Repository: timescale/timescaledb Branch: main Commit: 32a3cd5f1f8a Files: 2739 Total size: 37.2 MB Directory structure: gitextract_w6vwqnet/ ├── .clang-format ├── .codecov.yml ├── .dir-locals.el ├── .editorconfig ├── .git-blame-ignore-revs ├── .github/ │ ├── CODE_OF_CONDUCT.md │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report.yml │ │ ├── config.yml │ │ ├── enhancement.yml │ │ └── feature.yml │ ├── PULL_REQUEST_TEMPLATE/ │ │ └── pull_request_template.md │ ├── ci_settings.py │ ├── codespell-ignore-words │ ├── filters.yaml │ ├── gh_config_reader.py │ ├── gh_matrix_builder.py │ └── workflows/ │ ├── abi.yaml │ ├── apt-installcheck.yaml │ ├── apt-packages.yaml │ ├── backport-trigger.yaml │ ├── backport.yaml │ ├── catalog-updates-check.yaml │ ├── changelog-check.yaml │ ├── claude-code-review.yaml │ ├── coccinelle.yaml │ ├── code_style.yaml │ ├── coverity.yaml │ ├── docker-images.yaml │ ├── extras-diagnostic.yaml │ ├── homebrew.yaml │ ├── issue-handling.yaml │ ├── label-handling.yaml │ ├── label-released-prs.yaml │ ├── libfuzzer.yaml │ ├── linux-32bit-build-and-test.yaml │ ├── linux-build-and-test.yaml │ ├── loader_check.yml │ ├── memory-tests.yaml │ ├── nightly_cloud_smoke_test.yaml │ ├── pg_ladybug.yaml │ ├── pg_upgrade-test.yaml │ ├── pgspot.yaml │ ├── pr-approvals.yaml │ ├── pr-handling.yaml │ ├── pr-validation.yaml │ ├── prerelease-sanity.yaml │ ├── release_build_packages.yml │ ├── release_feature_freeze_ceremony.yaml │ ├── release_post_release_ceremony.yaml │ ├── rpm-packages.yaml │ ├── sanitizer-build-and-test.yaml │ ├── shellcheck.yaml │ ├── snapshot-abi.yaml │ ├── sqlsmith.yaml │ ├── stalebot.yaml │ ├── tests-fail-on-old-code.yaml │ ├── trigger-package-tests.yaml │ ├── trigger-prerelease-tests.yaml │ ├── update-test.yaml │ ├── windows-build-and-test.yaml │ └── windows-packages.yaml ├── .gitignore ├── .perltidyrc ├── .pull-review ├── .unreleased/ │ ├── chunk-param │ ├── columnar-function │ ├── constant-gapfill │ ├── direct-loss │ ├── in-any-chunk-exclusion │ ├── parameterized-merge │ ├── pr_8983 │ ├── pr_9142 │ ├── pr_9238 │ ├── pr_9253 │ ├── pr_9266 │ ├── pr_9267 │ ├── pr_9312 │ ├── pr_9334 │ ├── pr_9372 │ ├── pr_9374 │ ├── pr_9376 │ ├── pr_9378 │ ├── pr_9382 │ ├── pr_9399 │ ├── pr_9413 │ ├── pr_9414 │ ├── pr_9417 │ ├── saop-pushdown │ ├── text-minmax │ └── wrong-partition ├── .yamllint.yaml ├── CHANGELOG.md ├── CMakeLists.txt ├── CONTRIBUTING.md ├── LICENSE ├── LICENSE-APACHE ├── NOTICE ├── README.md ├── SECURITY.md ├── bootstrap ├── bootstrap.bat ├── cmake/ │ ├── GenerateScripts.cmake │ ├── GenerateTestSchedule.cmake │ ├── GitCommands.cmake │ └── ScriptFiles.cmake ├── coccinelle/ │ ├── README.md │ ├── attrnumbergetattroffset.cocci │ ├── bms_result.cocci │ ├── hash_create.cocci │ ├── heap_freetuple.cocci │ ├── hypertable_cache.cocci │ ├── hypertable_cache2.cocci │ ├── mcxt.cocci │ ├── namedata.cocci │ └── oidisvalid.cocci ├── coverage/ │ ├── CMakeLists.txt │ └── README.md ├── docs/ │ ├── BuildSource.md │ ├── MultiNodeDeprecation.md │ ├── StyleGuide.md │ └── getting-started/ │ ├── README.md │ ├── events-uuidv7/ │ │ └── README.md │ ├── financial-ticks/ │ │ └── README.md │ └── nyc-taxi/ │ ├── README.md │ ├── nyc-taxi-queries.sql │ ├── nyc-taxi-sample.csv │ └── nyc-taxi-schema.sql ├── scripts/ │ ├── CMakeLists.txt │ ├── backport.py │ ├── bundle_coredumps.sh │ ├── c_license_header-apache.h │ ├── c_license_header-timescale.h │ ├── changelog/ │ │ ├── generate.sh │ │ └── template.rfc822 │ ├── check_changelog_format.py │ ├── check_file_license.sh │ ├── check_license.sh │ ├── check_license_all.sh │ ├── check_missing_gitignore_for_template_tests.sh │ ├── check_orphaned_test_output_files.sh │ ├── check_sql_script.py │ ├── check_unnecessary_template_tests.sh │ ├── check_unreferenced_files.sh │ ├── check_updates.py │ ├── clang_format_all.sh │ ├── clang_format_wrapper.sh │ ├── cmake_format_all.sh │ ├── coccinelle.sh │ ├── delete_released_change_logs.sh │ ├── docker-build.sh │ ├── docker-run-tests.sh │ ├── dump_meta_data.sql │ ├── export_prefix_check.sh.in │ ├── githooks/ │ │ ├── .gitignore │ │ ├── commit_msg.py │ │ └── commit_msg_tests.py │ ├── label-released.py │ ├── license_apache.spec │ ├── license_tsl.spec │ ├── memory_leaks.sql │ ├── out_of_order_random_direct.sql │ ├── perltidy_format_all.sh │ ├── release/ │ │ ├── build_post_release_artefacts.sh │ │ ├── build_release_artefacts.sh │ │ ├── create_minor_release_branch.sh │ │ └── ready_fork_for_commit.sh │ ├── run_sql.sh │ ├── shellcheck-ci.sh │ ├── sql_license_apache.sql │ ├── sql_license_tsl.sql │ ├── start-local-docker.ps1 │ ├── start-local-docker.sh │ ├── suppressions/ │ │ ├── README.md │ │ ├── suppr_asan.txt │ │ ├── suppr_leak.txt │ │ └── suppr_ub.txt │ ├── test_downgrade.sh │ ├── test_license.sql │ ├── test_memory_spikes.py │ ├── test_pg_upgrade.py │ ├── test_regressions.sh │ ├── test_sanitizers.sh │ ├── test_update_from_version.sh │ ├── test_update_smoke.sh │ ├── test_updates.sh │ ├── ts_dump.sh │ ├── ts_restore.sh │ └── upload_ci_stats.sh ├── sql/ │ ├── CMakeLists.txt │ ├── bgw_scheduler.sql │ ├── bgw_startup.sql │ ├── bookend.sql │ ├── cagg_utils.sql │ ├── cat.cmake │ ├── chunk.sql │ ├── chunk_constraint.sql │ ├── comment_apache.sql │ ├── comment_tsl.sql │ ├── compat.sql │ ├── compression.sql │ ├── compression_defaults.sql │ ├── ddl_api.sql │ ├── ddl_triggers.sql │ ├── debug_build_utils.sql │ ├── debug_utils.sql │ ├── gapfill.sql │ ├── header.sql │ ├── histogram.sql │ ├── hypertable.sql │ ├── job_api.sql │ ├── job_stat_history_log_retention.sql │ ├── maintenance_utils.sql │ ├── metadata.sql │ ├── notice.sql │ ├── osm_api.sql │ ├── partitioning.sql │ ├── policy_api.sql │ ├── policy_internal.sql │ ├── pre_install/ │ │ ├── cache.sql │ │ ├── insert_data.sql │ │ ├── schemas.sql │ │ ├── tables.sql │ │ ├── types.functions.sql │ │ ├── types.post.sql │ │ └── types.pre.sql │ ├── restoring.sql │ ├── size_utils.sql │ ├── sparse_index.sql │ ├── time_bucket.sql │ ├── updates/ │ │ ├── 2.10.0--2.9.3.sql │ │ ├── 2.10.1--2.10.0.sql │ │ ├── 2.10.1--2.10.2.sql │ │ ├── 2.10.2--2.10.1.sql │ │ ├── 2.10.2--2.10.3.sql │ │ ├── 2.10.3--2.10.2.sql │ │ ├── 2.10.3--2.11.0.sql │ │ ├── 2.11.0--2.10.3.sql │ │ ├── 2.11.0--2.11.1.sql │ │ ├── 2.11.1--2.11.0.sql │ │ ├── 2.11.1--2.11.2.sql │ │ ├── 2.11.2--2.11.1.sql │ │ ├── 2.11.2--2.12.0.sql │ │ ├── 2.12.0--2.11.2.sql │ │ ├── 2.12.0-2.12.1.sql │ │ ├── 2.12.1--2.12.0.sql │ │ ├── 2.12.1--2.12.2.sql │ │ ├── 2.12.2--2.12.1.sql │ │ ├── 2.12.2--2.13.0.sql │ │ ├── 2.13.0--2.12.2.sql │ │ ├── 2.13.0--2.13.1.sql │ │ ├── 2.13.1--2.13.0.sql │ │ ├── 2.13.1--2.14.0.sql │ │ ├── 2.14.0--2.13.1.sql │ │ ├── 2.14.0--2.14.1.sql │ │ ├── 2.14.1--2.14.0.sql │ │ ├── 2.14.1--2.14.2.sql │ │ ├── 2.14.2--2.14.1.sql │ │ ├── 2.14.2--2.15.0.sql │ │ ├── 2.15.0--2.14.2.sql │ │ ├── 2.15.0--2.15.1.sql │ │ ├── 2.15.1--2.15.0.sql │ │ ├── 2.15.1--2.15.2.sql │ │ ├── 2.15.2--2.15.1.sql │ │ ├── 2.15.2--2.15.3.sql │ │ ├── 2.15.3--2.15.2.sql │ │ ├── 2.15.3--2.16.0.sql │ │ ├── 2.16.0--2.15.3.sql │ │ ├── 2.16.0--2.16.1.sql │ │ ├── 2.16.1--2.16.0.sql │ │ ├── 2.16.1--2.17.0.sql │ │ ├── 2.17.0--2.16.1.sql │ │ ├── 2.17.0--2.17.1.sql │ │ ├── 2.17.1--2.17.0.sql │ │ ├── 2.17.1--2.17.2.sql │ │ ├── 2.17.2--2.17.1.sql │ │ ├── 2.17.2--2.18.0.sql │ │ ├── 2.18.0--2.17.2.sql │ │ ├── 2.18.0--2.18.1.sql │ │ ├── 2.18.1--2.18.0.sql │ │ ├── 2.18.1--2.18.2.sql │ │ ├── 2.18.2--2.18.1.sql │ │ ├── 2.18.2--2.19.0.sql │ │ ├── 2.19.0--2.18.2.sql │ │ ├── 2.19.0--2.19.1.sql │ │ ├── 2.19.1--2.19.0.sql │ │ ├── 2.19.1--2.19.2.sql │ │ ├── 2.19.2--2.19.1.sql │ │ ├── 2.19.2--2.19.3.sql │ │ ├── 2.19.3--2.19.2.sql │ │ ├── 2.19.3--2.20.0.sql │ │ ├── 2.20.0--2.19.3.sql │ │ ├── 2.20.0--2.20.1.sql │ │ ├── 2.20.1--2.20.0.sql │ │ ├── 2.20.1--2.20.2.sql │ │ ├── 2.20.2--2.20.1.sql │ │ ├── 2.20.2--2.20.3.sql │ │ ├── 2.20.3--2.20.2.sql │ │ ├── 2.20.3--2.21.0.sql │ │ ├── 2.21.0--2.20.3.sql │ │ ├── 2.21.0--2.21.1.sql │ │ ├── 2.21.1--2.21.0.sql │ │ ├── 2.21.1--2.21.2.sql │ │ ├── 2.21.2--2.21.1.sql │ │ ├── 2.21.2--2.21.3.sql │ │ ├── 2.21.3--2.21.2.sql │ │ ├── 2.21.3--2.21.4.sql │ │ ├── 2.21.4--2.22.0.sql │ │ ├── 2.22.0--2.21.3.sql │ │ ├── 2.22.0--2.22.1.sql │ │ ├── 2.22.1--2.22.0.sql │ │ ├── 2.22.1--2.23.0.sql │ │ ├── 2.23.0--2.22.1.sql │ │ ├── 2.23.0--2.23.1.sql │ │ ├── 2.23.1--2.23.0.sql │ │ ├── 2.23.1--2.24.0.sql │ │ ├── 2.24.0--2.23.1.sql │ │ ├── 2.24.0--2.25.0.sql │ │ ├── 2.25.0--2.24.0.sql │ │ ├── 2.25.0--2.25.1.sql │ │ ├── 2.25.1--2.25.0.sql │ │ ├── 2.25.1--2.25.2.sql │ │ ├── 2.25.2--2.25.1.sql │ │ ├── 2.9.0--2.8.1.sql │ │ ├── 2.9.0--2.9.1.sql │ │ ├── 2.9.1--2.9.0.sql │ │ ├── 2.9.1--2.9.2.sql │ │ ├── 2.9.2--2.9.1.sql │ │ ├── 2.9.2--2.9.3.sql │ │ ├── 2.9.3--2.10.0.sql │ │ ├── 2.9.3--2.9.2.sql │ │ ├── README.md │ │ ├── latest-dev.sql │ │ ├── post-update.sql │ │ ├── pre-update.sql │ │ ├── pre-version-change.sql │ │ ├── reverse-dev.sql │ │ ├── set_post_update_stage.sql │ │ ├── unset_update_stage.sql │ │ └── version_check.sql │ ├── util_internal_table_ddl.sql │ ├── util_time.sql │ ├── uuidv7.sql │ ├── version.sql │ ├── views.sql │ ├── views_experimental.sql │ ├── with_telemetry.sql │ └── without_telemetry.sql ├── src/ │ ├── CMakeLists.txt │ ├── README.md │ ├── adts/ │ │ ├── README.md │ │ ├── bit_array.h │ │ ├── bit_array_impl.h │ │ ├── char_vec.h │ │ ├── uint64_vec.h │ │ └── vec.h │ ├── agg_bookend.c │ ├── annotations.h │ ├── bgw/ │ │ ├── CMakeLists.txt │ │ ├── README.md │ │ ├── job.c │ │ ├── job.h │ │ ├── job_stat.c │ │ ├── job_stat.h │ │ ├── job_stat_history.c │ │ ├── job_stat_history.h │ │ ├── launcher_interface.c │ │ ├── launcher_interface.h │ │ ├── scheduler.c │ │ ├── scheduler.h │ │ ├── timer.c │ │ ├── timer.h │ │ └── worker.h │ ├── bgw_policy/ │ │ ├── CMakeLists.txt │ │ ├── chunk_stats.c │ │ ├── chunk_stats.h │ │ ├── policy.c │ │ └── policy.h │ ├── bmslist_utils.c │ ├── bmslist_utils.h │ ├── build-defs.cmake │ ├── cache.c │ ├── cache.h │ ├── cache_invalidate.c │ ├── cache_invalidate.h │ ├── chunk.c │ ├── chunk.h │ ├── chunk_adaptive.c │ ├── chunk_adaptive.h │ ├── chunk_constraint.c │ ├── chunk_constraint.h │ ├── chunk_index.c │ ├── chunk_index.h │ ├── chunk_insert_state.c │ ├── chunk_insert_state.h │ ├── chunk_scan.c │ ├── chunk_scan.h │ ├── chunk_tuple_routing.c │ ├── chunk_tuple_routing.h │ ├── compat/ │ │ ├── CMakeLists.txt │ │ ├── compat-msvc-enter.h │ │ ├── compat-msvc-exit.h │ │ └── compat.h │ ├── config.h.in │ ├── constraint.c │ ├── constraint.h │ ├── copy.c │ ├── copy.h │ ├── cross_module_fn.c │ ├── cross_module_fn.h │ ├── custom_type_cache.c │ ├── custom_type_cache.h │ ├── debug_assert.h │ ├── debug_point.c │ ├── debug_point.h │ ├── dimension.c │ ├── dimension.h │ ├── dimension_slice.c │ ├── dimension_slice.h │ ├── dimension_vector.c │ ├── dimension_vector.h │ ├── error_utils.h │ ├── errors.h │ ├── estimate.c │ ├── estimate.h │ ├── event_trigger.c │ ├── event_trigger.h │ ├── export.h │ ├── expression_utils.c │ ├── expression_utils.h │ ├── extension.c │ ├── extension.h │ ├── extension_constants.c │ ├── extension_constants.h │ ├── extension_utils.c │ ├── foreach_ptr.h │ ├── foreign_key.c │ ├── foreign_key.h │ ├── func_cache.c │ ├── func_cache.h │ ├── gapfill.c │ ├── gitcommit.cmake │ ├── gitcommit.h.in │ ├── guc.c │ ├── guc.h │ ├── histogram.c │ ├── hypercube.c │ ├── hypercube.h │ ├── hypertable.c │ ├── hypertable.h │ ├── hypertable_cache.c │ ├── hypertable_cache.h │ ├── hypertable_restrict_info.c │ ├── hypertable_restrict_info.h │ ├── import/ │ │ ├── allpaths.c │ │ ├── allpaths.h │ │ ├── heapswap.c │ │ ├── heapswap.h │ │ ├── list.c │ │ ├── list.h │ │ ├── planner.c │ │ ├── planner.h │ │ ├── ts_explain.c │ │ └── ts_explain.h │ ├── indexing.c │ ├── indexing.h │ ├── init.c │ ├── jsonb_utils.c │ ├── jsonb_utils.h │ ├── license_guc.c │ ├── license_guc.h │ ├── loader/ │ │ ├── CMakeLists.txt │ │ ├── README.md │ │ ├── bgw_counter.c │ │ ├── bgw_counter.h │ │ ├── bgw_interface.c │ │ ├── bgw_interface.h │ │ ├── bgw_launcher.c │ │ ├── bgw_launcher.h │ │ ├── bgw_message_queue.c │ │ ├── bgw_message_queue.h │ │ ├── function_telemetry.c │ │ ├── function_telemetry.h │ │ ├── loader.c │ │ ├── loader.h │ │ ├── lwlocks.c │ │ └── lwlocks.h │ ├── net/ │ │ ├── CMakeLists.txt │ │ ├── conn.c │ │ ├── conn.h │ │ ├── conn_internal.h │ │ ├── conn_plain.c │ │ ├── conn_plain.h │ │ ├── conn_ssl.c │ │ ├── http.c │ │ ├── http.h │ │ ├── http_request.c │ │ └── http_response.c │ ├── nodes/ │ │ ├── CMakeLists.txt │ │ ├── chunk_append/ │ │ │ ├── CMakeLists.txt │ │ │ ├── chunk_append.c │ │ │ ├── chunk_append.h │ │ │ ├── exec.c │ │ │ ├── planner.c │ │ │ ├── transform.c │ │ │ └── transform.h │ │ ├── constraint_aware_append/ │ │ │ ├── CMakeLists.txt │ │ │ ├── constraint_aware_append.c │ │ │ └── constraint_aware_append.h │ │ ├── modify_hypertable.c │ │ ├── modify_hypertable.h │ │ ├── modify_hypertable_exec.c │ │ └── vector_agg.h │ ├── osm_callbacks.c │ ├── osm_callbacks.h │ ├── partition_chunk.c │ ├── partition_chunk.h │ ├── partitioning.c │ ├── partitioning.h │ ├── planner/ │ │ ├── CMakeLists.txt │ │ ├── agg_bookend.c │ │ ├── constify_now.c │ │ ├── constraint_cleanup.c │ │ ├── expand_hypertable.c │ │ ├── planner.c │ │ ├── planner.h │ │ └── space_constraint.c │ ├── process_utility.c │ ├── process_utility.h │ ├── scan_iterator.c │ ├── scan_iterator.h │ ├── scanner.c │ ├── scanner.h │ ├── sort_transform.c │ ├── sort_transform.h │ ├── subspace_store.README.md │ ├── subspace_store.c │ ├── subspace_store.h │ ├── telemetry/ │ │ ├── CMakeLists.txt │ │ ├── functions.c │ │ ├── functions.h │ │ ├── replication.c │ │ ├── replication.h │ │ ├── stats.c │ │ ├── stats.h │ │ ├── telemetry.c │ │ ├── telemetry.h │ │ ├── telemetry_metadata.c │ │ └── telemetry_metadata.h │ ├── time_bucket.c │ ├── time_bucket.h │ ├── time_utils.c │ ├── time_utils.h │ ├── timezones.c │ ├── timezones.h │ ├── trigger.c │ ├── trigger.h │ ├── ts_catalog/ │ │ ├── CMakeLists.txt │ │ ├── array_utils.c │ │ ├── array_utils.h │ │ ├── catalog.c │ │ ├── catalog.h │ │ ├── chunk_column_stats.c │ │ ├── chunk_column_stats.h │ │ ├── chunk_rewrite.c │ │ ├── chunk_rewrite.h │ │ ├── compression_chunk_size.c │ │ ├── compression_chunk_size.h │ │ ├── compression_settings.c │ │ ├── compression_settings.h │ │ ├── continuous_agg.c │ │ ├── continuous_agg.h │ │ ├── continuous_aggs_watermark.c │ │ ├── continuous_aggs_watermark.h │ │ ├── metadata.c │ │ ├── metadata.h │ │ ├── tablespace.c │ │ └── tablespace.h │ ├── tss_callbacks.c │ ├── tss_callbacks.h │ ├── utils.c │ ├── utils.h │ ├── uuid.c │ ├── uuid.h │ ├── version.c │ ├── version.h │ └── with_clause/ │ ├── CMakeLists.txt │ ├── alter_table_with_clause.c │ ├── alter_table_with_clause.h │ ├── create_materialized_view_with_clause.c │ ├── create_materialized_view_with_clause.h │ ├── create_table_with_clause.c │ ├── create_table_with_clause.h │ ├── with_clause_parser.c │ └── with_clause_parser.h ├── test/ │ ├── .gitignore │ ├── CMakeLists.txt │ ├── README.md │ ├── ci_rerun.sh │ ├── expected/ │ │ ├── agg_bookends-15.out │ │ ├── agg_bookends-16.out │ │ ├── agg_bookends-17.out │ │ ├── agg_bookends-18.out │ │ ├── alter.out │ │ ├── alternate_users.out │ │ ├── append-15.out │ │ ├── append-16.out │ │ ├── append-17.out │ │ ├── append-18.out │ │ ├── baserel_cache.out │ │ ├── bgw_launcher.out │ │ ├── c_unit_tests.out │ │ ├── catalog_corruption.out │ │ ├── chunk_adaptive.out │ │ ├── chunk_merge.out │ │ ├── chunk_publication.out │ │ ├── chunk_utils.out │ │ ├── chunks.out │ │ ├── cluster.out │ │ ├── constraint.out │ │ ├── copy.out │ │ ├── copy_memory_usage.out │ │ ├── copy_where.out │ │ ├── create_chunks.out │ │ ├── create_hypertable.out │ │ ├── create_table.out │ │ ├── create_table_with.out │ │ ├── cursor.out │ │ ├── custom_type.out │ │ ├── ddl.out │ │ ├── ddl_errors.out │ │ ├── ddl_extra.out │ │ ├── debug_utils.out │ │ ├── delete.out │ │ ├── drop_extension.out │ │ ├── drop_hypertable.out │ │ ├── drop_owned-15.out │ │ ├── drop_owned-16.out │ │ ├── drop_owned-17.out │ │ ├── drop_owned-18.out │ │ ├── drop_rename_hypertable.out │ │ ├── drop_schema.out │ │ ├── dump_meta.out │ │ ├── extension_scripts.out │ │ ├── generated_as_identity.out │ │ ├── grant_hypertable-15.out │ │ ├── grant_hypertable-16.out │ │ ├── grant_hypertable-17.out │ │ ├── grant_hypertable-18.out │ │ ├── hash.out │ │ ├── histogram_test-15.out │ │ ├── histogram_test-16.out │ │ ├── histogram_test-17.out │ │ ├── histogram_test-18.out │ │ ├── index.out │ │ ├── information_views.out │ │ ├── insert-15.out │ │ ├── insert-16.out │ │ ├── insert-17.out │ │ ├── insert-18.out │ │ ├── insert_many.out │ │ ├── insert_returning.out │ │ ├── insert_returning_old_new.out │ │ ├── insert_single.out │ │ ├── lateral.out │ │ ├── license.out │ │ ├── loader-oss.out │ │ ├── loader-tsl.out │ │ ├── merge.out │ │ ├── metadata.out │ │ ├── multi_transaction_index.out │ │ ├── net.out │ │ ├── null_exclusion-15.out │ │ ├── null_exclusion-16.out │ │ ├── null_exclusion-17.out │ │ ├── null_exclusion-18.out │ │ ├── parallel-15.out │ │ ├── parallel-16.out │ │ ├── parallel-17.out │ │ ├── parallel-18.out │ │ ├── partition.out │ │ ├── partition_coercion.out │ │ ├── partitioned_hypertable.out │ │ ├── partitioning.out │ │ ├── partitionwise-15.out │ │ ├── partitionwise-16.out │ │ ├── partitionwise-17.out │ │ ├── partitionwise-18.out │ │ ├── partitionwise.out │ │ ├── pg_dump.out │ │ ├── pg_dump_unprivileged.out │ │ ├── pg_join.out │ │ ├── plain.out │ │ ├── plan_expand_hypertable-15.out │ │ ├── plan_expand_hypertable-16.out │ │ ├── plan_expand_hypertable-17.out │ │ ├── plan_expand_hypertable-18.out │ │ ├── plan_hashagg-15.out │ │ ├── plan_hashagg-16.out │ │ ├── plan_hashagg-17.out │ │ ├── plan_hashagg-18.out │ │ ├── plan_hypertable_inline.out │ │ ├── plan_ordered_append-15.out │ │ ├── plan_ordered_append-16.out │ │ ├── plan_ordered_append-17.out │ │ ├── plan_ordered_append-18.out │ │ ├── query.out │ │ ├── relocate_extension.out │ │ ├── reloptions.out │ │ ├── repair.out │ │ ├── rowsecurity-15.out │ │ ├── rowsecurity-16.out │ │ ├── rowsecurity-17.out │ │ ├── rowsecurity-18.out │ │ ├── size_utils.out │ │ ├── sort_optimization.out │ │ ├── sql_query.out │ │ ├── symbol_conflict.out │ │ ├── tableam.out │ │ ├── tableam_alter.out │ │ ├── tableam_alter_defaults.out │ │ ├── tablespace.out │ │ ├── telemetry.out │ │ ├── test_tss_callbacks.out │ │ ├── test_utils.out │ │ ├── timestamp-15.out │ │ ├── timestamp-16.out │ │ ├── timestamp-17.out │ │ ├── timestamp-18.out │ │ ├── triggers.out │ │ ├── truncate.out │ │ ├── trusted_extension.out │ │ ├── ts_merge-15.out │ │ ├── ts_merge-16.out │ │ ├── ts_merge-17.out │ │ ├── ts_merge-18.out │ │ ├── update.out │ │ ├── upsert.out │ │ ├── util.out │ │ ├── uuid.out │ │ ├── vacuum.out │ │ ├── vacuum_parallel.out │ │ └── version.out │ ├── isolation/ │ │ ├── CMakeLists.txt │ │ ├── expected/ │ │ │ ├── concurrent_add_dimension.out │ │ │ ├── concurrent_query_and_drop_chunks.out │ │ │ ├── deadlock_dropchunks_select.out │ │ │ ├── dropchunks_race.out │ │ │ ├── insert_dropchunks_race.out │ │ │ ├── isolation_nop.out │ │ │ ├── multi_transaction_indexing.out │ │ │ ├── read_committed_insert.out │ │ │ ├── read_uncommitted_insert.out │ │ │ ├── repeatable_read_insert.out │ │ │ ├── serializable_insert.out │ │ │ └── serializable_insert_rollback.out │ │ └── specs/ │ │ ├── CMakeLists.txt │ │ ├── concurrent_add_dimension.spec │ │ ├── concurrent_query_and_drop_chunks.spec │ │ ├── deadlock_dropchunks_select.spec │ │ ├── dropchunks_race.spec │ │ ├── insert_dropchunks_race.spec │ │ ├── isolation_nop.spec │ │ ├── multi_transaction_indexing.spec │ │ ├── read_committed_insert.spec │ │ ├── read_uncommitted_insert.spec │ │ ├── repeatable_read_insert.spec │ │ ├── serializable_insert.spec │ │ └── serializable_insert_rollback.spec │ ├── perl/ │ │ ├── CMakeLists.txt │ │ ├── README.md │ │ └── TimescaleNode.pm │ ├── pg_prove.sh │ ├── pg_regress.sh │ ├── pgpass.conf.in │ ├── pgtest/ │ │ ├── CMakeLists.txt │ │ └── README.md │ ├── pgtest.conf.in │ ├── postgres-asan-instrumentation-PG18GE.patch │ ├── postgres-asan-instrumentation.patch │ ├── postgresql.conf.in │ ├── runner.sh │ ├── runner_cleanup_output.sh │ ├── runner_isolation.sh │ ├── runner_shared.sh │ ├── sql/ │ │ ├── .gitignore │ │ ├── CMakeLists.txt │ │ ├── agg_bookends.sql.in │ │ ├── alter.sql │ │ ├── alternate_users.sql │ │ ├── append.sql.in │ │ ├── baserel_cache.sql │ │ ├── bgw_launcher.sql │ │ ├── c_unit_tests.sql │ │ ├── catalog_corruption.sql │ │ ├── chunk_adaptive.sql │ │ ├── chunk_publication.sql │ │ ├── chunk_utils.sql │ │ ├── chunks.sql │ │ ├── cluster.sql │ │ ├── constraint.sql │ │ ├── copy.sql │ │ ├── copy_memory_usage.sql │ │ ├── copy_where.sql │ │ ├── create_chunks.sql │ │ ├── create_hypertable.sql │ │ ├── create_table.sql │ │ ├── create_table_with.sql │ │ ├── cursor.sql │ │ ├── custom_type.sql │ │ ├── data/ │ │ │ ├── alter.tsv │ │ │ ├── copy_data.csv │ │ │ ├── ds1_dev1_1.tsv │ │ │ ├── onek.data │ │ │ └── tenk.data │ │ ├── ddl.sql │ │ ├── ddl_errors.sql │ │ ├── ddl_extra.sql │ │ ├── debug_utils.sql │ │ ├── delete.sql │ │ ├── drop_extension.sql │ │ ├── drop_hypertable.sql │ │ ├── drop_owned.sql.in │ │ ├── drop_rename_hypertable.sql │ │ ├── drop_schema.sql │ │ ├── dump_meta.sql │ │ ├── extension_scripts.sql │ │ ├── generated_as_identity.sql │ │ ├── grant_hypertable.sql.in │ │ ├── hash.sql │ │ ├── histogram_test.sql.in │ │ ├── include/ │ │ │ ├── agg_bookends_load.sql │ │ │ ├── agg_bookends_query.sql │ │ │ ├── append_load.sql │ │ │ ├── append_query.sql │ │ │ ├── bgw_launcher_utils.sql │ │ │ ├── bgw_launcher_utils_cleanup.sql │ │ │ ├── ddl_ops_1.sql │ │ │ ├── ddl_ops_2.sql │ │ │ ├── insert_single.sql │ │ │ ├── insert_two_partitions.sql │ │ │ ├── join_load.sql │ │ │ ├── join_query.sql │ │ │ ├── plan_expand_hypertable_load.sql │ │ │ ├── plan_expand_hypertable_query.sql │ │ │ ├── plan_hashagg_load.sql │ │ │ ├── plan_hashagg_query.sql │ │ │ ├── plan_ordered_append_load.sql │ │ │ ├── plan_ordered_append_query.sql │ │ │ ├── query_load.sql │ │ │ ├── query_query.sql │ │ │ ├── query_result_test_equal.sql │ │ │ ├── test_utils.sql │ │ │ ├── ts_merge_load.sql │ │ │ ├── ts_merge_load_ht.sql │ │ │ └── ts_merge_query.sql │ │ ├── index.sql │ │ ├── information_views.sql │ │ ├── insert.sql.in │ │ ├── insert_many.sql │ │ ├── insert_returning.sql │ │ ├── insert_returning_old_new.sql │ │ ├── insert_single.sql │ │ ├── lateral.sql │ │ ├── license.sql │ │ ├── loader/ │ │ │ ├── CMakeLists.txt │ │ │ ├── timescaledb--0.0.0.sql │ │ │ ├── timescaledb--mock-1--mock-2.sql │ │ │ ├── timescaledb--mock-1.sql │ │ │ ├── timescaledb--mock-2--mock-3.sql │ │ │ ├── timescaledb--mock-2.sql │ │ │ ├── timescaledb--mock-3--mock-4.sql │ │ │ ├── timescaledb--mock-3.sql │ │ │ ├── timescaledb--mock-4.sql │ │ │ ├── timescaledb--mock-5--mock-6.sql │ │ │ ├── timescaledb--mock-5.sql │ │ │ ├── timescaledb--mock-6.sql │ │ │ ├── timescaledb--mock-broken--mock-5.sql │ │ │ ├── timescaledb--mock-broken.sql │ │ │ ├── timescaledb_osm--mock-1.sql │ │ │ └── timescaledb_osm.control │ │ ├── loader.sql.in │ │ ├── merge.sql │ │ ├── metadata.sql │ │ ├── multi_transaction_index.sql │ │ ├── net.sql │ │ ├── null_exclusion.sql.in │ │ ├── parallel.sql.in │ │ ├── partition.sql │ │ ├── partition_coercion.sql │ │ ├── partitioned_hypertable.sql │ │ ├── partitioning.sql │ │ ├── partitionwise.sql.in │ │ ├── pg_dump.sql │ │ ├── pg_dump_unprivileged.sql │ │ ├── pg_join.sql │ │ ├── plain.sql │ │ ├── plan_expand_hypertable.sql.in │ │ ├── plan_hashagg.sql.in │ │ ├── plan_hypertable_inline.sql │ │ ├── plan_ordered_append.sql.in │ │ ├── query.sql │ │ ├── relocate_extension.sql │ │ ├── reloptions.sql │ │ ├── repair.sql │ │ ├── rowsecurity.sql.in │ │ ├── size_utils.sql │ │ ├── sort_optimization.sql │ │ ├── sql_query.sql │ │ ├── symbol_conflict.sql │ │ ├── tableam.sql │ │ ├── tableam_alter.sql │ │ ├── tableam_alter_defaults.sql │ │ ├── tablespace.sql │ │ ├── telemetry.sql │ │ ├── test_tss_callbacks.sql │ │ ├── test_utils.sql │ │ ├── timestamp.sql.in │ │ ├── triggers.sql │ │ ├── truncate.sql │ │ ├── trusted_extension.sql │ │ ├── ts_merge.sql.in │ │ ├── update.sql │ │ ├── updates/ │ │ │ ├── catalog_missing_columns.sql │ │ │ ├── cleanup.bigint.sql │ │ │ ├── cleanup.chunk_skipping.sql │ │ │ ├── cleanup.compression.sql │ │ │ ├── cleanup.constraints.sql │ │ │ ├── cleanup.continuous_aggs.v2.sql │ │ │ ├── cleanup.policies.sql │ │ │ ├── cleanup.sparse_index.sql │ │ │ ├── cleanup.timestamp.sql │ │ │ ├── cleanup.v10.sql │ │ │ ├── cleanup.v7.sql │ │ │ ├── cleanup.v8.sql │ │ │ ├── cleanup.v9.sql │ │ │ ├── post.catalog.sql │ │ │ ├── post.chunk_skipping.sql │ │ │ ├── post.compression.sql │ │ │ ├── post.continuous_aggs.sql │ │ │ ├── post.continuous_aggs.v2.sql │ │ │ ├── post.continuous_aggs.v3.sql │ │ │ ├── post.functions.sql │ │ │ ├── post.insert.sql │ │ │ ├── post.integrity_test.sql │ │ │ ├── post.pg_upgrade.sql │ │ │ ├── post.policies.sql │ │ │ ├── post.repair.hierarchical_cagg.sql │ │ │ ├── post.repair.sql │ │ │ ├── post.sequences.sql │ │ │ ├── post.sparse_index.sql │ │ │ ├── post.v10.sql │ │ │ ├── post.v7.sql │ │ │ ├── post.v8.sql │ │ │ ├── post.v9.sql │ │ │ ├── pre.cleanup.sql │ │ │ ├── pre.smoke.sql │ │ │ ├── pre.testing.sql │ │ │ ├── setup.bigint.sql │ │ │ ├── setup.catalog.sql │ │ │ ├── setup.check.sql │ │ │ ├── setup.chunk_skipping.sql │ │ │ ├── setup.compression.sql │ │ │ ├── setup.constraints.sql │ │ │ ├── setup.continuous_aggs.sql │ │ │ ├── setup.databases.sql │ │ │ ├── setup.drop_meta.sql │ │ │ ├── setup.fix_sparse_index_migration.sql │ │ │ ├── setup.insert_bigint.v1.sql │ │ │ ├── setup.insert_bigint.v2.sql │ │ │ ├── setup.insert_timestamp.sql │ │ │ ├── setup.pg_upgrade.sql │ │ │ ├── setup.policies.sql │ │ │ ├── setup.post-downgrade.sql │ │ │ ├── setup.repair.sql │ │ │ ├── setup.roles.sql │ │ │ ├── setup.sparse_index.sql │ │ │ ├── setup.timestamp.sql │ │ │ ├── setup.v10.sql │ │ │ ├── setup.v7.sql │ │ │ ├── setup.v8.sql │ │ │ └── setup.v9.sql │ │ ├── upsert.sql │ │ ├── util.sql │ │ ├── utils/ │ │ │ ├── pg_dump_aux_dump.sh │ │ │ ├── pg_dump_aux_plain_dump.sh │ │ │ ├── pg_dump_aux_restore.sh │ │ │ ├── pg_dump_unprivileged.sh │ │ │ ├── test_fatal_command.sh │ │ │ ├── testsupport.sql │ │ │ └── testsupport_init.sql │ │ ├── uuid.sql │ │ ├── vacuum.sql │ │ ├── vacuum_parallel.sql │ │ └── version.sql │ ├── src/ │ │ ├── CMakeLists.txt │ │ ├── adt_tests.c │ │ ├── bgw/ │ │ │ ├── CMakeLists.txt │ │ │ ├── README.md │ │ │ ├── log.c │ │ │ ├── log.h │ │ │ ├── params.c │ │ │ ├── params.h │ │ │ ├── scheduler_mock.c │ │ │ ├── test_job_refresh.c │ │ │ ├── test_job_utils.c │ │ │ ├── timer_mock.c │ │ │ └── timer_mock.h │ │ ├── loader/ │ │ │ ├── CMakeLists.txt │ │ │ ├── config.h │ │ │ ├── init.c │ │ │ └── osm_init.c │ │ ├── metadata.c │ │ ├── net/ │ │ │ ├── CMakeLists.txt │ │ │ ├── conn_mock.c │ │ │ ├── conn_mock.h │ │ │ ├── test_conn.c │ │ │ └── test_http.c │ │ ├── symbol_conflict.c │ │ ├── telemetry/ │ │ │ ├── CMakeLists.txt │ │ │ ├── test_privacy.c │ │ │ └── test_telemetry.c │ │ ├── test_bmslist_utils.c │ │ ├── test_compression_settings.c │ │ ├── test_jsonb_utils.c │ │ ├── test_scanner.c │ │ ├── test_time_to_internal.c │ │ ├── test_time_utils.c │ │ ├── test_tss_callbacks.c │ │ ├── test_utils.c │ │ ├── test_utils.h │ │ └── test_with_clause_parser.c │ ├── t/ │ │ ├── 001_replication_telemetry.pl │ │ └── CMakeLists.txt │ └── test-defs.cmake ├── timescaledb.control.in ├── tsl/ │ ├── CMakeLists.txt │ ├── LICENSE-TIMESCALE │ ├── README.md │ ├── src/ │ │ ├── CMakeLists.txt │ │ ├── README.module.md │ │ ├── bgw_policy/ │ │ │ ├── CMakeLists.txt │ │ │ ├── compression_api.c │ │ │ ├── compression_api.h │ │ │ ├── continuous_aggregate_api.c │ │ │ ├── continuous_aggregate_api.h │ │ │ ├── job.c │ │ │ ├── job.h │ │ │ ├── job_api.c │ │ │ ├── job_api.h │ │ │ ├── policies_v2.c │ │ │ ├── policies_v2.h │ │ │ ├── policy_config.c │ │ │ ├── policy_config.h │ │ │ ├── policy_utils.c │ │ │ ├── policy_utils.h │ │ │ ├── process_hyper_inval_api.c │ │ │ ├── process_hyper_inval_api.h │ │ │ ├── reorder_api.c │ │ │ ├── reorder_api.h │ │ │ ├── retention_api.c │ │ │ └── retention_api.h │ │ ├── build-defs.cmake │ │ ├── chunk.c │ │ ├── chunk.h │ │ ├── chunk_api.c │ │ ├── chunk_api.h │ │ ├── chunk_merge.c │ │ ├── chunk_split.c │ │ ├── chunkwise_agg.c │ │ ├── chunkwise_agg.h │ │ ├── compression/ │ │ │ ├── CMakeLists.txt │ │ │ ├── README.md │ │ │ ├── algorithms/ │ │ │ │ ├── CMakeLists.txt │ │ │ │ ├── array.c │ │ │ │ ├── array.h │ │ │ │ ├── bool_compress.c │ │ │ │ ├── bool_compress.h │ │ │ │ ├── datum_serialize.c │ │ │ │ ├── datum_serialize.h │ │ │ │ ├── deltadelta.c │ │ │ │ ├── deltadelta.h │ │ │ │ ├── deltadelta_impl.c │ │ │ │ ├── dictionary.c │ │ │ │ ├── dictionary.h │ │ │ │ ├── dictionary_hash.h │ │ │ │ ├── float_utils.h │ │ │ │ ├── gorilla.c │ │ │ │ ├── gorilla.h │ │ │ │ ├── gorilla_impl.c │ │ │ │ ├── null.c │ │ │ │ ├── null.h │ │ │ │ ├── simple8b_rle.h │ │ │ │ ├── simple8b_rle_bitarray.h │ │ │ │ ├── simple8b_rle_bitmap.h │ │ │ │ ├── simple8b_rle_decompress_all.h │ │ │ │ ├── uuid_compress.c │ │ │ │ └── uuid_compress.h │ │ │ ├── api.c │ │ │ ├── api.h │ │ │ ├── arrow_c_data_interface.h │ │ │ ├── batch_metadata_builder.h │ │ │ ├── batch_metadata_builder_bloom1.c │ │ │ ├── batch_metadata_builder_minmax.c │ │ │ ├── batch_metadata_builder_minmax.h │ │ │ ├── city_combine.h │ │ │ ├── compression.c │ │ │ ├── compression.h │ │ │ ├── compression_dml.c │ │ │ ├── compression_dml.h │ │ │ ├── compression_scankey.c │ │ │ ├── compression_storage.c │ │ │ ├── compression_storage.h │ │ │ ├── create.c │ │ │ ├── create.h │ │ │ ├── recompress.c │ │ │ ├── recompress.h │ │ │ ├── sparse_index_bloom1.h │ │ │ └── wal_utils.h │ │ ├── continuous_aggs/ │ │ │ ├── CMakeLists.txt │ │ │ ├── README.md │ │ │ ├── common.c │ │ │ ├── common.h │ │ │ ├── create.c │ │ │ ├── create.h │ │ │ ├── finalize.c │ │ │ ├── finalize.h │ │ │ ├── insert.c │ │ │ ├── insert.h │ │ │ ├── invalidation.c │ │ │ ├── invalidation.h │ │ │ ├── invalidation_threshold.c │ │ │ ├── invalidation_threshold.h │ │ │ ├── materialize.c │ │ │ ├── materialize.h │ │ │ ├── options.c │ │ │ ├── options.h │ │ │ ├── planner.c │ │ │ ├── planner.h │ │ │ ├── refresh.c │ │ │ ├── refresh.h │ │ │ ├── utils.c │ │ │ └── utils.h │ │ ├── import/ │ │ │ ├── CMakeLists.txt │ │ │ ├── ts_like_match.c │ │ │ ├── umash.c │ │ │ └── umash.h │ │ ├── init.c │ │ ├── nodes/ │ │ │ ├── CMakeLists.txt │ │ │ ├── README.md │ │ │ ├── columnar_index_scan/ │ │ │ │ ├── CMakeLists.txt │ │ │ │ ├── columnar_index_scan.c │ │ │ │ ├── columnar_index_scan.h │ │ │ │ └── columnar_index_scan_exec.c │ │ │ ├── columnar_scan/ │ │ │ │ ├── CMakeLists.txt │ │ │ │ ├── batch_array.c │ │ │ │ ├── batch_array.h │ │ │ │ ├── batch_queue.h │ │ │ │ ├── batch_queue_fifo.c │ │ │ │ ├── batch_queue_fifo.h │ │ │ │ ├── batch_queue_heap.c │ │ │ │ ├── batch_queue_heap.h │ │ │ │ ├── columnar_scan.c │ │ │ │ ├── columnar_scan.h │ │ │ │ ├── compressed_batch.c │ │ │ │ ├── compressed_batch.h │ │ │ │ ├── decompress_context.h │ │ │ │ ├── detoaster.c │ │ │ │ ├── detoaster.h │ │ │ │ ├── exec.c │ │ │ │ ├── exec.h │ │ │ │ ├── planner.c │ │ │ │ ├── planner.h │ │ │ │ ├── pred_text.c │ │ │ │ ├── pred_text.h │ │ │ │ ├── pred_vector_array.c │ │ │ │ ├── pred_vector_const_arithmetic_all.c │ │ │ │ ├── pred_vector_const_arithmetic_single.c │ │ │ │ ├── pred_vector_const_arithmetic_type_pair.c │ │ │ │ ├── qual_pushdown.c │ │ │ │ ├── qual_pushdown.h │ │ │ │ ├── vector_dict.h │ │ │ │ ├── vector_predicates.c │ │ │ │ ├── vector_predicates.h │ │ │ │ └── vector_quals.h │ │ │ ├── gapfill/ │ │ │ │ ├── CMakeLists.txt │ │ │ │ ├── README.md │ │ │ │ ├── gapfill.h │ │ │ │ ├── gapfill_exec.c │ │ │ │ ├── gapfill_functions.c │ │ │ │ ├── gapfill_functions.h │ │ │ │ ├── gapfill_internal.h │ │ │ │ ├── gapfill_plan.c │ │ │ │ ├── interpolate.c │ │ │ │ ├── interpolate.h │ │ │ │ ├── locf.c │ │ │ │ └── locf.h │ │ │ ├── skip_scan/ │ │ │ │ ├── CMakeLists.txt │ │ │ │ ├── README.md │ │ │ │ ├── exec.c │ │ │ │ ├── planner.c │ │ │ │ └── skip_scan.h │ │ │ └── vector_agg/ │ │ │ ├── CMakeLists.txt │ │ │ ├── exec.c │ │ │ ├── exec.h │ │ │ ├── filter_word_iterator.h │ │ │ ├── function/ │ │ │ │ ├── CMakeLists.txt │ │ │ │ ├── agg_many_vector_helper.c │ │ │ │ ├── agg_scalar_helper.c │ │ │ │ ├── agg_vector_validity_helper.c │ │ │ │ ├── float48_accum_single.c │ │ │ │ ├── float48_accum_templates.c │ │ │ │ ├── float48_accum_types.c │ │ │ │ ├── functions.c │ │ │ │ ├── functions.h │ │ │ │ ├── int128_accum_single.c │ │ │ │ ├── int128_accum_templates.c │ │ │ │ ├── int24_avg_accum_single.c │ │ │ │ ├── int24_avg_accum_templates.c │ │ │ │ ├── int24_sum_single.c │ │ │ │ ├── int24_sum_templates.c │ │ │ │ ├── minmax_arithmetic_single.c │ │ │ │ ├── minmax_arithmetic_types.c │ │ │ │ ├── minmax_templates.c │ │ │ │ ├── minmax_text.c │ │ │ │ ├── sum_float_single.c │ │ │ │ ├── sum_float_templates.c │ │ │ │ └── template_helper.h │ │ │ ├── grouping_policy.h │ │ │ ├── grouping_policy_batch.c │ │ │ ├── grouping_policy_hash.c │ │ │ ├── grouping_policy_hash.h │ │ │ ├── hashing/ │ │ │ │ ├── CMakeLists.txt │ │ │ │ ├── batch_hashing_params.h │ │ │ │ ├── hash64.h │ │ │ │ ├── hash_strategy_common.c │ │ │ │ ├── hash_strategy_impl.c │ │ │ │ ├── hash_strategy_impl_single_fixed_key.c │ │ │ │ ├── hash_strategy_serialized.c │ │ │ │ ├── hash_strategy_single_fixed_2.c │ │ │ │ ├── hash_strategy_single_fixed_4.c │ │ │ │ ├── hash_strategy_single_fixed_8.c │ │ │ │ ├── hash_strategy_single_text.c │ │ │ │ ├── hashing_strategy.h │ │ │ │ ├── template_helper.h │ │ │ │ └── umash_fingerprint_key.h │ │ │ ├── plan.c │ │ │ ├── plan.h │ │ │ ├── plan_columnar_scan.c │ │ │ └── vector_slot.h │ │ ├── planner.c │ │ ├── planner.h │ │ ├── process_utility.c │ │ ├── process_utility.h │ │ ├── reorder.c │ │ └── reorder.h │ └── test/ │ ├── .gitignore │ ├── CMakeLists.txt │ ├── expected/ │ │ ├── agg_partials_pushdown.out │ │ ├── attach_chunk.out │ │ ├── bgw_custom.out │ │ ├── bgw_db_scheduler.out │ │ ├── bgw_db_scheduler_fixed.out │ │ ├── bgw_job_ddl.out │ │ ├── bgw_job_stat_history.out │ │ ├── bgw_job_stat_history_errors.out │ │ ├── bgw_job_stat_history_errors_permissions.out │ │ ├── bgw_policy.out │ │ ├── bgw_reorder_drop_chunks.out │ │ ├── bgw_scheduler_control.out │ │ ├── bgw_scheduler_restart.out │ │ ├── bgw_security.out │ │ ├── bgw_telemetry.out │ │ ├── cagg-15.out │ │ ├── cagg-16.out │ │ ├── cagg-17.out │ │ ├── cagg-18.out │ │ ├── cagg_bgw-15.out │ │ ├── cagg_bgw-16.out │ │ ├── cagg_bgw-17.out │ │ ├── cagg_bgw-18.out │ │ ├── cagg_bgw_drop_chunks.out │ │ ├── cagg_ddl-15.out │ │ ├── cagg_ddl-16.out │ │ ├── cagg_ddl-17.out │ │ ├── cagg_ddl-18.out │ │ ├── cagg_deprecated_bucket_ng.out │ │ ├── cagg_direct_compress.out │ │ ├── cagg_drop_chunks.out │ │ ├── cagg_dump.out │ │ ├── cagg_errors.out │ │ ├── cagg_exp_monthly.out │ │ ├── cagg_exp_next_gen.out │ │ ├── cagg_exp_origin.out │ │ ├── cagg_exp_timezone.out │ │ ├── cagg_invalidation.out │ │ ├── cagg_invalidation_variable_bucket.out │ │ ├── cagg_joins.out │ │ ├── cagg_multi.out │ │ ├── cagg_on_cagg.out │ │ ├── cagg_on_cagg_joins.out │ │ ├── cagg_permissions-15.out │ │ ├── cagg_permissions-16.out │ │ ├── cagg_permissions-17.out │ │ ├── cagg_permissions-18.out │ │ ├── cagg_planning.out │ │ ├── cagg_policy.out │ │ ├── cagg_policy_concurrent.out │ │ ├── cagg_policy_incremental.out │ │ ├── cagg_policy_move.out │ │ ├── cagg_policy_run.out │ │ ├── cagg_query-15.out │ │ ├── cagg_query-16.out │ │ ├── cagg_query-17.out │ │ ├── cagg_query-18.out │ │ ├── cagg_query_using_merge-15.out │ │ ├── cagg_query_using_merge-16.out │ │ ├── cagg_query_using_merge-17.out │ │ ├── cagg_query_using_merge-18.out │ │ ├── cagg_refresh_using_merge.out │ │ ├── cagg_refresh_using_trigger.out │ │ ├── cagg_tableam.out │ │ ├── cagg_union_view-15.out │ │ ├── cagg_union_view-16.out │ │ ├── cagg_union_view-17.out │ │ ├── cagg_union_view-18.out │ │ ├── cagg_usage-15.out │ │ ├── cagg_usage-16.out │ │ ├── cagg_usage-17.out │ │ ├── cagg_usage-18.out │ │ ├── cagg_utils.out │ │ ├── cagg_uuid.out │ │ ├── cagg_watermark.out │ │ ├── chunk_api.out │ │ ├── chunk_column_stats.out │ │ ├── chunk_merge.out │ │ ├── chunk_publication_compression.out │ │ ├── chunk_utils_compression.out │ │ ├── chunk_utils_internal.out │ │ ├── columnar_index_scan-15.out │ │ ├── columnar_index_scan-16.out │ │ ├── columnar_index_scan-17.out │ │ ├── columnar_index_scan-18.out │ │ ├── columnar_scan_cost.out │ │ ├── columnstore_aliases.out │ │ ├── compress_auto_sparse_index.out │ │ ├── compress_batch_size.out │ │ ├── compress_bgw_reorder_drop_chunks.out │ │ ├── compress_bitmap_scan.out │ │ ├── compress_bloom_dml.out │ │ ├── compress_bloom_hash.out │ │ ├── compress_bloom_hash_1.out │ │ ├── compress_bloom_legacy_v1.out │ │ ├── compress_bloom_sparse-15.out │ │ ├── compress_bloom_sparse-16.out │ │ ├── compress_bloom_sparse-17.out │ │ ├── compress_bloom_sparse-18.out │ │ ├── compress_bloom_sparse_debug.out │ │ ├── compress_compbloom_basics.out │ │ ├── compress_compbloom_config.out │ │ ├── compress_compbloom_index_add.out │ │ ├── compress_compbloom_index_drop.out │ │ ├── compress_compbloom_manual_config.out │ │ ├── compress_compbloom_upsert.out │ │ ├── compress_composite_bloom_debug.out │ │ ├── compress_default.out │ │ ├── compress_dml_copy.out │ │ ├── compress_explain.out │ │ ├── compress_float8_corrupt.out │ │ ├── compress_qualpushdown_complex.out │ │ ├── compress_qualpushdown_saop.out │ │ ├── compress_sort_transform.out │ │ ├── compress_sparse_config.out │ │ ├── compress_unordered_sort.out │ │ ├── compressed_collation.out │ │ ├── compressed_detoaster.out │ │ ├── compression.out │ │ ├── compression_algos.out │ │ ├── compression_allocation.out │ │ ├── compression_bgw.out │ │ ├── compression_bool_vectorized.out │ │ ├── compression_bools.out │ │ ├── compression_conflicts.out │ │ ├── compression_constraints.out │ │ ├── compression_create_compressed_table.out │ │ ├── compression_ddl.out │ │ ├── compression_defaults.out │ │ ├── compression_delete_bitmapscan-15.out │ │ ├── compression_delete_bitmapscan-16.out │ │ ├── compression_delete_bitmapscan-17.out │ │ ├── compression_delete_bitmapscan-18.out │ │ ├── compression_errors.out │ │ ├── compression_fks.out │ │ ├── compression_hypertable.out │ │ ├── compression_indexcreate.out │ │ ├── compression_indexscan.out │ │ ├── compression_insert.out │ │ ├── compression_merge.out │ │ ├── compression_null_dump_restore.out │ │ ├── compression_nulls_and_defaults.out │ │ ├── compression_permissions-15.out │ │ ├── compression_permissions-16.out │ │ ├── compression_permissions-17.out │ │ ├── compression_permissions-18.out │ │ ├── compression_policy.out │ │ ├── compression_qualpushdown.out │ │ ├── compression_segment_meta.out │ │ ├── compression_sequence_num_removal.out │ │ ├── compression_settings.out │ │ ├── compression_sorted_merge.out │ │ ├── compression_sorted_merge_columns.out │ │ ├── compression_sorted_merge_distinct.out │ │ ├── compression_sorted_merge_filter.out │ │ ├── compression_sorted_merge_unordered.out │ │ ├── compression_trigger.out │ │ ├── compression_update_delete-15.out │ │ ├── compression_update_delete-16.out │ │ ├── compression_update_delete-17.out │ │ ├── compression_update_delete-18.out │ │ ├── compression_uuid.out │ │ ├── create_table_with.out │ │ ├── decompress_index.out │ │ ├── decompress_memory.out │ │ ├── decompress_vector_qual.out │ │ ├── detach_chunk.out │ │ ├── direct_compress_copy.out │ │ ├── direct_compress_insert.out │ │ ├── feature_flags.out │ │ ├── fixed_schedules.out │ │ ├── foreign_keys_test-15.out │ │ ├── foreign_keys_test-16.out │ │ ├── foreign_keys_test-17.out │ │ ├── foreign_keys_test-18.out │ │ ├── hypertable_generalization.out │ │ ├── information_view_chunk_count.out │ │ ├── insert_memory_usage.out │ │ ├── jit.out │ │ ├── license_tsl.out │ │ ├── merge_append_partially_compressed.out │ │ ├── merge_chunks.out │ │ ├── merge_compress.out │ │ ├── modify_exclusion-15.out │ │ ├── modify_exclusion-16.out │ │ ├── modify_exclusion-17.out │ │ ├── modify_exclusion-18.out │ │ ├── move.out │ │ ├── ordered_append-15.out │ │ ├── ordered_append-16.out │ │ ├── ordered_append-17.out │ │ ├── ordered_append-18.out │ │ ├── plan_skip_scan-15.out │ │ ├── plan_skip_scan-16.out │ │ ├── plan_skip_scan-17.out │ │ ├── plan_skip_scan-18.out │ │ ├── plan_skip_scan_dagg-15.out │ │ ├── plan_skip_scan_dagg-16.out │ │ ├── plan_skip_scan_dagg-17.out │ │ ├── plan_skip_scan_dagg-18.out │ │ ├── plan_skip_scan_dagg.out │ │ ├── plan_skip_scan_notnull.out │ │ ├── policy_generalization.out │ │ ├── privilege_maintain.out │ │ ├── read_only.out │ │ ├── rebuild_columnstore_tests.out │ │ ├── recompress_chunk_segmentwise.out │ │ ├── recompression_integrity_tests.out │ │ ├── recompression_integrity_unordered.out │ │ ├── reorder.out │ │ ├── scheduler_fixed.out │ │ ├── size_utils_tsl.out │ │ ├── skip_scan.out │ │ ├── skip_scan_dagg.out │ │ ├── split_chunk.out │ │ ├── telemetry_stats.out │ │ ├── transparent_decompression-15.out │ │ ├── transparent_decompression-16.out │ │ ├── transparent_decompression-17.out │ │ ├── transparent_decompression-18.out │ │ ├── transparent_decompression_join_index.out │ │ ├── transparent_decompression_ordered_index-15.out │ │ ├── transparent_decompression_ordered_index-16.out │ │ ├── transparent_decompression_ordered_index-17.out │ │ ├── transparent_decompression_ordered_index-18.out │ │ ├── transparent_decompression_queries-15.out │ │ ├── transparent_decompression_queries-16.out │ │ ├── transparent_decompression_queries-17.out │ │ ├── transparent_decompression_queries-18.out │ │ ├── tsl_tables.out │ │ ├── uncompressed_size.out │ │ ├── unlogged.out │ │ ├── uuid_policies.out │ │ ├── vacuum.out │ │ ├── vector_agg_byte.out │ │ ├── vector_agg_default.out │ │ ├── vector_agg_expr.out │ │ ├── vector_agg_filter.out │ │ ├── vector_agg_functions.out │ │ ├── vector_agg_groupagg.out │ │ ├── vector_agg_grouping.out │ │ ├── vector_agg_memory.out │ │ ├── vector_agg_modify_hypertable.out │ │ ├── vector_agg_param.out │ │ ├── vector_agg_planning-15.out │ │ ├── vector_agg_planning-16.out │ │ ├── vector_agg_planning-17.out │ │ ├── vector_agg_planning-18.out │ │ ├── vector_agg_segmentby.out │ │ ├── vector_agg_text.out │ │ ├── vector_agg_uuid.out │ │ ├── vector_qual_default.out │ │ └── vectorized_aggregation.out │ ├── fuzzing/ │ │ └── compression/ │ │ ├── array-bool/ │ │ │ └── empty │ │ ├── array-text/ │ │ │ ├── 01accf1c403c681e8ccc10349c97a28ef2afbdd3 │ │ │ ├── 0d699bc41031c7525fa65c0ab267f34f608eef6a │ │ │ ├── 0dbf553220bcd27478f10999d679d564a11632a1 │ │ │ ├── 0e356ba505631fbf715758bed27d503f8b260e3a │ │ │ ├── 0f873e20f9cc905c940207795842a8f89598bb78 │ │ │ ├── 13f402104e20e5a38290bdc5fec85a46ae36bd73 │ │ │ ├── 143a44ebe20d00b1c6bdf12487758974467b504f │ │ │ ├── 1641a06baa3defcf9a1b704cb94ea3387f40f2ad │ │ │ ├── 2123898ce1d45564480b9ff51cf391b87dcc5a07 │ │ │ ├── 22e70b0d023eac54b28a067aac0ab8e4eb75887b │ │ │ ├── 3862930f38ef2ac7387e3e47191234094aee7c0a │ │ │ ├── 3f82cf837a5f3fae26f7cbb25ed3d903eed71687 │ │ │ ├── 428361124252a8847f1182747c936696bc43543b │ │ │ ├── 4cd1b3841a01a3abc7f1cec6325130fd109dee84 │ │ │ ├── 513033f491e3f9ae4cf779c239158c9063f2af4d │ │ │ ├── 58420143cbcd2fe40fd1409948b6a78d3bf14a32 │ │ │ ├── 592e2bafa4637d9786e9d14c5f1ca512e0076940 │ │ │ ├── 5ba93c9db0cff93f52b521d7420e43f6eda2784f │ │ │ ├── 5d1be7e9dda1ee8896be5b7e34a85ee16452a7b4 │ │ │ ├── 6a126964d691ada7de1d25c976c4e7b481858665 │ │ │ ├── 6c0295b5f6b25ca492bafd609ba5c9494785651f │ │ │ ├── 76023b236d960f02d7fb41c7a1fa4d28dafa7c2d │ │ │ ├── 9159cb8bcee7fcb95582f140960cdae72788d326 │ │ │ ├── 98e49024fd7e15859ec345ee83a01fec0656ad94 │ │ │ ├── a24f8ad32bdadaee87b839765599bc63dfcbd62a │ │ │ ├── a3d453f14af5370aae60089101d659fb12c3aff4 │ │ │ ├── a42c6cf1de3abfdea9b95f34687cbbe92b9a7383 │ │ │ ├── array1 │ │ │ ├── b6f695dd09d681d144c71e52ebe565a2567a23f9 │ │ │ ├── b71fa13e7c2fee50d39a87fc927c31256f8c4af3 │ │ │ ├── bc6960f7ed1b4b216c5cbc2721ea5136ffb80aae │ │ │ ├── c13eaced24e2a5039d3fbeef655fc3cf827a2be7 │ │ │ ├── c2588a11e70caad0b9c91272f81d48348b94f938 │ │ │ ├── c88b988789743b6aad8ef68278fc383847a37ddf │ │ │ ├── crash-1ac67d1b8ebedeb93c515b5244f6e60613c1af0b │ │ │ ├── crash-b6cfa8632a8bf28e90198ec167f3f63258880f77 │ │ │ ├── d57ef126bce1cf2bb5a63addf172a9cf9bedb7da │ │ │ ├── ebddebf24b791e82c4ac007197581ec4fa2886ee │ │ │ ├── ec536d9d30f08137531d4bfe676a545d897ae98b │ │ │ ├── ede4129ca9c4d8dbd649aa6f2d5c0038e1537716 │ │ │ ├── ef519c31ea5d415293021b0ca83dd655701d2c13 │ │ │ ├── f2f014ef49bdaf4ff29d4a7116feff81e7015283 │ │ │ ├── f41f46df995dd4c7690f27978152ead25ccd5c75 │ │ │ ├── length-saw │ │ │ └── with-nulls │ │ ├── array-uuid/ │ │ │ └── empty │ │ ├── bool-bool/ │ │ │ ├── 0187ae825e656c01623dd6d982ab50d490c0c417 │ │ │ ├── 03fa95c77415b6c8691a0dc1d13e195ccd1b897c │ │ │ ├── 050a6450bd7d5dfb7504532cee17be1b973174bc │ │ │ ├── 0699e368418710fc791c8301af3df8895bcfb10e │ │ │ ├── 0bcdbd74d4c3526b71a46282bcfecd7554e70cd9 │ │ │ ├── 11f4de6b8b45cf8051b1d17fa4cde9ad935cea41 │ │ │ ├── 168bf1eed49508508684c0f0f00b3bca3ed1de9c │ │ │ ├── 1957b4d9af53d57c71ce03fe726f3dd55cc5e495 │ │ │ ├── 1f3a7fa8e785a9db380099f29b52ef4598eade72 │ │ │ ├── 34a3bedc4867e71ae61074746c225378159130b2 │ │ │ ├── 3542b3da299b536e44e55dca08549bb5ae401461 │ │ │ ├── 3e8a424e6e48bde8488851a191893d2aab7fa962 │ │ │ ├── 50f65669e0378f3764acc65e9ec8664db7b77a63 │ │ │ ├── 5ba93c9db0cff93f52b521d7420e43f6eda2784f │ │ │ ├── 5e1dbb627121d05e51a038be3425414bc3d463f7 │ │ │ ├── 6326b8e4ed85d653f9a043fca18c638dd4df6d43 │ │ │ ├── 662b18f40ac12457327499dbe84ad32e1a995540 │ │ │ ├── 8dc00598417d4eb788a77ac6ccef3cb484905d8b │ │ │ ├── 96ddb4dc6bf60a475735388b8da21dc601275c4c │ │ │ ├── a26c1d3763601a41dc1e36869238eb2986cd1972 │ │ │ ├── ac6f615983f527a43aff8bbf3edebef8cc881c33 │ │ │ ├── e287fdeb24184e2c2f8e162ba90b1fe2dec90dcc │ │ │ └── empty │ │ ├── deltadelta-int8/ │ │ │ ├── 000226b14bbb5e685204e626484dba42d8996992 │ │ │ ├── 0005721f2bceba264a5cd791cf6a1e36aaed780e │ │ │ ├── 001edd3c9db1c2fe30c9f9d446a0b42b58569ddd │ │ │ ├── 002007b1d394a27c4332710e3af9c00dae04aac5 │ │ │ ├── 0043fbabf9a4d188fb2b4654914a2da5db3d7485 │ │ │ ├── 005c46a47f8bf11c68af4134ce6eb0c22db620ee │ │ │ ├── 00617fee7e086d32d20dd281ddafa8a726646744 │ │ │ ├── 00742699a5aad2ead456e90d61c5f9faba0293a6 │ │ │ ├── 0076eef47ad215c435ecdb57e2494af0f55fd81a │ │ │ ├── 008110c25d8a3a8a8170b7d3ba621cf06d4853c0 │ │ │ ├── 0081a9f0f585a9b67b2103220cc60d36e2dbbd03 │ │ │ ├── 0088c8e6e015602ba5e40a90986019b4ad512f65 │ │ │ ├── 009ffc43f38a56a761e7b83b0f2c8bd026a563bb │ │ │ ├── 00a2eb7a7e3fdc3bfcac0045554586d5fb5502f3 │ │ │ ├── 00d5f2c9add73bceda87ef3acd392122d8adb3e4 │ │ │ ├── 00f85251b4c47fb81acc131a198a1bc98ca779a1 │ │ │ ├── 015c684cdef94cdf964d0800d325c6849d014324 │ │ │ ├── 016dd5961834fbbcc799ffa91004247e868e2781 │ │ │ ├── 01e3938a0dcce30f75133d8ad80220d2d613f008 │ │ │ ├── 01ec3391b4d39420ac8414c675fc1269955e3c57 │ │ │ ├── 02484d2db80b46ff0ae20eafddb5dc45a53140b6 │ │ │ ├── 0270028dc334db0bd34fca6e6abbcb9ecdfac57a │ │ │ ├── 029aaf2ff41035a0bf05a6f97fb68d6a62e32974 │ │ │ ├── 02cabee9534673ed1f5b4ce6dc957952166cbe1b │ │ │ ├── 02d459bcf9a9e071c18db18c808245c16833ca09 │ │ │ ├── 02eb701fa0ab34abda28be9fbba19582790432ad │ │ │ ├── 03607062b7b4b60f979bdd121981128b7002a561 │ │ │ ├── 039c250391321f58181a4cf10a804f6e529eb57e │ │ │ ├── 03f6be1bf6529592f3e57d28e1e8ad46953f7c51 │ │ │ ├── 04096d38044e4074f2b0541b455ed0ecc9a059fe │ │ │ ├── 041ef2e92911ab4633fbc16bf5457fe25d723ae8 │ │ │ ├── 044d5359fc697f71c9f1cd7ab773f025f38ccae6 │ │ │ ├── 0459cc74fd756782cfb3c84c956abf1df632d453 │ │ │ ├── 04826e1568e927a9de9d23e013fac8f4704d230e │ │ │ ├── 048b8752a6490e051d3b4a9ac3c6279953cc8279 │ │ │ ├── 04bdfef2ca95152c9a0525bee938b2ed74c316e9 │ │ │ ├── 04f3abff8e30697834380a3c2f0e238cab099e2b │ │ │ ├── 05f24e5eb37b55471dd9eae8c4194b4f65e25067 │ │ │ ├── 05fb4342fad4ba26e0e2d21742bc65fe70cf3ff5 │ │ │ ├── 06033906c750496425efa6b85c31caa8b7781f34 │ │ │ ├── 063e248ade3d6d40b6cf4c96ef4ff0164c099f7c │ │ │ ├── 06449ffa405bcd2cf8652e4fc82f93fb701f053c │ │ │ ├── 0663f93bee6cc22be2a74ffd2719d6296e74334d │ │ │ ├── 0688f3c387bbd17d82c2b3f7dadf3042f768887b │ │ │ ├── 06905c0515b1cc8f21c39c69adf954607aa1ef97 │ │ │ ├── 06f2e370b0e257bc75726698ff4b05d31fbc82ca │ │ │ ├── 070b4ae2222399e9432f58f67029a7794526a992 │ │ │ ├── 0720f498d7a2de389fee84f09d7d5f8decfa76bd │ │ │ ├── 0761d21c1bed0dc463b92f5c114d170293f17735 │ │ │ ├── 078dd3d695a19d89d78ca5ce6cb5ad394b72800f │ │ │ ├── 07946f273216738b476a364fedf5eaae34c08ecc │ │ │ ├── 07c4739c2e79966c2678d5189115d31469f6cfa6 │ │ │ ├── 07d34e1cb03d70ba8e83a11c1cddad7fcb083855 │ │ │ ├── 07ddb4cff1428ceb1fc0edb6dc8ad7bfadcb8233 │ │ │ ├── 07e310a31ea039330c339d6a5140c067dee5b44f │ │ │ ├── 081e0108c111c717c98f33c9af02b536eaf50349 │ │ │ ├── 083014d7487b279086470af22a3716c7bfc00630 │ │ │ ├── 08e95e19e9ff57fc53b0959896df624e02fdf75f │ │ │ ├── 090a8933ed1391740a094bb7a19e913db10ca6c9 │ │ │ ├── 094f5fdd281c4a9d67f1fac1179708a0753071b3 │ │ │ ├── 09808f60497dfc988e43f40c8dcfc864b46e4c23 │ │ │ ├── 09c504d74d0697ff4bb25681dbd5f9bbe603154b │ │ │ ├── 0a1054b55fab3e07cb841de17cf76a14937f2d49 │ │ │ ├── 0a4e527b4885a1a89b1f7b899282c09023184075 │ │ │ ├── 0a606eef286c1a80546c24253468898684b711f4 │ │ │ ├── 0aa2ce83cc47ebd7457452ee44eec44ef5ae8945 │ │ │ ├── 0b4693f634921fc1aff558120380550054c7dc79 │ │ │ ├── 0b4c9c405bc395ee2064a3790820618c62af22eb │ │ │ ├── 0b8493a4edb47558fa3dfa9cb3b3ca0d3af16971 │ │ │ ├── 0b9dbabf75544aeaeb4301aa3e3b367681eee4bd │ │ │ ├── 0bb58d3d928b75d5bbbb534d8fbcc02e7cec61b7 │ │ │ ├── 0c1bf5358361ad43e3718d79b16153c848257dbc │ │ │ ├── 0c2cf9c105698a12da9f7c1de00897a6039adbd9 │ │ │ ├── 0ceae4f10fc61a7f91b69b3481882a5ce5e57c38 │ │ │ ├── 0d48d19aef5682445e6b8724da523f6ad334d5c8 │ │ │ ├── 0d87d016d0e51c749adb95b1192d3cbdacd8ff96 │ │ │ ├── 0da96dc1d241308ec033b9628a519fda371defc6 │ │ │ ├── 0dc7fd92992bdf76b8a7edf1819ebfbb1a7b6382 │ │ │ ├── 0e039859b8723df22cb8f4cf0831efd10f7df2d3 │ │ │ ├── 0e6947636358abbcaf4704335de9ce6bf1b3bb37 │ │ │ ├── 0ea3cbbe4df13a9c00137579b048da07ac4620da │ │ │ ├── 0f069ba7b764acb518096d549c272968ef139baf │ │ │ ├── 0f24070fee9fd323f32004b8a19e19e4899669c5 │ │ │ ├── 0f5b129fcbfde4a2b74de973536594536a702b17 │ │ │ ├── 0f6b55a491e86c632769c11bcd0e401e8d736c17 │ │ │ ├── 0fbc8b98efd055002d0a9b1f1585c16e1bcafbf5 │ │ │ ├── 0fe403cf9e514edaba7624592e57dedd44d8f974 │ │ │ ├── 10cb10b75109d9f5a51df8cc25e5922ab9304486 │ │ │ ├── 1234967d87289b5476e82e03a5fdaddfb2d89f5d │ │ │ ├── 12429bc13e7a80e8566e6355325baba779106594 │ │ │ ├── 12af146da39094e9e46dd07e119ef8104840a0d8 │ │ │ ├── 13197c60bcdca232644653ced7bea29b141dc3d0 │ │ │ ├── 133c2effbe5ad58905ac393dfdd16ca1fa2c7eb7 │ │ │ ├── 134b9385f7a8269fa53a25d14207c72ec1088f81 │ │ │ ├── 13f9787e9557c5d37424143c6bb87a34a91dc233 │ │ │ ├── 14259b5a7c8b7095dd40e602ce8fdc4d208c00a3 │ │ │ ├── 14964c2e5d120c7acf5fca81233b916b3b3016fc │ │ │ ├── 1573858c8eb6d027bbdaa303eaf5d57c978b3465 │ │ │ ├── 1669aab7adfb7e65162eda52612d49e0f252d270 │ │ │ ├── 1697392b1fb9f4b49c80030d6b3017d72d4ede4c │ │ │ ├── 176236dd152ebf4fa259212b539fbc49d9dd1799 │ │ │ ├── 17896100eabd31d729b38a703da3f31e57fdf4b8 │ │ │ ├── 17af15070570b101ef4c57ade5ad766e4924aabc │ │ │ ├── 17e00a1df1c9b937d50a5a890562587f1196b646 │ │ │ ├── 18c4bd81a648ee9d7db0dc68c4615a22ffd4cd72 │ │ │ ├── 19308702d029cceef2220db8c57fbae88ef17f44 │ │ │ ├── 1937ed2d8de2d8577fc48a407ded56d272e40096 │ │ │ ├── 1a8f7f5dd5f42e1ae89b55a47e1fa45cab816e62 │ │ │ ├── 1b110f70f70d47d01b6f55110d5719e3dd19d893 │ │ │ ├── 1b1f4b7a41866daf1fd4f2aaeb6aaecea3bea926 │ │ │ ├── 1b2d9c2d949b251a2ca3eecd2d8f29cf96334611 │ │ │ ├── 1b53e293c41a7b6bd041098fa91aab2e5ef4ab16 │ │ │ ├── 1ba5cd685ecccdac9caa92377346315ce6d6c293 │ │ │ ├── 1bc1d2f6da12c6cd0eaab779fe69b872c1b427b0 │ │ │ ├── 1c66535a7d66622db6aac4ee1df14b1db1369b41 │ │ │ ├── 1d1ae6b78379f1aea79abee08338cd4762ebab3c │ │ │ ├── 1d2e73934349bc80d13e3c1581eb915d7e658666 │ │ │ ├── 1d7c3373fc113bbe6f4eb12600fddde935ec69bb │ │ │ ├── 1dbe1fffa3a5edddbc95ecf5e496236386b1b928 │ │ │ ├── 1e783d97191cec0a7095c81805cc5de7a5d1355c │ │ │ ├── 1ea298a60f91377c5e8852dab8a10cdbc66a7702 │ │ │ ├── 1fce78dca10eecad74006cda5b0619c1530a90dd │ │ │ ├── 21d3312341b1626ba6bfd29568e2ef50c60538d8 │ │ │ ├── 2263ed26b47e2066384bab5c098b26f13faf5ee1 │ │ │ ├── 2344a89366d1eee87d8bb3e54b620b02c68abc96 │ │ │ ├── 237df2b50d5437a256391b6efde8d6ced3a006e4 │ │ │ ├── 2453ff3b056ab95e17a1d7dd99c82defc2b597d0 │ │ │ ├── 2477ff336402e5de52445520eb15e8704343ed00 │ │ │ ├── 24b6413c3745f8b7b2f729bae89ee399b0641beb │ │ │ ├── 24e6190084fd40e44ff278c02fb4e43c8cd29836 │ │ │ ├── 269f91653f9bbac218dfe0884c8c43ebbeccbcaf │ │ │ ├── 2723b24f11843b73282f94931b3f3ed696963ee2 │ │ │ ├── 275315c1e29d2d7fb15810a3d66953d5d68d8d62 │ │ │ ├── 2b20144d0f616bd4695c236e15ebaded83f5b598 │ │ │ ├── 2b40483bd9238d7036d56fa6dcd735fc52fbe195 │ │ │ ├── 2c6bd6e0bb02871d00ea542c1702fd81a0a86d21 │ │ │ ├── 2cbd4f4b339c999597a58fc5af53e4bfffc2b78c │ │ │ ├── 2cf2b0ae879c5a9120263a5ecb793425def7a8b0 │ │ │ ├── 2d9c2be54686acf5d5b2755384dd52ff4d19ee02 │ │ │ ├── 2dc8f286c1ea4f3dbdabc059f3a1243080cd62f8 │ │ │ ├── 2e18a24e3c46167c0f3d368b75760abdf8fac401 │ │ │ ├── 307ea4e260815f6a9e8b129307a38452976b692b │ │ │ ├── 311f3dceb49012061d4b72a831e347133c92bcdb │ │ │ ├── 313644ec39dc8694c470ab757ba111838ee53b1c │ │ │ ├── 31ae44f31abd373f0ff481806d3157f85ccae710 │ │ │ ├── 324102241d71e8c8a37d543dea5cb99ac7ae7c8d │ │ │ ├── 324ec9cb002617a880a56eb07f49657fc4483505 │ │ │ ├── 3493fcabf43e379422d876938aefff76d611ab95 │ │ │ ├── 349cbb7c3e3812603d102a39ed82b0bdac61410c │ │ │ ├── 350699ee4c96235151a3b75fdf0d54d482f71218 │ │ │ ├── 3625c8ad48799bd41b23ff6bc8a3c487702b3f00 │ │ │ ├── 36c0f8e79ee0a971f51204923c2f06cef5ae86e7 │ │ │ ├── 39764a75e150887f1b3d6c15ff2883112bce78d0 │ │ │ ├── 39d95b1ac8f9a92f57162e173e809bfd24dc38d8 │ │ │ ├── 3a2fa560af8026b2342777bf91ccd155a71af6d4 │ │ │ ├── 3c3b7a908a6b8303cc2ced4cdc2c548cedbb760f │ │ │ ├── 3c81c456a004ec4ccd7c80fe2eea2f0ef7ab7473 │ │ │ ├── 3cd038948d733c0b49ba6c78452cbd03186c69bb │ │ │ ├── 3d657fe063550eb5f8d8ef40b55590b321aa20f4 │ │ │ ├── 3da8fb2b32486ad7c716c76458668a6acfdf5f0e │ │ │ ├── 3e30f3f570dfee8350064c138494bd670d2fc795 │ │ │ ├── 3e8153e9034c230c77afe71a541d4f85ef1aefb8 │ │ │ ├── 3f8b816713d67daeb3c9a6dfd6a1fa9e651c96e7 │ │ │ ├── 40c238a192c1a36484f359cbf87e4a3c4cea8bd3 │ │ │ ├── 416d8c8f3c970b609e07c225a93db937ab8fd7a0 │ │ │ ├── 41ebd242b789740786e3ece0d4117dcadbf6a37e │ │ │ ├── 421c2d863ad0ae5d339f9a18293a38bc4b24d0fb │ │ │ ├── 444dc80536559d07351b2d3616cb40c3c1a9dcbd │ │ │ ├── 445df2c7e19680732514c77c9b4e8554dfd58a69 │ │ │ ├── 4815d6ffa9e96eac844fc31289df2d00932c95c1 │ │ │ ├── 487e4c6fb01756b58d769ca4a0015b5cb1017b7b │ │ │ ├── 48843aedaa180a1bb3b5106e8b55352276d0c586 │ │ │ ├── 492d805d4c8fdbc70e597d8d6e601238a3be324e │ │ │ ├── 4b81c48b8d565a4dcecfe7d694ea47a69deec3fa │ │ │ ├── 4c06c923331191aff0ec6cd7c19536e311be6d59 │ │ │ ├── 4c5b635d4e0e7685b5b3b3fe67fe1ead7aa8cb87 │ │ │ ├── 4c92c44bdf911e127f98b1f741a9429a416404b7 │ │ │ ├── 4cd9c9085b3974be6114e9c96b385fd5928169b3 │ │ │ ├── 4d56d33e33ce1f0d3a6e3e5582bd28665e3fa65a │ │ │ ├── 4d80df18bb50e97b97b10fe7ff50065944df7973 │ │ │ ├── 4e22436534f7ddda6023610945ad3fb84b08d5f2 │ │ │ ├── 4ed47503b5d6a6a97de63384162805adf4cc1a58 │ │ │ ├── 4ed96db01a061c6cf9f3236f9254989d12f65315 │ │ │ ├── 527d035f3b7718f9a3fc2ea09e79803d5169fc0a │ │ │ ├── 53157896277c12aaf92e63bd6eba876823c53644 │ │ │ ├── 5369e602a569f72fb2398c3dfa65806f870a4330 │ │ │ ├── 54acea7c142b7d17006615be6d40591bfdf6280c │ │ │ ├── 54f997c30666b3519baabf15d055a8468795076a │ │ │ ├── 55eb87d57c0f0f7fd3ea9d63c1eac58ee00199dd │ │ │ ├── 5958b905c02f53a22cdfabb51cd2823d13650241 │ │ │ ├── 5af121503212bc786fef31e2e3a51ce2031b1b84 │ │ │ ├── 5ba93c9db0cff93f52b521d7420e43f6eda2784f │ │ │ ├── 5de1bb11c456a3b400a4fd2b3c419ca129f099c5 │ │ │ ├── 62bad82a78a7694bf9f72575622d90eae3c1cba8 │ │ │ ├── 62dc4d708eb260cf29f94e5a9c65f89f6dd3fa43 │ │ │ ├── 640126de1ef49506804ad2af892b39fe0af37aa6 │ │ │ ├── 6565683c7f1f07d7cfa27c84426e1c386425a931 │ │ │ ├── 6629278a4f5b5143bac025355a79df241d0d32db │ │ │ ├── 673be270a1d5f97382728953dcbc1a749b8191fb │ │ │ ├── 67871c4d2cfe8eb0807cc8fc7c2b423d2c4be5d2 │ │ │ ├── 6bd8cd832b15cf0d5e3eea310d97b737faf8ee43 │ │ │ ├── 6c7b212dd6e420cc250464ab9517d2168e6c5edf │ │ │ ├── 6ddcfcba54e660ad5fca1bfc91670823cb71f1d2 │ │ │ ├── 6f0935dd5c60572c57d7bb0f5896da83cf4657a9 │ │ │ ├── 6f21d6eb06123ef2835d168b44b7574c39b06fc6 │ │ │ ├── 6fb7b33d08c092caa76e11decfc4e9431ace8d32 │ │ │ ├── 756e189c2c25c94854325a4c948a6df1181c1c13 │ │ │ ├── 75deb85a67a4c6b8a087733ba3a0cd938e63a41f │ │ │ ├── 79b7cb1655367697a866f041f8d7bca70b87ea5f │ │ │ ├── 7ca47c2e3eecc2acf0c7a0655af7c51546bde6d5 │ │ │ ├── 7d197122c22bef9423f360691dcb78dedd2c0c99 │ │ │ ├── 7d1aebf64a30affed22e9f885a0af617ef5bd572 │ │ │ ├── 7d2b0e79c750d7a3fa52f27aefb85a165b4b7133 │ │ │ ├── 7dd097f27180debdc68621b74e84cc448102d92e │ │ │ ├── 7e38e6f71b4a050f3fecdc08d51e3d1418f3fb0f │ │ │ ├── 80117ddd69955929af3ca5f981ba98a48012260b │ │ │ ├── 87446dfd5404badf9eddff3d071218025166ea84 │ │ │ ├── 8a7f547660cec12461032cfda4d1caf55bb07dec │ │ │ ├── 8d50f8e07c30c3521ccf5f399e3667d6da593ddf │ │ │ ├── 93012fe6454ae8c2c9049e784fdc31b444f31f69 │ │ │ ├── 94e9fdedcab956cafdb52ad634cf54487d840e3e │ │ │ ├── 96f00344ba748469ee00ffd0c2d3191a7978b644 │ │ │ ├── 9842926af7ca0a8cca12604f945414f07b01e13d │ │ │ ├── 99d7208df7ea6e626fa9c888e582f7c5432b243b │ │ │ ├── 9ff89f6c4cc178a314ac0b40ad0d823748491bd0 │ │ │ ├── a42c6cf1de3abfdea9b95f34687cbbe92b9a7383 │ │ │ ├── a58d0fc75ead12a9c29063a1d0db8fb80f395c31 │ │ │ ├── a7e9dc04db4e4b11302187e182dd349b563494df │ │ │ ├── ac2f350f4fa2ff262cb01d9d97974df5a2dff0be │ │ │ ├── af11ea42436ef96d9d8ffe0387fb5782ddbb348a │ │ │ ├── b2cf8461474181d99198df332c5dca521c64de2c │ │ │ ├── b461b7d5a152d72fe425d130a6181ea7b0b05f12 │ │ │ ├── b4f3a185d32f5ebe2b85bd678bba599d76f70a7a │ │ │ ├── ba00242b72f56b5d07786b7af10b9749a59a15ba │ │ │ ├── ba3d618f85780af05ad8e9ae2069603006ae9bbb │ │ │ ├── bcd8a144287c2ad0ca47a66cc3606aed7192d5ce │ │ │ ├── bde3b51ae38da58047d42a41371a217827d31f75 │ │ │ ├── c0df86d6b43ad023227f4618429d963fdc360e44 │ │ │ ├── c20513576febd09086017080392adbf18f0aabe7 │ │ │ ├── c39a1c24b6aabfd2656ac84c5c8fed16eede528e │ │ │ ├── c3b2bc6acbc107d0e9403e61661887935b73ce0c │ │ │ ├── c3e25a84177c1317713c0fa1ddcbaa90a7a9f1cc │ │ │ ├── c40202c42aa92830d9d7d3f2157a4fb0b53fbf11 │ │ │ ├── c67117cd08df138c69184b8717979f297eb3fced │ │ │ ├── c851a06d9dc94b4589a7dae6c527b747e1e474f6 │ │ │ ├── ce6480edc36e784863ce3125c7dbcf95f9cd20ca │ │ │ ├── d3130220f9b81ee04247af415825ddd82fdef93e │ │ │ ├── ddec96652840748881a1e8894c92f7fc3ada74be │ │ │ ├── debbb602a0332ccb345fb31c8c26db33eb8755d6 │ │ │ ├── defd2d06b29a1f7458ae0ee09a4d8551a7e13aec │ │ │ ├── e5137c545032f0c8c5ae6cabc5bbc2ede6370b06 │ │ │ ├── ed0f301384e5530814f95de4c53ac7e7301d35a5 │ │ │ ├── f2048a5d9e7244812355acd3308fe9758b84ef6a │ │ │ ├── f2076dbdb823721d4d275ee0d63c625d4e07d6cd │ │ │ ├── f3543673cd7f732b6680878a993b33199623cf62 │ │ │ ├── f39742a8bb9ccb0675a05436d4bdb86da25a9d2b │ │ │ ├── f8515665b1a6e5bd47075c62b27c76d612e45d8b │ │ │ └── fa0efb79a43b42c1f4ecfaed81dab190b9546bb8 │ │ ├── dictionary-bool/ │ │ │ └── empty │ │ ├── dictionary-text/ │ │ │ ├── 00e8dce1331978dafd0c87d03898913a00d5f427 │ │ │ ├── 03be2aaf02e2f78738dcd55209eb660c63460f73 │ │ │ ├── 06a9bb98f465ce2136ba9c5c3b15912de9101d7b │ │ │ ├── 0ce9675bea488114cd61e0902df0f65bb266b414 │ │ │ ├── 127d1cd7df6314984a355234cad8daee2a0a6f85 │ │ │ ├── 193ff1ad23aaeaa8feafcbe7168f259e5063e715 │ │ │ ├── 1a155dbc0885a4ce404a03ccad4f90e8dfb6838b │ │ │ ├── 1bdd8892fa9cd727ebf1b97101a04456b8ba7bc2 │ │ │ ├── 20294feb1598e5893bda9b1fe7b9568ea0af237c │ │ │ ├── 24a09483ef45200b6ed894e43934767755155e3d │ │ │ ├── 27de45122a7e9da969165f24f6942fc131cb17df │ │ │ ├── 29e8abf085d862cb208f5e476f628644de1c22a0 │ │ │ ├── 2a0fa91e546f986d25159ed1e7507ec4793df3a4 │ │ │ ├── 2d79b560f5c1dde8b7841ae6d77d616f26f5b3ab │ │ │ ├── 2dd04d16222832a133c966eb5e9c4e7ae4d31059 │ │ │ ├── 2e10aad1b62e9ad833ea94af977cd498ba7da057 │ │ │ ├── 2ee77e4ad0a5c13eb219c48f0e8964d9f6124737 │ │ │ ├── 3716f49a4dc3a527cc3682d04ae2036204c406ce │ │ │ ├── 373e78bec9ac0a14383172c8613ac9c3fbb04349 │ │ │ ├── 3902e1ec9e8894b6befc97144a957e69f696bc75 │ │ │ ├── 391c64d022add4f53fcc55761febf54e86465604 │ │ │ ├── 39e6777ccb030519cefc62205e2df23e703aa9fa │ │ │ ├── 3b2185a90d32decad650ceb443751a4023368cdd │ │ │ ├── 3e8223a2a8a5034ca9935ccb1a9c8f41f4dfd782 │ │ │ ├── 3f1762bf4bdff8f21a3de2f04afe2cc9bfd538a5 │ │ │ ├── 446dbbeac1d19ca3ac51def5be4e9fcccf97cdc6 │ │ │ ├── 44991dd092a92994af67db1b51c9ca42261c27d3 │ │ │ ├── 44a171e481ef28ccd2fa0cef7666d8107489c25b │ │ │ ├── 44dae141798a56015e84bb90b8d47f2d1d9db66e │ │ │ ├── 4875bd67fa63473075333ca3ccbc86eecf90fb49 │ │ │ ├── 48ddda607e4778f35289784b3f6c80234660856d │ │ │ ├── 4979455aba5b4c7d302af47c4fd941214e16d3a9 │ │ │ ├── 4e4ec17ed15eab3b2aaee34c46ca44e72789f384 │ │ │ ├── 4e8af02cd72c9425df8c3164b3a14bc1b70c6498 │ │ │ ├── 559b65125ca556ff1a57f82f9ae55a86b71c6296 │ │ │ ├── 55b0c7dbbd7470a644c43aaf8aaaa520631e3bb5 │ │ │ ├── 57a99548ae911ad4a20406a03b0a0ac7a9adc63a │ │ │ ├── 5a72bac420b736c0d530a9d4c861a374ad32f5a5 │ │ │ ├── 5ba93c9db0cff93f52b521d7420e43f6eda2784f │ │ │ ├── 5c9409528b92b40afa79d037eadcb73b859e94e6 │ │ │ ├── 5c99325fac6e6a77d673cf223fb3b3e62fb1e07e │ │ │ ├── 5e81672e813bd1e06ecba224f520328498f27ea8 │ │ │ ├── 626bf1b65f1a0c5fba061fac07a711f970b07a80 │ │ │ ├── 62ca0c60044ab7f33aa33d10a4ae715ee06b7748 │ │ │ ├── 6407ff9cce2be245bacff6693615b8e382ba2a96 │ │ │ ├── 664e56319f5a1ffc9bd3e9554f2358ace5a739ca │ │ │ ├── 66a4327f1269b132279cd6bedd1b82f5f112eda9 │ │ │ ├── 687464af84370f5d43d25acba442acc7fd14beec │ │ │ ├── 68c94392b09d47edea2a48f62808073bf83448f3 │ │ │ ├── 6af6e86bfe31a3941d3085227f051c05777657de │ │ │ ├── 6f04561347c9100edce326d87e065789d2d56185 │ │ │ ├── 726be829733ebbca258b51fc29a79e543de46677 │ │ │ ├── 75fc076b6fc8dac4f65c31fa4fd34ad236530422 │ │ │ ├── 76bbc1ead78624711303acd22377969f0962736b │ │ │ ├── 76fa9dc37fc42934404c72df57d296c663ee807d │ │ │ ├── 79ef4a8ba594c7a2b611bc33fc8b83fe8ac14998 │ │ │ ├── 7bbc7585698a7c3375bea9c3bcff8838722d8f64 │ │ │ ├── 7be90a9058961ac6ee07f764618805dfe8b6cfae │ │ │ ├── 8127c19b14b9a5750a4731aef2f900a72ec6d802 │ │ │ ├── 8426e28aabe6684eb7200bd032ff0dad5a5169af │ │ │ ├── 85e53271e14006f0265921d02d4d736cdc580b0b │ │ │ ├── 8a5c3216797d7a54adeafc86b708b942a9894b2f │ │ │ ├── 8c92cd8b3e908dad0b490baa09ee984fdf224b21 │ │ │ ├── 8f1eab4f75b343ac81f12c926a077aaa572cd002 │ │ │ ├── 910ae695ec5c0972098df24b9e4a910ef2c36959 │ │ │ ├── 91b36d91365f1f59f8874170ecde945d9eeb1316 │ │ │ ├── 95f1a48e7e1cbe4b91461f1251416245497ff131 │ │ │ ├── 9936dc85a37671ac657693400f490966090e391f │ │ │ ├── 994cc577406fe37f59e27ea1028a9d0a814af721 │ │ │ ├── 9a78211436f6d425ec38f5c4e02270801f3524f8 │ │ │ ├── 9a88a0ae40cf185ed2c9bf4ebde71b048030211d │ │ │ ├── 9b99593353a610c4bee0d6a94a01a3296080c0fb │ │ │ ├── a42f35cc555c689a38ef471b21fad93692f36a9c │ │ │ ├── a54a56388dd751dc1ea1727f8e2965b349b54800 │ │ │ ├── a707473dd0d734a745a15b98f20645839d69a660 │ │ │ ├── ae02ec1f395c202f6cd2965ea34d73dc35e10fdf │ │ │ ├── af094ea4a3607992332e87dbe90f1a0f0cf82e09 │ │ │ ├── af33a704edf520f6ccc1c6c51b06d39b5a7e82f8 │ │ │ ├── b18ecac8feda2826b91131b386b8842a1fca17e5 │ │ │ ├── b1eb62fe7596e0f62ef933c269b429851f73853b │ │ │ ├── b8f87323f660b627117ced725f6278ca8dc7fe42 │ │ │ ├── b931131d935fb27ebf7977285299dcf11bb52eb4 │ │ │ ├── ba200d8a4886abcdba4174f4b131db56e9128785 │ │ │ ├── bf8b4530d8d246dd74ac53a13471bba17941dff7 │ │ │ ├── c74adbd5fe4690b76e721fe2de60f04c5d494eae │ │ │ ├── c78643e37119bb0f817531ba2ff265d6ef53c64e │ │ │ ├── c92920944247d80c842eaa65fd01efec1c84c342 │ │ │ ├── crash-49789ae0866d7d630f2075dc26812433f4af1db3 │ │ │ ├── crash-5eeac6ca5053992914dfb318e02e4c657a65c7cf │ │ │ ├── crash-707526606a02c72364e1c8ea82357eead6c74f60 │ │ │ ├── crash-b0db762535226b28c0b55ffe00d5537fd8ef7e39 │ │ │ ├── crash-bd6e8aa1ebeb96cf8aa644c5ef6eb2214dee0ffc │ │ │ ├── crash-e5143387e8896dcfb0f95f8111538502cee38ce0 │ │ │ ├── d0f63f55c89c3220cd326e9395c12e2f4bd57942 │ │ │ ├── d421e94ef02d0e45a2f783b63f3fe6622b6776cd │ │ │ ├── dict1 │ │ │ ├── e0485f22a1d04b0df70035eafa33f1278a52b8a6 │ │ │ ├── e5c4a84e1935991b3103fecf70bf563eb82f7936 │ │ │ ├── e767ec96033f7d9da306711adab0a6557fd4b71e │ │ │ ├── eaafc08a833a503cc25218c854bc70b9655d6b38 │ │ │ ├── eb02ce7f9339084b7dfa707b412ee1b1f7046885 │ │ │ ├── ecbd22c462813a437898cfe2848a46e5d6a460c5 │ │ │ ├── eddf750270b16df6744f3bbfa6ee82271961f573 │ │ │ ├── ef8c99ef78bd1d6410375edc127cf4e708dc63e3 │ │ │ ├── efb13296f8f471aadfdf8083380d1e7ac9a6bbc5 │ │ │ ├── f0f2e7efda77af51c83fe7870bd04d1b93556116 │ │ │ ├── f2d42e12fb2ed451e7ba0b270a9065916a391fb1 │ │ │ ├── f6377e2f32fd888f1877518c26bf6be4e24f92bd │ │ │ ├── fc8f69146bf6f556444c5acd8053537a292db418 │ │ │ ├── fe56cff03603408c02ef6579df1958ba3cdbdd48 │ │ │ ├── fee17c65913f4eddcae3f69d1c2b6f318b938af2 │ │ │ └── with-nulls │ │ ├── dictionary-uuid/ │ │ │ ├── 31bba5d620d39268ee0d0b4acaf0b48ab78e7376 │ │ │ ├── 5ba93c9db0cff93f52b521d7420e43f6eda2784f │ │ │ ├── 974a2356cb6d533bc0c421623eae03c5f645cf64 │ │ │ ├── 9b99593353a610c4bee0d6a94a01a3296080c0fb │ │ │ ├── c4ea21bb365bbeeaf5f2c654883e56d11e43c44e │ │ │ └── empty │ │ ├── gorilla-float8/ │ │ │ ├── 000335f765ced2a4f901ac182723f543d684e01a │ │ │ ├── 001123a617c5cd66ad7122a7bf5a4df193fec5c5 │ │ │ ├── 0018f597ee9cd81341f63167e437891935ded799 │ │ │ ├── 008f5da38ab688b5013799fa2078080b87d1315a │ │ │ ├── 00c229a7320c63a40c31ea1268814fc13b15de90 │ │ │ ├── 00fa51c6e24f79b0ba39dd32e9f3b68561ae2eff │ │ │ ├── 00ffbad9a023d169b2b1943887702b3cac8b045c │ │ │ ├── 010e7a5025e9ff7ffacf3ce3e2176bcb8c29617b │ │ │ ├── 0189f688b66865cd51637e2f5f8347431fcc54ad │ │ │ ├── 018c00c72b324da47458186f1f1527945de2b3d7 │ │ │ ├── 019e2ae7986441e1220e4060aeee97063e5f5f7f │ │ │ ├── 01beb9afee5bdd566eebc3e11ab9a2bf937af072 │ │ │ ├── 01d284d9abc6f8798411204503b6e7092294497a │ │ │ ├── 01ee785469f584519839126202f99e66e0f6b62f │ │ │ ├── 02118691c5748861f850a0ddc6504520e9cf9d59 │ │ │ ├── 02210f01b6e9d651caacb78cbe1981c3d7d948cf │ │ │ ├── 025fa6c86875979f9890fd5821994db68b0d9e0d │ │ │ ├── 026d5ea723c4995420f38fc8e2f41f52fdc8934f │ │ │ ├── 02819a1d54865c2b35eb888056fa9d88d7da99a2 │ │ │ ├── 030401af4fb4f96e77e95d08e12261897120bfe8 │ │ │ ├── 0316e3ffcaf4e49f3ce450a4681df2010192c3d8 │ │ │ ├── 032a566a41de14096cb32800aa07ea600a9dc74d │ │ │ ├── 032d6f047e5c6e1f3cc3816ee1b8da3c8971bcbb │ │ │ ├── 033b463a196126041ec08b6e61b189973f1627f6 │ │ │ ├── 03b2ca74225ae7011134f9aa114040b106ec8caf │ │ │ ├── 03cda3811ac87f97a957ece7d963db8dda7aaa8d │ │ │ ├── 03e931d374e50b3e6612eb4a7b4eb6707c75f2ce │ │ │ ├── 043d96b51ea12b68bbbdc1f9bbe69bc3e29163b5 │ │ │ ├── 046d94fe08c43ec5b3ead226ef0206eb5c501e4b │ │ │ ├── 04867457469f20f351b2863fa3197d79d739d20b │ │ │ ├── 04f6eecba790bef1c51a27366a658141f06f765f │ │ │ ├── 055f0d536e9e4af4b9769b2fc4504d00516034f4 │ │ │ ├── 0561d22c62592505cf5bd0a3943cf9aae716e5ae │ │ │ ├── 05a4b23a5b4806fadc1512f3962eacfd56b0b00b │ │ │ ├── 05a5a98579e2643e226e5571ba5d24339d622641 │ │ │ ├── 05ad10df436f072a5bcae26764ce310ec7ce6e6b │ │ │ ├── 05e1c72278fac592cb882971bab84d568799ca57 │ │ │ ├── 06498d96e93a66c56b0b5cb271fc67e7dbaefad0 │ │ │ ├── 06512255ce67fd4de2b6472be237c8dc9375d1b9 │ │ │ ├── 068dd3742f3f3beb0d9a00141a05ec1f16054373 │ │ │ ├── 06e255ce19ca34158bb0ac82bcef108367ca3267 │ │ │ ├── 06e51dd399ecef706d99060188e41dccb79be52f │ │ │ ├── 06f43a625eaf0da2f2bf2fa9143a659fd5b2ff0b │ │ │ ├── 070bd26d7cc8fcba32ef29da3ad625824c38d343 │ │ │ ├── 07419f558bfcb7a18c7fa7598a9bda076e3bfe39 │ │ │ ├── 07574d2f3e74a3bc135f59c033cd37482bd4c521 │ │ │ ├── 07707fa8bdb6bd86a827d7a4822dc25d8ea58f8b │ │ │ ├── 078bb3b1836c346eb4025764c06c1c94da8aed31 │ │ │ ├── 07fcc85472883b813b69b0e4e4a795c9a9b1a3c6 │ │ │ ├── 08291961257285a1e2137a0bce9ea3b262f164ef │ │ │ ├── 08bf43b8834a2a3663d2c6d287628816347eddc4 │ │ │ ├── 08f9d53a5a7d64f61b817786ac3acff477f8cbf7 │ │ │ ├── 0933cb848b5bab240adf85aa088d013c633283e5 │ │ │ ├── 0942e894b6559b62b4b52f2df1b409f4a7de53b0 │ │ │ ├── 0943eafc6afd3a6ade048635011c148db8f4d8d0 │ │ │ ├── 0982c40ef6669e243d31221b3e54e4eb58fd6609 │ │ │ ├── 0998446366e5a5f8d4d1ed817ebdf627df5bf145 │ │ │ ├── 09b35f10899c4282030d0fc8dfadff0a1366ed2c │ │ │ ├── 09f6ee62d7b28e763bc3dcddb730b233650b757a │ │ │ ├── 0a062e7146194e5149c24338b968ee5bbaafd40d │ │ │ ├── 0b0ed38d0ad51a990cf2b16c727bca9f385046e1 │ │ │ ├── 0b2cfbcda117da0fb38239e0bbd69149bc4be902 │ │ │ ├── 0b4534d5ccbde3ccb4ffedf6c35f1c151371b81e │ │ │ ├── 0b88c467e94529331ef6b9672a55c0917048ac36 │ │ │ ├── 0bb1287fc41b93ec186a6cd0a4e2abbe5ac25d20 │ │ │ ├── 0bdbb69df444c03dffdf45540b123857e8d52c41 │ │ │ ├── 0cf821743f73799451e8bd59b4493a2af6af993e │ │ │ ├── 0dab00d8d9d07f05544561f2184043891ac024c1 │ │ │ ├── 0e229390a2419e54659c91fac8fda3776c8d3597 │ │ │ ├── 0e5a473aa26bb4611426ff7c887793ab5bb11b00 │ │ │ ├── 0ecd6dd291cd9f1ff004dbf59deeaae538be0d5e │ │ │ ├── 0ed43cdbd0d48f987df9e39916fe6d7dca351372 │ │ │ ├── 0f4095e80f0972faa28e52867819e9ad4def29cf │ │ │ ├── 1013fec78493721af7513209cc4208d26ec828f2 │ │ │ ├── 103541e080d45c3543c343127acae3b09e9f7947 │ │ │ ├── 103e54b97fda4fb0ccdbb6eefb570786f8b06c3d │ │ │ ├── 1070cb122d3b0f16e90476d2756732c5faef9180 │ │ │ ├── 10cbc61c2d50941b8ee84950485330b51020b8ca │ │ │ ├── 112faf041b40f005bf544dfad5f108aebfe9932c │ │ │ ├── 113bd1949f408fed9ee1bf839de2e7f7a93c841f │ │ │ ├── 114fd66ac1f1853ed91e1cf08abb64b9cf24ba24 │ │ │ ├── 125897cf81e066dcf7a848a33d66e32c65aeb5ad │ │ │ ├── 125b7489711c008100163278833976214efd3c06 │ │ │ ├── 129c9ac9ab87801820904a3b64bd6754e7de31cf │ │ │ ├── 12da17d3510e43fa87404a6cc4e89590b7ab7b6f │ │ │ ├── 12f2a8f93e7fdd944afbee2fbd14575db669d350 │ │ │ ├── 1326a4ac09a56678f346f2b1d6370f73b4a6391d │ │ │ ├── 13fbb8bb9ace2fee070b0a57cc23297e3929a2e3 │ │ │ ├── 14f5a764478056056709a1adf56bf1c4f844be46 │ │ │ ├── 14fa52c7a0e0f2dac186cc477b8590bd7ed5d5de │ │ │ ├── 150d861d63105edb32e1c8b9962228c6f0471bfc │ │ │ ├── 156ef15ae5a6999bdd7672a6659f1733e74fd268 │ │ │ ├── 158c9040ac3b2284eddb47727edb1ad8e64aaee2 │ │ │ ├── 15d9f065256954141540c7973b69945361a1779d │ │ │ ├── 161f9d06c9fbad0fdcd36f26d39872902081bd7c │ │ │ ├── 1632c0af768ebd94edd01ed87e4e6a22d38b9443 │ │ │ ├── 16e7bd9816834e5b609f45262f76eab1b6ec40b9 │ │ │ ├── 17a72a1d905026efdf6c6323d9f006002ab85980 │ │ │ ├── 17a9a0cda279f524a28f5f30e851116e072418d4 │ │ │ ├── 180e4be371f1056fc2144de0c0bf19805e4d608d │ │ │ ├── 189dcbb43b973255710dde7497763ea8a046ac16 │ │ │ ├── 1910a16a432c3dcc21d2646244678b93756aa1a0 │ │ │ ├── 19167267fdc1593d4be0c4cb3b601932c3f3adfc │ │ │ ├── 1acda981777221e8e6eaea2c496c7f876519d564 │ │ │ ├── 1aff404a6311174ae5eccd28a1aa2ad2d738e866 │ │ │ ├── 1b0af5886bcea028cd013ace38242c2a09614ec0 │ │ │ ├── 1b211d33180b5e5cda190ecd00168bb4948f4ea8 │ │ │ ├── 1b7bd0d9995cd6c889dfa823565c6d490ec5be8e │ │ │ ├── 1c66f09df9136cbf6cd9c0e9c85f4fe1e12700d1 │ │ │ ├── 1cb6ea9ed7ae98362e6fa46cbd0bb0e977fb24db │ │ │ ├── 1cd4b5622701839c902ccc031af388cf995f863b │ │ │ ├── 1d172fbc7ae373610545d1436e0a17dcccc5e055 │ │ │ ├── 1e809f15bc387deff02d07edd0b9b3011a172d3d │ │ │ ├── 1ebe7bc218890b4a5419154c4f8042d35c760ccb │ │ │ ├── 1ec71769042da0bfbad6967d3a9af3d5637966bd │ │ │ ├── 1f09f12d930daae8e5fd34e11e7b2303e1705b2e │ │ │ ├── 1f3237efa553f6f82174223a1c5df78475ef25a0 │ │ │ ├── 1f5cb95a3f9af8efe74c41ba64bcdf3e12da81f2 │ │ │ ├── 1f6810f6f160b2e7d5f27822be00fccda2c0d9ce │ │ │ ├── 1fadee5ce81d5a4ede3b8d2d508eada6aa3e5fdc │ │ │ ├── 20149c9075b835532b9903e2d838432de7c4c82b │ │ │ ├── 207f681dda06e17d6fb16b89274720956e5315b8 │ │ │ ├── 208964be3ec654f18b77217d9d5dc908ef8eff6e │ │ │ ├── 20bec61757d2a6e9ccf5674424f23d8dd4c59821 │ │ │ ├── 213bd18eb8b9792f5ba8d6453e7b0b74f0d2a12c │ │ │ ├── 217bf330784989cf4c3e21bcbe0d1adeed7dec2e │ │ │ ├── 224b4677ce8e3065a8be0e661b5d3be49602680e │ │ │ ├── 224d47f68da20713449bf600b84a3d750d495ed6 │ │ │ ├── 229389eeb6f917a0db71fcadbc97cf3432357ba6 │ │ │ ├── 2397935ec2daeeea90cc7530afb1bfa490d21607 │ │ │ ├── 239d654fb97da7931b992a93b3c51be8912cdc0d │ │ │ ├── 23a46cbb1893e9284d06e72558317d95ecee8e74 │ │ │ ├── 23c3074517a019bd7edd8d8b71fb159980a10ec0 │ │ │ ├── 24c3c3a4c50155712005e5fa934a99a3ec803318 │ │ │ ├── 24f67fc225eeebfcd06f121effe46d26251e785e │ │ │ ├── 26338cbc8a2c58872d0df57b1231785ba53c5128 │ │ │ ├── 276129fc7c40928aa2743aff77c15bfac126c00d │ │ │ ├── 27fff07708997c5e6e07a0f83406683a7d36daa9 │ │ │ ├── 2839e34ab20fb0105de4ebec1a25a807ba7642c9 │ │ │ ├── 28ac5313c590ed2b28e1ed0fc22f2bd1eab0df51 │ │ │ ├── 28e31c10ef08fdecf629dd265116e0f833cd57cd │ │ │ ├── 29b15bb9cb49218df433c4fe27794615c3a4451b │ │ │ ├── 2a221a24ad464e8da7ee51105400f1670b4320a1 │ │ │ ├── 2a86bc754e5703e9d6101f64eb5e06ccbd261952 │ │ │ ├── 2ad4a333d74699bc03523cba5cb3c108a2bc9527 │ │ │ ├── 2bf57e68f7436d449f0486a2c1ae3a281458bdcd │ │ │ ├── 2c0d33ffce3a3e585c845cf7c6aebd5f4917fddc │ │ │ ├── 2d0ff967eccc17e5d165712c758681ef3fe51caf │ │ │ ├── 2d3db2cf9bcfec159f6bccf801d74a1ce610013f │ │ │ ├── 2e29d5f8ccf89cf55a1876623bbbc0a2a0e76032 │ │ │ ├── 2e485e5fe51b9824721a41a67dfc1fd9ab8eb3f1 │ │ │ ├── 2e64ef301925f827368cf80e67a858f9c1906b44 │ │ │ ├── 2eb732cc04c8a4259763abb8c02b633c484aa08b │ │ │ ├── 3000a8aa02ef08fd7105151982b709ca87702b2a │ │ │ ├── 30b30fb9ab1f1031139fc72d478b5cf6fde75f70 │ │ │ ├── 30cb23ea8ce282a984424ba74cf9e692a0ea95c2 │ │ │ ├── 322398e8d8eec282d54ecf8270285109012daafa │ │ │ ├── 3488d3837745952ffee80820a15f98f3f39bacbd │ │ │ ├── 35551262d560ddd52f1b8ba9039b0ac05932e7f3 │ │ │ ├── 3579bff2e768d34cc21071a22ca286c84bdeb0be │ │ │ ├── 35cdca75d22e73fa72051421f170698c4c526838 │ │ │ ├── 360ce4c914f52af71a7f1f165b579b263aef7224 │ │ │ ├── 3612694abd743a7e55e65aa65263520ad36fab1d │ │ │ ├── 3661b2541a41109436af57eed8cd81f2b077dc64 │ │ │ ├── 36a015ecabd510fc11470a8ee201f74c98a752cb │ │ │ ├── 378b0717ae8bca467ed4ee58212ca50ceae5ec7d │ │ │ ├── 384e73c939df5029d4a47e7446f57a279e0f4e6d │ │ │ ├── 3890191defea00490c16ad251b95f9e07b4c74c9 │ │ │ ├── 38a0feff4559737aeed5f72ee302a3a0ed8629bb │ │ │ ├── 38cc6d4e011e8685c88b2f60ae85df9b6edf06d4 │ │ │ ├── 39e5a474f4810a8abd8b966bdf6a407e20501b25 │ │ │ ├── 39fbdcdb86bc1205438c1a8327ec6668ff0bc009 │ │ │ ├── 3a56ebf0102f26bc48241309d90eedb53bc5599b │ │ │ ├── 3bfafbcc633c3b157ca48d686020d806a9238d9a │ │ │ ├── 3c2a06759b1f89f8e8bf0d01162558169ccfcdb0 │ │ │ ├── 3c4845d3eab256e1cd051b0e02c954e9feb5d689 │ │ │ ├── 3dd30028e1743600a71c84026e90af3f7bb3181e │ │ │ ├── 3e1105e64284eed8dc2bccbe18ec3e6de2a082a0 │ │ │ ├── 4181be0e85942f5aaa5f9785e51b4a72055ab5d6 │ │ │ ├── 42805756addc26eff319d88bf58e37887339eb41 │ │ │ ├── 4463475e7855ef5ca2229888df0624efd85493c8 │ │ │ ├── 44dadbb9d63786197f95408954423a2490e54a72 │ │ │ ├── 45e5d985b27fb2210ba470b892756eaf053194e4 │ │ │ ├── 46945e0bf54f1dc350c0d9ec9b670cf687445be0 │ │ │ ├── 4780df7420a24ebae8d9bcacf6c62c27e0c2e811 │ │ │ ├── 47f3c798199bb53546b443fba5dc925d08d37718 │ │ │ ├── 4957a6bc77cc14caae8682e4499cafca63ac12b4 │ │ │ ├── 498988babdc605e5d03ed9448af24a714ff69e72 │ │ │ ├── 4a5a4312137e693b8122b4426d54fc6444cefb4d │ │ │ ├── 4a81750c78f1acc88612777149b7d47b27d1d58d │ │ │ ├── 4bc7279fedba6ac6deb9be39476e43dce7d69525 │ │ │ ├── 4be8eb811a92f7221820a72185b76e32ab95edf4 │ │ │ ├── 4bea5a628cf732e42825e3bcfb0d0f63819c37a8 │ │ │ ├── 4ca579f0ff4d06f56eebb6b94c1303de1ea289b4 │ │ │ ├── 4ca93515d0cb2e30ac9a09bc6917cf2f4041e309 │ │ │ ├── 4dbe572a6b29c977ba4f132bf27330f5bce8bbb4 │ │ │ ├── 4e452a93754eec25673cc41def71f43e12d6ad44 │ │ │ ├── 4edbe8bce1dd60b4447731e0183305889971ce1c │ │ │ ├── 4ef3a46a40596d12420c9acc7827a7956b93eff2 │ │ │ ├── 4f039fecd3612031008b5ac1ec016a9db9a0afe2 │ │ │ ├── 4f1ee2f2a2b9499943d866a11c5eb73dbb5578a6 │ │ │ ├── 4fcb4525eae80e58e4c0dafcad5dbf9bd6ed8800 │ │ │ ├── 514e88c8fedf058601f8c1a3857405a72bafd346 │ │ │ ├── 51628c735c1dfa273dd79264ffdff1de023ded3b │ │ │ ├── 5181e1d782cae994672eb604f0b88720d4171a6b │ │ │ ├── 5593e1771829808b4c1c17a151f5807590c87060 │ │ │ ├── 56152748eafa8a23aca552d2c63aefd328ef9d1c │ │ │ ├── 583bd88280acbe57634a229531e50afea8977c5e │ │ │ ├── 5b989097db45b584bfdfd0ea0eeb6e484b04ffa4 │ │ │ ├── 5b9f71ff1bb5eb987287e97e9223ad2ee384d043 │ │ │ ├── 5bcb10fcf0de7e1f1eea8b2a6c97b9f2efb38cfe │ │ │ ├── 5bdb91298fe95bcd0fd59b077e9b03708352adc0 │ │ │ ├── 5bf5ba2025d12dff0f8cf646bc0a67e512781a78 │ │ │ ├── 5cbe78de4aa4595bfd13dfc8f9d88bc740bda660 │ │ │ ├── 5d53743f0f6877c631ba1b78a2546351ffd945a5 │ │ │ ├── 5ddb48bfb461a6f541ea5263c124fafc84228405 │ │ │ ├── 5fcca6becd914f5a66441e491bb460cc0d399798 │ │ │ ├── 61b0a7eb4b781e6e980620119029784028b0dd02 │ │ │ ├── 638ee93f6f582a08b54e2d904b6e703e975fb779 │ │ │ ├── 63ddefed7ec1d06bb8e65ef0446fd66247ad9e5f │ │ │ ├── 640783010f7d68e0dd208d742c0ae5dcc35bca18 │ │ │ ├── 6409f517d99779cf628cec89e0ea781800ec1673 │ │ │ ├── 6472b46069dcd29079a739cd9f4d043d77b9db2b │ │ │ ├── 64e26698d0ae69059a828a29619dbb220834ce0b │ │ │ ├── 64ec1601cf03b88a5c1032569dc861d156d706ec │ │ │ ├── 65440ab3f4518a75309ca81383e9f4e6a60bbd6c │ │ │ ├── 669710a353b5b72bcf14eb140b57b0a6be809e7b │ │ │ ├── 683ee9ae1a3ddbdbee2336af93e7f408f5cc42b5 │ │ │ ├── 688934845f22049cb14668832efa33d45013b6b9 │ │ │ ├── 68cdb6aea2466f9256dc371bdcd482b139d28d1c │ │ │ ├── 68f451a137e92f09cfd5eb88a04dbd0b8798bb92 │ │ │ ├── 69bb7cec8b07da9cb2bd3d0aba9f55b0f41f65cc │ │ │ ├── 6b0165838e17a7309a523a909b6f83de6b46f51c │ │ │ ├── 6b49f3fa96695a790faa6eb4f051e5cf2c471209 │ │ │ ├── 6b85b749998063c521ffdb3faf1823f20e85d5e7 │ │ │ ├── 6f924d014dc9bc839ea89f3b98217f262b27c231 │ │ │ ├── 71c2d79e816ce7ffee8d3ac90eedb3159dea7fac │ │ │ ├── 735de4ce401995f6a6e4c47039ac8940bdc1ec8b │ │ │ ├── 778c71afaf6c5a1d3b1c045788d668189adbf96e │ │ │ ├── 77a4c5e5039c7180b2217342711f214970bea74a │ │ │ ├── 781835ce31e3ba1b697c5a40b8447f6b7f7a0144 │ │ │ ├── 78d4981a356b7e21562982b8cd64af8d7f9ff62b │ │ │ ├── 79f08f3117e3d9c49e1e12727b5bc72692a41bad │ │ │ ├── 7c5f32cd666fb03e67aa75a34b889236b40a0eb6 │ │ │ ├── 7c6d18a6fc69526d5c30f4cbd24725fea7423515 │ │ │ ├── 7e6f7bfb1169a82fc31aa11c760b1b1699d7b3a4 │ │ │ ├── 7ee8badf61cd2a56bcd4c5fdafe9395eb733e22a │ │ │ ├── 869028f499a7faea4cf993eb867ea9fd1c3f58e6 │ │ │ ├── 87330cfd941fe956757bce61baa636e6a6b7a0ad │ │ │ ├── 8887779003e7ccbdbc95ba305c34aa75fb255d69 │ │ │ ├── 8bec4981ecd6533ecadf49b978e34e2cdea28e67 │ │ │ ├── 8c08c6739ba900dd00089e5eb7e15c43b5d76e91 │ │ │ ├── 8e6d821b0593f6e74635125f995af5ce4daace70 │ │ │ ├── 94972df24f18202ef83823f34daf4225cac60c13 │ │ │ ├── 95873ef906927ff752a0f48e4fbed36b15003e14 │ │ │ ├── 9881ea067ad96598ca5de7dd0ddfb41a629f776e │ │ │ ├── 9c479051463aa32bdf5fe7ff36258586ea5606d4 │ │ │ ├── 9c5eba1b6a51edabf204e3c2c3e5358604f04c74 │ │ │ ├── 9dbe4a4d6f17605da53c17dcb26c3e5ee30b442b │ │ │ ├── 9f10b850126a00b70b0c7389754b3af7a78efd78 │ │ │ ├── a636879bbc41a43a4748f1aec41c5b8535f95ffb │ │ │ ├── a6c2d09ad472092bfa8e82470f1f493e0e2867a4 │ │ │ ├── a95439dceaaf041214315966c319d4bd99fad3b8 │ │ │ ├── a9fd565873b81f9bc753ef9ffb01279e58b8e995 │ │ │ ├── aa2fc4a7eb7ef7b4db1d14abba054bf82bc99708 │ │ │ ├── abfe928debc5210e137f2ccb2e35afba1fba01f4 │ │ │ ├── ad3f5bbed8a7c1aa1b1d37d9274e51f04ad9d793 │ │ │ ├── af5fe18bf35c2ce1542515dfd0a2187a912efed8 │ │ │ ├── b1951c3ecea275b55530cd1a7dee87896cba3af8 │ │ │ ├── b32871f39219d46cc598467522717179035628a2 │ │ │ ├── b5af72dc2bcef16a9533f28db6764b41e1dec249 │ │ │ ├── b71d21118459e909a1d13dd39c98bf324746204c │ │ │ ├── b961201e6499ce99d9809e6c45e2d3cb6c25368c │ │ │ ├── bbcd55dd0ecf9385e265652b46670796cfb06866 │ │ │ ├── bbe1e29361b5d285b1e3a796d9ea7a6ad330679d │ │ │ ├── bdd340d8b3d28bbd6352fe82bc946b8b662707f6 │ │ │ ├── c1240cef7cf639125a64e8471db1fb7977d2f090 │ │ │ ├── c4ddbbe3d51294ee8e9ad9476b74cea7024b928a │ │ │ ├── c5bd074e83ef42d118f9508c5fd5de1ecc77eb7f │ │ │ ├── c7470bb0cba41502153de8ba1dccfcb415ec37bf │ │ │ ├── c87f1ba99e627af168ddd46ec94d7f7c08a05c07 │ │ │ ├── c90fb91bdb14d13df37727cf0af752161786f9ee │ │ │ ├── cafcf0638aa1531c76b3da2ee1f7a3ce9a22c0cc │ │ │ ├── cbf49a84c11c6c588ae0d298407c77c05775e303 │ │ │ ├── cd4e9ead434c68f2162aace95bb32d69bcd441f3 │ │ │ ├── d1f6b7119295f8823e331ad4f26e8272b6b608e2 │ │ │ ├── d2860d9b6b66d6fc1515ee7e73432da15a2e3d6d │ │ │ ├── d325fb86fbb09b1410f4d2d7d2e9552206bbe4f1 │ │ │ ├── d36ce05a4a02233c33f3c5deb7559e0816b4b10b │ │ │ ├── d3708428e4607ba37bfb13ee8ba2dd236755adbd │ │ │ ├── d71949b44c5e54a1c86336f522a28b281860397f │ │ │ ├── d834d51078ec5d4319c520766d35faae90554491 │ │ │ ├── dbbfd75cb69ba487c576d62261849af7fee58b61 │ │ │ ├── dc17f72fa3dc031b3e2059844d774fc889ad9f93 │ │ │ ├── dc288af0c1589727824c9f9014b81fbc800f97c0 │ │ │ ├── dc9041a71824c1034bb22589a8e340b23c4e90d3 │ │ │ ├── e1e359dab552ab2d41d035879cf66261e9312d64 │ │ │ ├── e1e6fecdf18b3a1375531bbf5a103c2d8f56cdf0 │ │ │ ├── e229b64d990b03a9f44cbd00e71d3d8baa4319ab │ │ │ ├── e5ed00680dfe3dd03c24932186d653ad0dcf676c │ │ │ ├── e7b835e207feecbece96967fc8a45c3c8484c99e │ │ │ ├── ea35f5785fb9f197b4f308eb0e33bf2e51ed3853 │ │ │ ├── f0da90d63a403203967ac12b90eb9d63f0f4ceeb │ │ │ ├── f0ecab55d82b763c44cf359fcc00e191a96d70e7 │ │ │ ├── f29494e8b1819543bd357029a0de41a3eda2d451 │ │ │ ├── f3c4c54e20e1b109c2211ea1c75e448fca92d94b │ │ │ ├── f41088954ca36155c47d16133b8c7c472a50a211 │ │ │ ├── f6dde08a505b30f15b89ff107fffe86418e107c3 │ │ │ ├── f79f35266d31ad7f640363bdd2965e714625ac2d │ │ │ └── f8bf4cc7cb83512b86997321687281890693f5dd │ │ └── uuid-uuid/ │ │ ├── 00204831b0ca9ea0e111df6cef4b9d5d7baacda4 │ │ ├── 03e9ded76e11c2ab56b2b64783f9d5eab0f41e45 │ │ ├── 04ba82c9afdc3ec29f62660df56ce9a07f0c7c4e │ │ ├── 07f74222f10552358f3d355b18b317d94d47ffa8 │ │ ├── 0857d325561ee6748b28d758a8cd56537f2e591f │ │ ├── 0bebd7d18520dee5aa81b3759fbc531c2285e989 │ │ ├── 0e4bbd6dbf3ddae0d2615789ee20b4aa7e7bd098 │ │ ├── 125f55cce23c378eec2f06aa3b0e56337faaa870 │ │ ├── 13aadea7f739db7b7b613ded4f2d9f80c3a1cb2f │ │ ├── 1593b53d6247286af0708369568b11009e81a643 │ │ ├── 1add81bd28c7e785fcaec8a0e5a525d8443d1844 │ │ ├── 1fae8d67d206d97677ef96ad645fcded48950dfc │ │ ├── 264cc1c0f5af787d5803675e1ad1771250fd21a9 │ │ ├── 278f798603ac73124fe61d7a3c04475122a1886d │ │ ├── 29d118681004d0a5837ce19e1c83f8d32723607e │ │ ├── 2d2814db73500b8103051a18a69b28c921ef8f86 │ │ ├── 325fcbfdacebcf2259e782cc363e73c857482faf │ │ ├── 39cd2aeafea74e6f19c4aec657fb7ffa73262927 │ │ ├── 39deb22a28b76c591e8f277ead182d3c56409f12 │ │ ├── 3c651559e8fb3923967b977dcbeb3b742591e267 │ │ ├── 3cdf2936da2fc556bfa533ab1eb59ce710ac80e5 │ │ ├── 3f342aaf5a895c87a09dcff4608482ab3adf6d42 │ │ ├── 4820348eb8d1d6257f21e82769043a3f02fe4d05 │ │ ├── 4967f66d5fceaa599c1877d83aa24e5ed568029f │ │ ├── 4b02658a763a2875b5f1e7000154d9a6726393e8 │ │ ├── 4ee2d3a264bee8637d869d828e2ac90008ee03bb │ │ ├── 5ab314d48a22d74c0c6b025954cd3e54e18475d4 │ │ ├── 5d1be7e9dda1ee8896be5b7e34a85ee16452a7b4 │ │ ├── 6f7133c510007b76d2bcc3d85d1980e85fad67a4 │ │ ├── 7638706b19c5bbcf956375eb0c8b489187b2cca1 │ │ ├── 926eb1d08171eaf3c6f49b059133ebce13780c9b │ │ ├── a545d616805bfdc85a040218fc435bda325d8586 │ │ ├── a58315ace7c70ef7844b17b0985c9ff05fd85b8e │ │ ├── abc9df7cd29252cbe7b79407192c89d3c9a00d5b │ │ ├── acc9a1ac056149e11830bfbf59e3ee16226fe6c4 │ │ ├── c1cbd4f3b9b55ef1fbcba2ca8615aa139aea57aa │ │ ├── cd829577a89686ff93bd3d895a7ee384319668f1 │ │ ├── d0b5cf5009aedc6c54b6e0a8221cac55c943fb19 │ │ ├── de6c21a05e80a0be02b7cde74fc3629ef1d11499 │ │ ├── ec8eabcbce8137c58679fd59db76f92f0554867a │ │ └── empty │ ├── isolation/ │ │ ├── CMakeLists.txt │ │ ├── expected/ │ │ │ ├── attach_chunk_isolation.out │ │ │ ├── bgw_job_duplicate_race.out │ │ │ ├── bgw_job_stat_history_retention_isolation.out │ │ │ ├── cagg_concurrent_invalidation.out │ │ │ ├── cagg_concurrent_move.out │ │ │ ├── cagg_concurrent_refresh.out │ │ │ ├── cagg_insert.out │ │ │ ├── cagg_multi_iso.out │ │ │ ├── cagg_watermark_concurrent_update.out │ │ │ ├── cagg_watermark_concurrent_update_1.out │ │ │ ├── compression_chunk_race.out │ │ │ ├── compression_conflicts_iso.out │ │ │ ├── compression_ddl_iso.out │ │ │ ├── compression_dml_iso.out │ │ │ ├── compression_freeze.out │ │ │ ├── compression_merge_race.out │ │ │ ├── compression_recompress.out │ │ │ ├── concurrent_decompress_update.out │ │ │ ├── deadlock_drop_chunks_compress.out │ │ │ ├── deadlock_drop_index_vacuum.out │ │ │ ├── deadlock_recompress_chunk.out │ │ │ ├── decompression_chunk_and_parallel_query.out │ │ │ ├── decompression_chunk_and_parallel_query_wo_idx.out │ │ │ ├── delete_job_deadlock.out │ │ │ ├── detach_chunk_isolation.out │ │ │ ├── direct_compress_copy.out │ │ │ ├── fk_hypertable_lock.out │ │ │ ├── freeze_chunk.out │ │ │ ├── hypertable_row_lock.out │ │ │ ├── merge_chunks_concurrent.out │ │ │ ├── osm_range_updates_iso.out │ │ │ ├── parallel_compression.out │ │ │ ├── reorder_deadlock.out │ │ │ ├── reorder_vs_insert.out │ │ │ ├── reorder_vs_insert_other_chunk.out │ │ │ ├── reorder_vs_select.out │ │ │ └── split_chunk_concurrent.out │ │ └── specs/ │ │ ├── .gitignore │ │ ├── CMakeLists.txt │ │ ├── attach_chunk_isolation.spec │ │ ├── bgw_job_duplicate_race.spec │ │ ├── bgw_job_stat_history_retention_isolation.spec │ │ ├── cagg_concurrent_invalidation.spec │ │ ├── cagg_concurrent_move.spec │ │ ├── cagg_concurrent_refresh.spec │ │ ├── cagg_insert.spec │ │ ├── cagg_multi_iso.spec │ │ ├── cagg_watermark_concurrent_update.spec │ │ ├── compression_chunk_race.spec │ │ ├── compression_conflicts_iso.spec │ │ ├── compression_ddl_iso.spec │ │ ├── compression_dml_iso.spec │ │ ├── compression_freeze.spec │ │ ├── compression_merge_race.spec │ │ ├── compression_recompress.spec │ │ ├── concurrent_decompress_update.spec │ │ ├── deadlock_drop_chunks_compress.spec │ │ ├── deadlock_drop_index_vacuum.spec │ │ ├── deadlock_recompress_chunk.spec │ │ ├── decompression_chunk_and_parallel_query.in │ │ ├── decompression_chunk_and_parallel_query_wo_idx.spec │ │ ├── delete_job_deadlock.spec │ │ ├── detach_chunk_isolation.spec │ │ ├── direct_compress_copy.spec │ │ ├── fk_hypertable_lock.spec │ │ ├── freeze_chunk.spec │ │ ├── hypertable_row_lock.spec │ │ ├── merge_chunks_concurrent.spec │ │ ├── osm_range_updates_iso.spec │ │ ├── parallel_compression.spec │ │ ├── reorder_deadlock.spec.in │ │ ├── reorder_vs_insert.spec.in │ │ ├── reorder_vs_insert_other_chunk.spec.in │ │ ├── reorder_vs_select.spec.in │ │ └── split_chunk_concurrent.spec │ ├── postgresql.conf.in │ ├── shared/ │ │ ├── CMakeLists.txt │ │ ├── expected/ │ │ │ ├── build_info.out │ │ │ ├── cagg_compression.out │ │ │ ├── chunk_append_merge_append.out │ │ │ ├── chunkwise_agg_gather_sort.out │ │ │ ├── classify_relation.out │ │ │ ├── compat.out │ │ │ ├── compress_bloom_sparse_compat.out │ │ │ ├── compress_unique_index.out │ │ │ ├── compression_dml.out │ │ │ ├── compression_nulls_not_distinct.out │ │ │ ├── constify_now-15.out │ │ │ ├── constify_now-16.out │ │ │ ├── constify_now-17.out │ │ │ ├── constify_now-18.out │ │ │ ├── constify_timestamptz_op_interval-15.out │ │ │ ├── constify_timestamptz_op_interval-16.out │ │ │ ├── constify_timestamptz_op_interval-17.out │ │ │ ├── constify_timestamptz_op_interval-18.out │ │ │ ├── constraint_aware_append.out │ │ │ ├── constraint_exclusion_prepared-15.out │ │ │ ├── constraint_exclusion_prepared-16.out │ │ │ ├── constraint_exclusion_prepared-17.out │ │ │ ├── constraint_exclusion_prepared-18.out │ │ │ ├── decompress_join.out │ │ │ ├── decompress_placeholdervar.out │ │ │ ├── decompress_tracking.out │ │ │ ├── extension.out │ │ │ ├── gapfill-15.out │ │ │ ├── gapfill-16.out │ │ │ ├── gapfill-17.out │ │ │ ├── gapfill-18.out │ │ │ ├── gapfill_bug.out │ │ │ ├── generated_columns.out │ │ │ ├── lateral_subquery-15.out │ │ │ ├── lateral_subquery-16.out │ │ │ ├── lateral_subquery-17.out │ │ │ ├── lateral_subquery-18.out │ │ │ ├── memoize.out │ │ │ ├── merge_dml.out │ │ │ ├── ordered_append_join-15.out │ │ │ ├── ordered_append_join-16.out │ │ │ ├── ordered_append_join-17.out │ │ │ ├── ordered_append_join-18.out │ │ │ ├── parameterized_chunkappend.out │ │ │ ├── security_barrier.out │ │ │ ├── space_constraint-15.out │ │ │ ├── space_constraint-16.out │ │ │ ├── space_constraint-17.out │ │ │ ├── space_constraint-18.out │ │ │ ├── subtract_integer_from_now.out │ │ │ ├── timestamp_limits.out │ │ │ ├── transparent_decompress_chunk-15.out │ │ │ ├── transparent_decompress_chunk-16.out │ │ │ ├── transparent_decompress_chunk-17.out │ │ │ ├── transparent_decompress_chunk-18.out │ │ │ └── with_clause_parser.out │ │ └── sql/ │ │ ├── .gitignore │ │ ├── CMakeLists.txt │ │ ├── build_info.sql │ │ ├── cagg_compression.sql │ │ ├── chunk_append_merge_append.sql │ │ ├── chunkwise_agg_gather_sort.sql │ │ ├── classify_relation.sql │ │ ├── compat.sql │ │ ├── compress_bloom_sparse_compat.sql │ │ ├── compress_unique_index.sql │ │ ├── compression_dml.sql │ │ ├── compression_nulls_not_distinct.sql │ │ ├── constify_now.sql.in │ │ ├── constify_timestamptz_op_interval.sql.in │ │ ├── constraint_aware_append.sql │ │ ├── constraint_exclusion_prepared.sql.in │ │ ├── decompress_join.sql │ │ ├── decompress_placeholdervar.sql │ │ ├── decompress_tracking.sql │ │ ├── extension.sql │ │ ├── gapfill.sql.in │ │ ├── gapfill_bug.sql │ │ ├── generated_columns.sql │ │ ├── include/ │ │ │ ├── cagg_compression_query.sql │ │ │ ├── cagg_compression_setup.sql │ │ │ ├── constraint_exclusion_prepared_query.sql │ │ │ ├── gapfill_metrics_query.sql │ │ │ ├── memoize_query.sql │ │ │ ├── ordered_append_join.sql │ │ │ └── shared_setup.sql │ │ ├── lateral_subquery.sql.in │ │ ├── memoize.sql │ │ ├── merge_dml.sql │ │ ├── ordered_append_join.sql.in │ │ ├── parameterized_chunkappend.sql │ │ ├── security_barrier.sql │ │ ├── space_constraint.sql.in │ │ ├── subtract_integer_from_now.sql │ │ ├── timestamp_limits.sql │ │ ├── transparent_decompress_chunk.sql.in │ │ └── with_clause_parser.sql │ ├── sql/ │ │ ├── .gitignore │ │ ├── CMakeLists.txt │ │ ├── agg_partials_pushdown.sql │ │ ├── attach_chunk.sql │ │ ├── bgw_custom.sql │ │ ├── bgw_db_scheduler.sql │ │ ├── bgw_db_scheduler_fixed.sql │ │ ├── bgw_job_ddl.sql │ │ ├── bgw_job_stat_history.sql │ │ ├── bgw_job_stat_history_errors.sql │ │ ├── bgw_job_stat_history_errors_permissions.sql │ │ ├── bgw_policy.sql │ │ ├── bgw_reorder_drop_chunks.sql │ │ ├── bgw_scheduler_control.sql │ │ ├── bgw_scheduler_restart.sql │ │ ├── bgw_security.sql │ │ ├── bgw_telemetry.sql │ │ ├── cagg.sql.in │ │ ├── cagg_bgw.sql.in │ │ ├── cagg_bgw_drop_chunks.sql │ │ ├── cagg_ddl.sql.in │ │ ├── cagg_direct_compress.sql │ │ ├── cagg_drop_chunks.sql │ │ ├── cagg_dump.sql │ │ ├── cagg_errors.sql │ │ ├── cagg_invalidation.sql │ │ ├── cagg_invalidation_variable_bucket.sql │ │ ├── cagg_joins.sql │ │ ├── cagg_multi.sql │ │ ├── cagg_on_cagg.sql │ │ ├── cagg_on_cagg_joins.sql │ │ ├── cagg_permissions.sql.in │ │ ├── cagg_planning.sql │ │ ├── cagg_policy.sql │ │ ├── cagg_policy_concurrent.sql │ │ ├── cagg_policy_incremental.sql │ │ ├── cagg_policy_move.sql │ │ ├── cagg_policy_run.sql │ │ ├── cagg_query.sql.in │ │ ├── cagg_query_using_merge.sql.in │ │ ├── cagg_refresh_using_merge.sql │ │ ├── cagg_refresh_using_trigger.sql │ │ ├── cagg_tableam.sql │ │ ├── cagg_union_view.sql.in │ │ ├── cagg_usage.sql.in │ │ ├── cagg_utils.sql │ │ ├── cagg_uuid.sql │ │ ├── cagg_watermark.sql │ │ ├── chunk_api.sql │ │ ├── chunk_column_stats.sql │ │ ├── chunk_merge.sql │ │ ├── chunk_publication_compression.sql │ │ ├── chunk_utils_compression.sql │ │ ├── chunk_utils_internal.sql │ │ ├── columnar_index_scan.sql.in │ │ ├── columnar_scan_cost.sql │ │ ├── columnstore_aliases.sql │ │ ├── compress_auto_sparse_index.sql │ │ ├── compress_batch_size.sql │ │ ├── compress_bgw_reorder_drop_chunks.sql │ │ ├── compress_bitmap_scan.sql │ │ ├── compress_bloom_dml.sql │ │ ├── compress_bloom_hash.sql │ │ ├── compress_bloom_legacy_v1.sql │ │ ├── compress_bloom_sparse.sql.in │ │ ├── compress_bloom_sparse_debug.sql │ │ ├── compress_compbloom_basics.sql │ │ ├── compress_compbloom_config.sql │ │ ├── compress_compbloom_index_add.sql │ │ ├── compress_compbloom_index_drop.sql │ │ ├── compress_compbloom_manual_config.sql │ │ ├── compress_compbloom_upsert.sql │ │ ├── compress_composite_bloom_debug.sql │ │ ├── compress_default.sql │ │ ├── compress_dml_copy.sql │ │ ├── compress_explain.sql │ │ ├── compress_float8_corrupt.sql │ │ ├── compress_qualpushdown_complex.sql │ │ ├── compress_qualpushdown_saop.sql │ │ ├── compress_sort_transform.sql │ │ ├── compress_sparse_config.sql │ │ ├── compress_unordered_sort.sql │ │ ├── compressed_collation.sql │ │ ├── compressed_detoaster.sql │ │ ├── compression.sql │ │ ├── compression_algos.sql │ │ ├── compression_allocation.sql │ │ ├── compression_bgw.sql │ │ ├── compression_bool_vectorized.sql │ │ ├── compression_bools.sql │ │ ├── compression_conflicts.sql │ │ ├── compression_constraints.sql │ │ ├── compression_create_compressed_table.sql │ │ ├── compression_ddl.sql │ │ ├── compression_defaults.sql │ │ ├── compression_delete_bitmapscan.sql.in │ │ ├── compression_errors.sql │ │ ├── compression_fks.sql │ │ ├── compression_hypertable.sql │ │ ├── compression_indexcreate.sql │ │ ├── compression_indexscan.sql │ │ ├── compression_insert.sql │ │ ├── compression_merge.sql │ │ ├── compression_null_dump_restore.sql │ │ ├── compression_nulls_and_defaults.sql │ │ ├── compression_permissions.sql.in │ │ ├── compression_policy.sql │ │ ├── compression_qualpushdown.sql │ │ ├── compression_segment_meta.sql │ │ ├── compression_sequence_num_removal.sql │ │ ├── compression_settings.sql │ │ ├── compression_sorted_merge.sql │ │ ├── compression_sorted_merge_columns.sql │ │ ├── compression_sorted_merge_distinct.sql │ │ ├── compression_sorted_merge_filter.sql │ │ ├── compression_sorted_merge_unordered.sql │ │ ├── compression_trigger.sql │ │ ├── compression_update_delete.sql.in │ │ ├── compression_uuid.sql │ │ ├── create_table_with.sql │ │ ├── data/ │ │ │ ├── copy_data.csv │ │ │ └── magic.csv │ │ ├── decompress_index.sql │ │ ├── decompress_memory.sql │ │ ├── decompress_vector_qual.sql │ │ ├── detach_chunk.sql │ │ ├── direct_compress_copy.sql │ │ ├── direct_compress_insert.sql │ │ ├── feature_flags.sql │ │ ├── fixed_schedules.sql │ │ ├── foreign_keys_test.sql.in │ │ ├── hypertable_generalization.sql │ │ ├── include/ │ │ │ ├── aggregate_queries.sql │ │ │ ├── aggregate_table_create.sql │ │ │ ├── aggregate_table_populate.sql │ │ │ ├── cagg_on_cagg_common.sql │ │ │ ├── cagg_on_cagg_setup.sql │ │ │ ├── cagg_on_cagg_validations.sql │ │ │ ├── cagg_planning_query.sql │ │ │ ├── cagg_query_common.sql │ │ │ ├── cagg_refresh_common.sql │ │ │ ├── chunk_utils_internal_orderedappend.sql │ │ │ ├── cluster_test_setup.sql │ │ │ ├── columnar_index_scan_query.sql │ │ │ ├── compression_alter.sql │ │ │ ├── compression_test.sql │ │ │ ├── compression_test_hypertable.sql │ │ │ ├── compression_test_hypertable_segment_meta.sql │ │ │ ├── compression_test_merge.sql │ │ │ ├── compression_test_segment_meta.sql │ │ │ ├── compression_utils.sql │ │ │ ├── cont_agg_equal.sql │ │ │ ├── cont_agg_equal_deprecated.sql │ │ │ ├── cont_agg_test_equal.sql │ │ │ ├── debugsupport.sql │ │ │ ├── filter_exec.sql │ │ │ ├── foreign_keys.sql │ │ │ ├── jit_cleanup.sql │ │ │ ├── jit_load.sql │ │ │ ├── jit_query.sql │ │ │ ├── modify_exclusion_load.sql │ │ │ ├── ordered_append.sql │ │ │ ├── ordered_append_load.sql │ │ │ ├── rand_generator.sql │ │ │ ├── recompress_basic.sql │ │ │ ├── recompression_integrity_check.sql │ │ │ ├── scheduler_fixed_common.sql │ │ │ ├── skip_scan_comp_query.sql │ │ │ ├── skip_scan_dagg_comp_query.sql │ │ │ ├── skip_scan_dagg_load_comp_query.sql │ │ │ ├── skip_scan_dagg_query.sql │ │ │ ├── skip_scan_dagg_query_ht.sql │ │ │ ├── skip_scan_load.sql │ │ │ ├── skip_scan_load_comp_query.sql │ │ │ ├── skip_scan_multi_load.sql │ │ │ ├── skip_scan_multi_query.sql │ │ │ ├── skip_scan_notnull.sql │ │ │ ├── skip_scan_notnull_setup.sql │ │ │ ├── skip_scan_query.sql │ │ │ ├── skip_scan_query_ht.sql │ │ │ ├── transparent_decompression_constraintaware.sql │ │ │ ├── transparent_decompression_ordered.sql │ │ │ ├── transparent_decompression_ordered_index.sql │ │ │ ├── transparent_decompression_ordered_indexplan.sql │ │ │ ├── transparent_decompression_query.sql │ │ │ ├── transparent_decompression_systemcolumns.sql │ │ │ ├── transparent_decompression_undiffed.sql │ │ │ └── vector_agg_planning_query.sql │ │ ├── information_view_chunk_count.sql │ │ ├── insert_memory_usage.sql │ │ ├── jit.sql │ │ ├── license_tsl.sql │ │ ├── merge_append_partially_compressed.sql │ │ ├── merge_chunks.sql │ │ ├── merge_compress.sql │ │ ├── modify_exclusion.sql.in │ │ ├── move.sql │ │ ├── ordered_append.sql.in │ │ ├── plan_skip_scan.sql.in │ │ ├── plan_skip_scan_dagg.sql.in │ │ ├── plan_skip_scan_notnull.sql │ │ ├── policy_generalization.sql │ │ ├── privilege_maintain.sql │ │ ├── read_only.sql │ │ ├── rebuild_columnstore_tests.sql │ │ ├── recompress_chunk_segmentwise.sql │ │ ├── recompression_integrity_tests.sql │ │ ├── recompression_integrity_unordered.sql │ │ ├── reorder.sql │ │ ├── scheduler_fixed.sql │ │ ├── size_utils_tsl.sql │ │ ├── skip_scan.sql │ │ ├── skip_scan_dagg.sql │ │ ├── split_chunk.sql │ │ ├── telemetry_stats.sql │ │ ├── transparent_decompression.sql.in │ │ ├── transparent_decompression_join_index.sql │ │ ├── transparent_decompression_ordered_index.sql.in │ │ ├── transparent_decompression_queries.sql.in │ │ ├── tsl_tables.sql │ │ ├── uncompressed_size.sql │ │ ├── unlogged.sql │ │ ├── uuid_policies.sql │ │ ├── vacuum.sql │ │ ├── vector_agg_byte.sql │ │ ├── vector_agg_default.sql │ │ ├── vector_agg_expr.sql │ │ ├── vector_agg_filter.sql │ │ ├── vector_agg_functions.sql │ │ ├── vector_agg_groupagg.sql │ │ ├── vector_agg_grouping.sql │ │ ├── vector_agg_memory.sql │ │ ├── vector_agg_modify_hypertable.sql │ │ ├── vector_agg_param.sql │ │ ├── vector_agg_planning.sql.in │ │ ├── vector_agg_segmentby.sql │ │ ├── vector_agg_text.sql │ │ ├── vector_agg_uuid.sql │ │ ├── vector_qual_default.sql │ │ └── vectorized_aggregation.sql │ ├── src/ │ │ ├── CMakeLists.txt │ │ ├── compression_sql_test.c │ │ ├── compression_sql_test.h │ │ ├── compression_unit_test.c │ │ ├── decompress_arithmetic_test_impl.c │ │ ├── decompress_text_test_impl.c │ │ ├── test_chunk_stats.c │ │ ├── test_continuous_agg.c │ │ └── test_merge_chunk.c │ └── t/ │ ├── 001_job_crash_log.pl │ ├── 002_logrepl_decomp_marker.pl │ ├── 003_mvcc_cagg.pl │ ├── 004_truncate_or_delete_spin_lock.pl │ ├── 005_recompression_spin_lock_test.pl │ └── CMakeLists.txt └── version.config ================================================ FILE CONTENTS ================================================ ================================================ FILE: .clang-format ================================================ --- Language: Cpp # BasedOnStyle: LLVM AccessModifierOffset: -2 AlignAfterOpenBracket: Align AlignConsecutiveAssignments: false AlignConsecutiveDeclarations: false AlignEscapedNewlines: Right AlignOperands: true AlignTrailingComments: true AllowAllParametersOfDeclarationOnNextLine: true AllowShortBlocksOnASingleLine: false AllowShortCaseLabelsOnASingleLine: false AllowShortFunctionsOnASingleLine: All AllowShortIfStatementsOnASingleLine: false AllowShortLoopsOnASingleLine: false # AlwaysBreakAfterDefinitionReturnType: None # option is deprecated AlwaysBreakAfterReturnType: AllDefinitions AlwaysBreakBeforeMultilineStrings: false AlwaysBreakTemplateDeclarations: MultiLine BinPackArguments: false BinPackParameters: true BraceWrapping: AfterCaseLabel: true AfterClass: true AfterControlStatement: true AfterEnum: true AfterFunction: true AfterNamespace: true AfterObjCDeclaration: true AfterStruct: true AfterUnion: true AfterExternBlock: true BeforeCatch: true BeforeElse: true IndentBraces: false SplitEmptyFunction: true SplitEmptyRecord: true SplitEmptyNamespace: true BreakBeforeBinaryOperators: None BreakBeforeBraces: Custom BreakBeforeInheritanceComma: false # N/A C++ BreakInheritanceList: BeforeColon BreakBeforeTernaryOperators: false BreakConstructorInitializersBeforeComma: false BreakConstructorInitializers: BeforeColon BreakAfterJavaFieldAnnotations: false # N/A Java BreakStringLiterals: true ColumnLimit: 100 CommentPragmas: '^ TS Pragma:' #For future proofing CompactNamespaces: false # N/A c++ ConstructorInitializerAllOnOneLineOrOnePerLine: false # N/A C++ ConstructorInitializerIndentWidth: 40 # N/A C++ ContinuationIndentWidth: 4 Cpp11BracedListStyle: false # see catalog.c array struct assigns for an example DerivePointerAlignment: false # always use Right DisableFormat: false # haha # ExperimentalAutoDetectBinPacking: false #the docs say not to have this in config file FixNamespaceComments: true # N/A C++ ForEachMacros: - foreach - forboth - for_each_cell - for_both_cell - forthree IncludeBlocks: Preserve # separate include blocks will not be merged IncludeCategories: # we want to ensure c.h and postgres.h appear first - Regex: '^<(string|unistd)\.h>' Priority: -2 - Regex: '^' Priority: -1 - Regex: '^(<|")compat/compat\.h' Priority: -2 - Regex: '^"compat/compat-msvc-enter\.h' Priority: -1 - Regex: '^"compat/compat-msvc-exit\.h' Priority: 2000000000 - Regex: '.*' Priority: 1 IncludeIsMainRegex: '' # filename_ will be seen as the primary include IndentCaseLabels: true IndentPPDirectives: None # do not indent preprocessor directives after the '#' IndentWidth: 4 IndentWrappedFunctionNames: false # we do not indent the function name in the declaration JavaScriptQuotes: Double # N/A js JavaScriptWrapImports: true # N/A js KeepEmptyLinesAtTheStartOfBlocks: false MacroBlockBegin: '' # regex of macros that behave like '{' MacroBlockEnd: '' # regex of macros that behave like '}' MaxEmptyLinesToKeep: 1 NamespaceIndentation: None # N/A c++ ObjCBinPackProtocolList: Auto # N/A objC ObjCBlockIndentWidth: 2 # N/A objC ObjCSpaceAfterProperty: false # N/A objC ObjCSpaceBeforeProtocolList: true # N/A objC PenaltyBreakAssignment: 2 PenaltyBreakBeforeFirstCallParameter: 10000 PenaltyBreakComment: 300 PenaltyBreakFirstLessLess: 120 PenaltyBreakString: 1000 PenaltyBreakTemplateDeclaration: 10 PenaltyExcessCharacter: 1000000 PenaltyReturnTypeOnItsOwnLine: 60 PointerAlignment: Right # as in char *foo; ReflowComments: true # break up long comments into multiple lines SortIncludes: CaseInsensitive # sort includes SortUsingDeclarations: false # N/A c++ SpaceAfterCStyleCast: true SpaceAfterTemplateKeyword: false # N/A c++ SpaceBeforeAssignmentOperators: true SpaceBeforeCpp11BracedList: false SpaceBeforeCtorInitializerColon: true # N/A c++ SpaceBeforeInheritanceColon: false # N/A c++ SpaceBeforeParens: ControlStatements SpaceBeforeRangeBasedForLoopColon: true # N/A C++ SpaceInEmptyParentheses: false SpacesBeforeTrailingComments: 1 SpacesInAngles: false # N/A c++ SpacesInContainerLiterals: true # N/A c++ SpacesInCStyleCastParentheses: false SpacesInParentheses: false SpacesInSquareBrackets: false Standard: Cpp11 TabWidth: 4 UseTab: Always ... ================================================ FILE: .codecov.yml ================================================ # If it says that commit YAML is invalid again, # validate it with: # curl --data-binary @.codecov.yml https://codecov.io/validate # More docs: https://docs.codecov.com/docs/codecov-yaml#repository-yaml ignore: - "test/src" coverage: status: project: default: flags: - pr ================================================ FILE: .dir-locals.el ================================================ ;; see also src/tools/editors/emacs.samples in the PostgreSQL source ;; tree for more complete settings ((c-mode . ((c-basic-offset . 4) (c-file-style . "bsd") (fill-column . 78) (indent-tabs-mode . t) (tab-width . 4))) (diff-mode . ((tab-width . 4))) (dsssl-mode . ((indent-tabs-mode . nil))) (nxml-mode . ((indent-tabs-mode . nil))) (perl-mode . ((perl-indent-level . 4) (perl-continued-statement-offset . 4) (perl-continued-brace-offset . 4) (perl-brace-offset . 0) (perl-brace-imaginary-offset . 0) (perl-label-offset . -2) (indent-tabs-mode . t) (tab-width . 4))) (sgml-mode . ((fill-column . 78) (indent-tabs-mode . nil))) (sql-mode . ((sql-product . postgres))) ) ================================================ FILE: .editorconfig ================================================ root = true [*] charset = utf-8 trim_trailing_whitespace = true insert_final_newline = true end_of_line = lf [*.{c,h}] indent_style = tab indent_size = 4 [*.out] trim_trailing_whitespace = false insert_final_newline = false ================================================ FILE: .git-blame-ignore-revs ================================================ # This file contains a list of commits that are not likely what you # are looking for in a blame, such as mass reformatting or renaming. # You can set this file as a default ignore file for blame by running # the following command. # # $ git config blame.ignoreRevsFile .git-blame-ignore-revs # formatting with pgindent 32c45b75b27e3f690236a9d1c8d13a025316ad2f # Fix formatting to comply with pgindent 2ec065b53823e50dd1ac1d7cf925ae5f90e293ea # Fix formatting issues 3e42150e3b0402ae865d9a827d8d178568a0d27e # Run clang-format on code 34edba16a9385a4b0353e8e07a19dba98d7e3fb9 # Run pg_format on SQL files 3f5872ec61650519e2c5e6fe1dfb60d07696cac7 # Fix various misspellings 68aec9593c0f37dddbaa4f2e2b34a9ba3f5b11d9 # Switch to clang-format-14 7758f5959c8ed64499ab0e6bb66c30464b11dd81 # Remove trailing whitespaces from test code a4356f342f1732857a1d8057f71219b50f1919b2 # Cosmetic changes to create.c 230f368f4e5d146ce5f919cc5999b236997befaf # Adding python and yaml linters 9133319081aef92705f1405087822fc281d215d4 44cd71a602ba96029001de6e97a1b44488730080 f75a51def79796ff7fef58ec950c859fe4e71618 21a3f8206c0de98932867096637c7d1e3d04d925 # Clang-tidy f862212c8ca19b1af56c7608a68f22b7dd0c985e 05ba1cf22f0dc9232069b566dd23c3edb2cbaee4 ecf34132c69e1709cd393eab43b5fcbfd7c201db e75274ee7c6eef1dafc9b4f4d9f71e8e88f76813 a3ef0384655d57200e83ad7b13c91a31177b97c1 ================================================ FILE: .github/CODE_OF_CONDUCT.md ================================================ The Timescale Code of Conduct can be found at . ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.yml ================================================ --- name: Bug report description: Is something not working? Help us fix it! title: "[Bug]: " labels: ["bug", "triage"] body: - type: markdown attributes: value: | Thanks for taking the time to fill out this bug report! - type: dropdown id: type attributes: label: What type of bug is this? multiple: true options: - Configuration - Crash - Data corruption - Incorrect result - Locking issue - Performance issue - Unexpected error - Other validations: required: true - type: dropdown id: subsystem attributes: label: What subsystems and features are affected? description: You can pick multiple subsystems and features. multiple: true options: - Adaptive chunking - Background worker - Backup - Build system - Command processing - Compression - Configuration - Continuous aggregate - Data ingestion - Gapfill - Packaging - Partitioning - Platform/OS - Policy - Query executor - Query planner - Replication - Restore - SkipScan - Telemetry - User-Defined Action (UDA) - Other validations: required: true - type: textarea id: what-happened attributes: label: What happened? description: | Tell us what happened and also what you would have expected to happen instead. placeholder: "Describe the bug" validations: required: true - type: input id: timescaledb-version attributes: label: TimescaleDB version affected description: Let us know what version of TimescaleDB you are running. placeholder: "2.5.0" validations: required: true - type: input id: postgresql-version attributes: label: PostgreSQL version used description: Let us know what version of PostgreSQL you are running. placeholder: "17.4" validations: required: true - type: input id: os attributes: label: What operating system did you use? description: | Please provide OS, version, and architecture. For example: Windows 10 x64, Ubuntu 21.04 x64, Mac OS X 10.5 ARM, Rasperry Pi i386, etc. placeholder: "Ubuntu 21.04 x64" validations: required: true - type: dropdown id: installation attributes: label: What installation method did you use? multiple: true options: - Deb/Apt - Docker - Homebrew - RPM - Source - Other - Not applicable validations: required: true - type: dropdown id: platform attributes: label: What platform did you run on? multiple: true options: - Amazon Web Services (AWS) - Google Cloud Platform (GCP) - Managed Service for TimescaleDB (MST/Aiven) - Microsoft Azure Cloud - On prem/Self-hosted - Timescale Cloud - Other - Not applicable validations: required: true - type: textarea id: logs attributes: label: Relevant log output and stack trace description: | Please copy and paste any relevant log output or a stack trace. This will be automatically formatted into code, so no need for backticks. render: bash - type: textarea id: reproduce attributes: label: How can we reproduce the bug? description: | Please try to provide step-by-step instructions how to reproduce the issue. If possible, provide scripts that we can run to trigger the bug. render: bash validations: required: true ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ blank_issues_enabled: true contact_links: - name: Support Request url: https://www.timescale.com/support about: Support request or question relating to TimescaleDB - name: Timescale Community Slack url: https://slack.timescale.com/ about: Get free help from the TimescaleDB community - name: Timescale Community Forum url: https://forum.timescale.com/ about: Get free help from the TimescaleDB community - name: Documentation request url: https://github.com/timescale/docs/issues/new/choose about: Request a change to our documentation ================================================ FILE: .github/ISSUE_TEMPLATE/enhancement.yml ================================================ --- name: Enhancement description: Suggest an enhancement to existing functionality title: "[Enhancement]: <Title>" labels: [ "enhancement" ] body: - type: dropdown id: type attributes: label: What type of enhancement is this? multiple: true options: - API improvement - Configuration - Performance - Refactor - Tech debt reduction - User experience - Other validations: required: true - type: dropdown id: subsystem attributes: label: What subsystems and features will be improved? description: You can pick multiple subsystems and features. multiple: true options: - Adaptive chunking - Background worker - Build system - Command processing - Compression - Configuration - Continuous aggregate - Data ingestion - Gapfill - Packaging - Partitioning - Platform/OS - Policy - Query executor - Query planner - Replication - SkipScan - Telemetry - User-Defined Action (UDA) - Other validations: required: true - type: textarea id: what attributes: label: What does the enhancement do? description: | Give a high-level overview of how you suggest to improve an existing feature or functionality. validations: required: true - type: textarea id: implementation attributes: label: Implementation challenges description: | Share any ideas of how to implement the enhancement. validations: required: false ================================================ FILE: .github/ISSUE_TEMPLATE/feature.yml ================================================ --- name: Feature request description: Suggest a new feature for TimescaleDB title: "[Feature]: <Feature name>" labels: [ "feature-request" ] body: - type: markdown id: info attributes: value: | Only use this template to suggest a new feature that doesn't already exist in TimescaleDB. For enhancements to existing features, use the "Enhancement" issue template. For bugs, use the bug report template. - type: textarea id: what attributes: label: What problem does the new feature solve? description: | Describe the problem and why it is important to solve. Did you consider alternative solutions, perhaps outside the database? Why is it better to add the feature to TimescaleDB? validations: required: true - type: textarea id: how attributes: label: What does the feature do? description: | Give a high-level overview of what the feature does and how it would work. validations: required: true - type: textarea id: implementation attributes: label: Implementation challenges description: | If you have ideas of how to implement the feature, and any particularly challenging issues to overcome, then provide them here. validations: required: false ================================================ FILE: .github/PULL_REQUEST_TEMPLATE/pull_request_template.md ================================================ - [ ] Add [CHANGELOG](https://github.com/timescale/timescaledb/blob/main/CHANGELOG.md) updates - [ ] Needs [documentation](https://github.com/timescale/docs) updates Fixes #<issue number>. --- Cut everything below the above line. ## Guidelines for Pull requests Checklist for a pull request: - Try to follow [the seven rules of a great Git commit message](https://chris.beams.io/posts/git-commit/). - Ideally, the PR should be merged as a single commit. - Rebase on latest main code. - Reference the issue(s) resolved with Fixes #<issue number>, or Closes #<issue number>. - Make sure the PR has appropriate CHANGELOG updates, including thanks to people that reported issues. - Include appropriate documentation changes. - Two approvals are necessary to merge. ### The seven rules of a great Git commit message 1. Separate subject from body with a blank line 2. Limit the subject line to 50 characters 3. Capitalize the subject line 4. Do not end the subject line with a period 5. Use the imperative mood in the subject line 6. Wrap the body at 72 characters 7. Use the body to explain what and why vs. how ### Single commit PR The simplify reviewing and to protect against accidental merges of work-in-progress commits, the CI system enforces that a PR is merged as a single commit. However, sometimes a multi-commit PR is warranted if each commit is a distinct change that makes sense to submit in the same PR. The single-commit enforcement can be disabled by adding the following trailer to the PR description (note that the trailer should not be in the commit message): Disable-check: commit-count Generally, though, small single-commit PRs are preferred over large multi-commit PRs so that PRs are easier to review. It is also good to avoid multiple commits that have unrelated changes or a lot of work-in-progress changes that are not appropriate for a well-formatted commit log. ### Documentation If the code change adds a new feature, limitation, or change in behavior, the PR might need to include documentation or a separate follow-up PR to the [documentation repository](https://github.com/timescale/docs). ================================================ FILE: .github/ci_settings.py ================================================ #!/usr/bin/env python # This file and its contents are licensed under the Apache License 2.0. # Please see the included NOTICE for copyright information and # LICENSE-APACHE for a copy of the license. # Common settings for our CI jobs # # EARLIEST is the minimum postgres version required when building from source # LATEST is the maximum postgres version that is supported # ABI_MIN is the minimum postgres version required when the extension was build against LATEST # PG15_EARLIEST = "15.10" PG15_LATEST = "15.17" PG15_ABI_MIN = "15.10" PG16_EARLIEST = "16.6" PG16_LATEST = "16.13" PG16_ABI_MIN = "16.6" PG17_EARLIEST = "17.2" PG17_LATEST = "17.9" PG17_ABI_MIN = "17.2" PG18_EARLIEST = "18.0" PG18_LATEST = "18.3" PG18_ABI_MIN = "18.0" PG_LATEST = [PG15_LATEST, PG16_LATEST, PG17_LATEST, PG18_LATEST] PG_LATEST_ONLY = [PG17_LATEST] ================================================ FILE: .github/codespell-ignore-words ================================================ brin clos inh inout isnt larg relaction textin ================================================ FILE: .github/filters.yaml ================================================ shell: - '**.sh' - .github/workflows/shellcheck.yaml sql: - 'sql/**' - '.github/**' src: - 'CMakeLists.txt' - 'cmake/**' - 'coverage/**' - 'scripts/**' - 'sql/**' - 'src/**' - 'test/**' - 'tsl/**' - '.github/**' windows-workflow: - '.github/workflows/windows-build-and-test.yaml' ================================================ FILE: .github/gh_config_reader.py ================================================ #!/usr/bin/env python # This file and its contents are licensed under the Apache License 2.0. # Please see the included NOTICE for copyright information and # LICENSE-APACHE for a copy of the license. # We hash the .github directory to understand whether our Postgres build cache # can still be used, and the __pycache__ files interfere with that, so don't # create them. import sys sys.dont_write_bytecode = True import ci_settings import json import os # generate commands to set github action variables for key in dir(ci_settings): if not key.startswith("__"): value = getattr(ci_settings, key) with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: print(str.format("{0}={1}", key, json.dumps(value)), file=output) ================================================ FILE: .github/gh_matrix_builder.py ================================================ #!/usr/bin/env python # This file and its contents are licensed under the Apache License 2.0. # Please see the included NOTICE for copyright information and # LICENSE-APACHE for a copy of the license. # Python script to dynamically generate matrix for github action # Since we want to run additional test configurations when triggered # by a push to prerelease_test or by cron but github actions don't # allow a dynamic matrix via yaml configuration, we generate the matrix # with this python script. While we could always have the full matrix # and have if checks on every step that would make the actual checks # harder to browse because the workflow will have lots of entries and # only by navigating into the individual jobs would it be visible # if a job was actually run. # We hash the .github directory to understand whether our Postgres build cache # can still be used, and the __pycache__ files interfere with that, so don't # create them. import sys sys.dont_write_bytecode = True import json import os import random import subprocess from ci_settings import ( PG15_EARLIEST, PG15_LATEST, PG16_EARLIEST, PG16_LATEST, PG17_EARLIEST, PG17_LATEST, PG18_EARLIEST, PG18_LATEST, PG_LATEST, ) # github event type which is either push, pull_request or schedule event_type = sys.argv[1] pull_request = event_type == "pull_request" m = { "include": [], } # Ignored tests that are known to be flaky or have known issues. default_ignored_tests = { "bgw_db_scheduler", "bgw_db_scheduler_fixed", "bgw_job_stat_history", "bgw_launcher", "telemetry", "memoize", "net", } # Some tests are ignored on PG earlier than 17 due to broken MergeAppend cost model there. ignored_before_pg17 = default_ignored_tests | {"merge_append_partially_compressed"} # Some tests are ignored on PG earlier than 16 due to changes in default relation # size estimates. ignored_before_pg16 = default_ignored_tests | {"columnar_scan_cost"} # Tests that we do not run as part of a Flake tests flaky_exclude_tests = { # Not executed as a flake test since it easily exhausts available # background worker slots. "bgw_launcher", # Not executed as a flake test since it takes a very long time and # easily interferes with other tests. "bgw_scheduler_restart", } # helper functions to generate matrix entries # the release and apache config inherit from the # debug config to reduce repetition def build_debug_config(overrides): # llvm version and clang versions must match otherwise # there will be build errors this is true even when compiling # with gcc as clang is used to compile the llvm parts. # # Strictly speaking, WARNINGS_AS_ERRORS=ON is not needed here, but # we add it as a precaution. Intention is to have at least one # release and one debug build with WARNINGS_AS_ERRORS=ON so that we # capture warnings generated due to changes in the code base or the # compiler. base_config = dict( { "build_type": "Debug", "cc": "gcc", "clang": "clang", "coverage": False, "cxx": "g++", "extra_packages": "clang llvm llvm-dev", "ignored_tests": default_ignored_tests, "name": "Debug", "os": "ubuntu-22.04", "pg_extra_args": "--enable-debug --enable-cassert --with-llvm LLVM_CONFIG=llvm-config", "pg_extensions": "postgres_fdw test_decoding", "installcheck": True, "pginstallcheck": True, "tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON", } ) base_config.update(overrides) return base_config # We build this release configuration with WARNINGS_AS_ERRORS=ON to # make sure that we can build with -Werrors even for release # builds. This will capture some cases where warnings are generated # for release builds but not for debug builds. def build_release_config(overrides): release_config = dict( { "name": "Release", "build_type": "RelWithDebInfo", "tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON", "coverage": False, } ) release_config.update(overrides) return build_debug_config(release_config) def build_without_telemetry(overrides): config = dict( { "name": "ReleaseWithoutTelemetry", "coverage": False, } ) config.update(overrides) config = build_release_config(config) config["tsdb_build_args"] += " -DUSE_TELEMETRY=OFF" return config def build_apache_config(overrides): apache_config = dict( { "name": "ApacheOnly", "build_type": "RelWithDebInfo", "tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON -DAPACHE_ONLY=1", "coverage": False, } ) apache_config.update(overrides) return build_debug_config(apache_config) def macos_config(overrides): macos_ignored_tests = { "bgw_launcher", "pg_dump", "compression_bgw", "compressed_collation", } openssl_path = "/usr/local/opt/openssl@3" base_config = dict( { "cc": "clang", "clang": "clang", "coverage": False, "cxx": "clang++", "extra_packages": "", "ignored_tests": default_ignored_tests.union(macos_ignored_tests), "os": "macos-15-intel", "pg_extra_args": ( " --enable-debug" f" --with-libraries={openssl_path}/lib" f" --with-includes={openssl_path}/include" " --without-icu" ), "pg_extensions": "postgres_fdw test_decoding", "pginstallcheck": True, "tsdb_build_args": ( " -DASSERTIONS=ON" " -DREQUIRE_ALL_TESTS=ON" f" -DOPENSSL_ROOT_DIR={openssl_path}" ), } ) base_config.update(overrides) return base_config # always test debug build on latest of all supported pg versions m["include"].append( build_debug_config({"pg": PG15_LATEST, "ignored_tests": ignored_before_pg16}) ) m["include"].append( build_debug_config({"pg": PG16_LATEST, "ignored_tests": ignored_before_pg17}) ) m["include"].append(build_debug_config({"pg": PG17_LATEST})) m["include"].append(build_debug_config({"pg": PG18_LATEST, "coverage": True})) # Also test on ARM. The custom arm64 runner is only available in the # timescale/timescaledb repository. # See the available runners here: # https://github.com/timescale/timescaledb/actions/runners if os.environ.get("GITHUB_REPOSITORY") == "timescale/timescaledb": m["include"].append( build_debug_config( { "pg": PG18_LATEST, "os": "timescaledb-runner-arm64", # We need to enable ARM crypto extensions to build the vectorized grouping # code. The actual architecture for our ARM CI runner is reported as: # -imultiarch aarch64-linux-gnu - -mlittle-endian -mabi=lp64 -march=armv8.2-a+crypto+fp16+rcpc+dotprod "pg_extra_args": "--enable-debug --enable-cassert --without-llvm CFLAGS=-march=armv8.2-a+crypto", } ) ) # test timescaledb with release config on latest postgres release in MacOS # we only run compilation tests in pull requests. m["include"].append( build_release_config( macos_config( { "pg": PG18_LATEST, "installcheck": not pull_request, "pginstallcheck": not pull_request, } ) ) ) # Test latest postgres release without telemetry. Also run clang-tidy on it # because it's the fastest one. m["include"].append( build_without_telemetry( { "pg": PG18_LATEST, "cc": "clang", "cxx": "clang++", "tsdb_build_args": "-DLINTER=ON -DWARNINGS_AS_ERRORS=ON", } ) ) # if this is not a pull request e.g. a scheduled run or a push # to a specific branch like prerelease_test we add additional # entries to the matrix if not pull_request: # add debug test for first supported PG15 version m["include"].append( build_debug_config( { "pg": PG15_EARLIEST, "ignored_tests": ignored_before_pg16 | {"insert_single"}, } ) ) # add debug test for first supported PG16 version m["include"].append( build_debug_config( { "pg": PG16_EARLIEST, "ignored_tests": ignored_before_pg17 | {"insert_single"}, } ) ) # add debug test for first supported PG17 version if PG17_EARLIEST != PG17_LATEST: m["include"].append( build_debug_config( { "pg": PG17_EARLIEST, "ignored_tests": {"insert_single"}, } ) ) # add debug test for first supported PG18 version if PG18_EARLIEST != PG18_LATEST: m["include"].append(build_debug_config({"pg": PG18_EARLIEST})) # add debug tests for timescaledb on latest postgres release in MacOS m["include"].append( build_debug_config( macos_config({"pg": PG15_LATEST, "ignored_tests": ignored_before_pg16}) ) ) m["include"].append( build_debug_config( macos_config({"pg": PG16_LATEST, "ignored_tests": ignored_before_pg17}) ) ) m["include"].append(build_debug_config(macos_config({"pg": PG17_LATEST}))) m["include"].append(build_debug_config(macos_config({"pg": PG18_LATEST}))) # add release test for latest pg releases m["include"].append( build_release_config({"pg": PG15_LATEST, "ignored_tests": ignored_before_pg16}) ) m["include"].append( build_release_config({"pg": PG16_LATEST, "ignored_tests": ignored_before_pg17}) ) m["include"].append(build_release_config({"pg": PG17_LATEST})) m["include"].append(build_release_config({"pg": PG18_LATEST})) # add apache only test for latest pg versions for PG_LATEST_VER in PG_LATEST: m["include"].append(build_apache_config({"pg": PG_LATEST_VER})) # to discover issues with upcoming releases we run CI against # the stable branches of supported PG releases m["include"].append( build_debug_config( { "pg": 15, "ignored_tests": ignored_before_pg16 | { "bgw_custom", "bgw_scheduler_restart", "bgw_job_stat_history_errors_permissions", "bgw_job_stat_history_errors", "bgw_job_stat_history", "bgw_db_scheduler_fixed", "bgw_reorder_drop_chunks", "scheduler_fixed", "compress_bgw_reorder_drop_chunks", }, "snapshot": "snapshot", } ) ) m["include"].append( build_debug_config( { "pg": 16, "ignored_tests": ignored_before_pg17, "snapshot": "snapshot", } ) ) m["include"].append( build_debug_config( { "pg": 17, "snapshot": "snapshot", } ) ) m["include"].append( build_debug_config( { "pg": 18, "snapshot": "snapshot", } ) ) elif len(sys.argv) > 2: # Check if we need to check for the flaky tests. Determine which test files # have been changed in the PR. The sql files might include other files that # change independently, and might be .in templates, so it's easier to look # at the output files. They are also the same for the isolation tests. p = subprocess.Popen( f"git diff --name-only {sys.argv[2]} -- '**expected/*.out'", stdout=subprocess.PIPE, shell=True, ) output, err = p.communicate() p_status = p.wait() if p_status != 0: print( f'git diff failed: code {p_status}, output "{output}", stderr "{err}"', file=sys.stderr, ) sys.exit(1) tests = set() test_count = 1 for f in output.decode().split("\n"): print(f) if not f: continue test_count += 1 if test_count > 10: print( f"too many ({test_count}) changed tests, won't run the flaky check", file=sys.stderr, ) print("full list:", file=sys.stderr) print(output, file=sys.stderr) tests = set() break basename = os.path.basename(f) split = basename.split(".") name = split[0] ext = split[-1] if ext == "out": # Account for the version number. tests.add(name) else: # Should've been filtered out above. print( f"unknown extension '{ext}' for test output file '{f}'", file=sys.stderr ) sys.exit(1) if tests: to_run = [t for t in list(tests) if t not in flaky_exclude_tests] * 20 random.shuffle(to_run) installcheck_args = f'TESTS="{" ".join(to_run)}"' m["include"].append( build_debug_config( { "coverage": False, "installcheck_args": installcheck_args, "name": "Flaky Check Debug", "pg": PG18_LATEST, "pginstallcheck": False, } ) ) # generate command to set github action variable with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: print(str.format("matrix={0}", json.dumps(m, default=list)), file=output) ================================================ FILE: .github/workflows/abi.yaml ================================================ # Test minimum and maximum ABI compatible postgres version # # Build timescaledb against specific postgres version and then run our # tests with that library loaded in a different postgres version. # This is to detect changes in required minimum/maximum postgres versions # for our built packages. # This test is expected to fail when upstream does ABI incompatible changes # in a new minor postgresql version. name: ABI Test "on": schedule: # run daily 20:00 on main branch - cron: '0 20 * * *' push: branches: - prerelease_test - trigger/abi pull_request: paths: .github/workflows/abi.yaml workflow_dispatch: jobs: config: runs-on: ubuntu-latest outputs: pg15_abi_min: ${{ steps.config.outputs.pg15_abi_min }} pg16_abi_min: ${{ steps.config.outputs.pg16_abi_min }} pg17_abi_min: ${{ steps.config.outputs.pg17_abi_min }} pg18_abi_min: ${{ steps.config.outputs.pg18_abi_min }} pg15_latest: ${{ steps.config.outputs.pg15_latest }} pg16_latest: ${{ steps.config.outputs.pg16_latest }} pg17_latest: ${{ steps.config.outputs.pg17_latest }} pg18_latest: ${{ steps.config.outputs.pg18_latest }} steps: - name: Checkout source code uses: actions/checkout@v4 - name: Read configuration id: config run: python .github/gh_config_reader.py abi_test: name: ABI Test ${{ matrix.dir }} PG${{ matrix.pg }} runs-on: ubuntu-latest needs: config strategy: fail-fast: false matrix: pg: [ 15, 16, 17, 18 ] ignores: - 'net telemetry' include: - pg: 15 builder: ${{ fromJson(needs.config.outputs.pg15_abi_min) }} tester: ${{ fromJson(needs.config.outputs.pg15_latest) }} - pg: 16 builder: ${{ fromJson(needs.config.outputs.pg16_abi_min) }} tester: ${{ fromJson(needs.config.outputs.pg16_latest) }} - pg: 17 builder: ${{ fromJson(needs.config.outputs.pg17_abi_min) }} tester: ${{ fromJson(needs.config.outputs.pg17_latest) }} - pg: 18 builder: ${{ fromJson(needs.config.outputs.pg18_abi_min) }} tester: ${{ fromJson(needs.config.outputs.pg18_latest) }} steps: - name: Checkout TimescaleDB uses: actions/checkout@v4 - name: Build extension with ${{ matrix.builder }} run: | BUILDER_IMAGE="postgres:${{matrix.builder}}-alpine" docker pull ${BUILDER_IMAGE} docker buildx imagetools inspect ${BUILDER_IMAGE} docker run -i --rm -v $(pwd):/mnt -e EXTRA_PKGS="${EXTRA_PKGS}" ${BUILDER_IMAGE} bash <<"EOF" apk add cmake gcc make build-base git ${EXTRA_PKGS} # We run the same extension on different docker images, old versions # have OpenSSL 1.1 and the new versions have OpenSSL 3, so we try to # pin the 1.1. Note that depending on PG version, both images might # have 1.1 or 3, so we first try to install the versioned 1.1 package, # and if it's not present, it means the unversioned package is 1.1, so # we install it. apk add openssl1.1-compat-dev || apk add openssl-dev # Postgres is compiled with ICU, so the pg_locale.h depends on the ICU # headers and we have to install them apk add icu-dev git config --global --add safe.directory /mnt cd /mnt BUILD_DIR=build_abi BUILD_FORCE_REMOVE=true ./bootstrap make -C build_abi -j $(getconf _NPROCESSORS_ONLN) install mkdir -p build_abi/install_ext build_abi/install_lib cp `pg_config --sharedir`/extension/timescaledb*.{control,sql} build_abi/install_ext cp `pg_config --pkglibdir`/timescaledb*.so build_abi/install_lib EOF - name: Run tests on server ${{ matrix.tester }} run: | TEST_IMAGE="postgres:${{ matrix.tester }}-alpine" docker pull ${TEST_IMAGE} docker buildx imagetools inspect ${TEST_IMAGE} docker run -i --rm -v $(pwd):/mnt -e EXTRA_PKGS="${EXTRA_PKGS}" ${TEST_IMAGE} bash <<"EOF" apk add cmake gcc make build-base sudo coreutils ${EXTRA_PKGS} apk add openssl1.1-compat-dev || apk add openssl-dev cd /mnt cp build_abi/install_ext/* `pg_config --sharedir`/extension/ cp build_abi/install_lib/* `pg_config --pkglibdir` chown -R postgres /mnt set -o pipefail [ -f /usr/bin/gmake ] || ln -s /usr/bin/make /usr/bin/gmake sudo -u postgres make -j $(getconf _NPROCESSORS_ONLN) -C build_abi \ -k regresscheck regresscheck-t regresscheck-shared \ IGNORES="${{ matrix.ignores }}" | tee installcheck.log EOF - name: Show regression diffs if: always() id: collectlogs run: | sudo chmod a+rw . sudo find build_abi -name regression.diffs -exec cat {} + > regression.log sudo find build_abi -name postmaster.log -exec cat {} + > postmaster.log if [[ -s regression.log ]]; then echo "regression_diff=true" >>$GITHUB_OUTPUT; fi grep -e 'FAILED' -e 'failed (ignored)' -e 'not ok' installcheck.log || true cat regression.log - name: Save regression diffs if: always() && steps.collectlogs.outputs.regression_diff == 'true' uses: actions/upload-artifact@v4 with: name: Regression diff ABI Breakage ${{ matrix.dir }} PG${{ matrix.pg }} path: regression.log - name: Save postmaster.log if: always() uses: actions/upload-artifact@v4 with: name: PostgreSQL log ABI Breakage ${{ matrix.dir }} PG${{ matrix.pg }} path: postmaster.log ================================================ FILE: .github/workflows/apt-installcheck.yaml ================================================ # Test running make installcheck on our APT packages. name: "Packaging tests: Installcheck for APT" "on": schedule: # run daily 0:00 on main branch - cron: '0 0 * * *' pull_request: paths: .github/workflows/apt-installcheck.yaml push: tags: - '*' branches: - release_test - trigger/package_test workflow_dispatch: jobs: apt_tests: name: APT ${{ matrix.runner }} ${{ matrix.image }} PG${{ matrix.pg }} ${{ matrix.license }} container: image: ${{ matrix.image }} env: DEBIAN_FRONTEND: noninteractive strategy: fail-fast: false matrix: runner: - ubuntu-latest - timescaledb-runner-arm64 image: - ubuntu:22.04 pg: - 17 license: - "TSL" include: - runner: ubuntu-latest arch: AMD64 - runner: timescaledb-runner-arm64 arch: ARM - image: ubuntu:22.04 image_friendly: Ubuntu 22.04 runs-on: ${{ matrix.runner }} steps: - name: Add repositories run: | apt-get update apt-get install -y wget lsb-release gnupg sudo postgresql-common git cmake jq yes | /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh image_type=$(lsb_release -i -s | tr '[:upper:]' '[:lower:]') echo "deb https://packagecloud.io/timescale/timescaledb/${image_type}/ $(lsb_release -c -s) main" \ > /etc/apt/sources.list.d/timescaledb.list wget --quiet -O - https://packagecloud.io/timescale/timescaledb/gpgkey | gpg --dearmor -o /etc/apt/trusted.gpg.d/timescale_timescaledb.gpg - name: Install timescaledb run: | apt-get update apt-get install -y --no-install-recommends \ timescaledb-2${{ matrix.pkg_suffix }}-postgresql-${{ matrix.pg }} timescaledb-tools timescaledb-tune --quiet --yes - uses: actions/checkout@v4 - name: Get version of latest release id: versions run: | version=$(wget -q https://api.github.com/repos/timescale/timescaledb/releases/latest -O - | jq -r .tag_name) echo "version=${version}" echo "version=${version}" >>$GITHUB_OUTPUT grep PRETTY_NAME /etc/os-release >> $GITHUB_OUTPUT echo "arch=$(uname -m)" >> $GITHUB_OUTPUT - name: Test Installation id: installation run: | set -xeu pg_ctlcluster ${{ matrix.pg }} main start sudo -u postgres psql -X -c "CREATE EXTENSION timescaledb" \ -c "SELECT extname,extversion,version() FROM pg_extension WHERE extname='timescaledb'" installed_version=$(sudo -u postgres psql -X -t \ -c "SELECT extversion FROM pg_extension WHERE extname='timescaledb';" | sed -e 's! !!g') if [ "${{ steps.versions.outputs.version }}" != "$installed_version" ];then false fi sudo -u postgres psql -qtAX -c " select format(E'commit_hash=%s\ncommit_tag=%s', commit_hash, commit_tag) from _timescaledb_functions.get_git_commit(); " | tee -a $GITHUB_OUTPUT - name: Checkout the sources for version ${{ steps.installation.outputs.commit_tag }} uses: actions/checkout@v4 with: ref: ${{ steps.installation.outputs.commit_hash }} - name: Run make installcheck id: installcheck shell: bash run: | chown -R postgres:postgres . chmod g+s . sudo -u postgres git log -1 . apt install -y postgresql-server-dev-${{ matrix.pg }} sudo -u postgres cmake -B build -S . -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DTEST_PG_LOG_DIRECTORY="$(readlink -f .)" set -o pipefail sudo -u postgres LANG=C.UTF-8 make -k -C build installcheck | tee installcheck.log - name: Collect the logs if: always() run: | find . -name regression.diffs -exec cat {} + > regression.log find . -mindepth 2 -name initdb.log -exec cat {} + > initdb.log - name: Show regression diffs if: always() run: | grep -e 'FAILED' -e 'failed (ignored)' -e 'not ok' installcheck.log || true cat regression.log - name: Save the logs if: always() uses: actions/upload-artifact@v4 with: name: Installcheck logs ${{ matrix.runner }} ${{ matrix.image_friendly }} PG ${{ matrix.pg }} path: | postmaster.* initdb.log regression.log installcheck.log ================================================ FILE: .github/workflows/apt-packages.yaml ================================================ # Test installing our ubuntu and debian packages for the latest version. name: "Packaging tests: APT" "on": schedule: # run daily 0:00 on main branch - cron: '0 0 * * *' pull_request: paths: .github/workflows/apt-packages.yaml push: tags: - '*' branches: - release_test - trigger/package_test workflow_dispatch: jobs: apt_tests: name: APT ${{ matrix.arch }} ${{ matrix.image }} PG${{ matrix.pg }} ${{ matrix.license }} runs-on: ${{ matrix.runner }} container: image: ${{ matrix.image }} env: DEBIAN_FRONTEND: noninteractive strategy: fail-fast: false matrix: arch: [ "x86", "ARM" ] image: [ "debian:11-slim", "debian:12-slim", "debian:13-slim", "ubuntu:22.04", "ubuntu:24.04" ] pg: [ 15, 16, 17, 18 ] license: [ "TSL", "Apache"] include: - license: Apache pkg_suffix: "-oss" - arch: "x86" runner: "ubuntu-latest" - arch: "ARM" runner: "cloud-image-runner-ubuntu-24-arm64" exclude: - image: "debian:11-slim" pg: 18 steps: - name: Add repositories run: | apt-get update apt-get install -y wget lsb-release gnupg apt-transport-https sudo postgresql-common jq yes | /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh image_type=$(lsb_release -i -s | tr '[:upper:]' '[:lower:]') echo "deb https://packagecloud.io/timescale/timescaledb/${image_type}/ $(lsb_release -c -s) main" \ > /etc/apt/sources.list.d/timescaledb.list if [ "${image_type}" = "ubuntu" ]; then wget --quiet -O - https://packagecloud.io/timescale/timescaledb/gpgkey | apt-key add - else wget --quiet -O - https://packagecloud.io/timescale/timescaledb/gpgkey | gpg --dearmor -o /etc/apt/trusted.gpg.d/timescale_timescaledb.gpg fi - name: Install timescaledb run: | apt-get update apt-get install -y --no-install-recommends \ timescaledb-2${{ matrix.pkg_suffix }}-postgresql-${{ matrix.pg }} timescaledb-tools timescaledb-tune --quiet --yes - name: List available versions run: | apt-cache show timescaledb-2${{ matrix.pkg_suffix }}-postgresql-${{ matrix.pg }} \ | grep -e Version: -e Depends: | tr '\n' ' ' | sed -e 's! Version: !\n!g' -e 's!Version: !!' -e 's!$!\n!' - name: Show files in package run: | dpkg -L timescaledb-2${{ matrix.pkg_suffix }}-postgresql-${{ matrix.pg }} - uses: actions/checkout@v4 - name: Get version of latest release id: versions run: | version=$(wget -q https://api.github.com/repos/timescale/timescaledb/releases/latest -O - | jq -r .tag_name) echo "version=${version}" echo "version=${version}" >>$GITHUB_OUTPUT - name: Test Installation run: | pg_ctlcluster ${{ matrix.pg }} main start sudo -u postgres psql -X -c "CREATE EXTENSION timescaledb" \ -c "SELECT extname,extversion,version() FROM pg_extension WHERE extname='timescaledb'" installed_version=$(sudo -u postgres psql -X -t \ -c "SELECT extversion FROM pg_extension WHERE extname='timescaledb';" | sed -e 's! !!g') if [ "${{ steps.versions.outputs.version }}" != "$installed_version" ];then false fi - name: Test Downgrade run: | # Since this runs nightly on main we have to get the previous version # from the last released version and not current branch. prev_version=$(wget --quiet -O - \ https://raw.githubusercontent.com/timescale/timescaledb/${{ steps.versions.outputs.version }}/version.config \ | grep previous_version | sed -e 's!previous_version = !!') sudo -u postgres psql -X -c "ALTER EXTENSION timescaledb UPDATE TO '${prev_version}'" \ -c "SELECT extname,extversion,version() FROM pg_extension WHERE extname='timescaledb'" installed_version=$(sudo -u postgres psql -X -t \ -c "SELECT extversion FROM pg_extension WHERE extname='timescaledb';" | sed -e 's! !!g') if [ "$prev_version" != "$installed_version" ];then false fi - name: Install toolkit run: | apt-get install -y --no-install-recommends \ timescaledb-toolkit-postgresql-${{ matrix.pg }} - name: List available toolkit versions run: | apt-cache show timescaledb-toolkit-postgresql-${{ matrix.pg }} | grep -e Version: -e Depends: | tr '\n' ' ' | sed -e 's! Version: !\n!g' -e 's!Version: !!' -e 's!$!\n!' - name: PostgreSQL log if: always() run: | cat /var/log/postgresql/postgresql-${{ matrix.pg }}-main.log ================================================ FILE: .github/workflows/backport-trigger.yaml ================================================ # A helper workflow to trigger the run of the backport workflow on the main # branch, when a release branch or the main branch were changed. name: Trigger the Backport Workflow "on": push: branches: - main - ?.*.x pull_request: paths: .github/workflows/backport-trigger.yaml jobs: backport_trigger: runs-on: timescaledb-runner-arm64 steps: - name: Checkout TimescaleDB uses: actions/checkout@v4 - name: Trigger the Backport Workflow env: GH_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | gh workflow run backport.yaml --ref main ================================================ FILE: .github/workflows/backport.yaml ================================================ name: Backport Bug Fixes on: schedule: # Run weekdays 12:00 on main branch, so that it doesn't wreak havoc on # weekends. Good to have regular runs so that we can react to changes in # issue tags, or retry some spurious network errors, or whatever. - cron: '0 12 * * 1-5' workflow_dispatch: push: # This is also triggered from backport-trigger.yaml when the release branches # are updated. branches: # You can run and debug new versions of the backport script by pushing it # to this branch. workflow_dispatch can only be run through github cli for # branches that are not main, so it's inconvenient. - trigger/backport # The workflow needs the permission to push branches permissions: contents: write pull-requests: write issues: write actions: write statuses: write jobs: backport: name: Backport Bug Fixes runs-on: timescaledb-runner-arm64 steps: - name: Install Python Dependencies run: | pip install PyGithub requests - name: Checkout TimescaleDB uses: actions/checkout@v4 with: token: ${{ secrets.ORG_AUTOMATION_TOKEN }} - name: Run the Backport Script env: ORG_AUTOMATION_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | git remote --verbose scripts/backport.py 2>&1 git remote --verbose ================================================ FILE: .github/workflows/catalog-updates-check.yaml ================================================ name: Check for unsafe catalog updates "on": pull_request: push: branches: - main - ?.*.x jobs: check_catalog_correctly_updated: name: Check updates to latest-dev and reverse-dev are properly handled by PR runs-on: timescaledb-runner-arm64 steps: - name: Checkout source uses: actions/checkout@v4 with: fetch-depth: 0 - name: Install pglast run: | python -m pip install pglast - name: Check sql file contents run: | find sql -name '*.sql' -not -path 'sql/updates/*' -not -path 'sql/compat.sql' | xargs -IFILE python scripts/check_updates.py FILE - name: Check latest-dev contents run: | python scripts/check_updates.py --latest "sql/updates/latest-dev.sql" - name: Check for idempotency in SQL scripts if: always() run: | python scripts/check_sql_script.py sql/*.sql # To allow fixing previous mistakes we run the check against reverse-dev but don't # fail it on errors. - name: Check reverse-dev contents if: always() run: | python scripts/check_updates.py "sql/updates/reverse-dev.sql" || true ================================================ FILE: .github/workflows/changelog-check.yaml ================================================ name: Check for changelog entry file "on": pull_request: types: [opened, synchronize, reopened, edited] branches: - main jobs: # Check if the PR creates a separate file with changelog entry in the # ".unreleased" folder # # This check can be disabled by adding the following line in the PR text # # Disable-check: force-changelog-file # # The file having the changelog entry is expected to have lines in the # following format # # Fixes: #NNNN <bug description> (mandatory in case of bugfixes) # Thanks: @name <thank you note> (optional) # Implements: #NNNN <feature description> (mandatory in case of new features) check_changelog_file: name: Check for file with CHANGELOG entry runs-on: timescaledb-runner-arm64 steps: - name: Setup Python uses: actions/setup-python@v5 with: python-version: '3.13' - name: Install Python Dependencies run: | pip install PyGithub - name: Checkout source uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 - name: Check if the pull request adds file in ".unreleased" folder shell: bash --norc --noprofile {0} env: BODY: ${{ github.event.pull_request.body }} GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} run: | set -euo pipefail folder=".unreleased" # Get the list of modified files as a bash array. Exclude the # development-related files because they don't require a changelog. mapfile -t modified_files < <(gh pr view $PR_NUMBER --json files --jq ' [.files.[].path | select( (startswith(".github") or startswith("test") or startswith("tsl/test") or startswith("scripts")) | not)] | .[]') echo "Modified files: ${modified_files[@]}" # Get the changelog files as a bash array mapfile -t changelog_files < <(gh pr view $PR_NUMBER --json files --jq " [.files.[].path | select(startswith(\"${folder}\"))] | .[]") echo "Changelog files: ${changelog_files[@]}" if echo "$BODY" | egrep -qsi "Disable-check:[[:space:]]*force-changelog-file" then # skip changelog checks if forced : elif (( ${#modified_files[@]} > 0 && ${#changelog_files[@]} == 0 )) then # if no changelog files found, and the PR does not have the force disable check option echo "PR does not add a change log file in .unreleased/ folder" echo "Check .unreleased/template.rfc822 for the format of the change log file." echo echo "To disable changelog updated check, add this trailer to pull request message:" echo echo "Disable-check: force-changelog-file" echo echo "Trailers follow RFC2822 conventions, so no whitespace" echo "before field name and the check is case-insensitive for" echo "both the field name and the field body." exit 1 else # check the format of the files in .unreleased folder for file in "${changelog_files[@]}" do if ! scripts/check_changelog_format.py "${file}" then echo "Invalid CHANGELOG entries in ${file}." exit 1 fi done fi ================================================ FILE: .github/workflows/claude-code-review.yaml ================================================ name: Claude Code Review on: workflow_dispatch: # Manual trigger only, use @claude mentions for on-demand reviews jobs: claude-review: runs-on: ubuntu-latest permissions: contents: read pull-requests: read issues: read id-token: write steps: - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: 1 - name: Run Claude Code Review id: claude-review uses: anthropics/claude-code-action@v1 with: anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} prompt: | Please review this pull request and provide feedback on: - Code quality and best practices - Potential bugs or issues - Performance considerations - Security concerns - Test coverage Be constructive and helpful in your feedback. Use `gh pr comment` with your Bash tool to leave your review as a comment on the PR. # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md # or https://docs.anthropic.com/en/docs/claude-code/sdk#command-line for available options claude_args: '--allowed-tools "Bash(gh issue view:*),Bash(gh search:*),Bash(gh issue list:*),Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh pr list:*)"' ================================================ FILE: .github/workflows/coccinelle.yaml ================================================ # Check our codebase for defective programming patterns name: Coccinelle "on": pull_request: push: branches: - main - ?.*.x jobs: coccinelle: name: Coccinelle # coccinelle version in ubuntu-latest (20.04) is too old so we run # this in jammy (22.04) runs-on: ubuntu-22.04 steps: - name: Install Dependencies run: | sudo apt-get update sudo apt-get -y install coccinelle - name: Checkout TimescaleDB uses: actions/checkout@v4 - name: Run coccinelle run: | ./scripts/coccinelle.sh - name: Save coccinelle.diff if: always() uses: actions/upload-artifact@v4 with: name: coccinelle.diff path: coccinelle.diff ================================================ FILE: .github/workflows/code_style.yaml ================================================ name: Code style "on": push: branches: - main - ?.*.x pull_request: jobs: cmake_checks: name: Check CMake files runs-on: ubuntu-22.04 steps: - name: Install prerequisites run: pip install cmakelang - name: Checkout source uses: actions/checkout@v4 - name: Run format on CMake files run: | git reset --hard find -name CMakeLists.txt -exec cmake-format -i {} + find src test tsl -name '*.cmake' -exec cmake-format -i {} + git diff --exit-code perl_checks: name: Check Perl code in tree runs-on: ubuntu-22.04 steps: - name: Install prerequisites run: sudo apt install perltidy - name: Checkout source uses: actions/checkout@v4 - name: Check trailing whitespace if: always() run: | find . -name '*.p[lm]' -exec perl -pi -e 's/[ \t]+$//' {} + git diff --exit-code - name: Format Perl files, if needed if: always() run: | git reset --hard find . -name '*.p[lm]' -exec perltidy -b -bext=/ {} + git diff --exit-code yaml_checks: name: Check YAML code in tree runs-on: ubuntu-latest steps: - name: Install prerequisites run: | pip install yamllint - name: Checkout source uses: actions/checkout@v4 - name: Run yamllint run: | find . -type f \( -name "*.yaml" -or -name "*.yml" \) -print -exec yamllint {} \+ spelling_checks: name: Check spelling runs-on: ubuntu-latest steps: - name: Install prerequisites run: | pip install codespell - name: Checkout source uses: actions/checkout@v4 - name: Run codespell run: | find . -type f \( -name "*.c" -or -name "*.h" -or -name "*.yaml" -or -name "*.yml" -or -name "*.sh" -or -name "*.cmake" -or -name "*.py" -or -name "*.pl" -or -name "CMakeLists.txt" \) \ -exec codespell -I .github/codespell-ignore-words {} \+ cc_checks: name: Check code formatting runs-on: ubuntu-22.04 strategy: fail-fast: false env: LLVM_VER: 17 steps: - name: Install dependencies run: | gpg --batch --keyserver hkp://keyserver.ubuntu.com --recv-keys 15CF4D18AF4F7421 gpg --batch --export --export-options export-minimal --armor 15CF4D18AF4F7421 | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null . /etc/os-release echo "deb https://apt.llvm.org/${VERSION_CODENAME}/ llvm-toolchain-${VERSION_CODENAME}-${LLVM_VER} main" | sudo tee /etc/apt/sources.list.d/llvm.list >/dev/null sudo apt-get update sudo apt-get install clang-format-${LLVM_VER} - name: Checkout source uses: actions/checkout@v4 - name: Check trailing whitespace if: always() run: | git reset --hard find . -type f -regex '.*\.\(c\|h\|sql\|sql\.in\)$' -exec perl -pi -e 's/[ \t]+$//' {} + git diff --exit-code - name: Check code formatting if: always() run: | sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-${LLVM_VER} 100 sudo update-alternatives --set clang-format /usr/bin/clang-format-${LLVM_VER} git reset --hard ./scripts/clang_format_all.sh git diff --exit-code - name: FIXME annotations left in code (use TODO for long-term notes) if: always() run: | ! grep fixme -niR ./* python_checks: name: Check Python code in tree runs-on: ubuntu-latest steps: - name: Install prerequisites run: | pip install --upgrade pip pip install black prospector pylint dodgy \ mccabe pycodestyle pyflakes \ psutil pygithub pglast testgres more_itertools # pinning snowballstemmer to version 2.2.0 due to: # https://github.com/prospector-dev/prospector/issues/763 pip install --force-reinstall --no-deps snowballstemmer==2.2.0 pip list pip list --user - name: Checkout source uses: actions/checkout@v4 - name: Run prospector run: | git reset --hard find . -type f -name "*.py" -print -exec prospector {} + -exec black {} + git diff --exit-code misc_checks: name: Check license, update scripts, git hooks, missing gitignore entries and unnecessary template tests runs-on: ubuntu-24.04 strategy: fail-fast: false steps: - name: Checkout source if: always() uses: actions/checkout@v4 - name: Check license if: always() run: ./scripts/check_license_all.sh - name: Check git commit hooks if: always() run: | ./scripts/githooks/commit_msg_tests.py - name: Check for unreferenced test files if: always() run: ./scripts/check_unreferenced_files.sh - name: Check for missing gitignore entries for template test files if: always() run: | ./bootstrap ./scripts/check_missing_gitignore_for_template_tests.sh - name: Check for unnecessary template test files if: always() run: ./scripts/check_unnecessary_template_tests.sh - name: Check for orphaned output test files if: always() run: ./scripts/check_orphaned_test_output_files.sh ================================================ FILE: .github/workflows/coverity.yaml ================================================ name: Coverity "on": schedule: # run at 22:00 on every saturday - cron: '0 22 * * TUE,SAT' push: branches: - coverity_scan - trigger/coverity pull_request: paths: .github/workflows/coverity.yaml workflow_dispatch: jobs: coverity: name: Coverity ${{ matrix.pg }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: # run only on the 3 latest PG versions as we have rate limit on coverity pg: [16, 17, 18] os: [ubuntu-22.04] steps: - name: Install Dependencies run: | sudo apt-get update sudo apt-get install gnupg systemd-coredump gdb postgresql-common yes | sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh sudo apt-get update sudo apt-get install postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} - name: Checkout TimescaleDB uses: actions/checkout@v4 - name: Coverity tools run: | wget https://scan.coverity.com/download/linux64 \ --post-data "token=${{ secrets.COVERITY_TOKEN }}&project=timescale%2Ftimescaledb" \ -O coverity_tool.tgz -q tar xf coverity_tool.tgz mv cov-analysis-linux64-* coverity - name: Build TimescaleDB run: | PATH="$GITHUB_WORKSPACE/coverity/bin:/usr/lib/postgresql/${{ matrix.pg }}/bin:$PATH" ./bootstrap -DCMAKE_BUILD_TYPE=Release cov-build --dir cov-int make -C build - name: Upload report env: FORM_EMAIL: --form email=ci@timescale.com FORM_FILE: --form file=@timescaledb.tgz FORM_DESC: --form description="CI" FORM_TOKEN: --form token="${{ secrets.COVERITY_TOKEN }}" COVERITY_URL: https://scan.coverity.com/builds?project=timescale%2Ftimescaledb run: | tar czf timescaledb.tgz cov-int curl $FORM_TOKEN $FORM_EMAIL $FORM_DESC $FORM_FILE \ --form version="$(grep '^version' version.config | cut -b11-)-${{ matrix.pg }}" $COVERITY_URL ================================================ FILE: .github/workflows/docker-images.yaml ================================================ # Test our docker images are built with the most recent version # The main purpose of this test is to check the image is working # and the latest tag points to an image with the most recent # release. name: "Packaging tests: Docker images" "on": schedule: # run daily 0:00 on main branch - cron: '0 0 * * *' pull_request: paths: .github/workflows/docker-images.yaml push: tags: - '*' branches: - release_test - trigger/package_test workflow_dispatch: jobs: docker_tests: name: ${{ matrix.image }} runs-on: ubuntu-latest services: ts: image: timescale/${{ matrix.image }} ports: - 5432:5432 env: POSTGRES_HOST_AUTH_METHOD: trust POSTGRESQL_PASSWORD: ci env: PGHOST: localhost PGUSER: postgres PGPASSWORD: ci strategy: fail-fast: false matrix: image: [ "timescaledb:latest-pg15", "timescaledb:latest-pg16", "timescaledb:latest-pg17", "timescaledb:latest-pg18", "timescaledb-ha:pg15", "timescaledb-ha:pg16", "timescaledb-ha:pg17", "timescaledb-ha:pg18", ] steps: - uses: actions/checkout@v4 - name: Get version of latest release id: versions env: GH_TOKEN: ${{ github.token }} run: | version=$(gh release list --json tagName,isLatest --jq '.[] | select(.isLatest) | .tagName') echo "version=${version}" >>$GITHUB_OUTPUT - name: Wait for services to start run: | sleep 10 pg_isready -t 30 - name: Check version run: | psql -c "SELECT extname,extversion,version() FROM pg_extension WHERE extname='timescaledb'" installed_version=$(psql -X -t \ -c "SELECT extversion FROM pg_extension WHERE extname='timescaledb';" | sed -e 's! !!g') if [ "${{ steps.versions.outputs.version }}" != "$installed_version" ];then false fi - name: Create hypertable run: | psql -c "$(cat <<SQL CREATE TABLE metrics(time timestamptz, device text, metric text, value float); SELECT create_hypertable('metrics','time'); SQL )" ================================================ FILE: .github/workflows/extras-diagnostic.yaml ================================================ # Test diagnostic script from timescaledb-extras works name: "timescaledb-extras diagnostic" "on": schedule: # run daily 0:00 on main branch - cron: '0 0 * * *' pull_request: paths: .github/workflows/extras-diagnostic.yaml push: branches: - trigger/package_test workflow_dispatch: jobs: diagnostic_test: name: timescaledb-extras PG${{ matrix.pg }} nightly runs-on: ubuntu-latest services: ts: image: timescaledev/timescaledb:nightly-pg${{ matrix.pg }} ports: - 5432:5432 env: POSTGRES_HOST_AUTH_METHOD: trust POSTGRESQL_PASSWORD: ci env: PGHOST: localhost PGUSER: postgres PGPASSWORD: ci strategy: fail-fast: false matrix: pg: [15,16,17,18] steps: - uses: actions/checkout@v4 with: repository: 'timescale/timescaledb-extras' - name: Wait for services to start run: | sleep 10 pg_isready -t 30 - name: Prepare database run: | psql -v ON_ERROR_STOP=1 -c "$(cat <<SQL SELECT version(); SELECT extname, extversion from pg_extension; CREATE TABLE t(time timestamptz) WITH (tsdb.hypertable); INSERT INTO t SELECT '2025-01-01'; INSERT INTO t SELECT '2025-02-01'; SELECT compress_chunk(show_chunks('t')); SQL )" - name: Run diagnostic script run: | psql -v ON_ERROR_STOP=1 < diagnostic.sql ================================================ FILE: .github/workflows/homebrew.yaml ================================================ # Test installation of our homebrew tap for latest version name: "Packaging tests: Homebrew" "on": schedule: # run daily 20:00 on main branch - cron: '0 20 * * *' pull_request: paths: .github/workflows/homebrew.yaml push: tags: - '*' branches: - release_test - trigger/package_test - trigger/homebrew_test workflow_dispatch: jobs: homebrew: name: Homebrew runs-on: macos-latest strategy: fail-fast: false matrix: license: [ "TSL", "Apache"] include: - license: Apache install_options: "--with-oss-only" steps: - name: Setup run: | brew install postgresql@17 echo "/opt/homebrew/opt/postgresql@17/bin" >> $GITHUB_PATH brew tap timescale/tap brew info timescaledb - name: Install timescaledb run: | brew uninstall cmake brew install timescaledb ${{ matrix.install_options }} timescaledb-tune --quiet --yes timescaledb_move.sh brew services start postgresql@17 # checkout code to get version information - uses: actions/checkout@v4 - name: Test Installation env: GH_TOKEN: ${{ github.token }} run: | psql -X -c "CREATE EXTENSION timescaledb;" postgres psql -X -c "SELECT extname,extversion,version() FROM pg_extension WHERE extname='timescaledb';" postgres version=$(gh release list --json tagName,isLatest --jq '.[] | select(.isLatest) | .tagName') installed_version=$(psql -X -t \ -c "SELECT extversion FROM pg_extension WHERE extname='timescaledb';" \ postgres | sed -e 's! !!g') if [ "$version" != "$installed_version" ];then echo "Installed version \"${installed_version}\" does not match expected version \"${version}\"." false fi ================================================ FILE: .github/workflows/issue-handling.yaml ================================================ name: Process issue workflows "on": issues: types: [opened, closed, labeled] issue_comment: types: [created, edited] jobs: add-to-project: name: Add issue to projects # issue_comment is triggered when commenting on both issues and # pull requests. To avoid adding pull requests to the bug board, # filter out pull requests if: ${{ !github.event.issue.pull_request }} runs-on: timescaledb-runner-arm64 steps: - name: Add to bugs board uses: actions/add-to-project@v0.5.0 with: project-url: https://github.com/orgs/timescale/projects/55 github-token: ${{ secrets.ORG_AUTOMATION_TOKEN }} labeled: bug, needs-triage, flaky-test label-operator: OR - name: Add to CAggs board uses: actions/add-to-project@v0.5.0 with: project-url: https://github.com/orgs/timescale/projects/128 github-token: ${{ secrets.ORG_AUTOMATION_TOKEN }} labeled: continuous_aggregate notify-sec: name: Notify security channel runs-on: timescaledb-runner-arm64 if: >- github.event_name == 'issues' && github.event.action == 'opened' && ( contains(github.event.issue.labels.*.name, 'segfault') || contains(github.event.issue.labels.*.name, 'security') ) || github.event_name == 'issues' && github.event.action == 'labeled' && ( github.event.label.name == 'segfault' || github.event.label.name == 'security' ) env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} steps: - name: Post to Security Channel uses: slackapi/slack-github-action@v1.25.0 with: channel-id: '${{ secrets.SLACK_CHANNEL_SECURITY }}' payload: | { "text": "Issue #${{github.event.issue.number}} (${{github.event.issue.title}})> needs attention", "blocks": [ { "type": "section", "text": { "type": "mrkdwn", "text": "Issue <${{github.event.issue.html_url}}|#${{github.event.issue.number}} ${{github.event.issue.title}}> needs attention" } } ] } close-issue: name: Issue is closed runs-on: timescaledb-runner-arm64 if: github.event_name == 'issues' && github.event.action == 'closed' && contains(github.event.issues.issue.labels.*.name, 'bug') steps: - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 with: gh_token: ${{ secrets.ORG_AUTOMATION_TOKEN }} organization: timescale project_id: 55 resource_node_id: ${{ github.event.issue.node_id }} status_value: 'Done' - name: Remove waiting-for-author label uses: andymckay/labeler@3a4296e9dcdf9576b0456050db78cfd34853f260 with: remove-labels: 'waiting-for-author, no-activity' repo-token: ${{ secrets.ORG_AUTOMATION_TOKEN }} waiting-for-author: name: Waiting for Author runs-on: timescaledb-runner-arm64 if: github.event_name == 'issues' && github.event.action == 'labeled' && github.event.label.name == 'waiting-for-author' steps: - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 with: gh_token: ${{ secrets.ORG_AUTOMATION_TOKEN }} organization: timescale project_id: 55 resource_node_id: ${{ github.event.issue.node_id }} status_value: 'Waiting for Author' waiting-for-engineering: name: Waiting for Engineering runs-on: timescaledb-runner-arm64 if: github.event_name == 'issue_comment' && !github.event.issue.pull_request steps: - name: Install dependencies run: | sudo apt-get update sudo apt-get install jq - name: Get board column of issue id: extract_board_column continue-on-error: true run: | # The following GraphQL query requests all issues from a project. It uses the repository # to locate the issue and get the reference to the project. Then, a filter is applied # (number: $project) to get the reference to the desired project (i.e., the bug board). # Now, all issues from this project are requested. The reason for fetching all issues is # because the current implementation of the GitHub GraphQL API for projects does not # support server-side filters for issues and we can not restrict the query to our issue. # Therefore, we fetch all issues and apply a filter on the client side in the next step. gh api graphql --paginate -F issue=$ISSUE -F project=$PROJECT -F owner=$OWNER -F repo=$REPO -f query=' query board_column($issue: Int!, $project: Int!, $owner: String!, $repo: String!, $endCursor: String) { repository(owner: $owner, name: $repo) { issue(number: $issue) { projectV2(number: $project) { items(first: 100, after: $endCursor) { nodes { fieldValueByName(name: "Status") { ... on ProjectV2ItemFieldSingleSelectValue { name } } content { ... on Issue { id title number repository { name owner { login } } } } } pageInfo { hasNextPage endCursor } } } } } } ' > api_result # Get board column for issue board_column=$(jq -r ".data.repository.issue.projectV2.items.nodes[] | select (.content.number == $ISSUE and .content.repository.name == \"$REPO\" and .content.repository.owner.login == \"$OWNER\") | .fieldValueByName.name" api_result) echo "Issue is in column: $board_column" echo "issue_board_column=$board_column" >> "$GITHUB_OUTPUT" env: OWNER: timescale REPO: ${{ github.event.repository.name }} PROJECT: 55 ISSUE: ${{ github.event.issue.number }} GITHUB_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} - name: Check if organization member uses: tspascoal/get-user-teams-membership@v2 id: checkUserMember with: username: ${{ github.actor }} organization: timescale team: 'database-eng' GITHUB_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} - name: Remove waiting-for-author label if: >- steps.checkUserMember.outputs.isTeamMember == 'false' && steps.extract_board_column.outputs.issue_board_column == 'Waiting for Author' uses: andymckay/labeler@3a4296e9dcdf9576b0456050db78cfd34853f260 with: remove-labels: 'waiting-for-author, no-activity' repo-token: ${{ secrets.ORG_AUTOMATION_TOKEN }} - name: Move to waiting for engineering column if: ${{ steps.checkUserMember.outputs.isTeamMember == 'false' && steps.extract_board_column.outputs.issue_board_column == 'Waiting for Author' }} uses: leonsteinhaeuser/project-beta-automations@v2.0.0 with: gh_token: ${{ secrets.ORG_AUTOMATION_TOKEN }} organization: timescale project_id: 55 resource_node_id: ${{ github.event.issue.node_id }} status_value: 'Waiting for Engineering' ================================================ FILE: .github/workflows/label-handling.yaml ================================================ # # Collection of actions to run when a label is added # to an issue or pull request # name: Label Handling on: pull_request: types: - labeled issues: types: - labeled jobs: # # Ping the Database team if the label was applied on a PR # pr-upgrade-requires-restart: name: "PR: Ping database team if 'upgrade-requires-restart' label is set" runs-on: ubuntu-latest steps: - name: "PR: upgrade-requires-restart" if: github.event.label.name == 'upgrade-requires-restart' uses: actions/github-script@v7 with: script: | github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.payload.pull_request.number, body: "📣 the `upgrade-requires-restart` label was added, pinging @timescale/database-eng" }) # # If a bug issue is opened, try to parse the affected TimescaleDB # and Postgres version used, the add the labels to the issue. If # the labels don't exist, created them. The format must be semver # d.d.d+, any other format gets rejected and no label gets created. # TimescaleDB labels are prefixed with a `v`, e.g. `v2.23.0`` # Postgres labels are prefixed with `postgres`, e.g. `postgres-17.4` # issue-add-version-labels-for-bugs: name: "Issue: Add TimescaleDB and Postgres version labels" runs-on: ubuntu-latest # Only run if the issue has the "bug" label if: contains(github.event.issue.labels.*.name, 'bug') permissions: issues: write steps: - name: Extract versions from issue body id: extract-versions uses: actions/github-script@v7 with: script: | const issueBody = context.payload.issue.body || ''; const issueNumber = context.issue.number; const results = { issueNumber: issueNumber, timescaledb: { found: false }, postgres: { found: false } }; // Extract TimescaleDB version (with line break: "TimescaleDB version affected\nX.Y.Z") const timescaleRegex = /TimescaleDB version affected\s*[\r\n]+\s*(\d+\.\d+\.\d+)/i; const timescaleMatch = issueBody.match(timescaleRegex); if (timescaleMatch) { const version = timescaleMatch[1]; const formatRegex = /^\d+\.\d+\.\d+$/; if (formatRegex.test(version)) { console.log(`Found TimescaleDB version: ${version}`); results.timescaledb = { found: true, version: version, label: `v${version}` }; } } // Extract PostgreSQL version (X.Y or X.Y.Z format, create X.Y label) const postgresRegex = /PostgreSQL version used:\s*[\r\n]+\s*(\d+\.\d+(?:\.\d+)?)/i; const postgresMatch = issueBody.match(postgresRegex); if (postgresMatch) { const fullVersion = postgresMatch[1]; // Extract just X.Y from X.Y or X.Y.Z const majorMinor = fullVersion.match(/^(\d+\.\d+)/); if (majorMinor) { const version = majorMinor[1]; console.log(`Found PostgreSQL version: ${fullVersion}, using ${version} for label`); results.postgres = { found: true, version: version, fullVersion: fullVersion, label: `postgres-${version}` }; } } if (!results.timescaledb.found) { console.log('No valid TimescaleDB version found in issue body'); } if (!results.postgres.found) { console.log('No valid PostgreSQL version found in issue body'); } return results; result-encoding: json - name: Create and add TimescaleDB version label if: fromJson(steps.extract-versions.outputs.result).timescaledb.found == true uses: actions/github-script@v7 env: RESULTS: ${{ steps.extract-versions.outputs.result }} with: script: | const results = JSON.parse(process.env.RESULTS); const labelName = results.timescaledb.label; const owner = context.repo.owner; const repo = context.repo.repo; const issueNumber = results.issueNumber; // Check if label exists, create if it doesn't try { await github.rest.issues.getLabel({ owner, repo, name: labelName }); console.log(`Label ${labelName} already exists`); } catch (error) { if (error.status === 404) { console.log(`Label ${labelName} does not exist, creating it`); await github.rest.issues.createLabel({ owner, repo, name: labelName, color: '0366d6', // Blue color description: `TimescaleDB version ${results.timescaledb.version}` }); console.log(`Created label ${labelName}`); } else { throw error; } } // Add label to issue await github.rest.issues.addLabels({ owner, repo, issue_number: issueNumber, labels: [labelName] }); console.log(`Added label ${labelName} to issue #${issueNumber}`); - name: Create and add PostgreSQL version label if: fromJson(steps.extract-versions.outputs.result).postgres.found == true uses: actions/github-script@v7 env: RESULTS: ${{ steps.extract-versions.outputs.result }} with: script: | const results = JSON.parse(process.env.RESULTS); const labelName = results.postgres.label; const owner = context.repo.owner; const repo = context.repo.repo; const issueNumber = results.issueNumber; // Check if label exists, create if it doesn't try { await github.rest.issues.getLabel({ owner, repo, name: labelName }); console.log(`Label ${labelName} already exists`); } catch (error) { if (error.status === 404) { console.log(`Label ${labelName} does not exist, creating it`); await github.rest.issues.createLabel({ owner, repo, name: labelName, color: '336791', // PostgreSQL blue color description: `PostgreSQL version ${results.postgres.version}` }); console.log(`Created label ${labelName}`); } else { throw error; } } // Add label to issue await github.rest.issues.addLabels({ owner, repo, issue_number: issueNumber, labels: [labelName] }); console.log(`Added label ${labelName} to issue #${issueNumber}`); ================================================ FILE: .github/workflows/label-released-prs.yaml ================================================ # Apply "released" labels to the PRs that got into a particular release. name: Label Released PRs on: release: types: [published] push: branches: [trigger/label-released-prs] jobs: label-release: name: Label Released PRs runs-on: timescaledb-runner-arm64 env: GH_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} steps: - uses: actions/checkout@v4 - name: Install Python Dependencies run: | pip install PyGithub requests - name: Get latest release tag run: | echo "LATEST_TAG=$(gh release view --json tagName --jq '.tagName')" >> $GITHUB_ENV - name: Run label-released script run: | scripts/label-released.py --release "$LATEST_TAG" ================================================ FILE: .github/workflows/libfuzzer.yaml ================================================ name: Libfuzzer "on": schedule: # run daily 1:00 on main branch - cron: '0 1 * * *' push: branches: - trigger/libfuzzer pull_request: paths: - .github/workflows/libfuzzer.yaml - 'tsl/test/fuzzing/compression/**' workflow_dispatch: jobs: build: runs-on: ubuntu-22.04 name: Build PostgreSQL and TimescaleDB env: PG_SRC_DIR: pgbuild PG_INSTALL_DIR: postgresql steps: - name: Install Linux Dependencies run: | # Don't add ddebs here because the ddebs mirror is always 503 Service Unavailable. # If needed, install them before opening the core dump. sudo apt-get update sudo apt-get install 7zip clang lld llvm flex bison libipc-run-perl \ libtest-most-perl tree jq - name: Checkout TimescaleDB uses: actions/checkout@v4 - name: Read configuration id: config run: python -B .github/gh_config_reader.py # We are going to rebuild Postgres daily, so that it doesn't suddenly break # ages after the original problem. - name: Get date for build caching id: get-date run: | echo "date=$(date +"%d")" >> $GITHUB_OUTPUT # we cache the build directory instead of the install directory here # because extension installation will write files to install directory # leading to a tainted cache - name: Cache PostgreSQL id: cache-postgresql uses: actions/cache@v4 with: path: ~/${{ env.PG_SRC_DIR }} key: "postgresql-libfuzzer-${{ steps.get-date.outputs.date }}-${{ hashFiles('.github/**') }}" - name: Build PostgreSQL if: steps.cache-postgresql.outputs.cache-hit != 'true' run: | wget -q -O postgresql.tar.bz2 \ https://ftp.postgresql.org/pub/source/v${{ steps.config.outputs.PG15_LATEST }}/postgresql-${{ steps.config.outputs.PG15_LATEST }}.tar.bz2 mkdir -p ~/$PG_SRC_DIR tar --extract --file postgresql.tar.bz2 --directory ~/$PG_SRC_DIR --strip-components 1 cd ~/$PG_SRC_DIR CC=clang ./configure --prefix=$HOME/$PG_INSTALL_DIR --with-openssl \ --without-readline --without-zlib --without-libxml --enable-cassert \ --enable-debug CC=clang \ CFLAGS="-fuse-ld=lld -ggdb3 -O2 -fno-omit-frame-pointer" make -j$(getconf _NPROCESSORS_ONLN) - name: Install PostgreSQL run: | make -C ~/$PG_SRC_DIR install make -C ~/$PG_SRC_DIR/contrib/postgres_fdw install - name: Upload config.log if: always() uses: actions/upload-artifact@v4 with: name: config.log for PostgreSQL path: ~/${{ env.PG_SRC_DIR }}/config.log - name: Build TimescaleDB run: | set -e export LIBFUZZER_PATH=$(dirname "$(find $(llvm-config --libdir) -name libclang_rt.fuzzer_no_main-x86_64.a | head -1)") # Some pointers for the next time we have linking/undefined symbol problems: # http://web.archive.org/web/20200926071757/https://github.com/google/sanitizers/issues/111 # http://web.archive.org/web/20231101091231/https://github.com/cms-sw/cmssw/issues/40680 cmake -B build -S . -DASSERTIONS=ON -DLINTER=OFF -DCMAKE_VERBOSE_MAKEFILE=1 \ -DWARNINGS_AS_ERRORS=1 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_C_COMPILER=clang \ -DCMAKE_C_FLAGS="-fsanitize=fuzzer-no-link -lstdc++ -L$LIBFUZZER_PATH -l:libclang_rt.fuzzer_no_main-x86_64.a -static-libsan" \ -DCOMPRESSION_FUZZING=1 -DPG_PATH=$HOME/$PG_INSTALL_DIR make -C build -j$(getconf _NPROCESSORS_ONLN) install # Incredibly, the upload-artifact action can't preserve executable permissions: # https://github.com/actions/upload-artifact/issues/38 # It's also extremely slow. - name: Compress the installation directory run: 7z a install.7z $HOME/$PG_INSTALL_DIR - name: Save the installation directory uses: actions/upload-artifact@v4 with: name: fuzzing-install-dir path: install.7z if-no-files-found: error retention-days: 1 fuzz: needs: build strategy: fail-fast: false matrix: case: [ { algo: gorilla , pgtype: float8, bulk: false, runs: 500000000 }, { algo: deltadelta, pgtype: int8 , bulk: false, runs: 500000000 }, { algo: gorilla , pgtype: float8, bulk: true , runs: 1000000000 }, { algo: deltadelta, pgtype: int8 , bulk: true , runs: 1000000000 }, # array has a peculiar recv function that recompresses all input, so # fuzzing it is much slower. The dictionary recv also uses it. { algo: array , pgtype: text , bulk: false, runs: 10000000 }, { algo: array , pgtype: text , bulk: true , runs: 10000000 }, { algo: dictionary, pgtype: text , bulk: false, runs: 100000000 }, { algo: dictionary, pgtype: text , bulk: true , runs: 100000000 }, { algo: bool, pgtype: bool , bulk: false, runs: 500000000 }, { algo: bool, pgtype: bool , bulk: true , runs: 1000000000 }, # uuid tests are slower because it is more work to compare { algo: dictionary, pgtype: uuid , bulk: false, runs: 200000000 }, { algo: dictionary, pgtype: uuid , bulk: true , runs: 500000000 }, { algo: uuid, pgtype: uuid , bulk: false, runs: 200000000 }, { algo: uuid, pgtype: uuid , bulk: true , runs: 400000000 }, ] name: Fuzz decompression ${{ matrix.case.algo }} ${{ matrix.case.pgtype }} ${{ matrix.case.bulk && 'bulk' || 'rowbyrow' }} runs-on: ubuntu-22.04 env: PG_SRC_DIR: pgbuild PG_INSTALL_DIR: postgresql JOB_NAME: Fuzz decompression ${{ matrix.case.algo }} ${{ matrix.case.pgtype }} ${{ matrix.case.bulk && 'bulk' || 'rowbyrow' }} steps: - name: Install Linux dependencies run: | sudo apt update sudo apt install 7zip systemd-coredump gdb - name: Checkout TimescaleDB uses: actions/checkout@v4 - name: Download the installation directory uses: actions/download-artifact@v4 with: name: fuzzing-install-dir - name: Unpack the installation directory run: 7z x -o$HOME install.7z - name: initdb run: | # Have to do this before initializing the corpus, or initdb will complain. set -xeu export PGDATA=db export PGPORT=5432 export PGDATABASE=postgres export PATH=$HOME/$PG_INSTALL_DIR/bin:$PATH initdb echo "shared_preload_libraries = 'timescaledb'" >> $PGDATA/postgresql.conf - name: Set configuration id: config run: | set -x echo "cache_prefix=${{ format('libfuzzer-corpus-2-{0}-{1}', matrix.case.algo, matrix.case.pgtype) }}" >> $GITHUB_OUTPUT echo "name=${{ matrix.case.algo }} ${{ matrix.case.pgtype }} ${{ matrix.case.bulk && 'bulk' || 'rowbyrow' }}" >> $GITHUB_OUTPUT - name: Restore the cached fuzzing corpus (bulk) id: restore-corpus-cache-bulk uses: actions/cache/restore@v4 with: path: db/corpus-bulk key: "${{ steps.config.outputs.cache_prefix }}-bulk" # We save the row-by-row corpus separately from the bulk corpus, so that # they don't overwrite each other. Now we are going to combine them. - name: Restore the cached fuzzing corpus (rowbyrow) id: restore-corpus-cache-rowbyrow uses: actions/cache/restore@v4 with: path: db/corpus-rowbyrow key: "${{ steps.config.outputs.cache_prefix }}-rowbyrow" - name: Initialize the fuzzing corpus run: | # Combine the cached corpus from rowbyrow and bulk fuzzing, and from repository. mkdir -p db/corpus{,-rowbyrow,-bulk} find "tsl/test/fuzzing/compression/${{ matrix.case.algo }}-${{ matrix.case.pgtype }}" -type f -exec cp -n -t db/corpus {} + find "db/corpus-rowbyrow" -type f -exec cp -n -t db/corpus {} + find "db/corpus-bulk" -type f -exec cp -n -t db/corpus {} + ls db/corpus | wc -l - name: Run libfuzzer for compression run: | set -xeu export PGDATA=db export PGPORT=5432 export PGDATABASE=postgres export PATH=$HOME/$PG_INSTALL_DIR/bin:$PATH pg_ctl -o "-clogging_collector=true" -o "-clog_destination=jsonlog,stderr" \ -o "-clog_directory=$(readlink -f .)" -o "-clog_filename=postmaster.log" \ -o "-clog_error_verbosity=verbose" start psql -c "create extension timescaledb;" # Create the fuzzing functions export MODULE_NAME=$(basename $(find $HOME/$PG_INSTALL_DIR -name "timescaledb-tsl-*.so")) psql -a -c "create or replace function fuzz(algo cstring, pgtype regtype, bulk bool, runs int) returns int as '"$MODULE_NAME"', 'ts_fuzz_compression' language c; create or replace function ts_read_compressed_data_directory(algo cstring, pgtype regtype, path cstring, bulk bool) returns table(path text, bytes int, rows int, sqlstate text, location text) as '"$MODULE_NAME"', 'ts_read_compressed_data_directory' language c; " # Start more fuzzing processes in the background. We won't even monitor # their progress, because the server will panic if they find an error. for x in {2..$(getconf _NPROCESSORS_ONLN)} do psql -v ON_ERROR_STOP=1 -c "select fuzz('${{ matrix.case.algo }}', '${{ matrix.case.pgtype }}', '${{ matrix.case.bulk }}', ${{ matrix.case.runs }});" & done # Start the one fuzzing process that we will monitor, in foreground. # The LLVM fuzzing driver calls exit(), so we expect to lose the connection. ret=0 psql -v ON_ERROR_STOP=1 -c "select fuzz('${{ matrix.case.algo }}', '${{ matrix.case.pgtype }}', '${{ matrix.case.bulk }}', ${{ matrix.case.runs }});" || ret=$? if ! [ $ret -eq 2 ] then >&2 echo "Unexpected psql exit code $ret" exit 1 fi ls db/corpus | wc -l fn="ts_read_compressed_data_directory('${{ matrix.case.algo }}', '${{ matrix.case.pgtype }}', 'corpus', '${{ matrix.case.bulk }}')" # Show the statistics about fuzzing corpus psql -c "select count(*), location, min(sqlstate), min(path) from $fn group by location order by count(*) desc " # Save interesting cases because the caches are not available for download from UI mkdir -p interesting psql -qtAX -c "select distinct on (location) 'db/' || path from $fn order by location, bytes, path " | xargs cp -t interesting # Check that we don't have any internal errors errors=$(psql -qtAX --set=ON_ERROR_STOP=1 -c "select count(*) from $fn where sqlstate = 'XX000'") echo "Internal program errors: $errors" [ $errors -eq 0 ] || exit 1 # Shouldn't have any WARNINGS in the log. ! grep -F "] WARNING: " postmaster.log # Check that the server is still alive. psql -c "select 1" - name: Collect the logs if: always() id: collectlogs run: | # wait in case there are in-progress coredumps sleep 10 if coredumpctl -q list >/dev/null; then echo "coredumps=true" >>$GITHUB_OUTPUT; fi # print OOM killer information sudo journalctl --system -q --facility=kern --grep "Killed process" || true - name: Save PostgreSQL log if: always() uses: actions/upload-artifact@v4 with: name: PostgreSQL log for ${{ steps.config.outputs.name }} path: postgres.* - name: Save fuzzer-generated crash cases if: always() uses: actions/upload-artifact@v4 with: name: Crash cases for ${{ steps.config.outputs.name }} path: db/crash-* if-no-files-found: ignore - name: Save interesting cases if: always() uses: actions/upload-artifact@v4 with: name: Interesting cases for ${{ steps.config.outputs.name }} path: interesting/ # We use separate restore/save actions, because the default action won't # save the updated folder after the cache hit. We also want to save the # cache after fuzzing errors, and the default action doesn't save after # errors. # We can't overwrite the existing cache, so we add a unique suffix. The # cache is matched by key prefix, not exact key, and picks the newest # matching item, so this works. # The caches for rowbyrow and bulk fuzzing are saved separately, otherwise # the slower job would always overwrite the cache from the faster one. We # want to combine corpuses from bulk and rowbyrow fuzzing for better # coverage. # Note that the cache action cannot be restored on a path different from the # one it was saved from. To make our lives more interesting, it is not # directly documented anywhere, but we can deduce it from path influencing # the version. - name: Change corpus path to please the 'actions/cache' GitHub Action if: always() run: | rm -rf db/corpus-{bulk,rowbyrow} ||: mv -fT db/corpus{,-${{ matrix.case.bulk && 'bulk' || 'rowbyrow' }}} - name: Save fuzzer corpus if: always() uses: actions/cache/save@v4 with: path: db/corpus-${{ matrix.case.bulk && 'bulk' || 'rowbyrow' }} key: "${{ format('{0}-{1}-{2}-{3}', steps.config.outputs.cache_prefix, matrix.case.bulk && 'bulk' || 'rowbyrow', github.run_id, github.run_attempt) }}" - name: Stack trace if: always() && steps.collectlogs.outputs.coredumps == 'true' run: | sudo coredumpctl gdb <<<" set verbose on set trace-commands on show debug-file-directory printf "'"'"query = '%s'\n\n"'"'", debug_query_string frame function ExceptionalCondition printf "'"'"condition = '%s'\n"'"'", conditionName up 1 l info args info locals bt full " 2>&1 | tee stacktrace.log ./scripts/bundle_coredumps.sh exit 1 # Fail the job if we have core dumps. - name: Upload core dumps if: always() && steps.collectlogs.outputs.coredumps == 'true' uses: actions/upload-artifact@v4 with: name: Coredumps for ${{ steps.config.outputs.name }} path: coredumps - name: Upload test results to the database if: always() env: CI_STATS_DB: ${{ secrets.CI_STATS_DB }} GITHUB_EVENT_NAME: ${{ github.event_name }} GITHUB_REF_NAME: ${{ github.ref_name }} GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} GITHUB_RUN_ID: ${{ github.run_id }} GITHUB_RUN_NUMBER: ${{ github.run_number }} JOB_STATUS: ${{ job.status }} run: | if [[ "${{ github.event_name }}" == "pull_request" ]] ; then GITHUB_PR_NUMBER="${{ github.event.number }}" else GITHUB_PR_NUMBER=0 fi export GITHUB_PR_NUMBER scripts/upload_ci_stats.sh ================================================ FILE: .github/workflows/linux-32bit-build-and-test.yaml ================================================ name: Regression Linux i386 "on": schedule: # run daily 0:00 on main branch # Since we use the date as a part of the cache key to ensure no # stale cache entries hiding build failures we need to make sure # we have a cache entry present before workflows that depend on cache # are run. - cron: '0 0 * * *' push: branches: - main - ?.*.x - prerelease_test - trigger/regression pull_request: jobs: config: runs-on: ubuntu-latest outputs: pg_latest: ${{ github.event_name == 'pull_request' && steps.setter.outputs.PG_LATEST_ONLY || steps.setter.outputs.PG_LATEST }} steps: - name: Checkout source code uses: actions/checkout@v4 - name: Read configuration id: setter run: python .github/gh_config_reader.py check_paths: runs-on: ubuntu-latest outputs: run_ci: >- ${{ github.event_name == 'push' || (steps.filter.outputs.src == 'true' && !contains(github.event.pull_request.labels.*.name, 'skip-ci')) }} steps: - uses: actions/checkout@v4 - uses: dorny/paths-filter@v3 id: filter with: filters: .github/filters.yaml regress: if: needs.check_paths.outputs.run_ci == 'true' name: PG${{ matrix.pg }} ${{ matrix.build_type }} linux-i386 runs-on: ubuntu-latest needs: [config, check_paths] container: image: i386/debian:bookworm-slim options: --privileged --ulimit core=-1:-1 --ulimit fsize=-1:-1 env: PG_SRC_DIR: pgbuild PG_INSTALL_DIR: postgresql CLANG: clang-14 CC: clang-14 CXX: clang++-14 DEBIAN_FRONTEND: noninteractive # vectorized_aggregation has different output on i386 because int8 is by # reference and currently it cannot be used for vectorized hash grouping. # vector_agg_text and vector_agg_groupagg use the UMASH hashing library # that we can't compile on i386. # For the same reason of different hashing, we ignore the results of # several bloom filter tests as well. IGNORES: >- append-* bgw_db_scheduler* columnar_scan_cost compress_bloom_sparse_compat compress_bloom_sparse_debug compress_composite_bloom_debug compress_qualpushdown_saop compress_sort_transform compression_allocation merge_append_partially_compressed net pg_dump telemetry transparent_decompress_chunk-* transparent_decompression-* uncompressed_size vector_agg_byte vector_agg_expr vector_agg_filter vector_agg_groupagg vector_agg_grouping vector_agg_planning-* vector_agg_text vector_agg_uuid vectorized_aggregation ${{ matrix.pg < 16 && 'columnar_scan_cost' || '' }} SKIPS: chunk_adaptive histogram_test-* EXTENSIONS: "postgres_fdw test_decoding" strategy: matrix: pg: ${{ fromJson(needs.config.outputs.pg_latest) }} build_type: [ Debug ] fail-fast: false steps: # /__e/node16/bin/node (used by actions/checkout@v4) needs 64-bit libraries - name: Install 64-bit libraries for GitHub actions run: | apt-get update apt-get install -y lib64atomic1 lib64gcc-s1 lib64stdc++6 libc6-amd64 jq - name: Install build dependencies run: | PG_MAJOR=$(echo "${{ matrix.pg }}" | sed -e 's![.].*!!') echo '/tmp/core.%h.%e.%t' > /proc/sys/kernel/core_pattern apt-get install -y gcc make cmake libssl-dev libipc-run-perl \ libtest-most-perl sudo gdb git wget gawk lbzip2 flex bison lcov base-files \ locales clang-14 llvm-14 llvm-14-dev llvm-14-tools postgresql-client pkgconf \ icu-devtools - name: Checkout TimescaleDB uses: actions/checkout@v4 # We are going to rebuild Postgres daily, so that it doesn't suddenly break # ages after the original problem. - name: Get date for build caching id: get-date run: | echo "date=$(date +"%d")" >> $GITHUB_OUTPUT # we cache the build directory instead of the install directory here # because extension installation will write files to install directory # leading to a tainted cache - name: Cache PostgreSQL ${{ matrix.pg }} ${{ matrix.build_type }} id: cache-postgresql uses: actions/cache@v4 with: path: ~/${{ env.PG_SRC_DIR }} key: "linux-32-bit-postgresql-${{ matrix.pg }}-${{ matrix.cc }}\ -${{ steps.get-date.outputs.date }}-${{ hashFiles('.github/**') }}" - name: Build PostgreSQL ${{ matrix.pg }} if: steps.cache-postgresql.outputs.cache-hit != 'true' run: | wget -q -O postgresql.tar.bz2 \ https://ftp.postgresql.org/pub/source/v${{ matrix.pg }}/postgresql-${{ matrix.pg }}.tar.bz2 mkdir -p ~/$PG_SRC_DIR tar --extract --file postgresql.tar.bz2 --directory ~/$PG_SRC_DIR --strip-components 1 cd ~/$PG_SRC_DIR # When building on i386 with the clang compiler, Postgres requires -msse2 to be used ./configure --prefix=$HOME/$PG_INSTALL_DIR --with-openssl \ --without-readline --without-zlib --without-libxml --enable-cassert \ --enable-debug --with-llvm --without-icu LLVM_CONFIG=llvm-config-14 CFLAGS="-msse2" make -j $(getconf _NPROCESSORS_ONLN) for ext in ${EXTENSIONS}; do make -j $(getconf _NPROCESSORS_ONLN) -C contrib/${ext} done - name: Install PostgreSQL ${{ matrix.pg }} run: | useradd postgres cd ~/$PG_SRC_DIR make install for ext in ${EXTENSIONS}; do make -C contrib/${ext} install done chown -R postgres:postgres $HOME/$PG_INSTALL_DIR sed -i 's/^# *\(en_US.UTF-8\)/\1/' /etc/locale.gen locale-gen - name: Upload config.log if: always() && steps.cache-postgresql.outputs.cache-hit != 'true' uses: actions/upload-artifact@v4 with: name: config.log linux-i386 PG${{ matrix.pg }} path: ~/${{ env.PG_SRC_DIR }}/config.log - name: Build TimescaleDB run: | # The owner of the checkout directory and the files do not match. Add the directory to # Git's "safe.directory" setting. Otherwise git would complain about # 'detected dubious ownership in repository' git config --global --add safe.directory $(pwd) ./bootstrap -DCMAKE_BUILD_TYPE="${{ matrix.build_type }}" \ -DPG_SOURCE_DIR=~/$PG_SRC_DIR -DPG_PATH=~/$PG_INSTALL_DIR \ -DREQUIRE_ALL_TESTS=ON \ -DTEST_PG_LOG_DIRECTORY="$(readlink -f .)" make -j $(getconf _NPROCESSORS_ONLN) -C build make -C build install chown -R postgres:postgres . - name: Upload CMake Logs if: always() uses: actions/upload-artifact@v4 with: name: CMake Logs linux-i386 PG${{ matrix.pg }} path: | build/CMakeCache.txt build/CMakeFiles/CMakeConfigureLog.yaml build/CMakeFiles/CMakeError.log build/CMakeFiles/CMakeOutput.log build/compile_commands.json - name: make installcheck id: installcheck shell: bash run: | set -o pipefail export LANG=C.UTF-8 # # Even disabling parallel plans the Sort Method on 32bits # is always 'still in progress' for PG17 and PG18. So for now added # those tests to SKIPS. # PG_MAJOR=$(echo "${{ matrix.pg }}" | sed -e 's![.].*!!') if [ ${PG_MAJOR} -eq 17 ]; then SKIPS="${SKIPS} constraint_exclusion_prepared-17 ordered_append*" fi if [ ${PG_MAJOR} -eq 18 ]; then SKIPS="${SKIPS} constraint_exclusion_prepared-18 ordered_append*" fi # PostgreSQL cannot be run as root. So, switch to postgres user. runuser -u postgres -- \ make -k -C build installcheck IGNORES="${IGNORES}" \ SKIPS="${SKIPS}" PSQL="${HOME}/${PG_INSTALL_DIR}/bin/psql" \ | tee installcheck.log - name: Show regression diffs if: always() id: collectlogs shell: bash run: | find . -name regression.diffs -exec cat {} + > regression.log if [[ -s regression.log ]]; then echo "regression_diff=true" >>$GITHUB_OUTPUT; fi grep -e 'FAILED' -e 'failed (ignored)' -e 'not ok' installcheck.log || true cat regression.log - name: Save regression diffs if: always() && steps.collectlogs.outputs.regression_diff == 'true' uses: actions/upload-artifact@v4 with: name: Regression diff linux-i386 PG${{ matrix.pg }} path: | regression.log installcheck.log - name: Save PostgreSQL log if: always() uses: actions/upload-artifact@v4 with: name: PostgreSQL log linux-i386 PG${{ matrix.pg }} path: postmaster.* - name: Stack trace Linux if: always() shell: bash run: | # wait in case there are in-progress coredumps sleep 10 if compgen -G "/tmp/core*" > /dev/null; then PG_MAJOR=$(echo "${{ matrix.pg }}" | sed -e 's![.].*!!') for file in /tmp/core* do gdb "${HOME}/${PG_INSTALL_DIR}/bin/postgres" -c $file <<<" set verbose on set trace-commands on show debug-file-directory printf "'"'"query = '%s'\n\n"'"'", debug_query_string bt full # We try to find ExceptionalCondition frame to print the failed condition # for searching in logs. frame function ExceptionalCondition printf "'"'"condition = '%s'\n"'"'", conditionName # Hopefully now we should be around the failed assertion, print where # we are. up 1 list info args info locals " 2>&1 | tee -a stacktrace.log done echo "coredumps=true" >>$GITHUB_OUTPUT exit 1 fi - name: Coredumps if: always() && steps.coredumps.outputs.coredumps == 'true' uses: actions/upload-artifact@v4 with: name: Coredumps linux-i386 PG${{ matrix.pg }} path: coredumps - name: Save stacktraces if: always() && steps.coredumps.outputs.coredumps == 'true' uses: actions/upload-artifact@v4 with: name: Stacktraces linux-i386 PG${{ matrix.pg }} path: stacktrace.log - name: Save TAP test logs if: always() uses: actions/upload-artifact@v4 with: name: TAP test logs ${{ matrix.os }} ${{ matrix.name }} ${{ matrix.pg }} path: | build/test/tmp_check/log build/tsl/test/tmp_check/log - name: Upload test results to the database if: always() shell: bash env: # GitHub Actions allow you neither to use the env context for the job name, # nor to access the job name from the step context, so we have to # duplicate it to work around this nonsense. JOB_NAME: PG${{ matrix.pg }} ${{ matrix.build_type }} linux-i386 CI_STATS_DB: ${{ secrets.CI_STATS_DB }} GITHUB_EVENT_NAME: ${{ github.event_name }} GITHUB_REF_NAME: ${{ github.ref_name }} GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} GITHUB_RUN_ID: ${{ github.run_id }} GITHUB_RUN_NUMBER: ${{ github.run_number }} JOB_STATUS: ${{ job.status }} run: | if [[ "${{ github.event_name }}" == "pull_request" ]] ; then GITHUB_PR_NUMBER="${{ github.event.number }}" else GITHUB_PR_NUMBER=0 fi export GITHUB_PR_NUMBER scripts/upload_ci_stats.sh report_status: name: Regression Linux i386 summary needs: [check_paths, regress] if: always() runs-on: ubuntu-latest steps: - run: | # If the detector failed, or the matrix failed, exit with error if [[ "${{ needs.regress.result }}" == "failure" ]]; then exit 1 fi echo "All checks passed or were safely skipped." ================================================ FILE: .github/workflows/linux-build-and-test.yaml ================================================ name: Regression "on": schedule: # run daily 0:00 on main branch # Since we use the date as a part of the cache key to ensure no # stale cache entries hiding build failures we need to make sure # we have a cache entry present before workflows that depend on cache # are run. - cron: '0 0 * * *' push: branches: - main - ?.*.x - prerelease_test - trigger/regression pull_request: jobs: matrixbuilder: runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - name: Checkout source code uses: actions/checkout@v4 - name: Build matrix id: set-matrix run: | if [[ "${{ github.event_name }}" == "pull_request" ]] ; then git fetch origin ${{ github.base_ref }}:base .github/gh_matrix_builder.py ${{ github.event_name }} base else .github/gh_matrix_builder.py ${{ github.event_name }} fi check_paths: runs-on: ubuntu-latest outputs: run_ci: >- ${{ github.event_name == 'push' || (steps.filter.outputs.src == 'true' && !contains(github.event.pull_request.labels.*.name, 'skip-ci')) }} steps: - uses: actions/checkout@v4 - uses: dorny/paths-filter@v3 id: filter with: filters: .github/filters.yaml regress: if: needs.check_paths.outputs.run_ci == 'true' # Change the JOB_NAME variable below when changing the name. name: PG${{ matrix.pg }}${{ matrix.snapshot }} ${{ matrix.name }} ${{ matrix.os }} needs: [matrixbuilder, check_paths] runs-on: ${{ matrix.os }} env: PG_SRC_DIR: pgbuild PG_INSTALL_DIR: postgresql CLANG: ${{ matrix.clang }} CC: ${{ matrix.cc }} CXX: ${{ matrix.cxx }} # For some reason, on PG <= 15 with faketime the client backends get the # modified time, and the background workers the unmodified time, so it # doesn't work. FAKETIME: | ${{ (contains(matrix.os, 'ubuntu') && !startsWith(matrix.pg, '14.') && !startsWith(matrix.pg, '15.') ) && 'faketime -f +379d' || '' }} strategy: matrix: ${{ fromJson(needs.matrixbuilder.outputs.matrix) }} fail-fast: false steps: - name: Install Linux Dependencies if: runner.os == 'Linux' run: | # Don't add ddebs here because the ddebs mirror is always 503 Service Unavailable. # If needed, install them before opening the core dump. sudo apt-get update sudo apt-get install flex bison lcov systemd-coredump gdb libipc-run-perl \ libtest-most-perl pkgconf icu-devtools faketime jq ${{ matrix.extra_packages }} - name: Install macOS Dependencies if: runner.os == 'macOS' run: | # Disable the automatic dependency upgrade executed by `brew install` # https://docs.brew.sh/Manpage#install-options-formulacask- HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 brew install gawk brew --prefix openssl@3 # Install perl modules after last Homebew call, since Homebrew can change the perl version sudo perl -MCPAN -e "CPAN::Shell->notest('install', 'IPC::Run')" sudo perl -MCPAN -e "CPAN::Shell->notest('install', 'Test::Most')" - name: Setup macOS coredump directory if: runner.os == 'macOS' run: sudo chmod 777 /cores - name: Checkout TimescaleDB uses: actions/checkout@v4 # We are going to rebuild Postgres daily, so that it doesn't suddenly break # ages after the original problem. - name: Get date for build caching id: get-date run: | echo "date=$(date +"%d")" >> $GITHUB_OUTPUT # on macOS the path used is depending on the runner version leading to cache failure # when the runner version changes so we extract runner version from path and add it # as cache suffix - name: Cache suffix if: runner.os == 'macOS' run: echo "CACHE_SUFFIX=-${ImageVersion}" >> $GITHUB_ENV # we cache the build directory instead of the install directory here # because extension installation will write files to install directory # leading to a tainted cache - name: Cache PostgreSQL ${{ matrix.pg }} ${{ matrix.build_type }} id: cache-postgresql if: matrix.snapshot != 'snapshot' && runner.os == 'Linux' uses: actions/cache@v4 with: path: ~/${{ env.PG_SRC_DIR }} key: "${{ matrix.os }}-postgresql-${{ matrix.pg }}-${{ matrix.cc }}-${{ steps.get-date.outputs.date }}-${{ hashFiles('.github/**') }}${{ env.CACHE_SUFFIX }}" - name: Build PostgreSQL ${{ matrix.pg }}${{ matrix.snapshot }} if: steps.cache-postgresql.outputs.cache-hit != 'true' run: | if [ "${{ matrix.snapshot }}" = "snapshot" ]; then wget -q -O postgresql.tar.bz2 \ https://ftp.postgresql.org/pub/snapshot/${{ matrix.pg }}/postgresql-${{ matrix.pg }}-snapshot.tar.bz2 else wget -q -O postgresql.tar.bz2 \ https://ftp.postgresql.org/pub/source/v${{ matrix.pg }}/postgresql-${{ matrix.pg }}.tar.bz2 fi mkdir -p ~/$PG_SRC_DIR tar --extract --file postgresql.tar.bz2 --directory ~/$PG_SRC_DIR --strip-components 1 cd ~/$PG_SRC_DIR ./configure --prefix=$HOME/$PG_INSTALL_DIR --with-openssl \ --without-readline --without-zlib --without-libxml ${{ matrix.pg_extra_args }} make -j $(getconf _NPROCESSORS_ONLN) for ext in ${{ matrix.pg_extensions }}; do make -j $(getconf _NPROCESSORS_ONLN) -C contrib/${ext} done - name: Install PostgreSQL ${{ matrix.pg }} run: | cd ~/$PG_SRC_DIR make install for ext in ${{ matrix.pg_extensions }}; do make -C contrib/${ext} install done echo "$HOME/$PG_INSTALL_DIR/bin" >> "${GITHUB_PATH}" - name: Upload config.log if: always() && steps.cache-postgresql.outputs.cache-hit != 'true' uses: actions/upload-artifact@v4 with: name: config.log for PostgreSQL ${{ matrix.os }} ${{ matrix.name }} ${{ matrix.pg }} path: ~/${{ env.PG_SRC_DIR }}/config.log - name: Test telemetry without OpenSSL if: github.event_name != 'pull_request' && runner.os == 'Linux' && matrix.build_type == 'Debug' run: | BUILD_DIR=nossl ./bootstrap -DCMAKE_BUILD_TYPE=Debug \ -DPG_SOURCE_DIR=$HOME/$PG_SRC_DIR -DPG_PATH=$HOME/$PG_INSTALL_DIR \ ${{ matrix.tsdb_build_args }} -DCODECOVERAGE=${{ matrix.coverage }} -DUSE_OPENSSL=OFF \ -DTEST_PG_LOG_DIRECTORY="$(readlink -f .)" make -j $(getconf _NPROCESSORS_ONLN) -C nossl make -C nossl install make -C nossl regresscheck TESTS=telemetry - name: Build TimescaleDB run: | # Show the actual architecture this CI runner has "$CC" -march=native -E -v - </dev/null 2>&1 | grep cc1 ./bootstrap -DCMAKE_BUILD_TYPE="${{ matrix.build_type }}" \ -DPG_SOURCE_DIR=$HOME/$PG_SRC_DIR -DPG_PATH=$HOME/$PG_INSTALL_DIR \ ${{ matrix.tsdb_build_args }} -DCODECOVERAGE=${{ matrix.coverage }} \ -DTEST_PG_LOG_DIRECTORY="$(readlink -f .)" make -j $(getconf _NPROCESSORS_ONLN) -C build make -C build install - name: Upload CMake Logs if: always() uses: actions/upload-artifact@v4 with: name: CMake Logs ${{ matrix.os }} ${{ matrix.name }} ${{ matrix.pg }} path: | build/CMakeCache.txt build/CMakeFiles/CMakeConfigureLog.yaml build/CMakeFiles/CMakeError.log build/CMakeFiles/CMakeOutput.log build/compile_commands.json - name: Check exported symbols run: ./build/scripts/export_prefix_check.sh - name: make installcheck if: matrix.installcheck run: | set -o pipefail make -k -C build installcheck IGNORES="${{ join(matrix.ignored_tests, ' ') }}" \ SKIPS="${{ join(matrix.skipped_tests, ' ') }}" ${{ matrix.installcheck_args }} \ | tee installcheck.log - name: pginstallcheck if: matrix.pginstallcheck run: make -C build pginstallcheck - name: coverage if: matrix.coverage run: make -j $(getconf _NPROCESSORS_ONLN) -k -C build coverage - name: Send coverage report to Codecov.io app if: matrix.coverage uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} files: ./build/coverage/timescaledb-coverage.info - name: Save LCOV coverage report if: matrix.coverage uses: actions/upload-artifact@v4 with: name: LCOV coverage report ${{ matrix.os }} ${{ matrix.name }} ${{ matrix.pg }} path: ./build/coverage/lcov-report - name: Show regression diffs if: always() id: collectlogs run: | find . -name regression.diffs -exec cat {} + > regression.log if [[ "${{ runner.os }}" == "Linux" ]] ; then # wait in case there are in-progress coredumps sleep 10 if coredumpctl -q list >/dev/null; then echo "coredumps=true" >>$GITHUB_OUTPUT; fi # print OOM killer information sudo journalctl --system -q --facility=kern --grep "Killed process" || true elif [[ "${{ runner.os }}" == "macOS" ]] ; then if [ $(find /cores -type f | wc -l) -gt 0 ]; then echo "coredumps=true" >>$GITHUB_OUTPUT; fi fi if [[ -s regression.log ]]; then echo "regression_diff=true" >> $GITHUB_OUTPUT; fi grep -e 'FAILED' -e 'failed (ignored)' -e 'not ok' installcheck.log || true cat regression.log - name: Save regression diffs if: always() && steps.collectlogs.outputs.regression_diff == 'true' uses: actions/upload-artifact@v4 with: name: Regression diff ${{ matrix.os }} ${{ matrix.name }} ${{ matrix.pg }} path: | regression.log installcheck.log - name: Save PostgreSQL log if: always() uses: actions/upload-artifact@v4 with: name: PostgreSQL log ${{ matrix.os }} ${{ matrix.name }} ${{ matrix.pg }} path: postmaster.* - name: Stack trace Linux if: always() && steps.collectlogs.outputs.coredumps == 'true' && runner.os == 'Linux' run: | sudo coredumpctl debug --debugger=gdb --debugger-arguments='' <<<" set verbose on set trace-commands on show debug-file-directory printf "'"'"query = '%s'\n\n"'"'", debug_query_string bt full # We try to find ExceptionalCondition frame to print the failed condition # for searching in logs. frame function ExceptionalCondition printf "'"'"condition = '%s'\n"'"'", conditionName # Hopefully now we should be around the failed assertion, print where # we are. up 1 list info args info locals " 2>&1 | tee -a stacktrace.log ./scripts/bundle_coredumps.sh - name: Stack trace macOS if: always() && steps.collectlogs.outputs.coredumps == 'true' && runner.os == 'macOS' run: | ~/$PG_SRC_DIR/src/tools/ci/cores_backtrace.sh macos /cores - name: Coredumps if: always() && steps.collectlogs.outputs.coredumps == 'true' uses: actions/upload-artifact@v4 with: name: Coredumps ${{ matrix.os }} ${{ matrix.name }} ${{ matrix.pg }} path: coredumps - name: Save stacktraces if: always() && steps.collectlogs.outputs.coredumps == 'true' uses: actions/upload-artifact@v4 with: name: Stacktraces ${{ matrix.os }} ${{ matrix.name }} ${{ matrix.pg }} path: stacktrace.log - name: Save TAP test logs if: always() uses: actions/upload-artifact@v4 with: name: TAP test logs ${{ matrix.os }} ${{ matrix.name }} ${{ matrix.pg }} path: | build/test/tmp_check/log build/tsl/test/tmp_check/log - name: Check that no new internal program errors are introduced # We collect the same messages when uploading to the CI checks database, # but it runs on different conditions (not on forks and not on flaky check), # and requires different data (job date), so we do it separately here. # The jq --exit-code option is broken with select() on jq-1.6 which we see # on some machines, so we use grep instead (see https://github.com/jqlang/jq/issues/1139). if: always() && contains(matrix.name, 'Flaky') run: | ! jq 'select( (.state_code == "XX000" and .error_severity != "LOG" and (.message | test("TestFailure") | not)) or (.message | test("resource was not closed")) ) | [.message, .func_name, .statement] | @tsv ' -r postmaster.json | grep . - name: Upload test results to the database # Don't upload the results of the flaky check, because the db schema only # supports running one test once per job. if: always() && matrix.installcheck && (! contains(matrix.name, 'Flaky')) env: # GitHub Actions allow you neither to use the env context for the job name, # nor to access the job name from the step context, so we have to # duplicate it to work around this nonsense. JOB_NAME: PG${{ matrix.pg }}${{ matrix.snapshot }} ${{ matrix.name }} ${{ matrix.os }} CI_STATS_DB: ${{ secrets.CI_STATS_DB }} GITHUB_EVENT_NAME: ${{ github.event_name }} GITHUB_REF_NAME: ${{ github.ref_name }} GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} GITHUB_RUN_ID: ${{ github.run_id }} GITHUB_RUN_NUMBER: ${{ github.run_number }} JOB_STATUS: ${{ job.status }} run: | if [[ "${{ github.event_name }}" == "pull_request" ]] ; then GITHUB_PR_NUMBER="${{ github.event.number }}" else GITHUB_PR_NUMBER=0 fi export GITHUB_PR_NUMBER scripts/upload_ci_stats.sh report_status: name: Regression summary needs: [check_paths, regress] if: always() runs-on: ubuntu-latest steps: - run: | # If the detector failed, or the matrix failed, exit with error if [[ "${{ needs.regress.result }}" == "failure" ]]; then exit 1 fi echo "All checks passed or were safely skipped." ================================================ FILE: .github/workflows/loader_check.yml ================================================ name: Check for loader changes "on": pull_request: types: [opened, synchronize, reopened, labeled, unlabeled, edited] jobs: check_loader_change: name: Check for loader changes # Ignore loader changes if acknowledged already if: ${{ !contains(github.event.pull_request.labels.*.name, 'upgrade-requires-restart') }} runs-on: timescaledb-runner-arm64 steps: - name: Checkout source uses: actions/checkout@v4 - name: Check if the pull request changes the loader shell: bash --norc --noprofile {0} env: BODY: ${{ github.event.pull_request.body }} GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} run: | echo "$BODY" | egrep -qsi '^disable-check:.*\<loader-change\>' if [[ $? -ne 0 ]]; then # Get the list of modified files in this pull request files=$(gh pr view $PR_NUMBER --json files --jq '.files.[].path') # Check for loader changes if echo "${files}" | grep -Eq "^src/loader/.+$"; then echo "Warning: This PR changes the loader. Therefore, upgrading to the next TimescaleDB" echo "version requires a restart of PostgreSQL. Make sure to bump the loader version if" echo "necessary and coordinate the release with the cloud team before merging." echo echo "After the release is coordinated, add the 'upgrade-requires-restart' label" echo "to the PR to acknowledge this warning." echo echo "To disable this check, add this trailer to pull request message:" echo echo "Disable-check: loader-change" echo exit 1 fi fi check_loader_version_bump: name: Check if loader versions are incremented for required restart # If the label is present, validate the loader version is bumped if: ${{ contains(github.event.pull_request.labels.*.name, 'upgrade-requires-restart') }} runs-on: timescaledb-runner-arm64 steps: - name: Checkout source uses: actions/checkout@v4 with: fetch-depth: 0 - name: Checkout ${{ github.event.pull_request.base.ref }} branch run: git checkout ${{ github.event.pull_request.base.ref }} - name: Extract versions from ${{ github.event.pull_request.base.ref }} branch id: upstream-versions run: | # Extract version from bgw_interface.c BGW_VERSION_MAIN=$(grep -oP 'const int32 ts_bgw_loader_api_version = \K\d+' src/loader/bgw_interface.c || echo "0") echo "bgw_version_main=$BGW_VERSION_MAIN" >> $GITHUB_OUTPUT # Extract version from launcher_interface.c LAUNCHER_VERSION_MAIN=$(grep -oP '#define MIN_LOADER_API_VERSION \K\d+' src/bgw/launcher_interface.c || echo "0") echo "launcher_version_main=$LAUNCHER_VERSION_MAIN" >> $GITHUB_OUTPUT echo "${{ github.event.pull_request.base.ref }} branch:" echo "src/loader/bgw_interface.c:const int32 ts_bgw_loader_api_version = $BGW_VERSION_MAIN" echo "src/bgw/launcher_interface.c:#define MIN_LOADER_API_VERSION $LAUNCHER_VERSION_MAIN" - name: Checkout PR branch run: git checkout ${{ github.event.pull_request.head.sha }} - name: Extract versions from PR branch id: pr-versions run: | # Extract version from bgw_interface.c BGW_VERSION_PR=$(grep -oP 'const int32 ts_bgw_loader_api_version = \K\d+' src/loader/bgw_interface.c || echo "0") echo "bgw_version_pr=$BGW_VERSION_PR" >> $GITHUB_OUTPUT # Extract version from launcher_interface.c LAUNCHER_VERSION_PR=$(grep -oP '#define MIN_LOADER_API_VERSION \K\d+' src/bgw/launcher_interface.c || echo "0") echo "launcher_version_pr=$LAUNCHER_VERSION_PR" >> $GITHUB_OUTPUT echo "PR:" echo "src/loader/bgw_interface.c:const int32 ts_bgw_loader_api_version = $BGW_VERSION_PR" echo "src/bgw/launcher_interface.c:#define MIN_LOADER_API_VERSION $LAUNCHER_VERSION_PR" - name: Validate version increments run: | BGW_MAIN=${{ steps.upstream-versions.outputs.bgw_version_main }} BGW_PR=${{ steps.pr-versions.outputs.bgw_version_pr }} LAUNCHER_MAIN=${{ steps.upstream-versions.outputs.launcher_version_main }} LAUNCHER_PR=${{ steps.pr-versions.outputs.launcher_version_pr }} echo "Validating version increments..." echo "bgw_interface.c: $BGW_MAIN -> $BGW_PR (expected: $((BGW_MAIN + 1)))" echo "launcher_interface.c: $LAUNCHER_MAIN -> $LAUNCHER_PR (expected: $((LAUNCHER_MAIN + 1)))" VALIDATION_FAILED=false # Check bgw_interface version if [ "$BGW_PR" -ne "$((BGW_MAIN + 1))" ]; then echo "❌ ERROR: bgw_interface.c version should be incremented by 1, expected: $((BGW_MAIN + 1)), Found: $BGW_PR" VALIDATION_FAILED=true else echo "✅ bgw_interface.c version correctly incremented" fi # Check launcher_interface.c version if [ "$LAUNCHER_PR" -ne "$((LAUNCHER_MAIN + 1))" ]; then echo "❌ ERROR: launcher_interface.c version should be incremented by 1, expected: $((LAUNCHER_MAIN + 1)), Found: $LAUNCHER_PR" VALIDATION_FAILED=true else echo "✅ launcher_interface.c version correctly incremented" fi if [ "$VALIDATION_FAILED" = true ]; then echo "Version validation failed. Please ensure: Both version numbers are incremented by exactly 1 from ${{ github.event.pull_request.base.ref }} branch" exit 1 fi - name: Add comment to PR (if validation fails) if: failure() uses: actions/github-script@v7 with: script: | const bgwMain = '${{ steps.upstream-versions.outputs.bgw_version_main }}'; const bgwPr = '${{ steps.pr-versions.outputs.bgw_version_pr }}'; const launcherMain = '${{ steps.upstream-versions.outputs.launcher_version_main }}'; const launcherPr = '${{ steps.pr-versions.outputs.launcher_version_pr }}'; const comment = `## ❌ Loader Version Validation Failed The version numbers in your PR do not meet the requirements, to indicate a acknowledged loader change. | File | ${{ github.event.pull_request.base.ref }} | PR | Expected | |------|-------------|-----------|----------| | \`src/loader/bgw_interface.c:const int32 ts_bgw_loader_api_version\` | ${bgwMain} | ${bgwPr} | ${parseInt(bgwMain) + 1} | | \`src/bgw/launcher_interface.c:#define MIN_LOADER_API_VERSION\` | ${launcherMain} | ${launcherPr} | ${parseInt(launcherMain) + 1} | **Requirements:** Both version numbers must be incremented by 1 from the ${{ github.event.pull_request.base.ref }} branch. Please update the version numbers and push your changes.`; github.rest.issues.createComment({ issue_number: context.issue.number, owner: context.repo.owner, repo: context.repo.repo, body: comment }); ================================================ FILE: .github/workflows/memory-tests.yaml ================================================ name: Memory tests "on": push: branches: - main - ?.*.x - memory_test - trigger/memory_test pull_request: paths: .github/workflows/memory-tests.yaml jobs: memory_leak: name: Memory leak on insert PG${{ matrix.pg }} runs-on: ubuntu-22.04 strategy: matrix: pg: [15, 16, 17, 18] fail-fast: false steps: - name: Install Dependencies run: | sudo apt-get update sudo apt-get install gnupg systemd-coredump gdb postgresql-common python3-psutil yes | sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh sudo apt-get update sudo apt-get install postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} - name: Checkout TimescaleDB uses: actions/checkout@v4 - name: Build TimescaleDB run: | PATH="/usr/lib/postgresql/${{ matrix.pg }}/bin:$PATH" ./bootstrap -DCMAKE_BUILD_TYPE=Release make -C build sudo make -C build install - name: Setup database run: | sudo pg_createcluster ${{matrix.pg}} main sudo tee -a /etc/postgresql/${{ matrix.pg }}/main/postgresql.conf <<-CONF shared_preload_libraries = 'timescaledb' max_worker_processes = 0 log_destination = syslog max_wal_size = 8GB max_wal_senders = 0 wal_level = minimal checkpoint_timeout = 20min log_checkpoints = on bgwriter_lru_maxpages = 0 track_counts = off fsync = off port = 5432 CONF sudo grep port /etc/postgresql/${{ matrix.pg }}/main/postgresql.conf sudo systemctl start postgresql@${{ matrix.pg }}-main.service sudo -u postgres psql -X -c "CREATE USER runner SUPERUSER LOGIN;" - name: Run insert memory test run: | sudo -u postgres python ./scripts/test_memory_spikes.py & sleep 5 \ && psql -d postgres -v ECHO=all -X -f scripts/out_of_order_random_direct.sql - name: Run generic memory test run: | sudo -u postgres python ./scripts/test_memory_spikes.py & sleep 5 \ && psql -d postgres -v ECHO=all -X -f scripts/memory_leaks.sql - name: Postgres log if: always() run: | sudo journalctl -u postgresql@${{ matrix.pg }}-main.service ================================================ FILE: .github/workflows/nightly_cloud_smoke_test.yaml ================================================ name: Nightly - Update smoke test on release branch # QA: run the upgrade smoke tests with the last 10 releases # against the the next version. # Requires upstream GitHub action to deploy the last commit # from the release branch to the QA service in Cloud "on": #schedule: # - cron: '0 0 * * *' workflow_dispatch: pull_request: paths: - .github/workflows/nightly_cloud_smoke_test.yaml - scripts/test_update_smoke.sh - test/sql/updates/*.sql jobs: get-next-version: runs-on: ubuntu-latest outputs: next_version: ${{ steps.fetch.outputs.next_version }} steps: - name: Get next version # Get newest release branch with most recent commits id: fetch run: | RELEASE_BRANCH=$(git ls-remote --heads https://github.com/timescale/timescaledb.git | \ grep -Eo 'refs/heads/[2-9]+\.[0-9]+\.x' | \ sed 's|refs/heads/||' | \ sort -V | \ tail -n1) echo "current release branch: ${RELEASE_BRANCH}" curl --fail -o version.config "https://raw.githubusercontent.com/timescale/timescaledb/refs/heads/${RELEASE_BRANCH}/version.config" NEXT_VERSION=$(head -1 version.config | cut -d ' ' -f 3 | cut -d '-' -f 1) echo "next version: ${NEXT_VERSION} .." echo "next_version=${NEXT_VERSION}" >> "$GITHUB_OUTPUT" generate-matrix: runs-on: ubuntu-latest outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - name: Fetch TimescaleDB releases id: fetch-releases run: | releases=$(curl -s \ -H "Accept: application/vnd.github.v3+json" \ "https://api.github.com/repos/timescale/timescaledb/releases?per_page=10" \ | jq -r '.[].tag_name') # Convert to JSON array format for matrix matrix_json=$(echo "$releases" | jq -R -s -c 'split("\n") | map(select(length > 0))') echo "releases=$matrix_json" >> $GITHUB_OUTPUT # Also output for debugging echo "Found releases:" echo "$releases" - name: Set matrix output id: set-matrix run: | matrix='{"version": ${{ steps.fetch-releases.outputs.releases }}}' echo "matrix=$matrix" >> $GITHUB_OUTPUT echo $matrix test-version: needs: [generate-matrix, get-next-version] runs-on: ubuntu-latest strategy: # run sequentially matrix: ${{ fromJson(needs.generate-matrix.outputs.matrix) }} max-parallel: 1 fail-fast: false steps: - name: Checkout TimescaleDB uses: actions/checkout@v4 - name: Install Dependencies # we want the right version of Postgres for handling any dump file run: | sudo apt-get update sudo apt-get install gnupg postgresql-common yes | sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh sudo apt-get update sudo apt-get install postgresql-17 - name: "Run update smoke test with v${{ matrix.version }} to ${{needs.get-next-version.outputs.next_version}}" # Now run the test. Currently the cloud instance is always up. # only run the test if the versions are not equal run: | PATH="/usr/lib/postgresql/17/bin:$PATH" ./scripts/test_update_smoke.sh \ ${{ matrix.version }} \ ${{needs.get-next-version.outputs.next_version}} \ "${{ secrets.DB_TEAM_QA_SERVICE_CONNECTION_STRING }}" - name: Show logs if: always() run: | ls -l /tmp/smoketest*/* cat /tmp/smoketest*/* - name: Upload Artifacts # Save the logs, so if there is a failure we'll have a better # chance to understand what went wrong. if: failure() uses: actions/upload-artifact@v4 with: name: Cloud Update test smoke path: /tmp/smoketest*/* ================================================ FILE: .github/workflows/pg_ladybug.yaml ================================================ name: pg_ladybug "on": pull_request: push: branches: - main - ?.*.x jobs: pg_ladybug: runs-on: ubuntu-latest env: CC: clang-19 CXX: clang++-19 LLVM_CONFIG: llvm-config-19 steps: - name: Install dependencies run: | sudo apt-get update sudo apt-get purge llvm-16 llvm-17 llvm-18 clang-16 clang-17 clang-18 sudo apt-get install llvm-19 llvm-19-dev clang-19 libclang-19-dev clang-tidy-19 libcurl4-openssl-dev sudo ln -sf /usr/bin/clang-tidy-19 /usr/bin/clang-tidy - name: Checkout timescaledb uses: actions/checkout@v4 - name: Checkout pg_ladybug uses: actions/checkout@v4 with: repository: 'timescale/pg_ladybug' path: 'pg_ladybug' ref: '0.1.0' - name: build pg_ladybug run: | cd pg_ladybug cmake -S . -B build -DLLVM_ROOT=/usr/lib/llvm-19 make -C build -j $(getconf _NPROCESSORS_ONLN) sudo make -C build install - name: Verify pg_ladybug run: | clang-tidy --load /usr/local/lib/libPostgresCheck.so --checks='-*,postgres-*' --list-checks | grep postgres - name: Configure timescaledb run: | # installing postgres headers pulls in llvm-17 which confuses pg_ladybug build process so we install this here instead of at beginning sudo apt-get install postgresql-server-dev-16 ./bootstrap -DCMAKE_BUILD_TYPE=Debug -DLINTER=ON -DCLANG_TIDY_EXTRA_OPTS=",-*,postgres-*;--load=/usr/local/lib/libPostgresCheck.so" - name: Build timescaledb run: | make -C build -j $(getconf _NPROCESSORS_ONLN) ================================================ FILE: .github/workflows/pg_upgrade-test.yaml ================================================ name: pg_upgrade test "on": push: branches: - main - ?.*.x - prerelease_test pull_request: jobs: check_paths: runs-on: ubuntu-latest outputs: run_ci: >- ${{ github.event_name == 'push' || (steps.filter.outputs.sql == 'true' && !contains(github.event.pull_request.labels.*.name, 'skip-ci')) }} steps: - uses: actions/checkout@v4 - uses: dorny/paths-filter@v3 id: filter with: filters: .github/filters.yaml pg_upgrade_test: needs: check_paths if: needs.check_paths.outputs.run_ci == 'true' name: pg_upgrade test from PG${{ matrix.pg_version_old }} to PG${{ matrix.pg_version_new }} runs-on: 'ubuntu-latest' strategy: matrix: include: - pg_version_old: 15 # 15 to 16 pg_version_new: 16 - pg_version_old: 15 # 15 to 17 pg_version_new: 17 - pg_version_old: 15 # 15 to 18 pg_version_new: 18 - pg_version_old: 16 # 16 to 17 pg_version_new: 17 - pg_version_old: 16 # 16 to 18 pg_version_new: 18 - pg_version_old: 17 # 17 to 18 pg_version_new: 18 fail-fast: false env: OUTPUT_DIR: ${{ github.workspace }}/pg_upgrade_test steps: - name: Install Linux Dependencies run: | sudo apt-get update sudo apt-get install pip postgresql-common - name: Install testgres run: | pip install testgres - name: Install PostgreSQL ${{ matrix.pg_version_old}} and ${{ matrix.pg_version_new }} run: | yes | sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh echo "deb https://packagecloud.io/timescale/timescaledb/ubuntu/ $(lsb_release -c -s) main" | sudo tee /etc/apt/sources.list.d/timescaledb.list wget --quiet -O - https://packagecloud.io/timescale/timescaledb/gpgkey | sudo apt-key add - sudo apt-get update sudo apt-get install -y \ postgresql-${{ matrix.pg_version_old }} postgresql-server-dev-${{ matrix.pg_version_old }} \ postgresql-${{ matrix.pg_version_new }} postgresql-server-dev-${{ matrix.pg_version_new }} sudo apt-get install -y --no-install-recommends \ timescaledb-2-postgresql-${{ matrix.pg_version_old }} \ timescaledb-2-postgresql-${{ matrix.pg_version_new }} - name: Checkout TimescaleDB uses: actions/checkout@v4 - name: Build and install TimescaleDB on PostgreSQL ${{ matrix.pg_version_old}} env: BUILD_DIR: pg${{ matrix.pg_version_old }} run: | PATH="/usr/lib/postgresql/${{ matrix.pg_version_old }}/bin:$PATH" ./bootstrap -DCMAKE_BUILD_TYPE=Release -DWARNINGS_AS_ERRORS=OFF -DASSERTIONS=ON -DLINTER=OFF -DGENERATE_DOWNGRADE_SCRIPT=OFF -DREGRESS_CHECKS=OFF -DTAP_CHECKS=OFF make -j -C pg${{ matrix.pg_version_old }} sudo make -j -C pg${{ matrix.pg_version_old }} install - name: Build and install TimescaleDB on PostgreSQL ${{ matrix.pg_version_new}} env: BUILD_DIR: pg${{ matrix.pg_version_new }} run: | PATH="/usr/lib/postgresql/${{ matrix.pg_version_new }}/bin:$PATH" ./bootstrap -DCMAKE_BUILD_TYPE=Release -DWARNINGS_AS_ERRORS=OFF -DASSERTIONS=ON -DLINTER=OFF -DGENERATE_DOWNGRADE_SCRIPT=OFF -DREGRESS_CHECKS=OFF -DTAP_CHECKS=OFF make -j -C pg${{ matrix.pg_version_new }} sudo make -j -C pg${{ matrix.pg_version_new }} install - name: Run pg_upgrade test env: PGVERSIONOLD: ${{ matrix.pg_version_old }} PGVERSIONNEW: ${{ matrix.pg_version_new }} DIFFFILE: ${{ env.OUTPUT_DIR }}/upgrade_check.diff run: | scripts/test_pg_upgrade.py diff -u \ "${OUTPUT_DIR}/post.pg${PGVERSIONOLD}.log" \ "${OUTPUT_DIR}/post.pg${PGVERSIONNEW}.log" | \ tee "${DIFFFILE}" if [[ -s "${DIFFFILE}" ]]; then echo "pg_upgrade test for ${PGVERSIONOLD} -> ${PGVERSIONNEW} failed" exit 1 fi - name: Show pg_upgrade diffs if: always() env: DIFFFILE: ${{ env.OUTPUT_DIR }}/upgrade_check.diff DIROLD: pg${{ matrix.pg_version_old }} DIRNEW: pg${{ matrix.pg_version_new }} run: | cd ${OUTPUT_DIR} cat ${DIFFFILE} tar czf /tmp/pg_upgrade_artifacts.tgz \ ${DIFFFILE} \ ${OUTPUT_DIR}/*.log \ ${OUTPUT_DIR}/${DIROLD}/logs/* \ ${OUTPUT_DIR}/${DIRNEW}/logs/* \ ${OUTPUT_DIR}/${DIRNEW}/data/pg_upgrade_output.d/* - name: Upload pg_upgrade logs if: always() uses: actions/upload-artifact@v4 with: name: pg_upgrade logs from ${{ matrix.pg_version_old }} to ${{ matrix.pg_version_new }} path: /tmp/pg_upgrade_artifacts.tgz report_status: name: pg_upgrade summary needs: [check_paths, pg_upgrade_test] if: always() runs-on: ubuntu-latest steps: - run: | # If the detector failed, or the matrix failed, exit with error if [[ "${{ needs.pg_upgrade_test.result }}" == "failure" ]]; then exit 1 fi echo "All checks passed or were safely skipped." ================================================ FILE: .github/workflows/pgspot.yaml ================================================ # Test our extension sql scripts are following security best practices name: pgspot "on": pull_request: push: branches: - main - ?.*.x jobs: check_paths: runs-on: ubuntu-latest outputs: run_ci: >- ${{ github.event_name == 'push' || (steps.filter.outputs.sql == 'true' && !contains(github.event.pull_request.labels.*.name, 'skip-ci')) }} steps: - uses: actions/checkout@v4 - uses: dorny/paths-filter@v3 id: filter with: filters: .github/filters.yaml pgspot: needs: check_paths if: needs.check_paths.outputs.run_ci == 'true' runs-on: ubuntu-latest env: # policy_compression and policy_compression_execute do not have explicit search_path # because postgres does not allow it for procedures doing transaction control PGSPOT_OPTS: --ignore PS002 --proc-without-search-path '_timescaledb_internal.policy_compression(job_id integer,config jsonb)' --proc-without-search-path '_timescaledb_functions.policy_compression(job_id integer,config jsonb)' --proc-without-search-path '_timescaledb_functions.policy_compression_execute(job_id integer,htid integer,lag anyelement,maxchunks integer,verbose_log boolean,recompress_enabled boolean,reindex_enabled boolean,use_creation_time boolean)' steps: - name: Setup python 3.13 uses: actions/setup-python@v5 with: python-version: '3.13' - name: Checkout timescaledb uses: actions/checkout@v4 - name: Install pgspot run: | python -m pip install pgspot==0.9.2 - name: Build timescaledb sqlfiles run: | previous_version=$(grep '^previous_version = ' version.config | sed -e 's!^[^=]\+ = !!') git fetch --tags ./bootstrap -DGENERATE_DOWNGRADE_SCRIPT=ON git checkout ${previous_version} make -C build sqlfile sqlupdatescripts git checkout ${GITHUB_SHA} make -C build sqlfile sqlupdatescripts ls -la build/sql/timescaledb--*.sql - name: Run pgspot run: | version=$(grep '^version = ' version.config | sed -e 's!^[^=]\+ = !!') previous_version=$(grep '^previous_version = ' version.config | sed -e 's!^[^=]\+ = !!') # Show files used ls -la build/sql/timescaledb--${version}.sql build/sql/timescaledb--${previous_version}--${version}.sql \ build/sql/timescaledb--${version}--${previous_version}.sql # The next pgspot execution tests the installation script by itself pgspot ${{ env.PGSPOT_OPTS }} build/sql/timescaledb--${version}.sql # The next pgspot execution tests the update script to the latest version # we prepend the installation script here so pgspot can correctly keep track of created objects pgspot ${{ env.PGSPOT_OPTS }} -a build/sql/timescaledb--${previous_version}.sql \ build/sql/timescaledb--${previous_version}--${version}.sql # The next pgspot execution tests the downgrade script to the previous version # we prepend the installation script here so pgspot can correctly keep track of created objects pgspot ${{ env.PGSPOT_OPTS }} -a build/sql/timescaledb--${version}.sql \ build/sql/timescaledb--${version}--${previous_version}.sql ================================================ FILE: .github/workflows/pr-approvals.yaml ================================================ # All PRs except trivial ones should require 2 approvals, since this a global setting # we cannot make this decision from github configuration alone. So we set the required # approved in github to 1 and make this check required which will enforce 2 approvals # unless overwritten or only CI files are touched. name: PR Approval Check "on": pull_request: types: [opened, synchronize, reopened, edited, auto_merge_enabled, auto_merge_disabled] branches: [main] pull_request_review: jobs: check_approvals: name: Check for sufficient approvals runs-on: timescaledb-runner-arm64 steps: - name: Checkout source uses: actions/checkout@v4 - name: Check for sufficient approvals shell: bash --norc --noprofile {0} env: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} run: | set -eu echo "PR number is $PR_NUMBER" BODY=$(gh pr view $PR_NUMBER -q .body --json body) if ! echo "$BODY" | egrep -qsi '^disable-check:.*\<approval-count\>' then # Get the list of modified files in this pull request echo "Modified files: " gh pr view $PR_NUMBER --json files # Get the number of modified files, but exclude those that # are workflow files. These require only a single # reviewer. files=$(gh pr view $PR_NUMBER --json files --jq '[.files.[].path | select(startswith(".github") | not)] | length') # Get the number of approvals in this pull request echo "Reviews: " gh pr view $PR_NUMBER --json reviews approvals=$( gh pr view $PR_NUMBER --json reviews --jq ' [ .reviews.[] | select(.authorAssociation == "MEMBER" and .state == "APPROVED") ] | length ' ) echo "approvals: $approvals, files: $files" if [[ $approvals -lt 2 && $files -gt 0 ]] ; then echo "This pull request requires 2 approvals before merging." echo echo "For trivial changes, you may disable this check by adding this trailer to the pull request message:" echo echo "Disable-check: approval-count" echo exit 1 fi fi ================================================ FILE: .github/workflows/pr-handling.yaml ================================================ name: Assign PR to author and reviewers # This workflow runs on the pull_request_target event: # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target. # # So the secrets are accessible even if the PR was opened from an external # repository. This is done because the GitHub API key needs to be accessed # to modify the PRs. # # NOTE: Only API calls should be made in the actions defined here. The # committed code should _NOT_ be touched in any case. "on": pull_request_target: types: [ opened, reopened, ready_for_review ] jobs: assign-pr: name: Assign PR to author runs-on: ubuntu-latest steps: - uses: toshimaru/auto-author-assign@v2.1.0 ask-review: name: Run pull-review runs-on: ubuntu-latest if: ${{ !github.event.pull_request.draft && github.event.pull_request.base.ref == 'main' }} steps: - uses: actions/checkout@v4 - name: Run pull-review with docker env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} OWNER: ${{ github.repository_owner }} REPO: ${{ github.event.repository.name }} PULL_REQUEST_NUMBER : ${{ github.event.pull_request.number }} shell: bash run: | PR_URL="https://github.com/${OWNER}/${REPO}/pull/${PULL_REQUEST_NUMBER}" docker run ghcr.io/imsky/pull-review "$PR_URL" --github-token "${GITHUB_TOKEN}" ================================================ FILE: .github/workflows/pr-validation.yaml ================================================ name: Pull Request Validation "on": pull_request: types: [opened, synchronize, reopened, edited, auto_merge_enabled, auto_merge_disabled] branches: [main] jobs: # Count the number of commits in a pull request. This can be # disabled by adding a trailer line of the following form to the # pull request message: # # Disable-Check: commit-count # # The check is case-insensitive and ignores other contents on the # line as well, so it is possible to add several different checks if # that is necessary. # # It is assumed that the trailer is following RFC2822 conventions, # but this is currently not enforced. count_commits: name: Enforce single commit pull request runs-on: timescaledb-runner-arm64 steps: - name: Checkout source uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 0 - name: Dump GitHub context (for debugging) env: GITHUB_CONTEXT: ${{ toJSON(github) }} run: | echo "GITHUB_CONTEXT: $GITHUB_CONTEXT" - name: Check number of commits shell: bash --norc --noprofile {0} env: BODY: ${{ github.event.pull_request.body }} run: | echo "$BODY" | egrep -qsi '^disable-check:.*\<commit-count\>' if [[ $? -ne 0 ]] && [[ "${{ github.event.pull_request.auto_merge.merge_method }}" != "squash" ]] then base=${{ github.event.pull_request.base.sha }} head=${{ github.event.pull_request.head.sha }} count=`git rev-list --no-merges --count $base..$head` if [[ "$count" -ne 1 ]]; then echo "Found $count commits in pull request (there should be only one):" git log --format=format:'- %h %s' $base..$head echo echo "To disable commit count, add this trailer to pull request message:" echo echo "Disable-check: commit-count" echo echo "Trailers follow RFC2822 conventions, so no whitespace" echo "before field name and the check is case-insensitive for" echo "both the field name and the field body." exit 1 fi fi ================================================ FILE: .github/workflows/prerelease-sanity.yaml ================================================ name: Prerelease Sanity "on": push: branches: - prerelease_test pull_request: branches: "?.*.x" paths: - version.config - .github/workflows/prerelease-sanity.yaml workflow_dispatch: jobs: check_release_commit: name: Check Release Commit runs-on: timescaledb-runner-arm64 steps: - name: Checkout TimescaleDB uses: actions/checkout@v4 # GitHub creates an empty merge commit even for fast-fordward merges, which # makes it needlessly difficult to inspect the actual commit title. Since # we require the PRs to release branches to be up to date before merging, # we can just work with the PR head here. with: ref: ${{ github.event.pull_request.head.sha }} fetch-depth: 2 # The combined changelog must reference all changes, and the respective # change files in the .unreleased folder must be deleted. - name: No .unreleased files are left behind run: | ! compgen -G .unreleased/* # The messages of the release commit and tag must start with Release <version>. # If this is the release tag, it must point to the release commit # and not something else. - name: The release commit message references the respective version run: | required_title=$(sed -n "s/^version = /Release /p" version.config) echo $required_title tag_title=$(git log --oneline -1 --pretty=format:%s) echo $tag_title grep "$required_title" <<<"$tag_title" # Our reference might be a tag, so check the pointed-to commit as well, # using the ^0 git path specification to find it. commit_title=$(git log --oneline -1 --pretty=format:%s @^0) echo $commit_title grep "$required_title" <<<"$commit_title" # The release commit must modify the version.config - name: The release commit modifies the version.config run: | git log --oneline -1 @^0 git log --oneline -1 @^0~ ! git diff --exit-code @^0~ @^0 -- version.config ================================================ FILE: .github/workflows/release_build_packages.yml ================================================ name: "Packaging: Build distributions" "on": release: types: [published] jobs: update: runs-on: ubuntu-latest steps: - name: Initialize build summary run: | echo "# 📦 Distribution Package Build Summary" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "**Version:** \`${{ github.event.release.tag_name }}\`" >> $GITHUB_STEP_SUMMARY echo "**URL:** [${{ github.event.release.name }}](${{ github.event.release.html_url }})" >> $GITHUB_STEP_SUMMARY echo "**Triggered at:** $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "## 🚀 Build Status" >> $GITHUB_STEP_SUMMARY echo "| Package | Status | Workflow |" >> $GITHUB_STEP_SUMMARY echo "| --- | --- | --- |" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "HAS_FAILURES=false" >> $GITHUB_ENV - name: Build Docker Image env: GH_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | echo "🐳 Triggering Docker image build..." | tee -a build.log if gh workflow run docker-image.yml -R timescale/timescaledb-docker -f version=${{ github.event.release.tag_name }} -f tag_latest=true -f registry=prod; then echo "| 🐳 Docker Image | ✅ Triggered | [timescaledb-docker](https://github.com/timescale/timescaledb-docker/actions/workflows/docker-image.yml) |" >> $GITHUB_STEP_SUMMARY else echo "| 🐳 Docker Image | ❌ Failed | [timescaledb-docker](https://github.com/timescale/timescaledb-docker/actions/workflows/docker-image.yml) |" >> $GITHUB_STEP_SUMMARY echo "HAS_FAILURES=true" >> $GITHUB_ENV fi - name: Build APT Packages env: GH_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | echo "🐧 Triggering APT package build..." | tee -a build.log if gh workflow run timescaledb-apt.yml -R timescale/release-build-scripts -f version=${{ github.event.release.tag_name }} -f upload-artifacts=true; then echo "| 🐧 APT Packages | ✅ Triggered | [summary](https://github.com/timescale/release-build-scripts/actions/workflows/timescaledb-apt.yml) |" >> $GITHUB_STEP_SUMMARY else echo "| 🐧 APT Packages | ❌ Failed | [summary](https://github.com/timescale/release-build-scripts/actions/workflows/timescaledb-apt.yml) |" >> $GITHUB_STEP_SUMMARY echo "HAS_FAILURES=true" >> $GITHUB_ENV fi - name: Build RPM Packages env: GH_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | echo "⚙ Triggering RPM package build..." | tee -a build.log if gh workflow run timescaledb-rpm.yml -R timescale/release-build-scripts -f version=${{ github.event.release.tag_name }} -f upload-artifacts=true; then echo "| ⚙ RPM Packages | ✅ Triggered | [summary](https://github.com/timescale/release-build-scripts/actions/workflows/timescaledb-rpm.yml) |" >> $GITHUB_STEP_SUMMARY else echo "| ⚙ RPM Packages | ❌ Failed | [summary](https://github.com/timescale/release-build-scripts/actions/workflows/timescaledb-rpm.yml) |" >> $GITHUB_STEP_SUMMARY echo "HAS_FAILURES=true" >> $GITHUB_ENV fi - name: Build Homebrew Packages env: GH_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | echo "🍺 Triggering Homebrew package build..." | tee -a build.log if gh workflow run timescaledb-homebrew.yml -R timescale/release-build-scripts -f version=${{ github.event.release.tag_name }} -f upload-artifacts=true; then echo "| 🍺 Homebrew Packages | ✅ Triggered | [summary](https://github.com/timescale/release-build-scripts/actions/workflows/timescaledb-homebrew.yml) |" >> $GITHUB_STEP_SUMMARY else echo "| 🍺 Homebrew Packages | ❌ Failed | [summary](https://github.com/timescale/release-build-scripts/actions/workflows/timescaledb-homebrew.yml) |" >> $GITHUB_STEP_SUMMARY echo "HAS_FAILURES=true" >> $GITHUB_ENV fi - name: Build Windows Packages env: GH_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | echo "🪟 Triggering Windows package build..." | tee -a build.log if gh workflow run timescaledb-windows.yml -R timescale/release-build-scripts -f version=${{ github.event.release.tag_name }} -f upload-artifacts=true; then echo "| 🪟 Windows Packages | ✅ Triggered | [summary](https://github.com/timescale/release-build-scripts/actions/workflows/timescaledb-windows.yml) |" >> $GITHUB_STEP_SUMMARY else echo "| 🪟 Windows Packages | ❌ Failed | [summary](https://github.com/timescale/release-build-scripts/actions/workflows/timescaledb-windows.yml) |" >> $GITHUB_STEP_SUMMARY echo "HAS_FAILURES=true" >> $GITHUB_ENV fi - name: Generate final summary if: always() run: | echo "---" >> $GITHUB_STEP_SUMMARY echo "## 📋 Next Steps" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "1. Monitor the triggered workflows in their respective repositories" >> $GITHUB_STEP_SUMMARY echo "2. Check build artifacts once workflows complete" >> $GITHUB_STEP_SUMMARY echo "3. Verify package availability in distribution channels" >> $GITHUB_STEP_SUMMARY - name: Validate workflow dispatch status if: ${{ env.HAS_FAILURES == 'true' }} run: | echo "---" >> $GITHUB_STEP_SUMMARY echo "❌ one or more workflows could not get dispatched" >> $GITHUB_STEP_SUMMARY exit 1 ================================================ FILE: .github/workflows/release_feature_freeze_ceremony.yaml ================================================ name: Release - Feature Freeze Ceremony # # Feature Freeze # # prereqs: # 0. needs approval from another person to run the action (environment) # 1. ensure the correct branch is used to execute the action # a. if the run was triggered from the `main` branch, the NEXT_VERSION must be a new minor (end with a .0) # b. if NEXT_VERSION is a patch release, then the branch the action was triggered on must be a release branch: `^[0-9]+\.[0-9]+\.x$` # 2. ensure the next version does not exist as tag yet # # Tasks # 0. bump minor -dev version in version.config on main, if and only if it's a .0 release # 1. creates the release branch X.Y.x, if and if the branch does not exist yet # 2. generates CHANGELOG and opens PR for it # 3. generates the release artefacts for the next versions # 4. bumps the version in version.config to minor + 1 if and only if a .0 release is done # on: workflow_dispatch: # The workflow needs the permission to push branches permissions: contents: write pull-requests: write jobs: validate-conditions: name: Validate Run Conditions runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Check Release Conditions run: | NEXT_VERSION=$(head -1 version.config | cut -d ' ' -f 3 | cut -d '-' -f 1) RELEASE_BRANCH="${NEXT_VERSION/%.[0-9]/.x}" echo "Checking run conditions for release: $NEXT_VERSION" >> $GITHUB_STEP_SUMMARY echo "Current branch: ${{ github.ref_name }}" >> $GITHUB_STEP_SUMMARY echo "Expected release branch: $RELEASE_BRANCH" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY # Condition: version tag can not exist if git tag -l | grep -q "^$NEXT_VERSION$"; then echo "❌ \`$NEXT_VERSION\` tag already exists" >> $GITHUB_STEP_SUMMARY exit 1 fi # Condition: minor releases (.0) must be triggered from the main branch if [[ "$NEXT_VERSION" =~ \.0$ ]] && [[ "${{ github.ref_name }}" != "main" ]]; then echo "❌ for minor releases, \`$NEXT_VERSION\` the \`main\` branch is required, but got \`${{ github.ref_name }}\`" >> $GITHUB_STEP_SUMMARY exit 1 fi # Condition: patch releases must be triggered from the release branch if [[ ! "$NEXT_VERSION" =~ \.0$ ]] && [[ "${{ github.ref_name }}" != "$RELEASE_BRANCH" ]]; then echo "❌ the patch releases, \`$NEXT_VERSION\` the action must be started from the release branch \`$RELEASE_BRANCH\`, but got \`${{ github.ref_name }}\`" >> $GITHUB_STEP_SUMMARY exit 1 fi echo "✅ All release conditions satisfied for \`$NEXT_RELEASE\`" >> $GITHUB_STEP_SUMMARY release-feature-freeze-ceremony: name: Release - Feature Freeze runs-on: ubuntu-latest needs: validate-conditions environment: name: Release Ceremonies steps: - name: Checkout TimescaleDB uses: actions/checkout@v4 # git user config timescale-automation - run: | git config user.name "timescale-automation" git config user.email "123763385+github-actions[bot]@users.noreply.github.com" - name: Install Dependencies run: | sudo apt-get update sudo apt-get install pip pip install PyGithub requests # Read the next version and build the next versions - name: Set version configuration run: | NEXT_VERSION=$(head -1 version.config | cut -d ' ' -f 3 | cut -d '-' -f 1) CURRENT_VERSION=$(tail -1 version.config | cut -d ' ' -f 3 | cut -d '-' -f 1) RELEASE_BRANCH="${NEXT_VERSION/%.[0-9]/.x}" echo "RELEASE_BRANCH=${RELEASE_BRANCH}" >> $GITHUB_ENV echo "CURRENT_VERSION=${CURRENT_VERSION}" >> $GITHUB_ENV echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV echo "# 🚀 Release v${NEXT_VERSION}" >> $GITHUB_STEP_SUMMARY echo "Previous version: \`$CURRENT_VERSION\`" >> $GITHUB_STEP_SUMMARY echo "Release branch: \`$RELEASE_BRANCH\`" >> $GITHUB_STEP_SUMMARY echo "Workflow branch: \`${{ github.ref_name }}\`" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY # ------------------------------------------------------- # # Only running for new minor releases # # ------------------------------------------------------- # Bump minor in main for minor releases - uses: actions/checkout@v4 with: ref: main - name: Bump minor version on main if next release is a new minor if: endsWith(env.NEXT_VERSION, '.0') env: GITHUB_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | NEXT_MAIN_VERSION=$(head -1 version.config | cut -d ' ' -f 3 | cut -d '-' -f 1) NEXT_MAIN_VERSION=$(echo "$NEXT_MAIN_VERSION" | awk -F. '{printf "%d.%d.0-dev", $1, $2 + 1}') sed -i "s/^version = .*/version = $NEXT_MAIN_VERSION/" version.config BRANCH="release/bump-main-to-$NEXT_MAIN_VERSION" git checkout -b $BRANCH git add version.config git commit -m "Increase minor version $NEXT_MAIN_VERSION" git push origin $BRANCH PR_URL=$(gh pr create \ --title "Increase minor version $NEXT_MAIN_VERSION" \ --body "Bump to next minor for the upcoming ${{ env.CURRENT_VERSION }} release" \ --base main \ --head $BRANCH \ --label "release") echo "* ✅ minor version increase on main: $PR_URL" >> $GITHUB_STEP_SUMMARY # Create `2.XX.x` release branch of `main` # Release branch is a dependency for the next steps # only execute this step on .0 releases - name: Create Release Branch env: GITHUB_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | if git ls-remote --heads origin "${{ env.RELEASE_BRANCH }}" | grep -q "${{ env.RELEASE_BRANCH }}"; then echo "* ✓ the \`${{ env.RELEASE_BRANCH }}\` release branch exists" >> $GITHUB_STEP_SUMMARY else gh api repos/${{ github.repository }}/git/refs \ --method POST \ --field ref="refs/heads/${{ env.RELEASE_BRANCH }}" \ --field sha="${{ github.sha }}" echo "* ✅ the \`${{ env.RELEASE_BRANCH }}\` release branch created" >> $GITHUB_STEP_SUMMARY fi # ------------------------------------------------------- # # Runs for every release # # ------------------------------------------------------- # Generate the CHANGELOG and submit it in a separate PR - uses: actions/checkout@v4 with: ref: ${{ env.RELEASE_BRANCH }} - name: Generate and create Pull Request for CHANGELOG env: GITHUB_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | ./scripts/changelog/generate.sh BODY=$(head -n 19 CHANGELOG.md | tail -n 20) BRANCH="release/${{ env.NEXT_VERSION }}-changelog" git checkout -b $BRANCH git add .unreleased/* CHANGELOG.md git commit -m "CHANGELOG ${{ env.NEXT_VERSION }}" git push origin $BRANCH PR_URL=$(gh pr create \ --title "Release ${{ env.NEXT_VERSION }}" \ --body "$BODY" \ --base ${{ env.RELEASE_BRANCH }} \ --head $BRANCH \ --label "release,changelog") echo "* ✅ CHANGELOG: $PR_URL" >> $GITHUB_STEP_SUMMARY # Rework version.config and up & down grade scripts - uses: actions/checkout@v4 with: ref: ${{ env.RELEASE_BRANCH }} - name: Generate release artefacts env: GITHUB_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | ./scripts/release/build_release_artefacts.sh ${{ env.CURRENT_VERSION }} ${{ env.NEXT_VERSION }} BRANCH="release/${{ env.CURRENT_VERSION }}-${{ env.NEXT_VERSION }}" git checkout -b $BRANCH git add version.config sql/updates/*.sql sql/CMakeLists.txt git commit -m "release artefacts from ${{ env.CURRENT_VERSION }} to ${{ env.NEXT_VERSION }}" git push origin $BRANCH BODY="- CHANGELOG\n- set previous_version in version.config\n- adjust CMakeList.txt with up & down grade files" PR_URL=$(gh pr create \ --title "Release artefacts ${{ env.NEXT_VERSION }}" \ --body "${BODY//\\n/$\n}" \ --base ${{ env.RELEASE_BRANCH }} \ --head $BRANCH \ --label "release") echo "* ✅ Release artefacts: $PR_URL" >> $GITHUB_STEP_SUMMARY ================================================ FILE: .github/workflows/release_post_release_ceremony.yaml ================================================ name: Release - Post Release Ceremony # # Post Release Workflow # # 0. trigger the generate GUCs workflow in the docs repo # 1. generate the artefacts to forward port to main and open PR for it # 2. bump the next patch number + 1 on the release branch and open PR for it # "on": release: types: [published] # The workflow needs the permission to a PR permissions: contents: write pull-requests: write jobs: docs: name: Post Release Ceremonies runs-on: ubuntu-latest steps: - name: Generate list of GUCs for docs env: GITHUB_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | if gh workflow run tsdb-refresh-gucs-list.yaml -R timescale/docs -f tag=${{ github.event.release.tag_name }}; then echo "✅ Triggered generation of GUC list in docs" >> $GITHUB_STEP_SUMMARY else echo "❌ docs workflow to generate the GUCs failed" >> $GITHUB_STEP_SUMMARY fi - uses: actions/checkout@v4 with: ref: ${{ github.event.release.tag_name }} # git user config timescale-automation - run: | git config user.name "timescale-automation" git config user.email "123763385+github-actions[bot]@users.noreply.github.com" - name: Forward port changes to main env: GITHUB_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | ./scripts/release/build_post_release_artefacts.sh ${{ github.event.release.tag_name }} BRANCH="release/${{ github.event.release.tag_name }}-main" git checkout -b $BRANCH git add version.config sql/updates/*.sql sql/CMakeLists.txt .unreleased/* CHANGELOG.md git commit -m "forward port release artefacts of ${{ github.event.release.tag_name }} to main" git push origin $BRANCH printf '%b- CHANGELOG\n- set previous_version in version.config\n- adjust CMakeList.txt with up & down grade files\n- removed .unreleased files' > $BODY PR_URL=$(gh pr create \ --title "Forwardport ${{ github.event.release.tag_name }}" \ --body "$BODY" \ --base main \ --head $BRANCH \ --label "release") echo "✅ Forward ported release artefacts: $PR_URL" >> $GITHUB_STEP_SUMMARY - name: Get next versions run: | TAGGED_VERSION=$(head -1 version.config | cut -d ' ' -f 3 | cut -d '-' -f 1) if [[ "$TAGGED_VERSION" != "$${{ github.event.release.tag_name }}" ]]; then echo "The tag: ${{ github.event.release.tag_name }} and the release version ${TAGGED_VERSION} do not match." >&2 echo "❌ tagged version ${{ github.event.release.tag_name }} is not matching the current version $TAGGED_VERSION" >> $GITHUB_STEP_SUMMARY exit 1 fi RELEASE_BRANCH="${TAGGED_VERSION/%.d/.x}" NEXT_VERSION=$(echo "$TAGGED_VERSION" | awk -F. '{printf "%d.%d.%d", $1, $2, $3+1}') echo "TAGGED_VERSION=${TAGGED_VERSION}" >> $GITHUB_ENV echo "RELEASE_BRANCH=${RELEASE_BRANCH}" >> $GITHUB_ENV echo "NEXT_VERSION=${NEXT_VERSION}" >> $GITHUB_ENV - uses: actions/checkout@v4 with: ref: ${{ env.RELEASE_BRANCH }} - name: Bump to next version on release branch env: GITHUB_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | echo "Tagged version: ${{ env.TAGGED_VERSION }}" echo "Next version: ${{ env.NEXT_VERSION }}" # adjust version.config file sed -i \ -e "s/^version = .*/version = ${{ env.NEXT_VERSION }}/" \ -e "s/^previous_version = .*/previous_version = ${{ env.TAGGED_VERSION }}/" \ version.config echo "✅ setting next patch version ${{ env.TAGGED_VERSION }} in version.config" >> $GITHUB_STEP_SUMMARY cat version.config >> $GITHUB_STEP_SUMMARY BRANCH="release/bump-${{ github.event.release.tag_name }}-to-${{ env.NEXT_VERSION }}" git checkout -b $BRANCH git add version.config git commit -m "Bump to next patch version ${{ env.NEXT_VERSION }}" git push origin $BRANCH PR_URL=$(gh pr create \ --title "Forwardport ${{ github.event.release.tag_name }}" \ --body "Change to the next patch version ${{ env.NEXT_VERSION }} after the release of ${{ github.event.release.tag_name }}." \ --base $RELEASE_BRANCH \ --head $BRANCH \ --label "release") echo "✅ Bump to next patch version: $PR_URL" >> $GITHUB_STEP_SUMMARY ================================================ FILE: .github/workflows/rpm-packages.yaml ================================================ # Test rpm package installation for latest version name: "Packaging tests: RPM" "on": schedule: # run daily 0:00 on main branch - cron: '0 0 * * *' pull_request: paths: .github/workflows/rpm-packages.yaml push: tags: - '*' branches: - release_test - trigger/package_test workflow_dispatch: jobs: rpm_tests: name: RPM ${{ matrix.image }} PG${{ matrix.pg }} ${{ matrix.license }} runs-on: ubuntu-latest container: image: ${{ matrix.image }} strategy: fail-fast: false matrix: image: [ "rockylinux:8", "rockylinux:9" ] pg: [ 15, 16, 17, 18 ] license: [ "TSL", "Apache"] include: - license: Apache pkg_suffix: "-oss" exclude: - image: "rockylinux:8" pg: 18 steps: - name: Add postgres repositories run: "yum install -y \"https://download.postgresql.org/pub/repos/yum/reporpms/\ EL-$(rpm -E %{rhel})-x86_64/pgdg-redhat-repo-latest.noarch.rpm\"" - name: Add other repositories run: | tee /etc/yum.repos.d/timescale_timescaledb.repo <<EOL [timescale_timescaledb] name=timescale_timescaledb baseurl=https://packagecloud.io/timescale/timescaledb/el/$(rpm -E %{rhel})/\$basearch repo_gpgcheck=1 gpgcheck=0 enabled=1 gpgkey=https://packagecloud.io/timescale/timescaledb/gpgkey sslverify=1 sslcacert=/etc/pki/tls/certs/ca-bundle.crt metadata_expire=300 EOL - name: Install timescaledb run: | yum update -y if [[ "$(rpm -E %{rhel})" -eq "8" ]]; then dnf -qy module disable postgresql; fi yum install -y timescaledb-2${{ matrix.pkg_suffix }}-postgresql-${{ matrix.pg }} sudo wget jq sudo -u postgres /usr/pgsql-${{ matrix.pg }}/bin/initdb -D /var/lib/pgsql/${{ matrix.pg }}/data timescaledb-tune --quiet --yes --pg-config /usr/pgsql-${{ matrix.pg }}/bin/pg_config - name: List available versions run: | yum --showduplicates list timescaledb-2${{ matrix.pkg_suffix }}-postgresql-${{ matrix.pg }} - name: Show files in package run: | rpm -ql timescaledb-2${{ matrix.pkg_suffix }}-postgresql-${{ matrix.pg }} - uses: actions/checkout@v3 - name: Get version of latest release id: versions run: | version=$(wget -q https://api.github.com/repos/timescale/timescaledb/releases/latest -O - | jq -r .tag_name) echo "version=${version}" echo "version=${version}" >>$GITHUB_OUTPUT - name: Test Installation run: | sudo -u postgres /usr/pgsql-${{ matrix.pg }}/bin/pg_ctl -D /var/lib/pgsql/${{ matrix.pg }}/data start while ! /usr/pgsql-${{ matrix.pg }}/bin/pg_isready; do sleep 1; done sudo -u postgres psql -X -c "CREATE EXTENSION timescaledb" \ -c "SELECT extname,extversion,version() FROM pg_extension WHERE extname='timescaledb';" installed_version=$(sudo -u postgres psql -X -t \ -c "SELECT extversion FROM pg_extension WHERE extname='timescaledb';" | sed -e 's! !!g') if [ "${{ steps.versions.outputs.version }}" != "$installed_version" ];then false fi - name: Test Downgrade if: matrix.pg != '18' # pg18 only has 1 version so downgrade is not possible run: | # Since this runs nightly on main we have to get the previous version # from the last released version and not current branch. prev_version=$(wget --quiet -O - \ https://raw.githubusercontent.com/timescale/timescaledb/${{ steps.versions.outputs.version }}/version.config \ | grep previous_version | sed -e 's!previous_version = !!') yum downgrade -y timescaledb-2${{ matrix.pkg_suffix }}-postgresql-${{ matrix.pg }}-${prev_version} sudo -u postgres psql -X -c "ALTER EXTENSION timescaledb UPDATE TO '${prev_version}'" \ -c "SELECT extname,extversion,version() FROM pg_extension WHERE extname='timescaledb';" installed_version=$(sudo -u postgres psql -X -t \ -c "SELECT extversion FROM pg_extension WHERE extname='timescaledb';" | sed -e 's! !!g') if [ "$prev_version" != "$installed_version" ];then false fi - name: Install toolkit if: matrix.pg != 17 && matrix.image != 'rockylinux:9' # timescaledb-toolkit currently not available for PG17 and Rocky9 run: | yum install -y timescaledb-toolkit-postgresql-${{ matrix.pg }} - name: List available toolkit versions if: matrix.pg != 17 && matrix.image != 'rockylinux:9' # timescaledb-toolkit currently not available for PG17 and Rocky9 run: | yum --showduplicates list timescaledb-toolkit-postgresql-${{ matrix.pg }} ================================================ FILE: .github/workflows/sanitizer-build-and-test.yaml ================================================ # Run regression tests under memory sanitizer name: Sanitizer test "on": schedule: # run daily 0:00 on main branch - cron: '0 0 * * *' push: branches: - main - ?.*.x - trigger/sanitizer pull_request: paths: .github/workflows/sanitizer-build-and-test.yaml workflow_dispatch: env: name: "Sanitizer" PG_SRC_DIR: "pgbuild" PG_INSTALL_DIR: "postgresql" extra_packages: "clang-15 llvm-15 llvm-15-dev llvm-15-tools" llvm_config: "llvm-config-15" CLANG: "clang-15" CC: "clang-15" CXX: "clang-15" # gcc CFLAGS, disable inlining for function name pattern matching to work for suppressions # CFLAGS: "-g -fsanitize=address,undefined -fno-omit-frame-pointer -O1 -fno-inline" # CXXFLAGS: "-g -fsanitize=address,undefined -fno-omit-frame-pointer -O1 -fno-inline" # clang CFLAGS CFLAGS: "-g -fsanitize=address,undefined -fno-omit-frame-pointer -Og -fno-inline-functions" CXXFLAGS: "-g -fsanitize=address,undefined -fno-omit-frame-pointer -Og -fno-inline-functions" # We do not link libasan dynamically to avoid problems with libdl and our libraries. # clang does this by default, but we need to explicitly state that for gcc. # static gcc LDFLAGS # LDFLAGS: "-fsanitize=address,undefined -static-libasan -static-liblsan -static-libubsan" # static sanitizer clang LDFLAGS or dynamic sanitizer gcc LDFLAGS LDFLAGS: "-fsanitize=address,undefined" ASAN_OPTIONS: suppressions=${{ github.workspace }}/scripts/suppressions/suppr_asan.txt detect_odr_violation=0 log_path=${{ github.workspace }}/sanitizer_logs/sanitizer log_exe_name=true print_suppressions=false exitcode=27 detect_leaks=0 abort_on_error=1 LSAN_OPTIONS: suppressions=${{ github.workspace }}/scripts/suppressions/suppr_leak.txt print_suppressions=0 log_path=${{ github.workspace }}/sanitizer_logs/sanitizer log_exe_name=true print_suppressions=false exitcode=27 UBSAN_OPTIONS: suppressions=${{ github.workspace }}/scripts/suppressions/suppr_ub.txt print_stacktrace=1 halt_on_error=1 log_path=${{ github.workspace }}/sanitizer_logs/sanitizer log_exe_name=true print_suppressions=false exitcode=27 IGNORES: >- bgw_db_scheduler bgw_db_scheduler_fixed merge_append_partially_compressed net telemetry EXTENSIONS: "postgres_fdw test_decoding" jobs: config: runs-on: ubuntu-latest outputs: pg_latest: ${{ steps.setter.outputs.PG_LATEST }} steps: - name: Checkout source code uses: actions/checkout@v4 - name: Read configuration id: setter run: python .github/gh_config_reader.py sanitizer: # Change the JOB_NAME variable below when changing the name. # Don't use the env variable here because the env context is not accessible. name: PG${{ matrix.pg }} Sanitizer ${{ matrix.os }} runs-on: ${{ matrix.os }} needs: config strategy: fail-fast: false matrix: # "os" has to be in the matrix due to a bug in "env": https://github.community/t/how-to-use-env-context/16975 os: ["ubuntu-22.04"] pg: ${{ fromJson(needs.config.outputs.pg_latest) }} steps: - name: Install Linux Dependencies run: | sudo apt-get update sudo apt-get install flex bison lcov systemd-coredump gdb libipc-run-perl \ libtest-most-perl jq ${{ env.extra_packages }} - name: Checkout TimescaleDB uses: actions/checkout@v4 # We are going to rebuild Postgres daily, so that it doesn't suddenly break # ages after the original problem. - name: Get date for build caching id: get-date run: | echo "date=$(date +"%d")" >> $GITHUB_OUTPUT # Create a directory for sanitizer logs. This directory is referenced by # ASAN_OPTIONS, LSAN_OPTIONS, and UBSAN_OPTIONS - name: Create sanitizer log directory run: | mkdir ${{ github.workspace }}/sanitizer_logs # we cache the build directory instead of the install directory here # because extension installation will write files to install directory # leading to a tainted cache - name: Cache PostgreSQL ${{ matrix.pg }} id: cache-postgresql uses: actions/cache@v4 with: path: ~/${{ env.PG_SRC_DIR }} key: "${{ matrix.os }}-${{ env.name }}-postgresql-${{ matrix.pg }}-${{ env.CC }}\ -${{ steps.get-date.outputs.date }}-${{ hashFiles('.github/**') }}" - name: Build PostgreSQL ${{ matrix.pg }} if not in cache if: steps.cache-postgresql.outputs.cache-hit != 'true' run: | wget -q -O postgresql.tar.bz2 \ https://ftp.postgresql.org/pub/source/v${{ matrix.pg }}/postgresql-${{ matrix.pg }}.tar.bz2 mkdir -p ~/$PG_SRC_DIR tar --extract --file postgresql.tar.bz2 --directory ~/$PG_SRC_DIR --strip-components 1 # Add instrumentation to the Postgres memory contexts. For more details, see # https://github.com/timescale/eng-database/wiki/Using-Address-Sanitizer#adding-more-instrumentation PG_MAJOR=$(echo "${{ matrix.pg }}" | sed -e 's![.].*!!') if [ ${PG_MAJOR} -lt 18 ]; then patch -F5 -p1 -d ~/$PG_SRC_DIR < test/postgres-asan-instrumentation.patch else patch -F5 -p1 -d ~/$PG_SRC_DIR < test/postgres-asan-instrumentation-PG18GE.patch fi cd ~/$PG_SRC_DIR ./configure --prefix=$HOME/$PG_INSTALL_DIR --enable-debug --enable-cassert \ --with-openssl --without-readline --without-zlib --without-libxml make -j$(getconf _NPROCESSORS_ONLN) for ext in ${EXTENSIONS}; do make -j$(getconf _NPROCESSORS_ONLN) -C contrib/${ext} done - name: Upload config.log if: always() && steps.cache-postgresql.outputs.cache-hit != 'true' uses: actions/upload-artifact@v4 with: name: config.log for PostgreSQL ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }} path: ~/${{ env.PG_SRC_DIR }}/config.log - name: Install PostgreSQL ${{ matrix.pg }} run: | cd ~/$PG_SRC_DIR make install for ext in ${EXTENSIONS}; do make -C contrib/${ext} install done ~/$PG_INSTALL_DIR/bin/pg_config --version - name: Build TimescaleDB run: | ./bootstrap -DCMAKE_BUILD_TYPE=Debug -DPG_SOURCE_DIR=~/$PG_SRC_DIR \ -DPG_PATH=~/$PG_INSTALL_DIR -DCODECOVERAGE=OFF -DREQUIRE_ALL_TESTS=ON \ -DTEST_GROUP_SIZE=5 -DTEST_PG_LOG_DIRECTORY="$(readlink -f .)" -DTEST_TIMEOUT=240 make -j$(getconf _NPROCESSORS_ONLN) -C build make -C build install - name: make installcheck run: | set -o pipefail # IGNORE some test since they fail under ASAN. make -k -C build installcheck IGNORES="${IGNORES}" \ PSQL="${HOME}/${PG_INSTALL_DIR}/bin/psql" | tee installcheck.log - name: Show regression diffs if: always() id: collectlogs run: | find . -name regression.diffs -exec cat {} + > regression.log if [[ "${{ runner.os }}" == "Linux" ]] ; then # wait in case there are in-progress coredumps sleep 10 if coredumpctl -q list >/dev/null; then echo "coredumps=true" >>$GITHUB_OUTPUT; fi # print OOM killer information sudo journalctl --system -q --facility=kern --grep "Killed process" || true fi if [[ -s regression.log ]]; then echo "regression_diff=true" >>$GITHUB_OUTPUT; fi grep -e 'FAILED' -e 'failed (ignored)' -e 'not ok' installcheck.log || true cat regression.log - name: Save regression diffs if: always() && steps.collectlogs.outputs.regression_diff == 'true' uses: actions/upload-artifact@v4 with: name: Regression diff ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }} path: | regression.log installcheck.log - name: Save PostgreSQL log if: always() uses: actions/upload-artifact@v4 with: name: PostgreSQL log ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }} path: postmaster.* - name: Stack trace if: always() && steps.collectlogs.outputs.coredumps == 'true' run: | sudo coredumpctl gdb <<<" set verbose on set trace-commands on show debug-file-directory printf "'"'"query = '%s'\n\n"'"'", debug_query_string frame function ExceptionalCondition printf "'"'"condition = '%s'\n"'"'", conditionName up 1 l info args info locals bt full " 2>&1 | tee stacktrace.log ./scripts/bundle_coredumps.sh - name: Show sanitizer logs if: always() run: | tail -vn +1 ${{ github.workspace }}/sanitizer_logs/sanitizer* || : - name: Coredumps if: always() && steps.collectlogs.outputs.coredumps == 'true' uses: actions/upload-artifact@v4 with: name: Coredumps ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }} path: coredumps - name: Upload sanitizer logs if: always() uses: actions/upload-artifact@v4 with: name: sanitizer logs ${{ matrix.os }} ${{ env.name }} ${{ matrix.pg }} # The log_path sanitizer option means "Write logs to 'log_path.pid'". # https://github.com/google/sanitizers/wiki/SanitizerCommonFlags path: ${{ github.workspace }}/sanitizer_logs/* - name: Upload test results to the database if: always() env: # GitHub Actions allow you neither to use the env context for the job name, # nor to access the job name from the step context, so we have to # duplicate it to work around this nonsense. JOB_NAME: PG${{ matrix.pg }} ${{ env.name }} ${{ matrix.os }} CI_STATS_DB: ${{ secrets.CI_STATS_DB }} GITHUB_EVENT_NAME: ${{ github.event_name }} GITHUB_REF_NAME: ${{ github.ref_name }} GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} GITHUB_RUN_ID: ${{ github.run_id }} GITHUB_RUN_NUMBER: ${{ github.run_number }} JOB_STATUS: ${{ job.status }} run: | if [[ "${{ github.event_name }}" == "pull_request" ]] ; then GITHUB_PR_NUMBER="${{ github.event.number }}" else GITHUB_PR_NUMBER=0 fi export GITHUB_PR_NUMBER scripts/upload_ci_stats.sh ================================================ FILE: .github/workflows/shellcheck.yaml ================================================ # Test our shell scripts for bugs name: Shellcheck "on": pull_request: push: branches: - main - ?.*.x jobs: check_paths: runs-on: ubuntu-latest outputs: run_ci: >- ${{ github.event_name == 'push' || (steps.filter.outputs.shell == 'true' && !contains(github.event.pull_request.labels.*.name, 'skip-ci')) }} steps: - uses: actions/checkout@v4 - uses: dorny/paths-filter@v3 id: filter with: filters: .github/filters.yaml shellcheck: name: Shellcheck needs: [check_paths] if: needs.check_paths.outputs.run_ci == 'true' runs-on: ubuntu-latest steps: - name: Install Dependencies run: | sudo apt-get update sudo apt-get install shellcheck - name: Checkout TimescaleDB uses: actions/checkout@v4 with: fetch-depth: 0 - name: Run shellcheck run: scripts/shellcheck-ci.sh report_status: name: Shellcheck summary needs: [check_paths, shellcheck] if: always() runs-on: ubuntu-latest steps: - run: | # If the detector failed, or the matrix failed, exit with error if [[ "${{ needs.shellcheck.result }}" == "failure" ]]; then exit 1 fi echo "All checks passed or were safely skipped." ================================================ FILE: .github/workflows/snapshot-abi.yaml ================================================ # Test ABI versions against snapshot # # name: ABI Test Against Snapshot "on": schedule: # run daily 20:00 on main branch - cron: '0 20 * * *' push: branches: - ?.*.x - trigger/snapshot-abi pull_request: paths: .github/workflows/snapshot-abi.yaml workflow_dispatch: jobs: config: runs-on: ubuntu-latest outputs: pg15_abi_min: ${{ steps.config.outputs.pg15_abi_min }} pg16_abi_min: ${{ steps.config.outputs.pg16_abi_min }} pg17_abi_min: ${{ steps.config.outputs.pg17_abi_min }} pg18_abi_min: ${{ steps.config.outputs.pg18_abi_min }} steps: - name: Checkout source code uses: actions/checkout@v4 - name: Read configuration id: config run: python .github/gh_config_reader.py abi_snapshot_test: name: ABI Snapshot Test PG${{ matrix.pg }} runs-on: ubuntu-latest needs: config env: PG_SRC_DIR: pgbuild PG_INSTALL_DIR: postgresql PG_EXTENSIONS: postgres_fdw test_decoding strategy: fail-fast: false matrix: pg: [ 15, 16, 17, 18 ] ignores: - 'net telemetry bgw_launcher' include: - pg: 15 abi_min: ${{ fromJson(needs.config.outputs.pg15_abi_min) }} - pg: 16 abi_min: ${{ fromJson(needs.config.outputs.pg16_abi_min) }} - pg: 17 abi_min: ${{ fromJson(needs.config.outputs.pg17_abi_min) }} - pg: 18 abi_min: ${{ fromJson(needs.config.outputs.pg18_abi_min) }} steps: - name: Install Linux Dependencies run: | # Don't add ddebs here because the ddebs mirror is always 503 Service Unavailable. # If needed, install them before opening the core dump. sudo apt-get update sudo apt-get install flex bison lcov systemd-coredump gdb libipc-run-perl \ libtest-most-perl pkgconf icu-devtools clang-14 llvm-14 llvm-14-dev llvm-14-tools cmake - name: Checkout TimescaleDB uses: actions/checkout@v4 - name: Download Postgres ${{ matrix.pg }} run: | wget -q -O postgresql.tar.bz2 \ https://ftp.postgresql.org/pub/source/v${{ matrix.abi_min }}/postgresql-${{ matrix.abi_min }}.tar.bz2 mkdir -p ~/$PG_SRC_DIR tar --extract --file postgresql.tar.bz2 --directory ~/$PG_SRC_DIR --strip-components 1 cd ~/$PG_SRC_DIR ./configure --prefix=$HOME/$PG_INSTALL_DIR --with-openssl \ --without-readline --without-zlib --without-libxml --without-llvm make -j $(getconf _NPROCESSORS_ONLN) for ext in $PG_EXTENSIONS; do make -j $(getconf _NPROCESSORS_ONLN) -C contrib/${ext} done - name: Install postgresql ${{ matrix.pg }} run: | cd ~/$PG_SRC_DIR make install for ext in $PG_EXTENSIONS; do make -C contrib/${ext} install done echo "$HOME/$PG_INSTALL_DIR/bin" >> "${GITHUB_PATH}" - name: Build TimescaleDB run: | ./bootstrap -DCMAKE_BUILD_TYPE=Debug \ -DPG_SOURCE_DIR=~/$PG_SRC_DIR -DPG_PATH=~/$PG_INSTALL_DIR \ -DWARNINGS_AS_ERRORS=OFF -DREQUIRE_ALL_TESTS=ON \ -DTEST_PG_LOG_DIRECTORY="$(readlink -f .)" make -j $(getconf _NPROCESSORS_ONLN) -C build make -C build install mkdir -p build/install_ext build/install_lib cp `pg_config --sharedir`/extension/timescaledb*.{control,sql} build/install_ext cp `pg_config --pkglibdir`/timescaledb*.so build/install_lib - name: Download Postgres ${{ matrix.pg }}-snapshot run: | wget -q -O postgresql.tar.bz2 \ https://ftp.postgresql.org/pub/snapshot/${{ matrix.pg }}/postgresql-${{ matrix.pg }}-snapshot.tar.bz2 mkdir -p ~/$PG_SRC_DIR-snapshot tar --extract --file postgresql.tar.bz2 --directory ~/$PG_SRC_DIR-snapshot --strip-components 1 cd ~/$PG_SRC_DIR-snapshot ./configure --prefix=$HOME/$PG_INSTALL_DIR-snapshot --with-openssl \ --without-readline --without-zlib --without-libxml --without-llvm make -j $(getconf _NPROCESSORS_ONLN) for ext in $PG_EXTENSIONS; do make -j $(getconf _NPROCESSORS_ONLN) -C contrib/${ext} done - name: Install PostgreSQL ${{ matrix.pg }}-snapshot run: | cd ~/$PG_SRC_DIR-snapshot make install for ext in $PG_EXTENSIONS; do make -C contrib/${ext} install done echo "$HOME/$PG_INSTALL_DIR-snapshot/bin" >> "${GITHUB_PATH}" - name: Copy extension files to postgresql ${{ matrix.pg }}-snapshot run: | BUILD_DIR=build_snapshot ./bootstrap -DCMAKE_BUILD_TYPE=Debug \ -DPG_SOURCE_DIR=~/$PG_SRC_DIR-snapshot -DPG_PATH=~/$PG_INSTALL_DIR-snapshot \ -DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON \ -DTEST_PG_LOG_DIRECTORY="$(readlink -f .)" cp build/install_ext/* `pg_config --sharedir`/extension/ cp build/install_lib/* `pg_config --pkglibdir` - name: make regresscheck id: regresscheck run: | set -o pipefail make -k -C build_snapshot installcheck IGNORES="${{ matrix.ignores }}" | tee installcheck.log - name: Show regression diffs if: always() id: collectlogs run: | find . -name regression.diffs -exec cat {} + > regression.log if [[ "${{ runner.os }}" == "Linux" ]] ; then # wait in case there are in-progress coredumps sleep 10 if coredumpctl -q list >/dev/null; then echo "coredumps=true" >>$GITHUB_OUTPUT; fi # print OOM killer information sudo journalctl --system -q --facility=kern --grep "Killed process" || true elif [[ "${{ runner.os }}" == "macOS" ]] ; then if [ $(find /cores -type f | wc -l) -gt 0 ]; then echo "coredumps=true" >>$GITHUB_OUTPUT; fi fi if [[ -s regression.log ]]; then echo "regression_diff=true" >>$GITHUB_OUTPUT; fi grep -e 'FAILED' -e 'failed (ignored)' -e 'not ok' installcheck.log || true cat regression.log - name: Save regression diffs if: always() && steps.collectlogs.outputs.regression_diff == 'true' uses: actions/upload-artifact@v4 with: name: Regression diff Snapshot ABI Breakage PG${{ matrix.pg }} path: regression.log - name: Save postmaster.log if: always() uses: actions/upload-artifact@v4 with: name: PostgreSQL log Snapshot ABI Breakage PG${{ matrix.pg }} path: postmaster.log ================================================ FILE: .github/workflows/sqlsmith.yaml ================================================ name: SQLsmith "on": schedule: # run daily 2:00 on main branch - cron: '0 2 * * *' workflow_dispatch: push: branches: - sqlsmith - main - ?.*.x pull_request: paths: .github/workflows/sqlsmith.yaml jobs: sqlsmith: # Change the JOB_NAME variable below when changing the name. # Don't use the env variable here because the env context is not accessible. name: SQLsmith PG${{ matrix.pg }} runs-on: ${{ matrix.os }} strategy: matrix: os: ["ubuntu-22.04"] pg: [ "15", "16", "17", "18" ] build_type: ["Debug"] fail-fast: false env: PG_SRC_DIR: pgbuild PG_INSTALL_DIR: postgresql JOB_NAME: SQLsmith PG${{ matrix.pg }} steps: - name: Install Dependencies run: | sudo apt-get update sudo apt-get install gnupg systemd-coredump gdb postgresql-common \ build-essential autoconf autoconf-archive \ libboost-regex-dev libsqlite3-dev jq yes | sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh sudo apt-get purge postgresql* sudo apt-get install libpqxx-dev postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} # Make sure the system postgres service is not running. sudo systemctl mask postgresql sudo systemctl stop postgresql { psql -c "select 1" && exit 1 ; } || : - name: Checkout TimescaleDB uses: actions/checkout@v4 - name: Build TimescaleDB run: | ./bootstrap -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DWARNINGS_AS_ERRORS=OFF make -j$(getconf _NPROCESSORS_ONLN) -C build sudo make -C build install - name: Checkout sqlsmith uses: actions/checkout@v4 with: repository: 'timescale/sqlsmith' path: 'sqlsmith' ref: 'timescaledb' - name: Build SQLsmith run: | cd sqlsmith autoreconf -i ./configure make -j$(getconf _NPROCESSORS_ONLN) - name: Setup test environment run: | mkdir ~/pgdata /usr/lib/postgresql/${{ matrix.pg }}/bin/pg_ctl initdb -D ~/pgdata /usr/lib/postgresql/${{ matrix.pg }}/bin/pg_ctl -D ~/pgdata start \ -o "-cshared_preload_libraries=timescaledb" -o "-cmax_connections=200" \ -o "-cmax_prepared_transactions=100" -o "-cunix_socket_directories=/tmp" \ -o "-clog_statement=all" -o "-clogging_collector=true" \ -o "-clog_destination=jsonlog,stderr" -o "-clog_directory=$(readlink -f .)" \ -o "-clog_error_verbosity=verbose" -o "-clog_filename=postmaster.log" psql -h /tmp postgres -c 'CREATE DATABASE smith;' psql -h /tmp smith -c 'CREATE EXTENSION timescaledb;' psql -h /tmp smith -c '\i ${{ github.workspace }}/tsl/test/shared/sql/include/shared_setup.sql' psql -h /tmp smith -c '\i ${{ github.workspace }}/tsl/test/shared/sql/include/cagg_compression_setup.sql' # we run these in a loop to reinitialize the random number generator # 10000 queries seems to take roughly 4 minutes in CI # so total runtime should be around 200 minutes in nightly run and 40 minutes otherwise - name: Run SQLsmith run: | set -xeu set -o pipefail if [ "${{ github.event_name }}" == "schedule" ]; then LOOPS=20 else LOOPS=10 fi cd sqlsmith for _ in $(seq 1 $LOOPS) do ./sqlsmith --seed=$((16#$(openssl rand -hex 3))) --exclude-catalog \ --target="host=/tmp dbname=smith" --max-queries=10000 \ 2>&1 | tee -a sqlsmith.log psql "host=/tmp dbname=smith" -c "select 1" done - name: Check for coredumps if: always() id: collectlogs run: | # wait for in progress coredumps sleep 10 if coredumpctl list; then echo "coredumps=true" >>$GITHUB_OUTPUT false fi - name: Stack trace if: always() && steps.collectlogs.outputs.coredumps == 'true' run: | sudo coredumpctl gdb <<<" set verbose on set trace-commands on show debug-file-directory printf "'"'"query = '%s'\n\n"'"'", (char *) debug_query_string frame function ExceptionalCondition printf "'"'"condition = '%s'\n"'"'", (char *) conditionName up 1 l info args info locals bt full " 2>&1 | tee stacktrace.log ./scripts/bundle_coredumps.sh false - name: Upload Coredumps if: always() && steps.collectlogs.outputs.coredumps == 'true' uses: actions/upload-artifact@v4 with: name: Coredumps sqlsmith ${{ matrix.os }} PG${{ matrix.pg }} path: coredumps - name: Save PostgreSQL log if: always() && steps.collectlogs.outputs.coredumps == 'true' uses: actions/upload-artifact@v4 with: name: PostgreSQL log for PG${{ matrix.pg }} path: postgres.* - name: Upload test results to the database if: always() env: CI_STATS_DB: ${{ secrets.CI_STATS_DB }} GITHUB_EVENT_NAME: ${{ github.event_name }} GITHUB_REF_NAME: ${{ github.ref_name }} GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} GITHUB_RUN_ID: ${{ github.run_id }} GITHUB_RUN_NUMBER: ${{ github.run_number }} JOB_STATUS: ${{ job.status }} run: | if [[ "${{ github.event_name }}" == "pull_request" ]] ; then GITHUB_PR_NUMBER="${{ github.event.number }}" else GITHUB_PR_NUMBER=0 fi export GITHUB_PR_NUMBER scripts/upload_ci_stats.sh ================================================ FILE: .github/workflows/stalebot.yaml ================================================ name: 'Close stale issues and PRs' "on": schedule: - cron: '30 1 * * *' workflow_dispatch: pull_request: paths: .github/workflows/stalebot.yaml jobs: stale: runs-on: ubuntu-latest steps: - uses: actions/stale@v10 with: stale-issue-message: " Dear Author, \n\n This issue has been automatically marked as stale due to lack of activity. With only the issue description that is currently provided, we do not have enough information to take action. If you have or find the answers we would need, please reach out. Otherwise, this issue will be closed in 30 days. \n\n Thank you! " close-issue-message: " Dear Author, \n\n We are closing this issue due to lack of activity. Feel free to add a comment to this issue if you can provide more information and we will re-open it. \n\n Thank you! " stale-pr-message: " This pull request has been automatically marked as stale due to lack of activity. This pull request will be closed in 30 days. " close-pr-message: " We are closing this pull request due to lack of activity. " days-before-stale: 30 days-before-close: 30 stale-issue-label: 'no-activity' stale-pr-label: 'no-activity' close-issue-label: 'closed-by-bot' close-pr-label: 'closed-by-bot' # Process only issues that contain the label 'waiting-for-author' only-issue-labels: 'waiting-for-author' # Exclude issues with the 'in-progress' label exempt-issue-labels: 'in-progress' exempt-pr-labels: 'in-progress' operations-per-run: 100 ================================================ FILE: .github/workflows/tests-fail-on-old-code.yaml ================================================ # This workflow verifies that new tests fail when run against old code. # It helps ensure tests are actually testing the new functionality. name: Tests should fail on old code "on": pull_request: jobs: verify-tests-fail: name: Verify tests fail on old code runs-on: timescaledb-runner-arm64 env: PG_SRC_DIR: pgbuild PG_INSTALL_DIR: postgresql CODE_PATHS: src/ tsl/src/ test/src tsl/test/src steps: - name: Checkout TimescaleDB uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set PostgreSQL version id: pg run: | MAJOR=17 echo "major=$MAJOR" >> $GITHUB_OUTPUT # Choose the latest version from config that matches our major version. # This weird quote handling in sed is to remove the quotes around the # value that the config reader produces, i.e. version="17.7". GITHUB_OUTPUT=/dev/stdout .github/gh_config_reader.py \ | sed -n "s/PG${MAJOR}_LATEST=\"\([^\"]*\)\"/version=\1/p" \ >> $GITHUB_OUTPUT - name: Check for code and test changes id: check run: | git fetch origin ${{ github.base_ref }}:base echo "Looking at PR event ref $(git rev-parse @) vs base $(git rev-parse base)" # Code changes: anything under src/ directories readarray -t CODE_CHANGES < <(git diff --name-only base HEAD -- $CODE_PATHS) if ! ((${#CODE_CHANGES[@]})) then echo "No code changes found, skipping" echo "should_run=false" >> $GITHUB_OUTPUT exit 0 fi # Test changes: changed .out files under test/expected directories. # For versioned test outputs, we should only check the version that # matches the Postgers version we build. readarray -t CHANGED_TESTS < <( \ git diff --name-only base HEAD -- test/expected test/isolation/expected \ tsl/test/expected tsl/test/isolation/expected tsl/test/shared/expected \ | sed -n 's!^.*expected/\([^/ -]\+\(-${{ steps.pg.outputs.major }}\)\?\)\.out$!\1!gp') if ! ((${#CHANGED_TESTS[@]})) then echo "No test output changes found, skipping" echo "should_run=false" >> $GITHUB_OUTPUT exit 0 fi echo "Changed tests: ${CHANGED_TESTS[@]}" echo "should_run=true" >> $GITHUB_OUTPUT echo "changed_tests=${CHANGED_TESTS[@]}" >> $GITHUB_OUTPUT - name: Install Linux Dependencies if: steps.check.outputs.should_run == 'true' run: | sudo apt-get update sudo apt-get install flex bison cmake # We are going to rebuild Postgres daily, so that it doesn't suddenly break # ages after the original problem. - name: Get date for build caching if: steps.check.outputs.should_run == 'true' id: get-date run: | echo "date=$(date +"%d")" >> $GITHUB_OUTPUT # we cache the build directory instead of the install directory here # because extension installation will write files to install directory # leading to a tainted cache - name: Cache PostgreSQL ${{ steps.pg.outputs.version }} if: steps.check.outputs.should_run == 'true' id: cache-postgresql uses: actions/cache@v4 with: path: ~/${{ env.PG_SRC_DIR }} key: "pg-${{ steps.pg.outputs.version }}-${{ runner.os }}-${{ runner.arch }}-${{ steps.get-date.outputs.date }}-${{ hashFiles('.github/**') }}" - name: Build PostgreSQL ${{ steps.pg.outputs.version }} if: steps.check.outputs.should_run == 'true' && steps.cache-postgresql.outputs.cache-hit != 'true' run: | wget -q -O postgresql.tar.bz2 \ https://ftp.postgresql.org/pub/source/v${{ steps.pg.outputs.version }}/postgresql-${{ steps.pg.outputs.version }}.tar.bz2 mkdir -p ~/$PG_SRC_DIR tar --extract --file postgresql.tar.bz2 --directory ~/$PG_SRC_DIR --strip-components 1 cd ~/$PG_SRC_DIR ./configure --prefix=$HOME/$PG_INSTALL_DIR --with-openssl \ --without-readline --without-zlib --without-libxml make -j $(getconf _NPROCESSORS_ONLN) - name: Install PostgreSQL ${{ steps.pg.outputs.version }} if: steps.check.outputs.should_run == 'true' run: | cd ~/$PG_SRC_DIR make install echo "$HOME/$PG_INSTALL_DIR/bin" >> $GITHUB_PATH - name: Revert code changes if: steps.check.outputs.should_run == 'true' run: | git checkout base -- $CODE_PATHS git status - name: Build TimescaleDB if: steps.check.outputs.should_run == 'true' run: | cmake -B build -S . \ -DCMAKE_BUILD_TYPE=Debug \ -DPG_SOURCE_DIR=$HOME/$PG_SRC_DIR \ -DPG_PATH=$HOME/$PG_INSTALL_DIR make -j $(getconf _NPROCESSORS_ONLN) -C build make -C build install - name: make installcheck if: steps.check.outputs.should_run == 'true' run: | TESTS="${{ steps.check.outputs.changed_tests }}" echo "Running tests: $TESTS" # Run all changed tests together, allow failure make -C build -k installcheck TESTS="$TESTS" 2>&1 | tee installcheck.log || true - name: Show regression diffs if: always() && steps.check.outputs.should_run == 'true' id: collectlogs run: | find . -name regression.diffs -exec cat {} + > regression.log if [[ -s regression.log ]]; then echo "regression_diff=true" >> $GITHUB_OUTPUT; fi grep -e 'FAILED' -e 'failed (ignored)' -e 'not ok' installcheck.log || true cat regression.log - name: Save regression diffs if: always() && steps.collectlogs.outputs.regression_diff == 'true' uses: actions/upload-artifact@v4 with: name: Regression diff PG${{ steps.pg.outputs.version }} path: | regression.log installcheck.log - name: Save PostgreSQL log if: always() && steps.check.outputs.should_run == 'true' uses: actions/upload-artifact@v4 with: name: PostgreSQL log ${{ steps.pg.outputs.version }} path: postmaster.* - name: Verify tests failed on old code if: steps.check.outputs.should_run == 'true' run: | TESTS="${{ steps.check.outputs.changed_tests }}" # Here we switch on results because there can be variant expected files # using a _NNN suffix (pg_regress feature). readarray -t FAILED_TESTS < <( \ sed -n 's!^\+\+\+.*results/\([^/ -]\+\(-${{ steps.pg.outputs.major }}\)\?\)\.out[[:space:]].*$!\1!gp' regression.log) echo "Tests failing with the old code: ${FAILED_TESTS[@]}" # Check which changed tests did NOT fail PASSED_TESTS="" for TEST in $TESTS; do if ! echo "${FAILED_TESTS[@]}" | grep -qw "$TEST"; then PASSED_TESTS="$PASSED_TESTS $TEST" fi done if [[ -n "$PASSED_TESTS" ]]; then echo "WARNING: The following changed tests PASSED when they should have FAILED:" echo " $PASSED_TESTS" echo "" echo "This suggests these tests may not be testing the new code changes." echo "Please verify that your tests actually exercise the new functionality." exit 1 fi echo "All changed tests failed as expected." ================================================ FILE: .github/workflows/trigger-package-tests.yaml ================================================ # Trigger the Pre-Release tests # # Params: # ref: branch, tag or SHA to checkout, defaults to `main` # name: Trigger Packaging tests on: workflow_dispatch: inputs: ref: description: "branch, tag or SHA to checkout" required: true default: "main" jobs: trigger_tests: runs-on: ubuntu-latest steps: - name: Checkout TimescaleDB uses: actions/checkout@v4 with: ref: ${{ github.event.inputs.ref }} token: ${{ secrets.ORG_AUTOMATION_TOKEN }} - name: Push to package_test branch env: GH_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | git push origin HEAD:trigger/package_test --force ================================================ FILE: .github/workflows/trigger-prerelease-tests.yaml ================================================ # Trigger the Pre-Release tests # # Params: # ref: branch, tag or SHA to checkout, defaults to `main` # name: Trigger Pre-Release tests on: workflow_dispatch: inputs: ref: description: "branch, tag or SHA to checkout" required: true default: "main" jobs: trigger_tests: runs-on: ubuntu-latest steps: - name: Checkout TimescaleDB uses: actions/checkout@v4 with: ref: ${{ github.event.inputs.ref }} token: ${{ secrets.ORG_AUTOMATION_TOKEN }} - name: Push to pre-release branch env: GH_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} run: | git push origin HEAD:prerelease_test --force ================================================ FILE: .github/workflows/update-test.yaml ================================================ name: Update and Downgrade "on": push: branches: - main - ?.*.x - prerelease_test pull_request: workflow_dispatch: jobs: check_paths: runs-on: ubuntu-latest outputs: run_ci: >- ${{ github.event_name == 'push' || (steps.filter.outputs.sql == 'true' && !contains(github.event.pull_request.labels.*.name, 'skip-ci')) }} steps: - uses: actions/checkout@v4 - uses: dorny/paths-filter@v3 id: filter with: filters: .github/filters.yaml update_test: needs: check_paths if: needs.check_paths.outputs.run_ci == 'true' name: Update test PG${{ matrix.pg }} runs-on: 'ubuntu-latest' strategy: matrix: pg: [15, 16, 17, 18] fail-fast: false env: PG_VERSION: ${{ matrix.pg }} POSTGRES_HOST_AUTH_METHOD: trust steps: - name: Checkout TimescaleDB uses: actions/checkout@v4 - name: Install Dependencies run: | sudo apt-get update sudo apt-get install gnupg systemd-coredump gdb postgresql-common libkrb5-dev yes | sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh echo "deb https://packagecloud.io/timescale/timescaledb/ubuntu/ $(lsb_release -c -s) main" | sudo tee /etc/apt/sources.list.d/timescaledb.list wget --quiet -O - https://packagecloud.io/timescale/timescaledb/gpgkey | sudo apt-key add - sudo apt-get update sudo apt-get install postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} sudo apt-get install -y --no-install-recommends timescaledb-2-postgresql-${{ matrix.pg }} git fetch --tags - name: Update tests PG${{ matrix.pg }} run: | PATH="/usr/lib/postgresql/${{ matrix.pg }}/bin:$PATH" ./scripts/test_updates.sh - name: Downgrade tests PG${{ matrix.pg }} if: always() run: | PATH="/usr/lib/postgresql/${{ matrix.pg }}/bin:$PATH" ./scripts/test_downgrade.sh - name: Update diff if: failure() run: | find update_test -name "*.diff" | xargs -IFILE sh -c "echo '\nFILE\n';cat FILE" - name: Postgres Errors if: failure() run: | find update_test -name postgres.log -exec grep ERROR {} \; - name: Check for coredumps if: failure() id: collectlogs run: | # wait for in progress coredumps sleep 10 if coredumpctl list; then echo "coredumps=true" >>$GITHUB_OUTPUT false fi - name: Stack trace if: always() && steps.collectlogs.outputs.coredumps == 'true' run: | sudo coredumpctl gdb <<<" set verbose on set trace-commands on show debug-file-directory printf "'"'"query = '%s'\n\n"'"'", (char *) debug_query_string frame function ExceptionalCondition printf "'"'"condition = '%s'\n"'"'", (char *) conditionName up 1 l info args info locals bt full " 2>&1 | tee stacktrace.log ./scripts/bundle_coredumps.sh false - name: Upload Coredumps if: always() && steps.collectlogs.outputs.coredumps == 'true' uses: actions/upload-artifact@v4 with: name: Coredumps update-test ${{ matrix.os }} PG${{ matrix.pg }} path: coredumps - name: Upload Artifacts if: always() uses: actions/upload-artifact@v4 with: name: Update test PG${{ matrix.pg }} path: update_test report_status: name: Update and Downgrade summary needs: [check_paths, update_test] if: always() runs-on: ubuntu-latest steps: - run: | # If the detector failed, or the matrix failed, exit with error if [[ "${{ needs.update_test.result }}" == "failure" ]]; then exit 1 fi echo "All checks passed or were safely skipped." ================================================ FILE: .github/workflows/windows-build-and-test.yaml ================================================ # Test building the extension on Windows name: Regression Windows "on": schedule: # run daily 0:00 on main branch - cron: '0 0 * * *' push: branches: - main - ?.*.x - prerelease_test - trigger/windows_tests - trigger/regression pull_request: workflow_dispatch: jobs: config: runs-on: ubuntu-latest outputs: build_type: ${{ steps.build_type.outputs.build_type }} pg15_latest: ${{ steps.config.outputs.pg15_latest }} pg16_latest: ${{ steps.config.outputs.pg16_latest }} pg17_latest: ${{ steps.config.outputs.pg17_latest }} pg18_latest: ${{ steps.config.outputs.pg18_latest }} steps: - name: Checkout source code uses: actions/checkout@v4 - name: Read configuration id: config run: python .github/gh_config_reader.py - name: Set build_type id: build_type run: | if [[ "${{ github.event_name }}" == "pull_request" ]]; then echo "build_type=['Debug']" >>$GITHUB_OUTPUT else echo "build_type=['Debug','Release']" >>$GITHUB_OUTPUT fi check_paths: runs-on: ubuntu-latest outputs: run_ci: >- ${{ github.event_name == 'push' || (steps.filter.outputs.src == 'true' && !contains(github.event.pull_request.labels.*.name, 'skip-ci')) }} run_installcheck: ${{ github.event_name == 'push' || steps.filters.outputs.windows-workflow == 'true' }} steps: - uses: actions/checkout@v4 - uses: dorny/paths-filter@v3 id: filter with: filters: .github/filters.yaml build: # Change the JOB_NAME variable below when changing the name. name: PG${{ matrix.versions.pg }} ${{ matrix.build_type }} windows runs-on: windows-2025 needs: [config, check_paths] if: needs.check_paths.outputs.run_ci == 'true' strategy: fail-fast: false matrix: versions: - { pg: 15, pg_version: "${{ needs.config.outputs.pg15_latest }}" } - { pg: 16, pg_version: "${{ needs.config.outputs.pg16_latest }}" } - { pg: 17, pg_version: "${{ needs.config.outputs.pg17_latest }}" } - { pg: 18, pg_version: "${{ needs.config.outputs.pg18_latest }}" } build_type: ${{ fromJson(needs.config.outputs.build_type) }} pg_config: ["-cfsync=off -cstatement_timeout=60s"] event_name: ["${{ github.event_name }}"] exclude: - versions: { pg: 15, pg_version: "${{ needs.config.outputs.pg15_latest }}" } event_name: "pull_request" - versions: { pg: 16, pg_version: "${{ needs.config.outputs.pg16_latest }}" } event_name: "pull_request" env: # PostgreSQL configuration PGPORT: 55432 PGDATA: pgdata TABLESPACE1: D:\tablespace1\ TABLESPACE2: D:\tablespace2\ IGNORES: >- bgw_launcher chunk_adaptive compress_bloom_sparse_compat compress_bloom_sparse_debug compress_composite_bloom_debug compress_qualpushdown_saop compress_sort_transform compression_algos compression_uuid direct_compress_copy merge_append_partially_compressed metadata telemetry vector_agg_expr vector_agg_byte vector_agg_filter vector_agg_groupagg vector_agg_grouping vector_agg_planning-* vector_agg_text vector_agg_uuid vectorized_aggregation ${{ matrix.versions.pg < 16 && 'columnar_scan_cost' || '' }} SKIPS: >- create_hypertable create_table_with bgw_db_scheduler bgw_db_scheduler_fixed steps: - name: Remove Existing PostgreSQL shell: powershell run: | # Search for any installed application package containing 'PostgreSQL' in the name. $PostgresPackages = Get-Package -Name "*PostgreSQL*" -Provider Programs -ErrorAction SilentlyContinue if ($PostgresPackages) { Write-Host "Found $($PostgresPackages.Count) PostgreSQL package(s) to uninstall." # Use Uninstall-Package with -Force for a silent, non-interactive removal. $PostgresPackages | Uninstall-Package -Force # Manually remove common installation paths and data folders # to ensure no conflicts with the new installation. Remove-Item "C:\Program Files\PostgreSQL" -Recurse -Force -ErrorAction SilentlyContinue Remove-Item "C:\PostgreSQL" -Recurse -Force -ErrorAction SilentlyContinue } else { Write-Host "No registered PostgreSQL installation found. Skipping uninstall step." } - name: Setup WSL uses: Vampire/setup-wsl@v3.1.4 with: distribution: Debian - name: Configure apt retries and timeout in WSL shell: wsl-bash {0} run: | echo 'Acquire::Retries "10";' >> /etc/apt/apt.conf.d/99retries echo 'Acquire::http::Timeout "240";' >> /etc/apt/apt.conf.d/99retries echo 'Acquire::https::Timeout "240";' >> /etc/apt/apt.conf.d/99retries - name: Install additional packages in WSL shell: wsl-bash {0} run: | # Workaround for https://github.com/Vampire/setup-wsl/issues/76 sed -i 's#ftp.debian.org/debian bullseye-backports#archive.debian.org/debian bullseye-backports#' /etc/apt/sources.list apt-get update apt-get install --yes --no-install-recommends cmake gawk gcc git gnupg jq make postgresql-client postgresql-common tree - name: Configure git # Since we want to reuse the checkout in the WSL environment # we have to prevent git from changing the line ending in the # shell scripts as that would break them. run: | git config --global core.autocrlf false git config --global core.eol lf - name: Checkout TimescaleDB source uses: actions/checkout@v4 # We are going to rebuild Postgres daily, so that it doesn't suddenly break # ages after the original problem. - name: Get date for build caching id: get-date env: WSLENV: GITHUB_OUTPUT/p shell: wsl-bash {0} run: | echo "date=$(date +"%d")" >> $GITHUB_OUTPUT # Use a cache for the PostgreSQL installation to speed things up # and avoid unnecessary package downloads. Since we only save # the directory containing the binaries, the runs with a cache # hit won't have PostgreSQL installed with a running service # since the installer never runs. We therefore install with # --extract-only and launch our own test instance, which is # probably better anyway since it gives us more control. - name: Cache PostgreSQL installation uses: actions/cache@v4 id: cache-postgresql with: path: C:\Progra~1\PostgreSQL\${{ matrix.versions.pg }} key: "${{ runner.os }}-build-pg${{ matrix.pkg_version }}\ -${{ steps.get-date.outputs.date }}-${{ hashFiles('.github/**') }}" - name: Install PostgreSQL ${{ matrix.versions.pg }} (using ${{ matrix.versions.pg_version }}) if: github.event_name != 'schedule' && steps.cache-postgresql.outputs.cache-hit != 'true' run: | $version = ${{ matrix.versions.pg_version }} $version = $version.Replace('"', '') + "-1" winget install --id PostgreSQL.PostgreSQL.${{ matrix.versions.pg }} --version $version --force --accept-source-agreements --accept-package-agreements --disable-interactivity # This is for nightly builds. Here we pick the latest version of the package. - name: Install PostgreSQL ${{ matrix.versions.pg }} if: github.event_name == 'schedule' && steps.cache-postgresql.outputs.cache-hit != 'true' run: | winget install --id PostgreSQL.PostgreSQL.${{ matrix.versions.pg }} --force --accept-source-agreements --accept-package-agreements --disable-interactivity - name: Configure TimescaleDB run: cmake -B build_win -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} ` -DPG_CONFIG=C:\Progra~1/PostgreSQL/${{ matrix.versions.pg }}/bin/pg_config ` -DASSERTIONS=ON ` -DTEST_PG_LOG_DIRECTORY="log" - name: Build TimescaleDB run: cmake --build build_win --config ${{ matrix.build_type }} - name: Install TimescaleDB run: cmake --install build_win --config ${{ matrix.build_type }} - name: Enable crash dumps for postgres.exe (WER LocalDumps) if: needs.check_paths.outputs.run_installcheck == 'true' shell: powershell run: | $DumpDir = Join-Path $env:GITHUB_WORKSPACE "crashdumps" New-Item -ItemType Directory -Force -Path $DumpDir | Out-Null $Key = "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps\postgres.exe" New-Item -Path $Key -Force | Out-Null New-ItemProperty -Path $Key -Name DumpFolder -Value $DumpDir -PropertyType ExpandString -Force | Out-Null New-ItemProperty -Path $Key -Name DumpCount -Value 10 -PropertyType DWord -Force | Out-Null New-ItemProperty -Path $Key -Name DumpType -Value 2 -PropertyType DWord -Force | Out-Null Write-Host "Configured WER LocalDumps for postgres.exe => $DumpDir" - name: Setup postgres cluster if: needs.check_paths.outputs.run_installcheck == 'true' run: | C:\Progra~1/PostgreSQL/${{ matrix.versions.pg }}/bin/initdb -U postgres -A trust --locale=en_US --encoding=UTF8 mkdir -p ${{ env.TABLESPACE1 }}\_default mkdir -p ${{ env.TABLESPACE2 }}\_default icacls ${{ env.TABLESPACE1 }} /grant runneradmin:F /T icacls ${{ env.TABLESPACE2 }} /grant runneradmin:F /T copy build_win/test/postgresql.conf ${{ env.PGDATA }} icacls . /grant runneradmin:F /T C:\Progra~1/PostgreSQL/${{ matrix.versions.pg }}/bin/pg_ctl start -o "${{ matrix.pg_config }}" --log=postmaster.log C:\Progra~1/PostgreSQL/${{ matrix.versions.pg }}/bin/pg_isready -U postgres -d postgres --timeout=60 C:\Progra~1/PostgreSQL/${{ matrix.versions.pg }}/bin/psql -U postgres -d postgres -c 'CREATE USER root SUPERUSER LOGIN;' echo "PG version:" C:\Progra~1/PostgreSQL/${{ matrix.versions.pg }}/bin/psql -U postgres -d postgres -c 'SELECT version();' echo "Log configuration:" C:\Progra~1/PostgreSQL/${{ matrix.versions.pg }}/bin/psql -U postgres -d postgres -c 'SHOW logging_collector;' C:\Progra~1/PostgreSQL/${{ matrix.versions.pg }}/bin/psql -U postgres -d postgres -c 'SHOW log_filename;' C:\Progra~1/PostgreSQL/${{ matrix.versions.pg }}/bin/psql -U postgres -d postgres -c 'SHOW log_directory;' C:\Progra~1/PostgreSQL/${{ matrix.versions.pg }}/bin/psql -U postgres -d postgres -c 'SELECT pg_current_logfile();' echo "Data directory:" C:\Progra~1/PostgreSQL/${{ matrix.versions.pg }}/bin/psql -U postgres -d postgres -c 'SHOW data_directory;' - name: Install postgres for test runner if: needs.check_paths.outputs.run_installcheck == 'true' shell: wsl-bash {0} run: | yes | /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh || true echo 'Acquire::Retries "10";' >> /etc/apt/apt.conf.d/99retries echo 'Acquire::http::Timeout "240";' >> /etc/apt/apt.conf.d/99retries echo 'Acquire::https::Timeout "240";' >> /etc/apt/apt.conf.d/99retries apt-get install -y --force-yes postgresql-server-dev-${{ matrix.versions.pg }} - name: Run tests if: needs.check_paths.outputs.run_installcheck == 'true' shell: wsl-bash {0} env: WSLENV: "IGNORES:SKIPS" run: | export TEST_TABLESPACE1_PREFIX='${{ env.TABLESPACE1 }}' export TEST_TABLESPACE2_PREFIX='${{ env.TABLESPACE2 }}' cmake -B build_wsl -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DTEST_PGPORT_LOCAL=${{ env.PGPORT }} make -C build_wsl isolationchecklocal | tee -a installcheck.log make -C build_wsl regresschecklocal IGNORES="${IGNORES}" SKIPS="${SKIPS}" | tee -a installcheck.log - name: Setup postgres cluster for TSL tests if: needs.check_paths.outputs.run_installcheck == 'true' run: | C:\Progra~1/PostgreSQL/${{ matrix.versions.pg }}/bin/pg_ctl stop timeout 10 Remove-Item -Recurse ${{ env.PGDATA }} C:\Progra~1/PostgreSQL/${{ matrix.versions.pg }}/bin/initdb -U postgres -A trust --locale=en_US --encoding=UTF8 copy build_win/tsl/test/postgresql.conf ${{ env.PGDATA }} C:\Progra~1/PostgreSQL/${{ matrix.versions.pg }}/bin/pg_ctl start -o "${{ matrix.pg_config }}" --log="postmaster.log" C:\Progra~1/PostgreSQL/${{ matrix.versions.pg }}/bin/pg_isready -U postgres -d postgres --timeout=30 C:\Progra~1/PostgreSQL/${{ matrix.versions.pg }}/bin/psql -U postgres -d postgres -c 'CREATE USER root SUPERUSER LOGIN;' - name: Run TSL tests if: needs.check_paths.outputs.run_installcheck == 'true' shell: wsl-bash {0} env: WSLENV: "IGNORES:SKIPS" run: | export TEST_TABLESPACE1_PREFIX='${{ env.TABLESPACE1 }}' export TEST_TABLESPACE2_PREFIX='${{ env.TABLESPACE2 }}' make -C build_wsl isolationchecklocal-t | tee -a installcheck.log make -C build_wsl -k regresschecklocal-t IGNORES="${IGNORES}" SKIPS="${SKIPS}" | tee -a installcheck.log - name: Show regression diffs id: collectlogs if: always() env: WSLENV: GITHUB_OUTPUT/p shell: wsl-bash {0} run: | find . -name regression.diffs -exec cat {} + > regression.log if [[ -s regression.log ]]; then echo "regression_diff=true" >>$GITHUB_OUTPUT; fi grep -e 'FAILED' -e 'failed (ignored)' -e 'not ok' installcheck.log || true cat regression.log - name: Extract stack traces from crash dumps if: always() shell: powershell run: | $DumpDir = Join-Path $env:GITHUB_WORKSPACE "crashdumps" if (!(Test-Path $DumpDir)) { Write-Host "No crashdumps directory found."; exit 0 } $Dumps = Get-ChildItem -Path $DumpDir -Filter "*.dmp" -Recurse -ErrorAction SilentlyContinue if (!$Dumps) { Write-Host "No .dmp files found in $DumpDir."; exit 0 } $SymbolCache = Join-Path $DumpDir "symbols" New-Item -ItemType Directory -Force -Path $SymbolCache | Out-Null $Out = Join-Path $DumpDir "stacktraces.txt" foreach ($Dump in $Dumps) { Add-Content -Path $Out -Value ("==== " + $Dump.FullName + " ====") & cdb.exe -z $Dump.FullName -c ".symfix $SymbolCache; .reload; !analyze -v; kv; q" | Out-File -FilePath $Out -Append -Encoding utf8 } - name: Upload crash dumps and stack traces if: always() uses: actions/upload-artifact@v4 with: name: Crash dumps PG${{ matrix.versions.pg }} ${{ matrix.build_type }} windows path: | crashdumps\** if-no-files-found: ignore - name: Save regression diffs if: always() && steps.collectlogs.outputs.regression_diff == 'true' uses: actions/upload-artifact@v4 with: name: Regression ${{ matrix.versions.pg }} diff ${{ matrix.os }} ${{ matrix.build_type }} Build path: | regression.log installcheck.log - name: Save PostgreSQL log if: always() uses: actions/upload-artifact@v4 with: name: PostgreSQL ${{ matrix.versions.pg }} log ${{ matrix.os }} ${{ matrix.build_type }} Build path: ${{ env.PGDATA }}\log\postmaster.log - name: Upload CMake Logs if: always() uses: actions/upload-artifact@v4 with: name: CMake Logs ${{ matrix.versions.pg }} ${{ matrix.os }} ${{ matrix.build_type }} path: | build_win/CMakeCache.txt build_win/CMakeFiles/CMakeConfigureLog.yaml build_win/CMakeFiles/CMakeError.log build_win/CMakeFiles/CMakeOutput.log build_win/compile_commands.json build_wsl/CMakeCache.txt build_wsl/CMakeFiles/CMakeConfigureLog.yaml build_wsl/CMakeFiles/CMakeError.log build_wsl/CMakeFiles/CMakeOutput.log build_wsl/compile_commands.json - name: Upload test results to the database if: always() shell: wsl-bash {0} env: # Update when adding new variables. WSLENV: "JOB_NAME:CI_STATS_DB:GITHUB_EVENT_NAME:GITHUB_REF_NAME\ :GITHUB_REPOSITORY:GITHUB_RUN_ATTEMPT:GITHUB_RUN_ID:GITHUB_RUN_NUMBER:JOB_STATUS" # GitHub Actions allow you neither to use the env context for the job name, # nor to access the job name from the step context, so we have to # duplicate it to work around this nonsense. JOB_NAME: PG${{ matrix.versions.pg }} ${{ matrix.build_type }} ${{ matrix.os }} CI_STATS_DB: ${{ secrets.CI_STATS_DB }} GITHUB_EVENT_NAME: ${{ github.event_name }} GITHUB_REF_NAME: ${{ github.ref_name }} GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} GITHUB_RUN_ID: ${{ github.run_id }} GITHUB_RUN_NUMBER: ${{ github.run_number }} JOB_STATUS: ${{ job.status }} run: | if [[ "${{ github.event_name }}" == "pull_request" ]] ; then GITHUB_PR_NUMBER="${{ github.event.number }}" else GITHUB_PR_NUMBER=0 fi export GITHUB_PR_NUMBER scripts/upload_ci_stats.sh report_status: name: Regression Windows summary needs: [check_paths, build] if: always() runs-on: ubuntu-latest steps: - run: | # If the detector failed, or the matrix failed, exit with error if [[ "${{ needs.build.result }}" == "failure" ]]; then exit 1 fi echo "All checks passed or were safely skipped." ================================================ FILE: .github/workflows/windows-packages.yaml ================================================ # Test installation of windows package for latest version name: "Packaging tests: Windows" "on": schedule: # run daily 0:00 on main branch - cron: '0 0 * * *' pull_request: paths: .github/workflows/windows-packages.yaml push: tags: - '*' branches: - release_test - trigger/windows_packages workflow_dispatch: jobs: build: name: Windows package PG${{ matrix.pg }} runs-on: windows-2025 strategy: fail-fast: false matrix: pg: [ "15", "16", "17", "18" ] env: # PostgreSQL configuration PGPORT: 6543 PGDATA: pgdata steps: - name: Checkout TimescaleDB source uses: actions/checkout@v4 - name: Get version id: version env: GH_TOKEN: ${{ github.token }} run: | $version=gh release list --json tagName,isLatest --jq '.[] | select(.isLatest) | .tagName' echo "Determined version: " echo "version=$version" echo "version=$version" >>$env:GITHUB_OUTPUT - name: Install PostgreSQL ${{ matrix.pg }} run: | choco feature disable --name=usePackageExitCodes choco feature disable --name=showDownloadProgress choco uninstall postgresql --yes winget install --id PostgreSQL.PostgreSQL.${{ matrix.pg }} --accept-source-agreements --accept-package-agreements --disable-interactivity choco install wget - name: Download TimescaleDB run: "wget --quiet -O timescaledb.zip 'https://github.com/timescale/timescaledb/releases/download/\ ${{ steps.version.outputs.version }}/timescaledb-postgresql-${{ matrix.pg}}-windows-amd64.zip'" - name: Install TimescaleDB run: | tar -xf timescaledb.zip cd timescaledb ./setup.exe -yes-tune -pgconfig C:\Progra~1\PostgreSQL\${{ matrix.pg }}\bin\pg_config - name: Create DB run: | C:\Progra~1\PostgreSQL\${{ matrix.pg }}\bin\initdb -U postgres -A trust C:\Progra~1\PostgreSQL\${{ matrix.pg }}\bin\pg_ctl start -o "-cshared_preload_libraries=timescaledb" - name: Test creating extension run: | C:\Progra~1\PostgreSQL\${{ matrix.pg }}\bin\psql -U postgres -d postgres -X ` -c "CREATE EXTENSION timescaledb" ` -c "SELECT extname,extversion,version() FROM pg_extension WHERE extname='timescaledb'" $installed_version = C:\Progra~1\PostgreSQL\${{ matrix.pg }}\bin\psql -U postgres ` -d postgres -qtAX -c "SELECT extversion FROM pg_extension WHERE extname='timescaledb'" $installed_version = $installed_version.Trim() echo "Installed version is '${installed_version}'" if ("${installed_version}" -ne "${{ steps.version.outputs.version }}") { false } ================================================ FILE: .gitignore ================================================ \#*# .#* *~ **/CMakeFiles/ **/CMakeCache.txt /sql/tests/unit/testoutputs.tmp /sql/timescaledb--*.sql /sql/pre_install/*.gen /sql/updates/*.gen /data/ /src/*.o /src/*.so /src/*.d /.vscode/ /timescaledb.so *.bak *.backup typedef.list /test/testcluster /test/log /test/temp_schedule /build* /update_test **/GPATH **/GTAGS **/GRTAGS **/GSYMS tags /.vs /compile_commands.json /.DS_Store /.clangd /.cache /CMakeSettings.json /out/* /debug/ /cmake-build-debug/ /.idea/ coccinelle.diff .gdb_history ================================================ FILE: .perltidyrc ================================================ --add-whitespace --delete-old-whitespace --entab-leading-whitespace=4 --keep-old-blank-lines=2 --maximum-line-length=78 --nooutdent-long-comments --nooutdent-long-quotes --nospace-for-semicolon --opening-brace-on-new-line --output-line-ending=unix --paren-tightness=2 --paren-vertical-tightness=2 --paren-vertical-tightness-closing=2 --noblanks-before-comments ================================================ FILE: .pull-review ================================================ --- # pull-review config version (required) version: 1 # use review requests instead of assignees to assign reviewers to pull requests use_review_requests: true # maximum number of files to evaluate per pull request (set to 0 for no limit) max_files: 0 # evaluate all files of the PR # minimum number of reviewers to assign and notify for a pull request min_reviewers: 2 # maximum number of reviewers to assign and notify for a pull request max_reviewers: 2 # maximum number of files per reviewer (set to 0 for no limit) max_files_per_reviewer: 0 # maximum number of lines changed per reviewer (set to 0 for no limit) max_lines_per_reviewer: 0 # if at least a minimum number of reviewers is not found, assign a minimum number of reviewers randomly assign_min_reviewers_randomly: true # if the pull request changes code with fewer than a minimum number of authors, add extra reviewers if possible (set to 0 to disable) min_authors_of_changed_files: 0 # minimum percent of lines authored by a reviewer to require an extra reviewer (set to 0 to disable) min_percent_authorship_for_extra_reviewer: 0 # minimum number of lines that must be changed to add an extra reviewer (set to 0 to disable) min_lines_changed_for_extra_reviewer: 0 # require a user to be listed in the reviewers section in order to be assigned to a pull request require_notification: true # list of users and their app-specific usernames reviewers: Poroma-Banerjee: {} akuzm: {} antekresic: {} dbeck: {} kpan2034: {} melihmutlu: {} natalya-aksman: {} philkra: {} pnthao: {} svenklemm: {} # list of users who will never be notified review_blacklist: - timescale-automation - philkra # ignore changes to the test output files. # 1. because they usually will have equivalent .sql files # 2. because people that modify a test, don't necessarily change the same c-functionality at all # so tests should be less important than C files file_blacklist: - tsl/test/expected/*.out - test/expected/*.out label_blacklist: - is-auto-backport ================================================ FILE: .unreleased/chunk-param ================================================ Implements: #9368 Enable runtime chunk exclusion on inner side of nested loop join ================================================ FILE: .unreleased/columnar-function ================================================ Implements: #9117 Support functions like `time_bucket` in the columnar aggregation and grouping pipeline. ================================================ FILE: .unreleased/constant-gapfill ================================================ Fixes: #7629 Forbid non-constant timezone parameter in time_bucket_gapfill ================================================ FILE: .unreleased/direct-loss ================================================ Fixes: #9381 Data loss with direct compress with client-ordered data in an INSERT SELECT from a compressed hypertable. ================================================ FILE: .unreleased/in-any-chunk-exclusion ================================================ Implements: #9398 Fix chunk exclusion for IN/ANY on open (time) dimensions ================================================ FILE: .unreleased/parameterized-merge ================================================ Fixes: #9356 Potential crash when using a hypertable with partial compression or space partitioning in a nested loop join ================================================ FILE: .unreleased/pr_8983 ================================================ Implements: #8983 Add GUC for default chunk time interval ================================================ FILE: .unreleased/pr_9142 ================================================ Implements: #9142 Remove column `dropped` from _timescaledb_catalog.chunk ================================================ FILE: .unreleased/pr_9238 ================================================ Implements: #9238 Support non-partial aggregates with vectorized aggregation ================================================ FILE: .unreleased/pr_9253 ================================================ Implements: #9253 Support VectorAgg in subqueries and CTEs ================================================ FILE: .unreleased/pr_9266 ================================================ Implements: #9266 Add support for HAVING to vectorized aggregation ================================================ FILE: .unreleased/pr_9267 ================================================ Implements: #9267 Enable ColumnarIndexScan custom scan ================================================ FILE: .unreleased/pr_9312 ================================================ Implements: #9312 Remove advisory locks from bgw jobs and add graceful cancellation Thanks: @leppaott for reporting a deadlock when deleting jobs ================================================ FILE: .unreleased/pr_9334 ================================================ Implements: #9334 Fix out-of-range timestamp error in WHERE clauses ================================================ FILE: .unreleased/pr_9372 ================================================ Implements: #9372 Push down composite bloom filter checks to SELECT execution. ================================================ FILE: .unreleased/pr_9374 ================================================ Implements: #9374 Use bloom filters to eliminate decompression of unrelated compressed batches during UPSERTs. ================================================ FILE: .unreleased/pr_9376 ================================================ Fixes: #9376 Allow CREATE EXTENSION after drop in the same session Thanks: @janpio for reporting an issue with CREATE EXTENSION after dropping and recreating schema ================================================ FILE: .unreleased/pr_9378 ================================================ Fixes: #9378 Fix FK constraint failure when inserting into hypertable with referencing FK Thanks: @bronzinni for reporting an issue with foreign keys on hypertables ================================================ FILE: .unreleased/pr_9382 ================================================ Implements: #9382 Fix chunk creation failure after replica identity invalidation ================================================ FILE: .unreleased/pr_9399 ================================================ Implements: #9399 Use bloom filters to reduce decompression during UPDATE/DELETE commands. ================================================ FILE: .unreleased/pr_9413 ================================================ Fixes: #9413 Fix incorrect decompress markers on full batch delete ================================================ FILE: .unreleased/pr_9414 ================================================ Fixes: #9414 Fix NULL compression handling in estimate_uncompressed_size ================================================ FILE: .unreleased/pr_9417 ================================================ Fixes: #9417 Fix segfault in bloom1_contains ================================================ FILE: .unreleased/saop-pushdown ================================================ Implements: #9192 Push down scalar array operations into the columnar metadata scan by transforming them into an OR/AND clause. Setting: `enable_columnar_scan_filter_pushdown`: enables pushing the filters on columnar scan down to the compressed scan level. On by default. ================================================ FILE: .unreleased/text-minmax ================================================ Implements: #9104 Support min(text), max(text) for C collation in columnar aggregation pipeline ================================================ FILE: .unreleased/wrong-partition ================================================ Fixes: #9344 Wrong result or crash on cross-type comparison of partitioning column ================================================ FILE: .yamllint.yaml ================================================ rules: document-start: disable line-length: max: 250 ================================================ FILE: CHANGELOG.md ================================================ # TimescaleDB Changelog **Please note: When updating your database, you should connect using This page lists all the latest features and updates to TimescaleDB. When you use psql to update your database, use the -X flag and prevent any .psqlrc commands from accidentally triggering the load of a previous DB version.** ## 2.25.2 (2026-03-03) This release contains performance improvements and bug fixes since the 2.25.1 release and a fix for a security vulnerability ([#9331](https://github.com/timescale/timescaledb/pull/9331)). You can check the [security advisory](https://github.com/timescale/timescaledb/security/advisories/GHSA-vgp2-jj5c-828m) for more information on the vulnerability and the platforms that are affected. We recommend that you upgrade as soon as possible. **Bugfixes** * [#9276](https://github.com/timescale/timescaledb/pull/9276) Fix `NULL` and `DEFAULT` handling in uniqueness check on compressed chunks * [#9277](https://github.com/timescale/timescaledb/pull/9277) Fix SSL-related build errors * [#9279](https://github.com/timescale/timescaledb/pull/9279) Fix `EXPLAIN VERBOSE` corrupting targetlist of cached ModifyHypertable plans * [#9281](https://github.com/timescale/timescaledb/pull/9281) Fix real-time continuous aggregates on UUID hypertables * [#9283](https://github.com/timescale/timescaledb/pull/9283) Fix plan-time error when using `enum` in `orderby` compression setting * [#9290](https://github.com/timescale/timescaledb/pull/9290) Propagate `ALTER <object> OWNER TO` to policy jobs * [#9292](https://github.com/timescale/timescaledb/pull/9292) Fix continuous aggregate column rename * [#9293](https://github.com/timescale/timescaledb/pull/9293) Fix `time_bucket_gapfill` inside `LATERAL` subqueries * [#9294](https://github.com/timescale/timescaledb/pull/9294) Fix `DELETE`and `UPDATE` with `WHERE EXISTS` on hypertables * [#9303](https://github.com/timescale/timescaledb/pull/9303) Fix segfault in continuous aggregate creation on Postgres 18 * [#9308](https://github.com/timescale/timescaledb/pull/9308) Fix continuous aggregate offset/origin not applied in watermark and refresh window calculations * [#9314](https://github.com/timescale/timescaledb/pull/9314) Fix generated columns always `NULL` in compressed chunks * [#9321](https://github.com/timescale/timescaledb/pull/9321) Fix segfault when using OLD/NEW refs in `RETURNING` clause on Postgres 18 * [#9324](https://github.com/timescale/timescaledb/pull/9324) Potential violation of a foreign key constraint referencing a hypertable caused by concurrent `DELETE` of the key record * [#9327](https://github.com/timescale/timescaledb/pull/9327) Fix handling of generated columns with `NOT NULL` domain type * [#9331](https://github.com/timescale/timescaledb/pull/9331) Ensure `search_path` is set before anything else in SQL scripts * [#9339](https://github.com/timescale/timescaledb/pull/9339) Fix segmentwise recompression clearing unordered flag **Thanks** * @CaptainCuddleCube for reporting an issue with `time_bucket_gapfill` and `LATERAL` subqueries * @JacobBrejnbjerg for reporting an issue with generated columns in compressed chunks * @Kusumoto for reporting an issue with continuous aggregates on hypertables with UUID columns * @arfathyahiya for reporting an issue with renaming columns in continuous aggregates * @desertmark for reporting an issue with `DELETE`/`UPDATE` and subqueries * @flaviofernandes004 for reporting an issue with `RETURNING` clause and references to OLD/NEW * @tureba for fixing SSL-related build errors ## 2.25.1 (2026-02-17) This release contains performance improvements and bug fixes since the 2.25.0 release. We recommend that you upgrade at the next available opportunity. **Bugfixes** * [#9215](https://github.com/timescale/timescaledb/pull/9215) Add missing handling for em_parent to sort_transform * [#9223](https://github.com/timescale/timescaledb/pull/9223) Clean up orphaned entries in continuous aggregate invalidaton logs * [#9226](https://github.com/timescale/timescaledb/pull/9226) Fix invalidation and batching issues for variable bucket continuous aggregates. * [#9256](https://github.com/timescale/timescaledb/pull/9256) Error "record type has no extended hash function" on some queries using a sparse bloom filter index on a column of composite type. * [#9257](https://github.com/timescale/timescaledb/pull/9257) Handle type coercion for metadata column equivalence members **Thanks** * @emapple for reporting a crash in a query with nested joins and subqueries ## 2.25.0 (2026-01-29) This release contains performance improvements and bug fixes since the 2.24.0 release. We recommend that you upgrade at the next available opportunity. **Highlighted features in TimescaleDB v2.25.0** This release features multiple improvements for continuous aggregates on the columnstore: * Faster refreshes: You can now utilize direct compress during materialized view refreshes, resulting in higher throughput and reduced I/O usage. * Efficiency: The enablement of delete optimizations significantly lowers system resource requirements. * Smaller transactions: Adjusted defaults for `buckets_per_batch` to 10 reduces transaction sizes, requiring less WAL holding time. * Faster queries: Smarter defaults for `segmentby` and `orderby` yield improved query performance and better compression ratio on the columnstore. **Sunsetting announcements** * This release removes the WAL-based invalidation of continuous aggregates. This feature was introduced in [2.22.0](https://github.com/timescale/timescaledb/releases/tag/2.22.0) as tech preview to use logical decoding for building the invalidation logs. The feature was designed for high ingest workloads, reducing the write amplification. With the upcoming stream of improvements to continuous aggregates, this feature was deprioritized and removed. * The old continuous aggregate format, deprecated in version [2.10.0](https://github.com/timescale/timescaledb/releases/tag/2.10.0), has been fully removed from TimescaleDB in this release. Users still on the old format should read the [migration documentation](https://www.tigerdata.com/docs/use-timescale/latest/continuous-aggregates/migrate) to migrate to the new format. Users of Tiger Cloud have already been automatically migrated. **Features** * [#8777](https://github.com/timescale/timescaledb/pull/8777) Enable direct compress on continuous aggregate refresh using new GUC `timescaledb.enable_direct_compress_on_cagg_refresh` * [#9031](https://github.com/timescale/timescaledb/pull/9031) Change default `buckets_per_batch` on continuous aggregate refresh policy to `10` * [#9032](https://github.com/timescale/timescaledb/pull/9032) Add in-memory recompression for unordered chunks * [#9017](https://github.com/timescale/timescaledb/pull/9017) Move `bgw_job` table into schema `_timescaledb_catalog` * [#9033](https://github.com/timescale/timescaledb/pull/9033) Add `rebuild_columnstore` procedure * [#9038](https://github.com/timescale/timescaledb/pull/9038) Change default configuration for compressed continuous aggregates * [#9042](https://github.com/timescale/timescaledb/pull/9042) Enable batch sorted merge on unordered compressed chunks * [#9046](https://github.com/timescale/timescaledb/pull/9046) Allow non timescaledb namespace `SET` option for continuous aggregates * [#9059](https://github.com/timescale/timescaledb/pull/9059) Allow configuring `work_mem` for background worker jobs * [#9074](https://github.com/timescale/timescaledb/pull/9074) Add function to estimate uncompressed size of compressed chunk * [#9085](https://github.com/timescale/timescaledb/pull/9085) Don't register timescaledb-tune specific GUCs * [#9088](https://github.com/timescale/timescaledb/pull/9088) Add `ColumnarIndexScan` custom node * [#9090](https://github.com/timescale/timescaledb/pull/9090) Support direct batch delete on hypertables with continuous aggregates * [#9094](https://github.com/timescale/timescaledb/pull/9094) Enable the columnar pipeline for grouping without aggregation to speed up the queries of the form `select column from table group by column`. * [#9103](https://github.com/timescale/timescaledb/pull/9103) Support `FIRST` and `LAST` in `ColumnarIndexScan` * [#9108](https://github.com/timescale/timescaledb/pull/9108) Support multiple aggregates in `ColumnarIndexScan` * [#9111](https://github.com/timescale/timescaledb/pull/9111) Allow recompression with orderby/index changes * [#9113](https://github.com/timescale/timescaledb/pull/9113) Use `enable_columnarscan` to control columnarscan * [#9127](https://github.com/timescale/timescaledb/pull/9127) Remove primary dimension constraints from fully covered chunks * [#8710](https://github.com/timescale/timescaledb/pull/8710) Add SQL function to fetch continuous aggregate grouping columns * [#9133](https://github.com/timescale/timescaledb/pull/9133) Allow pushing down sort into columnar unordered chunks when it is possible * [#8229](https://github.com/timescale/timescaledb/pull/8229) Removed `time_bucket_ng` function * [#8859](https://github.com/timescale/timescaledb/pull/8859) Remove support for partial continuous aggregate format * [#9022](https://github.com/timescale/timescaledb/pull/9022) Remove WAL based invalidation * [#9016](https://github.com/timescale/timescaledb/pull/9016) Remove `_timescaledb_debug` schema * [#9030](https://github.com/timescale/timescaledb/pull/9030) Add new chunks to hypertable publication **Bug fixes** * [#8706](https://github.com/timescale/timescaledb/pull/8706) Fix planning performance regression on Postgres 16 and later on some join queries. * [#8986](https://github.com/timescale/timescaledb/pull/8986) Add pathkey replacement for `ColumnarScanPath` * [#8989](https://github.com/timescale/timescaledb/pull/8989) Ensure no XID is assigned during chunk query * [#8990](https://github.com/timescale/timescaledb/pull/8990) Fix `EquivalenceClass` index update for `RelOptInfo` * [#9007](https://github.com/timescale/timescaledb/pull/9007) Add validation for compression index key limits * [#9024](https://github.com/timescale/timescaledb/pull/9024) Recompress some chunks on `VACUUM FULL` * [#9045](https://github.com/timescale/timescaledb/pull/9045) Fix missing UUID check in compression policy * [#9056](https://github.com/timescale/timescaledb/pull/9056) Fix split chunk `relfrozenxid` * [#9058](https://github.com/timescale/timescaledb/pull/9058) Fix missing chunk column stats bug * [#9061](https://github.com/timescale/timescaledb/pull/9061) Fix update race with background worker jobs * [#9069](https://github.com/timescale/timescaledb/pull/9069) Fix applying multikey sort for columnstore when one numeric key is pinned to a Const of different type * [#9102](https://github.com/timescale/timescaledb/pull/9102) Support retention policies on UUIDv7-partitioned hypertables * [#9120](https://github.com/timescale/timescaledb/pull/9120) Fix for pre Postgres 17, where a `DELETE` from a partially compressed chunk may miss records if `BitmapHeapScan` is being used * [#9121](https://github.com/timescale/timescaledb/pull/9121) Allow any immutable constant expressions as default values for compressed columns * [#9121](https://github.com/timescale/timescaledb/pull/9121) Fix a potential "unexpected column type 'bool'" error for compressed bool columns with missing value * [#9144](https://github.com/timescale/timescaledb/pull/9144) Fix handling implicit constraints in `ALTER TABLE` * [#9155](https://github.com/timescale/timescaledb/pull/9155) Fix column generation during compressed chunk insert * [#9129](https://github.com/timescale/timescaledb/pull/9129) Fix `time_bucket` with timezone during DST * [#9177](https://github.com/timescale/timescaledb/pull/9177) Add alias for `bgw_job` * [#9176](https://github.com/timescale/timescaledb/pull/9176) Handle `NULL` values in continuous aggregate invalidation more gracefully * [#9175](https://github.com/timescale/timescaledb/pull/9175) Do not remove dimension constraints for OSM chunks **GUCs** * `enable_columnarindexscan`: Enable returning results directly from compression metadata without decompression. This feature is experimental, and in development towards a GA release. Not for production environments. Default: `false` * `enable_direct_compress_on_cagg_refresh`: Enable experimental support for direct compression during Continuous Aggregate refresh. Default: `false` * `enable_qual_filtering`: Filter qualifiers on chunks when complete chunk would be included by filter. Default: `true` **Thanks** * @t-aistleitner for reporting the planning performance regression on PG16 and later on some join queries. * @vahnrr for reporting a crash when adding columns and constraints to a hypertable at the same time * @cracksalad and @eyadmba for reporting a bug with timezone handling in `time_bucket` ## 2.24.0 (2025-12-03) This release contains performance improvements and bug fixes since the 2.23.1 release. We recommend that you upgrade at the next available opportunity. **Highlighted features in TimescaleDB v2.24.0** * **Direct Compress just got smarter and faster**: it now works seamlessly with hypertables generating continuous aggregates. Invalidation ranges are computed directly in-memory based on the ingested batches and written efficiently at transaction commit. This change reduces the IO footprint drastically by removing the write amplification of the invalidation logs. * **Continuous aggregates now speak UUIDv7**: hypertables partitioned by UUIDv7 are fully supported through an enhanced `time_bucket` that accepts UUIDv7 values and returns precise, timezone-aware timestamps — unlocking powerful time-series analytics on modern UUID-driven table schemas. * **Lightning-fast recompression**: the new `recompress := true` option on the `convert_to_columnstore` API enables pure in-memory recompression, delivering a **4–5× speed boost** over the previous disk-based process. **ARM support for bloom filters** The [sparse bloom filter indexes](https://www.tigerdata.com/blog/blocked-bloom-filters-speeding-up-point-lookups-in-tiger-postgres-native-columnstore) will stop working after upgrade to 2.24. If you are affected by this problem, the warning "bloom filter sparse indexes require action to re-enable" will appear in the Postgres log during upgrade. In versions before 2.24, the hashing scheme of the bloom filter sparse indexes used to depend on the build options of the TimescaleDB executables. These options are set by the package publishers and might differ between different package sources or even versions. After upgrading to a version with different options, the queries that use the bloom filter lookups could erroneously stop returning the rows that should in fact match the query conditions. The 2.24 release fixes this by using distinct column names for each hashing scheme. The bloom filter sparse indexes will be disabled on the compressed chunks created before upgrading to 2.24. To re-enable them, you have to decompress and then compress the affected chunks. If you were running the official APT package on AMD64 architecture, the hashing scheme did not change, and it is safe to use the existing bloom filter sparse indexes. To enable this, set the GUC `timescaledb.read_legacy_bloom1_v1 = on` in the server configuration. The chunks compressed after upgrade to 2.24 will use the new index format, and the bloom filter sparse indexes will continue working as usual for these chunks without any intervention. For more details, refer to the pull request [#8761](https://github.com/timescale/timescaledb/pull/8761). **Deprecations** * The next release of TimescaleDB will remove the deprecated partial continuous aggregates format. The new format was introduced in [`2.7.0`](https://github.com/timescale/timescaledb/releases/tag/2.7.0) and provides significant improvements in terms of performance and storage efficiency. Please use [`cagg_migrate(<CONTINUOUS_AGGREGATE_NAME>)`](https://www.tigerdata.com/docs/use-timescale/latest/continuous-aggregates/migrate) to migrate to the new format. Tiger Cloud users are migrated automatically. * In future releases the deprecated view `timescaledb_information.compression_settings` will be removed. Please use [`timescaledb_information.hypertable_columnstore_settings`](https://www.tigerdata.com/docs/api/latest/hypercore/hypertable_columnstore_settings) as a replacement. * The experimental view [`timescaledb_experimental.policies`](https://www.tigerdata.com/docs/api/latest/informational-views/policies) and the adjacent experimental functions [`add_policies`](https://www.tigerdata.com/docs/api/latest/continuous-aggregates/add_policies), [`alter_policies`](https://www.tigerdata.com/docs/api/latest/continuous-aggregates/alter_policies), [`show_policies`](https://www.tigerdata.com/docs/api/latest/continuous-aggregates/show_policies), [`remove_policies`](https://www.tigerdata.com/docs/api/latest/continuous-aggregates/remove_policies), and [`remove_all_policies`](https://www.tigerdata.com/docs/api/latest/continuous-aggregates/remove_all_policies) to manage continuous aggregates will be removed in an upcoming release. For replacements, please use the [Jobs API](https://www.tigerdata.com/docs/api/latest/jobs-automation). **Backward-Incompatible Changes** * [#8761](https://github.com/timescale/timescaledb/pull/8761) Fix matching rows in queries using the bloom filter sparse indexes potentially not returned after extension upgrade. The version of the bloom filter sparse indexes is changed. The existing indexes will stop working and will require action to re-enable. See the section above for details. **Features** * [#8465](https://github.com/timescale/timescaledb/pull/8465) Speed up the filters like `x = any(array[...])` using bloom filter sparse indexes. * [#8569](https://github.com/timescale/timescaledb/pull/8569) In-memory recompression * [#8754](https://github.com/timescale/timescaledb/pull/8754) Add concurrent mode for merging chunks * [#8786](https://github.com/timescale/timescaledb/pull/8786) Display chunks view range as timestamps for UUIDv7 * [#8819](https://github.com/timescale/timescaledb/pull/8819) Refactor chunk compression logic * [#8840](https://github.com/timescale/timescaledb/pull/8840) Allow `ALTER COLUMN TYPE` when compression is enabled but no compressed chunks exist * [#8908](https://github.com/timescale/timescaledb/pull/8908) Add time bucketing support for UUIDv7 * [#8909](https://github.com/timescale/timescaledb/pull/8909) Support direct compress on hypertables with continuous aggregates * [#8939](https://github.com/timescale/timescaledb/pull/8939) Support continuous aggregates on UUIDv7-partitioned hypertables * [#8959](https://github.com/timescale/timescaledb/pull/8959) Cap continuous aggregate invalidation interval range at chunk boundary * [#8975](https://github.com/timescale/timescaledb/pull/8975) Exclude date/time columns from default segmentby * [#8993](https://github.com/timescale/timescaledb/pull/8993) Add GUC for in-memory recompression **Bugfixes** * [#8839](https://github.com/timescale/timescaledb/pull/8839) Improve `_timescaledb_functions.cagg_watermark` error handling * [#8853](https://github.com/timescale/timescaledb/pull/8853) Change log level of continuous aggregate refresh messages to `DEBUG1` * [#8933](https://github.com/timescale/timescaledb/pull/8933) Potential crash or seemingly random errors when querying the compressed chunks created on releases before 2.15 and using the minmax sparse indexes. * [#8942](https://github.com/timescale/timescaledb/pull/8942) Fix lateral join handling for compressed chunks * [#8958](https://github.com/timescale/timescaledb/pull/8958) Fix `if_not_exists` behaviour when adding refresh policy * [#8969](https://github.com/timescale/timescaledb/pull/8969) Gracefully handle missing job stat in background worker * [#8988](https://github.com/timescale/timescaledb/pull/8988) Don't ignore additional filters on same column when building scankeys **GUCs** * `direct_compress_copy_tuple_sort_limit`: Number of tuples that can be sorted at once in a `COPY` operation. * `direct_compress_insert_tuple_sort_limit`: Number of tuples that can be sorted at once in an `INSERT` operation. * `read_legacy_bloom1_v1`: Enable reading the legacy `bloom1` version 1 sparse indexes for `SELECT` queries. * `enable_in_memory_recompression`: Enable in-memory recompression functionality. **Thanks** * @bezpechno for implementing `ALTER COLUMN TYPE` for hypertable with columnstore when no compressed chunks exist ## 2.23.1 (2025-11-13) This release contains performance improvements and bug fixes since the 2.23.0 release. We recommend that you upgrade at the next available opportunity. **Bugfixes** * [#8873](https://github.com/timescale/timescaledb/pull/8873) Don't error on failure to update job stats * [#8875](https://github.com/timescale/timescaledb/pull/8875) Fix decoding of UUID v7 timestamp microseconds * [#8879](https://github.com/timescale/timescaledb/pull/8879) Fix blocker for multiple hierarchical continuous aggregate policies * [#8882](https://github.com/timescale/timescaledb/pull/8882) Fix crash in policy creation **Thanks** * @alexanderlaw for reporting a crash when creating a policy * @leppaott for reporting an issue with hierarchical continuous aggregates ## 2.23.0 (2025-10-29) This release contains performance improvements and bug fixes since the 2.22.1 release. We recommend that you upgrade at the next available opportunity. **Highlighted features in TimescaleDB v2.23.0** * This release introduces full PostgreSQL 18 support for all existing features. TimescaleDB v2.23 is available for PostgreSQL 15, 16, 17, and 18. * UUIDv7 compression is now enabled by default on the columnstore. This feature was shipped in [v2.22.0](https://github.com/timescale/timescaledb/releases/tag/2.22.0). It saves you at least 30% of storage and delivers ~2× faster query performance with UUIDv7 columns in the filter conditions. * Added the ability to set hypertables to unlogged, addressing an open community request [#836](https://github.com/timescale/timescaledb/issues/836). This allows the tradeoff between durability and performance, with the latter being favourable for larger imports. * By allowing [set-returning functions](https://www.postgresql.org/docs/current/functions-srf.html) in continuous aggregates, this releases addresses a long standing blocker, raised by the community [#1717](https://github.com/timescale/timescaledb/issues/1717). **PostgreSQL 15 deprecation announcement** We will continue supporting PostgreSQL 15 until June 2026. Closer to that time, we will announce the specific TimescaleDB version in which PostgreSQL 15 support will not be included going forward. **Features** * [#8373](https://github.com/timescale/timescaledb/pull/8373) More precise estimates of row numbers for columnar storage based on Postgres statistics. * [#8581](https://github.com/timescale/timescaledb/pull/8581) Allow mixing Postgres and TimescaleDB options in `ALTER TABLE SET`. * [#8582](https://github.com/timescale/timescaledb/pull/8582) Make `partition_column` in `CREATE TABLE WITH` optional. * [#8588](https://github.com/timescale/timescaledb/pull/8588) Automatically create a columnstore policy when a hypertable with columnstore enabled is created via `CREATE TABLE WITH` statement. * [#8606](https://github.com/timescale/timescaledb/pull/8606) Add job history config parameters for maximum successes and failures to keep for each job. * [#8632](https://github.com/timescale/timescaledb/pull/8632) Remove `ChunkDispatch` custom node. * [#8637](https://github.com/timescale/timescaledb/pull/8637) Add `INSERT` support for direct compress. * [#8661](https://github.com/timescale/timescaledb/pull/8661) Allow `ALTER TABLE ONLY` to change `reloptions` to apply setting changes only to future chunks. * [#8703](https://github.com/timescale/timescaledb/pull/8703) Allow set-returning functions in continuous aggregates. * [#8734](https://github.com/timescale/timescaledb/pull/8734) Support direct compress when inserting into a chunk. * [#8741](https://github.com/timescale/timescaledb/pull/8741) Add support for unlogged hypertables. * [#8769](https://github.com/timescale/timescaledb/pull/8769) Remove continuous aggregate invalidation trigger. * [#8798](https://github.com/timescale/timescaledb/pull/8798) Enable UUIDv7 compression by default. * [#8804](https://github.com/timescale/timescaledb/pull/8804) Remove `insert_blocker` trigger. **Bugfixes** * [#8561](https://github.com/timescale/timescaledb/pull/8561) Show warning when direct compress is skipped due to triggers or unique constraints. * [#8567](https://github.com/timescale/timescaledb/pull/8567) Do not require a job to have executed to show status. * [#8654](https://github.com/timescale/timescaledb/pull/8654) Fix `approximate_row_count` for compressed chunks. * [#8704](https://github.com/timescale/timescaledb/pull/8704) Fix direct `DELETE` on compressed chunk. * [#8728](https://github.com/timescale/timescaledb/pull/8728) Don't block dropping hypertables with other objects. * [#8735](https://github.com/timescale/timescaledb/pull/8735) Fix `ColumnarScan` for `UNION` queries. * [#8739](https://github.com/timescale/timescaledb/pull/8739) Fix cached utility statements. * [#8742](https://github.com/timescale/timescaledb/pull/8742) Potential internal program error when grouping by `bool` columns of a compressed hypertable. * [#8743](https://github.com/timescale/timescaledb/pull/8743) Modify schedule interval for job history pruning. * [#8746](https://github.com/timescale/timescaledb/pull/8746) Support show/drop chunks with UUIDv7 partitioning. * [#8753](https://github.com/timescale/timescaledb/pull/8753) Allow sorts over decompressed index scans for `ChunkAppend`. * [#8758](https://github.com/timescale/timescaledb/pull/8758) Improve error message on catalog version mismatch. * [#8774](https://github.com/timescale/timescaledb/pull/8774) Add GUC for WAL based invalidation of continuous aggregates. * [#8782](https://github.com/timescale/timescaledb/pull/8782) Stops sparse index from allowing multiple options. * [#8799](https://github.com/timescale/timescaledb/pull/8799) Set `next_start` for `WITH` clause compression policy. * [#8807](https://github.com/timescale/timescaledb/pull/8807) Only warn but not fail the compression if bloom filter indexes are configured but disabled with a GUC. **GUCs** * `cagg_processing_wal_batch_size`: Batch size when processing WAL entries. * `enable_cagg_wal_based_invalidation`: Enable experimental invalidations for continuous aggregates using WAL. * `enable_direct_compress_insert`: Enable direct compression during `INSERT`. * `enable_direct_compress_insert_client_sorted`: Enable direct compress `INSERT` with presorted data. * `enable_direct_compress_insert_sort_batches`: Enable batch sorting during direct compress `INSERT`. **Thanks** * @brandonpurcell-dev For highlighting issues with `show_chunks()` and UUIDv7 partitioning * @moodgorning for reporting an issue with the `timescaledb_information.job_stats` view * @ruideyllot for reporting set-returning functions not working in continuous aggregates * @t-aistleitner for reporting an issue with utility statements in plpgsql functions ## 2.22.1 (2025-09-30) This release contains performance improvements and bug fixes since the [2.22.0](https://github.com/timescale/timescaledb/releases/tag/2.20.0) release. We recommend that you upgrade at the next available opportunity. This release blocks the ability to leverage **concurrent refresh policies** in **hierarchical continuous aggregates**, as potential deadlocks can occur. [Concurrent refresh policies](https://docs.tigerdata.com/use-timescale/latest/continuous-aggregates/refresh-policies/#add-concurrent-refresh-policies) were introduced in [2.21.0](https://github.com/timescale/timescaledb/releases/tag/2.20.0) and allow users to define multiple time ranges, to refresh, e.g. data from the last hour in policy and the last day in a second policy. If you are using this feature with **hierarchical** continuous aggregates, please [remove the existing policies](https://docs.tigerdata.com/api/latest/jobs-automation/delete_job/#samples) and [create a new policy](https://docs.tigerdata.com/use-timescale/latest/continuous-aggregates/refresh-policies/#change-the-refresh-policy) for the full range you want to refresh, of the continuous aggregate as follows: ``` --- Find the job ID's of the concurrent refresh policies SELECT * FROM timescaledb_information.jobs WHERE proc_name = 'policy_refresh_continuous_aggregate'; --- Remove the job SELECT delete_job("<job_id_of_concurrent_policy>"); --- Create new policy for hierarchical continuous aggregate SELECT add_continuous_aggregate_policy('<name_of_materialized_view>', start_offset => INTERVAL '1 month', end_offset => INTERVAL '1 day', schedule_interval => INTERVAL '1 hour'); ``` **Bugfixes** * [#7766](https://github.com/timescale/timescaledb/pull/7766) Load the OSM extension in the retention background worker to drop tiered chunks * [#8550](https://github.com/timescale/timescaledb/pull/8550) Error in gapfill with expressions over aggregates, groupby columns, and out-of-order columns * [#8593](https://github.com/timescale/timescaledb/pull/8593) Error on change of invalidation method for continuous aggregate * [#8599](https://github.com/timescale/timescaledb/pull/8599) Fix the attnum mismatch bug in chunk constraint checks * [#8607](https://github.com/timescale/timescaledb/pull/8607) Fix the interrupted continous aggregate refresh materialization phase that leaves behind pending materialization ranges * [#8638](https://github.com/timescale/timescaledb/pull/8638) `ALTER TABLE RESET` for `orderby` settings * [#8644](https://github.com/timescale/timescaledb/pull/8644) Fix the migration script for sparse index configuration * [#8657](https://github.com/timescale/timescaledb/pull/8657) Fix `CREATE TABLE WITH` when using UUIDv7 partitioning * [#8659](https://github.com/timescale/timescaledb/pull/8659) `ALTER TABLE` commands to foreign data wrapper chunks not propogated. * [#8693](https://github.com/timescale/timescaledb/pull/8693) Compressed index not chosen for `varchar` typed `segmentby` columns * [#8707](https://github.com/timescale/timescaledb/pull/8707) Block concurrent refresh policies for hierarchical continous aggregate due to potential deadlocks **Thanks** * @MKrkkl for reporting a bug in gapfill queries with expressions over aggregates and groupby columns * @brandonpurcell-dev for creating a test case that showed a bug in `CREATE TABLE WITH` when using UUIDv7 partitioning * @snyrkill for reporting a bug when interrupting a continous aggregate refresh ## 2.22.0 (2025-09-02) This release contains performance improvements and bug fixes since the 2.21.3 release. We recommend that you upgrade at the next available opportunity. **Highlighted features in TimescaleDB v2.22.0** * Sparse indexes on compressed hypertables can now be explicitly configured via `ALTER TABLE` rather than relying only on internal heuristics. Users can define indexes on multiple columns to improve query performance for their specific workloads. * [Tech Preview] Continuous aggregates now support the `timescaledb.invalidate_using` option, enabling invalidations to be collected either via triggers on the hypertable or directly from WAL using logical decoding. Aggregates inherit the hypertable’s method if none is specified. * UUIDv7 compression and vectorization are now supported. The compression algorithm leverages the timestamp portion for delta-delta compression while storing the random portion separately. The vectorized equality/inequality filters with bulk decompression deliver ~2× faster query performance. The feature is disabled by default (`timescaledb.enable_uuid_compression`) to simplify the downgrading experience, and will be enabled out of the box in the next minor release. * Hypertables can now be partitioned by UUIDv7 columns, leveraging their embedded timestamps for time-based chunking. We’ve also added utility functions to simplify working with UUIDv7, such as generating values or extracting timestamps - e.g., `uuid_timestamp()` returns a PostgreSQL timestamp from a UUIDv7. * SkipScan now supports multi-column indexes in not-null mode, improving performance for distinct and ordered queries across multiple keys. **Removal of the hypercore table access method** We made the decision to deprecate the hypercore table access method (TAM) with the 2.21.0 release. Hypercore TAM was an experiment and it did not show the performance improvements we hoped for. It is removed with this release. Upgrades to 2.22.0 and higher are blocked if TAM is still in use. Since TAM’s inception in [2.18.0](https://github.com/timescale/timescaledb/releases/tag/2.18.0), we learned that btrees were not the right architecture. Recent advancements in the columnstore, such as more performant backfilling, SkipScan, adding check constraints, and faster point queries, put the [columnstore](https://www.timescale.com/blog/hypercore-a-hybrid-row-storage-engine-for-real-time-analytics) close to or on par with TAM without needing to store an additional index. We apologize for the inconvenience this action potentially causes and are here to assist you during the migration process. Migration path ``` do $$ declare relid regclass; begin for relid in select cl.oid from pg_class cl join pg_am am on (am.oid = cl.relam) where am.amname = 'hypercore' loop raise notice 'converting % to heap', relid::regclass; execute format('alter table %s set access method heap', relid); end loop; end $$; ``` **Features** * [#8247](https://github.com/timescale/timescaledb/pull/8247) Add configurable alter settings for sparse indexes * [#8306](https://github.com/timescale/timescaledb/pull/8306) Add option for invalidation collection using WAL for continuous aggregates * [#8340](https://github.com/timescale/timescaledb/pull/8340) Improve selectivity estimates for sparse minmax indexes, so that an index scan on a table in the columnstore is chosen more often when it's beneficial. * [#8360](https://github.com/timescale/timescaledb/pull/8360) Continuous aggregate multi-hypertable invalidation processing * [#8364](https://github.com/timescale/timescaledb/pull/8364) Remove hypercore table access method * [#8371](https://github.com/timescale/timescaledb/pull/8371) Show available timescaledb `ALTER` options when encountering unsupported options * [#8376](https://github.com/timescale/timescaledb/pull/8376) Change `DecompressChunk` custom node name to `ColumnarScan` * [#8385](https://github.com/timescale/timescaledb/pull/8385) UUID v7 functions for testing pre PG18 * [#8393](https://github.com/timescale/timescaledb/pull/8393) Add specialized compression for UUIDs. Best suited for UUID v7, but still works with other UUID versions. This is experimental at the moment and backward compatibility is not guaranteed. * [#8398](https://github.com/timescale/timescaledb/pull/8398) Set default compression settings at compress time * [#8401](https://github.com/timescale/timescaledb/pull/8401) Support `ALTER TABLE RESET` for compression settings * [#8414](https://github.com/timescale/timescaledb/pull/8414) Vectorised filtering of UUID Eq and Ne filters, plus bulk decompression of UUIDs * [#8424](https://github.com/timescale/timescaledb/pull/8424) Block downgrade when orderby setting is `NULL` * [#8454](https://github.com/timescale/timescaledb/pull/8454) Remove internal unused index helper functions * [#8494](https://github.com/timescale/timescaledb/pull/8494) Improve job stat history retention policy * [#8496](https://github.com/timescale/timescaledb/pull/8496) Fix dropping chunks with foreign keys * [#8505](https://github.com/timescale/timescaledb/pull/8505) Add support for partitioning on UUIDv7 * [#8513](https://github.com/timescale/timescaledb/pull/8513) Support multikey SkipScan when all keys are guaranteed to be non-null * [#8514](https://github.com/timescale/timescaledb/pull/8514) Concurrent continuous aggregates improvements * [#8528](https://github.com/timescale/timescaledb/pull/8528) Add the `_timescaledb_functions.chunk_status_text` helper function * [#8529](https://github.com/timescale/timescaledb/pull/8529) Optimize direct compress status handling **Bugfixes** * [#8422](https://github.com/timescale/timescaledb/pull/8422) Don't require `columnstore=false` when using the TimescaleDB Apache 2 Edition * [#8493](https://github.com/timescale/timescaledb/pull/8493) Change log level of `not null` constraint message * [#8500](https://github.com/timescale/timescaledb/pull/8500) Fix uniqueness check with generated columns and hypercore * [#8545](https://github.com/timescale/timescaledb/pull/8545) Fix error in LOCF/Interpolate with out-of-order and repeated columns * [#8558](https://github.com/timescale/timescaledb/pull/8558) Error out on bad args when processing invalidation * [#8559](https://github.com/timescale/timescaledb/pull/8559) Fix `timestamp out of range` using `end_offset=NULL` on CAgg refresh policy **GUCs** * `enable_multikey_skipscan`: Enable SkipScan for multiple distinct keys, default: on * `enable_uuid_compression`: Enable UUID compression functionality, default: off * `cagg_processing_wal_batch_size`: Batch size when processing WAL entries, default: 10000 * `cagg_processing_low_work_mem`: Low working memory limit for continuous aggregate invalidation processing, default: 38.4MB * `cagg_processing_high_work_mem`: High working memory limit for continuous aggregate invalidation processing, default: 51.2MB **Thanks** * @CodeTherapist for reporting an issue where foreign key checks did not work after several insert statements * @moodgorning for reporting a bug in queries with LOCF/Interpolate using out-of-order columns * @nofalx for reporting an error when using `end_offset=NULL` on CAgg refresh policy * @pierreforstmann for fixing a bug that happened when dropping chunks with foreign keys * @Zaczero for reporting a bug with CREATE TABLE WITH when using the TimescaleDB Apache 2 Edition ## 2.21.4 (2025-09-25) This release contains performance improvements and bug fixes since the 2.21.3 release. We recommend that you upgrade at the next available opportunity. **Bugfixes** * [#8667](https://github.com/timescale/timescaledb/pull/8667) Fix wrong selectivity estimates uncovered by the recent Postgres minor releases 15.14, 16.10, 17.6. ## 2.21.3 (2025-08-12) This release contains performance improvements and bug fixes since the 2.21.2 release. We recommend that you upgrade at the next available opportunity. **Bugfixes** * [#8471](https://github.com/timescale/timescaledb/pull/8471) Fix MERGE behaviour with updated values ## 2.21.2 (2025-08-05) This release contains performance improvements and bug fixes since the 2.21.1 release. We recommend that you upgrade at the next available opportunity. **Bugfixes** * [#8418](https://github.com/timescale/timescaledb/pull/8418) Fix duplicate constraints in JOIN queries * [#8426](https://github.com/timescale/timescaledb/pull/8426) Fix chunk skipping min/max calculation * [#8434](https://github.com/timescale/timescaledb/pull/8434) Allow `show_chunks` to process more than 65535 chunks ## 2.21.1 (2025-07-22) This release contains one bug fix since the 2.21.0 release. We recommend that you upgrade at the next available opportunity. **Bugfixes** * [#8336](https://github.com/timescale/timescaledb/pull/8336) Fix generic plans for foreign key checks and prepared statements **Thanks** * @CodeTherapist for reporting the issue with foreign key checks not working after several `INSERT` statements ## 2.21.0 (2025-07-08) This release contains performance improvements and bug fixes since the 2.20.3 release. We recommend that you upgrade at the next available opportunity. **Highlighted features in TimescaleDB v2.21.0** * The attach & detach chunks feature allows manually adding or removing chunks from a hypertable with uncompressed chunks, similar to PostgreSQL’s partition management. * Continued improvement of backfilling into the columnstore, achieving up to 2.5x speedup for constrained tables, by introducing caching logic that boosts throughput for writes to compressed chunks, bringing `INSERT` performance close to that of uncompressed chunks. * Optimized `DELETE` operations on the columstore through batch-level deletions of non-segmentby keys in the filter condition, greatly improving performance to up to 42x faster in some cases, as well as reducing bloat, and lowering resource usage. * The heavy lock taken in Continuous Aggregate refresh was relaxed, enabling concurrent refreshes for non-overlapping ranges and eliminating the need for complex customer workarounds. * [tech preview] Direct Compress is an innovative TimescaleDB feature that improves high-volume data ingestion by compressing data in memory and writing it directly to disk, reducing I/O overhead, eliminating dependency on background compression jobs, and significantly boosting insert performance. **Sunsetting of the hypercore access method** We made the decision to deprecate hypercore access method (TAM) with the 2.21.0 release. It was an experiment, which did not show the signals we hoped for and will be sunsetted in TimescaleDB 2.22.0, scheduled for September 2025. Upgrading to 2.22.0 and higher will be blocked if TAM is still in use. Since TAM’s inception in [2.18.0](https://github.com/timescale/timescaledb/releases/tag/2.18.0), we learned that btrees were not the right architecture. The recent advancements in the columnstore—such as more performant backfilling, SkipScan, adding check constraints, and faster point queries—put the [columnstore](https://www.timescale.com/blog/hypercore-a-hybrid-row-storage-engine-for-real-time-analytics) close to or on par with TAM without the storage from the additional index. We apologize for the inconvenience this action potentially causes and are here to assist you during the migration process. Migration path ``` do $$ declare relid regclass; begin for relid in select cl.oid from pg_class cl join pg_am am on (am.oid = cl.relam) where am.amname = 'hypercore' loop raise notice 'converting % to heap', relid::regclass; execute format('alter table %s set access method heap', relid); end loop; end $$; ``` **Features** * [#8081](https://github.com/timescale/timescaledb/pull/8081) Use JSON error code for job configuration parsing * [#8100](https://github.com/timescale/timescaledb/pull/8100) Support splitting compressed chunks * [#8131](https://github.com/timescale/timescaledb/pull/8131) Add policy to process hypertable invalidations * [#8141](https://github.com/timescale/timescaledb/pull/8141) Add function to process hypertable invalidations * [#8165](https://github.com/timescale/timescaledb/pull/8165) Reindex recompressed chunks in compression policy * [#8178](https://github.com/timescale/timescaledb/pull/8178) Add columnstore option to `CREATE TABLE WITH` * [#8179](https://github.com/timescale/timescaledb/pull/8179) Implement direct `DELETE` on non-segmentby columns * [#8182](https://github.com/timescale/timescaledb/pull/8182) Cache information for repeated upserts into the same compressed chunk * [#8187](https://github.com/timescale/timescaledb/pull/8187) Allow concurrent Continuous Aggregate refreshes * [#8191](https://github.com/timescale/timescaledb/pull/8191) Add option to not process hypertable invalidations * [#8196](https://github.com/timescale/timescaledb/pull/8196) Show deprecation warning for TAM * [#8208](https://github.com/timescale/timescaledb/pull/8208) Use `NULL` compression for bool batches with all null values like the other compression algorithms * [#8223](https://github.com/timescale/timescaledb/pull/8223) Support for attach/detach chunk * [#8265](https://github.com/timescale/timescaledb/pull/8265) Set incremental Continous Aggregate refresh policy on by default * [#8274](https://github.com/timescale/timescaledb/pull/8274) Allow creating concurrent continuous aggregate refresh policies * [#8314](https://github.com/timescale/timescaledb/pull/8314) Add support for timescaledb_lake in loader * [#8209](https://github.com/timescale/timescaledb/pull/8209) Add experimental support for Direct Compress of `COPY` * [#8341](https://github.com/timescale/timescaledb/pull/8341) Allow quick migration from hypercore TAM to (columnstore) heap **Bugfixes** * [#8153](https://github.com/timescale/timescaledb/pull/8153) Restoring a database having NULL compressed data * [#8164](https://github.com/timescale/timescaledb/pull/8164) Check columns when creating new chunk from table * [#8294](https://github.com/timescale/timescaledb/pull/8294) The "vectorized predicate called for a null value" error for WHERE conditions like `x = any(null::int[])`. * [#8307](https://github.com/timescale/timescaledb/pull/8307) Fix missing catalog entries for bool and null compression in fresh installations * [#8323](https://github.com/timescale/timescaledb/pull/8323) Fix DML issue with expression indexes and BHS **GUCs** * `enable_direct_compress_copy`: Enable experimental support for direct compression during `COPY`, default: off * `enable_direct_compress_copy_sort_batches`: Enable batch sorting during direct compress `COPY`, default: on * `enable_direct_compress_copy_client_sorted`: Correct handling of data sorting by the user is required for this option, default: off ## 2.20.3 (2025-06-11) This release contains bug fixes since the 2.20.2 release. We recommend that you upgrade at the next available opportunity. **Bugfixes** * [#8107](https://github.com/timescale/timescaledb/pull/8107) Adjust SkipScan cost for quals needing full scan of indexed data. * [#8211](https://github.com/timescale/timescaledb/pull/8211) Fixed dump and restore when chunk skipping is enabled. * [#8216](https://github.com/timescale/timescaledb/pull/8216) Fix for dropped quals bug in SkipScan. * [#8230](https://github.com/timescale/timescaledb/pull/8230) Fix for inserting into compressed data when vectorised check is not available. * [#8236](https://github.com/timescale/timescaledb/pull/8236) Fixed the snapshot handling in background workers. **Thanks** * @ikaakkola for reporting that SkipScan is slow when non-index quals do not match any tuples. ## 2.20.2 (2025-06-02) This release contains bug fixes since the 2.20.1 release. We recommend that you upgrade at the next available opportunity. **Bugfixes** * [#8200](https://github.com/timescale/timescaledb/pull/8202) Fix `NULL` compression handling for vectorized constraint checking ## 2.20.1 (2025-05-27) This release contains performance improvements and bug fixes since the 2.20.0 release. We recommend that you upgrade at the next available opportunity. **Features** * [#8145](https://github.com/timescale/timescaledb/pull/8145) Log only if compression ratio warnings are enabled **Bugfixes** * [#7292](https://github.com/timescale/timescaledb/pull/7292) Intermittent "could not find pathkey item to sort" error when grouping or ordering by a time_bucket of an equality join variable. * [#8126](https://github.com/timescale/timescaledb/pull/8126) Allow setting bgw_log_level to FATAL and ERROR * [#8151](https://github.com/timescale/timescaledb/pull/8151) Treat null equal to null for merged CAgg refresh * [#8153](https://github.com/timescale/timescaledb/pull/8153) Restoring a database having NULL compressed data * [#8162](https://github.com/timescale/timescaledb/pull/8162) Fix setting compress_chunk_interval on continuous aggregates * [#8163](https://github.com/timescale/timescaledb/pull/8163) Fix gapfill crash with locf NULL values treated as missing * [#8171](https://github.com/timescale/timescaledb/pull/8171) Disable decompression limit during continuous aggregate refresh **Thanks** * @bobozaur, @kvc0, @ChadMoran, @PaddyKe for reporting the pathkey error. * @jlordiales for reporting an issue with setting compress_chunk_interval for continuous aggregates * @svanharmelen, @cmdjulian, @etaMS20 for reporting time_bucket_gapfill with locf crash when NULL values are treated as missing ## 2.20.0 (2025-05-15) This release contains performance improvements and bug fixes since the 2.19.3 release. We recommend that you upgrade at the next available opportunity. **Highlighted features in TimescaleDB v2.20.0** * The columnstore now leverages *bloom filters* to deliver up to 6x faster point queries on columns with high cardinality values, such as UUIDs. * Major *improvements to the columnstores' backfill process* enable `UPSERTS` with strict constraints to execute up to 10x faster. * *SkipScan is now supported in the columnstore*, including for DISTINCT queries. This enhancement leads to dramatic query performance improvements of 2000x to 2500x, especially for selective queries. * SIMD vectorization for the bool data type is now enabled by default. This change results in a 30–45% increase in performance for analytical queries with bool clauses on the columnstore. * *Continuous aggregates* now include experimental support for *window functions and non-immutable functions*, extending the analytics use cases they can solve. * Several quality-of-life improvements have been introduced: job names for continuous aggregates are now more descriptive, you can assign custom names to them, and it is now possible to add unique constraints along with `ADD COLUMN` operations in the columnstore. * Improved management and optimization of chunks with the ability to split large uncompressed chunks at a specified point in time using the `split_chunk` function. This new function complements the existing `merge_chunk` function that can be used to merge two small chunks into one larger chunk. * Enhancements to the default behavior of the columnstore now provide better *automatic assessments* of `segment by` and `order by` columns, reducing the need for manual configuration and simplifying initial setup. **PostgreSQL 14 support removal announcement** Following the deprecation announcement for PostgreSQL 14 in TimescaleDB v2.19.0, PostgreSQL 14 is no longer supported in TimescaleDB v2.20.0. The currently supported PostgreSQL major versions are 15, 16, and 17. **Features** * [#7638](https://github.com/timescale/timescaledb/pull/7638) Bloom filter sparse indexes for compressed columns. Can be disabled with the GUC `timescaledb.enable_sparse_index_bloom` * [#7756](https://github.com/timescale/timescaledb/pull/7756) Add warning for poor compression ratio * [#7762](https://github.com/timescale/timescaledb/pull/7762) Speed up the queries that use minmax sparse indexes on compressed tables by changing the index TOAST storage type to `MAIN`. This applies to newly compressed chunks * [#7785](https://github.com/timescale/timescaledb/pull/7785) Do `DELETE` instead of `TRUNCATE` when locks aren't acquired * [#7852](https://github.com/timescale/timescaledb/pull/7852) Allow creating foreign key constraints on compressed tables * [#7854](https://github.com/timescale/timescaledb/pull/7854) Remove support for PG14 * [#7864](https://github.com/timescale/timescaledb/pull/7854) Allow adding CHECK constraints to compressed chunks * [#7868](https://github.com/timescale/timescaledb/pull/7868) Allow adding columns with `CHECK` constraints to compressed chunks * [#7874](https://github.com/timescale/timescaledb/pull/7874) Support for SkipScan for distinct aggregates over the same column * [#7877](https://github.com/timescale/timescaledb/pull/7877) Remove blocker for unique constraints with `ADD COLUMN` * [#7878](https://github.com/timescale/timescaledb/pull/7878) Don't block non-immutable functions in continuous aggregates * [#7880](https://github.com/timescale/timescaledb/pull/7880) Add experimental support for window functions in continuous aggregates * [#7899](https://github.com/timescale/timescaledb/pull/7899) Vectorized decompression and filtering for boolean columns * [#7915](https://github.com/timescale/timescaledb/pull/7915) New option `refresh_newest_first` to continuous aggregate refresh policy API * [#7917](https://github.com/timescale/timescaledb/pull/7917) Remove `_timescaledb_functions.create_chunk_table` function * [#7929](https://github.com/timescale/timescaledb/pull/7929) Add `CREATE TABLE ... WITH` API for creating hypertables * [#7946](https://github.com/timescale/timescaledb/pull/7946) Add support for splitting a chunk * [#7958](https://github.com/timescale/timescaledb/pull/7958) Allow custom names for jobs * [#7972](https://github.com/timescale/timescaledb/pull/7972) Add vectorized filtering for constraint checking while backfilling into compressed chunks * [#7976](https://github.com/timescale/timescaledb/pull/7976) Include continuous aggregate name in jobs informational view * [#7977](https://github.com/timescale/timescaledb/pull/7977) Replace references to compression with columnstore * [#7981](https://github.com/timescale/timescaledb/pull/7981) Add columnstore as alias for `enable_columnstore `in `ALTER TABLE` * [#7983](https://github.com/timescale/timescaledb/pull/7983) Support for SkipScan over compressed data * [#7991](https://github.com/timescale/timescaledb/pull/7991) Improves default `segmentby` options * [#7992](https://github.com/timescale/timescaledb/pull/7992) Add API into hypertable invalidation log * [#8000](https://github.com/timescale/timescaledb/pull/8000) Add primary dimension info to information schema * [#8005](https://github.com/timescale/timescaledb/pull/8005) Support `ALTER TABLE SET (timescaledb.chunk_time_interval='1 day')` * [#8012](https://github.com/timescale/timescaledb/pull/8012) Add event triggers support on chunk creation * [#8014](https://github.com/timescale/timescaledb/pull/8014) Enable bool compression by default by setting `timescaledb.enable_bool_compression=true`. Note: for downgrading to `2.18` or earlier version, use [this downgrade script](https://github.com/timescale/timescaledb-extras/blob/master/utils/2.19.0-downgrade_new_compression_algorithms.sql) * [#8018](https://github.com/timescale/timescaledb/pull/8018) Add spin-lock during recompression on unique constraints * [#8026](https://github.com/timescale/timescaledb/pull/8026) Allow `WHERE` conditions that use nonvolatile functions to be pushed down to the compressed scan level. For example, conditions like `time > now()`, where `time` is a columnstore `orderby` column, will evaluate `now()` and use the sparse index on `time` to filter out the entire compressed batches that cannot contain matching rows. * [#8027](https://github.com/timescale/timescaledb/pull/8027) Add materialization invalidations API * [#8047](https://github.com/timescale/timescaledb/pull/8027) Support SkipScan for `SELECT DISTINCT` with multiple distincts when all but one distinct is pinned * [#8115](https://github.com/timescale/timescaledb/pull/8115) Add batch size limiting during compression **Bugfixes** * [#7862](https://github.com/timescale/timescaledb/pull/7862) Release cache pin when checking for `NOT NULL` * [#7909](https://github.com/timescale/timescaledb/pull/7909) Update compression stats when merging chunks * [#7928](https://github.com/timescale/timescaledb/pull/7928) Don't create a hypertable for implicitly published tables * [#7982](https://github.com/timescale/timescaledb/pull/7982) Fix crash in batch sort merge over eligible expressions * [#8008](https://github.com/timescale/timescaledb/pull/8008) Fix compression policy error message that shows number of successes * [#8031](https://github.com/timescale/timescaledb/pull/8031) Fix reporting of deleted tuples for direct batch delete * [#8033](https://github.com/timescale/timescaledb/pull/8033) Skip default `segmentby` if `orderby` is explicitly set * [#8061](https://github.com/timescale/timescaledb/pull/8061) Ensure settings for a compressed relation are found * [#7515](https://github.com/timescale/timescaledb/pull/7515) Add missing lock to Constraint-aware append * [#8067](https://github.com/timescale/timescaledb/pull/8067) Make sure hypercore TAM parent is vacuumed * [#8074](https://github.com/timescale/timescaledb/pull/8074) Fix memory leak in row compressor flush * [#8099](https://github.com/timescale/timescaledb/pull/8099) Block chunk merging on multi-dimensional hypertables * [#8106](https://github.com/timescale/timescaledb/pull/8106) Fix segfault when adding unique compression indexes to compressed chunks * [#8127](https://github.com/timescale/timescaledb/pull/8127) Read bit-packed version of booleans **GUCs** * `timescaledb.enable_sparse_index_bloom`: Enable creation of the bloom1 sparse index on compressed chunks; Default: `ON` * `timescaledb.compress_truncate_behaviour`: Defines how truncate behaves at the end of compression; Default: `truncate_only` * `timescaledb.enable_compression_ratio_warnings`: Enable warnings for poor compression ratio; Default: `ON` * `timescaledb.enable_event_triggers`: Enable event triggers for chunks creation; Default: `OFF` * `timescaledb.enable_cagg_window_functions`: Enable window functions in continuous aggregates; Default: `OFF` **Thanks** * @arajkumar for reporting that implicitly published tables were still able to create hypertables * @thotokraa for reporting an issue with unique expression indexes on compressed chunks ## 2.19.3 (2025-04-15) This release contains bug fixes since the 2.19.2 release. We recommend that you upgrade at the next available opportunity. **Bug fixes** * [#7893](https://github.com/timescale/timescaledb/pull/7893) Don't capture hard errors in with-clause parser * [#7903](https://github.com/timescale/timescaledb/pull/7903) Don't capture hard errors for old cagg format * [#7912](https://github.com/timescale/timescaledb/pull/7912) Don't capture errors estimating time max spread * [#7910](https://github.com/timescale/timescaledb/pull/7910) Fix not using SkipScan over one chunk * [#7913](https://github.com/timescale/timescaledb/pull/7913) Allow TAM chunk creation as non-owner * [#7935](https://github.com/timescale/timescaledb/pull/7935) Fix TAM segfault on DELETE using segmentby column * [#7954](https://github.com/timescale/timescaledb/pull/7954) Allow scheduler restarts to be disabled * [#7964](https://github.com/timescale/timescaledb/pull/7964) Crash when grouping by multiple columns of a compressed table, one of which is a UUID segmentby column. ## 2.19.2 (2025-04-07) This release contains bug fixes since the 2.19.1 release. We recommend that you upgrade at the next available opportunity. **Features** * [#7923](https://github.com/timescale/timescaledb/pull/7923) Add a GUC to set a compression batch size limit **Bugfixes** * [#7911](https://github.com/timescale/timescaledb/pull/7911) Don't create a Hypertable for published tables * [#7902](https://github.com/timescale/timescaledb/pull/7902) Fix crash in VectorAgg plan code **GUCs** * `compression_batch_size_limit`: set batch size limit, default: `1000` **Thanks** * @soedirgo for reporting that published tables don't get dropped when they have a hypertable* ## 2.19.1 (2025-04-01) This release contains bug fixes since the 2.19.0 release. We recommend that you upgrade at the next available opportunity. **Bugfixes** * [#7816](https://github.com/timescale/timescaledb/pull/7816) Fix `ORDER BY` for direct queries on partial chunks * [#7834](https://github.com/timescale/timescaledb/pull/7834) Avoid unnecessary scheduler restarts * [#7837](https://github.com/timescale/timescaledb/pull/7837) Ignore frozen chunks in compression policy * [#7850](https://github.com/timescale/timescaledb/pull/7850) Add `is_current_xact_tuple` to Arrow TTS * [#7890](https://github.com/timescale/timescaledb/pull/7890) Flush error state before doing anything else **Thanks** * @bjornuppeke for reporting a problem with `INSERT INTO ... ON CONFLICT DO NOTHING` on compressed chunks * @kav23alex for reporting a segmentation fault on `ALTER TABLE` with `DEFAULT` ## 2.19.0 (2025-03-18) This release contains performance improvements and bug fixes since the 2.18.2 release. We recommend that you upgrade at the next available opportunity. * Improved concurrency of `INSERT`, `UPDATE`, and `DELETE` operations on the columnstore by no longer blocking DML statements during the recompression of a chunk. * Improved system performance during continuous aggregate refreshes by breaking them into smaller batches. This reduces systems pressure and minimizes the risk of spilling to disk. * Faster and more up-to-date results for queries against continuous aggregates by materializing the most recent data first, as opposed to old data first in prior versions. * Faster analytical queries with SIMD vectorization of aggregations over text columns and `GROUP BY` over multiple columns. * Enable optimizing the chunk size for better query performance in the columnstore by merging them with `merge_chunk`. **Deprecation warning** This is the last minor release supporting PostgreSQL 14. Starting with the next minor version of TimescaleDB, only Postgres 15, 16, and 17 will be supported. **Downgrading of 2.19.0** This release introduces custom bool compression. If you enable this feature with `enable_bool_compression` and need to revert it, use [this script](https://github.com/timescale/timescaledb-extras/blob/master/utils/2.19.0-downgrade_new_compression_algorithms.sql) to convert the columns back to their previous state. TimescaleDB versions prior to 2.19.0 do not support this new type. **Features** * [#7586](https://github.com/timescale/timescaledb/pull/7586) Vectorized aggregation with grouping by a single text column * [#7632](https://github.com/timescale/timescaledb/pull/7632) Optimize recompression for chunks without segmentby * [#7655](https://github.com/timescale/timescaledb/pull/7655) Support vectorized aggregation on hypercore TAM * [#7669](https://github.com/timescale/timescaledb/pull/7669) Add support for merging compressed chunks * [#7701](https://github.com/timescale/timescaledb/pull/7701) Implement a custom compression algorithm for bool columns. It is in early access and can undergo backwards-incompatible changes. For testing, enable it using `timescaledb.enable_bool_compression = on` * [#7707](https://github.com/timescale/timescaledb/pull/7707) Support `ALTER COLUMN SET NOT NULL` on compressed chunks * [#7765](https://github.com/timescale/timescaledb/pull/7765) Allow `tsdb` as alias for `timescaledb` in `WITH` and `SET` clauses * [#7786](https://github.com/timescale/timescaledb/pull/7786) Show a warning for inefficient `compress_chunk_time_interval` configuration * [#7788](https://github.com/timescale/timescaledb/pull/7788) Add a callback to `mem_guard` for background workers * [#7789](https://github.com/timescale/timescaledb/pull/7789) Do not recompress segmentwise when default order by is empty * [#7790](https://github.com/timescale/timescaledb/pull/7790) Add a configurable incremental CAgg refresh policy **Bugfixes** * [#7665](https://github.com/timescale/timescaledb/pull/7665) Block merging of frozen chunks * [#7673](https://github.com/timescale/timescaledb/pull/7673) Don't abort additional `INSERT`s when hitting the first conflict * [#7714](https://github.com/timescale/timescaledb/pull/7714) Fix the wrong result when compressed `NULL` values were confused with default values. This happened for a very particular order of events. * [#7747](https://github.com/timescale/timescaledb/pull/7747) Block TAM rewrites with incompatible GUC setting * [#7748](https://github.com/timescale/timescaledb/pull/7748) Crash in the segmentwise recompression * [#7764](https://github.com/timescale/timescaledb/pull/7764) Fix compression settings handling in hypercore TAM * [#7768](https://github.com/timescale/timescaledb/pull/7768) Remove costing index scan of hypertable parent * [#7799](https://github.com/timescale/timescaledb/pull/7799) Handle the `DEFAULT` table access name in `ALTER TABLE` **GUCs** * `enable_bool_compression`: enable the BOOL compression algorithm, default: `OFF` * `enable_exclusive_locking_recompression`: enable exclusive locking during recompression (legacy mode), default: `OFF` **Thanks** * @bjornuppeke for reporting a problem with `INSERT INTO ... ON CONFLICT DO NOTHING` on compressed chunks * @kav23alex for reporting a segmentation fault on `ALTER TABLE with DEFAULT` ## 2.18.2 (2025-02-19) This release contains performance improvements and bug fixes since the 2.18.1 release. We recommend that you upgrade at the next available opportunity. **Bugfixes** * [#7686](https://github.com/timescale/timescaledb/pull/7686) Potential wrong aggregation result when using vectorized aggregation with hash grouping in reverse order * [#7694](https://github.com/timescale/timescaledb/pull/7694) Fix ExplainHook breaking call chain * [#7695](https://github.com/timescale/timescaledb/pull/7695) Block dropping internal compressed chunks with `drop_chunk()` * [#7711](https://github.com/timescale/timescaledb/pull/7711) License error when using hypercore handler * [#7712](https://github.com/timescale/timescaledb/pull/7712) Respect other extensions' ExecutorStart hooks **Thanks** * @davidmehren and @jflambert for reporting an issue with extension hooks * @jflambert for reporting a bug with license errors shown in autovacuum ## 2.18.1 (2025-02-10) This release contains performance improvements and bug fixes since the 2.18.0 release. We recommend that you upgrade at the next available opportunity. **Features** * [#7656](https://github.com/timescale/timescaledb/pull/7656) Remove limitation of compression policy for continuous aggregates **Bugfixes** * [#7600](https://github.com/timescale/timescaledb/pull/7600) Fix lock order when dropping index * [#7637](https://github.com/timescale/timescaledb/pull/7637) Allow EXPLAIN in read-only mode * [#7645](https://github.com/timescale/timescaledb/pull/7645) Fix DELETE on compressed chunk with non-btree operators * [#7649](https://github.com/timescale/timescaledb/pull/7649) Allow non-btree operator pushdown in UPDATE/DELETE queries on compressed chunks * [#7653](https://github.com/timescale/timescaledb/pull/7653) Push down orderby scankeys to Hypercore TAM * [#7665](https://github.com/timescale/timescaledb/pull/7665) Block merging of frozen chunks * [#7673](https://github.com/timescale/timescaledb/pull/7673) Don't abort additional INSERTs when hitting first conflict **GUCs** * `enable_hypercore_scankey_pushdown`: Push down qualifiers as scankeys when using Hypercore TAM introduced with [#7653](https://github.com/timescale/timescaledb/pull/7653) **Thanks** * @bjornuppeke for reporting a problem with INSERT INTO ... ON CONFLICT DO NOTHING on compressed chunks * @ikalafat for reporting a problem with EXPLAIN in read-only mode * Timescale community members Jacob and pantonis for reporting issues with slow queries. ## 2.18.0 (2025-01-23) This release introduces the ability to add secondary indexes to the columnstore, improves group by and filtering performance through columnstore vectorization, and contains the highly upvoted community request of transition table support. We recommend that you upgrade at the next available opportunity. **Highlighted features in TimescaleDB v2.18.0** * The ability to add secondary indexes to the columnstore through the new hypercore table access method. * Significant performance improvements through vectorization (`SIMD`) for aggregations using a group by with one column and/or using a filter clause when querying the columnstore. * Hypertables support triggers for transition tables, which is one of the most upvoted community feature requests. * Updated methods to manage Timescale's hybrid row-columnar store (hypercore) that highlight the usage of the columnstore which includes both an optimized columnar format as well as compression. **Dropping support for Bitnami images** After the recent change in Bitnami’s [LTS support policy](https://github.com/bitnami/containers/issues/75671), we are no longer building Bitnami images for TimescaleDB. We recommend using the [official TimescaleDB Docker image](https://hub.docker.com/r/timescale/timescaledb-ha) **Deprecation Notice** We are deprecating the following parameters, functions, procedures and views. They will be removed with the next major release of TimescaleDB. Please find the replacements in the table below: | Deprecated | Replacement | Type | | --- | --- | --- | | decompress_chunk | convert_to_rowstore | Procedure | | compress_chunk | convert_to_columnstore | Procedure | | add_compression_policy | add_columnstore_policy | Function | | remove_compression_policy | remove_columnstore_policy | Function | | hypertable_compression_stats | hypertable_columnstore_stats | Function | | chunk_compression_stats | chunk_columnstore_stats | Function | | hypertable_compression_settings | hypertable_columnstore_settings | View | | chunk_compression_settings | chunk_columnstore_settings | View | | compression_settings | columnstore_settings | View | | timescaledb.compress | timescaledb.enable_columnstore | Parameter | | timescaledb.compress_segmentby | timescaledb.segmentby | Parameter | | timescaledb.compress_orderby | timescaledb.orderby | Parameter | **Features** * #7341: Vectorized aggregation with grouping by one fixed-size by-value compressed column (such as arithmetic types). * #7104: Hypercore table access method. * #6901: Add hypertable support for transition tables. * #7482: Optimize recompression of partially compressed chunks. * #7458: Support vectorized aggregation with aggregate `filter` clauses that are also vectorizable. * #7433: Add support for merging chunks. * #7271: Push down `order by` in real-time continuous aggregate queries. * #7455: Support `drop not null` on compressed hypertables. * #7295: Support `alter table set access method` on hypertable. * #7411: Change parameter name to enable hypercore table access method. * #7436: Add index creation on `order by` columns. * #7443: Add hypercore function and view aliases. * #7521: Add optional `force` argument to `refresh_continuous_aggregate`. * #7528: Transform sorting on `time_bucket` to sorting on time for compressed chunks in some cases. * #7565: Add hint when hypertable creation fails. * #7390: Disable custom `hashagg` planner code. * #7587: Add `include_tiered_data` parameter to `add_continuous_aggregate_policy` API. * #7486: Prevent building against PostgreSQL versions with broken ABI. * #7412: Add [GUC](https://www.postgresql.org/docs/current/acronyms.html#:~:text=GUC) for the `hypercore_use_access_method` default. * #7413: Add GUC for segmentwise recompression. **Bugfixes** * #7378: Remove obsolete job referencing `policy_job_error_retention`. * #7409: Update `bgw_job` table when altering procedure. * #7410: Fix the `aggregated compressed column not found` error on aggregation query. * #7426: Fix `datetime` parsing error in chunk constraint creation. * #7432: Verify that the heap tuple is valid before using. * #7434: Fix the segfault when internally setting the replica identity for a given chunk. * #7488: Emit error for transition table trigger on chunks. * #7514: Fix the error: `invalid child of chunk append`. * #7517: Fix the performance regression on the `cagg_migrate` procedure. * #7527: Restart scheduler on error. * #7557: Fix null handling for in-memory tuple filtering. * #7566: Improve transaction check in CAGG refresh. * #7584: Fix NaN-handling for vectorized aggregation. * #7598: Match the Postgres NaN comparison behavior in WHERE clause over compressed tables. **Thanks** * @bharrisau for reporting the segfault when creating chunks. * @jakehedlund for reporting the incompatible NaN behavior in WHERE clause over compressed tables. * @k-rus for suggesting that we add a hint when hypertable creation fails. * @pgloader for reporting the issue in an internal background job. * @staticlibs for sending the pull request that improves the transaction check in CAGG refresh. * @uasiddiqi for reporting the `aggregated compressed column not found` error. ## 2.17.2 (2024-11-06) This release contains bug fixes since the 2.17.1 release. We recommend that you upgrade at the next available opportunity. **Bugfixes** * #7384 Fix "negative bitmapset member not allowed" and performance degradation on queries to compressed tables with ORDER BY clause matching the order of the compressed data * #7388 Use-after-free in vectorized grouping by segmentby columns **Thanks** * @dx034 for reporting an issue with negative bitmapset members due to large OIDs ## 2.17.1 (2024-10-21) This release contains performance improvements and bug fixes since the 2.17.0 release. We recommend that you upgrade at the next available opportunity. **Features** * #7360 Add chunk skipping GUC **Bugfixes** * #7335 Change log level used in compression * #7342 Fix collation for in-memory tuple filtering **Thanks** * @gmilamjr for reporting an issue with the log level of compression messages * @hackbnw for reporting an issue with collation during tuple filtering ## 2.17.0 (2024-10-08) This release adds support for PostgreSQL 17, significantly improves the performance of continuous aggregate refreshes, and contains performance improvements for analytical queries and delete operations over compressed hypertables. We recommend that you upgrade at the next available opportunity. **Highlighted features in TimescaleDB v2.17.0** * Full PostgreSQL 17 support for all existing features. TimescaleDB v2.17 is available for PostgreSQL 14, 15, 16, and 17. * Significant performance improvements for continuous aggregate policies: continuous aggregate refresh is now using `merge` instead of deleting old materialized data and re-inserting. This update can decrease dramatically the amount of data that must be written on the continuous aggregate in the presence of a small number of changes, reduce the `i/o` cost of refreshing a continuous aggregate, and generate fewer Write-Ahead Logs (`WAL`). Overall, continuous aggregate policies will be more lightweight, use less system resources, and complete faster. * Increased performance for real-time analytical queries over compressed hypertables: we are excited to introduce additional Single Instruction, Multiple Data (`SIMD`) vectorization optimization to our engine by supporting vectorized execution for queries that group by using the `segment_by` column(s) and aggregate using the basic aggregate functions (`sum`, `count`, `avg`, `min`, `max`). Stay tuned for more to come in follow-up releases! Support for grouping on additional columns, filtered aggregation, vectorized expressions, and `time_bucket` is coming soon. * Improved performance of deletes on compressed hypertables when a large amount of data is affected. This improvement speeds up operations that delete whole segments by skipping the decompression step. It is enabled for all deletes that filter by the `segment_by` column(s). **PostgreSQL 14 deprecation announcement** We will continue supporting PostgreSQL 14 until April 2025. Closer to that time, we will announce the specific version of TimescaleDB in which PostgreSQL 14 support will not be included going forward. **Features** * #6882: Allow delete of full segments on compressed chunks without decompression. * #7033: Use `merge` statement on continuous aggregates refresh. * #7126: Add functions to show the compression information. * #7147: Vectorize partial aggregation for `sum(int4)` with grouping on `segment by` columns. * #7204: Track additional extensions in telemetry. * #7207: Refactor the `decompress_batches_scan` functions for easier maintenance. * #7209: Add a function to drop the `osm` chunk. * #7275: Add support for the `returning` clause for `merge`. * #7200: Vectorize common aggregate functions like `min`, `max`, `sum`, `avg`, `stddev`, `variance` for compressed columns of arithmetic types, when there is grouping on `segment by` columns or no grouping. **Bug fixes** * #7187: Fix the string literal length for the `compressed_data_info` function. * #7191: Fix creating default indexes on chunks when migrating the data. * #7195: Fix the `segment by` and `order by` checks when dropping a column from a compressed hypertable. * #7201: Use the generic extension description when building `apt` and `rpm` loader packages. * #7227: Add an index to the `compression_chunk_size` catalog table. * #7229: Fix the foreign key constraints where the index and the constraint column order are different. * #7230: Do not propagate the foreign key constraints to the `osm` chunk. * #7234: Release the cache after accessing the cache entry. * #7258: Force English in the `pg_config` command executed by `cmake` to avoid the unexpected building errors. * #7270: Fix the memory leak in compressed DML batch filtering. * #7286: Fix the index column check while searching for the index. * #7290: Add check for null offset for continuous aggregates built on top of continuous aggregates. * #7301: Make foreign key behavior for hypertables consistent. * #7318: Fix chunk skipping range filtering. * #7320: Set the license specific extension comment in the install script. **Thanks** * @MiguelTubio for reporting and fixing the Windows build error. * @posuch for reporting the misleading extension description in the generic loader packages. * @snyrkill for discovering and reporting the issue with continuous aggregates built on top of continuous aggregates. ## 2.16.1 (2024-08-06) This release contains bug fixes since the 2.16.0 release. We recommend that you upgrade at the next available opportunity. **Bugfixes** * #7182 Fix untier_chunk for hypertables with foreign keys ## 2.16.0 (2024-07-31) This release contains significant performance improvements when working with compressed data, extended join support in continuous aggregates, and the ability to define foreign keys from regular tables towards hypertables. We recommend that you upgrade at the next available opportunity. In TimescaleDB v2.16.0 we: * Introduce multiple performance focused optimizations for data manipulation operations (DML) over compressed chunks. Improved upsert performance by more than 100x in some cases and more than 1000x in some update/delete scenarios. * Add the ability to define chunk skipping indexes on non-partitioning columns of compressed hypertables TimescaleDB v2.16.0 extends chunk exclusion to use those skipping (sparse) indexes when queries filter on the relevant columns, and prune chunks that do not include any relevant data for calculating the query response. * Offer new options for use cases that require foreign keys defined. You can now add foreign keys from regular tables towards hypertables. We have also removed some really annoying locks in the reverse direction that blocked access to referenced tables while compression was running. * Extend Continuous Aggregates to support more types of analytical queries. More types of joins are supported, additional equality operators on join clauses, and support for joins between multiple regular tables. **Highlighted features in this release** * Improved query performance through chunk exclusion on compressed hypertables. You can now define chunk skipping indexes on compressed chunks for any column with one of the following integer data types: `smallint`, `int`, `bigint`, `serial`, `bigserial`, `date`, `timestamp`, `timestamptz`. After you call `enable_chunk_skipping` on a column, TimescaleDB tracks the min and max values for that column. TimescaleDB uses that information to exclude chunks for queries that filter on that column, and would not find any data in those chunks. * Improved upsert performance on compressed hypertables. By using index scans to verify constraints during inserts on compressed chunks, TimescaleDB speeds up some ON CONFLICT clauses by more than 100x. * Improved performance of updates, deletes, and inserts on compressed hypertables. By filtering data while accessing the compressed data and before decompressing, TimescaleDB has improved performance for updates and deletes on all types of compressed chunks, as well as inserts into compressed chunks with unique constraints. By signaling constraint violations without decompressing, or decompressing only when matching records are found in the case of updates, deletes and upserts, TimescaleDB v2.16.0 speeds up those operations more than 1000x in some update/delete scenarios, and 10x for upserts. * You can add foreign keys from regular tables to hypertables, with support for all types of cascading options. This is useful for hypertables that partition using sequential IDs, and need to reference those IDs from other tables. * Lower locking requirements during compression for hypertables with foreign keys Advanced foreign key handling removes the need for locking referenced tables when new chunks are compressed. DML is no longer blocked on referenced tables while compression runs on a hypertable. * Improved support for queries on Continuous Aggregates `INNER/LEFT` and `LATERAL` joins are now supported. Plus, you can now join with multiple regular tables, and you can have more than one equality operator on join clauses. **PostgreSQL 13 support removal announcement** Following the deprecation announcement for PostgreSQL 13 in TimescaleDB v2.13, PostgreSQL 13 is no longer supported in TimescaleDB v2.16. The Currently supported PostgreSQL major versions are 14, 15 and 16. **Features** * #6880: Add support for the array operators used for compressed DML batch filtering. * #6895: Improve the compressed DML expression pushdown. * #6897: Add support for replica identity on compressed hypertables. * #6918: Remove support for PG13. * #6920: Rework compression activity wal markers. * #6989: Add support for foreign keys when converting plain tables to hypertables. * #7020: Add support for the chunk column statistics tracking. * #7048: Add an index scan for INSERT DML decompression. * #7075: Reduce decompression on the compressed INSERT. * #7101: Reduce decompressions for the compressed UPDATE/DELETE. * #7108 Reduce decompressions for INSERTs with UNIQUE constraints * #7116 Use DELETE instead of TRUNCATE after compression * #7134 Refactor foreign key handling for compressed hypertables * #7161 Fix `mergejoin input data is out of order` **Bugfixes** * #6987 Fix REASSIGN OWNED BY for background jobs * #7018: Fix `search_path` quoting in the compression defaults function. * #7046: Prevent locking for compressed tuples. * #7055: Fix the `scankey` for `segment by` columns, where the type `constant` is different to `variable`. * #7064: Fix the bug in the default `order by` calculation in compression. * #7069: Fix the index column name usage. * #7074: Fix the bug in the default `segment by` calculation in compression. **Thanks** * @jledentu For reporting a problem with mergejoin input order ## 2.15.3 (2024-07-02) This release contains bug fixes since the 2.15.2 release. Best practice is to upgrade at the next available opportunity. **Migrating from self-hosted TimescaleDB v2.14.x and earlier** After you run `ALTER EXTENSION`, you must run [this SQL script](https://github.com/timescale/timescaledb-extras/blob/master/utils/2.15.X-fix_hypertable_foreign_keys.sql). For more details, see the following pull request [#6797](https://github.com/timescale/timescaledb/pull/6797). If you are migrating from TimescaleDB v2.15.0, v2.15.1 or v2.15.2, no changes are required. **Bugfixes** * #7035: Fix the error when acquiring a tuple lock on the OSM chunks on the replica. * #7061: Fix the handling of multiple unique indexes in a compressed INSERT. * #7080: Fix the `corresponding equivalence member not found` error. * #7088: Fix the leaks in the DML functions. * #7091: Fix ORDER BY/GROUP BY expression not found in targetlist on PG16 **Thanks** * @Kazmirchuk for reporting the issue about leaks with the functions in DML. ## 2.15.2 (2024-06-07) This release contains bug fixes since the 2.15.1 release. Best practice is to upgrade at the next available opportunity. **Migrating from self-hosted TimescaleDB v2.14.x and earlier** After you run `ALTER EXTENSION`, you must run [this SQL script](https://github.com/timescale/timescaledb-extras/blob/master/utils/2.15.X-fix_hypertable_foreign_keys.sql). For more details, see the following pull request [#6797](https://github.com/timescale/timescaledb/pull/6797). If you are migrating from TimescaleDB v2.15.0 or v2.15.1, no changes are required. **Bugfixes** * #6975: Fix sort pushdown for partially compressed chunks. * #6976: Fix removal of metadata function and update script. * #6978: Fix segfault in `compress_chunk` with a primary space partition. * #6993: Disallow hash partitioning on primary column. **Thanks** * @gugu for reporting the issue with catalog corruption due to update. * @srieding for reporting an issue with partially compressed chunks and ordering on joined columns. ## 2.15.1 (2024-05-28) This release contains bug fixes since the 2.15.0 release. Best practice is to upgrade at the next available opportunity. **Migrating from self-hosted TimescaleDB v2.14.x and earlier** After you run `ALTER EXTENSION`, you must run [this SQL script](https://github.com/timescale/timescaledb-extras/blob/master/utils/2.15.X-fix_hypertable_foreign_keys.sql). For more details, see the following pull request [#6797](https://github.com/timescale/timescaledb/pull/6797). If you are migrating from TimescaleDB v2.15.0, no changes are required. **Bugfixes** * #6540: Segmentation fault when you backfill data using COPY into a compressed chunk. * #6858: `BEFORE UPDATE` trigger not working correctly. * #6908: Fix `time_bucket_gapfill()` with timezone behaviour around daylight savings time (DST) switches. * #6911: Fix dropped chunk metadata removal in the update script. * #6940: Fix `pg_upgrade` failure by removing `regprocedure` from the catalog table. * #6957: Fix then `segfault` in UNION queries that contain ordering on compressed chunks. **Thanks** * @DiAifU, @kiddhombre and @intermittentnrg for reporting the issues with gapfill and daylight saving time. * @edgarzamora for reporting the issue with update triggers. * @hongquan for reporting the issue with the update script. * @iliastsa and @SystemParadox for reporting the issue with COPY into a compressed chunk. ## 2.15.0 (2024-05-08) This release contains performance improvements and bug fixes since the 2.14.2 release. Best practice is to upgrade at the next available opportunity. In addition, it includes these noteworthy features: * Support `time_bucket` with `origin` and/or `offset` on Continuous Aggregate * Compression improvements: - Improve expression pushdown - Add minmax sparse indexes when compressing columns with btree indexes - Improve compression setting defaults - Vectorize filters in WHERE clause that contain text equality operators and LIKE expressions **Deprecation warning** * Starting on this release will not be possible to create Continuous Aggregate using `time_bucket_ng` anymore and it will be completely removed on the upcoming releases. * Recommend users to [migrate their old Continuous Aggregate format to the new one](https://docs.timescale.com/use-timescale/latest/continuous-aggregates/migrate/) because it support will be completely removed in next releases prevent them to migrate. * This is the last release supporting PostgreSQL 13. **Migrating from self-hosted TimescaleDB v2.14.x and earlier** After you run `ALTER EXTENSION`, you must run [this SQL script](https://github.com/timescale/timescaledb-extras/blob/master/utils/2.15.X-fix_hypertable_foreign_keys.sql). For more details, see the following pull request [#6797](https://github.com/timescale/timescaledb/pull/6797). **Features** * #6382: Support for `time_bucket` with `origin` and `offset` in CAggs. * #6696: Improve defaults for compression `segment_by` and `order_by`. * #6705: Add sparse minmax indexes for compressed columns that have uncompressed btree indexes. * #6754: Allow `DROP CONSTRAINT` on compressed hypertables. * #6767: Add metadata table `_timestaledb_internal.bgw_job_stat_history` for tracking job execution history. * #6798: Prevent usage of deprecated `time_bucket_ng` in CAgg definition. * #6810: Add telemetry for access methods. * #6811: Remove no longer relevant `timescaledb.allow_install_without_preload` GUC. * #6837: Add migration path for CAggs using `time_bucket_ng`. * #6865: Update the watermark when truncating a CAgg. **Bugfixes** * #6617: Fix error in show_chunks. * #6621: Remove metadata when dropping chunks. * #6677: Fix snapshot usage in CAgg invalidation scanner. * #6698: Define meaning of 0 retries for jobs as no retries. * #6717: Fix handling of compressed tables with primary or unique index in COPY path. * #6726: Fix constify cagg_watermark using window function when querying a CAgg. * #6729: Fix NULL start value handling in CAgg refresh. * #6732: Fix CAgg migration with custom timezone / date format settings. * #6752: Remove custom autovacuum setting from compressed chunks. * #6770: Fix plantime chunk exclusion for OSM chunk. * #6789: Fix deletes with subqueries and compression. * #6796: Fix a crash involving a view on a hypertable. * #6797: Fix foreign key constraint handling on compressed hypertables. * #6816: Fix handling of chunks with no contraints. * #6820: Fix a crash when the ts_hypertable_insert_blocker was called directly. * #6849: Use non-orderby compressed metadata in compressed DML. * #6867: Clean up compression settings when deleting compressed cagg. * #6869: Fix compressed DML with constraints of form value OP column. * #6870: Fix bool expression pushdown for queries on compressed chunks. **Thanks** * @brasic for reporting a crash when the ts_hypertable_insert_blocker was called directly. * @bvanelli for reporting an issue with the jobs retry count. * @djzurawsk for reporting error when dropping chunks. * @Dzuzepppe for reporting an issue with DELETEs using subquery on compressed chunk working incorrectly. * @hongquan for reporting a `timestamp out of range` error during CAgg migrations. * @kevcenteno for reporting an issue with the show_chunks API showing incorrect output when `created_before/created_after` was used with time-partitioned columns. * @mahipv for starting working on the job history PR. * @rovo89 for reporting constify cagg_watermark not working using window function when querying a CAgg. ## 2.14.2 (2024-02-20) This release contains bug fixes since the 2.14.1 release. We recommend that you upgrade at the next available opportunity. **Bugfixes** * #6655 Fix segfault in cagg_validate_query * #6660 Fix refresh on empty CAgg with variable bucket * #6670 Don't try to compress osm chunks **Thanks** * @kav23alex for reporting a segfault in cagg_validate_query ## 2.14.1 (2024-02-14) This release contains bug fixes since the 2.14.0 release. We recommend that you upgrade at the next available opportunity. **Features** * #6630 Add views for per chunk compression settings **Bugfixes** * #6636 Fixes extension update of compressed hypertables with dropped columns * #6637 Reset sequence numbers on non-rollup compression * #6639 Disable default indexscan for compression * #6651 Fix DecompressChunk path generation with per chunk settings **Thanks** * @anajavi for reporting an issue with extension update of compressed hypertables ## 2.14.0 (2024-02-08) This release contains performance improvements and bug fixes since the 2.13.1 release. We recommend that you upgrade at the next available opportunity. In addition, it includes these noteworthy features: * Ability to change compression settings on existing compressed hypertables at any time. New compression settings take effect on any new chunks that are compressed after the change. * Reduced locking requirements during chunk recompression * Limiting tuple decompression during DML operations to avoid decompressing a lot of tuples and causing storage issues (100k limit, configurable) * Helper functions for determining compression settings * Plan-time chunk exclusion for real-time Continuous Aggregate by constifying the cagg_watermark function call, leading to faster queries using real-time continuous aggregates **For this release only**, you will need to restart the database before running `ALTER EXTENSION` **Multi-node support removal announcement** Following the deprecation announcement for Multi-node in TimescaleDB 2.13, Multi-node is no longer supported starting with TimescaleDB 2.14. TimescaleDB 2.13 is the last version that includes multi-node support. Learn more about it [here](docs/MultiNodeDeprecation.md). If you want to migrate from multi-node TimescaleDB to single-node TimescaleDB, read the [migration documentation](https://docs.timescale.com/migrate/latest/multi-node-to-timescale-service/). **Deprecation notice: recompress_chunk procedure** TimescaleDB 2.14 is the last version that will include the recompress_chunk procedure. Its functionality will be replaced by the compress_chunk function, which, starting on TimescaleDB 2.14, works on both uncompressed and partially compressed chunks. The compress_chunk function should be used going forward to fully compress all types of chunks or even recompress old fully compressed chunks using new compression settings (through the newly introduced recompress optional parameter). **Features** * #6325 Add plan-time chunk exclusion for real-time CAggs * #6360 Remove support for creating Continuous Aggregates with old format * #6386 Add functions for determining compression defaults * #6410 Remove multinode public API * #6440 Allow SQLValueFunction pushdown into compressed scan * #6463 Support approximate hypertable size * #6513 Make compression settings per chunk * #6529 Remove reindex_relation from recompression * #6531 Fix if_not_exists behavior for CAgg policy with NULL offsets * #6545 Remove restrictions for changing compression settings * #6566 Limit tuple decompression during DML operations * #6579 Change compress_chunk and decompress_chunk to idempotent version by default * #6608 Add LWLock for OSM usage in loader * #6609 Deprecate recompress_chunk * #6609 Add optional recompress argument to compress_chunk **Bugfixes** * #6541 Inefficient join plans on compressed hypertables. * #6491 Enable now() plantime constification with BETWEEN * #6494 Fix create_hypertable referenced by fk succeeds * #6498 Suboptimal query plans when using time_bucket with query parameters * #6507 time_bucket_gapfill with timezones doesn't handle daylight savings * #6509 Make extension state available through function * #6512 Log extension state changes * #6522 Disallow triggers on CAggs * #6523 Reduce locking level on compressed chunk index during segmentwise recompression * #6531 Fix if_not_exists behavior for CAgg policy with NULL offsets * #6571 Fix pathtarget adjustment for MergeAppend paths in aggregation pushdown code * #6575 Fix compressed chunk not found during upserts * #6592 Fix recompression policy ignoring partially compressed chunks * #6610 Ensure qsort comparison function is transitive **Thanks** * @coney21 and @GStechschulte for reporting the problem with inefficient join plans on compressed hypertables. * @HollowMan6 for reporting triggers not working on materialized views of CAggs * @jbx1 for reporting suboptimal query plans when using time_bucket with query parameters * @JerkoNikolic for reporting the issue with gapfill and DST * @pdipesh02 for working on removing the old Continuous Aggregate format * @raymalt and @martinhale for reporting very slow query plans on realtime CAggs queries ## 2.13.1 (2024-01-09) This release contains bug fixes since the 2.13.0 release. We recommend that you upgrade at the next available opportunity. **Bugfixes** * #6365 Use numrows_pre_compression in approximate row count * #6377 Use processed group clauses in PG16 * #6384 Change bgw_log_level to use PGC_SUSET * #6393 Disable vectorized sum for expressions. * #6405 Read CAgg watermark from materialized data * #6408 Fix groupby pathkeys for gapfill in PG16 * #6428 Fix index matching during DML decompression * #6439 Fix compressed chunk permission handling on PG16 * #6443 Fix lost concurrent CAgg updates * #6454 Fix unique expression indexes on compressed chunks * #6465 Fix use of freed path in decompression sort logic **Thanks** * @MA-MacDonald for reporting an issue with gapfill in PG16 * @aarondglover for reporting an issue with unique expression indexes on compressed chunks * @adriangb for reporting an issue with security barrier views on pg16 ## 2.13.0 (2023-11-28) This release contains performance improvements, an improved hypertable DDL API and bug fixes since the 2.12.2 release. We recommend that you upgrade at the next available opportunity. In addition, it includes these noteworthy features: * Full PostgreSQL 16 support for all existing features * Vectorized aggregation execution for sum() * Track chunk creation time used in retention/compression policies **Deprecation notice: Multi-node support** TimescaleDB 2.13 is the last version that will include multi-node support. Multi-node support in 2.13 is available for PostgreSQL 13, 14 and 15. Learn more about it [here](docs/MultiNodeDeprecation.md). If you want to migrate from multi-node TimescaleDB to single-node TimescaleDB read the [migration documentation](https://docs.timescale.com/migrate/latest/multi-node-to-timescale-service/). **PostgreSQL 13 deprecation announcement** We will continue supporting PostgreSQL 13 until April 2024. Sooner to that time, we will announce the specific version of TimescaleDB in which PostgreSQL 13 support will not be included going forward. **Starting from TimescaleDB 2.13.0** * No Amazon Machine Images (AMI) are published. If you previously used AMI, please use another [installation method](https://docs.timescale.com/self-hosted/latest/install/) * Continuous Aggregates are materialized only (non-realtime) by default **Features** * #5575 Add chunk-wise sorted paths for compressed chunks * #5761 Simplify hypertable DDL API * #5890 Reduce WAL activity by freezing compressed tuples immediately * #6050 Vectorized aggregation execution for sum() * #6062 Add metadata for chunk creation time * #6077 Make Continous Aggregates materialized only (non-realtime) by default * #6177 Change show_chunks/drop_chunks using chunk creation time * #6178 Show batches/tuples decompressed during DML operations in EXPLAIN output * #6185 Keep track of catalog version * #6227 Use creation time in retention/compression policy * #6307 Add SQL function cagg_validate_query **Bugfixes** * #6188 Add GUC for setting background worker log level * #6222 Allow enabling compression on hypertable with unique expression index * #6240 Check if worker registration succeeded * #6254 Fix exception detail passing in compression_policy_execute * #6264 Fix missing bms_del_member result assignment * #6275 Fix negative bitmapset member not allowed in compression * #6280 Potential data loss when compressing a table with a partial index that matches compression order. * #6289 Add support for startup chunk exclusion with aggs * #6290 Repair relacl on upgrade * #6297 Fix segfault when creating a cagg using a NULL width in time bucket function * #6305 Make timescaledb_functions.makeaclitem strict * #6332 Fix typmod and collation for segmentby columns * #6339 Fix tablespace with constraints * #6343 Enable segmentwise recompression in compression policy **Thanks** * @fetchezar for reporting an issue with compression policy error messages * @jflambert for reporting the background worker log level issue * @torazem for reporting an issue with compression and large oids * @fetchezar for reporting an issue in the compression policy * @lyp-bobi for reporting an issue with tablespace with constraints * @pdipesh02 for contributing to the implementation of the metadata for chunk creation time, the generalized hypertable API, and show_chunks/drop_chunks using chunk creation time * @lkshminarayanan for all his work on PG16 support ## 2.12.2 (2023-10-19) This release contains bug fixes since the 2.12.1 release. We recommend that you upgrade at the next available opportunity. **Bugfixes** * #6155 Align gapfill bucket generation with time_bucket * #6181 Ensure fixed_schedule field is populated * #6210 Fix EXPLAIN ANALYZE for compressed DML ## 2.12.1 (2023-10-12) This release contains bug fixes since the 2.12.0 release. We recommend that you upgrade at the next available opportunity. **Bugfixes** * #6113 Fix planner distributed table count * #6117 Avoid decompressing batches using an empty slot * #6123 Fix concurrency errors in OSM API * #6142 do not throw an error when deprecation GUC cannot be read **Thanks** * @symbx for reporting a crash when selecting from empty hypertables ## 2.12.0 (2023-09-27) This release contains performance improvements for compressed hypertables and continuous aggregates and bug fixes since the 2.11.2 release. We recommend that you upgrade at the next available opportunity. This release moves all internal functions from the _timescaleb_internal schema into the _timescaledb_functions schema. This separates code from internal data objects and improves security by allowing more restrictive permissions for the code schema. If you are calling any of those internal functions you should adjust your code as soon as possible. This version also includes a compatibility layer that allows calling them in the old location but that layer will be removed in 2.14.0. **PostgreSQL 12 support removal announcement** Following the deprecation announcement for PostgreSQL 12 in TimescaleDB 2.10, PostgreSQL 12 is not supported starting with TimescaleDB 2.12. Currently supported PostgreSQL major versions are 13, 14 and 15. PostgreSQL 16 support will be added with a following TimescaleDB release. **Features** * #5137 Insert into index during chunk compression * #5150 MERGE support on hypertables * #5515 Make hypertables support replica identity * #5586 Index scan support during UPDATE/DELETE on compressed hypertables * #5596 Support for partial aggregations at chunk level * #5599 Enable ChunkAppend for partially compressed chunks * #5655 Improve the number of parallel workers for decompression * #5758 Enable altering job schedule type through `alter_job` * #5805 Make logrepl markers for (partial) decompressions * #5809 Relax invalidation threshold table-level lock to row-level when refreshing a Continuous Aggregate * #5839 Support CAgg names in chunk_detailed_size * #5852 Make set_chunk_time_interval CAggs aware * #5868 Allow ALTER TABLE ... REPLICA IDENTITY (FULL|INDEX) on materialized hypertables (continuous aggregates) * #5875 Add job exit status and runtime to log * #5909 CREATE INDEX ONLY ON hypertable creates index on chunks **Bugfixes** * #5860 Fix interval calculation for hierarchical CAggs * #5894 Check unique indexes when enabling compression * #5951 _timescaledb_internal.create_compressed_chunk doesn't account for existing uncompressed rows * #5988 Move functions to _timescaledb_functions schema * #5788 Chunk_create must add an existing table or fail * #5872 Fix duplicates on partially compressed chunk reads * #5918 Fix crash in COPY from program returning error * #5990 Place data in first/last function in correct mctx * #5991 Call eq_func correctly in time_bucket_gapfill * #6015 Correct row count in EXPLAIN ANALYZE INSERT .. ON CONFLICT output * #6035 Fix server crash on UPDATE of compressed chunk * #6044 Fix server crash when using duplicate segmentby column * #6045 Fix segfault in set_integer_now_func * #6053 Fix approximate_row_count for CAggs * #6081 Improve compressed DML datatype handling * #6084 Propagate parameter changes to decompress child nodes * #6102 Schedule compression policy more often **Thanks** * @ajcanterbury for reporting a problem with lateral joins on compressed chunks * @alexanderlaw for reporting multiple server crashes * @lukaskirner for reporting a bug with monthly continuous aggregates * @mrksngl for reporting a bug with unusual user names * @willsbit for reporting a crash in time_bucket_gapfill ## 2.11.2 (2023-08-09) This release contains bug fixes since the 2.11.1 release. We recommend that you upgrade at the next available opportunity. **Features** * #5923 Feature flags for TimescaleDB features **Bugfixes** * #5680 Fix DISTINCT query with JOIN on multiple segmentby columns * #5774 Fixed two bugs in decompression sorted merge code * #5786 Ensure pg_config --cppflags are passed * #5906 Fix quoting owners in sql scripts. * #5912 Fix crash in 1-step integer policy creation **Thanks** * @mrksngl for submitting a PR to fix extension upgrade scripts * @ericdevries for reporting an issue with DISTINCT queries using segmentby columns of compressed hypertable ## 2.11.1 (2023-06-29) This release contains bug fixes since the 2.11.0 release. We recommend that you upgrade at the next available opportunity. **Features** * #5679 Update the loader to add support for the OSM extension (used for data tiering on [Timescale](https://www.timescale.com/)) **Bugfixes** * #5705 Scheduler accidentally getting killed when calling `delete_job` * #5742 Fix Result node handling with ConstraintAwareAppend on compressed chunks * #5750 Ensure tlist is present in decompress chunk plan * #5754 Fixed handling of NULL values in bookend_sfunc * #5798 Fixed batch look ahead in compressed sorted merge * #5804 Mark cagg_watermark function as PARALLEL RESTRICTED * #5807 Copy job config JSONB structure into current MemoryContext * #5824 Improve continuous aggregate query chunk exclusion **Thanks** * @JamieD9 for reporting an issue with a wrong result ordering * @xvaara for reporting an issue with Result node handling in ConstraintAwareAppend ## 2.11.0 (2023-05-12) This release contains new features and bug fixes since the 2.10.3 release. We deem it moderate priority for upgrading. This release includes these noteworthy features: * Support for DML operations on compressed chunks: * UPDATE/DELETE support * Support for unique constraints on compressed chunks * Support for `ON CONFLICT DO UPDATE` * Support for `ON CONFLICT DO NOTHING` * Join support for Hierarchical Continuous Aggregates * Performance improvements for real-time Hierarchical Continuous Aggregates **Features** * #5212 Allow pushdown of reference table joins * #5261 Improve Realtime Continuous Aggregate performance * #5252 Improve unique constraint support on compressed hypertables * #5339 Support UPDATE/DELETE on compressed hypertables * #5344 Enable JOINS for Hierarchical Continuous Aggregates * #5361 Add parallel support for partialize_agg() * #5417 Refactor and optimize distributed COPY * #5454 Add support for ON CONFLICT DO UPDATE for compressed hypertables * #5547 Skip Ordered Append when only 1 child node is present * #5510 Propagate vacuum/analyze to compressed chunks * #5584 Reduce decompression during constraint checking * #5530 Optimize compressed chunk resorting * #5639 Support sending telemetry event reports **Bugfixes** * #5396 Fix SEGMENTBY columns predicates to be pushed down * #5427 Handle user-defined FDW options properly * #5442 Decompression may have lost DEFAULT values * #5459 Fix issue creating dimensional constraints * #5570 Improve interpolate error message on datatype mismatch * #5573 Fix unique constraint on compressed tables * #5615 Add permission checks to run_job() * #5614 Enable run_job() for telemetry job * #5578 Fix on-insert decompression after schema changes * #5613 Quote username identifier appropriately * #5525 Fix tablespace for compressed hypertable and corresponding toast * #5642 Fix ALTER TABLE SET with normal tables * #5666 Reduce memory usage for distributed analyze * #5668 Fix subtransaction resource owner **Thanks** * @kovetskiy and @DZDomi for reporting peformance regression in Realtime Continuous Aggregates * @ollz272 for reporting an issue with interpolate error messages ## 2.10.3 (2023-04-26) **Bugfixes** * #5583 Fix parameterization in DecompressChunk path generation * #5602 Fix broken CAgg with JOIN repair function ## 2.10.2 (2023-04-20) **Bugfixes** * #5410 Fix file trailer handling in the COPY fetcher * #5446 Add checks for malloc failure in libpq calls * #5233 Out of on_proc_exit slots on guc license change * #5428 Use consistent snapshots when scanning metadata * #5499 Do not segfault on large histogram() parameters * #5470 Ensure superuser perms during copy/move chunk * #5500 Fix when no FROM clause in continuous aggregate definition * #5433 Fix join rte in CAggs with joins * #5556 Fix duplicated entries on timescaledb_experimental.policies view * #5462 Fix segfault after column drop on compressed table * #5543 Copy scheduled_jobs list before sorting it * #5497 Allow named time_bucket arguments in Cagg definition * #5544 Fix refresh from beginning of Continuous Aggregate with variable time bucket * #5558 Use regrole for job owner * #5542 Enable indexscan on uncompressed part of partially compressed chunks **Thanks** * @nikolaps for reporting an issue with the COPY fetcher * @S-imo-n for reporting the issue on Background Worker Scheduler crash * @geezhu for reporting issue on segfault in historgram() * @mwahlhuetter for reporting the issue with joins in CAggs * @mwahlhuetter for reporting issue with duplicated entries on timescaledb_experimental.policies view * @H25E for reporting error refreshing from beginning of a Continuous Aggregate with variable time bucket ## 2.10.1 (2023-03-07) This release contains bug fixes since the 2.10.0 release. We recommend that you upgrade at the next available opportunity. **Bugfixes** * #5159 Support Continuous Aggregates names in hypertable_(detailed_)size * #5226 Fix concurrent locking with chunk_data_node table * #5317 Fix some incorrect memory handling * #5336 Use NameData and namestrcpy for names * #5343 Set PortalContext when starting job * #5360 Fix uninitialized bucket_info variable * #5362 Make copy fetcher more async * #5364 Fix num_chunks inconsistency in hypertables view * #5367 Fix column name handling in old-style continuous aggregates * #5378 Fix multinode DML HA performance regression * #5384 Fix Hierarchical Continuous Aggregates chunk_interval_size **Thanks** * @justinozavala for reporting an issue with PL/Python procedures in the background worker * @Medvecrab for discovering an issue with copying NameData when forming heap tuples. * @pushpeepkmonroe for discovering an issue in upgrading old-style continuous aggregates with renamed columns ## 2.10.0 (2023-02-21) This release contains new features and bug fixes since the 2.9.3 release. We deem it moderate priority for upgrading. This release includes these noteworthy features: * Joins in continuous aggregates * Re-architecture of how compression works: ~2x improvement on INSERT rate into compressed chunks. * Full PostgreSQL 15 support for all existing features. Support for the newly introduced MERGE command on hypertables will be introduced on a follow-up release. **PostgreSQL 12 deprecation announcement** We will continue supporting PostgreSQL 12 until July 2023. Sooner to that time, we will announce the specific version of TimescaleDB in which PostgreSQL 12 support will not be included going forward. **Old format of Continuous Aggregates deprecation announcement** TimescaleDB 2.7 introduced a new format for continuous aggregates that improves performance. All instances with Continuous Aggregates using the old format should [migrate to the new format](https://docs.timescale.com/api/latest/continuous-aggregates/cagg_migrate/) by July 2023, when support for the old format will be removed. Sooner to that time, we will announce the specific version of TimescaleDB in which support for this feature will not be included going forward. **Features** * #4874 Allow joins in continuous aggregates * #4926 Refactor INSERT into compressed chunks * #5241 Allow RETURNING clause when inserting into compressed chunks * #5245 Manage life-cycle of connections via memory contexts * #5246 Make connection establishment interruptible * #5253 Make data node command execution interruptible * #5262 Extend enabling compression on a continuous aggregrate with 'compress_segmentby' and 'compress_orderby' parameters **Bugfixes** * #5214 Fix use of prepared statement in async module * #5218 Add role-level security to job error log * #5239 Fix next_start calculation for fixed schedules * #5290 Fix enabling compression on continuous aggregates with columns requiring quotation **Thanks** * @henriquegelio for reporting the issue on fixed schedules ## 2.9.3 (2023-02-03) This release contains bug fixes since the 2.9.2 release and a fix for a security vulnerability (#5259). You can check the security advisory(https://github.com/timescale/timescaledb/security/advisories/GHSA-44jh-j22r-33wq) for more information on the vulnerability and the platforms that are affected. This release is high priority for upgrade. We strongly recommend that you upgrade as soon as possible. **Bugfixes** * #4804 Skip bucketing when start or end of refresh job is null * #5108 Fix column ordering in compressed table index not following the order of a multi-column segment by definition * #5187 Don't enable clang-tidy by default * #5255 Fix year not being considered as a multiple of day/month in hierarchical continuous aggregates * #5259 Lock down search_path in SPI calls **Thanks** * @ssmoss for reporting issues on continuous aggregates * @jaskij for reporting the compliation issue that occurred with clang ## 2.9.2 (2023-01-26) This release contains bug fixes since the 2.9.1 release. We recommend that you upgrade at the next available opportunity. **Bugfixes** * #5114 Fix issue with deleting data node and dropping the database on multi-node * #5133 Fix creating a CAgg on a CAgg where the time column is in a different order of the original hypertable * #5152 Fix adding column with NULL constraint to compressed hypertable * #5170 Fix CAgg on CAgg variable bucket size validation * #5180 Fix default data node availability status on multi-node * #5181 Fix ChunkAppend and ConstraintAwareAppend with TidRangeScan child subplan * #5193 Fix repartition behavior when attaching data node on multi-node **Thanks** * @salquier-appvizer for reporting error on CAgg on CAgg using different column order on the original hypertable * @ikkala for reporting error when adding column with NULL constraint to compressed hypertable * @ssmoss, @adbnexxtlab and @ivanzamanov for reporting error on CAgg on CAgg variable bucket size validation * @ronnyas for reporting a bug "Invalid child of chunk" on specific ctid filtering ## 2.9.1 (2022-12-23) This release contains bug fixes since the 2.9.0 release. This release is high priority for upgrade. We strongly recommend that you upgrade as soon as possible. **Bugfixes** * #5072 Fix CAgg on CAgg bucket size validation * #5101 Fix enabling compression on caggs with renamed columns * #5106 Fix building against PG15 on Windows * #5117 Fix postgres server restart on background worker exit * #5121 Fix privileges for job_errors in update script ## 2.9.0 (2022-12-15) This release adds major new features since the 2.8.1 release. We deem it moderate priority for upgrading. This release includes these noteworthy features: * Hierarchical Continuous Aggregates (aka Continuous Aggregate on top of another Continuous Aggregate) * Improve `time_bucket_gapfill` function to allow specifying the timezone to bucket * Introduce fixed schedules for background jobs and the ability to check job errors. * Use `alter_data_node()` to change the data node configuration. This function introduces the option to configure the availability of the data node. This release also includes several bug fixes. **Features** * #4476 Batch rows on access node for distributed COPY * #4567 Exponentially backoff when out of background workers * #4650 Show warnings when not following best practices * #4664 Introduce fixed schedules for background jobs * #4668 Hierarchical Continuous Aggregates * #4670 Add timezone support to time_bucket_gapfill * #4678 Add interface for troubleshooting job failures * #4718 Add ability to merge chunks while compressing * #4786 Extend the now() optimization to also apply to CURRENT_TIMESTAMP * #4820 Support parameterized data node scans in joins * #4830 Add function to change configuration of a data nodes * #4966 Handle DML activity when datanode is not available * #4971 Add function to drop stale chunks on a datanode **Bugfixes** * #4663 Don't error when compression metadata is missing * #4673 Fix now() constification for VIEWs * #4681 Fix compression_chunk_size primary key * #4696 Report warning when enabling compression on hypertable * #4745 Fix FK constraint violation error while insert into hypertable which references partitioned table * #4756 Improve compression job IO performance * #4770 Continue compressing other chunks after an error * #4794 Fix degraded performance seen on timescaledb_internal.hypertable_local_size() function * #4807 Fix segmentation fault during INSERT into compressed hypertable * #4822 Fix missing segmentby compression option in CAGGs * #4823 Fix a crash that could occur when using nested user-defined functions with hypertables * #4840 Fix performance regressions in the copy code * #4860 Block multi-statement DDL command in one query * #4898 Fix cagg migration failure when trying to resume * #4904 Remove BitmapScan support in DecompressChunk * #4906 Fix a performance regression in the query planner by speeding up frozen chunk state checks * #4910 Fix a typo in process_compressed_data_out * #4918 Cagg migration orphans cagg policy * #4941 Restrict usage of the old format (pre 2.7) of continuous aggregates in PostgreSQL 15. * #4955 Fix cagg migration for hypertables using timestamp without timezone * #4968 Check for interrupts in gapfill main loop * #4988 Fix cagg migration crash when refreshing the newly created cagg * #5054 Fix segfault after second ANALYZE * #5086 Reset baserel cache on invalid hypertable cache **Thanks** * @byazici for reporting a problem with segmentby on compressed caggs * @jflambert for reporting a crash with nested user-defined functions. * @jvanns for reporting hypertable FK reference to vanilla PostgreSQL partitioned table doesn't seem to work * @kou for fixing a typo in process_compressed_data_out * @kyrias for reporting a crash when ANALYZE is executed on extended query protocol mode with extension loaded. * @tobiasdirksen for requesting Continuous aggregate on top of another continuous aggregate * @xima for reporting a bug in Cagg migration * @xvaara for helping reproduce a bug with bitmap scans in transparent decompression ## 2.8.1 (2022-10-06) This release is a patch release. We recommend that you upgrade at the next available opportunity. **Bugfixes** * #4454 Keep locks after reading job status * #4658 Fix error when querying a compressed hypertable with compress_segmentby on an enum column * #4671 Fix a possible error while flushing the COPY data * #4675 Fix bad TupleTableSlot drop * #4676 Fix a deadlock when decompressing chunks and performing SELECTs * #4685 Fix chunk exclusion for space partitions in SELECT FOR UPDATE queries * #4694 Change parameter names of cagg_migrate procedure * #4698 Do not use row-by-row fetcher for parameterized plans * #4711 Remove support for procedures as custom checks * #4712 Fix assertion failure in constify_now * #4713 Fix Continuous Aggregate migration policies * #4720 Fix chunk exclusion for prepared statements and dst changes * #4726 Fix gapfill function signature * #4737 Fix join on time column of compressed chunk * #4738 Fix error when waiting for remote COPY to finish * #4739 Fix continuous aggregate migrate check constraint * #4760 Fix segfault when INNER JOINing hypertables * #4767 Fix permission issues on index creation for CAggs **Thanks** * @boxhock and @cocowalla for reporting a segfault when JOINing hypertables * @carobme for reporting constraint error during continuous aggregate migration * @choisnetm, @dustinsorensen, @jayadevanm and @joeyberkovitz for reporting a problem with JOINs on compressed hypertables * @daniel-k for reporting a background worker crash * @justinpryzby for reporting an error when compressing very wide tables * @maxtwardowski for reporting problems with chunk exclusion and space partitions * @yuezhihan for reporting GROUP BY error when having compress_segmentby on an enum column ## 2.8.0 (2022-08-30) This release adds major new features since the 2.7.2 release. We deem it moderate priority for upgrading. This release includes these noteworthy features: * time_bucket now supports bucketing by month, year and timezone * Improve performance of bulk SELECT and COPY for distributed hypertables * 1 step CAgg policy management * Migrate Continuous Aggregates to the new format **Features** * #4188 Use COPY protocol in row-by-row fetcher * #4307 Mark partialize_agg as parallel safe * #4380 Enable chunk exclusion for space dimensions in UPDATE/DELETE * #4384 Add schedule_interval to policies * #4390 Faster lookup of chunks by point * #4393 Support intervals with day component when constifying now() * #4397 Support intervals with month component when constifying now() * #4405 Support ON CONFLICT ON CONSTRAINT for hypertables * #4412 Add telemetry about replication * #4415 Drop remote data when detaching data node * #4416 Handle TRUNCATE TABLE on chunks * #4425 Add parameter check_config to alter_job * #4430 Create index on Continuous Aggregates * #4439 Allow ORDER BY on continuous aggregates * #4443 Add stateful partition mappings * #4484 Use non-blocking data node connections for COPY * #4495 Support add_dimension() with existing data * #4502 Add chunks to baserel cache on chunk exclusion * #4545 Add hypertable distributed argument and defaults * #4552 Migrate Continuous Aggregates to the new format * #4556 Add runtime exclusion for hypertables * #4561 Change get_git_commit to return full commit hash * #4563 1 step CAgg policy management * #4641 Allow bucketing by month, year, century in time_bucket and time_bucket_gapfill * #4642 Add timezone support to time_bucket **Bugfixes** * #4359 Create composite index on segmentby columns * #4374 Remove constified now() constraints from plan * #4416 Handle TRUNCATE TABLE on chunks * #4478 Synchronize chunk cache sizes * #4486 Adding boolean column with default value doesn't work on compressed table * #4512 Fix unaligned pointer access * #4519 Throw better error message on incompatible row fetcher settings * #4549 Fix dump_meta_data for windows * #4553 Fix timescaledb_post_restore GUC handling * #4573 Load TSL library on compressed_data_out call * #4575 Fix use of `get_partition_hash` and `get_partition_for_key` inside an IMMUTABLE function * #4577 Fix segfaults in compression code with corrupt data * #4580 Handle default privileges on CAggs properly * #4582 Fix assertion in GRANT .. ON ALL TABLES IN SCHEMA * #4583 Fix partitioning functions * #4589 Fix rename for distributed hypertable * #4601 Reset compression sequence when group resets * #4611 Fix a potential OOM when loading large data sets into a hypertable * #4624 Fix heap buffer overflow * #4627 Fix telemetry initialization * #4631 Ensure TSL library is loaded on database upgrades * #4646 Fix time_bucket_ng origin handling * #4647 Fix the error "SubPlan found with no parent plan" that occurred if using joins in RETURNING clause. **Thanks** * @AlmiS for reporting error on `get_partition_hash` executed inside an IMMUTABLE function * @Creatation for reporting an issue with renaming hypertables * @janko for reporting an issue when adding bool column with default value to compressed hypertable * @jayadevanm for reporting error of TRUNCATE TABLE on compressed chunk * @michaelkitson for reporting permission errors using default privileges on Continuous Aggregates * @mwahlhuetter for reporting error in joins in RETURNING clause * @ninjaltd and @mrksngl for reporting a potential OOM when loading large data sets into a hypertable * @PBudmark for reporting an issue with dump_meta_data.sql on Windows * @ssmoss for reporting an issue with time_bucket_ng origin handling ## 2.7.2 (2022-07-26) This release is a patch release. We recommend that you upgrade at the next available opportunity. Among other things this release fixes several memory leaks, handling of TOASTed values in GapFill and parameter handling in prepared statements. **Bugfixes** * #4517 Fix prepared statement param handling in ChunkAppend * #4522 Fix ANALYZE on dist hypertable for a set of nodes * #4526 Fix gapfill group comparison for TOASTed values * #4527 Handle stats properly for range types * #4532 Fix memory leak in function telemetry * #4534 Use explicit memory context with hash_create * #4538 Fix chunk creation on hypertables with non-default statistics **Thanks** * @3a6u9ka, @bgemmill, @hongquan, @stl-leonid-kalmaev and @victor-sudakov for reporting a memory leak * @hleung2021 and @laocaixw for reporting an issue with parameter handling in prepared statements ## 2.7.1 (2022-07-07) This release is a patch release. We recommend that you upgrade at the next available opportunity. **Bugfixes** * #4494 Handle timescaledb versions aptly in multinode * #4493 Segfault when executing IMMUTABLE functions * #4482 Fix race conditions during chunk (de)compression * #4367 Improved buffer management in the copy operator * #4375 Don't ask for orderby column if default already set * #4400 Use our implementation of `find_em_expr_for_rel` for PG15+ * #4408 Fix crash during insert into distributed hypertable * #4411 Add `shmem_request_hook` * #4437 Fix segfault in subscription_exec * #4442 Fix perms in copy/move chunk * #4450 Retain hypertable ownership on `attach_data_node` * #4451 Repair numeric partial state on the fly * #4463 Fix empty bytea handlng with distributed tables * #4469 Better superuser handling for `move_chunk` **Features** * #4244 Function telemetry * #4287 Add internal api for foreign table chunk * #4470 Block drop chunk if chunk is in frozen state * #4464 Add internal api to associate a hypertable with custom jobs **Thanks** * @xin-hedera Finding bug in empty bytea values for distributed tables * @jflambert for reporting a bug with IMMUTABLE functions * @nikugogoi for reporting a bug with CTEs and upserts on distributed hypertables ## 2.7.0 (2022-05-24) This release adds major new features since the 2.6.1 release. We deem it moderate priority for upgrading. This release includes these noteworthy features: * Optimize continuous aggregate query performance and storage * The following query clauses and functions can now be used in a continuous aggregate: FILTER, DISTINCT, ORDER BY as well as [Ordered-Set Aggregate](https://www.postgresql.org/docs/current/functions-aggregate.html#FUNCTIONS-ORDEREDSET-TABLE) and [Hypothetical-Set Aggregate](https://www.postgresql.org/docs/current/functions-aggregate.html#FUNCTIONS-HYPOTHETICAL-TABLE) * Optimize now() query planning time * Improve COPY insert performance * Improve performance of UPDATE/DELETE on PG14 by excluding chunks This release also includes several bug fixes. If you are upgrading from a previous version and were using compression with a non-default collation on a segmentby-column you should recompress those hypertables. **Features** * #4045 Custom origin's support in CAGGs * #4120 Add logging for retention policy * #4158 Allow ANALYZE command on a data node directly * #4169 Add support for chunk exclusion on DELETE to PG14 * #4209 Add support for chunk exclusion on UPDATE to PG14 * #4269 Continuous Aggregates finals form * #4301 Add support for bulk inserts in COPY operator * #4311 Support non-superuser move chunk operations * #4330 Add GUC "bgw_launcher_poll_time" * #4340 Enable now() usage in plan-time chunk exclusion **Bugfixes** * #3899 Fix segfault in Continuous Aggregates * #4225 Fix TRUNCATE error as non-owner on hypertable * #4236 Fix potential wrong order of results for compressed hypertable with a non-default collation * #4249 Fix option "timescaledb.create_group_indexes" * #4251 Fix INSERT into compressed chunks with dropped columns * #4255 Fix option "timescaledb.create_group_indexes" * #4259 Fix logic bug in extension update script * #4269 Fix bad Continuous Aggregate view definition reported in #4233 * #4289 Support moving compressed chunks between data nodes * #4300 Fix refresh window cap for cagg refresh policy * #4315 Fix memory leak in scheduler * #4323 Remove printouts from signal handlers * #4342 Fix move chunk cleanup logic * #4349 Fix crashes in functions using AlterTableInternal * #4358 Fix crash and other issues in telemetry reporter **Thanks** * @abrownsword for reporting a bug in the telemetry reporter and testing the fix * @jsoref for fixing various misspellings in code, comments and documentation * @yalon for reporting an error with ALTER TABLE RENAME on distributed hypertables * @zhuizhuhaomeng for reporting and fixing a memory leak in our scheduler ## 2.6.1 (2022-04-11) This release is patch release. We recommend that you upgrade at the next available opportunity. **Bugfixes** * #4121 Fix RENAME TO/SET SCHEMA on distributed hypertable * #4122 Fix segfault on INSERT into distributed hypertable * #4142 Ignore invalid relid when deleting hypertable * #4159 Fix ADD COLUMN IF NOT EXISTS error on compressed hypertable * #4161 Fix memory handling during scans * #4176 Fix remote EXPLAIN with parameterized queries * #4181 Fix spelling errors and omissions * #4186 Fix owner change for distributed hypertable * #4192 Abort sessions after extension reload * #4193 Fix relcache callback handling causing crashes * #4199 Remove signal-unsafe calls from signal handlers * #4219 Do not modify aggregation state in finalize **Thanks** * @abrownsword for reporting a crash in the telemetry reporter * @amalek215 for reporting a segmentation fault when running VACUUM FULL pg_class * @daydayup863 for reporting issue with remote explain * @krvajal for reporting an error with ADD COLUMN IF NOT EXISTS on compressed hypertables ## 2.6.0 (2022-02-16) This release is medium priority for upgrade. We recommend that you upgrade at the next available opportunity. This release adds major new features since the 2.5.2 release, including: * Compression in continuous aggregates * Experimental support for timezones in continuous aggregates * Experimental support for monthly buckets in continuous aggregates The release also includes several bug fixes. Telemetry reports now include new and more detailed statistics on regular tables and views, compression, distributed hypertables, and continuous aggregates, which will help us improve TimescaleDB. **Features** * #3768 Allow ALTER TABLE ADD COLUMN with DEFAULT on compressed hypertable * #3769 Allow ALTER TABLE DROP COLUMN on compressed hypertable * #3943 Optimize first/last * #3945 Add support for ALTER SCHEMA on multi-node * #3949 Add support for DROP SCHEMA on multi-node **Bugfixes** * #3808 Properly handle `max_retries` option * #3863 Fix remote transaction heal logic * #3869 Fix ALTER SET/DROP NULL constraint on distributed hypertable * #3944 Fix segfault in add_compression_policy * #3961 Fix crash in EXPLAIN VERBOSE on distributed hypertable * #4015 Eliminate float rounding instabilities in interpolate * #4019 Update ts_extension_oid in transitioning state * #4073 Fix buffer overflow in partition scheme * #4180 ALTER TABLE OWNER TO does not work for distributed hypertable **Improvements** * Query planning performance is improved for hypertables with a large number of chunks. **Thanks** * @fvannee for reporting a first/last memory leak * @mmouterde for reporting an issue with floats and interpolate ## 2.5.2 (2022-02-09) This release contains bug fixes since the 2.5.1 release. This release is high priority for upgrade. We strongly recommend that you upgrade as soon as possible. **Bugfixes** * #3900 Improve custom scan node registration * #3911 Fix role type deparsing for GRANT command * #3918 Fix DataNodeScan plans with one-time filter * #3921 Fix segfault on insert into internal compressed table * #3938 Fix subtract_integer_from_now on 32-bit platforms and improve error handling * #3939 Fix projection handling in time_bucket_gapfill * #3948 Avoid double PGclear() in data fetchers * #3979 Fix deparsing of index predicates * #4020 Fix ALTER TABLE EventTrigger initialization * #4024 Fix premature cache release call * #4037 Fix status for dropped chunks that have catalog entries * #4069 Fix riinfo NULL handling in ANY construct * #4071 Fix extension installation privilege escalation (CVE-2022-24128) **Thanks** * @carlocperez for reporting crash with NULL handling in ANY construct * @erikhh for reporting an issue with time_bucket_gapfill * @kancsuki for reporting drop column and partial index creation not working * @mmouterde for reporting an issue with floats and interpolate * Pedro Gallegos for reporting a possible privilege escalation during extension installation ## 2.5.1 (2021-12-02) This release contains bug fixes since the 2.5.0 release. We deem it medium priority to upgrade. **Bugfixes** * #3706 Test enabling dist compression within a procedure * #3734 Rework distributed DDL processing logic * #3737 Fix flaky pg_dump * #3739 Fix compression policy on tables using INTEGER * #3766 Fix segfault in ts_hist_sfunc * #3779 Support GRANT/REVOKE on distributed database * #3789 Fix time_bucket comparison transformation * #3797 Fix DISTINCT ON queries for distributed hyperatbles * #3799 Fix error printout on correct security label * #3801 Fail size utility functions when data nodes do not respond * #3809 Fix NULL pointer evaluation in fill_result_error() * #3811 Fix INSERT..SELECT involving dist hypertables * #3819 Fix reading garbage value from TSConnectionError * #3824 Remove pointers from CAGG lists for 64-bit archs * #3846 Eliminate deadlock in recompress chunk policy * #3881 Fix SkipScan crash due to pruned unique path * #3884 Fix create_distributed_restore_point memory issue **Thanks** * @cbisnett for reporting and fixing a typo in an error message * @CaptainCuddleCube for reporting bug on compression policy procedure on tables using INTEGER on time dimension * @phemmer for reporting bugs on multi-node ## 2.5.0 (2021-10-28) This release adds major new features since the 2.4.2 release. We deem it moderate priority for upgrading. This release includes these noteworthy features: * Continuous Aggregates for Distributed Hypertables * Support for PostgreSQL 14 * Experimental: Support for timezones in `time_bucket_ng()`, including the `origin` argument This release also includes several bug fixes. **Features** * #3034 Add support for PostgreSQL 14 * #3435 Add continuous aggregates for distributed hypertables * #3505 Add support for timezones in `time_bucket_ng()` **Bugfixes** * #3580 Fix memory context bug executing TRUNCATE * #3592 Allow alter column type on distributed hypertable * #3598 Improve evaluation of stable functions such as now() on access node * #3618 Fix execution of refresh_caggs from user actions * #3625 Add shared dependencies when creating chunk * #3626 Fix memory context bug executing TRUNCATE * #3627 Schema qualify UDTs in multi-node * #3638 Allow owner change of a data node * #3654 Fix index attnum mapping in reorder_chunk * #3661 Fix SkipScan path generation with constant DISTINCT column * #3667 Fix compress_policy for multi txn handling * #3673 Fix distributed hypertable DROP within a procedure * #3701 Allow anyone to use size utilities on distributed hypertables * #3708 Fix crash in get_aggsplit * #3709 Fix ordered append pathkey check * #3712 Fix GRANT/REVOKE ALL IN SCHEMA handling * #3717 Support transparent decompression on individual chunks * #3724 Fix inserts into compressed chunks on hypertables with caggs * #3727 Fix DirectFunctionCall crash in distributed_exec * #3728 Fix SkipScan with varchar column * #3733 Fix ANALYZE crash with custom statistics for custom types * #3747 Always reset expr context in DecompressChunk **Thanks** * @binakot and @sebvett for reporting an issue with DISTINCT queries * @hardikm10, @DavidPavlicek and @pafiti for reporting bugs on TRUNCATE * @mjf for reporting an issue with ordered append and JOINs * @phemmer for reporting the issues on multinode with aggregate queries and evaluation of now() * @abolognino for reporting an issue with INSERTs into compressed hypertables that have cagg * @tanglebones for reporting the ANALYZE crash with custom types on multinode * @amadeubarbosa and @felipenogueirajack for reporting crash using JSONB column in compressed chunks ## 2.4.2 (2021-09-21) This release contains bug fixes since the 2.4.1 release. We deem it high priority to upgrade. **Bugfixes** * #3437 Rename on all continuous aggregate objects * #3469 Use signal-safe functions in signal handler * #3520 Modify compression job processing logic * #3527 Fix time_bucket_ng behaviour with origin argument * #3532 Fix bootstrap with regresschecks disabled * #3574 Fix failure on job execution by background worker * #3590 Call cleanup functions on backend exit **Thanks** * @jankatins for reporting a crash with background workers * @LutzWeischerFujitsu for reporting an issue with bootstrap ## 2.4.1 (2021-08-19) This release contains bug fixes since the 2.4.0 release. We deem it high priority to upgrade. The release fixes continous aggregate refresh for postgres 12.8 and 13.4, a crash with ALTER TABLE commands and a crash with continuous aggregates with HAVING clause. **Bugfixes** * #3430 Fix havingqual processing for continuous aggregates * #3468 Disable tests by default if tools are not found * #3462 Fix crash while tracking alter table commands * #3489 Fix continuous agg bgw job failure for PG 12.8 and 13.4 * #3494 Improve error message when adding data nodes **Thanks** * @brianbenns for reporting a segfault with continuous aggregates * @usego for reporting an issue with continuous aggregate refresh on PG 13.4 ## 2.4.0 (2021-07-29) This release adds new experimental features since the 2.3.1 release. The experimental features in this release are: * APIs for chunk manipulation across data nodes in a distributed hypertable setup. This includes the ability to add a data node and move chunks to the new data node for cluster rebalancing. * The `time_bucket_ng` function, a newer version of `time_bucket`. This function supports years, months, days, hours, minutes, and seconds. We’re committed to developing these experiments, giving the community a chance to provide early feedback and influence the direction of TimescaleDB’s development. We’ll travel faster with your input! Please create your feedback as a GitHub issue (using the experimental-schema label), describe what you found, and tell us the steps or share the code snip to recreate it. This release also includes several bug fixes. PostgreSQL 11 deprecation announcement Timescale is working hard on our next exciting features. To make that possible, we require functionality that is available in Postgres 12 and above. Postgres 11 is not supported with TimescaleDB 2.4. **Experimental Features** * #3293 Add timescaledb_experimental schema * #3302 Add block_new_chunks and allow_new_chunks API to experimental schema. Add chunk based refresh_continuous_aggregate. * #3211 Introduce experimental time_bucket_ng function * #3366 Allow use of experimental time_bucket_ng function in continuous aggregates * #3408 Support for seconds, minutes and hours in time_bucket_ng * #3446 Implement cleanup for chunk copy/move. **Bugfixes** * #3401 Fix segfault for RelOptInfo without fdw_private * #3411 Verify compressed chunk validity for compressed path * #3416 Fix targetlist names for continuous aggregate views * #3434 Remove extension check from relcache invalidation callback * #3440 Fix remote_tx_heal_data_node to work with only current database **Thanks** * @fvannee for reporting an issue with hypertable expansion in functions * @amalek215 for reporting an issue with cache invalidation during pg_class vacuum full * @hardikm10 for reporting an issue with inserting into compressed chunks * @dberardo-com and @iancmcc for reporting an issue with extension updates after renaming columns of continuous aggregates. ## 2.3.1 (2021-07-05) This maintenance release contains bugfixes since the 2.3.0 release. We deem it moderate priority for upgrading. The release introduces the possibility of generating downgrade scripts, improves the trigger handling for distributed hypertables, adds some more randomness to chunk assignment to avoid thundering herd issues in chunk assignment, and fixes some issues in update handling as well as some other bugs. **Bugfixes** * #3279 Add some more randomness to chunk assignment * #3288 Fix failed update with parallel workers * #3300 Improve trigger handling on distributed hypertables * #3304 Remove paths that reference parent relids for compressed chunks * #3305 Fix pull_varnos miscomputation of relids set * #3310 Generate downgrade script * #3314 Fix heap buffer overflow in hypertable expansion * #3317 Fix heap buffer overflow in remote connection cache. * #3327 Make aggregates in caggs fully qualified * #3336 Fix pg_init_privs objsubid handling * #3345 Fix SkipScan distinct column identification * #3355 Fix heap buffer overflow when renaming compressed hypertable columns. * #3367 Improve DecompressChunk qual pushdown * #3377 Fix bad use of repalloc **Thanks** * @db-adrian for reporting an issue when accessing cagg view through postgres_fdw * @fncaldas and @pgwhalen for reporting an issue accessing caggs when public is not in search_path * @fvannee, @mglonnro and @ebreijo for reporting an issue with the upgrade script * @fvannee for reporting a performance regression with SkipScan ## 2.3.0 (2021-05-25) This release adds major new features since the 2.2.1 release. We deem it moderate priority for upgrading. This release adds support for inserting data into compressed chunks and improves performance when inserting data into distributed hypertables. Distributed hypertables now also support triggers and compression policies. The bug fixes in this release address issues related to the handling of privileges on compressed hypertables, locking, and triggers with transition tables. **Features** * #3116 Add distributed hypertable compression policies * #3162 Use COPY when executing distributed INSERTs * #3199 Add GENERATED column support on distributed hypertables * #3210 Add trigger support on distributed hypertables * #3230 Support for inserts into compressed chunks **Bugfixes** * #3213 Propagate grants to compressed hypertables * #3229 Use correct lock mode when updating chunk * #3243 Fix assertion failure in decompress_chunk_plan_create * #3250 Fix constraint triggers on hypertables * #3251 Fix segmentation fault due to incorrect call to chunk_scan_internal * #3252 Fix blocking triggers with transition tables **Thanks** * @yyjdelete for reporting a crash with decompress_chunk and identifying the bug in the code * @fabriziomello for documenting the prerequisites when compiling against PostgreSQL 13 ## 2.2.1 (2021-05-05) This maintenance release contains bugfixes since the 2.2.0 release. We deem it high priority for upgrading. This release extends Skip Scan to multinode by enabling the pushdown of `DISTINCT` to data nodes. It also fixes a number of bugs in the implementation of Skip Scan, in distributed hypertables, in creation of indexes, in compression, and in policies. **Features** * #3113 Pushdown "SELECT DISTINCT" in multi-node to allow use of Skip Scan **Bugfixes** * #3101 Use commit date in `get_git_commit()` * #3102 Fix `REINDEX TABLE` for distributed hypertables * #3104 Fix use after free in `add_reorder_policy` * #3106 Fix use after free in `chunk_api_get_chunk_stats` * #3109 Copy recreated object permissions on update * #3111 Fix `CMAKE_BUILD_TYPE` check * #3112 Use `%u` to format Oid instead of `%d` * #3118 Fix use after free in cache * #3123 Fix crash while using `REINDEX TABLE CONCURRENTLY` * #3135 Fix SkipScan path generation in `DISTINCT` queries with expressions * #3146 Fix SkipScan for IndexPaths without pathkeys * #3147 Skip ChunkAppend if AppendPath has no children * #3148 Make `SELECT DISTINCT` handle non-var targetlists * #3151 Fix `fdw_relinfo_get` assertion failure on `DELETE` * #3155 Inherit `CFLAGS` from PostgreSQL * #3169 Fix incorrect type cast in compression policy * #3183 Fix segfault in calculate_chunk_interval * #3185 Fix wrong datatype for integer based retention policy **Thanks** * @Dead2, @dv8472 and @einsibjarni for reporting an issue with multinode queries and views * @aelg for reporting an issue with policies on integer-based hypertables * @hperez75 for reporting an issue with Skip Scan * @nathanloisel for reporting an issue with compression on hypertables with integer-based timestamps * @xin-hedera for fixing an issue with compression on hypertables with integer-based timestamps ## 2.2.0 (2021-04-13) This release adds major new features since the 2.1.1 release. We deem it moderate priority for upgrading. This release adds the Skip Scan optimization, which significantly improves the performance of queries with DISTINCT ON. This optimization is not yet available for queries on distributed hypertables. This release also adds a function to create a distributed restore point, which allows performing a consistent restore of a multi-node cluster from a backup. The bug fixes in this release address issues with size and stats functions, high memory usage in distributed inserts, slow distributed ORDER BY queries, indexes involving INCLUDE, and single chunk query planning. **PostgreSQL 11 deprecation announcement** Timescale is working hard on our next exciting features. To make that possible, we require functionality that is unfortunately absent on PostgreSQL 11. For this reason, we will continue supporting PostgreSQL 11 until mid-June 2021. Sooner to that time, we will announce the specific version of TimescaleDB in which PostgreSQL 11 support will not be included going forward. **Major Features** * #2843 Add distributed restore point functionality * #3000 SkipScan to speed up SELECT DISTINCT **Bugfixes** * #2989 Refactor and harden size and stats functions * #3058 Reduce memory usage for distributed inserts * #3067 Fix extremely slow multi-node order by queries * #3082 Fix chunk index column name mapping * #3083 Keep Append pathkeys in ChunkAppend **Thanks** * @BowenGG for reporting an issue with indexes with INCLUDE * @fvannee for reporting an issue with ChunkAppend pathkeys * @pedrokost and @RobAtticus for reporting an issue with size functions on empty hypertables * @phemmer and @ryanbooz for reporting issues with slow multi-node order by queries * @stephane-moreau for reporting an issue with high memory usage during single-transaction inserts on a distributed hypertable. ## 2.1.1 (2021-03-29) This maintenance release contains bugfixes since the 2.1.0 release. We deem it high priority for upgrading. The bug fixes in this release address issues with CREATE INDEX and UPSERT for hypertables, custom jobs, and gapfill queries. This release marks TimescaleDB as a trusted extension in PG13, so that superuser privileges are not required anymore to install the extension. **Minor features** * #2998 Mark timescaledb as trusted extension **Bugfixes** * #2948 Fix off by 4 error in histogram deserialize * #2974 Fix index creation for hypertables with dropped columns * #2990 Fix segfault in job_config_check for cagg * #2987 Fix crash due to txns in emit_log_hook_callback * #3042 Commit end transaction for CREATE INDEX * #3053 Fix gapfill/hashagg planner interaction * #3059 Fix UPSERT on hypertables with columns with defaults **Thanks** * @eloyekunle and @kitwestneat for reporting an issue with UPSERT * @jocrau for reporting an issue with index creation * @kev009 for fixing a compilation issue * @majozv and @pehlert for reporting an issue with time_bucket_gapfill ## 2.1.0 (2021-02-22) This release adds major new features since the 2.0.2 release. We deem it moderate priority for upgrading. This release adds the long-awaited support for PostgreSQL 13 to TimescaleDB. The minimum required PostgreSQL 13 version is 13.2 due to a security vulnerability affecting TimescaleDB functionality present in earlier versions of PostgreSQL 13. This release also relaxes some restrictions for compressed hypertables; namely, TimescaleDB now supports adding columns to compressed hypertables and renaming columns of compressed hypertables. **Major Features** * #2779 Add support for PostgreSQL 13 **Minor features** * #2736 Support adding columns to hypertables with compression enabled * #2909 Support renaming columns of hypertables with compression enabled ## 2.0.2 (2021-02-19) This maintenance release contains bugfixes since the 2.0.1 release. We deem it high priority for upgrading. The bug fixes in this release address issues with joins, the status of background jobs, and disabling compression. It also includes enhancements to continuous aggregates, including improved validation of policies and optimizations for faster refreshes when there are a lot of invalidations. **Minor features** * #2926 Optimize cagg refresh for small invalidations **Bugfixes** * #2850 Set status for backend in background jobs * #2883 Fix join qual propagation for nested joins * #2884 Add GUC to control join qual propagation * #2885 Fix compressed chunk check when disabling compression * #2908 Fix changing column type of clustered hypertables * #2942 Validate continuous aggregate policy **Thanks** * @zeeshanshabbir93 for reporting an issue with joins * @Antiarchitect for reporting the issue with slow refreshes of continuous aggregates. * @diego-hermida for reporting the issue about being unable to disable compression * @mtin for reporting the issue about wrong job status ## 1.7.5 (2021-02-12) This maintenance release contains bugfixes since the 1.7.4 release. Most of these fixes were backported from the 2.0.0 and 2.0.1 releases. We deem it high priority for upgrading for users on TimescaleDB 1.7.4 or previous versions. In particular the fixes contained in this maintenance release address issues in continuous aggregates, compression, JOINs with hypertables, and when upgrading from previous versions. **Bugfixes** * #2502 Replace check function when updating * #2558 Repair dimension slice table on update * #2619 Fix segfault in decompress_chunk for chunks with dropped columns * #2664 Fix support for complex aggregate expression * #2800 Lock dimension slices when creating new chunk * #2860 Fix projection in ChunkAppend nodes * #2865 Apply volatile function quals at decompresschunk * #2851 Fix nested loop joins that involve compressed chunks * #2868 Fix corruption in gapfill plan * #2883 Fix join qual propagation for nested joins * #2885 Fix compressed chunk check when disabling compression * #2920 Fix repair in update scripts **Thanks** * @akamensky for reporting several issues including segfaults after version update * @alex88 for reporting an issue with joined hypertables * @dhodyn for reporting an issue when joining compressed chunks * @diego-hermida for reporting an issue with disabling compression * @Netskeh for reporting bug on time_bucket problem in continuous aggregates * @WarriorOfWire for reporting the bug with gapfill queries not being able to find pathkey item to sort * @zeeshanshabbir93 for reporting an issue with joins ## 2.0.1 (2021-01-28) This maintenance release contains bugfixes since the 2.0.0 release. We deem it high priority for upgrading. In particular the fixes contained in this maintenance release address issues in continuous aggregates, compression, JOINs with hypertables and when upgrading from previous versions. **Bugfixes** * #2772 Always validate existing database and extension * #2780 Fix config enum entries for remote data fetcher * #2806 Add check for dropped chunk on update * #2828 Improve cagg watermark caching * #2838 Fix catalog repair in update script * #2842 Do not mark job as started when setting next_start field * #2845 Fix continuous aggregate privileges during upgrade * #2851 Fix nested loop joins that involve compressed chunks * #2860 Fix projection in ChunkAppend nodes * #2861 Remove compression stat update from update script * #2865 Apply volatile function quals at decompresschunk node * #2866 Avoid partitionwise planning of partialize_agg * #2868 Fix corruption in gapfill plan * #2874 Fix partitionwise agg crash due to uninitialized memory **Thanks** * @alex88 for reporting an issue with joined hypertables * @brian-from-quantrocket for reporting an issue with extension update and dropped chunks * @dhodyn for reporting an issue when joining compressed chunks * @markatosi for reporting a segfault with partitionwise aggregates enabled * @PhilippJust for reporting an issue with add_job and initial_start * @sgorsh for reporting an issue when using pgAdmin on windows * @WarriorOfWire for reporting the bug with gapfill queries not being able to find pathkey item to sort ## 2.0.0 (2020-12-18) With this release, we are officially moving TimescaleDB 2.0 to GA, concluding several release candidates. TimescaleDB 2.0 adds the much-anticipated support for distributed hypertables (multi-node TimescaleDB), as well as new features and enhancements to core functionality to give users better clarity and more control and flexibility over their data. Multi-node architecture: In particular, with TimescaleDB 2.0, users can now create distributed hypertables across multiple instances of TimescaleDB, configured so that one instance serves as an access node and multiple others as data nodes. All queries for a distributed hypertable are issued to the access node, but inserted data and queries are pushed down across data nodes for greater scale and performance. Multi-node TimescaleDB can be self managed or, for easier operation, launched within Timescale's fully-managed cloud services. This release also adds: * Support for user-defined actions, allowing users to define, customize, and schedule automated tasks, which can be run by the built-in jobs scheduling framework now exposed to users. * Significant changes to continuous aggregates, which now separate the view creation from the policy. Users can now refresh individual regions of the continuous aggregate materialized view, or schedule automated refreshing via policy. * Redesigned informational views, including new (and more general) views for information about hypertable's dimensions and chunks, policies and user-defined actions, as well as support for multi-node TimescaleDB. * Moving all formerly enterprise features into our Community Edition, and updating Timescale License, which now provides additional (more permissive) rights to users and developers. Some of the changes above (e.g., continuous aggregates, updated informational views) do introduce breaking changes to APIs and are not backwards compatible. While the update scripts in TimescaleDB 2.0 will upgrade databases running TimescaleDB 1.x automatically, some of these API and feature changes may require changes to clients and/or upstream scripts that rely on the previous APIs. Before upgrading, we recommend reviewing upgrade documentation at docs.timescale.com for more details. **Major Features** TimescaleDB 2.0 moves the following major features to GA: * #1923 Add support for distributed hypertables * #2006 Add support for user-defined actions * #2125 #2221 Improve Continuous Aggregate API * #2084 #2089 #2098 #2417 Redesign informational views * #2435 Move enterprise features to community * #2437 Update Timescale License **Previous Release Candidates** * #2702 Release Candidate 4 (December 2, 2020) * #2637 Release Candidate 3 (November 12, 2020) * #2554 Release Candidate 2 (October 20, 2020) * #2478 Release Candidate 1 (October 1, 2020) **Minor Features** Since the last release candidate 4, there are several minor improvements: * #2746 Optimize locking for create chunk API * #2705 Block tableoid access on distributed hypertable * #2730 Do not allow unique index on compressed hypertables * #2764 Bootstrap data nodes with versioned extension **Bugfixes** Since the last release candidate 4, there are several bugfixes: * #2719 Support disabling compression on distributed hypertables * #2742 Fix compression status in chunks view for distributed chunks * #2751 Fix crash and cancel when adding data node * #2763 Fix check constraint on hypertable metadata table **Thanks** Thanks to all contributors for the TimescaleDB 2.0 release: * @airton-neto for reporting a bug in executing some queries with UNION * @nshah14285 for reporting an issue with propagating privileges * @kalman5 for reporting an issue with renaming constraints * @LbaNeXte for reporting a bug in decompression for queries with subqueries * @semtexzv for reporting an issue with continuous aggregates on int-based hypertables * @mr-ns for reporting an issue with privileges for creating chunks * @cloud-rocket for reporting an issue with setting an owner on continuous aggregate * @jocrau for reporting a bug during creating an index with transaction per chunk * @fvannee for reporting an issue with custom time types * @ArtificialPB for reporting a bug in executing queries with conditional ordering on compressed hypertable * @dutchgecko for reporting an issue with continuous aggregate datatype handling * @lambdaq for suggesting to improve error message in continuous aggregate creation * @francesco11112 for reporting memory issue on COPY * @Netskeh for reporting bug on time_bucket problem in continuous aggregates * @mr-ns for reporting the issue with CTEs on distributed hypertables * @akamensky for reporting an issue with recursive cache invalidation * @ryanbooz for reporting slow queries with real-time aggregation on continuous aggregates * @cevian for reporting an issue with disabling compression on distributed hypertables ## 2.0.0-rc4 (2020-12-02) This release candidate contains bugfixes since the previous release candidate, as well as additional minor features. It improves validation of configuration changes for background jobs, adds support for gapfill on distributed tables, contains improvements to the memory handling for large COPY, and contains improvements to compression for distributed hypertables. **Minor Features** * #2689 Check configuration in alter_job and add_job * #2696 Support gapfill on distributed hypertable * #2468 Show more information in get_git_commit * #2678 Include user actions into job stats view * #2664 Fix support for complex aggregate expression * #2672 Add hypertable to continuous aggregates view * #2662 Save compression metadata settings on access node * #2707 Introduce additional db for data node bootstrapping **Bugfixes** * #2688 Fix crash for concurrent drop and compress chunk * #2666 Fix timeout handling in async library * #2683 Fix crash in add_job when given NULL interval * #2698 Improve memory handling for remote COPY * #2555 Set metadata for chunks compressed before 2.0 **Thanks** * @francesco11112 for reporting memory issue on COPY * @Netskeh for reporting bug on time_bucket problem in continuous aggregates ## 2.0.0-rc3 (2020-11-12) This release candidate contains bugfixes since the previous release candidate, as well as additional minor features including support for "user-mapping" authentication between access/data nodes and an experimental API for refreshing continuous aggregates on individual chunks. **Minor Features** * #2627 Add optional user mappings support * #2635 Add API to refresh continuous aggregate on chunk **Bugfixes** * #2560 Fix SCHEMA DROP CASCADE with continuous aggregates * #2593 Set explicitly all lock parameters in alter_job * #2604 Fix chunk creation on hypertables with foreign key constraints * #2610 Support analyze of internal compression table * #2612 Optimize internal cagg_watermark function * #2613 Refresh correct partial during refresh on drop * #2617 Fix validation of available extensions on data node * #2619 Fix segfault in decompress_chunk for chunks with dropped columns * #2620 Fix DROP CASCADE for continuous aggregate * #2625 Fix subquery errors when using AsyncAppend * #2626 Fix incorrect total_table_pages setting for compressed scan * #2628 Stop recursion in cache invalidation **Thanks** * @mr-ns for reporting the issue with CTEs on distributed hypertables * @akamensky for reporting an issue with recursive cache invalidation * @ryanbooz for reporting slow queries with real-time aggregation on continuous aggregates ## 2.0.0-rc2 (2020-10-21) This release candidate contains bugfixes since the previous release candidate. **Minor Features** * #2520 Support non-transactional distibuted_exec **Bugfixes** * #2307 Overflow handling for refresh policy with integer time * #2503 Remove error for correct bootstrap of data node * #2507 Fix validation logic when adding a new data node * #2510 Fix outer join qual propagation * #2514 Lock dimension slices when creating new chunk * #2515 Add if_attached argument to detach_data_node() * #2517 Fix member access within misaligned address in chunk_update_colstats * #2525 Fix index creation on hypertables with dropped columns * #2543 Pass correct status to lock_job * #2544 Assume custom time type range is same as bigint * #2563 Fix DecompressChunk path generation * #2564 Improve continuous aggregate datatype handling * #2568 Change use of ssl_dir GUC * #2571 Make errors and messages conform to style guide * #2577 Exclude compressed chunks from ANALYZE/VACUUM ## 2.0.0-rc1 (2020-10-06) This release adds major new features and bugfixes since the 1.7.4 release. We deem it moderate priority for upgrading. This release adds the long-awaited support for distributed hypertables to TimescaleDB. With 2.0, users can create distributed hypertables across multiple instances of TimescaleDB, configured so that one instance serves as an access node and multiple others as data nodes. All queries for a distributed hypertable are issued to the access node, but inserted data and queries are pushed down across data nodes for greater scale and performance. This release also adds support for user-defined actions allowing users to define actions that are run by the TimescaleDB automation framework. In addition to these major new features, the 2.0 branch introduces _breaking_ changes to APIs and existing features, such as continuous aggregates. These changes are not backwards compatible and might require changes to clients and/or scripts that rely on the previous APIs. Please review our updated documentation and do proper testing to ensure compatibility with your existing applications. The noticeable breaking changes in APIs are: - Redefined functions for policies - A continuous aggregate is now created with `CREATE MATERIALIZED VIEW` instead of `CREATE VIEW` and automated refreshing requires adding a policy via `add_continuous_aggregate_policy` - Redesign of informational views, including new (and more general) views for information about policies and user-defined actions This release candidate is upgradable, so if you are on a previous release (e.g., 1.7.4) you can upgrade to the release candidate and later expect to be able to upgrade to the final 2.0 release. However, please carefully consider your compatibility requirements _before_ upgrading. **Major Features** * #1923 Add support for distributed hypertables * #2006 Add support for user-defined actions * #2435 Move enterprise features to community * #2437 Update Timescale License **Minor Features** * #2011 Constify TIMESTAMPTZ OP INTERVAL in constraints * #2105 Support moving compressed chunks **Bugfixes** * #1843 Improve handling of "dropped" chunks * #1886 Change ChunkAppend leader to use worker subplan * #2116 Propagate privileges from hypertables to chunks * #2263 Fix timestamp overflow in time_bucket optimization * #2270 Fix handling of non-reference counted TupleDescs in gapfill * #2325 Fix rename constraint/rename index * #2370 Fix detection of hypertables in subqueries * #2376 Fix caggs width expression handling on int based hypertables * #2416 Check insert privileges to create chunk * #2428 Allow owner change of continuous aggregate * #2436 Propagate grants in continuous aggregates ## 2.0.0-beta6 (2020-09-14) **For beta releases**, upgrading from an earlier version of the extension (including previous beta releases) is not supported. This beta release includes breaking changes to APIs. The most notable changes since the beta-5 release are the following, which will be reflected in forthcoming documentation for the 2.0 release. * Existing information views were reorganized. Retrieving information about sizes and statistics was moved to functions. New views were added to expose information, which was previously available only internally. * New ability to create custom jobs was added. * Continuous aggregate API was redesigned. Its policy creation is separated from the view creation. * compress_chunk_policy and drop_chunk_policy were renamed to compression_policy and retention_policy. ## 1.7.4 (2020-09-07) This maintenance release contains bugfixes since the 1.7.3 release. We deem it high priority for upgrading if TimescaleDB is deployed with replicas (synchronous or asynchronous). In particular the fixes contained in this maintenance release address an issue with running queries on compressed hypertables on standby nodes. **Bugfixes** * #2340 Remove tuple lock on select path ## 1.7.3 (2020-07-27) This maintenance release contains bugfixes since the 1.7.2 release. We deem it high priority for upgrading. In particular the fixes contained in this maintenance release address issues in compression, drop_chunks and the background worker scheduler. **Bugfixes** * #2059 Improve infering start and stop arguments from gapfill query * #2067 Support moving compressed chunks * #2068 Apply SET TABLESPACE for compressed chunks * #2090 Fix index creation with IF NOT EXISTS for existing indexes * #2092 Fix delete on tables involving hypertables with compression * #2164 Fix telemetry installed_time format * #2184 Fix background worker scheduler memory consumption * #2222 Fix `negative bitmapset member not allowed` in decompression * #2255 Propagate privileges from hypertables to chunks * #2256 Fix segfault in chunk_append with space partitioning * #2259 Fix recursion in cache processing * #2261 Lock dimension slice tuple when scanning **Thanks** * @akamensky for reporting an issue with drop_chunks and ChunkAppend with space partitioning * @dewetburger430 for reporting an issue with setting tablespace for compressed chunks * @fvannee for reporting an issue with cache invalidation * @nexces for reporting an issue with ChunkAppend on space-partitioned hypertables * @PichetGoulu for reporting an issue with index creation and IF NOT EXISTS * @prathamesh-sonpatki for contributing a typo fix * @sezaru for reporting an issue with background worker scheduler memory consumption ## 1.7.2 (2020-07-07) This maintenance release contains bugfixes since the 1.7.1 release. We deem it medium priority for upgrading. In particular the fixes contained in this maintenance release address bugs in continuous aggregates, drop_chunks and compression. **Features** * #1877 Add support for fast pruning of inlined functions **Bugfixes** * #1908 Fix drop_chunks with unique constraints when cascade_to_materializations is false * #1915 Check for database in extension_current_state * #1918 Unify chunk index creation * #1932 Change compression locking order * #1938 Fix gapfill locf treat_null_as_missing * #1982 Check for disabled telemetry earlier * #1984 Fix compression bit array left shift count * #1997 Add checks for read-only transactions * #2002 Reset restoring gucs rather than explicitly setting 'off' * #2028 Fix locking in drop_chunks * #2031 Enable compression for tables with compound foreign key * #2039 Fix segfault in create_trigger_handler * #2043 Fix segfault in cagg_update_view_definition * #2046 Use index tablespace during chunk creation * #2047 Better handling of chunk insert state destruction * #2049 Fix handling of PlaceHolderVar in DecompressChunk * #2051 Fix tuple concurrently deleted error with multiple continuous aggregates **Thanks** * @akamensky for reporting an issue with telemetry and an issue with drop_chunks * @darko408 for reporting an issue with decompression * @dmitri191 for reporting an issue with failing background workers * @eduardotsj for reporting an issue with indexes not inheriting tablespace settings * @fourseventy for reporting an issue with multiple continuous aggregrates * @fvannee for contributing optimizations for pruning inlined functions * @jflambert for reporting an issue with failing telemetry jobs * @nbouscal for reporting an issue with compression jobs locking referenced tables * @nicolai6120 for reporting an issue with locf and treat_null_as_missing * @nomanor for reporting an issue with expression index with table references * @olernov for contributing a fix for compressing tables with compound foreign keys * @werjo for reporting an issue with drop_chunks and unique constraints ## 2.0.0-beta5 (2020-06-08) This release adds new functionality on distributed hypertables, including (but not limited to) basic LIMIT pushdown, manual chunk compression, table access methods storage options, SERIAL columns, and altering of the replication factor. This release only supports PG11 and PG12. Thus, PG9.6 and PG10 are no longer supported. Note that the 2.0 major release will introduce breaking changes to user functions and APIs. In particular, this beta removes the cascade parameter from drop_chunks and changes the names of certain GUC parameters. Expect additional breaking changes to be introduced up until the 2.0 release. **For beta releases**, upgrading from an earlier version of the extension (including previous beta releases) is not supported. **Features** * #1877 Add support for fast pruning of inlined functions * #1922 Cleanup GUC names * #1923 Add repartition option on detach/delete_data_node * #1923 Allow ALTER TABLE SET on distributed hypertable * #1923 Allow SERIAL columns for distributed hypertables * #1923 Basic LIMIT push down support * #1923 Implement altering replication factor * #1923 Support compression on distributed hypertables * #1923 Support storage options for distributed hypertables * #1941 Change default prefix for distributed tables * #1943 Support table access methods for distributed hypertables * #1952 Remove cascade option from drop_chunks * #1955 Remove support for PG9.6 and PG10 **Bugfixes** * #1915 Check for database in extension_current_state * #1918 Unify chunk index creation * #1923 Fix insert batch size calculation for prepared statements * #1923 Fix port conversion issue in add_data_node * #1932 Change compression locking order * #1938 Fix gapfill locf treat_null_as_missing **Thanks** * @dmitri191 for reporting an issue with failing background workers * @fvannee for optimizing pruning of inlined functions * @nbouscal for reporting an issue with compression jobs locking referenced tables * @nicolai6120 for reporting an issue with locf and treat_null_as_missing * @nomanor for reporting an issue with expression index with table references ## 1.7.1 (2020-05-18) This maintenance release contains bugfixes since the 1.7.0 release. We deem it medium priority for upgrading and high priority for users with multiple continuous aggregates. In particular the fixes contained in this maintenance release address bugs in continuous aggregates with real-time aggregation and PostgreSQL 12 support. **Bugfixes** * #1834 Define strerror() for Windows * #1846 Fix segfault on COPY to hypertable * #1850 Fix scheduler failure due to bad next_start_time for jobs * #1851 Fix hypertable expansion for UNION ALL * #1854 Fix reorder policy job to skip compressed chunks * #1861 Fix qual pushdown for compressed hypertables where quals have casts * #1864 Fix issue with subplan selection in parallel ChunkAppend * #1868 Add support for WHERE, HAVING clauses with real time aggregates * #1869 Fix real time aggregate support for multiple continuous aggregates * #1871 Don't rely on timescaledb.restoring for upgrade * #1875 Fix hypertable detection in subqueries * #1884 Fix crash on SELECT WHERE NOT with empty table **Thanks** * @airton-neto for reporting an issue with queries over UNIONs of hypertables * @dhodyn for reporting an issue with UNION ALL queries * @frostwind for reporting an issue with casts in where clauses on compressed hypertables * @fvannee for reporting an issue with hypertable detection in inlined SQL functions and an issue with COPY * @hgiasac for reporting missing where clause with real time aggregates * @louisth for reporting an issue with real-time aggregation and multiple continuous aggregates * @michael-sayapin for reporting an issue with INSERTs and WHERE NOT EXISTS * @olernov for reporting and fixing an issue with compressed chunks in the reorder policy * @pehlert for reporting an issue with pg_upgrade ## 1.7.0 (2020-04-16) This release adds major new features and bugfixes since the 1.6.1 release. We deem it moderate priority for upgrading. This release adds the long-awaited support for PostgreSQL 12 to TimescaleDB. This release also adds a new default behavior when querying continuous aggregates that we call real-time aggregation. A query on a continuous aggregate will now combine materialized data with recent data that has yet to be materialized. Note that only newly created continuous aggregates will have this real-time query behavior, although it can be enabled on existing continuous aggregates with a configuration setting as follows: ALTER VIEW continuous_view_name SET (timescaledb.materialized_only=false); This release also moves several data management lifecycle features to the Community version of TimescaleDB (from Enterprise), including data reordering and data retention policies. **Major Features** * #1456 Add support for PostgreSQL 12 * #1685 Add support for real-time aggregation on continuous aggregates **Bugfixes** * #1665 Add ignore_invalidation_older_than to timescaledb_information.continuous_aggregates view * #1750 Handle undefined ignore_invalidation_older_than * #1757 Restrict watermark to max for continuous aggregates * #1769 Add rescan function to CompressChunkDml CustomScan node * #1785 Fix last_run_success value in continuous_aggregate_stats view * #1801 Include parallel leader in plan execution * #1808 Fix ts_hypertable_get_all for compressed tables * #1828 Ignore dropped chunks in compressed_chunk_stats **Licensing changes** * Reorder and policies around reorder and drop chunks are now accessible to community users, not just enterprise * Gapfill functionality no longer warns about expired license **Thanks** * @t0k4rt for reporting an issue with parallel chunk append plans * @alxndrdude for reporting an issue when trying to insert into compressed chunks * @Olernov for reporting and fixing an issue with show_chunks and drop_chunks for compressed hypertables * @mjb512 for reporting an issue with INSERTs in CTEs in cached plans * @dmarsh19 for reporting and fixing an issue with dropped chunks in compressed_chunk_stats ## 1.6.1 (2020-03-18) This maintenance release contains bugfixes since the 1.6.0 release. We deem it medium priority for upgrading. In particular the fixes contained in this maintenance release address bugs in continuous aggregates, time_bucket_gapfill, partial index handling and drop_chunks. **For this release only**, you will need to restart the database after upgrade before restoring a backup. **Minor Features** * #1666 Support drop_chunks API for continuous aggregates * #1711 Change log level for continuous aggregate materialization messages **Bugfixes** * #1630 Print notice for COPY TO on hypertable * #1648 Drop chunks from materialized hypertable * #1668 Cannot add dimension if hypertable has empty chunks * #1673 Fix crash when interrupting create_hypertable * #1674 Fix time_bucket_gapfill's interaction with GROUP BY * #1686 Fix order by queries on compressed hypertables that have char segment by column * #1687 Fix issue with disabling compression when foreign keys are present * #1688 Handle many BGW jobs better * #1698 Add logic to ignore dropped chunks in hypertable_relation_size * #1704 Fix bad plan for continuous aggregate materialization * #1709 Prevent starting background workers with NOLOGIN * #1713 Fix miscellaneous background worker issues * #1715 Fix issue with overly aggressive chunk exclusion in outer joins * #1719 Fix restoring/scheduler entrypoint to avoid BGW death * #1720 Add scheduler cache invalidations * #1727 Fix compressing INTERVAL columns * #1728 Handle Sort nodes in ConstraintAwareAppend * #1730 Fix partial index handling on hypertables * #1739 Use release OpenSSL DLLs for debug builds on Windows * #1740 Fix invalidation entries from multiple caggs on same hypertable * #1743 Fix continuous aggregate materialization timezone handling * #1748 Fix remove_drop_chunks_policy for continuous aggregates * #1756 Fix handling of dropped chunks in compression background worker **Thanks** * @RJPhillips01 for reporting an issue with drop chunks. * @b4eEx for reporting an issue with disabling compression. * @darko408 for reporting an issue with order by on compressed hypertables * @mrechte for reporting an issue with compressing INTERVAL columns * @tstaehli for reporting an issue with ConstraintAwareAppend * @chadshowalter for reporting an issue with partial index on hypertables * @geoffreybennett for reporting an issue with create_hypertable when interrupting operations * @alxndrdude for reporting an issue with background workers during restore * @zcavaliero for reporting and fixing an issue with dropped columns in hypertable_relation_size * @ismailakpolat for reporting an issue with cagg materialization on hypertables with TIMESTAMP column ## 1.6.0 (2020-01-14) This release adds major new features and bugfixes since the 1.5.1 release. We deem it moderate priority for upgrading. The major new feature in this release allows users to keep the aggregated data in a continuous aggregate while dropping the raw data with drop_chunks. This allows users to save storage by keeping only the aggregates. The semantics of the refresh_lag parameter for continuous aggregates has been changed to be relative to the current timestamp instead of the maximum value in the table. This change requires that an integer_now func be set on hypertables with integer-based time columns to use continuous aggregates on this table. We added a timescaledb.ignore_invalidation_older_than parameter for continuous aggregates. This parameter accept a time-interval (e.g. 1 month). If set, it limits the amount of time for which to process invalidation. Thus, if timescaledb.ignore_invalidation_older_than = '1 month', then any modifications for data older than 1 month from the current timestamp at modification time may not cause continuous aggregate to be updated. This limits the amount of work that a backfill can trigger. By default, all invalidations are processed. **Major Features** * #1589 Allow drop_chunks while keeping continuous aggregates **Minor Features** * #1568 Add ignore_invalidation_older_than option to continuous aggs * #1575 Reorder group-by clause for continuous aggregates * #1592 Improve continuous agg user messages **Bugfixes** * #1565 Fix partial select query for continuous aggregate * #1591 Fix locf treat_null_as_missing option * #1594 Fix error in compression constraint check * #1603 Add join info to compressed chunk * #1606 Fix constify params during runtime exclusion * #1607 Delete compression policy when drop hypertable * #1608 Add jobs to timescaledb_information.policy_stats * #1609 Fix bug with parent table in decompression * #1624 Fix drop_chunks for ApacheOnly * #1632 Check for NULL before dereferencing variable **Thanks** * @optijon for reporting an issue with locf treat_null_as_missing option * @acarrera42 for reporting an issue with constify params during runtime exclusion * @ChristopherZellermann for reporting an issue with the compression constraint check * @SimonDelamare for reporting an issue with joining hypertables with compression ## 2.0.0-beta4 (2019-12-19) **For beta releases**, upgrading from an earlier version of the extension (including previous beta releases) is not supported. This release includes user experience improvements for managing data nodes, more efficient statistics collection for distributed hypertables, and miscellaneous fixes and improvements. ## 1.5.1 (2019-11-12) This maintenance release contains bugfixes since the 1.5.0 release. We deem it low priority for upgrading. In particular the fixes contained in this maintenance release address potential segfaults and no other security vulnerabilities. The bugfixes are related to bloom indexes and updates from previous versions. **Bugfixes** * #1523 Fix bad SQL updates from previous updates * #1526 Fix hypertable model * #1530 Set active snapshots in multi-xact index create **Thanks** * @84660320 for reporting an issue with bloom indexes * @gumshoes @perhamm @jermudgeon @gmisagm for reporting the issue with updates ## 2.0.0-beta3 (2019-11-05) **For beta releases**, upgrading from an earlier version of the extension (including previous beta releases) is not supported. This release improves performance for queries executed on distributed hypertables, fixes minor issues and blocks a number of SQL API functions, which are not supported on distributed hypertables. It also adds information about distributed databases in the telemetry. ## 2.0.0-beta2 (2019-10-22) This release introduces *distributed hypertables*, a major new feature that allows hypertables to scale out across multiple nodes for increased performance and fault tolerance. Please review the documentation to learn how to configure and use distributed hypertables and what current limitations are. ## 1.5.0 (2019-10-31) This release adds major new features and bugfixes since the 1.4.2 release. We deem it moderate priority for upgrading. This release adds compression as a major new feature. Multiple type-specific compression options are available in this release (including DeltaDelta with run-length-encoding for integers and timestamps; Gorilla compression for floats; dictionary-based compression for any data type, but specifically for low-cardinality datasets; and other LZ-based techniques). Individual columns can be compressed with type-specific compression algorithms as Postgres' native row-based format are rolled up into columnar-like arrays on a per chunk basis. The query planner then handles transparent decompression for compressed chunks at execution time. This release also adds support for basic data tiering by supporting the migration of chunks between tablespaces, as well as support for parallel query coordination to the ChunkAppend node. Previously ChunkAppend would rely on parallel coordination in the underlying scans for parallel plans. More information can be found on [our blog](https://blog.timescale.com/blog/building-columnar-compression-in-a-row-oriented-database) or in this [tutorial](https://docs.timescale.com/latest/tutorials/compression-tutorial) **For this release only**, you will need to restart the database before running `ALTER EXTENSION` **Major Features** * #1393 Moving chunks between different tablespaces * #1433 Make ChunkAppend parallel aware * #1434 Introducing native compression, multiple compression algorithms, and hybrid row/columnar projections **Minor Features** * #1471 Allow setting reloptions on chunks * #1479 Add next_start option to alter_job_schedule * #1481 Add last_successful_finish to bgw_job_stats **Bugfixes** * #1444 Prevent LIMIT pushdown in JOINs * #1447 Fix runtime exclusion memory leak * #1464 Fix ordered append with expressions in ORDER BY clause with space partitioning * #1476 Fix logic for BGW rescheduling * #1477 Fix gapfill treat_null_as_missing * #1493 Prevent recursion in invalidation processing * #1498 Fix overflow in gapfill's interpolate * #1499 Fix error for exported_uuid in pg_restore * #1503 Fix bug with histogram function in parallel **Thanks** * @dhyun-obsec for reporting an issue with pg_restore * @rhaymo for reporting an issue with interpolate * @optijon for reporting an issue with locf treat_null_as_missing * @fvannee for reporting an issue with runtime exclusion * @Lectem for reporting an issue with histograms * @rogerdwan for reporting an issue with BGW rescheduling * @od0 for reporting an issue with alter_job_schedule ## 1.4.2 (2019-09-11) This maintenance release contains bugfixes since the 1.4.1 release. We deem it medium priority for upgrading. In particular the fixes contained in this maintenance release address 2 potential segfaults and no other security vulnerabilities. The bugfixes are related to background workers, OUTER JOINs, ordered append on space partitioned hypertables and expression indexes. **Bugfixes** * #1327 Fix chunk exclusion with ordered append * #1390 Fix deletes of background workers while a job is running * #1392 Fix cagg_agg_validate expression handling (segfault) * #1408 Fix ChunkAppend space partitioning support for ordered append * #1420 Fix OUTER JOIN qual propagation * #1422 Fix background worker error handling (segfault) * #1424 Fix ChunkAppend LIMIT pushdown * #1429 Fix expression index creation **Thanks** * @shahidhk for reporting an issue with OUTER JOINs * @cossbow and @xxGL1TCHxx for reporting reporting issues with ChunkAppend and space partitioning * @est for reporting an issue with CASE expressions in continuous aggregates * @devlucasc for reporting the issue with deleting a background worker while a job is running * @ryan-shaw for reporting an issue with expression indexes on hypertables with dropped columns ## 1.4.1 (2019-08-01) This maintenance release contains bugfixes since the 1.4.0 release. We deem it medium priority for upgrading. In particular the fixes contained in this maintenance release address 2 potential segfaults and no other security vulnerabilities. The bugfixes are related to queries with prepared statements, PL/pgSQL functions and interoperability with other extensions. More details below. **Bugfixes** * #1362 Fix ConstraintAwareAppend subquery exclusion * #1363 Mark drop_chunks as VOLATILE and not PARALLEL SAFE * #1369 Fix ChunkAppend with prepared statements * #1373 Only allow PARAM_EXTERN as time_bucket_gapfill arguments * #1380 Handle Result nodes gracefully in ChunkAppend **Thanks** * @overhacked for reporting an issue with drop_chunks and parallel queries * @fvannee for reporting an issue with ConstraintAwareAppend and subqueries * @rrb3942 for reporting a segfault with ChunkAppend and prepared statements * @mchesser for reporting a segfault with time_bucket_gapfill and subqueries * @lolizeppelin for reporting and helping debug an issue with ChunkAppend and Result nodes ## 1.4.0 (2019-07-18) This release contains major new functionality for continuous aggregates and adds performance improvements for analytical queries. In version 1.3.0 we added support for continuous aggregates which was initially limited to one continuous aggregate per hypertable. With this release, we remove this restriction and allow multiple continuous aggregates per hypertable. This release adds a new custom node ChunkAppend that can perform execution time constraint exclusion and is also used for ordered append. Ordered append no longer requires a LIMIT clause and now supports space partitioning and ordering by time_bucket. **Major features** * #1270 Use ChunkAppend to replace Append nodes * #1257 Support for multiple continuous aggregates **Minor features** * #1181 Remove LIMIT clause restriction from ordered append * #1273 Propagate quals to joined hypertables * #1317 Support time bucket functions in Ordered Append * #1331 Add warning message for REFRESH MATERIALIZED VIEW * #1332 Add job statistics columns to timescaledb_information.continuous_aggregate_stats view * #1326 Add architecture and bit size to telemetry **Bugfixes** * #1288 Do not remove Result nodes with one-time filter * #1300 Fix telemetry report return value * #1339 Fix continuous agg catalog table insert failure * #1344 Update continuous agg bgw job start time **Thanks** * @ik9999 for reporting a bug with continuous aggregates and negative refresh lag ## 1.3.2 (2019-06-24) This maintenance release contains bug and security fixes since the 1.3.1 release. We deem it moderate-to-high priority for upgrading. This release fixes some security vulnerabilities, specifically related to being able to elevate role-based permissions by database users that already have access to the database. We strongly recommend that users who rely on role-based permissions upgrade to this release as soon as possible. **Security Fixes** * #1311 Fix role-based permission checking logic **Bugfixes** * #1315 Fix potentially lost invalidations in continuous aggs * #1303 Fix handling of types with custom time partitioning * #1299 Arm32: Fix Datum to int cast issue * #1297 Arm32: Fix crashes due to long handling * #1019 Add ARM32 tests on travis **Thanks** * @hedayat for reporting the error with handling of types with custom time partitioning ## 1.3.1 (2019-06-10) This maintenance release contains bugfixes since the 1.3.0 release. We deem it low-to-moderate priority for upgrading. In particular, the fixes contained in this maintenance release do not address any security vulnerabilities, while the only one affecting system stability is related to TimescaleDB running on PostgreSQL 11. More details below. **Bugfixes** * #1220 Fix detecting JOINs for continuous aggs * #1221 Fix segfault in VACUUM on PG11 * #1228 ARM32 Fix: Pass int64 using Int64GetDatum when a Datum is required * #1232 Allow Param as time_bucket_gapfill arguments * #1236 Stop preventing REFRESH in transaction blocks * #1283 Fix constraint exclusion for OUTER JOIN **Thanks** * @od0 for reporting an error with continuous aggs and JOINs * @rickbatka for reporting an error when using time_bucket_gapfill in functions * @OneMoreSec for reporting the bug with VACUUM * @dvdrozdov @od0 @t0k4rt for reporting the issue with REFRESH in transaction blocks * @mhagander and @devrimgunduz for suggesting adding a CMAKE flag to control the default telemetry level ## 1.3.0 (2019-05-06) This release contains major new functionality that we call continuous aggregates. Aggregate queries which touch large swathes of time-series data can take a long time to compute because the system needs to scan large amounts of data on every query execution. Our continuous aggregates continuously calculate the results of a query in the background and materialize the results. Queries to the continuous aggregate view are then significantly faster as they do not need to touch the raw data in the hypertable, instead using the pre-computed aggregates in the view. Continuous aggregates are somewhat similar to PostgreSQL materialized views, but unlike a materialized view, continuous aggregates do not need to be refreshed manually; the view will be refreshed automatically in the background as new data is added, or old data is modified. Additionally, it does not need to re-calculate all of the data on every refresh. Only new and/or invalidated data will be calculated. Since this re-aggregation is automatic, it doesn’t add any maintenance burden to your database. Our continuous aggregate approach supports high-ingest rates by avoiding the high-write amplification associated with trigger-based approaches. Instead, we use invalidation techniques to track what data has changed, and then correct the materialized aggregate the next time that the automated process executes. More information can be found on [our docs overview](http://docs.timescale.com/using-timescaledb/continuous-aggregates) or in this [tutorial](http://docs.timescale.com/tutorials/continuous-aggs-tutorial). **Major Features** * #1184 Add continuous aggregate functionality **Minor Features** * #1005 Enable creating indexes with one transaction per chunk * #1007 Remove hypertable parent from query plans * #1038 Infer time_bucket_gapfill arguments from WHERE clause * #1062 Make constraint aware append parallel safe * #1067 Add treat_null_as_missing option to locf * #1112 Add support for window functions to gapfill * #1130 Add support for cross datatype chunk exclusion for time types * #1134 Add support for partitionwise aggregation * #1153 Add time_bucket support to chunk exclusion * #1170 Add functions for turning restoring on/off and setting license key * #1177 Add transformed time_bucket comparison to quals * #1182 Enable optimizing SELECTs within INSERTs * #1201 Add telemetry for policies: drop_chunk & reorder **Bugfixes** * #1010 Add session locks to CLUSTER * #1115 Fix ordered append optimization for join queries * #1123 Fix gapfill with prepared statements * #1125 Fix column handling for columns derived from GROUP BY columns * #1132 Adjust ordered append path cost * #1155 Limit initial max_open_chunks_per_insert to PG_INT16_MAX * #1167 Fix postgres.conf ApacheOnly license * #1183 Handle NULL in a check constraint name * #1195 Fix cascade in scheduled drop chunks * #1196 Fix UPSERT with prepared statements **Thanks** * @spickman for reporting a segfault with ordered append and JOINs * @comicfans for reporting a performance regression with ordered append * @Specter-Y for reporting a segfault with UPSERT and prepared statements * @erthalion submitting a bugfix for a segfault with validating check constraints ## 1.2.2 (2019-03-14) This release contains bugfixes. **Bugfixes** * #1097 Adjust ordered append plan cost * #1079 Stop background worker on ALTER DATABASE SET TABLESPACE and CREATE DATABASE WITH TEMPLATE * #1088 Fix ON CONFLICT when using prepared statements and functions * #1089 Fix compatibility with extensions that define planner_hook * #1057 Fix chunk exclusion constraint type inference * #1060 Fix sort_transform optimization **Thanks** * @esatterwhite for reporting a bug when using timescaledb with zombodb * @eeeebbbbrrrr for fixing compatibility with extensions that also define planner_hook * @naquad for reporting a segfault when using ON conflict in stored procedures * @aaronkaplan for reporting an issue with ALTER DATABASE SET TABLESPACE * @quetz for reporting an issue with CREATE DATABASE WITH TEMPLATE * @nbouscal for reporting an issue with ordered append resulting in bad plans ## 1.2.1 (2019-02-11) This release contains bugfixes. **Notable commits** * [2f6b58a] Fix tlist on hypertable inserts inside CTEs * [7973b4a] Stop background worker on rename database * [32cc645] Fix loading the tsl license in parallel workers **Thanks** * @jamessewell for reporting and helping debug a segfault in last() [034a0b0] * @piscopoc for reporting a segfault in time_bucket_gapfill [e6c68f8] ## 1.2.0 (2019-01-29) **This is our first release to include Timescale-Licensed features, in addition to new Apache-2 capabilities.** We are excited to be introducing new time-series analytical functions, advanced data lifecycle management capabilities, and improved performance. - **Time-series analytical functions**: Users can now use our `time_bucket_gapfill` function, to write complex gapfilling, last object carried forward, and interpolation queries. - **Advanced data lifecycle management**: We are introducing scheduled policies, which use our background worker framework to manage time-series data. In this release, we support scheduled `drop_chunks` and `reorder`. - **Improved performance**: We added support for ordered appends, which optimize a large range of queries - particularly those that are ordered by time and contain a LIMIT clause. Please note that ordered appends do not support ordering by `time_bucket` at this time. - **Postgres 11 Support**: We added beta support for PG11 in 1.1.0. We're happy to announce that our PG11 support is now out of beta, and fully supported. This release adds code under a new license, LICENSE_TIMESCALE. This code can be found in `tsl`. **For this release only**, you will need to restart the database before running `ALTER EXTENSION` **Notable commits** * [a531733] switch cis state when we switch chunks * [5c6b619] Make a copy of the ri_onConflict object in PG11 * [61e524e] Make slot for upserts be update for every chunk switch * [8a7c127] Fix for ExecSlotDescriptor during upserts * [fa61613] Change time_bucket_gapfill argument names * [01be394] Fix bgw_launcher restart when failing during launcher setup * [7b3929e] Add ordered append optimization * [a69f84c] Fix signal processing in background workers * [47b5b7d] Log which chunks are dropped by background workers * [4e1e15f] Add reorder command * [2e4bb5d] Recluster and drop chunks scheduling code * [ef43e52] Add alter_policy_schedule API function * [5ba740e] Add gapfill query support * [be7c74c] Add logic for automatic DB maintenance functions * [4ff6ac7] Initial Timescale-Licensed-Module and License-Key Implementation * [fc42539] Add new top-level licensing information * [31e9c5b] Fix time column handling in get_create_command * [1b8ceca] Avoid loading twice in parallel workers and load only from $libdir * [76d7875] Don't throw errors when extension is loaded but not installed yet * [eecd845] Add Timescale License (TSL) * [4b42b30] Free ChunkInsertStates when the es_per_tuple_exprcontext is freed **Thanks** * @fordred for reporting our docker-run.sh script was out of date * @JpWebster for reporting a deadlock between reads an drop_chunks * @chickenburgers for reporting an issue with our CMake * Dimtrj and Asbjørn D., on slack, for creating a reproducible testcase for an UPSERT bug * @skebanga for reporting a loader bug ## 1.1.1 (2018-12-20) This release contains bugfixes. **High-level changes** * Fix issue when upgrading with pg_upgrade * Fix a segfault that sometimes appeared in long COPYs * Other bug and stability fixes **Notable commits** * [f99b540] Avoid loading twice in parallel workers and load only from $libdir * [e310f7d] Don't throw errors when extension is loaded but not installed yet * [8498416] Free ChunkInsertStates when the es_per_tuple_exprcontext is freed * [937eefe] Set C standard to C11 **Thanks** * @costigator for reporting the pg_upgrade bug * @FireAndIce68 for reporting the parallel workers issue * @damirda for reporting the copy bug ## 1.1.0 (2018-12-13) Our 1.1 release introduces beta support for PG 11, as well as several performance optimizations aimed at improving chunk exclusion for read queries. We are also packaging our new timescale-tune tool (currently in beta) with our Debian and Linux releases. If you encounter any issues with our beta features, please file a Github issue. **Potential breaking changes** - In addition to optimizing first() / last() to utilize indices for non-group-by queries, we adjusted its sorting behavior to match that of PostgreSQL’s max() and min() functions. Previously, if the column being sorted had NULL values, a NULL would be returned. First() and last() now instead ignore NULL values. **Notable Commits** * [71f3a0c] Fix Datum conversion issues * [5aa1eda] Refactor compatibility functions and code to support PG11 * [e4a4f8e] Add support for functions on open (time) dimensions * [ed5067c] Fix interval_from_now_to_internal timestamptz handling * [019971c] Optimize FIRST/LAST aggregate functions * [83014ee] Implement drop_chunks in C * [9a34028] Implement show_chunks in C and have drop_chunks use it * [d461959] Add view to show hypertable information * [35dee48] Remove version-checking from client-side * [5b6a5f4] Change size utility and job functions to STRICT * [7e55d91] Add checks for NULL arguments to DDL functions * [c1db608] Fix upsert TLE translation when mapping variable numbers * [55a378e] Check extension exists for DROP OWNED and DROP EXTENSION * [0c8c085] Exclude unneeded chunks for IN/ANY/ALL operators * [f27c0a3] Move int time_bucket functions with offset to C **Thanks** * @did-g for some memory improvements ## 1.0.1 (2018-12-05) This commit contains bugfixes and optimizations for 1.0.0 **Notable commits** * [6553aa4] Make a number of size utility functions to `STRICT` * [bb1d748] Add checks for NULL arguments to `set_adaptive_chunking`, `set_number_partitions`, `set_chunk_time_interval`, `add_dimension`, and `create_hypertable` * [a534ed4] Fix upsert TLE translation when mapping variable numbers * [aecd55b] Check extension exists for DROP OWNED and DROP EXTENSION ## 1.0.0 (2018-10-30) **This is our 1.0 release!** For notable commits between 0.12.0/0.12.1 and this final 1.0 release, please see previous entries for the release candidates (rc1, rc2, and rc3). **Thanks** To all the external contributors who helped us debug the release candidates, as well as anyone who has contributed bug reports, PRs, or feedback on Slack, GitHub, and other channels. All input has been valuable and helped us create the product we have today! **Potential breaking changes** * To better align with the ISO standard so that time bucketing starts each week by default on a Monday (rather than Saturday), the `time_bucket` epoch/origin has been changed from January 1, 2000 to January 3, 2000. The function now includes an `origin` parameter that can be used to adjust this. * Error codes are now prefixed with `TS` instead of the prior `IO` prefix. If you were checking for these error codes by name, please update your code. ## 1.0.0-rc3 (2018-10-18) This release is our third 1.0 release candidate. We expect to only merge bug fixes between now and our final 1.0 release. This is a big milestone for us and signifies our maturity and enterprise readiness. **PLEASE NOTE** that release candidate (rc) builds will only be made available via GitHub and Docker, and _not_ on other release channels. Please help us test these release candidates out if you can! **Potential breaking change**: Starting with rc2, we updated our error codes to be prefixed with `TS` instead of the old `IO` prefix. If you were checking for these error codes by name, please update your checks. **Notable commits** * [f7ba13d] Handle and test tablespace changes to and from the default tablespace * [9ccda0d] Start stopped workers on restart message * [3e3bb0c] Add bool created to create_hypertable and add_dimension return value * [53ff656] Add state machine and polling to launcher * [d9b2dfe] Change return value of add_dimension to TABLE * [19299cf] Make all time_bucket function STRICT * [297d885] Add a version of time_bucket that takes an origin * [e74be30] Move time_bucket epoch to a Monday * [46564c1] Handle ALTER SCHEMA RENAME properly * [a83e283] Change return value of create_hypertable to TABLE * [aea7c7e] Add GRANTs to update script for pg_dump to work * [119963a] Replace hardcoded bash path in shell scripts **Thanks** * @jesperpedersen for several PRs that help improve documentation and some rough edges * @did-g for improvements to our build process * @skebanga for reporting an issue with ALTER SCHEMA RENAME * @p-alik for suggesting a way to improve our bash scripts' portability * @mx781 and @HeikoOnnebrink for reporting an issues with permission GRANTs and ownership when using pg_dump ## 1.0.0-rc2 (2018-09-27) This release is our second 1.0 release candidate. We expect to only merge bug fixes between now and our final 1.0 release. This is a big milestone for us and signifies our maturity and enterprise readiness. **PLEASE NOTE** that release candidate (rc) builds will only be made available via GitHub and Docker, and _not_ on other release channels. Please help us test these release candidates out if you can! **Potential breaking change**: We updated our error codes to be prefixed with `TS` instead of the old `IO` prefix. If you were checking for these error codes by name, please update your checks. **Notable commits** * [b43574f] Switch 'IO' error prefix to 'TS' * [9747885] Prefix public C functions with ts_ * [39510c3] Block unknown alter table commands on hypertables * [2408a83] Add support for ALTER TABLE SET TABLESPACE on hypertables * [41d9846] Enclose macro replacement list and arguments in parentheses * [cc59d51] Replace macro LEAST_TIMESTAMP by a static function * [281f363] Modify telemetry BGW to run every hour the first 12 hours * [a09b3ec] Add pg_isolation_regress support to the timescale build system * [2c267ba] Handle SIGTERM/SIGINT asynchronously * [5377e2d] Fix use-after-free bug for connections in the telemetry BGW * [248f662] Fix pg_dump for unprivileged users * [193fa4a] Stop background workers when extension is DROP OWNED * [625e3fa] Fix negative value handling in int time_bucket * [a33224b] Add loader API version function * [18b8068] Remove unnecessary index on dimension metadata table * [d09405d] Fix adaptive chunking when hypertables have multiple dimensions * [a81dc18] Block signals when writing to the log table in tests * [d5a6392] Fix adaptive chunking so it chooses correct index * [3489cca] Fix sigterm handling in background jobs * [2369ae9] Remove !WIN32 for sys/time.h and sys/socket.h, pg provides fills * [ebbb4ae] Also add sys/time.h for NetBSD. Fixes #700 * [1a9ae17] Fix build on FreeBSD wrt sockets * [8225cd2] Remove (redefined) macro PG_VERSION and replace with PACKAGE_VERSION * [2a07cf9] Release SpinLock even when we're about to Error due to over-decrementing * [b2a15b8] Make sure DB schedulers are not decremented if they were never incremented * [6731c86] Add support for pre-release version checks **Thanks** * @did-g for an improvement to our macros to make compiliers happy * @mx781 and @HeikoOnnebrink for reporting issues with working with pg_dump fully * @znbang and @timolson for reporting a bug that was causing telemetry to fail * @alanhamlett for reporting an issue with spinlocks when handling SIGTERMs * @oldgreen for reporting an issue with building on NetBSD * @kev009 for fixing build issues on FreeBSD and NetBSD * All the others who have helped us test and used these RCs! ## 0.12.1 (2018-09-19) **High-level changes** * Fixes for a few issues related to the new scheduler and background worker framework. * Fixed bug in adaptive chunking where the incorrect index could be used for determining the current interval. * Improved testing, code cleanup, and other housekeeping. **Notable commits** * [0f6f7fc] Fix adaptive chunking so it chooses correct index * [3ed79ed] Fix sigterm handling in background jobs * [bea098f] Remove !WIN32 for sys/time.h and sys/socket.h, pg provides fills * [9f62a1a] Also add sys/time.h for NetBSD. Fixes #700 * [95a982f] Fix build on FreeBSD wrt sockets * [fcb4a79] Remove (redefined) macro PG_VERSION and replace with PACKAGE_VERSION * [2634897] Release SpinLock even when we're about to Error due to over-decrementing * [1f30dbb] Make sure DB schedulers are not decremented if they were never incremented * [f518cd0] Add support for pre-release version checks * [acebaea] Don't start schedulers for template databases. * [f221a12] Fix use-after-free bug in telemetry test * [0dc5bbb] Use pg_config bindir directory for pg executables **Thanks** * @did-g for reporting a use-after-free bug in a test and for improving the robustness of another test * @kev009 for fixing build issues on FreeBSD and NetBSD ## 1.0.0-rc1 (2018-09-12) This release is our 1.0 release candidate. We expect to only merge bug fixes between now and our final 1.0 release. This is a big milestone for us and signifies our maturity and enterprise readiness. **PLEASE NOTE** that release candidate (rc) builds will only be made available via GitHub and Docker, and _not_ on other release channels. Please help us test these release candidates out if you can! **Notable commits** * [acebaea] Don't start schedulers for template databases. * [f221a12] Fix use-after-free bug in telemetry test * [2092b2a] Fix unused variable warning in Release build * [0dc5bbb] Use pg_config bindir directory for pg executables **Thanks** * @did-g for reporting a use-after-free bug in a test and for improving the robustness of another test ## 0.12.0 (2018-09-10) **High-level changes** *Scheduler framework:* This release introduces a background job framework and scheduler. Each database running within a PostgreSQL instance has a scheduler that schedules recurring jobs from a new jobs table while maintaining statistics that inform the scheduler's policy. Future releases will leverage this scheduler framework for more automated management of data retention, archiving, analytics, and the like. *Telemetry:* Using this new scheduler framework, TimescaleDB databases now send anonymized usage information to a telemetry server via HTTPS, as well as perform version checking to notify users if a newer version is available. For transparency, a new `get_telemetry_report` function details the exact JSON that is sent, and users may also opt out of this telemetry and version check. *Continued hardening:* This release addresses several issues around more robust backup and recovery, handling large numbers of chunks, and additional test coverage. **Notable commits** * [efab2aa] Fix net lib functionality on Windows and improve error handling * [71589c4] Fix issues when OpenSSL is not available * [a43cd04] Call the main telemetry function inside BGW executor * [faf481b] Add telemetry functionality * [45a2b76] Add new Connection and HTTP libraries * [b6fe657] Fix max_background_workers guc, errors on EXEC_BACKEND and formatting * [5d8c7cc] Add a scheduler for background jobs * [55a7141] Implement a cluster-wide launcher for background workers * [5bc705f] Update bootstrap to check for cmake and exit if not found * [98e56dd] Improve show_indexes test func to be more platform agnostic * [b928caa] Note how to recreate templated files * [8571e41] Use AttrNumberGetAttrOffset instead of Anum_name - 1 for array indexing * [d1710ef] Improve regression test script to cleanup more thoroughly * [fc3677f] Reduce number of open chunks per insert * [027b7b2] Hide extension symbols by default on Unix platforms * [6a3abe5] Fix SubspaceStore to ensure max_open_chunks_per_insert is obeyed **Thanks** @EvanCarroll for updates to the bootstrap script to check for cmake ## 0.11.0 (2018-08-08) **High-level changes** * **Adaptive chunking**: This feature, currently in beta, allows the database to automatically adapt a chunk's time interval, so that users do not need to manually set (and possibly manually change) this interval size. In this release, users can specify either a target chunk data size (in terms of MB or GB), and the chunk's time intervals will be automatically adapted. Alternatively, users can ask the database to just estimate a target size itself based on the platform's available memory and other parameters, and the system will adapt accordingly. This type of automation can simplify initial database test and operations. This feature is default off. Note: The default time interval for non-adaptive chunking has also been changed from 1 month to 1 week. * **Continued hardening**: This release addresses a number of less frequently used schema modifications, functions, or constraints. Unsupported functions are safely blocked, while we have added support for a number of new types of table alterations. This release also adds additional test coverage. * Support for additional types of time columns, if they are binary compatible (thanks @fvannee!). **Notable commits** * [9ba2e81] Fix segfault with custom partition types * [7e9bf25] Change default chunk size to one week * [506fa18] Add tests for custom types * [1d9ade7] add support for other types as timescale column * [570f2f8] Validate parameters when creating partition info * [148f2da] Use shared_buffers as the available cache memory * [e0a15c1] Add additional comments to explain algorithm * [d81dccb] Set the default chunk_time_interval to 1 day with adaptive chunking enabled * [2e7b32c] Add WARNING when doing min-max heap scan for adaptive chunking * [6b452a8] Update adaptive chunk algorithm to handle very small chunks. * [9c9cdca] Add support for adaptive chunk sizing * [7f8d17d] Handle DEFERRED and VALID options for constraints * [0c5c21b] Block using rules with hypertables * [37142e9] Block INSERTs on a hypertable's root table * [4daf087] Fix some ALTER TABLE corner case bugs on hypertables * [122f5f1] Block replica identity usage with hypertables * [8bf552e] Block unlogged tables from being used as hypertables * [a8c637e] Create aggregate functions only once to avoid dependency issues * [a97f2af] Add support for custom hypertable dimension types * [dfe026c] Refactor create_hypertable rel access. * [ed379c3] Validate existing indexes before adding a new dimension * [1f2d276] Fix and improve show_indexes test support function * [77b0035] Enforce IMMUTABLE partitioning functions * [cbc5e60] Block NO INHERIT constraints on hypertables * [e362e9c] Block mixing hypertables with postgres inheritance * [011f12b] Add support for CLUSTER ON and SET WITHOUT CLUSTER * [e947c6b] Improve handling of column settings * [fc4957b] Update statistics on parent table when doing ANALYZE * [82942bf] Enable backwards compatibility for loader for 0.9.0 and 0.9.1 **Thanks** * @Ngalstyan4 and @hjsuh18, our interns, for all of the PRs this summer * @fvannee for a PR adding support for binary compatible custom types as a time column * @fmacelw for reporting a bug where first() and last() hold reference across extension update * @soccerdroid for reporting a corner case bug in ALTER TABLE ## 0.10.1 (2018-07-12) **High-level changes** * Improved memory management for long-lived connections. * Fixed handling of dropping triggers that would lead to orphaned references in pg_depend. * Fixed pruning in CustomScan when the subplan is not a Scan type that caused a crash with LATERALs. * Corrected size reporting that was not accurately counting TOAST size * Updated error messages that more closely conform to PG style. * Corrected handling of table and schema name changes to chunks; TimescaleDB metadata catalogs are now properly updated **Notable commits** * [8b58500] Fix bug where dropping triggers caused dangling references in pg_depend, disallow disabling triggers on hypertables * [745b8ab] Fixing CustomScan pruning whenever the subplan is NOT of a Scan type. * [67a8a41] Make chunk identifiers formatting safe using format * [41af6ff] Fix misreported toast_size in chunk_relation_size funcs * [4f2f1a6] Update the error messages to conform with the style guide; Fix tests * [3c28f65] Release cache pin memory * [abe76fc] Add support for changing chunk schema and name **Thanks** * @mfuterko for updating our error messages to conform with PG error message style * @fvannee for reporting a crash when using certain LATERAL joins with aggregates * @linba708 for reporting a memory leak with long lived connections * @phlsmk for reporting an issue where dropping triggers prevented drop_chunks from working due to orphaned dependencies ## 0.10.0 (2018-06-27) **High-level changes** * Planning time improvement (**up to 15x**) when a hypertable has many chunks by only expanding (and taking locks on) chunks that will actually be used in a query, rather than on all chunks (as was the default PostgreSQL behavior). * Smarter use of HashAggregate by teaching the planner to better estimate the number of output rows when using time-based grouping. * New convenience function for getting the approximate number of rows in a hypertable (`hypertable_approximate_row_count`). * Fixed support for installing extension into non-`public` schemas * Other bug fixes and refactorings. **Notable commits** * [12bc117] Fix static analyzer warning when checking for index attributes * [7d9f49b] Fix missing NULL check when creating default indexes * [2e1f3b9] Improve memory allocation during cache lookups * [ca6e5ef] Fix upserts on altered tables. * [2de6b02] Add optimization to use HashAggregate more often * [4b4211f] Fix some external functions when setting a custom schema * [b7257fc] Optimize planning times when hypertables have many chunks * [c660fcd] Add hypertable_approximate_row_count convenience function * [9ce1576] Fix a compilation issue on pre 9.6.3 versions **Thanks** * @viragkothari for suggesting the addition of `hypertable_approximate_row_count` and @fvannee for providing the initial SQL used to build that function * 'mintekhab' from Slack for reporting a segfault when using upserts on an altered table * @mmouterde for reporting an issue where the extension implicitly expected to be installed in the `public` schema * @mfuterko for bringing some potential bugs to our attention via static analysis ## 0.9.2 (2018-05-04) **High-level changes** * Fixed handling of `DISCARD ALL` command when parallel workers are involved, which sometimes caused the extension to complain it was not preloaded * User permission bug fix where users locating TRIGGER permissions in a database could not insert data into a hypertable * Fixes for some issues with 32-bit architectures **Notable commits** * [256b394] Fix parsing of GrantRoleStmt * [b78953b] Fix datum conversion typo * [c7283ef] Fix bug with extension loader when DISCARD ALL is executed * [fe20e48] Fix chunk creation with user that lacks TRIGGER permission **Thanks** * @gumshoes, @manigandham, @wallies, & @cjrh for reporting a problem where sometimes the extension would appear to not be preloaded when it actually was * @thaxy for reporting a permissions issue when user creating a hypertable lacks TRIGGER permission * @bertmelis for reporting some bugs with 32-bit architectures ## 0.9.1 (2018-03-26) **High-level changes** * **For this release only**, you will need to restart the database before running `ALTER EXTENSION` * Several edge cases regarding CTEs addressed * Updated preloader with better error messaging and fixed edge case * ABI compatibility with latest PostgreSQL to help catch any breaking changes **Notable commits** * [40ce037] Fix crash on explain analyze with insert cte * [8378beb] Enable hypertable inserts within CTEs * [bdfda75] Fix double-loading of extension * [01ea77e] Fix EXPLAIN output for ConstraintAwareAppend inside CTE * [fc05637] Add no preload error to versioned library. * [38f8e0c] Add ABI compatibility tests * [744ca09] Fix Cache Pinning for Subtxns * [39010db] Move more drops into event trigger system * [fc36699] Do not fail add_dimension() on non-empty table with 'if_not_exists' **Thanks** * @The-Alchemist for pointing out broken links in the README * @chaintng for pointing out a broken link in the docs * @jgranstrom for reporting a edge case crash with UPSERTs in CTEs * @saosebastiao for reporting the lack of an error message when the library is not preloaded and trying to delete/modify a hypertable * @jbylund for reporting a cache invalidation issue with the preloader ## 0.9.0 (2018-03-05) **High-level changes** * Support for multiple extension versions on different databases in the same PostgreSQL instance. This allows different databases to be updated independently and provides for smoother updates between versions. No more spurious errors in the log as the extension is being updated, and new versions no longer require a restart of the database. * Streamlined update process for smaller binary/package sizes * Significant refactoring to simplify and improve codebase, including improvements to error handling, security/permissions, and more * Corrections to edge-case scenarios involving dropping schemas, hypertables, dimensions, and more * Correctness improvements through propagating reloptions from main table to chunk tables and blocking `ONLY` commands that try to alter hypertables (i.e., changes should be applied to chunks as well) * Addition of a `migrate_data` option to `create_hypertable` to allow non-empty tables to be turned into hypertables without separate creation & insertion steps. Note, this option may take a while if the original table has lots of data * Support for `ALTER TABLE RENAME CONSTRAINT` * Support for adjusting the number of partitions for a space dimension * Improvements to tablespace handling **Notable commits** * [4672719] Fix error in handling of RESET ALL * [9399308] Refactor/simplify update scripts and build process * [0e79df4] Fix handling of custom SQL-based partitioning functions * [f13969e] Fix possible memory safety issue and squash valgrind error. * [ef74491] Migrate table data when creating a hypertable * [2696582] Move index and constraints drop handling to event trigger * [d6baccb] Improve tablespace handling, including blocks for DROP and REVOKE * [b9a6f89] Handle DROP SCHEMA for hypertable and chunk schemas * [b534a5a] Add test case for adding metadata entries automatically * [6adce4c] Handle TRUNCATE without upcall and handle ONLY modifier * [71b1124] Delete orphaned dimension slices * [fa19a54] Handle deletes on metadata objects via native catalog API * [6e011d1] Refactor hypertable-related API functions * [5afd39a] Fix locking for serializing chunk creation * [6dd2c46] Add check for null in ca_append_rescan to prevent segfault * [71962b8] Refactor dimension-related API functions * [cc254a9] Fix CREATE EXTENSION IF NOT EXISTS and error messages * [d135256] Spread chunk indexes across tablespaces like chunks * [e85721a] Block ONLY hypertable on all ALTER TABLE commands. * [78d36b5] Handle subtxn for cache pinning * [26ef77f] Add subtxn abort logic to process_utility.c * [25f3284] Handle cache invalidation during subtxn rollback * [264956f] Block DROP NOT NULL on time-partitioned columns. * [ad7d361] Better accounting for number of items stored in a subspace * [12f92ea] Improve speed of out-of-order inserts * [87f055d] Add support for ALTER TABLE RENAME CONSTRAINT. * [da8cc79] Add support for multiple extension version in one pg instance * [68faddc] Make chunks inherit reloptions set on the hypertable * [4df8f28] Add proper permissions handling for associated (chunk) schemas * [21efcce] Refactor chunk table creation and unify constraint handling **Thanks** * @Anthares for a request to pass reloptions like fill factor to child chunks * @oldgreen for reporting an issue with subtransaction handling * @fvannee for a PR that fixed a bug with `ca_append_rescan` * @maksm90 for reporting an superfluous index being created in an internal catalog table * @Rashid987 for reporting an issue where deleting a chunk, then changing the time interval would not apply the change when a replacement chunk is created * RaedA from Slack for reporting compilation issues on Windows between 0.8.0 and this release * @haohello for a request to adjust the number of partitions for a given dimension * @LonghronShen and @devereaux for reporting an issue (and submitting a PR) for handling version identification when there is more to the version than just major and minor numbers * @carlospeon for reporting an issue with dropping hypertables * @gumshoes, @simpod, @jbylund, and @ryan-shaw for testing a pre-release version to verify our new update path works as expected * @gumshoes for reporting an issue with `RESET ALL` ## 0.8.0 (2017-12-19) **High-level changes** * TimescaleDB now builds and runs on Windows! Now in addition to using Docker, users can choose to build the extension from source and install on 64-bit Windows * Update functions `add_dimension` and `set_chunk_time_interval` to take `INTERVAL` types * Improved tablespace management including detaching tablespaces from hypertables and looking up tablespaces associated with a hypertable * Reduced memory usage for `INSERT`s with out-of-order data * Fixes inserts on 32-bit architectures, in particular ARM * Other correctness improvements including preventing attachment of PG10 partitions to hypertables, improved handling of space dimensions with one partition, and correctly working with `pg_upgrade` * Test and build improvements making those both more robust and easier to do **Notable commits** * [26971d2] Make `tablespace_show` function return Name instead of CString * [2fe447b] Make TimescaleDB work with pg_upgrade * [90c7a6f] Fix logic for one space partition * [6cfdd79] Prevent native partitioning attachment of hypertables * [438d79d] Fix trigger relcache handling for COPY * [cc1ad95] Reduce memory usage for out-of-order inserts * [a0f62c5] Improve bootstrap script's robustness * [00a096f] Modify tests to make more platform agnostic * [0e76b5f] Do not add tablespaces to hypertable objects * [176b75e] Add command to show tablespaces attached to a hypertable * [6e92383] Add function to detach tablespaces from hypertables * [e593876] Refactor tablespace handling * [c4a46ac] Add hypertable cache lookup on ID/pkey * [f38a578] Fix handling of long constraint names * [20c9b28] Unconditionally add pg_config --includedir to src build * [12dff61] Fixes insert for 32bit architecture * [e44e47e] Update add_dimension to take INTERVAL times * [0763e62] Update set_chunk_time_interval to take INTERVAL times * [87c4b4f] Fix test generator to work for PG 10.1 * [51854ac] Fix error message to reflect that drop_chunks can take a DATE interval * [66396fb] Add build support for Windows * [e1a0e81] Refactor and fix cache invalidation **Thanks** * @oldgreen for reporting an issue where `COPY` was warning of relcache reference leaks * @campeterson for pointing out some documentation typos * @jwdeitch for the PR to prevent attaching PG10 partitions to hypertables * @vjpr and @sztanpet for reporting bugs and suggesting improvements to the bootstrap script ## 0.7.1 (2017-11-29) **High-level changes** * Fix to the migration script for those coming from 0.6.1 (or earlier) * Fix edge case in `drop_chunks` when hypertable uses `TIMESTAMP` type * Query planning improvements & fixes * Permission fixes and support `SET ROLE` functionality **Notable commits** * [717299f] Change time handling in drop_chunks for TIMESTAMP times * [d8ec285] Do not append-optimize plans with result relations (DELETE/UPDATE) * [30b72ec] Handle empty append plans in ConstraintAwareAppend * [b35509b] Permission fixes and allow SET ROLE **Thanks** * @shaneodonnell for reporting a bug with empty append plans in ConstraintAwareAppend * @ryan-shaw for reporting a bug with query plans involving result relations and reporting an issue with our 0.6.1 to 0.7.0 migration script ## 0.7.0 (2017-11-21) **Please note: This update may take a long time (minutes, even hours) to complete, depending on the size of your database** **High-level changes** * **Initial PostgreSQL 10 support**. TimescaleDB now should work on both PostgreSQL 9.6 and 10. As this is our first release supporting PG10, we look forward to community feedback and testing. _Some release channels, like Ubuntu & RPM-based distros will remain on 9.6 for now_ * Support for `CLUSTER` on hypertables to recursively apply to chunks * Improve constraint handling of edge cases for `DATE` and `TIMESTAMP` * Fix `range_start` and `range_end` to properly handle the full 32-bit int space * Allow users to specify their desired partitioning function * Enforce `NOT NULL` constraint on time columns * Add testing infrastructure to use Coverity and test PostgreSQL regression tests in TimescaleDB * Switch to the CMake build system for better cross-platform support * Several other bug fixes, cleanups, and improvements **Notable commits** * [13e1cb5] Add reindex function * [6594018] Handle when create_hypertable is invoked on partitioned table * [818bdbc] Add coverity testing * [5d0cbc1] Recurse CLUSTER command to chunks * [9c7191e] Change TIMESTAMP partitioning to be completely tz-independent * [741b256] Mark IMMUTABLE functions as PARALLEL SAFE * [2ffb30d] Make aggregate serialize and deserialize functions STRICT * [c552410] Add build target to run the standard PostgreSQL regression tests * [291050b] Change DATE partitioning to be completely tz-independent * [ca0968a] Make all partitioning functions take anyelement argument * [a4e1e32] Change range_start and range_end semantics * [2dfbc82] Fix off-by-one error on range-end * [500563f] Add support for PostgreSQL 10 * [201a948] Check that time dimensions are set as NOT NULL. * [4532650] Allow setting partitioning function * [4a0a0d8] Fix column type change on plain tables * [cf009cc] Avoid string conversion in hash partitioning * [8151098] Improve update testing by adding a rerun test * [c420c11] Create a catalog entry for constraint-backed indexes * [ec746d1] Add ability to run regression test locally * [44f9fec] Add analyze to parallel test for stability * [9e0422a] Fix bug with pointer assignment after realloc * [114fa8d] Refactor functions used to recurse DDL commands to chunks * [b1ec4fa] Refactor build system to use CMake **Thanks** * @jgraichen for reporting an issue with `drop_chunks` not accepting `BIGINT` * @nathansgreen for reporting an edge case with constraints for `TIMESTAMP` * @jonmd for reporting a similar edge case for `DATE` * @jwdeitch for a PR to cover an error case in PG10 ## 0.6.1 (2017-11-07) **High-level changes** * Fix several memory bugs that caused segfaults * Fix bug when creating expression indexes * Plug a memory leak with constraint expressions * Several other bug fixes and stability improvements **Notable commits** * [2799075] Fix EXPLAIN for ConstraintAware and MergeAppend * [8084594] Use per-chunk memory context for cached chunks * [a13d9de] Do not convert tuples on insert unless needed * [da09f24] Limit growth of range table during chunk inserts * [85dee79] Fix issue with creating expression indexes * [844ff7f] Fix memory leak due to constraint expressions. * [e90d3ee] Consider precvious CIS state in copy FROM file to rel * [56d632f] Fix bug with pointer assignment after realloc * [f97d624] Make event trigger creation idempotent **Thanks** * @jwdeitch for submitting a patch to correct behavior in the COPY operation * @jgraichen for reporting a bug with expression indexes * @zixet for reporting a memory leak * @djk447 for reporting a bug in EXPLAIN with ConstraintAware and MergeAppend ## 0.6.0 (2017-10-12) **High-level changes** * Fix bugs where hypertable-specific handlers were affecting normal Postgres tables. * Make it so that all TimescaleDB commands can run as a normal user rather than a superuser. * Updates to the code to make the extension compileable on Windows; future changes will add steps to properly build. * Move `time_bucket` functions out of `public` schema (put in schema where extension is). * Several other bugs fixes. **Notable commits** * [1d73fb8] Fix bug with extension starting too early. * [fd390ec] Fix chunk index attribute mismatch and locking issue * [430ed8a] Fix bug with collected commands in index statement. * [614c2b7] Fix permissions bugs and run tests as normal user * [ce12104] Fix "ON CONFLICT ON CONSTRAINT" on plain PostgreSQL tables * [4c451e0] Fix rename and reindex bugs when objects are not relations * [c3ebc67] Fix permission problems with dropping hypertables and chunks * [040e815] Remove truncate and hypertable metadata triggers * [5c26328] Fix INSERT on hypertables using sub-selects with aggregates * [b57e2bf] Prepare C code for compiling on Windows * [a2bad2b] Fix constraint validation on regular tables * [fb5717f] Remove explicit schema for time_bucket * [04d01ce] Split DDL processing into start and end hooks **Thanks** * @oldgreen for reporting `time_bucket` being incorrectly put in the `public` schema and pointing out permission problems * @qlandman for reporting a bug with INSERT using sub-selects with aggregates * @min-mwei for reporting a deadlock issue during INSERTs * @ryan-shaw for reporting a bug where the extension sometimes used `pg_cache` too soon ## 0.5.0 (2017-09-20) **High-level changes** * Improved support for primary-key, foreign-key, unique, and exclusion constraints. * New histogram function added for getting the frequency of a column's values. * Add support for using `DATE` as partition column. * `chunk_time_interval` now supports `INTERVAL` data types * Block several unsupported and/or dangerous operations on hypertables and chunks, including dropping or otherwise altering a chunk directly. * Several bug fixes throughout the code. **Notable commits** * [afcb0b1] Fix NULL handling in first/last functions. * [d53c705] Add script to dump meta data that can be useful for debugging. * [aa904fa] Block adding constraints without a constraint name * [a13039f] Fix dump and restore for tables with triggers and constraints * [8cf8d3c] Improve the size utils functions. * [2767548] Block adding constraints using an existing index * [5cee104] Allow chunk_time_interval to be specified as an INTERVAL type * [6232f98] Add histogram function. * [2380033] Block ALTER TABLE and handle DROP TABLE on chunks * [72d6681] Move security checks for ALTER TABLE ALTER COLUMN to C * [19d3d89] Handle changing the type of dimension columns correctly. * [17c4ba9] Handle ALTER TABLE rename column * [66932cf] Forbid relocating extension after install. * [d2561cc] Add ability to partition by a date type * [48e0a61] Remove triggers from chunk and chunk_constraint * [4dcbe61] Add support for hypertable constraints **Thanks** * @raycheung for reporting a segfault in `first`/`last` * @meotimdihia, @noyez, and @andrew-blake for reporting issues with `UNQIUE` and other types of constraints ## 0.4.2 (2017-09-06) **High-level changes** * Provide scripts for backing up and restoring single hypertables **Notable commits** * [683c078] Add backup/restore scripts for single hypertables ## 0.4.1 (2017-09-04) **High-level changes** * Bug fix for a segmentation fault in the planner * Shortcut when constraint-aware append excludes all chunks * Fix edge case with negative timestamps when points fell right on the boundary * Fix behavior of `time_bucket` for `DATE` types by not converting to `TIMESTAMPTZ` * Make the output of `chunk_relation_size` consistent **Notable commits** * [50c8c4c] Fix possible segfault in planner * [e49e45c] Fix failure when constraint-aware append excludes all chunks * [c3b6fb9] Fix bug with negative dimension values * [3c69e4f] Fix semantics of time_bucket on DATE input * [0137c92] Fix output order of chunk dimensions and ranges in chunk_relation_size. * [645b530] Convert inserted tuples to the chunk's rowtype **Thanks** * @yadid for reporting a segfault (fixed in 50c8c4c) * @ryan-shaw for reporting tuples not being correctly converted to a chunk's rowtype (fixed in 645b530) * @yuezhihan for reporting GROUP BY error when setting compress_segmentby with an enum column ## 0.4.0 (2017-08-21) **High-level changes** * Exclude chunks when constraints can be constifyed even if they are considered mutable like `NOW()`. * Support for negative values in the dimension range which allows for pre-1970 dates. * Improve handling of default chunk times for integral date times by forcing it to be explicit rather than guessing the units of the time. * Improve memory usage for long-running `COPY` operations (previously it would grow unbounded). * `VACUUM` and `REINDEX` on hypertables now recurse down to chunks. **Notable commits** * [139fe34] Implement constraint-aware appends to exclude chunks at execution time * [2a51cf0] Add support for negative values in dimension range * [f2d5c3f] Error if add_dimension given both partition number and interval length * [f3df02d] Improve handling of non-TIMESTAMP/TZ timestamps * [6a5a7eb] Reduce memory usage on long-running COPY operations * [953346c] Make VACUUM and REINDEX recurse to chunks * [55bfdf7] Release all cache pins when a transaction ends ## 0.3.0 (2017-07-31) **High-level changes** * "Upserts" are now supported via normal `ON CONFLICT DO UPDATE`/`ON CONFLICT DO NOTHING` syntax. However, `ON CONFLICT ON CONSTRAINT` is not yet supported. * Improved support for user-defined triggers on hypertables. Now handles both INSERT BEFORE and INSERT AFTER triggers, and triggers can be named arbitrarily (before, a \_0\_ prefix was required to ensure correct execution priority). * `TRUNCATE` on a hypertable now deletes empty chunks. **Notable commits** * [23f9d3c] Add support for upserts (`ON CONFLICT DO UPDATE`) * [1f3dcd8] Make `INSERT`s use a custom plan instead of triggers * [f23bf58] Remove empty chunks on `TRUNCATE` hypertable. ## 0.2.0 (2017-07-12) **High-level changes** * Users can now define their own triggers on hypertables (except for `INSERT AFTER`) * Hypertables can now be renamed or moved to a different schema * Utility functions added so you can examine the size hypertables, chunks, and indices **Notable commits** * [83c75fd] Add support for triggers on hypertables for all triggers except `INSERT AFTER` * [e0eeeb9] Add hypertable, chunk, and indexes size utils functions. * [4d2a65d] Add infrastructure to build update script files. * [a5725d9] Add support to rename and change schema on hypertable. * [142f58c] Cleanup planner and process utility hooks ## 0.1.0 (2017-06-28) **IMPORTANT NOTE** Starting with this release, TimescaleDB will now support upgrading between extension versions using the typical `ALTER EXTENSION` command, unless otherwise noted in future release notes. This important step should make it easier to test TimescaleDB and be able to get the latest benefits from new versions of TimescaleDB. If you were previously using a version with the `-beta` tag, you will need to `DROP` any databases currently using TimescaleDB and re-create them in order to upgrade to this new version. To backup and migrate data, use `pg_dump` to save the table schemas and `COPY` to write hypertable data to CSV for re-importing after upgrading is complete. We describe a similar process on [our docs](http://docs.timescale.com/getting-started/setup/migrate-from-postgresql#different-db). **High-level changes** * More refactoring to stabilize and cleanup the code base for supporting upgrades (see above note) * Correct handling of ownership and permission propagation for hypertables * Multiple bug fixes **Notable commits** * [696cc4c] Provide API for adding hypertable dimensions * [97681c2] Fixes permission handling * [aca7f32] Fix extension drop handling * [9b8a447] Limit the SubspaceStore size; Add documentation. * [14ac892] Fix possible segfault * [0f4169c] Fix check constraint on dimension table * [71c5e78] Fix and refactor tablespace support * [5452dc5] Fix partiton functions; bug fixes (including memory) * [e75cd7e] Finer grained memory management * [3c460f0] Fix partitioning, memory, and tests * [fe51d8d] Add native scan for the chunk table * [fc68baa] Separate out subspace_store and add it to the hypertable object as well * [c8124b8] Use hypercube instead of dimension slice list * [f5d7786] Change the semantics of range_end to be exclusive * [700c9c8] Refactor insert path in C. * [0584c47] Created chunk_get_or_create in sql with an SPI connector in C * [7b8de0c] Refactor catalog for new schema and add native data types * [d3bdcba] Start refactoring to support any number of partitioning dimensions ## 0.0.12-beta (2017-06-21) **High-level changes** * A major cleanup and refactoring was done to remove legacy code and currently unused code paths. This change is **backwards incompatible** and will require a database to be re-initialized and data re-imported. This refactoring will allow us to provide upgrade paths starting with the next release. * `COPY` and `INSERT` commands now return the correct number of rows * Default indexes no longer duplicate existing indexes * Cleanup of the Docker image and build process * Chunks are now time-aligned across partitions **Notable commits** * [3192c8a] Remove Dockerfile and docker.mk * [2a01ebc] Ensure that chunks are aligned. * [73622bf] Fix default index creation duplication of indexes * [c8872fe] Fix command-tag for COPY and INSERT * [bfe58b6] Refactor towards supporting version upgrades * [db01c84] Make time-bucket function parallel safe * [18db11c] Fix timestamp test * [97bbb59] Make constraint exclusion work with non-text partition keys * [f2b42eb] Fix problems with partitioning logic for padded fields * [997029a] if_not_exist flag to create_hypertable now works on hypertables with data as well * [347a8bd] Reference the correct column when scanning partition epochs * [88a9849] Fix bug with timescaledb.allow_install_without_preload GUC not working ## 0.0.11-beta (2017-05-24) **High-level changes** * New `first(value, time)` and `last(value, time)` aggregates * Remove `setup_timescaledb()` function to streamline setup * Allow for use cases where restarting the server is not feasible by force loading the library * Disable time series optimizations on non-hypertables * Add some default indexes for hypertables if they do not exist * Add "if not exists" flag for `create_hypertable` * Several bug fixes and cleanups **Notable commits** * [8ccc8cc] Add if_not_exists flag to create_hypertable() * [2bc60c7] Fix time interval field name in hypertable cache entry * [4638688] Improve GUC handling * [cedcafc] Remove setup_timescaledb() and fix pg_dump/pg_restore. * [34ad9a0] Add error when timescaledb library is not preloaded. * [fc4ddd6] Fix bug with dropping chunks on tables with indexes * [32215ff] Add default indexes for hypertables * [b2900f9] Disable query optimization on regular tables (non-hypertables) * [f227db4] Fixes command tag return for COPY on hypertables. * [eb32081] Fix Invalid database ID error * [662be94] Add the first(value, time),last(value, time) aggregates * [384a8fb] Add regression tests for deleted unit tests * [31ee92a] Remove unit tests and sql/setup * [13d3acb] Fix bug with alter table add/drop column if exists * [f960c24] Fix bug with querying a row as a composite type ## 0.0.10-beta (2017-05-04) **High-level changes** * New `time_bucket` functions for doing roll-ups on varied intervals * Change default partition function (thanks @robin900) * Variety of bug fixes **Notable commits** * [1c4868d] Add documentation for chunk_time_interval argument * [55fd2f2] Fixes command tag return for `INSERT`s on hypertables. * [c3f930f] Add `time_bucket` functions * [b128ac2] Fix bug with `INSERT INTO...SELECT` * [e20edf8] Add better error checking for index creation. * [72f754a] use PostgreSQL's own `hash_any` function as default partfunc (thanks @robin900) * [39f4c0f] Remove sample data instructions and point to docs site * [9015314] Revised the `get_general_index_definition` function to handle cases where indexes have definitions other than just `CREATE INDEX` (thanks @bricklen) ================================================ FILE: CMakeLists.txt ================================================ cmake_minimum_required(VERSION 3.15) list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) include(CheckCCompilerFlag) include(CheckSymbolExists) include(GitCommands) include(GenerateScripts) include(CMakeDependentOption) option(APACHE_ONLY "only compile apache code" off) # This requires all tests to run. This defaults to OFF but can be enabled to # ensure that no tests are skipped because of missing tools. option(REQUIRE_ALL_TESTS "Require all tests to run." OFF) option(USE_OPENSSL "Enable use of OpenSSL if available" ON) option(SEND_TELEMETRY_DEFAULT "The default value for whether to send telemetry" ON) option( USE_TELEMETRY "Include telemetry functionality in the build. Disabling will exclude all telemetry code from the build." ON) option(REGRESS_CHECKS "PostgreSQL regress checks through installcheck" ON) option( ENABLE_OPTIMIZER_DEBUG "Enable OPTIMIZER_DEBUG when building. Requires Postgres server to be built with OPTIMIZER_DEBUG." OFF) option(ENABLE_DEBUG_UTILS "Enable debug utilities for the extension." ON) # Option to enable assertions. Note that if we include headers from a PostgreSQL # build that has assertions enabled, we might inherit that setting without # explicitly enabling assertions via the ASSERTIONS option defined here. Thus, # this option is mostly useful to enable assertions when the PostgreSQL we # compile against has it disabled. option(ASSERTIONS "Compile with assertion checks (default OFF)" OFF) # Function to call pg_config and extract values. function(GET_PG_CONFIG var) set(_temp) # Only call pg_config if the variable didn't already have a value. if(NOT ${var}) execute_process( COMMAND ${CMAKE_COMMAND} -E env LC_MESSAGES=C ${PG_CONFIG} ${ARGN} OUTPUT_VARIABLE _temp OUTPUT_STRIP_TRAILING_WHITESPACE) endif() # On Windows, fields that are not recorded will be given the value "not # recorded", so we translate this into <var>-NOTFOUND to make it undefined. # # It will then also show as, e.g., "PG_LDFLAGS-NOTFOUND" in any string # interpolation, making it obvious that it is an undefined CMake variable. if("${_temp}" STREQUAL "not recorded") set(_temp ${var}-NOTFOUND) endif() set(${var} ${_temp} PARENT_SCOPE) endfunction() # Search paths for Postgres binaries if(WIN32) find_path( PG_PATH postgres.exe PATHS "C:/PostgreSQL" "C:/Program Files/PostgreSQL" PATH_SUFFIXES bin 15/bin 16/bin DOC "The path to a PostgreSQL installation") elseif(UNIX) find_path( PG_PATH postgres PATHS $ENV{HOME} /opt/local/pgsql /usr/local/pgsql /usr/lib/postgresql PATH_SUFFIXES bin 15/bin 16/bin DOC "The path to a PostgreSQL installation") endif() find_program( PG_CONFIG pg_config HINTS ${PG_PATH} PATH_SUFFIXES bin DOC "The path to the pg_config of the PostgreSQL version to compile against") if(NOT PG_CONFIG) message(FATAL_ERROR "Unable to find 'pg_config'") endif() configure_file("version.config" "version.config" COPYONLY) file(READ version.config VERSION_CONFIG) if(VERSION_CONFIG MATCHES "(^|.*[^a-z])version[\t ]*=[\t ]*([0-9]+\\.[0-9]+\\.*[0-9]*)(-([a-z]+[0-9]*|dev))?.*" ) set(VERSION ${CMAKE_MATCH_2}) set(VERSION_MOD ${CMAKE_MATCH_4}) # This is used in config.h if(CMAKE_MATCH_3) set(PROJECT_VERSION_MOD ${CMAKE_MATCH_2}${CMAKE_MATCH_3}) else() set(PROJECT_VERSION_MOD ${CMAKE_MATCH_2}) endif() endif() if(VERSION_CONFIG MATCHES ".*previous_version[\t ]*=[\t ]*([0-9]+\\.[0-9]+\\.[0-9]+(-[a-z]+[0-9]*)?).*" ) set(PREVIOUS_VERSION ${CMAKE_MATCH_1}) else() message( FATAL_ERROR "Could not determine previous version from version.config") endif() # a hack to avoid change of SQL extschema variable set(extschema "@extschema@") # If not explicitly specified, try to use the same compiler as Postgres to avoid # mismatches and simplify configuration. if((NOT DEFINED CMAKE_C_COMPILER) AND (NOT DEFINED ENV{CC})) get_pg_config(PG_CC --cc) # The first word might be ccache or similar, so handle this. string(REPLACE " " ";" PG_CC_LIST ${PG_CC}) list(POP_FRONT PG_CC_LIST PG_CC_FIRST) if(PG_CC_LIST) # Got multi-word PG CC, treat the first word as ccache. find_program(PG_CCACHE_FOUND ${PG_CC_FIRST}) if((NOT DEFINED CMAKE_C_COMPILER_LAUNCHER) AND PG_CCACHE_FOUND) message( STATUS "Using ${PG_CCACHE_FOUND} as C compiler launcher based on pg_config CC" ) set(CMAKE_C_COMPILER_LAUNCHER ${PG_CCACHE_FOUND}) endif() set(PG_CC ${PG_CC_LIST}) else() # Single-word CC. set(PG_CC ${PG_CC_FIRST}) endif() find_program(PG_CC_FOUND ${PG_CC}) if(PG_CC_FOUND) message(STATUS "Using ${PG_CC_FOUND} as CC based on pg_config CC") set(CMAKE_C_COMPILER ${PG_CC_FOUND}) endif() endif() # Set project name, version, and language. Language needs to be set for compiler # checks project( timescaledb VERSION ${VERSION} LANGUAGES C) if(NOT CMAKE_BUILD_TYPE) # Default to Release builds set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel" FORCE) endif() set(SUPPORTED_BUILD_TYPES Debug Release RelWithDebInfo MinSizeRel) if(NOT CMAKE_BUILD_TYPE IN_LIST SUPPORTED_BUILD_TYPES) message( FATAL_ERROR "Bad CMAKE_BUILD_TYPE. Expected one of ${SUPPORTED_BUILD_TYPES}" ) endif() message( STATUS "TimescaleDB version ${PROJECT_VERSION_MOD}. Can be updated from version ${PREVIOUS_VERSION}" ) message(STATUS "Build type is ${CMAKE_BUILD_TYPE}") set(PROJECT_INSTALL_METHOD source CACHE STRING "Specify what install platform this binary is built for") message(STATUS "Install method is '${PROJECT_INSTALL_METHOD}'") # Build compilation database by default set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # Code coverage is optional and OFF by default option(CODECOVERAGE "Enable code coverage for the build" OFF) option(EXPERIMENTAL "Skip postgres version compatibility check" OFF) # Generate downgrade script option(GENERATE_DOWNGRADE_SCRIPT "Generate downgrade script. Defaults to not generate a downgrade script." OFF) if(CMAKE_BUILD_TYPE MATCHES Debug) # CMAKE_BUILD_TYPE is set at CMake configuration type. But usage of # CMAKE_C_FLAGS_DEBUG is determined at build time by running cmake --build . # --config Debug (at least on Windows). Therefore, we only set these flags if # the configuration-time CMAKE_BUILD_TYPE is set to Debug. Then Debug enabled # builds will only happen on Windows if both the configuration- and build-time # settings are Debug. set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -DDEBUG=1 -DTS_DEBUG=1") endif(CMAKE_BUILD_TYPE MATCHES Debug) set(SUPPORTED_COMPILERS "GNU" "Clang" "AppleClang" "MSVC") # Check for a supported compiler if(NOT CMAKE_C_COMPILER_ID IN_LIST SUPPORTED_COMPILERS) message( FATAL_ERROR "Unsupported compiler ${CMAKE_C_COMPILER_ID}. Supported compilers are: ${SUPPORTED_COMPILERS}" ) endif() # Option to treat warnings as errors when compiling (default on for debug # builds, off for all other build types) if(CMAKE_BUILD_TYPE STREQUAL Debug) option(WARNINGS_AS_ERRORS "Make compiler warnings into errors (default ON)" ON) else() option(WARNINGS_AS_ERRORS "Make compiler warnings into errors (default ON)" OFF) endif() if(WARNINGS_AS_ERRORS) if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang|AppleClang") add_compile_options(-Werror) elseif(CMAKE_C_COMPILER_ID MATCHES "MSVC") add_compile_options(/WX) endif() endif(WARNINGS_AS_ERRORS) if(CMAKE_C_COMPILER_ID MATCHES "GNU|AppleClang|Clang") # These two flags generate too many errors currently, but we probably want # these optimizations enabled. # # -fdelete-null-pointer-checks -Wnull-dereference # This flag avoid some subtle bugs related to standard conversions, but # currently does not compile because we are using too many implicit # conversions that potentially lose precision. # # -Wconversions # These flags are supported on all compilers. add_compile_options( -Wempty-body -Wvla -Wall -Wextra # The SQL function arguments macro PG_FUNCTION_ARGS often inroduces unused # arguments. -Wno-unused-parameter -Wundef -Wmissing-prototypes -Wpointer-arith -Werror=vla -Wendif-labels -fno-strict-aliasing -fno-omit-frame-pointer) # These flags are just supported on some of the compilers, so we check them # before adding them. check_c_compiler_flag(-Wno-unused-command-line-argument CC_SUPPORTS_NO_UNUSED_CLI_ARG) if(CC_SUPPORTS_NO_UNUSED_CLI_ARG) add_compile_options(-Wno-unused-command-line-argument) endif() check_c_compiler_flag(-Wno-format-truncation CC_SUPPORTS_NO_FORMAT_TRUNCATION) if(CC_SUPPORTS_NO_FORMAT_TRUNCATION) add_compile_options(-Wno-format-truncation) else() message(STATUS "Compiler does not support -Wno-format-truncation") endif() check_c_compiler_flag(-Wstringop-truncation CC_STRINGOP_TRUNCATION) if(CC_STRINGOP_TRUNCATION) add_compile_options(-Wno-stringop-truncation) else() message(STATUS "Compiler does not support -Wno-stringop-truncation") endif() check_c_compiler_flag(-Wimplicit-fallthrough CC_SUPPORTS_IMPLICIT_FALLTHROUGH) if(CC_SUPPORTS_IMPLICIT_FALLTHROUGH) add_compile_options(-Wimplicit-fallthrough) else() message(STATUS "Compiler does not support -Wimplicit-fallthrough") endif() check_c_compiler_flag(-Wnewline-eof CC_SUPPORTS_NEWLINE_EOF) if(CC_SUPPORTS_NEWLINE_EOF) add_compile_options(-Wnewline-eof) endif() # strict overflow check produces false positives on gcc < 8 if(CMAKE_COMPILER_IS_GNUCC AND CMAKE_C_COMPILER_VERSION VERSION_LESS 8) add_compile_options(-Wno-strict-overflow) endif() # -Wclobbered produces false positives on gcc < 9 if(CMAKE_COMPILER_IS_GNUCC AND CMAKE_C_COMPILER_VERSION VERSION_LESS 9) add_compile_options(-Wno-clobbered) endif() if(CMAKE_COMPILER_IS_GNUCC) add_compile_options( # Seems to be broken in GCC 11 with designated initializers. -Wno-missing-field-initializers) endif() # On UNIX, the compiler needs to support -fvisibility=hidden to hide symbols # by default check_c_compiler_flag(-fvisibility=hidden CC_SUPPORTS_VISIBILITY_HIDDEN) if(NOT CC_SUPPORTS_VISIBILITY_HIDDEN) message( FATAL_ERROR "The compiler ${CMAKE_C_COMPILER_ID} does not support -fvisibility=hidden" ) endif(NOT CC_SUPPORTS_VISIBILITY_HIDDEN) endif() # On Windows, default to only include Release builds so MSBuild.exe 'just works' if(WIN32 AND NOT CMAKE_CONFIGURATION_TYPES) set(CMAKE_CONFIGURATION_TYPES Release CACHE STRING "Semicolon separated list of supported configuration types, only supports Debug, Release, MinSizeRel, and RelWithDebInfo, anything else will be ignored." FORCE) endif() message(STATUS "Using compiler ${CMAKE_C_COMPILER_ID}") if(ENABLE_OPTIMIZER_DEBUG) message( STATUS "Enabling OPTIMIZER_DEBUG. Make sure that ${PG_SOURCE_DIR} is installed and built with OPTIMIZER_DEBUG." ) add_definitions(-DOPTIMIZER_DEBUG) endif() find_package(Git) # Check PostgreSQL version execute_process( COMMAND ${PG_CONFIG} --version OUTPUT_VARIABLE PG_VERSION_STRING OUTPUT_STRIP_TRAILING_WHITESPACE) if(NOT ${PG_VERSION_STRING} MATCHES "^PostgreSQL[ ]+([0-9]+)(\\.([0-9]+)|beta|devel|rc[0-9]+)") message(FATAL_ERROR "Could not parse PostgreSQL version ${PG_VERSION_STRING}") endif() set(PG_VERSION_MAJOR ${CMAKE_MATCH_1}) if(${CMAKE_MATCH_COUNT} GREATER "2") set(PG_VERSION_MINOR ${CMAKE_MATCH_3}) else() set(PG_VERSION_MINOR 0) endif() set(PG_VERSION "${PG_VERSION_MAJOR}.${PG_VERSION_MINOR}") message( STATUS "Compiling against PostgreSQL version ${PG_VERSION} using pg_config '${PG_CONFIG}'" ) # Ensure that PostgreSQL version is supported and consistent with src/compat.h # version check if((${PG_VERSION_MAJOR} LESS "15") OR (${PG_VERSION_MAJOR} GREATER "18") AND NOT (${EXPERIMENTAL})) message(FATAL_ERROR "TimescaleDB only supports PostgreSQL 15, 16, 17 and 18") endif() # Get PostgreSQL configuration from pg_config get_pg_config(PG_INCLUDEDIR --includedir) get_pg_config(PG_INCLUDEDIR_SERVER --includedir-server) get_pg_config(PG_LIBDIR --libdir) get_pg_config(PG_PKGLIBDIR --pkglibdir) get_pg_config(PG_SHAREDIR --sharedir) get_pg_config(PG_BINDIR --bindir) get_pg_config(PG_CFLAGS --cflags) get_pg_config(PG_CFLAGS_SL --cflags_sl) get_pg_config(PG_CPPFLAGS --cppflags) get_pg_config(PG_LDFLAGS --ldflags) get_pg_config(PG_LIBS --libs) separate_arguments(PG_CFLAGS) foreach(option ${PG_CFLAGS}) if(NOT ${option} MATCHES ^-W) set(filtered "${filtered} ${option}") endif() endforeach() set(PG_CFLAGS "${filtered} ${PG_CFLAGS_SL}") find_path( PG_SOURCE_DIR src/include/pg_config.h.in HINTS $ENV{HOME} $ENV{HOME}/projects $ENV{HOME}/Projects $ENV{HOME}/development $ENV{HOME}/Development $ENV{HOME}/workspace PATH_SUFFIXES postgres postgresql pgsql DOC "The path to the PostgreSQL source tree") option(PG_SOURCE_INCLUDES "Add PG source to include directories" OFF) if(PG_SOURCE_DIR) message(STATUS "Found PostgreSQL source in ${PG_SOURCE_DIR}") if(PG_SOURCE_INCLUDES) # Add the PostgreSQL source dir include directories to the build system # includes BEFORE the installed PG include files. This will allow the LSP # (e.g., clangd) to navigate to the PostgreSQL source instead of the install # path directory that only has the headers. include_directories(BEFORE SYSTEM ${PG_SOURCE_DIR}/src/include) endif(PG_SOURCE_INCLUDES) endif(PG_SOURCE_DIR) set(EXT_CONTROL_FILE ${PROJECT_NAME}.control) configure_file(${EXT_CONTROL_FILE}.in ${EXT_CONTROL_FILE}) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${EXT_CONTROL_FILE} DESTINATION "${PG_SHAREDIR}/extension") find_program( CLANG_FORMAT NAMES clang-format-17 clang-format PATHS /usr/bin /usr/local/bin /usr/local/opt/ /usr/local/opt/llvm/bin /opt/bin DOC "The path to clang-format") if(CLANG_FORMAT) execute_process( COMMAND ${CLANG_FORMAT} --version OUTPUT_VARIABLE CLANG_FORMAT_VERSION_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE) if(NOT ${CLANG_FORMAT_VERSION_OUTPUT} MATCHES "version[ ]+([0-9]+)\\.([0-9]+)(\\.([0-9]+))*") message( FATAL_ERROR "Could not parse clang-format version ${CLANG_FORMAT_VERSION_OUTPUT}") endif() if((${CMAKE_MATCH_1} LESS "17")) message(WARNING "clang-format version 17 or greater required") set(CLANG_FORMAT False) endif() endif() if(CLANG_FORMAT) message(STATUS "Using local clang-format") add_custom_target( clang-format COMMAND ${CMAKE_COMMAND} -E env CLANG_FORMAT=${CLANG_FORMAT} ${PROJECT_SOURCE_DIR}/scripts/clang_format_all.sh) endif() find_program( CMAKE_FORMAT NAMES cmake-format PATHS /usr/bin /usr/local/bin /usr/local/opt/ /usr/local/opt/llvm/bin /opt/bin DOC "The path to cmake-format") if(CMAKE_FORMAT) add_custom_target( cmake-format COMMAND ${CMAKE_COMMAND} -E env CMAKE_FORMAT=${CMAKE_FORMAT} ${PROJECT_SOURCE_DIR}/scripts/cmake_format_all.sh) endif() find_program( PERLTIDY NAMES perltidy PATHS /bin /usr/bin /usr/local/bin /usr/local/opt/ /opt/bin DOC "The path to perltidy") if(PERLTIDY) message(STATUS "Using perltidy ${PERLTIDY}") add_custom_target( perltidy COMMAND ${CMAKE_COMMAND} -E env PERLTIDY=${PERLTIDY} PERLTIDY_CONFIG="${PROJECT_SOURCE_DIR}/.perltidyrc" ${PROJECT_SOURCE_DIR}/scripts/perltidy_format_all.sh) endif() if(TARGET clang-format OR TARGET cmake-format OR TARGET perltidy) add_custom_target(format) if(TARGET clang-format) add_dependencies(format clang-format) endif() if(TARGET cmake-format) add_dependencies(format cmake-format) endif() if(TARGET perltidy) add_dependencies(format perltidy) endif() endif() if(REGRESS_CHECKS) find_program(PG_REGRESS pg_regress HINTS "${PG_BINDIR}" "${PG_PKGLIBDIR}/pgxs/src/test/regress/") if(NOT PG_REGRESS) message(STATUS "Regress checks disabled: program 'pg_regress' not found") endif() find_program( PG_ISOLATION_REGRESS NAMES pg_isolation_regress HINTS ${PG_BINDIR} ${PG_PKGLIBDIR}/pgxs/src/test/isolation ${PG_SOURCE_DIR}/src/test/isolation ${BINDIR}) if(NOT PG_ISOLATION_REGRESS) message( STATUS "Isolation regress checks disabled: 'pg_isolation_regress' not found") endif() else() message(STATUS "Regress checks and isolation checks disabled") endif() # Linter support via clang-tidy. Enabled when using clang as compiler option(LINTER "Enable linter support using clang-tidy" OFF) set(CLANG_TIDY_EXTRA_OPTS "" CACHE STRING "Additional options for clang-tidy") if(LINTER) find_program( CLANG_TIDY clang-tidy PATHS /usr/bin /usr/local/bin /usr/local/opt/ /usr/local/opt/llvm/bin /opt/bin DOC "The path to the clang-tidy linter" REQUIRED) execute_process(COMMAND ${CLANG_TIDY} --version OUTPUT_VARIABLE CLANG_TIDY_VERSION) message(STATUS "Using clang-tidy ${CLANG_TIDY}, ${CLANG_TIDY_VERSION}") string( CONCAT CMAKE_C_CLANG_TIDY "${CLANG_TIDY}" ";--checks=clang-diagnostic-*,clang-analyzer-*" ",-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling" ",-clang-analyzer-deadcode.DeadStores" ",bugprone-*" ",-bugprone-branch-clone" ",-bugprone-easily-swappable-parameters" ",-bugprone-implicit-widening-of-multiplication-result" ",-bugprone-narrowing-conversions" ",-bugprone-reserved-identifier" ",-bugprone-suspicious-include" ",readability-*" ",-readability-avoid-const-params-in-decls" ",-readability-braces-around-statements" ",-readability-else-after-return" ",-readability-function-cognitive-complexity" ",-readability-function-size" ",-readability-identifier-length" ",-readability-isolate-declaration" ",-readability-magic-numbers" ",-readability-math-missing-parentheses" ",-readability-non-const-parameter" ",-readability-redundant-casting" "${CLANG_TIDY_EXTRA_OPTS}") if(WARNINGS_AS_ERRORS) set(CMAKE_C_CLANG_TIDY "${CMAKE_C_CLANG_TIDY};--warnings-as-errors=*") else() set(CMAKE_C_CLANG_TIDY "${CMAKE_C_CLANG_TIDY};--quiet") endif(WARNINGS_AS_ERRORS) endif(LINTER) if(NOT EXISTS ${PG_INCLUDEDIR}/pg_config.h) message( FATAL_ERROR "Could not find pg_config.h in ${PG_INCLUDEDIR}. " "Make sure PG_PATH points to a valid PostgreSQL installation that includes development headers." ) endif() file(READ ${PG_INCLUDEDIR}/pg_config.h PG_CONFIG_H) string(REGEX MATCH "#define USE_ASSERT_CHECKING 1" PG_USE_ASSERT_CHECKING ${PG_CONFIG_H}) if(PG_USE_ASSERT_CHECKING AND NOT ASSERTIONS) message( STATUS "Assertion checks are OFF although enabled in PostgreSQL build (pg_config.h). " "The PostgreSQL setting for assertions will take precedence.") elseif(ASSERTIONS) message(STATUS "Assertion checks are ON") add_compile_definitions(USE_ASSERT_CHECKING=1) elseif(CMAKE_BUILD_TYPE MATCHES Debug) message( "Assertion checks are OFF in Debug build. Set -DASSERTIONS=ON to enable assertions." ) else() message(STATUS "Assertion checks are OFF") endif() # Check if PostgreSQL has OpenSSL enabled by inspecting pg_config.h. Right now, # a Postgres header will redefine an OpenSSL function if Postgres is not # installed --with-openssl, so in order for TimescaleDB to compile correctly # with OpenSSL, Postgres must also have OpenSSL enabled. check_symbol_exists(USE_OPENSSL ${PG_INCLUDEDIR}/pg_config.h PG_USE_OPENSSL) if(USE_OPENSSL AND (NOT PG_USE_OPENSSL)) message( FATAL_ERROR "PostgreSQL was built without OpenSSL support, which TimescaleDB needs for full compatibility. Please rebuild PostgreSQL using `--with-openssl` or if you want to continue without OpenSSL, re-run bootstrap with `-DUSE_OPENSSL=0`" ) endif(USE_OPENSSL AND (NOT PG_USE_OPENSSL)) # While we dont link directly against OpenSSL on non-Windows, doing this on # Windows causes linker errors. So on Windows we link directly against the # OpenSSL libraries. if(USE_OPENSSL AND MSVC) # Try to find a local OpenSSL installation find_package(OpenSSL) if(NOT OPENSSL_FOUND) message( FATAL_ERROR "TimescaleDB requires OpenSSL but it wasn't found. If you want to continue without OpenSSL, re-run bootstrap with `-DUSE_OPENSSL=0`" ) endif(NOT OPENSSL_FOUND) if(${OPENSSL_VERSION} VERSION_LESS "1.0") message(FATAL_ERROR "TimescaleDB requires OpenSSL version 1.0 or greater") endif() set(_libraries) foreach(_path ${OPENSSL_LIBRARIES}) if(EXISTS "${_path}") list(APPEND _libraries ${_path}) else() # check if a release version of the libraries are available if(CMAKE_BUILD_TYPE STREQUAL "Debug" AND MSVC) get_filename_component(_dir ${_path} DIRECTORY) get_filename_component(_name ${_path} NAME_WE) string(REGEX REPLACE "[Dd]$" "" _fixed ${_name}) get_filename_component(_ext ${_path} EXT) set(_new_path "${_dir}/${_fixed}${_ext}") if(EXISTS "${_new_path}") list(APPEND _libraries ${_new_path}) endif() endif() endif() endforeach() set(OPENSSL_LIBRARIES ${_libraries}) foreach(_path ${OPENSSL_LIBRARIES}) message(STATUS "OpenSSL libraries: ${_path}") endforeach() message(STATUS "Using OpenSSL version ${OPENSSL_VERSION}") endif(USE_OPENSSL AND MSVC) if(CODECOVERAGE) message(STATUS "Code coverage is enabled.") # Note that --coverage is synonym for the necessary compiler and linker flags # for the given compiler. For example, with GCC, --coverage translates to # -fprofile-arcs -ftest-coverage when compiling and -lgcov when linking add_compile_options(--coverage -O0) add_link_options(--coverage) # When a source file is recompiled, the compiler regenerates the .gcno file # but any .gcda file from a previous test run remains with a stale checksum. # lcov then silently produces incorrect coverage data. Delete the .gcda # alongside the .o to prevent this. The $$ escaping produces a literal $ for # the shell in both Ninja and Make generators. set(CMAKE_C_COMPILE_OBJECT "${CMAKE_C_COMPILE_OBJECT} && o=<OBJECT> && rm -f \$\${o%.o}.gcda") endif(CODECOVERAGE) # TAP test support option(TAP_CHECKS "Enable TAP test support" ON) if(TAP_CHECKS) find_package(Perl 5.8) if(PERL_FOUND) get_filename_component(PERL_BIN_PATH ${PERL_EXECUTABLE} DIRECTORY) find_program( PROVE prove HINTS ${PERL_BIN_PATH} PATHS "/usr/bin") if(NOT PROVE) message(STATUS "Not running TAP tests: 'prove' binary not found.") set(TAP_CHECKS OFF) endif() # Check for the IPC::Run module execute_process( COMMAND ${PERL_EXECUTABLE} -MIPC::Run -e "" ERROR_QUIET RESULT_VARIABLE PERL_MODULE_STATUS) if(PERL_MODULE_STATUS) message(STATUS "Not running TAP tests: IPC::Run Perl module not found.") set(TAP_CHECKS OFF) endif() else() message(STATUS "Not running TAP tests: Perl not found.") set(TAP_CHECKS OFF) endif() endif() if(UNIX) add_subdirectory(scripts) endif(UNIX) add_subdirectory(sql) add_subdirectory(test) add_subdirectory(src) if(NOT APACHE_ONLY) add_subdirectory(tsl) endif() add_custom_target(licensecheck COMMAND ${PROJECT_SOURCE_DIR}/scripts/check_license_all.sh) # This needs to be the last subdirectory so that other targets are already # defined if(CODECOVERAGE) add_subdirectory(coverage) endif() if(IS_DIRECTORY ${PROJECT_SOURCE_DIR}/.git) configure_file(${PROJECT_SOURCE_DIR}/scripts/githooks/commit_msg.py ${PROJECT_SOURCE_DIR}/.git/hooks/commit-msg COPYONLY) endif() ================================================ FILE: CONTRIBUTING.md ================================================ # Contributing to TimescaleDB We appreciate any help the community can provide to make TimescaleDB better! You can help in different ways: * Open an [issue](https://github.com/timescale/timescaledb/issues) with a bug report, build issue, feature request, suggestion, etc. * Fork this repository and submit a pull request For any particular improvement you want to make, it can be beneficial to begin discussion on the GitHub issues page. This is the best place to discuss your proposed improvement (and its implementation) with the core development team. Before we accept any code contributions, Timescale contributors need to sign the [Contributor License Agreement](https://cla-assistant.io/timescale/timescaledb) (CLA). By signing a CLA, we can ensure that the community is free and confident in its ability to use your contributions. ## Getting and building TimescaleDB Please follow our README for [instructions on installing from source](https://github.com/timescale/timescaledb/blob/main/docs/BuildSource.md). ## Style guide Before submitting any contributions, please ensure that it adheres to our [Style Guide](https://github.com/timescale/timescaledb/blob/main/docs/StyleGuide.md). ## Code review workflow * Sign the [Contributor License Agreement](https://cla-assistant.io/timescale/timescaledb) (CLA) if you're a new contributor. * Develop on your local branch: * Fork the repository and create a local feature branch to do work on, ideally on one thing at a time. Don't mix bug fixes with unrelated feature enhancements or stylistical changes. * Hack away. Add tests for non-trivial changes. * Run the [test suite](#testing) and make sure everything passes. * When committing, be sure to write good commit messages according to [these seven rules](https://chris.beams.io/posts/git-commit/#seven-rules). Doing `git commit` prints a message if any of the rules is violated. Stylistically, we use commit message titles in the imperative tense, e.g., `Add merge-append query optimization for time aggregate`. In the case of non-trivial changes, include a longer description in the commit message body explaining and detailing the changes. That is, a commit message should have a short title, followed by a empty line, and then followed by the longer description. * When committing, link which GitHub issue of [this repository](https://github.com/timescale/timescaledb/issues) is fixed or closed by the commit with a [linking keyword recognised by GitHub](https://docs.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword). For example, if the commit fixes bug 123, add a line at the end of the commit message with `Fixes #123`, if the commit implements feature request 321, add a line at the end of the commit message `Closes #321`. This will be recognized by GitHub. It will close the corresponding issue and place a hyperlink under the number. * Push your changes to an upstream branch: * Make sure that each commit in the pull request will represent a logical change to the code, will compile, and will pass tests. * Make sure that the pull request message contains all important information from the commit messages including which issues are fixed and closed. If a pull request contains one commit only, then repeating the commit message is preferred, which is done automatically by GitHub when it creates the pull request. * Rebase your local feature branch against main (`git fetch origin`, then `git rebase origin/main`) to make sure you're submitting your changes on top of the newest version of our code. * When finalizing your PR (i.e., it has been approved for merging), aim for the fewest number of commits that make sense. That is, squash any "fix up" commits into the commit they fix rather than keep them separate. Each commit should represent a clean, logical change and include a descriptive commit message. * Push your commit to your upstream feature branch: `git push -u <yourfork> my-feature-branch` * Create and manage pull request: * [Create a pull request using GitHub](https://help.github.com/articles/creating-a-pull-request). If you know a core developer well suited to reviewing your pull request, either mention them (preferably by GitHub name) in the PR's body or [assign them as a reviewer](https://help.github.com/articles/assigning-issues-and-pull-requests-to-other-github-users/). * If you get a test failure in the CI, check them under [Github Actions](https://github.com/timescale/timescaledb/actions) * Address feedback by amending your commit(s). If your change contains multiple commits, address each piece of feedback by amending that commit to which the particular feedback is aimed. * The PR is marked as accepted when the reviewer thinks it's ready to be merged. Most new contributors aren't allowed to merge themselves; in that case, we'll do it for you. ## Testing Every non-trivial change to the code base should be accompanied by a relevant addition to or modification of the test suite. Please check that the full test suite (including your test additions or changes) passes successfully on your local machine **before you open a pull request**. If you are running locally: ```bash # Use Debug build mode for full battery of tests ./bootstrap -DCMAKE_BUILD_TYPE=Debug cd build && make make installcheck ``` All submitted pull requests are also automatically run against our test suite via [Github Actions](https://github.com/timescale/timescaledb/actions) (that link shows the latest build status of the repository). ## Reviewing and accepting your contribution We appreciate everyone who is investing time in contributing to TimescaleDB and regret that we sometimes have to reject contributions even when they might appear to add value. If the contribution is accepted, we will merge the changes, acknowledge your contribution in the release, and take care of the backporting to the relevant branches, if necessary. Before you start, please discuss your change in a GitHub issue before spending much time on its implementation. We sometimes have to reject contributions that duplicate other efforts, take the wrong approach to solving a problem, or solve a problem which does not need solving. An up-front discussion often saves your time. A contribution is expected to address one specific change. Pull requests are expected to be small, if a PR is deemed too large or touches too many disparate parts of the system, you will be required to break it down into a series of smaller, digestible PRs before reviewing continues. Avoid adding unnecessary stylistic changes. TimescaleDB is complex system, requiring a deep understanding of Postgres internals, the system architecture and the potential secondary effects of changes. While we highly value community input, the core development team's primary responsibility is maintaining the health of the project. We reserve the right to respectfully close the PR. This is not a reflection of your skills as an engineer, but rather a necessity of resource allocation to keep the project stable and performant. We reserved the right to reject contributions, if the time required on reviews would outweigh the benefits of a change by preventing us from working on other beneficial changes instead. We sometimes reject contributions due to the low quality of the submission since low-quality submissions tend to take unreasonable effort to review properly. Quality is rather subjective so it is hard to describe exactly how to avoid this, but there are some basic steps you can take to reduce the chances of rejection: * Unit Tests: Every new function or modified logic path must have accompanying unit tests. * Integration Tests: Features that touch the storage engine, query planner, or sub system must include integration tests. * Edge Cases: You are expected to proactively test for edge cases, concurrency hazards, and out-of-memory (OOM) scenarios. * No regressions: Your code must pass all existing CI/CD pipelines, including e.g. fuzzing, without degrading current metrics. * Style Guide: Your code must be formatted according to the projects [style guide](#style-guide). We welcome the use of AI coding assistants (Copilot, Gemini, etc.) to enhance your productivity. However, AI-generated contributions adhere to the exact same rigorous standards as human-written code. You are fully responsible for the accuracy, safety, and performance of any AI-generated code you submit. If a PR appears to be low quality AI outputs without reflecting a proficient understanding of the change, we will close the PR immediately. We expect you to follow up on review comments, but recognise that everyone has many priorities for their time and may not be able to respond for several days. We will understand if you find yourself without the time to complete your contribution, but please let us know that you have stopped working on it and we can conclude how to handle the contribution. After two weeks of inactivity, we will reject the contribution, unless we complete it. If your contribution is rejected, we will close the pull request with a comment explaining why. ## License headers We require license headers on all C and SQL files, unless explicitly instructed otherwise. All C and SQL files in the [tsl](https://github.com/timescale/timescaledb/tree/main/tsl) directory require the following short license header of the [Timescale License Agreement](https://github.com/timescale/timescaledb/blob/main/tsl/LICENSE-TIMESCALE): ``` /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ ``` All C and SQL files in the [src](https://github.com/timescale/timescaledb/tree/main/src) directory require the following short license header of the [Apache License](https://github.com/timescale/timescaledb/blob/main/LICENSE-APACHE): ``` /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ ``` ================================================ FILE: LICENSE ================================================ Source code in this repository is variously licensed under the Apache License Version 2.0, an Apache compatible license, or the Timescale License. All source code should have information at the beginning of its respective file which specifies its licensing information. * Outside of the "tsl" directory, source code in a given file is licensed under the Apache License Version 2.0, unless otherwise noted (e.g., an Apache-compatible license). * Within the "tsl" folder, source code in a given file is licensed under the Timescale License, unless otherwise noted. When built, separate shared object files are generated for the Apache-licensed source code and the Timescale-licensed source code. The shared object binaries that contain `-tsl` in their name are licensed under the Timescale License. ================================================ FILE: LICENSE-APACHE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS ================================================ FILE: NOTICE ================================================ TimescaleDB (TM) Copyright (c) 2017-2026 Timescale, Inc. All Rights Reserved. Copyright (c) 2016-2017 440 Labs, Inc. dba Timescale. All Rights Reserved. Source code in this repository is variously licensed under the Apache License Version 2.0, an Apache-compatible license, or the Timescale License. Please see LICENSE for more information. * For a copy of the Apache License Version 2.0, please see LICENSE-APACHE as included in this repository's top-level directory. * For a copy of the Timescale License, please see LICENSE-TIMESCALE as included in this repository's "tsl" directory. * For a copy of all other Apache-compatible licenses and notices, please see below. ======================================================================== NOTICES ======================================================================== Certain files in this code base have been modified and/or copied, either partially or wholely, from source code from the PostgreSQL database management system, which is licensed under the open-source PostgreSQL License with the following copyright information. The PostgreSQL License ======================================================================== PostgreSQL Database Management System (formerly known as Postgres, then as Postgres95) Portions Copyright (c) 1996-2026, The PostgreSQL Global Development Group Portions Copyright (c) 1994, The Regents of the University of California Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this paragraph and the following two paragraphs appear in all copies. IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. ======================================================================== ================================================ FILE: README.md ================================================ <div align=center> <picture align=center> <source srcset="https://assets.timescale.com/timescale-web/brand/show/horizontal-black.svg"> <img alt="Tiger Data logo" > </picture> </div> <div align=center> <h3>TimescaleDB is a PostgreSQL extension for high-performance real-time analytics on time-series and event data</h3> [![Docs](https://img.shields.io/badge/Read_the_docs-black?style=for-the-badge&logo=readthedocs&logoColor=white)](https://docs.tigerdata.com/) [![SLACK](https://img.shields.io/badge/Ask_the_community-black?style=for-the-badge&logo=slack&logoColor=white)](https://timescaledb.slack.com/archives/C4GT3N90X) [![Try TimescaleDB for free](https://img.shields.io/badge/Try_Tiger_Cloud_for_free-black?style=for-the-badge&logo=timescale&logoColor=white)](https://console.cloud.timescale.com/signup) </div> ## Quick Start with TimescaleDB Get started with TimescaleDB in under 10 minutes. This guide will help you run TimescaleDB locally, create your first hypertable with columnstore enabled, write data to the columnstore, and see instant analytical query performance. ### What You'll Learn - How to run TimescaleDB with a one-line install or Docker command - How to create a hypertable with columnstore enabled - How to insert data directly to the columnstore - How to execute analytical queries ### Prerequisites - Docker installed on your machine - 8GB RAM recommended - `psql` client (included with PostgreSQL) or any PostgreSQL client like [pgAdmin](https://www.pgadmin.org/download/) ### Step 1: Start TimescaleDB You have two options to start TimescaleDB: #### Option 1: One-line install (Recommended) The easiest way to get started: > **Important:** This script is intended for local development and testing only. Do **not** use it for production deployments. For production-ready installation options, see the [TimescaleDB installation guide](https://docs.timescale.com/self-hosted/latest/install/). **Linux/Mac:** ```sh curl -sL https://tsdb.co/start-local | sh ``` This command: - Downloads and starts TimescaleDB (if not already downloaded) - Exposes PostgreSQL on port **6543** (a non-standard port to avoid conflicts with other PostgreSQL instances on port 5432) - Automatically tunes settings for your environment using timescaledb-tune - Sets up a persistent data volume #### Option 2: Manual Docker command also used for Windows Alternatively, you can run TimescaleDB directly with Docker: ```bash docker run -d --name timescaledb \ -p 6543:5432 \ -e POSTGRES_PASSWORD=password \ timescale/timescaledb-ha:pg18 ``` **Note:** We use port **6543** (mapped to container port 5432) to avoid conflicts if you have other PostgreSQL instances running on the standard port 5432. Wait about 1-2 minutes for TimescaleDB to download & initialize. ### Step 2: Connect to TimescaleDB Connect using `psql`: ```bash psql -h localhost -p 6543 -U postgres # When prompted, enter password: password ``` You should see the PostgreSQL prompt. Verify TimescaleDB is installed: ```sql SELECT extname, extversion FROM pg_extension WHERE extname = 'timescaledb'; ``` Expected output: ``` extname | extversion -------------+------------ timescaledb | 2.x.x ``` **Prefer a GUI?** If you'd rather use a graphical tool instead of the command line, you can download [pgAdmin](https://www.pgadmin.org/download/) and connect to TimescaleDB using the same connection details (host: `localhost`, port: `6543`, user: `postgres`, password: `password`). ### Step 3: Create Your First Hypertable Let's create a hypertable for IoT sensor data with columnstore enabled: ```sql -- Create a hypertable with automatic columnstore CREATE TABLE sensor_data ( time TIMESTAMPTZ NOT NULL, sensor_id TEXT NOT NULL, temperature DOUBLE PRECISION, humidity DOUBLE PRECISION, pressure DOUBLE PRECISION ) WITH ( tsdb.hypertable ); -- create index CREATE INDEX idx_sensor_id_time ON sensor_data(sensor_id, time DESC); ``` `tsdb.hypertable` - Converts this into a TimescaleDB hypertable See more: - [About hypertables](https://docs.tigerdata.com/use-timescale/latest/hypertables/) - [API reference](https://docs.tigerdata.com/api/latest/hypertable/) - [About columnstore](https://docs.tigerdata.com/use-timescale/latest/compression/about-compression/) - [Enable columnstore manually](https://docs.tigerdata.com/use-timescale/latest/compression/manual-compression/) - [API reference](https://docs.tigerdata.com/api/latest/compression/) ### Step 4: Insert Sample Data Let's add some sample sensor readings: ```sql -- Enable timing to see time to execute queries \timing on -- Insert sample data for multiple sensors -- SET timescaledb.enable_direct_compress_insert = on to insert data directly to the columnstore (columnnar format for performance) SET timescaledb.enable_direct_compress_insert = on; INSERT INTO sensor_data (time, sensor_id, temperature, humidity, pressure) SELECT time, 'sensor_' || ((random() * 9)::int + 1), 20 + (random() * 15), 40 + (random() * 30), 1000 + (random() * 50) FROM generate_series( NOW() - INTERVAL '90 days', NOW(), INTERVAL '1 seconds' ) AS time; -- Once data is inserted into the columnstore we optimize the order and structure -- this compacts and orders the data in the chunks for optimal query performance and compression DO $$ DECLARE ch TEXT; BEGIN FOR ch IN SELECT show_chunks('sensor_data') LOOP CALL convert_to_columnstore(ch, recompress := true); END LOOP; END $$; ``` This generates ~7,776,001 readings across 10 sensors over the past 90 days. Verify the data was inserted: ```sql SELECT COUNT(*) FROM sensor_data; ``` ### Step 5: Run Your First Analytical Queries Now let's run some analytical queries that showcase TimescaleDB's performance: ```sql -- Enable query timing to see performance \timing on -- Query 1: Average readings per sensor over the last 7 days SELECT sensor_id, COUNT(*) as readings, ROUND(AVG(temperature)::numeric, 2) as avg_temp, ROUND(AVG(humidity)::numeric, 2) as avg_humidity, ROUND(AVG(pressure)::numeric, 2) as avg_pressure FROM sensor_data WHERE time > NOW() - INTERVAL '7 days' GROUP BY sensor_id ORDER BY sensor_id; -- Query 2: Hourly averages using time_bucket -- Time buckets enable you to aggregate data in hypertables by time interval and calculate summary values. SELECT time_bucket('1 hour', time) AS hour, sensor_id, ROUND(AVG(temperature)::numeric, 2) as avg_temp, ROUND(AVG(humidity)::numeric, 2) as avg_humidity FROM sensor_data WHERE time > NOW() - INTERVAL '24 hours' GROUP BY hour, sensor_id ORDER BY hour DESC, sensor_id LIMIT 20; -- Query 3: Daily statistics across all sensors SELECT time_bucket('1 day', time) AS day, COUNT(*) as total_readings, ROUND(AVG(temperature)::numeric, 2) as avg_temp, ROUND(MIN(temperature)::numeric, 2) as min_temp, ROUND(MAX(temperature)::numeric, 2) as max_temp FROM sensor_data GROUP BY day ORDER BY day DESC LIMIT 10; -- Query 4: Latest reading for each sensor -- Highlights the value of Skipscan executing in under 100ms without skipscan it takes over 5sec SELECT DISTINCT ON (sensor_id) sensor_id, time, ROUND(temperature::numeric, 2) as temperature, ROUND(humidity::numeric, 2) as humidity, ROUND(pressure::numeric, 2) as pressure FROM sensor_data ORDER BY sensor_id, time DESC; ``` Notice how fast these analytical queries run, even with aggregations across millions of rows. This is the power of TimescaleDB's columnstore. ### What's Happening Behind the Scenes? TimescaleDB automatically: - **Partitions your data** into time-based chunks for efficient querying - **Write directly to columnstore** using columnar storage (90%+ compression typical) and faster vectorized queries - **Optimizes queries** by only scanning relevant time ranges and columns - **Enables time_bucket()** - a powerful function for time-series aggregation See more: - [Query data](https://docs.tigerdata.com/use-timescale/latest/query-data/) - [Write data](https://docs.tigerdata.com/use-timescale/latest/write-data/) - [About time buckets](https://docs.tigerdata.com/use-timescale/latest/time-buckets/about-time-buckets/) - [API reference](https://docs.tigerdata.com/api/latest/hyperfunctions/time_bucket/) - [All TimescaleDB features](https://docs.tigerdata.com/use-timescale/latest/) ### Next Steps Now that you've got the basics, explore more: ### Create Continuous Aggregates Continuous aggregates make real-time analytics run faster on very large datasets. They continuously and incrementally refresh a query in the background, so that when you run such query, only the data that has changed needs to be computed, not the entire dataset. This is what makes them different from regular PostgreSQL [materialized views](https://www.postgresql.org/docs/current/rules-materializedviews.html), which cannot be incrementally materialized and have to be rebuilt from scratch every time you want to refresh them. Let's create a continuous aggregate for hourly sensor statistics: #### Step 1: Create the Continuous Aggregate ```sql CREATE MATERIALIZED VIEW sensor_data_hourly WITH (timescaledb.continuous) AS SELECT time_bucket('1 hour', time) AS hour, sensor_id, AVG(temperature) AS avg_temp, AVG(humidity) AS avg_humidity, AVG(pressure) AS avg_pressure, MIN(temperature) AS min_temp, MAX(temperature) AS max_temp, COUNT(*) AS reading_count FROM sensor_data GROUP BY hour, sensor_id; ``` This creates a materialized view that pre-aggregates your sensor data into hourly buckets. The view is automatically populated with existing data. #### Step 2: Add a Refresh Policy To keep the continuous aggregate up-to-date as new data arrives, add a refresh policy: ```sql SELECT add_continuous_aggregate_policy( 'sensor_data_hourly', start_offset => INTERVAL '3 hours', end_offset => INTERVAL '1 hour', schedule_interval => INTERVAL '1 hour' ); ``` This policy: - Refreshes the continuous aggregate every hour - Processes data from 3 hours ago up to 1 hour ago (leaving the most recent hour for real-time queries) - Only processes new or changed data incrementally #### Step 3: Query the Continuous Aggregate Now you can query the pre-aggregated data for much faster results: ```sql -- Get hourly averages for the last 24 hours SELECT hour, sensor_id, ROUND(avg_temp::numeric, 2) AS avg_temp, ROUND(avg_humidity::numeric, 2) AS avg_humidity, reading_count FROM sensor_data_hourly WHERE hour > NOW() - INTERVAL '24 hours' ORDER BY hour DESC, sensor_id LIMIT 50; ``` #### Benefits of Continuous Aggregates - **Faster queries**: Pre-aggregated data means queries run in milliseconds instead of seconds - **Incremental refresh**: Only new/changed data is processed, not the entire dataset - **Automatic updates**: The refresh policy keeps your aggregates current without manual intervention - **Real-time option**: You can enable real-time aggregation to combine materialized and raw data #### Try It Yourself Compare the performance difference: ```sql -- Query the raw hypertable (slower on large datasets) \timing on SELECT time_bucket('1 hour', time) AS hour, AVG(temperature) AS avg_temp FROM sensor_data WHERE time > NOW() - INTERVAL '60 days' GROUP BY hour ORDER BY hour DESC LIMIT 24; -- Query the continuous aggregate (much faster) SELECT hour, avg_temp FROM sensor_data_hourly WHERE hour > NOW() - INTERVAL '60 days' ORDER BY hour DESC LIMIT 24; ``` Notice how the continuous aggregate query is significantly faster, especially as your dataset grows! See more: - [About continuous aggregates](https://docs.tigerdata.com/use-timescale/latest/continuous-aggregates/) - [API reference](https://docs.tigerdata.com/api/latest/continuous-aggregates/create_materialized_view/) - [TimescaleDB Documentation](https://docs.timescale.com) - [Time-series Best Practices](https://docs.timescale.com/use-timescale/latest/schema-management/) - [Continuous Aggregates](https://docs.timescale.com/use-timescale/latest/continuous-aggregates/) ## Examples Learn TimescaleDB with complete, standalone examples using real-world datasets. Each example includes sample data and analytical queries. - **[NYC Taxi Data](docs/getting-started/nyc-taxi/)** - Transportation and location-based analytics - **[Financial Market Data](docs/getting-started/financial-ticks/)** - Trading and market data analysis - **[Application Events](docs/getting-started/events-uuidv7/)** - Event logging with UUIDv7 Or try some of our workshops - **[AI Workshop: EV Charging Station Analysis](https://github.com/timescale/TigerData-Workshops/tree/main/AI-Workshop)** - Integrate PostgreSQL with AI capabilities for managing and analyzing EV charging station data - **[Time-Series Workshop: Financial Data Analysis](https://github.com/timescale/TigerData-Workshops/tree/main/TimeSeries-Workshop-Finance/)** - Work with cryptocurrency tick data, create candlestick charts ## Want TimescaleDB hosted and managed for you? Try Tiger Cloud [Tiger Cloud](https://docs.tigerdata.com/getting-started/latest/) is the modern PostgreSQL data platform for all your applications. It enhances PostgreSQL to handle time series, events, real-time analytics, and vector search—all in a single database alongside transactional workloads. You get one system that handles live data ingestion, late and out-of-order updates, and low latency queries, with the performance, reliability, and scalability your app needs. Ideal for IoT, crypto, finance, SaaS, and a myriad other domains, Tiger Cloud allows you to build data-heavy, mission-critical apps while retaining the familiarity and reliability of PostgreSQL. See [our whitepaper](https://docs.tigerdata.com/about/latest/whitepaper/) for a deep dive into Tiger Cloud's architecture and how it meets the needs of even the most demanding applications. A Tiger Cloud service is a single optimized 100% PostgreSQL database instance that you use as is, or extend with capabilities specific to your business needs. The available capabilities are: - **Time-series and analytics**: PostgreSQL with TimescaleDB. The PostgreSQL you know and love, supercharged with functionality for storing and querying time-series data at scale for real-time analytics and other use cases. Get faster time-based queries with hypertables, continuous aggregates, and columnar storage. Save on storage with native compression, data retention policies, and bottomless data tiering to Amazon S3. - **AI and vector**: PostgreSQL with vector extensions. Use PostgreSQL as a vector database with purpose built extensions for building AI applications from start to scale. Get fast and accurate similarity search with the pgvector and pgvectorscale extensions. Create vector embeddings and perform LLM reasoning on your data with the pgai extension. - **PostgreSQL**: the trusted industry-standard RDBMS. Ideal for applications requiring strong data consistency, complex relationships, and advanced querying capabilities. Get ACID compliance, extensive SQL support, JSON handling, and extensibility through custom functions, data types, and extensions. All services include all the cloud tooling you'd expect for production use: [automatic backups](https://docs.tigerdata.com/use-timescale/latest/backup-restore/backup-restore-cloud/), [high availability](https://docs.tigerdata.com/use-timescale/latest/ha-replicas/), [read replicas](https://docs.tigerdata.com/use-timescale/latest/ha-replicas/read-scaling/), [data forking](https://docs.tigerdata.com/use-timescale/latest/services/service-management/#fork-a-service), [connection pooling](https://docs.tigerdata.com/use-timescale/latest/services/connection-pooling/), [tiered storage](https://docs.tigerdata.com/use-timescale/latest/data-tiering/), [usage-based storage](https://docs.tigerdata.com/about/latest/pricing-and-account-management/), and much more. ## Check build status |Linux/macOS|Linux i386|Windows|Coverity|Code Coverage|OpenSSF| |:---:|:---:|:---:|:---:|:---:|:---:| |[![Build Status Linux/macOS](https://github.com/timescale/timescaledb/actions/workflows/linux-build-and-test.yaml/badge.svg?branch=main&event=schedule)](https://github.com/timescale/timescaledb/actions/workflows/linux-build-and-test.yaml?query=workflow%3ARegression+branch%3Amain+event%3Aschedule)|[![Build Status Linux i386](https://github.com/timescale/timescaledb/actions/workflows/linux-32bit-build-and-test.yaml/badge.svg?branch=main&event=schedule)](https://github.com/timescale/timescaledb/actions/workflows/linux-32bit-build-and-test.yaml?query=workflow%3ARegression+branch%3Amain+event%3Aschedule)|[![Windows build status](https://github.com/timescale/timescaledb/actions/workflows/windows-build-and-test.yaml/badge.svg?branch=main&event=schedule)](https://github.com/timescale/timescaledb/actions/workflows/windows-build-and-test.yaml?query=workflow%3ARegression+branch%3Amain+event%3Aschedule)|[![Coverity Scan Build Status](https://scan.coverity.com/projects/timescale-timescaledb/badge.svg)](https://scan.coverity.com/projects/timescale-timescaledb)|[![Code Coverage](https://codecov.io/gh/timescale/timescaledb/branch/main/graphs/badge.svg?branch=main)](https://codecov.io/gh/timescale/timescaledb)|[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/8012/badge)](https://www.bestpractices.dev/projects/8012)| ## Get involved We welcome contributions to TimescaleDB! See [Contributing](https://github.com/timescale/timescaledb/blob/main/CONTRIBUTING.md) and [Code style guide](https://github.com/timescale/timescaledb/blob/main/docs/StyleGuide.md) for details. ## Learn about Tiger Data Tiger Data is the fastest PostgreSQL for transactional, analytical and agentic workloads. To learn more about the company and its products, visit [tigerdata.com](https://www.tigerdata.com). ## Troubleshooting #### Docker container won't start ```bash # Check if container is running docker ps -a # View container logs (use the appropriate container name) # For one-line install: docker logs timescaledb-ha-pg18-quickstart # For manual Docker command: docker logs timescaledb # Stop and remove existing container # For one-line install: docker stop timescaledb-ha-pg18-quickstart && docker rm timescaledb-ha-pg18-quickstart # For manual Docker command: docker stop timescaledb && docker rm timescaledb # Start fresh # Option 1: Use the one-line install curl -sL https://tsdb.co/start-local | sh # Option 2: Use manual Docker command docker run -d --name timescaledb -p 6543:5432 -e POSTGRES_PASSWORD=password timescale/timescaledb-ha:pg18 ``` #### Can't connect with psql - Verify Docker container is running: `docker ps` - Check port 6543 isn't already in use: `lsof -i :6543` - Try using explicit host: `psql -h 127.0.0.1 -p 6543 -U postgres` #### TimescaleDB extension not found The `timescale/timescaledb-ha:pg18` image has TimescaleDB pre-installed and pre-loaded. If you see errors, ensure you're using the correct image. ## Clean Up When you're done experimenting: #### If you used the one-line install: ```bash # Stop the container docker stop timescaledb-ha-pg18-quickstart # Remove the container docker rm timescaledb-ha-pg18-quickstart # Remove the persistent data volume docker volume rm timescaledb_data # (Optional) Remove the Docker image docker rmi timescale/timescaledb-ha:pg18 ``` #### If you used the manual Docker command: ```bash # Stop the container docker stop timescaledb # Remove the container docker rm timescaledb # (Optional) Remove the Docker image docker rmi timescale/timescaledb-ha:pg18 ``` **Note:** If you created a named volume with the manual Docker command, you can remove it with `docker volume rm <volume_name>`. ================================================ FILE: SECURITY.md ================================================ # Security Policy We aim to keep TimescaleDB safe for everyone. Publicly disclosing security bugs in a public forum can put everyone in the Timescale community at risk, however. Therefore, we ask that people follow the below instructions to report security vulnerability. The entire Timescale community thanks you! ## Supported Versions The supported version is always the latest major release available in our repository. We also release regular minor versions with fixes and corrections alongside some new features as well as patchfix releases, that you should keep upgrading to. Vulnerability fixes are made available as part of these patchfix releases and you can read our list of [Security Advisories](https://github.com/timescale/timescaledb/security/advisories?state=published). You can also take a look at our [Support Policy](https://www.timescale.com/legal/support-policy). ## Reporting a Vulnerability If you find a vulnerability in our software, please email the Timescale Security Team at security@timescale.com. Please note that the e-mail address should only be used for reporting undisclosed security vulnerabilities in Timescale products and services. Regular bug reports should be submitted as GitHub issues, while other _questions_ around security, compliance, or functionality can be made either through our support (for customers) or community channels (e.g., [Timescale Slack](https://slack.timescale.com/), [Forums](https://www.timescale.com/forums), etc.) ================================================ FILE: bootstrap ================================================ #!/usr/bin/env bash # This bootstrap scripts set up the build environment for TimescaleDB # Any flags will be passed on to CMake, e.g., # ./bootstrap -DCMAKE_BUILD_TYPE="Debug" ## Check to make cmake is installed if ! command -v cmake >/dev/null 2>&1; then echo "cmake is required to build TimescaleDB. Please install via your system's preferred method." exit 1 fi BUILD_DIR=${BUILD_DIR:-./build} BUILD_FORCE_REMOVE=${BUILD_FORCE_REMOVE:-false} SRC_DIR=$(dirname $0) if [[ ! ${SRC_DIR} == /* ]]; then SRC_DIR=$(pwd)/${SRC_DIR} fi if [ ${BUILD_FORCE_REMOVE} == "true" ]; then rm -fr ${BUILD_DIR} elif [ -d ${BUILD_DIR} ]; then echo "Build system already initialized in ${BUILD_DIR}" read -r -n 1 -p "Do you want to remove it (this is IMMEDIATE and PERMANENT), y/n? " choice echo "" if [ $choice == "y" ]; then rm -fr ${BUILD_DIR} else exit fi fi set -e set -u mkdir -p ${BUILD_DIR} && \ cd ${BUILD_DIR} && \ cmake ${SRC_DIR} "$@" echo "TimescaleDB build system initialized in ${BUILD_DIR}. To compile, do:" echo -e "\033[1mcd ${BUILD_DIR} && make\033[0m" ================================================ FILE: bootstrap.bat ================================================ @echo off :: This bootstrap scripts set up the build environment for TimescaleDB :: Any flags will be passed on to CMake, e.g., :: ./bootstrap.bat -DCMAKE_BUILD_TYPE="Debug" :: Get source directory to build from set ORIG=%0 for %%F in (%ORIG%) do set SRC_DIR=%%~dpF SET BUILD_DIR=./build IF EXIST "%BUILD_DIR%" ( setlocal EnableDelayedExpansion ECHO Build system already initialized in %BUILD_DIR% SET /P resp="Do you want to remove it (this is IMMEDIATE and PERMANENT), y/n? " IF "!resp!" == "y" ( rd /s /q "%BUILD_DIR%" ) ELSE ( ECHO Exiting EXIT ) ) mkdir "%BUILD_DIR%" cd "%BUILD_DIR%" cmake %SRC_DIR% -A x64 %* ECHO --- ECHO TimescaleDB build system initialized in %BUILD_DIR%. ECHO To compile, do: ECHO cmake --build %BUILD_DIR% --config Release ================================================ FILE: cmake/GenerateScripts.cmake ================================================ # Functions to generate downgrade scripts # generate_downgrade_script(<options>) # # Create a downgrade script from a source version to a target version. The # ScriptFiles.cmake manifest is read from the target version's git tag to # determine which files make up the prolog and epilog. # # SOURCE_VERSION <version> # Version to generate downgrade script from. # # TARGET_VERSION <version> # Version to generate downgrade script to. # # OUTPUT_DIRECTORY <dir> # Output directory for script file. Defaults to CMAKE_CURRENT_BINARY_DIR. # # INPUT_DIRECTORY <dir> # Input directory for downgrade files. Defaults to CMAKE_CURRENT_SOURCE_DIR. # # FILES <file> ... # Downgrade-specific files to include between the prolog and epilog. # # The output script concatenates: # 1. Prolog from the target version (header.sql, pre-version-change.sql) # 2. Downgrade files from the source version # 3. Epilog from the target version (function defs, source files, post-update) # # All files undergo @VARIABLE@ template substitution with MODULE_PATHNAME set # to the target version's shared library path. function(generate_downgrade_script) set(options) set(oneValueArgs SOURCE_VERSION TARGET_VERSION OUTPUT_DIRECTORY INPUT_DIRECTORY FILES) cmake_parse_arguments(_downgrade "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) if(NOT _downgrade_OUTPUT_DIRECTORY) set(_downgrade_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) endif() if(NOT _downgrade_INPUT_DIRECTORY) set(_downgrade_INPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) endif() foreach(_downgrade_file ${_downgrade_FILES}) if(NOT EXISTS ${_downgrade_INPUT_DIRECTORY}/${_downgrade_file}) message(FATAL_ERROR "No downgrade file ${_downgrade_file} found!") endif() endforeach() # Fetch manifest with list of files for the prolog and epilog from the target # version. git_versioned_get(VERSION ${_downgrade_TARGET_VERSION} FILES ${CMAKE_SOURCE_DIR}/cmake/ScriptFiles.cmake) # This will include the variables in this scope, but not in the parent scope # so we can read them locally without affecting the parent scope. include( ${CMAKE_BINARY_DIR}/v${_downgrade_TARGET_VERSION}/cmake/ScriptFiles.cmake) set(_downgrade_PRE_FILES "header.sql;${PRE_DOWNGRADE_FILES}") set(_downgrade_POST_FILES "${PRE_INSTALL_FUNCTION_FILES};${SOURCE_FILES}" ${SET_POST_UPDATE_STAGE} ${POST_UPDATE_FILES} ${UNSET_UPDATE_STAGE}) # Fetch epilog from target version. git_versioned_get( VERSION ${_downgrade_TARGET_VERSION} FILES ${_downgrade_POST_FILES} RESULT_FILES _epilog_files IGNORE_ERRORS) # Assemble the full file list: prolog + downgrade files + epilog set(_files) foreach(_downgrade_file ${_downgrade_PRE_FILES}) get_filename_component(_downgrade_filename ${_downgrade_file} NAME) configure_file(${_downgrade_file} ${_downgrade_INPUT_DIRECTORY}/${_downgrade_filename} COPYONLY) list(APPEND _files ${_downgrade_INPUT_DIRECTORY}/${_downgrade_filename}) endforeach() foreach(_downgrade_file ${_downgrade_FILES}) list(APPEND _files ${_downgrade_INPUT_DIRECTORY}/${_downgrade_file}) endforeach() list(APPEND _files ${_epilog_files}) # Set template variables for the target version set(MODULE_PATHNAME "$libdir/timescaledb-${_downgrade_TARGET_VERSION}") set(LOADER_PATHNAME "$libdir/timescaledb") set(PROJECT_VERSION_MOD ${PREVIOUS_VERSION}) # Process all template files and concatenate into the output script set(_script timescaledb--${_downgrade_SOURCE_VERSION}--${_downgrade_TARGET_VERSION}.sql) set(_output ${_downgrade_OUTPUT_DIRECTORY}/${_script}) message(STATUS "Generating script ${_script}") file(WRITE ${_output} "") foreach(_file ${_files}) configure_file(${_file} ${_file}.gen @ONLY) file(READ ${_file}.gen _contents) file(APPEND ${_output} "${_contents}") endforeach() install(FILES ${_output} DESTINATION "${PG_SHAREDIR}/extension") endfunction() ================================================ FILE: cmake/GenerateTestSchedule.cmake ================================================ # generate_test_schedul(<output file> ...) # # A test schedule is generated for the files in TEST_FILES. The test schedule # groups the tests into groups of size GROUP_SIZE, with the exceptions of any # tests mentioned in the list SOLO, which will be in their own test group. # # TEST_FILES <file> ... # # Test files to generate a test schedule for. # # SOLO <test> ... # # Names of tests that should be run as solo tests. Note that these are test # names, not file names. # # GROUP_SIZE # # Size of each group in the test schedule. function(generate_test_schedule OUTPUT_FILE) set(options) set(oneValueArgs GROUP_SIZE) set(multiValueArgs TEST_FILES SOLO) cmake_parse_arguments(_generate "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) # Depending on the configuration we might end up # with an empty schedule here. On older cmake versions < 3.14 # trying to sort an empty list will produce an error, so # we check for empty list here. if(_generate_TEST_FILES) list(SORT _generate_TEST_FILES) endif() file(REMOVE ${OUTPUT_FILE}) # We put the solo tests in the test file first. Note that we do not generate # groups for solo tests that are not in the list of test files. foreach(_solo ${_generate_SOLO}) if("${_solo}.sql" IN_LIST _generate_TEST_FILES) file(APPEND ${OUTPUT_FILE} "test: ${_solo}\n") endif() endforeach() # Generate groups of tests set(_members 0) foreach(_file ${_generate_TEST_FILES}) string(REGEX REPLACE "(.+)\.sql" "\\1" _test ${_file}) if(NOT (_test IN_LIST _generate_SOLO)) if(_members EQUAL 0) file(APPEND ${OUTPUT_FILE} "test: ") endif() file(APPEND ${OUTPUT_FILE} "${_test} ") if(_members LESS _generate_GROUP_SIZE) math(EXPR _members "${_members} + 1") else() set(_members 0) file(APPEND ${OUTPUT_FILE} "\n") endif() endif() endforeach() file(APPEND ${OUTPUT_FILE} "\n") endfunction() ================================================ FILE: cmake/GitCommands.cmake ================================================ # Git helper functions and macros # git_get_versioned(VERSION <version> FILES <path> ...) # # Get files by name relative to the current source directory but read it from # branch VERSION. Note that the version referenced has to exists as a branch, # otherwise you get an error. # # VERSION <version> # # Version to read files from. Any object name is accepted as given in # gitrevision(7), but typically you should use a tag name. # # FILES <path> ... # # Paths to extract. These are relative or absolute paths to files to extract. If # relative, the path names are resolved relative the current source directory. # # RESULT_FILES <var> # # Variable for the list of the full paths of the retrieved files. # # IGNORE_ERRORS # # Ignore errors when fetching file from previous version. This is currently used # to ignore missing files, but we should probably be more selective in the # following versions. function(git_versioned_get) set(options IGNORE_ERRORS) set(oneValueArgs VERSION RESULT_FILES) set(multiValueArgs FILES) cmake_parse_arguments(_git_get "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) execute_process( COMMAND git show-ref --verify --quiet refs/tags/${_git_get_VERSION} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} RESULT_VARIABLE _version_missing) if(_version_missing) message( FATAL_ERROR "Version ${_git_get_VERSION} does not exist in repository.") endif() set(_result_files) foreach(_file ${_git_get_FILES}) # Build full path, if the path is not absolute. if(IS_ABSOLUTE ${_file}) set(_path "${_file}") else() set(_path "${CMAKE_CURRENT_SOURCE_DIR}/${_file}") endif() # Remove root source directory to get path relative to source root. This is # necessary for git-show to work correctly and then use that to generate a # full path for the version. (Yeah, we could use ./ before, but this is # less clear in the log what file is actually fetched.) string(REPLACE ${CMAKE_SOURCE_DIR}/ "" _relpath ${_path}) set(_fullpath "${CMAKE_BINARY_DIR}/v${_git_get_VERSION}/${_relpath}") get_filename_component(_dirname ${_fullpath} DIRECTORY) file(MAKE_DIRECTORY ${_dirname}) # We place the output file next to the final file to avoid cross-device # renames but give it a different name since it is created even if the # command fails. execute_process( COMMAND ${GIT_EXECUTABLE} show ${_git_get_VERSION}:${_relpath} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_FILE "${_fullpath}.part" RESULT_VARIABLE _error ERROR_VARIABLE _message) if(_error) if(_git_get_IGNORE_ERRORS) string(STRIP "${_message}" _stripped_message) message(STATUS "${_stripped_message} (ignored).") else() message(FATAL_ERROR "${_message}") endif() else() file(RENAME "${_fullpath}.part" "${_fullpath}") list(APPEND _result_files "${_fullpath}") endif() endforeach() set(${_git_get_RESULT_FILES} ${_result_files} PARENT_SCOPE) endfunction() ================================================ FILE: cmake/ScriptFiles.cmake ================================================ # File defining all variables used to generate script files. # # This is needed for the downgrade script since files can be added and removed # and it is necessary to get a list of all files available for a specific # version. # # We only care about files that are part of generating the prolog or epilog for # the update scripts, to the actual versioned files are not necessary to put # here. # Source files that define the schemas and tables for our metadata set(PRE_INSTALL_SOURCE_FILES header.sql # Must be first since it sets up the search path pre_install/schemas.sql pre_install/types.pre.sql pre_install/types.functions.sql pre_install/types.post.sql # Must be before tables.sql pre_install/tables.sql pre_install/cache.sql pre_install/insert_data.sql) # Source files that define functions and need to be rerun in update set(PRE_INSTALL_FUNCTION_FILES pre_install/types.functions.sql ) # The rest of the source files defining mostly functions set(SOURCE_FILES hypertable.sql chunk.sql util_time.sql util_internal_table_ddl.sql chunk_constraint.sql partitioning.sql ddl_api.sql ddl_triggers.sql bookend.sql time_bucket.sql version.sql size_utils.sql histogram.sql bgw_scheduler.sql metadata.sql uuidv7.sql views.sql views_experimental.sql gapfill.sql compression.sql maintenance_utils.sql restoring.sql job_api.sql policy_api.sql policy_internal.sql cagg_utils.sql job_stat_history_log_retention.sql osm_api.sql compression_defaults.sql sparse_index.sql) if(ENABLE_DEBUG_UTILS AND CMAKE_BUILD_TYPE MATCHES Debug) list(APPEND SOURCE_FILES debug_build_utils.sql) endif() if(ENABLE_DEBUG_UTILS) list(APPEND SOURCE_FILES debug_utils.sql) endif() if(USE_TELEMETRY) list(APPEND SOURCE_FILES with_telemetry.sql) else() list(APPEND SOURCE_FILES without_telemetry.sql) endif() # Compatibility layer for timescaledb 2.12 for internal functions that got moved into _timescaledb_functions list(APPEND SOURCE_FILES compat.sql) # These files need to be last in the scripts. list(APPEND SOURCE_FILES bgw_startup.sql) if(APACHE_ONLY) list(APPEND SOURCE_FILES comment_apache.sql) else() list(APPEND SOURCE_FILES comment_tsl.sql) endif() # These files should be prepended to update scripts so that they are executed # before anything else during updates set(PRE_UPDATE_FILES updates/pre-version-change.sql updates/pre-update.sql) set(PRE_DOWNGRADE_FILES updates/pre-version-change.sql) # The POST_UPDATE_FILES should be executed as the last part of the update # script. sets state for executing POST_UPDATE_FILES during ALTER EXTENSION set(SET_POST_UPDATE_STAGE updates/set_post_update_stage.sql) set(UNSET_UPDATE_STAGE updates/unset_update_stage.sql) set(POST_UPDATE_FILES updates/post-update.sql) ================================================ FILE: coccinelle/README.md ================================================ This directory contains scripts to check the codebase for defective programming patterns, eg use after free or not freeing resources. Coccinelle is a static code analysis program. It uses a semantic patch language which resembles unified diff output. The semantic patches may inline python or ocaml code for more advanced use cases. A coccinelle patch file consists of multiple blocks. Example block header: ``` @ name @ Expression var1; Expression var2; @@ ``` The block header may contain variable definitions. It may also contain required matches or non-matches in previous blocks. Examples for blocks with required previous matches: ``` @ b2 depends on name @ @@ @ b3 depends on name && !b2 @ @@ ``` Variables inside a block can also reference matches from previous blocks. ``` @ b2 depends on name @ Expression name.var1; @@ ``` var1 inside this block will be the match from the `name` block. - https://github.com/coccinelle/coccinelle - https://coccinelle.gitlabpages.inria.fr/website/ - https://coccinelle.gitlabpages.inria.fr/website/docs/main_grammar.html ================================================ FILE: coccinelle/attrnumbergetattroffset.cocci ================================================ // // find missing `AttrNumberGetAttrOffset` usage // // Postgres has `AttrNumberGetAttrOffset()` macro to proper access Datum array members // (see https://github.com/postgres/postgres/blob/master/src/include/access/attnum.h). // // For example: // `datum[attrno - 1]` should be `datum[AttrNumberGetAttrOffset(attrno)]` // @@ typedef Datum; expression attrno; Datum [] datum; @@ - datum[attrno - 1] + /* use AttrNumberGetAttrOffset() for accessing Datum array members */ + datum[AttrNumberGetAttrOffset(attrno)] @@ typedef bool; expression attrno; bool [] isnull; @@ - isnull[attrno - 1] + /* use AttrNumberGetAttrOffset() for accessing bool array members */ + isnull[AttrNumberGetAttrOffset(attrno)] ================================================ FILE: coccinelle/bms_result.cocci ================================================ // Some bitmap set operations recycle one of the input parameters or return // a reference to a new bitmap set. The following set of rules checks that the // returned reference is not discarded. @rule_1@ expression E1, E2; @@ + /* Result of bms_add_member has to be used */ + E1 = bms_add_member(E1, E2); - bms_add_member(E1, E2); @rule_2@ expression E1, E2; @@ + /* Result of bms_del_member has to be used */ + E1 = bms_del_member(E1, E2); - bms_del_member(E1, E2); @rule_3@ expression E1, E2; @@ + /* Result of bms_add_members has to be used */ + E1 = bms_add_members(E1, E2); - bms_add_members(E1, E2); @rule_4@ expression E1, E2, E3; @@ + /* Result of bms_add_range has to be used */ + E1 = bms_add_range(E1, E2, E3); - bms_add_range(E1, E2, E3); @rule_5@ expression E1, E2; @@ + /* Result of bms_int_members has to be used */ + E1 = bms_int_members(E1, E2); - bms_int_members(E1, E2); @rule_6@ expression E1, E2; @@ + /* Result of bms_del_members has to be used */ + E1 = bms_del_members(E1, E2); - bms_del_members(E1, E2); @rule_7@ expression E1, E2; @@ + /* Result of bms_join has to be used */ + E1 = bms_join(E1, E2); - bms_join(E1, E2); @rule_8@ expression E1, E2; @@ + /* Result of bms_union has to be used */ + E1 = bms_union(E1, E2); - bms_union(E1, E2); @rule_9@ expression E1, E2; @@ + /* Result of bms_intersect has to be used */ + E1 = bms_intersect(E1, E2); - bms_intersect(E1, E2); @rule_10@ expression E1, E2; @@ + /* Result of bms_difference has to be used */ + E1 = bms_difference(E1, E2); - bms_difference(E1, E2); ================================================ FILE: coccinelle/hash_create.cocci ================================================ // find hash_create calls without HASH_CONTEXT flag // // hash_create calls without HASH_CONTEXT flag will create the hash table in // TopMemoryContext which can introduce memory leaks if not intended. We want // to be explicit about the memory context our hash tables live in so we enforce // usage of the flag. @ hash_create @ position p; @@ hash_create@p(...) @safelist@ expression arg1, arg2, arg3; expression w1, w2; position hash_create.p; @@ ( hash_create@p(arg1,arg2,arg3, w1 | HASH_CONTEXT | w2) | hash_create@p(arg1,arg2,arg3, w1 | HASH_CONTEXT) | hash_create@p(arg1,arg2,arg3, HASH_CONTEXT | w2 ) ) @safelist2@ expression res; expression arg1, arg2, arg3; expression flags; position hash_create.p; @@ Assert(flags & HASH_CONTEXT); res = hash_create@p(arg1,arg2,arg3, flags); @ depends on !safelist && !safelist2 @ position hash_create.p; @@ + /* hash_create without HASH_CONTEXT flag */ hash_create@p(...) ================================================ FILE: coccinelle/heap_freetuple.cocci ================================================ // find heap_form_tuple with missing heap_freetuple calls @ heap_form_tuple @ identifier tuple; position p; @@ tuple@p = heap_form_tuple(...); @safelist@ expression tuple; position heap_form_tuple.p; @@ tuple@p = heap_form_tuple(...); ... ( return tuple; | heap_freetuple(tuple) | HeapTupleGetDatum(tuple) ) @depends on !safelist@ expression tuple; position heap_form_tuple.p; @@ + /* heap_form_tuple with missing heap_freetuple call */ tuple@p = heap_form_tuple(...); ================================================ FILE: coccinelle/hypertable_cache.cocci ================================================ // find hypertable uses after cache release @ cache_get @ expression cache, ht, htid, relid, rv, schema, table, flags; position p; @@ ( ht = ts_hypertable_cache_get_entry(cache,...); | ht = ts_hypertable_cache_get_cache_and_entry(relid, flags, &cache); | ht = ts_hypertable_cache_get_entry_rv(cache, rv); | ht = ts_hypertable_cache_get_entry_with_table(cache, relid, schema, table, flags); | ht = ts_hypertable_cache_get_entry_by_id(cache, htid); ) ... ts_cache_release(cache)@p; ... ht @ safelist1 depends on cache_get @ expression cache_get.cache; position cache_get.p; @@ ts_cache_release(cache)@p; ( PG_RETURN_DATUM | aclcheck_error(...) | ereport(ERROR,...) ) // // if variable gets reassigned it's not use after free // @ safelist2 depends on cache_get && !safelist1 @ expression cache_get.cache; expression cache_get.ht; position cache_get.p; @@ ts_cache_release(cache)@p; ... ( ht = ts_hypertable_cache_get_entry(...); | ht = ts_hypertable_cache_get_cache_and_entry(...); ) // print context of matched use after free @ match depends on cache_get && !safelist1 && !safelist2 @ expression cache_get.cache; expression cache_get.ht; position cache_get.p; position m; @@ * ts_cache_release(cache)@p; ... * ht@m ================================================ FILE: coccinelle/hypertable_cache2.cocci ================================================ // find use after free bugs due to premature cache releases // this will find bugs of the following form: // ht = ts_hypertable_cache_get_entry(cache) // dim = hyperspace_get_open_dimension(ht->space, ...) // ts_cache_release(cache) // usage of dim after cache release @ cache_get @ expression cache, htid, relid, rv, schema, table, flags; identifier dim, ht; position p; @@ ( ht = ts_hypertable_cache_get_entry(cache,...) | ht = ts_hypertable_cache_get_cache_and_entry(relid, flags, &cache) | ht = ts_hypertable_cache_get_entry_rv(cache, rv) | ht = ts_hypertable_cache_get_entry_with_table(cache, relid, schema, table, flags) | ht = ts_hypertable_cache_get_entry_by_id(cache, htid) ) ... ( dim = hyperspace_get_open_dimension(ht->space, ...) | dim = ts_hyperspace_get_dimension(ht->space, ...) | dim = ts_hyperspace_get_dimension_by_name(ht->space, ...) | dim = ts_hyperspace_get_mutable_dimension_by_name(ht->space, ...) | dim = ts_hyperspace_get_dimension_by_id(ht->space, ...) ) ... ts_cache_release(cache)@p; ... dim @ m1 depends on cache_get @ expression cache_get.cache; position cache_get.p; @@ ts_cache_release(cache)@p; PG_RETURN_DATUM @ m2 depends on cache_get && !m1 @ expression cache_get.cache; identifier cache_get.dim; position cache_get.p; @@ - ts_cache_release(cache)@p; ... + /* dim used after calling ts_cache_release */ dim ================================================ FILE: coccinelle/mcxt.cocci ================================================ // find MemoryContextSwitchTo missing a context switch back @ MemoryContextSwitch @ local idexpression oldctx; position p; @@ oldctx@p = MemoryContextSwitchTo(...); @safelist@ local idexpression MemoryContextSwitch.oldctx; position MemoryContextSwitch.p; @@ oldctx@p = MemoryContextSwitchTo(...); ... MemoryContextSwitchTo(oldctx) @depends on !safelist@ expression oldctx; position MemoryContextSwitch.p; @@ + /* MemoryContextSwitch missing context switch back */ oldctx@p = MemoryContextSwitchTo(...); ================================================ FILE: coccinelle/namedata.cocci ================================================ // NameData is a fixed-size type of 64 bytes. Using strlcpy to copy data into a // NameData struct can cause problems because any data that follows the initial // null-terminated string will also be part of the data. @rule_var_decl_struct@ symbol NAMEDATALEN; identifier I1, I2; @@ struct I1 { ... - char I2[NAMEDATALEN]; + /* You are declaring a char of length NAMEDATALEN, please consider using NameData instead. */ + NameData I2; ... } @rule_namedata_strlcpy@ identifier I1; expression E1; symbol NAMEDATALEN; @@ - strlcpy(I1, E1, NAMEDATALEN); + /* You are using strlcpy with NAMEDATALEN, please consider using NameData and namestrcpy instead. */ + namestrcpy(I1, E1); @rule_namedata_memcpy@ expression E1, E2; symbol NAMEDATALEN; @@ - memcpy(E1, E2, NAMEDATALEN); + /* You are using memcpy with NAMEDATALEN, please consider using NameData and namestrcpy instead. */ + namestrcpy(E1, E2); @@ typedef NameData; NameData E; @@ - E.data + /* Use NameStr rather than accessing data member directly */ + NameStr(E) @@ NameData *E; @@ - E->data + /* Use NameStr rather than accessing data member directly */ + NameStr(*E) @@ typedef Name; Name E; @@ - E->data + /* Use NameStr rather than accessing data member directly */ + NameStr(*E) ================================================ FILE: coccinelle/oidisvalid.cocci ================================================ // // find comparisons against `InvalidOid` // // Postgres has `OidIsValid()` macro to check if a given Oid is valid or not // (see https://github.com/postgres/postgres/blob/master/src/include/c.h). // // This script look for comparisons to `InvalidOid` and recommend the usage of // `OidIsValid()` instead. // // For example: // `oid != InvalidOid` should be `OidIsValid(oid)` // `oid == InvalidOid` should be `!OidIsValid(oid)` // @@ symbol InvalidOid; expression oid; @@ - (oid != InvalidOid) + /* use OidIsValid() instead of comparing against InvalidOid */ + OidIsValid(oid) @@ symbol InvalidOid; expression oid; @@ - (InvalidOid != oid) + /* use OidIsValid() instead of comparing against InvalidOid */ + OidIsValid(oid) @@ symbol InvalidOid; expression oid; @@ - (oid == InvalidOid) + /* use OidIsValid() instead of comparing against InvalidOid */ + !OidIsValid(oid) @@ symbol InvalidOid; expression oid; @@ - (InvalidOid == oid) + /* use OidIsValid() instead of comparing against InvalidOid */ + !OidIsValid(oid) ================================================ FILE: coverage/CMakeLists.txt ================================================ # CMake targets for generating code coverage reports # # Note that the targets in here are not needed to enable code coverage on # builds. To have builds generate code coverage metadata, one only has to enable # the --coverage option for the compiler and this is done in the top-level # CMakeLists.txt so that the option covers all build targets in the project. # # Given that all dependencies (lcov, gcov, genhtml) are installed, and CMake is # initialized with -DCODECOVERAGE=ON, it should be possible to generate a code # coverage report by running: # # $ make coverage # # The report is generated in REPORT_DIR and can be viewed in a web browser. # If we use clang, prefer the LLVM gcov. The "normal" gcov segfaults with the # coverage info generated by clang, and the LLVM gcov with GCC-generated # coverage gives weird errors like GCOV_TAG_COUNTER_ARCS mismatch. set(GCOV_NAMES "gcov") if(CMAKE_C_COMPILER_ID MATCHES "Clang|AppleClang") string(REGEX MATCH "^[0-9]+" CMAKE_C_COMPILER_VERSION_MAJOR ${CMAKE_C_COMPILER_VERSION}) list(PREPEND GCOV_NAMES "llvm-cov-${CMAKE_C_COMPILER_VERSION_MAJOR}") endif() find_program(GCOV NAMES ${GCOV_NAMES}) # Find lcov for html output find_program(LCOV lcov) if(NOT GCOV) message(STATUS "Install gcov to generate code coverage reports") elseif(NOT LCOV) message(STATUS "Install lcov to generate code coverage reports") else() execute_process( COMMAND ${LCOV} --version OUTPUT_VARIABLE LCOV_VERSION_OUTPUT RESULT_VARIABLE LCOV_VERSION_RESULT OUTPUT_STRIP_TRAILING_WHITESPACE) if(LCOV_VERSION_RESULT EQUAL 0) # You might need to parse LCOV_VERSION_OUTPUT to extract just the version # number For example, if the output is "lcov: LCOV version 1.15", you'd # extract "1.15" string(REGEX MATCH "LCOV version ([0-9]+\\.[0-9]+)" LCOV_VERSION_MATCH "${LCOV_VERSION_OUTPUT}") if(LCOV_VERSION_MATCH) set(LCOV_VERSION ${CMAKE_MATCH_1}) message(STATUS "Using lcov ${LCOV}: ${LCOV_VERSION}") else() message( FATAL_ERROR "Could not parse LCOV version from output: ${LCOV_VERSION_OUTPUT}") endif() else() message( FATAL_ERROR "Failed to get LCOV version. Error code: ${LCOV_VERSION_RESULT}") endif() message(STATUS "Using gcov ${GCOV}:") execute_process(COMMAND ${GCOV} --version) # Final tracefile for code coverage set(OUTPUT_FILE "timescaledb-coverage.info") # Common options for lcov capture commands. We disable branch coverage for # Assert and Ensure, because these are never supposed to be hit and always # have partial coverage. The ereport macro also has some internal conditions # that lead to partial coverage. set(LCOV_CAPTURE_OPTS --rc lcov_branch_coverage=1 --rc "lcov_excl_br_line=Assert\\(|Ensure\\(|ereport\\(" --no-external --base-directory ${CMAKE_SOURCE_DIR} --directory ${CMAKE_BINARY_DIR} --gcov-tool ${CMAKE_CURRENT_BINARY_DIR}/gcov) if(${LCOV_VERSION} VERSION_GREATER_EQUAL "2.0") list(APPEND LCOV_CAPTURE_OPTS --ignore-errors=mismatch,mismatch) endif() # Directory where to generate the HTML report set(REPORT_DIR "lcov-report") # We can't directly use llvm-cov as --gcov-tool, because it has to be called # like "llvm-cov gcov <gcov args>" for that. Thankfully, if its $0 is gcov, it # will understand that we want it to operate like gcov. Just create a symlink. add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/gcov COMMAND ln -s ${GCOV} ${CMAKE_CURRENT_BINARY_DIR}/gcov) # The baseline run needs to run before tests to learn what zero coverage looks # like add_custom_command( OUTPUT ${OUTPUT_FILE}.base COMMENT "Generating code coverage base file" COMMAND ${LCOV} --capture --initial ${LCOV_CAPTURE_OPTS} --output-file ${OUTPUT_FILE}.base DEPENDS timescaledb-tsl timescaledb timescaledb-loader ${CMAKE_CURRENT_BINARY_DIR}/gcov VERBATIM) add_custom_target(coverage_base DEPENDS ${OUTPUT_FILE}.base) # Ensure baseline file is generated before tests add_dependencies(installcheck coverage_base) # The test run needs to run after tests to analyze the test coverage add_custom_command( OUTPUT ${OUTPUT_FILE}.test COMMENT "Generating code coverage test file" COMMAND ${LCOV} --capture ${LCOV_CAPTURE_OPTS} --output-file ${OUTPUT_FILE}.test DEPENDS ${OUTPUT_FILE}.base coverage_base VERBATIM) # Make sure coverage_test runs after tests (installcheck) finish add_custom_target(coverage_test DEPENDS ${OUTPUT_FILE}.test) add_dependencies(installcheck-post-hook coverage_test) # Generate the final coverage file by combining the pre and post test # tracefiles add_custom_command( OUTPUT ${OUTPUT_FILE} COMMENT "Generating final code coverage file" COMMAND ${LCOV} ${LCOV_CAPTURE_OPTS} --add-tracefile ${OUTPUT_FILE}.base --add-tracefile ${OUTPUT_FILE}.test --output-file ${OUTPUT_FILE} DEPENDS ${OUTPUT_FILE}.test coverage_test VERBATIM) add_custom_target(coverage_final DEPENDS ${OUTPUT_FILE}) add_dependencies(coverage_final coverage_test) # Look for genhtml to produce HTML report. This tool is part of the lcov # suite, so should be installed if lcov is installed. Thus, this is an # over-cautious check just in case some distributions break these tools up in # separate packages. find_program(GENHTML genhtml) if(GENHTML) message( STATUS "Generate a code coverage report using the 'coverage' target after tests have run." ) add_custom_command( OUTPUT ${REPORT_DIR}/index.html COMMENT "Generating HTML code coverage report in ${CMAKE_CURRENT_BINARY_DIR}/${REPORT_DIR}" COMMAND ${GENHTML} --prefix ${CMAKE_SOURCE_DIR} --branch-coverage --ignore-errors source --legend --title "TimescaleDB" --output-directory ${REPORT_DIR} ${OUTPUT_FILE} DEPENDS ${OUTPUT_FILE}) add_custom_target(coverage DEPENDS ${REPORT_DIR}/index.html) add_dependencies(coverage coverage_final) add_custom_command( COMMAND echo "Open file://${CMAKE_CURRENT_BINARY_DIR}/${REPORT_DIR}/index.html in a browser to view the report" TARGET coverage POST_BUILD COMMENT) else() message(STATUS "Install genhtml to generate code coverage reports") endif(GENHTML) endif() ================================================ FILE: coverage/README.md ================================================ # Code coverage for TimescaleDB Code coverage can be enabled for TimescaleDB builds by setting the option `-DCODECOVEAGE=ON` when running CMake (it is off by default). This enables the necessary compiler option (`--coverage`) to generate code coverage statistics and should be enough for CI build reports using, e.g., `codecov.io`. In addition, local code coverage reports can be generated with the `lcov` tool, when this tool is installed on the build system. ## Generating local code coverage data files A code coverage report is generated in three steps using `lcov`: 1. A pre test baseline run to learn what zero coverage looks like (the `coverage_base` target). 2. A post test run to learn the test coverage (the `coverage_test` target). 3. A final run to combine the pre test and post test output files into a final data file (the `coverage_final` target). Each of these steps can be run manually using the mentioned targets, but should happen automatically as part of regular build and test steps. Optionally, this process can be extended with a filtering step to ignore certain paths that shouldn't be included in the final report. ## Producing a HTML-based code coverage report Once the complete test suite has run (`installcheck` target), it is possible to produce a HTML-based code coverage report that can be viewed in a web browser. This is automated by the `coverage` target. Thus, the complete steps to produce a code coverage report are: 1. `cmake --build` 2. `cmake --build --target installcheck` 3. `cmake --build --target coverage` ================================================ FILE: docs/BuildSource.md ================================================ ### Building from source #### Building from source (Unix-based systems) If you are building from source for **non-development purposes** (i.e., you want to run TimescaleDB, not submit a patch), you should **always use a release-tagged commit and not build from `main`**. See the Releases tab for the latest release. **Prerequisites**: - A standard PostgreSQL 15, 16, or 17 installation with development environment (header files) (e.g., `postgresql-server-dev-17` package for Linux, Postgres.app for MacOS) - C compiler (e.g., gcc or clang) - [CMake](https://cmake.org/) version 3.15 or greater ```bash git clone git@github.com:timescale/timescaledb.git cd timescaledb # Find the latest release and checkout, e.g. for 2.15.0: git checkout 2.15.0 # Bootstrap the build system ./bootstrap # To build the extension cd build && make # To install make install ``` Note, if you have multiple versions of PostgreSQL installed you can specify the path to `pg_config` that should be used by using `./bootstrap -DPG_CONFIG=/path/to/pg_config`. Please see our [additional configuration instructions](https://docs.timescale.com/self-hosted/latest/install/). #### Building from source (Windows) If you are building from source for **non-development purposes** (i.e., you want to run TimescaleDB, not submit a patch), you should **always use a release-tagged commit and not build from `main`**. See the Releases tab for the latest release. **Prerequisites**: - A standard [PostgreSQL 15, 16, or 17 64-bit installation](https://www.enterprisedb.com/downloads/postgres-postgresql-downloads#windows) - OpenSSL for Windows - Microsoft Visual Studio 2017 with CMake and Git components - OR Visual Studio 2015/2016 with [CMake](https://cmake.org/) version 3.15 or greater and Git - Make sure all relevant binaries are in your PATH: `pg_config` and `cmake` If using Visual Studio 2017 with the CMake and Git components, you should be able to simply clone the repo and open the folder in Visual Studio which will take care of the rest. If you are using an earlier version of Visual Studio, then it can be built in the following way: ```bash git clone git@github.com:timescale/timescaledb.git cd timescaledb # Find the latest release and checkout, e.g. for 2.15.0: git checkout 2.15.0 # Bootstrap the build system bootstrap.bat # To build the extension from command line cmake --build ./build --config Release # To install cmake --build ./build --config Release --target install # Alternatively, build in Visual Studio via its built-in support for # CMake or by opening the generated build/timescaledb.sln solution file. ``` ================================================ FILE: docs/MultiNodeDeprecation.md ================================================ ## Multi-node Deprecation Multi-node support has been deprecated. TimescaleDB 2.13 is the last version that will include multi-node support. Multi-node support in 2.13 is available for PostgreSQL 13, 14 and 15. This decision was not made lightly, and we want to provide a clear understanding of the reasoning behind this change and the path forward. ### Why we are ending multi-node support We began to work on multi-node support in 2018 and released the first version in 2020 to address the growing demand for higher scalability in TimescaleDB deployments. The distributed architecture of multi-node allowed for horizontal scalability of writes and reads to go beyond what a single node could provide. While we added many improvements since the initial launch, the architecture of multi-node came with a number of inherent limitations and challenges that have limited its adoption. Regrettably, only about 1% of current TimescaleDB deployments utilize multi-node, and the complexity involved in maintaining this feature has become a significant obstacle. It’s not an isolated feature that can be kept in the product with very little effort since adding new features required extra development and testing to ensure they also worked for multi-node. As we've evolved our single-node product and expanded our cloud offering to serve thousands of customers, we've identified more efficient solutions to meet the scalability needs of our users. First, we’ve been able and will continue to make big improvements in the write and read performance of single-node. We’ve scaled a single-node deployment to process 2 million inserts per second and have seen performance improvements of 10x for common queries. You can read a summary of the latest query performance improvements [here](https://www.timescale.com/blog/8-performance-improvements-in-recent-timescaledb-releases-for-faster-query-analytics/). And second, we are leveraging cloud technologies that have become very mature to provide higher scalability in a more accessible way. For example, our cloud offering uses object storage to deliver virtually [infinite storage capacity at a very low cost](https://www.timescale.com/blog/scaling-postgresql-for-cheap-introducing-tiered-storage-in-timescale/). For those reasons, we’ve decided to focus our efforts on improving single-node and leveraging cloud technologies to solve for high scalability and as a result we’ve ended support for multi-node. ### What this means for you We understand that this change may raise questions, and we are committed to supporting you through the transition. For current TimescaleDB multi-node users, please refer to our [migration documentation](https://docs.timescale.com/migrate/latest/multi-node-to-timescale-service/) for a step-by-step guide to transition to a single-node configuration. Alternatively, you can continue to use multi-node up to version 2.13. However, please be aware that future versions will no longer include this functionality. If you have any questions or feedback, you can share them in the #multi-node channel in our [community Slack](https://slack.timescale.com/). ================================================ FILE: docs/StyleGuide.md ================================================ # TimescaleDB code style guide Source code should follow the [PostgreSQL coding conventions](https://www.postgresql.org/docs/current/static/source.html). This includes [source formatting](https://www.postgresql.org/docs/current/static/source-format.html) with 4 column tab spacing and layout rules according to the BSD style. ## SQL and PL/pgSQL style There is no official SQL or PL/pgSQL style guide for PostgreSQL that we are aware of, apart from the general spacing and layout rules above. For now, try to follow the style of the surrounding code when making modifications. We might develop more stringent guidelines in the future. ## Error messages Error messages in TimescaleDB should obey the PostgreSQL [error message style guide](https://www.postgresql.org/docs/current/static/error-style-guide.html). ## C style While the PostgreSQL project mandates that [C code adheres to the C89 standard](https://www.postgresql.org/docs/current/static/source-conventions.html) with some exceptions, we've chosen to allow many C99 features for the clarity and convenience they provide. PostgreSQL sticks with C89 mostly for compatibility with many older compilers and platforms, such as Visual Studio on Windows. Visual Studio should support most of C99 nowadays, however. We might revisit this decision in the future if it turns out to be a problem for important platforms. Unfortunately, PostgreSQL does not have a consistent style for naming of functions, variables and types. This [mailing-list thread](https://www.postgresql.org/message-id/1221125165.5637.12.camel%40abbas-laptop) elaborates on the situation. Instead, we've tried our best to develop a consistent style that roughly follows the style of the PostgreSQL source code. ### Declarations before code C99 supports having code before a declaration, similar to C++, and we also support this in the code. For instance, the following code follows the guidelines: ```C void some_function() { prepare(); int result = fetch(); ... } ``` ### Function and variable names For clarity and consistency, we've chosen to go with lowercase under-score separated names for functions and variables. For instance, this piece of code is correct: ```C static void my_function_name(int my_parameter) { int my_variable; ... } ``` while this one is wrong: ```C static void MyFunctionName(int myParameter) { int myVariable; ... } ``` ### Type declarations New composite/aggregate types should be typedef'd and use UpperCamelCase naming. For instance, the following is correct: ```C typedef struct MyType { ... } MyType; ``` while the following is wrong: ```C typedef struct my_type { ... } my_type; ``` ### Modular code and namespacing When possible, code should be grouped into logical modules. Such modules typically resemble classes in object-oriented programming (OOP) languages and should use namespaced function and variable names that have the module name as prefix. TimescaleDB's [Cache](../src/cache.c) implementation is a good example of such a module where one would use ```C void cache_initialize(Cache *c) { ... } ``` rather than ```C void initialize_cache(Cache *c) { } ``` ### Object-orientated programming style Even though C is not an object-oriented programming language, it is fairly straight-forward to write C code with an OOP flavor. While we do not mandate that C code has an OOP flavor, we recommend it when it makes sense (e.g., to achieve modularity and code reuse). For example, TimescaleDB's [cache.c](../src/cache.c) module can be seen as a _base class_ with multiple derived _subclasses_, such as [hypertable_cache.c](../src/hypertable_cache.c) and [chunk_cache.c](../src/chunk_cache.c). Here's another example of subclassing using shapes: ```C typedef struct Shape { int color; void (*draw)(Shape *, Canvas *); } Shape; void shape_draw(Shape *shape) { /* open canvas for drawing */ Canvas *c = canvas_open(); /* other common shape code */ ... shape->draw(shape, c); canvas_close(c); } typedef struct Circle { Shape shape; float diameter; } Circle; Circle blue_circle = { .shape = { .color = BLUE, .draw = circle_draw, }, .diameter = 10.1, }; void circle_draw(Shape *shape, Canvas *canvas) { Circle *circle = (Circle *) shape; /* draw circle */ ... } ``` There are a couple of noteworthy take-aways from this example. * Non-static "member" methods should take a pointer to the object as first argument and be namespaced as described above. * Derived modules can expand the original type using struct embedding, in which case the "superclass" should be the first member of the "subclass", so that one can easily upcast a `Circle` to a `Shape` or, vice-versa, downcast as in `circle_draw()` above. * C++-style virtual functions can be implemented with functions pointers. Good use cases for such functions are module-specific initialization and cleanup, or function overriding in a subclass. ### Other noteworthy recommendations * Prefer static allocation and/or stack-allocated variables over heap/dynamic allocation when possible. Fortunately, PostgreSQL has a nice `MemoryContext` implementation that helps with heap allocation when needed. * Try to minimize the number of included headers in source files, especially when header files include other headers. Avoid circular header dependencies by predeclaring types (or use struct pointers). For a general guide to writing C functions in PostgreSQL, you can review the section on [C-language functions](https://www.postgresql.org/docs/current/static/xfunc-c.html) in the PostgreSQL documentation. ## Tools and editors We require running C code through clang-format before submitting a PR. This will ensure your code is properly formatted according to our style (which is similar to the PostgreSQL style but implement in clang-format). You can run clang-format on all of the TimescaleDB code using `make format` if you have clang-format (version >= 7) or docker installed. The following [Wiki post](https://wiki.postgresql.org/wiki/Developer_FAQ#What.27s_the_formatting_style_used_in_PostgreSQL_source_code.3F) contains links to style configuration files for various editors. ================================================ FILE: docs/getting-started/README.md ================================================ # TimescaleDB Examples This directory contains complete, standalone examples to help you get started with TimescaleDB using real-world use cases. Each example includes sample data and analytical queries to showcase TimescaleDB's capabilities. ## Available Examples ### [NYC Taxi Data](nyc-taxi/) Analyze New York City taxi trip data to understand transportation patterns. Great for learning about location-based analytics and high-volume time-series data. **What you'll learn:** - Handling high-cardinality data (locations, cab types) - Time-series aggregations with `time_bucket()` - Segmentation strategies for optimal compression - Revenue and usage pattern analysis **Use cases:** Transportation analytics, ride-sharing platforms, logistics optimization, urban planning --- ### [Financial Market Data](financial-ticks/) Work with financial tick and candlestick data for market analysis. Ideal for understanding high-frequency time-series data and multi-timeframe aggregations. **What you'll learn:** - OHLCV (Open, High, Low, Close, Volume) data modeling - Creating candlestick aggregations at multiple intervals - Continuous aggregates for different timeframes (1min, 5min, 1hour) - Real-time market analysis queries **Use cases:** Trading platforms, market data analysis, portfolio analytics, algorithmic trading --- ### [Application Events with UUIDv7](events-uuidv7/) Track and analyze application events using modern UUIDv7 identifiers. Perfect for understanding event-driven analytics and user behavior tracking. **What you'll learn:** - Using UUIDv7 for time-ordered unique identifiers - Efficient time-range queries with `to_uuidv7_boundary()` - Session tracking and user analytics - Event funnel and conversion analysis **Use cases:** Application monitoring, user behavior analytics, audit logging, event-driven architectures --- ## Workshops ### [AI Workshop: EV Charging Station Analysis](https://github.com/timescale/TigerData-Workshops/tree/main/AI-Workshop) Integrate PostgreSQL with AI capabilities for managing and analyzing EV charging station data. This workshop demonstrates how to combine time-series data with vector search and AI features. **What you'll learn:** - Integrating TimescaleDB with AI and vector extensions - Managing EV charging station time-series data - Vector search and similarity queries - AI-powered analytics on time-series data **Use cases:** EV infrastructure management, smart grid analytics, energy optimization, predictive maintenance --- ### [Time-Series Workshop: Financial Data Analysis](https://github.com/timescale/TigerData-Workshops/tree/main/TimeSeries-Workshop-Finance/) Work with cryptocurrency tick data and create candlestick charts. Learn advanced time-series analysis techniques for financial markets. **What you'll learn:** - Working with high-frequency cryptocurrency tick data - Creating candlestick aggregations and visualizations - Advanced time-series analysis patterns - Multi-timeframe financial analytics **Use cases:** Cryptocurrency trading, market analysis, financial data visualization, algorithmic trading --- ## How to Use These Examples Each example is completely standalone and self-contained. You can use any example as your starting point: 1. **Choose an example** that matches your use case or interests 2. **Navigate to the example directory** and read the README 3. **Follow the step-by-step guide** - each example includes: - Complete schema definition (SQL) - Sample data (CSV) included in the repository - Data loading instructions (both direct to columnstore and standard approaches) - Sample analytical queries (SQL) - Detailed explanations of what's happening 4. **Run the queries** and see TimescaleDB's columnstore performance in action 5. **Adapt to your needs** - use these patterns for your own data ## Quick Start Path **New to TimescaleDB?** We recommend this path: 1. Start with the [Quick Start Guide](../../README.md#quick-start-with-timescaledb) (10 minutes) 2. Try the [NYC Taxi](nyc-taxi/) example (20 minutes) 3. Explore other examples based on your use case 4. Ready for your data? See [Your Own Data Guide](../your-own-data/) ## Common Patterns Across All Examples All examples demonstrate these TimescaleDB features: - **Hypertables** - Automatic time-based partitioning with `tsdb.hypertable` - **Columnstore** - Hybrid row-columnar storage with `tsdb.enable_columnstore=true` - **Direct to Columnstore** - Instant analytical performance with `enable_direct_compress_copy` - **time_bucket()** - Powerful time-series aggregation function - **Compression** - Automatic 90%+ data compression - **Optimal indexing** - Best practices for time-series indexes ## Prerequisites All examples require: - Docker (for running TimescaleDB) - A PostgreSQL client (`psql` recommended) - 10-30 minutes of your time Each example works with the same Docker setup. You have two options: **Option 1: One-line install (Recommended)** ```sh curl -sL https://tsdb.co/start-local | sh ``` This command: - Downloads and starts TimescaleDB (if not already downloaded) - Exposes PostgreSQL on port **6543** (a non-standard port to avoid conflicts with other PostgreSQL instances on port 5432) - Automatically tunes settings for your environment using timescaledb-tune - Sets up a persistent data volume **Option 2: Manual Docker command** ```bash docker run -d --name timescaledb \ -p 6543:5432 \ -e POSTGRES_PASSWORD=password \ timescale/timescaledb-ha:pg18 ``` **Note:** We use port **6543** (mapped to container port 5432) to avoid conflicts if you have other PostgreSQL instances running on the standard port 5432. ## Example Selection Guide **Choose your example based on your use case:** | Your Use Case | Recommended Example | |--------------|---------------------| | Location-based services, GPS tracking | [NYC Taxi](nyc-taxi/) | | Financial trading, market data | [Financial Ticks](financial-ticks/) | | Application logs, user events | [Events with UUIDv7](events-uuidv7/) | | Cryptocurrency, volatile markets | [Time-Series Workshop: Financial Data Analysis](https://github.com/timescale/TigerData-Workshops/tree/main/TimeSeries-Workshop-Finance/) | | AI/ML with time-series data | [AI Workshop: EV Charging Station Analysis](https://github.com/timescale/TigerData-Workshops/tree/main/AI-Workshop) | | General time-series analytics | Start with [NYC Taxi](nyc-taxi/) | ## What Makes These Examples Different? - **Completely standalone** - No dependencies between examples - **Sample data included** - CSV files in the repo, ready to load - **Production-ready patterns** - Real-world schema designs and query patterns - **Instant performance** - Direct to columnstore examples for immediate results - **Copy-paste friendly** - All code works as-is in `psql` - **Explained thoroughly** - Comments and documentation explain the "why" ## Next Steps After trying these examples: 1. **Bring your own data** - See [Your Own Data Guide](../your-own-data/) 2. **Learn advanced features** - Explore [TimescaleDB Documentation](https://docs.timescale.com) 3. **Production deployment** - Check out [Timescale Cloud](https://www.timescale.com/cloud) for managed hosting 4. **Join the community** - Get help in [Timescale Community Forums](https://www.timescale.com/forum) ## Contributing Found an issue or want to improve an example? Contributions are welcome! Please open an issue or pull request on our [GitHub repository](https://github.com/timescale/timescaledb). --- **Ready to start?** Pick an example above and dive in! ================================================ FILE: docs/getting-started/events-uuidv7/README.md ================================================ # Application Events with UUIDv7 Example Get started with TimescaleDB using application event data leveraging UUIDv7 identifiers. This example demonstrates how to handle event logging and analytics using time-embedded UUIDs for partitioning—**no separate timestamp column needed!** ## What You'll Learn - Using UUIDv7 for time-ordered unique identifiers - Efficient time-range queries with `to_uuidv7_boundary()` - Session tracking and user analytics - Event funnel and conversion analysis ## Prerequisites - Docker installed - `psql` PostgreSQL client - 15-20 minutes ## Quick Start ### Step 1: Start TimescaleDB You have two options to start TimescaleDB: #### Option 1: One-line install (Recommended) The easiest way to get started: > **Important:** This script is intended for local development and testing only. Do **not** use it for production deployments. For production-ready installation options, see the [TimescaleDB installation guide](https://docs.timescale.com/self-hosted/latest/install/). **Linux/Mac:** ```sh curl -sL https://tsdb.co/start-local | sh ``` This command: - Downloads and starts TimescaleDB (if not already downloaded) - Exposes PostgreSQL on port **6543** (a non-standard port to avoid conflicts with other PostgreSQL instances on port 5432) - Automatically tunes settings for your environment using timescaledb-tune - Sets up a persistent data volume #### Option 2: Manual Docker command also used for Windows Alternatively, you can run TimescaleDB directly with Docker: ```bash docker run -d --name timescaledb \ -p 6543:5432 \ -e POSTGRES_PASSWORD=password \ timescale/timescaledb-ha:pg18 ``` **Note:** We use port **6543** (mapped to container port 5432) to avoid conflicts if you have other PostgreSQL instances running on the standard port 5432. Wait about 1-2 minutes for TimescaleDB to download & initialize. ### Step 2: Connect to TimescaleDB Connect using `psql`: ```bash psql -h localhost -p 6543 -U postgres # When prompted, enter password: password ``` You should see the PostgreSQL prompt. Verify TimescaleDB is installed: ```sql SELECT extname, extversion FROM pg_extension WHERE extname = 'timescaledb'; ``` Expected output: ``` extname | extversion -------------+------------ timescaledb | 2.x.x ``` **Prefer a GUI?** If you'd rather use a graphical tool instead of the command line, you can download [pgAdmin](https://www.pgadmin.org/download/) and connect to TimescaleDB using the same connection details (host: `localhost`, port: `6543`, user: `postgres`, password: `password`). ### Step 3: Create the Schema Create the optimized hypertable by running this SQL in your `psql` session: ```sql -- Create the app_events table with UUIDv7 partitioning -- Note: No separate timestamp column needed - the timestamp is embedded in the UUIDv7! -- Note: No PRIMARY KEY to allow direct compress during COPY CREATE TABLE IF NOT EXISTS app_events ( event_id UUID NOT NULL, user_id UUID NOT NULL, session_id UUID NOT NULL, event_type TEXT NOT NULL, event_name TEXT, device_type TEXT, country_code TEXT, category TEXT, page_path TEXT, referrer TEXT, viewport_width INTEGER, element_id TEXT, position_x INTEGER, position_y INTEGER, product_id TEXT, quantity INTEGER, revenue_cents INTEGER ) WITH ( tsdb.hypertable, tsdb.partition_column = 'event_id', tsdb.segmentby = 'user_id' ); -- Create index on event_id for lookups (not unique to allow direct compress) CREATE INDEX idx_app_events_event_id ON app_events(event_id); ``` This creates an `app_events` table with: - UUIDv7 partitioning on `event_id` (time is embedded in the UUID) - Segmentation by `user_id` for optimal compression - No separate timestamp column needed! ### Step 4: Load Sample Data First, download and decompress the sample data: ```bash # Download the sample data wget https://assets.timescale.com/timescaledb-datasets/events_uuid.csv.gz # Decompress the CSV file gunzip events_uuid.csv.gz # This will create events_uuid.csv ready for loading ``` We provide two approaches for loading data. Choose based on your needs: #### Option A: Direct to Columnstore (Recommended - Instant Performance) This approach writes data directly to the columnstore, bypassing the rowstore entirely. You get instant analytical performance. **From command line:** ```bash psql -h localhost -p 6543 -U postgres \ -v ON_ERROR_STOP=1 \ -c "SET timescaledb.enable_direct_compress_copy = on; COPY app_events FROM STDIN WITH (FORMAT csv, HEADER true);" \ < events_uuid.csv ``` This command reads the CSV file from your local filesystem and pipes it to PostgreSQL, which loads it directly into the columnstore. **Verify data loaded:** ```sql SELECT COUNT(*) FROM app_events; ``` #### Option B: Standard COPY (Fallback) This approach loads data into the rowstore first. Data will be converted to the columnstore by a background policy (12-24 hours) for faster querying. **From command line:** ```bash psql -h localhost -p 6543 -U postgres \ -v ON_ERROR_STOP=1 \ -c "COPY app_events FROM STDIN WITH (FORMAT csv, HEADER true);" \ < events_uuid.csv ``` **Verify data loaded:** ```sql SELECT COUNT(*) FROM app_events; ``` **Manually convert to columnstore (Optional):** If you loaded data using standard copy a background process will convert your rowstore data to the columnstore in 12-24 hours, you can manually convert it immediately to get the best query performance: ```sql DO $$ DECLARE ch TEXT; BEGIN FOR ch IN SELECT show_chunks('app_events') LOOP CALL convert_to_columnstore(ch); END LOOP; END $$; ``` ### Step 5: Run Sample Queries Now let's explore the data with some analytical queries. Run these in your `psql` session: **Query 1: Efficient Time Range Query (Chunk Pruning)** ```sql -- ✅ CORRECT: Uses boundary function for chunk exclusion \timing on SELECT COUNT(*), event_type FROM app_events WHERE event_id >= to_uuidv7_boundary(now() - interval '7 days') GROUP BY event_type; ``` **Why this is fast:** The `to_uuidv7_boundary()` function creates a UUID boundary value that TimescaleDB can use to exclude entire chunks without scanning them. **Query 2: Inefficient Query (Anti-Pattern)** ```sql -- ❌ WRONG: Scans ALL chunks, extracts timestamp from every row SELECT COUNT(*), event_type FROM app_events WHERE uuid_timestamp(event_id) >= now() - interval '7 days' GROUP BY event_type; ``` **Why this is slow:** The `uuid_timestamp()` function must be evaluated for every row, preventing chunk exclusion. **Query 3: SkipScan on Single Column (Distinct Users)** ```sql -- Demonstrates SkipScan optimization: uses compression index to skip repeated values \timing on SELECT DISTINCT ON (user_id) user_id, event_type, uuid_timestamp(event_id) as event_time FROM app_events WHERE event_id >= to_uuidv7_boundary(now() - interval '30 days') ORDER BY user_id, event_id DESC LIMIT 50; ``` **Why this uses SkipScan:** Since `user_id` is the segmentby column, TimescaleDB automatically creates a compression index on it. SkipScan can jump directly to the next unique `user_id` value instead of scanning all rows. The WHERE clause ensures chunk exclusion (only scans chunks with recent data), making it even faster. **Verify SkipScan is used:** Check the query plan with `EXPLAIN` - you should see `Custom Scan (SkipScan)` on the compressed chunks instead of a sequential scan. **Query 4: Funnel Analysis** ```sql WITH funnel AS ( SELECT user_id, session_id, MAX(CASE WHEN event_type = 'page_view' THEN 1 ELSE 0 END) as viewed, MAX(CASE WHEN event_type = 'add_to_cart' THEN 1 ELSE 0 END) as added_to_cart, MAX(CASE WHEN event_type = 'purchase' THEN 1 ELSE 0 END) as purchased FROM app_events WHERE event_id >= to_uuidv7_boundary(now() - interval '30 days') GROUP BY user_id, session_id ) SELECT SUM(viewed) as sessions_with_view, SUM(added_to_cart) as sessions_with_cart, SUM(purchased) as sessions_with_purchase, ROUND(100.0 * SUM(added_to_cart) / NULLIF(SUM(viewed), 0), 2) as view_to_cart_pct, ROUND(100.0 * SUM(purchased) / NULLIF(SUM(added_to_cart), 0), 2) as cart_to_purchase_pct FROM funnel; ``` **Query 5: Revenue by Country (Last 30 Days)** ```sql SELECT country_code, COUNT(*) as purchase_count, SUM(revenue_cents) / 100.0 as total_revenue, ROUND(AVG(revenue_cents) / 100.0, 2) as avg_order_value FROM app_events WHERE event_id >= to_uuidv7_boundary(now() - interval '30 days') AND event_type = 'purchase' GROUP BY country_code ORDER BY total_revenue DESC LIMIT 10; ``` ## What's Happening Behind the Scenes? ### UUIDv7 Partitioning When you create a table with `tsdb.partition_column = 'event_id'` where `event_id` is a UUIDv7: - TimescaleDB automatically partitions your data by the time-embedded UUID - **No separate timestamp column needed** - the timestamp is embedded in the UUID itself - Chunk exclusion works with `to_uuidv7_boundary()` for efficient time-range queries - Vectorized UUID compression provides 30% storage savings and 2x query performance ### Columnstore Compression With `tsdb.enable_columnstore=true`: - Data is stored in a hybrid row-columnar format - Analytical queries only scan the columns they need (massive speedup) - Typical compression ratios: 90%+ for time-series data - Compression happens transparently - no changes to your queries ### Direct to Columnstore When you use `SET timescaledb.enable_direct_compress_copy = on`: - Data loads directly into compressed columnstore format - Bypasses the rowstore entirely - Instant analytical performance - no waiting for background compression ### Segmentation The `tsdb.segmentby='user_id'` setting: - Groups data by user within each chunk - Improves compression ratios (similar data together) - Speeds up queries that filter by user_id - Better for user-based analytics ### UUIDv7 Functions TimescaleDB provides comprehensive UUIDv7 functionality across **all supported PostgreSQL versions** (including PostgreSQL 15, 16, and 17), while PostgreSQL only provides UUIDv7 support in PostgreSQL 18. For complete documentation on all available UUIDv7 functions, see the [UUIDv7 Functions API Reference](https://www.tigerdata.com/docs/api/latest/uuid-functions). ## Continuous Aggregates (Advanced) For real-time dashboards, you can create continuous aggregates that automatically update: ```sql -- Create a continuous aggregate for hourly event statistics CREATE MATERIALIZED VIEW app_events_hourly WITH (timescaledb.continuous) AS SELECT time_bucket('1 hour', uuid_timestamp(event_id)) AS hour, event_type, COUNT(*) as event_count, COUNT(DISTINCT user_id) as unique_users, SUM(revenue_cents) / 100.0 as total_revenue FROM app_events GROUP BY hour, event_type; -- Add a refresh policy to keep it updated SELECT add_continuous_aggregate_policy('app_events_hourly', start_offset => INTERVAL '2 hours', end_offset => INTERVAL '1 hour', schedule_interval => INTERVAL '1 hour'); ``` Now you can query `app_events_hourly` for instant results on pre-aggregated data. ## Troubleshooting ### Data didn't load - Check the CSV file path is correct - Ensure the CSV header matches the schema columns - Try loading a few rows first to test: `LIMIT 10` in your data file ### Direct to columnstore not working - Verify TimescaleDB version 2.24+: `SELECT extversion FROM pg_extension WHERE extname = 'timescaledb';` - Ensure you ran `SET timescaledb.enable_direct_compress_copy = on;` in the same session - Check for error messages in the output ### Queries seem slow - Verify you're using `to_uuidv7_boundary()` for time-range queries (not `uuid_timestamp()`) - Check if data is compressed: `SELECT * FROM timescaledb_information.chunks WHERE hypertable_name = 'app_events';` - Ensure chunk exclusion is working: Use `EXPLAIN ANALYZE` to see chunk pruning ### UUIDv7 functions not found - Verify TimescaleDB version supports UUIDv7: `SELECT extversion FROM pg_extension WHERE extname = 'timescaledb';` - UUIDv7 support requires TimescaleDB 2.24+ with the uuidv7 extension enabled ## Use Cases This Application Events example demonstrates patterns applicable to: - **SaaS analytics** - User behavior tracking, feature usage, conversion funnels - **E-commerce** - Shopping cart analysis, purchase patterns, product recommendations - **Application monitoring** - Error tracking, performance metrics, user sessions - **Audit logging** - Security events, compliance tracking, change history - **Event-driven architectures** - Microservices event sourcing, message queues - **A/B testing** - Experiment tracking, variant analysis, statistical significance ## Files in This Example - [`README.md`](README.md) - This file - Sample data files (to be added) ## Clean Up When you're done experimenting: #### If you used the one-line install: ```bash # Stop the container docker stop timescaledb-ha-pg18-quickstart # Remove the container docker rm timescaledb-ha-pg18-quickstart # Remove the persistent data volume docker volume rm timescaledb_data # (Optional) Remove the Docker image docker rmi timescale/timescaledb-ha:pg18 ``` #### If you used the manual Docker command: ```bash # Stop the container docker stop timescaledb # Remove the container docker rm timescaledb # (Optional) Remove the Docker image docker rmi timescale/timescaledb-ha:pg18 ``` **Note:** If you created a named volume with the manual Docker command, you can remove it with `docker volume rm <volume_name>`. --- **Questions?** Check out [TimescaleDB Documentation](https://docs.timescale.com) or the [TimescaleDB Community Forums](https://www.timescale.com/forum). ================================================ FILE: docs/getting-started/financial-ticks/README.md ================================================ # Financial Market Data Example This example will demonstrate financial tick and candlestick data analysis with TimescaleDB. The datasets corresponds to the stocks listed in the [S&P 500 index](https://en.wikipedia.org/wiki/List_of_S%26P_500_companies), with fictive prices and movements. The tick cadence is per second, over three business days, containing approx 35 million records in total and 503 tickers. ## Use Cases Trading platforms, market data analysis, portfolio analytics, algorithmic trading ## What You'll Learn - OHLCV (Open, High, Low, Close, Volume) data modeling - Candlestick aggregations at multiple intervals - Continuous aggregates for different timeframes (1min, 5min, 1hour) - Real-time market analysis queries ## Dataset Preview * timestamp * ticker * price * price delta * change percentage * volume ```csv 2025-11-12 14:30:00+00:00,NVDA,38.25,0.25,0.65359,5095712 2025-11-12 14:30:00+00:00,AAPL,152.03,0.03,0.01973,6466554 2025-11-12 14:30:00+00:00,MSFT,129.23,0.23,0.17798,4417848 2025-11-12 14:30:00+00:00,GOOG,174.93,-0.07,-0.04002,19602229 2025-11-12 14:30:00+00:00,GOOGL,71.21,0.21,0.2949,3149482 2025-11-12 14:30:00+00:00,AMZN,95.95,-0.05,-0.05211,12150474 2025-11-12 14:30:00+00:00,AVGO,196.15,0.15,0.07647,6166047 2025-11-12 14:30:00+00:00,META,133.22,0.22,0.16514,12230004 2025-11-12 14:30:00+00:00,TSLA,82.0,0.0,0.0,10298937 2025-11-12 14:30:00+00:00,BRK.B,56.84,-0.16,-0.28149,10980047 ``` ## Prerequisites - Docker installed - `psql` PostgreSQL client - 15-20 minutes ## Quick Start First download the dataset: ```sh curl -L https://assets.timescale.com/timescaledb-datasets/sp500_stock_prices_3d_1s.tar.gz | tar -xzf - ``` ### Step 1: Start TimescaleDB You have two options to start TimescaleDB: #### Option 1: One-line install (Recommended) The easiest way to get started: > **Important:** This script is intended for local development and testing only. Do **not** use it for production deployments. For production-ready installation options, see the [TimescaleDB installation guide](https://docs.timescale.com/self-hosted/latest/install/). **Linux/Mac:** ```sh curl -sL https://tsdb.co/start-local | sh ``` This command: - Downloads and starts TimescaleDB (if not already downloaded) - Exposes PostgreSQL on port **6543** (a non-standard port to avoid conflicts with other PostgreSQL instances on port 5432) - Automatically tunes settings for your environment using timescaledb-tune - Sets up a persistent data volume #### Option 2: Manual Docker command also used for Windows Alternatively, you can run TimescaleDB directly with Docker: ```bash docker run -d --name timescaledb \ -p 6543:5432 \ -e POSTGRES_PASSWORD=password \ timescale/timescaledb-ha:pg18 ``` **Note:** We use port **6543** (mapped to container port 5432) to avoid conflicts if you have other PostgreSQL instances running on the standard port 5432. Wait about 1-2 minutes for TimescaleDB to download & initialize. ### Step 2: Connect to TimescaleDB Connect using `psql`: ```sh psql "postgres://postgres:password@localhost:6543/postgres" ``` You should see the PostgreSQL prompt. Verify TimescaleDB is installed: ```sql SELECT extname, extversion FROM pg_extension WHERE extname = 'timescaledb'; ``` Expected output: ``` extname | extversion -------------+------------ timescaledb | 2.x.x ``` **Prefer a GUI?** If you'd rather use a graphical tool instead of the command line, you can download [pgAdmin](https://www.pgadmin.org/download/) and connect to TimescaleDB using the same connection details (host: `localhost`, port: `6543`, user: `postgres`, password: `password`). ### Step 3: Create the Schema Create the optimized hypertable by running this SQL in your `psql` session: ```sql -- Create the stock_prices table with the column ts as partitioning -- Note: for optimized query performance grouping on ticker, we select this column to segment by CREATE TABLE stock_prices ( ts TIMESTAMPTZ NOT NULL, ticker TEXT NOT NULL, price DOUBLE PRECISION NOT NULL, change_delta DOUBLE PRECISION NOT NULL, change_percentage DOUBLE PRECISION NOT NULL, volume BIGINT NOT NULL CHECK (volume >= 0) ) WITH ( timescaledb.hypertable, timescaledb.segmentby='ticker' ); ``` This creates a `stock_prices` table with: - partitioned by timestamp on column `ts` - Segmentation by `ticker` for optimal compression and query performance ### Step 4: Load Sample Data into TimescaleDB This approach writes data directly to the columnstore, bypassing the rowstore entirely. You get instant analytical performance. **From psql:** ```sql -- Enable direct to columnstore for this session SET timescaledb.enable_direct_compress_copy = on; -- Load data directly into columnstore (if you have a CSV file) \COPY stock_prices FROM 'sp500_stock_prices_3d_1s.csv' WITH (FORMAT csv, HEADER true); -- Verify data loaded SELECT COUNT(*) FROM stock_prices; ``` ### Step 5: Run Sample Queries Now let's explore the data with some analytical queries. Run these in your `psql` session: ```sql -- Activate time measuring \timing on ``` **Query 1: OHLCV per hour of AAPL** Aggregating raw 1 second tick data into 15-minute "candlesticks" (Open, High, Low, Close, Volume) of the ticker `AAPL`. ```sql SELECT time_bucket('1 hour', ts) AS hour_bucket, ticker, FIRST(price, ts) AS open_price, MAX(price) AS high_price, MIN(price) AS low_price, LAST(price, ts) AS close_price, AVG(price) AS avg_price, SUM(volume) AS sum_volume FROM stock_prices WHERE ticker = 'AAPL' GROUP BY hour_bucket, ticker ORDER BY hour_bucket DESC; ``` **Query 2: Trend Analysis: Simple Moving Average (SMA) of MSFT** Calculating a "smoothing" line to see trends over noise for the ticker `MSFT` over 4 hours. ```sql WITH candles AS ( SELECT time_bucket('1 hour', ts) AS bucket, ticker, LAST(price, ts) AS close_price FROM stock_prices WHERE ticker = 'MSFT' GROUP BY bucket, ticker ) SELECT bucket, ticker, close_price, AVG(close_price) OVER ( PARTITION BY ticker ORDER BY bucket ROWS BETWEEN 3 PRECEDING AND CURRENT ROW ) AS sma_4hours FROM candles ORDER BY bucket DESC; ``` **Query 3: Hour-over-Hour Return** Comparing the current price to the price exactly one hour ago to calculate percentage growth. ```sql WITH hourly_close AS ( SELECT time_bucket('1 hour', ts) AS bucket, ticker, LAST(price, ts) AS closing_price FROM stock_prices GROUP BY bucket, ticker ) SELECT bucket, ticker, closing_price, LAG(closing_price, 1) OVER (PARTITION BY ticker ORDER BY bucket) AS prev_close, ((closing_price - LAG(closing_price, 1) OVER (PARTITION BY ticker ORDER BY bucket)) / LAG(closing_price, 1) OVER (PARTITION BY ticker ORDER BY bucket)) * 100 AS hourly_return_pct FROM hourly_close; ``` **Query 4: Price volatility** ```sql SELECT ticker, AVG(price) AS avg_price, STDDEV(price) AS price_volatility, MAX(price) - MIN(price) AS price_spread FROM stock_prices WHERE ts > NOW() - INTERVAL '7 days' GROUP BY ticker HAVING count(*) > 10 ORDER BY price_volatility DESC; ``` ## What's Happening Behind the Scenes? ### Columnstore Compression With `tsdb.enable_columnstore=true`: - Data is stored in a hybrid row-columnar format - Analytical queries only scan the columns they need (massive speedup) - Typical compression ratios: 90%+ for time-series data - Compression happens transparently - no changes to your queries ### Direct to Columnstore When you use `SET timescaledb.enable_direct_compress_copy = on`: - Data loads directly into compressed columnstore format - Bypasses the rowstore entirely - Instant analytical performance - no waiting for background compression ### Segmentation The `tsdb.segmentby='ticker'` setting: - Groups data by user within each chunk - Improves compression ratios (similar data together) - Speeds up queries that filter by ticker - Better for ticker-based analytics ## Continuous Aggregates (Advanced) For real-time dashboards, you can create continuous aggregates that automatically update: ```sql -- Create a continuous aggregate for hourly candlesticks CREATE MATERIALIZED VIEW candlesticks_hourly WITH (timescaledb.continuous) AS SELECT time_bucket('1 hour', ts) AS hour, ticker, FIRST(price, ts) AS open_price, MAX(price) AS high_price, MIN(price) AS low_price, LAST(price, ts) AS close_price, AVG(price) AS avg_price, SUM(volume) AS sum_volume FROM stock_prices GROUP BY hour, ticker ORDER BY hour DESC, ticker ASC; -- Add a refresh policy to keep it updated SELECT add_continuous_aggregate_policy('candlesticks_hourly', start_offset => INTERVAL '2 hours', end_offset => INTERVAL '1 hour', schedule_interval => INTERVAL '1 hour'); ``` Now you can query `candlesticks_hourly` for instant results on pre-aggregated data. ```sql SELECT * from candlesticks_hourly WHERE ticker = 'NFLX'; ``` --- **Questions?** Check out [TimescaleDB Documentation](https://docs.timescale.com) or the [TimescaleDB Community Forums](https://www.timescale.com/forum). ================================================ FILE: docs/getting-started/nyc-taxi/README.md ================================================ # NYC Taxi Data Example Get started with TimescaleDB using New York City taxi trip data. This example demonstrates how to handle high-volume transportation data with location-based analytics and time-series aggregations. ## What You'll Learn - How to model high-volume transportation data with lat/lon coordinates - Time-series aggregations with `time_bucket()` - Optimal segmentation strategies for compression - Revenue and usage pattern analysis - Loading data with direct to columnstore for instant performance ## Prerequisites - Docker installed - `psql` PostgreSQL client - 15-20 minutes ## Quick Start ### Step 1: Start TimescaleDB You have two options to start TimescaleDB: #### Option 1: One-line install (Recommended) The easiest way to get started: > **Important:** This script is intended for local development and testing only. Do **not** use it for production deployments. For production-ready installation options, see the [TimescaleDB installation guide](https://docs.timescale.com/self-hosted/latest/install/). **Linux/Mac:** ```sh curl -sL https://tsdb.co/start-local | sh ``` This command: - Downloads and starts TimescaleDB (if not already downloaded) - Exposes PostgreSQL on port **6543** (a non-standard port to avoid conflicts with other PostgreSQL instances on port 5432) - Automatically tunes settings for your environment using timescaledb-tune - Sets up a persistent data volume #### Option 2: Manual Docker command also used for Windows Alternatively, you can run TimescaleDB directly with Docker: ```bash docker run -d --name timescaledb \ -p 6543:5432 \ -e POSTGRES_PASSWORD=password \ timescale/timescaledb-ha:pg18 ``` **Note:** We use port **6543** (mapped to container port 5432) to avoid conflicts if you have other PostgreSQL instances running on the standard port 5432. Wait about 1-2 minutes for TimescaleDB to download & initialize. ### Step 2: Connect to TimescaleDB Connect using `psql`: ```bash psql -h localhost -p 6543 -U postgres # When prompted, enter password: password ``` You should see the PostgreSQL prompt. Verify TimescaleDB is installed: ```sql SELECT extname, extversion FROM pg_extension WHERE extname = 'timescaledb'; ``` Expected output: ``` extname | extversion -------------+------------ timescaledb | 2.x.x ``` **Prefer a GUI?** If you'd rather use a graphical tool instead of the command line, you can download [pgAdmin](https://www.pgadmin.org/download/) and connect to TimescaleDB using the same connection details (host: `localhost`, port: `6543`, user: `postgres`, password: `password`). ### Step 3: Create the Schema Create the optimized hypertable by running this SQL in your `psql` session: ```sql -- Create the hypertable with optimal settings for NYC Taxi data -- This automatically enables columnstore for fast analytical queries CREATE TABLE trips ( vendor_id TEXT, pickup_boroname VARCHAR, pickup_datetime TIMESTAMP WITHOUT TIME ZONE NOT NULL, dropoff_datetime TIMESTAMP WITHOUT TIME ZONE NOT NULL, passenger_count NUMERIC, trip_distance NUMERIC, pickup_longitude NUMERIC, pickup_latitude NUMERIC, rate_code INTEGER, dropoff_longitude NUMERIC, dropoff_latitude NUMERIC, payment_type VARCHAR, fare_amount NUMERIC, extra NUMERIC, mta_tax NUMERIC, tip_amount NUMERIC, tolls_amount NUMERIC, improvement_surcharge NUMERIC, total_amount NUMERIC ) WITH ( tsdb.hypertable, tsdb.partition_column='pickup_datetime', tsdb.enable_columnstore=true, tsdb.segmentby='pickup_boroname', tsdb.orderby='pickup_datetime DESC' ); -- Create indexes CREATE INDEX idx_trips_pickup_time ON trips (pickup_datetime DESC); CREATE INDEX idx_trips_borough_time ON trips (pickup_boroname, pickup_datetime DESC); ``` This creates a `trips` table with: - Automatic time-based partitioning on `pickup_datetime` - Columnstore enabled for fast analytical queries - Segmentation by `pickup_boroname` for optimal compression (6 boroughs) - Full trip details including fares, distances, and coordinates ### Step 4: Load Sample Data First, download and decompress the sample data: ```bash # Download the sample data wget https://assets.timescale.com/timescaledb-datasets/nyc_taxi_sample_nov_dec_2015.csv.gz # Decompress the CSV file gunzip nyc_taxi_sample_nov_dec_2015.csv.gz # This will create nyc_taxi_sample_nov_dec_2015.csv ready for loading ``` We provide two approaches for loading data. Choose based on your needs: #### Option A: Direct to Columnstore (Recommended - Instant Performance) This approach writes data directly to the columnstore, bypassing the rowstore entirely. You get instant analytical performance. **From command line:** ```bash psql -h localhost -p 6543 -U postgres \ -v ON_ERROR_STOP=1 \ -c "SET timescaledb.enable_direct_compress_copy = on; COPY trips FROM STDIN WITH (FORMAT csv, HEADER true);" \ < nyc_taxi_sample_nov_dec_2015.csv ``` This command reads the CSV file from your local filesystem and pipes it to PostgreSQL, which loads it directly into the columnstore. **Verify data loaded:** ```sql SELECT COUNT(*) FROM trips; ``` #### Option B: Standard COPY (Fallback) This approach loads data into the rowstore first. Data will be converted to the columnstore by a background policy (12-24 hours) for faster querying. **From command line:** ```bash psql -h localhost -p 6543 -U postgres \ -v ON_ERROR_STOP=1 \ -c "COPY trips FROM STDIN WITH (FORMAT csv, HEADER true);" \ < nyc_taxi_sample_nov_dec_2015.csv ``` **Verify data loaded:** ```sql SELECT COUNT(*) FROM trips; ``` **Manually convert to columnstore (Optional):** If you loaded data using standard copy a background process will convert your rowstore data to the columnstore in 12-24 hours, you can manually convert it immediately to get the best query performance: ```sql DO $$ DECLARE ch TEXT; BEGIN FOR ch IN SELECT show_chunks('trips') LOOP CALL convert_to_columnstore(ch); END LOOP; END $$; ``` ### Step 5: Run Sample Queries Now let's explore the data with some analytical queries. Run these in your `psql` session: **Query 1: Overall statistics** ```sql \timing on SELECT COUNT(*) as total_trips, ROUND(SUM(fare_amount)::numeric, 2) as total_revenue, ROUND(AVG(fare_amount)::numeric, 2) as avg_fare, ROUND(AVG(trip_distance)::numeric, 2) as avg_distance FROM trips; ``` **Query 2: Breakdown by vendor** ```sql SELECT vendor_id, COUNT(*) as trips, ROUND(AVG(fare_amount)::numeric, 2) as avg_fare, ROUND(AVG(tip_amount)::numeric, 2) as avg_tip, ROUND(AVG(passenger_count)::numeric, 2) as avg_passengers FROM trips GROUP BY vendor_id ORDER BY trips DESC; ``` **Query 3: Hourly patterns using time_bucket** ```sql SELECT time_bucket('1 hour', pickup_datetime) AS hour, COUNT(*) as trips, ROUND(AVG(fare_amount)::numeric, 2) as avg_fare, ROUND(SUM(tip_amount)::numeric, 2) as total_tips FROM trips GROUP BY hour ORDER BY hour DESC LIMIT 24; ``` **Query 4: Payment type analysis** ```sql SELECT payment_type, COUNT(*) as trip_count, ROUND(SUM(fare_amount)::numeric, 2) as total_revenue, ROUND(AVG(trip_distance)::numeric, 2) as avg_distance, ROUND(AVG(tip_amount)::numeric, 2) as avg_tip FROM trips GROUP BY payment_type ORDER BY total_revenue DESC; ``` **Query 5: Daily statistics by borough** ```sql SELECT time_bucket('1 day', pickup_datetime) AS day, pickup_boroname, COUNT(*) as trips, ROUND(AVG(fare_amount)::numeric, 2) as avg_fare, ROUND(MAX(fare_amount)::numeric, 2) as max_fare FROM trips GROUP BY day, pickup_boroname ORDER BY day DESC, pickup_boroname LIMIT 20; ``` **Query 6: Trips by distance category** ```sql SELECT CASE WHEN trip_distance < 1 THEN 'Short (< 1 mile)' WHEN trip_distance < 5 THEN 'Medium (1-5 miles)' WHEN trip_distance < 10 THEN 'Long (5-10 miles)' ELSE 'Very Long (> 10 miles)' END as distance_category, COUNT(*) as trips, ROUND(AVG(fare_amount)::numeric, 2) as avg_fare, ROUND(AVG(tip_amount)::numeric, 2) as avg_tip FROM trips GROUP BY distance_category ORDER BY trips DESC; ``` ## What's Happening Behind the Scenes? ### Hypertables When you create a table with `tsdb.hypertable`, TimescaleDB automatically: - Partitions your data into time-based chunks (default: 7 days per chunk) - Manages chunk lifecycle automatically - Optimizes queries to scan only relevant chunks ### Columnstore Compression With `tsdb.enable_columnstore=true`: - Data is stored in a hybrid row-columnar format - Analytical queries only scan the columns they need (massive speedup) - Typical compression ratios: 90%+ for time-series data - Compression happens transparently - no changes to your queries ### Direct to Columnstore When you use `SET timescaledb.enable_direct_compress_copy = on`: - Data loads directly into compressed columnstore format - Bypasses the rowstore entirely - Instant analytical performance - no waiting for background compression - Perfect for bulk data loads and migrations ### Segmentation The `tsdb.segmentby='pickup_boroname'` setting: - Groups data by pickup borough within each chunk (6 unique values: Manhattan, Brooklyn, Queens, Bronx, Staten Island, EWR) - Improves compression ratios (similar data together) - Speeds up queries that filter by pickup_boroname - Better cardinality than vendor_id (6 values vs 2) for optimal compression - Automatically optimized without manual tuning ### time_bucket() Function TimescaleDB's `time_bucket()` is like PostgreSQL's `date_trunc()` but more powerful: - Works with any interval: `5 minutes`, `1 hour`, `1 day`, etc. - Optimized for time-series queries - Integrates seamlessly with continuous aggregates - Essential for time-series analytics ## Sample Queries Explained See [`nyc-taxi-queries.sql`](nyc-taxi-queries.sql) for the complete set of queries. Each query demonstrates: 1. **Total trips and revenue** - Simple aggregations across all data 2. **Breakdown by vendor** - Segmentation analysis by taxi vendor 3. **Hourly patterns** - Using `time_bucket()` for time-based aggregation 4. **Payment type analysis** - Analyzing payment methods 5. **Daily statistics** - Multi-dimensional aggregation (time + borough) 6. **Distance categories** - CASE statement with aggregations ## Schema Design Choices ### Why these settings? **partition_column='pickup_datetime'** - Time is the natural partition key for time-series data - Enables automatic chunk pruning for time-range queries - Default chunk interval (7 days) works well for taxi data **segmentby='pickup_boroname'** - Optimal cardinality with 6 borough values (Manhattan, Brooklyn, Queens, Bronx, Staten Island, EWR) - Frequently used in WHERE clauses and GROUP BY for location-based analytics - Improves compression by grouping geographically similar trips - Better than vendor_id (only 2 values) for compression efficiency **orderby='pickup_datetime DESC'** - Most queries want recent data first - Optimizes for "latest trips" queries - Improves query performance for time-range scans ## Continuous Aggregates (Advanced) For real-time dashboards, you can create continuous aggregates that automatically update: ```sql -- Create a continuous aggregate for hourly statistics by borough CREATE MATERIALIZED VIEW trips_hourly WITH (timescaledb.continuous) AS SELECT time_bucket('1 hour', pickup_datetime) AS hour, pickup_boroname, COUNT(*) as trip_count, AVG(fare_amount) as avg_fare, SUM(fare_amount) as total_revenue, AVG(trip_distance) as avg_distance FROM trips GROUP BY hour, pickup_boroname; -- Add a refresh policy to keep it updated SELECT add_continuous_aggregate_policy('trips_hourly', start_offset => INTERVAL '2 hours', end_offset => INTERVAL '1 hour', schedule_interval => INTERVAL '1 hour'); ``` Now you can query `trips_hourly` for instant results on pre-aggregated data. ## Troubleshooting ### Data didn't load - Check the CSV file path is correct - Ensure the CSV header matches the schema columns - Try loading a few rows first to test: `LIMIT 10` in your data file ### Direct to columnstore not working - Verify TimescaleDB version 2.24+: `SELECT extversion FROM pg_extension WHERE extname = 'timescaledb';` - Ensure you ran `SET timescaledb.enable_direct_compress_copy = on;` in the same session - Check for error messages in the output ### Queries seem slow - Verify columnstore is enabled: `SELECT * FROM timescaledb_information.hypertables WHERE hypertable_name = 'trips';` - Check if data is compressed: `SELECT * FROM timescaledb_information.chunks WHERE hypertable_name = 'trips';` - Ensure you're querying with time ranges (enables chunk exclusion) ### Out of memory during load - Reduce batch size in COPY command - Increase Docker memory allocation - Consider loading data in smaller time-range batches ## Use Cases This NYC Taxi example demonstrates patterns applicable to: - **Ride-sharing platforms** - Track trips, drivers, pricing - **Fleet management** - Vehicle tracking, route optimization - **Delivery services** - Order tracking, delivery times, driver analytics - **Public transportation** - Route analysis, passenger counts, schedule optimization - **Urban planning** - Traffic patterns, popular routes, demand forecasting - **Logistics** - Shipment tracking, route efficiency, cost analysis ## Clean Up When you're done experimenting: #### If you used the one-line install: ```bash # Stop the container docker stop timescaledb-ha-pg18-quickstart # Remove the container docker rm timescaledb-ha-pg18-quickstart # Remove the persistent data volume docker volume rm timescaledb_data # (Optional) Remove the Docker image docker rmi timescale/timescaledb-ha:pg18 ``` #### If you used the manual Docker command: ```bash # Stop the container docker stop timescaledb # Remove the container docker rm timescaledb # (Optional) Remove the Docker image docker rmi timescale/timescaledb-ha:pg18 ``` **Note:** If you created a named volume with the manual Docker command, you can remove it with `docker volume rm <volume_name>`. ## Contributing Found an issue or want to improve this example? Contributions welcome! Open an issue or PR on [GitHub](https://github.com/timescale/timescaledb). --- **Questions?** Check out [Timescale Community Forums](https://www.timescale.com/forum) or [TimescaleDB Documentation](https://docs.timescale.com). ================================================ FILE: docs/getting-started/nyc-taxi/nyc-taxi-queries.sql ================================================ -- Sample analytical queries for NYC Taxi dataset -- These showcase TimescaleDB's columnstore performance \echo '=== Sample Queries for NYC Taxi Data ===' \echo '' -- Query 1: Total trips and revenue \echo 'Query 1: Overall statistics' SELECT COUNT(*) as total_trips, SUM(fare_amount) as total_revenue, AVG(fare_amount) as avg_fare, AVG(trip_distance) as avg_distance FROM trips; -- Query 2: Trips by vendor \echo '' \echo 'Query 2: Breakdown by vendor' SELECT vendor_id, COUNT(*) as trips, AVG(fare_amount) as avg_fare, AVG(tip_amount) as avg_tip, AVG(passenger_count) as avg_passengers FROM trips GROUP BY vendor_id ORDER BY trips DESC; -- Query 3: Hourly patterns using time_bucket \echo '' \echo 'Query 3: Hourly trip patterns (using time_bucket)' SELECT time_bucket('1 hour', pickup_datetime) AS hour, COUNT(*) as trips, AVG(fare_amount) as avg_fare, SUM(tip_amount) as total_tips FROM trips GROUP BY hour ORDER BY hour DESC LIMIT 24; -- Query 4: Payment type analysis \echo '' \echo 'Query 4: Payment type analysis' SELECT payment_type, COUNT(*) as trip_count, SUM(fare_amount) as total_revenue, AVG(trip_distance) as avg_distance, AVG(tip_amount) as avg_tip FROM trips GROUP BY payment_type ORDER BY total_revenue DESC; -- Query 5: Daily aggregation with time_bucket \echo '' \echo 'Query 5: Daily statistics by borough' SELECT time_bucket('1 day', pickup_datetime) AS day, pickup_boroname, COUNT(*) as trips, AVG(fare_amount) as avg_fare, MAX(fare_amount) as max_fare FROM trips GROUP BY day, pickup_boroname ORDER BY day DESC, pickup_boroname LIMIT 20; -- Query 6: Distance-based analysis \echo '' \echo 'Query 6: Trips by distance category' SELECT CASE WHEN trip_distance < 1 THEN 'Short (< 1 mile)' WHEN trip_distance < 5 THEN 'Medium (1-5 miles)' WHEN trip_distance < 10 THEN 'Long (5-10 miles)' ELSE 'Very Long (> 10 miles)' END as distance_category, COUNT(*) as trips, AVG(fare_amount) as avg_fare, AVG(tip_amount) as avg_tip FROM trips GROUP BY distance_category ORDER BY trips DESC; \echo '' \echo '=== Query examples complete! ===' \echo 'Notice how fast these analytical queries run on compressed columnar data.' ================================================ FILE: docs/getting-started/nyc-taxi/nyc-taxi-sample.csv ================================================ vendor_id,pickup_boroname,pickup_datetime,dropoff_datetime,passenger_count,trip_distance,pickup_longitude,pickup_latitude,rate_code,dropoff_longitude,dropoff_latitude,payment_type,fare_amount,extra,mta_tax,tip_amount,tolls_amount,improvement_surcharge,total_amount # Placeholder: This file will contain ~1000 rows of NYC taxi trip data # Sample row format: CMT,Manhattan,2024-01-15 08:30:00,2024-01-15 08:45:00,1,2.5,-73.9812,40.7685,1,-73.9580,40.7784,1,12.50,0.50,0.50,2.50,0.00,0.30,16.30 ================================================ FILE: docs/getting-started/nyc-taxi/nyc-taxi-schema.sql ================================================ -- TimescaleDB NYC Taxi Example Schema -- -- This schema demonstrates optimal design for high-volume transportation data. -- NYC Taxi data includes timestamps, locations, fares, and trip details. -- Enable timing to show query performance \timing on -- Create the hypertable with optimal settings for NYC Taxi data -- This automatically enables columnstore for fast analytical queries CREATE TABLE trips ( vendor_id TEXT, pickup_boroname VARCHAR, pickup_datetime TIMESTAMP WITHOUT TIME ZONE NOT NULL, dropoff_datetime TIMESTAMP WITHOUT TIME ZONE NOT NULL, passenger_count NUMERIC, trip_distance NUMERIC, pickup_longitude NUMERIC, pickup_latitude NUMERIC, rate_code INTEGER, dropoff_longitude NUMERIC, dropoff_latitude NUMERIC, payment_type VARCHAR, fare_amount NUMERIC, extra NUMERIC, mta_tax NUMERIC, tip_amount NUMERIC, tolls_amount NUMERIC, improvement_surcharge NUMERIC, total_amount NUMERIC ) WITH ( tsdb.hypertable, tsdb.partition_column='pickup_datetime', tsdb.enable_columnstore=true, tsdb.segmentby='pickup_boroname', tsdb.orderby='pickup_datetime DESC' ); -- Create indexes for common query patterns on rowstore data -- Note: These indexes primarily help with uncompressed rowstore data. -- Columnstore queries use internal structures (min/max stats) for pruning. CREATE INDEX idx_trips_pickup_time ON trips (pickup_datetime DESC); CREATE INDEX idx_trips_borough_time ON trips (pickup_boroname, pickup_datetime DESC); -- Add helpful table comments COMMENT ON TABLE trips IS 'NYC Taxi trip data with automatic partitioning and columnstore compression'; COMMENT ON COLUMN trips.vendor_id IS 'Taxi vendor ID'; COMMENT ON COLUMN trips.pickup_boroname IS 'Pickup borough name (Manhattan, Brooklyn, Queens, Bronx, Staten Island, EWR)'; COMMENT ON COLUMN trips.pickup_datetime IS 'Timestamp when the trip started'; COMMENT ON COLUMN trips.dropoff_datetime IS 'Timestamp when the trip ended'; COMMENT ON COLUMN trips.passenger_count IS 'Number of passengers'; COMMENT ON COLUMN trips.trip_distance IS 'Trip distance in miles'; COMMENT ON COLUMN trips.pickup_longitude IS 'Pickup location longitude'; COMMENT ON COLUMN trips.pickup_latitude IS 'Pickup location latitude'; COMMENT ON COLUMN trips.rate_code IS 'Rate code for the trip'; COMMENT ON COLUMN trips.dropoff_longitude IS 'Dropoff location longitude'; COMMENT ON COLUMN trips.dropoff_latitude IS 'Dropoff location latitude'; COMMENT ON COLUMN trips.payment_type IS 'Payment type (e.g., Credit card, Cash, No charge, Dispute, Unknown, Voided trip)'; COMMENT ON COLUMN trips.fare_amount IS 'Base fare amount'; COMMENT ON COLUMN trips.extra IS 'Extra charges'; COMMENT ON COLUMN trips.mta_tax IS 'MTA tax'; COMMENT ON COLUMN trips.tip_amount IS 'Tip amount'; COMMENT ON COLUMN trips.tolls_amount IS 'Tolls amount'; COMMENT ON COLUMN trips.improvement_surcharge IS 'Improvement surcharge'; COMMENT ON COLUMN trips.total_amount IS 'Total trip amount'; \echo '' \echo '=== NYC Taxi hypertable created successfully! ===' \echo '' \echo 'Table: trips' \echo 'Features enabled:' \echo ' - Automatic time-based partitioning' \echo ' - Columnstore compression for fast analytics' \echo ' - Optimized segmentation by pickup_boroname (6 boroughs)' \echo '' \echo 'Next: Load sample data using nyc-taxi-sample.csv' \echo '' ================================================ FILE: scripts/CMakeLists.txt ================================================ find_program( NM NAMES nm PATHS /usr/bin /usr/local/bin /opt/local/bin) if(NM) message(STATUS "Using nm ${NM}") else() message(STATUS "Install nm to be able to run export checks") endif(NM) configure_file(export_prefix_check.sh.in export_prefix_check.sh @ONLY) ================================================ FILE: scripts/backport.py ================================================ #!/usr/bin/env python3 import os import random import re import string import subprocess import sys from github import Github # This is PyGithub. import requests # Limit our history search and fetch depth to this value, not to get stuck in # case of a bug. HISTORY_DEPTH = 1000 def run_query(query): """A simple function to use requests.post to make the GraphQL API call.""" token = os.environ.get("ORG_AUTOMATION_TOKEN") request = requests.post( "https://api.github.com/graphql", json={"query": query}, headers={"Authorization": f"Bearer {token}"} if token else None, timeout=20, ) response = request.json() # Have to work around the unique GraphQL convention of returning 200 for errors. if request.status_code != 200 or "errors" in response: raise ValueError( f"Query failed to run by returning code of {request.status_code}." f"\nQuery: '{query}'" f"\nResponse: '{request.json()}'" ) return response def get_referenced_issue(pr_number): """Get the number of issue fixed by the given pull request. Returns None if no issue is fixed, or more than one issue""" # We only need the first issue here. We also request only the first 30 labels, # because GitHub requires some small restriction there that is counted # towards the GraphQL API usage quota. ref_result = run_query(string.Template(""" query { repository(owner: "timescale", name: "timescaledb") { pullRequest(number: $pr_number) { closingIssuesReferences(first: 1) { nodes { number, title, labels (first: 30) { nodes { name } } } } } } } """).substitute({"pr_number": pr_number})) # The above returns: # {'data': {'repository': {'pullRequest': {'closingIssuesReferences': {'nodes': [{'number': 6819, # 'title': '[Bug]: Segfault when `ts_insert_blocker` function is called', # 'labels': {'nodes': [{'name': 'bug'}]}}]}}}}} # # We can have {'nodes': [None]} in case it references an inaccessible repository, # just ignore it. ref_nodes = ref_result["data"]["repository"]["pullRequest"][ "closingIssuesReferences" ]["nodes"] if not ref_nodes or len(ref_nodes) != 1 or not ref_nodes[0]: return None, None, None number = ref_nodes[0]["number"] title = ref_nodes[0]["title"] labels = {x["name"] for x in ref_nodes[0]["labels"]["nodes"]} return number, title, labels def set_auto_merge(pr_number): """Enable auto-merge for the given PR""" # We first have to find out the PR id, which is some base64 string, different # from its number. query = string.Template("""query { repository(owner: "$owner", name: "$name") { pullRequest(number: $pr_number) { id } } }""").substitute( pr_number=pr_number, owner=source_repo.owner.login, name=source_repo.name ) result = run_query(query) pr_id = result["data"]["repository"]["pullRequest"]["id"] query = string.Template("""mutation { enablePullRequestAutoMerge( input: { pullRequestId: "$pr_id", mergeMethod: REBASE } ) { clientMutationId } }""").substitute(pr_id=pr_id) run_query(query) def git_output(command): """Get output from the git command, checking for the successful exit code""" return subprocess.check_output(f"git {command}", shell=True, text=True) def git_check(command): """Run a git command, checking for the successful exit code""" subprocess.run(f"git {command}", shell=True, check=True) def git_returncode(command): """Run a git command, returning the exit code""" return subprocess.run(f"git {command}", shell=True, check=False).returncode # The token has to have the "access public repositories" permission, or else creating a PR returns 404. github = Github(os.environ.get("ORG_AUTOMATION_TOKEN")) source_remote = "origin" source_repo_name = os.environ.get("GITHUB_REPOSITORY") # This is set in GitHub Actions. if not source_repo_name: source_repo_name = "timescale/timescaledb" print( f"Will look at '{source_repo_name}' (git remote '{source_remote}') for bug fixes." ) source_repo = github.get_repo(source_repo_name) # Fetch the main branch. Apparently the local repo can be shallow in some cases # in Github Actions, so specify the depth. --unshallow will complain on normal # repositories, this is why we don't use it here. git_check( f"fetch --quiet --depth={HISTORY_DEPTH} {source_remote} main:refs/remotes/{source_remote}/main" ) # Find out what is the branch corresponding to the previous version compared to # main. We will backport to that branch. version_config = dict( [ re.match(r"^(.+)\s+=\s+(.+)$", line).group(1, 2) for line in git_output(f"show {source_remote}/main:version.config").splitlines() if line ] ) version = version_config["version"].split("-")[0] # Split off the 'dev' suffix. version_parts = version.split(".") # Split the three version numbers. version_parts[1] = str(int(version_parts[1]) - 1) version_parts[2] = "x" backport_target = ".".join(version_parts) backported_label = f"backported-{backport_target}" print(f"Will backport to {backport_target}.") # Fetch the target branch. Apparently the local repo can be shallow in some cases # in Github Actions, so specify the depth. --unshallow will complain on normal # repositories, this is why we don't use it here. git_check( f"fetch --quiet --depth={HISTORY_DEPTH} {source_remote} {backport_target}:refs/remotes/{source_remote}/{backport_target}" ) # Find out which commits are unique to main and target branch. Also build sets of # the titles of these commits. We will compare the titles to check whether a # commit was backported. main_commits = [ line.split("\t") for line in git_output( f'log -{HISTORY_DEPTH} --abbrev=12 --pretty="format:%h\t%s" {source_remote}/{backport_target}..{source_remote}/main' ).splitlines() if line ] print(f"Have {len(main_commits)} new commits in the main branch.") branch_commits = [ line.split("\t") for line in git_output( f'log -{HISTORY_DEPTH} --abbrev=12 --pretty="format:%h\t%s" {source_remote}/main..{source_remote}/{backport_target}' ).splitlines() if line ] branch_commit_titles = {x[1] for x in branch_commits} # We will do backports per-PR, because one PR, though not often, might contain # many commits. So as the first step, go through the commits unique to main, find # out which of them have to be backported, and remember the corresponding PRs. # We also have to remember which commits to backport. The list from PR itself is # not what we need, these are the original commits from the PR branch, and we # need the resulting commits in master. class PRInfo: """Information about the PR to be backported.""" def __init__(self, pygithub_pr_, issue_number_): self.pygithub_pr = pygithub_pr_ self.pygithub_commits = [] self.issue_number = issue_number_ def should_backport_by_labels(number, title, labels): """Should we backport the given PR/issue, judging by the labels? Note that this works in ternary logic: True means we must, False means we must not (tags to disable backport take precedence), and None means weak no (no tags to either request or disable backport)""" stopper_labels = labels.intersection( ["disable-auto-backport", "auto-backport-not-done"] ) if stopper_labels: print( f"#{number} '{title}' is labeled as '{list(stopper_labels)[0]}' which prevents automated backporting." ) return False force_labels = labels.intersection(["bug", "force-auto-backport"]) if force_labels: print( f"#{number} '{title}' is labeled as '{list(force_labels)[0]}' which requests automated backporting." ) return True return None # Go through the commits unique to main, and build a dict(pr number -> PRInfo) # of PRs that we will consider for backporting. prs_to_backport = {} for commit_sha, commit_title in main_commits: print() pygithub_commit = source_repo.get_commit(sha=commit_sha) pulls = pygithub_commit.get_pulls() if not pulls or pulls.totalCount == 0: print(f"{commit_sha[:9]} '{commit_title}' does not belong to a PR.") continue if pulls.totalCount > 1: # What would that mean? Just play it safe and skip it. print( f"{commit_sha[:9]} '{commit_title}' references multiple PRs: {', '.join([pull.number for pull in pulls])}" ) continue pull = pulls[0] # If a commit with the same title is already in the branch, mark the PR with # a corresponding tag. This makes it easier to check what was backported # when looking at the release milestone. Note that we do this before other # checks -- maybe it was backported manually regardless of the usual # conditions. if commit_title in branch_commit_titles: print(f"{commit_sha[:9]} '{commit_title}' is already in the branch.") if backported_label not in {label.name for label in pull.labels}: pull.add_to_labels(backported_label) continue # Next, we're going to look at the labels of both the PR and the linked # issue, if any, to understand whether we should backport the fix. We have # labels to request backport like "bug", and labels to prevent backport # like "disable-auto-backport", on both issue and the PR. We're going to use # the ternary False/None/True logic to combine them properly. issue_number, issue_title, issue_labels = get_referenced_issue(pull.number) if not issue_number: should_backport_issue_ternary = None print( f"{commit_sha[:9]} belongs to the PR #{pull.number} '{pull.title}' that does not close an issue." ) else: issue = source_repo.get_issue(number=issue_number) should_backport_issue_ternary = should_backport_by_labels( issue_number, issue_title, issue_labels ) print( f"{commit_sha[:9]} belongs to the PR #{pull.number} '{pull.title}' " f"that references the issue #{issue.number} '{issue.title}'." ) pull_labels = {label.name for label in pull.labels} should_backport_pr_ternary = should_backport_by_labels( pull.number, pull.title, pull_labels ) # We backport if either the PR or the issue labels request the backport, and # none of them prevent it. I'm writing it with `is True` because I don't # remember python rules for ternary logic with None (do you?). if ( should_backport_pr_ternary is True or should_backport_issue_ternary is True ) and ( should_backport_pr_ternary is not False and should_backport_issue_ternary is not False ): print(f"{commit_sha[:9]} '{commit_title}' will be considered for backporting.") else: continue # Remember the PR and the corresponding resulting commit in main. if pull.number not in prs_to_backport: prs_to_backport[pull.number] = PRInfo(pull, issue_number) # We're traversing the history backwards, and want to have the list of # commits in forward order. prs_to_backport[pull.number].pygithub_commits.insert(0, pygithub_commit) def branch_has_open_pr(repo, branch): """Check whether the given branch has an open PR.""" # There's no way to search by branch name + fork name, but in the case the # branch name is probably unique. We'll bail out if we find more than one PR. template = """query { repository(name: "$repo_name", owner: "$repo_owner") { pullRequests(headRefName: "$branch", first: 2) { nodes { closed }}}}""" params = { "branch": branch, "repo_name": repo.name, "repo_owner": repo.owner.login, } query = string.Template(template).substitute(params) result = run_query(query) # This returns: # {'data': {'repository': {'pullRequests': {'nodes': [{'closed': True}]}}}} prs = result["data"]["repository"]["pullRequests"]["nodes"] if not prs or len(prs) != 1 or not prs[0]: return None return not prs[0]["closed"] def report_backport_not_done(original_pr, reason, details=None): """If something prevents us from backporting the PR automatically, report it in a comment to original PR, and add a label preventing further attempts.""" print( f"Will not backport the PR #{original_pr.number} '{original_pr.title}': {reason}" ) github_comment = f"Automated backport to {backport_target} not done: {reason}." if details: github_comment += f"\n\n{details}" # Link to the job if we're running in the Github Action environment. if "GITHUB_REPOSITORY" in os.environ: github_comment += ( "\n\n" f"[Job log](https://github.com/{os.environ.get('GITHUB_REPOSITORY')}" f"/actions/runs/{os.environ.get('GITHUB_RUN_ID')}" f"/attempts/{os.environ.get('GITHUB_RUN_ATTEMPT')})" ) original_pr.create_issue_comment(github_comment) original_pr.add_to_labels("auto-backport-not-done") # Set git name and email corresponding to the token user. token_user = github.get_user() os.environ["GIT_COMMITTER_NAME"] = token_user.name os.environ["GIT_AUTHOR_NAME"] = token_user.name # This is an email that is used by Github when you opt to hide your real email # address. It is required so that the commits are recognized by Github as made # by the user. That is, if you use a wrong e-mail, there won't be a clickable # profile picture next to the commit in the Github interface. os.environ["GIT_COMMITTER_EMAIL"] = ( f"{token_user.id}+{token_user.login}@users.noreply.github.com" ) os.environ["GIT_AUTHOR_EMAIL"] = os.environ["GIT_COMMITTER_EMAIL"] print( f"Will commit as {os.environ['GIT_COMMITTER_NAME']} <{os.environ['GIT_COMMITTER_EMAIL']}>" ) # Fetch all branches from the repository, because we use the presence # of the backport branches to determine that a backport exists. It's not convenient # to query for branch existence through the PyGithub API. git_check(f"fetch {source_remote}") # Now, go over the list of PRs that we have collected, and try to backport # each of them. Do it in randomized order to avoid getting stuck on a single # error. print(f"Have {len(prs_to_backport)} PRs to backport.") for index, pr_info in enumerate( random.sample(list(prs_to_backport.values()), len(prs_to_backport)) ): print() # Don't want to have an endless loop that modifies the repository in an # unattended script. The already backported/conflicted PRs shouldn't even # get into this list, so the low number is OK, it will still make progress. if index > 5: print(f"{index} PRs processed, stopping as a precaution.") sys.exit(0) original_pr = pr_info.pygithub_pr backport_branch = f"backport/{backport_target}/{original_pr.number}" # If there is already a backport branch for this PR, this probably means # that we already created the backport PR. Update it, because the PR might # not auto-merge when the branch is not up to date with the target branch, # depending on the branch protection settings. We want to update to the # recent target automatically to minimize the amount of manual work. if ( git_returncode(f"rev-parse {source_remote}/{backport_branch} > /dev/null 2>&1") == 0 ): print( f'Backport branch {backport_branch} for PR #{original_pr.number}: "{original_pr.title}" already exists.' ) if not branch_has_open_pr(source_repo, backport_branch): # The PR can be closed manually when the backport is not needed, or # can not exist when there was some error. We are only interested in # the most certain case when there is an open backport PR. continue print(f"Updating the branch {backport_branch} because it has an open PR.") git_check("reset --hard") git_check("clean -xfd") git_check( f"checkout --quiet --force --detach {source_remote}/{backport_branch} > /dev/null" ) # Use merge and no force-push, so that the simultaneous changes made by # other users are not accidentally overwritten. git_check(f"merge --quiet --no-edit {source_remote}/{backport_target}") git_check(f"push {source_remote} @:{backport_branch}") continue # Try to cherry-pick the commits. git_check("reset --hard") git_check("clean -xfd") git_check( f"checkout --quiet --force --detach {source_remote}/{backport_target} > /dev/null" ) commit_shas = [commit.sha for commit in pr_info.pygithub_commits] if git_returncode(f"cherry-pick --quiet -m 1 -x {' '.join(commit_shas)}") != 0: details = f"### Git status\n\n```\n{git_output('status')}\n```" git_check("cherry-pick --abort") report_backport_not_done(original_pr, "cherry-pick failed", details) continue changed_files = {file.filename for file in original_pr.get_files()} # Push the backport branch. git_check(f"push {source_remote} @:refs/heads/{backport_branch}") # Prepare description for the backport PR. backport_description = ( f"This is an automated backport of #{original_pr.number}: {original_pr.title}." ) if pr_info.issue_number: backport_description += f"\nThe original issue is #{pr_info.issue_number}." # Do not merge the PR automatically if it changes some particularly # conflict-prone files that are better to review manually. Also mention this # in the description. stopper_files = changed_files.intersection( ["sql/updates/latest-dev.sql", "sql/updates/reverse-dev.sql"] ) if stopper_files: backport_description += ( "\n" f"This PR will not be merged automatically, because it modifies '{list(stopper_files)[0]}' " "which is conflict-prone. Please review these changes manually." ) else: backport_description += ( "\n" "This PR will be merged automatically after all the relevant CI checks pass." ) backport_description += ( " If this fix should not be backported, or will be backported manually, " "just close this PR. You can use the backport branch to add your " "changes, it won't be modified automatically anymore." "\n" "\n" "For more details, please see the [documentation]" "(https://github.com/timescale/eng-database/wiki/Releasing-TimescaleDB#automated-cherry-picking-of-bug-fixes)" ) # Add original PR description. Comment out the Github issue reference # keywords like 'Fixes #1234', to avoid having multiple PRs saying they fix # a given issue. The backport PR is going to reference the fixed issue as # "Original issue #xxxx". original_description = re.sub( r"((fix|clos|resolv)[esd]+)(\s+#[0-9]+)", r"`\1`\3", original_pr.body or "", # Match "" if pr_body is None flags=re.IGNORECASE, ) backport_description += ( "\n" "\n" "## Original description" "\n" f"### {original_pr.title}" "\n" f"{original_description}" ) # Create the backport PR. backport_pr = source_repo.create_pull( title=f"Backport to {backport_target}: #{original_pr.number}: {original_pr.title}", body=backport_description, # We're creating PR from the token user's fork. head=backport_branch, base=backport_target, ) backport_pr.add_to_labels("is-auto-backport") backport_pr.add_to_assignees(original_pr.user.login) if not stopper_files: set_auto_merge(backport_pr.number) print( f"Created backport PR #{backport_pr.number} for #{original_pr.number}: {original_pr.title}" ) ================================================ FILE: scripts/bundle_coredumps.sh ================================================ #!/bin/bash TARGET=coredumps COREDUMP_DIR=/var/lib/systemd/coredump set -e mkdir -p "$TARGET" # get information from gdb info=$(echo "info sharedlibrary" | coredumpctl gdb) executable=$(echo "$info" | grep Executable | sed -e 's!^[^/]\+!!') cp "$executable" "$TARGET" cp ${COREDUMP_DIR}/* "$TARGET" # copy libraries extracted from gdb info echo "$info" | grep '^0x' | sed -e 's!^[^/]\+!!' | xargs -ILIB cp "LIB" "$TARGET" ================================================ FILE: scripts/c_license_header-apache.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ ================================================ FILE: scripts/c_license_header-timescale.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ ================================================ FILE: scripts/changelog/generate.sh ================================================ #!/usr/bin/env bash set -eu # # This script build a CHANGELOG.md for a new release # echo_changelog() { echo "${1}" # skip the template and release notes files grep -i "${2}" .unreleased/* | \ cut -d: -f3- | sort | uniq | sed -e 's/^[[:space:]]*//' -e 's/^/* /' -e 's!#\([0-9][0-9]*\)![#\1](https://github.com/timescale/timescaledb/pull/\1)!g' echo } # Build a delta of the GUCs between two releases # # Param: previous release # Param: release branch # echo_gucs() { echo "**GUCs**" git diff ${1}..${2} src/guc.c | grep EXTOPTION | grep -o '"[^"]*"' | sed 's/^/* /' | sort | sed 's/"/`/g' } get_version_config_var() { grep "${1}" version.config | awk '{print $3}' | sed 's/-dev//' } RELEASE_NEXT=$(get_version_config_var '^version') RELEASE_PREVIOUS=$(get_version_config_var '^previous_version') RELEASE_BRANCH="${RELEASE_NEXT/%.[0-9]/.x}" echo "Building CHANGELOG" { echo "## ${RELEASE_NEXT} ($(date +"%Y-%m-%d"))" echo "" echo "This release contains performance improvements and bug fixes since the ${RELEASE_PREVIOUS} release. We recommend that you upgrade at the next available opportunity." echo "" echo "**Highlighted features in TimescaleDB v${RELEASE_NEXT}**" echo "* " echo "" echo_changelog '**Backward-Incompatible Changes**' '^Backward-Incompatible Change:' echo_changelog '**Features**' '^Implements:' echo_changelog '**Bugfixes**' '^Fixes:' echo_changelog '**New Settings**' '^Setting:' echo_gucs $RELEASE_PREVIOUS $RELEASE_BRANCH echo "" echo_changelog '**Thanks**' '^Thanks:' } > CHANGELOG_next.md RELEASE_NOTE_START=$(grep -n $RELEASE_PREVIOUS CHANGELOG.md | cut -d ':' -f 1 | head -1) CHANGELOG_HEADER_LINES=$((RELEASE_NOTE_START - 1)) mv CHANGELOG.md CHANGELOG.md.tmp head -n $CHANGELOG_HEADER_LINES CHANGELOG.md.tmp > CHANGELOG.md cat CHANGELOG_next.md >> CHANGELOG.md CHANGELOG_LENGTH=$(wc -l < CHANGELOG.md.tmp) CHANGELOG_ENTRIES=$((CHANGELOG_LENGTH-CHANGELOG_HEADER_LINES)) tail -n "$CHANGELOG_ENTRIES" CHANGELOG.md.tmp >> CHANGELOG.md rm -f CHANGELOG.md.tmp CHANGELOG_next.md # Remove the CHANGELOG generating # Fresh start for next version echo "Deleting all .unreleased files" rm -f .unreleased/* echo "done." ================================================ FILE: scripts/changelog/template.rfc822 ================================================ Backward-Incompatible Change: #NNNNN <one line description of the feature> Implements: #NNNNN <one line description of the feature> Fixes: #NNNNN <one line description of the Issue> Setting: <one-line description of the new or changed setting (GUC)> Thanks: @name <Thank you note> ================================================ FILE: scripts/check_changelog_format.py ================================================ #!/usr/bin/env python3 import sys import re import os import github # this is PyGithub. import requests import string def run_query(query): """A simple function to use requests.post to make the GraphQL API call.""" request = requests.post( "https://api.github.com/graphql", json={"query": query}, headers={"Authorization": f'Bearer {os.environ.get("GH_TOKEN")}'}, timeout=20, ) response = request.json() # Have to work around the unique GraphQL convention of returning 200 for errors. if request.status_code != 200 or "errors" in response: raise ValueError( f"Query failed to run by returning code of {request.status_code}." f"\nQuery: '{query}'" f"\nResponse: '{request.json()}'" ) return response def get_referenced_issues(pr_number): """Get the numbers of issue fixed by the given pull request.""" ref_result = run_query(string.Template(""" query { repository(owner: "timescale", name: "timescaledb") { pullRequest(number: $pr_number) { closingIssuesReferences(first: 100) { edges { node { number } } } } } } """).substitute({"pr_number": pr_number})) # The above returns {'data': {'repository': {'pullRequest': {'closingIssuesReferences': {'edges': [{'node': {'number': 4944}}]}}}}} ref_edges = ref_result["data"]["repository"]["pullRequest"][ "closingIssuesReferences" ]["edges"] if not ref_edges: return [] return [edge["node"]["number"] for edge in ref_edges if edge] # Check if a line matches any of the specified patterns def is_valid_line(line): patterns = [ r"^Fixes:\s*.*$", r"^Implements:\s*.*$", r"^Thanks:\s*.*$", r"^Backward-Incompatible Change:\s*.*$", r"^Setting:\s*.*$", ] for pattern in patterns: if re.match(pattern, line): return True return False def main(): github_token = os.environ.get("GH_TOKEN") if not github_token: print("Please populate the GH_TOKEN environment variable.") sys.exit(1) github_obj = github.Github(github_token) repo = github_obj.get_repo("timescale/timescaledb") # Get the file name from the command line argument if len(sys.argv) != 2: print("Please provide a file name as a command-line argument.") sys.exit(1) file_name = sys.argv[1] # Check if the file exists if not os.path.exists(file_name): print(f"{file_name} does not exist") sys.exit(1) this_pr_number = int(os.environ["PR_NUMBER"]) pr_issues = set(get_referenced_issues(this_pr_number)) # Read the file and check non-empty lines changelog_issues = set() with open(file_name, "r", encoding="utf-8") as file: for line in file: line = line.strip() if not is_valid_line(line): print(f'Invalid entry in change log: "{line}"') sys.exit(1) # The referenced issue number should be valid. for issue_number in re.findall("#([0-9]+)", line): issue_number = int(issue_number) try: issue = repo.get_issue(number=issue_number) except github.UnknownObjectException: print( f"The changelog entry references an invalid issue #{issue_number}:\n{line}" ) sys.exit(1) as_pr = None try: as_pr = issue.as_pull_request() except github.UnknownObjectException: # Not a pull request pass # Accept references to PR itself. if as_pr: if issue_number != this_pr_number: print( f"The changelog for PR #{this_pr_number} references another PR #{issue_number}" ) sys.exit(1) changelog_issues = pr_issues else: changelog_issues.add(issue_number) if changelog_issues != pr_issues: print( "Instead of " + (f"the issues {pr_issues}" if pr_issues else "no issues") + f" linked to the PR #{this_pr_number}, the changelog references " + (f"the issues {changelog_issues}" if changelog_issues else "no issues") ) sys.exit(1) if __name__ == "__main__": main() ================================================ FILE: scripts/check_file_license.sh ================================================ #!/bin/bash get_c_license() { awk 'BEGIN {ORS=""}{if($1 == "*/") {print; exit;}} {print}' $1 } get_sql_license() { awk 'BEGIN {ORS=""}{if($1 == "") {print; exit;}} {print}' $1 } check_file() { FLAG=${1} FILE=${2} SCRIPTPATH="$( cd "$(dirname "${0}")" || exit ; pwd -P )" TIMESCALE_LOCATION=$(dirname ${SCRIPTPATH}) LICENSE_FILE= LICENSE_STRING= FIRST_COMMENT= case ${FLAG} in ('-c') LICENSE_FILE="${SCRIPTPATH}/c_license_header-apache.h" LICENSE_STRING=$(get_c_license ${LICENSE_FILE}) FIRST_COMMENT=$(get_c_license ${FILE}) ;; ('-e') LICENSE_FILE="${SCRIPTPATH}/c_license_header-timescale.h" LICENSE_STRING=$(get_c_license ${LICENSE_FILE}) FIRST_COMMENT=$(get_c_license ${FILE}) ;; ('-i') LICENSE_FILE="${SCRIPTPATH}/license_apache.spec" LICENSE_STRING=$(get_sql_license ${LICENSE_FILE}) FIRST_COMMENT=$(get_sql_license ${FILE}) ;; ('-j') LICENSE_FILE="${SCRIPTPATH}/license_tsl.spec" LICENSE_STRING=$(get_sql_license ${LICENSE_FILE}) FIRST_COMMENT=$(get_sql_license ${FILE}) ;; ('-s') LICENSE_FILE="${SCRIPTPATH}/sql_license_apache.sql" LICENSE_STRING=$(get_sql_license ${LICENSE_FILE}) FIRST_COMMENT=$(get_sql_license ${FILE}) ;; ('-t') LICENSE_FILE="${SCRIPTPATH}/sql_license_tsl.sql" LICENSE_STRING=$(get_sql_license ${LICENSE_FILE}) FIRST_COMMENT=$(get_sql_license ${FILE}) ;; ('-p') LICENSE_FILE="${SCRIPTPATH}/license_tsl.spec" LICENSE_STRING=$(get_sql_license ${LICENSE_FILE}) FIRST_COMMENT=$(get_sql_license ${FILE}) ;; ("--") return 0; ;; (*) echo "Unknown flag" ${1} return 1; esac if [[ "${FIRST_COMMENT}" != "${LICENSE_STRING}" ]]; then echo ${FILE#"$TIMESCALE_LOCATION/"} "lacks a license header. Add"; echo cat ${LICENSE_FILE} echo echo "to the top of the file"; return 1; fi } args=$(getopt "c:e:i:j:p:s:t:" "$@"); errcode=$?; set -- $args if [[ ${errcode} != 0 ]]; then echo 'Usage: check_file_license ((-c|-e|-i|-j|-s|-t) <filename> ...)' return 2 fi ERRORCODE=0 while [[ ${1} ]]; do if [[ ${1} == "--" ]]; then break; fi check_file ${1} ${2} FILE_ERR=${?} ERRORCODE=$((FILE_ERR | ERRORCODE)); shift; shift; done exit ${ERRORCODE}; ================================================ FILE: scripts/check_license.sh ================================================ #! /bin/bash SCRIPT_DIR=$(dirname ${0}) SRC_DIR=$(dirname ${SCRIPT_DIR}) # we skip license checks for: # - the update script fragments, because the generated update scripts will # contain the license at top, and we don't want to repeat it in the middle # - test/sql/dump which contains auto-generated code # - src/chunk_adatptive since it's still in BETA check_file() { SUFFIX0= SUFFIX1= if [[ ${1} == '-c' || ${1} == '-e' ]]; then SUFFIX0='*.c' SUFFIX1='*.h' elif [[ ${1} == '-i' || ${1} == '-j' ]]; then SUFFIX0='*.spec' SUFFIX1='*.spec.in' elif [[ ${1} == '-p' ]]; then SUFFIX0='*.pl' SUFFIX1='*.pm' else SUFFIX0='*.sql' SUFFIX1='*.sql.in' fi find $2 -type f \( -name "${SUFFIX0}" -or -name "${SUFFIX1}" \) -and -not -path "${SRC_DIR}/sql/updates/*.sql" -and -not -path "${SRC_DIR}/test/sql/dump/*.sql" -and -not -path "${SRC_DIR}/src/chunk_adaptive.*" -print0 | xargs -0 -n1 "$(dirname ${0})/check_file_license.sh" ${1} } args=$(getopt "c:e:i:j:p:s:t:" "$@"); set -- $args ERRORCODE=0 while [[ ${1} ]]; do if [[ ${1} == "--" ]]; then break; fi check_file ${1} ${2} FILE_ERR=${?} ERRORCODE=$((FILE_ERR | ERRORCODE)); shift; shift; done exit ${ERRORCODE}; ================================================ FILE: scripts/check_license_all.sh ================================================ #!/bin/bash SCRIPT_DIR=$(dirname $0) BASE_DIR=$(dirname ${SCRIPT_DIR}) SRC_DIR=$BASE_DIR ${SCRIPT_DIR}/check_license.sh -c ${BASE_DIR}/src -s ${BASE_DIR}/sql -c ${BASE_DIR}/test -s ${BASE_DIR}/test -i ${BASE_DIR}/test exit_apache=$? SRC_DIR=$BASE_DIR ${SCRIPT_DIR}/check_license.sh -e ${BASE_DIR}/tsl/src -t ${BASE_DIR}/tsl/test -e ${BASE_DIR}/tsl/test -j ${BASE_DIR}/tsl/test -p ${BASE_DIR}/tsl/test -p ${BASE_DIR}/test/perl exit_tsl=$? if [ ${exit_apache} -ne 0 ] || [ ${exit_tsl} -ne 0 ]; then exit 1 fi ================================================ FILE: scripts/check_missing_gitignore_for_template_tests.sh ================================================ #!/bin/bash ERROR=0 for FILE in $(git ls-files | grep '\.sql\.in') do DIRNAME=$(dirname "${FILE}") FILENAME=$(basename "${FILE}" .sql.in) GITIGNORE=${DIRNAME}/.gitignore if [ -f "${GITIGNORE}" ]; then if git ls-files --others --exclude-standard $DIRNAME | grep --silent "${FILENAME}-"; then echo "Missing entry in ${GITIGNORE} for template file ${FILE}" ERROR=1 fi fi done exit ${ERROR} ================================================ FILE: scripts/check_orphaned_test_output_files.sh ================================================ #!/bin/bash ERROR=0 for FILE in $(git ls-files test/expected/*-[0-9][0-9].out tsl/test/expected/*-[0-9][0-9].out) do DIRNAME=$(dirname "${FILE}" | sed 's/expected/sql/g') TESTOUTPUTNAME=$(basename "${FILE}" .out | sed 's/-[0-9][0-9]//g') if [ ! -f "${DIRNAME}/${TESTOUTPUTNAME}.sql.in" ]; then echo "ERROR: template SQL test does not found: \"${DIRNAME}/${TESTOUTPUTNAME}.sql.in\"" echo "HINT: Please remove the output test file \"${FILE}\"" ERROR=1 fi done exit ${ERROR} ================================================ FILE: scripts/check_sql_script.py ================================================ #!/usr/bin/env python # Check SQL script components for problematic patterns. This script is # intended to be run on the scripts that are added to every update script, # but not the compiled update script or the pre_install scripts. # # This script will find patterns that are not idempotent and therefore # should be moved to the pre_install part. from pglast import parse_sql from pglast.visitors import Visitor, Skip, Continue from pglast.stream import RawStream import sys import re import argparse parser = argparse.ArgumentParser() parser.add_argument("filename", type=argparse.FileType("r"), nargs="+") args = parser.parse_args() class SQLVisitor(Visitor): def __init__(self, file): self.errors = 0 self.file = file super().__init__() def error(self, node, hint): self.errors += 1 print( f"Invalid statement found in sql script({self.file}):\n", RawStream()(node), ) print(hint, "\n") def visit_RawStmt(self, _ancestors, _node): # Statements are nested in RawStmt so we need to let the visitor descend return Continue def visit(self, _ancestors, node): self.error(node, "Consider moving the statement into a pre_install script") # We are only interested in checking top-level statements return Skip def visit_CommentStmt(self, _ancestors, _node): return Skip def visit_GrantStmt(self, _ancestors, _node): return Skip def visit_SelectStmt(self, _ancestors, _node): return Skip def visit_InsertStmt(self, _ancestors, _node): return Skip def visit_DeleteStmt(self, _ancestors, _node): return Skip def visit_DoStmt(self, _ancestors, _node): return Skip def visit_CreateEventTrigStmt(self, _ancestors, _node): return Skip def visit_VariableSetStmt(self, _ancestors, node): if not node.is_local: self.error(node, "Consider using SET LOCAL instead of SET") return Skip def visit_CreateTrigStmt(self, _ancestors, node): if not node.replace: self.error(node, "Consider using CREATE OR REPLACE TRIGGER") return Skip def visit_DefineStmt(self, _ancestors, node): if not node.replace: self.error(node, "Consider using CREATE OR REPLACE") return Skip def visit_DropStmt(self, _ancestors, node): if not node.missing_ok: self.error(node, "Consider using DROP IF EXISTS") return Skip def visit_ViewStmt(self, _ancestors, node): if not node.replace: self.error(node, "Consider using CREATE OR REPLACE VIEW") return Skip def visit_CreateFunctionStmt(self, _ancestors, node): if not node.replace: fn_str = ("FUNCTION", "PROCEDURE")[node.is_procedure is True] self.error(node, f"Consider using CREATE OR REPLACE {fn_str}") return Skip # copied from pgspot def visit_sql(sql, file): # @extschema@ is placeholder in extension scripts for # the schema the extension gets installed in sql = sql.replace("@extschema@", "extschema") sql = sql.replace("@extowner@", "extowner") sql = sql.replace("@database_owner@", "database_owner") # postgres contrib modules are protected by psql meta commands to # prevent running extension files in psql. # The SQL parser will error on those since they are not valid # SQL, so we comment out all psql meta commands before parsing. sql = re.sub(r"^\\", "-- \\\\", sql, flags=re.MULTILINE) visitor = SQLVisitor(file) for stmt in parse_sql(sql): visitor(stmt) return visitor.errors def main(args): errors = 0 error_files = [] for file in args.filename: sql = file.read() result = visit_sql(sql, file.name) if result > 0: errors += result error_files.append(file.name) if errors > 0: numbering = "errors" if errors > 1 else "error" print( f"{errors} {numbering} detected in {len(error_files)} files({', '.join(error_files)})" ) sys.exit(1) sys.exit(0) if __name__ == "__main__": main(args) sys.exit(0) ================================================ FILE: scripts/check_unnecessary_template_tests.sh ================================================ #!/bin/bash ERROR=0 for FILE in $(git ls-files test/sql/*.sql.in tsl/test/sql/*.sql.in) do DIRNAME=$(dirname "${FILE}" | sed 's/sql/expected/g') TESTNAME=$(basename "${FILE}" .sql.in) if diff --from-file ${DIRNAME}/${TESTNAME}-*.out > /dev/null 2>&1; then echo "ERROR: all template output test files are equal: \"${DIRNAME}/${TESTNAME}-*.out\"" echo "HINT: Please turn template test file \"${FILE}\" into a regular test file" ERROR=1 fi done exit ${ERROR} ================================================ FILE: scripts/check_unreferenced_files.sh ================================================ #!/bin/bash SCRIPT_DIR=$(dirname ${0}) BASE_DIR=$(pwd)/$SCRIPT_DIR/.. unreferenced=0 function get_filenames { echo ./*.$2 ./*.$2.in | xargs git ls-files | xargs -IFILE basename FILE } function check_directory { cd $BASE_DIR/$1 || exit test_files=$(get_filenames $1 $2) for file in $test_files; do output=$(grep --files-without-match $file CMakeLists.txt) # return value from grep --files-without-match seems to differ # between grep versions so we use output instead of return value if [ "$output" != "" ]; then echo -e "\nUnreferenced file in $1: $file\n" unreferenced=1 fi done } check_directory test/sql sql check_directory tsl/test/sql sql check_directory tsl/test/shared/sql sql check_directory test/isolation/specs spec check_directory tsl/test/isolation/specs spec exit $unreferenced ================================================ FILE: scripts/check_updates.py ================================================ #!/usr/bin/env python # Check SQL update script for undesirable patterns. This script is # intended to be run on the compiled update script or subsets of # the update script (e.g. latest-dev.sql and reverse-dev.sql) from pglast import parse_sql from pglast.ast import ColumnDef from pglast.visitors import Visitor from pglast import enums import sys import re import argparse parser = argparse.ArgumentParser() parser.add_argument("filename") parser.add_argument("--latest", action="store_true", help="process latest-dev.sql") args = parser.parse_args() class SQLVisitor(Visitor): def __init__(self): self.errors = 0 self.catalog_schemata = [ "_timescaledb_catalog", "_timescaledb_config", "_timescaledb_internal", ] super().__init__() def error(self, msg, hint=None): self.errors += 1 print(msg) if hint: print(hint) print() # ALTER TABLE _timescaledb_catalog.<tablename> ADD/DROP COLUMN def visit_AlterTableStmt(self, ancestors, node): # pylint: disable=unused-argument if ( "schemaname" in node.relation and node.relation.schemaname in self.catalog_schemata ): schema = node.relation.schemaname table = node.relation.relname for cmd in node.cmds: if cmd.subtype in ( enums.AlterTableType.AT_AddColumn, enums.AlterTableType.AT_DropColumn, ): if cmd.subtype == enums.AlterTableType.AT_AddColumn: subcmd = "ADD" column = cmd.def_.colname else: subcmd = "DROP" column = cmd.name self.error( f"Attempting to {subcmd} COLUMN {column} to catalog table {schema}.{table}", "Tables need to be rebuilt in update script to ensure consistent attribute numbers", ) # ALTER TABLE _timescaledb_catalog.<tablename> RENAME TO def visit_RenameStmt(self, ancestors, node): # pylint: disable=unused-argument if ( node.renameType == enums.ObjectType.OBJECT_TABLE and node.relation.schemaname in self.catalog_schemata ): self.error( f"Attempting to RENAME catalog table {node.relation.schemaname}.{node.relation.relname}", "Catalog tables should be rebuilt in update scripts to ensure consistent naming for dependent objects", ) # CREATE TEMP | TEMPORARY TABLE .. # CREATE TABLE IF NOT EXISTS .. def visit_CreateStmt(self, ancestors, node): # pylint: disable=unused-argument if node.relation.relpersistence == "t": schema = ( node.relation.schemaname + "." if node.relation.schemaname is not None else "" ) self.error( f"Attempting to CREATE TEMPORARY TABLE {schema}{node.relation.relname}" "Creating temporary tables is blocked in pg_extwlist context" ) if node.if_not_exists: self.error( f"Attempting to CREATE TABLE IF NOT EXISTS {node.relation.relname}" ) # We have to be careful with the column types we use in our catalog to only allow types # that are safe to use in catalog tables and not cause problems during extension upgrade, # pg_upgrade or dump/restore. if node.tableElts is not None: for coldef in node.tableElts: if isinstance(coldef, ColumnDef): if coldef.typeName.arrayBounds is not None: if coldef.typeName.names[-1].sval not in [ "bool", "text", ]: self.error( f"Attempting to CREATE TABLE {node.relation.relname} with blocked array type {coldef.typeName.names[-1].sval}" ) else: if coldef.typeName.names[-1].sval not in [ "bool", "int2", "int4", "int8", "interval", "jsonb", "name", "regclass", "regtype", "regrole", "serial", "text", "timestamptz", ]: self.error( f"Attempting to CREATE TABLE {node.relation.relname} with blocked type {coldef.typeName.names[-1].sval}" ) # CREATE SCHEMA IF NOT EXISTS .. def visit_CreateSchemaStmt( self, ancestors, node ): # pylint: disable=unused-argument if node.if_not_exists: self.error(f"Attempting to CREATE SCHEMA IF NOT EXISTS {node.schemaname}") # CREATE MATERIALIZED VIEW IF NOT EXISTS .. def visit_CreateTableAsStmt( self, ancestors, node ): # pylint: disable=unused-argument if node.if_not_exists: self.error( f"Attempting to CREATE MATERIALIZED VIEW IF NOT EXISTS {node.into.rel.relname}" ) # CREATE FOREIGN TABLE IF NOT EXISTS .. def visit_CreateForeignTableStmt( self, ancestors, node ): # pylint: disable=unused-argument if node.base.if_not_exists: self.error( f"Attempting to CREATE FOREIGN TABLE IF NOT EXISTS {node.base.relation.relname}" ) # CREATE FUNCTION / PROCEDURE _timescaledb_internal... def visit_CreateFunctionStmt( self, ancestors, node ): # pylint: disable=unused-argument if args.latest: # C functions should only appear in actual function definition but not # in latest-dev.sql as that would introduce a dependency on the library. # In that case, we want to use a dedicated placeholder function. lang = [elem for elem in node.options if elem.defname == "language"] code = [elem for elem in node.options if elem.defname == "as"][0].arg if ( lang and lang[0].arg.sval == "c" and code[-1].sval != "ts_update_placeholder" and node.returnType.names[0].sval not in ["table_am_handler", "index_am_handler"] ): functype = "procedure" if node.is_procedure else "function" self.error( f"Attempting to create {functype} {node.funcname[-1].sval} with language 'c'", "latest-dev should link C functions to ts_update_placeholder", ) if len(node.funcname) == 2 and node.funcname[0].sval == "_timescaledb_internal": functype = "procedure" if node.is_procedure else "function" self.error( f"Attempting to create {functype} {node.funcname[1].sval} in the internal schema", "_timescaledb_functions should be used as schema for internal functions", ) # copied from pgspot def visit_sql(sql): # @extschema@ is placeholder in extension scripts for # the schema the extension gets installed in sql = sql.replace("@extschema@", "extschema") sql = sql.replace("@extowner@", "extowner") sql = sql.replace("@database_owner@", "database_owner") # postgres contrib modules are protected by psql meta commands to # prevent running extension files in psql. # The SQL parser will error on those since they are not valid # SQL, so we comment out all psql meta commands before parsing. sql = re.sub(r"^\\", "-- \\\\", sql, flags=re.MULTILINE) visitor = SQLVisitor() # try: for stmt in parse_sql(sql): visitor(stmt) return visitor.errors def main(args): file = args.filename with open(file, "r", encoding="utf-8") as f: sql = f.read() errors = visit_sql(sql) if errors > 0: numbering = "errors" if errors > 1 else "error" print(f"{errors} {numbering} detected in file {file}") sys.exit(1) sys.exit(0) if __name__ == "__main__": main(args) sys.exit(0) ================================================ FILE: scripts/clang_format_all.sh ================================================ #!/bin/bash # we need to convert script dir to an absolute path SCRIPT_DIR=$(cd "$(dirname $0)" || exit; pwd) BASE_DIR=$(dirname $SCRIPT_DIR) find ${BASE_DIR} \( -path "${BASE_DIR}/src/*" -or -path "${BASE_DIR}/test/*" -or -path "${BASE_DIR}/tsl/*" \) \ -and -not \( -path "*/.*" -or -path "*CMake*" -or -path "${BASE_DIR}/src/import/*" -or -path "${BASE_DIR}/tsl/src/import/*" \) \ -and \( -name '*.c' -or -name '*.h' \) -print0 | xargs -0 ${SCRIPT_DIR}/clang_format_wrapper.sh -style=file -i ================================================ FILE: scripts/clang_format_wrapper.sh ================================================ #!/bin/bash # clang-format misunderstand sql function written in C because they have the signature # Datum my_func(PG_FUNCTION_ARGS) # and clang-format does not interpret PG_FUNCTION_ARGS as a type, name pair. # This script replaces PG_FUNCTION_ARGS with "PG_FUNCTION_ARGS fake_var_for_clang" to # make it look like a proper function for clang and then converts it back after clang runs. SCRIPT_DIR=$(cd "$(dirname $0)"; pwd) BASE_DIR=$(dirname $SCRIPT_DIR) TEMP_DIR="/tmp/timescaledb_format" OPTIONS=("${@}") if [ -e ${TEMP_DIR} ] then echo "error: ${TEMP_DIR} already exists" echo " delete ${TEMP_DIR} to format" exit 1 fi # shellcheck disable=SC2317 cleanup() { echo "cleaning" rm -rf ${TEMP_DIR} } trap cleanup EXIT SIGINT SIGTERM if ! mkdir ${TEMP_DIR} then echo "error: could not create temporary directory ${TEMP_DIR}" exit 1 fi #from this point on, if we get a failure we end up in an invalid state, so exit set -e CLANG_FORMAT_FLAGS="" FILE_NAMES="" for opt in "${OPTIONS[@]}" do if [[ "${opt:0:1}" != "-" ]] then file_path=${opt#"$BASE_DIR/"} FILE_NAMES="${FILE_NAMES} $file_path" else CLANG_FORMAT_FLAGS="${CLANG_FORMAT_FLAGS} $opt" fi done echo "copying to ${TEMP_DIR}" for name in ${FILE_NAMES} do # sed -i have different semantics on mac and linux, don't use mkdir -p "$(dirname "${TEMP_DIR}/${name}")" sed -e 's/(PG_FUNCTION_ARGS)/(PG_FUNCTION_ARGS fake_var_for_clang)/' ${BASE_DIR}/${name} > ${TEMP_DIR}/${name} done cp ${BASE_DIR}/.clang-format ${TEMP_DIR}/.clang-format CURR_DIR=${PWD} echo "formatting" cd ${TEMP_DIR} ${CLANG_FORMAT:-clang-format} --version ${CLANG_FORMAT:-clang-format} -Wno-error=unknown ${CLANG_FORMAT_FLAGS} ${FILE_NAMES} cd ${CURR_DIR} echo "copying back" for name in ${FILE_NAMES} do if sed -e 's/PG_FUNCTION_ARGS fake_var_for_clang/PG_FUNCTION_ARGS/' ${TEMP_DIR}/${name} > ${TEMP_DIR}/replace_file; then if ! cmp -s ${TEMP_DIR}/replace_file ${BASE_DIR}/${name}; then echo "Updating ${BASE_DIR}/${name}" mv ${TEMP_DIR}/replace_file ${BASE_DIR}/${name} fi fi done exit 0 ================================================ FILE: scripts/cmake_format_all.sh ================================================ #!/bin/bash SCRIPTDIR=$(cd "$(dirname $0)" || exit; pwd) BASEDIR=$(dirname $SCRIPTDIR) find $BASEDIR -name CMakeLists.txt -exec cmake-format -i {} + find $BASEDIR/src $BASEDIR/test $BASEDIR/tsl -name '*.cmake' -exec cmake-format -i {} + ================================================ FILE: scripts/coccinelle.sh ================================================ #!/bin/bash set -o pipefail SCRIPT_DIR=$(dirname "${0}") FAILED=false true > coccinelle.diff for f in "${SCRIPT_DIR}"/../coccinelle/*.cocci; do spatch --very-quiet --include-headers --sp-file "$f" --dir "${SCRIPT_DIR}"/.. | tee -a coccinelle.diff rc=$? if [ $rc -ne 0 ]; then FAILED=true fi done if [ $FAILED = true ] || [ -s coccinelle.diff ] ; then exit 1; fi ================================================ FILE: scripts/delete_released_change_logs.sh ================================================ #!/bin/bash # Check if both old_version and new_version have been provided if [[ -z "$1" || -z "$2" ]]; then echo "Usage: $0 <old_version> <new_version>" exit 1 fi # Assign the values to variables for easier use later OLD_VERSION="$1" NEW_VERSION="$2" # Run git diff and remove the listed files git diff --name-only "$OLD_VERSION" "$NEW_VERSION" .unreleased | grep -v '.unreleased/template.rfc822' | xargs rm ================================================ FILE: scripts/docker-build.sh ================================================ #!/usr/bin/env bash # # This script builds a development TimescaleDB image from the # currently checked out source on the host. # SCRIPT_DIR=$(dirname $0) BASE_DIR=${PWD}/${SCRIPT_DIR}/.. PG_VERSION=${PG_VERSION:-17.4} PG_IMAGE_TAG=${PG_IMAGE_TAG:-${PG_VERSION}-alpine} BUILD_CONTAINER_NAME=${BUILD_CONTAINER_NAME:-pgbuild} BUILD_IMAGE_NAME=${BUILD_IMAGE_NAME:-$USER/pgbuild} IMAGE_NAME=${IMAGE_NAME:-$USER/timescaledb} GIT_ID=$(git -C ${BASE_DIR} describe --dirty --always | sed -e "s|/|_|g") TAG_NAME=${TAG_NAME:-$GIT_ID} BUILD_TYPE=${BUILD_TYPE:-Release} USE_OPENSSL=${USE_OPENSSL:-true} PUSH_PG_IMAGE=${PUSH_PG_IMAGE:-false} GENERATE_DOWNGRADE_SCRIPT=${GENERATE_DOWNGRADE_SCRIPT:-OFF} # Full image identifiers PG_IMAGE=${BUILD_IMAGE_NAME}:${PG_IMAGE_TAG} TS_IMAGE=${IMAGE_NAME}:${TAG_NAME} trap remove_build_container EXIT image_exists() { [[ $(docker image ls -f "reference=$1" -q) ]] } postgres_build_image_exists() { image_exists ${PG_IMAGE} } timescaledb_image_exists() { image_exists ${TS_IMAGE} } remove_build_container() { local container=${1:-${BUILD_CONTAINER_NAME}} docker rm -vf "$(docker ps -a -q -f name=${container} 2>/dev/null)" 2>/dev/null } fetch_postgres_build_image() { local image=${1:-${PG_IMAGE}} docker pull ${image} } create_postgres_build_image() { local image=${1:-${PG_IMAGE}} echo "Creating new PostgreSQL build image ${image}" # Run a Postgres container docker run -d --name ${BUILD_CONTAINER_NAME} --env POSTGRES_HOST_AUTH_METHOD=trust -v ${BASE_DIR}:/src postgres:${PG_IMAGE_TAG} # Install build dependencies docker exec -u root ${BUILD_CONTAINER_NAME} /bin/bash -c "apk add --no-cache --virtual .build-deps postgresql-dev gdb coreutils dpkg-dev gcc git libc-dev make cmake util-linux-dev diffutils libssl3 && mkdir -p /build/debug" docker commit -a $USER -m "TimescaleDB build base image version $PG_IMAGE_TAG" ${BUILD_CONTAINER_NAME} ${image} remove_build_container ${BUILD_CONTAINER_NAME} if ${PUSH_PG_IMAGE}; then docker push ${image} fi } run_postgres_build_image() { local image=${1:-${PG_IMAGE}} echo "Starting image ${image}" docker run -d --name ${BUILD_CONTAINER_NAME} -v ${BASE_DIR}:/src ${image} } build_timescaledb() { echo "Building TimescaleDB image \"${TS_IMAGE}\" with USE_OPENSSL=${USE_OPENSSL} BUILD_TYPE=${BUILD_TYPE}" run_postgres_build_image ${PG_IMAGE} # Build and install the extension with debug symbols and assertions tar -c -C ${BASE_DIR} {cmake,src,sql,test,scripts,tsl,version.config,CMakeLists.txt,timescaledb.control.in} | docker cp - ${BUILD_CONTAINER_NAME}:/build/ if ! docker exec -u root ${BUILD_CONTAINER_NAME} /bin/bash -c " \ cd /build/debug \ && git config --global --add safe.directory /src \ && cmake -DGENERATE_DOWNGRADE_SCRIPT=${GENERATE_DOWNGRADE_SCRIPT} -DUSE_OPENSSL=${USE_OPENSSL} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} /src \ && make -j $(getconf _NPROCESSORS_ONLN) && make install \ && echo \"shared_preload_libraries = 'timescaledb'\" >> /usr/local/share/postgresql/postgresql.conf.sample \ && echo \"timescaledb.telemetry_level=off\" >> /usr/local/share/postgresql/postgresql.conf.sample \ && cd / && rm -rf /build" then echo "Building timescaledb failed" return 1 fi docker commit -a $USER -m "TimescaleDB development image" ${BUILD_CONTAINER_NAME} ${TS_IMAGE} } message_and_exit() { echo echo "Run 'docker run -d --name some-timescaledb -p 5432:5432 ${TS_IMAGE}' to launch" exit } remove_build_container if timescaledb_image_exists; then echo "Image \"${TS_IMAGE}\" already exists." message_and_exit fi if ! postgres_build_image_exists; then if ! fetch_postgres_build_image "$@"; then create_postgres_build_image "$@" || exit 1 fi fi build_timescaledb || exit 1 message_and_exit ================================================ FILE: scripts/docker-run-tests.sh ================================================ #!/usr/bin/env bash # # This script runs the TimescaleDB tests through a standard PostgreSQL # container, first installing the extension via a mounted host volume. # SCRIPT_DIR=$(dirname $0) BASE_DIR=${PWD}/${SCRIPT_DIR}/.. PG_IMAGE_TAG=${PG_IMAGE_TAG:-12.0-alpine} CONTAINER_NAME=${CONTAINER_NAME:-pgtest} case $1 in clean) docker rm -f "$(docker ps -a -q -f name=${CONTAINER_NAME} 2>/dev/null)" 2>/dev/null ;; esac if [ "$(docker ps -q -f name=${CONTAINER_NAME} 2>/dev/null | wc -l)" == "0" ]; then echo "Creating container ${CONTAINER_NAME}" docker rm ${CONTAINER_NAME} 2>/dev/null # Run a Postgres container docker run -u postgres -d --name ${CONTAINER_NAME} -v ${BASE_DIR}:/src postgres:${PG_IMAGE_TAG} # Install build dependencies docker exec -u root -it ${CONTAINER_NAME} /bin/bash -c "apk add --no-cache --virtual .build-deps coreutils dpkg-dev gcc libc-dev make util-linux-dev diffutils cmake bison flex && mkdir -p /build/debug" fi # Copy the source files to build directory docker exec -u root -it ${CONTAINER_NAME} /bin/bash -c "cp -a /src/{src,sql,scripts,test,tsl,CMakeLists.txt,timescaledb.control.in,version.config} /build/ &&cd /build/debug/ && CFLAGS=-Werror cmake .. -DCMAKE_BUILD_TYPE=Debug && make -C /build/debug install && chown -R postgres /build/debug" # Run tests if ! docker exec -u postgres -it ${CONTAINER_NAME} /bin/bash -c "make -C /build/debug installcheck TEST_INSTANCE_OPTS='--temp-instance=/tmp/pgdata --temp-config=/build/test/postgresql.conf'" then docker exec -it ${CONTAINER_NAME} cat /build/debug/test/regression.diffs fi ================================================ FILE: scripts/dump_meta_data.sql ================================================ -- -- This file is licensed under the Apache License, see LICENSE-APACHE -- at the top level directory of the TimescaleDB distribution. -- This script will dump relevant meta data from internal TimescaleDB tables -- that can help our engineers trouble shoot. -- -- usage: -- psql [your connect flags] -d your_timescale_db < dump_meta_data.sql > dumpfile.txt \echo 'TimescaleDB meta data dump' \echo '<exclude_from_test>' \echo 'Date, git commit, and extension version can change without it being an error.' \echo 'Adding this tag allows us to run regression tests on this script file.' select now(); \echo 'Postgres version' select version(); \echo 'Build tag' \set ON_ERROR_STOP 0 SELECT * FROM _timescaledb_internal.get_git_commit(); \set ON_ERROR_STOP 1 \dx \echo '</exclude_from_test>' \echo 'List of tables' SELECT n.nspname as "Schema", c.relname as "Name", CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 't' THEN 'TOAST table' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'partitioned table' WHEN 'I' THEN 'partitioned index' END as "Type", pg_catalog.pg_get_userbyid(c.relowner) as "Owner" FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace LEFT JOIN pg_catalog.pg_am am ON am.oid = c.relam WHERE c.relkind IN ('r','p','') AND n.nspname <> 'pg_catalog' AND n.nspname !~ '^pg_toast' AND n.nspname <> 'information_schema' AND pg_catalog.pg_table_is_visible(c.oid) ORDER BY 1,2; \echo 'List of hypertables' SELECT * FROM _timescaledb_catalog.hypertable; \echo 'List of chunk indexes' SELECT ch.id AS chunk_id, ci.indexrelid::regclass::text AS index_name, h.id AS hypertable_id FROM _timescaledb_catalog.hypertable h JOIN _timescaledb_catalog.chunk ch ON ch.hypertable_id = h.id JOIN pg_index ci ON ci.indrelid = format('%I.%I', ch.schema_name, ch.table_name)::regclass ORDER BY h.id, ch.id, ci.indrelid::regclass::text; \echo 'Size of hypertables' SELECT hypertable, table_bytes, index_bytes, toast_bytes, total_bytes FROM ( SELECT *, total_bytes-index_bytes-COALESCE(toast_bytes,0) AS table_bytes FROM ( SELECT pgc.oid::regclass::text as hypertable, sum(pg_total_relation_size('"' || c.schema_name || '"."' || c.table_name || '"'))::bigint as total_bytes, sum(pg_indexes_size('"' || c.schema_name || '"."' || c.table_name || '"'))::bigint AS index_bytes, sum(pg_total_relation_size(reltoastrelid))::bigint AS toast_bytes FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.chunk c, pg_class pgc, pg_namespace pns WHERE c.hypertable_id = h.id AND pgc.relname = h.table_name AND pns.oid = pgc.relnamespace AND pns.nspname = h.schema_name AND relkind = 'r' GROUP BY pgc.oid ) sub1 ) sub2; \echo 'Chunk sizes:' SELECT chunk_id, chunk_table, partitioning_columns, partitioning_column_types, partitioning_hash_functions, ranges, table_bytes, index_bytes, toast_bytes, total_bytes FROM ( SELECT *, total_bytes-index_bytes-COALESCE(toast_bytes,0) AS table_bytes FROM ( SELECT c.id as chunk_id, '"' || c.schema_name || '"."' || c.table_name || '"' as chunk_table, pg_total_relation_size('"' || c.schema_name || '"."' || c.table_name || '"') AS total_bytes, pg_indexes_size('"' || c.schema_name || '"."' || c.table_name || '"') AS index_bytes, pg_total_relation_size(reltoastrelid) AS toast_bytes, array_agg(d.column_name ORDER BY d.interval_length, d.column_name ASC) as partitioning_columns, array_agg(d.column_type ORDER BY d.interval_length, d.column_name ASC) as partitioning_column_types, array_agg(d.partitioning_func_schema || '.' || d.partitioning_func ORDER BY d.interval_length, d.column_name ASC) as partitioning_hash_functions, array_agg('[' || _timescaledb_functions.range_value_to_pretty(range_start, column_type) || ',' || _timescaledb_functions.range_value_to_pretty(range_end, column_type) || ')' ORDER BY d.interval_length, d.column_name ASC) as ranges FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _timescaledb_catalog.dimension d, _timescaledb_catalog.dimension_slice ds, pg_class pgc, pg_namespace pns WHERE pgc.relname = h.table_name AND pns.oid = pgc.relnamespace AND pns.nspname = h.schema_name AND relkind = 'r' AND c.hypertable_id = h.id AND c.id = cc.chunk_id AND cc.dimension_slice_id = ds.id AND ds.dimension_id = d.id GROUP BY c.id, pgc.reltoastrelid, pgc.oid ORDER BY c.id ) sub1 ) sub2; \echo 'Hypertable index sizes' SET search_path TO pg_catalog, pg_temp; SELECT i.indrelid::regclass AS hypertable, i.indexrelid::regclass AS index_name, public.hypertable_index_size(i.indexrelid::regclass) AS index_bytes FROM _timescaledb_catalog.hypertable h JOIN pg_index i ON i.indrelid = format('%I.%I',h.schema_name,h.table_name)::regclass ORDER BY i.indrelid::regclass::text, i.indexrelid::regclass::text; RESET search_path; ================================================ FILE: scripts/export_prefix_check.sh.in ================================================ #!/bin/bash if [ ! -x "@NM@" ]; then echo "Cannot check export format because nm is not installed" >&2 exit 1 fi DEFINED_ONLY="--defined-only" EXTERN_ONLY="--extern-only" if nm --help | grep -q ' \-defined\-only'; then # The flag can be either --defined-only or -defined-only # depending on nm-llvm version. If there is " -defined-only" # string in the --help output, we assume this is the right # way to specify the flag. DEFINED_ONLY="-defined-only" EXTERN_ONLY="-extern-only" fi # this script outputs all symbols not starting with an allowed prefix # exported symbols are allowed to start with # ts_ for regular timescaledb functions # pg_finfo for metadata defined by PG_FUNCTION_INFO_V1 # we also whitelist a couple of special symbols # _.*_init module startup functions # _.*_fini module shutdown functions # Pg_magic_func used by postgres to check extension compatibility # timescaledb_hello used to test that our name collision resistance works # loader_hello used to test that our name collision resistance works # test_symbol_conflict used to test that our name collision resistance works # tsl_license_update_check used to check for valid license # __sanitizer.* __sancov.* llvm sanitizer # _ZN11__sanitizer.* llvm sanitizer # __asan.* __odr_asan.* address sanitizer # __ubsan.* _ZN7__ubsan.* undefined behaviour sanitizer # internal_sigreturn gcc # OnPrint gcc # backtrace_uncompress_zdebug libbacktrace # reset_fn_list used by llvm code coverage infrastructure # all of these symbols start with an additional leading '_' on macos exports=$(find @CMAKE_BINARY_DIR@ -not -path '*/\.*' -name '*.so' -print0 \ | xargs -0 @NM@ ${DEFINED_ONLY} ${EXTERN_ONLY} \ | sed -e 's:^/.*$::' -e '/^$/d' -e 's/^[a-f0-9]* [A-Za-z] //' \ | grep -v \ -e '^_\?__gcov_' \ -e '^__bss_start$' \ -e '^_edata$' \ -e '^_end$' \ -e '^_fini$' \ -e '^_init$' \ -e '^_\?flush_fn_list$' \ -e '^_\?writeout_fn_list$' \ -e '^_\?lprofDirMode$' \ -e '^_\?mangle_path$' \ -e '^_\?ts_' \ -e '^_\?tsl_license_update_check$' \ -e '^_\?pg_finfo' \ -e '^_\?_.*_init$' \ -e '^_\?_.*_fini$' \ -e '^_\?Pg_magic_func$' \ -e '^_\?timescaledb_' \ -e '^_\?loader_hello$' \ -e '^_\?test_symbol_conflict$' \ -e '^_\?__sanitizer' \ -e '^_\?__sancov' \ -e '^_\?__asan' \ -e '^_\?__odr_asan' \ -e '^_\?__ubsan' \ -e '^_\?_ZN7__ubsan' \ -e '^_\?_ZN11__sanitizer' \ -e '^_\?internal_sigreturn$' \ -e '^_\?OnPrint$' \ -e '^_\?backtrace_uncompress_zdebug$' \ -e '^reset_fn_list$' \ ) num_exports=$(echo "${exports}" | grep -v '^$' -c) echo "${exports}" if [ ${num_exports} -gt 0 ]; then exit 1 else exit 0 fi ================================================ FILE: scripts/githooks/.gitignore ================================================ /__pycache__/ /*.pyc /*~ ================================================ FILE: scripts/githooks/commit_msg.py ================================================ #!/usr/bin/env python3 # Check a Git commit message according to the seven rules of a good commit message: # https://chris.beams.io/posts/git-commit/ import sys class GitCommitMessage: "Represents a parsed Git commit message" rules = [ "Separate subject from body with a blank line", "Limit the subject line to 50 characters", "Capitalize the subject line", "Do not end the subject line with a period", "Use the imperative mood in the subject line", "Wrap the body at 72 characters", "Use the body to explain what and why vs. how", ] valid_rules = [False, False, False, False, False, False, False] def __init__(self, filename=None): lines = [] if filename is not None: with open(filename, "r", encoding="utf-8") as f: for line in f: if line.startswith( "# ------------------------ >8 ------------------------" ): break if not line.startswith("#"): lines.append(line) self.parse_lines(lines) def parse_lines(self, lines): self.body_lines = [] self.subject = [] if not lines or len(lines) == 0: return self self.subject = lines[0] self.subject_words = self.subject.split() self.has_subject_body_separator = False if len(lines) > 1: self.has_subject_body_separator = len(lines[1].strip()) == 0 if self.has_subject_body_separator: self.body_lines = lines[2:] else: self.body_lines = lines[1:] return self def check_subject_body_separtor(self): "Rule 1: Separate subject from body with a blank line" if len(self.body_lines) > 0: return self.has_subject_body_separator return True def check_subject_limit(self): "Rule 2: Limit the subject line to 50 characters" return len(self.subject.rstrip("\n")) <= 50 def check_subject_capitalized(self): "Rule 3: Capitalize the subject line" return len(self.subject) > 0 and self.subject[0].isupper() def check_subject_no_period(self): "Rule 4: Do not end the subject line with a period" return not self.subject.endswith(".") common_first_words = [ "Add", "Adjust", "Support", "Change", "Remove", "Fix", "Print", "Track", "Refactor", "Combine", "Release", "Set", "Stop", "Make", "Mark", "Enable", "Check", "Exclude", "Format", "Correct", ] def check_subject_imperative(self): """Rule 5: Use the imperative mood in the subject line. We can only check for common mistakes here, like using the -ing form of a verb or non-imperative version of common verbs """ firstword = self.subject_words[0] if firstword.endswith("ing"): return False for word in self.common_first_words: if firstword.startswith(word) and firstword != word: return False return True def check_body_limit(self): "Rule 6: Wrap the body at 72 characters" if len(self.body_lines) == 0: return True for line in self.body_lines: if len(line.rstrip("\n")) > 72: return False return True def check_body_uses_why(self): "Rule 7: Use the body to explain what and why vs. how" # Not enforceable return True rule_funcs = [ check_subject_body_separtor, check_subject_limit, check_subject_capitalized, check_subject_no_period, check_subject_imperative, check_body_limit, check_body_uses_why, ] def check_the_seven_rules(self): "validates the commit message against the seven rules" num_violations = 0 for i, func in enumerate(self.rule_funcs): res = func(self) self.valid_rules[i] = res if not res: num_violations += 1 if num_violations > 0: print() print("**** WARNING ****") print() print( "The commit message does not seem to comply with the project's guidelines." ) print('Please try to follow the "Seven rules of a great commit message":') print("https://chris.beams.io/posts/git-commit/") print() print("The following rules are violated:\n") for i in range(len(self.rule_funcs)): if not self.valid_rules[i]: print(f'\t* Rule {i+1}: "{self.rules[i]}"') # Extra sanity checks beyond the seven rules if len(self.body_lines) == 0: print() print("NOTE: the commit message has no body.") print("It is recommended to add a body with a description of your") print( "changes, even if they are small. Explain what and why instead of how:" ) print("https://chris.beams.io/posts/git-commit/#why-not-how") if len(self.subject_words) < 3: print() print("Warning: the subject line has less than three words.") print("Consider using a more explanatory subject line.") if num_violations > 0: print() print("Run 'git commit --amend' to change the commit message") print() return num_violations def main(): if len(sys.argv) != 2: print("Unexpected number of arguments") sys.exit(1) msg = GitCommitMessage(sys.argv[1]) return msg.check_the_seven_rules() if __name__ == "__main__": main() # Always exit with success. We could also fail the commit if with # a non-zero exit code, but that might be a bit excessive and we'd # have to save the failed commit message to a file so that it can # be recovered. sys.exit(0) ================================================ FILE: scripts/githooks/commit_msg_tests.py ================================================ #!/usr/bin/env python3 import unittest from commit_msg import GitCommitMessage class TestCommitMsg(unittest.TestCase): def testNoInput(self): m = GitCommitMessage().parse_lines([]) self.assertEqual(len(m.body_lines), 0) m = GitCommitMessage().parse_lines(None) self.assertEqual(len(m.body_lines), 0) def testParsing(self): m = GitCommitMessage().parse_lines( ["This is a subject line", " ", "body line 1", "body line 2"] ) self.assertEqual(m.subject, "This is a subject line") self.assertTrue(m.has_subject_body_separator) self.assertEqual(m.body_lines[0], "body line 1") self.assertEqual(m.body_lines[1], "body line 2") def testNonImperative(self): m = GitCommitMessage().parse_lines(["Adds new file"]) self.assertFalse(m.check_subject_imperative()) m.parse_lines(["Adding new file"]) self.assertFalse(m.check_subject_imperative()) # Default to accept m.parse_lines(["Foo bar"]) self.assertTrue(m.check_subject_imperative()) def testSubjectBodySeparator(self): m = GitCommitMessage().parse_lines(["This is a subject line"]) self.assertTrue(m.check_subject_body_separtor()) m = GitCommitMessage().parse_lines(["This is a subject line", "body"]) self.assertFalse(m.check_subject_body_separtor()) m = GitCommitMessage().parse_lines(["This is a subject line", "", "body"]) self.assertTrue(m.check_subject_body_separtor()) m = GitCommitMessage().parse_lines(["This is a subject line", " ", "body"]) self.assertTrue(m.check_subject_body_separtor()) def testSubjectLimit(self): m = GitCommitMessage().parse_lines( ["This subject line is exactly 48 characters long"] ) self.assertTrue(m.check_subject_limit()) m = GitCommitMessage().parse_lines( ["This is a very long subject line that will obviously exceed the limit"] ) self.assertFalse(m.check_subject_limit()) m = GitCommitMessage().parse_lines( ["This 50-character subject line ends with an LF EOL\n"] ) self.assertTrue(m.check_subject_limit()) def testSubjectCapitalized(self): m = GitCommitMessage().parse_lines(["This subject line is capitalized"]) self.assertTrue(m.check_subject_capitalized()) m = GitCommitMessage().parse_lines(["this subject line is not capitalized"]) self.assertFalse(m.check_subject_capitalized()) def testSubjectNoPeriod(self): m = GitCommitMessage().parse_lines(["This subject line ends with a period."]) self.assertFalse(m.check_subject_no_period()) m = GitCommitMessage().parse_lines( ["This subject line does not end with a period"] ) self.assertTrue(m.check_subject_no_period()) def testBodyLimit(self): m = GitCommitMessage().parse_lines( ["This is a subject line", " ", "A short body line"] ) self.assertTrue(m.check_body_limit()) m = GitCommitMessage().parse_lines( [ "This is a subject line", " ", "A very long body line which certainly exceeds the 72 char recommended limit", ] ) self.assertFalse(m.check_body_limit()) m = GitCommitMessage().parse_lines( [ "This is a subject line", " ", "A body line with exactly 72 characters, followed by an EOL (Unix-style).\n", ] ) self.assertTrue(m.check_body_limit()) def testCheckAllRules(self): m = GitCommitMessage().parse_lines( ["This is a subject line", "", "A short body line"] ) self.assertEqual(0, m.check_the_seven_rules()) if __name__ == "__main__": unittest.main() ================================================ FILE: scripts/label-released.py ================================================ #!/usr/bin/env python3 """ Look at commits between the given release and the previous one, and label all PRs that made these commits with the "released-..." label. """ import os import sys import argparse import re import subprocess import requests import github # This is PyGithub. import more_itertools OWNER = "timescale" REPO = "timescaledb" TOKEN = os.environ.get("GH_TOKEN") if not TOKEN: print("Specify the GitHub token in GH_TOKEN environment variable.", file=sys.stderr) sys.exit(1) def git_check(cmd: str): subprocess.run(f"git {cmd}", shell=True, check=True) def git_output(cmd: str) -> str: return subprocess.check_output(f"git {cmd}", shell=True, text=True).strip() def run_query(query, params): """A simple function to use requests.post to make the GraphQL API call.""" request = requests.post( "https://api.github.com/graphql", json={"query": query, "variables": params}, headers={"Authorization": f"Bearer {TOKEN}"} if TOKEN else None, timeout=20, ) response = request.json() # Have to work around the unique GraphQL convention of returning 200 for errors. if request.status_code != 200 or "errors" in response: raise ValueError( f"Query failed to run by returning code of {request.status_code}." f"\nQuery: '{query}'" f"\nResponse: '{request.json()}'" ) return response["data"] parser = argparse.ArgumentParser() parser.add_argument("--release", required=True, help="Tag to process, e.g. 2.20.0") parser.add_argument("--dry-run", action="store_true") args = parser.parse_args() target_tag = args.release dry_run = args.dry_run # Create the label if needed gh = github.Github(TOKEN) repo = gh.get_repo(f"{OWNER}/{REPO}") label_name = f"released-{target_tag}" try: pygithub_label = repo.get_label(label_name) label_id = pygithub_label.raw_data["node_id"] except github.UnknownObjectException: if not args.dry_run: pygithub_label = repo.create_label( label_name, "d3d3d3", f"Released in {target_tag}" ) label_id = pygithub_label.raw_data["node_id"] else: label_id = "<dry run>" print(f"Label will be: {label_name}") # Make sure the branches are present in the repository git_check("fetch --depth=1000 origin main:refs/remotes/origin/main") git_check(f"fetch --depth=1000 origin tag {target_tag}") # Read the previous release from version.config. try: cfg = git_output(f"show {target_tag}:version.config") except subprocess.CalledProcessError: sys.exit(f"Error: cannot read version.config at '{target_tag}'") m = re.search(r"^(?:previous_version|update_from_version)\s*=\s*(\S+)", cfg, re.M) if not m: sys.exit("Error: version.config missing previous_version/update_from_version") prev_tag = m.group(1) git_check(f"fetch --depth=1000 origin tag {prev_tag}") print(f"Comparing tags: {prev_tag} → {target_tag}") if dry_run: print("[dry-run] no changes will be made") def fetch_commits_with_prs(starting_commit, cutoff_date): """ Fetch the commits starting from the given one until the cutoff date, with the associated PRs, using paginated GraphQL request. """ GQL_COMMITS = """ query CommitsWithPRs( $owner: String!, $repo: String!, $expr: String!, $since: GitTimestamp!, $first: Int!, $after: String ) { repository(owner:$owner,name:$repo) { object(expression:$expr) { ... on Commit { history(first:$first, since:$since, after:$after) { pageInfo { hasNextPage endCursor } nodes { oid messageHeadline # We only handle the commits with one PR, so fetch at most two. associatedPullRequests(first:2) { nodes { id number title baseRefName } } } } } } } } """ nodes = [] after = None while True: params = { "owner": OWNER, "repo": REPO, "expr": starting_commit, "since": cutoff_date, "first": 100, "after": after, } hist = run_query(GQL_COMMITS, params)["repository"]["object"]["history"] nodes.extend(hist["nodes"]) page_info = hist["pageInfo"] if not page_info["hasNextPage"]: break after = page_info["endCursor"] print("Fetching next page...") print(f"Fetched {len(nodes)} commits with PR info starting from {starting_commit}") return nodes target_release = git_output(f"rev-parse {target_tag}^0") print(f"Target release commit {git_output(f'log -1 --oneline {target_release}')}") prev_release = git_output(f"rev-parse {prev_tag}^0") print(f"Prev release commit {git_output(f'log -1 --oneline {prev_release}')}") prev_release_fork_sha = git_output(f"merge-base {target_release} {prev_release}") print( f"Prev release fork point {git_output(f'log -1 --oneline {prev_release_fork_sha}')}" ) prev_release_fork_date = git_output(f"show -s --format=%cI {prev_release_fork_sha}") print(f"Prev release fork date {prev_release_fork_date}") main_fork_sha = git_output(f"merge-base {target_release} origin/main") print(f"Main fork point {git_output(f'log -1 --oneline {main_fork_sha}')}") main_fork_date = git_output(f"show -s --format=%cI {main_fork_sha}") print(f"Main fork date {main_fork_date}") # This is the relationship between the various refs we've built above: # For a patch release: # (current release branch) -(backports)---prev_release_fork_sha---target_release---> # / ^ ^ ^ # (main) >----main_fork_sha----------------> # # For a minor release: # (prev release branch) prev_release--> # / # (current release branch) / -(backports)-target_release--> # / / ^ ^ ^ # (main) >-prev_release_fork_sha---main_fork_sha---------------> # # We can't use the commit SHAs for the commit lookups due to API limitations, so # we use the dates instead. # Now, perform the lookups for release branch commits and the potentially # backported main commits with the respective PRs. branch_commit_nodes = fetch_commits_with_prs(target_release, prev_release_fork_date) main_commit_nodes = fetch_commits_with_prs("main", main_fork_date) # We're going to match the commits by title to account for backports. main_commit_title_to_pr = { node["messageHeadline"]: prs[0] for node in main_commit_nodes if (prs := node["associatedPullRequests"]["nodes"]) and len(prs) == 1 } print(f"On main ({len(main_commit_title_to_pr)} PRs):") print( "\n".join( [ f'#{pr["number"]}: {pr["title"]} <- {title}' for title, pr in main_commit_title_to_pr.items() ] ) ) print() branch_commit_title_to_pr = { node["messageHeadline"]: prs[0] for node in branch_commit_nodes if (prs := node["associatedPullRequests"]["nodes"]) and len(prs) == 1 } print(f"On branch ({len(branch_commit_title_to_pr)} PRs):") print( "\n".join( [ f'#{pr["number"]}: {pr["title"]} <- {title}' for title, pr in branch_commit_title_to_pr.items() ] ) ) print() # The commits with same titles in main and release branch, but with different PRs, # are backported commits. We have to label the PR to main in that case, and not # the backport PR. backported_titles = { title: pr for title, pr in main_commit_title_to_pr.items() if title in branch_commit_title_to_pr and pr["number"] != branch_commit_title_to_pr[title]["number"] } print(f"Backported ({len(backported_titles)} PRs):") print( "\n".join([f'#{pr["number"]}: {pr["title"]}' for pr in backported_titles.values()]) ) print() branch_commit_title_to_pr.update(backported_titles) print(f"To label as {label_name} ({len(branch_commit_title_to_pr)} PRs):") print( "\n".join( [f'#{pr["number"]}: {pr["title"]}' for pr in branch_commit_title_to_pr.values()] ) ) print() # Label the PRs in bulk using GraphQL. ids = list({pr["id"] for pr in branch_commit_title_to_pr.values()}) for chunk in more_itertools.chunked(ids, 10): parts = [ f'p{j}: addLabelsToLabelable(input: {{labelableId:"{nid}",labelIds:["{label_id}"]}}) {{ clientMutationId }}' for j, nid in enumerate(chunk) ] gql = "mutation BulkLabel {\n" + "\n".join(parts) + "\n}" if dry_run: print(f"\nDry-run for {len(chunk)} PRs:\n{gql}") else: run_query(gql, {}) print(f"Labeled {len(chunk)} PRs.") ================================================ FILE: scripts/license_apache.spec ================================================ # This file and its contents are licensed under the Apache License 2.0. # Please see the included NOTICE for copyright information and # LICENSE-APACHE for a copy of the license. ================================================ FILE: scripts/license_tsl.spec ================================================ # This file and its contents are licensed under the Timescale License. # Please see the included NOTICE for copyright information and # LICENSE-TIMESCALE for a copy of the license. ================================================ FILE: scripts/memory_leaks.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. DROP DATABASE IF EXISTS test; CREATE DATABASE test; \c test \timing DROP EXTENSION IF EXISTS timescaledb CASCADE; CREATE EXTENSION timescaledb; SELECT pg_backend_pid(); SELECT pg_sleep(5); CREATE TABLE metrics(time timestamptz, device text, value float); SELECT create_hypertable('metrics', 'time'); INSERT INTO metrics SELECT '2020-01-01'::timestamptz + format('%ss',time)::interval, format('Device %s',time%10), random() FROM generate_series(1,1000000) g(time); \o /dev/null \set ECHO none \timing off \echo Starting 1 million SELECT queries SELECT format($$SELECT * FROM metrics WHERE device = 'Device %s' ORDER BY time DESC LIMIT 1;$$,time % 10) FROM generate_series(1,1000000) g(time) \gexec \echo Starting 10k UPDATE queries SELECT format($$UPDATE metrics SET value = %s WHERE time < '2020-01-02' AND device = 'Device %s';$$,time,time % 10) FROM generate_series(1,10000) g(time) \gexec \echo Starting 50k INSERTs with batches of 1000 SELECT format($$INSERT INTO metrics SELECT '2021-01-01'::timestamptz + '%ss'::interval + format('%%s', time)::interval, format('Device %%s',time %% 10), random() FROM generate_series(1,1000) g(time);$$,time) FROM generate_series(1,50000) g(time) \gexec ================================================ FILE: scripts/out_of_order_random_direct.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. DROP DATABASE IF EXISTS test; CREATE DATABASE test; \c test \timing DROP EXTENSION IF EXISTS timescaledb CASCADE; CREATE EXTENSION timescaledb; SELECT pg_backend_pid(); CREATE TABLE test_hyper (i bigint, j double precision, k bigint, ts timestamp); SELECT create_hypertable('test_hyper', 'i', chunk_time_interval=>10); INSERT INTO test_hyper SELECT random()*100, random()*100, x*10, _timescaledb_functions.to_timestamp(x) FROM generate_series(1,100000000) AS x; ================================================ FILE: scripts/perltidy_format_all.sh ================================================ #!/bin/bash SCRIPTDIR=$(cd "$(dirname $0)" || exit; pwd) BASEDIR=$(dirname $SCRIPTDIR) PERLTIDY=${PERLTIDY:-perltidy} PERLTIDY_CONFIG=${PERLTIDY_CONFIG:-$BASEDIR/.perltidyrc} if [ ! -f "$PERLTIDY_CONFIG" ]; then echo "perltidy config '$PERLTIDY_CONFIG' not found" exit 1 fi find "$BASEDIR"/test -name '*.p[lm]' -exec "$PERLTIDY" --profile="$PERLTIDY_CONFIG" -b -bext=/ {} + find "$BASEDIR"/tsl/test -name '*.p[lm]' -exec "$PERLTIDY" --profile="$PERLTIDY_CONFIG" -b -bext=/ {} + ================================================ FILE: scripts/release/build_post_release_artefacts.sh ================================================ #!/bin/bash # # Build the necessary artefacts for the forwardporting the # the release to `main` (Minor & Patch) # - up & down files for the next version # - latest.sql & reverse.sql # - sets new version in version.config # - adjusts the CMakeLists.txt # # Param: <published_version> # set -eu if [ "$#" -ne 1 ]; then echo "${0} <published_version>" exit 2 fi PUBLISHED_VERSION=$1 PREVIOUS_VERSION=$(tail -1 version.config | cut -d ' ' -f 3 | cut -d '-' -f 1) echo "fetch the up & downgrade files" URL_UPGRADE="https://raw.githubusercontent.com/timescale/timescaledb/refs/tags/${PUBLISHED_VERSION}/sql/updates/${PREVIOUS_VERSION}--${PUBLISHED_VERSION}.sql" URL_DOWNGRADE="https://raw.githubusercontent.com/timescale/timescaledb/refs/tags/${PUBLISHED_VERSION}/sql/updates/${PUBLISHED_VERSION}--${PREVIOUS_VERSION}.sql" echo $URL_UPGRADE curl --fail -o sql/updates/${PREVIOUS_VERSION}--${PUBLISHED_VERSION}.sql $URL_UPGRADE curl --fail -o sql/updates/${PUBLISHED_VERSION}--${PREVIOUS_VERSION}.sql $URL_DOWNGRADE # find last up & down grade files LAST_UPDATE_FILE=$(find sql/updates/*--${PREVIOUS_VERSION}.sql | head -1 | cut -d '/' -f 3) LAST_DOWNGRADE_FILE=$(find sql/updates/${PREVIOUS_VERSION}--*.sql | head -1 | cut -d '/' -f 3) echo "register upgrade sql file" gawk -i inplace '/'${LAST_UPDATE_FILE}')/ { print; print " updates/'${PREVIOUS_VERSION}'--'${PUBLISHED_VERSION}'.sql)"; next }1' ./sql/CMakeLists.txt sed -i.bak "s/${LAST_UPDATE_FILE})/${LAST_UPDATE_FILE}/g" ./sql/CMakeLists.txt echo "register downgrade sql file" gawk -i inplace '/'${LAST_DOWNGRADE_FILE}')/ { print; print " '${PUBLISHED_VERSION}'--'${PREVIOUS_VERSION}'.sql)"; next }1' ./sql/CMakeLists.txt sed -i.bak "s/${LAST_DOWNGRADE_FILE})/${LAST_DOWNGRADE_FILE}/g" ./sql/CMakeLists.txt # CHANGELOG echo "fetching CHANGELOG" URL_CHANGELOG="https://raw.githubusercontent.com/timescale/timescaledb/refs/tags/${PUBLISHED_VERSION}/CHANGELOG.md" curl --silent --fail -o CHANGELOG.md $URL_CHANGELOG # .unreleased files echo "remove .unreleased files" #git diff ${PUBLISHED_VERSION}..main .unreleased/* | xargs rm -v # Set new previous version echo "Set new previous version version.config to ${PUBLISHED_VERSION}" sed -i '' -e "s/^previous_version = .*/previous_version = $PUBLISHED_VERSION/" version.config tail -n 1 version.config ================================================ FILE: scripts/release/build_release_artefacts.sh ================================================ #!/bin/bash # # Build the necessary artefacts for the next release (Minor & Patch) # - up & down files for the next version # - latest.sql & reverse.sql # - sets new version in version.config # - adjusts the CMakeLists.txt # # Param: <current_version> # Param: <nextt_version> # set -eu if [ "$#" -ne 2 ]; then echo "${0} <current_version> <next_version>" exit 2 fi CURRENT_VERSION=$1 NEXT_VERSION=$2 UPDATE_FILE="$CURRENT_VERSION--$NEXT_VERSION.sql" DOWNGRADE_FILE="$NEXT_VERSION--$CURRENT_VERSION.sql" LAST_UPDATE_FILE=$(find sql/updates/*--${CURRENT_VERSION}.sql | head -1 | cut -d '/' -f 3) LAST_DOWNGRADE_FILE=$(find sql/updates/${CURRENT_VERSION}--*.sql | head -1 | cut -d '/' -f 3) # prepare next up & down files echo "generate up and downgrade files" cp ./sql/updates/latest-dev.sql ./sql/updates/$UPDATE_FILE cp ./sql/updates/reverse-dev.sql ./sql/updates/$DOWNGRADE_FILE echo "truncate dev up & downgrade paths" truncate -s 0 ./sql/updates/latest-dev.sql truncate -s 0 ./sql/updates/reverse-dev.sql # CMakeLists echo "register upgrade sql file" gawk -i inplace '/'${LAST_UPDATE_FILE}')/ { print; print " updates/'${UPDATE_FILE}')"; next }1' ./sql/CMakeLists.txt sed -i.bak "s/${LAST_UPDATE_FILE})/${LAST_UPDATE_FILE}/g" ./sql/CMakeLists.txt echo "register downgrade sql file" gawk -i inplace '/'${LAST_DOWNGRADE_FILE}')/ { print; print " '${DOWNGRADE_FILE}')"; next }1' ./sql/CMakeLists.txt sed -i.bak "s/${LAST_DOWNGRADE_FILE})/${LAST_DOWNGRADE_FILE}/g" ./sql/CMakeLists.txt echo "register reverse path" sed -i.bak "s/FILE reverse-dev.sql)/FILE ${DOWNGRADE_FILE})/g" ./sql/CMakeLists.txt # Set only next minor release version in version.config # and create this as a separate PR on `main` echo "Set next minor release version.config" sed -i.bak "s/${NEXT_VERSION}-dev/${NEXT_VERSION}/g" version.config rm version.config.bak head -n1 version.config ================================================ FILE: scripts/release/create_minor_release_branch.sh ================================================ #!/bin/bash set -eu # Folder, where we have cloned repositories' sources SOURCES_DIR="timescaledb" FORK_DIR="timescaledb" echo "---- Deriving the release related versions from main ----" cd ~/"$SOURCES_DIR"/"$FORK_DIR" git fetch --all NEW_PATCH_VERSION="0" NEW_VERSION=$(head -1 version.config | cut -d ' ' -f 3 | cut -d '-' -f 1) RELEASE_BRANCH="${NEW_VERSION/%.$NEW_PATCH_VERSION/.x}" echo "RELEASE_BRANCH is $RELEASE_BRANCH" echo "NEW_VERSION is $NEW_VERSION" echo "---- Creating the version branch from main ----" git fetch --all git checkout -b "$RELEASE_BRANCH" origin/main git push origin "$RELEASE_BRANCH":"$RELEASE_BRANCH" ================================================ FILE: scripts/release/ready_fork_for_commit.sh ================================================ #!/bin/bash set -eu echo "---- Setting git user parameters as per current global git configuration ----" # GITHUB_USERNAMES GH_EMAIL=$(git config user.email) GH_FULL_USERNAME=$(git config user.name) GH_USERNAME=$(gh auth status | grep 'Logged in to' |cut -d ' ' -f 9) echo "GH_EMAIL is $GH_EMAIL" echo "GH_FULL_USERNAME is $GH_FULL_USERNAME" echo "GH_USERNAME is $GH_USERNAME" # Folder, where we have cloned repositories' sources SOURCES_DIR="sources" # Derived Variables FORK_DIR="$GH_USERNAME-timescaledb" echo "---- Updating fork with upstream for user $GH_USERNAME ----" gh repo sync "$GH_USERNAME/timescaledb" -b main echo "---- Cloning the fork to $FORK_DIR ----" cd cd "$SOURCES_DIR" rm -rf "$FORK_DIR" git clone git@github.com:"$GH_USERNAME"/timescaledb.git "$FORK_DIR" cd "$FORK_DIR" git branch git pull && git diff HEAD git log -n 2 echo "---- Configuring the fork for commit ----" git config user.name "$GH_FULL_USERNAME" git config user.email "$GH_EMAIL" git remote add upstream git@github.com:timescale/timescaledb.git git config -l git remote -v echo "---- Updating tags from upstream on the fork ----" git fetch --tags upstream git push --tags origin main # Check the needed branch name here - could it be 2.14.x ? # git push -f --tags origin main ================================================ FILE: scripts/run_sql.sh ================================================ #!/usr/bin/env bash # To avoid pw writing, add localhost:5432:*:postgres:test to ~/.pgpass set -u set -e PWD=$(pwd) DIR=$(dirname $0) export PGUSER=${PGUSER:-postgres} export PGHOST=${PGHOST:-localhost} export PGDATABASE=${PGDATABASE:-timescaledb} psql -v ON_ERROR_STOP=1 -q -X -f $DIR/sql/$1 ================================================ FILE: scripts/shellcheck-ci.sh ================================================ #!/bin/bash set -e find ./* .github -type f -executable \ -exec bash -c '[ "$(file --brief --mime-type "$1")" == "text/x-shellscript" ]' sh {} \; \ -not -exec shellcheck -x --exclude=SC2086 {} \; \ -print \ | grep . \ && exit 1 \ || exit 0 ================================================ FILE: scripts/sql_license_apache.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. ================================================ FILE: scripts/sql_license_tsl.sql ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. ================================================ FILE: scripts/start-local-docker.ps1 ================================================ #requires -Version 5.1 $ErrorActionPreference = "Stop" # --- Configuration --- $IMAGE_NAME = "timescale/timescaledb-ha:pg18" $CONTAINER_NAME = "timescaledb-ha-pg18-quickstart" $DB_PASSWORD = "password" $DB_PORT = 6543 $VOLUME_NAME = "timescaledb_data" # Catch-all so users see the error and return to command prompt trap { Write-Host "" Write-Host ("[ERROR] " + $_.Exception.Message) -ForegroundColor Red return # Return to command prompt instead of exiting } function Fail($msg) { throw $msg # IMPORTANT: do NOT use exit here; it kills the host instantly when invoked via iwr|iex } function Info($msg) { Write-Host "[INFO] $msg" -ForegroundColor Cyan } function Success($msg) { Write-Host "[SUCCESS] $msg" -ForegroundColor Green } function Test-Command($name) { return [bool](Get-Command $name -ErrorAction SilentlyContinue) } Clear-Host Write-Host "" Write-Host " _____ _ __ ____ ____ " -ForegroundColor Yellow Write-Host " |_ _(_)_ __ ___ ___ ___ ___ __ _| | ___| _ \| __ ) " -ForegroundColor Yellow Write-Host " | | | | '_ \` _ \ / _ \/ __|/ __/ _\` | |/ _ \ | | | _ \ " -ForegroundColor Yellow Write-Host " | | | | | | | | | __/\__ \ (_| (_| | | __/ |_| | |_) |" -ForegroundColor Yellow Write-Host " |_| |_|_| |_| |_|\___||___/\___\__,_|_|\___|____/|____/ " -ForegroundColor Yellow Write-Host "" Write-Host "1. System Check" -ForegroundColor White if (-not (Test-Command docker)) { Fail "Docker is not found. Install Docker Desktop first.`nDownload Docker: https://www.docker.com/get-started/" } Success "Docker found" try { docker info *> $null } catch { Fail "Docker is installed but not running. Start Docker Desktop and try again.`nDownload Docker: https://www.docker.com/get-started/" } Success "Docker is running" # Cleanup old container $existing = (docker ps -aq -f "name=^/$CONTAINER_NAME$").Trim() if ($existing) { Info "Found existing container. Removing it .." docker rm -f $CONTAINER_NAME *> $null Success "Removed existing container" } Write-Host "" Write-Host "2. Deployment" -ForegroundColor White Info "Pulling image ($IMAGE_NAME) .." docker pull $IMAGE_NAME | Out-Host Success "Image pulled ($IMAGE_NAME)" Info "Starting TimescaleDB on port $DB_PORT .." docker run -d ` --name $CONTAINER_NAME ` -p "$DB_PORT`:5432" ` -e "POSTGRES_PASSWORD=$DB_PASSWORD" ` -v "$VOLUME_NAME`:/home/postgres/pgdata/data" ` $IMAGE_NAME *> $null Success "Container started ($CONTAINER_NAME)" # Wait for readiness Info "Waiting for database to accept connections .." $retries = 30 $ready = $false for ($i=0; $i -lt $retries; $i++) { try { docker exec $CONTAINER_NAME pg_isready -U postgres *> $null if ($LASTEXITCODE -eq 0) { $ready = $true; break } } catch {} Start-Sleep -Seconds 1 } if (-not $ready) { Fail "Database failed to start in time. Check logs with: docker logs $CONTAINER_NAME" } Success "Database is ready!" # Get versions without host psql: use psql inside container $tsdbVersion = (docker exec $CONTAINER_NAME psql -U postgres -t -A -c "select extversion from pg_extension where extname='timescaledb';" 2>$null).Trim() $pgVersion = (docker exec $CONTAINER_NAME psql -U postgres -t -A -c "select split_part(version(),' ',2);" 2>$null).Trim() Write-Host "" Write-Host "╔══════════════════════════════════════════════════════════╗" -ForegroundColor Green Write-Host "║ 🚀 SETUP COMPLETED SUCCESSFULLY ║" -ForegroundColor Green Write-Host "╚══════════════════════════════════════════════════════════╝" -ForegroundColor Green Write-Host "" Write-Host (" Postgres: {0}" -f ($(if ($pgVersion) { $pgVersion } else { "(unknown)" }))) Write-Host (" TimescaleDB: {0}" -f ($(if ($tsdbVersion) { $tsdbVersion } else { "(unknown)" }))) Write-Host "" Write-Host " Container: $CONTAINER_NAME" Write-Host " Port: $DB_PORT" Write-Host " User: postgres" Write-Host " Password: $DB_PASSWORD" Write-Host "" Write-Host " Connect:" Write-Host " psql \"postgres://postgres:$DB_PASSWORD@localhost:$DB_PORT/postgres\"" Write-Host "" Write-Host " Getting Started:" Write-Host " * Quick start: https://tsdb.co/quick-start" Write-Host " * NYC taxis: https://tsdb.co/quick-start-nyc-taxis" Write-Host " * S&P 500 stocks: https://tsdb.co/quick-start-sp500-stocks" Write-Host " * Sensor devices: https://tsdb.co/quick-start-sensors" Write-Host "" # Success: optionally pause if launched in a transient window # Pause-OnExit 0 ================================================ FILE: scripts/start-local-docker.sh ================================================ #!/bin/sh set -e # # Getting Started script with Docker # # URL: https://tsdb.co/start-local # # --- Configuration --- IMAGE_NAME="timescale/timescaledb-ha:pg18" CONTAINER_NAME="timescaledb-ha-pg18-quickstart" DB_PASSWORD="password" DB_PORT="6543" # Colors RESET="\033[0m" BOLD="\033[1m" COLOR_GREEN="\033[32m" COLOR_BLUE="\033[34m" COLOR_RED="\033[31m" COLOR_CYAN="\033[36m" COLOR_YELLOW="\033[33m" # --- Formatting Helpers --- # Hide cursor for cleaner UI, restore on exit cleanup() { tput cnorm # restore cursor } trap cleanup EXIT INT TERM success() { printf "%b✔%b %s\n" "${COLOR_GREEN}" "${RESET}" "$1"; } error() { printf "%b✖%b %s\n" "${COLOR_RED}" "${RESET}" "$1"; exit 1; } # --- Spinner Function --- # Usage: Run command in background, then call: spinner $! "Loading text..." spinner() { local pid=$1 local text=$2 local delay=0.1 local spinstr='|/-\' # Hide cursor tput civis printf "%s " "$text" while kill -0 "$pid" 2>/dev/null; do # Loop through spin string characters # Note: POSIX sh handles string slicing differently, so we use a simpler approach printf "\b%s" "|" sleep $delay printf "\b%s" "/" sleep $delay printf "\b%s" "-" sleep $delay printf "\b%s" "\\" sleep $delay done # Clear spinner character printf "\b " # Restore cursor tput cnorm } header() { clear echo "" printf "%b _____ _ __ ____ ____ %b\n" "${COLOR_YELLOW}" "${RESET}" printf "%b |_ _(_)_ __ ___ ___ ___ ___ __ _| | ___| _ \| __ ) %b\n" "${COLOR_YELLOW}" "${RESET}" printf "%b | | | | '_ \` _ \ / _ \/ __|/ __/ _\` | |/ _ \ | | | _ \ %b\n" "${COLOR_YELLOW}" "${RESET}" printf "%b | | | | | | | | | __/\__ \ (_| (_| | | __/ |_| | |_) |%b\n" "${COLOR_YELLOW}" "${RESET}" printf "%b |_| |_|_| |_| |_|\___||___/\___\__,_|_|\___|____/|____/ %b\n" "${COLOR_YELLOW}" "${RESET}" echo "" } get_timescaledb_version() { psql "postgres://postgres:$DB_PASSWORD@localhost:$DB_PORT/postgres" \ -c "select extversion from pg_extension where extname = 'timescaledb';" \ | awk 'NR==3 {print $1}' } get_postgres_version() { psql "postgres://postgres:$DB_PASSWORD@localhost:$DB_PORT/postgres" \ -t \ -c "select version();" \ | awk '{print $2}' } # --------------------------------------------------------------- # header # --- 1. Check for Docker --- printf "%b1. System Check%b\n" "${BOLD}" "${RESET}" if ! command -v docker > /dev/null 2>&1; then error "Docker is not found. Please install Docker Desktop (Windows/Mac) or Docker Engine (Linux) first." else success "Docker found" fi if ! docker info > /dev/null 2>&1; then error "Docker is installed but not running. Please start Docker and try again." else success "Docker is running" fi # Cleanup Old Containers if [ "$(docker ps -aq -f name=^/${CONTAINER_NAME}$)" ]; then success "Found existing container. Removing it .." docker rm -f $CONTAINER_NAME > /dev/null fi # --- 2. Pull and Run --- echo "" printf "%b2. Deployment%b\n" "${BOLD}" "${RESET}" # Start pull in background printf "%b::%b " "${COLOR_CYAN}" "${RESET}" docker pull -q $IMAGE_NAME > /dev/null 2>&1 & PID=$! # Run spinner attached to that PID spinner $PID "Pulling image ($IMAGE_NAME) .." # Wait for PID to capture exit code wait $PID EXIT_CODE=$? if [ $EXIT_CODE -eq 0 ]; then printf "\r%b::%b Image pulled ($IMAGE_NAME) \n" "${COLOR_GREEN}" "${RESET}" printf "%b✔%b Success\n" "${COLOR_GREEN}" "${RESET}" else printf "\r%b✖%b Failed to pull image.\n" "${COLOR_RED}" "${RESET}" exit 1 fi success "Starting TimescaleDB on port $DB_PORT\n" docker run -d \ --name "$CONTAINER_NAME" \ -p ${DB_PORT}:5432 \ -e POSTGRES_PASSWORD="$DB_PASSWORD" \ -v ${CONTAINER_NAME}_data:/home/postgres/pgdata/data \ "$IMAGE_NAME" > /dev/null # --- 4. Wait for Healthcheck --- printf "%b::%b Waiting for database to accept connections .. " "${COLOR_CYAN}" "${RESET}" RETRIES=30 SUCCESS=false # We use a custom loop here to animate the spinner while sleeping tput civis # Hide cursor while [ $RETRIES -gt 0 ]; do # Check DB if docker exec $CONTAINER_NAME pg_isready -U postgres > /dev/null 2>&1; then SUCCESS=true break fi # Animate spinner for 1 second (4 * 0.25s) for i in 1 2 3 4; do printf "\b|" ; sleep 0.1 printf "\b/" ; sleep 0.1 printf "\b-" ; sleep 0.1 printf "\b\\" ; sleep 0.1 done RETRIES=$((RETRIES-1)) done tput cnorm # Restore cursor if [ $RETRIES -eq 0 ]; then printf "\n" error "Database failed to start in time. Check logs with: docker logs $CONTAINER_NAME" fi # --- 5. Success --- if [ "$SUCCESS" = true ]; then printf "\r%b::%b Database is ready! \n" "${COLOR_GREEN}" "${RESET}" echo "" TSDB_VERSION=$(get_timescaledb_version) POSTGRES_VERSION=$(get_postgres_version) printf "${COLOR_GREEN}╔══════════════════════════════════════════════════════════╗${RESET}" echo "" printf "${COLOR_GREEN}║ 🚀 SETUP COMPLETED SUCCESSFULLY ║${RESET}" echo "" printf "${COLOR_GREEN}╚══════════════════════════════════════════════════════════╝${RESET}" echo "" printf " %bPostgres:%b $POSTGRES_VERSION\n" "${BOLD}" "${RESET}" printf " %bTimescaleDB:%b $TSDB_VERSION\n" "${BOLD}" "${RESET}" echo "" printf " %bContainer:%b $CONTAINER_NAME\n" "${BOLD}" "${RESET}" printf " %bPort:%b $DB_PORT\n" "${BOLD}" "${RESET}" printf " %bUser:%b postgres\n" "${BOLD}" "${RESET}" printf " %bPassword:%b $DB_PASSWORD\n" "${BOLD}" "${RESET}" echo "" printf "%b Connect:%b\n" "${BOLD}" "${RESET}" echo " psql \"postgres://postgres:$DB_PASSWORD@localhost:$DB_PORT/postgres\"" echo "" printf "%b Getting Started:%b\n" "${BOLD}" "${RESET}" echo " * Examples: https://tsdb.co/quick-start-examples" echo " * NYC taxis: https://tsdb.co/quick-start-nyc-taxis" echo " * S&P 500 stocks: https://tsdb.co/quick-start-sp500-stocks" echo " * Events: https://tsdb.co/quick-start-events-uuid" echo "" else printf "\r%b✖%b Database timed out.\n" "${COLOR_RED}" "${RESET}" echo " Check logs with: docker logs $CONTAINER_NAME" exit 1 fi ================================================ FILE: scripts/suppressions/README.md ================================================ # Suppressions for Clang Sanitizers # This folder contains [supression files](https://clang.llvm.org/docs/SanitizerSpecialCaseList.html) for running timescale using Clang's [AddressSanitizer](https://clang.llvm.org/docs/AddressSanitizer.html) and [UndefinedBehaviorSanitizer](https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html), which we use as part of timescale's regission suite. There are a few places OSs have UB and where postgres has benign memory leaks, in order to run these sanitizers, we suppress these warning. For ease of use, we provide a script in [/scripts/test_sanitizers.sh] to run our regression tests with sanitizers enabled. ================================================ FILE: scripts/suppressions/suppr_asan.txt ================================================ #suppress getaddrinfo due to internal error on macos interceptor_via_fun:getaddrinfo ================================================ FILE: scripts/suppressions/suppr_leak.txt ================================================ leak:save_ps_display_args leak:psql/startup.c #don't care about the frontend leaks leak:fe_memutils.c leak:fe-connect.c leak:fe-exec.c leak:initdb.c #annoingly have to supress strdup because it leaks in PostmasterMain -D option leak:strdup leak:ProcessConfigFileInternal leak:internal_load_library leak:pg_timezone_abbrev_initialize leak:check_timezone_abbreviations leak:check_session_authorization leak:InitializeGUCOptions leak:CheckMyDatabase #pg_dump leak:getSchemaData leak:dumpDumpableObject #should live as long as the process leak:ShmemInitHash #test only functions leak:deserialize_test_parameters leak:ts_params_get leak:test_job_dispatcher #openssl leak:CRYPTO_zalloc ================================================ FILE: scripts/suppressions/suppr_ub.txt ================================================ alignment:pg_comp_crc32c_sse42 alignment:array_cmp alignment:array_iter_setup alignment:array_out alignment:array_unnest alignment:array_eq alignment:AllocSetCheck nonnull-attribute:TransactionIdSetPageStatus nonnull-attribute:SerializeTransactionState nonnull-attribute:initscan nonnull-attribute:SetTransactionSnapshot nonnull-attribute:shm_mq_receive #care, gcc cannot parse the path, only the filename nonnull-attribute:*/src/fe_utils/print.c nonnull-attribute:print_aligned_text #PG13.3-copyfuncs.c:2374:2: runtime error: null pointer passed as argument 2, which is declared to never be null nonnull-attribute:_copyAppendRelInfo #PG13.3-copyfuncs.c:1190:2: runtime error: null pointer passed as argument 2, which is declared to never be null nonnull-attribute:_copyLimit nonnull-attribute:*/src/backend/nodes/copyfuncs.c #division by 0, looks like a real postgres ug float-divide-by-zero:_bt_vacuum_needs_cleanup # It complains about casting -nan to int in histogram_test.sql. It is in the # postgres code and doesn't lead to bugs, because it is caught by a check later. float-cast-overflow:width_bucket_float8 ================================================ FILE: scripts/test_downgrade.sh ================================================ #!/bin/bash set -e SCRIPT_DIR=$(dirname $0) PG_MAJOR_VERSION=$(pg_config --version | awk '{print $2}' | awk -F. '{print $1}') PG_EXTENSION_DIR=$(pg_config --sharedir)/extension if [ "${CI:-false}" == true ]; then GIT_REF=${GIT_REF:-$(git rev-parse HEAD)} else GIT_REF=$(git branch --show-current) fi BUILD_DIR="build_update_pg${PG_MAJOR_VERSION}" CURRENT_VERSION=$(grep '^version ' version.config | awk '{ print $3 }') PREV_VERSION=$(grep '^previous_version ' version.config | awk '{ print $3 }') if [ ! -d "${BUILD_DIR}" ]; then echo "Initializing build directory" BUILD_DIR="${BUILD_DIR}" ./bootstrap -DCMAKE_BUILD_TYPE=Release -DWARNINGS_AS_ERRORS=OFF -DASSERTIONS=ON -DLINTER=ON -DGENERATE_DOWNGRADE_SCRIPT=ON -DREGRESS_CHECKS=OFF -DTAP_CHECKS=OFF fi if [ ! -f "${PG_EXTENSION_DIR}/timescaledb--${PREV_VERSION}.sql" ]; then echo "Building ${PREV_VERSION}" git checkout ${PREV_VERSION} make -C "${BUILD_DIR}" -j "$(getconf _NPROCESSORS_ONLN)" > /dev/null sudo make -C "${BUILD_DIR}" install > /dev/null git checkout ${GIT_REF} fi # We want to use the latest loader for all the tests so we build it last make -C "${BUILD_DIR}" -j "$(getconf _NPROCESSORS_ONLN)" sudo make -C "${BUILD_DIR}" install > /dev/null set +e FROM_VERSION=${CURRENT_VERSION} TO_VERSION=${PREV_VERSION} TEST_REPAIR=false "${SCRIPT_DIR}/test_update_from_version.sh" return_code=$? if [ $return_code -ne 0 ]; then echo -e "\nFailed downgrade from ${CURRENT_VERSION} to ${PREV_VERSION}\n" exit 1 fi ================================================ FILE: scripts/test_license.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. ================================================ FILE: scripts/test_memory_spikes.py ================================================ #!/usr/bin/env python # This file and its contents are licensed under the Apache License 2.0. # Please see the included NOTICE for copyright information and # LICENSE-APACHE for a copy of the license. # Python script to check if there are memory spikes when running queries import psutil import time import sys from datetime import datetime DEFAULT_MEMCAP = 300 # in MB THRESHOLD_RATIO = 1.5 # ratio above which considered memory spike WAIT_TO_STABILIZE = 15 # wait in seconds before considering memory stable CHECK_INTERVAL = 15 DEBUG = False # finds processes with name as argument def find_procs_by_name(name): # Return a list of processes matching 'name' ls = [] for p in psutil.process_iter(): if p.name() == name: ls.append(p) return ls # return human readable form of number of bytes n def bytes2human(n): # http://code.activestate.com/recipes/578019 # >>> bytes2human(10000) # '9.8K' # >>> bytes2human(100001221) # '95.4M' symbols = ("K", "M", "G", "T", "P", "E", "Z", "Y") prefix = {} for i, s in enumerate(symbols): prefix[s] = 1 << (i + 1) * 10 for s in reversed(symbols): if n >= prefix[s]: value = float(n) / prefix[s] return f"{value:.1f}{s}" return f"{n}B" # prints pid of processes def process_details(process): return f"{process.pid} {''.join(process.cmdline()).strip()}" def process_stats(): processes = find_procs_by_name("postgres") for p in processes: print(p, p.num_ctx_switches(), p.cpu_times(), p.memory_info(), flush=True) # return process id of new postgres process created when running SQL file def find_new_process(): # get postgres processes that are running before insertion starts base_process = find_procs_by_name("postgres") print("Processes running before inserts run: ") for p in base_process: print(process_details(p)) process_count = len(base_process) print( f"Waiting {WAIT_TO_STABILIZE} seconds for process running inserts to start", flush=True, ) time.sleep( WAIT_TO_STABILIZE ) # wait WAIT_TO_STABILIZE seconds to get process that runs the inserts # continuously check for creation of new postgres process timeout = time.time() + 60 while True: # prevent infinite loop if time.time() > timeout: print( "Timed out on finding new process, should force quit SQL inserts", flush=True, ) sys.exit(4) process = find_procs_by_name("postgres") cnt = len(process) print("process count ", cnt) if cnt > process_count: process = find_procs_by_name("postgres") difference_set = set(process) - set(base_process) new_process = None # We assume that the backend is the first 'new' process to start, so it will have # the lower PID for p in difference_set: print(f"found process: {process_details(p)}") if new_process is None or p.pid < new_process.pid: new_process = p print(f"new_process: {process_details(new_process)}") return new_process.pid time.sleep(1) def main(): # get process created from running insert test sql file pid = find_new_process() p = psutil.Process(pid) print('*** Check this pid is the same as "pg_backend_pid" from SQL command ***') print("New process backend process:", pid) print( f"Waiting {WAIT_TO_STABILIZE} seconds for memory consumption to stabilize", flush=True, ) time.sleep(WAIT_TO_STABILIZE) # Calculate average memory consumption from 5 values over 15 seconds total = 0 for _ in range(0, 5): total += p.memory_info().rss time.sleep(3) avg = total / 5 print("Average memory consumption: ", bytes2human(avg), flush=True) cap = int(sys.argv[1] if len(sys.argv) > 1 else DEFAULT_MEMCAP) * 1024 * 1024 upper_threshold = min(cap, avg * THRESHOLD_RATIO) # check if memory consumption goes above threshold until process terminates every 30 seconds timeout = time.time() + 45 * 60 while True: # insert finished if not psutil.pid_exists(pid): break # prevent infinite loop if time.time() > timeout: print("Timed out on running inserts (over 45 mins)") print("Killing postgres process") p.kill() sys.exit(4) rss = p.memory_info().rss stamp = datetime.now().strftime("%H:%M:%S") print(f"{stamp} Memory used by process {p.pid}: {bytes2human(rss)}", flush=True) if DEBUG: process_stats() # exit with error if memory above threshold if rss > upper_threshold: print("Memory consumption exceeded upper threshold") print("Killing postgres process", flush=True) p.kill() sys.exit(4) time.sleep(CHECK_INTERVAL) print("No memory leaks detected", flush=True) sys.exit(0) # success if __name__ == "__main__": main() ================================================ FILE: scripts/test_pg_upgrade.py ================================================ #!/usr/bin/env python3 import os import sys from shutil import rmtree from testgres import get_new_node, PostgresNode # Accessor functions def set_default_conf(node: PostgresNode, unix_socket_dir): node.default_conf(fsync=True, allow_streaming=False, allow_logical=False) node.append_conf(unix_socket_directories=unix_socket_dir) node.append_conf(timezone="GMT") node.append_conf(client_min_messages="warning") node.append_conf(max_prepared_transactions="100") node.append_conf(max_worker_processes="0") node.append_conf(shared_preload_libraries="timescaledb") node.append_conf("timescaledb.telemetry_level=off") def write_bytes_to_file(filename, nbytes): with open(filename, "wb") as f: f.write(nbytes) f.close() def getenv_or_error(envvar): return os.getenv(envvar) or sys.exit(f"Environment variable {envvar} not defined") def getenv_or_default(envvar, default): return os.getenv(envvar) or default def initialize_node(working_dir, prefix, port, bin_dir, base_dir): node = get_new_node(prefix=prefix, port=port, bin_dir=bin_dir, base_dir=base_dir) node.init(initdb_params=["--data-checksums"]) set_default_conf(node, working_dir) return node # Globals pg_version_old = getenv_or_error("PGVERSIONOLD") pg_version_new = getenv_or_error("PGVERSIONNEW") pg_node_old = f"pg{pg_version_old}" pg_node_new = f"pg{pg_version_new}" pg_port_old = int(getenv_or_default("PGPORTOLD", "54321")) pg_port_new = int(getenv_or_default("PGPORTNEW", "54322")) test_version = getenv_or_default("TEST_VERSION", "v8") pg_bin_old = getenv_or_default("PGBINOLD", f"/usr/lib/postgresql/{pg_version_old}/bin") pg_bin_new = getenv_or_default("PGBINNEW", f"/usr/lib/postgresql/{pg_version_new}/bin") working_dir = getenv_or_default( "OUTPUT_DIR", f"/tmp/pg_upgrade_output/{pg_version_old}_to_{pg_version_new}", ) pg_data_old = getenv_or_default("PGDATAOLD", f"{working_dir}/{pg_node_old}") pg_data_new = getenv_or_default("PGDATAOLD", f"{working_dir}/{pg_node_new}") pg_database_test = getenv_or_default("PGDATABASE", "pg_upgrade_test") if os.path.exists(working_dir): rmtree(working_dir) os.makedirs(working_dir) # Real testing code print(f"Initializing nodes {pg_node_old} and {pg_node_new}") node_old = initialize_node( working_dir, pg_node_old, pg_port_old, pg_bin_old, pg_data_old ) node_old.start() node_new = initialize_node( working_dir, pg_node_new, pg_port_new, pg_bin_new, pg_data_new ) print(f"Creating {pg_database_test} database on node {pg_node_old}") node_old.safe_psql(filename="test/sql/updates/setup.roles.sql") node_old.safe_psql(query=f"CREATE DATABASE {pg_database_test};") node_old.safe_psql(dbname=pg_database_test, query="CREATE EXTENSION timescaledb;") node_old.safe_psql(dbname=pg_database_test, filename="test/sql/updates/pre.testing.sql") node_old.safe_psql( dbname=pg_database_test, filename=f"test/sql/updates/setup.{test_version}.sql" ) node_old.safe_psql( dbname=pg_database_test, filename="test/sql/updates/setup.pg_upgrade.sql" ) node_old.safe_psql(dbname=pg_database_test, query="CHECKPOINT") node_old.safe_psql(dbname=pg_database_test, filename="test/sql/updates/setup.check.sql") # Run over the old node to check the output code, old_out, old_err = node_old.psql( dbname=pg_database_test, filename="test/sql/updates/post.pg_upgrade.sql" ) # Save output to log write_bytes_to_file(f"{working_dir}/post.{pg_node_old}.log", old_out) node_old.stop() print(f"Upgrading node {pg_node_old} to {pg_node_new}") res = node_new.upgrade_from(old_node=node_old, options=["--retain"]) node_new.start() code, new_out, new_err = node_new.psql( dbname=pg_database_test, filename="test/sql/updates/post.pg_upgrade.sql", ) # Save output to log write_bytes_to_file(f"{working_dir}/post.pg{pg_version_new}.log", new_out) node_new.stop() print(f"Finish upgrading node {pg_node_old} to {pg_node_new}") ================================================ FILE: scripts/test_regressions.sh ================================================ #!/bin/bash set -eu # # Run regression check # # Param: <next_version> # Param: <connection_string> # if [ "$#" -ne 2 ]; then echo "${0} <next_version> <connection_string>" exit 2 fi #NEXT_VERSION=$1 #CONNECTION_STRING=$2 #psql "$CONNECTION_STRING" -c "ALTER EXTENSION timescaledb UPDATE TO '${NEXT_VERSION}'" #export TEST_PGPORT_REMOTE="$PORT" #export TEST_PGHOST="$HOSTNAME" #export TEST_PGUSER="$SERVICE_USER" #export TEST_DBNAME="$DBNAME" #export PGUSER="$SERVICE_USER" #export PGPASSWORD="$PASSWORD" #export PGPORT="$PORT" #export USER="tsdbadmin" #$SERVICE_USER" #Run the test #make regresscheck-shared #exit 2 # timescaledb Version/Tag related parameters, more changeable: #NEW_DEV_VERSION="2.17.0-dev" #COUNT="0" # PG Tag related parameters , usually less changeable: #PG_MAJOR_VERSION=16 #PG_MINOR_VERSION=4 #create required users on the service. #This is blocked , for now. #psql "$SERVICE_URL" -c "CREATE USER default_perm_user" #psql "$SERVICE_URL" -c "GRANT default_perm_user TO tsdbadmin" #psql "$SERVICE_URL" -c "CREATE USER default_perm_user_2" #psql "$SERVICE_URL" -c "GRANT default_perm_user_2 TO tsdbadmin" #Modify some GUCs, as needed on the service. #psql tsdb -c "ALTER database tsdb set timescaledb_experimental.enable_distributed_ddl to on;" #psql tsdb -c "ALTER database tsdb set timezone to 'US/Pacific';" #psql tsdb -c "ALTER database tsdb set timescaledb.telemetry_level to off;" #psql tsdb -c "ALTER database tsdb set extra_float_digits to 0;" #psql tsdb -c "ALTER database tsdb set random_page_cost to 1.0;" #psql tsdb -c "ALTER database tsdb set datestyle to 'Postgres, MDY';" #psql tsdb -c "ALTER system set autovacuum to false;" #psql tsdb -c "ALTER system set wal_level to 'logical';" #psql tsdb -c "SELECT _timescaledb_functions.stop_background_workers();" #psql tsdb -c "ALTER database tsdb set max_parallel_workers to 8;" #psql tsdb -c "ALTER database tsdb set max_parallel_workers_per_gather to 2;" # This does not switch off Query Identifier , right now. Need to check further. #psql tsdb -c "SET compute_query_id = off;" #sleeps are to allow the server to restart after these GUC parameters' change. #sc service parameters -E "$ENVIRONMENT" -R "$REGION" -S "$SERVICE_ID" set -p max_connections -n 200 #sleep 120 #sc service parameters -E "$ENVIRONMENT" -R "$REGION" -S "$SERVICE_ID" set -p max_worker_processes -n 24 #sleep 120 #Set the parameters #export TEST_PGPORT_REMOTE="$PORT" #export TEST_PGHOST="$HOSTNAME" #export TEST_PGUSER="$SERVICE_USER" #export TEST_DBNAME="$DBNAME" #export PGUSER="$SERVICE_USER" #export PGPASSWORD="$PASSWORD" #export PGPORT="$PORT" #export USER="$SERVICE_USER" #Run the test #make regresscheck-shared ================================================ FILE: scripts/test_sanitizers.sh ================================================ #!/bin/bash set -e set -o pipefail DO_CLEANUP=true SCRIPT_DIR=${SCRIPT_DIR:-$(dirname "$0")} EXCLUDE_PATTERN=${EXCLUDE_PATTERN:-'^$'} # tests matching regex pattern will be excluded INCLUDE_PATTERN=${INCLUDE_PATTERN:-'.*'} # tests matching regex pattern will be included TEST_MAX=${TEST_MAX:-$((2**16))} TEST_MIN=${TEST_MIN:-$((-1))} USE_REMOTE=${USE_REMOTE:-false} REMOTE_TAG=${REMOTE_TAG:-'latest'} PUSH_IMAGE=${PUSH_IMAGE:-false} REMOTE_ORG=${REMOTE_ORG:-'timescaledev'} REMOTE_NAME=${REMOTE_NAME:-'postgres-dev-clang'} TIMESCALE_DIR=${TIMESCALE_DIR:-${PWD}/${SCRIPT_DIR}/..} while getopts "d" opt; do case $opt in d) DO_CLEANUP=false echo "!!Debug mode: Containers and temporary directory will be left on disk" echo ;; *) echo "Unknown flag '$opt'" exit 1 ;; esac done shift $((OPTIND-1)) if [ "$DO_CLEANUP" == "true" ] ; then trap cleanup EXIT fi cleanup() { # Save status here so that we can return the status of the last # command in the script and not the last command of the cleanup # function status="$?" set +e # do not exit immediately on failure in cleanup handler if [[ $status -eq 0 ]]; then echo "All tests passed" docker rm -vf timescaledb-san 2>/dev/null else # docker logs timescaledb-san # only print respective postmaster.log when regression.diffs exists docker_exec timescaledb-san "if [ -f /tsdb_build/timescaledb/build/test/regression.diffs ]; then cat /tsdb_build/timescaledb/build/test/regression.diffs /tsdb_build/timescaledb/build/test/log/postmaster.log; fi" docker_exec timescaledb-san "if [ -f /tsdb_build/timescaledb/build/tsl/test/regression.diffs ]; then cat /tsdb_build/timescaledb/build/tsl/test/regression.diffs /tsdb_build/timescaledb/build/tsl/test/log/postmaster.log; fi" fi echo "Exit status is $status" exit $status } docker_exec() { # Echo to stderr >&2 echo -e "\033[1m$1\033[0m: $2" docker exec "$1" /bin/bash -c "$2" } docker rm -f timescaledb-san 2>/dev/null || true docker run -d --privileged --name timescaledb-san --env POSTGRES_HOST_AUTH_METHOD=trust -v "${TIMESCALE_DIR}":/timescaledb "${REMOTE_ORG}/${REMOTE_NAME}":"${REMOTE_TAG}" # Run these commands as root to copy the source into the # container. Make sure that all files in the copy is owned by user # 'postgres', which we use to run tests below. docker exec -i timescaledb-san /bin/bash -Oe <<EOF mkdir /tsdb_build chown postgres /tsdb_build cp -R /timescaledb tsdb_build chown -R postgres:postgres /tsdb_build EOF # Build TimescaleDB as 'postgres' user docker exec -i -u postgres -w /tsdb_build/timescaledb timescaledb-san /bin/bash -Oe <<EOF export CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -O2" export PG_SOURCE_DIR="/usr/src/postgresql/" export BUILD_FORCE_REMOVE=true ./bootstrap -DCMAKE_BUILD_TYPE='Debug' -DTEST_GROUP_SIZE=1 cd build make EOF # Install TimescaleDB as root docker exec -i -w /tsdb_build/timescaledb/build timescaledb-san /bin/bash <<EOF make install EOF echo "Testing" # Echo to stderr >&2 echo -e "\033[1m$1\033[0m: $2" # Run tests as 'postgres' user. # # IGNORE some test since they fail under ASAN. docker exec -i -u postgres -w /tsdb_build/timescaledb/build timescaledb-san /bin/bash <<EOF make -k regresscheck regresscheck-t IGNORES='bgw_db_scheduler bgw_db_scheduler_fixed bgw_launcher cluster-11 continuous_aggs_ddl-11' EOF ================================================ FILE: scripts/test_update_from_version.sh ================================================ #!/usr/bin/env bash # During the update test the following databases will be created: # - baseline: fresh installation of $TO_VERSION # - updated: install $FROM_VERSION, update to $TO_VERSION # - restored: restore from updated dump # - repair: install $FROM_VERSION, update to $TO_VERSION and run integrity tests set -xeu FROM_VERSION=${FROM_VERSION:-$(grep '^previous_version ' version.config | awk '{ print $3 }')} TO_VERSION=${TO_VERSION:-$(grep '^version ' version.config | awk '{ print $3 }')} TEST_REPAIR=${TEST_REPAIR:-false} TEST_VERSION=${TEST_VERSION:-v10} OUTPUT_DIR=${OUTPUT_DIR:-update_test/${FROM_VERSION}_to_${TO_VERSION}} PGDATA="${OUTPUT_DIR}/data" # Get an unused port to allow for parallel execution PGHOST=localhost PGPORT=${PGPORT:-$(python -c 'import socket; s=socket.socket(); s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) ; s.bind(("", 0)); print(s.getsockname()[1]); s.close()')} PGDATABASE=postgres export PGHOST PGPORT PGDATABASE PGDATA run_sql() { local db=${2:-$PGDATABASE} psql -X -q --echo-queries -d "${db}" -v ON_ERROR_STOP=1 -v VERBOSITY=verbose -v WITH_ROLES=true -v WITH_SUPERUSER=true -v WITH_CHUNK=true -c "${1}" } run_sql_file() { local db=${2:-$PGDATABASE} psql -X -q --echo-queries -d "${db}" -v ON_ERROR_STOP=1 -v VERBOSITY=verbose -v WITH_ROLES=true -v WITH_SUPERUSER=true -v WITH_CHUNK=true -f "${1}" } check_version() { psql -X -c "DO \$\$BEGIN PERFORM from pg_available_extension_versions WHERE name='timescaledb' AND version='$1'; IF NOT FOUND THEN RAISE 'Version $1 not available'; END IF; END\$\$;" > /dev/null } trap cleanup EXIT cleanup() { # Save status here so that we can return the status of the last # command in the script and not the last command of the cleanup # function local status="$?" set +e # do not exit immediately on failure in cleanup handler pg_ctl stop > /dev/null rm -rf "${PGDATA}" exit ${status} } mkdir -p "${OUTPUT_DIR}" rm -rf "${OUTPUT_DIR}/data" mkdir -p "${OUTPUT_DIR}/data" UNIX_SOCKET_DIR=$(readlink -f "${OUTPUT_DIR}") initdb > "${OUTPUT_DIR}/initdb.log" 2>&1 # Don't try to wrap the settings, pg_ctl can't handle newlines there. pg_ctl -l "${OUTPUT_DIR}/postgres.log" start -o "-c unix_socket_directories=${UNIX_SOCKET_DIR} -c timezone=GMT -c client_min_messages=warning -c port=${PGPORT} -c max_prepared_transactions=100 -c shared_preload_libraries=timescaledb -c timescaledb.telemetry_level=off -c timescaledb.enable_compression_ratio_warnings=off -c max_worker_processes=0 -c log_statement=all" pg_isready -t 30 > /dev/null echo -e "\nUpdate test version ${TEST_VERSION} for ${FROM_VERSION} -> ${TO_VERSION}\n" # caller should ensure that the versions are available check_version "${FROM_VERSION}" check_version "${TO_VERSION}" run_sql_file test/sql/updates/setup.roles.sql > /dev/null echo "Creating baseline database" { run_sql "CREATE DATABASE baseline;" PGDATABASE=baseline run_sql "CREATE EXTENSION timescaledb VERSION \"${TO_VERSION}\";" run_sql_file test/sql/updates/pre.testing.sql run_sql_file test/sql/updates/setup.${TEST_VERSION}.sql run_sql "CHECKPOINT;" run_sql_file test/sql/updates/setup.check.sql } > "${OUTPUT_DIR}/baseline.log" 2>&1 echo "Creating updated database" { run_sql "CREATE DATABASE updated;" > "${OUTPUT_DIR}/updated.log" PGDATABASE=updated run_sql "CREATE EXTENSION timescaledb VERSION \"${FROM_VERSION}\";" run_sql_file test/sql/updates/pre.testing.sql run_sql_file test/sql/updates/setup.${TEST_VERSION}.sql run_sql "CHECKPOINT;" >> "${OUTPUT_DIR}/updated.log" run_sql "ALTER EXTENSION timescaledb UPDATE TO \"${TO_VERSION}\";" run_sql_file test/sql/updates/setup.check.sql } > "${OUTPUT_DIR}/updated.log" 2>&1 echo "Creating restored database" { run_sql "CREATE DATABASE restored;" PGDATABASE=restored run_sql "CREATE EXTENSION timescaledb VERSION \"${TO_VERSION}\";" run_sql "ALTER DATABASE restored SET timescaledb.restoring='on';" pg_dump -Fc -d updated > "${OUTPUT_DIR}/updated.dump" pg_restore -d restored "${OUTPUT_DIR}/updated.dump" run_sql "ALTER DATABASE restored RESET timescaledb.restoring;" } > "${OUTPUT_DIR}/restored.log" 2>&1 run_sql_file test/sql/updates/post.${TEST_VERSION}.sql baseline > "${OUTPUT_DIR}/post.baseline.log" run_sql_file test/sql/updates/post.${TEST_VERSION}.sql updated > "${OUTPUT_DIR}/post.updated.log" run_sql_file test/sql/updates/post.${TEST_VERSION}.sql restored > "${OUTPUT_DIR}/post.restored.log" sed -i -E -e 's!"*_ts_meta_v2_bl[0-9A-Za-z]+_([_0-9A-Za-z]+)"*!regress-test-bloom_\1!g' \ "${OUTPUT_DIR}"/post.*.log if [ "${TEST_REPAIR}" = "true" ]; then echo "Creating repair database" { run_sql "CREATE DATABASE repair;" PGDATABASE=repair run_sql "CREATE EXTENSION timescaledb VERSION \"${FROM_VERSION}\";" run_sql_file test/sql/updates/setup.repair.sql baseline run_sql "ALTER EXTENSION timescaledb UPDATE TO \"${TO_VERSION}\";" run_sql_file test/sql/updates/post.repair.sql baseline run_sql_file test/sql/updates/post.integrity_test.sql baseline } > "${OUTPUT_DIR}/repair.log" 2>&1 fi diff -u "${OUTPUT_DIR}/post.baseline.log" "${OUTPUT_DIR}/post.updated.log" | tee "${OUTPUT_DIR}/baseline_vs_updated.diff" if [ ! -s "${OUTPUT_DIR}/baseline_vs_updated.diff" ]; then rm "${OUTPUT_DIR}/baseline_vs_updated.diff" fi diff -u "${OUTPUT_DIR}/post.baseline.log" "${OUTPUT_DIR}/post.restored.log" | tee "${OUTPUT_DIR}/baseline_vs_restored.diff" if [ ! -s "${OUTPUT_DIR}/baseline_vs_restored.diff" ]; then rm "${OUTPUT_DIR}/baseline_vs_restored.diff" fi if [ -f "${OUTPUT_DIR}/baseline_vs_updated.diff" ] || [ -f "${OUTPUT_DIR}/baseline_vs_restored.diff" ]; then echo "Update test for ${FROM_VERSION} -> ${TO_VERSION} failed" exit 1 fi ================================================ FILE: scripts/test_update_smoke.sh ================================================ #!/bin/bash # shellcheck disable=SC2129,SC2230 # SC2129: Consider using { cmd1; cmd2; } >> file instead of individual redirects. # SC2230: which is non-standard. Use builtin 'command -v' instead. # Run smoke tests to test that updating between versions work. # # Usage: # bash test_update_smoke.sh postgres://... # # This is based on our update tests but doing some tweaks to ensure we # can run it on Forge (or any other PostgreSQL server that only permit # a single user and single database). # # In particular, we cannot create new roles and we cannot create new # databases. # # Info on the parameters: # - current_version is the version to update from # # - next_version is the version to update to # # - connection_string is the URL to use for the connection # # - dbname if not defined use "tsdb" as database name # if [ "$#" -ne 3 ]; then echo "${0} <current_version> <next_version> <connection_string>" exit 2 fi CURRENT_VERSION=$1 NEXT_VERSION=$2 CONNECTION_STRING=$3 echo "testing upgrade path from v${CURRENT_VERSION} to ${NEXT_VERSION} .." SCRIPT_DIR=$(dirname $0) BASE_DIR=${PWD}/${SCRIPT_DIR}/.. SCRATCHDIR=$(mktemp -d -t "smoketest-${CURRENT_VERSION}-${NEXT_VERSION}-XXXX") LOGFILE="$SCRATCHDIR/update-test.log" DUMPFILE="$SCRATCHDIR/smoke.dump" UPGRADE_OUT="$SCRATCHDIR/upgrade.out" CLEAN_OUT="$SCRATCHDIR/clean.out" RESTORE_OUT="$SCRATCHDIR/restore.out" TEST_VERSION=${TEST_VERSION:-v7} # We do not have superuser privileges when running smoke tests. WITH_SUPERUSER=false WITH_ROLES=false shift $((OPTIND-1)) echo "**** pg_dump at " "$(which pg_dump)" echo "**** pg_restore at" "$(which pg_restore)" # Extra options to pass to psql PGOPTS="-v TEST_VERSION=${TEST_VERSION} -v WITH_SUPERUSER=${WITH_SUPERUSER} -v WITH_ROLES=${WITH_ROLES} -v WITH_CHUNK=false" PSQL="psql -a -qX $PGOPTS" # If we are providing a URI for the connection, we parse it here and # set the PG??? variables since that is the only reliable way to # provide connection information to psql, pg_dump, and pg_restore. # # To work with Forge, we need to only set PGPASSWORD when the password # is available and leave it unset otherwise. If the user has either # set PGPASSWORD or has the password in a .pgpass file, it will be # picked up and used for the connection. # shellcheck disable=SC2207 # Prefer mapfile or read -a to split command output (or quote to avoid splitting). parts=($(echo $CONNECTION_STRING | perl -mURI::Split=uri_split -ne '@F = uri_split($_); print join(" ", split(qr/[:@]/, $F[1]), substr($F[2], 1))')) export PGUSER=${parts[0]} if [[ ${#parts[@]} -eq 5 ]]; then # Cloud has 5 fields export PGPASSWORD=${parts[1]} export PGHOST=${parts[2]} export PGPORT=${parts[3]} export PGDATABASE=${parts[4]} elif [[ ${#parts[@]} -eq 4 ]]; then # Forge has 4 fields export PGHOST=${parts[1]} export PGPORT=${parts[2]} export PGDATABASE=${parts[3]} else echo "Malformed URL '$CONNECTION_STRING'" 1>&2 exit 2 fi err_trap() { exit 3 } exit_trap() { exit_code=$? if [ "$exit_code" != "0" ]; then echo "!!!! FAILED !!!!" else echo "**** passed ****" fi echo "**** logs can be found in $SCRATCHDIR" } set -e set -o pipefail trap exit_trap EXIT trap err_trap ERR missing_versions() { $PSQL -v ECHO=none -t <<-EOF SELECT * FROM (VALUES ('$1'), ('$2')) AS foo EXCEPT SELECT version FROM pg_available_extension_versions WHERE name = 'timescaledb' AND version IN ('$1', '$2'); EOF } echo "**** Scratch directory: ${SCRATCHDIR}" echo "**** Update files in directory ${BASE_DIR}/test/sql/updates" cd ${BASE_DIR}/test/sql/updates $PSQL -c '\conninfo' $PSQL -c "ALTER DATABASE ${PGDATABASE} SET timescaledb.enable_compression_ratio_warnings = 'off'"; # shellcheck disable=SC2207 # Prefer mapfile or read -a to split command output (or quote to avoid splitting). missing=($(missing_versions $CURRENT_VERSION $NEXT_VERSION)) if [[ ${#missing[@]} -gt 0 ]]; then echo "ERROR: Missing version(s) ${missing[*]} of 'timescaledb'" echo "Available versions: " "$($PSQL -tc "SELECT version FROM pg_available_extension_versions WHERE name = 'timescaledb'")" exit 1 fi # For the comments below, we assume the upgrade is from 1.7.5 to 2.0.2 # (this is just an example, the real value is given by variables # above). # Create a 1.7.5 version Upgrade echo "---- Connecting to ${FORGE_CONNINFO} and running setup ----" $PSQL -f cleanup.${TEST_VERSION}.sql >>$LOGFILE 2>&1 $PSQL -c "DROP EXTENSION IF EXISTS timescaledb CASCADE" >>$LOGFILE 2>&1 $PSQL -f pre.cleanup.sql >>$LOGFILE 2>&1 $PSQL -c "CREATE EXTENSION timescaledb VERSION '${CURRENT_VERSION}'" >>$LOGFILE 2>&1 $PSQL -c "\dx" # Run setup on Upgrade $PSQL -f pre.smoke.sql >>$LOGFILE 2>&1 $PSQL -f setup.${TEST_VERSION}.sql >>$LOGFILE 2>&1 # Run update on Upgrade. You now have a 2.0.2 version in Upgrade. $PSQL -c "ALTER EXTENSION timescaledb UPDATE TO '${NEXT_VERSION}'" >>$LOGFILE 2>&1 echo -n "Dumping the contents of Upgrade..." pg_dump -Fc -f $DUMPFILE >>$LOGFILE 2>&1 echo "done" # Run the post scripts on Upgrade to get UpgradeOut to compare # with. We can now discard Upgrade database. echo -n "Collecting post-update status..." $PSQL -f post.${TEST_VERSION}.sql >$UPGRADE_OUT echo "done" $PSQL -f cleanup.${TEST_VERSION}.sql >>$LOGFILE 2>&1 echo "---- Create a ${NEXT_VERSION} version Clean ----" $PSQL -c "DROP EXTENSION IF EXISTS timescaledb CASCADE" >>$LOGFILE 2>&1 $PSQL -f pre.cleanup.sql >>$LOGFILE 2>&1 $PSQL -c "CREATE EXTENSION timescaledb VERSION '${NEXT_VERSION}'" >>$LOGFILE 2>&1 $PSQL -c "\dx" echo "---- Run the setup scripts on Clean, with post-update actions ----" $PSQL -f pre.smoke.sql >>$LOGFILE 2>&1 $PSQL -f setup.${TEST_VERSION}.sql >>$LOGFILE 2>&1 echo "---- Run the post scripts on Clean to get output CleanOut ----" $PSQL -f post.${TEST_VERSION}.sql >$CLEAN_OUT $PSQL -f cleanup.${TEST_VERSION}.sql >>$LOGFILE 2>&1 echo "---- Create a ${NEXT_VERSION} version Restore ----" $PSQL -c "DROP EXTENSION IF EXISTS timescaledb CASCADE" >>$LOGFILE 2>&1 $PSQL -f pre.cleanup.sql >>$LOGFILE 2>&1 $PSQL -c "CREATE EXTENSION timescaledb VERSION '${NEXT_VERSION}'" >>$LOGFILE 2>&1 $PSQL -c "\dx" echo "---- Restore the UpgradeDump into Restore ----" echo -n "Restoring dump..." $PSQL -c "SELECT timescaledb_pre_restore()" >>$LOGFILE 2>&1 pg_restore -d $PGDATABASE $DUMPFILE >>$LOGFILE 2>&1 || true $PSQL -c "SELECT timescaledb_post_restore()" >>$LOGFILE 2>&1 echo "done" echo "---- Run the post scripts on Restore to get a RestoreOut ----" $PSQL -f post.${TEST_VERSION}.sql >$RESTORE_OUT echo "---- Compare UpgradeOut with CleanOut and make sure they are identical ----" diff -u $UPGRADE_OUT $CLEAN_OUT | tee $SCRATCHDIR/upgrade-clean.diff echo "---- Compare RestoreOut with CleanOut and make sure they are identical ----" diff -u $RESTORE_OUT $CLEAN_OUT | tee $SCRATCHDIR/restore-clean.diff $PSQL -f cleanup.${TEST_VERSION}.sql >>$LOGFILE 2>&1 ================================================ FILE: scripts/test_updates.sh ================================================ #!/bin/bash set -eu SCRIPT_DIR=$(readlink -f "$(dirname $0)") PG_MAJOR_VERSION=$(pg_config --version | awk '{print $2}' | awk -F. '{print $1}') PG_EXTENSION_DIR=$(pg_config --sharedir)/extension if [ "${CI:-false}" == true ]; then GIT_REF=${GIT_REF:-$(git rev-parse HEAD)} else GIT_REF=$(git branch --show-current) fi BUILD_DIR="build_update_pg${PG_MAJOR_VERSION}" VERSIONS="" FAILED_VERSIONS="" ALL_VERSIONS=$(git tag --sort=taggerdate | grep -P '^[2]\.[0-9]+\.[0-9]+$') MAX_VERSION=$(grep '^previous_version ' version.config | awk '{ print $3 }') # major version is always 2 atm max_minor_version=$(echo "${MAX_VERSION}" | awk -F. '{print $2}') max_patch_version=$(echo "${MAX_VERSION}" | awk -F. '{print $3}') # Filter versions depending on the current postgres version # Minimum version for valid update paths are as follows: # PG 14 v8 2.5+ # PG 15 v8 2.9+ # PG 16 v8 2.13+ # PG 17 v8 2.17+ for version in ${ALL_VERSIONS}; do minor_version=$(echo "${version}" | awk -F. '{print $2}') patch_version=$(echo "${version}" | awk -F. '{print $3}') # skip versions that are newer than the max version # We might have a tag for a newer version defined already but the post release # adjustment have not been merged yet. So we want to skip those versions. if [ "${minor_version}" -gt "${max_minor_version}" ]; then continue elif [ "${minor_version}" -eq "${max_minor_version}" ] && [ "${patch_version}" -gt "${max_patch_version}" ]; then continue fi if [ "${minor_version}" -le 8 ]; then # not part of any valid update path continue elif [ "${minor_version}" -le 12 ]; then if [ "${PG_MAJOR_VERSION}" -le 15 ]; then VERSIONS="${VERSIONS} ${version}" fi elif [ "${minor_version}" -le 16 ]; then if [ "${PG_MAJOR_VERSION}" -le 16 ]; then VERSIONS="${VERSIONS} ${version}" fi elif [ "${minor_version}" -le 22 ]; then if [ "${PG_MAJOR_VERSION}" -le 17 ]; then VERSIONS="${VERSIONS} ${version}" fi else VERSIONS="${VERSIONS} ${version}" fi done FAIL_COUNT=0 if [ ! -d "${BUILD_DIR}" ]; then echo "Initializing build directory" BUILD_DIR="${BUILD_DIR}" ./bootstrap -DCMAKE_BUILD_TYPE=Release -DWARNINGS_AS_ERRORS=OFF -DASSERTIONS=ON -DLINTER=OFF -DGENERATE_DOWNGRADE_SCRIPT=ON -DREGRESS_CHECKS=OFF -DTAP_CHECKS=OFF fi for version in ${VERSIONS}; do if [ ! -f "${PG_EXTENSION_DIR}/timescaledb--${version}.sql" ]; then echo "Building ${version}" git checkout ${version} make -C "${BUILD_DIR}" -j "$(getconf _NPROCESSORS_ONLN)" > /dev/null sudo make -C "${BUILD_DIR}" install > /dev/null git checkout ${GIT_REF} fi done # We want to use the latest loader for all the tests so we build it last git checkout ${GIT_REF} make -C "${BUILD_DIR}" -j "$(getconf _NPROCESSORS_ONLN)" sudo make -C "${BUILD_DIR}" install set +e if [ -n "${VERSIONS}" ]; then for version in ${VERSIONS}; do ts_minor_version=$(echo "${version}" | awk -F. '{print $2}') if [ "${ts_minor_version}" -ge 10 ]; then TEST_REPAIR=true else TEST_REPAIR=false fi if [ "${ts_minor_version}" -ge 20 ]; then TEST_VERSION=v10 elif [ "${ts_minor_version}" -ge 16 ]; then TEST_VERSION=v9 else TEST_VERSION=v8 fi export TEST_VERSION TEST_REPAIR FROM_VERSION=${version} "${SCRIPT_DIR}/test_update_from_version.sh" return_code=$? if [ $return_code -ne 0 ]; then FAIL_COUNT=$((FAIL_COUNT + 1)) FAILED_VERSIONS="${FAILED_VERSIONS} ${version}" fi done fi echo -e "\nUpdate test finished for ${VERSIONS}\n" if [ $FAIL_COUNT -gt 0 ]; then echo -e "Failed versions: ${FAILED_VERSIONS}\n" echo -e "Postgres errors:\n" find update_test -name postgres.log -exec grep ERROR {} \; else echo -e "All tests succeeded.\n" fi exit $FAIL_COUNT ================================================ FILE: scripts/ts_dump.sh ================================================ #!/usr/bin/env bash # This file and its contents are licensed under the Apache License 2.0. # Please see the included NOTICE for copyright information and # LICENSE-APACHE for a copy of the license. # This script is used for backing up a single hypertable into an easy-to-restore # tarball. The tarball contains two files: (1) a .sql file for recreating the # hypertable and its indices and (2) a .csv file containing the data as CSV. # # Because pg_dump/pg_restore dump all of TimescaleDB's internal tables when # used, this script is useful if you want a backup that can be restored # regardless of TimescaleDB version running, or as part of a process where you # do not want to always backup all your hypertables at once. if [[ -z "$1" || -z "$2" ]]; then echo "Usage: $0 hypertable output_name [pg_dump CONNECTION OPTIONS]" echo " hypertable - Hypertable to backup" echo " output_name - Output files will be stored in an archive named [output_name].tar.gz" echo " " echo "Any connection options for pg_dump/psql (e.g. -d database, -U username) should be listed at the end" exit 1 fi HYPERTABLE=$1 PREFIX=$2 shift 2 set -e echo "Backing up schema as $PREFIX-schema.sql..." pg_dump "$@" --schema-only -t $HYPERTABLE -f $PREFIX-schema.sql echo >> $PREFIX-schema.sql "-- -- Restore to hypertable --" psql "$@" -qAtX -c "SELECT _timescaledb_functions.get_create_command('$HYPERTABLE');" >> $PREFIX-schema.sql echo "Backing up data as $PREFIX-data.csv..." psql "$@" -c "\COPY (SELECT * FROM $HYPERTABLE) TO $PREFIX-data.csv DELIMITER ',' CSV" echo "Archiving and removing previous files..." tar -czvf $PREFIX.tar.gz $PREFIX-data.csv $PREFIX-schema.sql rm $PREFIX-data.csv rm $PREFIX-schema.sql ================================================ FILE: scripts/ts_restore.sh ================================================ #!/usr/bin/env bash # This file and its contents are licensed under the Apache License 2.0. # Please see the included NOTICE for copyright information and # LICENSE-APACHE for a copy of the license. # This script is used for restoring a hypertable from a tarball made with # ts_dump.sh. It unarchives the tarball, producing a schema file and data file, # which are then restore separately. if [[ -z "$1" || -z "$2" ]]; then echo "Usage: $0 hypertable tarfile_name" echo " hypertable - Hypertable to restore" echo " tarfile_name - Name of the tarball created by ts_dump.sh to restore" echo " " echo "Any connection options for pg_restore/psql (e.g. -d database, -U username) should be listed at the end" exit 1 fi HYPERTABLE=$1 TARFILE=$2 PREFIX="${TARFILE/.tar.gz/}" shift 2 echo "Unarchiving tarball..." tar xvzf $TARFILE echo "Restoring hypertable's schema..." if ! psql -q -v "ON_ERROR_STOP=1" "$@" < $PREFIX-schema.sql then echo "Restoring schema failed, exiting." rm $PREFIX-data.csv rm $PREFIX-schema.sql exit $? fi echo "Restoring hypertable's data..." if ! psql "$@" -v "ON_ERROR_STOP=1" -c "\COPY $HYPERTABLE FROM $PREFIX-data.csv DELIMITER ',' CSV" then echo "Restoring data failed, exiting." fi rm $PREFIX-data.csv rm $PREFIX-schema.sql ================================================ FILE: scripts/upload_ci_stats.sh ================================================ #!/usr/bin/env bash set -xue set -o pipefail if [ -z "${CI_STATS_DB:-}" ] then # The secret with the stats db connection string is not accessible in forks. echo "The statistics database connection string is not specified" exit 0 fi PSQL=${PSQL:-psql} PSQL=("${PSQL}" "${CI_STATS_DB}" -qtAX "--set=ON_ERROR_STOP=1") # The tables we are going to use. This schema is here just as a reminder, you'll # have to create them manually. After you manually change the actual DB schema, # don't forget to append the needed migration code below. : " create extension if not exists timescaledb; create table job( job_date timestamptz, -- Serves as a unique id. commit_sha text, job_name text, repository text, ref_name text, event_name text, pr_number int, job_status text, url text, run_attempt int, run_id bigint, run_number int, pg_version text generated always as ( substring(job_name from 'PG([0-9]+(\.[0-9]+)*)') ) stored ); create unique index on job(job_date); select create_hypertable('job', 'job_date'); create table test( job_date timestamptz, test_name text, test_status text, test_duration float ); create unique index on test(job_date, test_name); select create_hypertable('test', 'job_date'); create table log( job_date timestamptz, test_name text, log_contents text ); create unique index on log(job_date, test_name); select create_hypertable('log', 'job_date'); create table ipe( job_date timestamptz, error text, location text, statement text ); select create_hypertable('ipe', 'job_date'); " # Create the job record. COMMIT_SHA=$(git -C "$(dirname "${BASH_SOURCE[0]}")" rev-parse @) export COMMIT_SHA JOB_NAME="${JOB_NAME:-test-job}" export JOB_NAME JOB_DATE=$("${PSQL[@]}" -c " insert into job( job_date, commit_sha, job_name, repository, ref_name, event_name, pr_number, job_status, url, run_attempt, run_id, run_number ) values ( now(), '$COMMIT_SHA', '$JOB_NAME', '$GITHUB_REPOSITORY', '$GITHUB_REF_NAME', '$GITHUB_EVENT_NAME', '$GITHUB_PR_NUMBER', '$JOB_STATUS', 'https://github.com/timescale/timescaledb/actions/runs/$GITHUB_RUN_ID/attempts/$GITHUB_RUN_ATTEMPT', '$GITHUB_RUN_ATTEMPT', '$GITHUB_RUN_ID', '$GITHUB_RUN_NUMBER') returning job_date; ") export JOB_DATE # Parse the installcheck.log to find the individual test results. Note that this # file might not exist for failed checks or non-regression checks like SQLSmith. # We still want to save the other logs. if [ -f 'installcheck.log' ] then gawk -v OFS='\t' ' match($0, /^(test| ) ([^ ]+)[ ]+\.\.\.[ ]+([^ ]+) (|\(.*\))[ ]+([0-9]+) ms$/, a) { print ENVIRON["JOB_DATE"], a[2], tolower(a[3] (a[4] ? (" " a[4]) : "")), a[5]; } match($0, /^([^0-9]+) [0-9]+ +[-+] ([^ ]+) +([0-9]+) ms/, a) { print ENVIRON["JOB_DATE"], a[2], a[1], a[3]; } ' installcheck.log > tests.tsv # Save the test results into the database. "${PSQL[@]}" -c "\copy test from tests.tsv" # Split the regression.diffs into per-test files. gawk ' match($0, /^(diff|\+\+\+|\-\-\-) .*\/(.*)[.]out/, a) { file = a[2] ".diff"; next; } { if (file) print $0 > file; } ' regression.log fi # Snip the long sequences of "+" or "-" changes in the diffs. for x in *.diff; do if ! [ -f "$x" ] ; then continue ; fi gawk -v max_context_lines=10 -v min_context_lines=2 ' /^-/ { new_sign = "-" } /^+/ { new_sign = "+" } /^[^+-]/ { new_sign = " " } { if (old_sign != new_sign) { to_print = lines_buffered > max_context_lines ? min_context_lines : lines_buffered; if (lines_buffered > to_print) print "<" lines_buffered - to_print " lines skipped>"; for (i = 0; i < to_print; i++) { print buf[(NR + i - to_print) % max_context_lines] } printf("c %04d: %s\n", NR, $0); old_sign = new_sign; lines_printed = 0; lines_buffered = 0; } else { if (lines_printed >= min_context_lines) { lines_buffered++; buf[NR % max_context_lines] = sprintf("b %04d: %s", NR, $0) } else { lines_printed++; printf("p %04d: %s\n", NR, $0); } } } END { to_print = lines_buffered > max_context_lines ? min_context_lines : lines_buffered; if (lines_buffered > to_print) print "<" lines_buffered - to_print " lines skipped>"; for (i = 0; i < to_print; i++) { print buf[(NR + 1 + i - to_print) % max_context_lines] } }' "$x" > "$x.tmp" mv "$x.tmp" "$x" done # Save a snippet of logs where a backend was terminated by signal. grep -C40 "was terminated by signal" postmaster.log > postgres-failure.log ||: # Find internal program errors and resource owner leak warnings in the sever log. # We do the same thing in Flaky Check and error out if we find any, not to # introduce these errors for the new tests. jq 'select( (.state_code == "XX000" and .error_severity != "LOG") or (.message | test("resource was not closed")) ) | [env.JOB_DATE, .message, .func_name, .statement] | @tsv ' -r postmaster.json > ipe.tsv ||: "${PSQL[@]}" -c "\copy ipe from ipe.tsv" # Upload the logs. # Note that the sanitizer setting log_path means "write logs to 'log_path.pid'". for x in sanitizer* sanitizer/* {sqlsmith/sqlsmith,sanitizer,stacktrace,postgres-failure}.log *.diff do if ! [ -e "$x" ]; then continue ; fi "${PSQL[@]}" <<<" \set contents \`cat $x\` insert into log values ('$JOB_DATE', '$(basename "$x" .diff)', :'contents'); " done ================================================ FILE: sql/CMakeLists.txt ================================================ set(INSTALL_FILE ${PROJECT_NAME}--${PROJECT_VERSION_MOD}.sql) include(ScriptFiles) # These files represent the modifications that happen in each version, excluding # new objects or updates to functions. We use them to build a path (update # script) from every historical version to the current version. Note that not # all of these files may exist on disk, in case they would have no contents. # There still needs to be an entry here to build an update script for that # version. Thus, for every new release, an entry should be added here. set(MOD_FILES updates/2.9.0--2.9.1.sql updates/2.9.1--2.9.2.sql updates/2.9.2--2.9.3.sql updates/2.9.3--2.10.0.sql updates/2.10.0--2.10.1.sql updates/2.10.1--2.10.2.sql updates/2.10.2--2.10.3.sql updates/2.10.3--2.11.0.sql updates/2.11.0--2.11.1.sql updates/2.11.1--2.11.2.sql updates/2.11.2--2.12.0.sql updates/2.12.0--2.12.1.sql updates/2.12.1--2.12.2.sql updates/2.12.2--2.13.0.sql updates/2.13.0--2.13.1.sql updates/2.13.1--2.14.0.sql updates/2.14.0--2.14.1.sql updates/2.14.1--2.14.2.sql updates/2.14.2--2.15.0.sql updates/2.15.0--2.15.1.sql updates/2.15.1--2.15.2.sql updates/2.15.2--2.15.3.sql updates/2.15.3--2.16.0.sql updates/2.16.0--2.16.1.sql updates/2.16.1--2.17.0.sql updates/2.17.0--2.17.1.sql updates/2.17.1--2.17.2.sql updates/2.17.2--2.18.0.sql updates/2.18.0--2.18.1.sql updates/2.18.1--2.18.2.sql updates/2.18.2--2.19.0.sql updates/2.19.0--2.19.1.sql updates/2.19.1--2.19.2.sql updates/2.19.2--2.19.3.sql updates/2.19.3--2.20.0.sql updates/2.20.0--2.20.1.sql updates/2.20.1--2.20.2.sql updates/2.20.2--2.20.3.sql updates/2.20.3--2.21.0.sql updates/2.21.0--2.21.1.sql updates/2.21.1--2.21.2.sql updates/2.21.2--2.21.3.sql updates/2.21.3--2.21.4.sql updates/2.21.4--2.22.0.sql updates/2.22.0--2.22.1.sql updates/2.22.1--2.23.0.sql updates/2.23.0--2.23.1.sql updates/2.23.1--2.24.0.sql updates/2.24.0--2.25.0.sql updates/2.25.0--2.25.1.sql updates/2.25.1--2.25.2.sql) # The downgrade file to generate a downgrade script for the current version, as # specified in version.config set(CURRENT_REV_FILE reverse-dev.sql) set(MODULE_PATHNAME "$libdir/timescaledb-${PROJECT_VERSION_MOD}") set(LOADER_PATHNAME "$libdir/timescaledb") set(TS_MODULE_PATHNAME ${MODULE_PATHNAME} PARENT_SCOPE) set(TSL_MODULE_PATHNAME "$libdir/timescaledb-tsl-${PROJECT_VERSION_MOD}" PARENT_SCOPE) if(NOT GENERATE_DOWNGRADE_SCRIPT) message( STATUS "Not generating downgrade script: downgrade generation disabled.") elseif(NOT GIT_FOUND) message(STATUS "Not generating downgrade script: Git not available.") else() generate_downgrade_script( SOURCE_VERSION ${PROJECT_VERSION_MOD} TARGET_VERSION ${PREVIOUS_VERSION} INPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/updates FILES ${CURRENT_REV_FILE}) endif() # Function to replace @VARIABLE@ in source files, producing output files in the # build dir. When SUFFIX is provided, it is appended to each output filename # (used for version-specific copies like version_check.sql). function(version_files SRC_FILE_LIST OUTPUT_FILE_LIST) cmake_parse_arguments(_vf "" "SUFFIX" "" ${ARGN}) set(result "") foreach(unversioned_file ${SRC_FILE_LIST}) set(versioned_file ${unversioned_file}${_vf_SUFFIX}) list(APPEND result ${CMAKE_CURRENT_BINARY_DIR}/${versioned_file}) if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${unversioned_file}) configure_file(${unversioned_file} ${versioned_file} @ONLY) endif() endforeach(unversioned_file) set(${OUTPUT_FILE_LIST} "${result}" PARENT_SCOPE) endfunction() # Create versioned files (replacing MODULE_PATHNAME) in the build directory of # all our source files version_files("${PRE_UPDATE_FILES}" PRE_UPDATE_FILES_VERSIONED) version_files("${POST_UPDATE_FILES}" POST_UPDATE_FILES_VERSIONED) version_files("${PRE_INSTALL_SOURCE_FILES}" PRE_INSTALL_SOURCE_FILES_VERSIONED) version_files("${PRE_INSTALL_FUNCTION_FILES}" PRE_INSTALL_FUNCTION_FILES_VERSIONED) version_files("${SOURCE_FILES}" SOURCE_FILES_VERSIONED) version_files("${MOD_FILES}" MOD_FILES_VERSIONED) version_files("updates/latest-dev.sql" LASTEST_MOD_VERSIONED) version_files("notice.sql" NOTICE_FILE) # Function to concatenate all files in SRC_FILE_LIST into file OUTPUT_FILE. When # STRIP_REPLACE is set, strips `OR REPLACE` from the output to prevent privilege # escalation attacks in CREATE scripts. function(cat_files SRC_FILE_LIST OUTPUT_FILE) if(WIN32) set("SRC_ARG" "-DSRC_FILE_LIST=${SRC_FILE_LIST}") else() set("SRC_ARG" "'-DSRC_FILE_LIST=${SRC_FILE_LIST}'") endif() set(_extra_args) if(STRIP_REPLACE) list(APPEND _extra_args "-DSTRIP_REPLACE=ON") endif() add_custom_command( OUTPUT ${OUTPUT_FILE} DEPENDS ${SRC_FILE_LIST} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMAND ${CMAKE_COMMAND} "${SRC_ARG}" "-DOUTPUT_FILE=${OUTPUT_FILE}" ${_extra_args} -P cat.cmake COMMENT "Generating ${OUTPUT_FILE}") endfunction() # Generate the extension file used with CREATE EXTENSION set(STRIP_REPLACE ON) cat_files( "${PRE_INSTALL_SOURCE_FILES_VERSIONED};${SOURCE_FILES_VERSIONED};${NOTICE_FILE}" ${CMAKE_CURRENT_BINARY_DIR}/${INSTALL_FILE}) set(STRIP_REPLACE OFF) add_custom_target(sqlfile ALL DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${INSTALL_FILE}) # Generate the update files used with ALTER EXTENSION <name> UPDATE set(MOD_FILE_REGEX "([0-9]+\\.[0-9]+\\.*[0-9]+[-a-z0-9]*)--([0-9]+\\.[0-9]+\\.*[0-9]+[-a-z0-9]*).sql" ) # We'd like to process the updates in reverse (descending) order if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/updates/${PREVIOUS_VERSION}--${PROJECT_VERSION_MOD}.sql" ) set(MOD_FILES_LIST ${MOD_FILES_VERSIONED}) else() set(MOD_FILES_LIST "${MOD_FILES_VERSIONED};updates/${PREVIOUS_VERSION}--${PROJECT_VERSION_MOD}.sql" ) endif() list(REVERSE MOD_FILES_LIST) # Variable that will hold the list of update scripts from every previous version # to the current version set(UPDATE_SCRIPTS "") # A list of current modfiles. We append to this list for every previous version # that moves us further away from the current version, thus making the update # path longer as we move back in history set(CURR_MOD_FILES "${LASTEST_MOD_VERSIONED}") # Generate the post-update file once (same for all update scripts) set(POST_FILES_PROCESSED ${POST_UPDATE_FILES_VERSIONED}.processed) cat_files( "${SET_POST_UPDATE_STAGE};${POST_UPDATE_FILES_VERSIONED};${UNSET_UPDATE_STAGE}" ${POST_FILES_PROCESSED}) # Now loop through the modfiles and generate the update files foreach(transition_mod_file ${MOD_FILES_LIST}) if(NOT (${transition_mod_file} MATCHES ${MOD_FILE_REGEX})) message(FATAL_ERROR "Cannot parse update file name ${transition_mod_file}") endif() set(START_VERSION ${CMAKE_MATCH_1}) set(END_VERSION ${CMAKE_MATCH_2}) set(PRE_FILES ${PRE_UPDATE_FILES_VERSIONED}) # Check for version-specific update code with fixes if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/updates/${START_VERSION}.sql) version_files("updates/${START_VERSION}.sql" ORIGIN_MOD_FILE) list(APPEND PRE_FILES ${ORIGIN_MOD_FILE}) endif() version_files("updates/version_check.sql" VERSION_CHECK_VERSIONED SUFFIX -${START_VERSION}) list(PREPEND PRE_FILES "header.sql;${VERSION_CHECK_VERSIONED}") # There might not have been any changes in the modfile, in which case the # modfile need not be present if(EXISTS ${transition_mod_file}) # Prepend the modfile as we are moving through the versions in descending # order list(INSERT CURR_MOD_FILES 0 ${transition_mod_file}) endif() set(UPDATE_SCRIPT ${CMAKE_CURRENT_BINARY_DIR}/timescaledb--${START_VERSION}--${PROJECT_VERSION_MOD}.sql ) list(APPEND UPDATE_SCRIPTS ${UPDATE_SCRIPT}) if(CURR_MOD_FILES) cat_files( "${PRE_FILES};${CURR_MOD_FILES};${PRE_INSTALL_FUNCTION_FILES_VERSIONED};${SOURCE_FILES_VERSIONED};${POST_FILES_PROCESSED}" ${UPDATE_SCRIPT}) else() cat_files( "${PRE_FILES};${PRE_INSTALL_FUNCTION_FILES_VERSIONED};${SOURCE_FILES_VERSIONED};${POST_FILES_PROCESSED}" ${UPDATE_SCRIPT}) endif() endforeach(transition_mod_file) add_custom_target(sqlupdatescripts ALL DEPENDS ${UPDATE_SCRIPTS}) # Install target for the extension file and update scripts install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${INSTALL_FILE} ${UPDATE_SCRIPTS} DESTINATION "${PG_SHAREDIR}/extension") ================================================ FILE: sql/bgw_scheduler.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE OR REPLACE FUNCTION _timescaledb_functions.restart_background_workers() RETURNS BOOL AS '@LOADER_PATHNAME@', 'ts_bgw_db_workers_restart' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION _timescaledb_functions.stop_background_workers() RETURNS BOOL AS '@LOADER_PATHNAME@', 'ts_bgw_db_workers_stop' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION _timescaledb_functions.start_background_workers() RETURNS BOOL AS '@LOADER_PATHNAME@', 'ts_bgw_db_workers_start' LANGUAGE C VOLATILE; ================================================ FILE: sql/bgw_startup.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SELECT _timescaledb_functions.restart_background_workers(); ================================================ FILE: sql/bookend.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE OR REPLACE FUNCTION _timescaledb_functions.first_sfunc(internal, anyelement, "any") RETURNS internal AS '@MODULE_PATHNAME@', 'ts_first_sfunc' LANGUAGE C IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.first_combinefunc(internal, internal) RETURNS internal AS '@MODULE_PATHNAME@', 'ts_first_combinefunc' LANGUAGE C IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.last_sfunc(internal, anyelement, "any") RETURNS internal AS '@MODULE_PATHNAME@', 'ts_last_sfunc' LANGUAGE C IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.last_combinefunc(internal, internal) RETURNS internal AS '@MODULE_PATHNAME@', 'ts_last_combinefunc' LANGUAGE C IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.bookend_finalfunc(internal, anyelement, "any") RETURNS anyelement AS '@MODULE_PATHNAME@', 'ts_bookend_finalfunc' LANGUAGE C IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.bookend_serializefunc(internal) RETURNS bytea AS '@MODULE_PATHNAME@', 'ts_bookend_serializefunc' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.bookend_deserializefunc(bytea, internal) RETURNS internal AS '@MODULE_PATHNAME@', 'ts_bookend_deserializefunc' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; -- We started using CREATE OR REPLACE AGGREGATE for aggregate creation once the syntax was fully supported -- as it is easier to support idempotent changes this way. This will allow for changes to functions supporting -- the aggregate, and, for instance, the definition and inclusion of inverse functions for window function -- support. However, it should still be noted that changes to the data structures used for the internal -- state of the aggregate must be backwards compatible and the old format must be accepted by any new functions -- in order for them to continue working with Continuous Aggregates, where old states may have been materialized. --This aggregate returns the "first" value of the first argument when ordered by the second argument. --Ex. first(temp, time) returns the temp value for the row with the lowest time CREATE OR REPLACE AGGREGATE @extschema@.first(anyelement, "any") ( SFUNC = _timescaledb_functions.first_sfunc, STYPE = internal, COMBINEFUNC = _timescaledb_functions.first_combinefunc, SERIALFUNC = _timescaledb_functions.bookend_serializefunc, DESERIALFUNC = _timescaledb_functions.bookend_deserializefunc, PARALLEL = SAFE, FINALFUNC = _timescaledb_functions.bookend_finalfunc, FINALFUNC_EXTRA ); --This aggregate returns the "last" value of the first argument when ordered by the second argument. --Ex. last(temp, time) returns the temp value for the row with highest time CREATE OR REPLACE AGGREGATE @extschema@.last(anyelement, "any") ( SFUNC = _timescaledb_functions.last_sfunc, STYPE = internal, COMBINEFUNC = _timescaledb_functions.last_combinefunc, SERIALFUNC = _timescaledb_functions.bookend_serializefunc, DESERIALFUNC = _timescaledb_functions.bookend_deserializefunc, PARALLEL = SAFE, FINALFUNC = _timescaledb_functions.bookend_finalfunc, FINALFUNC_EXTRA ); ================================================ FILE: sql/cagg_utils.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE OR REPLACE FUNCTION _timescaledb_functions.cagg_validate_query( query TEXT, OUT is_valid BOOLEAN, OUT error_level TEXT, OUT error_code TEXT, OUT error_message TEXT, OUT error_detail TEXT, OUT error_hint TEXT ) RETURNS RECORD AS '@MODULE_PATHNAME@', 'ts_continuous_agg_validate_query' LANGUAGE C STRICT VOLATILE; CREATE OR REPLACE FUNCTION _timescaledb_functions.cagg_get_bucket_function_info( mat_hypertable_id INTEGER, -- The bucket function OUT bucket_func REGPROCEDURE, -- `bucket_width` argument of the function, e.g. "1 month" OUT bucket_width TEXT, -- optional `origin` argument of the function provided by the user OUT bucket_origin TEXT, -- optional `offset` argument of the function provided by the user OUT bucket_offset TEXT, -- optional `timezone` argument of the function provided by the user OUT bucket_timezone TEXT, -- fixed or variable sized bucket OUT bucket_fixed_width BOOLEAN ) RETURNS RECORD AS '@MODULE_PATHNAME@', 'ts_continuous_agg_get_bucket_function_info' LANGUAGE C STRICT VOLATILE; CREATE OR REPLACE FUNCTION _timescaledb_functions.cagg_get_grouping_columns( cagg REGCLASS ) RETURNS TEXT[] AS '@MODULE_PATHNAME@', 'ts_continuous_agg_get_grouping_columns' LANGUAGE C STRICT VOLATILE; ================================================ FILE: sql/cat.cmake ================================================ IF(POLICY CMP0012) CMAKE_POLICY(SET CMP0012 NEW) ENDIF() if (NOT DEFINED STRIP_REPLACE) set(STRIP_REPLACE OFF) endif() function(append_file IN_FILE OUT_FILE STRIP_REPLACE) file(READ ${IN_FILE} CONTENTS) if (${STRIP_REPLACE}) string(REPLACE " OR REPLACE " " " CONTENTS "${CONTENTS}") endif() file(APPEND ${OUT_FILE} "${CONTENTS}") endfunction() # Function to concatenate all files in SRC_FILE_LIST into file OUTPUT_FILE function(cat SRC_FILE_LIST OUTPUT_FILE STRIP_REPLACE) file(WRITE ${OUTPUT_FILE} "") foreach(SRC_FILE ${SRC_FILE_LIST}) append_file(${SRC_FILE} ${OUTPUT_FILE} ${STRIP_REPLACE}) endforeach() endfunction() cat("${SRC_FILE_LIST}" "${OUTPUT_FILE}" "${STRIP_REPLACE}") ================================================ FILE: sql/chunk.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Built-in function for calculating the next chunk interval when -- using adaptive chunking. The function can be replaced by a -- user-defined function with the same signature. -- -- The parameters passed to the function are as follows: -- -- dimension_id: the ID of the dimension to calculate the interval for -- dimension_coord: the coordinate / point on the dimensional axis -- where the tuple that triggered this chunk creation falls. -- chunk_target_size: the target size in bytes that the chunk should have. -- -- The function should return the new interval in dimension-specific -- time (ususally microseconds). CREATE OR REPLACE FUNCTION _timescaledb_functions.calculate_chunk_interval( dimension_id INTEGER, dimension_coord BIGINT, chunk_target_size BIGINT ) RETURNS BIGINT AS '@MODULE_PATHNAME@', 'ts_calculate_chunk_interval' LANGUAGE C; -- Get the status of the chunk CREATE OR REPLACE FUNCTION _timescaledb_functions.chunk_status(REGCLASS) RETURNS INT AS '@MODULE_PATHNAME@', 'ts_chunk_status' LANGUAGE C; -- Get the status of the chunk as text array CREATE OR REPLACE FUNCTION _timescaledb_functions.chunk_status_text(chunk_status int) RETURNS TEXT[] AS '@MODULE_PATHNAME@', 'ts_chunk_status_text' LANGUAGE C STRICT IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.chunk_status_text(chunk regclass) RETURNS TEXT[] AS $$ SELECT _timescaledb_functions.chunk_status_text(_timescaledb_functions.chunk_status($1)); $$ LANGUAGE SQL STRICT IMMUTABLE PARALLEL SAFE SET search_path TO pg_catalog, pg_temp;; --given a chunk's relid, return the id. Error out if not a chunk relid. CREATE OR REPLACE FUNCTION _timescaledb_functions.chunk_id_from_relid(relid OID) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_chunk_id_from_relid' LANGUAGE C STABLE STRICT PARALLEL SAFE; -- Show the definition of a chunk. CREATE OR REPLACE FUNCTION _timescaledb_functions.show_chunk(chunk REGCLASS) RETURNS TABLE(chunk_id INTEGER, hypertable_id INTEGER, schema_name NAME, table_name NAME, relkind "char", slices JSONB) AS '@MODULE_PATHNAME@', 'ts_chunk_show' LANGUAGE C VOLATILE; -- Create a chunk with the given dimensional constraints (slices) as -- given in the JSONB. If chunk_table is a valid relation, it will be -- attached to the hypertable and used as the data table for the new -- chunk. Note that schema_name and table_name need not be the same as -- the existing schema and name for chunk_table. The provided chunk -- table will be renamed and/or moved as necessary. CREATE OR REPLACE FUNCTION _timescaledb_functions.create_chunk( hypertable REGCLASS, slices JSONB, schema_name NAME = NULL, table_name NAME = NULL, chunk_table REGCLASS = NULL) RETURNS TABLE(chunk_id INTEGER, hypertable_id INTEGER, schema_name NAME, table_name NAME, relkind "char", slices JSONB, created BOOLEAN) AS '@MODULE_PATHNAME@', 'ts_chunk_create' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION _timescaledb_functions.freeze_chunk( chunk REGCLASS) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_chunk_freeze_chunk' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION _timescaledb_functions.unfreeze_chunk( chunk REGCLASS) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_chunk_unfreeze_chunk' LANGUAGE C VOLATILE; --wrapper for ts_chunk_drop --drops the chunk table and its entry in the chunk catalog CREATE OR REPLACE FUNCTION _timescaledb_functions.drop_chunk( chunk REGCLASS) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_chunk_drop_single_chunk' LANGUAGE C VOLATILE; -- internal API used by OSM extension to attach a table as a chunk of the hypertable CREATE OR REPLACE FUNCTION _timescaledb_functions.attach_osm_table_chunk( hypertable REGCLASS, chunk REGCLASS) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_chunk_attach_osm_table_chunk' LANGUAGE C VOLATILE; -- internal API used by OSM extension to drop an OSM chunk table from the hypertable CREATE OR REPLACE FUNCTION _timescaledb_functions.drop_osm_chunk(hypertable REGCLASS) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_chunk_drop_osm_chunk' LANGUAGE C VOLATILE; CREATE OR REPLACE PROCEDURE @extschema@.detach_chunk(chunk REGCLASS) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_detach_chunk'; CREATE OR REPLACE PROCEDURE @extschema@.attach_chunk(hypertable REGCLASS, chunk REGCLASS, slices JSONB) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_attach_chunk'; ================================================ FILE: sql/chunk_constraint.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- create constraint on newly created chunk based on hypertable constraint CREATE OR REPLACE FUNCTION _timescaledb_functions.chunk_constraint_add_table_constraint( chunk_constraint_row _timescaledb_catalog.chunk_constraint ) RETURNS VOID LANGUAGE PLPGSQL AS $BODY$ DECLARE chunk_row _timescaledb_catalog.chunk; hypertable_row _timescaledb_catalog.hypertable; constraint_oid OID; constraint_type CHAR; check_sql TEXT; def TEXT; indx_tablespace NAME; tablespace_def TEXT; BEGIN SELECT * INTO STRICT chunk_row FROM _timescaledb_catalog.chunk c WHERE c.id = chunk_constraint_row.chunk_id; SELECT * INTO STRICT hypertable_row FROM _timescaledb_catalog.hypertable h WHERE h.id = chunk_row.hypertable_id; IF chunk_constraint_row.dimension_slice_id IS NOT NULL THEN RAISE 'cannot create dimension constraint %', chunk_constraint_row; ELSIF chunk_constraint_row.hypertable_constraint_name IS NOT NULL THEN SELECT oid, contype INTO STRICT constraint_oid, constraint_type FROM pg_constraint WHERE conname=chunk_constraint_row.hypertable_constraint_name AND conrelid = format('%I.%I', hypertable_row.schema_name, hypertable_row.table_name)::regclass::oid; IF constraint_type IN ('p','u') THEN -- since primary keys and unique constraints are backed by an index -- they might have an index tablespace assigned -- the tablspace is not part of the constraint definition so -- we have to append it explicitly to preserve it SELECT T.spcname INTO indx_tablespace FROM pg_constraint C, pg_class I, pg_tablespace T WHERE C.oid = constraint_oid AND C.contype IN ('p', 'u') AND I.oid = C.conindid AND I.reltablespace = T.oid; def := pg_get_constraintdef(constraint_oid); ELSIF constraint_type = 't' THEN -- constraint triggers are copied separately with normal triggers def := NULL; ELSE def := pg_get_constraintdef(constraint_oid); END IF; ELSE RAISE 'unknown constraint type'; END IF; IF def IS NOT NULL THEN -- to allow for custom types with operators outside of pg_catalog -- we set search_path to @extschema@ SET LOCAL search_path TO @extschema@, pg_temp; EXECUTE pg_catalog.format( $$ ALTER TABLE %I.%I ADD CONSTRAINT %I %s $$, chunk_row.schema_name, chunk_row.table_name, chunk_constraint_row.constraint_name, def ); -- if constraint (primary or unique) needs a tablespace then add it -- via a separate ALTER INDEX SET TABLESPACE command. We cannot append it -- to the "def" string above since it leads to a SYNTAX error when -- "DEFERRABLE" or "INITIALLY DEFERRED" are used in the constraint IF indx_tablespace IS NOT NULL THEN EXECUTE pg_catalog.format( $$ ALTER INDEX %I.%I SET TABLESPACE %I $$, chunk_row.schema_name, chunk_constraint_row.constraint_name, indx_tablespace ); END IF; END IF; END $BODY$ SET search_path TO pg_catalog, pg_temp; -- Clone fk constraint from a hypertable to a compressed chunk CREATE OR REPLACE FUNCTION _timescaledb_functions.constraint_clone( constraint_oid OID, target_oid REGCLASS ) RETURNS VOID LANGUAGE PLPGSQL AS $BODY$ DECLARE constraint_name NAME; def TEXT; BEGIN def := pg_get_constraintdef(constraint_oid); SELECT conname INTO STRICT constraint_name FROM pg_constraint WHERE oid = constraint_oid; IF def IS NULL THEN RAISE 'constraint not found'; END IF; -- to allow for custom types with operators outside of pg_catalog -- we set search_path to @extschema@ SET LOCAL search_path TO @extschema@, pg_temp; EXECUTE pg_catalog.format($$ ALTER TABLE %s ADD CONSTRAINT %I %s $$, target_oid::pg_catalog.text, constraint_name, def); END $BODY$ SET search_path TO pg_catalog, pg_temp; ================================================ FILE: sql/comment_apache.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. COMMENT ON EXTENSION timescaledb IS 'Enables scalable inserts and complex queries for time-series data (Apache 2 Edition)'; ================================================ FILE: sql/comment_tsl.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. COMMENT ON EXTENSION timescaledb IS 'Enables scalable inserts and complex queries for time-series data (Community Edition)'; ================================================ FILE: sql/compat.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- TimescaleDB 2.12 moved all functions present in _timescaledb_internal into -- _timescaledb_functions. This file contains a compatibility layer to allow -- for more flexibility when migrating for any users calling these internal -- functions. -- This compatibility layer will be removed in a future versions. CREATE OR REPLACE FUNCTION _timescaledb_internal.alter_job_set_hypertable_id(job_id integer, hypertable regclass) RETURNS integer LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.alter_job_set_hypertable_id(integer,regclass) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.alter_job_set_hypertable_id($1,$2); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.attach_osm_table_chunk(hypertable regclass, chunk regclass) RETURNS boolean LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.attach_osm_table_chunk(regclass,regclass) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.attach_osm_table_chunk($1,$2); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.cagg_watermark(hypertable_id integer) RETURNS bigint LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.cagg_watermark(integer) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.cagg_watermark($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.cagg_watermark_materialized(hypertable_id integer) RETURNS bigint LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.cagg_watermark_materialized(integer) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.cagg_watermark_materialized($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.calculate_chunk_interval(dimension_id integer,dimension_coord bigint,chunk_target_size bigint) RETURNS bigint LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.calculate_chunk_interval(integer,bigint,bigint) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.calculate_chunk_interval($1,$2,$3); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.chunk_constraint_add_table_constraint(chunk_constraint_row _timescaledb_catalog.chunk_constraint) RETURNS void LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.chunk_constraint_add_table_constraint(_timescaledb_catalog.chunk_constraint) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; PERFORM _timescaledb_functions.chunk_constraint_add_table_constraint($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.chunk_id_from_relid(relid oid) RETURNS integer LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.chunk_id_from_relid(oid) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.chunk_id_from_relid($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.chunk_status(regclass) RETURNS integer LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.chunk_status(regclass) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.chunk_status($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.chunks_local_size(schema_name_in name,table_name_in name) RETURNS TABLE (chunk_id integer, chunk_schema NAME, chunk_name NAME, table_bytes bigint, index_bytes bigint, toast_bytes bigint, total_bytes bigint) LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.chunks_local_size(name,name) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN QUERY SELECT * FROM _timescaledb_functions.chunks_local_size($1,$2); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.compressed_chunk_local_stats(schema_name_in name,table_name_in name) RETURNS TABLE (chunk_schema name, chunk_name name, compression_status text, before_compression_table_bytes bigint, before_compression_index_bytes bigint, before_compression_toast_bytes bigint, before_compression_total_bytes bigint, after_compression_table_bytes bigint, after_compression_index_bytes bigint, after_compression_toast_bytes bigint, after_compression_total_bytes bigint) LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.compressed_chunk_local_stats(name,name) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN QUERY SELECT * FROM _timescaledb_functions.compressed_chunk_local_stats($1,$2); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.compressed_chunk_remote_stats(schema_name_in name,table_name_in name) RETURNS TABLE ( chunk_schema name, chunk_name name, compression_status text, before_compression_table_bytes bigint, before_compression_index_bytes bigint, before_compression_toast_bytes bigint, before_compression_total_bytes bigint, after_compression_table_bytes bigint, after_compression_index_bytes bigint, after_compression_toast_bytes bigint, after_compression_total_bytes bigint, node_name name) LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.compressed_chunk_remote_stats(name,name) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN QUERY SELECT * FROM _timescaledb_functions.compressed_chunk_remote_stats($1,$2); END$$ SET search_path TO pg_catalog,pg_temp; -- we have to prefix slices, schema_name and table_name parameter with _ here to not clash with output names otherwise plpgsql will complain CREATE OR REPLACE FUNCTION _timescaledb_internal.create_chunk(hypertable regclass,_slices jsonb,_schema_name name=NULL,_table_name name=NULL,chunk_table regclass=NULL) RETURNS TABLE(chunk_id INTEGER, hypertable_id INTEGER, schema_name NAME, table_name NAME, relkind "char", slices JSONB, created BOOLEAN) LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.create_chunk(regclass,jsonb,name,name,regclass) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN QUERY SELECT * FROM _timescaledb_functions.create_chunk($1,$2,$3,$4,$5); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.create_compressed_chunk(chunk regclass,chunk_table regclass,uncompressed_heap_size bigint,uncompressed_toast_size bigint,uncompressed_index_size bigint,compressed_heap_size bigint,compressed_toast_size bigint,compressed_index_size bigint,numrows_pre_compression bigint,numrows_post_compression bigint) RETURNS regclass LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.create_compressed_chunk(regclass,regclass,bigint,bigint,bigint,bigint,bigint,bigint,bigint,bigint) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.create_compressed_chunk($1,$2,$3,$4,$5,$6,$7,$8,$9,$10); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.drop_chunk(chunk regclass) RETURNS boolean LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.drop_chunk(regclass) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.drop_chunk($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.freeze_chunk(chunk regclass) RETURNS boolean LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.freeze_chunk(regclass) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.freeze_chunk($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.generate_uuid() RETURNS uuid LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.generate_uuid() is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.generate_uuid(); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.get_approx_row_count(relation regclass) RETURNS bigint LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.get_approx_row_count(regclass) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.get_approx_row_count($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.get_compressed_chunk_index_for_recompression(uncompressed_chunk regclass) RETURNS regclass LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.get_compressed_chunk_index_for_recompression(regclass) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.get_compressed_chunk_index_for_recompression($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.get_create_command(table_name name) RETURNS text LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.get_create_command(name) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.get_create_command($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.get_git_commit() RETURNS TABLE(commit_tag TEXT, commit_hash TEXT, commit_time TIMESTAMPTZ) LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.get_git_commit() is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN QUERY SELECT * FROM _timescaledb_functions.get_git_commit(); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.get_os_info() RETURNS TABLE(sysname TEXT, version TEXT, release TEXT, version_pretty TEXT) LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.get_os_info() is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN QUERY SELECT * FROM _timescaledb_functions.get_os_info(); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.get_partition_for_key(val anyelement) RETURNS integer LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.get_partition_for_key(anyelement) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.get_partition_for_key($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.get_partition_hash(val anyelement) RETURNS integer LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.get_partition_hash(anyelement) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.get_partition_hash($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.hypertable_local_size(schema_name_in name,table_name_in name) RETURNS TABLE ( table_bytes BIGINT, index_bytes BIGINT, toast_bytes BIGINT, total_bytes BIGINT) LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.hypertable_local_size(name,name) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN QUERY SELECT * FROM _timescaledb_functions.hypertable_local_size($1,$2); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.interval_to_usec(chunk_interval interval) RETURNS bigint LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.interval_to_usec(interval) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.interval_to_usec($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.policy_compression_check(config jsonb) RETURNS void LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.policy_compression_check(jsonb) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; PERFORM _timescaledb_functions.policy_compression_check($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.policy_job_stat_history_retention(job_id integer,config jsonb) RETURNS integer LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.policy_job_stat_history_retention(integer,jsonb) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.policy_job_stat_history_retention($1,$2); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.policy_job_stat_history_retention_check(config jsonb) RETURNS void LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.policy_job_stat_history_retention_check(jsonb) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; PERFORM _timescaledb_functions.policy_job_stat_history_retention_check($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.policy_refresh_continuous_aggregate_check(config jsonb) RETURNS void LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.policy_refresh_continuous_aggregate_check(jsonb) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; PERFORM _timescaledb_functions.policy_refresh_continuous_aggregate_check($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.policy_reorder_check(config jsonb) RETURNS void LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.policy_reorder_check(jsonb) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; PERFORM _timescaledb_functions.policy_reorder_check($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.policy_retention_check(config jsonb) RETURNS void LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.policy_retention_check(jsonb) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; PERFORM _timescaledb_functions.policy_retention_check($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.process_ddl_event() RETURNS event_trigger LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.process_ddl_event() is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.process_ddl_event(); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.range_value_to_pretty(time_value bigint,column_type regtype) RETURNS text LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.range_value_to_pretty(bigint,regtype) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.range_value_to_pretty($1,$2); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.recompress_chunk_segmentwise(uncompressed_chunk regclass,if_compressed boolean=false) RETURNS regclass LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.recompress_chunk_segmentwise(regclass,boolean) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.recompress_chunk_segmentwise($1,$2); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.relation_size(relation regclass) RETURNS TABLE (total_size BIGINT, heap_size BIGINT, index_size BIGINT, toast_size BIGINT) LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.relation_size(regclass) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN QUERY SELECT * FROM _timescaledb_functions.relation_size($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.restart_background_workers() RETURNS boolean LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.restart_background_workers() is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.restart_background_workers(); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.show_chunk(chunk regclass) RETURNS TABLE(chunk_id INTEGER, hypertable_id INTEGER, schema_name NAME, table_name NAME, relkind "char", slices JSONB) LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.show_chunk(regclass) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN QUERY SELECT * FROM _timescaledb_functions.show_chunk($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.start_background_workers() RETURNS boolean LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.start_background_workers() is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.start_background_workers(); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.stop_background_workers() RETURNS boolean LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.stop_background_workers() is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.stop_background_workers(); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.subtract_integer_from_now(hypertable_relid regclass,lag bigint) RETURNS bigint LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.subtract_integer_from_now(regclass,bigint) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.subtract_integer_from_now($1,$2); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.time_to_internal(time_val anyelement) RETURNS bigint LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.time_to_internal(anyelement) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.time_to_internal($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.to_date(unixtime_us bigint) RETURNS date LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.to_date(bigint) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.to_date($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.to_interval(unixtime_us bigint) RETURNS interval LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.to_interval(bigint) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.to_interval($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.to_timestamp(unixtime_us bigint) RETURNS timestamp with time zone LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.to_timestamp(bigint) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.to_timestamp($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.to_timestamp_without_timezone(unixtime_us bigint) RETURNS timestamp without time zone LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.to_timestamp_without_timezone(bigint) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.to_timestamp_without_timezone($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.to_unix_microseconds(ts timestamp with time zone) RETURNS bigint LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.to_unix_microseconds(timestamp with time zone) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.to_unix_microseconds($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.tsl_loaded() RETURNS boolean LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.tsl_loaded() is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.tsl_loaded(); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.unfreeze_chunk(chunk regclass) RETURNS boolean LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'function _timescaledb_internal.unfreeze_chunk(regclass) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; RETURN _timescaledb_functions.unfreeze_chunk($1); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE PROCEDURE _timescaledb_internal.policy_compression(job_id integer,config jsonb) LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'procedure _timescaledb_internal.policy_compression(integer,jsonb) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; CALL _timescaledb_functions.policy_compression($1,$2); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE PROCEDURE _timescaledb_internal.policy_recompression(job_id integer,config jsonb) LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'procedure _timescaledb_internal.policy_recompression(integer,jsonb) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; CALL _timescaledb_functions.policy_recompression($1,$2); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE PROCEDURE _timescaledb_internal.policy_refresh_continuous_aggregate(job_id integer,config jsonb) LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'procedure _timescaledb_internal.policy_refresh_continuous_aggregate(integer,jsonb) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; CALL _timescaledb_functions.policy_refresh_continuous_aggregate($1,$2); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE PROCEDURE _timescaledb_internal.policy_reorder(job_id integer,config jsonb) LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'procedure _timescaledb_internal.policy_reorder(integer,jsonb) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; CALL _timescaledb_functions.policy_reorder($1,$2); END$$ SET search_path TO pg_catalog,pg_temp; CREATE OR REPLACE PROCEDURE _timescaledb_internal.policy_retention(job_id integer,config jsonb) LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'procedure _timescaledb_internal.policy_retention(integer,jsonb) is deprecated and has been moved to _timescaledb_functions schema. this compatibility function will be removed in a future version.'; END IF; CALL _timescaledb_functions.policy_retention($1,$2); END$$ SET search_path TO pg_catalog,pg_temp; ================================================ FILE: sql/compression.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE OR REPLACE FUNCTION _timescaledb_functions.compressed_data_to_array(_timescaledb_internal.compressed_data, ANYELEMENT) RETURNS ANYARRAY AS '@MODULE_PATHNAME@', 'ts_compressed_data_to_array' LANGUAGE C IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.compressed_data_column_size(_timescaledb_internal.compressed_data, ANYELEMENT) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_compressed_data_column_size' LANGUAGE C IMMUTABLE PARALLEL SAFE; ================================================ FILE: sql/compression_defaults.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- This function return a jsonb with the following keys: -- - columns: an array of column names that shold be used for segment by -- - confidence: a number between 0 and 10 (most confident) indicating how sure we are. -- - message: a message that should be displayed to the user to evaluate the result. CREATE OR REPLACE FUNCTION _timescaledb_functions.get_segmentby_defaults( relation regclass ) RETURNS JSONB LANGUAGE PLPGSQL AS $BODY$ DECLARE _table_name NAME; _schema_name NAME; _hypertable_row _timescaledb_catalog.hypertable; _segmentby NAME; _cnt int; BEGIN SELECT n.nspname, c.relname INTO STRICT _schema_name, _table_name FROM pg_class c INNER JOIN pg_namespace n ON (n.oid = c.relnamespace) WHERE c.oid = relation; SELECT * INTO STRICT _hypertable_row FROM _timescaledb_catalog.hypertable h WHERE h.table_name = _table_name AND h.schema_name = _schema_name; --STEP 1 if column stats exist use unique indexes. --Pick the column that comes first in any such indexes --Select the column such that tuples are segmented evenly across distinct values. --Note: this will only pick a column that is NOT unique in a multi-column unique index. with index_attr as ( SELECT a.attnum, min(a.pos) as pos FROM ( SELECT indkey, indnkeyatts FROM pg_catalog.pg_index WHERE indisunique AND indrelid = relation ) i INNER JOIN LATERAL ( SELECT * FROM unnest(i.indkey) WITH ORDINALITY ) a(attnum, pos) ON TRUE WHERE a.pos <= i.indnkeyatts GROUP BY a.attnum ), stats_with_stddev as ( SELECT a.attname, i.pos, ROUND(stddev_pop(freqs)::numeric, 5) as freq_stddev FROM index_attr i INNER JOIN pg_attribute a ON a.attnum = i.attnum AND a.attrelid = relation INNER JOIN pg_type t ON t.oid = a.atttypid INNER JOIN pg_stats s ON s.attname = a.attname AND s.schemaname = _schema_name AND s.tablename = _table_name AND s.inherited = true LEFT JOIN LATERAL unnest(s.most_common_freqs) as freqs ON TRUE WHERE a.attname NOT IN ( SELECT column_name FROM _timescaledb_catalog.dimension d WHERE d.hypertable_id = _hypertable_row.id ) AND s.n_distinct > 1 -- exclude date/time type category AND t.typcategory NOT IN ('D') GROUP BY a.attname, i.pos ) SELECT attname INTO _segmentby FROM stats_with_stddev ORDER BY pos ASC, freq_stddev ASC NULLS LAST LIMIT 1; IF FOUND THEN return json_build_object('columns', json_build_array(_segmentby), 'confidence', 10); END IF; --STEP 2 if column stats exist and no unique indexes use non-unique indexes. --Pick the column that comes first in any such indexes --Select the column such that tuples are segmented evenly across distinct values. with index_attr as ( SELECT a.attnum, min(a.pos) as pos FROM (select indkey, indnkeyatts from pg_catalog.pg_index where NOT indisunique and indrelid = relation) i INNER JOIN LATERAL (select * from unnest(i.indkey) with ordinality) a(attnum, pos) ON (TRUE) WHERE a.pos <= i.indnkeyatts GROUP BY 1 ), stats_with_stddev as ( SELECT a.attname, i.pos, ROUND(stddev_pop(freqs)::numeric, 5) as freq_stddev FROM index_attr i INNER JOIN pg_attribute a ON a.attnum = i.attnum AND a.attrelid = relation INNER JOIN pg_type t ON t.oid = a.atttypid INNER JOIN pg_stats s ON s.attname = a.attname AND s.schemaname = _schema_name AND s.tablename = _table_name AND s.inherited = true LEFT JOIN LATERAL unnest(s.most_common_freqs) as freqs ON TRUE WHERE a.attname NOT IN ( SELECT column_name FROM _timescaledb_catalog.dimension d WHERE d.hypertable_id = _hypertable_row.id ) AND s.n_distinct > 1 AND t.typcategory NOT IN ('D') GROUP BY a.attname, i.pos ) SELECT attname INTO _segmentby FROM stats_with_stddev ORDER BY pos ASC, freq_stddev ASC NULLS LAST LIMIT 1; IF FOUND THEN return json_build_object('columns', json_build_array(_segmentby), 'confidence', 8); END IF; --STEP 3 if column stats exist but there are no indexes --Select the column such that tuples are segmented evenly across distinct values. with stats_with_stddev as ( SELECT a.attname, ROUND(stddev_pop(freqs)::numeric, 5) as freq_stddev FROM pg_attribute a INNER JOIN pg_type t ON t.oid = a.atttypid INNER JOIN pg_stats s ON s.attname = a.attname AND s.schemaname = _schema_name AND s.tablename = _table_name AND s.inherited = true LEFT JOIN LATERAL unnest(s.most_common_freqs) as freqs ON TRUE WHERE a.attrelid = relation AND a.attname NOT IN ( SELECT column_name FROM _timescaledb_catalog.dimension d WHERE d.hypertable_id = _hypertable_row.id ) AND s.n_distinct > 1 AND t.typcategory NOT IN ('D') GROUP BY a.attname ) SELECT attname INTO _segmentby FROM stats_with_stddev ORDER BY freq_stddev ASC NULLS LAST LIMIT 1; IF FOUND THEN return json_build_object('columns', json_build_array(_segmentby), 'confidence', 7); END IF; --STEP 4 if column stats do not exist use non-unique indexes. Pick the column that comes first in any such indexes. Ties are broken arbitrarily. with index_attr as ( SELECT a.attnum, min(a.pos) as pos FROM (select indkey, indnkeyatts from pg_catalog.pg_index where NOT indisunique and indrelid = relation) i INNER JOIN LATERAL (select * from unnest(i.indkey) with ordinality) a(attnum, pos) ON (TRUE) WHERE a.pos <= i.indnkeyatts GROUP BY 1 ) SELECT a.attname INTO _segmentby FROM index_attr i INNER JOIN pg_attribute a on (a.attnum = i.attnum AND a.attrelid = relation) INNER JOIN pg_type t ON t.oid = a.atttypid LEFT JOIN pg_catalog.pg_attrdef ad ON (ad.adrelid = relation AND ad.adnum = a.attnum) LEFT JOIN pg_stats s ON s.attname = a.attname AND s.schemaname = _schema_name AND s.tablename = _table_name AND s.inherited = true WHERE a.attname NOT IN (SELECT column_name FROM _timescaledb_catalog.dimension d WHERE d.hypertable_id = _hypertable_row.id) AND s.n_distinct is null AND a.attidentity = '' AND (ad.adbin IS NULL OR pg_get_expr(adbin, adrelid) not like 'nextval%') AND t.typcategory NOT IN ('D') ORDER BY i.pos LIMIT 1; IF FOUND THEN return json_build_object( 'columns', json_build_array(_segmentby), 'confidence', 5, 'message', 'Please make sure '|| _segmentby||' is not a unique column and appropriate for a segment by'); END IF; --STEP 5 if column stats do not exist and no non-unique indexes, use unique indexes. Pick the column that comes first in any such indexes. Ties are broken arbitrarily. with index_attr as ( SELECT a.attnum, min(a.pos) as pos FROM (select indkey, indnkeyatts from pg_catalog.pg_index where indisunique and indrelid = relation) i INNER JOIN LATERAL (select * from unnest(i.indkey) with ordinality) a(attnum, pos) ON (TRUE) WHERE a.pos <= i.indnkeyatts GROUP BY 1 ) SELECT a.attname INTO _segmentby FROM index_attr i INNER JOIN pg_attribute a on (a.attnum = i.attnum AND a.attrelid = relation) INNER JOIN pg_type t ON t.oid = a.atttypid LEFT JOIN pg_catalog.pg_attrdef ad ON (ad.adrelid = relation AND ad.adnum = a.attnum) LEFT JOIN pg_stats s ON s.attname = a.attname AND s.schemaname = _schema_name AND s.tablename = _table_name AND s.inherited = true WHERE a.attname NOT IN (SELECT column_name FROM _timescaledb_catalog.dimension d WHERE d.hypertable_id = _hypertable_row.id) AND s.n_distinct is null AND a.attidentity = '' AND (ad.adbin IS NULL OR pg_get_expr(adbin, adrelid) not like 'nextval%') AND t.typcategory NOT IN ('D') ORDER BY i.pos LIMIT 1; IF FOUND THEN return json_build_object( 'columns', json_build_array(_segmentby), 'confidence', 5, 'message', 'Please make sure '|| _segmentby||' is not a unique column and appropriate for a segment by'); END IF; --are there any indexed columns that are not dimemsions and are not serial/identity? with index_attr as ( SELECT a.attnum, min(a.pos) as pos FROM (select indkey, indnkeyatts from pg_catalog.pg_index where indisunique and indrelid = relation) i INNER JOIN LATERAL (select * from unnest(i.indkey) with ordinality) a(attnum, pos) ON (TRUE) WHERE a.pos <= i.indnkeyatts GROUP BY 1 ) SELECT count(*) INTO STRICT _cnt FROM index_attr i INNER JOIN pg_attribute a on (a.attnum = i.attnum AND a.attrelid = relation) INNER JOIN pg_type t ON t.oid = a.atttypid LEFT JOIN pg_catalog.pg_attrdef ad ON (ad.adrelid = relation AND ad.adnum = a.attnum) WHERE a.attname NOT IN (SELECT column_name FROM _timescaledb_catalog.dimension d WHERE d.hypertable_id = _hypertable_row.id) AND a.attidentity = '' AND (ad.adbin IS NULL OR pg_get_expr(adbin, adrelid) not like 'nextval%') AND t.typcategory NOT IN ('D'); IF _cnt > 0 THEN --there are many potential candidates. We do not have enough information to choose one. return json_build_object( 'columns', json_build_array(), 'confidence', 0, 'message', 'Several columns are potential segment by candidates and we do not have enough information to choose one. Please use the segment_by option to explicitly specify the segment_by column'); ELSE --there are no potential candidates. There is a good chance no segment by is the correct choice. return json_build_object( 'columns', json_build_array(), 'confidence', 5, 'message', 'You do not have any indexes on columns that can be used for segment_by and thus we are not using segment_by for converting to columnstore. Please make sure you are not missing any indexes'); END IF; END $BODY$ SET search_path TO pg_catalog, pg_temp; -- This function return a jsonb with the following keys: -- - clauses: an array of column names and sort order key words that shold be used for order by. -- - confidence: a number between 0 and 10 (most confident) indicating how sure we are. -- - message: a message that should be shown to the user to evaluate the result. CREATE OR REPLACE FUNCTION _timescaledb_functions.get_orderby_defaults( relation regclass, segment_by_cols text[] ) RETURNS JSONB LANGUAGE PLPGSQL AS $BODY$ DECLARE _table_name NAME; _schema_name NAME; _hypertable_row _timescaledb_catalog.hypertable; _orderby_names NAME[]; _dimension_names NAME[]; _first_index_attrs NAME[]; _orderby_clauses text[]; _confidence int; BEGIN SELECT n.nspname, c.relname INTO STRICT _schema_name, _table_name FROM pg_class c INNER JOIN pg_namespace n ON (n.oid = c.relnamespace) WHERE c.oid = relation; SELECT * INTO STRICT _hypertable_row FROM _timescaledb_catalog.hypertable h WHERE h.table_name = _table_name AND h.schema_name = _schema_name; --start with the unique index columns minus the segment by columns with index_attr as ( SELECT a.attnum, min(a.pos) as pos FROM --is there a better way to pick the right unique index if there are multiple? (select indkey, indnkeyatts from pg_catalog.pg_index where indisunique and indrelid = relation limit 1) i INNER JOIN LATERAL (select * from unnest(i.indkey) with ordinality) a(attnum, pos) ON (TRUE) WHERE a.pos <= i.indnkeyatts GROUP BY 1 ) SELECT array_agg(a.attname ORDER BY i.pos) INTO _orderby_names FROM index_attr i INNER JOIN pg_attribute a on (a.attnum = i.attnum AND a.attrelid = relation) WHERE NOT(a.attname::text = ANY (segment_by_cols)); if _orderby_names is null then _orderby_names := array[]::name[]; _confidence := 5; else _confidence := 8; end if; --add dimension colomns to the end. A dimension column like time should probably always be part of the order by. SELECT array_agg(d.column_name) INTO _dimension_names FROM _timescaledb_catalog.dimension d WHERE d.hypertable_id = _hypertable_row.id AND NOT(d.column_name::text = ANY (_orderby_names)) AND NOT(d.column_name::text = ANY (segment_by_cols)); _orderby_names := _orderby_names || _dimension_names; --add the first attribute of any index with index_attr as ( SELECT a.attnum, min(a.pos) as pos FROM (select indkey, indnkeyatts from pg_catalog.pg_index where indrelid = relation) i INNER JOIN LATERAL (select * from unnest(i.indkey) with ordinality) a(attnum, pos) ON (TRUE) WHERE a.pos = 1 GROUP BY 1 ) SELECT array_agg(a.attname ORDER BY i.pos) INTO _first_index_attrs FROM index_attr i INNER JOIN pg_attribute a on (a.attnum = i.attnum AND a.attrelid = relation) WHERE NOT(a.attname::text = ANY (_orderby_names)) AND NOT(a.attname::text = ANY (segment_by_cols)); _orderby_names := _orderby_names || _first_index_attrs; --add DESC to any dimensions SELECT coalesce(array_agg( CASE WHEN d.column_name IS NULL THEN format('%I', a.colname) ELSE format('%I DESC', a.colname) END ORDER BY pos), array[]::text[]) INTO STRICT _orderby_clauses FROM unnest(_orderby_names) WITH ORDINALITY as a(colname, pos) LEFT JOIN _timescaledb_catalog.dimension d ON (d.column_name = a.colname AND d.hypertable_id = _hypertable_row.id); return json_build_object('clauses', _orderby_clauses, 'confidence', _confidence); END $BODY$ SET search_path TO pg_catalog, pg_temp; ================================================ FILE: sql/ddl_api.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- This file defines DDL functions for adding and manipulating hypertables. -- Converts a regular postgres table to a hypertable. -- -- relation - The OID of the table to be converted -- time_column_name - Name of the column that contains time for a given record -- partitioning_column - Name of the column to partition data by -- number_partitions - (Optional) Number of partitions for data -- associated_schema_name - (Optional) Schema for internal hypertable tables -- associated_table_prefix - (Optional) Prefix for internal hypertable table names -- chunk_time_interval - (Optional) Initial time interval for a chunk -- create_default_indexes - (Optional) Whether or not to create the default indexes -- if_not_exists - (Optional) Do not fail if table is already a hypertable -- partitioning_func - (Optional) The partitioning function to use for spatial partitioning -- migrate_data - (Optional) Set to true to migrate any existing data in the table to chunks -- chunk_target_size - (Optional) The target size for chunks (e.g., '1000MB', 'estimate', or 'off') -- chunk_sizing_func - (Optional) A function to calculate the chunk time interval for new chunks -- time_partitioning_func - (Optional) The partitioning function to use for "time" partitioning CREATE OR REPLACE FUNCTION @extschema@.create_hypertable( relation REGCLASS, time_column_name NAME, partitioning_column NAME = NULL, number_partitions INTEGER = NULL, associated_schema_name NAME = NULL, associated_table_prefix NAME = NULL, chunk_time_interval ANYELEMENT = NULL::bigint, create_default_indexes BOOLEAN = TRUE, if_not_exists BOOLEAN = FALSE, partitioning_func REGPROC = NULL, migrate_data BOOLEAN = FALSE, chunk_target_size TEXT = NULL, chunk_sizing_func REGPROC = '_timescaledb_functions.calculate_chunk_interval'::regproc, time_partitioning_func REGPROC = NULL ) RETURNS TABLE(hypertable_id INT, schema_name NAME, table_name NAME, created BOOL) AS '@MODULE_PATHNAME@', 'ts_hypertable_create' LANGUAGE C VOLATILE; -- A generalized hypertable creation API that can be used to convert a PostgreSQL table -- with TIME/SERIAL/BIGSERIAL columns to a hypertable. -- -- relation - The OID of the table to be converted -- dimension - The dimension to use for partitioning -- create_default_indexes (Optional) Whether or not to create the default indexes -- if_not_exists (Optional) Do not fail if table is already a hypertable -- migrate_data (Optional) Set to true to migrate any existing data in the table to chunks CREATE OR REPLACE FUNCTION @extschema@.create_hypertable( relation REGCLASS, dimension _timescaledb_internal.dimension_info, create_default_indexes BOOLEAN = TRUE, if_not_exists BOOLEAN = FALSE, migrate_data BOOLEAN = FALSE ) RETURNS TABLE(hypertable_id INT, created BOOL) AS '@MODULE_PATHNAME@', 'ts_hypertable_create_general' LANGUAGE C VOLATILE; -- Set adaptive chunking. To disable, set chunk_target_size => 'off'. CREATE OR REPLACE FUNCTION @extschema@.set_adaptive_chunking( hypertable REGCLASS, chunk_target_size TEXT, INOUT chunk_sizing_func REGPROC = '_timescaledb_functions.calculate_chunk_interval'::regproc, OUT chunk_target_size BIGINT ) RETURNS RECORD AS '@MODULE_PATHNAME@', 'ts_chunk_adaptive_set' LANGUAGE C VOLATILE; -- Update chunk_time_interval for a hypertable [DEPRECATED]. -- -- hypertable - The OID of the table corresponding to a hypertable whose time -- interval should be updated -- chunk_time_interval - The new time interval. For hypertables with integral -- time columns, this must be an integral type. For hypertables with a -- TIMESTAMP/TIMESTAMPTZ/DATE type, it can be integral which is treated as -- microseconds, or an INTERVAL type. CREATE OR REPLACE FUNCTION @extschema@.set_chunk_time_interval( hypertable REGCLASS, chunk_time_interval ANYELEMENT, dimension_name NAME = NULL ) RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_dimension_set_interval' LANGUAGE C VOLATILE; -- Update partition_interval for a hypertable. -- -- hypertable - The OID of the table corresponding to a hypertable whose -- partition interval should be updated -- partition_interval - The new interval. For hypertables with integral/serial/bigserial -- time columns, this must be an integral type. For hypertables with a -- TIMESTAMP/TIMESTAMPTZ/DATE type, it can be integral which is treated as -- microseconds, or an INTERVAL type. CREATE OR REPLACE FUNCTION @extschema@.set_partitioning_interval( hypertable REGCLASS, partition_interval ANYELEMENT, dimension_name NAME = NULL ) RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_dimension_set_interval' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION @extschema@.set_number_partitions( hypertable REGCLASS, number_partitions INTEGER, dimension_name NAME = NULL ) RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_dimension_set_num_slices' LANGUAGE C VOLATILE; -- Drop chunks older than the given timestamp for the specific -- hypertable or continuous aggregate. CREATE OR REPLACE FUNCTION @extschema@.drop_chunks( relation REGCLASS, older_than "any" = NULL, newer_than "any" = NULL, verbose BOOLEAN = FALSE, created_before "any" = NULL, created_after "any" = NULL ) RETURNS SETOF TEXT AS '@MODULE_PATHNAME@', 'ts_chunk_drop_chunks' LANGUAGE C VOLATILE PARALLEL UNSAFE; -- show chunks older than or newer than a specific time. -- `relation` must be a valid hypertable or continuous aggregate. CREATE OR REPLACE FUNCTION @extschema@.show_chunks( relation REGCLASS, older_than "any" = NULL, newer_than "any" = NULL, created_before "any" = NULL, created_after "any" = NULL ) RETURNS SETOF REGCLASS AS '@MODULE_PATHNAME@', 'ts_chunk_show_chunks' LANGUAGE C STABLE PARALLEL SAFE; -- Add a dimension (of partitioning) to a hypertable [DEPRECATED] -- -- hypertable - OID of the table to add a dimension to -- column_name - NAME of the column to use in partitioning for this dimension -- number_partitions - Number of partitions, for non-time dimensions -- chunk_time_interval - Size of intervals for time dimensions (can be integral or INTERVAL) -- partitioning_func - Function used to partition the column -- if_not_exists - If set, and the dimension already exists, generate a notice instead of an error CREATE OR REPLACE FUNCTION @extschema@.add_dimension( hypertable REGCLASS, column_name NAME, number_partitions INTEGER = NULL, chunk_time_interval ANYELEMENT = NULL::BIGINT, partitioning_func REGPROC = NULL, if_not_exists BOOLEAN = FALSE ) RETURNS TABLE(dimension_id INT, schema_name NAME, table_name NAME, column_name NAME, created BOOL) AS '@MODULE_PATHNAME@', 'ts_dimension_add' LANGUAGE C VOLATILE; -- Add a dimension (of partitioning) to a hypertable. -- -- hypertable - OID of the table to add a dimension to -- dimension - Dimension to add -- if_not_exists - If set, and the dimension already exists, generate a notice instead of an error CREATE OR REPLACE FUNCTION @extschema@.add_dimension( hypertable REGCLASS, dimension _timescaledb_internal.dimension_info, if_not_exists BOOLEAN = FALSE ) RETURNS TABLE(dimension_id INT, created BOOL) AS '@MODULE_PATHNAME@', 'ts_dimension_add_general' LANGUAGE C VOLATILE; -- Enable tracking of statistics on a column of a hypertable. -- -- hypertable - OID of the table to which the column belongs to -- column_name - The column to track statistics for -- if_not_exists - If set, and the entry already exists, generate a notice instead of an error -- Returns the "id" of the entry created. The "enabled" field -- is set to true if entry is created or exists already. CREATE OR REPLACE FUNCTION @extschema@.enable_chunk_skipping( hypertable REGCLASS, column_name NAME, if_not_exists BOOLEAN = FALSE ) RETURNS TABLE(column_stats_id INT, enabled BOOL) AS '@MODULE_PATHNAME@', 'ts_chunk_column_stats_enable' LANGUAGE C VOLATILE; -- Disable tracking of statistics on a column of a hypertable. -- -- hypertable - OID of the table to remove from -- column_name - NAME of the column on which the stats are tracked -- if_not_exists - If set, and the entry does not exist, -- generate a notice instead of an error. The "disabled" field -- is set to true if entry is deleted successfully. CREATE OR REPLACE FUNCTION @extschema@.disable_chunk_skipping( hypertable REGCLASS, column_name NAME, if_not_exists BOOLEAN = FALSE ) RETURNS TABLE(hypertable_id INT, column_name NAME, disabled BOOL) AS '@MODULE_PATHNAME@', 'ts_chunk_column_stats_disable' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION @extschema@.by_hash(column_name NAME, number_partitions INTEGER, partition_func regproc = NULL) RETURNS _timescaledb_internal.dimension_info LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_hash_dimension'; CREATE OR REPLACE FUNCTION @extschema@.by_range(column_name NAME, partition_interval ANYELEMENT = NULL::bigint, partition_func regproc = NULL) RETURNS _timescaledb_internal.dimension_info LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_range_dimension'; CREATE OR REPLACE FUNCTION @extschema@.attach_tablespace( tablespace NAME, hypertable REGCLASS, if_not_attached BOOLEAN = false ) RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_tablespace_attach' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION @extschema@.detach_tablespace( tablespace NAME, hypertable REGCLASS = NULL, if_attached BOOLEAN = false ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_tablespace_detach' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION @extschema@.detach_tablespaces(hypertable REGCLASS) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_tablespace_detach_all_from_hypertable' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION @extschema@.show_tablespaces(hypertable REGCLASS) RETURNS SETOF NAME AS '@MODULE_PATHNAME@', 'ts_tablespace_show' LANGUAGE C VOLATILE STRICT; -- Refresh a continuous aggregate across the given window. CREATE OR REPLACE PROCEDURE @extschema@.refresh_continuous_aggregate( continuous_aggregate REGCLASS, window_start "any", window_end "any", force BOOLEAN = FALSE, options JSONB = NULL ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_continuous_agg_refresh'; ================================================ FILE: sql/ddl_triggers.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. DROP EVENT TRIGGER IF EXISTS timescaledb_ddl_command_end; CREATE OR REPLACE FUNCTION _timescaledb_functions.process_ddl_event() RETURNS event_trigger AS '@MODULE_PATHNAME@', 'ts_timescaledb_process_ddl_event' LANGUAGE C; --EVENT TRIGGER MUST exclude the ALTER EXTENSION tag. CREATE EVENT TRIGGER timescaledb_ddl_command_end ON ddl_command_end WHEN TAG IN ('ALTER TABLE','CREATE TRIGGER','CREATE TABLE','CREATE INDEX','ALTER INDEX', 'DROP TABLE', 'DROP INDEX', 'DROP SCHEMA') EXECUTE FUNCTION _timescaledb_functions.process_ddl_event(); DROP EVENT TRIGGER IF EXISTS timescaledb_ddl_sql_drop; CREATE EVENT TRIGGER timescaledb_ddl_sql_drop ON sql_drop EXECUTE FUNCTION _timescaledb_functions.process_ddl_event(); ================================================ FILE: sql/debug_build_utils.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- This file contains utility functions that are only used in debug -- builds for debugging and testing. CREATE OR REPLACE FUNCTION debug_waitpoint_enable(TEXT) RETURNS VOID LANGUAGE C VOLATILE STRICT AS '@MODULE_PATHNAME@', 'ts_debug_point_enable'; CREATE OR REPLACE FUNCTION debug_waitpoint_release(TEXT) RETURNS VOID LANGUAGE C VOLATILE STRICT AS '@MODULE_PATHNAME@', 'ts_debug_point_release'; CREATE OR REPLACE FUNCTION debug_waitpoint_id(TEXT) RETURNS BIGINT LANGUAGE C VOLATILE STRICT AS '@MODULE_PATHNAME@', 'ts_debug_point_id'; CREATE OR REPLACE FUNCTION ts_now_mock() RETURNS TIMESTAMPTZ AS '@MODULE_PATHNAME@', 'ts_now_mock' LANGUAGE C STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION clear_hypertable_cache() RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_hypertable_cache_clear' LANGUAGE C; ================================================ FILE: sql/debug_utils.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- This file contains utility functions and views that are used for -- debugging in release builds. These are all placed in the schema -- _timescaledb_debug. CREATE OR REPLACE FUNCTION _timescaledb_functions.extension_state() RETURNS TEXT AS '@MODULE_PATHNAME@', 'ts_extension_get_state' LANGUAGE C; ================================================ FILE: sql/gapfill.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE OR REPLACE FUNCTION @extschema@.time_bucket_gapfill(bucket_width SMALLINT, ts SMALLINT, start SMALLINT=NULL, finish SMALLINT=NULL) RETURNS SMALLINT AS '@MODULE_PATHNAME@', 'ts_gapfill_int16_bucket' LANGUAGE C VOLATILE PARALLEL SAFE; CREATE OR REPLACE FUNCTION @extschema@.time_bucket_gapfill(bucket_width INT, ts INT, start INT=NULL, finish INT=NULL) RETURNS INT AS '@MODULE_PATHNAME@', 'ts_gapfill_int32_bucket' LANGUAGE C VOLATILE PARALLEL SAFE; CREATE OR REPLACE FUNCTION @extschema@.time_bucket_gapfill(bucket_width BIGINT, ts BIGINT, start BIGINT=NULL, finish BIGINT=NULL) RETURNS BIGINT AS '@MODULE_PATHNAME@', 'ts_gapfill_int64_bucket' LANGUAGE C VOLATILE PARALLEL SAFE; CREATE OR REPLACE FUNCTION @extschema@.time_bucket_gapfill(bucket_width INTERVAL, ts DATE, start DATE=NULL, finish DATE=NULL) RETURNS DATE AS '@MODULE_PATHNAME@', 'ts_gapfill_date_bucket' LANGUAGE C VOLATILE PARALLEL SAFE; CREATE OR REPLACE FUNCTION @extschema@.time_bucket_gapfill(bucket_width INTERVAL, ts TIMESTAMP, start TIMESTAMP=NULL, finish TIMESTAMP=NULL) RETURNS TIMESTAMP AS '@MODULE_PATHNAME@', 'ts_gapfill_timestamp_bucket' LANGUAGE C VOLATILE PARALLEL SAFE; CREATE OR REPLACE FUNCTION @extschema@.time_bucket_gapfill(bucket_width INTERVAL, ts TIMESTAMPTZ, start TIMESTAMPTZ=NULL, finish TIMESTAMPTZ=NULL) RETURNS TIMESTAMPTZ AS '@MODULE_PATHNAME@', 'ts_gapfill_timestamptz_bucket' LANGUAGE C VOLATILE PARALLEL SAFE; CREATE OR REPLACE FUNCTION @extschema@.time_bucket_gapfill(bucket_width INTERVAL, ts TIMESTAMPTZ, timezone TEXT, start TIMESTAMPTZ=NULL, finish TIMESTAMPTZ=NULL) RETURNS TIMESTAMPTZ AS '@MODULE_PATHNAME@', 'ts_gapfill_timestamptz_timezone_bucket' LANGUAGE C VOLATILE PARALLEL SAFE; -- locf function CREATE OR REPLACE FUNCTION @extschema@.locf(value ANYELEMENT, prev ANYELEMENT=NULL, treat_null_as_missing BOOL=false) RETURNS ANYELEMENT AS '@MODULE_PATHNAME@', 'ts_gapfill_marker' LANGUAGE C VOLATILE PARALLEL SAFE; -- interpolate functions CREATE OR REPLACE FUNCTION @extschema@.interpolate(value SMALLINT,prev RECORD=NULL,next RECORD=NULL) RETURNS SMALLINT AS '@MODULE_PATHNAME@', 'ts_gapfill_marker' LANGUAGE C VOLATILE PARALLEL SAFE; CREATE OR REPLACE FUNCTION @extschema@.interpolate(value INT,prev RECORD=NULL,next RECORD=NULL) RETURNS INT AS '@MODULE_PATHNAME@', 'ts_gapfill_marker' LANGUAGE C VOLATILE PARALLEL SAFE; CREATE OR REPLACE FUNCTION @extschema@.interpolate(value BIGINT,prev RECORD=NULL,next RECORD=NULL) RETURNS BIGINT AS '@MODULE_PATHNAME@', 'ts_gapfill_marker' LANGUAGE C VOLATILE PARALLEL SAFE; CREATE OR REPLACE FUNCTION @extschema@.interpolate(value REAL,prev RECORD=NULL,next RECORD=NULL) RETURNS REAL AS '@MODULE_PATHNAME@', 'ts_gapfill_marker' LANGUAGE C VOLATILE PARALLEL SAFE; CREATE OR REPLACE FUNCTION @extschema@.interpolate(value FLOAT,prev RECORD=NULL,next RECORD=NULL) RETURNS FLOAT AS '@MODULE_PATHNAME@', 'ts_gapfill_marker' LANGUAGE C VOLATILE PARALLEL SAFE; ================================================ FILE: sql/header.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- This file is always prepended to all installation and upgrade/downgrade scripts. SET LOCAL search_path TO pg_catalog, pg_temp; ================================================ FILE: sql/histogram.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE OR REPLACE FUNCTION _timescaledb_functions.hist_sfunc (state INTERNAL, val DOUBLE PRECISION, MIN DOUBLE PRECISION, MAX DOUBLE PRECISION, nbuckets INTEGER) RETURNS INTERNAL AS '@MODULE_PATHNAME@', 'ts_hist_sfunc' LANGUAGE C IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.hist_combinefunc(state1 INTERNAL, state2 INTERNAL) RETURNS INTERNAL AS '@MODULE_PATHNAME@', 'ts_hist_combinefunc' LANGUAGE C IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.hist_serializefunc(INTERNAL) RETURNS bytea AS '@MODULE_PATHNAME@', 'ts_hist_serializefunc' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.hist_deserializefunc(bytea, INTERNAL) RETURNS INTERNAL AS '@MODULE_PATHNAME@', 'ts_hist_deserializefunc' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.hist_finalfunc(state INTERNAL, val DOUBLE PRECISION, MIN DOUBLE PRECISION, MAX DOUBLE PRECISION, nbuckets INTEGER) RETURNS INTEGER[] AS '@MODULE_PATHNAME@', 'ts_hist_finalfunc' LANGUAGE C IMMUTABLE PARALLEL SAFE; -- We started using CREATE OR REPLACE AGGREGATE for aggregate creation once the syntax was fully supported -- as it is easier to support idempotent changes this way. This will allow for changes to functions supporting -- the aggregate, and, for instance, the definition and inclusion of inverse functions for window function -- support. However, it should still be noted that changes to the data structures used for the internal -- state of the aggregate must be backwards compatible and the old format must be accepted by any new functions -- in order for them to continue working with Continuous Aggregates, where old states may have been materialized. -- This aggregate partitions the dataset into a specified number of buckets (nbuckets) ranging -- from the inputted min to max values. CREATE OR REPLACE AGGREGATE @extschema@.histogram (DOUBLE PRECISION, DOUBLE PRECISION, DOUBLE PRECISION, INTEGER) ( SFUNC = _timescaledb_functions.hist_sfunc, STYPE = INTERNAL, COMBINEFUNC = _timescaledb_functions.hist_combinefunc, SERIALFUNC = _timescaledb_functions.hist_serializefunc, DESERIALFUNC = _timescaledb_functions.hist_deserializefunc, PARALLEL = SAFE, FINALFUNC = _timescaledb_functions.hist_finalfunc, FINALFUNC_EXTRA ); ================================================ FILE: sql/hypertable.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE OR REPLACE FUNCTION @extschema@.set_integer_now_func(hypertable REGCLASS, integer_now_func REGPROC, replace_if_exists BOOL = false) RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_hypertable_set_integer_now_func' LANGUAGE C VOLATILE STRICT; ================================================ FILE: sql/job_api.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE OR REPLACE FUNCTION @extschema@.add_job( proc REGPROC, schedule_interval INTERVAL, config JSONB DEFAULT NULL, initial_start TIMESTAMPTZ DEFAULT NULL, scheduled BOOL DEFAULT true, check_config REGPROC DEFAULT NULL, fixed_schedule BOOL DEFAULT TRUE, timezone TEXT DEFAULT NULL, job_name TEXT DEFAULT NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_job_add' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION @extschema@.delete_job(job_id INTEGER) RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_job_delete' LANGUAGE C VOLATILE STRICT; CREATE OR REPLACE PROCEDURE @extschema@.run_job(job_id INTEGER) AS '@MODULE_PATHNAME@', 'ts_job_run' LANGUAGE C; -- Returns the updated job schedule values CREATE OR REPLACE FUNCTION @extschema@.alter_job( job_id INTEGER, schedule_interval INTERVAL = NULL, max_runtime INTERVAL = NULL, max_retries INTEGER = NULL, retry_period INTERVAL = NULL, scheduled BOOL = NULL, config JSONB = NULL, next_start TIMESTAMPTZ = NULL, if_exists BOOL = FALSE, check_config REGPROC = NULL, fixed_schedule BOOL = NULL, initial_start TIMESTAMPTZ = NULL, timezone TEXT DEFAULT NULL, job_name TEXT DEFAULT NULL ) RETURNS TABLE (job_id INTEGER, schedule_interval INTERVAL, max_runtime INTERVAL, max_retries INTEGER, retry_period INTERVAL, scheduled BOOL, config JSONB, next_start TIMESTAMPTZ, check_config TEXT, fixed_schedule BOOL, initial_start TIMESTAMPTZ, timezone TEXT, application_name name) AS '@MODULE_PATHNAME@', 'ts_job_alter' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION _timescaledb_functions.alter_job_set_hypertable_id( job_id INTEGER, hypertable REGCLASS ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_job_alter_set_hypertable_id' LANGUAGE C VOLATILE; ================================================ FILE: sql/job_stat_history_log_retention.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- A retention policy is set up for the table _timescaledb_internal.job_errors (Error Log Retention Policy [2]) -- By default, it will run once a month and and drop rows older than a month. -- We use binary search on the id column to figure out which rows should be retained from the job history table -- Doing it this way allows us to use the index on the `id` column, which (empirically) we found is faster than querying on the `execution_finish` column directly without an index -- This works because `execution_finish` is always ordered. -- We can consider alternative approaches to simplify this in the future. CREATE OR REPLACE FUNCTION _timescaledb_functions.job_history_bsearch(search_point TIMESTAMPTZ) RETURNS BIGINT AS $$ DECLARE id_lower BIGINT; id_upper BIGINT; id_middle BIGINT DEFAULT 0; target_tz TIMESTAMPTZ; BEGIN SELECT COALESCE(min(id), 0), COALESCE(max(id), 0) INTO id_lower, id_upper FROM _timescaledb_internal.bgw_job_stat_history; IF id_lower = 0 AND id_upper = 0 THEN RETURN NULL; END IF; -- We want the first entry in the table where execution_finish is >= search_point WHILE id_lower < id_upper LOOP id_middle := id_lower + (id_upper - id_lower) / 2; SELECT execution_finish INTO target_tz FROM _timescaledb_internal.bgw_job_stat_history WHERE id = id_middle; -- If the id_middle is not found, shift to a previous id that's still in the search space IF NOT FOUND THEN SELECT execution_finish, id INTO target_tz, id_middle FROM _timescaledb_internal.bgw_job_stat_history WHERE id <= id_middle AND id >= id_lower ORDER BY id LIMIT 1; IF NOT FOUND THEN id_middle := id_lower; END IF; END IF; IF target_tz >= search_point THEN id_upper := id_middle; ELSE id_lower := id_middle + 1; END IF; END LOOP; -- Handle the case where no ids need to be deleted and return NULL instead SELECT execution_finish INTO target_tz FROM _timescaledb_internal.bgw_job_stat_history WHERE id = id_lower; IF target_tz < search_point THEN RETURN NULL; END IF; RETURN id_lower; END $$ LANGUAGE plpgsql SET search_path TO pg_catalog, pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_functions.policy_job_stat_history_retention(job_id integer, config JSONB) RETURNS VOID LANGUAGE PLPGSQL AS $BODY$ DECLARE search_point TIMESTAMPTZ; id_found BIGINT; BEGIN PERFORM set_config('lock_timeout', coalesce(config->>'lock_timeout', '5s'), true /* is local */); -- We need to prevent concurrent changes on this table when running this retention job -- We take an AccessExclusiveLock at the start since we TRUNCATE later LOCK TABLE _timescaledb_internal.bgw_job_stat_history IN ACCESS EXCLUSIVE MODE; search_point := now() - (config->>'drop_after')::interval; id_found := _timescaledb_functions.job_history_bsearch(search_point); IF id_found IS NULL THEN RETURN; END IF; -- Build a table that contains only rows younger than the max age -- and satisfy the constraints on number of successfull and failures -- for each job. Since the table is ordered, we can use the "id" -- column to find out what records to remove. CREATE TEMP TABLE __tmp_bgw_job_stat_history ON COMMIT DROP AS WITH enumerated AS ( SELECT *, row_number() OVER ( PARTITION BY j.job_id, j.succeeded ORDER BY j.execution_finish DESC ) AS row_number FROM _timescaledb_internal.bgw_job_stat_history j WHERE id >= id_found) SELECT id, e.job_id, pid, execution_start, execution_finish, succeeded, data FROM enumerated e WHERE succeeded AND row_number <= (config->>'max_successes_per_job')::int OR NOT succeeded AND row_number <= (config->>'max_failures_per_job')::int ORDER BY id; TRUNCATE _timescaledb_internal.bgw_job_stat_history; INSERT INTO _timescaledb_internal.bgw_job_stat_history SELECT * FROM __tmp_bgw_job_stat_history; END $BODY$ SET search_path TO pg_catalog, pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_functions.policy_job_stat_history_retention_check(config JSONB) RETURNS VOID LANGUAGE PLPGSQL AS $BODY$ BEGIN IF config IS NULL THEN RAISE EXCEPTION 'config cannot be NULL, and must contain drop_after'; END IF; IF NOT (config ? 'drop_after') THEN RAISE EXCEPTION 'drop_after interval not provided'; END IF; IF NOT (config ? 'max_successes_per_job') THEN RAISE EXCEPTION 'max_successes_per_job not provided'; END IF; IF NOT (config ? 'max_failures_per_job') THEN RAISE EXCEPTION 'max_failures_per_job not provided'; END IF; IF (config->>'max_successes_per_job')::integer < 10 THEN RAISE EXCEPTION 'max_successes_per_job has to be at least 10'; END IF; IF (config->>'max_failures_per_job')::integer < 10 THEN RAISE EXCEPTION 'max_failures_per_job has to be at least 10'; END IF; END $BODY$ SET search_path TO pg_catalog, pg_temp; INSERT INTO _timescaledb_catalog.bgw_job ( id, application_name, schedule_interval, max_runtime, max_retries, retry_period, proc_schema, proc_name, owner, scheduled, config, check_schema, check_name, fixed_schedule, initial_start ) VALUES ( 3, 'Job History Log Retention Policy [3]', INTERVAL '6 hours', INTERVAL '1 hour', -1, INTERVAL '1h', '_timescaledb_functions', 'policy_job_stat_history_retention', pg_catalog.quote_ident(current_role)::regrole, true, '{"drop_after":"1 month","max_successes_per_job":1000,"max_failures_per_job":1000}', '_timescaledb_functions', 'policy_job_stat_history_retention_check', true, '2000-01-01 00:00:00+00'::timestamptz ) ON CONFLICT (id) DO NOTHING; ================================================ FILE: sql/maintenance_utils.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- chunk - the OID of the chunk to be CLUSTERed -- index - the OID of the index to be CLUSTERed on, or NULL to use the index -- last used CREATE OR REPLACE FUNCTION @extschema@.reorder_chunk( chunk REGCLASS, index REGCLASS=NULL, verbose BOOLEAN=FALSE ) RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_reorder_chunk' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION @extschema@.move_chunk( chunk REGCLASS, destination_tablespace Name, index_destination_tablespace Name=NULL, reorder_index REGCLASS=NULL, verbose BOOLEAN=FALSE ) RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_move_chunk' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION _timescaledb_functions.create_compressed_chunk( chunk REGCLASS, chunk_table REGCLASS, uncompressed_heap_size BIGINT, uncompressed_toast_size BIGINT, uncompressed_index_size BIGINT, compressed_heap_size BIGINT, compressed_toast_size BIGINT, compressed_index_size BIGINT, numrows_pre_compression BIGINT, numrows_post_compression BIGINT ) RETURNS REGCLASS AS '@MODULE_PATHNAME@', 'ts_create_compressed_chunk' LANGUAGE C STRICT VOLATILE; CREATE OR REPLACE FUNCTION @extschema@.compress_chunk( uncompressed_chunk REGCLASS, if_not_compressed BOOLEAN = true, recompress BOOLEAN = false ) RETURNS REGCLASS AS '@MODULE_PATHNAME@', 'ts_compress_chunk' LANGUAGE C VOLATILE; -- Alias for compress_chunk above. CREATE OR REPLACE PROCEDURE @extschema@.convert_to_columnstore( chunk REGCLASS, if_not_columnstore BOOLEAN = true, recompress BOOLEAN = false ) AS '@MODULE_PATHNAME@', 'ts_compress_chunk' LANGUAGE C; CREATE OR REPLACE FUNCTION @extschema@.decompress_chunk( uncompressed_chunk REGCLASS, if_compressed BOOLEAN = true ) RETURNS REGCLASS AS '@MODULE_PATHNAME@', 'ts_decompress_chunk' LANGUAGE C STRICT VOLATILE; CREATE OR REPLACE PROCEDURE @extschema@.convert_to_rowstore( chunk REGCLASS, if_columnstore BOOLEAN = true ) AS '@MODULE_PATHNAME@', 'ts_decompress_chunk' LANGUAGE C; CREATE OR REPLACE PROCEDURE _timescaledb_functions.rebuild_columnstore( chunk REGCLASS ) AS '@MODULE_PATHNAME@', 'ts_rebuild_columnstore' LANGUAGE C; CREATE OR REPLACE PROCEDURE _timescaledb_functions.chunk_rewrite_cleanup() LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_chunk_rewrite_cleanup'; CREATE OR REPLACE PROCEDURE @extschema@.merge_chunks( chunk1 REGCLASS, chunk2 REGCLASS, concurrently BOOLEAN = false ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_merge_two_chunks'; CREATE OR REPLACE PROCEDURE @extschema@.merge_chunks( chunks REGCLASS[] ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_merge_chunks'; CREATE OR REPLACE PROCEDURE @extschema@.merge_chunks_concurrently( chunks REGCLASS[] ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_merge_chunks_concurrently'; CREATE OR REPLACE PROCEDURE @extschema@.split_chunk( chunk REGCLASS, split_at "any" = NULL ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_split_chunk'; CREATE OR REPLACE FUNCTION _timescaledb_functions.recompress_chunk_segmentwise( uncompressed_chunk REGCLASS, if_compressed BOOLEAN = true ) RETURNS REGCLASS AS '@MODULE_PATHNAME@', 'ts_recompress_chunk_segmentwise' LANGUAGE C STRICT VOLATILE; -- find the index on the compressed chunk that can be used to recompress efficiently -- this index must contain all the segmentby columns and the meta_sequence_number column last CREATE OR REPLACE FUNCTION _timescaledb_functions.get_compressed_chunk_index_for_recompression( uncompressed_chunk REGCLASS ) RETURNS REGCLASS AS '@MODULE_PATHNAME@', 'ts_get_compressed_chunk_index_for_recompression' LANGUAGE C STRICT VOLATILE; -- Recompress a chunk -- -- Will give an error if the chunk was not already compressed. In this -- case, the user should use compress_chunk instead. Note that this -- function cannot be executed in an explicit transaction since it -- contains transaction control commands. -- -- Parameters: -- chunk: Chunk to recompress. -- if_not_compressed: Print notice instead of error if chunk is already compressed. CREATE OR REPLACE PROCEDURE @extschema@.recompress_chunk(chunk REGCLASS, if_not_compressed BOOLEAN = true) LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('timescaledb.enable_deprecation_warnings', true)::bool THEN RAISE WARNING 'procedure @extschema@.recompress_chunk(regclass,boolean) is deprecated and the functionality is now included in @extschema@.compress_chunk. this compatibility function will be removed in a future version.'; END IF; PERFORM @extschema@.compress_chunk(chunk, if_not_compressed); END$$ SET search_path TO pg_catalog,pg_temp; -- A version of makeaclitem that accepts a comma-separated list of -- privileges rather than just a single privilege. This is copied from -- PG16, but since we need to support earlier versions, we provide it -- with the extension. -- -- This is intended for internal usage and interface might change. CREATE OR REPLACE FUNCTION _timescaledb_functions.makeaclitem(regrole, regrole, text, bool) RETURNS AclItem AS '@MODULE_PATHNAME@', 'ts_makeaclitem' LANGUAGE C STABLE PARALLEL SAFE STRICT; -- Repair relation ACL by removing roles that do not exist in pg_authid. CREATE OR REPLACE PROCEDURE _timescaledb_functions.repair_relation_acls() LANGUAGE SQL AS $$ WITH badrels AS ( SELECT oid::regclass FROM (SELECT oid, (aclexplode(relacl)).* FROM pg_class) AS rels WHERE rels.grantee != 0 AND rels.grantee NOT IN (SELECT oid FROM pg_authid) ), pickacls AS ( SELECT oid::regclass, _timescaledb_functions.makeaclitem( b.grantee, b.grantor, string_agg(b.privilege_type, ','), b.is_grantable ) AS acl FROM (SELECT oid, (aclexplode(relacl)).* AS a FROM pg_class) AS b WHERE b.grantee IN (SELECT oid FROM pg_authid) GROUP BY oid, b.grantee, b.grantor, b.is_grantable ), cleanacls AS ( SELECT oid, array_agg(acl) AS acl FROM pickacls GROUP BY oid ) UPDATE pg_class c SET relacl = (SELECT acl FROM cleanacls n WHERE c.oid = n.oid) WHERE oid IN (SELECT oid FROM badrels) $$ SET search_path TO pg_catalog, pg_temp; -- Remove chunk metadata when marked as dropped CREATE OR REPLACE FUNCTION _timescaledb_functions.remove_dropped_chunk_metadata(_hypertable_id INTEGER) RETURNS INTEGER LANGUAGE plpgsql AS $$ DECLARE _chunk_id INTEGER; _removed INTEGER := 0; BEGIN FOR _chunk_id IN SELECT id FROM _timescaledb_catalog.chunk WHERE hypertable_id = _hypertable_id AND NOT EXISTS ( SELECT FROM information_schema.tables WHERE tables.table_schema = chunk.schema_name AND tables.table_name = chunk.table_name ) LOOP _removed := _removed + 1; RAISE INFO 'Removing metadata of chunk % from hypertable %', _chunk_id, _hypertable_id; WITH _dimension_slice_remove AS ( DELETE FROM _timescaledb_catalog.dimension_slice USING _timescaledb_catalog.chunk_constraint WHERE dimension_slice.id = chunk_constraint.dimension_slice_id AND chunk_constraint.chunk_id = _chunk_id AND NOT EXISTS ( SELECT FROM _timescaledb_catalog.chunk_constraint cc WHERE cc.chunk_id <> _chunk_id AND cc.dimension_slice_id = dimension_slice.id ) RETURNING _timescaledb_catalog.dimension_slice.id ) DELETE FROM _timescaledb_catalog.chunk_constraint USING _dimension_slice_remove WHERE chunk_constraint.dimension_slice_id = _dimension_slice_remove.id; DELETE FROM _timescaledb_catalog.chunk_constraint WHERE chunk_constraint.chunk_id = _chunk_id; DELETE FROM _timescaledb_internal.bgw_policy_chunk_stats WHERE bgw_policy_chunk_stats.chunk_id = _chunk_id; DELETE FROM _timescaledb_catalog.compression_chunk_size WHERE compression_chunk_size.chunk_id = _chunk_id OR compression_chunk_size.compressed_chunk_id = _chunk_id; DELETE FROM _timescaledb_catalog.chunk WHERE chunk.id = _chunk_id OR chunk.compressed_chunk_id = _chunk_id; END LOOP; RETURN _removed; END; $$ SET search_path TO pg_catalog, pg_temp; ================================================ FILE: sql/metadata.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE OR REPLACE FUNCTION _timescaledb_functions.generate_uuid() RETURNS UUID AS '@MODULE_PATHNAME@', 'ts_uuid_generate' LANGUAGE C VOLATILE STRICT; -- Trigger to change INSERT into UPDATE if key already exists. -- -- During extension installation we create 3 entries in the metadata table which are -- included in dumps. To allow loading logical dumps we need this trigger to turn INSERTs -- into UPDATEs if the key already exists. CREATE OR REPLACE FUNCTION _timescaledb_functions.metadata_insert_trigger() RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ BEGIN IF EXISTS (SELECT FROM _timescaledb_catalog.metadata WHERE key = NEW.key) THEN UPDATE _timescaledb_catalog.metadata SET value = NEW.value WHERE key = NEW.key; RETURN NULL; END IF; RETURN NEW; END $$ SET search_path TO pg_catalog, pg_temp; CREATE OR REPLACE TRIGGER metadata_insert_trigger BEFORE INSERT ON _timescaledb_catalog.metadata FOR EACH ROW EXECUTE PROCEDURE _timescaledb_functions.metadata_insert_trigger(); -- Insert uuid and install_timestamp on database creation since the trigger -- will turn these into UPDATEs on conflicts we can't use ON CONFLICT DO NOTHING. DO $$ BEGIN IF (NOT EXISTS (SELECT FROM _timescaledb_catalog.metadata WHERE key = 'uuid')) THEN INSERT INTO _timescaledb_catalog.metadata SELECT 'uuid', _timescaledb_functions.generate_uuid(), TRUE; END IF; IF (NOT EXISTS (SELECT FROM _timescaledb_catalog.metadata WHERE key = 'install_timestamp')) THEN INSERT INTO _timescaledb_catalog.metadata SELECT 'install_timestamp', now(), TRUE; END IF; END $$; -- Install catalog version on database installation and upgrade. -- This allows us to detect catalog mismatches in dump/restore cycle. INSERT INTO _timescaledb_catalog.metadata (key, value, include_in_telemetry) SELECT 'timescaledb_version', '@PROJECT_VERSION_MOD@', FALSE; ================================================ FILE: sql/notice.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. DO language plpgsql $$ DECLARE telemetry_string TEXT; telemetry_level text; BEGIN telemetry_level := current_setting('timescaledb.telemetry_level', true); CASE telemetry_level WHEN 'off' THEN telemetry_string = E'Note: Please enable telemetry to help us improve our product by running: ALTER DATABASE "' || current_database() || E'" SET timescaledb.telemetry_level = ''basic'';'; WHEN 'basic' THEN telemetry_string = E'Note: TimescaleDB collects anonymous reports to better understand and assist our users.\nFor more information and how to disable, please see our docs https://docs.timescale.com/timescaledb/latest/how-to-guides/configuration/telemetry.'; ELSE telemetry_string = E''; END CASE; RAISE NOTICE E'%\n%\n', E'\nWELCOME TO\n' || E' _____ _ _ ____________ \n' || E'|_ _(_) | | | _ \\ ___ \\ \n' || E' | | _ _ __ ___ ___ ___ ___ __ _| | ___| | | | |_/ / \n' || ' | | | | _ ` _ \ / _ \/ __|/ __/ _` | |/ _ \ | | | ___ \ ' || E'\n' || ' | | | | | | | | | __/\__ \ (_| (_| | | __/ |/ /| |_/ /' || E'\n' || ' |_| |_|_| |_| |_|\___||___/\___\__,_|_|\___|___/ \____/' || E'\n' || E' Running version ' || '@PROJECT_VERSION_MOD@' || E'\n' || E'For more information on TimescaleDB, please visit the following links:\n\n' || E' 1. Getting started: https://docs.timescale.com/timescaledb/latest/getting-started\n' || E' 2. API reference documentation: https://docs.timescale.com/api/latest\n', telemetry_string; END; $$; ================================================ FILE: sql/osm_api.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- This function updates the dimension slice range stored in the catalog with the min and max -- values that the OSM chunk contains. Since there is only one OSM chunk per hypertable with -- only a time dimension, the hypertable is used to determine the corresponding slice CREATE OR REPLACE FUNCTION _timescaledb_functions.hypertable_osm_range_update( hypertable REGCLASS, range_start ANYELEMENT = NULL::bigint, range_end ANYELEMENT = NULL, empty BOOL = false ) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_hypertable_osm_range_update' LANGUAGE C VOLATILE; ================================================ FILE: sql/partitioning.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Deprecated partition hash function CREATE OR REPLACE FUNCTION _timescaledb_functions.get_partition_for_key(val anyelement) RETURNS int AS '@MODULE_PATHNAME@', 'ts_get_partition_for_key' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.get_partition_hash(val anyelement) RETURNS int AS '@MODULE_PATHNAME@', 'ts_get_partition_hash' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; ================================================ FILE: sql/policy_api.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Add a retention policy to a hypertable or continuous aggregate. -- The retention_window (typically an INTERVAL) determines the -- window beyond which data is dropped at the time -- of execution of the policy (e.g., '1 week'). Note that the retention -- window will always align with chunk boundaries, thus the window -- might be larger than the given one, but never smaller. In other -- words, some data beyond the retention window -- might be kept, but data within the window will never be deleted. CREATE OR REPLACE FUNCTION @extschema@.add_retention_policy( relation REGCLASS, drop_after "any" = NULL, if_not_exists BOOL = false, schedule_interval INTERVAL = NULL, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL, drop_created_before INTERVAL = NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_policy_retention_add' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION @extschema@.remove_retention_policy( relation REGCLASS, if_exists BOOL = false ) RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_policy_retention_remove' LANGUAGE C VOLATILE STRICT; /* reorder policy */ CREATE OR REPLACE FUNCTION @extschema@.add_reorder_policy( hypertable REGCLASS, index_name NAME, if_not_exists BOOL = false, initial_start timestamptz = NULL, timezone TEXT = NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_policy_reorder_add' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION @extschema@.remove_reorder_policy(hypertable REGCLASS, if_exists BOOL = false) RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_policy_reorder_remove' LANGUAGE C VOLATILE STRICT; /* compression policy */ CREATE OR REPLACE FUNCTION @extschema@.add_compression_policy( hypertable REGCLASS, compress_after "any" = NULL, if_not_exists BOOL = false, schedule_interval INTERVAL = NULL, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL, compress_created_before INTERVAL = NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_policy_compression_add' LANGUAGE C VOLATILE; -- not strict because we need to set different default values for schedule_interval CREATE OR REPLACE PROCEDURE @extschema@.add_columnstore_policy( hypertable REGCLASS, after "any" = NULL, if_not_exists BOOL = false, schedule_interval INTERVAL = NULL, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL, created_before INTERVAL = NULL ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_policy_compression_add'; CREATE OR REPLACE FUNCTION @extschema@.remove_compression_policy(hypertable REGCLASS, if_exists BOOL = false) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_policy_compression_remove' LANGUAGE C VOLATILE STRICT; CREATE OR REPLACE PROCEDURE @extschema@.remove_columnstore_policy( hypertable REGCLASS, if_exists BOOL = false ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_policy_compression_remove'; /* continuous aggregates policy */ CREATE OR REPLACE FUNCTION @extschema@.add_continuous_aggregate_policy( continuous_aggregate REGCLASS, start_offset "any", end_offset "any", schedule_interval INTERVAL, if_not_exists BOOL = false, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL, include_tiered_data BOOL = NULL, buckets_per_batch INTEGER = NULL, max_batches_per_execution INTEGER = NULL, refresh_newest_first BOOL = NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_policy_refresh_cagg_add' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION @extschema@.remove_continuous_aggregate_policy( continuous_aggregate REGCLASS, if_not_exists BOOL = false, -- deprecating this argument, if_exists overrides it if_exists BOOL = NULL) -- when NULL get the value from if_not_exists RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_policy_refresh_cagg_remove' LANGUAGE C VOLATILE; CREATE OR REPLACE PROCEDURE @extschema@.add_process_hypertable_invalidations_policy( hypertable REGCLASS, schedule_interval INTERVAL, if_not_exists BOOL = false, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_policy_process_hyper_inval_add'; CREATE OR REPLACE PROCEDURE @extschema@.remove_process_hypertable_invalidations_policy( hypertable REGCLASS, if_exists BOOL = false ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_policy_process_hyper_inval_remove'; /* 1 step policies */ /* Add policies */ /* Unsupported drop_created_before/compress_created_before in add/alter for caggs */ CREATE OR REPLACE FUNCTION timescaledb_experimental.add_policies( relation REGCLASS, if_not_exists BOOL = false, refresh_start_offset "any" = NULL, refresh_end_offset "any" = NULL, compress_after "any" = NULL, drop_after "any" = NULL ) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_policies_add' LANGUAGE C VOLATILE; /* Remove policies */ CREATE OR REPLACE FUNCTION timescaledb_experimental.remove_policies( relation REGCLASS, if_exists BOOL = false, VARIADIC policy_names TEXT[] = NULL) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_policies_remove' LANGUAGE C VOLATILE; /* Remove all policies */ CREATE OR REPLACE FUNCTION timescaledb_experimental.remove_all_policies( relation REGCLASS, if_exists BOOL = false) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_policies_remove_all' LANGUAGE C VOLATILE; /* Alter policies */ CREATE OR REPLACE FUNCTION timescaledb_experimental.alter_policies( relation REGCLASS, if_exists BOOL = false, refresh_start_offset "any" = NULL, refresh_end_offset "any" = NULL, compress_after "any" = NULL, drop_after "any" = NULL) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_policies_alter' LANGUAGE C VOLATILE; /* Show policies info */ CREATE OR REPLACE FUNCTION timescaledb_experimental.show_policies( relation REGCLASS) RETURNS SETOF JSONB AS '@MODULE_PATHNAME@', 'ts_policies_show' LANGUAGE C VOLATILE; ================================================ FILE: sql/policy_internal.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE OR REPLACE PROCEDURE _timescaledb_functions.policy_retention(job_id INTEGER, config JSONB) AS '@MODULE_PATHNAME@', 'ts_policy_retention_proc' LANGUAGE C; CREATE OR REPLACE FUNCTION _timescaledb_functions.policy_retention_check(config JSONB) RETURNS void AS '@MODULE_PATHNAME@', 'ts_policy_retention_check' LANGUAGE C; CREATE OR REPLACE PROCEDURE _timescaledb_functions.policy_reorder(job_id INTEGER, config JSONB) AS '@MODULE_PATHNAME@', 'ts_policy_reorder_proc' LANGUAGE C; CREATE OR REPLACE FUNCTION _timescaledb_functions.policy_reorder_check(config JSONB) RETURNS void AS '@MODULE_PATHNAME@', 'ts_policy_reorder_check' LANGUAGE C; CREATE OR REPLACE PROCEDURE _timescaledb_functions.policy_recompression(job_id INTEGER, config JSONB) AS '@MODULE_PATHNAME@', 'ts_policy_recompression_proc' LANGUAGE C; CREATE OR REPLACE FUNCTION _timescaledb_functions.policy_compression_check(config JSONB) RETURNS void AS '@MODULE_PATHNAME@', 'ts_policy_compression_check' LANGUAGE C; CREATE OR REPLACE PROCEDURE _timescaledb_functions.policy_refresh_continuous_aggregate(job_id INTEGER, config JSONB) AS '@MODULE_PATHNAME@', 'ts_policy_refresh_cagg_proc' LANGUAGE C; CREATE OR REPLACE FUNCTION _timescaledb_functions.policy_refresh_continuous_aggregate_check(config JSONB) RETURNS void AS '@MODULE_PATHNAME@', 'ts_policy_refresh_cagg_check' LANGUAGE C; CREATE OR REPLACE PROCEDURE _timescaledb_functions.policy_process_hypertable_invalidations(job_id INTEGER, config JSONB) AS '@MODULE_PATHNAME@', 'ts_policy_process_hyper_inval_proc' LANGUAGE C; CREATE OR REPLACE FUNCTION _timescaledb_functions.policy_process_hypertable_invalidations_check(config JSONB) RETURNS void AS '@MODULE_PATHNAME@', 'ts_policy_process_hyper_inval_check' LANGUAGE C; CREATE OR REPLACE PROCEDURE _timescaledb_functions.policy_compression_execute( job_id INTEGER, htid INTEGER, lag ANYELEMENT, maxchunks INTEGER, verbose_log BOOLEAN, recompress_enabled BOOLEAN, reindex_enabled BOOLEAN, use_creation_time BOOLEAN ) AS $$ DECLARE htoid REGCLASS; chunk_rec RECORD; idx_rec RECORD; numchunks_compressed INTEGER := 0; _message text; _detail text; _sqlstate text; -- fully compressed chunk status status_fully_compressed int := 1; -- chunk status bits: bit_compressed int := 1; bit_compressed_unordered int := 2; bit_frozen int := 4; bit_compressed_partial int := 8; creation_lag INTERVAL := NULL; chunks_failure INTEGER := 0; BEGIN -- procedures with SET clause cannot execute transaction -- control so we adjust search_path in procedure body SET LOCAL search_path TO pg_catalog, pg_temp; SELECT format('%I.%I', schema_name, table_name) INTO htoid FROM _timescaledb_catalog.hypertable WHERE id = htid; -- for the integer cases, we have to compute the lag w.r.t -- the integer_now function and then pass on to show_chunks IF pg_typeof(lag) IN ('BIGINT'::regtype, 'INTEGER'::regtype, 'SMALLINT'::regtype) THEN -- cannot have use_creation_time set with this IF use_creation_time IS TRUE THEN RAISE EXCEPTION 'job % cannot use creation time with integer_now function', job_id; END IF; lag := _timescaledb_functions.subtract_integer_from_now(htoid, lag::BIGINT); END IF; -- if use_creation_time has been specified then the lag needs to be used with the -- "compress_created_before" argument. Otherwise the usual "older_than" argument -- is good enough IF use_creation_time IS TRUE THEN creation_lag := lag; lag := NULL; END IF; FOR chunk_rec IN SELECT show.oid, ch.schema_name, ch.table_name, ch.status FROM @extschema@.show_chunks(htoid, older_than => lag, created_before => creation_lag) AS show(oid) INNER JOIN pg_class pgc ON pgc.oid = show.oid INNER JOIN pg_namespace pgns ON pgc.relnamespace = pgns.oid INNER JOIN _timescaledb_catalog.chunk ch ON ch.table_name = pgc.relname AND ch.schema_name = pgns.nspname AND ch.hypertable_id = htid WHERE NOT ch.osm_chunk -- Checking for chunks which are not fully compressed and not frozen AND ch.status != status_fully_compressed AND ch.status & bit_frozen = 0 LOOP BEGIN IF chunk_rec.status = bit_compressed OR recompress_enabled IS TRUE THEN PERFORM @extschema@.compress_chunk(chunk_rec.oid); numchunks_compressed := numchunks_compressed + 1; END IF; EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS _message = MESSAGE_TEXT, _detail = PG_EXCEPTION_DETAIL, _sqlstate = RETURNED_SQLSTATE; RAISE WARNING 'converting chunk "%" to columnstore failed when recompress columnstore policy is executed', chunk_rec.oid::regclass::text USING DETAIL = format('Message: (%s), Detail: (%s).', _message, _detail), ERRCODE = _sqlstate; chunks_failure := chunks_failure + 1; END; COMMIT; -- went through recompression successfully now reindex indexes IF (chunk_rec.status & bit_compressed_partial = bit_compressed_partial) AND (reindex_enabled IS TRUE) THEN FOR idx_rec IN SELECT idx.schemaname, idx.indexname FROM pg_indexes idx JOIN _timescaledb_catalog.chunk ch ON ch.schema_name = idx.schemaname AND ch.table_name = idx.tablename WHERE idx.schemaname = chunk_rec.schema_name AND idx.tablename = chunk_rec.table_name AND ch.status = status_fully_compressed LOOP BEGIN EXECUTE format('REINDEX INDEX %I.%I;', idx_rec.schemaname, idx_rec.indexname); EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS _message = MESSAGE_TEXT, _detail = PG_EXCEPTION_DETAIL, _sqlstate = RETURNED_SQLSTATE; RAISE WARNING 'reindexing index "%.%" for chunk "%" to columnstore failed when columnstore policy is executed', idx_rec.schemaname, idx_rec.indexname, chunk_rec.oid::regclass::text USING DETAIL = format('Message: (%s), Detail: (%s).', _message, _detail), ERRCODE = _sqlstate; chunks_failure := chunks_failure + 1; END; COMMIT; END LOOP; END IF; -- SET LOCAL is only active until end of transaction. -- While we could use SET at the start of the function we do not -- want to bleed out search_path to caller, so we do SET LOCAL -- again after COMMIT SET LOCAL search_path TO pg_catalog, pg_temp; IF verbose_log THEN RAISE LOG 'job % completed processing chunk %.%', job_id, chunk_rec.schema_name, chunk_rec.table_name; END IF; IF maxchunks > 0 AND numchunks_compressed >= maxchunks THEN EXIT; END IF; END LOOP; IF chunks_failure > 0 THEN RAISE EXCEPTION 'columnstore policy failure' USING DETAIL = format('Failed to convert %L chunks to columnstore. Successfully converted %L chunks.', chunks_failure, numchunks_compressed), ERRCODE = 'data_exception'; END IF; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE _timescaledb_functions.policy_compression(job_id INTEGER, config JSONB) AS $$ DECLARE dimtype REGTYPE; dimtypeinput REGPROC; compress_after TEXT; compress_created_before TEXT; lag_value TEXT; lag_bigint_value BIGINT; htid INTEGER; htoid REGCLASS; chunk_rec RECORD; verbose_log BOOL; maxchunks INTEGER := 0; numchunks INTEGER := 1; recompress_enabled BOOL; reindex_enabled BOOL; use_creation_time BOOL := FALSE; BEGIN -- procedures with SET clause cannot execute transaction -- control so we adjust search_path in procedure body SET LOCAL search_path TO pg_catalog, pg_temp; IF config IS NULL THEN RAISE EXCEPTION 'job % has null config', job_id; END IF; htid := jsonb_object_field_text(config, 'hypertable_id')::INTEGER; IF htid is NULL THEN RAISE EXCEPTION 'job % config must have hypertable_id', job_id; END IF; verbose_log := COALESCE(jsonb_object_field_text(config, 'verbose_log')::BOOLEAN, FALSE); maxchunks := COALESCE(jsonb_object_field_text(config, 'maxchunks_to_compress')::INTEGER, 0); recompress_enabled := COALESCE(jsonb_object_field_text(config, 'recompress')::BOOLEAN, TRUE); reindex_enabled := COALESCE(jsonb_object_field_text(config, 'reindex')::BOOLEAN, TRUE); -- find primary dimension type -- SELECT dim.column_type INTO dimtype FROM _timescaledb_catalog.hypertable ht JOIN _timescaledb_catalog.dimension dim ON ht.id = dim.hypertable_id WHERE ht.id = htid ORDER BY dim.id LIMIT 1; compress_after := jsonb_object_field_text(config, 'compress_after'); IF compress_after IS NULL THEN compress_created_before := jsonb_object_field_text(config, 'compress_created_before'); IF compress_created_before IS NULL THEN RAISE EXCEPTION 'job % config must have compress_after or compress_created_before', job_id; END IF; lag_value := compress_created_before; use_creation_time := true; dimtype := 'INTERVAL' ::regtype; ELSE lag_value := compress_after; END IF; -- execute the properly type casts for the lag value CASE dimtype WHEN 'TIMESTAMP'::regtype, 'TIMESTAMPTZ'::regtype, 'DATE'::regtype, 'INTERVAL' ::regtype, 'UUID'::regtype THEN CALL _timescaledb_functions.policy_compression_execute(job_id, htid, lag_value::INTERVAL, maxchunks, verbose_log, recompress_enabled, reindex_enabled, use_creation_time); WHEN 'BIGINT'::regtype THEN CALL _timescaledb_functions.policy_compression_execute(job_id, htid, lag_value::BIGINT, maxchunks, verbose_log, recompress_enabled, reindex_enabled, use_creation_time); WHEN 'INTEGER'::regtype THEN CALL _timescaledb_functions.policy_compression_execute(job_id, htid, lag_value::INTEGER, maxchunks, verbose_log, recompress_enabled, reindex_enabled, use_creation_time); WHEN 'SMALLINT'::regtype THEN CALL _timescaledb_functions.policy_compression_execute(job_id, htid, lag_value::SMALLINT, maxchunks, verbose_log, recompress_enabled, reindex_enabled, use_creation_time); END CASE; COMMIT; END; $$ LANGUAGE PLPGSQL; ================================================ FILE: sql/pre_install/cache.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- This file contains infrastructure for cache invalidation of TimescaleDB -- metadata caches kept in C. Please look at cache_invalidate.c for a -- description of how this works. CREATE TABLE _timescaledb_cache.cache_inval_hypertable(); -- For notifying the scheduler of changes to the bgw_job table. CREATE TABLE _timescaledb_cache.cache_inval_bgw_job(); -- This is pretty subtle. We create this dummy cache_inval_extension table -- solely for the purpose of getting a relcache invalidation event when it is -- deleted on DROP extension. It has no related triggers. When the table is -- invalidated, all backends will be notified and will know that they must -- invalidate all cached information, including catalog table and index OIDs, -- etc. CREATE TABLE _timescaledb_cache.cache_inval_extension(); GRANT SELECT ON ALL TABLES IN SCHEMA _timescaledb_cache TO PUBLIC; ================================================ FILE: sql/pre_install/insert_data.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --insert data for compression_algorithm -- insert into _timescaledb_catalog.compression_algorithm( id, version, name, description) values ( 0, 1, 'COMPRESSION_ALGORITHM_NONE', 'no compression'), ( 1, 1, 'COMPRESSION_ALGORITHM_ARRAY', 'array'), ( 2, 1, 'COMPRESSION_ALGORITHM_DICTIONARY', 'dictionary'), ( 3, 1, 'COMPRESSION_ALGORITHM_GORILLA', 'gorilla'), ( 4, 1, 'COMPRESSION_ALGORITHM_DELTADELTA', 'deltadelta'), ( 5, 1, 'COMPRESSION_ALGORITHM_BOOL', 'bool'), ( 6, 1, 'COMPRESSION_ALGORITHM_NULL', 'null'), ( 7, 1, 'COMPRESSION_ALGORITHM_UUID', 'uuid'); ================================================ FILE: sql/pre_install/schemas.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SET LOCAL search_path TO pg_catalog, pg_temp; CREATE SCHEMA _timescaledb_catalog; CREATE SCHEMA _timescaledb_config; CREATE SCHEMA _timescaledb_functions; CREATE SCHEMA _timescaledb_internal; CREATE SCHEMA _timescaledb_cache; CREATE SCHEMA timescaledb_experimental; CREATE SCHEMA timescaledb_information; GRANT USAGE ON SCHEMA _timescaledb_cache, _timescaledb_catalog, _timescaledb_config, _timescaledb_functions, _timescaledb_internal, timescaledb_experimental, timescaledb_information TO PUBLIC; ================================================ FILE: sql/pre_install/tables.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --NOTICE: UPGRADE-SCRIPT-NEEDED contents in this file are not auto-upgraded. -- This file contains table definitions for various abstractions and data -- structures for representing hypertables and lower-level concepts. -- Hypertable -- ========== -- -- The hypertable is an abstraction that represents a table that is -- partitioned in N dimensions, where each dimension maps to a column -- in the table. A dimension can either be 'open' or 'closed', which -- reflects the scheme that divides the dimension's keyspace into -- "slices". -- -- Conceptually, a partition -- called a "chunk", is a hypercube in -- the N-dimensional space. A chunk stores a subset of the -- hypertable's tuples on disk in its own distinct table. The slices -- that span the chunk's hypercube each correspond to a constraint on -- the chunk's table, enabling constraint exclusion during queries on -- the hypertable's data. -- -- -- Open dimensions ------------------ -- An open dimension does on-demand slicing, creating a new slice -- based on a configurable interval whenever a tuple falls outside the -- existing slices. Open dimensions fit well with columns that are -- incrementally increasing, such as time-based ones. -- -- Closed dimensions -------------------- -- A closed dimension completely divides its keyspace into a -- configurable number of slices. The number of slices can be -- reconfigured, but the new partitioning only affects newly created -- chunks. -- The unique constraint is table_name +schema_name. The ordering is -- important as we want index access when we filter by table_name CREATE SEQUENCE _timescaledb_catalog.hypertable_id_seq MINVALUE 1; CREATE TABLE _timescaledb_catalog.hypertable ( id INTEGER NOT NULL DEFAULT nextval('_timescaledb_catalog.hypertable_id_seq'), schema_name name NOT NULL, table_name name NOT NULL, associated_schema_name name NOT NULL, associated_table_prefix name NOT NULL, num_dimensions smallint NOT NULL, chunk_sizing_func_schema name NOT NULL, chunk_sizing_func_name name NOT NULL, chunk_target_size bigint NOT NULL, -- size in bytes compression_state smallint NOT NULL DEFAULT 0, compressed_hypertable_id integer, status int NOT NULL DEFAULT 0, -- table constraints CONSTRAINT hypertable_pkey PRIMARY KEY (id), CONSTRAINT hypertable_associated_schema_name_associated_table_prefix_key UNIQUE (associated_schema_name, associated_table_prefix), CONSTRAINT hypertable_table_name_schema_name_key UNIQUE (table_name, schema_name), CONSTRAINT hypertable_schema_name_check CHECK (schema_name != '_timescaledb_catalog'), -- internal compressed hypertables have compression state = 2 CONSTRAINT hypertable_dim_compress_check CHECK (num_dimensions > 0 OR compression_state = 2), CONSTRAINT hypertable_chunk_target_size_check CHECK (chunk_target_size >= 0), CONSTRAINT hypertable_compress_check CHECK ( (compression_state = 0 OR compression_state = 1 ) OR (compression_state = 2 AND compressed_hypertable_id IS NULL)), CONSTRAINT hypertable_compressed_hypertable_id_fkey FOREIGN KEY (compressed_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ); ALTER SEQUENCE _timescaledb_catalog.hypertable_id_seq OWNED BY _timescaledb_catalog.hypertable.id; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable_id_seq', ''); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable', 'WHERE id >= 1'); -- The tablespace table maps tablespaces to hypertables. -- This allows spreading a hypertable's chunks across multiple disks. CREATE TABLE _timescaledb_catalog.tablespace ( id serial NOT NULL, hypertable_id int NOT NULL, tablespace_name name NOT NULL, -- table constraints CONSTRAINT tablespace_pkey PRIMARY KEY (id), CONSTRAINT tablespace_hypertable_id_tablespace_name_key UNIQUE (hypertable_id, tablespace_name), CONSTRAINT tablespace_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE ); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.tablespace', ''); -- A dimension represents an axis along which data is partitioned. CREATE TABLE _timescaledb_catalog.dimension ( id serial NOT NULL , hypertable_id integer NOT NULL, column_name name NOT NULL, column_type REGTYPE NOT NULL, aligned boolean NOT NULL, -- closed dimensions num_slices smallint NULL, partitioning_func_schema name NULL, partitioning_func name NULL, -- open dimensions (e.g., time) interval_length bigint NULL, -- compress interval is used by rollup procedure during compression -- in order to merge multiple chunks into a single one compress_interval_length bigint NULL, integer_now_func_schema name NULL, integer_now_func name NULL, -- table constraints CONSTRAINT dimension_pkey PRIMARY KEY (id), CONSTRAINT dimension_hypertable_id_column_name_key UNIQUE (hypertable_id, column_name), CONSTRAINT dimension_check CHECK ((partitioning_func_schema IS NULL AND partitioning_func IS NULL) OR (partitioning_func_schema IS NOT NULL AND partitioning_func IS NOT NULL)), CONSTRAINT dimension_check1 CHECK ((num_slices IS NULL AND interval_length IS NOT NULL) OR (num_slices IS NOT NULL AND interval_length IS NULL)), CONSTRAINT dimension_check2 CHECK ((integer_now_func_schema IS NULL AND integer_now_func IS NULL) OR (integer_now_func_schema IS NOT NULL AND integer_now_func IS NOT NULL)), CONSTRAINT dimension_interval_length_check CHECK (interval_length IS NULL OR interval_length > 0), CONSTRAINT dimension_compress_interval_length_check CHECK (compress_interval_length IS NULL OR compress_interval_length > 0), CONSTRAINT dimension_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE ); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.dimension', ''); SELECT pg_catalog.pg_extension_config_dump(pg_get_serial_sequence('_timescaledb_catalog.dimension', 'id'), ''); -- A dimension slice defines a keyspace range along a dimension -- axis. A chunk references a slice in each of its dimensions, forming -- a hypercube. CREATE TABLE _timescaledb_catalog.dimension_slice ( id serial NOT NULL, dimension_id integer NOT NULL, range_start bigint NOT NULL, range_end bigint NOT NULL, -- table constraints CONSTRAINT dimension_slice_pkey PRIMARY KEY (id), CONSTRAINT dimension_slice_dimension_id_range_start_range_end_key UNIQUE (dimension_id, range_start, range_end), CONSTRAINT dimension_slice_check CHECK (range_start <= range_end), CONSTRAINT dimension_slice_dimension_id_fkey FOREIGN KEY (dimension_id) REFERENCES _timescaledb_catalog.dimension (id) ON DELETE CASCADE ); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.dimension_slice', ''); SELECT pg_catalog.pg_extension_config_dump(pg_get_serial_sequence('_timescaledb_catalog.dimension_slice', 'id'), ''); -- A chunk is a partition (hypercube) in an N-dimensional -- hyperspace. Each chunk is associated with N constraints that define -- the chunk's hypercube. Tuples that fall within the chunk's -- hypercube are stored in the chunk's data table, as given by -- 'schema_name' and 'table_name'. CREATE SEQUENCE _timescaledb_catalog.chunk_id_seq MINVALUE 1; CREATE TABLE _timescaledb_catalog.chunk ( id integer NOT NULL DEFAULT nextval('_timescaledb_catalog.chunk_id_seq'), hypertable_id int NOT NULL, schema_name name NOT NULL, table_name name NOT NULL, compressed_chunk_id integer , status integer NOT NULL DEFAULT 0, osm_chunk boolean NOT NULL DEFAULT FALSE, creation_time timestamptz NOT NULL, -- table constraints CONSTRAINT chunk_pkey PRIMARY KEY (id), CONSTRAINT chunk_schema_name_table_name_key UNIQUE (schema_name, table_name), CONSTRAINT chunk_compressed_chunk_id_fkey FOREIGN KEY (compressed_chunk_id) REFERENCES _timescaledb_catalog.chunk (id), CONSTRAINT chunk_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ); ALTER SEQUENCE _timescaledb_catalog.chunk_id_seq OWNED BY _timescaledb_catalog.chunk.id; CREATE INDEX chunk_hypertable_id_idx ON _timescaledb_catalog.chunk (hypertable_id); CREATE INDEX chunk_compressed_chunk_id_idx ON _timescaledb_catalog.chunk (compressed_chunk_id); --we could use a partial index (where osm_chunk is true). However, the catalog code --does not work with partial/functional indexes. So we instead have a full index here. --Another option would be to use the status field to identify a OSM chunk. However bit --operations only work on varbit datatype and not integer datatype. CREATE INDEX chunk_osm_chunk_idx ON _timescaledb_catalog.chunk (osm_chunk, hypertable_id); CREATE INDEX chunk_hypertable_id_creation_time_idx ON _timescaledb_catalog.chunk(hypertable_id, creation_time); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk', ''); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk_id_seq', ''); -- A chunk constraint maps a dimension slice to a chunk. Each -- constraint associated with a chunk will also be a table constraint -- on the chunk's data table. CREATE TABLE _timescaledb_catalog.chunk_constraint ( chunk_id integer NOT NULL, dimension_slice_id integer NULL, constraint_name name NOT NULL, hypertable_constraint_name name NULL, -- table constraints CONSTRAINT chunk_constraint_chunk_id_constraint_name_key UNIQUE (chunk_id, constraint_name), CONSTRAINT chunk_constraint_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk (id), CONSTRAINT chunk_constraint_dimension_slice_id_fkey FOREIGN KEY (dimension_slice_id) REFERENCES _timescaledb_catalog.dimension_slice (id) ); CREATE INDEX chunk_constraint_dimension_slice_id_idx ON _timescaledb_catalog.chunk_constraint (dimension_slice_id); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk_constraint', ''); CREATE SEQUENCE _timescaledb_catalog.chunk_constraint_name; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk_constraint_name', ''); -- Track statistics for columns of chunks from a hypertable. -- Currently, we track the min/max range for a given column across chunks. -- More statistics (like bloom filters) can be added in the future. -- -- A "special" entry for a column with invalid chunk_id, PG_INT64_MAX, -- PG_INT64_MIN indicates that min/max ranges could be computed for this column -- for chunks. -- -- The ranges can overlap across chunks. The values could be out-of-date if -- modifications/changes occur in the corresponding chunk and such entries -- should be marked as "invalid" to ensure that the chunk is in -- appropriate state to be able to use these values. Thus these entries -- are different from dimension_slice which is used for tracking partitioning -- column ranges which have different characteristics. -- -- Currently this catalog supports datatypes like INT, SERIAL, BIGSERIAL, -- DATE, TIMESTAMP etc. by storing the ranges in bigint columns. In the -- future, we could support additional datatypes (which support btree style -- >, <, = comparators) by storing their textual representation. -- CREATE TABLE _timescaledb_catalog.chunk_column_stats ( id serial NOT NULL, hypertable_id integer NOT NULL, chunk_id integer NULL, column_name name NOT NULL, range_start bigint NOT NULL, range_end bigint NOT NULL, valid boolean NOT NULL, -- table constraints CONSTRAINT chunk_column_stats_pkey PRIMARY KEY (id), CONSTRAINT chunk_column_stats_ht_id_chunk_id_colname_key UNIQUE (hypertable_id, chunk_id, column_name), CONSTRAINT chunk_column_stats_range_check CHECK (range_start <= range_end), CONSTRAINT chunk_column_stats_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id), CONSTRAINT chunk_column_stats_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk_column_stats', ''); SELECT pg_catalog.pg_extension_config_dump(pg_get_serial_sequence('_timescaledb_catalog.chunk_column_stats', 'id'), ''); -- Default jobs are given the id space [1,1000). User-installed jobs and any jobs created inside tests -- are given the id space [1000, INT_MAX). That way, we do not pg_dump jobs that are always default-installed -- inside other .sql scripts. This avoids insertion conflicts during pg_restore. CREATE SEQUENCE _timescaledb_catalog.bgw_job_id_seq MINVALUE 1000; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.bgw_job_id_seq', ''); -- We put columns that can be null or have variable length -- last. This allow us to read the important fields above in the -- scheduler without materializing these fields below, which the -- scheduler does not neeed. CREATE TABLE _timescaledb_catalog.bgw_job ( id integer NOT NULL DEFAULT nextval('_timescaledb_catalog.bgw_job_id_seq'), application_name name NOT NULL, schedule_interval interval NOT NULL, max_runtime interval NOT NULL, max_retries integer NOT NULL, retry_period interval NOT NULL, proc_schema name NOT NULL, proc_name name NOT NULL, owner regrole NOT NULL DEFAULT pg_catalog.quote_ident(current_role)::regrole, scheduled bool NOT NULL DEFAULT TRUE, fixed_schedule bool not null default true, initial_start timestamptz, hypertable_id integer, config jsonb, check_schema name, check_name name, timezone text, -- table constraints CONSTRAINT bgw_job_pkey PRIMARY KEY (id), CONSTRAINT bgw_job_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE ); ALTER SEQUENCE _timescaledb_catalog.bgw_job_id_seq OWNED BY _timescaledb_catalog.bgw_job.id; CREATE INDEX bgw_job_proc_hypertable_id_idx ON _timescaledb_catalog.bgw_job (proc_schema, proc_name, hypertable_id); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.bgw_job', 'WHERE id >= 1000'); CREATE TABLE _timescaledb_internal.bgw_job_stat ( job_id integer NOT NULL, last_start timestamptz NOT NULL DEFAULT NOW(), last_finish timestamptz NOT NULL, next_start timestamptz NOT NULL, last_successful_finish timestamptz NOT NULL, last_run_success bool NOT NULL, total_runs bigint NOT NULL, total_duration interval NOT NULL, total_duration_failures interval NOT NULL, total_successes bigint NOT NULL, total_failures bigint NOT NULL, total_crashes bigint NOT NULL, consecutive_failures int NOT NULL, consecutive_crashes int NOT NULL, flags int NOT NULL DEFAULT 0, -- table constraints CONSTRAINT bgw_job_stat_pkey PRIMARY KEY (job_id), CONSTRAINT bgw_job_stat_job_id_fkey FOREIGN KEY (job_id) REFERENCES _timescaledb_catalog.bgw_job (id) ON DELETE CASCADE ); CREATE SEQUENCE _timescaledb_internal.bgw_job_stat_history_id_seq MINVALUE 1; CREATE TABLE _timescaledb_internal.bgw_job_stat_history ( id BIGINT NOT NULL DEFAULT nextval('_timescaledb_internal.bgw_job_stat_history_id_seq'), job_id INTEGER NOT NULL, pid INTEGER, execution_start TIMESTAMPTZ NOT NULL DEFAULT NOW(), execution_finish TIMESTAMPTZ, succeeded boolean, data jsonb, -- table constraints CONSTRAINT bgw_job_stat_history_pkey PRIMARY KEY (id) ); ALTER SEQUENCE _timescaledb_internal.bgw_job_stat_history_id_seq OWNED BY _timescaledb_internal.bgw_job_stat_history.id; CREATE INDEX bgw_job_stat_history_execution_start_idx ON _timescaledb_internal.bgw_job_stat_history (execution_start); CREATE INDEX bgw_job_stat_history_job_id_execution_start_idx ON _timescaledb_internal.bgw_job_stat_history(job_id, execution_start DESC); --The job_stat table is not dumped by pg_dump on purpose because --the statistics probably aren't very meaningful across instances. -- Now we define a special stats table for each job/chunk pair. This will be used by the scheduler -- to determine whether to run a specific job on a specific chunk. CREATE TABLE _timescaledb_internal.bgw_policy_chunk_stats ( job_id integer NOT NULL, chunk_id integer NOT NULL, num_times_job_run integer, last_time_job_run timestamptz, -- table constraints CONSTRAINT bgw_policy_chunk_stats_job_id_chunk_id_key UNIQUE (job_id, chunk_id), CONSTRAINT bgw_policy_chunk_stats_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ON DELETE CASCADE, CONSTRAINT bgw_policy_chunk_stats_job_id_fkey FOREIGN KEY (job_id) REFERENCES _timescaledb_catalog.bgw_job (id) ON DELETE CASCADE ); CREATE TABLE _timescaledb_catalog.metadata ( key NAME NOT NULL, value text NOT NULL, include_in_telemetry boolean NOT NULL, -- table constraints CONSTRAINT metadata_pkey PRIMARY KEY (key) ); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.metadata', $$ WHERE key <> 'uuid' $$); -- Log with events that will be sent out with the telemetry. The log -- will be flushed after it has been sent out. We do not save it to -- backups since it should not contain important data. CREATE TABLE _timescaledb_catalog.telemetry_event ( created timestamptz NOT NULL DEFAULT current_timestamp, tag name NOT NULL, body jsonb NOT NULL ); CREATE TABLE _timescaledb_catalog.continuous_agg ( mat_hypertable_id integer NOT NULL, raw_hypertable_id integer NOT NULL, parent_mat_hypertable_id integer, user_view_schema name NOT NULL, user_view_name name NOT NULL, partial_view_schema name NOT NULL, partial_view_name name NOT NULL, direct_view_schema name NOT NULL, direct_view_name name NOT NULL, materialized_only bool NOT NULL DEFAULT FALSE, -- table constraints CONSTRAINT continuous_agg_pkey PRIMARY KEY (mat_hypertable_id), CONSTRAINT continuous_agg_partial_view_schema_partial_view_name_key UNIQUE (partial_view_schema, partial_view_name), CONSTRAINT continuous_agg_user_view_schema_user_view_name_key UNIQUE (user_view_schema, user_view_name), CONSTRAINT continuous_agg_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE, CONSTRAINT continuous_agg_raw_hypertable_id_fkey FOREIGN KEY (raw_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE, CONSTRAINT continuous_agg_parent_mat_hypertable_id_fkey FOREIGN KEY (parent_mat_hypertable_id) REFERENCES _timescaledb_catalog.continuous_agg (mat_hypertable_id) ON DELETE CASCADE ); CREATE INDEX continuous_agg_raw_hypertable_id_idx ON _timescaledb_catalog.continuous_agg (raw_hypertable_id); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_agg', ''); -- See the comments for ContinuousAggsBucketFunction structure. CREATE TABLE _timescaledb_catalog.continuous_aggs_bucket_function ( mat_hypertable_id integer NOT NULL, -- The bucket function bucket_func text NOT NULL, -- `bucket_width` argument of the function, e.g. "1 month" bucket_width text NOT NULL, -- optional `origin` argument of the function provided by the user bucket_origin text, -- optional `offset` argument of the function provided by the user bucket_offset text, -- optional `timezone` argument of the function provided by the user bucket_timezone text, -- fixed or variable sized bucket bucket_fixed_width bool NOT NULL, -- table constraints CONSTRAINT continuous_aggs_bucket_function_pkey PRIMARY KEY (mat_hypertable_id), CONSTRAINT continuous_aggs_bucket_function_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE, CONSTRAINT continuous_aggs_bucket_function_func_check CHECK (pg_catalog.to_regprocedure(bucket_func) IS DISTINCT FROM 0) ); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_aggs_bucket_function', ''); CREATE TABLE _timescaledb_catalog.continuous_aggs_invalidation_threshold ( hypertable_id integer NOT NULL, watermark bigint NOT NULL, -- table constraints CONSTRAINT continuous_aggs_invalidation_threshold_pkey PRIMARY KEY (hypertable_id), CONSTRAINT continuous_aggs_invalidation_threshold_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE ); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_aggs_invalidation_threshold', ''); CREATE TABLE _timescaledb_catalog.continuous_aggs_watermark ( mat_hypertable_id integer NOT NULL, watermark bigint NOT NULL, -- table constraints CONSTRAINT continuous_aggs_watermark_pkey PRIMARY KEY (mat_hypertable_id), CONSTRAINT continuous_aggs_watermark_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.continuous_agg (mat_hypertable_id) ON DELETE CASCADE ); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_aggs_watermark', ''); -- this does not have an FK on the materialization table since INSERTs to this -- table are performance critical CREATE TABLE _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log ( hypertable_id integer NOT NULL, lowest_modified_value bigint NOT NULL, greatest_modified_value bigint NOT NULL ); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_aggs_hypertable_invalidation_log', ''); CREATE INDEX continuous_aggs_hypertable_invalidation_log_idx ON _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log (hypertable_id, lowest_modified_value ASC); -- per cagg copy of invalidation log CREATE TABLE _timescaledb_catalog.continuous_aggs_materialization_invalidation_log ( materialization_id integer, lowest_modified_value bigint NOT NULL, greatest_modified_value bigint NOT NULL, -- table constraints CONSTRAINT continuous_aggs_materialization_invalid_materialization_id_fkey FOREIGN KEY (materialization_id) REFERENCES _timescaledb_catalog.continuous_agg (mat_hypertable_id) ON DELETE CASCADE ); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_aggs_materialization_invalidation_log', ''); CREATE INDEX continuous_aggs_materialization_invalidation_log_idx ON _timescaledb_catalog.continuous_aggs_materialization_invalidation_log (materialization_id, lowest_modified_value ASC); -- cagg materialization ranges CREATE TABLE _timescaledb_catalog.continuous_aggs_materialization_ranges ( materialization_id integer, lowest_modified_value bigint NOT NULL, greatest_modified_value bigint NOT NULL, -- table constraints CONSTRAINT continuous_aggs_materialization_ranges_materialization_id_fkey FOREIGN KEY (materialization_id) REFERENCES _timescaledb_catalog.continuous_agg (mat_hypertable_id) ON DELETE CASCADE ); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_aggs_materialization_ranges', ''); CREATE INDEX continuous_aggs_materialization_ranges_idx ON _timescaledb_catalog.continuous_aggs_materialization_ranges (materialization_id, lowest_modified_value ASC); /* the source of this data is the enum from the source code that lists * the algorithms. This table is NOT dumped. */ CREATE TABLE _timescaledb_catalog.compression_algorithm ( id smallint NOT NULL, version smallint NOT NULL, name name NOT NULL, description text, -- table constraints CONSTRAINT compression_algorithm_pkey PRIMARY KEY (id) ); CREATE TABLE _timescaledb_catalog.compression_settings ( relid regclass NOT NULL, compress_relid regclass NULL, segmentby text[], orderby text[], orderby_desc bool[], orderby_nullsfirst bool[], index jsonb, CONSTRAINT compression_settings_pkey PRIMARY KEY (relid), CONSTRAINT compression_settings_check_segmentby CHECK (array_ndims(segmentby) = 1), CONSTRAINT compression_settings_check_orderby_null CHECK ((orderby IS NULL AND orderby_desc IS NULL AND orderby_nullsfirst IS NULL) OR (orderby IS NOT NULL AND orderby_desc IS NOT NULL AND orderby_nullsfirst IS NOT NULL)), CONSTRAINT compression_settings_check_orderby_cardinality CHECK (array_ndims(orderby) = 1 AND array_ndims(orderby_desc) = 1 AND array_ndims(orderby_nullsfirst) = 1 AND cardinality(orderby) = cardinality(orderby_desc) AND cardinality(orderby) = cardinality(orderby_nullsfirst)) ); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.compression_settings', ''); CREATE INDEX compression_settings_compress_relid_idx ON _timescaledb_catalog.compression_settings (compress_relid); CREATE TABLE _timescaledb_catalog.compression_chunk_size ( chunk_id integer NOT NULL, compressed_chunk_id integer NOT NULL, uncompressed_heap_size bigint NOT NULL, uncompressed_toast_size bigint NOT NULL, uncompressed_index_size bigint NOT NULL, compressed_heap_size bigint NOT NULL, compressed_toast_size bigint NOT NULL, compressed_index_size bigint NOT NULL, numrows_pre_compression bigint, numrows_post_compression bigint, numrows_frozen_immediately bigint, -- table constraints CONSTRAINT compression_chunk_size_pkey PRIMARY KEY (chunk_id), CONSTRAINT compression_chunk_size_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ON DELETE CASCADE, CONSTRAINT compression_chunk_size_compressed_chunk_id_fkey FOREIGN KEY (compressed_chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ON DELETE CASCADE ); -- Create index on the compressed_chunk_id to speed up maintainance -- operations during upgrades. This is mostly relevant for very large -- number of chunks. CREATE INDEX compression_chunk_size_idx ON _timescaledb_catalog.compression_chunk_size (compressed_chunk_id); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.compression_chunk_size', ''); CREATE TABLE _timescaledb_catalog.chunk_rewrite ( chunk_relid REGCLASS NOT NULL, new_relid REGCLASS NOT NULL, CONSTRAINT chunk_rewrite_key UNIQUE (chunk_relid) ); -- Set table permissions -- We need to grant SELECT to PUBLIC for all tables even those not -- marked as being dumped because pg_dump will try to access all -- tables initially to detect inheritance chains and then decide -- which objects actually need to be dumped. GRANT SELECT ON ALL TABLES IN SCHEMA _timescaledb_catalog TO PUBLIC; GRANT SELECT ON ALL TABLES IN SCHEMA _timescaledb_internal TO PUBLIC; GRANT SELECT ON ALL SEQUENCES IN SCHEMA _timescaledb_catalog TO PUBLIC; GRANT SELECT ON ALL SEQUENCES IN SCHEMA _timescaledb_internal TO PUBLIC; -- We want to restrict access to the bgw_job_stat_history to only work through -- the job_errors and job_history views. REVOKE ALL ON _timescaledb_internal.bgw_job_stat_history FROM PUBLIC; ================================================ FILE: sql/pre_install/types.functions.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Functions have to be run in 2 places: -- 1) In pre-install between types.pre.sql and types.post.sql to set up the types. -- 2) On every update to make sure the function points to the correct versioned.so -- PostgreSQL composite types do not support constraint checks. That is why any table having a ts_interval column must use the following -- function for constraint validation. -- This function needs to be defined before executing pre_install/tables.sql because it is used as -- validation constraint for columns of type ts_interval. --the textual input/output is simply base64 encoding of the binary representation CREATE OR REPLACE FUNCTION _timescaledb_functions.compressed_data_in(CSTRING) RETURNS _timescaledb_internal.compressed_data AS '@MODULE_PATHNAME@', 'ts_compressed_data_in' LANGUAGE C IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION _timescaledb_functions.compressed_data_out(_timescaledb_internal.compressed_data) RETURNS CSTRING AS '@MODULE_PATHNAME@', 'ts_compressed_data_out' LANGUAGE C IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION _timescaledb_functions.compressed_data_send(_timescaledb_internal.compressed_data) RETURNS BYTEA AS '@MODULE_PATHNAME@', 'ts_compressed_data_send' LANGUAGE C IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION _timescaledb_functions.compressed_data_recv(internal) RETURNS _timescaledb_internal.compressed_data AS '@MODULE_PATHNAME@', 'ts_compressed_data_recv' LANGUAGE C IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION _timescaledb_functions.compressed_data_info(_timescaledb_internal.compressed_data) RETURNS TABLE (algorithm name, has_nulls bool) LANGUAGE C STRICT IMMUTABLE AS '@MODULE_PATHNAME@', 'ts_compressed_data_info'; CREATE OR REPLACE FUNCTION _timescaledb_functions.compressed_data_has_nulls(_timescaledb_internal.compressed_data) RETURNS BOOL LANGUAGE C STRICT IMMUTABLE AS '@MODULE_PATHNAME@', 'ts_compressed_data_has_nulls'; CREATE OR REPLACE FUNCTION _timescaledb_functions.dimension_info_in(cstring) RETURNS _timescaledb_internal.dimension_info LANGUAGE C STRICT IMMUTABLE AS '@MODULE_PATHNAME@', 'ts_dimension_info_in'; CREATE OR REPLACE FUNCTION _timescaledb_functions.dimension_info_out(_timescaledb_internal.dimension_info) RETURNS cstring LANGUAGE C STRICT IMMUTABLE AS '@MODULE_PATHNAME@', 'ts_dimension_info_out'; -- Type for bloom filters used by the sparse indexes on compressed hypertables. CREATE OR REPLACE FUNCTION _timescaledb_functions.bloom1in(cstring) RETURNS _timescaledb_internal.bloom1 AS 'byteain' LANGUAGE INTERNAL STRICT IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.bloom1out(_timescaledb_internal.bloom1) RETURNS cstring AS 'byteaout' LANGUAGE INTERNAL STRICT IMMUTABLE PARALLEL SAFE; ================================================ FILE: sql/pre_install/types.post.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- -- The general compressed_data type; -- CREATE TYPE _timescaledb_internal.compressed_data ( INTERNALLENGTH = VARIABLE, STORAGE = EXTERNAL, ALIGNMENT = double, --needed for alignment in ARRAY type compression INPUT = _timescaledb_functions.compressed_data_in, OUTPUT = _timescaledb_functions.compressed_data_out, RECEIVE = _timescaledb_functions.compressed_data_recv, SEND = _timescaledb_functions.compressed_data_send ); -- Dimension type used in create_hypertable, add_dimension, etc. It is -- deliberately an opaque type. CREATE TYPE _timescaledb_internal.dimension_info ( INPUT = _timescaledb_functions.dimension_info_in, OUTPUT = _timescaledb_functions.dimension_info_out, INTERNALLENGTH = VARIABLE ); -- Type for bloom filters used by the sparse indexes on compressed hypertables. CREATE TYPE _timescaledb_internal.bloom1 ( INPUT = _timescaledb_functions.bloom1in, OUTPUT = _timescaledb_functions.bloom1out, LIKE = bytea ); ================================================ FILE: sql/pre_install/types.pre.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- -- The general compressed_data type; -- CREATE TYPE _timescaledb_internal.compressed_data; --placeholder to allow creation of functions below CREATE TYPE _timescaledb_internal.dimension_info; -- Type for bloom filters used by the sparse indexes on compressed hypertables. CREATE TYPE _timescaledb_internal.bloom1; ================================================ FILE: sql/restoring.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE OR REPLACE FUNCTION @extschema@.timescaledb_pre_restore() RETURNS BOOL AS $BODY$ DECLARE db text; BEGIN SELECT current_database() INTO db; EXECUTE format($$ALTER DATABASE %I SET timescaledb.restoring ='on'$$, db); SET SESSION timescaledb.restoring = 'on'; PERFORM _timescaledb_functions.stop_background_workers(); RETURN true; END $BODY$ LANGUAGE PLPGSQL SET search_path TO pg_catalog, pg_temp; CREATE OR REPLACE FUNCTION @extschema@.timescaledb_post_restore() RETURNS BOOL AS $BODY$ DECLARE db text; catalog_version text; BEGIN SELECT m.value INTO catalog_version FROM pg_extension x JOIN _timescaledb_catalog.metadata m ON m.key='timescaledb_version' WHERE x.extname='timescaledb' AND x.extversion <> m.value; -- check that a loaded dump is compatible with the currently running code IF FOUND THEN RAISE EXCEPTION 'catalog version mismatch, expected "%" seen "%"', '@PROJECT_VERSION_MOD@', catalog_version; END IF; SELECT current_database() INTO db; EXECUTE format($$ALTER DATABASE %I RESET timescaledb.restoring $$, db); -- we cannot use reset here because the reset_val might not be off SET timescaledb.restoring TO off; PERFORM _timescaledb_functions.restart_background_workers(); RETURN true; END $BODY$ LANGUAGE PLPGSQL SET search_path TO pg_catalog, pg_temp; ================================================ FILE: sql/size_utils.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- This file contains utility functions to get the relation size -- of hypertables, chunks, and indexes on hypertables. CREATE OR REPLACE FUNCTION _timescaledb_functions.index_matches(index1 regclass, index2 regclass) RETURNS BOOLEAN AS '@MODULE_PATHNAME@', 'ts_index_matches' LANGUAGE C STRICT IMMUTABLE; CREATE OR REPLACE FUNCTION _timescaledb_functions.relation_size(relation REGCLASS) RETURNS TABLE (total_size BIGINT, heap_size BIGINT, index_size BIGINT, toast_size BIGINT) AS '@MODULE_PATHNAME@', 'ts_relation_size' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION _timescaledb_functions.relation_approximate_size(relation REGCLASS) RETURNS TABLE (total_size BIGINT, heap_size BIGINT, index_size BIGINT, toast_size BIGINT) AS '@MODULE_PATHNAME@', 'ts_relation_approximate_size' LANGUAGE C STRICT VOLATILE; CREATE OR REPLACE VIEW _timescaledb_internal.hypertable_chunk_local_size AS SELECT h.schema_name AS hypertable_schema, h.table_name AS hypertable_name, h.id AS hypertable_id, c.id AS chunk_id, c.schema_name AS chunk_schema, c.table_name AS chunk_name, COALESCE((relsize).total_size, 0) AS total_bytes, COALESCE((relsize).heap_size, 0) AS heap_bytes, COALESCE((relsize).index_size, 0) AS index_bytes, COALESCE((relsize).toast_size, 0) AS toast_bytes, COALESCE((relcompsize).total_size, 0) AS compressed_total_size, COALESCE((relcompsize).heap_size, 0) AS compressed_heap_size, COALESCE((relcompsize).index_size, 0) AS compressed_index_size, COALESCE((relcompsize).toast_size, 0) AS compressed_toast_size FROM _timescaledb_catalog.hypertable h JOIN _timescaledb_catalog.chunk c ON h.id = c.hypertable_id JOIN pg_class cl ON cl.relname = c.table_name AND cl.relkind = 'r' JOIN pg_namespace n ON n.oid = cl.relnamespace AND n.nspname = c.schema_name JOIN LATERAL _timescaledb_functions.relation_size(cl.oid) AS relsize ON TRUE LEFT JOIN _timescaledb_catalog.compression_settings cs ON cs.relid = cl.oid LEFT JOIN LATERAL _timescaledb_functions.relation_size(cs.compress_relid) AS relcompsize ON TRUE; GRANT SELECT ON _timescaledb_internal.hypertable_chunk_local_size TO PUBLIC; CREATE OR REPLACE FUNCTION _timescaledb_functions.hypertable_local_size( schema_name_in name, table_name_in name) RETURNS TABLE ( table_bytes BIGINT, index_bytes BIGINT, toast_bytes BIGINT, total_bytes BIGINT) LANGUAGE SQL VOLATILE STRICT AS $BODY$ /* get the main hypertable id and sizes */ WITH _hypertable_sizes AS ( SELECT id, COALESCE((relsize).total_size, 0) AS total_bytes, COALESCE((relsize).heap_size, 0) AS heap_bytes, COALESCE((relsize).index_size, 0) AS index_bytes, COALESCE((relsize).toast_size, 0) AS toast_bytes, 0::BIGINT AS compressed_total_size, 0::BIGINT AS compressed_index_size, 0::BIGINT AS compressed_toast_size, 0::BIGINT AS compressed_heap_size FROM _timescaledb_catalog.hypertable ht JOIN pg_class c ON relname = ht.table_name AND c.relkind = 'r' JOIN pg_namespace n ON n.oid = c.relnamespace AND n.nspname = ht.schema_name JOIN LATERAL _timescaledb_functions.relation_size(c.oid) AS relsize ON TRUE WHERE schema_name = schema_name_in AND table_name = table_name_in ), /* calculate the size of the hypertable chunks */ _chunk_sizes AS ( SELECT chunk_id, COALESCE(ch.total_bytes, 0) AS total_bytes, COALESCE(ch.heap_bytes, 0) AS heap_bytes, COALESCE(ch.index_bytes, 0) AS index_bytes, COALESCE(ch.toast_bytes, 0) AS toast_bytes, COALESCE(ch.compressed_total_size, 0) AS compressed_total_size, COALESCE(ch.compressed_index_size, 0) AS compressed_index_size, COALESCE(ch.compressed_toast_size, 0) AS compressed_toast_size, COALESCE(ch.compressed_heap_size, 0) AS compressed_heap_size FROM _timescaledb_internal.hypertable_chunk_local_size ch JOIN _hypertable_sizes ht ON ht.id = ch.hypertable_id WHERE hypertable_schema = schema_name_in AND hypertable_name = table_name_in ) /* calculate the SUM of the hypertable and chunk sizes */ SELECT (SUM(heap_bytes) + SUM(compressed_heap_size))::BIGINT AS heap_bytes, (SUM(index_bytes) + SUM(compressed_index_size))::BIGINT AS index_bytes, (SUM(toast_bytes) + SUM(compressed_toast_size))::BIGINT AS toast_bytes, (SUM(total_bytes) + SUM(compressed_total_size))::BIGINT AS total_bytes FROM (SELECT * FROM _hypertable_sizes UNION ALL SELECT * FROM _chunk_sizes) AS sizes; $BODY$ SET search_path TO pg_catalog, pg_temp; -- Get relation size of hypertable -- like pg_relation_size(hypertable) -- -- hypertable - hypertable to get size of -- -- Returns: -- table_bytes - Disk space used by hypertable (like pg_relation_size(hypertable)) -- index_bytes - Disk space used by indexes -- toast_bytes - Disk space of toast tables -- total_bytes - Total disk space used by the specified table, including all indexes and TOAST data CREATE OR REPLACE FUNCTION @extschema@.hypertable_detailed_size( hypertable REGCLASS) RETURNS TABLE (table_bytes BIGINT, index_bytes BIGINT, toast_bytes BIGINT, total_bytes BIGINT, node_name NAME) LANGUAGE PLPGSQL VOLATILE STRICT AS $BODY$ DECLARE table_name NAME = NULL; schema_name NAME = NULL; BEGIN SELECT relname, nspname INTO table_name, schema_name FROM pg_class c INNER JOIN pg_namespace n ON (n.OID = c.relnamespace) INNER JOIN _timescaledb_catalog.hypertable ht ON (ht.schema_name = n.nspname AND ht.table_name = c.relname) WHERE c.OID = hypertable; IF table_name IS NULL THEN SELECT h.schema_name, h.table_name INTO schema_name, table_name FROM pg_class c INNER JOIN pg_namespace n ON (n.OID = c.relnamespace) INNER JOIN _timescaledb_catalog.continuous_agg a ON (a.user_view_schema = n.nspname AND a.user_view_name = c.relname) INNER JOIN _timescaledb_catalog.hypertable h ON h.id = a.mat_hypertable_id WHERE c.OID = hypertable; IF table_name IS NULL THEN RETURN; END IF; END IF; RETURN QUERY SELECT *, NULL::name FROM _timescaledb_functions.hypertable_local_size(schema_name, table_name); END; $BODY$ SET search_path TO pg_catalog, pg_temp; --- returns total-bytes for a hypertable (includes table + index) CREATE OR REPLACE FUNCTION @extschema@.hypertable_size( hypertable REGCLASS) RETURNS BIGINT LANGUAGE SQL VOLATILE STRICT AS $BODY$ SELECT total_bytes::bigint FROM @extschema@.hypertable_detailed_size(hypertable); $BODY$ SET search_path TO pg_catalog, pg_temp; -- Get approximate relation size of hypertable -- -- hypertable - hypertable to get approximate size of -- -- Returns: -- table_bytes - Approximate disk space used by hypertable -- index_bytes - Approximate disk space used by indexes -- toast_bytes - Approximate disk space of toast tables -- total_bytes - Total approximate disk space used by the specified table, including all indexes and TOAST data CREATE OR REPLACE FUNCTION @extschema@.hypertable_approximate_detailed_size(relation REGCLASS) RETURNS TABLE (table_bytes BIGINT, index_bytes BIGINT, toast_bytes BIGINT, total_bytes BIGINT) AS '@MODULE_PATHNAME@', 'ts_hypertable_approximate_size' LANGUAGE C VOLATILE; --- returns approximate total-bytes for a hypertable (includes table + index) CREATE OR REPLACE FUNCTION @extschema@.hypertable_approximate_size( hypertable REGCLASS) RETURNS BIGINT LANGUAGE SQL VOLATILE STRICT AS $BODY$ SELECT sum(total_bytes)::bigint FROM @extschema@.hypertable_approximate_detailed_size(hypertable); $BODY$ SET search_path TO pg_catalog, pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_functions.chunks_local_size( schema_name_in name, table_name_in name) RETURNS TABLE ( chunk_id integer, chunk_schema NAME, chunk_name NAME, table_bytes bigint, index_bytes bigint, toast_bytes bigint, total_bytes bigint) LANGUAGE SQL VOLATILE STRICT AS $BODY$ SELECT ch.chunk_id, ch.chunk_schema, ch.chunk_name, (ch.total_bytes - COALESCE( ch.index_bytes , 0 ) - COALESCE( ch.toast_bytes, 0 ) + COALESCE( ch.compressed_heap_size , 0 ))::bigint as heap_bytes, (COALESCE( ch.index_bytes, 0 ) + COALESCE( ch.compressed_index_size , 0) )::bigint as index_bytes, (COALESCE( ch.toast_bytes, 0 ) + COALESCE( ch.compressed_toast_size, 0 ))::bigint as toast_bytes, (ch.total_bytes + COALESCE( ch.compressed_total_size, 0 ))::bigint as total_bytes FROM _timescaledb_internal.hypertable_chunk_local_size ch WHERE ch.hypertable_schema = schema_name_in AND ch.hypertable_name = table_name_in; $BODY$ SET search_path TO pg_catalog, pg_temp; -- Get relation size of the chunks of an hypertable -- hypertable - hypertable to get size of -- -- Returns: -- chunk_schema - schema name for chunk -- chunk_name - chunk table name -- table_bytes - Disk space used by chunk table -- index_bytes - Disk space used by indexes -- toast_bytes - Disk space of toast tables -- total_bytes - Disk space used in total -- node_name - node on which chunk lives if this is -- a distributed hypertable. CREATE OR REPLACE FUNCTION @extschema@.chunks_detailed_size( hypertable REGCLASS ) RETURNS TABLE ( chunk_schema NAME, chunk_name NAME, table_bytes BIGINT, index_bytes BIGINT, toast_bytes BIGINT, total_bytes BIGINT, node_name NAME) LANGUAGE PLPGSQL VOLATILE STRICT AS $BODY$ DECLARE table_name NAME; schema_name NAME; BEGIN SELECT relname, nspname INTO table_name, schema_name FROM pg_class c INNER JOIN pg_namespace n ON (n.OID = c.relnamespace) INNER JOIN _timescaledb_catalog.hypertable ht ON (ht.schema_name = n.nspname AND ht.table_name = c.relname) WHERE c.OID = hypertable; IF table_name IS NULL THEN SELECT h.schema_name, h.table_name INTO schema_name, table_name FROM pg_class c INNER JOIN pg_namespace n ON (n.OID = c.relnamespace) INNER JOIN _timescaledb_catalog.continuous_agg a ON (a.user_view_schema = n.nspname AND a.user_view_name = c.relname) INNER JOIN _timescaledb_catalog.hypertable h ON h.id = a.mat_hypertable_id WHERE c.OID = hypertable; IF table_name IS NULL THEN RETURN; END IF; END IF; RETURN QUERY SELECT chl.chunk_schema, chl.chunk_name, chl.table_bytes, chl.index_bytes, chl.toast_bytes, chl.total_bytes, NULL::NAME FROM _timescaledb_functions.chunks_local_size(schema_name, table_name) chl; END; $BODY$ SET search_path TO pg_catalog, pg_temp; ---------- end of detailed size functions ------ CREATE OR REPLACE FUNCTION _timescaledb_functions.range_value_to_pretty( time_value BIGINT, column_type REGTYPE ) RETURNS TEXT LANGUAGE PLPGSQL STABLE AS $BODY$ DECLARE BEGIN IF NOT (time_value > (-9223372036854775808)::bigint AND time_value < 9223372036854775807::bigint) THEN RETURN ''; END IF; IF time_value IS NULL THEN RETURN format('%L', NULL); END IF; CASE column_type WHEN 'BIGINT'::regtype, 'INTEGER'::regtype, 'SMALLINT'::regtype THEN RETURN format('%L', time_value); -- scale determined by user. WHEN 'TIMESTAMP'::regtype, 'TIMESTAMPTZ'::regtype THEN -- assume time_value is in microsec RETURN format('%1$L', _timescaledb_functions.to_timestamp(time_value)); -- microseconds WHEN 'DATE'::regtype THEN RETURN format('%L', timezone('UTC',_timescaledb_functions.to_timestamp(time_value))::date); ELSE RETURN time_value; END CASE; END $BODY$ SET search_path TO pg_catalog, pg_temp; -- Convenience function to return approximate row count -- -- relation - table or hypertable to get approximate row count for -- -- Returns: -- Estimated number of rows according to catalog tables CREATE OR REPLACE FUNCTION @extschema@.approximate_row_count(relation REGCLASS) RETURNS BIGINT LANGUAGE PLPGSQL VOLATILE STRICT AS $BODY$ DECLARE v_mat_ht REGCLASS = NULL; v_name NAME = NULL; v_schema NAME = NULL; v_hypertable_id INTEGER; BEGIN -- Check if input relation is continuous aggregate view then -- get the corresponding materialized hypertable and schema name SELECT format('%I.%I', ht.schema_name, ht.table_name)::regclass INTO v_mat_ht FROM pg_class c JOIN pg_namespace n ON (n.OID = c.relnamespace) JOIN _timescaledb_catalog.continuous_agg a ON (a.user_view_schema = n.nspname AND a.user_view_name = c.relname) JOIN _timescaledb_catalog.hypertable ht ON (a.mat_hypertable_id = ht.id) WHERE c.OID = relation; IF FOUND THEN relation = v_mat_ht; END IF; SELECT nspname, relname FROM pg_class c INNER JOIN pg_namespace n ON (n.OID = c.relnamespace) INTO v_schema, v_name WHERE c.OID = relation; -- for hypertables return the sum of the row counts of all chunks SELECT id FROM _timescaledb_catalog.hypertable INTO v_hypertable_id WHERE table_name = v_name AND schema_name = v_schema; IF FOUND THEN RETURN (SELECT coalesce(sum(_timescaledb_functions.get_approx_row_count(format('%I.%I',schema_name,table_name))),0) FROM _timescaledb_catalog.chunk WHERE hypertable_id = v_hypertable_id); END IF; IF EXISTS (SELECT FROM pg_inherits WHERE inhparent = relation) THEN RETURN ( SELECT _timescaledb_functions.get_approx_row_count(relation) + COALESCE(SUM(@extschema@.approximate_row_count(i.inhrelid)),0) FROM pg_inherits i WHERE i.inhparent = relation ); END IF; -- Check for input relation is Plain RELATION RETURN _timescaledb_functions.get_approx_row_count(relation); END; $BODY$ SET search_path TO pg_catalog, pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_functions.estimate_compressed_batch_size(relation REGCLASS) RETURNS FLOAT8 AS '@MODULE_PATHNAME@', 'ts_estimate_compressed_batch_size' LANGUAGE C STRICT STABLE; CREATE OR REPLACE FUNCTION _timescaledb_functions.get_approx_row_count(relation REGCLASS) RETURNS BIGINT LANGUAGE PLPGSQL VOLATILE STRICT AS $BODY$ DECLARE v_schema NAME; v_name NAME; v_chunk_id INTEGER; v_oid OID; row_count BIGINT = 0; BEGIN SELECT nspname, relname INTO v_schema, v_name FROM pg_class c JOIN pg_namespace n ON (n.OID = c.relnamespace) WHERE c.OID = relation; -- we only need to check if the relation has a compressed chunk if it is a chunk SELECT compress_relid FROM _timescaledb_catalog.compression_settings INTO v_oid WHERE relid = relation; IF v_oid IS NOT NULL THEN row_count := (SELECT CASE WHEN reltuples IS NULL THEN 0 WHEN reltuples < 0 THEN 0 ELSE reltuples * _timescaledb_functions.estimate_compressed_batch_size(oid) END FROM pg_class WHERE oid = v_oid); END IF; row_count := COALESCE((SELECT row_count + CASE WHEN reltuples < 0 OR relkind = 'p' THEN 0 ELSE reltuples END FROM pg_class WHERE oid = relation), 0); RETURN row_count; END $BODY$ SET search_path TO pg_catalog, pg_temp; -------- stats related to compression ------ CREATE OR REPLACE VIEW _timescaledb_internal.compressed_chunk_stats AS SELECT srcht.schema_name AS hypertable_schema, srcht.table_name AS hypertable_name, srcch.schema_name AS chunk_schema, srcch.table_name AS chunk_name, CASE WHEN srcch.status & 1 = 1 THEN 'Compressed'::text ELSE 'Uncompressed'::text END AS compression_status, map.uncompressed_heap_size, map.uncompressed_index_size, map.uncompressed_toast_size, map.uncompressed_heap_size + map.uncompressed_toast_size + map.uncompressed_index_size AS uncompressed_total_size, map.compressed_heap_size, map.compressed_index_size, map.compressed_toast_size, map.compressed_heap_size + map.compressed_toast_size + map.compressed_index_size AS compressed_total_size FROM _timescaledb_catalog.hypertable AS srcht JOIN _timescaledb_catalog.chunk AS srcch ON srcht.id = srcch.hypertable_id AND srcht.compressed_hypertable_id IS NOT NULL LEFT JOIN _timescaledb_catalog.compression_chunk_size map ON srcch.id = map.chunk_id; GRANT SELECT ON _timescaledb_internal.compressed_chunk_stats TO PUBLIC; CREATE OR REPLACE FUNCTION _timescaledb_functions.compressed_chunk_local_stats(schema_name_in name, table_name_in name) RETURNS TABLE ( chunk_schema name, chunk_name name, compression_status text, before_compression_table_bytes bigint, before_compression_index_bytes bigint, before_compression_toast_bytes bigint, before_compression_total_bytes bigint, after_compression_table_bytes bigint, after_compression_index_bytes bigint, after_compression_toast_bytes bigint, after_compression_total_bytes bigint) LANGUAGE SQL STABLE STRICT AS $BODY$ SELECT ch.chunk_schema, ch.chunk_name, ch.compression_status, ch.uncompressed_heap_size, ch.uncompressed_index_size, ch.uncompressed_toast_size, ch.uncompressed_total_size, ch.compressed_heap_size, ch.compressed_index_size, ch.compressed_toast_size, ch.compressed_total_size FROM _timescaledb_internal.compressed_chunk_stats ch WHERE ch.hypertable_schema = schema_name_in AND ch.hypertable_name = table_name_in; $BODY$ SET search_path TO pg_catalog, pg_temp; -- Get per chunk compression statistics for a hypertable that has -- compression enabled CREATE OR REPLACE FUNCTION @extschema@.chunk_compression_stats (hypertable REGCLASS) RETURNS TABLE ( chunk_schema name, chunk_name name, compression_status text, before_compression_table_bytes bigint, before_compression_index_bytes bigint, before_compression_toast_bytes bigint, before_compression_total_bytes bigint, after_compression_table_bytes bigint, after_compression_index_bytes bigint, after_compression_toast_bytes bigint, after_compression_total_bytes bigint, node_name name) LANGUAGE PLPGSQL STABLE STRICT AS $BODY$ DECLARE table_name name; schema_name name; BEGIN SELECT relname, nspname INTO table_name, schema_name FROM pg_class c INNER JOIN pg_namespace n ON (n.OID = c.relnamespace) INNER JOIN _timescaledb_catalog.hypertable ht ON (ht.schema_name = n.nspname AND ht.table_name = c.relname) WHERE c.OID = hypertable; IF table_name IS NULL THEN RETURN; END IF; RETURN QUERY SELECT *, NULL::name FROM _timescaledb_functions.compressed_chunk_local_stats(schema_name, table_name); END; $BODY$ SET search_path TO pg_catalog, pg_temp; CREATE OR REPLACE FUNCTION @extschema@.chunk_columnstore_stats (hypertable REGCLASS) RETURNS TABLE ( chunk_schema name, chunk_name name, compression_status text, before_compression_table_bytes bigint, before_compression_index_bytes bigint, before_compression_toast_bytes bigint, before_compression_total_bytes bigint, after_compression_table_bytes bigint, after_compression_index_bytes bigint, after_compression_toast_bytes bigint, after_compression_total_bytes bigint, node_name name) LANGUAGE SQL STABLE STRICT AS 'SELECT * FROM @extschema@.chunk_compression_stats($1)' SET search_path TO pg_catalog, pg_temp; -- Get compression statistics for a hypertable that has -- compression enabled CREATE OR REPLACE FUNCTION @extschema@.hypertable_compression_stats (hypertable REGCLASS) RETURNS TABLE ( total_chunks bigint, number_compressed_chunks bigint, before_compression_table_bytes bigint, before_compression_index_bytes bigint, before_compression_toast_bytes bigint, before_compression_total_bytes bigint, after_compression_table_bytes bigint, after_compression_index_bytes bigint, after_compression_toast_bytes bigint, after_compression_total_bytes bigint, node_name name) LANGUAGE SQL STABLE STRICT AS $BODY$ SELECT count(*)::bigint AS total_chunks, (count(*) FILTER (WHERE ch.compression_status = 'Compressed'))::bigint AS number_compressed_chunks, sum(ch.before_compression_table_bytes)::bigint AS before_compression_table_bytes, sum(ch.before_compression_index_bytes)::bigint AS before_compression_index_bytes, sum(ch.before_compression_toast_bytes)::bigint AS before_compression_toast_bytes, sum(ch.before_compression_total_bytes)::bigint AS before_compression_total_bytes, sum(ch.after_compression_table_bytes)::bigint AS after_compression_table_bytes, sum(ch.after_compression_index_bytes)::bigint AS after_compression_index_bytes, sum(ch.after_compression_toast_bytes)::bigint AS after_compression_toast_bytes, sum(ch.after_compression_total_bytes)::bigint AS after_compression_total_bytes, ch.node_name FROM @extschema@.chunk_compression_stats(hypertable) ch GROUP BY ch.node_name; $BODY$ SET search_path TO pg_catalog, pg_temp; CREATE OR REPLACE FUNCTION @extschema@.hypertable_columnstore_stats (hypertable REGCLASS) RETURNS TABLE ( total_chunks bigint, number_compressed_chunks bigint, before_compression_table_bytes bigint, before_compression_index_bytes bigint, before_compression_toast_bytes bigint, before_compression_total_bytes bigint, after_compression_table_bytes bigint, after_compression_index_bytes bigint, after_compression_toast_bytes bigint, after_compression_total_bytes bigint, node_name name) LANGUAGE SQL STABLE STRICT AS 'SELECT * FROM @extschema@.hypertable_compression_stats($1)' SET search_path TO pg_catalog, pg_temp; -------------Get index size for hypertables ------- CREATE OR REPLACE FUNCTION @extschema@.hypertable_index_size( index_name REGCLASS ) RETURNS BIGINT LANGUAGE SQL VOLATILE STRICT AS $BODY$ SELECT pg_relation_size(ht_i.indexrelid) + COALESCE(sum(pg_relation_size(ch_i.indexrelid)), 0) FROM pg_index ht_i LEFT JOIN pg_inherits ch on ch.inhparent = ht_i.indrelid LEFT JOIN pg_index ch_i on ch_i.indrelid = ch.inhrelid and _timescaledb_functions.index_matches(ht_i.indexrelid, ch_i.indexrelid) WHERE ht_i.indexrelid = index_name GROUP BY ht_i.indexrelid; $BODY$ SET search_path TO pg_catalog, pg_temp; -------------End index size for hypertables ------- CREATE OR REPLACE FUNCTION _timescaledb_functions.estimate_uncompressed_size(IN regclass, OUT tuples bigint, OUT relation_size bigint, OUT index_size bigint, OUT total_size bigint) AS $$ DECLARE v_compressed_chunk regclass; v_uncompressed_chunk regclass; v_index regclass; v_fixed_column_size integer; v_num_varlen_columns integer; v_tuple_header integer; v_tuple_data integer; v_index_header integer; v_index_size bigint; v_columns integer; v_varlen_query text:= ''; v_multiplier decimal:=1.15; -- multiplier to account for page header, fill factor and alignment padding v_index_multiplier decimal:=1.25; -- multiplier to account for page header, fill factor and alignment padding BEGIN v_compressed_chunk := $1; SELECT relid INTO v_uncompressed_chunk FROM _timescaledb_catalog.compression_settings WHERE compress_relid = v_compressed_chunk; IF NOT FOUND THEN RETURN; END IF; SELECT count(*), sum(attlen) FILTER(WHERE attlen > 0), count(*) FILTER(WHERE attlen = -1) FROM pg_attribute INTO v_columns, v_fixed_column_size, v_num_varlen_columns WHERE attrelid = v_uncompressed_chunk AND attnum > 0 AND NOT attisdropped; -- header size = MAXALIGN(Header + NullBitmap) + MAXALIGN(Data) v_tuple_header := 23; -- Heap tuple header v_tuple_header := v_tuple_header + ((v_columns + 7) / 8); -- Null bitmap size v_tuple_header := v_tuple_header + 7 & ~7; -- align to 8 bytes v_tuple_data := v_fixed_column_size; -- Fixed-length column sizes v_tuple_data := v_tuple_data + 7 & ~7; -- align to 8 bytes IF v_num_varlen_columns > 0 THEN SELECT ' + (' || string_agg(format('sum(_timescaledb_functions.compressed_data_column_size(%I,NULL::%s))', attname, pg_catalog.format_type(atttypid, atttypmod)), ' + ') || ')' FROM pg_attribute INTO v_varlen_query WHERE attrelid = v_uncompressed_chunk AND attnum > 0 AND NOT attisdropped AND attlen = -1; END IF; EXECUTE format('SELECT sum(_ts_meta_count) FROM %s', v_compressed_chunk) INTO tuples; -- we can optimize the following query if all columns are fixed size EXECUTE format('SELECT ((%s * (%s + %s)) %s) * %s FROM %s', tuples, v_tuple_header, v_tuple_data, v_varlen_query, v_multiplier, v_compressed_chunk) INTO relation_size; index_size := 0; FOR v_index, v_varlen_query, v_columns IN SELECT i.indexrelid::regclass, (SELECT ' + (' || string_agg(format('sum(_timescaledb_functions.compressed_data_column_size(%I,NULL::%s))', attname, pg_catalog.format_type(atttypid, atttypmod)), ' + ' ORDER BY attnum) || ')' FROM pg_attribute att WHERE att.attrelid=i.indrelid AND attnum =ANY(i.indkey)), array_length(i.indkey,1) FROM pg_index i WHERE i.indrelid = v_uncompressed_chunk LOOP v_index_header := 8; -- Index tuple header -- v_compressed_chunk is a regclass, which will be properly escaped when cast to `text` EXECUTE format('SELECT ((%s * %s) %s) * %s FROM %s', tuples, v_index_header, v_varlen_query, v_index_multiplier, v_compressed_chunk) INTO v_index_size; index_size := index_size + v_index_size; END LOOP; total_size := relation_size + index_size; END $$ LANGUAGE plpgsql SET search_path TO pg_catalog, pg_temp; ================================================ FILE: sql/sparse_index.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE OR REPLACE FUNCTION _timescaledb_functions.bloom1_contains(_timescaledb_internal.bloom1, anyelement) RETURNS bool AS '@MODULE_PATHNAME@', 'ts_bloom1_contains' LANGUAGE C IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.bloom1_contains_any(_timescaledb_internal.bloom1, anyarray) RETURNS bool AS '@MODULE_PATHNAME@', 'ts_bloom1_contains_any' LANGUAGE C IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.jsonb_get_matching_index_entry( config jsonb, attr_name text, target_type text ) RETURNS jsonb AS $$ DECLARE elem jsonb; attr_count int := 0; BEGIN -- Return NULL if any input is NULL IF config IS NULL OR attr_name IS NULL OR target_type IS NULL THEN RETURN NULL; END IF; FOR elem IN SELECT * FROM jsonb_array_elements(config) LOOP IF elem->>'column' = attr_name THEN attr_count := attr_count + 1; IF elem->>'type' = target_type THEN IF attr_count > 2 THEN RAISE EXCEPTION 'Found % sparse index entries for attribute "%"', attr_count, attr_name; END IF; RETURN elem; END IF; END IF; END LOOP; IF attr_count > 2 THEN RAISE EXCEPTION 'Found % sparse index entries for attribute "%"', attr_count, attr_name; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL SET search_path TO pg_catalog, pg_temp; ================================================ FILE: sql/time_bucket.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- time_bucket returns the left edge of the bucket where ts falls into. -- Buckets span an interval of time equal to the bucket_width and are aligned with the epoch. CREATE OR REPLACE FUNCTION @extschema@.time_bucket(bucket_width INTERVAL, ts TIMESTAMP) RETURNS TIMESTAMP AS '@MODULE_PATHNAME@', 'ts_timestamp_bucket' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; -- bucketing of timestamptz happens at UTC time CREATE OR REPLACE FUNCTION @extschema@.time_bucket(bucket_width INTERVAL, ts TIMESTAMPTZ) RETURNS TIMESTAMPTZ AS '@MODULE_PATHNAME@', 'ts_timestamptz_bucket' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; --bucketing on date should not do any timezone conversion CREATE OR REPLACE FUNCTION @extschema@.time_bucket(bucket_width INTERVAL, ts DATE) RETURNS DATE AS '@MODULE_PATHNAME@', 'ts_date_bucket' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION @extschema@.time_bucket(bucket_width INTERVAL, ts UUID) RETURNS TIMESTAMPTZ AS '@MODULE_PATHNAME@', 'ts_uuid_bucket' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; --bucketing with origin CREATE OR REPLACE FUNCTION @extschema@.time_bucket(bucket_width INTERVAL, ts TIMESTAMP, origin TIMESTAMP) RETURNS TIMESTAMP AS '@MODULE_PATHNAME@', 'ts_timestamp_bucket' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION @extschema@.time_bucket(bucket_width INTERVAL, ts TIMESTAMPTZ, origin TIMESTAMPTZ) RETURNS TIMESTAMPTZ AS '@MODULE_PATHNAME@', 'ts_timestamptz_bucket' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION @extschema@.time_bucket(bucket_width INTERVAL, ts DATE, origin DATE) RETURNS DATE AS '@MODULE_PATHNAME@', 'ts_date_bucket' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION @extschema@.time_bucket(bucket_width INTERVAL, ts UUID, origin TIMESTAMPTZ) RETURNS TIMESTAMPTZ AS '@MODULE_PATHNAME@', 'ts_uuid_bucket' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; --bucketing with offset CREATE OR REPLACE FUNCTION @extschema@.time_bucket(bucket_width INTERVAL, ts TIMESTAMP, "offset" INTERVAL) RETURNS TIMESTAMP AS '@MODULE_PATHNAME@', 'ts_timestamp_offset_bucket' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION @extschema@.time_bucket(bucket_width INTERVAL, ts TIMESTAMPTZ, "offset" INTERVAL) RETURNS TIMESTAMPTZ AS '@MODULE_PATHNAME@', 'ts_timestamptz_offset_bucket' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION @extschema@.time_bucket(bucket_width INTERVAL, ts DATE, "offset" INTERVAL) RETURNS DATE AS '@MODULE_PATHNAME@', 'ts_date_offset_bucket' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION @extschema@.time_bucket(bucket_width INTERVAL, ts UUID, "offset" INTERVAL) RETURNS TIMESTAMPTZ AS '@MODULE_PATHNAME@', 'ts_uuid_offset_bucket' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; -- bucketing with timezone CREATE OR REPLACE FUNCTION @extschema@.time_bucket(bucket_width INTERVAL, ts TIMESTAMPTZ, timezone TEXT, origin TIMESTAMPTZ DEFAULT NULL, "offset" INTERVAL DEFAULT NULL) RETURNS TIMESTAMPTZ AS '@MODULE_PATHNAME@', 'ts_timestamptz_timezone_bucket' LANGUAGE C IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION @extschema@.time_bucket(bucket_width INTERVAL, ts UUID, timezone TEXT, origin TIMESTAMPTZ DEFAULT NULL, "offset" INTERVAL DEFAULT NULL) RETURNS TIMESTAMPTZ AS '@MODULE_PATHNAME@', 'ts_uuid_timezone_bucket' LANGUAGE C IMMUTABLE PARALLEL SAFE; -- bucketing of int CREATE OR REPLACE FUNCTION @extschema@.time_bucket(bucket_width SMALLINT, ts SMALLINT) RETURNS SMALLINT AS '@MODULE_PATHNAME@', 'ts_int16_bucket' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION @extschema@.time_bucket(bucket_width INT, ts INT) RETURNS INT AS '@MODULE_PATHNAME@', 'ts_int32_bucket' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION @extschema@.time_bucket(bucket_width BIGINT, ts BIGINT) RETURNS BIGINT AS '@MODULE_PATHNAME@', 'ts_int64_bucket' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; -- bucketing of int with offset CREATE OR REPLACE FUNCTION @extschema@.time_bucket(bucket_width SMALLINT, ts SMALLINT, "offset" SMALLINT) RETURNS SMALLINT AS '@MODULE_PATHNAME@', 'ts_int16_bucket' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION @extschema@.time_bucket(bucket_width INT, ts INT, "offset" INT) RETURNS INT AS '@MODULE_PATHNAME@', 'ts_int32_bucket' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION @extschema@.time_bucket(bucket_width BIGINT, ts BIGINT, "offset" BIGINT) RETURNS BIGINT AS '@MODULE_PATHNAME@', 'ts_int64_bucket' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; -- This will align a range to a bucket size. It is similar to -- time_bucket(), but takes a range and produces a range that starts -- and ends at bucket boundaries. CREATE OR REPLACE FUNCTION _timescaledb_functions.align_to_bucket(width interval, rng anyrange) RETURNS anyrange AS $body$ BEGIN RETURN _timescaledb_functions.make_range_from_internal_time( rng, @extschema@.time_bucket(width, lower(rng)), @extschema@.time_bucket(width, upper(rng) - '1 microsecond'::interval) + width ); END $body$ LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE SET search_path TO pg_catalog, pg_temp; ================================================ FILE: sql/updates/2.10.0--2.9.3.sql ================================================ GRANT ALL ON _timescaledb_internal.job_errors TO PUBLIC; ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.job_errors; DROP VIEW timescaledb_information.job_errors; ================================================ FILE: sql/updates/2.10.1--2.10.0.sql ================================================ ================================================ FILE: sql/updates/2.10.1--2.10.2.sql ================================================ DROP FUNCTION _timescaledb_internal.ping_data_node(NAME); -- We only create stub here to not introduce shared library dependencies in update chains -- the proper function definition will be created at end of update script when all functions -- are recreated. CREATE OR REPLACE FUNCTION _timescaledb_internal.ping_data_node(node_name NAME, timeout INTERVAL = NULL) RETURNS BOOLEAN AS $$SELECT false;$$ LANGUAGE SQL VOLATILE; -- drop dependent views DROP VIEW IF EXISTS timescaledb_information.job_errors; DROP VIEW IF EXISTS timescaledb_information.job_stats; DROP VIEW IF EXISTS timescaledb_information.jobs; DROP VIEW IF EXISTS timescaledb_experimental.policies; ALTER TABLE _timescaledb_config.bgw_job ALTER COLUMN owner DROP DEFAULT, ALTER COLUMN owner TYPE regrole USING pg_catalog.quote_ident(owner)::regrole, ALTER COLUMN owner SET DEFAULT pg_catalog.quote_ident(current_role)::regrole; CREATE TABLE _timescaledb_config.bgw_job_tmp AS SELECT * FROM _timescaledb_config.bgw_job; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_config.bgw_job; ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_config.bgw_job_id_seq; ALTER TABLE _timescaledb_internal.bgw_job_stat DROP CONSTRAINT IF EXISTS bgw_job_stat_job_id_fkey; ALTER TABLE _timescaledb_internal.bgw_policy_chunk_stats DROP CONSTRAINT IF EXISTS bgw_policy_chunk_stats_job_id_fkey; CREATE TABLE _timescaledb_internal.tmp_bgw_job_seq_value AS SELECT last_value, is_called FROM _timescaledb_config.bgw_job_id_seq; DROP TABLE _timescaledb_config.bgw_job; CREATE SEQUENCE _timescaledb_config.bgw_job_id_seq MINVALUE 1000; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_config.bgw_job_id_seq', ''); SELECT pg_catalog.setval('_timescaledb_config.bgw_job_id_seq', last_value, is_called) FROM _timescaledb_internal.tmp_bgw_job_seq_value; DROP TABLE _timescaledb_internal.tmp_bgw_job_seq_value; CREATE TABLE _timescaledb_config.bgw_job ( id integer NOT NULL DEFAULT nextval('_timescaledb_config.bgw_job_id_seq'), application_name name NOT NULL, schedule_interval interval NOT NULL, max_runtime interval NOT NULL, max_retries integer NOT NULL, retry_period interval NOT NULL, proc_schema name NOT NULL, proc_name name NOT NULL, owner regrole NOT NULL DEFAULT pg_catalog.quote_ident(current_role)::regrole, scheduled bool NOT NULL DEFAULT TRUE, fixed_schedule bool not null default true, initial_start timestamptz, hypertable_id integer, config jsonb, check_schema name, check_name name, timezone text, CONSTRAINT bgw_job_pkey PRIMARY KEY (id), CONSTRAINT bgw_job_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE ); ALTER SEQUENCE _timescaledb_config.bgw_job_id_seq OWNED BY _timescaledb_config.bgw_job.id; CREATE INDEX bgw_job_proc_hypertable_id_idx ON _timescaledb_config.bgw_job(proc_schema,proc_name,hypertable_id); INSERT INTO _timescaledb_config.bgw_job SELECT * FROM _timescaledb_config.bgw_job_tmp ORDER BY id; DROP TABLE _timescaledb_config.bgw_job_tmp; ALTER TABLE _timescaledb_internal.bgw_job_stat ADD CONSTRAINT bgw_job_stat_job_id_fkey FOREIGN KEY(job_id) REFERENCES _timescaledb_config.bgw_job(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_internal.bgw_policy_chunk_stats ADD CONSTRAINT bgw_policy_chunk_stats_job_id_fkey FOREIGN KEY(job_id) REFERENCES _timescaledb_config.bgw_job(id) ON DELETE CASCADE; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_config.bgw_job', 'WHERE id >= 1000'); GRANT SELECT ON _timescaledb_config.bgw_job TO PUBLIC; GRANT SELECT ON _timescaledb_config.bgw_job_id_seq TO PUBLIC; ================================================ FILE: sql/updates/2.10.2--2.10.1.sql ================================================ DROP FUNCTION _timescaledb_internal.ping_data_node(NAME, INTERVAL); CREATE OR REPLACE FUNCTION _timescaledb_internal.ping_data_node(node_name NAME) RETURNS BOOLEAN AS '@MODULE_PATHNAME@', 'ts_data_node_ping' LANGUAGE C VOLATILE; -- drop dependent views DROP VIEW IF EXISTS timescaledb_information.job_errors; DROP VIEW IF EXISTS timescaledb_information.job_stats; DROP VIEW IF EXISTS timescaledb_information.jobs; DROP VIEW IF EXISTS timescaledb_experimental.policies; ALTER TABLE _timescaledb_config.bgw_job ALTER COLUMN owner DROP DEFAULT, ALTER COLUMN owner TYPE name USING owner::name, ALTER COLUMN owner SET DEFAULT current_role; CREATE TABLE _timescaledb_config.bgw_job_tmp AS SELECT * FROM _timescaledb_config.bgw_job; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_config.bgw_job; ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_config.bgw_job_id_seq; ALTER TABLE _timescaledb_internal.bgw_job_stat DROP CONSTRAINT IF EXISTS bgw_job_stat_job_id_fkey; ALTER TABLE _timescaledb_internal.bgw_policy_chunk_stats DROP CONSTRAINT IF EXISTS bgw_policy_chunk_stats_job_id_fkey; CREATE TABLE _timescaledb_internal.tmp_bgw_job_seq_value AS SELECT last_value, is_called FROM _timescaledb_config.bgw_job_id_seq; DROP TABLE _timescaledb_config.bgw_job; CREATE SEQUENCE _timescaledb_config.bgw_job_id_seq MINVALUE 1000; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_config.bgw_job_id_seq', ''); SELECT pg_catalog.setval('_timescaledb_config.bgw_job_id_seq', last_value, is_called) FROM _timescaledb_internal.tmp_bgw_job_seq_value; DROP TABLE _timescaledb_internal.tmp_bgw_job_seq_value; CREATE TABLE _timescaledb_config.bgw_job ( id integer NOT NULL DEFAULT nextval('_timescaledb_config.bgw_job_id_seq'), application_name name NOT NULL, schedule_interval interval NOT NULL, max_runtime interval NOT NULL, max_retries integer NOT NULL, retry_period interval NOT NULL, proc_schema name NOT NULL, proc_name name NOT NULL, owner name NOT NULL DEFAULT current_role, scheduled bool NOT NULL DEFAULT TRUE, fixed_schedule bool not null default true, initial_start timestamptz, hypertable_id integer, config jsonb, check_schema name, check_name name, timezone text, CONSTRAINT bgw_job_pkey PRIMARY KEY (id), CONSTRAINT bgw_job_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE ); ALTER SEQUENCE _timescaledb_config.bgw_job_id_seq OWNED BY _timescaledb_config.bgw_job.id; CREATE INDEX bgw_job_proc_hypertable_id_idx ON _timescaledb_config.bgw_job(proc_schema,proc_name,hypertable_id); INSERT INTO _timescaledb_config.bgw_job SELECT * FROM _timescaledb_config.bgw_job_tmp ORDER BY id; DROP TABLE _timescaledb_config.bgw_job_tmp; ALTER TABLE _timescaledb_internal.bgw_job_stat ADD CONSTRAINT bgw_job_stat_job_id_fkey FOREIGN KEY(job_id) REFERENCES _timescaledb_config.bgw_job(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_internal.bgw_policy_chunk_stats ADD CONSTRAINT bgw_policy_chunk_stats_job_id_fkey FOREIGN KEY(job_id) REFERENCES _timescaledb_config.bgw_job(id) ON DELETE CASCADE; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_config.bgw_job', 'WHERE id >= 1000'); GRANT SELECT ON _timescaledb_config.bgw_job TO PUBLIC; GRANT SELECT ON _timescaledb_config.bgw_job_id_seq TO PUBLIC; ================================================ FILE: sql/updates/2.10.2--2.10.3.sql ================================================ ================================================ FILE: sql/updates/2.10.3--2.10.2.sql ================================================ ================================================ FILE: sql/updates/2.10.3--2.11.0.sql ================================================ CREATE TABLE _timescaledb_catalog.continuous_aggs_watermark ( mat_hypertable_id integer NOT NULL, watermark bigint NOT NULL, -- table constraints CONSTRAINT continuous_aggs_watermark_pkey PRIMARY KEY (mat_hypertable_id), CONSTRAINT continuous_aggs_watermark_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.continuous_agg (mat_hypertable_id) ON DELETE CASCADE ); GRANT SELECT ON _timescaledb_catalog.continuous_aggs_watermark TO PUBLIC; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_aggs_watermark', ''); CREATE FUNCTION _timescaledb_internal.cagg_watermark_materialized(hypertable_id INTEGER) RETURNS INT8 AS '@MODULE_PATHNAME@', 'ts_continuous_agg_watermark_materialized' LANGUAGE C STABLE STRICT PARALLEL SAFE; CREATE FUNCTION _timescaledb_internal.recompress_chunk_segmentwise(REGCLASS, BOOLEAN) RETURNS REGCLASS AS '@MODULE_PATHNAME@', 'ts_recompress_chunk_segmentwise' LANGUAGE C STRICT VOLATILE; CREATE FUNCTION _timescaledb_internal.get_compressed_chunk_index_for_recompression(REGCLASS) RETURNS REGCLASS AS '@MODULE_PATHNAME@', 'ts_get_compressed_chunk_index_for_recompression' LANGUAGE C STRICT VOLATILE; DROP FUNCTION _timescaledb_internal.dimension_is_finite; DROP FUNCTION _timescaledb_internal.dimension_slice_get_constraint_sql; CREATE SCHEMA _timescaledb_functions; GRANT USAGE ON SCHEMA _timescaledb_functions TO PUBLIC; -- migrate histogram support functions into _timescaledb_functions schema ALTER FUNCTION _timescaledb_internal.hist_sfunc (state INTERNAL, val DOUBLE PRECISION, MIN DOUBLE PRECISION, MAX DOUBLE PRECISION, nbuckets INTEGER) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.hist_combinefunc(state1 INTERNAL, state2 INTERNAL) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.hist_serializefunc(INTERNAL) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.hist_deserializefunc(bytea, INTERNAL) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.hist_finalfunc(state INTERNAL, val DOUBLE PRECISION, MIN DOUBLE PRECISION, MAX DOUBLE PRECISION, nbuckets INTEGER) SET SCHEMA _timescaledb_functions; -- migrate first/last support functions into _timescaledb_functions schema ALTER FUNCTION _timescaledb_internal.first_sfunc(internal, anyelement, "any") SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.first_combinefunc(internal, internal) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.last_sfunc(internal, anyelement, "any") SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.last_combinefunc(internal, internal) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.bookend_finalfunc(internal, anyelement, "any") SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.bookend_serializefunc(internal) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.bookend_deserializefunc(bytea, internal) SET SCHEMA _timescaledb_functions; DROP FUNCTION IF EXISTS _timescaledb_internal.is_main_table(regclass); DROP FUNCTION IF EXISTS _timescaledb_internal.is_main_table(name, name); DROP FUNCTION IF EXISTS _timescaledb_internal.hypertable_from_main_table(regclass); DROP FUNCTION IF EXISTS _timescaledb_internal.main_table_from_hypertable(integer); DROP FUNCTION IF EXISTS _timescaledb_internal.time_literal_sql(bigint, regtype); ALTER FUNCTION _timescaledb_internal.compressed_data_in(CSTRING) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.compressed_data_out(_timescaledb_internal.compressed_data) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.compressed_data_send(_timescaledb_internal.compressed_data) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.compressed_data_recv(internal) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.rxid_in(cstring) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.rxid_out(@extschema@.rxid) SET SCHEMA _timescaledb_functions; ALTER TABLE _timescaledb_config.bgw_job ALTER COLUMN owner SET DEFAULT pg_catalog.quote_ident(current_role)::regrole; ALTER TABLE _timescaledb_catalog.continuous_agg_migrate_plan ADD COLUMN user_view_definition TEXT, DROP CONSTRAINT continuous_agg_migrate_plan_mat_hypertable_id_fkey; -- Log with events that will be sent out with the telemetry. The log -- will be flushed after it has been sent out. We do not save it to -- backups since it should not contain important data. CREATE TABLE _timescaledb_catalog.telemetry_event ( created timestamptz NOT NULL DEFAULT current_timestamp, tag name NOT NULL, body jsonb NOT NULL ); GRANT SELECT ON _timescaledb_catalog.telemetry_event TO PUBLIC; ================================================ FILE: sql/updates/2.11.0--2.10.3.sql ================================================ DROP FUNCTION IF EXISTS _timescaledb_internal.get_approx_row_count(REGCLASS); ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_aggs_watermark; DROP TABLE IF EXISTS _timescaledb_catalog.continuous_aggs_watermark; DROP FUNCTION IF EXISTS _timescaledb_internal.cagg_watermark_materialized(hypertable_id INTEGER); DROP FUNCTION _timescaledb_internal.recompress_chunk_segmentwise(REGCLASS, BOOLEAN); DROP FUNCTION _timescaledb_internal.get_compressed_chunk_index_for_recompression(REGCLASS); CREATE OR REPLACE FUNCTION _timescaledb_internal.dimension_is_finite( val BIGINT ) RETURNS BOOLEAN LANGUAGE SQL IMMUTABLE PARALLEL SAFE AS $BODY$ --end values of bigint reserved for infinite SELECT val > (-9223372036854775808)::bigint AND val < 9223372036854775807::bigint $BODY$ SET search_path TO pg_catalog, pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_internal.dimension_slice_get_constraint_sql( dimension_slice_id INTEGER ) RETURNS TEXT LANGUAGE PLPGSQL VOLATILE AS $BODY$ DECLARE dimension_slice_row _timescaledb_catalog.dimension_slice; dimension_row _timescaledb_catalog.dimension; dimension_def TEXT; dimtype REGTYPE; parts TEXT[]; BEGIN SELECT * INTO STRICT dimension_slice_row FROM _timescaledb_catalog.dimension_slice WHERE id = dimension_slice_id; SELECT * INTO STRICT dimension_row FROM _timescaledb_catalog.dimension WHERE id = dimension_slice_row.dimension_id; IF dimension_row.partitioning_func_schema IS NOT NULL AND dimension_row.partitioning_func IS NOT NULL THEN SELECT prorettype INTO STRICT dimtype FROM pg_catalog.pg_proc pro WHERE pro.oid = format('%I.%I', dimension_row.partitioning_func_schema, dimension_row.partitioning_func)::regproc::oid; dimension_def := format('%1$I.%2$I(%3$I)', dimension_row.partitioning_func_schema, dimension_row.partitioning_func, dimension_row.column_name); ELSE dimension_def := format('%1$I', dimension_row.column_name); dimtype := dimension_row.column_type; END IF; IF dimension_row.num_slices IS NOT NULL THEN IF _timescaledb_internal.dimension_is_finite(dimension_slice_row.range_start) THEN parts = parts || format(' %1$s >= %2$L ', dimension_def, dimension_slice_row.range_start); END IF; IF _timescaledb_internal.dimension_is_finite(dimension_slice_row.range_end) THEN parts = parts || format(' %1$s < %2$L ', dimension_def, dimension_slice_row.range_end); END IF; IF array_length(parts, 1) = 0 THEN RETURN NULL; END IF; return array_to_string(parts, 'AND'); ELSE -- only works with time for now IF _timescaledb_internal.time_literal_sql(dimension_slice_row.range_start, dimtype) = _timescaledb_internal.time_literal_sql(dimension_slice_row.range_end, dimtype) THEN RAISE 'time-based constraints have the same start and end values for column "%": %', dimension_row.column_name, _timescaledb_internal.time_literal_sql(dimension_slice_row.range_end, dimtype); END IF; parts = ARRAY[]::text[]; IF _timescaledb_internal.dimension_is_finite(dimension_slice_row.range_start) THEN parts = parts || format(' %1$s >= %2$s ', dimension_def, _timescaledb_internal.time_literal_sql(dimension_slice_row.range_start, dimtype)); END IF; IF _timescaledb_internal.dimension_is_finite(dimension_slice_row.range_end) THEN parts = parts || format(' %1$s < %2$s ', dimension_def, _timescaledb_internal.time_literal_sql(dimension_slice_row.range_end, dimtype)); END IF; return array_to_string(parts, 'AND'); END IF; END $BODY$ SET search_path TO pg_catalog, pg_temp; ALTER FUNCTION _timescaledb_functions.hist_sfunc (state INTERNAL, val DOUBLE PRECISION, MIN DOUBLE PRECISION, MAX DOUBLE PRECISION, nbuckets INTEGER) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.hist_combinefunc(state1 INTERNAL, state2 INTERNAL) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.hist_serializefunc(INTERNAL) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.hist_deserializefunc(bytea, INTERNAL) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.hist_finalfunc(state INTERNAL, val DOUBLE PRECISION, MIN DOUBLE PRECISION, MAX DOUBLE PRECISION, nbuckets INTEGER) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.first_sfunc(internal, anyelement, "any") SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.first_combinefunc(internal, internal) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.last_sfunc(internal, anyelement, "any") SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.last_combinefunc(internal, internal) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.bookend_finalfunc(internal, anyelement, "any") SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.bookend_serializefunc(internal) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.bookend_deserializefunc(bytea, internal) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.compressed_data_in(CSTRING) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.compressed_data_out(_timescaledb_internal.compressed_data) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.compressed_data_send(_timescaledb_internal.compressed_data) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.compressed_data_recv(internal) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.rxid_in(cstring) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.rxid_out(@extschema@.rxid) SET SCHEMA _timescaledb_internal; DROP SCHEMA _timescaledb_functions; CREATE FUNCTION _timescaledb_internal.is_main_table( table_oid regclass ) RETURNS bool LANGUAGE SQL STABLE AS $BODY$ SELECT EXISTS(SELECT 1 FROM _timescaledb_catalog.hypertable WHERE table_name = relname AND schema_name = nspname) FROM pg_class c INNER JOIN pg_namespace n ON (n.OID = c.relnamespace) WHERE c.OID = table_oid; $BODY$ SET search_path TO pg_catalog, pg_temp; -- Check if given table is a hypertable's main table CREATE FUNCTION _timescaledb_internal.is_main_table( schema_name NAME, table_name NAME ) RETURNS BOOLEAN LANGUAGE SQL STABLE AS $BODY$ SELECT EXISTS( SELECT 1 FROM _timescaledb_catalog.hypertable h WHERE h.schema_name = is_main_table.schema_name AND h.table_name = is_main_table.table_name ); $BODY$ SET search_path TO pg_catalog, pg_temp; -- Get a hypertable given its main table OID CREATE FUNCTION _timescaledb_internal.hypertable_from_main_table( table_oid regclass ) RETURNS _timescaledb_catalog.hypertable LANGUAGE SQL STABLE AS $BODY$ SELECT h.* FROM pg_class c INNER JOIN pg_namespace n ON (n.OID = c.relnamespace) INNER JOIN _timescaledb_catalog.hypertable h ON (h.table_name = c.relname AND h.schema_name = n.nspname) WHERE c.OID = table_oid; $BODY$ SET search_path TO pg_catalog, pg_temp; CREATE FUNCTION _timescaledb_internal.main_table_from_hypertable( hypertable_id int ) RETURNS regclass LANGUAGE SQL STABLE AS $BODY$ SELECT format('%I.%I',h.schema_name, h.table_name)::regclass FROM _timescaledb_catalog.hypertable h WHERE id = hypertable_id; $BODY$ SET search_path TO pg_catalog, pg_temp; -- Gets the sql code for representing the literal for the given time value (in the internal representation) as the column_type. CREATE FUNCTION _timescaledb_internal.time_literal_sql( time_value BIGINT, column_type REGTYPE ) RETURNS text LANGUAGE PLPGSQL STABLE AS $BODY$ DECLARE ret text; BEGIN IF time_value IS NULL THEN RETURN format('%L', NULL); END IF; CASE column_type WHEN 'BIGINT'::regtype, 'INTEGER'::regtype, 'SMALLINT'::regtype THEN RETURN format('%L', time_value); -- scale determined by user. WHEN 'TIMESTAMP'::regtype THEN --the time_value for timestamps w/o tz does not depend on local timezones. So perform at UTC. RETURN format('TIMESTAMP %1$L', timezone('UTC',_timescaledb_internal.to_timestamp(time_value))); -- microseconds WHEN 'TIMESTAMPTZ'::regtype THEN -- assume time_value is in microsec RETURN format('TIMESTAMPTZ %1$L', _timescaledb_internal.to_timestamp(time_value)); -- microseconds WHEN 'DATE'::regtype THEN RETURN format('%L', timezone('UTC',_timescaledb_internal.to_timestamp(time_value))::date); ELSE EXECUTE 'SELECT format(''%L'', $1::' || column_type::text || ')' into ret using time_value; RETURN ret; END CASE; END $BODY$ SET search_path TO pg_catalog, pg_temp; ALTER TABLE _timescaledb_config.bgw_job ALTER COLUMN owner SET DEFAULT pg_catalog.quote_ident(current_role)::regrole; -- Rebuild the _timescaledb_catalog.continuous_agg_migrate_plan_step definition ALTER TABLE _timescaledb_catalog.continuous_agg_migrate_plan_step DROP CONSTRAINT continuous_agg_migrate_plan_step_mat_hypertable_id_fkey; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_agg_migrate_plan; CREATE TABLE _timescaledb_catalog._tmp_continuous_agg_migrate_plan AS SELECT mat_hypertable_id, start_ts, end_ts FROM _timescaledb_catalog.continuous_agg_migrate_plan; DROP TABLE _timescaledb_catalog.continuous_agg_migrate_plan; CREATE TABLE _timescaledb_catalog.continuous_agg_migrate_plan ( mat_hypertable_id integer NOT NULL, start_ts TIMESTAMPTZ NOT NULL DEFAULT pg_catalog.now(), end_ts TIMESTAMPTZ, -- table constraints CONSTRAINT continuous_agg_migrate_plan_pkey PRIMARY KEY (mat_hypertable_id), CONSTRAINT continuous_agg_migrate_plan_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.continuous_agg (mat_hypertable_id) ); INSERT INTO _timescaledb_catalog.continuous_agg_migrate_plan SELECT * FROM _timescaledb_catalog._tmp_continuous_agg_migrate_plan; DROP TABLE _timescaledb_catalog._tmp_continuous_agg_migrate_plan; ALTER TABLE _timescaledb_catalog.continuous_agg_migrate_plan_step ADD CONSTRAINT continuous_agg_migrate_plan_step_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.continuous_agg_migrate_plan (mat_hypertable_id) ON DELETE CASCADE; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_agg_migrate_plan', ''); GRANT SELECT ON TABLE _timescaledb_catalog.continuous_agg_migrate_plan TO PUBLIC; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.telemetry_event; DROP TABLE IF EXISTS _timescaledb_catalog.telemetry_event; ================================================ FILE: sql/updates/2.11.0--2.11.1.sql ================================================ ================================================ FILE: sql/updates/2.11.1--2.11.0.sql ================================================ ================================================ FILE: sql/updates/2.11.1--2.11.2.sql ================================================ ================================================ FILE: sql/updates/2.11.2--2.11.1.sql ================================================ ================================================ FILE: sql/updates/2.11.2--2.12.0.sql ================================================ DROP FUNCTION IF EXISTS @extschema@.alter_job( INTEGER, INTERVAL, INTERVAL, INTEGER, INTERVAL, BOOL, JSONB, TIMESTAMPTZ, BOOL, REGPROC ); CREATE FUNCTION @extschema@.alter_job( job_id INTEGER, schedule_interval INTERVAL = NULL, max_runtime INTERVAL = NULL, max_retries INTEGER = NULL, retry_period INTERVAL = NULL, scheduled BOOL = NULL, config JSONB = NULL, next_start TIMESTAMPTZ = NULL, if_exists BOOL = FALSE, check_config REGPROC = NULL, fixed_schedule BOOL = NULL, initial_start TIMESTAMPTZ = NULL, timezone TEXT DEFAULT NULL ) RETURNS TABLE (job_id INTEGER, schedule_interval INTERVAL, max_runtime INTERVAL, max_retries INTEGER, retry_period INTERVAL, scheduled BOOL, config JSONB, next_start TIMESTAMPTZ, check_config TEXT, fixed_schedule BOOL, initial_start TIMESTAMPTZ, timezone TEXT) AS '@MODULE_PATHNAME@', 'ts_job_alter' LANGUAGE C VOLATILE; -- when upgrading from old versions on PG13 this function might not be present -- since there is no ALTER FUNCTION IF EXISTS we have to work around it with a DO block DO $$ DECLARE foid regprocedure; funcs text[] = '{ drop_dist_ht_invalidation_trigger, subtract_integer_from_now, get_approx_row_count, chunk_status, create_chunk,create_chunk_table, freeze_chunk,unfreeze_chunk,drop_chunk, attach_osm_table_chunk }'; BEGIN FOR foid IN SELECT oid FROM pg_proc WHERE proname = ANY(funcs) AND pronamespace = '_timescaledb_internal'::regnamespace LOOP EXECUTE format('ALTER FUNCTION %s SET SCHEMA _timescaledb_functions', foid); END LOOP; END; $$; DROP FUNCTION IF EXISTS _timescaledb_internal.get_time_type(integer); ALTER FUNCTION _timescaledb_internal.insert_blocker() SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.continuous_agg_invalidation_trigger() SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.get_create_command(name) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.to_unix_microseconds(timestamptz) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.to_timestamp(bigint) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.to_timestamp_without_timezone(bigint) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.to_date(bigint) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.to_interval(bigint) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.interval_to_usec(interval) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.time_to_internal(anyelement) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.set_dist_id(uuid) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.set_peer_dist_id(uuid) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.validate_as_data_node() SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.show_connection_cache() SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.ping_data_node(name, interval) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.remote_txn_heal_data_node(oid) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.relation_size(regclass) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.data_node_hypertable_info(name, name, name) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.data_node_chunk_info(name, name, name) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.hypertable_local_size(name, name) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.hypertable_remote_size(name, name) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.chunks_local_size(name, name) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.chunks_remote_size(name, name) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.range_value_to_pretty(bigint, regtype) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.data_node_compressed_chunk_stats(name, name, name) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.compressed_chunk_local_stats(name, name) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.compressed_chunk_remote_stats(name, name) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.indexes_local_size(name, name) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.data_node_index_size(name, name, name) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.indexes_remote_size(name, name, name) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.generate_uuid() SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.get_git_commit() SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.get_os_info() SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.tsl_loaded() SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.calculate_chunk_interval(int, bigint, bigint) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.chunks_in(record, integer[]) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.chunk_id_from_relid(oid) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.show_chunk(regclass) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.get_chunk_relstats(regclass) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.get_chunk_colstats(regclass) SET SCHEMA _timescaledb_functions; UPDATE _timescaledb_catalog.hypertable SET chunk_sizing_func_schema = '_timescaledb_functions' WHERE chunk_sizing_func_schema = '_timescaledb_internal' AND chunk_sizing_func_name = 'calculate_chunk_interval'; DO $$ DECLARE foid regprocedure; kind text; funcs text[] = '{ policy_compression_check,policy_compression_execute,policy_compression, policy_job_error_retention_check,policy_job_error_retention, policy_recompression, policy_refresh_continuous_aggregate_check,policy_refresh_continuous_aggregate, policy_reorder_check,policy_reorder,policy_retention_check,policy_retention, cagg_watermark, cagg_watermark_materialized, cagg_migrate_plan_exists, cagg_migrate_pre_validation, cagg_migrate_create_plan, cagg_migrate_execute_create_new_cagg, cagg_migrate_execute_disable_policies, cagg_migrate_execute_enable_policies, cagg_migrate_execute_copy_policies, cagg_migrate_execute_refresh_new_cagg, cagg_migrate_execute_copy_data, cagg_migrate_execute_override_cagg, cagg_migrate_execute_drop_old_cagg, cagg_migrate_execute_plan, finalize_agg, hypertable_invalidation_log_delete, invalidation_cagg_log_add_entry, invalidation_hyper_log_add_entry, invalidation_process_cagg_log, invalidation_process_hypertable_log, materialization_invalidation_log_delete, alter_job_set_hypertable_id, set_chunk_default_data_node, create_compressed_chunk, get_compressed_chunk_index_for_recompression, recompress_chunk_segmentwise, chunk_drop_replica, chunk_index_clone, chunk_index_replace, create_chunk_replica_table, drop_stale_chunks, chunk_constraint_add_table_constraint, hypertable_constraint_add_table_fk_constraint, health, wait_subscription_sync }'; BEGIN FOR foid, kind IN SELECT oid, CASE WHEN prokind = 'f' THEN 'FUNCTION' WHEN prokind = 'a' THEN 'AGGREGATE' ELSE 'PROCEDURE' END FROM pg_proc WHERE proname = ANY(funcs) AND pronamespace = '_timescaledb_internal'::regnamespace LOOP EXECUTE format('ALTER %s %s SET SCHEMA _timescaledb_functions', kind, foid); END LOOP; END; $$; UPDATE _timescaledb_config.bgw_job SET proc_schema = '_timescaledb_functions' WHERE proc_schema = '_timescaledb_internal'; UPDATE _timescaledb_config.bgw_job SET check_schema = '_timescaledb_functions' WHERE check_schema = '_timescaledb_internal'; ALTER FUNCTION _timescaledb_internal.start_background_workers() SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.stop_background_workers() SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.restart_background_workers() SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.process_ddl_event() SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.get_partition_for_key(val anyelement) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.get_partition_hash(val anyelement) SET SCHEMA _timescaledb_functions; UPDATE _timescaledb_catalog.dimension SET partitioning_func_schema = '_timescaledb_functions' WHERE partitioning_func_schema = '_timescaledb_internal' AND partitioning_func IN ('get_partition_for_key','get_partition_hash'); ALTER FUNCTION _timescaledb_internal.finalize_agg_ffunc(internal,text,name,name,name[],bytea,anyelement) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.finalize_agg_sfunc(internal,text,name,name,name[],bytea,anyelement) SET SCHEMA _timescaledb_functions; ALTER FUNCTION _timescaledb_internal.partialize_agg(anyelement) SET SCHEMA _timescaledb_functions; -- Fix osm chunk ranges UPDATE _timescaledb_catalog.dimension_slice ds SET range_start = 9223372036854775806 FROM _timescaledb_catalog.chunk_constraint cc INNER JOIN _timescaledb_catalog.chunk c ON c.id = cc.chunk_id AND c.osm_chunk WHERE cc.dimension_slice_id = ds.id AND ds.range_start <> 9223372036854775806; -- OSM support - table must be rebuilt to ensure consistent attribute numbers -- we cannot just ALTER TABLE .. ADD COLUMN ALTER TABLE _timescaledb_config.bgw_job DROP CONSTRAINT bgw_job_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.chunk DROP CONSTRAINT chunk_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.chunk_index DROP CONSTRAINT chunk_index_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.continuous_agg DROP CONSTRAINT continuous_agg_mat_hypertable_id_fkey, DROP CONSTRAINT continuous_agg_raw_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.continuous_aggs_bucket_function DROP CONSTRAINT continuous_aggs_bucket_function_mat_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.continuous_aggs_invalidation_threshold DROP CONSTRAINT continuous_aggs_invalidation_threshold_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.dimension DROP CONSTRAINT dimension_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.hypertable DROP CONSTRAINT hypertable_compressed_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.hypertable_compression DROP CONSTRAINT hypertable_compression_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.hypertable_data_node DROP CONSTRAINT hypertable_data_node_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.tablespace DROP CONSTRAINT tablespace_hypertable_id_fkey; DROP VIEW IF EXISTS timescaledb_information.hypertables; DROP VIEW IF EXISTS timescaledb_information.job_stats; DROP VIEW IF EXISTS timescaledb_information.jobs; DROP VIEW IF EXISTS timescaledb_information.continuous_aggregates; DROP VIEW IF EXISTS timescaledb_information.chunks; DROP VIEW IF EXISTS timescaledb_information.dimensions; DROP VIEW IF EXISTS timescaledb_information.compression_settings; DROP VIEW IF EXISTS _timescaledb_internal.hypertable_chunk_local_size; DROP VIEW IF EXISTS _timescaledb_internal.compressed_chunk_stats; DROP VIEW IF EXISTS timescaledb_experimental.chunk_replication_status; DROP VIEW IF EXISTS timescaledb_experimental.policies; -- recreate table CREATE TABLE _timescaledb_catalog.hypertable_tmp AS SELECT * FROM _timescaledb_catalog.hypertable; CREATE TABLE _timescaledb_catalog.tmp_hypertable_seq_value AS SELECT last_value, is_called FROM _timescaledb_catalog.hypertable_id_seq; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.hypertable; ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_catalog.hypertable_id_seq; SET timescaledb.restoring = on; -- must disable the hooks otherwise we can't do anything without the table _timescaledb_catalog.hypertable DROP TABLE _timescaledb_catalog.hypertable; CREATE SEQUENCE _timescaledb_catalog.hypertable_id_seq MINVALUE 1; SELECT setval('_timescaledb_catalog.hypertable_id_seq', last_value, is_called) FROM _timescaledb_catalog.tmp_hypertable_seq_value; DROP TABLE _timescaledb_catalog.tmp_hypertable_seq_value; CREATE TABLE _timescaledb_catalog.hypertable ( id INTEGER PRIMARY KEY NOT NULL DEFAULT nextval('_timescaledb_catalog.hypertable_id_seq'), schema_name name NOT NULL, table_name name NOT NULL, associated_schema_name name NOT NULL, associated_table_prefix name NOT NULL, num_dimensions smallint NOT NULL, chunk_sizing_func_schema name NOT NULL, chunk_sizing_func_name name NOT NULL, chunk_target_size bigint NOT NULL, -- size in bytes compression_state smallint NOT NULL DEFAULT 0, compressed_hypertable_id integer, replication_factor smallint NULL, status integer NOT NULL DEFAULT 0 ); SET timescaledb.restoring = off; INSERT INTO _timescaledb_catalog.hypertable ( id, schema_name, table_name, associated_schema_name, associated_table_prefix, num_dimensions, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size, compression_state, compressed_hypertable_id, replication_factor ) SELECT id, schema_name, table_name, associated_schema_name, associated_table_prefix, num_dimensions, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size, compression_state, compressed_hypertable_id, replication_factor FROM _timescaledb_catalog.hypertable_tmp ORDER BY id; UPDATE _timescaledb_catalog.hypertable h SET status = 3 WHERE EXISTS ( SELECT FROM _timescaledb_catalog.chunk c WHERE c.osm_chunk AND c.hypertable_id = h.id ); ALTER SEQUENCE _timescaledb_catalog.hypertable_id_seq OWNED BY _timescaledb_catalog.hypertable.id; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable', 'WHERE id >= 1'); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable_id_seq', ''); GRANT SELECT ON _timescaledb_catalog.hypertable TO PUBLIC; GRANT SELECT ON _timescaledb_catalog.hypertable_id_seq TO PUBLIC; DROP TABLE _timescaledb_catalog.hypertable_tmp; -- now add any constraints ALTER TABLE _timescaledb_catalog.hypertable ADD CONSTRAINT hypertable_associated_schema_name_associated_table_prefix_key UNIQUE (associated_schema_name, associated_table_prefix), ADD CONSTRAINT hypertable_table_name_schema_name_key UNIQUE (table_name, schema_name), ADD CONSTRAINT hypertable_schema_name_check CHECK (schema_name != '_timescaledb_catalog'), ADD CONSTRAINT hypertable_dim_compress_check CHECK (num_dimensions > 0 OR compression_state = 2), ADD CONSTRAINT hypertable_chunk_target_size_check CHECK (chunk_target_size >= 0), ADD CONSTRAINT hypertable_compress_check CHECK ( (compression_state = 0 OR compression_state = 1 ) OR (compression_state = 2 AND compressed_hypertable_id IS NULL)), ADD CONSTRAINT hypertable_replication_factor_check CHECK (replication_factor > 0 OR replication_factor = -1), ADD CONSTRAINT hypertable_compressed_hypertable_id_fkey FOREIGN KEY (compressed_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id); GRANT SELECT ON TABLE _timescaledb_catalog.hypertable TO PUBLIC; -- 3. reestablish constraints on other tables ALTER TABLE _timescaledb_config.bgw_job ADD CONSTRAINT bgw_job_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.chunk ADD CONSTRAINT chunk_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id); ALTER TABLE _timescaledb_catalog.chunk_index ADD CONSTRAINT chunk_index_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.continuous_agg ADD CONSTRAINT continuous_agg_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE, ADD CONSTRAINT continuous_agg_raw_hypertable_id_fkey FOREIGN KEY (raw_hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.continuous_aggs_bucket_function ADD CONSTRAINT continuous_aggs_bucket_function_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.continuous_aggs_invalidation_threshold ADD CONSTRAINT continuous_aggs_invalidation_threshold_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.dimension ADD CONSTRAINT dimension_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.hypertable_compression ADD CONSTRAINT hypertable_compression_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.hypertable_data_node ADD CONSTRAINT hypertable_data_node_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id); ALTER TABLE _timescaledb_catalog.tablespace ADD CONSTRAINT tablespace_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ================================================ FILE: sql/updates/2.12.0--2.11.2.sql ================================================ -- remove compatibility wrapper functions -- this needs to happen before we move the actual functions back into _timescaledb_internal DROP FUNCTION _timescaledb_internal.alter_job_set_hypertable_id(integer,regclass); DROP FUNCTION _timescaledb_internal.attach_osm_table_chunk(regclass,regclass); DROP FUNCTION _timescaledb_internal.cagg_migrate_plan_exists(integer); DROP FUNCTION _timescaledb_internal.cagg_migrate_pre_validation(text,text,text); DROP FUNCTION _timescaledb_internal.cagg_watermark(integer); DROP FUNCTION _timescaledb_internal.cagg_watermark_materialized(integer); DROP FUNCTION _timescaledb_internal.calculate_chunk_interval(integer,bigint,bigint); DROP FUNCTION _timescaledb_internal.chunk_constraint_add_table_constraint(_timescaledb_catalog.chunk_constraint); DROP FUNCTION _timescaledb_internal.chunk_drop_replica(regclass,name); DROP FUNCTION _timescaledb_internal.chunk_id_from_relid(oid); DROP FUNCTION _timescaledb_internal.chunk_index_clone(oid); DROP FUNCTION _timescaledb_internal.chunk_index_replace(oid,oid); DROP FUNCTION _timescaledb_internal.chunk_status(regclass); DROP FUNCTION _timescaledb_internal.chunks_in(record,integer[]); DROP FUNCTION _timescaledb_internal.chunks_local_size(name,name); DROP FUNCTION _timescaledb_internal.chunks_remote_size(name,name); DROP FUNCTION _timescaledb_internal.compressed_chunk_local_stats(name,name); DROP FUNCTION _timescaledb_internal.compressed_chunk_remote_stats(name,name); DROP FUNCTION _timescaledb_internal.continuous_agg_invalidation_trigger(); DROP FUNCTION _timescaledb_internal.create_chunk(regclass,jsonb,name,name,regclass); DROP FUNCTION _timescaledb_internal.create_chunk_replica_table(regclass,name); DROP FUNCTION _timescaledb_internal.create_chunk_table(regclass,jsonb,name,name); DROP FUNCTION _timescaledb_internal.create_compressed_chunk(regclass,regclass,bigint,bigint,bigint,bigint,bigint,bigint,bigint,bigint); DROP FUNCTION _timescaledb_internal.data_node_chunk_info(name,name,name); DROP FUNCTION _timescaledb_internal.data_node_compressed_chunk_stats(name,name,name); DROP FUNCTION _timescaledb_internal.data_node_hypertable_info(name,name,name); DROP FUNCTION _timescaledb_internal.data_node_index_size(name,name,name); DROP FUNCTION _timescaledb_internal.drop_chunk(regclass); DROP FUNCTION _timescaledb_internal.drop_dist_ht_invalidation_trigger(integer); DROP FUNCTION _timescaledb_internal.drop_stale_chunks(name,integer[]); DROP AGGREGATE _timescaledb_internal.finalize_agg(agg_name TEXT, inner_agg_collation_schema NAME, inner_agg_collation_name NAME, inner_agg_input_types NAME[][], inner_agg_serialized_state BYTEA, return_type_dummy_val anyelement); DROP FUNCTION _timescaledb_internal.finalize_agg_ffunc(internal, text, name, name, name[][], bytea, anyelement); DROP FUNCTION _timescaledb_internal.finalize_agg_sfunc(internal, text, name, name, name[][], bytea, anyelement); DROP FUNCTION _timescaledb_internal.freeze_chunk(regclass); DROP FUNCTION _timescaledb_internal.generate_uuid(); DROP FUNCTION _timescaledb_internal.get_approx_row_count(regclass); DROP FUNCTION _timescaledb_internal.get_chunk_colstats(regclass); DROP FUNCTION _timescaledb_internal.get_chunk_relstats(regclass); DROP FUNCTION _timescaledb_internal.get_compressed_chunk_index_for_recompression(regclass); DROP FUNCTION _timescaledb_internal.get_create_command(name); DROP FUNCTION _timescaledb_internal.get_git_commit(); DROP FUNCTION _timescaledb_internal.get_os_info(); DROP FUNCTION _timescaledb_internal.get_partition_for_key(anyelement); DROP FUNCTION _timescaledb_internal.get_partition_hash(anyelement); DROP FUNCTION _timescaledb_internal.health(); DROP FUNCTION _timescaledb_internal.hypertable_constraint_add_table_fk_constraint(name,name,name,integer); DROP FUNCTION _timescaledb_internal.hypertable_invalidation_log_delete(integer); DROP FUNCTION _timescaledb_internal.hypertable_local_size(name,name); DROP FUNCTION _timescaledb_internal.hypertable_remote_size(name,name); DROP FUNCTION _timescaledb_internal.indexes_local_size(name,name); DROP FUNCTION _timescaledb_internal.indexes_remote_size(name,name,name); DROP FUNCTION _timescaledb_internal.insert_blocker(); DROP FUNCTION _timescaledb_internal.interval_to_usec(interval); DROP FUNCTION _timescaledb_internal.invalidation_cagg_log_add_entry(integer,bigint,bigint); DROP FUNCTION _timescaledb_internal.invalidation_hyper_log_add_entry(integer,bigint,bigint); DROP FUNCTION _timescaledb_internal.invalidation_process_cagg_log(integer,integer,regtype,bigint,bigint,integer[],bigint[],bigint[]); DROP FUNCTION _timescaledb_internal.invalidation_process_cagg_log(integer,integer,regtype,bigint,bigint,integer[],bigint[],bigint[],text[]); DROP FUNCTION _timescaledb_internal.invalidation_process_hypertable_log(integer,integer,regtype,integer[],bigint[],bigint[]); DROP FUNCTION _timescaledb_internal.invalidation_process_hypertable_log(integer,integer,regtype,integer[],bigint[],bigint[],text[]); DROP FUNCTION _timescaledb_internal.materialization_invalidation_log_delete(integer); DROP FUNCTION _timescaledb_internal.partialize_agg(anyelement); DROP FUNCTION _timescaledb_internal.ping_data_node(name,interval); DROP FUNCTION _timescaledb_internal.policy_compression_check(jsonb); DROP FUNCTION _timescaledb_internal.policy_job_error_retention(integer,jsonb); DROP FUNCTION _timescaledb_internal.policy_job_error_retention_check(jsonb); DROP FUNCTION _timescaledb_internal.policy_refresh_continuous_aggregate_check(jsonb); DROP FUNCTION _timescaledb_internal.policy_reorder_check(jsonb); DROP FUNCTION _timescaledb_internal.policy_retention_check(jsonb); DROP FUNCTION _timescaledb_internal.process_ddl_event(); DROP FUNCTION _timescaledb_internal.range_value_to_pretty(bigint,regtype); DROP FUNCTION _timescaledb_internal.recompress_chunk_segmentwise(regclass,boolean); DROP FUNCTION _timescaledb_internal.relation_size(regclass); DROP FUNCTION _timescaledb_internal.remote_txn_heal_data_node(oid); DROP FUNCTION _timescaledb_internal.set_chunk_default_data_node(regclass,name); DROP FUNCTION _timescaledb_internal.set_dist_id(uuid); DROP FUNCTION _timescaledb_internal.set_peer_dist_id(uuid); DROP FUNCTION _timescaledb_internal.show_chunk(regclass); DROP FUNCTION _timescaledb_internal.show_connection_cache(); DROP FUNCTION _timescaledb_internal.start_background_workers(); DROP FUNCTION _timescaledb_internal.stop_background_workers(); DROP FUNCTION _timescaledb_internal.subtract_integer_from_now(regclass,bigint); DROP FUNCTION _timescaledb_internal.time_to_internal(anyelement); DROP FUNCTION _timescaledb_internal.to_date(bigint); DROP FUNCTION _timescaledb_internal.to_interval(bigint); DROP FUNCTION _timescaledb_internal.to_timestamp(bigint); DROP FUNCTION _timescaledb_internal.to_timestamp_without_timezone(bigint); DROP FUNCTION _timescaledb_internal.to_unix_microseconds(timestamp with time zone); DROP FUNCTION _timescaledb_internal.tsl_loaded(); DROP FUNCTION _timescaledb_internal.unfreeze_chunk(regclass); DROP FUNCTION _timescaledb_internal.validate_as_data_node(); DROP PROCEDURE _timescaledb_internal.cagg_migrate_create_plan(_timescaledb_catalog.continuous_agg,text,boolean,boolean); DROP PROCEDURE _timescaledb_internal.cagg_migrate_execute_copy_data(_timescaledb_catalog.continuous_agg,_timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE _timescaledb_internal.cagg_migrate_execute_copy_policies(_timescaledb_catalog.continuous_agg,_timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE _timescaledb_internal.cagg_migrate_execute_create_new_cagg(_timescaledb_catalog.continuous_agg,_timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE _timescaledb_internal.cagg_migrate_execute_disable_policies(_timescaledb_catalog.continuous_agg,_timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE _timescaledb_internal.cagg_migrate_execute_drop_old_cagg(_timescaledb_catalog.continuous_agg,_timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE _timescaledb_internal.cagg_migrate_execute_enable_policies(_timescaledb_catalog.continuous_agg,_timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE _timescaledb_internal.cagg_migrate_execute_override_cagg(_timescaledb_catalog.continuous_agg,_timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE _timescaledb_internal.cagg_migrate_execute_plan(_timescaledb_catalog.continuous_agg); DROP PROCEDURE _timescaledb_internal.cagg_migrate_execute_refresh_new_cagg(_timescaledb_catalog.continuous_agg,_timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE _timescaledb_internal.policy_compression(integer,jsonb); DROP PROCEDURE _timescaledb_internal.policy_compression_execute(integer,integer,anyelement,integer,boolean,boolean); DROP PROCEDURE _timescaledb_internal.policy_recompression(integer,jsonb); DROP PROCEDURE _timescaledb_internal.policy_refresh_continuous_aggregate(integer,jsonb); DROP PROCEDURE _timescaledb_internal.policy_reorder(integer,jsonb); DROP PROCEDURE _timescaledb_internal.policy_retention(integer,jsonb); DROP PROCEDURE _timescaledb_internal.wait_subscription_sync(name,name,integer,numeric); DROP FUNCTION IF EXISTS @extschema@.alter_job( INTEGER, INTERVAL, INTERVAL, INTEGER, INTERVAL, BOOL, JSONB, TIMESTAMPTZ, BOOL, REGPROC, BOOL, TIMESTAMPTZ, TEXT ); CREATE FUNCTION @extschema@.alter_job( job_id INTEGER, schedule_interval INTERVAL = NULL, max_runtime INTERVAL = NULL, max_retries INTEGER = NULL, retry_period INTERVAL = NULL, scheduled BOOL = NULL, config JSONB = NULL, next_start TIMESTAMPTZ = NULL, if_exists BOOL = FALSE, check_config REGPROC = NULL ) RETURNS TABLE (job_id INTEGER, schedule_interval INTERVAL, max_runtime INTERVAL, max_retries INTEGER, retry_period INTERVAL, scheduled BOOL, config JSONB, next_start TIMESTAMPTZ, check_config TEXT) AS '@MODULE_PATHNAME@', 'ts_job_alter' LANGUAGE C VOLATILE; ALTER FUNCTION _timescaledb_functions.insert_blocker() SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.continuous_agg_invalidation_trigger() SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.drop_dist_ht_invalidation_trigger(integer) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.get_create_command(name) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.to_unix_microseconds(timestamptz) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.to_timestamp(bigint) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.to_timestamp_without_timezone(bigint) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.to_date(bigint) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.to_interval(bigint) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.interval_to_usec(interval) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.time_to_internal(anyelement) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.subtract_integer_from_now(regclass, bigint) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.set_dist_id(uuid) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.set_peer_dist_id(uuid) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.validate_as_data_node() SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.show_connection_cache() SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.ping_data_node(name, interval) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.remote_txn_heal_data_node(oid) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.relation_size(regclass) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.data_node_hypertable_info(name, name, name) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.data_node_chunk_info(name, name, name) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.hypertable_local_size(name, name) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.hypertable_remote_size(name, name) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.chunks_local_size(name, name) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.chunks_remote_size(name, name) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.range_value_to_pretty(bigint, regtype) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.get_approx_row_count(regclass) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.data_node_compressed_chunk_stats(name, name, name) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.compressed_chunk_local_stats(name, name) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.compressed_chunk_remote_stats(name, name) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.indexes_local_size(name, name) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.data_node_index_size(name, name, name) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.indexes_remote_size(name, name, name) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.generate_uuid() SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.get_git_commit() SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.get_os_info() SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.tsl_loaded() SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.calculate_chunk_interval(int, bigint, bigint) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.chunk_status(regclass) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.chunks_in(record, integer[]) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.chunk_id_from_relid(oid) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.show_chunk(regclass) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.create_chunk(regclass, jsonb, name, name, regclass) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.set_chunk_default_data_node(regclass, name) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.get_chunk_relstats(regclass) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.get_chunk_colstats(regclass) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.create_chunk_table(regclass, jsonb, name, name) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.freeze_chunk(regclass) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.unfreeze_chunk(regclass) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.drop_chunk(regclass) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.attach_osm_table_chunk(regclass, regclass) SET SCHEMA _timescaledb_internal; UPDATE _timescaledb_catalog.hypertable SET chunk_sizing_func_schema = '_timescaledb_internal' WHERE chunk_sizing_func_schema = '_timescaledb_functions' AND chunk_sizing_func_name = 'calculate_chunk_interval'; ALTER FUNCTION _timescaledb_functions.policy_compression_check(jsonb) SET SCHEMA _timescaledb_internal; ALTER PROCEDURE _timescaledb_functions.policy_compression_execute(integer,integer,anyelement,integer,boolean,boolean) SET SCHEMA _timescaledb_internal; ALTER PROCEDURE _timescaledb_functions.policy_compression(integer,jsonb) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.policy_job_error_retention_check(jsonb) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.policy_job_error_retention(integer,jsonb) SET SCHEMA _timescaledb_internal; ALTER PROCEDURE _timescaledb_functions.policy_recompression(integer,jsonb) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.policy_refresh_continuous_aggregate_check(jsonb) SET SCHEMA _timescaledb_internal; ALTER PROCEDURE _timescaledb_functions.policy_refresh_continuous_aggregate(integer,jsonb) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.policy_reorder_check(jsonb) SET SCHEMA _timescaledb_internal; ALTER PROCEDURE _timescaledb_functions.policy_reorder(integer,jsonb) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.policy_retention_check(jsonb) SET SCHEMA _timescaledb_internal; ALTER PROCEDURE _timescaledb_functions.policy_retention(integer,jsonb) SET SCHEMA _timescaledb_internal; UPDATE _timescaledb_config.bgw_job SET proc_schema = '_timescaledb_internal' WHERE proc_schema = '_timescaledb_functions'; UPDATE _timescaledb_config.bgw_job SET check_schema = '_timescaledb_internal' WHERE check_schema = '_timescaledb_functions'; ALTER FUNCTION _timescaledb_functions.cagg_migrate_plan_exists(INTEGER) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.cagg_migrate_pre_validation(TEXT, TEXT, TEXT) SET SCHEMA _timescaledb_internal; ALTER PROCEDURE _timescaledb_functions.cagg_migrate_create_plan(_timescaledb_catalog.continuous_agg, TEXT, BOOLEAN, BOOLEAN) SET SCHEMA _timescaledb_internal; ALTER PROCEDURE _timescaledb_functions.cagg_migrate_execute_create_new_cagg(_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step) SET SCHEMA _timescaledb_internal; ALTER PROCEDURE _timescaledb_functions.cagg_migrate_execute_disable_policies(_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step) SET SCHEMA _timescaledb_internal; ALTER PROCEDURE _timescaledb_functions.cagg_migrate_execute_enable_policies(_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step) SET SCHEMA _timescaledb_internal; ALTER PROCEDURE _timescaledb_functions.cagg_migrate_execute_copy_policies(_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step) SET SCHEMA _timescaledb_internal; ALTER PROCEDURE _timescaledb_functions.cagg_migrate_execute_refresh_new_cagg(_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step) SET SCHEMA _timescaledb_internal; ALTER PROCEDURE _timescaledb_functions.cagg_migrate_execute_copy_data(_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step) SET SCHEMA _timescaledb_internal; ALTER PROCEDURE _timescaledb_functions.cagg_migrate_execute_override_cagg(_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step) SET SCHEMA _timescaledb_internal; ALTER PROCEDURE _timescaledb_functions.cagg_migrate_execute_drop_old_cagg(_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step) SET SCHEMA _timescaledb_internal; ALTER PROCEDURE _timescaledb_functions.cagg_migrate_execute_plan(_timescaledb_catalog.continuous_agg) SET SCHEMA _timescaledb_internal; -- pre-update of previous version will have created an additional copy of restart_background_workers -- since restart_background_workers was handled differently from other functions in previous versions DROP FUNCTION _timescaledb_internal.restart_background_workers(); ALTER FUNCTION _timescaledb_functions.start_background_workers() SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.stop_background_workers() SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.restart_background_workers() SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.alter_job_set_hypertable_id(integer,regclass) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.cagg_watermark(integer) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.cagg_watermark_materialized(integer) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.hypertable_invalidation_log_delete(integer) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.invalidation_cagg_log_add_entry(integer,bigint,bigint) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.invalidation_hyper_log_add_entry(integer,bigint,bigint) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.invalidation_process_cagg_log(integer,integer,regtype,bigint,bigint,integer[],bigint[],bigint[]) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.invalidation_process_cagg_log(integer,integer,regtype,bigint,bigint,integer[],bigint[],bigint[],text[]) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.invalidation_process_hypertable_log(integer,integer,regtype,integer[],bigint[],bigint[]) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.invalidation_process_hypertable_log(integer,integer,regtype,integer[],bigint[],bigint[],text[]) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.materialization_invalidation_log_delete(integer) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.chunk_constraint_add_table_constraint(_timescaledb_catalog.chunk_constraint) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.chunk_drop_replica(regclass,name) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.chunk_index_clone(oid) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.chunk_index_replace(oid,oid) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.create_chunk_replica_table(regclass,name) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.create_compressed_chunk(regclass,regclass,bigint,bigint,bigint,bigint,bigint,bigint,bigint,bigint) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.drop_stale_chunks(name,integer[]) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.get_compressed_chunk_index_for_recompression(regclass) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.health() SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.hypertable_constraint_add_table_fk_constraint(name,name,name,integer) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.process_ddl_event() SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.recompress_chunk_segmentwise(regclass,boolean) SET SCHEMA _timescaledb_internal; ALTER PROCEDURE _timescaledb_functions.wait_subscription_sync(name,name,integer,numeric) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.get_partition_for_key(val anyelement) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.get_partition_hash(val anyelement) SET SCHEMA _timescaledb_internal; UPDATE _timescaledb_catalog.dimension SET partitioning_func_schema = '_timescaledb_internal' WHERE partitioning_func_schema = '_timescaledb_functions' AND partitioning_func IN ('get_partition_for_key','get_partition_hash'); ALTER FUNCTION _timescaledb_functions.finalize_agg_ffunc(internal,text,name,name,name[],bytea,anyelement) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.finalize_agg_sfunc(internal,text,name,name,name[],bytea,anyelement) SET SCHEMA _timescaledb_internal; ALTER FUNCTION _timescaledb_functions.partialize_agg(anyelement) SET SCHEMA _timescaledb_internal; ALTER AGGREGATE _timescaledb_functions.finalize_agg(text,name,name,name[][],bytea,anyelement) SET SCHEMA _timescaledb_internal; DROP FUNCTION _timescaledb_functions.hypertable_osm_range_update(regclass, anyelement, anyelement, boolean); -- recreate the _timescaledb_catalog.hypertable table as new field was added -- 1. drop CONSTRAINTS from other tables referencing the existing one ALTER TABLE _timescaledb_config.bgw_job DROP CONSTRAINT bgw_job_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.chunk DROP CONSTRAINT chunk_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.chunk_index DROP CONSTRAINT chunk_index_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.continuous_agg DROP CONSTRAINT continuous_agg_mat_hypertable_id_fkey, DROP CONSTRAINT continuous_agg_raw_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.continuous_aggs_bucket_function DROP CONSTRAINT continuous_aggs_bucket_function_mat_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.continuous_aggs_invalidation_threshold DROP CONSTRAINT continuous_aggs_invalidation_threshold_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.dimension DROP CONSTRAINT dimension_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.hypertable DROP CONSTRAINT hypertable_compressed_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.hypertable_compression DROP CONSTRAINT hypertable_compression_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.hypertable_data_node DROP CONSTRAINT hypertable_data_node_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.tablespace DROP CONSTRAINT tablespace_hypertable_id_fkey; -- drop dependent views ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.hypertables; ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.job_stats; ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.jobs; ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.continuous_aggregates; ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.chunks; ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.dimensions; ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.compression_settings; ALTER EXTENSION timescaledb DROP VIEW _timescaledb_internal.hypertable_chunk_local_size; ALTER EXTENSION timescaledb DROP VIEW _timescaledb_internal.compressed_chunk_stats; ALTER EXTENSION timescaledb DROP VIEW timescaledb_experimental.chunk_replication_status; ALTER EXTENSION timescaledb DROP VIEW timescaledb_experimental.policies; DROP VIEW timescaledb_information.hypertables; DROP VIEW timescaledb_information.job_stats; DROP VIEW timescaledb_information.jobs; DROP VIEW timescaledb_information.continuous_aggregates; DROP VIEW timescaledb_information.chunks; DROP VIEW timescaledb_information.dimensions; DROP VIEW timescaledb_information.compression_settings; DROP VIEW _timescaledb_internal.hypertable_chunk_local_size; DROP VIEW _timescaledb_internal.compressed_chunk_stats; DROP VIEW timescaledb_experimental.chunk_replication_status; DROP VIEW timescaledb_experimental.policies; -- recreate table CREATE TABLE _timescaledb_catalog.hypertable_tmp AS SELECT * FROM _timescaledb_catalog.hypertable; CREATE TABLE _timescaledb_catalog.tmp_hypertable_seq_value AS SELECT last_value, is_called FROM _timescaledb_catalog.hypertable_id_seq; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.hypertable; ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_catalog.hypertable_id_seq; SET timescaledb.restoring = on; -- must disable the hooks otherwise we can't do anything without the table _timescaledb_catalog.hypertable DROP TABLE _timescaledb_catalog.hypertable; CREATE SEQUENCE _timescaledb_catalog.hypertable_id_seq MINVALUE 1; SELECT setval('_timescaledb_catalog.hypertable_id_seq', last_value, is_called) FROM _timescaledb_catalog.tmp_hypertable_seq_value; DROP TABLE _timescaledb_catalog.tmp_hypertable_seq_value; CREATE TABLE _timescaledb_catalog.hypertable ( id INTEGER PRIMARY KEY NOT NULL DEFAULT nextval('_timescaledb_catalog.hypertable_id_seq'), schema_name name NOT NULL, table_name name NOT NULL, associated_schema_name name NOT NULL, associated_table_prefix name NOT NULL, num_dimensions smallint NOT NULL, chunk_sizing_func_schema name NOT NULL, chunk_sizing_func_name name NOT NULL, chunk_target_size bigint NOT NULL, -- size in bytes compression_state smallint NOT NULL DEFAULT 0, compressed_hypertable_id integer, replication_factor smallint NULL ); SET timescaledb.restoring = off; INSERT INTO _timescaledb_catalog.hypertable ( id, schema_name, table_name, associated_schema_name, associated_table_prefix, num_dimensions, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size, compression_state, compressed_hypertable_id, replication_factor ) SELECT id, schema_name, table_name, associated_schema_name, associated_table_prefix, num_dimensions, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size, compression_state, compressed_hypertable_id, replication_factor FROM _timescaledb_catalog.hypertable_tmp ORDER BY id; ALTER SEQUENCE _timescaledb_catalog.hypertable_id_seq OWNED BY _timescaledb_catalog.hypertable.id; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable', 'WHERE id >= 1'); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable_id_seq', ''); GRANT SELECT ON _timescaledb_catalog.hypertable TO PUBLIC; GRANT SELECT ON _timescaledb_catalog.hypertable_id_seq TO PUBLIC; DROP TABLE _timescaledb_catalog.hypertable_tmp; -- now add any constraints ALTER TABLE _timescaledb_catalog.hypertable -- ADD CONSTRAINT hypertable_pkey PRIMARY KEY (id), ADD CONSTRAINT hypertable_associated_schema_name_associated_table_prefix_key UNIQUE (associated_schema_name, associated_table_prefix), ADD CONSTRAINT hypertable_table_name_schema_name_key UNIQUE (table_name, schema_name), ADD CONSTRAINT hypertable_schema_name_check CHECK (schema_name != '_timescaledb_catalog'), -- internal compressed hypertables have compression state = 2 ADD CONSTRAINT hypertable_dim_compress_check CHECK (num_dimensions > 0 OR compression_state = 2), ADD CONSTRAINT hypertable_chunk_target_size_check CHECK (chunk_target_size >= 0), ADD CONSTRAINT hypertable_compress_check CHECK ( (compression_state = 0 OR compression_state = 1 ) OR (compression_state = 2 AND compressed_hypertable_id IS NULL)), -- replication_factor NULL: regular hypertable -- replication_factor > 0: distributed hypertable on access node -- replication_factor -1: distributed hypertable on data node, which is part of a larger table ADD CONSTRAINT hypertable_replication_factor_check CHECK (replication_factor > 0 OR replication_factor = -1), ADD CONSTRAINT hypertable_compressed_hypertable_id_fkey FOREIGN KEY (compressed_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id); GRANT SELECT ON TABLE _timescaledb_catalog.hypertable TO PUBLIC; -- 3. reestablish constraints on other tables ALTER TABLE _timescaledb_config.bgw_job ADD CONSTRAINT bgw_job_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.chunk ADD CONSTRAINT chunk_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id); ALTER TABLE _timescaledb_catalog.chunk_index ADD CONSTRAINT chunk_index_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.continuous_agg ADD CONSTRAINT continuous_agg_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE, ADD CONSTRAINT continuous_agg_raw_hypertable_id_fkey FOREIGN KEY (raw_hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.continuous_aggs_bucket_function ADD CONSTRAINT continuous_aggs_bucket_function_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.continuous_aggs_invalidation_threshold ADD CONSTRAINT continuous_aggs_invalidation_threshold_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.dimension ADD CONSTRAINT dimension_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.hypertable_compression ADD CONSTRAINT hypertable_compression_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.hypertable_data_node ADD CONSTRAINT hypertable_data_node_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id); ALTER TABLE _timescaledb_catalog.tablespace ADD CONSTRAINT tablespace_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ================================================ FILE: sql/updates/2.12.0-2.12.1.sql ================================================ ================================================ FILE: sql/updates/2.12.1--2.12.0.sql ================================================ ================================================ FILE: sql/updates/2.12.1--2.12.2.sql ================================================ ================================================ FILE: sql/updates/2.12.2--2.12.1.sql ================================================ ================================================ FILE: sql/updates/2.12.2--2.13.0.sql ================================================ CREATE TYPE _timescaledb_internal.dimension_info; CREATE OR REPLACE FUNCTION _timescaledb_functions.dimension_info_in(cstring) RETURNS _timescaledb_internal.dimension_info LANGUAGE C STRICT IMMUTABLE AS '@MODULE_PATHNAME@', 'ts_dimension_info_in'; CREATE OR REPLACE FUNCTION _timescaledb_functions.dimension_info_out(_timescaledb_internal.dimension_info) RETURNS cstring LANGUAGE C STRICT IMMUTABLE AS '@MODULE_PATHNAME@', 'ts_dimension_info_out'; CREATE TYPE _timescaledb_internal.dimension_info ( INPUT = _timescaledb_functions.dimension_info_in, OUTPUT = _timescaledb_functions.dimension_info_out, INTERNALLENGTH = VARIABLE ); CREATE FUNCTION @extschema@.create_hypertable( relation REGCLASS, dimension _timescaledb_internal.dimension_info, create_default_indexes BOOLEAN = TRUE, if_not_exists BOOLEAN = FALSE, migrate_data BOOLEAN = FALSE ) RETURNS TABLE(hypertable_id INT, created BOOL) AS '@MODULE_PATHNAME@', 'ts_hypertable_create_general' LANGUAGE C VOLATILE; CREATE FUNCTION @extschema@.add_dimension( hypertable REGCLASS, dimension _timescaledb_internal.dimension_info, if_not_exists BOOLEAN = FALSE ) RETURNS TABLE(dimension_id INT, created BOOL) AS '@MODULE_PATHNAME@', 'ts_dimension_add_general' LANGUAGE C VOLATILE; CREATE FUNCTION @extschema@.set_partitioning_interval( hypertable REGCLASS, partition_interval ANYELEMENT, dimension_name NAME = NULL ) RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_dimension_set_interval' LANGUAGE C VOLATILE; CREATE FUNCTION @extschema@.by_hash(column_name NAME, number_partitions INTEGER, partition_func regproc = NULL) RETURNS _timescaledb_internal.dimension_info LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_hash_dimension'; CREATE FUNCTION @extschema@.by_range(column_name NAME, partition_interval ANYELEMENT = NULL::bigint, partition_func regproc = NULL) RETURNS _timescaledb_internal.dimension_info LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_range_dimension'; -- -- Rebuild the catalog table `_timescaledb_catalog.chunk` to -- add new column `creation_time` -- CREATE TABLE _timescaledb_internal.chunk_tmp AS SELECT * from _timescaledb_catalog.chunk; CREATE TABLE _timescaledb_internal.tmp_chunk_seq_value AS SELECT last_value, is_called FROM _timescaledb_catalog.chunk_id_seq; --drop foreign keys on chunk table ALTER TABLE _timescaledb_catalog.chunk_constraint DROP CONSTRAINT chunk_constraint_chunk_id_fkey; ALTER TABLE _timescaledb_catalog.chunk_index DROP CONSTRAINT chunk_index_chunk_id_fkey; ALTER TABLE _timescaledb_catalog.chunk_data_node DROP CONSTRAINT chunk_data_node_chunk_id_fkey; ALTER TABLE _timescaledb_internal.bgw_policy_chunk_stats DROP CONSTRAINT bgw_policy_chunk_stats_chunk_id_fkey; ALTER TABLE _timescaledb_catalog.compression_chunk_size DROP CONSTRAINT compression_chunk_size_chunk_id_fkey; ALTER TABLE _timescaledb_catalog.compression_chunk_size DROP CONSTRAINT compression_chunk_size_compressed_chunk_id_fkey; ALTER TABLE _timescaledb_catalog.chunk_copy_operation DROP CONSTRAINT chunk_copy_operation_chunk_id_fkey; --drop dependent views DROP VIEW IF EXISTS timescaledb_information.hypertables; DROP VIEW IF EXISTS timescaledb_information.chunks; DROP VIEW IF EXISTS _timescaledb_internal.hypertable_chunk_local_size; DROP VIEW IF EXISTS _timescaledb_internal.compressed_chunk_stats; DROP VIEW IF EXISTS timescaledb_experimental.chunk_replication_status; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.chunk; ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_catalog.chunk_id_seq; DROP TABLE _timescaledb_catalog.chunk; CREATE SEQUENCE _timescaledb_catalog.chunk_id_seq MINVALUE 1; -- now create table without self referential foreign key CREATE TABLE _timescaledb_catalog.chunk ( id integer NOT NULL DEFAULT nextval('_timescaledb_catalog.chunk_id_seq'), hypertable_id int NOT NULL, schema_name name NOT NULL, table_name name NOT NULL, compressed_chunk_id integer , dropped boolean NOT NULL DEFAULT FALSE, status integer NOT NULL DEFAULT 0, osm_chunk boolean NOT NULL DEFAULT FALSE, creation_time timestamptz, -- table constraints CONSTRAINT chunk_pkey PRIMARY KEY (id), CONSTRAINT chunk_schema_name_table_name_key UNIQUE (schema_name, table_name) ); INSERT INTO _timescaledb_catalog.chunk ( id, hypertable_id, schema_name, table_name, compressed_chunk_id, dropped, status, osm_chunk) SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, dropped, status, osm_chunk FROM _timescaledb_internal.chunk_tmp; -- update creation_time for chunks UPDATE _timescaledb_catalog.chunk c SET creation_time = (pg_catalog.pg_stat_file(pg_catalog.pg_relation_filepath(r.oid))).modification FROM pg_class r, pg_namespace n WHERE r.relnamespace = n.oid AND r.relname = c.table_name AND n.nspname = c.schema_name AND r.relkind = 'r' AND c.dropped IS FALSE; -- Make sure that there are no record with empty creation time UPDATE _timescaledb_catalog.chunk SET creation_time = now() WHERE creation_time IS NULL; --add indexes to the chunk table CREATE INDEX chunk_hypertable_id_idx ON _timescaledb_catalog.chunk (hypertable_id); CREATE INDEX chunk_compressed_chunk_id_idx ON _timescaledb_catalog.chunk (compressed_chunk_id); CREATE INDEX chunk_osm_chunk_idx ON _timescaledb_catalog.chunk (osm_chunk, hypertable_id); CREATE INDEX chunk_hypertable_id_creation_time_idx ON _timescaledb_catalog.chunk(hypertable_id, creation_time); ALTER SEQUENCE _timescaledb_catalog.chunk_id_seq OWNED BY _timescaledb_catalog.chunk.id; SELECT setval('_timescaledb_catalog.chunk_id_seq', last_value, is_called) FROM _timescaledb_internal.tmp_chunk_seq_value; -- add self referential foreign key ALTER TABLE _timescaledb_catalog.chunk ADD CONSTRAINT chunk_compressed_chunk_id_fkey FOREIGN KEY ( compressed_chunk_id ) REFERENCES _timescaledb_catalog.chunk( id ); --add foreign key constraint ALTER TABLE _timescaledb_catalog.chunk ADD CONSTRAINT chunk_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk', ''); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk_id_seq', ''); -- Add non-null constraint ALTER TABLE _timescaledb_catalog.chunk ALTER COLUMN creation_time SET NOT NULL; --add the foreign key constraints ALTER TABLE _timescaledb_catalog.chunk_constraint ADD CONSTRAINT chunk_constraint_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id); ALTER TABLE _timescaledb_catalog.chunk_index ADD CONSTRAINT chunk_index_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.chunk_data_node ADD CONSTRAINT chunk_data_node_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id); ALTER TABLE _timescaledb_internal.bgw_policy_chunk_stats ADD CONSTRAINT bgw_policy_chunk_stats_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.compression_chunk_size ADD CONSTRAINT compression_chunk_size_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.compression_chunk_size ADD CONSTRAINT compression_chunk_size_compressed_chunk_id_fkey FOREIGN KEY (compressed_chunk_id) REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.chunk_copy_operation ADD CONSTRAINT chunk_copy_operation_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ON DELETE CASCADE; --cleanup DROP TABLE _timescaledb_internal.chunk_tmp; DROP TABLE _timescaledb_internal.tmp_chunk_seq_value; GRANT SELECT ON _timescaledb_catalog.chunk_id_seq TO PUBLIC; GRANT SELECT ON _timescaledb_catalog.chunk TO PUBLIC; -- end recreate _timescaledb_catalog.chunk table -- -- -- Rebuild the catalog table `_timescaledb_catalog.compression_chunk_size` to -- add new column `numrows_frozen_immediately` -- CREATE TABLE _timescaledb_internal.compression_chunk_size_tmp AS SELECT * from _timescaledb_catalog.compression_chunk_size; -- Drop depended views -- We assume that '_timescaledb_internal.compressed_chunk_stats' was already dropped in this update -- (see above) -- Drop table ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.compression_chunk_size; DROP TABLE _timescaledb_catalog.compression_chunk_size; CREATE TABLE _timescaledb_catalog.compression_chunk_size ( chunk_id integer NOT NULL, compressed_chunk_id integer NOT NULL, uncompressed_heap_size bigint NOT NULL, uncompressed_toast_size bigint NOT NULL, uncompressed_index_size bigint NOT NULL, compressed_heap_size bigint NOT NULL, compressed_toast_size bigint NOT NULL, compressed_index_size bigint NOT NULL, numrows_pre_compression bigint, numrows_post_compression bigint, numrows_frozen_immediately bigint, -- table constraints CONSTRAINT compression_chunk_size_pkey PRIMARY KEY (chunk_id), CONSTRAINT compression_chunk_size_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ON DELETE CASCADE, CONSTRAINT compression_chunk_size_compressed_chunk_id_fkey FOREIGN KEY (compressed_chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ON DELETE CASCADE ); INSERT INTO _timescaledb_catalog.compression_chunk_size (chunk_id, compressed_chunk_id, uncompressed_heap_size, uncompressed_toast_size, uncompressed_index_size, compressed_heap_size, compressed_toast_size, compressed_index_size, numrows_pre_compression, numrows_post_compression, numrows_frozen_immediately) SELECT chunk_id, compressed_chunk_id, uncompressed_heap_size, uncompressed_toast_size, uncompressed_index_size, compressed_heap_size, compressed_toast_size, compressed_index_size, numrows_pre_compression, numrows_post_compression, 0 FROM _timescaledb_internal.compression_chunk_size_tmp; DROP TABLE _timescaledb_internal.compression_chunk_size_tmp; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.compression_chunk_size', ''); GRANT SELECT ON _timescaledb_catalog.compression_chunk_size TO PUBLIC; -- End modify `_timescaledb_catalog.compression_chunk_size` DROP FUNCTION @extschema@.drop_chunks(REGCLASS, "any", "any", BOOL); CREATE FUNCTION @extschema@.drop_chunks( relation REGCLASS, older_than "any" = NULL, newer_than "any" = NULL, verbose BOOLEAN = FALSE, created_before "any" = NULL, created_after "any" = NULL ) RETURNS SETOF TEXT AS '@MODULE_PATHNAME@', 'ts_chunk_drop_chunks' LANGUAGE C VOLATILE PARALLEL UNSAFE; DROP FUNCTION @extschema@.show_chunks(REGCLASS, "any", "any"); CREATE FUNCTION @extschema@.show_chunks( relation REGCLASS, older_than "any" = NULL, newer_than "any" = NULL, created_before "any" = NULL, created_after "any" = NULL ) RETURNS SETOF REGCLASS AS '@MODULE_PATHNAME@', 'ts_chunk_show_chunks' LANGUAGE C STABLE PARALLEL SAFE; DROP FUNCTION @extschema@.add_retention_policy(REGCLASS, "any", BOOL, INTERVAL, TIMESTAMPTZ, TEXT); CREATE FUNCTION @extschema@.add_retention_policy( relation REGCLASS, drop_after "any" = NULL, if_not_exists BOOL = false, schedule_interval INTERVAL = NULL, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL, drop_created_before INTERVAL = NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_policy_retention_add' LANGUAGE C VOLATILE; DROP FUNCTION @extschema@.add_compression_policy(REGCLASS, "any", BOOL, INTERVAL, TIMESTAMPTZ, TEXT); CREATE FUNCTION @extschema@.add_compression_policy( hypertable REGCLASS, compress_after "any" = NULL, if_not_exists BOOL = false, schedule_interval INTERVAL = NULL, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL, compress_created_before INTERVAL = NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_policy_compression_add' LANGUAGE C VOLATILE; DROP PROCEDURE IF EXISTS _timescaledb_functions.policy_compression_execute(INTEGER, INTEGER, ANYELEMENT, INTEGER, BOOLEAN, BOOLEAN); DROP PROCEDURE IF EXISTS _timescaledb_internal.policy_compression_execute(INTEGER, INTEGER, ANYELEMENT, INTEGER, BOOLEAN, BOOLEAN); CREATE PROCEDURE _timescaledb_functions.policy_compression_execute( job_id INTEGER, htid INTEGER, lag ANYELEMENT, maxchunks INTEGER, verbose_log BOOLEAN, recompress_enabled BOOLEAN, use_creation_time BOOLEAN) AS $$ DECLARE htoid REGCLASS; chunk_rec RECORD; numchunks INTEGER := 1; _message text; _detail text; -- chunk status bits: bit_compressed int := 1; bit_compressed_unordered int := 2; bit_frozen int := 4; bit_compressed_partial int := 8; creation_lag INTERVAL := NULL; BEGIN -- procedures with SET clause cannot execute transaction -- control so we adjust search_path in procedure body SET LOCAL search_path TO pg_catalog, pg_temp; SELECT format('%I.%I', schema_name, table_name) INTO htoid FROM _timescaledb_catalog.hypertable WHERE id = htid; -- for the integer cases, we have to compute the lag w.r.t -- the integer_now function and then pass on to show_chunks IF pg_typeof(lag) IN ('BIGINT'::regtype, 'INTEGER'::regtype, 'SMALLINT'::regtype) THEN -- cannot have use_creation_time set with this IF use_creation_time IS TRUE THEN RAISE EXCEPTION 'job % cannot use creation time with integer_now function', job_id; END IF; lag := _timescaledb_functions.subtract_integer_from_now(htoid, lag::BIGINT); END IF; -- if use_creation_time has been specified then the lag needs to be used with the -- "compress_created_before" argument. Otherwise the usual "older_than" argument -- is good enough IF use_creation_time IS TRUE THEN creation_lag := lag; lag := NULL; END IF; FOR chunk_rec IN SELECT show.oid, ch.schema_name, ch.table_name, ch.status FROM @extschema@.show_chunks(htoid, older_than => lag, created_before => creation_lag) AS show(oid) INNER JOIN pg_class pgc ON pgc.oid = show.oid INNER JOIN pg_namespace pgns ON pgc.relnamespace = pgns.oid INNER JOIN _timescaledb_catalog.chunk ch ON ch.table_name = pgc.relname AND ch.schema_name = pgns.nspname AND ch.hypertable_id = htid WHERE ch.dropped IS FALSE AND ( ch.status = 0 OR ( ch.status & bit_compressed > 0 AND ( ch.status & bit_compressed_unordered > 0 OR ch.status & bit_compressed_partial > 0 ) ) ) LOOP IF chunk_rec.status = 0 THEN BEGIN PERFORM @extschema@.compress_chunk( chunk_rec.oid ); EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS _message = MESSAGE_TEXT, _detail = PG_EXCEPTION_DETAIL; RAISE WARNING 'compressing chunk "%" failed when compression policy is executed', chunk_rec.oid::regclass::text USING DETAIL = format('Message: (%s), Detail: (%s).', _message, _detail), ERRCODE = sqlstate; END; ELSIF ( chunk_rec.status & bit_compressed > 0 AND ( chunk_rec.status & bit_compressed_unordered > 0 OR chunk_rec.status & bit_compressed_partial > 0 ) ) AND recompress_enabled IS TRUE THEN BEGIN PERFORM @extschema@.decompress_chunk(chunk_rec.oid, if_compressed => true); EXCEPTION WHEN OTHERS THEN RAISE WARNING 'decompressing chunk "%" failed when compression policy is executed', chunk_rec.oid::regclass::text USING DETAIL = format('Message: (%s), Detail: (%s).', _message, _detail), ERRCODE = sqlstate; END; -- SET LOCAL is only active until end of transaction. -- While we could use SET at the start of the function we do not -- want to bleed out search_path to caller, so we do SET LOCAL -- again after COMMIT BEGIN PERFORM @extschema@.compress_chunk(chunk_rec.oid); EXCEPTION WHEN OTHERS THEN RAISE WARNING 'compressing chunk "%" failed when compression policy is executed', chunk_rec.oid::regclass::text USING DETAIL = format('Message: (%s), Detail: (%s).', _message, _detail), ERRCODE = sqlstate; END; END IF; COMMIT; -- SET LOCAL is only active until end of transaction. -- While we could use SET at the start of the function we do not -- want to bleed out search_path to caller, so we do SET LOCAL -- again after COMMIT SET LOCAL search_path TO pg_catalog, pg_temp; IF verbose_log THEN RAISE LOG 'job % completed processing chunk %.%', job_id, chunk_rec.schema_name, chunk_rec.table_name; END IF; numchunks := numchunks + 1; IF maxchunks > 0 AND numchunks >= maxchunks THEN EXIT; END IF; END LOOP; END; $$ LANGUAGE PLPGSQL; -- fix atttypmod and attcollation for segmentby columns DO $$ DECLARE htc_id INTEGER; htc REGCLASS; _attname NAME; _atttypmod INTEGER; _attcollation OID; BEGIN -- find any segmentby columns where typmod and collation in -- the compressed hypertable does not match the uncompressed -- hypertable values FOR htc_id, htc, _attname, _atttypmod, _attcollation IN SELECT cat.htc_id, cat.htc, pga.attname, ht_mod, ht_coll FROM pg_attribute pga INNER JOIN ( SELECT htc.id AS htc_id, format('%I.%I',htc.schema_name,htc.table_name) AS htc, att_ht.atttypmod AS ht_mod, att_ht.attcollation AS ht_coll, c.attname FROM _timescaledb_catalog.hypertable_compression c INNER JOIN _timescaledb_catalog.hypertable ht ON ht.id=c.hypertable_id INNER JOIN pg_attribute att_ht ON att_ht.attname = c.attname AND att_ht.attrelid = format('%I.%I',ht.schema_name,ht.table_name)::regclass INNER JOIN _timescaledb_catalog.hypertable htc ON htc.id=ht.compressed_hypertable_id WHERE c.segmentby_column_index > 0 ) cat ON cat.htc::regclass = pga.attrelid AND cat.attname = pga.attname WHERE pga.atttypmod <> ht_mod OR pga.attcollation <> ht_coll LOOP -- fix typmod and collation for the compressed hypertable and all compressed chunks UPDATE pg_attribute SET atttypmod = _atttypmod, attcollation = _attcollation WHERE attname = _attname AND attrelid IN ( SELECT format('%I.%I',schema_name,table_name)::regclass from _timescaledb_catalog.chunk WHERE hypertable_id = htc_id AND NOT dropped UNION ALL SELECT htc ); END LOOP; END $$; ================================================ FILE: sql/updates/2.13.0--2.12.2.sql ================================================ -- API changes related to hypertable generalization DROP FUNCTION IF EXISTS @extschema@.add_dimension(regclass,dimension_info,boolean); DROP FUNCTION IF EXISTS @extschema@.create_hypertable(regclass,dimension_info,boolean,boolean,boolean); DROP FUNCTION IF EXISTS @extschema@.set_partitioning_interval(regclass,anyelement,name); DROP FUNCTION IF EXISTS @extschema@.by_hash(name,integer,regproc); DROP FUNCTION IF EXISTS @extschema@.by_range(name,anyelement,regproc); DROP TYPE IF EXISTS _timescaledb_internal.dimension_info CASCADE; -- -- Rebuild the catalog table `_timescaledb_catalog.chunk` -- -- We need to recreate the catalog from scratch because when we drop a column -- Postgres marks `pg_attribute.attisdropped=TRUE` instead of removing it from -- the `pg_catalog.pg_attribute` table. -- -- If we downgrade and upgrade the extension without rebuilding the catalog table it -- will mess up `pg_attribute.attnum` and we will end up with issues when trying -- to update data in those catalog tables. -- Recreate _timescaledb_catalog.chunk table -- CREATE TABLE _timescaledb_internal.chunk_tmp AS SELECT * from _timescaledb_catalog.chunk; CREATE TABLE _timescaledb_internal.tmp_chunk_seq_value AS SELECT last_value, is_called FROM _timescaledb_catalog.chunk_id_seq; --drop foreign keys on chunk table ALTER TABLE _timescaledb_catalog.chunk_constraint DROP CONSTRAINT chunk_constraint_chunk_id_fkey; ALTER TABLE _timescaledb_catalog.chunk_index DROP CONSTRAINT chunk_index_chunk_id_fkey; ALTER TABLE _timescaledb_catalog.chunk_data_node DROP CONSTRAINT chunk_data_node_chunk_id_fkey; ALTER TABLE _timescaledb_internal.bgw_policy_chunk_stats DROP CONSTRAINT bgw_policy_chunk_stats_chunk_id_fkey; ALTER TABLE _timescaledb_catalog.compression_chunk_size DROP CONSTRAINT compression_chunk_size_chunk_id_fkey; ALTER TABLE _timescaledb_catalog.compression_chunk_size DROP CONSTRAINT compression_chunk_size_compressed_chunk_id_fkey; ALTER TABLE _timescaledb_catalog.chunk_copy_operation DROP CONSTRAINT chunk_copy_operation_chunk_id_fkey; --drop dependent views DROP VIEW IF EXISTS timescaledb_information.hypertables; DROP VIEW IF EXISTS timescaledb_information.chunks; DROP VIEW IF EXISTS _timescaledb_internal.hypertable_chunk_local_size; DROP VIEW IF EXISTS _timescaledb_internal.compressed_chunk_stats; DROP VIEW IF EXISTS timescaledb_experimental.chunk_replication_status; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.chunk; ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_catalog.chunk_id_seq; DROP TABLE _timescaledb_catalog.chunk; CREATE SEQUENCE _timescaledb_catalog.chunk_id_seq MINVALUE 1; -- now create table without self referential foreign key CREATE TABLE _timescaledb_catalog.chunk ( id integer NOT NULL DEFAULT nextval('_timescaledb_catalog.chunk_id_seq'), hypertable_id int NOT NULL, schema_name name NOT NULL, table_name name NOT NULL, compressed_chunk_id integer , dropped boolean NOT NULL DEFAULT FALSE, status integer NOT NULL DEFAULT 0, osm_chunk boolean NOT NULL DEFAULT FALSE, -- table constraints CONSTRAINT chunk_pkey PRIMARY KEY (id), CONSTRAINT chunk_schema_name_table_name_key UNIQUE (schema_name, table_name) ); INSERT INTO _timescaledb_catalog.chunk ( id, hypertable_id, schema_name, table_name, compressed_chunk_id, dropped, status, osm_chunk) SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, dropped, status, osm_chunk FROM _timescaledb_internal.chunk_tmp; --add indexes to the chunk table CREATE INDEX chunk_hypertable_id_idx ON _timescaledb_catalog.chunk (hypertable_id); CREATE INDEX chunk_compressed_chunk_id_idx ON _timescaledb_catalog.chunk (compressed_chunk_id); CREATE INDEX chunk_osm_chunk_idx ON _timescaledb_catalog.chunk (osm_chunk, hypertable_id); ALTER SEQUENCE _timescaledb_catalog.chunk_id_seq OWNED BY _timescaledb_catalog.chunk.id; SELECT setval('_timescaledb_catalog.chunk_id_seq', last_value, is_called) FROM _timescaledb_internal.tmp_chunk_seq_value; -- add self referential foreign key ALTER TABLE _timescaledb_catalog.chunk ADD CONSTRAINT chunk_compressed_chunk_id_fkey FOREIGN KEY ( compressed_chunk_id ) REFERENCES _timescaledb_catalog.chunk( id ); --add foreign key constraint ALTER TABLE _timescaledb_catalog.chunk ADD CONSTRAINT chunk_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk', ''); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk_id_seq', ''); --add the foreign key constraints ALTER TABLE _timescaledb_catalog.chunk_constraint ADD CONSTRAINT chunk_constraint_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id); ALTER TABLE _timescaledb_catalog.chunk_index ADD CONSTRAINT chunk_index_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.chunk_data_node ADD CONSTRAINT chunk_data_node_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id); ALTER TABLE _timescaledb_internal.bgw_policy_chunk_stats ADD CONSTRAINT bgw_policy_chunk_stats_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.compression_chunk_size ADD CONSTRAINT compression_chunk_size_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.compression_chunk_size ADD CONSTRAINT compression_chunk_size_compressed_chunk_id_fkey FOREIGN KEY (compressed_chunk_id) REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.chunk_copy_operation ADD CONSTRAINT chunk_copy_operation_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ON DELETE CASCADE; --cleanup DROP TABLE _timescaledb_internal.chunk_tmp; DROP TABLE _timescaledb_internal.tmp_chunk_seq_value; GRANT SELECT ON _timescaledb_catalog.chunk_id_seq TO PUBLIC; GRANT SELECT ON _timescaledb_catalog.chunk TO PUBLIC; -- end recreate _timescaledb_catalog.chunk table -- -- -- Rebuild the catalog table `_timescaledb_catalog.compression_chunk_size` to -- remove column `numrows_frozen_immediately` -- CREATE TABLE _timescaledb_internal.compression_chunk_size_tmp AS SELECT * from _timescaledb_catalog.compression_chunk_size; -- Drop depended views -- We assume that '_timescaledb_internal.compressed_chunk_stats' was already dropped in this update -- (see above) -- Drop table ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.compression_chunk_size; DROP TABLE _timescaledb_catalog.compression_chunk_size; CREATE TABLE _timescaledb_catalog.compression_chunk_size ( chunk_id integer NOT NULL, compressed_chunk_id integer NOT NULL, uncompressed_heap_size bigint NOT NULL, uncompressed_toast_size bigint NOT NULL, uncompressed_index_size bigint NOT NULL, compressed_heap_size bigint NOT NULL, compressed_toast_size bigint NOT NULL, compressed_index_size bigint NOT NULL, numrows_pre_compression bigint, numrows_post_compression bigint, -- table constraints CONSTRAINT compression_chunk_size_pkey PRIMARY KEY (chunk_id), CONSTRAINT compression_chunk_size_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ON DELETE CASCADE, CONSTRAINT compression_chunk_size_compressed_chunk_id_fkey FOREIGN KEY (compressed_chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ON DELETE CASCADE ); INSERT INTO _timescaledb_catalog.compression_chunk_size (chunk_id, compressed_chunk_id, uncompressed_heap_size, uncompressed_toast_size, uncompressed_index_size, compressed_heap_size, compressed_toast_size, compressed_index_size, numrows_pre_compression, numrows_post_compression) SELECT chunk_id, compressed_chunk_id, uncompressed_heap_size, uncompressed_toast_size, uncompressed_index_size, compressed_heap_size, compressed_toast_size, compressed_index_size, numrows_pre_compression, numrows_post_compression FROM _timescaledb_internal.compression_chunk_size_tmp; DROP TABLE _timescaledb_internal.compression_chunk_size_tmp; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.compression_chunk_size', ''); GRANT SELECT ON _timescaledb_catalog.compression_chunk_size TO PUBLIC; -- End modify `_timescaledb_catalog.compression_chunk_size` DROP FUNCTION @extschema@.drop_chunks(REGCLASS, "any", "any", BOOL, "any", "any"); CREATE FUNCTION @extschema@.drop_chunks( relation REGCLASS, older_than "any" = NULL, newer_than "any" = NULL, verbose BOOLEAN = FALSE ) RETURNS SETOF TEXT AS '@MODULE_PATHNAME@', 'ts_chunk_drop_chunks' LANGUAGE C VOLATILE PARALLEL UNSAFE; DROP FUNCTION @extschema@.show_chunks(REGCLASS, "any", "any", "any", "any"); CREATE FUNCTION @extschema@.show_chunks( relation REGCLASS, older_than "any" = NULL, newer_than "any" = NULL ) RETURNS SETOF REGCLASS AS '@MODULE_PATHNAME@', 'ts_chunk_show_chunks' LANGUAGE C STABLE PARALLEL SAFE; DROP PROCEDURE IF EXISTS _timescaledb_functions.repair_relation_acls(); DROP FUNCTION IF EXISTS _timescaledb_functions.makeaclitem(regrole, regrole, text, bool); DROP FUNCTION IF EXISTS _timescaledb_functions.cagg_validate_query(TEXT); DROP FUNCTION @extschema@.add_retention_policy(REGCLASS, "any", BOOL, INTERVAL, TIMESTAMPTZ, TEXT, INTERVAL); CREATE FUNCTION @extschema@.add_retention_policy( relation REGCLASS, drop_after "any", if_not_exists BOOL = false, schedule_interval INTERVAL = NULL, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_policy_retention_add' LANGUAGE C VOLATILE; DROP FUNCTION @extschema@.add_compression_policy(REGCLASS, "any", BOOL, INTERVAL, TIMESTAMPTZ, TEXT, INTERVAL); CREATE FUNCTION @extschema@.add_compression_policy( hypertable REGCLASS, compress_after "any", if_not_exists BOOL = false, schedule_interval INTERVAL = NULL, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_policy_compression_add' LANGUAGE C VOLATILE; DROP PROCEDURE IF EXISTS _timescaledb_functions.policy_compression_execute(INTEGER, INTEGER, ANYELEMENT, INTEGER, BOOLEAN, BOOLEAN, BOOLEAN); DROP PROCEDURE IF EXISTS _timescaledb_internal.policy_compression_execute(INTEGER, INTEGER, ANYELEMENT, INTEGER, BOOLEAN, BOOLEAN, BOOLEAN); CREATE PROCEDURE _timescaledb_functions.policy_compression_execute( job_id INTEGER, htid INTEGER, lag ANYELEMENT, maxchunks INTEGER, verbose_log BOOLEAN, recompress_enabled BOOLEAN) AS $$ DECLARE htoid REGCLASS; chunk_rec RECORD; numchunks INTEGER := 1; _message text; _detail text; -- chunk status bits: bit_compressed int := 1; bit_compressed_unordered int := 2; bit_frozen int := 4; bit_compressed_partial int := 8; BEGIN -- procedures with SET clause cannot execute transaction -- control so we adjust search_path in procedure body SET LOCAL search_path TO pg_catalog, pg_temp; SELECT format('%I.%I', schema_name, table_name) INTO htoid FROM _timescaledb_catalog.hypertable WHERE id = htid; -- for the integer cases, we have to compute the lag w.r.t -- the integer_now function and then pass on to show_chunks IF pg_typeof(lag) IN ('BIGINT'::regtype, 'INTEGER'::regtype, 'SMALLINT'::regtype) THEN lag := _timescaledb_functions.subtract_integer_from_now(htoid, lag::BIGINT); END IF; FOR chunk_rec IN SELECT show.oid, ch.schema_name, ch.table_name, ch.status FROM @extschema@.show_chunks(htoid, older_than => lag) AS show(oid) INNER JOIN pg_class pgc ON pgc.oid = show.oid INNER JOIN pg_namespace pgns ON pgc.relnamespace = pgns.oid INNER JOIN _timescaledb_catalog.chunk ch ON ch.table_name = pgc.relname AND ch.schema_name = pgns.nspname AND ch.hypertable_id = htid WHERE ch.dropped IS FALSE AND ( ch.status = 0 OR ( ch.status & bit_compressed > 0 AND ( ch.status & bit_compressed_unordered > 0 OR ch.status & bit_compressed_partial > 0 ) ) ) LOOP IF chunk_rec.status = 0 THEN BEGIN PERFORM @extschema@.compress_chunk( chunk_rec.oid ); EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS _message = MESSAGE_TEXT, _detail = PG_EXCEPTION_DETAIL; RAISE WARNING 'compressing chunk "%" failed when compression policy is executed', chunk_rec.oid::regclass::text USING DETAIL = format('Message: (%s), Detail: (%s).', _message, _detail), ERRCODE = sqlstate; END; ELSIF ( chunk_rec.status & bit_compressed > 0 AND ( chunk_rec.status & bit_compressed_unordered > 0 OR chunk_rec.status & bit_compressed_partial > 0 ) ) AND recompress_enabled IS TRUE THEN BEGIN PERFORM @extschema@.decompress_chunk(chunk_rec.oid, if_compressed => true); EXCEPTION WHEN OTHERS THEN RAISE WARNING 'decompressing chunk "%" failed when compression policy is executed', chunk_rec.oid::regclass::text USING DETAIL = format('Message: (%s), Detail: (%s).', _message, _detail), ERRCODE = sqlstate; END; -- SET LOCAL is only active until end of transaction. -- While we could use SET at the start of the function we do not -- want to bleed out search_path to caller, so we do SET LOCAL -- again after COMMIT BEGIN PERFORM @extschema@.compress_chunk(chunk_rec.oid); EXCEPTION WHEN OTHERS THEN RAISE WARNING 'compressing chunk "%" failed when compression policy is executed', chunk_rec.oid::regclass::text USING DETAIL = format('Message: (%s), Detail: (%s).', _message, _detail), ERRCODE = sqlstate; END; END IF; COMMIT; -- SET LOCAL is only active until end of transaction. -- While we could use SET at the start of the function we do not -- want to bleed out search_path to caller, so we do SET LOCAL -- again after COMMIT SET LOCAL search_path TO pg_catalog, pg_temp; IF verbose_log THEN RAISE LOG 'job % completed processing chunk %.%', job_id, chunk_rec.schema_name, chunk_rec.table_name; END IF; numchunks := numchunks + 1; IF maxchunks > 0 AND numchunks >= maxchunks THEN EXIT; END IF; END LOOP; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION _timescaledb_functions.chunk_constraint_add_table_constraint( chunk_constraint_row _timescaledb_catalog.chunk_constraint ); CREATE FUNCTION _timescaledb_functions.chunk_constraint_add_table_constraint( chunk_constraint_row _timescaledb_catalog.chunk_constraint ) RETURNS VOID LANGUAGE PLPGSQL AS $BODY$ DECLARE chunk_row _timescaledb_catalog.chunk; hypertable_row _timescaledb_catalog.hypertable; constraint_oid OID; constraint_type CHAR; check_sql TEXT; def TEXT; indx_tablespace NAME; tablespace_def TEXT; BEGIN SELECT * INTO STRICT chunk_row FROM _timescaledb_catalog.chunk c WHERE c.id = chunk_constraint_row.chunk_id; SELECT * INTO STRICT hypertable_row FROM _timescaledb_catalog.hypertable h WHERE h.id = chunk_row.hypertable_id; IF chunk_constraint_row.dimension_slice_id IS NOT NULL THEN RAISE 'cannot create dimension constraint %', chunk_constraint_row; ELSIF chunk_constraint_row.hypertable_constraint_name IS NOT NULL THEN SELECT oid, contype INTO STRICT constraint_oid, constraint_type FROM pg_constraint WHERE conname=chunk_constraint_row.hypertable_constraint_name AND conrelid = format('%I.%I', hypertable_row.schema_name, hypertable_row.table_name)::regclass::oid; IF constraint_type IN ('p','u') THEN -- since primary keys and unique constraints are backed by an index -- they might have an index tablespace assigned -- the tablspace is not part of the constraint definition so -- we have to append it explicitly to preserve it SELECT T.spcname INTO indx_tablespace FROM pg_constraint C, pg_class I, pg_tablespace T WHERE C.oid = constraint_oid AND C.contype IN ('p', 'u') AND I.oid = C.conindid AND I.reltablespace = T.oid; def := pg_get_constraintdef(constraint_oid); IF indx_tablespace IS NOT NULL THEN def := format('%s USING INDEX TABLESPACE %I', def, indx_tablespace); END IF; ELSIF constraint_type = 't' THEN -- constraint triggers are copied separately with normal triggers def := NULL; ELSE def := pg_get_constraintdef(constraint_oid); END IF; ELSE RAISE 'unknown constraint type'; END IF; IF def IS NOT NULL THEN -- to allow for custom types with operators outside of pg_catalog -- we set search_path to @extschema@ SET LOCAL search_path TO @extschema@, pg_temp; EXECUTE pg_catalog.format( $$ ALTER TABLE %I.%I ADD CONSTRAINT %I %s $$, chunk_row.schema_name, chunk_row.table_name, chunk_constraint_row.constraint_name, def ); END IF; END $BODY$ SET search_path TO pg_catalog, pg_temp; ================================================ FILE: sql/updates/2.13.0--2.13.1.sql ================================================ ================================================ FILE: sql/updates/2.13.1--2.13.0.sql ================================================ -- Manually drop the following functions / procedures since 'OR REPLACE' is missing in 2.13.0 DROP PROCEDURE IF EXISTS _timescaledb_functions.repair_relation_acls(); DROP FUNCTION IF EXISTS _timescaledb_functions.makeaclitem(regrole, regrole, text, bool); ================================================ FILE: sql/updates/2.13.1--2.14.0.sql ================================================ -- ERROR if trying to update the extension while multinode is present DO $$ DECLARE data_nodes TEXT; dist_hypertables TEXT; BEGIN SELECT string_agg(format('%I.%I', schema_name, table_name), ', ') INTO dist_hypertables FROM _timescaledb_catalog.hypertable WHERE replication_factor > 0; IF dist_hypertables IS NOT NULL THEN RAISE USING ERRCODE = 'feature_not_supported', MESSAGE = 'cannot upgrade because multi-node has been removed in 2.14.0', DETAIL = 'The following distributed hypertables should be migrated to regular: '||dist_hypertables; END IF; SELECT string_agg(format('%I', srv.srvname), ', ') INTO data_nodes FROM pg_foreign_server srv JOIN pg_foreign_data_wrapper fdw ON srv.srvfdw = fdw.oid AND fdw.fdwname = 'timescaledb_fdw'; IF data_nodes IS NOT NULL THEN RAISE USING ERRCODE = 'feature_not_supported', MESSAGE = 'cannot upgrade because multi-node has been removed in 2.14.0', DETAIL = 'The following data nodes should be removed: '||data_nodes; END IF; IF EXISTS(SELECT FROM _timescaledb_catalog.metadata WHERE key = 'dist_uuid') THEN RAISE USING ERRCODE = 'feature_not_supported', MESSAGE = 'cannot upgrade because multi-node has been removed in 2.14.0', DETAIL = 'This node appears to be part of a multi-node installation'; END IF; END $$; DROP FUNCTION IF EXISTS _timescaledb_functions.ping_data_node; DROP FUNCTION IF EXISTS _timescaledb_internal.ping_data_node; DROP FUNCTION IF EXISTS _timescaledb_functions.remote_txn_heal_data_node; DROP FUNCTION IF EXISTS _timescaledb_internal.remote_txn_heal_data_node; DROP FUNCTION IF EXISTS _timescaledb_functions.set_dist_id; DROP FUNCTION IF EXISTS _timescaledb_internal.set_dist_id; DROP FUNCTION IF EXISTS _timescaledb_functions.set_peer_dist_id; DROP FUNCTION IF EXISTS _timescaledb_internal.set_peer_dist_id; DROP FUNCTION IF EXISTS _timescaledb_functions.validate_as_data_node; DROP FUNCTION IF EXISTS _timescaledb_internal.validate_as_data_node; DROP FUNCTION IF EXISTS _timescaledb_functions.show_connection_cache; DROP FUNCTION IF EXISTS _timescaledb_internal.show_connection_cache; DROP FUNCTION IF EXISTS @extschema@.create_hypertable(relation REGCLASS, time_column_name NAME, partitioning_column NAME, number_partitions INTEGER, associated_schema_name NAME, associated_table_prefix NAME, chunk_time_interval ANYELEMENT, create_default_indexes BOOLEAN, if_not_exists BOOLEAN, partitioning_func REGPROC, migrate_data BOOLEAN, chunk_target_size TEXT, chunk_sizing_func REGPROC, time_partitioning_func REGPROC, replication_factor INTEGER, data_nodes NAME[], distributed BOOLEAN); CREATE FUNCTION @extschema@.create_hypertable( relation REGCLASS, time_column_name NAME, partitioning_column NAME = NULL, number_partitions INTEGER = NULL, associated_schema_name NAME = NULL, associated_table_prefix NAME = NULL, chunk_time_interval ANYELEMENT = NULL::bigint, create_default_indexes BOOLEAN = TRUE, if_not_exists BOOLEAN = FALSE, partitioning_func REGPROC = NULL, migrate_data BOOLEAN = FALSE, chunk_target_size TEXT = NULL, chunk_sizing_func REGPROC = '_timescaledb_functions.calculate_chunk_interval'::regproc, time_partitioning_func REGPROC = NULL ) RETURNS TABLE(hypertable_id INT, schema_name NAME, table_name NAME, created BOOL) AS '@MODULE_PATHNAME@', 'ts_hypertable_create' LANGUAGE C VOLATILE; DROP FUNCTION IF EXISTS @extschema@.create_distributed_hypertable; DROP FUNCTION IF EXISTS @extschema@.add_data_node; DROP FUNCTION IF EXISTS @extschema@.delete_data_node; DROP FUNCTION IF EXISTS @extschema@.attach_data_node; DROP FUNCTION IF EXISTS @extschema@.detach_data_node; DROP FUNCTION IF EXISTS @extschema@.alter_data_node; DROP PROCEDURE IF EXISTS @extschema@.distributed_exec; DROP FUNCTION IF EXISTS @extschema@.create_distributed_restore_point; DROP FUNCTION IF EXISTS @extschema@.set_replication_factor; CREATE TABLE _timescaledb_catalog.compression_settings ( relid regclass NOT NULL, segmentby text[], orderby text[], orderby_desc bool[], orderby_nullsfirst bool[], CONSTRAINT compression_settings_pkey PRIMARY KEY (relid), CONSTRAINT compression_settings_check_segmentby CHECK (array_ndims(segmentby) = 1), CONSTRAINT compression_settings_check_orderby_null CHECK ( (orderby IS NULL AND orderby_desc IS NULL AND orderby_nullsfirst IS NULL) OR (orderby IS NOT NULL AND orderby_desc IS NOT NULL AND orderby_nullsfirst IS NOT NULL) ), CONSTRAINT compression_settings_check_orderby_cardinality CHECK (array_ndims(orderby) = 1 AND array_ndims(orderby_desc) = 1 AND array_ndims(orderby_nullsfirst) = 1 AND cardinality(orderby) = cardinality(orderby_desc) AND cardinality(orderby) = cardinality(orderby_nullsfirst)) ); INSERT INTO _timescaledb_catalog.compression_settings(relid, segmentby, orderby, orderby_desc, orderby_nullsfirst) SELECT format('%I.%I', ht.schema_name, ht.table_name)::regclass, array_agg(attname ORDER BY segmentby_column_index) FILTER(WHERE segmentby_column_index >= 1) AS compress_segmentby, array_agg(attname ORDER BY orderby_column_index) FILTER(WHERE orderby_column_index >= 1) AS compress_orderby, array_agg(NOT orderby_asc ORDER BY orderby_column_index) FILTER(WHERE orderby_column_index >= 1) AS compress_orderby_desc, array_agg(orderby_nullsfirst ORDER BY orderby_column_index) FILTER(WHERE orderby_column_index >= 1) AS compress_orderby_nullsfirst FROM _timescaledb_catalog.hypertable_compression hc INNER JOIN _timescaledb_catalog.hypertable ht ON ht.id = hc.hypertable_id GROUP BY hypertable_id, ht.schema_name, ht.table_name; INSERT INTO _timescaledb_catalog.compression_settings SELECT format('%I.%I',ch.schema_name,ch.table_name)::regclass,s.segmentby,s.orderby,s.orderby_desc,s.orderby_nullsfirst FROM _timescaledb_catalog.hypertable ht1 INNER JOIN _timescaledb_catalog.hypertable ht2 ON ht2.id = ht1.compressed_hypertable_id INNER JOIN _timescaledb_catalog.compression_settings s ON s.relid = format('%I.%I',ht1.schema_name,ht1.table_name)::regclass INNER JOIN _timescaledb_catalog.chunk ch ON ch.hypertable_id = ht2.id ON CONFLICT DO NOTHING; GRANT SELECT ON _timescaledb_catalog.compression_settings TO PUBLIC; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.compression_settings', ''); ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.hypertable_compression; DROP VIEW IF EXISTS timescaledb_information.compression_settings; DROP TABLE _timescaledb_catalog.hypertable_compression; DROP FOREIGN DATA WRAPPER IF EXISTS timescaledb_fdw; DROP FUNCTION IF EXISTS @extschema@.timescaledb_fdw_handler(); DROP FUNCTION IF EXISTS @extschema@.timescaledb_fdw_validator(text[], oid); DROP FUNCTION IF EXISTS _timescaledb_functions.create_chunk_replica_table; DROP FUNCTION IF EXISTS _timescaledb_functions.chunk_drop_replica; DROP PROCEDURE IF EXISTS _timescaledb_functions.wait_subscription_sync; DROP FUNCTION IF EXISTS _timescaledb_functions.health; DROP FUNCTION IF EXISTS _timescaledb_functions.drop_stale_chunks; DROP FUNCTION IF EXISTS _timescaledb_internal.create_chunk_replica_table; DROP FUNCTION IF EXISTS _timescaledb_internal.chunk_drop_replica; DROP PROCEDURE IF EXISTS _timescaledb_internal.wait_subscription_sync; DROP FUNCTION IF EXISTS _timescaledb_internal.health; DROP FUNCTION IF EXISTS _timescaledb_internal.drop_stale_chunks; ALTER TABLE _timescaledb_catalog.remote_txn DROP CONSTRAINT remote_txn_remote_transaction_id_check; DROP TYPE IF EXISTS @extschema@.rxid CASCADE; DROP FUNCTION IF EXISTS _timescaledb_functions.rxid_in; DROP FUNCTION IF EXISTS _timescaledb_functions.rxid_out; DROP FUNCTION IF EXISTS _timescaledb_functions.data_node_hypertable_info; DROP FUNCTION IF EXISTS _timescaledb_functions.data_node_chunk_info; DROP FUNCTION IF EXISTS _timescaledb_functions.data_node_compressed_chunk_stats; DROP FUNCTION IF EXISTS _timescaledb_functions.data_node_index_size; DROP FUNCTION IF EXISTS _timescaledb_internal.data_node_hypertable_info; DROP FUNCTION IF EXISTS _timescaledb_internal.data_node_chunk_info; DROP FUNCTION IF EXISTS _timescaledb_internal.data_node_compressed_chunk_stats; DROP FUNCTION IF EXISTS _timescaledb_internal.data_node_index_size; DROP FUNCTION IF EXISTS timescaledb_experimental.block_new_chunks; DROP FUNCTION IF EXISTS timescaledb_experimental.allow_new_chunks; DROP FUNCTION IF EXISTS timescaledb_experimental.subscription_exec; DROP PROCEDURE IF EXISTS timescaledb_experimental.move_chunk; DROP PROCEDURE IF EXISTS timescaledb_experimental.copy_chunk; DROP PROCEDURE IF EXISTS timescaledb_experimental.cleanup_copy_chunk_operation; DROP FUNCTION IF EXISTS _timescaledb_functions.set_chunk_default_data_node; DROP FUNCTION IF EXISTS _timescaledb_internal.set_chunk_default_data_node; DROP FUNCTION IF EXISTS _timescaledb_functions.drop_dist_ht_invalidation_trigger; DROP FUNCTION IF EXISTS _timescaledb_internal.drop_dist_ht_invalidation_trigger; -- remove multinode catalog tables DROP VIEW IF EXISTS timescaledb_information.chunks; DROP VIEW IF EXISTS timescaledb_information.data_nodes; DROP VIEW IF EXISTS timescaledb_information.hypertables; DROP VIEW IF EXISTS timescaledb_experimental.chunk_replication_status; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.remote_txn; DROP TABLE _timescaledb_catalog.remote_txn; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.hypertable_data_node; DROP TABLE _timescaledb_catalog.hypertable_data_node; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.chunk_data_node; DROP TABLE _timescaledb_catalog.chunk_data_node; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.chunk_copy_operation; DROP TABLE _timescaledb_catalog.chunk_copy_operation; ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_catalog.chunk_copy_operation_id_seq; DROP SEQUENCE _timescaledb_catalog.chunk_copy_operation_id_seq; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.dimension_partition; DROP TABLE _timescaledb_catalog.dimension_partition; DROP FUNCTION IF EXISTS _timescaledb_functions.hypertable_remote_size; DROP FUNCTION IF EXISTS _timescaledb_internal.hypertable_remote_size; DROP FUNCTION IF EXISTS _timescaledb_functions.chunks_remote_size; DROP FUNCTION IF EXISTS _timescaledb_internal.chunks_remote_size; DROP FUNCTION IF EXISTS _timescaledb_functions.indexes_remote_size; DROP FUNCTION IF EXISTS _timescaledb_internal.indexes_remote_size; DROP FUNCTION IF EXISTS _timescaledb_functions.compressed_chunk_remote_stats; DROP FUNCTION IF EXISTS _timescaledb_internal.compressed_chunk_remote_stats; -- rebuild _timescaledb_catalog.hypertable ALTER TABLE _timescaledb_config.bgw_job DROP CONSTRAINT bgw_job_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.chunk DROP CONSTRAINT chunk_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.chunk_index DROP CONSTRAINT chunk_index_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.continuous_agg DROP CONSTRAINT continuous_agg_mat_hypertable_id_fkey, DROP CONSTRAINT continuous_agg_raw_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.continuous_aggs_bucket_function DROP CONSTRAINT continuous_aggs_bucket_function_mat_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.continuous_aggs_invalidation_threshold DROP CONSTRAINT continuous_aggs_invalidation_threshold_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.dimension DROP CONSTRAINT dimension_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.hypertable DROP CONSTRAINT hypertable_compressed_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.tablespace DROP CONSTRAINT tablespace_hypertable_id_fkey; DROP VIEW IF EXISTS timescaledb_information.hypertables; DROP VIEW IF EXISTS timescaledb_information.job_stats; DROP VIEW IF EXISTS timescaledb_information.jobs; DROP VIEW IF EXISTS timescaledb_information.continuous_aggregates; DROP VIEW IF EXISTS timescaledb_information.chunks; DROP VIEW IF EXISTS timescaledb_information.dimensions; DROP VIEW IF EXISTS timescaledb_information.compression_settings; DROP VIEW IF EXISTS _timescaledb_internal.hypertable_chunk_local_size; DROP VIEW IF EXISTS _timescaledb_internal.compressed_chunk_stats; DROP VIEW IF EXISTS timescaledb_experimental.chunk_replication_status; DROP VIEW IF EXISTS timescaledb_experimental.policies; -- recreate table CREATE TABLE _timescaledb_catalog.hypertable_tmp AS SELECT * FROM _timescaledb_catalog.hypertable; CREATE TABLE _timescaledb_catalog.tmp_hypertable_seq_value AS SELECT last_value, is_called FROM _timescaledb_catalog.hypertable_id_seq; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.hypertable; ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_catalog.hypertable_id_seq; SET timescaledb.restoring = on; -- must disable the hooks otherwise we can't do anything without the table _timescaledb_catalog.hypertable DROP TABLE _timescaledb_catalog.hypertable; CREATE SEQUENCE _timescaledb_catalog.hypertable_id_seq MINVALUE 1; SELECT setval('_timescaledb_catalog.hypertable_id_seq', last_value, is_called) FROM _timescaledb_catalog.tmp_hypertable_seq_value; DROP TABLE _timescaledb_catalog.tmp_hypertable_seq_value; CREATE TABLE _timescaledb_catalog.hypertable ( id INTEGER PRIMARY KEY NOT NULL DEFAULT nextval('_timescaledb_catalog.hypertable_id_seq'), schema_name name NOT NULL, table_name name NOT NULL, associated_schema_name name NOT NULL, associated_table_prefix name NOT NULL, num_dimensions smallint NOT NULL, chunk_sizing_func_schema name NOT NULL, chunk_sizing_func_name name NOT NULL, chunk_target_size bigint NOT NULL, -- size in bytes compression_state smallint NOT NULL DEFAULT 0, compressed_hypertable_id integer, status integer NOT NULL DEFAULT 0 ); SET timescaledb.restoring = off; INSERT INTO _timescaledb_catalog.hypertable ( id, schema_name, table_name, associated_schema_name, associated_table_prefix, num_dimensions, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size, compression_state, compressed_hypertable_id ) SELECT id, schema_name, table_name, associated_schema_name, associated_table_prefix, num_dimensions, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size, compression_state, compressed_hypertable_id FROM _timescaledb_catalog.hypertable_tmp ORDER BY id; UPDATE _timescaledb_catalog.hypertable h SET status = 3 WHERE EXISTS ( SELECT FROM _timescaledb_catalog.chunk c WHERE c.osm_chunk AND c.hypertable_id = h.id ); ALTER SEQUENCE _timescaledb_catalog.hypertable_id_seq OWNED BY _timescaledb_catalog.hypertable.id; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable_id_seq', ''); GRANT SELECT ON _timescaledb_catalog.hypertable TO PUBLIC; GRANT SELECT ON _timescaledb_catalog.hypertable_id_seq TO PUBLIC; DROP TABLE _timescaledb_catalog.hypertable_tmp; -- now add any constraints ALTER TABLE _timescaledb_catalog.hypertable ADD CONSTRAINT hypertable_associated_schema_name_associated_table_prefix_key UNIQUE (associated_schema_name, associated_table_prefix), ADD CONSTRAINT hypertable_table_name_schema_name_key UNIQUE (table_name, schema_name), ADD CONSTRAINT hypertable_schema_name_check CHECK (schema_name != '_timescaledb_catalog'), ADD CONSTRAINT hypertable_dim_compress_check CHECK (num_dimensions > 0 OR compression_state = 2), ADD CONSTRAINT hypertable_chunk_target_size_check CHECK (chunk_target_size >= 0), ADD CONSTRAINT hypertable_compress_check CHECK ( (compression_state = 0 OR compression_state = 1 ) OR (compression_state = 2 AND compressed_hypertable_id IS NULL)), ADD CONSTRAINT hypertable_compressed_hypertable_id_fkey FOREIGN KEY (compressed_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id); GRANT SELECT ON TABLE _timescaledb_catalog.hypertable TO PUBLIC; -- 3. reestablish constraints on other tables ALTER TABLE _timescaledb_config.bgw_job ADD CONSTRAINT bgw_job_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.chunk ADD CONSTRAINT chunk_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id); ALTER TABLE _timescaledb_catalog.chunk_index ADD CONSTRAINT chunk_index_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.continuous_agg ADD CONSTRAINT continuous_agg_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE, ADD CONSTRAINT continuous_agg_raw_hypertable_id_fkey FOREIGN KEY (raw_hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.continuous_aggs_bucket_function ADD CONSTRAINT continuous_aggs_bucket_function_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.continuous_aggs_invalidation_threshold ADD CONSTRAINT continuous_aggs_invalidation_threshold_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.dimension ADD CONSTRAINT dimension_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.tablespace ADD CONSTRAINT tablespace_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; CREATE SCHEMA _timescaledb_debug; -- Migrate existing compressed hypertables to new internal format DO $$ DECLARE chunk regclass; hypertable regclass; ht_id integer; index regclass; column_name name; cmd text; BEGIN SET timescaledb.restoring TO ON; -- Detach compressed chunks from their parent hypertables FOR chunk, hypertable, ht_id IN SELECT format('%I.%I',ch.schema_name,ch.table_name)::regclass chunk, format('%I.%I',ht.schema_name,ht.table_name)::regclass hypertable, ht.id FROM _timescaledb_catalog.chunk ch INNER JOIN _timescaledb_catalog.hypertable ht_uncomp ON ch.hypertable_id = ht_uncomp.compressed_hypertable_id INNER JOIN _timescaledb_catalog.hypertable ht ON ht.id = ht_uncomp.compressed_hypertable_id LOOP cmd := format('ALTER TABLE %s NO INHERIT %s', chunk, hypertable); EXECUTE cmd; -- remove references to indexes from the compressed hypertable DELETE FROM _timescaledb_catalog.chunk_index WHERE hypertable_id = ht_id; END LOOP; FOR hypertable IN SELECT format('%I.%I',ht.schema_name,ht.table_name)::regclass hypertable FROM _timescaledb_catalog.hypertable ht_uncomp INNER JOIN _timescaledb_catalog.hypertable ht ON ht.id = ht_uncomp.compressed_hypertable_id LOOP -- remove indexes from the compressed hypertable (but not chunks) FOR index IN SELECT indexrelid::regclass FROM pg_index WHERE indrelid = hypertable LOOP cmd := format('DROP INDEX %s', index); EXECUTE cmd; END LOOP; -- remove columns from the compressed hypertable (but not chunks) FOR column_name IN SELECT attname FROM pg_attribute WHERE attrelid = hypertable AND attnum > 0 AND NOT attisdropped LOOP cmd := format('ALTER TABLE %s DROP COLUMN %I', hypertable, column_name); EXECUTE cmd; END LOOP; END LOOP; SET timescaledb.restoring TO OFF; END $$; DROP FUNCTION IF EXISTS _timescaledb_internal.hypertable_constraint_add_table_fk_constraint; DROP FUNCTION IF EXISTS _timescaledb_functions.hypertable_constraint_add_table_fk_constraint; -- only define stub here, actual code will be filled in at end of update script CREATE FUNCTION _timescaledb_functions.constraint_clone(constraint_oid OID,target_oid REGCLASS) RETURNS VOID LANGUAGE PLPGSQL AS $$BEGIN END$$ SET search_path TO pg_catalog, pg_temp; DROP FUNCTION IF EXISTS _timescaledb_functions.chunks_in; DROP FUNCTION IF EXISTS _timescaledb_internal.chunks_in; CREATE FUNCTION _timescaledb_functions.metadata_insert_trigger() RETURNS TRIGGER LANGUAGE PLPGSQL AS $$ BEGIN IF EXISTS (SELECT FROM _timescaledb_catalog.metadata WHERE key = NEW.key) THEN UPDATE _timescaledb_catalog.metadata SET value = NEW.value WHERE key = NEW.key; RETURN NULL; END IF; RETURN NEW; END $$ SET search_path TO pg_catalog, pg_temp; CREATE TRIGGER metadata_insert_trigger BEFORE INSERT ON _timescaledb_catalog.metadata FOR EACH ROW EXECUTE PROCEDURE _timescaledb_functions.metadata_insert_trigger(); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.metadata', $$ WHERE key <> 'uuid' $$); -- Remove unwanted entries from extconfig and extcondition in pg_extension -- We use ALTER EXTENSION DROP TABLE to remove these entries. ALTER EXTENSION timescaledb DROP TABLE _timescaledb_cache.cache_inval_hypertable; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_cache.cache_inval_extension; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_cache.cache_inval_bgw_job; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_internal.job_errors; -- Associate the above tables back to keep the dependencies safe ALTER EXTENSION timescaledb ADD TABLE _timescaledb_cache.cache_inval_hypertable; ALTER EXTENSION timescaledb ADD TABLE _timescaledb_cache.cache_inval_extension; ALTER EXTENSION timescaledb ADD TABLE _timescaledb_cache.cache_inval_bgw_job; ALTER EXTENSION timescaledb ADD TABLE _timescaledb_internal.job_errors; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.hypertable; ALTER EXTENSION timescaledb ADD TABLE _timescaledb_catalog.hypertable; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable', 'WHERE id >= 1'); CREATE FUNCTION _timescaledb_functions.relation_approximate_size(relation REGCLASS) RETURNS TABLE (total_size BIGINT, heap_size BIGINT, index_size BIGINT, toast_size BIGINT) AS '@MODULE_PATHNAME@', 'ts_relation_approximate_size' LANGUAGE C STRICT VOLATILE; CREATE FUNCTION @extschema@.hypertable_approximate_detailed_size(relation REGCLASS) RETURNS TABLE (table_bytes BIGINT, index_bytes BIGINT, toast_bytes BIGINT, total_bytes BIGINT) AS '@MODULE_PATHNAME@', 'ts_hypertable_approximate_size' LANGUAGE C VOLATILE; --- returns approximate total-bytes for a hypertable (includes table + index) CREATE FUNCTION @extschema@.hypertable_approximate_size( hypertable REGCLASS) RETURNS BIGINT LANGUAGE SQL VOLATILE STRICT AS $BODY$ SELECT sum(total_bytes)::bigint FROM @extschema@.hypertable_approximate_detailed_size(hypertable); $BODY$ SET search_path TO pg_catalog, pg_temp; DROP FUNCTION IF EXISTS @extschema@.compress_chunk; CREATE FUNCTION @extschema@.compress_chunk(uncompressed_chunk REGCLASS, if_not_compressed BOOLEAN = true, recompress BOOLEAN = false) RETURNS REGCLASS AS '' LANGUAGE SQL SET search_path TO pg_catalog, pg_temp; ================================================ FILE: sql/updates/2.14.0--2.13.1.sql ================================================ -- check whether we can safely downgrade compression setup DO $$ DECLARE hypertable regclass; ht_uncomp regclass; chunk_relids oid[]; ht_id integer; BEGIN FOR hypertable, ht_uncomp, ht_id IN SELECT format('%I.%I',ht.schema_name,ht.table_name)::regclass, format('%I.%I',ht_uncomp.schema_name,ht_uncomp.table_name)::regclass, ht.id FROM _timescaledb_catalog.hypertable ht_uncomp INNER JOIN _timescaledb_catalog.hypertable ht ON ht.id = ht_uncomp.compressed_hypertable_id LOOP -- hypertables need to at least have 1 compressed chunk so we can restore the columns IF NOT EXISTS(SELECT FROM _timescaledb_catalog.chunk WHERE hypertable_id = ht_id) THEN RAISE USING ERRCODE = 'feature_not_supported', MESSAGE = 'Cannot downgrade compressed hypertables with no compressed chunks. Disable compression on the affected hypertable before downgrading.', DETAIL = 'The following hypertable is affected: '|| ht_uncomp::text; END IF; chunk_relids := array(SELECT format('%I.%I',schema_name,table_name)::regclass FROM _timescaledb_catalog.chunk WHERE hypertable_id = ht_id); -- any hypertable with distinct compression settings cannot be downgraded IF EXISTS ( SELECT FROM ( SELECT DISTINCT segmentby, orderby, orderby_desc, orderby_nullsfirst FROM _timescaledb_catalog.compression_settings WHERE relid = hypertable OR relid = ANY(chunk_relids) ) dist_settings HAVING count(*) > 1 ) THEN RAISE USING ERRCODE = 'feature_not_supported', MESSAGE = 'Cannot downgrade hypertables with distinct compression settings. Decompress the affected hypertable before downgrading.', DETAIL = 'The following hypertable is affected: '|| ht_uncomp::text; END IF; END LOOP; END $$; CREATE FUNCTION _timescaledb_functions.tmp_resolve_indkeys(oid,int2[]) RETURNS text[] LANGUAGE SQL AS $$ SELECT array_agg(attname) FROM ( SELECT attname FROM (SELECT unnest($2) attnum) indkeys JOIN LATERAL ( SELECT attname FROM pg_attribute att WHERE att.attnum=indkeys.attnum AND att.attrelid=$1 ) r ON true ) resolve; $$ SET search_path TO pg_catalog, pg_temp; DO $$ DECLARE chunk regclass; hypertable regclass; ht_id integer; chunk_id integer; _index regclass; ht_index regclass; chunk_index regclass; index_name name; chunk_index_name name; _indkey text[]; column_name name; column_type regtype; cmd text; BEGIN SET timescaledb.restoring TO ON; FOR hypertable, ht_id IN SELECT format('%I.%I',ht.schema_name,ht.table_name)::regclass, ht.id FROM _timescaledb_catalog.hypertable ht_uncomp INNER JOIN _timescaledb_catalog.hypertable ht ON ht.id = ht_uncomp.compressed_hypertable_id LOOP -- get first chunk which we use as template for restoring columns and indexes SELECT format('%I.%I',schema_name,table_name)::regclass INTO STRICT chunk FROM _timescaledb_catalog.chunk WHERE hypertable_id = ht_id ORDER by id LIMIT 1; -- restore columns from the compressed hypertable FOR column_name, column_type IN SELECT attname, atttypid::regtype FROM pg_attribute WHERE attrelid = chunk AND attnum > 0 AND NOT attisdropped LOOP cmd := format('ALTER TABLE %s ADD COLUMN %I %s', hypertable, column_name, column_type); EXECUTE cmd; END LOOP; -- restore indexes on the compressed hypertable FOR _index, _indkey IN SELECT indexrelid::regclass, _timescaledb_functions.tmp_resolve_indkeys(indrelid, indkey) FROM pg_index WHERE indrelid = chunk LOOP SELECT relname INTO STRICT index_name FROM pg_class WHERE oid = _index; cmd := pg_get_indexdef(_index); cmd := replace(cmd, format(' INDEX %s ON ', index_name), ' INDEX ON '); cmd := replace(cmd, chunk::text, hypertable::text); EXECUTE cmd; -- get indexrelid of index we just created on hypertable SELECT indexrelid INTO STRICT ht_index FROM pg_index WHERE indrelid = hypertable AND _timescaledb_functions.tmp_resolve_indkeys(hypertable, indkey) = _indkey; SELECT relname INTO STRICT index_name FROM pg_class WHERE oid = ht_index; -- restore indexes in our catalog FOR chunk, chunk_id IN SELECT format('%I.%I',schema_name,table_name)::regclass, id FROM _timescaledb_catalog.chunk WHERE hypertable_id = ht_id LOOP SELECT indexrelid INTO STRICT chunk_index FROM pg_index WHERE indrelid = chunk AND _timescaledb_functions.tmp_resolve_indkeys(chunk, indkey) = _indkey; SELECT relname INTO STRICT chunk_index_name FROM pg_class WHERE oid = chunk_index; INSERT INTO _timescaledb_catalog.chunk_index (chunk_id, index_name, hypertable_id, hypertable_index_name) VALUES (chunk_id, chunk_index_name, ht_id, index_name); END LOOP; END LOOP; -- restore inheritance cmd := format('ALTER TABLE %s INHERIT %s', chunk, hypertable); EXECUTE cmd; END LOOP; SET timescaledb.restoring TO OFF; END $$; DROP FUNCTION _timescaledb_functions.tmp_resolve_indkeys; CREATE FUNCTION _timescaledb_functions.ping_data_node(node_name NAME, timeout INTERVAL = NULL) RETURNS BOOLEAN AS '@MODULE_PATHNAME@', 'ts_data_node_ping' LANGUAGE C VOLATILE; CREATE FUNCTION _timescaledb_functions.remote_txn_heal_data_node(foreign_server_oid oid) RETURNS INT AS '@MODULE_PATHNAME@', 'ts_remote_txn_heal_data_node' LANGUAGE C STRICT; CREATE FUNCTION _timescaledb_functions.set_dist_id(dist_id UUID) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_dist_set_id' LANGUAGE C VOLATILE STRICT; CREATE FUNCTION _timescaledb_functions.set_peer_dist_id(dist_id UUID) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_dist_set_peer_id' LANGUAGE C VOLATILE STRICT; -- Function to validate that a node has local settings to function as -- a data node. Throws error if validation fails. CREATE FUNCTION _timescaledb_functions.validate_as_data_node() RETURNS void AS '@MODULE_PATHNAME@', 'ts_dist_validate_as_data_node' LANGUAGE C VOLATILE STRICT; CREATE FUNCTION _timescaledb_functions.show_connection_cache() RETURNS TABLE ( node_name name, user_name name, host text, port int, database name, backend_pid int, connection_status text, transaction_status text, transaction_depth int, processing boolean, invalidated boolean) AS '@MODULE_PATHNAME@', 'ts_remote_connection_cache_show' LANGUAGE C VOLATILE STRICT; DROP FUNCTION IF EXISTS @extschema@.create_hypertable(relation REGCLASS, time_column_name NAME, partitioning_column NAME, number_partitions INTEGER, associated_schema_name NAME, associated_table_prefix NAME, chunk_time_interval ANYELEMENT, create_default_indexes BOOLEAN, if_not_exists BOOLEAN, partitioning_func REGPROC, migrate_data BOOLEAN, chunk_target_size TEXT, chunk_sizing_func REGPROC, time_partitioning_func REGPROC); CREATE FUNCTION @extschema@.create_hypertable( relation REGCLASS, time_column_name NAME, partitioning_column NAME = NULL, number_partitions INTEGER = NULL, associated_schema_name NAME = NULL, associated_table_prefix NAME = NULL, chunk_time_interval ANYELEMENT = NULL::bigint, create_default_indexes BOOLEAN = TRUE, if_not_exists BOOLEAN = FALSE, partitioning_func REGPROC = NULL, migrate_data BOOLEAN = FALSE, chunk_target_size TEXT = NULL, chunk_sizing_func REGPROC = '_timescaledb_functions.calculate_chunk_interval'::regproc, time_partitioning_func REGPROC = NULL, replication_factor INTEGER = NULL, data_nodes NAME[] = NULL, distributed BOOLEAN = NULL ) RETURNS TABLE(hypertable_id INT, schema_name NAME, table_name NAME, created BOOL) AS '@MODULE_PATHNAME@', 'ts_hypertable_create' LANGUAGE C VOLATILE; CREATE FUNCTION @extschema@.create_distributed_hypertable( relation REGCLASS, time_column_name NAME, partitioning_column NAME = NULL, number_partitions INTEGER = NULL, associated_schema_name NAME = NULL, associated_table_prefix NAME = NULL, chunk_time_interval ANYELEMENT = NULL::bigint, create_default_indexes BOOLEAN = TRUE, if_not_exists BOOLEAN = FALSE, partitioning_func REGPROC = NULL, migrate_data BOOLEAN = FALSE, chunk_target_size TEXT = NULL, chunk_sizing_func REGPROC = '_timescaledb_functions.calculate_chunk_interval'::regproc, time_partitioning_func REGPROC = NULL, replication_factor INTEGER = NULL, data_nodes NAME[] = NULL ) RETURNS TABLE(hypertable_id INT, schema_name NAME, table_name NAME, created BOOL) AS '@MODULE_PATHNAME@', 'ts_hypertable_distributed_create' LANGUAGE C VOLATILE; CREATE FUNCTION @extschema@.add_data_node( node_name NAME, host TEXT, database NAME = NULL, port INTEGER = NULL, if_not_exists BOOLEAN = FALSE, bootstrap BOOLEAN = TRUE, password TEXT = NULL ) RETURNS TABLE(node_name NAME, host TEXT, port INTEGER, database NAME, node_created BOOL, database_created BOOL, extension_created BOOL) AS '@MODULE_PATHNAME@', 'ts_data_node_add' LANGUAGE C VOLATILE; CREATE FUNCTION @extschema@.delete_data_node( node_name NAME, if_exists BOOLEAN = FALSE, force BOOLEAN = FALSE, repartition BOOLEAN = TRUE, drop_database BOOLEAN = FALSE ) RETURNS BOOLEAN AS '@MODULE_PATHNAME@', 'ts_data_node_delete' LANGUAGE C VOLATILE; CREATE FUNCTION @extschema@.attach_data_node( node_name NAME, hypertable REGCLASS, if_not_attached BOOLEAN = FALSE, repartition BOOLEAN = TRUE ) RETURNS TABLE(hypertable_id INTEGER, node_hypertable_id INTEGER, node_name NAME) AS '@MODULE_PATHNAME@', 'ts_data_node_attach' LANGUAGE C VOLATILE; CREATE FUNCTION @extschema@.detach_data_node( node_name NAME, hypertable REGCLASS = NULL, if_attached BOOLEAN = FALSE, force BOOLEAN = FALSE, repartition BOOLEAN = TRUE, drop_remote_data BOOLEAN = FALSE ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_data_node_detach' LANGUAGE C VOLATILE; CREATE FUNCTION @extschema@.alter_data_node( node_name NAME, host TEXT = NULL, database NAME = NULL, port INTEGER = NULL, available BOOLEAN = NULL ) RETURNS TABLE(node_name NAME, host TEXT, port INTEGER, database NAME, available BOOLEAN) AS '@MODULE_PATHNAME@', 'ts_data_node_alter' LANGUAGE C VOLATILE; CREATE PROCEDURE @extschema@.distributed_exec( query TEXT, node_list name[] = NULL, transactional BOOLEAN = TRUE) AS '@MODULE_PATHNAME@', 'ts_distributed_exec' LANGUAGE C; CREATE FUNCTION @extschema@.create_distributed_restore_point( name TEXT ) RETURNS TABLE(node_name NAME, node_type TEXT, restore_point pg_lsn) AS '@MODULE_PATHNAME@', 'ts_create_distributed_restore_point' LANGUAGE C VOLATILE STRICT; CREATE FUNCTION @extschema@.set_replication_factor( hypertable REGCLASS, replication_factor INTEGER ) RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_hypertable_distributed_set_replication_factor' LANGUAGE C VOLATILE; CREATE TABLE _timescaledb_catalog.hypertable_compression ( hypertable_id integer NOT NULL, attname name NOT NULL, compression_algorithm_id smallint, segmentby_column_index smallint, orderby_column_index smallint, orderby_asc boolean, orderby_nullsfirst boolean, -- table constraints CONSTRAINT hypertable_compression_pkey PRIMARY KEY (hypertable_id, attname), CONSTRAINT hypertable_compression_hypertable_id_orderby_column_index_key UNIQUE (hypertable_id, orderby_column_index), CONSTRAINT hypertable_compression_hypertable_id_segmentby_column_index_key UNIQUE (hypertable_id, segmentby_column_index), CONSTRAINT hypertable_compression_compression_algorithm_id_fkey FOREIGN KEY (compression_algorithm_id) REFERENCES _timescaledb_catalog.compression_algorithm (id) ); INSERT INTO _timescaledb_catalog.hypertable_compression( hypertable_id, attname, compression_algorithm_id, segmentby_column_index, orderby_column_index, orderby_asc, orderby_nullsfirst ) SELECT ht.id, att.attname, CASE WHEN att.attname = ANY(cs.segmentby) THEN 0 WHEN att.atttypid IN ('numeric'::regtype) THEN 1 WHEN att.atttypid IN ('float4'::regtype,'float8'::regtype) THEN 3 WHEN att.atttypid IN ('int2'::regtype,'int4'::regtype,'int8'::regtype,'date'::regtype,'timestamp'::regtype,'timestamptz'::regtype) THEN 4 WHEN EXISTS(SELECT FROM pg_operator op WHERE op.oprname = '=' AND op.oprkind = 'b' AND op.oprcanhash = true AND op.oprleft = att.atttypid AND op.oprright = att.atttypid) THEN 2 ELSE 1 END AS compression_algorithm_id, CASE WHEN att.attname = ANY(cs.segmentby) THEN array_position(cs.segmentby, att.attname::text) ELSE NULL END AS segmentby_column_index, CASE WHEN att.attname = ANY(cs.orderby) THEN array_position(cs.orderby, att.attname::text) ELSE NULL END AS orderby_column_index, CASE WHEN att.attname = ANY(cs.orderby) THEN NOT cs.orderby_desc[array_position(cs.orderby, att.attname::text)] ELSE false END AS orderby_asc, CASE WHEN att.attname = ANY(cs.orderby) THEN cs.orderby_nullsfirst[array_position(cs.orderby, att.attname::text)] ELSE false END AS orderby_nullsfirst FROM _timescaledb_catalog.hypertable ht INNER JOIN _timescaledb_catalog.compression_settings cs ON cs.relid = format('%I.%I',ht.schema_name,ht.table_name)::regclass LEFT JOIN pg_attribute att ON att.attrelid = format('%I.%I',ht.schema_name,ht.table_name)::regclass AND attnum > 0 WHERE compressed_hypertable_id IS NOT NULL; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable_compression', ''); GRANT SELECT ON _timescaledb_catalog.hypertable_compression TO PUBLIC; ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.compression_settings; DROP VIEW timescaledb_information.compression_settings; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.compression_settings; DROP TABLE _timescaledb_catalog.compression_settings; CREATE FUNCTION @extschema@.timescaledb_fdw_handler() RETURNS fdw_handler AS '@MODULE_PATHNAME@', 'ts_timescaledb_fdw_handler' LANGUAGE C STRICT; CREATE FUNCTION @extschema@.timescaledb_fdw_validator(text[], oid) RETURNS void AS '@MODULE_PATHNAME@', 'ts_timescaledb_fdw_validator' LANGUAGE C STRICT; CREATE FOREIGN DATA WRAPPER timescaledb_fdw HANDLER @extschema@.timescaledb_fdw_handler VALIDATOR @extschema@.timescaledb_fdw_validator; CREATE FUNCTION _timescaledb_functions.create_chunk_replica_table( chunk REGCLASS, data_node_name NAME ) RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_chunk_create_replica_table' LANGUAGE C VOLATILE; CREATE FUNCTION _timescaledb_functions.chunk_drop_replica( chunk REGCLASS, node_name NAME ) RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_chunk_drop_replica' LANGUAGE C VOLATILE; CREATE PROCEDURE _timescaledb_functions.wait_subscription_sync( schema_name NAME, table_name NAME, retry_count INT DEFAULT 18000, retry_delay_ms NUMERIC DEFAULT 0.200 ) LANGUAGE PLPGSQL AS $BODY$ DECLARE in_sync BOOLEAN; BEGIN FOR i in 1 .. retry_count LOOP SELECT pgs.srsubstate = 'r' INTO in_sync FROM pg_subscription_rel pgs JOIN pg_class pgc ON relname = table_name JOIN pg_namespace n ON (n.OID = pgc.relnamespace) WHERE pgs.srrelid = pgc.oid AND schema_name = n.nspname; if (in_sync IS NULL OR NOT in_sync) THEN PERFORM pg_sleep(retry_delay_ms); ELSE RETURN; END IF; END LOOP; RAISE 'subscription sync wait timedout'; END $BODY$ SET search_path TO pg_catalog, pg_temp; CREATE FUNCTION _timescaledb_functions.health() RETURNS TABLE (node_name NAME, healthy BOOL, in_recovery BOOL, error TEXT) AS '@MODULE_PATHNAME@', 'ts_health_check' LANGUAGE C VOLATILE; CREATE FUNCTION _timescaledb_functions.drop_stale_chunks( node_name NAME, chunks integer[] = NULL ) RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_chunks_drop_stale' LANGUAGE C VOLATILE; CREATE FUNCTION _timescaledb_functions.rxid_in(cstring) RETURNS @extschema@.rxid AS '@MODULE_PATHNAME@', 'ts_remote_txn_id_in' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION _timescaledb_functions.rxid_out(@extschema@.rxid) RETURNS cstring AS '@MODULE_PATHNAME@', 'ts_remote_txn_id_out' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE TYPE @extschema@.rxid ( internallength = 16, input = _timescaledb_functions.rxid_in, output = _timescaledb_functions.rxid_out ); CREATE FUNCTION _timescaledb_functions.data_node_hypertable_info( node_name NAME, schema_name_in name, table_name_in name ) RETURNS TABLE ( table_bytes bigint, index_bytes bigint, toast_bytes bigint, total_bytes bigint) AS '@MODULE_PATHNAME@', 'ts_dist_remote_hypertable_info' LANGUAGE C VOLATILE STRICT; CREATE FUNCTION _timescaledb_functions.data_node_chunk_info( node_name NAME, schema_name_in name, table_name_in name ) RETURNS TABLE ( chunk_id integer, chunk_schema name, chunk_name name, table_bytes bigint, index_bytes bigint, toast_bytes bigint, total_bytes bigint) AS '@MODULE_PATHNAME@', 'ts_dist_remote_chunk_info' LANGUAGE C VOLATILE STRICT; CREATE FUNCTION _timescaledb_functions.data_node_compressed_chunk_stats(node_name name, schema_name_in name, table_name_in name) RETURNS TABLE ( chunk_schema name, chunk_name name, compression_status text, before_compression_table_bytes bigint, before_compression_index_bytes bigint, before_compression_toast_bytes bigint, before_compression_total_bytes bigint, after_compression_table_bytes bigint, after_compression_index_bytes bigint, after_compression_toast_bytes bigint, after_compression_total_bytes bigint ) AS '@MODULE_PATHNAME@' , 'ts_dist_remote_compressed_chunk_info' LANGUAGE C VOLATILE STRICT; CREATE FUNCTION _timescaledb_functions.data_node_index_size(node_name name, schema_name_in name, index_name_in name) RETURNS TABLE ( hypertable_id INTEGER, total_bytes BIGINT) AS '@MODULE_PATHNAME@' , 'ts_dist_remote_hypertable_index_info' LANGUAGE C VOLATILE STRICT; CREATE FUNCTION timescaledb_experimental.block_new_chunks(data_node_name NAME, hypertable REGCLASS = NULL, force BOOLEAN = FALSE) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_data_node_block_new_chunks' LANGUAGE C VOLATILE; CREATE FUNCTION timescaledb_experimental.allow_new_chunks(data_node_name NAME, hypertable REGCLASS = NULL) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_data_node_allow_new_chunks' LANGUAGE C VOLATILE; CREATE PROCEDURE timescaledb_experimental.move_chunk( chunk REGCLASS, source_node NAME = NULL, destination_node NAME = NULL, operation_id NAME = NULL) AS '@MODULE_PATHNAME@', 'ts_move_chunk_proc' LANGUAGE C; CREATE PROCEDURE timescaledb_experimental.copy_chunk( chunk REGCLASS, source_node NAME = NULL, destination_node NAME = NULL, operation_id NAME = NULL) AS '@MODULE_PATHNAME@', 'ts_copy_chunk_proc' LANGUAGE C; CREATE FUNCTION timescaledb_experimental.subscription_exec( subscription_command TEXT ) RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_subscription_exec' LANGUAGE C VOLATILE; CREATE PROCEDURE timescaledb_experimental.cleanup_copy_chunk_operation( operation_id NAME) AS '@MODULE_PATHNAME@', 'ts_copy_chunk_cleanup_proc' LANGUAGE C; CREATE FUNCTION _timescaledb_functions.set_chunk_default_data_node(chunk REGCLASS, node_name NAME) RETURNS BOOLEAN AS '@MODULE_PATHNAME@', 'ts_chunk_set_default_data_node' LANGUAGE C VOLATILE; CREATE FUNCTION _timescaledb_functions.drop_dist_ht_invalidation_trigger( raw_hypertable_id INTEGER ) RETURNS VOID AS '@MODULE_PATHNAME@', 'ts_drop_dist_ht_invalidation_trigger' LANGUAGE C STRICT VOLATILE; -- restore multinode catalog tables CREATE TABLE _timescaledb_catalog.remote_txn ( data_node_name name, --this is really only to allow us to cleanup stuff on a per-node basis. remote_transaction_id text NOT NULL, -- table constraints CONSTRAINT remote_txn_pkey PRIMARY KEY (remote_transaction_id) ); ALTER TABLE _timescaledb_catalog.remote_txn ADD CONSTRAINT remote_txn_remote_transaction_id_check CHECK (remote_transaction_id::@extschema@.rxid IS NOT NULL); CREATE INDEX remote_txn_data_node_name_idx ON _timescaledb_catalog.remote_txn (data_node_name); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.remote_txn', ''); GRANT SELECT ON TABLE _timescaledb_catalog.remote_txn TO PUBLIC; CREATE TABLE _timescaledb_catalog.hypertable_data_node ( hypertable_id integer NOT NULL, node_hypertable_id integer NULL, node_name name NOT NULL, block_chunks boolean NOT NULL, -- table constraints CONSTRAINT hypertable_data_node_hypertable_id_node_name_key UNIQUE (hypertable_id, node_name), CONSTRAINT hypertable_data_node_node_hypertable_id_node_name_key UNIQUE (node_hypertable_id, node_name), CONSTRAINT hypertable_data_node_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable_data_node', ''); GRANT SELECT ON TABLE _timescaledb_catalog.hypertable_data_node TO PUBLIC; CREATE TABLE _timescaledb_catalog.chunk_data_node ( chunk_id integer NOT NULL, node_chunk_id integer NOT NULL, node_name name NOT NULL, -- table constraints CONSTRAINT chunk_data_node_chunk_id_node_name_key UNIQUE (chunk_id, node_name), CONSTRAINT chunk_data_node_node_chunk_id_node_name_key UNIQUE (node_chunk_id, node_name), CONSTRAINT chunk_data_node_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ); CREATE INDEX chunk_data_node_node_name_idx ON _timescaledb_catalog.chunk_data_node (node_name); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk_data_node', ''); GRANT SELECT ON TABLE _timescaledb_catalog.chunk_data_node TO PUBLIC; CREATE SEQUENCE _timescaledb_catalog.chunk_copy_operation_id_seq MINVALUE 1; GRANT SELECT ON SEQUENCE _timescaledb_catalog.chunk_copy_operation_id_seq TO PUBLIC; CREATE TABLE _timescaledb_catalog.chunk_copy_operation ( operation_id name NOT NULL, -- the publisher/subscriber identifier used backend_pid integer NOT NULL, -- the pid of the backend running this activity completed_stage name NOT NULL, -- the completed stage/step time_start timestamptz NOT NULL DEFAULT NOW(), -- start time of the activity chunk_id integer NOT NULL, compress_chunk_name name NOT NULL, source_node_name name NOT NULL, dest_node_name name NOT NULL, delete_on_source_node bool NOT NULL, -- is a move or copy activity -- table constraints CONSTRAINT chunk_copy_operation_pkey PRIMARY KEY (operation_id), CONSTRAINT chunk_copy_operation_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ON DELETE CASCADE ); GRANT SELECT ON TABLE _timescaledb_catalog.chunk_copy_operation TO PUBLIC; CREATE TABLE _timescaledb_catalog.dimension_partition ( dimension_id integer NOT NULL REFERENCES _timescaledb_catalog.dimension (id) ON DELETE CASCADE, range_start bigint NOT NULL, data_nodes name[] NULL, UNIQUE (dimension_id, range_start) ); GRANT SELECT ON TABLE _timescaledb_catalog.dimension_partition TO PUBLIC; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.dimension_partition', ''); CREATE FUNCTION _timescaledb_functions.hypertable_remote_size( schema_name_in name, table_name_in name) RETURNS TABLE ( table_bytes bigint, index_bytes bigint, toast_bytes bigint, total_bytes bigint, node_name NAME) LANGUAGE SQL VOLATILE STRICT AS $BODY$ $BODY$ SET search_path TO pg_catalog, pg_temp; CREATE FUNCTION _timescaledb_functions.chunks_remote_size( schema_name_in name, table_name_in name) RETURNS TABLE ( chunk_id integer, chunk_schema NAME, chunk_name NAME, table_bytes bigint, index_bytes bigint, toast_bytes bigint, total_bytes bigint, node_name NAME) LANGUAGE SQL VOLATILE STRICT AS $BODY$ $BODY$ SET search_path TO pg_catalog, pg_temp; CREATE FUNCTION _timescaledb_functions.indexes_remote_size( schema_name_in NAME, table_name_in NAME, index_name_in NAME ) RETURNS BIGINT LANGUAGE SQL VOLATILE STRICT AS $BODY$ $BODY$ SET search_path TO pg_catalog, pg_temp; CREATE FUNCTION _timescaledb_functions.compressed_chunk_remote_stats(schema_name_in name, table_name_in name) RETURNS TABLE ( chunk_schema name, chunk_name name, compression_status text, before_compression_table_bytes bigint, before_compression_index_bytes bigint, before_compression_toast_bytes bigint, before_compression_total_bytes bigint, after_compression_table_bytes bigint, after_compression_index_bytes bigint, after_compression_toast_bytes bigint, after_compression_total_bytes bigint, node_name name) LANGUAGE SQL STABLE STRICT AS $BODY$ $BODY$ SET search_path TO pg_catalog, pg_temp; -- recreate the _timescaledb_catalog.hypertable table as new field was added -- 1. drop CONSTRAINTS from other tables referencing the existing one ALTER TABLE _timescaledb_config.bgw_job DROP CONSTRAINT bgw_job_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.chunk DROP CONSTRAINT chunk_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.chunk_index DROP CONSTRAINT chunk_index_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.continuous_agg DROP CONSTRAINT continuous_agg_mat_hypertable_id_fkey, DROP CONSTRAINT continuous_agg_raw_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.continuous_aggs_bucket_function DROP CONSTRAINT continuous_aggs_bucket_function_mat_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.continuous_aggs_invalidation_threshold DROP CONSTRAINT continuous_aggs_invalidation_threshold_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.dimension DROP CONSTRAINT dimension_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.hypertable DROP CONSTRAINT hypertable_compressed_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.hypertable_data_node DROP CONSTRAINT hypertable_data_node_hypertable_id_fkey; ALTER TABLE _timescaledb_catalog.tablespace DROP CONSTRAINT tablespace_hypertable_id_fkey; -- drop dependent views ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.hypertables; ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.job_stats; ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.jobs; ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.continuous_aggregates; ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.chunks; ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.dimensions; ALTER EXTENSION timescaledb DROP VIEW _timescaledb_internal.hypertable_chunk_local_size; ALTER EXTENSION timescaledb DROP VIEW _timescaledb_internal.compressed_chunk_stats; ALTER EXTENSION timescaledb DROP VIEW timescaledb_experimental.policies; DROP VIEW timescaledb_information.hypertables; DROP VIEW timescaledb_information.job_stats; DROP VIEW timescaledb_information.jobs; DROP VIEW timescaledb_information.continuous_aggregates; DROP VIEW timescaledb_information.chunks; DROP VIEW timescaledb_information.dimensions; DROP VIEW _timescaledb_internal.hypertable_chunk_local_size; DROP VIEW _timescaledb_internal.compressed_chunk_stats; DROP VIEW timescaledb_experimental.policies; -- recreate table CREATE TABLE _timescaledb_catalog.hypertable_tmp AS SELECT * FROM _timescaledb_catalog.hypertable; CREATE TABLE _timescaledb_catalog.tmp_hypertable_seq_value AS SELECT last_value, is_called FROM _timescaledb_catalog.hypertable_id_seq; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.hypertable; ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_catalog.hypertable_id_seq; SET timescaledb.restoring = on; -- must disable the hooks otherwise we can't do anything without the table _timescaledb_catalog.hypertable DROP TABLE _timescaledb_catalog.hypertable; CREATE SEQUENCE _timescaledb_catalog.hypertable_id_seq MINVALUE 1; SELECT setval('_timescaledb_catalog.hypertable_id_seq', last_value, is_called) FROM _timescaledb_catalog.tmp_hypertable_seq_value; DROP TABLE _timescaledb_catalog.tmp_hypertable_seq_value; CREATE TABLE _timescaledb_catalog.hypertable ( id INTEGER PRIMARY KEY NOT NULL DEFAULT nextval('_timescaledb_catalog.hypertable_id_seq'), schema_name name NOT NULL, table_name name NOT NULL, associated_schema_name name NOT NULL, associated_table_prefix name NOT NULL, num_dimensions smallint NOT NULL, chunk_sizing_func_schema name NOT NULL, chunk_sizing_func_name name NOT NULL, chunk_target_size bigint NOT NULL, -- size in bytes compression_state smallint NOT NULL DEFAULT 0, compressed_hypertable_id integer, replication_factor smallint NULL, status int NOT NULL DEFAULT 0 ); SET timescaledb.restoring = off; INSERT INTO _timescaledb_catalog.hypertable ( id, schema_name, table_name, associated_schema_name, associated_table_prefix, num_dimensions, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size, compression_state, compressed_hypertable_id ) SELECT id, schema_name, table_name, associated_schema_name, associated_table_prefix, num_dimensions, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size, compression_state, compressed_hypertable_id FROM _timescaledb_catalog.hypertable_tmp ORDER BY id; ALTER SEQUENCE _timescaledb_catalog.hypertable_id_seq OWNED BY _timescaledb_catalog.hypertable.id; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable_id_seq', ''); GRANT SELECT ON _timescaledb_catalog.hypertable TO PUBLIC; GRANT SELECT ON _timescaledb_catalog.hypertable_id_seq TO PUBLIC; DROP TABLE _timescaledb_catalog.hypertable_tmp; -- now add any constraints ALTER TABLE _timescaledb_catalog.hypertable -- ADD CONSTRAINT hypertable_pkey PRIMARY KEY (id), ADD CONSTRAINT hypertable_associated_schema_name_associated_table_prefix_key UNIQUE (associated_schema_name, associated_table_prefix), ADD CONSTRAINT hypertable_table_name_schema_name_key UNIQUE (table_name, schema_name), ADD CONSTRAINT hypertable_schema_name_check CHECK (schema_name != '_timescaledb_catalog'), -- internal compressed hypertables have compression state = 2 ADD CONSTRAINT hypertable_dim_compress_check CHECK (num_dimensions > 0 OR compression_state = 2), ADD CONSTRAINT hypertable_chunk_target_size_check CHECK (chunk_target_size >= 0), ADD CONSTRAINT hypertable_compress_check CHECK ( (compression_state = 0 OR compression_state = 1 ) OR (compression_state = 2 AND compressed_hypertable_id IS NULL)), ADD CONSTRAINT hypertable_replication_factor_check CHECK (replication_factor > 0 OR replication_factor = -1), ADD CONSTRAINT hypertable_compressed_hypertable_id_fkey FOREIGN KEY (compressed_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id); GRANT SELECT ON TABLE _timescaledb_catalog.hypertable TO PUBLIC; -- 3. reestablish constraints on other tables ALTER TABLE _timescaledb_config.bgw_job ADD CONSTRAINT bgw_job_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.chunk ADD CONSTRAINT chunk_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id); ALTER TABLE _timescaledb_catalog.chunk_index ADD CONSTRAINT chunk_index_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.continuous_agg ADD CONSTRAINT continuous_agg_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE, ADD CONSTRAINT continuous_agg_raw_hypertable_id_fkey FOREIGN KEY (raw_hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.continuous_aggs_bucket_function ADD CONSTRAINT continuous_aggs_bucket_function_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.continuous_aggs_invalidation_threshold ADD CONSTRAINT continuous_aggs_invalidation_threshold_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.dimension ADD CONSTRAINT dimension_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.hypertable_compression ADD CONSTRAINT hypertable_compression_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.hypertable_data_node ADD CONSTRAINT hypertable_data_node_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id); ALTER TABLE _timescaledb_catalog.tablespace ADD CONSTRAINT tablespace_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) ON DELETE CASCADE; DROP FUNCTION IF EXISTS _timescaledb_debug.extension_state; DROP SCHEMA IF EXISTS _timescaledb_debug; DROP FUNCTION IF EXISTS _timescaledb_internal.hypertable_constraint_add_table_fk_constraint; DROP FUNCTION _timescaledb_functions.constraint_clone; CREATE FUNCTION _timescaledb_functions.hypertable_constraint_add_table_fk_constraint(user_ht_constraint_name name,user_ht_schema_name name,user_ht_table_name name,compress_ht_id integer) RETURNS void LANGUAGE PLPGSQL AS $$BEGIN END$$ SET search_path TO pg_catalog,pg_temp; CREATE FUNCTION _timescaledb_functions.chunks_in(record RECORD, chunks INTEGER[]) RETURNS BOOL AS 'BEGIN END' LANGUAGE PLPGSQL SET search_path TO pg_catalog,pg_temp; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.metadata', $$ WHERE KEY = 'exported_uuid' $$); DROP TRIGGER metadata_insert_trigger ON _timescaledb_catalog.metadata; DROP FUNCTION _timescaledb_functions.metadata_insert_trigger(); DROP FUNCTION IF EXISTS _timescaledb_functions.get_orderby_defaults(regclass,text[]); DROP FUNCTION IF EXISTS _timescaledb_functions.get_segmentby_defaults(regclass); --- re-include in the pg_dump config SELECT pg_catalog.pg_extension_config_dump('_timescaledb_cache.cache_inval_hypertable', ''); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_cache.cache_inval_extension', ''); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_cache.cache_inval_bgw_job', ''); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_internal.job_errors', ''); -- Remove unwanted entry from extconfig and extcondition in pg_extension ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.hypertable; -- Associate the above table back to keep the dependencies safe ALTER EXTENSION timescaledb ADD TABLE _timescaledb_catalog.hypertable; -- include this now in the config SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.hypertable', ''); DROP FUNCTION IF EXISTS _timescaledb_functions.relation_approximate_size(relation REGCLASS); DROP FUNCTION IF EXISTS @extschema@.hypertable_approximate_detailed_size(relation REGCLASS); DROP FUNCTION IF EXISTS @extschema@.hypertable_approximate_size(hypertable REGCLASS); DROP FUNCTION IF EXISTS @extschema@.compress_chunk; CREATE FUNCTION @extschema@.compress_chunk(uncompressed_chunk REGCLASS, if_not_compressed BOOLEAN = true) RETURNS REGCLASS AS '' LANGUAGE SQL SET search_path TO pg_catalog,pg_temp; ================================================ FILE: sql/updates/2.14.0--2.14.1.sql ================================================ CREATE VIEW timescaledb_information.hypertable_compression_settings AS SELECT format('%I.%I',ht.schema_name,ht.table_name)::regclass AS hypertable, array_to_string(segmentby,',') AS segmentby, un.orderby, d.compress_interval_length FROM _timescaledb_catalog.hypertable ht JOIN LATERAL ( SELECT CASE WHEN d.column_type = ANY(ARRAY['timestamp','timestamptz','date']::regtype[]) THEN _timescaledb_functions.to_interval(d.compress_interval_length)::text ELSE d.compress_interval_length::text END AS compress_interval_length FROM _timescaledb_catalog.dimension d WHERE d.hypertable_id = ht.id ORDER BY id LIMIT 1 ) d ON true LEFT JOIN _timescaledb_catalog.compression_settings s ON format('%I.%I',ht.schema_name,ht.table_name)::regclass = s.relid LEFT JOIN LATERAL ( SELECT string_agg( format('%I%s%s',orderby, CASE WHEN "desc" THEN ' DESC' ELSE '' END, CASE WHEN nullsfirst AND NOT "desc" THEN ' NULLS FIRST' WHEN NOT nullsfirst AND "desc" THEN ' NULLS LAST' ELSE '' END ) ,',') AS orderby FROM unnest(s.orderby, s.orderby_desc, s.orderby_nullsfirst) un(orderby, "desc", nullsfirst) ) un ON true; CREATE VIEW timescaledb_information.chunk_compression_settings AS SELECT format('%I.%I',ht.schema_name,ht.table_name)::regclass AS hypertable, format('%I.%I',ch.schema_name,ch.table_name)::regclass AS chunk, array_to_string(segmentby,',') AS segmentby, un.orderby FROM _timescaledb_catalog.hypertable ht INNER JOIN _timescaledb_catalog.chunk ch ON ch.hypertable_id = ht.id INNER JOIN _timescaledb_catalog.chunk ch2 ON ch2.id = ch.compressed_chunk_id LEFT JOIN _timescaledb_catalog.compression_settings s ON format('%I.%I',ch2.schema_name,ch2.table_name)::regclass = s.relid LEFT JOIN LATERAL ( SELECT string_agg( format('%I%s%s',orderby, CASE WHEN "desc" THEN ' DESC' ELSE '' END, CASE WHEN nullsfirst AND NOT "desc" THEN ' NULLS FIRST' WHEN NOT nullsfirst AND "desc" THEN ' NULLS LAST' ELSE '' END ),',') AS orderby FROM unnest(s.orderby, s.orderby_desc, s.orderby_nullsfirst) un(orderby, "desc", nullsfirst) ) un ON true; INSERT INTO _timescaledb_catalog.compression_settings SELECT format('%I.%I',ch.schema_name,ch.table_name)::regclass,s.segmentby,s.orderby,s.orderby_desc,s.orderby_nullsfirst FROM _timescaledb_catalog.hypertable ht1 INNER JOIN _timescaledb_catalog.hypertable ht2 ON ht2.id = ht1.compressed_hypertable_id INNER JOIN _timescaledb_catalog.compression_settings s ON s.relid = format('%I.%I',ht1.schema_name,ht1.table_name)::regclass INNER JOIN _timescaledb_catalog.chunk ch ON ch.hypertable_id = ht2.id ON CONFLICT DO NOTHING; ================================================ FILE: sql/updates/2.14.1--2.14.0.sql ================================================ DROP VIEW IF EXISTS timescaledb_information.hypertable_compression_settings; DROP VIEW IF EXISTS timescaledb_information.chunk_compression_settings; ================================================ FILE: sql/updates/2.14.1--2.14.2.sql ================================================ ================================================ FILE: sql/updates/2.14.2--2.14.1.sql ================================================ ================================================ FILE: sql/updates/2.14.2--2.15.0.sql ================================================ -- Remove multi-node CAGG support DROP FUNCTION IF EXISTS _timescaledb_internal.invalidation_cagg_log_add_entry(integer,bigint,bigint); DROP FUNCTION IF EXISTS _timescaledb_internal.invalidation_hyper_log_add_entry(integer,bigint,bigint); DROP FUNCTION IF EXISTS _timescaledb_internal.materialization_invalidation_log_delete(integer); DROP FUNCTION IF EXISTS _timescaledb_internal.invalidation_process_cagg_log(integer,integer,regtype,bigint,bigint,integer[],bigint[],bigint[]); DROP FUNCTION IF EXISTS _timescaledb_internal.invalidation_process_cagg_log(integer,integer,regtype,bigint,bigint,integer[],bigint[],bigint[],text[]); DROP FUNCTION IF EXISTS _timescaledb_internal.invalidation_process_hypertable_log(integer,integer,regtype,integer[],bigint[],bigint[]); DROP FUNCTION IF EXISTS _timescaledb_internal.invalidation_process_hypertable_log(integer,integer,regtype,integer[],bigint[],bigint[],text[]); DROP FUNCTION IF EXISTS _timescaledb_internal.hypertable_invalidation_log_delete(integer); DROP FUNCTION IF EXISTS _timescaledb_functions.invalidation_cagg_log_add_entry(integer,bigint,bigint); DROP FUNCTION IF EXISTS _timescaledb_functions.invalidation_hyper_log_add_entry(integer,bigint,bigint); DROP FUNCTION IF EXISTS _timescaledb_functions.materialization_invalidation_log_delete(integer); DROP FUNCTION IF EXISTS _timescaledb_functions.invalidation_process_cagg_log(integer,integer,regtype,bigint,bigint,integer[],bigint[],bigint[]); DROP FUNCTION IF EXISTS _timescaledb_functions.invalidation_process_cagg_log(integer,integer,regtype,bigint,bigint,integer[],bigint[],bigint[],text[]); DROP FUNCTION IF EXISTS _timescaledb_functions.invalidation_process_hypertable_log(integer,integer,regtype,integer[],bigint[],bigint[]); DROP FUNCTION IF EXISTS _timescaledb_functions.invalidation_process_hypertable_log(integer,integer,regtype,integer[],bigint[],bigint[],text[]); DROP FUNCTION IF EXISTS _timescaledb_functions.hypertable_invalidation_log_delete(integer); -- Remove chunk metadata when marked as dropped CREATE FUNCTION _timescaledb_functions.remove_dropped_chunk_metadata(_hypertable_id INTEGER) RETURNS INTEGER LANGUAGE plpgsql AS $$ DECLARE _chunk_id INTEGER; _removed INTEGER := 0; BEGIN FOR _chunk_id IN SELECT id FROM _timescaledb_catalog.chunk WHERE hypertable_id = _hypertable_id AND dropped IS TRUE AND NOT EXISTS ( SELECT FROM information_schema.tables WHERE tables.table_schema = chunk.schema_name AND tables.table_name = chunk.table_name ) AND NOT EXISTS ( SELECT FROM _timescaledb_catalog.hypertable JOIN _timescaledb_catalog.continuous_agg ON continuous_agg.raw_hypertable_id = hypertable.id WHERE hypertable.id = chunk.hypertable_id -- for the old caggs format we need to keep chunk metadata for dropped chunks AND continuous_agg.finalized IS FALSE ) LOOP _removed := _removed + 1; RAISE INFO 'Removing metadata of chunk % from hypertable %', _chunk_id, _hypertable_id; WITH _dimension_slice_remove AS ( DELETE FROM _timescaledb_catalog.dimension_slice USING _timescaledb_catalog.chunk_constraint WHERE dimension_slice.id = chunk_constraint.dimension_slice_id AND chunk_constraint.chunk_id = _chunk_id AND NOT EXISTS ( SELECT FROM _timescaledb_catalog.chunk_constraint cc WHERE cc.chunk_id <> _chunk_id AND cc.dimension_slice_id = dimension_slice.id ) RETURNING _timescaledb_catalog.dimension_slice.id ) DELETE FROM _timescaledb_catalog.chunk_constraint USING _dimension_slice_remove WHERE chunk_constraint.dimension_slice_id = _dimension_slice_remove.id; DELETE FROM _timescaledb_catalog.chunk_constraint WHERE chunk_constraint.chunk_id = _chunk_id; DELETE FROM _timescaledb_internal.bgw_policy_chunk_stats WHERE bgw_policy_chunk_stats.chunk_id = _chunk_id; DELETE FROM _timescaledb_catalog.chunk_index WHERE chunk_index.chunk_id = _chunk_id; DELETE FROM _timescaledb_catalog.compression_chunk_size WHERE compression_chunk_size.chunk_id = _chunk_id OR compression_chunk_size.compressed_chunk_id = _chunk_id; DELETE FROM _timescaledb_catalog.chunk WHERE chunk.id = _chunk_id OR chunk.compressed_chunk_id = _chunk_id; END LOOP; RETURN _removed; END; $$ SET search_path TO pg_catalog, pg_temp; SELECT _timescaledb_functions.remove_dropped_chunk_metadata(id) AS chunks_metadata_removed FROM _timescaledb_catalog.hypertable; -- -- Rebuild the catalog table `_timescaledb_catalog.continuous_aggs_bucket_function` -- CREATE OR REPLACE FUNCTION _timescaledb_functions.cagg_get_bucket_function( mat_hypertable_id INTEGER ) RETURNS regprocedure AS '@MODULE_PATHNAME@', 'ts_continuous_agg_get_bucket_function' LANGUAGE C STRICT VOLATILE; -- Since we need now the regclass of the used bucket function, we have to recover it -- by parsing the view query by calling 'cagg_get_bucket_function'. CREATE TABLE _timescaledb_catalog._tmp_continuous_aggs_bucket_function AS SELECT mat_hypertable_id, _timescaledb_functions.cagg_get_bucket_function(mat_hypertable_id), bucket_width, origin, NULL::text AS bucket_offset, timezone, false AS bucket_fixed_width FROM _timescaledb_catalog.continuous_aggs_bucket_function ORDER BY mat_hypertable_id; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_aggs_bucket_function; DROP TABLE _timescaledb_catalog.continuous_aggs_bucket_function; CREATE TABLE _timescaledb_catalog.continuous_aggs_bucket_function ( mat_hypertable_id integer NOT NULL, -- The bucket function bucket_func regprocedure NOT NULL, -- `bucket_width` argument of the function, e.g. "1 month" bucket_width text NOT NULL, -- optional `origin` argument of the function provided by the user bucket_origin text, -- optional `offset` argument of the function provided by the user bucket_offset text, -- optional `timezone` argument of the function provided by the user bucket_timezone text, -- fixed or variable sized bucket bucket_fixed_width bool NOT NULL, -- table constraints CONSTRAINT continuous_aggs_bucket_function_pkey PRIMARY KEY (mat_hypertable_id), CONSTRAINT continuous_aggs_bucket_function_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE ); INSERT INTO _timescaledb_catalog.continuous_aggs_bucket_function SELECT * FROM _timescaledb_catalog._tmp_continuous_aggs_bucket_function; DROP TABLE _timescaledb_catalog._tmp_continuous_aggs_bucket_function; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_aggs_bucket_function', ''); GRANT SELECT ON TABLE _timescaledb_catalog.continuous_aggs_bucket_function TO PUBLIC; ANALYZE _timescaledb_catalog.continuous_aggs_bucket_function; ALTER EXTENSION timescaledb DROP FUNCTION _timescaledb_functions.cagg_get_bucket_function(INTEGER); DROP FUNCTION IF EXISTS _timescaledb_functions.cagg_get_bucket_function(INTEGER); -- -- End rebuild the catalog table `_timescaledb_catalog.continuous_aggs_bucket_function` -- -- Convert _timescaledb_catalog.continuous_aggs_bucket_function.bucket_origin to TimestampTZ UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_origin = bucket_origin::timestamp::timestamptz::text WHERE length(bucket_origin) > 1; -- Historically, we have used empty strings for undefined bucket_origin and timezone -- attributes. This is now replaced by proper NULL values. We use TRIM() to ensure we handle empty string well. UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_origin = NULL WHERE TRIM(bucket_origin) = ''; UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_timezone = NULL WHERE TRIM(bucket_timezone) = ''; -- So far, there were no difference between 0 and -1 retries. Since now on, 0 means no retries. Updating the retry -- count of existing jobs to -1 to keep the current semantics. UPDATE _timescaledb_config.bgw_job SET max_retries = -1 WHERE max_retries = 0; DROP FUNCTION IF EXISTS _timescaledb_functions.get_chunk_relstats; DROP FUNCTION IF EXISTS _timescaledb_functions.get_chunk_colstats; DROP FUNCTION IF EXISTS _timescaledb_internal.get_chunk_relstats; DROP FUNCTION IF EXISTS _timescaledb_internal.get_chunk_colstats; -- In older TSDB versions, we disabled autovacuum for compressed chunks -- to keep the statistics. However, this restriction was removed in -- #5118 but no migration was performed to remove the custom -- autovacuum setting for existing chunks. DO $$ DECLARE chunk regclass; BEGIN FOR chunk IN SELECT pg_catalog.format('%I.%I', schema_name, table_name)::regclass FROM _timescaledb_catalog.chunk c JOIN pg_catalog.pg_class AS pc ON (pc.oid=format('%I.%I', schema_name, table_name)::regclass) CROSS JOIN unnest(reloptions) AS u(option) WHERE dropped = false AND osm_chunk = false AND option LIKE 'autovacuum_enabled%' LOOP EXECUTE pg_catalog.format('ALTER TABLE %s RESET (autovacuum_enabled);', chunk::text); END LOOP; END $$; -- -- Rebuild the catalog table `_timescaledb_catalog.continuous_agg` -- -- (1) Create missing entries in _timescaledb_catalog.continuous_aggs_bucket_function CREATE OR REPLACE FUNCTION _timescaledb_functions.cagg_get_bucket_function( mat_hypertable_id INTEGER ) RETURNS regprocedure AS '@MODULE_PATHNAME@', 'ts_continuous_agg_get_bucket_function' LANGUAGE C STRICT VOLATILE; -- Make sure function points to the new version of TSDB CREATE OR REPLACE FUNCTION _timescaledb_functions.to_interval(unixtime_us BIGINT) RETURNS INTERVAL AS '@MODULE_PATHNAME@', 'ts_pg_unix_microseconds_to_interval' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; -- We need to create entries in continuous_aggs_bucket_function for all CAggs that were treated so far -- as fixed indicated by a bucket_width != -1 INSERT INTO _timescaledb_catalog.continuous_aggs_bucket_function SELECT mat_hypertable_id, _timescaledb_functions.cagg_get_bucket_function(mat_hypertable_id), -- Intervals needs to be converted into the proper interval format -- Function name could be prefixed with 'public.'. Therefore LIKE instead of starts_with is used CASE WHEN _timescaledb_functions.cagg_get_bucket_function(mat_hypertable_id)::text LIKE '%time_bucket(interval,%' THEN _timescaledb_functions.to_interval(bucket_width)::text ELSE bucket_width::text END, NULL, -- bucket_origin NULL, -- bucket_offset NULL, -- bucket_timezone true -- bucket_fixed_width FROM _timescaledb_catalog.continuous_agg WHERE bucket_width != -1; ALTER EXTENSION timescaledb DROP FUNCTION _timescaledb_functions.cagg_get_bucket_function(INTEGER); DROP FUNCTION IF EXISTS _timescaledb_functions.cagg_get_bucket_function(INTEGER); -- (2) Rebuild catalog table DROP VIEW IF EXISTS timescaledb_experimental.policies; DROP VIEW IF EXISTS timescaledb_information.hypertables; DROP VIEW IF EXISTS timescaledb_information.continuous_aggregates; DROP PROCEDURE IF EXISTS @extschema@.cagg_migrate (REGCLASS, BOOLEAN, BOOLEAN); DROP FUNCTION IF EXISTS _timescaledb_internal.cagg_migrate_pre_validation (TEXT, TEXT, TEXT); DROP FUNCTION IF EXISTS _timescaledb_functions.cagg_migrate_pre_validation (TEXT, TEXT, TEXT); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_create_plan (_timescaledb_catalog.continuous_agg, TEXT, BOOLEAN, BOOLEAN); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_create_plan (_timescaledb_catalog.continuous_agg, TEXT, BOOLEAN, BOOLEAN); DROP FUNCTION IF EXISTS _timescaledb_functions.cagg_migrate_plan_exists (INTEGER); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_plan (_timescaledb_catalog.continuous_agg); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_plan (_timescaledb_catalog.continuous_agg); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_create_new_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_create_new_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_disable_policies (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_disable_policies (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_enable_policies (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_enable_policies (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_copy_policies (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_copy_policies (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_refresh_new_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_refresh_new_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_copy_data (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_copy_data (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_override_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_override_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_drop_old_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_drop_old_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_invalidation_log DROP CONSTRAINT continuous_aggs_materialization_invalid_materialization_id_fkey; ALTER TABLE _timescaledb_catalog.continuous_aggs_watermark DROP CONSTRAINT continuous_aggs_watermark_mat_hypertable_id_fkey; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_agg; CREATE TABLE _timescaledb_catalog._tmp_continuous_agg AS SELECT mat_hypertable_id, raw_hypertable_id, parent_mat_hypertable_id, user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name, materialized_only, finalized FROM _timescaledb_catalog.continuous_agg ORDER BY mat_hypertable_id; DROP TABLE _timescaledb_catalog.continuous_agg; CREATE TABLE _timescaledb_catalog.continuous_agg ( mat_hypertable_id integer NOT NULL, raw_hypertable_id integer NOT NULL, parent_mat_hypertable_id integer, user_view_schema name NOT NULL, user_view_name name NOT NULL, partial_view_schema name NOT NULL, partial_view_name name NOT NULL, direct_view_schema name NOT NULL, direct_view_name name NOT NULL, materialized_only bool NOT NULL DEFAULT FALSE, finalized bool NOT NULL DEFAULT TRUE, -- table constraints CONSTRAINT continuous_agg_pkey PRIMARY KEY (mat_hypertable_id), CONSTRAINT continuous_agg_partial_view_schema_partial_view_name_key UNIQUE (partial_view_schema, partial_view_name), CONSTRAINT continuous_agg_user_view_schema_user_view_name_key UNIQUE (user_view_schema, user_view_name), CONSTRAINT continuous_agg_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE, CONSTRAINT continuous_agg_raw_hypertable_id_fkey FOREIGN KEY (raw_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE, CONSTRAINT continuous_agg_parent_mat_hypertable_id_fkey FOREIGN KEY (parent_mat_hypertable_id) REFERENCES _timescaledb_catalog.continuous_agg (mat_hypertable_id) ON DELETE CASCADE ); INSERT INTO _timescaledb_catalog.continuous_agg SELECT * FROM _timescaledb_catalog._tmp_continuous_agg; DROP TABLE _timescaledb_catalog._tmp_continuous_agg; CREATE INDEX continuous_agg_raw_hypertable_id_idx ON _timescaledb_catalog.continuous_agg (raw_hypertable_id); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_agg', ''); GRANT SELECT ON TABLE _timescaledb_catalog.continuous_agg TO PUBLIC; ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_invalidation_log ADD CONSTRAINT continuous_aggs_materialization_invalid_materialization_id_fkey FOREIGN KEY (materialization_id) REFERENCES _timescaledb_catalog.continuous_agg(mat_hypertable_id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.continuous_aggs_watermark ADD CONSTRAINT continuous_aggs_watermark_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.continuous_agg (mat_hypertable_id) ON DELETE CASCADE; ANALYZE _timescaledb_catalog.continuous_agg; -- -- END Rebuild the catalog table `_timescaledb_catalog.continuous_agg` -- -- -- START bgw_job_stat_history -- DROP VIEW IF EXISTS timescaledb_information.job_errors; CREATE SEQUENCE _timescaledb_internal.bgw_job_stat_history_id_seq MINVALUE 1; CREATE TABLE _timescaledb_internal.bgw_job_stat_history ( id INTEGER NOT NULL DEFAULT nextval('_timescaledb_internal.bgw_job_stat_history_id_seq'), job_id INTEGER NOT NULL, pid INTEGER, execution_start TIMESTAMPTZ NOT NULL DEFAULT NOW(), execution_finish TIMESTAMPTZ, succeeded boolean NOT NULL DEFAULT FALSE, data jsonb, -- table constraints CONSTRAINT bgw_job_stat_history_pkey PRIMARY KEY (id) ); ALTER SEQUENCE _timescaledb_internal.bgw_job_stat_history_id_seq OWNED BY _timescaledb_internal.bgw_job_stat_history.id; CREATE INDEX bgw_job_stat_history_job_id_idx ON _timescaledb_internal.bgw_job_stat_history (job_id); REVOKE ALL ON _timescaledb_internal.bgw_job_stat_history FROM PUBLIC; INSERT INTO _timescaledb_internal.bgw_job_stat_history (job_id, pid, execution_start, execution_finish, data) SELECT job_errors.job_id, job_errors.pid, job_errors.start_time, job_errors.finish_time, jsonb_build_object('job', to_jsonb(bgw_job.*)) || jsonb_build_object('error_data', job_errors.error_data) FROM _timescaledb_internal.job_errors LEFT JOIN _timescaledb_config.bgw_job ON bgw_job.id = job_errors.job_id ORDER BY job_errors.job_id, job_errors.start_time; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_internal.job_errors; DROP TABLE _timescaledb_internal.job_errors; UPDATE _timescaledb_config.bgw_job SET scheduled = false WHERE id = 2; INSERT INTO _timescaledb_config.bgw_job ( id, application_name, schedule_interval, max_runtime, max_retries, retry_period, proc_schema, proc_name, owner, scheduled, config, check_schema, check_name, fixed_schedule, initial_start ) VALUES ( 3, 'Job History Log Retention Policy [3]', INTERVAL '1 month', INTERVAL '1 hour', -1, INTERVAL '1h', '_timescaledb_functions', 'policy_job_stat_history_retention', pg_catalog.quote_ident(current_role)::regrole, true, '{"drop_after":"1 month"}', '_timescaledb_functions', 'policy_job_stat_history_retention_check', true, '2000-01-01 00:00:00+00'::timestamptz ) ON CONFLICT (id) DO NOTHING; DROP FUNCTION IF EXISTS _timescaledb_internal.policy_job_error_retention(job_id integer,config jsonb); DROP FUNCTION IF EXISTS _timescaledb_internal.policy_job_error_retention_check(config jsonb); DROP FUNCTION IF EXISTS _timescaledb_functions.policy_job_error_retention(job_id integer,config jsonb); DROP FUNCTION IF EXISTS _timescaledb_functions.policy_job_error_retention_check(config jsonb); -- -- END bgw_job_stat_history -- -- Migrate existing CAggs using time_bucket_ng to time_bucket DO $$ DECLARE cagg_name regclass; caggs text; BEGIN SELECT string_agg(pg_catalog.format('%I.%I', user_view_schema, user_view_name), ', ') INTO caggs FROM _timescaledb_catalog.continuous_agg cagg JOIN _timescaledb_catalog.continuous_aggs_bucket_function AS bf ON (cagg.mat_hypertable_id = bf.mat_hypertable_id) WHERE bf.bucket_func::text LIKE '%time_bucket_ng%'; IF caggs IS NOT NULL THEN RAISE WARNING 'continuous aggregates with time_bucket_ng found, please use _timescaledb_functions.cagg_migrate_to_time_bucket to migrate caggs manually after extension update' USING DETAIL = format('Continuous Aggregates: %s', caggs); END IF; END $$; ================================================ FILE: sql/updates/2.15.0--2.14.2.sql ================================================ DROP FUNCTION IF EXISTS _timescaledb_functions.remove_dropped_chunk_metadata(INTEGER); -- -- Rebuild the catalog table `_timescaledb_catalog.continuous_agg` -- DROP VIEW IF EXISTS timescaledb_experimental.policies; DROP VIEW IF EXISTS timescaledb_information.hypertables; DROP VIEW IF EXISTS timescaledb_information.continuous_aggregates; DROP PROCEDURE IF EXISTS @extschema@.cagg_migrate (REGCLASS, BOOLEAN, BOOLEAN); DROP FUNCTION IF EXISTS _timescaledb_internal.cagg_migrate_pre_validation (TEXT, TEXT, TEXT); DROP FUNCTION IF EXISTS _timescaledb_functions.cagg_migrate_pre_validation (TEXT, TEXT, TEXT); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_create_plan (_timescaledb_catalog.continuous_agg, TEXT, BOOLEAN, BOOLEAN); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_create_plan (_timescaledb_catalog.continuous_agg, TEXT, BOOLEAN, BOOLEAN); DROP FUNCTION IF EXISTS _timescaledb_functions.cagg_migrate_plan_exists (INTEGER); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_plan (_timescaledb_catalog.continuous_agg); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_plan (_timescaledb_catalog.continuous_agg); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_create_new_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_create_new_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_disable_policies (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_disable_policies (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_enable_policies (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_enable_policies (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_copy_policies (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_copy_policies (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_refresh_new_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_refresh_new_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_copy_data (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_copy_data (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_override_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_override_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_drop_old_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_drop_old_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_invalidation_log DROP CONSTRAINT continuous_aggs_materialization_invalid_materialization_id_fkey; ALTER TABLE _timescaledb_catalog.continuous_aggs_watermark DROP CONSTRAINT continuous_aggs_watermark_mat_hypertable_id_fkey; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_agg; CREATE TABLE _timescaledb_catalog._tmp_continuous_agg AS SELECT mat_hypertable_id, raw_hypertable_id, parent_mat_hypertable_id, user_view_schema, user_view_name, partial_view_schema, partial_view_name, -1::bigint as bucket_width, -- -1 means variable width. Will be modified soon if not variable. direct_view_schema, direct_view_name, materialized_only, finalized FROM _timescaledb_catalog.continuous_agg ORDER BY mat_hypertable_id; -- Migrate CAggs with fixed bucket on interval back WITH fixed_buckets AS ( SELECT * FROM _timescaledb_catalog.continuous_aggs_bucket_function WHERE bucket_fixed_width = true AND bucket_func::text LIKE '%time_bucket(interval%' ) UPDATE _timescaledb_catalog._tmp_continuous_agg cagg SET bucket_width = _timescaledb_functions.interval_to_usec(fb.bucket_width::interval) FROM fixed_buckets fb WHERE cagg.mat_hypertable_id = fb.mat_hypertable_id; -- Migrate CAggs with fixed bucket on integer back WITH fixed_buckets AS ( SELECT * FROM _timescaledb_catalog.continuous_aggs_bucket_function WHERE bucket_fixed_width = true AND bucket_func::text NOT LIKE '%time_bucket(interval%' ) UPDATE _timescaledb_catalog._tmp_continuous_agg cagg SET bucket_width = fb.bucket_width::bigint FROM fixed_buckets fb WHERE cagg.mat_hypertable_id = fb.mat_hypertable_id; DELETE FROM _timescaledb_catalog.continuous_aggs_bucket_function WHERE bucket_fixed_width = true; DROP TABLE _timescaledb_catalog.continuous_agg; CREATE TABLE _timescaledb_catalog.continuous_agg ( mat_hypertable_id integer NOT NULL, raw_hypertable_id integer NOT NULL, parent_mat_hypertable_id integer, user_view_schema name NOT NULL, user_view_name name NOT NULL, partial_view_schema name NOT NULL, partial_view_name name NOT NULL, bucket_width bigint NOT NULL, direct_view_schema name NOT NULL, direct_view_name name NOT NULL, materialized_only bool NOT NULL DEFAULT FALSE, finalized bool NOT NULL DEFAULT TRUE, -- table constraints CONSTRAINT continuous_agg_pkey PRIMARY KEY (mat_hypertable_id), CONSTRAINT continuous_agg_partial_view_schema_partial_view_name_key UNIQUE (partial_view_schema, partial_view_name), CONSTRAINT continuous_agg_user_view_schema_user_view_name_key UNIQUE (user_view_schema, user_view_name), CONSTRAINT continuous_agg_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE, CONSTRAINT continuous_agg_raw_hypertable_id_fkey FOREIGN KEY (raw_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE, CONSTRAINT continuous_agg_parent_mat_hypertable_id_fkey FOREIGN KEY (parent_mat_hypertable_id) REFERENCES _timescaledb_catalog.continuous_agg (mat_hypertable_id) ON DELETE CASCADE ); INSERT INTO _timescaledb_catalog.continuous_agg SELECT * FROM _timescaledb_catalog._tmp_continuous_agg; DROP TABLE _timescaledb_catalog._tmp_continuous_agg; CREATE INDEX continuous_agg_raw_hypertable_id_idx ON _timescaledb_catalog.continuous_agg (raw_hypertable_id); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_agg', ''); GRANT SELECT ON TABLE _timescaledb_catalog.continuous_agg TO PUBLIC; ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_invalidation_log ADD CONSTRAINT continuous_aggs_materialization_invalid_materialization_id_fkey FOREIGN KEY (materialization_id) REFERENCES _timescaledb_catalog.continuous_agg(mat_hypertable_id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.continuous_aggs_watermark ADD CONSTRAINT continuous_aggs_watermark_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.continuous_agg (mat_hypertable_id) ON DELETE CASCADE; ANALYZE _timescaledb_catalog.continuous_agg; -- -- END Rebuild the catalog table `_timescaledb_catalog.continuous_agg` -- -- -- Rebuild the catalog table `_timescaledb_catalog.continuous_aggs_bucket_function` -- UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_origin = '' WHERE bucket_origin IS NULL; UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_timezone = '' WHERE bucket_timezone IS NULL; CREATE TABLE _timescaledb_catalog._tmp_continuous_aggs_bucket_function AS SELECT mat_hypertable_id, CASE WHEN bucket_func::text like 'timescaledb_experimental%' THEN true ELSE false END, split_part(bucket_func::regproc::text, '.', 2), bucket_width, bucket_origin, bucket_timezone FROM _timescaledb_catalog.continuous_aggs_bucket_function ORDER BY mat_hypertable_id; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_aggs_bucket_function; DROP TABLE _timescaledb_catalog.continuous_aggs_bucket_function; CREATE TABLE _timescaledb_catalog.continuous_aggs_bucket_function ( mat_hypertable_id integer NOT NULL, -- The schema of the function. Equals TRUE for "timescaledb_experimental", FALSE otherwise. experimental bool NOT NULL, -- Name of the bucketing function, e.g. "time_bucket" or "time_bucket_ng" name text NOT NULL, -- `bucket_width` argument of the function, e.g. "1 month" bucket_width text NOT NULL, -- `origin` argument of the function provided by the user origin text NOT NULL, -- `timezone` argument of the function provided by the user timezone text NOT NULL, -- table constraints CONSTRAINT continuous_aggs_bucket_function_pkey PRIMARY KEY (mat_hypertable_id), CONSTRAINT continuous_aggs_bucket_function_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE ); INSERT INTO _timescaledb_catalog.continuous_aggs_bucket_function SELECT * FROM _timescaledb_catalog._tmp_continuous_aggs_bucket_function; DROP TABLE _timescaledb_catalog._tmp_continuous_aggs_bucket_function; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_aggs_bucket_function', ''); GRANT SELECT ON TABLE _timescaledb_catalog.continuous_aggs_bucket_function TO PUBLIC; ANALYZE _timescaledb_catalog.continuous_aggs_bucket_function; -- -- End rebuild the catalog table `_timescaledb_catalog.continuous_aggs_bucket_function` -- -- Convert _timescaledb_catalog.continuous_aggs_bucket_function.origin back to Timestamp UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET origin = origin::timestamptz::timestamp::text WHERE length(origin) > 1; -- only create stub CREATE FUNCTION _timescaledb_functions.get_chunk_relstats(relid REGCLASS) RETURNS TABLE(chunk_id INTEGER, hypertable_id INTEGER, num_pages INTEGER, num_tuples REAL, num_allvisible INTEGER) AS $$BEGIN END$$ LANGUAGE plpgsql SET search_path = pg_catalog, pg_temp; CREATE FUNCTION _timescaledb_functions.get_chunk_colstats(relid REGCLASS) RETURNS TABLE(chunk_id INTEGER, hypertable_id INTEGER, att_num INTEGER, nullfrac REAL, width INTEGER, distinctval REAL, slotkind INTEGER[], slotopstrings CSTRING[], slotcollations OID[], slot1numbers FLOAT4[], slot2numbers FLOAT4[], slot3numbers FLOAT4[], slot4numbers FLOAT4[], slot5numbers FLOAT4[], slotvaluetypetrings CSTRING[], slot1values CSTRING[], slot2values CSTRING[], slot3values CSTRING[], slot4values CSTRING[], slot5values CSTRING[]) AS $$BEGIN END$$ LANGUAGE plpgsql SET search_path = pg_catalog, pg_temp; -- -- START bgw_job_stat_history -- DROP VIEW IF EXISTS timescaledb_information.job_errors; ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.job_history; DROP VIEW IF EXISTS timescaledb_information.job_history; CREATE TABLE _timescaledb_internal.job_errors ( job_id integer not null, pid integer, start_time timestamptz, finish_time timestamptz, error_data jsonb ); INSERT INTO _timescaledb_internal.job_errors (job_id, pid, start_time, finish_time, error_data) SELECT job_id, pid, execution_start, execution_finish, data->'error_data' FROM _timescaledb_internal.bgw_job_stat_history WHERE succeeded IS FALSE ORDER BY job_id, execution_start; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_internal.bgw_job_stat_history; DROP TABLE IF EXISTS _timescaledb_internal.bgw_job_stat_history; REVOKE ALL ON _timescaledb_internal.job_errors FROM PUBLIC; DROP FUNCTION IF EXISTS _timescaledb_internal.policy_job_stat_history_retention(job_id integer,config jsonb); DROP FUNCTION IF EXISTS _timescaledb_internal.policy_job_stat_history_retention_check(config jsonb); DROP FUNCTION IF EXISTS _timescaledb_functions.policy_job_stat_history_retention(job_id integer,config jsonb); DROP FUNCTION IF EXISTS _timescaledb_functions.policy_job_stat_history_retention_check(config jsonb); CREATE OR REPLACE FUNCTION _timescaledb_functions.policy_job_error_retention(job_id integer, config JSONB) RETURNS integer LANGUAGE PLPGSQL AS $BODY$ DECLARE drop_after INTERVAL; numrows INTEGER; BEGIN SELECT config->>'drop_after' INTO STRICT drop_after; WITH deleted AS (DELETE FROM _timescaledb_internal.job_errors WHERE finish_time < (now() - drop_after) RETURNING *) SELECT count(*) FROM deleted INTO numrows; RETURN numrows; END; $BODY$ SET search_path TO pg_catalog, pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_functions.policy_job_error_retention_check(config JSONB) RETURNS VOID LANGUAGE PLPGSQL AS $BODY$ DECLARE drop_after interval; BEGIN IF config IS NULL THEN RAISE EXCEPTION 'config cannot be NULL, and must contain drop_after'; END IF; SELECT config->>'drop_after' INTO STRICT drop_after; IF drop_after IS NULL THEN RAISE EXCEPTION 'drop_after interval not provided'; END IF ; END; $BODY$ SET search_path TO pg_catalog, pg_temp; UPDATE _timescaledb_config.bgw_job SET scheduled = true WHERE id = 2; DELETE FROM _timescaledb_config.bgw_job WHERE id = 3; DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_to_time_bucket(cagg REGCLASS); ================================================ FILE: sql/updates/2.15.0--2.15.1.sql ================================================ CREATE TABLE _timescaledb_catalog._tmp_continuous_aggs_bucket_function AS SELECT mat_hypertable_id, bucket_func::text AS bucket_func, bucket_width, bucket_origin, bucket_offset, bucket_timezone, bucket_fixed_width FROM _timescaledb_catalog.continuous_aggs_bucket_function ORDER BY mat_hypertable_id; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_aggs_bucket_function; DROP TABLE _timescaledb_catalog.continuous_aggs_bucket_function; CREATE TABLE _timescaledb_catalog.continuous_aggs_bucket_function ( mat_hypertable_id integer NOT NULL, -- The bucket function bucket_func text NOT NULL, -- `bucket_width` argument of the function, e.g. "1 month" bucket_width text NOT NULL, -- optional `origin` argument of the function provided by the user bucket_origin text, -- optional `offset` argument of the function provided by the user bucket_offset text, -- optional `timezone` argument of the function provided by the user bucket_timezone text, -- fixed or variable sized bucket bucket_fixed_width bool NOT NULL, -- table constraints CONSTRAINT continuous_aggs_bucket_function_pkey PRIMARY KEY (mat_hypertable_id), CONSTRAINT continuous_aggs_bucket_function_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE, CONSTRAINT continuous_aggs_bucket_function_func_check CHECK (pg_catalog.to_regprocedure(bucket_func) IS DISTINCT FROM 0) ); INSERT INTO _timescaledb_catalog.continuous_aggs_bucket_function SELECT * FROM _timescaledb_catalog._tmp_continuous_aggs_bucket_function; DROP TABLE _timescaledb_catalog._tmp_continuous_aggs_bucket_function; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_aggs_bucket_function', ''); GRANT SELECT ON TABLE _timescaledb_catalog.continuous_aggs_bucket_function TO PUBLIC; ANALYZE _timescaledb_catalog.continuous_aggs_bucket_function; ================================================ FILE: sql/updates/2.15.1--2.15.0.sql ================================================ CREATE TABLE _timescaledb_catalog._tmp_continuous_aggs_bucket_function AS SELECT mat_hypertable_id, pg_catalog.to_regprocedure(bucket_func) AS bucket_func, bucket_width, bucket_origin, bucket_offset, bucket_timezone, bucket_fixed_width FROM _timescaledb_catalog.continuous_aggs_bucket_function ORDER BY mat_hypertable_id; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_aggs_bucket_function; DROP TABLE _timescaledb_catalog.continuous_aggs_bucket_function; CREATE TABLE _timescaledb_catalog.continuous_aggs_bucket_function ( mat_hypertable_id integer NOT NULL, -- The bucket function bucket_func regprocedure NOT NULL, -- `bucket_width` argument of the function, e.g. "1 month" bucket_width text NOT NULL, -- optional `origin` argument of the function provided by the user bucket_origin text, -- optional `offset` argument of the function provided by the user bucket_offset text, -- optional `timezone` argument of the function provided by the user bucket_timezone text, -- fixed or variable sized bucket bucket_fixed_width bool NOT NULL, -- table constraints CONSTRAINT continuous_aggs_bucket_function_pkey PRIMARY KEY (mat_hypertable_id), CONSTRAINT continuous_aggs_bucket_function_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE ); INSERT INTO _timescaledb_catalog.continuous_aggs_bucket_function SELECT * FROM _timescaledb_catalog._tmp_continuous_aggs_bucket_function; DROP TABLE _timescaledb_catalog._tmp_continuous_aggs_bucket_function; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_aggs_bucket_function', ''); GRANT SELECT ON TABLE _timescaledb_catalog.continuous_aggs_bucket_function TO PUBLIC; ANALYZE _timescaledb_catalog.continuous_aggs_bucket_function; ================================================ FILE: sql/updates/2.15.1--2.15.2.sql ================================================ ================================================ FILE: sql/updates/2.15.2--2.15.1.sql ================================================ ================================================ FILE: sql/updates/2.15.2--2.15.3.sql ================================================ ================================================ FILE: sql/updates/2.15.3--2.15.2.sql ================================================ ================================================ FILE: sql/updates/2.15.3--2.16.0.sql ================================================ -- Enable tracking of statistics on a column of a hypertable. -- -- hypertable - OID of the table to which the column belongs to -- column_name - The column to track statistics for -- if_not_exists - If set, and the entry already exists, generate a notice instead of an error CREATE FUNCTION @extschema@.enable_chunk_skipping( hypertable REGCLASS, column_name NAME, if_not_exists BOOLEAN = FALSE ) RETURNS TABLE(column_stats_id INT, enabled BOOL) AS 'SELECT NULL,NULL' LANGUAGE SQL VOLATILE SET search_path = pg_catalog, pg_temp; -- Disable tracking of statistics on a column of a hypertable. -- -- hypertable - OID of the table to remove from -- column_name - NAME of the column on which the stats are tracked -- if_not_exists - If set, and the entry does not exist, -- generate a notice instead of an error CREATE FUNCTION @extschema@.disable_chunk_skipping( hypertable REGCLASS, column_name NAME, if_not_exists BOOLEAN = FALSE ) RETURNS TABLE(hypertable_id INT, column_name NAME, disabled BOOL) AS 'SELECT NULL,NULL,NULL' LANGUAGE SQL VOLATILE SET search_path = pg_catalog, pg_temp; -- Track statistics for columns of chunks from a hypertable. -- Currently, we track the min/max range for a given column across chunks. -- More statistics (like bloom filters) will be added in the future. -- -- A "special" entry for a column with invalid chunk_id, PG_INT64_MAX, -- PG_INT64_MIN indicates that min/max ranges could be computed for this column -- for chunks. -- -- The ranges can overlap across chunks. The values could be out-of-date if -- modifications/changes occur in the corresponding chunk and such entries -- should be marked as "invalid" to ensure that the chunk is in -- appropriate state to be able to use these values. Thus these entries -- are different from dimension_slice which is used for tracking partitioning -- column ranges which have different characteristics. -- -- Currently this catalog supports datatypes like INT, SERIAL, BIGSERIAL, -- DATE, TIMESTAMP etc. by storing the ranges in bigint columns. In the -- future, we could support additional datatypes (which support btree style -- >, <, = comparators) by storing their textual representation. -- CREATE TABLE _timescaledb_catalog.chunk_column_stats ( id serial NOT NULL, hypertable_id integer NOT NULL, chunk_id integer NOT NULL, column_name name NOT NULL, range_start bigint NOT NULL, range_end bigint NOT NULL, valid boolean NOT NULL, -- table constraints CONSTRAINT chunk_column_stats_pkey PRIMARY KEY (id), CONSTRAINT chunk_column_stats_ht_id_chunk_id_colname_key UNIQUE (hypertable_id, chunk_id, column_name), CONSTRAINT chunk_column_stats_range_check CHECK (range_start <= range_end), CONSTRAINT chunk_column_stats_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id), CONSTRAINT chunk_column_stats_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk_column_stats', ''); SELECT pg_catalog.pg_extension_config_dump(pg_get_serial_sequence('_timescaledb_catalog.chunk_column_stats', 'id'), ''); GRANT SELECT ON _timescaledb_catalog.chunk_column_stats TO PUBLIC; GRANT SELECT ON _timescaledb_catalog.chunk_column_stats_id_seq TO PUBLIC; -- Remove foreign key constraints from compressed chunks DO $$ DECLARE conrelid regclass; conname name; BEGIN FOR conrelid, conname IN SELECT con.conrelid::regclass, con.conname FROM _timescaledb_catalog.chunk ch JOIN pg_constraint con ON con.conrelid = format('%I.%I',schema_name,table_name)::regclass AND con.contype='f' WHERE NOT ch.dropped AND EXISTS(SELECT FROM _timescaledb_catalog.chunk ch2 WHERE NOT ch2.dropped AND ch2.compressed_chunk_id=ch.id) LOOP EXECUTE format('ALTER TABLE %s DROP CONSTRAINT %I', conrelid, conname); END LOOP; END $$; ================================================ FILE: sql/updates/2.16.0--2.15.3.sql ================================================ DROP FUNCTION IF EXISTS _timescaledb_functions.cagg_get_bucket_function_info(INTEGER); -- remove chunk column statistics related objects DROP FUNCTION IF EXISTS @extschema@.enable_chunk_skipping(REGCLASS, NAME, BOOLEAN); DROP FUNCTION IF EXISTS @extschema@.disable_chunk_skipping(REGCLASS, NAME, BOOLEAN); ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.chunk_column_stats; ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_catalog.chunk_column_stats_id_seq; DROP TABLE IF EXISTS _timescaledb_catalog.chunk_column_stats; -- Add foreign key constraints back to compressed chunks DO $$ DECLARE chunkrelid regclass; conname name; conoid oid; BEGIN FOR chunkrelid, conname, conoid IN SELECT format('%I.%I',ch.schema_name,ch.table_name)::regclass, con.conname, con.oid FROM _timescaledb_catalog.hypertable ht JOIN pg_constraint con ON con.contype = 'f' AND con.conrelid=format('%I.%I',ht.schema_name,ht.table_name)::regclass JOIN _timescaledb_catalog.chunk ch on ch.hypertable_id=ht.compressed_hypertable_id and not ch.dropped LOOP EXECUTE format('ALTER TABLE %s ADD CONSTRAINT %I %s', chunkrelid, conname, pg_get_constraintdef(conoid)); END LOOP; END $$; ================================================ FILE: sql/updates/2.16.0--2.16.1.sql ================================================ ================================================ FILE: sql/updates/2.16.1--2.16.0.sql ================================================ ================================================ FILE: sql/updates/2.16.1--2.17.0.sql ================================================ CREATE FUNCTION _timescaledb_functions.compressed_data_info(_timescaledb_internal.compressed_data) RETURNS TABLE (algorithm name, has_nulls bool) AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C STRICT IMMUTABLE SET search_path = pg_catalog, pg_temp; CREATE INDEX compression_chunk_size_idx ON _timescaledb_catalog.compression_chunk_size (compressed_chunk_id); CREATE FUNCTION _timescaledb_functions.drop_osm_chunk(hypertable REGCLASS) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; ================================================ FILE: sql/updates/2.17.0--2.16.1.sql ================================================ -- check whether we can safely downgrade the existing compression setup CREATE OR REPLACE FUNCTION _timescaledb_functions.add_sequence_number_metadata_column( comp_ch_schema_name text, comp_ch_table_name text ) RETURNS BOOL LANGUAGE PLPGSQL AS $BODY$ DECLARE chunk_schema_name text; chunk_table_name text; index_name text; segmentby_columns text; BEGIN SELECT ch.schema_name, ch.table_name INTO STRICT chunk_schema_name, chunk_table_name FROM _timescaledb_catalog.chunk ch INNER JOIN _timescaledb_catalog.chunk comp_ch ON ch.compressed_chunk_id = comp_ch.id WHERE comp_ch.schema_name = comp_ch_schema_name AND comp_ch.table_name = comp_ch_table_name; IF NOT FOUND THEN RAISE USING ERRCODE = 'feature_not_supported', MESSAGE = 'Cannot migrate compressed chunk to version 2.16.1, chunk not found'; END IF; -- Add sequence number column to compressed chunk EXECUTE format('ALTER TABLE %s.%s ADD COLUMN _ts_meta_sequence_num INT DEFAULT NULL', comp_ch_schema_name, comp_ch_table_name); -- Remove all indexes from compressed chunk FOR index_name IN SELECT format('%s.%s', i.schemaname, i.indexname) FROM pg_indexes i WHERE i.schemaname = comp_ch_schema_name AND i.tablename = comp_ch_table_name LOOP EXECUTE format('DROP INDEX %s;', index_name); END LOOP; -- Fetch the segmentby columns from compression settings SELECT string_agg(cs.segmentby_column, ',') INTO segmentby_columns FROM ( SELECT unnest(segmentby) FROM _timescaledb_catalog.compression_settings WHERE relid = format('%s.%s', comp_ch_schema_name, comp_ch_table_name)::regclass::oid AND segmentby IS NOT NULL ) AS cs(segmentby_column); -- Create compressed chunk index based on sequence num metadata column -- If there is no segmentby columns, we can skip creating the index IF FOUND AND segmentby_columns IS NOT NULL THEN EXECUTE format('CREATE INDEX ON %s.%s (%s, _ts_meta_sequence_num);', comp_ch_schema_name, comp_ch_table_name, segmentby_columns); END IF; -- Mark compressed chunk as unordered -- Marking the chunk status bit (2) makes it unordered -- and disables some optimizations. In order to re-enable -- them, you need to recompress these chunks. UPDATE _timescaledb_catalog.chunk SET status = status | 2 -- set unordered bit WHERE schema_name = chunk_schema_name AND table_name = chunk_table_name; RETURN true; END $BODY$ SET search_path TO pg_catalog, pg_temp; DO $$ DECLARE chunk_count int; chunk_record record; BEGIN -- if we find chunks which don't have sequence number metadata column in -- compressed chunk, we need to stop downgrade and have the user run -- a migration script to re-add the missing columns SELECT count(*) INTO STRICT chunk_count FROM _timescaledb_catalog.chunk ch INNER JOIN _timescaledb_catalog.chunk uncomp_ch ON uncomp_ch.compressed_chunk_id = ch.id WHERE not exists ( SELECT FROM pg_attribute att WHERE attrelid=format('%I.%I',ch.schema_name,ch.table_name)::regclass AND attname='_ts_meta_sequence_num') AND NOT uncomp_ch.dropped; -- Doing the migration if we find 10 or less chunks that need to be migrated IF chunk_count > 10 THEN RAISE USING ERRCODE = 'feature_not_supported', MESSAGE = 'Cannot downgrade compressed hypertables with chunks that do not contain sequence numbers. Run timescaledb--2.17-2.16.1.sql migration script before downgrading.', DETAIL = 'Number of chunks that need to be migrated: '|| chunk_count::text; ELSIF chunk_count > 0 THEN FOR chunk_record IN SELECT comp_ch.* FROM _timescaledb_catalog.chunk ch INNER JOIN _timescaledb_catalog.chunk comp_ch ON ch.compressed_chunk_id = comp_ch.id WHERE not exists ( SELECT FROM pg_attribute att WHERE attrelid=format('%I.%I',comp_ch.schema_name,comp_ch.table_name)::regclass AND attname='_ts_meta_sequence_num') AND NOT ch.dropped LOOP PERFORM _timescaledb_functions.add_sequence_number_metadata_column(chunk_record.schema_name, chunk_record.table_name); RAISE LOG 'Migrated compressed chunk %s.%s to version 2.16.1', chunk_record.schema_name, chunk_record.table_name; END LOOP; RAISE LOG 'Migration successful!'; END IF; END $$; DROP FUNCTION _timescaledb_functions.add_sequence_number_metadata_column(text, text); DROP FUNCTION _timescaledb_functions.compressed_data_info(_timescaledb_internal.compressed_data); DROP INDEX _timescaledb_catalog.compression_chunk_size_idx; DROP FUNCTION IF EXISTS _timescaledb_functions.drop_osm_chunk(REGCLASS); ================================================ FILE: sql/updates/2.17.0--2.17.1.sql ================================================ ================================================ FILE: sql/updates/2.17.1--2.17.0.sql ================================================ ================================================ FILE: sql/updates/2.17.1--2.17.2.sql ================================================ ================================================ FILE: sql/updates/2.17.2--2.17.1.sql ================================================ ================================================ FILE: sql/updates/2.17.2--2.18.0.sql ================================================ -- remove obsolete job DELETE FROM _timescaledb_config.bgw_job WHERE id = 2; -- Hypercore updates CREATE FUNCTION _timescaledb_debug.is_compressed_tid(tid) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C STRICT; DROP FUNCTION IF EXISTS @extschema@.compress_chunk(uncompressed_chunk REGCLASS, if_not_compressed BOOLEAN, recompress BOOLEAN); CREATE FUNCTION @extschema@.compress_chunk( uncompressed_chunk REGCLASS, if_not_compressed BOOLEAN = true, recompress BOOLEAN = false, hypercore_use_access_method BOOL = NULL ) RETURNS REGCLASS AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; DROP FUNCTION IF EXISTS @extschema@.add_compression_policy(hypertable REGCLASS, compress_after "any", if_not_exists BOOL, schedule_interval INTERVAL, initial_start TIMESTAMPTZ, timezone TEXT, compress_created_before INTERVAL); CREATE FUNCTION @extschema@.add_compression_policy( hypertable REGCLASS, compress_after "any" = NULL, if_not_exists BOOL = false, schedule_interval INTERVAL = NULL, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL, compress_created_before INTERVAL = NULL, hypercore_use_access_method BOOL = NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; DROP FUNCTION IF EXISTS timescaledb_experimental.add_policies(relation REGCLASS, if_not_exists BOOL, refresh_start_offset "any", refresh_end_offset "any", compress_after "any", drop_after "any"); CREATE FUNCTION timescaledb_experimental.add_policies( relation REGCLASS, if_not_exists BOOL = false, refresh_start_offset "any" = NULL, refresh_end_offset "any" = NULL, compress_after "any" = NULL, drop_after "any" = NULL, hypercore_use_access_method BOOL = NULL) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; DROP PROCEDURE IF EXISTS _timescaledb_functions.policy_compression_execute(job_id INTEGER, htid INTEGER, lag ANYELEMENT, maxchunks INTEGER, verbose_log BOOLEAN, recompress_enabled BOOLEAN, use_creation_time BOOLEAN); DROP PROCEDURE IF EXISTS _timescaledb_functions.policy_compression(job_id INTEGER, config JSONB); CREATE PROCEDURE @extschema@.convert_to_columnstore( chunk REGCLASS, if_not_columnstore BOOLEAN = true, recompress BOOLEAN = false, hypercore_use_access_method BOOL = NULL) AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C; CREATE PROCEDURE @extschema@.convert_to_rowstore( chunk REGCLASS, if_columnstore BOOLEAN = true) AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C; CREATE PROCEDURE @extschema@.add_columnstore_policy( hypertable REGCLASS, after "any" = NULL, if_not_exists BOOL = false, schedule_interval INTERVAL = NULL, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL, created_before INTERVAL = NULL, hypercore_use_access_method BOOL = NULL ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; CREATE PROCEDURE @extschema@.remove_columnstore_policy( hypertable REGCLASS, if_exists BOOL = false ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; CREATE FUNCTION @extschema@.chunk_columnstore_stats (hypertable REGCLASS) RETURNS TABLE ( chunk_schema name, chunk_name name, compression_status text, before_compression_table_bytes bigint, before_compression_index_bytes bigint, before_compression_toast_bytes bigint, before_compression_total_bytes bigint, after_compression_table_bytes bigint, after_compression_index_bytes bigint, after_compression_toast_bytes bigint, after_compression_total_bytes bigint, node_name name) LANGUAGE SQL STABLE STRICT AS 'SELECT * FROM @extschema@.chunk_compression_stats($1)' SET search_path TO pg_catalog, pg_temp; CREATE FUNCTION @extschema@.hypertable_columnstore_stats (hypertable REGCLASS) RETURNS TABLE ( total_chunks bigint, number_compressed_chunks bigint, before_compression_table_bytes bigint, before_compression_index_bytes bigint, before_compression_toast_bytes bigint, before_compression_total_bytes bigint, after_compression_table_bytes bigint, after_compression_index_bytes bigint, after_compression_toast_bytes bigint, after_compression_total_bytes bigint, node_name name) LANGUAGE SQL STABLE STRICT AS 'SELECT * FROM @extschema@.hypertable_compression_stats($1)' SET search_path TO pg_catalog, pg_temp; -- Recreate `refresh_continuous_aggregate` procedure to add `force` argument DROP PROCEDURE IF EXISTS @extschema@.refresh_continuous_aggregate (continuous_aggregate REGCLASS, window_start "any", window_end "any"); CREATE PROCEDURE @extschema@.refresh_continuous_aggregate( continuous_aggregate REGCLASS, window_start "any", window_end "any", force BOOLEAN = FALSE ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; -- Add `include_tiered_data` argument to `add_continuous_aggregate_policy` DROP FUNCTION @extschema@.add_continuous_aggregate_policy( continuous_aggregate REGCLASS, start_offset "any", end_offset "any", schedule_interval INTERVAL, if_not_exists BOOL, initial_start TIMESTAMPTZ, timezone TEXT ); CREATE FUNCTION @extschema@.add_continuous_aggregate_policy( continuous_aggregate REGCLASS, start_offset "any", end_offset "any", schedule_interval INTERVAL, if_not_exists BOOL = false, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL, include_tiered_data BOOL = NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; -- Merge chunks CREATE PROCEDURE @extschema@.merge_chunks( chunk1 REGCLASS, chunk2 REGCLASS ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; CREATE PROCEDURE @extschema@.merge_chunks( chunks REGCLASS[] ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; CREATE FUNCTION ts_hypercore_handler(internal) RETURNS table_am_handler AS 'heap_tableam_handler' LANGUAGE internal; CREATE FUNCTION ts_hypercore_proxy_handler(internal) RETURNS index_am_handler AS 'bthandler' LANGUAGE internal; CREATE ACCESS METHOD hypercore TYPE TABLE HANDLER ts_hypercore_handler; COMMENT ON ACCESS METHOD hypercore IS 'Storage engine using hybrid row/columnar compression'; CREATE ACCESS METHOD hypercore_proxy TYPE INDEX HANDLER ts_hypercore_proxy_handler; COMMENT ON ACCESS METHOD hypercore_proxy IS 'Hypercore proxy index access method'; CREATE OPERATOR CLASS int4_ops DEFAULT FOR TYPE int4 USING hypercore_proxy AS OPERATOR 1 = (int4, int4), FUNCTION 1 hashint4(int4); ================================================ FILE: sql/updates/2.18.0--2.17.2.sql ================================================ -- Hypercore AM DROP ACCESS METHOD IF EXISTS hypercore_proxy; DROP FUNCTION IF EXISTS ts_hypercore_proxy_handler; DROP ACCESS METHOD IF EXISTS hypercore; DROP FUNCTION IF EXISTS ts_hypercore_handler; DROP FUNCTION IF EXISTS _timescaledb_debug.is_compressed_tid; DROP FUNCTION IF EXISTS @extschema@.compress_chunk(uncompressed_chunk REGCLASS, if_not_compressed BOOLEAN, recompress BOOLEAN, hypercore_use_access_method BOOL); CREATE FUNCTION @extschema@.compress_chunk( uncompressed_chunk REGCLASS, if_not_compressed BOOLEAN = true, recompress BOOLEAN = false ) RETURNS REGCLASS AS '@MODULE_PATHNAME@', 'ts_compress_chunk' LANGUAGE C STRICT VOLATILE; DROP FUNCTION IF EXISTS @extschema@.add_compression_policy(hypertable REGCLASS, compress_after "any", if_not_exists BOOL, schedule_interval INTERVAL, initial_start TIMESTAMPTZ, timezone TEXT, compress_created_before INTERVAL, hypercore_use_access_method BOOL); CREATE FUNCTION @extschema@.add_compression_policy( hypertable REGCLASS, compress_after "any" = NULL, if_not_exists BOOL = false, schedule_interval INTERVAL = NULL, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL, compress_created_before INTERVAL = NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_policy_compression_add' LANGUAGE C VOLATILE; DROP FUNCTION IF EXISTS timescaledb_experimental.add_policies(relation REGCLASS, if_not_exists BOOL, refresh_start_offset "any", refresh_end_offset "any", compress_after "any", drop_after "any", hypercore_use_access_method BOOL); CREATE FUNCTION timescaledb_experimental.add_policies( relation REGCLASS, if_not_exists BOOL = false, refresh_start_offset "any" = NULL, refresh_end_offset "any" = NULL, compress_after "any" = NULL, drop_after "any" = NULL) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_policies_add' LANGUAGE C VOLATILE; DROP PROCEDURE IF EXISTS _timescaledb_functions.policy_compression_execute(job_id INTEGER, htid INTEGER, lag ANYELEMENT, maxchunks INTEGER, verbose_log BOOLEAN, recompress_enabled BOOLEAN, use_creation_time BOOLEAN, useam BOOLEAN); DROP PROCEDURE IF EXISTS _timescaledb_functions.policy_compression(job_id INTEGER, config JSONB); DROP PROCEDURE IF EXISTS @extschema@.convert_to_columnstore(REGCLASS, BOOLEAN, BOOLEAN, BOOLEAN); DROP PROCEDURE IF EXISTS @extschema@.convert_to_rowstore(REGCLASS, BOOLEAN); DROP PROCEDURE IF EXISTS @extschema@.add_columnstore_policy(REGCLASS, "any", BOOL, INTERVAL, TIMESTAMPTZ, TEXT, INTERVAL, BOOL); DROP PROCEDURE IF EXISTS @extschema@.remove_columnstore_policy(REGCLASS, BOOL); DROP FUNCTION IF EXISTS @extschema@.hypertable_columnstore_stats(REGCLASS); DROP FUNCTION IF EXISTS @extschema@.chunk_columnstore_stats(REGCLASS); ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.hypertable_columnstore_settings; ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.chunk_columnstore_settings; DROP VIEW timescaledb_information.hypertable_columnstore_settings; DROP VIEW timescaledb_information.chunk_columnstore_settings; DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_update_watermark(INTEGER); -- Recreate `refresh_continuous_aggregate` procedure to remove the `force` argument DROP PROCEDURE IF EXISTS @extschema@.refresh_continuous_aggregate (continuous_aggregate REGCLASS, window_start "any", window_end "any", force BOOLEAN); CREATE PROCEDURE @extschema@.refresh_continuous_aggregate( continuous_aggregate REGCLASS, window_start "any", window_end "any" ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_continuous_agg_refresh'; -- Remove `include_tiered_data` argument from `add_continuous_aggregate_policy` DROP FUNCTION @extschema@.add_continuous_aggregate_policy( continuous_aggregate REGCLASS, start_offset "any", end_offset "any", schedule_interval INTERVAL, if_not_exists BOOL, initial_start TIMESTAMPTZ, timezone TEXT, include_tiered_data BOOL ); CREATE FUNCTION @extschema@.add_continuous_aggregate_policy( continuous_aggregate REGCLASS, start_offset "any", end_offset "any", schedule_interval INTERVAL, if_not_exists BOOL = false, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_policy_refresh_cagg_add' LANGUAGE C VOLATILE; -- Merge chunks DROP PROCEDURE IF EXISTS @extschema@.merge_chunks(chunk1 REGCLASS, chunk2 REGCLASS); DROP PROCEDURE IF EXISTS @extschema@.merge_chunks(chunks REGCLASS[]); ================================================ FILE: sql/updates/2.18.0--2.18.1.sql ================================================ ================================================ FILE: sql/updates/2.18.1--2.18.0.sql ================================================ ================================================ FILE: sql/updates/2.18.1--2.18.2.sql ================================================ ALTER TABLE _timescaledb_internal.bgw_job_stat_history ALTER COLUMN succeeded DROP NOT NULL, ALTER COLUMN succeeded DROP DEFAULT; ================================================ FILE: sql/updates/2.18.2--2.18.1.sql ================================================ UPDATE _timescaledb_internal.bgw_job_stat_history SET succeeded = FALSE WHERE succeeded IS NULL; ALTER TABLE _timescaledb_internal.bgw_job_stat_history ALTER COLUMN succeeded SET NOT NULL, ALTER COLUMN succeeded SET DEFAULT FALSE; ================================================ FILE: sql/updates/2.18.2--2.19.0.sql ================================================ CREATE FUNCTION _timescaledb_functions.compressed_data_has_nulls(_timescaledb_internal.compressed_data) RETURNS BOOL LANGUAGE C STRICT IMMUTABLE AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; INSERT INTO _timescaledb_catalog.compression_algorithm( id, version, name, description) values ( 5, 1, 'COMPRESSION_ALGORITHM_BOOL', 'bool'), ( 6, 1, 'COMPRESSION_ALGORITHM_NULL', 'null') ; ------------------------------- -- Update compression settings ------------------------------- CREATE TABLE _timescaledb_catalog.tempsettings (LIKE _timescaledb_catalog.compression_settings); INSERT INTO _timescaledb_catalog.tempsettings SELECT * FROM _timescaledb_catalog.compression_settings; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.compression_settings; DROP TABLE _timescaledb_catalog.compression_settings CASCADE; CREATE TABLE _timescaledb_catalog.compression_settings ( relid regclass NOT NULL, compress_relid regclass NULL, segmentby text[], orderby text[], orderby_desc bool[], orderby_nullsfirst bool[], CONSTRAINT compression_settings_pkey PRIMARY KEY (relid), CONSTRAINT compression_settings_check_segmentby CHECK (array_ndims(segmentby) = 1), CONSTRAINT compression_settings_check_orderby_null CHECK ((orderby IS NULL AND orderby_desc IS NULL AND orderby_nullsfirst IS NULL) OR (orderby IS NOT NULL AND orderby_desc IS NOT NULL AND orderby_nullsfirst IS NOT NULL)), CONSTRAINT compression_settings_check_orderby_cardinality CHECK (array_ndims(orderby) = 1 AND array_ndims(orderby_desc) = 1 AND array_ndims(orderby_nullsfirst) = 1 AND cardinality(orderby) = cardinality(orderby_desc) AND cardinality(orderby) = cardinality(orderby_nullsfirst)) ); -- Insert updated settings INSERT INTO _timescaledb_catalog.compression_settings SELECT CASE WHEN h.schema_name IS NOT NULL THEN cs.relid ELSE format('%I.%I', ch.schema_name, ch.table_name)::regclass END AS relid, CASE WHEN h.schema_name IS NOT NULL THEN NULL ELSE cs.relid END AS compress_relid, cs.segmentby, cs.orderby, cs.orderby_desc, cs.orderby_nullsfirst FROM _timescaledb_catalog.tempsettings cs INNER JOIN pg_class c ON (cs.relid = c.oid) INNER JOIN pg_namespace ns ON (ns.oid = c.relnamespace) LEFT JOIN _timescaledb_catalog.hypertable h ON (h.schema_name = ns.nspname AND h.table_name = c.relname) LEFT JOIN _timescaledb_catalog.chunk cch ON (cch.schema_name = ns.nspname AND cch.table_name = c.relname) LEFT JOIN _timescaledb_catalog.chunk ch ON (cch.id = ch.compressed_chunk_id); -- Add index on secondary compressed relid key CREATE INDEX compression_settings_compress_relid_idx ON _timescaledb_catalog.compression_settings (compress_relid); DROP TABLE _timescaledb_catalog.tempsettings CASCADE; GRANT SELECT ON _timescaledb_catalog.compression_settings TO PUBLIC; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.compression_settings', ''); -- New add_continuous_aggregate_policy API for incremental refresh policy DROP FUNCTION @extschema@.add_continuous_aggregate_policy( continuous_aggregate REGCLASS, start_offset "any", end_offset "any", schedule_interval INTERVAL, if_not_exists BOOL, initial_start TIMESTAMPTZ, timezone TEXT, include_tiered_data BOOL ); CREATE FUNCTION @extschema@.add_continuous_aggregate_policy( continuous_aggregate REGCLASS, start_offset "any", end_offset "any", schedule_interval INTERVAL, if_not_exists BOOL = false, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL, include_tiered_data BOOL = NULL, buckets_per_batch INTEGER = NULL, max_batches_per_execution INTEGER = NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; ================================================ FILE: sql/updates/2.19.0--2.18.2.sql ================================================ DROP FUNCTION IF EXISTS _timescaledb_functions.compressed_data_has_nulls(_timescaledb_internal.compressed_data); DELETE FROM _timescaledb_catalog.compression_algorithm WHERE id = 5 AND version = 1 AND name = 'COMPRESSION_ALGORITHM_BOOL'; DELETE FROM _timescaledb_catalog.compression_algorithm WHERE id = 6 AND version = 1 AND name = 'COMPRESSION_ALGORITHM_NULL'; -- Update compression settings CREATE TABLE _timescaledb_catalog.tempsettings (LIKE _timescaledb_catalog.compression_settings); INSERT INTO _timescaledb_catalog.tempsettings SELECT * FROM _timescaledb_catalog.compression_settings; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.compression_settings; DROP TABLE _timescaledb_catalog.compression_settings CASCADE; CREATE TABLE _timescaledb_catalog.compression_settings ( relid regclass NOT NULL, segmentby text[], orderby text[], orderby_desc bool[], orderby_nullsfirst bool[], CONSTRAINT compression_settings_pkey PRIMARY KEY (relid), CONSTRAINT compression_settings_check_segmentby CHECK (array_ndims(segmentby) = 1), CONSTRAINT compression_settings_check_orderby_null CHECK ((orderby IS NULL AND orderby_desc IS NULL AND orderby_nullsfirst IS NULL) OR (orderby IS NOT NULL AND orderby_desc IS NOT NULL AND orderby_nullsfirst IS NOT NULL)), CONSTRAINT compression_settings_check_orderby_cardinality CHECK (array_ndims(orderby) = 1 AND array_ndims(orderby_desc) = 1 AND array_ndims(orderby_nullsfirst) = 1 AND cardinality(orderby) = cardinality(orderby_desc) AND cardinality(orderby) = cardinality(orderby_nullsfirst)) ); -- Revert information in compression settings INSERT INTO _timescaledb_catalog.compression_settings SELECT CASE WHEN cs.compress_relid IS NULL THEN cs.relid ELSE cs.compress_relid END as relid, cs.segmentby, cs.orderby, cs.orderby_desc, cs.orderby_nullsfirst FROM _timescaledb_catalog.tempsettings cs; DROP TABLE _timescaledb_catalog.tempsettings CASCADE; GRANT SELECT ON _timescaledb_catalog.compression_settings TO PUBLIC; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.compression_settings', ''); -- Revert add_continuous_aggregate_policy API for incremental refresh policy DROP FUNCTION @extschema@.add_continuous_aggregate_policy( continuous_aggregate REGCLASS, start_offset "any", end_offset "any", schedule_interval INTERVAL, if_not_exists BOOL, initial_start TIMESTAMPTZ, timezone TEXT, include_tiered_data BOOL, buckets_per_batch INTEGER, max_batches_per_execution INTEGER ); CREATE FUNCTION @extschema@.add_continuous_aggregate_policy( continuous_aggregate REGCLASS, start_offset "any", end_offset "any", schedule_interval INTERVAL, if_not_exists BOOL = false, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL, include_tiered_data BOOL = NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; ================================================ FILE: sql/updates/2.19.0--2.19.1.sql ================================================ ================================================ FILE: sql/updates/2.19.1--2.19.0.sql ================================================ ================================================ FILE: sql/updates/2.19.1--2.19.2.sql ================================================ ================================================ FILE: sql/updates/2.19.2--2.19.1.sql ================================================ ================================================ FILE: sql/updates/2.19.2--2.19.3.sql ================================================ ================================================ FILE: sql/updates/2.19.3--2.19.2.sql ================================================ ================================================ FILE: sql/updates/2.19.3--2.20.0.sql ================================================ -- Type for bloom filters used by the sparse indexes on compressed hypertables. CREATE TYPE _timescaledb_internal.bloom1; CREATE FUNCTION _timescaledb_functions.bloom1in(cstring) RETURNS _timescaledb_internal.bloom1 AS 'byteain' LANGUAGE INTERNAL STRICT IMMUTABLE PARALLEL SAFE; CREATE FUNCTION _timescaledb_functions.bloom1out(_timescaledb_internal.bloom1) RETURNS cstring AS 'byteaout' LANGUAGE INTERNAL STRICT IMMUTABLE PARALLEL SAFE; CREATE TYPE _timescaledb_internal.bloom1 ( INPUT = _timescaledb_functions.bloom1in, OUTPUT = _timescaledb_functions.bloom1out, LIKE = bytea ); CREATE FUNCTION _timescaledb_functions.bloom1_contains(_timescaledb_internal.bloom1, anyelement) RETURNS bool AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; DROP FUNCTION IF EXISTS _timescaledb_internal.create_chunk_table; DROP FUNCTION IF EXISTS _timescaledb_functions.create_chunk_table; -- New option `refresh_newest_first` for incremental cagg refresh policy DROP FUNCTION @extschema@.add_continuous_aggregate_policy( continuous_aggregate REGCLASS, start_offset "any", end_offset "any", schedule_interval INTERVAL, if_not_exists BOOL, initial_start TIMESTAMPTZ, timezone TEXT, include_tiered_data BOOL, buckets_per_batch INTEGER, max_batches_per_execution INTEGER ); CREATE FUNCTION @extschema@.add_continuous_aggregate_policy( continuous_aggregate REGCLASS, start_offset "any", end_offset "any", schedule_interval INTERVAL, if_not_exists BOOL = false, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL, include_tiered_data BOOL = NULL, buckets_per_batch INTEGER = NULL, max_batches_per_execution INTEGER = NULL, refresh_newest_first BOOL = NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; UPDATE _timescaledb_catalog.hypertable SET chunk_sizing_func_schema = '_timescaledb_functions' WHERE chunk_sizing_func_schema = '_timescaledb_internal' AND chunk_sizing_func_name = 'calculate_chunk_interval'; DROP VIEW IF EXISTS timescaledb_information.hypertables; -- Rename Columnstore Policy jobs to Compression Policy UPDATE _timescaledb_config.bgw_job SET application_name = replace(application_name, 'Compression Policy', 'Columnstore Policy') WHERE application_name LIKE '%Compression Policy%'; -- Split chunk CREATE PROCEDURE @extschema@.split_chunk( chunk REGCLASS, split_at "any" = NULL ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; CREATE FUNCTION _timescaledb_functions.align_to_bucket(width INTERVAL, rng ANYRANGE) RETURNS ANYRANGE AS $body$ BEGIN RETURN _timescaledb_functions.make_range_from_internal_time( rng, @extschema@.time_bucket(width, lower(rng)), @extschema@.time_bucket(width, upper(rng) - '1 microsecond'::interval) + width ); END $body$ LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE SET search_path TO pg_catalog, pg_temp; CREATE FUNCTION _timescaledb_functions.make_multirange_from_internal_time( base TSTZRANGE, low_usec BIGINT, high_usec BIGINT ) RETURNS TSTZMULTIRANGE AS $body$ select multirange(tstzrange(_timescaledb_functions.to_timestamp(low_usec), _timescaledb_functions.to_timestamp(high_usec))); $body$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET search_path TO pg_catalog, pg_temp; CREATE FUNCTION _timescaledb_functions.make_multirange_from_internal_time( base TSRANGE, low_usec BIGINT, high_usec BIGINT ) RETURNS TSMULTIRANGE AS $body$ select multirange(tsrange(_timescaledb_functions.to_timestamp_without_timezone(low_usec), _timescaledb_functions.to_timestamp_without_timezone(high_usec))); $body$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET search_path TO pg_catalog, pg_temp; CREATE FUNCTION _timescaledb_functions.make_range_from_internal_time( base ANYRANGE, low_usec ANYELEMENT, high_usec ANYELEMENT ) RETURNS anyrange AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.get_internal_time_min(REGTYPE) RETURNS BIGINT AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.get_internal_time_max(REGTYPE) RETURNS BIGINT AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; DROP FUNCTION IF EXISTS @extschema@.add_job( proc REGPROC, schedule_interval INTERVAL, config JSONB, initial_start TIMESTAMPTZ, scheduled BOOL, check_config REGPROC, fixed_schedule BOOL, timezone TEXT ); CREATE FUNCTION @extschema@.add_job( proc REGPROC, schedule_interval INTERVAL, config JSONB DEFAULT NULL, initial_start TIMESTAMPTZ DEFAULT NULL, scheduled BOOL DEFAULT true, check_config REGPROC DEFAULT NULL, fixed_schedule BOOL DEFAULT TRUE, timezone TEXT DEFAULT NULL, job_name TEXT DEFAULT NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; DROP FUNCTION IF EXISTS @extschema@.alter_job( job_id INTEGER, schedule_interval INTERVAL, max_runtime INTERVAL, max_retries INTEGER, retry_period INTERVAL, scheduled BOOL, config JSONB, next_start TIMESTAMPTZ, if_exists BOOL, check_config REGPROC, fixed_schedule BOOL, initial_start TIMESTAMPTZ, timezone TEXT ); CREATE FUNCTION @extschema@.alter_job( job_id INTEGER, schedule_interval INTERVAL = NULL, max_runtime INTERVAL = NULL, max_retries INTEGER = NULL, retry_period INTERVAL = NULL, scheduled BOOL = NULL, config JSONB = NULL, next_start TIMESTAMPTZ = NULL, if_exists BOOL = FALSE, check_config REGPROC = NULL, fixed_schedule BOOL = NULL, initial_start TIMESTAMPTZ = NULL, timezone TEXT DEFAULT NULL, job_name TEXT DEFAULT NULL ) RETURNS TABLE (job_id INTEGER, schedule_interval INTERVAL, max_runtime INTERVAL, max_retries INTEGER, retry_period INTERVAL, scheduled BOOL, config JSONB, next_start TIMESTAMPTZ, check_config TEXT, fixed_schedule BOOL, initial_start TIMESTAMPTZ, timezone TEXT, application_name name) AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; ================================================ FILE: sql/updates/2.20.0--2.19.3.sql ================================================ -- Drop the type used by the bloom sparse indexes on compressed hypertables. DROP TYPE _timescaledb_internal.bloom1 CASCADE; CREATE FUNCTION _timescaledb_internal.create_chunk_table(hypertable REGCLASS, slices JSONB, schema_name NAME, table_name NAME) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; CREATE FUNCTION _timescaledb_functions.create_chunk_table(hypertable REGCLASS, slices JSONB, schema_name NAME, table_name NAME) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; DROP FUNCTION _timescaledb_functions.get_hypertable_id(REGCLASS, REGTYPE); DROP FUNCTION _timescaledb_functions.get_hypertable_invalidations(REGCLASS, TIMESTAMPTZ, INTERVAL[]); DROP FUNCTION _timescaledb_functions.get_hypertable_invalidations(REGCLASS, TIMESTAMP, INTERVAL[]); DROP PROCEDURE _timescaledb_functions.accept_hypertable_invalidations(REGCLASS, TEXT); -- Revert new option `refresh_newest_first` from incremental cagg refresh policy DROP FUNCTION @extschema@.add_continuous_aggregate_policy( continuous_aggregate REGCLASS, start_offset "any", end_offset "any", schedule_interval INTERVAL, if_not_exists BOOL, initial_start TIMESTAMPTZ, timezone TEXT, include_tiered_data BOOL, buckets_per_batch INTEGER, max_batches_per_execution INTEGER, refresh_newest_first BOOL ); CREATE FUNCTION @extschema@.add_continuous_aggregate_policy( continuous_aggregate REGCLASS, start_offset "any", end_offset "any", schedule_interval INTERVAL, if_not_exists BOOL = false, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL, include_tiered_data BOOL = NULL, buckets_per_batch INTEGER = NULL, max_batches_per_execution INTEGER = NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; DROP VIEW IF EXISTS timescaledb_information.hypertables; -- Rename Columnstore Policy jobs to Compression Policy UPDATE _timescaledb_config.bgw_job SET application_name = replace(application_name, 'Columnstore Policy', 'Compression Policy') WHERE application_name LIKE '%Columnstore Policy%'; CREATE OR REPLACE PROCEDURE _timescaledb_functions.policy_compression_execute( job_id INTEGER, htid INTEGER, lag ANYELEMENT, maxchunks INTEGER, verbose_log BOOLEAN, recompress_enabled BOOLEAN, use_creation_time BOOLEAN, useam BOOLEAN = NULL) AS $$ DECLARE htoid REGCLASS; chunk_rec RECORD; numchunks INTEGER := 1; _message text; _detail text; _sqlstate text; -- fully compressed chunk status status_fully_compressed int := 1; -- chunk status bits: bit_compressed int := 1; bit_compressed_unordered int := 2; bit_frozen int := 4; bit_compressed_partial int := 8; creation_lag INTERVAL := NULL; chunks_failure INTEGER := 0; BEGIN -- procedures with SET clause cannot execute transaction -- control so we adjust search_path in procedure body SET LOCAL search_path TO pg_catalog, pg_temp; SELECT format('%I.%I', schema_name, table_name) INTO htoid FROM _timescaledb_catalog.hypertable WHERE id = htid; -- for the integer cases, we have to compute the lag w.r.t -- the integer_now function and then pass on to show_chunks IF pg_typeof(lag) IN ('BIGINT'::regtype, 'INTEGER'::regtype, 'SMALLINT'::regtype) THEN -- cannot have use_creation_time set with this IF use_creation_time IS TRUE THEN RAISE EXCEPTION 'job % cannot use creation time with integer_now function', job_id; END IF; lag := _timescaledb_functions.subtract_integer_from_now(htoid, lag::BIGINT); END IF; -- if use_creation_time has been specified then the lag needs to be used with the -- "compress_created_before" argument. Otherwise the usual "older_than" argument -- is good enough IF use_creation_time IS TRUE THEN creation_lag := lag; lag := NULL; END IF; FOR chunk_rec IN SELECT show.oid, ch.schema_name, ch.table_name, ch.status FROM @extschema@.show_chunks(htoid, older_than => lag, created_before => creation_lag) AS show(oid) INNER JOIN pg_class pgc ON pgc.oid = show.oid INNER JOIN pg_namespace pgns ON pgc.relnamespace = pgns.oid INNER JOIN _timescaledb_catalog.chunk ch ON ch.table_name = pgc.relname AND ch.schema_name = pgns.nspname AND ch.hypertable_id = htid WHERE NOT ch.dropped AND NOT ch.osm_chunk -- Checking for chunks which are not fully compressed and not frozen AND ch.status != status_fully_compressed AND ch.status & bit_frozen = 0 LOOP BEGIN IF chunk_rec.status = bit_compressed OR recompress_enabled IS TRUE THEN PERFORM @extschema@.compress_chunk(chunk_rec.oid, hypercore_use_access_method => useam); numchunks := numchunks + 1; END IF; EXCEPTION WHEN OTHERS THEN GET STACKED DIAGNOSTICS _message = MESSAGE_TEXT, _detail = PG_EXCEPTION_DETAIL, _sqlstate = RETURNED_SQLSTATE; RAISE WARNING 'compressing chunk "%" failed when compression policy is executed', chunk_rec.oid::regclass::text USING DETAIL = format('Message: (%s), Detail: (%s).', _message, _detail), ERRCODE = _sqlstate; chunks_failure := chunks_failure + 1; END; COMMIT; -- SET LOCAL is only active until end of transaction. -- While we could use SET at the start of the function we do not -- want to bleed out search_path to caller, so we do SET LOCAL -- again after COMMIT SET LOCAL search_path TO pg_catalog, pg_temp; IF verbose_log THEN RAISE LOG 'job % completed processing chunk %.%', job_id, chunk_rec.schema_name, chunk_rec.table_name; END IF; IF maxchunks > 0 AND numchunks >= maxchunks THEN EXIT; END IF; END LOOP; IF chunks_failure > 0 THEN RAISE EXCEPTION 'compression policy failure' USING DETAIL = format('Failed to compress %L chunks. Successfully compressed %L chunks.', chunks_failure, numchunks - chunks_failure); END IF; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE _timescaledb_functions.policy_compression(job_id INTEGER, config JSONB) AS $$ DECLARE dimtype REGTYPE; dimtypeinput REGPROC; compress_after TEXT; compress_created_before TEXT; lag_value TEXT; lag_bigint_value BIGINT; htid INTEGER; htoid REGCLASS; chunk_rec RECORD; verbose_log BOOL; maxchunks INTEGER := 0; numchunks INTEGER := 1; recompress_enabled BOOL; use_creation_time BOOL := FALSE; hypercore_use_access_method BOOL; BEGIN -- procedures with SET clause cannot execute transaction -- control so we adjust search_path in procedure body SET LOCAL search_path TO pg_catalog, pg_temp; IF config IS NULL THEN RAISE EXCEPTION 'job % has null config', job_id; END IF; htid := jsonb_object_field_text(config, 'hypertable_id')::INTEGER; IF htid is NULL THEN RAISE EXCEPTION 'job % config must have hypertable_id', job_id; END IF; verbose_log := COALESCE(jsonb_object_field_text(config, 'verbose_log')::BOOLEAN, FALSE); maxchunks := COALESCE(jsonb_object_field_text(config, 'maxchunks_to_compress')::INTEGER, 0); recompress_enabled := COALESCE(jsonb_object_field_text(config, 'recompress')::BOOLEAN, TRUE); -- find primary dimension type -- SELECT dim.column_type INTO dimtype FROM _timescaledb_catalog.hypertable ht JOIN _timescaledb_catalog.dimension dim ON ht.id = dim.hypertable_id WHERE ht.id = htid ORDER BY dim.id LIMIT 1; compress_after := jsonb_object_field_text(config, 'compress_after'); IF compress_after IS NULL THEN compress_created_before := jsonb_object_field_text(config, 'compress_created_before'); IF compress_created_before IS NULL THEN RAISE EXCEPTION 'job % config must have compress_after or compress_created_before', job_id; END IF; lag_value := compress_created_before; use_creation_time := true; dimtype := 'INTERVAL' ::regtype; ELSE lag_value := compress_after; END IF; hypercore_use_access_method := jsonb_object_field_text(config, 'hypercore_use_access_method')::bool; -- execute the properly type casts for the lag value CASE dimtype WHEN 'TIMESTAMP'::regtype, 'TIMESTAMPTZ'::regtype, 'DATE'::regtype, 'INTERVAL' ::regtype THEN CALL _timescaledb_functions.policy_compression_execute( job_id, htid, lag_value::INTERVAL, maxchunks, verbose_log, recompress_enabled, use_creation_time, hypercore_use_access_method ); WHEN 'BIGINT'::regtype THEN CALL _timescaledb_functions.policy_compression_execute( job_id, htid, lag_value::BIGINT, maxchunks, verbose_log, recompress_enabled, use_creation_time, hypercore_use_access_method ); WHEN 'INTEGER'::regtype THEN CALL _timescaledb_functions.policy_compression_execute( job_id, htid, lag_value::INTEGER, maxchunks, verbose_log, recompress_enabled, use_creation_time, hypercore_use_access_method ); WHEN 'SMALLINT'::regtype THEN CALL _timescaledb_functions.policy_compression_execute( job_id, htid, lag_value::SMALLINT, maxchunks, verbose_log, recompress_enabled, use_creation_time, hypercore_use_access_method ); END CASE; END; $$ LANGUAGE PLPGSQL; -- Split chunk DROP PROCEDURE IF EXISTS @extschema@.split_chunk(chunk REGCLASS, split_at "any"); DROP FUNCTION _timescaledb_functions.align_to_bucket(INTERVAL, ANYRANGE); DROP FUNCTION _timescaledb_functions.make_multirange_from_internal_time(TSTZRANGE, BIGINT, BIGINT); DROP FUNCTION _timescaledb_functions.make_multirange_from_internal_time(TSRANGE, BIGINT, BIGINT); DROP FUNCTION _timescaledb_functions.make_range_from_internal_time(ANYRANGE, ANYELEMENT, ANYELEMENT); DROP FUNCTION _timescaledb_functions.get_internal_time_min(REGTYPE); DROP FUNCTION _timescaledb_functions.get_internal_time_max(REGTYPE); DROP PROCEDURE _timescaledb_functions.add_materialization_invalidations(REGCLASS,TSRANGE); DROP PROCEDURE _timescaledb_functions.add_materialization_invalidations(REGCLASS,TSTZRANGE); DROP FUNCTION _timescaledb_functions.get_raw_materialization_ranges(REGTYPE); DROP FUNCTION _timescaledb_functions.get_materialization_invalidations(REGCLASS, TSTZRANGE); DROP FUNCTION _timescaledb_functions.get_materialization_invalidations(REGCLASS, TSRANGE); DROP FUNCTION _timescaledb_functions.get_materialization_info(REGCLASS); DROP FUNCTION IF EXISTS @extschema@.add_job( proc REGPROC, schedule_interval INTERVAL, config JSONB, initial_start TIMESTAMPTZ, scheduled BOOL, check_config REGPROC, fixed_schedule BOOL, timezone TEXT, job_name TEXT ); CREATE FUNCTION @extschema@.add_job( proc REGPROC, schedule_interval INTERVAL, config JSONB DEFAULT NULL, initial_start TIMESTAMPTZ DEFAULT NULL, scheduled BOOL DEFAULT true, check_config REGPROC DEFAULT NULL, fixed_schedule BOOL DEFAULT TRUE, timezone TEXT DEFAULT NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; DROP FUNCTION IF EXISTS @extschema@.alter_job( job_id INTEGER, schedule_interval INTERVAL, max_runtime INTERVAL, max_retries INTEGER, retry_period INTERVAL, scheduled BOOL, config JSONB, next_start TIMESTAMPTZ, if_exists BOOL, check_config REGPROC, fixed_schedule BOOL, initial_start TIMESTAMPTZ, timezone TEXT, job_name TEXT ); CREATE FUNCTION @extschema@.alter_job( job_id INTEGER, schedule_interval INTERVAL = NULL, max_runtime INTERVAL = NULL, max_retries INTEGER = NULL, retry_period INTERVAL = NULL, scheduled BOOL = NULL, config JSONB = NULL, next_start TIMESTAMPTZ = NULL, if_exists BOOL = FALSE, check_config REGPROC = NULL, fixed_schedule BOOL = NULL, initial_start TIMESTAMPTZ = NULL, timezone TEXT DEFAULT NULL ) RETURNS TABLE (job_id INTEGER, schedule_interval INTERVAL, max_runtime INTERVAL, max_retries INTEGER, retry_period INTERVAL, scheduled BOOL, config JSONB, next_start TIMESTAMPTZ, check_config TEXT, fixed_schedule BOOL, initial_start TIMESTAMPTZ, timezone TEXT) AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; ================================================ FILE: sql/updates/2.20.0--2.20.1.sql ================================================ ================================================ FILE: sql/updates/2.20.1--2.20.0.sql ================================================ ================================================ FILE: sql/updates/2.20.1--2.20.2.sql ================================================ ================================================ FILE: sql/updates/2.20.2--2.20.1.sql ================================================ ================================================ FILE: sql/updates/2.20.2--2.20.3.sql ================================================ -- Make chunk_id use NULL to mark special entries instead of 0 -- (Invalid chunk) since that doesn't work with the FK constraint on -- chunk_id. ALTER TABLE _timescaledb_catalog.chunk_column_stats ALTER COLUMN chunk_id DROP NOT NULL; UPDATE _timescaledb_catalog.chunk_column_stats SET chunk_id = NULL WHERE chunk_id = 0; ================================================ FILE: sql/updates/2.20.3--2.20.2.sql ================================================ -- Add back the chunk_column_stats NOT NULL constraint. But first -- delete all entries with with NULL since they will no longer be -- allowed. Note that reverting chunk_id back to 0 because it will -- violate the FK constraint. Even if we would revert, the downgrade -- tests for "restore" would fail due to violating entries. Removing -- the entries effectively means that collecting column stats for -- those columns will be disabled. It can be enabled again after -- downgrade. We emit a warning if anything was disabled. DO $$ DECLARE num_null_chunk_ids int; BEGIN SELECT count(*) INTO num_null_chunk_ids FROM _timescaledb_catalog.chunk_column_stats WHERE chunk_id IS NULL; IF num_null_chunk_ids > 0 THEN RAISE WARNING 'chunk skipping has been disabled for all hypertables' USING HINT = 'Use enable_chunk_skipping() to re-enable chunk skipping'; END IF; END $$; DELETE FROM _timescaledb_catalog.chunk_column_stats WHERE chunk_id IS NULL; ALTER TABLE _timescaledb_catalog.chunk_column_stats ALTER COLUMN chunk_id SET NOT NULL; ================================================ FILE: sql/updates/2.20.3--2.21.0.sql ================================================ CREATE PROCEDURE _timescaledb_functions.process_hypertable_invalidations( hypertable REGCLASS ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; CREATE PROCEDURE @extschema@.add_process_hypertable_invalidations_policy( hypertable REGCLASS, schedule_interval INTERVAL, if_not_exists BOOL = false, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; CREATE PROCEDURE @extschema@.remove_process_hypertable_invalidations_policy( hypertable REGCLASS, if_exists BOOL = false ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; DROP PROCEDURE IF EXISTS _timescaledb_functions.policy_compression(job_id INTEGER, config JSONB); DROP PROCEDURE IF EXISTS _timescaledb_internal.policy_compression_execute( INTEGER, INTEGER, ANYELEMENT, INTEGER, BOOLEAN, BOOLEAN, BOOLEAN ); DROP PROCEDURE IF EXISTS _timescaledb_functions.policy_compression_execute( INTEGER, INTEGER, ANYELEMENT, INTEGER, BOOLEAN, BOOLEAN, BOOLEAN, BOOLEAN ); CREATE PROCEDURE _timescaledb_functions.policy_compression_execute( job_id INTEGER, htid INTEGER, lag ANYELEMENT, maxchunks INTEGER, verbose_log BOOLEAN, recompress_enabled BOOLEAN, reindex_enabled BOOLEAN, use_creation_time BOOLEAN, useam BOOLEAN = NULL) AS $$ BEGIN -- empty body END; $$ LANGUAGE PLPGSQL; DROP PROCEDURE @extschema@.refresh_continuous_aggregate( continuous_aggregate REGCLASS, window_start "any", window_end "any", force BOOLEAN ); CREATE PROCEDURE @extschema@.refresh_continuous_aggregate( continuous_aggregate REGCLASS, window_start "any", window_end "any", force BOOLEAN = FALSE, options JSONB = NULL ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; -- since we forgot to add the compression algorithms in the previous release to the preinstall script -- we add them here with an ON CONFLICT DO NOTHING clause INSERT INTO _timescaledb_catalog.compression_algorithm( id, version, name, description) values ( 5, 1, 'COMPRESSION_ALGORITHM_BOOL', 'bool'), ( 6, 1, 'COMPRESSION_ALGORITHM_NULL', 'null') ON CONFLICT (id) DO NOTHING ; CREATE PROCEDURE @extschema@.detach_chunk( chunk REGCLASS ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; CREATE PROCEDURE @extschema@.attach_chunk( hypertable REGCLASS, chunk REGCLASS, slices JSONB ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; ================================================ FILE: sql/updates/2.21.0--2.20.3.sql ================================================ DROP PROCEDURE _timescaledb_functions.process_hypertable_invalidations(REGCLASS); DROP PROCEDURE @extschema@.add_process_hypertable_invalidations_policy(REGCLASS, INTERVAL, BOOL, TIMESTAMPTZ, TEXT); DROP PROCEDURE @extschema@.remove_process_hypertable_invalidations_policy(REGCLASS, BOOL); DROP PROCEDURE _timescaledb_functions.policy_process_hypertable_invalidations(INTEGER, JSONB); DROP FUNCTION _timescaledb_functions.policy_process_hypertable_invalidations_check(JSONB); DROP PROCEDURE IF EXISTS _timescaledb_functions.policy_compression(job_id INTEGER, config JSONB); DROP PROCEDURE IF EXISTS _timescaledb_functions.policy_compression_execute( INTEGER, INTEGER, ANYELEMENT, INTEGER, BOOLEAN, BOOLEAN, BOOLEAN, BOOLEAN, BOOLEAN ); CREATE PROCEDURE _timescaledb_functions.policy_compression_execute( job_id INTEGER, htid INTEGER, lag ANYELEMENT, maxchunks INTEGER, verbose_log BOOLEAN, recompress_enabled BOOLEAN, use_creation_time BOOLEAN, useam BOOLEAN = NULL) AS $$ BEGIN -- empty body END; $$ LANGUAGE PLPGSQL; DROP PROCEDURE @extschema@.refresh_continuous_aggregate( continuous_aggregate REGCLASS, window_start "any", window_end "any", force BOOLEAN, options JSONB ); CREATE PROCEDURE @extschema@.refresh_continuous_aggregate( continuous_aggregate REGCLASS, window_start "any", window_end "any", force BOOLEAN = FALSE ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; DROP PROCEDURE IF EXISTS @extschema@.detach_chunk(REGCLASS); DROP PROCEDURE IF EXISTS @extschema@.attach_chunk(REGCLASS, REGCLASS, JSONB); ================================================ FILE: sql/updates/2.21.0--2.21.1.sql ================================================ ================================================ FILE: sql/updates/2.21.1--2.21.0.sql ================================================ ================================================ FILE: sql/updates/2.21.1--2.21.2.sql ================================================ ================================================ FILE: sql/updates/2.21.2--2.21.1.sql ================================================ ================================================ FILE: sql/updates/2.21.2--2.21.3.sql ================================================ ================================================ FILE: sql/updates/2.21.3--2.21.2.sql ================================================ ================================================ FILE: sql/updates/2.21.3--2.21.4.sql ================================================ ================================================ FILE: sql/updates/2.21.4--2.22.0.sql ================================================ -- block upgrade if hypercore access method is still in use DO $$ BEGIN IF EXISTS(SELECT from pg_class c join pg_am am ON c.relam=am.oid AND am.amname='hypercore') THEN RAISE EXCEPTION 'TimescaleDB does no longer support the hypercore table access method. Convert all tables to heap access method before upgrading.'; END IF; END $$; DROP OPERATOR CLASS IF EXISTS int4_ops USING hypercore_proxy; DROP ACCESS METHOD IF EXISTS hypercore_proxy; DROP ACCESS METHOD IF EXISTS hypercore; DROP FUNCTION IF EXISTS ts_hypercore_proxy_handler; DROP FUNCTION IF EXISTS ts_hypercore_handler; DROP FUNCTION IF EXISTS _timescaledb_debug.is_compressed_tid; DROP PROCEDURE IF EXISTS _timescaledb_functions.policy_compression_execute; DROP FUNCTION IF EXISTS @extschema@.add_compression_policy; DROP PROCEDURE IF EXISTS @extschema@.add_columnstore_policy; DROP FUNCTION IF EXISTS timescaledb_experimental.add_policies; DROP FUNCTION IF EXISTS @extschema@.compress_chunk; DROP PROCEDURE IF EXISTS @extschema@.convert_to_columnstore; CREATE FUNCTION @extschema@.compress_chunk( uncompressed_chunk REGCLASS, if_not_compressed BOOLEAN = true, recompress BOOLEAN = false ) RETURNS REGCLASS AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; CREATE PROCEDURE @extschema@.convert_to_columnstore( chunk REGCLASS, if_not_columnstore BOOLEAN = true, recompress BOOLEAN = false ) AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C; CREATE FUNCTION @extschema@.add_compression_policy( hypertable REGCLASS, compress_after "any" = NULL, if_not_exists BOOL = false, schedule_interval INTERVAL = NULL, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL, compress_created_before INTERVAL = NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; CREATE PROCEDURE @extschema@.add_columnstore_policy( hypertable REGCLASS, after "any" = NULL, if_not_exists BOOL = false, schedule_interval INTERVAL = NULL, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL, created_before INTERVAL = NULL ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; CREATE FUNCTION timescaledb_experimental.add_policies( relation REGCLASS, if_not_exists BOOL = false, refresh_start_offset "any" = NULL, refresh_end_offset "any" = NULL, compress_after "any" = NULL, drop_after "any" = NULL ) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; CREATE PROCEDURE _timescaledb_functions.policy_compression_execute( job_id INTEGER, htid INTEGER, lag ANYELEMENT, maxchunks INTEGER, verbose_log BOOLEAN, recompress_enabled BOOLEAN, reindex_enabled BOOLEAN, use_creation_time BOOLEAN ) AS $$ BEGIN END $$ LANGUAGE PLPGSQL; INSERT INTO _timescaledb_catalog.compression_algorithm( id, version, name, description) values ( 7, 1, 'COMPRESSION_ALGORITHM_UUID', 'uuid'); DROP FUNCTION IF EXISTS _timescaledb_internal.chunk_index_clone(oid); DROP FUNCTION IF EXISTS _timescaledb_functions.chunk_index_clone(oid); DROP FUNCTION IF EXISTS _timescaledb_internal.chunk_index_replace(oid,oid); DROP FUNCTION IF EXISTS _timescaledb_functions.chunk_index_replace(oid,oid); -- Update compression settings CREATE TABLE _timescaledb_catalog.tempsettings (LIKE _timescaledb_catalog.compression_settings); INSERT INTO _timescaledb_catalog.tempsettings SELECT * FROM _timescaledb_catalog.compression_settings; DROP VIEW IF EXISTS timescaledb_information.hypertable_columnstore_settings; DROP VIEW IF EXISTS timescaledb_information.chunk_columnstore_settings; DROP VIEW IF EXISTS timescaledb_information.hypertable_compression_settings; DROP VIEW IF EXISTS timescaledb_information.chunk_compression_settings; DROP VIEW IF EXISTS timescaledb_information.compression_settings; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.compression_settings; DROP TABLE _timescaledb_catalog.compression_settings; CREATE TABLE _timescaledb_catalog.compression_settings ( relid regclass NOT NULL, compress_relid regclass NULL, segmentby text[], orderby text[], orderby_desc bool[], orderby_nullsfirst bool[], index jsonb, CONSTRAINT compression_settings_pkey PRIMARY KEY (relid), CONSTRAINT compression_settings_check_segmentby CHECK (array_ndims(segmentby) = 1), CONSTRAINT compression_settings_check_orderby_null CHECK ((orderby IS NULL AND orderby_desc IS NULL AND orderby_nullsfirst IS NULL) OR (orderby IS NOT NULL AND orderby_desc IS NOT NULL AND orderby_nullsfirst IS NOT NULL)), CONSTRAINT compression_settings_check_orderby_cardinality CHECK (array_ndims(orderby) = 1 AND array_ndims(orderby_desc) = 1 AND array_ndims(orderby_nullsfirst) = 1 AND cardinality(orderby) = cardinality(orderby_desc) AND cardinality(orderby) = cardinality(orderby_nullsfirst)) ); -- Insert updated settings INSERT INTO _timescaledb_catalog.compression_settings SELECT cs.relid, cs.compress_relid, cs.segmentby, cs.orderby, cs.orderby_desc, cs.orderby_nullsfirst FROM _timescaledb_catalog.tempsettings cs; -- Add index on secondary compressed relid key CREATE INDEX compression_settings_compress_relid_idx ON _timescaledb_catalog.compression_settings (compress_relid); DROP TABLE _timescaledb_catalog.tempsettings; GRANT SELECT ON _timescaledb_catalog.compression_settings TO PUBLIC; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.compression_settings', ''); CREATE FUNCTION _timescaledb_functions.jsonb_get_matching_index_entry( config jsonb, attr_name text, target_type text ) RETURNS jsonb AS $$ BEGIN --empty body RETURN NULL; END; $$ LANGUAGE PLPGSQL SET search_path TO pg_catalog, pg_temp; --migration script CREATE FUNCTION process_single_table_index_migrate( IN target_schema TEXT, IN target_table TEXT ) RETURNS jsonb AS $$ DECLARE col_name TEXT; json_entries jsonb := '[]'; type_part TEXT; sparse_type TEXT; col_part TEXT; BEGIN -- Step 1: Loop over relevant column names FOR col_name IN SELECT column_name FROM information_schema.columns WHERE table_name = target_table AND table_schema = target_schema AND column_name LIKE '_ts_meta_v2_%' LOOP -- Step 2: Parse column name: _ts_meta_v2_<type>_<col> col_name := regexp_replace(col_name, '^_ts_meta_v2_', ''); type_part := split_part(col_name, '_', 1); col_part := substring(col_name FROM length(type_part) + 2); -- skip type and underscore IF type_part = 'bloom1' THEN sparse_type := 'bloom'; ELSIF type_part = 'min' THEN sparse_type := 'minmax'; ELSE CONTINUE; END IF; json_entries := json_entries || jsonb_build_object('type', sparse_type, 'source', 'default', 'column', col_part); END LOOP; RETURN json_entries; END; $$ LANGUAGE PLPGSQL SET search_path TO pg_catalog, pg_temp; DO $$ DECLARE rec RECORD; schema_name TEXT; table_name TEXT; json_entries JSONB; rows_updated INT; BEGIN FOR rec IN SELECT relid, compress_relid FROM _timescaledb_catalog.compression_settings WHERE index IS NULL LOOP IF rec.compress_relid IS NULL THEN CONTINUE; ELSE -- Get schema and table name from compress_relid SELECT n.nspname, c.relname INTO schema_name, table_name FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid WHERE c.oid = rec.compress_relid; END IF; -- Call procedure for that table json_entries := process_single_table_index_migrate(schema_name, table_name); IF jsonb_array_length(json_entries) > 0 THEN UPDATE _timescaledb_catalog.compression_settings SET index = json_entries WHERE relid = rec.relid AND index IS NULL; END IF; END LOOP; END $$; DROP FUNCTION process_single_table_index_migrate(text, text); -- add orderby minmax columns to sparse index settings UPDATE _timescaledb_catalog.compression_settings cs SET index = COALESCE(index, '[]'::jsonb) || ( SELECT jsonb_agg(jsonb_build_object( 'type', 'minmax', 'source', 'orderby', 'column', elem)) FROM unnest(cs.orderby) AS elem ) WHERE cs.orderby IS NOT NULL AND cardinality(cs.orderby) > 0 AND compress_relid IS NOT NULL; DROP FUNCTION IF EXISTS _timescaledb_internal.indexes_local_size; DROP FUNCTION IF EXISTS _timescaledb_functions.indexes_local_size; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.chunk_index; DROP TABLE IF EXISTS _timescaledb_catalog.chunk_index; -- cagg materialization ranges CREATE TABLE _timescaledb_catalog.continuous_aggs_materialization_ranges ( materialization_id integer, lowest_modified_value bigint NOT NULL, greatest_modified_value bigint NOT NULL, -- table constraints CONSTRAINT continuous_aggs_materialization_ranges_materialization_id_fkey FOREIGN KEY (materialization_id) REFERENCES _timescaledb_catalog.continuous_agg (mat_hypertable_id) ON DELETE CASCADE ); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_aggs_materialization_ranges', ''); CREATE INDEX continuous_aggs_materialization_ranges_idx ON _timescaledb_catalog.continuous_aggs_materialization_ranges (materialization_id, lowest_modified_value ASC); GRANT SELECT ON TABLE _timescaledb_catalog.continuous_aggs_materialization_ranges TO PUBLIC; ================================================ FILE: sql/updates/2.22.0--2.21.3.sql ================================================ ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.continuous_aggregates; DROP VIEW timescaledb_information.continuous_aggregates; DROP PROCEDURE _timescaledb_functions.process_hypertable_invalidations(REGCLASS[]); DROP PROCEDURE _timescaledb_functions.process_hypertable_invalidations(NAME); DROP FUNCTION _timescaledb_functions.cagg_parse_invalidation_record(BYTEA); DROP FUNCTION _timescaledb_functions.has_invalidation_trigger(regclass); DROP FUNCTION _timescaledb_functions.invalidation_plugin_name(); CREATE FUNCTION ts_hypercore_handler(internal) RETURNS table_am_handler AS '@MODULE_PATHNAME@', 'ts_hypercore_handler' LANGUAGE C; CREATE FUNCTION ts_hypercore_proxy_handler(internal) RETURNS index_am_handler AS '@MODULE_PATHNAME@', 'ts_hypercore_proxy_handler' LANGUAGE C; CREATE ACCESS METHOD hypercore TYPE TABLE HANDLER ts_hypercore_handler; COMMENT ON ACCESS METHOD hypercore IS 'Storage engine using hybrid row/columnar compression'; CREATE ACCESS METHOD hypercore_proxy TYPE INDEX HANDLER ts_hypercore_proxy_handler; COMMENT ON ACCESS METHOD hypercore_proxy IS 'Hypercore proxy index access method'; CREATE OPERATOR CLASS int4_ops DEFAULT FOR TYPE int4 USING hypercore_proxy AS OPERATOR 1 = (int4, int4), FUNCTION 1 hashint4(int4); CREATE FUNCTION _timescaledb_debug.is_compressed_tid(tid) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C STRICT; DROP PROCEDURE IF EXISTS _timescaledb_functions.policy_compression_execute; DROP FUNCTION IF EXISTS @extschema@.add_compression_policy; DROP PROCEDURE IF EXISTS @extschema@.add_columnstore_policy; DROP FUNCTION IF EXISTS timescaledb_experimental.add_policies; DROP FUNCTION IF EXISTS @extschema@.compress_chunk; DROP PROCEDURE IF EXISTS @extschema@.convert_to_columnstore; DO $$ DECLARE remove_these text[]; BEGIN SELECT array_agg(format('%I.%I', user_view_schema, user_view_name)) INTO remove_these FROM _timescaledb_catalog.continuous_agg JOIN _timescaledb_catalog.hypertable ht ON raw_hypertable_id = ht.id WHERE 'ts_cagg_invalidation_trigger' NOT IN ( SELECT tgname FROM pg_trigger WHERE tgrelid = format('%I.%I', ht.schema_name, ht.table_name)::regclass ); IF array_length(remove_these, 1) > 0 THEN RAISE EXCEPTION 'cannot downgrade because there are continuous aggregates using WAL-based invalidation collection' USING ERRCODE = 'object_not_in_prerequisite_state', DETAIL = format('Please remove these CAggs before downgrade: %s.', array_to_string(remove_these, ',')); END IF; END $$; CREATE FUNCTION @extschema@.compress_chunk( uncompressed_chunk REGCLASS, if_not_compressed BOOLEAN = true, recompress BOOLEAN = false, hypercore_use_access_method BOOL = NULL ) RETURNS REGCLASS AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; CREATE PROCEDURE @extschema@.convert_to_columnstore( chunk REGCLASS, if_not_columnstore BOOLEAN = true, recompress BOOLEAN = false, hypercore_use_access_method BOOL = NULL ) AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C; CREATE FUNCTION @extschema@.add_compression_policy( hypertable REGCLASS, compress_after "any" = NULL, if_not_exists BOOL = false, schedule_interval INTERVAL = NULL, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL, compress_created_before INTERVAL = NULL, hypercore_use_access_method BOOL = NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; CREATE PROCEDURE @extschema@.add_columnstore_policy( hypertable REGCLASS, after "any" = NULL, if_not_exists BOOL = false, schedule_interval INTERVAL = NULL, initial_start TIMESTAMPTZ = NULL, timezone TEXT = NULL, created_before INTERVAL = NULL, hypercore_use_access_method BOOL = NULL ) LANGUAGE C AS '@MODULE_PATHNAME@', 'ts_update_placeholder'; CREATE OR REPLACE FUNCTION timescaledb_experimental.add_policies( relation REGCLASS, if_not_exists BOOL = false, refresh_start_offset "any" = NULL, refresh_end_offset "any" = NULL, compress_after "any" = NULL, drop_after "any" = NULL, hypercore_use_access_method BOOL = NULL) RETURNS BOOL AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C VOLATILE; CREATE PROCEDURE _timescaledb_functions.policy_compression_execute( job_id INTEGER, htid INTEGER, lag ANYELEMENT, maxchunks INTEGER, verbose_log BOOLEAN, recompress_enabled BOOLEAN, reindex_enabled BOOLEAN, use_creation_time BOOLEAN, useam BOOLEAN = NULL) AS $$ BEGIN END $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS _timescaledb_functions.generate_uuid_v7; DROP FUNCTION IF EXISTS _timescaledb_functions.uuid_v7_from_timestamptz; DROP FUNCTION IF EXISTS _timescaledb_functions.uuid_v7_from_timestamptz_zeroed; DROP FUNCTION IF EXISTS _timescaledb_functions.timestamptz_from_uuid_v7; DROP FUNCTION IF EXISTS _timescaledb_functions.timestamptz_from_uuid_v7_with_microseconds; DROP FUNCTION IF EXISTS _timescaledb_functions.uuid_version; DELETE FROM _timescaledb_catalog.compression_algorithm WHERE id = 7 AND version = 1 AND name = 'COMPRESSION_ALGORITHM_UUID'; -- downgrade compression settings CREATE TABLE _timescaledb_catalog.tempsettings (LIKE _timescaledb_catalog.compression_settings); INSERT INTO _timescaledb_catalog.tempsettings SELECT * FROM _timescaledb_catalog.compression_settings; DROP VIEW timescaledb_information.hypertable_columnstore_settings; DROP VIEW timescaledb_information.chunk_columnstore_settings; DROP VIEW timescaledb_information.hypertable_compression_settings; DROP VIEW timescaledb_information.chunk_compression_settings; DROP VIEW timescaledb_information.compression_settings; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.compression_settings; DROP TABLE _timescaledb_catalog.compression_settings; CREATE TABLE _timescaledb_catalog.compression_settings ( relid regclass NOT NULL, compress_relid regclass NULL, segmentby text[], orderby text[], orderby_desc bool[], orderby_nullsfirst bool[], CONSTRAINT compression_settings_pkey PRIMARY KEY (relid), CONSTRAINT compression_settings_check_segmentby CHECK (array_ndims(segmentby) = 1), CONSTRAINT compression_settings_check_orderby_null CHECK ((orderby IS NULL AND orderby_desc IS NULL AND orderby_nullsfirst IS NULL) OR (orderby IS NOT NULL AND orderby_desc IS NOT NULL AND orderby_nullsfirst IS NOT NULL)), CONSTRAINT compression_settings_check_orderby_cardinality CHECK (array_ndims(orderby) = 1 AND array_ndims(orderby_desc) = 1 AND array_ndims(orderby_nullsfirst) = 1 AND cardinality(orderby) = cardinality(orderby_desc) AND cardinality(orderby) = cardinality(orderby_nullsfirst)) ); -- Revert information in compression settings INSERT INTO _timescaledb_catalog.compression_settings SELECT cs.relid, cs.compress_relid, cs.segmentby, cs.orderby, cs.orderby_desc, cs.orderby_nullsfirst FROM _timescaledb_catalog.tempsettings cs; DROP TABLE _timescaledb_catalog.tempsettings; CREATE INDEX compression_settings_compress_relid_idx ON _timescaledb_catalog.compression_settings (compress_relid); GRANT SELECT ON _timescaledb_catalog.compression_settings TO PUBLIC; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.compression_settings', ''); DROP FUNCTION IF EXISTS _timescaledb_functions.jsonb_get_matching_index_entry(jsonb, text, text); -- block downgrade if a table has NULL orderby setting (not allowed in 2.21) DO $$ BEGIN IF EXISTS ( SELECT 1 FROM _timescaledb_catalog.compression_settings WHERE orderby IS NULL ) THEN RAISE EXCEPTION 'TimescaleDB 2.21 can not have NULL columnstore orderby settings. Use ALTER TABLE to configure them before downgrading.'; END IF; END $$; -- remove empty segmentby UPDATE _timescaledb_catalog.compression_settings SET segmentby = NULL WHERE segmentby = '{}'; DROP FUNCTION IF EXISTS _timescaledb_functions.index_matches; CREATE TABLE _timescaledb_catalog.chunk_index ( chunk_id integer NOT NULL, index_name name NOT NULL, hypertable_id integer NOT NULL, hypertable_index_name name NOT NULL, -- table constraints CONSTRAINT chunk_index_chunk_id_index_name_key UNIQUE (chunk_id, index_name), CONSTRAINT chunk_index_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ON DELETE CASCADE, CONSTRAINT chunk_index_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE ); CREATE INDEX chunk_index_hypertable_id_hypertable_index_name_idx ON _timescaledb_catalog.chunk_index (hypertable_id, hypertable_index_name); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk_index', ''); CREATE OR REPLACE FUNCTION _timescaledb_functions.temp_index_keycolumns(oid) RETURNS text[] AS $$ SELECT array_agg(att.attname ORDER BY array_position(idx.indkey, att.attnum)) AS index_columns FROM pg_index AS idx JOIN pg_attribute AS att ON att.attrelid = idx.indrelid WHERE idx.indexrelid = $1 AND att.attnum = ANY(idx.indkey); $$ LANGUAGE SQL IMMUTABLE SET search_path TO pg_catalog, pg_temp; INSERT INTO _timescaledb_catalog.chunk_index (chunk_id, index_name, hypertable_id, hypertable_index_name) SELECT ch.id, ch_ci.relname, h.id, ht_ci.relname FROM _timescaledb_catalog.hypertable h JOIN pg_index ht_i ON ht_i.indrelid = format('%I.%I',h.schema_name,h.table_name)::regclass JOIN pg_class ht_ci ON ht_ci.oid=ht_i.indexrelid JOIN _timescaledb_catalog.chunk ch ON ch.hypertable_id=h.id AND NOT ch.dropped JOIN pg_index ch_i ON ch_i.indrelid=format('%I.%I',ch.schema_name,ch.table_name)::regclass AND ht_i.indnatts = ch_i.indnatts AND ht_i.indnkeyatts = ch_i.indnkeyatts AND ht_i.indisunique = ch_i.indisunique AND ht_i.indnullsnotdistinct = ch_i.indnullsnotdistinct AND ht_i.indisprimary = ch_i.indisprimary AND ht_i.indisexclusion = ch_i.indisexclusion AND ht_i.indimmediate = ch_i.indimmediate AND ht_i.indcollation=ch_i.indcollation AND ht_i.indclass=ch_i.indclass AND ht_i.indoption = ch_i.indoption AND ht_i.indexprs IS NOT DISTINCT FROM ch_i.indexprs AND ht_i.indpred IS NOT DISTINCT FROM ch_i.indpred AND _timescaledb_functions.temp_index_keycolumns(ht_i.indexrelid) = _timescaledb_functions.temp_index_keycolumns(ch_i.indexrelid) JOIN pg_class ch_ci ON ch_ci.oid=ch_i.indexrelid; DROP FUNCTION IF EXISTS _timescaledb_functions.temp_index_keycolumns(oid); GRANT SELECT ON TABLE _timescaledb_catalog.chunk_index TO PUBLIC; DROP FUNCTION IF EXISTS _timescaledb_functions.chunk_status_text(regclass); DROP FUNCTION IF EXISTS _timescaledb_functions.chunk_status_text(int); DO $$ DECLARE caggs_to_refresh TEXT; BEGIN IF EXISTS (SELECT FROM _timescaledb_catalog.continuous_aggs_materialization_ranges LIMIT 1) THEN SELECT string_agg(format('%I.%I', user_view_schema, user_view_name), ', ' ORDER BY user_view_schema, user_view_name) INTO caggs_to_refresh FROM _timescaledb_catalog.continuous_aggs_materialization_ranges JOIN _timescaledb_catalog.continuous_agg ON materialization_id = mat_hypertable_id; RAISE EXCEPTION 'cannot downgrade because there are pending CAgg refreshes' USING ERRCODE = 'object_not_in_prerequisite_state', DETAIL = format('Please refresh the CAggs before downgrade: %s.', caggs_to_refresh); END IF; END; $$; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_aggs_materialization_ranges; DROP TABLE IF EXISTS _timescaledb_catalog.continuous_aggs_materialization_ranges; DROP FUNCTION _timescaledb_functions.job_history_bsearch(TIMESTAMPTZ); DROP FUNCTION IF EXISTS @extschema@.generate_uuidv7(); DROP FUNCTION IF EXISTS @extschema@.to_uuidv7(timestamptz); DROP FUNCTION IF EXISTS @extschema@.to_uuidv7_boundary(timestamptz); DROP FUNCTION IF EXISTS @extschema@.uuid_timestamp(uuid); DROP FUNCTION IF EXISTS @extschema@.uuid_timestamp_micros(uuid); DROP FUNCTION IF EXISTS @extschema@.uuid_version(uuid); ================================================ FILE: sql/updates/2.22.0--2.22.1.sql ================================================ -- Fix wrong migration by removing all sparse index configurations -- which only contain auto sparse indexing definitions on hypertable DO $$ DECLARE rec RECORD; num_config INT; BEGIN IF NOT EXISTS ( SELECT column_name FROM information_schema.columns WHERE table_schema = '_timescaledb_catalog' AND table_name = 'compression_settings' AND column_name = 'index') THEN RETURN; END IF; FOR rec IN SELECT relid, compress_relid, index FROM _timescaledb_catalog.compression_settings WHERE compress_relid IS NULL LOOP num_config:=0; SELECT count(*) INTO num_config FROM jsonb_array_elements(rec.index) AS idx WHERE idx::jsonb @> '{"storage":"config"}'::jsonb; IF num_config = 0 THEN UPDATE _timescaledb_catalog.compression_settings SET index = NULL WHERE relid = rec.relid; END IF; END LOOP; END $$; ================================================ FILE: sql/updates/2.22.1--2.22.0.sql ================================================ ================================================ FILE: sql/updates/2.22.1--2.23.0.sql ================================================ DO $$ BEGIN UPDATE _timescaledb_config.bgw_job SET config = config || '{"max_successes_per_job": 1000, "max_failures_per_job": 1000}', schedule_interval = '6 hours' WHERE id = 3; -- system job retention RAISE WARNING 'job history configuration modified' USING DETAIL = 'The job history will only keep the last 1000 successes and failures and run once each day.'; END $$; DROP VIEW IF EXISTS timescaledb_information.job_stats; DROP VIEW IF EXISTS timescaledb_information.continuous_aggregates; -- remove cagg trigger from all hypertables and chunks DO $$ DECLARE rel regclass; BEGIN FOR rel IN SELECT format('%I.%I', schema_name, table_name)::regclass FROM _timescaledb_catalog.hypertable ht LOOP EXECUTE format('DROP TRIGGER IF EXISTS ts_cagg_invalidation_trigger ON %s;', rel); END LOOP; FOR rel IN SELECT format('%I.%I', schema_name, table_name)::regclass FROM _timescaledb_catalog.chunk ch LOOP EXECUTE format('DROP TRIGGER IF EXISTS ts_cagg_invalidation_trigger ON %s;', rel); END LOOP; END $$; DROP FUNCTION IF EXISTS _timescaledb_internal.continuous_agg_invalidation_trigger(); DROP FUNCTION IF EXISTS _timescaledb_functions.continuous_agg_invalidation_trigger(); DROP FUNCTION IF EXISTS _timescaledb_functions.has_invalidation_trigger(regclass); -- remove ts_insert_blocker trigger from all hypertables DO $$ DECLARE rel regclass; BEGIN FOR rel IN SELECT format('%I.%I', schema_name, table_name)::regclass FROM _timescaledb_catalog.hypertable ht LOOP EXECUTE format('DROP TRIGGER IF EXISTS ts_insert_blocker ON %s;', rel); END LOOP; END $$; DROP FUNCTION IF EXISTS _timescaledb_internal.insert_blocker(); DROP FUNCTION IF EXISTS _timescaledb_functions.insert_blocker(); ================================================ FILE: sql/updates/2.23.0--2.22.1.sql ================================================ DO $$ BEGIN UPDATE _timescaledb_config.bgw_job SET config = config - 'max_successes_per_job' - 'max_failures_per_job', schedule_interval = '1 month' WHERE id = 3; -- system job retention RAISE WARNING 'job history configuration modified' USING DETAIL = 'The job history will keep full history for each job and run once each month.'; END $$; DROP VIEW IF EXISTS timescaledb_information.job_stats; DROP VIEW IF EXISTS timescaledb_information.continuous_aggregates; CREATE OR REPLACE FUNCTION _timescaledb_functions.continuous_agg_invalidation_trigger() RETURNS TRIGGER AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C; -- add cagg trigger to hypertables with caggs and their chunks DO $$ DECLARE v_hypertable regclass; v_chunk regclass; v_hypertable_id integer; BEGIN FOR v_hypertable_id, v_hypertable IN SELECT ht.id, format('%I.%I', schema_name, table_name)::regclass FROM _timescaledb_catalog.hypertable ht WHERE compression_state <> 2 AND EXISTS (SELECT FROM _timescaledb_catalog.continuous_agg agg WHERE agg.raw_hypertable_id = ht.id) LOOP EXECUTE format('CREATE TRIGGER ts_cagg_invalidation_trigger AFTER INSERT OR UPDATE OR DELETE ON %s FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.continuous_agg_invalidation_trigger(''%s'');', v_hypertable, v_hypertable_id); FOR v_chunk IN SELECT format('%I.%I', schema_name, table_name)::regclass FROM _timescaledb_catalog.chunk ch WHERE ch.hypertable_id = v_hypertable_id LOOP EXECUTE format('CREATE TRIGGER ts_cagg_invalidation_trigger AFTER INSERT OR UPDATE OR DELETE ON %s FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.continuous_agg_invalidation_trigger(''%s'');', v_chunk, v_hypertable_id); END LOOP; END LOOP; END $$; CREATE OR REPLACE FUNCTION _timescaledb_functions.insert_blocker() RETURNS TRIGGER AS '@MODULE_PATHNAME@', 'ts_update_placeholder' LANGUAGE C; -- add ts_insert_blocker trigger to hypertables DO $$ DECLARE v_hypertable regclass; BEGIN FOR v_hypertable IN SELECT format('%I.%I', schema_name, table_name)::regclass FROM _timescaledb_catalog.hypertable ht LOOP EXECUTE format('CREATE TRIGGER ts_insert_blocker BEFORE INSERT ON %s FOR EACH ROW EXECUTE FUNCTION _timescaledb_functions.insert_blocker();', v_hypertable); END LOOP; END $$; ================================================ FILE: sql/updates/2.23.0--2.23.1.sql ================================================ ================================================ FILE: sql/updates/2.23.1--2.23.0.sql ================================================ ================================================ FILE: sql/updates/2.23.1--2.24.0.sql ================================================ DROP FUNCTION IF EXISTS _timescaledb_functions.policy_job_stat_history_retention; DROP VIEW IF EXISTS timescaledb_information.chunks; -- Add support for concurrent merge_chunks() CREATE TABLE _timescaledb_catalog.chunk_rewrite ( chunk_relid REGCLASS NOT NULL, new_relid REGCLASS NOT NULL, CONSTRAINT chunk_rewrite_key UNIQUE (chunk_relid) ); GRANT SELECT ON _timescaledb_catalog.chunk_rewrite TO PUBLIC; DROP PROCEDURE IF EXISTS @extschema@.merge_chunks(REGCLASS, REGCLASS); -- Check whether the database has the sparse bloom filter indexes on compressed -- chunks, which will require manual action to re-enable. DO $$ DECLARE num_chunks_with_bloom int; BEGIN SELECT count(*) INTO num_chunks_with_bloom FROM pg_attribute WHERE attname LIKE '_ts_meta_v2_bloom1_%'; IF num_chunks_with_bloom > 0 THEN RAISE WARNING 'bloom filter sparse indexes require action to re-enable' USING HINT = 'See the changelog for details.'; END IF; END $$; ================================================ FILE: sql/updates/2.24.0--2.23.1.sql ================================================ DROP FUNCTION _timescaledb_functions.bloom1_contains_any(_timescaledb_internal.bloom1, anyarray); DROP FUNCTION IF EXISTS _timescaledb_functions.policy_job_stat_history_retention; DROP VIEW IF EXISTS timescaledb_information.chunks; -- Revert support for concurrent merge chunks() DROP PROCEDURE IF EXISTS _timescaledb_functions.chunk_rewrite_cleanup(); DROP PROCEDURE IF EXISTS @extschema@.merge_chunks_concurrently(REGCLASS[]); DROP PROCEDURE IF EXISTS @extschema@.merge_chunks(REGCLASS, REGCLASS, BOOLEAN); DROP TABLE IF EXISTS _timescaledb_catalog.chunk_rewrite; -- Remove UUID time_bucket functions DROP FUNCTION IF EXISTS @extschema@.time_bucket(INTERVAL, UUID); DROP FUNCTION IF EXISTS @extschema@.time_bucket(INTERVAL, UUID, TIMESTAMPTZ); DROP FUNCTION IF EXISTS @extschema@.time_bucket(INTERVAL, UUID, INTERVAL); DROP FUNCTION IF EXISTS @extschema@.time_bucket(INTERVAL, UUID, TEXT, TIMESTAMPTZ, INTERVAL); ================================================ FILE: sql/updates/2.24.0--2.25.0.sql ================================================ DROP VIEW IF EXISTS timescaledb_information.dimensions; -- Block update if CAggs in old format are found DO $$ DECLARE caggs text; BEGIN SELECT string_agg(format('%I.%I', user_view_schema, user_view_name), ', ') INTO caggs FROM _timescaledb_catalog.continuous_agg WHERE finalized IS FALSE GROUP BY user_view_schema, user_view_name ORDER BY user_view_schema, user_view_name; IF caggs IS NOT NULL THEN RAISE EXCEPTION 'continuous aggregates with old format found, update blocked' USING DETAIL = format('Continuous Aggregates: %s', caggs), HINT = 'You should use `cagg_migrate` procedure to migrate to the new format.'; END IF; END $$; -- Block update if CAggs using time_bucket_ng are found DO $$ DECLARE caggs text; BEGIN SELECT string_agg(pg_catalog.format('%I.%I', user_view_schema, user_view_name), ', ') INTO caggs FROM _timescaledb_catalog.continuous_agg cagg JOIN _timescaledb_catalog.continuous_aggs_bucket_function AS bf ON (cagg.mat_hypertable_id = bf.mat_hypertable_id) WHERE bf.bucket_func::text LIKE '%time_bucket_ng%'; IF caggs IS NOT NULL THEN RAISE EXCEPTION 'continuous aggregates using time_bucket_ng found, update blocked' USING DETAIL = format('Continuous Aggregates: %s', caggs), HINT = 'time_bucket_ng has been removed. Please migrate the continuous aggregates using `cagg_migrate` before updating'; END IF; END $$; -- -- Rebuild the catalog table `_timescaledb_catalog.continuous_agg` to remove `finalized` column -- -- (1) Remove cagg migration functions and procedures from public and internal schemas DROP PROCEDURE IF EXISTS @extschema@.cagg_migrate (REGCLASS, BOOLEAN, BOOLEAN); DROP FUNCTION IF EXISTS _timescaledb_internal.cagg_migrate_pre_validation (TEXT, TEXT, TEXT); DROP FUNCTION IF EXISTS _timescaledb_functions.cagg_migrate_pre_validation (TEXT, TEXT, TEXT); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_create_plan (_timescaledb_catalog.continuous_agg, TEXT, BOOLEAN, BOOLEAN); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_create_plan (_timescaledb_catalog.continuous_agg, TEXT, BOOLEAN, BOOLEAN); DROP FUNCTION IF EXISTS _timescaledb_internal.cagg_migrate_plan_exists (INTEGER); DROP FUNCTION IF EXISTS _timescaledb_functions.cagg_migrate_plan_exists (INTEGER); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_plan (_timescaledb_catalog.continuous_agg); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_plan (_timescaledb_catalog.continuous_agg); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_create_new_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_create_new_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_disable_policies (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_disable_policies (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_enable_policies (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_enable_policies (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_copy_policies (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_copy_policies (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_refresh_new_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_refresh_new_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_update_watermark(integer); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_copy_data (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_copy_data (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_override_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_override_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_drop_old_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_execute_drop_old_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); -- (2) Rebuild catalog table DROP VIEW IF EXISTS timescaledb_experimental.policies; DROP VIEW IF EXISTS timescaledb_information.hypertables; DROP VIEW IF EXISTS timescaledb_information.continuous_aggregates; DROP VIEW IF EXISTS timescaledb_information.jobs; ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_ranges DROP CONSTRAINT continuous_aggs_materialization_ranges_materialization_id_fkey; ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_invalidation_log DROP CONSTRAINT continuous_aggs_materialization_invalid_materialization_id_fkey; ALTER TABLE _timescaledb_catalog.continuous_aggs_watermark DROP CONSTRAINT continuous_aggs_watermark_mat_hypertable_id_fkey; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_agg; CREATE TABLE _timescaledb_catalog._tmp_continuous_agg AS SELECT mat_hypertable_id, raw_hypertable_id, parent_mat_hypertable_id, user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name, materialized_only FROM _timescaledb_catalog.continuous_agg ORDER BY mat_hypertable_id; DROP TABLE _timescaledb_catalog.continuous_agg; CREATE TABLE _timescaledb_catalog.continuous_agg ( mat_hypertable_id integer NOT NULL, raw_hypertable_id integer NOT NULL, parent_mat_hypertable_id integer, user_view_schema name NOT NULL, user_view_name name NOT NULL, partial_view_schema name NOT NULL, partial_view_name name NOT NULL, direct_view_schema name NOT NULL, direct_view_name name NOT NULL, materialized_only bool NOT NULL DEFAULT FALSE, -- table constraints CONSTRAINT continuous_agg_pkey PRIMARY KEY (mat_hypertable_id), CONSTRAINT continuous_agg_partial_view_schema_partial_view_name_key UNIQUE (partial_view_schema, partial_view_name), CONSTRAINT continuous_agg_user_view_schema_user_view_name_key UNIQUE (user_view_schema, user_view_name), CONSTRAINT continuous_agg_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE, CONSTRAINT continuous_agg_raw_hypertable_id_fkey FOREIGN KEY (raw_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE, CONSTRAINT continuous_agg_parent_mat_hypertable_id_fkey FOREIGN KEY (parent_mat_hypertable_id) REFERENCES _timescaledb_catalog.continuous_agg (mat_hypertable_id) ON DELETE CASCADE ); INSERT INTO _timescaledb_catalog.continuous_agg SELECT * FROM _timescaledb_catalog._tmp_continuous_agg; DROP TABLE _timescaledb_catalog._tmp_continuous_agg; CREATE INDEX continuous_agg_raw_hypertable_id_idx ON _timescaledb_catalog.continuous_agg (raw_hypertable_id); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_agg', ''); GRANT SELECT ON TABLE _timescaledb_catalog.continuous_agg TO PUBLIC; -- clean up orphaned entries in related tables DELETE FROM _timescaledb_catalog.continuous_aggs_materialization_ranges range WHERE NOT EXISTS ( SELECT FROM _timescaledb_catalog.continuous_agg ca WHERE ca.mat_hypertable_id = range.materialization_id ); DELETE FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log inval WHERE NOT EXISTS ( SELECT FROM _timescaledb_catalog.continuous_agg ca WHERE ca.mat_hypertable_id = inval.materialization_id ); DELETE FROM _timescaledb_catalog.continuous_aggs_watermark wm WHERE NOT EXISTS ( SELECT FROM _timescaledb_catalog.continuous_agg ca WHERE ca.mat_hypertable_id = wm.mat_hypertable_id ); ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_ranges ADD CONSTRAINT continuous_aggs_materialization_ranges_materialization_id_fkey FOREIGN KEY (materialization_id) REFERENCES _timescaledb_catalog.continuous_agg(mat_hypertable_id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_invalidation_log ADD CONSTRAINT continuous_aggs_materialization_invalid_materialization_id_fkey FOREIGN KEY (materialization_id) REFERENCES _timescaledb_catalog.continuous_agg(mat_hypertable_id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.continuous_aggs_watermark ADD CONSTRAINT continuous_aggs_watermark_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.continuous_agg (mat_hypertable_id) ON DELETE CASCADE; ANALYZE _timescaledb_catalog.continuous_agg; -- -- END Rebuild the catalog table `_timescaledb_catalog.continuous_agg` -- DROP FUNCTION IF EXISTS _timescaledb_debug.extension_state(); DROP SCHEMA IF EXISTS _timescaledb_debug; ALTER TABLE _timescaledb_config.bgw_job SET SCHEMA _timescaledb_catalog; -- Remove legacy partialize/finalize aggregate functions. It should be -- conditional because on 2.12.0 we moved from internal to functions schema DO $$ DECLARE foid regprocedure; fkind text; fargs text; funcs text[] = '{finalize_agg, finalize_agg_sfunc, finalize_agg_ffunc, partialize_agg}'; BEGIN FOR foid, fkind, fargs IN SELECT p.oid, CASE WHEN p.prokind = 'f' THEN 'FUNCTION' WHEN p.prokind = 'a' THEN 'AGGREGATE' ELSE 'PROCEDURE' END, pg_catalog.pg_get_function_arguments(p.oid) FROM pg_catalog.pg_proc AS p WHERE p.proname = ANY(funcs) AND p.pronamespace IN ('_timescaledb_internal'::regnamespace, '_timescaledb_functions'::regnamespace) ORDER BY p.proname LOOP EXECUTE format('ALTER EXTENSION timescaledb DROP %s %s (%s);', fkind, foid::regproc, fargs); EXECUTE format('DROP %s %s (%s);', fkind, foid::regproc, fargs); END LOOP; END; $$ LANGUAGE plpgsql; DROP FUNCTION IF EXISTS _timescaledb_functions.cagg_parse_invalidation_record(bytea); DROP FUNCTION IF EXISTS _timescaledb_functions.get_hypertable_id(regclass, regtype); DROP FUNCTION IF EXISTS _timescaledb_functions.get_hypertable_invalidations(regclass,timestamp without time zone,interval[]); DROP FUNCTION IF EXISTS _timescaledb_functions.get_hypertable_invalidations(regclass,timestamp with time zone,interval[]); DROP FUNCTION IF EXISTS _timescaledb_functions.get_materialization_info(regclass); DROP FUNCTION IF EXISTS _timescaledb_functions.get_materialization_invalidations(regclass,tsrange); DROP FUNCTION IF EXISTS _timescaledb_functions.get_materialization_invalidations(regclass,tstzrange); DROP FUNCTION IF EXISTS _timescaledb_functions.get_raw_materialization_ranges(regtype); DROP FUNCTION IF EXISTS _timescaledb_functions.invalidation_plugin_name(); DROP PROCEDURE IF EXISTS _timescaledb_functions.accept_hypertable_invalidations(regclass,text); DROP PROCEDURE IF EXISTS _timescaledb_functions.add_materialization_invalidations(regclass,tsrange); DROP PROCEDURE IF EXISTS _timescaledb_functions.add_materialization_invalidations(regclass,tstzrange); DROP PROCEDURE IF EXISTS _timescaledb_functions.process_hypertable_invalidations(name); DROP PROCEDURE IF EXISTS _timescaledb_functions.process_hypertable_invalidations(regclass); DROP PROCEDURE IF EXISTS _timescaledb_functions.process_hypertable_invalidations(regclass[]); DROP PROCEDURE IF EXISTS _timescaledb_functions.cagg_migrate_to_time_bucket(regclass); DROP FUNCTION timescaledb_experimental.time_bucket_ng(bucket_width INTERVAL, ts DATE); DROP FUNCTION timescaledb_experimental.time_bucket_ng(bucket_width INTERVAL, ts DATE, origin DATE); DROP FUNCTION timescaledb_experimental.time_bucket_ng(bucket_width INTERVAL, ts TIMESTAMP); DROP FUNCTION timescaledb_experimental.time_bucket_ng(bucket_width INTERVAL, ts TIMESTAMP, origin TIMESTAMP); DROP FUNCTION timescaledb_experimental.time_bucket_ng(bucket_width INTERVAL, ts TIMESTAMPTZ, timezone TEXT); DROP FUNCTION timescaledb_experimental.time_bucket_ng(bucket_width INTERVAL, ts TIMESTAMPTZ, origin TIMESTAMPTZ, timezone TEXT); DROP FUNCTION timescaledb_experimental.time_bucket_ng(bucket_width INTERVAL, ts TIMESTAMPTZ); DROP FUNCTION timescaledb_experimental.time_bucket_ng(bucket_width INTERVAL, ts TIMESTAMPTZ, origin TIMESTAMPTZ); ================================================ FILE: sql/updates/2.25.0--2.24.0.sql ================================================ DROP VIEW IF EXISTS timescaledb_information.dimensions; -- -- Rebuild the catalog table `_timescaledb_catalog.continuous_agg` to add `finalized` column -- DROP VIEW IF EXISTS timescaledb_experimental.policies; DROP VIEW IF EXISTS timescaledb_information.hypertables; DROP VIEW IF EXISTS timescaledb_information.continuous_aggregates; DROP VIEW IF EXISTS timescaledb_information.jobs; ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_ranges DROP CONSTRAINT continuous_aggs_materialization_ranges_materialization_id_fkey; ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_invalidation_log DROP CONSTRAINT continuous_aggs_materialization_invalid_materialization_id_fkey; ALTER TABLE _timescaledb_catalog.continuous_aggs_watermark DROP CONSTRAINT continuous_aggs_watermark_mat_hypertable_id_fkey; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_agg; CREATE TABLE _timescaledb_catalog._tmp_continuous_agg AS SELECT mat_hypertable_id, raw_hypertable_id, parent_mat_hypertable_id, user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name, materialized_only, true AS finalized FROM _timescaledb_catalog.continuous_agg ORDER BY mat_hypertable_id; DROP TABLE _timescaledb_catalog.continuous_agg; CREATE TABLE _timescaledb_catalog.continuous_agg ( mat_hypertable_id integer NOT NULL, raw_hypertable_id integer NOT NULL, parent_mat_hypertable_id integer, user_view_schema name NOT NULL, user_view_name name NOT NULL, partial_view_schema name NOT NULL, partial_view_name name NOT NULL, direct_view_schema name NOT NULL, direct_view_name name NOT NULL, materialized_only bool NOT NULL DEFAULT FALSE, finalized bool NOT NULL DEFAULT TRUE, -- table constraints CONSTRAINT continuous_agg_pkey PRIMARY KEY (mat_hypertable_id), CONSTRAINT continuous_agg_partial_view_schema_partial_view_name_key UNIQUE (partial_view_schema, partial_view_name), CONSTRAINT continuous_agg_user_view_schema_user_view_name_key UNIQUE (user_view_schema, user_view_name), CONSTRAINT continuous_agg_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE, CONSTRAINT continuous_agg_raw_hypertable_id_fkey FOREIGN KEY (raw_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE, CONSTRAINT continuous_agg_parent_mat_hypertable_id_fkey FOREIGN KEY (parent_mat_hypertable_id) REFERENCES _timescaledb_catalog.continuous_agg (mat_hypertable_id) ON DELETE CASCADE ); INSERT INTO _timescaledb_catalog.continuous_agg SELECT * FROM _timescaledb_catalog._tmp_continuous_agg; DROP TABLE _timescaledb_catalog._tmp_continuous_agg; CREATE INDEX continuous_agg_raw_hypertable_id_idx ON _timescaledb_catalog.continuous_agg (raw_hypertable_id); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_agg', ''); GRANT SELECT ON TABLE _timescaledb_catalog.continuous_agg TO PUBLIC; ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_ranges ADD CONSTRAINT continuous_aggs_materialization_ranges_materialization_id_fkey FOREIGN KEY (materialization_id) REFERENCES _timescaledb_catalog.continuous_agg(mat_hypertable_id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_invalidation_log ADD CONSTRAINT continuous_aggs_materialization_invalid_materialization_id_fkey FOREIGN KEY (materialization_id) REFERENCES _timescaledb_catalog.continuous_agg(mat_hypertable_id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.continuous_aggs_watermark ADD CONSTRAINT continuous_aggs_watermark_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.continuous_agg (mat_hypertable_id) ON DELETE CASCADE; ANALYZE _timescaledb_catalog.continuous_agg; -- -- END Rebuild the catalog table `_timescaledb_catalog.continuous_agg` -- DROP FUNCTION IF EXISTS _timescaledb_functions.extension_state(); CREATE SCHEMA _timescaledb_debug; GRANT USAGE ON SCHEMA _timescaledb_debug TO PUBLIC; DROP VIEW IF EXISTS _timescaledb_config.bgw_job; ALTER TABLE _timescaledb_catalog.bgw_job SET SCHEMA _timescaledb_config; DROP FUNCTION IF EXISTS _timescaledb_functions.estimate_compressed_batch_size(REGCLASS); DROP PROCEDURE IF EXISTS _timescaledb_functions.rebuild_columnstore(REGCLASS); DROP FUNCTION IF EXISTS _timescaledb_functions.cagg_get_grouping_columns; DROP FUNCTION IF EXISTS _timescaledb_functions.compressed_data_to_array(_timescaledb_internal.compressed_data, ANYELEMENT); DROP FUNCTION IF EXISTS _timescaledb_functions.compressed_data_column_size(_timescaledb_internal.compressed_data, ANYELEMENT); DROP FUNCTION IF EXISTS _timescaledb_functions.estimate_uncompressed_size; ================================================ FILE: sql/updates/2.25.0--2.25.1.sql ================================================ ================================================ FILE: sql/updates/2.25.1--2.25.0.sql ================================================ ================================================ FILE: sql/updates/2.25.1--2.25.2.sql ================================================ ================================================ FILE: sql/updates/2.25.2--2.25.1.sql ================================================ ================================================ FILE: sql/updates/2.9.0--2.8.1.sql ================================================ -- gapfill with timezone support DROP FUNCTION @extschema@.time_bucket_gapfill(INTERVAL,TIMESTAMPTZ,TEXT,TIMESTAMPTZ,TIMESTAMPTZ); ALTER TABLE _timescaledb_catalog.compression_chunk_size DROP CONSTRAINT compression_chunk_size_pkey; ALTER TABLE _timescaledb_catalog.compression_chunk_size ADD CONSTRAINT compression_chunk_size_pkey PRIMARY KEY(chunk_id,compressed_chunk_id); DROP FUNCTION _timescaledb_internal.policy_job_error_retention(integer, JSONB); DROP FUNCTION _timescaledb_internal.policy_job_error_retention_check(JSONB); DELETE FROM _timescaledb_config.bgw_job WHERE id = 2; ALTER EXTENSION timescaledb DROP VIEW timescaledb_information.job_errors; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_internal.job_errors; DROP VIEW timescaledb_information.job_errors; DROP TABLE _timescaledb_internal.job_errors; -- drop dependent views DROP VIEW IF EXISTS timescaledb_information.job_stats; DROP VIEW IF EXISTS timescaledb_information.jobs; CREATE TABLE _timescaledb_internal._tmp_bgw_job_stat AS SELECT * FROM _timescaledb_internal.bgw_job_stat; DROP TABLE _timescaledb_internal.bgw_job_stat; CREATE TABLE _timescaledb_internal.bgw_job_stat ( job_id integer NOT NULL, last_start timestamptz NOT NULL DEFAULT NOW(), last_finish timestamptz NOT NULL, next_start timestamptz NOT NULL, last_successful_finish timestamptz NOT NULL, last_run_success bool NOT NULL, total_runs bigint NOT NULL, total_duration interval NOT NULL, total_successes bigint NOT NULL, total_failures bigint NOT NULL, total_crashes bigint NOT NULL, consecutive_failures int NOT NULL, consecutive_crashes int NOT NULL, -- table constraints CONSTRAINT bgw_job_stat_pkey PRIMARY KEY (job_id), CONSTRAINT bgw_job_stat_job_id_fkey FOREIGN KEY (job_id) REFERENCES _timescaledb_config.bgw_job (id) ON DELETE CASCADE ); INSERT INTO _timescaledb_internal.bgw_job_stat SELECT job_id, last_start, last_finish, next_start, last_successful_finish, last_run_success, total_runs, total_duration, total_successes, total_failures, total_crashes, consecutive_failures, consecutive_crashes FROM _timescaledb_internal._tmp_bgw_job_stat; DROP TABLE _timescaledb_internal._tmp_bgw_job_stat; GRANT SELECT ON TABLE _timescaledb_internal.bgw_job_stat TO PUBLIC; DROP VIEW _timescaledb_internal.hypertable_chunk_local_size; DROP FUNCTION _timescaledb_internal.hypertable_local_size(name, name); CREATE FUNCTION _timescaledb_internal.hypertable_local_size( schema_name_in name, table_name_in name) RETURNS TABLE ( table_bytes BIGINT, index_bytes BIGINT, toast_bytes BIGINT, total_bytes BIGINT) LANGUAGE SQL VOLATILE STRICT AS $BODY$ /* get the main hypertable id and sizes */ WITH _hypertable AS ( SELECT id, _timescaledb_internal.relation_size(format('%I.%I', schema_name, table_name)::regclass) AS relsize FROM _timescaledb_catalog.hypertable WHERE schema_name = schema_name_in AND table_name = table_name_in ), /* project the size of the parent hypertable */ _hypertable_sizes AS ( SELECT id, COALESCE((relsize).total_size, 0) AS total_bytes, COALESCE((relsize).heap_size, 0) AS heap_bytes, COALESCE((relsize).index_size, 0) AS index_bytes, COALESCE((relsize).toast_size, 0) AS toast_bytes, 0::BIGINT AS compressed_total_size, 0::BIGINT AS compressed_index_size, 0::BIGINT AS compressed_toast_size, 0::BIGINT AS compressed_heap_size FROM _hypertable ), /* calculate the size of the hypertable chunks */ _chunk_sizes AS ( SELECT chunk_id, COALESCE(ch.total_bytes, 0) AS total_bytes, COALESCE(ch.heap_bytes, 0) AS heap_bytes, COALESCE(ch.index_bytes, 0) AS index_bytes, COALESCE(ch.toast_bytes, 0) AS toast_bytes, COALESCE(ch.compressed_total_size, 0) AS compressed_total_size, COALESCE(ch.compressed_index_size, 0) AS compressed_index_size, COALESCE(ch.compressed_toast_size, 0) AS compressed_toast_size, COALESCE(ch.compressed_heap_size, 0) AS compressed_heap_size FROM _timescaledb_internal.hypertable_chunk_local_size ch JOIN _hypertable_sizes ht ON ht.id = ch.hypertable_id ) /* calculate the SUM of the hypertable and chunk sizes */ SELECT (SUM(heap_bytes) + SUM(compressed_heap_size))::BIGINT AS heap_bytes, (SUM(index_bytes) + SUM(compressed_index_size))::BIGINT AS index_bytes, (SUM(toast_bytes) + SUM(compressed_toast_size))::BIGINT AS toast_bytes, (SUM(total_bytes) + SUM(compressed_total_size))::BIGINT AS total_bytes FROM (SELECT * FROM _hypertable_sizes UNION ALL SELECT * FROM _chunk_sizes) AS sizes; $BODY$ SET search_path TO pg_catalog, pg_temp; DROP VIEW IF EXISTS timescaledb_information.job_stats; DROP VIEW IF EXISTS timescaledb_information.jobs; DROP VIEW IF EXISTS timescaledb_experimental.policies; -- fixed schedule DROP FUNCTION IF EXISTS @extschema@.add_retention_policy(REGCLASS, "any", BOOL, INTERVAL, TIMESTAMPTZ, BOOL); DROP FUNCTION IF EXISTS @extschema@.add_compression_policy(REGCLASS, "any", BOOL, INTERVAL); -- fixed schedule changes -- drop and recreate functions with modified signatures, modified views, modified tables DROP FUNCTION IF EXISTS @extschema@.add_job(REGPROC, INTERVAL, JSONB, TIMESTAMPTZ, BOOL, REGPROC, BOOL, TEXT); DROP FUNCTION IF EXISTS @extschema@.add_continuous_aggregate_policy(REGCLASS, "any", "any", INTERVAL, BOOL, TIMESTAMPTZ, TEXT); DROP FUNCTION IF EXISTS @extschema@.add_compression_policy(REGCLASS, "any", BOOL, INTERVAL, TIMESTAMPTZ, TEXT); DROP FUNCTION IF EXISTS @extschema@.add_retention_policy(REGCLASS, "any", BOOL, INTERVAL, TIMESTAMPTZ, TEXT); DROP FUNCTION IF EXISTS @extschema@.add_reorder_policy(REGCLASS, NAME, BOOL, TIMESTAMPTZ, TEXT); -- recreate functions with the previous signature CREATE FUNCTION @extschema@.add_job( proc REGPROC, schedule_interval INTERVAL, config JSONB DEFAULT NULL, initial_start TIMESTAMPTZ DEFAULT NULL, scheduled BOOL DEFAULT true, check_config REGPROC DEFAULT NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_job_add' LANGUAGE C VOLATILE; CREATE FUNCTION @extschema@.add_compression_policy(hypertable REGCLASS, compress_after "any", if_not_exists BOOL = false, schedule_interval INTERVAL = NULL) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_policy_compression_add' LANGUAGE C VOLATILE STRICT; CREATE FUNCTION @extschema@.add_retention_policy( relation REGCLASS, drop_after "any", if_not_exists BOOL = false, schedule_interval INTERVAL = NULL ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_policy_retention_add' LANGUAGE C VOLATILE STRICT; CREATE FUNCTION @extschema@.add_continuous_aggregate_policy(continuous_aggregate REGCLASS, start_offset "any", end_offset "any", schedule_interval INTERVAL, if_not_exists BOOL = false) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_policy_refresh_cagg_add' LANGUAGE C VOLATILE; CREATE FUNCTION @extschema@.add_reorder_policy( hypertable REGCLASS, index_name NAME, if_not_exists BOOL = false ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_policy_reorder_add' LANGUAGE C VOLATILE STRICT; DROP VIEW IF EXISTS timescaledb_information.jobs; DROP VIEW IF EXISTS timescaledb_information.job_stats; -- now need to rebuild the table ALTER TABLE _timescaledb_internal.bgw_job_stat DROP CONSTRAINT bgw_job_stat_job_id_fkey; ALTER TABLE _timescaledb_internal.bgw_policy_chunk_stats DROP CONSTRAINT bgw_policy_chunk_stats_chunk_id_fkey, DROP CONSTRAINT bgw_policy_chunk_stats_job_id_fkey; CREATE TABLE _timescaledb_config.bgw_job_tmp AS SELECT * FROM _timescaledb_config.bgw_job; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_config.bgw_job; ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_config.bgw_job_id_seq; CREATE TABLE _timescaledb_internal.tmp_bgw_job_seq_value AS SELECT last_value, is_called FROM _timescaledb_config.bgw_job_id_seq; DROP TABLE _timescaledb_config.bgw_job; CREATE SEQUENCE _timescaledb_config.bgw_job_id_seq MINVALUE 1000; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_config.bgw_job_id_seq', ''); SELECT setval('_timescaledb_config.bgw_job_id_seq', last_value, is_called) FROM _timescaledb_internal.tmp_bgw_job_seq_value; DROP TABLE _timescaledb_internal.tmp_bgw_job_seq_value; CREATE TABLE _timescaledb_config.bgw_job ( id integer PRIMARY KEY DEFAULT nextval('_timescaledb_config.bgw_job_id_seq'), application_name name NOT NULL, schedule_interval interval NOT NULL, max_runtime interval NOT NULL, max_retries integer NOT NULL, retry_period interval NOT NULL, proc_schema name NOT NULL, proc_name name NOT NULL, owner name NOT NULL DEFAULT CURRENT_ROLE, scheduled bool NOT NULL DEFAULT TRUE, hypertable_id integer REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE, config jsonb, check_schema NAME, check_name NAME ); INSERT INTO _timescaledb_config.bgw_job(id, application_name, schedule_interval, max_runtime, max_retries, retry_period, proc_schema, proc_name, owner, scheduled, hypertable_id, config) SELECT id, application_name, schedule_interval, max_runtime, max_retries, retry_period, proc_schema, proc_name, owner, scheduled, hypertable_id, config FROM _timescaledb_config.bgw_job_tmp ORDER BY id; ALTER SEQUENCE _timescaledb_config.bgw_job_id_seq OWNED BY _timescaledb_config.bgw_job.id; CREATE INDEX bgw_job_proc_hypertable_id_idx ON _timescaledb_config.bgw_job(proc_schema,proc_name,hypertable_id); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_config.bgw_job', 'WHERE id >= 1000'); GRANT SELECT ON _timescaledb_config.bgw_job TO PUBLIC; GRANT SELECT ON _timescaledb_config.bgw_job_id_seq TO PUBLIC; DROP TABLE _timescaledb_config.bgw_job_tmp; ALTER TABLE _timescaledb_internal.bgw_job_stat ADD CONSTRAINT bgw_job_stat_job_id_fkey FOREIGN KEY(job_id) REFERENCES _timescaledb_config.bgw_job(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_internal.bgw_policy_chunk_stats ADD CONSTRAINT bgw_policy_chunk_stats_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE, ADD CONSTRAINT bgw_policy_chunk_stats_job_id_fkey FOREIGN KEY(job_id) REFERENCES _timescaledb_config.bgw_job(id) ON DELETE CASCADE; DROP FUNCTION _timescaledb_internal.health; -- Recreate _timescaledb_catalog.dimension table without the compress_interval_length column -- CREATE TABLE _timescaledb_internal.dimension_tmp AS SELECT * from _timescaledb_catalog.dimension; CREATE TABLE _timescaledb_internal.tmp_dimension_seq_value AS SELECT last_value, is_called FROM _timescaledb_catalog.dimension_id_seq; --drop foreign keys on dimension table ALTER TABLE _timescaledb_catalog.dimension_partition DROP CONSTRAINT dimension_partition_dimension_id_fkey; ALTER TABLE _timescaledb_catalog.dimension_slice DROP CONSTRAINT dimension_slice_dimension_id_fkey; --drop dependent views DROP VIEW IF EXISTS timescaledb_information.chunks; DROP VIEW IF EXISTS timescaledb_information.dimensions; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.dimension; ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_catalog.dimension_id_seq; DROP TABLE _timescaledb_catalog.dimension; CREATE TABLE _timescaledb_catalog.dimension ( id serial NOT NULL , hypertable_id integer NOT NULL, column_name name NOT NULL, column_type REGTYPE NOT NULL, aligned boolean NOT NULL, -- closed dimensions num_slices smallint NULL, partitioning_func_schema name NULL, partitioning_func name NULL, -- open dimensions (e.g., time) interval_length bigint NULL, integer_now_func_schema name NULL, integer_now_func name NULL, -- table constraints CONSTRAINT dimension_pkey PRIMARY KEY (id), CONSTRAINT dimension_hypertable_id_column_name_key UNIQUE (hypertable_id, column_name), CONSTRAINT dimension_check CHECK ((partitioning_func_schema IS NULL AND partitioning_func IS NULL) OR (partitioning_func_schema IS NOT NULL AND partitioning_func IS NOT NULL)), CONSTRAINT dimension_check1 CHECK ((num_slices IS NULL AND interval_length IS NOT NULL) OR (num_slices IS NOT NULL AND interval_length IS NULL)), CONSTRAINT dimension_check2 CHECK ((integer_now_func_schema IS NULL AND integer_now_func IS NULL) OR (integer_now_func_schema IS NOT NULL AND integer_now_func IS NOT NULL)), CONSTRAINT dimension_interval_length_check CHECK (interval_length IS NULL OR interval_length > 0), CONSTRAINT dimension_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE ); INSERT INTO _timescaledb_catalog.dimension ( id, hypertable_id, column_name, column_type, aligned, num_slices, partitioning_func_schema, partitioning_func, interval_length, integer_now_func_schema, integer_now_func) SELECT id, hypertable_id, column_name, column_type, aligned, num_slices, partitioning_func_schema, partitioning_func, interval_length, integer_now_func_schema, integer_now_func FROM _timescaledb_internal.dimension_tmp; ALTER SEQUENCE _timescaledb_catalog.dimension_id_seq OWNED BY _timescaledb_catalog.dimension.id; SELECT setval('_timescaledb_catalog.dimension_id_seq', last_value, is_called) FROM _timescaledb_internal.tmp_dimension_seq_value; SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.dimension', ''); SELECT pg_catalog.pg_extension_config_dump(pg_get_serial_sequence('_timescaledb_catalog.dimension', 'id'), ''); --add the foreign key constraints ALTER TABLE _timescaledb_catalog.dimension_partition ADD CONSTRAINT dimension_partition_dimension_id_fkey FOREIGN KEY (dimension_id) REFERENCES _timescaledb_catalog.dimension(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.dimension_slice ADD CONSTRAINT dimension_slice_dimension_id_fkey FOREIGN KEY (dimension_id) REFERENCES _timescaledb_catalog.dimension(id) ON DELETE CASCADE; --cleanup DROP TABLE _timescaledb_internal.dimension_tmp; DROP TABLE _timescaledb_internal.tmp_dimension_seq_value; GRANT SELECT ON _timescaledb_catalog.dimension_id_seq TO PUBLIC; GRANT SELECT ON _timescaledb_catalog.dimension TO PUBLIC; -- end recreate _timescaledb_catalog.dimension table -- -- changes related to alter_data_node() DROP INDEX _timescaledb_catalog.chunk_data_node_node_name_idx; DROP FUNCTION @extschema@.alter_data_node; -- -- Prevent downgrading if there are hierarchical continuous aggregates -- DO $$ DECLARE caggs_hierarchical TEXT; caggs_count INTEGER; BEGIN SELECT string_agg(format('%I.%I', user_view_schema, user_view_name), ', '), count(*) INTO caggs_hierarchical, caggs_count FROM _timescaledb_catalog.continuous_agg WHERE parent_mat_hypertable_id IS NOT NULL; IF caggs_count > 0 THEN RAISE EXCEPTION 'Downgrade is not possible because there are % hierarchical continuous aggregates: %', caggs_count, caggs_nested USING HINT = 'Remove the corresponding continuous aggregates manually before downgrading'; END IF; END; $$ LANGUAGE 'plpgsql'; -- -- Rebuild the catalog table `_timescaledb_catalog.continuous_agg` -- DROP VIEW IF EXISTS timescaledb_information.hypertables; DROP VIEW IF EXISTS timescaledb_information.continuous_aggregates; DROP PROCEDURE IF EXISTS @extschema@.cagg_migrate (REGCLASS, BOOLEAN, BOOLEAN); DROP FUNCTION IF EXISTS _timescaledb_internal.cagg_migrate_pre_validation (TEXT, TEXT, TEXT); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_create_plan (_timescaledb_catalog.continuous_agg, TEXT, BOOLEAN, BOOLEAN); DROP FUNCTION IF EXISTS _timescaledb_internal.cagg_migrate_plan_exists (INTEGER); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_plan (_timescaledb_catalog.continuous_agg); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_create_new_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_disable_policies (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_enable_policies (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_copy_policies (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_refresh_new_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_copy_data (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_override_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); DROP PROCEDURE IF EXISTS _timescaledb_internal.cagg_migrate_execute_drop_old_cagg (_timescaledb_catalog.continuous_agg, _timescaledb_catalog.continuous_agg_migrate_plan_step); ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_agg; ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_invalidation_log DROP CONSTRAINT continuous_aggs_materialization_invalid_materialization_id_fkey; ALTER TABLE _timescaledb_catalog.continuous_agg_migrate_plan DROP CONSTRAINT continuous_agg_migrate_plan_mat_hypertable_id_fkey; CREATE TABLE _timescaledb_catalog._tmp_continuous_agg AS SELECT mat_hypertable_id, raw_hypertable_id, user_view_schema, user_view_name, partial_view_schema, partial_view_name, bucket_width, direct_view_schema, direct_view_name, materialized_only, finalized FROM _timescaledb_catalog.continuous_agg ORDER BY mat_hypertable_id; DROP TABLE _timescaledb_catalog.continuous_agg; CREATE TABLE _timescaledb_catalog.continuous_agg ( mat_hypertable_id integer NOT NULL, raw_hypertable_id integer NOT NULL, user_view_schema name NOT NULL, user_view_name name NOT NULL, partial_view_schema name NOT NULL, partial_view_name name NOT NULL, bucket_width bigint NOT NULL, direct_view_schema name NOT NULL, direct_view_name name NOT NULL, materialized_only bool NOT NULL DEFAULT FALSE, finalized bool NOT NULL DEFAULT TRUE, -- table constraints CONSTRAINT continuous_agg_pkey PRIMARY KEY (mat_hypertable_id), CONSTRAINT continuous_agg_partial_view_schema_partial_view_name_key UNIQUE (partial_view_schema, partial_view_name), CONSTRAINT continuous_agg_user_view_schema_user_view_name_key UNIQUE (user_view_schema, user_view_name), CONSTRAINT continuous_agg_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE, CONSTRAINT continuous_agg_raw_hypertable_id_fkey FOREIGN KEY (raw_hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id) ON DELETE CASCADE ); INSERT INTO _timescaledb_catalog.continuous_agg SELECT * FROM _timescaledb_catalog._tmp_continuous_agg; DROP TABLE _timescaledb_catalog._tmp_continuous_agg; CREATE INDEX continuous_agg_raw_hypertable_id_idx ON _timescaledb_catalog.continuous_agg (raw_hypertable_id); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_agg', ''); GRANT SELECT ON TABLE _timescaledb_catalog.continuous_agg TO PUBLIC; ALTER TABLE _timescaledb_catalog.continuous_aggs_materialization_invalidation_log ADD CONSTRAINT continuous_aggs_materialization_invalid_materialization_id_fkey FOREIGN KEY (materialization_id) REFERENCES _timescaledb_catalog.continuous_agg(mat_hypertable_id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.continuous_agg_migrate_plan ADD CONSTRAINT continuous_agg_migrate_plan_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.continuous_agg (mat_hypertable_id); ANALYZE _timescaledb_catalog.continuous_agg; -- changes related to drop_stale_chunks() DROP FUNCTION _timescaledb_internal.drop_stale_chunks; ================================================ FILE: sql/updates/2.9.0--2.9.1.sql ================================================ GRANT SELECT ON _timescaledb_internal.job_errors to PUBLIC; ================================================ FILE: sql/updates/2.9.1--2.9.0.sql ================================================ ================================================ FILE: sql/updates/2.9.1--2.9.2.sql ================================================ ================================================ FILE: sql/updates/2.9.2--2.9.1.sql ================================================ ================================================ FILE: sql/updates/2.9.2--2.9.3.sql ================================================ ================================================ FILE: sql/updates/2.9.3--2.10.0.sql ================================================ CREATE OR REPLACE VIEW timescaledb_information.job_errors WITH (security_barrier = true) AS SELECT job_id, error_data ->> 'proc_schema' as proc_schema, error_data ->> 'proc_name' as proc_name, pid, start_time, finish_time, error_data ->> 'sqlerrcode' AS sqlerrcode, CASE WHEN error_data ->>'message' IS NOT NULL THEN CASE WHEN error_data ->>'detail' IS NOT NULL THEN CASE WHEN error_data ->>'hint' IS NOT NULL THEN concat(error_data ->>'message', '. ', error_data ->>'detail', '. ', error_data->>'hint') ELSE concat(error_data ->>'message', ' ', error_data ->>'detail') END ELSE CASE WHEN error_data ->>'hint' IS NOT NULL THEN concat(error_data ->>'message', '. ', error_data->>'hint') ELSE error_data ->>'message' END END ELSE 'job crash detected, see server logs' END AS err_message FROM _timescaledb_internal.job_errors LEFT JOIN _timescaledb_config.bgw_job ON (bgw_job.id = job_errors.job_id) WHERE pg_catalog.pg_has_role(current_user, (SELECT pg_catalog.pg_get_userbyid(datdba) FROM pg_catalog.pg_database WHERE datname = current_database()), 'MEMBER') IS TRUE OR pg_catalog.pg_has_role(current_user, owner, 'MEMBER') IS TRUE; REVOKE ALL ON _timescaledb_internal.job_errors FROM PUBLIC; ================================================ FILE: sql/updates/2.9.3--2.9.2.sql ================================================ ================================================ FILE: sql/updates/README.md ================================================ ## General principles for statements in update/downgrade scripts 1. The `search_path` for these scripts will be locked down to `pg_catalog, pg_temp`. Locking down `search_path` happens in `pre-update.sql`. Therefore all object references need to be fully qualified unless they reference objects from `pg_catalog`. Use `@extschema@` to refer to the target schema of the installation (resolves to `public` by default). 2. Creating objects must not use IF NOT EXISTS as this will introduce privilege escalation vulnerabilities. 3. All functions should have explicit `search_path`. Setting explicit `search_path` will prevent SQL function inlining for functions and transaction control for procedures so for some functions/procedures it is acceptable to not have explicit `search_path`. Special care needs to be taken with those functions/procedures by either setting `search_path` in function body or having only fully qualified object references including operators. 4. When generating the install scripts `CREATE OR REPLACE` will be changed to `CREATE` to prevent users from precreating extension objects. Since we need `CREATE OR REPLACE` for update scripts and we don't want to maintain two versions of the sql files containing the function definitions we use `CREATE OR REPLACE` in those. 5. Any object added in a new version needs to have an equivalent `CREATE` statement in the update script without `OR REPLACE` to prevent precreation of the object. 6. The creation of new metadata tables need to be part of modfiles, similar to `ALTER`s of such tables. Otherwise, later modfiles cannot rely on those tables being present. ## Extension updates This directory contains "modfiles" (SQL scripts) with modifications that are applied when updating from one version of the extension to another. The actual update scripts are compiled from modfiles by concatenating them with the current source code (which should come at the end of the resulting update script). Update scripts can "jump" several versions by using multiple modfiles in order. There are two types of modfiles: * Transition modfiles named `<from>-<to>.sql`, where `from` and `to` indicate the (adjacent) versions transitioning between. Transition modfiles are concatenated to form the lineage from an origin version to any later version. * Origin modfiles named <version>.sql, which are included only in update scripts that origin at the particular version given in the name. So, for instance, `0.7.0.sql` is only included in the script moving from `0.7.0` to the current version, but not in, e.g., the update script for `0.4.0` to the current version. These files typically contain fixes for bugs that are specific to the origin version, but are no longer present in the transition modfiles. Notes on post_update.sql We use a special config var (timescaledb.update_script_stage ) to notify that dependencies have been setup and now timescaledb specific queries can be enabled. This is useful if we want to, for example, modify objects that need timescaledb specific syntax as part of the extension update). The scripts in post_update.sql are executed as part of the `ALTER EXTENSION` stmt. Note that modfiles that contain no changes need not exist as a file. Transition modfiles must, however, be listed in the `CMakeLists.txt` file in the parent directory for an update script to be built for that version. ## Extension downgrades You can enable the generation of a downgrade file by setting `GENERATE_DOWNGRADE_SCRIPT` to `ON`, for example: ``` ./bootstrap -DGENERATE_DOWNGRADE_SCRIPT=ON ``` To support downgrades to previous versions of the extension, it is necessary to execute CMake from a Git repository since the generation of a downgrade script requires access to the previous version files that are used to generate an update script. In addition, we only generate a downgrade script to the immediate preceeding version and not to any other preceeding versions. The source and target versions are found in be found in the file `version.config` file in the root of the source tree, where `version` is the source version and `previous_version` is the target version. Note that we have a separate field for the downgrade. A downgrade file consists of: - A prolog that is retrieved from the target version. - A version-specific piece of code that exists on the source version. - An epilog that is retrieved from the target version. The prolog consists of the files mentioned in the `PRE_UPDATE_FILES` variable in the target version of `cmake/ScriptFiles.cmake`. The version-specific code is found in the source version of the file `sql/updates/reverse-dev.sql`. The epilog consists of the files in variables `SOURCE_FILES`, `SET_POST_UPDATE_STAGE`, `POST_UPDATE_FILES`, and `UNSET_UPDATE_STAGE` in that order. For downgrades to work correctly, some rules need to be followed: 1. If you add new objects in `sql/updates/latest-dev.sql`, you need to remove them in the version-specific downgrade file. The `sql/updates/pre-update.sql` in the target version do not know about objects created in the source version, so they need to be dropped explicitly. 2. Since `sql/updates/pre-update.sql` can be executed on a later version of the extension, it might be that some objects have been removed and do not exist. Hence `DROP` calls need to use `IF NOT EXISTS`. Note that, in contrast to update scripts, downgrade scripts are not built by composing several downgrade scripts into a more extensive downgrade script. The downgrade scripts are intended to be use only in special cases and are not intended to be use to move up and down between versions at will, which is why we only generate a downgrade script to the immediately preceeding version. ### When releasing a new version When releasing a new version, please rename the file `reverse-dev.sql` to `<version>--<previous_version>.sql` and add that name to `REV_FILES` variable in the `sql/CMakeLists.txt`. This will allow generation of downgrade scripts for any version in that list, but it is currently not added. ================================================ FILE: sql/updates/latest-dev.sql ================================================ -- -- Rebuild the catalog table `_timescaledb_catalog.chunk` to drop column `dropped` -- CREATE TABLE _timescaledb_internal.tmp_chunk AS SELECT * from _timescaledb_catalog.chunk WHERE NOT dropped; CREATE TABLE _timescaledb_internal.tmp_chunk_seq_value AS SELECT last_value, is_called FROM _timescaledb_catalog.chunk_id_seq; --drop foreign keys on chunk table ALTER TABLE _timescaledb_catalog.chunk_constraint DROP CONSTRAINT chunk_constraint_chunk_id_fkey; ALTER TABLE _timescaledb_catalog.chunk_column_stats DROP CONSTRAINT chunk_column_stats_chunk_id_fkey; ALTER TABLE _timescaledb_internal.bgw_policy_chunk_stats DROP CONSTRAINT bgw_policy_chunk_stats_chunk_id_fkey; ALTER TABLE _timescaledb_catalog.compression_chunk_size DROP CONSTRAINT compression_chunk_size_chunk_id_fkey; ALTER TABLE _timescaledb_catalog.compression_chunk_size DROP CONSTRAINT compression_chunk_size_compressed_chunk_id_fkey; --drop dependent views DROP VIEW IF EXISTS timescaledb_information.hypertables; DROP VIEW IF EXISTS timescaledb_information.chunks; DROP VIEW IF EXISTS _timescaledb_internal.hypertable_chunk_local_size; DROP VIEW IF EXISTS _timescaledb_internal.compressed_chunk_stats; DROP VIEW IF EXISTS timescaledb_information.chunk_columnstore_settings; DROP VIEW IF EXISTS timescaledb_information.chunk_compression_settings; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.chunk; ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_catalog.chunk_id_seq; DROP TABLE _timescaledb_catalog.chunk; CREATE SEQUENCE _timescaledb_catalog.chunk_id_seq MINVALUE 1; -- now create table without self referential foreign key CREATE TABLE _timescaledb_catalog.chunk ( id integer NOT NULL DEFAULT nextval('_timescaledb_catalog.chunk_id_seq'), hypertable_id int NOT NULL, schema_name name NOT NULL, table_name name NOT NULL, compressed_chunk_id integer , status integer NOT NULL DEFAULT 0, osm_chunk boolean NOT NULL DEFAULT FALSE, creation_time timestamptz NOT NULL, -- table constraints CONSTRAINT chunk_pkey PRIMARY KEY (id), CONSTRAINT chunk_schema_name_table_name_key UNIQUE (schema_name, table_name) ); INSERT INTO _timescaledb_catalog.chunk( id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk, creation_time) SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk, creation_time FROM _timescaledb_internal.tmp_chunk; --add indexes to the chunk table CREATE INDEX chunk_hypertable_id_idx ON _timescaledb_catalog.chunk (hypertable_id); CREATE INDEX chunk_compressed_chunk_id_idx ON _timescaledb_catalog.chunk (compressed_chunk_id); CREATE INDEX chunk_osm_chunk_idx ON _timescaledb_catalog.chunk (osm_chunk, hypertable_id); CREATE INDEX chunk_hypertable_id_creation_time_idx ON _timescaledb_catalog.chunk(hypertable_id, creation_time); ALTER SEQUENCE _timescaledb_catalog.chunk_id_seq OWNED BY _timescaledb_catalog.chunk.id; SELECT setval('_timescaledb_catalog.chunk_id_seq', last_value, is_called) FROM _timescaledb_internal.tmp_chunk_seq_value; -- add self referential foreign key ALTER TABLE _timescaledb_catalog.chunk ADD CONSTRAINT chunk_compressed_chunk_id_fkey FOREIGN KEY ( compressed_chunk_id ) REFERENCES _timescaledb_catalog.chunk( id ); --add foreign key constraint ALTER TABLE _timescaledb_catalog.chunk ADD CONSTRAINT chunk_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk', ''); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk_id_seq', ''); --add the foreign key constraints ALTER TABLE _timescaledb_catalog.chunk_constraint ADD CONSTRAINT chunk_constraint_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id); ALTER TABLE _timescaledb_catalog.chunk_column_stats ADD CONSTRAINT chunk_column_stats_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk (id); ALTER TABLE _timescaledb_internal.bgw_policy_chunk_stats ADD CONSTRAINT bgw_policy_chunk_stats_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.compression_chunk_size ADD CONSTRAINT compression_chunk_size_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.compression_chunk_size ADD CONSTRAINT compression_chunk_size_compressed_chunk_id_fkey FOREIGN KEY (compressed_chunk_id) REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE; --cleanup DROP TABLE _timescaledb_internal.tmp_chunk; DROP TABLE _timescaledb_internal.tmp_chunk_seq_value; GRANT SELECT ON _timescaledb_catalog.chunk_id_seq TO PUBLIC; GRANT SELECT ON _timescaledb_catalog.chunk TO PUBLIC; -- end rebuild _timescaledb_catalog.chunk table -- -- drop the catalog tables for continuous aggregate migration plans ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_agg_migrate_plan; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.continuous_agg_migrate_plan_step; ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_catalog.continuous_agg_migrate_plan_step_step_id_seq; DROP TABLE _timescaledb_catalog.continuous_agg_migrate_plan_step; DROP TABLE _timescaledb_catalog.continuous_agg_migrate_plan; -- -- Add this index to speed up queries for recent job history -- This statement is idempotent to allow the index to have been precreated. -- CREATE INDEX IF NOT EXISTS bgw_job_stat_history_execution_start_idx ON _timescaledb_internal.bgw_job_stat_history(execution_start); CREATE INDEX IF NOT EXISTS bgw_job_stat_history_job_id_execution_start_idx ON _timescaledb_internal.bgw_job_stat_history(job_id, execution_start DESC); DROP INDEX IF EXISTS _timescaledb_internal.bgw_job_stat_history_job_id_idx; ================================================ FILE: sql/updates/post-update.sql ================================================ -- For objects that are newly created, we need to set the initprivs to -- the initprivs for some table that was created in the installation -- of the TimescaleDB extension and not as part of any update. -- -- We chose the "chunk" catalog table for this since that is created -- in the first version of TimescaleDB and should have the correct -- initprivs, but we could use any other table that existed in the -- first installation. INSERT INTO _timescaledb_internal.saved_privs SELECT nspname, relname, relacl, (SELECT tmpini FROM _timescaledb_internal.saved_privs WHERE tmpnsp = '_timescaledb_catalog' AND tmpname = 'chunk') FROM pg_class JOIN pg_namespace ns ON ns.oid = relnamespace LEFT JOIN _timescaledb_internal.saved_privs ON tmpnsp = nspname AND tmpname = relname WHERE relkind IN ('r', 'v') AND nspname IN ('_timescaledb_catalog', '_timescaledb_config') OR nspname = '_timescaledb_internal' AND relname IN ('hypertable_chunk_local_size', 'compressed_chunk_stats', 'bgw_job_stat', 'bgw_policy_chunk_stats', 'job_errors') ON CONFLICT DO NOTHING; -- The above is good enough for tables and views. However sequences need to -- use the "chunk_id_seq" catalog sequence as a template INSERT INTO _timescaledb_internal.saved_privs SELECT nspname, relname, relacl, (SELECT tmpini FROM _timescaledb_internal.saved_privs WHERE tmpnsp = '_timescaledb_catalog' AND tmpname = 'chunk_id_seq') FROM pg_class JOIN pg_namespace ns ON ns.oid = relnamespace LEFT JOIN _timescaledb_internal.saved_privs ON tmpnsp = nspname AND tmpname = relname WHERE relkind IN ('S') AND nspname IN ('_timescaledb_catalog', '_timescaledb_config') OR nspname = '_timescaledb_internal' AND relname IN ('hypertable_chunk_local_size', 'compressed_chunk_stats', 'bgw_job_stat', 'bgw_policy_chunk_stats') ON CONFLICT DO NOTHING; -- We can now copy back saved initprivs. WITH to_update AS ( SELECT objoid, tmpini FROM pg_class cl JOIN pg_namespace ns ON ns.oid = relnamespace JOIN pg_init_privs ip ON ip.objoid = cl.oid AND ip.objsubid = 0 JOIN _timescaledb_internal.saved_privs ON tmpnsp = nspname AND tmpname = relname) UPDATE pg_init_privs SET initprivs = tmpini FROM to_update WHERE to_update.objoid = pg_init_privs.objoid AND classoid = 'pg_class'::regclass AND objsubid = 0; -- Can only restore permissions on views after they have been rebuilt, -- so we restore for all types of objects here. WITH to_update AS ( SELECT cl.oid, tmpacl FROM pg_class cl JOIN pg_namespace ns ON ns.oid = relnamespace JOIN _timescaledb_internal.saved_privs ON tmpnsp = nspname AND tmpname = relname) UPDATE pg_class cl SET relacl = tmpacl FROM to_update WHERE cl.oid = to_update.oid; DROP TABLE _timescaledb_internal.saved_privs; -- Create watermark record when required DO $$ DECLARE ts_version TEXT; BEGIN SELECT extversion INTO ts_version FROM pg_extension WHERE extname = 'timescaledb'; IF ts_version >= '2.11.0' THEN INSERT INTO _timescaledb_catalog.continuous_aggs_watermark (mat_hypertable_id, watermark) SELECT a.mat_hypertable_id, _timescaledb_functions.cagg_watermark_materialized(a.mat_hypertable_id) FROM _timescaledb_catalog.continuous_agg a LEFT JOIN _timescaledb_catalog.continuous_aggs_watermark b ON b.mat_hypertable_id = a.mat_hypertable_id WHERE b.mat_hypertable_id IS NULL ORDER BY 1; END IF; END; $$; -- Repair relations that have relacl entries for users that do not -- exist in pg_authid CALL _timescaledb_functions.repair_relation_acls(); -- Cleanup orphaned compression settings WITH orphaned_settings AS ( SELECT cs.relid, cl.relname FROM _timescaledb_catalog.compression_settings cs LEFT JOIN pg_class cl ON (cs.relid = cl.oid) WHERE cl.relname IS NULL ) DELETE FROM _timescaledb_catalog.compression_settings AS cs USING orphaned_settings AS os WHERE cs.relid = os.relid; ================================================ FILE: sql/updates/pre-update.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- This file is always prepended to all upgrade scripts. ================================================ FILE: sql/updates/pre-version-change.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- This file is always prepended to all upgrade and downgrade scripts. -- This file must avoid referencing extension objects directly as that -- would limit the things we can alter in extension update/downgrade -- itself. SET LOCAL search_path TO pg_catalog, pg_temp; -- Disable parallel execution for the duration of the update process. -- This avoids version mismatch errors that would have beeen triggered by the -- parallel workers in ts_extension_check_version(). SET LOCAL max_parallel_workers = 0; -- Triggers should be disabled during upgrades to avoid having them -- invoke functions that might load an old version of the shared -- library before those functions have been updated. DROP EVENT TRIGGER IF EXISTS timescaledb_ddl_command_end; DROP EVENT TRIGGER IF EXISTS timescaledb_ddl_sql_drop; -- Since we want to call the new version of restart_background_workers we -- create a function that points to that version. The proper restart_background_workers -- may either be in _timescaledb_internal or in _timescaledb_functions -- depending on the version we are upgrading from and we can't make -- the move in this location as the new schema might not have been set up. DO $$ BEGIN IF EXISTS (SELECT FROM pg_namespace WHERE nspname='_timescaledb_functions') THEN CREATE FUNCTION _timescaledb_functions._tmp_restart_background_workers() RETURNS BOOL AS '@LOADER_PATHNAME@', 'ts_bgw_db_workers_restart' LANGUAGE C VOLATILE; PERFORM _timescaledb_functions._tmp_restart_background_workers(); DROP FUNCTION _timescaledb_functions._tmp_restart_background_workers(); ELSE -- timescaledb < 2.11 does not have _timescaledb_functions schema CREATE FUNCTION _timescaledb_internal._tmp_restart_background_workers() RETURNS BOOL AS '@LOADER_PATHNAME@', 'ts_bgw_db_workers_restart' LANGUAGE C VOLATILE; PERFORM _timescaledb_internal._tmp_restart_background_workers(); DROP FUNCTION _timescaledb_internal._tmp_restart_background_workers(); END IF; END $$; -- Table for ACL and initprivs of tables. CREATE TABLE _timescaledb_internal.saved_privs( tmpnsp name, tmpname name, tmpacl aclitem[], tmpini aclitem[], UNIQUE (tmpnsp, tmpname)); -- We save away both the ACL and the initprivs for all tables and -- views in the extension (but not for chunks and internal objects) so -- that we can restore them to the proper state after the update. INSERT INTO _timescaledb_internal.saved_privs SELECT nspname, relname, relacl, initprivs FROM pg_class cl JOIN pg_namespace ns ON ns.oid = relnamespace JOIN pg_init_privs ip ON ip.objoid = cl.oid AND ip.objsubid = 0 AND ip.classoid = 'pg_class'::regclass WHERE nspname IN ('_timescaledb_catalog', '_timescaledb_config') OR ( relname IN ('hypertable_chunk_local_size', 'compressed_chunk_stats', 'bgw_job_stat', 'bgw_policy_chunk_stats') AND nspname = '_timescaledb_internal' ) ; ================================================ FILE: sql/updates/reverse-dev.sql ================================================ -- -- Rebuild the catalog table `_timescaledb_catalog.chunk` to add column `dropped` -- CREATE TABLE _timescaledb_internal.tmp_chunk AS SELECT * from _timescaledb_catalog.chunk; CREATE TABLE _timescaledb_internal.tmp_chunk_seq_value AS SELECT last_value, is_called FROM _timescaledb_catalog.chunk_id_seq; --drop foreign keys on chunk table ALTER TABLE _timescaledb_catalog.chunk_constraint DROP CONSTRAINT chunk_constraint_chunk_id_fkey; ALTER TABLE _timescaledb_catalog.chunk_column_stats DROP CONSTRAINT chunk_column_stats_chunk_id_fkey; ALTER TABLE _timescaledb_internal.bgw_policy_chunk_stats DROP CONSTRAINT bgw_policy_chunk_stats_chunk_id_fkey; ALTER TABLE _timescaledb_catalog.compression_chunk_size DROP CONSTRAINT compression_chunk_size_chunk_id_fkey; ALTER TABLE _timescaledb_catalog.compression_chunk_size DROP CONSTRAINT compression_chunk_size_compressed_chunk_id_fkey; --drop dependent views DROP VIEW IF EXISTS timescaledb_information.hypertables; DROP VIEW IF EXISTS timescaledb_information.chunks; DROP VIEW IF EXISTS _timescaledb_internal.hypertable_chunk_local_size; DROP VIEW IF EXISTS _timescaledb_internal.compressed_chunk_stats; DROP VIEW IF EXISTS timescaledb_information.chunk_columnstore_settings; DROP VIEW IF EXISTS timescaledb_information.chunk_compression_settings; ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.chunk; ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_catalog.chunk_id_seq; DROP TABLE _timescaledb_catalog.chunk; CREATE SEQUENCE _timescaledb_catalog.chunk_id_seq MINVALUE 1; -- now create table without self referential foreign key CREATE TABLE _timescaledb_catalog.chunk ( id integer NOT NULL DEFAULT nextval('_timescaledb_catalog.chunk_id_seq'), hypertable_id int NOT NULL, schema_name name NOT NULL, table_name name NOT NULL, compressed_chunk_id integer , dropped boolean NOT NULL DEFAULT FALSE, status integer NOT NULL DEFAULT 0, osm_chunk boolean NOT NULL DEFAULT FALSE, creation_time timestamptz NOT NULL, -- table constraints CONSTRAINT chunk_pkey PRIMARY KEY (id), CONSTRAINT chunk_schema_name_table_name_key UNIQUE (schema_name, table_name) ); INSERT INTO _timescaledb_catalog.chunk( id, hypertable_id, schema_name, table_name, compressed_chunk_id, dropped, status, osm_chunk, creation_time) SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, false, status, osm_chunk, creation_time FROM _timescaledb_internal.tmp_chunk; --add indexes to the chunk table CREATE INDEX chunk_hypertable_id_idx ON _timescaledb_catalog.chunk (hypertable_id); CREATE INDEX chunk_compressed_chunk_id_idx ON _timescaledb_catalog.chunk (compressed_chunk_id); CREATE INDEX chunk_osm_chunk_idx ON _timescaledb_catalog.chunk (osm_chunk, hypertable_id); CREATE INDEX chunk_hypertable_id_creation_time_idx ON _timescaledb_catalog.chunk(hypertable_id, creation_time); ALTER SEQUENCE _timescaledb_catalog.chunk_id_seq OWNED BY _timescaledb_catalog.chunk.id; SELECT setval('_timescaledb_catalog.chunk_id_seq', last_value, is_called) FROM _timescaledb_internal.tmp_chunk_seq_value; -- add self referential foreign key ALTER TABLE _timescaledb_catalog.chunk ADD CONSTRAINT chunk_compressed_chunk_id_fkey FOREIGN KEY ( compressed_chunk_id ) REFERENCES _timescaledb_catalog.chunk( id ); --add foreign key constraint ALTER TABLE _timescaledb_catalog.chunk ADD CONSTRAINT chunk_hypertable_id_fkey FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk', ''); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk_id_seq', ''); --add the foreign key constraints ALTER TABLE _timescaledb_catalog.chunk_constraint ADD CONSTRAINT chunk_constraint_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id); ALTER TABLE _timescaledb_catalog.chunk_column_stats ADD CONSTRAINT chunk_column_stats_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk (id); ALTER TABLE _timescaledb_internal.bgw_policy_chunk_stats ADD CONSTRAINT bgw_policy_chunk_stats_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.compression_chunk_size ADD CONSTRAINT compression_chunk_size_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE; ALTER TABLE _timescaledb_catalog.compression_chunk_size ADD CONSTRAINT compression_chunk_size_compressed_chunk_id_fkey FOREIGN KEY (compressed_chunk_id) REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE; --cleanup DROP TABLE _timescaledb_internal.tmp_chunk; DROP TABLE _timescaledb_internal.tmp_chunk_seq_value; GRANT SELECT ON _timescaledb_catalog.chunk_id_seq TO PUBLIC; GRANT SELECT ON _timescaledb_catalog.chunk TO PUBLIC; -- end recreate _timescaledb_catalog.chunk table -- -- Rebuild the catalog tables for continuous aggregate migration plans CREATE TABLE _timescaledb_catalog.continuous_agg_migrate_plan ( mat_hypertable_id integer NOT NULL, start_ts TIMESTAMPTZ NOT NULL DEFAULT pg_catalog.now(), end_ts TIMESTAMPTZ, user_view_definition TEXT, -- table constraints CONSTRAINT continuous_agg_migrate_plan_pkey PRIMARY KEY (mat_hypertable_id) ); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_agg_migrate_plan', ''); CREATE TABLE _timescaledb_catalog.continuous_agg_migrate_plan_step ( mat_hypertable_id integer NOT NULL, step_id serial NOT NULL, status TEXT NOT NULL DEFAULT 'NOT STARTED', -- NOT STARTED, STARTED, FINISHED, CANCELED start_ts TIMESTAMPTZ, end_ts TIMESTAMPTZ, type TEXT NOT NULL, config JSONB, -- table constraints CONSTRAINT continuous_agg_migrate_plan_step_pkey PRIMARY KEY (mat_hypertable_id, step_id), CONSTRAINT continuous_agg_migrate_plan_step_mat_hypertable_id_fkey FOREIGN KEY (mat_hypertable_id) REFERENCES _timescaledb_catalog.continuous_agg_migrate_plan (mat_hypertable_id) ON DELETE CASCADE, CONSTRAINT continuous_agg_migrate_plan_step_check CHECK (start_ts <= end_ts), CONSTRAINT continuous_agg_migrate_plan_step_check2 CHECK (type IN ('CREATE NEW CAGG', 'DISABLE POLICIES', 'COPY POLICIES', 'ENABLE POLICIES', 'SAVE WATERMARK', 'REFRESH NEW CAGG', 'COPY DATA', 'OVERRIDE CAGG', 'DROP OLD CAGG')) ); SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.continuous_agg_migrate_plan_step', ''); SELECT pg_catalog.pg_extension_config_dump(pg_get_serial_sequence('_timescaledb_catalog.continuous_agg_migrate_plan_step', 'step_id'), ''); GRANT SELECT ON ALL TABLES IN SCHEMA _timescaledb_catalog TO PUBLIC; GRANT SELECT ON ALL SEQUENCES IN SCHEMA _timescaledb_catalog TO PUBLIC; CREATE INDEX bgw_job_stat_history_job_id_idx ON _timescaledb_internal.bgw_job_stat_history (job_id); DROP INDEX _timescaledb_internal.bgw_job_stat_history_execution_start_idx; DROP INDEX _timescaledb_internal.bgw_job_stat_history_job_id_execution_start_idx; ================================================ FILE: sql/updates/set_post_update_stage.sql ================================================ set timescaledb.update_script_stage = 'post'; ================================================ FILE: sql/updates/unset_update_stage.sql ================================================ set timescaledb.update_script_stage = ''; ================================================ FILE: sql/updates/version_check.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. DO $$ DECLARE catalog_version TEXT; BEGIN SELECT value INTO catalog_version FROM _timescaledb_catalog.metadata WHERE key='timescaledb_version' AND value <> '@START_VERSION@'; IF FOUND THEN RAISE EXCEPTION 'catalog version mismatch' USING DETAIL = format('current extension version is "%s" but catalog version is "%s"', '@START_VERSION@', catalog_version), HINT = 'Make sure the TimescaleDB version used to dump the database is the same as the one used to restore it.'; END IF; END$$; ================================================ FILE: sql/util_internal_table_ddl.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- This file contains functions associated with creating new -- hypertables. -- Outputs the create_hypertable command to recreate the given hypertable. -- -- This is currently used internally for our single hypertable backup tool -- so that it knows how to restore the hypertable without user intervention. -- -- It only works for hypertables with up to 2 dimensions. CREATE OR REPLACE FUNCTION _timescaledb_functions.get_create_command( table_name NAME ) RETURNS TEXT LANGUAGE PLPGSQL VOLATILE AS $BODY$ DECLARE h_id INTEGER; schema_name NAME; time_column NAME; time_interval BIGINT; space_column NAME; space_partitions INTEGER; dimension_cnt INTEGER; dimension_row record; ret TEXT; BEGIN SELECT h.id, h.schema_name FROM _timescaledb_catalog.hypertable AS h WHERE h.table_name = get_create_command.table_name INTO h_id, schema_name; IF h_id IS NULL THEN RAISE EXCEPTION 'hypertable "%" not found', table_name USING ERRCODE = 'TS101'; END IF; SELECT COUNT(*) FROM _timescaledb_catalog.dimension d WHERE d.hypertable_id = h_id INTO STRICT dimension_cnt; IF dimension_cnt > 2 THEN RAISE EXCEPTION 'get_create_command only supports hypertables with up to 2 dimensions' USING ERRCODE = 'TS101'; END IF; FOR dimension_row IN SELECT * FROM _timescaledb_catalog.dimension d WHERE d.hypertable_id = h_id LOOP IF dimension_row.interval_length IS NOT NULL THEN time_column := dimension_row.column_name; time_interval := dimension_row.interval_length; ELSIF dimension_row.num_slices IS NOT NULL THEN space_column := dimension_row.column_name; space_partitions := dimension_row.num_slices; END IF; END LOOP; ret := format($$SELECT create_hypertable('%I.%I', '%s'$$, schema_name, table_name, time_column); IF space_column IS NOT NULL THEN ret := ret || format($$, '%I', %s$$, space_column, space_partitions); END IF; ret := ret || format($$, chunk_time_interval => %s, create_default_indexes=>FALSE);$$, time_interval); RETURN ret; END $BODY$ SET search_path TO pg_catalog, pg_temp; ================================================ FILE: sql/util_time.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- This file contains utilities for time conversion. -- Return the minimum for the type. For time types, it will be the -- Unix timestamp in microseconds. CREATE OR REPLACE FUNCTION _timescaledb_functions.get_internal_time_min(REGTYPE) RETURNS BIGINT AS '@MODULE_PATHNAME@', 'ts_get_internal_time_min' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; -- Return the minimum for the type. For time types, it will be the -- Unix timestamp in microseconds. CREATE OR REPLACE FUNCTION _timescaledb_functions.get_internal_time_max(REGTYPE) RETURNS BIGINT AS '@MODULE_PATHNAME@', 'ts_get_internal_time_max' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.to_unix_microseconds(ts TIMESTAMPTZ) RETURNS BIGINT AS '@MODULE_PATHNAME@', 'ts_pg_timestamp_to_unix_microseconds' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.to_timestamp(unixtime_us BIGINT) RETURNS TIMESTAMPTZ AS '@MODULE_PATHNAME@', 'ts_pg_unix_microseconds_to_timestamp' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.to_timestamp_without_timezone(unixtime_us BIGINT) RETURNS TIMESTAMP AS '@MODULE_PATHNAME@', 'ts_pg_unix_microseconds_to_timestamp' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.to_date(unixtime_us BIGINT) RETURNS DATE AS '@MODULE_PATHNAME@', 'ts_pg_unix_microseconds_to_date' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.to_interval(unixtime_us BIGINT) RETURNS INTERVAL AS '@MODULE_PATHNAME@', 'ts_pg_unix_microseconds_to_interval' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; -- Time can be represented in a hypertable as an int* (bigint/integer/smallint) or as a timestamp type ( -- with or without timezones). In metatables and other internal systems all time values are stored as bigint. -- Converting from int* columns to internal representation is a cast to bigint. -- Converting from timestamps to internal representation is conversion to epoch (in microseconds). CREATE OR REPLACE FUNCTION _timescaledb_functions.interval_to_usec( chunk_interval INTERVAL ) RETURNS BIGINT LANGUAGE SQL IMMUTABLE PARALLEL SAFE AS $BODY$ SELECT (int_sec * 1000000)::bigint from extract(epoch from chunk_interval) as int_sec; $BODY$ SET search_path TO pg_catalog, pg_temp; CREATE OR REPLACE FUNCTION _timescaledb_functions.time_to_internal(time_val ANYELEMENT) RETURNS BIGINT AS '@MODULE_PATHNAME@', 'ts_time_to_internal' LANGUAGE C VOLATILE STRICT; CREATE OR REPLACE FUNCTION _timescaledb_functions.cagg_watermark(hypertable_id INTEGER) RETURNS INT8 AS '@MODULE_PATHNAME@', 'ts_continuous_agg_watermark' LANGUAGE C STABLE STRICT PARALLEL RESTRICTED; CREATE OR REPLACE FUNCTION _timescaledb_functions.cagg_watermark_materialized(hypertable_id INTEGER) RETURNS INT8 AS '@MODULE_PATHNAME@', 'ts_continuous_agg_watermark_materialized' LANGUAGE C STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.subtract_integer_from_now( hypertable_relid REGCLASS, lag INT8 ) RETURNS INT8 AS '@MODULE_PATHNAME@', 'ts_subtract_integer_from_now' LANGUAGE C STABLE STRICT; -- Convert integer UNIX timestamps in microsecond to a timestamp range. CREATE OR REPLACE FUNCTION _timescaledb_functions.make_multirange_from_internal_time( base tstzrange, low_usec bigint, high_usec bigint ) RETURNS TSTZMULTIRANGE AS $body$ select multirange(tstzrange(_timescaledb_functions.to_timestamp(low_usec), _timescaledb_functions.to_timestamp(high_usec))); $body$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET search_path TO pg_catalog, pg_temp; -- Convert integer UNIX timestamps in microsecond to a timestamp range. CREATE OR REPLACE FUNCTION _timescaledb_functions.make_multirange_from_internal_time( base TSRANGE, low_usec bigint, high_usec bigint ) RETURNS TSMULTIRANGE AS $body$ select multirange(tsrange(_timescaledb_functions.to_timestamp_without_timezone(low_usec), _timescaledb_functions.to_timestamp_without_timezone(high_usec))); $body$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET search_path TO pg_catalog, pg_temp; -- Helper function to construct a range given an existing type from -- UNIX timestamps in microsecond precision. CREATE OR REPLACE FUNCTION _timescaledb_functions.make_range_from_internal_time( base anyrange, low_usec anyelement, high_usec anyelement ) RETURNS anyrange AS '@MODULE_PATHNAME@', 'ts_make_range_from_internal_time' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; ================================================ FILE: sql/uuidv7.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE OR REPLACE FUNCTION @extschema@.generate_uuidv7() RETURNS UUID AS '@MODULE_PATHNAME@', 'ts_uuid_generate_v7' LANGUAGE C VOLATILE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION @extschema@.to_uuidv7( ts TIMESTAMPTZ ) RETURNS UUID AS '@MODULE_PATHNAME@', 'ts_uuid_v7_from_timestamptz' LANGUAGE C VOLATILE STRICT PARALLEL SAFE; -- -- Produce a boundary UUIDv7 from a timestamp, with all otherwise -- random bits in the resulting UUID set to zero. Useful for -- time-range queries directly on a UUID column. -- CREATE OR REPLACE FUNCTION @extschema@.to_uuidv7_boundary( ts TIMESTAMPTZ ) RETURNS UUID AS '@MODULE_PATHNAME@', 'ts_uuid_v7_from_timestamptz_boundary' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; -- -- Get the v7 UUID timestamp with millisecond precision. -- CREATE OR REPLACE FUNCTION @extschema@.uuid_timestamp( uuid UUID ) RETURNS TIMESTAMPTZ AS '@MODULE_PATHNAME@', 'ts_timestamptz_from_uuid_v7' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; -- -- Get the v7 UUID timestamp with microsecond precision using the -- (optional) rand_a bits. -- CREATE OR REPLACE FUNCTION @extschema@.uuid_timestamp_micros( uuid UUID ) RETURNS TIMESTAMPTZ AS '@MODULE_PATHNAME@', 'ts_timestamptz_from_uuid_v7_with_microseconds' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION @extschema@.uuid_version( uuid UUID ) RETURNS INTEGER AS '@MODULE_PATHNAME@', 'ts_uuid_version' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; ================================================ FILE: sql/version.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE OR REPLACE FUNCTION _timescaledb_functions.get_git_commit() RETURNS TABLE(commit_tag TEXT, commit_hash TEXT, commit_time TIMESTAMPTZ) AS '@MODULE_PATHNAME@', 'ts_get_git_commit' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.get_os_info() RETURNS TABLE(sysname TEXT, version TEXT, release TEXT, version_pretty TEXT) AS '@MODULE_PATHNAME@', 'ts_get_os_info' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_functions.tsl_loaded() RETURNS BOOLEAN AS '@MODULE_PATHNAME@', 'ts_tsl_loaded' LANGUAGE C; ================================================ FILE: sql/views.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Convenience view to list all hypertables CREATE OR REPLACE VIEW timescaledb_information.hypertables AS WITH hypertable_info AS ( SELECT hypertable_id, schema_name, table_name, num_dimensions, compression_state, column_name, column_type, interval_length, (compression_state = 1) AS compression_enabled, row_number() OVER (PARTITION BY hypertable_id ORDER BY di.id) AS dimension_num FROM _timescaledb_catalog.hypertable ht JOIN _timescaledb_catalog.dimension di ON ht.id = di.hypertable_id ) SELECT ht.schema_name AS hypertable_schema, ht.table_name AS hypertable_name, t.tableowner AS owner, ht.num_dimensions, ( SELECT count(1) FROM _timescaledb_catalog.chunk ch WHERE ch.hypertable_id = ht.hypertable_id AND ch.osm_chunk IS FALSE ) AS num_chunks, ht.compression_enabled, srchtbs.tablespace_list AS tablespaces, ht.column_name AS primary_dimension, ht.column_type AS primary_dimension_type FROM hypertable_info ht JOIN pg_tables t ON ht.table_name = t.tablename AND ht.schema_name = t.schemaname LEFT JOIN _timescaledb_catalog.continuous_agg ca ON ca.mat_hypertable_id = ht.hypertable_id LEFT JOIN ( SELECT hypertable_id, array_agg(tablespace_name ORDER BY id) AS tablespace_list FROM _timescaledb_catalog.tablespace GROUP BY hypertable_id) srchtbs ON ht.hypertable_id = srchtbs.hypertable_id WHERE ht.compression_state != 2 --> no internal compression tables AND ca.mat_hypertable_id IS NULL AND ht.interval_length IS NOT NULL AND ht.dimension_num = 1; -- Get status of existing jobs. -- -- Note that we will always list all jobs that are available in the -- database, but some fields might be null if, for example, the job -- has not yet executed, or there is no hypertable associated with the -- job. CREATE OR REPLACE VIEW timescaledb_information.job_stats AS SELECT ht.schema_name AS hypertable_schema, ht.table_name AS hypertable_name, j.id AS job_id, js.last_start AS last_run_started_at, js.last_successful_finish AS last_successful_finish, CASE WHEN js.last_finish < '4714-11-24 00:00:00+00 BC' THEN NULL WHEN js.last_finish IS NOT NULL THEN CASE WHEN js.last_run_success = 't' THEN 'Success' WHEN js.last_run_success = 'f' THEN 'Failed' END END AS last_run_status, CASE WHEN pgs.state = 'active' THEN 'Running' WHEN j.scheduled = FALSE THEN 'Paused' ELSE 'Scheduled' END AS job_status, CASE WHEN js.last_finish > js.last_start THEN (js.last_finish - js.last_start) END AS last_run_duration, CASE WHEN j.scheduled THEN js.next_start END AS next_start, js.total_runs, js.total_successes, js.total_failures FROM _timescaledb_catalog.bgw_job j LEFT JOIN _timescaledb_internal.bgw_job_stat js ON j.id = js.job_id LEFT JOIN _timescaledb_catalog.hypertable ht ON j.hypertable_id = ht.id LEFT JOIN pg_stat_activity pgs ON pgs.datname = current_database() AND pgs.application_name = j.application_name ORDER BY ht.schema_name, ht.table_name; -- view for background worker jobs CREATE OR REPLACE VIEW timescaledb_information.jobs AS SELECT j.id AS job_id, j.application_name, j.schedule_interval, j.max_runtime, j.max_retries, j.retry_period, j.proc_schema, j.proc_name, j.owner, j.scheduled, j.fixed_schedule, j.config, js.next_start, j.initial_start, COALESCE(ca.user_view_schema, ht.schema_name) AS hypertable_schema, COALESCE(ca.user_view_name, ht.table_name) AS hypertable_name, j.check_schema, j.check_name FROM _timescaledb_catalog.bgw_job j LEFT JOIN _timescaledb_catalog.hypertable ht ON ht.id = j.hypertable_id LEFT JOIN _timescaledb_internal.bgw_job_stat js ON js.job_id = j.id LEFT JOIN _timescaledb_catalog.continuous_agg ca ON ca.mat_hypertable_id = j.hypertable_id; -- views for continuous aggregate queries --- CREATE OR REPLACE VIEW timescaledb_information.continuous_aggregates AS SELECT ht.schema_name AS hypertable_schema, ht.table_name AS hypertable_name, cagg.user_view_schema AS view_schema, cagg.user_view_name AS view_name, viewinfo.viewowner AS view_owner, cagg.materialized_only, CASE WHEN mat_ht.compressed_hypertable_id IS NOT NULL THEN TRUE ELSE FALSE END AS compression_enabled, mat_ht.schema_name AS materialization_hypertable_schema, mat_ht.table_name AS materialization_hypertable_name, directview.viewdefinition AS view_definition FROM _timescaledb_catalog.continuous_agg cagg, _timescaledb_catalog.hypertable ht, LATERAL ( SELECT C.oid, pg_get_userbyid(C.relowner) AS viewowner FROM pg_class C LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace) WHERE C.relkind = 'v' AND C.relname = cagg.user_view_name AND N.nspname = cagg.user_view_schema) viewinfo, LATERAL ( SELECT pg_get_viewdef(C.oid) AS viewdefinition FROM pg_class C LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace) WHERE C.relkind = 'v' AND C.relname = cagg.direct_view_name AND N.nspname = cagg.direct_view_schema) directview, LATERAL ( SELECT schema_name, table_name, compressed_hypertable_id FROM _timescaledb_catalog.hypertable WHERE cagg.mat_hypertable_id = id) mat_ht WHERE cagg.raw_hypertable_id = ht.id; -- chunks metadata view, shows information about the primary dimension column -- query plans with CTEs are not always optimized by PG. So use in-line -- tables. CREATE OR REPLACE VIEW timescaledb_information.chunks AS SELECT hypertable_schema, hypertable_name, schema_name AS chunk_schema, chunk_name, primary_dimension, primary_dimension_type, range_start, range_end, integer_range_start AS range_start_integer, integer_range_end AS range_end_integer, is_compressed, chunk_table_space AS chunk_tablespace, creation_time AS chunk_creation_time FROM ( SELECT ht.schema_name AS hypertable_schema, ht.table_name AS hypertable_name, srcch.schema_name AS schema_name, srcch.table_name AS chunk_name, dim.column_name AS primary_dimension, dim.column_type AS primary_dimension_type, row_number() OVER (PARTITION BY chcons.chunk_id ORDER BY dim.id) AS chunk_dimension_num, CASE WHEN dim.column_type = ANY(ARRAY['timestamp','timestamptz','date', 'uuid']::regtype[]) THEN _timescaledb_functions.to_timestamp(dimsl.range_start) ELSE NULL END AS range_start, CASE WHEN dim.column_type = ANY(ARRAY['timestamp','timestamptz','date', 'uuid']::regtype[]) THEN _timescaledb_functions.to_timestamp(dimsl.range_end) ELSE NULL END AS range_end, CASE WHEN dim.column_type = ANY(ARRAY['timestamp','timestamptz','date', 'uuid']::regtype[]) THEN NULL ELSE dimsl.range_start END AS integer_range_start, CASE WHEN dim.column_type = ANY(ARRAY['timestamp','timestamptz','date', 'uuid']::regtype[]) THEN NULL ELSE dimsl.range_end END AS integer_range_end, CASE WHEN (srcch.status & 1 = 1) THEN TRUE ELSE FALSE END AS is_compressed, pgtab.spcname AS chunk_table_space, srcch.creation_time AS creation_time FROM _timescaledb_catalog.chunk srcch INNER JOIN _timescaledb_catalog.hypertable ht ON ht.id = srcch.hypertable_id INNER JOIN _timescaledb_catalog.chunk_constraint chcons ON srcch.id = chcons.chunk_id INNER JOIN _timescaledb_catalog.dimension dim ON srcch.hypertable_id = dim.hypertable_id INNER JOIN _timescaledb_catalog.dimension_slice dimsl ON dim.id = dimsl.dimension_id AND chcons.dimension_slice_id = dimsl.id INNER JOIN ( SELECT relname, reltablespace, nspname AS schema_name FROM pg_class, pg_namespace WHERE pg_class.relnamespace = pg_namespace.oid) cl ON srcch.table_name = cl.relname AND srcch.schema_name = cl.schema_name LEFT OUTER JOIN pg_tablespace pgtab ON pgtab.oid = reltablespace WHERE srcch.osm_chunk IS FALSE AND ht.compression_state != 2 ) finalq WHERE chunk_dimension_num = 1; -- hypertable's dimension information -- CTEs aren't used in the query as PG does not always optimize them -- as expected. CREATE OR REPLACE VIEW timescaledb_information.dimensions AS SELECT ht.schema_name AS hypertable_schema, ht.table_name AS hypertable_name, rank() OVER (PARTITION BY hypertable_id ORDER BY dim.id) AS dimension_number, dim.column_name, dim.column_type, CASE WHEN dim.interval_length IS NULL THEN 'Space' ELSE 'Time' END AS dimension_type, CASE WHEN dim.interval_length IS NOT NULL THEN CASE WHEN dim.column_type = ANY(ARRAY['timestamp','timestamptz','date', 'uuid']::regtype[]) THEN _timescaledb_functions.to_interval(dim.interval_length) ELSE NULL END END AS time_interval, CASE WHEN dim.interval_length IS NOT NULL THEN CASE WHEN dim.column_type = ANY(ARRAY['timestamp','timestamptz','date', 'uuid']::regtype[]) THEN NULL ELSE dim.interval_length END END AS integer_interval, dim.integer_now_func, dim.num_slices AS num_partitions FROM _timescaledb_catalog.hypertable ht, _timescaledb_catalog.dimension dim WHERE dim.hypertable_id = ht.id; ---compression parameters information --- CREATE OR REPLACE VIEW timescaledb_information.compression_settings AS SELECT schema_name AS hypertable_schema, table_name AS hypertable_name, (unnest(cs.segmentby))::name COLLATE "C" AS attname, generate_series(1,array_length(cs.segmentby,1))::smallint AS segmentby_column_index, NULL::smallint AS orderby_column_index, NULL::bool AS orderby_asc, NULL::bool AS orderby_nullsfirst FROM _timescaledb_catalog.hypertable ht INNER JOIN _timescaledb_catalog.compression_settings cs ON cs.relid = format('%I.%I',ht.schema_name,ht.table_name)::regclass AND cs.segmentby IS NOT NULL WHERE compressed_hypertable_id IS NOT NULL UNION ALL SELECT schema_name AS hypertable_schema, table_name AS hypertable_name, (unnest(cs.orderby))::name COLLATE "C" AS attname, NULL::smallint AS segmentby_column_index, generate_series(1,array_length(cs.orderby,1))::smallint AS orderby_column_index, unnest(array_replace(array_replace(array_replace(cs.orderby_desc,false,NULL),true,false),NULL,true)) AS orderby_asc, unnest(cs.orderby_nullsfirst) AS orderby_nullsfirst FROM _timescaledb_catalog.hypertable ht INNER JOIN _timescaledb_catalog.compression_settings cs ON cs.relid = format('%I.%I',ht.schema_name,ht.table_name)::regclass AND cs.orderby IS NOT NULL WHERE compressed_hypertable_id IS NOT NULL ORDER BY hypertable_name, segmentby_column_index, orderby_column_index; -- Job errors view that adds a security barrier on the bgw_job_stat_history -- table in _timescaledb_internal. The view only allows users to view -- log entries belonging to jobs that are owned by any of the users -- role. A special case is added so that the superuser or the database -- owner can see all job log entries, even those that do not have an -- associated job. -- -- Note that we have to use a sub-select here since pg_database_owner -- does not exist before PostgreSQL 14. CREATE OR REPLACE VIEW timescaledb_information.job_errors WITH (security_barrier = true) AS SELECT h.job_id, h.data->'job'->>'proc_schema' as proc_schema, h.data->'job'->>'proc_name' as proc_name, h.pid, h.execution_start AS start_time, h.execution_finish AS finish_time, h.data->'error_data'->>'sqlerrcode' AS sqlerrcode, CASE WHEN h.succeeded IS NULL AND h.execution_finish IS NULL AND h.pid IS NULL THEN 'job crash detected, see server logs' WHEN h.data->'error_data'->>'message' IS NOT NULL THEN CASE WHEN h.data->'error_data'->>'detail' IS NOT NULL THEN CASE WHEN h.data->'error_data'->>'hint' IS NOT NULL THEN concat(h.data->'error_data'->>'message', '. ', h.data->'error_data'->>'detail', '. ', h.data->'error_data'->>'hint') ELSE concat(h.data->'error_data'->>'message', ' ', h.data->'error_data'->>'detail') END ELSE CASE WHEN h.data->'error_data'->>'hint' IS NOT NULL THEN concat(h.data->'error_data'->>'message', '. ', h.data->'error_data'->>'hint') ELSE h.data->'error_data'->>'message' END END END AS err_message FROM _timescaledb_internal.bgw_job_stat_history h LEFT JOIN _timescaledb_catalog.bgw_job j ON (j.id = h.job_id) WHERE h.succeeded IS FALSE OR h.succeeded IS NULL AND (pg_catalog.pg_has_role(current_user, (SELECT pg_catalog.pg_get_userbyid(datdba) FROM pg_catalog.pg_database WHERE datname = current_database()), 'MEMBER') IS TRUE OR pg_catalog.pg_has_role(current_user, owner, 'MEMBER') IS TRUE); CREATE OR REPLACE VIEW timescaledb_information.job_history WITH (security_barrier = true) AS SELECT h.id, h.job_id, h.succeeded, coalesce(h.data->'job'->>'proc_schema', j.proc_schema) as proc_schema, coalesce(h.data->'job'->>'proc_name', j.proc_name) as proc_name, h.pid, h.execution_start AS start_time, h.execution_finish AS finish_time, h.data->'job'->'config' AS config, h.data->'error_data'->>'sqlerrcode' AS sqlerrcode, CASE WHEN h.succeeded IS NULL AND h.execution_finish IS NULL AND h.pid IS NULL THEN 'job crash detected, see server logs' WHEN h.succeeded IS FALSE AND h.data->'error_data'->>'message' IS NOT NULL THEN CASE WHEN h.data->'error_data'->>'detail' IS NOT NULL THEN CASE WHEN h.data->'error_data'->>'hint' IS NOT NULL THEN concat(h.data->'error_data'->>'message', '. ', h.data->'error_data'->>'detail', '. ', h.data->'error_data'->>'hint') ELSE concat(h.data->'error_data'->>'message', ' ', h.data->'error_data'->>'detail') END ELSE CASE WHEN h.data->'error_data'->>'hint' IS NOT NULL THEN concat(h.data->'error_data'->>'message', '. ', h.data->'error_data'->>'hint') ELSE h.data->'error_data'->>'message' END END END AS err_message FROM _timescaledb_internal.bgw_job_stat_history h LEFT JOIN _timescaledb_catalog.bgw_job j ON (j.id = h.job_id) WHERE (pg_catalog.pg_has_role(current_user, (SELECT pg_catalog.pg_get_userbyid(datdba) FROM pg_catalog.pg_database WHERE datname = current_database()), 'MEMBER') IS TRUE OR pg_catalog.pg_has_role(current_user, owner, 'MEMBER') IS TRUE); CREATE OR REPLACE VIEW timescaledb_information.hypertable_compression_settings AS SELECT format('%I.%I',ht.schema_name,ht.table_name)::regclass AS hypertable, array_to_string(segmentby,',') AS segmentby, un.orderby, d.compress_interval_length, s.index AS index FROM _timescaledb_catalog.hypertable ht JOIN LATERAL ( SELECT CASE WHEN d.column_type = ANY(ARRAY['timestamp','timestamptz','date']::regtype[]) THEN _timescaledb_functions.to_interval(d.compress_interval_length)::text ELSE d.compress_interval_length::text END AS compress_interval_length FROM _timescaledb_catalog.dimension d WHERE d.hypertable_id = ht.id ORDER BY id LIMIT 1 ) d ON true LEFT JOIN _timescaledb_catalog.compression_settings s ON format('%I.%I',ht.schema_name,ht.table_name)::regclass = s.relid LEFT JOIN LATERAL ( SELECT string_agg( format('%I%s%s',orderby, CASE WHEN "desc" THEN ' DESC' ELSE '' END, CASE WHEN nullsfirst AND NOT "desc" THEN ' NULLS FIRST' WHEN NOT nullsfirst AND "desc" THEN ' NULLS LAST' ELSE '' END ) ,',') AS orderby FROM unnest(s.orderby, s.orderby_desc, s.orderby_nullsfirst) un(orderby, "desc", nullsfirst) ) un ON true; CREATE OR REPLACE VIEW timescaledb_information.chunk_compression_settings AS SELECT format('%I.%I',ht.schema_name,ht.table_name)::regclass AS hypertable, format('%I.%I',ch.schema_name,ch.table_name)::regclass AS chunk, array_to_string(segmentby,',') AS segmentby, un.orderby, s.index AS index FROM _timescaledb_catalog.hypertable ht INNER JOIN _timescaledb_catalog.chunk ch ON ch.hypertable_id = ht.id INNER JOIN _timescaledb_catalog.compression_settings s ON (format('%I.%I',ch.schema_name,ch.table_name)::regclass = s.relid) LEFT JOIN LATERAL ( SELECT string_agg( format('%I%s%s',orderby, CASE WHEN "desc" THEN ' DESC' ELSE '' END, CASE WHEN nullsfirst AND NOT "desc" THEN ' NULLS FIRST' WHEN NOT nullsfirst AND "desc" THEN ' NULLS LAST' ELSE '' END ) ,',') AS orderby FROM unnest(s.orderby, s.orderby_desc, s.orderby_nullsfirst) un(orderby, "desc", nullsfirst) ) un ON true; CREATE OR REPLACE VIEW timescaledb_information.hypertable_columnstore_settings AS SELECT * FROM timescaledb_information.hypertable_compression_settings; CREATE OR REPLACE VIEW timescaledb_information.chunk_columnstore_settings AS SELECT * FROM timescaledb_information.chunk_compression_settings; --temporary alias for bgw_job CREATE OR REPLACE VIEW _timescaledb_config.bgw_job AS SELECT * from _timescaledb_catalog.bgw_job; GRANT SELECT ON ALL TABLES IN SCHEMA _timescaledb_config TO PUBLIC; GRANT SELECT ON ALL TABLES IN SCHEMA timescaledb_information TO PUBLIC; ================================================ FILE: sql/views_experimental.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE OR REPLACE VIEW timescaledb_experimental.policies AS SELECT ca.user_view_name AS relation_name, ca.user_view_schema AS relation_schema, j.schedule_interval, j.proc_schema, j.proc_name, j.config, ht.schema_name AS hypertable_schema, ht.table_name AS hypertable_name FROM _timescaledb_catalog.bgw_job j JOIN _timescaledb_catalog.continuous_agg ca ON ca.mat_hypertable_id = j.hypertable_id JOIN _timescaledb_catalog.hypertable ht ON ht.id = ca.mat_hypertable_id; GRANT SELECT ON ALL TABLES IN SCHEMA timescaledb_experimental TO PUBLIC; ================================================ FILE: sql/with_telemetry.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE OR REPLACE FUNCTION @extschema@.get_telemetry_report() RETURNS jsonb AS '@MODULE_PATHNAME@', 'ts_telemetry_get_report_jsonb' LANGUAGE C STABLE PARALLEL SAFE; INSERT INTO _timescaledb_catalog.bgw_job (id, application_name, schedule_interval, max_runtime, max_retries, retry_period, proc_schema, proc_name, owner, scheduled, fixed_schedule) VALUES (1, 'Telemetry Reporter [1]', INTERVAL '24h', INTERVAL '100s', -1, INTERVAL '1h', '_timescaledb_functions', 'policy_telemetry', pg_catalog.quote_ident(current_role)::regrole, true, false) ON CONFLICT (id) DO NOTHING; ================================================ FILE: sql/without_telemetry.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- We drop the telemetry function here since we have a non-telemetry -- build and the function might exist. DROP FUNCTION IF EXISTS @extschema@.get_telemetry_report; -- We delete the telemetry job since this is a non-telemetry build. DELETE FROM _timescaledb_catalog.bgw_job WHERE id = 1; ================================================ FILE: src/CMakeLists.txt ================================================ set(SOURCES uuid.c agg_bookend.c bmslist_utils.c func_cache.c cache.c cache_invalidate.c chunk.c chunk_adaptive.c chunk_constraint.c chunk_index.c chunk_insert_state.c chunk_scan.c chunk_tuple_routing.c constraint.c cross_module_fn.c copy.c dimension.c dimension_slice.c dimension_vector.c estimate.c event_trigger.c extension.c extension_constants.c expression_utils.c foreign_key.c gapfill.c guc.c histogram.c hypercube.c hypertable.c hypertable_cache.c hypertable_restrict_info.c indexing.c init.c jsonb_utils.c license_guc.c osm_callbacks.c partitioning.c partition_chunk.c process_utility.c scanner.c scan_iterator.c sort_transform.c subspace_store.c timezones.c time_bucket.c time_utils.c custom_type_cache.c trigger.c utils.c version.c tss_callbacks.c) # Add test source code in Debug builds if(CMAKE_BUILD_TYPE MATCHES Debug) set(TS_DEBUG 1) set(DEBUG 1) list(APPEND SOURCES debug_point.c) endif(CMAKE_BUILD_TYPE MATCHES Debug) include(build-defs.cmake) set(GITREV_TMP ${CMAKE_CURRENT_BINARY_DIR}/tmp_gitcommit.h) set(GITREV_FILE ${CMAKE_CURRENT_BINARY_DIR}/gitcommit.h) # The commands for generating gitcommit.h need to be executed on every make run # and not on cmake run to detect branch switches, commit changes or local # modifications. That's why we add the commands in a custom target and run them # on every make run. We do the generation part in a temporary file and only # overwrite the actual file when the content is different to not trigger # unnecessary recompilations. add_custom_target( gitcheck COMMAND ${CMAKE_COMMAND} "-DGIT_FOUND=${GIT_FOUND}" "-DSOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}" "-DGIT_EXECUTABLE=${GIT_EXECUTABLE}" "-DINPUT_FILE=${CMAKE_CURRENT_SOURCE_DIR}/gitcommit.h.in" "-DOUTPUT_FILE=${GITREV_TMP}" -P ${CMAKE_CURRENT_SOURCE_DIR}/gitcommit.cmake COMMAND ${CMAKE_COMMAND} -E copy_if_different ${GITREV_TMP} ${GITREV_FILE}) if(CMAKE_BUILD_TYPE MATCHES Debug) add_library(${PROJECT_NAME} MODULE ${SOURCES} ${GITCOMMIT_H} $<TARGET_OBJECTS:${TESTS_LIB_NAME}>) else() add_library(${PROJECT_NAME} MODULE ${SOURCES} ${GITCOMMIT_H}) endif() if(USE_TELEMETRY) if(SEND_TELEMETRY_DEFAULT) set(TELEMETRY_DEFAULT TELEMETRY_BASIC) else() set(TELEMETRY_DEFAULT TELEMETRY_OFF) endif() endif() set_target_properties( ${PROJECT_NAME} PROPERTIES OUTPUT_NAME ${PROJECT_NAME}-${PROJECT_VERSION_MOD} PREFIX "") install(TARGETS ${PROJECT_NAME} DESTINATION ${PG_PKGLIBDIR}) if(USE_OPENSSL) set(TS_USE_OPENSSL ${USE_OPENSSL}) target_include_directories(${PROJECT_NAME} SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR}) if(MSVC) target_link_libraries(${PROJECT_NAME} ${OPENSSL_LIBRARIES}) endif(MSVC) endif(USE_OPENSSL) configure_file(config.h.in config.h) add_dependencies(${PROJECT_NAME} gitcheck) include_directories(${CMAKE_CURRENT_SOURCE_DIR}) add_subdirectory(bgw) add_subdirectory(net) if(USE_TELEMETRY) add_subdirectory(telemetry) endif() add_subdirectory(loader) add_subdirectory(bgw_policy) add_subdirectory(compat) add_subdirectory(ts_catalog) add_subdirectory(nodes) add_subdirectory(planner) add_subdirectory(with_clause) # Don't run clang-tidy on the files we copied from Postgres. We don't want to # introduce changes there unless absolutely necessary. CMake can only access the # file properties if the target was added in the same directory, so just move it # all here. set(IMPORTED_SOURCES import/allpaths.c import/heapswap.c import/list.c import/planner.c import/ts_explain.c) set_source_files_properties(${IMPORTED_SOURCES} PROPERTIES SKIP_LINTING ON) target_sources(${PROJECT_NAME} PRIVATE ${IMPORTED_SOURCES}) ================================================ FILE: src/README.md ================================================ # Basic TimescaleDB Features - [TimescaleDB Abstract Data Types](adts/README.md) - [TimescaleDB Scheduler](bgw/README.md) - [TimescaleDB Multi-version Loader](loader/README.md) ================================================ FILE: src/adts/README.md ================================================ # A Collection of ADTs This directory contains a collection of Abstract Data Types. These ADTS are containers that can store data of any other type. These ADTs use macros as oppossed to void pointers for performance reasons, as well as for better type safety. This approach to ADTs follows Postgres convention. ## Vector A dynamic vector implementation that can store any type. It handles growing/shrinking the memory for you. ## Bit Array A dynamic vector to store bits. The API allows appending and iterating an arbitrary amount of bits. It stores the bits in a vector of uint64 and has methods to serialize/deserialize. ================================================ FILE: src/adts/bit_array.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <lib/stringinfo.h> #include <libpq/pqformat.h> #include "bit_array_impl.h" #include <adts/uint64_vec.h> /******************* *** Public API *** *******************/ /* Array to hold blobs of bits of arbitrary sizes. The interface * expects you to read the same amount of bits, in the same order, as what * was written. */ typedef struct BitArray BitArray; typedef struct BitArrayIterator BitArrayIterator; /* Main Interface */ static void bit_array_init(BitArray *array, int expected_bits); /* Append num_bits to the array */ static void bit_array_append(BitArray *array, uint8 num_bits, uint64 bits); static void bit_array_iterator_init(BitArrayIterator *iter, const BitArray *array); /* return next num_bits from the iterator; must have been written as num_bits */ pg_attribute_always_inline static uint64 bit_array_iter_next(BitArrayIterator *iter, uint8 num_bits); static void bit_array_iterator_init_rev(BitArrayIterator *iter, const BitArray *array); /* return last num_bits in forward-order (not reverse-order); must have been written as num_bits */ static uint64 bit_array_iter_next_rev(BitArrayIterator *iter, uint8 num_bits); /* I/O */ static inline void bit_array_send(StringInfo buffer, const BitArray *data); static inline BitArray bit_array_recv(const StringInfo buffer); static char *bytes_store_bit_array_and_advance(char *dest, size_t expected_size, const BitArray *array, uint32 *num_buckets, uint8 *bits_in_last_bucket); static size_t bit_array_output(const BitArray *array, uint64 *data, size_t max_n_bytes, uint64 *num_bits_out); static void bit_array_wrap(BitArray *dst, uint64 *data, uint64 num_bits); /* Accessors / Info */ static uint64 bit_array_num_bits(const BitArray *array); static uint32 bit_array_num_buckets(const BitArray *array); static uint64 *bit_array_buckets(const BitArray *array); static size_t bit_array_data_bytes_used(const BitArray *array); ================================================ FILE: src/adts/bit_array_impl.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <lib/stringinfo.h> #include <libpq/pqformat.h> #include "compat/compat.h" #include "adts/uint64_vec.h" #define BITS_PER_BUCKET 64 typedef struct BitArray { uint64_vec buckets; uint8 bits_used_in_last_bucket; } BitArray; typedef struct BitArrayIterator { const BitArray *array; uint8 bits_used_in_current_bucket; /* note that current_bucket should be signed since it sometimes gets decremented and must be * able to hold UINT32_MAX */ int64 current_bucket; } BitArrayIterator; /************************ *** Private Helpers *** ************************/ static void bit_array_append_bucket(BitArray *array, uint8 bits_used, uint64 bucket); static uint64 bit_array_low_bits_mask(uint8 bits_used); static inline void bit_array_wrap_internal(BitArray *array, uint32 num_buckets, uint8 bits_used_in_last_bucket, uint64 *buckets); /************************ *** Implementation *** ************************/ static inline void bit_array_init(BitArray *array, int expected_bits) { *array = (BitArray){ .bits_used_in_last_bucket = 0, }; uint64_vec_init(&array->buckets, CurrentMemoryContext, expected_bits / 64); } /* This initializes the bit array by wrapping buckets. Note, that the bit array will * point to buckets instead of creating a new copy. */ static inline void bit_array_wrap_internal(BitArray *array, uint32 num_buckets, uint8 bits_used_in_last_bucket, uint64 *buckets) { *array = (BitArray){ .bits_used_in_last_bucket = bits_used_in_last_bucket, .buckets = (uint64_vec){ .data = buckets, .num_elements = num_buckets, .max_elements = num_buckets, }, }; } static inline size_t bit_array_data_bytes_used(const BitArray *array) { return array->buckets.num_elements * sizeof(*array->buckets.data); } static inline uint32 bit_array_num_buckets(const BitArray *array) { return array->buckets.num_elements; } static inline uint64 bit_array_num_bits(const BitArray *array) { return (BITS_PER_BUCKET * (array->buckets.num_elements - UINT64CONST(1))) + array->bits_used_in_last_bucket; } static inline uint64 * bit_array_buckets(const BitArray *array) { return array->buckets.data; } static inline BitArray bit_array_recv(const StringInfo buffer) { uint32 i; uint32 num_elements = pq_getmsgint32(buffer); uint8 bits_used_in_last_bucket = pq_getmsgbyte(buffer); BitArray array; CheckCompressedData(num_elements <= GLOBAL_MAX_ROWS_PER_COMPRESSION); CheckCompressedData(bits_used_in_last_bucket <= BITS_PER_BUCKET); array = (BitArray){ .bits_used_in_last_bucket = bits_used_in_last_bucket, .buckets = { .num_elements = num_elements, .max_elements = num_elements, .ctx = CurrentMemoryContext, .data = palloc(num_elements * sizeof(uint64)), }, }; for (i = 0; i < num_elements; i++) array.buckets.data[i] = pq_getmsgint64(buffer); return array; } static inline void bit_array_send(StringInfo buffer, const BitArray *data) { pq_sendint32(buffer, data->buckets.num_elements); pq_sendbyte(buffer, data->bits_used_in_last_bucket); for (uint32 i = 0; i < data->buckets.num_elements; i++) pq_sendint64(buffer, data->buckets.data[i]); } static inline size_t bit_array_output(const BitArray *array, uint64 *dst, size_t max_n_bytes, uint64 *num_bits_out) { size_t size = bit_array_data_bytes_used(array); if (max_n_bytes < size) elog(ERROR, "not enough memory to serialize bit array"); if (num_bits_out != NULL) *num_bits_out = bit_array_num_bits(array); memcpy(dst, array->buckets.data, size); return size; } static inline char * bytes_store_bit_array_and_advance(char *dest, size_t expected_size, const BitArray *array, uint32 *num_buckets_out, uint8 *bits_in_last_bucket_out) { size_t size = bit_array_data_bytes_used(array); if (expected_size != size) elog(ERROR, "the size to serialize does not match the bit array"); *num_buckets_out = bit_array_num_buckets(array); *bits_in_last_bucket_out = array->bits_used_in_last_bucket; if (size > 0) { Assert(array->buckets.data != NULL); memcpy(dest, array->buckets.data, size); } return dest + size; } static inline void bit_array_wrap(BitArray *dst, uint64 *data, uint64 num_bits) { uint32 num_buckets = num_bits / BITS_PER_BUCKET; uint8 bits_used_in_last_bucket = num_bits % BITS_PER_BUCKET; if (bits_used_in_last_bucket == 0) { /* last bucket uses all bits */ if (num_buckets > 0) bits_used_in_last_bucket = BITS_PER_BUCKET; } else num_buckets += 1; bit_array_wrap_internal(dst, num_buckets, bits_used_in_last_bucket, data); } static inline void bit_array_append(BitArray *array, uint8 num_bits, uint64 bits) { /* Fill bits from LSB to MSB */ uint8 bits_remaining_in_last_bucket; uint8 num_bits_for_new_bucket; uint64 bits_for_new_bucket; Assert(num_bits <= 64); if (num_bits == 0) return; if (array->buckets.num_elements == 0) bit_array_append_bucket(array, 0, 0); bits &= bit_array_low_bits_mask(num_bits); bits_remaining_in_last_bucket = 64 - array->bits_used_in_last_bucket; if (bits_remaining_in_last_bucket >= num_bits) { uint64 *bucket = uint64_vec_last(&array->buckets); /* mask out any unused high bits, probably unneeded */ *bucket |= bits << array->bits_used_in_last_bucket; array->bits_used_in_last_bucket += num_bits; return; } /* When splitting an integer across buckets, the low-order bits go into the first bucket and * the high-order bits go into the second bucket */ num_bits_for_new_bucket = num_bits - bits_remaining_in_last_bucket; if (bits_remaining_in_last_bucket > 0) { uint64 bits_for_current_bucket = bits & bit_array_low_bits_mask(bits_remaining_in_last_bucket); uint64 *current_bucket = uint64_vec_last(&array->buckets); *current_bucket |= bits_for_current_bucket << array->bits_used_in_last_bucket; bits >>= bits_remaining_in_last_bucket; } /* We zero out the high bits of the new bucket, to ensure that unused bits are always 0 */ bits_for_new_bucket = bits & bit_array_low_bits_mask(num_bits_for_new_bucket); bit_array_append_bucket(array, num_bits_for_new_bucket, bits_for_new_bucket); } static inline void bit_array_iterator_init(BitArrayIterator *iter, const BitArray *array) { *iter = (BitArrayIterator){ .array = array, }; } pg_attribute_always_inline static uint64 bit_array_iter_next(BitArrayIterator *iter, uint8 num_bits) { uint8 bits_remaining_in_current_bucket; uint8 num_bits_from_next_bucket; uint64 value = 0; uint64 value_from_next_bucket; CheckCompressedData(num_bits <= 64); if (num_bits == 0) return 0; CheckCompressedData(iter->current_bucket < iter->array->buckets.num_elements); bits_remaining_in_current_bucket = 64 - iter->bits_used_in_current_bucket; if (bits_remaining_in_current_bucket >= num_bits) { value = *uint64_vec_get(&iter->array->buckets, iter->current_bucket); value >>= iter->bits_used_in_current_bucket; value &= bit_array_low_bits_mask(num_bits); iter->bits_used_in_current_bucket += num_bits; return value; } num_bits_from_next_bucket = num_bits - bits_remaining_in_current_bucket; if (bits_remaining_in_current_bucket > 0) { /* The first bucket has the low-order bits */ value = *uint64_vec_get(&iter->array->buckets, iter->current_bucket); value >>= iter->bits_used_in_current_bucket; } /* The second bucket has the high-order bits */ CheckCompressedData(iter->current_bucket + 1 < iter->array->buckets.num_elements); value_from_next_bucket = *uint64_vec_get(&iter->array->buckets, iter->current_bucket + 1) & bit_array_low_bits_mask(num_bits_from_next_bucket); value_from_next_bucket <<= bits_remaining_in_current_bucket; value |= value_from_next_bucket; iter->current_bucket += 1; iter->bits_used_in_current_bucket = num_bits_from_next_bucket; return value; } static inline void bit_array_iterator_init_rev(BitArrayIterator *iter, const BitArray *array) { *iter = (BitArrayIterator){ .array = array, .current_bucket = array->buckets.num_elements - 1, .bits_used_in_current_bucket = array->bits_used_in_last_bucket, }; } static inline uint64 bit_array_iter_next_rev(BitArrayIterator *iter, uint8 num_bits) { uint8 bits_remaining_in_current_bucket; uint8 num_bits_from_previous_bucket; uint64 value = 0; uint64 bits_from_previous; Assert(num_bits <= BITS_PER_BUCKET); if (num_bits == 0) return 0; Assert(iter->current_bucket >= 0); bits_remaining_in_current_bucket = iter->bits_used_in_current_bucket; if (bits_remaining_in_current_bucket >= num_bits) { value = *uint64_vec_get(&iter->array->buckets, iter->current_bucket); value >>= (iter->bits_used_in_current_bucket - num_bits); value &= bit_array_low_bits_mask(num_bits); iter->bits_used_in_current_bucket -= num_bits; return value; } Assert(iter->current_bucket - 1 >= 0); num_bits_from_previous_bucket = num_bits - bits_remaining_in_current_bucket; Assert(num_bits <= BITS_PER_BUCKET); if (bits_remaining_in_current_bucket > 0) { /* The current bucket has the high-order bits (it's the second bucket) */ value |= *uint64_vec_get(&iter->array->buckets, iter->current_bucket) & bit_array_low_bits_mask(bits_remaining_in_current_bucket); value <<= num_bits_from_previous_bucket; } /* The previous bucket has the low-order bits (it's the first bucket) */ bits_from_previous = *uint64_vec_get(&iter->array->buckets, iter->current_bucket - 1); bits_from_previous >>= BITS_PER_BUCKET - num_bits_from_previous_bucket; bits_from_previous &= bit_array_low_bits_mask(num_bits_from_previous_bucket); value |= bits_from_previous; iter->current_bucket -= 1; iter->bits_used_in_current_bucket = BITS_PER_BUCKET - num_bits_from_previous_bucket; return value; } /************************ *** Private Helpers *** ************************/ static void bit_array_append_bucket(BitArray *array, uint8 bits_used, uint64 bucket) { uint64_vec_append(&array->buckets, bucket); array->bits_used_in_last_bucket = bits_used; } static uint64 bit_array_low_bits_mask(uint8 bits_used) { Assert(bits_used > 0); return ~0ULL >> (64 - bits_used); } ================================================ FILE: src/adts/char_vec.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #define VEC_PREFIX char #define VEC_ELEMENT_TYPE char #define VEC_DECLARE 1 #define VEC_DEFINE 1 #define VEC_SCOPE static inline #include <adts/vec.h> ================================================ FILE: src/adts/uint64_vec.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #define VEC_PREFIX uint64 #define VEC_ELEMENT_TYPE uint64 #define VEC_DECLARE 1 #define VEC_DEFINE 1 #define VEC_SCOPE static inline #include <adts/vec.h> ================================================ FILE: src/adts/vec.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ /* NOTE header guard deliberately omitted, as it is valid to include this header * multiple times in a single file */ /* * To generate a vector and associated functions for a use case several * macros have to be #define'ed before this file is included. Including * the file #undef's all those, so a new vectors can be generated afterwards. * The relevant parameters are: * - VEC_PREFIX - prefix for all symbol names generated. A prefix of 'foo' * will result in vector table type 'foo_vec' and functions like * 'foo_vec_append'/'foo_vec_at' and so forth. This is usually the * name of the stored element type * - VEC_ELEMENT_TYPE - type of the contained elements. * - VEC_DECLARE - if defined function prototypes and type declarations are * generated * - VEC_DEFINE - if defined function definitions are generated * - VEC_SCOPE - in which scope (e.g. extern, static inline) do function * declarations reside */ #define VEC_MAKE_PREFIX(a) CppConcat(a, _) #define VEC_MAKE_NAME(name) VEC_MAKE_NAME_(VEC_MAKE_PREFIX(VEC_PREFIX), name) #define VEC_MAKE_NAME_(a, b) CppConcat(a, b) /* name macros for: */ /* type declarations */ #define VEC_TYPE VEC_MAKE_NAME(vec) /* function declarations */ #define VEC_INIT VEC_MAKE_NAME(vec_init) #define VEC_CREATE VEC_MAKE_NAME(vec_create) #define VEC_FREE_DATA VEC_MAKE_NAME(vec_free_data) #define VEC_CLEAR VEC_MAKE_NAME(vec_clear) #define VEC_AT VEC_MAKE_NAME(vec_at) #define VEC_GET VEC_MAKE_NAME(vec_get) #define VEC_LAST VEC_MAKE_NAME(vec_last) #define VEC_APPEND VEC_MAKE_NAME(vec_append) #define VEC_APPEND_ARRAY VEC_MAKE_NAME(vec_append_array) #define VEC_APPEND_ZEROS VEC_MAKE_NAME(vec_append_zeros) #define VEC_DELETE VEC_MAKE_NAME(vec_delete) #define VEC_DELETE_RANGE VEC_MAKE_NAME(vec_delete_range) #define VEC_RESERVE VEC_MAKE_NAME(vec_reserve) #define VEC_FREE VEC_MAKE_NAME(vec_free) /* generate forward declarations necessary to use the vector */ #ifdef VEC_DECLARE /* type definitions */ typedef struct VEC_TYPE { /* size of the elements array */ uint32 max_elements; /* number of elements currently used */ uint32 num_elements; /* the actual data */ VEC_ELEMENT_TYPE *data; /* memory context to use for allocations */ MemoryContext ctx; } VEC_TYPE; /* externally visible function prototypes */ VEC_SCOPE void VEC_INIT(VEC_TYPE *vec, MemoryContext ctx, uint32 nelements); VEC_SCOPE VEC_TYPE *VEC_CREATE(MemoryContext ctx, uint32 nelements); VEC_SCOPE void VEC_FREE_DATA(VEC_TYPE *vec); VEC_SCOPE void VEC_CLEAR(VEC_TYPE *vec); VEC_SCOPE VEC_ELEMENT_TYPE *VEC_AT(VEC_TYPE *vec, uint32 index); VEC_SCOPE const VEC_ELEMENT_TYPE *VEC_GET(const VEC_TYPE *vec, uint32 index); VEC_SCOPE VEC_ELEMENT_TYPE *VEC_LAST(VEC_TYPE *vec); VEC_SCOPE void VEC_APPEND(VEC_TYPE *vec, VEC_ELEMENT_TYPE element); VEC_SCOPE VEC_ELEMENT_TYPE *VEC_APPEND_ARRAY(VEC_TYPE *vec, VEC_ELEMENT_TYPE *elements, uint32 num_elements); VEC_SCOPE VEC_ELEMENT_TYPE *VEC_APPEND_ZEROS(VEC_TYPE *vec, uint32 num_elements); VEC_SCOPE void VEC_DELETE(VEC_TYPE *vec, uint32 index); VEC_SCOPE void VEC_DELETE_RANGE(VEC_TYPE *vec, uint32 start, uint32 len); VEC_SCOPE void VEC_RESERVE(VEC_TYPE *vec, uint32 additional); VEC_SCOPE void VEC_FREE(VEC_TYPE *vec); #endif /* VEC_DECLARE */ /* generate implementation of the vector */ #ifdef VEC_DEFINE #include <utils/memutils.h> /* * Allocate space so the vector can store least `additional` new elements. * * Usually this will automatically be called by appends/inserts, when * necessary. But resizing to the exact input size can be advantageous * performance-wise, when known at some point. */ VEC_SCOPE void VEC_RESERVE(VEC_TYPE *vec, uint32 additional) { uint64 num_new_elements = additional; uint64 max_element_limit = MaxAllocSize / sizeof(VEC_ELEMENT_TYPE); uint64 num_elements; uint64 num_bytes; /* this doesn't handle integer overflow or >MaxAllocSize allocations */ if (num_new_elements == 0 || vec->num_elements + num_new_elements <= vec->max_elements) return; num_elements = vec->num_elements + num_new_elements; if (num_new_elements < vec->num_elements) { /* Follow the usual doubling progression of allocation sizes. */ num_elements = vec->num_elements * 2; } if (num_elements >= max_element_limit) { if (vec->num_elements + num_new_elements >= max_element_limit) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("vector allocation overflow when trying to allocate %ld bytes", (long) ((vec->num_elements + num_new_elements) * sizeof(VEC_ELEMENT_TYPE))))); /* Clamp num_elements to max allocation size if they can fit*/ num_elements = max_element_limit; } Assert(num_elements > vec->num_elements); vec->max_elements = num_elements; num_bytes = vec->max_elements * sizeof(VEC_ELEMENT_TYPE); if (vec->data == NULL) vec->data = MemoryContextAlloc(vec->ctx, num_bytes); else vec->data = repalloc(vec->data, num_bytes); } /* * Initialize a vector with enough space for `nelements`. Memory is allocated * from the passed-in context. */ VEC_SCOPE void VEC_INIT(VEC_TYPE *vec, MemoryContext ctx, uint32 nelements) { *vec = (VEC_TYPE){ .ctx = ctx, }; if (nelements > 0) VEC_RESERVE(vec, nelements); } /* * Create a vector with enough space for `nelements`. Memory for the vector, and * its elements, is allocated from the passed-in context. */ VEC_SCOPE VEC_TYPE * VEC_CREATE(MemoryContext ctx, uint32 nelements) { VEC_TYPE *vec = MemoryContextAlloc(ctx, sizeof(*vec)); VEC_INIT(vec, ctx, nelements); return vec; } /* free the underlying array */ VEC_SCOPE void VEC_FREE_DATA(VEC_TYPE *vec) { if (vec->data != NULL) pfree(vec->data); /* zero out all the vec data except the memory context so it can be reused */ *vec = (VEC_TYPE){ .ctx = vec->ctx, }; } /* free an allocated vector, and its data */ VEC_SCOPE void VEC_FREE(VEC_TYPE *vec) { if (vec == NULL) return; VEC_FREE_DATA(vec); pfree(vec); } /* clear a vector, but don't free the underlying array */ VEC_SCOPE void VEC_CLEAR(VEC_TYPE *vec) { vec->num_elements = 0; } VEC_SCOPE VEC_ELEMENT_TYPE * VEC_AT(VEC_TYPE *vec, uint32 index) { Assert(index < vec->num_elements); return &vec->data[index]; } VEC_SCOPE const VEC_ELEMENT_TYPE * VEC_GET(const VEC_TYPE *vec, uint32 index) { Assert(index < vec->num_elements); return &vec->data[index]; } /* return a pointer to the last element in the vector */ VEC_SCOPE VEC_ELEMENT_TYPE * VEC_LAST(VEC_TYPE *vec) { Assert(vec->num_elements > 0); return VEC_AT(vec, vec->num_elements - 1); } VEC_SCOPE VEC_ELEMENT_TYPE * VEC_APPEND_ARRAY(VEC_TYPE *vec, VEC_ELEMENT_TYPE *elements, uint32 num_elements) { VEC_ELEMENT_TYPE *first_new_element; VEC_RESERVE(vec, num_elements); Assert(vec->num_elements < vec->max_elements); first_new_element = vec->data + vec->num_elements; memcpy(first_new_element, elements, sizeof(*elements) * num_elements); vec->num_elements += num_elements; return first_new_element; } VEC_SCOPE void VEC_APPEND(VEC_TYPE *vec, VEC_ELEMENT_TYPE element) { VEC_APPEND_ARRAY(vec, &element, 1); } VEC_SCOPE VEC_ELEMENT_TYPE * VEC_APPEND_ZEROS(VEC_TYPE *vec, uint32 num_elements) { VEC_ELEMENT_TYPE *first_new_element; VEC_RESERVE(vec, num_elements); Assert(vec->num_elements + num_elements <= vec->max_elements); first_new_element = vec->data + vec->num_elements; memset(first_new_element, 0, sizeof(*first_new_element) * num_elements); vec->num_elements += num_elements; return first_new_element; } VEC_SCOPE void VEC_DELETE_RANGE(VEC_TYPE *vec, uint32 start, uint32 len) { if (start > vec->num_elements) elog(ERROR, "trying to delete starting past the end of a vector"); if (start + (uint64) len > (uint64) vec->num_elements) elog(ERROR, "trying to delete past the end of a vector"); if (start + (uint64) len < (uint64) vec->num_elements) { /* backshift the elements after the deletion that still remain */ uint64 backshit_elements = vec->num_elements - (start + len); uint64 backshit_bytes = backshit_elements * sizeof(*vec->data); memmove(&vec->data[start], &vec->data[start + len], backshit_bytes); } vec->num_elements -= len; } VEC_SCOPE void VEC_DELETE(VEC_TYPE *vec, uint32 index) { VEC_DELETE_RANGE(vec, index, 1); } #endif /* undefine external parameters, so next vector can be defined */ #undef VEC_PREFIX #undef VEC_ELEMENT_TYPE #undef VEC_SCOPE #undef VEC_DECLARE #undef VEC_DEFINE /* undefine locally declared macros */ #undef VEC_MAKE_PREFIX #undef VEC_MAKE_NAME #undef VEC_MAKE_NAME /* types */ #undef VEC_TYPE /* external function names */ #undef VEC_INIT #undef VEC_CREATE #undef VEC_FREE_DATA #undef VEC_CLEAR #undef VEC_AT #undef VEC_GET #undef VEC_LAST #undef VEC_APPEND #undef VEC_APPEND_ARRAY #undef VEC_APPEND_ZEROS #undef VEC_DELETE #undef VEC_DELETE_RANGE #undef VEC_RESERVE #undef VEC_FREE ================================================ FILE: src/agg_bookend.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/htup_details.h> #include <catalog/namespace.h> #include <catalog/pg_type.h> #include <fmgr.h> #include <lib/stringinfo.h> #include <libpq/pqformat.h> #include <nodes/value.h> #include <utils/datum.h> #include <utils/lsyscache.h> #include <utils/syscache.h> #include "export.h" /* bookend aggregates first and last: * first(value, cmp) returns the value for the row with the smallest cmp element. * last(value, cmp) returns the value for the row with the biggest cmp element. * * Usage: * SELECT first(metric, time), last(metric, time) FROM metric GROUP BY hostname. */ TS_FUNCTION_INFO_V1(ts_first_sfunc); TS_FUNCTION_INFO_V1(ts_first_combinefunc); TS_FUNCTION_INFO_V1(ts_last_sfunc); TS_FUNCTION_INFO_V1(ts_last_combinefunc); TS_FUNCTION_INFO_V1(ts_bookend_finalfunc); TS_FUNCTION_INFO_V1(ts_bookend_serializefunc); TS_FUNCTION_INFO_V1(ts_bookend_deserializefunc); /* A PolyDatum represents a polymorphic datum */ typedef struct PolyDatum { bool is_null; Datum datum; } PolyDatum; typedef struct TypeInfoCache { Oid typoid; int16 typlen; bool typbyval; } TypeInfoCache; /* PolyDatumIOState is internal state used by polydatum_serialize and polydatum_deserialize */ typedef struct PolyDatumIOState { TypeInfoCache type; FmgrInfo proc; Oid typeioparam; } PolyDatumIOState; static PolyDatum polydatum_from_arg(int argno, FunctionCallInfo fcinfo) { PolyDatum value; value.is_null = PG_ARGISNULL(argno); if (!value.is_null) value.datum = PG_GETARG_DATUM(argno); else value.datum = PointerGetDatum(NULL); return value; } /* Serialize type as namespace name string + type name string. * Don't simple send Oid since this state may be needed across pg_dumps. */ static void polydatum_serialize_type(StringInfo buf, Oid type_oid) { HeapTuple tup; Form_pg_type type_tuple; char *namespace_name; tup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(type_oid)); if (!HeapTupleIsValid(tup)) elog(ERROR, "cache lookup failed for type %u", type_oid); type_tuple = (Form_pg_type) GETSTRUCT(tup); namespace_name = get_namespace_name(type_tuple->typnamespace); /* send qualified type name */ pq_sendstring(buf, namespace_name); pq_sendstring(buf, NameStr(type_tuple->typname)); ReleaseSysCache(tup); } /* serializes the polydatum pd unto buf */ static void polydatum_serialize(PolyDatum *pd, StringInfo buf, PolyDatumIOState *state, FunctionCallInfo fcinfo) { bytea *outputbytes; Assert(OidIsValid(state->type.typoid)); polydatum_serialize_type(buf, state->type.typoid); if (pd->is_null) { /* emit -1 data length to signify a NULL */ pq_sendint32(buf, -1); return; } outputbytes = SendFunctionCall(&state->proc, pd->datum); pq_sendint32(buf, VARSIZE(outputbytes) - VARHDRSZ); pq_sendbytes(buf, VARDATA(outputbytes), VARSIZE(outputbytes) - VARHDRSZ); } static Oid polydatum_deserialize_type(StringInfo buf) { const char *schema_name = pq_getmsgstring(buf); const char *type_name = pq_getmsgstring(buf); Oid schema_oid = LookupExplicitNamespace(schema_name, false); Oid type_oid = GetSysCacheOid2(TYPENAMENSP, Anum_pg_type_oid, PointerGetDatum(type_name), ObjectIdGetDatum(schema_oid)); if (!OidIsValid(type_oid)) elog(ERROR, "cache lookup failed for type %s.%s", schema_name, type_name); return type_oid; } /* * Deserialize the PolyDatum where the binary representation is in buf. * If a not-null PolyDatum is passed in, fill in it's fields, otherwise palloc. * */ static PolyDatum * polydatum_deserialize(MemoryContext mem_ctx, PolyDatum *result, StringInfo buf, PolyDatumIOState *state, FunctionCallInfo fcinfo) { int itemlen; StringInfoData item_buf; StringInfo bufptr; char csave; Assert(result != NULL); MemoryContext old_context = MemoryContextSwitchTo(mem_ctx); Oid deserialized_type = polydatum_deserialize_type(buf); /* Following is copied/adapted from record_recv in core postgres */ /* Get and check the item length */ itemlen = pq_getmsgint(buf, 4); if (itemlen < -1 || itemlen > (buf->len - buf->cursor)) ereport(ERROR, (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION), errmsg("insufficient data left in message %d %d", itemlen, buf->len))); if (itemlen == -1) { /* -1 length means NULL */ result->is_null = true; bufptr = NULL; csave = 0; } else { /* * Rather than copying data around, we just set up a phony StringInfo * pointing to the correct portion of the input buffer. We assume we * can scribble on the input buffer so as to maintain the convention * that StringInfos have a trailing null. */ item_buf.data = &buf->data[buf->cursor]; item_buf.maxlen = itemlen + 1; item_buf.len = itemlen; item_buf.cursor = 0; buf->cursor += itemlen; csave = buf->data[buf->cursor]; buf->data[buf->cursor] = '\0'; bufptr = &item_buf; result->is_null = false; } /* Now call the column's receiveproc */ if (state->type.typoid != deserialized_type) { Assert(!OidIsValid(state->type.typoid)); Oid func; getTypeBinaryInputInfo(deserialized_type, &func, &state->typeioparam); fmgr_info_cxt(func, &state->proc, fcinfo->flinfo->fn_mcxt); state->type.typoid = deserialized_type; get_typlenbyval(state->type.typoid, &state->type.typlen, &state->type.typbyval); } result->datum = ReceiveFunctionCall(&state->proc, bufptr, state->typeioparam, -1); if (bufptr) { /* Trouble if it didn't eat the whole buffer */ if (item_buf.cursor != itemlen) ereport(ERROR, (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION), errmsg("improper binary format in polydata"))); buf->data[buf->cursor] = csave; } MemoryContextSwitchTo(old_context); return result; } typedef struct TransCache { TypeInfoCache value_type_cache; TypeInfoCache cmp_type_cache; FmgrInfo cmp_proc; } TransCache; /* Internal state for bookend aggregates */ typedef struct InternalCmpAggStore { TransCache aggstate_type_cache; PolyDatum value; PolyDatum cmp; /* the comparison element. e.g. time */ } InternalCmpAggStore; inline static InternalCmpAggStore * init_store(MemoryContext aggcontext) { InternalCmpAggStore *state = (InternalCmpAggStore *) MemoryContextAllocZero(aggcontext, sizeof(InternalCmpAggStore)); state->value.is_null = true; state->cmp.is_null = true; return state; } /* State used to cache data for serialize/deserialize operations */ typedef struct InternalCmpAggStoreIOState { PolyDatumIOState value; PolyDatumIOState cmp; /* the comparison element. e.g. time */ } InternalCmpAggStoreIOState; inline static void typeinfocache_polydatumcopy(TypeInfoCache *tic, PolyDatum input, PolyDatum *output) { Assert(OidIsValid(tic->typoid)); if (!tic->typbyval && !output->is_null) { pfree(DatumGetPointer(output->datum)); } *output = input; if (!input.is_null) { output->datum = datumCopy(input.datum, tic->typbyval, tic->typlen); output->is_null = false; } else { output->datum = PointerGetDatum(NULL); output->is_null = true; } } inline static void cmpproc_init(FunctionCallInfo fcinfo, FmgrInfo *cmp_proc, Oid type_oid, char *opname) { Oid cmp_op, cmp_regproc; if (!OidIsValid(type_oid)) elog(ERROR, "could not determine the type of the comparison_element"); cmp_op = OpernameGetOprid(list_make1(makeString(opname)), type_oid, type_oid); if (!OidIsValid(cmp_op)) elog(ERROR, "could not find a %s operator for type %d", opname, type_oid); cmp_regproc = get_opcode(cmp_op); if (!OidIsValid(cmp_regproc)) elog(ERROR, "could not find the procedure for the %s operator for type %d", opname, type_oid); fmgr_info_cxt(cmp_regproc, cmp_proc, fcinfo->flinfo->fn_mcxt); } inline static bool cmpproc_cmp(FmgrInfo *cmp_proc, FunctionCallInfo fcinfo, PolyDatum left, PolyDatum right) { return DatumGetBool(FunctionCall2Coll(cmp_proc, fcinfo->fncollation, left.datum, right.datum)); } /* * bookend_sfunc - internal function called by ts_last_sfunc and ts_first_sfunc; */ static inline Datum bookend_sfunc(MemoryContext aggcontext, InternalCmpAggStore *state, char *opname, FunctionCallInfo fcinfo) { PolyDatum value = polydatum_from_arg(1, fcinfo); PolyDatum cmp = polydatum_from_arg(2, fcinfo); MemoryContext old_context; old_context = MemoryContextSwitchTo(aggcontext); if (state == NULL) { state = init_store(aggcontext); TransCache *cache = &state->aggstate_type_cache; TypeInfoCache *v = &cache->value_type_cache; v->typoid = get_fn_expr_argtype(fcinfo->flinfo, 1); get_typlenbyval(v->typoid, &v->typlen, &v->typbyval); TypeInfoCache *c = &cache->cmp_type_cache; c->typoid = get_fn_expr_argtype(fcinfo->flinfo, 2); get_typlenbyval(c->typoid, &c->typlen, &c->typbyval); typeinfocache_polydatumcopy(&cache->value_type_cache, value, &state->value); typeinfocache_polydatumcopy(&cache->cmp_type_cache, cmp, &state->cmp); } else if (!cmp.is_null) { TransCache *cache = &state->aggstate_type_cache; if (cache->cmp_proc.fn_addr == NULL) { cmpproc_init(fcinfo, &cache->cmp_proc, cache->cmp_type_cache.typoid, opname); } /* only do comparison if cmp is not NULL */ if (state->cmp.is_null || cmpproc_cmp(&cache->cmp_proc, fcinfo, cmp, state->cmp)) { typeinfocache_polydatumcopy(&cache->value_type_cache, value, &state->value); typeinfocache_polydatumcopy(&cache->cmp_type_cache, cmp, &state->cmp); } } MemoryContextSwitchTo(old_context); PG_RETURN_POINTER(state); } /* bookend_combinefunc - internal function called by ts_last_combinefunc and ts_first_combinefunc; * fmgr args are: (internal internal_state, internal2 internal_state) */ static inline Datum bookend_combinefunc(MemoryContext aggcontext, InternalCmpAggStore *state1, InternalCmpAggStore *state2, char *opname, FunctionCallInfo fcinfo) { MemoryContext old_context; if (state2 == NULL) PG_RETURN_POINTER(state1); /* * manually copy all fields from state2 to state1, as per other combine * func like int8_avg_combine */ if (state1 == NULL) { old_context = MemoryContextSwitchTo(aggcontext); state1 = init_store(aggcontext); Assert(OidIsValid(state2->aggstate_type_cache.value_type_cache.typoid)); Assert(OidIsValid(state2->aggstate_type_cache.cmp_type_cache.typoid)); TransCache *cache1 = &state1->aggstate_type_cache; TransCache *cache2 = &state2->aggstate_type_cache; /* * Initialize the type information from the right-hand state. Note that * we will have to re-lookup the comparison procedure on demand, because * the comparison procedure from the right-hand state might have been * allocated in a different memory context. */ cache1->value_type_cache = cache2->value_type_cache; cache1->cmp_type_cache = cache2->cmp_type_cache; typeinfocache_polydatumcopy(&cache1->value_type_cache, state2->value, &state1->value); typeinfocache_polydatumcopy(&cache1->cmp_type_cache, state2->cmp, &state1->cmp); MemoryContextSwitchTo(old_context); PG_RETURN_POINTER(state1); } if (state1->cmp.is_null && state2->cmp.is_null) { PG_RETURN_POINTER(state1); } else if (state1->cmp.is_null != state2->cmp.is_null) { if (state1->cmp.is_null) PG_RETURN_POINTER(state2); else PG_RETURN_POINTER(state1); } TransCache *cache1 = &state1->aggstate_type_cache; if (cache1->cmp_proc.fn_addr == NULL) { cmpproc_init(fcinfo, &cache1->cmp_proc, cache1->cmp_type_cache.typoid, opname); } if (cmpproc_cmp(&cache1->cmp_proc, fcinfo, state2->cmp, state1->cmp)) { old_context = MemoryContextSwitchTo(aggcontext); typeinfocache_polydatumcopy(&cache1->value_type_cache, state2->value, &state1->value); typeinfocache_polydatumcopy(&cache1->cmp_type_cache, state2->cmp, &state1->cmp); MemoryContextSwitchTo(old_context); } PG_RETURN_POINTER(state1); } /* first(internal internal_state, anyelement value, "any" comparison_element) */ Datum ts_first_sfunc(PG_FUNCTION_ARGS) { InternalCmpAggStore *store = PG_ARGISNULL(0) ? NULL : (InternalCmpAggStore *) PG_GETARG_POINTER(0); MemoryContext aggcontext; if (!AggCheckCallContext(fcinfo, &aggcontext)) { /* cannot be called directly because of internal-type argument */ elog(ERROR, "first_sfun called in non-aggregate context"); } return bookend_sfunc(aggcontext, store, "<", fcinfo); } /* last(internal internal_state, anyelement value, "any" comparison_element) */ Datum ts_last_sfunc(PG_FUNCTION_ARGS) { InternalCmpAggStore *store = PG_ARGISNULL(0) ? NULL : (InternalCmpAggStore *) PG_GETARG_POINTER(0); MemoryContext aggcontext; if (!AggCheckCallContext(fcinfo, &aggcontext)) { /* cannot be called directly because of internal-type argument */ elog(ERROR, "last_sfun called in non-aggregate context"); } return bookend_sfunc(aggcontext, store, ">", fcinfo); } /* first_combinerfunc(internal, internal) => internal */ Datum ts_first_combinefunc(PG_FUNCTION_ARGS) { MemoryContext aggcontext; InternalCmpAggStore *state1 = PG_ARGISNULL(0) ? NULL : (InternalCmpAggStore *) PG_GETARG_POINTER(0); InternalCmpAggStore *state2 = PG_ARGISNULL(1) ? NULL : (InternalCmpAggStore *) PG_GETARG_POINTER(1); if (!AggCheckCallContext(fcinfo, &aggcontext)) { /* cannot be called directly because of internal-type argument */ elog(ERROR, "ts_first_combinefunc called in non-aggregate context"); } return bookend_combinefunc(aggcontext, state1, state2, "<", fcinfo); } /* last_combinerfunc(internal, internal) => internal */ Datum ts_last_combinefunc(PG_FUNCTION_ARGS) { MemoryContext aggcontext; InternalCmpAggStore *state1 = PG_ARGISNULL(0) ? NULL : (InternalCmpAggStore *) PG_GETARG_POINTER(0); InternalCmpAggStore *state2 = PG_ARGISNULL(1) ? NULL : (InternalCmpAggStore *) PG_GETARG_POINTER(1); if (!AggCheckCallContext(fcinfo, &aggcontext)) { /* cannot be called directly because of internal-type argument */ elog(ERROR, "ts_last_combinefunc called in non-aggregate context"); } return bookend_combinefunc(aggcontext, state1, state2, ">", fcinfo); } /* ts_bookend_serializefunc(internal) => bytea */ Datum ts_bookend_serializefunc(PG_FUNCTION_ARGS) { StringInfoData buf; InternalCmpAggStoreIOState *my_extra; InternalCmpAggStore *state; Assert(!PG_ARGISNULL(0)); state = (InternalCmpAggStore *) PG_GETARG_POINTER(0); my_extra = (InternalCmpAggStoreIOState *) fcinfo->flinfo->fn_extra; if (my_extra == NULL) { fcinfo->flinfo->fn_extra = MemoryContextAllocZero(fcinfo->flinfo->fn_mcxt, sizeof(InternalCmpAggStoreIOState)); my_extra = (InternalCmpAggStoreIOState *) fcinfo->flinfo->fn_extra; Oid func; bool is_varlena; my_extra->value.type = state->aggstate_type_cache.value_type_cache; Assert(OidIsValid(my_extra->value.type.typoid)); getTypeBinaryOutputInfo(my_extra->value.type.typoid, &func, &is_varlena); fmgr_info_cxt(func, &my_extra->value.proc, fcinfo->flinfo->fn_mcxt); my_extra->cmp.type = state->aggstate_type_cache.cmp_type_cache; Assert(OidIsValid(my_extra->cmp.type.typoid)); getTypeBinaryOutputInfo(my_extra->cmp.type.typoid, &func, &is_varlena); fmgr_info_cxt(func, &my_extra->cmp.proc, fcinfo->flinfo->fn_mcxt); } pq_begintypsend(&buf); polydatum_serialize(&state->value, &buf, &my_extra->value, fcinfo); polydatum_serialize(&state->cmp, &buf, &my_extra->cmp, fcinfo); PG_RETURN_BYTEA_P(pq_endtypsend(&buf)); } /* ts_bookend_deserializefunc(bytea, internal) => internal */ Datum ts_bookend_deserializefunc(PG_FUNCTION_ARGS) { MemoryContext aggcontext; bytea *sstate; StringInfoData buf; InternalCmpAggStore *result; InternalCmpAggStoreIOState *my_extra; if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "aggregate function called in non-aggregate context"); sstate = PG_GETARG_BYTEA_P(0); /* * Copy the bytea into a StringInfo so that we can "receive" it using the * standard recv-function infrastructure. */ initStringInfo(&buf); appendBinaryStringInfo(&buf, VARDATA(sstate), VARSIZE(sstate) - VARHDRSZ); my_extra = (InternalCmpAggStoreIOState *) fcinfo->flinfo->fn_extra; if (my_extra == NULL) { fcinfo->flinfo->fn_extra = MemoryContextAllocZero(fcinfo->flinfo->fn_mcxt, sizeof(InternalCmpAggStoreIOState)); my_extra = (InternalCmpAggStoreIOState *) fcinfo->flinfo->fn_extra; } result = MemoryContextAllocZero(aggcontext, sizeof(InternalCmpAggStore)); polydatum_deserialize(aggcontext, &result->value, &buf, &my_extra->value, fcinfo); polydatum_deserialize(aggcontext, &result->cmp, &buf, &my_extra->cmp, fcinfo); result->aggstate_type_cache.value_type_cache = my_extra->value.type; result->aggstate_type_cache.cmp_type_cache = my_extra->cmp.type; PG_RETURN_POINTER(result); } /* ts_bookend_finalfunc(internal, anyelement, "any") => anyelement */ Datum ts_bookend_finalfunc(PG_FUNCTION_ARGS) { InternalCmpAggStore *state; if (!AggCheckCallContext(fcinfo, NULL)) { /* cannot be called directly because of internal-type argument */ elog(ERROR, "ts_bookend_finalfunc called in non-aggregate context"); } state = PG_ARGISNULL(0) ? NULL : (InternalCmpAggStore *) PG_GETARG_POINTER(0); if (state == NULL || state->value.is_null || state->cmp.is_null) PG_RETURN_NULL(); PG_RETURN_DATUM(state->value.datum); } ================================================ FILE: src/annotations.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once /* Supported since clang 12 and GCC 7 */ #if defined __has_attribute #if __has_attribute(fallthrough) #define TS_FALLTHROUGH __attribute__((fallthrough)) #else #define TS_FALLTHROUGH /* FALLTHROUGH */ #endif #else #define TS_FALLTHROUGH /* FALLTHROUGH */ #endif #ifdef __has_attribute #if __has_attribute(used) #define TS_USED __attribute__((used)) #else #define TS_USED #endif #else #define TS_USED #endif ================================================ FILE: src/bgw/CMakeLists.txt ================================================ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/job.c ${CMAKE_CURRENT_SOURCE_DIR}/job_stat.c ${CMAKE_CURRENT_SOURCE_DIR}/job_stat_history.c ${CMAKE_CURRENT_SOURCE_DIR}/launcher_interface.c ${CMAKE_CURRENT_SOURCE_DIR}/scheduler.c ${CMAKE_CURRENT_SOURCE_DIR}/timer.c) target_sources(${PROJECT_NAME} PRIVATE ${SOURCES}) ================================================ FILE: src/bgw/README.md ================================================ # Background worker jobs TimescaleDB needs to run multiple background jobs. This module implements a simple scheduler so that jobs inserted into a jobs table can be run on a schedule. Each database in an instance runs it's own scheduler because different databases may run different TimescaleDB extension versions which may require different scheduler logic. ## Schedules The scheduler allows you to set a `schedule_interval` for every job. That defines the interval the scheduler will wait after a job finishes to start it again, if the job is successful. If the job fails, the scheduler uses `retry_period` in an exponential backoff to decide when to run the job again. ## Design The scheduler itself is a background job that continuously runs and waits for a time when jobs need to be scheduled. It then launches jobs as new background workers that it controls through the background worker handle. Aggregate statistics about a job are kept in the job stat catalog table. These statistics include the start and finish times of the last run of the job as well as whether or not the job succeeded. The `next_start` is used to figure out when next to run a job after a scheduler is restarted. The statistics table also tracks consecutive failures and crashes for the job which are used for calculating the exponential backoff after a crash or failure (which is used to set the `next_start` after the crash/failure). Note also that there is a minimum time after the database scheduler starts up and a crashed job is restarted. This is to allow the operator enough time to disable the job if needed. Note that the number of crashes is an overestimate of the actual number of crashes for a job. This is so that we are conservative and never miss a crash and fail to use the appropriate backoff logic. There is some complexity in ensuring that all crashes are counted. A crash in Postgres causes *all* processes to quit immediately therefore we cannot write anything to the database once any process has crashed. Thus, we must be able to deduce that a crash occurred from a commit that happened before any crash. We accomplish this by committing a changes to the stats table before a job starts and undoing the change after it finishes. If a job crashed, it will be left in an intermediate state from which we deduce that it could have been the crashing process. ## Scheduler State Machine The scheduler implements a state machine for each job. Each job starts in the `SCHEDULED` state. As soon as a job starts it enters the `STARTING` state. If the scheduler determines the job should be terminated (e.g. it has reached a timeout), it moves the job to a TERMINATING state. Once a background worker has for a job has stopped, the job returns to the `SCHEDULED` state. The states and associated transitions are as follows. ```ditaa +---------+ +--------+ +---> |SCHEDULED+-------> |DISABLED| | +----+----+ +--------+ | | | | | v | +---+----+ +<-----+STARTING| | +---+----+ | | | | | v | +---+-------+ +<-----+TERMINATING| +-----------+ ``` ## Limitations This first implementation has two limitations: - The list of jobs to be run is read from the database when the scheduler is first started. We do not update this list if the jobs table changes. - There is no prioritization for when to run jobs. ================================================ FILE: src/bgw/job.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <unistd.h> #include <access/xact.h> #include <catalog/pg_authid.h> #include <executor/execdebug.h> #include <executor/instrument.h> #include <miscadmin.h> #include <nodes/makefuncs.h> #include <parser/parse_func.h> #include <parser/parser.h> #include <pgstat.h> #include <postmaster/bgworker.h> #include <storage/ipc.h> #include <storage/lock.h> #include <tcop/tcopprot.h> #include <utils/acl.h> #include <utils/backend_status.h> #include <utils/builtins.h> #include <utils/elog.h> #include <utils/jsonb.h> #include <utils/lsyscache.h> #include <utils/memutils.h> #include <utils/snapmgr.h> #include <utils/syscache.h> #include <utils/timestamp.h> #include "compat/compat.h" #include "bgw/job_stat_history.h" #include "bgw/scheduler.h" #include "bgw_policy/chunk_stats.h" #include "bgw_policy/policy.h" #include "config.h" #include "cross_module_fn.h" #include "debug_assert.h" #include "debug_point.h" #include "extension.h" #include "job.h" #include "job_stat.h" #include "license_guc.h" #include "scan_iterator.h" #include "scanner.h" #include "tss_callbacks.h" #include "utils.h" #ifdef USE_TELEMETRY #include "telemetry/telemetry.h" #endif static scheduler_test_hook_type scheduler_test_hook = NULL; static char *job_entrypoint_function_name = "ts_bgw_job_entrypoint"; /* * Get the mem_guard callbacks. * * You might get a NULL pointer back if there are no mem_guard installed, so * check before using. */ MGCallbacks * ts_get_mem_guard_callbacks(void) { static MGCallbacks **mem_guard_callback_ptr = NULL; if (mem_guard_callback_ptr) return *mem_guard_callback_ptr; mem_guard_callback_ptr = (MGCallbacks **) find_rendezvous_variable(MG_CALLBACKS_VAR_NAME); return *mem_guard_callback_ptr; } BackgroundWorkerHandle * ts_bgw_job_start(BgwJob *job, Oid user_oid) { BgwParams bgw_params = { .job_id = Int32GetDatum(job->fd.id), .job_history_id = job->job_history.id, .job_history_execution_start = job->job_history.execution_start, .user_oid = user_oid, }; strlcpy(bgw_params.bgw_main, job_entrypoint_function_name, sizeof(bgw_params.bgw_main)); return ts_bgw_start_worker(NameStr(job->fd.application_name), &bgw_params); } static void job_execute_function(FuncExpr *funcexpr) { bool isnull; EState *estate = CreateExecutorState(); ExprContext *econtext = CreateExprContext(estate); ExprState *es = ExecPrepareExpr((Expr *) funcexpr, estate); ExecEvalExpr(es, econtext, &isnull); FreeExprContext(econtext, true); FreeExecutorState(estate); } /** * Run configuration check validation function. * * This will run the configuration check validation function registered for * the job. If a new job is added, the job_id is going to be zero. */ void ts_bgw_job_run_config_check(Oid check, int32 job_id, Jsonb *config) { /* Nothing to check if there is no check function provided */ if (!OidIsValid(check)) return; /* NULL config may be valid */ Const *arg; if (config == NULL) arg = makeNullConst(JSONBOID, -1, InvalidOid); else arg = makeConst(JSONBOID, -1, InvalidOid, -1, JsonbPGetDatum(config), false, false); List *args = list_make1(arg); FuncExpr *funcexpr = makeFuncExpr(check, VOIDOID, args, InvalidOid, InvalidOid, COERCE_EXPLICIT_CALL); if (get_func_prokind(check) == PROKIND_FUNCTION) job_execute_function(funcexpr); else ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported function type"), errdetail("Only functions are allowed as custom configuration checks"), errhint("Use a FUNCTION instead"))); } /* Run the check function on a configuration. It will generate errors if there * is anything wrong with the configuration, otherwise just return. If the * check function does not exist, no checking will be done.*/ static void job_config_check(BgwJob *job, Jsonb *config) { Oid proc; List *funcname; /* Both should either be empty or contain a schema and name */ Assert((strlen(NameStr(job->fd.check_schema)) == 0) == (strlen(NameStr(job->fd.check_schema)) == 0)); /* If there is no function, just return */ if (strlen(NameStr(job->fd.check_name)) == 0) return; funcname = list_make2(makeString(NameStr(job->fd.check_schema)), makeString(NameStr(job->fd.check_name))); Oid argtypes[] = { JSONBOID }; /* Only functions allowed as custom checks, as procedures can cause errors with COMMIT * statements */ proc = LookupFuncName(funcname, 1, argtypes, true); /* a check function has been registered but it can't be found anymore because it was dropped or renamed. Allow alter_job to run if that's the case without validating the config but also print a warning */ if (OidIsValid(proc)) ts_bgw_job_run_config_check(proc, job->fd.id, config); else elog(WARNING, "function %s.%s(config jsonb) not found, skipping config validation for " "job %d", NameStr(job->fd.check_schema), NameStr(job->fd.check_name), job->fd.id); } static BgwJob * bgw_job_from_tupleinfo(TupleInfo *ti, size_t alloc_size) { BgwJob *job; bool should_free; HeapTuple tuple; MemoryContext old_ctx; Datum values[Natts_bgw_job] = { 0 }; bool nulls[Natts_bgw_job] = { false }; /* * allow for embedding with arbitrary alloc_size, which means we can't use * the STRUCT_FROM_TUPLE macro */ Assert(alloc_size >= sizeof(BgwJob)); job = MemoryContextAllocZero(ti->mctx, alloc_size); tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); old_ctx = MemoryContextSwitchTo(ti->mctx); /* * Using heap_deform_tuple instead of GETSTRUCT since the tuple can * contain NULL values. Some of these cannot really be null, but we check * anyway since it is cheap and will avoid problems in the future. Note * that the job structure is zeroed, so we only need to update the field * if it is non-NULL. */ heap_deform_tuple(tuple, ts_scanner_get_tupledesc(ti), values, nulls); if (!nulls[AttrNumberGetAttrOffset(Anum_bgw_job_id)]) job->fd.id = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_bgw_job_id)]); if (!nulls[AttrNumberGetAttrOffset(Anum_bgw_job_application_name)]) namestrcpy(&job->fd.application_name, DatumGetCString(values[AttrNumberGetAttrOffset(Anum_bgw_job_application_name)])); if (!nulls[AttrNumberGetAttrOffset(Anum_bgw_job_schedule_interval)]) job->fd.schedule_interval = *DatumGetIntervalP(values[AttrNumberGetAttrOffset(Anum_bgw_job_schedule_interval)]); if (!nulls[AttrNumberGetAttrOffset(Anum_bgw_job_max_runtime)]) job->fd.max_runtime = *DatumGetIntervalP(values[AttrNumberGetAttrOffset(Anum_bgw_job_max_runtime)]); if (!nulls[AttrNumberGetAttrOffset(Anum_bgw_job_max_retries)]) job->fd.max_retries = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_bgw_job_max_retries)]); if (!nulls[AttrNumberGetAttrOffset(Anum_bgw_job_fixed_schedule)]) job->fd.fixed_schedule = DatumGetBool(values[AttrNumberGetAttrOffset(Anum_bgw_job_fixed_schedule)]); if (!nulls[AttrNumberGetAttrOffset(Anum_bgw_job_initial_start)]) { job->fd.initial_start = DatumGetTimestampTz(values[AttrNumberGetAttrOffset(Anum_bgw_job_initial_start)]); } else job->fd.initial_start = DT_NOBEGIN; if (!nulls[AttrNumberGetAttrOffset(Anum_bgw_job_timezone)]) job->fd.timezone = DatumGetTextPCopy(values[AttrNumberGetAttrOffset(Anum_bgw_job_timezone)]); if (!nulls[AttrNumberGetAttrOffset(Anum_bgw_job_retry_period)]) job->fd.retry_period = *DatumGetIntervalP(values[AttrNumberGetAttrOffset(Anum_bgw_job_retry_period)]); if (!nulls[AttrNumberGetAttrOffset(Anum_bgw_job_proc_schema)]) namestrcpy(&job->fd.proc_schema, DatumGetCString(values[AttrNumberGetAttrOffset(Anum_bgw_job_proc_schema)])); if (!nulls[AttrNumberGetAttrOffset(Anum_bgw_job_proc_name)]) namestrcpy(&job->fd.proc_name, DatumGetCString(values[AttrNumberGetAttrOffset(Anum_bgw_job_proc_name)])); if (!nulls[AttrNumberGetAttrOffset(Anum_bgw_job_check_schema)]) namestrcpy(&job->fd.check_schema, DatumGetCString(values[AttrNumberGetAttrOffset(Anum_bgw_job_check_schema)])); if (!nulls[AttrNumberGetAttrOffset(Anum_bgw_job_check_name)]) namestrcpy(&job->fd.check_name, DatumGetCString(values[AttrNumberGetAttrOffset(Anum_bgw_job_check_name)])); if (!nulls[AttrNumberGetAttrOffset(Anum_bgw_job_owner)]) job->fd.owner = DatumGetObjectId(values[AttrNumberGetAttrOffset(Anum_bgw_job_owner)]); if (!nulls[AttrNumberGetAttrOffset(Anum_bgw_job_scheduled)]) job->fd.scheduled = DatumGetBool(values[AttrNumberGetAttrOffset(Anum_bgw_job_scheduled)]); if (!nulls[AttrNumberGetAttrOffset(Anum_bgw_job_hypertable_id)]) job->fd.hypertable_id = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_bgw_job_hypertable_id)]); if (!nulls[AttrNumberGetAttrOffset(Anum_bgw_job_config)]) job->fd.config = DatumGetJsonbPCopy(values[AttrNumberGetAttrOffset(Anum_bgw_job_config)]); MemoryContextSwitchTo(old_ctx); if (should_free) heap_freetuple(tuple); return job; } typedef struct AccumData { List *list; size_t alloc_size; } AccumData; static ScanTupleResult bgw_job_accum_tuple_found(TupleInfo *ti, void *data) { AccumData *list_data = data; BgwJob *job = bgw_job_from_tupleinfo(ti, list_data->alloc_size); MemoryContext orig = MemoryContextSwitchTo(ti->mctx); list_data->list = lappend(list_data->list, job); MemoryContextSwitchTo(orig); return SCAN_CONTINUE; } static ScanFilterResult bgw_job_filter_scheduled(const TupleInfo *ti, void *data) { bool isnull; Datum scheduled = slot_getattr(ti->slot, Anum_bgw_job_scheduled, &isnull); Ensure(!isnull, "scheduled column was null"); return DatumGetBool(scheduled); } /* This function is meant to be used by the scheduler only * it does not include the config field which saves us from * detoasting and makes memory management in the scheduler * simpler as otherwise the config field would have to be * freed separately when freeing jobs which would prevent * the use of list_free_deep. * The scheduler does not need the config field only the * individual jobs do. * The scheduler requires jobs to be sorted by id * which is guaranteed by the index scan on the primary key */ List * ts_bgw_job_get_scheduled(size_t alloc_size, MemoryContext mctx) { MemoryContext old_ctx; List *jobs = NIL; ScanIterator iterator = ts_scan_iterator_create_with_catalog_snapshot(BGW_JOB, AccessShareLock, mctx); iterator.ctx.index = catalog_get_index(ts_catalog_get(), BGW_JOB, BGW_JOB_PKEY_IDX); iterator.ctx.filter = bgw_job_filter_scheduled; ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); bool should_free, isnull, initial_start_isnull, timezone_isnull; Datum value, initial_start, timezone; BgwJob *job = MemoryContextAllocZero(mctx, alloc_size); HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); /* * Note that the nullable columns might have variable width, so we * handle them below. We can only use memcpy for the non-nullable fixed * width starting part of the BgwJob struct. */ memcpy(job, GETSTRUCT(tuple), offsetof(FormData_bgw_job, initial_start)); if (should_free) heap_freetuple(tuple); #ifdef USE_TELEMETRY /* ignore telemetry jobs if telemetry is disabled */ if (!ts_telemetry_on() && ts_is_telemetry_job(job)) { pfree(job); continue; } #endif /* handle NULL columns */ initial_start = slot_getattr(ti->slot, Anum_bgw_job_initial_start, &initial_start_isnull); if (!initial_start_isnull) job->fd.initial_start = DatumGetTimestampTz(initial_start); else job->fd.initial_start = DT_NOBEGIN; value = slot_getattr(ti->slot, Anum_bgw_job_hypertable_id, &isnull); job->fd.hypertable_id = isnull ? 0 : DatumGetInt32(value); /* We skip config, check_name, and check_schema since the scheduler * doesn't need these, it saves us from detoasting, and simplifies * freeing job lists in the scheduler as otherwise the config field * would have to be freed separately when freeing a job. */ job->fd.config = NULL; old_ctx = MemoryContextSwitchTo(mctx); timezone = slot_getattr(ti->slot, Anum_bgw_job_timezone, &timezone_isnull); if (!timezone_isnull) { /* We use DatumGetTextPCopy to move the detoasted value into our memory context */ job->fd.timezone = DatumGetTextPCopy(timezone); } else { job->fd.timezone = NULL; } jobs = lappend(jobs, job); MemoryContextSwitchTo(old_ctx); } return jobs; } List * ts_bgw_job_get_all(size_t alloc_size, MemoryContext mctx) { Catalog *catalog = ts_catalog_get(); AccumData list_data = { .list = NIL, .alloc_size = sizeof(BgwJob), }; ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, BGW_JOB), .data = &list_data, .tuple_found = bgw_job_accum_tuple_found, .lockmode = AccessShareLock, .result_mctx = mctx, .scandirection = ForwardScanDirection, .use_catalog_snapshot = true, }; ts_scanner_scan(&scanctx); return list_data.list; } static void init_scan_by_proc_name(ScanKeyData *scankey, const char *proc_name) { ScanKeyInit(scankey, Anum_bgw_job_proc_hypertable_id_idx_proc_name, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(proc_name)); } static void init_scan_by_proc_schema(ScanKeyData *scankey, const char *proc_schema) { ScanKeyInit(scankey, Anum_bgw_job_proc_hypertable_id_idx_proc_schema, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(proc_schema)); } static void init_scan_by_hypertable_id(ScanKeyData *scankey, int32 hypertable_id) { ScanKeyInit(scankey, Anum_bgw_job_proc_hypertable_id_idx_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); } List * ts_bgw_job_find_by_proc_and_hypertable_id(const char *proc_name, const char *proc_schema, int32 hypertable_id) { Catalog *catalog = ts_catalog_get(); ScanKeyData scankey[3]; AccumData list_data = { .list = NIL, .alloc_size = sizeof(BgwJob), }; ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, BGW_JOB), .index = catalog_get_index(ts_catalog_get(), BGW_JOB, BGW_JOB_PROC_HYPERTABLE_ID_IDX), .data = &list_data, .scankey = scankey, .nkeys = sizeof(scankey) / sizeof(*scankey), .tuple_found = bgw_job_accum_tuple_found, .lockmode = AccessShareLock, .scandirection = ForwardScanDirection, .use_catalog_snapshot = true, }; init_scan_by_proc_schema(&scankey[0], proc_schema); init_scan_by_proc_name(&scankey[1], proc_name); init_scan_by_hypertable_id(&scankey[2], hypertable_id); ts_scanner_scan(&scanctx); return list_data.list; } List * ts_bgw_job_find_by_hypertable_id(int32 hypertable_id) { Catalog *catalog = ts_catalog_get(); ScanKeyData scankey[1]; AccumData list_data = { .list = NIL, .alloc_size = sizeof(BgwJob), }; ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, BGW_JOB), .index = catalog_get_index(ts_catalog_get(), BGW_JOB, BGW_JOB_PROC_HYPERTABLE_ID_IDX), .data = &list_data, .scankey = scankey, .nkeys = sizeof(scankey) / sizeof(*scankey), .tuple_found = bgw_job_accum_tuple_found, .lockmode = AccessShareLock, .scandirection = ForwardScanDirection, .use_catalog_snapshot = true, }; init_scan_by_hypertable_id(&scankey[0], hypertable_id); ts_scanner_scan(&scanctx); return list_data.list; } static void init_scan_by_job_id(ScanIterator *iterator, int32 job_id) { iterator->ctx.index = catalog_get_index(ts_catalog_get(), BGW_JOB, BGW_JOB_PKEY_IDX); ts_scan_iterator_scan_key_init(iterator, Anum_bgw_job_pkey_idx_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(job_id)); } BgwJob * ts_bgw_job_find(int32 bgw_job_id, MemoryContext mctx, bool fail_if_not_found) { ScanIterator iterator = ts_scan_iterator_create_with_catalog_snapshot(BGW_JOB, AccessShareLock, mctx); int num_found = 0; BgwJob *job = NULL; init_scan_by_job_id(&iterator, bgw_job_id); ts_scanner_foreach(&iterator) { Assert(num_found == 0); job = bgw_job_from_tupleinfo(ts_scan_iterator_tuple_info(&iterator), sizeof(BgwJob)); num_found++; DEBUG_WAITPOINT("bgw_job_find_during_scan"); } if (num_found == 0 && fail_if_not_found) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("job %d not found", bgw_job_id))); return job; } static ScanTupleResult bgw_job_tuple_delete(TupleInfo *ti, void *data) { CatalogSecurityContext sec_ctx; bool isnull_job_id; Datum datum = slot_getattr(ti->slot, Anum_bgw_job_id, &isnull_job_id); int32 job_id = DatumGetInt32(datum); Ensure(!isnull_job_id, "job id was null"); /* Also delete the bgw_stat entry */ ts_bgw_job_stat_delete(job_id); /* Delete any stats in bgw_policy_chunk_stats related to this job */ ts_bgw_policy_chunk_stats_delete_row_only_by_job_id(job_id); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); ts_catalog_restore_user(&sec_ctx); return SCAN_CONTINUE; } static bool bgw_job_delete_scan(ScanKeyData *scankey, int32 job_id) { Catalog *catalog = ts_catalog_get(); ScannerCtx scanctx; scanctx = (ScannerCtx){ .table = catalog_get_table_id(catalog, BGW_JOB), .index = catalog_get_index(catalog, BGW_JOB, BGW_JOB_PKEY_IDX), .nkeys = 1, .scankey = scankey, .data = NULL, .limit = 1, .tuple_found = bgw_job_tuple_delete, .lockmode = RowExclusiveLock, .scandirection = ForwardScanDirection, .result_mctx = CurrentMemoryContext, .use_catalog_snapshot = true, }; return ts_scanner_scan(&scanctx); } /* * Cancel the background worker running the given job, identified by its * application_name. This is best-effort: if the worker is not found or * already exited, we simply do nothing. */ static void cancel_worker_for_job(const char *appname) { const int num_backends = pgstat_fetch_stat_numbackends(); for (int i = 1; i <= num_backends; i++) { const LocalPgBackendStatus *local_beentry = pgstat_get_local_beentry_by_index_compat(i); const PgBackendStatus *beentry = &local_beentry->backendStatus; if (beentry->st_databaseid == MyDatabaseId && beentry->st_appname[0] != '\0' && strcmp(beentry->st_appname, appname) == 0) { DirectFunctionCall1(pg_cancel_backend, Int32GetDatum(beentry->st_procpid)); break; } } } /* * Delete the job identified by `job_id`. If the job is currently running, * we send SIGINT to the worker for prompt cancellation. */ bool ts_bgw_job_delete_by_id(int32 job_id) { ScanKeyData scankey[1]; NameData appname = { .data = { 0 } }; bool result; /* Look up the job's application_name before deleting */ BgwJob *job = ts_bgw_job_find(job_id, CurrentMemoryContext, false); if (job != NULL) { namestrcpy(&appname, NameStr(job->fd.application_name)); pfree(job); } ScanKeyInit(&scankey[0], Anum_bgw_job_pkey_idx_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(job_id)); result = bgw_job_delete_scan(scankey, job_id); /* Send SIGINT to the running worker for prompt cancellation */ if (result && NameStr(appname)[0] != '\0') cancel_worker_for_job(NameStr(appname)); return result; } /* This function only updates the fields modifiable with alter_job. */ static ScanTupleResult bgw_job_tuple_update_by_id(TupleInfo *ti, void *const data) { BgwJob *updated_job = (BgwJob *) data; bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); HeapTuple new_tuple; Datum values[Natts_bgw_job] = { 0 }; bool isnull[Natts_bgw_job] = { 0 }; bool doReplace[Natts_bgw_job] = { 0 }; values[AttrNumberGetAttrOffset(Anum_bgw_job_application_name)] = NameGetDatum(&updated_job->fd.application_name); doReplace[AttrNumberGetAttrOffset(Anum_bgw_job_application_name)] = true; Datum old_schedule_interval = slot_getattr(ti->slot, Anum_bgw_job_schedule_interval, &isnull[0]); Assert(!isnull[0]); /* when we update the schedule interval, modify the next start time as well*/ if (!DatumGetBool(DirectFunctionCall2(interval_eq, old_schedule_interval, IntervalPGetDatum(&updated_job->fd.schedule_interval)))) { BgwJobStat *stat = ts_bgw_job_stat_find(updated_job->fd.id); if (stat != NULL) { TimestampTz next_start = DatumGetTimestampTz( DirectFunctionCall2(timestamptz_pl_interval, TimestampTzGetDatum(stat->fd.last_finish), IntervalPGetDatum(&updated_job->fd.schedule_interval))); /* allow DT_NOBEGIN for next_start here through allow_unset=true in the case that * last_finish is DT_NOBEGIN, * This means the value is counted as unset which is what we want */ ts_bgw_job_stat_update_next_start(updated_job->fd.id, next_start, true); } values[AttrNumberGetAttrOffset(Anum_bgw_job_schedule_interval)] = IntervalPGetDatum(&updated_job->fd.schedule_interval); doReplace[AttrNumberGetAttrOffset(Anum_bgw_job_schedule_interval)] = true; } values[AttrNumberGetAttrOffset(Anum_bgw_job_max_runtime)] = IntervalPGetDatum(&updated_job->fd.max_runtime); doReplace[AttrNumberGetAttrOffset(Anum_bgw_job_max_runtime)] = true; values[AttrNumberGetAttrOffset(Anum_bgw_job_max_retries)] = Int32GetDatum(updated_job->fd.max_retries); doReplace[AttrNumberGetAttrOffset(Anum_bgw_job_max_retries)] = true; values[AttrNumberGetAttrOffset(Anum_bgw_job_retry_period)] = IntervalPGetDatum(&updated_job->fd.retry_period); doReplace[AttrNumberGetAttrOffset(Anum_bgw_job_retry_period)] = true; values[AttrNumberGetAttrOffset(Anum_bgw_job_scheduled)] = BoolGetDatum(updated_job->fd.scheduled); doReplace[AttrNumberGetAttrOffset(Anum_bgw_job_scheduled)] = true; values[AttrNumberGetAttrOffset(Anum_bgw_job_fixed_schedule)] = BoolGetDatum(updated_job->fd.fixed_schedule); doReplace[AttrNumberGetAttrOffset(Anum_bgw_job_fixed_schedule)] = true; doReplace[AttrNumberGetAttrOffset(Anum_bgw_job_config)] = true; values[AttrNumberGetAttrOffset(Anum_bgw_job_check_schema)] = NameGetDatum(&updated_job->fd.check_schema); doReplace[AttrNumberGetAttrOffset(Anum_bgw_job_check_schema)] = true; values[AttrNumberGetAttrOffset(Anum_bgw_job_check_name)] = NameGetDatum(&updated_job->fd.check_name); doReplace[AttrNumberGetAttrOffset(Anum_bgw_job_check_name)] = true; if (strlen(NameStr(updated_job->fd.check_name)) == 0) { isnull[AttrNumberGetAttrOffset(Anum_bgw_job_check_name)] = true; isnull[AttrNumberGetAttrOffset(Anum_bgw_job_check_schema)] = true; } if (updated_job->fd.config) { job_config_check(updated_job, updated_job->fd.config); values[AttrNumberGetAttrOffset(Anum_bgw_job_config)] = JsonbPGetDatum(updated_job->fd.config); } else isnull[AttrNumberGetAttrOffset(Anum_bgw_job_config)] = true; if (updated_job->fd.hypertable_id != INVALID_HYPERTABLE_ID) { values[AttrNumberGetAttrOffset(Anum_bgw_job_hypertable_id)] = Int32GetDatum(updated_job->fd.hypertable_id); doReplace[AttrNumberGetAttrOffset(Anum_bgw_job_hypertable_id)] = true; } else isnull[AttrNumberGetAttrOffset(Anum_bgw_job_hypertable_id)] = true; if (TIMESTAMP_NOT_FINITE(updated_job->fd.initial_start)) isnull[AttrNumberGetAttrOffset(Anum_bgw_job_initial_start)] = true; else values[AttrNumberGetAttrOffset(Anum_bgw_job_initial_start)] = TimestampTzGetDatum(updated_job->fd.initial_start); doReplace[AttrNumberGetAttrOffset(Anum_bgw_job_initial_start)] = true; if (updated_job->fd.timezone) { values[AttrNumberGetAttrOffset(Anum_bgw_job_timezone)] = PointerGetDatum(updated_job->fd.timezone); } else isnull[AttrNumberGetAttrOffset(Anum_bgw_job_timezone)] = true; doReplace[AttrNumberGetAttrOffset(Anum_bgw_job_timezone)] = true; new_tuple = heap_modify_tuple(tuple, ts_scanner_get_tupledesc(ti), values, isnull, doReplace); ts_catalog_update(ti->scanrel, new_tuple); heap_freetuple(new_tuple); if (should_free) heap_freetuple(tuple); return SCAN_DONE; } /* * Overwrite job with specified job_id with the given fields * * This function only updates the fields modifiable with alter_job. */ bool ts_bgw_job_update_by_id(int32 job_id, BgwJob *job) { ScanKeyData scankey[1]; Catalog *catalog = ts_catalog_get(); ScanTupLock scantuplock = { .waitpolicy = LockWaitBlock, .lockmode = LockTupleExclusive, }; ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, BGW_JOB), .index = catalog_get_index(catalog, BGW_JOB, BGW_JOB_PKEY_IDX), .nkeys = 1, .scankey = scankey, .data = job, .limit = 1, .tuple_found = bgw_job_tuple_update_by_id, .lockmode = RowExclusiveLock, .scandirection = ForwardScanDirection, .result_mctx = CurrentMemoryContext, .tuplock = &scantuplock, .use_catalog_snapshot = true }; ScanKeyInit(&scankey[0], Anum_bgw_job_pkey_idx_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(job_id)); return ts_scanner_scan(&scanctx); } static void ts_bgw_job_check_max_retries(BgwJob *job) { BgwJobStat *job_stat; job_stat = ts_bgw_job_stat_find(job->fd.id); /* stop to execute failing jobs after reached the "max_retries" option */ if (job->fd.max_retries >= 0 && job_stat->fd.consecutive_failures >= job->fd.max_retries) { ereport(WARNING, (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED), errmsg("job %d reached max_retries after %d consecutive failures", job->fd.id, job_stat->fd.consecutive_failures), errdetail("Job %d unscheduled as max_retries reached %d, consecutive failures %d.", job->fd.id, job->fd.max_retries, job_stat->fd.consecutive_failures), errhint("Use alter_job(%d, scheduled => TRUE) SQL function to reschedule.", job->fd.id))); if (job->fd.scheduled) { job->fd.scheduled = false; ts_bgw_job_update_by_id(job->fd.id, job); } } } void ts_bgw_job_permission_check(BgwJob *job, const char *cmd) { if (!has_privs_of_role(GetUserId(), job->fd.owner)) { const char *owner_name = GetUserNameFromId(job->fd.owner, false); const char *user_name = GetUserNameFromId(GetUserId(), false); ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("insufficient permissions to %s job %d", cmd, job->fd.id), errdetail("Job %d is owned by role \"%s\" but user \"%s\" does not belong to that " "role.", job->fd.id, owner_name, user_name))); } } void ts_bgw_job_validate_job_owner(Oid owner) { HeapTuple role_tup = SearchSysCache1(AUTHOID, ObjectIdGetDatum(owner)); if (!HeapTupleIsValid(role_tup)) elog(ERROR, "cache lookup failed for role %u", owner); Form_pg_authid rform = (Form_pg_authid) GETSTRUCT(role_tup); if (!rform->rolcanlogin) { ReleaseSysCache(role_tup); ereport(ERROR, (errcode(ERRCODE_INVALID_AUTHORIZATION_SPECIFICATION), errmsg("permission denied to start background process as role \"%s\"", NameStr(rform->rolname)), errhint("Hypertable owner must have LOGIN permission to run background tasks."))); } ReleaseSysCache(role_tup); } /* * Is the job the telemetry job? */ #ifdef USE_TELEMETRY bool ts_is_telemetry_job(BgwJob *job) { return namestrcmp(&job->fd.proc_schema, FUNCTIONS_SCHEMA_NAME) == 0 && namestrcmp(&job->fd.proc_name, "policy_telemetry") == 0; } #endif JobResult ts_bgw_job_execute(BgwJob *job) { #ifdef USE_TELEMETRY /* The telemetry job has a separate code path since we want to be able to * use telemetry even if the TSL code is not installed. */ if (ts_is_telemetry_job(job)) { /* * In the first 12 hours, we want telemetry to ping every * hour. After that initial period, we default to the * schedule_interval listed in the job table. */ Interval one_hour = { .time = 1 * USECS_PER_HOUR }; return ts_bgw_job_run_and_set_next_start(job, ts_telemetry_main_wrapper, TELEMETRY_INITIAL_NUM_RUNS, &one_hour, /* atomic */ true, /* mark */ false); } #endif #ifdef TS_DEBUG if (scheduler_test_hook != NULL) return scheduler_test_hook(job); #endif return ts_cm_functions->job_execute(job); } bool ts_bgw_job_has_timeout(BgwJob *job) { Interval zero_val = { .time = 0, }; return DatumGetBool(DirectFunctionCall2(interval_gt, IntervalPGetDatum(&job->fd.max_runtime), IntervalPGetDatum(&zero_val))); } /* Return the timestamp at which to kill the job due to a timeout */ TimestampTz ts_bgw_job_timeout_at(BgwJob *job, TimestampTz start_time) { /* timestamptz plus interval */ return DatumGetTimestampTz(DirectFunctionCall2(timestamptz_pl_interval, TimestampTzGetDatum(start_time), IntervalPGetDatum(&job->fd.max_runtime))); } TS_FUNCTION_INFO_V1(ts_bgw_job_entrypoint); static void zero_guc(const char *guc_name) { int config_change = set_config_option(guc_name, "0", PGC_SUSET, PGC_S_SESSION, GUC_ACTION_SET, true, 0, false); if (config_change == 0) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("guc \"%s\" does not exist", guc_name))); else if (config_change < 0) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not set \"%s\" guc", guc_name))); } Oid ts_bgw_job_get_funcid(BgwJob *job) { ObjectWithArgs *object = makeNode(ObjectWithArgs); object->objname = list_make2(makeString(NameStr(job->fd.proc_schema)), makeString(NameStr(job->fd.proc_name))); object->objargs = list_make2(SystemTypeName("int4"), SystemTypeName("jsonb")); /* Return InvalidOid if don't found */ return LookupFuncWithArgs(OBJECT_ROUTINE, object, true); } const char * ts_bgw_job_function_call_string(BgwJob *job) { Oid funcid = ts_bgw_job_get_funcid(job); /* If do not found the function or procedure then fallback to PROKIND_FUNCTION */ char prokind = OidIsValid(funcid) ? get_func_prokind(funcid) : PROKIND_FUNCTION; StringInfoData stmt; initStringInfo(&stmt); char *jsonb_str = "NULL"; if (job->fd.config) jsonb_str = quote_literal_cstr( JsonbToCString(NULL, &job->fd.config->root, VARSIZE(job->fd.config))); switch (prokind) { case PROKIND_FUNCTION: appendStringInfo(&stmt, "SELECT %s.%s('%d', %s)", quote_identifier(NameStr(job->fd.proc_schema)), quote_identifier(NameStr(job->fd.proc_name)), job->fd.id, jsonb_str); break; case PROKIND_PROCEDURE: appendStringInfo(&stmt, "CALL %s.%s('%d', %s)", quote_identifier(NameStr(job->fd.proc_schema)), quote_identifier(NameStr(job->fd.proc_name)), job->fd.id, jsonb_str); break; default: ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("unsupported function type: %c", prokind))); break; } return stmt.data; } extern Datum ts_bgw_job_entrypoint(PG_FUNCTION_ARGS) { Oid db_oid = DatumGetObjectId(MyBgworkerEntry->bgw_main_arg); BgwParams params; BgwJob *job; JobResult volatile res = JOB_FAILURE_IN_EXECUTION; instr_time start; instr_time duration; memcpy(¶ms, MyBgworkerEntry->bgw_extra, sizeof(BgwParams)); Ensure(OidIsValid(params.user_oid) && params.job_id != 0, "job id or user oid was zero - job_id: %d, user_oid: %d", params.job_id, params.user_oid); BackgroundWorkerBlockSignals(); /* Setup any signal handlers here */ /* * do not use the default `bgworker_die` sigterm handler because it does * not respect critical sections */ pqsignal(SIGTERM, die); BackgroundWorkerUnblockSignals(); /* * Set up mem_guard before starting to allocate (any significant amounts * of) memory but after we have unblocked signals since we have no control * over how the callback behaves. */ MGCallbacks *callbacks = ts_get_mem_guard_callbacks(); if (callbacks && callbacks->version_num == MG_CALLBACKS_VERSION && callbacks->toggle_allocation_blocking && !callbacks->enabled) callbacks->toggle_allocation_blocking(/*enable=*/true); BackgroundWorkerInitializeConnectionByOid(db_oid, params.user_oid, 0); log_min_messages = ts_guc_bgw_log_level; elog(DEBUG2, "job %d started execution", params.job_id); ts_license_enable_module_loading(); INSTR_TIME_SET_CURRENT(start); StartTransactionCommand(); PushActiveSnapshot(GetTransactionSnapshot()); job = ts_bgw_job_find(params.job_id, TopMemoryContext, false); if (job == NULL) /* If the job is not found, we can't proceed */ ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("job %d not found when running the background worker", params.job_id))); /* get parameters from bgworker */ job->job_history.id = params.job_history_id; job->job_history.execution_start = params.job_history_execution_start; ts_bgw_job_stat_history_update(JOB_STAT_HISTORY_UPDATE_PID, job, JOB_SUCCESS, NULL); PopActiveSnapshot(); CommitTransactionCommand(); elog(DEBUG2, "job %d (%s) found", params.job_id, NameStr(job->fd.application_name)); pgstat_report_appname(NameStr(job->fd.application_name)); MemoryContext oldcontext = CurrentMemoryContext; bool job_failed = false; if (scheduler_test_hook == NULL) ts_begin_tss_store_callback(); PG_TRY(); { /* * we do not necessarily have a valid parallel worker context in * background workers, so disable parallel execution by default */ zero_guc("max_parallel_workers_per_gather"); zero_guc("max_parallel_workers"); zero_guc("max_parallel_maintenance_workers"); res = ts_bgw_job_execute(job); /* The job is responsible for committing or aborting it's own txns */ if (IsTransactionState()) ereport(ERROR, (errcode(ERRCODE_INVALID_TRANSACTION_STATE), errmsg("TimescaleDB background job \"%s\" failed to end the transaction", NameStr(job->fd.application_name)))); } PG_CATCH(); { ErrorData *edata; NameData proc_schema = { .data = { 0 } }, proc_name = { .data = { 0 } }; if (IsTransactionState()) /* If there was an error, rollback what was done before the error */ AbortCurrentTransaction(); StartTransactionCommand(); PushActiveSnapshot(GetTransactionSnapshot()); /* Free the old job if it exists, it's no longer needed, and since it's * in the TopMemoryContext it won't be freed otherwise. */ if (job != NULL) { pfree(job); } /* switch away from error context to not lose the data */ MemoryContextSwitchTo(oldcontext); job_failed = true; edata = CopyErrorData(); FlushErrorState(); /* * Note that the mark_start happens in the scheduler right before the * job is launched. */ job = ts_bgw_job_find(params.job_id, TopMemoryContext, false); if (job != NULL) { namestrcpy(&proc_name, NameStr(job->fd.proc_name)); namestrcpy(&proc_schema, NameStr(job->fd.proc_schema)); job->job_history.id = params.job_history_id; job->job_history.execution_start = params.job_history_execution_start; ts_bgw_job_stat_mark_end(job, JOB_FAILURE_IN_EXECUTION, ts_errdata_to_jsonb(edata, &proc_schema, &proc_name)); ts_bgw_job_check_max_retries(job); pfree(job); } /* * the rethrow will log the error; but also log which job threw the * error */ elog(LOG, "job %d threw an error", params.job_id); PopActiveSnapshot(); CommitTransactionCommand(); ReThrowError(edata); } PG_END_TRY(); Assert(!IsTransactionState()); StartTransactionCommand(); PushActiveSnapshot(GetTransactionSnapshot()); /* * Note that the mark_start happens in the scheduler right before the job * is launched */ ts_bgw_job_stat_mark_end(job, res, NULL); if (!job_failed && ts_is_tss_enabled() && scheduler_test_hook == NULL) { const char *stmt = ts_bgw_job_function_call_string(job); ts_end_tss_store_callback(stmt, -1, (int) strlen(stmt), 0, 0); } PopActiveSnapshot(); CommitTransactionCommand(); INSTR_TIME_SET_CURRENT(duration); INSTR_TIME_SUBTRACT(duration, start); elog(DEBUG1, "job %d (%s) exiting with %s: execution time %.2f ms", params.job_id, NameStr(job->fd.application_name), (res == JOB_SUCCESS ? "success" : "failure"), INSTR_TIME_GET_MILLISEC(duration)); if (job != NULL) { pfree(job); job = NULL; } PG_RETURN_VOID(); } void ts_bgw_job_set_scheduler_test_hook(scheduler_test_hook_type hook) { scheduler_test_hook = hook; } void ts_bgw_job_set_job_entrypoint_function_name(char *func_name) { job_entrypoint_function_name = func_name; } /* * Run job and set next start. * * job: Job to run and update * func: Function to execute for the job * initial_runs: Limit on the number of runs to do * next_interval: Interval to use when computing next start * atomic: Should be executed as a single transaction. * mark: Mark the start and end of the function execution in job_stats */ bool ts_bgw_job_run_and_set_next_start(BgwJob *job, job_main_func func, int64 initial_runs, Interval *next_interval, bool atomic, bool mark) { BgwJobStat *job_stat; bool result; if (atomic) StartTransactionCommand(); if (mark) ts_bgw_job_stat_mark_start(job); result = func(); if (mark) ts_bgw_job_stat_mark_end(job, result ? JOB_SUCCESS : JOB_FAILURE_IN_EXECUTION, NULL); /* Now update next_start. */ job_stat = ts_bgw_job_stat_find(job->fd.id); /* * Note that setting next_start explicitly from this function will * override any backoff calculation due to failure. */ Ensure(job_stat != NULL, "job status for job %d not found", job->fd.id); if (job_stat->fd.total_runs < initial_runs) { TimestampTz next_start = DatumGetTimestampTz(DirectFunctionCall2(timestamptz_pl_interval, TimestampTzGetDatum(job_stat->fd.last_start), IntervalPGetDatum(next_interval))); ts_bgw_job_stat_set_next_start(job->fd.id, next_start); } if (atomic) CommitTransactionCommand(); return result; } /* Insert a new job in the bgw_job relation */ int ts_bgw_job_insert_relation(Name application_name, Interval *schedule_interval, Interval *max_runtime, int32 max_retries, Interval *retry_period, Name proc_schema, Name proc_name, Name check_schema, Name check_name, Oid owner, bool scheduled, bool fixed_schedule, int32 hypertable_id, Jsonb *config, TimestampTz initial_start, const char *timezone) { Catalog *catalog = ts_catalog_get(); Relation rel; TupleDesc desc; Datum values[Natts_bgw_job] = { 0 }; CatalogSecurityContext sec_ctx; bool nulls[Natts_bgw_job] = { false }; int32 job_id; char app_name[NAMEDATALEN]; int name_len; rel = table_open(catalog_get_table_id(catalog, BGW_JOB), RowExclusiveLock); desc = RelationGetDescr(rel); values[AttrNumberGetAttrOffset(Anum_bgw_job_schedule_interval)] = IntervalPGetDatum(schedule_interval); values[AttrNumberGetAttrOffset(Anum_bgw_job_max_runtime)] = IntervalPGetDatum(max_runtime); values[AttrNumberGetAttrOffset(Anum_bgw_job_max_retries)] = Int32GetDatum(max_retries); values[AttrNumberGetAttrOffset(Anum_bgw_job_retry_period)] = IntervalPGetDatum(retry_period); values[AttrNumberGetAttrOffset(Anum_bgw_job_proc_schema)] = NameGetDatum(proc_schema); values[AttrNumberGetAttrOffset(Anum_bgw_job_proc_name)] = NameGetDatum(proc_name); if (strlen(NameStr(*check_schema)) > 0) values[AttrNumberGetAttrOffset(Anum_bgw_job_check_schema)] = NameGetDatum(check_schema); else nulls[AttrNumberGetAttrOffset(Anum_bgw_job_check_schema)] = true; if (strlen(NameStr(*check_name)) > 0) values[AttrNumberGetAttrOffset(Anum_bgw_job_check_name)] = NameGetDatum(check_name); else nulls[AttrNumberGetAttrOffset(Anum_bgw_job_check_name)] = true; values[AttrNumberGetAttrOffset(Anum_bgw_job_owner)] = ObjectIdGetDatum(owner); values[AttrNumberGetAttrOffset(Anum_bgw_job_scheduled)] = BoolGetDatum(scheduled); values[AttrNumberGetAttrOffset(Anum_bgw_job_fixed_schedule)] = BoolGetDatum(fixed_schedule); /* initial_start must have a value if the schedule is fixed */ Assert(!fixed_schedule || !TIMESTAMP_NOT_FINITE(initial_start)); if (TIMESTAMP_NOT_FINITE(initial_start)) { nulls[AttrNumberGetAttrOffset(Anum_bgw_job_initial_start)] = true; values[AttrNumberGetAttrOffset(Anum_bgw_job_initial_start)] = TimestampTzGetDatum(initial_start); } else { nulls[AttrNumberGetAttrOffset(Anum_bgw_job_initial_start)] = false; values[AttrNumberGetAttrOffset(Anum_bgw_job_initial_start)] = TimestampTzGetDatum(initial_start); } if (hypertable_id == INVALID_HYPERTABLE_ID) nulls[AttrNumberGetAttrOffset(Anum_bgw_job_hypertable_id)] = true; else values[AttrNumberGetAttrOffset(Anum_bgw_job_hypertable_id)] = Int32GetDatum(hypertable_id); if (config == NULL) nulls[AttrNumberGetAttrOffset(Anum_bgw_job_config)] = true; else values[AttrNumberGetAttrOffset(Anum_bgw_job_config)] = JsonbPGetDatum(config); if (timezone == NULL) nulls[AttrNumberGetAttrOffset(Anum_bgw_job_timezone)] = true; else values[AttrNumberGetAttrOffset(Anum_bgw_job_timezone)] = CStringGetTextDatum(timezone); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); job_id = DatumGetInt32(ts_catalog_table_next_seq_id(catalog, BGW_JOB)); name_len = snprintf(app_name, NAMEDATALEN, "%s [%d]", NameStr(*application_name), job_id); if (name_len >= NAMEDATALEN) ereport(ERROR, (errcode(ERRCODE_NAME_TOO_LONG), errmsg("application name too long."))); values[AttrNumberGetAttrOffset(Anum_bgw_job_id)] = Int32GetDatum(job_id); values[AttrNumberGetAttrOffset(Anum_bgw_job_application_name)] = CStringGetDatum(app_name); ts_catalog_insert_values(rel, desc, values, nulls); ts_catalog_restore_user(&sec_ctx); /* This is where we would add a call to recordDependencyOnOwner, but it * cannot support dependencies on anything but built-in classes since * getObjectClass() have a lot of hard-coded checks in place. * * Instead we have a check in process_utility.c that prevents dropping the * user if there is a dependent job. */ table_close(rel, NoLock); return values[AttrNumberGetAttrOffset(Anum_bgw_job_id)]; } /* * This function ensures the schedule interval is acceptable in the case * of fixed job schedules. Intervals that mix months with day and time * components are not acceptable since internally we use time_bucket and * cannot bucket by such an interval. */ void ts_bgw_job_validate_schedule_interval(Interval *schedule_interval) { bool has_month, has_day, has_time; has_month = schedule_interval->month; has_day = schedule_interval->day; has_time = schedule_interval->time; if (has_month && (has_day || has_time)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("month intervals cannot have day or time component"), errdetail("Fixed schedule jobs do not support such schedule intervals."), errhint("Express the interval in terms of days or time instead."))); } char * ts_bgw_job_validate_timezone(Datum timezone) { DirectFunctionCall2(timestamp_zone, timezone, TimestampGetDatum(ts_timer_get_current_timestamp())); return TextDatumGetCString(timezone); } ================================================ FILE: src/bgw/job.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <postmaster/bgworker.h> #include "export.h" #include "ts_catalog/catalog.h" #define TELEMETRY_INITIAL_NUM_RUNS 12 #define SCHEDULER_APPNAME "TimescaleDB Background Worker Scheduler" /* * This is copied from mem_guard and have to be the same as the type in * mem_guard. * * These are intended as an interim solution and will be removed once we have * a stable plugin ABI for TimescaleDB. */ #define MG_CALLBACKS_VERSION 1 #define MG_CALLBACKS_VAR_NAME "mg_callbacks" typedef void (*mg_toggle_allocation_blocking)(bool enable); typedef size_t (*mg_get_allocated_memory)(); typedef size_t (*mg_get_total_allocated_memory)(); typedef bool (*mg_enabled)(); typedef struct MGCallbacks { int64 version_num; mg_toggle_allocation_blocking toggle_allocation_blocking; mg_get_allocated_memory get_allocated_memory; mg_get_total_allocated_memory get_total_allocated_memory; mg_enabled enabled; } MGCallbacks; typedef struct BgwJobHistory { int64 id; TimestampTz execution_start; } BgwJobHistory; typedef struct BgwJob { FormData_bgw_job fd; BgwJobHistory job_history; } BgwJob; /* Positive result numbers reserved for success */ typedef enum JobResult { JOB_FAILURE_TO_START = -1, JOB_FAILURE_IN_EXECUTION = 0, JOB_SUCCESS = 1, } JobResult; typedef bool job_main_func(void); typedef bool (*scheduler_test_hook_type)(BgwJob *job); extern BackgroundWorkerHandle *ts_bgw_job_start(BgwJob *job, Oid user_oid); extern List *ts_bgw_job_get_all(size_t alloc_size, MemoryContext mctx); extern List *ts_bgw_job_get_scheduled(size_t alloc_size, MemoryContext mctx); extern TSDLLEXPORT List *ts_bgw_job_find_by_hypertable_id(int32 hypertable_id); extern TSDLLEXPORT List *ts_bgw_job_find_by_proc_and_hypertable_id(const char *proc_name, const char *proc_schema, int32 hypertable_id); TSDLLEXPORT BgwJob *ts_bgw_job_find(int job_id, MemoryContext mctx, bool fail_if_not_found); extern bool ts_bgw_job_has_timeout(BgwJob *job); extern TimestampTz ts_bgw_job_timeout_at(BgwJob *job, TimestampTz start_time); extern TSDLLEXPORT bool ts_bgw_job_delete_by_id(int32 job_id); extern TSDLLEXPORT bool ts_bgw_job_update_by_id(int32 job_id, BgwJob *job); extern TSDLLEXPORT int32 ts_bgw_job_insert_relation( Name application_name, Interval *schedule_interval, Interval *max_runtime, int32 max_retries, Interval *retry_period, Name proc_schema, Name proc_name, Name check_schema, Name check_name, Oid owner, bool scheduled, bool fixed_schedule, int32 hypertable_id, Jsonb *config, TimestampTz initial_start, const char *timezone); extern TSDLLEXPORT void ts_bgw_job_permission_check(BgwJob *job, const char *cmd); extern TSDLLEXPORT void ts_bgw_job_validate_job_owner(Oid owner); extern JobResult ts_bgw_job_execute(BgwJob *job); extern TSDLLEXPORT void ts_bgw_job_run_config_check(Oid check, int32 job_id, Jsonb *config); extern TSDLLEXPORT Datum ts_bgw_job_entrypoint(PG_FUNCTION_ARGS); extern void ts_bgw_job_set_scheduler_test_hook(scheduler_test_hook_type hook); extern void ts_bgw_job_set_job_entrypoint_function_name(char *func_name); extern TSDLLEXPORT bool ts_bgw_job_run_and_set_next_start(BgwJob *job, job_main_func func, int64 initial_runs, Interval *next_interval, bool atomic, bool mark); extern TSDLLEXPORT void ts_bgw_job_validate_schedule_interval(Interval *schedule_interval); extern TSDLLEXPORT char *ts_bgw_job_validate_timezone(Datum timezone); extern TSDLLEXPORT bool ts_is_telemetry_job(BgwJob *job); ScanTupleResult ts_bgw_job_change_owner(TupleInfo *ti, void *data); extern TSDLLEXPORT Oid ts_bgw_job_get_funcid(BgwJob *job); extern TSDLLEXPORT const char *ts_bgw_job_function_call_string(BgwJob *job); extern TSDLLEXPORT MGCallbacks *ts_get_mem_guard_callbacks(void); ================================================ FILE: src/bgw/job_stat.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <math.h> #include <stdlib.h> #include <utils/builtins.h> #include <utils/fmgrprotos.h> #include <utils/resowner.h> #include "guc.h" #include "job.h" #include "job_stat.h" #include "job_stat_history.h" #include "jsonb_utils.h" #include "scanner.h" #include "time_bucket.h" #include "timer.h" #include "utils.h" #define MAX_INTERVALS_BACKOFF 5 #define MAX_FAILURES_MULTIPLIER 20 #define MIN_WAIT_AFTER_CRASH_MS (5 * 60 * 1000) static bool bgw_job_stat_next_start_was_set(FormData_bgw_job_stat *fd) { return fd->next_start != DT_NOBEGIN; } static ScanTupleResult bgw_job_stat_tuple_found(TupleInfo *ti, void *const data) { BgwJobStat **job_stat_pp = (BgwJobStat **) data; *job_stat_pp = STRUCT_FROM_SLOT(ti->slot, ti->mctx, BgwJobStat, FormData_bgw_job_stat); /* * Return SCAN_CONTINUE because we check for multiple tuples as an error * condition. */ return SCAN_CONTINUE; } static bool bgw_job_stat_scan_one(int indexid, ScanKeyData scankey[], int nkeys, tuple_found_func tuple_found, tuple_filter_func tuple_filter, void *data, LOCKMODE lockmode) { Catalog *catalog = ts_catalog_get(); ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, BGW_JOB_STAT), .index = catalog_get_index(catalog, BGW_JOB_STAT, indexid), .nkeys = nkeys, .scankey = scankey, .flags = SCANNER_F_KEEPLOCK, .tuple_found = tuple_found, .filter = tuple_filter, .data = data, .lockmode = lockmode, .scandirection = ForwardScanDirection, }; return ts_scanner_scan_one(&scanctx, false, "bgw job stat"); } static inline bool bgw_job_stat_scan_job_id(int32 bgw_job_id, tuple_found_func tuple_found, tuple_filter_func tuple_filter, void *data, LOCKMODE lockmode) { ScanKeyData scankey[1]; ScanKeyInit(&scankey[0], Anum_bgw_job_stat_pkey_idx_job_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(bgw_job_id)); return bgw_job_stat_scan_one(BGW_JOB_STAT_PKEY_IDX, scankey, 1, tuple_found, tuple_filter, data, lockmode); } TSDLLEXPORT BgwJobStat * ts_bgw_job_stat_find(int32 bgw_job_id) { BgwJobStat *job_stat = NULL; bgw_job_stat_scan_job_id(bgw_job_id, bgw_job_stat_tuple_found, NULL, (void *) &job_stat, AccessShareLock); return job_stat; } static ScanTupleResult bgw_job_stat_tuple_delete(TupleInfo *ti, void *const data) { ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); return SCAN_CONTINUE; } void ts_bgw_job_stat_delete(int32 bgw_job_id) { bgw_job_stat_scan_job_id(bgw_job_id, bgw_job_stat_tuple_delete, NULL, NULL, ShareRowExclusiveLock); } /* Mark the start of a job. This should be done in a separate transaction by the scheduler * before the bgw for a job is launched. This ensures that the job is counted as started * before /any/ job specific code is executed. A job that has been started but never ended * is assumed to have crashed. We use this conservative design since no process in the database * instance can write once a crash happened in any job. Therefore our only choice is to deduce * a crash from the lack of a write (the marked end write in this case). */ static ScanTupleResult bgw_job_stat_tuple_mark_start(TupleInfo *ti, void *const data) { bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); HeapTuple new_tuple = heap_copytuple(tuple); FormData_bgw_job_stat *fd = (FormData_bgw_job_stat *) GETSTRUCT(new_tuple); if (should_free) heap_freetuple(tuple); fd->last_start = ts_timer_get_current_timestamp(); fd->last_finish = DT_NOBEGIN; fd->next_start = DT_NOBEGIN; fd->total_runs++; /* * This is undone by any of the end marks. This is so that we count * crashes conservatively. Pretty much the crash is incremented in the * beginning and then decremented during `bgw_job_stat_tuple_mark_end`. * Thus, it only remains incremented if the job is never marked as having * ended. This happens when: 1) the job crashes 2) another process crashes * while the job is running 3) the scheduler gets a SIGTERM while the job * is running * * Unfortunately, 3 cannot be helped because when a scheduler gets a * SIGTERM it sends SIGTERMS to it's any running jobs as well. Since you * aren't supposed to write to the DB Once you get a sigterm, neither the * job nor the scheduler can mark the end of a job. */ fd->last_run_success = false; fd->total_crashes++; fd->consecutive_crashes++; fd->flags = ts_clear_flags_32(fd->flags, LAST_CRASH_REPORTED); ts_catalog_update(ti->scanrel, new_tuple); heap_freetuple(new_tuple); return SCAN_DONE; } typedef struct { JobResult result; BgwJob *job; } JobResultCtx; /* * logic is the following * Ideally we would return * time_bucket(schedule_interval, finish_time, origin => initial_start, timezone). * That is what we return when the schedule interval does not have month components. * However, when there is a month component in the schedule interval, * then supplying the origin in time_bucket * does not work and the returned bucket is aligned on the start of the month. * In those cases, we only have month components. So we compute the difference in * months between the initial_start's timebucket and the finish time's bucket. */ TimestampTz ts_get_next_scheduled_execution_slot(BgwJob *job, TimestampTz finish_time) { Assert(job->fd.fixed_schedule == true); Datum schedint_datum = IntervalPGetDatum(&job->fd.schedule_interval); Datum timebucket_fini, timebucket_init, result; Interval one_month = { .day = 0, .time = 0, .month = 1, }; if (job->fd.schedule_interval.month > 0) { if (job->fd.timezone == NULL) { timebucket_init = DirectFunctionCall2(ts_timestamptz_bucket, schedint_datum, TimestampTzGetDatum(job->fd.initial_start)); timebucket_fini = DirectFunctionCall2(ts_timestamptz_bucket, schedint_datum, TimestampTzGetDatum(finish_time)); } else { char *tz = text_to_cstring(job->fd.timezone); timebucket_fini = DirectFunctionCall3(ts_timestamptz_timezone_bucket, schedint_datum, TimestampTzGetDatum(finish_time), CStringGetTextDatum(tz)); timebucket_init = DirectFunctionCall3(ts_timestamptz_timezone_bucket, schedint_datum, TimestampTzGetDatum(job->fd.initial_start), CStringGetTextDatum(tz)); } /* always the next bucket */ timebucket_fini = DirectFunctionCall2(timestamptz_pl_interval, timebucket_fini, schedint_datum); /* get the number of months between them */ Datum year_init = DirectFunctionCall2(timestamptz_part, CStringGetTextDatum("year"), timebucket_init); Datum year_fini = DirectFunctionCall2(timestamptz_part, CStringGetTextDatum("year"), timebucket_fini); Datum month_init = DirectFunctionCall2(timestamptz_part, CStringGetTextDatum("month"), timebucket_init); Datum month_fini = DirectFunctionCall2(timestamptz_part, CStringGetTextDatum("month"), timebucket_fini); /* convert everything to months */ float8 month_diff = (DatumGetFloat8(year_fini) * 12) + DatumGetFloat8(month_fini) - ((DatumGetFloat8(year_init) * 12) + DatumGetFloat8(month_init)); Datum months_to_add = DirectFunctionCall2(interval_mul, IntervalPGetDatum(&one_month), Float8GetDatum(month_diff)); result = DirectFunctionCall2(timestamptz_pl_interval, TimestampTzGetDatum(job->fd.initial_start), months_to_add); } else { if (job->fd.timezone == NULL) { /* it is safe to use the origin in time_bucket calculation */ timebucket_fini = DirectFunctionCall3(ts_timestamptz_bucket, schedint_datum, TimestampTzGetDatum(finish_time), TimestampTzGetDatum(job->fd.initial_start)); result = timebucket_fini; } else { char *tz = text_to_cstring(job->fd.timezone); timebucket_fini = DirectFunctionCall4(ts_timestamptz_timezone_bucket, schedint_datum, TimestampTzGetDatum(finish_time), CStringGetTextDatum(tz), TimestampTzGetDatum(job->fd.initial_start)); result = timebucket_fini; } } while (DatumGetTimestampTz(result) <= finish_time) { result = DirectFunctionCall2(timestamptz_pl_interval, result, schedint_datum); } return DatumGetTimestampTz(result); } static TimestampTz calculate_next_start_on_success_fixed(TimestampTz finish_time, BgwJob *job) { TimestampTz next_slot; next_slot = ts_get_next_scheduled_execution_slot(job, finish_time); return next_slot; } static TimestampTz calculate_next_start_on_success_drifting(TimestampTz last_finish, BgwJob *job) { TimestampTz ts; ts = DatumGetTimestampTz(DirectFunctionCall2(timestamptz_pl_interval, TimestampTzGetDatum(last_finish), IntervalPGetDatum(&job->fd.schedule_interval))); return ts; } static TimestampTz calculate_next_start_on_success(TimestampTz finish_time, BgwJob *job) { /* next_start is the previously calculated next_start for this job */ TimestampTz ts; TimestampTz last_finish = finish_time; if (!IS_VALID_TIMESTAMP(finish_time)) { last_finish = ts_timer_get_current_timestamp(); } /* calculate next_start differently depending on drift/no drift */ if (job->fd.fixed_schedule) ts = calculate_next_start_on_success_fixed(last_finish, job); else ts = calculate_next_start_on_success_drifting(last_finish, job); return ts; } static float8 calculate_jitter_percent() { /* returns a number in the range [-0.125, 0.125] */ uint8 percent = rand(); return ldexp((double) (16 - (int) (percent % 32)), -7); } /* For failures we have backoff based on consecutive failures * along with a ceiling at schedule_interval * MAX_INTERVALS_BACKOFF / 1 minute * for jobs failing at runtime / for jobs failing to launch. * We also limit the backoff in case of consecutive failures as we don't * want to pass in input that leads to out of range timestamps and don't want to * put off the next start time for the job indefinitely */ static TimestampTz calculate_next_start_on_failure(TimestampTz finish_time, int consecutive_failures, BgwJob *job, bool launch_failure) { float8 jitter = calculate_jitter_percent(); /* * Have to be declared volatile because they are modified between * setjmp/longjmp calls. */ volatile TimestampTz res = 0; volatile bool res_set = false; volatile TimestampTz last_finish = finish_time; /* consecutive failures includes this failure */ float8 multiplier = (consecutive_failures > MAX_FAILURES_MULTIPLIER ? MAX_FAILURES_MULTIPLIER : consecutive_failures); Assert(consecutive_failures > 0 && multiplier < 63); MemoryContext oldctx = CurrentMemoryContext; ResourceOwner oldowner = CurrentResourceOwner; /* 2^(consecutive_failures) - 1, at most 2^20 */ int64 max_slots = (INT64CONST(1) << (int64) multiplier) - INT64CONST(1); int64 rand_backoff = rand() % (max_slots * USECS_PER_SEC); if (!IS_VALID_TIMESTAMP(finish_time)) { elog(LOG, "%s: invalid finish time", __func__); last_finish = ts_timer_get_current_timestamp(); } PG_TRY(); { Datum ival; /* ival_max is the ceiling = MAX_INTERVALS_BACKOFF * schedule_interval */ Datum ival_max; // max wait time to launch job is 1 minute Interval interval_max = { .time = 60000000 }; Interval retry_ival = { .time = 2000000 }; retry_ival.time += rand_backoff; BeginInternalSubTransaction("next start on failure"); if (launch_failure) { // random backoff seconds in [2, 2 + 2^f] ival = IntervalPGetDatum(&retry_ival); ival_max = IntervalPGetDatum(&interval_max); } else { /* ival = retry_period * (consecutive_failures) */ ival = DirectFunctionCall2(interval_mul, IntervalPGetDatum(&job->fd.retry_period), Float8GetDatum(multiplier)); /* ival_max is the ceiling = MAX_INTERVALS_BACKOFF * schedule_interval */ ival_max = DirectFunctionCall2(interval_mul, IntervalPGetDatum(&job->fd.schedule_interval), Float8GetDatum(MAX_INTERVALS_BACKOFF)); } if (DatumGetInt32(DirectFunctionCall2(interval_cmp, ival, ival_max)) > 0) ival = ival_max; /* Add some random jitter to prevent stampeding-herds, interval will be within about +-13% */ ival = DirectFunctionCall2(interval_mul, ival, Float8GetDatum(1.0 + jitter)); res = DatumGetTimestampTz( DirectFunctionCall2(timestamptz_pl_interval, TimestampTzGetDatum(last_finish), ival)); res_set = true; ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldctx); CurrentResourceOwner = oldowner; } PG_CATCH(); { RollbackAndReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldctx); CurrentResourceOwner = oldowner; ErrorData *errdata = CopyErrorData(); FlushErrorState(); ereport(LOG, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not calculate next start on failure: resetting value"), errdetail("Error: %s.", errdata->message))); FreeErrorData(errdata); } PG_END_TRY(); Assert(CurrentMemoryContext == oldctx); if (!res_set) { TimestampTz nowt; /* job->fd_retry_period is a valid non-null value */ nowt = ts_timer_get_current_timestamp(); res = DatumGetTimestampTz(DirectFunctionCall2(timestamptz_pl_interval, TimestampTzGetDatum(nowt), IntervalPGetDatum(&job->fd.retry_period))); } /* for fixed_schedules, we make sure that if the calculated next_start time * surpasses the next scheduled slot, then next_start will be set to the value * of the next scheduled slot, so we don't get off track */ if (job->fd.fixed_schedule) { TimestampTz next_slot = ts_get_next_scheduled_execution_slot(job, finish_time); if (res > next_slot) res = next_slot; } return res; } static TimestampTz calculate_next_start_on_failed_launch(int consecutive_failed_launches, BgwJob *job) { TimestampTz now = ts_timer_get_current_timestamp(); TimestampTz failure_calc = calculate_next_start_on_failure(now, consecutive_failed_launches, job, true); return failure_calc; } /* For crashes, the logic is the similar as for failures except we also have * a minimum wait after a crash that we wait, so that if an operator needs to disable the job, * there will be enough time before another crash. */ static TimestampTz calculate_next_start_on_crash(int consecutive_crashes, BgwJob *job) { TimestampTz now = ts_timer_get_current_timestamp(); TimestampTz failure_calc = calculate_next_start_on_failure(now, consecutive_crashes, job, false); TimestampTz min_time = TimestampTzPlusMilliseconds(now, MIN_WAIT_AFTER_CRASH_MS); if (min_time > failure_calc) return min_time; return failure_calc; } static ScanTupleResult bgw_job_stat_tuple_mark_end(TupleInfo *ti, void *const data) { JobResultCtx *result_ctx = data; bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); HeapTuple new_tuple = heap_copytuple(tuple); FormData_bgw_job_stat *fd = (FormData_bgw_job_stat *) GETSTRUCT(new_tuple); Interval *duration; if (should_free) heap_freetuple(tuple); fd->last_finish = ts_timer_get_current_timestamp(); duration = DatumGetIntervalP(DirectFunctionCall2(timestamp_mi, TimestampTzGetDatum(fd->last_finish), TimestampTzGetDatum(fd->last_start))); /* undo marking created by start marks */ fd->last_run_success = result_ctx->result == JOB_SUCCESS ? true : false; fd->total_crashes--; fd->consecutive_crashes = 0; fd->flags = ts_clear_flags_32(fd->flags, LAST_CRASH_REPORTED); if (result_ctx->result == JOB_SUCCESS) { fd->total_success++; fd->consecutive_failures = 0; fd->last_successful_finish = fd->last_finish; fd->total_duration = *DatumGetIntervalP(DirectFunctionCall2(interval_pl, IntervalPGetDatum(&fd->total_duration), IntervalPGetDatum(duration))); /* Mark the next start at the end if the job itself hasn't */ if (!bgw_job_stat_next_start_was_set(fd)) fd->next_start = calculate_next_start_on_success(fd->last_finish, result_ctx->job); } else { fd->total_failures++; fd->consecutive_failures++; fd->total_duration_failures = *DatumGetIntervalP(DirectFunctionCall2(interval_pl, IntervalPGetDatum(&fd->total_duration_failures), IntervalPGetDatum(duration))); /* * Mark the next start at the end if the job itself hasn't (this may * have happened before failure) and the failure was not in starting. * If the failure was in starting, then next_start should have been * restored in `on_failure_to_start_job` and thus we don't change it here. * Even if it wasn't restored, then keep it as DT_NOBEGIN to mark it as highest priority. */ if (!bgw_job_stat_next_start_was_set(fd) && result_ctx->result != JOB_FAILURE_TO_START) fd->next_start = calculate_next_start_on_failure(fd->last_finish, fd->consecutive_failures, result_ctx->job, false); } ts_catalog_update(ti->scanrel, new_tuple); heap_freetuple(new_tuple); return SCAN_DONE; } static ScanTupleResult bgw_job_stat_tuple_mark_crash_reported(TupleInfo *ti, void *const data) { bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); HeapTuple new_tuple = heap_copytuple(tuple); FormData_bgw_job_stat *fd = (FormData_bgw_job_stat *) GETSTRUCT(new_tuple); if (should_free) heap_freetuple(tuple); fd->flags = ts_set_flags_32(fd->flags, LAST_CRASH_REPORTED); ts_catalog_update(ti->scanrel, new_tuple); heap_freetuple(new_tuple); return SCAN_DONE; } static ScanTupleResult bgw_job_stat_tuple_set_next_start(TupleInfo *ti, void *const data) { TimestampTz *next_start = data; bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); HeapTuple new_tuple = heap_copytuple(tuple); FormData_bgw_job_stat *fd = (FormData_bgw_job_stat *) GETSTRUCT(new_tuple); if (should_free) heap_freetuple(tuple); fd->next_start = *next_start; ts_catalog_update(ti->scanrel, new_tuple); heap_freetuple(new_tuple); return SCAN_DONE; } static bool bgw_job_stat_insert_relation(Relation rel, int32 bgw_job_id, bool mark_start, TimestampTz next_start) { TupleDesc desc = RelationGetDescr(rel); Datum values[Natts_bgw_job_stat]; bool nulls[Natts_bgw_job_stat] = { false }; CatalogSecurityContext sec_ctx; Interval zero_ival = { .time = 0, }; values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_job_id)] = Int32GetDatum(bgw_job_id); if (mark_start) values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_last_start)] = TimestampGetDatum(ts_timer_get_current_timestamp()); else values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_last_start)] = TimestampGetDatum(DT_NOBEGIN); values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_last_finish)] = TimestampGetDatum(DT_NOBEGIN); values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_next_start)] = TimestampGetDatum(next_start); values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_last_successful_finish)] = TimestampGetDatum(DT_NOBEGIN); values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_total_runs)] = Int64GetDatum((mark_start ? 1 : 0)); values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_total_duration)] = IntervalPGetDatum(&zero_ival); values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_total_duration_failures)] = IntervalPGetDatum(&zero_ival); values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_total_success)] = Int64GetDatum(0); values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_total_failures)] = Int64GetDatum(0); values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_consecutive_failures)] = Int32GetDatum(0); values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_flags)] = Int32GetDatum(JOB_STAT_FLAGS_DEFAULT); if (mark_start) { /* This is udone by any of the end marks */ values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_last_run_success)] = BoolGetDatum(false); values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_total_crashes)] = Int64GetDatum(1); values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_consecutive_crashes)] = Int32GetDatum(1); } else { values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_last_run_success)] = BoolGetDatum(true); values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_total_crashes)] = Int64GetDatum(0); values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_consecutive_crashes)] = Int32GetDatum(0); } ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_insert_values(rel, desc, values, nulls); ts_catalog_restore_user(&sec_ctx); return true; } void ts_bgw_job_stat_mark_start(BgwJob *job) { /* We grab a ShareRowExclusiveLock here because we need to ensure that no * job races and adds a job when we insert the relation as well since that * can trigger a failure when inserting a row for the job. We use the * RowExclusiveLock in the scan since we cannot use NoLock (relation_open * requires a lock that it not NoLock). */ Relation rel = table_open(catalog_get_table_id(ts_catalog_get(), BGW_JOB_STAT), ShareRowExclusiveLock); if (!bgw_job_stat_scan_job_id(job->fd.id, bgw_job_stat_tuple_mark_start, NULL, NULL, RowExclusiveLock)) bgw_job_stat_insert_relation(rel, job->fd.id, true, DT_NOBEGIN); table_close(rel, NoLock); /* We need to capture the execution start because failures are always logged */ job->job_history.execution_start = ts_timer_get_current_timestamp(); job->job_history.id = INVALID_BGW_JOB_STAT_HISTORY_ID; ts_bgw_job_stat_history_update(JOB_STAT_HISTORY_UPDATE_START, job, JOB_SUCCESS, NULL); pgstat_report_activity(STATE_IDLE, NULL); } void ts_bgw_job_stat_mark_end(BgwJob *job, JobResult result, Jsonb *edata) { JobResultCtx res = { .job = job, .result = result, }; if (!bgw_job_stat_scan_job_id(job->fd.id, bgw_job_stat_tuple_mark_end, NULL, &res, ShareRowExclusiveLock)) { if (!ts_bgw_job_find(job->fd.id, CurrentMemoryContext, false)) { elog(WARNING, "skipping job statistics update for job %d, job was deleted during execution", job->fd.id); return; } ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unable to find job statistics for job %d", job->fd.id))); } ts_bgw_job_stat_history_update(JOB_STAT_HISTORY_UPDATE_END, job, result, edata); pgstat_report_activity(STATE_IDLE, NULL); } void ts_bgw_job_stat_mark_crash_reported(BgwJob *job, JobResult result) { if (!bgw_job_stat_scan_job_id(job->fd.id, bgw_job_stat_tuple_mark_crash_reported, NULL, NULL, RowExclusiveLock)) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unable to find job statistics for job %d", job->fd.id))); } ts_bgw_job_stat_history_update(JOB_STAT_HISTORY_UPDATE_END, job, result, NULL); pgstat_report_activity(STATE_IDLE, NULL); } bool ts_bgw_job_stat_end_was_marked(BgwJobStat *jobstat) { return !TIMESTAMP_IS_NOBEGIN(jobstat->fd.last_finish); } TSDLLEXPORT void ts_bgw_job_stat_set_next_start(int32 job_id, TimestampTz next_start) { /* Cannot use DT_NOBEGIN as that's the value used to indicate "not set" */ if (next_start == DT_NOBEGIN) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot set next start to -infinity"))); } if (!bgw_job_stat_scan_job_id(job_id, bgw_job_stat_tuple_set_next_start, NULL, &next_start, ShareRowExclusiveLock)) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unable to find job statistics for job %d", job_id))); } } /* update next_start if job stat exists */ TSDLLEXPORT bool ts_bgw_job_stat_update_next_start(int32 job_id, TimestampTz next_start, bool allow_unset) { bool found = false; /* Cannot use DT_NOBEGIN as that's the value used to indicate "not set" */ if (!allow_unset && next_start == DT_NOBEGIN) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot set next start to -infinity"))); } found = bgw_job_stat_scan_job_id(job_id, bgw_job_stat_tuple_set_next_start, NULL, &next_start, ShareRowExclusiveLock); return found; } TSDLLEXPORT void ts_bgw_job_stat_upsert_next_start(int32 bgw_job_id, TimestampTz next_start) { /* Cannot use DT_NOBEGIN as that's the value used to indicate "not set" */ if (next_start == DT_NOBEGIN) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot set next start to -infinity"))); } /* We grab a ShareRowExclusiveLock here because we need to ensure that no * job races and adds a job when we insert the relation as well since that * can trigger a failure when inserting a row for the job. We use the * RowExclusiveLock in the scan since we cannot use NoLock (relation_open * requires a lock that it not NoLock). */ Relation rel = table_open(catalog_get_table_id(ts_catalog_get(), BGW_JOB_STAT), ShareRowExclusiveLock); if (!bgw_job_stat_scan_job_id(bgw_job_id, bgw_job_stat_tuple_set_next_start, NULL, &next_start, RowExclusiveLock)) bgw_job_stat_insert_relation(rel, bgw_job_id, false, next_start); table_close(rel, NoLock); } bool ts_bgw_job_stat_should_execute(BgwJobStat *jobstat, BgwJob *job) { /* * Stub to allow the system to disable jobs based on the number of crashes * or failures. */ return true; } TimestampTz ts_bgw_job_stat_next_start(BgwJobStat *jobstat, BgwJob *job, int32 consecutive_failed_launches) { /* give the system some room to breathe, wait before trying to launch again */ if (consecutive_failed_launches > 0) return calculate_next_start_on_failed_launch(consecutive_failed_launches, job); if (jobstat == NULL) /* Never previously run - run right away */ return DT_NOBEGIN; if (jobstat->fd.consecutive_crashes > 0) { /* Update the errors table regarding the crash */ if (!ts_flags_are_set_32(jobstat->fd.flags, LAST_CRASH_REPORTED)) { ts_bgw_job_stat_mark_crash_reported(job, JOB_FAILURE_IN_EXECUTION); } return calculate_next_start_on_crash(jobstat->fd.consecutive_crashes, job); } return jobstat->fd.next_start; } ================================================ FILE: src/bgw/job_stat.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include "job.h" #include "ts_catalog/catalog.h" #define JOB_STAT_FLAGS_DEFAULT 0 #define LAST_CRASH_REPORTED 1 typedef struct BgwJobStat { FormData_bgw_job_stat fd; } BgwJobStat; extern TSDLLEXPORT BgwJobStat *ts_bgw_job_stat_find(int job_id); extern void ts_bgw_job_stat_delete(int job_id); extern TSDLLEXPORT void ts_bgw_job_stat_mark_start(BgwJob *job); extern void ts_bgw_job_stat_mark_end(BgwJob *job, JobResult result, Jsonb *edata); extern bool ts_bgw_job_stat_end_was_marked(BgwJobStat *jobstat); extern TSDLLEXPORT void ts_bgw_job_stat_set_next_start(int32 job_id, TimestampTz next_start); extern TSDLLEXPORT bool ts_bgw_job_stat_update_next_start(int32 job_id, TimestampTz next_start, bool allow_unset); extern TSDLLEXPORT void ts_bgw_job_stat_upsert_next_start(int32 bgw_job_id, TimestampTz next_start); extern bool ts_bgw_job_stat_should_execute(BgwJobStat *jobstat, BgwJob *job); extern TimestampTz ts_bgw_job_stat_next_start(BgwJobStat *jobstat, BgwJob *job, int32 consecutive_failed_launches); extern TSDLLEXPORT void ts_bgw_job_stat_mark_crash_reported(BgwJob *job, JobResult result); extern TSDLLEXPORT TimestampTz ts_get_next_scheduled_execution_slot(BgwJob *job, TimestampTz finish_time); ================================================ FILE: src/bgw/job_stat_history.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <utils/jsonb.h> #include "compat/compat.h" #include "guc.h" #include "hypertable.h" #include "job_stat_history.h" #include "jsonb_utils.h" #include "timer.h" #include "utils.h" typedef struct BgwJobStatHistoryContext { JobResult result; BgwJobStatHistoryUpdateType update_type; BgwJob *job; Jsonb *edata; } BgwJobStatHistoryContext; static Jsonb * build_job_info(BgwJob *job) { JsonbParseState *parse_state = NULL; pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); /* all fields that is possible to change with `alter_job` API */ ts_jsonb_add_interval(parse_state, "schedule_interval", &job->fd.schedule_interval); ts_jsonb_add_interval(parse_state, "max_runtime", &job->fd.max_runtime); ts_jsonb_add_int32(parse_state, "max_retries", job->fd.max_retries); ts_jsonb_add_interval(parse_state, "retry_period", &job->fd.retry_period); ts_jsonb_add_str(parse_state, "proc_schema", NameStr(job->fd.proc_schema)); ts_jsonb_add_str(parse_state, "proc_name", NameStr(job->fd.proc_name)); ts_jsonb_add_str(parse_state, "owner", GetUserNameFromId(job->fd.owner, false)); ts_jsonb_add_bool(parse_state, "scheduled", job->fd.scheduled); ts_jsonb_add_bool(parse_state, "fixed_schedule", job->fd.fixed_schedule); if (job->fd.initial_start) ts_jsonb_add_interval(parse_state, "initial_start", &job->fd.retry_period); if (job->fd.hypertable_id != INVALID_HYPERTABLE_ID) ts_jsonb_add_int32(parse_state, "hypertable_id", job->fd.hypertable_id); if (job->fd.config != NULL) { /* config information jsonb*/ JsonbValue value = { 0 }; JsonbToJsonbValue(job->fd.config, &value); ts_jsonb_add_value(parse_state, "config", &value); } if (strlen(NameStr(job->fd.check_schema)) > 0) ts_jsonb_add_str(parse_state, "check_schema", NameStr(job->fd.check_schema)); if (strlen(NameStr(job->fd.check_name)) > 0) ts_jsonb_add_str(parse_state, "check_name", NameStr(job->fd.check_name)); if (job->fd.timezone != NULL) ts_jsonb_add_str(parse_state, "timezone", text_to_cstring(job->fd.timezone)); return JsonbValueToJsonb(pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL)); } static Jsonb * ts_bgw_job_stat_history_build_data_info(BgwJobStatHistoryContext *context) { JsonbParseState *parse_state = NULL; JsonbValue value = { 0 }; pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); Assert(context != NULL && context->job != NULL); /* job information jsonb */ JsonbToJsonbValue(build_job_info(context->job), &value); ts_jsonb_add_value(parse_state, "job", &value); if (context->edata != NULL) { /* error information jsonb */ JsonbToJsonbValue(context->edata, &value); ts_jsonb_add_value(parse_state, "error_data", &value); } return JsonbValueToJsonb(pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL)); } static void bgw_job_stat_history_insert(BgwJobStatHistoryContext *context, bool track_only_errors) { Assert(context != NULL); Relation rel = table_open(catalog_get_table_id(ts_catalog_get(), BGW_JOB_STAT_HISTORY), ShareRowExclusiveLock); TupleDesc desc = RelationGetDescr(rel); NullableDatum values[Natts_bgw_job_stat_history] = { { 0 } }; CatalogSecurityContext sec_ctx; ts_datum_set_int32(Anum_bgw_job_stat_history_job_id, values, context->job->fd.id, false); ts_datum_set_timestamptz(Anum_bgw_job_stat_history_execution_start, values, context->job->job_history.execution_start, false); if (track_only_errors) { /* In case of logging only ERRORs */ ts_datum_set_int32(Anum_bgw_job_stat_history_pid, values, MyProcPid, false); ts_datum_set_timestamptz(Anum_bgw_job_stat_history_execution_finish, values, ts_timer_get_current_timestamp(), false); ts_datum_set_bool(Anum_bgw_job_stat_history_succeeded, values, false, false); } else { /* When tracking history first we INSERT the job without the FINISH execution timestamp, * PID and SUCCEED flag because it will be marked once the job finishes */ ts_datum_set_int32(Anum_bgw_job_stat_history_pid, values, 0, true); ts_datum_set_timestamptz(Anum_bgw_job_stat_history_execution_finish, values, 0, true); ts_datum_set_bool(Anum_bgw_job_stat_history_succeeded, values, false, true); } ts_datum_set_jsonb(Anum_bgw_job_stat_history_data, values, ts_bgw_job_stat_history_build_data_info(context)); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); if (context->job->job_history.id == INVALID_BGW_JOB_STAT_HISTORY_ID) { /* We need to get a new job id to mark the end later */ context->job->job_history.id = ts_catalog_table_next_seq_id(ts_catalog_get(), BGW_JOB_STAT_HISTORY); } ts_datum_set_int64(Anum_bgw_job_stat_history_id, values, context->job->job_history.id, false); ts_catalog_insert_datums(rel, desc, values); ts_catalog_restore_user(&sec_ctx); table_close(rel, NoLock); } static void bgw_job_stat_history_mark_start(BgwJobStatHistoryContext *context) { /* Don't mark the start in case of the GUC be disabled */ if (!ts_guc_enable_job_execution_logging) return; bgw_job_stat_history_insert(context, false); } static void bgw_job_stat_history_update_entry(int64 bgw_job_history_id, tuple_found_func tuple_found, tuple_filter_func tuple_filter, void *data, LOCKMODE lockmode) { if (bgw_job_history_id == INVALID_BGW_JOB_STAT_HISTORY_ID) return; ScanKeyData scankey[1]; ScanKeyInit(&scankey[0], Anum_bgw_job_stat_history_pkey_idx_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(bgw_job_history_id)); Catalog *catalog = ts_catalog_get(); ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, BGW_JOB_STAT_HISTORY), .index = catalog_get_index(catalog, BGW_JOB_STAT_HISTORY, BGW_JOB_STAT_HISTORY_PKEY_IDX), .nkeys = 1, .scankey = scankey, .flags = SCANNER_F_KEEPLOCK, .tuple_found = tuple_found, .filter = tuple_filter, .data = data, .lockmode = lockmode, .scandirection = ForwardScanDirection, }; int num_found = ts_scanner_scan(&scanctx); /* We do not want to raise an error in case there is something wrong with history entries */ if (num_found == 0) /* This might happen due to job history retention deleting entries */ ereport(DEBUG1, (errmsg("could not find job stat history entry with id " INT64_FORMAT, bgw_job_history_id))); else if (num_found > 1) ereport(DEBUG1, (errmsg("found multiple job stat history entries with id " INT64_FORMAT, bgw_job_history_id))); } static ScanTupleResult bgw_job_stat_history_tuple_update(TupleInfo *ti, void *const data) { bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); BgwJobStatHistoryContext *context = (BgwJobStatHistoryContext *) data; Jsonb *job_history_data = NULL; Datum values[Natts_bgw_job_stat_history] = { 0 }; bool nulls[Natts_bgw_job_stat_history] = { 0 }; bool doReplace[Natts_bgw_job_stat_history] = { 0 }; switch (context->update_type) { case JOB_STAT_HISTORY_UPDATE_PID: { values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_history_pid)] = Int32GetDatum(MyProcPid); doReplace[AttrNumberGetAttrOffset(Anum_bgw_job_stat_history_pid)] = true; break; } case JOB_STAT_HISTORY_UPDATE_END: { values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_history_execution_finish)] = TimestampTzGetDatum(ts_timer_get_current_timestamp()); doReplace[AttrNumberGetAttrOffset(Anum_bgw_job_stat_history_execution_finish)] = true; values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_history_succeeded)] = BoolGetDatum((context->result == JOB_SUCCESS)); doReplace[AttrNumberGetAttrOffset(Anum_bgw_job_stat_history_succeeded)] = true; job_history_data = ts_bgw_job_stat_history_build_data_info(context); if (job_history_data != NULL) { values[AttrNumberGetAttrOffset(Anum_bgw_job_stat_history_data)] = JsonbPGetDatum(job_history_data); doReplace[AttrNumberGetAttrOffset(Anum_bgw_job_stat_history_data)] = true; } break; } case JOB_STAT_HISTORY_UPDATE_START: pg_unreachable(); break; } HeapTuple new_tuple = heap_modify_tuple(tuple, ts_scanner_get_tupledesc(ti), values, nulls, doReplace); ts_catalog_update(ti->scanrel, new_tuple); heap_freetuple(new_tuple); if (should_free) heap_freetuple(tuple); return SCAN_DONE; } static void bgw_job_stat_history_update(BgwJobStatHistoryContext *context) { /* Don't execute in case of the GUC is false and the job succeeded, because failures are always * logged */ if (!ts_guc_enable_job_execution_logging && context->result == JOB_SUCCESS) return; /* Re-read the job information because it can change during the execution by using the * `alter_job` API inside the function/procedure (i.e. job config) */ BgwJob *new_job = ts_bgw_job_find(context->job->fd.id, CurrentMemoryContext, true); /* Set the job history information */ new_job->job_history = context->job->job_history; /* Use the newly loaded job in the current context to use this information to register the * execution history */ context->job = new_job; /* Failures are always logged so in case of the GUC is false and a failure happens then we need * to insert all the information in the job error history table */ if (!ts_guc_enable_job_execution_logging && context->result != JOB_SUCCESS) { bgw_job_stat_history_insert(context, true); } else { /* Mark the end of the previous inserted start execution */ bgw_job_stat_history_update_entry(new_job->job_history.id, bgw_job_stat_history_tuple_update, NULL, context, RowExclusiveLock); } } void ts_bgw_job_stat_history_update(BgwJobStatHistoryUpdateType update_type, BgwJob *job, JobResult result, Jsonb *edata) { BgwJobStatHistoryContext context = { .result = result, .update_type = update_type, .job = job, .edata = edata, }; switch (update_type) { case JOB_STAT_HISTORY_UPDATE_START: bgw_job_stat_history_mark_start(&context); break; case JOB_STAT_HISTORY_UPDATE_END: case JOB_STAT_HISTORY_UPDATE_PID: bgw_job_stat_history_update(&context); break; } } ================================================ FILE: src/bgw/job_stat_history.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include "job.h" #include "job_stat.h" #include "ts_catalog/catalog.h" #define INVALID_BGW_JOB_STAT_HISTORY_ID 0 typedef enum BgwJobStatHistoryUpdateType { JOB_STAT_HISTORY_UPDATE_START, JOB_STAT_HISTORY_UPDATE_END, JOB_STAT_HISTORY_UPDATE_PID, } BgwJobStatHistoryUpdateType; extern void ts_bgw_job_stat_history_update(BgwJobStatHistoryUpdateType update_type, BgwJob *job, JobResult result, Jsonb *edata); ================================================ FILE: src/bgw/launcher_interface.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <fmgr.h> #include "compat/compat.h" #include "extension.h" #include "launcher_interface.h" #define MIN_LOADER_API_VERSION 4 extern bool ts_bgw_worker_reserve(void) { PGFunction reserve = load_external_function(EXTENSION_SO, "ts_bgw_worker_reserve", true, NULL); return DatumGetBool( DirectFunctionCall1(reserve, BoolGetDatum(false))); /* no function call zero */ } extern void ts_bgw_worker_release(void) { PGFunction release = load_external_function(EXTENSION_SO, "ts_bgw_worker_release", true, NULL); DirectFunctionCall1(release, BoolGetDatum(false)); /* no function call zero */ } extern int ts_bgw_num_unreserved(void) { PGFunction unreserved = load_external_function(EXTENSION_SO, "ts_bgw_num_unreserved", true, NULL); return DatumGetInt32( DirectFunctionCall1(unreserved, BoolGetDatum(false))); /* no function call zero */ } extern int ts_bgw_loader_api_version(void) { void **versionptr = find_rendezvous_variable(RENDEZVOUS_BGW_LOADER_API_VERSION); if (*versionptr == NULL) return 0; return *((int32 *) *versionptr); } extern void ts_bgw_check_loader_api_version() { int version = ts_bgw_loader_api_version(); if (version < MIN_LOADER_API_VERSION) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("loader version out-of-date"), errhint("Please restart the database to upgrade the loader version."))); } ================================================ FILE: src/bgw/launcher_interface.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> extern bool ts_bgw_worker_reserve(void); extern void ts_bgw_worker_release(void); extern int ts_bgw_num_unreserved(void); extern int ts_bgw_loader_api_version(void); extern void ts_bgw_check_loader_api_version(void); ================================================ FILE: src/bgw/scheduler.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ /* * This is a scheduler that takes background jobs and schedules them appropriately * * Limitations: For now the jobs are only loaded when the scheduler starts and are not * updated if the jobs table changes * */ #include <postgres.h> #include <access/xact.h> #include <miscadmin.h> #include <nodes/pg_list.h> #include <pgstat.h> #include <postmaster/bgworker.h> #include <storage/ipc.h> #include <storage/latch.h> #include <storage/lwlock.h> #include <storage/proc.h> #include <storage/shmem.h> #include <tcop/tcopprot.h> #include <utils/acl.h> #include <utils/inval.h> #include <utils/jsonb.h> #include <utils/memutils.h> #include <utils/snapmgr.h> #include <utils/timestamp.h> #include "compat/compat.h" #include "extension.h" #include "guc.h" #include "job.h" #include "job_stat.h" #include "launcher_interface.h" #include "scheduler.h" #include "timer.h" #include "version.h" #include "worker.h" #define START_RETRY_MS (1 * INT64CONST(1000)) /* 1 seconds */ #define ONE_SECOND_IN_MICROSECONDS 1000000 static TimestampTz least_timestamp(TimestampTz left, TimestampTz right) { return (left < right ? left : right); } TS_FUNCTION_INFO_V1(ts_bgw_scheduler_main); /* * Global so the invalidate cache message can set. Don't need to protect * access with a lock because it's accessed only by the scheduler process. */ static bool jobs_list_needs_update; /* has to be global to shutdown jobs on exit */ static List *scheduled_jobs = NIL; static MemoryContext scheduler_mctx; static MemoryContext scratch_mctx; /* See the README for a state transition diagram */ typedef enum JobState { /* terminal state for now. Later we may have path to JOB_STATE_SCHEDULED */ JOB_STATE_DISABLED, /* * This is the initial state. next states: JOB_STATE_STARTED, * JOB_STATE_DISABLED. This job is not running and has been scheduled to * be started at a later time. */ JOB_STATE_SCHEDULED, /* * next states: JOB_STATE_TERMINATING, JOB_STATE_SCHEDULED. This job has * been started by the scheduler and is either running or finished (and * the finish has not yet been detected by the scheduler). */ JOB_STATE_STARTED, /* * next states: JOB_STATE_SCHEDULED. The scheduler has explicitly sent a * terminate to this job but has not yet detected that it has stopped. */ JOB_STATE_TERMINATING } JobState; typedef struct ScheduledBgwJob { BgwJob job; TimestampTz next_start; TimestampTz timeout_at; JobState state; BackgroundWorkerHandle *handle; bool reserved_worker; /* * We say "may" here since under normal circumstances the job itself will * perform the mark_end */ bool may_need_mark_end; int32 consecutive_failed_launches; } ScheduledBgwJob; static void on_failure_to_start_job(ScheduledBgwJob *sjob); static volatile sig_atomic_t got_SIGHUP = false; BackgroundWorkerHandle * ts_bgw_start_worker(const char *name, const BgwParams *bgw_params) { BackgroundWorker worker = { .bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION, .bgw_start_time = BgWorkerStart_RecoveryFinished, .bgw_restart_time = BGW_NEVER_RESTART, .bgw_notify_pid = MyProcPid, .bgw_main_arg = ObjectIdGetDatum(MyDatabaseId), }; BackgroundWorkerHandle *handle = NULL; strlcpy(worker.bgw_name, name, BGW_MAXLEN); strlcpy(worker.bgw_library_name, ts_extension_get_so_name(), BGW_MAXLEN); strlcpy(worker.bgw_function_name, bgw_params->bgw_main, BGW_MAXLEN); memcpy(worker.bgw_extra, bgw_params, sizeof(*bgw_params)); /* handle needs to be allocated in long-lived memory context */ MemoryContextSwitchTo(scheduler_mctx); if (!RegisterDynamicBackgroundWorker(&worker, &handle)) { elog(NOTICE, "unable to register background worker"); handle = NULL; } MemoryContextSwitchTo(scratch_mctx); return handle; } #ifdef USE_ASSERT_CHECKING static void assert_that_worker_has_stopped(ScheduledBgwJob *sjob) { pid_t pid; BgwHandleStatus status; Assert(sjob->reserved_worker); status = GetBackgroundWorkerPid(sjob->handle, &pid); Assert(BGWH_STOPPED == status); } #endif static void mark_job_as_started(ScheduledBgwJob *sjob) { Assert(!sjob->may_need_mark_end); sjob->consecutive_failed_launches = 0; ts_bgw_job_stat_mark_start(&sjob->job); sjob->may_need_mark_end = true; } static void mark_job_as_ended(ScheduledBgwJob *sjob, JobResult res, Jsonb *edata) { Assert(sjob->may_need_mark_end); ts_bgw_job_stat_mark_end(&sjob->job, res, edata); sjob->may_need_mark_end = false; } static ErrorData * makeJobErrorData(ScheduledBgwJob *sjob, JobResult res) { ErrorData *edata = (ErrorData *) palloc0(sizeof(ErrorData)); edata->elevel = ERROR; edata->sqlerrcode = ERRCODE_INTERNAL_ERROR; edata->hint = NULL; Assert(res != JOB_SUCCESS); switch (res) { case JOB_FAILURE_TO_START: edata->message = "failed to start job"; edata->detail = psprintf("Job %d (\"%s\") failed to start", sjob->job.fd.id, NameStr(sjob->job.fd.application_name)); break; case JOB_FAILURE_IN_EXECUTION: edata->message = "failed to execute job"; edata->detail = psprintf("Job %d (\"%s\") failed to execute.", sjob->job.fd.id, NameStr(sjob->job.fd.application_name)); break; default: pg_unreachable(); break; } return edata; } static void worker_state_cleanup(ScheduledBgwJob *sjob) { /* * This function needs to be safe wrt failures occurring at any point in * the job starting process. */ if (sjob->handle != NULL) { #ifdef USE_ASSERT_CHECKING /* Sanity check: worker has stopped (if it was started) */ assert_that_worker_has_stopped(sjob); #endif pfree(sjob->handle); sjob->handle = NULL; } /* * first cleanup reserved workers before accessing db. Want to minimize * the possibility of errors before worker is released */ if (sjob->reserved_worker) { ts_bgw_worker_release(); sjob->reserved_worker = false; } if (sjob->may_need_mark_end) { BgwJobStat *job_stat; if (!ts_bgw_job_find(sjob->job.fd.id, CurrentMemoryContext, false)) { elog(WARNING, "scheduler detected that job %d was deleted after job quit", sjob->job.fd.id); ts_bgw_job_cache_invalidate_callback(); sjob->may_need_mark_end = false; return; } job_stat = ts_bgw_job_stat_find(sjob->job.fd.id); if (job_stat && !ts_bgw_job_stat_end_was_marked(job_stat)) { /* * Usually the job process will mark the end, but if the job gets * a signal (cancel or terminate), it won't be able to so we * should. */ elog(LOG, "job %d failed", sjob->job.fd.id); ErrorData *edata = makeJobErrorData(sjob, JOB_FAILURE_IN_EXECUTION); mark_job_as_ended(sjob, JOB_FAILURE_IN_EXECUTION, ts_errdata_to_jsonb(edata, &sjob->job.fd.proc_schema, &sjob->job.fd.proc_name)); } else { sjob->may_need_mark_end = false; } } } /* Set the state of the job. * This function is responsible for setting all of the variables in ScheduledBgwJob * except for the job itself. */ static void scheduled_bgw_job_transition_state_to(ScheduledBgwJob *sjob, JobState new_state) { #ifdef USE_ASSERT_CHECKING JobState prev_state = sjob->state; #endif BgwJobStat *job_stat; switch (new_state) { case JOB_STATE_DISABLED: Assert(prev_state == JOB_STATE_STARTED || prev_state == JOB_STATE_TERMINATING); sjob->handle = NULL; break; case JOB_STATE_SCHEDULED: /* prev_state can be any value, including itself */ worker_state_cleanup(sjob); job_stat = ts_bgw_job_stat_find(sjob->job.fd.id); Assert(!sjob->reserved_worker); sjob->next_start = ts_bgw_job_stat_next_start(job_stat, &sjob->job, sjob->consecutive_failed_launches); break; case JOB_STATE_STARTED: Assert(prev_state == JOB_STATE_SCHEDULED); Assert(sjob->handle == NULL); Assert(!sjob->reserved_worker); StartTransactionCommand(); PushActiveSnapshot(GetTransactionSnapshot()); if (!ts_bgw_job_find(sjob->job.fd.id, CurrentMemoryContext, false)) { elog(WARNING, "scheduler detected that job %d was deleted when starting job", sjob->job.fd.id); ts_bgw_job_cache_invalidate_callback(); CommitTransactionCommand(); MemoryContextSwitchTo(scratch_mctx); return; } /* If we are unable to reserve a worker go back to the scheduled state */ sjob->reserved_worker = ts_bgw_worker_reserve(); if (!sjob->reserved_worker) { elog(WARNING, "failed to launch job %d \"%s\": out of background workers", sjob->job.fd.id, NameStr(sjob->job.fd.application_name)); sjob->consecutive_failed_launches++; scheduled_bgw_job_transition_state_to(sjob, JOB_STATE_SCHEDULED); PopActiveSnapshot(); CommitTransactionCommand(); MemoryContextSwitchTo(scratch_mctx); return; } /* * start the job before you can encounter any errors so that they * are always registered */ mark_job_as_started(sjob); if (ts_bgw_job_has_timeout(&sjob->job)) sjob->timeout_at = ts_bgw_job_timeout_at(&sjob->job, ts_timer_get_current_timestamp()); else sjob->timeout_at = DT_NOEND; PopActiveSnapshot(); CommitTransactionCommand(); MemoryContextSwitchTo(scratch_mctx); elog(DEBUG1, "launching job %d \"%s\"", sjob->job.fd.id, NameStr(sjob->job.fd.application_name)); sjob->handle = ts_bgw_job_start(&sjob->job, sjob->job.fd.owner); if (sjob->handle == NULL) { elog(WARNING, "failed to launch job %d \"%s\": failed to start a background worker", sjob->job.fd.id, NameStr(sjob->job.fd.application_name)); on_failure_to_start_job(sjob); return; } Assert(sjob->reserved_worker); break; case JOB_STATE_TERMINATING: Assert(prev_state == JOB_STATE_STARTED); Assert(sjob->handle != NULL); Assert(sjob->reserved_worker); TerminateBackgroundWorker(sjob->handle); break; } sjob->state = new_state; } static void on_failure_to_start_job(ScheduledBgwJob *sjob) { StartTransactionCommand(); PushActiveSnapshot(GetTransactionSnapshot()); if (!ts_bgw_job_find(sjob->job.fd.id, CurrentMemoryContext, false)) { elog(WARNING, "scheduler detected that job %d was deleted while failing to start", sjob->job.fd.id); ts_bgw_job_cache_invalidate_callback(); } else { /* restore the original next_start to maintain priority (it is unset during mark_start) */ if (sjob->next_start != DT_NOBEGIN) ts_bgw_job_stat_set_next_start(sjob->job.fd.id, sjob->next_start); ErrorData *edata = makeJobErrorData(sjob, JOB_FAILURE_TO_START); mark_job_as_ended(sjob, JOB_FAILURE_TO_START, ts_errdata_to_jsonb(edata, &sjob->job.fd.proc_schema, &sjob->job.fd.proc_name)); } scheduled_bgw_job_transition_state_to(sjob, JOB_STATE_SCHEDULED); PopActiveSnapshot(); CommitTransactionCommand(); MemoryContextSwitchTo(scratch_mctx); } static inline void bgw_scheduler_on_postmaster_death(void) { /* * Don't call exit hooks cause we want to bail out quickly. We don't care * about cleaning up shared memory in this case anyway since it's * potentially corrupt. */ on_exit_reset(); ereport(FATAL, (errcode(ERRCODE_ADMIN_SHUTDOWN), errmsg("postmaster exited while TimescaleDB scheduler was working"))); } /* * This function starts a job. * To correctly count crashes we need to mark the start of a job in a separate * txn before we kick off the actual job. Thus this function cannot be run * from within a transaction. */ static void scheduled_ts_bgw_job_start(ScheduledBgwJob *sjob, register_background_worker_callback_type bgw_register) { pid_t pid; BgwHandleStatus status; scheduled_bgw_job_transition_state_to(sjob, JOB_STATE_STARTED); if (sjob->state != JOB_STATE_STARTED) return; Assert(sjob->handle != NULL); if (bgw_register != NULL) bgw_register(sjob->handle, scheduler_mctx); status = WaitForBackgroundWorkerStartup(sjob->handle, &pid); switch (status) { case BGWH_POSTMASTER_DIED: bgw_scheduler_on_postmaster_death(); break; case BGWH_STARTED: /* all good */ break; case BGWH_STOPPED: StartTransactionCommand(); scheduled_bgw_job_transition_state_to(sjob, JOB_STATE_SCHEDULED); CommitTransactionCommand(); MemoryContextSwitchTo(scratch_mctx); break; case BGWH_NOT_YET_STARTED: /* should not be possible */ elog(ERROR, "unexpected bgworker state %d", status); break; } } static void terminate_and_cleanup_job(ScheduledBgwJob *sjob) { if (sjob->handle != NULL) { pid_t pid; BgwHandleStatus status; status = GetBackgroundWorkerPid(sjob->handle, &pid); if (status == BGWH_STARTED) { /* Try graceful cancellation first (SIGINT via pg_cancel_backend) */ DirectFunctionCall1(pg_cancel_backend, Int32GetDatum(pid)); /* Poll for up to 3 seconds for the worker to exit */ for (int i = 0; i < 30; i++) { pg_usleep(100000); /* 100ms */ status = GetBackgroundWorkerPid(sjob->handle, &pid); if (status == BGWH_STOPPED) break; } if (status != BGWH_STOPPED) { elog(WARNING, "job %d did not exit after SIGINT, sending SIGTERM", sjob->job.fd.id); TerminateBackgroundWorker(sjob->handle); WaitForBackgroundWorkerShutdown(sjob->handle); } } else if (status != BGWH_STOPPED) { TerminateBackgroundWorker(sjob->handle); WaitForBackgroundWorkerShutdown(sjob->handle); } } sjob->may_need_mark_end = false; worker_state_cleanup(sjob); } /* * Update the given job list with whatever is in the bgw_job table. For overlapping jobs, * copy over any existing scheduler info from the given jobs list. * Assume that both lists are ordered by job ID. * Note that this function call will destroy cur_jobs_list and return a new list. */ List * ts_update_scheduled_jobs_list(List *cur_jobs_list, MemoryContext mctx) { List *new_jobs = ts_bgw_job_get_scheduled(sizeof(ScheduledBgwJob), mctx); ListCell *new_ptr = list_head(new_jobs); ListCell *cur_ptr = list_head(cur_jobs_list); elog(DEBUG2, "updating scheduled jobs list"); while (cur_ptr != NULL && new_ptr != NULL) { ScheduledBgwJob *new_sjob = lfirst(new_ptr); ScheduledBgwJob *cur_sjob = lfirst(cur_ptr); if (cur_sjob->job.fd.id < new_sjob->job.fd.id) { /* * We don't need cur_sjob anymore. Make sure to clean up the job * state. Then keep advancing cur pointer until we catch up. */ terminate_and_cleanup_job(cur_sjob); cur_ptr = lnext(cur_jobs_list, cur_ptr); continue; } if (cur_sjob->job.fd.id == new_sjob->job.fd.id) { /* * Then this job already exists. Copy over any state and advance * both pointers. */ cur_sjob->job = new_sjob->job; *new_sjob = *cur_sjob; /* reload the scheduling information from the job_stats */ if (cur_sjob->state == JOB_STATE_SCHEDULED) scheduled_bgw_job_transition_state_to(new_sjob, JOB_STATE_SCHEDULED); cur_ptr = lnext(cur_jobs_list, cur_ptr); new_ptr = lnext(new_jobs, new_ptr); } else if (cur_sjob->job.fd.id > new_sjob->job.fd.id) { scheduled_bgw_job_transition_state_to(new_sjob, JOB_STATE_SCHEDULED); elog(DEBUG1, "sjob %d was new, its fixed_schedule is %d", new_sjob->job.fd.id, new_sjob->job.fd.fixed_schedule); /* Advance the new_job list until we catch up to cur_list */ new_ptr = lnext(new_jobs, new_ptr); } } /* If there's more stuff in cur_list, clean it all up */ if (cur_ptr != NULL) { ListCell *ptr; for_each_cell (ptr, cur_jobs_list, cur_ptr) terminate_and_cleanup_job(lfirst(ptr)); } if (new_ptr != NULL) { /* Then there are more new jobs. Initialize all of them. */ ListCell *ptr; for_each_cell (ptr, new_jobs, new_ptr) scheduled_bgw_job_transition_state_to(lfirst(ptr), JOB_STATE_SCHEDULED); } /* Free the old list */ list_free_deep(cur_jobs_list); return new_jobs; } #ifdef TS_DEBUG /* Only used by test code */ void ts_populate_scheduled_job_tuple(ScheduledBgwJob *sjob, Datum *values) { if (sjob == NULL) return; values[0] = Int32GetDatum(sjob->job.fd.id); values[1] = NameGetDatum(&sjob->job.fd.application_name); values[2] = IntervalPGetDatum(&sjob->job.fd.schedule_interval); values[3] = IntervalPGetDatum(&sjob->job.fd.max_runtime); values[4] = Int32GetDatum(sjob->job.fd.max_retries); values[5] = IntervalPGetDatum(&sjob->job.fd.retry_period); values[6] = TimestampTzGetDatum(sjob->next_start); values[7] = TimestampTzGetDatum(sjob->timeout_at); values[8] = BoolGetDatum(sjob->reserved_worker); values[9] = BoolGetDatum(sjob->may_need_mark_end); } #endif static int cmp_next_start(const ListCell *left_cell, const ListCell *right_cell) { ScheduledBgwJob *left_sjob = lfirst(left_cell); ScheduledBgwJob *right_sjob = lfirst(right_cell); if (left_sjob->next_start < right_sjob->next_start) return -1; if (left_sjob->next_start > right_sjob->next_start) return 1; return 0; } static void start_scheduled_jobs(register_background_worker_callback_type bgw_register) { List *ordered_scheduled_jobs; ListCell *lc; Assert(CurrentMemoryContext == scratch_mctx); /* Order jobs by increasing next_start */ /* list_sort does in-place sort - so make a copy and sort that */ ordered_scheduled_jobs = list_copy(scheduled_jobs); list_sort(ordered_scheduled_jobs, cmp_next_start); foreach (lc, ordered_scheduled_jobs) { ScheduledBgwJob *sjob = lfirst(lc); int64 job_start_diff = sjob->next_start - ts_timer_get_current_timestamp(); if (sjob->state == JOB_STATE_SCHEDULED && (job_start_diff <= 0 || sjob->next_start == DT_NOBEGIN)) { elog(DEBUG2, "starting scheduled job %d", sjob->job.fd.id); scheduled_ts_bgw_job_start(sjob, bgw_register); } else { elog(DEBUG5, "starting scheduled job %d in " INT64_FORMAT " seconds", sjob->job.fd.id, job_start_diff / ONE_SECOND_IN_MICROSECONDS); } } list_free(ordered_scheduled_jobs); } /* Returns the earliest time the scheduler should start a job that is waiting to be started */ static TimestampTz earliest_wakeup_to_start_next_job() { ListCell *lc; TimestampTz earliest = DT_NOEND; TimestampTz now = ts_timer_get_current_timestamp(); foreach (lc, scheduled_jobs) { ScheduledBgwJob *sjob = lfirst(lc); if (sjob->state == JOB_STATE_SCHEDULED) { TimestampTz start = sjob->next_start; /* if the start is less than now, this means we tried and failed to start it already, so * use the retry period */ if (start < now) start = TimestampTzPlusMilliseconds(now, START_RETRY_MS); earliest = least_timestamp(earliest, start); } } return earliest; } /* Returns the earliest time the scheduler needs to kill a job according to its timeout */ static TimestampTz earliest_job_timeout() { ListCell *lc; TimestampTz earliest = DT_NOEND; foreach (lc, scheduled_jobs) { ScheduledBgwJob *sjob = lfirst(lc); if (sjob->state == JOB_STATE_STARTED) earliest = least_timestamp(earliest, sjob->timeout_at); } return earliest; } /* Special exit function only used in shmem_exit_callback. * Do not call the normal cleanup function (worker_state_cleanup), because * 1) we do not wait for the BGW to terminate, * 2) we cannot access the database at this time, so we should not be * trying to update the bgw_stat table. */ static void terminate_all_jobs_and_release_workers() { ListCell *lc; foreach (lc, scheduled_jobs) { ScheduledBgwJob *sjob = lfirst(lc); /* * Clean up the background workers. Don't worry about state of the * sjobs, because this callback might have interrupted a state * transition. */ if (sjob->handle != NULL) TerminateBackgroundWorker(sjob->handle); if (sjob->reserved_worker) { ts_bgw_worker_release(); sjob->reserved_worker = false; } } } static void wait_for_all_jobs_to_shutdown() { ListCell *lc; foreach (lc, scheduled_jobs) { ScheduledBgwJob *sjob = lfirst(lc); if (sjob->state == JOB_STATE_STARTED || sjob->state == JOB_STATE_TERMINATING) WaitForBackgroundWorkerShutdown(sjob->handle); } } static void check_for_stopped_and_timed_out_jobs() { ListCell *lc; foreach (lc, scheduled_jobs) { BgwHandleStatus status; pid_t pid; ScheduledBgwJob *sjob = lfirst(lc); TimestampTz now = ts_timer_get_current_timestamp(); if (sjob->state != JOB_STATE_STARTED && sjob->state != JOB_STATE_TERMINATING) continue; status = GetBackgroundWorkerPid(sjob->handle, &pid); switch (status) { case BGWH_POSTMASTER_DIED: bgw_scheduler_on_postmaster_death(); break; case BGWH_NOT_YET_STARTED: elog(ERROR, "unexpected bgworker state %d", status); break; case BGWH_STARTED: /* still running */ if (sjob->state == JOB_STATE_STARTED && now >= sjob->timeout_at) { elog(WARNING, "terminating background worker \"%s\" due to timeout", NameStr(sjob->job.fd.application_name)); scheduled_bgw_job_transition_state_to(sjob, JOB_STATE_TERMINATING); Assert(sjob->state != JOB_STATE_STARTED); } break; case BGWH_STOPPED: StartTransactionCommand(); PushActiveSnapshot(GetTransactionSnapshot()); scheduled_bgw_job_transition_state_to(sjob, JOB_STATE_SCHEDULED); PopActiveSnapshot(); CommitTransactionCommand(); MemoryContextSwitchTo(scratch_mctx); Assert(sjob->state != JOB_STATE_STARTED); break; } } } /* This is the guts of the scheduler which runs the main loop. * The parameter ttl_ms gives a maximum time to run the loop (after which * the loop will exit). This functionality is used to ease testing. * In production, ttl_ms should be < 0 to signal that the loop should * run forever (or until the process gets a signal). * * The scheduler uses 2 memory contexts for its operation: scheduler_mctx * for long-lived objects and scratch_mctx for short-lived objects. * After every iteration of the scheduling main loop scratch_mctx gets * reset. Special care needs to be taken in regards to memory contexts * since StartTransactionCommand creates and switches to a transaction * memory context which gets deleted on CommitTransactionCommand which * switches CurrentMemoryContext back to TopMemoryContext. So operations * wrapped in Start/CommitTransactionCommit will not happen in scratch_mctx * but will get freed on CommitTransactionCommand. */ void ts_bgw_scheduler_process(int32 run_for_interval_ms, register_background_worker_callback_type bgw_register) { TimestampTz start = ts_timer_get_current_timestamp(); TimestampTz quit_time = DT_NOEND; log_min_messages = ts_guc_bgw_log_level; pgstat_report_activity(STATE_RUNNING, NULL); /* If we are restoring or upgrading, don't schedule anything. Just * exit. */ if (ts_guc_restoring || IsBinaryUpgrade) { ereport(LOG, errmsg("scheduler for database %u exiting with exit status %d", MyDatabaseId, ts_debug_bgw_scheduler_exit_status), errdetail("the database is restoring or upgrading")); terminate_all_jobs_and_release_workers(); goto scheduler_exit; } /* txn to read the list of jobs from the DB */ StartTransactionCommand(); PushActiveSnapshot(GetTransactionSnapshot()); scheduled_jobs = ts_update_scheduled_jobs_list(scheduled_jobs, scheduler_mctx); PopActiveSnapshot(); CommitTransactionCommand(); MemoryContextSwitchTo(scratch_mctx); jobs_list_needs_update = false; if (run_for_interval_ms > 0) quit_time = TimestampTzPlusMilliseconds(start, run_for_interval_ms); elog(DEBUG1, "database scheduler for database %u starting", MyDatabaseId); /* * on SIGTERM the process will usually die from the CHECK_FOR_INTERRUPTS * in the die() called from the sigterm handler. Child reaping is then * handled in the before_shmem_exit, * bgw_scheduler_before_shmem_exit_callback. */ while (quit_time > ts_timer_get_current_timestamp() && !ProcDiePending && !ts_shutdown_bgw) { TimestampTz next_wakeup = quit_time; Assert(CurrentMemoryContext == scratch_mctx); /* start jobs, and then check when to next wake up */ elog(DEBUG5, "scheduler wakeup in database %u", MyDatabaseId); start_scheduled_jobs(bgw_register); next_wakeup = least_timestamp(next_wakeup, earliest_wakeup_to_start_next_job()); next_wakeup = least_timestamp(next_wakeup, earliest_job_timeout()); pgstat_report_activity(STATE_IDLE, NULL); ts_timer_wait(next_wakeup); pgstat_report_activity(STATE_RUNNING, NULL); CHECK_FOR_INTERRUPTS(); if (got_SIGHUP) { got_SIGHUP = false; ProcessConfigFile(PGC_SIGHUP); log_min_messages = ts_guc_bgw_log_level; } /* * Process any cache invalidation message that indicates we need to * update the jobs list */ AcceptInvalidationMessages(); if (jobs_list_needs_update) { StartTransactionCommand(); Assert(CurrentMemoryContext == CurTransactionContext); scheduled_jobs = ts_update_scheduled_jobs_list(scheduled_jobs, scheduler_mctx); CommitTransactionCommand(); MemoryContextSwitchTo(scratch_mctx); jobs_list_needs_update = false; } check_for_stopped_and_timed_out_jobs(); MemoryContextReset(scratch_mctx); } elog(DEBUG1, "scheduler for database %u exiting with exit status %d", MyDatabaseId, ts_debug_bgw_scheduler_exit_status); #ifdef TS_DEBUG if (ts_shutdown_bgw) elog(WARNING, "bgw scheduler stopped due to shutdown_bgw guc"); #endif scheduler_exit: CHECK_FOR_INTERRUPTS(); wait_for_all_jobs_to_shutdown(); check_for_stopped_and_timed_out_jobs(); scheduled_jobs = NIL; proc_exit(ts_debug_bgw_scheduler_exit_status); } static void bgw_scheduler_before_shmem_exit_callback(int code, Datum arg) { terminate_all_jobs_and_release_workers(); } void ts_bgw_scheduler_setup_callbacks() { before_shmem_exit(bgw_scheduler_before_shmem_exit_callback, PointerGetDatum(NULL)); } /* some of the scheduler mock code calls functions from this file without going through * the main loop so we need a way to setup the memory contexts */ void ts_bgw_scheduler_setup_mctx() { scheduler_mctx = AllocSetContextCreate(TopMemoryContext, "Scheduler", ALLOCSET_DEFAULT_SIZES); scratch_mctx = AllocSetContextCreate(scheduler_mctx, "SchedulerScratch", ALLOCSET_DEFAULT_SIZES); MemoryContextSwitchTo(scratch_mctx); } static void handle_sighup(SIGNAL_ARGS) { /* based on av_sighup_handler */ int save_errno = errno; got_SIGHUP = true; SetLatch(MyLatch); errno = save_errno; } /* * Register SIGTERM and SIGHUP handlers for bgw_scheduler. * This function _must_ be called with signals blocked, i.e., after calling * BackgroundWorkerBlockSignals */ void ts_bgw_scheduler_register_signal_handlers(void) { /* * do not use the default `bgworker_die` sigterm handler because it does * not respect critical sections */ pqsignal(SIGTERM, die); pqsignal(SIGHUP, handle_sighup); /* Some SIGHUPS may already have been dropped, so we must load the file here */ got_SIGHUP = false; ProcessConfigFile(PGC_SIGHUP); log_min_messages = ts_guc_bgw_log_level; } Datum ts_bgw_scheduler_main(PG_FUNCTION_ARGS) { BackgroundWorkerBlockSignals(); /* Setup any signal handlers here */ ts_bgw_scheduler_register_signal_handlers(); BackgroundWorkerUnblockSignals(); ts_bgw_scheduler_setup_callbacks(); pgstat_report_appname(SCHEDULER_APPNAME); ts_bgw_scheduler_setup_mctx(); ts_bgw_scheduler_process(-1, NULL); Assert(scheduled_jobs == NIL); MemoryContextSwitchTo(TopMemoryContext); MemoryContextDelete(scheduler_mctx); PG_RETURN_VOID(); }; void ts_bgw_job_cache_invalidate_callback() { jobs_list_needs_update = true; } ================================================ FILE: src/bgw/scheduler.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include "compat/compat.h" #include <postgres.h> #include <fmgr.h> #include <postmaster/bgworker.h> #include "timer.h" #include "worker.h" typedef struct ScheduledBgwJob ScheduledBgwJob; /* callback used in testing */ typedef void (*register_background_worker_callback_type)(BackgroundWorkerHandle *, MemoryContext scheduler_ctx); /* Exposed for testing */ extern List *ts_update_scheduled_jobs_list(List *cur_jobs_list, MemoryContext mctx); #ifdef TS_DEBUG extern void ts_populate_scheduled_job_tuple(ScheduledBgwJob *sjob, Datum *values); #endif extern void ts_bgw_scheduler_process(int32 run_for_interval_ms, register_background_worker_callback_type bgw_register); /* exposed for access by mock */ extern void ts_bgw_scheduler_setup_callbacks(void); extern void ts_bgw_job_cache_invalidate_callback(void); extern void ts_bgw_scheduler_register_signal_handlers(void); extern void ts_bgw_scheduler_setup_mctx(void); extern BackgroundWorkerHandle *ts_bgw_start_worker(const char *name, const BgwParams *bgw_params); ================================================ FILE: src/bgw/timer.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <miscadmin.h> #include <pgstat.h> #include <postmaster/bgworker.h> #include <storage/ipc.h> #include <storage/latch.h> #include <storage/lwlock.h> #include <storage/proc.h> #include <storage/shmem.h> #include <utils/jsonb.h> #include <utils/memutils.h> #include <utils/snapmgr.h> #include <utils/timestamp.h> #include "compat/compat.h" #include "config.h" #include "timer.h" #define MAX_TIMEOUT (5 * INT64CONST(1000)) #define MILLISECS_PER_SEC INT64CONST(1000) #define USECS_PER_MILLISEC INT64CONST(1000) static inline void on_postmaster_death(void) { /* * Don't call exit hooks cause we want to bail out quickly. We don't care * about cleaning up shared memory in this case anyway since it's * potentially corrupt. */ on_exit_reset(); ereport(FATAL, (errcode(ERRCODE_ADMIN_SHUTDOWN), errmsg("postmaster exited while timescaledb scheduler was working"))); } static int64 get_timeout_millisec(TimestampTz by_time) { long timeout_sec = 0; int timeout_usec = 0; if (TIMESTAMP_IS_NOBEGIN(by_time)) return 0; if (TIMESTAMP_IS_NOEND(by_time)) return PG_INT64_MAX; TimestampDifference(GetCurrentTimestamp(), by_time, &timeout_sec, &timeout_usec); if (timeout_sec < 0 || timeout_usec < 0) return 0; return (int64) ((timeout_sec * MILLISECS_PER_SEC) + (((int64) timeout_usec) / USECS_PER_MILLISEC)); } static bool wait_using_wait_latch(TimestampTz until) { int wl_rc; int64 timeout = get_timeout_millisec(until); Assert(timeout >= 0 && "get_timeout_millisec underflow"); if (timeout > MAX_TIMEOUT) timeout = MAX_TIMEOUT; /* Wait latch requires timeout to be <= INT_MAX */ if (timeout > (int64) INT_MAX) timeout = INT_MAX; wl_rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, timeout, PG_WAIT_EXTENSION); ResetLatch(MyLatch); if (wl_rc & WL_POSTMASTER_DEATH) on_postmaster_death(); return true; } static const Timer standard_timer = { .get_current_timestamp = GetCurrentTimestamp, .wait = wait_using_wait_latch, }; static const Timer *current_timer_implementation = &standard_timer; static inline const Timer * timer_get() { return current_timer_implementation; } bool ts_timer_wait(TimestampTz until) { return timer_get()->wait(until); } TimestampTz ts_timer_get_current_timestamp() { return timer_get()->get_current_timestamp(); } #ifdef TS_DEBUG void ts_timer_set(const Timer *timer) { current_timer_implementation = timer; } const Timer * ts_get_standard_timer() { return &standard_timer; } #endif ================================================ FILE: src/bgw/timer.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <utils/timestamp.h> #include "config.h" #include "export.h" typedef struct Timer { TimestampTz (*get_current_timestamp)(); bool (*wait)(TimestampTz until); } Timer; extern bool ts_timer_wait(TimestampTz until); extern TSDLLEXPORT TimestampTz ts_timer_get_current_timestamp(void); #ifdef TS_DEBUG extern void ts_timer_set(const Timer *timer); extern const Timer *ts_get_standard_timer(void); #endif ================================================ FILE: src/bgw/worker.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <postmaster/bgworker.h> /** * Parameters to background workers. * * Do not add data here that cannot be simply copied to the background worker * using memcpy(3). If it is necessary to add fields that cannot simply be * copied, we need to start using the send and recv functions for the types. * * Only one of `job_id` and `ttl` is passed currently, with `job_id` being used * for normal jobs and `ttl` being used for tests. * * The `bgw_main` is the function to execute when starting the job and is * different depending on whether this is a test runner or the real runner. * * @see ts_bgw_db_scheduler_test_main * @see ts_bgw_job_entrypoint */ typedef struct BgwParams { /** User oid to run the job as. Used when initializing the database * connection. */ Oid user_oid; /** Job id to use for the worker when executing the job */ int32 job_id; /** Job history information to use for the worker when recording the job execution */ int64 job_history_id; TimestampTz job_history_execution_start; /** Time to live. Only used in tests. */ int32 ttl; /** Name of function to call when starting the background worker. */ char bgw_main[BGW_MAXLEN]; } BgwParams; /** * Compile-time check to ensure that the size of BgwParams fit into the bgw_extra field * of BackgroundWorker. */ StaticAssertDecl(sizeof(BgwParams) <= sizeof(((BackgroundWorker *) 0)->bgw_extra), "sizeof(BgwParams) exceeds sizeof(bgw_extra) field of BackgroundWorker"); ================================================ FILE: src/bgw_policy/CMakeLists.txt ================================================ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/policy.c ${CMAKE_CURRENT_SOURCE_DIR}/chunk_stats.c) target_sources(${PROJECT_NAME} PRIVATE ${SOURCES}) ================================================ FILE: src/bgw_policy/chunk_stats.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <executor/tuptable.h> #include "bgw/job.h" #include "chunk_stats.h" #include "policy.h" #include "ts_catalog/catalog.h" #include "utils.h" #include "compat/compat.h" static ScanTupleResult bgw_policy_chunk_stats_tuple_found(TupleInfo *ti, void *const data) { BgwPolicyChunkStats **chunk_stats = (BgwPolicyChunkStats **) data; *chunk_stats = STRUCT_FROM_SLOT(ti->slot, ti->mctx, BgwPolicyChunkStats, FormData_bgw_policy_chunk_stats); return SCAN_CONTINUE; } /* Cascades deletes via the job delete function */ static ScanTupleResult bgw_policy_chunk_stats_delete_via_job_tuple_found(TupleInfo *ti, void *const data) { bool isnull; Datum job_id = slot_getattr(ti->slot, Anum_bgw_policy_chunk_stats_job_id, &isnull); Assert(!isnull); /* This call will actually delete the row for us */ ts_bgw_job_delete_by_id(DatumGetInt32(job_id)); return SCAN_CONTINUE; } /* * Delete all chunk_stat rows associated with this job_id. * To prevent infinite recursive calls from the job <-> policy tables, we do not cascade deletes in * this function. Instead, the caller must be responsible for making sure that the delete cascades * to the job corresponding to this policy. */ void ts_bgw_policy_chunk_stats_delete_row_only_by_job_id(int32 job_id) { ScanKeyData scankey[1]; ScanKeyInit(&scankey[0], Anum_bgw_policy_chunk_stats_job_id_chunk_id_idx_job_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(job_id)); ts_catalog_scan_all(BGW_POLICY_CHUNK_STATS, BGW_POLICY_CHUNK_STATS_JOB_ID_CHUNK_ID_IDX, scankey, 1, ts_bgw_policy_delete_row_only_tuple_found, RowExclusiveLock, NULL); } /* * Delete all chunk_stat rows associated with this chunk_id. * Deletes are cascaded via ...delete_via_job_tuple_found. */ void ts_bgw_policy_chunk_stats_delete_by_chunk_id(int32 chunk_id) { ScanKeyData scankey[1]; ScanKeyInit(&scankey[0], Anum_bgw_policy_chunk_stats_chunk_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(chunk_id)); ts_catalog_scan_all(BGW_POLICY_CHUNK_STATS, InvalidOid, scankey, 1, bgw_policy_chunk_stats_delete_via_job_tuple_found, RowExclusiveLock, NULL); } static void ts_bgw_policy_chunk_stats_insert_with_relation(Relation rel, BgwPolicyChunkStats *chunk_stats) { TupleDesc tupdesc; CatalogSecurityContext sec_ctx; Datum values[Natts_bgw_policy_chunk_stats]; bool nulls[Natts_bgw_policy_chunk_stats] = { false }; tupdesc = RelationGetDescr(rel); values[AttrNumberGetAttrOffset(Anum_bgw_policy_chunk_stats_job_id)] = Int32GetDatum(chunk_stats->fd.job_id); values[AttrNumberGetAttrOffset(Anum_bgw_policy_chunk_stats_chunk_id)] = Int32GetDatum(chunk_stats->fd.chunk_id); values[AttrNumberGetAttrOffset(Anum_bgw_policy_chunk_stats_num_times_job_run)] = Int32GetDatum(chunk_stats->fd.num_times_job_run); values[AttrNumberGetAttrOffset(Anum_bgw_policy_chunk_stats_last_time_job_run)] = TimestampTzGetDatum(chunk_stats->fd.last_time_job_run); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_insert_values(rel, tupdesc, values, nulls); ts_catalog_restore_user(&sec_ctx); } void ts_bgw_policy_chunk_stats_insert(BgwPolicyChunkStats *chunk_stats) { Catalog *catalog = ts_catalog_get(); Relation rel = table_open(catalog_get_table_id(catalog, BGW_POLICY_CHUNK_STATS), RowExclusiveLock); ts_bgw_policy_chunk_stats_insert_with_relation(rel, chunk_stats); table_close(rel, RowExclusiveLock); } BgwPolicyChunkStats * ts_bgw_policy_chunk_stats_find(int32 job_id, int32 chunk_id) { ScanKeyData scankeys[2]; BgwPolicyChunkStats *stats = NULL; ScanKeyInit(&scankeys[0], Anum_bgw_policy_chunk_stats_job_id_chunk_id_idx_job_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(job_id)); ScanKeyInit(&scankeys[1], Anum_bgw_policy_chunk_stats_job_id_chunk_id_idx_chunk_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(chunk_id)); ts_catalog_scan_one(BGW_POLICY_CHUNK_STATS, BGW_POLICY_CHUNK_STATS_JOB_ID_CHUNK_ID_IDX, scankeys, 2, bgw_policy_chunk_stats_tuple_found, AccessShareLock, BGW_POLICY_CHUNK_STATS_TABLE_NAME, (void *) &stats); return stats; } static ScanTupleResult bgw_policy_chunk_stats_update_tuple_found(TupleInfo *ti, void *const data) { TimestampTz *updated_last_time_job_run = data; bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); HeapTuple new_tuple = heap_copytuple(tuple); BgwPolicyChunkStats *chunk_stats = (BgwPolicyChunkStats *) GETSTRUCT(new_tuple); if (should_free) heap_freetuple(tuple); chunk_stats->fd.num_times_job_run++; chunk_stats->fd.last_time_job_run = *updated_last_time_job_run; ts_catalog_update(ti->scanrel, new_tuple); heap_freetuple(new_tuple); return SCAN_CONTINUE; } /* This function also increments num_times_job_run by 1. */ void ts_bgw_policy_chunk_stats_record_job_run(int32 job_id, int32 chunk_id, TimestampTz last_time_job_run) { bool updated; ScanKeyData scankeys[2]; ScanKeyInit(&scankeys[0], Anum_bgw_policy_chunk_stats_job_id_chunk_id_idx_job_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(job_id)); ScanKeyInit(&scankeys[1], Anum_bgw_policy_chunk_stats_job_id_chunk_id_idx_chunk_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(chunk_id)); updated = ts_catalog_scan_one(BGW_POLICY_CHUNK_STATS, BGW_POLICY_CHUNK_STATS_JOB_ID_CHUNK_ID_IDX, scankeys, 2, bgw_policy_chunk_stats_update_tuple_found, RowExclusiveLock, BGW_POLICY_CHUNK_STATS_TABLE_NAME, &last_time_job_run); if (!updated) { BgwPolicyChunkStats new_stat = { .fd = { .job_id = job_id, .chunk_id = chunk_id, .num_times_job_run = 1, .last_time_job_run = last_time_job_run, }, }; ts_bgw_policy_chunk_stats_insert(&new_stat); } } ================================================ FILE: src/bgw_policy/chunk_stats.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include "export.h" #include "ts_catalog/catalog.h" typedef struct BgwPolicyChunkStats { FormData_bgw_policy_chunk_stats fd; } BgwPolicyChunkStats; extern TSDLLEXPORT void ts_bgw_policy_chunk_stats_insert(BgwPolicyChunkStats *chunk_stats); extern BgwPolicyChunkStats *ts_bgw_policy_chunk_stats_find(int32 job_id, int32 chunk_id); extern void ts_bgw_policy_chunk_stats_delete_row_only_by_job_id(int32 job_id); extern void ts_bgw_policy_chunk_stats_delete_by_chunk_id(int32 chunk_id); extern TSDLLEXPORT void ts_bgw_policy_chunk_stats_record_job_run(int32 job_id, int32 chunk_id, TimestampTz last_time_job_run); ================================================ FILE: src/bgw_policy/policy.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <utils/builtins.h> #include "bgw/job.h" #include "policy.h" void ts_bgw_policy_delete_by_hypertable_id(int32 hypertable_id) { List *jobs; ListCell *lc; jobs = ts_bgw_job_find_by_hypertable_id(hypertable_id); foreach (lc, jobs) { BgwJob *job = lfirst(lc); ts_bgw_job_delete_by_id(job->fd.id); } } /* This function does NOT cascade deletes to the bgw_job table. */ ScanTupleResult ts_bgw_policy_delete_row_only_tuple_found(TupleInfo *ti, void *const data) { CatalogSecurityContext sec_ctx; ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); ts_catalog_restore_user(&sec_ctx); return SCAN_CONTINUE; } ================================================ FILE: src/bgw_policy/policy.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include "config.h" #include "export.h" #include "scanner.h" #include "ts_catalog/catalog.h" extern ScanTupleResult ts_bgw_policy_delete_row_only_tuple_found(TupleInfo *ti, void *const data); extern void ts_bgw_policy_delete_by_hypertable_id(int32 hypertable_id); ================================================ FILE: src/bmslist_utils.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include "bmslist_utils.h" TsBmsList ts_bmslist_create(void) { /* Create a new empty list is NULL */ return NIL; } TsBmsList ts_bmslist_add_member(TsBmsList bmslist, const int *items, int num_items) { Assert(items != NULL); Assert(num_items > 0); if (items == NULL || num_items == 0) return bmslist; /* Create a new Bitmapset for the items */ int first_item = items[0]; Bitmapset *set = bms_make_singleton(first_item); for (int i = 1; i < num_items; i++) { int item = items[i]; set = bms_add_member(set, item); } /* Add the new set to the list */ return lappend(bmslist, set); } TsBmsList ts_bmslist_add_set(TsBmsList bmslist, Bitmapset *set) { Assert(set != NULL); return lappend(bmslist, set); } bool ts_bmslist_contains_items(TsBmsList bmslist, const int *items, int num_items) { bool result = false; Assert(items != NULL); Assert(num_items > 0); if (items == NULL || num_items == 0) return false; /* Create a new Bitmapset for the items */ int first_item = items[0]; Bitmapset *set = bms_make_singleton(first_item); for (int i = 1; i < num_items; i++) { int item = items[i]; set = bms_add_member(set, item); } result = ts_bmslist_contains_set(bmslist, set); bms_free(set); return result; } bool ts_bmslist_contains_set(TsBmsList bmslist, Bitmapset *set) { ListCell *lc; Assert(set != NULL); if (set == NULL) return false; foreach (lc, bmslist) { Bitmapset *item = (Bitmapset *) lfirst(lc); if (bms_equal(item, set)) return true; } return false; } void ts_bmslist_free(TsBmsList bmslist) { list_free_deep(bmslist); } ================================================ FILE: src/bmslist_utils.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once /* * Utility functions to simplify working with lists of Bitmapsets. * It builds on the Bitmapset and List implementations in Postgres. * It is merely a convenience layer on top of the Postgres functionality. */ #include "nodes/bitmapset.h" #include "nodes/pg_list.h" #include "postgres.h" #include "export.h" typedef List *TsBmsList; /* Just a wrapper around the List type */ extern TSDLLEXPORT TsBmsList ts_bmslist_create(void); /* Convert the items to a Bitmapset and add it to the list. It doesn't verify if * the items are already in the list, so it may add duplicates. */ extern TSDLLEXPORT TsBmsList ts_bmslist_add_member(TsBmsList bmslist, const int *items, int num_items); extern TSDLLEXPORT TsBmsList ts_bmslist_add_set(TsBmsList bmslist, Bitmapset *set); /* Checks if the list contains the given items or set. */ extern TSDLLEXPORT bool ts_bmslist_contains_items(TsBmsList bmslist, const int *items, int num_items); extern TSDLLEXPORT bool ts_bmslist_contains_set(TsBmsList bmslist, Bitmapset *set); /* Frees the list and all the Bitmapsets it contains. */ extern TSDLLEXPORT void ts_bmslist_free(TsBmsList bmslist); ================================================ FILE: src/build-defs.cmake ================================================ # Hide symbols by default in shared libraries if(NOT USE_DEFAULT_VISIBILITY) set(CMAKE_C_VISIBILITY_PRESET "hidden") endif() if(UNIX) set(CMAKE_C_STANDARD 11) set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -L${PG_LIBDIR}") set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -L${PG_LIBDIR}") set(CMAKE_C_FLAGS "${PG_CFLAGS} ${PG_CPPFLAGS} ${CMAKE_C_FLAGS}") set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g") endif() if(APPLE) if((${PG_VERSION_MAJOR} GREATER_EQUAL "16")) set(CMAKE_SHARED_MODULE_SUFFIX ".dylib") endif() set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -multiply_defined suppress") set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -multiply_defined suppress -Wl,-undefined,dynamic_lookup -bundle_loader ${PG_BINDIR}/postgres" ) elseif(WIN32) set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /MANIFEST:NO") set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} /MANIFEST:NO") endif() # PG_LDFLAGS can have strange values if not found, so we just add the flags if # they are defined. if(PG_LDFLAGS) set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${PG_LDFLAGS}") set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${PG_LDFLAGS}") endif() if(APACHE_ONLY) add_definitions(-DAPACHE_ONLY) endif() include_directories(${PROJECT_SOURCE_DIR}/src ${PROJECT_BINARY_DIR}/src) include_directories(SYSTEM ${PG_INCLUDEDIR_SERVER}) # Only Windows and FreeBSD need the base include/ dir instead of # include/server/, and including both causes problems on Ubuntu where they # frequently get out of sync if(WIN32 OR (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")) include_directories(SYSTEM ${PG_INCLUDEDIR}) endif() if(WIN32) set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${PG_LIBDIR}/postgres.lib ws2_32.lib Version.lib" ) set(CMAKE_C_FLAGS "-D_CRT_SECURE_NO_WARNINGS") include_directories(SYSTEM ${PG_INCLUDEDIR_SERVER}/port/win32) if(MSVC) include_directories(SYSTEM ${PG_INCLUDEDIR_SERVER}/port/win32_msvc) endif(MSVC) endif(WIN32) # Name of library with test-specific code set(TESTS_LIB_NAME ${PROJECT_NAME}-tests) ================================================ FILE: src/cache.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <storage/ipc.h> #include "compat/compat.h" #include "cache.h" /* List of pinned caches. A cache occurs once in this list for every pin * taken */ static List *pinned_caches = NIL; static MemoryContext pinned_caches_mctx = NULL; typedef struct CachePin { Cache *cache; SubTransactionId subtxnid; } CachePin; static void cache_reset_pinned_caches(void) { if (NULL != pinned_caches_mctx) MemoryContextDelete(pinned_caches_mctx); pinned_caches_mctx = AllocSetContextCreate(CacheMemoryContext, "Cache pins", ALLOCSET_DEFAULT_SIZES); pinned_caches = NIL; } void ts_cache_init(Cache *cache) { if (cache->htab != NULL) { elog(ERROR, "cache %s is already initialized", cache->name); return; } /* * The cache object should have been created in its own context so that * cache_destroy can just delete the context to free everything. */ Assert(GetMemoryChunkContext(cache) == ts_cache_memory_ctx(cache)); /* * We always want to be explicit about the memory context our hash table * ends up in to ensure it's not accidentally put in TopMemoryContext. */ Assert(cache->flags & HASH_CONTEXT); cache->htab = hash_create(cache->name, cache->numelements, &cache->hctl, cache->flags); cache->refcount = 1; cache->handle_txn_callbacks = true; cache->release_on_commit = true; } static void cache_destroy(Cache **cache_ptr) { Cache *cache = *cache_ptr; if (cache == NULL) return; if (cache->refcount > 0) { /* will be destroyed later */ return; } if (cache->pre_destroy_hook != NULL) cache->pre_destroy_hook(cache); hash_destroy(cache->htab); MemoryContextDelete(cache->hctl.hcxt); *cache_ptr = NULL; } void ts_cache_invalidate(Cache **cache_ptr) { Cache *cache = *cache_ptr; if (cache == NULL) return; cache->refcount--; cache_destroy(cache_ptr); } /* * Pinning is needed if any items returned by the cache may need to survive * invalidation events (i.e. AcceptInvalidationMessages() may be called). * * Invalidation messages may be processed on any internal function that takes a * lock (e.g. table_open). * * Each call to cache_pin MUST BE paired with a call to cache_release. * */ Cache * ts_cache_pin(Cache *cache) { MemoryContext old = MemoryContextSwitchTo(pinned_caches_mctx); CachePin *cp = palloc(sizeof(CachePin)); cp->cache = cache; cp->subtxnid = GetCurrentSubTransactionId(); if (cache->handle_txn_callbacks) pinned_caches = lappend(pinned_caches, cp); MemoryContextSwitchTo(old); cache->refcount++; return cache; } static void remove_pin(Cache *cache, SubTransactionId subtxnid) { ListCell *lc; foreach (lc, pinned_caches) { CachePin *cp = lfirst(lc); if (cp->cache == cache && cp->subtxnid == subtxnid) { // free cache memory and then remove the pin cache_destroy(&cp->cache); pinned_caches = list_delete_cell(pinned_caches, lc); pfree(cp); return; } } /* should never reach here: there should always be a pin to remove */ Assert(false); } static int cache_release_subtxn(Cache **cache_ptr, SubTransactionId subtxnid) { Cache *cache = *cache_ptr; int refcount = cache->refcount - 1; Assert(cache->refcount > 0); cache->refcount--; if (cache->handle_txn_callbacks) { remove_pin(cache, subtxnid); } else { cache_destroy(cache_ptr); } return refcount; } int ts_cache_release(Cache **cache) { return cache_release_subtxn(cache, GetCurrentSubTransactionId()); } MemoryContext ts_cache_memory_ctx(Cache *cache) { return cache->hctl.hcxt; } void * ts_cache_fetch(Cache *cache, CacheQuery *query) { HASHACTION action; bool found; if (cache->htab == NULL || cache->valid_result == NULL) elog(ERROR, "cache \"%s\" is not initialized", cache->name); if (query->flags & CACHE_FLAG_NOCREATE) action = HASH_FIND; else if (cache->create_entry == NULL) elog(ERROR, "cache \"%s\" does not support creating new entries", cache->name); else action = HASH_ENTER; query->result = hash_search(cache->htab, cache->get_key(query), action, &found); if (found) { cache->stats.hits++; if (cache->update_entry != NULL) query->result = cache->update_entry(cache, query); } else { cache->stats.misses++; if (action == HASH_ENTER) { cache->stats.numelements++; query->result = cache->create_entry(cache, query); } } if (!(query->flags & CACHE_FLAG_MISSING_OK) && !cache->valid_result(query->result)) { if (cache->missing_error != NULL) cache->missing_error(cache, query); else elog(ERROR, "failed to find entry in cache \"%s\"", cache->name); } return query->result; } static void release_all_pinned_caches() { ListCell *lc; /* * release once for every occurrence of a cache in the pinned caches list. * On abort, release irrespective of cache->release_on_commit. */ foreach (lc, pinned_caches) { CachePin *cp = lfirst(lc); cp->cache->refcount--; cache_destroy(&cp->cache); } cache_reset_pinned_caches(); } static void release_subtxn_pinned_caches(SubTransactionId subtxnid, bool abort) { ListCell *lc; /* * Need a copy because cache_release will modify pinned_caches. * * This needs to be allocated in pinned cache memory context. * Otherwise leaks ensue if CurTransactionContext (which is the * CurrentMemoryContext) gets used! */ MemoryContext old = MemoryContextSwitchTo(pinned_caches_mctx); List *pinned_caches_copy = list_copy(pinned_caches); MemoryContextSwitchTo(old); /* Only release caches created in subtxn */ foreach (lc, pinned_caches_copy) { CachePin *cp = lfirst(lc); if (cp->subtxnid == subtxnid && cp->cache) { /* * This assert makes sure that that we don't have a cache leak * when running with debugging */ Assert(abort); cache_release_subtxn(&cp->cache, subtxnid); } } list_free(pinned_caches_copy); } /* * Transaction end callback that cleans up any pinned caches. This is a * safeguard that protects against indefinitely pinned caches (memory leaks) * that may occur if a transaction ends (normally or abnormally) while a pin is * held. Without this, a ts_cache_pin() call always needs to be paired with a * ts_cache_release() call and wrapped in a PG_TRY() block to capture and handle * any exceptions that occur. * * Note that this checks that ts_cache_release() is always called by the end * of a non-aborted transaction unless cache->release_on_commit is set to true. * */ static void cache_xact_end(XactEvent event, void *arg) { switch (event) { case XACT_EVENT_ABORT: case XACT_EVENT_PARALLEL_ABORT: release_all_pinned_caches(); break; default: { /* * Make a copy of the list of pinned caches since * ts_cache_release() can manipulate the original list. */ List *pinned_caches_copy = list_copy(pinned_caches); ListCell *lc; /* * Only caches left should be marked as non-released */ foreach (lc, pinned_caches_copy) { CachePin *cp = lfirst(lc); /* * This assert makes sure that that we don't have a cache * leak when running with debugging */ Assert(!cp->cache->release_on_commit); /* * This may still happen in optimized environments where * Assert is turned off. In that case, release. */ if (cp->cache->release_on_commit) ts_cache_release(&cp->cache); } list_free(pinned_caches_copy); } break; } } static void cache_subxact_abort(SubXactEvent event, SubTransactionId subtxn_id, SubTransactionId parentSubid, void *arg) { /* * Note that cache->release_on_commit is irrelevant here since can't have * cross-commit operations in subtxns */ /* * In subtxns, caches should have already been released, unless an abort * happened. Be careful to only release caches that were created in the * same subtxn. */ switch (event) { case SUBXACT_EVENT_START_SUB: case SUBXACT_EVENT_PRE_COMMIT_SUB: /* do nothing */ break; case SUBXACT_EVENT_COMMIT_SUB: release_subtxn_pinned_caches(subtxn_id, false); break; case SUBXACT_EVENT_ABORT_SUB: release_subtxn_pinned_caches(subtxn_id, true); break; } } void _cache_init(void) { cache_reset_pinned_caches(); RegisterXactCallback(cache_xact_end, NULL); RegisterSubXactCallback(cache_subxact_abort, NULL); } void _cache_fini(void) { release_all_pinned_caches(); MemoryContextDelete(pinned_caches_mctx); pinned_caches_mctx = NULL; pinned_caches = NIL; UnregisterXactCallback(cache_xact_end, NULL); UnregisterSubXactCallback(cache_subxact_abort, NULL); } ================================================ FILE: src/cache.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <utils/hsearch.h> #include <utils/memutils.h> #include "export.h" typedef enum CacheQueryFlags { CACHE_FLAG_NONE = 0, CACHE_FLAG_MISSING_OK = 1 << 0, CACHE_FLAG_NOCREATE = 1 << 1, } CacheQueryFlags; #define CACHE_FLAG_CHECK (CACHE_FLAG_MISSING_OK | CACHE_FLAG_NOCREATE) typedef struct CacheQuery { /* CacheQueryFlags as defined above */ const unsigned int flags; void *result; void *data; } CacheQuery; typedef struct CacheStats { long numelements; uint64 hits; uint64 misses; } CacheStats; typedef struct Cache { HASHCTL hctl; HTAB *htab; int refcount; const char *name; long numelements; int flags; CacheStats stats; void *(*get_key)(struct CacheQuery *); void *(*create_entry)(struct Cache *, CacheQuery *); void *(*update_entry)(struct Cache *, CacheQuery *); void (*missing_error)(const struct Cache *, const CacheQuery *); bool (*valid_result)(const void *); void (*remove_entry)(void *entry); void (*pre_destroy_hook)(struct Cache *); bool handle_txn_callbacks; /* Auto-release caches on (sub)txn * aborts and commits. Should be off * if cache used in txn callbacks */ bool release_on_commit; /* This should be false if doing * cross-commit operations like CLUSTER or * VACUUM */ } Cache; extern TSDLLEXPORT void ts_cache_init(Cache *cache); extern TSDLLEXPORT void ts_cache_invalidate(Cache **cache_ptr); extern TSDLLEXPORT void *ts_cache_fetch(Cache *cache, CacheQuery *query); extern TSDLLEXPORT MemoryContext ts_cache_memory_ctx(Cache *cache); extern TSDLLEXPORT Cache *ts_cache_pin(Cache *cache); extern TSDLLEXPORT int ts_cache_release(Cache **cache_ptr); extern void _cache_init(void); extern void _cache_fini(void); ================================================ FILE: src/cache_invalidate.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <catalog/namespace.h> #include <miscadmin.h> #include <nodes/nodes.h> #include <utils/inval.h> #include <utils/lsyscache.h> #include <utils/syscache.h> #include "compat/compat.h" #include "annotations.h" #include "extension.h" #include "hypertable_cache.h" #include "ts_catalog/catalog.h" #include "bgw/scheduler.h" #include "cache_invalidate.h" #include "cross_module_fn.h" /* * Notes on the way cache invalidation works. * * Since our caches are stored in per-process (per-backend memory), we need a * way to signal all backends that they should invalidate their caches. For this * we use the PostgreSQL relcache mechanism that propagates relation cache * invalidation events to all backends. We register a callback with this * mechanism to receive events on all backends whenever a relation cache entry * is invalidated. * * To know which events should trigger invalidation of our caches, we use dummy * (empty) tables. We can trigger relcache invalidation events for these tables * to signal other backends. If the received table OID is a dummy table, we know * that this is an event that we care about. * * Caches for catalog tables should be invalidated on: * * 1. INSERT/UPDATE/DELETE on a catalog table * 2. Aborted transactions that taint the caches * * Generally, INSERTS do not warrant cache invalidation, unless it is an insert * of a subobject that belongs to an object that might already be in the cache * (e.g., a new dimension of a hypertable), or when replacing an existing entry * (e.g., when replacing a negative hypertable entry with a positive one). Note, * also, that INSERTS can taint the cache if the transaction that did the INSERT * fails. This is why we also need to invalidate caches on transaction failure. */ void _cache_invalidate_init(void); void _cache_invalidate_fini(void); static inline void cache_invalidate_relcache_all(void) { ts_hypertable_cache_invalidate_callback(); ts_bgw_job_cache_invalidate_callback(); } static Oid hypertable_proxy_table_oid = InvalidOid; static Oid bgw_proxy_table_oid = InvalidOid; void ts_cache_invalidate_set_proxy_tables(Oid hypertable_proxy_oid, Oid bgw_proxy_oid) { hypertable_proxy_table_oid = hypertable_proxy_oid; bgw_proxy_table_oid = bgw_proxy_oid; } /* * This function is called when any relcache is invalidated. * Should route the invalidation to the correct cache. * * NOTE that the callback should not call any functions that could invoke the * relcache or syscache to query information during the invalidation. That * might lead to bad things happening. */ static void cache_invalidate_relcache_callback(Datum arg, Oid relid) { if (!OidIsValid(relid)) { cache_invalidate_relcache_all(); } else if (ts_extension_is_proxy_table_relid(relid)) { ts_extension_invalidate(); cache_invalidate_relcache_all(); ts_cache_invalidate_set_proxy_tables(InvalidOid, InvalidOid); } else if (relid == hypertable_proxy_table_oid) { ts_hypertable_cache_invalidate_callback(); } else if (relid == bgw_proxy_table_oid) { ts_bgw_job_cache_invalidate_callback(); } } TS_FUNCTION_INFO_V1(ts_timescaledb_invalidate_cache); /* * Force a cache invalidation for a catalog table. * * This function is used for debugging purposes and triggers a cache * invalidation. * * The first argument should be the catalog table that has changed, warranting a * cache invalidation. */ Datum ts_timescaledb_invalidate_cache(PG_FUNCTION_ARGS) { ts_catalog_invalidate_cache(PG_GETARG_OID(0), CMD_UPDATE); PG_RETURN_VOID(); } static void cache_invalidate_xact_end(XactEvent event, void *arg) { switch (event) { case XACT_EVENT_ABORT: case XACT_EVENT_PARALLEL_ABORT: /* * Invalidate caches on aborted transactions to purge entries that * have been added during the transaction and are now no longer * valid. Note that we need not signal other backends of this * change since the transaction hasn't been committed and other * backends cannot have the invalid state. */ cache_invalidate_relcache_all(); break; default: break; } } static void cache_invalidate_subxact_end(SubXactEvent event, SubTransactionId mySubid, SubTransactionId parentSubid, void *arg) { switch (event) { case SUBXACT_EVENT_ABORT_SUB: /* * Invalidate caches on aborted sub transactions. See notes above * in cache_invalidate_xact_end. */ cache_invalidate_relcache_all(); break; default: break; } } void _cache_invalidate_init(void) { RegisterXactCallback(cache_invalidate_xact_end, NULL); RegisterSubXactCallback(cache_invalidate_subxact_end, NULL); CacheRegisterRelcacheCallback(cache_invalidate_relcache_callback, PointerGetDatum(NULL)); } void _cache_invalidate_fini(void) { UnregisterXactCallback(cache_invalidate_xact_end, NULL); UnregisterSubXactCallback(cache_invalidate_subxact_end, NULL); /* No way to unregister relcache callback */ } ================================================ FILE: src/cache_invalidate.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> extern void ts_cache_invalidate_set_proxy_tables(Oid hypertable_proxy_oid, Oid bgw_proxy_oid); ================================================ FILE: src/chunk.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/htup.h> #include <access/htup_details.h> #include <access/reloptions.h> #include <access/table.h> #include <access/tableam.h> #include <access/tupdesc.h> #include <access/xact.h> #include <catalog/indexing.h> #include <catalog/namespace.h> #include <catalog/pg_class.h> #include <catalog/pg_constraint.h> #include <catalog/pg_inherits.h> #include <catalog/pg_opfamily.h> #include <catalog/pg_publication.h> #include <catalog/pg_publication_rel_d.h> #include <catalog/pg_trigger.h> #include <catalog/pg_type.h> #include <catalog/pg_type_d.h> #include <catalog/toasting.h> #include <commands/defrem.h> #include <commands/publicationcmds.h> #include <commands/tablecmds.h> #include <commands/trigger.h> #include <executor/executor.h> #include <fmgr.h> #include <foreign/fdwapi.h> #include <funcapi.h> #include <miscadmin.h> #include <nodes/execnodes.h> #include <nodes/lockoptions.h> #include <nodes/makefuncs.h> #include <nodes/value.h> #include <parser/parse_node.h> #include <storage/lmgr.h> #include <storage/lockdefs.h> #include <tcop/tcopprot.h> #include <utils/acl.h> #include <utils/array.h> #include <utils/builtins.h> #include <utils/datum.h> #include <utils/elog.h> #include <utils/hsearch.h> #include <utils/inval.h> #include <utils/lsyscache.h> #include <utils/palloc.h> #include <utils/syscache.h> #include <utils/timestamp.h> #include <utils/typcache.h> #include "chunk.h" #include "compat/compat.h" #include "bgw_policy/chunk_stats.h" #include "cache.h" #include "chunk_index.h" #include "cross_module_fn.h" #include "debug_assert.h" #include "debug_point.h" #include "dimension.h" #include "dimension_slice.h" #include "dimension_vector.h" #include "errors.h" #include "foreign_key.h" #include "guc.h" #include "hypercube.h" #include "hypertable.h" #include "hypertable_cache.h" #include "osm_callbacks.h" #include "partition_chunk.h" #include "process_utility.h" #include "scan_iterator.h" #include "scanner.h" #include "time_utils.h" #include "trigger.h" #include "ts_catalog/catalog.h" #include "ts_catalog/chunk_column_stats.h" #include "ts_catalog/chunk_rewrite.h" #include "ts_catalog/compression_chunk_size.h" #include "ts_catalog/compression_settings.h" #include "ts_catalog/continuous_agg.h" #include "ts_catalog/continuous_aggs_watermark.h" #include "utils.h" TS_FUNCTION_INFO_V1(ts_chunk_show_chunks); TS_FUNCTION_INFO_V1(ts_chunk_drop_chunks); TS_FUNCTION_INFO_V1(ts_chunk_drop_single_chunk); TS_FUNCTION_INFO_V1(ts_chunk_attach_osm_table_chunk); TS_FUNCTION_INFO_V1(ts_chunk_drop_osm_chunk); TS_FUNCTION_INFO_V1(ts_chunk_id_from_relid); TS_FUNCTION_INFO_V1(ts_chunk_show); TS_FUNCTION_INFO_V1(ts_chunk_create); TS_FUNCTION_INFO_V1(ts_chunk_status); static bool ts_chunk_add_status(Chunk *chunk, int32 status); static const char * DatumGetNameString(Datum datum) { Name name = DatumGetName(datum); return pstrdup(NameStr(*name)); } /* Used when processing scanned chunks */ typedef enum ChunkResult { CHUNK_DONE, CHUNK_IGNORED, CHUNK_PROCESSED } ChunkResult; /* * Context for scanning and building a chunk from a stub. * * If found, the chunk will be created and the chunk pointer member is set in * the result. Optionally, a caller can pre-allocate the chunk member's memory, * which is useful if one, e.g., wants to fill in an memory-aligned array of * chunks. * */ typedef struct ChunkStubScanCtx { ChunkStub *stub; Chunk *chunk; LOCKMODE chunk_lockmode; const ScanTupLock *slice_lock; } ChunkStubScanCtx; static bool chunk_stub_is_valid(const ChunkStub *stub, int16 expected_slices) { return stub && stub->id > 0 && stub->constraints && expected_slices == stub->cube->num_slices && stub->cube->num_slices == stub->constraints->num_dimension_constraints; } typedef ChunkResult (*on_chunk_stub_func)(ChunkScanCtx *ctx, ChunkStub *stub); static void chunk_scan_ctx_init(ChunkScanCtx *ctx, const Hypertable *ht, const Point *point); static void chunk_scan_ctx_destroy(ChunkScanCtx *ctx); static void chunk_collision_scan(ChunkScanCtx *scanctx, const Hypercube *cube); static int chunk_scan_ctx_foreach_chunk_stub(ChunkScanCtx *ctx, on_chunk_stub_func on_chunk, uint64 limit); static Datum show_chunks_return_srf(FunctionCallInfo fcinfo); static int chunk_cmp(const void *ch1, const void *ch2); static void init_scan_by_qualified_table_name(ScanIterator *iterator, const char *schema_name, const char *table_name); static Chunk *get_chunks_in_time_range(Hypertable *ht, int64 older_than, int64 newer_than, MemoryContext mctx, uint64 *num_chunks_returned, ScanTupLock *tuplock); static Chunk *get_chunks_in_creation_time_range(Hypertable *ht, int64 older_than, int64 newer_than, MemoryContext mctx, uint64 *num_chunks_returned, ScanTupLock *tupLock); static HeapTuple chunk_formdata_make_tuple(const FormData_chunk *fd, TupleDesc desc) { Datum values[Natts_chunk]; bool nulls[Natts_chunk] = { false }; memset(values, 0, sizeof(Datum) * Natts_chunk); values[AttrNumberGetAttrOffset(Anum_chunk_id)] = Int32GetDatum(fd->id); values[AttrNumberGetAttrOffset(Anum_chunk_hypertable_id)] = Int32GetDatum(fd->hypertable_id); values[AttrNumberGetAttrOffset(Anum_chunk_schema_name)] = NameGetDatum(&fd->schema_name); values[AttrNumberGetAttrOffset(Anum_chunk_table_name)] = NameGetDatum(&fd->table_name); /*when we insert a chunk the compressed chunk id is always NULL */ if (fd->compressed_chunk_id == INVALID_CHUNK_ID) nulls[AttrNumberGetAttrOffset(Anum_chunk_compressed_chunk_id)] = true; else { values[AttrNumberGetAttrOffset(Anum_chunk_compressed_chunk_id)] = Int32GetDatum(fd->compressed_chunk_id); } values[AttrNumberGetAttrOffset(Anum_chunk_status)] = Int32GetDatum(fd->status); values[AttrNumberGetAttrOffset(Anum_chunk_osm_chunk)] = BoolGetDatum(fd->osm_chunk); values[AttrNumberGetAttrOffset(Anum_chunk_creation_time)] = Int64GetDatum(fd->creation_time); return heap_form_tuple(desc, values, nulls); } void ts_chunk_formdata_fill(FormData_chunk *fd, const TupleInfo *ti) { bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); bool nulls[Natts_chunk]; Datum values[Natts_chunk]; memset(fd, 0, sizeof(FormData_chunk)); heap_deform_tuple(tuple, ts_scanner_get_tupledesc(ti), values, nulls); Assert(!nulls[AttrNumberGetAttrOffset(Anum_chunk_id)]); Assert(!nulls[AttrNumberGetAttrOffset(Anum_chunk_hypertable_id)]); Assert(!nulls[AttrNumberGetAttrOffset(Anum_chunk_schema_name)]); Assert(!nulls[AttrNumberGetAttrOffset(Anum_chunk_table_name)]); Assert(!nulls[AttrNumberGetAttrOffset(Anum_chunk_status)]); Assert(!nulls[AttrNumberGetAttrOffset(Anum_chunk_osm_chunk)]); Assert(!nulls[AttrNumberGetAttrOffset(Anum_chunk_creation_time)]); fd->id = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_chunk_id)]); fd->hypertable_id = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_chunk_hypertable_id)]); namestrcpy(&fd->schema_name, DatumGetCString(values[AttrNumberGetAttrOffset(Anum_chunk_schema_name)])); namestrcpy(&fd->table_name, DatumGetCString(values[AttrNumberGetAttrOffset(Anum_chunk_table_name)])); if (nulls[AttrNumberGetAttrOffset(Anum_chunk_compressed_chunk_id)]) fd->compressed_chunk_id = INVALID_CHUNK_ID; else fd->compressed_chunk_id = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_chunk_compressed_chunk_id)]); fd->status = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_chunk_status)]); fd->osm_chunk = DatumGetBool(values[AttrNumberGetAttrOffset(Anum_chunk_osm_chunk)]); fd->creation_time = DatumGetInt64(values[AttrNumberGetAttrOffset(Anum_chunk_creation_time)]); if (should_free) heap_freetuple(tuple); } int64 ts_chunk_primary_dimension_start(const Chunk *chunk) { return chunk->cube->slices[0]->fd.range_start; } int64 ts_chunk_primary_dimension_end(const Chunk *chunk) { return chunk->cube->slices[0]->fd.range_end; } static void chunk_insert_relation(Relation rel, const Chunk *chunk) { HeapTuple new_tuple; CatalogSecurityContext sec_ctx; new_tuple = chunk_formdata_make_tuple(&chunk->fd, RelationGetDescr(rel)); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_insert(rel, new_tuple); ts_catalog_restore_user(&sec_ctx); heap_freetuple(new_tuple); } void ts_chunk_insert_lock(const Chunk *chunk, LOCKMODE lock) { Catalog *catalog = ts_catalog_get(); Relation rel; rel = table_open(catalog_get_table_id(catalog, CHUNK), lock); chunk_insert_relation(rel, chunk); table_close(rel, lock); } typedef struct CollisionInfo { Hypercube *cube; ChunkStub *colliding_chunk; } CollisionInfo; /*- * Align a chunk's hypercube in 'aligned' dimensions. * * Alignment ensures that chunks line up in a particular dimension, i.e., their * ranges should either be identical or not overlap at all. * * Non-aligned: * * ' [---------] <- existing slice * ' [---------] <- calculated (new) slice * * To align the slices above there are two cases depending on where the * insertion point happens: * * Case 1 (reuse slice): * * ' [---------] * ' [--x------] * * The insertion point x falls within the range of the existing slice. We should * reuse the existing slice rather than creating a new one. * * Case 2 (cut to align): * * ' [---------] * ' [-------x-] * * The insertion point falls outside the range of the existing slice and we need * to cut the new slice to line up. * * ' [---------] * ' cut [---] * ' * * Note that slice reuse (case 1) happens already when calculating the tentative * hypercube for the chunk, and is thus already performed once reaching this * function. Thus, we deal only with case 2 here. Also note that a new slice * might overlap in complicated ways, requiring multiple cuts. For instance, * consider the following situation: * * ' [------] [-] [---] * ' [---x-------] <- calculated slice * * This should but cut-to-align as follows: * * ' [------] [-] [---] * ' [x] * * After a chunk collision scan, this function is called for each chunk in the * chunk scan context. Chunks in the scan context may have only a partial set of * slices if they only overlap in some, but not all, dimensions (see * illustrations below). Still, partial chunks may still be of interest for * alignment in a particular dimension. Thus, if a chunk has an overlapping * slice in an aligned dimension, we cut to not overlap with that slice. */ static ChunkResult do_dimension_alignment(ChunkScanCtx *scanctx, ChunkStub *stub) { CollisionInfo *info = scanctx->data; Hypercube *cube = info->cube; const Hyperspace *space = scanctx->ht->space; ChunkResult res = CHUNK_IGNORED; int i; for (i = 0; i < space->num_dimensions; i++) { const Dimension *dim = &space->dimensions[i]; const DimensionSlice *chunk_slice; DimensionSlice *cube_slice; int64 coord = scanctx->point->coordinates[i]; if (!dim->fd.aligned) continue; /* * The stub might not have a slice for each dimension, so we cannot * use array indexing. Fetch slice by dimension ID instead. */ chunk_slice = ts_hypercube_get_slice_by_dimension_id(stub->cube, dim->fd.id); if (NULL == chunk_slice) continue; cube_slice = cube->slices[i]; /* * Only cut-to-align if the slices collide and are not identical * (i.e., if we are reusing an existing slice we should not cut it) */ if (!ts_dimension_slices_equal(cube_slice, chunk_slice) && ts_dimension_slices_collide(cube_slice, chunk_slice)) { ts_dimension_slice_cut(cube_slice, chunk_slice, coord); res = CHUNK_PROCESSED; } } return res; } /* * Calculate, and potentially set, a new chunk interval for an open dimension. */ static bool calculate_and_set_new_chunk_interval(const Hypertable *ht, const Point *p) { Hyperspace *hs = ht->space; Dimension *dim = NULL; Datum datum; int64 chunk_interval, coord; int i; if (!OidIsValid(ht->chunk_sizing_func) || ht->fd.chunk_target_size <= 0) return false; /* Find first open dimension */ for (i = 0; i < hs->num_dimensions; i++) { dim = &hs->dimensions[i]; if (IS_OPEN_DIMENSION(dim)) break; dim = NULL; } /* Nothing to do if no open dimension */ if (NULL == dim) { elog(WARNING, "adaptive chunking enabled on hypertable \"%s\" without an open (time) dimension", get_rel_name(ht->main_table_relid)); return false; } coord = p->coordinates[i]; datum = OidFunctionCall3(ht->chunk_sizing_func, Int32GetDatum(dim->fd.id), Int64GetDatum(coord), Int64GetDatum(ht->fd.chunk_target_size)); chunk_interval = DatumGetInt64(datum); /* Check if the function didn't set and interval or nothing changed */ if (chunk_interval <= 0 || chunk_interval == dim->fd.interval_length) return false; /* Update the dimension */ ts_dimension_set_chunk_interval(dim, chunk_interval); return true; } /* * Resolve chunk collisions. * * After a chunk collision scan, this function is called for each chunk in the * chunk scan context. We only care about chunks that have a full set of * slices/constraints that overlap with our tentative hypercube, i.e., they * fully collide. We resolve those collisions by cutting the hypercube. */ static ChunkResult do_collision_resolution(ChunkScanCtx *scanctx, ChunkStub *stub) { CollisionInfo *info = scanctx->data; Hypercube *cube = info->cube; const Hyperspace *space = scanctx->ht->space; ChunkResult res = CHUNK_IGNORED; int i; if (stub->cube->num_slices != space->num_dimensions || !ts_hypercubes_collide(cube, stub->cube)) return CHUNK_IGNORED; for (i = 0; i < space->num_dimensions; i++) { DimensionSlice *cube_slice = cube->slices[i]; DimensionSlice *chunk_slice = stub->cube->slices[i]; int64 coord = scanctx->point->coordinates[i]; /* * Only cut if we aren't reusing an existing slice and there is a * collision */ if (!ts_dimension_slices_equal(cube_slice, chunk_slice) && ts_dimension_slices_collide(cube_slice, chunk_slice)) { ts_dimension_slice_cut(cube_slice, chunk_slice, coord); res = CHUNK_PROCESSED; /* * Redo the collision check after each cut since cutting in one * dimension might have resolved the collision in another */ if (!ts_hypercubes_collide(cube, stub->cube)) return res; } } Assert(!ts_hypercubes_collide(cube, stub->cube)); return res; } static ChunkResult check_for_collisions(ChunkScanCtx *scanctx, ChunkStub *stub) { CollisionInfo *info = scanctx->data; Hypercube *cube = info->cube; const Hyperspace *space = scanctx->ht->space; /* Check if this chunk collides with our hypercube */ if (stub->cube->num_slices == space->num_dimensions && ts_hypercubes_collide(cube, stub->cube)) { info->colliding_chunk = stub; return CHUNK_DONE; } return CHUNK_IGNORED; } /* * Check if a (tentative) chunk collides with existing chunks. * * Return the colliding chunk. Note that the chunk is a stub and not a full * chunk. */ static ChunkStub * chunk_collides(const Hypertable *ht, const Hypercube *hc) { ChunkScanCtx scanctx; CollisionInfo info = { .cube = (Hypercube *) hc, .colliding_chunk = NULL, }; chunk_scan_ctx_init(&scanctx, ht, NULL); /* Scan for all chunks that collide with the hypercube of the new chunk */ chunk_collision_scan(&scanctx, hc); scanctx.data = &info; /* Find chunks that collide */ chunk_scan_ctx_foreach_chunk_stub(&scanctx, check_for_collisions, 0); chunk_scan_ctx_destroy(&scanctx); return info.colliding_chunk; } /*- * Resolve collisions and perform alignment. * * Chunks collide only if their hypercubes overlap in all dimensions. For * instance, the 2D chunks below collide because they overlap in both the X and * Y dimensions: * * ' _____ * ' | | * ' | ___|__ * ' |_|__| | * ' | | * ' |_____| * * While the following chunks do not collide, although they still overlap in the * X dimension: * * ' _____ * ' | | * ' | | * ' |____| * ' ______ * ' | | * ' | *| * ' |_____| * * For the collision case above we obviously want to cut our hypercube to no * longer collide with existing chunks. However, the second case might still be * of interest for alignment in case X is an 'aligned' dimension. If '*' is the * insertion point, then we still want to cut the hypercube to ensure that the * dimension remains aligned, like so: * * ' _____ * ' | | * ' | | * ' |____| * ' ___ * ' | | * ' |*| * ' |_| * * * We perform alignment first as that might actually resolve chunk * collisions. After alignment we check for any remaining collisions. */ static void chunk_collision_resolve(const Hypertable *ht, Hypercube *cube, const Point *p) { ChunkScanCtx scanctx; CollisionInfo info = { .cube = cube, .colliding_chunk = NULL, }; chunk_scan_ctx_init(&scanctx, ht, p); /* Scan for all chunks that collide with the hypercube of the new chunk */ chunk_collision_scan(&scanctx, cube); scanctx.data = &info; /* Cut the hypercube in any aligned dimensions */ chunk_scan_ctx_foreach_chunk_stub(&scanctx, do_dimension_alignment, 0); /* * If there are any remaining collisions with chunks, then cut-to-fit to * resolve those collisions */ chunk_scan_ctx_foreach_chunk_stub(&scanctx, do_collision_resolution, 0); chunk_scan_ctx_destroy(&scanctx); } static int chunk_add_constraints(const Chunk *chunk) { int num_added; num_added = ts_chunk_constraints_add_dimension_constraints(chunk->constraints, chunk->fd.id, chunk->cube); num_added += ts_chunk_constraints_add_inheritable_constraints(chunk->constraints, chunk->fd.id, chunk->relkind, chunk->hypertable_relid, chunk->table_id); return num_added; } /* applies the attributes and statistics target for columns on the hypertable to columns on the chunk */ static void set_attoptions(Relation ht_rel, Oid chunk_oid) { TupleDesc tupleDesc = RelationGetDescr(ht_rel); int natts = tupleDesc->natts; int attno; List *alter_cmds = NIL; for (attno = 1; attno <= natts; attno++) { Form_pg_attribute attribute = TupleDescAttr(tupleDesc, attno - 1); char *attributeName = NameStr(attribute->attname); HeapTuple tuple; Datum options; bool isnull; /* Ignore dropped */ if (attribute->attisdropped) continue; tuple = SearchSysCacheAttName(RelationGetRelid(ht_rel), attributeName); Assert(tuple != NULL); /* * Pass down the attribute options (ALTER TABLE ALTER COLUMN SET * attribute_option) */ options = SysCacheGetAttr(ATTNAME, tuple, Anum_pg_attribute_attoptions, &isnull); if (!isnull) { AlterTableCmd *cmd = makeNode(AlterTableCmd); cmd->subtype = AT_SetOptions; cmd->name = attributeName; cmd->def = (Node *) untransformRelOptions(options); alter_cmds = lappend(alter_cmds, cmd); } /* * Pass down the attribute options (ALTER TABLE ALTER COLUMN SET * STATISTICS) */ options = SysCacheGetAttr(ATTNAME, tuple, Anum_pg_attribute_attstattarget, &isnull); if (!isnull) { int32 target = DatumGetInt32(options); /* Don't do anything if it's set to the default */ if (target != -1) { AlterTableCmd *cmd = makeNode(AlterTableCmd); cmd->subtype = AT_SetStatistics; cmd->name = attributeName; cmd->def = (Node *) makeInteger(target); alter_cmds = lappend(alter_cmds, cmd); } } ReleaseSysCache(tuple); } if (alter_cmds != NIL) { AlterTableInternal(chunk_oid, alter_cmds, false); list_free_deep(alter_cmds); } } static void create_toast_table(CreateStmt *stmt, Oid chunk_oid) { /* similar to tcop/utility.c */ #if PG18_LT char *validnsps[] = HEAP_RELOPT_NAMESPACES; #else const char *const validnsps[] = HEAP_RELOPT_NAMESPACES; #endif Datum toast_options = transformRelOptions(UnassignedDatum, stmt->options, "toast", validnsps, true, false); (void) heap_reloptions(RELKIND_TOASTVALUE, toast_options, true); NewRelationCreateToastTable(chunk_oid, toast_options); } static void copy_hypertable_acl_to_relid(const Hypertable *ht, const Oid owner_id, const Oid relid) { ts_copy_relation_acl(ht->main_table_relid, relid, owner_id); } /* * Create a chunk's table. * * A chunk inherits from the main hypertable and will have the same owner. Since * chunks can be created either in the TimescaleDB internal schema or in a * user-specified schema, some care has to be taken to use the right * permissions, depending on the case: * * 1. if the chunk is created in the internal schema, we create it as the * catalog/schema owner (i.e., anyone can create chunks there via inserting into * a hypertable, but can not do it via CREATE TABLE). * * 2. if the chunk is created in a user-specified "associated schema", then we * shouldn't use the catalog owner to create the table since that typically * implies super-user permissions. If we would allow that, anyone can specify * someone else's schema in create_hypertable() and create chunks in it without * having the proper permissions to do so. With this logic, the hypertable owner * must have permissions to create tables in the associated schema, or else * table creation will fail. If the schema doesn't yet exist, the table owner * instead needs the proper permissions on the database to create the schema. */ Oid ts_chunk_create_table(const Chunk *chunk, const Hypertable *ht, const char *tablespacename) { Relation rel; ObjectAddress address; int sec_ctx; char *amname = NULL; amname = get_am_name(ts_get_rel_am(chunk->hypertable_relid)); /* * CreateStmt node to create the chunk table */ CreateStmt stmt = { .type = T_CreateStmt, .relation = makeRangeVar((char *) NameStr(chunk->fd.schema_name), (char *) NameStr(chunk->fd.table_name), 0), .tablespacename = tablespacename ? (char *) tablespacename : NULL, .options = (chunk->relkind == RELKIND_RELATION) ? ts_get_reloptions(ht->main_table_relid) : NIL, .accessMethod = amname, }; /* * If partitioned hypertables are enabled, create the chunk as a standalone * table with the same columns as the hypertable to attach it as a partition * later. Otherwise, create it as an inherited table. */ if (is_partitioning_allowed(ht->main_table_relid)) { List *attlist = NIL; List *constraints = NIL; ts_partition_chunk_prepare_attributes(ht->main_table_relid, &attlist, &constraints); stmt.tableElts = attlist; stmt.constraints = constraints; } else { stmt.inhRelations = list_make1(makeRangeVar((char *) NameStr(ht->fd.schema_name), (char *) NameStr(ht->fd.table_name), 0)); } Oid uid, saved_uid; Assert(chunk->hypertable_relid == ht->main_table_relid); rel = table_open(ht->main_table_relid, AccessShareLock); /* Inherit the persistence (LOGGED or UNLOGGED) from the parent hypertable */ stmt.relation->relpersistence = rel->rd_rel->relpersistence; /* * If the chunk is created in the internal schema, become the catalog * owner, otherwise become the hypertable owner */ if (namestrcmp((Name) &chunk->fd.schema_name, INTERNAL_SCHEMA_NAME) == 0) uid = ts_catalog_database_info_get()->owner_uid; else uid = rel->rd_rel->relowner; GetUserIdAndSecContext(&saved_uid, &sec_ctx); if (uid != saved_uid) SetUserIdAndSecContext(uid, sec_ctx | SECURITY_LOCAL_USERID_CHANGE); /* Prepare event trigger state and invoke ddl_command_start triggers */ if (ts_guc_enable_event_triggers) { EventTriggerBeginCompleteQuery(); EventTriggerDDLCommandStart((Node *) &stmt); } address = DefineRelation(&stmt, chunk->relkind, rel->rd_rel->relowner, NULL, NULL); /* Invoke ddl_command_end triggers and clean up the event trigger state */ if (ts_guc_enable_event_triggers) { EventTriggerCollectSimpleCommand(address, InvalidObjectAddress, (Node *) &stmt); EventTriggerDDLCommandEnd((Node *) &stmt); EventTriggerEndCompleteQuery(); } /* Make the newly defined relation visible so that we can update the * ACL. */ CommandCounterIncrement(); /* Copy acl from hypertable to chunk relation record */ copy_hypertable_acl_to_relid(ht, rel->rd_rel->relowner, address.objectId); if (chunk->relkind == RELKIND_RELATION) { /* * need to create a toast table explicitly for some of the option * setting to work */ create_toast_table(&stmt, address.objectId); /* * Some options require being table owner to set for example statistics * so we have to set them before restoring security context */ set_attoptions(rel, address.objectId); if (uid != saved_uid) SetUserIdAndSecContext(saved_uid, sec_ctx); } else elog(ERROR, "invalid relkind \"%c\" when creating chunk", chunk->relkind); /* Insert the table into the cache to attach it as partition later */ if (is_partitioning_allowed(ht->main_table_relid)) ts_partition_cache_insert_chunk(ht, address.objectId); table_close(rel, AccessShareLock); return address.objectId; } static int32 get_next_chunk_id() { int32 chunk_id; CatalogSecurityContext sec_ctx; const Catalog *catalog = ts_catalog_get(); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); chunk_id = ts_catalog_table_next_seq_id(catalog, CHUNK); ts_catalog_restore_user(&sec_ctx); return chunk_id; } /* * Create a chunk object from the dimensional constraints in the given hypercube. * * The chunk object is then used to create the actual chunk table and update the * metadata separately. * * The table name for the chunk can be given explicitly, or generated if * table_name is NULL. If the table name is generated, it will use the given * prefix or, if NULL, use the hypertable's associated table prefix. Similarly, * if schema_name is NULL it will use the hypertable's associated schema for * the chunk. */ static Chunk * chunk_create_object(const Hypertable *ht, Hypercube *cube, const char *schema_name, const char *table_name, const char *prefix, int32 chunk_id) { const Hyperspace *hs = ht->space; Chunk *chunk; const char relkind = RELKIND_RELATION; if (NULL == schema_name || schema_name[0] == '\0') schema_name = NameStr(ht->fd.associated_schema_name); /* Create a new chunk based on the hypercube */ chunk = ts_chunk_create_base(chunk_id, hs->num_dimensions, relkind); chunk->fd.hypertable_id = hs->hypertable_id; chunk->cube = cube; chunk->hypertable_relid = ht->main_table_relid; namestrcpy(&chunk->fd.schema_name, schema_name); if (NULL == table_name || table_name[0] == '\0') { int len; if (NULL == prefix) prefix = NameStr(ht->fd.associated_table_prefix); len = snprintf(NameStr(chunk->fd.table_name), NAMEDATALEN, "%s_%d_chunk", prefix, chunk->fd.id); if (len >= NAMEDATALEN) elog(ERROR, "chunk table name too long"); } else namestrcpy(&chunk->fd.table_name, table_name); return chunk; } static void chunk_insert_into_metadata_after_lock(const Chunk *chunk) { /* Insert chunk */ ts_chunk_insert_lock(chunk, RowExclusiveLock); /* Add metadata for dimensional and inheritable constraints */ ts_chunk_constraints_insert_metadata(chunk->constraints); } /* * Ensure the replica identity setting of a chunk matches that of the root * table. */ static void chunk_set_replica_identity(const Chunk *chunk) { Relation ht_rel = relation_open(chunk->hypertable_relid, AccessShareLock); Relation ch_rel = relation_open(chunk->table_id, AccessShareLock); /* Do nothing if REPLICA IDENTITY of hypertable and chunk are equal */ if (ht_rel->rd_rel->relreplident == ch_rel->rd_rel->relreplident) { table_close(ch_rel, NoLock); table_close(ht_rel, NoLock); return; } ReplicaIdentityStmt stmt = { .type = T_ReplicaIdentityStmt, .identity_type = ht_rel->rd_rel->relreplident, }; AlterTableCmd cmd = { .type = T_AlterTableCmd, .def = (Node *) &stmt, .subtype = AT_ReplicaIdentity, }; CatalogSecurityContext sec_ctx; if (stmt.identity_type == REPLICA_IDENTITY_INDEX) { /* Use RelationGetReplicaIndex() instead of rd_replidindex * directly to ensure the index list is loaded after any * relcache invalidation. */ Oid ht_indexoid = RelationGetReplicaIndex(ht_rel); Oid chunk_index_relid = InvalidOid; if (OidIsValid(ht_indexoid)) chunk_index_relid = ts_chunk_index_get_by_hypertable_indexrelid(ch_rel, ht_indexoid); if (OidIsValid(chunk_index_relid)) stmt.name = get_rel_name(chunk_index_relid); else stmt.identity_type = REPLICA_IDENTITY_NOTHING; } ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_alter_table_with_event_trigger(chunk->table_id, NULL, list_make1(&cmd), false); ts_catalog_restore_user(&sec_ctx); table_close(ch_rel, NoLock); table_close(ht_rel, NoLock); } static void chunk_create_table_constraints(const Hypertable *ht, const Chunk *chunk) { /* Do not create any of these for partitioned hypertables */ if (is_partitioning_allowed(ht->main_table_relid)) return; /* Create the chunk's constraints, triggers, and indexes */ ts_chunk_constraints_create(ht, chunk); if (chunk->relkind == RELKIND_RELATION && !IS_OSM_CHUNK(chunk)) { ts_trigger_create_all_on_chunk(chunk); ts_chunk_index_create_all(chunk->fd.hypertable_id, chunk->hypertable_relid, chunk->fd.id, chunk->table_id, InvalidOid); chunk_set_replica_identity(chunk); } /* Copy FK constraints after indexes are created, since FK validation * requires the supporting unique index to exist on the chunk. */ ts_chunk_copy_referencing_fk(ht, chunk); } static Oid chunk_create_table(Chunk *chunk, const Hypertable *ht) { /* Create the actual table relation for the chunk */ const char *tablespace = ts_hypertable_select_tablespace_name(ht, chunk); chunk->table_id = ts_chunk_create_table(chunk, ht, tablespace); Assert(OidIsValid(chunk->table_id)); return chunk->table_id; } /* * Creates only a table for a chunk. * Either table name or chunk id needs to be provided. */ static Chunk * chunk_create_only_table_after_lock(const Hypertable *ht, Hypercube *cube, const char *schema_name, const char *table_name, const char *prefix, int32 chunk_id) { Chunk *chunk; Assert(table_name != NULL || chunk_id != INVALID_CHUNK_ID); chunk = chunk_create_object(ht, cube, schema_name, table_name, prefix, chunk_id); Assert(chunk != NULL); chunk_create_table(chunk, ht); return chunk; } static void get_hypertable_publication_filters(Oid puboid, const Chunk *chunk, List **columns, Node **whereClause) { HeapTuple pubtuple; Datum datum; bool isnull; *columns = NIL; *whereClause = NULL; /* Get filters for hypertable, chunk should inherit them */ pubtuple = SearchSysCache2(PUBLICATIONRELMAP, ObjectIdGetDatum(chunk->hypertable_relid), ObjectIdGetDatum(puboid)); if (!HeapTupleIsValid(pubtuple)) return; datum = SysCacheGetAttr(PUBLICATIONRELMAP, pubtuple, Anum_pg_publication_rel_prqual, &isnull); if (!isnull) { char *prqual_str = TextDatumGetCString(datum); *whereClause = stringToNode(prqual_str); } datum = SysCacheGetAttr(PUBLICATIONRELMAP, pubtuple, Anum_pg_publication_rel_prattrs, &isnull); if (!isnull) { ArrayType *arr = DatumGetArrayTypeP(datum); int nelems = ARR_DIMS(arr)[0]; int16 *attnums = (int16 *) ARR_DATA_PTR(arr); for (int i = 0; i < nelems; i++) { char *colname = get_attname(chunk->hypertable_relid, attnums[i], false); *columns = lappend(*columns, makeString(colname)); } } ReleaseSysCache(pubtuple); } static void chunk_add_to_publication(Oid puboid, const Chunk *chunk) { PublicationRelInfo pri = { 0 }; Relation chunk_rel; List *columns = NIL; Node *whereClause = NULL; get_hypertable_publication_filters(puboid, chunk, &columns, &whereClause); chunk_rel = table_open(chunk->table_id, AccessShareLock); pri.relation = chunk_rel; pri.columns = columns; pri.whereClause = whereClause; publication_add_relation(puboid, &pri, true); table_close(chunk_rel, AccessShareLock); } static void chunk_add_to_publications(const Chunk *chunk) { List *puboids; ListCell *lc; puboids = GetRelationPublications(chunk->hypertable_relid); foreach (lc, puboids) { Oid puboid = lfirst_oid(lc); chunk_add_to_publication(puboid, chunk); } } static Chunk * chunk_create_from_hypercube_after_lock(const Hypertable *ht, Hypercube *cube, const char *schema_name, const char *table_name, const char *prefix) { chunk_insert_check_hook_type osm_chunk_insert_hook = ts_get_osm_chunk_insert_hook(); if (osm_chunk_insert_hook) { /* OSM only uses first dimension. */ Dimension *dim = &ht->space->dimensions[0]; /* convert to PG timestamp from timescaledb internal format */ int64 range_start = ts_internal_to_time_int64(cube->slices[0]->fd.range_start, dim->fd.column_type); int64 range_end = ts_internal_to_time_int64(cube->slices[0]->fd.range_end, dim->fd.column_type); int chunk_exists = osm_chunk_insert_hook(ht->main_table_relid, range_start, range_end); if (chunk_exists) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("Cannot insert into tiered chunk range of %s.%s - attempt to create " "new chunk " "with range [%s %s] failed", NameStr(ht->fd.schema_name), NameStr(ht->fd.table_name), ts_internal_to_time_string(cube->slices[0]->fd.range_start, dim->fd.column_type), ts_internal_to_time_string(cube->slices[0]->fd.range_end, dim->fd.column_type)), errhint( "Hypertable has tiered data with time range that overlaps the insert"))); } } /* Insert any new dimension slices into metadata */ ts_dimension_slice_insert_multi(cube->slices, cube->num_slices); Chunk *chunk = chunk_create_only_table_after_lock(ht, cube, schema_name, table_name, prefix, get_next_chunk_id()); /* Insert any new chunk column stats entries into the catalog */ ts_chunk_column_stats_insert(ht, chunk); chunk_add_constraints(chunk); chunk_insert_into_metadata_after_lock(chunk); chunk_create_table_constraints(ht, chunk); /* Add chunk to publications if hypertable is in any publications */ if (ts_guc_enable_chunk_auto_publication) chunk_add_to_publications(chunk); return chunk; } /* * Make a chunk table inherit a hypertable. * * Execution happens via high-level ALTER TABLE statement. This includes * numerous checks to ensure that the chunk table has all the prerequisites to * properly inherit the hypertable. */ static void chunk_add_inheritance(Chunk *chunk, const Hypertable *ht) { AlterTableCmd altercmd = { .type = T_AlterTableCmd, .subtype = AT_AddInherit, .def = (Node *) makeRangeVar((char *) NameStr(ht->fd.schema_name), (char *) NameStr(ht->fd.table_name), 0), .missing_ok = false, }; AlterTableStmt alterstmt = { .type = T_AlterTableStmt, .cmds = list_make1(&altercmd), .missing_ok = false, .objtype = OBJECT_TABLE, .relation = makeRangeVar((char *) NameStr(chunk->fd.schema_name), (char *) NameStr(chunk->fd.table_name), 0), }; LOCKMODE lockmode = AlterTableGetLockLevel(alterstmt.cmds); AlterTableUtilityContext atcontext = { .relid = AlterTableLookupRelation(&alterstmt, lockmode), }; AlterTable(&alterstmt, lockmode, &atcontext); } static Chunk * chunk_create_from_hypercube_and_table_after_lock(const Hypertable *ht, Hypercube *cube, Oid chunk_table_relid, const char *schema_name, const char *table_name, const char *prefix) { Oid current_chunk_schemaid = get_rel_namespace(chunk_table_relid); Oid new_chunk_schemaid = InvalidOid; Chunk *chunk; Assert(OidIsValid(chunk_table_relid)); Assert(OidIsValid(current_chunk_schemaid)); Assert(OidIsValid(ht->main_table_relid)); Relation ht_rel = table_open(ht->main_table_relid, AccessShareLock); Relation chunk_rel = table_open(chunk_table_relid, AccessShareLock); TupleDesc tupdesc = RelationGetDescr(chunk_rel); for (int attno = 0; attno < tupdesc->natts; attno++) { Form_pg_attribute att = TupleDescAttr(tupdesc, attno); AttrNumber ht_attnum = InvalidAttrNumber; /* Ignore dropped */ if (att->attisdropped) continue; ht_attnum = get_attnum(ht->main_table_relid, NameStr(att->attname)); /* Try to find the column in parent (matching on column name) */ if (ht_attnum == InvalidAttrNumber) ereport(ERROR, (errcode(ERRCODE_DATATYPE_MISMATCH), errmsg("table \"%s\" contains column \"%s\" not found in parent \"%s\"", RelationGetRelationName(chunk_rel), NameStr(att->attname), RelationGetRelationName(ht_rel)), errdetail("The new chunk can contain only the columns present in parent."))); /* * PG16 and later does not allow generated columns on child tables if the parent * column is not generated. This is a change from PG15 and earlier, where the * child column could be generated even if the parent was not. * We check if a generated column is also generated in the parent here to disallow * this behavior in PG15 too. * * The case when the parent column is generated and the child column is not is handled * by Postgres code, which will throw an error. * * This check can be removed once we drop support for PG15. */ if (att->attgenerated && !get_attgenerated(ht->main_table_relid, ht_attnum)) { ereport(ERROR, (errcode(ERRCODE_DATATYPE_MISMATCH), errmsg("column \"%s\" in chunk table must not be a generated column", NameStr(att->attname)), errdetail("Chunk column must be generated if and only if parent column is " "also generated"))); } /* Check that the chunk column has the same expression as the hypertable column */ if (att->attgenerated && get_attgenerated(ht->main_table_relid, ht_attnum)) { char *chunk_expr = ts_get_attr_expr(chunk_rel, attno + 1); char *ht_expr = ts_get_attr_expr(ht_rel, ht_attnum); if (strcmp(chunk_expr, ht_expr) != 0) ereport(ERROR, (errcode(ERRCODE_DATATYPE_MISMATCH), errmsg("chunk column \"%s\" must have the same expression as the " "hypertable column.", NameStr(att->attname)))); } } table_close(ht_rel, NoLock); /* Insert any new dimension slices into metadata */ ts_dimension_slice_insert_multi(cube->slices, cube->num_slices); chunk = chunk_create_object(ht, cube, schema_name, table_name, prefix, get_next_chunk_id()); chunk->table_id = chunk_table_relid; chunk->hypertable_relid = ht->main_table_relid; new_chunk_schemaid = get_namespace_oid(NameStr(chunk->fd.schema_name), false); if (current_chunk_schemaid != new_chunk_schemaid) { ObjectAddresses *objects; CheckSetNamespace(current_chunk_schemaid, new_chunk_schemaid); objects = new_object_addresses(); AlterTableNamespaceInternal(chunk_rel, current_chunk_schemaid, new_chunk_schemaid, objects); free_object_addresses(objects); /* Make changes visible */ CommandCounterIncrement(); } table_close(chunk_rel, NoLock); if (namestrcmp(&chunk->fd.table_name, get_rel_name(chunk_table_relid)) != 0) { /* Renaming will acquire and keep an AccessExclusivelock on the chunk * table */ RenameRelationInternal(chunk_table_relid, NameStr(chunk->fd.table_name), true, false); /* Make changes visible */ CommandCounterIncrement(); } /* Note that we do not automatically add constrains and triggers to the * chunk table when the chunk is created from an existing table. However, * PostgreSQL currently validates that CHECK constraints exists, but no * validation is done for other objects, including triggers, UNIQUE, * PRIMARY KEY, and FOREIGN KEY constraints. We might want to either * enforce that these constraints exist prior to creating the chunk from a * table, or we ensure that they are automatically added when the chunk is * created. However, for the latter case, we risk duplicating constraints * and triggers if some of them already exist on the chunk table prior to * creating the chunk from it. */ chunk_add_constraints(chunk); ts_chunk_constraint_check_violated(chunk, ht->space); chunk_insert_into_metadata_after_lock(chunk); chunk_add_inheritance(chunk, ht); chunk_create_table_constraints(ht, chunk); /* Add chunk to publications if hypertable is in any publications */ if (ts_guc_enable_chunk_auto_publication) chunk_add_to_publications(chunk); return chunk; } static Chunk * chunk_create_from_point_after_lock(const Hypertable *ht, const Point *p, const char *schema_name, const char *table_name, const char *prefix) { Hyperspace *hs = ht->space; Hypercube *cube; ScanTupLock tuplock = { .lockmode = LockTupleKeyShare, .waitpolicy = LockWaitBlock, }; /* * If the user has enabled adaptive chunking, call the function to * calculate and set the new chunk time interval. */ calculate_and_set_new_chunk_interval(ht, p); /* Calculate the hypercube for a new chunk that covers the tuple's point. * * We lock the tuple in KEY SHARE mode since we are concerned with * ensuring that it is not deleted (or the key value changed) while we are * adding chunk constraints (in `ts_chunk_constraints_insert_metadata` * called in `chunk_create_metadata_after_lock`). The range of a dimension * slice does not change, but we should use the weakest lock possible to * not unnecessarily block other operations. */ cube = ts_hypercube_calculate_from_point(hs, p, &tuplock); /* Resolve collisions with other chunks by cutting the new hypercube */ chunk_collision_resolve(ht, cube, p); return chunk_create_from_hypercube_after_lock(ht, cube, schema_name, table_name, prefix); } Chunk * ts_chunk_find_or_create_without_cuts(const Hypertable *ht, Hypercube *hc, const char *schema_name, const char *table_name, Oid chunk_table_relid, bool *created) { ChunkStub *stub; Chunk *chunk = NULL; DEBUG_WAITPOINT("find_or_create_chunk_start"); stub = chunk_collides(ht, hc); if (NULL == stub) { /* Serialize chunk creation around the root hypertable */ LockRelationOid(ht->main_table_relid, ShareUpdateExclusiveLock); /* Check again after lock */ stub = chunk_collides(ht, hc); if (NULL == stub) { ScanTupLock tuplock = { .lockmode = LockTupleKeyShare, .waitpolicy = LockWaitBlock, }; /* Lock all slices that already exist to ensure they remain when we * commit since we won't create those slices ourselves. */ ts_hypercube_find_existing_slices(hc, &tuplock); if (OidIsValid(chunk_table_relid)) chunk = chunk_create_from_hypercube_and_table_after_lock(ht, hc, chunk_table_relid, schema_name, table_name, NULL); else chunk = chunk_create_from_hypercube_after_lock(ht, hc, schema_name, table_name, NULL); if (NULL != created) *created = true; ASSERT_IS_VALID_CHUNK(chunk); DEBUG_WAITPOINT("find_or_create_chunk_created"); return chunk; } /* We didn't need the lock, so release it */ UnlockRelationOid(ht->main_table_relid, ShareUpdateExclusiveLock); } Assert(NULL != stub); /* We can only use an existing chunk if it has identical dimensional * constraints. Otherwise, throw an error */ if (OidIsValid(chunk_table_relid) || !ts_hypercube_equal(stub->cube, hc)) ereport(ERROR, (errcode(ERRCODE_TS_CHUNK_COLLISION), errmsg("chunk creation failed due to collision"))); /* chunk_collides only returned a stub, so we need to lookup the full * chunk. */ chunk = ts_chunk_get_by_id(stub->id, true); if (NULL != created) *created = false; DEBUG_WAITPOINT("find_or_create_chunk_found"); ASSERT_IS_VALID_CHUNK(chunk); return chunk; } /* * Find the chunk containing the given point, locking all its dimension slices * for share. NULL if not found. */ Chunk * ts_chunk_find_for_point(const Hypertable *ht, const Point *p, LOCKMODE lockmode) { ScanTupLock slice_lock = { .lockmode = LockTupleKeyShare, .waitpolicy = LockWaitBlock, .lockflags = TUPLE_LOCK_FLAG_FIND_LAST_VERSION, }; int32 chunk_id = ts_chunk_point_find_chunk_id(ht, p, NULL); if (chunk_id == INVALID_CHUNK_ID) return NULL; /* The chunk might be dropped, so we don't fail if we haven't found it. */ return ts_chunk_get_by_id_with_slice_lock(chunk_id, lockmode, lockmode == NoLock ? NULL : &slice_lock, /* fail_if_not_found = */ false); } /* * Create a chunk through insertion of a tuple at a given point. * * If some other process managed to create the chunk before us, the existing * chunk is locked with "chunk_lockmode". */ Chunk * ts_chunk_create_for_point(const Hypertable *ht, const Point *p, const char *schema, const char *prefix, LOCKMODE chunk_lockmode) { /* * Serialize chunk creation around a lock on the "main table" to avoid * multiple processes trying to create the same chunk. We use a * ShareUpdateExclusiveLock, which is the weakest lock possible that * conflicts with itself. The lock needs to be held until transaction end. */ LockRelationOid(ht->main_table_relid, ShareUpdateExclusiveLock); DEBUG_WAITPOINT("chunk_create_for_point"); /* * Recheck if someone else created the chunk before we got the table * lock. The returned chunk will have all slices locked so that they * aren't removed. */ int chunk_id = ts_chunk_point_find_chunk_id(ht, p, NULL); if (chunk_id != INVALID_CHUNK_ID) { ScanTupLock slice_lock = { .lockmode = LockTupleKeyShare, .waitpolicy = LockWaitBlock, .lockflags = TUPLE_LOCK_FLAG_FIND_LAST_VERSION, }; /* The chunk might be dropped, so we don't fail if we haven't found it. */ Chunk *chunk = ts_chunk_get_by_id_with_slice_lock(chunk_id, chunk_lockmode, &slice_lock, /* fail_if_not_found = */ false); if (chunk != NULL) { /* * Chunk was not created by us but by someone else, so we can * release the lock early. */ UnlockRelationOid(ht->main_table_relid, ShareUpdateExclusiveLock); return chunk; } } /* Create the chunk normally. */ Chunk *chunk = chunk_create_from_point_after_lock(ht, p, schema, NULL, prefix); ASSERT_IS_VALID_CHUNK(chunk); return chunk; } static void scan_add_chunk_context(ChunkScanCtx *ctx, int32 chunk_id, List *dimension_vecs, List **l_chunk_ids) { bool found = false; ChunkScanEntry *entry = hash_search(ctx->htab, &chunk_id, HASH_ENTER, &found); if (!found) { entry->stub = NULL; entry->num_dimension_constraints = 0; } entry->num_dimension_constraints++; /* * A chunk is complete when we've found slices for all required dimensions, * i.e., a complete subspace. */ if (entry->num_dimension_constraints == list_length(dimension_vecs)) { *l_chunk_ids = lappend_int(*l_chunk_ids, entry->chunk_id); } } /* * Find the chunks that belong to the subspace identified by the given dimension * vectors. We might be restricting only some dimensions, so this subspace is * not a hypercube, but a hyperplane of some order. * Returns a list of matching chunk ids. */ List * ts_chunk_id_find_in_subspace(Hypertable *ht, List *dimension_vecs) { List *chunk_ids = NIL; ChunkScanCtx ctx; chunk_scan_ctx_init(&ctx, ht, /* point = */ NULL); ScanIterator iterator = ts_chunk_constraint_scan_iterator_create(CurrentMemoryContext); ListCell *lc; foreach (lc, dimension_vecs) { const DimensionVec *vec = lfirst(lc); /* * If it's an entry of type DIMENSION_TYPE_STATS then we need to get * the chunks using the _timescaledb_catalog.chunk_column_stats catalog. */ Assert(vec->dri != NULL); if (vec->dri->dimension->type == DIMENSION_TYPE_STATS) { ListCell *lc; List *range_chunk_ids; Assert(vec->num_slices == 0); range_chunk_ids = ts_chunk_column_stats_get_chunk_ids_by_scan(vec->dri); /* add these chunks to the context appropriately. */ foreach (lc, range_chunk_ids) { int32 chunk_id = lfirst_int(lc); scan_add_chunk_context(&ctx, chunk_id, dimension_vecs, &chunk_ids); } continue; } /* * We shouldn't see a dimension with zero matching dimension slices. * That would mean that no chunks match at all, this should have been * handled earlier by gather_restriction_dimension_vectors(). */ Assert(vec->num_slices > 0); for (int i = 0; i < vec->num_slices; i++) { const DimensionSlice *slice = vec->slices[i]; ts_chunk_constraint_scan_iterator_set_slice_id(&iterator, slice->fd.id); ts_scan_iterator_start_or_restart_scan(&iterator); while (ts_scan_iterator_next(&iterator) != NULL) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); bool PG_USED_FOR_ASSERTS_ONLY isnull = true; Datum datum = slot_getattr(ti->slot, Anum_chunk_constraint_chunk_id, &isnull); Assert(!isnull); int32 current_chunk_id = DatumGetInt32(datum); Assert(current_chunk_id != INVALID_CHUNK_ID); /* * We have only the dimension constraints here, because we're searching * by dimension slice id. */ Assert(!slot_attisnull(ts_scan_iterator_slot(&iterator), Anum_chunk_constraint_dimension_slice_id)); scan_add_chunk_context(&ctx, current_chunk_id, dimension_vecs, &chunk_ids); } } } ts_scan_iterator_close(&iterator); chunk_scan_ctx_destroy(&ctx); return chunk_ids; } ChunkStub * ts_chunk_stub_create(int32 id, int16 num_constraints) { ChunkStub *stub; stub = palloc0(sizeof(*stub)); stub->id = id; if (num_constraints > 0) stub->constraints = ts_chunk_constraints_alloc(num_constraints, CurrentMemoryContext); return stub; } Chunk * ts_chunk_create_base(int32 id, int16 num_constraints, const char relkind) { Chunk *chunk; chunk = palloc0(sizeof(Chunk)); chunk->fd.id = id; chunk->fd.compressed_chunk_id = INVALID_CHUNK_ID; chunk->relkind = relkind; chunk->fd.creation_time = GetCurrentTimestamp(); if (num_constraints > 0) chunk->constraints = ts_chunk_constraints_alloc(num_constraints, CurrentMemoryContext); return chunk; } /* * Build a chunk from a chunk tuple and a stub. * * The stub allows the chunk to be constructed more efficiently. But if the stub * is not "valid", dimension slices and constraints are fully * rescanned/recreated. */ Chunk * ts_chunk_build_from_tuple_and_stub(Chunk **chunkptr, TupleInfo *ti, const ChunkStub *stub, const ScanTupLock *slice_lock) { Chunk *chunk = NULL; int num_constraints_hint = stub ? stub->constraints->num_constraints : 2; if (chunkptr == NULL) chunkptr = &chunk; if (*chunkptr == NULL) *chunkptr = MemoryContextAllocZero(ti->mctx, sizeof(Chunk)); chunk = *chunkptr; ts_chunk_formdata_fill(&chunk->fd, ti); /* * When searching for the chunk stub matching the dimensional point, we * only scanned for dimensional constraints. We now need to rescan the * constraints to also get the inherited constraints. */ chunk->constraints = ts_chunk_constraint_scan_by_chunk_id(chunk->fd.id, num_constraints_hint, ti->mctx); /* If a stub is provided then reuse its hypercube. Note that stubs that * are results of a point or range scan might be incomplete (in terms of * number of slices and constraints). Only a chunk stub that matches in * all dimensions will have a complete hypercube. Thus, we need to check * the validity of the stub before we can reuse it. */ if (chunk_stub_is_valid(stub, chunk->constraints->num_dimension_constraints)) { MemoryContext oldctx = MemoryContextSwitchTo(ti->mctx); chunk->cube = ts_hypercube_copy(stub->cube); MemoryContextSwitchTo(oldctx); /* * The hypercube slices were filled in during the scan. Now we need to * sort them in dimension order. */ ts_hypercube_slice_sort(chunk->cube); } else { ScanIterator it = ts_dimension_slice_scan_iterator_create(slice_lock, ti->mctx); chunk->cube = ts_hypercube_from_constraints(chunk->constraints, &it); ts_scan_iterator_close(&it); } chunk->hypertable_relid = ts_hypertable_id_to_relid(chunk->fd.hypertable_id, false); ts_get_rel_info_by_name(NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name), &chunk->table_id, &chunk->relkind); Ensure(chunk->relkind > 0, "relkind for chunk \"%s\".\"%s\" is invalid", NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name)); return chunk; } static ScanTupleResult chunk_tuple_found(TupleInfo *ti, void *arg) { ChunkStubScanCtx *stubctx = arg; /* * The chunk table could also have been dropped concurrently. Try to * acquire the requested lock in order to guarantee that the chunk table * still exists. */ if (stubctx->chunk_lockmode != NoLock) { Datum schema_name; Datum table_name; const RangeVar *rv; bool isnull; schema_name = slot_getattr(ti->slot, Anum_chunk_schema_name, &isnull); Assert(!isnull); table_name = slot_getattr(ti->slot, Anum_chunk_table_name, &isnull); Assert(!isnull); rv = makeRangeVar(NameStr(*DatumGetName(schema_name)), NameStr(*DatumGetName(table_name)), -1); Relation rel = table_openrv_extended(rv, stubctx->chunk_lockmode, true); if (!rel) return SCAN_DONE; table_close(rel, NoLock); } ts_chunk_build_from_tuple_and_stub(&stubctx->chunk, ti, stubctx->stub, stubctx->slice_lock); return SCAN_DONE; } /* Create a chunk by scanning on chunk ID. A stub must be provided as input. */ static Chunk * chunk_create_from_stub(ChunkStubScanCtx *stubctx) { ScanKeyData scankey[1]; Catalog *catalog = ts_catalog_get(); int num_found; ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, CHUNK), .index = catalog_get_index(catalog, CHUNK, CHUNK_ID_INDEX), .nkeys = 1, .scankey = scankey, .data = stubctx, .tuple_found = chunk_tuple_found, .lockmode = AccessShareLock, .scandirection = ForwardScanDirection, }; /* * Perform an index scan on chunk ID. */ ScanKeyInit(&scankey[0], Anum_chunk_idx_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(stubctx->stub->id)); num_found = ts_scanner_scan(&scanctx); Assert(num_found == 0 || num_found == 1); if (num_found != 1) elog(ERROR, "no chunk found with ID %d", stubctx->stub->id); Assert(stubctx->chunk != NULL); return stubctx->chunk; } /* * Initialize a chunk scan context. * * A chunk scan context is used to join chunk-related information from metadata * tables during scans. */ static void chunk_scan_ctx_init(ChunkScanCtx *ctx, const Hypertable *ht, const Point *point) { struct HASHCTL hctl = { .keysize = sizeof(int32), .entrysize = sizeof(ChunkScanEntry), .hcxt = CurrentMemoryContext, }; memset(ctx, 0, sizeof(*ctx)); ctx->htab = hash_create("chunk-scan-context", 20, &hctl, HASH_ELEM | HASH_CONTEXT | HASH_BLOBS); ctx->ht = ht; ctx->point = point; ctx->lockmode = NoLock; } /* * Destroy the chunk scan context. * * This will free the hash table in the context, but not the chunks within since * they are not allocated on the hash tables memory context. */ static void chunk_scan_ctx_destroy(ChunkScanCtx *ctx) { hash_destroy(ctx->htab); } static inline void dimension_slice_and_chunk_constraint_join(ChunkScanCtx *scanctx, const DimensionVec *vec) { int i; for (i = 0; i < vec->num_slices; i++) { /* * For each dimension slice, find matching constraints. These will be * saved in the scan context */ ts_chunk_constraint_scan_by_dimension_slice(vec->slices[i], scanctx, CurrentMemoryContext); } } /* * Scan for chunks that collide with the given hypercube. * * Collisions are determined using axis-aligned bounding box collision detection * generalized to N dimensions. Slices are collected in the scan context's hash * table according to the chunk IDs they are associated with. A slice might * represent the dimensional bound of multiple chunks, and thus is added to all * the hash table slots of those chunks. At the end of the scan, those chunks * that have a full set of slices are the ones that actually collide with the * given hypercube. * * Chunks in the scan context that do not collide (do not have a full set of * slices), might still be important for ensuring alignment in those dimensions * that require alignment. */ static void chunk_collision_scan(ChunkScanCtx *scanctx, const Hypercube *cube) { int i; /* Scan all dimensions for colliding slices */ for (i = 0; i < scanctx->ht->space->num_dimensions; i++) { DimensionVec *vec; DimensionSlice *slice = cube->slices[i]; vec = dimension_slice_collision_scan(slice->fd.dimension_id, slice->fd.range_start, slice->fd.range_end); /* Add the slices to all the chunks they are associated with */ dimension_slice_and_chunk_constraint_join(scanctx, vec); } } /* * Apply a function to each stub in the scan context's hash table. If the limit * is greater than zero only a limited number of chunks will be processed. * * The chunk handler function (on_chunk_func) should return CHUNK_PROCESSED if * the chunk should be considered processed and count towards the given * limit. CHUNK_IGNORE can be returned to have a chunk NOT count towards the * limit. CHUNK_DONE counts the chunk but aborts processing irrespective of * whether the limit is reached or not. * * Returns the number of processed chunks. */ static int chunk_scan_ctx_foreach_chunk_stub(ChunkScanCtx *ctx, on_chunk_stub_func on_chunk, uint64 limit) { HASH_SEQ_STATUS status; ChunkScanEntry *entry; ctx->num_processed = 0; hash_seq_init(&status, ctx->htab); for (entry = hash_seq_search(&status); entry != NULL; entry = hash_seq_search(&status)) { switch (on_chunk(ctx, entry->stub)) { case CHUNK_DONE: ctx->num_processed++; hash_seq_term(&status); return ctx->num_processed; case CHUNK_PROCESSED: ctx->num_processed++; if (limit > 0 && ctx->num_processed == limit) { hash_seq_term(&status); return ctx->num_processed; } break; case CHUNK_IGNORED: break; } } return ctx->num_processed; } typedef struct ChunkScanCtxAddChunkData { Chunk *chunks; uint64 max_chunks; uint64 num_chunks; } ChunkScanCtxAddChunkData; static ChunkResult chunk_scan_context_add_chunk(ChunkScanCtx *scanctx, ChunkStub *stub) { ChunkScanCtxAddChunkData *data = scanctx->data; ChunkStubScanCtx stubctx = { .chunk = &data->chunks[data->num_chunks], .stub = stub, }; Assert(data->num_chunks < data->max_chunks); chunk_create_from_stub(&stubctx); data->num_chunks++; return CHUNK_PROCESSED; } TM_Result ts_chunk_lock_for_creating_compressed_chunk(int32 chunk_id, int32 *compressed_chunk_id) { ScanIterator iterator; bool found = false; TM_Result lockresult; ScanTupLock tuplock = { .lockmode = LockTupleExclusive, .waitpolicy = LockWaitBlock, .lockflags = TUPLE_LOCK_FLAG_FIND_LAST_VERSION, }; iterator = ts_scan_iterator_create(CHUNK, RowShareLock, CurrentMemoryContext); ts_chunk_scan_iterator_set_chunk_id(&iterator, chunk_id); iterator.ctx.tuplock = &tuplock; ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); lockresult = ti->lockresult; if (lockresult == TM_Ok && compressed_chunk_id) { bool isnull; Datum value = slot_getattr(ti->slot, Anum_chunk_compressed_chunk_id, &isnull); *compressed_chunk_id = isnull ? INVALID_CHUNK_ID : DatumGetInt32(value); } found = true; } ts_scan_iterator_close(&iterator); if (!found) elog(ERROR, "chunk with ID %d does not exist", chunk_id); return lockresult; } /* * Scan for the chunk that encloses the given point. * * In each dimension there can be one or more slices that match the point's * coordinate in that dimension. Slices are collected in the scan context's hash * table according to the chunk IDs they are associated with. A slice might * represent the dimensional bound of multiple chunks, and thus is added to all * the hash table slots of those chunks. At the end of the scan there will be at * most one chunk that has a complete set of slices, since a point cannot belong * to two chunks. * * This involves: * * 1) For each dimension: * - Find all dimension slices that match the dimension * 2) For each dimension slice: * - Find all chunk constraints matching the dimension slice * 3) For each matching chunk constraint * - Insert a chunk stub into a hash table and add the constraint to the chunk * - If chunk already exists in hash table, add the constraint to the chunk * 4) At the end of the scan, only one chunk in the hash table should have * N number of constraints. This is the matching chunk. * * NOTE: this function allocates transient data, e.g., dimension slice, * constraints and chunks, that in the end are not part of the returned * chunk. Therefore, this scan should be executed on a transient memory * context. The returned chunk needs to be copied into another memory context in * case it needs to live beyond the lifetime of the other data. * * The slices can be locked by specifying an optional slice lock. */ int32 ts_chunk_point_find_chunk_id(const Hypertable *ht, const Point *p, const ScanTupLock *slice_lock) { int32 matching_chunk_id = 0; /* The scan context will keep the state accumulated during the scan */ ChunkScanCtx ctx; chunk_scan_ctx_init(&ctx, ht, p); /* Scan all dimensions for slices enclosing the point */ List *all_slices = NIL; for (int dimension_index = 0; dimension_index < ctx.ht->space->num_dimensions; dimension_index++) { ts_dimension_slice_scan_list(ctx.ht->space->dimensions[dimension_index].fd.id, p->coordinates[dimension_index], &all_slices, slice_lock); } /* Find constraints matching dimension slices. */ ScanIterator iterator = ts_chunk_constraint_scan_iterator_create(CurrentMemoryContext); ListCell *lc; foreach (lc, all_slices) { DimensionSlice *slice = (DimensionSlice *) lfirst(lc); ts_chunk_constraint_scan_iterator_set_slice_id(&iterator, slice->fd.id); ts_scan_iterator_start_or_restart_scan(&iterator); while (ts_scan_iterator_next(&iterator) != NULL) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); bool PG_USED_FOR_ASSERTS_ONLY isnull = true; Datum datum = slot_getattr(ti->slot, Anum_chunk_constraint_chunk_id, &isnull); Assert(!isnull); int32 current_chunk_id = DatumGetInt32(datum); Assert(current_chunk_id != INVALID_CHUNK_ID); bool found = false; ChunkScanEntry *entry = hash_search(ctx.htab, ¤t_chunk_id, HASH_ENTER, &found); if (!found) { entry->stub = NULL; entry->num_dimension_constraints = 0; } /* * We have only the dimension constraints here, because we're searching * by dimension slice id. */ Assert(!slot_attisnull(ts_scan_iterator_slot(&iterator), Anum_chunk_constraint_dimension_slice_id)); entry->num_dimension_constraints++; /* * A chunk is complete when we've found slices for all its dimensions, * i.e., a complete hypercube. Only one chunk matches a given hyperspace * point, so we can stop early. */ if (entry->num_dimension_constraints == ctx.ht->space->num_dimensions) { matching_chunk_id = entry->chunk_id; break; } } if (matching_chunk_id != INVALID_CHUNK_ID) { break; } } ts_scan_iterator_close(&iterator); chunk_scan_ctx_destroy(&ctx); return matching_chunk_id; } /* * Find all the chunks in hyperspace that include elements (dimension slices) * calculated by given range constraints and return the corresponding * ChunkScanCxt. It is the caller's responsibility to destroy this context after * usage. */ static void chunks_find_all_in_range_limit(const Hypertable *ht, const Dimension *time_dim, StrategyNumber start_strategy, int64 start_value, StrategyNumber end_strategy, int64 end_value, int limit, uint64 *num_found, ScanTupLock *tuplock, ChunkScanCtx *ctx) { DimensionVec *slices; Assert(ht != NULL); /* must have been checked earlier that this is the case */ Assert(time_dim != NULL); slices = ts_dimension_slice_scan_range_limit(time_dim->fd.id, start_strategy, start_value, end_strategy, end_value, limit, tuplock); /* The scan context will keep the state accumulated during the scan */ chunk_scan_ctx_init(ctx, ht, NULL); /* No abort when the first chunk is found */ ctx->early_abort = false; /* Scan for chunks that are in range */ dimension_slice_and_chunk_constraint_join(ctx, slices); *num_found += hash_get_num_entries(ctx->htab); } /* show_chunks SQL function handler */ Datum ts_chunk_show_chunks(PG_FUNCTION_ARGS) { /* * show_chunks_return_srf is called even when it is not the first call but only * after doing some computation first */ if (SRF_IS_FIRSTCALL()) { FuncCallContext *funcctx; Oid relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); Hypertable *ht; const Dimension *time_dim; Cache *hcache; int64 older_than = PG_INT64_MAX; int64 newer_than = PG_INT64_MIN; int64 created_before = PG_INT64_MAX; int64 created_after = PG_INT64_MIN; Oid time_type; Oid arg_type; bool older_newer = false; bool before_after = false; hcache = ts_hypertable_cache_pin(); ht = ts_resolve_hypertable_from_table_or_cagg(hcache, relid, true); Assert(ht != NULL); time_dim = hyperspace_get_open_dimension(ht->space, 0); if (!time_dim) time_dim = hyperspace_get_closed_dimension(ht->space, 0); if (time_dim && IS_CLOSED_DIMENSION(time_dim) && (!PG_ARGISNULL(1) || !PG_ARGISNULL(2))) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot specify \"older_than\" or \"newer_than\" for " "\"closed\"-like partitioning types"), errhint("Use \"created_before\" and/or \"created_after\" which rely on the " "chunk creation time values."))); if (time_dim) time_type = ts_dimension_get_partition_type(time_dim); else time_type = InvalidOid; /* * Treat UUID (v7) as a timestamptz type. The expected input is an interval or absolute * timestamptz. */ if (IS_UUID_TYPE(time_type)) time_type = TIMESTAMPTZOID; /* note that arg_types will be the same for all specified "ANY" elements for a given call */ arg_type = InvalidOid; if (!PG_ARGISNULL(1)) { arg_type = get_fn_expr_argtype(fcinfo->flinfo, 1); older_than = ts_time_value_from_arg(PG_GETARG_DATUM(1), arg_type, time_type, true); older_newer = true; } if (!PG_ARGISNULL(2)) { arg_type = get_fn_expr_argtype(fcinfo->flinfo, 2); newer_than = ts_time_value_from_arg(PG_GETARG_DATUM(2), arg_type, time_type, true); older_newer = true; } /* * We cannot have a mix of [older_than/newer_than] and [created_before/created_after]. * So, check that first. Note that created_before/created_after have a type of * TIMESTAMPTZOID regardless of the partitioning type. */ if (!PG_ARGISNULL(3)) { if (older_newer) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot specify \"older_than\" or \"newer_than\" together with " "\"created_before\"" "or \"created_after\""))); arg_type = get_fn_expr_argtype(fcinfo->flinfo, 3); /* We use the existing function for various type/conversion checks */ created_before = ts_time_value_from_arg(PG_GETARG_DATUM(3), arg_type, TIMESTAMPTZOID, false); /* convert into int64 format for comparisons */ created_before = ts_internal_to_time_int64(created_before, TIMESTAMPTZOID); before_after = true; } if (!PG_ARGISNULL(4)) { if (older_newer) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot specify \"older_than\" or \"newer_than\" together with " "\"created_before\"" "or \"created_after\""))); arg_type = get_fn_expr_argtype(fcinfo->flinfo, 4); /* We use the existing function for various type/conversion checks */ created_after = ts_time_value_from_arg(PG_GETARG_DATUM(4), arg_type, TIMESTAMPTZOID, false); /* convert into int64 format for comparisons */ created_after = ts_internal_to_time_int64(created_after, TIMESTAMPTZOID); before_after = true; } /* if both have not been specified then default to older_newer */ if (!older_newer && !before_after) older_newer = true; funcctx = SRF_FIRSTCALL_INIT(); /* * For INTEGER type dimensions, we support querying using intervals or any * timestamp or date input. For such INTEGER dimensions, we get the chunks * using their creation time values. */ if (IS_INTEGER_TYPE(time_type) && (arg_type == INTERVALOID || IS_TIMESTAMP_TYPE(arg_type))) { /* check that we use proper inputs for such cases */ if (older_newer) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot specify \"older_than\" and/or \"newer_than\" for " "\"integer\"-like partitioning types"), errhint( "Use \"created_before\" and/or \"created_after\" which rely on the " "chunk creation time values."))); funcctx->user_fctx = get_chunks_in_creation_time_range(ht, created_before, created_after, funcctx->multi_call_memory_ctx, &funcctx->max_calls, NULL); } else { /* check that we use proper inputs for such cases */ if (!older_newer) { funcctx->user_fctx = get_chunks_in_creation_time_range(ht, created_before, created_after, funcctx->multi_call_memory_ctx, &funcctx->max_calls, NULL); } else funcctx->user_fctx = get_chunks_in_time_range(ht, older_than, newer_than, funcctx->multi_call_memory_ctx, &funcctx->max_calls, NULL); } ts_cache_release(&hcache); } return show_chunks_return_srf(fcinfo); } static Chunk * get_chunks_in_time_range(Hypertable *ht, int64 older_than, int64 newer_than, MemoryContext mctx, uint64 *num_chunks_returned, ScanTupLock *tuplock) { MemoryContext oldcontext; ChunkScanCtx chunk_scan_ctx; Chunk *chunks; ChunkScanCtxAddChunkData data; const Dimension *time_dim; StrategyNumber start_strategy; StrategyNumber end_strategy; uint64 num_chunks = 0; if (older_than <= newer_than) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid time range"), errhint("The start of the time range must be before the end."))); if (TS_HYPERTABLE_IS_INTERNAL_COMPRESSION_TABLE(ht)) ereport(ERROR, (errcode(ERRCODE_TS_OPERATION_NOT_SUPPORTED), errmsg("invalid operation on compressed hypertable"))); start_strategy = (newer_than == PG_INT64_MIN) ? InvalidStrategy : BTGreaterEqualStrategyNumber; end_strategy = (older_than == PG_INT64_MAX) ? InvalidStrategy : BTLessStrategyNumber; time_dim = hyperspace_get_open_dimension(ht->space, 0); if (time_dim == NULL) time_dim = hyperspace_get_closed_dimension(ht->space, 0); Ensure(time_dim != NULL, "partitioning dimension not found for hypertable \"%s\".\"%s\"", NameStr(ht->fd.schema_name), NameStr(ht->fd.table_name)); oldcontext = MemoryContextSwitchTo(mctx); chunks_find_all_in_range_limit(ht, time_dim, start_strategy, newer_than, end_strategy, older_than, -1, &num_chunks, tuplock, &chunk_scan_ctx); MemoryContextSwitchTo(oldcontext); chunks = MemoryContextAllocZero(mctx, sizeof(Chunk) * num_chunks); data = (ChunkScanCtxAddChunkData){ .chunks = chunks, .max_chunks = num_chunks, .num_chunks = 0, }; /* Get all the chunks from the context */ chunk_scan_ctx.data = &data; chunk_scan_ctx_foreach_chunk_stub(&chunk_scan_ctx, chunk_scan_context_add_chunk, 0); /* * only affects ctx.htab Got all the chunk already so can now safely * destroy the context */ chunk_scan_ctx_destroy(&chunk_scan_ctx); *num_chunks_returned = data.num_chunks; qsort(chunks, *num_chunks_returned, sizeof(Chunk), chunk_cmp); #ifdef USE_ASSERT_CHECKING do { uint64 i = 0; /* Assert that we never return dropped chunks */ for (i = 0; i < *num_chunks_returned; i++) ASSERT_IS_VALID_CHUNK(&chunks[i]); } while (false); #endif return chunks; } Chunk * ts_chunk_copy(const Chunk *chunk) { Chunk *copy; ASSERT_IS_VALID_CHUNK(chunk); copy = palloc(sizeof(Chunk)); memcpy(copy, chunk, sizeof(Chunk)); if (NULL != chunk->constraints) copy->constraints = ts_chunk_constraints_copy(chunk->constraints); if (NULL != chunk->cube) copy->cube = ts_hypercube_copy(chunk->cube); return copy; } static int chunk_scan_internal(int indexid, ScanKeyData scankey[], int nkeys, tuple_found_func tuple_found, void *data, int limit, ScanDirection scandir, LOCKMODE lockmode, MemoryContext mctx) { Catalog *catalog = ts_catalog_get(); ScannerCtx ctx = { .table = catalog_get_table_id(catalog, CHUNK), .index = catalog_get_index(catalog, CHUNK, indexid), .nkeys = nkeys, .data = data, .scankey = scankey, .tuple_found = tuple_found, .limit = limit, .lockmode = lockmode, .scandirection = scandir, .result_mctx = mctx, }; return ts_scanner_scan(&ctx); } /* * Get a window of chunks that "precedes" the given dimensional point. * * For instance, if the dimension is "time", then given a point in time the * function returns the recent chunks that come before the chunk that includes * that point. The count parameter determines the number or slices the window * should include in the given dimension. Note, that with multi-dimensional * partitioning, there might be multiple chunks in each dimensional slice that * all precede the given point. For instance, the example below shows two * different situations that each go "back" two slices (count = 2) in the * x-dimension, but returns two vs. eight chunks due to different * partitioning. * * '_____________ * '| | | * | * '|___|___|___| * ' * ' * '____ ________ * '| | | * | * '|___|___|___| * '| | | | * '|___|___|___| * '| | | | * '|___|___|___| * '| | | | * '|___|___|___| * * Note that the returned chunks will be allocated on the given memory * context, including the list itself. So, beware of not leaking the list if * the chunks are later cached somewhere else. */ List * ts_chunk_get_window(int32 dimension_id, int64 point, int count, MemoryContext mctx) { List *chunks = NIL; DimensionVec *dimvec; int i; /* Scan for "count" slices that precede the point in the given dimension */ dimvec = ts_dimension_slice_scan_by_dimension_before_point(dimension_id, point, count, BackwardScanDirection, mctx); /* * For each slice, join with any constraints that reference the slice. * There might be multiple constraints for each slice in case of * multi-dimensional partitioning. */ for (i = 0; i < dimvec->num_slices; i++) { DimensionSlice *slice = dimvec->slices[i]; ChunkConstraints *ccs = ts_chunk_constraints_alloc(1, mctx); int j; ts_chunk_constraint_scan_by_dimension_slice_id(slice->fd.id, ccs, mctx); /* For each constraint, find the corresponding chunk */ for (j = 0; j < ccs->num_constraints; j++) { ChunkConstraint *cc = &ccs->constraints[j]; Chunk *chunk = ts_chunk_get_by_id(cc->fd.chunk_id, false); MemoryContext old; ScanIterator it; /* Dropped chunks do not contain valid data and must not be returned */ if (!chunk) continue; chunk->constraints = ts_chunk_constraint_scan_by_chunk_id(chunk->fd.id, 1, mctx); it = ts_dimension_slice_scan_iterator_create(NULL, mctx); chunk->cube = ts_hypercube_from_constraints(chunk->constraints, &it); ts_scan_iterator_close(&it); /* Allocate the list on the same memory context as the chunks */ old = MemoryContextSwitchTo(mctx); chunks = lappend(chunks, chunk); MemoryContextSwitchTo(old); } } #ifdef USE_ASSERT_CHECKING /* Assert that we never return dropped chunks */ do { ListCell *lc; foreach (lc, chunks) { Chunk *chunk = lfirst(lc); ASSERT_IS_VALID_CHUNK(chunk); } } while (false); #endif return chunks; } static Chunk * chunk_scan_find(int indexid, ScanKeyData scankey[], int nkeys, MemoryContext mctx, LOCKMODE chunk_lockmode, const ScanTupLock *slice_lock, bool fail_if_not_found, const DisplayKeyData displaykey[]) { ChunkStubScanCtx stubctx = { .slice_lock = slice_lock, .chunk_lockmode = chunk_lockmode, }; Chunk *chunk; int num_found; num_found = chunk_scan_internal(indexid, scankey, nkeys, chunk_tuple_found, &stubctx, 1, ForwardScanDirection, AccessShareLock, mctx); Assert(num_found == 0 || num_found == 1); chunk = stubctx.chunk; switch (num_found) { case 0: if (fail_if_not_found) { int i = 0; StringInfoData info; initStringInfo(&info); while (i < nkeys) { appendStringInfo(&info, "%s: %s", displaykey[i].name, displaykey[i].as_string(scankey[i].sk_argument)); if (++i < nkeys) appendStringInfoString(&info, ", "); } ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("chunk not found"), errdetail("%s", info.data))); } break; case 1: if (chunk) { ASSERT_IS_VALID_CHUNK(chunk); } break; default: elog(ERROR, "expected a single chunk, found %d", num_found); } return chunk; } Chunk * ts_chunk_get_by_name_with_memory_context(const char *schema_name, const char *table_name, LOCKMODE chunk_lockmode, const ScanTupLock *slice_lock, MemoryContext mctx, bool fail_if_not_found) { NameData schema, table; ScanKeyData scankey[2]; static const DisplayKeyData displaykey[2] = { [0] = { .name = "schema_name", .as_string = DatumGetNameString }, [1] = { .name = "table_name", .as_string = DatumGetNameString }, }; /* Early check for rogue input */ if (schema_name == NULL || table_name == NULL) { if (fail_if_not_found) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("chunk not found"), errdetail("schema_name: %s, table_name: %s", schema_name ? schema_name : "<null>", table_name ? table_name : "<null>"))); else return NULL; } namestrcpy(&schema, schema_name); namestrcpy(&table, table_name); /* * Check that the table actually exists and get a lock, unless no lock * requested. */ if (chunk_lockmode != NoLock) { RangeVar *rv = makeRangeVar(NameStr(schema), NameStr(table), -1); Relation rel = table_openrv_extended(rv, chunk_lockmode, true); if (!rel) { if (fail_if_not_found) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("chunk not found"), errdetail("schema_name: %s, table_name: %s", schema_name, table_name))); return NULL; } table_close(rel, NoLock); } /* * Perform an index scan on chunk name. */ ScanKeyInit(&scankey[0], Anum_chunk_schema_name_idx_schema_name, BTEqualStrategyNumber, F_NAMEEQ, NameGetDatum(&schema)); ScanKeyInit(&scankey[1], Anum_chunk_schema_name_idx_table_name, BTEqualStrategyNumber, F_NAMEEQ, NameGetDatum(&table)); return chunk_scan_find(CHUNK_SCHEMA_NAME_INDEX, scankey, 2, mctx, chunk_lockmode, slice_lock, fail_if_not_found, displaykey); } Chunk * ts_chunk_get_by_relid_locked(Oid relid, LOCKMODE chunk_lockmode, const ScanTupLock *slice_lock, bool fail_if_not_found) { char *schema; char *table; if (!OidIsValid(relid)) { if (fail_if_not_found) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("invalid Oid"))); else return NULL; } schema = get_namespace_name(get_rel_namespace(relid)); table = get_rel_name(relid); return chunk_get_by_name(schema, table, chunk_lockmode, slice_lock, fail_if_not_found); } Chunk * ts_chunk_get_by_relid(Oid relid, bool fail_if_not_found) { ScanTupLock slice_lock = { .lockmode = LockTupleKeyShare, .waitpolicy = LockWaitBlock, .lockflags = TUPLE_LOCK_FLAG_FIND_LAST_VERSION, }; return ts_chunk_get_by_relid_locked(relid, NoLock, &slice_lock, fail_if_not_found); } void ts_chunk_free(Chunk *chunk) { if (chunk->cube) { ts_hypercube_free(chunk->cube); } if (chunk->constraints) { ChunkConstraints *c = chunk->constraints; pfree(c->constraints); pfree(c); } pfree(chunk); } static const char * DatumGetInt32AsString(Datum datum) { char *buf = (char *) palloc(12); /* sign, 10 digits, '\0' */ pg_ltoa(DatumGetInt32(datum), buf); return buf; } Chunk * ts_chunk_get_by_id_with_slice_lock(int32 id, LOCKMODE chunk_lockmode, const ScanTupLock *slice_lock, bool fail_if_not_found) { ScanKeyData scankey[1]; static const DisplayKeyData displaykey[1] = { [0] = { .name = "id", .as_string = DatumGetInt32AsString }, }; /* * Perform an index scan on chunk id. */ ScanKeyInit(&scankey[0], Anum_chunk_idx_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(id)); return chunk_scan_find(CHUNK_ID_INDEX, scankey, 1, CurrentMemoryContext, chunk_lockmode, slice_lock, fail_if_not_found, displaykey); } Chunk * ts_chunk_get_by_id(int32 id, bool fail_if_not_found) { ScanTupLock slice_lock = { .lockmode = LockTupleKeyShare, .waitpolicy = LockWaitBlock, .lockflags = TUPLE_LOCK_FLAG_FIND_LAST_VERSION, }; return ts_chunk_get_by_id_with_slice_lock(id, NoLock, &slice_lock, fail_if_not_found); } /* * Simple scans provide lightweight ways to access chunk information without the * overhead of getting a full chunk (i.e., no extra metadata, like constraints, * are joined in). This function forms the basis of a number of lookup functions * that, e.g., translates a chunk relid to a chunk_id, or vice versa. */ static bool chunk_simple_scan(ScanIterator *iterator, FormData_chunk *form, bool missing_ok, const DisplayKeyData displaykey[]) { int count = 0; ts_scanner_foreach(iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(iterator); ts_chunk_formdata_fill(form, ti); count++; } Assert(count == 0 || count == 1); if (count == 0 && !missing_ok) { int i = 0; StringInfoData info; initStringInfo(&info); while (i < iterator->ctx.nkeys) { appendStringInfo(&info, "%s: %s", displaykey[i].name, displaykey[i].as_string(iterator->ctx.scankey[i].sk_argument)); if (++i < iterator->ctx.nkeys) appendStringInfoString(&info, ", "); } ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("chunk not found"), errdetail("%s", info.data))); } return count == 1; } static bool chunk_simple_scan_by_name(const char *schema, const char *table, FormData_chunk *form, bool missing_ok) { ScanIterator iterator; static const DisplayKeyData displaykey[] = { [0] = { .name = "schema_name", .as_string = DatumGetNameString }, [1] = { .name = "table_name", .as_string = DatumGetNameString }, }; if (schema == NULL || table == NULL) return false; iterator = ts_scan_iterator_create(CHUNK, AccessShareLock, CurrentMemoryContext); init_scan_by_qualified_table_name(&iterator, schema, table); return chunk_simple_scan(&iterator, form, missing_ok, displaykey); } bool ts_chunk_simple_scan_by_reloid(Oid reloid, FormData_chunk *form, bool missing_ok) { bool found = false; if (OidIsValid(reloid)) { const char *table = get_rel_name(reloid); if (table != NULL) { Oid nspid = get_rel_namespace(reloid); const char *schema = get_namespace_name(nspid); found = chunk_simple_scan_by_name(schema, table, form, missing_ok); } } if (!found && !missing_ok) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("chunk with reloid %u not found", reloid))); return found; } static bool chunk_simple_scan_by_id(int32 chunk_id, FormData_chunk *form, bool missing_ok) { ScanIterator iterator; static const DisplayKeyData displaykey[] = { [0] = { .name = "id", .as_string = DatumGetInt32AsString }, }; iterator = ts_scan_iterator_create(CHUNK, AccessShareLock, CurrentMemoryContext); ts_chunk_scan_iterator_set_chunk_id(&iterator, chunk_id); return chunk_simple_scan(&iterator, form, missing_ok, displaykey); } /* * Lookup a Chunk ID from a chunk's relid. */ Datum ts_chunk_id_from_relid(PG_FUNCTION_ARGS) { static Oid last_relid = InvalidOid; static int32 last_id = 0; Oid relid = PG_GETARG_OID(0); FormData_chunk form; if (last_relid == relid) return last_id; ts_chunk_simple_scan_by_reloid(relid, &form, false); last_relid = relid; last_id = form.id; PG_RETURN_INT32(last_id); } bool ts_chunk_exists_relid(Oid relid) { FormData_chunk form; return ts_chunk_simple_scan_by_reloid(relid, &form, true); } /* * Returns 0 if there is no chunk with such reloid. */ int32 ts_chunk_get_hypertable_id_by_reloid(Oid reloid) { FormData_chunk form; if (ts_chunk_simple_scan_by_reloid(reloid, &form, /* missing_ok = */ true)) { return form.hypertable_id; } return 0; } FormData_chunk ts_chunk_get_formdata(int32 chunk_id) { FormData_chunk fd; chunk_simple_scan_by_id(chunk_id, &fd, /* missing_ok = */ false); return fd; } /* * Get the relid of a chunk given its ID. */ Oid ts_chunk_get_relid(int32 chunk_id, bool missing_ok) { FormData_chunk form = { 0 }; Oid relid = InvalidOid; if (chunk_simple_scan_by_id(chunk_id, &form, missing_ok)) relid = ts_get_relation_relid(NameStr(form.schema_name), NameStr(form.table_name), true); if (!OidIsValid(relid) && !missing_ok) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_SCHEMA), errmsg("chunk with id %d not found", chunk_id))); return relid; } /* * Get the schema (namespace) of a chunk given its ID. * * This is a lightweight way to get the schema of a chunk without creating a * full Chunk object that joins in constraints, etc. */ Oid ts_chunk_get_schema_id(int32 chunk_id, bool missing_ok) { FormData_chunk form = { 0 }; if (!chunk_simple_scan_by_id(chunk_id, &form, missing_ok)) return InvalidOid; return get_namespace_oid(NameStr(form.schema_name), missing_ok); } bool ts_chunk_get_id(const char *schema, const char *table, int32 *chunk_id, bool missing_ok) { FormData_chunk form = { 0 }; if (!chunk_simple_scan_by_name(schema, table, &form, missing_ok)) return false; if (NULL != chunk_id) *chunk_id = form.id; return true; } /* Delete the chunk tuple. * * relid: Required when deleting via an event trigger hook, because at that * point the relation is gone and it is no longer possible to resolve the Oid * from the PG catalog. * */ static void chunk_tuple_delete(TupleInfo *ti, Oid relid, DropBehavior behavior, bool detach) { FormData_chunk form; CatalogSecurityContext sec_ctx; int i; ts_chunk_formdata_fill(&form, ti); ChunkConstraints *ccs; /* * Do not drop any constraint if detaching * We will still need to delete dimension slices for the chunk */ ccs = ts_chunk_constraints_alloc(2, ti->mctx); ts_chunk_constraint_delete_dimensional_constraints(form.id, ccs); ts_chunk_constraint_delete_by_chunk_id(form.id, ccs, !detach); /* Check for dimension slices that are orphaned by the chunk deletion */ for (i = 0; i < ccs->num_constraints; i++) { ChunkConstraint *cc = &ccs->constraints[i]; /* * Delete the dimension slice if there are no remaining constraints * referencing it */ if (is_dimension_constraint(cc)) { /* * Dimension slices are shared between chunk constraints and * subsequently between chunks as well. Since different chunks * can reference the same dimension slice (through the chunk * constraint), we must lock the dimension slice in FOR UPDATE * mode *prior* to scanning the chunk constraints table. If we * do not do that, we can have the following scenario: * * - T1: Prepares to create a chunk that uses an existing dimension slice X * - T2: Deletes a chunk and dimension slice X because it is not * references by a chunk constraint. * - T1: Adds a chunk constraint referencing dimension * slice X (which is about to be deleted by T2). */ ScanTupLock tuplock = { .lockmode = LockTupleExclusive, .waitpolicy = LockWaitBlock }; DimensionSlice *slice = ts_dimension_slice_scan_by_id_and_lock(cc->fd.dimension_slice_id, &tuplock, CurrentMemoryContext, AccessShareLock); /* If the slice is not found in the scan above, the table is * broken so we do not delete the slice. We proceed * anyway since users need to be able to drop broken tables or * remove broken chunks. */ if (!slice) { const Hypertable *const ht = ts_hypertable_get_by_id(form.hypertable_id); ereport(WARNING, (errmsg("unexpected state for chunk %s.%s, dropping anyway", quote_identifier(NameStr(form.schema_name)), quote_identifier(NameStr(form.table_name))), errdetail("The integrity of hypertable %s.%s might be " "compromised " "since one of its chunks lacked a dimension slice.", quote_identifier(NameStr(ht->fd.schema_name)), quote_identifier(NameStr(ht->fd.table_name))))); } else if (ts_chunk_constraint_scan_by_dimension_slice_id(slice->fd.id, NULL, CurrentMemoryContext) == 0) ts_dimension_slice_delete_by_id(cc->fd.dimension_slice_id, false); } } /* * Even tough we keep foreign key constraints on the chunk, we still * need to drop the referencing foreign keys since such keys are possibly * intended to reference the hypertable, not the chunk. */ if (detach) ts_chunk_drop_referencing_fk_by_chunk_id(form.id); ts_compression_chunk_size_delete(form.id); /* Delete any row in bgw_policy_chunk-stats corresponding to this chunk */ ts_bgw_policy_chunk_stats_delete_by_chunk_id(form.id); /* Delete any rows in _timescaledb_catalog.chunk_column_stats corresponding to this chunk */ ts_chunk_column_stats_delete_by_chunk_id(form.id); if (!OidIsValid(relid)) { /* * If the chunk is deleted as a result of deleting the Hypertable, and * it is cleaned up in the DROP eventtrigger hook, it might not be * possible to resolve the relid because the relation is already gone * in pg_catalog. But that's OK, because compression settings will be * cleaned up when processing the eventtrigger. */ relid = ts_get_relation_relid(NameStr(form.schema_name), NameStr(form.table_name), true); } /* * Cleanup dependent catalogs. */ if (OidIsValid(relid)) { ts_chunk_rewrite_delete(relid, false); } if (form.compressed_chunk_id != INVALID_CHUNK_ID) { Chunk *compressed_chunk = ts_chunk_get_by_id(form.compressed_chunk_id, false); if (OidIsValid(relid)) ts_compression_settings_delete(relid); /* The chunk may have been deleted by a CASCADE */ if (compressed_chunk != NULL) { /* Plain drop without preserving catalog row because this is the compressed * chunk */ ts_chunk_drop(compressed_chunk, behavior, DEBUG1); } } else if (OidIsValid(relid)) { /* If there is no compressed chunk ID, this might be the actual * compressed chunk */ ts_compression_settings_delete_by_compress_relid(relid); } ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); ts_catalog_restore_user(&sec_ctx); } static void init_scan_by_qualified_table_name(ScanIterator *iterator, const char *schema_name, const char *table_name) { iterator->ctx.index = catalog_get_index(ts_catalog_get(), CHUNK, CHUNK_SCHEMA_NAME_INDEX); ts_scan_iterator_scan_key_init(iterator, Anum_chunk_schema_name_idx_schema_name, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(schema_name)); ts_scan_iterator_scan_key_init(iterator, Anum_chunk_schema_name_idx_table_name, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(table_name)); } static int chunk_delete(ScanIterator *iterator, Oid relid, DropBehavior behavior, bool detach) { int count = 0; ts_scanner_foreach(iterator) { chunk_tuple_delete(ts_scan_iterator_tuple_info(iterator), relid, behavior, detach); count++; } return count; } static int ts_chunk_delete_by_name_internal(const char *schema, const char *table, Oid relid, DropBehavior behavior) { ScanIterator iterator = ts_scan_iterator_create(CHUNK, RowExclusiveLock, CurrentMemoryContext); int count; init_scan_by_qualified_table_name(&iterator, schema, table); count = chunk_delete(&iterator, relid, behavior, false); /* (schema,table) names and (hypertable_id) are unique so should only have * dropped one chunk or none (if not found) */ Assert(count == 1 || count == 0); return count; } int ts_chunk_delete_by_name(const char *schema, const char *table, DropBehavior behavior) { Oid relid = ts_get_relation_relid(schema, table, false); return ts_chunk_delete_by_name_internal(schema, table, relid, behavior); } int ts_chunk_delete_by_relid_and_relname(Oid relid, const char *schemaname, const char *tablename, DropBehavior behavior) { if (!OidIsValid(relid)) return 0; return ts_chunk_delete_by_name_internal(schemaname, tablename, relid, behavior); } static void init_scan_by_hypertable_id(ScanIterator *iterator, int32 hypertable_id) { iterator->ctx.index = catalog_get_index(ts_catalog_get(), CHUNK, CHUNK_HYPERTABLE_ID_INDEX); ts_scan_iterator_scan_key_init(iterator, Anum_chunk_hypertable_id_idx_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); } int ts_chunk_delete_by_hypertable_id(int32 hypertable_id) { ScanIterator iterator = ts_scan_iterator_create(CHUNK, RowExclusiveLock, CurrentMemoryContext); init_scan_by_hypertable_id(&iterator, hypertable_id); return chunk_delete(&iterator, InvalidOid, DROP_RESTRICT, false); } bool ts_chunk_exists_with_compression(int32 hypertable_id) { ScanIterator iterator = ts_scan_iterator_create(CHUNK, AccessShareLock, CurrentMemoryContext); bool found = false; init_scan_by_hypertable_id(&iterator, hypertable_id); ts_scanner_foreach(&iterator) { bool isnull_chunk_id = slot_attisnull(ts_scan_iterator_slot(&iterator), Anum_chunk_compressed_chunk_id); if (!isnull_chunk_id) { found = true; break; } } ts_scan_iterator_close(&iterator); return found; } static void init_scan_by_compressed_chunk_id(ScanIterator *iterator, int32 compressed_chunk_id) { iterator->ctx.index = catalog_get_index(ts_catalog_get(), CHUNK, CHUNK_COMPRESSED_CHUNK_ID_INDEX); ts_scan_iterator_scan_key_init(iterator, Anum_chunk_compressed_chunk_id_idx_compressed_chunk_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(compressed_chunk_id)); } Chunk * ts_chunk_get_compressed_chunk_parent(const Chunk *chunk) { ScanIterator iterator = ts_scan_iterator_create(CHUNK, AccessShareLock, CurrentMemoryContext); Oid parent_id = InvalidOid; init_scan_by_compressed_chunk_id(&iterator, chunk->fd.id); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); Datum datum; bool isnull; Assert(!OidIsValid(parent_id)); datum = slot_getattr(ti->slot, Anum_chunk_id, &isnull); if (!isnull) parent_id = DatumGetObjectId(datum); } if (OidIsValid(parent_id)) return ts_chunk_get_by_id(parent_id, true); return NULL; } bool ts_chunk_contains_compressed_data(const Chunk *chunk) { Chunk *parent_chunk = ts_chunk_get_compressed_chunk_parent(chunk); return parent_chunk != NULL; } List * ts_chunk_get_chunk_ids_by_hypertable_id(int32 hypertable_id) { List *chunkids = NIL; ScanIterator iterator = ts_scan_iterator_create(CHUNK, AccessShareLock, CurrentMemoryContext); init_scan_by_hypertable_id(&iterator, hypertable_id); ts_scanner_foreach(&iterator) { bool isnull; Datum id = slot_getattr(ts_scan_iterator_slot(&iterator), Anum_chunk_id, &isnull); if (!isnull) chunkids = lappend_int(chunkids, DatumGetInt32(id)); } return chunkids; } /* Return list of chunks that belong to the given hypertable. * * The returned chunk objects will not have any constraints or dimension * information filled in. */ List * ts_chunk_get_by_hypertable_id(int32 hypertable_id) { List *chunks = NIL; Oid hypertable_relid = ts_hypertable_id_to_relid(hypertable_id, false); ScanIterator iterator = ts_scan_iterator_create(CHUNK, RowExclusiveLock, CurrentMemoryContext); init_scan_by_hypertable_id(&iterator, hypertable_id); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); Chunk *chunk = palloc0(sizeof(Chunk)); ts_chunk_formdata_fill(&chunk->fd, ti); chunk->hypertable_relid = hypertable_relid; chunk->table_id = ts_get_relation_relid(NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name), false); chunks = lappend(chunks, chunk); } return chunks; } static ChunkResult chunk_recreate_constraint(ChunkScanCtx *ctx, ChunkStub *stub) { ChunkStubScanCtx stubctx = { .stub = stub, }; Chunk *chunk = chunk_create_from_stub(&stubctx); ts_chunk_constraints_recreate(ctx->ht, chunk); return CHUNK_PROCESSED; } void ts_chunk_recreate_all_constraints_for_dimension(Hypertable *ht, int32 dimension_id) { DimensionVec *slices; ChunkScanCtx chunkctx; int i; slices = ts_dimension_slice_scan_by_dimension(dimension_id, 0); if (NULL == slices) return; chunk_scan_ctx_init(&chunkctx, ht, NULL); for (i = 0; i < slices->num_slices; i++) ts_chunk_constraint_scan_by_dimension_slice(slices->slices[i], &chunkctx, CurrentMemoryContext); chunk_scan_ctx_foreach_chunk_stub(&chunkctx, chunk_recreate_constraint, 0); chunk_scan_ctx_destroy(&chunkctx); } /* * Chunk catalog updates are done in three steps. * This is achieved by following this sequence: * 1: call lock_chunk_tuple: this finds most recent version of tuple, * locks it, fills TID and data * 2: make changes to the data * 3: call chunk_update_catalog_tuple with the TID and updated data * * This is equivalent to SELECT for UPDATE, followed by UPDATE * * All callers who want to update chunk tuples should respect this so that locks * are acquired correctly. * */ static void chunk_update_catalog_tuple(ItemPointer tid, FormData_chunk *update) { HeapTuple new_tuple; CatalogSecurityContext sec_ctx; Catalog *catalog = ts_catalog_get(); Oid table = catalog_get_table_id(catalog, CHUNK); Relation chunk_rel = relation_open(table, RowExclusiveLock); new_tuple = chunk_formdata_make_tuple(update, chunk_rel->rd_att); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_update_tid(chunk_rel, tid, new_tuple); ts_catalog_restore_user(&sec_ctx); heap_freetuple(new_tuple); relation_close(chunk_rel, NoLock); } /* * This function locks the timescaledb_catalog.chunk tuple (corresponding to chunk_id ) in * LockTupleExclusiveMode. It blocks till the lock is acquired. The tid and data (corresponding to * the locked tuple) are returned via tid and form arguments. Anyone updating/deleting a chunk entry * from the catalog table is expected to first call this function. Refer to * chunk_update_catalog_tuple() for more details. */ static bool lock_chunk_tuple(int32 chunk_id, ItemPointer tid, FormData_chunk *form) { ScanTupLock scantuplock = { .waitpolicy = LockWaitBlock, .lockmode = LockTupleExclusive, }; ScanIterator iterator = ts_scan_iterator_create(CHUNK, RowShareLock, CurrentMemoryContext); iterator.ctx.index = catalog_get_index(ts_catalog_get(), CHUNK, CHUNK_ID_INDEX); iterator.ctx.tuplock = &scantuplock; /* Keeping the lock since we presumably want to update the tuple */ iterator.ctx.flags = SCANNER_F_KEEPLOCK; /* see table_tuple_lock for details about flags that are set in TupleExclusive mode */ scantuplock.lockflags = TUPLE_LOCK_FLAG_LOCK_UPDATE_IN_PROGRESS; if (!IsolationUsesXactSnapshot()) { /* in read committed mode, we follow all updates to this tuple */ scantuplock.lockflags |= TUPLE_LOCK_FLAG_FIND_LAST_VERSION; } ts_scan_iterator_scan_key_init(&iterator, Anum_chunk_idx_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(chunk_id)); bool success = false; ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); if (ti->lockresult != TM_Ok) { if (IsolationUsesXactSnapshot()) { /* For Repeatable Read and Serializable isolation level report error * if we cannot lock the tuple */ ereport(ERROR, (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("could not serialize access due to concurrent update"))); } else { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("unable to lock chunk catalog tuple, lock result is %d for chunk " "ID (%d)", ti->lockresult, chunk_id))); } } ts_chunk_formdata_fill(form, ti); ItemPointer result_tid = ts_scanner_get_tuple_tid(ti); tid->ip_blkid = result_tid->ip_blkid; tid->ip_posid = result_tid->ip_posid; success = true; break; } ts_scan_iterator_close(&iterator); return success; } bool ts_chunk_set_name(Chunk *chunk, const char *newname) { FormData_chunk form; ItemPointerData tid; bool PG_USED_FOR_ASSERTS_ONLY found; found = lock_chunk_tuple(chunk->fd.id, &tid, &form); Assert(found); namestrcpy(&form.table_name, newname); chunk_update_catalog_tuple(&tid, &form); return true; } bool ts_chunk_set_schema(Chunk *chunk, const char *newschema) { FormData_chunk form; ItemPointerData tid; bool PG_USED_FOR_ASSERTS_ONLY found; found = lock_chunk_tuple(chunk->fd.id, &tid, &form); Assert(found); namestrcpy(&form.schema_name, newschema); chunk_update_catalog_tuple(&tid, &form); return true; } bool ts_chunk_set_unordered(Chunk *chunk) { Assert(ts_chunk_is_compressed(chunk)); return ts_chunk_add_status(chunk, CHUNK_STATUS_COMPRESSED_UNORDERED); } bool ts_chunk_set_partial(Chunk *chunk) { bool set_status; Assert(ts_chunk_is_compressed(chunk)); set_status = ts_chunk_add_status(chunk, CHUNK_STATUS_COMPRESSED_PARTIAL); if (set_status) { /* * If the status was set then convert the corresponding * _timescaledb_catalog.chunk_column_stats entries "INVALID". */ ts_chunk_column_stats_set_invalid(chunk->fd.hypertable_id, chunk->fd.id); /* changed chunk status, so invalidate plans involving this chunk */ CacheInvalidateRelcacheByRelid(chunk->table_id); } return set_status; } /* No inserts, updates, and deletes are permitted on a frozen chunk. * Compression policies etc do not run on a frozen chunk. * Only valid operation is dropping the chunk */ bool ts_chunk_set_frozen(Chunk *chunk) { return ts_chunk_add_status(chunk, CHUNK_STATUS_FROZEN); } bool ts_chunk_unset_frozen(Chunk *chunk) { return ts_chunk_clear_status(chunk, CHUNK_STATUS_FROZEN); } bool ts_chunk_is_frozen(const Chunk *chunk) { return ts_flags_are_set_32(chunk->fd.status, CHUNK_STATUS_FROZEN); } /* only caller used to be ts_chunk_unset_frozen. This code was in PG14 block as we run into * a "defined but unset" error in CI/CD builds for PG < 14. But now called from recompress as well */ bool ts_chunk_clear_status(Chunk *chunk, int32 status) { /* only frozen status can be cleared for a frozen chunk */ if (status != CHUNK_STATUS_FROZEN && ts_flags_are_set_32(chunk->fd.status, CHUNK_STATUS_FROZEN)) { /* chunk in frozen state cannot be modified */ ereport(ERROR, (errcode(ERRCODE_TS_OPERATION_NOT_SUPPORTED), errmsg("cannot modify frozen chunk status"), errdetail("chunk id = %d attempt to clear status %d , current status %x ", chunk->fd.id, status, chunk->fd.status))); } FormData_chunk form; ItemPointerData tid; bool PG_USED_FOR_ASSERTS_ONLY found; found = lock_chunk_tuple(chunk->fd.id, &tid, &form); Assert(found); /* applying the flags after locking the metadata tuple */ int32 old_status = form.status; form.status = ts_clear_flags_32(form.status, status); chunk->fd.status = form.status; /* Row-level locks are released at transaction end or during savepoint rollback */ if (old_status != form.status) chunk_update_catalog_tuple(&tid, &form); return true; } static bool ts_chunk_add_status(Chunk *chunk, int32 status) { bool status_set = false; if (ts_flags_are_set_32(chunk->fd.status, CHUNK_STATUS_FROZEN)) { /* chunk in frozen state cannot be modified */ ereport(ERROR, (errcode(ERRCODE_TS_OPERATION_NOT_SUPPORTED), errmsg("cannot modify frozen chunk status"), errdetail("chunk id = %d attempt to set status %d , current status %x ", chunk->fd.id, status, chunk->fd.status))); } FormData_chunk form; ItemPointerData tid; bool PG_USED_FOR_ASSERTS_ONLY found; found = lock_chunk_tuple(chunk->fd.id, &tid, &form); Assert(found); /* Somebody could update the status before we are able to lock it so check again */ if (ts_flags_are_set_32(form.status, CHUNK_STATUS_FROZEN)) { /* chunk in frozen state cannot be modified */ ereport(ERROR, (errcode(ERRCODE_TS_OPERATION_NOT_SUPPORTED), errmsg("cannot modify frozen chunk status"), errdetail("chunk id = %d attempt to set status %d , current status %d ", chunk->fd.id, status, form.status))); } /* applying the flags after locking the metadata tuple */ int32 old_status = form.status; form.status = ts_set_flags_32(form.status, status); chunk->fd.status = form.status; /* Row-level locks are released at transaction end or during savepoint rollback */ if (old_status != form.status) { chunk_update_catalog_tuple(&tid, &form); status_set = true; } return status_set; } /*Assume permissions are already checked */ bool ts_chunk_set_compressed_chunk(Chunk *chunk, int32 compressed_chunk_id) { uint32 flags = CHUNK_STATUS_COMPRESSED; uint32 mstatus = ts_set_flags_32(chunk->fd.status, flags); if (ts_flags_are_set_32(chunk->fd.status, CHUNK_STATUS_FROZEN)) { /* chunk in frozen state cannot be modified */ ereport(ERROR, (errcode(ERRCODE_TS_OPERATION_NOT_SUPPORTED), errmsg("cannot modify frozen chunk status"), errdetail("chunk id = %d attempt to set status %d , current status %d ", chunk->fd.id, mstatus, chunk->fd.status))); } FormData_chunk form; ItemPointerData tid; bool PG_USED_FOR_ASSERTS_ONLY found; found = lock_chunk_tuple(chunk->fd.id, &tid, &form); Assert(found); /* Somebody could update the status before we are able to lock it so check again */ if (ts_flags_are_set_32(form.status, CHUNK_STATUS_FROZEN)) { /* chunk in frozen state cannot be modified */ ereport(ERROR, (errcode(ERRCODE_TS_OPERATION_NOT_SUPPORTED), errmsg("cannot modify frozen chunk status"), errdetail("chunk id = %d attempt to set status %d , current status %d ", chunk->fd.id, mstatus, form.status))); } /* re-applying the flags after locking the metadata tuple */ form.status = ts_set_flags_32(form.status, flags); form.compressed_chunk_id = compressed_chunk_id; chunk->fd.compressed_chunk_id = form.compressed_chunk_id; chunk->fd.status = form.status; chunk_update_catalog_tuple(&tid, &form); return true; } /*Assume permissions are already checked */ bool ts_chunk_clear_compressed_chunk(Chunk *chunk) { uint32 flags = CHUNK_STATUS_COMPRESSED | CHUNK_STATUS_COMPRESSED_UNORDERED | CHUNK_STATUS_COMPRESSED_PARTIAL; uint32 mstatus = ts_clear_flags_32(chunk->fd.status, flags); if (ts_flags_are_set_32(chunk->fd.status, CHUNK_STATUS_FROZEN)) { /* chunk in frozen state cannot be modified */ ereport(ERROR, (errcode(ERRCODE_TS_OPERATION_NOT_SUPPORTED), errmsg("cannot modify frozen chunk status"), errdetail("chunk id = %d attempt to set status %d , current status %d ", chunk->fd.id, mstatus, chunk->fd.status))); } FormData_chunk form; ItemPointerData tid; bool PG_USED_FOR_ASSERTS_ONLY found; found = lock_chunk_tuple(chunk->fd.id, &tid, &form); Assert(found); /* Somebody could update the status before we are able to lock it so check again */ if (ts_flags_are_set_32(form.status, CHUNK_STATUS_FROZEN)) { /* chunk in frozen state cannot be modified */ ereport(ERROR, (errcode(ERRCODE_TS_OPERATION_NOT_SUPPORTED), errmsg("cannot modify frozen chunk status"), errdetail("chunk id = %d attempt to set status %d , current status %d ", chunk->fd.id, mstatus, form.status))); } /* re-applying the flags after locking the metadata tuple */ form.status = ts_clear_flags_32(form.status, flags); form.compressed_chunk_id = INVALID_CHUNK_ID; chunk->fd.compressed_chunk_id = form.compressed_chunk_id; chunk->fd.status = form.status; chunk_update_catalog_tuple(&tid, &form); return true; } /* Used as a tuple found function */ static ScanTupleResult chunk_rename_schema_name(TupleInfo *ti, void *data) { FormData_chunk form; HeapTuple new_tuple; CatalogSecurityContext sec_ctx; ts_chunk_formdata_fill(&form, ti); /* Rename schema name */ namestrcpy(&form.schema_name, (char *) data); new_tuple = chunk_formdata_make_tuple(&form, ts_scanner_get_tupledesc(ti)); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_update_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti), new_tuple); ts_catalog_restore_user(&sec_ctx); heap_freetuple(new_tuple); return SCAN_CONTINUE; } /* Go through the internal chunk table and rename all matching schemas */ void ts_chunks_rename_schema_name(char *old_schema, char *new_schema) { NameData old_schema_name; ScanKeyData scankey[1]; Catalog *catalog = ts_catalog_get(); ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, CHUNK), .index = catalog_get_index(catalog, CHUNK, CHUNK_SCHEMA_NAME_INDEX), .nkeys = 1, .scankey = scankey, .tuple_found = chunk_rename_schema_name, .data = new_schema, .lockmode = RowExclusiveLock, .scandirection = ForwardScanDirection, }; namestrcpy(&old_schema_name, old_schema); ScanKeyInit(&scankey[0], Anum_chunk_schema_name_idx_schema_name, BTEqualStrategyNumber, F_NAMEEQ, NameGetDatum(&old_schema_name)); ts_scanner_scan(&scanctx); } static int chunk_cmp(const void *ch1, const void *ch2) { const Chunk *v1 = ((const Chunk *) ch1); const Chunk *v2 = ((const Chunk *) ch2); if (v1->fd.hypertable_id < v2->fd.hypertable_id) return -1; if (v1->fd.hypertable_id > v2->fd.hypertable_id) return 1; if (v1->table_id < v2->table_id) return -1; if (v1->table_id > v2->table_id) return 1; return 0; } /* * This is a helper set returning function (SRF) that takes a set returning function context * and as argument and returns oids extracted from funcctx->user_fctx (which is Chunk* * array). Note that the caller needs to be registered as a set returning function for this * to work. */ static Datum show_chunks_return_srf(FunctionCallInfo fcinfo) { FuncCallContext *funcctx; uint64 call_cntr; TupleDesc tupdesc; Chunk *result_set; Chunk *curr_chunk; /* stuff done only on the first call of the function */ if (SRF_IS_FIRSTCALL()) { /* Build a tuple descriptor for our result type */ /* not quite necessary */ if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_SCALAR) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context " "that cannot accept type record"))); } /* stuff done on every call of the function */ funcctx = SRF_PERCALL_SETUP(); call_cntr = funcctx->call_cntr; result_set = (Chunk *) funcctx->user_fctx; /* * skip if it's an OSM chunk. Ideally this check could be done deep down in * functions like "chunk_scan_context_add_chunk", "chunk_tuple_dropped_filter" * etc. but they are used by other APIs like drop_chunks, chunk_scan_find, etc * which need access to the OSM chunk. Trying to unify scan functions across * all such usages seems to be too much of an overhaul as compared to this. * * Check the index appropriately first. */ if (call_cntr < funcctx->max_calls) { curr_chunk = &result_set[call_cntr]; if (IS_OSM_CHUNK(curr_chunk)) { call_cntr = ++funcctx->call_cntr; } } /* do when there is more left to send */ if (call_cntr < funcctx->max_calls) SRF_RETURN_NEXT(funcctx, result_set[call_cntr].table_id); else /* do when there is no more left */ SRF_RETURN_DONE(funcctx); } void ts_chunk_drop(const Chunk *chunk, DropBehavior behavior, int32 log_level) { ObjectAddress objaddr = { .classId = RelationRelationId, .objectId = chunk->table_id, }; if (log_level >= 0) elog(log_level, "dropping chunk %s.%s", NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name)); /* Remove the chunk from the chunk table */ ts_chunk_delete_by_relid_and_relname(chunk->table_id, NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name), behavior); /* Drop the table */ performDeletion(&objaddr, behavior, 0); } static void lock_referenced_tables(Oid table_relid) { List *fk_relids = NIL; ListCell *lf; List *cachedfkeys = NIL; Relation table_rel = table_open(table_relid, AccessShareLock); /* this list is from the relcache and can disappear with a cache flush, so * no further catalog access till we save the fk relids */ cachedfkeys = RelationGetFKeyList(table_rel); foreach (lf, cachedfkeys) { ForeignKeyCacheInfo *cachedfk = lfirst_node(ForeignKeyCacheInfo, lf); /* conrelid should always be that of the table we're considering */ Assert(cachedfk->conrelid == RelationGetRelid(table_rel)); fk_relids = lappend_oid(fk_relids, cachedfk->confrelid); } table_close(table_rel, AccessShareLock); foreach (lf, fk_relids) LockRelationOid(lfirst_oid(lf), AccessExclusiveLock); } List * ts_chunk_do_drop_chunks(Hypertable *ht, int64 older_than, int64 newer_than, int32 log_level, Oid time_type, Oid arg_type, bool older_newer) { uint64 num_chunks = 0; Chunk *chunks; const char *schema_name, *table_name; const int32 hypertable_id = ht->fd.id; bool has_continuous_aggs, is_materialization_hypertable; const MemoryContext oldcontext = CurrentMemoryContext; ScanTupLock tuplock = { .waitpolicy = LockWaitBlock, .lockmode = LockTupleExclusive, }; ts_hypertable_permissions_check(ht->main_table_relid, GetUserId()); /* We have a FK between hypertable H and PAR. Hypertable H has number of * chunks C1, C2, etc. When we execute "drop table C", PG acquires locks * on C and PAR. If we have a query as "select * from hypertable", this * acquires a lock on C and PAR as well. But the order of the locks is not * the same and results in deadlocks. - github issue #865 We hope to * alleviate the problem by acquiring a lock on PAR before executing the * drop table stmt. This is not fool-proof as we could have multiple * fkrelids and the order of lock acquisition for these could differ as * well. Do not unlock - let the transaction semantics take care of it. */ lock_referenced_tables(ht->main_table_relid); is_materialization_hypertable = false; switch (ts_continuous_agg_hypertable_status(hypertable_id)) { case HypertableIsMaterialization: has_continuous_aggs = false; is_materialization_hypertable = true; break; case HypertableIsMaterializationAndRaw: has_continuous_aggs = true; is_materialization_hypertable = true; break; case HypertableIsRawTable: has_continuous_aggs = true; break; default: has_continuous_aggs = false; break; } PG_TRY(); { /* * For INTEGER type dimensions, we support querying using intervals or any * timestamp or date input. For such INTEGER dimensions, we get the chunks * using their creation time values. */ if (IS_INTEGER_TYPE(time_type) && (arg_type == INTERVALOID || IS_TIMESTAMP_TYPE(arg_type))) { chunks = get_chunks_in_creation_time_range(ht, older_than, newer_than, CurrentMemoryContext, &num_chunks, &tuplock); } else { if (!older_newer) chunks = get_chunks_in_creation_time_range(ht, older_than, newer_than, CurrentMemoryContext, &num_chunks, &tuplock); else chunks = get_chunks_in_time_range(ht, older_than, newer_than, CurrentMemoryContext, &num_chunks, &tuplock); } } PG_CATCH(); { ErrorData *edata; MemoryContextSwitchTo(oldcontext); edata = CopyErrorData(); FlushErrorState(); if (edata->sqlerrcode == ERRCODE_LOCK_NOT_AVAILABLE) { edata->detail = edata->message; edata->message = psprintf("some chunks could not be read since they are being concurrently updated"); } ReThrowError(edata); } PG_END_TRY(); DEBUG_WAITPOINT("drop_chunks_chunks_found"); int32 osm_chunk_id = ts_chunk_get_osm_chunk_id(ht->fd.id); if (has_continuous_aggs) { /* Exclusively lock all chunks, and invalidate the continuous * aggregates in the regions covered by the chunks. We do this in two * steps: first lock all the chunks and then invalidate the * regions. Since we are going to drop the chunks, there is no point * in allowing inserts into them. * * Locking prevents further modification of the dropped region during * this transaction, which allows moving the invalidation threshold * without having to worry about new invalidations while * refreshing. */ for (uint64 i = 0; i < num_chunks; i++) { LockRelationOid(chunks[i].table_id, ExclusiveLock); Assert(hyperspace_get_open_dimension(ht->space, 0)->fd.id == chunks[i].cube->slices[0]->fd.dimension_id); } DEBUG_WAITPOINT("drop_chunks_locked"); /* Invalidate the dropped region to indicate that it was modified. * * The invalidation will allow the refresh command on a continuous * aggregate to see that this region was dropped and and will * therefore be able to refresh accordingly.*/ for (uint64 i = 0; i < num_chunks; i++) { if (osm_chunk_id == chunks[i].fd.id) { // we do not rebuild continuous aggs if tiered data is dropped */ continue; } int64 start = ts_chunk_primary_dimension_start(&chunks[i]); int64 end = ts_chunk_primary_dimension_end(&chunks[i]); ts_cm_functions->continuous_agg_invalidate_raw_ht(ht, start, end); } } List *dropped_chunk_names = NIL; for (uint64 i = 0; i < num_chunks; i++) { char *chunk_name; ASSERT_IS_VALID_CHUNK(&chunks[i]); /* frozen chunks are skipped. Not dropped. */ if (!ts_chunk_validate_chunk_status_for_operation(&chunks[i], CHUNK_DROP, false /*throw_error */) || osm_chunk_id == chunks[i].fd.id) { continue; } /* store chunk name for output */ schema_name = quote_identifier(NameStr(chunks[i].fd.schema_name)); table_name = quote_identifier(NameStr(chunks[i].fd.table_name)); chunk_name = psprintf("%s.%s", schema_name, table_name); dropped_chunk_names = lappend(dropped_chunk_names, chunk_name); ts_chunk_drop(chunks + i, DROP_RESTRICT, log_level); } // if we have tiered chunks cascade drop to tiering layer as well if (osm_chunk_id != INVALID_CHUNK_ID) { Chunk *osm_chunk = ts_chunk_get_by_id(osm_chunk_id, true); hypertable_drop_chunks_hook_type osm_drop_chunks_hook = ts_get_osm_hypertable_drop_chunks_hook(); /* * The OSM library may not be loaded at the moment if * `ts_chunk_do_drop_chunks` is called from the a background worker * (e.g. from a retention policy). We call `GetFdwRoutineByRelId` to * ensure the library is loaded. */ if (!osm_drop_chunks_hook && GetFdwRoutineByRelId(osm_chunk->table_id)) osm_drop_chunks_hook = ts_get_osm_hypertable_drop_chunks_hook(); if (osm_drop_chunks_hook) { ListCell *lc; Dimension *dim = &ht->space->dimensions[0]; /* convert to PG timestamp from timescaledb internal format */ int64 range_start = ts_internal_to_time_int64(newer_than, dim->fd.column_type); int64 range_end = ts_internal_to_time_int64(older_than, dim->fd.column_type); List *osm_dropped_names = osm_drop_chunks_hook(osm_chunk->table_id, NameStr(ht->fd.schema_name), NameStr(ht->fd.table_name), range_start, range_end); foreach (lc, osm_dropped_names) { dropped_chunk_names = lappend(dropped_chunk_names, lfirst(lc)); } } } /* When dropping chunks for a given CAgg then force set the watermark */ if (is_materialization_hypertable) { bool isnull; int64 watermark = ts_hypertable_get_open_dim_max_value(ht, 0, &isnull); ts_cagg_watermark_update(ht, watermark, isnull, true); } DEBUG_WAITPOINT("drop_chunks_end"); return dropped_chunk_names; } /* * This is a helper set returning function (SRF) that takes a set returning function context * and as argument and returns cstrings extracted from funcctx->user_fctx (which is a List). * Note that the caller needs to be registered as a set returning function for this to work. */ static Datum list_return_srf(FunctionCallInfo fcinfo) { FuncCallContext *funcctx; uint64 call_cntr; TupleDesc tupdesc; List *result_set; Datum retval; /* stuff done only on the first call of the function */ if (SRF_IS_FIRSTCALL()) { /* Build a tuple descriptor for our result type */ /* not quite necessary */ if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_SCALAR) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context " "that cannot accept type record"))); } /* stuff done on every call of the function */ funcctx = SRF_PERCALL_SETUP(); call_cntr = funcctx->call_cntr; result_set = castNode(List, funcctx->user_fctx); /* do when there is more left to send */ if (call_cntr < funcctx->max_calls) { /* store return value and increment linked list */ retval = CStringGetTextDatum(linitial(result_set)); funcctx->user_fctx = list_delete_first(result_set); SRF_RETURN_NEXT(funcctx, retval); } else /* do when there is no more left */ SRF_RETURN_DONE(funcctx); } Datum ts_chunk_drop_single_chunk(PG_FUNCTION_ARGS) { Oid chunk_relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); char *chunk_table_name = get_rel_name(chunk_relid); char *chunk_schema_name = get_namespace_name(get_rel_namespace(chunk_relid)); ScanTupLock tuplock = { .lockmode = LockTupleKeyShare, .waitpolicy = LockWaitBlock, .lockflags = TUPLE_LOCK_FLAG_FIND_LAST_VERSION, }; const Chunk *ch = ts_chunk_get_by_name_with_memory_context(chunk_schema_name, chunk_table_name, NoLock, &tuplock, CurrentMemoryContext, true); Assert(ch != NULL); ts_chunk_validate_chunk_status_for_operation(ch, CHUNK_DROP, true /*throw_error */); if (ts_chunk_contains_compressed_data(ch)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("dropping compressed chunks not supported"), errhint("Please drop the corresponding chunk on the uncompressed hypertable " "instead."))); /* do not drop any chunk dependencies */ ts_chunk_drop(ch, DROP_RESTRICT, LOG); PG_RETURN_BOOL(true); } Datum ts_chunk_drop_chunks(PG_FUNCTION_ARGS) { MemoryContext oldcontext; FuncCallContext *funcctx; Hypertable *ht; List *dc_temp = NIL; List *dc_names = NIL; Oid relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); /* * Marked volatile to suppress the -Wclobbered warning. The warning is * actually incorrect because these values are not used after longjmp. */ volatile int64 older_than = PG_INT64_MAX; volatile int64 newer_than = PG_INT64_MIN; volatile int64 created_before = PG_INT64_MAX; volatile int64 created_after = PG_INT64_MIN; volatile bool older_newer = false; volatile bool before_after = false; bool verbose; int elevel; Cache *hcache; const Dimension *time_dim; Oid time_type; Oid arg_type; TS_PREVENT_FUNC_IF_READ_ONLY(); arg_type = InvalidOid; /* * When past the first call of the SRF, dropping has already been completed, * so we just return the next chunk in the list of dropped chunks. */ if (!SRF_IS_FIRSTCALL()) return list_return_srf(fcinfo); if (PG_ARGISNULL(0)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid hypertable or continuous aggregate"), errhint("Specify a hypertable or continuous aggregate."))); /* Find either the hypertable or view, or error out if the relid is * neither. * * We should improve the printout since it can either be a proper relid * that does not refer to a hypertable or a continuous aggregate, or a * relid that does not refer to anything at all. */ hcache = ts_hypertable_cache_pin(); ht = ts_resolve_hypertable_from_table_or_cagg(hcache, relid, false); Assert(ht != NULL); time_dim = hyperspace_get_open_dimension(ht->space, 0); if (!time_dim) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("hypertable has no open partitioning dimension"))); time_type = ts_dimension_get_partition_type(time_dim); /* * Treat UUID (v7) as a timestamptz type. The expected input is an interval or absolute * timestamptz. */ if (IS_UUID_TYPE(time_type)) time_type = TIMESTAMPTZOID; /* note that arg_types will be the same for all specified "ANY" elements for a given call */ if (!PG_ARGISNULL(1)) { arg_type = get_fn_expr_argtype(fcinfo->flinfo, 1); older_than = ts_time_value_from_arg(PG_GETARG_DATUM(1), arg_type, time_type, true); older_newer = true; } if (!PG_ARGISNULL(2)) { arg_type = get_fn_expr_argtype(fcinfo->flinfo, 2); newer_than = ts_time_value_from_arg(PG_GETARG_DATUM(2), arg_type, time_type, true); older_newer = true; } /* * We cannot have a mix of [older_than/newer_than] and [created_before/created_after]. * So, check that first. Note that created_before/created_after have a type of * TIMESTAMPTZOID regardless of the partitioning type. */ if (!PG_ARGISNULL(4)) { if (older_newer) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot specify \"older_than\" or \"newer_than\" together with " "\"created_before\"" "or \"created_after\""), errhint("\"older_than\" and/or \"newer_than\" is recommended with " "\"time\"-like partitioning" " and \"created_before\" and/or \"created_after\" is recommended " "with \"integer\"-like" " partitioning."))); arg_type = get_fn_expr_argtype(fcinfo->flinfo, 4); /* We use the existing function for various type/conversion checks */ created_before = ts_time_value_from_arg(PG_GETARG_DATUM(4), arg_type, TIMESTAMPTZOID, false); /* convert into int64 format for comparisons */ created_before = ts_internal_to_time_int64(created_before, TIMESTAMPTZOID); before_after = true; older_than = created_before; } if (!PG_ARGISNULL(5)) { if (older_newer) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot specify \"older_than\" or \"newer_than\" together with " "\"created_before\"" " or \"created_after\""), errhint("\"older_than\" and/or \"newer_than\" is recommended with " "\"time\"-like partitioning" " and \"created_before\" and/or \"created_after\" is recommended " "with \"integer\"-like" " partitioning."))); arg_type = get_fn_expr_argtype(fcinfo->flinfo, 5); /* We use the existing function for various type/conversion checks */ created_after = ts_time_value_from_arg(PG_GETARG_DATUM(5), arg_type, TIMESTAMPTZOID, false); /* convert into int64 format for comparisons */ created_after = ts_internal_to_time_int64(created_after, TIMESTAMPTZOID); before_after = true; newer_than = created_after; } /* if both have not been specified then error out */ if (!older_newer && !before_after) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid time range for dropping chunks"), errhint("At least one of older_than/newer_than or created_before/created_after" " must be provided."))); /* * For INTEGER type dimensions, we support querying using intervals or any * timestamp or date input. For such INTEGER dimensions, we get the chunks * using their creation time values. */ if (IS_INTEGER_TYPE(time_type) && (arg_type == INTERVALOID || IS_TIMESTAMP_TYPE(arg_type))) { /* check that we use proper inputs for such cases */ if (older_newer) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot specify \"older_than\" and/or \"newer_than\" for " "\"integer\"-like partitioning types"), errhint("Use \"created_before\" and/or \"created_after\" which rely on the " "chunk creation time values."))); } verbose = PG_ARGISNULL(3) ? false : PG_GETARG_BOOL(3); elevel = verbose ? INFO : DEBUG2; /* Initial multi function call setup */ funcctx = SRF_FIRSTCALL_INIT(); /* Drop chunks and store their names for return */ oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); PG_TRY(); { dc_temp = ts_chunk_do_drop_chunks(ht, older_than, newer_than, elevel, time_type, arg_type, older_newer); } PG_CATCH(); { /* An error is raised if there are dependent objects, but the original * message is not very helpful in suggesting that you should use * CASCADE (we don't support it), so we replace the hint with a more * accurate hint for our situation. */ ErrorData *edata; MemoryContextSwitchTo(oldcontext); edata = CopyErrorData(); FlushErrorState(); if (edata->sqlerrcode == ERRCODE_DEPENDENT_OBJECTS_STILL_EXIST) edata->hint = pstrdup("Use DROP ... to drop the dependent objects."); ts_cache_release(&hcache); ReThrowError(edata); } PG_END_TRY(); ts_cache_release(&hcache); dc_names = list_concat(dc_names, dc_temp); MemoryContextSwitchTo(oldcontext); /* store data for multi function call */ funcctx->max_calls = list_length(dc_names); funcctx->user_fctx = dc_names; return list_return_srf(fcinfo); } /* Return the compression status for the chunk */ ChunkCompressionStatus ts_chunk_get_compression_status(int32 chunk_id) { ChunkCompressionStatus st = CHUNK_COMPRESS_NONE; ScanIterator iterator = ts_scan_iterator_create(CHUNK, AccessShareLock, CurrentMemoryContext); iterator.ctx.index = catalog_get_index(ts_catalog_get(), CHUNK, CHUNK_ID_INDEX); ts_scan_iterator_scan_key_init(&iterator, Anum_chunk_idx_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(chunk_id)); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); bool status_isnull; Datum status; status = slot_getattr(ti->slot, Anum_chunk_status, &status_isnull); Assert(!status_isnull); bool status_is_compressed = ts_flags_are_set_32(DatumGetInt32(status), CHUNK_STATUS_COMPRESSED); bool status_is_unordered = ts_flags_are_set_32(DatumGetInt32(status), CHUNK_STATUS_COMPRESSED_UNORDERED); bool status_is_partial = ts_flags_are_set_32(DatumGetInt32(status), CHUNK_STATUS_COMPRESSED_PARTIAL); if (status_is_compressed) { if (status_is_unordered || status_is_partial) st = CHUNK_COMPRESS_UNORDERED; else st = CHUNK_COMPRESS_ORDERED; } else { Assert(!status_is_unordered); st = CHUNK_COMPRESS_NONE; } } ts_scan_iterator_close(&iterator); return st; } /* Note that only a compressed chunk can have unordered flag set */ bool ts_chunk_is_unordered(const Chunk *chunk) { return ts_flags_are_set_32(chunk->fd.status, CHUNK_STATUS_COMPRESSED_UNORDERED); } bool ts_chunk_is_compressed(const Chunk *chunk) { return ts_flags_are_set_32(chunk->fd.status, CHUNK_STATUS_COMPRESSED); } bool ts_chunk_needs_compression(const Chunk *chunk) { return !ts_chunk_is_compressed(chunk); } bool ts_chunk_needs_recompression(const Chunk *chunk) { Assert(ts_chunk_is_compressed(chunk)); return ts_chunk_is_partial(chunk) || ts_chunk_is_unordered(chunk); } /* Note that only a compressed chunk can have partial flag set */ bool ts_chunk_is_partial(const Chunk *chunk) { return ts_flags_are_set_32(chunk->fd.status, CHUNK_STATUS_COMPRESSED_PARTIAL); } static const char * get_chunk_operation_str(ChunkOperation cmd) { switch (cmd) { case CHUNK_INSERT: return "Insert"; case CHUNK_DELETE: return "Delete"; case CHUNK_UPDATE: return "Update"; case CHUNK_COMPRESS: return "compress_chunk"; case CHUNK_DECOMPRESS: return "decompress_chunk"; case CHUNK_DROP: return "drop_chunk"; default: return "Unsupported"; } } bool ts_chunk_validate_chunk_status_for_operation(const Chunk *chunk, ChunkOperation cmd, bool throw_error) { Oid chunk_relid = chunk->table_id; int32 chunk_status = chunk->fd.status; /* * Block everything but DELETE on OSM chunks. */ if (chunk->fd.osm_chunk) { switch (cmd) { case CHUNK_DROP: return true; break; default: if (throw_error) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("%s not permitted on tiered chunk \"%s\" ", get_chunk_operation_str(cmd), get_rel_name(chunk_relid)))); return false; break; } } /* Handle frozen chunks */ if (ts_flags_are_set_32(chunk_status, CHUNK_STATUS_FROZEN)) { /* Data modification is not permitted on a frozen chunk */ switch (cmd) { case CHUNK_INSERT: case CHUNK_DELETE: case CHUNK_UPDATE: case CHUNK_COMPRESS: case CHUNK_DECOMPRESS: case CHUNK_DROP: { if (throw_error) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("%s not permitted on frozen chunk \"%s\" ", get_chunk_operation_str(cmd), get_rel_name(chunk_relid)))); return false; break; } default: break; /*supported operations */ } } /* Handle unfrozen chunks */ else { switch (cmd) { /* supported operations */ case CHUNK_INSERT: case CHUNK_DELETE: case CHUNK_UPDATE: break; /* Only uncompressed chunks can be compressed */ case CHUNK_COMPRESS: { if (ts_flags_are_set_32(chunk_status, CHUNK_STATUS_COMPRESSED)) ereport((throw_error ? ERROR : NOTICE), (errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("chunk \"%s\" is already compressed", get_rel_name(chunk_relid)))); return false; } /* Only compressed chunks can be decompressed */ case CHUNK_DECOMPRESS: { if (!ts_flags_are_set_32(chunk_status, CHUNK_STATUS_COMPRESSED)) ereport((throw_error ? ERROR : NOTICE), (errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("chunk \"%s\" is already decompressed", get_rel_name(chunk_relid)))); return false; } default: break; } } return true; } Datum ts_chunk_show(PG_FUNCTION_ARGS) { return ts_cm_functions->show_chunk(fcinfo); } Datum ts_chunk_create(PG_FUNCTION_ARGS) { return ts_cm_functions->create_chunk(fcinfo); } /** * Get the chunk status. * * Values returned are documented above and is a bitwise or of the * CHUNK_STATUS_XXX values. * * @see CHUNK_STATUS_DEFAULT * @see CHUNK_STATUS_COMPRESSED * @see CHUNK_STATUS_COMPRESSED_UNORDERED * @see CHUNK_STATUS_FROZEN * @see CHUNK_STATUS_COMPRESSED_PARTIAL */ Datum ts_chunk_status(PG_FUNCTION_ARGS) { Oid chunk_relid = PG_GETARG_OID(0); Chunk *chunk = ts_chunk_get_by_relid(chunk_relid, /* fail_if_not_found */ true); PG_RETURN_INT32(chunk->fd.status); } TS_FUNCTION_INFO_V1(ts_chunk_status_text); Datum ts_chunk_status_text(PG_FUNCTION_ARGS) { int32 status = PG_GETARG_INT32(0); ArrayBuildState *astate = initArrayResult(TEXTOID, CurrentMemoryContext, false); if (status & CHUNK_STATUS_COMPRESSED) astate = accumArrayResult(astate, CStringGetTextDatum("COMPRESSED"), false, TEXTOID, CurrentMemoryContext); if (status & CHUNK_STATUS_COMPRESSED_UNORDERED) astate = accumArrayResult(astate, CStringGetTextDatum("UNORDERED"), false, TEXTOID, CurrentMemoryContext); if (status & CHUNK_STATUS_FROZEN) astate = accumArrayResult(astate, CStringGetTextDatum("FROZEN"), false, TEXTOID, CurrentMemoryContext); if (status & CHUNK_STATUS_COMPRESSED_PARTIAL) astate = accumArrayResult(astate, CStringGetTextDatum("PARTIAL"), false, TEXTOID, CurrentMemoryContext); if (status < 0 || status > (CHUNK_STATUS_COMPRESSED | CHUNK_STATUS_COMPRESSED_UNORDERED | CHUNK_STATUS_FROZEN | CHUNK_STATUS_COMPRESSED_PARTIAL)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid chunk status %d", status))); PG_RETURN_DATUM(makeArrayResult(astate, CurrentMemoryContext)); } /* * Lock the chunk if the lockmode demands it. * * Also check that the chunk relation actually exists after the lock is * acquired. Return true if no locking is necessary or the chunk relation * exists and the lock was successfully acquired. Otherwise return false. */ bool ts_chunk_lock_if_exists(Oid chunk_oid, LOCKMODE chunk_lockmode) { /* No lock is requested, so assume relation exists */ if (chunk_lockmode != NoLock) { /* Get the lock to synchronize against concurrent drop */ LockRelationOid(chunk_oid, chunk_lockmode); /* * Now that we have the lock, double-check to see if the relation * really exists or not. If not, assume it was dropped while we * waited to acquire lock, and ignore it. */ if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(chunk_oid))) { /* Release useless lock */ UnlockRelationOid(chunk_oid, chunk_lockmode); /* And ignore this relation */ return false; } } return true; } ScanIterator ts_chunk_scan_iterator_create(MemoryContext result_mcxt) { ScanIterator it = ts_scan_iterator_create(CHUNK, AccessShareLock, result_mcxt); it.ctx.flags |= SCANNER_F_NOEND_AND_NOCLOSE; return it; } void ts_chunk_scan_iterator_set_chunk_id(ScanIterator *it, int32 chunk_id) { it->ctx.index = catalog_get_index(ts_catalog_get(), CHUNK, CHUNK_ID_INDEX); ts_scan_iterator_scan_key_reset(it); ts_scan_iterator_scan_key_init(it, Anum_chunk_idx_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(chunk_id)); } /* * Create a hypercube for the OSM chunk * The initial range for the OSM chunk will be from INT64_MAX - 1 to INT64_MAX. * This range was chosen to minimize interference with tuple routing and * occupy a range outside of potential values as there must be no overlap * between the hypercube occupied by the osm chunk and actual chunks. */ static Hypercube * fill_hypercube_for_osm_chunk(Hyperspace *hs) { Hypercube *cube = ts_hypercube_alloc(hs->num_dimensions); Assert(hs->num_dimensions == 1); // does not work with partitioned range for (int i = 0; i < hs->num_dimensions; i++) { const Dimension *dim = &hs->dimensions[i]; Assert(dim->type == DIMENSION_TYPE_OPEN); cube->slices[i] = ts_dimension_slice_create(dim->fd.id, PG_INT64_MAX - 1, PG_INT64_MAX); cube->num_slices++; } Assert(cube->num_slices == 1); return cube; } /* adds foreign table as a chunk to the hypertable. * creates a dummy chunk constraint for the time dimension. * These constraints are recorded in the chunk-dimension slice metadata. * They are NOT added as CHECK constraints on the foreign table. * * Does not add any inheritable constraints or indexes that are already * defined on the hypertable. * * This is used to add an OSM table as a chunk. * Set the osm_chunk flag to true. */ static void add_foreign_table_as_chunk(Oid relid, Hypertable *parent_ht) { Hyperspace *hs = parent_ht->space; Catalog *catalog = ts_catalog_get(); CatalogSecurityContext sec_ctx; Chunk *chunk; char *relschema = get_namespace_name(get_rel_namespace(relid)); char *relname = get_rel_name(relid); Oid ht_ownerid = ts_rel_get_owner(parent_ht->main_table_relid); if (!has_privs_of_role(GetUserId(), ht_ownerid)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("must be owner of hypertable \"%s\"", get_rel_name(parent_ht->main_table_relid)))); Assert(get_rel_relkind(relid) == RELKIND_FOREIGN_TABLE); if (hs->num_dimensions > 1) elog(ERROR, "cannot attach a foreign table to a hypertable that has more than 1 dimension"); /* Create a new chunk based on the hypercube */ ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); chunk = ts_chunk_create_base(ts_catalog_table_next_seq_id(catalog, CHUNK), hs->num_dimensions, RELKIND_RELATION); ts_catalog_restore_user(&sec_ctx); /* fill in the correct table_name for the chunk*/ chunk->fd.hypertable_id = hs->hypertable_id; chunk->fd.osm_chunk = true; /* this is an OSM chunk */ chunk->cube = fill_hypercube_for_osm_chunk(hs); chunk->hypertable_relid = parent_ht->main_table_relid; chunk->constraints = ts_chunk_constraints_alloc(1, CurrentMemoryContext); namestrcpy(&chunk->fd.schema_name, relschema); namestrcpy(&chunk->fd.table_name, relname); /* Insert chunk */ ts_chunk_insert_lock(chunk, RowExclusiveLock); /* insert dimension slices if they do not exist. */ ts_dimension_slice_insert_multi(chunk->cube->slices, chunk->cube->num_slices); /* check constraints are not automatically created for foreign tables. * See: ts_chunk_constraints_add_dimension_constraints. * Collect all the check constraints from the hypertable and add them to the * foreign table. Otherwise, cannot add as child of the hypertable (pg inheritance * code will error. Note that the name of the check constraint on the hypertable * and the foreign table chunk should match. */ ts_chunk_constraints_add_inheritable_check_constraints(chunk->constraints, chunk->fd.id, chunk->relkind, chunk->hypertable_relid); chunk_create_table_constraints(parent_ht, chunk); /* Add dimension constraints for the chunk */ ts_chunk_constraints_add_dimension_constraints(chunk->constraints, chunk->fd.id, chunk->cube); ts_chunk_constraints_insert_metadata(chunk->constraints); chunk_add_inheritance(chunk, parent_ht); /* * Update hypertable entry with tiering status information. * XXX: For compatibility reasons, we set the noncontiguous flag, but * this should be reverted as soon as the newer version of the OSM extension * is rolled out. * Noncontiguous flag should not be set since the chunk should be empty upon * creation, with an invalid range assigned, so ordered append should be allowed. * Once the data is moved into the OSM chunk, then our catalog should be * updated with proper API calls from the OSM extension. */ parent_ht->fd.status = ts_set_flags_32(parent_ht->fd.status, HYPERTABLE_STATUS_OSM | HYPERTABLE_STATUS_OSM_CHUNK_NONCONTIGUOUS); ts_hypertable_update_status_osm(parent_ht); } void ts_chunk_merge_on_dimension(const Hypertable *ht, Chunk *chunk, const Chunk *merge_chunk, int32 dimension_id) { const DimensionSlice *slice, *merge_slice; int num_ccs = 0; bool dimension_slice_found = false; if (chunk->hypertable_relid != merge_chunk->hypertable_relid) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("cannot merge chunks from different hypertables"), errhint("chunk 1: \"%s\", chunk 2: \"%s\"", get_rel_name(chunk->table_id), get_rel_name(merge_chunk->table_id)))); for (int i = 0; i < chunk->cube->num_slices; i++) { if (chunk->cube->slices[i]->fd.dimension_id == dimension_id) { slice = chunk->cube->slices[i]; merge_slice = merge_chunk->cube->slices[i]; dimension_slice_found = true; } else if (chunk->cube->slices[i]->fd.id != merge_chunk->cube->slices[i]->fd.id) { /* If the slices do not match (except on time dimension), we cannot merge the chunks. */ ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("cannot merge chunks with different partitioning schemas"), errhint("chunk 1: \"%s\", chunk 2: \"%s\" have different slices on " "dimension ID %d", get_rel_name(chunk->table_id), get_rel_name(merge_chunk->table_id), chunk->cube->slices[i]->fd.dimension_id))); } } if (!dimension_slice_found) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("cannot find slice for merging dimension"), errhint("chunk 1: \"%s\", chunk 2: \"%s\", dimension ID %d", get_rel_name(chunk->table_id), get_rel_name(merge_chunk->table_id), dimension_id))); if (slice->fd.range_end != merge_slice->fd.range_start) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("cannot merge non-adjacent chunks over supplied dimension"), errhint("chunk 1: \"%s\", chunk 2: \"%s\", dimension ID %d", get_rel_name(chunk->table_id), get_rel_name(merge_chunk->table_id), dimension_id))); num_ccs = ts_chunk_constraint_scan_by_dimension_slice_id(slice->fd.id, NULL, CurrentMemoryContext); /* There should always be an associated chunk constraint to a dimension slice. * This can only occur when the catalog metadata is corrupt. */ if (num_ccs <= 0) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("missing chunk constraint for dimension slice"), errhint("chunk: \"%s\", slice ID %d", get_rel_name(chunk->table_id), slice->fd.id))); DimensionSlice *new_slice = ts_dimension_slice_create(dimension_id, slice->fd.range_start, merge_slice->fd.range_end); /* Only if there is exactly one chunk constraint for the merged dimension slice * we can go ahead and delete it since we are dropping the chunk. */ if (num_ccs == 1) ts_dimension_slice_delete_by_id(slice->fd.id, false); /* Check for dimension slice already exists, if not create a new one. */ ScanTupLock tuplock = { .lockmode = LockTupleKeyShare, .waitpolicy = LockWaitBlock, }; if (!ts_dimension_slice_scan_for_existing(new_slice, &tuplock)) { ts_dimension_slice_insert(new_slice); } ts_chunk_constraint_update_slice_id(chunk->fd.id, slice->fd.id, new_slice->fd.id); ChunkConstraints *ccs = ts_chunk_constraints_alloc(1, CurrentMemoryContext); ScanIterator iterator = ts_scan_iterator_create(CHUNK_CONSTRAINT, AccessShareLock, CurrentMemoryContext); ts_chunk_constraint_scan_iterator_set_slice_id(&iterator, new_slice->fd.id); ts_scanner_foreach(&iterator) { bool isnull; Datum d; d = slot_getattr(ts_scan_iterator_slot(&iterator), Anum_chunk_constraint_chunk_id, &isnull); if (!isnull && DatumGetInt32(d) == chunk->fd.id) { num_ccs++; ts_chunk_constraints_add_from_tuple(ccs, ts_scan_iterator_tuple_info(&iterator)); } } if (num_ccs <= 0) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("missing chunk constraint for merged dimension slice"), errhint("chunk: \"%s\", slice ID %d", get_rel_name(chunk->table_id), new_slice->fd.id))); /* Update the slice in the chunk's hypercube. Needed to make recreate constraints work. */ for (int i = 0; i < chunk->cube->num_slices; i++) { if (chunk->cube->slices[i]->fd.dimension_id == dimension_id) { chunk->cube->slices[i] = new_slice; break; } } /* Delete the old constraint */ for (int i = 0; i < chunk->constraints->num_constraints; i++) { const ChunkConstraint *cc = &chunk->constraints->constraints[i]; if (cc->fd.dimension_slice_id == slice->fd.id) { ObjectAddress constrobj = { .classId = ConstraintRelationId, .objectId = get_relation_constraint_oid(chunk->table_id, NameStr(cc->fd.constraint_name), false), }; performDeletion(&constrobj, DROP_RESTRICT, 0); break; } } /* We have to recreate the chunk constraints since we are changing * table constraints when updating the slice. */ ChunkConstraints *oldccs = chunk->constraints; chunk->constraints = ccs; ts_process_utility_set_expect_chunk_modification(true); ts_chunk_constraints_create(ht, chunk); ts_chunk_copy_referencing_fk(ht, chunk); ts_process_utility_set_expect_chunk_modification(false); chunk->constraints = oldccs; ts_chunk_drop(merge_chunk, DROP_RESTRICT, 1); } /* Internal API used by OSM extension. OSM table is a foreign table that is * attached as a chunk of the hypertable. A chunk needs dimension constraints. We * add dummy constraints for the OSM chunk and then attach it to the hypertable. * OSM extension is responsible for maintaining any constraints on this table. */ Datum ts_chunk_attach_osm_table_chunk(PG_FUNCTION_ARGS) { Oid hypertable_relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); Oid ftable_relid = PG_ARGISNULL(1) ? InvalidOid : PG_GETARG_OID(1); bool ret = false; Cache *hcache; Hypertable *ht = ts_hypertable_cache_get_cache_and_entry(hypertable_relid, CACHE_FLAG_MISSING_OK, &hcache); if (!ht) { char *name = get_rel_name(hypertable_relid); if (!name) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("invalid Oid"))); else ereport(ERROR, (errcode(ERRCODE_TS_HYPERTABLE_NOT_EXIST), errmsg("\"%s\" is not a hypertable", name))); } if (get_rel_relkind(ftable_relid) == RELKIND_FOREIGN_TABLE) { add_foreign_table_as_chunk(ftable_relid, ht); ret = true; } ts_cache_release(&hcache); PG_RETURN_BOOL(ret); } static ScanTupleResult chunk_tuple_osm_chunk_found(TupleInfo *ti, void *arg) { bool isnull; Datum osm_chunk = slot_getattr(ti->slot, Anum_chunk_osm_chunk, &isnull); Assert(!isnull); bool is_osm_chunk = DatumGetBool(osm_chunk); if (!is_osm_chunk) return SCAN_CONTINUE; int *chunk_id = (int *) arg; Datum chunk_id_datum = slot_getattr(ti->slot, Anum_chunk_id, &isnull); Assert(!isnull); *chunk_id = DatumGetInt32(chunk_id_datum); return SCAN_DONE; } /* get OSM chunk id associated with the hypertable */ int ts_chunk_get_osm_chunk_id(int hypertable_id) { int chunk_id = INVALID_CHUNK_ID; ScanKeyData scankey[2]; bool is_osm_chunk = true; Catalog *catalog = ts_catalog_get(); ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, CHUNK), .index = catalog_get_index(catalog, CHUNK, CHUNK_OSM_CHUNK_INDEX), .nkeys = 2, .scankey = scankey, .data = &chunk_id, .tuple_found = chunk_tuple_osm_chunk_found, .lockmode = AccessShareLock, .scandirection = ForwardScanDirection, }; /* * Perform an index scan on hypertable ID + osm_chunk */ ScanKeyInit(&scankey[0], Anum_chunk_osm_chunk_idx_osm_chunk, BTEqualStrategyNumber, F_BOOLEQ, BoolGetDatum(is_osm_chunk)); ScanKeyInit(&scankey[1], Anum_chunk_osm_chunk_idx_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); int num_found = ts_scanner_scan(&scanctx); if (num_found > 1) { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("More than 1 OSM chunk found for hypertable (%d)", hypertable_id))); } return chunk_id; } /* Upon creation, OSM chunks are assigned an invalid range [INT64_MAX -1, infinity) */ bool ts_osm_chunk_range_is_invalid(int64 range_start, int64 range_end) { return ((range_end == PG_INT64_MAX) && (range_start == range_end - 1)); } int32 ts_chunk_get_osm_slice_id(int32 chunk_id, int32 time_dim_id) { Chunk *chunk = ts_chunk_get_by_id(chunk_id, true); const DimensionSlice *ds = ts_hypercube_get_slice_by_dimension_id(chunk->cube, time_dim_id); const int slice_id = ds->fd.id; return slice_id; } /* * Initialization and access method for ChunkVec. This needs to be extended to support additional * operations. */ static ChunkVec * chunk_vec_expand(ChunkVec *chunks, uint32 new_capacity) { if (chunks != NULL && chunks->capacity >= new_capacity) return chunks; if (chunks == NULL) chunks = palloc(CHUNK_VEC_SIZE(new_capacity)); else chunks = repalloc(chunks, CHUNK_VEC_SIZE(new_capacity)); chunks->capacity = new_capacity; return chunks; } ChunkVec * ts_chunk_vec_create(int32 capacity) { ChunkVec *chunks = chunk_vec_expand(NULL, capacity); chunks->num_chunks = 0; return chunks; } ChunkVec * ts_chunk_vec_sort(ChunkVec **chunks) { ChunkVec *vec = *chunks; if (vec->num_chunks > 1) qsort(&vec->chunks, vec->num_chunks, sizeof(Chunk), chunk_cmp); return vec; } ChunkVec * ts_chunk_vec_add_from_tuple(ChunkVec **chunks, TupleInfo *ti) { Chunk *chunkptr = NULL; ChunkVec *vec; int num_constraints_hint; ScanIterator it; vec = *chunks; num_constraints_hint = 2; if (vec->num_chunks + 1 > vec->capacity) *chunks = vec = chunk_vec_expand(vec, vec->capacity + DEFAULT_CHUNK_VEC_SIZE); chunkptr = &vec->chunks[vec->num_chunks++]; ts_chunk_formdata_fill(&chunkptr->fd, ti); /* * Get the chunk constraints, hypercube slices and relation details. */ chunkptr->constraints = ts_chunk_constraint_scan_by_chunk_id(chunkptr->fd.id, num_constraints_hint, ti->mctx); it = ts_dimension_slice_scan_iterator_create(NULL, ti->mctx); chunkptr->cube = ts_hypercube_from_constraints(chunkptr->constraints, &it); ts_scan_iterator_close(&it); chunkptr->table_id = ts_get_relation_relid(NameStr(chunkptr->fd.schema_name), NameStr(chunkptr->fd.table_name), true); chunkptr->hypertable_relid = ts_hypertable_id_to_relid(chunkptr->fd.hypertable_id, false); chunkptr->relkind = get_rel_relkind(chunkptr->table_id); return vec; } /* * Prepare for an index scan for all the chunks matching the given hypertable_id and range criteria. */ static int chunk_creation_time_set_range(ScanIterator *it, Oid hypertable_id, StrategyNumber start_strategy, int64 start_value, StrategyNumber end_strategy, int64 end_value) { TypeCacheEntry *tce; it->ctx.index = catalog_get_index(ts_catalog_get(), CHUNK, CHUNK_HYPERTABLE_ID_CREATION_TIME_INDEX); ts_scan_iterator_scan_key_reset(it); ts_scan_iterator_scan_key_init(it, Anum_chunk_hypertable_id_creation_time_idx_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); tce = lookup_type_cache(TIMESTAMPTZOID, TYPECACHE_BTREE_OPFAMILY); /* If both are valid strategies then a proper scan within these limits will be performed */ if (start_strategy != InvalidStrategy) { Oid opno = get_opfamily_member(tce->btree_opf, TIMESTAMPTZOID, TIMESTAMPTZOID, start_strategy); Oid proc = get_opcode(opno); Assert(OidIsValid(proc)); ts_scan_iterator_scan_key_init(it, Anum_chunk_hypertable_id_creation_time_idx_creation_time, start_strategy, proc, Int64GetDatum(start_value)); } if (end_strategy != InvalidStrategy) { Oid opno = get_opfamily_member(tce->btree_opf, TIMESTAMPTZOID, TIMESTAMPTZOID, end_strategy); Oid proc = get_opcode(opno); Assert(OidIsValid(proc)); ts_scan_iterator_scan_key_init(it, Anum_chunk_hypertable_id_creation_time_idx_creation_time, end_strategy, proc, Int64GetDatum(end_value)); } return it->ctx.nkeys; } /* * Perform an index scan for given hypertable_id and chunk creation time range. * * Returns an array of chunks using ChunkVec the encloses all the chunks satisfying the range * criteria. */ static Chunk * get_chunks_in_creation_time_range_limit(Hypertable *ht, StrategyNumber start_strategy, int64 start_value, StrategyNumber end_strategy, int64 end_value, int limit, uint64 *num_chunks, ScanTupLock *tupLock) { ScanIterator it; ChunkVec *chunk_vec = NULL; it = ts_scan_iterator_create(CHUNK, AccessShareLock, CurrentMemoryContext); it.ctx.flags |= SCANNER_F_NOEND_AND_NOCLOSE; it.ctx.tuplock = tupLock; chunk_creation_time_set_range(&it, ht->fd.id, start_strategy, start_value, end_strategy, end_value); it.ctx.limit = limit; #ifdef TS_DEBUG /* allow testing of the chunk vec expansion in debug builds */ if (limit == -1) limit = 1; #endif chunk_vec = ts_chunk_vec_create(limit > 0 ? limit : DEFAULT_CHUNK_VEC_SIZE); ts_scanner_foreach(&it) { TupleInfo *ti = ts_scan_iterator_tuple_info(&it); ts_chunk_vec_add_from_tuple(&chunk_vec, ti); } ts_scan_iterator_close(&it); ts_chunk_vec_sort(&chunk_vec); *num_chunks = chunk_vec->num_chunks; return chunk_vec->chunks; } Chunk * get_chunks_in_creation_time_range(Hypertable *ht, int64 older_than, int64 newer_than, MemoryContext mctx, uint64 *num_chunks_returned, ScanTupLock *tupLock) { MemoryContext oldcontext; Chunk *chunks = NULL; StrategyNumber start_strategy; StrategyNumber end_strategy; uint64 num_chunks = 0; if (older_than <= newer_than) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid chunk creation time range"), errhint("The start of the time range must be before the end."))); start_strategy = (newer_than == PG_INT64_MIN) ? InvalidStrategy : BTGreaterEqualStrategyNumber; end_strategy = (older_than == PG_INT64_MAX) ? InvalidStrategy : BTLessStrategyNumber; oldcontext = MemoryContextSwitchTo(mctx); chunks = get_chunks_in_creation_time_range_limit(ht, start_strategy, newer_than, end_strategy, older_than, -1, &num_chunks, tupLock); MemoryContextSwitchTo(oldcontext); *num_chunks_returned = num_chunks; return chunks; } Datum ts_chunk_drop_osm_chunk(PG_FUNCTION_ARGS) { Oid hypertable_relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); Cache *hcache = ts_hypertable_cache_pin(); Hypertable *ht = ts_resolve_hypertable_from_table_or_cagg(hcache, hypertable_relid, true); int32 osm_chunk_id = ts_chunk_get_osm_chunk_id(ht->fd.id); Chunk *osm_chunk = ts_chunk_get_by_id(osm_chunk_id, true); ts_chunk_validate_chunk_status_for_operation(osm_chunk, CHUNK_DROP, true); /* do not drop any chunk dependencies */ ts_chunk_drop(osm_chunk, DROP_RESTRICT, LOG); /* reset hypertable OSM status */ ht->fd.status = ts_clear_flags_32(ht->fd.status, HYPERTABLE_STATUS_OSM | HYPERTABLE_STATUS_OSM_CHUNK_NONCONTIGUOUS); ts_hypertable_update_status_osm(ht); ts_cache_release(&hcache); PG_RETURN_BOOL(true); } TS_FUNCTION_INFO_V1(ts_merge_two_chunks); Datum ts_merge_two_chunks(PG_FUNCTION_ARGS) { Datum chunks[2] = { PG_GETARG_DATUM(0), PG_GETARG_DATUM(1) }; bool concurrently = PG_ARGISNULL(2) ? false : PG_GETARG_BOOL(2); ArrayType *chunk_array = construct_array(chunks, 2, REGCLASSOID, sizeof(Oid), true, TYPALIGN_INT); return DirectFunctionCall2(ts_cm_functions->merge_chunks, PointerGetDatum(chunk_array), BoolGetDatum(concurrently)); } TS_FUNCTION_INFO_V1(ts_merge_chunks_concurrently); Datum ts_merge_chunks_concurrently(PG_FUNCTION_ARGS) { return DirectFunctionCall2(ts_cm_functions->merge_chunks, PG_GETARG_DATUM(0), BoolGetDatum(true)); } void ts_chunk_detach_by_relid(Oid relid) { ScanIterator iterator = ts_scan_iterator_create(CHUNK, RowExclusiveLock, CurrentMemoryContext); char *schema; char *table; int PG_USED_FOR_ASSERTS_ONLY count; Assert(OidIsValid(relid)); schema = get_namespace_name(get_rel_namespace(relid)); table = get_rel_name(relid); init_scan_by_qualified_table_name(&iterator, schema, table); count = chunk_delete(&iterator, relid, DROP_RESTRICT, true); /* * (schema,table) names and (hypertable_id) are unique so should only have * dropped one chunk */ Assert(count == 1); } ================================================ FILE: src/chunk.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <access/htup.h> #include <access/tupdesc.h> #include <foreign/foreign.h> #include <utils/hsearch.h> #include "chunk_constraint.h" #include "export.h" #include "hypertable.h" #include "ts_catalog/catalog.h" #define INVALID_CHUNK_ID 0 #define IS_OSM_CHUNK(chunk) ((chunk)->fd.osm_chunk == true) /* Should match definitions in ddl_api.sql */ #define DROP_CHUNKS_FUNCNAME "drop_chunks" #define DROP_CHUNKS_NARGS 6 #define COMPRESS_CHUNK_FUNCNAME "compress_chunk" #define COMPRESS_CHUNK_NARGS 2 #define DECOMPRESS_CHUNK_FUNCNAME "decompress_chunk" #define RECOMPRESS_CHUNK_FUNCNAME "recompress_chunk" #define RECOMPRESS_CHUNK_NARGS 2 typedef enum ChunkCompressionStatus { CHUNK_COMPRESS_NONE = 0, CHUNK_COMPRESS_UNORDERED, CHUNK_COMPRESS_ORDERED, } ChunkCompressionStatus; typedef enum ChunkOperation { CHUNK_DROP = 0, CHUNK_INSERT, CHUNK_UPDATE, CHUNK_DELETE, CHUNK_SELECT, CHUNK_COMPRESS, CHUNK_DECOMPRESS, } ChunkOperation; typedef struct Hypercube Hypercube; typedef struct Point Point; typedef struct Hyperspace Hyperspace; typedef struct Hypertable Hypertable; /* * A chunk represents a table that stores data, part of a partitioned * table. * * Conceptually, a chunk is a hypercube in an N-dimensional space. The * boundaries of the cube is represented by a collection of slices from the N * distinct dimensions. */ typedef struct Chunk { FormData_chunk fd; char relkind; Oid table_id; Oid hypertable_relid; /* * The hypercube defines the chunks position in the N-dimensional space. * Each of the N slices in the cube corresponds to a constraint on the * chunk table. */ Hypercube *cube; ChunkConstraints *constraints; } Chunk; /* This structure is used during the join of the chunk constraints to find * chunks that match all constraints. It is a stripped down version of the chunk * since we don't want to fill in all the fields until we find a match. */ typedef struct ChunkStub { int32 id; Hypercube *cube; ChunkConstraints *constraints; } ChunkStub; /* * ChunkScanCtx is used to scan for chunks in a hypertable's N-dimensional * hyperspace. * * For every matching constraint, a corresponding chunk will be created in the * context's hash table, keyed on the chunk ID. */ typedef struct ChunkScanCtx { HTAB *htab; char relkind; /* Create chunks of this relkind */ const Hypertable *ht; const Point *point; unsigned int num_complete_chunks; uint64 num_processed; bool early_abort; LOCKMODE lockmode; void *data; } ChunkScanCtx; /* Returns true if the stub has a full set of constraints, otherwise * false. Used to find a stub matching a point in an N-dimensional * hyperspace. */ static inline bool chunk_stub_is_complete(const ChunkStub *stub, const Hyperspace *space) { return space->num_dimensions == stub->constraints->num_dimension_constraints; } /* The hash table entry for the ChunkScanCtx */ typedef struct ChunkScanEntry { int32 chunk_id; ChunkStub *stub; /* Used for fast chunk search where we don't want to build chunk stubs. */ int32 num_dimension_constraints; } ChunkScanEntry; /* * Information to be able to display a scan key details for error messages. */ typedef struct DisplayKeyData { const char *name; const char *(*as_string)(Datum); } DisplayKeyData; /* * Chunk vector is collection of chunks for a given hypertable_id. */ typedef struct ChunkVec { uint32 capacity; uint32 num_chunks; Chunk chunks[FLEXIBLE_ARRAY_MEMBER]; } ChunkVec; extern ChunkVec *ts_chunk_vec_create(int32 capacity); extern ChunkVec *ts_chunk_vec_sort(ChunkVec **chunks); extern ChunkVec *ts_chunk_vec_add_from_tuple(ChunkVec **chunks, TupleInfo *ti); #define CHUNK_VEC_SIZE(num_chunks) (sizeof(ChunkVec) + (sizeof(Chunk) * num_chunks)) #define DEFAULT_CHUNK_VEC_SIZE 10 extern void ts_chunk_formdata_fill(FormData_chunk *fd, const TupleInfo *ti); extern int32 ts_chunk_point_find_chunk_id(const Hypertable *ht, const Point *p, const ScanTupLock *slice_lock); extern Chunk *ts_chunk_find_for_point(const Hypertable *ht, const Point *p, LOCKMODE lockmode); extern Chunk *ts_chunk_create_for_point(const Hypertable *ht, const Point *p, const char *schema, const char *prefix, LOCKMODE chunk_lockmode); List *ts_chunk_id_find_in_subspace(Hypertable *ht, List *dimension_vecs); extern TSDLLEXPORT Chunk *ts_chunk_create_base(int32 id, int16 num_constraints, const char relkind); extern TSDLLEXPORT ChunkStub *ts_chunk_stub_create(int32 id, int16 num_constraints); extern TSDLLEXPORT Chunk *ts_chunk_copy(const Chunk *chunk); extern TSDLLEXPORT Chunk * ts_chunk_get_by_name_with_memory_context(const char *schema_name, const char *table_name, LOCKMODE chunk_lockmode, const ScanTupLock *slice_lock, MemoryContext mctx, bool fail_if_not_found); extern TSDLLEXPORT void ts_chunk_insert_lock(const Chunk *chunk, LOCKMODE lock); extern TSDLLEXPORT Oid ts_chunk_create_table(const Chunk *chunk, const Hypertable *ht, const char *tablespacename); extern TSDLLEXPORT Chunk *ts_chunk_get_by_id_with_slice_lock(int32 id, LOCKMODE chunk_lockmode, const ScanTupLock *slice_lock, bool fail_if_not_found); extern TSDLLEXPORT Chunk *ts_chunk_get_by_id(int32 id, bool fail_if_not_found); extern TSDLLEXPORT Chunk *ts_chunk_get_by_relid_locked(Oid relid, LOCKMODE lockmode, const ScanTupLock *slice_lock, bool fail_if_not_found); extern TSDLLEXPORT Chunk *ts_chunk_get_by_relid(Oid relid, bool fail_if_not_found); extern TSDLLEXPORT void ts_chunk_free(Chunk *chunk); extern bool ts_chunk_exists(const char *schema_name, const char *table_name); extern TSDLLEXPORT int32 ts_chunk_get_hypertable_id_by_reloid(Oid reloid); extern TSDLLEXPORT FormData_chunk ts_chunk_get_formdata(int32 chunk_id); extern TSDLLEXPORT bool ts_chunk_simple_scan_by_reloid(Oid reloid, FormData_chunk *form, bool missing_ok); extern TSDLLEXPORT Oid ts_chunk_get_relid(int32 chunk_id, bool missing_ok); extern Oid ts_chunk_get_schema_id(int32 chunk_id, bool missing_ok); extern TSDLLEXPORT bool ts_chunk_get_id(const char *schema, const char *table, int32 *chunk_id, bool missing_ok); extern bool ts_chunk_exists_relid(Oid relid); extern TSDLLEXPORT bool ts_chunk_exists_with_compression(int32 hypertable_id); extern void ts_chunk_recreate_all_constraints_for_dimension(Hypertable *ht, int32 dimension_id); extern int ts_chunk_delete_by_hypertable_id(int32 hypertable_id); extern TSDLLEXPORT int ts_chunk_delete_by_name(const char *schema, const char *table, DropBehavior behavior); extern int ts_chunk_delete_by_relid_and_relname(Oid relid, const char *schemaname, const char *tablename, DropBehavior behavior); extern bool ts_chunk_set_name(Chunk *chunk, const char *newname); extern bool ts_chunk_set_schema(Chunk *chunk, const char *newschema); extern TSDLLEXPORT List *ts_chunk_get_window(int32 dimension_id, int64 point, int count, MemoryContext mctx); extern void ts_chunks_rename_schema_name(char *old_schema, char *new_schema); extern TSDLLEXPORT bool ts_chunk_set_partial(Chunk *chunk); extern TSDLLEXPORT bool ts_chunk_set_unordered(Chunk *chunk); extern TSDLLEXPORT bool ts_chunk_set_frozen(Chunk *chunk); extern TSDLLEXPORT bool ts_chunk_unset_frozen(Chunk *chunk); extern TSDLLEXPORT bool ts_chunk_is_frozen(const Chunk *chunk); extern TSDLLEXPORT bool ts_chunk_set_compressed_chunk(Chunk *chunk, int32 compressed_chunk_id); extern TSDLLEXPORT bool ts_chunk_clear_compressed_chunk(Chunk *chunk); extern TSDLLEXPORT void ts_chunk_drop(const Chunk *chunk, DropBehavior behavior, int32 log_level); extern TSDLLEXPORT List *ts_chunk_do_drop_chunks(Hypertable *ht, int64 older_than, int64 newer_than, int32 log_level, Oid time_type, Oid arg_type, bool older_newer); extern TSDLLEXPORT Chunk * ts_chunk_find_or_create_without_cuts(const Hypertable *ht, Hypercube *hc, const char *schema_name, const char *table_name, Oid chunk_table_relid, bool *created); extern TSDLLEXPORT Chunk *ts_chunk_get_compressed_chunk_parent(const Chunk *chunk); extern TSDLLEXPORT bool ts_chunk_is_unordered(const Chunk *chunk); extern TSDLLEXPORT bool ts_chunk_is_partial(const Chunk *chunk); extern TSDLLEXPORT bool ts_chunk_is_compressed(const Chunk *chunk); extern TSDLLEXPORT bool ts_chunk_needs_compression(const Chunk *chunk); extern TSDLLEXPORT bool ts_chunk_needs_recompression(const Chunk *chunk); extern TSDLLEXPORT bool ts_chunk_validate_chunk_status_for_operation(const Chunk *chunk, ChunkOperation cmd, bool throw_error); extern TSDLLEXPORT bool ts_chunk_contains_compressed_data(const Chunk *chunk); extern TSDLLEXPORT ChunkCompressionStatus ts_chunk_get_compression_status(int32 chunk_id); extern TSDLLEXPORT Datum ts_chunk_id_from_relid(PG_FUNCTION_ARGS); extern TSDLLEXPORT Datum ts_chunk_status_text(PG_FUNCTION_ARGS); extern TSDLLEXPORT List *ts_chunk_get_chunk_ids_by_hypertable_id(int32 hypertable_id); extern TSDLLEXPORT List *ts_chunk_get_by_hypertable_id(int32 hypertable_id); extern TSDLLEXPORT int64 ts_chunk_primary_dimension_start(const Chunk *chunk); extern TSDLLEXPORT int64 ts_chunk_primary_dimension_end(const Chunk *chunk); extern Chunk *ts_chunk_build_from_tuple_and_stub(Chunk **chunkptr, TupleInfo *ti, const ChunkStub *stub, const ScanTupLock *slice_lock); extern TM_Result ts_chunk_lock_for_creating_compressed_chunk(int32 chunk_id, int32 *compressed_chunk_id); extern ScanIterator ts_chunk_scan_iterator_create(MemoryContext result_mcxt); extern void ts_chunk_scan_iterator_set_chunk_id(ScanIterator *it, int32 chunk_id); extern bool ts_chunk_lock_if_exists(Oid chunk_oid, LOCKMODE chunk_lockmode); int ts_chunk_get_osm_chunk_id(int hypertable_id); extern TSDLLEXPORT void ts_chunk_merge_on_dimension(const Hypertable *ht, Chunk *chunk, const Chunk *merge_chunk, int32 dimension_id); extern TSDLLEXPORT void ts_chunk_detach_by_relid(Oid relid); #define chunk_get_by_name(schema_name, table_name, chunk_lockmode, slice_lock, fail_if_not_found) \ ts_chunk_get_by_name_with_memory_context(schema_name, \ table_name, \ chunk_lockmode, \ slice_lock, \ CurrentMemoryContext, \ fail_if_not_found) /* * Sanity checks for chunk. * * The individual checks are split into separate Asserts so it's * easier to tell from a stacktrace which one failed. */ #define ASSERT_IS_VALID_CHUNK(chunk) \ do \ { \ Assert(chunk); \ Assert((chunk)->fd.id > 0); \ Assert((chunk)->fd.hypertable_id > 0); \ Assert(OidIsValid((chunk)->table_id)); \ Assert(OidIsValid((chunk)->hypertable_relid)); \ Assert((chunk)->cube); \ Assert((chunk)->relkind == RELKIND_RELATION || (chunk)->relkind == RELKIND_FOREIGN_TABLE); \ } while (0) /* * The chunk status field values are persisted in the database and must never be changed. * Those values are used as flags and must always be powers of 2 to allow bitwise operations. * When adding new status values we must make sure to add special handling for these values * to the downgrade script as previous versions will not know how to deal with those. */ #define CHUNK_STATUS_DEFAULT 0 /* * Setting a chunk status field as CHUNK_STATUS_COMPRESSED means that the corresponding * compressed_chunk_id field points to a chunk that holds the compressed data. Otherwise, * the corresponding compressed_chunk_id is NULL. */ #define CHUNK_STATUS_COMPRESSED 1 /* * When inserting into a compressed chunk the configured compress_orderby is not retained. * Any such chunks need an explicit Sort step to produce ordered output until the chunk * ordering has been restored by recompress_chunk. This flag can only exist on compressed * chunks. */ #define CHUNK_STATUS_COMPRESSED_UNORDERED 2 /* * A chunk is in frozen state (i.e no inserts/updates/deletes into this chunk are * permitted. Other chunk level operations like dropping chunk etc. are also blocked. * */ #define CHUNK_STATUS_FROZEN 4 /* * A chunk is in this state when it is compressed but also has uncompressed tuples * in the uncompressed chunk. */ #define CHUNK_STATUS_COMPRESSED_PARTIAL 8 extern TSDLLEXPORT bool ts_chunk_clear_status(Chunk *chunk, int32 status); extern bool ts_osm_chunk_range_is_invalid(int64 range_start, int64 range_end); extern int32 ts_chunk_get_osm_slice_id(int32 chunk_id, int32 time_dim_id); ================================================ FILE: src/chunk_adaptive.c ================================================ /* * The contents and feature provided by this file (and its associated header * file) -- adaptive chunking -- are currently in BETA. * Feedback, suggestions, and bugs should be reported * on the GitHub repository issues page, or in our public slack. */ #include <postgres.h> #include <catalog/pg_proc.h> #include <catalog/pg_type.h> #include <funcapi.h> #include <math.h> #include <miscadmin.h> #include <parser/parse_func.h> #include <utils/acl.h> #include <utils/array.h> #include <utils/builtins.h> #include <utils/guc.h> #include <utils/lsyscache.h> #include <utils/snapmgr.h> #include <utils/syscache.h> #include <utils/typcache.h> #include "compat/compat.h" #include "chunk.h" #include "chunk_adaptive.h" #include "errors.h" #include "hypercube.h" #include "hypertable_cache.h" #include "utils.h" #define DEFAULT_CHUNK_SIZING_FN_NAME "calculate_chunk_interval" /* This can be set to a positive number (and non-zero) value from tests to * simulate memory cache size. This makes it possible to run tests * deterministically. */ static int64 fixed_memory_cache_size = -1; /* * Takes a PostgreSQL text representation of data (e.g., 40MB) and converts it * into a int64 for calculations */ static int64 convert_text_memory_amount_to_bytes(const char *memory_amount) { const char *hintmsg; int nblocks; int64 bytes; if (NULL == memory_amount) elog(ERROR, "invalid memory amount"); if (!parse_int(memory_amount, &nblocks, GUC_UNIT_BLOCKS, &hintmsg)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid data amount"), errhint("%s", hintmsg))); bytes = nblocks; bytes *= BLCKSZ; return bytes; } /* * Exposed for testing purposes to be able to simulate a different memory * cache size in tests. */ TS_FUNCTION_INFO_V1(ts_set_memory_cache_size); Datum ts_set_memory_cache_size(PG_FUNCTION_ARGS) { const char *memory_amount = text_to_cstring(PG_GETARG_TEXT_P(0)); fixed_memory_cache_size = convert_text_memory_amount_to_bytes(memory_amount); PG_RETURN_INT64(fixed_memory_cache_size); } /* * Get the amount of cache memory for chunks. * We use shared_buffers converted to bytes. */ static int64 get_memory_cache_size(void) { const char *val; const char *hintmsg; int shared_buffers; int64 memory_bytes; if (fixed_memory_cache_size > 0) return fixed_memory_cache_size; val = GetConfigOption("shared_buffers", false, false); if (NULL == val) elog(ERROR, "missing configuration for 'shared_buffers'"); if (!parse_int(val, &shared_buffers, GUC_UNIT_BLOCKS, &hintmsg)) elog(ERROR, "could not parse 'shared_buffers' setting: %s", hintmsg); memory_bytes = shared_buffers; /* Value is in blocks, so convert to bytes */ memory_bytes *= BLCKSZ; return memory_bytes; } /* * For chunk sizing, we don't want to set chunk size exactly the same as the * available cache memory, since chunk sizes won't be exact. We therefore give * some slack here. */ #define DEFAULT_CACHE_MEMORY_SLACK 0.9 extern inline int64 ts_chunk_calculate_initial_chunk_target_size(void) { return (int64) ((double) get_memory_cache_size() * DEFAULT_CACHE_MEMORY_SLACK); } typedef enum MinMaxResult { MINMAX_NO_INDEX, MINMAX_NO_TUPLES, MINMAX_FOUND, } MinMaxResult; /* * Use a heap scan to find the min and max of a given column of a chunk. This * could be a rather costly operation. Should figure out how to keep min-max * stats cached. */ static MinMaxResult minmax_heapscan(Relation rel, Oid atttype, AttrNumber attnum, Datum minmax[2]) { TupleTableSlot *slot = table_slot_create(rel, NULL); TableScanDesc scan; TypeCacheEntry *tce; bool nulls[2] = { true, true }; /* Lookup the tuple comparison function from the type cache */ tce = lookup_type_cache(atttype, TYPECACHE_CMP_PROC | TYPECACHE_CMP_PROC_FINFO); if (NULL == tce || !OidIsValid(tce->cmp_proc)) elog(ERROR, "no comparison function for type %u", atttype); PushActiveSnapshot(GetTransactionSnapshot()); scan = table_beginscan(rel, GetActiveSnapshot(), 0, NULL); while (table_scan_getnextslot(scan, ForwardScanDirection, slot)) { bool isnull; Datum value = slot_getattr(slot, attnum, &isnull); if (isnull) continue; /* Check for new min */ if (nulls[0] || DatumGetInt32(FunctionCall2(&tce->cmp_proc_finfo, value, minmax[0])) < 0) { nulls[0] = false; minmax[0] = value; } /* Check for new max */ if (nulls[1] || DatumGetInt32(FunctionCall2(&tce->cmp_proc_finfo, value, minmax[1])) > 0) { nulls[1] = false; minmax[1] = value; } } table_endscan(scan); ExecDropSingleTupleTableSlot(slot); PopActiveSnapshot(); return (nulls[0] || nulls[1]) ? MINMAX_NO_TUPLES : MINMAX_FOUND; } /* * Use an index scan to find the min and max of a given column of a chunk. */ static MinMaxResult minmax_indexscan(Relation rel, Relation idxrel, AttrNumber attnum, Datum minmax[2]) { PushActiveSnapshot(GetTransactionSnapshot()); IndexScanDesc scan = index_beginscan_compat(rel, idxrel, GetActiveSnapshot(), NULL, 0, 0); TupleTableSlot *slot = table_slot_create(rel, NULL); bool nulls[2] = { true, true }; int i; ScanDirection directions[2] = { ForwardScanDirection /* min */, BackwardScanDirection /* max */ }; int16 option = idxrel->rd_indoption[0]; bool index_orderby_asc = ((option & INDOPTION_DESC) == 0); /* default index ordering is ASC, check if that's not the case */ if (!index_orderby_asc) { directions[0] = BackwardScanDirection; directions[1] = ForwardScanDirection; } for (i = 0; i < 2; i++) { bool found_tuple; bool isnull; index_rescan(scan, NULL, 0, NULL, 0); found_tuple = index_getnext_slot(scan, directions[i], slot); if (!found_tuple) break; minmax[i] = slot_getattr(slot, attnum, &isnull); nulls[i] = isnull; } index_endscan(scan); ExecDropSingleTupleTableSlot(slot); PopActiveSnapshot(); Assert((nulls[0] && nulls[1]) || (!nulls[0] && !nulls[1])); return nulls[0] ? MINMAX_NO_TUPLES : MINMAX_FOUND; } /* * Do a scan for min and max using and index on the given column. */ static MinMaxResult relation_minmax_indexscan(Relation rel, Oid atttype, Name attname, AttrNumber attnum, Datum minmax[2]) { List *indexlist = RelationGetIndexList(rel); ListCell *lc; MinMaxResult res = MINMAX_NO_INDEX; foreach (lc, indexlist) { Relation idxrel; Form_pg_attribute idxattr; idxrel = index_open(lfirst_oid(lc), AccessShareLock); idxattr = TupleDescAttr(idxrel->rd_att, 0); if (idxattr->atttypid == atttype && namestrcmp(&idxattr->attname, NameStr(*attname)) == 0) res = minmax_indexscan(rel, idxrel, attnum, minmax); index_close(idxrel, AccessShareLock); if (res == MINMAX_FOUND) break; } return res; } /* * Determines if a table has an appropriate index for finding the minimum and * maximum time value. This would be an index whose first column is the same as * the column used for time partitioning. */ static bool table_has_minmax_index(Oid relid, Oid atttype, Name attname, AttrNumber attnum) { Datum minmax[2]; Relation rel = table_open(relid, AccessShareLock); MinMaxResult res = relation_minmax_indexscan(rel, atttype, attname, attnum, minmax); table_close(rel, AccessShareLock); return res != MINMAX_NO_INDEX; } /* * Get the min and max value for a given column of a chunk. * * Returns true iff min and max is found, otherwise false. */ static bool chunk_get_minmax(Oid relid, Oid atttype, AttrNumber attnum, const char *call_context, Datum minmax[2]) { Relation rel = table_open(relid, AccessShareLock); NameData attname; MinMaxResult res; namestrcpy(&attname, get_attname(relid, attnum, false)); res = relation_minmax_indexscan(rel, atttype, &attname, attnum, minmax); if (res == MINMAX_NO_INDEX) { ereport(WARNING, (errmsg("no index on \"%s\" found for %s on chunk \"%s\"", NameStr(attname), call_context, get_rel_name(relid)), errdetail("%s works best with an index on the dimension.", call_context))); res = minmax_heapscan(rel, atttype, attnum, minmax); } table_close(rel, AccessShareLock); return res == MINMAX_FOUND; } #define CHUNK_SIZING_FUNC_NARGS 3 #define DEFAULT_CHUNK_WINDOW 3 /* Tuples must have filled this fraction of the chunk interval to use it to * estimate a new chunk time interval */ #define INTERVAL_FILLFACTOR_THRESH 0.5 /* A chunk must fill this (extrapolated) fraction of the target size to use it * to estimate a new chunk time interval. */ #define SIZE_FILLFACTOR_THRESH 0.15 /* The calculated chunk time interval must differ this much to actually change * the interval */ #define INTERVAL_MIN_CHANGE_THRESH 0.15 /* More than this number of intervals must be undersized in order to use the * undersized calculation path */ #define NUM_UNDERSIZED_INTERVALS 1 /* Threshold to boost to if there are only undersized intervals to make * predictions from. This should be slightly above the SIZE_FILLFACTOR_THRESH * so that the next chunks made with this are likely to meet that threshold * and be used in normal prediction mode */ #define UNDERSIZED_FILLFACTOR_THRESH (SIZE_FILLFACTOR_THRESH * 1.1) TS_FUNCTION_INFO_V1(ts_calculate_chunk_interval); /* * Calculate a new interval for a chunk in a given dimension. * * This function implements the main algorithm for adaptive chunking. Given a * dimension, coordinate (point) on the dimensional axis (e.g., point in time), * and a chunk target size (in bytes), the function should return a new * interval that best fills the chunk to the target size. * * The intuition behind the current implementation is to look back at the recent * past chunks in the dimension and look at how close they are to the target * size (the fillfactor) and then use that information to calculate a new * interval. I.e., if the fillfactor of a past chunk was below 1.0 we increase * the interval, and if it was above 1.0 we decrease it. Thus, for each past * chunk, we calculate the interval that would have filled the chunk to the * target size. Then, to calculate the new chunk interval, we average the * intervals of the past chunks. * * Note, however, that there are a couple of caveats. First, we cannot look back * at the most recently created chunks, because there is no guarantee that data * was written exactly in order of the dimension we are looking at. Therefore, * we "look back" along the dimension axis instead of by, e.g., chunk * ID. Second, chunks can be filled unevenly. Below are three examples of how * chunks can be filled ('*' represents data): * *' |--------| *' | * * * *| 1. Evenly filled (ideal) *' |--------| * *' |--------| *' | ****| 2. Partially filled *' |--------| * *' |--------| *' | * * **| 3. Unevenly filled *' |--------| * * Chunk (1) above represents the ideal situation. The chunk is evenly filled * across the entire chunk interval. This indicates a steady stream of data at * an even rate. Given the size and interval of this chunk, it would be * straightforward to calculate a new interval to hit a given target size. * * Chunk (2) has the same amount of data as (1), but it is reasonable to believe * that the following chunk will be fully filled with about twice the amount of * data. It is common for the first chunk in a hypertable to look like * this. Thus, to be able to use the first chunk for prediction, we compensate * by finding the MIN and MAX dimension values of the data in the chunk and then * use max-min (difference) as the interval instead of the chunk's actual * interval (i.e., since we are more interested in data rate/density we pretend * that this is a smaller chunk in terms of the given dimension.) * * Chunk (3) is probably a common real world scenario. We don't do anything * special to handle this case. * * We use a number of thresholds to avoid changing intervals * unnecessarily. I.e., if we are close to the target interval, we avoid * changing the interval since there might be a natural variance in the * fillfactor across chunks. This is intended to avoid flip-flopping or unstable * behavior. * * Additionally, two other thresholds govern much of the algorithm's behavior. * First is the SIZE_FILLFACTOR_THRESH, which is the minimum percentage of * the extrapolated size a chunk should fill to be used in computing a new * target size. We want a minimum so as to not overreact to a chunk that is too * small to get an accurate extrapolation from. For example, a chunk that is * only a percentage point or two of the extrapolated size (or less!) may not * contain enough data to give a true sense of the data rate, i.e., if it was * made in a particularly bursty or slow period. * * However, in the event that an initial chunk size was set * way too small, the algorithm will never adjust because * _all_ the chunks fall below this threshold. Therefore we have another * threshold -- NUM_UNDERSIZED_INTERVALS -- that helps our algorithm make * progress to the correct estimate. If there are _no_ chunks that * meet SIZE_FILLFACTOR_THRESH, and at least NUM_UNDERSIZED_INTERVALS chunks * we are sufficiently full, we use those chunks to adjust the target chunk * size so that the next chunks created at least meet SIZE_FILLFACTOR_THRESH. * This will then allow the algorithm to work in the normal way to adjust * further if needed. */ Datum ts_calculate_chunk_interval(PG_FUNCTION_ARGS) { int32 dimension_id = PG_GETARG_INT32(0); int64 dimension_coord = PG_GETARG_INT64(1); int64 chunk_target_size_bytes = PG_GETARG_INT64(2); int64 chunk_interval = 0; int64 undersized_intervals = 0; int64 current_interval; int32 hypertable_id; Hypertable *ht; const Dimension *dim; List *chunks = NIL; ListCell *lc; int num_intervals = 0; int num_undersized_intervals = 0; double interval_diff; double undersized_fillfactor = 0.0; AclResult acl_result; if (PG_NARGS() != CHUNK_SIZING_FUNC_NARGS) elog(ERROR, "invalid number of arguments"); if (chunk_target_size_bytes < 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("chunk_target_size must be positive"))); elog(DEBUG1, "[adaptive] chunk_target_size_bytes=" UINT64_FORMAT, chunk_target_size_bytes); hypertable_id = ts_dimension_get_hypertable_id(dimension_id); if (hypertable_id <= 0) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("could not find a matching hypertable for dimension %u", dimension_id))); ht = ts_hypertable_get_by_id(hypertable_id); Assert(ht != NULL); acl_result = pg_class_aclcheck(ht->main_table_relid, GetUserId(), ACL_SELECT); if (acl_result != ACLCHECK_OK) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permission denied for table %s", NameStr(ht->fd.table_name)))); dim = ts_hyperspace_get_dimension_by_id(ht->space, dimension_id); Assert(dim != NULL); current_interval = dim->fd.interval_length; /* Get a window of recent chunks */ chunks = ts_chunk_get_window(dimension_id, dimension_coord, DEFAULT_CHUNK_WINDOW, CurrentMemoryContext); foreach (lc, chunks) { Chunk *chunk = lfirst(lc); const DimensionSlice *slice = ts_hypercube_get_slice_by_dimension_id(chunk->cube, dimension_id); int64 chunk_size, slice_interval; Datum minmax[2]; AttrNumber attno = ts_map_attno(ht->main_table_relid, chunk->table_id, dim->column_attno); Assert(NULL != slice); chunk_size = DatumGetInt64( DirectFunctionCall1(pg_total_relation_size, ObjectIdGetDatum(chunk->table_id))); slice_interval = slice->fd.range_end - slice->fd.range_start; if (chunk_get_minmax(chunk->table_id, dim->fd.column_type, attno, "adaptive chunking", minmax)) { int64 min = ts_time_value_to_internal(minmax[0], dim->fd.column_type); int64 max = ts_time_value_to_internal(minmax[1], dim->fd.column_type); double interval_fillfactor, size_fillfactor; int64 extrapolated_chunk_size; /* * The fillfactor of the slice interval that the data actually * spans */ interval_fillfactor = ((double) max - min) / slice_interval; /* * Extrapolate the size the chunk would have if it spanned the * entire interval */ extrapolated_chunk_size = chunk_size / interval_fillfactor; size_fillfactor = ((double) extrapolated_chunk_size) / chunk_target_size_bytes; elog(DEBUG2, "[adaptive] slice_interval=" UINT64_FORMAT " interval_fillfactor=%lf" " current_chunk_size=" UINT64_FORMAT " extrapolated_chunk_size=" UINT64_FORMAT " size_fillfactor=%lf", slice_interval, interval_fillfactor, chunk_size, extrapolated_chunk_size, size_fillfactor); /* * If the chunk is sufficiently filled with data and its * extrapolated size is large enough to make a good estimate, use * it */ if (interval_fillfactor > INTERVAL_FILLFACTOR_THRESH && size_fillfactor > SIZE_FILLFACTOR_THRESH) { chunk_interval += (slice_interval / size_fillfactor); num_intervals++; } /* * If the chunk is sufficiently filled with data but its * extrapolated size is too small, track it and maybe use it if it * is all we have */ else if (interval_fillfactor > INTERVAL_FILLFACTOR_THRESH) { elog(DEBUG2, "[adaptive] chunk sufficiently full, " "but undersized. may use for prediction."); undersized_intervals += slice_interval; undersized_fillfactor += size_fillfactor; num_undersized_intervals++; } } } elog(DEBUG1, "[adaptive] current interval=" UINT64_FORMAT " num_intervals=%d num_undersized_intervals=%d", current_interval, num_intervals, num_undersized_intervals); /* * No full sized intervals, but enough undersized intervals to adjust * higher. We only want to do this if there are no sufficiently sized * intervals to use for a normal adjustment. This keeps us from getting * stuck with a really small interval size. */ if (num_intervals == 0 && num_undersized_intervals > NUM_UNDERSIZED_INTERVALS) { double avg_fillfactor = undersized_fillfactor / num_undersized_intervals; double incr_factor = UNDERSIZED_FILLFACTOR_THRESH / avg_fillfactor; int64 avg_interval = undersized_intervals / num_undersized_intervals; elog(DEBUG1, "[adaptive] no sufficiently large intervals found, but " "some undersized ones found. increase interval to probe for better" " threshold. factor=%lf", incr_factor); chunk_interval = (int64) (avg_interval * incr_factor); } /* No data & insufficient amount of undersized chunks, keep old interval */ else if (num_intervals == 0) { elog(DEBUG1, "[adaptive] no sufficiently large intervals found, " "nor enough undersized chunks to estimate. " "use previous size of " UINT64_FORMAT, current_interval); PG_RETURN_INT64(current_interval); } else chunk_interval /= num_intervals; /* * If the interval hasn't really changed much from before, we keep the old * interval to ensure we do not have fluctuating behavior around the * target size. */ interval_diff = fabs(1.0 - ((double) chunk_interval / current_interval)); if (interval_diff <= INTERVAL_MIN_CHANGE_THRESH) { elog(DEBUG1, "[adaptive] calculated chunk interval=" UINT64_FORMAT ", but is below change threshold, keeping old interval", chunk_interval); chunk_interval = current_interval; } else { elog(LOG, "[adaptive] calculated chunk interval=" UINT64_FORMAT " for hypertable %d, making change", chunk_interval, hypertable_id); } PG_RETURN_INT64(chunk_interval); } /* * Validate that the provided function in the catalog can be used for * determining a new chunk size, i.e., has form (int,bigint,bigint) -> bigint. * * Parameter 'info' will be updated with the function's information */ void ts_chunk_sizing_func_validate(regproc func, ChunkSizingInfo *info) { HeapTuple tuple; Form_pg_proc form; Oid *typearr; if (!OidIsValid(func)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_FUNCTION), (errmsg("invalid chunk sizing function")))); tuple = SearchSysCache1(PROCOID, ObjectIdGetDatum(func)); if (!HeapTupleIsValid(tuple)) elog(ERROR, "cache lookup failed for function %u", func); form = (Form_pg_proc) GETSTRUCT(tuple); typearr = form->proargtypes.values; if (form->pronargs != CHUNK_SIZING_FUNC_NARGS || typearr[0] != INT4OID || typearr[1] != INT8OID || typearr[2] != INT8OID || form->prorettype != INT8OID) { ReleaseSysCache(tuple); ereport(ERROR, (errcode(ERRCODE_INVALID_FUNCTION_DEFINITION), errmsg("invalid function signature"), errhint("A chunk sizing function's signature should be (int, bigint, bigint) -> " "bigint"))); } if (NULL != info) { info->func = func; namestrcpy(&info->func_schema, get_namespace_name(form->pronamespace)); namestrcpy(&info->func_name, NameStr(form->proname)); } ReleaseSysCache(tuple); } /* * Parse the target size text into an integer amount of bytes. * * 'off' / 'disable' - returns a target of 0 * 'estimate' - returns a target based on number of bytes in shared memory * 'XXMB' / etc - converts from PostgreSQL pretty text into number of bytes */ static int64 chunk_target_size_in_bytes(const text *target_size_text) { const char *target_size = text_to_cstring(target_size_text); int64 target_size_bytes = 0; if (pg_strcasecmp(target_size, "off") == 0 || pg_strcasecmp(target_size, "disable") == 0) return 0; if (pg_strcasecmp(target_size, "estimate") == 0) target_size_bytes = ts_chunk_calculate_initial_chunk_target_size(); else target_size_bytes = convert_text_memory_amount_to_bytes(target_size); /* Disable if target size is zero or less */ if (target_size_bytes <= 0) target_size_bytes = 0; return target_size_bytes; } #define MB (1024 * 1024) void ts_chunk_adaptive_sizing_info_validate(ChunkSizingInfo *info) { AttrNumber attnum; NameData attname; Oid atttype; if (!OidIsValid(info->table_relid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("table does not exist"))); ts_hypertable_permissions_check(info->table_relid, GetUserId()); if (NULL == info->colname) ereport(ERROR, (errcode(ERRCODE_TS_DIMENSION_NOT_EXIST), errmsg("no open dimension found for adaptive chunking"))); attnum = get_attnum(info->table_relid, info->colname); namestrcpy(&attname, info->colname); atttype = get_atttype(info->table_relid, attnum); if (!OidIsValid(atttype)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("column \"%s\" does not exist", info->colname))); ts_chunk_sizing_func_validate(info->func, info); if (NULL == info->target_size) info->target_size_bytes = 0; else info->target_size_bytes = chunk_target_size_in_bytes(info->target_size); /* Don't validate further if disabled */ if (info->target_size_bytes <= 0 || !OidIsValid(info->func)) return; /* Warn of small target sizes */ if (info->target_size_bytes > 0 && info->target_size_bytes < (10 * MB)) elog(WARNING, "target chunk size for adaptive chunking is less than 10 MB"); if (info->check_for_index && !table_has_minmax_index(info->table_relid, atttype, &attname, attnum)) ereport(WARNING, (errmsg("no index on \"%s\" found for adaptive chunking on hypertable \"%s\"", info->colname, get_rel_name(info->table_relid)), errdetail("Adaptive chunking works best with an index on the dimension being " "adapted."))); } TS_FUNCTION_INFO_V1(ts_chunk_adaptive_set); /* * Change the settings for adaptive chunking. */ Datum ts_chunk_adaptive_set(PG_FUNCTION_ARGS) { ChunkSizingInfo info = { .table_relid = PG_GETARG_OID(0), .target_size = PG_ARGISNULL(1) ? NULL : PG_GETARG_TEXT_P(1), .func = PG_ARGISNULL(2) ? InvalidOid : PG_GETARG_OID(2), .colname = NULL, .check_for_index = true, }; Hypertable *ht; const Dimension *dim; Cache *hcache; HeapTuple tuple; TupleDesc tupdesc; Datum values[2]; bool nulls[2] = { false, false }; TS_PREVENT_FUNC_IF_READ_ONLY(); if (PG_ARGISNULL(0)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid hypertable: cannot be NULL"))); if (!OidIsValid(info.table_relid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("table does not exist"))); ts_hypertable_permissions_check(info.table_relid, GetUserId()); ht = ts_hypertable_cache_get_cache_and_entry(info.table_relid, CACHE_FLAG_NONE, &hcache); /* Get the first open dimension that we will adapt on */ dim = ts_hyperspace_get_dimension(ht->space, DIMENSION_TYPE_OPEN, 0); if (NULL == dim) ereport(ERROR, (errcode(ERRCODE_TS_DIMENSION_NOT_EXIST), errmsg("no open dimension found for adaptive chunking"))); info.colname = NameStr(dim->fd.column_name); ts_chunk_adaptive_sizing_info_validate(&info); if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) elog(ERROR, "function returning record called in context that cannot accept type record"); tupdesc = BlessTupleDesc(tupdesc); if (OidIsValid(info.func)) { ht->chunk_sizing_func = info.func; values[0] = ObjectIdGetDatum(info.func); } else if (OidIsValid(ht->chunk_sizing_func)) { ts_chunk_sizing_func_validate(ht->chunk_sizing_func, &info); values[0] = ObjectIdGetDatum(ht->chunk_sizing_func); } else ereport(ERROR, (errcode(ERRCODE_UNDEFINED_FUNCTION), errmsg("invalid chunk sizing function"))); values[1] = Int64GetDatum(info.target_size_bytes); /* Update the hypertable entry */ ht->fd.chunk_target_size = info.target_size_bytes; ts_hypertable_update_chunk_sizing(ht); ts_cache_release(&hcache); tuple = heap_form_tuple(tupdesc, values, nulls); PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); } static Oid get_default_chunk_sizing_fn_oid() { Oid chunkfnargtypes[] = { INT4OID, INT8OID, INT8OID }; List *funcname = list_make2(makeString(FUNCTIONS_SCHEMA_NAME), makeString(DEFAULT_CHUNK_SIZING_FN_NAME)); int nargs = sizeof(chunkfnargtypes) / sizeof(chunkfnargtypes[0]); Oid chunkfnoid = LookupFuncName(funcname, nargs, chunkfnargtypes, false); return chunkfnoid; } ChunkSizingInfo * ts_chunk_sizing_info_get_default_disabled(Oid table_relid) { ChunkSizingInfo *chunk_sizing_info = palloc(sizeof(*chunk_sizing_info)); *chunk_sizing_info = (ChunkSizingInfo){ .table_relid = table_relid, .target_size = NULL, .func = get_default_chunk_sizing_fn_oid(), .colname = NULL, .check_for_index = false, }; return chunk_sizing_info; } ================================================ FILE: src/chunk_adaptive.h ================================================ /* * NOTE: adaptive chunking is still in BETA */ #pragma once #include <postgres.h> #include "export.h" typedef struct ChunkSizingInfo { Oid table_relid; /* Set manually */ Oid func; text *target_size; const char *colname; /* The column of the dimension we are adapting * on */ bool check_for_index; /* Set if we should check for an index on * the dimension we are adapting on */ /* Validated info */ NameData func_name; NameData func_schema; int64 target_size_bytes; } ChunkSizingInfo; extern void ts_chunk_adaptive_sizing_info_validate(ChunkSizingInfo *info); extern void ts_chunk_sizing_func_validate(regproc func, ChunkSizingInfo *info); extern TSDLLEXPORT ChunkSizingInfo *ts_chunk_sizing_info_get_default_disabled(Oid table_relid); extern TSDLLEXPORT int64 ts_chunk_calculate_initial_chunk_target_size(void); ================================================ FILE: src/chunk_constraint.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/heapam.h> #include <access/xact.h> #include <catalog/dependency.h> #include <catalog/heap.h> #include <catalog/indexing.h> #include <catalog/objectaddress.h> #include <catalog/pg_constraint.h> #include <commands/tablecmds.h> #include <funcapi.h> #include <nodes/makefuncs.h> #include <storage/lockdefs.h> #include <utils/builtins.h> #include <utils/hsearch.h> #include <utils/lsyscache.h> #include <utils/palloc.h> #include <utils/rel.h> #include <utils/relcache.h> #include <utils/snapmgr.h> #include <utils/syscache.h> #include "compat/compat.h" #include "chunk.h" #include "chunk_constraint.h" #include "chunk_index.h" #include "constraint.h" #include "debug_assert.h" #include "dimension_slice.h" #include "dimension_vector.h" #include "errors.h" #include "export.h" #include "foreign_key.h" #include "hypercube.h" #include "hypertable.h" #include "partitioning.h" #include "process_utility.h" #include "scan_iterator.h" #include "scanner.h" #define DEFAULT_EXTRA_CONSTRAINTS_SIZE 4 #define CHUNK_CONSTRAINTS_SIZE(num_constraints) (sizeof(ChunkConstraint) * (num_constraints)) ChunkConstraints * ts_chunk_constraints_alloc(int size_hint, MemoryContext mctx) { ChunkConstraints *ccs = MemoryContextAlloc(mctx, sizeof(ChunkConstraints)); ccs->mctx = mctx; ccs->capacity = size_hint + DEFAULT_EXTRA_CONSTRAINTS_SIZE; ccs->num_constraints = 0; ccs->num_dimension_constraints = 0; ccs->constraints = MemoryContextAllocZero(mctx, CHUNK_CONSTRAINTS_SIZE(ccs->capacity)); return ccs; } ChunkConstraints * ts_chunk_constraints_copy(ChunkConstraints *chunk_constraints) { ChunkConstraints *copy = palloc(sizeof(ChunkConstraints)); memcpy(copy, chunk_constraints, sizeof(ChunkConstraints)); copy->constraints = palloc0(CHUNK_CONSTRAINTS_SIZE(chunk_constraints->capacity)); memcpy(copy->constraints, chunk_constraints->constraints, CHUNK_CONSTRAINTS_SIZE(chunk_constraints->num_constraints)); return copy; } static void chunk_constraints_expand(ChunkConstraints *ccs, int16 new_capacity) { MemoryContext old; if (new_capacity <= ccs->capacity) return; old = MemoryContextSwitchTo(ccs->mctx); ccs->capacity = new_capacity; Assert(ccs->constraints); /* repalloc() does not work with NULL argument */ ccs->constraints = repalloc(ccs->constraints, CHUNK_CONSTRAINTS_SIZE(new_capacity)); MemoryContextSwitchTo(old); } static void chunk_constraint_dimension_choose_name(Name dst, int32 dimension_slice_id) { snprintf(NameStr(*dst), NAMEDATALEN, "constraint_%d", dimension_slice_id); } static void chunk_constraint_choose_name(Name dst, const char *hypertable_constraint_name, int32 chunk_id) { char constrname[NAMEDATALEN]; CatalogSecurityContext sec_ctx; Assert(hypertable_constraint_name != NULL); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); snprintf(constrname, NAMEDATALEN, "%d_" INT64_FORMAT "_%s", chunk_id, ts_catalog_table_next_seq_id(ts_catalog_get(), CHUNK_CONSTRAINT), hypertable_constraint_name); ts_catalog_restore_user(&sec_ctx); namestrcpy(dst, constrname); } ChunkConstraint * ts_chunk_constraints_add(ChunkConstraints *ccs, int32 chunk_id, int32 dimension_slice_id, const char *constraint_name, const char *hypertable_constraint_name) { ChunkConstraint *cc; chunk_constraints_expand(ccs, ccs->num_constraints + 1); cc = &ccs->constraints[ccs->num_constraints++]; cc->fd.chunk_id = chunk_id; cc->fd.dimension_slice_id = dimension_slice_id; if (NULL == constraint_name) { if (is_dimension_constraint(cc)) { chunk_constraint_dimension_choose_name(&cc->fd.constraint_name, cc->fd.dimension_slice_id); namestrcpy(&cc->fd.hypertable_constraint_name, ""); } else chunk_constraint_choose_name(&cc->fd.constraint_name, hypertable_constraint_name, cc->fd.chunk_id); } else namestrcpy(&cc->fd.constraint_name, constraint_name); if (NULL != hypertable_constraint_name) namestrcpy(&cc->fd.hypertable_constraint_name, hypertable_constraint_name); if (is_dimension_constraint(cc)) ccs->num_dimension_constraints++; return cc; } static void chunk_constraint_fill_tuple_values(const ChunkConstraint *cc, Datum values[Natts_chunk_constraint], bool nulls[Natts_chunk_constraint]) { memset(values, 0, sizeof(Datum) * Natts_chunk_constraint); values[AttrNumberGetAttrOffset(Anum_chunk_constraint_chunk_id)] = Int32GetDatum(cc->fd.chunk_id); values[AttrNumberGetAttrOffset(Anum_chunk_constraint_dimension_slice_id)] = Int32GetDatum(cc->fd.dimension_slice_id); values[AttrNumberGetAttrOffset(Anum_chunk_constraint_constraint_name)] = NameGetDatum(&cc->fd.constraint_name); values[AttrNumberGetAttrOffset(Anum_chunk_constraint_hypertable_constraint_name)] = NameGetDatum(&cc->fd.hypertable_constraint_name); if (is_dimension_constraint(cc)) nulls[AttrNumberGetAttrOffset(Anum_chunk_constraint_hypertable_constraint_name)] = true; else nulls[AttrNumberGetAttrOffset(Anum_chunk_constraint_dimension_slice_id)] = true; } static void chunk_constraint_insert_relation(const Relation rel, const ChunkConstraint *cc) { TupleDesc desc = RelationGetDescr(rel); Datum values[Natts_chunk_constraint]; bool nulls[Natts_chunk_constraint] = { false }; chunk_constraint_fill_tuple_values(cc, values, nulls); ts_catalog_insert_values(rel, desc, values, nulls); } /* * Insert multiple chunk constraints into the metadata catalog. */ void ts_chunk_constraints_insert_metadata(const ChunkConstraints *ccs) { Catalog *catalog = ts_catalog_get(); CatalogSecurityContext sec_ctx; Relation rel; int i; rel = table_open(catalog_get_table_id(catalog, CHUNK_CONSTRAINT), RowExclusiveLock); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); for (i = 0; i < ccs->num_constraints; i++) chunk_constraint_insert_relation(rel, &ccs->constraints[i]); ts_catalog_restore_user(&sec_ctx); table_close(rel, RowExclusiveLock); } /* * Insert a single chunk constraints into the metadata catalog. */ void ts_chunk_constraint_insert(ChunkConstraint *constraint) { Catalog *catalog = ts_catalog_get(); CatalogSecurityContext sec_ctx; Relation rel; rel = table_open(catalog_get_table_id(catalog, CHUNK_CONSTRAINT), RowExclusiveLock); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); chunk_constraint_insert_relation(rel, constraint); ts_catalog_restore_user(&sec_ctx); table_close(rel, RowExclusiveLock); } ChunkConstraint * ts_chunk_constraints_add_from_tuple(ChunkConstraints *ccs, const TupleInfo *ti) { bool nulls[Natts_chunk_constraint]; Datum values[Natts_chunk_constraint]; ChunkConstraint *constraints; int32 dimension_slice_id; Name constraint_name; Name hypertable_constraint_name; bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); MemoryContext oldcxt; heap_deform_tuple(tuple, ts_scanner_get_tupledesc(ti), values, nulls); oldcxt = MemoryContextSwitchTo(ccs->mctx); constraint_name = DatumGetName(values[AttrNumberGetAttrOffset(Anum_chunk_constraint_constraint_name)]); if (nulls[AttrNumberGetAttrOffset(Anum_chunk_constraint_dimension_slice_id)]) { dimension_slice_id = 0; hypertable_constraint_name = DatumGetName( values[AttrNumberGetAttrOffset(Anum_chunk_constraint_hypertable_constraint_name)]); } else { dimension_slice_id = DatumGetInt32( values[AttrNumberGetAttrOffset(Anum_chunk_constraint_dimension_slice_id)]); hypertable_constraint_name = DatumGetName(DirectFunctionCall1(namein, CStringGetDatum(""))); } constraints = ts_chunk_constraints_add(ccs, DatumGetInt32(values[AttrNumberGetAttrOffset( Anum_chunk_constraint_chunk_id)]), dimension_slice_id, NameStr(*constraint_name), NameStr(*hypertable_constraint_name)); MemoryContextSwitchTo(oldcxt); if (should_free) heap_freetuple(tuple); return constraints; } /* * Create a dimensional CHECK constraint for a partitioning dimension. */ Constraint * ts_chunk_constraint_dimensional_create(const Dimension *dim, const DimensionSlice *slice, const char *name) { Constraint *constr = NULL; Node *dimdef; ColumnRef *colref; List *compexprs = NIL; Oid type; if (slice->fd.range_start == PG_INT64_MIN && slice->fd.range_end == PG_INT64_MAX) return NULL; colref = makeNode(ColumnRef); colref->fields = list_make1(makeString(pstrdup(NameStr(dim->fd.column_name)))); colref->location = -1; /* Convert the dimensional ranges to the appropriate text/string * representation for the time type. For dimensions with a * partitioning/time function, use the function's output type. */ if (dim->partitioning != NULL) { /* Both open and closed dimensions can have a partitioning function */ PartitioningInfo *partinfo = dim->partitioning; List *funcname = list_make2(makeString(NameStr(partinfo->partfunc.schema)), makeString(NameStr(partinfo->partfunc.name))); dimdef = (Node *) makeFuncCall(funcname, list_make1(colref), COERCE_EXPLICIT_CALL, -1); if (IS_OPEN_DIMENSION(dim)) { /* The dimension has a time function to compute the time value so * need to convert the range values to the time type returned by * the partitioning function. */ type = partinfo->partfunc.rettype; } else { /* Closed dimension, just use the INT8 type */ type = INT8OID; } } else { /* Must be open dimension, since no partitioning function */ Assert(IS_OPEN_DIMENSION(dim)); dimdef = (Node *) colref; type = dim->fd.column_type; } /* * We are forcing ISO datestyle here to prevent parsing errors with * certain timezone/datestyle combinations. */ int current_datestyle = DateStyle; DateStyle = USE_ISO_DATES; char *start_str = ts_internal_to_time_string(slice->fd.range_start, type); char *end_str = ts_internal_to_time_string(slice->fd.range_end, type); DateStyle = current_datestyle; /* Elide range constraint for +INF or -INF */ if (slice->fd.range_start != PG_INT64_MIN) { A_Const *start_const = makeNode(A_Const); memcpy(&start_const->val, makeString(start_str), sizeof(start_const->val)); start_const->location = -1; A_Expr *ge_expr = makeSimpleA_Expr(AEXPR_OP, ">=", dimdef, (Node *) start_const, -1); compexprs = lappend(compexprs, ge_expr); } if (slice->fd.range_end != PG_INT64_MAX) { A_Const *end_const = makeNode(A_Const); memcpy(&end_const->val, makeString(end_str), sizeof(end_const->val)); end_const->location = -1; A_Expr *lt_expr = makeSimpleA_Expr(AEXPR_OP, "<", dimdef, (Node *) end_const, -1); compexprs = lappend(compexprs, lt_expr); } constr = makeNode(Constraint); constr->contype = CONSTR_CHECK; constr->conname = name ? pstrdup(name) : NULL; constr->deferrable = false; constr->skip_validation = true; constr->initially_valid = true; #if PG18_GE constr->is_enforced = true; #endif Assert(list_length(compexprs) >= 1); if (list_length(compexprs) == 2) constr->raw_expr = (Node *) makeBoolExpr(AND_EXPR, compexprs, -1); else if (list_length(compexprs) == 1) constr->raw_expr = linitial(compexprs); return constr; } /* * Add a constraint to a chunk table. */ static Oid chunk_constraint_create_on_table(const ChunkConstraint *cc, Oid chunk_oid) { HeapTuple tuple; Datum values[Natts_chunk_constraint]; bool nulls[Natts_chunk_constraint] = { false }; CatalogSecurityContext sec_ctx; Relation rel; chunk_constraint_fill_tuple_values(cc, values, nulls); rel = RelationIdGetRelation(catalog_get_table_id(ts_catalog_get(), CHUNK_CONSTRAINT)); tuple = heap_form_tuple(RelationGetDescr(rel), values, nulls); RelationClose(rel); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); CatalogInternalCall1(DDL_ADD_CHUNK_CONSTRAINT, HeapTupleGetDatum(tuple)); ts_catalog_restore_user(&sec_ctx); heap_freetuple(tuple); return get_relation_constraint_oid(chunk_oid, NameStr(cc->fd.constraint_name), true); } /* * Create a non-dimensional constraint on a chunk table (foreign key, trigger * constraint, etc.), including adding relevant metadata to the catalog. */ static Oid create_non_dimensional_constraint(const ChunkConstraint *cc, Oid chunk_oid, int32 chunk_id, Oid hypertable_oid, int32 hypertable_id) { Oid chunk_constraint_oid; Assert(!is_dimension_constraint(cc)); /* * If we're creating constraints for a new chunk from an existing * table or attaching a chunk to an existing hypertable, we might * have constraints already created. If so, skip creating * such constraints, we only needed their metadata to be added. */ if (ConstraintNameIsUsed(CONSTRAINT_RELATION, chunk_oid, NameStr(cc->fd.constraint_name))) { chunk_constraint_oid = get_relation_constraint_oid(chunk_oid, NameStr(cc->fd.constraint_name), true); } else { ts_process_utility_set_expect_chunk_modification(true); chunk_constraint_oid = chunk_constraint_create_on_table(cc, chunk_oid); ts_process_utility_set_expect_chunk_modification(false); } return chunk_constraint_oid; } static const DimensionSlice * get_slice_with_id(const Hypercube *cube, int32 id) { int i; for (i = 0; i < cube->num_slices; i++) { const DimensionSlice *slice = cube->slices[i]; if (slice->fd.id == id) return slice; } return NULL; } /* * Create a set of constraints on a chunk table. */ void ts_chunk_constraints_create(const Hypertable *ht, const Chunk *chunk) { const ChunkConstraints *ccs = chunk->constraints; List *newconstrs = NIL; int i; for (i = 0; i < ccs->num_constraints; i++) { const ChunkConstraint *cc = &ccs->constraints[i]; if (is_dimension_constraint(cc)) { const DimensionSlice *slice = get_slice_with_id(chunk->cube, cc->fd.dimension_slice_id); const Dimension *dim; Constraint *constr; dim = ts_hyperspace_get_dimension_by_id(ht->space, slice->fd.dimension_id); Assert(dim); constr = ts_chunk_constraint_dimensional_create(dim, slice, NameStr(cc->fd.constraint_name)); /* In some cases, a CHECK constraint is not needed. For instance, * if the range is -INF to +INF. */ if (constr != NULL) newconstrs = lappend(newconstrs, constr); } else { create_non_dimensional_constraint(cc, chunk->table_id, chunk->fd.id, ht->main_table_relid, ht->fd.id); } } if (newconstrs != NIL) { List PG_USED_FOR_ASSERTS_ONLY *cookedconstrs = NIL; Relation rel = table_open(chunk->table_id, AccessExclusiveLock); cookedconstrs = AddRelationNewConstraints(rel, NIL /* List *newColDefaults */, newconstrs, false /* allow_merge */, true /* is_local */, false /* is_internal */, NULL /* query string */); table_close(rel, NoLock); Assert(list_length(cookedconstrs) == list_length(newconstrs)); CommandCounterIncrement(); } } ScanIterator ts_chunk_constraint_scan_iterator_create(MemoryContext result_mcxt) { ScanIterator it = ts_scan_iterator_create(CHUNK_CONSTRAINT, AccessShareLock, result_mcxt); it.ctx.flags |= SCANNER_F_NOEND_AND_NOCLOSE; return it; } void ts_chunk_constraint_scan_iterator_set_slice_id(ScanIterator *it, int32 slice_id) { it->ctx.index = catalog_get_index(ts_catalog_get(), CHUNK_CONSTRAINT, CHUNK_CONSTRAINT_DIMENSION_SLICE_ID_IDX); ts_scan_iterator_scan_key_reset(it); ts_scan_iterator_scan_key_init(it, Anum_chunk_constraint_dimension_slice_id_idx_dimension_slice_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(slice_id)); } void ts_chunk_constraint_scan_iterator_set_chunk_id(ScanIterator *it, int32 chunk_id) { it->ctx.index = catalog_get_index(ts_catalog_get(), CHUNK_CONSTRAINT, CHUNK_CONSTRAINT_CHUNK_ID_CONSTRAINT_NAME_IDX); ts_scan_iterator_scan_key_reset(it); ts_scan_iterator_scan_key_init(it, Anum_chunk_constraint_chunk_id_constraint_name_idx_chunk_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(chunk_id)); } static void init_scan_by_chunk_id_constraint_name(ScanIterator *iterator, int32 chunk_id, const char *constraint_name) { iterator->ctx.index = catalog_get_index(ts_catalog_get(), CHUNK_CONSTRAINT, CHUNK_CONSTRAINT_CHUNK_ID_CONSTRAINT_NAME_IDX); ts_scan_iterator_scan_key_reset(iterator); ts_scan_iterator_scan_key_init(iterator, Anum_chunk_constraint_chunk_id_constraint_name_idx_chunk_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(chunk_id)); ts_scan_iterator_scan_key_init( iterator, Anum_chunk_constraint_chunk_id_constraint_name_idx_constraint_name, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(constraint_name)); } /* * Scan all the chunk's constraints given its chunk ID. * * Returns a set of chunk constraints. */ ChunkConstraints * ts_chunk_constraint_scan_by_chunk_id(int32 chunk_id, Size num_constraints_hint, MemoryContext mctx) { ChunkConstraints *constraints = ts_chunk_constraints_alloc(num_constraints_hint, mctx); ScanIterator iterator = ts_scan_iterator_create(CHUNK_CONSTRAINT, AccessShareLock, mctx); int num_found = 0; ts_chunk_constraint_scan_iterator_set_chunk_id(&iterator, chunk_id); ts_scanner_foreach(&iterator) { num_found++; ts_chunk_constraints_add_from_tuple(constraints, ts_scan_iterator_tuple_info(&iterator)); } if (num_found != constraints->num_constraints) elog(ERROR, "unexpected number of constraints found for chunk ID %d", chunk_id); return constraints; } /* * Scan for all chunk constraints that match the given slice ID. The chunk * constraints are saved in the chunk scan context. */ int ts_chunk_constraint_scan_by_dimension_slice(const DimensionSlice *slice, ChunkScanCtx *ctx, MemoryContext mctx) { ScanIterator iterator = ts_scan_iterator_create(CHUNK_CONSTRAINT, AccessShareLock, mctx); int count = 0; ts_chunk_constraint_scan_iterator_set_slice_id(&iterator, slice->fd.id); ts_scanner_foreach(&iterator) { const Hyperspace *hs = ctx->ht->space; ChunkStub *stub; ChunkScanEntry *entry; bool found; TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); Datum datum = slot_getattr(ti->slot, Anum_chunk_constraint_chunk_id, &found); int32 chunk_id = DatumGetInt32(datum); if (slot_attisnull(ts_scan_iterator_slot(&iterator), Anum_chunk_constraint_dimension_slice_id)) continue; count++; Assert(!slot_attisnull(ti->slot, Anum_chunk_constraint_dimension_slice_id)); entry = hash_search(ctx->htab, &chunk_id, HASH_ENTER, &found); if (!found) { stub = ts_chunk_stub_create(chunk_id, hs->num_dimensions); stub->cube = ts_hypercube_alloc(hs->num_dimensions); entry->stub = stub; } else stub = entry->stub; ts_chunk_constraints_add_from_tuple(stub->constraints, ti); ts_hypercube_add_slice(stub->cube, slice); /* A stub is complete when we've added slices for all its dimensions, * i.e., a complete hypercube */ if (chunk_stub_is_complete(stub, ctx->ht->space)) { ctx->num_complete_chunks++; if (ctx->early_abort) { ts_scan_iterator_close(&iterator); break; } } } return count; } /* * Similar to chunk_constraint_scan_by_dimension_slice, but stores only chunk_ids * in a list, which is easier to traverse and provides deterministic chunk selection. */ int ts_chunk_constraint_scan_by_dimension_slice_to_list(const DimensionSlice *slice, List **list, MemoryContext mctx) { ScanIterator iterator = ts_scan_iterator_create(CHUNK_CONSTRAINT, AccessShareLock, mctx); int count = 0; ts_chunk_constraint_scan_iterator_set_slice_id(&iterator, slice->fd.id); ts_scanner_foreach(&iterator) { bool is_null; TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); Datum chunk_id; if (slot_attisnull(ti->slot, Anum_chunk_constraint_dimension_slice_id)) continue; count++; chunk_id = slot_getattr(ti->slot, Anum_chunk_constraint_chunk_id, &is_null); Assert(!is_null); *list = lappend_int(*list, DatumGetInt32(chunk_id)); } return count; } /* * Scan for chunk constraints given a dimension slice ID. * * Optionally, collect all chunk constraints if ChunkConstraints is non-NULL. */ int ts_chunk_constraint_scan_by_dimension_slice_id(int32 dimension_slice_id, ChunkConstraints *ccs, MemoryContext mctx) { ScanIterator iterator = ts_scan_iterator_create(CHUNK_CONSTRAINT, AccessShareLock, mctx); int count = 0; ts_chunk_constraint_scan_iterator_set_slice_id(&iterator, dimension_slice_id); ts_scanner_foreach(&iterator) { if (slot_attisnull(ts_scan_iterator_slot(&iterator), Anum_chunk_constraint_dimension_slice_id)) continue; count++; if (ccs != NULL) ts_chunk_constraints_add_from_tuple(ccs, ts_scan_iterator_tuple_info(&iterator)); } return count; } static bool chunk_constraint_need_on_chunk(Form_pg_constraint conform) { if (conform->contype == CONSTRAINT_CHECK #if PG18_GE /* Avoid NOT NULL constraints * https://github.com/postgres/postgres/commit/b0e96f31 */ || conform->contype == CONSTRAINT_NOTNULL #endif ) { /* * check and not null constraints handled by regular inheritance (from * docs): All check constraints and not-null constraints on a parent * table are automatically inherited by its children, unless * explicitly specified otherwise with NO INHERIT clauses. Other types * of constraints (unique, primary key, and foreign key constraints) * are not inherited." */ return false; } /* Check if the foreign key constraint references a partition in a partitioned table. In that case, we shouldn't include this constraint as we will end up checking the foreign key constraint once for every partition, which obviously leads to foreign key constraint violation. Instead, we only include constraints referencing the parent table of the partitioned table. */ if (conform->contype == CONSTRAINT_FOREIGN && OidIsValid(conform->conparentid)) return false; return true; } int ts_chunk_constraints_add_dimension_constraints(ChunkConstraints *ccs, int32 chunk_id, const Hypercube *cube) { int i; for (i = 0; i < cube->num_slices; i++) ts_chunk_constraints_add(ccs, chunk_id, cube->slices[i]->fd.id, NULL, NULL); return cube->num_slices; } typedef struct ConstraintContext { int num_added; char chunk_relkind; ChunkConstraints *ccs; int32 chunk_id; Oid chunk_relid; } ConstraintContext; static ConstraintProcessStatus chunk_constraint_add(HeapTuple constraint_tuple, void *arg) { ConstraintContext *cc = arg; Form_pg_constraint constraint = (Form_pg_constraint) GETSTRUCT(constraint_tuple); if (cc->chunk_relkind != RELKIND_FOREIGN_TABLE && chunk_constraint_need_on_chunk(constraint)) { /* If the chunk already has an equivalent constraint, use the existing one. */ Relation chunk = table_open(cc->chunk_relid, AccessShareLock); Form_pg_constraint matching_const = ts_constraint_find_matching(constraint_tuple, chunk); table_close(chunk, NoLock); if (matching_const != NULL) ts_chunk_constraints_add(cc->ccs, cc->chunk_id, 0, NameStr(matching_const->conname), NameStr(constraint->conname)); else ts_chunk_constraints_add(cc->ccs, cc->chunk_id, 0, NULL, NameStr(constraint->conname)); return CONSTR_PROCESSED; } return CONSTR_IGNORED; } int ts_chunk_constraints_add_inheritable_constraints(ChunkConstraints *ccs, int32 chunk_id, const char chunk_relkind, Oid hypertable_oid, Oid table_id) { /* This should never be called with NULL ccs. */ Ensure(ccs, "ccs must not be NULL"); ConstraintContext cc = { .chunk_relkind = chunk_relkind, .ccs = ccs, .chunk_id = chunk_id, .chunk_relid = table_id, }; return ts_constraint_process(hypertable_oid, chunk_constraint_add, &cc); } /* check constraints have the same name as the one on the hypertable */ static ConstraintProcessStatus chunk_constraint_add_check(HeapTuple constraint_tuple, void *arg) { ConstraintContext *cc = arg; Form_pg_constraint constraint = (Form_pg_constraint) GETSTRUCT(constraint_tuple); if (constraint->contype == CONSTRAINT_CHECK) { ts_chunk_constraints_add(cc->ccs, cc->chunk_id, 0, NameStr(constraint->conname), NameStr(constraint->conname)); return CONSTR_PROCESSED; } return CONSTR_IGNORED; } /* Adds only inheritable check constraints */ int ts_chunk_constraints_add_inheritable_check_constraints(ChunkConstraints *ccs, int32 chunk_id, const char chunk_relkind, Oid hypertable_oid) { ConstraintContext cc = { .chunk_relkind = chunk_relkind, .ccs = ccs, .chunk_id = chunk_id, }; return ts_constraint_process(hypertable_oid, chunk_constraint_add_check, &cc); } void ts_chunk_constraint_create_on_chunk(const Hypertable *ht, const Chunk *chunk, Oid constraint_oid) { HeapTuple tuple; Form_pg_constraint con; tuple = SearchSysCache1(CONSTROID, ObjectIdGetDatum(constraint_oid)); if (!HeapTupleIsValid(tuple)) elog(ERROR, "cache lookup failed for constraint %u", constraint_oid); con = (Form_pg_constraint) GETSTRUCT(tuple); if (chunk->relkind != RELKIND_FOREIGN_TABLE && chunk_constraint_need_on_chunk(con)) { ChunkConstraint *cc = ts_chunk_constraints_add(chunk->constraints, chunk->fd.id, 0, NULL, NameStr(con->conname)); ts_chunk_constraint_insert(cc); create_non_dimensional_constraint(cc, chunk->table_id, chunk->fd.id, ht->main_table_relid, ht->fd.id); } ReleaseSysCache(tuple); } static bool hypertable_constraint_matches_tuple(TupleInfo *ti, const char *hypertable_constraint_name) { bool isnull; Datum name = slot_getattr(ti->slot, Anum_chunk_constraint_hypertable_constraint_name, &isnull); return !isnull && namestrcmp(DatumGetName(name), hypertable_constraint_name) == 0; } static void chunk_constraint_drop_constraint(TupleInfo *ti) { bool isnull; Datum constrname = slot_getattr(ti->slot, Anum_chunk_constraint_constraint_name, &isnull); int32 chunk_id = DatumGetInt32(slot_getattr(ti->slot, Anum_chunk_constraint_chunk_id, &isnull)); /* Get the chunk relid. Note that, at this point, the chunk table can be * deleted already. */ Oid chunk_relid = ts_chunk_get_relid(chunk_id, true); if (OidIsValid(chunk_relid)) { ObjectAddress constrobj = { .classId = ConstraintRelationId, .objectId = get_relation_constraint_oid(chunk_relid, NameStr(*DatumGetName(constrname)), true), }; if (OidIsValid(constrobj.objectId)) /* must use DROP_CASCADE if regular table references a hypertable */ performDeletion(&constrobj, DROP_CASCADE, 0); } } int ts_chunk_constraint_delete_by_hypertable_constraint_name(int32 chunk_id, const char *hypertable_constraint_name) { ScanIterator iterator = ts_scan_iterator_create(CHUNK_CONSTRAINT, RowExclusiveLock, CurrentMemoryContext); int count = 0; ts_chunk_constraint_scan_iterator_set_chunk_id(&iterator, chunk_id); ts_scanner_foreach(&iterator) { if (!hypertable_constraint_matches_tuple(ts_scan_iterator_tuple_info(&iterator), hypertable_constraint_name)) continue; count++; TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); chunk_constraint_drop_constraint(ti); } return count; } int ts_chunk_constraint_delete_by_constraint_name(int32 chunk_id, const char *constraint_name) { ScanIterator iterator = ts_scan_iterator_create(CHUNK_CONSTRAINT, RowExclusiveLock, CurrentMemoryContext); int count = 0; init_scan_by_chunk_id_constraint_name(&iterator, chunk_id, constraint_name); ts_scanner_foreach(&iterator) { count++; TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); } return count; } /* * Delete all constraints for a chunk. Optionally, collect the deleted constraints. */ int ts_chunk_constraint_delete_by_chunk_id(int32 chunk_id, ChunkConstraints *ccs, bool drop_constraint) { ScanIterator iterator = ts_scan_iterator_create(CHUNK_CONSTRAINT, RowExclusiveLock, CurrentMemoryContext); int count = 0; ts_chunk_constraint_scan_iterator_set_chunk_id(&iterator, chunk_id); ts_scanner_foreach(&iterator) { count++; ts_chunk_constraints_add_from_tuple(ccs, ts_scan_iterator_tuple_info(&iterator)); TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); if (drop_constraint) chunk_constraint_drop_constraint(ts_scan_iterator_tuple_info(&iterator)); } return count; } int ts_chunk_constraint_delete_by_dimension_slice_id(int32 dimension_slice_id) { ScanIterator iterator = ts_scan_iterator_create(CHUNK_CONSTRAINT, RowExclusiveLock, CurrentMemoryContext); int count = 0; ts_chunk_constraint_scan_iterator_set_slice_id(&iterator, dimension_slice_id); ts_scanner_foreach(&iterator) { count++; TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); chunk_constraint_drop_constraint(ti); } return count; } void ts_chunk_constraints_recreate(const Hypertable *ht, const Chunk *chunk) { const ChunkConstraints *ccs = chunk->constraints; int i; for (i = 0; i < ccs->num_constraints; i++) { const ChunkConstraint *cc = &ccs->constraints[i]; ObjectAddress constrobj = { .classId = ConstraintRelationId, .objectId = get_relation_constraint_oid(chunk->table_id, NameStr(cc->fd.constraint_name), false), }; performDeletion(&constrobj, DROP_RESTRICT, 0); } ts_chunk_constraints_create(ht, chunk); ts_chunk_copy_referencing_fk(ht, chunk); } static void chunk_constraint_rename_on_chunk_table(int32 chunk_id, const char *old_name, const char *new_name) { Oid chunk_relid = ts_chunk_get_relid(chunk_id, false); Oid nspid = get_rel_namespace(chunk_relid); RenameStmt rename = { .renameType = OBJECT_TABCONSTRAINT, .relation = makeRangeVar(get_namespace_name(nspid), get_rel_name(chunk_relid), 0), .subname = pstrdup(old_name), .newname = pstrdup(new_name), }; RenameConstraint(&rename); } static void chunk_constraint_rename_hypertable_from_tuple(TupleInfo *ti, const char *new_name) { bool nulls[Natts_chunk_constraint]; Datum values[Natts_chunk_constraint]; bool doReplace[Natts_chunk_constraint] = { false }; HeapTuple tuple, new_tuple; TupleDesc tupdesc = ts_scanner_get_tupledesc(ti); NameData new_hypertable_constraint_name; NameData new_chunk_constraint_name; Name old_chunk_constraint_name; int32 chunk_id; bool should_free; tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); heap_deform_tuple(tuple, tupdesc, values, nulls); chunk_id = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_chunk_constraint_chunk_id)]); namestrcpy(&new_hypertable_constraint_name, new_name); chunk_constraint_choose_name(&new_chunk_constraint_name, new_name, chunk_id); values[AttrNumberGetAttrOffset(Anum_chunk_constraint_hypertable_constraint_name)] = NameGetDatum(&new_hypertable_constraint_name); doReplace[AttrNumberGetAttrOffset(Anum_chunk_constraint_hypertable_constraint_name)] = true; old_chunk_constraint_name = DatumGetName(values[AttrNumberGetAttrOffset(Anum_chunk_constraint_constraint_name)]); values[AttrNumberGetAttrOffset(Anum_chunk_constraint_constraint_name)] = NameGetDatum(&new_chunk_constraint_name); doReplace[AttrNumberGetAttrOffset(Anum_chunk_constraint_constraint_name)] = true; chunk_constraint_rename_on_chunk_table(chunk_id, NameStr(*old_chunk_constraint_name), NameStr(new_chunk_constraint_name)); new_tuple = heap_modify_tuple(tuple, tupdesc, values, nulls, doReplace); ts_catalog_update(ti->scanrel, new_tuple); heap_freetuple(new_tuple); if (should_free) heap_freetuple(tuple); } /* * Adjust internal metadata after index/constraint rename */ int ts_chunk_constraint_adjust_meta(int32 chunk_id, const char *ht_name, const char *chunk_old_name, const char *chunk_new_name) { ScanIterator iterator = ts_scan_iterator_create(CHUNK_CONSTRAINT, RowExclusiveLock, CurrentMemoryContext); int count = 0; init_scan_by_chunk_id_constraint_name(&iterator, chunk_id, chunk_old_name); ts_scanner_foreach(&iterator) { bool nulls[Natts_chunk_constraint]; bool doReplace[Natts_chunk_constraint] = { false }; Datum values[Natts_chunk_constraint]; bool should_free; TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); HeapTuple new_tuple; heap_deform_tuple(tuple, ts_scanner_get_tupledesc(ti), values, nulls); /* * The constraint names are of Postgres type 'name' which is fixed-width * 64-byte type. The input strings might not have the necessary padding * after them. */ NameData ht_constraint_namedata; namestrcpy(&ht_constraint_namedata, ht_name); NameData new_namedata; namestrcpy(&new_namedata, chunk_new_name); values[AttrNumberGetAttrOffset(Anum_chunk_constraint_hypertable_constraint_name)] = NameGetDatum(&ht_constraint_namedata); doReplace[AttrNumberGetAttrOffset(Anum_chunk_constraint_hypertable_constraint_name)] = true; values[AttrNumberGetAttrOffset(Anum_chunk_constraint_constraint_name)] = NameGetDatum(&new_namedata); doReplace[AttrNumberGetAttrOffset(Anum_chunk_constraint_constraint_name)] = true; new_tuple = heap_modify_tuple(tuple, ts_scanner_get_tupledesc(ti), values, nulls, doReplace); ts_catalog_update(ti->scanrel, new_tuple); heap_freetuple(new_tuple); if (should_free) heap_freetuple(tuple); count++; } return count; } bool ts_chunk_constraint_update_slice_id(int32 chunk_id, int32 old_slice_id, int32 new_slice_id) { ScanIterator iterator = ts_scan_iterator_create(CHUNK_CONSTRAINT, RowExclusiveLock, CurrentMemoryContext); ts_chunk_constraint_scan_iterator_set_slice_id(&iterator, old_slice_id); ts_scanner_foreach(&iterator) { bool replIsnull[Natts_chunk_constraint]; bool repl[Natts_chunk_constraint] = { false }; Datum values[Natts_chunk_constraint]; bool should_free, isnull; TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); int32 current_chunk_id = DatumGetInt32(slot_getattr(ti->slot, Anum_chunk_constraint_chunk_id, &isnull)); if (isnull || current_chunk_id != chunk_id) continue; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); HeapTuple new_tuple; heap_deform_tuple(tuple, ts_scanner_get_tupledesc(ti), values, replIsnull); values[AttrNumberGetAttrOffset(Anum_chunk_constraint_dimension_slice_id)] = Int32GetDatum(new_slice_id); repl[AttrNumberGetAttrOffset(Anum_chunk_constraint_dimension_slice_id)] = true; new_tuple = heap_modify_tuple(tuple, ts_scanner_get_tupledesc(ti), values, replIsnull, repl); ts_catalog_update(ti->scanrel, new_tuple); heap_freetuple(new_tuple); if (should_free) heap_freetuple(tuple); ts_scan_iterator_close(&iterator); return true; } return false; } int ts_chunk_constraint_rename_hypertable_constraint(int32 chunk_id, const char *old_name, const char *new_name) { ScanIterator iterator = ts_scan_iterator_create(CHUNK_CONSTRAINT, RowExclusiveLock, CurrentMemoryContext); int count = 0; ts_chunk_constraint_scan_iterator_set_chunk_id(&iterator, chunk_id); ts_scanner_foreach(&iterator) { if (!hypertable_constraint_matches_tuple(ts_scan_iterator_tuple_info(&iterator), old_name)) continue; count++; chunk_constraint_rename_hypertable_from_tuple(ts_scan_iterator_tuple_info(&iterator), new_name); } return count; } char * ts_chunk_constraint_get_name_from_hypertable_constraint(Oid chunk_relid, const char *hypertable_constraint_name) { ScanIterator iterator = ts_scan_iterator_create(CHUNK_CONSTRAINT, RowExclusiveLock, CurrentMemoryContext); Datum chunk_id = DirectFunctionCall1(ts_chunk_id_from_relid, ObjectIdGetDatum(chunk_relid)); ts_chunk_constraint_scan_iterator_set_chunk_id(&iterator, DatumGetInt32(chunk_id)); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); MemoryContext oldmctx; bool isnull; Datum datum; char *name; if (!hypertable_constraint_matches_tuple(ti, hypertable_constraint_name)) continue; datum = slot_getattr(ti->slot, Anum_chunk_constraint_constraint_name, &isnull); Assert(!isnull); oldmctx = MemoryContextSwitchTo(ti->mctx); name = pstrdup(NameStr(*DatumGetName(datum))); MemoryContextSwitchTo(oldmctx); ts_scan_iterator_close(&iterator); return name; } return NULL; } int ts_chunk_constraint_delete_dimensional_constraints(int32 chunk_id, ChunkConstraints *ccs) { ScanIterator iterator = ts_scan_iterator_create(CHUNK_CONSTRAINT, RowExclusiveLock, CurrentMemoryContext); int count = 0; ts_chunk_constraint_scan_iterator_set_chunk_id(&iterator, chunk_id); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); bool isnull; int32 slice_id = DatumGetInt32( slot_getattr(ti->slot, Anum_chunk_constraint_dimension_slice_id, &isnull)); if (isnull || slice_id == 0) continue; count++; ts_chunk_constraints_add_from_tuple(ccs, ti); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); chunk_constraint_drop_constraint(ti); } return count; } /* * Drop a constraint using a pg_constraint heap tuple. */ void ts_chunk_constraint_drop_from_tuple(HeapTuple constraint_tuple) { FormData_pg_constraint *constr = (FormData_pg_constraint *) GETSTRUCT(constraint_tuple); ObjectAddress constrobj = { .classId = ConstraintRelationId, .objectId = constr->oid, }; if (OidIsValid(constr->conparentid)) { deleteDependencyRecordsForClass(constrobj.classId, constrobj.objectId, ConstraintRelationId, DEPENDENCY_INTERNAL); CommandCounterIncrement(); } if (OidIsValid(constrobj.objectId)) performDeletion(&constrobj, DROP_RESTRICT, 0); } static void check_chunk_constraint_violated(Oid chunk_relid, const Dimension *dim, const DimensionSlice *slice) { Relation rel; TupleTableSlot *slot; TableScanDesc scandesc; bool isnull; int attno = get_attnum(chunk_relid, NameStr(dim->fd.column_name)); Ensure(attno != InvalidAttrNumber, "invalid attribute number"); PushActiveSnapshot(GetLatestSnapshot()); rel = table_open(chunk_relid, AccessShareLock); scandesc = table_beginscan(rel, GetActiveSnapshot(), 0, NULL); slot = table_slot_create(rel, NULL); while (table_scan_getnextslot(scandesc, ForwardScanDirection, slot)) { Datum datum; int64 value; datum = slot_getattr(slot, attno, &isnull); Assert(!isnull); if (NULL != dim->partitioning) { Oid collation = TupleDescAttr(slot->tts_tupleDescriptor, AttrNumberGetAttrOffset(attno)) ->attcollation; datum = ts_partitioning_func_apply(dim->partitioning, collation, datum); } if (dim->type == DIMENSION_TYPE_OPEN) value = ts_time_value_to_internal(datum, ts_dimension_get_partition_type(dim)); else if (dim->type == DIMENSION_TYPE_CLOSED) value = (int64) DatumGetInt32(datum); else elog(ERROR, "invalid dimension type when checking constraint"); if (value < slice->fd.range_start || value >= slice->fd.range_end) ereport(ERROR, (errcode(ERRCODE_CHECK_VIOLATION), errmsg("dimension constraint for column \"%s\" violated by some row", NameStr(dim->fd.column_name)))); } ExecDropSingleTupleTableSlot(slot); table_endscan(scandesc); table_close(rel, NoLock); PopActiveSnapshot(); } /* * Check whether the chunk has any row that violates any of its dimensional constraints */ void ts_chunk_constraint_check_violated(const Chunk *chunk, const Hyperspace *hs) { const ChunkConstraints *ccs = chunk->constraints; for (int i = 0; i < ccs->num_constraints; i++) { const ChunkConstraint *cc = &ccs->constraints[i]; if (is_dimension_constraint(cc)) { const DimensionSlice *slice = get_slice_with_id(chunk->cube, cc->fd.dimension_slice_id); const Dimension *dim; dim = ts_hyperspace_get_dimension_by_id(hs, slice->fd.dimension_id); Assert(dim); /* Check if the chunk has any row that violates the constraint */ check_chunk_constraint_violated(chunk->table_id, dim, slice); } } } ================================================ FILE: src/chunk_constraint.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/pg_list.h> #include "hypertable.h" #include "ts_catalog/catalog.h" typedef struct ChunkConstraint { FormData_chunk_constraint fd; } ChunkConstraint; typedef struct ChunkConstraints { MemoryContext mctx; int16 capacity; int16 num_constraints; int16 num_dimension_constraints; ChunkConstraint *constraints; } ChunkConstraints; #define chunk_constraints_get(cc, i) &((cc)->constraints[i]) #define is_dimension_constraint(cc) ((cc)->fd.dimension_slice_id > 0) typedef struct Chunk Chunk; typedef struct DimensionSlice DimensionSlice; typedef struct Hypercube Hypercube; typedef struct ChunkScanCtx ChunkScanCtx; extern TSDLLEXPORT ChunkConstraints *ts_chunk_constraints_alloc(int size_hint, MemoryContext mctx); extern ChunkConstraints * ts_chunk_constraint_scan_by_chunk_id(int32 chunk_id, Size num_constraints_hint, MemoryContext mctx); extern ChunkConstraints *ts_chunk_constraints_copy(ChunkConstraints *chunk_constraints); extern int ts_chunk_constraint_scan_by_dimension_slice(const DimensionSlice *slice, ChunkScanCtx *ctx, MemoryContext mctx); extern int ts_chunk_constraint_scan_by_dimension_slice_to_list(const DimensionSlice *slice, List **list, MemoryContext mctx); extern int TSDLLEXPORT ts_chunk_constraint_scan_by_dimension_slice_id(int32 dimension_slice_id, ChunkConstraints *ccs, MemoryContext mctx); extern ChunkConstraint *ts_chunk_constraints_add(ChunkConstraints *ccs, int32 chunk_id, int32 dimension_slice_id, const char *constraint_name, const char *hypertable_constraint_name); extern int ts_chunk_constraints_add_dimension_constraints(ChunkConstraints *ccs, int32 chunk_id, const Hypercube *cube); extern TSDLLEXPORT int ts_chunk_constraints_add_inheritable_constraints(ChunkConstraints *ccs, int32 chunk_id, const char chunk_relkind, Oid hypertable_oid, Oid table_id); extern TSDLLEXPORT int ts_chunk_constraints_add_inheritable_check_constraints( ChunkConstraints *ccs, int32 chunk_id, const char chunk_relkind, Oid hypertable_oid); extern TSDLLEXPORT void ts_chunk_constraints_insert_metadata(const ChunkConstraints *ccs); extern TSDLLEXPORT Constraint *ts_chunk_constraint_dimensional_create(const Dimension *dim, const DimensionSlice *slice, const char *name); extern TSDLLEXPORT void ts_chunk_constraints_create(const Hypertable *ht, const Chunk *chunk); extern void ts_chunk_constraint_create_on_chunk(const Hypertable *ht, const Chunk *chunk, Oid constraint_oid); extern int ts_chunk_constraint_delete_by_hypertable_constraint_name(int32 chunk_id, const char *hypertable_constraint_name); extern int ts_chunk_constraint_delete_by_chunk_id(int32 chunk_id, ChunkConstraints *ccs, bool drop_constraint); extern int ts_chunk_constraint_delete_by_dimension_slice_id(int32 dimension_slice_id); extern int ts_chunk_constraint_delete_by_constraint_name(int32 chunk_id, const char *constraint_name); extern void ts_chunk_constraints_recreate(const Hypertable *ht, const Chunk *chunk); extern int ts_chunk_constraint_rename_hypertable_constraint(int32 chunk_id, const char *old_name, const char *new_name); extern int ts_chunk_constraint_adjust_meta(int32 chunk_id, const char *ht_name, const char *chunk_old_name, const char *chunk_new_name); extern TSDLLEXPORT bool ts_chunk_constraint_update_slice_id(int32 chunk_id, int32 old_slice_id, int32 new_slice_id); extern char * ts_chunk_constraint_get_name_from_hypertable_constraint(Oid chunk_relid, const char *hypertable_constraint_name); extern void ts_chunk_constraint_insert(ChunkConstraint *constraint); extern ChunkConstraint *ts_chunk_constraints_add_from_tuple(ChunkConstraints *ccs, const TupleInfo *ti); extern ScanIterator ts_chunk_constraint_scan_iterator_create(MemoryContext result_mcxt); extern void ts_chunk_constraint_scan_iterator_set_slice_id(ScanIterator *it, int32 slice_id); extern void ts_chunk_constraint_scan_iterator_set_chunk_id(ScanIterator *it, int32 chunk_id); extern int ts_chunk_constraint_delete_dimensional_constraints(int32 chunk_id, ChunkConstraints *ccs); extern TSDLLEXPORT void ts_chunk_constraint_drop_from_tuple(HeapTuple constraint_tuple); extern TSDLLEXPORT void ts_chunk_constraint_check_violated(const Chunk *chunk, const Hyperspace *hs); ================================================ FILE: src/chunk_index.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/htup_details.h> #include <access/xact.h> #include <catalog/dependency.h> #include <catalog/index.h> #include <catalog/indexing.h> #include <catalog/namespace.h> #include <catalog/objectaddress.h> #include <catalog/pg_constraint.h> #include <catalog/pg_depend.h> #include <catalog/pg_index.h> #include <commands/cluster.h> #include <commands/defrem.h> #include <commands/tablecmds.h> #include <commands/tablespace.h> #include <miscadmin.h> #include <nodes/parsenodes.h> #include <optimizer/optimizer.h> #include <utils/builtins.h> #include <utils/fmgroids.h> #include <utils/lsyscache.h> #include <utils/rel.h> #include <utils/syscache.h> #include "chunk.h" #include "chunk_index.h" #include "hypertable.h" #include "hypertable_cache.h" #include "indexing.h" #include "scan_iterator.h" #include "scanner.h" #include "ts_catalog/catalog.h" static Oid ts_chunk_index_create_post_adjustment(int32 hypertable_id, Relation template_indexrel, Relation chunkrel, IndexInfo *indexinfo, bool isconstraint, Oid index_tablespace); static List * create_index_colnames(Relation indexrel) { List *colnames = NIL; int i; for (i = 0; i < indexrel->rd_att->natts; i++) { Form_pg_attribute idxattr = TupleDescAttr(indexrel->rd_att, i); colnames = lappend(colnames, pstrdup(NameStr(idxattr->attname))); } return colnames; } /* * Pick a name for a chunk index. * * The chunk's index name will the original index name prefixed with the chunk's * table name, modulo any conflict resolution we need to do. */ static char * chunk_index_choose_name(const char *tabname, const char *main_index_name, Oid namespaceid) { char buf[10]; char *label = NULL; char *idxname; int n = 0; for (;;) { /* makeObjectName will ensure the index name fits within a NAME type */ idxname = makeObjectName(tabname, main_index_name, label); if (!OidIsValid(get_relname_relid(idxname, namespaceid))) break; /* found a conflict, so try a new name component */ pfree(idxname); snprintf(buf, sizeof(buf), "%d", ++n); label = buf; } return idxname; } static void adjust_expr_attnos(Oid ht_relid, IndexInfo *ii, Relation chunkrel) { List *vars = NIL; ListCell *lc; /* Get a list of references to all Vars in the expression */ if (ii->ii_Expressions != NIL) vars = list_concat(vars, pull_var_clause((Node *) ii->ii_Expressions, 0)); /* Get a list of references to all Vars in the predicate */ if (ii->ii_Predicate != NIL) vars = list_concat(vars, pull_var_clause((Node *) ii->ii_Predicate, 0)); foreach (lc, vars) { Var *var = lfirst_node(Var, lc); var->varattno = ts_map_attno(ht_relid, chunkrel->rd_id, var->varattno); var->varattnosyn = var->varattno; } } /* * Adjust column reference attribute numbers for regular indexes. */ static void chunk_adjust_colref_attnos(IndexInfo *ii, Oid ht_relid, Relation chunkrel) { int i; for (i = 0; i < ii->ii_NumIndexAttrs; i++) { /* zeroes indicate expressions */ if (ii->ii_IndexAttrNumbers[i] == 0) continue; /* we must not use get_attname on the index here as the index column names * are independent of parent relation column names. Instead we need to look * up the attno of the referenced hypertable column and do the matching * with the hypertable column name */ ii->ii_IndexAttrNumbers[i] = ts_map_attno(ht_relid, chunkrel->rd_id, ii->ii_IndexAttrNumbers[i]); } } void ts_adjust_indexinfo_attnos(IndexInfo *indexinfo, Oid ht_relid, Relation chunkrel) { /* * Adjust a hypertable's index attribute numbers to match a chunk. * * A hypertable's IndexInfo for one of its indexes references the attributes * (columns) in the hypertable by number. These numbers might not be the same * for the corresponding attribute in one of its chunks. To be able to use an * IndexInfo from a hypertable's index to create a corresponding index on a * chunk, we need to adjust the attribute numbers to match the chunk. * * We need to handle 3 places: * - direct column references in ii_IndexAttrNumbers * - references in expressions in ii_Expressions * - references in expressions in ii_Predicate */ chunk_adjust_colref_attnos(indexinfo, ht_relid, chunkrel); if (indexinfo->ii_Expressions || indexinfo->ii_Predicate) adjust_expr_attnos(ht_relid, indexinfo, chunkrel); } #define CHUNK_INDEX_TABLESPACE_OFFSET 1 /* * Pick a chunk index's tablespace at an offset from the chunk's tablespace in * order to avoid colocating chunks and their indexes in the same tablespace. * This hopefully leads to more I/O parallelism. */ static Oid chunk_index_select_tablespace(int32 hypertable_id, Relation chunkrel) { Tablespace *tspc; Oid tablespace_oid = InvalidOid; tspc = ts_hypertable_get_tablespace_at_offset_from(hypertable_id, chunkrel->rd_rel->reltablespace, CHUNK_INDEX_TABLESPACE_OFFSET); if (NULL != tspc) tablespace_oid = tspc->tablespace_oid; return tablespace_oid; } Oid ts_chunk_index_get_tablespace(int32 hypertable_id, Relation template_indexrel, Relation chunkrel) { /* * Determine the index's tablespace. Use the main index's tablespace, or, * if not set, select one at an offset from the chunk's tablespace. */ if (OidIsValid(template_indexrel->rd_rel->reltablespace)) return template_indexrel->rd_rel->reltablespace; else return chunk_index_select_tablespace(hypertable_id, chunkrel); } /* * Create a chunk index based on the configuration of the "parent" index. */ static Oid chunk_relation_index_create(Relation htrel, Relation template_indexrel, Relation chunkrel, bool isconstraint, Oid index_tablespace) { IndexInfo *indexinfo = BuildIndexInfo(template_indexrel); int32 hypertable_id; bool skip_mapping = false; /* * If the supplied template index is not on the hypertable we must not do attnum * mapping based on the hypertable. Ideally we would check for the template being * on the chunk but we cannot do that since when we rebuild a chunk the new chunk * has a different id. But the template index should always be either on the * hypertable or on a relation with the same physical layout as chunkrel. */ if (IndexGetRelation(template_indexrel->rd_id, false) != htrel->rd_id) skip_mapping = true; /* * Convert the IndexInfo's attnos to match the chunk instead of the * hypertable */ if (!skip_mapping && chunk_index_need_attnos_adjustment(RelationGetDescr(htrel), RelationGetDescr(chunkrel))) ts_adjust_indexinfo_attnos(indexinfo, htrel->rd_id, chunkrel); hypertable_id = ts_hypertable_relid_to_id(htrel->rd_id); Assert(hypertable_id != INVALID_HYPERTABLE_ID); return ts_chunk_index_create_post_adjustment(hypertable_id, template_indexrel, chunkrel, indexinfo, isconstraint, index_tablespace); } static Oid ts_chunk_index_create_post_adjustment(int32 hypertable_id, Relation template_indexrel, Relation chunkrel, IndexInfo *indexinfo, bool isconstraint, Oid index_tablespace) { Oid chunk_indexrelid = InvalidOid; const char *indexname; HeapTuple tuple; bool isnull; Datum reloptions; Datum indclass; oidvector *indclassoid; List *colnames = create_index_colnames(template_indexrel); Oid tablespace; bits16 flags = 0; tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(RelationGetRelid(template_indexrel))); if (!HeapTupleIsValid(tuple)) elog(ERROR, "cache lookup failed for index relation %u", RelationGetRelid(template_indexrel)); reloptions = SysCacheGetAttr(RELOID, tuple, Anum_pg_class_reloptions, &isnull); indclass = SysCacheGetAttr(INDEXRELID, template_indexrel->rd_indextuple, Anum_pg_index_indclass, &isnull); Assert(!isnull); indclassoid = (oidvector *) DatumGetPointer(indclass); indexname = chunk_index_choose_name(get_rel_name(RelationGetRelid(chunkrel)), get_rel_name(RelationGetRelid(template_indexrel)), get_rel_namespace(RelationGetRelid(chunkrel))); if (OidIsValid(index_tablespace)) tablespace = index_tablespace; else tablespace = ts_chunk_index_get_tablespace(hypertable_id, template_indexrel, chunkrel); /* assign flags for index creation and constraint creation */ if (isconstraint) flags |= INDEX_CREATE_ADD_CONSTRAINT; if (template_indexrel->rd_index->indisprimary) flags |= INDEX_CREATE_IS_PRIMARY; chunk_indexrelid = index_create_compat(chunkrel, indexname, InvalidOid, InvalidOid, InvalidOid, InvalidOid, indexinfo, colnames, template_indexrel->rd_rel->relam, tablespace, template_indexrel->rd_indcollation, indclassoid->values, NULL, /* opclassOptions */ template_indexrel->rd_indoption, NULL, /* stattargets */ reloptions, flags, 0, /* constr_flags constant and 0 * for now */ false, /* allow system table mods */ false, /* is internal */ NULL); /* constraintId */ ReleaseSysCache(tuple); return chunk_indexrelid; } static Oid chunk_index_find_matching(Relation chunk_rel, Oid ht_indexoid) { List *indexlist = RelationGetIndexList(chunk_rel); ListCell *lc; foreach (lc, indexlist) { Oid indexoid = lfirst_oid(lc); if (ts_indexing_compare(indexoid, ht_indexoid)) return indexoid; } list_free(indexlist); return InvalidOid; } /* * Create a new chunk index as a child of a parent hypertable index. * * The chunk index is created based on the information from the parent index * relation. This function is typically called when a new chunk is created and * it should, for each hypertable index, have a corresponding index of its own. */ static void chunk_index_create(Relation hypertable_rel, int32 hypertable_id, Relation hypertable_idxrel, int32 chunk_id, Relation chunkrel, Oid constraint_oid, Oid index_tblspc) { Oid chunk_indexrelid; if (OidIsValid(constraint_oid)) { /* * If there is an associated constraint then that constraint created * both the index and the catalog entry for the index */ return; } chunk_indexrelid = chunk_index_find_matching(chunkrel, RelationGetRelid(hypertable_idxrel)); if (!OidIsValid(chunk_indexrelid)) { chunk_indexrelid = chunk_relation_index_create(hypertable_rel, hypertable_idxrel, chunkrel, false, index_tblspc); } } void ts_chunk_index_create_from_adjusted_index_info(int32 hypertable_id, Relation hypertable_idxrel, int32 chunk_id, Relation chunkrel, IndexInfo *indexinfo) { ts_chunk_index_create_post_adjustment(hypertable_id, hypertable_idxrel, chunkrel, indexinfo, false, false); } /* * Create all indexes on a chunk, given the indexes that exists on the chunk's * hypertable. */ void ts_chunk_index_create_all(int32 hypertable_id, Oid hypertable_relid, int32 chunk_id, Oid chunkrelid, Oid index_tblspc) { Relation htrel; Relation chunkrel; List *indexlist; ListCell *lc; const char chunk_relkind = get_rel_relkind(chunkrelid); /* Foreign table chunks don't support indexes */ if (chunk_relkind == RELKIND_FOREIGN_TABLE) return; Assert(chunk_relkind == RELKIND_RELATION); htrel = table_open(hypertable_relid, AccessShareLock); /* Need ShareLock on the heap relation we are creating indexes on */ chunkrel = table_open(chunkrelid, ShareLock); /* * We should only add those indexes that aren't created from constraints, * since those are added separately. * * Ideally, we should just be able to check the index relation's rd_index * struct for the flags indisunique, indisprimary, indisexclusion to * figure out if this is a constraint-supporting index. However, * indisunique is true both for plain unique indexes and those created * from constraints. Instead, we prune the main table's index list, * removing those indexes that are supporting a constraint. */ indexlist = RelationGetIndexList(htrel); foreach (lc, indexlist) { Oid hypertable_idxoid = lfirst_oid(lc); Relation hypertable_idxrel = index_open(hypertable_idxoid, AccessShareLock); chunk_index_create(htrel, hypertable_id, hypertable_idxrel, chunk_id, chunkrel, get_index_constraint(hypertable_idxoid), index_tblspc); index_close(hypertable_idxrel, AccessShareLock); } table_close(chunkrel, NoLock); table_close(htrel, AccessShareLock); } List * ts_chunk_index_get_mappings(Hypertable *ht, Oid hypertable_indexrelid) { List *mappings = NIL; List *chunks = ts_chunk_get_by_hypertable_id(ht->fd.id); ListCell *lc; foreach (lc, chunks) { Chunk *chunk = lfirst(lc); if (!OidIsValid(chunk->table_id)) continue; Relation chunk_rel = table_open(chunk->table_id, AccessShareLock); Oid chunk_indexrelid = ts_chunk_index_get_by_hypertable_indexrelid(chunk_rel, hypertable_indexrelid); table_close(chunk_rel, AccessShareLock); if (OidIsValid(chunk_indexrelid)) { ChunkIndexMapping *cim = palloc0(sizeof(ChunkIndexMapping)); cim->chunkoid = chunk->table_id; cim->indexoid = chunk_indexrelid; cim->parent_indexoid = hypertable_indexrelid; cim->hypertableoid = ht->fd.id; mappings = lappend(mappings, cim); } } return mappings; } TSDLLEXPORT Oid ts_chunk_index_get_by_hypertable_indexrelid(Relation chunk_rel, Oid ht_indexoid) { List *indexlist = RelationGetIndexList(chunk_rel); ListCell *lc; Oid chunk_index_oid = InvalidOid; foreach (lc, indexlist) { Oid indexoid = lfirst_oid(lc); if (ts_indexing_compare(indexoid, ht_indexoid)) { chunk_index_oid = indexoid; break; } } list_free(indexlist); return chunk_index_oid; } void ts_chunk_index_rename(Hypertable *ht, Oid hypertable_indexrelid, const char *ht_name) { ListCell *lc; List *chunks = ts_chunk_get_by_hypertable_id(ht->fd.id); foreach (lc, chunks) { Chunk *chunk = lfirst(lc); if (!OidIsValid(chunk->table_id)) continue; Relation chunk_rel = table_open(chunk->table_id, AccessExclusiveLock); Oid chunk_indexrelid = ts_chunk_index_get_by_hypertable_indexrelid(chunk_rel, hypertable_indexrelid); table_close(chunk_rel, NoLock); /* If there is no matching index on the chunk, skip it */ if (OidIsValid(chunk_indexrelid)) { Oid chunk_schemaoid = get_namespace_oid(NameStr(chunk->fd.schema_name), false); const char *chunk_old_name = get_rel_name(chunk_indexrelid); const char *chunk_new_name = chunk_index_choose_name(NameStr(chunk->fd.table_name), ht_name, chunk_schemaoid); /* * Index might also have a constraint which we track separately in our catalog * and needs to be updated too */ ts_chunk_constraint_adjust_meta(chunk->fd.id, ht_name, chunk_old_name, chunk_new_name); RenameRelationInternal(chunk_indexrelid, chunk_new_name, false, true); } } } void ts_chunk_index_set_tablespace(Hypertable *ht, Oid hypertable_indexrelid, char *tablespace) { List *chunks = ts_chunk_get_by_hypertable_id(ht->fd.id); ListCell *lc; foreach (lc, chunks) { Chunk *chunk = lfirst(lc); Relation chunk_rel = table_open(chunk->table_id, AccessExclusiveLock); Oid chunk_indexrelid = ts_chunk_index_get_by_hypertable_indexrelid(chunk_rel, hypertable_indexrelid); table_close(chunk_rel, NoLock); if (OidIsValid(chunk_indexrelid)) { AlterTableCmd *cmd = makeNode(AlterTableCmd); List *cmds = NIL; cmd->subtype = AT_SetTableSpace; cmd->name = tablespace; cmds = lappend(cmds, cmd); ts_alter_table_with_event_trigger(chunk_indexrelid, NULL, cmds, false); } } } TSDLLEXPORT void ts_chunk_index_mark_clustered(Oid chunkrelid, Oid indexrelid) { Relation rel = table_open(chunkrelid, AccessShareLock); mark_index_clustered(rel, indexrelid, true); CommandCounterIncrement(); table_close(rel, AccessShareLock); } static Oid chunk_index_duplicate_index(Relation hypertable_rel, Chunk *src_chunk, Oid chunk_index_oid, Relation dest_chunk_rel, Oid index_tablespace) { Relation chunk_index_rel = index_open(chunk_index_oid, AccessShareLock); Oid constraint_oid; Oid new_chunk_indexrelid; constraint_oid = get_index_constraint(chunk_index_oid); new_chunk_indexrelid = chunk_relation_index_create(hypertable_rel, chunk_index_rel, dest_chunk_rel, OidIsValid(constraint_oid), index_tablespace); index_close(chunk_index_rel, NoLock); return new_chunk_indexrelid; } /* * Create versions of every index over src_chunkrelid over chunkrelid. * Returns the relids of the new indexes created. * New indexes are in the same order as RelationGetIndexList. */ TSDLLEXPORT List * ts_chunk_index_duplicate(Oid src_chunkrelid, Oid dest_chunkrelid, List **src_index_oids, Oid index_tablespace) { Relation hypertable_rel = NULL; Relation src_chunk_rel; Relation dest_chunk_rel; List *index_oids; ListCell *index_elem; List *new_index_oids = NIL; Chunk *src_chunk; src_chunk_rel = table_open(src_chunkrelid, AccessShareLock); dest_chunk_rel = table_open(dest_chunkrelid, ShareLock); src_chunk = ts_chunk_get_by_relid(src_chunkrelid, true); hypertable_rel = table_open(src_chunk->hypertable_relid, AccessShareLock); index_oids = RelationGetIndexList(src_chunk_rel); foreach (index_elem, index_oids) { Oid chunk_index_oid = lfirst_oid(index_elem); Oid new_chunk_indexrelid = chunk_index_duplicate_index(hypertable_rel, src_chunk, chunk_index_oid, dest_chunk_rel, index_tablespace); new_index_oids = lappend_oid(new_index_oids, new_chunk_indexrelid); } table_close(hypertable_rel, AccessShareLock); table_close(dest_chunk_rel, NoLock); table_close(src_chunk_rel, NoLock); if (src_index_oids != NULL) *src_index_oids = index_oids; return new_index_oids; } void ts_chunk_index_move_all(Oid chunk_relid, Oid index_tblspc) { Relation chunkrel; List *indexlist; ListCell *lc; const char chunk_relkind = get_rel_relkind(chunk_relid); /* execute ALTER INDEX .. SET TABLESPACE for each index on the chunk */ AlterTableCmd cmd = { .type = T_AlterTableCmd, .subtype = AT_SetTableSpace, .name = get_tablespace_name(index_tblspc) }; /* Foreign table chunks don't support indexes */ if (chunk_relkind == RELKIND_FOREIGN_TABLE) return; Assert(chunk_relkind == RELKIND_RELATION); chunkrel = table_open(chunk_relid, AccessShareLock); indexlist = RelationGetIndexList(chunkrel); foreach (lc, indexlist) { Oid chunk_idxoid = lfirst_oid(lc); ts_alter_table_with_event_trigger(chunk_idxoid, NULL, list_make1(&cmd), false); } table_close(chunkrel, AccessShareLock); } ================================================ FILE: src/chunk_index.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <fmgr.h> #include <nodes/execnodes.h> #include <nodes/parsenodes.h> #include <utils/relcache.h> #include "compat/compat.h" #include "export.h" typedef struct Chunk Chunk; typedef struct Hypertable Hypertable; typedef struct ChunkIndexMapping { Oid chunkoid; Oid parent_indexoid; Oid indexoid; Oid hypertableoid; } ChunkIndexMapping; extern void ts_chunk_index_create(Relation hypertable_rel, int32 hypertable_id, Relation hypertable_idxrel, int32 chunk_id, Relation chunkrel); void ts_adjust_indexinfo_attnos(IndexInfo *indexinfo, Oid ht_relid, Relation chunkrel); extern void ts_chunk_index_create_from_adjusted_index_info(int32 hypertable_id, Relation hypertable_idxrel, int32 chunk_id, Relation chunkrel, IndexInfo *indexinfo); extern TSDLLEXPORT void ts_chunk_index_create_all(int32 hypertable_id, Oid hypertable_relid, int32 chunk_id, Oid chunkrelid, Oid index_tblspc); extern TSDLLEXPORT void ts_chunk_index_move_all(Oid chunk_relid, Oid index_tblspc); extern void ts_chunk_index_rename(Hypertable *ht, Oid hypertable_indexrelid, const char *ht_name); extern void ts_chunk_index_set_tablespace(Hypertable *ht, Oid hypertable_indexrelid, char *tablespace); extern List *ts_chunk_index_get_mappings(Hypertable *ht, Oid hypertable_indexrelid); extern TSDLLEXPORT Oid ts_chunk_index_get_by_hypertable_indexrelid(Relation chunk_rel, Oid ht_indexoid); extern TSDLLEXPORT void ts_chunk_index_mark_clustered(Oid chunkrelid, Oid indexrelid); extern TSDLLEXPORT List *ts_chunk_index_duplicate(Oid src_chunkrelid, Oid dest_chunkrelid, List **src_index_oids, Oid index_tablespace); extern Oid ts_chunk_index_get_tablespace(int32 hypertable_id, Relation template_indexrel, Relation chunkrel); static inline bool chunk_index_columns_changed(int hypertable_natts, TupleDesc chunkdesc) { /* * We should be able to safely assume that the only reason the number of * attributes differ is because we have removed columns in the base table, * leaving junk attributes that aren't inherited by the chunk. */ return hypertable_natts != chunkdesc->natts; } static inline bool chunk_index_need_attnos_adjustment(TupleDesc htdesc, TupleDesc chunkdesc) { return chunk_index_columns_changed(htdesc->natts, chunkdesc); } ================================================ FILE: src/chunk_insert_state.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/attnum.h> #include <access/xact.h> #include <catalog/pg_type.h> #include <executor/tuptable.h> #include <foreign/fdwapi.h> #include <miscadmin.h> #include <nodes/execnodes.h> #include <nodes/makefuncs.h> #include <nodes/nodes.h> #include <nodes/plannodes.h> #include <optimizer/optimizer.h> #include <parser/parsetree.h> #include <rewrite/rewriteManip.h> #include <utils/builtins.h> #include <utils/guc.h> #include <utils/lsyscache.h> #include <utils/memutils.h> #include <utils/rel.h> #include <utils/rls.h> #include "compat/compat.h" #include "chunk_index.h" #include "chunk_insert_state.h" #include "chunk_tuple_routing.h" #include "debug_point.h" #include "errors.h" #include "indexing.h" #include "nodes/modify_hypertable.h" #include "ts_catalog/continuous_agg.h" /* Just like ExecPrepareExpr except that it doesn't switch to the query memory context */ static inline ExprState * prepare_constr_expr(Expr *node) { ExprState *result; node = expression_planner(node); result = ExecInitExpr(node, NULL); return result; } /* * Create the constraint exprs inside the current memory context. If this * is not done here, then ExecRelCheck will do it for you but put it into * the query memory context, which will cause a memory leak. * * See the comment in `ts_chunk_insert_state_destroy` for more information * on the implications of this. */ static inline void create_chunk_rri_constraint_expr(ResultRelInfo *rri, Relation rel) { int ncheck, i; ConstrCheck *check; Assert(rel->rd_att->constr != NULL && rri->ri_CheckConstraintExprs == NULL); ncheck = rel->rd_att->constr->num_check; check = rel->rd_att->constr->check; rri->ri_CheckConstraintExprs = (ExprState **) palloc(ncheck * sizeof(ExprState *)); for (i = 0; i < ncheck; i++) { Expr *checkconstr = stringToNode(check[i].ccbin); rri->ri_CheckConstraintExprs[i] = prepare_constr_expr(checkconstr); } } /* * Create a new ResultRelInfo for a chunk. * * The ResultRelInfo holds the executor state (e.g., open relation, indexes, and * options) for the result relation where tuples will be stored. * * The Hypertable ResultRelInfo is used as a template for the chunk's new ResultRelInfo. */ ResultRelInfo * create_chunk_result_relation_info(ResultRelInfo *ht_rri, Relation rel, EState *estate) { ResultRelInfo *rri; rri = makeNode(ResultRelInfo); InitResultRelInfo(rri, rel, ht_rri->ri_RangeTableIndex, NULL, estate->es_instrument); /* Copy options from the main table's (hypertable's) result relation info */ rri->ri_WithCheckOptions = ht_rri->ri_WithCheckOptions; rri->ri_WithCheckOptionExprs = ht_rri->ri_WithCheckOptionExprs; rri->ri_projectReturning = ht_rri->ri_projectReturning; rri->ri_FdwState = NULL; rri->ri_usesFdwDirectModify = ht_rri->ri_usesFdwDirectModify; if (RelationGetForm(rel)->relkind == RELKIND_FOREIGN_TABLE) rri->ri_FdwRoutine = GetFdwRoutineForRelation(rel, true); create_chunk_rri_constraint_expr(rri, rel); return rri; } static ProjectionInfo * get_adjusted_projection_info_returning(ProjectionInfo *orig, List *returning_clauses, TupleConversionMap *map, Index varno, Oid rowtype, TupleDesc chunk_desc) { bool found_whole_row; Assert(returning_clauses != NIL); /* map hypertable attnos -> chunk attnos */ if (map != NULL) returning_clauses = castNode(List, map_variable_attnos((Node *) returning_clauses, varno, 0, map->attrMap, rowtype, &found_whole_row)); return ExecBuildProjectionInfo(returning_clauses, orig->pi_exprContext, orig->pi_state.resultslot, orig->pi_state.parent, chunk_desc); } static List * translate_clause(List *inclause, TupleConversionMap *chunk_map, Index varno, Relation hyper_rel, Relation chunk_rel) { List *clause = copyObject(inclause); bool found_whole_row; /* nothing to do here if the chunk_map is NULL */ if (!chunk_map) return list_copy(clause); /* map hypertable attnos -> chunk attnos for the "excluded" table */ clause = castNode(List, map_variable_attnos((Node *) clause, INNER_VAR, 0, chunk_map->attrMap, RelationGetForm(chunk_rel)->reltype, &found_whole_row)); /* map hypertable attnos -> chunk attnos for the hypertable */ clause = castNode(List, map_variable_attnos((Node *) clause, varno, 0, chunk_map->attrMap, RelationGetForm(chunk_rel)->reltype, &found_whole_row)); return clause; } /* * adjust_chunk_colnos * Adjust the list of UPDATE target column numbers to account for * attribute differences between the parent and the partition. * * adapted from postgres adjust_partition_colnos */ static List * adjust_chunk_colnos(List *colnos, ResultRelInfo *chunk_rri) { List *new_colnos = NIL; TupleConversionMap *map = ExecGetChildToRootMap(chunk_rri); AttrMap *attrMap; ListCell *lc; Assert(map != NULL); /* else we shouldn't be here */ attrMap = map->attrMap; foreach (lc, colnos) { AttrNumber parentattrno = lfirst_int(lc); if (parentattrno <= 0 || parentattrno > attrMap->maplen || attrMap->attnums[parentattrno - 1] == 0) elog(ERROR, "unexpected attno %d in target column list", parentattrno); new_colnos = lappend_int(new_colnos, attrMap->attnums[parentattrno - 1]); } return new_colnos; } /* * Setup ON CONFLICT state for a chunk. * * Mostly, this is about mapping attribute numbers from the hypertable root to * a chunk, accounting for differences in the tuple descriptors due to dropped * columns, etc. */ static void setup_on_conflict_state(ResultRelInfo *ht_rri, ModifyTableState *mtstate, ChunkInsertState *state, TupleConversionMap *chunk_map) { TupleConversionMap *map = state->hyper_to_chunk_map; ResultRelInfo *chunk_rri = state->result_relation_info; Relation chunk_rel = state->result_relation_info->ri_RelationDesc; Relation hyper_rel = ht_rri->ri_RelationDesc; ModifyTable *mt = castNode(ModifyTable, mtstate->ps.plan); OnConflictSetState *onconfl = makeNode(OnConflictSetState); memcpy(onconfl, ht_rri->ri_onConflict, sizeof(OnConflictSetState)); chunk_rri->ri_onConflict = onconfl; #if PG16_LT chunk_rri->ri_RootToPartitionMap = map; #else chunk_rri->ri_RootToChildMap = map; chunk_rri->ri_RootToChildMapValid = true; #endif Assert(mt->onConflictSet); Assert(ht_rri->ri_onConflict != NULL); /* * Need a separate existing slot for each partition, as the * partition could be of a different AM, even if the tuple * descriptors match. */ onconfl->oc_Existing = table_slot_create(chunk_rri->ri_RelationDesc, NULL); state->existing_slot = onconfl->oc_Existing; /* * If the chunk's tuple descriptor matches exactly the hypertable * (the common case), we can reuse most of the parent's ON * CONFLICT SET state, skipping a bunch of work. Otherwise, we * need to create state specific to this partition. */ if (!map) { /* * It's safe to reuse these from the hypertable, as we * only process one tuple at a time (therefore we won't * overwrite needed data in slots), and the results of * projections are independent of the underlying storage. * Projections and where clauses themselves don't store state * / are independent of the underlying storage. */ onconfl->oc_ProjSlot = ht_rri->ri_onConflict->oc_ProjSlot; onconfl->oc_ProjInfo = ht_rri->ri_onConflict->oc_ProjInfo; onconfl->oc_WhereClause = ht_rri->ri_onConflict->oc_WhereClause; state->conflproj_slot = onconfl->oc_ProjSlot; } else { List *onconflset; List *onconflcols; /* * Translate expressions in onConflictSet to account for * different attribute numbers. For that, map partition * varattnos twice: first to catch the EXCLUDED * pseudo-relation (INNER_VAR), and second to handle the main * target relation (firstVarno). */ onconflset = copyObject(mt->onConflictSet); Assert(map->outdesc == RelationGetDescr(chunk_rel)); if (!chunk_map) chunk_map = convert_tuples_by_name(RelationGetDescr(chunk_rel), RelationGetDescr(hyper_rel)); onconflset = translate_clause(onconflset, chunk_map, ht_rri->ri_RangeTableIndex, hyper_rel, chunk_rel); chunk_rri->ri_ChildToRootMap = chunk_map; chunk_rri->ri_ChildToRootMapValid = true; /* Finally, adjust the target colnos to match the chunk. */ if (chunk_map) onconflcols = adjust_chunk_colnos(mt->onConflictCols, chunk_rri); else onconflcols = mt->onConflictCols; /* create the tuple slot for the UPDATE SET projection */ onconfl->oc_ProjSlot = table_slot_create(chunk_rel, NULL); state->conflproj_slot = onconfl->oc_ProjSlot; /* build UPDATE SET projection state */ onconfl->oc_ProjInfo = ExecBuildUpdateProjection(onconflset, true, onconflcols, RelationGetDescr(chunk_rel), mtstate->ps.ps_ExprContext, onconfl->oc_ProjSlot, &mtstate->ps); Node *onconflict_where = mt->onConflictWhere; /* * Map attribute numbers in the WHERE clause, if it exists. */ if (onconflict_where && chunk_map) { List *clause = translate_clause(castNode(List, onconflict_where), chunk_map, ht_rri->ri_RangeTableIndex, hyper_rel, chunk_rel); chunk_rri->ri_onConflict->oc_WhereClause = ExecInitQual(clause, NULL); } } } /* Translate hypertable indexes to chunk indexes in the arbiter clause */ static void set_arbiter_indexes(ChunkInsertState *state, List *ht_arbiter_indexes) { List *chunk_arbiter_indexes = NIL; ListCell *lc; foreach (lc, ht_arbiter_indexes) { Oid hypertable_index = lfirst_oid(lc); Oid chunk_index_oid = ts_chunk_index_get_by_hypertable_indexrelid(state->rel, hypertable_index); if (!OidIsValid(chunk_index_oid)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("could not find arbiter index for hypertable index \"%s\" on chunk " "\"%s\"", get_rel_name(hypertable_index), get_rel_name(RelationGetRelid(state->rel))))); } chunk_arbiter_indexes = lappend_oid(chunk_arbiter_indexes, chunk_index_oid); } state->result_relation_info->ri_onConflictArbiterIndexes = chunk_arbiter_indexes; } /* Change the projections to work with chunks instead of hypertables */ static void adjust_projections(ResultRelInfo *ht_rri, ModifyTableState *mtstate, ChunkInsertState *cis, Oid rowtype) { ResultRelInfo *chunk_rri = cis->result_relation_info; Relation hyper_rel = ht_rri->ri_RelationDesc; Relation chunk_rel = cis->rel; TupleConversionMap *chunk_map = NULL; OnConflictAction onConflictAction = ONCONFLICT_NONE; List *returningLists = NIL; if (mtstate) { ModifyTable *mt = castNode(ModifyTable, mtstate->ps.plan); onConflictAction = mt->onConflictAction; returningLists = mt->returningLists; } if (returningLists) { /* * We need the opposite map from cis->hyper_to_chunk_map. The map needs * to have the hypertable_desc in the out spot for map_variable_attnos * to work correctly in mapping hypertable attnos->chunk attnos. */ chunk_map = convert_tuples_by_name(RelationGetDescr(chunk_rel), RelationGetDescr(hyper_rel)); chunk_rri->ri_projectReturning = get_adjusted_projection_info_returning(chunk_rri->ri_projectReturning, linitial(returningLists), chunk_map, ht_rri->ri_RangeTableIndex, rowtype, RelationGetDescr(chunk_rel)); } /* Set the chunk's arbiter indexes for ON CONFLICT statements */ if (onConflictAction != ONCONFLICT_NONE) { set_arbiter_indexes(cis, ht_rri->ri_onConflictArbiterIndexes); if (onConflictAction == ONCONFLICT_UPDATE) setup_on_conflict_state(ht_rri, mtstate, cis, chunk_map); } } /* * Create new insert chunk state. * * This is essentially a ResultRelInfo for a chunk. Initialization of the * ResultRelInfo should be similar to ExecInitModifyTable(). */ extern ChunkInsertState * ts_chunk_insert_state_create(Oid chunk_relid, const ChunkTupleRouting *ctr) { ChunkInsertState *state; Relation rel, parent_rel; MemoryContext cis_context = AllocSetContextCreate(ctr->estate->es_query_cxt, "chunk insert state memory context", ALLOCSET_DEFAULT_SIZES); ResultRelInfo *relinfo; const Chunk *chunk; MemoryContext old_mcxt = MemoryContextSwitchTo(ctr->estate->es_per_tuple_exprcontext->ecxt_per_tuple_memory); /* * Since we insert data and won't modify metadata, a RowExclusiveLock * should be sufficient. This should conflict with any metadata-modifying * operations as they should take higher-level locks (ShareLock and * above). */ rel = table_open(chunk_relid, RowExclusiveLock); /* * A concurrent chunk operation (e.g., compression) might have changed the * chunk metadata before we got a lock, so re-read it. * * This works even in higher levels of isolation since catalog data is * always read from latest snapshot. */ chunk = ts_chunk_get_by_relid(chunk_relid, true); Assert(chunk->relkind == RELKIND_RELATION); ts_chunk_validate_chunk_status_for_operation(chunk, CHUNK_INSERT, true); MemoryContextSwitchTo(cis_context); if (ctr->single_chunk_insert) relinfo = ctr->root_rri; else relinfo = create_chunk_result_relation_info(ctr->root_rri, rel, ctr->estate); if (ctr->mht_state) CheckValidResultRelCompat(relinfo, ctr->mht_state->mt->operation, ctr->mht_state->mt->onConflictAction, NIL); state = palloc0(sizeof(ChunkInsertState)); state->counters = ctr->counters; if (ctr->mht_state) state->onConflictAction = ctr->mht_state->mt->onConflictAction; state->mctx = cis_context; state->rel = rel; state->result_relation_info = relinfo; state->estate = ctr->estate; ts_set_compression_status(state, chunk); if (relinfo->ri_RelationDesc->rd_rel->relhasindex && relinfo->ri_IndexRelationDescs == NULL) ExecOpenIndices(relinfo, state->onConflictAction != ONCONFLICT_NONE); if (relinfo->ri_TrigDesc != NULL) { TriggerDesc *tg = relinfo->ri_TrigDesc; /* instead of triggers can only be created on VIEWs */ Assert(!tg->trig_insert_instead_row); /* * A statement that targets a parent table in an inheritance or * partitioning hierarchy does not cause the statement-level triggers * of affected child tables to be fired; only the parent table's * statement-level triggers are fired. However, row-level triggers * of any affected child tables will be fired. * During chunk creation we only copy ROW trigger to chunks so * statement triggers should not exist on chunks. */ if (tg->trig_insert_after_statement || tg->trig_insert_before_statement) elog(ERROR, "statement trigger on chunk table not supported"); } if (!ctr->single_chunk_insert) { parent_rel = table_open(ctr->hypertable->main_table_relid, AccessShareLock); /* Set tuple conversion map, if tuple needs conversion. */ state->hyper_to_chunk_map = convert_tuples_by_name(RelationGetDescr(parent_rel), RelationGetDescr(rel)); if (ctr->mht_state) adjust_projections(ctr->root_rri, linitial_node(ModifyTableState, ctr->mht_state->cscan_state.custom_ps), state, RelationGetForm(rel)->reltype); table_close(parent_rel, AccessShareLock); } /* Need a tuple table slot to store tuples going into this chunk. We don't * want this slot tied to the executor's tuple table, since that would tie * the slot's lifetime to the entire length of the execution and we want * to be able to dynamically create and destroy chunk insert * state. Otherwise, memory might blow up when there are many chunks being * inserted into. This also means that the slot needs to be destroyed with * the chunk insert state. */ state->slot = MakeSingleTupleTableSlot(RelationGetDescr(relinfo->ri_RelationDesc), table_slot_callbacks(relinfo->ri_RelationDesc)); state->hypertable_relid = chunk->hypertable_relid; state->chunk_id = chunk->fd.id; MemoryContextSwitchTo(old_mcxt); return state; } void ts_set_compression_status(ChunkInsertState *state, const Chunk *chunk) { state->chunk_compressed = ts_chunk_is_compressed(chunk); if (state->chunk_compressed) { state->chunk_partial = ts_chunk_is_partial(chunk); } } extern void ts_chunk_insert_state_destroy(ChunkInsertState *state, bool single_chunk_insert) { /* * Check if we need to mark the chunk as partial. * We need to change chunk status to partial in the following cases: * - rowstore insert into compressed chunk * - columnstore insert into uncompressed chunk that is not a new chunk (flagged as * needs_partial in chunk_tuple_routing.c) */ if (state->chunk_compressed && !state->chunk_partial && (!state->columnstore_insert || state->needs_partial)) { Oid chunk_relid = RelationGetRelid(state->result_relation_info->ri_RelationDesc); Chunk *chunk = ts_chunk_get_by_relid(chunk_relid, true); ts_chunk_set_partial(chunk); } table_close(state->rel, NoLock); if (state->slot) ExecDropSingleTupleTableSlot(state->slot); /* * Clean up per-chunk tuple table slots created for ON CONFLICT handling. */ if (NULL != state->existing_slot) ExecDropSingleTupleTableSlot(state->existing_slot); /* The ON CONFLICT projection slot is only chunk specific in case the * tuple descriptor didn't match the hypertable */ if (NULL != state->hyper_to_chunk_map && NULL != state->conflproj_slot) ExecDropSingleTupleTableSlot(state->conflproj_slot); if (!single_chunk_insert) { ExecCloseIndices(state->result_relation_info); MemoryContextDelete(state->mctx); } } ================================================ FILE: src/chunk_insert_state.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <access/tupconvert.h> #include <funcapi.h> #include "bmslist_utils.h" #include "cache.h" #include "chunk.h" typedef struct ChunkTupleRouting ChunkTupleRouting; typedef struct CompressionSettings CompressionSettings; typedef struct tuple_filtering_constraints tuple_filtering_constraints; typedef struct Bloom1Hasher Bloom1Hasher; /* * Bundle the ScanKey and the attribute numbers together * to be able to update the scankey by replacing the * `sk_argument` field with the value from the actual slot. */ typedef struct ScanKeyWithAttnos { int num_scankeys; ScanKeyData *scankeys; AttrNumber *attnos; } ScanKeyWithAttnos; /* * Holds information to cache scan keys and other * information needed for repeated calls of * `decompress_batches_for_insert` on the same chunk. */ typedef struct CachedDecompressionState { bool has_primary_or_unique_index; CompressionSettings *compression_settings; tuple_filtering_constraints *constraints; /* Columns that needs to be checked manually because * heap scan doesn't support SK_SEARCHNULL: */ Bitmapset *columns_with_null_check; ScanKeyWithAttnos heap_scankeys; ScanKeyWithAttnos index_scankeys; ScanKeyWithAttnos mem_scankeys; Oid index_relid; /* * Bloom information for UPSERT bloom optimization. * This is the best bloom filter match for the chunk out * of the (potentially) multiple bloom filters for the * chunk, based on the number of columns in the bloom filter. */ char *bloom_column_name; Bitmapset *bloom_insert_attnums; AttrNumber upsert_bloom_attnum; Bloom1Hasher *bloom_hasher; /* Pre-computed bloom filter checks for UPDATE/DELETE (List of BloomFilterCheck) */ List *bloom_filters; } CachedDecompressionState; typedef struct SharedCounters { /* Number of batches deleted */ int64 batches_deleted; /* Number of batches decompressed into the uncompressed table */ int64 batches_decompressed; /* Number of tuples decompressed */ int64 tuples_decompressed; /* Number of batches scanned */ int64 batches_scanned; /* Number of batches checked by bloom */ int64 batches_checked_by_bloom; /* Number of batches pruned by bloom */ int64 batches_pruned_by_bloom; /* Number of batches without bloom */ int64 batches_without_bloom; /* Number of batches bloom false positives */ int64 batches_bloom_false_positives; /* Number of batches skipped by pre-decompression filters */ int64 batches_filtered_compressed; /* Number of batches filtered after decompression */ int64 batches_filtered_decompressed; } SharedCounters; typedef struct ChunkInsertState { Relation rel; ResultRelInfo *result_relation_info; /* When the tuple descriptors for the main hypertable (root) and a chunk * differs, it is necessary to convert tuples to chunk format before * insertion, ON CONFLICT, or RETURNING handling. The table AM (storage format) * can also differ between the hypertable root and each chunk (as well as * between each chunk, in theory). * * The ResultRelInfo keeps per-relation slots for these purposes. The slots * here simply points to the per-relation slots in the ResultRelInfo. */ /* Pointer to slot for projected tuple in ON CONFLICT handling */ TupleTableSlot *conflproj_slot; /* Pointer to slot for tuple replaced in ON CONFLICT DO UPDATE * handling. Note that this slot's tuple descriptor is always the same as * the chunk rel's. */ TupleTableSlot *existing_slot; /* Slot for inserted/new tuples going into the chunk */ TupleTableSlot *slot; /* Map for converting tuple from hypertable (root table) format to chunk format */ TupleConversionMap *hyper_to_chunk_map; MemoryContext mctx; EState *estate; Oid hypertable_relid; int32 chunk_id; Oid user_id; /* for tracking compressed chunks */ bool chunk_compressed; bool chunk_partial; bool columnstore_insert; bool needs_partial; /* To speedup repeated calls of `decompress_batches_for_insert` */ CachedDecompressionState *cached_decompression_state; OnConflictAction onConflictAction; /* Should this INSERT be skipped due to ON CONFLICT DO NOTHING */ bool skip_current_tuple; SharedCounters *counters; } ChunkInsertState; extern ChunkInsertState *ts_chunk_insert_state_create(Oid chunk_relid, const ChunkTupleRouting *ctr); extern void ts_chunk_insert_state_destroy(ChunkInsertState *state, bool single_chunk_insert); ResultRelInfo *create_chunk_result_relation_info(ResultRelInfo *ht_rri, Relation rel, EState *estate); void ts_set_compression_status(ChunkInsertState *state, const Chunk *chunk); ================================================ FILE: src/chunk_scan.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/namespace.h> #include <storage/lmgr.h> #include <utils/builtins.h> #include <utils/syscache.h> #include "chunk.h" #include "chunk_constraint.h" #include "chunk_scan.h" #include "debug_point.h" #include "dimension_vector.h" #include "guc.h" #include "hypercube.h" #include "hypertable.h" #include "scan_iterator.h" #include "utils.h" /* * Scan for chunks matching a query. * * Given a number of dimension slices that match a query (a vector of slices * is given for each dimension), find the chunks that reference one slice in * each of the given dimensions. The matching chunks are built across multiple * scans: * * 1. Dimensional chunk constraints * 2. Chunk metadata * 3. Additional chunk constraints * 4. Chunk data nodes * * For performance, try not to interleave scans of different metadata tables * in order to maintain data locality while scanning. Also, keep scanned * tables and indexes open until all the metadata is scanned for all chunks. */ Chunk ** ts_chunk_scan_by_chunk_ids(const Hyperspace *hs, const List *chunk_ids, unsigned int *num_chunks) { MemoryContext work_mcxt = AllocSetContextCreate(CurrentMemoryContext, "chunk-scan-work", ALLOCSET_DEFAULT_SIZES); Chunk **locked_chunks = NULL; int locked_chunk_count = 0; ListCell *lc; Assert(OidIsValid(hs->main_table_relid)); MemoryContext orig_mcxt = MemoryContextSwitchTo(work_mcxt); /* * For each matching chunk, fill in the metadata from the "chunk" table. * Make sure to filter out "dropped" chunks. */ ScanIterator chunk_it = ts_chunk_scan_iterator_create(orig_mcxt); locked_chunks = (Chunk **) MemoryContextAlloc(orig_mcxt, sizeof(Chunk *) * list_length(chunk_ids)); foreach (lc, chunk_ids) { int chunk_id = lfirst_int(lc); Assert(CurrentMemoryContext == work_mcxt); ts_chunk_scan_iterator_set_chunk_id(&chunk_it, chunk_id); ts_scan_iterator_start_or_restart_scan(&chunk_it); TupleInfo *ti = ts_scan_iterator_next(&chunk_it); if (ti == NULL) { continue; } bool isnull; /* We found a chunk. First, try to lock it. */ Name schema_name = DatumGetName(slot_getattr(ti->slot, Anum_chunk_schema_name, &isnull)); Assert(!isnull); Name table_name = DatumGetName(slot_getattr(ti->slot, Anum_chunk_table_name, &isnull)); Assert(!isnull); Oid chunk_reloid = ts_get_relation_relid(NameStr(*schema_name), NameStr(*table_name), /* return_invalid = */ false); Assert(OidIsValid(chunk_reloid)); /* Only one chunk should match */ Assert(ts_scan_iterator_next(&chunk_it) == NULL); DEBUG_WAITPOINT("hypertable_expansion_before_lock_chunk"); if (!ts_chunk_lock_if_exists(chunk_reloid, AccessShareLock)) { continue; } /* * Now after we have locked the chunk, we have to reread its metadata. * It might have been modified concurrently by decompression, for * example. */ ts_chunk_scan_iterator_set_chunk_id(&chunk_it, chunk_id); ts_scan_iterator_start_or_restart_scan(&chunk_it); ti = ts_scan_iterator_next(&chunk_it); Assert(ti != NULL); Chunk *chunk = MemoryContextAllocZero(orig_mcxt, sizeof(Chunk)); ts_chunk_formdata_fill(&chunk->fd, ti); chunk->constraints = NULL; chunk->cube = NULL; chunk->hypertable_relid = hs->main_table_relid; chunk->table_id = chunk_reloid; locked_chunks[locked_chunk_count] = chunk; locked_chunk_count++; /* Only one chunk should match */ Assert(ts_scan_iterator_next(&chunk_it) == NULL); } ts_scan_iterator_close(&chunk_it); Assert(locked_chunk_count == 0 || locked_chunks != NULL); Assert(locked_chunk_count <= list_length(chunk_ids)); Assert(CurrentMemoryContext == work_mcxt); for (int i = 0; i < locked_chunk_count; i++) { Chunk *chunk = locked_chunks[i]; chunk->relkind = get_rel_relkind(chunk->table_id); } /* * Fetch the chunk constraints. */ ScanIterator constr_it = ts_chunk_constraint_scan_iterator_create(orig_mcxt); for (int i = 0; i < locked_chunk_count; i++) { Chunk *chunk = locked_chunks[i]; chunk->constraints = ts_chunk_constraints_alloc(/* size_hint = */ 0, orig_mcxt); ts_chunk_constraint_scan_iterator_set_chunk_id(&constr_it, chunk->fd.id); ts_scan_iterator_start_or_restart_scan(&constr_it); while (ts_scan_iterator_next(&constr_it) != NULL) { TupleInfo *constr_ti = ts_scan_iterator_tuple_info(&constr_it); ts_chunk_constraints_add_from_tuple(chunk->constraints, constr_ti); } } ts_scan_iterator_close(&constr_it); /* * Build hypercubes for the chunks by finding and combining the dimension * slices that match the chunk constraints. */ ScanIterator slice_iterator = ts_dimension_slice_scan_iterator_create(NULL, orig_mcxt); for (int chunk_index = 0; chunk_index < locked_chunk_count; chunk_index++) { Chunk *chunk = locked_chunks[chunk_index]; ChunkConstraints *constraints = chunk->constraints; MemoryContextSwitchTo(orig_mcxt); Hypercube *cube = ts_hypercube_alloc(constraints->num_dimension_constraints); MemoryContextSwitchTo(work_mcxt); for (int constraint_index = 0; constraint_index < constraints->num_constraints; constraint_index++) { ChunkConstraint *constraint = &constraints->constraints[constraint_index]; if (!is_dimension_constraint(constraint)) { continue; } /* * Find the slice by id. Don't have to lock it because the chunk is * locked. */ const int slice_id = constraint->fd.dimension_slice_id; DimensionSlice *slice_ptr = ts_dimension_slice_scan_iterator_get_by_id(&slice_iterator, slice_id); if (slice_ptr == NULL) { elog(ERROR, "dimension slice %d is not found", slice_id); } MemoryContextSwitchTo(orig_mcxt); DimensionSlice *slice_copy = ts_dimension_slice_create(slice_ptr->fd.dimension_id, slice_ptr->fd.range_start, slice_ptr->fd.range_end); slice_copy->fd.id = slice_ptr->fd.id; MemoryContextSwitchTo(work_mcxt); Assert(cube->capacity > cube->num_slices); cube->slices[cube->num_slices++] = slice_copy; } if (cube->num_slices == 0) { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("chunk %s has no dimension slices", get_rel_name(chunk->table_id)))); } ts_hypercube_slice_sort(cube); chunk->cube = cube; } ts_scan_iterator_close(&slice_iterator); Assert(CurrentMemoryContext == work_mcxt); MemoryContextSwitchTo(orig_mcxt); MemoryContextDelete(work_mcxt); #ifdef USE_ASSERT_CHECKING /* Assert that we always return valid chunks */ for (int i = 0; i < locked_chunk_count; i++) { ASSERT_IS_VALID_CHUNK(locked_chunks[i]); } #endif *num_chunks = locked_chunk_count; Assert(*num_chunks == 0 || locked_chunks != NULL); return locked_chunks; } ================================================ FILE: src/chunk_scan.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include "hypertable.h" extern Chunk **ts_chunk_scan_by_chunk_ids(const Hyperspace *hs, const List *chunk_ids, unsigned int *num_chunks); ================================================ FILE: src/chunk_tuple_routing.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/tableam.h> #include <storage/lockdefs.h> #include <utils/rls.h> #include "chunk_insert_state.h" #include "chunk_tuple_routing.h" #include "cross_module_fn.h" #include "debug_point.h" #include "guc.h" #include "hypercube.h" #include "subspace_store.h" ChunkTupleRouting * ts_chunk_tuple_routing_create(EState *estate, Hypertable *ht, ResultRelInfo *rri) { ChunkTupleRouting *ctr; Assert(ht); /* * Here we attempt to expend as little effort as possible in setting up * the ChunkTupleRouting. Each partition's ResultRelInfo is built on * demand, only when we actually need to route a tuple to that partition. * The reason for this is that a common case is for INSERT to insert a * single tuple into a partitioned table and this must be fast. */ ctr = (ChunkTupleRouting *) palloc0(sizeof(ChunkTupleRouting)); ctr->root_rri = rri; ctr->root_rel = rri->ri_RelationDesc; ctr->estate = estate; ctr->counters = palloc0(sizeof(SharedCounters)); ctr->hypertable = ht; /* * If the relid of ResultRelInfo does not match the Hypertable this is an operation * directly on a chunk. */ ctr->single_chunk_insert = ht->main_table_relid != RelationGetRelid(rri->ri_RelationDesc); ctr->subspace = ts_subspace_store_init(ctr->hypertable->space, estate->es_query_cxt, ts_guc_max_open_chunks_per_insert); ctr->has_dropped_attrs = false; return ctr; } void ts_chunk_tuple_routing_destroy(ChunkTupleRouting *ctr) { ts_subspace_store_free(ctr->subspace); pfree(ctr); } static void destroy_chunk_insert_state_single_chunk(void *cis) { ts_chunk_insert_state_destroy((ChunkInsertState *) cis, true); } static void destroy_chunk_insert_state(void *cis) { ts_chunk_insert_state_destroy((ChunkInsertState *) cis, false); } extern ChunkInsertState * ts_chunk_tuple_routing_find_chunk(ChunkTupleRouting *ctr, Point *point) { Chunk *chunk = NULL; ChunkInsertState *cis = NULL; cis = ts_subspace_store_get(ctr->subspace, point); /* * The chunk search functions may leak memory, so switch to a temporary * memory context. */ MemoryContext old_context = MemoryContextSwitchTo(GetPerTupleMemoryContext(ctr->estate)); if (!cis) { bool chunk_created = false; bool needs_partial = false; const LOCKMODE lockmode = RowExclusiveLock; /* * Normally, for every row of the chunk except the first one, we expect * the chunk to exist already. The "create" function would take a lock * on the hypertable to serialize the concurrent chunk creation. Here we * first use the "find" function to try to find the chunk without * locking the hypertable. This serves as a fast path for the usual case * where the chunk already exists. */ DEBUG_WAITPOINT("chunk_insert_before_lock"); chunk = ts_hypertable_find_chunk_for_point(ctr->hypertable, point, lockmode); /* * When inserting directly into a chunk, we should always find the chunk and * the returned chunk should match the relid we are inserting into. */ if (ctr->single_chunk_insert) { if (!chunk || chunk->table_id != RelationGetRelid(ctr->root_rri->ri_RelationDesc)) ereport(ERROR, (errcode(ERRCODE_CHECK_VIOLATION), errmsg("new row for relation \"%s\" violates chunk constraint", RelationGetRelationName(ctr->root_rri->ri_RelationDesc)))); } if (chunk && ts_chunk_is_frozen(chunk)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot INSERT into frozen chunk \"%s\"", get_rel_name(chunk->table_id)))); if (chunk && IS_OSM_CHUNK(chunk)) { const Dimension *time_dim = hyperspace_get_open_dimension(ctr->hypertable->space, 0); Assert(time_dim != NULL); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("Cannot insert into tiered chunk range of %s.%s - attempt to create " "new chunk " "with range [%s %s] failed", NameStr(ctr->hypertable->fd.schema_name), NameStr(ctr->hypertable->fd.table_name), ts_internal_to_time_string(chunk->cube->slices[0]->fd.range_start, time_dim->fd.column_type), ts_internal_to_time_string(chunk->cube->slices[0]->fd.range_end, time_dim->fd.column_type)), errhint( "Hypertable has tiered data with time range that overlaps the insert"))); } if (!chunk) { chunk = ts_hypertable_create_chunk_for_point(ctr->hypertable, point, lockmode); chunk_created = true; } Ensure(chunk, "no chunk found or created"); #ifdef USE_ASSERT_CHECKING /* Ensure we always hold a lock on the chunk table at this point */ Relation chunk_rel = RelationIdGetRelation(chunk->table_id); Assert(CheckRelationLockedByMe(chunk_rel, lockmode, true)); RelationClose(chunk_rel); #endif if (ctr->create_compressed_chunk && !chunk->fd.compressed_chunk_id) { /* * When creating a compressed chunk, the operation must be * synchronized with other operations. A RowExclusiveLock is * already held on the chunk table itself so it will conflict with * explicit compress calls like compress_chunk() or * convert_to_columnstore() that take at least * ExclusiveLock. However, it is also necessary to synchronize * with other concurrent inserts doing the same thing. * * We don't want to do a lock upgrade on the chunk table since * that increases the risk of deadlocks. * * Instead we synchronize around a tuple lock on the chunk * metadata row since this is the row getting updated with new * compression status. */ TM_Result lockres; DEBUG_WAITPOINT("insert_create_compressed"); lockres = ts_chunk_lock_for_creating_compressed_chunk(chunk->fd.id, &chunk->fd.compressed_chunk_id); /* * Since the locking function blocks and follows the update chain, * the only reasonable return value is TM_Ok. Everything else is * an error. */ Ensure(lockres == TM_Ok, "could not lock chunk row for creating " "compressed chunk. Lock result %d", lockres); /* recheck whether compressed chunk exists after acquiring the lock */ if (!chunk->fd.compressed_chunk_id) { Hypertable *compressed_ht = ts_hypertable_get_by_id(ctr->hypertable->fd.compressed_hypertable_id); Chunk *compressed_chunk = ts_cm_functions->compression_chunk_create(compressed_ht, chunk); ts_chunk_set_compressed_chunk(chunk, compressed_chunk->fd.id); chunk->fd.compressed_chunk_id = compressed_chunk->fd.id; /* mark chunk as partial unless completely new chunk */ if (!chunk_created) needs_partial = true; } } cis = ts_chunk_insert_state_create(chunk->table_id, ctr); cis->needs_partial = needs_partial; ts_subspace_store_add(ctr->subspace, chunk->cube, cis, ctr->single_chunk_insert ? destroy_chunk_insert_state_single_chunk : destroy_chunk_insert_state); } MemoryContextSwitchTo(old_context); Assert(cis != NULL); return cis; } extern void ts_chunk_tuple_routing_decompress_for_insert(ChunkInsertState *cis, ResultRelInfo *root_rri, TupleTableSlot *slot, EState *estate, bool update_counter) { if (!cis->chunk_compressed || (cis->cached_decompression_state && !cis->cached_decompression_state->has_primary_or_unique_index)) return; /* * If this is an INSERT into a compressed chunk with UNIQUE or * PRIMARY KEY constraints we need to make sure any batches that could * potentially lead to a conflict are in the decompressed chunk so * postgres can do proper constraint checking. */ ts_cm_functions->init_decompress_state_for_insert(cis, slot); /* If we are dealing with generated stored columns, generate the values * so can use it for uniqueness checks. */ Relation resultRelationDesc = cis->rel; if (resultRelationDesc->rd_att->constr && resultRelationDesc->rd_att->constr->has_generated_stored) { slot->tts_tableOid = RelationGetRelid(resultRelationDesc); ExecComputeStoredGenerated(root_rri, estate, slot, CMD_INSERT); } ts_cm_functions->decompress_batches_for_insert(cis, slot); /* mark rows visible */ if (update_counter) estate->es_output_cid = GetCurrentCommandId(true); if (ts_guc_max_tuples_decompressed_per_dml > 0) { if (cis->counters->tuples_decompressed > ts_guc_max_tuples_decompressed_per_dml) { ereport(ERROR, (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED), errmsg("tuple decompression limit exceeded by operation"), errdetail("current limit: %d, tuples decompressed: %lld", ts_guc_max_tuples_decompressed_per_dml, (long long int) cis->counters->tuples_decompressed), errhint("Consider increasing " "timescaledb.max_tuples_decompressed_per_dml_transaction or set " "to 0 (unlimited)."))); } } } ================================================ FILE: src/chunk_tuple_routing.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <executor/nodeModifyTable.h> #include "chunk_insert_state.h" #include "hypertable.h" typedef struct ModifyHypertableState ModifyHypertableState; typedef struct ChunkTupleRouting { Hypertable *hypertable; /* * When single_chunk_insert is true, root_rel and root_rri point to the * chunk being inserted into. Otherwise, they point to the hypertable. */ Relation root_rel; ResultRelInfo *root_rri; bool single_chunk_insert; SubspaceStore *subspace; EState *estate; bool create_compressed_chunk; bool has_dropped_attrs; ModifyHypertableState *mht_state; /* state for the ModifyHypertable custom scan node */ ChunkInsertState *cis; SharedCounters *counters; /* shared counters for the current statement */ } ChunkTupleRouting; ChunkTupleRouting *ts_chunk_tuple_routing_create(EState *estate, Hypertable *ht, ResultRelInfo *rri); void ts_chunk_tuple_routing_destroy(ChunkTupleRouting *ctr); ChunkInsertState *ts_chunk_tuple_routing_find_chunk(ChunkTupleRouting *ctr, Point *point); extern void ts_chunk_tuple_routing_decompress_for_insert(ChunkInsertState *cis, ResultRelInfo *root_rri, TupleTableSlot *slot, EState *estate, bool update_counter); ================================================ FILE: src/compat/CMakeLists.txt ================================================ set(SOURCES) target_sources(${PROJECT_NAME} PRIVATE ${SOURCES}) ================================================ FILE: src/compat/compat-msvc-enter.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> /* * Not all exported data symbols in PostgreSQL are marked with PGDLLIMPORT, * which causes errors during linking. This hack turns all extern symbols into * properly exported symbols so we can use them in our code. Only necessary * for files that use these incorrectly unlabeled data symbols (e.g. extension.c) * * NOTE: Applies to data symbols only, not functions */ #ifdef _MSC_VER #undef PGDLLIMPORT #define PGDLLIMPORT #define extern extern _declspec(dllimport) #include <catalog/genbki.h> #undef DECLARE_TOAST #undef DECLARE_INDEX #undef DECLARE_UNIQUE_INDEX #undef DECLARE_UNIQUE_INDEX_PKEY #undef DECLARE_FOREIGN_KEY #undef DECLARE_FOREIGN_KEY_OPT #undef DECLARE_ARRAY_FOREIGN_KEY #undef DECLARE_ARRAY_FOREIGN_KEY_OPT #define DECLARE_TOAST(name, toastoid, indexoid) #define DECLARE_INDEX(name, oid, decl) #define DECLARE_UNIQUE_INDEX(name, oid, decl) #define DECLARE_UNIQUE_INDEX_PKEY(name, oid, decl) #define DECLARE_FOREIGN_KEY(cols, reftbl, refcols) #define DECLARE_FOREIGN_KEY_OPT(cols, reftbl, refcols) #define DECLARE_ARRAY_FOREIGN_KEY(cols, reftbl, refcols) #define DECLARE_ARRAY_FOREIGN_KEY_OPT(cols, reftbl, refcols) #endif /* _MSC_VER */ ================================================ FILE: src/compat/compat-msvc-exit.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once /* * Included after all files that need compatibility are included, this undoes * the 'extern' macro so as not to break other headers (e.g. Windows headers). */ #ifdef _MSC_VER #undef extern #undef PGDLLIMPORT #define PGDLLIMPORT __declspec(dllexport) #endif /* _MSC_VER */ ================================================ FILE: src/compat/compat.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <commands/cluster.h> #include <commands/defrem.h> #include <commands/explain.h> #include <commands/trigger.h> #include <commands/vacuum.h> #include <executor/executor.h> #include <executor/tuptable.h> #include <nodes/execnodes.h> #include <nodes/nodes.h> #include <optimizer/restrictinfo.h> #include <pgstat.h> #include <storage/lmgr.h> #include <utils/jsonb.h> #include <utils/lsyscache.h> #include <utils/rel.h> #include "export.h" #define PG_MAJOR_MIN 15 /* * Prevent building against upstream versions that had ABI breaking change (15.9, 16.5, 17.1) * that was reverted in the following release. */ #define is_supported_pg_version_15(version) ((version >= 150010) && (version < 160000)) #define is_supported_pg_version_16(version) ((version >= 160006) && (version < 170000)) #define is_supported_pg_version_17(version) ((version >= 170002) && (version < 180000)) #define is_supported_pg_version_18(version) ((version >= 180000) && (version < 190000)) /* * To compile with an unsupported version, use -DEXPERIMENTAL=ON with cmake. * (Useful when testing with unreleased versions) */ #define is_supported_pg_version(version) \ (is_supported_pg_version_15(version) || is_supported_pg_version_16(version) || \ is_supported_pg_version_17(version) || is_supported_pg_version_18(version)) #define PG15 is_supported_pg_version_15(PG_VERSION_NUM) #define PG16 is_supported_pg_version_16(PG_VERSION_NUM) #define PG17 is_supported_pg_version_17(PG_VERSION_NUM) #define PG18 is_supported_pg_version_18(PG_VERSION_NUM) #define PG15_LT (PG_VERSION_NUM < 150000) #define PG15_GE (PG_VERSION_NUM >= 150000) #define PG16_LT (PG_VERSION_NUM < 160000) #define PG16_GE (PG_VERSION_NUM >= 160000) #define PG17_LT (PG_VERSION_NUM < 170000) #define PG17_GE (PG_VERSION_NUM >= 170000) #define PG18_LT (PG_VERSION_NUM < 180000) #define PG18_GE (PG_VERSION_NUM >= 180000) #if !(is_supported_pg_version(PG_VERSION_NUM)) #error "Unsupported PostgreSQL version" #endif #if ((PG_VERSION_NUM >= 150009 && PG_VERSION_NUM < 160000) || \ (PG_VERSION_NUM >= 160005 && PG_VERSION_NUM < 170000) || (PG_VERSION_NUM >= 170001)) /* * The above versions introduced a fix for potentially losing updates to * pg_class and pg_database due to inplace updates done to those catalog * tables by PostgreSQL. The fix requires taking a lock on the tuple via * SearchSysCacheLocked1(). For older PG versions, we just map the new * function to the unlocked version and the unlocking of the tuple is a noop. * * https://github.com/postgres/postgres/commit/3b7a689e1a805c4dac2f35ff14fd5c9fdbddf150 * * Here's an excerpt from README.tuplock that explains the need for additional * tuple locks: * * If IsInplaceUpdateRelation() returns true for a table, the table is a * system catalog that receives systable_inplace_update_begin() calls. * Preparing a heap_update() of these tables follows additional locking rules, * to ensure we don't lose the effects of an inplace update. In particular, * consider a moment when a backend has fetched the old tuple to modify, not * yet having called heap_update(). Another backend's inplace update starting * then can't conclude until the heap_update() places its new tuple in a * buffer. We enforce that using locktags as follows. While DDL code is the * main audience, the executor follows these rules to make e.g. "MERGE INTO * pg_class" safer. Locking rules are per-catalog: * * pg_class heap_update() callers: before copying the tuple to modify, take a * lock on the tuple, a ShareUpdateExclusiveLock on the relation, or a * ShareRowExclusiveLock or stricter on the relation. */ #define SYSCACHE_TUPLE_LOCK_NEEDED 1 #define AssertSufficientPgClassUpdateLockHeld(relid) \ Assert(CheckRelationOidLockedByMe(relid, ShareUpdateExclusiveLock, false) || \ CheckRelationOidLockedByMe(relid, ShareRowExclusiveLock, true)); #define UnlockSysCacheTuple(rel, tid) UnlockTuple(rel, tid, InplaceUpdateTupleLock); #else #define SearchSysCacheLockedCopy1(rel, datum) SearchSysCacheCopy1(rel, datum) #define UnlockSysCacheTuple(rel, tid) #define AssertSufficientPgClassUpdateLockHeld(relid) #endif /* * The following are compatibility functions for different versions of * PostgreSQL. Each compatibility function (or group) has its own logic for * which versions get different behavior and is separated from others by a * comment with its name and any clarifying notes about supported behavior. Each * compatibility define is separated out by function so that it is easier to see * what changed about its behavior, and at what version, but closely related * functions that changed at the same time may be grouped together into a single * block. Compatibility functions are organized in alphabetical order. * * Wherever reasonable, we try to achieve forwards compatibility so that we can * take advantage of features added in newer PG versions. This avoids some * future tech debt, though may not always be possible. * * We append "compat" to the name of the function or define if we change the behavior * of something that existed in a previous version. If we are merely backpatching * behavior from a later version to an earlier version and not changing the * behavior of the new version we simply adopt the new version's name. */ #if PG16_LT #define ExecInsertIndexTuplesCompat(rri, \ slot, \ estate, \ update, \ noDupErr, \ specConflict, \ arbiterIndexes, \ onlySummarizing) \ ExecInsertIndexTuples(rri, slot, estate, update, noDupErr, specConflict, arbiterIndexes) #else #define ExecInsertIndexTuplesCompat(rri, \ slot, \ estate, \ update, \ noDupErr, \ specConflict, \ arbiterIndexes, \ onlySummarizing) \ ExecInsertIndexTuples(rri, \ slot, \ estate, \ update, \ noDupErr, \ specConflict, \ arbiterIndexes, \ onlySummarizing) #endif /* * PG16 removed outerjoin_delayed, nullable_relids arguments from make_restrictinfo * https://github.com/postgres/postgres/commit/b448f1c8d8 * * PG16 adds three new parameter - has_clone, is_clone and incompatible_relids, as a * part of fixing the filtering of "cloned" outer-join quals * https://github.com/postgres/postgres/commit/991a3df227 */ #if PG16_LT #define make_restrictinfo_compat(root, \ clause, \ is_pushed_down, \ has_clone, \ is_clone, \ outerjoin_delayed, \ pseudoconstant, \ security_level, \ required_relids, \ incompatible_relids, \ outer_relids, \ nullable_relids) \ make_restrictinfo(root, \ clause, \ is_pushed_down, \ outerjoin_delayed, \ pseudoconstant, \ security_level, \ required_relids, \ outer_relids, \ nullable_relids) #else #define make_restrictinfo_compat(root, \ clause, \ is_pushed_down, \ has_clone, \ is_clone, \ outerjoin_delayed, \ pseudoconstant, \ security_level, \ required_relids, \ incompatible_relids, \ outer_relids, \ nullable_relids) \ make_restrictinfo(root, \ clause, \ is_pushed_down, \ has_clone, \ is_clone, \ pseudoconstant, \ security_level, \ required_relids, \ incompatible_relids, \ outer_relids) #endif /* fmgr * In a9c35cf postgres changed how it calls SQL functions so that the number of * argument-slots allocated is chosen dynamically, instead of being fixed. This * change was ABI-breaking, so we cannot backport this optimization, however, * we do backport the interface, so that all our code will be compatible with * new versions. */ /* convenience macro to allocate FunctionCallInfoData on the heap */ #define HEAP_FCINFO(nargs) palloc(SizeForFunctionCallInfo(nargs)) /* getting arguments has a different API, so these macros unify the versions */ #define FC_ARG(fcinfo, n) ((fcinfo)->args[(n)].value) #define FC_NULL(fcinfo, n) ((fcinfo)->args[(n)].isnull) #define FC_FN_OID(fcinfo) ((fcinfo)->flinfo->fn_oid) /* convenience setters */ #define FC_SET_ARG(fcinfo, n, val) \ do \ { \ short _n = (n); \ FunctionCallInfo _fcinfo = (fcinfo); \ FC_ARG(_fcinfo, _n) = (val); \ FC_NULL(_fcinfo, _n) = false; \ } while (0) #define FC_SET_NULL(fcinfo, n) \ do \ { \ short _n = (n); \ FunctionCallInfo _fcinfo = (fcinfo); \ FC_ARG(_fcinfo, _n) = 0; \ FC_NULL(_fcinfo, _n) = true; \ } while (0) /* create this function for symmetry with pq_sendint32 */ #define pq_getmsgint32(buf) pq_getmsgint(buf, 4) #define ts_tuptableslot_set_table_oid(slot, table_oid) (slot)->tts_tableOid = table_oid static inline ClusterParams * get_cluster_options(const ClusterStmt *stmt) { ListCell *lc; ClusterParams *params = palloc0(sizeof(ClusterParams)); bool verbose = false; /* Parse option list */ foreach (lc, stmt->params) { DefElem *opt = (DefElem *) lfirst(lc); if (strcmp(opt->defname, "verbose") == 0) verbose = defGetBoolean(opt); else ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("unrecognized CLUSTER option \"%s\"", opt->defname), parser_errposition(NULL, opt->location))); } params->options = (verbose ? CLUOPT_VERBOSE : 0); return params; } #include <catalog/index.h> static inline int get_reindex_options(ReindexStmt *stmt) { ListCell *lc; bool concurrently = false; bool verbose = false; /* Parse option list */ foreach (lc, stmt->params) { DefElem *opt = (DefElem *) lfirst(lc); if (strcmp(opt->defname, "verbose") == 0) verbose = defGetBoolean(opt); else if (strcmp(opt->defname, "concurrently") == 0) concurrently = defGetBoolean(opt); else ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("unrecognized REINDEX option \"%s\"", opt->defname), parser_errposition(NULL, opt->location))); } return (verbose ? REINDEXOPT_VERBOSE : 0) | (concurrently ? REINDEXOPT_CONCURRENTLY : 0); } /* * define some list macros for convenience */ #define lfifth(l) lfirst(list_nth_cell(l, 4)) #define lfifth_int(l) lfirst_int(list_nth_cell(l, 4)) #if PG16_LT /* * PG15 consolidate VACUUM xid cutoff logic. * * https://github.com/postgres/postgres/commit/efa4a946 * * PG16 introduced VacuumCutoffs so define here for previous PG versions. */ struct VacuumCutoffs { TransactionId relfrozenxid; MultiXactId relminmxid; TransactionId OldestXmin; MultiXactId OldestMxact; TransactionId FreezeLimit; MultiXactId MultiXactCutoff; }; static inline bool vacuum_get_cutoffs(Relation rel, const VacuumParams *params, struct VacuumCutoffs *cutoffs) { return vacuum_set_xid_limits(rel, 0, 0, 0, 0, &cutoffs->OldestXmin, &cutoffs->OldestMxact, &cutoffs->FreezeLimit, &cutoffs->MultiXactCutoff); } #endif /* * PG16 adds TMResult argument to ExecBRUpdateTriggers * https://github.com/postgres/postgres/commit/7103ebb7 * this was backported to PG15 in * https://github.com/postgres/postgres/commit/7d9a75713ab9 */ #if PG15 #define ExecBRUpdateTriggers(estate, \ epqstate, \ resultRelInfo, \ tupleid, \ oldtuple, \ slot, \ result, \ tmfdp) \ ExecBRUpdateTriggersNew(estate, epqstate, resultRelInfo, tupleid, oldtuple, slot, result, tmfdp) #endif /* * PG16 adds TMResult argument to ExecBRDeleteTriggers * https://github.com/postgres/postgres/commit/9321c79c * this was backported to PG15 in * https://github.com/postgres/postgres/commit/7d9a75713ab9 */ #if PG15 #define ExecBRDeleteTriggers(estate, \ epqstate, \ relinfo, \ tupleid, \ fdw_trigtuple, \ epqslot, \ tmresult, \ tmfd) \ ExecBRDeleteTriggersNew(estate, \ epqstate, \ relinfo, \ tupleid, \ fdw_trigtuple, \ epqslot, \ tmresult, \ tmfd) #endif #if PG16_GE #define pgstat_get_local_beentry_by_index_compat(idx) pgstat_get_local_beentry_by_index(idx) #else #define pgstat_get_local_beentry_by_index_compat(idx) pgstat_fetch_stat_local_beentry(idx) #endif /* * PG16 adds a new parameter to DefineIndex, total_parts, that takes * in the total number of direct and indirect partitions of the relation. * * https://github.com/postgres/postgres/commit/27f5c712 */ #if PG16_LT #define DefineIndexCompat(relationId, \ stmt, \ indexRelationId, \ parentIndexId, \ parentConstraintId, \ total_parts, \ is_alter_table, \ check_rights, \ check_not_in_use, \ skip_build, \ quiet) \ DefineIndex(relationId, \ stmt, \ indexRelationId, \ parentIndexId, \ parentConstraintId, \ is_alter_table, \ check_rights, \ check_not_in_use, \ skip_build, \ quiet) #else #define DefineIndexCompat(relationId, \ stmt, \ indexRelationId, \ parentIndexId, \ parentConstraintId, \ total_parts, \ is_alter_table, \ check_rights, \ check_not_in_use, \ skip_build, \ quiet) \ DefineIndex(relationId, \ stmt, \ indexRelationId, \ parentIndexId, \ parentConstraintId, \ total_parts, \ is_alter_table, \ check_rights, \ check_not_in_use, \ skip_build, \ quiet) #endif #if PG16_LT #include <catalog/pg_database_d.h> #include <catalog/pg_foreign_server_d.h> #include <catalog/pg_namespace_d.h> #include <catalog/pg_proc_d.h> #include <catalog/pg_tablespace_d.h> #include <utils/acl.h> /* * PG16 replaces most aclcheck functions with a common object_aclcheck() function * https://github.com/postgres/postgres/commit/c727f511 */ static inline AclResult object_aclcheck(Oid classid, Oid objectid, Oid roleid, AclMode mode) { switch (classid) { case DatabaseRelationId: return pg_database_aclcheck(objectid, roleid, mode); case ForeignServerRelationId: return pg_foreign_server_aclcheck(objectid, roleid, mode); case NamespaceRelationId: return pg_namespace_aclcheck(objectid, roleid, mode); case ProcedureRelationId: return pg_proc_aclcheck(objectid, roleid, mode); case TableSpaceRelationId: return pg_tablespace_aclcheck(objectid, roleid, mode); default: Assert(false); } return ACLCHECK_NOT_OWNER; } /* * PG16 replaces pg_foo_ownercheck() functions with a common object_ownercheck() function * https://github.com/postgres/postgres/commit/afbfc029 */ static inline bool object_ownercheck(Oid classid, Oid objectid, Oid roleid) { switch (classid) { case RelationRelationId: return pg_class_ownercheck(objectid, roleid); default: Assert(false); } return false; } #endif #if PG17_LT /* * Backport of RestrictSearchPath() from PG17 * * We skip the check for IsBootstrapProcessingMode() since it creates problems * on Windows builds and we don't need it for our use case. */ #include <utils/guc.h> static inline void RestrictSearchPath(void) { set_config_option("search_path", "pg_catalog, pg_temp", PGC_USERSET, PGC_S_SESSION, GUC_ACTION_SAVE, true, 0, false); } #endif #if PG17_LT /* This macro was renamed in PG17, see 414f6c0fb79a */ #define WAIT_EVENT_MESSAGE_QUEUE_INTERNAL WAIT_EVENT_MQ_INTERNAL /* 'flush' argument was added in 173b56f1ef59 */ #define LogLogicalMessageCompat(prefix, message, size, transactional, flush) \ LogLogicalMessage(prefix, message, size, transactional) /* 'stmt' argument was added in f21848de2013 */ #define reindex_relation_compat(stmt, relid, flags, params) reindex_relation(relid, flags, params) /* 'vacuum_is_relation_owner' was renamed to 'vacuum_is_permitted_for_relation' in ecb0fd33720f */ #define vacuum_is_permitted_for_relation_compat(relid, reltuple, options) \ vacuum_is_relation_owner(relid, reltuple, options) /* * 'BackendIdGetProc' was renamed to 'ProcNumberGetProc' in 024c52111757. * Also 'backendId' was renamed to 'procNumber' */ #define VirtualTransactionGetProcCompat(vxid) BackendIdGetProc((vxid)->backendId) /* * 'opclassOptions' argument was added in 784162357130. * Previously indexInfo->ii_OpclassOptions was used instead. * On top of that 'stattargets' argument was added in 6a004f1be87d. */ #define index_create_compat(heapRelation, \ indexRelationName, \ indexRelationId, \ parentIndexRelid, \ parentConstraintId, \ relFileNumber, \ indexInfo, \ indexColNames, \ accessMethodId, \ tableSpaceId, \ collationIds, \ opclassIds, \ opclassOptions, \ coloptions, \ stattargets, \ reloptions, \ flags, \ constr_flags, \ allow_system_table_mods, \ is_internal, \ constraintId) \ index_create(heapRelation, \ indexRelationName, \ indexRelationId, \ parentIndexRelid, \ parentConstraintId, \ relFileNumber, \ indexInfo, \ indexColNames, \ accessMethodId, \ tableSpaceId, \ collationIds, \ opclassIds, \ coloptions, \ reloptions, \ flags, \ constr_flags, \ allow_system_table_mods, \ is_internal, \ constraintId) #else /* PG17_GE */ #define LogLogicalMessageCompat(prefix, message, size, transactional, flush) \ LogLogicalMessage(prefix, message, size, transactional, flush) #define reindex_relation_compat(stmt, relid, flags, params) \ reindex_relation(stmt, relid, flags, params) #define vacuum_is_permitted_for_relation_compat(relid, reltuple, options) \ vacuum_is_permitted_for_relation(relid, reltuple, options) #define VirtualTransactionGetProcCompat(vxid) ProcNumberGetProc(vxid->procNumber) #define index_create_compat(heapRelation, \ indexRelationName, \ indexRelationId, \ parentIndexRelid, \ parentConstraintId, \ relFileNumber, \ indexInfo, \ indexColNames, \ accessMethodId, \ tableSpaceId, \ collationIds, \ opclassIds, \ opclassOptions, \ coloptions, \ stattargets, \ reloptions, \ flags, \ constr_flags, \ allow_system_table_mods, \ is_internal, \ constraintId) \ index_create(heapRelation, \ indexRelationName, \ indexRelationId, \ parentIndexRelid, \ parentConstraintId, \ relFileNumber, \ indexInfo, \ indexColNames, \ accessMethodId, \ tableSpaceId, \ collationIds, \ opclassIds, \ opclassOptions, \ coloptions, \ stattargets, \ reloptions, \ flags, \ constr_flags, \ allow_system_table_mods, \ is_internal, \ constraintId) #endif #if PG17_LT /* 'mergeActions' argument was added in 5f2e179bd31e */ #define CheckValidResultRelCompat(resultRelInfo, operation, onConflictAction, mergeActions) \ CheckValidResultRel(resultRelInfo, operation) #elif PG18_LT #define CheckValidResultRelCompat(resultRelInfo, operation, onConflictAction, mergeActions) \ CheckValidResultRel(resultRelInfo, operation, mergeActions) #else #define CheckValidResultRelCompat(resultRelInfo, operation, onConflictAction, mergeActions) \ CheckValidResultRel(resultRelInfo, operation, onConflictAction, mergeActions) #endif #if PG17_LT /* * Overflow-aware comparison functions to be used in qsort. Introduced in PG * 17 and included here for older PG versions. */ static inline int pg_cmp_u32(uint32 a, uint32 b) { return (a > b) - (a < b); } #endif #if PG16_LT /* * Similarly, wrappers around labs()/llabs() matching our int64. * * Introduced on PG16: * https://github.com/postgres/postgres/commit/357cfefb09115292cfb98d504199e6df8201c957 */ #ifdef HAVE_LONG_INT_64 #define i64abs(i) labs(i) #else #define i64abs(i) llabs(i) #endif #endif /* * PG18 adds IndexScanInstrumentation parameter to index_beginscan * https://github.com/postgres/postgres/commit/0fbceae8 */ #if PG18_LT #define index_beginscan_compat(heapRelation, \ indexRelation, \ snapshot, \ instrument, \ nkeys, \ norderbys) \ index_beginscan(heapRelation, indexRelation, snapshot, nkeys, norderbys) #else #define index_beginscan_compat(heapRelation, \ indexRelation, \ snapshot, \ instrument, \ nkeys, \ norderbys) \ index_beginscan(heapRelation, indexRelation, snapshot, instrument, nkeys, norderbys) #endif #if PG16_LT #define make_range_compat(typcache, lower, upper, empty, escontext) \ make_range(typcache, lower, upper, empty) #else #define make_range_compat(typcache, lower, upper, empty, escontext) \ make_range(typcache, lower, upper, empty, escontext) #endif /* Copied from PG17. We can remove it once we deprecate older versions. */ #if PG17_LT static inline void initReadOnlyStringInfo(StringInfo str, char *data, int len) { str->data = data; str->len = len; str->maxlen = 0; /* read-only */ str->cursor = 0; } #endif /* * PG18 renames ri_ConstraintExprs to ri_CheckConstraintExprs * Add macros so we can use the new naming for older versions. * https://github.com/postgres/postgres/commit/9a9ead11 */ #if PG18_LT #define ri_CheckConstraintExprs ri_ConstraintExprs #endif /* * PG18 renames ec_derives to ec_derives_list * Add macros so we can use the new naming for older versions. * https://github.com/postgres/postgres/commit/88f55bc9 */ #if PG18_LT #define ec_derives_list ec_derives #endif /* PG18 introduces new CompareType for ordering operations * Add macros so we can use the new naming for older versions. * https://github.com/postgres/postgres/commit/8123e91f */ #if PG18_LT #define CompareType int16 #define COMPARE_LT BTLessStrategyNumber #define COMPARE_GT BTGreaterStrategyNumber #define pk_cmptype pk_strategy #endif /* PG18 adds is_merge_delete param to ExecBR{Delete|Update}Triggers function. * This has been backported to 17.6 but with a new name (ExecBR{Delete|Update}TriggersNew)j * Add compat function to cover 3 versions (pre 17.6, 17.6 - 18, post 18) * https://github.com/postgres/postgres/commit/5022ff25 */ #if PG_VERSION_NUM < 170006 #define ExecBRDeleteTriggersCompat(estate, \ epqstate, \ relinfo, \ tupleid, \ fdw_trigtuple, \ epqslot, \ tmresult, \ tmfd, \ is_merge_delete) \ ExecBRDeleteTriggers(estate, epqstate, relinfo, tupleid, fdw_trigtuple, epqslot, tmresult, tmfd) #define ExecBRUpdateTriggersCompat(estate, \ epqstate, \ relinfo, \ tupleid, \ fdw_trigtuple, \ epqslot, \ tmresult, \ tmfd, \ is_merge_delete) \ ExecBRUpdateTriggers(estate, epqstate, relinfo, tupleid, fdw_trigtuple, epqslot, tmresult, tmfd) #endif #if PG_VERSION_NUM >= 170006 && PG_VERSION_NUM < 180000 #define ExecBRDeleteTriggersCompat(estate, \ epqstate, \ relinfo, \ tupleid, \ fdw_trigtuple, \ epqslot, \ tmresult, \ tmfd, \ is_merge_delete) \ ExecBRDeleteTriggersNew(estate, \ epqstate, \ relinfo, \ tupleid, \ fdw_trigtuple, \ epqslot, \ tmresult, \ tmfd, \ is_merge_delete) #define ExecBRUpdateTriggersCompat(estate, \ epqstate, \ relinfo, \ tupleid, \ fdw_trigtuple, \ epqslot, \ tmresult, \ tmfd, \ is_merge_delete) \ ExecBRUpdateTriggersNew(estate, \ epqstate, \ relinfo, \ tupleid, \ fdw_trigtuple, \ epqslot, \ tmresult, \ tmfd, \ is_merge_delete) #endif #if PG18_GE #define ExecBRDeleteTriggersCompat(estate, \ epqstate, \ relinfo, \ tupleid, \ fdw_trigtuple, \ epqslot, \ tmresult, \ tmfd, \ is_merge_delete) \ ExecBRDeleteTriggers(estate, \ epqstate, \ relinfo, \ tupleid, \ fdw_trigtuple, \ epqslot, \ tmresult, \ tmfd, \ is_merge_delete) #define ExecBRUpdateTriggersCompat(estate, \ epqstate, \ relinfo, \ tupleid, \ fdw_trigtuple, \ epqslot, \ tmresult, \ tmfd, \ is_merge_delete) \ ExecBRUpdateTriggers(estate, \ epqstate, \ relinfo, \ tupleid, \ fdw_trigtuple, \ epqslot, \ tmresult, \ tmfd, \ is_merge_delete) #endif /* PG16 removes create_new_ph parameter from add_vars_to_targetlist * https://github.com/postgres/postgres/commit/2489d76c4906 */ #if PG16_LT #define add_vars_to_targetlist_compat(root, vars, where_needed) \ add_vars_to_targetlist(root, vars, where_needed, false) #else #define add_vars_to_targetlist_compat(root, vars, where_needed) \ add_vars_to_targetlist(root, vars, where_needed) #endif /* PG16 consolidates ItemPointer to datum functions so backported it to PG15 * https://github.com/postgres/postgres/commit/bd944884e92a */ #if PG16_LT static inline ItemPointer DatumGetItemPointer(Datum X) { return (ItemPointer) DatumGetPointer(X); } static inline Datum ItemPointerGetDatum(const ItemPointerData *X) { return PointerGetDatum(X); } #endif ================================================ FILE: src/config.h.in ================================================ #ifndef TIMESCALEDB_CONFIG_H #define TIMESCALEDB_CONFIG_H #define TIMESCALEDB_VERSION "@PROJECT_VERSION@" #define TIMESCALEDB_VERSION_MOD "@PROJECT_VERSION_MOD@" #define TIMESCALEDB_MAJOR_VERSION "@PROJECT_VERSION_MAJOR@" #define TIMESCALEDB_MINOR_VERSION "@PROJECT_VERSION_MINOR@" #define TIMESCALEDB_PATCH_VERSION "@PROJECT_VERSION_PATCH@" #define TIMESCALEDB_MOD_VERSION "@VERSION_MOD@" #define BUILD_OS_NAME "@CMAKE_SYSTEM_NAME@" #define BUILD_OS_VERSION "@CMAKE_SYSTEM_VERSION@" #define BUILD_PROCESSOR "@CMAKE_SYSTEM_PROCESSOR@" #define BUILD_POINTER_BYTES @CMAKE_SIZEOF_VOID_P@ /* * Value should be set in package release scripts. Otherwise * defaults to "source" */ #define TIMESCALEDB_INSTALL_METHOD "@PROJECT_INSTALL_METHOD@" /* Platform */ #ifndef WIN32 #cmakedefine WIN32 #endif #ifndef MSVC #cmakedefine MSVC #endif #ifndef UNIX #cmakedefine UNIX #endif #ifndef APPLE #cmakedefine APPLE #endif #ifndef DEBUG #cmakedefine DEBUG #endif #ifndef TS_DEBUG #cmakedefine TS_DEBUG #endif #ifndef USE_TELEMETRY #cmakedefine USE_TELEMETRY #endif #ifndef TELEMETRY_DEFAULT #cmakedefine TELEMETRY_DEFAULT @TELEMETRY_DEFAULT@ #endif /* Avoid conflicts with USE_OPENSSL defined by PostgreSQL */ #cmakedefine TS_USE_OPENSSL #endif /* TIMESCALEDB_CONFIG_H */ ================================================ FILE: src/constraint.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/genam.h> #include <access/table.h> #include <catalog/index.h> #include <catalog/indexing.h> #include <catalog/pg_constraint.h> #include <utils/builtins.h> #include <utils/fmgroids.h> #include <utils/rel.h> #include "constraint.h" #include "indexing.h" /* * Process constraints that belong to a given relation. * * Returns the number of constraints processed. */ int ts_constraint_process(Oid relid, constraint_func process_func, void *ctx) { ScanKeyData skey; Relation rel; SysScanDesc scan; HeapTuple htup; bool should_continue = true; int count = 0; ScanKeyInit(&skey, Anum_pg_constraint_conrelid, BTEqualStrategyNumber, F_OIDEQ, relid); rel = table_open(ConstraintRelationId, AccessShareLock); scan = systable_beginscan(rel, ConstraintRelidTypidNameIndexId, true, NULL, 1, &skey); while (HeapTupleIsValid(htup = systable_getnext(scan)) && should_continue) { switch (process_func(htup, ctx)) { case CONSTR_PROCESSED: count++; break; case CONSTR_PROCESSED_DONE: count++; should_continue = false; break; case CONSTR_IGNORED: break; case CONSTR_IGNORED_DONE: should_continue = false; break; } } systable_endscan(scan); table_close(rel, AccessShareLock); return count; } /* * Search for an existing constraint in the chunk that matches the given * hypertable constraint. This is used to avoid creating duplicate constraints * when creating chunks. * * Returns true if a matching constraint is found, false otherwise. */ Form_pg_constraint ts_constraint_find_matching(HeapTuple ht_tup, Relation chunk_rel) { ScanKeyData skey; Relation constraint_rel; SysScanDesc scan; HeapTuple chunk_tup; Form_pg_constraint ht_con = (Form_pg_constraint) GETSTRUCT(ht_tup); Form_pg_constraint chunk_con; Relation ht_rel = RelationIdGetRelation(ht_con->conrelid); Form_pg_constraint result = NULL; constraint_rel = table_open(ConstraintRelationId, RowExclusiveLock); /* Search for a constraint matching this one */ ScanKeyInit(&skey, Anum_pg_constraint_conrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(RelationGetRelid(chunk_rel))); scan = systable_beginscan(constraint_rel, ConstraintRelidTypidNameIndexId, true, NULL, 1, &skey); while (HeapTupleIsValid(chunk_tup = systable_getnext(scan))) { chunk_con = (Form_pg_constraint) GETSTRUCT(chunk_tup); /* Check constraints are handled by CreateInheritance() */ if (ht_con->contype != chunk_con->contype || ht_con->contype == CONSTRAINT_CHECK) continue; /* * Get the main name for chunk constraint and check whether it matches * the hypertable constraint name. If names match, then we compare indexes * to ensure that the constraints are equivalent. If the names do not * match, we assume that the constraints are not equivalent. */ if (ht_con->contype == CONSTRAINT_PRIMARY || ht_con->contype == CONSTRAINT_UNIQUE || ht_con->contype == CONSTRAINT_EXCLUSION) { if (ts_indexing_compare(ht_con->conindid, chunk_con->conindid)) { /* * pfree'ing this form is up to the caller. It is not expected to * consume a lot of memory, as only one form is allocated per * constraint. */ result = (Form_pg_constraint) palloc(sizeof(FormData_pg_constraint)); memcpy(result, chunk_con, sizeof(FormData_pg_constraint)); break; } } } systable_endscan(scan); table_close(constraint_rel, RowExclusiveLock); RelationClose(ht_rel); return result; } ================================================ FILE: src/constraint.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <access/htup.h> #include "export.h" /* * Return status for constraint processing function. * * PROCESSED - count the constraint as processed * IGNORED - the constraint wasn't processed * DONE - stop processing constraints */ typedef enum ConstraintProcessStatus { CONSTR_PROCESSED, CONSTR_PROCESSED_DONE, CONSTR_IGNORED, CONSTR_IGNORED_DONE, } ConstraintProcessStatus; typedef ConstraintProcessStatus (*constraint_func)(HeapTuple constraint_tuple, void *ctx); extern TSDLLEXPORT int ts_constraint_process(Oid relid, constraint_func process_func, void *ctx); extern TSDLLEXPORT Form_pg_constraint ts_constraint_find_matching(HeapTuple ht_tup, Relation chunk_rel); ================================================ FILE: src/copy.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ /* * This file contains source code that was copied and/or modified from * the PostgreSQL database, which is licensed under the open-source * PostgreSQL License. Please see the NOTICE at the top level * directory for a copy of the PostgreSQL License. * * The code copies data to a hypertable or migrates existing data from * a table to a hypertable when create_hypertable(..., migrate_data => * 'true', ...) is called. * * Unfortunately, there aren't any good hooks in the regular COPY code to * insert our chunk dispatching. So, most of this code is a straight-up * copy of the regular PostgreSQL source code for the COPY command * (command/copy.c and command/copyfrom.c), albeit with minor modifications. */ #include <postgres.h> #include <access/heapam.h> #include <access/hio.h> #include <access/sysattr.h> #include <access/xact.h> #include <catalog/pg_trigger_d.h> #include <commands/copy.h> #include <commands/copyfrom_internal.h> #include <commands/tablecmds.h> #include <commands/trigger.h> #include <executor/executor.h> #include <executor/nodeModifyTable.h> #include <miscadmin.h> #include <nodes/makefuncs.h> #include <optimizer/optimizer.h> #include <parser/parse_coerce.h> #include <parser/parse_collate.h> #include <parser/parse_expr.h> #include <parser/parse_relation.h> #include <storage/bufmgr.h> #include <storage/smgr.h> #include <utils/builtins.h> #include <utils/elog.h> #include <utils/guc.h> #include <utils/hsearch.h> #include <utils/lsyscache.h> #include <utils/rel.h> #include <utils/rls.h> #include "compat/compat.h" #include "chunk_insert_state.h" #include "copy.h" #include "cross_module_fn.h" #include "dimension.h" #include "hypertable.h" #include "indexing.h" #include "subspace_store.h" /* * Represents the insert method to be used during COPY FROM. */ typedef enum TSCopyInsertMethod { TS_CIM_SINGLE, /* use table_tuple_insert or ExecForeignInsert */ TS_CIM_MULTI_CONDITIONAL, /* use table_multi_insert or * ExecForeignBatchInsert only if valid */ TS_CIM_COMPRESSION, /* use compression for the insert */ } TSCopyInsertMethod; /* * No more than this many tuples per TSCopyMultiInsertBuffer * * Caution: Don't make this too big, as we could end up with this many * TSCopyMultiInsertBuffer items stored in TSCopyMultiInsertInfo's * multiInsertBuffers list. Increasing this can cause quadratic growth in * memory requirements during copies into partitioned tables with a large * number of partitions. */ #define MAX_BUFFERED_TUPLES 1000 /* * Flush buffers if there are >= this many bytes, as counted by the input * size, of tuples stored. */ #define MAX_BUFFERED_BYTES 65535 /* Trim the list of buffers back down to this number after flushing */ #define MAX_PARTITION_BUFFERS 32 /* Stores multi-insert data related to a single relation in CopyFrom. */ typedef struct TSCopyMultiInsertBuffer { TSCopyInsertMethod method; /* The insert method to use */ /* * Tuple description for inserted tuple slots. We use a copy of the result * relation tupdesc to disable reference counting for this tupdesc. It is * not needed and is wasting a lot of CPU in ResourceOwner. */ TupleDesc tupdesc; TupleTableSlot *slots[MAX_BUFFERED_TUPLES]; /* Array to store tuples */ Point *point; /* The point in space of this buffer */ BulkInsertState bistate; /* BulkInsertState for this buffer */ int nused; /* number of 'slots' containing tuples */ uint64 linenos[MAX_BUFFERED_TUPLES]; /* Line # of tuple in copy * stream */ bool can_skip_constraints; /* Whether we can skip constraint * checks for this relation */ RowCompressor *compressor; /* compressor for the chunk */ BulkWriter *bulk_writer; /* BulkWriter for the compressed chunk */ } TSCopyMultiInsertBuffer; /* * Stores one or many TSCopyMultiInsertBuffers and details about the size and * number of tuples which are stored in them. This allows multiple buffers to * exist at once when COPYing into a partitioned table. * * The HTAB is used to store the relationship between a chunk and a * TSCopyMultiInsertBuffer beyond the lifetime of the ChunkInsertState. * * Chunks can be closed (e.g., due to timescaledb.max_open_chunks_per_insert). * When ts_chunk_dispatch_get_chunk_insert_state is called again for a closed * chunk, a new ChunkInsertState is returned. */ typedef struct TSCopyMultiInsertInfo { HTAB *multiInsertBuffers; /* Maps the chunk ids to the buffers (chunkid -> TSCopyMultiInsertBuffer) */ int bufferedTuples; /* number of tuples buffered over all buffers */ int bufferedBytes; /* number of bytes from all buffered tuples */ CopyChunkState *ccstate; /* Copy chunk state for this TSCopyMultiInsertInfo */ EState *estate; /* Executor state used for COPY */ CommandId mycid; /* Command Id used for COPY */ int ti_options; /* table insert options */ Hypertable *ht; /* The hypertable for the inserts */ bool has_continuous_aggregate; } TSCopyMultiInsertInfo; /* * The entry of the multiInsertBuffers HTAB. */ typedef struct MultiInsertBufferEntry { int32 key; TSCopyMultiInsertBuffer *buffer; } MultiInsertBufferEntry; static CopyChunkState * copy_chunk_state_create(Hypertable *ht, Relation rel, CopyFromFunc from_func, CopyFromState cstate, TableScanDesc scandesc) { CopyChunkState *ccstate; EState *estate = CreateExecutorState(); ccstate = palloc(sizeof(CopyChunkState)); ccstate->rel = rel; ccstate->estate = estate; ccstate->cstate = cstate; ccstate->scandesc = scandesc; ccstate->next_copy_from = from_func; ccstate->where_clause = NULL; return ccstate; } /* * Determine whether we can skip constraints checks for this relation. * We will skip constraints checks if: * 1. The relation has CHECK constraints that match the number of dimensions * 2. The relation has no NOT NULL constraints on non-partitioning columns */ static bool can_skip_constraint_check(Hypertable *ht, TupleDesc tupledesc) { /* * When the number of constraints does not match the number of dimensions then there are * additional constraints that we need to check during COPY. Partitioning constraints would * have already been checked by tuple routing. */ Assert(tupledesc->constr->num_check >= ht->space->num_dimensions); if (tupledesc->constr && tupledesc->constr->num_check != ht->space->num_dimensions) return false; for (int i = 0; i < tupledesc->natts; i++) { Form_pg_attribute att = TupleDescAttr(tupledesc, i); if (att->attisdropped) continue; /* * If we have NOT NULL constraints on non-partitioning columns, we cannot skip * constraints and have to check them. */ if (att->attnotnull) { if (ts_is_partitioning_column_name(ht, att->attname)) continue; return false; } } return true; } /* * Allocate memory and initialize a new TSCopyMultiInsertBuffer for this * ResultRelInfo. */ static TSCopyMultiInsertBuffer * TSCopyMultiInsertBufferInit(TSCopyMultiInsertInfo *miinfo, ChunkInsertState *cis, Point *point, TSCopyInsertMethod method) { TSCopyMultiInsertBuffer *buffer; buffer = (TSCopyMultiInsertBuffer *) palloc0(sizeof(TSCopyMultiInsertBuffer)); buffer->method = method; buffer->point = palloc(POINT_SIZE(point->num_coords)); memcpy(buffer->point, point, POINT_SIZE(point->num_coords)); buffer->can_skip_constraints = can_skip_constraint_check(miinfo->ht, cis->rel->rd_att); /* * Downgrade the insert method when triggers are present. */ if (method != TS_CIM_SINGLE && cis->result_relation_info->ri_TrigDesc) { /* If there are BEFORE INSERT row triggers, we cannot use * multi-insert, as the tuples may be inserted in an out-of-order manner, * which might violate the semantics of the triggers. * * For compressed inserts we fall back to TS_CIM_SINGLE when any triggers are present. * This is a safety measure. We might actually safely allow some of these in the future. */ if (method == TS_CIM_MULTI_CONDITIONAL && (cis->result_relation_info->ri_TrigDesc->trig_insert_before_row || cis->result_relation_info->ri_TrigDesc->trig_insert_instead_row)) { buffer->method = TS_CIM_SINGLE; } else if (method == TS_CIM_COMPRESSION) { buffer->method = TS_CIM_SINGLE; } } switch (buffer->method) { case TS_CIM_SINGLE: break; case TS_CIM_MULTI_CONDITIONAL: buffer->bistate = GetBulkInsertState(); /* * Make a non-refcounted copy of tupdesc to avoid spending CPU in * ResourceOwner when creating a big number of table slots. This happens * because each new slot pins its tuple descriptor using PinTupleDesc, and * for reference-counting tuples this involves adding a new reference to * ResourceOwner, which is not very efficient for a large number of * references. */ buffer->tupdesc = CreateTupleDescCopyConstr(cis->rel->rd_att); Assert(buffer->tupdesc->tdrefcount == -1); break; case TS_CIM_COMPRESSION: { bool sort = ts_guc_enable_direct_compress_copy_sort_batches && !ts_guc_enable_direct_compress_copy_client_sorted; buffer->compressor = ts_cm_functions->compressor_init(cis->rel, &buffer->bulk_writer, sort, ts_guc_direct_compress_copy_tuple_sort_limit); if (miinfo->has_continuous_aggregate) { ts_cm_functions->compressor_set_invalidation(buffer->compressor, miinfo->ht, RelationGetRelid(cis->rel)); } /* * The sorting done in the compressor is only a local sort for the * currently ingested batch and will produce overlapping batches for * multiple independent insert streams. Therefore we still need to * mark the chunk as unordered until we adjust the rest of the code to * be able to deal with overlapping batches. */ if (!ts_guc_enable_direct_compress_copy_client_sorted) { Chunk *chunk = ts_chunk_get_by_id(cis->chunk_id, true); if (!ts_chunk_is_unordered(chunk)) ts_chunk_set_unordered(chunk); } cis->columnstore_insert = true; break; } } return buffer; } /* * Get the existing TSCopyMultiInsertBuffer for the chunk or create a new one. */ static inline TSCopyMultiInsertBuffer * TSCopyMultiInsertInfoGetOrSetupBuffer(TSCopyMultiInsertInfo *miinfo, ChunkInsertState *cis, Point *point, TSCopyInsertMethod method) { bool found; int32 chunk_id; Assert(miinfo != NULL); Assert(cis != NULL); Assert(point != NULL); chunk_id = cis->chunk_id; MultiInsertBufferEntry *entry = hash_search(miinfo->multiInsertBuffers, &chunk_id, HASH_ENTER, &found); /* No insert buffer for this chunk exists, create a new one */ if (!found) { entry->buffer = TSCopyMultiInsertBufferInit(miinfo, cis, point, method); } return entry->buffer; } /* * Create a new HTAB that maps from the chunk_id to the multi-insert buffers. */ static HTAB * TSCopyCreateNewInsertBufferHashMap() { struct HASHCTL hctl = { .keysize = sizeof(int32), .entrysize = sizeof(MultiInsertBufferEntry), .hcxt = CurrentMemoryContext, }; return hash_create("COPY insert buffer", 20, &hctl, HASH_ELEM | HASH_CONTEXT | HASH_BLOBS); } /* * Initialize an already allocated TSCopyMultiInsertInfo. */ static void TSCopyMultiInsertInfoInit(TSCopyMultiInsertInfo *miinfo, ResultRelInfo *rri, CopyChunkState *ccstate, EState *estate, CommandId mycid, int ti_options, Hypertable *ht) { miinfo->multiInsertBuffers = TSCopyCreateNewInsertBufferHashMap(); miinfo->bufferedTuples = 0; miinfo->bufferedBytes = 0; miinfo->ccstate = ccstate; miinfo->estate = estate; miinfo->mycid = mycid; miinfo->ti_options = ti_options; miinfo->ht = ht; miinfo->has_continuous_aggregate = ts_hypertable_has_continuous_aggregates(ht->fd.id); } /* * Returns true if the buffers are full. */ static inline bool TSCopyMultiInsertInfoIsFull(TSCopyMultiInsertInfo *miinfo) { if (miinfo->bufferedTuples >= MAX_BUFFERED_TUPLES || miinfo->bufferedBytes >= MAX_BUFFERED_BYTES) return true; return false; } /* * Write the tuples stored in 'buffer' out to the table. */ static inline int TSCopyMultiInsertBufferFlush(TSCopyMultiInsertInfo *miinfo, TSCopyMultiInsertBuffer *buffer) { MemoryContext oldcontext; int i; Assert(miinfo != NULL); Assert(buffer != NULL); EState *estate = miinfo->estate; CommandId mycid = miinfo->mycid; int ti_options = miinfo->ti_options; int nused = buffer->nused; TupleTableSlot **slots = buffer->slots; if (buffer->method == TS_CIM_COMPRESSION) { ts_cm_functions->compressor_flush(buffer->compressor, buffer->bulk_writer); } /* * table_multi_insert and reinitialization of the chunk insert state may * leak memory, so switch to short-lived memory context before calling it. */ oldcontext = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate)); /* * A chunk can be closed while buffering the tuples. Even when the chunk * insert state is moved to the copy memory context, the underlying * table is closed and pointers (e.g., result_relation_info point) to invalid * addresses. Re-reading the chunk insert state ensures that the table is * open and the pointers are valid. * * No callback on changed chunk is needed, the bulk insert state buffer is * freed in TSCopyMultiInsertBufferCleanup(). */ ChunkInsertState *cis = ts_chunk_tuple_routing_find_chunk(miinfo->ccstate->ctr, buffer->point); ResultRelInfo *resultRelInfo = cis->result_relation_info; /* * Add context information to the copy state, which is used to display * error messages with additional details. */ uint64 save_cur_lineno = 0; bool line_buf_valid = false; CopyFromState cstate = miinfo->ccstate->cstate; /* cstate can be NULL in calls that are invoked from timescaledb_move_from_table_to_chunks. */ if (cstate != NULL) { line_buf_valid = cstate->line_buf_valid; save_cur_lineno = cstate->cur_lineno; cstate->line_buf_valid = false; } table_multi_insert(resultRelInfo->ri_RelationDesc, slots, nused, mycid, ti_options, buffer->bistate); MemoryContextSwitchTo(oldcontext); for (i = 0; i < nused; i++) { if (cstate != NULL) cstate->cur_lineno = buffer->linenos[i]; /* * If there are any indexes, update them for all the inserted tuples, * and run AFTER ROW INSERT triggers. */ if (resultRelInfo->ri_NumIndices > 0) { List *recheckIndexes; recheckIndexes = ExecInsertIndexTuplesCompat(resultRelInfo, buffer->slots[i], estate, false, false, NULL, NIL, false); ExecARInsertTriggers(estate, resultRelInfo, slots[i], recheckIndexes, NULL /* transition capture */); list_free(recheckIndexes); } /* * There's no indexes, but see if we need to run AFTER ROW INSERT * triggers anyway. */ else if (resultRelInfo->ri_TrigDesc != NULL && (resultRelInfo->ri_TrigDesc->trig_insert_after_row || resultRelInfo->ri_TrigDesc->trig_insert_new_table)) { ExecARInsertTriggers(estate, resultRelInfo, slots[i], NIL, NULL /* transition capture */); } if (miinfo->has_continuous_aggregate) { bool should_free; HeapTuple tuple = ExecFetchSlotHeapTuple(slots[i], false, &should_free); ts_cm_functions->continuous_agg_dml_invalidate(miinfo->ht->fd.id, resultRelInfo->ri_RelationDesc, tuple, NULL, false); if (should_free) heap_freetuple(tuple); } ExecClearTuple(slots[i]); } /* Mark that all slots are free */ buffer->nused = 0; /* Chunk could be closed on a subsequent call of ts_chunk_dispatch_get_chunk_insert_state * (e.g., due to timescaledb.max_open_chunks_per_insert). So, ensure the bulk insert is * finished after the flush is complete. */ ResultRelInfo *result_relation_info = cis->result_relation_info; Assert(result_relation_info != NULL); table_finish_bulk_insert(result_relation_info->ri_RelationDesc, miinfo->ti_options); /* Reset cur_lineno and line_buf_valid to what they were */ if (cstate != NULL) { cstate->line_buf_valid = line_buf_valid; cstate->cur_lineno = save_cur_lineno; } return cis->chunk_id; } /* * Drop used slots and free member for this buffer. * * The buffer must be flushed before cleanup. */ static inline void TSCopyMultiInsertBufferCleanup(TSCopyMultiInsertInfo *miinfo, TSCopyMultiInsertBuffer *buffer) { int i; /* Ensure buffer was flushed */ Assert(buffer->nused == 0); switch (buffer->method) { case TS_CIM_SINGLE: break; case TS_CIM_MULTI_CONDITIONAL: FreeBulkInsertState(buffer->bistate); /* Since we only create slots on demand, just drop the non-null ones. */ for (i = 0; i < MAX_BUFFERED_TUPLES && buffer->slots[i] != NULL; i++) ExecDropSingleTupleTableSlot(buffer->slots[i]); FreeTupleDesc(buffer->tupdesc); break; case TS_CIM_COMPRESSION: ts_cm_functions->compressor_free(buffer->compressor, buffer->bulk_writer); break; } pfree(buffer->point); pfree(buffer); } /* list_sort comparator to sort TSCopyMultiInsertBuffer by usage */ static int TSCmpBuffersByUsage(const ListCell *a, const ListCell *b) { int b1 = ((const TSCopyMultiInsertBuffer *) lfirst(a))->nused; int b2 = ((const TSCopyMultiInsertBuffer *) lfirst(b))->nused; Assert(b1 >= 0); Assert(b2 >= 0); if (b1 > b2) { return 1; } if (b1 == b2) { return 0; } return -1; } /* * Flush all buffers by writing the tuples to the chunks. In addition, trim down the * amount of multi-insert buffers to MAX_PARTITION_BUFFERS by deleting the least used * buffers (the buffers that store least tuples). */ static inline void TSCopyMultiInsertInfoFlush(TSCopyMultiInsertInfo *miinfo, ChunkInsertState *cur_cis) { HASH_SEQ_STATUS status; MultiInsertBufferEntry *entry; int current_multi_insert_buffers; int buffers_to_delete; bool found; int32 flushed_chunk_id; List *buffer_list = NIL; ListCell *lc; current_multi_insert_buffers = hash_get_num_entries(miinfo->multiInsertBuffers); int current_chunk_id = cur_cis ? cur_cis->chunk_id : 0; /* Create a list of buffers that can be sorted by usage */ hash_seq_init(&status, miinfo->multiInsertBuffers); for (entry = hash_seq_search(&status); entry != NULL; entry = hash_seq_search(&status)) { buffer_list = lappend(buffer_list, entry->buffer); } buffers_to_delete = Max(current_multi_insert_buffers - MAX_PARTITION_BUFFERS, 0); /* Sorting is only needed if we want to remove the least used buffers */ if (buffers_to_delete > 0) list_sort(buffer_list, TSCmpBuffersByUsage); /* Flush buffers and delete them if needed */ foreach (lc, buffer_list) { TSCopyMultiInsertBuffer *buffer = (TSCopyMultiInsertBuffer *) lfirst(lc); flushed_chunk_id = TSCopyMultiInsertBufferFlush(miinfo, buffer); if (buffers_to_delete > 0) { /* * Reduce active multi-insert buffers. However, the current used buffer * should not be deleted because it might reused for the next insert. */ if (current_chunk_id == 0 || flushed_chunk_id != current_chunk_id) { TSCopyMultiInsertBufferCleanup(miinfo, buffer); hash_search(miinfo->multiInsertBuffers, &flushed_chunk_id, HASH_REMOVE, &found); Assert(found); buffers_to_delete--; } } } list_free(buffer_list); /* All buffers have been flushed */ miinfo->bufferedTuples = 0; miinfo->bufferedBytes = 0; } /* * All existing buffers are flushed and the multi-insert states * are freed. So, delete old hash map and create a new one for further * inserts. */ static inline void TSCopyMultiInsertInfoFlushAndCleanup(TSCopyMultiInsertInfo *miinfo) { TSCopyMultiInsertInfoFlush(miinfo, NULL); HASH_SEQ_STATUS status; MultiInsertBufferEntry *entry; hash_seq_init(&status, miinfo->multiInsertBuffers); for (entry = hash_seq_search(&status); entry != NULL; entry = hash_seq_search(&status)) { TSCopyMultiInsertBuffer *buffer = entry->buffer; TSCopyMultiInsertBufferCleanup(miinfo, buffer); } hash_destroy(miinfo->multiInsertBuffers); } /* * Get the next TupleTableSlot that the next tuple should be stored in. * * Callers must ensure that the buffer is not full. * * Note: 'miinfo' is unused but has been included for consistency with the * other functions in this area. */ static inline TupleTableSlot * TSCopyMultiInsertInfoNextFreeSlot(TSCopyMultiInsertInfo *miinfo, ResultRelInfo *result_relation_info, TSCopyMultiInsertBuffer *buffer) { int nused = buffer->nused; Assert(buffer != NULL); Assert(nused < MAX_BUFFERED_TUPLES); if (buffer->slots[nused] == NULL) { const TupleTableSlotOps *tts_cb = table_slot_callbacks(result_relation_info->ri_RelationDesc); buffer->slots[nused] = MakeSingleTupleTableSlot(buffer->tupdesc, tts_cb); } return buffer->slots[nused]; } /* * Record the previously reserved TupleTableSlot that was reserved by * TSCopyMultiInsertInfoNextFreeSlot as being consumed. */ static inline void TSCopyMultiInsertInfoStore(TSCopyMultiInsertInfo *miinfo, ResultRelInfo *rri, TSCopyMultiInsertBuffer *buffer, TupleTableSlot *slot, CopyFromState cstate) { Assert(buffer != NULL); Assert(slot == buffer->slots[buffer->nused]); /* Store the line number so we can properly report any errors later */ uint64 lineno = 0; /* The structure CopyFromState is private in PG < 14. So we can not access * the members like the line number or the size of the tuple. */ if (cstate != NULL) lineno = cstate->cur_lineno; buffer->linenos[buffer->nused] = lineno; /* Record this slot as being used */ buffer->nused++; /* Update how many tuples are stored and their size */ miinfo->bufferedTuples++; /* * Note: There is no reliable way to determine the in-memory size of a virtual * tuple. So, we perform flushing in PG < 14 only based on the number of buffered * tuples and not based on the size. */ if (cstate != NULL) { int tuplen = cstate->line_buf.len; miinfo->bufferedBytes += tuplen; } } static void copy_chunk_state_destroy(CopyChunkState *ccstate) { ts_chunk_tuple_routing_destroy(ccstate->ctr); FreeExecutorState(ccstate->estate); } static bool next_copy_from(CopyChunkState *ccstate, ExprContext *econtext, Datum *values, bool *nulls) { Assert(ccstate->cstate != NULL); return NextCopyFrom(ccstate->cstate, econtext, values, nulls); } /* * Error context callback when copying from table to chunk. */ static void copy_table_to_chunk_error_callback(void *arg) { TableScanDesc scandesc = (TableScanDesc) arg; errcontext("copying from table %s", RelationGetRelationName(scandesc->rs_rd)); } static TSCopyInsertMethod choose_copy_method(Hypertable *ht, CopyChunkState *ccstate, ResultRelInfo *resultRelInfo) { /* * Multi-insert buffers (TS_CIM_MULTI_CONDITIONAL) can only be used if no triggers are * defined on the target table. Otherwise, the tuples may be inserted in an out-of-order * manner, which might violate the semantics of the triggers. So, they are inserted * tuple-per-tuple (TS_CIM_SINGLE). However, the ts_block trigger on the hypertable can * be ignored. */ /* Before INSERT Triggers */ bool has_before_insert_row_trig = (resultRelInfo->ri_TrigDesc && resultRelInfo->ri_TrigDesc->trig_insert_before_row); /* Instead of INSERT Triggers */ bool has_instead_insert_row_trig = (resultRelInfo->ri_TrigDesc && resultRelInfo->ri_TrigDesc->trig_insert_instead_row); bool has_after_insert_statement_trig = (resultRelInfo->ri_TrigDesc && resultRelInfo->ri_TrigDesc->trig_insert_new_table); /* Depending on the configured trigger, enable or disable the multi-insert buffers */ if (has_after_insert_statement_trig || has_before_insert_row_trig || has_instead_insert_row_trig) { ereport(DEBUG1, (errmsg("Using normal unbuffered copy operation (TS_CIM_SINGLE) " "because triggers are defined on the destination table."))); if (ts_guc_enable_direct_compress_copy) ereport(WARNING, (errmsg("disabling direct compress copy due to presence of triggers on the " "destination table"))); return TS_CIM_SINGLE; } if (TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(ht) && ts_guc_enable_direct_compress_copy) { if (ts_indexing_relation_has_primary_or_unique_index(ccstate->rel)) { ereport(WARNING, (errmsg("disabling direct compress because the destination table has unique " "constraints"))); } else if (resultRelInfo->ri_TrigDesc && resultRelInfo->ri_TrigDesc->numtriggers > 1) { ereport(WARNING, (errmsg( "disabling direct compress because the destination table has triggers"))); } else { ccstate->ctr->create_compressed_chunk = true; ereport(DEBUG1, (errmsg("Using compressed copy operation (TS_CIM_COMPRESSION)."))); return TS_CIM_COMPRESSION; } } ereport(DEBUG1, (errmsg("Using optimized multi-buffer copy operation (TS_CIM_MULTI_CONDITIONAL)."))); return TS_CIM_MULTI_CONDITIONAL; } /* * Use COPY FROM to copy data from file to relation. */ static uint64 copyfrom(CopyChunkState *ccstate, ParseState *pstate, Hypertable *ht, MemoryContext copycontext, void (*callback)(void *), void *arg) { ResultRelInfo *resultRelInfo; ResultRelInfo *saved_resultRelInfo = NULL; EState *estate = ccstate->estate; /* for ExecConstraints() */ ExprContext *econtext; TupleTableSlot *singleslot; MemoryContext oldcontext = CurrentMemoryContext; ErrorContextCallback errcallback = { .callback = callback, .arg = arg, }; CommandId mycid = GetCurrentCommandId(true); TSCopyInsertMethod insertMethod; /* The insert method for the table */ TSCopyMultiInsertInfo multiInsertInfo = { 0 }; /* pacify compiler */ int ti_options = 0; /* start with default options for insert */ BulkInsertState bistate = NULL; uint64 processed = 0; ExprState *qualexpr = NULL; Assert(pstate->p_rtable); if (ccstate->rel->rd_rel->relkind != RELKIND_RELATION) { if (ccstate->rel->rd_rel->relkind == RELKIND_VIEW) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot copy to view \"%s\"", RelationGetRelationName(ccstate->rel)))); else if (ccstate->rel->rd_rel->relkind == RELKIND_MATVIEW) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot copy to materialized view \"%s\"", RelationGetRelationName(ccstate->rel)))); else if (ccstate->rel->rd_rel->relkind == RELKIND_FOREIGN_TABLE) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot copy to foreign table \"%s\"", RelationGetRelationName(ccstate->rel)))); else if (ccstate->rel->rd_rel->relkind == RELKIND_SEQUENCE) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot copy to sequence \"%s\"", RelationGetRelationName(ccstate->rel)))); else ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot copy to non-table relation \"%s\"", RelationGetRelationName(ccstate->rel)))); } /*---------- * Check to see if we can avoid writing WAL * * If archive logging/streaming is not enabled *and* either * - table was created in same transaction as this COPY * - data is being written to relfilenode created in this transaction * then we can skip writing WAL. It's safe because if the transaction * doesn't commit, we'll discard the table (or the new relfilenode file). * If it does commit, we'll have done the heap_sync at the bottom of this * routine first. * * As mentioned in comments in utils/rel.h, the in-same-transaction test * is not always set correctly, since in rare cases rd_newRelfilenodeSubid * can be cleared before the end of the transaction. The exact case is * when a relation sets a new relfilenode twice in same transaction, yet * the second one fails in an aborted subtransaction, e.g. * * BEGIN; * TRUNCATE t; * SAVEPOINT save; * TRUNCATE t; * ROLLBACK TO save; * COPY ... * * Also, if the target file is new-in-transaction, we assume that checking * FSM for free space is a waste of time, even if we must use WAL because * of archiving. This could possibly be wrong, but it's unlikely. * * The comments for heap_insert and RelationGetBufferForTuple specify that * skipping WAL logging is only safe if we ensure that our tuples do not * go into pages containing tuples from any other transactions --- but this * must be the case if we have a new table or new relfilenode, so we need * no additional work to enforce that. *---------- */ /* createSubid is creation check, newRelfilenodeSubid is truncation check */ if (ccstate->rel->rd_createSubid != InvalidSubTransactionId || #if PG16_LT ccstate->rel->rd_newRelfilenodeSubid != InvalidSubTransactionId) #else ccstate->rel->rd_newRelfilelocatorSubid != InvalidSubTransactionId) #endif { ti_options |= HEAP_INSERT_SKIP_FSM; } /* * We need a ResultRelInfo so we can use the regular executor's * index-entry-making machinery. (There used to be a huge amount of code * here that basically duplicated execUtils.c ...) * * WARNING. The dummy rangetable index is decremented by 1 (unchecked) * inside `ExecConstraints` so unless you want to have a overflow, keep it * above zero. See `rt_fetch` in parsetree.h. */ resultRelInfo = makeNode(ResultRelInfo); #if PG16_LT ExecInitRangeTable(estate, pstate->p_rtable); #elif PG18_LT Assert(pstate->p_rteperminfos != NULL); ExecInitRangeTable(estate, pstate->p_rtable, pstate->p_rteperminfos); #else /* * PG18+ adds unpruned relids to ExecInitRangeTable * We initialize it with 1 similar to upstream behavior, * but since this is copy no pruning is expected to happen. */ Assert(pstate->p_rteperminfos != NULL); ExecInitRangeTable(estate, pstate->p_rtable, pstate->p_rteperminfos, bms_make_singleton(1)); #endif ExecInitResultRelation(estate, resultRelInfo, 1); CheckValidResultRelCompat(resultRelInfo, CMD_INSERT, ONCONFLICT_NONE, NIL); ExecOpenIndices(resultRelInfo, false); ccstate->ctr = ts_chunk_tuple_routing_create(estate, ht, resultRelInfo); singleslot = table_slot_create(resultRelInfo->ri_RelationDesc, &estate->es_tupleTable); /* Prepare to catch AFTER triggers. */ AfterTriggerBeginQuery(); /* * If there are any triggers with transition tables on the named relation, * we need to be prepared to capture transition tuples. Note that * ccstate->cstate is null when we migrate from an existing table in a * call from create_hypertable(), so we do not need a transition capture * state in this case. */ if (ccstate->cstate) ccstate->cstate->transition_capture = MakeTransitionCaptureState(ccstate->rel->trigdesc, RelationGetRelid(ccstate->rel), CMD_INSERT); if (ccstate->where_clause) qualexpr = ExecInitQual(castNode(List, ccstate->where_clause), NULL); /* * Check BEFORE STATEMENT insertion triggers. It's debatable whether we * should do this for COPY, since it's not really an "INSERT" statement as * such. However, executing these triggers maintains consistency with the * EACH ROW triggers that we already fire on COPY. */ ExecBSInsertTriggers(estate, resultRelInfo); bistate = GetBulkInsertState(); econtext = GetPerTupleExprContext(estate); /* Set up callback to identify error line number. * * It is not necessary to add an entry to the error context stack if we do * not have a CopyFromState or callback. In that case, we just use the existing * error already on the context stack. */ if (ccstate->cstate && callback) { errcallback.previous = error_context_stack; error_context_stack = &errcallback; } insertMethod = choose_copy_method(ht, ccstate, resultRelInfo); TSCopyMultiInsertInfoInit(&multiInsertInfo, resultRelInfo, ccstate, estate, mycid, ti_options, ht); TSCopyMultiInsertBuffer *buffer = NULL; int reset_count = 0; /* Reset the per-tuple exprcontext every 100 tuples */ Oid prev_chunk_oid = InvalidOid; /* Previous chunk OID to detect chunk changes */ for (;;) { TupleTableSlot *myslot = NULL; bool skip_tuple; Point *point = NULL; ChunkInsertState *cis = NULL; CHECK_FOR_INTERRUPTS(); /* * Reset the per-tuple exprcontext. We do this after every tuple, to * clean-up after expression evaluations etc. */ if (reset_count == 100) { ResetPerTupleExprContext(estate); reset_count = 0; } else reset_count++; myslot = singleslot; Assert(myslot != NULL); /* Switch into its memory context */ MemoryContextSwitchTo(GetPerTupleMemoryContext(estate)); ExecClearTuple(myslot); if (!ccstate->next_copy_from(ccstate, econtext, myslot->tts_values, myslot->tts_isnull)) break; ExecStoreVirtualTuple(myslot); /* Calculate the tuple's point in the N-dimensional hyperspace */ point = ts_hyperspace_calculate_point(ht->space, myslot); /* Find or create the insert state matching the point */ cis = ts_chunk_tuple_routing_find_chunk(ccstate->ctr, point); if (OidIsValid(prev_chunk_oid) && prev_chunk_oid != cis->rel->rd_id) { ReleaseBulkInsertStatePin(bistate); } prev_chunk_oid = cis->rel->rd_id; Assert(cis != NULL); ts_chunk_tuple_routing_decompress_for_insert(cis, ccstate->ctr->root_rri, myslot, ccstate->ctr->estate, false); /* Triggers and stuff need to be invoked in query context. */ MemoryContextSwitchTo(oldcontext); buffer = TSCopyMultiInsertInfoGetOrSetupBuffer(&multiInsertInfo, cis, point, insertMethod); /* * If the insert method has changed, we need to flush the * multi-insert info to ensure that the tuples are * visible to the triggers. */ if (insertMethod != buffer->method) TSCopyMultiInsertInfoFlush(&multiInsertInfo, cis); /* Convert the tuple to match the chunk's rowtype */ if (buffer->method == TS_CIM_SINGLE) { if (NULL != cis->hyper_to_chunk_map) myslot = execute_attr_map_slot(cis->hyper_to_chunk_map->attrMap, myslot, cis->slot); } else if (buffer->method == TS_CIM_COMPRESSION) { if (NULL != cis->hyper_to_chunk_map) myslot = execute_attr_map_slot(cis->hyper_to_chunk_map->attrMap, myslot, cis->slot); } else { /* * Prepare to queue up tuple for later batch insert into * current chunk. */ TupleTableSlot *batchslot; batchslot = TSCopyMultiInsertInfoNextFreeSlot(&multiInsertInfo, cis->result_relation_info, buffer); if (NULL != cis->hyper_to_chunk_map) myslot = execute_attr_map_slot(cis->hyper_to_chunk_map->attrMap, myslot, batchslot); else { /* * This looks more expensive than it is (Believe me, I * optimized it away. Twice.). The input is in virtual * form, and we'll materialize the slot below - for most * slot types the copy performs the work materialization * would later require anyway. */ ExecCopySlot(batchslot, myslot); myslot = batchslot; } } if (qualexpr != NULL) { econtext->ecxt_scantuple = myslot; if (!ExecQual(qualexpr, econtext)) continue; } /* * Set the result relation in the executor state to the target chunk. * This makes sure that the tuple gets inserted into the correct * chunk. */ saved_resultRelInfo = resultRelInfo; resultRelInfo = cis->result_relation_info; /* Set the right relation for triggers */ ts_tuptableslot_set_table_oid(myslot, RelationGetRelid(resultRelInfo->ri_RelationDesc)); skip_tuple = false; /* BEFORE ROW INSERT Triggers */ if (resultRelInfo->ri_TrigDesc && resultRelInfo->ri_TrigDesc->trig_insert_before_row) skip_tuple = !ExecBRInsertTriggers(estate, resultRelInfo, myslot); if (!skip_tuple) { /* Note that PostgreSQL's copy path would check INSTEAD OF * INSERT/UPDATE/DELETE triggers here, but such triggers can only * exist on views and chunks cannot be views. */ List *recheckIndexes = NIL; /* Compute stored generated columns */ if (resultRelInfo->ri_RelationDesc->rd_att->constr && resultRelInfo->ri_RelationDesc->rd_att->constr->has_generated_stored) ExecComputeStoredGenerated(resultRelInfo, estate, myslot, CMD_INSERT); /* * If the target is a plain table, check the constraints of * the tuple. Since we check the constraints during tuple routing * we only need to check if we have additional constraints beyond * partitioning constraints. */ if (!buffer->can_skip_constraints) { Assert(resultRelInfo->ri_RangeTableIndex > 0 && estate->es_range_table); ExecConstraints(resultRelInfo, myslot, estate); } if (buffer->method == TS_CIM_SINGLE) { /* OK, store the tuple and create index entries for it */ table_tuple_insert(resultRelInfo->ri_RelationDesc, myslot, mycid, ti_options, bistate); if (resultRelInfo->ri_NumIndices > 0) recheckIndexes = ExecInsertIndexTuplesCompat(resultRelInfo, myslot, estate, false, false, NULL, NIL, false); /* AFTER ROW INSERT Triggers. We do not need to do this if we * are migrating data from an existing table in a call from * create_hypertable(). */ if (ccstate->cstate) ExecARInsertTriggers(estate, resultRelInfo, myslot, recheckIndexes, ccstate->cstate->transition_capture); } else if (buffer->method == TS_CIM_COMPRESSION) { ts_cm_functions->compressor_add_slot(buffer->compressor, buffer->bulk_writer, myslot); } else { /* * The slot previously might point into the per-tuple * context. For batching it needs to be longer lived. */ ExecMaterializeSlot(myslot); /* Add this tuple to the tuple buffer */ TSCopyMultiInsertInfoStore(&multiInsertInfo, resultRelInfo, buffer, myslot, ccstate->cstate); /* * If enough inserts have queued up, then flush all * buffers out to their tables. */ if (TSCopyMultiInsertInfoIsFull(&multiInsertInfo)) { ereport(DEBUG2, (errmsg("flush called with %d bytes and %d buffered tuples", multiInsertInfo.bufferedBytes, multiInsertInfo.bufferedTuples))); TSCopyMultiInsertInfoFlush(&multiInsertInfo, cis); } } list_free(recheckIndexes); /* * We count only tuples not suppressed by a BEFORE INSERT trigger; * this is the same definition used by execMain.c for counting * tuples inserted by an INSERT command. */ processed++; } resultRelInfo = saved_resultRelInfo; } /* Flush any remaining buffered tuples */ if (insertMethod != TS_CIM_SINGLE) TSCopyMultiInsertInfoFlushAndCleanup(&multiInsertInfo); /* Done, clean up */ if (ccstate->cstate && callback) error_context_stack = errcallback.previous; FreeBulkInsertState(bistate); MemoryContextSwitchTo(oldcontext); /* * Execute AFTER STATEMENT insertion triggers. * * We do not need to do this ccstate->cstate is NULL, which is the case * when migrating data from an existing table in a call from * create_hypertable(). */ if (ccstate->cstate) ExecASInsertTriggers(estate, resultRelInfo, ccstate->cstate->transition_capture); /* Handle queued AFTER triggers */ AfterTriggerEndQuery(estate); ExecResetTupleTable(estate->es_tupleTable, false); ExecCloseResultRelations(estate); ExecCloseRangeTableRelations(estate); /* * If we skipped writing WAL, then we need to sync the heap (but not * indexes since those use WAL anyway) */ if (!RelationNeedsWAL(ccstate->rel)) smgrimmedsync(RelationGetSmgr(ccstate->rel), MAIN_FORKNUM); return processed; } /* * CopyGetAttnums - build an integer list of attnums to be copied * * The input attnamelist is either the user-specified column list, * or NIL if there was none (in which case we want all the non-dropped * columns). * * rel can be NULL ... it's only used for error reports. */ static List * timescaledb_CopyGetAttnums(TupleDesc tupDesc, Relation rel, List *attnamelist) { List *attnums = NIL; if (attnamelist == NIL) { /* Generate default column list */ int attr_count = tupDesc->natts; int i; for (i = 0; i < attr_count; i++) { Form_pg_attribute attr = TupleDescAttr(tupDesc, i); if (attr->attisdropped) continue; attnums = lappend_int(attnums, i + 1); } } else { /* Validate the user-supplied list and extract attnums */ ListCell *l; foreach (l, attnamelist) { char *name = strVal(lfirst(l)); int attnum; int i; /* Lookup column name */ attnum = InvalidAttrNumber; for (i = 0; i < tupDesc->natts; i++) { Form_pg_attribute attr = TupleDescAttr(tupDesc, i); if (attr->attisdropped) continue; if (namestrcmp(&(attr->attname), name) == 0) { attnum = attr->attnum; break; } } if (attnum == InvalidAttrNumber) { if (rel != NULL) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("column \"%s\" of relation \"%s\" does not exist", name, RelationGetRelationName(rel)))); else ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("column \"%s\" does not exist", name))); } /* Check for duplicates */ if (list_member_int(attnums, attnum)) ereport(ERROR, (errcode(ERRCODE_DUPLICATE_COLUMN), errmsg("column \"%s\" specified more than once", name))); attnums = lappend_int(attnums, attnum); } } return attnums; } static void copy_constraints_and_check(ParseState *pstate, Relation rel, List *attnums) { ListCell *cur; char *xactReadOnly; ParseNamespaceItem *nsitem = addRangeTableEntryForRelation(pstate, rel, RowExclusiveLock, NULL, false, false); RangeTblEntry *rte = nsitem->p_rte; addNSItemToQuery(pstate, nsitem, true, true, true); #if PG16_LT rte->requiredPerms = ACL_INSERT; foreach (cur, attnums) { int attno = lfirst_int(cur) - FirstLowInvalidHeapAttributeNumber; rte->insertedCols = bms_add_member(rte->insertedCols, attno); } ExecCheckRTPerms(pstate->p_rtable, true); #else RTEPermissionInfo *perminfo = nsitem->p_perminfo; perminfo->requiredPerms = ACL_INSERT; foreach (cur, attnums) { int attno = lfirst_int(cur) - FirstLowInvalidHeapAttributeNumber; perminfo->insertedCols = bms_add_member(perminfo->insertedCols, attno); } ExecCheckPermissions(pstate->p_rtable, list_make1(perminfo), true); #endif /* * Permission check for row security policies. * * check_enable_rls will ereport(ERROR) if the user has requested * something invalid and will otherwise indicate if we should enable RLS * (returns RLS_ENABLED) or not for this COPY statement. * * If the relation has a row security policy and we are to apply it then * perform a "query" copy and allow the normal query processing to handle * the policies. * * If RLS is not enabled for this, then just fall through to the normal * non-filtering relation handling. */ if (check_enable_rls(rte->relid, InvalidOid, false) == RLS_ENABLED) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("COPY FROM not supported with row-level security"), errhint("Use INSERT statements instead."))); } /* check read-only transaction and parallel mode */ xactReadOnly = GetConfigOptionByName("transaction_read_only", NULL, false); if (strncmp(xactReadOnly, "on", sizeof("on")) == 0 && !rel->rd_islocaltemp) PreventCommandIfReadOnly("COPY FROM"); PreventCommandIfParallelMode("COPY FROM"); } void timescaledb_DoCopy(const CopyStmt *stmt, const char *queryString, uint64 *processed, Hypertable *ht) { CopyChunkState *ccstate; CopyFromState cstate; bool pipe = (stmt->filename == NULL); Relation rel; List *attnums = NIL; Node *where_clause = NULL; ParseState *pstate; MemoryContext copycontext = NULL; /* Disallow COPY to/from file or program except to superusers. */ if (!pipe && !superuser()) { if (stmt->is_program) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("must be superuser to COPY to or from an external program"), errhint("Anyone can COPY to stdout or from stdin. " "psql's \\copy command also works for anyone."))); else ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("must be superuser to COPY to or from a file"), errhint("Anyone can COPY to stdout or from stdin. " "psql's \\copy command also works for anyone."))); } if (!stmt->is_from || NULL == stmt->relation) elog(ERROR, "timescale DoCopy should only be called for COPY FROM"); Assert(!stmt->query); /* * We never actually write to the main table, but we need RowExclusiveLock * to ensure no one else is. Because of the check above, we know that * `stmt->relation` is defined, so we are guaranteed to have a relation * available. */ rel = table_openrv(stmt->relation, RowExclusiveLock); attnums = timescaledb_CopyGetAttnums(RelationGetDescr(rel), rel, stmt->attlist); pstate = make_parsestate(NULL); pstate->p_sourcetext = queryString; copy_constraints_and_check(pstate, rel, attnums); cstate = BeginCopyFrom(pstate, rel, NULL, stmt->filename, stmt->is_program, NULL, stmt->attlist, stmt->options); if (stmt->whereClause) { where_clause = transformExpr(pstate, stmt->whereClause, EXPR_KIND_COPY_WHERE); where_clause = coerce_to_boolean(pstate, where_clause, "WHERE"); assign_expr_collations(pstate, where_clause); where_clause = eval_const_expressions(NULL, where_clause); where_clause = (Node *) canonicalize_qual((Expr *) where_clause, false); where_clause = (Node *) make_ands_implicit((Expr *) where_clause); } ccstate = copy_chunk_state_create(ht, rel, next_copy_from, cstate, NULL); ccstate->where_clause = where_clause; copycontext = cstate->copycontext; *processed = copyfrom(ccstate, pstate, ht, copycontext, CopyFromErrorCallback, cstate); copy_chunk_state_destroy(ccstate); EndCopyFrom(cstate); free_parsestate(pstate); table_close(rel, NoLock); } static bool next_copy_from_table_to_chunks(CopyChunkState *ccstate, ExprContext *econtext, Datum *values, bool *nulls) { TableScanDesc scandesc = ccstate->scandesc; HeapTuple tuple; Assert(scandesc != NULL); tuple = heap_getnext(scandesc, ForwardScanDirection); if (!HeapTupleIsValid(tuple)) return false; heap_deform_tuple(tuple, RelationGetDescr(ccstate->rel), values, nulls); return true; } /* * Move data from the given hypertable's main table to chunks. * * The data moving is essentially a COPY from the main table to the chunks * followed by a TRUNCATE on the main table. */ void timescaledb_move_from_table_to_chunks(Hypertable *ht, LOCKMODE lockmode) { Relation rel; CopyChunkState *ccstate; TableScanDesc scandesc; ParseState *pstate = make_parsestate(NULL); Snapshot snapshot; List *attnums = NIL; MemoryContext copycontext; RangeVar rv = { .schemaname = NameStr(ht->fd.schema_name), .relname = NameStr(ht->fd.table_name), .inh = false, /* Don't recurse */ }; TruncateStmt stmt = { .type = T_TruncateStmt, .relations = list_make1(&rv), .behavior = DROP_RESTRICT, }; int i; rel = table_open(ht->main_table_relid, lockmode); for (i = 0; i < rel->rd_att->natts; i++) { Form_pg_attribute attr = TupleDescAttr(rel->rd_att, i); attnums = lappend_int(attnums, attr->attnum); } copycontext = AllocSetContextCreate(CurrentMemoryContext, "COPY", ALLOCSET_DEFAULT_SIZES); copy_constraints_and_check(pstate, rel, attnums); snapshot = RegisterSnapshot(GetLatestSnapshot()); scandesc = table_beginscan(rel, snapshot, 0, NULL); ccstate = copy_chunk_state_create(ht, rel, next_copy_from_table_to_chunks, NULL, scandesc); copyfrom(ccstate, pstate, ht, copycontext, copy_table_to_chunk_error_callback, scandesc); copy_chunk_state_destroy(ccstate); table_endscan(scandesc); UnregisterSnapshot(snapshot); table_close(rel, lockmode); if (MemoryContextIsValid(copycontext)) MemoryContextDelete(copycontext); ExecuteTruncate(&stmt); } ================================================ FILE: src/copy.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <access/relscan.h> #include <access/xact.h> #include <commands/copy.h> #include <executor/executor.h> #include <nodes/parsenodes.h> #include <storage/lockdefs.h> #include "chunk_tuple_routing.h" typedef struct CopyChunkState CopyChunkState; typedef struct Hypertable Hypertable; typedef bool (*CopyFromFunc)(CopyChunkState *ccstate, ExprContext *econtext, Datum *values, bool *nulls); typedef struct CopyChunkState { Relation rel; EState *estate; ChunkTupleRouting *ctr; CopyFromFunc next_copy_from; CopyFromState cstate; TableScanDesc scandesc; Node *where_clause; } CopyChunkState; extern void timescaledb_DoCopy(const CopyStmt *stmt, const char *queryString, uint64 *processed, Hypertable *ht); extern void timescaledb_move_from_table_to_chunks(Hypertable *ht, LOCKMODE lockmode); ================================================ FILE: src/cross_module_fn.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/amapi.h> #include <fmgr.h> #include <utils/lsyscache.h> #include <utils/timestamp.h> #include "bgw/job.h" #include "cross_module_fn.h" #include "export.h" #include "guc.h" #include "license_guc.h" #define CROSSMODULE_WRAPPER(func) \ TS_FUNCTION_INFO_V1(ts_##func); \ Datum ts_##func(PG_FUNCTION_ARGS) \ { \ PG_RETURN_DATUM(ts_cm_functions->func(fcinfo)); \ } /* bgw policy functions */ CROSSMODULE_WRAPPER(policy_compression_add); CROSSMODULE_WRAPPER(policy_compression_remove); CROSSMODULE_WRAPPER(policy_recompression_proc); CROSSMODULE_WRAPPER(policy_compression_check); CROSSMODULE_WRAPPER(policy_refresh_cagg_add); CROSSMODULE_WRAPPER(policy_refresh_cagg_proc); CROSSMODULE_WRAPPER(policy_refresh_cagg_check); CROSSMODULE_WRAPPER(policy_process_hyper_inval_remove); CROSSMODULE_WRAPPER(policy_process_hyper_inval_add); CROSSMODULE_WRAPPER(policy_process_hyper_inval_proc); CROSSMODULE_WRAPPER(policy_process_hyper_inval_check); CROSSMODULE_WRAPPER(policy_refresh_cagg_remove); CROSSMODULE_WRAPPER(policy_reorder_add); CROSSMODULE_WRAPPER(policy_reorder_proc); CROSSMODULE_WRAPPER(policy_reorder_check); CROSSMODULE_WRAPPER(policy_reorder_remove); CROSSMODULE_WRAPPER(policy_retention_add); CROSSMODULE_WRAPPER(policy_retention_proc); CROSSMODULE_WRAPPER(policy_retention_check); CROSSMODULE_WRAPPER(policy_retention_remove); CROSSMODULE_WRAPPER(job_add); CROSSMODULE_WRAPPER(job_delete); CROSSMODULE_WRAPPER(job_run); CROSSMODULE_WRAPPER(job_alter); CROSSMODULE_WRAPPER(job_alter_set_hypertable_id); CROSSMODULE_WRAPPER(reorder_chunk); CROSSMODULE_WRAPPER(move_chunk); CROSSMODULE_WRAPPER(policies_add); CROSSMODULE_WRAPPER(policies_remove); CROSSMODULE_WRAPPER(policies_remove_all); CROSSMODULE_WRAPPER(policies_alter); CROSSMODULE_WRAPPER(policies_show); /* compression functions */ CROSSMODULE_WRAPPER(compressed_data_decompress_forward); CROSSMODULE_WRAPPER(compressed_data_decompress_reverse); CROSSMODULE_WRAPPER(compressed_data_column_size); CROSSMODULE_WRAPPER(compressed_data_to_array); CROSSMODULE_WRAPPER(compressed_data_send); CROSSMODULE_WRAPPER(compressed_data_recv); CROSSMODULE_WRAPPER(compressed_data_in); CROSSMODULE_WRAPPER(compressed_data_out); CROSSMODULE_WRAPPER(compressed_data_info); CROSSMODULE_WRAPPER(compressed_data_has_nulls); CROSSMODULE_WRAPPER(deltadelta_compressor_append); CROSSMODULE_WRAPPER(deltadelta_compressor_finish); CROSSMODULE_WRAPPER(gorilla_compressor_append); CROSSMODULE_WRAPPER(gorilla_compressor_finish); CROSSMODULE_WRAPPER(dictionary_compressor_append); CROSSMODULE_WRAPPER(dictionary_compressor_finish); CROSSMODULE_WRAPPER(array_compressor_append); CROSSMODULE_WRAPPER(array_compressor_finish); CROSSMODULE_WRAPPER(bool_compressor_append); CROSSMODULE_WRAPPER(bool_compressor_finish); CROSSMODULE_WRAPPER(uuid_compressor_append); CROSSMODULE_WRAPPER(uuid_compressor_finish); CROSSMODULE_WRAPPER(create_compressed_chunk); CROSSMODULE_WRAPPER(compress_chunk); CROSSMODULE_WRAPPER(decompress_chunk); CROSSMODULE_WRAPPER(rebuild_columnstore); CROSSMODULE_WRAPPER(bloom1_contains); CROSSMODULE_WRAPPER(bloom1_contains_any); /* continuous aggregate */ CROSSMODULE_WRAPPER(continuous_agg_refresh); CROSSMODULE_WRAPPER(continuous_agg_validate_query); CROSSMODULE_WRAPPER(continuous_agg_get_bucket_function); CROSSMODULE_WRAPPER(continuous_agg_get_bucket_function_info); CROSSMODULE_WRAPPER(continuous_agg_get_grouping_columns); CROSSMODULE_WRAPPER(chunk_freeze_chunk); CROSSMODULE_WRAPPER(chunk_unfreeze_chunk); CROSSMODULE_WRAPPER(recompress_chunk_segmentwise); CROSSMODULE_WRAPPER(get_compressed_chunk_index_for_recompression); CROSSMODULE_WRAPPER(merge_chunks); CROSSMODULE_WRAPPER(split_chunk); CROSSMODULE_WRAPPER(detach_chunk); CROSSMODULE_WRAPPER(attach_chunk); CROSSMODULE_WRAPPER(estimate_compressed_batch_size); /* * casting a function pointer to a pointer of another type is undefined * behavior, so we need one of these for every function type we have */ static void error_no_default_fn_community(void) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("functionality not supported under the current \"%s\" license. Learn more at " "https://tsdb.co/pdbir1r3", ts_guc_license), errhint("To access all features and the best time-series experience, try out " "Timescale Cloud."))); } static bool error_no_default_fn_bool_void_community(void) { error_no_default_fn_community(); pg_unreachable(); } static bool job_execute_default_fn(BgwJob *job) { error_no_default_fn_community(); pg_unreachable(); } static void tsl_postprocess_plan_stub(PlannedStmt *stmt) { } static bool process_compress_table_default(Hypertable *ht, WithClauseResult *with_clause_options) { error_no_default_fn_community(); pg_unreachable(); } static void columnstore_setup_default(Hypertable *ht, WithClauseResult *with_clause_options) { error_no_default_fn_community(); pg_unreachable(); } static Datum error_no_default_fn_pg_community(PG_FUNCTION_ARGS) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function \"%s\" is not supported under the current \"%s\" license", fcinfo->flinfo ? get_func_name(fcinfo->flinfo->fn_oid) : "unknown", ts_guc_license), errhint("Upgrade your license to 'timescale' to use this free community feature."))); pg_unreachable(); } static void error_no_default_fn_chunk_insert_state_community(ChunkInsertState *cis, TupleTableSlot *slot) { error_no_default_fn_community(); pg_unreachable(); } /* * TSL library is not loaded by the replication worker for some reason, * so a call to `compressed_data_in` and `compressed_data_out` functions would * produce a misleading error saying that your license is "timescale" and you * should upgrade to "timescale" license, even if you have already upgraded. * * As a workaround, we try to load the TSL module it in this function. * It will still error out in the "apache" version */ static Datum process_compressed_data_in(PG_FUNCTION_ARGS) { ts_license_enable_module_loading(); if (ts_cm_functions->compressed_data_in != process_compressed_data_in) return ts_cm_functions->compressed_data_in(fcinfo); error_no_default_fn_pg_community(fcinfo); pg_unreachable(); } static Datum process_compressed_data_out(PG_FUNCTION_ARGS) { ts_license_enable_module_loading(); if (ts_cm_functions->compressed_data_out != process_compressed_data_out) return ts_cm_functions->compressed_data_out(fcinfo); error_no_default_fn_pg_community(fcinfo); pg_unreachable(); } static DDLResult process_cagg_viewstmt_default(Node *stmt, const char *query_string, void *pstmt, WithClauseResult *with_clause_options) { return error_no_default_fn_bool_void_community(); } static void continuous_agg_update_options_default(ContinuousAgg *cagg, WithClauseResult *with_clause_options) { error_no_default_fn_community(); pg_unreachable(); } static void continuous_agg_invalidate_raw_ht_all_default(const Hypertable *raw_ht, int64 start, int64 end) { error_no_default_fn_community(); pg_unreachable(); } static void continuous_agg_invalidate_mat_ht_all_default(const Hypertable *raw_ht, const Hypertable *mat_ht, int64 start, int64 end) { error_no_default_fn_community(); pg_unreachable(); } static void continuous_agg_dml_invalidate_default(int32 hypertable_id, Relation chunk_rel, HeapTuple chunk_tuple, HeapTuple chunk_newtuple, bool update) { error_no_default_fn_community(); pg_unreachable(); } TS_FUNCTION_INFO_V1(ts_tsl_loaded); PGDLLEXPORT Datum ts_tsl_loaded(PG_FUNCTION_ARGS) { PG_RETURN_BOOL(ts_cm_functions != &ts_cm_functions_default); } static void preprocess_query_tsl_default_fn_community(Query *parse, int *cursor_opts) { /* No op in community licensed code */ } static PGFunction bloom1_get_hash_function_default(Oid type, FmgrInfo **finfo) { error_no_default_fn_community(); pg_unreachable(); } /* * Define cross-module functions' default values: * If the submodule isn't activated, using one of the cm functions will throw an * exception. */ TSDLLEXPORT CrossModuleFunctions ts_cm_functions_default = { .create_upper_paths_hook = NULL, .set_rel_pathlist_dml = NULL, .set_rel_pathlist_query = NULL, .sort_transform_replace_pathkeys = NULL, .process_altertable_cmd = NULL, .process_rename_cmd = NULL, /* gapfill */ .gapfill_marker = error_no_default_fn_pg_community, .gapfill_int16_time_bucket = error_no_default_fn_pg_community, .gapfill_int32_time_bucket = error_no_default_fn_pg_community, .gapfill_int64_time_bucket = error_no_default_fn_pg_community, .gapfill_date_time_bucket = error_no_default_fn_pg_community, .gapfill_timestamp_time_bucket = error_no_default_fn_pg_community, .gapfill_timestamptz_time_bucket = error_no_default_fn_pg_community, .gapfill_timestamptz_timezone_time_bucket = error_no_default_fn_pg_community, /* bgw policies */ .policy_compression_add = error_no_default_fn_pg_community, .policy_compression_remove = error_no_default_fn_pg_community, .policy_recompression_proc = error_no_default_fn_pg_community, .policy_compression_check = error_no_default_fn_pg_community, .policy_refresh_cagg_add = error_no_default_fn_pg_community, .policy_refresh_cagg_proc = error_no_default_fn_pg_community, .policy_refresh_cagg_check = error_no_default_fn_pg_community, .policy_refresh_cagg_remove = error_no_default_fn_pg_community, .policy_process_hyper_inval_add = error_no_default_fn_pg_community, .policy_process_hyper_inval_proc = error_no_default_fn_pg_community, .policy_process_hyper_inval_check = error_no_default_fn_pg_community, .policy_process_hyper_inval_remove = error_no_default_fn_pg_community, .policy_reorder_add = error_no_default_fn_pg_community, .policy_reorder_proc = error_no_default_fn_pg_community, .policy_reorder_check = error_no_default_fn_pg_community, .policy_reorder_remove = error_no_default_fn_pg_community, .policy_retention_add = error_no_default_fn_pg_community, .policy_retention_proc = error_no_default_fn_pg_community, .policy_retention_check = error_no_default_fn_pg_community, .policy_retention_remove = error_no_default_fn_pg_community, .job_add = error_no_default_fn_pg_community, .job_alter = error_no_default_fn_pg_community, .job_alter_set_hypertable_id = error_no_default_fn_pg_community, .job_delete = error_no_default_fn_pg_community, .job_run = error_no_default_fn_pg_community, .job_execute = job_execute_default_fn, .reorder_chunk = error_no_default_fn_pg_community, .move_chunk = error_no_default_fn_pg_community, .policies_add = error_no_default_fn_pg_community, .policies_remove = error_no_default_fn_pg_community, .policies_remove_all = error_no_default_fn_pg_community, .policies_alter = error_no_default_fn_pg_community, .policies_show = error_no_default_fn_pg_community, .tsl_postprocess_plan = tsl_postprocess_plan_stub, .process_cagg_viewstmt = process_cagg_viewstmt_default, .continuous_agg_refresh = error_no_default_fn_pg_community, .continuous_agg_invalidate_raw_ht = continuous_agg_invalidate_raw_ht_all_default, .continuous_agg_invalidate_mat_ht = continuous_agg_invalidate_mat_ht_all_default, .continuous_agg_dml_invalidate = continuous_agg_dml_invalidate_default, .continuous_agg_update_options = continuous_agg_update_options_default, .continuous_agg_validate_query = error_no_default_fn_pg_community, .continuous_agg_get_bucket_function = error_no_default_fn_pg_community, .continuous_agg_get_bucket_function_info = error_no_default_fn_pg_community, .continuous_agg_get_grouping_columns = error_no_default_fn_pg_community, /* compression */ .compressed_data_send = error_no_default_fn_pg_community, .compressed_data_recv = error_no_default_fn_pg_community, .compressed_data_in = process_compressed_data_in, .compressed_data_out = process_compressed_data_out, .process_compress_table = process_compress_table_default, .create_compressed_chunk = error_no_default_fn_pg_community, .compress_chunk = error_no_default_fn_pg_community, .decompress_chunk = error_no_default_fn_pg_community, .rebuild_columnstore = error_no_default_fn_pg_community, .compressed_data_decompress_forward = error_no_default_fn_pg_community, .compressed_data_decompress_reverse = error_no_default_fn_pg_community, .compressed_data_column_size = error_no_default_fn_pg_community, .compressed_data_to_array = error_no_default_fn_pg_community, .deltadelta_compressor_append = error_no_default_fn_pg_community, .deltadelta_compressor_finish = error_no_default_fn_pg_community, .gorilla_compressor_append = error_no_default_fn_pg_community, .gorilla_compressor_finish = error_no_default_fn_pg_community, .dictionary_compressor_append = error_no_default_fn_pg_community, .dictionary_compressor_finish = error_no_default_fn_pg_community, .array_compressor_append = error_no_default_fn_pg_community, .array_compressor_finish = error_no_default_fn_pg_community, .bool_compressor_append = error_no_default_fn_pg_community, .bool_compressor_finish = error_no_default_fn_pg_community, .uuid_compressor_append = error_no_default_fn_pg_community, .uuid_compressor_finish = error_no_default_fn_pg_community, .bloom1_contains = error_no_default_fn_pg_community, .bloom1_contains_any = error_no_default_fn_pg_community, .bloom1_get_hash_function = bloom1_get_hash_function_default, .decompress_batches_for_insert = error_no_default_fn_chunk_insert_state_community, .init_decompress_state_for_insert = error_no_default_fn_chunk_insert_state_community, .columnstore_setup = columnstore_setup_default, .show_chunk = error_no_default_fn_pg_community, .create_chunk = error_no_default_fn_pg_community, .chunk_freeze_chunk = error_no_default_fn_pg_community, .chunk_unfreeze_chunk = error_no_default_fn_pg_community, .recompress_chunk_segmentwise = error_no_default_fn_pg_community, .get_compressed_chunk_index_for_recompression = error_no_default_fn_pg_community, .preprocess_query_tsl = preprocess_query_tsl_default_fn_community, .merge_chunks = error_no_default_fn_pg_community, .split_chunk = error_no_default_fn_pg_community, .detach_chunk = error_no_default_fn_pg_community, .attach_chunk = error_no_default_fn_pg_community, }; TSDLLEXPORT CrossModuleFunctions *ts_cm_functions = &ts_cm_functions_default; ================================================ FILE: src/cross_module_fn.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <commands/event_trigger.h> #include <fmgr.h> #include <optimizer/planner.h> #include <utils/array.h> #include <utils/jsonb.h> #include <utils/timestamp.h> #include "compat/compat.h" #include "bgw/job.h" #include "export.h" #include "planner/planner.h" #include "process_utility.h" #include "ts_catalog/continuous_agg.h" #include "with_clause/with_clause_parser.h" /* * To define a cross-module function add it to this struct, add a default * version in to ts_cm_functions_default cross_module_fn.c, and the overridden * version to tsl_cm_functions tsl/src/init.c. * This will allow the function to be called from this codebase as * ts_cm_functions-><function name> */ typedef struct JsonbParseState JsonbParseState; typedef struct Hypertable Hypertable; typedef struct Chunk Chunk; typedef struct ChunkInsertState ChunkInsertState; typedef struct CopyChunkState CopyChunkState; typedef struct ModifyHypertableState ModifyHypertableState; typedef struct RowCompressor RowCompressor; typedef struct BulkWriter BulkWriter; typedef struct CrossModuleFunctions { PGFunction policy_compression_add; PGFunction policy_compression_remove; PGFunction policy_recompression_proc; PGFunction policy_compression_check; PGFunction policy_refresh_cagg_add; PGFunction policy_refresh_cagg_proc; PGFunction policy_refresh_cagg_check; PGFunction policy_refresh_cagg_remove; PGFunction policy_process_hyper_inval_add; PGFunction policy_process_hyper_inval_proc; PGFunction policy_process_hyper_inval_check; PGFunction policy_process_hyper_inval_remove; PGFunction policy_reorder_add; PGFunction policy_reorder_proc; PGFunction policy_reorder_check; PGFunction policy_reorder_remove; PGFunction policy_retention_add; PGFunction policy_retention_proc; PGFunction policy_retention_check; PGFunction policy_retention_remove; PGFunction policies_add; PGFunction policies_remove; PGFunction policies_remove_all; PGFunction policies_alter; PGFunction policies_show; PGFunction job_add; PGFunction job_alter; PGFunction job_alter_set_hypertable_id; PGFunction job_delete; PGFunction job_run; bool (*job_execute)(BgwJob *job); void (*create_upper_paths_hook)(PlannerInfo *, UpperRelationKind, RelOptInfo *, RelOptInfo *, TsRelType input_reltype, Hypertable *ht, void *extra); void (*set_rel_pathlist_dml)(PlannerInfo *, RelOptInfo *, Index, RangeTblEntry *, Hypertable *); void (*set_rel_pathlist_query)(PlannerInfo *, RelOptInfo *, Index, RangeTblEntry *, Hypertable *); void (*sort_transform_replace_pathkeys)(void *path, List *transformed_pathkeys, List *original_pathkeys); /* gapfill */ PGFunction gapfill_marker; PGFunction gapfill_int16_time_bucket; PGFunction gapfill_int32_time_bucket; PGFunction gapfill_int64_time_bucket; PGFunction gapfill_date_time_bucket; PGFunction gapfill_timestamp_time_bucket; PGFunction gapfill_timestamptz_time_bucket; PGFunction gapfill_timestamptz_timezone_time_bucket; PGFunction reorder_chunk; PGFunction move_chunk; /* Vectorized queries */ void (*tsl_postprocess_plan)(PlannedStmt *stmt); /* Continuous Aggregates */ DDLResult (*process_cagg_viewstmt)(Node *stmt, const char *query_string, void *pstmt, WithClauseResult *with_clause_options); PGFunction continuous_agg_refresh; void (*continuous_agg_invalidate_raw_ht)(const Hypertable *raw_ht, int64 start, int64 end); void (*continuous_agg_invalidate_mat_ht)(const Hypertable *raw_ht, const Hypertable *mat_ht, int64 start, int64 end); void (*continuous_agg_dml_invalidate)(int32 hypertable_id, Relation chunk_rel, HeapTuple chunk_tuple, HeapTuple chunk_newtuple, bool update); void (*continuous_agg_update_options)(ContinuousAgg *cagg, WithClauseResult *with_clause_options); PGFunction continuous_agg_validate_query; PGFunction continuous_agg_get_bucket_function; PGFunction continuous_agg_get_bucket_function_info; PGFunction continuous_agg_get_grouping_columns; PGFunction compressed_data_send; PGFunction compressed_data_recv; PGFunction compressed_data_in; PGFunction compressed_data_out; PGFunction compressed_data_info; PGFunction compressed_data_has_nulls; bool (*process_compress_table)(Hypertable *ht, WithClauseResult *with_clause_options); void (*process_altertable_cmd)(Hypertable *ht, const AlterTableCmd *cmd); void (*process_rename_cmd)(Oid relid, Cache *hcache, const RenameStmt *stmt); PGFunction create_compressed_chunk; PGFunction compress_chunk; PGFunction decompress_chunk; PGFunction rebuild_columnstore; void (*decompress_batches_for_insert)(ChunkInsertState *state, TupleTableSlot *slot); void (*init_decompress_state_for_insert)(ChunkInsertState *state, TupleTableSlot *slot); bool (*decompress_target_segments)(ModifyHypertableState *ht_state); void (*columnstore_setup)(Hypertable *ht, WithClauseResult *with_clause_options); RowCompressor *(*compressor_init)(Relation in_rel, BulkWriter **bulk_writer, bool sort, int tuple_sort_limit); void (*compressor_set_invalidation)(RowCompressor *compressor, Hypertable *ht, Oid chunk_relid); void (*compressor_add_slot)(RowCompressor *compressor, BulkWriter *bulk_writer, TupleTableSlot *slot); void (*compressor_flush)(RowCompressor *compressor, BulkWriter *bulk_writer); void (*compressor_free)(RowCompressor *compressor, BulkWriter *bulk_writer); Chunk *(*compression_chunk_create)(Hypertable *ht, Chunk *src_chunk); /* The compression functions below are not installed in SQL as part of create extension; * They are installed and tested during testing scripts. They are exposed in cross-module * functions because they may be very useful for debugging customer problems if the sql * stub is installed on the customer's machine. */ PGFunction compressed_data_decompress_forward; PGFunction compressed_data_decompress_reverse; PGFunction compressed_data_column_size; PGFunction compressed_data_to_array; PGFunction deltadelta_compressor_append; PGFunction deltadelta_compressor_finish; PGFunction gorilla_compressor_append; PGFunction gorilla_compressor_finish; PGFunction dictionary_compressor_append; PGFunction dictionary_compressor_finish; PGFunction array_compressor_append; PGFunction array_compressor_finish; PGFunction bool_compressor_append; PGFunction bool_compressor_finish; PGFunction uuid_compressor_append; PGFunction uuid_compressor_finish; PGFunction bloom1_contains; PGFunction bloom1_contains_any; PGFunction (*bloom1_get_hash_function)(Oid type, FmgrInfo **finfo); PGFunction create_chunk; PGFunction show_chunk; PGFunction chunk_freeze_chunk; PGFunction chunk_unfreeze_chunk; PGFunction recompress_chunk_segmentwise; PGFunction get_compressed_chunk_index_for_recompression; void (*preprocess_query_tsl)(Query *parse, int *cursor_opts); PGFunction merge_chunks; PGFunction split_chunk; PGFunction detach_chunk; PGFunction attach_chunk; PGFunction estimate_compressed_batch_size; } CrossModuleFunctions; extern TSDLLEXPORT CrossModuleFunctions *ts_cm_functions; extern TSDLLEXPORT CrossModuleFunctions ts_cm_functions_default; ================================================ FILE: src/custom_type_cache.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/namespace.h> #include <catalog/pg_type.h> #include <utils/syscache.h> #include "custom_type_cache.h" #include "extension_constants.h" #include "ts_catalog/catalog.h" /* Information about functions that we put in the cache */ static CustomTypeInfo typeinfo[_CUSTOM_TYPE_MAX_INDEX] = { [CUSTOM_TYPE_COMPRESSED_DATA] = { .schema_name = INTERNAL_SCHEMA_NAME, .type_name = "compressed_data", .type_oid = InvalidOid, }, [CUSTOM_TYPE_BLOOM1] = { .schema_name = INTERNAL_SCHEMA_NAME, .type_name = "bloom1", .type_oid = InvalidOid, } }; extern CustomTypeInfo * ts_custom_type_cache_get(CustomType type) { CustomTypeInfo *tinfo; if (type >= _CUSTOM_TYPE_MAX_INDEX) elog(ERROR, "invalid timescaledb type %d", type); tinfo = &typeinfo[type]; if (!OidIsValid(tinfo->type_oid)) { Oid schema_oid = LookupExplicitNamespace(tinfo->schema_name, false); Oid type_oid = GetSysCacheOid2(TYPENAMENSP, Anum_pg_type_oid, CStringGetDatum(tinfo->type_name), ObjectIdGetDatum(schema_oid)); if (!OidIsValid(type_oid)) elog(ERROR, "unknown timescaledb type %s", tinfo->type_name); tinfo->type_oid = type_oid; } return tinfo; } ================================================ FILE: src/custom_type_cache.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include "compat/compat.h" #include <postgres.h> typedef enum CustomType { CUSTOM_TYPE_COMPRESSED_DATA = 0, CUSTOM_TYPE_BLOOM1, _CUSTOM_TYPE_MAX_INDEX } CustomType; typedef struct CustomTypeInfo { const char *schema_name; const char *type_name; Oid type_oid; } CustomTypeInfo; extern TSDLLEXPORT CustomTypeInfo *ts_custom_type_cache_get(CustomType type); ================================================ FILE: src/debug_assert.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> /* * Macro that expands to an assert in debug builds and to an ereport in * release builds. * * This allow you to start the debugger if internal assumptions are violated * in debug builds, but a release build will just print an error and abort the * transaction but and not crash the server. * * The error code is automatically set to ERRCODE_INTERNAL_ERROR and the error * details contains the assertion that failed in text format. * * The macro should be used for checks that are not expected to occur in * normal execution, or which can occur in odd corner-cases for conditions out * of our control (e.g., unexpected changes to the metadata) so if you have a * test that trigger the error, this macro should not be used. */ #define Ensure(COND, FMT, ...) \ do \ { \ if (unlikely(!(COND))) \ { \ Assert(false); \ ereport(ERROR, \ (errcode(ERRCODE_INTERNAL_ERROR), \ errdetail("Assertion '" #COND "' failed."), \ errmsg(FMT, ##__VA_ARGS__))); \ } \ } while (0) ================================================ FILE: src/debug_point.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include "debug_point.h" #include <postgres.h> #include <fmgr.h> #include <access/hash.h> #include <access/xact.h> #include <miscadmin.h> #include <storage/ipc.h> #include <storage/lock.h> #include <utils/builtins.h> #include "annotations.h" #include "export.h" TS_FUNCTION_INFO_V1(ts_debug_point_enable); TS_FUNCTION_INFO_V1(ts_debug_point_release); TS_FUNCTION_INFO_V1(ts_debug_point_id); /* * Debug points only exist in debug code and are intended to allow * more controlled testing of the code. * * Debug points can be used as a wait point (1) or as a way to * introduce error injections (2). * * (1) When used as wait points, execution will halt until the debug points are * explicitly released. * * When waiting on a debug point, there is an attempt to take a shared lock on it. * If the debug point is enabled by locking using an exclusive * lock, this will block all waiters. Once the exclusive lock is released, all * waiters will be able to proceed. * * (2) is similar to (1), but, instead of waiting for the debug point to be * released, it will generate an error immediately. * */ /* Tag for debug points. * * Each debug point is identified by a string that is hashed to a 8-byte * number and used with the normal advisory locks available in PostgreSQL. */ typedef struct DebugPoint { const char *name; LOCKTAG tag; } DebugPoint; static uint64 debug_point_name_to_id(const char *name) { return DatumGetUInt32(hash_any((const unsigned char *) name, strlen(name))); } static void debug_point_init(DebugPoint *point, const char *name) { /* Use 64-bit hashing to get two independent 32-bit hashes */ uint64 hash = debug_point_name_to_id(name); SET_LOCKTAG_ADVISORY(point->tag, MyDatabaseId, (uint32) (hash >> 32), (uint32) hash, 1); point->name = pstrdup(name); ereport(DEBUG3, (errmsg("initializing debug point '%s' to use " UINT64_FORMAT, point->name, hash))); } static void debug_point_enable(const DebugPoint *point) { LockAcquireResult lock_acquire_result; ereport(DEBUG1, (errmsg("enabling debug point \"%s\"", point->name))); lock_acquire_result = LockAcquire(&point->tag, ExclusiveLock, true, true); switch (lock_acquire_result) { case LOCKACQUIRE_ALREADY_HELD: case LOCKACQUIRE_ALREADY_CLEAR: LockRelease(&point->tag, ExclusiveLock, true); TS_FALLTHROUGH; case LOCKACQUIRE_NOT_AVAIL: ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("debug point \"%s\" already enabled", point->name))); break; case LOCKACQUIRE_OK: break; } } static void debug_point_release(const DebugPoint *point) { ereport(DEBUG1, (errmsg("releasing debug point \"%s\"", point->name))); if (!LockRelease(&point->tag, ExclusiveLock, true)) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("cannot release debug point \"%s\"", point->name))); } /* * Enable a debug point to block when being reached. * * This function will always succeed since we will not lock the debug point if * it is already locked. A notice will be printed if the debug point is already * enabled. */ Datum ts_debug_point_enable(PG_FUNCTION_ARGS) { text *name = PG_GETARG_TEXT_PP(0); DebugPoint point; if (PG_ARGISNULL(0)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("no name provided"))); debug_point_init(&point, text_to_cstring(name)); debug_point_enable(&point); PG_RETURN_VOID(); } /* * Release a debug point allowing execution to proceed. */ Datum ts_debug_point_release(PG_FUNCTION_ARGS) { text *name = PG_GETARG_TEXT_PP(0); DebugPoint point; if (PG_ARGISNULL(0)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("no name provided"))); debug_point_init(&point, text_to_cstring(name)); debug_point_release(&point); PG_RETURN_VOID(); } /* * Get the debug point identifier from the name. */ Datum ts_debug_point_id(PG_FUNCTION_ARGS) { text *name = PG_GETARG_TEXT_PP(0); if (PG_ARGISNULL(0)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("no name provided"))); PG_RETURN_UINT64(debug_point_name_to_id(text_to_cstring(name))); } /* * Wait for the debug point to be released. * * This is handled by first trying to get a shared lock, which will not block * other sessions that try to grab the same lock but will block if an * exclusive lock is already taken, and then release the lock immediately * after. * * This function can decide to block while taking the shared lock or it can * have a retry loop to take the share lock. This retry loop option is useful * in cases where this function gets called from deep down inside a transaction * where interrupts are not being served currently. */ void ts_debug_point_wait(const char *name, bool blocking) { DebugPoint point; LockAcquireResult lock_acquire_result pg_attribute_unused(); bool lock_release_result pg_attribute_unused(); /* Ensure that we are in a transaction before trying for locks */ if (!IsTransactionState()) return; debug_point_init(&point, name); ereport(DEBUG3, (errmsg("waiting on debug point '%s'", point.name))); if (blocking) lock_acquire_result = LockAcquire(&point.tag, ShareLock, true, false); else { /* * Trying to wait indefinitely here could lead to hangs. The current * behavior is to retry for retry_count and return with a warning * if that's crossed. * * If required, in future, we could take an additional option to decide * if the caller wants to retry indefinitely or return with a warning. * But the current behavior based on the "blocking" argument is ok for * now. */ unsigned int retry_count = 1000; /* try to acquire the lock without waiting. */ do { /* try to acquire the lock without waiting. */ lock_acquire_result = LockAcquire(&point.tag, ShareLock, true, true); if (lock_acquire_result == LOCKACQUIRE_OK) break; /* don't dare to take a lock when the proc is exiting! */ if (proc_exit_inprogress || ProcDiePending) return; if (retry_count == 0) { elog(WARNING, "timeout while acquiring debug point lock"); return; } retry_count--; /* retry after some time */ pg_usleep(100L); } while (lock_acquire_result == LOCKACQUIRE_NOT_AVAIL); } Assert(lock_acquire_result == LOCKACQUIRE_OK); lock_release_result = LockRelease(&point.tag, ShareLock, true); Assert(lock_release_result); ereport(DEBUG3, (errmsg("proceeding after debug point '%s'", point.name))); } /* * Produce an error in case if the debug point is enabled. * * The idea is to enable the debug point separately first which * acquires a ShareLock on this tag. With the debug point enabled, this function * when invoked will not get the exclusive lock and will be able to raise * the error as desired. */ void ts_debug_point_raise_error_if_enabled(const char *name) { DebugPoint point; LockAcquireResult lock_acquire_result; debug_point_init(&point, name); lock_acquire_result = LockAcquire(&point.tag, ExclusiveLock, true, true); switch (lock_acquire_result) { case LOCKACQUIRE_OK: case LOCKACQUIRE_ALREADY_HELD: case LOCKACQUIRE_ALREADY_CLEAR: /* Release/decrement lock count */ LockRelease(&point.tag, ExclusiveLock, true); if (lock_acquire_result == LOCKACQUIRE_OK) return; break; case LOCKACQUIRE_NOT_AVAIL: break; } ereport(ERROR, (errcode(ERRCODE_TRIGGERED_ACTION_EXCEPTION), errmsg("error injected at debug point '%s'", point.name))); } ================================================ FILE: src/debug_point.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include "export.h" extern TSDLLEXPORT void ts_debug_point_wait(const char *name, bool blocking); extern TSDLLEXPORT void ts_debug_point_raise_error_if_enabled(const char *name); #ifdef TS_DEBUG #define DEBUG_WAITPOINT(NAME) ts_debug_point_wait((NAME), true) #define DEBUG_RETRY_WAITPOINT(NAME) ts_debug_point_wait((NAME), false) #define DEBUG_ERROR_INJECTION(NAME) ts_debug_point_raise_error_if_enabled((NAME)) #else #define DEBUG_WAITPOINT(NAME) #define DEBUG_RETRY_WAITPOINT(NAME) #define DEBUG_ERROR_INJECTION(NAME) #endif ================================================ FILE: src/dimension.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/relscan.h> #include <catalog/namespace.h> #include <catalog/pg_type.h> #include <commands/tablecmds.h> #include <funcapi.h> #include <miscadmin.h> #include <nodes/makefuncs.h> #include <storage/lmgr.h> #include <utils/builtins.h> #include <utils/lsyscache.h> #include <utils/syscache.h> #include <utils/timestamp.h> #include "compat/compat.h" #include "cross_module_fn.h" #include "debug_point.h" #include "dimension.h" #include "dimension_slice.h" #include "dimension_vector.h" #include "error_utils.h" #include "errors.h" #include "hypertable.h" #include "hypertable_cache.h" #include "indexing.h" #include "partitioning.h" #include "scanner.h" #include "time_utils.h" #include "ts_catalog/catalog.h" #include "utils.h" /* add_dimension record attribute numbers */ enum Anum_add_dimension { Anum_add_dimension_id = 1, Anum_add_dimension_schema_name, Anum_add_dimension_table_name, Anum_add_dimension_column_name, Anum_add_dimension_created, _Anum_add_dimension_max, }; #define Natts_add_dimension (_Anum_add_dimension_max - 1) /* * Generic add dimension attributes */ enum Anum_generic_add_dimension { Anum_generic_add_dimension_id = 1, Anum_generic_add_dimension_created, _Anum_generic_add_dimension_max, }; #define Natts_generic_add_dimension (_Anum_generic_add_dimension_max - 1) static int cmp_dimension_id(const void *left, const void *right) { const Dimension *diml = (Dimension *) left; const Dimension *dimr = (Dimension *) right; if (diml->fd.id < dimr->fd.id) return -1; if (diml->fd.id > dimr->fd.id) return 1; return 0; } TS_FUNCTION_INFO_V1(ts_hash_dimension); TS_FUNCTION_INFO_V1(ts_range_dimension); PG_FUNCTION_INFO_V1(ts_dimension_info_in); PG_FUNCTION_INFO_V1(ts_dimension_info_out); const Dimension * ts_hyperspace_get_dimension_by_id(const Hyperspace *hs, int32 id) { Dimension dim = { .fd.id = id, }; return bsearch(&dim, hs->dimensions, hs->num_dimensions, sizeof(Dimension), cmp_dimension_id); } Dimension * ts_hyperspace_get_mutable_dimension_by_name(Hyperspace *hs, DimensionType type, const char *name) { int i; for (i = 0; i < hs->num_dimensions; i++) { Dimension *dim = &hs->dimensions[i]; if ((type == DIMENSION_TYPE_ANY || dim->type == type) && namestrcmp(&dim->fd.column_name, name) == 0) return dim; } return NULL; } const Dimension * ts_hyperspace_get_dimension_by_name(const Hyperspace *hs, DimensionType type, const char *name) { return ts_hyperspace_get_mutable_dimension_by_name((Hyperspace *) hs, type, name); } Dimension * ts_hyperspace_get_mutable_dimension(Hyperspace *hs, DimensionType type, Index n) { int i; for (i = 0; i < hs->num_dimensions; i++) { if (type == DIMENSION_TYPE_ANY || hs->dimensions[i].type == type) { if (n == 0) return &hs->dimensions[i]; n--; } } return NULL; } const Dimension * ts_hyperspace_get_dimension(const Hyperspace *hs, DimensionType type, Index n) { return ts_hyperspace_get_mutable_dimension((Hyperspace *) hs, type, n); } static int hyperspace_get_num_dimensions_by_type(Hyperspace *hs, DimensionType type) { int i; int n = 0; for (i = 0; i < hs->num_dimensions; i++) { if (type == DIMENSION_TYPE_ANY || hs->dimensions[i].type == type) n++; } return n; } static inline DimensionType dimension_type(TupleInfo *ti) { if (slot_attisnull(ti->slot, Anum_dimension_interval_length) && !slot_attisnull(ti->slot, Anum_dimension_num_slices)) return DIMENSION_TYPE_CLOSED; if (!slot_attisnull(ti->slot, Anum_dimension_interval_length) && slot_attisnull(ti->slot, Anum_dimension_num_slices)) return DIMENSION_TYPE_OPEN; elog(ERROR, "invalid partitioning dimension"); /* suppress compiler warning on MSVC */ return DIMENSION_TYPE_ANY; } static void dimension_fill_in_from_tuple(Dimension *d, TupleInfo *ti, Oid main_table_relid) { Datum values[Natts_dimension]; bool isnull[Natts_dimension]; bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); /* * With need to use heap_deform_tuple() rather than GETSTRUCT(), since * optional values may be omitted from the tuple. */ heap_deform_tuple(tuple, ts_scanner_get_tupledesc(ti), values, isnull); d->type = dimension_type(ti); d->fd.id = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_dimension_id)]); d->fd.hypertable_id = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_dimension_hypertable_id)]); d->fd.aligned = DatumGetBool(values[AttrNumberGetAttrOffset(Anum_dimension_aligned)]); d->fd.column_type = DatumGetObjectId(values[AttrNumberGetAttrOffset(Anum_dimension_column_type)]); namestrcpy(&d->fd.column_name, DatumGetCString(values[AttrNumberGetAttrOffset(Anum_dimension_column_name)])); if (!isnull[AttrNumberGetAttrOffset(Anum_dimension_partitioning_func_schema)] && !isnull[AttrNumberGetAttrOffset(Anum_dimension_partitioning_func)]) { MemoryContext old; d->fd.num_slices = DatumGetInt16(values[AttrNumberGetAttrOffset(Anum_dimension_num_slices)]); namestrcpy(&d->fd.partitioning_func_schema, DatumGetCString( values[AttrNumberGetAttrOffset(Anum_dimension_partitioning_func_schema)])); namestrcpy(&d->fd.partitioning_func, DatumGetCString( values[AttrNumberGetAttrOffset(Anum_dimension_partitioning_func)])); old = MemoryContextSwitchTo(ti->mctx); d->partitioning = ts_partitioning_info_create(NameStr(d->fd.partitioning_func_schema), NameStr(d->fd.partitioning_func), NameStr(d->fd.column_name), d->type, main_table_relid); MemoryContextSwitchTo(old); } if (!isnull[AttrNumberGetAttrOffset(Anum_dimension_integer_now_func_schema)] && !isnull[AttrNumberGetAttrOffset(Anum_dimension_integer_now_func)]) { namestrcpy(&d->fd.integer_now_func_schema, DatumGetCString( values[AttrNumberGetAttrOffset(Anum_dimension_integer_now_func_schema)])); namestrcpy(&d->fd.integer_now_func, DatumGetCString( values[AttrNumberGetAttrOffset(Anum_dimension_integer_now_func)])); } if (IS_CLOSED_DIMENSION(d)) d->fd.num_slices = DatumGetInt16(values[AttrNumberGetAttrOffset(Anum_dimension_num_slices)]); else { d->fd.interval_length = DatumGetInt64(values[AttrNumberGetAttrOffset(Anum_dimension_interval_length)]); if (!isnull[AttrNumberGetAttrOffset(Anum_dimension_compress_interval_length)]) d->fd.compress_interval_length = DatumGetInt64( values[AttrNumberGetAttrOffset(Anum_dimension_compress_interval_length)]); } d->column_attno = get_attnum(main_table_relid, NameStr(d->fd.column_name)); d->main_table_relid = main_table_relid; if (should_free) heap_freetuple(tuple); } static Datum create_range_datum(FunctionCallInfo fcinfo, DimensionSlice *slice) { TupleDesc tupdesc; Datum values[2]; bool nulls[2] = { false }; HeapTuple tuple; if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) elog(ERROR, "function returning record called in context that cannot accept type record"); tupdesc = BlessTupleDesc(tupdesc); values[0] = Int64GetDatum(slice->fd.range_start); values[1] = Int64GetDatum(slice->fd.range_end); tuple = heap_form_tuple(tupdesc, values, nulls); return HeapTupleGetDatum(tuple); } static DimensionSlice * calculate_open_range_default(const Dimension *dim, int64 value) { int64 range_start, range_end; Oid dimtype = ts_dimension_get_partition_type(dim); if (value < 0) { const int64 dim_min = ts_time_get_min(dimtype); range_end = ((value + 1) / dim->fd.interval_length) * dim->fd.interval_length; /* prevent integer underflow */ if (dim_min - range_end > -dim->fd.interval_length) range_start = DIMENSION_SLICE_MINVALUE; else range_start = range_end - dim->fd.interval_length; } else { const int64 dim_end = ts_time_get_max(dimtype); range_start = (value / dim->fd.interval_length) * dim->fd.interval_length; /* prevent integer overflow */ if (dim_end - range_start < dim->fd.interval_length) range_end = DIMENSION_SLICE_MAXVALUE; else range_end = range_start + dim->fd.interval_length; } return ts_dimension_slice_create(dim->fd.id, range_start, range_end); } TS_FUNCTION_INFO_V1(ts_dimension_calculate_open_range_default); /* * Expose open dimension range calculation for testing purposes. */ Datum ts_dimension_calculate_open_range_default(PG_FUNCTION_ARGS) { int64 value = PG_GETARG_INT64(0); Dimension dim = { .type = DIMENSION_TYPE_OPEN, .fd.id = 0, .fd.interval_length = PG_GETARG_INT64(1), .fd.column_type = TypenameGetTypid(PG_GETARG_CSTRING(2)), }; DimensionSlice *slice = calculate_open_range_default(&dim, value); PG_RETURN_DATUM(create_range_datum(fcinfo, slice)); } static int64 calculate_closed_range_interval(const Dimension *dim) { Assert(NULL != dim && IS_CLOSED_DIMENSION(dim)); return DIMENSION_SLICE_CLOSED_MAX / ((int64) dim->fd.num_slices); } static DimensionSlice * calculate_closed_range_default(const Dimension *dim, int64 value) { int64 range_start, range_end; /* The interval that divides the dimension into N equal sized slices */ int64 interval = calculate_closed_range_interval(dim); int64 last_start = interval * (dim->fd.num_slices - 1); if (value < 0) elog(ERROR, "invalid value " INT64_FORMAT " for closed dimension", value); if (value >= last_start) { /* put overflow from integer-division errors in last range */ range_start = last_start; range_end = DIMENSION_SLICE_MAXVALUE; } else { range_start = (value / interval) * interval; range_end = range_start + interval; } if (0 == range_start) { range_start = DIMENSION_SLICE_MINVALUE; } return ts_dimension_slice_create(dim->fd.id, range_start, range_end); } TS_FUNCTION_INFO_V1(ts_dimension_calculate_closed_range_default); /* * Exposed closed dimension range calculation for testing purposes. */ Datum ts_dimension_calculate_closed_range_default(PG_FUNCTION_ARGS) { int64 value = PG_GETARG_INT64(0); Dimension dim = { .type = DIMENSION_TYPE_CLOSED, .fd.id = 0, .fd.num_slices = PG_GETARG_INT16(1), }; DimensionSlice *slice = calculate_closed_range_default(&dim, value); PG_RETURN_DATUM(create_range_datum(fcinfo, slice)); } DimensionSlice * ts_dimension_calculate_default_slice(const Dimension *dim, int64 value) { if (IS_OPEN_DIMENSION(dim)) return calculate_open_range_default(dim, value); return calculate_closed_range_default(dim, value); } /* * Get the ordinal value of a slice in an open dimension. * * Note that, for an open dimension, we can only deal with already created * slices and cannot account for, e.g., gaps in the dimension where future * slices might be created and thus changing the ordinal value for a slice. * * For instance, the ordinal value of slice D below is 2 (zero indexed): * * ' | A | B | <gap> | D | E | * * but when slice C is later created the ordinal value of D will be 3: * * ' | A | B | C | D | E | */ static int ts_dimension_get_open_slice_ordinal(const Dimension *dim, const DimensionSlice *slice) { DimensionVec *vec; int i; Assert(NULL != dim && IS_OPEN_DIMENSION(dim)); Assert(NULL != slice); vec = ts_dimension_get_slices(dim); Assert(NULL != vec); /* Find the index (ordinal) of the chunk's slice in the open dimension */ i = ts_dimension_vec_find_slice_index(vec, slice->fd.id); if (i >= 0) return i; /* * Returns the number of slices if the slice not found, i.e., i = -1. * Dimension slice might not exist if a chunk table is created without * modifying metadata. */ return vec->num_slices; } /* * Get the ordinal value of a slice in a closed dimension. * * For closed dimensions, we calculate the ordinal value of a slice based on * the assumption that the dimension is fully partitioned in equal size slices * as given by the current partitioning configuration. In reality, though, * slices are created lazily so a closed dimension might have less slices in * time interval than the configuration suggests. Further, during time * intervals where repartitioning happens, there might be an unexpected number * of slices due to a mix of slices from both the old and the new partitioning * configuration. As a result, the ordinal value of a given slice might not * actually match the partitioning settings at a given point in time. In this case, we will return * the ordinal of current slice most overlapping the given slice (or first fully overlapped slice). */ static int ts_dimension_get_closed_slice_ordinal(const Dimension *dim, const DimensionSlice *target_slice) { int64 current_slice_size; int64 target_slice_size; int candidate_slice_ordinal; int64 target_overlap_with_candidate_slice; Assert(NULL != dim && IS_CLOSED_DIMENSION(dim)); Assert(NULL != target_slice); Assert(dim->fd.num_slices > 0); /* Slicing assumes partitioning functions use the range [0, INT32_MAX], though the first slice * uses INT64_MIN as its lower bound, and the last slice uses INT64_MAX as its upper bound. */ if (target_slice->fd.range_start == DIMENSION_SLICE_MINVALUE) return 0; if (target_slice->fd.range_end == DIMENSION_SLICE_MAXVALUE) return dim->fd.num_slices - 1; Assert(target_slice->fd.range_start > 0); Assert(target_slice->fd.range_end < DIMENSION_SLICE_CLOSED_MAX); /* Given a target slice starting from some point p, determine a candidate slice in the current * partitioning configuration that contains p. If that slice contains over half of our target * slice, return it's ordinal. Otherwise return the ordinal for the next slice. */ current_slice_size = calculate_closed_range_interval(dim); target_slice_size = target_slice->fd.range_end - target_slice->fd.range_start; candidate_slice_ordinal = target_slice->fd.range_start / current_slice_size; target_overlap_with_candidate_slice = current_slice_size - (target_slice->fd.range_start % current_slice_size); /* Note that if the candidate slice wholly contains the target slice, * target_overlap_with_candidate_slice will actually be greater than target_slice_size. This * doesn't affect the correctness of the following check. */ if (target_overlap_with_candidate_slice >= target_slice_size / 2) return candidate_slice_ordinal; else return candidate_slice_ordinal + 1; } /* * Get the ordinal value of a slice in a dimension. * * This function returns the ordinal value of a slice (starting at 0) in the * dimension it belongs to. In other words, the "earliest" slice along the * dimensional axis gets the lowest ordinal value and the "latest" the largest. */ int ts_dimension_get_slice_ordinal(const Dimension *dim, const DimensionSlice *slice) { Assert(NULL != dim); Assert(NULL != slice); Assert(dim->fd.id == slice->fd.dimension_id); switch (dim->type) { case DIMENSION_TYPE_OPEN: return ts_dimension_get_open_slice_ordinal(dim, slice); case DIMENSION_TYPE_CLOSED: return ts_dimension_get_closed_slice_ordinal(dim, slice); default: Assert(false); break; } pg_unreachable(); return -1; } static Hyperspace * hyperspace_create(int32 hypertable_id, Oid main_table_relid, uint16 num_dimensions, MemoryContext mctx) { Hyperspace *hs = MemoryContextAllocZero(mctx, HYPERSPACE_SIZE(num_dimensions)); hs->hypertable_id = hypertable_id; hs->main_table_relid = main_table_relid; hs->capacity = num_dimensions; hs->num_dimensions = 0; return hs; } static ScanTupleResult dimension_tuple_found(TupleInfo *ti, void *data) { Hyperspace *hs = data; Dimension *d = &hs->dimensions[hs->num_dimensions++]; dimension_fill_in_from_tuple(d, ti, hs->main_table_relid); return SCAN_CONTINUE; } static int dimension_scan_internal(ScanKeyData *scankey, int nkeys, tuple_found_func tuple_found, void *data, int limit, int dimension_index, LOCKMODE lockmode, MemoryContext mctx) { Catalog *catalog = ts_catalog_get(); ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, DIMENSION), .index = catalog_get_index(catalog, DIMENSION, dimension_index), .nkeys = nkeys, .limit = limit, .scankey = scankey, .data = data, .tuple_found = tuple_found, .lockmode = lockmode, .scandirection = ForwardScanDirection, .result_mctx = mctx, }; return ts_scanner_scan(&scanctx); } Hyperspace * ts_dimension_scan(int32 hypertable_id, Oid main_table_relid, int16 num_dimensions, MemoryContext mctx) { Hyperspace *space = hyperspace_create(hypertable_id, main_table_relid, num_dimensions, mctx); ScanKeyData scankey[1]; /* Perform an index scan on hypertable_id. */ ScanKeyInit(&scankey[0], Anum_dimension_hypertable_id_column_name_idx_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); dimension_scan_internal(scankey, 1, dimension_tuple_found, space, num_dimensions, DIMENSION_HYPERTABLE_ID_COLUMN_NAME_IDX, AccessShareLock, mctx); /* Sort dimensions in ascending order to allow binary search lookups */ qsort(space->dimensions, space->num_dimensions, sizeof(Dimension), cmp_dimension_id); return space; } static ScanTupleResult dimension_find_hypertable_id_tuple_found(TupleInfo *ti, void *data) { int32 *hypertable_id = data; bool isnull = false; Datum datum = slot_getattr(ti->slot, Anum_dimension_hypertable_id, &isnull); Assert(!isnull); *hypertable_id = DatumGetInt32(datum); return SCAN_DONE; } int32 ts_dimension_get_hypertable_id(int32 dimension_id) { int32 hypertable_id; ScanKeyData scankey[1]; int ret; /* Perform an index scan dimension_id. */ ScanKeyInit(&scankey[0], Anum_dimension_id_idx_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(dimension_id)); ret = dimension_scan_internal(scankey, 1, dimension_find_hypertable_id_tuple_found, &hypertable_id, 1, DIMENSION_ID_IDX, AccessShareLock, CurrentMemoryContext); if (ret == 1) return hypertable_id; return -1; } DimensionVec * ts_dimension_get_slices(const Dimension *dim) { return ts_dimension_slice_scan_by_dimension(dim->fd.id, 0); } static int dimension_scan_update(int32 dimension_id, tuple_found_func tuple_found, void *data, LOCKMODE lockmode) { Catalog *catalog = ts_catalog_get(); ScanKeyData scankey[1]; ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, DIMENSION), .index = catalog_get_index(catalog, DIMENSION, DIMENSION_ID_IDX), .nkeys = 1, .limit = 1, .scankey = scankey, .data = data, .tuple_found = tuple_found, .lockmode = lockmode, .scandirection = ForwardScanDirection, }; ScanKeyInit(&scankey[0], Anum_dimension_id_idx_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(dimension_id)); return ts_scanner_scan(&scanctx); } static ScanTupleResult dimension_tuple_delete(TupleInfo *ti, void *data) { CatalogSecurityContext sec_ctx; bool isnull; Datum dimension_id = slot_getattr(ti->slot, Anum_dimension_id, &isnull); bool *delete_slices = data; Assert(!isnull); /* delete dimension slices */ if (NULL != delete_slices && *delete_slices) ts_dimension_slice_delete_by_dimension_id(DatumGetInt32(dimension_id), false); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); ts_catalog_restore_user(&sec_ctx); return SCAN_CONTINUE; } int ts_dimension_delete_by_hypertable_id(int32 hypertable_id, bool delete_slices) { ScanKeyData scankey[1]; /* Perform an index scan to delete based on hypertable_id */ ScanKeyInit(&scankey[0], Anum_dimension_hypertable_id_column_name_idx_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); return dimension_scan_internal(scankey, 1, dimension_tuple_delete, &delete_slices, 0, DIMENSION_HYPERTABLE_ID_COLUMN_NAME_IDX, RowExclusiveLock, CurrentMemoryContext); } static ScanTupleResult dimension_tuple_update(TupleInfo *ti, void *data) { Dimension *dim = data; Datum values[Natts_dimension]; bool nulls[Natts_dimension]; CatalogSecurityContext sec_ctx; bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); HeapTuple new_tuple; heap_deform_tuple(tuple, ts_scanner_get_tupledesc(ti), values, nulls); Assert((dim->fd.num_slices <= 0 && dim->fd.interval_length > 0) || (dim->fd.num_slices > 0 && dim->fd.interval_length <= 0)); values[AttrNumberGetAttrOffset(Anum_dimension_column_name)] = NameGetDatum(&dim->fd.column_name); values[AttrNumberGetAttrOffset(Anum_dimension_column_type)] = ObjectIdGetDatum(dim->fd.column_type); values[AttrNumberGetAttrOffset(Anum_dimension_num_slices)] = Int16GetDatum(dim->fd.num_slices); if (!nulls[AttrNumberGetAttrOffset(Anum_dimension_partitioning_func)] && !nulls[AttrNumberGetAttrOffset(Anum_dimension_partitioning_func_schema)]) { values[AttrNumberGetAttrOffset(Anum_dimension_partitioning_func)] = NameGetDatum(&dim->fd.partitioning_func); values[AttrNumberGetAttrOffset(Anum_dimension_partitioning_func_schema)] = NameGetDatum(&dim->fd.partitioning_func_schema); } if (*NameStr(dim->fd.integer_now_func) != '\0' && *NameStr(dim->fd.integer_now_func_schema) != '\0') { values[AttrNumberGetAttrOffset(Anum_dimension_integer_now_func)] = NameGetDatum(&dim->fd.integer_now_func); values[AttrNumberGetAttrOffset(Anum_dimension_integer_now_func_schema)] = NameGetDatum(&dim->fd.integer_now_func_schema); nulls[AttrNumberGetAttrOffset(Anum_dimension_integer_now_func)] = false; nulls[AttrNumberGetAttrOffset(Anum_dimension_integer_now_func_schema)] = false; } if (!nulls[AttrNumberGetAttrOffset(Anum_dimension_interval_length)]) values[AttrNumberGetAttrOffset(Anum_dimension_interval_length)] = Int64GetDatum(dim->fd.interval_length); if (dim->fd.compress_interval_length > 0) { values[AttrNumberGetAttrOffset(Anum_dimension_compress_interval_length)] = Int64GetDatum(dim->fd.compress_interval_length); nulls[AttrNumberGetAttrOffset(Anum_dimension_compress_interval_length)] = false; } else { nulls[AttrNumberGetAttrOffset(Anum_dimension_compress_interval_length)] = true; } new_tuple = heap_form_tuple(ts_scanner_get_tupledesc(ti), values, nulls); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_update_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti), new_tuple); ts_catalog_restore_user(&sec_ctx); heap_freetuple(new_tuple); if (should_free) heap_freetuple(tuple); return SCAN_DONE; } static int32 dimension_insert_relation(Relation rel, int32 hypertable_id, Name colname, Oid coltype, int16 num_slices, regproc partitioning_func, int64 interval_length) { TupleDesc desc = RelationGetDescr(rel); Datum values[Natts_dimension]; bool nulls[Natts_dimension] = { false }; CatalogSecurityContext sec_ctx; int32 dimension_id; values[AttrNumberGetAttrOffset(Anum_dimension_hypertable_id)] = Int32GetDatum(hypertable_id); values[AttrNumberGetAttrOffset(Anum_dimension_column_name)] = NameGetDatum(colname); values[AttrNumberGetAttrOffset(Anum_dimension_column_type)] = ObjectIdGetDatum(coltype); if (OidIsValid(partitioning_func)) { Oid pronamespace = get_func_namespace(partitioning_func); values[AttrNumberGetAttrOffset(Anum_dimension_partitioning_func)] = DirectFunctionCall1(namein, CStringGetDatum(get_func_name(partitioning_func))); values[AttrNumberGetAttrOffset(Anum_dimension_partitioning_func_schema)] = DirectFunctionCall1(namein, CStringGetDatum(get_namespace_name(pronamespace))); } else { nulls[AttrNumberGetAttrOffset(Anum_dimension_partitioning_func)] = true; nulls[AttrNumberGetAttrOffset(Anum_dimension_partitioning_func_schema)] = true; } if (num_slices > 0) { /* Closed (hash) dimension */ Assert(num_slices > 0 && interval_length <= 0); values[AttrNumberGetAttrOffset(Anum_dimension_num_slices)] = Int16GetDatum(num_slices); values[AttrNumberGetAttrOffset(Anum_dimension_aligned)] = BoolGetDatum(false); nulls[AttrNumberGetAttrOffset(Anum_dimension_interval_length)] = true; } else { /* Open (time) dimension */ Assert(num_slices <= 0 && interval_length > 0); values[AttrNumberGetAttrOffset(Anum_dimension_interval_length)] = Int64GetDatum(interval_length); values[AttrNumberGetAttrOffset(Anum_dimension_aligned)] = BoolGetDatum(true); nulls[AttrNumberGetAttrOffset(Anum_dimension_num_slices)] = true; } /* no integer_now function by default */ nulls[AttrNumberGetAttrOffset(Anum_dimension_integer_now_func_schema)] = true; nulls[AttrNumberGetAttrOffset(Anum_dimension_integer_now_func)] = true; /* no compress interval length by default */ nulls[AttrNumberGetAttrOffset(Anum_dimension_compress_interval_length)] = true; ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); dimension_id = Int32GetDatum(ts_catalog_table_next_seq_id(ts_catalog_get(), DIMENSION)); values[AttrNumberGetAttrOffset(Anum_dimension_id)] = dimension_id; ts_catalog_insert_values(rel, desc, values, nulls); ts_catalog_restore_user(&sec_ctx); return dimension_id; } static int32 dimension_insert(int32 hypertable_id, Name colname, Oid coltype, int16 num_slices, regproc partitioning_func, int64 interval_length) { Catalog *catalog = ts_catalog_get(); Relation rel; int32 dimension_id; rel = table_open(catalog_get_table_id(catalog, DIMENSION), RowExclusiveLock); dimension_id = dimension_insert_relation(rel, hypertable_id, colname, coltype, num_slices, partitioning_func, interval_length); table_close(rel, RowExclusiveLock); return dimension_id; } int ts_dimension_set_type(Dimension *dim, Oid newtype) { if (!IS_VALID_OPEN_DIM_TYPE(newtype)) ereport(ERROR, (errcode(ERRCODE_INVALID_TABLE_DEFINITION), errmsg("cannot change data type of hypertable column \"%s\" from %s to %s", NameStr(dim->fd.column_name), format_type_be(dim->fd.column_type), format_type_be(newtype)), errhint("Use an integer, timestamp, or date type."))); dim->fd.column_type = newtype; return dimension_scan_update(dim->fd.id, dimension_tuple_update, dim, RowExclusiveLock); } TSDLLEXPORT Oid ts_dimension_get_partition_type(const Dimension *dim) { Assert(dim != NULL); return dim->partitioning != NULL ? dim->partitioning->partfunc.rettype : dim->fd.column_type; } int ts_dimension_set_name(Dimension *dim, const char *newname) { namestrcpy(&dim->fd.column_name, newname); return dimension_scan_update(dim->fd.id, dimension_tuple_update, dim, RowExclusiveLock); } int ts_dimension_set_chunk_interval(Dimension *dim, int64 chunk_interval) { Assert(IS_OPEN_DIMENSION(dim)); dim->fd.interval_length = chunk_interval; return dimension_scan_update(dim->fd.id, dimension_tuple_update, dim, RowExclusiveLock); } int ts_dimension_set_compress_interval(Dimension *dim, int64 compress_interval) { if (!IS_OPEN_DIMENSION(dim)) ereport(ERROR, (errmsg("trying to set compress interval on closed dimension"), errhint("dimension ID %d", dim->fd.id))); dim->fd.compress_interval_length = compress_interval; return dimension_scan_update(dim->fd.id, dimension_tuple_update, dim, RowExclusiveLock); } /* * Apply any dimension-specific transformations on a value, i.e., apply * partitioning function. Optionally get the type of the resulting value via * the restype parameter. */ Datum ts_dimension_transform_value(const Dimension *dim, Oid collation, Datum value, Oid const_datum_type, Oid *restype) { if (NULL != dim->partitioning) value = ts_partitioning_func_apply(dim->partitioning, collation, value); if (NULL != restype) { if (NULL != dim->partitioning) *restype = dim->partitioning->partfunc.rettype; else if (OidIsValid(const_datum_type)) *restype = const_datum_type; else *restype = dim->fd.column_type; } return value; } Point * ts_point_create(int16 num_dimensions) { Point *p = palloc0(POINT_SIZE(num_dimensions)); p->cardinality = num_dimensions; p->num_coords = 0; return p; } TSDLLEXPORT Point * ts_hyperspace_calculate_point(const Hyperspace *hs, TupleTableSlot *slot) { Point *p = ts_point_create(hs->num_dimensions); int i; for (i = 0; i < hs->num_dimensions; i++) { const Dimension *d = &hs->dimensions[i]; Datum datum; bool isnull; Oid dimtype; if (NULL != d->partitioning) datum = ts_partitioning_func_apply_slot(d->partitioning, slot, &isnull); else datum = slot_getattr(slot, d->column_attno, &isnull); switch (d->type) { case DIMENSION_TYPE_OPEN: dimtype = ts_dimension_get_partition_type(d); if (isnull) ereport(ERROR, (errcode(ERRCODE_NOT_NULL_VIOLATION), errmsg("NULL value in column \"%s\" violates not-null constraint", NameStr(d->fd.column_name)), errhint("Columns used for time partitioning cannot be NULL."))); p->coordinates[p->num_coords++] = ts_time_value_to_internal(datum, dimtype); break; case DIMENSION_TYPE_CLOSED: p->coordinates[p->num_coords++] = (int64) DatumGetInt32(datum); break; case DIMENSION_TYPE_STATS: case DIMENSION_TYPE_ANY: elog(ERROR, "invalid dimension type when inserting tuple"); break; } } return p; } #define INT_TYPE_MAX(type) \ (int64)(((type) == INT2OID) ? PG_INT16_MAX : \ (((type) == INT4OID) ? PG_INT32_MAX : PG_INT64_MAX)) #define IS_VALID_NUM_SLICES(num_slices) ((num_slices) >= 1 && (num_slices) <= PG_INT16_MAX) static int64 get_validated_integer_interval(Oid dimtype, int64 value) { if (value < 1 || value > INT_TYPE_MAX(dimtype)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid interval: must be between 1 and " INT64_FORMAT, INT_TYPE_MAX(dimtype)))); if (IS_TIMESTAMP_TYPE(dimtype) && value < USECS_PER_SEC) ereport(WARNING, (errcode(ERRCODE_AMBIGUOUS_PARAMETER), errmsg("unexpected interval: smaller than one second"), errhint("The interval is specified in microseconds."))); return value; } /* * The default chunk interval to use by hypertables. This value is set via the * corresponding GUC in the assign hook, so do not assign a value here. */ extern Interval *default_chunk_time_interval; /* * Get the default chunk interval based on dimension type. */ static ChunkInterval get_default_interval(Oid dimtype, bool adaptive_chunking) { ChunkInterval chunk_interval = { .type = InvalidOid, }; switch (dimtype) { case INT2OID: chunk_interval.type = INT2OID; chunk_interval.integer_interval = DEFAULT_SMALLINT_INTERVAL; break; case INT4OID: chunk_interval.type = INT4OID; chunk_interval.integer_interval = DEFAULT_INT_INTERVAL; break; case INT8OID: chunk_interval.type = INT8OID; chunk_interval.integer_interval = DEFAULT_BIGINT_INTERVAL; break; case TIMESTAMPOID: case TIMESTAMPTZOID: case DATEOID: case UUIDOID: if (default_chunk_time_interval != NULL) { chunk_interval.type = INTERVALOID; chunk_interval.interval = *default_chunk_time_interval; } else { chunk_interval.type = INT8OID; if (adaptive_chunking) chunk_interval.integer_interval = DEFAULT_CHUNK_TIME_INTERVAL_ADAPTIVE; else chunk_interval.integer_interval = DEFAULT_CHUNK_TIME_INTERVAL; } break; default: ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot get default interval for %s dimension", format_type_be(dimtype)), errhint("Use a valid dimension type."))); } return chunk_interval; } static int64 dimension_interval_to_internal(const char *colname, Oid dimtype, const ChunkInterval *chunk_interval, bool adaptive_chunking) { int64 interval; Assert(chunk_interval != NULL); if (!IS_VALID_OPEN_DIM_TYPE(dimtype)) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("invalid type for dimension \"%s\"", colname), errhint("Use an integer, timestamp, or date type."))); switch (chunk_interval->type) { case INT2OID: interval = get_validated_integer_interval(dimtype, chunk_interval->integer_interval); break; case INT4OID: interval = get_validated_integer_interval(dimtype, chunk_interval->integer_interval); break; case INT8OID: interval = get_validated_integer_interval(dimtype, chunk_interval->integer_interval); break; case INTERVALOID: if (!IS_TIMESTAMP_TYPE(dimtype) && !IS_UUID_TYPE(dimtype)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid interval type for %s dimension", format_type_be(dimtype)), errhint("Use an interval of type integer."))); interval = interval_to_usec(&chunk_interval->interval); break; default: ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid interval type for %s dimension", format_type_be(dimtype)), IS_TIMESTAMP_TYPE(dimtype) ? errhint("Use an interval of type integer or interval.") : errhint("Use an interval of type integer."))); } if (dimtype == DATEOID && (interval <= 0 || interval % USECS_PER_DAY != 0)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid interval for %s dimension", format_type_be(dimtype)), errhint("Use an interval that is a multiple of one day."))); return interval; } TS_FUNCTION_INFO_V1(ts_dimension_interval_to_internal_test); /* * Exposed for testing purposes. */ Datum ts_dimension_interval_to_internal_test(PG_FUNCTION_ARGS) { Oid dimtype = PG_GETARG_OID(0); Oid argtype = PG_ARGISNULL(1) ? InvalidOid : get_fn_expr_argtype(fcinfo->flinfo, 1); ChunkInterval chunk_interval; if (!OidIsValid(argtype)) chunk_interval = get_default_interval(dimtype, false); else chunk_interval_set(&chunk_interval, PG_GETARG_DATUM(1), argtype); PG_RETURN_INT64(dimension_interval_to_internal("testcol", dimtype, &chunk_interval, false)); } static void dimension_add_not_null_on_column(Oid table_relid, char *colname) { AlterTableCmd cmd = { .type = T_AlterTableCmd, .subtype = AT_SetNotNull, .name = colname, .missing_ok = false, }; ereport(DEBUG1, (errmsg("adding not-null constraint to column \"%s\"", colname), errdetail("Dimensions cannot have NULL values."))); ts_alter_table_with_event_trigger(table_relid, (Node *) &cmd, list_make1(&cmd), false); } void ts_dimension_update(const Hypertable *ht, const NameData *dimname, DimensionType dimtype, Datum *interval, Oid *intervaltype, int16 *num_slices, Oid *integer_now_func) { Dimension *dim; if (NULL == ht) ereport(ERROR, (errcode(ERRCODE_TS_HYPERTABLE_NOT_EXIST), errmsg("invalid hypertable"))); if (dimtype == DIMENSION_TYPE_ANY) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid dimension type"))); if (NULL == dimname) { if (hyperspace_get_num_dimensions_by_type(ht->space, dimtype) > 1) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("hypertable \"%s\" has multiple %s dimensions", get_rel_name(ht->main_table_relid), dimtype == DIMENSION_TYPE_OPEN ? "time" : "space"), errhint("An explicit dimension name must be specified."))); dim = ts_hyperspace_get_mutable_dimension(ht->space, dimtype, 0); } else dim = ts_hyperspace_get_mutable_dimension_by_name(ht->space, dimtype, NameStr(*dimname)); if (NULL == dim) ereport(ERROR, (errcode(ERRCODE_TS_DIMENSION_NOT_EXIST), errmsg("hypertable \"%s\" does not have a matching dimension", get_rel_name(ht->main_table_relid)))); Assert(dim->type == dimtype); if (interval) { Oid dimtype = ts_dimension_get_partition_type(dim); ChunkInterval chunk_interval; chunk_interval_set(&chunk_interval, *interval, *intervaltype); dim->fd.interval_length = dimension_interval_to_internal(NameStr(dim->fd.column_name), dimtype, &chunk_interval, hypertable_adaptive_chunking_enabled(ht)); } if (num_slices) { Assert(IS_CLOSED_DIMENSION(dim)); dim->fd.num_slices = *num_slices; } if (integer_now_func) { Oid pronamespace = get_func_namespace(*integer_now_func); namestrcpy(&dim->fd.integer_now_func_schema, get_namespace_name(pronamespace)); namestrcpy(&dim->fd.integer_now_func, get_func_name(*integer_now_func)); } dimension_scan_update(dim->fd.id, dimension_tuple_update, dim, RowExclusiveLock); } TS_FUNCTION_INFO_V1(ts_dimension_set_num_slices); Datum ts_dimension_set_num_slices(PG_FUNCTION_ARGS) { Oid table_relid = PG_GETARG_OID(0); int32 num_slices_arg = PG_ARGISNULL(1) ? -1 : PG_GETARG_INT32(1); Name colname = PG_ARGISNULL(2) ? NULL : PG_GETARG_NAME(2); Cache *hcache = ts_hypertable_cache_pin(); int16 num_slices; Hypertable *ht; TS_PREVENT_FUNC_IF_READ_ONLY(); if (PG_ARGISNULL(0)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("hypertable cannot be NULL"))); /* Verify that we're dealing with a hypertable or fail */ ht = ts_hypertable_cache_get_entry(hcache, table_relid, CACHE_FLAG_NONE); ts_hypertable_permissions_check(table_relid, GetUserId()); if (PG_ARGISNULL(1) || !IS_VALID_NUM_SLICES(num_slices_arg)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid number of partitions: must be between 1 and %d", PG_INT16_MAX))); /* * Our catalog stores num_slices as a smallint (int16). However, function * argument is an integer (int32) so that the user need not cast it to a * smallint. We therefore convert to int16 here after checking that * num_slices cannot be > INT16_MAX. */ num_slices = num_slices_arg & 0xffff; ts_dimension_update(ht, colname, DIMENSION_TYPE_CLOSED, NULL, NULL, &num_slices, NULL); ts_cache_release(&hcache); PG_RETURN_VOID(); } TS_FUNCTION_INFO_V1(ts_dimension_set_interval); /* * Update chunk_time_interval for a hypertable. * * hypertable - The OID of the table corresponding to a hypertable whose time * interval should be updated * chunk_time_interval - The new time interval. For hypertables with integral * time columns, this must be an integral type. For hypertables with a * TIMESTAMP/TIMESTAMPTZ/DATE type, it can be integral which is treated as * microseconds, or an INTERVAL type. * dimension_name - The name of the dimension */ Datum ts_dimension_set_interval(PG_FUNCTION_ARGS) { Oid table_relid = PG_GETARG_OID(0); Datum interval = PG_GETARG_DATUM(1); Oid intervaltype = InvalidOid; Name colname = PG_ARGISNULL(2) ? NULL : PG_GETARG_NAME(2); Cache *hcache = ts_hypertable_cache_pin(); Hypertable *ht; TS_PREVENT_FUNC_IF_READ_ONLY(); if (PG_ARGISNULL(0)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("hypertable cannot be NULL"))); ht = ts_resolve_hypertable_from_table_or_cagg(hcache, table_relid, true); ts_hypertable_permissions_check(table_relid, GetUserId()); if (PG_ARGISNULL(1)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid interval: an explicit interval must be specified"))); intervaltype = get_fn_expr_argtype(fcinfo->flinfo, 1); ts_dimension_update(ht, colname, DIMENSION_TYPE_OPEN, &interval, &intervaltype, NULL, NULL); ts_cache_release(&hcache); PG_RETURN_VOID(); } DimensionInfo * ts_dimension_info_create_open(Oid table_relid, Name column_name, Datum interval, Oid interval_type, regproc partitioning_func) { DimensionInfo *info = palloc(sizeof(*info)); *info = (DimensionInfo){ .type = DIMENSION_TYPE_OPEN, .table_relid = table_relid, .partitioning_func = partitioning_func, }; chunk_interval_set(&info->chunk_interval, interval, interval_type); namestrcpy(&info->colname, NameStr(*column_name)); return info; } DimensionInfo * ts_dimension_info_create_closed(Oid table_relid, Name column_name, int32 num_slices, regproc partitioning_func) { DimensionInfo *info = palloc(sizeof(*info)); *info = (DimensionInfo){ .type = DIMENSION_TYPE_CLOSED, .table_relid = table_relid, .num_slices = num_slices, .num_slices_is_set = (num_slices > 0), .partitioning_func = partitioning_func, }; namestrcpy(&info->colname, NameStr(*column_name)); return info; } /* Validate the configuration of an open ("time") dimension */ static void dimension_info_validate_open(DimensionInfo *info) { Oid dimtype = info->coltype; Assert(info->type == DIMENSION_TYPE_OPEN); if (OidIsValid(info->partitioning_func)) { if (!ts_partitioning_func_is_valid(info->partitioning_func, info->type, info->coltype)) ereport(ERROR, (errcode(ERRCODE_INVALID_FUNCTION_DEFINITION), errmsg("invalid partitioning function"), errhint("A valid partitioning function for open (time) dimensions must be " "IMMUTABLE, " "take the column type as input, and return an integer or " "timestamp type."))); dimtype = get_func_rettype(info->partitioning_func); } /* * Validate the dimension type before trying to get the default interval. * This ensures we give a clear "invalid type" error rather than a confusing * "cannot get default interval" error for unsupported types. */ if (!IS_VALID_OPEN_DIM_TYPE(dimtype)) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("invalid type for dimension \"%s\"", NameStr(info->colname)), errhint("Use an integer, timestamp, or date type."))); if (!OidIsValid(info->chunk_interval.type)) info->chunk_interval = get_default_interval(dimtype, info->adaptive_chunking); info->interval = dimension_interval_to_internal(NameStr(info->colname), dimtype, &info->chunk_interval, info->adaptive_chunking); } /* Validate the configuration of a closed ("space") dimension */ static void dimension_info_validate_closed(DimensionInfo *info) { Assert(info->type == DIMENSION_TYPE_CLOSED); if (!OidIsValid(info->partitioning_func)) info->partitioning_func = ts_partitioning_func_get_closed_default(); else if (!ts_partitioning_func_is_valid(info->partitioning_func, info->type, info->coltype)) ereport(ERROR, (errcode(ERRCODE_INVALID_FUNCTION_DEFINITION), errmsg("invalid partitioning function"), errhint("A valid partitioning function for closed (space) dimensions must be " "IMMUTABLE " "and have the signature (anyelement) -> integer."))); if (!info->num_slices_is_set || !IS_VALID_NUM_SLICES(info->num_slices)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid number of partitions for dimension \"%s\"", NameStr(info->colname)), errhint("A closed (space) dimension must specify between 1 and %d partitions.", PG_INT16_MAX))); } void ts_dimension_info_validate(DimensionInfo *info) { const Dimension *dim; HeapTuple tuple; Datum datum; bool isnull = false; bool isgenerated; if (!DIMENSION_INFO_IS_SET(info)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid dimension info"))); if (info->num_slices_is_set && OidIsValid(info->chunk_interval.type)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot specify both the number of partitions and an interval"))); /* Check that the column exists and get its NOT NULL status */ tuple = SearchSysCacheAttName(info->table_relid, NameStr(info->colname)); if (!HeapTupleIsValid(tuple)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("column \"%s\" does not exist", NameStr(info->colname)))); datum = SysCacheGetAttr(ATTNAME, tuple, Anum_pg_attribute_atttypid, &isnull); Assert(!isnull); info->coltype = DatumGetObjectId(datum); datum = SysCacheGetAttr(ATTNAME, tuple, Anum_pg_attribute_attnotnull, &isnull); Assert(!isnull); info->set_not_null = !DatumGetBool(datum); /* check that the column is not generated */ datum = SysCacheGetAttr(ATTNAME, tuple, Anum_pg_attribute_attgenerated, &isnull); Assert(!isnull); isgenerated = (DatumGetChar(datum) == ATTRIBUTE_GENERATED_STORED); if (isgenerated) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("invalid partitioning column"), errhint("Generated columns cannot be used as partitioning dimensions."))); ReleaseSysCache(tuple); if (NULL != info->ht) { /* Check if the dimension already exists */ dim = ts_hyperspace_get_dimension_by_name(info->ht->space, DIMENSION_TYPE_ANY, NameStr(info->colname)); if (NULL != dim) { if (!info->if_not_exists) ereport(ERROR, (errcode(ERRCODE_TS_DUPLICATE_DIMENSION), errmsg("column \"%s\" is already a dimension", NameStr(info->colname)))); info->dimension_id = dim->fd.id; info->skip = true; ereport(NOTICE, (errmsg("column \"%s\" is already a dimension, skipping", NameStr(info->colname)))); return; } } switch (info->type) { case DIMENSION_TYPE_CLOSED: dimension_info_validate_closed(info); break; case DIMENSION_TYPE_OPEN: dimension_info_validate_open(info); break; case DIMENSION_TYPE_STATS: case DIMENSION_TYPE_ANY: elog(ERROR, "invalid dimension type in configuration"); break; } } int32 ts_dimension_add_from_info(DimensionInfo *info) { if (info->set_not_null && info->type == DIMENSION_TYPE_OPEN) dimension_add_not_null_on_column(info->table_relid, NameStr(info->colname)); Assert(info->ht != NULL); info->dimension_id = dimension_insert(info->ht->fd.id, &info->colname, info->coltype, info->num_slices, info->partitioning_func, info->interval); return info->dimension_id; } /* * Create a datum to be returned by add_dimension DDL function */ static Datum dimension_create_datum(FunctionCallInfo fcinfo, DimensionInfo *info, bool is_generic) { TupleDesc tupdesc; HeapTuple tuple; if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in " "context that cannot accept type record"))); tupdesc = BlessTupleDesc(tupdesc); if (is_generic) { Datum values[Natts_generic_add_dimension]; bool nulls[Natts_generic_add_dimension] = { false }; Assert(tupdesc->natts == Natts_generic_add_dimension); values[AttrNumberGetAttrOffset(Anum_generic_add_dimension_id)] = info->dimension_id; values[AttrNumberGetAttrOffset(Anum_generic_add_dimension_created)] = BoolGetDatum(!info->skip); tuple = heap_form_tuple(tupdesc, values, nulls); } else { Datum values[Natts_add_dimension]; bool nulls[Natts_add_dimension] = { false }; Assert(tupdesc->natts == Natts_add_dimension); values[AttrNumberGetAttrOffset(Anum_add_dimension_id)] = info->dimension_id; values[AttrNumberGetAttrOffset(Anum_add_dimension_schema_name)] = NameGetDatum(&info->ht->fd.schema_name); values[AttrNumberGetAttrOffset(Anum_add_dimension_table_name)] = NameGetDatum(&info->ht->fd.table_name); values[AttrNumberGetAttrOffset(Anum_add_dimension_column_name)] = NameGetDatum(&info->colname); values[AttrNumberGetAttrOffset(Anum_add_dimension_created)] = BoolGetDatum(!info->skip); tuple = heap_form_tuple(tupdesc, values, nulls); } return HeapTupleGetDatum(tuple); } /* * Add a new dimension to a hypertable. * * Arguments: * 0. Relation ID of table * 1. Column name * 2. Number of partitions / slices in close ('space') dimensions * 3. Interval for open ('time') dimensions * 4. Partitioning function * 5. IF NOT EXISTS option (bool) */ static Datum ts_dimension_add_internal(FunctionCallInfo fcinfo, DimensionInfo *info, bool is_generic) { Cache *hcache; Datum retval = 0; Assert(DIMENSION_INFO_IS_SET(info)); if (!DIMENSION_INFO_IS_VALID(info)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("must specify either the number of partitions or an interval"))); ts_hypertable_permissions_check(info->table_relid, GetUserId()); /* * The hypertable catalog table has a CHECK(num_dimensions > 0), which * means, that when this function is called from create_hypertable() * instead of directly, num_dimension is already set to one. We therefore * need to lock the hypertable tuple here so that we can set the correct * number of dimensions once we've added the new dimension. * * This lock is also used to serialize access from concurrent add_dimension() * call and a chunk creation. */ LockRelationOid(info->table_relid, ShareUpdateExclusiveLock); DEBUG_WAITPOINT("add_dimension_ht_lock"); info->ht = ts_hypertable_cache_get_cache_and_entry(info->table_relid, CACHE_FLAG_NONE, &hcache); if (info->num_slices_is_set && OidIsValid(info->chunk_interval.type)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot specify both the number of partitions and an interval"))); if (!info->num_slices_is_set && !OidIsValid(info->chunk_interval.type)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot omit both the number of partitions and the interval"))); ts_dimension_info_validate(info); if (!info->skip) { int32 dimension_id; /* * Note that space->num_dimensions reflects the actual number of * dimension rows and not the num_dimensions in the hypertable catalog * table. */ ts_hypertable_set_num_dimensions(info->ht, info->ht->space->num_dimensions + 1); dimension_id = ts_dimension_add_from_info(info); /* Verify that existing indexes are compatible with a hypertable */ /* * Need to get a fresh copy of hypertable from the database as cache * does not reflect the changes in the previous 2 lines which add a * new dimension */ info->ht = ts_hypertable_get_by_id(info->ht->fd.id); ts_indexing_verify_indexes(info->ht); /* * If the hypertable has chunks, to make it compatible * we add artificial dimension slice which will cover -inf / inf * range. * * Newly created chunks will have a proper slice range according to * the created dimension and its partitioning. */ if (ts_hypertable_has_chunks(info->table_relid, AccessShareLock)) { ListCell *lc; List *chunk_id_list = ts_chunk_get_chunk_ids_by_hypertable_id(info->ht->fd.id); DimensionSlice *slice; slice = ts_dimension_slice_create(dimension_id, DIMENSION_SLICE_MINVALUE, DIMENSION_SLICE_MAXVALUE); ts_dimension_slice_insert_multi(&slice, 1); foreach (lc, chunk_id_list) { int32 chunk_id = lfirst_int(lc); Chunk *chunk = ts_chunk_get_by_id(chunk_id, true); ChunkConstraint *cc = ts_chunk_constraints_add(chunk->constraints, chunk->fd.id, slice->fd.id, NULL, NULL); ts_chunk_constraint_insert(cc); } } } retval = dimension_create_datum(fcinfo, info, is_generic); ts_cache_release(&hcache); PG_RETURN_DATUM(retval); } TS_FUNCTION_INFO_V1(ts_dimension_add); TS_FUNCTION_INFO_V1(ts_dimension_add_general); Datum ts_dimension_add(PG_FUNCTION_ARGS) { Oid interval_type = PG_ARGISNULL(3) ? InvalidOid : get_fn_expr_argtype(fcinfo->flinfo, 3); DimensionInfo info = { .type = PG_ARGISNULL(2) ? DIMENSION_TYPE_OPEN : DIMENSION_TYPE_CLOSED, .table_relid = PG_GETARG_OID(0), .num_slices = PG_ARGISNULL(2) ? DatumGetInt32(-1) : PG_GETARG_INT32(2), .num_slices_is_set = !PG_ARGISNULL(2), .chunk_interval.type = interval_type, .partitioning_func = PG_ARGISNULL(4) ? InvalidOid : PG_GETARG_OID(4), .if_not_exists = PG_ARGISNULL(5) ? false : PG_GETARG_BOOL(5), }; TS_PREVENT_FUNC_IF_READ_ONLY(); if (!PG_ARGISNULL(1)) namestrcpy(&info.colname, NameStr(*PG_GETARG_NAME(1))); if (!PG_ARGISNULL(3)) chunk_interval_set(&info.chunk_interval, PG_GETARG_DATUM(3), interval_type); if (PG_ARGISNULL(0)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("hypertable cannot be NULL"))); return ts_dimension_add_internal(fcinfo, &info, false); } TSDLLEXPORT Datum ts_dimension_info_in(PG_FUNCTION_ARGS) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot construct type \"dimension_info\" from string"), errdetail("Type dimension_info cannot be constructed from string. You need to " "use constructor function."), errhint("Use \"by_range\" or \"by_hash\" to construct dimension types."))); PG_RETURN_VOID(); /* keep compiler quiet */ } /* * Get the interval value as a Datum from a ChunkInterval. * On 32-bit platforms, int64 is pass-by-reference so we need Int64GetDatumFast. */ static Datum chunk_interval_get_datum(const ChunkInterval *ci) { switch (ci->type) { case INT2OID: return Int16GetDatum((int16) ci->integer_interval); case INT4OID: return Int32GetDatum((int32) ci->integer_interval); case INT8OID: /* int64 is pass-by-ref on 32-bit, pass-by-val on 64-bit */ return Int64GetDatumFast(ci->integer_interval); case INTERVALOID: /* Interval is always pass-by-ref */ return IntervalPGetDatum(&((ChunkInterval *) ci)->interval); default: ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unsupported chunk interval type %d", ci->type))); return UnassignedDatum; /* keep compiler quiet */ } } TSDLLEXPORT Datum ts_dimension_info_out(PG_FUNCTION_ARGS) { DimensionInfo *info = (DimensionInfo *) PG_GETARG_POINTER(0); StringInfoData str; const char *partfuncname = OidIsValid(info->partitioning_func) ? get_func_name(info->partitioning_func) : "-"; initStringInfo(&str); switch (info->type) { case DIMENSION_TYPE_CLOSED: appendStringInfo(&str, "hash//%s//%d//%s", NameStr(info->colname), info->num_slices, partfuncname); break; case DIMENSION_TYPE_OPEN: { const char *argvalstr = "-"; if (OidIsValid(info->chunk_interval.type)) { argvalstr = ts_datum_to_string(chunk_interval_get_datum(&info->chunk_interval), info->chunk_interval.type); } appendStringInfo(&str, "range//%s//%s//%s", NameStr(info->colname), argvalstr, partfuncname); break; } case DIMENSION_TYPE_STATS: appendStringInfo(&str, "range"); break; case DIMENSION_TYPE_ANY: appendStringInfo(&str, "any"); break; } PG_RETURN_CSTRING(str.data); } static DimensionInfo * make_dimension_info(Name colname, DimensionType dimtype) { size_t size = sizeof(DimensionInfo); DimensionInfo *info = palloc0(size); SET_VARSIZE(info, size); info->type = dimtype; namestrcpy(&info->colname, NameStr(*colname)); return info; } /* * DimensionInfo for a hash dimension. * * This structure is only partially filled in when constructed. The rest will * be filled in by ts_dimension_add_general. */ Datum ts_hash_dimension(PG_FUNCTION_ARGS) { Ensure(PG_NARGS() > 2, "expected at most 3 arguments, invoked with %d arguments", PG_NARGS()); Name column_name; GETARG_NOTNULL_NULLABLE(column_name, 0, "column_name", NAME); DimensionInfo *info = make_dimension_info(column_name, DIMENSION_TYPE_CLOSED); info->num_slices = PG_ARGISNULL(1) ? DatumGetInt32(-1) : PG_GETARG_INT32(1); info->num_slices_is_set = !PG_ARGISNULL(1); info->partitioning_func = PG_ARGISNULL(2) ? InvalidOid : PG_GETARG_OID(2); PG_RETURN_POINTER(info); } /* * DimensionInfo for a hash dimension. * * This structure is only partially filled in when constructed. The rest will * be filled in by ts_dimension_add_general. */ Datum ts_range_dimension(PG_FUNCTION_ARGS) { Ensure(PG_NARGS() > 2, "expected at most 3 arguments, invoked with %d arguments", PG_NARGS()); Name column_name; GETARG_NOTNULL_NULLABLE(column_name, 0, "column_name", NAME); DimensionInfo *info = make_dimension_info(column_name, DIMENSION_TYPE_OPEN); Datum interval_datum = PG_ARGISNULL(1) ? Int32GetDatum(-1) : PG_GETARG_DATUM(1); Oid interval_type = PG_ARGISNULL(1) ? InvalidOid : get_fn_expr_argtype(fcinfo->flinfo, 1); info->partitioning_func = PG_ARGISNULL(2) ? InvalidOid : PG_GETARG_OID(2); chunk_interval_set(&info->chunk_interval, interval_datum, interval_type); PG_RETURN_POINTER(info); } Datum ts_dimension_add_general(PG_FUNCTION_ARGS) { DimensionInfo *info = NULL; GETARG_NOTNULL_POINTER(info, 1, "dimension", DimensionInfo); info->table_relid = PG_GETARG_OID(0); if (PG_GETARG_BOOL(2)) info->if_not_exists = true; return ts_dimension_add_internal(fcinfo, info, true); } /* Used as a tuple found function */ static ScanTupleResult dimension_rename_schema_name(TupleInfo *ti, void *data) { /* Dimension table may contain null valued columns that is why we do not use * FormData_dimension *dimension = (FormData_dimension *) GETSTRUCT(tuple); * pattern here */ Datum values[Natts_dimension]; bool nulls[Natts_dimension]; bool doReplace[Natts_dimension] = { false }; bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); HeapTuple new_tuple; /* contains [old_name,new_name] in that order */ char **names = (char **) data; Name schemaname; heap_deform_tuple(tuple, ts_scanner_get_tupledesc(ti), values, nulls); Assert(!nulls[AttrNumberGetAttrOffset(Anum_dimension_partitioning_func_schema)] || !nulls[AttrNumberGetAttrOffset(Anum_dimension_integer_now_func_schema)]); /* Rename schema names */ if (!nulls[AttrNumberGetAttrOffset(Anum_dimension_partitioning_func_schema)]) { schemaname = DatumGetName(values[AttrNumberGetAttrOffset(Anum_dimension_partitioning_func_schema)]); if (namestrcmp(schemaname, names[0]) == 0) { namestrcpy(schemaname, (const char *) names[1]); values[AttrNumberGetAttrOffset(Anum_dimension_partitioning_func_schema)] = NameGetDatum(schemaname); doReplace[AttrNumberGetAttrOffset(Anum_dimension_partitioning_func_schema)] = true; } } if (!nulls[AttrNumberGetAttrOffset(Anum_dimension_integer_now_func_schema)]) { schemaname = DatumGetName(values[AttrNumberGetAttrOffset(Anum_dimension_integer_now_func_schema)]); if (namestrcmp(schemaname, names[0]) == 0) { namestrcpy(schemaname, (const char *) names[1]); values[AttrNumberGetAttrOffset(Anum_dimension_integer_now_func_schema)] = NameGetDatum(schemaname); doReplace[AttrNumberGetAttrOffset(Anum_dimension_integer_now_func_schema)] = true; } } new_tuple = heap_modify_tuple(tuple, ts_scanner_get_tupledesc(ti), values, nulls, doReplace); ts_catalog_update(ti->scanrel, new_tuple); heap_freetuple(new_tuple); if (should_free) heap_freetuple(tuple); return SCAN_CONTINUE; } /* Go through internal dimensions table and rename all relevant schema */ void ts_dimensions_rename_schema_name(const char *old_name, const char *new_name) { NameData old_schema_name; ScanKeyData scankey[1]; Catalog *catalog = ts_catalog_get(); char *names[2] = { (char *) old_name, (char *) new_name }; ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, DIMENSION), .index = InvalidOid, .nkeys = 1, .scankey = scankey, .tuple_found = dimension_rename_schema_name, .data = (void *) names, .lockmode = RowExclusiveLock, .scandirection = ForwardScanDirection, }; namestrcpy(&old_schema_name, old_name); ScanKeyInit(&scankey[0], Anum_dimension_partitioning_func_schema, BTEqualStrategyNumber, F_NAMEEQ, NameGetDatum(&old_schema_name)); ts_scanner_scan(&scanctx); ScanKeyInit(&scankey[0], Anum_dimension_integer_now_func_schema, BTEqualStrategyNumber, F_NAMEEQ, NameGetDatum(&old_schema_name)); ts_scanner_scan(&scanctx); } ================================================ FILE: src/dimension.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <access/attnum.h> #include <access/htup_details.h> #include <catalog/pg_type.h> #include <executor/tuptable.h> #include "export.h" #include "time_utils.h" #include "ts_catalog/catalog.h" #include "utils.h" typedef struct PartitioningInfo PartitioningInfo; typedef struct DimensionSlice DimensionSlice; typedef struct DimensionVec DimensionVec; /* * The chunk interval of an open partitioning dimension. * * The type can either be INTERVALOID or an INT(2|4|8)OID. */ typedef struct ChunkInterval { Oid type; /* Interval value storage */ union { int64 integer_interval; /* For INT8OID, INT4OID, INT2OID */ Interval interval; /* For INTERVALOID */ }; } ChunkInterval; static inline void chunk_interval_set(ChunkInterval *chunk_interval, Datum interval, Oid type) { chunk_interval->type = type; /* Store interval value in appropriate union field */ if (type == INT8OID) chunk_interval->integer_interval = DatumGetInt64(interval); else if (type == INTERVALOID) chunk_interval->interval = *DatumGetIntervalP(interval); else if (type == INT4OID) chunk_interval->integer_interval = DatumGetInt32(interval); else if (type == INT2OID) chunk_interval->integer_interval = DatumGetInt16(interval); } typedef enum DimensionType { DIMENSION_TYPE_OPEN, DIMENSION_TYPE_CLOSED, DIMENSION_TYPE_STATS, DIMENSION_TYPE_ANY, } DimensionType; typedef struct Dimension { FormData_dimension fd; DimensionType type; AttrNumber column_attno; Oid main_table_relid; PartitioningInfo *partitioning; } Dimension; #define IS_OPEN_DIMENSION(d) ((d)->type == DIMENSION_TYPE_OPEN) #define IS_CLOSED_DIMENSION(d) ((d)->type == DIMENSION_TYPE_CLOSED) #define IS_VALID_OPEN_DIM_TYPE(type) \ (IS_INTEGER_TYPE(type) || IS_TIMESTAMP_TYPE(type) || IS_UUID_TYPE(type) || \ ts_type_is_int8_binary_compatible(type)) /* * A hyperspace defines how to partition in a N-dimensional space. */ typedef struct Hyperspace { int32 hypertable_id; Oid main_table_relid; uint16 capacity; uint16 num_dimensions; /* Open dimensions should be stored before closed dimensions */ Dimension dimensions[FLEXIBLE_ARRAY_MEMBER]; } Hyperspace; #define HYPERSPACE_SIZE(num_dimensions) \ (sizeof(Hyperspace) + (sizeof(Dimension) * (num_dimensions))) /* * A point in an N-dimensional hyperspace. */ typedef struct Point { int16 cardinality; uint8 num_coords; /* Open dimension coordinates are stored before the closed coordinates */ int64 coordinates[FLEXIBLE_ARRAY_MEMBER]; } Point; #define POINT_SIZE(cardinality) (sizeof(Point) + (sizeof(int64) * (cardinality))) #define DEFAULT_CHUNK_TIME_INTERVAL (USECS_PER_DAY * 7) /* 7 days w/o adaptive */ #define DEFAULT_CHUNK_TIME_INTERVAL_ADAPTIVE \ (USECS_PER_DAY) /* 1 day with adaptive \ * chunking enabled */ /* Default intervals for integer types */ #define DEFAULT_SMALLINT_INTERVAL 10000 #define DEFAULT_INT_INTERVAL 100000 #define DEFAULT_BIGINT_INTERVAL 1000000 typedef struct Hypertable Hypertable; /* * Dimension information used to validate, create and update dimensions. * * This structure is used both partially filled in from the dimension info * constructors as well as when building dimension info for the storage into * the dimension table. * * @see ts_hash_dimension * @see ts_range_dimension */ typedef struct DimensionInfo { /* We declare the SQL type dimension_info with INTERNALLENGTH = VARIABLE. * So, PostgreSQL expects a proper length info field (varlena header). */ int32 vl_len_; Oid table_relid; int32 dimension_id; NameData colname; Oid coltype; DimensionType type; ChunkInterval chunk_interval; int64 interval; int32 num_slices; regproc partitioning_func; bool if_not_exists; bool skip; bool set_not_null; bool num_slices_is_set; bool adaptive_chunking; /* True if adaptive chunking is enabled */ Hypertable *ht; } DimensionInfo; #define DIMENSION_INFO_IS_SET(di) (di != NULL && OidIsValid((di)->table_relid)) #define DIMENSION_INFO_IS_VALID(di) \ (info->num_slices_is_set || OidIsValid(info->chunk_interval.type)) extern Hyperspace *ts_dimension_scan(int32 hypertable_id, Oid main_table_relid, int16 num_dimension, MemoryContext mctx); extern DimensionSlice *ts_dimension_calculate_default_slice(const Dimension *dim, int64 value); extern TSDLLEXPORT Point *ts_hyperspace_calculate_point(const Hyperspace *h, TupleTableSlot *slot); extern int ts_dimension_get_slice_ordinal(const Dimension *dim, const DimensionSlice *slice); extern TSDLLEXPORT const Dimension *ts_hyperspace_get_dimension_by_id(const Hyperspace *hs, int32 id); extern TSDLLEXPORT const Dimension *ts_hyperspace_get_dimension(const Hyperspace *hs, DimensionType type, Index n); extern TSDLLEXPORT Dimension *ts_hyperspace_get_mutable_dimension(Hyperspace *hs, DimensionType type, Index n); extern TSDLLEXPORT const Dimension * ts_hyperspace_get_dimension_by_name(const Hyperspace *hs, DimensionType type, const char *name); extern TSDLLEXPORT Dimension * ts_hyperspace_get_mutable_dimension_by_name(Hyperspace *hs, DimensionType type, const char *name); extern DimensionVec *ts_dimension_get_slices(const Dimension *dim); extern int32 ts_dimension_get_hypertable_id(int32 dimension_id); extern int ts_dimension_set_type(Dimension *dim, Oid newtype); extern TSDLLEXPORT Oid ts_dimension_get_partition_type(const Dimension *dim); extern int ts_dimension_set_name(Dimension *dim, const char *newname); extern TSDLLEXPORT int ts_dimension_set_chunk_interval(Dimension *dim, int64 chunk_interval); extern int ts_dimension_set_compress_interval(Dimension *dim, int64 compress_interval); extern Datum ts_dimension_transform_value(const Dimension *dim, Oid collation, Datum value, Oid const_datum_type, Oid *restype); extern int ts_dimension_delete_by_hypertable_id(int32 hypertable_id, bool delete_slices); extern TSDLLEXPORT DimensionInfo *ts_dimension_info_create_open(Oid table_relid, Name column_name, Datum interval, Oid interval_type, regproc partitioning_func); extern TSDLLEXPORT DimensionInfo *ts_dimension_info_create_closed(Oid table_relid, Name column_name, int32 num_slices, regproc partitioning_func); extern void ts_dimension_info_validate(DimensionInfo *info); extern int32 ts_dimension_add_from_info(DimensionInfo *info); extern void ts_dimensions_rename_schema_name(const char *old_name, const char *new_name); extern TSDLLEXPORT void ts_dimension_update(const Hypertable *ht, const NameData *dimname, DimensionType dimtype, Datum *interval, Oid *intervaltype, int16 *num_slices, Oid *integer_now_func); extern TSDLLEXPORT Point *ts_point_create(int16 num_dimensions); extern TSDLLEXPORT bool ts_is_equality_operator(Oid opno, Oid left, Oid right); extern TSDLLEXPORT Datum ts_dimension_info_in(PG_FUNCTION_ARGS); extern TSDLLEXPORT Datum ts_dimension_info_out(PG_FUNCTION_ARGS); #define hyperspace_get_open_dimension(space, i) \ ts_hyperspace_get_dimension(space, DIMENSION_TYPE_OPEN, i) #define hyperspace_get_closed_dimension(space, i) \ ts_hyperspace_get_dimension(space, DIMENSION_TYPE_CLOSED, i) ================================================ FILE: src/dimension_slice.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/heapam.h> #include <access/relscan.h> #include <access/xact.h> #include <catalog/indexing.h> #include <catalog/pg_opfamily.h> #include <catalog/pg_type.h> #include <funcapi.h> #include <utils/builtins.h> #include <utils/lsyscache.h> #include <utils/rel.h> #include "bgw_policy/chunk_stats.h" #include "chunk.h" #include "chunk_constraint.h" #include "dimension.h" #include "dimension_slice.h" #include "dimension_vector.h" #include "hypertable.h" #include "scanner.h" #include "ts_catalog/catalog.h" #include "compat/compat.h" static inline DimensionSlice * dimension_slice_alloc(void) { return palloc0(sizeof(DimensionSlice)); } static inline DimensionSlice * dimension_slice_from_form_data(const Form_dimension_slice fd) { DimensionSlice *slice = dimension_slice_alloc(); memcpy(&slice->fd, fd, sizeof(FormData_dimension_slice)); slice->storage_free = NULL; slice->storage = NULL; return slice; } static inline DimensionSlice * dimension_slice_from_slot(TupleTableSlot *slot) { bool should_free; HeapTuple tuple = ExecFetchSlotHeapTuple(slot, false, &should_free); DimensionSlice *slice; slice = dimension_slice_from_form_data((Form_dimension_slice) GETSTRUCT(tuple)); if (should_free) heap_freetuple(tuple); return slice; } static HeapTuple dimension_slice_formdata_make_tuple(const FormData_dimension_slice *fd, TupleDesc desc) { Datum values[Natts_dimension_slice]; bool nulls[Natts_dimension_slice] = { false }; memset(values, 0, sizeof(Datum) * Natts_dimension_slice); values[AttrNumberGetAttrOffset(Anum_dimension_slice_id)] = Int32GetDatum(fd->id); values[AttrNumberGetAttrOffset(Anum_dimension_slice_dimension_id)] = Int32GetDatum(fd->dimension_id); values[AttrNumberGetAttrOffset(Anum_dimension_slice_range_start)] = Int64GetDatum(fd->range_start); values[AttrNumberGetAttrOffset(Anum_dimension_slice_range_end)] = Int64GetDatum(fd->range_end); return heap_form_tuple(desc, values, nulls); } static inline void dimension_slice_formdata_fill(FormData_dimension_slice *fd, const TupleInfo *ti) { bool nulls[Natts_dimension_slice]; Datum values[Natts_dimension_slice]; bool should_free; HeapTuple tuple; tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); heap_deform_tuple(tuple, ts_scanner_get_tupledesc(ti), values, nulls); Assert(!nulls[AttrNumberGetAttrOffset(Anum_dimension_slice_id)]); Assert(!nulls[AttrNumberGetAttrOffset(Anum_dimension_slice_dimension_id)]); Assert(!nulls[AttrNumberGetAttrOffset(Anum_dimension_slice_range_start)]); Assert(!nulls[AttrNumberGetAttrOffset(Anum_dimension_slice_range_end)]); fd->id = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_dimension_slice_id)]); fd->dimension_id = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_dimension_slice_dimension_id)]); fd->range_start = DatumGetInt64(values[AttrNumberGetAttrOffset(Anum_dimension_slice_range_start)]); fd->range_end = DatumGetInt64(values[AttrNumberGetAttrOffset(Anum_dimension_slice_range_end)]); if (should_free) heap_freetuple(tuple); } static bool lock_dimension_slice_tuple(int32 dimension_slice_id, ItemPointer tid, FormData_dimension_slice *form) { bool success = false; ScanTupLock scantuplock = { .waitpolicy = LockWaitBlock, .lockmode = LockTupleExclusive, }; ScanIterator iterator = ts_scan_iterator_create(DIMENSION_SLICE, RowShareLock, CurrentMemoryContext); iterator.ctx.index = catalog_get_index(ts_catalog_get(), DIMENSION_SLICE, DIMENSION_SLICE_ID_IDX); iterator.ctx.tuplock = &scantuplock; /* Keeping the lock since we presumably want to update the tuple */ iterator.ctx.flags = SCANNER_F_KEEPLOCK; /* see table_tuple_lock for details about flags that are set in TupleExclusive mode */ scantuplock.lockflags = TUPLE_LOCK_FLAG_LOCK_UPDATE_IN_PROGRESS; if (!IsolationUsesXactSnapshot()) { /* in read committed mode, we follow all updates to this tuple */ scantuplock.lockflags |= TUPLE_LOCK_FLAG_FIND_LAST_VERSION; } ts_scan_iterator_scan_key_init(&iterator, Anum_dimension_slice_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(dimension_slice_id)); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); if (ti->lockresult != TM_Ok) { if (IsolationUsesXactSnapshot()) { /* For Repeatable Read and Serializable isolation level report error * if we cannot lock the tuple */ ereport(ERROR, (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("could not serialize access due to concurrent update"))); } else { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("unable to lock hypertable catalog tuple, lock result is %d for " "hypertable " "ID (%d)", ti->lockresult, dimension_slice_id))); } } dimension_slice_formdata_fill(form, ti); ItemPointer result_tid = ts_scanner_get_tuple_tid(ti); tid->ip_blkid = result_tid->ip_blkid; tid->ip_posid = result_tid->ip_posid; success = true; break; } ts_scan_iterator_close(&iterator); return success; } /* update the tuple at this tid. The assumption is that we already hold a * tuple exclusive lock and no other transaction can modify this tuple * The sequence of operations for any update is: * lock the tuple using lock_hypertable_tuple. * then update the required fields * call dimension_slice_update_catalog_tuple to complete the update. * This ensures correct tuple locking and tuple updates in the presence of * concurrent transactions. Failure to follow this results in catalog corruption */ static void dimension_slice_update_catalog_tuple(ItemPointer tid, FormData_dimension_slice *update) { HeapTuple new_tuple; CatalogSecurityContext sec_ctx; Catalog *catalog = ts_catalog_get(); Oid table = catalog_get_table_id(catalog, DIMENSION_SLICE); Relation dimension_slice_rel = relation_open(table, RowExclusiveLock); new_tuple = dimension_slice_formdata_make_tuple(update, dimension_slice_rel->rd_att); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_update_tid(dimension_slice_rel, tid, new_tuple); ts_catalog_restore_user(&sec_ctx); heap_freetuple(new_tuple); relation_close(dimension_slice_rel, NoLock); } /* delete the tuple at this tid. The assumption is that we already hold a * tuple exclusive lock and no other transaction can modify this tuple * The sequence of operations for any delete is: * lock the tuple using lock_hypertable_tuple. * call dimension_slice_delete_catalog_tuple to complete the delete. * This ensures correct tuple locking and tuple deletes in the presence of * concurrent transactions. Failure to follow this results in catalog corruption */ static void dimension_slice_delete_catalog_tuple(ItemPointer tid) { CatalogSecurityContext sec_ctx; Catalog *catalog = ts_catalog_get(); Oid table = catalog_get_table_id(catalog, DIMENSION_SLICE); Relation dimension_slice_rel = relation_open(table, RowExclusiveLock); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_delete_tid(dimension_slice_rel, tid); ts_catalog_restore_user(&sec_ctx); relation_close(dimension_slice_rel, NoLock); } DimensionSlice * ts_dimension_slice_create(int dimension_id, int64 range_start, int64 range_end) { DimensionSlice *slice = dimension_slice_alloc(); slice->fd.dimension_id = dimension_id; slice->fd.range_start = range_start; slice->fd.range_end = range_end; return slice; } int ts_dimension_slice_cmp(const DimensionSlice *left, const DimensionSlice *right) { int res = DIMENSION_SLICE_RANGE_START_CMP(left, right); if (res == 0) res = DIMENSION_SLICE_RANGE_END_CMP(left, right); return res; } int ts_dimension_slice_cmp_coordinate(const DimensionSlice *slice, int64 coord) { coord = REMAP_LAST_COORDINATE(coord); if (coord < slice->fd.range_start) return -1; if (coord >= slice->fd.range_end) return 1; return 0; } static bool tuple_is_deleted(TupleInfo *ti) { #ifdef USE_ASSERT_CHECKING if (ti->lockresult == TM_Deleted) Assert(ItemPointerEquals(ts_scanner_get_tuple_tid(ti), &ti->lockfd.ctid)); #endif return ti->lockresult == TM_Deleted; } static void lock_result_ok_or_abort(TupleInfo *ti) { switch (ti->lockresult) { /* Updating a tuple in the same transaction before taking a lock is OK * even though it is not expected in this case */ case TM_SelfModified: case TM_Ok: break; case TM_Deleted: case TM_Updated: ereport(ERROR, (errcode(ERRCODE_LOCK_NOT_AVAILABLE), errmsg("chunk %s by other transaction", tuple_is_deleted(ti) ? "deleted" : "updated"), errhint("Retry the operation again."))); pg_unreachable(); break; case TM_BeingModified: ereport(ERROR, (errcode(ERRCODE_LOCK_NOT_AVAILABLE), errmsg("chunk updated by other transaction"), errhint("Retry the operation again."))); pg_unreachable(); break; case TM_Invisible: elog(ERROR, "attempt to lock invisible tuple"); pg_unreachable(); break; case TM_WouldBlock: default: elog(ERROR, "unexpected tuple lock status: %d", ti->lockresult); pg_unreachable(); break; } } static ScanTupleResult dimension_vec_tuple_found_list(TupleInfo *ti, void *data) { List **slices = (List **) data; DimensionSlice *slice; MemoryContext old; switch (ti->lockresult) { case TM_SelfModified: case TM_Ok: break; case TM_Deleted: case TM_Updated: /* Treat as not found */ return SCAN_CONTINUE; default: elog(ERROR, "unexpected tuple lock status: %d", ti->lockresult); pg_unreachable(); break; } old = MemoryContextSwitchTo(ti->mctx); slice = dimension_slice_from_slot(ti->slot); Assert(NULL != slice); *slices = lappend(*slices, slice); MemoryContextSwitchTo(old); return SCAN_CONTINUE; } static ScanTupleResult dimension_vec_tuple_found(TupleInfo *ti, void *data) { DimensionVec **slices = (DimensionVec **) data; DimensionSlice *slice; MemoryContext old; switch (ti->lockresult) { case TM_SelfModified: case TM_Ok: break; case TM_Deleted: case TM_Updated: /* Treat as not found */ return SCAN_CONTINUE; default: elog(ERROR, "unexpected tuple lock status: %d", ti->lockresult); pg_unreachable(); break; } old = MemoryContextSwitchTo(ti->mctx); slice = dimension_slice_from_slot(ti->slot); Assert(NULL != slice); *slices = ts_dimension_vec_add_slice(slices, slice); MemoryContextSwitchTo(old); return SCAN_CONTINUE; } static int dimension_slice_scan_limit_direction_internal(int indexid, ScanKeyData *scankey, int nkeys, tuple_found_func on_tuple_found, void *scandata, int limit, ScanDirection scandir, LOCKMODE lockmode, const ScanTupLock *tuplock, MemoryContext mctx) { Catalog *catalog = ts_catalog_get(); ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, DIMENSION_SLICE), .index = catalog_get_index(catalog, DIMENSION_SLICE, indexid), .nkeys = nkeys, .scankey = scankey, .data = scandata, .limit = limit, .tuplock = tuplock, .tuple_found = on_tuple_found, .lockmode = lockmode, .scandirection = scandir, .result_mctx = mctx, }; return ts_scanner_scan(&scanctx); } static int dimension_slice_scan_limit_internal(int indexid, ScanKeyData *scankey, int nkeys, tuple_found_func on_tuple_found, void *scandata, int limit, LOCKMODE lockmode, const ScanTupLock *tuplock, MemoryContext mctx) { /* * We have =, <=, > ops for index columns, so backwards scan direction is * more appropriate. Forward direction wouldn't be able to use the second * column to find a starting point for the scan. Unfortunately we can't do * anything about the third column, we'll be checking for it with a * sequential scan over index pages. Ideally we need some other index type * than btree for this. */ return dimension_slice_scan_limit_direction_internal(indexid, scankey, nkeys, on_tuple_found, scandata, limit, BackwardScanDirection, lockmode, tuplock, mctx); } /* * Scan for slices that enclose the coordinate in the given dimension. * * Returns a dimension vector of slices that enclose the coordinate. */ DimensionVec * ts_dimension_slice_scan_limit(int32 dimension_id, int64 coordinate, int limit, const ScanTupLock *slice_lock) { ScanKeyData scankey[3]; DimensionVec *slices = ts_dimension_vec_create(limit > 0 ? limit : DIMENSION_VEC_DEFAULT_SIZE); coordinate = REMAP_LAST_COORDINATE(coordinate); /* * Perform an index scan for slices matching the dimension's ID and which * enclose the coordinate. */ ScanKeyInit(&scankey[0], Anum_dimension_slice_dimension_id_range_start_range_end_idx_dimension_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(dimension_id)); ScanKeyInit(&scankey[1], Anum_dimension_slice_dimension_id_range_start_range_end_idx_range_start, BTLessEqualStrategyNumber, F_INT8LE, Int64GetDatum(coordinate)); ScanKeyInit(&scankey[2], Anum_dimension_slice_dimension_id_range_start_range_end_idx_range_end, BTGreaterStrategyNumber, F_INT8GT, Int64GetDatum(coordinate)); dimension_slice_scan_limit_internal(DIMENSION_SLICE_DIMENSION_ID_RANGE_START_RANGE_END_IDX, scankey, 3, dimension_vec_tuple_found, (void *) &slices, limit, AccessShareLock, slice_lock, CurrentMemoryContext); return ts_dimension_vec_sort(&slices); } void ts_dimension_slice_scan_list(int32 dimension_id, int64 coordinate, List **matching_dimension_slices, const ScanTupLock *slice_lock) { coordinate = REMAP_LAST_COORDINATE(coordinate); /* * Perform an index scan for slices matching the dimension's ID and which * enclose the coordinate. */ ScanKeyData scankey[3]; ScanKeyInit(&scankey[0], Anum_dimension_slice_dimension_id_range_start_range_end_idx_dimension_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(dimension_id)); ScanKeyInit(&scankey[1], Anum_dimension_slice_dimension_id_range_start_range_end_idx_range_start, BTLessEqualStrategyNumber, F_INT8LE, Int64GetDatum(coordinate)); ScanKeyInit(&scankey[2], Anum_dimension_slice_dimension_id_range_start_range_end_idx_range_end, BTGreaterStrategyNumber, F_INT8GT, Int64GetDatum(coordinate)); dimension_slice_scan_limit_internal(DIMENSION_SLICE_DIMENSION_ID_RANGE_START_RANGE_END_IDX, scankey, 3, dimension_vec_tuple_found_list, (void *) matching_dimension_slices, /* limit = */ 0, AccessShareLock, slice_lock, CurrentMemoryContext); } int ts_dimension_slice_scan_iterator_set_range(ScanIterator *it, int32 dimension_id, StrategyNumber start_strategy, int64 start_value, StrategyNumber end_strategy, int64 end_value) { Catalog *catalog = ts_catalog_get(); it->ctx.index = catalog_get_index(catalog, DIMENSION_SLICE, DIMENSION_SLICE_DIMENSION_ID_RANGE_START_RANGE_END_IDX); ts_scan_iterator_scan_key_reset(it); ts_scan_iterator_scan_key_init( it, Anum_dimension_slice_dimension_id_range_start_range_end_idx_dimension_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(dimension_id)); /* * Perform an index scan for slices matching the dimension's ID and which * enclose the coordinate. */ if (start_strategy != InvalidStrategy) { Oid opno = get_opfamily_member(INTEGER_BTREE_FAM_OID, INT8OID, INT8OID, start_strategy); Oid proc = get_opcode(opno); Assert(OidIsValid(proc)); ts_scan_iterator_scan_key_init( it, Anum_dimension_slice_dimension_id_range_start_range_end_idx_range_start, start_strategy, proc, Int64GetDatum(start_value)); } if (end_strategy != InvalidStrategy) { Oid opno = get_opfamily_member(INTEGER_BTREE_FAM_OID, INT8OID, INT8OID, end_strategy); Oid proc = get_opcode(opno); Assert(OidIsValid(proc)); /* * range_end is stored as exclusive, so add 1 to the value being * searched. Also avoid overflow */ if (end_value != PG_INT64_MAX) { end_value++; /* * If getting as input INT64_MAX-1, need to remap the incremented * value back to INT64_MAX-1 */ end_value = REMAP_LAST_COORDINATE(end_value); } else { /* * The point with INT64_MAX gets mapped to INT64_MAX-1 so * incrementing that gets you to INT_64MAX */ end_value = PG_INT64_MAX; } ts_scan_iterator_scan_key_init( it, Anum_dimension_slice_dimension_id_range_start_range_end_idx_range_end, end_strategy, proc, Int64GetDatum(end_value)); } return it->ctx.nkeys; } /* * Look for all dimension slices where (lower_bound, upper_bound) of the dimension_slice contains * the given (start_value, end_value) range * */ DimensionVec * ts_dimension_slice_scan_range_limit(int32 dimension_id, StrategyNumber start_strategy, int64 start_value, StrategyNumber end_strategy, int64 end_value, int limit, const ScanTupLock *tuplock) { DimensionVec *slices = ts_dimension_vec_create(limit > 0 ? limit : DIMENSION_VEC_DEFAULT_SIZE); ScanIterator it = ts_dimension_slice_scan_iterator_create(tuplock, CurrentMemoryContext); ts_dimension_slice_scan_iterator_set_range(&it, dimension_id, start_strategy, start_value, end_strategy, end_value); it.ctx.limit = limit; ts_scanner_foreach(&it) { const TupleInfo *ti = ts_scan_iterator_tuple_info(&it); DimensionSlice *slice; MemoryContext old; switch (ti->lockresult) { case TM_SelfModified: case TM_Ok: old = MemoryContextSwitchTo(ti->mctx); slice = dimension_slice_from_slot(ti->slot); Assert(NULL != slice); slices = ts_dimension_vec_add_slice(&slices, slice); MemoryContextSwitchTo(old); break; case TM_Deleted: case TM_Updated: /* Treat as not found */ break; default: elog(ERROR, "unexpected tuple lock status: %d", ti->lockresult); pg_unreachable(); break; } } Assert(limit <= 0 || slices->num_slices <= limit); ts_scan_iterator_close(&it); return ts_dimension_vec_sort(&slices); } /* * Scan for slices that collide/overlap with the given range. * * Returns a dimension vector of colliding slices. */ DimensionVec * ts_dimension_slice_collision_scan_limit(int32 dimension_id, int64 range_start, int64 range_end, int limit) { ScanKeyData scankey[3]; DimensionVec *slices = ts_dimension_vec_create(limit > 0 ? limit : DIMENSION_VEC_DEFAULT_SIZE); ScanKeyInit(&scankey[0], Anum_dimension_slice_dimension_id_range_start_range_end_idx_dimension_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(dimension_id)); ScanKeyInit(&scankey[1], Anum_dimension_slice_dimension_id_range_start_range_end_idx_range_start, BTLessStrategyNumber, F_INT8LT, Int64GetDatum(range_end)); ScanKeyInit(&scankey[2], Anum_dimension_slice_dimension_id_range_start_range_end_idx_range_end, BTGreaterStrategyNumber, F_INT8GT, Int64GetDatum(range_start)); dimension_slice_scan_limit_internal(DIMENSION_SLICE_DIMENSION_ID_RANGE_START_RANGE_END_IDX, scankey, 3, dimension_vec_tuple_found, (void *) &slices, limit, AccessShareLock, NULL, CurrentMemoryContext); return ts_dimension_vec_sort(&slices); } DimensionVec * ts_dimension_slice_scan_by_dimension(int32 dimension_id, int limit) { ScanKeyData scankey[1]; DimensionVec *slices = ts_dimension_vec_create(limit > 0 ? limit : DIMENSION_VEC_DEFAULT_SIZE); ScanKeyInit(&scankey[0], Anum_dimension_slice_dimension_id_range_start_range_end_idx_dimension_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(dimension_id)); dimension_slice_scan_limit_internal(DIMENSION_SLICE_DIMENSION_ID_RANGE_START_RANGE_END_IDX, scankey, 1, dimension_vec_tuple_found, (void *) &slices, limit, AccessShareLock, NULL, CurrentMemoryContext); return ts_dimension_vec_sort(&slices); } /* * Return slices that occur "before" the given point. * * The slices will be allocated on the given memory context. Note, however, that * the returned dimension vector is allocated on the current memory context. */ DimensionVec * ts_dimension_slice_scan_by_dimension_before_point(int32 dimension_id, int64 point, int limit, ScanDirection scandir, MemoryContext mctx) { ScanKeyData scankey[3]; DimensionVec *slices = ts_dimension_vec_create(limit > 0 ? limit : DIMENSION_VEC_DEFAULT_SIZE); ScanKeyInit(&scankey[0], Anum_dimension_slice_dimension_id_range_start_range_end_idx_dimension_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(dimension_id)); ScanKeyInit(&scankey[1], Anum_dimension_slice_dimension_id_range_start_range_end_idx_range_start, BTLessStrategyNumber, F_INT8LT, Int64GetDatum(point)); ScanKeyInit(&scankey[2], Anum_dimension_slice_dimension_id_range_start_range_end_idx_range_end, BTLessStrategyNumber, F_INT8LT, Int64GetDatum(point)); dimension_slice_scan_limit_direction_internal( DIMENSION_SLICE_DIMENSION_ID_RANGE_START_RANGE_END_IDX, scankey, 3, dimension_vec_tuple_found, (void *) &slices, limit, scandir, AccessShareLock, NULL, mctx); return ts_dimension_vec_sort(&slices); } static ScanTupleResult dimension_slice_tuple_delete(TupleInfo *ti, void *data) { bool isnull; Datum dimension_slice_id = slot_getattr(ti->slot, Anum_dimension_slice_id, &isnull); if (ti->lockresult != TM_Ok) { if (IsolationUsesXactSnapshot()) { /* For Repeatable Read and Serializable isolation level report error * if we cannot lock the tuple */ ereport(ERROR, (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("could not serialize access due to concurrent update"))); } else { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("unable to lock hypertable catalog tuple, lock result is %d for " "hypertable " "ID (%d)", ti->lockresult, DatumGetInt32(dimension_slice_id)))); } } bool *delete_constraints = data; CatalogSecurityContext sec_ctx; Assert(!isnull); /* delete chunk constraints */ if (NULL != delete_constraints && *delete_constraints) ts_chunk_constraint_delete_by_dimension_slice_id(DatumGetInt32(dimension_slice_id)); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); ts_catalog_restore_user(&sec_ctx); return SCAN_CONTINUE; } int ts_dimension_slice_delete_by_dimension_id(int32 dimension_id, bool delete_constraints) { ScanKeyData scankey[1]; ScanKeyInit(&scankey[0], Anum_dimension_slice_dimension_id_range_start_range_end_idx_dimension_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(dimension_id)); ScanTupLock scantuplock = { .waitpolicy = LockWaitBlock, .lockmode = LockTupleExclusive, }; return dimension_slice_scan_limit_internal( DIMENSION_SLICE_DIMENSION_ID_RANGE_START_RANGE_END_IDX, scankey, 1, dimension_slice_tuple_delete, (void *) &delete_constraints, 0, RowExclusiveLock, &scantuplock, CurrentMemoryContext); } int ts_dimension_slice_delete_by_id(int32 dimension_slice_id, bool delete_constraints) { FormData_dimension_slice form; ItemPointerData tid; /* lock the tuple entry in the catalog table */ bool found = lock_dimension_slice_tuple(dimension_slice_id, &tid, &form); Ensure(found, "dimension slice id %d not found", dimension_slice_id); dimension_slice_delete_catalog_tuple(&tid); return true; } static ScanTupleResult dimension_slice_fill(TupleInfo *ti, void *data) { switch (ti->lockresult) { case TM_SelfModified: case TM_Ok: { DimensionSlice **slice = (DimensionSlice **) data; bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); memcpy(&(*slice)->fd, GETSTRUCT(tuple), sizeof(FormData_dimension_slice)); if (should_free) heap_freetuple(tuple); break; } case TM_Deleted: case TM_Updated: /* Same as not found */ break; default: elog(ERROR, "unexpected tuple lock status: %d", ti->lockresult); pg_unreachable(); break; } return SCAN_DONE; } /* * Scan for an existing slice that exactly matches the given slice's dimension * and range. If a match is found, the given slice is updated with slice ID * and the tuple is locked. * * Returns true if the dimension slice was found (and locked), false * otherwise. */ bool ts_dimension_slice_scan_for_existing(const DimensionSlice *slice, const ScanTupLock *tuplock) { ScanKeyData scankey[3]; ScanKeyInit(&scankey[0], Anum_dimension_slice_dimension_id_range_start_range_end_idx_dimension_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(slice->fd.dimension_id)); ScanKeyInit(&scankey[1], Anum_dimension_slice_dimension_id_range_start_range_end_idx_range_start, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(slice->fd.range_start)); ScanKeyInit(&scankey[2], Anum_dimension_slice_dimension_id_range_start_range_end_idx_range_end, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum(slice->fd.range_end)); return dimension_slice_scan_limit_internal( DIMENSION_SLICE_DIMENSION_ID_RANGE_START_RANGE_END_IDX, scankey, 3, dimension_slice_fill, (void *) &slice, 1, AccessShareLock, tuplock, CurrentMemoryContext); } DimensionSlice * ts_dimension_slice_from_tuple(TupleInfo *ti) { DimensionSlice *slice; MemoryContext old; lock_result_ok_or_abort(ti); old = MemoryContextSwitchTo(ti->mctx); slice = dimension_slice_from_slot(ti->slot); MemoryContextSwitchTo(old); return slice; } static ScanTupleResult dimension_slice_tuple_found(TupleInfo *ti, void *data) { DimensionSlice **slice = (DimensionSlice **) data; *slice = ts_dimension_slice_from_tuple(ti); return SCAN_DONE; } /* Scan for a slice by dimension slice id. * * If you're scanning for a tuple, you have to provide a lock, since, otherwise, * concurrent threads can do bad things with the tuple and you probably want * it to not change nor disappear. */ DimensionSlice * ts_dimension_slice_scan_by_id_and_lock(int32 dimension_slice_id, const ScanTupLock *tuplock, MemoryContext mctx, LOCKMODE lockmode) { DimensionSlice *slice = NULL; ScanKeyData scankey[1]; ScanKeyInit(&scankey[0], Anum_dimension_slice_id_idx_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(dimension_slice_id)); dimension_slice_scan_limit_internal(DIMENSION_SLICE_ID_IDX, scankey, 1, dimension_slice_tuple_found, (void *) &slice, 1, lockmode, tuplock, mctx); return slice; } ScanIterator ts_dimension_slice_scan_iterator_create(const ScanTupLock *tuplock, MemoryContext result_mcxt) { ScanIterator it = ts_scan_iterator_create(DIMENSION_SLICE, AccessShareLock, result_mcxt); it.ctx.flags |= SCANNER_F_NOEND_AND_NOCLOSE; it.ctx.tuplock = RecoveryInProgress() ? NULL : tuplock; return it; } void ts_dimension_slice_scan_iterator_set_slice_id(ScanIterator *it, int32 slice_id) { it->ctx.index = catalog_get_index(ts_catalog_get(), DIMENSION_SLICE, DIMENSION_SLICE_ID_IDX); ts_scan_iterator_scan_key_reset(it); ts_scan_iterator_scan_key_init(it, Anum_dimension_slice_id_idx_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(slice_id)); } DimensionSlice * ts_dimension_slice_scan_iterator_get_by_id(ScanIterator *it, int32 slice_id) { TupleInfo *ti; ts_dimension_slice_scan_iterator_set_slice_id(it, slice_id); ts_scan_iterator_start_or_restart_scan(it); ti = ts_scan_iterator_next(it); if (!ti) return NULL; DimensionSlice *slice = ts_dimension_slice_from_tuple(ti); /* There should be only one slice with the given id */ Assert(ts_scan_iterator_next(it) == NULL); return slice; } DimensionSlice * ts_dimension_slice_copy(const DimensionSlice *original) { DimensionSlice *new = palloc(sizeof(DimensionSlice)); Assert(original->storage == NULL); Assert(original->storage_free == NULL); memcpy(new, original, sizeof(DimensionSlice)); return new; } /* * Check if two dimensions slices overlap by doing collision detection in one * dimension. * * Returns true if the slices collide, otherwise false. */ bool ts_dimension_slices_collide(const DimensionSlice *slice1, const DimensionSlice *slice2) { Assert(slice1->fd.dimension_id == slice2->fd.dimension_id); return (slice1->fd.range_start < slice2->fd.range_end && slice1->fd.range_end > slice2->fd.range_start); } /* * Check whether two slices are identical. * * We require by assertion that the slices are in the same dimension and we only * compare the ranges (i.e., the slice ID is not important for equality). * * Returns true if the slices have identical ranges, otherwise false. */ bool ts_dimension_slices_equal(const DimensionSlice *slice1, const DimensionSlice *slice2) { Assert(slice1->fd.dimension_id == slice2->fd.dimension_id); return slice1->fd.range_start == slice2->fd.range_start && slice1->fd.range_end == slice2->fd.range_end; } /*- * Cut a slice that collides with another slice. The coordinate is the point of * insertion, and determines which end of the slice to cut. * * Case where we cut "after" the coordinate: * * ' [-x--------] * ' [--------] * * Case where we cut "before" the coordinate: * * ' [------x--] * ' [--------] * * Returns true if the slice was cut, otherwise false. */ bool ts_dimension_slice_cut(DimensionSlice *to_cut, const DimensionSlice *other, int64 coord) { Assert(to_cut->fd.dimension_id == other->fd.dimension_id); coord = REMAP_LAST_COORDINATE(coord); if (other->fd.range_end <= coord && other->fd.range_end > to_cut->fd.range_start) { /* Cut "before" the coordinate */ to_cut->fd.range_start = other->fd.range_end; return true; } else if (other->fd.range_start > coord && other->fd.range_start < to_cut->fd.range_end) { /* Cut "after" the coordinate */ to_cut->fd.range_end = other->fd.range_start; return true; } return false; } void ts_dimension_slice_free(DimensionSlice *slice) { if (slice->storage_free != NULL) slice->storage_free(slice->storage); pfree(slice); } static bool dimension_slice_insert_relation(const Relation rel, DimensionSlice *slice) { TupleDesc desc = RelationGetDescr(rel); Datum values[Natts_dimension_slice]; bool nulls[Natts_dimension_slice] = { false }; CatalogSecurityContext sec_ctx; if (slice->fd.id > 0) /* Slice already exists in table */ return false; ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); memset(values, 0, sizeof(values)); slice->fd.id = ts_catalog_table_next_seq_id(ts_catalog_get(), DIMENSION_SLICE); values[AttrNumberGetAttrOffset(Anum_dimension_slice_id)] = Int32GetDatum(slice->fd.id); values[AttrNumberGetAttrOffset(Anum_dimension_slice_dimension_id)] = Int32GetDatum(slice->fd.dimension_id); values[AttrNumberGetAttrOffset(Anum_dimension_slice_range_start)] = Int64GetDatum(slice->fd.range_start); values[AttrNumberGetAttrOffset(Anum_dimension_slice_range_end)] = Int64GetDatum(slice->fd.range_end); ts_catalog_insert_values(rel, desc, values, nulls); ts_catalog_restore_user(&sec_ctx); return true; } /* * Insert slices into the catalog. * * Only slices that don't already exist in the catalog will be inserted. Note * that all slices that already exist (i.e., have a valid ID) MUST be locked * with a tuple lock (e.g., FOR KEY SHARE) prior to calling this function * since they won't be created. Otherwise it is not possible to guarantee that * all slices still exist once the transaction commits. * * Returns the number of slices inserted. */ int ts_dimension_slice_insert_multi(DimensionSlice **slices, Size num_slices) { Catalog *catalog = ts_catalog_get(); Relation rel; Size i, n = 0; rel = table_open(catalog_get_table_id(catalog, DIMENSION_SLICE), RowExclusiveLock); for (i = 0; i < num_slices; i++) { if (slices[i]->fd.id == 0) { dimension_slice_insert_relation(rel, slices[i]); n++; } } table_close(rel, RowExclusiveLock); return n; } void ts_dimension_slice_insert(DimensionSlice *slice) { Catalog *catalog = ts_catalog_get(); Relation rel; rel = table_open(catalog_get_table_id(catalog, DIMENSION_SLICE), RowExclusiveLock); dimension_slice_insert_relation(rel, slice); /* Keeping a row lock to prevent VACUUM or ALTER TABLE from running while working on the table. * This is known to cause issues in certain situations. */ table_close(rel, NoLock); } static ScanTupleResult dimension_slice_nth_tuple_found(TupleInfo *ti, void *data) { DimensionSlice **slice = (DimensionSlice **) data; MemoryContext old = MemoryContextSwitchTo(ti->mctx); *slice = dimension_slice_from_slot(ti->slot); MemoryContextSwitchTo(old); return SCAN_CONTINUE; } DimensionSlice * ts_dimension_slice_nth_latest_slice(int32 dimension_id, int n) { ScanKeyData scankey[1]; int num_tuples; DimensionSlice *ret = NULL; ScanKeyInit(&scankey[0], Anum_dimension_slice_dimension_id_range_start_range_end_idx_dimension_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(dimension_id)); num_tuples = dimension_slice_scan_limit_direction_internal( DIMENSION_SLICE_DIMENSION_ID_RANGE_START_RANGE_END_IDX, scankey, 1, dimension_slice_nth_tuple_found, (void *) &ret, n, BackwardScanDirection, AccessShareLock, NULL, CurrentMemoryContext); if (num_tuples < n) return NULL; return ret; } DimensionSlice * ts_dimension_slice_nth_earliest_slice(int32 dimension_id, int n) { ScanKeyData scankey[1]; int num_tuples; DimensionSlice *ret = NULL; ScanKeyInit(&scankey[0], Anum_dimension_slice_dimension_id_range_start_range_end_idx_dimension_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(dimension_id)); num_tuples = dimension_slice_scan_limit_direction_internal( DIMENSION_SLICE_DIMENSION_ID_RANGE_START_RANGE_END_IDX, scankey, 1, dimension_slice_nth_tuple_found, (void *) &ret, n, ForwardScanDirection, AccessShareLock, NULL, CurrentMemoryContext); if (num_tuples < n) return NULL; return ret; } int32 ts_dimension_slice_oldest_valid_chunk_for_reorder(int32 job_id, int32 dimension_id, StrategyNumber start_strategy, int64 start_value, StrategyNumber end_strategy, int64 end_value) { int32 result_chunk_id = -1; ScanIterator it = ts_dimension_slice_scan_iterator_create(NULL, CurrentMemoryContext); bool done = false; ts_dimension_slice_scan_iterator_set_range(&it, dimension_id, start_strategy, start_value, end_strategy, end_value); ts_scan_iterator_start_scan(&it); while (!done) { const TupleInfo *ti = ts_scan_iterator_next(&it); ListCell *lc; DimensionSlice *slice; List *chunk_ids = NIL; if (NULL == ti) break; slice = dimension_slice_from_slot(ti->slot); ts_chunk_constraint_scan_by_dimension_slice_to_list(slice, &chunk_ids, CurrentMemoryContext); foreach (lc, chunk_ids) { /* Look for a chunk that a) doesn't have a job stat (reorder ) and b) is not compressed * (should not reorder a compressed chunk) */ int32 chunk_id = lfirst_int(lc); BgwPolicyChunkStats *chunk_stat = ts_bgw_policy_chunk_stats_find(job_id, chunk_id); if ((chunk_stat == NULL || chunk_stat->fd.num_times_job_run == 0) && ts_chunk_get_compression_status(chunk_id) == CHUNK_COMPRESS_NONE) { /* Save the chunk_id */ result_chunk_id = chunk_id; done = true; break; } } } ts_scan_iterator_close(&it); return result_chunk_id; } List * ts_dimension_slice_get_chunkids_to_compress(int32 dimension_id, StrategyNumber start_strategy, int64 start_value, StrategyNumber end_strategy, int64 end_value, bool compress, bool recompress, int32 numchunks) { List *chunk_ids = NIL; int32 maxchunks = numchunks > 0 ? numchunks : -1; ScanIterator it = ts_dimension_slice_scan_iterator_create(NULL, CurrentMemoryContext); bool done = false; ts_dimension_slice_scan_iterator_set_range(&it, dimension_id, start_strategy, start_value, end_strategy, end_value); ts_scan_iterator_start_scan(&it); while (!done) { DimensionSlice *slice; TupleInfo *ti; ListCell *lc; List *slice_chunk_ids = NIL; ti = ts_scan_iterator_next(&it); if (NULL == ti) break; slice = dimension_slice_from_slot(ti->slot); ts_chunk_constraint_scan_by_dimension_slice_to_list(slice, &slice_chunk_ids, CurrentMemoryContext); foreach (lc, slice_chunk_ids) { int32 chunk_id = lfirst_int(lc); ChunkCompressionStatus st = ts_chunk_get_compression_status(chunk_id); if ((compress && st == CHUNK_COMPRESS_NONE) || (recompress && st == CHUNK_COMPRESS_UNORDERED)) { /* found a chunk that is not compressed or needs recompress * caller needs to check the correct chunk status */ chunk_ids = lappend_int(chunk_ids, chunk_id); if (maxchunks > 0 && list_length(chunk_ids) >= maxchunks) { done = true; break; } } } } ts_scan_iterator_close(&it); return chunk_ids; } /* This function checks for overlap between the range we want to update for the OSM chunk and the chunks currently in timescaledb (not managed by OSM) */ bool ts_osm_chunk_range_overlaps(int32 osm_dimension_slice_id, int32 dimension_id, int64 range_start, int64 range_end) { bool res; DimensionVec *vec = dimension_slice_collision_scan(dimension_id, range_start, range_end); /* there is only one dimension slice for the OSM chunk. The OSM chunk may not * necessarily appear in the list of overlapping ranges because when first tiered, * it is given a range [max, infinity) */ if (vec->num_slices >= 2 || (vec->num_slices == 1 && vec->slices[0]->fd.id != osm_dimension_slice_id)) res = true; else res = false; pfree(vec); return res; } int ts_dimension_slice_range_update(DimensionSlice *slice) { FormData_dimension_slice form; ItemPointerData tid; /* lock the tuple entry in the catalog table */ bool found = lock_dimension_slice_tuple(slice->fd.id, &tid, &form); Ensure(found, "hypertable id %d not found", slice->fd.id); if (form.range_start != slice->fd.range_start || form.range_end != slice->fd.range_end) { form.range_start = slice->fd.range_start; form.range_end = slice->fd.range_end; dimension_slice_update_catalog_tuple(&tid, &form); } return true; } ================================================ FILE: src/dimension_slice.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/pg_list.h> #include "chunk_constraint.h" /* Put DIMENSION_SLICE_MAXVALUE point in same slice as DIMENSION_SLICE_MAXVALUE-1, always */ /* This avoids the problem with coord < range_end where coord and range_end is an int64 */ #define REMAP_LAST_COORDINATE(coord) \ (((coord) == DIMENSION_SLICE_MAXVALUE) ? DIMENSION_SLICE_MAXVALUE - 1 : (coord)) #define DIMENSION_SLICE_MAXVALUE ((int64) PG_INT64_MAX) #define DIMENSION_SLICE_MINVALUE ((int64) PG_INT64_MIN) /* partition functions return int32 */ #define DIMENSION_SLICE_CLOSED_MAX ((int64) PG_INT32_MAX) #define VALUE_GT(v1, v2) ((v1) > (v2)) #define VALUE_LT(v1, v2) ((v1) < (v2)) /* * Compare two values, returning -1, 1, 0 if the left one is, less, greater, * or equal to the right one, respectively. */ #define VALUE_CMP(v1, v2) VALUE_GT(v1, v2) - VALUE_LT(v1, v2) /* Compare the range start values of two slices */ #define DIMENSION_SLICE_RANGE_START_CMP(s1, s2) \ VALUE_CMP((s1)->fd.range_start, (s2)->fd.range_start) /* Compare the range end values of two slices */ #define DIMENSION_SLICE_RANGE_END_CMP(s1, s2) VALUE_CMP((s1)->fd.range_end, (s2)->fd.range_end) typedef struct DimensionSlice { FormData_dimension_slice fd; void (*storage_free)(void *); void *storage; } DimensionSlice; typedef struct DimensionVec DimensionVec; typedef struct Hypercube Hypercube; extern DimensionVec *ts_dimension_slice_scan_limit(int32 dimension_id, int64 coordinate, int limit, const ScanTupLock *slice_lock); extern void ts_dimension_slice_scan_list(int32 dimension_id, int64 coordinate, List **matching_dimension_slices, const ScanTupLock *slice_lock); extern DimensionVec * ts_dimension_slice_scan_range_limit(int32 dimension_id, StrategyNumber start_strategy, int64 start_value, StrategyNumber end_strategy, int64 end_value, int limit, const ScanTupLock *tuplock); extern DimensionVec *ts_dimension_slice_collision_scan_limit(int32 dimension_id, int64 range_start, int64 range_end, int limit); extern TSDLLEXPORT bool ts_dimension_slice_scan_for_existing(const DimensionSlice *slice, const ScanTupLock *tuplock); extern DimensionSlice *ts_dimension_slice_scan_by_id_and_lock(int32 dimension_slice_id, const ScanTupLock *tuplock, MemoryContext mctx, LOCKMODE lockmode); extern DimensionVec *ts_dimension_slice_scan_by_dimension(int32 dimension_id, int limit); extern DimensionVec *ts_dimension_slice_scan_by_dimension_before_point(int32 dimension_id, int64 point, int limit, ScanDirection scandir, MemoryContext mctx); extern int ts_dimension_slice_delete_by_dimension_id(int32 dimension_id, bool delete_constraints); extern TSDLLEXPORT int ts_dimension_slice_delete_by_id(int32 dimension_slice_id, bool delete_constraints); extern TSDLLEXPORT DimensionSlice *ts_dimension_slice_create(int dimension_id, int64 range_start, int64 range_end); extern TSDLLEXPORT DimensionSlice *ts_dimension_slice_copy(const DimensionSlice *original); extern TSDLLEXPORT bool ts_dimension_slices_collide(const DimensionSlice *slice1, const DimensionSlice *slice2); extern TSDLLEXPORT bool ts_dimension_slices_equal(const DimensionSlice *slice1, const DimensionSlice *slice2); extern bool ts_dimension_slice_cut(DimensionSlice *to_cut, const DimensionSlice *other, int64 coord); extern void ts_dimension_slice_free(DimensionSlice *slice); extern int ts_dimension_slice_insert_multi(DimensionSlice **slice, Size num_slices); extern TSDLLEXPORT void ts_dimension_slice_insert(DimensionSlice *slice); extern int ts_dimension_slice_cmp(const DimensionSlice *left, const DimensionSlice *right); extern int ts_dimension_slice_cmp_coordinate(const DimensionSlice *slice, int64 coord); extern TSDLLEXPORT DimensionSlice *ts_dimension_slice_nth_latest_slice(int32 dimension_id, int n); extern TSDLLEXPORT DimensionSlice *ts_dimension_slice_nth_earliest_slice(int32 dimension_id, int n); extern TSDLLEXPORT int32 ts_dimension_slice_oldest_valid_chunk_for_reorder( int32 job_id, int32 dimension_id, StrategyNumber start_strategy, int64 start_value, StrategyNumber end_strategy, int64 end_value); extern TSDLLEXPORT List *ts_dimension_slice_get_chunkids_to_compress( int32 dimension_id, StrategyNumber start_strategy, int64 start_value, StrategyNumber end_strategy, int64 end_value, bool compress, bool recompress, int32 numchunks); extern DimensionSlice *ts_dimension_slice_from_tuple(TupleInfo *ti); extern ScanIterator ts_dimension_slice_scan_iterator_create(const ScanTupLock *tuplock, MemoryContext result_mcxt); extern void ts_dimension_slice_scan_iterator_set_slice_id(ScanIterator *it, int32 slice_id); extern DimensionSlice *ts_dimension_slice_scan_iterator_get_by_id(ScanIterator *it, int32 slice_id); extern int ts_dimension_slice_scan_iterator_set_range(ScanIterator *it, int32 dimension_id, StrategyNumber start_strategy, int64 start_value, StrategyNumber end_strategy, int64 end_value); extern bool ts_osm_chunk_range_overlaps(int32 osm_dimension_slice_id, int32 dimension_id, int64 range_start, int64 range_end); extern int ts_dimension_slice_range_update(DimensionSlice *slice); #define dimension_slice_insert(slice) ts_dimension_slice_insert_multi(&(slice), 1) #define dimension_slice_scan(dimension_id, coordinate, tuplock) \ ts_dimension_slice_scan_limit(dimension_id, coordinate, 0, tuplock) #define dimension_slice_collision_scan(dimension_id, range_start, range_end) \ ts_dimension_slice_collision_scan_limit(dimension_id, range_start, range_end, 0) DimensionSlice *ts_chunk_get_osm_slice_and_lock(int32 osm_chunk_id, int32 time_dim_id, LockTupleMode tuplockmode, LOCKMODE tablelockmode); ================================================ FILE: src/dimension_vector.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include "dimension_vector.h" static int cmp_slices(const void *left, const void *right) { const DimensionSlice *left_slice = *((DimensionSlice **) left); const DimensionSlice *right_slice = *((DimensionSlice **) right); return ts_dimension_slice_cmp(left_slice, right_slice); } static int cmp_coordinate_and_slice(const void *left, const void *right) { int64 coord = *((int64 *) left); const DimensionSlice *slice = *((DimensionSlice **) right); return ts_dimension_slice_cmp_coordinate(slice, coord); } static DimensionVec * dimension_vec_expand(DimensionVec *vec, int32 new_capacity) { if (vec != NULL && vec->capacity >= new_capacity) return vec; if (NULL == vec) vec = palloc(DIMENSION_VEC_SIZE(new_capacity)); else vec = repalloc(vec, DIMENSION_VEC_SIZE(new_capacity)); vec->capacity = new_capacity; return vec; } DimensionVec * ts_dimension_vec_create(int32 initial_num_slices) { DimensionVec *vec = dimension_vec_expand(NULL, initial_num_slices); vec->capacity = initial_num_slices; vec->num_slices = 0; return vec; } DimensionVec * ts_dimension_vec_sort(DimensionVec **vecptr) { DimensionVec *vec = *vecptr; if (vec->num_slices > 1) qsort((void *) vec->slices, vec->num_slices, sizeof(DimensionSlice *), cmp_slices); return vec; } DimensionVec * ts_dimension_vec_add_slice(DimensionVec **vecptr, DimensionSlice *slice) { DimensionVec *vec = *vecptr; /* Ensure consistent dimension */ Assert(vec->num_slices == 0 || vec->slices[0]->fd.dimension_id == slice->fd.dimension_id); if (vec->num_slices + 1 > vec->capacity) *vecptr = vec = dimension_vec_expand(vec, vec->capacity + 10); vec->slices[vec->num_slices++] = slice; return vec; } DimensionVec * ts_dimension_vec_add_unique_slice(DimensionVec **vecptr, DimensionSlice *slice) { DimensionVec *vec = *vecptr; int32 existing_slice_index = ts_dimension_vec_find_slice_index(vec, slice->fd.id); if (existing_slice_index == -1) return ts_dimension_vec_add_slice(vecptr, slice); return vec; } DimensionVec * ts_dimension_vec_add_slice_sort(DimensionVec **vecptr, DimensionSlice *slice) { *vecptr = ts_dimension_vec_add_slice(vecptr, slice); return ts_dimension_vec_sort(vecptr); } void ts_dimension_vec_remove_slice(DimensionVec **vecptr, int32 index) { DimensionVec *vec = *vecptr; ts_dimension_slice_free(vec->slices[index]); memmove((void *) &vec->slices[index], (void *) &vec->slices[index + 1], sizeof(DimensionSlice *) * (vec->num_slices - index - 1)); vec->num_slices--; } #if defined(USE_ASSERT_CHECKING) static inline bool dimension_vec_is_sorted(const DimensionVec *vec) { int i; if (vec->num_slices < 2) return true; for (i = 1; i < vec->num_slices; i++) if (cmp_slices((void *) &vec->slices[i - 1], (void *) &vec->slices[i]) > 0) return false; return true; } #endif DimensionSlice * ts_dimension_vec_find_slice(const DimensionVec *vec, int64 coordinate) { DimensionSlice **res; if (vec->num_slices == 0) return NULL; Assert(dimension_vec_is_sorted(vec)); res = (DimensionSlice **) bsearch(&coordinate, (void *) vec->slices, vec->num_slices, sizeof(DimensionSlice *), cmp_coordinate_and_slice); if (res == NULL) return NULL; return *res; } int ts_dimension_vec_find_slice_index(const DimensionVec *vec, int32 dimension_slice_id) { int i; for (i = 0; i < vec->num_slices; i++) if (dimension_slice_id == vec->slices[i]->fd.id) return i; return -1; } const DimensionSlice * ts_dimension_vec_get(const DimensionVec *vec, int32 index) { if (index >= vec->num_slices) return NULL; return vec->slices[index]; } void ts_dimension_vec_free(DimensionVec *vec) { int i; for (i = 0; i < vec->num_slices; i++) ts_dimension_slice_free(vec->slices[i]); pfree(vec); } ================================================ FILE: src/dimension_vector.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include "dimension_slice.h" #include "hypertable_restrict_info.h" /* * DimensionVec is a collection of slices (ranges) along one dimension for a * time range. */ typedef struct DimensionVec { int32 capacity; /* The capacity of the slices array */ int32 num_slices; /* The current number of slices in slices * array */ DimensionRestrictInfo *dri; /* corresponding restrictinfo */ DimensionSlice *slices[FLEXIBLE_ARRAY_MEMBER]; } DimensionVec; #define DIMENSION_VEC_SIZE(num_slices) \ (sizeof(DimensionVec) + (sizeof(DimensionSlice *) * num_slices)) #define DIMENSION_VEC_DEFAULT_SIZE 10 extern DimensionVec *ts_dimension_vec_create(int32 initial_num_slices); extern DimensionVec *ts_dimension_vec_sort(DimensionVec **vec); extern DimensionVec *ts_dimension_vec_add_slice_sort(DimensionVec **vec, DimensionSlice *slice); extern DimensionVec *ts_dimension_vec_add_slice(DimensionVec **vecptr, DimensionSlice *slice); extern DimensionVec *ts_dimension_vec_add_unique_slice(DimensionVec **vecptr, DimensionSlice *slice); extern void ts_dimension_vec_remove_slice(DimensionVec **vecptr, int32 index); extern DimensionSlice *ts_dimension_vec_find_slice(const DimensionVec *vec, int64 coordinate); extern int ts_dimension_vec_find_slice_index(const DimensionVec *vec, int32 dimension_slice_id); extern const DimensionSlice *ts_dimension_vec_get(const DimensionVec *vec, int32 index); extern void ts_dimension_vec_free(DimensionVec *vec); ================================================ FILE: src/error_utils.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #define GETARG_NOTNULL_OID(var, arg, name) \ { \ var = PG_ARGISNULL(arg) ? InvalidOid : PG_GETARG_OID(arg); \ if (!OidIsValid(var)) \ ereport(ERROR, \ (errcode(ERRCODE_INVALID_PARAMETER_VALUE), \ errmsg("%s cannot be NULL", name))); \ } #define GETARG_NOTNULL_POINTER(var, arg, name, type) \ { \ if (PG_ARGISNULL(arg)) \ ereport(ERROR, \ (errcode(ERRCODE_INVALID_PARAMETER_VALUE), \ errmsg("%s cannot be NULL", name))); \ var = (type *) PG_GETARG_POINTER(arg); \ } #define GETARG_NOTNULL_NULLABLE(var, arg, name, type) \ { \ if (PG_ARGISNULL(arg)) \ ereport(ERROR, \ (errcode(ERRCODE_INVALID_PARAMETER_VALUE), \ errmsg("%s cannot be NULL", name))); \ var = PG_GETARG_##type(arg); \ } ================================================ FILE: src/errors.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once /* Defines error codes used -- PREFIX TS */ /* -- TS000 - GROUP: query errors -- TS001 - hypertable does not exist -- TS002 - column does not exist */ #define ERRCODE_TS_QUERY_ERRORS MAKE_SQLSTATE('T', 'S', '0', '0', '0') #define ERRCODE_TS_HYPERTABLE_NOT_EXIST MAKE_SQLSTATE('T', 'S', '0', '0', '1') #define ERRCODE_TS_DIMENSION_NOT_EXIST MAKE_SQLSTATE('T', 'S', '0', '0', '2') #define ERRCODE_TS_CHUNK_NOT_EXIST MAKE_SQLSTATE('T', 'S', '0', '0', '3') /* --TS100 - GROUP: DDL errors --TS101 - operation not supported --TS102 - bad hypertable definition --TS103 - bad hypertable index definition --TS110 - hypertable already exists --TS120 - node already exists --TS130 - user already exists --TS140 - tablespace already attached --TS150 - tablespace not attached --TS160 - duplicate dimension */ #define ERRCODE_TS_DDL_ERRORS MAKE_SQLSTATE('T', 'S', '1', '0', '0') #define ERRCODE_TS_OPERATION_NOT_SUPPORTED MAKE_SQLSTATE('T', 'S', '1', '0', '1') #define ERRCODE_TS_BAD_HYPERTABLE_DEFINITION MAKE_SQLSTATE('T', 'S', '1', '0', '2') #define ERRCODE_TS_BAD_HYPERTABLE_INDEX_DEFINITION MAKE_SQLSTATE('T', 'S', '1', '0', '3') #define ERRCODE_TS_HYPERTABLE_EXISTS MAKE_SQLSTATE('T', 'S', '1', '1', '0') #define ERRCODE_TS_NODE_EXISTS MAKE_SQLSTATE('T', 'S', '1', '2', '0') #define ERRCODE_TS_USER_EXISTS MAKE_SQLSTATE('T', 'S', '1', '3', '0') #define ERRCODE_TS_TABLESPACE_ALREADY_ATTACHED MAKE_SQLSTATE('T', 'S', '1', '4', '0') #define ERRCODE_TS_TABLESPACE_NOT_ATTACHED MAKE_SQLSTATE('T', 'S', '1', '5', '0') #define ERRCODE_TS_DUPLICATE_DIMENSION MAKE_SQLSTATE('T', 'S', '1', '6', '0') #define ERRCODE_TS_INSUFFICIENT_NUM_DATA_NODES MAKE_SQLSTATE('T', 'S', '1', '7', '0') /* --IO500 - GROUP: internal error --IO501 - unexpected state/event */ #define ERRCODE_TS_UNEXPECTED MAKE_SQLSTATE('T', 'S', '5', '0', '1') #define ERRCODE_TS_CHUNK_COLLISION MAKE_SQLSTATE('T', 'S', '5', '0', '3') ================================================ FILE: src/estimate.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_type.h> #include <optimizer/optimizer.h> #include <parser/parse_oper.h> #include <utils/selfuncs.h> #include "compat/compat.h" #include "estimate.h" #include "func_cache.h" #include "import/planner.h" #include "utils.h" /* * This module contains functions for estimating, e.g., the number of groups * formed in various grouping expressions that involve time bucketing. */ static double estimate_max_spread_expr(PlannerInfo *root, Expr *expr); static double group_estimate_opexpr(PlannerInfo *root, OpExpr *opexpr, double path_rows); /* Estimate the max spread on a time var in terms of the internal time representation. * Note that this will happen on the hypertable var in most cases. Therefore this is * a huge overestimate in many cases where there is a WHERE clause on time. */ static double estimate_max_spread_var(PlannerInfo *root, Var *var) { VariableStatData vardata; Oid ltop; Datum max_datum, min_datum; int64 max, min; bool valid; examine_variable(root, (Node *) var, 0, &vardata); get_sort_group_operators(var->vartype, true, false, false, <op, NULL, NULL, NULL); valid = ts_get_variable_range(root, &vardata, ltop, &min_datum, &max_datum); ReleaseVariableStats(vardata); if (!valid) return INVALID_ESTIMATE; max = ts_time_value_to_internal(max_datum, var->vartype); min = ts_time_value_to_internal(min_datum, var->vartype); return (double) (max - min); } static double estimate_max_spread_opexpr(PlannerInfo *root, OpExpr *opexpr) { char *function_name = get_opname(opexpr->opno); Expr *left; Expr *right; Expr *nonconst; if (list_length(opexpr->args) != 2 || strlen(function_name) != 1) return INVALID_ESTIMATE; left = linitial(opexpr->args); right = lsecond(opexpr->args); if (IsA(left, Const)) nonconst = right; else if (IsA(right, Const)) nonconst = left; else return INVALID_ESTIMATE; /* adding or subtracting a constant doesn't affect the range */ if (function_name[0] == '-' || function_name[0] == '+') return estimate_max_spread_expr(root, nonconst); return INVALID_ESTIMATE; } /* estimate the max spread (max(value)-min(value)) of the expr */ static double estimate_max_spread_expr(PlannerInfo *root, Expr *expr) { switch (nodeTag(expr)) { case T_Var: return estimate_max_spread_var(root, (Var *) expr); case T_OpExpr: return estimate_max_spread_opexpr(root, (OpExpr *) expr); default: return INVALID_ESTIMATE; } } /* * Return an estimate for the number of groups formed when expr is divided * into intervals of size interval_period. */ double ts_estimate_group_expr_interval(PlannerInfo *root, Expr *expr, double interval_period) { double max_period; if (interval_period <= 0) return INVALID_ESTIMATE; max_period = estimate_max_spread_expr(root, expr); if (!IS_VALID_ESTIMATE(max_period)) return INVALID_ESTIMATE; return clamp_row_est(max_period / interval_period); } /* if performing integer division number of groups is less than the spread divided by the divisor. * Note that this is an overestimate. */ static double group_estimate_integer_division(PlannerInfo *root, Oid opno, Node *left, Node *right) { char *function_name = get_opname(opno); /* only handle division */ if (function_name[0] == '/' && function_name[1] == '\0' && IsA(right, Const)) { Const *c = (Const *) right; if (c->consttype != INT2OID && c->consttype != INT4OID && c->consttype != INT8OID) return INVALID_ESTIMATE; return ts_estimate_group_expr_interval(root, (Expr *) left, (double) c->constvalue); } return INVALID_ESTIMATE; } static double group_estimate_funcexpr(PlannerInfo *root, FuncExpr *group_estimate_func, double path_rows) { FuncInfo *func_est = ts_func_cache_get_bucketing_func(group_estimate_func->funcid); if (func_est && func_est->group_estimate) return func_est->group_estimate(root, group_estimate_func, path_rows); return INVALID_ESTIMATE; } /* Get a custom estimate for the number of groups of an expression. Return INVALID_ESTIMATE if we * don't have any extra knowledge and should just use the default estimate */ static double group_estimate_expr(PlannerInfo *root, Node *expr, double path_rows) { switch (nodeTag(expr)) { case T_FuncExpr: return group_estimate_funcexpr(root, (FuncExpr *) expr, path_rows); case T_OpExpr: return group_estimate_opexpr(root, (OpExpr *) expr, path_rows); default: return INVALID_ESTIMATE; } } static double group_estimate_opexpr(PlannerInfo *root, OpExpr *opexpr, double path_rows) { Node *first; Node *second; double estimate; if (list_length(opexpr->args) != 2) return INVALID_ESTIMATE; first = eval_const_expressions(root, linitial(opexpr->args)); second = eval_const_expressions(root, lsecond(opexpr->args)); estimate = group_estimate_integer_division(root, opexpr->opno, first, second); if (IS_VALID_ESTIMATE(estimate)) return estimate; if (IsA(first, Const)) return group_estimate_expr(root, second, path_rows); if (IsA(second, Const)) return group_estimate_expr(root, first, path_rows); return INVALID_ESTIMATE; } /* * Get a custom estimate for the number of groups in a query. Return * INVALID_ESTIMATE if we don't have any extra knowledge and should just use * the default estimate. This works by getting a custom estimate for any * groups where a custom estimate exists and multiplying that by the standard * estimate of the groups for which custom estimates don't exist. */ double ts_estimate_group(PlannerInfo *root, double path_rows) { Query *parse = root->parse; double d_num_groups = 1; List *group_exprs; ListCell *lc; bool found = false; List *new_group_expr = NIL; Assert(parse->groupClause && !parse->groupingSets); group_exprs = get_sortgrouplist_exprs(parse->groupClause, parse->targetList); foreach (lc, group_exprs) { Node *item = lfirst(lc); double estimate = group_estimate_expr(root, item, path_rows); if (IS_VALID_ESTIMATE(estimate)) { found = true; d_num_groups *= estimate; } else new_group_expr = lappend(new_group_expr, item); } /* nothing custom */ if (!found) return INVALID_ESTIMATE; /* multiply by default estimates */ if (new_group_expr != NIL) d_num_groups *= estimate_num_groups(root, new_group_expr, path_rows, NULL, NULL); if (d_num_groups > path_rows) return INVALID_ESTIMATE; return clamp_row_est(d_num_groups); } ================================================ FILE: src/estimate.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #define INVALID_ESTIMATE (-1) #define IS_VALID_ESTIMATE(est) ((est) >= 0) extern double ts_estimate_group_expr_interval(PlannerInfo *root, Expr *expr, double interval_period); extern double ts_estimate_group(PlannerInfo *root, double path_rows); ================================================ FILE: src/event_trigger.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/htup_details.h> #include <catalog/pg_constraint.h> #include <catalog/pg_foreign_server.h> #include <catalog/pg_namespace.h> #include <catalog/pg_trigger.h> #include <commands/event_trigger.h> #include <executor/executor.h> #include <utils/array.h> #include <utils/builtins.h> #include "compat/compat.h" #include "event_trigger.h" #define DDL_INFO_NATTS 9 #define DROPPED_OBJECTS_NATTS 12 /* Function manager info for the event "pg_event_trigger_ddl_commands", which is * used to retrieve information on executed DDL commands in an event * trigger. The function manager info is initialized on extension load. */ static FmgrInfo ddl_commands_fmgrinfo; static FmgrInfo dropped_objects_fmgrinfo; /* * Get a list of executed DDL commands in an event trigger. * * This function calls the function pg_ts_event_trigger_ddl_commands(), which is * part of the event trigger API, and retrieves the DDL commands executed in * relation to the event trigger. It is only valid to call this function from * within an event trigger. */ List * ts_event_trigger_ddl_commands(void) { ReturnSetInfo rsinfo; LOCAL_FCINFO(fcinfo, 1); TupleTableSlot *slot; EState *estate = CreateExecutorState(); List *objects = NIL; InitFunctionCallInfoData(*fcinfo, &ddl_commands_fmgrinfo, 1, InvalidOid, NULL, NULL); MemSet(&rsinfo, 0, sizeof(rsinfo)); rsinfo.type = T_ReturnSetInfo; rsinfo.allowedModes = SFRM_Materialize; rsinfo.econtext = CreateExprContext(estate); FC_SET_NULL(fcinfo, 0); fcinfo->resultinfo = (fmNodePtr) &rsinfo; FunctionCallInvoke(fcinfo); slot = MakeSingleTupleTableSlot(rsinfo.setDesc, &TTSOpsMinimalTuple); while (tuplestore_gettupleslot(rsinfo.setResult, true, false, slot)) { bool should_free; HeapTuple tuple = ExecFetchSlotHeapTuple(slot, false, &should_free); CollectedCommand *cmd; Datum values[DDL_INFO_NATTS]; bool nulls[DDL_INFO_NATTS]; heap_deform_tuple(tuple, rsinfo.setDesc, values, nulls); if (should_free) heap_freetuple(tuple); if (rsinfo.setDesc->natts > 8 && !nulls[8]) { cmd = (CollectedCommand *) DatumGetPointer(values[8]); objects = lappend(objects, cmd); } } ExecDropSingleTupleTableSlot(slot); FreeExprContext(rsinfo.econtext, false); FreeExecutorState(estate); return objects; } /* Given a TEXT[] of addrnames return a list of heap allocated char * * * similar to textarray_to_strvaluelist */ static List * extract_addrnames(ArrayType *arr) { Datum *elems; bool *nulls; int nelems; List *list = NIL; int i; deconstruct_array(arr, TEXTOID, -1, false, TYPALIGN_INT, &elems, &nulls, &nelems); for (i = 0; i < nelems; i++) { if (nulls[i]) elog(ERROR, "unexpected NULL in name list"); /* TextDatumGetCString heap allocates the string */ list = lappend(list, TextDatumGetCString(elems[i])); } return list; } static EventTriggerDropTableConstraint * make_event_trigger_drop_table_constraint(const char *constraint_name, const char *schema, const char *table) { EventTriggerDropTableConstraint *obj = palloc(sizeof(EventTriggerDropTableConstraint)); *obj = (EventTriggerDropTableConstraint){ .obj = { .type = EVENT_TRIGGER_DROP_TABLE_CONSTRAINT, }, .constraint_name = constraint_name, .schema = schema, .table = table, }; return obj; } static EventTriggerDropRelation * make_event_trigger_drop_index(const char *index_name, const char *schema) { EventTriggerDropRelation *obj = palloc(sizeof(EventTriggerDropRelation)); *obj = (EventTriggerDropRelation){ .obj = { .type = EVENT_TRIGGER_DROP_INDEX, }, .name = index_name, .schema = schema, }; return obj; } static EventTriggerDropRelation * make_event_trigger_drop_table(Oid relid, const char *table_name, const char *schema, char relkind) { EventTriggerDropRelation *obj = palloc(sizeof(EventTriggerDropRelation)); *obj = (EventTriggerDropRelation){ .obj = { .type = (relkind == RELKIND_RELATION) ? EVENT_TRIGGER_DROP_TABLE : EVENT_TRIGGER_DROP_FOREIGN_TABLE, }, .relid = relid, .name = table_name, .schema = schema, }; return obj; } static EventTriggerDropView * make_event_trigger_drop_view(char *view_name, char *schema) { EventTriggerDropView *obj = palloc(sizeof(*obj)); *obj = (EventTriggerDropView){ .obj = { .type = EVENT_TRIGGER_DROP_VIEW }, .view_name = view_name, .schema = schema, }; return obj; } static EventTriggerDropSchema * make_event_trigger_drop_schema(const char *schema) { EventTriggerDropSchema *obj = palloc(sizeof(EventTriggerDropSchema)); *obj = (EventTriggerDropSchema){ .obj = { .type = EVENT_TRIGGER_DROP_SCHEMA, }, .schema = schema, }; return obj; } static EventTriggerDropTrigger * make_event_trigger_drop_trigger(const char *trigger_name, const char *schema, const char *table) { EventTriggerDropTrigger *obj = palloc(sizeof(EventTriggerDropTrigger)); *obj = (EventTriggerDropTrigger){ .obj = { .type = EVENT_TRIGGER_DROP_TRIGGER, }, .trigger_name = trigger_name, .schema = schema, .table = table }; return obj; } static EventTriggerDropForeignServer * make_event_trigger_drop_foreign_server(const char *server_name) { EventTriggerDropForeignServer *obj = palloc(sizeof(EventTriggerDropForeignServer)); *obj = (EventTriggerDropForeignServer){ .obj = { .type = EVENT_TRIGGER_DROP_FOREIGN_SERVER, }, .servername = server_name, }; return obj; } List * ts_event_trigger_dropped_objects(void) { ReturnSetInfo rsinfo; LOCAL_FCINFO(fcinfo, 0); TupleTableSlot *slot; EState *estate = CreateExecutorState(); List *objects = NIL; InitFunctionCallInfoData(*fcinfo, &dropped_objects_fmgrinfo, 0, InvalidOid, NULL, NULL); MemSet(&rsinfo, 0, sizeof(rsinfo)); rsinfo.type = T_ReturnSetInfo; rsinfo.allowedModes = SFRM_Materialize; rsinfo.econtext = CreateExprContext(estate); fcinfo->resultinfo = (fmNodePtr) &rsinfo; FunctionCallInvoke(fcinfo); slot = MakeSingleTupleTableSlot(rsinfo.setDesc, &TTSOpsMinimalTuple); while (tuplestore_gettupleslot(rsinfo.setResult, true, false, slot)) { bool should_free; HeapTuple tuple = ExecFetchSlotHeapTuple(slot, false, &should_free); Datum values[DROPPED_OBJECTS_NATTS]; bool nulls[DROPPED_OBJECTS_NATTS]; Oid class_id; char *objtype; List *addrnames = NIL; void *eventobj = NULL; heap_deform_tuple(tuple, rsinfo.setDesc, values, nulls); class_id = DatumGetObjectId(values[0]); switch (class_id) { case ConstraintRelationId: objtype = TextDatumGetCString(values[6]); if (objtype != NULL && strcmp(objtype, "table constraint") == 0) { addrnames = extract_addrnames(DatumGetArrayTypeP(values[10])); eventobj = make_event_trigger_drop_table_constraint(lthird(addrnames), linitial(addrnames), lsecond(addrnames)); } break; case RelationRelationId: objtype = TextDatumGetCString(values[6]); if (objtype == NULL) break; addrnames = extract_addrnames(DatumGetArrayTypeP(values[10])); if (strcmp(objtype, "index") == 0) eventobj = make_event_trigger_drop_index(lsecond(addrnames), linitial(addrnames)); else if (strcmp(objtype, "table") == 0) { eventobj = make_event_trigger_drop_table(DatumGetInt32(values[1]), lsecond(addrnames), linitial(addrnames), RELKIND_RELATION); } else if (strcmp(objtype, "view") == 0) { List *addrnames = extract_addrnames(DatumGetArrayTypeP(values[10])); objects = lappend(objects, make_event_trigger_drop_view(lsecond(addrnames), linitial(addrnames))); } else if (strcmp(objtype, "foreign table") == 0) eventobj = make_event_trigger_drop_table(DatumGetInt32(values[1]), lsecond(addrnames), linitial(addrnames), RELKIND_FOREIGN_TABLE); break; case NamespaceRelationId: addrnames = extract_addrnames(DatumGetArrayTypeP(values[10])); eventobj = make_event_trigger_drop_schema(linitial(addrnames)); break; case TriggerRelationId: addrnames = extract_addrnames(DatumGetArrayTypeP(values[10])); eventobj = make_event_trigger_drop_trigger(lthird(addrnames), linitial(addrnames), lsecond(addrnames)); break; case ForeignServerRelationId: addrnames = extract_addrnames(DatumGetArrayTypeP(values[10])); eventobj = make_event_trigger_drop_foreign_server(linitial(addrnames)); break; default: break; } if (NULL != eventobj) objects = lappend(objects, eventobj); if (should_free) heap_freetuple(tuple); } ExecDropSingleTupleTableSlot(slot); FreeExprContext(rsinfo.econtext, false); FreeExecutorState(estate); return objects; } void _event_trigger_init(void) { fmgr_info(fmgr_internal_function("pg_event_trigger_ddl_commands"), &ddl_commands_fmgrinfo); fmgr_info(fmgr_internal_function("pg_event_trigger_dropped_objects"), &dropped_objects_fmgrinfo); } void _event_trigger_fini(void) { /* Nothing to do */ } ================================================ FILE: src/event_trigger.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/pg_list.h> typedef enum EventTriggerDropType { EVENT_TRIGGER_DROP_TABLE_CONSTRAINT, EVENT_TRIGGER_DROP_INDEX, EVENT_TRIGGER_DROP_TABLE, EVENT_TRIGGER_DROP_VIEW, EVENT_TRIGGER_DROP_FOREIGN_TABLE, EVENT_TRIGGER_DROP_SCHEMA, EVENT_TRIGGER_DROP_TRIGGER, EVENT_TRIGGER_DROP_FOREIGN_SERVER, } EventTriggerDropType; typedef struct EventTriggerDropObject { EventTriggerDropType type; } EventTriggerDropObject; typedef struct EventTriggerDropTableConstraint { EventTriggerDropObject obj; const char *constraint_name; const char *schema; const char *table; } EventTriggerDropTableConstraint; typedef struct EventTriggerDropRelation { EventTriggerDropObject obj; Oid relid; const char *name; const char *schema; } EventTriggerDropRelation; typedef struct EventTriggerDropView { EventTriggerDropObject obj; char *view_name; char *schema; } EventTriggerDropView; typedef struct EventTriggerDropSchema { EventTriggerDropObject obj; const char *schema; } EventTriggerDropSchema; typedef struct EventTriggerDropTrigger { EventTriggerDropObject obj; const char *trigger_name; const char *schema; const char *table; } EventTriggerDropTrigger; typedef struct EventTriggerDropForeignServer { EventTriggerDropObject obj; const char *servername; } EventTriggerDropForeignServer; extern List *ts_event_trigger_dropped_objects(void); extern List *ts_event_trigger_ddl_commands(void); extern void _event_trigger_init(void); extern void _event_trigger_fini(void); ================================================ FILE: src/export.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include "config.h" /* Definitions for symbol exports */ #if defined(_WIN32) && !defined(WIN32) #define WIN32 #endif /* * PGDLLEXPORT is defined as en empty macro until PG15. * Since PG16, a macro HAVE_VISIBILITY_ATTRIBUTE is defined if the compiler has * support for visibility attribute and the PGDLLEXPORT macro is defined as the * same. So, skip redefining PGDLLEXPORT if HAVE_VISIBILITY_ATTRIBUTE is defined. * If not, undef the empty PGDLLEXPORT macro and redefine it properly. */ #if !defined(WIN32) && !defined(__CYGWIN__) && !defined(HAVE_VISIBILITY_ATTRIBUTE) #if __GNUC__ >= 4 /* PGDLLEXPORT is defined but will be empty. Redefine it. */ #undef PGDLLEXPORT #define PGDLLEXPORT __attribute__((visibility("default"))) #else #error "Unsupported GNUC version" #endif /* __GNUC__ */ #endif /* * On windows, symbols shared across modules have to be marked "export" in the * main TimescaleDb module and "import" in the submodule. Since we want to use the * same headers, we TSDLLEXPORT functions as "export" in the main module and * "import" in submodules. */ #ifndef TS_SUBMODULE /* In the core timescaledb TSDLLEXPORT is export */ #define TSDLLEXPORT PGDLLEXPORT #elif defined(PGDLLIMPORT) /* In submodules it works as imports */ #define TSDLLEXPORT PGDLLIMPORT #else /* If there is no IMPORT defined, it's a nop */ #define TSDLLEXPORT #endif #define TS_FUNCTION_INFO_V1(fn) \ PGDLLEXPORT Datum fn(PG_FUNCTION_ARGS); \ PG_FUNCTION_INFO_V1(fn) ================================================ FILE: src/expression_utils.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_type.h> #include <nodes/extensible.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <nodes/plannodes.h> #include <nodes/primnodes.h> #include <utils/lsyscache.h> #include "debug_assert.h" #include "export.h" #include "expression_utils.h" /* * This function is meant to extract the expression components to be used in a ScanKey. * * It will work on the following expression types: * - Var OP Expr * * Var OP Var is not supported as that will not work with scankeys. * */ bool TSDLLEXPORT ts_extract_expr_args(Expr *expr, Var **var, Expr **arg_value, Oid *opno, Oid *opcode) { List *args; Oid expr_opno, expr_opcode; switch (nodeTag(expr)) { case T_OpExpr: { OpExpr *opexpr = castNode(OpExpr, expr); args = opexpr->args; expr_opno = opexpr->opno; expr_opcode = opexpr->opfuncid; if (opexpr->opresulttype != BOOLOID) return false; break; } case T_ScalarArrayOpExpr: { ScalarArrayOpExpr *sa_opexpr = castNode(ScalarArrayOpExpr, expr); args = sa_opexpr->args; expr_opno = sa_opexpr->opno; expr_opcode = sa_opexpr->opfuncid; break; } default: return false; } if (list_length(args) != 2) return false; Expr *leftop = linitial(args); Expr *rightop = lsecond(args); if (IsA(leftop, RelabelType)) leftop = castNode(RelabelType, leftop)->arg; if (IsA(rightop, RelabelType)) rightop = castNode(RelabelType, rightop)->arg; if (IsA(leftop, Var) && !IsA(rightop, Var)) { /* ignore system columns */ if (castNode(Var, leftop)->varattno <= 0) return false; *var = castNode(Var, leftop); *arg_value = rightop; *opno = expr_opno; if (opcode) *opcode = expr_opcode; return true; } else if (IsA(rightop, Var) && !IsA(leftop, Var)) { /* ignore system columns */ if (castNode(Var, rightop)->varattno <= 0) return false; *var = castNode(Var, rightop); *arg_value = leftop; expr_opno = get_commutator(expr_opno); if (!OidIsValid(expr_opno)) return false; if (opcode) { expr_opcode = get_opcode(expr_opno); if (!OidIsValid(expr_opcode)) return false; *opcode = expr_opcode; } *opno = expr_opno; return true; } return false; } /* * Build an output targetlist for a custom node that just references all the * custom scan targetlist entries. */ List * ts_build_trivial_custom_output_targetlist(List *scan_targetlist) { List *result = NIL; ListCell *lc; foreach (lc, scan_targetlist) { TargetEntry *scan_entry = (TargetEntry *) lfirst(lc); Var *var = makeVar(INDEX_VAR, scan_entry->resno, exprType((Node *) scan_entry->expr), exprTypmod((Node *) scan_entry->expr), exprCollation((Node *) scan_entry->expr), /* varlevelsup = */ 0); TargetEntry *output_entry = makeTargetEntry((Expr *) var, scan_entry->resno, scan_entry->resname, scan_entry->resjunk); result = lappend(result, output_entry); } return result; } static Node * resolve_outer_special_vars_mutator(Node *node, void *context) { if (node == NULL) { return NULL; } if (!IsA(node, Var)) { return expression_tree_mutator(node, resolve_outer_special_vars_mutator, context); } Var *var = castNode(Var, node); CustomScan *custom = castNode(CustomScan, context); if ((Index) var->varno == (Index) custom->scan.scanrelid) { /* * This is already the uncompressed chunk var. We can see it referenced * by expressions in the output targetlist of the child scan node. */ return (Node *) copyObject(var); } if (var->varno == OUTER_VAR) { /* * Reference into the output targetlist of the child scan node. */ TargetEntry *columnar_scan_tentry = castNode(TargetEntry, list_nth(custom->scan.plan.targetlist, var->varattno - 1)); return resolve_outer_special_vars_mutator((Node *) columnar_scan_tentry->expr, context); } if (var->varno == INDEX_VAR) { /* * This is a reference into the custom scan targetlist, we have to resolve * it as well. */ var = castNode(Var, castNode(TargetEntry, list_nth(custom->custom_scan_tlist, var->varattno - 1)) ->expr); Assert(var->varno > 0); return (Node *) copyObject(var); } Ensure(false, "encountered unexpected varno %d as an aggregate argument", var->varno); return node; } /* * Walk a plan tree recursively descending into child plans. * At each leaf, call the user-supplied callback which may replace the node. */ Plan * ts_plan_tree_walker(Plan *plan, ts_plan_tree_walkerfunc func, void *context) { if (!plan) return NULL; if (IsA(plan, List)) { ListCell *lc; foreach (lc, castNode(List, plan)) { lfirst(lc) = ts_plan_tree_walker(lfirst(lc), func, context); } return plan; } if (plan->lefttree) plan->lefttree = ts_plan_tree_walker(plan->lefttree, func, context); if (plan->righttree) plan->righttree = ts_plan_tree_walker(plan->righttree, func, context); if (IsA(plan, Append)) { Append *append = castNode(Append, plan); append->appendplans = (List *) ts_plan_tree_walker((Plan *) append->appendplans, func, context); } else if (IsA(plan, MergeAppend)) { MergeAppend *append = castNode(MergeAppend, plan); append->mergeplans = (List *) ts_plan_tree_walker((Plan *) append->mergeplans, func, context); } else if (IsA(plan, CustomScan)) { CustomScan *custom = castNode(CustomScan, plan); custom->custom_plans = (List *) ts_plan_tree_walker((Plan *) custom->custom_plans, func, context); } if (IsA(plan, SubqueryScan)) { SubqueryScan *subquery = castNode(SubqueryScan, plan); subquery->subplan = ts_plan_tree_walker(subquery->subplan, func, context); return plan; } return func(plan, context); } /* * Resolve the OUTER_VAR special variables, that are used in the output * targetlists of aggregation nodes, replacing them with the uncompressed chunk * variables. */ Node * ts_resolve_outer_special_vars(Node *node, Plan *childplan) { return resolve_outer_special_vars_mutator(node, childplan); } ================================================ FILE: src/expression_utils.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <nodes/primnodes.h> #include <utils/lsyscache.h> #include "export.h" bool TSDLLEXPORT ts_extract_expr_args(Expr *expr, Var **var, Expr **arg_value, Oid *opno, Oid *opcode); TSDLLEXPORT List *ts_build_trivial_custom_output_targetlist(List *scan_targetlist); TSDLLEXPORT Node *ts_resolve_outer_special_vars(Node *node, Plan *childplan); typedef Plan *(*ts_plan_tree_walkerfunc)(Plan *, void *); extern TSDLLEXPORT Plan *ts_plan_tree_walker(Plan *plan, ts_plan_tree_walkerfunc func, void *context); ================================================ FILE: src/extension.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/transam.h> #include <access/xact.h> #include <catalog/namespace.h> #include <catalog/objectaccess.h> #include <commands/event_trigger.h> #include <fmgr.h> #include <utils/inval.h> #include <utils/lsyscache.h> #if PG_VERSION_NUM < 150000 /* * Some externs are mislabeled when building on Windows so we try to fix them * with this hack. This is only needed for versions < 15. */ #include "compat/compat-msvc-enter.h" #include <commands/extension.h> #include <miscadmin.h> #include "compat/compat-msvc-exit.h" #endif #include <access/relscan.h> #include <catalog/indexing.h> #include <catalog/pg_extension.h> #include <utils/builtins.h> #include <utils/fmgroids.h> #include "compat/compat.h" #include "extension.h" #include "extension_utils.c" #include "guc.h" #include "ts_catalog/catalog.h" #define TS_UPDATE_SCRIPT_CONFIG_VAR MAKE_EXTOPTION("update_script_stage") #define POST_UPDATE "post" /* * The name of the experimental schema. * * Call ts_extension_schema_name() or ts_experimental_schema_name() for * consistency. Don't use this macro directly. */ #define TS_EXPERIMENTAL_SCHEMA_NAME "timescaledb_experimental" static Oid extension_proxy_oid = InvalidOid; /* * ExtensionState tracks the state of extension metadata in the backend. * * Since we want to cache extension metadata to speed up common checks (e.g., * check for presence of the extension itself), we also need to track the * extension state to know when the metadata is valid. * * We use a proxy_table to be notified of extension drops/creates. Namely, * we rely on the fact that postgres will internally create RelCacheInvalidation * events when any tables are created or dropped. We rely on the following properties * of Postgres's dependency management: * * The proxy table will be created before the extension itself. * * The proxy table will be dropped before the extension itself. */ static enum ExtensionState extstate = EXTENSION_STATE_UNKNOWN; /* * Looking up the extension oid is a catalog lookup that can be costly, and we * often need it during the planning, so we cache it here. We update it when * the extension status is updated. */ static Oid ts_extension_oid = InvalidOid; static const char *extstate_str[] = { [EXTENSION_STATE_UNKNOWN] = "unknown", [EXTENSION_STATE_TRANSITIONING] = "transitioning", [EXTENSION_STATE_CREATED] = "created", [EXTENSION_STATE_NOT_INSTALLED] = "not installed", }; static bool extension_loader_present() { void **presentptr = find_rendezvous_variable(RENDEZVOUS_LOADER_PRESENT_NAME); return (*presentptr != NULL && *((bool *) *presentptr)); } void ts_extension_check_version(const char *so_version) { char *sql_version; if (!IsNormalProcessingMode() || !IsTransactionState() || !extension_exists(EXTENSION_NAME)) return; sql_version = extension_version(EXTENSION_NAME); if (strcmp(sql_version, so_version) != 0) { /* * Throw a FATAL error here so that clients will be forced to reconnect * when they have the wrong extension version loaded. */ ereport(FATAL, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("extension \"%s\" version mismatch: shared library version %s; SQL version " "%s", EXTENSION_NAME, so_version, sql_version))); } if (!process_shared_preload_libraries_in_progress && !extension_loader_present()) { extension_load_without_preload(); } } void ts_extension_check_server_version() { /* * This is a load-time check for the correct server version since the * extension may be distributed as a binary */ char *server_version_num_guc = GetConfigOptionByName("server_version_num", NULL, false); long server_version_num = strtol(server_version_num_guc, NULL, 10); if (!is_supported_pg_version(server_version_num)) { char *server_version_guc = GetConfigOptionByName("server_version", NULL, false); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("extension \"%s\" does not support postgres version %s", EXTENSION_NAME, server_version_guc))); } } /* Sets a new state, returning whether the state has changed */ static bool extension_set_state(enum ExtensionState newstate) { if (newstate == extstate) { return false; } switch (newstate) { case EXTENSION_STATE_TRANSITIONING: case EXTENSION_STATE_UNKNOWN: break; case EXTENSION_STATE_CREATED: ts_extension_check_version(TIMESCALEDB_VERSION_MOD); extension_proxy_oid = ts_get_relation_relid(CACHE_SCHEMA_NAME, EXTENSION_PROXY_TABLE, true); ts_catalog_reset(); break; case EXTENSION_STATE_NOT_INSTALLED: extension_proxy_oid = InvalidOid; ts_catalog_reset(); break; } elog(DEBUG1, "extension state changed: %s to %s", extstate_str[extstate], extstate_str[newstate]); extstate = newstate; return true; } /* Updates the state based on the current state, returning whether there had been a change. */ static void extension_update_state() { enum ExtensionState new_state = extension_current_state(EXTENSION_NAME, CACHE_SCHEMA_NAME, EXTENSION_PROXY_TABLE); /* Never actually set the state to "not installed" since there is no good * way to get out of it in case the extension is installed again in * another backend. After the extension has been dropped, the proxy table * no longer exists and when the extension is reinstalled, the proxy table * will have a different relid. Therefore, there is no way to identify the * invalidation on the proxy table when CREATE EXTENSION is issued in * another backend. Nor is it allowed to lookup the new relid in the * invalidation callback, since that may lead to bad behavior. * * Instead, set the state to "unknown" so that a "slow path" lookup of the * actual state has to be made next time the state is queried. */ if (new_state == EXTENSION_STATE_NOT_INSTALLED) new_state = EXTENSION_STATE_UNKNOWN; extension_set_state(new_state); /* * Update the extension oid. Note that it is only safe to run * get_extension_oid() when the extension state is 'CREATED' or * 'TRANSITIONING', because otherwise we might not be even able to do a * catalog lookup because we are not in transaction state, and the like. */ if (new_state == EXTENSION_STATE_CREATED || new_state == EXTENSION_STATE_TRANSITIONING) { ts_extension_oid = get_extension_oid(EXTENSION_NAME, true /* missing_ok */); Assert(OidIsValid(ts_extension_oid)); } else { ts_extension_oid = InvalidOid; } } Oid ts_extension_schema_oid(void) { Datum result; Relation rel; SysScanDesc scandesc; HeapTuple tuple; ScanKeyData entry[1]; bool is_null = true; Oid schema = InvalidOid; rel = table_open(ExtensionRelationId, AccessShareLock); ScanKeyInit(&entry[0], Anum_pg_extension_extname, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(EXTENSION_NAME)); scandesc = systable_beginscan(rel, ExtensionNameIndexId, true, NULL, 1, entry); tuple = systable_getnext(scandesc); /* We assume that there can be at most one matching tuple */ if (HeapTupleIsValid(tuple)) { result = heap_getattr(tuple, Anum_pg_extension_extnamespace, RelationGetDescr(rel), &is_null); if (!is_null) schema = DatumGetObjectId(result); } systable_endscan(scandesc); table_close(rel, AccessShareLock); if (!OidIsValid(schema)) elog(ERROR, "extension schema not found"); return schema; } char * ts_extension_schema_name(void) { return get_namespace_name(ts_extension_schema_oid()); } const char * ts_experimental_schema_name(void) { return TS_EXPERIMENTAL_SCHEMA_NAME; } /* * Invalidate the state of the extension (i.e., whether the extension is * installed or not in the current database). * * Since this function is called from a relcache invalidation callback, it * must not, directly or indirectly, call functions that use the cache. This * includes, e.g., table scans. * * Instead, the function just invalidates the state so that the true state is * resolved lazily when needed. */ void ts_extension_invalidate(void) { elog(DEBUG1, "extension state invalidated: %s to %s", extstate_str[extstate], extstate_str[EXTENSION_STATE_UNKNOWN]); extstate = EXTENSION_STATE_UNKNOWN; extension_proxy_oid = InvalidOid; } bool ts_extension_is_loaded(void) { if (EXTENSION_STATE_UNKNOWN == extstate || EXTENSION_STATE_TRANSITIONING == extstate) { /* status may have updated without a relcache invalidate event */ extension_update_state(); } switch (extstate) { case EXTENSION_STATE_CREATED: Assert(OidIsValid(ts_extension_oid)); Assert(OidIsValid(extension_proxy_oid)); return true; case EXTENSION_STATE_NOT_INSTALLED: case EXTENSION_STATE_UNKNOWN: case EXTENSION_STATE_TRANSITIONING: /* * Turn off extension during upgrade scripts. This is necessary so * that, for example, the catalog does not go looking for things * that aren't yet there. */ if (extstate == EXTENSION_STATE_TRANSITIONING) { /* when we are updating the extension, we execute * scripts in post_update.sql after setting up the * the dependencies. At this stage, TS * specific functionality is permitted as we now have * all catalogs and functions in place */ const char *update_script_stage = GetConfigOption(TS_UPDATE_SCRIPT_CONFIG_VAR, true, false); if (update_script_stage && (strncmp(update_script_stage, POST_UPDATE, strlen(POST_UPDATE)) == 0) && (strlen(POST_UPDATE) == strlen(update_script_stage))) return true; } return false; default: elog(ERROR, "unknown state: %d", extstate); return false; } } bool ts_extension_is_loaded_and_not_upgrading(void) { /* When restoring deactivate extension. * * We are using IsBinaryUpgrade (and ts_guc_restoring). If a user set * `ts_guc_restoring` for a database, it will be stored in * `pg_db_role_settings` and be included in a dump, which will cause * `pg_upgrade` to fail. * * See dumpDatabaseConfig in pg_dump.c. */ if (ts_guc_restoring || IsBinaryUpgrade) return false; return ts_extension_is_loaded(); } const char * ts_extension_get_so_name(void) { return EXTENSION_NAME "-" TIMESCALEDB_VERSION_MOD; } bool ts_extension_is_proxy_table_relid(Oid relid) { return relid == extension_proxy_oid; } TS_FUNCTION_INFO_V1(ts_extension_get_state); Datum ts_extension_get_state(PG_FUNCTION_ARGS) { PG_RETURN_TEXT_P(cstring_to_text(extstate_str[extstate])); } ================================================ FILE: src/extension.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/parsenodes.h> #include "export.h" #include "extension_constants.h" extern void ts_extension_invalidate(void); extern TSDLLEXPORT bool ts_extension_is_loaded(void); extern bool ts_extension_is_loaded_and_not_upgrading(void); extern void ts_extension_check_version(const char *so_version); extern void ts_extension_check_server_version(void); extern TSDLLEXPORT Oid ts_extension_schema_oid(void); extern TSDLLEXPORT char *ts_extension_schema_name(void); extern const char *ts_experimental_schema_name(void); extern const char *ts_extension_get_so_name(void); extern bool ts_extension_is_proxy_table_relid(Oid relid); ================================================ FILE: src/extension_constants.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include "extension_constants.h" const char *const ts_extension_schema_names[] = { [TS_CATALOG_SCHEMA] = CATALOG_SCHEMA_NAME, [TS_FUNCTIONS_SCHEMA] = FUNCTIONS_SCHEMA_NAME, [TS_INTERNAL_SCHEMA] = INTERNAL_SCHEMA_NAME, [TS_CACHE_SCHEMA] = CACHE_SCHEMA_NAME, [TS_EXPERIMENTAL_SCHEMA] = EXPERIMENTAL_SCHEMA_NAME, [TS_INFORMATION_SCHEMA] = INFORMATION_SCHEMA_NAME, }; ================================================ FILE: src/extension_constants.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once /* No function definitions here, only potentially globally available defines as this is used by the * loader*/ #define EXTENSION_NAME "timescaledb" /* Name of the actual extension */ #define EXTENSION_NAMESPACE "timescaledb" /* Namespace for extension objects */ #define EXTENSION_NAMESPACE_ALIAS "tsdb" /* Namespace for extension objects */ #define TSL_LIBRARY_NAME "timescaledb-tsl" #define TS_LIBDIR "$libdir/" #define EXTENSION_SO TS_LIBDIR "" EXTENSION_NAME #define EXTENSION_TSL_SO TS_LIBDIR TSL_LIBRARY_NAME "-" TIMESCALEDB_VERSION_MOD #define MAKE_EXTOPTION(NAME) (EXTENSION_NAMESPACE "." NAME) #define MAX_VERSION_LEN (NAMEDATALEN + 1) #define MAX_SO_NAME_LEN \ (8 + NAMEDATALEN + 1 + MAX_VERSION_LEN) /* "$libdir/"+extname+"-"+version \ * */ typedef enum TsExtensionSchemas { TS_CATALOG_SCHEMA, TS_FUNCTIONS_SCHEMA, TS_INTERNAL_SCHEMA, TS_CACHE_SCHEMA, TS_EXPERIMENTAL_SCHEMA, TS_INFORMATION_SCHEMA, _TS_MAX_SCHEMA, } TsExtensionSchemas; #define NUM_TIMESCALEDB_SCHEMAS _TS_MAX_SCHEMA #define CATALOG_SCHEMA_NAME "_timescaledb_catalog" #define FUNCTIONS_SCHEMA_NAME "_timescaledb_functions" #define INTERNAL_SCHEMA_NAME "_timescaledb_internal" #define CACHE_SCHEMA_NAME "_timescaledb_cache" #define EXPERIMENTAL_SCHEMA_NAME "timescaledb_experimental" #define INFORMATION_SCHEMA_NAME "timescaledb_information" extern const char *const ts_extension_schema_names[]; #define RENDEZVOUS_BGW_LOADER_API_VERSION MAKE_EXTOPTION("bgw_loader_api_version") ================================================ FILE: src/extension_utils.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ /* This file will be used by the versioned timescaledb extension and the loader * Because we want the loader not to export symbols all files here should be static * and be included via #include "extension_utils.c" instead of the regular linking process */ #include <postgres.h> #include <access/genam.h> #include <access/relscan.h> #include <access/table.h> #include <access/xact.h> #include <catalog/indexing.h> #include <catalog/namespace.h> #include <catalog/pg_authid.h> #include <catalog/pg_extension.h> #include <commands/extension.h> #include <miscadmin.h> #include <parser/analyze.h> #include <utils/acl.h> #include <utils/builtins.h> #include <utils/fmgroids.h> #include <utils/guc.h> #include <utils/lsyscache.h> #include <utils/rel.h> #include "compat/compat.h" #include "extension_constants.h" #include "utils.h" #define EXTENSION_PROXY_TABLE "cache_inval_extension" #define RENDEZVOUS_LOADER_PRESENT_NAME MAKE_EXTOPTION("loader_present") enum ExtensionState { /* * NOT_INSTALLED means that this backend knows that the extension is not * present. In this state we know that the proxy table is not present. * This state is never saved since there is no real way to get out of it * since we cannot signal via the proxy table as its relid is not known * post installation without a full lookup, which is not allowed in the * relcache callback. */ EXTENSION_STATE_NOT_INSTALLED, /* * UNKNOWN state is used only if we cannot be sure what the state is. This * can happen in two cases: 1) at the start of a backend or 2) We got a * relcache event outside of a transaction and thus could not check the * cache for the presence/absence of the proxy table or extension. */ EXTENSION_STATE_UNKNOWN, /* * TRANSITIONING only occurs in the middle of a CREATE EXTENSION or ALTER * EXTENSION UPDATE */ EXTENSION_STATE_TRANSITIONING, /* * CREATED means we know the extension is loaded, metadata is up-to-date, * and we therefore do not need a full check until a RelCacheInvalidation * on the proxy table. */ EXTENSION_STATE_CREATED, }; static char * extension_version(char const *const extension_name) { Datum result; Relation rel; SysScanDesc scandesc; HeapTuple tuple; ScanKeyData entry[1]; bool is_null = true; char *sql_version = NULL; rel = table_open(ExtensionRelationId, AccessShareLock); ScanKeyInit(&entry[0], Anum_pg_extension_extname, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(extension_name)); scandesc = systable_beginscan(rel, ExtensionNameIndexId, true, NULL, 1, entry); tuple = systable_getnext(scandesc); /* We assume that there can be at most one matching tuple */ if (HeapTupleIsValid(tuple)) { result = heap_getattr(tuple, Anum_pg_extension_extversion, RelationGetDescr(rel), &is_null); if (!is_null) { sql_version = pstrdup(TextDatumGetCString(result)); } } systable_endscan(scandesc); table_close(rel, AccessShareLock); if (sql_version == NULL) { elog(ERROR, "extension not found while getting version"); } return sql_version; } inline static bool extension_exists(char const *const extension_name) { return OidIsValid(get_extension_oid(extension_name, true)); } inline static bool extension_is_transitioning(char const *const extension_name) { /* * Determine whether the extension is being created or upgraded (as a * misnomer creating_extension is true during upgrades) */ if (creating_extension) { return get_extension_oid(extension_name, true) == CurrentExtensionObject; } return false; } /* Returns the recomputed current state. * * (schema_name, table_name) refer to a table that is owned by the extension. * thus it is created with the extension and dropped with the extension. */ static enum ExtensionState extension_current_state(char const *const extension_name, char const *const schema_name, char const *const table_name) { /* * NormalProcessingMode necessary to avoid accessing cache before its * ready (which may result in an infinite loop). More concretely we need * RelationCacheInitializePhase3 to have been already called. */ if (!IsNormalProcessingMode() || !IsTransactionState() || !OidIsValid(MyDatabaseId)) return EXTENSION_STATE_UNKNOWN; /* * NOTE: do not check for (schema_name, table_name) existing here. Want to be in * TRANSITIONING state even before that table is created */ if (extension_is_transitioning(extension_name)) return EXTENSION_STATE_TRANSITIONING; /* * We use syscache to check (schema_name, table_name) exists. Must come first. * * A table that is owned by the extension is dropped before the extension itself is dropped. * this logic lets us detect that an extension is being dropped early on in the drop process and * return `EXTENSION_STATE_NOT_INSTALLED` if that is the case. * It is best to use a table created early on in the extension creation process because * that means it will be dropped early in the drop process. */ if (OidIsValid(ts_get_relation_relid(schema_name, table_name, true))) { Assert(extension_exists(extension_name)); return EXTENSION_STATE_CREATED; } return EXTENSION_STATE_NOT_INSTALLED; } /* Handle extension load request without loader present */ static void extension_load_without_preload() { /* * These are FATAL because otherwise the loader ends up in a weird * half-loaded state after an ERROR */ /* Only privileged users can get the value of `config file` */ if (has_privs_of_role(GetUserId(), ROLE_PG_READ_ALL_SETTINGS)) { char *config_file = GetConfigOptionByName("config_file", NULL, false); ereport(FATAL, (errmsg("extension \"%s\" must be preloaded", EXTENSION_NAME), errhint("Please preload the timescaledb library via " "shared_preload_libraries.\n\n" "This can be done by editing the config file at: %1$s\n" "and adding 'timescaledb' to the list in the shared_preload_libraries " "config.\n" " # Modify postgresql.conf:\n shared_preload_libraries = " "'timescaledb'\n\n" "Another way to do this, if not preloading other libraries, is with " "the command:\n" " echo \"shared_preload_libraries = 'timescaledb'\" >> %1$s \n\n" "(Will require a database restart.)\n\n", config_file))); } else { ereport(FATAL, (errmsg("extension \"%s\" must be preloaded", EXTENSION_NAME), errhint("Please preload the timescaledb library via shared_preload_libraries.\n\n" "This can be done by editing the postgres config file \n" "and adding 'timescaledb' to the list in the shared_preload_libraries " "config.\n" " # Modify postgresql.conf:\n shared_preload_libraries = " "'timescaledb'\n\n" "Another way to do this, if not preloading other libraries, is with the " "command:\n" " echo \"shared_preload_libraries = 'timescaledb'\" >> " "/path/to/config/file \n\n" "(Will require a database restart.)\n\n"))); } } ================================================ FILE: src/foreach_ptr.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <compat/compat.h> #include <postgres.h> #include <nodes/pg_list.h> #ifdef PG17_LT /* In PG16 foreach_ptr is not available, this is a straight copy of the * Postgres code that defines foreach_ptr and foreach_internal. * * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California */ #ifndef foreach_ptr #define foreach_ptr(type, var, lst) foreach_internal(type, *, var, lst, lfirst) #endif #ifndef foreach_internal #define foreach_internal(type, pointer, var, lst, func) \ for (type pointer var = 0, pointer var##__outerloop = (type pointer) 1; var##__outerloop; \ var##__outerloop = 0) \ for (ForEachState var##__state = { (lst), 0 }; \ (var##__state.l != NIL && var##__state.i < var##__state.l->length && \ (var = (type pointer) func(&var##__state.l->elements[var##__state.i]), true)); \ var##__state.i++) #endif #endif /* PG17_LT */ ================================================ FILE: src/foreign_key.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ /* * The table referenced by a foreign constraint is supposed to have a * constraint that prevents removing the referenced rows. The constraint * is enforced by a pair of update and delete triggers. Normally this * is done by the postgres addFkRecurseReferenced(), but it doesn't work * for hypertables because they use inheritance, and that function only * recurses into declarative partitioning hierarchy. */ #include <postgres.h> #include "access/attmap.h" #include "catalog/pg_trigger.h" #include "commands/trigger.h" #include "parser/parser.h" #include "compat/compat.h" #include "chunk.h" #include "chunk_constraint.h" #include "foreign_key.h" #include "hypertable.h" static HeapTuple relation_get_fk_constraint(Oid conrelid, Oid confrelid); static List *relation_get_referencing_fk(Oid reloid); static Oid get_fk_index(Relation rel, int nkeys, AttrNumber *confkeys); static void constraint_get_trigger(Oid conoid, Oid *updtrigoid, Oid *deltrigoid); static char *ChooseForeignKeyConstraintNameAddition(int numkeys, AttrNumber *keys, Oid relid); static void createForeignKeyActionTriggers(Form_pg_constraint fk, Oid relid, Oid refRelOid, Oid constraintOid, Oid indexOid, Oid parentDelTrigger, Oid parentUpdTrigger); static void clone_constraint_on_chunk(const Chunk *chunk, Relation parentRel, Form_pg_constraint fk, int numfks, AttrNumber *conkey, AttrNumber *confkey, Oid *conpfeqop, Oid *conppeqop, Oid *conffeqop, int numfkdelsetcols, AttrNumber *confdelsetcols, Oid parentDelTrigger, Oid parentUpdTrigger); /* * Copy foreign key constraint fk_tuple to all chunks. */ static void propagate_fk(Relation ht_rel, HeapTuple fk_tuple, List *chunks) { Form_pg_constraint fk = (Form_pg_constraint) GETSTRUCT(fk_tuple); int numfks; AttrNumber conkey[INDEX_MAX_KEYS]; AttrNumber confkey[INDEX_MAX_KEYS]; Oid conpfeqop[INDEX_MAX_KEYS]; Oid conppeqop[INDEX_MAX_KEYS]; Oid conffeqop[INDEX_MAX_KEYS]; int numfkdelsetcols; AttrNumber confdelsetcols[INDEX_MAX_KEYS]; DeconstructFkConstraintRow(fk_tuple, &numfks, conkey, confkey, conpfeqop, conppeqop, conffeqop, &numfkdelsetcols, confdelsetcols); Oid parentDelTrigger, parentUpdTrigger; constraint_get_trigger(fk->oid, &parentUpdTrigger, &parentDelTrigger); ListCell *lc; foreach (lc, chunks) { Chunk *chunk = lfirst(lc); if (chunk->fd.osm_chunk) continue; clone_constraint_on_chunk(chunk, ht_rel, fk, numfks, conkey, confkey, conpfeqop, conppeqop, conffeqop, numfkdelsetcols, confdelsetcols, parentDelTrigger, parentUpdTrigger); } } /* * Copy all foreign key constraints from the main table to a chunk. */ void ts_chunk_copy_referencing_fk(const Hypertable *ht, const Chunk *chunk) { ListCell *lc; List *chunks = list_make1((Chunk *) chunk); List *fks = relation_get_referencing_fk(ht->main_table_relid); Relation ht_rel = table_open(ht->main_table_relid, AccessShareLock); foreach (lc, fks) { HeapTuple fk_tuple = lfirst(lc); propagate_fk(ht_rel, fk_tuple, chunks); } table_close(ht_rel, NoLock); } /* * Copy one foreign key constraint from the main table to all chunks. */ void ts_fk_propagate(Oid conrelid, Hypertable *ht) { HeapTuple fk_tuple = relation_get_fk_constraint(conrelid, ht->main_table_relid); if (!fk_tuple) elog(ERROR, "foreign key constraint not found"); Relation ht_rel = table_open(ht->main_table_relid, AccessShareLock); List *chunks = ts_chunk_get_by_hypertable_id(ht->fd.id); propagate_fk(ht_rel, fk_tuple, chunks); table_close(ht_rel, NoLock); } /* * Clone a single constraint to a single chunk. */ static void clone_constraint_on_chunk(const Chunk *chunk, Relation parentRel, Form_pg_constraint fk, int numfks, AttrNumber *conkey, AttrNumber *confkey, Oid *conpfeqop, Oid *conppeqop, Oid *conffeqop, int numfkdelsetcols, AttrNumber *confdelsetcols, Oid parentDelTrigger, Oid parentUpdTrigger) { AttrNumber mapped_confkey[INDEX_MAX_KEYS]; Relation pkrel = table_open(chunk->table_id, AccessShareLock); /* Map the foreign key columns on the hypertable side to the chunk columns */ #if PG16_GE AttrMap *attmap = build_attrmap_by_name(RelationGetDescr(pkrel), RelationGetDescr(parentRel), false); #else AttrMap *attmap = build_attrmap_by_name(RelationGetDescr(pkrel), RelationGetDescr(parentRel)); #endif for (int i = 0; i < numfks; i++) mapped_confkey[i] = attmap->attnums[confkey[i] - 1]; Oid indexoid = get_fk_index(pkrel, numfks, mapped_confkey); /* Since postgres accepted the constraint, there should be a supporting index. */ Ensure(OidIsValid(indexoid), "index for constraint not found on chunk"); table_close(pkrel, NoLock); char *conname_addition = ChooseForeignKeyConstraintNameAddition(numfks, confkey, parentRel->rd_id); char *conname = ChooseConstraintName(get_rel_name(fk->conrelid), conname_addition, "fkey", fk->connamespace, NIL); Oid conoid = CreateConstraintEntry(conname, fk->connamespace, CONSTRAINT_FOREIGN, fk->condeferrable, fk->condeferred, #if PG18_GE true, /* isEnforced */ #endif fk->convalidated, fk->oid, fk->conrelid, conkey, numfks, numfks, InvalidOid, indexoid, chunk->table_id, mapped_confkey, conpfeqop, conppeqop, conffeqop, numfks, fk->confupdtype, fk->confdeltype, confdelsetcols, numfkdelsetcols, fk->confmatchtype, NULL, NULL, NULL, false, 1, false, #if PG18_GE false, /* conPeriod */ #endif false); ObjectAddress address, referenced; ObjectAddressSet(address, ConstraintRelationId, conoid); ObjectAddressSet(referenced, ConstraintRelationId, fk->oid); recordDependencyOn(&address, &referenced, DEPENDENCY_INTERNAL); CommandCounterIncrement(); createForeignKeyActionTriggers(fk, fk->conrelid, chunk->table_id, conoid, indexoid, parentDelTrigger, parentUpdTrigger); } /* * Generate the column-name portion of the constraint name for a new foreign * key given the list of column names that reference the referenced * table. This will be passed to ChooseConstraintName along with the parent * table name and the "fkey" suffix. * * We know that less than NAMEDATALEN characters will actually be used, so we * can truncate the result once we've generated that many. * * This function is based on a static function by the same name in tablecmds.c in PostgreSQL. */ static char * ChooseForeignKeyConstraintNameAddition(int numkeys, AttrNumber *keys, Oid relid) { char buf[NAMEDATALEN * 2]; int buflen = 0; buf[0] = '\0'; for (int i = 0; i < numkeys; i++) { char *name = get_attname(relid, keys[i], false); if (buflen > 0) buf[buflen++] = '_'; /* insert _ between names */ /* * At this point we have buflen <= NAMEDATALEN. name should be less * than NAMEDATALEN already, but use strlcpy for paranoia. */ strlcpy(buf + buflen, name, NAMEDATALEN); buflen += strlen(buf + buflen); if (buflen >= NAMEDATALEN) break; } return pstrdup(buf); } /* * createForeignKeyActionTriggers * Create the referenced-side "action" triggers that implement a foreign * key. * This function is based on a static function by the same name in tablecmds.c in PostgreSQL. */ static void createForeignKeyActionTriggers(Form_pg_constraint fk, Oid relid, Oid refRelOid, Oid constraintOid, Oid indexOid, Oid parentDelTrigger, Oid parentUpdTrigger) { CreateTrigStmt *fk_trigger; /* * Build and execute a CREATE CONSTRAINT TRIGGER statement for the ON * DELETE action on the referenced table. */ fk_trigger = makeNode(CreateTrigStmt); fk_trigger->replace = false; fk_trigger->isconstraint = true; fk_trigger->trigname = "RI_ConstraintTrigger_a"; fk_trigger->relation = NULL; fk_trigger->args = NIL; fk_trigger->row = true; fk_trigger->timing = TRIGGER_TYPE_AFTER; fk_trigger->events = TRIGGER_TYPE_DELETE; fk_trigger->columns = NIL; fk_trigger->whenClause = NULL; fk_trigger->transitionRels = NIL; fk_trigger->constrrel = NULL; switch (fk->confdeltype) { case FKCONSTR_ACTION_NOACTION: fk_trigger->deferrable = fk->condeferrable; fk_trigger->initdeferred = fk->condeferred; fk_trigger->funcname = SystemFuncName("RI_FKey_noaction_del"); break; case FKCONSTR_ACTION_RESTRICT: fk_trigger->deferrable = false; fk_trigger->initdeferred = false; fk_trigger->funcname = SystemFuncName("RI_FKey_restrict_del"); break; case FKCONSTR_ACTION_CASCADE: fk_trigger->deferrable = false; fk_trigger->initdeferred = false; fk_trigger->funcname = SystemFuncName("RI_FKey_cascade_del"); break; case FKCONSTR_ACTION_SETNULL: fk_trigger->deferrable = false; fk_trigger->initdeferred = false; fk_trigger->funcname = SystemFuncName("RI_FKey_setnull_del"); break; case FKCONSTR_ACTION_SETDEFAULT: fk_trigger->deferrable = false; fk_trigger->initdeferred = false; fk_trigger->funcname = SystemFuncName("RI_FKey_setdefault_del"); break; default: elog(ERROR, "unrecognized FK action type: %d", (int) fk->confdeltype); break; } /* * clang will complain here about swapped arguments but this is intentional * as this is the reverse trigger from the referenced table back to the * referencing table. So we disable that specific warning for the next call. * * NOLINTBEGIN(readability-suspicious-call-argument) */ CreateTrigger(fk_trigger, NULL, refRelOid, relid, constraintOid, indexOid, InvalidOid, parentDelTrigger, NULL, true, false); /* NOLINTEND(readability-suspicious-call-argument) */ /* Make changes-so-far visible */ CommandCounterIncrement(); /* * Build and execute a CREATE CONSTRAINT TRIGGER statement for the ON * UPDATE action on the referenced table. */ fk_trigger = makeNode(CreateTrigStmt); fk_trigger->replace = false; fk_trigger->isconstraint = true; fk_trigger->trigname = "RI_ConstraintTrigger_a"; fk_trigger->relation = NULL; fk_trigger->args = NIL; fk_trigger->row = true; fk_trigger->timing = TRIGGER_TYPE_AFTER; fk_trigger->events = TRIGGER_TYPE_UPDATE; fk_trigger->columns = NIL; fk_trigger->whenClause = NULL; fk_trigger->transitionRels = NIL; fk_trigger->constrrel = NULL; switch (fk->confupdtype) { case FKCONSTR_ACTION_NOACTION: fk_trigger->deferrable = fk->condeferrable; fk_trigger->initdeferred = fk->condeferred; fk_trigger->funcname = SystemFuncName("RI_FKey_noaction_upd"); break; case FKCONSTR_ACTION_RESTRICT: fk_trigger->deferrable = false; fk_trigger->initdeferred = false; fk_trigger->funcname = SystemFuncName("RI_FKey_restrict_upd"); break; case FKCONSTR_ACTION_CASCADE: fk_trigger->deferrable = false; fk_trigger->initdeferred = false; fk_trigger->funcname = SystemFuncName("RI_FKey_cascade_upd"); break; case FKCONSTR_ACTION_SETNULL: fk_trigger->deferrable = false; fk_trigger->initdeferred = false; fk_trigger->funcname = SystemFuncName("RI_FKey_setnull_upd"); break; case FKCONSTR_ACTION_SETDEFAULT: fk_trigger->deferrable = false; fk_trigger->initdeferred = false; fk_trigger->funcname = SystemFuncName("RI_FKey_setdefault_upd"); break; default: elog(ERROR, "unrecognized FK action type: %d", (int) fk->confupdtype); break; } /* * clang will complain here about swapped arguments but this is intentional * as this is the reverse trigger from the referenced table back to the * referencing table. * * NOLINTBEGIN(readability-suspicious-call-argument) */ CreateTrigger(fk_trigger, NULL, refRelOid, relid, constraintOid, indexOid, InvalidOid, parentUpdTrigger, NULL, true, false); /* NOLINTEND(readability-suspicious-call-argument) */ /* Make changes-so-far visible */ CommandCounterIncrement(); } /* * Return a list of foreign key pg_constraint heap tuples referencing reloid. */ static List * relation_get_referencing_fk(Oid reloid) { List *result = NIL; Relation conrel; SysScanDesc conscan; ScanKeyData skey[2]; HeapTuple htup; /* Prepare to scan pg_constraint for entries having confrelid = this rel. */ ScanKeyInit(&skey[0], Anum_pg_constraint_confrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(reloid)); ScanKeyInit(&skey[1], Anum_pg_constraint_contype, BTEqualStrategyNumber, F_CHAREQ, CharGetDatum(CONSTRAINT_FOREIGN)); conrel = table_open(ConstraintRelationId, AccessShareLock); conscan = systable_beginscan(conrel, InvalidOid, false, NULL, 2, skey); while (HeapTupleIsValid(htup = systable_getnext(conscan))) { result = lappend(result, heap_copytuple(htup)); } systable_endscan(conscan); table_close(conrel, AccessShareLock); return result; } /* * Return a list of foreign key pg_constraint heap tuples referencing reloid. */ static HeapTuple relation_get_fk_constraint(Oid conrelid, Oid confrelid) { Relation conrel; SysScanDesc conscan; ScanKeyData skey[3]; /* Prepare to scan pg_constraint for entries having confrelid = this rel. */ ScanKeyInit(&skey[0], Anum_pg_constraint_conrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(conrelid)); ScanKeyInit(&skey[1], Anum_pg_constraint_confrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(confrelid)); ScanKeyInit(&skey[2], Anum_pg_constraint_contype, BTEqualStrategyNumber, F_CHAREQ, CharGetDatum(CONSTRAINT_FOREIGN)); conrel = table_open(ConstraintRelationId, AccessShareLock); conscan = systable_beginscan(conrel, InvalidOid, false, NULL, 3, skey); HeapTuple htup = systable_getnext(conscan); if (HeapTupleIsValid(htup)) { htup = heap_copytuple(htup); } systable_endscan(conscan); table_close(conrel, AccessShareLock); return htup; } /* Get the UPDATE and DELETE trigger OIDs for the given constraint OID */ static void constraint_get_trigger(Oid conoid, Oid *updtrigoid, Oid *deltrigoid) { Relation rel; SysScanDesc scan; ScanKeyData skey[1]; HeapTuple htup; *updtrigoid = InvalidOid; *deltrigoid = InvalidOid; ScanKeyInit(&skey[0], Anum_pg_trigger_tgconstraint, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(conoid)); rel = table_open(TriggerRelationId, AccessShareLock); scan = systable_beginscan(rel, TriggerConstraintIndexId, true, NULL, 1, skey); while (HeapTupleIsValid(htup = systable_getnext(scan))) { Form_pg_trigger trigform = (Form_pg_trigger) GETSTRUCT(htup); if ((trigform->tgtype & TRIGGER_TYPE_UPDATE) == TRIGGER_TYPE_UPDATE) *updtrigoid = trigform->oid; if ((trigform->tgtype & TRIGGER_TYPE_DELETE) == TRIGGER_TYPE_DELETE) *deltrigoid = trigform->oid; } systable_endscan(scan); table_close(rel, AccessShareLock); } /* * Return the oid of the index supporting the foreign key constraint. */ static Oid get_fk_index(Relation rel, int nkeys, AttrNumber *confkeys) { Oid indexoid = InvalidOid; List *indexes = RelationGetIndexList(rel); ListCell *lc; foreach (lc, indexes) { Oid indexoid = lfirst_oid(lc); Relation indexrel = index_open(indexoid, AccessShareLock); if (!indexrel->rd_index->indisunique || indexrel->rd_index->indnkeyatts != nkeys) { index_close(indexrel, AccessShareLock); continue; } Bitmapset *con_keys = NULL; Bitmapset *ind_keys = NULL; for (int i = 0; i < nkeys; i++) { /* * Since ordering of the constraint definition and index definition can differ, * we need to check that all the columns in the constraint are present in the index */ con_keys = bms_add_member(con_keys, confkeys[i]); ind_keys = bms_add_member(ind_keys, indexrel->rd_index->indkey.values[i]); } bool match = bms_equal(con_keys, ind_keys); index_close(indexrel, AccessShareLock); bms_free(con_keys); bms_free(ind_keys); if (match) { return indexoid; } } return indexoid; } void ts_chunk_drop_referencing_fk_by_chunk_id(Oid chunk_id) { Chunk *chunk = ts_chunk_get_by_id(chunk_id, true); List *fks = relation_get_referencing_fk(chunk->table_id); ListCell *lc; foreach (lc, fks) { HeapTuple fk_tuple = lfirst(lc); ts_chunk_constraint_drop_from_tuple(fk_tuple); } } ================================================ FILE: src/foreign_key.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_constraint.h> #include <nodes/parsenodes.h> #include "chunk.h" #include "export.h" #include "hypertable.h" extern TSDLLEXPORT void ts_fk_propagate(Oid conrelid, Hypertable *ht); extern TSDLLEXPORT void ts_chunk_copy_referencing_fk(const Hypertable *ht, const Chunk *chunk); extern TSDLLEXPORT void ts_chunk_drop_referencing_fk_by_chunk_id(Oid chunk_id); ================================================ FILE: src/func_cache.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/heapam.h> #include <access/htup.h> #include <catalog/pg_namespace_d.h> #include <catalog/pg_proc.h> #include <catalog/pg_type.h> #include <miscadmin.h> #include <nodes/pathnodes.h> #include <optimizer/optimizer.h> #include <parser/parse_oper.h> #include <utils/builtins.h> #include <utils/hsearch.h> #include <utils/lsyscache.h> #include <utils/rel.h> #include <utils/selfuncs.h> #include <utils/syscache.h> #include "compat/compat.h" #include "cache.h" #include "estimate.h" #include "extension.h" #include "func_cache.h" #include "sort_transform.h" #include "utils.h" /* * func_cache - a cache for quick identification of, and access to, functions * useful for TimescaleDB. The function info is used in various query * optimizations, for instance, we provide custom group estimate functions for * use when grouping on time buckets. We also provide functions that allow * sorting time buckets using an index on the non-bucketed expression/column. */ static Expr * date_trunc_sort_transform(FuncExpr *func) { /* * date_trunc (const, var) => var * * proof: date_trunc(c, time1) >= date_trunc(c,time2) iff time1 > time2 */ Expr *second; if (list_length(func->args) != 2 || !IsA(linitial(func->args), Const)) return (Expr *) func; second = ts_sort_transform_expr(lsecond(func->args)); if (!IsA(second, Var)) return (Expr *) func; return (Expr *) copyObject(second); } /* * Check that time_bucket has a const offset, if an offset is supplied */ #define time_bucket_has_const_offset(func) \ (list_length((func)->args) == 2 || IsA(lthird((func)->args), Const)) #define time_bucket_has_const_period(func) IsA(linitial((func)->args), Const) #define time_bucket_has_const_timezone(func) IsA(lthird((func)->args), Const) static Expr * do_sort_transform(FuncExpr *func) { Expr *second = ts_sort_transform_expr(lsecond(func->args)); if (!IsA(second, Var)) return (Expr *) func; return (Expr *) copyObject(second); } static Expr * time_bucket_gapfill_sort_transform(FuncExpr *func) { /* * time_bucket(const, var, const) => var * * proof: time_bucket(const1, time1) >= time_bucket(const1,time2) iff time1 * > time2 */ Assert(list_length(func->args) == 4 || list_length(func->args) == 5); if (!time_bucket_has_const_period(func) || (list_length(func->args) == 5 && !time_bucket_has_const_timezone(func))) return (Expr *) func; return do_sort_transform(func); } static Expr * time_bucket_sort_transform(FuncExpr *func) { Assert(list_length(func->args) >= 2); /* * If period and offset are not constants we must not do the optimization */ if (!time_bucket_has_const_offset(func)) return (Expr *) func; if (!time_bucket_has_const_period(func)) return (Expr *) func; return do_sort_transform(func); } /* * time_bucket with timezone will always have 5 args. For the sort * optimization to apply all args need to be Const except timestamp. */ static Expr * time_bucket_tz_sort_transform(FuncExpr *func) { Assert(list_length(func->args) == 5); if (!IsA(linitial((func)->args), Const) || !IsA(lthird(func->args), Const) || !IsA(lfourth(func->args), Const) || !IsA(lfifth(func->args), Const)) return (Expr *) func; return do_sort_transform(func); } /* For time_bucket this estimate currently works by seeing how many possible * buckets there will be if the data spans the entire hypertable. Note that * this is an overestimate. * */ static double time_bucket_group_estimate(PlannerInfo *root, FuncExpr *expr, double path_rows) { Node *first_arg = eval_const_expressions(root, linitial(expr->args)); Expr *second_arg = lsecond(expr->args); Const *c; double period; if (!IsA(first_arg, Const)) return INVALID_ESTIMATE; c = (Const *) first_arg; switch (c->consttype) { case INT2OID: period = (double) DatumGetInt16(c->constvalue); break; case INT4OID: period = (double) DatumGetInt32(c->constvalue); break; case INT8OID: period = (double) DatumGetInt64(c->constvalue); break; case INTERVALOID: period = (double) ts_get_interval_period_approx(DatumGetIntervalP(c->constvalue)); break; default: return INVALID_ESTIMATE; } return ts_estimate_group_expr_interval(root, second_arg, period); } /* For date_trunc this estimate currently works by seeing how many possible * buckets there will be if the data spans the entire hypertable. Note that * this is an overestimate. * */ static double date_trunc_group_estimate(PlannerInfo *root, FuncExpr *expr, double path_rows) { Node *first_arg = eval_const_expressions(root, linitial(expr->args)); Expr *second_arg = lsecond(expr->args); Const *c; text *interval; if (!IsA(first_arg, Const)) return INVALID_ESTIMATE; c = (Const *) first_arg; interval = DatumGetTextPP(c->constvalue); return ts_estimate_group_expr_interval(root, second_arg, (double) ts_date_trunc_interval_period_approx(interval)); } typedef struct FuncEntry { Oid funcid; FuncInfo *funcinfo; } FuncEntry; /* Information about functions that we put in the cache */ static FuncInfo funcinfo[] = { { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = true, .funcname = "time_bucket", .nargs = 2, .arg_types = { INTERVALOID, TIMESTAMPOID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_sort_transform, }, /* Interval Bucket with origin */ { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = true, .funcname = "time_bucket", .nargs = 3, .arg_types = { INTERVALOID, TIMESTAMPOID, TIMESTAMPOID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_sort_transform, }, /* Interval Bucket with offset */ { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = true, .funcname = "time_bucket", .nargs = 3, .arg_types = { INTERVALOID, TIMESTAMPOID, INTERVALOID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_sort_transform, }, { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = true, .funcname = "time_bucket", .nargs = 2, .arg_types = { INTERVALOID, TIMESTAMPTZOID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_sort_transform, }, /* Interval Bucket with origin */ { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = true, .funcname = "time_bucket", .nargs = 3, .arg_types = { INTERVALOID, TIMESTAMPTZOID, TIMESTAMPTZOID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_sort_transform, }, /* Interval Bucket with offset */ { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = true, .funcname = "time_bucket", .nargs = 3, .arg_types = { INTERVALOID, TIMESTAMPTZOID, INTERVALOID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_sort_transform, }, { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = true, .funcname = "time_bucket", .nargs = 2, .arg_types = { INTERVALOID, UUIDOID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_sort_transform, }, /* Interval Bucket with origin */ { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = true, .funcname = "time_bucket", .nargs = 3, .arg_types = { INTERVALOID, UUIDOID, TIMESTAMPTZOID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_sort_transform, }, /* Interval Bucket with offset */ { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = true, .funcname = "time_bucket", .nargs = 3, .arg_types = { INTERVALOID, UUIDOID, INTERVALOID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_sort_transform, }, { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = true, .funcname = "time_bucket", .nargs = 2, .arg_types = { INTERVALOID, DATEOID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_sort_transform, }, /* Interval Bucket with origin */ { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = true, .funcname = "time_bucket", .nargs = 3, .arg_types = { INTERVALOID, DATEOID, DATEOID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_sort_transform, }, /* Interval Bucket with offset */ { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = true, .funcname = "time_bucket", .nargs = 3, .arg_types = { INTERVALOID, DATEOID, INTERVALOID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_sort_transform, }, { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = true, .funcname = "time_bucket", .nargs = 2, .arg_types = { INT2OID, INT2OID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_sort_transform, }, /* Int2 Bucket with offset */ { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = true, .funcname = "time_bucket", .nargs = 3, .arg_types = { INT2OID, INT2OID, INT2OID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_sort_transform, }, { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = true, .funcname = "time_bucket", .nargs = 2, .arg_types = { INT4OID, INT4OID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_sort_transform, }, /* Int4 Bucket with offset */ { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = true, .funcname = "time_bucket", .nargs = 3, .arg_types = { INT4OID, INT4OID, INT4OID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_sort_transform, }, { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = true, .funcname = "time_bucket", .nargs = 2, .arg_types = { INT8OID, INT8OID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_sort_transform, }, /* Int8 Bucket with offset */ { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = true, .funcname = "time_bucket", .nargs = 3, .arg_types = { INT8OID, INT8OID, INT8OID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_sort_transform, }, { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = true, .funcname = "time_bucket", .nargs = 5, .arg_types = { INTERVALOID, TIMESTAMPTZOID, TEXTOID, TIMESTAMPTZOID, INTERVALOID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_tz_sort_transform, }, { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = false, .funcname = "time_bucket_gapfill", .nargs = 4, .arg_types = { INTERVALOID, TIMESTAMPOID, TIMESTAMPOID, TIMESTAMPOID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_gapfill_sort_transform, }, { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = false, .funcname = "time_bucket_gapfill", .nargs = 4, .arg_types = { INTERVALOID, TIMESTAMPTZOID, TIMESTAMPTZOID, TIMESTAMPTZOID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_gapfill_sort_transform, }, { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = false, .funcname = "time_bucket_gapfill", .nargs = 5, .arg_types = { INTERVALOID, TIMESTAMPTZOID, TEXTOID, TIMESTAMPTZOID, TIMESTAMPTZOID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_gapfill_sort_transform, }, { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = false, .funcname = "time_bucket_gapfill", .nargs = 4, .arg_types = { INTERVALOID, DATEOID, DATEOID, DATEOID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_gapfill_sort_transform, }, { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = false, .funcname = "time_bucket_gapfill", .nargs = 4, .arg_types = { INT2OID, INT2OID, INT2OID, INT2OID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_gapfill_sort_transform, }, { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = false, .funcname = "time_bucket_gapfill", .nargs = 4, .arg_types = { INT4OID, INT4OID, INT4OID, INT4OID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_gapfill_sort_transform, }, { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = true, .allowed_in_cagg_definition = false, .funcname = "time_bucket_gapfill", .nargs = 4, .arg_types = { INT8OID, INT8OID, INT8OID, INT8OID }, .group_estimate = time_bucket_group_estimate, .sort_transform = time_bucket_gapfill_sort_transform, }, { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = false, .allowed_in_cagg_definition = false, .funcname = "first", .nargs = 2, .arg_types = { ANYELEMENTOID, ANYOID }, .group_estimate = NULL, .sort_transform = NULL, }, { .origin = ORIGIN_TIMESCALE, .is_bucketing_func = false, .allowed_in_cagg_definition = false, .funcname = "last", .nargs = 2, .arg_types = { ANYELEMENTOID, ANYOID }, .group_estimate = NULL, .sort_transform = NULL, }, { .origin = ORIGIN_POSTGRES, .is_bucketing_func = true, .allowed_in_cagg_definition = false, .funcname = "date_trunc", .nargs = 2, .arg_types = { TEXTOID, TIMESTAMPOID }, .group_estimate = date_trunc_group_estimate, .sort_transform = date_trunc_sort_transform, }, { .origin = ORIGIN_POSTGRES, .is_bucketing_func = true, .allowed_in_cagg_definition = false, .funcname = "date_trunc", .nargs = 2, .arg_types = { TEXTOID, TIMESTAMPTZOID }, .group_estimate = date_trunc_group_estimate, .sort_transform = date_trunc_sort_transform, }, }; #define _MAX_CACHE_FUNCTIONS (sizeof(funcinfo) / sizeof(funcinfo[0])) Oid ts_first_func_oid = InvalidOid; Oid ts_last_func_oid = InvalidOid; static HTAB *func_hash = NULL; static Oid proc_get_oid(HeapTuple tuple) { Form_pg_proc form = (Form_pg_proc) GETSTRUCT(tuple); return form->oid; } void ts_func_cache_init() { Ensure(!func_hash, "function cache already initialized"); HASHCTL hashctl = { .keysize = sizeof(Oid), .entrysize = sizeof(FuncEntry), .hcxt = CacheMemoryContext, }; Oid extension_nsp = ts_extension_schema_oid(); Oid experimental_nsp = get_namespace_oid(ts_experimental_schema_name(), false); HeapTuple tuple; Relation rel; func_hash = hash_create("func_cache", _MAX_CACHE_FUNCTIONS, &hashctl, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); rel = table_open(ProcedureRelationId, AccessShareLock); for (size_t i = 0; i < _MAX_CACHE_FUNCTIONS; i++) { FuncInfo *finfo = &funcinfo[i]; Oid namespaceoid = PG_CATALOG_NAMESPACE; oidvector *paramtypes = buildoidvector(finfo->arg_types, finfo->nargs); FuncEntry *fentry; bool hash_found; Oid funcid; if (finfo->origin == ORIGIN_TIMESCALE) { namespaceoid = extension_nsp; } else if (finfo->origin == ORIGIN_TIMESCALE_EXPERIMENTAL) { namespaceoid = experimental_nsp; } tuple = SearchSysCache3(PROCNAMEARGSNSP, PointerGetDatum(finfo->funcname), PointerGetDatum(paramtypes), ObjectIdGetDatum(namespaceoid)); if (!HeapTupleIsValid(tuple)) { /* The function cache could be accessed during an extension upgrade. Not all expected * functions have to exist at this point. */ elog(ts_extension_is_loaded_and_not_upgrading() ? ERROR : NOTICE, "cache lookup failed for function \"%s\" with %d args", finfo->funcname, finfo->nargs); continue; } funcid = proc_get_oid(tuple); /* Special handling for first/last to set up named variables for their oids */ if (strcmp(finfo->funcname, "first") == 0) ts_first_func_oid = funcid; else if (strcmp(finfo->funcname, "last") == 0) ts_last_func_oid = funcid; fentry = hash_search(func_hash, &funcid, HASH_ENTER, &hash_found); Assert(!hash_found); fentry->funcid = funcid; fentry->funcinfo = finfo; ReleaseSysCache(tuple); } table_close(rel, AccessShareLock); } FuncInfo * ts_func_cache_get(Oid funcid) { FuncEntry *entry; if (!func_hash) ts_func_cache_init(); entry = hash_search(func_hash, &funcid, HASH_FIND, NULL); return (NULL == entry) ? NULL : entry->funcinfo; } FuncInfo * ts_func_cache_get_bucketing_func(Oid funcid) { FuncInfo *finfo = ts_func_cache_get(funcid); if (NULL == finfo) return NULL; return finfo->is_bucketing_func ? finfo : NULL; } ================================================ FILE: src/func_cache.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/primnodes.h> #include "export.h" #define FUNC_CACHE_MAX_FUNC_ARGS 10 typedef Expr *(*sort_transform_func)(FuncExpr *func); typedef double (*group_estimate_func)(PlannerInfo *root, FuncExpr *expr, double path_rows); /* Describes the function origin */ typedef enum { /* * Function is provided by PostgreSQL. */ ORIGIN_POSTGRES = 0, /* * Function is provided by TimescaleDB. */ ORIGIN_TIMESCALE = 1, /* * Function is provided by TimescaleDB and is experimental. * It should be looked for in the experimental schema. */ ORIGIN_TIMESCALE_EXPERIMENTAL = 2, } FuncOrigin; typedef struct FuncInfo { const char *funcname; FuncOrigin origin; bool is_bucketing_func; bool allowed_in_cagg_definition; int nargs; Oid arg_types[FUNC_CACHE_MAX_FUNC_ARGS]; group_estimate_func group_estimate; sort_transform_func sort_transform; } FuncInfo; extern TSDLLEXPORT void ts_func_cache_init(void); extern TSDLLEXPORT FuncInfo *ts_func_cache_get(Oid funcid); extern TSDLLEXPORT FuncInfo *ts_func_cache_get_bucketing_func(Oid funcid); extern TSDLLEXPORT Oid ts_first_func_oid; extern TSDLLEXPORT Oid ts_last_func_oid; ================================================ FILE: src/gapfill.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <fmgr.h> #include "compat/compat.h" #include "cross_module_fn.h" #include "export.h" #include "license_guc.h" /* * stub function to trigger locf and interpolate in gapfill node */ TS_FUNCTION_INFO_V1(ts_gapfill_marker); Datum ts_gapfill_marker(PG_FUNCTION_ARGS) { PG_RETURN_DATUM(ts_cm_functions->gapfill_marker(fcinfo)); } #define GAPFILL_TIMEBUCKET_WRAPPER(datatype) \ TS_FUNCTION_INFO_V1(ts_gapfill_##datatype##_bucket); \ Datum ts_gapfill_##datatype##_bucket(PG_FUNCTION_ARGS) \ { \ return ts_cm_functions->gapfill_##datatype##_time_bucket(fcinfo); \ } GAPFILL_TIMEBUCKET_WRAPPER(int16); GAPFILL_TIMEBUCKET_WRAPPER(int32); GAPFILL_TIMEBUCKET_WRAPPER(int64); GAPFILL_TIMEBUCKET_WRAPPER(date); GAPFILL_TIMEBUCKET_WRAPPER(timestamp); GAPFILL_TIMEBUCKET_WRAPPER(timestamptz); GAPFILL_TIMEBUCKET_WRAPPER(timestamptz_timezone); ================================================ FILE: src/gitcommit.cmake ================================================ # The commands for generating gitcommit.h need to be executed on every make run # and not on cmake run to detect branch switches, commit changes or local # modifications. if(GIT_FOUND) # We use "git describe" to generate the tag. It will find the latest tag and # also add some additional information if we are not on the tag. execute_process( COMMAND ${GIT_EXECUTABLE} describe --dirty --always --tags WORKING_DIRECTORY ${SOURCE_DIR} OUTPUT_VARIABLE EXT_GIT_COMMIT_TAG RESULT_VARIABLE _describe_RESULT OUTPUT_STRIP_TRAILING_WHITESPACE) # Fetch the commit HASH of head (short version) using rev-parse execute_process( COMMAND ${GIT_EXECUTABLE} rev-parse HEAD OUTPUT_VARIABLE EXT_GIT_COMMIT_HASH WORKING_DIRECTORY ${SOURCE_DIR} RESULT_VARIABLE _revparse_RESULT OUTPUT_STRIP_TRAILING_WHITESPACE) # Fetch the date of the head commit execute_process( COMMAND ${GIT_EXECUTABLE} log -1 --format=%cI WORKING_DIRECTORY ${SOURCE_DIR} OUTPUT_VARIABLE EXT_GIT_COMMIT_TIME RESULT_VARIABLE _log_RESULT OUTPUT_STRIP_TRAILING_WHITESPACE) # Results are non-zero if there were an error if(_describe_RESULT OR _revparse_RESULT OR _log_RESULT) message(STATUS "Unable to get git commit information") else() message( STATUS "Building commit ${EXT_GIT_COMMIT_TAG} (${EXT_GIT_COMMIT_HASH}), ${EXT_GIT_COMMIT_TIME}" ) endif() endif() file(REMOVE ${OUTPUT_FILE}) configure_file(${INPUT_FILE} ${OUTPUT_FILE}) ================================================ FILE: src/gitcommit.h.in ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #ifndef GITCOMMIT_H_ #define GITCOMMIT_H_ #cmakedefine EXT_GIT_COMMIT_TAG "@EXT_GIT_COMMIT_TAG@" #cmakedefine EXT_GIT_COMMIT_HASH "@EXT_GIT_COMMIT_HASH@" #cmakedefine EXT_GIT_COMMIT_TIME "@EXT_GIT_COMMIT_TIME@" #endif /* GITCOMMIT_H_ */ ================================================ FILE: src/guc.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <miscadmin.h> #include <parser/parse_func.h> #include <utils/fmgrprotos.h> #include <utils/guc.h> #include <utils/regproc.h> #include <utils/timestamp.h> #include <utils/varlena.h> #include "compat/compat.h" #include "config.h" #include "extension.h" #include "guc.h" #include "hypertable_cache.h" #include "license_guc.h" #ifdef USE_TELEMETRY #include "telemetry/telemetry.h" #endif #ifdef USE_TELEMETRY /* Define which level means on. We use this object to have at least one object * of type TelemetryLevel in the code, otherwise pgindent won't work for the * type */ static const TelemetryLevel on_level = TELEMETRY_NO_FUNCTIONS; bool ts_telemetry_on() { return ts_guc_telemetry_level >= on_level; } bool ts_function_telemetry_on() { return ts_guc_telemetry_level > TELEMETRY_NO_FUNCTIONS; } static const struct config_enum_entry telemetry_level_options[] = { { "off", TELEMETRY_OFF, false }, { "no_functions", TELEMETRY_NO_FUNCTIONS, false }, { "basic", TELEMETRY_BASIC, false }, { NULL, 0, false } }; #endif /* Copied from contrib/auto_explain/auto_explain.c */ static const struct config_enum_entry loglevel_options[] = { { "debug5", DEBUG5, false }, { "debug4", DEBUG4, false }, { "debug3", DEBUG3, false }, { "debug2", DEBUG2, false }, { "debug1", DEBUG1, false }, { "debug", DEBUG2, true }, { "info", INFO, false }, { "notice", NOTICE, false }, { "warning", WARNING, false }, { "log", LOG, false }, { "error", ERROR, false }, { "fatal", FATAL, false }, { NULL, 0, false } }; static const struct config_enum_entry compress_truncate_behaviour_options[] = { { "truncate_only", COMPRESS_TRUNCATE_ONLY, false }, { "truncate_or_delete", COMPRESS_TRUNCATE_OR_DELETE, false }, { "truncate_disabled", COMPRESS_TRUNCATE_DISABLED, false }, { NULL, 0, false } }; bool ts_guc_enable_direct_compress_copy = false; bool ts_guc_enable_direct_compress_copy_sort_batches = true; bool ts_guc_enable_direct_compress_copy_client_sorted = false; int ts_guc_direct_compress_copy_tuple_sort_limit = 100000; TSDLLEXPORT bool ts_guc_enable_direct_compress_insert = false; bool ts_guc_enable_direct_compress_insert_sort_batches = true; TSDLLEXPORT bool ts_guc_enable_direct_compress_insert_client_sorted = false; TSDLLEXPORT bool ts_guc_enable_direct_compress_on_cagg_refresh = false; int ts_guc_direct_compress_insert_tuple_sort_limit = 10000; bool ts_guc_enable_deprecation_warnings = true; bool ts_guc_enable_optimizations = true; bool ts_guc_restoring = false; bool ts_guc_enable_constraint_aware_append = true; bool ts_guc_enable_ordered_append = true; bool ts_guc_enable_chunk_append = true; bool ts_guc_enable_parallel_chunk_append = true; bool ts_guc_enable_runtime_exclusion = true; bool ts_guc_enable_constraint_exclusion = true; bool ts_guc_enable_qual_propagation = true; TSDLLEXPORT bool ts_guc_enable_columnar_scan_filter_pushdown = true; bool ts_guc_enable_qual_filtering = true; bool ts_guc_enable_cagg_reorder_groupby = true; TSDLLEXPORT bool ts_guc_enable_cagg_window_functions = false; bool ts_guc_enable_now_constify = true; bool ts_guc_enable_foreign_key_propagation = true; #if PG16_GE TSDLLEXPORT bool ts_guc_enable_cagg_sort_pushdown = true; #endif TSDLLEXPORT bool ts_guc_enable_cagg_watermark_constify = true; TSDLLEXPORT int ts_guc_cagg_max_individual_materializations = 10; bool ts_guc_enable_osm_reads = true; TSDLLEXPORT bool ts_guc_enable_compressed_direct_batch_delete = true; TSDLLEXPORT bool ts_guc_enable_dml_decompression = true; TSDLLEXPORT bool ts_guc_enable_dml_decompression_tuple_filtering = true; TSDLLEXPORT bool ts_guc_enable_dml_bloom_filter = true; TSDLLEXPORT int ts_guc_max_tuples_decompressed_per_dml = 100000; TSDLLEXPORT bool ts_guc_enable_compression_wal_markers = false; TSDLLEXPORT bool ts_guc_enable_decompression_sorted_merge = true; bool ts_guc_enable_chunkwise_aggregation = true; bool ts_guc_enable_vectorized_aggregation = true; TSDLLEXPORT bool ts_guc_enable_compression_indexscan = false; TSDLLEXPORT bool ts_guc_enable_bulk_decompression = true; TSDLLEXPORT bool ts_guc_auto_sparse_indexes = true; TSDLLEXPORT bool ts_guc_enable_sparse_index_bloom = true; TSDLLEXPORT bool ts_guc_enable_composite_bloom_indexes = true; TSDLLEXPORT bool ts_guc_read_legacy_bloom1_v1 = false; bool ts_guc_enable_chunk_skipping = false; TSDLLEXPORT bool ts_guc_enable_segmentwise_recompression = true; TSDLLEXPORT bool ts_guc_enable_in_memory_recompression = true; TSDLLEXPORT bool ts_guc_enable_exclusive_locking_recompression = false; TSDLLEXPORT bool ts_guc_enable_bool_compression = true; TSDLLEXPORT bool ts_guc_enable_uuid_compression = true; TSDLLEXPORT int ts_guc_compression_batch_size_limit = TARGET_COMPRESSED_BATCH_SIZE; TSDLLEXPORT bool ts_guc_compression_enable_compressor_batch_limit = false; TSDLLEXPORT CompressTruncateBehaviour ts_guc_compress_truncate_behaviour = COMPRESS_TRUNCATE_ONLY; bool ts_guc_enable_event_triggers = false; bool ts_guc_enable_chunk_auto_publication = false; bool ts_guc_debug_skip_scan_info = false; /* Only settable in debug mode for testing */ TSDLLEXPORT bool ts_guc_enable_null_compression = true; TSDLLEXPORT bool ts_guc_enable_compression_ratio_warnings = true; /* Enable of disable columnar scans for columnar-oriented storage engines. If * disabled, regular sequence scans will be used instead. */ TSDLLEXPORT bool ts_guc_enable_columnarscan = true; TSDLLEXPORT bool ts_guc_enable_columnarindexscan = true; TSDLLEXPORT int ts_guc_bgw_log_level = WARNING; TSDLLEXPORT bool ts_guc_enable_skip_scan = true; #if PG16_GE TSDLLEXPORT bool ts_guc_enable_skip_scan_for_distinct_aggregates = true; #endif TSDLLEXPORT bool ts_guc_enable_compressed_skip_scan = true; TSDLLEXPORT bool ts_guc_enable_multikey_skip_scan = true; TSDLLEXPORT double ts_guc_skip_scan_run_cost_multiplier = 1.0; static char *ts_guc_default_segmentby_fn = NULL; static char *ts_guc_default_orderby_fn = NULL; TSDLLEXPORT bool ts_guc_enable_job_execution_logging = false; bool ts_guc_enable_tss_callbacks = true; TSDLLEXPORT bool ts_guc_enable_delete_after_compression = false; TSDLLEXPORT bool ts_guc_enable_merge_on_cagg_refresh = false; bool ts_guc_enable_partitioned_hypertables = false; /* default value of ts_guc_max_open_chunks_per_insert and * ts_guc_max_cached_chunks_per_hypertable will be set as their respective boot-value when the * GUC mechanism starts up */ int ts_guc_max_open_chunks_per_insert; int ts_guc_max_cached_chunks_per_hypertable; #ifdef USE_TELEMETRY TelemetryLevel ts_guc_telemetry_level = TELEMETRY_DEFAULT; char *ts_telemetry_cloud = NULL; #endif TSDLLEXPORT char *ts_guc_license = TS_LICENSE_DEFAULT; /* * Exit code for the scheduler. * * Normally it exits with a zero which means that it will not restart. If an * error is raised, it exits with error code 1, which will trigger a * restart. * * This variable exists to be able to trigger a restart for a normal exit, * which is useful when debugging. * * See backend/postmaster/bgworker.c */ int ts_debug_bgw_scheduler_exit_status = 0; #ifdef TS_DEBUG bool ts_shutdown_bgw = false; char *ts_current_timestamp_mock = NULL; #endif int ts_guc_debug_toast_tuple_target = 128; static const struct config_enum_entry debug_require_options[] = { { "allow", DRO_Allow, false }, { "forbid", DRO_Forbid, false }, { "require", DRO_Require, false }, { "force", DRO_Force, false }, { NULL, 0, false } }; #ifdef TS_DEBUG bool ts_guc_debug_have_int128; DebugRequireOption ts_guc_debug_require_vector_qual = DRO_Allow; DebugRequireOption ts_guc_debug_require_vector_agg = DRO_Allow; #endif DebugRequireOption ts_guc_debug_require_batch_sorted_merge = false; bool ts_guc_debug_compression_path_info = false; bool ts_guc_enable_rowlevel_compression_locking = false; static bool ts_guc_enable_hypertable_create = true; static bool ts_guc_enable_hypertable_compression = true; static bool ts_guc_enable_cagg_create = true; static bool ts_guc_enable_policy_create = true; static char *ts_guc_default_chunk_time_interval = NULL; typedef struct { const char *name; const char *description; bool *enable; } FeatureFlag; static FeatureFlag ts_feature_flags[] = { [FEATURE_HYPERTABLE] = { MAKE_EXTOPTION("enable_hypertable_create"), "Enable creation of hypertable", &ts_guc_enable_hypertable_create }, [FEATURE_HYPERTABLE_COMPRESSION] = { MAKE_EXTOPTION("enable_hypertable_compression"), "Enable hypertable compression functions", &ts_guc_enable_hypertable_compression }, [FEATURE_CAGG] = { MAKE_EXTOPTION("enable_cagg_create"), "Enable creation of continuous aggregate", &ts_guc_enable_cagg_create }, [FEATURE_POLICY] = { MAKE_EXTOPTION("enable_policy_create"), "Enable creation of policies and user-defined actions", &ts_guc_enable_policy_create } }; static void ts_feature_flag_add(FeatureFlagType type) { FeatureFlag *flag = &ts_feature_flags[type]; int flag_context = PGC_SIGHUP; #ifdef TS_DEBUG flag_context = PGC_USERSET; #endif DefineCustomBoolVariable(flag->name, flag->description, NULL, flag->enable, true, flag_context, GUC_SUPERUSER_ONLY, NULL, NULL, NULL); } void ts_feature_flag_check(FeatureFlagType type) { FeatureFlag *flag = &ts_feature_flags[type]; if (likely(*flag->enable)) return; ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("You are using a PostgreSQL service. This feature is only available on " "Time-series and analytics services. " "https://docs.timescale.com/use-timescale/latest/services/"))); } /* * We have to understand if we have finished initializing the GUCs, so that we * know when it's OK to check their values for mutual consistency. */ static bool gucs_are_initialized = false; /* * Warn about the mismatched cache sizes that can lead to cache thrashing. */ static void validate_chunk_cache_sizes(int hypertable_chunks, int insert_chunks) { /* * Note that this callback is also called when the individual GUCs are * initialized, so we are going to see temporary mismatched values here. * That's why we also have to check that the GUC initialization have * finished. */ if (gucs_are_initialized && insert_chunks > hypertable_chunks) { ereport(WARNING, (errmsg("insert cache size is larger than hypertable chunk cache size"), errdetail("insert cache size is %d, hypertable chunk cache size is %d", insert_chunks, hypertable_chunks), errhint("This is a configuration problem. Either increase " "timescaledb.max_cached_chunks_per_hypertable (preferred) or decrease " "timescaledb.max_open_chunks_per_insert."))); } } static void assign_max_cached_chunks_per_hypertable_hook(int newval, void *extra) { /* invalidate the hypertable cache to reset */ ts_hypertable_cache_invalidate_callback(); validate_chunk_cache_sizes(newval, ts_guc_max_open_chunks_per_insert); } static void assign_max_open_chunks_per_insert_hook(int newval, void *extra) { validate_chunk_cache_sizes(ts_guc_max_cached_chunks_per_hypertable, newval); } static Oid get_segmentby_func(char *input_name) { List *namelist = NIL; if (strlen(input_name) == 0) { return InvalidOid; } #if PG16_LT namelist = stringToQualifiedNameList(input_name); #else namelist = stringToQualifiedNameList(input_name, NULL); #endif Oid argtyp[] = { REGCLASSOID }; return LookupFuncName(namelist, lengthof(argtyp), argtyp, true); } static bool check_segmentby_func(char **newval, void **extra, GucSource source) { /* if the extension doesn't exist you can't check for the function, have to take it on faith */ if (ts_extension_is_loaded_and_not_upgrading()) { Oid segment_func_oid = get_segmentby_func(*newval); if (strlen(*newval) > 0 && !OidIsValid(segment_func_oid)) { GUC_check_errdetail("Function \"%s\" does not exist.", *newval); return false; } } return true; } Oid ts_guc_default_segmentby_fn_oid() { return get_segmentby_func(ts_guc_default_segmentby_fn); } static Oid get_orderby_func(char *input_name) { List *namelist = NIL; if (strlen(input_name) == 0) { return InvalidOid; } #if PG16_LT namelist = stringToQualifiedNameList(input_name); #else namelist = stringToQualifiedNameList(input_name, NULL); #endif Oid argtyp[] = { REGCLASSOID, TEXTARRAYOID }; return LookupFuncName(namelist, lengthof(argtyp), argtyp, true); } static bool check_orderby_func(char **newval, void **extra, GucSource source) { /* if the extension doesn't exist you can't check for the function, have to take it on faith */ if (ts_extension_is_loaded_and_not_upgrading()) { Oid func_oid = get_orderby_func(*newval); if (strlen(*newval) > 0 && !OidIsValid(func_oid)) { GUC_check_errdetail("Function \"%s\" does not exist.", *newval); return false; } } return true; } Oid ts_guc_default_orderby_fn_oid() { return get_orderby_func(ts_guc_default_orderby_fn); } /* * Assign hook for chunk skipping. * * When chunk skipping is enabled, we need to clear the hypertable cache. * Otherwise there might be cached entries without a valid range_space entry, * which could lead to column stats not being created. */ static void chunk_skipping_assign_hook(bool newval, void *extra) { if (newval) ts_hypertable_cache_invalidate_callback(); } #if PG16_LT /* * guc_malloc is not public in PostgreSQL < 16. */ static void * guc_malloc(int elevel, size_t size) { void *data; /* Avoid unportable behavior of malloc(0) */ if (size == 0) size = 1; data = malloc(size); if (data == NULL) ereport(elevel, (errcode(ERRCODE_OUT_OF_MEMORY), errmsg("out of memory"))); return data; } #endif static bool check_default_chunk_time_interval(char **newval, void **extra, GucSource source) { /* * If GUC is unset, we treat that as a valid value for "no default chunk interval". * The chunk interval is instead computed in the legacy way using hard-coded defaults. */ if (*newval == NULL) { Assert(*extra == NULL); return true; } /* Test that the text value is a valid Interval */ LOCAL_FCINFO(fcinfo, 3); InitFunctionCallInfoData(*fcinfo, NULL, 3, InvalidOid, NULL, NULL); fcinfo->args[0].value = CStringGetDatum(*newval); fcinfo->args[0].isnull = false; fcinfo->args[1].value = ObjectIdGetDatum(INTERVALOID); fcinfo->args[1].isnull = false; fcinfo->args[2].value = Int32GetDatum(-1); fcinfo->args[2].isnull = false; Datum interval = interval_in(fcinfo); if (fcinfo->isnull) { GUC_check_errdetail("The default chunk interval must be a valid INTERVAL."); return false; } Interval *parsed = DatumGetIntervalP(interval); /* Save the new Interval in extra. The old extra is freed automatically. */ *extra = guc_malloc(ERROR, sizeof(Interval)); memcpy(*extra, parsed, sizeof(Interval)); pfree(parsed); return true; } Interval *default_chunk_time_interval = NULL; static void assign_default_chunk_time_interval(const char *newval, void *extra) { default_chunk_time_interval = extra; } void _guc_init(void) { DefineCustomBoolVariable(MAKE_EXTOPTION("enable_deprecation_warnings"), "Enable warnings when using deprecated functionality", NULL, &ts_guc_enable_deprecation_warnings, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_direct_compress_copy"), "Enable direct compression during COPY", "Enable experimental support for direct compression during COPY", &ts_guc_enable_direct_compress_copy, false, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_direct_compress_copy_sort_batches"), "Enable batch sorting during direct compress COPY", NULL, &ts_guc_enable_direct_compress_copy_sort_batches, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_direct_compress_copy_client_sorted"), "Enable direct compress COPY with presorted data", "Correct handling of data sorting by the user is required for this " "option.", &ts_guc_enable_direct_compress_copy_client_sorted, false, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomIntVariable(MAKE_EXTOPTION("direct_compress_copy_tuple_sort_limit"), "Number of tuples that can be sorted at once in a COPY operation", "This is mainly used to keep the memory footprint down for " "operations like importing large amounts of data in " "single transaction. Setting this to 0 would make it unlimited.", &ts_guc_direct_compress_copy_tuple_sort_limit, 100000, 0, 2147483647, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_direct_compress_insert"), "Enable direct compression during INSERT", "Enable experimental support for direct compression during INSERT", &ts_guc_enable_direct_compress_insert, false, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_direct_compress_insert_sort_batches"), "Enable batch sorting during direct compress INSERT", NULL, &ts_guc_enable_direct_compress_insert_sort_batches, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_direct_compress_insert_client_sorted"), "Enable direct compress INSERT with presorted data", "Correct handling of data sorting by the user is required for this " "option.", &ts_guc_enable_direct_compress_insert_client_sorted, false, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_direct_compress_on_cagg_refresh"), "Enable direct compress on Continuous Aggregate refresh", "Enable experimental support for direct compression during Continuous " "Aggregate refresh", &ts_guc_enable_direct_compress_on_cagg_refresh, false, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomIntVariable(MAKE_EXTOPTION("direct_compress_insert_tuple_sort_limit"), "Number of tuples that can be sorted at once in an INSERT operation", "This is mainly used to keep the memory footprint down for " "operations like importing large amounts of data in " "single transaction. Setting this to 0 would make it unlimited.", &ts_guc_direct_compress_insert_tuple_sort_limit, 10000, 0, 2147483647, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_optimizations"), "Enable TimescaleDB query optimizations", NULL, &ts_guc_enable_optimizations, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("restoring"), "Enable restoring mode for timescaledb", "In restoring mode all timescaledb internal hooks are disabled. This " "mode is required for restoring logical dumps of databases with " "timescaledb.", &ts_guc_restoring, false, PGC_SUSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_constraint_aware_append"), "Enable constraint-aware append scans", "Enable constraint exclusion at execution time", &ts_guc_enable_constraint_aware_append, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_ordered_append"), "Enable ordered append scans", "Enable ordered append optimization for queries that are ordered by " "the time dimension", &ts_guc_enable_ordered_append, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_chunk_append"), "Enable chunk append node", "Enable using chunk append node", &ts_guc_enable_chunk_append, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_parallel_chunk_append"), "Enable parallel chunk append node", "Enable using parallel aware chunk append node", &ts_guc_enable_parallel_chunk_append, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_runtime_exclusion"), "Enable runtime chunk exclusion", "Enable runtime chunk exclusion in ChunkAppend node", &ts_guc_enable_runtime_exclusion, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_constraint_exclusion"), "Enable constraint exclusion", "Enable planner constraint exclusion", &ts_guc_enable_constraint_exclusion, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_foreign_key_propagation"), "Enable foreign key propagation", "Adjust foreign key lookup queries to target whole hypertable", &ts_guc_enable_foreign_key_propagation, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_qual_filtering"), "Enable qualifier filtering for chunks", "Filter qualifiers on chunks when complete chunk would be included by " "filter", &ts_guc_enable_qual_filtering, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_qual_propagation"), "Enable qualifier propagation", "Enable propagation of qualifiers in JOINs", &ts_guc_enable_qual_propagation, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_columnar_scan_filter_pushdown"), "Enable columnar scan filter pushdown", "Enable pushing down the filters into the compressed scan part of the " "columnar scan", &ts_guc_enable_columnar_scan_filter_pushdown, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_dml_decompression"), "Enable DML decompression", "Enable DML decompression when modifying compressed hypertable", &ts_guc_enable_dml_decompression, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_dml_decompression_tuple_filtering"), "Enable DML decompression tuple filtering", "Recheck tuples during DML decompression to only decompress batches " "with matching tuples", &ts_guc_enable_dml_decompression_tuple_filtering, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_dml_bloom_filter"), "Enable bloom filter pruning for DML on compressed chunks", "When enabled, bloom filters are used to skip compressed batches " "that definitely do not contain matching rows during DELETE and " "UPDATE operations, reducing decompression overhead.", &ts_guc_enable_dml_bloom_filter, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_compressed_direct_batch_delete"), "Enable direct deletion of compressed batches", "Enable direct batch deletion in compressed chunks", &ts_guc_enable_compressed_direct_batch_delete, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomIntVariable(MAKE_EXTOPTION("max_tuples_decompressed_per_dml_transaction"), "The max number of tuples that can be decompressed during an " "INSERT, UPDATE, or DELETE.", " If the number of tuples exceeds this value, an error will " "be thrown and transaction rolled back. " "Setting this to 0 sets this value to unlimited number of " "tuples decompressed.", &ts_guc_max_tuples_decompressed_per_dml, 100000, 0, 2147483647, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_skipscan"), "Enable SkipScan", "Enable SkipScan for DISTINCT queries", &ts_guc_enable_skip_scan, true, PGC_USERSET, 0, NULL, NULL, NULL); #if PG16_GE DefineCustomBoolVariable(MAKE_EXTOPTION("enable_skipscan_for_distinct_aggregates"), "Enable SkipScan for DISTINCT aggregates", "Enable SkipScan for DISTINCT aggregates", &ts_guc_enable_skip_scan_for_distinct_aggregates, true, PGC_USERSET, 0, NULL, NULL, NULL); #endif DefineCustomBoolVariable(MAKE_EXTOPTION("enable_compressed_skipscan"), "Enable SkipScan for compressed chunks", "Enable SkipScan for distinct inputs over compressed chunks", &ts_guc_enable_compressed_skip_scan, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_multikey_skipscan"), "Enable SkipScan for multiple distinct keys", "Enable SkipScan for multiple distinct inputs", &ts_guc_enable_multikey_skip_scan, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomRealVariable(MAKE_EXTOPTION("skip_scan_run_cost_multiplier"), "Multiplier for SkipScan run cost as an option to make the cost " "smaller so that SkipScan can be chosen", "Default is 1.0 i.e. regularly estimated SkipScan run cost, 0.0 will " "make SkipScan to have run cost = 0", &ts_guc_skip_scan_run_cost_multiplier, 1.0, 0.0, 1.0, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("debug_skip_scan_info"), "Print debug info about SkipScan", "Print debug info about SkipScan distinct columns", &ts_guc_debug_skip_scan_info, false, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_compression_wal_markers"), "Enable WAL markers for compression ops", "Enable the generation of markers in the WAL stream which mark the " "start and end of compression operations", &ts_guc_enable_compression_wal_markers, true, PGC_SIGHUP, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_decompression_sorted_merge"), "Enable compressed batches heap merge", "Enable the merge of compressed batches to preserve the compression " "order by", &ts_guc_enable_decompression_sorted_merge, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_cagg_reorder_groupby"), "Enable group by reordering", "Enable group by clause reordering for continuous aggregates", &ts_guc_enable_cagg_reorder_groupby, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_cagg_window_functions"), "Enable window functions in continuous aggregates", "Allow window functions in continuous aggregate views", &ts_guc_enable_cagg_window_functions, false, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_now_constify"), "Enable now() constify", "Enable constifying now() in query constraints", &ts_guc_enable_now_constify, true, PGC_USERSET, 0, NULL, NULL, NULL); #if PG16_GE DefineCustomBoolVariable(MAKE_EXTOPTION("enable_cagg_sort_pushdown"), "Enable sort pushdown for continuous aggregates", "Enable pushdown of ORDER BY clause for continuous aggregates", &ts_guc_enable_cagg_sort_pushdown, true, PGC_USERSET, 0, NULL, NULL, NULL); #endif DefineCustomBoolVariable(MAKE_EXTOPTION("enable_cagg_watermark_constify"), "Enable cagg watermark constify", "Enable constifying cagg watermark for real-time caggs", &ts_guc_enable_cagg_watermark_constify, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_merge_on_cagg_refresh"), "Enable MERGE statement on cagg refresh", "Enable MERGE statement on cagg refresh", &ts_guc_enable_merge_on_cagg_refresh, false, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_chunk_skipping"), "Enable chunk skipping functionality", "Enable using chunk column stats to filter chunks based on column " "filters", &ts_guc_enable_chunk_skipping, false, PGC_USERSET, 0, NULL, chunk_skipping_assign_hook, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_segmentwise_recompression"), "Enable segmentwise recompression functionality", "Enable segmentwise recompression", &ts_guc_enable_segmentwise_recompression, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_in_memory_recompression"), "Enable in-memory recompression functionality", "Enable in-memory recompression", &ts_guc_enable_in_memory_recompression, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_exclusive_locking_recompression"), "Enable exclusive locking recompression", "Enable getting exclusive lock on chunk during segmentwise " "recompression", &ts_guc_enable_exclusive_locking_recompression, false, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_bool_compression"), "Enable bool compression functionality", "Enable bool compression", &ts_guc_enable_bool_compression, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_uuid_compression"), "Enable uuid compression functionality", "Enable uuid compression", &ts_guc_enable_uuid_compression, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomIntVariable(MAKE_EXTOPTION("compression_batch_size_limit"), "The max number of tuples that can be batched together during " "compression", "Setting this option to a number between 1 and 32767 will force " "compression " "to limit the size of compressed batches to that amount of " "uncompressed tuples. The setting influences only the compression " "process itself. The value of the setting is taken from the context " "of the session where the compression is performed. It is not " "persisted in any way.", &ts_guc_compression_batch_size_limit, TARGET_COMPRESSED_BATCH_SIZE, 1, GLOBAL_MAX_ROWS_PER_COMPRESSION, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_compressor_batch_limit"), "Enable compressor batch limit", "Enable compressor batch limit for compressors which " "can go over the allocation limit (1 GB). This feature will " "limit those compressors by reducing the size of the batch and thus " "avoid hitting the limit.", &ts_guc_compression_enable_compressor_batch_limit, false, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_event_triggers"), "Enable event triggers for chunks creation", "Enable event triggers for chunks creation", &ts_guc_enable_event_triggers, false, PGC_SUSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_chunk_auto_publication"), "Enable automatic chunk publication", "Enable automatically adding newly created chunks to the publication " "of their hypertable", &ts_guc_enable_chunk_auto_publication, false, PGC_USERSET, 0, NULL, NULL, NULL); #ifdef TS_DEBUG DefineCustomBoolVariable(MAKE_EXTOPTION("enable_null_compression"), "Debug only flag to enable NULL compression", "Enable null compression", &ts_guc_enable_null_compression, true, PGC_USERSET, 0, NULL, NULL, NULL); #endif DefineCustomBoolVariable(MAKE_EXTOPTION("enable_compression_ratio_warnings"), "Enable warnings for poor compression ratio", "Enable warnings for poor compression ratio", &ts_guc_enable_compression_ratio_warnings, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_tiered_reads"), "Enable tiered data reads", "Enable reading of tiered data by including a foreign table " "representing the data in the object storage into the query plan", &ts_guc_enable_osm_reads, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_chunkwise_aggregation"), "Enable chunk-wise aggregation", "Enable the pushdown of aggregations to the" " chunk level", &ts_guc_enable_chunkwise_aggregation, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_vectorized_aggregation"), "Enable vectorized aggregation", "Enable vectorized aggregation for compressed data", &ts_guc_enable_vectorized_aggregation, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_compression_indexscan"), "Enable compression to take indexscan path", "Enable indexscan during compression, if matching index is found", &ts_guc_enable_compression_indexscan, false, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_bulk_decompression"), "Enable decompression of the entire compressed batches", "Increases throughput of decompression, but might increase query " "memory usage", &ts_guc_enable_bulk_decompression, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("auto_sparse_indexes"), "Create sparse indexes on compressed chunks", "The hypertable columns that are used as index keys will have " "suitable sparse indexes when compressed. Must be set at the moment " "of chunk compression, e.g. when the `compress_chunk()` is called.", &ts_guc_auto_sparse_indexes, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_sparse_index_bloom"), "Enable creation of the bloom1 sparse index on compressed chunks", "This sparse index speeds up the equality queries on compressed " "columns, and can be disabled when not desired.", &ts_guc_enable_sparse_index_bloom, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_composite_bloom_indexes"), "Enable creation of the bloom1 composite index on compressed chunks", "This composite index speeds up the equality queries on compressed " "columns, and can be disabled when not desired.", &ts_guc_enable_composite_bloom_indexes, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("read_legacy_bloom1_v1"), "Enable reading the legacy bloom1 version 1 sparse indexes for SELECT " "queries", "These legacy indexes might give false negatives if they were built " "by the TimescaleDB extension compiled with different build options.", &ts_guc_read_legacy_bloom1_v1, false, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_columnarscan"), "Enable ColumnarScan for columnar storage", "Transparently decompress columnar data using ColumnarScan custom " "node. Disabling columnar scan will ignore data stored in columnar " "format in queries.", &ts_guc_enable_columnarscan, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_columnarindexscan"), "Enable metadata-only optimization for ColumnarScans", "Enable experimental support for returning results directly from " "compression metadata without decompression", &ts_guc_enable_columnarindexscan, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomIntVariable(MAKE_EXTOPTION("max_open_chunks_per_insert"), "Maximum open chunks per insert", "Maximum number of open chunk tables per insert", &ts_guc_max_open_chunks_per_insert, 1024, 0, PG_INT16_MAX, PGC_USERSET, 0, NULL, assign_max_open_chunks_per_insert_hook, NULL); DefineCustomIntVariable(MAKE_EXTOPTION("max_cached_chunks_per_hypertable"), "Maximum cached chunks", "Maximum number of chunks stored in the cache", &ts_guc_max_cached_chunks_per_hypertable, 1024, 0, 65536, PGC_USERSET, 0, NULL, assign_max_cached_chunks_per_hypertable_hook, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_job_execution_logging"), "Enable job execution logging", "Retain job run status in logging table", &ts_guc_enable_job_execution_logging, false, PGC_SIGHUP, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_tss_callbacks"), "Enable ts_stat_statements callbacks", "Enable ts_stat_statements callbacks", &ts_guc_enable_tss_callbacks, true, PGC_SUSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable(MAKE_EXTOPTION("enable_delete_after_compression"), "Delete all rows after compression instead of truncate", "Delete all rows after compression instead of truncate", &ts_guc_enable_delete_after_compression, false, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomEnumVariable(MAKE_EXTOPTION("compress_truncate_behaviour"), "Define behaviour of truncate after compression", "Defines how truncate behaves at the end of compression. " "'truncate_only' forces truncation. 'truncate_disabled' deletes rows " "instead of truncate. 'truncate_or_delete' allows falling back to " "deletion.", (int *) &ts_guc_compress_truncate_behaviour, COMPRESS_TRUNCATE_ONLY, compress_truncate_behaviour_options, PGC_USERSET, 0, NULL, NULL, NULL); #ifdef TS_DEBUG DefineCustomBoolVariable(MAKE_EXTOPTION("enable_partitioned_hypertables"), "Enable hypertables using declarative partitioning", "Enable experimental support for creating hypertables using " "PostgreSQL's native declarative partitioning", &ts_guc_enable_partitioned_hypertables, false, PGC_USERSET, 0, NULL, NULL, NULL); #endif #ifdef USE_TELEMETRY DefineCustomEnumVariable(MAKE_EXTOPTION("telemetry_level"), "Telemetry settings level", "Level used to determine which telemetry to send", (int *) &ts_guc_telemetry_level, TELEMETRY_DEFAULT, telemetry_level_options, PGC_USERSET, 0, NULL, NULL, NULL); #endif DefineCustomStringVariable(/* name= */ MAKE_EXTOPTION("compression_segmentby_default_function"), /* short_desc= */ "Function that sets default segment_by", /* long_desc= */ "Function to use for calculating default segment_by setting for " "compression", /* valueAddr= */ &ts_guc_default_segmentby_fn, /* Value= */ "_timescaledb_functions.get_segmentby_defaults", /* context= */ PGC_USERSET, /* flags= */ 0, /* check_hook= */ check_segmentby_func, /* assign_hook= */ NULL, /* show_hook= */ NULL); DefineCustomStringVariable(/* name= */ MAKE_EXTOPTION("compression_orderby_default_function"), /* short_desc= */ "Function that sets default order_by", /* long_desc= */ "Function to use for calculating default order_by setting for " "compression", /* valueAddr= */ &ts_guc_default_orderby_fn, /* Value= */ "_timescaledb_functions.get_orderby_defaults", /* context= */ PGC_USERSET, /* flags= */ 0, /* check_hook= */ check_orderby_func, /* assign_hook= */ NULL, /* show_hook= */ NULL); DefineCustomStringVariable(/* name= */ MAKE_EXTOPTION("license"), /* short_desc= */ "TimescaleDB license type", /* long_desc= */ "Determines which features are enabled", /* valueAddr= */ &ts_guc_license, /* bootValue= */ TS_LICENSE_DEFAULT, /* context= */ PGC_SUSET, /* flags= */ 0, /* check_hook= */ ts_license_guc_check_hook, /* assign_hook= */ ts_license_guc_assign_hook, /* show_hook= */ NULL); DefineCustomEnumVariable(MAKE_EXTOPTION("bgw_log_level"), "Log level for the background worker subsystem", "Log level for the scheduler and workers of the background worker " "subsystem. Requires configuration reload to change.", /* valueAddr= */ &ts_guc_bgw_log_level, /* bootValue= */ WARNING, /* options= */ loglevel_options, /* context= */ PGC_SUSET, 0, NULL, NULL, NULL); /* this information is useful in general on customer deployments */ DefineCustomBoolVariable(/* name= */ MAKE_EXTOPTION("debug_compression_path_info"), /* short_desc= */ "show various compression-related debug info", /* long_desc= */ "this is for debugging/information purposes", /* valueAddr= */ &ts_guc_debug_compression_path_info, /* bootValue= */ false, /* context= */ PGC_USERSET, /* flags= */ 0, /* check_hook= */ NULL, /* assign_hook= */ NULL, /* show_hook= */ NULL); DefineCustomBoolVariable(/* name= */ MAKE_EXTOPTION("enable_rowlevel_compression_locking"), /* short_desc= */ "Use rowlevel locking during compression", /* long_desc= */ "Use only if you know what you are doing", /* valueAddr= */ &ts_guc_enable_rowlevel_compression_locking, /* bootValue= */ false, /* context= */ PGC_USERSET, /* flags= */ 0, /* check_hook= */ NULL, /* assign_hook= */ NULL, /* show_hook= */ NULL); #ifdef USE_TELEMETRY DefineCustomStringVariable(/* name= */ "timescaledb_telemetry.cloud", /* short_desc= */ "cloud provider", /* long_desc= */ "cloud provider used for this instance", /* valueAddr= */ &ts_telemetry_cloud, /* bootValue= */ NULL, /* context= */ PGC_SIGHUP, /* flags= */ 0, /* check_hook= */ NULL, /* assign_hook= */ NULL, /* show_hook= */ NULL); #endif DefineCustomIntVariable(/* name= */ MAKE_EXTOPTION("debug_bgw_scheduler_exit_status"), /* short_desc= */ "exit status to use when shutting down the scheduler", /* long_desc= */ "this is for debugging purposes", /* valueAddr= */ &ts_debug_bgw_scheduler_exit_status, /* bootValue= */ 0, /* minValue= */ 0, /* maxValue= */ 255, /* context= */ PGC_SIGHUP, /* flags= */ 0, /* check_hook= */ NULL, /* assign_hook= */ NULL, /* show_hook= */ NULL); DefineCustomEnumVariable(/* name= */ MAKE_EXTOPTION("debug_require_batch_sorted_merge"), /* short_desc= */ "require batch sorted merge in ColumnarScan node", /* long_desc= */ "this is for debugging purposes", /* valueAddr= */ (int *) &ts_guc_debug_require_batch_sorted_merge, /* bootValue= */ DRO_Allow, /* options = */ debug_require_options, /* context= */ PGC_USERSET, /* flags= */ 0, /* check_hook= */ NULL, /* assign_hook= */ NULL, /* show_hook= */ NULL); DefineCustomStringVariable(/* name= */ MAKE_EXTOPTION("default_chunk_time_interval"), /* short_desc= */ "Default chunk time interval for new hypertables", /* long_desc= */ "Chunk time interval to use for a new hypertable, unless a specific " "chunk time interval is set on the hypertable. The default chunk " "interval is only used for hypertables with a compatible time " "type, e.g., timestamp, date, and UUID (v7). Hypertables using an " "integer partitioning column have hard-coded defaults." "Expert-level setting. These parameters are optimized for internal " "workflows; incorrect configurations can negatively impact query " "performance and system efficiency.", /* valueAddr= */ &ts_guc_default_chunk_time_interval, NULL, /* context= */ PGC_USERSET, /* flags= */ 0, /* check_hook= */ check_default_chunk_time_interval, /* assign_hook= */ assign_default_chunk_time_interval, /* show_hook= */ NULL); #ifdef TS_DEBUG DefineCustomBoolVariable(/* name= */ MAKE_EXTOPTION("shutdown_bgw_scheduler"), /* short_desc= */ "immediately shutdown the bgw scheduler", /* long_desc= */ "this is for debugging purposes", /* valueAddr= */ &ts_shutdown_bgw, /* bootValue= */ false, /* context= */ PGC_SIGHUP, /* flags= */ 0, /* check_hook= */ NULL, /* assign_hook= */ NULL, /* show_hook= */ NULL); DefineCustomStringVariable(/* name= */ MAKE_EXTOPTION("current_timestamp_mock"), /* short_desc= */ "set the current timestamp", /* long_desc= */ "this is for debugging purposes", /* valueAddr= */ &ts_current_timestamp_mock, /* bootValue= */ NULL, /* context= */ PGC_USERSET, /* flags= */ 0, /* check_hook= */ NULL, /* assign_hook= */ NULL, /* show_hook= */ NULL); DefineCustomIntVariable(/* name= */ MAKE_EXTOPTION("debug_toast_tuple_target"), /* short_desc= */ "set toast tuple target on compressed chunks", /* long_desc= */ "this is for debugging purposes", /* valueAddr= */ &ts_guc_debug_toast_tuple_target, /* bootValue = */ 128, /* minValue = */ 1, /* maxValue = */ 65535, /* context= */ PGC_USERSET, /* flags= */ 0, /* check_hook= */ NULL, /* assign_hook= */ NULL, /* show_hook= */ NULL); DefineCustomBoolVariable(/* name= */ MAKE_EXTOPTION("debug_have_int128"), /* short_desc= */ "whether we have int128 support", /* long_desc= */ "this is for debugging purposes", /* valueAddr= */ &ts_guc_debug_have_int128, #ifdef HAVE_INT128 /* bootValue= */ true, #else /* bootValue= */ false, #endif /* context= */ PGC_INTERNAL, /* flags= */ 0, /* check_hook= */ NULL, /* assign_hook= */ NULL, /* show_hook= */ NULL); DefineCustomEnumVariable(/* name= */ MAKE_EXTOPTION("debug_require_vector_agg"), /* short_desc= */ "ensure that vectorized aggregation is used or not", /* long_desc= */ "this is for debugging purposes", /* valueAddr= */ (int *) &ts_guc_debug_require_vector_agg, /* bootValue= */ DRO_Allow, /* options = */ debug_require_options, /* context= */ PGC_USERSET, /* flags= */ 0, /* check_hook= */ NULL, /* assign_hook= */ NULL, /* show_hook= */ NULL); DefineCustomEnumVariable(/* name= */ MAKE_EXTOPTION("debug_require_vector_qual"), /* short_desc= */ "ensure that non-vectorized or vectorized filters are used in " "ColumnarScan node", /* long_desc= */ "this is for debugging purposes, to let us check if the vectorized " "quals are used or not. EXPLAIN differs after PG15 for custom nodes, " "and " "using the test templates is a pain", /* valueAddr= */ (int *) &ts_guc_debug_require_vector_qual, /* bootValue= */ DRO_Allow, /* options = */ debug_require_options, /* context= */ PGC_USERSET, /* flags= */ 0, /* check_hook= */ NULL, /* assign_hook= */ NULL, /* show_hook= */ NULL); #endif /* register feature flags */ ts_feature_flag_add(FEATURE_HYPERTABLE); ts_feature_flag_add(FEATURE_HYPERTABLE_COMPRESSION); ts_feature_flag_add(FEATURE_CAGG); ts_feature_flag_add(FEATURE_POLICY); gucs_are_initialized = true; validate_chunk_cache_sizes(ts_guc_max_cached_chunks_per_hypertable, ts_guc_max_open_chunks_per_insert); } ================================================ FILE: src/guc.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include "compat/compat.h" #include "config.h" #include "export.h" #ifdef USE_TELEMETRY extern bool ts_telemetry_on(void); extern bool ts_function_telemetry_on(void); #endif extern bool ts_guc_enable_deprecation_warnings; extern bool ts_guc_enable_optimizations; extern bool ts_guc_enable_constraint_aware_append; extern bool ts_guc_enable_ordered_append; extern bool ts_guc_enable_chunk_append; extern bool ts_guc_enable_parallel_chunk_append; extern bool ts_guc_enable_qual_propagation; extern TSDLLEXPORT bool ts_guc_enable_columnar_scan_filter_pushdown; extern bool ts_guc_enable_qual_filtering; extern bool ts_guc_enable_runtime_exclusion; extern bool ts_guc_enable_constraint_exclusion; extern bool ts_guc_enable_cagg_reorder_groupby; extern TSDLLEXPORT bool ts_guc_enable_cagg_window_functions; extern TSDLLEXPORT int ts_guc_cagg_max_individual_materializations; extern bool ts_guc_enable_now_constify; extern bool ts_guc_enable_foreign_key_propagation; extern TSDLLEXPORT bool ts_guc_enable_osm_reads; #if PG16_GE extern TSDLLEXPORT bool ts_guc_enable_cagg_sort_pushdown; #endif extern TSDLLEXPORT bool ts_guc_enable_cagg_watermark_constify; extern TSDLLEXPORT bool ts_guc_enable_dml_decompression; extern TSDLLEXPORT bool ts_guc_enable_dml_decompression_tuple_filtering; extern TSDLLEXPORT bool ts_guc_enable_dml_bloom_filter; extern bool ts_guc_enable_direct_compress_copy; extern bool ts_guc_enable_direct_compress_copy_sort_batches; extern bool ts_guc_enable_direct_compress_copy_client_sorted; extern int ts_guc_direct_compress_copy_tuple_sort_limit; extern TSDLLEXPORT bool ts_guc_enable_direct_compress_insert; extern bool ts_guc_enable_direct_compress_insert_sort_batches; extern TSDLLEXPORT bool ts_guc_enable_direct_compress_insert_client_sorted; extern TSDLLEXPORT bool ts_guc_enable_direct_compress_on_cagg_refresh; extern int ts_guc_direct_compress_insert_tuple_sort_limit; extern TSDLLEXPORT bool ts_guc_enable_compressed_direct_batch_delete; extern TSDLLEXPORT int ts_guc_max_tuples_decompressed_per_dml; extern TSDLLEXPORT bool ts_guc_enable_compression_wal_markers; extern TSDLLEXPORT bool ts_guc_enable_decompression_sorted_merge; extern TSDLLEXPORT bool ts_guc_enable_skip_scan; extern TSDLLEXPORT bool ts_guc_enable_chunkwise_aggregation; extern TSDLLEXPORT bool ts_guc_enable_vectorized_aggregation; extern bool ts_guc_restoring; extern int ts_guc_max_open_chunks_per_insert; extern int ts_guc_max_cached_chunks_per_hypertable; extern TSDLLEXPORT bool ts_guc_enable_job_execution_logging; extern bool ts_guc_enable_tss_callbacks; extern TSDLLEXPORT bool ts_guc_enable_delete_after_compression; extern TSDLLEXPORT bool ts_guc_enable_merge_on_cagg_refresh; extern bool ts_guc_enable_chunk_skipping; extern TSDLLEXPORT bool ts_guc_enable_segmentwise_recompression; extern TSDLLEXPORT bool ts_guc_enable_in_memory_recompression; extern TSDLLEXPORT bool ts_guc_enable_exclusive_locking_recompression; extern TSDLLEXPORT bool ts_guc_enable_bool_compression; extern TSDLLEXPORT bool ts_guc_enable_uuid_compression; extern TSDLLEXPORT int ts_guc_compression_batch_size_limit; extern TSDLLEXPORT bool ts_guc_compression_enable_compressor_batch_limit; #if PG16_GE extern TSDLLEXPORT bool ts_guc_enable_skip_scan_for_distinct_aggregates; #endif extern bool ts_guc_enable_event_triggers; extern bool ts_guc_enable_chunk_auto_publication; extern TSDLLEXPORT bool ts_guc_enable_compressed_skip_scan; extern TSDLLEXPORT bool ts_guc_enable_multikey_skip_scan; extern TSDLLEXPORT double ts_guc_skip_scan_run_cost_multiplier; extern TSDLLEXPORT bool ts_guc_debug_skip_scan_info; /* Only settable in debug mode for testing */ extern TSDLLEXPORT bool ts_guc_enable_null_compression; extern TSDLLEXPORT bool ts_guc_enable_compression_ratio_warnings; typedef enum CompressTruncateBehaviour { COMPRESS_TRUNCATE_ONLY, COMPRESS_TRUNCATE_OR_DELETE, COMPRESS_TRUNCATE_DISABLED, } CompressTruncateBehaviour; extern TSDLLEXPORT CompressTruncateBehaviour ts_guc_compress_truncate_behaviour; #ifdef USE_TELEMETRY typedef enum TelemetryLevel { TELEMETRY_OFF, TELEMETRY_NO_FUNCTIONS, TELEMETRY_BASIC, } TelemetryLevel; extern TelemetryLevel ts_guc_telemetry_level; extern char *ts_telemetry_cloud; #endif extern TSDLLEXPORT char *ts_guc_license; extern TSDLLEXPORT bool ts_guc_enable_compression_indexscan; extern TSDLLEXPORT bool ts_guc_enable_bulk_decompression; extern TSDLLEXPORT bool ts_guc_auto_sparse_indexes; extern TSDLLEXPORT bool ts_guc_enable_sparse_index_bloom; extern TSDLLEXPORT bool ts_guc_enable_composite_bloom_indexes; extern TSDLLEXPORT bool ts_guc_read_legacy_bloom1_v1; extern TSDLLEXPORT bool ts_guc_enable_columnarscan; extern TSDLLEXPORT bool ts_guc_enable_columnarindexscan; extern TSDLLEXPORT int ts_guc_bgw_log_level; /* * Exit code to use when scheduler exits. * * Used for debugging. */ extern TSDLLEXPORT int ts_debug_bgw_scheduler_exit_status; #ifdef TS_DEBUG extern bool ts_shutdown_bgw; extern char *ts_current_timestamp_mock; #else #define ts_shutdown_bgw false #endif extern TSDLLEXPORT int ts_guc_debug_toast_tuple_target; typedef enum DebugRequireOption { DRO_Allow = 0, DRO_Forbid, DRO_Require, DRO_Force, } DebugRequireOption; #ifdef TS_DEBUG extern TSDLLEXPORT DebugRequireOption ts_guc_debug_require_vector_qual; extern TSDLLEXPORT DebugRequireOption ts_guc_debug_require_vector_agg; #endif extern TSDLLEXPORT bool ts_guc_debug_compression_path_info; extern TSDLLEXPORT bool ts_guc_enable_rowlevel_compression_locking; extern TSDLLEXPORT DebugRequireOption ts_guc_debug_require_batch_sorted_merge; extern bool ts_guc_enable_partitioned_hypertables; void _guc_init(void); typedef enum { FEATURE_HYPERTABLE, FEATURE_HYPERTABLE_COMPRESSION, FEATURE_CAGG, FEATURE_POLICY } FeatureFlagType; extern TSDLLEXPORT void ts_feature_flag_check(FeatureFlagType); extern TSDLLEXPORT Oid ts_guc_default_segmentby_fn_oid(void); extern TSDLLEXPORT Oid ts_guc_default_orderby_fn_oid(void); #define TARGET_COMPRESSED_BATCH_SIZE 1000 /* * We use this limit for sanity checks in case the compressed data is corrupt. */ #define GLOBAL_MAX_ROWS_PER_COMPRESSION INT16_MAX ================================================ FILE: src/histogram.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_type.h> #include <libpq/pqformat.h> #include <netinet/in.h> #include <nodes/makefuncs.h> #include <utils/array.h> #include <utils/builtins.h> #include <utils/lsyscache.h> #include "compat/compat.h" #include "debug_assert.h" #include "utils.h" /* aggregate histogram: * histogram(state, val, min, max, nbuckets) returns the histogram array with nbuckets * * Usage: * SELECT grouping_element, histogram(field, min, max, nbuckets) FROM table GROUP BY *grouping_element. * * Description: * Histogram generates a histogram array based off of a specified range passed into the function. * Values falling outside of this range are bucketed into the 0 or nbucket+1 buckets depending on * if they are below or above the range, respectively. The resultant histogram therefore contains * nbucket+2 buckets accounting for buckets outside the range. */ TS_FUNCTION_INFO_V1(ts_hist_sfunc); TS_FUNCTION_INFO_V1(ts_hist_combinefunc); TS_FUNCTION_INFO_V1(ts_hist_serializefunc); TS_FUNCTION_INFO_V1(ts_hist_deserializefunc); TS_FUNCTION_INFO_V1(ts_hist_finalfunc); #define HISTOGRAM_SIZE(state, nbuckets) \ (sizeof(*(state)) + ((nbuckets) * sizeof(*(state)->buckets))) typedef struct Histogram { int32 nbuckets; Datum buckets[FLEXIBLE_ARRAY_MEMBER]; } Histogram; /* histogram(state, val, min, max, nbuckets) */ Datum ts_hist_sfunc(PG_FUNCTION_ARGS) { MemoryContext aggcontext; Histogram *state = (Histogram *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0)); Datum val_datum = PG_GETARG_DATUM(1); Datum min_datum = PG_GETARG_DATUM(2); Datum max_datum = PG_GETARG_DATUM(3); double min = DatumGetFloat8(min_datum); double max = DatumGetFloat8(max_datum); int nbuckets; if (!AggCheckCallContext(fcinfo, &aggcontext)) { /* cannot be called directly because of internal-type argument */ ereport(ERROR, (errmsg("ts_hist_sfunc called in non-aggregate context"))); } if (min > max) { /* cannot generate a histogram with incompatible bounds */ ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("lower bound cannot exceed upper bound"))); } if (state == NULL) { nbuckets = PG_GETARG_INT32(4) + 2; /* Allocate memory to a new histogram state array */ state = MemoryContextAllocZero(aggcontext, HISTOGRAM_SIZE(state, nbuckets)); state->nbuckets = nbuckets; } /* Since the number of buckets is an argument to the calls it might differ * from the number we initialized with so we need to make sure we check * against what we actually have. */ nbuckets = state->nbuckets - 2; if (nbuckets != PG_GETARG_INT32(4)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("number of buckets must not change between calls"))); int32 bucket = DatumGetInt32(DirectFunctionCall4(width_bucket_float8, val_datum, min_datum, max_datum, Int32GetDatum(nbuckets))); /* Increment the proper histogram bucket */ if (bucket < 0 || bucket >= state->nbuckets) { ereport(ERROR, (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), errmsg("index %d from \"width_bucket\" out of range", bucket), errhint("You probably have a floating point overflow."))); } if (DatumGetInt32(state->buckets[bucket]) >= PG_INT32_MAX - 1) { elog(ERROR, "overflow in histogram"); } state->buckets[bucket] = Int32GetDatum(DatumGetInt32(state->buckets[bucket]) + 1); PG_RETURN_POINTER(state); } /* Make a copy of the histogram state */ static inline Histogram * copy_state(MemoryContext aggcontext, Histogram *state) { Histogram *copy; Size bucket_bytes = state->nbuckets * sizeof(*copy->buckets); copy = MemoryContextAlloc(aggcontext, sizeof(*copy) + bucket_bytes); copy->nbuckets = state->nbuckets; memcpy(copy->buckets, state->buckets, bucket_bytes); return copy; } /* ts_hist_combinefunc(internal, internal) => internal */ Datum ts_hist_combinefunc(PG_FUNCTION_ARGS) { MemoryContext aggcontext; Histogram *state1 = (Histogram *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0)); Histogram *state2 = (Histogram *) (PG_ARGISNULL(1) ? NULL : PG_GETARG_POINTER(1)); Histogram *result; if (!AggCheckCallContext(fcinfo, &aggcontext)) { /* cannot be called directly because of internal-type argument */ elog(ERROR, "ts_hist_combinefunc called in non-aggregate context"); } if (state1 == NULL && state2 == NULL) { PG_RETURN_NULL(); } else if (state2 == NULL) { result = copy_state(aggcontext, state1); } else if (state1 == NULL) { result = copy_state(aggcontext, state2); } else { /* Since number of buckets is part of the aggregation call the initialization * might be different in the partials so we error out if they are not identical. */ if (state1->nbuckets != state2->nbuckets) elog(ERROR, "number of buckets must not change between calls"); result = copy_state(aggcontext, state1); /* Combine values from state1 and state2 when both states are non-null */ for (int32 i = 0; i < state1->nbuckets; i++) { /* Perform addition using int64 to check for overflow */ int64 val = (int64) DatumGetInt32(result->buckets[i]); int64 other = (int64) DatumGetInt32(state2->buckets[i]); if (val + other >= PG_INT32_MAX) elog(ERROR, "overflow in histogram combine"); result->buckets[i] = Int32GetDatum((int32) (val + other)); } } PG_RETURN_POINTER(result); } /* ts_hist_serializefunc(internal) => bytea */ Datum ts_hist_serializefunc(PG_FUNCTION_ARGS) { Histogram *state; StringInfoData buf; Assert(!PG_ARGISNULL(0)); state = (Histogram *) PG_GETARG_POINTER(0); pq_begintypsend(&buf); pq_sendint32(&buf, state->nbuckets); for (int32 i = 0; i < state->nbuckets; i++) pq_sendint32(&buf, DatumGetInt32(state->buckets[i])); PG_RETURN_BYTEA_P(pq_endtypsend(&buf)); } /* ts_hist_deserializefunc(bytea *, internal) => internal */ Datum ts_hist_deserializefunc(PG_FUNCTION_ARGS) { MemoryContext aggcontext; bytea *serialized; int32 nbuckets; int32 i; StringInfoData buf; Histogram *state; if (!AggCheckCallContext(fcinfo, &aggcontext)) elog(ERROR, "ts_hist_deserializefunc called in non-aggregate context"); Assert(!PG_ARGISNULL(0)); serialized = PG_GETARG_BYTEA_P(0); buf.data = VARDATA(serialized); buf.len = VARSIZE(serialized) - VARHDRSZ; buf.maxlen = VARSIZE(serialized) - VARHDRSZ; buf.cursor = 0; /* used by pq_getmsgint*/ nbuckets = pq_getmsgint(&buf, 4); state = MemoryContextAllocZero(aggcontext, HISTOGRAM_SIZE(state, nbuckets)); state->nbuckets = nbuckets; for (i = 0; i < state->nbuckets; i++) state->buckets[i] = Int32GetDatum(pq_getmsgint(&buf, 4)); PG_RETURN_POINTER(state); } /* hist_finalfunc(internal, val REAL, MIN REAL, MAX REAL, nbuckets INTEGER) => INTEGER[] */ Datum ts_hist_finalfunc(PG_FUNCTION_ARGS) { Histogram *state; int dims[1]; int lbs[1]; if (!AggCheckCallContext(fcinfo, NULL)) { /* cannot be called directly because of internal-type argument */ elog(ERROR, "ts_hist_finalfunc called in non-aggregate context"); } state = (Histogram *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0)); if (state == NULL) PG_RETURN_NULL(); dims[0] = state->nbuckets; lbs[0] = 1; PG_RETURN_ARRAYTYPE_P( construct_md_array(state->buckets, NULL, 1, dims, lbs, INT4OID, 4, true, 'i')); } ================================================ FILE: src/hypercube.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <utils/jsonb.h> #include <utils/numeric.h> #include "dimension_vector.h" #include "export.h" #include "hypercube.h" /* * A hypercube represents the partition bounds of a hypertable chunk. * * A hypercube consists of N slices that each represent a range in a particular * dimension that make up the hypercube. When a new tuple is inserted into a * hypertable, and no chunk exists that can hold that tuple, we need to * calculate a new hypercube that encloses the point corresponding to the * tuple. When calculating the hypercube, we need to account for alignment * requirements in dimensions marked as "aligned" and also ensure that there are * no collisions with existing chunks. Alignment issues and collisions can occur * when the partitioning configuration has changed (e.g., the time interval or * number of partitions in a particular dimension changed). */ Hypercube * ts_hypercube_alloc(int16 num_dimensions) { Hypercube *hc = palloc0(HYPERCUBE_SIZE(num_dimensions)); hc->capacity = num_dimensions; return hc; } void ts_hypercube_free(Hypercube *hc) { int i; for (i = 0; i < hc->num_slices; i++) ts_dimension_slice_free(hc->slices[i]); pfree(hc); } #if defined(USE_ASSERT_CHECKING) static inline bool hypercube_is_sorted(const Hypercube *hc) { int i; if (hc->num_slices < 2) return true; for (i = 1; i < hc->num_slices; i++) if (hc->slices[i]->fd.dimension_id < hc->slices[i - 1]->fd.dimension_id) return false; return true; } #endif Hypercube * ts_hypercube_copy(const Hypercube *hc) { Hypercube *copy; size_t nbytes = HYPERCUBE_SIZE(hc->capacity); int i; copy = palloc(nbytes); memcpy(copy, hc, nbytes); for (i = 0; i < hc->num_slices; i++) copy->slices[i] = ts_dimension_slice_copy(hc->slices[i]); return copy; } bool ts_hypercube_equal(const Hypercube *hc1, const Hypercube *hc2) { int i; if (hc1->num_slices != hc2->num_slices) return false; for (i = 0; i < hc1->num_slices; i++) if (ts_dimension_slice_cmp(hc1->slices[i], hc2->slices[i]) != 0) return false; return true; } static int cmp_slices_by_dimension_id(const void *left, const void *right) { const DimensionSlice *left_slice = *((DimensionSlice **) left); const DimensionSlice *right_slice = *((DimensionSlice **) right); if (left_slice->fd.dimension_id == right_slice->fd.dimension_id) return 0; if (left_slice->fd.dimension_id < right_slice->fd.dimension_id) return -1; return 1; } DimensionSlice * ts_hypercube_add_slice_from_range(Hypercube *hc, int32 dimension_id, int64 start, int64 end) { DimensionSlice *slice; Assert(hc->capacity > hc->num_slices); slice = ts_dimension_slice_create(dimension_id, start, end); hc->slices[hc->num_slices++] = slice; /* Check if we require a sort to maintain dimension order */ if (hc->num_slices > 1 && slice->fd.dimension_id < hc->slices[hc->num_slices - 2]->fd.dimension_id) ts_hypercube_slice_sort(hc); Assert(hypercube_is_sorted(hc)); return slice; } DimensionSlice * ts_hypercube_add_slice(Hypercube *hc, const DimensionSlice *slice) { DimensionSlice *new_slice; new_slice = ts_hypercube_add_slice_from_range(hc, slice->fd.dimension_id, slice->fd.range_start, slice->fd.range_end); new_slice->fd.id = slice->fd.id; return new_slice; } /* * Sort the hypercubes slices in ascending dimension ID order. This allows us to * iterate slices in a consistent order. */ void ts_hypercube_slice_sort(Hypercube *hc) { qsort((void *) hc->slices, hc->num_slices, sizeof(DimensionSlice *), cmp_slices_by_dimension_id); } const DimensionSlice * ts_hypercube_get_slice_by_dimension_id(const Hypercube *hc, int32 dimension_id) { DimensionSlice slice = { .fd.dimension_id = dimension_id, }; void *ptr = &slice; if (hc->num_slices == 0) return NULL; Assert(hypercube_is_sorted(hc)); ptr = bsearch((void *) &ptr, (void *) hc->slices, hc->num_slices, sizeof(DimensionSlice *), cmp_slices_by_dimension_id); if (NULL == ptr) return NULL; return *((DimensionSlice **) ptr); } /* * Given a set of constraints, build the corresponding hypercube. */ Hypercube * ts_hypercube_from_constraints(const ChunkConstraints *constraints, ScanIterator *slice_it) { Hypercube *hc; int i; MemoryContext old; old = MemoryContextSwitchTo(ts_scan_iterator_get_result_memory_context(slice_it)); hc = ts_hypercube_alloc(constraints->num_dimension_constraints); MemoryContextSwitchTo(old); for (i = 0; i < constraints->num_constraints; i++) { ChunkConstraint *cc = chunk_constraints_get(constraints, i); if (is_dimension_constraint(cc)) { DimensionSlice *slice; Assert(hc->num_slices < constraints->num_dimension_constraints); /* When building the hypercube, we reference the dimension slices * to construct the hypercube. * * However, we cannot add a tuple lock when running in recovery * mode since that prevents SELECT statements (which reach this * point) from running on a read-only secondary (which runs in * ephemeral recovery mode), so we only take the lock if we are not * in recovery mode. */ slice = ts_dimension_slice_scan_iterator_get_by_id(slice_it, cc->fd.dimension_slice_id); if (!slice) { elog(ERROR, "could not find dimension slice with id %d", cc->fd.dimension_slice_id); } hc->slices[hc->num_slices++] = slice; } } ts_hypercube_slice_sort(hc); Assert(hypercube_is_sorted(hc)); return hc; } /* * Find slices in the hypercube that already exists in metadata. * * If a slice exists in metadata, the slice ID will be filled in on the * existing slice in the hypercube. Optionally, also lock the slice when * found. */ int ts_hypercube_find_existing_slices(const Hypercube *cube, const ScanTupLock *tuplock) { int i; int num_found = 0; for (i = 0; i < cube->num_slices; i++) { /* * Check if there's already an existing slice with the calculated * range. If a slice already exists, use that slice's ID instead * of a new one. */ bool found = ts_dimension_slice_scan_for_existing(cube->slices[i], tuplock); if (found) num_found++; } return num_found; } /* * Calculate the hypercube that encloses the given point. * * The hypercube's dimensions are calculated one by one, and depend on the * current partitioning in each dimension of the N-dimensional hyperspace, * including any alignment requirements. * * For non-aligned dimensions, we simply calculate the hypercube's slice range * in that dimension given current partitioning configuration. If there is * already an identical slice for that dimension, we will reuse it rather than * creating a new one. * * For aligned dimensions, we first try to find an existing slice that covers * the insertion point. If an existing slice is found, we reuse it or otherwise * we calculate a new slice as described for non-aligned dimensions. * * If a hypercube has dimension slices that are not reused ones, we might need * to cut them to ensure alignment and avoid collisions with other chunk * hypercubes. This happens in a later step. */ Hypercube * ts_hypercube_calculate_from_point(const Hyperspace *hs, const Point *p, const ScanTupLock *tuplock) { Hypercube *cube; int i; cube = ts_hypercube_alloc(hs->num_dimensions); /* For each dimension, calculate the hypercube's slice in that dimension */ for (i = 0; i < hs->num_dimensions; i++) { const Dimension *dim = &hs->dimensions[i]; int64 value = p->coordinates[i]; bool found = false; bool check_for_existing_slice = false; /* Assert that dimensions are in ascending order */ Assert(i == 0 || dim->fd.id > hs->dimensions[i - 1].fd.id); if (dim->fd.aligned) { DimensionVec *vec; vec = ts_dimension_slice_scan_limit(dim->fd.id, value, 1, tuplock); if (vec->num_slices > 0) { cube->slices[i] = vec->slices[0]; found = true; } } if (!found) { /* * No existing slice found, or we are not aligning, so calculate * the range of a new slice */ cube->slices[i] = ts_dimension_calculate_default_slice(dim, value); check_for_existing_slice = true; } if (check_for_existing_slice) { /* * Check if there's already an existing slice with the calculated * range. If a slice already exists, use that slice's ID instead * of a new one. */ ts_dimension_slice_scan_for_existing(cube->slices[i], tuplock); } } cube->num_slices = hs->num_dimensions; Assert(hypercube_is_sorted(cube)); return cube; } /* * Check if two hypercubes collide (overlap). * * This is basically an axis-aligned bounding box collision detection, * generalized to N dimensions. We check for dimension slice collisions in each * dimension and only if all dimensions collide there is a hypercube collision. */ bool ts_hypercubes_collide(const Hypercube *cube1, const Hypercube *cube2) { int i; Assert(cube1->num_slices == cube2->num_slices); for (i = 0; i < cube1->num_slices; i++) if (!ts_dimension_slices_collide(cube1->slices[i], cube2->slices[i])) return false; return true; } ================================================ FILE: src/hypercube.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include "dimension_slice.h" #include "scan_iterator.h" /* * Hypercube is a collection of slices from N distinct dimensions, i.e., the * N-dimensional analogue of a cube. */ typedef struct Hypercube { int16 capacity; /* capacity of slices[] */ int16 num_slices; /* actual number of slices (should equal * capacity after create) */ /* Slices are stored in dimension order */ DimensionSlice *slices[FLEXIBLE_ARRAY_MEMBER]; } Hypercube; #define HYPERCUBE_SIZE(num_dimensions) \ (sizeof(Hypercube) + (sizeof(DimensionSlice *) * (num_dimensions))) extern TSDLLEXPORT Hypercube *ts_hypercube_alloc(int16 num_dimensions); extern TSDLLEXPORT void ts_hypercube_free(Hypercube *hc); extern TSDLLEXPORT DimensionSlice * ts_hypercube_add_slice_from_range(Hypercube *hc, int32 dimension_id, int64 start, int64 end); extern TSDLLEXPORT DimensionSlice *ts_hypercube_add_slice(Hypercube *hc, const DimensionSlice *slice); extern Hypercube *ts_hypercube_from_constraints(const ChunkConstraints *constraints, ScanIterator *slice_it); extern int ts_hypercube_find_existing_slices(const Hypercube *cube, const ScanTupLock *tuplock); extern Hypercube *ts_hypercube_calculate_from_point(const Hyperspace *hs, const Point *p, const ScanTupLock *tuplock); extern bool ts_hypercubes_collide(const Hypercube *cube1, const Hypercube *cube2); extern TSDLLEXPORT const DimensionSlice *ts_hypercube_get_slice_by_dimension_id(const Hypercube *hc, int32 dimension_id); extern TSDLLEXPORT Hypercube *ts_hypercube_copy(const Hypercube *hc); extern bool ts_hypercube_equal(const Hypercube *hc1, const Hypercube *hc2); extern void ts_hypercube_slice_sort(Hypercube *hc); ================================================ FILE: src/hypertable.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/heapam.h> #include <access/htup_details.h> #include <access/relscan.h> #include <catalog/indexing.h> #include <catalog/namespace.h> #include <catalog/pg_collation.h> #include <catalog/pg_constraint.h> #include <catalog/pg_database_d.h> #include <catalog/pg_inherits.h> #include <catalog/pg_namespace_d.h> #include <catalog/pg_proc.h> #include <catalog/pg_type.h> #include <commands/dbcommands.h> #include <commands/schemacmds.h> #include <commands/tablecmds.h> #include <commands/tablespace.h> #include <commands/trigger.h> #include <executor/spi.h> #include <funcapi.h> #include <lib/stringinfo.h> #include <miscadmin.h> #include <nodes/makefuncs.h> #include <nodes/memnodes.h> #include <nodes/parsenodes.h> #include <nodes/value.h> #include <parser/parse_coerce.h> #include <parser/parse_func.h> #include <storage/lmgr.h> #include <utils/acl.h> #include <utils/builtins.h> #include <utils/lsyscache.h> #include <utils/memutils.h> #include <utils/snapmgr.h> #include <utils/syscache.h> #include "hypertable.h" #include "compat/compat.h" #include "bgw_policy/policy.h" #include "chunk.h" #include "chunk_adaptive.h" #include "copy.h" #include "cross_module_fn.h" #include "debug_assert.h" #include "dimension.h" #include "dimension_slice.h" #include "dimension_vector.h" #include "error_utils.h" #include "errors.h" #include "extension.h" #include "guc.h" #include "hypercube.h" #include "hypertable_cache.h" #include "indexing.h" #include "license_guc.h" #include "osm_callbacks.h" #include "partition_chunk.h" #include "scan_iterator.h" #include "scanner.h" #include "subspace_store.h" #include "trigger.h" #include "ts_catalog/catalog.h" #include "ts_catalog/chunk_column_stats.h" #include "ts_catalog/compression_settings.h" #include "ts_catalog/continuous_agg.h" #include "ts_catalog/metadata.h" #include "utils.h" Oid ts_rel_get_owner(Oid relid) { HeapTuple tuple; Oid ownerid; if (!OidIsValid(relid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("invalid relation OID"))); tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); if (!HeapTupleIsValid(tuple)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("relation with OID %u does not exist", relid))); ownerid = ((Form_pg_class) GETSTRUCT(tuple))->relowner; ReleaseSysCache(tuple); return ownerid; } bool ts_hypertable_has_privs_of(Oid hypertable_oid, Oid userid) { return has_privs_of_role(userid, ts_rel_get_owner(hypertable_oid)); } /* * The error output for permission denied errors such as these changed in PG11, * it modifies places where relation is specified to note the specific object * type we note that permissions are denied for the hypertable for all PG * versions so that tests need not change due to one word changes in error * messages and because it is more clear this way. */ Oid ts_hypertable_permissions_check(Oid hypertable_oid, Oid userid) { Oid ownerid = ts_rel_get_owner(hypertable_oid); if (!has_privs_of_role(userid, ownerid)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("must be owner of hypertable \"%s\"", get_rel_name(hypertable_oid)))); return ownerid; } void ts_hypertable_permissions_check_by_id(int32 hypertable_id) { Oid table_relid = ts_hypertable_id_to_relid(hypertable_id, false); ts_hypertable_permissions_check(table_relid, GetUserId()); } static Oid get_chunk_sizing_func_oid(const FormData_hypertable *fd) { Oid argtype[] = { INT4OID, INT8OID, INT8OID }; return LookupFuncName(list_make2(makeString((char *) NameStr(fd->chunk_sizing_func_schema)), makeString((char *) NameStr(fd->chunk_sizing_func_name))), sizeof(argtype) / sizeof(argtype[0]), argtype, false); } static HeapTuple hypertable_formdata_make_tuple(const FormData_hypertable *fd, TupleDesc desc) { Datum values[Natts_hypertable]; bool nulls[Natts_hypertable] = { false }; memset(values, 0, sizeof(Datum) * Natts_hypertable); values[AttrNumberGetAttrOffset(Anum_hypertable_id)] = Int32GetDatum(fd->id); values[AttrNumberGetAttrOffset(Anum_hypertable_schema_name)] = NameGetDatum(&fd->schema_name); values[AttrNumberGetAttrOffset(Anum_hypertable_table_name)] = NameGetDatum(&fd->table_name); values[AttrNumberGetAttrOffset(Anum_hypertable_associated_schema_name)] = NameGetDatum(&fd->associated_schema_name); Assert(&fd->associated_table_prefix != NULL); values[AttrNumberGetAttrOffset(Anum_hypertable_associated_table_prefix)] = NameGetDatum(&fd->associated_table_prefix); values[AttrNumberGetAttrOffset(Anum_hypertable_num_dimensions)] = Int16GetDatum(fd->num_dimensions); values[AttrNumberGetAttrOffset(Anum_hypertable_chunk_sizing_func_schema)] = NameGetDatum(&fd->chunk_sizing_func_schema); values[AttrNumberGetAttrOffset(Anum_hypertable_chunk_sizing_func_name)] = NameGetDatum(&fd->chunk_sizing_func_name); values[AttrNumberGetAttrOffset(Anum_hypertable_chunk_target_size)] = Int64GetDatum(fd->chunk_target_size); values[AttrNumberGetAttrOffset(Anum_hypertable_compression_state)] = Int16GetDatum(fd->compression_state); if (fd->compressed_hypertable_id == INVALID_HYPERTABLE_ID) nulls[AttrNumberGetAttrOffset(Anum_hypertable_compressed_hypertable_id)] = true; else values[AttrNumberGetAttrOffset(Anum_hypertable_compressed_hypertable_id)] = Int32GetDatum(fd->compressed_hypertable_id); values[AttrNumberGetAttrOffset(Anum_hypertable_status)] = Int32GetDatum(fd->status); return heap_form_tuple(desc, values, nulls); } void ts_hypertable_formdata_fill(FormData_hypertable *fd, const TupleInfo *ti) { bool nulls[Natts_hypertable]; Datum values[Natts_hypertable]; bool should_free; HeapTuple tuple; tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); heap_deform_tuple(tuple, ts_scanner_get_tupledesc(ti), values, nulls); Assert(!nulls[AttrNumberGetAttrOffset(Anum_hypertable_id)]); Assert(!nulls[AttrNumberGetAttrOffset(Anum_hypertable_schema_name)]); Assert(!nulls[AttrNumberGetAttrOffset(Anum_hypertable_table_name)]); Assert(!nulls[AttrNumberGetAttrOffset(Anum_hypertable_associated_schema_name)]); Assert(!nulls[AttrNumberGetAttrOffset(Anum_hypertable_associated_table_prefix)]); Assert(!nulls[AttrNumberGetAttrOffset(Anum_hypertable_num_dimensions)]); Assert(!nulls[AttrNumberGetAttrOffset(Anum_hypertable_chunk_sizing_func_schema)]); Assert(!nulls[AttrNumberGetAttrOffset(Anum_hypertable_chunk_sizing_func_name)]); Assert(!nulls[AttrNumberGetAttrOffset(Anum_hypertable_chunk_target_size)]); Assert(!nulls[AttrNumberGetAttrOffset(Anum_hypertable_compression_state)]); Assert(!nulls[AttrNumberGetAttrOffset(Anum_hypertable_status)]); fd->id = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_hypertable_id)]); namestrcpy(&fd->schema_name, DatumGetCString(values[AttrNumberGetAttrOffset(Anum_hypertable_schema_name)])); namestrcpy(&fd->table_name, DatumGetCString(values[AttrNumberGetAttrOffset(Anum_hypertable_table_name)])); namestrcpy(&fd->associated_schema_name, DatumGetCString( values[AttrNumberGetAttrOffset(Anum_hypertable_associated_schema_name)])); namestrcpy(&fd->associated_table_prefix, DatumGetCString( values[AttrNumberGetAttrOffset(Anum_hypertable_associated_table_prefix)])); fd->num_dimensions = DatumGetInt16(values[AttrNumberGetAttrOffset(Anum_hypertable_num_dimensions)]); namestrcpy(&fd->chunk_sizing_func_schema, DatumGetCString( values[AttrNumberGetAttrOffset(Anum_hypertable_chunk_sizing_func_schema)])); namestrcpy(&fd->chunk_sizing_func_name, DatumGetCString( values[AttrNumberGetAttrOffset(Anum_hypertable_chunk_sizing_func_name)])); fd->chunk_target_size = DatumGetInt64(values[AttrNumberGetAttrOffset(Anum_hypertable_chunk_target_size)]); fd->compression_state = DatumGetInt16(values[AttrNumberGetAttrOffset(Anum_hypertable_compression_state)]); if (nulls[AttrNumberGetAttrOffset(Anum_hypertable_compressed_hypertable_id)]) fd->compressed_hypertable_id = INVALID_HYPERTABLE_ID; else fd->compressed_hypertable_id = DatumGetInt32( values[AttrNumberGetAttrOffset(Anum_hypertable_compressed_hypertable_id)]); fd->status = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_hypertable_status)]); if (should_free) heap_freetuple(tuple); } Hypertable * ts_hypertable_from_tupleinfo(const TupleInfo *ti) { Hypertable *h = MemoryContextAllocZero(ti->mctx, sizeof(Hypertable)); ts_hypertable_formdata_fill(&h->fd, ti); h->main_table_relid = ts_get_relation_relid(NameStr(h->fd.schema_name), NameStr(h->fd.table_name), true); h->space = ts_dimension_scan(h->fd.id, h->main_table_relid, h->fd.num_dimensions, ti->mctx); h->chunk_cache = ts_subspace_store_init(h->space, ti->mctx, ts_guc_max_cached_chunks_per_hypertable); h->chunk_sizing_func = get_chunk_sizing_func_oid(&h->fd); if (ts_guc_enable_chunk_skipping) { h->range_space = ts_chunk_column_stats_range_space_scan(h->fd.id, h->main_table_relid, ti->mctx); } return h; } /* * Find either the hypertable or the materialized hypertable, if the relid is * a continuous aggregate, for the relid. * * If allow_matht is false, relid should be a cagg or a hypertable. * If allow_matht is true, materialized hypertable is also permitted as relid */ Hypertable * ts_resolve_hypertable_from_table_or_cagg(Cache *hcache, Oid relid, bool allow_matht) { const char *rel_name; Hypertable *ht; rel_name = get_rel_name(relid); if (!rel_name) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("invalid hypertable or continuous aggregate"))); ht = ts_hypertable_cache_get_entry(hcache, relid, CACHE_FLAG_MISSING_OK); if (ht) { const ContinuousAggHypertableStatus status = ts_continuous_agg_hypertable_status(ht->fd.id); switch (status) { case HypertableIsMaterialization: case HypertableIsMaterializationAndRaw: if (!allow_matht) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("operation not supported on materialized hypertable"), errhint("Try the operation on the continuous aggregate instead."), errdetail("Hypertable \"%s\" is a materialized hypertable.", rel_name))); } break; default: break; } } else { ContinuousAgg *const cagg = ts_continuous_agg_find_by_relid(relid); if (!cagg) ereport(ERROR, (errcode(ERRCODE_TS_HYPERTABLE_NOT_EXIST), errmsg("\"%s\" is not a hypertable or a continuous aggregate", rel_name), errhint("The operation is only possible on a hypertable or continuous" " aggregate."))); ht = ts_hypertable_get_by_id(cagg->data.mat_hypertable_id); if (!ht) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("no materialized table for continuous aggregate"), errdetail("Continuous aggregate \"%s\" had a materialized hypertable" " with id %d but it was not found in the hypertable " "catalog.", rel_name, cagg->data.mat_hypertable_id))); } return ht; } static ScanTupleResult hypertable_tuple_get_relid(TupleInfo *ti, void *data) { Oid *relid = data; FormData_hypertable fd; Oid schema_oid; ts_hypertable_formdata_fill(&fd, ti); schema_oid = get_namespace_oid(NameStr(fd.schema_name), true); if (OidIsValid(schema_oid)) *relid = get_relname_relid(NameStr(fd.table_name), schema_oid); return SCAN_DONE; } Oid ts_hypertable_id_to_relid(int32 hypertable_id, bool return_invalid) { Catalog *catalog = ts_catalog_get(); Oid relid = InvalidOid; ScanKeyData scankey[1]; ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, HYPERTABLE), .index = catalog_get_index(catalog, HYPERTABLE, HYPERTABLE_ID_INDEX), .nkeys = 1, .scankey = scankey, .tuple_found = hypertable_tuple_get_relid, .data = &relid, .lockmode = AccessShareLock, .scandirection = ForwardScanDirection, }; /* Perform an index scan on the hypertable pkey. */ ScanKeyInit(&scankey[0], Anum_hypertable_pkey_idx_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); ts_scanner_scan(&scanctx); if (!OidIsValid(relid) && !return_invalid) ereport(ERROR, (errcode(ERRCODE_TS_HYPERTABLE_NOT_EXIST), errmsg("hypertable with id %d does not exist", hypertable_id))); return relid; } int32 ts_hypertable_relid_to_id(Oid relid) { Cache *hcache; Hypertable *ht = ts_hypertable_cache_get_cache_and_entry(relid, CACHE_FLAG_MISSING_OK, &hcache); int result = ht ? ht->fd.id : INVALID_HYPERTABLE_ID; ts_cache_release(&hcache); return result; } static int hypertable_scan_limit_internal(ScanKeyData *scankey, int num_scankeys, int indexid, tuple_found_func on_tuple_found, void *scandata, int limit, LOCKMODE lock, MemoryContext mctx, tuple_filter_func filter) { Catalog *catalog = ts_catalog_get(); ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, HYPERTABLE), .index = catalog_get_index(catalog, HYPERTABLE, indexid), .nkeys = num_scankeys, .scankey = scankey, .data = scandata, .limit = limit, .tuple_found = on_tuple_found, .lockmode = lock, .filter = filter, .scandirection = ForwardScanDirection, .result_mctx = mctx, }; return ts_scanner_scan(&scanctx); } /* update the tuple at this tid. The assumption is that we already hold a * tuple exclusive lock and no other transaction can modify this tuple * The sequence of operations for any update is: * lock the tuple using lock_hypertable_tuple. * then update the required fields * call hypertable_update_catalog_tuple to complete the update. * This ensures correct tuple locking and tuple updates in the presence of * concurrent transactions. Failure to follow this results in catalog corruption */ static void hypertable_update_catalog_tuple(ItemPointer tid, FormData_hypertable *update) { HeapTuple new_tuple; CatalogSecurityContext sec_ctx; Catalog *catalog = ts_catalog_get(); Oid table = catalog_get_table_id(catalog, HYPERTABLE); Relation hypertable_rel = relation_open(table, RowExclusiveLock); new_tuple = hypertable_formdata_make_tuple(update, hypertable_rel->rd_att); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_update_tid(hypertable_rel, tid, new_tuple); ts_catalog_restore_user(&sec_ctx); heap_freetuple(new_tuple); relation_close(hypertable_rel, NoLock); } static bool lock_hypertable_tuple(int32 htid, ItemPointer tid, FormData_hypertable *form) { bool success = false; ScanTupLock scantuplock = { .waitpolicy = LockWaitBlock, .lockmode = LockTupleExclusive, }; ScanIterator iterator = ts_scan_iterator_create(HYPERTABLE, RowShareLock, CurrentMemoryContext); iterator.ctx.index = catalog_get_index(ts_catalog_get(), HYPERTABLE, HYPERTABLE_ID_INDEX); iterator.ctx.tuplock = &scantuplock; /* Keeping the lock since we presumably want to update the tuple */ iterator.ctx.flags = SCANNER_F_KEEPLOCK; /* see table_tuple_lock for details about flags that are set in TupleExclusive mode */ scantuplock.lockflags = TUPLE_LOCK_FLAG_LOCK_UPDATE_IN_PROGRESS; if (!IsolationUsesXactSnapshot()) { /* in read committed mode, we follow all updates to this tuple */ scantuplock.lockflags |= TUPLE_LOCK_FLAG_FIND_LAST_VERSION; } ts_scan_iterator_scan_key_init(&iterator, Anum_hypertable_pkey_idx_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(htid)); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); if (ti->lockresult != TM_Ok) { if (IsolationUsesXactSnapshot()) { /* For Repeatable Read and Serializable isolation level report error * if we cannot lock the tuple */ ereport(ERROR, (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("could not serialize access due to concurrent update"))); } else { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("unable to lock hypertable catalog tuple, lock result is %d for " "hypertable " "ID (%d)", ti->lockresult, htid))); } } ts_hypertable_formdata_fill(form, ti); ItemPointer result_tid = ts_scanner_get_tuple_tid(ti); tid->ip_blkid = result_tid->ip_blkid; tid->ip_posid = result_tid->ip_posid; success = true; break; } ts_scan_iterator_close(&iterator); return success; } int ts_hypertable_scan_with_memory_context(const char *schema, const char *table, tuple_found_func tuple_found, void *data, LOCKMODE lockmode, MemoryContext mctx) { ScanKeyData scankey[2]; NameData schema_name = { .data = { 0 } }; NameData table_name = { .data = { 0 } }; if (schema) namestrcpy(&schema_name, schema); if (table) namestrcpy(&table_name, table); /* Perform an index scan on schema and table. */ ScanKeyInit(&scankey[0], Anum_hypertable_name_idx_table, BTEqualStrategyNumber, F_NAMEEQ, NameGetDatum(&table_name)); ScanKeyInit(&scankey[1], Anum_hypertable_name_idx_schema, BTEqualStrategyNumber, F_NAMEEQ, NameGetDatum(&schema_name)); return hypertable_scan_limit_internal(scankey, 2, HYPERTABLE_NAME_INDEX, tuple_found, data, 1, lockmode, mctx, NULL); } TSDLLEXPORT ObjectAddress ts_hypertable_create_trigger(const Hypertable *ht, CreateTrigStmt *stmt, const char *query) { ObjectAddress root_trigger_addr; List *chunks; ListCell *lc; int sec_ctx; Oid saved_uid; Oid owner; Assert(ht != NULL); /* create the trigger on the root table */ /* ACL permissions checks happen within this call */ root_trigger_addr = CreateTrigger(stmt, query, InvalidOid, InvalidOid, InvalidOid, InvalidOid, InvalidOid, InvalidOid, NULL, false, false); /* and forward it to the chunks */ CommandCounterIncrement(); if (!stmt->row) return root_trigger_addr; /* switch to the hypertable owner's role -- note that this logic must be the same as * `ts_trigger_create_all_on_chunk` */ owner = ts_rel_get_owner(ht->main_table_relid); GetUserIdAndSecContext(&saved_uid, &sec_ctx); if (saved_uid != owner) SetUserIdAndSecContext(owner, sec_ctx | SECURITY_LOCAL_USERID_CHANGE); chunks = find_inheritance_children(ht->main_table_relid, NoLock); foreach (lc, chunks) { Oid chunk_oid = lfirst_oid(lc); char *relschema = get_namespace_name(get_rel_namespace(chunk_oid)); char *relname = get_rel_name(chunk_oid); char relkind = get_rel_relkind(chunk_oid); Assert(relkind == RELKIND_RELATION || relkind == RELKIND_FOREIGN_TABLE); /* Only create triggers on standard relations and not on, e.g., foreign * table chunks */ if (relkind == RELKIND_RELATION) ts_trigger_create_on_chunk(root_trigger_addr.objectId, relschema, relname); } if (saved_uid != owner) SetUserIdAndSecContext(saved_uid, sec_ctx); return root_trigger_addr; } TSDLLEXPORT void ts_hypertable_drop_invalidation_replication_slot(const char *slot_name) { CatalogSecurityContext sec_ctx; NameData slot; namestrcpy(&slot, slot_name); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); DirectFunctionCall1(pg_drop_replication_slot, NameGetDatum(&slot)); ts_catalog_restore_user(&sec_ctx); } /* based on RemoveObjects */ TSDLLEXPORT void ts_hypertable_drop_trigger(Oid relid, const char *trigger_name) { List *chunks = find_inheritance_children(relid, NoLock); ListCell *lc; if (OidIsValid(relid)) { ObjectAddress objaddr = { .classId = TriggerRelationId, .objectId = get_trigger_oid(relid, trigger_name, true), }; if (OidIsValid(objaddr.objectId)) performDeletion(&objaddr, DROP_RESTRICT, 0); } foreach (lc, chunks) { Oid chunk_oid = lfirst_oid(lc); ObjectAddress objaddr = { .classId = TriggerRelationId, .objectId = get_trigger_oid(chunk_oid, trigger_name, true), }; if (OidIsValid(objaddr.objectId)) performDeletion(&objaddr, DROP_RESTRICT, 0); } } static ScanTupleResult hypertable_tuple_delete(TupleInfo *ti, void *data) { CatalogSecurityContext sec_ctx; bool isnull; bool compressed_hypertable_id_isnull; int hypertable_id = DatumGetInt32(slot_getattr(ti->slot, Anum_hypertable_id, &isnull)); int compressed_hypertable_id = DatumGetInt32(slot_getattr(ti->slot, Anum_hypertable_compressed_hypertable_id, &compressed_hypertable_id_isnull)); ts_tablespace_delete(hypertable_id, NULL, InvalidOid); ts_chunk_delete_by_hypertable_id(hypertable_id); ts_dimension_delete_by_hypertable_id(hypertable_id, true); /* Also remove any policy argument / job that uses this hypertable */ ts_bgw_policy_delete_by_hypertable_id(hypertable_id); /* Also remove any rows in _timescaledb_catalog.chunk_column_stats corresponding to this * hypertable */ ts_chunk_column_stats_delete_by_hypertable_id(hypertable_id); /* Remove any dependent continuous aggs */ ts_continuous_agg_drop_hypertable_callback(hypertable_id); if (!compressed_hypertable_id_isnull) { Hypertable *compressed_hypertable = ts_hypertable_get_by_id(compressed_hypertable_id); /* The hypertable may have already been deleted by a cascade */ if (compressed_hypertable != NULL) ts_hypertable_drop(compressed_hypertable, DROP_RESTRICT); } hypertable_drop_hook_type osm_htdrop_hook = ts_get_osm_hypertable_drop_hook(); /* Invoke the OSM callback if set */ if (osm_htdrop_hook) { Name schema_name = DatumGetName(slot_getattr(ti->slot, Anum_hypertable_schema_name, &isnull)); Name table_name = DatumGetName(slot_getattr(ti->slot, Anum_hypertable_table_name, &isnull)); osm_htdrop_hook(NameStr(*schema_name), NameStr(*table_name)); } ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); ts_catalog_restore_user(&sec_ctx); return SCAN_CONTINUE; } int ts_hypertable_delete_by_name(const char *schema_name, const char *table_name) { ScanKeyData scankey[2]; ScanKeyInit(&scankey[0], Anum_hypertable_name_idx_table, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(table_name)); ScanKeyInit(&scankey[1], Anum_hypertable_name_idx_schema, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(schema_name)); return hypertable_scan_limit_internal(scankey, 2, HYPERTABLE_NAME_INDEX, hypertable_tuple_delete, NULL, 0, RowExclusiveLock, CurrentMemoryContext, NULL); } int ts_hypertable_delete_by_id(int32 hypertable_id) { ScanKeyData scankey[1]; ScanKeyInit(&scankey[0], Anum_hypertable_pkey_idx_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); return hypertable_scan_limit_internal(scankey, 1, HYPERTABLE_ID_INDEX, hypertable_tuple_delete, NULL, 1, RowExclusiveLock, CurrentMemoryContext, NULL); } void ts_hypertable_drop(Hypertable *hypertable, DropBehavior behavior) { /* The actual table might have been deleted already, but we still need to * clean up the catalog entry. */ if (OidIsValid(hypertable->main_table_relid)) { ObjectAddress hypertable_addr = (ObjectAddress){ .classId = RelationRelationId, .objectId = hypertable->main_table_relid, }; /* Drop the postgres table */ ts_compression_settings_delete(hypertable->main_table_relid); performDeletion(&hypertable_addr, behavior, 0); } /* Clean up catalog */ ts_hypertable_delete_by_name(NameStr(hypertable->fd.schema_name), NameStr(hypertable->fd.table_name)); } static ScanTupleResult reset_associated_tuple_found(TupleInfo *ti, void *data) { HeapTuple new_tuple; FormData_hypertable fd; CatalogSecurityContext sec_ctx; ts_hypertable_formdata_fill(&fd, ti); namestrcpy(&fd.associated_schema_name, INTERNAL_SCHEMA_NAME); new_tuple = hypertable_formdata_make_tuple(&fd, ts_scanner_get_tupledesc(ti)); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_update_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti), new_tuple); ts_catalog_restore_user(&sec_ctx); heap_freetuple(new_tuple); return SCAN_CONTINUE; } /* * Reset the matching associated schema to the internal schema. */ int ts_hypertable_reset_associated_schema_name(const char *associated_schema) { ScanKeyData scankey[1]; ScanKeyInit(&scankey[0], Anum_hypertable_associated_schema_name, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(associated_schema)); return hypertable_scan_limit_internal(scankey, 1, INVALID_INDEXID, reset_associated_tuple_found, NULL, 0, RowExclusiveLock, CurrentMemoryContext, NULL); } int ts_hypertable_set_name(Hypertable *ht, const char *newname) { FormData_hypertable form; ItemPointerData tid; /* lock the tuple entry in the catalog table */ bool found = lock_hypertable_tuple(ht->fd.id, &tid, &form); Ensure(found, "hypertable id %d not found", ht->fd.id); namestrcpy(&form.table_name, newname); hypertable_update_catalog_tuple(&tid, &form); return true; } int ts_hypertable_set_schema(Hypertable *ht, const char *newname) { FormData_hypertable form; ItemPointerData tid; /* lock the tuple entry in the catalog table */ bool found = lock_hypertable_tuple(ht->fd.id, &tid, &form); Ensure(found, "hypertable id %d not found", ht->fd.id); namestrcpy(&form.schema_name, newname); hypertable_update_catalog_tuple(&tid, &form); return true; } int ts_hypertable_set_num_dimensions(Hypertable *ht, int16 num_dimensions) { FormData_hypertable form; ItemPointerData tid; /* lock the tuple entry in the catalog table */ bool found = lock_hypertable_tuple(ht->fd.id, &tid, &form); Ensure(found, "hypertable id %d not found", ht->fd.id); Assert(num_dimensions > 0); form.num_dimensions = num_dimensions; hypertable_update_catalog_tuple(&tid, &form); return true; } #define DEFAULT_ASSOCIATED_TABLE_PREFIX_FORMAT "_hyper_%d" static const size_t MAXIMUM_PREFIX_LENGTH = NAMEDATALEN - 16; static void hypertable_insert_relation(Relation rel, FormData_hypertable *fd) { HeapTuple new_tuple; CatalogSecurityContext sec_ctx; new_tuple = hypertable_formdata_make_tuple(fd, RelationGetDescr(rel)); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_insert(rel, new_tuple); ts_catalog_restore_user(&sec_ctx); heap_freetuple(new_tuple); } static void hypertable_insert(int32 hypertable_id, Name schema_name, Name table_name, Name associated_schema_name, Name associated_table_prefix, Name chunk_sizing_func_schema, Name chunk_sizing_func_name, int64 chunk_target_size, int16 num_dimensions, bool compressed) { Catalog *catalog = ts_catalog_get(); Relation rel; FormData_hypertable fd; fd.id = hypertable_id; if (fd.id == INVALID_HYPERTABLE_ID) { CatalogSecurityContext sec_ctx; ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); fd.id = ts_catalog_table_next_seq_id(ts_catalog_get(), HYPERTABLE); ts_catalog_restore_user(&sec_ctx); } namestrcpy(&fd.schema_name, NameStr(*schema_name)); namestrcpy(&fd.table_name, NameStr(*table_name)); namestrcpy(&fd.associated_schema_name, NameStr(*associated_schema_name)); if (NULL == associated_table_prefix) { NameData default_associated_table_prefix; memset(NameStr(default_associated_table_prefix), '\0', NAMEDATALEN); snprintf(NameStr(default_associated_table_prefix), NAMEDATALEN, DEFAULT_ASSOCIATED_TABLE_PREFIX_FORMAT, fd.id); namestrcpy(&fd.associated_table_prefix, NameStr(default_associated_table_prefix)); } else { namestrcpy(&fd.associated_table_prefix, NameStr(*associated_table_prefix)); } if (strnlen(NameStr(fd.associated_table_prefix), NAMEDATALEN) > MAXIMUM_PREFIX_LENGTH) elog(ERROR, "associated_table_prefix too long"); fd.num_dimensions = num_dimensions; namestrcpy(&fd.chunk_sizing_func_schema, NameStr(*chunk_sizing_func_schema)); namestrcpy(&fd.chunk_sizing_func_name, NameStr(*chunk_sizing_func_name)); fd.chunk_target_size = chunk_target_size; if (fd.chunk_target_size < 0) fd.chunk_target_size = 0; if (compressed) fd.compression_state = HypertableInternalCompressionTable; else fd.compression_state = HypertableCompressionOff; /* when creating a hypertable, there is never an associated compressed dual */ fd.compressed_hypertable_id = INVALID_HYPERTABLE_ID; /* new hypertable does not have OSM chunk */ fd.status = HYPERTABLE_STATUS_DEFAULT; rel = table_open(catalog_get_table_id(catalog, HYPERTABLE), RowExclusiveLock); hypertable_insert_relation(rel, &fd); table_close(rel, RowExclusiveLock); } static ScanTupleResult hypertable_tuple_found(TupleInfo *ti, void *data) { Hypertable **entry = (Hypertable **) data; *entry = ts_hypertable_from_tupleinfo(ti); return SCAN_DONE; } Hypertable * ts_hypertable_get_by_name(const char *schema, const char *name) { Hypertable *ht = NULL; hypertable_scan(schema, name, hypertable_tuple_found, (void *) &ht, AccessShareLock); return ht; } Hypertable * ts_hypertable_get_by_id(int32 hypertable_id) { ScanKeyData scankey[1]; Hypertable *ht = NULL; ScanKeyInit(&scankey[0], Anum_hypertable_pkey_idx_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); hypertable_scan_limit_internal(scankey, 1, HYPERTABLE_ID_INDEX, hypertable_tuple_found, (void *) &ht, 1, AccessShareLock, CurrentMemoryContext, NULL); return ht; } static void hypertable_chunk_store_free(void *entry) { ts_chunk_free((Chunk *) entry); } /* * Add the chunk to the cache that allows fast lookup of chunks * for a given hyperspace Point. */ Chunk * ts_hypertable_chunk_store_add(const Hypertable *h, const Chunk *input_chunk) { MemoryContext old_mcxt; /* Add the chunk to the subspace store */ old_mcxt = MemoryContextSwitchTo(ts_subspace_store_mcxt(h->chunk_cache)); Chunk *cached_chunk = ts_chunk_copy(input_chunk); ts_subspace_store_add(h->chunk_cache, cached_chunk->cube, cached_chunk, hypertable_chunk_store_free); MemoryContextSwitchTo(old_mcxt); return cached_chunk; } /* * Create a chunk for the point, given that it does not exist yet. * * If the chunk already exists (i.e., another process beat us to it), then * lock the chunk with the specified lockmode. If the chunk is created, it * will always be locked with AccessExclusivelock. */ Chunk * ts_hypertable_create_chunk_for_point(const Hypertable *h, const Point *point, LOCKMODE chunk_lockmode) { Assert(ts_subspace_store_get(h->chunk_cache, point) == NULL); Chunk *chunk = ts_chunk_create_for_point(h, point, NameStr(h->fd.associated_schema_name), NameStr(h->fd.associated_table_prefix), chunk_lockmode); /* Also add the chunk to the hypertable's chunk store */ Chunk *cached_chunk = ts_hypertable_chunk_store_add(h, chunk); return cached_chunk; } /* * Find the chunk responsible for the given point. * * In case of a cache miss, a point scan will try to find a matching chunk. A * matching chunk will be locked in the given lockmode (unless NoLock is * specified) and added to the cache. * * If lockmode is higher than NoLock, all dimension slices will also be locked * in LockTupleKeyShare. * * If no chunk is found, NULL is returned. The returned chunk is owned by the * cache and may become invalid after some subsequent call to this function. * Leaks memory, so call in a short-lived context. */ Chunk * ts_hypertable_find_chunk_for_point(const Hypertable *h, const Point *point, LOCKMODE lockmode) { Chunk *chunk = ts_subspace_store_get(h->chunk_cache, point); if (!chunk) { chunk = ts_chunk_find_for_point(h, point, lockmode); if (chunk) chunk = ts_hypertable_chunk_store_add(h, chunk); } else if (!ts_chunk_lock_if_exists(chunk->table_id, lockmode)) { return NULL; } #ifdef USE_ASSERT_CHECKING if (chunk) { Relation chunk_rel = RelationIdGetRelation(chunk->table_id); Assert(CheckRelationLockedByMe(chunk_rel, lockmode, true)); RelationClose(chunk_rel); } #endif return chunk; } bool ts_hypertable_has_tablespace(const Hypertable *ht, Oid tspc_oid) { Tablespaces *tspcs = ts_tablespace_scan(ht->fd.id); return ts_tablespaces_contain(tspcs, tspc_oid); } static int hypertable_get_chunk_round_robin_index(const Hypertable *ht, const Hypercube *hc) { const Dimension *dim; const DimensionSlice *slice; int offset = 0; Assert(NULL != ht); Assert(NULL != hc); dim = hyperspace_get_closed_dimension(ht->space, 0); if (NULL == dim) { dim = hyperspace_get_open_dimension(ht->space, 0); /* Add some randomness between hypertables so that * if there is no space partitions, but multiple hypertables * the initial index is different for different hypertables. * This protects against creating a lot of chunks on the same * data node when many hypertables are created at roughly * the same time, e.g., from a bootstrap script. */ offset = (int) ht->fd.id; } Assert(NULL != dim); slice = ts_hypercube_get_slice_by_dimension_id(hc, dim->fd.id); Assert(NULL != slice); return ts_dimension_get_slice_ordinal(dim, slice) + offset; } /* * Select a tablespace to use for a given chunk. * * Selection happens based on the first closed (space) dimension, if available, * otherwise the first closed (time) one. * * We try to do "sticky" selection to consistently pick the same tablespace for * chunks in the same closed (space) dimension. This ensures chunks in the same * "space" partition will live on the same disk. */ Tablespace * ts_hypertable_select_tablespace(const Hypertable *ht, const Chunk *chunk) { Tablespaces *tspcs = ts_tablespace_scan(ht->fd.id); int i; if (NULL == tspcs || tspcs->num_tablespaces == 0) return NULL; i = hypertable_get_chunk_round_robin_index(ht, chunk->cube); /* Use the index of the slice to find the tablespace */ return &tspcs->tablespaces[i % tspcs->num_tablespaces]; } const char * ts_hypertable_select_tablespace_name(const Hypertable *ht, const Chunk *chunk) { Tablespace *tspc = ts_hypertable_select_tablespace(ht, chunk); Oid main_tspc_oid; if (tspc != NULL) return NameStr(tspc->fd.tablespace_name); /* Use main table tablespace, if any */ main_tspc_oid = get_rel_tablespace(ht->main_table_relid); if (OidIsValid(main_tspc_oid)) return get_tablespace_name(main_tspc_oid); return NULL; } /* * Get the tablespace at an offset from the given tablespace. */ Tablespace * ts_hypertable_get_tablespace_at_offset_from(int32 hypertable_id, Oid tablespace_oid, int16 offset) { Tablespaces *tspcs = ts_tablespace_scan(hypertable_id); int i = 0; if (NULL == tspcs || tspcs->num_tablespaces == 0) return NULL; for (i = 0; i < tspcs->num_tablespaces; i++) { if (tablespace_oid == tspcs->tablespaces[i].tablespace_oid) return &tspcs->tablespaces[(i + offset) % tspcs->num_tablespaces]; } return NULL; } static inline Oid hypertable_relid_lookup(Oid relid) { Cache *hcache; Hypertable *ht = ts_hypertable_cache_get_cache_and_entry(relid, CACHE_FLAG_MISSING_OK, &hcache); Oid result = (ht == NULL) ? InvalidOid : ht->main_table_relid; ts_cache_release(&hcache); return result; } /* * Returns a hypertable's relation ID (OID) iff the given RangeVar corresponds to * a hypertable, otherwise InvalidOid. */ Oid ts_hypertable_relid(RangeVar *rv) { return hypertable_relid_lookup(RangeVarGetRelid(rv, NoLock, true)); } bool ts_is_hypertable(Oid relid) { if (!OidIsValid(relid)) return false; return OidIsValid(hypertable_relid_lookup(relid)); } /* * Check that the current user can create chunks in a hypertable's associated * schema. * * This function is typically called from create_hypertable() to verify that the * table owner has CREATE permissions for the schema (if it already exists) or * the database (if the schema does not exist and needs to be created). */ static Oid hypertable_check_associated_schema_permissions(const char *schema_name, Oid user_oid) { Oid schema_oid; /* * If the schema name is NULL, it implies the internal catalog schema and * anyone should be able to create chunks there. */ if (NULL == schema_name) return InvalidOid; schema_oid = get_namespace_oid(schema_name, true); /* Anyone can create chunks in the internal schema */ if (strncmp(schema_name, INTERNAL_SCHEMA_NAME, NAMEDATALEN) == 0) { Assert(OidIsValid(schema_oid)); return schema_oid; } if (!OidIsValid(schema_oid)) { /* * Schema does not exist, so we must check that the user has * privileges to create the schema in the current database */ if (object_aclcheck(DatabaseRelationId, MyDatabaseId, user_oid, ACL_CREATE) != ACLCHECK_OK) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permissions denied: cannot create schema \"%s\" in database \"%s\"", schema_name, get_database_name(MyDatabaseId)))); } else if (object_aclcheck(NamespaceRelationId, schema_oid, user_oid, ACL_CREATE) != ACLCHECK_OK) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permissions denied: cannot create chunks in schema \"%s\"", schema_name))); return schema_oid; } inline static bool table_has_rules(Relation rel) { return rel->rd_rules != NULL; } bool ts_hypertable_has_chunks(Oid table_relid, LOCKMODE lockmode) { return find_inheritance_children(table_relid, lockmode) != NIL; } static void hypertable_create_schema(const char *schema_name) { CreateSchemaStmt stmt = { .schemaname = (char *) schema_name, .authrole = NULL, .schemaElts = NIL, .if_not_exists = true, }; CreateSchemaCommand(&stmt, "(generated CREATE SCHEMA command)", -1, -1); } /* * Check that existing table constraints are supported. * * Hypertables do not support the following constraints: * * - NO INHERIT constraints cannot be enforced on a hypertable since they only * exist on the parent table, which will have no tuples. * - FOREIGN KEY constraints referencing a hypertable. */ static void hypertable_validate_constraints(Oid relid) { Relation catalog; SysScanDesc scan; ScanKeyData scankey; HeapTuple tuple; catalog = table_open(ConstraintRelationId, AccessShareLock); ScanKeyInit(&scankey, Anum_pg_constraint_conrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); scan = systable_beginscan(catalog, ConstraintRelidTypidNameIndexId, true, NULL, 1, &scankey); while (HeapTupleIsValid(tuple = systable_getnext(scan))) { Form_pg_constraint form = (Form_pg_constraint) GETSTRUCT(tuple); if (form->contype == CONSTRAINT_FOREIGN) { if (ts_hypertable_relid_to_id(form->confrelid) != INVALID_HYPERTABLE_ID && !is_partitioning_allowed(form->confrelid)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("hypertables cannot be used as foreign key references of " "hypertables"))); } if (form->contype == CONSTRAINT_CHECK && form->connoinherit) ereport(ERROR, (errcode(ERRCODE_INVALID_TABLE_DEFINITION), errmsg("cannot have NO INHERIT constraints on hypertable \"%s\"", get_rel_name(relid)), errhint("Remove all NO INHERIT constraints from table \"%s\" before " "making it a hypertable.", get_rel_name(relid)))); } systable_endscan(scan); /* Check for foreign keys that reference this table */ ScanKeyInit(&scankey, Anum_pg_constraint_confrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); scan = systable_beginscan(catalog, 0, false, NULL, 1, &scankey); while (HeapTupleIsValid(tuple = systable_getnext(scan))) { Form_pg_constraint form = (Form_pg_constraint) GETSTRUCT(tuple); /* * Hypertable <-> hypertable foreign keys are not supported. */ if (form->contype == CONSTRAINT_FOREIGN && ts_hypertable_relid_to_id(form->conrelid) != INVALID_HYPERTABLE_ID) ereport(ERROR, (errcode(ERRCODE_INVALID_TABLE_DEFINITION), errmsg("cannot have FOREIGN KEY constraints to hypertable \"%s\"", get_rel_name(relid)), errhint("Remove all FOREIGN KEY constraints to table \"%s\" before " "making it a hypertable.", get_rel_name(relid)))); } systable_endscan(scan); table_close(catalog, AccessShareLock); } static Datum create_hypertable_datum(FunctionCallInfo fcinfo, const Hypertable *ht, bool created, bool is_generic) { TupleDesc tupdesc; HeapTuple tuple; if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in " "context that cannot accept type record"))); tupdesc = BlessTupleDesc(tupdesc); if (is_generic) { Datum values[Natts_generic_create_hypertable]; bool nulls[Natts_generic_create_hypertable] = { false }; values[AttrNumberGetAttrOffset(Anum_generic_create_hypertable_id)] = Int32GetDatum(ht->fd.id); values[AttrNumberGetAttrOffset(Anum_generic_create_hypertable_created)] = BoolGetDatum(created); tuple = heap_form_tuple(tupdesc, values, nulls); } else { Datum values[Natts_create_hypertable]; bool nulls[Natts_create_hypertable] = { false }; values[AttrNumberGetAttrOffset(Anum_create_hypertable_id)] = Int32GetDatum(ht->fd.id); values[AttrNumberGetAttrOffset(Anum_create_hypertable_schema_name)] = NameGetDatum(&ht->fd.schema_name); values[AttrNumberGetAttrOffset(Anum_create_hypertable_table_name)] = NameGetDatum(&ht->fd.table_name); values[AttrNumberGetAttrOffset(Anum_create_hypertable_created)] = BoolGetDatum(created); tuple = heap_form_tuple(tupdesc, values, nulls); } return HeapTupleGetDatum(tuple); } TS_FUNCTION_INFO_V1(ts_hypertable_create); TS_FUNCTION_INFO_V1(ts_hypertable_create_general); /* * Create a hypertable from an existing table. The specific version of create hypertable API * process the function arguments before calling this function. */ static Datum ts_hypertable_create_internal(FunctionCallInfo fcinfo, Oid table_relid, DimensionInfo *open_dim_info, DimensionInfo *closed_dim_info, Name associated_schema_name, Name associated_table_prefix, bool create_default_indexes, bool if_not_exists, bool migrate_data, text *target_size, Oid sizing_func, bool is_generic) { Cache *hcache; Hypertable *ht; Datum retval; bool created; uint32 flags = 0; ts_feature_flag_check(FEATURE_HYPERTABLE); ChunkSizingInfo chunk_sizing_info = { .table_relid = table_relid, .target_size = target_size, .func = sizing_func, .colname = NameStr(open_dim_info->colname), .check_for_index = !create_default_indexes, }; TS_PREVENT_FUNC_IF_READ_ONLY(); ht = ts_hypertable_cache_get_cache_and_entry(table_relid, CACHE_FLAG_MISSING_OK, &hcache); if (ht) { if (if_not_exists) ereport(NOTICE, (errcode(ERRCODE_TS_HYPERTABLE_EXISTS), errmsg("table \"%s\" is already a hypertable, skipping", get_rel_name(table_relid)))); else ereport(ERROR, (errcode(ERRCODE_TS_HYPERTABLE_EXISTS), errmsg("table \"%s\" is already a hypertable", get_rel_name(table_relid)))); created = false; } else { /* Release previously pinned cache */ ts_cache_release(&hcache); if (closed_dim_info && !closed_dim_info->num_slices_is_set) { /* If the number of partitions isn't specified, default to setting it * to the number of data nodes */ int16 num_partitions = closed_dim_info->num_slices; closed_dim_info->num_slices = num_partitions; closed_dim_info->num_slices_is_set = true; } if (if_not_exists) flags |= HYPERTABLE_CREATE_IF_NOT_EXISTS; if (!create_default_indexes) flags |= HYPERTABLE_CREATE_DISABLE_DEFAULT_INDEXES; if (migrate_data) flags |= HYPERTABLE_CREATE_MIGRATE_DATA; created = ts_hypertable_create_from_info(table_relid, INVALID_HYPERTABLE_ID, flags, open_dim_info, closed_dim_info, associated_schema_name, associated_table_prefix, &chunk_sizing_info); Assert(created); ht = ts_hypertable_cache_get_cache_and_entry(table_relid, CACHE_FLAG_NONE, &hcache); } retval = create_hypertable_datum(fcinfo, ht, created, is_generic); ts_cache_release(&hcache); PG_RETURN_DATUM(retval); } /* * Process create_hypertable parameters for time specific implementation. * * Arguments: * relation REGCLASS * time_column_name NAME * partitioning_column NAME = NULL * number_partitions INTEGER = NULL * associated_schema_name NAME = NULL * associated_table_prefix NAME = NULL * chunk_time_interval anyelement = NULL::BIGINT * create_default_indexes BOOLEAN = TRUE * if_not_exists BOOLEAN = FALSE * partitioning_func REGPROC = NULL * migrate_data BOOLEAN = FALSE * chunk_target_size TEXT = NULL * chunk_sizing_func OID = NULL * time_partitioning_func REGPROC = NULL */ Datum ts_hypertable_create(PG_FUNCTION_ARGS) { Oid table_relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); Name open_dim_name = PG_ARGISNULL(1) ? NULL : PG_GETARG_NAME(1); Name closed_dim_name = PG_ARGISNULL(2) ? NULL : PG_GETARG_NAME(2); int16 num_partitions = PG_ARGISNULL(3) ? -1 : PG_GETARG_INT16(3); Name associated_schema_name = PG_ARGISNULL(4) ? NULL : PG_GETARG_NAME(4); Name associated_table_prefix = PG_ARGISNULL(5) ? NULL : PG_GETARG_NAME(5); Datum default_interval = PG_ARGISNULL(6) ? UnassignedDatum : PG_GETARG_DATUM(6); Oid interval_type = PG_ARGISNULL(6) ? InvalidOid : get_fn_expr_argtype(fcinfo->flinfo, 6); bool create_default_indexes = PG_ARGISNULL(7) ? false : PG_GETARG_BOOL(7); /* Defaults to true in the sql code */ bool if_not_exists = PG_ARGISNULL(8) ? false : PG_GETARG_BOOL(8); regproc closed_partitioning_func = PG_ARGISNULL(9) ? InvalidOid : PG_GETARG_OID(9); bool migrate_data = PG_ARGISNULL(10) ? false : PG_GETARG_BOOL(10); text *target_size = PG_ARGISNULL(11) ? NULL : PG_GETARG_TEXT_P(11); Oid sizing_func = PG_ARGISNULL(12) ? InvalidOid : PG_GETARG_OID(12); regproc open_partitioning_func = PG_ARGISNULL(13) ? InvalidOid : PG_GETARG_OID(13); if (!OidIsValid(table_relid)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("relation cannot be NULL"))); if (get_rel_name(table_relid) == NULL) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("relation with oid %d not found", table_relid))); } if (!open_dim_name) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("partition column cannot be NULL"))); DimensionInfo *open_dim_info = ts_dimension_info_create_open(table_relid, open_dim_name, /* column name */ default_interval, /* interval */ interval_type, /* interval type */ open_partitioning_func /* partitioning func */ ); DimensionInfo *closed_dim_info = NULL; if (closed_dim_name) closed_dim_info = ts_dimension_info_create_closed(table_relid, closed_dim_name, /* column name */ num_partitions, /* number partitions */ closed_partitioning_func /* partitioning func */ ); return ts_hypertable_create_internal(fcinfo, table_relid, open_dim_info, closed_dim_info, associated_schema_name, associated_table_prefix, create_default_indexes, if_not_exists, migrate_data, target_size, sizing_func, false); } static Oid get_sizing_func_oid() { const char *sizing_func_name = "calculate_chunk_interval"; const int sizing_func_nargs = 3; static Oid sizing_func_arg_types[] = { INT4OID, INT8OID, INT8OID }; return ts_get_function_oid(sizing_func_name, FUNCTIONS_SCHEMA_NAME, sizing_func_nargs, sizing_func_arg_types); } /* * Process create_hypertable parameters for generic implementation. * * Arguments: * relation REGCLASS * dimension dimension_info * create_default_indexes BOOLEAN = TRUE * if_not_exists BOOLEAN = FALSE * migrate_data BOOLEAN = FALSE */ Datum ts_hypertable_create_general(PG_FUNCTION_ARGS) { Oid table_relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); DimensionInfo *dim_info = NULL; GETARG_NOTNULL_POINTER(dim_info, 1, "dimension", DimensionInfo); bool create_default_indexes = PG_ARGISNULL(2) ? false : PG_GETARG_BOOL(2); bool if_not_exists = PG_ARGISNULL(3) ? false : PG_GETARG_BOOL(3); bool migrate_data = PG_ARGISNULL(4) ? false : PG_GETARG_BOOL(4); /* * We do not support closed (hash) dimensions for the main partitioning * column. Check that first. The behavior then becomes consistent with the * earlier "ts_hypertable_create" implementation. */ if (IS_CLOSED_DIMENSION(dim_info)) { ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot partition using a closed dimension on primary column"), errhint("Use range partitioning on the primary column."))); } /* * Current implementation requires to provide a valid chunk sizing function * that is being used to populate hypertable catalog information. */ Oid sizing_func = get_sizing_func_oid(); /* * Fill in the rest of the info. */ dim_info->table_relid = table_relid; return ts_hypertable_create_internal(fcinfo, table_relid, dim_info, NULL, /* closed_dim_info */ NULL, /* associated_schema_name */ NULL, /* associated_table_prefix */ create_default_indexes, if_not_exists, migrate_data, NULL, sizing_func, true); } /* Go through columns of parent table and check for column data types. */ static void ts_validate_basetable_columns(Relation *rel) { int attno; TupleDesc tupdesc; tupdesc = RelationGetDescr(*rel); for (attno = 1; attno <= tupdesc->natts; attno++) { Form_pg_attribute attr = TupleDescAttr(tupdesc, attno - 1); /* skip dropped columns */ if (attr->attisdropped) continue; Oid typid = attr->atttypid; switch (typid) { case CHAROID: case VARCHAROID: ereport(WARNING, (errmsg("column type \"%s\" used for \"%s\" does not follow best practices", format_type_be(typid), NameStr(attr->attname)), errhint("Use datatype TEXT instead."))); break; case TIMESTAMPOID: ereport(WARNING, (errmsg("column type \"%s\" used for \"%s\" does not follow best practices", format_type_be(typid), NameStr(attr->attname)), errhint("Use datatype TIMESTAMPTZ instead."))); break; default: break; } } } /* Creates a new hypertable. * * Flags are one of HypertableCreateFlags. * All parameters after tim_dim_info can be NUL * returns 'true' if new hypertable was created, false if 'if_not_exists' and the hypertable already * exists. */ bool ts_hypertable_create_from_info(Oid table_relid, int32 hypertable_id, uint32 flags, DimensionInfo *time_dim_info, DimensionInfo *closed_dim_info, Name associated_schema_name, Name associated_table_prefix, ChunkSizingInfo *chunk_sizing_info) { Cache *hcache; Hypertable *ht; Oid associated_schema_oid; Oid user_oid = GetUserId(); Oid tspc_oid = get_rel_tablespace(table_relid); bool table_has_data; NameData schema_name, table_name, default_associated_schema_name; Relation rel; bool if_not_exists = (flags & HYPERTABLE_CREATE_IF_NOT_EXISTS) != 0; /* quick exit in the easy if-not-exists case to avoid all locking */ if (if_not_exists && ts_is_hypertable(table_relid)) { ereport(NOTICE, (errcode(ERRCODE_TS_HYPERTABLE_EXISTS), errmsg("table \"%s\" is already a hypertable, skipping", get_rel_name(table_relid)))); return false; } /* * Serialize hypertable creation to avoid having multiple transactions * creating the same hypertable simultaneously. The lock should conflict * with itself and RowExclusive, to prevent simultaneous inserts on the * table. Also since TRUNCATE (part of data migrations) takes an * AccessExclusiveLock take that lock level here too so that we don't have * lock upgrades, which are susceptible to deadlocks. If we aren't * migrating data, then shouldn't have much contention on the table thus * not worth optimizing. */ rel = table_open(table_relid, AccessExclusiveLock); /* recheck after getting lock */ if (ts_is_hypertable(table_relid)) { /* * Unlock and return. Note that unlocking is analogous to what PG does * for ALTER TABLE ADD COLUMN IF NOT EXIST */ table_close(rel, AccessExclusiveLock); if (if_not_exists) { ereport(NOTICE, (errcode(ERRCODE_TS_HYPERTABLE_EXISTS), errmsg("table \"%s\" is already a hypertable, skipping", get_rel_name(table_relid)))); return false; } ereport(ERROR, (errcode(ERRCODE_TS_HYPERTABLE_EXISTS), errmsg("table \"%s\" is already a hypertable", get_rel_name(table_relid)))); } /* * Hypertables also get created as part of caggs. Report warnings * only for hypertables created via call to create_hypertable(). */ if (hypertable_id == INVALID_HYPERTABLE_ID) ts_validate_basetable_columns(&rel); /* * Check that the user has permissions to make this table into a * hypertable */ ts_hypertable_permissions_check(table_relid, user_oid); /* Is this the right kind of relation? */ switch (get_rel_relkind(table_relid)) { case RELKIND_PARTITIONED_TABLE: if (!ts_guc_enable_partitioned_hypertables) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("table \"%s\" is already partitioned", get_rel_name(table_relid)), errdetail( "It is not possible to turn partitioned tables into hypertables."))); break; case RELKIND_MATVIEW: case RELKIND_RELATION: break; default: ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("invalid relation type"))); } /* * Check that the table is not part of any publication */ if (GetRelationPublications(table_relid) != NIL || GetAllTablesPublications() != NIL) { ereport(ERROR, (errcode(ERRCODE_TS_OPERATION_NOT_SUPPORTED), errmsg("cannot create hypertable for table \"%s\" because it is part of a " "publication", get_rel_name(table_relid)))); } /* Check that the table doesn't have any unsupported constraints */ hypertable_validate_constraints(table_relid); /* No need to check for data in partitioned tables */ table_has_data = get_rel_relkind(table_relid) == RELKIND_PARTITIONED_TABLE ? false : ts_relation_has_tuples(rel); if ((flags & HYPERTABLE_CREATE_MIGRATE_DATA) == 0 && table_has_data) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("table \"%s\" is not empty", get_rel_name(table_relid)), errhint("You can migrate data by specifying 'migrate_data => true' when calling " "this function."))); if (is_inheritance_table(table_relid)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("table \"%s\" is already partitioned", get_rel_name(table_relid)), errdetail( "It is not possible to turn tables that use inheritance into hypertables."))); if (rel->rd_rel->relpersistence == RELPERSISTENCE_TEMP) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("table \"%s\" cannot be temporary", get_rel_name(table_relid)), errdetail("It is not supported to turn temporary tables into hypertables."))); if (table_has_rules(rel)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("hypertables do not support rules"), errdetail("Table \"%s\" has attached rules, which do not work on hypertables.", get_rel_name(table_relid)), errhint("Remove the rules before creating a hypertable."))); /* * Must close the relation to decrease the reference count for the relation * as PG18+ will check the reference count when adding constraints for the table. */ table_close(rel, NoLock); /* * Create the associated schema where chunks are stored, or, check * permissions if it already exists */ if (NULL == associated_schema_name) { namestrcpy(&default_associated_schema_name, INTERNAL_SCHEMA_NAME); associated_schema_name = &default_associated_schema_name; } associated_schema_oid = hypertable_check_associated_schema_permissions(NameStr(*associated_schema_name), user_oid); /* Create the associated schema if it doesn't already exist */ if (!OidIsValid(associated_schema_oid)) hypertable_create_schema(NameStr(*associated_schema_name)); /* * Hypertables do not support arbitrary triggers, so if the table already * has unsupported triggers we bail out */ ts_check_unsupported_triggers(table_relid); if (NULL == chunk_sizing_info) chunk_sizing_info = ts_chunk_sizing_info_get_default_disabled(table_relid); /* Validate and set chunk sizing information */ if (OidIsValid(chunk_sizing_info->func)) { ts_chunk_adaptive_sizing_info_validate(chunk_sizing_info); if (chunk_sizing_info->target_size_bytes > 0) { ereport(NOTICE, (errcode(ERRCODE_WARNING), errmsg("adaptive chunking is a BETA feature and is not recommended for " "production deployments"))); time_dim_info->adaptive_chunking = true; } } else { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("chunk sizing function cannot be NULL"))); } /* Validate that the dimensions are OK */ ts_dimension_info_validate(time_dim_info); if (DIMENSION_INFO_IS_SET(closed_dim_info)) ts_dimension_info_validate(closed_dim_info); /* Checks pass, now we can create the catalog information */ namestrcpy(&schema_name, get_namespace_name(get_rel_namespace(table_relid))); namestrcpy(&table_name, get_rel_name(table_relid)); hypertable_insert(hypertable_id, &schema_name, &table_name, associated_schema_name, associated_table_prefix, &chunk_sizing_info->func_schema, &chunk_sizing_info->func_name, chunk_sizing_info->target_size_bytes, DIMENSION_INFO_IS_SET(closed_dim_info) ? 2 : 1, false); /* Get the a Hypertable object via the cache */ time_dim_info->ht = ts_hypertable_cache_get_cache_and_entry(table_relid, CACHE_FLAG_NONE, &hcache); /* Add validated dimensions */ ts_dimension_add_from_info(time_dim_info); if (DIMENSION_INFO_IS_SET(closed_dim_info)) { closed_dim_info->ht = time_dim_info->ht; ts_dimension_add_from_info(closed_dim_info); } /* Refresh the cache to get the updated hypertable with added dimensions */ ts_cache_release(&hcache); ht = ts_hypertable_cache_get_cache_and_entry(table_relid, CACHE_FLAG_NONE, &hcache); /* Verify that existing indexes are compatible with a hypertable */ ts_indexing_verify_indexes(ht); /* Attach tablespace, if any */ if (OidIsValid(tspc_oid)) { NameData tspc_name; namestrcpy(&tspc_name, get_tablespace_name(tspc_oid)); ts_tablespace_attach_internal(&tspc_name, table_relid, false); } if ((flags & HYPERTABLE_CREATE_DISABLE_DEFAULT_INDEXES) == 0) ts_indexing_create_default_indexes(ht); /* * Migrate data from the main table to chunks */ if (table_has_data) { ereport(NOTICE, (errmsg("migrating data to chunks"), errdetail("Migration might take a while depending on the amount of data."))); timescaledb_move_from_table_to_chunks(ht, RowExclusiveLock); } ts_cache_release(&hcache); return true; } /* Used as a tuple found function */ static ScanTupleResult hypertable_rename_schema_name(TupleInfo *ti, void *data) { const char **schema_names = (const char **) data; const char *old_schema_name = schema_names[0]; const char *new_schema_name = schema_names[1]; bool updated = false; FormData_hypertable fd; ts_hypertable_formdata_fill(&fd, ti); /* * Because we are doing a heap scan with no scankey, we don't know which * schema name to change, if any */ if (namestrcmp(&fd.schema_name, old_schema_name) == 0) { namestrcpy(&fd.schema_name, new_schema_name); updated = true; } if (namestrcmp(&fd.associated_schema_name, old_schema_name) == 0) { namestrcpy(&fd.associated_schema_name, new_schema_name); updated = true; } if (namestrcmp(&fd.chunk_sizing_func_schema, old_schema_name) == 0) { namestrcpy(&fd.chunk_sizing_func_schema, new_schema_name); updated = true; } /* Only update the catalog if we explicitly something */ if (updated) { HeapTuple new_tuple = hypertable_formdata_make_tuple(&fd, ts_scanner_get_tupledesc(ti)); ts_catalog_update_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti), new_tuple); heap_freetuple(new_tuple); } /* Keep going so we can change the name for all hypertables */ return SCAN_CONTINUE; } /* Go through internal hypertable table and rename all matching schemas */ void ts_hypertables_rename_schema_name(const char *old_name, const char *new_name) { const char *schema_names[2] = { old_name, new_name }; Catalog *catalog = ts_catalog_get(); ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, HYPERTABLE), .index = InvalidOid, .tuple_found = hypertable_rename_schema_name, .data = (void *) schema_names, .lockmode = RowExclusiveLock, .scandirection = ForwardScanDirection, }; ts_scanner_scan(&scanctx); } bool ts_is_partitioning_column(const Hypertable *ht, AttrNumber column_attno) { uint16 i; for (i = 0; i < ht->space->num_dimensions; i++) { if (column_attno == ht->space->dimensions[i].column_attno) return true; } return false; } bool ts_is_partitioning_column_name(const Hypertable *ht, NameData column_name) { uint16 i; for (i = 0; i < ht->space->num_dimensions; i++) { if (namestrcmp(&ht->space->dimensions[i].fd.column_name, NameStr(column_name)) == 0) return true; } return false; } static void integer_now_func_validate(Oid now_func_oid, Oid open_dim_type) { HeapTuple tuple; Form_pg_proc now_func; /* this function should only be called for hypertables with an open integer time dimension */ Assert(IS_INTEGER_TYPE(open_dim_type)); if (!OidIsValid(now_func_oid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_FUNCTION), (errmsg("invalid custom time function")))); tuple = SearchSysCache1(PROCOID, ObjectIdGetDatum(now_func_oid)); if (!HeapTupleIsValid(tuple)) { ereport(ERROR, (errcode(ERRCODE_NO_DATA_FOUND), errmsg("cache lookup failed for function %u", now_func_oid))); } now_func = (Form_pg_proc) GETSTRUCT(tuple); if ((now_func->provolatile != PROVOLATILE_IMMUTABLE && now_func->provolatile != PROVOLATILE_STABLE) || now_func->pronargs != 0) { ReleaseSysCache(tuple); ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid custom time function"), errhint("A custom time function must take no arguments and be STABLE."))); } if (now_func->prorettype != open_dim_type) { ReleaseSysCache(tuple); ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid custom time function"), errhint("The return type of the custom time function must be the same as" " the type of the time column of the hypertable."))); } ReleaseSysCache(tuple); } TS_FUNCTION_INFO_V1(ts_hypertable_set_integer_now_func); Datum ts_hypertable_set_integer_now_func(PG_FUNCTION_ARGS) { Oid table_relid = PG_GETARG_OID(0); Oid now_func_oid = PG_GETARG_OID(1); bool replace_if_exists = PG_GETARG_BOOL(2); Hypertable *hypertable; Cache *hcache; const Dimension *open_dim; Oid open_dim_type; AclResult aclresult; ts_hypertable_permissions_check(table_relid, GetUserId()); hypertable = ts_hypertable_cache_get_cache_and_entry(table_relid, CACHE_FLAG_NONE, &hcache); if (TS_HYPERTABLE_IS_INTERNAL_COMPRESSION_TABLE(hypertable)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("custom time function not supported on internal columnstore table"))); /* validate that the open dimension uses numeric type */ open_dim = hyperspace_get_open_dimension(hypertable->space, 0); if (!replace_if_exists) if (*NameStr(open_dim->fd.integer_now_func_schema) != '\0' || *NameStr(open_dim->fd.integer_now_func) != '\0') ereport(ERROR, (errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("custom time function already set for hypertable \"%s\"", get_rel_name(table_relid)))); open_dim_type = ts_dimension_get_partition_type(open_dim); if (!IS_INTEGER_TYPE(open_dim_type)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("custom time function not supported"), errhint("A custom time function can only be set for hypertables" " that have integer time dimensions."))); integer_now_func_validate(now_func_oid, open_dim_type); aclresult = object_aclcheck(ProcedureRelationId, now_func_oid, GetUserId(), ACL_EXECUTE); if (aclresult != ACLCHECK_OK) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permission denied for function %s", get_func_name(now_func_oid)))); ts_dimension_update(hypertable, &open_dim->fd.column_name, DIMENSION_TYPE_OPEN, NULL, NULL, NULL, &now_func_oid); ts_cache_release(&hcache); PG_RETURN_NULL(); } /*Assume permissions are already checked * set compression state as enabled */ bool ts_hypertable_set_compressed(Hypertable *ht, int32 compressed_hypertable_id) { FormData_hypertable form; ItemPointerData tid; /* lock the tuple entry in the catalog table */ bool found = lock_hypertable_tuple(ht->fd.id, &tid, &form); Ensure(found, "hypertable id %d not found", ht->fd.id); Assert(!TS_HYPERTABLE_IS_INTERNAL_COMPRESSION_TABLE(ht)); form.compression_state = HypertableCompressionEnabled; form.compressed_hypertable_id = compressed_hypertable_id; hypertable_update_catalog_tuple(&tid, &form); return true; } /* set compression_state as disabled and remove any * associated compressed hypertable id */ bool ts_hypertable_unset_compressed(Hypertable *ht) { FormData_hypertable form; ItemPointerData tid; /* lock the tuple entry in the catalog table */ bool found = lock_hypertable_tuple(ht->fd.id, &tid, &form); Ensure(found, "hypertable id %d not found", ht->fd.id); Assert(!TS_HYPERTABLE_IS_INTERNAL_COMPRESSION_TABLE(ht)); form.compression_state = HypertableCompressionOff; form.compressed_hypertable_id = INVALID_HYPERTABLE_ID; hypertable_update_catalog_tuple(&tid, &form); return true; } bool ts_hypertable_set_compress_interval(Hypertable *ht, int64 compress_interval) { Assert(!TS_HYPERTABLE_IS_INTERNAL_COMPRESSION_TABLE(ht)); Dimension *time_dimension = ts_hyperspace_get_mutable_dimension(ht->space, DIMENSION_TYPE_OPEN, 0); return ts_dimension_set_compress_interval(time_dimension, compress_interval) > 0; } /* create a compressed hypertable * table_relid - already created table which we are going to * set up as a compressed hypertable * hypertable_id - id to be used while creating hypertable with * compression property set * NOTE: * compressed hypertable has no dimensions. */ bool ts_hypertable_create_compressed(Oid table_relid, int32 hypertable_id) { Oid user_oid = GetUserId(); Oid tspc_oid = get_rel_tablespace(table_relid); NameData schema_name, table_name, associated_schema_name; ChunkSizingInfo *chunk_sizing_info; LockRelationOid(table_relid, AccessExclusiveLock); /* * Check that the user has permissions to make this table to a compressed * hypertable */ ts_hypertable_permissions_check(table_relid, user_oid); if (ts_is_hypertable(table_relid)) { ereport(ERROR, (errcode(ERRCODE_TS_HYPERTABLE_EXISTS), errmsg("table \"%s\" is already a hypertable", get_rel_name(table_relid)))); } namestrcpy(&schema_name, get_namespace_name(get_rel_namespace(table_relid))); namestrcpy(&table_name, get_rel_name(table_relid)); /* we don't use the chunking size info for managing the compressed table. * But need this to satisfy hypertable constraints */ chunk_sizing_info = ts_chunk_sizing_info_get_default_disabled(table_relid); ts_chunk_sizing_func_validate(chunk_sizing_info->func, chunk_sizing_info); /* Checks pass, now we can create the catalog information */ namestrcpy(&schema_name, get_namespace_name(get_rel_namespace(table_relid))); namestrcpy(&table_name, get_rel_name(table_relid)); namestrcpy(&associated_schema_name, INTERNAL_SCHEMA_NAME); /* compressed hypertable has no dimensions of its own , shares the original hypertable dims*/ hypertable_insert(hypertable_id, &schema_name, &table_name, &associated_schema_name, NULL, &chunk_sizing_info->func_schema, &chunk_sizing_info->func_name, chunk_sizing_info->target_size_bytes, 0 /*num_dimensions*/, true); /* No indexes are created for the compressed hypertable here */ /* Attach tablespace, if any */ if (OidIsValid(tspc_oid)) { NameData tspc_name; namestrcpy(&tspc_name, get_tablespace_name(tspc_oid)); ts_tablespace_attach_internal(&tspc_name, table_relid, false); } return true; } /* * Construct an expression for a dimensional column which is compatible with the max() function. * Normally, this is just the column name, but in the case of UUIDv7 there is no max() function * defined for the type so in that case the expression extracts the timestamp from the UUID. */ static const char * get_expr_for_dim_max(const char *colname, Oid timetype) { if (timetype == UUIDOID) { StringInfoData expr; initStringInfo(&expr); appendStringInfo(&expr, "%s.uuid_timestamp(%s)", ts_extension_schema_name(), quote_identifier(colname)); return expr.data; } return quote_identifier(colname); } /* * Get the max value of an open dimension. */ int64 ts_hypertable_get_open_dim_max_value(const Hypertable *ht, int dimension_index, bool *isnull) { StringInfoData command; const Dimension *dim; int res; bool max_isnull; Datum maxdat; Oid timetype; dim = hyperspace_get_open_dimension(ht->space, dimension_index); if (NULL == dim) elog(ERROR, "invalid open dimension index %d", dimension_index); timetype = ts_dimension_get_partition_type(dim); /* * Query for the last bucket in the materialized hypertable. * Since this might be run as part of a parallel operation * we cannot use SET search_path here to lock down the * search_path and instead have to fully schema-qualify * everything. */ initStringInfo(&command); appendStringInfo(&command, "SELECT pg_catalog.max(%s) FROM %s.%s", get_expr_for_dim_max(NameStr(dim->fd.column_name), timetype), quote_identifier(NameStr(ht->fd.schema_name)), quote_identifier(NameStr(ht->fd.table_name))); if (SPI_connect() != SPI_OK_CONNECT) elog(ERROR, "could not connect to SPI"); res = SPI_execute(command.data, true /* read_only */, 0 /*count*/); if (res < 0) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), (errmsg("could not find the maximum time value for hypertable \"%s\"", get_rel_name(ht->main_table_relid))))); /* In most cases the result type is the same as the time type. However, with UUIDs we first * extract the timestamptz so the result type is timestamptz instead. */ Oid result_type = timetype == UUIDOID ? TIMESTAMPTZOID : timetype; Ensure(SPI_gettypeid(SPI_tuptable->tupdesc, 1) == result_type, "partition types for result (%d) and dimension (%d) do not match", SPI_gettypeid(SPI_tuptable->tupdesc, 1), ts_dimension_get_partition_type(dim)); maxdat = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &max_isnull); if (isnull) *isnull = max_isnull; int64 max_value = max_isnull ? ts_time_get_min(result_type) : ts_time_value_to_internal(maxdat, result_type); res = SPI_finish(); if (res != SPI_OK_FINISH) elog(ERROR, "SPI_finish failed: %s", SPI_result_code_string(res)); return max_value; } bool ts_hypertable_has_compression_table(const Hypertable *ht) { if (ht->fd.compressed_hypertable_id != INVALID_HYPERTABLE_ID) { Assert(ht->fd.compression_state == HypertableCompressionEnabled); return true; } return false; } /* * hypertable status update is done in two steps, similar to * chunk_update_status * This is again equivalent to a SELECT FOR UPDATE, followed by UPDATE * 1. RowShareLock to SELECT for UPDATE * 2. UPDATE status using RowExclusiveLock */ bool ts_hypertable_update_status_osm(Hypertable *ht) { FormData_hypertable form; ItemPointerData tid; /* lock the tuple entry in the catalog table */ bool found = lock_hypertable_tuple(ht->fd.id, &tid, &form); Ensure(found, "hypertable id %d not found", ht->fd.id); if (form.status != ht->fd.status) { form.status = ht->fd.status; hypertable_update_catalog_tuple(&tid, &form); } return true; } bool ts_hypertable_update_chunk_sizing(Hypertable *ht) { FormData_hypertable form; ItemPointerData tid; /* lock the tuple entry in the catalog table */ bool found = lock_hypertable_tuple(ht->fd.id, &tid, &form); Ensure(found, "hypertable id %d not found", ht->fd.id); if (OidIsValid(ht->chunk_sizing_func)) { const Dimension *dim = ts_hyperspace_get_dimension(ht->space, DIMENSION_TYPE_OPEN, 0); ChunkSizingInfo info = { .table_relid = ht->main_table_relid, .colname = dim == NULL ? NULL : NameStr(dim->fd.column_name), .func = ht->chunk_sizing_func, }; ts_chunk_adaptive_sizing_info_validate(&info); namestrcpy(&form.chunk_sizing_func_schema, NameStr(info.func_schema)); namestrcpy(&form.chunk_sizing_func_name, NameStr(info.func_name)); } else elog(ERROR, "chunk sizing function cannot be NULL"); form.chunk_target_size = ht->fd.chunk_target_size; hypertable_update_catalog_tuple(&tid, &form); return true; } DimensionSlice * ts_chunk_get_osm_slice_and_lock(int32 osm_chunk_id, int32 time_dim_id, LockTupleMode tuplockmode, LOCKMODE tablelockmode) { ChunkConstraints *constraints = ts_chunk_constraint_scan_by_chunk_id(osm_chunk_id, 1, CurrentMemoryContext); for (int i = 0; i < constraints->num_constraints; i++) { ChunkConstraint *cc = chunk_constraints_get(constraints, i); if (is_dimension_constraint(cc)) { ScanTupLock tuplock = { .lockmode = tuplockmode, .waitpolicy = LockWaitBlock, }; /* * We cannot acquire a tuple lock when running in recovery mode * since that prevents scans on tiered hypertables from running * on a read-only secondary. Acquiring a tuple lock requires * assigning a transaction id for the current transaction state * which is not possible in recovery mode. So we only acquire the * lock if we are not in recovery mode. */ ScanTupLock *const tuplock_ptr = RecoveryInProgress() ? NULL : &tuplock; if (!IsolationUsesXactSnapshot()) { /* in read committed mode, we follow all updates to this tuple */ tuplock.lockflags |= TUPLE_LOCK_FLAG_FIND_LAST_VERSION; } DimensionSlice *dimslice = ts_dimension_slice_scan_by_id_and_lock(cc->fd.dimension_slice_id, tuplock_ptr, CurrentMemoryContext, tablelockmode); if (dimslice->fd.dimension_id == time_dim_id) return dimslice; } } return NULL; } /* * hypertable_osm_range_update * 0 hypertable REGCLASS, * 1 range_start=NULL::bigint, * 2 range_end=NULL, * 3 empty=false * If empty is set to true then the range will be set to invalid range * but the overlap flag will be unset, indicating that no data is managed * by OSM and therefore timescaledb optimizations can be applied. */ TS_FUNCTION_INFO_V1(ts_hypertable_osm_range_update); Datum ts_hypertable_osm_range_update(PG_FUNCTION_ARGS) { Oid relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); Hypertable *ht; const Dimension *time_dim; Cache *hcache; Oid time_type; /* required for resolving the argument types, should match the hypertable partitioning column type */ /* * This function is not meant to be run on a read-only secondary. It is * only used by OSM to update chunk's range in timescaledb catalog when * tiering configuration changes (a new chunk is created, a chunk drop * etc); OSM would already be holding a lock on a dimension slice tuple * by this moment (which is not possible on read-only instance). * Technically this function can be executed from SQL (e.g. from psql) when * in recovery mode; in that instance an ERROR would be thrown when trying * to update the dimension slice tuple, no harm will be done. */ Assert(!RecoveryInProgress()); hcache = ts_hypertable_cache_pin(); ht = ts_resolve_hypertable_from_table_or_cagg(hcache, relid, true); Assert(ht != NULL); time_dim = hyperspace_get_open_dimension(ht->space, 0); if (time_dim == NULL) ereport(ERROR, errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("could not find time dimension for hypertable %s.%s", quote_identifier(NameStr(ht->fd.schema_name)), quote_identifier(NameStr(ht->fd.table_name)))); time_type = ts_dimension_get_partition_type(time_dim); int32 osm_chunk_id = ts_chunk_get_osm_chunk_id(ht->fd.id); if (osm_chunk_id == INVALID_CHUNK_ID) ereport(ERROR, errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("no OSM chunk found for hypertable %s.%s", quote_identifier(NameStr(ht->fd.schema_name)), quote_identifier(NameStr(ht->fd.table_name)))); /* * range_start, range_end arguments must be converted to internal representation * a NULL start value is interpreted as INT64_MAX - 1 and a NULL end value is * interpreted as INT64_MAX. * Passing both start and end NULL values will reset the range to the default range an * OSM chunk is given upon creation, which is [INT64_MAX - 1, INT64_MAX] */ if ((PG_ARGISNULL(1) && !PG_ARGISNULL(2)) || (!PG_ARGISNULL(1) && PG_ARGISNULL(2))) ereport(ERROR, errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("range_start and range_end parameters must be both NULL or both non-NULL")); Oid argtypes[2]; for (int i = 0; i < 2; i++) { argtypes[i] = get_fn_expr_argtype(fcinfo->flinfo, i + 1); if (!can_coerce_type(1, &argtypes[i], &time_type, COERCION_IMPLICIT) && !PG_ARGISNULL(i + 1)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid time argument type \"%s\"", format_type_be(argtypes[i])), errhint("Try casting the argument to \"%s\".", format_type_be(time_type)))); } int64 range_start_internal, range_end_internal; if (PG_ARGISNULL(1)) range_start_internal = PG_INT64_MAX - 1; else range_start_internal = ts_time_value_to_internal(PG_GETARG_DATUM(1), get_fn_expr_argtype(fcinfo->flinfo, 1)); if (PG_ARGISNULL(2)) range_end_internal = PG_INT64_MAX; else range_end_internal = ts_time_value_to_internal(PG_GETARG_DATUM(2), get_fn_expr_argtype(fcinfo->flinfo, 2)); if (range_start_internal > range_end_internal) ereport(ERROR, errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("dimension slice range_end cannot be less than range_start")); bool osm_chunk_empty = PG_GETARG_BOOL(3); bool overlap = false, range_invalid = false; /* Lock tuple FOR UPDATE */ DimensionSlice *slice = ts_chunk_get_osm_slice_and_lock(osm_chunk_id, time_dim->fd.id, LockTupleExclusive, RowShareLock); if (!slice) ereport(ERROR, errmsg("could not find time dimension slice for chunk %d", osm_chunk_id)); int32 dimension_slice_id = slice->fd.id; overlap = ts_osm_chunk_range_overlaps(dimension_slice_id, slice->fd.dimension_id, range_start_internal, range_end_internal); /* * It should not be possible for OSM chunks to overlap with the range * managed by timescaledb. OSM extension should update the range of the * OSM chunk to [INT64_MAX -1, infinity) when it detects that it is * noncontiguous, so we should not end up detecting overlaps anyway. * But throw an error in case we encounter this situation. */ if (overlap) ereport(ERROR, errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("attempting to set overlapping range for tiered chunk of %s.%s", NameStr(ht->fd.schema_name), NameStr(ht->fd.table_name)), errhint("Range should be set to invalid for tiered chunk")); range_invalid = ts_osm_chunk_range_is_invalid(range_start_internal, range_end_internal); /* Update the hypertable flags regarding the validity of the OSM range */ if (range_invalid) { /* range is set to infinity so the OSM chunk is considered last */ range_start_internal = PG_INT64_MAX - 1; range_end_internal = PG_INT64_MAX; if (!osm_chunk_empty) ht->fd.status = ts_set_flags_32(ht->fd.status, HYPERTABLE_STATUS_OSM_CHUNK_NONCONTIGUOUS); else ht->fd.status = ts_clear_flags_32(ht->fd.status, HYPERTABLE_STATUS_OSM_CHUNK_NONCONTIGUOUS); } else ht->fd.status = ts_clear_flags_32(ht->fd.status, HYPERTABLE_STATUS_OSM_CHUNK_NONCONTIGUOUS); ts_hypertable_update_status_osm(ht); ts_cache_release(&hcache); slice->fd.range_start = range_start_internal; slice->fd.range_end = range_end_internal; ts_dimension_slice_range_update(slice); PG_RETURN_BOOL(overlap); } TSDLLEXPORT bool ts_hypertable_has_continuous_aggregates(int32 hypertable_id) { bool found = false; ScanIterator iterator = ts_scan_iterator_create(CONTINUOUS_AGG, AccessShareLock, CurrentMemoryContext); iterator.ctx.limit = 1; /* we only need to know if there is at least one */ iterator.ctx.index = catalog_get_index(ts_catalog_get(), CONTINUOUS_AGG, CONTINUOUS_AGG_RAW_HYPERTABLE_ID_IDX); ts_scan_iterator_scan_key_init(&iterator, Anum_continuous_agg_raw_hypertable_id_idx_raw_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); ts_scan_iterator_start_scan(&iterator); if (ts_scan_iterator_next(&iterator)) found = true; ts_scan_iterator_close(&iterator); return found; } ================================================ FILE: src/hypertable.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/primnodes.h> #include <utils/array.h> #include "chunk_adaptive.h" #include "dimension.h" #include "export.h" #include "hypertable_cache.h" #include "scan_iterator.h" #include "scanner.h" #include "ts_catalog/catalog.h" #include "ts_catalog/tablespace.h" #define INVALID_HYPERTABLE_ID 0 typedef struct SubspaceStore SubspaceStore; typedef struct Chunk Chunk; typedef struct Hypercube Hypercube; typedef struct ChunkRangeSpace ChunkRangeSpace; enum { HypertableCompressionOff = 0, HypertableCompressionEnabled = 1, HypertableInternalCompressionTable = 2, }; #define TS_HYPERTABLE_HAS_COMPRESSION_TABLE(ht) ts_hypertable_has_compression_table(ht) #define TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(ht) \ ((ht)->fd.compression_state == HypertableCompressionEnabled) #define TS_HYPERTABLE_IS_INTERNAL_COMPRESSION_TABLE(ht) \ ((ht)->fd.compression_state == HypertableInternalCompressionTable) typedef struct Hypertable { FormData_hypertable fd; Oid main_table_relid; Oid chunk_sizing_func; Hyperspace *space; SubspaceStore *chunk_cache; ChunkRangeSpace *range_space; } Hypertable; /* create_hypertable record attribute numbers */ enum Anum_create_hypertable { Anum_create_hypertable_id = 1, Anum_create_hypertable_schema_name, Anum_create_hypertable_table_name, Anum_create_hypertable_created, _Anum_create_hypertable_max, }; #define Natts_create_hypertable (_Anum_create_hypertable_max - 1) /* Create a generic hypertable */ enum Anum_generic_create_hypertable { Anum_generic_create_hypertable_id = 1, Anum_generic_create_hypertable_created, _Anum_generic_create_hypertable_max, }; #define Natts_generic_create_hypertable (_Anum_generic_create_hypertable_max - 1) extern TSDLLEXPORT Oid ts_rel_get_owner(Oid relid); typedef enum HypertableCreateFlags { HYPERTABLE_CREATE_DISABLE_DEFAULT_INDEXES = 1 << 0, HYPERTABLE_CREATE_IF_NOT_EXISTS = 1 << 1, HYPERTABLE_CREATE_MIGRATE_DATA = 1 << 2, } HypertableCreateFlags; extern TSDLLEXPORT bool ts_hypertable_create_from_info(Oid table_relid, int32 hypertable_id, uint32 flags, DimensionInfo *time_dim_info, DimensionInfo *closed_dim_info, Name associated_schema_name, Name associated_table_prefix, ChunkSizingInfo *chunk_sizing_info); extern TSDLLEXPORT bool ts_hypertable_create_compressed(Oid table_relid, int32 hypertable_id); extern TSDLLEXPORT Hypertable *ts_hypertable_get_by_id(int32 hypertable_id); extern Hypertable *ts_hypertable_get_by_name(const char *schema, const char *name); extern TSDLLEXPORT bool ts_hypertable_get_attributes_by_name(const char *schema, const char *name, FormData_hypertable *form); extern TSDLLEXPORT bool ts_hypertable_has_privs_of(Oid hypertable_oid, Oid userid); extern TSDLLEXPORT Oid ts_hypertable_permissions_check(Oid hypertable_oid, Oid userid); extern TSDLLEXPORT void ts_hypertable_permissions_check_by_id(int32 hypertable_id); extern Hypertable *ts_hypertable_from_tupleinfo(const TupleInfo *ti); extern Hypertable *ts_resolve_hypertable_from_table_or_cagg(Cache *hcache, Oid relid, bool allow_matht); extern int ts_hypertable_scan_with_memory_context(const char *schema, const char *table, tuple_found_func tuple_found, void *data, LOCKMODE lockmode, MemoryContext mctx); extern bool ts_hypertable_update_status_osm(Hypertable *ht); extern bool ts_hypertable_update_chunk_sizing(Hypertable *ht); extern int ts_hypertable_set_name(Hypertable *ht, const char *newname); extern int ts_hypertable_set_schema(Hypertable *ht, const char *newname); extern int ts_hypertable_set_num_dimensions(Hypertable *ht, int16 num_dimensions); extern int ts_hypertable_delete_by_name(const char *schema_name, const char *table_name); extern int ts_hypertable_delete_by_id(int32 hypertable_id); extern TSDLLEXPORT ObjectAddress ts_hypertable_create_trigger(const Hypertable *ht, CreateTrigStmt *stmt, const char *query); extern TSDLLEXPORT void ts_hypertable_drop_invalidation_replication_slot(const char *slot_name); extern TSDLLEXPORT void ts_hypertable_drop_trigger(Oid relid, const char *trigger_name); extern TSDLLEXPORT void ts_hypertable_drop(Hypertable *hypertable, DropBehavior behavior); extern int ts_hypertable_reset_associated_schema_name(const char *associated_schema); extern TSDLLEXPORT Oid ts_hypertable_id_to_relid(int32 hypertable_id, bool return_invalid); extern TSDLLEXPORT int32 ts_hypertable_relid_to_id(Oid relid); extern TSDLLEXPORT Chunk *ts_hypertable_find_chunk_for_point(const Hypertable *h, const Point *point, LOCKMODE lockmode); extern TSDLLEXPORT Chunk *ts_hypertable_chunk_store_add(const Hypertable *h, const Chunk *input_chunk); extern TSDLLEXPORT Chunk *ts_hypertable_create_chunk_for_point(const Hypertable *h, const Point *point, LOCKMODE chunk_lockmode); extern Oid ts_hypertable_relid(RangeVar *rv); extern TSDLLEXPORT bool ts_is_hypertable(Oid relid); extern bool ts_hypertable_has_tablespace(const Hypertable *ht, Oid tspc_oid); extern Tablespace *ts_hypertable_select_tablespace(const Hypertable *ht, const Chunk *chunk); extern const char *ts_hypertable_select_tablespace_name(const Hypertable *ht, const Chunk *chunk); extern Tablespace *ts_hypertable_get_tablespace_at_offset_from(int32 hypertable_id, Oid tablespace_oid, int16 offset); extern TSDLLEXPORT bool ts_hypertable_has_chunks(Oid table_relid, LOCKMODE lockmode); extern void ts_hypertables_rename_schema_name(const char *old_name, const char *new_name); extern bool ts_is_partitioning_column(const Hypertable *ht, AttrNumber column_attno); extern bool ts_is_partitioning_column_name(const Hypertable *ht, NameData column_name); extern TSDLLEXPORT bool ts_hypertable_set_compressed(Hypertable *ht, int32 compressed_hypertable_id); extern TSDLLEXPORT bool ts_hypertable_unset_compressed(Hypertable *ht); extern TSDLLEXPORT bool ts_hypertable_set_compress_interval(Hypertable *ht, int64 compress_interval); extern TSDLLEXPORT int64 ts_hypertable_get_open_dim_max_value(const Hypertable *ht, int dimension_index, bool *isnull); extern TSDLLEXPORT bool ts_hypertable_has_compression_table(const Hypertable *ht); extern TSDLLEXPORT bool ts_hypertable_has_continuous_aggregates(int32 hypertable_id); extern TSDLLEXPORT void ts_hypertable_formdata_fill(FormData_hypertable *fd, const TupleInfo *ti); #define hypertable_scan(schema, table, tuple_found, data, lockmode) \ ts_hypertable_scan_with_memory_context(schema, \ table, \ tuple_found, \ data, \ lockmode, \ CurrentMemoryContext) #define hypertable_adaptive_chunking_enabled(ht) \ (OidIsValid((ht)->chunk_sizing_func) && (ht)->fd.chunk_target_size > 0) ================================================ FILE: src/hypertable_cache.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/namespace.h> #include <utils/builtins.h> #include <utils/catcache.h> #include <utils/lsyscache.h> #include "cache.h" #include "dimension.h" #include "errors.h" #include "hypertable.h" #include "hypertable_cache.h" #include "scanner.h" #include "ts_catalog/catalog.h" #include "ts_catalog/tablespace.h" static void *hypertable_cache_create_entry(Cache *cache, CacheQuery *query); static void hypertable_cache_missing_error(const Cache *cache, const CacheQuery *query); typedef struct HypertableCacheQuery { CacheQuery q; Oid relid; const char *schema; const char *table; } HypertableCacheQuery; static void * hypertable_cache_get_key(CacheQuery *query) { return &((HypertableCacheQuery *) query)->relid; } typedef struct { Oid relid; Hypertable *hypertable; } HypertableCacheEntry; static bool hypertable_cache_valid_result(const void *result) { if (result == NULL) return false; return ((HypertableCacheEntry *) result)->hypertable != NULL; } static Cache * hypertable_cache_create() { MemoryContext ctx = AllocSetContextCreate(CacheMemoryContext, "Hypertable cache", ALLOCSET_DEFAULT_SIZES); Cache *cache = MemoryContextAlloc(ctx, sizeof(Cache)); Cache template = { .hctl = { .keysize = sizeof(Oid), .entrysize = sizeof(HypertableCacheEntry), .hcxt = ctx, }, .name = "hypertable_cache", .numelements = 16, .flags = HASH_ELEM | HASH_CONTEXT | HASH_BLOBS, .get_key = hypertable_cache_get_key, .create_entry = hypertable_cache_create_entry, .missing_error = hypertable_cache_missing_error, .valid_result = hypertable_cache_valid_result, }; *cache = template; ts_cache_init(cache); return cache; } static Cache *hypertable_cache_current = NULL; static ScanTupleResult hypertable_tuple_found(TupleInfo *ti, void *data) { HypertableCacheEntry *entry = data; entry->hypertable = ts_hypertable_from_tupleinfo(ti); return SCAN_DONE; } static void * hypertable_cache_create_entry(Cache *cache, CacheQuery *query) { HypertableCacheQuery *hq = (HypertableCacheQuery *) query; HypertableCacheEntry *cache_entry = query->result; int number_found; if (NULL == hq->schema) hq->schema = get_namespace_name(get_rel_namespace(hq->relid)); if (NULL == hq->table) hq->table = get_rel_name(hq->relid); number_found = ts_hypertable_scan_with_memory_context(hq->schema, hq->table, hypertable_tuple_found, query->result, AccessShareLock, ts_cache_memory_ctx(cache)); switch (number_found) { case 0: /* Negative cache entry: table is not a hypertable */ cache_entry->hypertable = NULL; break; case 1: Assert(strncmp(NameStr(cache_entry->hypertable->fd.schema_name), hq->schema, NAMEDATALEN) == 0); Assert(strncmp(NameStr(cache_entry->hypertable->fd.table_name), hq->table, NAMEDATALEN) == 0); break; default: elog(ERROR, "got an unexpected number of records: %d", number_found); break; } return cache_entry->hypertable == NULL ? NULL : cache_entry; } static void hypertable_cache_missing_error(const Cache *cache, const CacheQuery *query) { HypertableCacheQuery *hq = (HypertableCacheQuery *) query; const char *const rel_name = get_rel_name(hq->relid); if (rel_name == NULL) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("OID %u does not refer to a table", hq->relid))); else ereport(ERROR, (errcode(ERRCODE_TS_HYPERTABLE_NOT_EXIST), errmsg("table \"%s\" is not a hypertable", rel_name))); } void ts_hypertable_cache_invalidate_callback(void) { ts_cache_invalidate(&hypertable_cache_current); hypertable_cache_current = hypertable_cache_create(); } #ifdef TS_DEBUG TS_FUNCTION_INFO_V1(ts_hypertable_cache_clear); /* * Force a cache clearing. This function is used for debugging purposes. */ Datum ts_hypertable_cache_clear(PG_FUNCTION_ARGS) { ts_hypertable_cache_invalidate_callback(); PG_RETURN_VOID(); } #endif /* Get hypertable cache entry. If the entry is not in the cache, add it. */ Hypertable * ts_hypertable_cache_get_entry(Cache *const cache, const Oid relid, const unsigned int flags) { if (!OidIsValid(relid)) { if (flags & CACHE_FLAG_MISSING_OK) return NULL; else ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("invalid Oid"))); } return ts_hypertable_cache_get_entry_with_table(cache, relid, NULL, NULL, flags); } /* * Returns cache into the argument and hypertable as the function result. * If hypertable is not found, fails with an error. */ Hypertable * ts_hypertable_cache_get_cache_and_entry(const Oid relid, const unsigned int flags, Cache **const cache) { *cache = ts_hypertable_cache_pin(); return ts_hypertable_cache_get_entry(*cache, relid, flags); } Hypertable * ts_hypertable_cache_get_entry_rv(Cache *cache, const RangeVar *rv) { return ts_hypertable_cache_get_entry(cache, RangeVarGetRelid(rv, NoLock, true), CACHE_FLAG_MISSING_OK); } TSDLLEXPORT Hypertable * ts_hypertable_cache_get_entry_by_id(Cache *cache, const int32 hypertable_id) { return ts_hypertable_cache_get_entry(cache, ts_hypertable_id_to_relid(hypertable_id, true), CACHE_FLAG_MISSING_OK); } Hypertable * ts_hypertable_cache_get_entry_with_table(Cache *cache, const Oid relid, const char *schema, const char *table, const unsigned int flags) { HypertableCacheQuery query = { .q.flags = flags, .relid = relid, .schema = schema, .table = table, }; HypertableCacheEntry *entry = ts_cache_fetch(cache, &query.q); Assert((flags & CACHE_FLAG_MISSING_OK) ? true : (entry != NULL && entry->hypertable != NULL)); return entry == NULL ? NULL : entry->hypertable; } extern TSDLLEXPORT Cache * ts_hypertable_cache_pin() { return ts_cache_pin(hypertable_cache_current); } void _hypertable_cache_init(void) { CreateCacheMemoryContext(); hypertable_cache_current = hypertable_cache_create(); } void _hypertable_cache_fini(void) { ts_cache_invalidate(&hypertable_cache_current); } ================================================ FILE: src/hypertable_cache.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include "cache.h" #include "export.h" #include "hypertable.h" /* When a hypertable entry ht is fetched using the cache * i.e. ts_hypertable_cache_get_entry and variants, all related information such as * hyperspaces, dimensions etc are also fetched into the cache. These are allocated in * the cache's memory context. * If the cache pin is released by calling ts_cache_release or variants, the memory * associated with hypertable, its space dimensions etc. have also been released. * As a best practice, call ts_cache_release right before returning from the function * where the cache entry was acquired. This prevents inadvertent errors if someone * modifies this function later and uses an indirectly linked object from the cache. * Example: * void my_func(...) * { * * Hypertable * ht = ts_hypertable_cache_get_xxx(...) * ...... * * if ( error ) * { * elog(ERROR, ... ); <----- ts_cache_release not needed here. * } * * ..... * ts_cache_release(); * return ..; * } * Note that any exceptions/errors i.e. elog/ereport etc. will trigger an automatic * cache release. So there is no need for additional ts_cache_release() calls. */ extern TSDLLEXPORT Hypertable *ts_hypertable_cache_get_entry(Cache *const cache, const Oid relid, const unsigned int flags); extern TSDLLEXPORT Hypertable *ts_hypertable_cache_get_cache_and_entry(const Oid relid, const unsigned int flags, Cache **const cache); extern TSDLLEXPORT Hypertable *ts_hypertable_cache_get_entry_rv(Cache *cache, const RangeVar *rv); extern TSDLLEXPORT Hypertable * ts_hypertable_cache_get_entry_with_table(Cache *cache, const Oid relid, const char *schema, const char *table, const unsigned int flags); extern TSDLLEXPORT Hypertable *ts_hypertable_cache_get_entry_by_id(Cache *cache, const int32 hypertable_id); extern void ts_hypertable_cache_invalidate_callback(void); extern TSDLLEXPORT Cache *ts_hypertable_cache_pin(void); extern void _hypertable_cache_init(void); extern void _hypertable_cache_fini(void); ================================================ FILE: src/hypertable_restrict_info.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_inherits.h> #include <optimizer/optimizer.h> #include <parser/parse_coerce.h> #include <parser/parsetree.h> #include <tcop/tcopprot.h> #include <utils/array.h> #include <utils/builtins.h> #include <utils/lsyscache.h> #include <utils/typcache.h> #include "hypertable_restrict_info.h" #include "chunk.h" #include "chunk_scan.h" #include "dimension.h" #include "dimension_slice.h" #include "dimension_vector.h" #include "expression_utils.h" #include "guc.h" #include "hypercube.h" #include "partitioning.h" #include "scan_iterator.h" #include "ts_catalog/chunk_column_stats.h" #include "utils.h" typedef struct DimensionValues { List *values; bool use_or; /* ORed or ANDed values */ Oid type; /* Oid type for values */ } DimensionValues; static DimensionValues *dimension_values_create(List *values, Oid type, bool use_or); static DimensionRestrictInfoOpen * dimension_restrict_info_open_create(const Dimension *d) { DimensionRestrictInfoOpen *new = palloc(sizeof(DimensionRestrictInfoOpen)); new->base.dimension = d; new->lower_strategy = InvalidStrategy; new->upper_strategy = InvalidStrategy; return new; } static DimensionRestrictInfoClosed * dimension_restrict_info_closed_create(const Dimension *d) { DimensionRestrictInfoClosed *new = palloc(sizeof(DimensionRestrictInfoClosed)); new->partitions = NIL; new->base.dimension = d; new->strategy = InvalidStrategy; return new; } static DimensionRestrictInfo * dimension_restrict_info_create(const Dimension *d) { switch (d->type) { case DIMENSION_TYPE_OPEN: return &dimension_restrict_info_open_create(d)->base; case DIMENSION_TYPE_CLOSED: return &dimension_restrict_info_closed_create(d)->base; default: elog(ERROR, "unknown dimension type"); return NULL; } } /* * Given a column from a hypertable, create a DimensionRestrictInfo entry * representing it. This gets used by the usual hypertable restrict info * machinery to identify if the query clauses are using expressions involving * this column. Note that this column is NOT a partitioning column but we are * tracking ranges for this column in the _timescaledb_catalog.chunk_column_stats * catalog table. * * The idea is to do chunk exclusion when queries have WHERE clauses using this * column. The logic at the "Dimension" entry level are the same, so we reuse the * same representation to benefit from it. */ static DimensionRestrictInfo * chunk_column_stats_restrict_info_create(const Hypertable *ht, const Form_chunk_column_stats d) { /* create a dummy dimension structure for this range entry */ Dimension *dim = ts_chunk_column_stats_fill_dummy_dimension(d, ht->main_table_relid); /* similar to open dimensions */ return &dimension_restrict_info_open_create(dim)->base; } /* * Check if the restriction on this dimension is trivial, that is, the entire * range of the dimension matches. */ static bool dimension_restrict_info_is_trivial(const DimensionRestrictInfo *dri) { switch (dri->dimension->type) { case DIMENSION_TYPE_OPEN: case DIMENSION_TYPE_STATS: { DimensionRestrictInfoOpen *open = (DimensionRestrictInfoOpen *) dri; return open->lower_strategy == InvalidStrategy && open->upper_strategy == InvalidStrategy; } case DIMENSION_TYPE_CLOSED: return ((DimensionRestrictInfoClosed *) dri)->strategy == InvalidStrategy; default: Assert(false); return false; } } /* * Add restriction for open (time) dimension. * Values are expected to be int64 (already converted by caller). */ static bool dimension_restrict_info_open_add(DimensionRestrictInfoOpen *dri, StrategyNumber strategy, DimensionValues *dimvalues) { ListCell *item; bool restriction_added = false; /* * For IN/ANY with multiple equality values on an open dimension, * use the bounding range [min, max] as an over-approximation. * This may include extra chunks, which PG constraint exclusion * will prune later. Much better than returning all chunks. */ if (dimvalues->use_or && list_length(dimvalues->values) > 1) { if (strategy != BTEqualStrategyNumber) return false; int64 min_val = PG_INT64_MAX; int64 max_val = PG_INT64_MIN; ListCell *lc; foreach (lc, dimvalues->values) { int64 value = DatumGetInt64(PointerGetDatum(lfirst(lc))); if (value < min_val) min_val = value; if (value > max_val) max_val = value; } dri->lower_bound = min_val; dri->upper_bound = max_val; dri->lower_strategy = BTGreaterEqualStrategyNumber; dri->upper_strategy = BTLessEqualStrategyNumber; return true; } foreach (item, dimvalues->values) { int64 value = DatumGetInt64(PointerGetDatum(lfirst(item))); switch (strategy) { case BTLessEqualStrategyNumber: case BTLessStrategyNumber: if (dri->upper_strategy == InvalidStrategy || value < dri->upper_bound) { dri->upper_strategy = strategy; dri->upper_bound = value; restriction_added = true; } break; case BTGreaterEqualStrategyNumber: case BTGreaterStrategyNumber: if (dri->lower_strategy == InvalidStrategy || value > dri->lower_bound) { dri->lower_strategy = strategy; dri->lower_bound = value; restriction_added = true; } break; case BTEqualStrategyNumber: dri->lower_bound = value; dri->upper_bound = value; dri->lower_strategy = BTGreaterEqualStrategyNumber; dri->upper_strategy = BTLessEqualStrategyNumber; restriction_added = true; break; default: /* unsupported strategy */ break; } } return restriction_added; } static List * dimension_restrict_info_get_partitions(DimensionRestrictInfoClosed *dri, Oid collation, List *values, Oid value_type) { List *partitions = NIL; ListCell *item; foreach (item, values) { Datum value = ts_dimension_transform_value(dri->base.dimension, collation, PointerGetDatum(lfirst(item)), value_type, NULL); partitions = list_append_unique_int(partitions, DatumGetInt32(value)); } return partitions; } static bool dimension_restrict_info_closed_add(DimensionRestrictInfoClosed *dri, StrategyNumber strategy, Oid collation, DimensionValues *dimvalues) { List *partitions; bool restriction_added = false; if (strategy != BTEqualStrategyNumber) { return false; } partitions = dimension_restrict_info_get_partitions(dri, collation, dimvalues->values, dimvalues->type); /* the intersection is empty when using ALL operator (ANDing values) */ if (list_length(partitions) > 1 && !dimvalues->use_or) { dri->strategy = strategy; dri->partitions = NIL; return true; } if (dri->strategy == InvalidStrategy) /* first time through */ { dri->partitions = partitions; dri->strategy = strategy; restriction_added = true; } else { /* intersection with NULL is NULL */ if (dri->partitions == NIL) return true; /* * We are always ANDing the expressions thus intersection is used. */ dri->partitions = list_intersection_int(dri->partitions, partitions); /* no intersection is also a restriction */ restriction_added = true; } return restriction_added; } HypertableRestrictInfo * ts_hypertable_restrict_info_create(RelOptInfo *rel, Hypertable *ht) { /* If chunk skipping is disabled, we have to empty range_space * in case it was cached earlier. */ ChunkRangeSpace *range_space = ht->range_space; if (!ts_guc_enable_chunk_skipping) range_space = NULL; int num_dimensions = ht->space->num_dimensions + (range_space ? range_space->num_range_cols : 0); HypertableRestrictInfo *res = palloc0(sizeof(HypertableRestrictInfo) + (sizeof(DimensionRestrictInfo *) * num_dimensions)); int i; int range_index = 0; res->num_dimensions = num_dimensions; for (i = 0; i < ht->space->num_dimensions; i++) { DimensionRestrictInfo *dri = dimension_restrict_info_create(&ht->space->dimensions[i]); res->dimension_restriction[i] = dri; range_index++; } /* * We convert the range_space entries into dummy "DimensionRestrictInfo" entries. This allows * the hypertable restrict info machinery to consider these as well. */ for (i = 0; range_space != NULL && i < range_space->num_range_cols; i++) { DimensionRestrictInfo *dri = chunk_column_stats_restrict_info_create(ht, &ht->range_space->range_cols[i]); res->dimension_restriction[range_index++] = dri; } return res; } static DimensionRestrictInfo * hypertable_restrict_info_get(HypertableRestrictInfo *hri, AttrNumber attno) { int i; for (i = 0; i < hri->num_dimensions; i++) { if (hri->dimension_restriction[i]->dimension->column_attno == attno) return hri->dimension_restriction[i]; } return NULL; } typedef DimensionValues *(*get_dimension_values)(Const *c, bool use_or); static void hypertable_restrict_info_add_expr(HypertableRestrictInfo *hri, PlannerInfo *root, Var *v, Expr *expr, Oid op_oid, get_dimension_values func_get_dim_values, bool use_or) { DimensionRestrictInfo *dri; Const *c; RangeTblEntry *rte; Oid columntype; TypeCacheEntry *tce; int strategy; Oid lefttype, righttype; DimensionValues *dimvalues; dri = hypertable_restrict_info_get(hri, v->varattno); /* the attribute is not a dimension */ if (dri == NULL) return; expr = (Expr *) eval_const_expressions(root, (Node *) expr); if (!IsA(expr, Const) || !OidIsValid(op_oid) || !op_strict(op_oid)) return; c = (Const *) expr; /* quick check for a NULL constant */ if (c->constisnull) return; rte = rt_fetch(v->varno, root->parse->rtable); columntype = get_atttype(rte->relid, dri->dimension->column_attno); tce = lookup_type_cache(columntype, TYPECACHE_BTREE_OPFAMILY); if (!op_in_opfamily(op_oid, tce->btree_opf)) return; get_op_opfamily_properties(op_oid, tce->btree_opf, false, &strategy, &lefttype, &righttype); /* * For arrays (ScalarArrayOpExpr), we work with the element type. * Non-constant arrays were already filtered out above by the IsA(expr, Const) * check after eval_const_expressions. */ Oid consttype = c->consttype; Oid const_element_type = get_element_type(consttype); bool is_array = OidIsValid(const_element_type); if (is_array) consttype = const_element_type; /* * Coerce literal values to column type if needed. Coercion is required when * types differ and we use a partitioning function. The partitioning functions * always expect the column type. It is always used for closed dimensions * (space partitioning), and can be set for open dimensions too. * * Open dimensions without custom partitioning function don't need coercion * because the ts_time_value_to_internal_or_infinite() handles the cross-type * comparisons (e.g., date vs timestamp) and integer types directly. * * In Postgres, the cross-type integer inequalities (e.g. int4 column <= int8 * literal) work without coercion using cross-type functions like int48le(). * However, our partition function interface uses the column type, not the * literal type. * * We only use implicit coercions because narrowing casts (int8 -> int4) can * fail at runtime with "integer out of range". When no implicit coercion * exists, we skip chunk exclusion for this clause - correct but slower. */ bool needs_coercion = (consttype != columntype) && (IS_CLOSED_DIMENSION(dri->dimension) || dri->dimension->partitioning != NULL); if (needs_coercion) { Oid funcid; CoercionPathType pathtype = find_coercion_pathway(columntype, consttype, COERCION_IMPLICIT, &funcid); if (pathtype != COERCION_PATH_FUNC) { /* * No usable implicit coercion, skip this clause for TimescaleDB * chunk exclusion. It might be still handled by Postgres constraint * exclusion. * * COERCION_PATH_RELABELTYPE (binary compatible) won't occur * here because PostgreSQL coerces such literals at parse time and * eval_const_expressions() folds any remaining RelabelType(Const). */ return; } Assert(OidIsValid(funcid)); if (is_array) { ArrayIterator iterator = array_create_iterator(DatumGetArrayTypeP(c->constvalue), 0, NULL); Datum elem = (Datum) NULL; bool isnull; List *values = NIL; while (array_iterate(iterator, &elem, &isnull)) { if (!isnull) { Datum coerced = OidFunctionCall1Coll(funcid, c->constcollid, elem); values = lappend(values, DatumGetPointer(coerced)); } } array_free_iterator(iterator); dimvalues = dimension_values_create(values, columntype, use_or); } else { Datum coerced = OidFunctionCall1Coll(funcid, c->constcollid, c->constvalue); dimvalues = dimension_values_create(list_make1(DatumGetPointer(coerced)), columntype, use_or); } } else { dimvalues = func_get_dim_values(c, use_or); } /* * Add restriction based on dimension type. */ if (IS_CLOSED_DIMENSION(dri->dimension)) { if (dimension_restrict_info_closed_add((DimensionRestrictInfoClosed *) dri, strategy, c->constcollid, dimvalues)) hri->num_base_restrictions++; } else { /* Open and stats dimensions: convert values to int64 */ List *int64_values = NIL; ListCell *lc; Oid valuetype = dimvalues->type; foreach (lc, dimvalues->values) { Datum value = PointerGetDatum(lfirst(lc)); int64 internal; if (dri->dimension->partitioning != NULL) { /* Apply partitioning function first, then convert result to int64 */ Oid restype; value = ts_dimension_transform_value(dri->dimension, c->constcollid, value, valuetype, &restype); internal = ts_time_value_to_internal_or_infinite(value, restype); } else { internal = ts_time_value_to_internal_or_infinite(value, valuetype); } int64_values = lappend(int64_values, DatumGetPointer(Int64GetDatum(internal))); } dimvalues->values = int64_values; dimvalues->type = INT8OID; if (dimension_restrict_info_open_add((DimensionRestrictInfoOpen *) dri, strategy, dimvalues)) hri->num_base_restrictions++; } } static DimensionValues * dimension_values_create(List *values, Oid type, bool use_or) { DimensionValues *dimvalues; dimvalues = palloc(sizeof(DimensionValues)); dimvalues->values = values; dimvalues->use_or = use_or; dimvalues->type = type; return dimvalues; } static DimensionValues * dimension_values_create_from_array(Const *c, bool user_or) { ArrayIterator iterator = array_create_iterator(DatumGetArrayTypeP(c->constvalue), 0, NULL); Datum elem = (Datum) NULL; bool isnull; List *values = NIL; Oid base_el_type; while (array_iterate(iterator, &elem, &isnull)) { if (!isnull) values = lappend(values, DatumGetPointer(elem)); } /* it's an array type, lets get the base element type */ base_el_type = get_element_type(c->consttype); if (!OidIsValid(base_el_type)) elog(ERROR, "invalid base element type for array type: \"%s\"", format_type_be(c->consttype)); return dimension_values_create(values, base_el_type, user_or); } static DimensionValues * dimension_values_create_from_single_element(Const *c, bool user_or) { return dimension_values_create(list_make1(DatumGetPointer(c->constvalue)), c->consttype, user_or); } static void hypertable_restrict_info_add_restrict_info(HypertableRestrictInfo *hri, PlannerInfo *root, RestrictInfo *ri) { Oid opno; Var *var; Expr *arg_value; Expr *e = ri->clause; /* Same as constraint_exclusion */ if (contain_mutable_functions((Node *) e)) return; if (ts_extract_expr_args(e, &var, &arg_value, &opno, NULL)) { get_dimension_values value_func; bool use_or; switch (nodeTag(e)) { case T_OpExpr: { value_func = dimension_values_create_from_single_element; use_or = false; break; } case T_ScalarArrayOpExpr: { value_func = dimension_values_create_from_array; use_or = castNode(ScalarArrayOpExpr, e)->useOr; break; } default: /* we don't support other node types */ return; } hypertable_restrict_info_add_expr(hri, root, var, arg_value, opno, value_func, use_or); } } void ts_hypertable_restrict_info_add(HypertableRestrictInfo *hri, PlannerInfo *root, List *base_restrict_infos) { ListCell *lc; foreach (lc, base_restrict_infos) { RestrictInfo *ri = lfirst(lc); hypertable_restrict_info_add_restrict_info(hri, root, ri); } } /* * Scan for dimension slices matching query constraints. * * Matching slices are appended to to the given dimension vector. Note that we * keep the table and index open as long as we do not change the number of * scan keys. If the keys change, but the number of keys is the same, we can * simply "rescan". If the number of keys change, however, we need to end the * scan and start again. */ static DimensionVec * scan_and_append_slices(ScanIterator *it, int old_nkeys, DimensionVec **dv, bool unique) { if (old_nkeys != -1 && old_nkeys != it->ctx.nkeys) ts_scan_iterator_end(it); ts_scan_iterator_start_or_restart_scan(it); while (ts_scan_iterator_next(it)) { TupleInfo *ti = ts_scan_iterator_tuple_info(it); DimensionSlice *slice = ts_dimension_slice_from_tuple(ti); if (NULL != slice) { if (unique) *dv = ts_dimension_vec_add_unique_slice(dv, slice); else *dv = ts_dimension_vec_add_slice(dv, slice); } } return *dv; } /* search dimension_slice catalog table for slices that meet hri restriction */ static List * gather_restriction_dimension_vectors(const HypertableRestrictInfo *hri) { List *dimension_vecs = NIL; ScanIterator it; int i; int old_nkeys = -1; it = ts_dimension_slice_scan_iterator_create(NULL, CurrentMemoryContext); for (i = 0; i < hri->num_dimensions; i++) { DimensionRestrictInfo *dri = hri->dimension_restriction[i]; DimensionVec *dv; Assert(NULL != dri); /* dimension ranges don't need dimension slices */ dv = ts_dimension_vec_create( dri->dimension->type == DIMENSION_TYPE_STATS ? 1 : DIMENSION_VEC_DEFAULT_SIZE); dv->dri = dri; switch (dri->dimension->type) { case DIMENSION_TYPE_OPEN: { const DimensionRestrictInfoOpen *open = (const DimensionRestrictInfoOpen *) dri; ts_dimension_slice_scan_iterator_set_range(&it, open->base.dimension->fd.id, open->upper_strategy, open->upper_bound, open->lower_strategy, open->lower_bound); /* * If we have a condition on the second index column * range_start, use a backward scan direction, so that the index * is able to use the second column as well to choose the * starting point for the scan. * If not, prefer forward direction, because backwards scan is * slightly slower for some reason. * Ideally we need some other index type than btree for this, * because the btree index is not so suited for queries like * "find an interval that contains a given point", which is what * we're doing here. * There is a comment in the Postgres code (_bt_start()) that * explains the logic of selecting a starting point for a btree * index scan in more detail. */ it.ctx.scandirection = open->upper_strategy != InvalidStrategy ? BackwardScanDirection : ForwardScanDirection; dv = scan_and_append_slices(&it, old_nkeys, &dv, false); break; } case DIMENSION_TYPE_CLOSED: { const DimensionRestrictInfoClosed *closed = (const DimensionRestrictInfoClosed *) dri; /* Shouldn't have trivial restriction infos here. */ Assert(closed->strategy == BTEqualStrategyNumber); ListCell *cell; foreach (cell, closed->partitions) { int32 partition = lfirst_int(cell); /* * slice_end >= value && slice_start <= value. * See the comment about scan direction above. */ it.ctx.scandirection = BackwardScanDirection; ts_dimension_slice_scan_iterator_set_range(&it, dri->dimension->fd.id, BTLessEqualStrategyNumber, partition, BTGreaterEqualStrategyNumber, partition); dv = scan_and_append_slices(&it, old_nkeys, &dv, true); } break; } case DIMENSION_TYPE_STATS: { /* an empty dv will be appended for this as a placeholder */ break; } default: elog(ERROR, "unknown dimension type"); return NULL; } Assert(dv->num_slices >= 0); /* * If there is a dimension where no slices match, the result will be * empty. But only do so if it's not a DIMENSION_TYPE_STATS entry. * * For DIMENSION_TYPE_STATS entries, we get the list of chunks * directly later on from "chunk_column_stats" catalog. They do not * have dimension slices. */ if (dv->num_slices == 0 && dri->dimension->type != DIMENSION_TYPE_STATS) { ts_scan_iterator_close(&it); return NIL; } dv = ts_dimension_vec_sort(&dv); dimension_vecs = lappend(dimension_vecs, dv); old_nkeys = it.ctx.nkeys; } ts_scan_iterator_close(&it); Assert(list_length(dimension_vecs) == hri->num_dimensions); return dimension_vecs; } Chunk ** ts_hypertable_restrict_info_get_chunks(HypertableRestrictInfo *hri, Hypertable *ht, bool include_osm, unsigned int *num_chunks) { /* * Remove the dimensions for which we don't have a restriction, that is, * the entire range of the dimension matches. Such dimensions do not * influence the result set, because their every slice matches, so we can * just ignore them when searching for the matching chunks. */ const int old_dimensions = hri->num_dimensions; hri->num_dimensions = 0; for (int i = 0; i < old_dimensions; i++) { DimensionRestrictInfo *dri = hri->dimension_restriction[i]; if (!dimension_restrict_info_is_trivial(dri)) { hri->dimension_restriction[hri->num_dimensions] = dri; hri->num_dimensions++; } } List *chunk_ids = NIL; if (hri->num_dimensions == 0) { /* * No restrictions on hyperspace. Just enumerate all the chunks. */ chunk_ids = ts_chunk_get_chunk_ids_by_hypertable_id(ht->fd.id); /* * If the hypertable has an OSM chunk it would end up in the list * as well. We need to remove it when OSM reads are disabled via GUC * variable. */ if (!include_osm || !ts_guc_enable_osm_reads) { int32 osm_chunk_id = ts_chunk_get_osm_chunk_id(ht->fd.id); chunk_ids = list_delete_int(chunk_ids, osm_chunk_id); } } else { /* * Have some restrictions, enumerate the matching dimension slices. */ List *dimension_vectors = gather_restriction_dimension_vectors(hri); if (list_length(dimension_vectors) == 0) { /* * No dimension slices match for some dimension for which there is * a restriction. This means that no chunks match. */ chunk_ids = NIL; } else { /* Find the chunks matching these dimension ranges/slices. */ chunk_ids = ts_chunk_id_find_in_subspace(ht, dimension_vectors); } int32 osm_chunk_id = ts_chunk_get_osm_chunk_id(ht->fd.id); if (osm_chunk_id != INVALID_CHUNK_ID) { if (!ts_guc_enable_osm_reads) { chunk_ids = list_delete_int(chunk_ids, osm_chunk_id); } else { /* * At this point the OSM chunk was either: * 1. added to the list because it has a valid range that agrees with the * restrictions; * 2. not added because it has a valid range and it was excluded; * 3. not added because it has an invalid range and it was excluded. * If the chunk's range is invalid, only then should we consider adding it, * otherwise the exclusion logic should have correctly included or excluded it from * the list. Also, if the range is invalid but the NONCONTIGUOUS flag is not set, * indicating that the chunk is empty, we don't need to do a scan so we do not add * it either. */ const Dimension *time_dim = hyperspace_get_open_dimension(ht->space, 0); DimensionSlice *slice = ts_chunk_get_osm_slice_and_lock(osm_chunk_id, time_dim->fd.id, LockTupleKeyShare, RowShareLock); bool range_invalid = ts_osm_chunk_range_is_invalid(slice->fd.range_start, slice->fd.range_end); if (range_invalid && ts_flags_are_set_32(ht->fd.status, HYPERTABLE_STATUS_OSM_CHUNK_NONCONTIGUOUS)) chunk_ids = list_append_unique_int(chunk_ids, osm_chunk_id); } } } /* * Sort the ids to have more favorable (closer to sequential) data access * patterns to our catalog tables and indexes. * We don't care about the locking order here, because this code uses * AccessShareLock that doesn't conflict with itself. */ list_sort(chunk_ids, list_int_cmp); return ts_chunk_scan_by_chunk_ids(ht->space, chunk_ids, num_chunks); } /* * Compare two chunks along first dimension and chunk ID (in that priority and * order). */ static int chunk_cmp_impl(const Chunk *c1, const Chunk *c2) { int cmp = ts_dimension_slice_cmp(c1->cube->slices[0], c2->cube->slices[0]); if (cmp == 0) cmp = VALUE_CMP(c1->fd.id, c2->fd.id); return cmp; } static int chunk_cmp(const void *c1, const void *c2) { return chunk_cmp_impl(*((const Chunk **) c1), *((const Chunk **) c2)); } static int chunk_cmp_reverse(const void *c1, const void *c2) { return chunk_cmp_impl(*((const Chunk **) c2), *((const Chunk **) c1)); } /* * get chunk oids ordered by time dimension * * if "chunks" is NULL, we get all the chunks from the catalog. Otherwise we * restrict ourselves to the passed in chunks list. * * nested_oids is a list of lists, chunks that occupy the same time slice will be * in the same list. In the list [[1,2,3],[4,5,6]] chunks 1, 2 and 3 are space partitions of * the same time slice and 4, 5 and 6 are space partitions of the next time slice. * */ Chunk ** ts_hypertable_restrict_info_get_chunks_ordered(HypertableRestrictInfo *hri, Hypertable *ht, bool include_osm, Chunk **chunks, bool reverse, List **nested_oids, unsigned int *num_chunks) { List *slot_chunk_oids = NIL; DimensionSlice *slice = NULL; unsigned int i; if (chunks == NULL) { chunks = ts_hypertable_restrict_info_get_chunks(hri, ht, include_osm, num_chunks); } if (*num_chunks == 0) return NULL; Assert(ht->space->num_dimensions > 0); Assert(IS_OPEN_DIMENSION(&ht->space->dimensions[0])); if (reverse) qsort((void *) chunks, *num_chunks, sizeof(Chunk *), chunk_cmp_reverse); else qsort((void *) chunks, *num_chunks, sizeof(Chunk *), chunk_cmp); for (i = 0; i < *num_chunks; i++) { Chunk *chunk = chunks[i]; if (NULL != slice && ts_dimension_slice_cmp(slice, chunk->cube->slices[0]) != 0 && slot_chunk_oids != NIL) { *nested_oids = lappend(*nested_oids, slot_chunk_oids); slot_chunk_oids = NIL; } if (NULL != nested_oids) slot_chunk_oids = lappend_oid(slot_chunk_oids, chunk->table_id); slice = chunk->cube->slices[0]; } if (slot_chunk_oids != NIL) *nested_oids = lappend(*nested_oids, slot_chunk_oids); return chunks; } ================================================ FILE: src/hypertable_restrict_info.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include "hypertable.h" typedef struct DimensionRestrictInfo { const Dimension *dimension; } DimensionRestrictInfo; typedef struct DimensionRestrictInfoOpen { DimensionRestrictInfo base; int64 lower_bound; /* internal time representation */ StrategyNumber lower_strategy; int64 upper_bound; /* internal time representation */ StrategyNumber upper_strategy; } DimensionRestrictInfoOpen; typedef struct DimensionRestrictInfoClosed { DimensionRestrictInfo base; List *partitions; /* hash values */ StrategyNumber strategy; /* either Invalid or equal */ } DimensionRestrictInfoClosed; /* HypertableRestrictInfo represents restrictions on a hypertable. It uses * range exclusion logic to figure out which chunks can match the description */ typedef struct HypertableRestrictInfo { int num_base_restrictions; /* number of base restrictions * successfully added */ int num_dimensions; DimensionRestrictInfo *dimension_restriction[FLEXIBLE_ARRAY_MEMBER]; /* array of dimension * restrictions */ } HypertableRestrictInfo; extern HypertableRestrictInfo *ts_hypertable_restrict_info_create(RelOptInfo *rel, Hypertable *ht); /* Add restrictions based on a List of RestrictInfo */ extern void ts_hypertable_restrict_info_add(HypertableRestrictInfo *hri, PlannerInfo *root, List *base_restrict_infos); /* Get a list of chunk oids for chunks whose constraints match the restriction clauses */ extern Chunk **ts_hypertable_restrict_info_get_chunks(HypertableRestrictInfo *hri, Hypertable *ht, bool include_osm, unsigned int *num_chunks); extern Chunk **ts_hypertable_restrict_info_get_chunks_ordered(HypertableRestrictInfo *hri, Hypertable *ht, bool include_osm, Chunk **chunks, bool reverse, List **nested_oids, unsigned int *num_chunks); ================================================ FILE: src/import/allpaths.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ /* * This file contains source code that was copied and/or modified from * the PostgreSQL database, which is licensed under the open-source * PostgreSQL License. Please see the NOTICE at the top level * directory for a copy of the PostgreSQL License. */ #include <postgres.h> #include <access/tsmapi.h> #include <catalog/pg_proc.h> #include <foreign/fdwapi.h> #include <miscadmin.h> #include <nodes/nodeFuncs.h> #include <nodes/parsenodes.h> #include <nodes/plannodes.h> #include <optimizer/appendinfo.h> #include <optimizer/clauses.h> #include <optimizer/cost.h> #include <optimizer/optimizer.h> #include <optimizer/pathnode.h> #include <optimizer/paths.h> #include <optimizer/plancat.h> #include <optimizer/planner.h> #include <optimizer/prep.h> #include <utils/lsyscache.h> #include <utils/rel.h> #include <math.h> #include "allpaths.h" #include "chunk.h" #include "cross_module_fn.h" #include "planner/planner.h" static void set_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, Index rti, RangeTblEntry *rte); /* copied from allpaths.c */ static void set_foreign_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) { /* Call the FDW's GetForeignPaths function to generate path(s) */ rel->fdwroutine->GetForeignPaths(root, rel, rte->relid); } /* copied from allpaths.c */ static void set_tablesample_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) { Relids required_outer; Path *path; /* * We don't support pushing join clauses into the quals of a samplescan, * but it could still have required parameterization due to LATERAL refs * in its tlist or TABLESAMPLE arguments. */ required_outer = rel->lateral_relids; /* Consider sampled scan */ path = create_samplescan_path(root, rel, required_outer); /* * If the sampling method does not support repeatable scans, we must avoid * plans that would scan the rel multiple times. Ideally, we'd simply * avoid putting the rel on the inside of a nestloop join; but adding such * a consideration to the planner seems like a great deal of complication * to support an uncommon usage of second-rate sampling methods. Instead, * if there is a risk that the query might perform an unsafe join, just * wrap the SampleScan in a Materialize node. We can check for joins by * counting the membership of all_baserels (note that this correctly * counts inheritance trees as single rels). If we're inside a subquery, * we can't easily check whether a join might occur in the outer query, so * just assume one is possible. * * GetTsmRoutine is relatively expensive compared to the other tests here, * so check repeatable_across_scans last, even though that's a bit odd. */ if ((root->query_level > 1 || bms_membership(root->all_baserels) != BMS_SINGLETON) && !(GetTsmRoutine(rte->tablesample->tsmhandler)->repeatable_across_scans)) { path = (Path *) create_material_path(rel, path); } add_path(rel, path); /* For the moment, at least, there are no other paths to consider */ } /* copied from allpaths.c */ static void ts_create_plain_partial_paths(PlannerInfo *root, RelOptInfo *rel) { int parallel_workers; parallel_workers = compute_parallel_worker(rel, rel->pages, -1, max_parallel_workers_per_gather); /* If any limit was set to zero, the user doesn't want a parallel scan. */ if (parallel_workers <= 0) return; /* Add an unordered partial path based on a parallel sequential scan. */ add_partial_path(rel, create_seqscan_path(root, rel, NULL, parallel_workers)); } /* copied from allpaths.c */ static void set_plain_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) { Relids required_outer; /* * We don't support pushing join clauses into the quals of a seqscan, but * it could still have required parameterization due to LATERAL refs in * its tlist. */ required_outer = rel->lateral_relids; /* Consider sequential scan */ add_path(rel, create_seqscan_path(root, rel, required_outer, 0)); /* If appropriate, consider parallel sequential scan */ if (rel->consider_parallel && required_outer == NULL) ts_create_plain_partial_paths(root, rel); /* Consider index scans */ create_index_paths(root, rel); /* Consider TID scans */ create_tidscan_paths(root, rel); } /* copied from allpaths.c */ void ts_set_append_rel_pathlist(PlannerInfo *root, RelOptInfo *parent_rel, Index parent_rt_index, RangeTblEntry *parent_rte) { List *live_childrels = NIL; ListCell *l; /* * Generate access paths for each member relation, and remember the * non-dummy children. */ foreach (l, root->append_rel_list) { AppendRelInfo *appinfo = (AppendRelInfo *) lfirst(l); /* append_rel_list contains all append rels; ignore others */ if (appinfo->parent_relid != parent_rt_index) continue; /* Re-locate the child RTE and RelOptInfo */ const int child_rt_index = appinfo->child_relid; RelOptInfo *child_rel = root->simple_rel_array[child_rt_index]; /* * If set_append_rel_size() decided the parent appendrel was * parallel-unsafe at some point after visiting this child rel, we * need to propagate the unsafety marking down to the child, so that * we don't generate useless partial paths for it. */ if (!parent_rel->consider_parallel) child_rel->consider_parallel = false; /* * We want to disable planning the index scans on uncompressed chunk * tables of fully compressed chunks. It would be expensive and useless * because the uncompressed chunk tables are empty in this case. * * Note about the 'if' condition: compressed chunk tables expanded from * normal hypertables always have the type TS_REL_CHUNK_STANDALONE. The * direct select from a compressed chunk table would also produce this * type. Another possibility is a direct select from an internal * compression hypertable, where the compressed chunks would have the * type TS_REL_CHUNK_CHILD. We have to filter out all these cases here. * * For standalone chunks or UPDATE/DELETE, we do the same thing in * timescaledb_get_relation_info_hook(). */ Hypertable *ht; TsRelType reltype = ts_classify_relation(root, child_rel, &ht); if (reltype == TS_REL_CHUNK_CHILD && !TS_HYPERTABLE_IS_INTERNAL_COMPRESSION_TABLE(ht)) { const Chunk *chunk = ts_planner_chunk_fetch(root, child_rel); /* * This function is called only in tandem with our own hypertable * expansion, so the Chunk struct must be initialized already. */ Assert(chunk != NULL); if (!ts_chunk_is_partial(chunk) && ts_chunk_is_compressed(chunk)) { child_rel->indexlist = NIL; } } /* * Compute the child's access paths. */ RangeTblEntry *child_rte = root->simple_rte_array[child_rt_index]; set_rel_pathlist(root, child_rel, child_rt_index, child_rte); /* * If child is dummy, ignore it. */ if (IS_DUMMY_REL(child_rel)) continue; /* * Child is live, so add it to the live_childrels list for use below. */ live_childrels = lappend(live_childrels, child_rel); /* If consider startup costs on chunks because we can apply SkipScan, should also consider * startup costs on a hypertable */ if (child_rel->consider_startup) parent_rel->consider_startup = true; } /* Add paths to the append relation. */ add_paths_to_append_rel(root, parent_rel, live_childrels); } /* based on the function in allpaths.c, with the irrelevant branches removed */ static void set_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, Index rti, RangeTblEntry *rte) { if (IS_DUMMY_REL(rel)) { /* We already proved the relation empty, so nothing more to do */ } else { Assert(!rte->inh); switch (rel->rtekind) { case RTE_RELATION: if (rte->relkind == RELKIND_FOREIGN_TABLE) { /* Foreign table */ set_foreign_pathlist(root, rel, rte); } else if (rte->tablesample != NULL) { /* Sampled relation */ set_tablesample_rel_pathlist(root, rel, rte); } else { /* Plain relation */ set_plain_rel_pathlist(root, rel, rte); } break; case RTE_SUBQUERY: case RTE_FUNCTION: case RTE_TABLEFUNC: case RTE_VALUES: case RTE_CTE: case RTE_NAMEDTUPLESTORE: case RTE_RESULT: default: elog(ERROR, "unexpected rtekind: %d", (int) rel->rtekind); break; } } /* * Allow a plugin to editorialize on the set of Paths for this base * relation. It could add new paths (such as CustomPaths) by calling * add_path(), or add_partial_path() if parallel aware. It could also * delete or modify paths added by the core code. */ if (set_rel_pathlist_hook) (*set_rel_pathlist_hook)(root, rel, rti, rte); /* * If this is a baserel, we should normally consider gathering any partial * paths we may have created for it. We have to do this after calling the * set_rel_pathlist_hook, else it cannot add partial paths to be included * here. * * However, if this is an inheritance child, skip it. Otherwise, we could * end up with a very large number of gather nodes, each trying to grab * its own pool of workers. Instead, we'll consider gathering partial * paths for the parent appendrel. * * Also, if this is the topmost scan/join rel (that is, the only baserel), * we postpone gathering until the final scan/join targetlist is available * (see grouping_planner). */ if (rel->reloptkind == RELOPT_BASEREL && bms_membership(root->all_baserels) != BMS_SINGLETON) generate_gather_paths(root, rel, false); /* Now find the cheapest of the paths for this rel */ set_cheapest(rel); #ifdef OPTIMIZER_DEBUG debug_print_rel(root, rel); #endif } /* * set_dummy_rel_pathlist, copied from allpaths.c. * * This was a public function prior to PG12. */ static void set_dummy_rel_pathlist(RelOptInfo *rel) { /* Set dummy size estimates --- we leave attr_widths[] as zeroes */ rel->rows = 0; rel->reltarget->width = 0; /* Discard any pre-existing paths; no further need for them */ rel->pathlist = NIL; rel->partial_pathlist = NIL; /* Set up the dummy path */ add_path(rel, (Path *) create_append_path(NULL, rel, NIL, NIL, NIL, rel->lateral_relids, 0, false, -1)); /* * We set the cheapest-path fields immediately, just in case they were * pointing at some discarded path. This is redundant when we're called * from set_rel_size(), but not when called from elsewhere, and doing it * twice is harmless anyway. */ set_cheapest(rel); } /* copied from allpaths.c */ static void set_rel_consider_parallel(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) { /* * The flag has previously been initialized to false, so we can just * return if it becomes clear that we can't safely set it. */ Assert(!rel->consider_parallel); /* Don't call this if parallelism is disallowed for the entire query. */ Assert(root->glob->parallelModeOK); /* This should only be called for baserels and appendrel children. */ Assert(IS_SIMPLE_REL(rel)); /* Assorted checks based on rtekind. */ switch (rte->rtekind) { case RTE_RELATION: /* * Currently, parallel workers can't access the leader's temporary * tables. We could possibly relax this if the wrote all of its * local buffers at the start of the query and made no changes * thereafter (maybe we could allow hint bit changes), and if we * taught the workers to read them. Writing a large number of * temporary buffers could be expensive, though, and we don't have * the rest of the necessary infrastructure right now anyway. So * for now, bail out if we see a temporary table. */ if (get_rel_persistence(rte->relid) == RELPERSISTENCE_TEMP) return; /* * Table sampling can be pushed down to workers if the sample * function and its arguments are safe. */ if (rte->tablesample != NULL) { char proparallel = func_parallel(rte->tablesample->tsmhandler); if (proparallel != PROPARALLEL_SAFE) return; if (!is_parallel_safe(root, (Node *) rte->tablesample->args)) return; } /* * Ask FDWs whether they can support performing a ForeignScan * within a worker. Most often, the answer will be no. For * example, if the nature of the FDW is such that it opens a TCP * connection with a remote server, each parallel worker would end * up with a separate connection, and these connections might not * be appropriately coordinated between workers and the leader. */ if (rte->relkind == RELKIND_FOREIGN_TABLE) { Assert(rel->fdwroutine); if (!rel->fdwroutine->IsForeignScanParallelSafe) return; if (!rel->fdwroutine->IsForeignScanParallelSafe(root, rel, rte)) return; } /* * There are additional considerations for appendrels, which we'll * deal with in set_append_rel_size and set_append_rel_pathlist. * For now, just set consider_parallel based on the rel's own * quals and targetlist. */ break; case RTE_SUBQUERY: /* * There's no intrinsic problem with scanning a subquery-in-FROM * (as distinct from a SubPlan or InitPlan) in a parallel worker. * If the subquery doesn't happen to have any parallel-safe paths, * then flagging it as consider_parallel won't change anything, * but that's true for plain tables, too. We must set * consider_parallel based on the rel's own quals and targetlist, * so that if a subquery path is parallel-safe but the quals and * projection we're sticking onto it are not, we correctly mark * the SubqueryScanPath as not parallel-safe. (Note that * set_subquery_pathlist() might push some of these quals down * into the subquery itself, but that doesn't change anything.) * * We can't push sub-select containing LIMIT/OFFSET to workers as * there is no guarantee that the row order will be fully * deterministic, and applying LIMIT/OFFSET will lead to * inconsistent results at the top-level. (In some cases, where * the result is ordered, we could relax this restriction. But it * doesn't currently seem worth expending extra effort to do so.) */ { Query *subquery = castNode(Query, rte->subquery); if (limit_needed(subquery)) return; } break; case RTE_JOIN: /* Shouldn't happen; we're only considering baserels here. */ Assert(false); return; case RTE_FUNCTION: /* Check for parallel-restricted functions. */ if (!is_parallel_safe(root, (Node *) rte->functions)) return; break; case RTE_TABLEFUNC: /* not parallel safe */ return; case RTE_VALUES: /* Check for parallel-restricted functions. */ if (!is_parallel_safe(root, (Node *) rte->values_lists)) return; break; case RTE_CTE: /* * CTE tuplestores aren't shared among parallel workers, so we * force all CTE scans to happen in the leader. Also, populating * the CTE would require executing a subplan that's not available * in the worker, might be parallel-restricted, and must get * executed only once. */ return; case RTE_NAMEDTUPLESTORE: /* * tuplestore cannot be shared, at least without more * infrastructure to support that. */ return; case RTE_RESULT: /* RESULT RTEs, in themselves, are no problem. */ break; #if PG18_GE case RTE_GROUP: /* Shouldn't happen; we're only considering baserels here. */ Assert(false); return; #endif } /* * If there's anything in baserestrictinfo that's parallel-restricted, we * give up on parallelizing access to this relation. We could consider * instead postponing application of the restricted quals until we're * above all the parallelism in the plan tree, but it's not clear that * that would be a win in very many cases, and it might be tricky to make * outer join clauses work correctly. It would likely break equivalence * classes, too. */ if (!is_parallel_safe(root, (Node *) rel->baserestrictinfo)) return; /* * Likewise, if the relation's outputs are not parallel-safe, give up. * (Usually, they're just Vars, but sometimes they're not.) */ if (!is_parallel_safe(root, (Node *) rel->reltarget->exprs)) return; /* We have a winner. */ rel->consider_parallel = true; } /* copied from allpaths.c, REL_18_3 */ static void ts_set_append_rel_size(PlannerInfo *root, RelOptInfo *rel, Index rti, RangeTblEntry *rte) { int parentRTindex = rti; bool has_live_children; double parent_tuples; double parent_rows; double parent_size; double *parent_attrsizes; int nattrs; ListCell *l; /* Guard against stack overflow due to overly deep inheritance tree. */ check_stack_depth(); Assert(IS_SIMPLE_REL(rel)); /* * If this is a partitioned baserel, set the consider_partitionwise_join * flag; currently, we only consider partitionwise joins with the baserel * if its targetlist doesn't contain a whole-row Var. */ if (enable_partitionwise_join && rel->reloptkind == RELOPT_BASEREL && rte->relkind == RELKIND_PARTITIONED_TABLE && #if PG16_GE bms_is_empty(rel->attr_needed[InvalidAttrNumber - rel->min_attr])) #else rel->attr_needed[InvalidAttrNumber - rel->min_attr] == NULL) #endif rel->consider_partitionwise_join = true; /* * Initialize to compute size estimates for whole append relation. * * We handle tuples estimates by setting "tuples" to the total number of * tuples accumulated from each live child, rather than using "rows". * Although an appendrel itself doesn't directly enforce any quals, its * child relations may. Therefore, setting "tuples" equal to "rows" for * an appendrel isn't always appropriate, and can lead to inaccurate cost * estimates. For example, when estimating the number of distinct values * from an appendrel, we would be unable to adjust the estimate based on * the restriction selectivity (see estimate_num_groups). * * We handle width estimates by weighting the widths of different child * rels proportionally to their number of rows. This is sensible because * the use of width estimates is mainly to compute the total relation * "footprint" if we have to sort or hash it. To do this, we sum the * total equivalent size (in "double" arithmetic) and then divide by the * total rowcount estimate. This is done separately for the total rel * width and each attribute. * * Note: if you consider changing this logic, beware that child rels could * have zero rows and/or width, if they were excluded by constraints. */ has_live_children = false; parent_tuples = 0; parent_rows = 0; parent_size = 0; nattrs = rel->max_attr - rel->min_attr + 1; parent_attrsizes = (double *) palloc0(nattrs * sizeof(double)); foreach(l, root->append_rel_list) { AppendRelInfo *appinfo = (AppendRelInfo *) lfirst(l); int childRTindex; RangeTblEntry *childRTE; RelOptInfo *childrel; #if PG16_GE List *childrinfos; ListCell *lc; #endif ListCell *parentvars; ListCell *childvars; /* append_rel_list contains all append rels; ignore others */ if (appinfo->parent_relid != (Index) parentRTindex) continue; childRTindex = appinfo->child_relid; childRTE = root->simple_rte_array[childRTindex]; /* * The child rel's RelOptInfo was already created during * add_other_rels_to_query. */ childrel = find_base_rel(root, childRTindex); Assert(childrel->reloptkind == RELOPT_OTHER_MEMBER_REL); /* We may have already proven the child to be dummy. */ if (IS_DUMMY_REL(childrel)) continue; /* * We have to copy the parent's targetlist and quals to the child, * with appropriate substitution of variables. However, the * baserestrictinfo quals were already copied/substituted when the * child RelOptInfo was built. So we don't need any additional setup * before applying constraint exclusion. */ if (relation_excluded_by_constraints(root, childrel, childRTE)) { /* * This child need not be scanned, so we can omit it from the * appendrel. */ set_dummy_rel_pathlist(childrel); continue; } /* * Constraint exclusion failed, so copy the parent's join quals and * targetlist to the child, with appropriate variable substitutions. * * We skip join quals that came from above outer joins that can null * this rel, since they would be of no value while generating paths * for the child. This saves some effort while processing the child * rel, and it also avoids an implementation restriction in * adjust_appendrel_attrs (it can't apply nullingrels to a non-Var). */ #if PG16_GE childrinfos = NIL; foreach(lc, rel->joininfo) { RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); if (!bms_overlap(rinfo->clause_relids, rel->nulling_relids)) childrinfos = lappend(childrinfos, adjust_appendrel_attrs(root, (Node *) rinfo, 1, &appinfo)); } childrel->joininfo = childrinfos; #else childrel->joininfo = (List *) adjust_appendrel_attrs(root, (Node *) rel->joininfo, 1, &appinfo); #endif /* * Now for the child's targetlist. * * NB: the resulting childrel->reltarget->exprs may contain arbitrary * expressions, which otherwise would not occur in a rel's targetlist. * Code that might be looking at an appendrel child must cope with * such. (Normally, a rel's targetlist would only include Vars and * PlaceHolderVars.) XXX we do not bother to update the cost or width * fields of childrel->reltarget; not clear if that would be useful. */ childrel->reltarget->exprs = (List *) adjust_appendrel_attrs(root, (Node *) rel->reltarget->exprs, 1, &appinfo); /* * We have to make child entries in the EquivalenceClass data * structures as well. This is needed either if the parent * participates in some eclass joins (because we will want to consider * inner-indexscan joins on the individual children) or if the parent * has useful pathkeys (because we should try to build MergeAppend * paths that produce those sort orderings). */ if (rel->has_eclass_joins || has_useful_pathkeys(root, rel)) add_child_rel_equivalences(root, appinfo, rel, childrel); childrel->has_eclass_joins = rel->has_eclass_joins; /* * Note: we could compute appropriate attr_needed data for the child's * variables, by transforming the parent's attr_needed through the * translated_vars mapping. However, currently there's no need * because attr_needed is only examined for base relations not * otherrels. So we just leave the child's attr_needed empty. */ /* * If we consider partitionwise joins with the parent rel, do the same * for partitioned child rels. * * Note: here we abuse the consider_partitionwise_join flag by setting * it for child rels that are not themselves partitioned. We do so to * tell try_partitionwise_join() that the child rel is sufficiently * valid to be used as a per-partition input, even if it later gets * proven to be dummy. (It's not usable until we've set up the * reltarget and EC entries, which we just did.) */ if (rel->consider_partitionwise_join) childrel->consider_partitionwise_join = true; /* * If parallelism is allowable for this query in general, see whether * it's allowable for this childrel in particular. But if we've * already decided the appendrel is not parallel-safe as a whole, * there's no point in considering parallelism for this child. For * consistency, do this before calling set_rel_size() for the child. */ if (root->glob->parallelModeOK && rel->consider_parallel) set_rel_consider_parallel(root, childrel, childRTE); /* * Compute the child's size. */ ts_set_rel_size(root, childrel, childRTindex, childRTE); /* * It is possible that constraint exclusion detected a contradiction * within a child subquery, even though we didn't prove one above. If * so, we can skip this child. */ if (IS_DUMMY_REL(childrel)) continue; /* We have at least one live child. */ has_live_children = true; /* * If any live child is not parallel-safe, treat the whole appendrel * as not parallel-safe. In future we might be able to generate plans * in which some children are farmed out to workers while others are * not; but we don't have that today, so it's a waste to consider * partial paths anywhere in the appendrel unless it's all safe. * (Child rels visited before this one will be unmarked in * set_append_rel_pathlist().) */ if (!childrel->consider_parallel) rel->consider_parallel = false; /* * Accumulate size information from each live child. */ Assert(childrel->rows > 0); parent_tuples += childrel->tuples; parent_rows += childrel->rows; parent_size += childrel->reltarget->width * childrel->rows; /* * Accumulate per-column estimates too. We need not do anything for * PlaceHolderVars in the parent list. If child expression isn't a * Var, or we didn't record a width estimate for it, we have to fall * back on a datatype-based estimate. * * By construction, child's targetlist is 1-to-1 with parent's. */ forboth(parentvars, rel->reltarget->exprs, childvars, childrel->reltarget->exprs) { Var *parentvar = (Var *) lfirst(parentvars); Node *childvar = (Node *) lfirst(childvars); if (IsA(parentvar, Var) && parentvar->varno == parentRTindex) { int pndx = parentvar->varattno - rel->min_attr; int32 child_width = 0; if (IsA(childvar, Var) && (Index) ((Var *) childvar)->varno == childrel->relid) { int cndx = ((Var *) childvar)->varattno - childrel->min_attr; child_width = childrel->attr_widths[cndx]; } if (child_width <= 0) child_width = get_typavgwidth(exprType(childvar), exprTypmod(childvar)); Assert(child_width > 0); parent_attrsizes[pndx] += child_width * childrel->rows; } } } if (has_live_children) { /* * Save the finished size estimates. */ int i; Assert(parent_rows > 0); rel->tuples = parent_tuples; rel->rows = parent_rows; rel->reltarget->width = rint(parent_size / parent_rows); for (i = 0; i < nattrs; i++) rel->attr_widths[i] = rint(parent_attrsizes[i] / parent_rows); /* * Note that we leave rel->pages as zero; this is important to avoid * double-counting the appendrel tree in total_table_pages. */ } else { /* * All children were excluded by constraints, so mark the whole * appendrel dummy. We must do this in this phase so that the rel's * dummy-ness is visible when we generate paths for other rels. */ set_dummy_rel_pathlist(rel); } pfree(parent_attrsizes); } /* copied from allpaths.c */ static void set_foreign_size(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) { /* Mark rel with estimated output rows, width, etc */ set_foreign_size_estimates(root, rel); /* Let FDW adjust the size estimates, if it can */ rel->fdwroutine->GetForeignRelSize(root, rel, rte->relid); /* ... but do not let it set the rows estimate to zero */ rel->rows = clamp_row_est(rel->rows); } /* copied from allpaths.c */ static void set_tablesample_rel_size(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) { TableSampleClause *tsc = rte->tablesample; TsmRoutine *tsm; BlockNumber pages; double tuples; /* * Test any partial indexes of rel for applicability. We must do this * first since partial unique indexes can affect size estimates. */ check_index_predicates(root, rel); /* * Call the sampling method's estimation function to estimate the number * of pages it will read and the number of tuples it will return. (Note: * we assume the function returns sane values.) */ tsm = GetTsmRoutine(tsc->tsmhandler); tsm->SampleScanGetSampleSize(root, rel, tsc->args, &pages, &tuples); /* * For the moment, because we will only consider a SampleScan path for the * rel, it's okay to just overwrite the pages and tuples estimates for the * whole relation. If we ever consider multiple path types for sampled * rels, we'll need more complication. */ rel->pages = pages; rel->tuples = tuples; /* Mark rel with estimated output rows, width, etc */ set_baserel_size_estimates(root, rel); } /* copied from allpaths.c */ static void set_plain_rel_size(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte) { /* * Test any partial indexes of rel for applicability. We must do this * first since partial unique indexes can affect size estimates. */ check_index_predicates(root, rel); /* Mark rel with estimated output rows, width, etc */ set_baserel_size_estimates(root, rel); } /* extracted from the same function in allpaths.c * assumes that the root table is either excluded by constraints, or is an * inheritance base table, and that chunks are regular tables */ void ts_set_rel_size(PlannerInfo *root, RelOptInfo *rel, Index rti, RangeTblEntry *rte) { if (rel->reloptkind == RELOPT_BASEREL && relation_excluded_by_constraints(root, rel, rte)) { /* * We proved we don't need to scan the rel via constraint exclusion, * so set up a single dummy path for it. Here we only check this for * regular baserels; if it's an otherrel, CE was already checked in * set_append_rel_size(). * * In this case, we go ahead and set up the relation's path right away * instead of leaving it for set_rel_pathlist to do. This is because * we don't have a convention for marking a rel as dummy except by * assigning a dummy path to it. */ set_dummy_rel_pathlist(rel); } else if (rte->inh) { /* It's an "append relation", process accordingly */ ts_set_append_rel_size(root, rel, rti, rte); } else { switch (rel->rtekind) { case RTE_RELATION: if (rte->relkind == RELKIND_FOREIGN_TABLE) { /* Foreign table */ set_foreign_size(root, rel, rte); } else if (rte->relkind == RELKIND_PARTITIONED_TABLE) { /* * We could get here if asked to scan a partitioned table * with ONLY. In that case we shouldn't scan any of the * partitions, so mark it as a dummy rel. */ set_dummy_rel_pathlist(rel); } else if (rte->tablesample != NULL) { /* Sampled relation */ set_tablesample_rel_size(root, rel, rte); } else { /* Plain relation */ set_plain_rel_size(root, rel, rte); } break; case RTE_SUBQUERY: case RTE_FUNCTION: case RTE_TABLEFUNC: case RTE_VALUES: case RTE_CTE: case RTE_NAMEDTUPLESTORE: case RTE_RESULT: default: elog(ERROR, "unexpected rtekind: %d", (int) rel->rtekind); break; } } /* * We insist that all non-dummy rels have a nonzero rowcount estimate. */ Assert(rel->rows > 0 || IS_DUMMY_REL(rel)); } ================================================ FILE: src/import/allpaths.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/pathnodes.h> #include "export.h" extern void ts_set_rel_size(PlannerInfo *root, RelOptInfo *rel, Index rti, RangeTblEntry *rte); extern void ts_set_append_rel_pathlist(PlannerInfo *root, RelOptInfo *parent_rel, Index parent_rt_index, RangeTblEntry *parent_rte); ================================================ FILE: src/import/heapswap.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ /* * This file contains source code that was copied and/or modified from * the PostgreSQL database, which is licensed under the open-source * PostgreSQL License. Please see the NOTICE at the top level * directory for a copy of the PostgreSQL License. */ #include <postgres.h> #include <access/htup.h> #include <access/multixact.h> #include <access/relation.h> #include <access/table.h> #include <access/toast_internals.h> #include <access/xact.h> #include <catalog/catalog.h> #include <catalog/dependency.h> #include <catalog/heap.h> #include <catalog/index.h> #include <catalog/indexing.h> #include <catalog/objectaccess.h> #include <catalog/pg_am.h> #include <catalog/pg_class.h> #include <commands/defrem.h> #include <commands/progress.h> #include <commands/tablecmds.h> #include <executor/spi.h> #include <nodes/makefuncs.h> #include <nodes/value.h> #include <storage/lmgr.h> #include <storage/lockdefs.h> #include <utils/backend_progress.h> #include <utils/inval.h> #include <utils/lsyscache.h> #include <utils/rel.h> #include <utils/relcache.h> #include <utils/relmapper.h> #include <utils/snapmgr.h> #include <utils/syscache.h> #include "compat/compat.h" #include "heapswap.h" #if PG16_LT typedef Oid RelFileNumber; #define RelFileNumberIsValid OidIsValid #define RelationMapOidToFilenumber RelationMapOidToFilenode #define rd_newRelfilelocatorSubid rd_newRelfilenodeSubid #define rd_firstRelfilelocatorSubid rd_firstRelfilenodeSubid #define RelationAssumeNewRelfilelocator RelationAssumeNewRelfilenode #endif /** * The code in this file is imported from PostgreSQL and slightly modified to: * * 1. Make swap_relation_files() a public function. * 2. Optionally build indexes in finish_heap_swap(). * * The above changes are needed to decouple index building from heap swaps, needed for reorder and * merge chunks. */ /* * Swap the physical files of two given relations. * * We swap the physical identity (reltablespace, relfilenumber) while keeping * the same logical identities of the two relations. relpersistence is also * swapped, which is critical since it determines where buffers live for each * relation. * * We can swap associated TOAST data in either of two ways: recursively swap * the physical content of the toast tables (and their indexes), or swap the * TOAST links in the given relations' pg_class entries. The former is needed * to manage rewrites of shared catalogs (where we cannot change the pg_class * links) while the latter is the only way to handle cases in which a toast * table is added or removed altogether. * * Additionally, the first relation is marked with relfrozenxid set to * frozenXid. It seems a bit ugly to have this here, but the caller would * have to do it anyway, so having it here saves a heap_update. Note: in * the swap-toast-links case, we assume we don't need to change the toast * table's relfrozenxid: the new version of the toast table should already * have relfrozenxid set to RecentXmin, which is good enough. * * Lastly, if r2 and its toast table and toast index (if any) are mapped, * their OIDs are emitted into mapped_tables[]. This is hacky but beats * having to look the information up again later in finish_heap_swap. */ void ts_swap_relation_files(Oid r1, Oid r2, bool target_is_pg_class, bool swap_toast_by_content, bool is_internal, TransactionId frozenXid, MultiXactId cutoffMulti, Oid *mapped_tables) { Relation relRelation; HeapTuple reltup1, reltup2; Form_pg_class relform1, relform2; RelFileNumber relfilenumber1, relfilenumber2; RelFileNumber swaptemp; char swptmpchr; Oid relam1, relam2; /* We need writable copies of both pg_class tuples. */ relRelation = table_open(RelationRelationId, RowExclusiveLock); reltup1 = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(r1)); if (!HeapTupleIsValid(reltup1)) elog(ERROR, "cache lookup failed for relation %u", r1); relform1 = (Form_pg_class) GETSTRUCT(reltup1); reltup2 = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(r2)); if (!HeapTupleIsValid(reltup2)) elog(ERROR, "cache lookup failed for relation %u", r2); relform2 = (Form_pg_class) GETSTRUCT(reltup2); relfilenumber1 = relform1->relfilenode; relfilenumber2 = relform2->relfilenode; relam1 = relform1->relam; relam2 = relform2->relam; if (RelFileNumberIsValid(relfilenumber1) && RelFileNumberIsValid(relfilenumber2)) { /* * Normal non-mapped relations: swap relfilenumbers, reltablespaces, * relpersistence */ Assert(!target_is_pg_class); swaptemp = relform1->relfilenode; relform1->relfilenode = relform2->relfilenode; relform2->relfilenode = swaptemp; swaptemp = relform1->reltablespace; relform1->reltablespace = relform2->reltablespace; relform2->reltablespace = swaptemp; swaptemp = relform1->relam; relform1->relam = relform2->relam; relform2->relam = swaptemp; swptmpchr = relform1->relpersistence; relform1->relpersistence = relform2->relpersistence; relform2->relpersistence = swptmpchr; /* Also swap toast links, if we're swapping by links */ if (!swap_toast_by_content) { swaptemp = relform1->reltoastrelid; relform1->reltoastrelid = relform2->reltoastrelid; relform2->reltoastrelid = swaptemp; } } else { /* * Mapped-relation case. Here we have to swap the relation mappings * instead of modifying the pg_class columns. Both must be mapped. */ if (RelFileNumberIsValid(relfilenumber1) || RelFileNumberIsValid(relfilenumber2)) elog(ERROR, "cannot swap mapped relation \"%s\" with non-mapped relation", NameStr(relform1->relname)); /* * We can't change the tablespace nor persistence of a mapped rel, and * we can't handle toast link swapping for one either, because we must * not apply any critical changes to its pg_class row. These cases * should be prevented by upstream permissions tests, so these checks * are non-user-facing emergency backstop. */ if (relform1->reltablespace != relform2->reltablespace) elog(ERROR, "cannot change tablespace of mapped relation \"%s\"", NameStr(relform1->relname)); if (relform1->relpersistence != relform2->relpersistence) elog(ERROR, "cannot change persistence of mapped relation \"%s\"", NameStr(relform1->relname)); if (relform1->relam != relform2->relam) elog(ERROR, "cannot change access method of mapped relation \"%s\"", NameStr(relform1->relname)); if (!swap_toast_by_content && (relform1->reltoastrelid || relform2->reltoastrelid)) elog(ERROR, "cannot swap toast by links for mapped relation \"%s\"", NameStr(relform1->relname)); /* * Fetch the mappings --- shouldn't fail, but be paranoid */ relfilenumber1 = RelationMapOidToFilenumber(r1, relform1->relisshared); if (!RelFileNumberIsValid(relfilenumber1)) elog(ERROR, "could not find relation mapping for relation \"%s\", OID %u", NameStr(relform1->relname), r1); relfilenumber2 = RelationMapOidToFilenumber(r2, relform2->relisshared); if (!RelFileNumberIsValid(relfilenumber2)) elog(ERROR, "could not find relation mapping for relation \"%s\", OID %u", NameStr(relform2->relname), r2); /* * Send replacement mappings to relmapper. Note these won't actually * take effect until CommandCounterIncrement. */ RelationMapUpdateMap(r1, relfilenumber2, relform1->relisshared, false); RelationMapUpdateMap(r2, relfilenumber1, relform2->relisshared, false); /* Pass OIDs of mapped r2 tables back to caller */ *mapped_tables++ = r2; } /* * Recognize that rel1's relfilenumber (swapped from rel2) is new in this * subtransaction. The rel2 storage (swapped from rel1) may or may not be * new. */ { Relation rel1, rel2; rel1 = relation_open(r1, NoLock); rel2 = relation_open(r2, NoLock); rel2->rd_createSubid = rel1->rd_createSubid; rel2->rd_newRelfilelocatorSubid = rel1->rd_newRelfilelocatorSubid; rel2->rd_firstRelfilelocatorSubid = rel1->rd_firstRelfilelocatorSubid; RelationAssumeNewRelfilelocator(rel1); relation_close(rel1, NoLock); relation_close(rel2, NoLock); } /* * In the case of a shared catalog, these next few steps will only affect * our own database's pg_class row; but that's okay, because they are all * noncritical updates. That's also an important fact for the case of a * mapped catalog, because it's possible that we'll commit the map change * and then fail to commit the pg_class update. */ /* set rel1's frozen Xid and minimum MultiXid */ if (relform1->relkind != RELKIND_INDEX) { Assert(!TransactionIdIsValid(frozenXid) || TransactionIdIsNormal(frozenXid)); relform1->relfrozenxid = frozenXid; relform1->relminmxid = cutoffMulti; } /* swap size statistics too, since new rel has freshly-updated stats */ { int32 swap_pages; float4 swap_tuples; int32 swap_allvisible; swap_pages = relform1->relpages; relform1->relpages = relform2->relpages; relform2->relpages = swap_pages; swap_tuples = relform1->reltuples; relform1->reltuples = relform2->reltuples; relform2->reltuples = swap_tuples; swap_allvisible = relform1->relallvisible; relform1->relallvisible = relform2->relallvisible; relform2->relallvisible = swap_allvisible; } /* * Update the tuples in pg_class --- unless the target relation of the * swap is pg_class itself. In that case, there is zero point in making * changes because we'd be updating the old data that we're about to throw * away. Because the real work being done here for a mapped relation is * just to change the relation map settings, it's all right to not update * the pg_class rows in this case. The most important changes will instead * performed later, in finish_heap_swap() itself. */ if (!target_is_pg_class) { CatalogIndexState indstate; indstate = CatalogOpenIndexes(relRelation); CatalogTupleUpdateWithInfo(relRelation, &reltup1->t_self, reltup1, indstate); CatalogTupleUpdateWithInfo(relRelation, &reltup2->t_self, reltup2, indstate); CatalogCloseIndexes(indstate); } else { /* no update ... but we do still need relcache inval */ CacheInvalidateRelcacheByTuple(reltup1); CacheInvalidateRelcacheByTuple(reltup2); } /* * Now that pg_class has been updated with its relevant information for * the swap, update the dependency of the relations to point to their new * table AM, if it has changed. */ if (relam1 != relam2) { if (changeDependencyFor(RelationRelationId, r1, AccessMethodRelationId, relam1, relam2) != 1) elog(ERROR, "could not change access method dependency for relation \"%s.%s\"", get_namespace_name(get_rel_namespace(r1)), get_rel_name(r1)); if (changeDependencyFor(RelationRelationId, r2, AccessMethodRelationId, relam2, relam1) != 1) elog(ERROR, "could not change access method dependency for relation \"%s.%s\"", get_namespace_name(get_rel_namespace(r2)), get_rel_name(r2)); } /* * Post alter hook for modified relations. The change to r2 is always * internal, but r1 depends on the invocation context. */ InvokeObjectPostAlterHookArg(RelationRelationId, r1, 0, InvalidOid, is_internal); InvokeObjectPostAlterHookArg(RelationRelationId, r2, 0, InvalidOid, true); /* * If we have toast tables associated with the relations being swapped, * deal with them too. */ if (relform1->reltoastrelid || relform2->reltoastrelid) { if (swap_toast_by_content) { if (relform1->reltoastrelid && relform2->reltoastrelid) { /* Recursively swap the contents of the toast tables */ ts_swap_relation_files(relform1->reltoastrelid, relform2->reltoastrelid, target_is_pg_class, swap_toast_by_content, is_internal, frozenXid, cutoffMulti, mapped_tables); } else { /* caller messed up */ elog(ERROR, "cannot swap toast files by content when there's only one"); } } else { /* * We swapped the ownership links, so we need to change dependency * data to match. * * NOTE: it is possible that only one table has a toast table. * * NOTE: at present, a TOAST table's only dependency is the one on * its owning table. If more are ever created, we'd need to use * something more selective than deleteDependencyRecordsFor() to * get rid of just the link we want. */ ObjectAddress baseobject, toastobject; long count; /* * We disallow this case for system catalogs, to avoid the * possibility that the catalog we're rebuilding is one of the * ones the dependency changes would change. It's too late to be * making any data changes to the target catalog. */ if (IsSystemClass(r1, relform1)) elog(ERROR, "cannot swap toast files by links for system catalogs"); /* Delete old dependencies */ if (relform1->reltoastrelid) { count = deleteDependencyRecordsFor(RelationRelationId, relform1->reltoastrelid, false); if (count != 1) elog(ERROR, "expected one dependency record for TOAST table, found %ld", count); } if (relform2->reltoastrelid) { count = deleteDependencyRecordsFor(RelationRelationId, relform2->reltoastrelid, false); if (count != 1) elog(ERROR, "expected one dependency record for TOAST table, found %ld", count); } /* Register new dependencies */ baseobject.classId = RelationRelationId; baseobject.objectSubId = 0; toastobject.classId = RelationRelationId; toastobject.objectSubId = 0; if (relform1->reltoastrelid) { baseobject.objectId = r1; toastobject.objectId = relform1->reltoastrelid; recordDependencyOn(&toastobject, &baseobject, DEPENDENCY_INTERNAL); } if (relform2->reltoastrelid) { baseobject.objectId = r2; toastobject.objectId = relform2->reltoastrelid; recordDependencyOn(&toastobject, &baseobject, DEPENDENCY_INTERNAL); } } } /* * If we're swapping two toast tables by content, do the same for their * valid index. The swap can actually be safely done only if the relations * have indexes. */ if (swap_toast_by_content && relform1->relkind == RELKIND_TOASTVALUE && relform2->relkind == RELKIND_TOASTVALUE) { Oid toastIndex1, toastIndex2; /* Get valid index for each relation */ toastIndex1 = toast_get_valid_index(r1, AccessExclusiveLock); toastIndex2 = toast_get_valid_index(r2, AccessExclusiveLock); ts_swap_relation_files(toastIndex1, toastIndex2, target_is_pg_class, swap_toast_by_content, is_internal, InvalidTransactionId, InvalidMultiXactId, mapped_tables); } /* Clean up. */ heap_freetuple(reltup1); heap_freetuple(reltup2); table_close(relRelation, RowExclusiveLock); /* * Close both relcache entries' smgr links. We need this kludge because * both links will be invalidated during upcoming CommandCounterIncrement. * Whichever of the rels is the second to be cleared will have a dangling * reference to the other's smgr entry. Rather than trying to avoid this * by ordering operations just so, it's easiest to close the links first. * (Fortunately, since one of the entries is local in our transaction, * it's sufficient to clear out our own relcache this way; the problem * cannot arise for other backends when they see our update on the * non-transient relation.) * * Caution: the placement of this step interacts with the decision to * handle toast rels by recursion. When we are trying to rebuild pg_class * itself, the smgr close on pg_class must happen after all accesses in * this function. */ #if PG17_LT /* Not needed as of 21d9c3ee4ef7 in the upstream */ RelationCloseSmgrByOid(r1); RelationCloseSmgrByOid(r2); #endif } /* * Remove the transient table that was built by make_new_heap, and finish * cleaning up (including rebuilding all indexes on the old heap). */ void ts_finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap, bool is_system_catalog, bool swap_toast_by_content, bool check_constraints, bool is_internal, bool reindex, TransactionId frozenXid, MultiXactId cutoffMulti, char newrelpersistence) { ObjectAddress object; Oid mapped_tables[4]; int i; /* Report that we are now swapping relation files */ pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, PROGRESS_CLUSTER_PHASE_SWAP_REL_FILES); /* Zero out possible results from swapped_relation_files */ memset(mapped_tables, 0, sizeof(mapped_tables)); /* * Swap the contents of the heap relations (including any toast tables). * Also set old heap's relfrozenxid to frozenXid. */ ts_swap_relation_files(OIDOldHeap, OIDNewHeap, (OIDOldHeap == RelationRelationId), swap_toast_by_content, is_internal, frozenXid, cutoffMulti, mapped_tables); /* * If it's a system catalog, queue a sinval message to flush all catcaches * on the catalog when we reach CommandCounterIncrement. */ if (is_system_catalog) CacheInvalidateCatalog(OIDOldHeap); if (reindex) { int reindex_flags; ReindexParams reindex_params = { 0 }; /* * Rebuild each index on the relation (but not the toast table, which * is all-new at this point). It is important to do this before the * DROP step because if we are processing a system catalog that will * be used during DROP, we want to have its indexes available. There * is no advantage to the other order anyway because this is all * transactional, so no chance to reclaim disk space before commit. We * do not need a final CommandCounterIncrement() because * reindex_relation does it. * * Note: because index_build is called via reindex_relation, it will * never set indcheckxmin true for the indexes. This is OK even * though in some sense we are building new indexes rather than * rebuilding existing ones, because the new heap won't contain any * HOT chains at all, let alone broken ones, so it can't be necessary * to set indcheckxmin. */ reindex_flags = REINDEX_REL_SUPPRESS_INDEX_USE; if (check_constraints) reindex_flags |= REINDEX_REL_CHECK_CONSTRAINTS; /* * Ensure that the indexes have the same persistence as the parent * relation. */ if (newrelpersistence == RELPERSISTENCE_UNLOGGED) reindex_flags |= REINDEX_REL_FORCE_INDEXES_UNLOGGED; else if (newrelpersistence == RELPERSISTENCE_PERMANENT) reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT; /* Report that we are now reindexing relations */ pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, PROGRESS_CLUSTER_PHASE_REBUILD_INDEX); #if PG17_LT reindex_relation(OIDOldHeap, reindex_flags, &reindex_params); #else reindex_relation(NULL, OIDOldHeap, reindex_flags, &reindex_params); #endif } else { /* Must make changes visible after swap. Normally reindex does it * implicitly. */ CommandCounterIncrement(); } /* Report that we are now doing clean up */ pgstat_progress_update_param(PROGRESS_CLUSTER_PHASE, PROGRESS_CLUSTER_PHASE_FINAL_CLEANUP); /* * If the relation being rebuilt is pg_class, swap_relation_files() * couldn't update pg_class's own pg_class entry (check comments in * swap_relation_files()), thus relfrozenxid was not updated. That's * annoying because a potential reason for doing a VACUUM FULL is a * imminent or actual anti-wraparound shutdown. So, now that we can * access the new relation using its indices, update relfrozenxid. * pg_class doesn't have a toast relation, so we don't need to update the * corresponding toast relation. Not that there's little point moving all * relfrozenxid updates here since swap_relation_files() needs to write to * pg_class for non-mapped relations anyway. */ if (OIDOldHeap == RelationRelationId) { Relation relRelation; HeapTuple reltup; Form_pg_class relform; relRelation = table_open(RelationRelationId, RowExclusiveLock); reltup = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(OIDOldHeap)); if (!HeapTupleIsValid(reltup)) elog(ERROR, "cache lookup failed for relation %u", OIDOldHeap); relform = (Form_pg_class) GETSTRUCT(reltup); relform->relfrozenxid = frozenXid; relform->relminmxid = cutoffMulti; CatalogTupleUpdate(relRelation, &reltup->t_self, reltup); table_close(relRelation, RowExclusiveLock); } /* Destroy new heap with old filenumber */ object.classId = RelationRelationId; object.objectId = OIDNewHeap; object.objectSubId = 0; /* * The new relation is local to our transaction and we know nothing * depends on it, so DROP_RESTRICT should be OK. */ performDeletion(&object, DROP_RESTRICT, PERFORM_DELETION_INTERNAL); /* performDeletion does CommandCounterIncrement at end */ /* * Now we must remove any relation mapping entries that we set up for the * transient table, as well as its toast table and toast index if any. If * we fail to do this before commit, the relmapper will complain about new * permanent map entries being added post-bootstrap. */ for (i = 0; OidIsValid(mapped_tables[i]); i++) RelationMapRemoveMapping(mapped_tables[i]); /* * At this point, everything is kosher except that, if we did toast swap * by links, the toast table's name corresponds to the transient table. * The name is irrelevant to the backend because it's referenced by OID, * but users looking at the catalogs could be confused. Rename it to * prevent this problem. * * Note no lock required on the relation, because we already hold an * exclusive lock on it. */ if (!swap_toast_by_content) { Relation newrel; newrel = table_open(OIDOldHeap, NoLock); if (OidIsValid(newrel->rd_rel->reltoastrelid)) { Oid toastidx; char NewToastName[NAMEDATALEN]; /* Get the associated valid index to be renamed */ toastidx = toast_get_valid_index(newrel->rd_rel->reltoastrelid, NoLock); /* rename the toast table ... */ snprintf(NewToastName, NAMEDATALEN, "pg_toast_%u", OIDOldHeap); RenameRelationInternal(newrel->rd_rel->reltoastrelid, NewToastName, true, false); /* ... and its valid index too. */ snprintf(NewToastName, NAMEDATALEN, "pg_toast_%u_index", OIDOldHeap); RenameRelationInternal(toastidx, NewToastName, true, true); /* * Reset the relrewrite for the toast. The command-counter * increment is required here as we are about to update the tuple * that is updated as part of RenameRelationInternal. */ CommandCounterIncrement(); ResetRelRewrite(newrel->rd_rel->reltoastrelid); } relation_close(newrel, NoLock); } /* if it's not a catalog table, clear any missing attribute settings */ if (!is_system_catalog) { Relation newrel; newrel = table_open(OIDOldHeap, NoLock); RelationClearMissing(newrel); relation_close(newrel, NoLock); } } ================================================ FILE: src/import/heapswap.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ /* * This file contains source code that was copied and/or modified from * the PostgreSQL database, which is licensed under the open-source * PostgreSQL License. Please see the NOTICE at the top level * directory for a copy of the PostgreSQL License. */ #pragma once #include <postgres.h> #include <nodes/pg_list.h> #include <utils/rel.h> #include "export.h" extern TSDLLEXPORT void ts_finish_heap_swap(Oid OIDOldHeap, Oid OIDNewHeap, bool is_system_catalog, bool swap_toast_by_content, bool check_constraints, bool is_internal, bool reindex, TransactionId frozenXid, MultiXactId cutoffMulti, char newrelpersistence); extern TSDLLEXPORT void ts_swap_relation_files(Oid r1, Oid r2, bool target_is_pg_class, bool swap_toast_by_content, bool is_internal, TransactionId frozenXid, MultiXactId cutoffMulti, Oid *mapped_tables); ================================================ FILE: src/import/list.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <nodes/pg_list.h> #include <port/pg_bitutils.h> #include "import/list.h" /* * This file contains source code that was copied and/or modified from * the PostgreSQL database, which is licensed under the open-source * PostgreSQL License. Please see the NOTICE at the top level * directory for a copy of the PostgreSQL License. * * Copied from PostgreSQL 15.0 (2a7ce2e2ce474504a707ec03e128fde66cfb8b48) * without modifications. */ /* Overhead for the fixed part of a List header, measured in ListCells */ #define LIST_HEADER_OVERHEAD \ ((int) (((offsetof(List, initial_elements) - 1) / sizeof(ListCell)) + 1)) /* * Return a freshly allocated List with room for at least min_size cells. * * Since empty non-NIL lists are invalid, new_list() sets the initial length * to min_size, effectively marking that number of cells as valid; the caller * is responsible for filling in their data. */ List * ts_new_list(NodeTag type, int min_size) { List *newlist; int max_size; Assert(min_size > 0); /* * We allocate all the requested cells, and possibly some more, as part of * the same palloc request as the List header. This is a big win for the * typical case of short fixed-length lists. It can lose if we allocate a * moderately long list and then it gets extended; we'll be wasting more * initial_elements[] space than if we'd made the header small. However, * rounding up the request as we do in the normal code path provides some * defense against small extensions. */ #ifndef DEBUG_LIST_MEMORY_USAGE /* * Normally, we set up a list with some extra cells, to allow it to grow * without a repalloc. Prefer cell counts chosen to make the total * allocation a power-of-2, since palloc would round it up to that anyway. * (That stops being true for very large allocations, but very long lists * are infrequent, so it doesn't seem worth special logic for such cases.) * * The minimum allocation is 8 ListCell units, providing either 4 or 5 * available ListCells depending on the machine's word width. Counting * palloc's overhead, this uses the same amount of space as a one-cell * list did in the old implementation, and less space for any longer list. * * We needn't worry about integer overflow; no caller passes min_size * that's more than twice the size of an existing list, so the size limits * within palloc will ensure that we don't overflow here. */ max_size = pg_nextpower2_32(Max(8, min_size + LIST_HEADER_OVERHEAD)); max_size -= LIST_HEADER_OVERHEAD; #else /* * For debugging, don't allow any extra space. This forces any cell * addition to go through enlarge_list() and thus move the existing data. */ max_size = min_size; #endif newlist = (List *) palloc(offsetof(List, initial_elements) + (max_size * sizeof(ListCell))); newlist->type = type; newlist->length = min_size; newlist->max_length = max_size; newlist->elements = newlist->initial_elements; return newlist; } ================================================ FILE: src/import/list.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include "export.h" /* * This file contains source code that was copied and/or modified from * the PostgreSQL database, which is licensed under the open-source * PostgreSQL License. Please see the NOTICE at the top level * directory for a copy of the PostgreSQL License. */ extern TSDLLEXPORT List *ts_new_list(NodeTag type, int min_size); ================================================ FILE: src/import/planner.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ /* * This file contains source code that was copied and/or modified from * the PostgreSQL database, which is licensed under the open-source * PostgreSQL License. Please see the NOTICE at the top level * directory for a copy of the PostgreSQL License. * * These function were copied from the PostgreSQL core planner, since * they were declared static in the core planner, but we need them for * our manipulations. */ #include <postgres.h> #include <access/htup_details.h> #include <catalog/pg_collation.h> #include <catalog/pg_statistic.h> #include <executor/nodeAgg.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <optimizer/clauses.h> #include <optimizer/cost.h> #include <optimizer/optimizer.h> #include <optimizer/paramassign.h> #include <optimizer/paths.h> #include <optimizer/placeholder.h> #include <optimizer/planmain.h> #include <optimizer/planner.h> #include <optimizer/tlist.h> #include <parser/parsetree.h> #include <utils/datum.h> #include <utils/lsyscache.h> #include <utils/rel.h> #include <utils/syscache.h> #include "compat/compat.h" #include "planner.h" static Node *replace_nestloop_params_mutator(Node *node, PlannerInfo *root); static Plan *inject_projection_plan(Plan *subplan, List *tlist, bool parallel_safe); /* copied verbatim from optimizer/util/appendinfo.c at REL_17_6 */ void ts_make_inh_translation_list(Relation oldrelation, Relation newrelation, Index newvarno, AppendRelInfo *appinfo) { List *vars = NIL; AttrNumber *pcolnos; TupleDesc old_tupdesc = RelationGetDescr(oldrelation); TupleDesc new_tupdesc = RelationGetDescr(newrelation); Oid new_relid = RelationGetRelid(newrelation); int oldnatts = old_tupdesc->natts; int newnatts = new_tupdesc->natts; int old_attno; int new_attno = 0; /* Initialize reverse-translation array with all entries zero */ appinfo->num_child_cols = newnatts; appinfo->parent_colnos = pcolnos = (AttrNumber *) palloc0(newnatts * sizeof(AttrNumber)); for (old_attno = 0; old_attno < oldnatts; old_attno++) { Form_pg_attribute att; char *attname; Oid atttypid; int32 atttypmod; Oid attcollation; att = TupleDescAttr(old_tupdesc, old_attno); if (att->attisdropped) { /* Just put NULL into this list entry */ vars = lappend(vars, NULL); continue; } attname = NameStr(att->attname); atttypid = att->atttypid; atttypmod = att->atttypmod; attcollation = att->attcollation; /* * When we are generating the "translation list" for the parent table * of an inheritance set, no need to search for matches. */ if (oldrelation == newrelation) { vars = lappend(vars, makeVar(newvarno, (AttrNumber) (old_attno + 1), atttypid, atttypmod, attcollation, 0)); pcolnos[old_attno] = old_attno + 1; continue; } /* * Otherwise we have to search for the matching column by name. * There's no guarantee it'll have the same column position, because * of cases like ALTER TABLE ADD COLUMN and multiple inheritance. * However, in simple cases, the relative order of columns is mostly * the same in both relations, so try the column of newrelation that * follows immediately after the one that we just found, and if that * fails, let syscache handle it. */ if (new_attno >= newnatts || (att = TupleDescAttr(new_tupdesc, new_attno))->attisdropped || strcmp(attname, NameStr(att->attname)) != 0) { HeapTuple newtup; newtup = SearchSysCacheAttName(new_relid, attname); if (!HeapTupleIsValid(newtup)) elog(ERROR, "could not find inherited attribute \"%s\" of relation \"%s\"", attname, RelationGetRelationName(newrelation)); new_attno = ((Form_pg_attribute) GETSTRUCT(newtup))->attnum - 1; Assert(new_attno >= 0 && new_attno < newnatts); ReleaseSysCache(newtup); att = TupleDescAttr(new_tupdesc, new_attno); } /* Found it, check type and collation match */ if (atttypid != att->atttypid || atttypmod != att->atttypmod) elog(ERROR, "attribute \"%s\" of relation \"%s\" does not match parent's type", attname, RelationGetRelationName(newrelation)); if (attcollation != att->attcollation) elog(ERROR, "attribute \"%s\" of relation \"%s\" does not match parent's collation", attname, RelationGetRelationName(newrelation)); vars = lappend(vars, makeVar(newvarno, (AttrNumber) (new_attno + 1), atttypid, atttypmod, attcollation, 0)); pcolnos[new_attno] = old_attno + 1; new_attno++; } appinfo->translated_vars = vars; } /* copied verbatim from planner.c */ struct PathTarget * ts_make_partial_grouping_target(struct PlannerInfo *root, PathTarget *grouping_target) { struct Query *parse = root->parse; PathTarget *partial_target; struct List *non_group_cols; struct List *non_group_exprs; int i; ListCell *lc; partial_target = create_empty_pathtarget(); non_group_cols = NIL; i = 0; foreach (lc, grouping_target->exprs) { struct Expr *expr = (struct Expr *) lfirst(lc); unsigned int sgref = get_pathtarget_sortgroupref(grouping_target, i); if (sgref && parse->groupClause && get_sortgroupref_clause_noerr(sgref, parse->groupClause) != NULL) { /* * It's a grouping column, so add it to the partial_target as-is. * (This allows the upper agg step to repeat the grouping calcs.) */ add_column_to_pathtarget(partial_target, expr, sgref); } else { /* * Non-grouping column, so just remember the expression for later * call to pull_var_clause. */ non_group_cols = lappend(non_group_cols, expr); } i++; } /* * If there's a HAVING clause, we'll need the Vars/Aggrefs it uses, too. */ if (parse->havingQual) non_group_cols = lappend(non_group_cols, parse->havingQual); /* * Pull out all the Vars, PlaceHolderVars, and Aggrefs mentioned in * non-group cols (plus HAVING), and add them to the partial_target if not * already present. (An expression used directly as a GROUP BY item will * be present already.) Note this includes Vars used in resjunk items, so * we are covering the needs of ORDER BY and window specifications. */ non_group_exprs = pull_var_clause((struct Node *) non_group_cols, PVC_INCLUDE_AGGREGATES | PVC_RECURSE_WINDOWFUNCS | PVC_INCLUDE_PLACEHOLDERS); add_new_columns_to_pathtarget(partial_target, non_group_exprs); /* * Adjust Aggrefs to put them in partial mode. At this point all Aggrefs * are at the top level of the target list, so we can just scan the list * rather than recursing through the expression trees. */ foreach (lc, partial_target->exprs) { struct Aggref *aggref = (struct Aggref *) lfirst(lc); if (IsA(aggref, Aggref)) { struct Aggref *newaggref; /* * We shouldn't need to copy the substructure of the Aggref node, * but flat-copy the node itself to avoid damaging other trees. */ newaggref = makeNode(Aggref); memcpy(newaggref, aggref, sizeof(struct Aggref)); /* For now, assume serialization is required */ mark_partial_aggref(newaggref, AGGSPLIT_INITIAL_SERIAL); lfirst(lc) = newaggref; } } /* clean up cruft */ list_free(non_group_exprs); list_free(non_group_cols); /* XXX this causes some redundant cost calculation ... */ return set_pathtarget_cost_width(root, partial_target); } /* copied verbatim from selfuncs.c */ bool ts_get_variable_range(PlannerInfo *root, VariableStatData *vardata, Oid sortop, Datum *min, Datum *max) { Datum tmin = 0; Datum tmax = 0; bool have_data = false; int16 typLen; bool typByVal; Oid opfuncoid; AttStatsSlot sslot; int i; /* * XXX It's very tempting to try to use the actual column min and max, if * we can get them relatively-cheaply with an index probe. However, since * this function is called many times during join planning, that could * have unpleasant effects on planning speed. Need more investigation * before enabling this. */ #ifdef NOT_USED if (get_actual_variable_range(root, vardata, sortop, min, max)) return true; #endif if (!HeapTupleIsValid(vardata->statsTuple)) { /* no stats available, so default result */ return false; } /* * If we can't apply the sortop to the stats data, just fail. In * principle, if there's a histogram and no MCVs, we could return the * histogram endpoints without ever applying the sortop ... but it's * probably not worth trying, because whatever the caller wants to do with * the endpoints would likely fail the security check too. */ opfuncoid = get_opcode(sortop); if (!statistic_proc_security_check(vardata, opfuncoid)) return false; get_typlenbyval(vardata->atttype, &typLen, &typByVal); /* * If there is a histogram, grab the first and last values. * * If there is a histogram that is sorted with some other operator than * the one we want, fail --- this suggests that there is data we can't * use. */ if (get_attstatsslot(&sslot, vardata->statsTuple, STATISTIC_KIND_HISTOGRAM, sortop, ATTSTATSSLOT_VALUES)) { if (sslot.nvalues > 0) { tmin = datumCopy(sslot.values[0], typByVal, typLen); tmax = datumCopy(sslot.values[sslot.nvalues - 1], typByVal, typLen); have_data = true; } free_attstatsslot(&sslot); } else if (get_attstatsslot(&sslot, vardata->statsTuple, STATISTIC_KIND_HISTOGRAM, InvalidOid, 0)) { free_attstatsslot(&sslot); return false; } /* * If we have most-common-values info, look for extreme MCVs. This is * needed even if we also have a histogram, since the histogram excludes * the MCVs. However, usually the MCVs will not be the extreme values, so * avoid unnecessary data copying. */ if (get_attstatsslot(&sslot, vardata->statsTuple, STATISTIC_KIND_MCV, InvalidOid, ATTSTATSSLOT_VALUES)) { bool tmin_is_mcv = false; bool tmax_is_mcv = false; FmgrInfo opproc; fmgr_info(opfuncoid, &opproc); for (i = 0; i < sslot.nvalues; i++) { if (!have_data) { tmin = tmax = sslot.values[i]; tmin_is_mcv = tmax_is_mcv = have_data = true; continue; } if (DatumGetBool( FunctionCall2Coll(&opproc, DEFAULT_COLLATION_OID, sslot.values[i], tmin))) { tmin = sslot.values[i]; tmin_is_mcv = true; } if (DatumGetBool( FunctionCall2Coll(&opproc, DEFAULT_COLLATION_OID, tmax, sslot.values[i]))) { tmax = sslot.values[i]; tmax_is_mcv = true; } } if (tmin_is_mcv) tmin = datumCopy(tmin, typByVal, typLen); if (tmax_is_mcv) tmax = datumCopy(tmax, typByVal, typLen); free_attstatsslot(&sslot); } *min = tmin; *max = tmax; return have_data; } /* * ts_make_sort --- basic routine to build a Sort plan node * * Caller must have built the sortColIdx, sortOperators, collations, and * nullsFirst arrays already. */ Sort * ts_make_sort(Plan *lefttree, int numCols, AttrNumber *sortColIdx, Oid *sortOperators, Oid *collations, bool *nullsFirst) { Sort *node = makeNode(Sort); Plan *plan = &node->plan; plan->targetlist = lefttree->targetlist; plan->qual = NIL; plan->lefttree = lefttree; plan->righttree = NULL; node->numCols = numCols; node->sortColIdx = sortColIdx; node->sortOperators = sortOperators; node->collations = collations; node->nullsFirst = nullsFirst; return node; } /* * make_sort_from_pathkeys * Create sort plan to sort according to given pathkeys * * 'lefttree' is the node which yields input tuples * 'pathkeys' is the list of pathkeys by which the result is to be sorted * 'relids' is the set of relations required by prepare_sort_from_pathkeys() */ Sort * ts_make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids) { int numsortkeys; AttrNumber *sortColIdx; Oid *sortOperators; Oid *collations; bool *nullsFirst; /* Compute sort column info, and adjust lefttree as needed */ lefttree = ts_prepare_sort_from_pathkeys(lefttree, pathkeys, relids, NULL, false, &numsortkeys, &sortColIdx, &sortOperators, &collations, &nullsFirst); /* Now build the Sort node */ return ts_make_sort(lefttree, numsortkeys, sortColIdx, sortOperators, collations, nullsFirst); } /* * ts_prepare_sort_from_pathkeys * Prepare to sort according to given pathkeys * * This is used to set up for Sort, MergeAppend, and Gather Merge nodes. It * calculates the executor's representation of the sort key information, and * adjusts the plan targetlist if needed to add resjunk sort columns. * * Input parameters: * 'lefttree' is the plan node which yields input tuples * 'pathkeys' is the list of pathkeys by which the result is to be sorted * 'relids' identifies the child relation being sorted, if any * 'reqColIdx' is NULL or an array of required sort key column numbers * 'adjust_tlist_in_place' is true if lefttree must be modified in-place * * We must convert the pathkey information into arrays of sort key column * numbers, sort operator OIDs, collation OIDs, and nulls-first flags, * which is the representation the executor wants. These are returned into * the output parameters *p_numsortkeys etc. * * When looking for matches to an EquivalenceClass's members, we will only * consider child EC members if they belong to given 'relids'. This protects * against possible incorrect matches to child expressions that contain no * Vars. * * If reqColIdx isn't NULL then it contains sort key column numbers that * we should match. This is used when making child plans for a MergeAppend; * it's an error if we can't match the columns. * * If the pathkeys include expressions that aren't simple Vars, we will * usually need to add resjunk items to the input plan's targetlist to * compute these expressions, since a Sort or MergeAppend node itself won't * do any such calculations. If the input plan type isn't one that can do * projections, this means adding a Result node just to do the projection. * However, the caller can pass adjust_tlist_in_place = true to force the * lefttree tlist to be modified in-place regardless of whether the node type * can project --- we use this for fixing the tlist of MergeAppend itself. * * Returns the node which is to be the input to the Sort (either lefttree, * or a Result stacked atop lefttree). * * static function copied from createplan.c */ Plan * ts_prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids, const AttrNumber *reqColIdx, bool adjust_tlist_in_place, int *p_numsortkeys, AttrNumber **p_sortColIdx, Oid **p_sortOperators, Oid **p_collations, bool **p_nullsFirst) { List *tlist = lefttree->targetlist; ListCell *i; int numsortkeys; AttrNumber *sortColIdx; Oid *sortOperators; Oid *collations; bool *nullsFirst; /* * We will need at most list_length(pathkeys) sort columns; possibly less */ numsortkeys = list_length(pathkeys); sortColIdx = (AttrNumber *) palloc(numsortkeys * sizeof(AttrNumber)); sortOperators = (Oid *) palloc(numsortkeys * sizeof(Oid)); collations = (Oid *) palloc(numsortkeys * sizeof(Oid)); nullsFirst = (bool *) palloc(numsortkeys * sizeof(bool)); numsortkeys = 0; foreach (i, pathkeys) { PathKey *pathkey = (PathKey *) lfirst(i); EquivalenceClass *ec = pathkey->pk_eclass; EquivalenceMember *em; TargetEntry *tle = NULL; Oid pk_datatype = InvalidOid; Oid sortop; ListCell *j; if (ec->ec_has_volatile) { /* * If the pathkey's EquivalenceClass is volatile, then it must * have come from an ORDER BY clause, and we have to match it to * that same targetlist entry. */ if (ec->ec_sortref == 0) /* can't happen */ elog(ERROR, "volatile EquivalenceClass has no sortref"); tle = get_sortgroupref_tle(ec->ec_sortref, tlist); Assert(tle); Assert(list_length(ec->ec_members) == 1); pk_datatype = ((EquivalenceMember *) linitial(ec->ec_members))->em_datatype; } else if (reqColIdx != NULL) { /* * If we are given a sort column number to match, only consider * the single TLE at that position. It's possible that there is * no such TLE, in which case fall through and generate a resjunk * targetentry (we assume this must have happened in the parent * plan as well). If there is a TLE but it doesn't match the * pathkey's EC, we do the same, which is probably the wrong thing * but we'll leave it to caller to complain about the mismatch. */ tle = get_tle_by_resno(tlist, reqColIdx[numsortkeys]); if (tle) { em = find_ec_member_matching_expr(ec, tle->expr, relids); if (em) { /* found expr at right place in tlist */ pk_datatype = em->em_datatype; } else tle = NULL; } } else { /* * Otherwise, we can sort by any non-constant expression listed in * the pathkey's EquivalenceClass. For now, we take the first * tlist item found in the EC. If there's no match, we'll generate * a resjunk entry using the first EC member that is an expression * in the input's vars. (The non-const restriction only matters * if the EC is below_outer_join; but if it isn't, it won't * contain consts anyway, else we'd have discarded the pathkey as * redundant.) * * XXX if we have a choice, is there any way of figuring out which * might be cheapest to execute? (For example, int4lt is likely * much cheaper to execute than numericlt, but both might appear * in the same equivalence class...) Not clear that we ever will * have an interesting choice in practice, so it may not matter. */ foreach (j, tlist) { tle = (TargetEntry *) lfirst(j); em = find_ec_member_matching_expr(ec, tle->expr, relids); if (em) { /* found expr already in tlist */ pk_datatype = em->em_datatype; break; } tle = NULL; } } if (!tle) { /* * No matching tlist item; look for a computable expression. */ em = find_computable_ec_member(NULL, ec, tlist, relids, false); if (!em) elog(ERROR, "could not find pathkey item to sort"); pk_datatype = em->em_datatype; /* * Do we need to insert a Result node? */ if (!adjust_tlist_in_place && !is_projection_capable_plan(lefttree)) { /* copy needed so we don't modify input's tlist below */ tlist = copyObject(tlist); lefttree = inject_projection_plan(lefttree, tlist, lefttree->parallel_safe); } /* Don't bother testing is_projection_capable_plan again */ adjust_tlist_in_place = true; /* * Add resjunk entry to input's tlist */ tle = makeTargetEntry(copyObject(em->em_expr), list_length(tlist) + 1, NULL, true); tlist = lappend(tlist, tle); lefttree->targetlist = tlist; /* just in case NIL before */ } /* * Look up the correct sort operator from the PathKey's slightly * abstracted representation. */ sortop = get_opfamily_member(pathkey->pk_opfamily, pk_datatype, pk_datatype, pathkey->pk_cmptype); if (!OidIsValid(sortop)) /* should not happen */ elog(ERROR, "missing operator %d(%u,%u) in opfamily %u", pathkey->pk_cmptype, pk_datatype, pk_datatype, pathkey->pk_opfamily); /* Add the column to the sort arrays */ sortColIdx[numsortkeys] = tle->resno; sortOperators[numsortkeys] = sortop; collations[numsortkeys] = ec->ec_collation; nullsFirst[numsortkeys] = pathkey->pk_nulls_first; numsortkeys++; } /* Return results */ *p_numsortkeys = numsortkeys; *p_sortColIdx = sortColIdx; *p_sortOperators = sortOperators; *p_collations = collations; *p_nullsFirst = nullsFirst; return lefttree; } /* * copied verbatim from createplan.c */ List * ts_build_path_tlist(PlannerInfo *root, Path *path) { List *tlist = NIL; Index *sortgrouprefs = path->pathtarget->sortgrouprefs; int resno = 1; ListCell *v; foreach (v, path->pathtarget->exprs) { Node *node = (Node *) lfirst(v); TargetEntry *tle; /* * If it's a parameterized path, there might be lateral references in * the tlist, which need to be replaced with Params. There's no need * to remake the TargetEntry nodes, so apply this to each list item * separately. */ if (path->param_info) node = ts_replace_nestloop_params(root, node); tle = makeTargetEntry((Expr *) node, resno, NULL, false); if (sortgrouprefs) tle->ressortgroupref = sortgrouprefs[resno - 1]; tlist = lappend(tlist, tle); resno++; } return tlist; } /* * ts_replace_nestloop_params * Replace outer-relation Vars and PlaceHolderVars in the given expression * with nestloop Params * * All Vars and PlaceHolderVars belonging to the relation(s) identified by * root->curOuterRels are replaced by Params, and entries are added to * root->curOuterParams if not already present. */ Node * ts_replace_nestloop_params(PlannerInfo *root, Node *expr) { /* No setup needed for tree walk, so away we go */ return replace_nestloop_params_mutator(expr, root); } static Node * replace_nestloop_params_mutator(Node *node, PlannerInfo *root) { if (node == NULL) return NULL; if (IsA(node, Var)) { Var *var = (Var *) node; /* Upper-level Vars should be long gone at this point */ Assert(var->varlevelsup == 0); /* If not to be replaced, we can just return the Var unmodified */ if (!bms_is_member(var->varno, root->curOuterRels)) return node; /* Replace the Var with a nestloop Param */ return (Node *) replace_nestloop_param_var(root, var); } if (IsA(node, PlaceHolderVar)) { PlaceHolderVar *phv = (PlaceHolderVar *) node; /* Upper-level PlaceHolderVars should be long gone at this point */ Assert(phv->phlevelsup == 0); #if PG16_LT /* * Check whether we need to replace the PHV. We use bms_overlap as a * cheap/quick test to see if the PHV might be evaluated in the outer * rels, and then grab its PlaceHolderInfo to tell for sure. */ if (!bms_overlap(phv->phrels, root->curOuterRels) || !bms_is_subset(find_placeholder_info(root, phv, false)->ph_eval_at, root->curOuterRels)) #else /* Check whether we need to replace the PHV */ if (!bms_is_subset(find_placeholder_info(root, phv)->ph_eval_at, root->curOuterRels)) #endif { /* * We can't replace the whole PHV, but we might still need to * replace Vars or PHVs within its expression, in case it ends up * actually getting evaluated here. (It might get evaluated in * this plan node, or some child node; in the latter case we don't * really need to process the expression here, but we haven't got * enough info to tell if that's the case.) Flat-copy the PHV * node and then recurse on its expression. * * Note that after doing this, we might have different * representations of the contents of the same PHV in different * parts of the plan tree. This is OK because equal() will just * match on phid/phlevelsup, so setrefs.c will still recognize an * upper-level reference to a lower-level copy of the same PHV. */ PlaceHolderVar *newphv = makeNode(PlaceHolderVar); memcpy(newphv, phv, sizeof(PlaceHolderVar)); newphv->phexpr = (Expr *) replace_nestloop_params_mutator((Node *) phv->phexpr, root); return (Node *) newphv; } /* Replace the PlaceHolderVar with a nestloop Param */ return (Node *) replace_nestloop_param_placeholdervar(root, phv); } return expression_tree_mutator(node, replace_nestloop_params_mutator, (void *) root); } /* * make_result * Build a Result plan node */ static Result * make_result(List *tlist, Node *resconstantqual, Plan *subplan) { Result *node = makeNode(Result); Plan *plan = &node->plan; plan->targetlist = tlist; plan->qual = NIL; plan->lefttree = subplan; plan->righttree = NULL; node->resconstantqual = resconstantqual; return node; } /* * Copy cost and size info from a lower plan node to an inserted node. * (Most callers alter the info after copying it.) */ static void copy_plan_costsize(Plan *dest, Plan *src) { dest->startup_cost = src->startup_cost; dest->total_cost = src->total_cost; dest->plan_rows = src->plan_rows; dest->plan_width = src->plan_width; /* Assume the inserted node is not parallel-aware. */ dest->parallel_aware = false; /* Assume the inserted node is parallel-safe, if child plan is. */ dest->parallel_safe = src->parallel_safe; } /* * inject_projection_plan * Insert a Result node to do a projection step. * * This is used in a few places where we decide on-the-fly that we need a * projection step as part of the tree generated for some Path node. * We should try to get rid of this in favor of doing it more honestly. * * One reason it's ugly is we have to be told the right parallel_safe marking * to apply (since the tlist might be unsafe even if the child plan is safe). */ static Plan * inject_projection_plan(Plan *subplan, List *tlist, bool parallel_safe) { Plan *plan; plan = (Plan *) make_result(tlist, NULL, subplan); /* * In principle, we should charge tlist eval cost plus cpu_per_tuple per * row for the Result node. But the former has probably been factored in * already and the latter was not accounted for during Path construction, * so being formally correct might just make the EXPLAIN output look less * consistent not more so. Hence, just copy the subplan's cost. */ copy_plan_costsize(plan, subplan); plan->parallel_safe = parallel_safe; return plan; } /* In PG18, child ems are not added to ec_members * but need to be maintained in separate Lists. * * https://github.com/postgres/postgres/commit/d69d45a5 */ /* copied from add_child_eq_member */ #if PG18_GE void ts_add_child_eq_member(PlannerInfo *root, EquivalenceClass *ec, EquivalenceMember *em, int child_relid) { /* * Allocate the array to store child members; an array of Lists indexed by * relid, or expand the existing one, if necessary. */ if (unlikely(ec->ec_childmembers_size < root->simple_rel_array_size)) { ec->ec_relids = bms_add_members(ec->ec_relids, em->em_relids); if (ec->ec_childmembers == NULL) ec->ec_childmembers = palloc0_array(List *, root->simple_rel_array_size); else ec->ec_childmembers = repalloc0_array(ec->ec_childmembers, List *, ec->ec_childmembers_size, root->simple_rel_array_size); ec->ec_childmembers_size = root->simple_rel_array_size; } /* add member to the ec_childmembers List for the given child_relid */ ec->ec_childmembers[child_relid] = lappend(ec->ec_childmembers[child_relid], em); } #endif ================================================ FILE: src/import/planner.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once /* * This file contains source code that was copied and/or modified from * the PostgreSQL database, which is licensed under the open-source * PostgreSQL License. Please see the NOTICE at the top level * directory for a copy of the PostgreSQL License. * * These function were copied from the PostgreSQL core planner, since * they were declared static in the core planner, but we need them for * our manipulations. */ #include <postgres.h> #include <nodes/execnodes.h> #include <utils/rel.h> #include <utils/selfuncs.h> #include "export.h" extern TSDLLEXPORT void ts_make_inh_translation_list(Relation oldrelation, Relation newrelation, Index newvarno, AppendRelInfo *appinfo); extern TSDLLEXPORT struct PathTarget *ts_make_partial_grouping_target(struct PlannerInfo *root, PathTarget *grouping_target); extern bool ts_get_variable_range(PlannerInfo *root, VariableStatData *vardata, Oid sortop, Datum *min, Datum *max); extern TSDLLEXPORT Plan * ts_prepare_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids, const AttrNumber *reqColIdx, bool adjust_tlist_in_place, int *p_numsortkeys, AttrNumber **p_sortColIdx, Oid **p_sortOperators, Oid **p_collations, bool **p_nullsFirst); extern TSDLLEXPORT Sort *ts_make_sort_from_pathkeys(Plan *lefttree, List *pathkeys, Relids relids); extern TSDLLEXPORT Sort *ts_make_sort(Plan *lefttree, int numCols, AttrNumber *sortColIdx, Oid *sortOperators, Oid *collations, bool *nullsFirst); extern TSDLLEXPORT PathKey *ts_make_pathkey_from_sortop(PlannerInfo *root, Expr *expr, Relids nullable_relids, Oid ordering_op, bool nulls_first, Index sortref, bool create_it); extern TSDLLEXPORT List *ts_build_path_tlist(PlannerInfo *root, Path *path); extern TSDLLEXPORT Node *ts_replace_nestloop_params(PlannerInfo *root, Node *expr); extern void ts_ExecSetTupleBound(int64 tuples_needed, PlanState *child_node); #if PG18_GE /* In PG18, child ems are not added to ec_members * but need to be maintained in separate Lists. * * https://github.com/postgres/postgres/commit/d69d45a5 */ /* copied from add_child_eq_member */ extern TSDLLEXPORT void ts_add_child_eq_member(PlannerInfo *root, EquivalenceClass *ec, EquivalenceMember *em, int child_relid); #endif ================================================ FILE: src/import/ts_explain.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ /* * This file contains source code that was copied and/or modified from * the PostgreSQL database, which is licensed under the open-source * PostgreSQL License. Please see the NOTICE at the top level * directory for a copy of the PostgreSQL License. */ #include "ts_explain.h" #include <commands/explain.h> #include <nodes/makefuncs.h> #include <utils/ruleutils.h> #include "compat/compat.h" /* * Show a generic expression */ static void ts_show_expression(Node *node, const char *qlabel, PlanState *planstate, List *ancestors, bool useprefix, ExplainState *es) { List *context; char *exprstr; /* Set up deparsing context */ context = set_deparse_context_plan(es->deparse_cxt, planstate->plan, ancestors); /* Deparse the expression */ exprstr = deparse_expression(node, context, useprefix, false); /* And add to es->str */ ExplainPropertyText(qlabel, exprstr, es); } /* * Show a qualifier expression (which is a List with implicit AND semantics) */ static void ts_show_qual(List *qual, const char *qlabel, PlanState *planstate, List *ancestors, bool useprefix, ExplainState *es) { Node *node; /* No work if empty qual */ if (qual == NIL) return; /* Convert AND list to explicit AND */ node = (Node *) make_ands_explicit(qual); /* And show it */ ts_show_expression(node, qlabel, planstate, ancestors, useprefix, es); } /* * Show a qualifier expression for a scan plan node */ void ts_show_scan_qual(List *qual, const char *qlabel, PlanState *planstate, List *ancestors, ExplainState *es) { bool useprefix; useprefix = (IsA(planstate->plan, SubqueryScan) || es->verbose); ts_show_qual(qual, qlabel, planstate, ancestors, useprefix, es); } /* * If it's EXPLAIN ANALYZE, show instrumentation information for a plan node * * "which" identifies which instrumentation counter to print */ void ts_show_instrumentation_count(const char *qlabel, int which, PlanState *planstate, ExplainState *es) { double nfiltered; double nloops; if (!es->analyze || !planstate->instrument) return; if (which == 2) nfiltered = planstate->instrument->nfiltered2; else nfiltered = planstate->instrument->nfiltered1; nloops = planstate->instrument->nloops; /* In text mode, suppress zero counts; they're not interesting enough */ if (nfiltered > 0 || es->format != EXPLAIN_FORMAT_TEXT) { if (nloops > 0) ExplainPropertyFloat(qlabel, NULL, nfiltered / nloops, 0, es); else ExplainPropertyFloat(qlabel, NULL, 0.0, 0, es); } } ================================================ FILE: src/import/ts_explain.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once /* * This file contains source code that was copied and/or modified from * the PostgreSQL database, which is licensed under the open-source * PostgreSQL License. Please see the NOTICE at the top level * directory for a copy of the PostgreSQL License. */ #include <postgres.h> #include <commands/explain.h> #include <nodes/execnodes.h> #include <nodes/pg_list.h> #include <compat/compat.h> #include "export.h" #if PG18_GE #include <commands/explain_format.h> #include <commands/explain_state.h> #endif extern TSDLLEXPORT void ts_show_scan_qual(List *qual, const char *qlabel, PlanState *planstate, List *ancestors, ExplainState *es); extern TSDLLEXPORT void ts_show_instrumentation_count(const char *qlabel, int which, PlanState *planstate, ExplainState *es); ================================================ FILE: src/indexing.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <catalog/index.h> #include <catalog/indexing.h> #include <catalog/namespace.h> #include <catalog/pg_inherits.h> #include <commands/defrem.h> #include <commands/event_trigger.h> #include <commands/tablecmds.h> #include <commands/tablespace.h> #include <fmgr.h> #include <nodes/makefuncs.h> #include <nodes/parsenodes.h> #include <nodes/value.h> #include <parser/parse_utilcmd.h> #include <utils/builtins.h> #include <utils/lsyscache.h> #include <utils/syscache.h> #include "annotations.h" #include "dimension.h" #include "errors.h" #include "guc.h" #include "hypertable_cache.h" #include "indexing.h" #include "partitioning.h" static bool index_has_attribute(const List *indexelems, const char *attrname) { ListCell *lc; foreach (lc, indexelems) { Node *node = lfirst(lc); const char *colname = NULL; /* * The type of the element varies depending on whether the list is * from an index or a constraint */ switch (nodeTag(node)) { case T_IndexElem: colname = ((IndexElem *) node)->name; break; case T_String: colname = strVal(node); break; case T_List: { List *pair = lfirst_node(List, lc); if (list_length(pair) == 2 && IsA(linitial(pair), IndexElem) && IsA(lsecond(pair), List)) { colname = ((IndexElem *) linitial(pair))->name; break; } } TS_FALLTHROUGH; default: elog(ERROR, "unsupported index list element"); } if (colname != NULL && strncmp(colname, attrname, NAMEDATALEN) == 0) return true; } return false; } /* * Verify that index columns cover all partitioning dimensions. * * A UNIQUE, PRIMARY KEY or EXCLUSION index on a chunk must cover all * partitioning dimensions to guarantee uniqueness (or exclusion) across the * entire hypertable. Therefore we check that all dimensions are present among * the index columns. */ void ts_indexing_verify_columns(const Hyperspace *hs, const List *indexelems) { int i; for (i = 0; i < hs->num_dimensions; i++) { const Dimension *dim = &hs->dimensions[i]; if (!index_has_attribute(indexelems, NameStr(dim->fd.column_name))) ereport(ERROR, (errcode(ERRCODE_TS_BAD_HYPERTABLE_INDEX_DEFINITION), errmsg("cannot create a unique index without the column \"%s\" (used in " "partitioning)", NameStr(dim->fd.column_name)), errhint( "If you're creating a hypertable on a table with a primary key, ensure " "the partitioning column is part of the primary or composite key."))); } } /* * Verify index columns. * * We only care about UNIQUE, PRIMARY KEY or EXCLUSION indexes. */ void ts_indexing_verify_index(const Hyperspace *hs, const IndexStmt *stmt) { if (stmt->unique || stmt->excludeOpNames != NULL) ts_indexing_verify_columns(hs, stmt->indexParams); } /* * Build a list of string Values representing column names that an index covers. */ static List * build_indexcolumn_list(const Relation idxrel) { List *columns = NIL; int i; for (i = 0; i < idxrel->rd_att->natts; i++) { Form_pg_attribute idxattr = TupleDescAttr(idxrel->rd_att, i); columns = lappend(columns, makeString(NameStr(idxattr->attname))); } return columns; } static void create_default_index(const Hypertable *ht, List *indexelems) { IndexStmt stmt = { .type = T_IndexStmt, .accessMethod = DEFAULT_INDEX_TYPE, .idxname = NULL, .relation = makeRangeVar((char *) NameStr(ht->fd.schema_name), (char *) NameStr(ht->fd.table_name), 0), .tableSpace = get_tablespace_name(get_rel_tablespace(ht->main_table_relid)), .indexParams = indexelems, }; DefineIndexCompat(ht->main_table_relid, &stmt, InvalidOid, /* indexRelationId */ InvalidOid, /* parentIndexId */ InvalidOid, /* parentConstraintId */ -1, /* total_parts */ false, /* is_alter_table */ false, /* check_rights */ false, /* check_not_in_use */ false, /* skip_build */ true); /* quiet */ } static const Node * get_open_dim_expr(const Dimension *dim) { if (dim == NULL || dim->partitioning == NULL) return NULL; return dim->partitioning->partfunc.func_fmgr.fn_expr; } static const char * get_open_dim_name(const Dimension *dim) { if (dim == NULL || dim->partitioning != NULL) return NULL; return NameStr(dim->fd.column_name); } static void create_default_indexes(const Hypertable *ht, const Dimension *time_dim, const Dimension *space_dim, bool has_time_idx, bool has_time_space_idx) { const char *dimname = get_open_dim_name(time_dim); IndexElem telem = { .type = T_IndexElem, .name = dimname ? (char *) dimname : NULL, .ordering = SORTBY_DESC, .expr = (Node *) get_open_dim_expr(time_dim), }; /* In case we'd allow tables that are only space partitioned */ if (NULL == time_dim) return; /* Create ("time") index */ if (!has_time_idx) create_default_index(ht, list_make1(&telem)); /* Create ("space", "time") index */ if (space_dim != NULL && !has_time_space_idx) { IndexElem selem = { .type = T_IndexElem, .name = pstrdup(NameStr(space_dim->fd.column_name)), .ordering = SORTBY_ASC, }; create_default_index(ht, list_make2(&selem, &telem)); } } /* * Verify that unique, primary and exclusion indexes on a hypertable cover all * partitioning columns and create any default indexes. * * Default indexes are assumed to cover the first open ("time") dimension, and, * optionally, the first closed ("space") dimension. */ static void indexing_create_and_verify_hypertable_indexes(const Hypertable *ht, bool create_default, bool verify) { Relation tblrel = table_open(ht->main_table_relid, AccessShareLock); const Dimension *time_dim = ts_hyperspace_get_dimension(ht->space, DIMENSION_TYPE_OPEN, 0); const Dimension *space_dim = ts_hyperspace_get_dimension(ht->space, DIMENSION_TYPE_CLOSED, 0); List *indexlist = RelationGetIndexList(tblrel); bool has_time_idx = false; bool has_time_space_idx = false; ListCell *lc; foreach (lc, indexlist) { Relation idxrel = index_open(lfirst_oid(lc), AccessShareLock); if (verify && (idxrel->rd_index->indisunique || idxrel->rd_index->indisexclusion)) ts_indexing_verify_columns(ht->space, build_indexcolumn_list(idxrel)); /* Check for existence of "default" indexes */ if (create_default && NULL != time_dim) { Form_pg_attribute idxattr_time, idxattr_space; switch (idxrel->rd_att->natts) { case 1: /* ("time") index */ idxattr_time = TupleDescAttr(idxrel->rd_att, 0); if (namestrcmp(&idxattr_time->attname, NameStr(time_dim->fd.column_name)) == 0) has_time_idx = true; break; case 2: /* ("space", "time") index */ idxattr_space = TupleDescAttr(idxrel->rd_att, 0); idxattr_time = TupleDescAttr(idxrel->rd_att, 1); if (space_dim != NULL && namestrcmp(&idxattr_space->attname, NameStr(space_dim->fd.column_name)) == 0 && namestrcmp(&idxattr_time->attname, NameStr(time_dim->fd.column_name)) == 0) has_time_space_idx = true; break; default: break; } } index_close(idxrel, AccessShareLock); } if (create_default) create_default_indexes(ht, time_dim, space_dim, has_time_idx, has_time_space_idx); table_close(tblrel, AccessShareLock); } bool TSDLLEXPORT ts_indexing_relation_has_primary_or_unique_index(Relation htrel) { List *indexoidlist = RelationGetIndexList(htrel); ListCell *lc; bool result = false; if (OidIsValid(htrel->rd_pkindex)) return true; foreach (lc, indexoidlist) { Oid indexoid = lfirst_oid(lc); HeapTuple index_tuple; Form_pg_index index; index_tuple = SearchSysCache1(INDEXRELID, ObjectIdGetDatum(indexoid)); if (!HeapTupleIsValid(index_tuple)) /* should not happen */ elog(ERROR, "cache lookup failed for index %u in \"%s\" ", indexoid, RelationGetRelationName(htrel)); index = (Form_pg_index) GETSTRUCT(index_tuple); result = index->indisunique; ReleaseSysCache(index_tuple); if (result) break; } list_free(indexoidlist); return result; } /* create the index on the root table of a hypertable. * based on postgres CREATE INDEX * https://github.com/postgres/postgres/blob/ebfe20dc706bd3238a9bdf3b44cd8f82337e86a8/src/backend/tcop/utility.c#L1291-L1374 * despite not allowing CONCURRENT index creation now, we expect to do so soon, so this code * retains those code paths */ extern ObjectAddress ts_indexing_root_table_create_index(IndexStmt *stmt, const char *queryString, bool is_multitransaction) { Oid relid; LOCKMODE lockmode; ObjectAddress root_table_address; int total_parts = -1; if (stmt->concurrent) PreventInTransactionBlock(true, "CREATE INDEX CONCURRENTLY"); /* * Look up the relation OID just once, right here at the * beginning, so that we don't end up repeating the name * lookup later and latching onto a different relation * partway through. To avoid lock upgrade hazards, it's * important that we take the strongest lock that will * eventually be needed here, so the lockmode calculation * needs to match what DefineIndex() does. */ lockmode = stmt->concurrent ? ShareUpdateExclusiveLock : ShareLock; relid = RangeVarGetRelidExtended(stmt->relation, lockmode, 0, RangeVarCallbackOwnsRelation, NULL); /* * single-transaction CREATE INDEX on a hypertable tables recurses to * chunks, so we must acquire locks early to avoid deadlocks. * * We also take the opportunity to verify that all * chunks are something we can put an index on, to * avoid building some indexes only to fail later. */ if (!is_multitransaction) { ListCell *lc; List *inheritors = NIL; inheritors = find_all_inheritors(relid, lockmode, NULL); foreach (lc, inheritors) { char relkind = get_rel_relkind(lfirst_oid(lc)); if (relkind != RELKIND_RELATION && relkind != RELKIND_MATVIEW && relkind != RELKIND_FOREIGN_TABLE && !(relkind == RELKIND_PARTITIONED_TABLE && ts_guc_enable_partitioned_hypertables)) ereport(ERROR, (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), errmsg("cannot create index on hypertable \"%s\"", stmt->relation->relname), errdetail("Table \"%s\" contains chunks of the wrong type.", stmt->relation->relname))); } total_parts = list_length(inheritors) - 1; list_free(inheritors); } /* Run parse analysis ... */ stmt = transformIndexStmt(relid, stmt, queryString); /* ... and do it */ EventTriggerAlterTableStart((Node *) stmt); (void) total_parts; root_table_address = DefineIndexCompat(relid, /* OID of heap relation */ stmt, InvalidOid, /* no predefined OID */ InvalidOid, /* parentIndexId */ InvalidOid, /* parentConstraintId */ total_parts, /* total_parts */ false, /* is_alter_table */ true, /* check_rights */ false, /* check_not_in_use */ false, /* skip_build */ false); /* quiet */ return root_table_address; } void ts_indexing_verify_indexes(const Hypertable *ht) { indexing_create_and_verify_hypertable_indexes(ht, false, true); } void ts_indexing_create_default_indexes(const Hypertable *ht) { indexing_create_and_verify_hypertable_indexes(ht, true, false); } TSDLLEXPORT Oid ts_indexing_find_clustered_index(Oid table_relid) { Relation rel; ListCell *index; Oid index_relid = InvalidOid; rel = table_open(table_relid, AccessShareLock); /* We need to find the index that has indisclustered set. */ foreach (index, RelationGetIndexList(rel)) { HeapTuple idxtuple; Form_pg_index indexForm; index_relid = lfirst_oid(index); idxtuple = SearchSysCache1(INDEXRELID, ObjectIdGetDatum(index_relid)); if (!HeapTupleIsValid(idxtuple)) elog(ERROR, "cache lookup failed for index %u when looking for a clustered index", index_relid); indexForm = (Form_pg_index) GETSTRUCT(idxtuple); if (indexForm->indisclustered) { ReleaseSysCache(idxtuple); break; } ReleaseSysCache(idxtuple); index_relid = InvalidOid; } table_close(rel, AccessShareLock); return index_relid; } typedef enum IndexValidity { IndexInvalid = 0, IndexValid, } IndexValidity; static bool ts_indexing_mark_as(Oid index_id, IndexValidity validity) { Relation pg_index; HeapTuple indexTuple; HeapTuple new_tuple; Form_pg_index indexForm; bool was_valid; /* Open pg_index and fetch a writable copy of the index's tuple */ pg_index = table_open(IndexRelationId, RowExclusiveLock); indexTuple = SearchSysCacheCopy1(INDEXRELID, ObjectIdGetDatum(index_id)); if (!HeapTupleIsValid(indexTuple)) elog(ERROR, "cache lookup failed when marking index %u", index_id); new_tuple = heap_copytuple(indexTuple); indexForm = (Form_pg_index) GETSTRUCT(new_tuple); was_valid = indexForm->indisvalid; /* Perform the requested state change on the copy */ switch (validity) { case IndexValid: Assert(indexForm->indislive); Assert(indexForm->indisready); indexForm->indisvalid = true; break; case IndexInvalid: indexForm->indisvalid = false; indexForm->indisclustered = false; break; } /* ... and write it back */ CatalogTupleUpdate(pg_index, &indexTuple->t_self, new_tuple); table_close(pg_index, RowExclusiveLock); return was_valid; } void ts_indexing_mark_as_valid(Oid index_id) { ts_indexing_mark_as(index_id, IndexValid); } /* returns if the index was valid */ bool ts_indexing_mark_as_invalid(Oid index_id) { return ts_indexing_mark_as(index_id, IndexInvalid); } TS_FUNCTION_INFO_V1(ts_index_matches); Datum ts_index_matches(PG_FUNCTION_ARGS) { Oid index1 = PG_GETARG_OID(0); Oid index2 = PG_GETARG_OID(1); bool result; if (!OidIsValid(index1) || !OidIsValid(index2)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid index"))); result = ts_indexing_compare(index1, index2); PG_RETURN_BOOL(result); } /* Returns true if the indexes are equivalent */ bool ts_indexing_compare(Oid index1, Oid index2) { Relation indexrel1 = index_open(index1, AccessShareLock); Relation indexrel2 = index_open(index2, AccessShareLock); Relation rel1 = table_open(indexrel1->rd_index->indrelid, AccessShareLock); Relation rel2 = table_open(indexrel2->rd_index->indrelid, AccessShareLock); if (indexrel1->rd_rel->relkind != RELKIND_INDEX || indexrel2->rd_rel->relkind != RELKIND_INDEX) { ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("expected both \"%s\" and \"%s\" to be indexes", RelationGetRelationName(indexrel1), RelationGetRelationName(indexrel2)))); } IndexInfo *info1 = BuildIndexInfo(indexrel1); IndexInfo *info2 = BuildIndexInfo(indexrel2); #if PG16_GE AttrMap *attmap = build_attrmap_by_name(RelationGetDescr(rel1), RelationGetDescr(rel2), false); #else AttrMap *attmap = build_attrmap_by_name(RelationGetDescr(rel1), RelationGetDescr(rel2)); #endif bool result = CompareIndexInfo(info1, info2, indexrel1->rd_indcollation, indexrel2->rd_indcollation, indexrel1->rd_opfamily, indexrel2->rd_opfamily, attmap); if (result) { /* * CompareIndexInfo does not compare indoption, which means it will * consider two indexes with different ASC/DESC or NULLS FIRST/LAST * options as equivalent. */ for (int i = 0; i < IndexRelationGetNumberOfKeyAttributes(indexrel1); i++) { if (indexrel1->rd_indoption[i] != indexrel2->rd_indoption[i]) { result = false; break; } } } index_close(indexrel1, NoLock); index_close(indexrel2, NoLock); table_close(rel1, NoLock); table_close(rel2, NoLock); return result; } ================================================ FILE: src/indexing.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/parsenodes.h> #include <nodes/pg_list.h> #include "dimension.h" #include "export.h" extern void ts_indexing_verify_columns(const Hyperspace *hs, const List *indexelems); extern void ts_indexing_verify_index(const Hyperspace *hs, const IndexStmt *stmt); extern void ts_indexing_verify_indexes(const Hypertable *ht); extern void ts_indexing_create_default_indexes(const Hypertable *ht); extern ObjectAddress ts_indexing_root_table_create_index(IndexStmt *stmt, const char *queryString, bool is_multitransaction); extern TSDLLEXPORT Oid ts_indexing_find_clustered_index(Oid table_relid); extern void ts_indexing_mark_as_valid(Oid index_id); extern bool ts_indexing_mark_as_invalid(Oid index_id); extern bool TSDLLEXPORT ts_indexing_relation_has_primary_or_unique_index(Relation htrel); extern TSDLLEXPORT bool ts_indexing_compare(Oid index1, Oid index2); extern TSDLLEXPORT Datum ts_index_matches(PG_FUNCTION_ARGS); ================================================ FILE: src/init.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <commands/extension.h> #include <miscadmin.h> #include <parser/analyze.h> #include <storage/ipc.h> #include <utils/guc.h> #include "compat/compat.h" #include "bgw/launcher_interface.h" #include "config.h" #include "extension.h" #include "guc.h" #include "license_guc.h" #include "nodes/constraint_aware_append/constraint_aware_append.h" #include "partition_chunk.h" #include "ts_catalog/catalog.h" #include "version.h" #ifdef PG_MODULE_MAGIC PG_MODULE_MAGIC; #endif extern void _hypertable_cache_init(void); extern void _hypertable_cache_fini(void); extern void _cache_invalidate_init(void); extern void _cache_invalidate_fini(void); extern void _planner_init(void); extern void _planner_fini(void); extern void _process_utility_init(void); extern void _process_utility_fini(void); extern void _event_trigger_init(void); extern void _event_trigger_fini(void); extern void _conn_plain_init(); extern void _conn_plain_fini(); extern void _executor_init(void); extern void _executor_fini(void); #ifdef TS_USE_OPENSSL extern void _conn_ssl_init(); extern void _conn_ssl_fini(); #endif #ifdef TS_DEBUG extern void _conn_mock_init(); extern void _conn_mock_fini(); #endif extern void _chunk_append_init(); #if PG16_LT extern void TSDLLEXPORT _PG_init(void); #endif TS_FUNCTION_INFO_V1(ts_post_load_init); /* Called when the backend exits */ static void cleanup_on_pg_proc_exit(int code, Datum arg) { /* * Order of items should be strict reverse order of _PG_init. Please * document any exceptions. */ #ifdef TS_DEBUG _conn_mock_fini(); #endif #ifdef TS_USE_OPENSSL _conn_ssl_fini(); #endif _conn_plain_fini(); _process_utility_fini(); _event_trigger_fini(); _planner_fini(); _cache_invalidate_fini(); _hypertable_cache_fini(); _cache_fini(); _executor_fini(); } void _PG_init(void) { static bool init_done = false; /* * Check extension_is loaded to catch certain errors such as calls to * functions defined on the wrong extension version */ ts_extension_check_version(TIMESCALEDB_VERSION_MOD); ts_extension_check_server_version(); ts_bgw_check_loader_api_version(); /* We can call _PG_init() several times if we do an eager load, so abort * init if we do. */ if (init_done) return; _cache_init(); _hypertable_cache_init(); _cache_invalidate_init(); _planner_init(); _constraint_aware_append_init(); _chunk_append_init(); _event_trigger_init(); _process_utility_init(); _guc_init(); _conn_plain_init(); _executor_init(); #ifdef TS_USE_OPENSSL _conn_ssl_init(); #endif #ifdef TS_DEBUG _conn_mock_init(); #endif /* Register a cleanup function to be called when the backend exits */ on_proc_exit(cleanup_on_pg_proc_exit, 0); init_done = true; } TSDLLEXPORT Datum ts_post_load_init(PG_FUNCTION_ARGS) { /* * Unfortunately, if we load the tsl during _PG_init parallel workers try * to load the tsl before timescale itself, causing link-time errors. To * prevent this we defer loading until here. */ ts_license_enable_module_loading(); PG_RETURN_VOID(); } ================================================ FILE: src/jsonb_utils.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <common/jsonapi.h> #include <fmgr.h> #include <utils/builtins.h> #include <utils/fmgroids.h> #include <utils/json.h> #include <utils/jsonb.h> #include "compat/compat.h" #include "export.h" #include "jsonb_utils.h" #include "utils.h" static void ts_jsonb_add_pair(JsonbParseState *state, JsonbValue *key, JsonbValue *value); void ts_jsonb_add_null(JsonbParseState *state, const char *key) { JsonbValue json_value; json_value.type = jbvNull; ts_jsonb_add_value(state, key, &json_value); } void ts_jsonb_add_bool(JsonbParseState *state, const char *key, bool boolean) { JsonbValue json_value; json_value.type = jbvBool; json_value.val.boolean = boolean; ts_jsonb_add_value(state, key, &json_value); } void ts_jsonb_add_str(JsonbParseState *state, const char *key, const char *value) { JsonbValue json_value; Assert(value != NULL); /* If there is a null entry, don't add it to the JSON */ if (value == NULL) return; json_value.type = jbvString; json_value.val.string.val = (char *) value; json_value.val.string.len = strlen(value); ts_jsonb_add_value(state, key, &json_value); } static void ts_jsonb_add_str_element(JsonbParseState *state, const char *elem) { JsonbValue json_value; Assert(elem != NULL); /* If there is a null entry, don't add it to the JSON */ if (elem == NULL) return; json_value.type = jbvString; json_value.val.string.val = (char *) elem; json_value.val.string.len = strlen(elem); pushJsonbValue(&state, WJB_ELEM, &json_value); } void ts_jsonb_add_str_array(JsonbParseState *state, const char *key, const char **values, int num_values) { JsonbValue json_key; Assert(key != NULL); Assert(values != NULL); Assert(num_values > 0); Assert(key[0] != '\0'); if (key == NULL || values == NULL || num_values <= 0 || key[0] == '\0') return; json_key.type = jbvString; json_key.val.string.val = (char *) key; json_key.val.string.len = strlen(key); pushJsonbValue(&state, WJB_KEY, &json_key); pushJsonbValue(&state, WJB_BEGIN_ARRAY, NULL); for (int i = 0; i < num_values; i++) { if (values[i] == NULL || values[i][0] == '\0') continue; ts_jsonb_add_str_element(state, values[i]); } pushJsonbValue(&state, WJB_END_ARRAY, NULL); } static PGFunction get_convert_func(Oid typeid) { switch (typeid) { case INT2OID: return int2_numeric; case INT4OID: return int4_numeric; case INT8OID: return int8_numeric; default: return NULL; } } void ts_jsonb_set_value_by_type(JsonbValue *value, Oid typeid, Datum datum) { switch (typeid) { case INT2OID: case INT4OID: case INT8OID: case NUMERICOID: { PGFunction func = get_convert_func(typeid); value->type = jbvNumeric; value->val.numeric = DatumGetNumeric(func ? DirectFunctionCall1(func, datum) : datum); break; } default: { char *str = ts_datum_to_string(datum, typeid); value->type = jbvString; value->val.string.val = str; value->val.string.len = strlen(str); break; } } } void ts_jsonb_add_int32(JsonbParseState *state, const char *key, const int32 int_value) { JsonbValue json_value; ts_jsonb_set_value_by_type(&json_value, INT4OID, Int32GetDatum(int_value)); ts_jsonb_add_value(state, key, &json_value); } void ts_jsonb_add_int64(JsonbParseState *state, const char *key, const int64 int_value) { JsonbValue json_value; ts_jsonb_set_value_by_type(&json_value, INT8OID, Int64GetDatum(int_value)); ts_jsonb_add_value(state, key, &json_value); } void ts_jsonb_add_interval(JsonbParseState *state, const char *key, Interval *interval) { JsonbValue json_value; ts_jsonb_set_value_by_type(&json_value, INTERVALOID, IntervalPGetDatum(interval)); ts_jsonb_add_value(state, key, &json_value); } void ts_jsonb_add_value(JsonbParseState *state, const char *key, JsonbValue *value) { JsonbValue json_key; Assert(key != NULL); if (value == NULL) return; json_key.type = jbvString; json_key.val.string.val = (char *) key; json_key.val.string.len = strlen(key); ts_jsonb_add_pair(state, &json_key, value); } static void ts_jsonb_add_pair(JsonbParseState *state, JsonbValue *key, JsonbValue *value) { Assert(state != NULL); Assert(key != NULL); if (value == NULL) return; pushJsonbValue(&state, WJB_KEY, key); pushJsonbValue(&state, WJB_VALUE, value); } char * ts_jsonb_get_str_field(const Jsonb *jsonb, const char *key) { /* * `jsonb_object_field_text` returns NULL when the field is not found so * we cannot use `DirectFunctionCall` */ LOCAL_FCINFO(fcinfo, 2); Datum result; InitFunctionCallInfoData(*fcinfo, NULL, 2, InvalidOid, NULL, NULL); FC_SET_ARG(fcinfo, 0, PointerGetDatum(jsonb)); FC_SET_ARG(fcinfo, 1, PointerGetDatum(cstring_to_text(key))); result = jsonb_object_field_text(fcinfo); if (fcinfo->isnull) return NULL; return text_to_cstring(DatumGetTextP(result)); } bool ts_jsonb_get_bool_field(const Jsonb *json, const char *key, bool *field_found) { Datum bool_datum; char *bool_str = ts_jsonb_get_str_field(json, key); if (bool_str == NULL) { *field_found = false; return false; } bool_datum = DirectFunctionCall1(boolin, CStringGetDatum(bool_str)); *field_found = true; return DatumGetBool(bool_datum); } int32 ts_jsonb_get_int32_field(const Jsonb *json, const char *key, bool *field_found) { Datum int_datum; char *int_str = ts_jsonb_get_str_field(json, key); if (int_str == NULL) { *field_found = false; return 0; } int_datum = DirectFunctionCall1(int4in, CStringGetDatum(int_str)); *field_found = true; return DatumGetInt32(int_datum); } int64 ts_jsonb_get_int64_field(const Jsonb *json, const char *key, bool *field_found) { Datum int_datum; char *int_str = ts_jsonb_get_str_field(json, key); if (int_str == NULL) { *field_found = false; return 0; } int_datum = DirectFunctionCall1(int8in, CStringGetDatum(int_str)); *field_found = true; return DatumGetInt64(int_datum); } Interval * ts_jsonb_get_interval_field(const Jsonb *json, const char *key) { Datum interval_datum; char *interval_str = ts_jsonb_get_str_field(json, key); if (interval_str == NULL) return NULL; interval_datum = DirectFunctionCall3(interval_in, CStringGetDatum(interval_str), InvalidOid, -1); return DatumGetIntervalP(interval_datum); } bool ts_jsonb_equal(const Jsonb *left, const Jsonb *right) { /* Quick exit if both are NULL or point to same thing. */ if (left == right) return true; if (left == NULL || right == NULL) return false; Assert(left != NULL && right != NULL); Datum result = DirectFunctionCall2(jsonb_eq, PointerGetDatum(left), PointerGetDatum(right)); return DatumGetBool(result); } /* * searches for any occurrences of a matching key value pair. compatible with nested and * array jsonbs */ bool ts_jsonb_has_key_value_str_field(Jsonb *jb, const char *key, const char *value) { JsonbIterator *it; JsonbValue v; JsonbIteratorToken r; if (jb == NULL || JB_ROOT_COUNT(jb) == 0) return false; if (JB_ROOT_IS_SCALAR(jb)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot find from scalar"))); it = JsonbIteratorInit(&jb->root); while ((r = JsonbIteratorNext(&it, &v, false)) != WJB_DONE) { if (r == WJB_KEY && v.type == jbvString && ((int) strlen(key) == v.val.string.len) && strncmp(key, v.val.string.val, v.val.string.len) == 0) { r = JsonbIteratorNext(&it, &v, false); Assert(r == WJB_VALUE || r == WJB_BEGIN_ARRAY); if (v.type == jbvArray) { /* iterate over the array members and consume them all as this function should only * match single values and not arrays */ int i = 0; int n_elems = v.val.array.nElems; while (i < n_elems && (r = JsonbIteratorNext(&it, &v, false)) == WJB_ELEM) { ++i; } continue; } else if (v.type != jbvString) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Jsonb value is of type \"%s\", but expected type \"string\"", JsonbTypeName(&v)))); } if (v.type == jbvString && ((int) strlen(value) == v.val.string.len) && strncmp(value, v.val.string.val, v.val.string.len) == 0) { return true; } } } return false; } ================================================ FILE: src/jsonb_utils.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <utils/datetime.h> #include <utils/json.h> #include <utils/jsonb.h> #include "export.h" extern TSDLLEXPORT void ts_jsonb_add_null(JsonbParseState *state, const char *key); extern TSDLLEXPORT void ts_jsonb_add_bool(JsonbParseState *state, const char *key, bool boolean); extern TSDLLEXPORT void ts_jsonb_add_str(JsonbParseState *state, const char *key, const char *value); extern TSDLLEXPORT void ts_jsonb_add_str_array(JsonbParseState *state, const char *key, const char **values, int num_values); extern TSDLLEXPORT void ts_jsonb_add_interval(JsonbParseState *state, const char *key, Interval *interval); extern TSDLLEXPORT void ts_jsonb_add_int32(JsonbParseState *state, const char *key, const int32 value); extern TSDLLEXPORT void ts_jsonb_add_int64(JsonbParseState *state, const char *key, const int64 value); extern TSDLLEXPORT void ts_jsonb_set_value_by_type(JsonbValue *value, Oid typeid, Datum datum); extern void ts_jsonb_add_value(JsonbParseState *state, const char *key, JsonbValue *value); extern TSDLLEXPORT char *ts_jsonb_get_str_field(const Jsonb *jsonb, const char *key); extern TSDLLEXPORT Interval *ts_jsonb_get_interval_field(const Jsonb *jsonb, const char *key); extern TSDLLEXPORT bool ts_jsonb_get_bool_field(const Jsonb *json, const char *key, bool *field_found); extern TSDLLEXPORT int32 ts_jsonb_get_int32_field(const Jsonb *json, const char *key, bool *field_found); extern TSDLLEXPORT int64 ts_jsonb_get_int64_field(const Jsonb *json, const char *key, bool *field_found); extern TSDLLEXPORT bool ts_jsonb_equal(const Jsonb *left, const Jsonb *right); extern TSDLLEXPORT bool ts_jsonb_has_key_value_str_field(Jsonb *jb, const char *key, const char *value); ================================================ FILE: src/license_guc.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_authid.h> #include <fmgr.h> #include <miscadmin.h> #include <utils/acl.h> #include <utils/builtins.h> #include <utils/guc.h> #include "cross_module_fn.h" #include "export.h" #include "extension_constants.h" #include "license_guc.h" static bool load_enabled = false; static GucSource load_source = PGC_S_DEFAULT; static void *tsl_handle = NULL; static PGFunction tsl_init_fn = NULL; static bool tsl_register_proc_exit = false; /* * License Functions. * * License validation is performed via guc update-hooks. * In this file we check if the type of license supplied warrants loading an * additional module. * * GUC checks work in two parts: * * 1. In the check function, all validation of the new value is performed * and any auxiliary state is setup but not installed. This function * is not allowed to throw exceptions. * * 2. In the assign function all user-visible state is installed. This * function *MUST NOT FAIL* as it can be called from such places as * transaction commitment, and will cause database restarts if it fails. * * Therefore license validation also works in two parts, corresponding to * check and assign: * * 1. In the first stage we check the license type, load the submodule into * memory if needed (but don't link any of the cross-module functions yet). * * 2. In the second stage we link all of the cross-module functions and init * tsl module. * * In order for restoring libraries to work (e.g. in parallel workers), loading * the submodule must happen strictly after the main timescaledb module is * loaded. In order to ensure that the initial value doesn't break this, we * disable loading submodules until the post_load_init. * * No license change from user session is allowed. License can be changed only * if it is set from server configuration file or the server command line. */ typedef enum { LICENSE_UNDEF, LICENSE_APACHE, LICENSE_TIMESCALE } LicenseType; static LicenseType license_type_of(const char *string) { if (string == NULL) return LICENSE_UNDEF; if (strcmp(string, TS_LICENSE_TIMESCALE) == 0) return LICENSE_TIMESCALE; if (strcmp(string, TS_LICENSE_APACHE) == 0) return LICENSE_APACHE; return LICENSE_UNDEF; } bool ts_license_is_apache(void) { return license_type_of(ts_guc_license) == LICENSE_APACHE; } TSDLLEXPORT void ts_license_enable_module_loading(void) { int result; if (load_enabled) return; load_enabled = true; /* re-set the license to actually load the submodule if needed */ result = set_config_option(MAKE_EXTOPTION("license"), ts_guc_license, PGC_SUSET, load_source, GUC_ACTION_SET, true, 0, false); if (result <= 0) elog(ERROR, "invalid value for timescaledb.license: \"%s\"", ts_guc_license); } /* * TSL module load function. * * Load the module, but do not start it. Set tsl_handle and * tsl_init_fn module init function pointer (tsl/src/init.c). * * This function is idempotent, and will not reload the module * if called multiple times. */ static bool tsl_module_load(void) { void *function; void *handle; if (tsl_handle != NULL) return true; function = load_external_function(EXTENSION_TSL_SO, "ts_module_init", false, &handle); if (function == NULL || handle == NULL) return false; tsl_init_fn = function; tsl_handle = handle; /* the on_proc_exit callback is registered by the tsl_init_fn after load */ tsl_register_proc_exit = true; return true; } static void tsl_module_init(void) { Assert(tsl_handle != NULL); Assert(tsl_init_fn != NULL); DirectFunctionCall1(tsl_init_fn, BoolGetDatum(tsl_register_proc_exit)); /* register the on_proc_exit only when the module is reloaded */ if (tsl_register_proc_exit) tsl_register_proc_exit = false; } /* * Check hook function set by license guc. * * Used to validate license string before the assign hook * ts_license_guc_assign_hook() call. */ bool ts_license_guc_check_hook(char **newval, void **extra, GucSource source) { LicenseType type = license_type_of(*newval); /* Allow setting a license only if is is set from postgresql.conf * or the server command line */ switch (type) { case LICENSE_APACHE: case LICENSE_TIMESCALE: if (source == PGC_S_FILE || source == PGC_S_ARGV || source == PGC_S_DEFAULT) break; GUC_check_errdetail("Cannot change a license in a running session."); GUC_check_errhint( "Change the license in the configuration file or server command line."); return false; case LICENSE_UNDEF: GUC_check_errdetail("Unrecognized license type."); GUC_check_errhint("Supported license types are 'timescale' or 'apache'."); return false; } /* If loading is delayed, save the GucSource for later retry * in the ts_license_enable_module_loading() */ if (!load_enabled) { load_source = source; return true; } if (type == LICENSE_TIMESCALE && !tsl_module_load()) { GUC_check_errdetail("Could not find TSL timescaledb module."); GUC_check_errhint("Check that \"%s\" is available.", EXTENSION_TSL_SO); return false; } return true; } /* * Assign hook function set by license guc, executed right after * ts_license_guc_check_hook() hook call. * * Executes tsl module init function (tsl/src/init.c) which sets the * cross-module function pointers. */ void ts_license_guc_assign_hook(const char *newval, void *extra) { if (load_enabled && license_type_of(newval) == LICENSE_TIMESCALE) tsl_module_init(); } ================================================ FILE: src/license_guc.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <fmgr.h> #include <utils/guc.h> #include <export.h> #include <guc.h> #define TS_LICENSE_APACHE "apache" #define TS_LICENSE_TIMESCALE "timescale" /* * If compiled with APACHE_ONLY, default to using only Apache code. */ #ifdef APACHE_ONLY #define TS_LICENSE_DEFAULT TS_LICENSE_APACHE #else #define TS_LICENSE_DEFAULT TS_LICENSE_TIMESCALE #endif extern bool ts_license_guc_check_hook(char **newval, void **extra, GucSource source); extern void ts_license_guc_assign_hook(const char *newval, void *extra); extern TSDLLEXPORT void ts_license_enable_module_loading(void); extern bool ts_license_is_apache(void); ================================================ FILE: src/loader/CMakeLists.txt ================================================ set(SOURCES loader.c bgw_message_queue.c bgw_counter.c bgw_launcher.c bgw_interface.c function_telemetry.c lwlocks.c) set(TEST_SOURCES ${PROJECT_SOURCE_DIR}/test/src/symbol_conflict.c) add_library(${PROJECT_NAME}-loader MODULE ${SOURCES}) if(CMAKE_BUILD_TYPE MATCHES Debug) # Include code for tests in Debug build target_sources(${PROJECT_NAME}-loader PRIVATE ${TEST_SOURCES}) # This define generates extension-specific code for symbol conflict testing target_compile_definitions(${PROJECT_NAME}-loader PUBLIC MODULE_NAME=loader) endif(CMAKE_BUILD_TYPE MATCHES Debug) set_target_properties(${PROJECT_NAME}-loader PROPERTIES OUTPUT_NAME ${PROJECT_NAME} PREFIX "") install(TARGETS ${PROJECT_NAME}-loader DESTINATION ${PG_PKGLIBDIR}) ================================================ FILE: src/loader/README.md ================================================ # Loader The loader has two main purposes: 1) Load the correct versioned library for each database. Multiple databases in the same Postgres instance may have different versions of TimescaleDB installed. The loader is responsible for loading the shared library corresponding to the correct TimescaleDB version for the database as soon as possible. For example, a database containing TimescaleDB version 0.8.0 will have timescaledb-0.8.0.so loaded. 2) Starting a background task called the launcher at server startup. The launcher is responsible for launching schedulers (one for each database) that are responsible for checking whether the TimescaleDB extension is installed in a database. In case of no TimescaleDB extension, the scheduler exits until it is reactivated for that database, which happens, for instance, when the extension is installed. If a scheduler finds an extension, its task is to schedule jobs for that database. The launcher controls when schedulers are started up or shut down in response to events that necessitate such actions. It also instantiates a counter from which TimescaleDB background workers are allocated to be sure we are not using more `worker_processes` than we should. # Messages the launcher may receive The launcher implements a simple message queue to be notified when it should take certain actions, like starting or restarting a scheduler for a given database. ## Message types sent to the launcher: `start`: Used to start the scheduler by the user. It is meant to be an idempotent start, as in, if it is run multiple times, it is the same as if it were run once. It is used mainly to reactivate a scheduler that the user had stopped. It does not reset the vxid of a scheduler and the started scheduler will not wait on txn finish. `stop`: Used to stop the scheduler immediately. It does not wait on a vxid and it is idempotent. `restart`: Used to either stop and restart the scheduler if it is running or start it if it is not. Technically, this would be better named `force_restart` as that better describes the action to start or restart the scheduler. The scheduler is immediately restarted, but waits on the vxid of the txn that sent the message. It is not idempotent, and will restart newly started schedulers, even while they are waiting. However, if the scheduler is already started or allocated, its "slot" is never released back to the pool, so as not to allow a job worker to "steal" a scheduler's slot during a restart. ## When/which messages are sent: Server startup: no message sent. However, the launcher takes essentially the `start` action for each database (without the message handling/signalling bit). It cannot figure out whether a scheduler should exist for a given database because it can only connect to shared catalogs. The scheduler is responsible for shutting down if it should not exist (because either TimescaleDB is not installed in the database or the version of TimescaleDB installed does not have a scheduler function to call). `CREATE DATABASE`: essentially the same as server startup. The launcher checks for new databases each time it wakes up and will start schedulers for any that it has not seen before. `CREATE EXTENSION`: the create script sends a `restart` message. It does not use the `start` message because we need to wait waiting on the vxid of the process that is running `CREATE EXTENSION`. There is also the possibility that the idempotency of the `start` action, even if it waited on a vxid, would cause race conditions in cases where the server has just started or the database has been created. `ALTER EXTENSION UPDATE`: the pre-update script sends a `restart` message. This ensures that the current scheduler is shut down as the action starts, it then waits on the vxid of the calling txn to figure out the correct version of the extension to use. `DROP EXTENSION`: sends a `restart` message, which is necessary because a rollback of the drop extension command can still happen. The scheduler therefore waits on the vxid of the txn running `DROP EXTENSION` and then will take the correct action depending on whether the extension exists when the txn finishes. `DROP DATABASE`: sends a `stop` message, causing immediate shutdown of the scheduler. This is necessary as the database cannot be dropped if there are any open connections to it (the scheduler maintains a connection to the db). # Launcher per-DB state machine The following is the state machine that the launcher maintains for each database. The CAPITAL labels are the possible states, and the `lowercase` names for messages that trigger the accompanying transitions. Transitions without labels are taken automatically whenever available resources exist. ``` stop ENABLED+--------------+ + ^--------------| | start/restart || | || | || v +v ALLOCATED+------> DISABLED ^+ stop ^ || | restart || | || | +v | STARTED+--------------+ stop / scheduler quit ``` ## The following is a detailed description of the transitions Note that `set vxid` sets a vxid variable on the scheduler. This variable is passed down to the scheduler and the scheduler waits on that vxid when it first starts. Transitions that happen automatically (at least once per poll period). * `ENABLED->ALLOCATED`: Reserved slot for worker * `ALLOCATED->STARTED`: Scheduler started * `STARTED->DISABLED`: Iff scheduler has stopped. Release slot. Transitions that happen upon getting a STOP MESSAGE: * `ENABLED->DISABLED`: No action * `ALLOCATED->DISABLED`: Release slot. * `STARTED->DISABLED`: Terminate scheduler & release slot * `DISABLED->DISABLED`: No Action Transitions that happen upon getting a START MESSAGE * Database not yet registered: Register, set to ENABLED and take ENABLED action below. * `ENABLED->ENABLED`: Try automatic transitions * `ALLOCATED->ALLOCATED`: Try automatic transitions * `STARTED->STARTED`: No action * `DISABLED->ENABLED`: Try automatic transitions Transitions that happen upon getting a RESTART MESSAGE * Database not yet registered: Register it set to ENABLED, take ENABLED actions * `ENABLED->ENABLED`: Set vxid, try automatic transitions * `ALLOCATED->ALLOCATED`: Set vxid, try automatic transitions * `STARTED->ALLOCATED`: Terminate scheduler, do /not/ release slot, set vxid, then try automatic transitions * `DISABLED->ENABLED`: Set vxid, try automatic transitions ================================================ FILE: src/loader/bgw_counter.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ /* needed for initializing shared memory and using various locks */ #include <postgres.h> #include <miscadmin.h> #include <storage/ipc.h> #include <storage/latch.h> #include <storage/lwlock.h> #include <storage/shmem.h> #include <storage/spin.h> #include <utils/guc.h> #include <utils/hsearch.h> #include "bgw_counter.h" #include "extension_constants.h" #define BGW_COUNTER_STATE_NAME "ts_bgw_counter_state" int ts_guc_max_background_workers = 16; /* * We need a bit of shared state here to deal with keeping track of the total * number of background workers we've launched across the instance since we * don't want to exceed some configured value. We considered, briefly, the * possibility of using pg_sema for this, unfortunately it does not appear to * be accessible to code outside of postgres core in any meaningful way. So * we're not using that. */ typedef struct CounterState { /* * Using an slock because we're only taking it for very brief periods to * read a single value so no need for an lwlock */ slock_t mutex; /* controls modification of total_workers */ int total_workers; } CounterState; static CounterState *ct = NULL; static void bgw_counter_state_init() { bool found; LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); ct = ShmemInitStruct(BGW_COUNTER_STATE_NAME, sizeof(CounterState), &found); if (!found) { memset(ct, 0, sizeof(CounterState)); SpinLockInit(&ct->mutex); ct->total_workers = 0; } LWLockRelease(AddinShmemInitLock); } extern void ts_bgw_counter_setup_gucs(void) { DefineCustomIntVariable(MAKE_EXTOPTION("max_background_workers"), "Maximum background worker processes allocated to TimescaleDB", "Max background worker processes allocated to TimescaleDB - set to at " "least 1 + number of databases in Postgres instance to use background " "workers ", &ts_guc_max_background_workers, ts_guc_max_background_workers, 0, 1000, /* no reasonable way to have more than * 1000 background workers */ PGC_POSTMASTER, 0, NULL, NULL, NULL); } /* * This gets called by the loader (and therefore the postmaster) at * shared_preload_libraries time */ extern void ts_bgw_counter_shmem_alloc(void) { RequestAddinShmemSpace(sizeof(CounterState)); } extern void ts_bgw_counter_shmem_startup(void) { bgw_counter_state_init(); } extern void ts_bgw_counter_reinit(void) { /* set counter back to zero on startup */ SpinLockAcquire(&ct->mutex); ct->total_workers = 0; SpinLockRelease(&ct->mutex); } extern bool ts_bgw_total_workers_increment_by(int increment_by) { bool incremented = false; int max_workers = ts_guc_max_background_workers; SpinLockAcquire(&ct->mutex); if (ct->total_workers + increment_by <= max_workers) { ct->total_workers += increment_by; incremented = true; } SpinLockRelease(&ct->mutex); return incremented; } extern bool ts_bgw_total_workers_increment() { return ts_bgw_total_workers_increment_by(1); } extern void ts_bgw_total_workers_decrement_by(int decrement_by) { /* * Launcher is 1 worker, and when it dies we reinitialize, so we should * never be below 1 */ SpinLockAcquire(&ct->mutex); if (ct->total_workers - decrement_by >= 1) { ct->total_workers -= decrement_by; SpinLockRelease(&ct->mutex); } else { SpinLockRelease(&ct->mutex); ereport(FATAL, (errmsg("TimescaleDB background worker cannot decrement workers below 1"), errhint("The background worker scheduler is in an invalid state and may not be " "keeping track of workers allocated to TimescaleDB properly, please " "submit a bug report."))); } } extern void ts_bgw_total_workers_decrement() { ts_bgw_total_workers_decrement_by(1); } extern int ts_bgw_total_workers_get() { int nworkers; SpinLockAcquire(&ct->mutex); nworkers = ct->total_workers; SpinLockRelease(&ct->mutex); return nworkers; } ================================================ FILE: src/loader/bgw_counter.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> extern int ts_guc_max_background_workers; extern void ts_bgw_counter_shmem_alloc(void); extern void ts_bgw_counter_shmem_startup(void); extern void ts_bgw_counter_setup_gucs(void); extern void ts_bgw_counter_reinit(void); extern bool ts_bgw_total_workers_increment(void); extern void ts_bgw_total_workers_decrement(void); extern int ts_bgw_total_workers_get(void); extern bool ts_bgw_total_workers_increment_by(int increment_by); extern void ts_bgw_total_workers_decrement_by(int decrement_by); ================================================ FILE: src/loader/bgw_interface.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <fmgr.h> #include <miscadmin.h> #include "../compat/compat.h" #include "../extension_constants.h" #include "bgw_counter.h" #include "bgw_interface.h" #include "bgw_message_queue.h" /* This is where versioned-extension facing functions live. They shouldn't live anywhere else. */ /* All loader changes should always be backward compatible. * Update ts_bgw_loader_api_version if the loader changes are needed for newer extension updates. * e.g. adding a LWLock to loader is required for some future change coming to OSM extension version * xxxx. RENDEZVOUS_BGW_LOADER_API_VERSION is used to verify if the loader in use is compatible with * the current TimescaleDB version. This check happens in bgw/bgw_launcher.c When * ts_bgw_loader_api_version is updated, check the compatibility in bgw/bgw_launcher.c as well */ const int32 ts_bgw_loader_api_version = 4; TS_FUNCTION_INFO_V1(ts_bgw_worker_reserve); TS_FUNCTION_INFO_V1(ts_bgw_worker_release); TS_FUNCTION_INFO_V1(ts_bgw_num_unreserved); TS_FUNCTION_INFO_V1(ts_bgw_db_workers_start); TS_FUNCTION_INFO_V1(ts_bgw_db_workers_stop); TS_FUNCTION_INFO_V1(ts_bgw_db_workers_restart); void ts_bgw_interface_register_api_version() { void **versionptr = find_rendezvous_variable(RENDEZVOUS_BGW_LOADER_API_VERSION); /* Cast away the const to store in the rendezvous variable */ *versionptr = (void *) &ts_bgw_loader_api_version; } Datum ts_bgw_worker_reserve(PG_FUNCTION_ARGS) { PG_RETURN_BOOL(ts_bgw_total_workers_increment()); } Datum ts_bgw_worker_release(PG_FUNCTION_ARGS) { ts_bgw_total_workers_decrement(); PG_RETURN_VOID(); } Datum ts_bgw_num_unreserved(PG_FUNCTION_ARGS) { int unreserved_workers; unreserved_workers = ts_guc_max_background_workers - ts_bgw_total_workers_get(); PG_RETURN_INT32(unreserved_workers); } Datum ts_bgw_db_workers_start(PG_FUNCTION_ARGS) { if (!superuser()) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), (errmsg("must be superuser to start background workers")))); PG_RETURN_BOOL(ts_bgw_message_send_and_wait(START, MyDatabaseId)); } Datum ts_bgw_db_workers_stop(PG_FUNCTION_ARGS) { if (!superuser()) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), (errmsg("must be superuser to stop background workers")))); PG_RETURN_BOOL(ts_bgw_message_send_and_wait(STOP, MyDatabaseId)); } Datum ts_bgw_db_workers_restart(PG_FUNCTION_ARGS) { if (!superuser()) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), (errmsg("must be superuser to restart background workers")))); PG_RETURN_BOOL(ts_bgw_message_send_and_wait(RESTART, MyDatabaseId)); } ================================================ FILE: src/loader/bgw_interface.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> extern void ts_bgw_interface_register_api_version(void); extern const int32 ts_bgw_loader_api_version; ================================================ FILE: src/loader/bgw_launcher.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> /* BGW includes below */ /* These are always necessary for a bgworker */ #include <miscadmin.h> #include <postmaster/bgworker.h> #include <storage/ipc.h> #include <storage/latch.h> #include <storage/lwlock.h> #include <storage/proc.h> #include <storage/shmem.h> /* for setting our wait event during waitlatch*/ #include <pgstat.h> /* needed for getting database list*/ #include <access/heapam.h> #include <access/htup_details.h> #include <access/xact.h> #include <catalog/pg_database.h> #include <utils/snapmgr.h> /* and checking db list for whether we're in a template*/ #include <utils/syscache.h> /* for calling external function*/ #include <fmgr.h> /* for signal handling (specifically die() function) */ #include <tcop/tcopprot.h> /* for looking up sending proc information for message handling */ #include <storage/procarray.h> /* for allocating the htab storage */ #include <utils/memutils.h> /* for getting settings correct before loading the versioned scheduler */ #include "catalog/pg_db_role_setting.h" #include "../compat/compat.h" #include "../extension_constants.h" #include "bgw_counter.h" #include "bgw_launcher.h" #include "bgw_message_queue.h" #include "loader.h" #define BGW_DB_SCHEDULER_FUNCNAME "ts_bgw_scheduler_main" #define BGW_ENTRYPOINT_FUNCNAME "ts_bgw_db_scheduler_entrypoint" typedef enum AckResult { ACK_FAILURE = 0, ACK_SUCCESS, } AckResult; /* See state machine in README.md */ typedef enum SchedulerState { /* Scheduler should be started but has not been allocated or started */ ENABLED = 0, /* The scheduler has been allocated a spot in timescaleDB's worker counter */ ALLOCATED, /* Scheduler has been started */ STARTED, /* * Scheduler is stopped and should not be started automatically. START and * RESTART messages can re-enable the scheduler. */ DISABLED } SchedulerState; #ifdef TS_DEBUG #define BGW_LAUNCHER_RESTART_TIME_S 0 #else #define BGW_LAUNCHER_RESTART_TIME_S 60 #endif static volatile sig_atomic_t got_SIGHUP = false; int ts_guc_bgw_scheduler_restart_time_sec = BGW_NEVER_RESTART; static void launcher_sighup(SIGNAL_ARGS) { /* based on av_sighup_handler */ int save_errno = errno; got_SIGHUP = true; SetLatch(MyLatch); errno = save_errno; } /* * Main bgw launcher for the cluster. * * Run through the TimescaleDB loader, so needs to have a small footprint as * any interactions it has will need to remain backwards compatible for the * foreseeable future. * * Notes: multiple databases in an instance (PG cluster) can have TimescaleDB * installed. They are not necessarily the same version of TimescaleDB (though * they could be) Shared memory is allocated and background workers are * registered at shared_preload_libraries time We do not know what databases * exist, nor which databases TimescaleDB is installed in (if any) at * shared_preload_libraries time. */ TS_FUNCTION_INFO_V1(ts_bgw_cluster_launcher_main); TS_FUNCTION_INFO_V1(ts_bgw_db_scheduler_entrypoint); typedef struct DbHashEntry { Oid db_oid; /* key for the hash table, must be first */ BackgroundWorkerHandle *db_scheduler_handle; /* needed to shut down * properly */ SchedulerState state; VirtualTransactionId vxid; int state_transition_failures; } DbHashEntry; static void scheduler_state_trans_enabled_to_allocated(DbHashEntry *entry); static void bgw_on_postmaster_death(void) { on_exit_reset(); /* don't call exit hooks cause we want to bail * out quickly */ ereport(FATAL, (errcode(ERRCODE_ADMIN_SHUTDOWN), errmsg("postmaster exited while TimescaleDB background worker launcher was working"))); } static void report_bgw_limit_exceeded(DbHashEntry *entry) { if (entry->state_transition_failures == 0) ereport(LOG, (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED), errmsg("TimescaleDB background worker limit of %d exceeded", ts_guc_max_background_workers), errhint("Consider increasing timescaledb.max_background_workers."))); entry->state_transition_failures++; } static void report_error_on_worker_register_failure(DbHashEntry *entry) { if (entry->state_transition_failures == 0) ereport(LOG, (errcode(ERRCODE_INSUFFICIENT_RESOURCES), errmsg("no available background worker slots"), errhint("Consider increasing max_worker_processes in tandem with " "timescaledb.max_background_workers."))); entry->state_transition_failures++; } /* * Aliasing a few things in bgworker.h so that we exit correctly on postmaster * death so we don't have to duplicate code basically telling it we shouldn't * call exit hooks cause we want to bail out quickly - similar to how the * quickdie function works when we receive a sigquit. This should work * similarly because postmaster death is a similar severity of issue. * Additionally, we're wrapping these calls to make sure we never have a NULL * handle, if we have a null handle, we return normal things. */ static BgwHandleStatus get_background_worker_pid(BackgroundWorkerHandle *handle, pid_t *pidp) { BgwHandleStatus status; pid_t pid; if (handle == NULL) status = BGWH_STOPPED; else { status = GetBackgroundWorkerPid(handle, &pid); if (pidp != NULL) *pidp = pid; } if (status == BGWH_POSTMASTER_DIED) bgw_on_postmaster_death(); return status; } static void wait_for_background_worker_startup(BackgroundWorkerHandle *handle, pid_t *pidp) { BgwHandleStatus status; if (handle == NULL) status = BGWH_STOPPED; else status = WaitForBackgroundWorkerStartup(handle, pidp); /* * We don't care whether we get BGWH_STOPPED or BGWH_STARTED here, because * the worker could have started and stopped very quickly before we read * it. We can't get BGWH_NOT_YET_STARTED as that's what we're waiting for. * We do care if the Postmaster died however. */ if (status == BGWH_POSTMASTER_DIED) bgw_on_postmaster_death(); Assert(status == BGWH_STOPPED || status == BGWH_STARTED); } static void wait_for_background_worker_shutdown(BackgroundWorkerHandle *handle) { BgwHandleStatus status; if (handle == NULL) status = BGWH_STOPPED; else status = WaitForBackgroundWorkerShutdown(handle); /* We can only ever get BGWH_STOPPED stopped unless the Postmaster died. */ if (status == BGWH_POSTMASTER_DIED) bgw_on_postmaster_death(); Assert(status == BGWH_STOPPED); } static void terminate_background_worker(BackgroundWorkerHandle *handle) { if (handle == NULL) return; else TerminateBackgroundWorker(handle); } static bool check_scheduler_restart_time(int *newval, void **extra, GucSource source) { if (*newval == -1 || *newval >= 10) return true; GUC_check_errdetail("Scheduler restart time must be be either -1 or at least 10 seconds."); return false; } extern void ts_bgw_cluster_launcher_init(void) { BackgroundWorker worker; DefineCustomIntVariable(/* name= */ MAKE_EXTOPTION("bgw_scheduler_restart_time"), /* short_desc= */ "Restart time for scheduler in seconds", /* long_desc= */ "The number of seconds until the scheduler restart on failure, or zero " "if it should never restart.", /* valueAddr= */ &ts_guc_bgw_scheduler_restart_time_sec, /* bootValue= */ BGW_NEVER_RESTART, /* minValue= */ -1, /* maxValue= */ 3600, /* context= */ PGC_SIGHUP, /* flags= */ GUC_UNIT_S, /* check_hook= */ check_scheduler_restart_time, /* assign_hook= */ NULL, /* show_hook= */ NULL); memset(&worker, 0, sizeof(worker)); /* set up worker settings for our main worker */ snprintf(worker.bgw_name, BGW_MAXLEN, TS_BGW_TYPE_LAUNCHER); worker.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; worker.bgw_restart_time = BGW_LAUNCHER_RESTART_TIME_S; /* * Starting at BgWorkerStart_RecoveryFinished means we won't ever get * started on a hot_standby see * https://www.postgresql.org/docs/10/static/bgworker.html as it's not * documented in bgworker.c. */ worker.bgw_start_time = BgWorkerStart_RecoveryFinished; worker.bgw_notify_pid = 0; snprintf(worker.bgw_library_name, BGW_MAXLEN, EXTENSION_NAME); snprintf(worker.bgw_function_name, BGW_MAXLEN, "ts_bgw_cluster_launcher_main"); RegisterBackgroundWorker(&worker); } /* * Register a background worker that calls the main TimescaleDB background * worker launcher library (i.e. loader) and uses the scheduler entrypoint * function. The scheduler entrypoint will deal with starting a new worker, * and waiting on any txns that it needs to, if we pass along a vxid in the * bgw_extra field of the BgWorker. */ static bool register_entrypoint_for_db(Oid db_id, VirtualTransactionId vxid, BackgroundWorkerHandle **handle) { BackgroundWorker worker; int restart_time_sec = ts_guc_bgw_scheduler_restart_time_sec; /* BGW_NEVER_RESTART is typically -1, but we check that explicitly here in * case PostgreSQL changes it. Compiler should optimize this away if they * are the same. */ if (restart_time_sec == -1) restart_time_sec = BGW_NEVER_RESTART; memset(&worker, 0, sizeof(worker)); snprintf(worker.bgw_type, BGW_MAXLEN, TS_BGW_TYPE_SCHEDULER); snprintf(worker.bgw_name, BGW_MAXLEN, "%s for database %d", TS_BGW_TYPE_SCHEDULER, db_id); worker.bgw_flags = BGWORKER_SHMEM_ACCESS | BGWORKER_BACKEND_DATABASE_CONNECTION; worker.bgw_restart_time = restart_time_sec; worker.bgw_start_time = BgWorkerStart_RecoveryFinished; snprintf(worker.bgw_library_name, BGW_MAXLEN, EXTENSION_NAME); snprintf(worker.bgw_function_name, BGW_MAXLEN, BGW_ENTRYPOINT_FUNCNAME); worker.bgw_notify_pid = MyProcPid; worker.bgw_main_arg = ObjectIdGetDatum(db_id); memcpy(worker.bgw_extra, &vxid, sizeof(VirtualTransactionId)); return RegisterDynamicBackgroundWorker(&worker, handle); } /* Initializes the launcher's hash table of schedulers. * Return value is guaranteed to be not-null, because otherwise the function * will have thrown an error. */ static HTAB * init_database_htab(void) { HASHCTL info = { .keysize = sizeof(Oid), .entrysize = sizeof(DbHashEntry), .hcxt = TopMemoryContext }; return hash_create("launcher_db_htab", ts_guc_max_background_workers, &info, HASH_BLOBS | HASH_CONTEXT | HASH_ELEM); } /* Insert a scheduler entry into the hash table. Correctly set entry values. */ static DbHashEntry * db_hash_entry_create_if_not_exists(HTAB *db_htab, Oid db_oid) { DbHashEntry *db_he; bool found; db_he = (DbHashEntry *) hash_search(db_htab, &db_oid, HASH_ENTER, &found); if (!found) { db_he->db_scheduler_handle = NULL; db_he->state = ENABLED; SetInvalidVirtualTransactionId(db_he->vxid); db_he->state_transition_failures = 0; /* * Try to allocate a spot right away to give schedulers priority over * other bgws. This is especially important on initial server startup * where we want to reserve slots for all schedulers before starting * any. This is done so that background workers started by schedulers * don't race for open slots with other schedulers on startup. */ scheduler_state_trans_enabled_to_allocated(db_he); } return db_he; } /* * Result from signalling a backend. * * Error codes are non-zero, and success is zero. */ enum SignalBackendResult { SIGNAL_BACKEND_SUCCESS = 0, SIGNAL_BACKEND_ERROR, SIGNAL_BACKEND_NOPERMISSION, SIGNAL_BACKEND_NOSUPERUSER, }; /* * Terminate a background worker. * * This is copied from pg_signal_backend() in * src/backend/storage/ipc/signalfuncs.c but tweaked to not require a database * connection since the launcher does not have one. */ static enum SignalBackendResult ts_signal_backend(int pid, int sig) { PGPROC *proc = BackendPidGetProc(pid); if (unlikely(proc == NULL)) { ereport(WARNING, (errmsg("PID %d is not a PostgreSQL backend process", pid))); return SIGNAL_BACKEND_ERROR; } if (unlikely(kill(pid, sig))) { /* Again, just a warning to allow loops */ ereport(WARNING, (errmsg("could not send signal to process %d: %m", pid))); return SIGNAL_BACKEND_ERROR; } return SIGNAL_BACKEND_SUCCESS; } /* * Terminate backends by backend type. * * We iterate through all backends and mark those that match the given backend * type as terminated. * * Note that there is potentially a delay between marking backends as * terminated and their actual termination, so the backends have to be able to * run even if there are multiple instances accessing the same data. * * Parts of this code is taken from pg_stat_get_activity() in * src/backend/utils/adt/pgstatfuncs.c. */ static void terminate_backends_by_backend_type(const char *backend_type) { Assert(backend_type); const int num_backends = pgstat_fetch_stat_numbackends(); for (int curr_backend = 1; curr_backend <= num_backends; ++curr_backend) { const LocalPgBackendStatus *local_beentry = pgstat_get_local_beentry_by_index_compat(curr_backend); const PgBackendStatus *beentry = &local_beentry->backendStatus; const char *bgw_type = GetBackgroundWorkerTypeByPid(beentry->st_procpid); if (bgw_type && strcmp(backend_type, bgw_type) == 0) { int error = ts_signal_backend(beentry->st_procpid, SIGTERM); if (error) elog(LOG, "failed to terminate backend with pid %d", beentry->st_procpid); } } } /* * Model this on autovacuum.c -> get_database_list. * * Note that we are not doing all the things around memory context that they * do, because the hashtable we're using to store db entries is automatically * created in its own memory context (a child of TopMemoryContext) This can * get called at two different times 1) when the cluster launcher starts and * is looking for dbs and 2) if it restarts due to a postmaster signal. */ static void populate_database_htab(HTAB *db_htab) { Relation rel; TableScanDesc scan; HeapTuple tup; /* * by this time we should already be connected to the db, and only have * access to shared catalogs */ StartTransactionCommand(); (void) GetTransactionSnapshot(); rel = table_open(DatabaseRelationId, AccessShareLock); scan = table_beginscan_catalog(rel, 0, NULL); while (HeapTupleIsValid(tup = heap_getnext(scan, ForwardScanDirection))) { Form_pg_database pgdb = (Form_pg_database) GETSTRUCT(tup); if (!pgdb->datallowconn || pgdb->datistemplate) continue; /* don't bother with dbs that don't allow * connections or are templates */ db_hash_entry_create_if_not_exists(db_htab, pgdb->oid); } table_endscan(scan); table_close(rel, AccessShareLock); CommitTransactionCommand(); } static void scheduler_modify_state(DbHashEntry *entry, SchedulerState new_state) { Assert(entry->state != new_state); entry->state_transition_failures = 0; entry->state = new_state; } /* TRANSITION FUNCTIONS */ static void scheduler_state_trans_disabled_to_enabled(DbHashEntry *entry) { Assert(entry->state == DISABLED); Assert(entry->db_scheduler_handle == NULL); scheduler_modify_state(entry, ENABLED); } static void scheduler_state_trans_enabled_to_allocated(DbHashEntry *entry) { Assert(entry->state == ENABLED); Assert(entry->db_scheduler_handle == NULL); /* Reserve a spot for this scheduler with BGW counter */ if (!ts_bgw_total_workers_increment()) { report_bgw_limit_exceeded(entry); return; } scheduler_modify_state(entry, ALLOCATED); } static void scheduler_state_trans_started_to_allocated(DbHashEntry *entry) { Assert(entry->state == STARTED); Assert(get_background_worker_pid(entry->db_scheduler_handle, NULL) == BGWH_STOPPED); if (entry->db_scheduler_handle != NULL) { pfree(entry->db_scheduler_handle); entry->db_scheduler_handle = NULL; } scheduler_modify_state(entry, ALLOCATED); } static void scheduler_state_trans_allocated_to_started(DbHashEntry *entry) { pid_t worker_pid; bool worker_registered; Assert(entry->state == ALLOCATED); Assert(entry->db_scheduler_handle == NULL); worker_registered = register_entrypoint_for_db(entry->db_oid, entry->vxid, &entry->db_scheduler_handle); if (!worker_registered) { report_error_on_worker_register_failure(entry); return; } wait_for_background_worker_startup(entry->db_scheduler_handle, &worker_pid); SetInvalidVirtualTransactionId(entry->vxid); scheduler_modify_state(entry, STARTED); } static void scheduler_state_trans_enabled_to_disabled(DbHashEntry *entry) { Assert(entry->state == ENABLED); Assert(entry->db_scheduler_handle == NULL); scheduler_modify_state(entry, DISABLED); } static void scheduler_state_trans_allocated_to_disabled(DbHashEntry *entry) { Assert(entry->state == ALLOCATED); Assert(entry->db_scheduler_handle == NULL); ts_bgw_total_workers_decrement(); scheduler_modify_state(entry, DISABLED); } static void scheduler_state_trans_started_to_disabled(DbHashEntry *entry) { Assert(entry->state == STARTED); Assert(get_background_worker_pid(entry->db_scheduler_handle, NULL) == BGWH_STOPPED); ts_bgw_total_workers_decrement(); if (entry->db_scheduler_handle != NULL) { pfree(entry->db_scheduler_handle); entry->db_scheduler_handle = NULL; } scheduler_modify_state(entry, DISABLED); } static void scheduler_state_trans_automatic(DbHashEntry *entry) { switch (entry->state) { case ENABLED: scheduler_state_trans_enabled_to_allocated(entry); if (entry->state == ALLOCATED) scheduler_state_trans_allocated_to_started(entry); break; case ALLOCATED: scheduler_state_trans_allocated_to_started(entry); break; case STARTED: if (get_background_worker_pid(entry->db_scheduler_handle, NULL) == BGWH_STOPPED) scheduler_state_trans_started_to_disabled(entry); break; case DISABLED: break; } } static void scheduler_state_trans_automatic_all(HTAB *db_htab) { HASH_SEQ_STATUS hash_seq; DbHashEntry *current_entry; hash_seq_init(&hash_seq, db_htab); while ((current_entry = hash_seq_search(&hash_seq)) != NULL) scheduler_state_trans_automatic(current_entry); } /* This is called when we're going to shut down so we don't leave things messy*/ static void launcher_pre_shmem_cleanup(int code, Datum arg) { HTAB *db_htab = *(HTAB **) DatumGetPointer(arg); HASH_SEQ_STATUS hash_seq; DbHashEntry *current_entry; /* db_htab will be NULL if we fail during init_database_htab */ if (db_htab != NULL) { hash_seq_init(&hash_seq, db_htab); /* * Stop everyone (or at least tell the Postmaster we don't care about * them anymore) */ while ((current_entry = hash_seq_search(&hash_seq)) != NULL) { if (current_entry->db_scheduler_handle != NULL) { terminate_background_worker(current_entry->db_scheduler_handle); pfree(current_entry->db_scheduler_handle); } } hash_destroy(db_htab); } /* * Reset our pid in the queue so that others know we've died and don't * wait forever */ ts_bgw_message_queue_shmem_cleanup(); } /* ************* * Actions for message types we could receive off of the bgw_message_queue. ************* */ /* * This should be idempotent. If we find the background worker and it's not * stopped, do nothing. In order to maintain idempotency, a scheduler in the * ENABLED, ALLOCATED or STARTED state cannot get a new vxid to wait on. (We * cannot pass in a new vxid to wait on for an already-started scheduler in any * case). This means that actions like restart, which are not idempotent, will * not have their effects changed by subsequent start actions, no matter the * state they are in when the start action is received. */ static AckResult message_start_action(HTAB *db_htab, BgwMessage *message) { DbHashEntry *entry; entry = db_hash_entry_create_if_not_exists(db_htab, message->db_oid); if (entry->state == DISABLED) scheduler_state_trans_disabled_to_enabled(entry); scheduler_state_trans_automatic(entry); return (entry->state == STARTED ? ACK_SUCCESS : ACK_FAILURE); } static AckResult message_stop_action(HTAB *db_htab, BgwMessage *message) { DbHashEntry *entry; /* * If the entry does not exist try to create it so we can put it in the * DISABLED state. Otherwise, it will be created during the next poll and * then will end up in the ENABLED state and proceed to being STARTED. But * this is not the behavior we want. */ entry = db_hash_entry_create_if_not_exists(db_htab, message->db_oid); switch (entry->state) { case ENABLED: scheduler_state_trans_enabled_to_disabled(entry); break; case ALLOCATED: scheduler_state_trans_allocated_to_disabled(entry); break; case STARTED: terminate_background_worker(entry->db_scheduler_handle); wait_for_background_worker_shutdown(entry->db_scheduler_handle); scheduler_state_trans_started_to_disabled(entry); break; case DISABLED: break; } return entry->state == DISABLED ? ACK_SUCCESS : ACK_FAILURE; } /* * This function will stop and restart a scheduler in the STARTED state, ENABLE * a scheduler if it does not exist or is in the DISABLED state and set the vxid * to wait on for a scheduler in any state. It is not idempotent. Additionally, * one might think that this function would simply be a combination of stop and * start above, but it is not as we maintain the worker's "slot" by never * releasing the worker from our "pool" of background workers as stopping and * starting would. We don't want a race condition where some other db steals * the scheduler of the other by requesting a worker at the wrong time. (This is * accomplished by moving from STARTED to ALLOCATED after shutting down the * worker, never releasing the entry and transitioning all the way back to * ENABLED). */ static AckResult message_restart_action(HTAB *db_htab, BgwMessage *message, VirtualTransactionId vxid) { DbHashEntry *entry; entry = db_hash_entry_create_if_not_exists(db_htab, message->db_oid); entry->vxid = vxid; switch (entry->state) { case ENABLED: break; case ALLOCATED: break; case STARTED: terminate_background_worker(entry->db_scheduler_handle); wait_for_background_worker_shutdown(entry->db_scheduler_handle); scheduler_state_trans_started_to_allocated(entry); break; case DISABLED: scheduler_state_trans_disabled_to_enabled(entry); } scheduler_state_trans_automatic(entry); return entry->state == STARTED ? ACK_SUCCESS : ACK_FAILURE; } /* * Handle 1 message. */ static bool launcher_handle_message(HTAB *db_htab) { BgwMessage *message = ts_bgw_message_receive(); PGPROC *sender; VirtualTransactionId vxid; AckResult action_result = ACK_FAILURE; if (message == NULL) return false; sender = BackendPidGetProc(message->sender_pid); if (sender == NULL) { ereport(LOG, (errmsg("TimescaleDB background worker launcher received message from non-existent " "backend"))); return true; } GET_VXID_FROM_PGPROC(vxid, *sender); switch (message->message_type) { case START: action_result = message_start_action(db_htab, message); break; case STOP: action_result = message_stop_action(db_htab, message); break; case RESTART: action_result = message_restart_action(db_htab, message, vxid); break; } ts_bgw_message_send_ack(message, action_result); return true; } /* * The default, `bgworker_die()`, can't be used due to the fact that it * handles signals synchronously, rather than waiting for a * CHECK_FOR_INTERRUPTS(). `die()` (which is arguably misnamed) sets flags * that will cause the backend to exit on the next call to * CHECK_FOR_INTERRUPTS(), which can happen either in our code or in functions * within the Postgres codebase that we call. This means that we don't need to * wait for the next time control is returned to our loop to exit, which would * be necessary if we set our own flag and checked it in a loop * condition. However, because it cannot exit 0, the launcher will be * restarted by the postmaster, even when it has received a SIGTERM, which we * decided was the proper behavior. If users want to disable the launcher, * they can set `timescaledb.max_background_workers = 0` and then we will * `proc_exit(0)` before doing anything else. */ extern Datum ts_bgw_cluster_launcher_main(PG_FUNCTION_ARGS) { HTAB **htab_storage; HTAB *db_htab; pqsignal(SIGINT, StatementCancelHandler); pqsignal(SIGTERM, die); pqsignal(SIGHUP, launcher_sighup); /* Some SIGHUPS may already have been dropped, so we must load the file here */ got_SIGHUP = false; ProcessConfigFile(PGC_SIGHUP); BackgroundWorkerUnblockSignals(); ereport(DEBUG1, (errmsg("TimescaleDB background worker launcher started"))); /* set counter back to zero on restart */ ts_bgw_counter_reinit(); if (!ts_bgw_total_workers_increment()) { /* * Should be the first thing happening so if we already exceeded our * limits it means we have a limit of 0 and we should just exit We * have to exit(0) because if we exit in error we get restarted by the * postmaster. */ ereport(LOG, (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED), errmsg("TimescaleDB background worker is set to 0"), errhint("TimescaleDB background worker launcher shutting down."))); proc_exit(0); } /* Connect to the db, no db name yet, so can only access shared catalogs */ BackgroundWorkerInitializeConnection(NULL, NULL, 0); pgstat_report_appname(MyBgworkerEntry->bgw_name); ereport(LOG, (errmsg("TimescaleDB background worker launcher connected to shared catalogs"))); htab_storage = (HTAB **) MemoryContextAllocZero(TopMemoryContext, sizeof(void *)); /* * We must setup the cleanup function _before_ initializing any state it * touches (specifically the bgw_message_queue and db_htab). Failing to do * this can cause cascading failures when the launcher fails in * init_database_htab (eg. due to running out of shared memory) but * doesn't deregister itself from the shared bgw_message_queue. */ before_shmem_exit(launcher_pre_shmem_cleanup, PointerGetDatum((void *) htab_storage)); ts_bgw_message_queue_set_reader(); db_htab = init_database_htab(); *htab_storage = db_htab; /* * If the launcher was restarted and discovers old schedulers, these has * to be terminated to avoid exhausting the worker slots. * * We cannot easily pick up the old schedulers since we do not have access * to the slots array in PostgreSQL, so instead we scan for something that * looks like schedulers for databases, and kill them. New ones will then * be spawned below. */ terminate_backends_by_backend_type(TS_BGW_TYPE_SCHEDULER); populate_database_htab(db_htab); while (true) { int wl_rc; bool handled_msgs = false; CHECK_FOR_INTERRUPTS(); populate_database_htab(db_htab); handled_msgs = launcher_handle_message(db_htab); scheduler_state_trans_automatic_all(db_htab); if (handled_msgs) continue; wl_rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_POSTMASTER_DEATH | WL_TIMEOUT, (long) ts_guc_bgw_launcher_poll_time, PG_WAIT_EXTENSION); ResetLatch(MyLatch); if (wl_rc & WL_POSTMASTER_DEATH) bgw_on_postmaster_death(); if (got_SIGHUP) { got_SIGHUP = false; ProcessConfigFile(PGC_SIGHUP); } } PG_RETURN_VOID(); } /* * Inside the entrypoint, we must check again if we're in a template db * even though we excluded template dbs in populate_database_htab because * we can be called on, say, CREATE EXTENSION in a template db and then * we'll not stop til next server shutdown so if we hit this point and are * in a template db, we throw an error and shut down Check in the syscache * rather than searching through the entire database catalog again. * Modelled on autovacuum.c -> do_autovacuum. */ static void database_checks(void) { Form_pg_database pgdb; HeapTuple tuple; tuple = SearchSysCache1(DATABASEOID, ObjectIdGetDatum(MyDatabaseId)); if (!HeapTupleIsValid(tuple)) ereport(ERROR, (errmsg("TimescaleDB background worker failed to find entry for database in " "syscache"))); pgdb = (Form_pg_database) GETSTRUCT(tuple); if (!pgdb->datallowconn) ereport(ERROR, (errmsg("background worker \"%s\" trying to connect to database that does not " "allow connections, exiting", MyBgworkerEntry->bgw_name))); if (pgdb->datistemplate) ereport(ERROR, (errmsg("background worker \"%s\" trying to connect to template database, exiting", MyBgworkerEntry->bgw_name))); ReleaseSysCache(tuple); } /* * Before we morph into the scheduler, we also need to reload configs from their * defaults if the database default has changed. Defaults are changed in the * post_restore function where we change the db default for the restoring guc * wait until the txn commits and then must see if the txn made the change. * Checks for changes are normally run at connection startup, but because we * have to connect in order to wait on the txn we have to re-run after the wait. * This function is based on the postgres function in postinit.c by the same * name. */ static void process_settings(Oid databaseid) { Relation relsetting; Snapshot snapshot; if (!IsUnderPostmaster) return; relsetting = table_open(DbRoleSettingRelationId, AccessShareLock); /* read all the settings under the same snapshot for efficiency */ snapshot = RegisterSnapshot(GetCatalogSnapshot(DbRoleSettingRelationId)); /* Later settings are ignored if set earlier. */ ApplySetting(snapshot, databaseid, InvalidOid, relsetting, PGC_S_DATABASE); ApplySetting(snapshot, InvalidOid, InvalidOid, relsetting, PGC_S_GLOBAL); UnregisterSnapshot(snapshot); table_close(relsetting, AccessShareLock); } /* * Get the versioned scheduler for the database. * * This captures any errors generated while fetching information and print * them out, but does not propagate the error further since that might trigger * a restart. */ static PGFunction get_versioned_scheduler() { volatile PGFunction versioned_scheduler_main = NULL; PG_TRY(); { bool ts_installed = false; char version[MAX_VERSION_LEN]; /* * now we can start our transaction and get the version currently * installed */ StartTransactionCommand(); (void) GetTransactionSnapshot(); /* * Check whether a database is a template database and raise an error if * so, as we don't want to run in template dbs. */ database_checks(); /* Process any config changes caused by an ALTER DATABASE */ process_settings(MyDatabaseId); ts_installed = ts_loader_extension_exists(); if (ts_installed) strlcpy(version, ts_loader_extension_version(), MAX_VERSION_LEN); ts_loader_extension_check(); CommitTransactionCommand(); if (ts_installed) { char soname[MAX_SO_NAME_LEN]; snprintf(soname, MAX_SO_NAME_LEN, "%s-%s", EXTENSION_SO, version); versioned_scheduler_main = load_external_function(soname, BGW_DB_SCHEDULER_FUNCNAME, false, NULL); if (versioned_scheduler_main == NULL) ereport(ERROR, (errmsg("TimescaleDB version %s does not have a background worker, exiting", soname))); } } PG_CATCH(); { EmitErrorReport(); FlushErrorState(); } PG_END_TRY(); return versioned_scheduler_main; } /* * This can be run either from the cluster launcher at db_startup time, or * in the case of an install/uninstall/update of the extension, in the * first case, we have no vxid that we're waiting on. In the second case, * we do, because we have to wait so that we see the effects of said txn. * So we wait for it to finish, then we morph into the new db_scheduler * worker using whatever version is now installed (or exit gracefully if * no version is now installed). */ extern Datum ts_bgw_db_scheduler_entrypoint(PG_FUNCTION_ARGS) { Oid db_id = DatumGetObjectId(MyBgworkerEntry->bgw_main_arg); VirtualTransactionId vxid; pqsignal(SIGINT, StatementCancelHandler); pqsignal(SIGTERM, die); BackgroundWorkerUnblockSignals(); /* * Connecting to a database that does not allow connections will generate * a FATAL error, which might trigger restarts, so we override this check * and do it ourselves. */ BackgroundWorkerInitializeConnectionByOid(db_id, InvalidOid, BGWORKER_BYPASS_ALLOWCONN); pgstat_report_appname(MyBgworkerEntry->bgw_name); /* * Wait until whatever vxid that potentially called us finishes before we * happens in a txn so it's cleaned up correctly if we get a sigkill in * the meantime, but we will need stop after and take a new txn so we can * see the correct state after its effects */ StartTransactionCommand(); (void) GetTransactionSnapshot(); memcpy(&vxid, MyBgworkerEntry->bgw_extra, sizeof(VirtualTransactionId)); if (VirtualTransactionIdIsValid(vxid)) VirtualXactLock(vxid, true); CommitTransactionCommand(); /* * Essentially we morph into the versioned worker here, if there is one. * * If an error is generated here, we should trigger a restart (if the * scheduler is configured for that). */ PGFunction versioned_scheduler_main = get_versioned_scheduler(); if (versioned_scheduler_main) DirectFunctionCall1(versioned_scheduler_main, ObjectIdGetDatum(InvalidOid)); PG_RETURN_VOID(); } ================================================ FILE: src/loader/bgw_launcher.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <fmgr.h> #define TS_BGW_TYPE_LAUNCHER "TimescaleDB Background Worker Launcher" #define TS_BGW_TYPE_SCHEDULER "TimescaleDB Background Worker Scheduler" extern int ts_guc_bgw_scheduler_restart_time_sec; extern void ts_bgw_cluster_launcher_init(void); /*called by postmaster at launcher bgw startup*/ TSDLLEXPORT extern Datum ts_bgw_cluster_launcher_main(PG_FUNCTION_ARGS); TSDLLEXPORT extern Datum ts_bgw_db_scheduler_entrypoint(PG_FUNCTION_ARGS); ================================================ FILE: src/loader/bgw_message_queue.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <miscadmin.h> #include <pgstat.h> #include <storage/lwlock.h> #include <storage/proc.h> #include <storage/procarray.h> #include <storage/shm_mq.h> #include <storage/shmem.h> #include <storage/spin.h> #include "../compat/compat.h" #include "bgw_message_queue.h" #define BGW_MQ_MAX_MESSAGES 16 #define BGW_MQ_NAME "ts_bgw_message_queue" #define BGW_MQ_TRANCHE_NAME "ts_bgw_mq_tranche" #define BGW_MQ_NUM_WAITS 100 /* WaitLatch expects a long */ #define BGW_MQ_WAIT_INTERVAL 1000L #define BGW_ACK_RETRIES 20 /* WaitLatch expects a long */ #define BGW_ACK_WAIT_INTERVAL 100L #define BGW_ACK_QUEUE_SIZE (MAXALIGN(shm_mq_minimum_size + sizeof(int))) /* We're using a relatively simple implementation of a circular queue similar to: * http://opendatastructures.org/ods-python/2_3_ArrayQueue_Array_Based_.html */ typedef struct MessageQueue { pid_t reader_pid; /* Should only be set once at cluster launcher * startup */ slock_t mutex; /* Controls access to the reader pid */ LWLock *lock; /* Pointer to the lock to control * adding/removing elements from queue */ uint8 read_upto; uint8 num_elements; BgwMessage buffer[BGW_MQ_MAX_MESSAGES]; } MessageQueue; typedef enum QueueResponseType { MESSAGE_SENT = 0, QUEUE_FULL, READER_DETACHED } QueueResponseType; static MessageQueue *mq = NULL; /* * This is run during the shmem_startup_hook. * On Linux, it's only run once, but in EXEC_BACKEND mode / on Windows/ other systems * that do forking differently, it is run in every backend at startup */ static void queue_init() { bool found; LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); mq = ShmemInitStruct(BGW_MQ_NAME, sizeof(MessageQueue), &found); if (!found) { memset(mq, 0, sizeof(MessageQueue)); mq->reader_pid = InvalidPid; SpinLockInit(&mq->mutex); mq->lock = &(GetNamedLWLockTranche(BGW_MQ_TRANCHE_NAME))->lock; } LWLockRelease(AddinShmemInitLock); } /* This gets called when shared memory is initialized in a backend * (shmem_startup_hook) */ extern void ts_bgw_message_queue_shmem_startup(void) { queue_init(); } /* This is called in the loader during server startup to allocate a shared * memory segment*/ extern void ts_bgw_message_queue_alloc(void) { RequestAddinShmemSpace(sizeof(MessageQueue)); RequestNamedLWLockTranche(BGW_MQ_TRANCHE_NAME, 1); } /* * Notes on managing the queue/locking: We decided that for this application, * simplicity of locking scheme was more important than being very good about * concurrency as the frequency of these messages will be low and the number * of messages on this queue should be low, given that they mostly happen when * we update the extension. Therefore we decided to simply take an exclusive * lock whenever we were modifying anything in the shared memory segment to * avoid collisions. */ static pid_t queue_get_reader(MessageQueue *queue) { pid_t reader; volatile MessageQueue *vq = queue; SpinLockAcquire(&vq->mutex); reader = vq->reader_pid; SpinLockRelease(&vq->mutex); return reader; } static void queue_set_reader(MessageQueue *queue) { volatile MessageQueue *vq = queue; pid_t reader_pid; SpinLockAcquire(&vq->mutex); if (vq->reader_pid == InvalidPid) { vq->reader_pid = MyProcPid; } reader_pid = vq->reader_pid; SpinLockRelease(&vq->mutex); if (reader_pid != MyProcPid) ereport(ERROR, (errmsg("only one reader allowed for TimescaleDB background worker message queue"), errhint("Current process is %d.", reader_pid))); } static void queue_reset_reader(MessageQueue *queue) { volatile MessageQueue *vq = queue; bool reset = false; SpinLockAcquire(&vq->mutex); if (vq->reader_pid == MyProcPid) { reset = true; vq->reader_pid = InvalidPid; } SpinLockRelease(&vq->mutex); if (!reset) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("multiple TimescaleDB background worker launchers have been started when " "only one is allowed"))); } /* Add a message to the queue - we can do this if the queue is not full */ static QueueResponseType queue_add(MessageQueue *queue, BgwMessage *message) { QueueResponseType message_result = QUEUE_FULL; LWLockAcquire(queue->lock, LW_EXCLUSIVE); if (queue->num_elements < BGW_MQ_MAX_MESSAGES) { memcpy(&queue->buffer[(queue->read_upto + queue->num_elements) % BGW_MQ_MAX_MESSAGES], message, sizeof(BgwMessage)); queue->num_elements++; message_result = MESSAGE_SENT; } LWLockRelease(queue->lock); if (queue_get_reader(queue) != InvalidPid) SetLatch(&BackendPidGetProc(queue_get_reader(queue))->procLatch); else message_result = READER_DETACHED; return message_result; } static BgwMessage * queue_remove(MessageQueue *queue) { BgwMessage *message = NULL; LWLockAcquire(queue->lock, LW_EXCLUSIVE); if (queue_get_reader(queue) != MyProcPid) ereport(ERROR, (errmsg( "cannot read if not reader for TimescaleDB background worker message queue"))); if (queue->num_elements > 0) { message = palloc(sizeof(BgwMessage)); memcpy(message, &queue->buffer[queue->read_upto], sizeof(BgwMessage)); queue->read_upto = (queue->read_upto + 1) % BGW_MQ_MAX_MESSAGES; queue->num_elements--; } LWLockRelease(queue->lock); return message; } /* Construct a message */ static BgwMessage * bgw_message_create(BgwMessageType message_type, Oid db_oid) { BgwMessage *message = palloc(sizeof(BgwMessage)); dsm_segment *seg; seg = dsm_create(BGW_ACK_QUEUE_SIZE, 0); *message = (BgwMessage){ .message_type = message_type, .sender_pid = MyProcPid, .db_oid = db_oid, .ack_dsm_handle = dsm_segment_handle(seg) }; return message; } /* * Our own version of shm_mq_wait_for_attach that waits with a timeout so that * should our counterparty die before attaching, we don't end up hanging. */ static shm_mq_result ts_shm_mq_wait_for_attach(MessageQueue *queue, shm_mq_handle *ack_queue_handle) { int n; PGPROC *reader_proc; for (n = 1; n <= BGW_MQ_NUM_WAITS; n++) { /* The reader is the sender on the ack queue */ reader_proc = shm_mq_get_sender(shm_mq_get_queue(ack_queue_handle)); if (reader_proc != NULL) return SHM_MQ_SUCCESS; else if (queue_get_reader(queue) == InvalidPid) return SHM_MQ_DETACHED; /* Reader died after we enqueued our * message */ WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, BGW_MQ_WAIT_INTERVAL, WAIT_EVENT_MESSAGE_QUEUE_INTERNAL); ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); } return SHM_MQ_DETACHED; } static bool enqueue_message_wait_for_ack(MessageQueue *queue, BgwMessage *message, shm_mq_handle *ack_queue_handle) { Size bytes_received = 0; QueueResponseType send_result; bool *data = NULL; shm_mq_result mq_res; bool ack_received = false; int n; /* * We don't want the process restarting workers to really distinguish the * reasons workers might or might not be restarted, and we don't really * want them to error when workers can't be started, as there are multiple * valid reasons for that. So we'll simply return false for the ack even * if we can't attach to the queue etc. */ send_result = queue_add(queue, message); if (send_result != MESSAGE_SENT) return false; mq_res = ts_shm_mq_wait_for_attach(queue, ack_queue_handle); if (mq_res != SHM_MQ_SUCCESS) return false; /* Get a response, non-blocking, with retries */ for (n = 1; n <= BGW_ACK_RETRIES; n++) { mq_res = shm_mq_receive(ack_queue_handle, &bytes_received, (void **) &data, true); if (mq_res != SHM_MQ_WOULD_BLOCK) break; ereport(DEBUG1, (errmsg("TimescaleDB ack message receive failure, retrying"))); WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, BGW_ACK_WAIT_INTERVAL, WAIT_EVENT_MESSAGE_QUEUE_INTERNAL); ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); } if (mq_res != SHM_MQ_SUCCESS) return false; ack_received = (bytes_received != 0) && *data; return ack_received; } /* * Write element to queue, wait/error if queue is full * consumes message and deallocates */ extern bool ts_bgw_message_send_and_wait(BgwMessageType message_type, Oid db_oid) { shm_mq *ack_queue; dsm_segment *seg; shm_mq_handle *ack_queue_handle; BgwMessage *message; bool ack_received = false; message = bgw_message_create(message_type, db_oid); seg = dsm_find_mapping(message->ack_dsm_handle); if (seg == NULL) ereport(ERROR, (errmsg("TimescaleDB background worker dynamic shared memory segment not mapped"))); ack_queue = shm_mq_create(dsm_segment_address(seg), BGW_ACK_QUEUE_SIZE); shm_mq_set_receiver(ack_queue, MyProc); ack_queue_handle = shm_mq_attach(ack_queue, seg, NULL); if (ack_queue_handle != NULL) ack_received = enqueue_message_wait_for_ack(mq, message, ack_queue_handle); dsm_detach(seg); /* Queue detach happens in dsm detach callback */ pfree(message); return ack_received; } /* * Called only by the launcher */ extern BgwMessage * ts_bgw_message_receive(void) { return queue_remove(mq); } extern void ts_bgw_message_queue_set_reader(void) { queue_set_reader(mq); } typedef enum MessageAckSent { ACK_SENT = 0, DSM_SEGMENT_UNAVAILABLE, QUEUE_NOT_ATTACHED, SEND_FAILURE } MessageAckSent; static const char *message_ack_sent_err[] = { [ACK_SENT] = "Sent ack successfully", [DSM_SEGMENT_UNAVAILABLE] = "DSM Segment unavailable", [QUEUE_NOT_ATTACHED] = "Ack queue unable to attach", [SEND_FAILURE] = "Unable to send ack on queue" }; static MessageAckSent send_ack(dsm_segment *seg, bool success) { shm_mq *ack_queue; shm_mq_handle *ack_queue_handle; shm_mq_result ack_res; int n; ack_queue = dsm_segment_address(seg); if (ack_queue == NULL) return DSM_SEGMENT_UNAVAILABLE; shm_mq_set_sender(ack_queue, MyProc); ack_queue_handle = shm_mq_attach(ack_queue, seg, NULL); if (ack_queue_handle == NULL) return QUEUE_NOT_ATTACHED; /* Send the message off, non blocking, with retries */ for (n = 1; n <= BGW_ACK_RETRIES; n++) { ack_res = shm_mq_send(ack_queue_handle, sizeof(bool), &success, true, true); if (ack_res != SHM_MQ_WOULD_BLOCK) break; ereport(DEBUG1, (errmsg("TimescaleDB ack message send failure, retrying"))); WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, BGW_ACK_WAIT_INTERVAL, WAIT_EVENT_MESSAGE_QUEUE_INTERNAL); ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); } /* we are responsible for pfree'ing the handle, the dsm infrastructure only * deals with the queue itself */ pfree(ack_queue_handle); if (ack_res != SHM_MQ_SUCCESS) return SEND_FAILURE; return ACK_SENT; } /* * Called by launcher once it has taken action based on the contents of the message * consumes message and deallocates */ extern void ts_bgw_message_send_ack(BgwMessage *message, bool success) { dsm_segment *seg; /* * PG 9.6 does not check to see if we had a CurrentResourceOwner inside of * dsm.c->dsm_create_descriptor. Basically, it assumed we were in a * transaction if we ever attached to the dsm, whereas PG 10 addressed * that and did proper NULL checking. So, if we are in 9.6, we start a * transaction and then commit it at the end of ack sending, to be sure * everything is cleaned up properly etc. */ seg = dsm_attach(message->ack_dsm_handle); if (seg != NULL) { MessageAckSent ack_res; ack_res = send_ack(seg, success); if (ack_res != ACK_SENT) ereport(DEBUG1, (errmsg("TimescaleDB background worker launcher unable to send ack to backend " "pid %d", message->sender_pid), errhint("Reason: %s", message_ack_sent_err[ack_res]))); dsm_detach(seg); } pfree(message); } /* * This gets called before shmem exit in the launcher (even if we're exiting * in error, but not if we're exiting due to possible shmem corruption) */ static void queue_shmem_cleanup(MessageQueue *queue) { queue_reset_reader(queue); } extern void ts_bgw_message_queue_shmem_cleanup(void) { queue_shmem_cleanup(mq); } ================================================ FILE: src/loader/bgw_message_queue.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <storage/dsm.h> typedef enum BgwMessageType { STOP = 0, START, RESTART } BgwMessageType; typedef struct BgwMessage { BgwMessageType message_type; pid_t sender_pid; Oid db_oid; dsm_handle ack_dsm_handle; } BgwMessage; extern bool ts_bgw_message_send_and_wait(BgwMessageType message, Oid db_oid); /* called only by the launcher*/ extern void ts_bgw_message_queue_set_reader(void); extern BgwMessage *ts_bgw_message_receive(void); extern void ts_bgw_message_send_ack(BgwMessage *message, bool success); /*called at server startup*/ extern void ts_bgw_message_queue_alloc(void); /*called in every backend during shmem startup hook*/ extern void ts_bgw_message_queue_shmem_startup(void); extern void ts_bgw_message_queue_shmem_cleanup(void); ================================================ FILE: src/loader/function_telemetry.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <fmgr.h> #include <port/atomics.h> #include <storage/lwlock.h> #include <storage/shmem.h> #include "loader/function_telemetry.h" // Function telemetry hash table size. Sized to be large enough that we're // unlikely to run out of entries, but small enough that it won't have a // noticeable impact. #define FN_TELEMETRY_HASH_SIZE 10000 static FnTelemetryRendezvous rendezvous; void ts_function_telemetry_shmem_startup() { FnTelemetryRendezvous **rendezvous_ptr; HASHCTL hash_info; HTAB *function_telemetry_hash; LWLock **lock; bool found; // NOTE: dshash would be better once it's stable hash_info.keysize = sizeof(Oid); hash_info.entrysize = sizeof(FnTelemetryHashEntry); LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); /* * GetNamedLWLockTranche must only be run once on windows, otherwise it * segfaults. Since the shmem_startup_hook is run on every backend, we use * a ShmemInitStruct to detect if this function has been called before. */ lock = (LWLock **) ShmemInitStruct("fn_telemetry_detect_first_run", sizeof(LWLock *), &found); if (!found) *lock = &(GetNamedLWLockTranche(FN_TELEMETRY_LWLOCK_TRANCHE_NAME))->lock; function_telemetry_hash = ShmemInitHash("timescaledb function telemetry hash", FN_TELEMETRY_HASH_SIZE, FN_TELEMETRY_HASH_SIZE, &hash_info, HASH_ELEM | HASH_BLOBS); LWLockRelease(AddinShmemInitLock); rendezvous.lock = *lock; rendezvous.function_counts = function_telemetry_hash; rendezvous_ptr = (FnTelemetryRendezvous **) find_rendezvous_variable(RENDEZVOUS_FUNCTION_TELEMENTRY); *rendezvous_ptr = &rendezvous; } void ts_function_telemetry_shmem_alloc() { Size size = hash_estimate_size(FN_TELEMETRY_HASH_SIZE, sizeof(FnTelemetryHashEntry)); RequestAddinShmemSpace(add_size(size, sizeof(LWLock *))); RequestNamedLWLockTranche(FN_TELEMETRY_LWLOCK_TRANCHE_NAME, 1); } ================================================ FILE: src/loader/function_telemetry.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #define RENDEZVOUS_FUNCTION_TELEMENTRY "ts_function_telemetry" #define FN_TELEMETRY_LWLOCK_TRANCHE_NAME "ts_fn_telemetry_lwlock_tranche" typedef struct FnTelemetryRendezvous { LWLock *lock; HTAB *function_counts; } FnTelemetryRendezvous; typedef struct FnTelemetryHashEntry { Oid key; pg_atomic_uint64 count; } FnTelemetryHashEntry; extern void ts_function_telemetry_shmem_startup(void); extern void ts_function_telemetry_shmem_alloc(void); ================================================ FILE: src/loader/loader.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/heapam.h> #include <access/parallel.h> #include <access/xact.h> #include <catalog/pg_database.h> #include <commands/dbcommands.h> #include <commands/defrem.h> #include <commands/user.h> #include <nodes/print.h> #include <parser/analyze.h> #include <pg_config.h> #include <postmaster/bgworker.h> #include <storage/ipc.h> #include <tcop/utility.h> #include <utils/guc.h> #include <utils/inval.h> #if PG_VERSION_NUM < 150000 #include "compat/compat-msvc-enter.h" #include <commands/extension.h> #include <miscadmin.h> #include "compat/compat-msvc-exit.h" #endif #include "compat/compat.h" #include "config.h" #include "export.h" #include "extension_constants.h" #include "extension_utils.c" #include "loader/bgw_counter.h" #include "loader/bgw_interface.h" #include "loader/bgw_launcher.h" #include "loader/bgw_message_queue.h" #include "loader/function_telemetry.h" #include "loader/loader.h" #include "loader/lwlocks.h" /* * Loading process: * * 1. _PG_init starts up cluster-wide background worker stuff, and sets the * post_parse_analyze_hook (a postgres-defined hook which is called after * every statement is parsed) to our function post_analyze_hook * 2. When a command is run with timescale not loaded, post_analyze_hook: * a. Gets the extension version. * b. Loads the versioned extension. * c. Grabs the post_parse_analyze_hook from the versioned extension * (src/init.c:post_analyze_hook) and stores it in * extension_post_parse_analyze_hook. * d. Sets the post_parse_analyze_hook back to what it was before we * loaded the versioned extension (this hook eventually called our * post_analyze_hook, but may not be our function, for instance, if * another extension is loaded). * e. Calls extension_post_parse_analyze_hook. * f. Calls the prev_post_parse_analyze_hook. * * Some notes on design: * * We do not check for the installation of the extension upon loading the extension and instead rely * on a hook for a few reasons: * * 1) We probably can't: * - The shared_preload_libraries is called in PostmasterMain which is way before InitPostgres is * called. Note: This happens even before the fork of the backend, so we don't even know which * database this is for. * - This means we cannot query for the existence of the extension yet because the caches are * initialized in InitPostgres. * * 2) We actually don't want to load the extension in two cases: * a) We are upgrading the extension. * b) We set the guc timescaledb.disable_load. * * 3) We include a section for the bgw launcher and some workers below the rest, separated with its * own notes, some function definitions are included as they are referenced by other loader * functions. * */ #ifdef PG_MODULE_MAGIC PG_MODULE_MAGIC; #endif #define POST_LOAD_INIT_FN "ts_post_load_init" #define GUC_LAUNCHER_POLL_TIME_MS MAKE_EXTOPTION("bgw_launcher_poll_time") /* * The loader really shouldn't load if we're in a parallel worker as there is a * separate infrastructure for loading libraries inside of parallel workers. The * issue is that IsParallelWorker() doesn't work on Windows because the var used * is not dll exported correctly, so we have an alternate macro that looks for * the parallel worker flags in MyBgworkerEntry, if it exists. */ #define CalledInParallelWorker() \ (MyBgworkerEntry != NULL && (MyBgworkerEntry->bgw_flags & BGWORKER_CLASS_PARALLEL) != 0) #if PG16_LT extern void TSDLLEXPORT _PG_init(void); #endif /* was the versioned-extension loaded*/ static bool loader_present = true; int ts_guc_bgw_launcher_poll_time = BGW_LAUNCHER_POLL_TIME_MS; /* This is the hook that existed before the loader was installed */ static post_parse_analyze_hook_type prev_post_parse_analyze_hook; static shmem_startup_hook_type prev_shmem_startup_hook; static shmem_request_hook_type prev_shmem_request_hook; typedef struct TsExtension { /* * Static data */ /* Name of the extension (must be part of the so file name) */ char const *const name; /* Name of the schema for table_name. */ char const *const schema_name; /* Name of the table whose existence indicates the extension is loaded. */ char const *const table_name; /* Name of the GUC for disabling loading this extension. */ char const *const guc_disable_load_name; /* * Run-time state */ /* Current value of this extension's disable GUC. */ bool guc_disable_load; /* Shared object library version loaded; empty if none. */ char soversion[MAX_VERSION_LEN]; /* TODO Remove. Neither timescaledb nor OSM actually have this hook, * never have, and we don't plan to add them. */ post_parse_analyze_hook_type post_parse_analyze_hook; } TsExtension; TsExtension extensions[] = { /* Redundant default initializers are here because we compile with * `-Werror -Wmissing-field-initializers` for our PG13 build... */ { .name = EXTENSION_NAME, .schema_name = CACHE_SCHEMA_NAME, .table_name = EXTENSION_PROXY_TABLE, .guc_disable_load_name = MAKE_EXTOPTION("disable_load"), .guc_disable_load = false, .soversion = "", .post_parse_analyze_hook = NULL, }, { .name = "timescaledb_osm", .schema_name = "_osm_catalog", .table_name = "metadata", .guc_disable_load_name = "timescaledb_osm.disable_load", .guc_disable_load = false, .soversion = "", .post_parse_analyze_hook = NULL, }, { .name = "timescaledb_lake", .schema_name = "_timescaledb_lake_catalog", .table_name = "metadata", .guc_disable_load_name = "timescaledb_lake.disable_load", .guc_disable_load = false, .soversion = "", .post_parse_analyze_hook = NULL, }, }; inline static void extension_check(TsExtension * /*ext*/); static void call_extension_post_parse_analyze_hook(ParseState *pstate, Query *query, TsExtension const * /*ext*/, JumbleState *jstate); static bool extension_is_loaded(TsExtension const *const ext) { /* The extension is loaded when the version is set to a non-null string */ return ext->soversion[0] != '\0'; } extern char * ts_loader_extension_version(void) { return extension_version(EXTENSION_NAME); } extern bool ts_loader_extension_exists(void) { return extension_exists(EXTENSION_NAME); } static bool drop_statement_drops_extension(DropStmt const *const stmt, TsExtension const *const ext) { if (!extension_exists(ext->name)) return false; if (stmt->removeType == OBJECT_EXTENSION) { if (list_length(stmt->objects) == 1) { char *ext_name; void *name = linitial(stmt->objects); ext_name = strVal(name); if (strcmp(ext_name, ext->name) == 0) return true; } } return false; } static Oid extension_owner(TsExtension const *const ext) { Datum result; Relation rel; SysScanDesc scandesc; HeapTuple tuple; ScanKeyData entry[1]; bool is_null = true; Oid extension_owner = InvalidOid; rel = table_open(ExtensionRelationId, AccessShareLock); ScanKeyInit(&entry[0], Anum_pg_extension_extname, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(ext->name)); scandesc = systable_beginscan(rel, ExtensionNameIndexId, true, NULL, 1, entry); tuple = systable_getnext(scandesc); /* We assume that there can be at most one matching tuple */ if (HeapTupleIsValid(tuple)) { result = heap_getattr(tuple, Anum_pg_extension_extowner, RelationGetDescr(rel), &is_null); if (!is_null) extension_owner = ObjectIdGetDatum(result); } systable_endscan(scandesc); table_close(rel, AccessShareLock); if (!OidIsValid(extension_owner)) elog(ERROR, "extension not found while getting owner"); return extension_owner; } static bool drop_owned_statement_drops_extension(DropOwnedStmt const *const stmt, TsExtension const *const ext) { Oid extension_owner_oid; List *role_ids; ListCell *lc; if (!extension_exists(ext->name)) return false; Assert(IsTransactionState()); extension_owner_oid = extension_owner(ext); role_ids = roleSpecsToIds(stmt->roles); /* Check privileges */ foreach (lc, role_ids) { Oid role_id = lfirst_oid(lc); if (role_id == extension_owner_oid) return true; } return false; } static bool should_load_on_variable_set(Node const *const utility_stmt, TsExtension const *const ext) { VariableSetStmt *stmt = (VariableSetStmt *) utility_stmt; switch (stmt->kind) { case VAR_SET_VALUE: case VAR_SET_DEFAULT: case VAR_RESET: /* Do not load when setting the guc to disable load */ return stmt->name == NULL || strcmp(stmt->name, ext->guc_disable_load_name) != 0; default: return true; } } static bool should_load_on_alter_extension(Node const *const utility_stmt, TsExtension const *const ext) { AlterExtensionStmt *stmt = (AlterExtensionStmt *) utility_stmt; if (strcmp(stmt->extname, ext->name) != 0) return true; /* disallow loading two .so from different versions */ if (extension_is_loaded(ext)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("extension \"%s\" cannot be updated after the old version has already been " "loaded", stmt->extname), errhint("Start a new session and execute ALTER EXTENSION as the first command. " "Make sure to pass the \"-X\" flag to psql."))); /* do not load the current (old) version's .so */ return false; } static bool should_load_on_create_extension(Node const *const utility_stmt, TsExtension const *const ext) { CreateExtensionStmt *stmt = (CreateExtensionStmt *) utility_stmt; if (strcmp(stmt->extname, ext->name) != 0) return false; /* If set, a library has already been loaded */ if (!extension_is_loaded(ext)) return true; /* * If the extension exists and the create statement has an IF NOT EXISTS * option, we continue without loading and let CREATE EXTENSION bail out * with a standard NOTICE. We can only do this if the extension actually * exists (is created), or else we might potentially load the shared * library of another version of the extension. Loading typically happens * on CREATE EXTENSION (via CREATE FUNCTION as SQL files are installed) * even if we do not explicitly load the library here. If we load another * version of the library, in addition to the currently loaded version, we * might taint the backend. */ if (extension_exists(ext->name) && stmt->if_not_exists) return false; /* * If the extension does not exist (e.g., was dropped via DROP SCHEMA * CASCADE) but the same version of the shared library is already loaded * in this session, allow the CREATE EXTENSION to proceed without * reloading. The .so is already in memory with all hooks in place, so * CREATE EXTENSION just needs to install the SQL objects. * * We only allow this when no explicit VERSION is specified (meaning the * default version from the control file will be used, which matches the * loaded .so) or when the specified VERSION matches the loaded version. */ if (!extension_exists(ext->name)) { char *requested_version = NULL; ListCell *lc; foreach (lc, stmt->options) { DefElem *d = (DefElem *) lfirst(lc); if (strcmp(d->defname, "new_version") == 0) { requested_version = defGetString(d); break; } } if (requested_version == NULL || strcmp(requested_version, ext->soversion) == 0) return false; } /* disallow loading two .so from different versions */ ereport(ERROR, (errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("extension \"%s\" has already been loaded with another version", stmt->extname), errdetail("The loaded version is \"%s\".", ext->soversion), errhint("Start a new session and execute CREATE EXTENSION as the first command. " "Make sure to pass the \"-X\" flag to psql."))); return false; } static bool load_utility_cmd(Node const *const utility_stmt, TsExtension const *const ext) { switch (nodeTag(utility_stmt)) { case T_VariableSetStmt: return should_load_on_variable_set(utility_stmt, ext); case T_AlterExtensionStmt: return should_load_on_alter_extension(utility_stmt, ext); case T_CreateExtensionStmt: return should_load_on_create_extension(utility_stmt, ext); case T_DropStmt: return !drop_statement_drops_extension((DropStmt *) utility_stmt, ext); default: return true; } } static void stop_workers_on_db_drop(DropdbStmt *drop_db_statement) { /* * Don't check if extension exists here because even though the current * database might not have TimescaleDB installed the database we are * dropping might. */ Oid dropped_db_oid = get_database_oid(drop_db_statement->dbname, drop_db_statement->missing_ok); if (OidIsValid(dropped_db_oid)) { ereport(LOG, (errmsg("TimescaleDB background worker scheduler for database %u will be stopped", dropped_db_oid))); ts_bgw_message_send_and_wait(STOP, dropped_db_oid); } } static bool database_allowconn(const Oid db_oid) { Relation pg_database; ScanKeyData entry[1]; SysScanDesc scan; HeapTuple dbtuple; bool allowconn = false; pg_database = table_open(DatabaseRelationId, AccessShareLock); ScanKeyInit(&entry[0], Anum_pg_database_oid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(db_oid)); scan = systable_beginscan(pg_database, DatabaseOidIndexId, true, NULL, 1, entry); dbtuple = systable_getnext(scan); /* We assume that there can be at most one matching tuple */ if (!HeapTupleIsValid(dbtuple)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_DATABASE), errmsg("database with OID \"%u\" does not exist", db_oid))); allowconn = ((Form_pg_database) GETSTRUCT(dbtuple))->datallowconn; systable_endscan(scan); table_close(pg_database, AccessShareLock); return allowconn; } static void post_analyze_hook(ParseState *pstate, Query *query, JumbleState *jstate) { if (query->commandType == CMD_UTILITY) { switch (nodeTag(query->utilityStmt)) { case T_AlterDatabaseStmt: { /* * On ALTER DATABASE SET TABLESPACE we need to stop background * workers for the command to succeed. */ AlterDatabaseStmt *stmt = (AlterDatabaseStmt *) query->utilityStmt; if (list_length(stmt->options) == 1) { DefElem *option = linitial(stmt->options); if (option->defname && strcmp(option->defname, "tablespace") == 0) { Oid db_oid = get_database_oid(stmt->dbname, false); if (OidIsValid(db_oid)) { ts_bgw_message_send_and_wait(RESTART, db_oid); ereport(WARNING, (errmsg("you may need to manually restart any running " "background workers after this command"))); } } } break; } case T_CreatedbStmt: { /* * If we create a database and the database used as template * has background workers we need to stop those background * workers connected to the template database. */ CreatedbStmt *stmt = (CreatedbStmt *) query->utilityStmt; ListCell *lc; foreach (lc, stmt->options) { DefElem *option = lfirst(lc); if (option->defname != NULL && option->arg != NULL && strcmp(option->defname, "template") == 0) { Oid db_oid = get_database_oid(defGetString(option), false); if (OidIsValid(db_oid) && database_allowconn(db_oid)) ts_bgw_message_send_and_wait(RESTART, db_oid); } } break; } case T_DropdbStmt: { DropdbStmt *stmt = (DropdbStmt *) query->utilityStmt; /* * If we drop a database, we need to intercept and stop any of our * schedulers that might be connected to said db. */ stop_workers_on_db_drop(stmt); break; } case T_DropStmt: for (size_t i = 0; i < sizeof(extensions) / sizeof(TsExtension); ++i) { if (drop_statement_drops_extension((DropStmt *) query->utilityStmt, &extensions[i])) { /* * if we drop the extension we should restart (in case of * a rollback) the scheduler */ ts_bgw_message_send_and_wait(RESTART, MyDatabaseId); break; } } break; case T_DropOwnedStmt: for (size_t i = 0; i < sizeof(extensions) / sizeof(TsExtension); ++i) { if (drop_owned_statement_drops_extension((DropOwnedStmt *) query->utilityStmt, &extensions[i])) { ts_bgw_message_send_and_wait(RESTART, MyDatabaseId); break; } } break; case T_RenameStmt: if (((RenameStmt *) query->utilityStmt)->renameType == OBJECT_DATABASE) { RenameStmt *stmt = (RenameStmt *) query->utilityStmt; Oid db_oid = get_database_oid(stmt->subname, stmt->missing_ok); if (OidIsValid(db_oid)) { ts_bgw_message_send_and_wait(STOP, db_oid); ereport(WARNING, (errmsg("you need to manually restart any running " "background workers after this command"))); } } break; default: break; } } for (size_t i = 0; i < sizeof(extensions) / sizeof(TsExtension); ++i) { TsExtension *const ext = &extensions[i]; /* timescaledb.disable_load prevents loading of all extensions. * timescaledb_osm.disable_load prevents loading of timescaledb_osm. * If we ever had a third extension to load, we might need to make * this smarter, but not today. */ bool const disable_load = extensions[0].guc_disable_load || ext->guc_disable_load; if (!disable_load && (query->commandType != CMD_UTILITY || load_utility_cmd(query->utilityStmt, ext))) { extension_check(ext); } /* * Call the extension's hook. This is necessary since the extension is * installed during the hook. If we did not do this the extension's hook * would not be called during the first command because the extension * would not have yet been installed. Thus the loader captures the * extension hook and calls it explicitly after the check for installing * the extension. */ call_extension_post_parse_analyze_hook(pstate, query, ext, jstate); } if (prev_post_parse_analyze_hook != NULL) { prev_post_parse_analyze_hook(pstate, query, jstate); } } static void timescaledb_shmem_startup_hook(void) { if (prev_shmem_startup_hook) prev_shmem_startup_hook(); ts_bgw_counter_shmem_startup(); ts_bgw_message_queue_shmem_startup(); ts_lwlocks_shmem_startup(); ts_function_telemetry_shmem_startup(); } /* * PG15 requires all shared memory requests to be requested in a dedicated * hook. We group all our shared memory requests in this function and use * it as a normal function for PG < 14 and as a hook for PG 15+. */ static void timescaledb_shmem_request_hook(void) { if (prev_shmem_request_hook) prev_shmem_request_hook(); ts_bgw_counter_shmem_alloc(); ts_bgw_message_queue_alloc(); ts_lwlocks_shmem_alloc(); ts_function_telemetry_shmem_alloc(); } static void extension_mark_loader_present() { void **presentptr = find_rendezvous_variable(RENDEZVOUS_LOADER_PRESENT_NAME); *presentptr = &loader_present; } void _PG_init(void) { if (!process_shared_preload_libraries_in_progress) { extension_load_without_preload(); } extension_mark_loader_present(); elog(INFO, "timescaledb loaded"); ts_bgw_cluster_launcher_init(); ts_bgw_counter_setup_gucs(); ts_bgw_interface_register_api_version(); /* This is a safety-valve variable to prevent loading the full extension */ for (size_t i = 0; i < sizeof(extensions) / sizeof(TsExtension); ++i) { TsExtension *const ext = &extensions[i]; DefineCustomBoolVariable(ext->guc_disable_load_name, "Disable the loading of the actual extension", NULL, &ext->guc_disable_load, false, PGC_USERSET, 0, NULL, NULL, NULL); } DefineCustomIntVariable(GUC_LAUNCHER_POLL_TIME_MS, "Launcher timeout value in milliseconds", "Configure the time the launcher waits " "to look for new TimescaleDB instances", &ts_guc_bgw_launcher_poll_time, BGW_LAUNCHER_POLL_TIME_MS, /* 10 ms or 60 seconds */ 10, /* min: 10ms */ PG_INT32_MAX, /* PG_INT16_MAX would be too small */ PGC_POSTMASTER, 0, NULL, NULL, NULL); /* * Cannot check for extension here since not inside a transaction yet. Nor * do we even have an assigned database yet. * Using the post_parse_analyze_hook since it's the earliest available * hook. */ prev_post_parse_analyze_hook = post_parse_analyze_hook; /* register shmem startup hook for the background worker stuff */ prev_shmem_startup_hook = shmem_startup_hook; post_parse_analyze_hook = post_analyze_hook; shmem_startup_hook = timescaledb_shmem_startup_hook; prev_shmem_request_hook = shmem_request_hook; shmem_request_hook = timescaledb_shmem_request_hook; } inline static void do_load(TsExtension *const ext) { char *version = extension_version(ext->name); char soname[MAX_SO_NAME_LEN]; post_parse_analyze_hook_type old_hook; /* If the right version of the library is already loaded, we will just * skip the actual loading. If the wrong version of the library is loaded, * we need to kill the session since it will not be able to continue * operate. */ if (extension_is_loaded(ext)) { if (strcmp(ext->soversion, version) == 0) return; ereport(FATAL, (errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("\"%s\" already loaded with a different version", ext->name), errdetail("The new version is \"%s\", this session is using version \"%s\". The " "session will be restarted.", version, ext->soversion))); } strlcpy(ext->soversion, version, MAX_VERSION_LEN); snprintf(soname, MAX_SO_NAME_LEN, "%s%s-%s", TS_LIBDIR, ext->name, version); /* * In a parallel worker, we're not responsible for loading libraries, it's * handled by the parallel worker infrastructure which restores the * library state. */ if (CalledInParallelWorker()) { return; } /* * Set the config option to let versions 0.9.0 and 0.9.1 know that the * loader was preloaded, newer versions use rendezvous variables instead. */ if ((strcmp(version, "0.9.0") == 0 || strcmp(version, "0.9.1") == 0) && strcmp(ext->name, EXTENSION_NAME) == 0) { SetConfigOption(MAKE_EXTOPTION("loader_present"), "on", PGC_USERSET, PGC_S_SESSION); } /* * we need to capture the loaded extension's post analyze hook, giving it * a NULL as previous */ old_hook = post_parse_analyze_hook; post_parse_analyze_hook = NULL; /* * We want to call the post_parse_analyze_hook from the versioned * extension after we've loaded the versioned so. When the file is loaded * it sets post_parse_analyze_hook, which we capture and store in * extension_post_parse_analyze_hook to call at the end _PG_init */ PG_TRY(); { PGFunction ts_post_load_init = load_external_function(soname, POST_LOAD_INIT_FN, false, NULL); if (ts_post_load_init != NULL) { DirectFunctionCall1(ts_post_load_init, CharGetDatum(0)); } } PG_CATCH(); { ext->post_parse_analyze_hook = post_parse_analyze_hook; post_parse_analyze_hook = old_hook; PG_RE_THROW(); } PG_END_TRY(); ext->post_parse_analyze_hook = post_parse_analyze_hook; post_parse_analyze_hook = old_hook; } inline static void extension_check(TsExtension *const ext) { switch (extension_current_state(ext->name, ext->schema_name, ext->table_name)) { case EXTENSION_STATE_TRANSITIONING: /* * Always load as soon as the extension is transitioning. This is * necessary so that the extension load before any CREATE FUNCTION * calls. Otherwise, the CREATE FUNCTION calls will load the .so * without capturing the post_parse_analyze_hook. */ case EXTENSION_STATE_CREATED: do_load(ext); return; case EXTENSION_STATE_UNKNOWN: case EXTENSION_STATE_NOT_INSTALLED: return; } } extern void ts_loader_extension_check(void) { for (size_t i = 0; i < sizeof(extensions) / sizeof(TsExtension); ++i) { extension_check(&extensions[i]); } } static void call_extension_post_parse_analyze_hook(ParseState *pstate, Query *query, TsExtension const *const ext, JumbleState *jstate) { if (extension_is_loaded(ext) && ext->post_parse_analyze_hook != NULL) { ext->post_parse_analyze_hook(pstate, query, jstate); } } ================================================ FILE: src/loader/loader.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> extern char *ts_loader_extension_version(void); extern bool ts_loader_extension_exists(void); extern void ts_loader_extension_check(void); /* WaitLatch expects a long, so make sure to cast the value */ /* Default value for timescaledb.launcher_poll_time */ #ifdef TS_DEBUG #define BGW_LAUNCHER_POLL_TIME_MS 10 #else #define BGW_LAUNCHER_POLL_TIME_MS 60000 #endif /* GUC to control launcher timeout */ extern int ts_guc_bgw_launcher_poll_time; ================================================ FILE: src/loader/lwlocks.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <fmgr.h> #include <miscadmin.h> #include <storage/lwlock.h> #include <storage/shmem.h> #include "loader/lwlocks.h" #define TS_LWLOCKS_SHMEM_NAME "ts_lwlocks_shmem" #define CHUNK_APPEND_LWLOCK_TRANCHE_NAME "ts_chunk_append_lwlock_tranche" #define OSM_PARALLEL_LWLOCK_TRANCHE_NAME "ts_osm_parallel_lwlock_tranche" /* * since shared memory can only be setup in a library loaded as * shared_preload_libraries we have to setup this struct here */ typedef struct TSLWLocks { LWLock *chunk_append; LWLock *osm_parallel_lwlock; } TSLWLocks; static TSLWLocks *ts_lwlocks = NULL; void ts_lwlocks_shmem_startup() { bool found; LWLock **lock_pointer, **osm_lock_pointer; LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); ts_lwlocks = ShmemInitStruct(TS_LWLOCKS_SHMEM_NAME, sizeof(TSLWLocks), &found); if (!found) { memset(ts_lwlocks, 0, sizeof(TSLWLocks)); ts_lwlocks->chunk_append = &(GetNamedLWLockTranche(CHUNK_APPEND_LWLOCK_TRANCHE_NAME))->lock; ts_lwlocks->osm_parallel_lwlock = &(GetNamedLWLockTranche(OSM_PARALLEL_LWLOCK_TRANCHE_NAME))->lock; } LWLockRelease(AddinShmemInitLock); /* * We use a lock specific rendezvous variable to decouple the struct * from the individual lock users to have no constraints on the struct * across timescaledb versions. */ lock_pointer = (LWLock **) find_rendezvous_variable(RENDEZVOUS_CHUNK_APPEND_LWLOCK); *lock_pointer = ts_lwlocks->chunk_append; osm_lock_pointer = (LWLock **) find_rendezvous_variable(RENDEZVOUS_OSM_PARALLEL_LWLOCK); *osm_lock_pointer = ts_lwlocks->osm_parallel_lwlock; } /* * from postgres code comments: * Extensions (or core code) can obtain an LWLocks by calling * RequestNamedLWLockTranche() during postmaster startup. Subsequently, * call GetNamedLWLockTranche() to obtain a pointer to an array containing * the number of LWLocks requested. */ void ts_lwlocks_shmem_alloc() { RequestNamedLWLockTranche(CHUNK_APPEND_LWLOCK_TRANCHE_NAME, 1); RequestNamedLWLockTranche(OSM_PARALLEL_LWLOCK_TRANCHE_NAME, 1); RequestAddinShmemSpace(sizeof(TSLWLocks)); } ================================================ FILE: src/loader/lwlocks.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #define RENDEZVOUS_CHUNK_APPEND_LWLOCK "ts_chunk_append_lwlock" #define RENDEZVOUS_OSM_PARALLEL_LWLOCK "ts_osm_parallel_lwlock" void ts_lwlocks_shmem_startup(void); void ts_lwlocks_shmem_alloc(void); ================================================ FILE: src/net/CMakeLists.txt ================================================ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/conn.c ${CMAKE_CURRENT_SOURCE_DIR}/conn_plain.c ${CMAKE_CURRENT_SOURCE_DIR}/http.c ${CMAKE_CURRENT_SOURCE_DIR}/http_response.c ${CMAKE_CURRENT_SOURCE_DIR}/http_request.c) if(USE_OPENSSL) list(APPEND SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/conn_ssl.c) endif(USE_OPENSSL) target_sources(${PROJECT_NAME} PRIVATE ${SOURCES}) ================================================ FILE: src/net/conn.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include "conn_internal.h" #include "debug_assert.h" static ConnOps *conn_ops[_CONNECTION_MAX] = { NULL }; static const char *conn_names[] = { [CONNECTION_PLAIN] = "PLAIN", [CONNECTION_SSL] = "SSL", [CONNECTION_MOCK] = "MOCK", }; static Connection * connection_internal_create(ConnectionType type, ConnOps *ops) { Connection *conn = palloc(ops->size); if (NULL == conn) return NULL; memset(conn, 0, ops->size); conn->ops = ops; conn->type = type; return conn; } Connection * ts_connection_create(ConnectionType type) { Connection *conn; if (type == _CONNECTION_MAX) { elog(NOTICE, "invalid connection type"); return NULL; } if (NULL == conn_ops[type]) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("%s connections are not supported", conn_names[type]), errhint("Enable %s support when compiling the extension.", conn_names[type]))); conn = connection_internal_create(type, conn_ops[type]); Ensure(conn, "unable to create connection"); if (NULL != conn->ops->init) if (conn->ops->init(conn) < 0) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("%s connection could not be initialized", conn_names[type]))); return conn; } /* * Connect to a remote endpoint (host, service/port). * * The connection will be made to the host's service endpoint given by * 'servname' (e.g., 'http'), unless a valid port number is given. */ int ts_connection_connect(Connection *conn, const char *host, const char *servname, int port) { /* Windows defines 'connect()' as a macro, so we need to undef it here to use it in ops->connect */ #ifdef WIN32 #undef connect #endif return conn->ops->connect(conn, host, servname, port); } ssize_t ts_connection_write(Connection *conn, const char *buf, size_t writelen) { return conn->ops->write(conn, buf, writelen); } ssize_t ts_connection_read(Connection *conn, char *buf, size_t buflen) { return conn->ops->read(conn, buf, buflen); } void ts_connection_close(Connection *conn) { if (NULL != conn->ops) conn->ops->close(conn); } int ts_connection_set_timeout_millis(Connection *conn, unsigned long millis) { if (NULL != conn->ops->set_timeout) return conn->ops->set_timeout(conn, millis); return -1; } void ts_connection_destroy(Connection *conn) { if (conn == NULL) return; ts_connection_close(conn); conn->ops = NULL; pfree(conn); } int ts_connection_register(ConnectionType type, ConnOps *ops) { if (type == _CONNECTION_MAX) return -1; conn_ops[type] = ops; return 0; } const char * ts_connection_get_and_clear_error(Connection *conn) { if (NULL != conn->ops->errmsg) return conn->ops->errmsg(conn); return "unknown connection error"; } ================================================ FILE: src/net/conn.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> typedef struct ConnOps ConnOps; typedef enum ConnectionType { CONNECTION_PLAIN, CONNECTION_SSL, CONNECTION_MOCK, _CONNECTION_MAX, } ConnectionType; typedef struct Connection { ConnectionType type; #ifdef WIN32 SOCKET sock; #else int sock; #endif ConnOps *ops; int err; } Connection; extern Connection *ts_connection_create(ConnectionType type); extern int ts_connection_connect(Connection *conn, const char *host, const char *servname, int port); extern ssize_t ts_connection_read(Connection *conn, char *buf, size_t buflen); extern ssize_t ts_connection_write(Connection *conn, const char *buf, size_t writelen); extern void ts_connection_close(Connection *conn); extern void ts_connection_destroy(Connection *conn); extern int ts_connection_set_timeout_millis(Connection *conn, unsigned long millis); extern const char *ts_connection_get_and_clear_error(Connection *conn); ================================================ FILE: src/net/conn_internal.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include "conn.h" typedef struct ConnOps { size_t size; /* Size of the connection object */ int (*init)(Connection *conn); int (*connect)(Connection *conn, const char *host, const char *servname, int port); void (*close)(Connection *conn); ssize_t (*write)(Connection *conn, const char *buf, size_t writelen); ssize_t (*read)(Connection *conn, char *buf, size_t readlen); int (*set_timeout)(Connection *conn, unsigned long millis); const char *(*errmsg)(Connection *conn); } ConnOps; extern int ts_connection_register(ConnectionType type, ConnOps *ops); ================================================ FILE: src/net/conn_plain.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <unistd.h> #include <postgres.h> #include <sys/socket.h> #include <sys/time.h> #include "compat/compat.h" #include "conn_internal.h" #include "conn_plain.h" #include "port.h" #define DEFAULT_TIMEOUT_MSEC 3000 #define MAX_PORT 65535 static void set_error(int err) { #ifdef WIN32 WSASetLastError(err); #else errno = err; #endif } static int get_error(void) { #ifdef WIN32 return WSAGetLastError(); #else return errno; #endif } /* Create socket and connect */ int ts_plain_connect(Connection *conn, const char *host, const char *servname, int port) { char strport[6]; struct addrinfo *ainfo, hints = { .ai_family = AF_UNSPEC, .ai_socktype = SOCK_STREAM, }; int ret; if (NULL == servname && (port <= 0 || port > MAX_PORT)) { set_error(EINVAL); return -1; } /* Explicit port given. Use it instead of servname */ if (port > 0 && port <= MAX_PORT) { snprintf(strport, sizeof(strport), "%d", port); servname = strport; hints.ai_flags = AI_NUMERICSERV; } /* Lookup the endpoint ip address */ ret = getaddrinfo(host, servname, &hints, &ainfo); if (ret != 0) { ret = SOCKET_ERROR; #ifdef WIN32 WSASetLastError(WSAHOST_NOT_FOUND); #else /* * The closest match for a name resolution error. Strictly, this error * should not be used here, but to fix we need to support using * gai_strerror() */ errno = EADDRNOTAVAIL; #endif goto out; } #ifdef WIN32 /* * PostgreSQL redefines the socket() call on Windows and creates a * non-blocking socket by default. We avoid this by calling WSASocket * directly. */ conn->sock = WSASocket(ainfo->ai_family, ainfo->ai_socktype, ainfo->ai_protocol, NULL, 0, WSA_FLAG_OVERLAPPED); if (conn->sock == INVALID_SOCKET) ret = SOCKET_ERROR; #else ret = conn->sock = socket(ainfo->ai_family, ainfo->ai_socktype, ainfo->ai_protocol); #endif if (IS_SOCKET_ERROR(ret)) goto out_addrinfo; /* * Set send / recv timeout so that write and read don't block forever. Set * separately so that one of the actions failing doesn't block the other. */ if (ts_plain_set_timeout(conn, DEFAULT_TIMEOUT_MSEC) < 0) { ret = SOCKET_ERROR; goto out_addrinfo; } #ifdef WIN32 ret = WSAConnect(conn->sock, ainfo->ai_addr, ainfo->ai_addrlen, NULL, NULL, NULL, NULL); #else /* connect the socket */ ret = connect(conn->sock, ainfo->ai_addr, ainfo->ai_addrlen); #endif out_addrinfo: freeaddrinfo(ainfo); out: if (IS_SOCKET_ERROR(ret)) { conn->err = ret; return -1; } return 0; } static ssize_t plain_write(Connection *conn, const char *buf, size_t writelen) { ssize_t ret; #ifdef WIN32 DWORD b; WSABUF wbuf = { .len = writelen, .buf = (char *) buf, }; conn->err = WSASend(conn->sock, &wbuf, 1, &b, 0, NULL, NULL); if (IS_SOCKET_ERROR(conn->err)) ret = -1; else ret = b; #else ret = send(conn->sock, buf, writelen, 0); if (ret < 0) conn->err = ret; #endif return ret; } static ssize_t plain_read(Connection *conn, char *buf, size_t buflen) { ssize_t ret; #ifdef WIN32 DWORD b, flags = 0; WSABUF wbuf = { .len = buflen, .buf = buf, }; conn->err = WSARecv(conn->sock, &wbuf, 1, &b, &flags, NULL, NULL); if (IS_SOCKET_ERROR(conn->err)) ret = -1; else ret = b; #else ret = recv(conn->sock, buf, buflen, 0); if (ret < 0) conn->err = ret; #endif return ret; } void ts_plain_close(Connection *conn) { #ifdef WIN32 closesocket(conn->sock); #else close(conn->sock); #endif } int ts_plain_set_timeout(Connection *conn, unsigned long millis) { #ifdef WIN32 /* Timeout is in milliseconds on Windows */ DWORD timeout = millis; int optlen = sizeof(DWORD); #else /* we deliberately use a long constant here instead of a fixed width one because tv_sec is * declared as a long */ struct timeval timeout = { .tv_sec = millis / 1000L, .tv_usec = (millis % 1000L) * 1000L, }; int optlen = sizeof(struct timeval); #endif /* * Set send / recv timeout so that write and read don't block forever. Set * separately so that one of the actions failing doesn't block the other. */ conn->err = setsockopt(conn->sock, SOL_SOCKET, SO_RCVTIMEO, (const char *) &timeout, optlen); if (conn->err != 0) return -1; conn->err = setsockopt(conn->sock, SOL_SOCKET, SO_SNDTIMEO, (const char *) &timeout, optlen); if (conn->err != 0) return -1; return 0; } const char * ts_plain_errmsg(Connection *conn) { const char *errmsg = "no connection error"; if (IS_SOCKET_ERROR(conn->err)) errmsg = strerror(get_error()); conn->err = 0; return errmsg; } static ConnOps plain_ops = { .size = sizeof(Connection), .init = NULL, .connect = ts_plain_connect, .close = ts_plain_close, .write = plain_write, .read = plain_read, .errmsg = ts_plain_errmsg, }; extern void _conn_plain_init(void); extern void _conn_plain_fini(void); void _conn_plain_init(void) { /* * WSAStartup is required on Windows before using the Winsock API. * However, PostgreSQL already handles this for us, so it is disabled here * by default. Set WSA_STARTUP_ENABLED to perform this initialization * anyway. */ #if defined(WIN32) && defined(WSA_STARTUP_ENABLED) WSADATA wsadata; int res; res = WSAStartup(MAKEWORD(2, 2), &wsadata); if (res != 0) { elog(ERROR, "WSAStartup failed: %d", res); return; } #endif ts_connection_register(CONNECTION_PLAIN, &plain_ops); } void _conn_plain_fini(void) { #if defined(WIN32) && defined(WSA_STARTUP_ENABLED) int ret = WSACleanup(); if (ret != 0) elog(WARNING, "WSACleanup failed"); #endif } ================================================ FILE: src/net/conn_plain.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once typedef struct Connection Connection; #ifdef WIN32 #define IS_SOCKET_ERROR(err) (err == SOCKET_ERROR) #else #define SOCKET_ERROR -1 #define IS_SOCKET_ERROR(err) (err < 0) #endif extern int ts_plain_connect(Connection *conn, const char *host, const char *servname, int port); extern void ts_plain_close(Connection *conn); extern int ts_plain_set_timeout(Connection *conn, unsigned long millis); extern const char *ts_plain_errmsg(Connection *conn); ================================================ FILE: src/net/conn_ssl.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <openssl/err.h> #include <openssl/ssl.h> #include "conn_internal.h" #include "conn_plain.h" typedef struct SSLConnection { Connection conn; SSL_CTX *ssl_ctx; SSL *ssl; unsigned long errcode; } SSLConnection; static void ssl_set_error(SSLConnection *conn, int err) { conn->errcode = ERR_get_error(); conn->conn.err = err; } static SSL_CTX * ssl_ctx_create(void) { SSL_CTX *ctx; int options; #if (OPENSSL_VERSION_NUMBER >= 0x1010000fL) /* OpenSSL >= v1.1 */ ctx = SSL_CTX_new(TLS_method()); options = SSL_OP_NO_SSLv3 | SSL_OP_NO_TLSv1 | SSL_OP_NO_TLSv1_1; #elif (OPENSSL_VERSION_NUMBER >= 0x1000000fL) /* OpenSSL >= v1.0 */ ctx = SSL_CTX_new(SSLv23_method()); options = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_TLSv1 | SSL_OP_NO_TLSv1_1; #else #error "Unsupported OpenSSL version" #endif /* * Because we have a blocking socket, we don't want to be bothered with * retries. */ if (NULL != ctx) { SSL_CTX_set_options(ctx, options); SSL_CTX_set_mode(ctx, SSL_MODE_AUTO_RETRY); } return ctx; } static int ssl_setup(SSLConnection *conn, const char *host) { int ret; conn->ssl_ctx = ssl_ctx_create(); if (NULL == conn->ssl_ctx) { ssl_set_error(conn, -1); return -1; } ERR_clear_error(); conn->ssl = SSL_new(conn->ssl_ctx); if (conn->ssl == NULL) { ssl_set_error(conn, -1); return -1; } ERR_clear_error(); ret = SSL_set_fd(conn->ssl, conn->conn.sock); if (ret == 0) { ssl_set_error(conn, -1); return -1; } /* * Tell the server during the handshake which hostname we are attempting * to connect to in case the server supports multiple hosts. */ if (!SSL_set_tlsext_host_name(conn->ssl, host)) { ssl_set_error(conn, -1); return -1; } ret = SSL_connect(conn->ssl); if (ret <= 0) { ssl_set_error(conn, ret); ret = -1; } return ret; } static int ssl_connect(Connection *conn, const char *host, const char *servname, int port) { int ret; /* First do the base connection setup */ ret = ts_plain_connect(conn, host, servname, port); if (ret < 0) return -1; return ssl_setup((SSLConnection *) conn, host); } static ssize_t ssl_write(Connection *conn, const char *buf, size_t writelen) { SSLConnection *sslconn = (SSLConnection *) conn; int ret = SSL_write(sslconn->ssl, buf, writelen); if (ret < 0) ssl_set_error(sslconn, ret); return ret; } static ssize_t ssl_read(Connection *conn, char *buf, size_t buflen) { SSLConnection *sslconn = (SSLConnection *) conn; int ret = SSL_read(sslconn->ssl, buf, buflen); if (ret < 0) ssl_set_error(sslconn, ret); return ret; } static void ssl_close(Connection *conn) { SSLConnection *sslconn = (SSLConnection *) conn; if (sslconn->ssl != NULL) { SSL_free(sslconn->ssl); sslconn->ssl = NULL; } if (sslconn->ssl_ctx != NULL) { SSL_CTX_free(sslconn->ssl_ctx); sslconn->ssl_ctx = NULL; } ts_plain_close(conn); } static const char * ssl_errmsg(Connection *conn) { SSLConnection *sslconn = (SSLConnection *) conn; const char *reason; static char errbuf[32]; int err = conn->err; unsigned long ecode = sslconn->errcode; /* Clear errors */ conn->err = 0; sslconn->errcode = 0; if (NULL != sslconn->ssl) { int sslerr = SSL_get_error(sslconn->ssl, err); switch (sslerr) { case SSL_ERROR_NONE: case SSL_ERROR_SSL: /* ecode should be set and handled below */ break; case SSL_ERROR_ZERO_RETURN: return "SSL error zero return"; case SSL_ERROR_WANT_READ: return "SSL error want read"; case SSL_ERROR_WANT_WRITE: return "SSL error want write"; case SSL_ERROR_WANT_CONNECT: return "SSL error want connect"; case SSL_ERROR_WANT_ACCEPT: return "SSL error want accept"; case SSL_ERROR_WANT_X509_LOOKUP: return "SSL error want X509 lookup"; case SSL_ERROR_SYSCALL: if (ecode == 0) { if (err == 0) return "EOF in SSL operation"; else if (IS_SOCKET_ERROR(err)) { /* reset error for plan_errmsg() */ conn->err = err; return ts_plain_errmsg(conn); } else return "unknown SSL syscall error"; } return "SSL error syscall"; default: break; } } if (ecode == 0) { /* Assume this was an error of the underlying socket */ if (IS_SOCKET_ERROR(err)) { /* reset error for plan_errmsg() */ conn->err = err; return ts_plain_errmsg(conn); } return "no SSL error"; } reason = ERR_reason_error_string(ecode); if (NULL != reason) return reason; snprintf(errbuf, sizeof(errbuf), "SSL error code %lu", ecode); return errbuf; } static ConnOps ssl_ops = { .size = sizeof(SSLConnection), .init = NULL, .connect = ssl_connect, .close = ssl_close, .write = ssl_write, .read = ssl_read, .set_timeout = ts_plain_set_timeout, .errmsg = ssl_errmsg, }; extern void _conn_ssl_init(void); extern void _conn_ssl_fini(void); void _conn_ssl_init(void) { #if (OPENSSL_VERSION_NUMBER < 0x1010000fL) /* OpenSSL < 1.1.0 requires explicit initialization */ SSL_library_init(); /* Always returns 1 */ SSL_load_error_strings(); #endif ts_connection_register(CONNECTION_SSL, &ssl_ops); } void _conn_ssl_fini(void) { #if (OPENSSL_VERSION_NUMBER < 0x1010000fL) /* OpenSSL < 1.1.0 requires explicit cleanup */ ERR_free_strings(); #endif } ================================================ FILE: src/net/http.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include "conn.h" #include "http.h" static const char *http_error_strings[] = { [HTTP_ERROR_NONE] = "no HTTP error", [HTTP_ERROR_WRITE] = "HTTP connection write error", [HTTP_ERROR_READ] = "HTTP connection read error", [HTTP_ERROR_CONN_CLOSED] = "HTTP connection closed", [HTTP_ERROR_REQUEST_BUILD] = "could not build HTTP request", [HTTP_ERROR_RESPONSE_PARSE] = "could not parse HTTP response", [HTTP_ERROR_RESPONSE_INCOMPLETE] = "incomplete HTTP response", [HTTP_ERROR_INVALID_BUFFER_STATE] = "invalid HTTP buffer state", [HTTP_ERROR_UNKNOWN] = "unknown HTTP error", }; static const char *http_version_strings[] = { [HTTP_VERSION_10] = "HTTP/1.0", [HTTP_VERSION_11] = "HTTP/1.1", [HTTP_VERSION_INVALID] = "invalid HTTP version", }; const char * ts_http_strerror(HttpError http_errno) { return http_error_strings[http_errno]; } HttpVersion ts_http_version_from_string(const char *version) { int i; for (i = 0; i < HTTP_VERSION_INVALID; i++) if (pg_strcasecmp(http_version_strings[i], version) == 0) return i; return HTTP_VERSION_INVALID; } const char * ts_http_version_string(HttpVersion version) { return http_version_strings[version]; } /* * Send an HTTP request and receive the HTTP response on the given connection. * * Returns HTTP_ERROR_NONE (0) on success or a HTTP-specific error on failure. */ HttpError ts_http_send_and_recv(Connection *conn, HttpRequest *req, HttpResponseState *state) { const char *built_request; size_t request_len; off_t write_off = 0; HttpError err = HTTP_ERROR_NONE; int ret; built_request = ts_http_request_build(req, &request_len); if (NULL == built_request) return HTTP_ERROR_REQUEST_BUILD; while (request_len > 0) { ret = ts_connection_write(conn, built_request + write_off, request_len); if (ret < 0 || (size_t) ret > request_len) return HTTP_ERROR_WRITE; if (ret == 0) return HTTP_ERROR_CONN_CLOSED; write_off += ret; request_len -= ret; } while (err == HTTP_ERROR_NONE && !ts_http_response_state_is_done(state)) { ssize_t remaining = 0; char *buf = ts_http_response_state_next_buffer(state, &remaining); if (remaining < 0) err = HTTP_ERROR_INVALID_BUFFER_STATE; else if (remaining == 0) err = HTTP_ERROR_RESPONSE_INCOMPLETE; else { ssize_t bytes_read = ts_connection_read(conn, buf, remaining); if (bytes_read < 0) err = HTTP_ERROR_READ; /* Check for error or closed socket/EOF (ret == 0) */ else if (bytes_read == 0) err = HTTP_ERROR_CONN_CLOSED; else if (!ts_http_response_state_parse(state, bytes_read)) err = HTTP_ERROR_RESPONSE_PARSE; } } return err; } ================================================ FILE: src/net/http.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <utils/jsonb.h> #define HTTP_HOST "Host" #define HTTP_CONTENT_LENGTH "Content-Length" #define HTTP_CONTENT_TYPE "Content-Type" #define MAX_RAW_BUFFER_SIZE 4096 #define MAX_REQUEST_DATA_SIZE 2048 typedef struct HttpHeader { char *name; int name_len; char *value; int value_len; struct HttpHeader *next; } HttpHeader; /******* http_request.c *******/ /* We can add more methods later, but for now we do not need others */ typedef enum HttpRequestMethod { HTTP_GET, HTTP_POST, } HttpRequestMethod; typedef enum HttpVersion { HTTP_VERSION_10, HTTP_VERSION_11, HTTP_VERSION_INVALID, } HttpVersion; typedef enum HttpError { HTTP_ERROR_NONE = 0, HTTP_ERROR_WRITE, /* Connection write error, check errno */ HTTP_ERROR_READ, /* Connection read error, check errno */ HTTP_ERROR_CONN_CLOSED, HTTP_ERROR_REQUEST_BUILD, HTTP_ERROR_RESPONSE_PARSE, HTTP_ERROR_RESPONSE_INCOMPLETE, HTTP_ERROR_INVALID_BUFFER_STATE, HTTP_ERROR_UNKNOWN, /* Should always be last */ } HttpError; /* NOTE: HttpRequest* structs are all responsible */ /* for allocating and deallocating the char* */ typedef struct HttpRequest HttpRequest; typedef struct Connection Connection; extern HttpVersion ts_http_version_from_string(const char *version); extern const char *ts_http_version_string(HttpVersion version); extern void ts_http_request_init(HttpRequest *req, HttpRequestMethod method); extern HttpRequest *ts_http_request_create(HttpRequestMethod method); extern void ts_http_request_destroy(HttpRequest *req); /* Assume that uri is null-terminated */ extern void ts_http_request_set_uri(HttpRequest *req, const char *uri); extern void ts_http_request_set_version(HttpRequest *req, HttpVersion version); /* Assume that name and value are null-terminated */ extern void ts_http_request_set_header(HttpRequest *req, const char *name, const char *value); extern void ts_http_request_set_body_jsonb(HttpRequest *req, const Jsonb *json); /* Serialize the request into char *dst. Return the length of request in optional size pointer*/ extern const char *ts_http_request_build(HttpRequest *req, size_t *buf_size); /******* http_response.c *******/ typedef struct HttpResponseState HttpResponseState; extern void ts_http_response_state_init(HttpResponseState *state); extern HttpResponseState *ts_http_response_state_create(void); extern void ts_http_response_state_destroy(HttpResponseState *state); /* Accessor Functions */ extern bool ts_http_response_state_is_done(HttpResponseState *state); extern bool ts_http_response_state_valid_status(HttpResponseState *state); extern char *ts_http_response_state_next_buffer(HttpResponseState *state, ssize_t *bufsize); extern ssize_t ts_http_response_state_buffer_remaining(HttpResponseState *state); extern const char *ts_http_response_state_body_start(HttpResponseState *state); extern size_t ts_http_response_state_content_length(HttpResponseState *state); extern int ts_http_response_state_status_code(HttpResponseState *state); /* Returns false if encountered an error during parsing */ extern bool ts_http_response_state_parse(HttpResponseState *state, size_t bytes); extern const char *ts_http_strerror(HttpError http_errno); extern HttpError ts_http_send_and_recv(Connection *conn, HttpRequest *req, HttpResponseState *state); ================================================ FILE: src/net/http_request.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <lib/stringinfo.h> #include <utils/memutils.h> #include "http.h" #define SPACE ' ' #define COLON ':' #define CARRIAGE '\r' #define NEW_LINE '\n' /* So that http_response.c can find this function */ HttpHeader *ts_http_header_create(const char *name, size_t name_len, const char *value, size_t value_len, HttpHeader *next); HttpHeader * ts_http_header_create(const char *name, size_t name_len, const char *value, size_t value_len, HttpHeader *next) { HttpHeader *new_header = palloc(sizeof(HttpHeader)); memset(new_header, 0, sizeof(*new_header)); new_header->name = palloc(name_len + 1); if (name_len > 0) memcpy(new_header->name, name, name_len); new_header->name[name_len] = '\0'; new_header->name_len = name_len; new_header->value = palloc(value_len + 1); if (value_len > 0) memcpy(new_header->value, value, value_len); new_header->value[value_len] = '\0'; new_header->value_len = value_len; new_header->next = next; return new_header; } /* NOTE: The setter functions for HttpRequest should all */ /* ensure that every char * in this struct is null-terminated */ typedef struct HttpRequest { HttpRequestMethod method; char *uri; size_t uri_len; HttpVersion version; HttpHeader *headers; char *body; size_t body_len; MemoryContext context; } HttpRequest; static const char *http_method_strings[] = { [HTTP_GET] = "GET", [HTTP_POST] = "POST" }; #define METHOD_STRING(x) http_method_strings[x] #define VERSION_STRING(x) ts_http_version_string(x) /* appendBinaryStringInfo is UB if data is NULL. This function wraps it in a check that datalen > 0 */ static void appendOptionalBinaryStringInfo(StringInfo str, const char *data, int datalen) { if (datalen <= 0) return; Assert(data != NULL); appendBinaryStringInfo(str, data, datalen); } void ts_http_request_init(HttpRequest *req, HttpRequestMethod method) { req->method = method; } HttpRequest * ts_http_request_create(HttpRequestMethod method) { MemoryContext request_context = AllocSetContextCreate(CurrentMemoryContext, "Http Request", ALLOCSET_DEFAULT_SIZES); MemoryContext old = MemoryContextSwitchTo(request_context); HttpRequest *req = palloc0(sizeof(HttpRequest)); req->context = request_context; ts_http_request_init(req, method); MemoryContextSwitchTo(old); return req; } void ts_http_request_destroy(HttpRequest *req) { MemoryContextDelete(req->context); } void ts_http_request_set_uri(HttpRequest *req, const char *uri) { MemoryContext old = MemoryContextSwitchTo(req->context); int uri_len = strlen(uri); req->uri = palloc(uri_len + 1); memcpy(req->uri, uri, uri_len); req->uri[uri_len] = '\0'; req->uri_len = uri_len; MemoryContextSwitchTo(old); } void ts_http_request_set_version(HttpRequest *req, HttpVersion version) { req->version = version; } static void set_header(HttpRequest *req, const char *name, const char *value) { int name_len = strlen(name); int value_len = strlen(value); req->headers = ts_http_header_create(name, name_len, value, value_len, req->headers); } void ts_http_request_set_header(HttpRequest *req, const char *name, const char *value) { MemoryContext old = MemoryContextSwitchTo(req->context); set_header(req, name, value); MemoryContextSwitchTo(old); } void ts_http_request_set_body_jsonb(HttpRequest *req, const Jsonb *json) { MemoryContext old = MemoryContextSwitchTo(req->context); StringInfoData jtext; initStringInfo(&jtext); char content_length[10]; JsonbToCString(&jtext, (JsonbContainer *) &json->root, VARSIZE(json)); req->body = jtext.data; req->body_len = jtext.len; snprintf(content_length, sizeof(content_length), "%d", jtext.len); set_header(req, HTTP_CONTENT_TYPE, "application/json"); set_header(req, HTTP_CONTENT_LENGTH, content_length); MemoryContextSwitchTo(old); } static void http_request_serialize_method(HttpRequest *req, StringInfo buf) { const char *method = METHOD_STRING(req->method); appendStringInfoString(buf, method); } static void http_request_serialize_version(HttpRequest *req, StringInfo buf) { const char *version = VERSION_STRING(req->version); appendStringInfoString(buf, version); } static void http_request_serialize_uri(HttpRequest *req, StringInfo buf) { appendOptionalBinaryStringInfo(buf, req->uri, req->uri_len); } static void http_request_serialize_char(char to_serialize, StringInfo buf) { appendStringInfoChar(buf, to_serialize); } static void http_request_serialize_body(HttpRequest *req, StringInfo buf) { appendOptionalBinaryStringInfo(buf, req->body, req->body_len); } static void http_header_serialize(HttpHeader *header, StringInfo buf) { appendOptionalBinaryStringInfo(buf, header->name, header->name_len); http_request_serialize_char(COLON, buf); http_request_serialize_char(SPACE, buf); appendOptionalBinaryStringInfo(buf, header->value, header->value_len); } static int http_header_get_content_length(HttpHeader *header) { int content_length = -1; if (!strncmp(header->name, HTTP_CONTENT_LENGTH, header->name_len)) sscanf(header->value, "%d", &content_length); return content_length; } const char * ts_http_request_build(HttpRequest *req, size_t *buf_size) { /* serialize into this buf, which is allocated on caller's memory context */ StringInfoData buf; HttpHeader *cur_header; int content_length = 0; bool verified_content_length = false; initStringInfo(&buf); http_request_serialize_method(req, &buf); http_request_serialize_char(SPACE, &buf); http_request_serialize_uri(req, &buf); http_request_serialize_char(SPACE, &buf); http_request_serialize_version(req, &buf); http_request_serialize_char(CARRIAGE, &buf); http_request_serialize_char(NEW_LINE, &buf); cur_header = req->headers; while (cur_header != NULL) { content_length = http_header_get_content_length(cur_header); if (content_length != -1) { /* make sure it's equal to body_len */ if ((size_t) content_length != req->body_len) { return NULL; } else verified_content_length = true; } http_header_serialize(cur_header, &buf); http_request_serialize_char(CARRIAGE, &buf); http_request_serialize_char(NEW_LINE, &buf); cur_header = cur_header->next; } http_request_serialize_char(CARRIAGE, &buf); http_request_serialize_char(NEW_LINE, &buf); if (!verified_content_length) { /* Then there was no header field for Content-Length */ if (req->body_len != 0) { return NULL; } } http_request_serialize_body(req, &buf); /* Now everything lives in buf.data */ if (buf_size != NULL) *buf_size = buf.len; return buf.data; } ================================================ FILE: src/net/http_response.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <lib/stringinfo.h> #include <utils/memutils.h> #include "http.h" #define CARRIAGE_RETURN '\r' #define NEW_LINE '\n' #define SEP_CHAR ':' #define HTTP_VERSION_BUFFER_SIZE 128 extern HttpHeader *ts_http_header_create(const char *name, size_t name_len, const char *value, size_t value_len, HttpHeader *next); typedef enum HttpParseState { HTTP_STATE_STATUS, HTTP_STATE_INTERM, /* received a single \r */ HTTP_STATE_HEADER_NAME, /* received \r\n */ HTTP_STATE_HEADER_VALUE, HTTP_STATE_ALMOST_DONE, HTTP_STATE_BODY, HTTP_STATE_ERROR, HTTP_STATE_DONE, } HttpParseState; typedef struct HttpResponseState { MemoryContext context; char version[HTTP_VERSION_BUFFER_SIZE]; char raw_buffer[MAX_RAW_BUFFER_SIZE]; /* The next read should copy data into the buffer starting here */ off_t offset; off_t parse_offset; size_t cur_header_name_len; size_t cur_header_value_len; char *cur_header_name; char *cur_header_value; HttpHeader *headers; int status_code; size_t content_length; char *body_start; HttpParseState state; } HttpResponseState; void ts_http_response_state_init(HttpResponseState *state) { state->status_code = -1; state->state = HTTP_STATE_STATUS; } HttpResponseState * ts_http_response_state_create() { MemoryContext context = AllocSetContextCreate(CurrentMemoryContext, "Http Response", ALLOCSET_DEFAULT_SIZES); MemoryContext old = MemoryContextSwitchTo(context); HttpResponseState *ret = palloc(sizeof(HttpResponseState)); memset(ret, 0, sizeof(*ret)); ret->context = context; ts_http_response_state_init(ret); MemoryContextSwitchTo(old); return ret; } void ts_http_response_state_destroy(HttpResponseState *state) { MemoryContextDelete(state->context); } bool ts_http_response_state_valid_status(HttpResponseState *state) { /* If the status code hasn't been parsed yet, return */ if (state->status_code == -1) return true; /* If it's a bad status code, then bad! */ if (state->status_code / 100 == 2) return true; return false; } bool ts_http_response_state_is_done(HttpResponseState *state) { return (state->state == HTTP_STATE_DONE); } /* * Return the remaining buffer space. * * Returns 0 or a positive number, or -1 for invalid state. * */ ssize_t ts_http_response_state_buffer_remaining(HttpResponseState *state) { Assert(state->offset <= MAX_RAW_BUFFER_SIZE); return MAX_RAW_BUFFER_SIZE - state->offset; } /* * Return a pointer to the next buffer to write to. * * Optionally, return the buffer size via the bufsize parameter. */ char * ts_http_response_state_next_buffer(HttpResponseState *state, ssize_t *bufsize) { Assert(state->offset <= MAX_RAW_BUFFER_SIZE); if (NULL != bufsize) *bufsize = ts_http_response_state_buffer_remaining(state); /* * This should not happen, be we return NULL in this case and let caller * deal with it */ if (state->offset > MAX_RAW_BUFFER_SIZE) return NULL; return state->raw_buffer + state->offset; } const char * ts_http_response_state_body_start(HttpResponseState *state) { return state->body_start; } int ts_http_response_state_status_code(HttpResponseState *state) { return state->status_code; } size_t ts_http_response_state_content_length(HttpResponseState *state) { return state->content_length; } static bool http_parse_version(HttpResponseState *state) { return ts_http_version_from_string(state->version) != HTTP_VERSION_INVALID; } static void http_parse_status(HttpResponseState *state, const char next) { char *raw_buf = palloc(state->parse_offset + 1); switch (next) { case CARRIAGE_RETURN: /* * Then we are at the end of status and can use sscanf * * Need a second %s inside the sscanf so that we make sure to get * all of the digits of the status code */ memcpy(raw_buf, state->raw_buffer, state->parse_offset); raw_buf[state->parse_offset] = '\0'; state->state = HTTP_STATE_ERROR; memset(state->version, '\0', sizeof(state->version)); if (sscanf(raw_buf, "%127s%*[ ]%d%*[ ]%*s", state->version, &state->status_code) == 2) { if (http_parse_version(state)) state->state = HTTP_STATE_INTERM; else state->state = HTTP_STATE_ERROR; } break; case NEW_LINE: state->state = HTTP_STATE_ERROR; break; default: /* Don't try to parse Status line until we see '\r' */ break; } pfree(raw_buf); } static void http_response_state_add_header(HttpResponseState *state, const char *name, size_t name_len, const char *value, size_t value_len) { MemoryContext old = MemoryContextSwitchTo(state->context); HttpHeader *new_header = ts_http_header_create(name, name_len, value, value_len, state->headers); state->headers = new_header; MemoryContextSwitchTo(old); } static void http_parse_interm(HttpResponseState *state, const char next) { int temp_length; switch (next) { case NEW_LINE: state->state = HTTP_STATE_HEADER_NAME; /* Store another header */ http_response_state_add_header(state, state->cur_header_name, state->cur_header_name_len, state->cur_header_value, state->cur_header_value_len); /* Check if the line we just read is Content-Length */ if (state->cur_header_name != NULL && strncmp(HTTP_CONTENT_LENGTH, state->cur_header_name, state->cur_header_name_len) == 0) { if (sscanf(state->cur_header_value, "%d", &temp_length) == 1) { state->content_length = temp_length; } else { state->state = HTTP_STATE_ERROR; break; } } state->cur_header_name_len = 0; state->cur_header_value_len = 0; break; default: state->state = HTTP_STATE_ERROR; break; } } static void http_parse_header_name(HttpResponseState *state, const char next) { switch (next) { case SEP_CHAR: state->state = HTTP_STATE_HEADER_VALUE; state->cur_header_value = state->raw_buffer + state->parse_offset + 1; break; case CARRIAGE_RETURN: if (state->cur_header_name_len == 0) { state->state = HTTP_STATE_ALMOST_DONE; break; } else { /* * I'm guessing getting a carriage return in the middle of * field */ /* name is bad... */ state->state = HTTP_STATE_ERROR; break; } default: /* Header names are only alphabetic chars */ if (('a' <= next && next <= 'z') || ('A' <= next && next <= 'Z') || next == '-') { /* Good, then the next call will save this char */ state->cur_header_name_len++; break; } state->state = HTTP_STATE_ERROR; break; } } /* We do not customize to header_name. Assume all non \r or \n chars are allowed. */ static void http_parse_header_value(HttpResponseState *state, const char next) { /* Allow everything except... \r, \n */ switch (next) { case CARRIAGE_RETURN: state->state = HTTP_STATE_INTERM; break; case NEW_LINE: /* \n is not allowed */ state->state = HTTP_STATE_ERROR; break; default: state->cur_header_value_len++; break; } } static void http_parse_almost_done(HttpResponseState *state, const char next) { /* Don't do anything, this is intermediate state */ switch (next) { case NEW_LINE: state->state = HTTP_STATE_BODY; state->body_start = state->raw_buffer + state->parse_offset + 1; /* Special case if there is no body */ if (state->content_length == 0) state->state = HTTP_STATE_DONE; break; default: state->state = HTTP_STATE_ERROR; break; } } bool ts_http_response_state_parse(HttpResponseState *state, size_t bytes) { state->offset += bytes; if (state->offset > MAX_RAW_BUFFER_SIZE) state->offset = MAX_RAW_BUFFER_SIZE; /* Each state function will do the state AND transition */ while (state->parse_offset < state->offset) { char next = state->raw_buffer[state->parse_offset]; switch (state->state) { case HTTP_STATE_STATUS: http_parse_status(state, next); state->parse_offset++; break; case HTTP_STATE_INTERM: http_parse_interm(state, next); state->parse_offset++; state->cur_header_name = state->raw_buffer + state->parse_offset; break; case HTTP_STATE_HEADER_NAME: http_parse_header_name(state, next); state->parse_offset++; break; case HTTP_STATE_HEADER_VALUE: http_parse_header_value(state, next); state->parse_offset++; break; case HTTP_STATE_ALMOST_DONE: http_parse_almost_done(state, next); state->parse_offset++; break; case HTTP_STATE_BODY: /* Stay here until we have read content_length */ if ((state->body_start + state->content_length) <= (state->raw_buffer + state->offset)) { /* Then we are done */ state->state = HTTP_STATE_DONE; return true; } state->parse_offset++; break; case HTTP_STATE_ERROR: return false; case HTTP_STATE_DONE: return true; } } return true; } ================================================ FILE: src/nodes/CMakeLists.txt ================================================ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/modify_hypertable.c ${CMAKE_CURRENT_SOURCE_DIR}/modify_hypertable_exec.c) target_sources(${PROJECT_NAME} PRIVATE ${SOURCES}) add_subdirectory(chunk_append) add_subdirectory(constraint_aware_append) ================================================ FILE: src/nodes/chunk_append/CMakeLists.txt ================================================ # Add all *.c to sources in upperlevel directory set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/chunk_append.c ${CMAKE_CURRENT_SOURCE_DIR}/exec.c ${CMAKE_CURRENT_SOURCE_DIR}/planner.c ${CMAKE_CURRENT_SOURCE_DIR}/transform.c) target_sources(${PROJECT_NAME} PRIVATE ${SOURCES}) ================================================ FILE: src/nodes/chunk_append/chunk_append.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <nodes/nodeFuncs.h> #include <optimizer/optimizer.h> #include <optimizer/pathnode.h> #include <optimizer/paths.h> #include <optimizer/tlist.h> #include <utils/builtins.h> #include <utils/typcache.h> #include "func_cache.h" #include "guc.h" #include "nodes/chunk_append/chunk_append.h" #include "planner/planner.h" static Var *find_equality_join_var(Var *sort_var, Index ht_relid, List *join_conditions); static CustomPathMethods chunk_append_path_methods = { .CustomName = "ChunkAppend", .PlanCustomPath = ts_chunk_append_plan_create, }; bool ts_is_chunk_append_path(Path *path) { return IsA(path, CustomPath) && castNode(CustomPath, path)->methods == &chunk_append_path_methods; } static bool has_joins(FromExpr *jointree) { return list_length(jointree->fromlist) != 1 || !IsA(linitial(jointree->fromlist), RangeTblRef); } /* * Create the appropriate subpath for the outer MergeAppend * node depending on the number of paths in the current group: * Combine two or more group members into a mergeAppend node * or append a single member as is. * Members of the same group contain data of the same chunk, * so they are combined into a MergeAppend node corresponding * to that single chunk. */ static void create_group_subpath(PlannerInfo *root, RelOptInfo *rel, List *group, List *pathkeys, Relids required_outer, List *partitioned_rels, List **nested_children) { if (list_length(group) > 1) { MergeAppendPath *append = create_merge_append_path(root, rel, group, pathkeys, required_outer); *nested_children = lappend(*nested_children, append); } else { /* If group only has 1 member we can add it directly */ *nested_children = lappend(*nested_children, linitial(group)); } } ChunkAppendPath * ts_chunk_append_path_copy(ChunkAppendPath *ca, List *subpaths, PathTarget *pathtarget) { ListCell *lc; double total_cost = 0, rows = 0; ChunkAppendPath *new = palloc(sizeof(ChunkAppendPath)); memcpy(new, ca, sizeof(ChunkAppendPath)); new->cpath.custom_paths = subpaths; foreach (lc, subpaths) { Path *child = lfirst(lc); total_cost += child->total_cost; rows += child->rows; } new->cpath.path.total_cost = total_cost; new->cpath.path.rows = rows; new->cpath.path.pathtarget = copy_pathtarget(pathtarget); return new; } Path * ts_chunk_append_path_create(PlannerInfo *root, RelOptInfo *rel, Hypertable *ht, Path *subpath, bool parallel_aware, bool ordered, List *nested_oids) { ChunkAppendPath *path; ListCell *lc; double rows = 0.0; Cost total_cost = 0.0; List *children = NIL; path = (ChunkAppendPath *) newNode(sizeof(ChunkAppendPath), T_CustomPath); path->cpath.path.pathtype = T_CustomScan; path->cpath.path.parent = rel; path->cpath.path.pathtarget = rel->reltarget; path->cpath.path.param_info = subpath->param_info; /* * We keep the pathkeys from the original path here because * the original path was either a MergeAppendPath and this * will become an ordered append or the original path is an * AppendPath and since we do not reorder children the order * will be kept intact. For the AppendPath case with pathkeys * it was most likely an Append with only a single child. * We could skip the ChunkAppend path creation if there is * only a single child but we decided earlier that ChunkAppend * would be beneficial for this query so we treat it the same * as if it had multiple children. */ Assert(IsA(subpath, AppendPath) || IsA(subpath, MergeAppendPath)); path->cpath.path.pathkeys = subpath->pathkeys; path->cpath.path.parallel_aware = ts_guc_enable_parallel_chunk_append ? parallel_aware : false; path->cpath.path.parallel_safe = subpath->parallel_safe; path->cpath.path.parallel_workers = subpath->parallel_workers; /* * Set flags. We can set CUSTOMPATH_SUPPORT_BACKWARD_SCAN and * CUSTOMPATH_SUPPORT_MARK_RESTORE. The only interesting flag is the first * one (backward scan), but since we are not scanning a real relation we * need not indicate that we support backward scans. Lower-level index * scanning nodes will scan backward if necessary, so once tuples get to * this node they will be in a given order already. */ path->cpath.flags = 0; path->cpath.methods = &chunk_append_path_methods; /* * Figure out whether there's a hard limit on the number of rows that * query_planner's result subplan needs to return. Even if we know a * hard limit overall, it doesn't apply if the query has any * grouping/aggregation operations, or SRFs in the tlist. */ if (root->parse->groupClause || root->parse->groupingSets || root->parse->distinctClause || root->parse->hasAggs || root->parse->hasWindowFuncs || root->hasHavingQual || has_joins(root->parse->jointree) || root->limit_tuples > PG_INT32_MAX || root->parse->hasTargetSRFs || !pathkeys_contained_in(root->sort_pathkeys, subpath->pathkeys)) path->limit_tuples = -1; else path->limit_tuples = (int) root->limit_tuples; /* * check if we should do startup and runtime exclusion */ foreach (lc, rel->baserestrictinfo) { RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); /* * The external parameters (e.g. from parameterized prepared statements) * are constant during query run time, so we can use them for startup * exclusion. * The join parameters have multiple values, so they are only used for * runtime exclusion. */ if (contain_mutable_functions((Node *) rinfo->clause) || ts_contains_external_param((Node *) rinfo->clause)) { path->startup_exclusion = true; } if (ts_guc_enable_runtime_exclusion && ts_contains_join_param((Node *) rinfo->clause)) { ListCell *lc_var; /* We have two types of exclusion: * * Parent exclusion fires if the entire hypertable can be excluded. * This happens if doing things like joining against a parameter * value that is an empty array or NULL. It doesn't happen often, * but when it does, it speeds up the query immensely. It's also cheap * to check for this condition as you check this once per hypertable * at runtime. * * Child exclusion works by seeing if there is a contradiction between * the chunks constraints and the expression on parameter values. For example, * it can evaluate whether a time parameter from a subquery falls outside * the range of the chunk. It is more widely applicable than the parent * exclusion but is also more expensive to evaluate since you have to perform * the check on every chunk. Child exclusion can only apply if one of the quals * involves a partitioning column. * */ path->runtime_exclusion_parent = true; foreach (lc_var, pull_var_clause((Node *) rinfo->clause, 0)) { Var *var = lfirst(lc_var); /* * varattno 0 is whole row and varattno less than zero are * system columns so we skip those even though * ts_is_partitioning_column would return the correct * answer for those as well */ if ((Index) var->varno == rel->relid && var->varattno > 0 && ts_is_partitioning_column(ht, var->varattno)) { path->runtime_exclusion_children = true; break; } } } } /* * For parameterized paths (e.g., inner side of LATERAL joins), also check * ppi_clauses for runtime exclusion. Any clause in ppi_clauses references * outer relations by definition. */ if (ts_guc_enable_runtime_exclusion && subpath->param_info != NULL && subpath->param_info->ppi_clauses != NIL) { path->runtime_exclusion_parent = true; /* Check if any clause involves a partitioning column for child exclusion */ foreach (lc, subpath->param_info->ppi_clauses) { RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); ListCell *lc_var; foreach (lc_var, pull_var_clause((Node *) rinfo->clause, 0)) { Var *var = lfirst(lc_var); if ((Index) var->varno == rel->relid && var->varattno > 0 && ts_is_partitioning_column(ht, var->varattno)) { path->runtime_exclusion_children = true; break; } } if (path->runtime_exclusion_children) break; } } /* * Our strategy is to use child exclusion if possible (if a partitioning * column is used) and fall back to parent exclusion if we can't use child * exclusion. Please note: there is no point to using both child and parent * exclusion at the same time since child exclusion would always exclude * the same chunks that parent exclusion would. */ if (path->runtime_exclusion_parent && path->runtime_exclusion_children) path->runtime_exclusion_parent = false; /* * Make sure our subpath is either an Append or MergeAppend node */ switch (nodeTag(subpath)) { case T_AppendPath: { AppendPath *append = castNode(AppendPath, subpath); if (append->path.parallel_aware && append->first_partial_path > 0) path->first_partial_path = append->first_partial_path; children = append->subpaths; break; } case T_MergeAppendPath: /* * check if ordered append is applicable, only assert ordered here * checked properly in ts_ordered_append_should_optimize */ Assert(ordered); /* * we only push down LIMIT for ordered append */ path->pushdown_limit = true; children = castNode(MergeAppendPath, subpath)->subpaths; break; default: elog(ERROR, "invalid child of chunk append: %s", ts_get_node_name((Node *) subpath)); break; } if (!ordered) { path->cpath.custom_paths = children; } else if (ht->space->num_dimensions == 1) { List *nested_children = NIL; /* * Convert the sort nodes that refer to the same chunk into a single * mergeAppend node to combine compressed and uncompressed chunk output. * * NB: We assume that the sort nodes referring the same chunk appear * one after the other and so we iterate through the children examining * consecutive pairs. Is it possible that this assumption is wrong? */ List *group = NIL; Oid relid = InvalidOid; foreach (lc, children) { Path *child = (Path *) lfirst(lc); /* Check if this is in new group */ if (child->parent->relid != relid) { /* if previous group had members, process them */ if (group) { create_group_subpath(root, rel, group, path->cpath.path.pathkeys, PATH_REQ_OUTER(subpath), NIL, &nested_children); group = NIL; } relid = child->parent->relid; } /* Form the new group */ group = lappend(group, child); } if (group) { create_group_subpath(root, rel, group, path->cpath.path.pathkeys, PATH_REQ_OUTER(subpath), NIL, &nested_children); } path->cpath.custom_paths = nested_children; children = nested_children; } else { /* * For space partitioning we need to change the shape of the plan * into a MergeAppend for each time slice with all space partitions below * The final plan for space partitioning will look like this: * * Custom Scan (ChunkAppend) * Hypertable: space * -> Merge Append * Sort Key: _hyper_9_56_chunk."time" * -> Index Scan * -> Index Scan * -> Index Scan * -> Merge Append * Sort Key: _hyper_9_55_chunk."time" * -> Index Scan * -> Index Scan * -> Index Scan * * We do not check sort order at this stage but injecting of Sort * nodes happens when the plan is created instead. */ ListCell *flat = list_head(children); List *nested_children = NIL; bool has_scan_childs = false; foreach (lc, nested_oids) { ListCell *lc_oid; List *current_oids = lfirst(lc); List *merge_childs = NIL; MergeAppendPath *append; /* * For each lc_oid, there will be 0, 1, or 2 matches in flat_list: 0 matches * if child was pruned, 1 match if the chunk is uncompressed or fully compressed, * 2 matches if the chunk is partially compressed. * If there are 2 matches they will also be consecutive (see assumption above) */ foreach (lc_oid, current_oids) { bool is_not_pruned = true; #ifdef USE_ASSERT_CHECKING int nmatches = 0; #endif /* Before entering the "DO" loop, check for a valid path entry */ if (flat == NULL) break; do { Path *child = (Path *) lfirst(flat); Oid parent_relid = child->parent->relid; is_not_pruned = lfirst_oid(lc_oid) == root->simple_rte_array[parent_relid]->relid; /* postgres may have pruned away some children already */ if (is_not_pruned) { #ifdef USE_ASSERT_CHECKING nmatches++; #endif merge_childs = lappend(merge_childs, child); flat = lnext(children, flat); if (flat == NULL) break; } /* if current one matched then need to check next one for match */ } while (is_not_pruned); #ifdef USE_ASSERT_CHECKING Assert(nmatches <= 2); #endif } if (list_length(merge_childs) > 1) { append = create_merge_append_path(root, rel, merge_childs, path->cpath.path.pathkeys, PATH_REQ_OUTER(subpath)); nested_children = lappend(nested_children, append); } else if (list_length(merge_childs) == 1) { has_scan_childs = true; nested_children = lappend(nested_children, linitial(merge_childs)); } } Assert(flat == NULL); /* * if we do not have scans as direct children of this * node we disable startup and runtime exclusion * in this node */ if (!has_scan_childs) { path->startup_exclusion = false; path->runtime_exclusion_parent = false; path->runtime_exclusion_children = false; } path->cpath.custom_paths = nested_children; } foreach (lc, path->cpath.custom_paths) { Path *child = lfirst(lc); /* * If there is a LIMIT clause we only include as many chunks as * planner thinks are needed to satisfy LIMIT clause. * We do this to prevent planner choosing parallel plan which might * otherwise look preferable cost wise. */ if (!path->pushdown_limit || path->limit_tuples == -1 || rows < path->limit_tuples) { total_cost += child->total_cost; rows += child->rows; } } path->cpath.path.rows = rows; path->cpath.path.total_cost = total_cost; if (path->cpath.custom_paths != NIL) path->cpath.path.startup_cost = ((Path *) linitial(path->cpath.custom_paths))->startup_cost; return &path->cpath.path; } /* * Check if conditions for doing ordered append optimization are fulfilled */ bool ts_ordered_append_should_optimize(PlannerInfo *root, RelOptInfo *rel, Hypertable *ht, int *order_attno, bool *reverse) { SortGroupClause *sort = linitial(root->parse->sortClause); TargetEntry *tle = get_sortgroupref_tle(sort->tleSortGroupRef, root->parse->targetList); RangeTblEntry *rte = root->simple_rte_array[rel->relid]; TypeCacheEntry *tce; char *column; Index ht_relid = rel->relid; Index sort_relid; Var *ht_var; Var *sort_var; /* these are checked in caller so we only Assert */ Assert(ts_guc_enable_optimizations && ts_guc_enable_ordered_append && ts_guc_enable_chunk_append); /* * only do this optimization for queries with an ORDER BY clause, * caller checked this, so only asserting */ Assert(root->parse->sortClause != NIL); if (IsA(tle->expr, Var)) { /* direct column reference */ sort_var = castNode(Var, tle->expr); } else if (IsA(tle->expr, FuncExpr) && list_length(root->parse->sortClause) == 1) { /* * check for bucketing functions * * If ORDER BY clause only has 1 expression and the expression is a * bucketing function we can still do Ordered Append, the 1 expression * limit could only be safely removed if we ensure chunk boundaries * are not crossed. * * The following example demonstrates this requirement: * * Chunk 1 has (time, device_id) * 0 1 * 0 2 * * Chunk 2 has (time, device_id) * 10 1 * 10 2 * * The ORDER BY clause is time_bucket(100,time), device_id * The result when transforming to an ordered append would be the following: * (time_bucket(100, time), device_id) * 0 1 * 0 2 * 0 1 * 0 2 * * The order of the device_ids is wrong so we cannot safely remove the MergeAppend * unless we eliminate the possibility that a bucket spans multiple chunks. */ FuncInfo *info = ts_func_cache_get_bucketing_func(castNode(FuncExpr, tle->expr)->funcid); Expr *transformed; if (!info || !info->sort_transform) return false; transformed = info->sort_transform(castNode(FuncExpr, tle->expr)); if (!IsA(transformed, Var)) return false; sort_var = castNode(Var, transformed); } else return false; /* ordered append won't work for system columns / whole row orderings */ if (sort_var->varattno <= 0) return false; sort_relid = sort_var->varno; tce = lookup_type_cache(sort_var->vartype, TYPECACHE_EQ_OPR | TYPECACHE_LT_OPR | TYPECACHE_GT_OPR); /* check sort operation is either less than or greater than */ if (sort->sortop != tce->lt_opr && sort->sortop != tce->gt_opr) return false; /* * check the ORDER BY column actually belongs to our hypertable */ if (sort_relid == ht_relid) { /* ORDER BY column belongs to our hypertable */ ht_var = sort_var; } else { /* * If the ORDER BY does not match our hypertable, but we are joining * against another hypertable on the time column, then doing an ordered * append here is still beneficial, because we can skip the sort * step for the MergeJoin. */ Bitmapset *outer_relids = root->simple_rel_array[sort_relid]->relids; Bitmapset *inner_relids = root->simple_rel_array[ht_relid]->relids; List *join_conditions = generate_join_implied_equalities(root, bms_union(outer_relids, inner_relids), outer_relids, rel #if PG16_GE , /* sjinfo = */ NULL #endif ); /* * The outer join clauses don't form ECs and stay in joininfo, and we * want to check them too. * There are also non-equality join conditions in joininfo, but they're * not relevant for MergeJoin anyway and will be skipped. */ join_conditions = list_concat(join_conditions, rel->joininfo); if (join_conditions == NIL) return false; ht_var = find_equality_join_var(sort_var, ht_relid, join_conditions); if (ht_var == NULL) return false; } /* Check hypertable column is the first dimension of the hypertable */ column = strVal(list_nth(rte->eref->colnames, AttrNumberGetAttrOffset(ht_var->varattno))); if (namestrcmp(&ht->space->dimensions[0].fd.column_name, column) != 0) return false; Assert(order_attno != NULL && reverse != NULL); *order_attno = ht_var->varattno; *reverse = sort->sortop == tce->lt_opr ? false : true; return true; } /* * Find equality join between column referenced by sort_var and Relation * with relid ht_relid */ static Var * find_equality_join_var(Var *sort_var, Index ht_relid, List *join_conditions) { ListCell *lc; Index sort_relid = sort_var->varno; foreach (lc, join_conditions) { RestrictInfo *ri = castNode(RestrictInfo, lfirst(lc)); /* * Only interested in join clauses here. */ if (!ri->can_join) { continue; } /* * The clause must be a mergejoinable equality operator. */ if (ri->mergeopfamilies == NIL) { continue; } OpExpr *op = castNode(OpExpr, ri->clause); if (!IsA(linitial(op->args), Var)) { continue; } Var *left = castNode(Var, linitial(op->args)); if (!IsA(lsecond(op->args), Var)) { continue; } Var *right = castNode(Var, lsecond(op->args)); /* Is this a join condition referencing our hypertable */ if (((Index) left->varno == sort_relid && (Index) right->varno == ht_relid && left->varattno == sort_var->varattno)) { return right; } if (((Index) left->varno == ht_relid && (Index) right->varno == sort_relid && right->varattno == sort_var->varattno)) { return left; } } return NULL; } ================================================ FILE: src/nodes/chunk_append/chunk_append.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/extensible.h> #include "hypertable.h" /* * Indexes into the settings list (first element of custom_private). */ typedef enum { CAS_StartupExclusion = 0, CAS_RuntimeExclusionParent = 1, CAS_RuntimeExclusionChildren = 2, CAS_Limit = 3, CAS_FirstPartialPath = 4, CAS_Count } ChunkAppendSettingsIndex; /* * Indexes into custom_private for ChunkAppend. */ typedef enum { CAP_Settings = 0, CAP_ChunkRIClauses = 1, CAP_RTIndexes = 2, CAP_SortOptions = 3, CAP_ParentClauses = 4, CAP_Count } ChunkAppendPrivateIndex; typedef struct ChunkAppendPath { CustomPath cpath; bool startup_exclusion; bool runtime_exclusion_parent; bool runtime_exclusion_children; bool pushdown_limit; int limit_tuples; int first_partial_path; } ChunkAppendPath; extern TSDLLEXPORT ChunkAppendPath *ts_chunk_append_path_copy(ChunkAppendPath *ca, List *subpaths, PathTarget *pathtarget); extern Path *ts_chunk_append_path_create(PlannerInfo *root, RelOptInfo *rel, Hypertable *ht, Path *subpath, bool parallel_aware, bool ordered, List *nested_oids); extern Plan *ts_chunk_append_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *path, List *tlist, List *clauses, List *custom_plans); extern Node *ts_chunk_append_state_create(CustomScan *cscan); extern bool ts_ordered_append_should_optimize(PlannerInfo *root, RelOptInfo *rel, Hypertable *ht, int *order_attno, bool *reverse); extern TSDLLEXPORT bool ts_is_chunk_append_path(Path *path); extern TSDLLEXPORT bool ts_is_chunk_append_plan(Plan *plan); extern Scan *ts_chunk_append_get_scan_plan(Plan *plan); void _chunk_append_init(void); extern TSDLLEXPORT List *ts_constify_restrictinfos(PlannerInfo *root, List *restrictinfos); extern TSDLLEXPORT List *ts_constify_restrictinfo_params(PlannerInfo *root, EState *state, List *restrictinfos); ================================================ FILE: src/nodes/chunk_append/exec.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_collation.h> #include <executor/executor.h> #include <executor/nodeSubplan.h> #include <fmgr.h> #include <miscadmin.h> #include <nodes/bitmapset.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <optimizer/cost.h> #include <optimizer/optimizer.h> #include <optimizer/plancat.h> #include <optimizer/prep.h> #include <optimizer/restrictinfo.h> #include <parser/parsetree.h> #include <rewrite/rewriteManip.h> #include <utils/builtins.h> #include <utils/memutils.h> #include <utils/ruleutils.h> #include <utils/typcache.h> #include <math.h> #include "compat/compat.h" #include "loader/lwlocks.h" #include "nodes/chunk_append/chunk_append.h" #include "planner/planner.h" #include "transform.h" #include "ts_catalog/chunk_column_stats.h" #if PG18_GE #include <commands/explain_format.h> #endif #define INVALID_SUBPLAN_INDEX (-1) #define NO_MATCHING_SUBPLANS (-2) typedef enum SubplanState { SUBPLAN_STATE_INCLUDED = 1 << 0, /* Used and not removed by startup exclusion */ SUBPLAN_STATE_FINISHED = 1 << 1, /* The subplan is finished */ } SubplanState; /* ParallelChunkAppendState is stored in shared memory to coordinate the parallel workers. * * subplan_state is accessed by two different indexes. This is done because a C struct can have only * one FLEXIBLE_ARRAY_MEMBER, two pieces of information must be stored per subplan in shared memory, * and computing a mapping between both indexes is avoided in the current implementation. * * The first index is the position of the subplan in initial_subplans. This index is used to * get/set the flag SUBPLAN_STATE_INCLUDED. * * The second index is the position of a subplan in filtered_subplans. This index is used to get/set * the flag SUBPLAN_STATE_FINISHED. */ typedef struct ParallelChunkAppendState { int next_plan; int filtered_first_partial_plan; uint32 subplan_state[FLEXIBLE_ARRAY_MEMBER]; /* See SubplanState */ } ParallelChunkAppendState; typedef struct ChunkAppendState { CustomScanState csstate; PlanState **subplanstates; MemoryContext exclusion_ctx; int num_subplans; int first_partial_plan; int filtered_first_partial_plan; int current; Oid ht_reloid; bool startup_exclusion; bool runtime_exclusion_parent; bool runtime_exclusion_children; bool runtime_initialized; uint32 limit; #ifdef USE_ASSERT_CHECKING bool init_done; #endif /* list of subplans after planning */ List *initial_subplans; /* list of constraints indexed like initial_subplans */ List *initial_constraints; /* list of restrictinfo clauses indexed like initial_subplans */ List *initial_ri_clauses; /* List of restrictinfo clauses on the parent hypertable */ List *initial_parent_clauses; /* list of subplans after startup exclusion */ List *filtered_subplans; /* list of relation constraints after startup exclusion */ List *filtered_constraints; /* list of restrictinfo clauses after startup exclusion */ List *filtered_ri_clauses; /* included subplans by startup exclusion */ Bitmapset *included_subplans_by_se; /* valid subplans for runtime exclusion */ Bitmapset *valid_subplans; Bitmapset *params; /* sort options if this append is ordered, only used for EXPLAIN */ List *sort_options; /* number of loops and exclusions for EXPLAIN */ int runtime_number_loops; int runtime_number_exclusions_parent; int runtime_number_exclusions_children; LWLock *lock; ParallelContext *pcxt; ParallelChunkAppendState *pstate; EState *estate; int eflags; void (*choose_next_subplan)(struct ChunkAppendState *); } ChunkAppendState; static TupleTableSlot *chunk_append_exec(CustomScanState *node); static void chunk_append_begin(CustomScanState *node, EState *estate, int eflags); static void chunk_append_end(CustomScanState *node); static void chunk_append_rescan(CustomScanState *node); static void chunk_append_explain(CustomScanState *node, List *ancestors, ExplainState *es); static Size chunk_append_estimate_dsm(CustomScanState *node, ParallelContext *pcxt); static void chunk_append_initialize_dsm(CustomScanState *node, ParallelContext *pcxt, void *coordinate); static void chunk_append_reinitialize_dsm(CustomScanState *node, ParallelContext *pcxt, void *coordinate); static void chunk_append_initialize_worker(CustomScanState *node, shm_toc *toc, void *coordinate); static CustomExecMethods chunk_append_state_methods = { .BeginCustomScan = chunk_append_begin, .ExecCustomScan = chunk_append_exec, .EndCustomScan = chunk_append_end, .ReScanCustomScan = chunk_append_rescan, .ExplainCustomScan = chunk_append_explain, .EstimateDSMCustomScan = chunk_append_estimate_dsm, .InitializeDSMCustomScan = chunk_append_initialize_dsm, .ReInitializeDSMCustomScan = chunk_append_reinitialize_dsm, .InitializeWorkerCustomScan = chunk_append_initialize_worker, }; static void choose_next_subplan_non_parallel(ChunkAppendState *state); static void choose_next_subplan_for_worker(ChunkAppendState *state); static bool can_exclude_chunk(List *constraints, List *baserestrictinfo); static void do_startup_exclusion(ChunkAppendState *state); static Node *constify_param_mutator(Node *node, void *context); static void initialize_constraints(ChunkAppendState *state, List *initial_rt_indexes); static LWLock *chunk_append_get_lock_pointer(void); static void show_sort_group_keys(ChunkAppendState *planstate, List *ancestors, ExplainState *es); static void show_sortorder_options(StringInfo buf, Node *sortexpr, Oid sortOperator, Oid collation, bool nullsFirst); static void perform_plan_init(ChunkAppendState *state, EState *estate, int eflags); Node * ts_chunk_append_state_create(CustomScan *cscan) { ChunkAppendState *state; Assert(list_length(cscan->custom_private) == CAP_Count); List *settings = list_nth(cscan->custom_private, CAP_Settings); Assert(list_length(settings) == CAS_Count); state = (ChunkAppendState *) newNode(sizeof(ChunkAppendState), T_CustomScanState); state->csstate.methods = &chunk_append_state_methods; state->initial_subplans = cscan->custom_plans; state->initial_ri_clauses = list_nth(cscan->custom_private, CAP_ChunkRIClauses); state->sort_options = list_nth(cscan->custom_private, CAP_SortOptions); state->initial_parent_clauses = list_nth(cscan->custom_private, CAP_ParentClauses); state->startup_exclusion = list_nth_int(settings, CAS_StartupExclusion); state->runtime_exclusion_parent = list_nth_int(settings, CAS_RuntimeExclusionParent); state->runtime_exclusion_children = list_nth_int(settings, CAS_RuntimeExclusionChildren); state->limit = list_nth_int(settings, CAS_Limit); state->first_partial_plan = list_nth_int(settings, CAS_FirstPartialPath); state->filtered_subplans = state->initial_subplans; state->filtered_ri_clauses = state->initial_ri_clauses; state->filtered_first_partial_plan = state->first_partial_plan; state->current = INVALID_SUBPLAN_INDEX; state->choose_next_subplan = choose_next_subplan_non_parallel; state->exclusion_ctx = AllocSetContextCreate(CurrentMemoryContext, "ChunkApppend exclusion", ALLOCSET_DEFAULT_SIZES); return (Node *) state; } static void do_startup_exclusion(ChunkAppendState *state) { List *filtered_children = NIL; List *filtered_ri_clauses = NIL; List *filtered_constraints = NIL; ListCell *lc_plan; ListCell *lc_clauses; ListCell *lc_constraints; int i = -1; int filtered_first_partial_plan = state->first_partial_plan; /* * create skeleton plannerinfo for estimate_expression_value */ PlannerGlobal glob = { .boundParams = state->csstate.ss.ps.state->es_param_list_info, }; PlannerInfo root = { .glob = &glob, }; /* Reset included subplans */ state->included_subplans_by_se = NULL; /* * clauses and constraints should always have the same length as initial_subplans */ Assert(list_length(state->initial_subplans) == list_length(state->initial_ri_clauses)); Assert(list_length(state->initial_subplans) == list_length(state->initial_constraints)); forthree (lc_plan, state->initial_subplans, lc_constraints, state->initial_constraints, lc_clauses, state->initial_ri_clauses) { List *restrictinfos = NIL; List *ri_clauses = lfirst(lc_clauses); ListCell *lc; Scan *scan = ts_chunk_append_get_scan_plan(lfirst(lc_plan)); i++; /* * If this is a base rel (chunk), check if it can be * excluded from the scan. Otherwise, fall through. */ if (scan != NULL && scan->scanrelid) { foreach (lc, ri_clauses) { RestrictInfo *ri = makeNode(RestrictInfo); ri->clause = lfirst(lc); restrictinfos = lappend(restrictinfos, ri); } restrictinfos = ts_constify_restrictinfos(&root, restrictinfos); if (can_exclude_chunk(lfirst(lc_constraints), restrictinfos)) { if (i < state->first_partial_plan) filtered_first_partial_plan--; continue; } /* * if this node does runtime exclusion on the children we keep the constified * expressions to save us some work during runtime exclusion */ if (state->runtime_exclusion_children) { List *const_ri_clauses = NIL; foreach (lc, restrictinfos) { RestrictInfo *ri = lfirst(lc); const_ri_clauses = lappend(const_ri_clauses, ri->clause); } ri_clauses = const_ri_clauses; } } state->included_subplans_by_se = bms_add_member(state->included_subplans_by_se, i); filtered_children = lappend(filtered_children, lfirst(lc_plan)); filtered_ri_clauses = lappend(filtered_ri_clauses, ri_clauses); filtered_constraints = lappend(filtered_constraints, lfirst(lc_constraints)); } state->filtered_subplans = filtered_children; state->filtered_ri_clauses = filtered_ri_clauses; state->filtered_constraints = filtered_constraints; state->filtered_first_partial_plan = filtered_first_partial_plan; Assert(list_length(state->filtered_subplans) == bms_num_members(state->included_subplans_by_se)); } /* * Complete initialization of the supplied CustomScanState. * Standard fields have been initialized by ExecInitCustomScan, * but any private fields should be initialized here. */ static void chunk_append_begin(CustomScanState *node, EState *estate, int eflags) { CustomScan *cscan = castNode(CustomScan, node->ss.ps.plan); ChunkAppendState *state = (ChunkAppendState *) node; /* CustomScan hard-codes the scan and result tuple slot to a fixed * TTSOpsVirtual ops (meaning it expects the slot ops of the child tuple to * also have this type). Oddly, when reading slots from subscan nodes * (children), there is no knowing what tuple slot ops the child slot will * have (e.g., for ChunkAppend it is common that the child is a * seqscan/indexscan that produces a TTSOpsBufferHeapTuple * slot). Unfortunately, any mismatch between slot types when projecting is * asserted by PostgreSQL. To avoid this issue, we mark the scanops as * non-fixed and reinitialize the projection state with this new setting. * * Alternatively, we could copy the child tuple into the scan slot to get * the expected ops before projection, but this would require materializing * and copying the tuple unnecessarily. */ node->ss.ps.scanopsfixed = false; /* Since we sometimes return the scan slot directly from the subnode, the * result slot is not fixed either. */ node->ss.ps.resultopsfixed = false; ExecAssignScanProjectionInfoWithVarno(&node->ss, INDEX_VAR); initialize_constraints(state, list_nth(cscan->custom_private, CAP_RTIndexes)); /* In parallel mode with a parallel_aware plan, the parallel leader performs the startup * exclusion and stores the result in shared memory (the flag SUBPLAN_STATE_INCLUDED of * pstate->subplan_state is set for all included plans). * * The parallel workers use the information from shared memory to include the same plans as the * parallel leader. This ensures that all workers work on the same subplans and we have an * agreement about the number of subplans. This is necessary to ensure that the parallel workers * work correctly and that the next subplan to be processed in the shared memory * (pstate->next_plan) pointers to the same plan in all workers. * * If the workers perform the startup exclusion individually, they may choose different subplans * (e.g., due to a "constant" function that claims to be constant but returns different * results). In that case, we have a disagreement about the plans between the workers. This * would lead to hard-to-debug problems and out-of-bounds reads when pstate->next_plan is used * for subplan selection. * */ if (IsParallelWorker() && node->ss.ps.plan->parallel_aware) { /* We are inside a parallel worker running a parallel plan. Chunk exclusion was performed by * the leader, and based on it, we will initialize the included subplans later, in * chunk_append_initialize_worker. We have to store estate and eflags here that are needed * for that initialization. * * Note: When force_parallel_mode debug GUC is set, a normal sequential ChunkAppend plan can * run inside a parallel worker. In this case, we have to perform the chunk exclusion right * away. We distinguish it by that the parallel_aware flag of the plan is not set. */ state->estate = estate; state->eflags = eflags; return; } if (state->startup_exclusion) do_startup_exclusion(state); perform_plan_init(state, estate, eflags); } /* * Perform an initialization of the filtered_subplans. */ static void perform_plan_init(ChunkAppendState *state, EState *estate, int eflags) { ListCell *lc; int i; #ifdef USE_ASSERT_CHECKING Assert(state->init_done == false); state->init_done = true; #endif state->num_subplans = list_length(state->filtered_subplans); if (state->num_subplans == 0) { state->current = NO_MATCHING_SUBPLANS; return; } state->subplanstates = (PlanState **) palloc0(state->num_subplans * sizeof(PlanState *)); i = 0; foreach (lc, state->filtered_subplans) { /* * we use an array for the states but put it in custom_ps as well * so explain and planstate_tree_walker can find it */ state->subplanstates[i] = ExecInitNode(lfirst(lc), estate, eflags); state->csstate.custom_ps = lappend(state->csstate.custom_ps, state->subplanstates[i]); /* * pass down limit to child nodes */ if (state->limit) ExecSetTupleBound(state->limit, state->subplanstates[i]); i++; } if (state->runtime_exclusion_parent || state->runtime_exclusion_children) { state->params = state->subplanstates[0]->plan->allParam; /* * make sure all params are initialized for runtime exclusion */ state->csstate.ss.ps.chgParam = bms_copy(state->subplanstates[0]->plan->allParam); } } static bool can_exclude_constraints_using_clauses(ChunkAppendState *state, List *constraints, List *clauses, PlannerInfo *root, PlanState *ps) { bool can_exclude; ListCell *lc; MemoryContext old = MemoryContextSwitchTo(state->exclusion_ctx); List *restrictinfos = NIL; foreach (lc, clauses) { RestrictInfo *ri = makeNode(RestrictInfo); ri->clause = lfirst(lc); restrictinfos = lappend(restrictinfos, ri); } restrictinfos = ts_constify_restrictinfo_params(root, ps->state, restrictinfos); can_exclude = can_exclude_chunk(constraints, restrictinfos); MemoryContextReset(state->exclusion_ctx); MemoryContextSwitchTo(old); return can_exclude; } /* * build bitmap of valid subplans for runtime exclusion */ static void initialize_runtime_exclusion(ChunkAppendState *state) { ListCell *lc_clauses, *lc_constraints; int i = 0; PlannerGlobal glob = { .boundParams = state->csstate.ss.ps.state->es_param_list_info, }; PlannerInfo root = { .glob = &glob, }; state->runtime_initialized = true; if (state->num_subplans == 0) { return; } state->runtime_number_loops++; if (state->runtime_exclusion_parent) { /* try to exclude all the chunks using the parents clauses. * here, all constraints are true but exclusion can still * happen because of things like ANY(empty set), and NULL * inference */ if (can_exclude_constraints_using_clauses(state, list_make1(makeBoolConst(true, false)), state->initial_parent_clauses, &root, &state->csstate.ss.ps)) { state->runtime_number_exclusions_parent++; return; } } if (!state->runtime_exclusion_children) { for (i = 0; i < state->num_subplans; i++) { state->valid_subplans = bms_add_member(state->valid_subplans, i); } return; } Assert(state->num_subplans == list_length(state->filtered_ri_clauses)); lc_clauses = list_head(state->filtered_ri_clauses); lc_constraints = list_head(state->filtered_constraints); /* * mark subplans as active/inactive in valid_subplans */ for (i = 0; i < state->num_subplans; i++) { PlanState *ps = state->subplanstates[i]; Scan *scan = ts_chunk_append_get_scan_plan(ps->plan); if (scan == NULL || scan->scanrelid == 0) { state->valid_subplans = bms_add_member(state->valid_subplans, i); } else { bool can_exclude = can_exclude_constraints_using_clauses(state, lfirst(lc_constraints), lfirst(lc_clauses), &root, ps); if (!can_exclude) state->valid_subplans = bms_add_member(state->valid_subplans, i); else state->runtime_number_exclusions_children++; } lc_clauses = lnext(state->filtered_ri_clauses, lc_clauses); lc_constraints = lnext(state->filtered_constraints, lc_constraints); } } /* * Fetch the next scan tuple. * * If any tuples remain, it should fill ps_ResultTupleSlot with the next * tuple in the current scan direction, and then return the tuple slot. * If not, NULL or an empty slot should be returned. */ static TupleTableSlot * chunk_append_exec(CustomScanState *node) { ChunkAppendState *state = (ChunkAppendState *) node; ExprContext *econtext = node->ss.ps.ps_ExprContext; ProjectionInfo *projinfo = node->ss.ps.ps_ProjInfo; TupleTableSlot *subslot; Assert(state->init_done == true); if (state->current == INVALID_SUBPLAN_INDEX) state->choose_next_subplan(state); while (true) { PlanState *subnode; CHECK_FOR_INTERRUPTS(); if (state->current == NO_MATCHING_SUBPLANS) return ExecClearTuple(node->ss.ps.ps_ResultTupleSlot); Assert(state->current >= 0 && state->current < state->num_subplans); subnode = state->subplanstates[state->current]; /* * get a tuple from the subplan */ subslot = ExecProcNode(subnode); if (!TupIsNull(subslot)) { /* * If the subplan gave us something check if we need * to do projection otherwise return as is. */ if (projinfo == NULL) return subslot; ResetExprContext(econtext); econtext->ecxt_scantuple = subslot; return ExecProject(projinfo); } state->choose_next_subplan(state); /* loop back and try to get a tuple from the new subplan */ } } static int get_next_subplan(ChunkAppendState *state, int last_plan) { if (last_plan == NO_MATCHING_SUBPLANS) return NO_MATCHING_SUBPLANS; if (state->runtime_exclusion_parent || state->runtime_exclusion_children) { if (!state->runtime_initialized) initialize_runtime_exclusion(state); /* * bms_next_member will return -2 (NO_MATCHING_SUBPLANS) if there are * no more members */ return bms_next_member(state->valid_subplans, last_plan); } else { int next_plan = last_plan + 1; if (next_plan >= state->num_subplans) return NO_MATCHING_SUBPLANS; return next_plan; } } static void choose_next_subplan_non_parallel(ChunkAppendState *state) { state->current = get_next_subplan(state, state->current); } static void choose_next_subplan_for_worker(ChunkAppendState *state) { ParallelChunkAppendState *pstate = state->pstate; int next_plan; int start; LWLockAcquire(state->lock, LW_EXCLUSIVE); /* mark just completed subplan as finished */ if (state->current >= 0) pstate->subplan_state[state->current] = ts_set_flags_32(pstate->subplan_state[state->current], SUBPLAN_STATE_FINISHED); if (pstate->next_plan == INVALID_SUBPLAN_INDEX) next_plan = get_next_subplan(state, INVALID_SUBPLAN_INDEX); else next_plan = pstate->next_plan; if (next_plan == NO_MATCHING_SUBPLANS) { /* all subplans are finished */ pstate->next_plan = NO_MATCHING_SUBPLANS; state->current = NO_MATCHING_SUBPLANS; LWLockRelease(state->lock); return; } start = next_plan; /* skip finished subplans */ while (ts_flags_are_set_32(pstate->subplan_state[next_plan], SUBPLAN_STATE_FINISHED)) { next_plan = get_next_subplan(state, next_plan); /* wrap around if we reach end of subplan list */ if (next_plan < 0) next_plan = get_next_subplan(state, INVALID_SUBPLAN_INDEX); if (next_plan == start || next_plan < 0) { /* * back at start of search so all subplans are finished * * next_plan should not be < 0 because this means there * are no valid subplans and then the function would * have returned at the check before the while loop but * static analysis marked this so might as well include * that in the check */ Assert(next_plan >= 0); pstate->next_plan = NO_MATCHING_SUBPLANS; state->current = NO_MATCHING_SUBPLANS; LWLockRelease(state->lock); return; } } Assert(next_plan >= 0 && next_plan < state->num_subplans); state->current = next_plan; /* * if this is not a partial plan we mark it as finished * immediately so it does not get assigned another worker */ if (next_plan < state->filtered_first_partial_plan) pstate->subplan_state[next_plan] = ts_set_flags_32(pstate->subplan_state[next_plan], SUBPLAN_STATE_FINISHED); /* advance next_plan for next worker */ pstate->next_plan = get_next_subplan(state, state->current); /* * if we reach the end of the list of subplans we set next_plan * to INVALID_SUBPLAN_INDEX to allow rechecking unfinished subplans * on next call */ if (pstate->next_plan < 0) pstate->next_plan = INVALID_SUBPLAN_INDEX; LWLockRelease(state->lock); } /* * Clean up any private data associated with the CustomScanState. * * This method is required, but it does not need to do anything if there * is no associated data or it will be cleaned up automatically. */ static void chunk_append_end(CustomScanState *node) { ChunkAppendState *state = (ChunkAppendState *) node; int i; for (i = 0; i < state->num_subplans; i++) { ExecEndNode(state->subplanstates[i]); } } /* * Rewind the current scan to the beginning and prepare to rescan the relation. */ static void chunk_append_rescan(CustomScanState *node) { ChunkAppendState *state = (ChunkAppendState *) node; int i; for (i = 0; i < state->num_subplans; i++) { if (node->ss.ps.chgParam != NULL) UpdateChangedParamSet(state->subplanstates[i], node->ss.ps.chgParam); ExecReScan(state->subplanstates[i]); } state->current = INVALID_SUBPLAN_INDEX; /* * detect changed params and reset runtime exclusion state */ if ((state->runtime_exclusion_parent || state->runtime_exclusion_children) && bms_overlap(node->ss.ps.chgParam, state->params)) { bms_free(state->valid_subplans); state->valid_subplans = NULL; state->runtime_initialized = false; } } /* * Estimate the amount of dynamic shared memory that will be required * for parallel operation. * This may be higher than the amount that will actually be used, * but it must not be lower. The return value is in bytes. * This callback is optional, and need only be supplied if this * custom scan provider supports parallel execution. */ static Size chunk_append_estimate_dsm(CustomScanState *node, ParallelContext *pcxt) { ChunkAppendState *state = (ChunkAppendState *) node; return add_size(offsetof(ParallelChunkAppendState, subplan_state), sizeof(uint32) * list_length(state->initial_subplans)); } /* * Initialize the parallel state. */ static void init_pstate(ChunkAppendState *state, ParallelChunkAppendState *pstate) { Assert(state != NULL); Assert(pstate != NULL); Assert(state->csstate.pscan_len > 0); /* The parallel worker state has to be (re-)initialized by the parallel leader */ Assert(!IsParallelWorker()); memset(pstate, 0, state->csstate.pscan_len); pstate->next_plan = INVALID_SUBPLAN_INDEX; pstate->filtered_first_partial_plan = state->filtered_first_partial_plan; /* Mark active subplans in parallel state */ int plan = -1; while ((plan = bms_next_member(state->included_subplans_by_se, plan)) >= 0) { pstate->subplan_state[plan] = ts_set_flags_32(pstate->subplan_state[plan], SUBPLAN_STATE_INCLUDED); } } /* * Initialize the dynamic shared memory that will be required for * parallel operation. * coordinate points to a shared memory area of size equal to the return * value of EstimateDSMCustomScan. * This callback is optional, and need only be supplied if this custom scan * provider supports parallel execution. */ static void chunk_append_initialize_dsm(CustomScanState *node, ParallelContext *pcxt, void *coordinate) { ChunkAppendState *state = (ChunkAppendState *) node; ParallelChunkAppendState *pstate = (ParallelChunkAppendState *) coordinate; init_pstate(state, pstate); state->lock = chunk_append_get_lock_pointer(); /* * Leader should use the same subplan selection as normal worker threads. If the user wishes to * disallow running plans on the leader they should do so via the parallel_leader_participation * GUC. */ state->choose_next_subplan = choose_next_subplan_for_worker; state->current = INVALID_SUBPLAN_INDEX; state->pcxt = pcxt; state->pstate = pstate; } /* * Re-initialize the dynamic shared memory required for parallel operation * when the custom-scan plan node is about to be re-scanned. * This callback is optional, and need only be supplied if this custom scan * provider supports parallel execution. * Recommended practice is that this callback reset only shared state, * while the ReScanCustomScan callback resets only local state. * Currently, this callback will be called before ReScanCustomScan, * but it's best not to rely on that ordering. */ static void chunk_append_reinitialize_dsm(CustomScanState *node, ParallelContext *pcxt, void *coordinate) { ChunkAppendState *state = (ChunkAppendState *) node; ParallelChunkAppendState *pstate = (ParallelChunkAppendState *) coordinate; init_pstate(state, pstate); } /* * Initialize a parallel worker's local state based on the shared state * set up by the leader during InitializeDSMCustomScan. * * This callback is optional, and need only be supplied if this custom scan * provider supports parallel execution. */ static void chunk_append_initialize_worker(CustomScanState *node, shm_toc *toc, void *coordinate) { ChunkAppendState *state = (ChunkAppendState *) node; ParallelChunkAppendState *pstate = (ParallelChunkAppendState *) coordinate; Assert(IsParallelWorker()); Assert(node->ss.ps.plan->parallel_aware); Assert(pstate != NULL); Assert(state->estate != NULL); /* Read information about included plans by startup exclusion from the parallel state */ state->filtered_first_partial_plan = pstate->filtered_first_partial_plan; List *filtered_subplans = NIL; List *filtered_ri_clauses = NIL; List *filtered_constraints = NIL; for (int plan = 0; plan < list_length(state->initial_subplans); plan++) { if (ts_flags_are_set_32(pstate->subplan_state[plan], SUBPLAN_STATE_INCLUDED)) { filtered_subplans = lappend(filtered_subplans, list_nth(state->filtered_subplans, plan)); filtered_ri_clauses = lappend(filtered_ri_clauses, list_nth(state->filtered_ri_clauses, plan)); filtered_constraints = lappend(filtered_constraints, list_nth(state->filtered_constraints, plan)); } } state->filtered_subplans = filtered_subplans; state->filtered_ri_clauses = filtered_ri_clauses; state->filtered_constraints = filtered_constraints; Assert(list_length(state->filtered_subplans) == list_length(state->filtered_ri_clauses)); Assert(list_length(state->filtered_ri_clauses) == list_length(state->filtered_constraints)); state->lock = chunk_append_get_lock_pointer(); state->choose_next_subplan = choose_next_subplan_for_worker; state->current = INVALID_SUBPLAN_INDEX; state->pstate = pstate; perform_plan_init(state, state->estate, state->eflags); Assert(state->num_subplans == list_length(state->filtered_subplans)); } /* * get a pointer to the LWLock used for coordinating * parallel workers */ static LWLock * chunk_append_get_lock_pointer() { LWLock **lock = (LWLock **) find_rendezvous_variable(RENDEZVOUS_CHUNK_APPEND_LWLOCK); if (*lock == NULL) elog(ERROR, "LWLock for coordinating parallel workers not initialized"); return *lock; } /* * Convert restriction clauses to constants expressions (i.e., if there are * mutable functions, they need to be evaluated to constants). For instance, * something like: * * ...WHERE time > now - interval '1 hour' * * becomes * * ...WHERE time > '2017-06-02 11:26:43.935712+02' */ List * ts_constify_restrictinfos(PlannerInfo *root, List *restrictinfos) { List *additional_list = NIL; ListCell *lc; foreach (lc, restrictinfos) { RestrictInfo *rinfo = lfirst(lc); Expr *constified = (Expr *) estimate_expression_value(root, (Node *) rinfo->clause); /* * Note that we have to use equal() here, because the expression mutators * always return a deep copy of the expression tree, even if nothing was * modified. */ if (!equal(rinfo->clause, constified)) { /* * We have constified something, so try applying the time_bucket * transformations again. This might allow us to exclude chunks * based on a parameterized time_bucket expression. */ Expr *additional_clause = ts_transform_time_bucket_comparison(constified); if (additional_clause != NULL) { /* * We successfully added a filter clause based on a * parameterized time_bucket comparison, but it might contain * stable operators like comparison of timestamp to timestamptz, * so we have to evaluate them as well. */ additional_clause = ts_transform_cross_datatype_comparison(additional_clause); additional_clause = (Expr *) estimate_expression_value(root, (Node *) additional_clause); additional_list = lappend(additional_list, make_simple_restrictinfo(root, additional_clause)); } } rinfo->clause = constified; } return list_concat(restrictinfos, additional_list); } List * ts_constify_restrictinfo_params(PlannerInfo *root, EState *state, List *restrictinfos) { ListCell *lc; foreach (lc, restrictinfos) { RestrictInfo *rinfo = lfirst(lc); rinfo->clause = (Expr *) constify_param_mutator((Node *) rinfo->clause, state); rinfo->clause = (Expr *) estimate_expression_value(root, (Node *) rinfo->clause); } return restrictinfos; } static Node * constify_param_mutator(Node *node, void *context) { if (node == NULL) return NULL; /* Don't descend into subplans to constify their parameters, because they may not be valid yet */ if (IsA(node, SubPlan)) return node; if (IsA(node, Param)) { Param *param = castNode(Param, node); EState *estate = (EState *) context; if (param->paramkind == PARAM_EXEC) { TypeCacheEntry *tce = lookup_type_cache(param->paramtype, 0); ParamExecData prm = estate->es_param_exec_vals[param->paramid]; if (prm.execPlan != NULL) { ExprContext *econtext = GetPerTupleExprContext(estate); ExecSetParamPlan(prm.execPlan, econtext); // reload prm as it may have been changed by ExecSetParamPlan call above. prm = estate->es_param_exec_vals[param->paramid]; } if (prm.execPlan == NULL) return (Node *) makeConst(param->paramtype, param->paramtypmod, param->paramcollid, tce->typlen, prm.value, prm.isnull, tce->typbyval); } return node; } return expression_tree_mutator(node, constify_param_mutator, context); } /* * stripped down version of postgres get_relation_constraints */ static List * ca_get_relation_constraints(Oid relationObjectId, Index varno, bool include_notnull) { List *result = NIL; Relation relation; TupleConstr *constr; /* * We assume the relation has already been safely locked. */ relation = table_open(relationObjectId, AccessShareLock); constr = relation->rd_att->constr; if (constr != NULL) { int num_check = constr->num_check; int i; for (i = 0; i < num_check; i++) { Node *cexpr; /* * If this constraint hasn't been fully validated yet, we must * ignore it here. */ if (!constr->check[i].ccvalid) continue; cexpr = stringToNode(constr->check[i].ccbin); /* * Run each expression through const-simplification and * canonicalization. This is not just an optimization, but is * necessary, because we will be comparing it to * similarly-processed qual clauses, and may fail to detect valid * matches without this. This must match the processing done to * qual clauses in preprocess_expression()! (We can skip the * stuff involving subqueries, however, since we don't allow any * in check constraints.) */ cexpr = eval_const_expressions(NULL, cexpr); cexpr = (Node *) canonicalize_qual((Expr *) cexpr, true); /* Fix Vars to have the desired varno */ if (varno != 1) ChangeVarNodes(cexpr, 1, varno, 0); /* * Finally, convert to implicit-AND format (that is, a List) and * append the resulting item(s) to our output list. */ result = list_concat(result, make_ands_implicit((Expr *) cexpr)); } /* Add NOT NULL constraints in expression form, if requested */ if (include_notnull && constr->has_not_null) { int natts = relation->rd_att->natts; for (i = 1; i <= natts; i++) { Form_pg_attribute att = TupleDescAttr(relation->rd_att, i - 1); if (att->attnotnull && !att->attisdropped) { NullTest *ntest = makeNode(NullTest); ntest->arg = (Expr *) makeVar(varno, i, att->atttypid, att->atttypmod, att->attcollation, 0); ntest->nulltesttype = IS_NOT_NULL; /* * argisrow=false is correct even for a composite column, * because attnotnull does not represent a SQL-spec IS NOT * NULL test in such a case, just IS DISTINCT FROM NULL. */ ntest->argisrow = false; ntest->location = -1; result = lappend(result, ntest); } } } if (ts_guc_enable_chunk_skipping) { /* Add column range min/max ranges in 'CHECK CONSTRAINT' form */ result = list_concat(result, ts_chunk_column_stats_construct_check_constraints(relation, relationObjectId, varno)); } } table_close(relation, NoLock); return result; } /* * Exclude child relations (chunks) at execution time based on constraints. * * constraints is the list of constraint expressions of the relation * baserestrictinfo is the list of RestrictInfos */ static bool can_exclude_chunk(List *constraints, List *baserestrictinfo) { /* * Detect constant-FALSE-or-NULL restriction clauses. If we have such a * clause, no rows from the chunk are going to match. Unlike the postgres * analog of this code in relation_excluded_by_constraints, we can't expect * a single const false restrictinfo in this case, because we don't try to * fold the restrictinfos after evaluating the mutable functions. * We have to check this separately from the subsequent predicate_refuted_by. * That function can also work with the normal CHECK constraints, and they * don't fail if the constraint evaluates to null given the restriction info. * That's why it has to prove that the CHECK constraint evaluates to false, * and this doesn't follow from having a const null restrictinfo. */ ListCell *lc; foreach (lc, baserestrictinfo) { RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); Expr *clause = rinfo->clause; if (clause && IsA(clause, Const) && (((Const *) clause)->constisnull || !DatumGetBool(((Const *) clause)->constvalue))) return true; } /* * The constraints are effectively ANDed together, so we can just try to * refute the entire collection at once. This may allow us to make proofs * that would fail if we took them individually. * * Note: we use rel->baserestrictinfo, not safe_restrictions as might seem * an obvious optimization. Some of the clauses might be OR clauses that * have volatile and nonvolatile subclauses, and it's OK to make * deductions with the nonvolatile parts. * * We need strong refutation because we have to prove that the constraints * would yield false, not just NULL. */ if (predicate_refuted_by(constraints, baserestrictinfo, false)) return true; return false; } /* * Fetch the constraints for a relation and adjust range table indexes * if necessary. */ static void initialize_constraints(ChunkAppendState *state, List *initial_rt_indexes) { ListCell *lc_clauses, *lc_plan, *lc_relid; List *constraints = NIL; EState *estate = state->csstate.ss.ps.state; if (initial_rt_indexes == NIL) return; Assert(list_length(state->initial_subplans) == list_length(state->initial_ri_clauses)); Assert(list_length(state->initial_subplans) == list_length(initial_rt_indexes)); forthree (lc_plan, state->initial_subplans, lc_clauses, state->initial_ri_clauses, lc_relid, initial_rt_indexes) { Scan *scan = ts_chunk_append_get_scan_plan(lfirst(lc_plan)); Index initial_index = lfirst_oid(lc_relid); List *relation_constraints = NIL; if (scan != NULL && scan->scanrelid > 0) { Index rt_index = scan->scanrelid; RangeTblEntry *rte = rt_fetch(rt_index, estate->es_range_table); relation_constraints = ca_get_relation_constraints(rte->relid, rt_index, true); /* * Adjust the RangeTableEntry indexes in the restrictinfo * clauses because during planning subquery indexes may be * different from the final index after flattening. */ if (rt_index != initial_index) ChangeVarNodes(lfirst(lc_clauses), initial_index, scan->scanrelid, 0); } constraints = lappend(constraints, relation_constraints); } state->initial_constraints = constraints; state->filtered_constraints = constraints; } /* * Output additional information for EXPLAIN of a custom-scan plan node. * This callback is optional. Common data stored in the ScanState, * such as the target list and scan relation, will be shown even without * this callback, but the callback allows the display of additional, * private state. */ static void chunk_append_explain(CustomScanState *node, List *ancestors, ExplainState *es) { ChunkAppendState *state = (ChunkAppendState *) node; if (state->sort_options != NIL) show_sort_group_keys(state, ancestors, es); if (es->verbose || es->format != EXPLAIN_FORMAT_TEXT) ExplainPropertyBool("Startup Exclusion", state->startup_exclusion, es); if (es->verbose || es->format != EXPLAIN_FORMAT_TEXT) ExplainPropertyBool("Runtime Exclusion", (state->runtime_exclusion_parent || state->runtime_exclusion_children), es); if (state->startup_exclusion) ExplainPropertyInteger("Chunks excluded during startup", NULL, list_length(state->initial_subplans) - list_length(node->custom_ps), es); if (state->runtime_exclusion_parent && state->runtime_number_loops > 0) { int avg_excluded = state->runtime_number_exclusions_parent / state->runtime_number_loops; ExplainPropertyInteger("Hypertables excluded during runtime", NULL, avg_excluded, es); } if (state->runtime_exclusion_children && state->runtime_number_loops > 0) { int avg_excluded = state->runtime_number_exclusions_children / state->runtime_number_loops; ExplainPropertyInteger("Chunks excluded during runtime", NULL, avg_excluded, es); } } /* * adjusted from postgresql explain.c * since we have to keep the state in custom_private our sort state * is in lists instead of arrays */ static void show_sort_group_keys(ChunkAppendState *state, List *ancestors, ExplainState *es) { Plan *plan = state->csstate.ss.ps.plan; List *context; List *result = NIL; StringInfoData sortkeybuf; bool useprefix; int keyno; int nkeys = list_length(linitial(state->sort_options)); List *sort_indexes = linitial(state->sort_options); List *sort_ops = lsecond(state->sort_options); List *sort_collations = lthird(state->sort_options); List *sort_nulls = lfourth(state->sort_options); if (nkeys <= 0) return; initStringInfo(&sortkeybuf); /* Set up deparsing context */ context = set_deparse_context_plan(es->deparse_cxt, plan, ancestors); useprefix = (list_length(es->rtable) > 1 || es->verbose); for (keyno = 0; keyno < nkeys; keyno++) { /* find key expression in tlist */ AttrNumber keyresno = list_nth_oid(sort_indexes, keyno); TargetEntry *target = get_tle_by_resno(castNode(CustomScan, plan)->custom_scan_tlist, keyresno); char *exprstr; if (!target) elog(ERROR, "no tlist entry for key %d", keyresno); /* Deparse the expression, showing any top-level cast */ exprstr = deparse_expression((Node *) target->expr, context, useprefix, true); resetStringInfo(&sortkeybuf); appendStringInfoString(&sortkeybuf, exprstr); /* Append sort order information, if relevant */ if (sort_ops != NIL) show_sortorder_options(&sortkeybuf, (Node *) target->expr, list_nth_oid(sort_ops, keyno), list_nth_oid(sort_collations, keyno), list_nth_oid(sort_nulls, keyno)); /* Emit one property-list item per sort key */ result = lappend(result, pstrdup(sortkeybuf.data)); } ExplainPropertyList("Order", result, es); } /* copied verbatim from postgresql explain.c */ static void show_sortorder_options(StringInfo buf, Node *sortexpr, Oid sortOperator, Oid collation, bool nullsFirst) { Oid sortcoltype = exprType(sortexpr); bool reverse = false; TypeCacheEntry *typentry; typentry = lookup_type_cache(sortcoltype, TYPECACHE_LT_OPR | TYPECACHE_GT_OPR); /* * Print COLLATE if it's not default. There are some cases where this is * redundant, eg if expression is a column whose declared collation is * that collation, but it's hard to distinguish that here. */ if (OidIsValid(collation) && collation != DEFAULT_COLLATION_OID) { char *collname = get_collation_name(collation); if (collname == NULL) elog(ERROR, "cache lookup failed for collation %u", collation); appendStringInfo(buf, " COLLATE %s", quote_identifier(collname)); } /* Print direction if not ASC, or USING if non-default sort operator */ if (sortOperator == typentry->gt_opr) { appendStringInfoString(buf, " DESC"); reverse = true; } else if (sortOperator != typentry->lt_opr) { char *opname = get_opname(sortOperator); if (opname == NULL) elog(ERROR, "cache lookup failed for operator %u", sortOperator); appendStringInfo(buf, " USING %s", opname); /* Determine whether operator would be considered ASC or DESC */ (void) get_equality_op_for_ordering_op(sortOperator, &reverse); } /* Add NULLS FIRST/LAST only if it wouldn't be default */ if (nullsFirst && !reverse) { appendStringInfoString(buf, " NULLS FIRST"); } else if (!nullsFirst && reverse) { appendStringInfoString(buf, " NULLS LAST"); } } ================================================ FILE: src/nodes/chunk_append/planner.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_namespace.h> #include <nodes/extensible.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <optimizer/appendinfo.h> #include <optimizer/optimizer.h> #include <optimizer/pathnode.h> #include <optimizer/paths.h> #include <optimizer/placeholder.h> #include <optimizer/planmain.h> #include <optimizer/prep.h> #include <optimizer/subselect.h> #include <optimizer/tlist.h> #include <parser/parsetree.h> #include "guc.h" #include "import/planner.h" #include "nodes/chunk_append/chunk_append.h" #include "nodes/chunk_append/transform.h" #include "nodes/modify_hypertable.h" #include "nodes/vector_agg.h" static Sort *make_sort(Plan *lefttree, int numCols, AttrNumber *sortColIdx, Oid *sortOperators, Oid *collations, bool *nullsFirst); static Plan *adjust_childscan(PlannerInfo *root, Plan *plan, Path *path, List *pathkeys, List *tlist, AttrNumber *sortColIdx); static CustomScanMethods chunk_append_plan_methods = { .CustomName = "ChunkAppend", .CreateCustomScanState = ts_chunk_append_state_create, }; bool ts_is_chunk_append_plan(Plan *plan) { return IsA(plan, CustomScan) && castNode(CustomScan, plan)->methods == &chunk_append_plan_methods; } void _chunk_append_init(void) { TryRegisterCustomScanMethods(&chunk_append_plan_methods); } static Plan * adjust_childscan(PlannerInfo *root, Plan *plan, Path *path, List *pathkeys, List *tlist, AttrNumber *sortColIdx) { int childSortCols; Oid *sortOperators; Oid *collations; bool *nullsFirst; AttrNumber *childColIdx; /* Compute sort column info, and adjust subplan's tlist as needed */ plan = ts_prepare_sort_from_pathkeys(plan, pathkeys, path->parent->relids, sortColIdx, true, &childSortCols, &childColIdx, &sortOperators, &collations, &nullsFirst); /* inject sort node if child sort order does not match desired order */ if (!pathkeys_contained_in(pathkeys, path->pathkeys)) { Assert(!IsA(plan, Sort)); plan = (Plan *) make_sort(plan, childSortCols, childColIdx, sortOperators, collations, nullsFirst); } return plan; } Plan * ts_chunk_append_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *path, List *tlist, List *clauses, List *custom_plans) { ListCell *lc_child; List *parent_clauses = NIL; List *chunk_ri_clauses = NIL; List *chunk_rt_indexes = NIL; List *sort_options = NIL; List *custom_private = NIL; uint32 limit = 0; List *orig_tlist = NIL; ChunkAppendPath *capath = (ChunkAppendPath *) path; CustomScan *cscan = makeNode(CustomScan); cscan->flags = path->flags; cscan->methods = &chunk_append_plan_methods; cscan->scan.scanrelid = rel->relid; orig_tlist = ts_build_path_tlist(root, (Path *) path); tlist = orig_tlist; /* * If this is a child of ModifyHypertable we need to adjust * targetlists to not have any ROWID_VAR references as postgres * asserts that scan targetlists do not have them in setrefs.c * * We keep orig_tlist unaltered to let adjust_appendrel_attrs() * replace ROWID_VARs for chunks' targetlists (it would assert * trying to modify a "wholerow" target entry that has already * been adjusted by ts_replace_rowid_vars(); we see these in * foreign tables). */ if (root->parse->commandType != CMD_SELECT) tlist = ts_replace_rowid_vars(root, tlist, rel->relid); cscan->scan.plan.targetlist = tlist; /* * For parameterized paths (e.g., LATERAL joins), transform outer-relation * Vars to NestLoop Params. We store clauses in custom_private rather than * custom_exprs because chunk_ri_clauses reference chunk relations (after * adjust_appendrel_attrs), but setrefs.c processing of custom_exprs expects * Vars to reference the parent relation (scanrelid). Since custom_private * bypasses setrefs, we must transform outer Vars to Params ourselves. */ if (path->path.param_info && root->curOuterRels) { List *transformed_clauses = NIL; ListCell *lc; foreach (lc, clauses) { RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); RestrictInfo *newrinfo = makeNode(RestrictInfo); memcpy(newrinfo, rinfo, sizeof(RestrictInfo)); newrinfo->clause = (Expr *) ts_replace_nestloop_params(root, (Node *) rinfo->clause); transformed_clauses = lappend(transformed_clauses, newrinfo); } clauses = transformed_clauses; } ListCell *lc_plan, *lc_path; forboth (lc_path, path->custom_paths, lc_plan, custom_plans) { Plan *child_plan = lfirst(lc_plan); Path *child_path = lfirst(lc_path); /* push down targetlist to children */ if (child_path->parent->reloptkind == RELOPT_OTHER_MEMBER_REL) { /* if this is an append child we need to adjust targetlist references */ AppendRelInfo *appinfo = ts_get_appendrelinfo(root, child_path->parent->relid, false); child_plan->targetlist = castNode(List, adjust_appendrel_attrs(root, (Node *) orig_tlist, 1, &appinfo)); } else { /* * This can also be a MergeAppend path building the entire * hypertable, in case we have a single partial chunk. */ child_plan->targetlist = tlist; } } if (path->path.pathkeys != NIL) { /* * If this is an ordered append node we need to ensure the columns * required for sorting are present in the targetlist and all children * return sorted output. Children not returning sorted output will be * wrapped in a sort node. */ int numCols; AttrNumber *sortColIdx; Oid *sortOperators; Oid *collations; bool *nullsFirst; List *pathkeys = path->path.pathkeys; List *sort_indexes = NIL; List *sort_ops = NIL; List *sort_collations = NIL; List *sort_nulls = NIL; int i; /* Compute sort column info, and adjust MergeAppend's tlist as needed */ ts_prepare_sort_from_pathkeys(&cscan->scan.plan, pathkeys, path->path.parent->relids, NULL, true, &numCols, &sortColIdx, &sortOperators, &collations, &nullsFirst); /* * collect sort information to make available to explain */ for (i = 0; i < numCols; i++) { sort_indexes = lappend_oid(sort_indexes, sortColIdx[i]); sort_ops = lappend_oid(sort_ops, sortOperators[i]); sort_collations = lappend_oid(sort_collations, collations[i]); sort_nulls = lappend_oid(sort_nulls, nullsFirst[i]); } sort_options = list_make4(sort_indexes, sort_ops, sort_collations, sort_nulls); forboth (lc_path, path->custom_paths, lc_plan, custom_plans) { /* * If the planner injected a Result node to do projection * we can safely remove the Result node if it does not have * a one-time filter because ChunkAppend can do projection. */ if (IsA(lfirst(lc_plan), Result) && castNode(Result, lfirst(lc_plan))->resconstantqual == NULL) lfirst(lc_plan) = ((Plan *) lfirst(lc_plan))->lefttree; /* * This could be a MergeAppend due to space partitioning, or * due to partially compressed chunks. The MergeAppend plan adds * sort to it children, and has the proper sorting itself, so no * need to do anything for it. * We can also have plain chunk scans here which might require a * Sort. */ if (!IsA(lfirst(lc_plan), MergeAppend)) { lfirst(lc_plan) = adjust_childscan(root, lfirst(lc_plan), lfirst(lc_path), path->path.pathkeys, orig_tlist, sortColIdx); } } } /* decouple input tlist from output tlist in case output tlist gets modified later */ cscan->custom_scan_tlist = list_copy(tlist); cscan->custom_plans = custom_plans; /* * If we do either startup or runtime exclusion, we need to pass restrictinfo * clauses into executor. */ if (capath->startup_exclusion || capath->runtime_exclusion_children) { foreach (lc_child, cscan->custom_plans) { Scan *scan = ts_chunk_append_get_scan_plan(lfirst(lc_child)); if (scan == NULL || scan->scanrelid == 0) { chunk_ri_clauses = lappend(chunk_ri_clauses, NIL); chunk_rt_indexes = lappend_oid(chunk_rt_indexes, 0); } else { List *chunk_clauses = NIL; ListCell *lc; AppendRelInfo *appinfo = ts_get_appendrelinfo(root, scan->scanrelid, false); foreach (lc, clauses) { Node *clause = (Node *) ts_transform_cross_datatype_comparison( castNode(RestrictInfo, lfirst(lc))->clause); clause = adjust_appendrel_attrs(root, clause, 1, &appinfo); chunk_clauses = lappend(chunk_clauses, clause); } chunk_ri_clauses = lappend(chunk_ri_clauses, chunk_clauses); chunk_rt_indexes = lappend_oid(chunk_rt_indexes, scan->scanrelid); } } Assert(list_length(cscan->custom_plans) == list_length(chunk_ri_clauses)); Assert(list_length(chunk_ri_clauses) == list_length(chunk_rt_indexes)); } /* pass down the parent clauses if doing parent exclusion */ if (capath->runtime_exclusion_parent) { ListCell *lc; foreach (lc, clauses) { parent_clauses = lappend(parent_clauses, castNode(RestrictInfo, lfirst(lc))->clause); } } if (capath->pushdown_limit && capath->limit_tuples > 0) limit = capath->limit_tuples; custom_private = list_make1(list_make5_int(capath->startup_exclusion, capath->runtime_exclusion_parent, capath->runtime_exclusion_children, limit, capath->first_partial_path)); custom_private = lappend(custom_private, chunk_ri_clauses); custom_private = lappend(custom_private, chunk_rt_indexes); custom_private = lappend(custom_private, sort_options); custom_private = lappend(custom_private, parent_clauses); cscan->custom_private = custom_private; return &cscan->scan.plan; } /* * make_sort --- basic routine to build a Sort plan node * * Caller must have built the sortColIdx, sortOperators, collations, and * nullsFirst arrays already. */ static Sort * make_sort(Plan *lefttree, int numCols, AttrNumber *sortColIdx, Oid *sortOperators, Oid *collations, bool *nullsFirst) { Sort *node = makeNode(Sort); Plan *plan = &node->plan; plan->targetlist = lefttree->targetlist; plan->qual = NIL; plan->lefttree = lefttree; plan->righttree = NULL; node->numCols = numCols; node->sortColIdx = sortColIdx; node->sortOperators = sortOperators; node->collations = collations; node->nullsFirst = nullsFirst; return node; } Scan * ts_chunk_append_get_scan_plan(Plan *plan) { if (plan == NULL) return NULL; switch (nodeTag(plan)) { case T_BitmapHeapScan: case T_BitmapIndexScan: case T_CteScan: case T_ForeignScan: case T_FunctionScan: case T_IndexOnlyScan: case T_IndexScan: case T_SampleScan: case T_SeqScan: case T_SubqueryScan: case T_TidScan: case T_ValuesScan: case T_WorkTableScan: case T_TidRangeScan: return (Scan *) plan; case T_CustomScan: { CustomScan *custom = castNode(CustomScan, plan); if (custom->scan.scanrelid > 0) { /* * The custom plan node is a scan itself. This handles the * ColumnarScan node. */ return (Scan *) plan; } if (strcmp(custom->methods->CustomName, VECTOR_AGG_NODE_NAME) == 0) { /* * This is a vectorized aggregation node, we have to recurse * into its child, similar to the normal aggregation node. * * Unfortunately we have to hardcode the node name here, because * we can't depend on the TSL library. */ return ts_chunk_append_get_scan_plan(linitial(custom->custom_plans)); } break; } case T_Sort: case T_Result: case T_Agg: if (plan->lefttree != NULL) { Assert(plan->righttree == NULL); /* Let ts_chunk_append_get_scan_plan handle the subplan */ return ts_chunk_append_get_scan_plan(plan->lefttree); } break; default: break; } return NULL; } ================================================ FILE: src/nodes/chunk_append/transform.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_namespace.h> #include <catalog/pg_type.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <optimizer/optimizer.h> #include <utils/lsyscache.h> #include "nodes/chunk_append/transform.h" #include "utils.h" #define DATATYPE_PAIR(left, right, type1, type2) \ (((left) == (type1) && (right) == (type2)) || ((left) == (type2) && (right) == (type1))) /* * Cross datatype comparisons between DATE/TIMESTAMP/TIMESTAMPTZ * are not immutable which prevents their usage for chunk exclusion. * Unfortunately estimate_expression_value will not estimate those * expressions which makes them unusable for execution time chunk * exclusion with constraint aware append. * To circumvent this we inject casts and use an operator * with the same datatype on both sides when constifying * restrictinfo. This allows estimate_expression_value * to evaluate those expressions and makes them accessible for * execution time chunk exclusion. * * The following transformations are done: * TIMESTAMP OP TIMESTAMPTZ => TIMESTAMP OP (TIMESTAMPTZ::TIMESTAMP) * TIMESTAMPTZ OP DATE => TIMESTAMPTZ OP (DATE::TIMESTAMPTZ) * * No transformation is required for TIMESTAMP OP DATE because * those operators are marked immutable. */ Expr * ts_transform_cross_datatype_comparison(Expr *clause) { if (!IsA(clause, OpExpr) || list_length(castNode(OpExpr, clause)->args) != 2) { return clause; } OpExpr *op = castNode(OpExpr, clause); Oid left_type = exprType(linitial(op->args)); Oid right_type = exprType(lsecond(op->args)); /* * Postgres doesn't allow non-bool or set returning functions in the WHERE * clause. */ Assert(op->opresulttype == BOOLOID && !op->opretset); if (!IsA(linitial(op->args), Var) && !IsA(lsecond(op->args), Var)) return clause; if (DATATYPE_PAIR(left_type, right_type, TIMESTAMPOID, TIMESTAMPTZOID) || DATATYPE_PAIR(left_type, right_type, TIMESTAMPTZOID, DATEOID)) { char *opname = get_opname(op->opno); Oid source_type, target_type, opno, cast_oid; /* * if Var is on left side we put cast on right side otherwise * it will be left */ if (IsA(linitial(op->args), Var)) { source_type = right_type; target_type = left_type; } else { source_type = left_type; target_type = right_type; } opno = ts_get_operator(opname, PG_CATALOG_NAMESPACE, target_type, target_type); cast_oid = ts_get_cast_func(source_type, target_type); if (OidIsValid(opno) && OidIsValid(cast_oid)) { Expr *left = copyObject(linitial(op->args)); Expr *right = copyObject(lsecond(op->args)); if (source_type == left_type) left = (Expr *) makeFuncExpr(cast_oid, target_type, list_make1(left), InvalidOid, InvalidOid, 0); else right = (Expr *) makeFuncExpr(cast_oid, target_type, list_make1(right), InvalidOid, InvalidOid, 0); return make_opclause(opno, BOOLOID, false, left, right, InvalidOid, InvalidOid); } } return clause; } ================================================ FILE: src/nodes/chunk_append/transform.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/extensible.h> #include "export.h" extern TSDLLEXPORT Expr *ts_transform_cross_datatype_comparison(Expr *clause); ================================================ FILE: src/nodes/constraint_aware_append/CMakeLists.txt ================================================ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/constraint_aware_append.c) target_sources(${PROJECT_NAME} PRIVATE ${SOURCES}) ================================================ FILE: src/nodes/constraint_aware_append/constraint_aware_append.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_cast.h> #include <catalog/pg_class.h> #include <catalog/pg_namespace.h> #include <catalog/pg_operator.h> #include <commands/explain.h> #include <executor/executor.h> #include <nodes/extensible.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <nodes/nodes.h> #include <nodes/parsenodes.h> #include <nodes/plannodes.h> #include <optimizer/appendinfo.h> #include <optimizer/cost.h> #include <optimizer/optimizer.h> #include <optimizer/plancat.h> #include <parser/parsetree.h> #include <rewrite/rewriteManip.h> #include <storage/lockdefs.h> #include <utils/lsyscache.h> #include <utils/memutils.h> #include <utils/syscache.h> #include "compat/compat.h" #include "constraint_aware_append.h" #include "debug_assert.h" #include "guc.h" #include "nodes/chunk_append/transform.h" #include "utils.h" #if PG18_GE #include <commands/explain_format.h> #endif /* * Exclude child relations (chunks) at execution time based on constraints. * * This functions tries to reuse as much functionality as possible from standard * constraint exclusion in PostgreSQL that normally happens at planning * time. Therefore, we need to fake a number of planning-related data * structures. */ static bool excluded_by_constraint(PlannerInfo *root, RangeTblEntry *rte, Index rt_index, List *restrictinfos) { RelOptInfo rel = { .type = T_RelOptInfo, .relid = rt_index, .reloptkind = RELOPT_OTHER_MEMBER_REL, .baserestrictinfo = restrictinfos, }; return relation_excluded_by_constraints(root, &rel, rte); } static Plan * get_plans_for_exclusion(Plan *plan) { /* Optimization: If we want to be able to prune */ /* when the node is a T_Result or T_Sort, then we need to peek */ /* into the subplans of this Result node. */ switch (nodeTag(plan)) { case T_Result: case T_Sort: Ensure(plan->lefttree != NULL, "subplan is null"); return get_plans_for_exclusion(plan->lefttree); default: return plan; } } static bool can_exclude_chunk(PlannerInfo *root, RangeTblEntry *rte, Index rt_index, List *restrictinfos) { return rte->rtekind == RTE_RELATION && rte->relkind == RELKIND_RELATION && !rte->inh && excluded_by_constraint(root, rte, rt_index, restrictinfos); } /* * Convert restriction clauses to constants expressions (i.e., if there are * mutable functions, they need to be evaluated to constants). For instance, * something like: * * ...WHERE time > now - interval '1 hour' * * becomes * * ...WHERE time > '2017-06-02 11:26:43.935712+02' */ static List * constify_restrictinfos(PlannerInfo *root, List *restrictinfos) { ListCell *lc; foreach (lc, restrictinfos) { RestrictInfo *rinfo = lfirst(lc); rinfo->clause = (Expr *) estimate_expression_value(root, (Node *) rinfo->clause); } return restrictinfos; } /* * Initialize the scan state and prune any subplans from the Append node below * us in the plan tree. Pruning happens by evaluating the subplan's table * constraints against a folded version of the restriction clauses in the query. */ static void ca_append_begin(CustomScanState *node, EState *estate, int eflags) { ConstraintAwareAppendState *state = (ConstraintAwareAppendState *) node; CustomScan *cscan = (CustomScan *) node->ss.ps.plan; Plan *subplan = copyObject(state->subplan); List *chunk_ri_clauses = lsecond(cscan->custom_private); List *chunk_relids = lthird(cscan->custom_private); List **appendplans, *old_appendplans; ListCell *lc_plan; ListCell *lc_clauses; ListCell *lc_relid; /* * create skeleton plannerinfo to reuse some PostgreSQL planner functions */ Query parse = { .resultRelation = InvalidOid, }; PlannerGlobal glob = { .boundParams = NULL, }; PlannerInfo root = { .glob = &glob, .parse = &parse, }; /* CustomScan hard-codes the scan and result tuple slot to a fixed * TTSOpsVirtual ops (meaning it expects the slot ops of the child tuple to * also have this type). Oddly, when reading slots from subscan nodes * (children), there is no knowing what tuple slot ops the child slot will * have (e.g., for ChunkAppend it is common that the child is a * seqscan/indexscan that produces a TTSOpsBufferHeapTuple * slot). Unfortunately, any mismatch between slot types when projecting is * asserted by PostgreSQL. To avoid this issue, we mark the scanops as * non-fixed and reinitialize the projection state with this new setting. * * Alternatively, we could copy the child tuple into the scan slot to get * the expected ops before projection, but this would require materializing * and copying the tuple unnecessarily. */ node->ss.ps.scanopsfixed = false; /* Since we sometimes return the scan slot directly from the subnode, the * result slot is not fixed either. */ node->ss.ps.resultopsfixed = false; ExecAssignScanProjectionInfoWithVarno(&node->ss, INDEX_VAR); switch (nodeTag(subplan)) { case T_Append: { Append *append = (Append *) subplan; old_appendplans = append->appendplans; append->appendplans = NIL; appendplans = &append->appendplans; break; } case T_MergeAppend: { MergeAppend *append = (MergeAppend *) subplan; old_appendplans = append->mergeplans; append->mergeplans = NIL; appendplans = &append->mergeplans; break; } case T_Result: /* * Append plans are turned into a Result node if empty. This can * happen if children are pruned first by constraint exclusion * while we also remove the main table from the appendplans list, * leaving an empty list. In that case, there is nothing to do. */ return; default: elog(ERROR, "invalid child of constraint-aware append: %s", ts_get_node_name((Node *) subplan)); } /* * clauses should always have the same length as appendplans because * that's the base for building the lists */ Assert(list_length(old_appendplans) == list_length(chunk_ri_clauses)); Assert(list_length(chunk_relids) == list_length(chunk_ri_clauses)); forthree (lc_plan, old_appendplans, lc_clauses, chunk_ri_clauses, lc_relid, chunk_relids) { Plan *plan = get_plans_for_exclusion(lfirst(lc_plan)); switch (nodeTag(plan)) { case T_SeqScan: case T_SampleScan: case T_IndexScan: case T_IndexOnlyScan: case T_BitmapIndexScan: case T_BitmapHeapScan: case T_TidScan: case T_SubqueryScan: case T_FunctionScan: case T_ValuesScan: case T_CteScan: case T_WorkTableScan: case T_ForeignScan: case T_CustomScan: case T_TidRangeScan: { /* * If this is a base rel (chunk), check if it can be * excluded from the scan. Otherwise, fall through. */ Index scanrelid = ((Scan *) plan)->scanrelid; List *restrictinfos = NIL; List *ri_clauses = lfirst(lc_clauses); ListCell *lc; RangeTblEntry *rte; Assert(scanrelid); foreach (lc, ri_clauses) { RestrictInfo *ri = makeNode(RestrictInfo); ri->clause = lfirst(lc); /* * The index of the RangeTblEntry might have changed between planning * because of flattening, so we need to adjust the expressions * for the RestrictInfos if they are not equal. */ if (lfirst_oid(lc_relid) != scanrelid) ChangeVarNodes((Node *) ri->clause, lfirst_oid(lc_relid), scanrelid, 0); restrictinfos = lappend(restrictinfos, ri); } /* * The function excluded_by_constraint(), which is called when * excluding chunks, assumes that a relation is already locked * when called. In most cases the relation is already locked * when getting here, but not in case of some parallel scans * where the parallel worker hasn't locked it yet. */ rte = rt_fetch(scanrelid, estate->es_range_table); LockRelationOid(rte->relid, AccessShareLock); restrictinfos = constify_restrictinfos(&root, restrictinfos); if (can_exclude_chunk(&root, rte, scanrelid, restrictinfos)) continue; *appendplans = lappend(*appendplans, lfirst(lc_plan)); break; } default: elog(ERROR, "invalid child of constraint-aware append: %s", ts_get_node_name((Node *) plan)); break; } } state->num_append_subplans = list_length(*appendplans); state->num_chunks_excluded = list_length(old_appendplans) - state->num_append_subplans; if (state->num_append_subplans > 0) node->custom_ps = list_make1(ExecInitNode(subplan, estate, eflags)); } static TupleTableSlot * ca_append_exec(CustomScanState *node) { ConstraintAwareAppendState *state = (ConstraintAwareAppendState *) node; TupleTableSlot *subslot; ExprContext *econtext = node->ss.ps.ps_ExprContext; /* * Check if all append subplans were pruned. In that case there is nothing * to do. */ if (state->num_append_subplans == 0) return NULL; ResetExprContext(econtext); while (true) { subslot = ExecProcNode(linitial(node->custom_ps)); if (TupIsNull(subslot)) return NULL; if (!node->ss.ps.ps_ProjInfo) return subslot; econtext->ecxt_scantuple = subslot; return ExecProject(node->ss.ps.ps_ProjInfo); } } static void ca_append_end(CustomScanState *node) { if (node->custom_ps != NIL) { ExecEndNode(linitial(node->custom_ps)); } } static void ca_append_rescan(CustomScanState *node) { if (node->custom_ps != NIL) { ExecReScan(linitial(node->custom_ps)); } } static void ca_append_explain(CustomScanState *node, List *ancestors, ExplainState *es) { CustomScan *cscan = (CustomScan *) node->ss.ps.plan; ConstraintAwareAppendState *state = (ConstraintAwareAppendState *) node; Oid relid = linitial_oid(linitial(cscan->custom_private)); ExplainPropertyText("Hypertable", get_rel_name(relid), es); ExplainPropertyInteger("Chunks excluded during startup", NULL, state->num_chunks_excluded, es); } static CustomExecMethods constraint_aware_append_state_methods = { .BeginCustomScan = ca_append_begin, .ExecCustomScan = ca_append_exec, .EndCustomScan = ca_append_end, .ReScanCustomScan = ca_append_rescan, .ExplainCustomScan = ca_append_explain, }; static Node * constraint_aware_append_state_create(CustomScan *cscan) { ConstraintAwareAppendState *state; Append *append = linitial(cscan->custom_plans); state = (ConstraintAwareAppendState *) newNode(sizeof(ConstraintAwareAppendState), T_CustomScanState); state->csstate.methods = &constraint_aware_append_state_methods; state->subplan = &append->plan; return (Node *) state; } static CustomScanMethods constraint_aware_append_plan_methods = { .CustomName = "ConstraintAwareAppend", .CreateCustomScanState = constraint_aware_append_state_create, }; static Plan * constraint_aware_append_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *path, List *tlist, List *clauses, List *custom_plans) { CustomScan *cscan = makeNode(CustomScan); Plan *subplan; RangeTblEntry *rte = planner_rt_fetch(rel->relid, root); List *chunk_ri_clauses = NIL; List *chunk_relids = NIL; List *children = NIL; ListCell *lc_child; /* * Postgres will inject Result nodes above mergeappend when target lists don't match * because the nodes themselves do not perform projection. The ConstraintAwareAppend * node can do this projection itself, however, so just throw away the result node * Removing the Result node is only safe if there is no one-time filter */ if (IsA(linitial(custom_plans), Result) && castNode(Result, linitial(custom_plans))->resconstantqual == NULL) { Result *result = castNode(Result, linitial(custom_plans)); if (result->plan.righttree != NULL) elog(ERROR, "unexpected right tree below result node in constraint aware append"); custom_plans = list_make1(result->plan.lefttree); } subplan = linitial(custom_plans); cscan->scan.scanrelid = 0; /* Not a real relation we are scanning */ cscan->scan.plan.targetlist = tlist; /* Target list we expect as output */ cscan->custom_plans = custom_plans; /* * create per chunk RestrictInfo * * We also need to walk the expression trees of the restriction clauses and * update any Vars that reference the main table to instead reference the child * table (chunk) we want to exclude. */ switch (nodeTag(linitial(custom_plans))) { case T_MergeAppend: children = castNode(MergeAppend, linitial(custom_plans))->mergeplans; break; case T_Append: children = castNode(Append, linitial(custom_plans))->appendplans; break; default: elog(ERROR, "invalid child of constraint-aware append: %s", ts_get_node_name((Node *) linitial(custom_plans))); break; } /* * we only iterate over the child chunks of this node * so the list of metadata exactly matches the list of * child nodes in the executor */ foreach (lc_child, children) { Plan *plan = get_plans_for_exclusion(lfirst(lc_child)); switch (nodeTag(plan)) { case T_SeqScan: case T_SampleScan: case T_IndexScan: case T_IndexOnlyScan: case T_BitmapIndexScan: case T_BitmapHeapScan: case T_TidScan: case T_SubqueryScan: case T_FunctionScan: case T_ValuesScan: case T_CteScan: case T_WorkTableScan: case T_ForeignScan: case T_CustomScan: case T_TidRangeScan: { List *chunk_clauses = NIL; ListCell *lc; Index scanrelid = ((Scan *) plan)->scanrelid; AppendRelInfo *appinfo = ts_get_appendrelinfo(root, scanrelid, false); foreach (lc, clauses) { Node *clause = (Node *) ts_transform_cross_datatype_comparison( castNode(RestrictInfo, lfirst(lc))->clause); clause = adjust_appendrel_attrs(root, clause, 1, &appinfo); chunk_clauses = lappend(chunk_clauses, clause); } chunk_ri_clauses = lappend(chunk_ri_clauses, chunk_clauses); chunk_relids = lappend_oid(chunk_relids, scanrelid); break; } default: elog(ERROR, "invalid child of constraint-aware append: %s", ts_get_node_name((Node *) plan)); break; } } cscan->custom_private = list_make3(list_make1_oid(rte->relid), chunk_ri_clauses, chunk_relids); cscan->custom_scan_tlist = subplan->targetlist; /* Target list of tuples * we expect as input */ cscan->flags = path->flags; cscan->methods = &constraint_aware_append_plan_methods; return &cscan->scan.plan; } static CustomPathMethods constraint_aware_append_path_methods = { .CustomName = "ConstraintAwareAppend", .PlanCustomPath = constraint_aware_append_plan_create, }; Path * ts_constraint_aware_append_path_create(PlannerInfo *root, Path *subpath) { ConstraintAwareAppendPath *path; path = (ConstraintAwareAppendPath *) newNode(sizeof(ConstraintAwareAppendPath), T_CustomPath); path->cpath.path.pathtype = T_CustomScan; path->cpath.path.rows = subpath->rows; path->cpath.path.startup_cost = subpath->startup_cost; path->cpath.path.total_cost = subpath->total_cost; path->cpath.path.parent = subpath->parent; path->cpath.path.pathkeys = subpath->pathkeys; path->cpath.path.param_info = subpath->param_info; path->cpath.path.pathtarget = subpath->pathtarget; path->cpath.path.parallel_aware = false; path->cpath.path.parallel_safe = subpath->parallel_safe; path->cpath.path.parallel_workers = subpath->parallel_workers; /* * Set flags. We can set CUSTOMPATH_SUPPORT_BACKWARD_SCAN and * CUSTOMPATH_SUPPORT_MARK_RESTORE. The only interesting flag is the first * one (backward scan), but since we are not scanning a real relation we * need not indicate that we support backward scans. Lower-level index * scanning nodes will scan backward if necessary, so once tuples get to * this node they will be in a given order already. */ path->cpath.flags = 0; path->cpath.custom_paths = list_make1(subpath); path->cpath.methods = &constraint_aware_append_path_methods; /* * Make sure our subpath is either an Append or MergeAppend node */ switch (nodeTag(subpath)) { case T_AppendPath: case T_MergeAppendPath: break; default: elog(ERROR, "invalid child of constraint-aware append: %s", ts_get_node_name((Node *) subpath)); break; } return &path->cpath.path; } bool ts_constraint_aware_append_possible(Path *path) { RelOptInfo *rel = path->parent; ListCell *lc; int num_children; if (!ts_guc_enable_optimizations || !ts_guc_enable_constraint_aware_append || constraint_exclusion == CONSTRAINT_EXCLUSION_OFF) return false; switch (nodeTag(path)) { case T_AppendPath: num_children = list_length(castNode(AppendPath, path)->subpaths); break; case T_MergeAppendPath: num_children = list_length(castNode(MergeAppendPath, path)->subpaths); break; default: return false; } /* Never use constraint-aware append with only one child, since PostgreSQL * will later prune the (Merge)Append node from such plans, leaving us with * an unexpected child node. */ if (num_children <= 1) return false; /* * If there are clauses that have mutable functions, this path is ripe for * execution-time optimization. */ foreach (lc, rel->baserestrictinfo) { RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); if (contain_mutable_functions((Node *) rinfo->clause)) return true; } return false; } bool ts_is_constraint_aware_append_path(Path *path) { return IsA(path, CustomPath) && castNode(CustomPath, path)->methods == &constraint_aware_append_path_methods; } void _constraint_aware_append_init(void) { TryRegisterCustomScanMethods(&constraint_aware_append_plan_methods); } ================================================ FILE: src/nodes/constraint_aware_append/constraint_aware_append.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/extensible.h> typedef struct ConstraintAwareAppendPath { CustomPath cpath; } ConstraintAwareAppendPath; typedef struct ConstraintAwareAppendState { CustomScanState csstate; Plan *subplan; Size num_append_subplans; Size num_chunks_excluded; } ConstraintAwareAppendState; typedef struct Hypertable Hypertable; extern bool ts_constraint_aware_append_possible(Path *path); extern TSDLLEXPORT Path *ts_constraint_aware_append_path_create(PlannerInfo *root, Path *subpath); extern TSDLLEXPORT bool ts_is_constraint_aware_append_path(Path *path); extern void _constraint_aware_append_init(void); ================================================ FILE: src/nodes/modify_hypertable.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <nodes/execnodes.h> #include <nodes/makefuncs.h> #include <utils/syscache.h> #include "compat/compat.h" #include "chunk_tuple_routing.h" #include "cross_module_fn.h" #include "guc.h" #include "indexing.h" #include "nodes/chunk_append/chunk_append.h" #include "nodes/modify_hypertable.h" #if PG18_GE #include <commands/explain_format.h> #endif static AttrNumber rel_get_natts(Oid relid) { HeapTuple tp = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); if (!HeapTupleIsValid(tp)) elog(ERROR, "cache lookup failed for relation %u", relid); AttrNumber natts = ((Form_pg_class) GETSTRUCT(tp))->relnatts; ReleaseSysCache(tp); return natts; } static bool rel_has_dropped_attrs(Oid relid) { AttrNumber natts = rel_get_natts(relid); for (AttrNumber attno = 1; attno <= natts; attno++) { HeapTuple tp = SearchSysCache2(ATTNUM, ObjectIdGetDatum(relid), Int16GetDatum(attno)); if (!HeapTupleIsValid(tp)) continue; Form_pg_attribute att_tup = (Form_pg_attribute) GETSTRUCT(tp); bool result = att_tup->attisdropped || att_tup->atthasmissing; ReleaseSysCache(tp); if (result) return true; } return false; } static bool should_use_direct_compress(ModifyHypertableState *state) { if (!ts_guc_enable_direct_compress_insert) return false; ModifyTableState *mtstate = linitial_node(ModifyTableState, state->cscan_state.custom_ps); ResultRelInfo *resultRelInfo = mtstate->resultRelInfo; Hypertable *ht = state->ctr->hypertable; if (!TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(ht)) return false; if (resultRelInfo->ri_TrigDesc) { ereport(WARNING, (errmsg("disabling direct compress because the destination table has triggers"))); return false; } if (ts_indexing_relation_has_primary_or_unique_index(state->ctr->root_rel)) { ereport(WARNING, (errmsg("disabling direct compress because the destination table has unique " "constraints"))); return false; } Plan *subplan = mtstate->ps.plan->lefttree; if (subplan->plan_rows < 10) { ereport(WARNING, (errmsg("disabling direct compress because of too small batch size"))); return false; } return true; } /* * ModifyHypertable is a plan node that implements DML for hypertables. * It is a wrapper around the ModifyTable plan node that calls the wrapped ModifyTable * plan. */ static void modify_hypertable_begin(CustomScanState *node, EState *estate, int eflags) { ModifyHypertableState *state = (ModifyHypertableState *) node; ModifyTableState *mtstate; PlanState *ps; ModifyTable *mt = castNode(ModifyTable, &state->mt->plan); /* * To make statement trigger defined on the hypertable work * we need to set the hypertable as the rootRelation otherwise * statement trigger defined only on the hypertable will not fire. */ if (mt->operation == CMD_DELETE || mt->operation == CMD_UPDATE || mt->operation == CMD_MERGE) mt->rootRelation = mt->nominalRelation; ps = ExecInitNode(&mt->plan, estate, eflags); node->custom_ps = list_make1(ps); mtstate = castNode(ModifyTableState, ps); /* * If this is not the primary ModifyTable node, postgres added it to the * beginning of es_auxmodifytables, to be executed by ExecPostprocessPlan. * Unfortunately that strips off the HypertableInsert node leading to * tuple routing not working in INSERTs inside CTEs. To make INSERTs * inside CTEs work we have to fix es_auxmodifytables and add back the * ModifyHypertableState. */ if (estate->es_auxmodifytables && linitial(estate->es_auxmodifytables) == mtstate) linitial(estate->es_auxmodifytables) = node; state->ht = ts_hypertable_cache_get_cache_and_entry(RelationGetRelid( mtstate->resultRelInfo->ri_RelationDesc), CACHE_FLAG_MISSING_OK, &state->ht_cache); /* * If we are inserting into a chunk directly, rri will point to the chunk * itself, so we need to get the hypertable from the chunk. */ if (!state->ht) { Chunk *chunk = ts_chunk_get_by_relid(RelationGetRelid(mtstate->resultRelInfo->ri_RelationDesc), true); state->ht = ts_hypertable_cache_get_entry(state->ht_cache, chunk->hypertable_relid, CACHE_FLAG_NONE); } state->has_continuous_aggregate = ts_hypertable_has_continuous_aggregates(state->ht->fd.id); if (mtstate->operation == CMD_INSERT || mtstate->operation == CMD_MERGE) { /* setup chunk tuple routing state for INSERT/MERGE */ state->ctr = ts_chunk_tuple_routing_create(estate, state->ht, mtstate->resultRelInfo); state->ctr->mht_state = state; if (mtstate->operation == CMD_INSERT && should_use_direct_compress(state)) { state->columnstore_insert = true; state->ctr->create_compressed_chunk = true; } if (mtstate->operation == CMD_MERGE) state->ctr->has_dropped_attrs = rel_has_dropped_attrs(state->ctr->hypertable->main_table_relid); /* setup per tuple exprcontext for tuple routing */ if (!estate->es_per_tuple_exprcontext) estate->es_per_tuple_exprcontext = CreateExprContext(estate); } } static TupleTableSlot * modify_hypertable_exec(CustomScanState *node) { ModifyTableState *mtstate = linitial_node(ModifyTableState, node->custom_ps); return ExecModifyTable(node, &mtstate->ps); } static void modify_hypertable_end(CustomScanState *node) { ModifyHypertableState *state = (ModifyHypertableState *) node; /* * Restore targetlists that were temporarily nullified during EXPLAIN * VERBOSE (see modify_hypertable_explain). This prevents corruption of * cached plans for prepared statements. */ if (state->explain_saved_tlist) { ModifyTableState *mtstate = linitial_node(ModifyTableState, node->custom_ps); Plan *lefttree = mtstate->ps.plan->lefttree; lefttree->targetlist = state->explain_saved_tlist; if (IsA(lefttree, CustomScan) && state->explain_saved_custom_scan_tlist) { castNode(CustomScan, lefttree)->custom_scan_tlist = state->explain_saved_custom_scan_tlist; } state->explain_saved_tlist = NULL; state->explain_saved_custom_scan_tlist = NULL; } if (state->compressor) { ts_cm_functions->compressor_flush(state->compressor, state->bulk_writer); ts_cm_functions->compressor_free(state->compressor, state->bulk_writer); state->compressor = NULL; state->bulk_writer = NULL; } ExecEndNode(linitial(node->custom_ps)); if (state->ctr) ts_chunk_tuple_routing_destroy(state->ctr); ts_cache_release(&state->ht_cache); } static void modify_hypertable_rescan(CustomScanState *node) { ExecReScan(linitial(node->custom_ps)); } /* * Check if the plan is a ChunkAppend, possibly wrapped in one or more * Result nodes (for projection and/or pseudoconstant gating quals like EXISTS). */ static bool is_chunk_append_or_projection(Plan *plan) { while (IsA(plan, Result) && plan->lefttree != NULL) plan = plan->lefttree; return ts_is_chunk_append_plan(plan); } static void modify_hypertable_explain(CustomScanState *node, List *ancestors, ExplainState *es) { ModifyHypertableState *state = (ModifyHypertableState *) node; ModifyTableState *mtstate = linitial_node(ModifyTableState, node->custom_ps); /* * The targetlist for this node will have references that cannot be resolved by * EXPLAIN. So for EXPLAIN VERBOSE we clear the targetlist so that EXPLAIN does not * complain. PostgreSQL does something equivalent and does not print the targetlist * for ModifyTable for EXPLAIN VERBOSE. * * We save the original pointers and restore them in modify_hypertable_end * to avoid corrupting cached Plan trees (e.g. for prepared statements). */ const CmdType operation = ((ModifyTable *) mtstate->ps.plan)->operation; if ((operation == CMD_MERGE || operation == CMD_DELETE) && es->verbose && is_chunk_append_or_projection(mtstate->ps.plan->lefttree)) { Plan *lefttree = mtstate->ps.plan->lefttree; state->explain_saved_tlist = lefttree->targetlist; lefttree->targetlist = NULL; if (IsA(lefttree, CustomScan)) { state->explain_saved_custom_scan_tlist = castNode(CustomScan, lefttree)->custom_scan_tlist; castNode(CustomScan, lefttree)->custom_scan_tlist = NULL; } } /* * Since we hijack the ModifyTable node, instrumentation on ModifyTable will * be missing so we set it to instrumentation of ModifyHypertable node. */ if (mtstate->ps.instrument) { /* * INSERT .. ON CONFLICT statements record few metrics in the ModifyTable node. * So, copy them into ModifyHypertable node before replacing them. */ node->ss.ps.instrument->ntuples2 = mtstate->ps.instrument->ntuples2; node->ss.ps.instrument->nfiltered1 = mtstate->ps.instrument->nfiltered1; } mtstate->ps.instrument = node->ss.ps.instrument; /* * For INSERT we have to read the number of decompressed batches and * tuples from the ChunkTupleRouting state below the ModifyTable. */ if ((mtstate->operation == CMD_INSERT || mtstate->operation == CMD_MERGE) && outerPlanState(mtstate)) { SharedCounters *counters = state->ctr->counters; state->batches_deleted += counters->batches_deleted; state->batches_filtered_decompressed += counters->batches_filtered_decompressed; state->batches_decompressed += counters->batches_decompressed; state->tuples_decompressed += counters->tuples_decompressed; state->batches_scanned += counters->batches_scanned; state->batches_checked_by_bloom += counters->batches_checked_by_bloom; state->batches_pruned_by_bloom += counters->batches_pruned_by_bloom; state->batches_without_bloom += counters->batches_without_bloom; state->batches_bloom_false_positives += counters->batches_bloom_false_positives; } if (state->batches_scanned > 0) ExplainPropertyInteger("Batches scanned", NULL, state->batches_scanned, es); if (state->batches_filtered_compressed > 0) ExplainPropertyInteger("Compressed batches filtered", NULL, state->batches_filtered_compressed, es); if (state->batches_filtered_decompressed > 0) ExplainPropertyInteger("Batches filtered after decompression", NULL, state->batches_filtered_decompressed, es); if (state->batches_decompressed > 0) ExplainPropertyInteger("Batches decompressed", NULL, state->batches_decompressed, es); if (state->tuples_decompressed > 0) ExplainPropertyInteger("Tuples decompressed", NULL, state->tuples_decompressed, es); if (state->batches_deleted > 0) ExplainPropertyInteger("Batches deleted", NULL, state->batches_deleted, es); if (state->batches_checked_by_bloom > 0) ExplainPropertyInteger("Batches checked by bloom", NULL, state->batches_checked_by_bloom, es); if (state->batches_pruned_by_bloom > 0) ExplainPropertyInteger("Batches pruned by bloom", NULL, state->batches_pruned_by_bloom, es); if (state->batches_without_bloom > 0) ExplainPropertyInteger("Batches without bloom", NULL, state->batches_without_bloom, es); if (state->batches_bloom_false_positives > 0) ExplainPropertyInteger("Batches bloom false positives", NULL, state->batches_bloom_false_positives, es); if (ts_guc_enable_direct_compress_insert && state->mt->operation == CMD_INSERT) ExplainPropertyBool("Direct Compress", state->columnstore_insert, es); } static CustomExecMethods modify_hypertable_state_methods = { .CustomName = "ModifyHypertableState", .BeginCustomScan = modify_hypertable_begin, .EndCustomScan = modify_hypertable_end, .ExecCustomScan = modify_hypertable_exec, .ReScanCustomScan = modify_hypertable_rescan, .ExplainCustomScan = modify_hypertable_explain, }; static Node * modify_hypertable_state_create(CustomScan *cscan) { ModifyHypertableState *state; ModifyTable *mt = castNode(ModifyTable, linitial(cscan->custom_plans)); state = (ModifyHypertableState *) newNode(sizeof(ModifyHypertableState), T_CustomScanState); state->cscan_state.methods = &modify_hypertable_state_methods; state->mt = mt; state->mt->arbiterIndexes = linitial(cscan->custom_private); return (Node *) state; } static CustomScanMethods modify_hypertable_plan_methods = { .CustomName = "ModifyHypertable", .CreateCustomScanState = modify_hypertable_state_create, }; bool ts_is_modify_hypertable_plan(Plan *plan) { return IsA(plan, CustomScan) && castNode(CustomScan, plan)->methods == &modify_hypertable_plan_methods; } /* * Make a targetlist to meet CustomScan expectations. * * When a CustomScan isn't scanning a real relation (scanrelid=0), it will build * a virtual TupleDesc for the scan "input" based on custom_scan_tlist. The * "output" targetlist is then expected to reference the attributes of the * input's TupleDesc. Without projection, the targetlist will be only Vars with * varno set to INDEX_VAR (to indicate reference to the TupleDesc instead of a * real relation) and matching the order of the attributes in the TupleDesc. * * Any other order, or non-Vars, will lead to the CustomScan performing * projection. * * Since the CustomScan for hypertable insert just wraps ModifyTable, no * projection is needed, so we'll build a targetlist to avoid this. */ static List * make_var_targetlist(const List *tlist) { List *new_tlist = NIL; ListCell *lc; int resno = 1; foreach (lc, tlist) { TargetEntry *tle = lfirst_node(TargetEntry, lc); Var *var = makeVarFromTargetEntry(INDEX_VAR, tle); var->varattno = resno; new_tlist = lappend(new_tlist, makeTargetEntry(&var->xpr, resno, tle->resname, false)); resno++; } return new_tlist; } /* * Construct the HypertableInsert's target list based on the ModifyTable's * target list, which now exists after having been created by * set_plan_references(). */ void ts_modify_hypertable_fixup_tlist(Plan *plan) { if (IsA(plan, CustomScan)) { CustomScan *cscan = (CustomScan *) plan; if (cscan->methods == &modify_hypertable_plan_methods) { ModifyTable *mt = linitial_node(ModifyTable, cscan->custom_plans); if (mt->plan.targetlist == NIL) { cscan->custom_scan_tlist = NIL; cscan->scan.plan.targetlist = NIL; } else { /* The input is the output of the child ModifyTable node */ cscan->custom_scan_tlist = mt->plan.targetlist; /* The output is a direct mapping of the input */ cscan->scan.plan.targetlist = make_var_targetlist(mt->plan.targetlist); } } } } List * ts_replace_rowid_vars(PlannerInfo *root, List *tlist, int varno) { ListCell *lc; tlist = list_copy(tlist); foreach (lc, tlist) { TargetEntry *tle = lfirst_node(TargetEntry, lc); if (IsA(tle->expr, Var) && castNode(Var, tle->expr)->varno == ROWID_VAR) { tle = copyObject(tle); Var *var = castNode(Var, copyObject(tle->expr)); RowIdentityVarInfo *ridinfo = (RowIdentityVarInfo *) list_nth(root->row_identity_vars, var->varattno - 1); var = copyObject(ridinfo->rowidvar); var->varno = varno; var->varnosyn = 0; var->varattnosyn = 0; tle->expr = (Expr *) var; lfirst(lc) = tle; } } return tlist; } static Plan * modify_hypertable_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *best_path, List *tlist, List *clauses, List *custom_plans) { CustomScan *cscan = makeNode(CustomScan); ModifyTable *mt = linitial_node(ModifyTable, custom_plans); cscan->methods = &modify_hypertable_plan_methods; cscan->custom_plans = custom_plans; cscan->scan.scanrelid = 0; /* Copy costs, etc., from the original plan */ cscan->scan.plan.startup_cost = mt->plan.startup_cost; cscan->scan.plan.total_cost = mt->plan.total_cost; cscan->scan.plan.plan_rows = mt->plan.plan_rows; cscan->scan.plan.plan_width = mt->plan.plan_width; /* The tlist is always NIL since the ModifyTable subplan doesn't have its * targetlist set until set_plan_references (setrefs.c) is run */ Assert(tlist == NIL); /* Target list handling here needs special attention. Intuitively, we'd like * to adopt the target list of the ModifyTable subplan we wrap without * further projection. For a CustomScan this means setting the "input" * custom_scan_tlist to the ModifyTable's target list and having an "output" * targetlist that references the TupleDesc that is created from the * custom_scan_tlist at execution time. Now, while this seems * straight-forward, there are several things with how ModifyTable nodes are * handled in the planner that complicates this: * * - First, ModifyTable doesn't have a targetlist set at this point, and * it is only set later in set_plan_references (setrefs.c) if there's a * RETURNING clause. * * - Second, top-level plan nodes, except for ModifyTable nodes, need to * have a targetlist matching root->processed_tlist. This is asserted in * apply_tlist_labeling, which is called in create_plan (createplan.c) * immediately after this function returns. ModifyTable is exempted * because it doesn't always have a targetlist that matches * processed_tlist. So, even if we had access to ModifyTable's * targetlist here we wouldn't be able to use it since we're a * CustomScan and thus not exempted. * * - Third, a CustomScan's targetlist should reference the attributes of the * TupleDesc that gets created from the custom_scan_tlist at the start of * execution. This means we need to make the targetlist into all Vars with * attribute numbers that correspond to the TupleDesc instead of result * relation in the ModifyTable. * * To get around these issues, we set the targetlist here to * root->processed_tlist, and at the end of planning when the ModifyTable's * targetlist is set, we go back and fix up the CustomScan's targetlist. */ cscan->scan.plan.targetlist = copyObject(root->processed_tlist); /* * For UPDATE/DELETE/MERGE processed_tlist will have ROWID_VAR. We * need to remove those because set_customscan_references will bail * if it sees ROWID_VAR entries in the targetlist. */ if (mt->operation == CMD_UPDATE || mt->operation == CMD_DELETE || mt->operation == CMD_MERGE) { cscan->scan.plan.targetlist = ts_replace_rowid_vars(root, cscan->scan.plan.targetlist, mt->nominalRelation); /* * When the ModifyTable's lefttree contains a ChunkAppend (possibly * wrapped in one or more Result nodes for projection and/or * pseudoconstant gating quals like EXISTS), ChunkAppend will have * already replaced ROWID_VAR entries in its own targetlist to avoid * assertions in set_customscan_references. However, the wrapping * Result nodes' targetlists still contain the original ROWID_VAR * entries. When set_plan_references later calls set_upper_references * on these Result nodes, it tries to resolve the ROWID_VAR entries * against the child's (already replaced) targetlist so we have to * replace ROWID_VAR entries in all Result nodes' targetlists between * ModifyTable and ChunkAppend. */ if (is_chunk_append_or_projection(mt->plan.lefttree)) { Plan *plan = mt->plan.lefttree; while (IsA(plan, Result) && plan->lefttree != NULL) { plan->targetlist = ts_replace_rowid_vars(root, plan->targetlist, mt->nominalRelation); plan = plan->lefttree; } } } cscan->custom_scan_tlist = cscan->scan.plan.targetlist; /* * we save the original list of arbiter indexes here * because we modify that list during execution and * we still need the original list in case that plan * gets reused. */ cscan->custom_private = list_make1(mt->arbiterIndexes); return &cscan->scan.plan; } static CustomPathMethods modify_hypertable_path_methods = { .CustomName = "ModifyHypertablePath", .PlanCustomPath = modify_hypertable_plan_create, }; Path * ts_modify_hypertable_path_create(PlannerInfo *root, ModifyTablePath *mtpath, RelOptInfo *rel) { ModifyHypertablePath *mht_path = palloc0(sizeof(ModifyHypertablePath)); /* Copy costs, etc. */ memcpy(&mht_path->cpath.path, &mtpath->path, sizeof(Path)); mht_path->cpath.path.type = T_CustomPath; mht_path->cpath.path.pathtype = T_CustomScan; mht_path->cpath.custom_paths = list_make1(mtpath); mht_path->cpath.methods = &modify_hypertable_path_methods; return &mht_path->cpath.path; } ================================================ FILE: src/nodes/modify_hypertable.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <foreign/fdwapi.h> #include <nodes/execnodes.h> #include "chunk_tuple_routing.h" #include "hypertable.h" /* Forward declarations */ typedef struct ModifyTableContext ModifyTableContext; typedef struct RowCompressor RowCompressor; typedef struct BulkWriter BulkWriter; typedef struct ModifyHypertablePath { CustomPath cpath; } ModifyHypertablePath; /* * State for the hypertable_modify custom scan node. * * This struct definition is also used in ts_stat_statements, so any new fields * should only be added at the end of the struct. */ typedef struct ModifyHypertableState { CustomScanState cscan_state; ModifyTable *mt; ChunkTupleRouting *ctr; Hypertable *ht; Cache *ht_cache; bool has_continuous_aggregate; RowCompressor *compressor; BulkWriter *bulk_writer; Oid compressor_relid; bool columnstore_insert; bool comp_chunks_processed; Snapshot snapshot; int64 tuples_decompressed; int64 batches_decompressed; int64 batches_filtered_decompressed; int64 batches_deleted; int64 tuples_deleted; int64 batches_scanned; /* bloom stats */ int64 batches_checked_by_bloom; int64 batches_pruned_by_bloom; int64 batches_without_bloom; int64 batches_bloom_false_positives; /* bloom, betadata and null filters */ int64 batches_filtered_compressed; /* * When EXPLAIN VERBOSE is used, we temporarily nullify the targetlist of the * lefttree of the ModifyTable to avoid printing out the full targetlist since * they can't be resolved by EXPLAIN. To not corrupt cached plans we need to * restore them to their original value afterwards. */ List *explain_saved_tlist; List *explain_saved_custom_scan_tlist; } ModifyHypertableState; extern TSDLLEXPORT bool ts_is_modify_hypertable_plan(Plan *plan); extern void ts_modify_hypertable_fixup_tlist(Plan *plan); extern Path *ts_modify_hypertable_path_create(PlannerInfo *root, ModifyTablePath *mtpath, RelOptInfo *input_rel); extern List *ts_replace_rowid_vars(PlannerInfo *root, List *tlist, int varno); TupleTableSlot *ExecModifyTable(CustomScanState *cs_node, PlanState *pstate); TupleTableSlot *ExecInsert(ModifyTableContext *context, ResultRelInfo *resultRelInfo, ChunkTupleRouting *ctr, TupleTableSlot *slot, bool canSetTag); ================================================ FILE: src/nodes/modify_hypertable_exec.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ /* * This file is based on postgresql backend/executor/nodeModifyTable.c * Ordering of functions in this file is based on the order of functions in * nodeModifyTable.c */ /* clang-format off */ /* INTERFACE ROUTINES * ExecInitModifyTable - initialize the ModifyTable node * ExecModifyTable - retrieve the next tuple from the node * ExecEndModifyTable - shut down the ModifyTable node * ExecReScanModifyTable - rescan the ModifyTable node * * NOTES * The ModifyTable node receives input from its outerPlan, which is * the data to insert for INSERT cases, the changed columns' new * values plus row-locating info for UPDATE and MERGE cases, or just the * row-locating info for DELETE cases. * * The relation to modify can be an ordinary table, a foreign table, or a * view. If it's a view, either it has sufficient INSTEAD OF triggers or * this node executes only MERGE ... DO NOTHING. If the original MERGE * targeted a view not in one of those two categories, earlier processing * already pointed the ModifyTable result relation to an underlying * relation of that other view. This node does process * ri_WithCheckOptions, which may have expressions from those other, * automatically updatable views. * * MERGE runs a join between the source relation and the target table. * If any WHEN NOT MATCHED [BY TARGET] clauses are present, then the join * is an outer join that might output tuples without a matching target * tuple. In this case, any unmatched target tuples will have NULL * row-locating info, and only INSERT can be run. But for matched target * tuples, the row-locating info is used to determine the tuple to UPDATE * or DELETE. When all clauses are WHEN MATCHED or WHEN NOT MATCHED BY * SOURCE, all tuples produced by the join will include a matching target * tuple, so all tuples contain row-locating info. * * If the query specifies RETURNING, then the ModifyTable returns a * RETURNING tuple after completing each row insert, update, or delete. * It must be called again to continue the operation. Without RETURNING, * we just loop within the node until all the work is done, then * return NULL. This avoids useless call/return overhead. */ #include <postgres.h> #include <access/tupdesc.h> #include <access/xact.h> #include <catalog/pg_attribute.h> #include <catalog/pg_type.h> #include <executor/execPartition.h> #include <executor/nodeModifyTable.h> #include <foreign/foreign.h> #include <nodes/execnodes.h> #include <nodes/extensible.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <nodes/pg_list.h> #include <nodes/plannodes.h> #include <optimizer/optimizer.h> #include <optimizer/plancat.h> #include <parser/parsetree.h> #include <storage/lmgr.h> #include <utils/builtins.h> #include <utils/lsyscache.h> #include <utils/rel.h> #include <utils/snapmgr.h> #include "cross_module_fn.h" #include "guc.h" #include "hypertable_cache.h" #include "modify_hypertable.h" #include "nodes/chunk_append/chunk_append.h" #include "utils.h" /* * Context struct for a ModifyTable operation, containing basic execution * state and some output variables populated by ExecUpdateAct() and * ExecDeleteAct() to report the result of their actions to callers. */ typedef struct ModifyTableContext { /* Operation state */ ModifyHypertableState *ht_state; ModifyTableState *mtstate; EPQState *epqstate; EState *estate; /* * Slot containing tuple obtained from ModifyTable's subplan. Used to * access "junk" columns that are not going to be stored. */ TupleTableSlot *planSlot; /* MERGE specific */ MergeActionState *relaction; /* MERGE action in progress */ /* * Information about the changes that were made concurrently to a tuple * being updated or deleted */ TM_FailureData tmfd; /* * The tuple produced by EvalPlanQual to retry from, if a * cross-partition UPDATE requires it */ TupleTableSlot *cpUpdateRetrySlot; /* * The tuple projected by the INSERT's RETURNING clause, when doing a * cross-partition UPDATE */ TupleTableSlot *cpUpdateReturningSlot; /* * Lock mode to acquire on the latest tuple version before performing * EvalPlanQual on it */ LockTupleMode lockmode; } ModifyTableContext; /* * Context struct containing output data specific to UPDATE operations. */ typedef struct UpdateContext { bool crossPartUpdate; /* was it a cross-partition update? */ #if PG16_LT bool updateIndexes; /* index update required? */ #else TU_UpdateIndexes updateIndexes; /* Which index updates are required? */ #endif /* * Lock mode to acquire on the latest tuple version before performing * EvalPlanQual on it */ LockTupleMode lockmode; } UpdateContext; static void ExecBatchInsert(ModifyTableState *mtstate, ResultRelInfo *resultRelInfo, TupleTableSlot **slots, TupleTableSlot **planSlots, int numSlots, EState *estate, bool canSetTag); static void ExecPendingInserts(EState *estate); /* static void ExecCrossPartitionUpdateForeignKey(ModifyTableContext *context, ResultRelInfo *sourcePartInfo, ResultRelInfo *destPartInfo, ItemPointer tupleid, TupleTableSlot *oldslot, TupleTableSlot *newslot); */ static bool ExecOnConflictUpdate(ModifyTableContext *context, ResultRelInfo *resultRelInfo, ItemPointer conflictTid, TupleTableSlot *excludedSlot, bool canSetTag, TupleTableSlot **returning); static TupleTableSlot *ExecPrepareTupleRouting(ModifyTableState *mtstate, EState *estate, ChunkTupleRouting *ctr, ResultRelInfo *targetRelInfo, TupleTableSlot *slot, ResultRelInfo **partRelInfo); static TupleTableSlot *ExecMerge(ModifyTableContext *context, ResultRelInfo *resultRelInfo, ChunkTupleRouting *ctr, ItemPointer tupleid, HeapTuple oldtuple, bool canSetTag); /* static void ExecInitMerge(ModifyTableState *mtstate, EState *estate); */ static TupleTableSlot *ExecMergeMatched(ModifyTableContext *context, ResultRelInfo *resultRelInfo, ItemPointer tupleid, HeapTuple oldtuple, bool canSetTag, bool *matched); static TupleTableSlot *ExecMergeNotMatched(ModifyTableContext *context, ResultRelInfo *resultRelInfo, ChunkTupleRouting *ctr, bool canSetTag); /* * Verify that the tuples to be produced by INSERT match the * target relation's rowtype * * We do this to guard against stale plans. If plan invalidation is * functioning properly then we should never get a failure here, but better * safe than sorry. Note that this is called after we have obtained lock * on the target rel, so the rowtype can't change underneath us. * * The plan output is represented by its targetlist, because that makes * handling the dropped-column case easier. * * We used to use this for UPDATE as well, but now the equivalent checks * are done in ExecBuildUpdateProjection. */ static void ExecCheckPlanOutput(Relation resultRel, List *targetList) { TupleDesc resultDesc = RelationGetDescr(resultRel); int attno = 0; ListCell *lc; foreach(lc, targetList) { TargetEntry *tle = (TargetEntry *) lfirst(lc); Form_pg_attribute attr; Assert(!tle->resjunk); /* caller removed junk items already */ if (attno >= resultDesc->natts) ereport(ERROR, (errcode(ERRCODE_DATATYPE_MISMATCH), errmsg("table row type and query-specified row type do not match"), errdetail("Query has too many columns."))); attr = TupleDescAttr(resultDesc, attno); attno++; /* * Special cases here should match planner's expand_insert_targetlist. */ if (attr->attisdropped) { /* * For a dropped column, we can't check atttypid (it's likely 0). * In any case the planner has most likely inserted an INT4 null. * What we insist on is just *some* NULL constant. */ if (!IsA(tle->expr, Const) || !((Const *) tle->expr)->constisnull) ereport(ERROR, (errcode(ERRCODE_DATATYPE_MISMATCH), errmsg("table row type and query-specified row type do not match"), errdetail("Query provides a value for a dropped column at ordinal position %d.", attno))); } else if (attr->attgenerated) { /* * For a generated column, the planner will have inserted a null * of the column's base type (to avoid possibly failing on domain * not-null constraints). It doesn't seem worth insisting on that * exact type though, since a null value is type-independent. As * above, just insist on *some* NULL constant. */ if (!IsA(tle->expr, Const) || !((Const *) tle->expr)->constisnull) ereport(ERROR, (errcode(ERRCODE_DATATYPE_MISMATCH), errmsg("table row type and query-specified row type do not match"), errdetail("Query provides a value for a generated column at ordinal position %d.", attno))); } else { /* Normal case: demand type match */ if (exprType((Node *) tle->expr) != attr->atttypid) ereport(ERROR, (errcode(ERRCODE_DATATYPE_MISMATCH), errmsg("table row type and query-specified row type do not match"), errdetail("Table has type %s at ordinal position %d, but query expects %s.", format_type_be(attr->atttypid), attno, format_type_be(exprType((Node *) tle->expr))))); } } if (attno != resultDesc->natts) ereport(ERROR, (errcode(ERRCODE_DATATYPE_MISMATCH), errmsg("table row type and query-specified row type do not match"), errdetail("Query has too few columns."))); } /* * ExecProcessReturning --- evaluate a RETURNING list * * resultRelInfo: current result rel * cmdType: operation performed (INSERT, UPDATE, or DELETE) * oldSlot: slot holding old tuple deleted or updated * newSlot: slot holding new tuple inserted or updated * planSlot: slot holding tuple returned by top subplan node * * Note: If oldSlot and newSlot are NULL, the FDW should have already provided * econtext's scan tuple and its old & new tuples are not needed (FDW direct- * modify is disabled if the RETURNING list refers to any OLD/NEW values). * * Returns a slot holding the result tuple */ static TupleTableSlot * ExecProcessReturning(ResultRelInfo *resultRelInfo, CmdType cmdType, TupleTableSlot *oldSlot, TupleTableSlot *newSlot, TupleTableSlot *planSlot) { ProjectionInfo *projectReturning = resultRelInfo->ri_projectReturning; ExprContext *econtext = projectReturning->pi_exprContext; TupleTableSlot *tupleSlot = (cmdType == CMD_DELETE) ? oldSlot : newSlot; /* Make tuple and any needed join variables available to ExecProject */ if (tupleSlot) econtext->ecxt_scantuple = tupleSlot; econtext->ecxt_outertuple = planSlot; #if PG18_GE { EState *estate = econtext->ecxt_estate; /* Make old/new tuples available to ExecProject, if required */ if (oldSlot) econtext->ecxt_oldtuple = oldSlot; else if (projectReturning->pi_state.flags & EEO_FLAG_HAS_OLD) econtext->ecxt_oldtuple = ExecGetAllNullSlot(estate, resultRelInfo); else econtext->ecxt_oldtuple = NULL; if (newSlot) econtext->ecxt_newtuple = newSlot; else if (projectReturning->pi_state.flags & EEO_FLAG_HAS_NEW) econtext->ecxt_newtuple = ExecGetAllNullSlot(estate, resultRelInfo); else econtext->ecxt_newtuple = NULL; /* * Tell ExecProject whether or not the OLD/NEW rows actually exist. * This is required to evaluate ReturningExpr nodes and also in * ExecEvalSysVar() and ExecEvalWholeRowVar(). */ if (oldSlot == NULL) projectReturning->pi_state.flags |= EEO_FLAG_OLD_IS_NULL; else projectReturning->pi_state.flags &= ~EEO_FLAG_OLD_IS_NULL; if (newSlot == NULL) projectReturning->pi_state.flags |= EEO_FLAG_NEW_IS_NULL; else projectReturning->pi_state.flags &= ~EEO_FLAG_NEW_IS_NULL; } #else /* * RETURNING expressions might reference the tableoid column, so * reinitialize tts_tableOid before evaluating them. */ econtext->ecxt_scantuple->tts_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc); #endif /* Compute the RETURNING expressions */ return ExecProject(projectReturning); } /* * ExecCheckTupleVisible -- verify tuple is visible * * It would not be consistent with guarantees of the higher isolation levels to * proceed with avoiding insertion (taking speculative insertion's alternative * path) on the basis of another tuple that is not visible to MVCC snapshot. * Check for the need to raise a serialization failure, and do so as necessary. */ static void ExecCheckTupleVisible(EState *estate, Relation rel, TupleTableSlot *slot) { if (!IsolationUsesXactSnapshot()) return; if (!table_tuple_satisfies_snapshot(rel, slot, estate->es_snapshot)) { Datum xminDatum; TransactionId xmin; bool isnull; xminDatum = slot_getsysattr(slot, MinTransactionIdAttributeNumber, &isnull); Assert(!isnull); xmin = DatumGetTransactionId(xminDatum); /* * We should not raise a serialization failure if the conflict is * against a tuple inserted by our own transaction, even if it's not * visible to our snapshot. (This would happen, for example, if * conflicting keys are proposed for insertion in a single command.) */ if (!TransactionIdIsCurrentTransactionId(xmin)) ereport(ERROR, (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("could not serialize access due to concurrent update"))); } } /* * ExecCheckTIDVisible -- convenience variant of ExecCheckTupleVisible() */ static void ExecCheckTIDVisible(EState *estate, ResultRelInfo *relinfo, ItemPointer tid, TupleTableSlot *tempSlot) { Relation rel = relinfo->ri_RelationDesc; /* Redundantly check isolation level */ if (!IsolationUsesXactSnapshot()) return; if (!table_tuple_fetch_row_version(rel, tid, SnapshotAny, tempSlot)) elog(ERROR, "failed to fetch conflicting tuple for ON CONFLICT"); ExecCheckTupleVisible(estate, rel, tempSlot); ExecClearTuple(tempSlot); } /* * ExecInitInsertProjection * Do one-time initialization of projection data for INSERT tuples. * * INSERT queries may need a projection to filter out junk attrs in the tlist. * * This is also a convenient place to verify that the * output of an INSERT matches the target table. */ static void ExecInitInsertProjection(ModifyTableState *mtstate, ResultRelInfo *resultRelInfo) { ModifyTable *node = (ModifyTable *) mtstate->ps.plan; Plan *subplan = outerPlan(node); EState *estate = mtstate->ps.state; List *insertTargetList = NIL; bool need_projection = false; ListCell *l; /* Extract non-junk columns of the subplan's result tlist. */ foreach(l, subplan->targetlist) { TargetEntry *tle = (TargetEntry *) lfirst(l); if (!tle->resjunk) insertTargetList = lappend(insertTargetList, tle); else need_projection = true; } /* * The junk-free list must produce a tuple suitable for the result * relation. */ ExecCheckPlanOutput(resultRelInfo->ri_RelationDesc, insertTargetList); /* We'll need a slot matching the table's format. */ resultRelInfo->ri_newTupleSlot = table_slot_create(resultRelInfo->ri_RelationDesc, &estate->es_tupleTable); /* Build ProjectionInfo if needed (it probably isn't). */ if (need_projection) { TupleDesc relDesc = RelationGetDescr(resultRelInfo->ri_RelationDesc); /* need an expression context to do the projection */ if (mtstate->ps.ps_ExprContext == NULL) ExecAssignExprContext(estate, &mtstate->ps); resultRelInfo->ri_projectNew = ExecBuildProjectionInfo(insertTargetList, mtstate->ps.ps_ExprContext, resultRelInfo->ri_newTupleSlot, &mtstate->ps, relDesc); } resultRelInfo->ri_projectNewInfoValid = true; } /* * ExecInitUpdateProjection * Do one-time initialization of projection data for UPDATE tuples. * * UPDATE always needs a projection, because (1) there's always some junk * attrs, and (2) we may need to merge values of not-updated columns from * the old tuple into the final tuple. In UPDATE, the tuple arriving from * the subplan contains only new values for the changed columns, plus row * identity info in the junk attrs. * * This is "one-time" for any given result rel, but we might touch more than * one result rel in the course of an inherited UPDATE, and each one needs * its own projection due to possible column order variation. * * This is also a convenient place to verify that the output of an UPDATE * matches the target table (ExecBuildUpdateProjection does that). */ static void ExecInitUpdateProjection(ModifyTableState *mtstate, ResultRelInfo *resultRelInfo) { ModifyTable *node = (ModifyTable *) mtstate->ps.plan; Plan *subplan = outerPlan(node); EState *estate = mtstate->ps.state; TupleDesc relDesc = RelationGetDescr(resultRelInfo->ri_RelationDesc); int whichrel; List *updateColnos; /* * Usually, mt_lastResultIndex matches the target rel. If it happens not * to, we can get the index the hard way with an integer division. */ whichrel = mtstate->mt_lastResultIndex; if (resultRelInfo != mtstate->resultRelInfo + whichrel) { whichrel = resultRelInfo - mtstate->resultRelInfo; Assert(whichrel >= 0 && whichrel < mtstate->mt_nrels); } updateColnos = (List *) list_nth(node->updateColnosLists, whichrel); /* * For UPDATE, we use the old tuple to fill up missing values in the tuple * produced by the subplan to get the new tuple. We need two slots, both * matching the table's desired format. */ resultRelInfo->ri_oldTupleSlot = table_slot_create(resultRelInfo->ri_RelationDesc, &estate->es_tupleTable); resultRelInfo->ri_newTupleSlot = table_slot_create(resultRelInfo->ri_RelationDesc, &estate->es_tupleTable); /* need an expression context to do the projection */ if (mtstate->ps.ps_ExprContext == NULL) ExecAssignExprContext(estate, &mtstate->ps); resultRelInfo->ri_projectNew = ExecBuildUpdateProjection(subplan->targetlist, false, /* subplan did the evaluation */ updateColnos, relDesc, mtstate->ps.ps_ExprContext, resultRelInfo->ri_newTupleSlot, &mtstate->ps); resultRelInfo->ri_projectNewInfoValid = true; } /* * ExecGetInsertNewTuple * This prepares a "new" tuple ready to be inserted into given result * relation, by removing any junk columns of the plan's output tuple * and (if necessary) coercing the tuple to the right tuple format. */ static TupleTableSlot * ExecGetInsertNewTuple(ResultRelInfo *relinfo, TupleTableSlot *planSlot) { ProjectionInfo *newProj = relinfo->ri_projectNew; ExprContext *econtext; /* * If there's no projection to be done, just make sure the slot is of the * right type for the target rel. If the planSlot is the right type we * can use it as-is, else copy the data into ri_newTupleSlot. */ if (newProj == NULL) { if (relinfo->ri_newTupleSlot->tts_ops != planSlot->tts_ops) { ExecCopySlot(relinfo->ri_newTupleSlot, planSlot); return relinfo->ri_newTupleSlot; } else return planSlot; } /* * Else project; since the projection output slot is ri_newTupleSlot, this * will also fix any slot-type problem. * * Note: currently, this is dead code, because INSERT cases don't receive * any junk columns so there's never a projection to be done. */ econtext = newProj->pi_exprContext; econtext->ecxt_outertuple = planSlot; return ExecProject(newProj); } /* * ExecPrepareTupleRouting --- prepare for routing one tuple * * Determine the partition in which the tuple in slot is to be inserted, * and return its ResultRelInfo in *partRelInfo. The return value is * a slot holding the tuple of the partition rowtype. * * This also sets the transition table information in mtstate based on the * selected partition. */ static TupleTableSlot * ExecPrepareTupleRouting(ModifyTableState *mtstate, EState *estate, ChunkTupleRouting *ctr, ResultRelInfo *targetRelInfo, TupleTableSlot *slot, ResultRelInfo **partRelInfo) { ChunkInsertState *cis = ctr->cis; /* Convert the tuple to the chunk's rowtype, if necessary */ if (cis->hyper_to_chunk_map != NULL && ctr->has_dropped_attrs == false) slot = execute_attr_map_slot(cis->hyper_to_chunk_map->attrMap, slot, cis->slot); *partRelInfo = cis->result_relation_info; return slot; } /* ---------------------------------------------------------------- * ExecInsert * * For INSERT, we have to insert the tuple into the target relation * (or partition thereof) and insert appropriate tuples into the index * relations. * * slot contains the new tuple value to be stored. * planSlot is the output of the ModifyTable's subplan; we use it * to access "junk" columns that are not going to be stored. * * Returns RETURNING result if any, otherwise NULL. * * This may change the currently active tuple conversion map in * mtstate->mt_transition_capture, so the callers must take care to * save the previous value to avoid losing track of it. * ---------------------------------------------------------------- * * copied and modified version of ExecInsert from executor/nodeModifyTable.c */ TupleTableSlot * ExecInsert(ModifyTableContext *context, ResultRelInfo *resultRelInfo, ChunkTupleRouting *ctr, TupleTableSlot *slot, bool canSetTag) { ModifyTableState *mtstate = context->mtstate; EState *estate = context->estate; Relation resultRelationDesc; List *recheckIndexes = NIL; TupleTableSlot *planSlot = context->planSlot; TupleTableSlot *result = NULL; TransitionCaptureState *ar_insert_trig_tcs; ModifyTable *node = (ModifyTable *) mtstate->ps.plan; OnConflictAction onconflict = node->onConflictAction; /* * If the input result relation is a partitioned table, find the leaf * partition to insert the tuple into. */ if (ctr) { ResultRelInfo *partRelInfo; slot = ExecPrepareTupleRouting(mtstate, estate, ctr, resultRelInfo, slot, &partRelInfo); resultRelInfo = partRelInfo; } ExecMaterializeSlot(slot); resultRelationDesc = resultRelInfo->ri_RelationDesc; /* * Open the table's indexes, if we have not done so already, so that we * can add new index entries for the inserted tuple. */ if (resultRelationDesc->rd_rel->relhasindex && resultRelInfo->ri_IndexRelationDescs == NULL) ExecOpenIndices(resultRelInfo, onconflict != ONCONFLICT_NONE); /* * BEFORE ROW INSERT Triggers. * * Note: We fire BEFORE ROW TRIGGERS for every attempted insertion in an * INSERT ... ON CONFLICT statement. We cannot check for constraint * violations before firing these triggers, because they can change the * values to insert. Also, they can run arbitrary user-defined code with * side-effects that we can't cancel by just not inserting the tuple. */ if (resultRelInfo->ri_TrigDesc && resultRelInfo->ri_TrigDesc->trig_insert_before_row) { if (!ExecBRInsertTriggers(estate, resultRelInfo, slot)) return NULL; /* "do nothing" */ } /* INSTEAD OF ROW INSERT Triggers */ if (resultRelInfo->ri_TrigDesc && resultRelInfo->ri_TrigDesc->trig_insert_instead_row) { if (!ExecIRInsertTriggers(estate, resultRelInfo, slot)) return NULL; /* "do nothing" */ } else if (resultRelInfo->ri_FdwRoutine) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("inserting into foreign tables not supported in hypertable context"))); } else { WCOKind wco_kind; /* * Constraints and GENERATED expressions might reference the tableoid * column, so (re-)initialize tts_tableOid before evaluating them. */ slot->tts_tableOid = RelationGetRelid(resultRelationDesc); /* * Compute stored generated columns */ if (resultRelationDesc->rd_att->constr && resultRelationDesc->rd_att->constr->has_generated_stored) ExecComputeStoredGenerated(resultRelInfo, estate, slot, CMD_INSERT); /* * Check any RLS WITH CHECK policies. * * Normally we should check INSERT policies. But if the insert is the * result of a partition key update that moved the tuple to a new * partition, we should instead check UPDATE policies, because we are * executing policies defined on the target table, and not those * defined on the child partitions. * * If we're running MERGE, we refer to the action that we're executing * to know if we're doing an INSERT or UPDATE to a partition table. */ if (mtstate->operation == CMD_UPDATE) wco_kind = WCO_RLS_UPDATE_CHECK; else if (mtstate->operation == CMD_MERGE) wco_kind = ( #if PG17_GE mtstate->mt_merge_action->mas_action->commandType #else context->relaction->mas_action->commandType #endif == CMD_UPDATE) ? WCO_RLS_UPDATE_CHECK : WCO_RLS_INSERT_CHECK; else wco_kind = WCO_RLS_INSERT_CHECK; /* * ExecWithCheckOptions() will skip any WCOs which are not of the kind * we are looking for at this point. */ if (resultRelInfo->ri_WithCheckOptions != NIL) ExecWithCheckOptions(wco_kind, resultRelInfo, slot, estate); /* * Check the constraints of the tuple. */ if (resultRelationDesc->rd_att->constr) ExecConstraints(resultRelInfo, slot, estate); /* * Also check the tuple against the partition constraint, if there is * one; except that if we got here via tuple-routing, we don't need to * if there's no BR trigger defined on the partition. */ if (resultRelationDesc->rd_rel->relispartition && (resultRelInfo->ri_RootResultRelInfo == NULL || (resultRelInfo->ri_TrigDesc && resultRelInfo->ri_TrigDesc->trig_insert_before_row))) ExecPartitionCheck(resultRelInfo, slot, estate, true); if (onconflict != ONCONFLICT_NONE && resultRelInfo->ri_NumIndices > 0) { /* Perform a speculative insertion. */ uint32 specToken; ItemPointerData conflictTid; bool specConflict; List *arbiterIndexes; arbiterIndexes = resultRelInfo->ri_onConflictArbiterIndexes; /* * Do a non-conclusive check for conflicts first. * * We're not holding any locks yet, so this doesn't guarantee that * the later insert won't conflict. But it avoids leaving behind * a lot of canceled speculative insertions, if you run a lot of * INSERT ON CONFLICT statements that do conflict. * * We loop back here if we find a conflict below, either during * the pre-check, or when we re-check after inserting the tuple * speculatively. */ vlock: specConflict = false; if (!ExecCheckIndexConstraints(resultRelInfo, slot, estate, &conflictTid, #if PG18_GE NULL, #endif arbiterIndexes)) { /* committed conflict tuple found */ if (onconflict == ONCONFLICT_UPDATE) { /* * In case of ON CONFLICT DO UPDATE, execute the UPDATE * part. Be prepared to retry if the UPDATE fails because * of another concurrent UPDATE/DELETE to the conflict * tuple. */ TupleTableSlot *returning = NULL; if (ExecOnConflictUpdate(context, resultRelInfo, &conflictTid, slot, canSetTag, &returning)) { InstrCountTuples2(&mtstate->ps, 1); return returning; } else goto vlock; } else { /* * In case of ON CONFLICT DO NOTHING, do nothing. However, * verify that the tuple is visible to the executor's MVCC * snapshot at higher isolation levels. * * Using ExecGetReturningSlot() to store the tuple for the * recheck isn't that pretty, but we can't trivially use * the input slot, because it might not be of a compatible * type. As there's no conflicting usage of * ExecGetReturningSlot() in the DO NOTHING case... */ Assert(onconflict == ONCONFLICT_NOTHING); ExecCheckTIDVisible(estate, resultRelInfo, &conflictTid, ExecGetReturningSlot(estate, resultRelInfo)); InstrCountTuples2(&mtstate->ps, 1); return NULL; } } /* * Before we start insertion proper, acquire our "speculative * insertion lock". Others can use that to wait for us to decide * if we're going to go ahead with the insertion, instead of * waiting for the whole transaction to complete. */ specToken = SpeculativeInsertionLockAcquire(GetCurrentTransactionId()); /* insert the tuple, with the speculative token */ table_tuple_insert_speculative(resultRelationDesc, slot, estate->es_output_cid, 0, NULL, specToken); /* insert index entries for tuple */ recheckIndexes = ExecInsertIndexTuplesCompat(resultRelInfo, slot, estate, false, true, &specConflict, arbiterIndexes, false); /* adjust the tuple's state accordingly */ table_tuple_complete_speculative(resultRelationDesc, slot, specToken, !specConflict); /* * Wake up anyone waiting for our decision. They will re-check * the tuple, see that it's no longer speculative, and wait on our * XID as if this was a regularly inserted tuple all along. Or if * we killed the tuple, they will see it's dead, and proceed as if * the tuple never existed. */ SpeculativeInsertionLockRelease(GetCurrentTransactionId()); /* * If there was a conflict, start from the beginning. We'll do * the pre-check again, which will now find the conflicting tuple * (unless it aborts before we get there). */ if (specConflict) { list_free(recheckIndexes); goto vlock; } /* Since there was no insertion conflict, we're done */ } else { /* insert the tuple normally */ table_tuple_insert(resultRelationDesc, slot, estate->es_output_cid, 0, NULL); /* insert index entries for tuple */ if (resultRelInfo->ri_NumIndices > 0) recheckIndexes = ExecInsertIndexTuplesCompat(resultRelInfo, slot, estate, false, false, NULL, NIL, false); } } if (context->ht_state->has_continuous_aggregate) { bool should_free; HeapTuple tuple = ExecFetchSlotHeapTuple(slot, false, &should_free); ts_cm_functions->continuous_agg_dml_invalidate(context->ht_state->ht->fd.id, resultRelationDesc, tuple, NULL, false); if (should_free) heap_freetuple(tuple); } if (canSetTag) (estate->es_processed)++; /* * If this insert is the result of a partition key update that moved the * tuple to a new partition, put this row into the transition NEW TABLE, * if there is one. We need to do this separately for DELETE and INSERT * because they happen on different tables. */ ar_insert_trig_tcs = mtstate->mt_transition_capture; if (mtstate->operation == CMD_UPDATE && mtstate->mt_transition_capture && mtstate->mt_transition_capture->tcs_update_new_table) { ExecARUpdateTriggers(estate, resultRelInfo, NULL, /* src_partinfo */ NULL, /* dst_partinfo */ NULL, NULL, slot, NULL, mtstate->mt_transition_capture, false /* is_crosspart_update */ ); /* * We've already captured the NEW TABLE row, so make sure any AR * INSERT trigger fired below doesn't capture it again. */ ar_insert_trig_tcs = NULL; } /* AFTER ROW INSERT Triggers */ ExecARInsertTriggers(estate, resultRelInfo, slot, recheckIndexes, ar_insert_trig_tcs); list_free(recheckIndexes); /* * Check any WITH CHECK OPTION constraints from parent views. We are * required to do this after testing all constraints and uniqueness * violations per the SQL spec, so we do it after actually inserting the * record into the heap and all indexes. * * ExecWithCheckOptions will elog(ERROR) if a violation is found, so the * tuple will never be seen, if it violates the WITH CHECK OPTION. * * ExecWithCheckOptions() will skip any WCOs which are not of the kind we * are looking for at this point. */ if (resultRelInfo->ri_WithCheckOptions != NIL) ExecWithCheckOptions(WCO_VIEW_CHECK, resultRelInfo, slot, estate); /* Process RETURNING if present */ if (resultRelInfo->ri_projectReturning) result = ExecProcessReturning(resultRelInfo, CMD_INSERT, NULL, slot, planSlot); return result; } /* ---------------------------------------------------------------- * ExecBatchInsert * * Insert multiple tuples in an efficient way. * Currently, this handles inserting into a foreign table without * RETURNING clause. * ---------------------------------------------------------------- */ static void ExecBatchInsert(ModifyTableState *mtstate, ResultRelInfo *resultRelInfo, TupleTableSlot **slots, TupleTableSlot **planSlots, int numSlots, EState *estate, bool canSetTag) { int i; int numInserted = numSlots; TupleTableSlot *slot = NULL; TupleTableSlot **rslots; /* * insert into foreign table: let the FDW do it */ rslots = resultRelInfo->ri_FdwRoutine->ExecForeignBatchInsert(estate, resultRelInfo, slots, planSlots, &numInserted); for (i = 0; i < numInserted; i++) { slot = rslots[i]; /* * AFTER ROW Triggers might reference the tableoid column, so * (re-)initialize tts_tableOid before evaluating them. */ slot->tts_tableOid = RelationGetRelid(resultRelInfo->ri_RelationDesc); /* AFTER ROW INSERT Triggers */ ExecARInsertTriggers(estate, resultRelInfo, slot, NIL, mtstate->mt_transition_capture); /* * Check any WITH CHECK OPTION constraints from parent views. See the * comment in ExecInsert. */ if (resultRelInfo->ri_WithCheckOptions != NIL) ExecWithCheckOptions(WCO_VIEW_CHECK, resultRelInfo, slot, estate); } if (canSetTag && numInserted > 0) estate->es_processed += numInserted; /* Clean up all the slots, ready for the next batch */ for (i = 0; i < numSlots; i++) { ExecClearTuple(slots[i]); ExecClearTuple(planSlots[i]); } resultRelInfo->ri_NumSlots = 0; } /* * ExecPendingInserts -- flushes all pending inserts to the foreign tables */ static void ExecPendingInserts(EState *estate) { ListCell *l1, *l2; forboth(l1, estate->es_insert_pending_result_relations, l2, estate->es_insert_pending_modifytables) { ResultRelInfo *resultRelInfo = (ResultRelInfo *) lfirst(l1); ModifyTableState *mtstate = (ModifyTableState *) lfirst(l2); Assert(mtstate); ExecBatchInsert(mtstate, resultRelInfo, resultRelInfo->ri_Slots, resultRelInfo->ri_PlanSlots, resultRelInfo->ri_NumSlots, estate, mtstate->canSetTag); } list_free(estate->es_insert_pending_result_relations); list_free(estate->es_insert_pending_modifytables); estate->es_insert_pending_result_relations = NIL; estate->es_insert_pending_modifytables = NIL; } /* * ExecDeletePrologue -- subroutine for ExecDelete * * Prepare executor state for DELETE. Actually, the only thing we have to do * here is execute BEFORE ROW triggers. We return false if one of them makes * the delete a no-op; otherwise, return true. */ static bool ExecDeletePrologue(ModifyTableContext *context, ResultRelInfo *resultRelInfo, ItemPointer tupleid, HeapTuple oldtuple, TupleTableSlot **epqreturnslot, TM_Result *result) { if (result) *result = TM_Ok; /* BEFORE ROW DELETE triggers */ if (resultRelInfo->ri_TrigDesc && resultRelInfo->ri_TrigDesc->trig_delete_before_row) { /* Flush any pending inserts, so rows are visible to the triggers */ if (context->estate->es_insert_pending_result_relations != NIL) ExecPendingInserts(context->estate); return ExecBRDeleteTriggersCompat(context->estate, context->epqstate, resultRelInfo, tupleid, oldtuple, epqreturnslot, result, &context->tmfd, context->mtstate->operation == CMD_MERGE); } return true; } /* * ExecDeleteAct -- subroutine for ExecDelete * * Actually delete the tuple from a plain table. * * Caller is in charge of doing EvalPlanQual as necessary */ static TM_Result ExecDeleteAct(ModifyTableContext *context, ResultRelInfo *resultRelInfo, ItemPointer tupleid, bool changingPart) { EState *estate = context->estate; return table_tuple_delete(resultRelInfo->ri_RelationDesc, tupleid, estate->es_output_cid, estate->es_snapshot, estate->es_crosscheck_snapshot, true /* wait for commit */ , &context->tmfd, changingPart); } /* * ExecDeleteEpilogue -- subroutine for ExecDelete * * Closing steps of tuple deletion; this invokes AFTER FOR EACH ROW triggers, * including the UPDATE triggers if the deletion is being done as part of a * cross-partition tuple move. */ static void ExecDeleteEpilogue(ModifyTableContext *context, ResultRelInfo *resultRelInfo, ItemPointer tupleid, HeapTuple oldtuple, bool changingPart) { ModifyTableState *mtstate = context->mtstate; EState *estate = context->estate; TransitionCaptureState *ar_delete_trig_tcs; /* * If this delete is the result of a partition key update that moved the * tuple to a new partition, put this row into the transition OLD TABLE, * if there is one. We need to do this separately for DELETE and INSERT * because they happen on different tables. */ ar_delete_trig_tcs = mtstate->mt_transition_capture; if (mtstate->operation == CMD_UPDATE && mtstate->mt_transition_capture && mtstate->mt_transition_capture->tcs_update_old_table) { ExecARUpdateTriggers(estate, resultRelInfo, NULL, NULL, tupleid, oldtuple, NULL, NULL, mtstate->mt_transition_capture, false); /* * We've already captured the OLD TABLE row, so make sure any AR * DELETE trigger fired below doesn't capture it again. */ ar_delete_trig_tcs = NULL; } /* AFTER ROW DELETE Triggers */ ExecARDeleteTriggers(estate, resultRelInfo, tupleid, oldtuple, ar_delete_trig_tcs, changingPart); } /* ---------------------------------------------------------------- * ExecDelete * * DELETE is like UPDATE, except that we delete the tuple and no * index modifications are needed. * * When deleting from a table, tupleid identifies the tuple to delete and * oldtuple is NULL. When deleting through a view INSTEAD OF trigger, * oldtuple is passed to the triggers and identifies what to delete, and * tupleid is invalid. When deleting from a foreign table, tupleid is * invalid; the FDW has to figure out which row to delete using data from * the planSlot. oldtuple is passed to foreign table triggers; it is * NULL when the foreign table has no relevant triggers. We use * tupleDeleted to indicate whether the tuple is actually deleted, * callers can use it to decide whether to continue the operation. When * this DELETE is a part of an UPDATE of partition-key, then the slot * returned by EvalPlanQual() is passed back using output parameter * epqreturnslot. * * Returns RETURNING result if any, otherwise NULL. * ---------------------------------------------------------------- */ static TupleTableSlot * ExecDelete(ModifyTableContext *context, ResultRelInfo *resultRelInfo, ItemPointer tupleid, HeapTuple oldtuple, bool processReturning, bool changingPart, bool canSetTag, TM_Result *tmresult, bool *tupleDeleted, TupleTableSlot **epqreturnslot) { EState *estate = context->estate; Relation resultRelationDesc = resultRelInfo->ri_RelationDesc; TupleTableSlot *slot = NULL; TM_Result result; if (tupleDeleted) *tupleDeleted = false; /* * Prepare for the delete. This includes BEFORE ROW triggers, so we're * done if it says we are. */ if (!ExecDeletePrologue(context, resultRelInfo, tupleid, oldtuple, epqreturnslot, tmresult)) return NULL; /* INSTEAD OF ROW DELETE Triggers */ if (resultRelInfo->ri_TrigDesc && resultRelInfo->ri_TrigDesc->trig_delete_instead_row) { bool dodelete; Assert(oldtuple != NULL); dodelete = ExecIRDeleteTriggers(estate, resultRelInfo, oldtuple); if (!dodelete) /* "do nothing" */ return NULL; } else if (resultRelInfo->ri_FdwRoutine) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("deleting from foreign tables not supported in hypertable context"))); } else { /* * delete the tuple * * Note: if context->estate->es_crosscheck_snapshot isn't * InvalidSnapshot, we check that the row to be deleted is visible to * that snapshot, and throw a can't-serialize error if not. This is a * special-case behavior needed for referential integrity updates in * transaction-snapshot mode transactions. */ ldelete: result = ExecDeleteAct(context, resultRelInfo, tupleid, changingPart); if (tmresult) *tmresult = result; switch (result) { case TM_SelfModified: /* * The target tuple was already updated or deleted by the * current command, or by a later command in the current * transaction. The former case is possible in a join DELETE * where multiple tuples join to the same target tuple. This * is somewhat questionable, but Postgres has always allowed * it: we just ignore additional deletion attempts. * * The latter case arises if the tuple is modified by a * command in a BEFORE trigger, or perhaps by a command in a * volatile function used in the query. In such situations we * should not ignore the deletion, but it is equally unsafe to * proceed. We don't want to discard the original DELETE * while keeping the triggered actions based on its deletion; * and it would be no better to allow the original DELETE * while discarding updates that it triggered. The row update * carries some information that might be important according * to business rules; so throwing an error is the only safe * course. * * If a trigger actually intends this type of interaction, it * can re-execute the DELETE and then return NULL to cancel * the outer delete. */ if (context->tmfd.cmax != estate->es_output_cid) ereport(ERROR, (errcode(ERRCODE_TRIGGERED_DATA_CHANGE_VIOLATION), errmsg("tuple to be deleted was already modified by an operation triggered by the current command"), errhint("Consider using an AFTER trigger instead of a BEFORE trigger to propagate changes to other rows."))); /* Else, already deleted by self; nothing to do */ return NULL; case TM_Ok: break; case TM_Updated: { TupleTableSlot *inputslot; TupleTableSlot *epqslot; if (IsolationUsesXactSnapshot()) ereport(ERROR, (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("could not serialize access due to concurrent update"))); /* * Already know that we're going to need to do EPQ, so * fetch tuple directly into the right slot. */ EvalPlanQualBegin(context->epqstate); inputslot = EvalPlanQualSlot(context->epqstate, resultRelationDesc, resultRelInfo->ri_RangeTableIndex); result = table_tuple_lock(resultRelationDesc, tupleid, estate->es_snapshot, inputslot, estate->es_output_cid, LockTupleExclusive, LockWaitBlock, TUPLE_LOCK_FLAG_FIND_LAST_VERSION, &context->tmfd); switch (result) { case TM_Ok: Assert(context->tmfd.traversed); epqslot = EvalPlanQual(context->epqstate, resultRelationDesc, resultRelInfo->ri_RangeTableIndex, inputslot); if (TupIsNull(epqslot)) /* Tuple not passing quals anymore, exiting... */ return NULL; /* * If requested, skip delete and pass back the * updated row. */ if (epqreturnslot) { *epqreturnslot = epqslot; return NULL; } else goto ldelete; case TM_SelfModified: /* * This can be reached when following an update * chain from a tuple updated by another session, * reaching a tuple that was already updated in * this transaction. If previously updated by this * command, ignore the delete, otherwise error * out. * * See also TM_SelfModified response to * table_tuple_delete() above. */ if (context->tmfd.cmax != estate->es_output_cid) ereport(ERROR, (errcode(ERRCODE_TRIGGERED_DATA_CHANGE_VIOLATION), errmsg("tuple to be deleted was already modified by an operation triggered by the current command"), errhint("Consider using an AFTER trigger instead of a BEFORE trigger to propagate changes to other rows."))); return NULL; case TM_Deleted: /* tuple already deleted; nothing to do */ return NULL; default: /* * TM_Invisible should be impossible because we're * waiting for updated row versions, and would * already have errored out if the first version * is invisible. * * TM_Updated should be impossible, because we're * locking the latest version via * TUPLE_LOCK_FLAG_FIND_LAST_VERSION. */ elog(ERROR, "unexpected table_tuple_lock status: %u", result); return NULL; } Assert(false); break; } case TM_Deleted: if (IsolationUsesXactSnapshot()) ereport(ERROR, (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("could not serialize access due to concurrent delete"))); /* tuple already deleted; nothing to do */ return NULL; default: elog(ERROR, "unrecognized table_tuple_delete status: %u", result); return NULL; } /* * Note: Normally one would think that we have to delete index tuples * associated with the heap tuple now... * * ... but in POSTGRES, we have no need to do this because VACUUM will * take care of it later. We can't delete index tuples immediately * anyway, since the tuple is still visible to other transactions. */ } if (context->ht_state->has_continuous_aggregate) { bool should_free; TupleTableSlot *cagg_slot = table_slot_create(resultRelationDesc, NULL); table_tuple_fetch_row_version(resultRelationDesc, tupleid, SnapshotAny, cagg_slot); HeapTuple tuple = ExecFetchSlotHeapTuple(cagg_slot, false, &should_free); ts_cm_functions->continuous_agg_dml_invalidate(context->ht_state->ht->fd.id, resultRelationDesc, tuple, NULL, false); if (should_free) heap_freetuple(tuple); ExecDropSingleTupleTableSlot(cagg_slot); } if (canSetTag) (estate->es_processed)++; /* Tell caller that the delete actually happened. */ if (tupleDeleted) *tupleDeleted = true; ExecDeleteEpilogue(context, resultRelInfo, tupleid, oldtuple, changingPart); /* Process RETURNING if present and if requested */ if (processReturning && resultRelInfo->ri_projectReturning) { /* * We have to put the target tuple into a slot, which means first we * gotta fetch it. We can use the trigger tuple slot. */ TupleTableSlot *rslot; if (resultRelInfo->ri_FdwRoutine) { /* FDW must have provided a slot containing the deleted row */ Assert(!TupIsNull(slot)); } else { slot = ExecGetReturningSlot(estate, resultRelInfo); if (oldtuple != NULL) { ExecForceStoreHeapTuple(oldtuple, slot, false); } else { if (!table_tuple_fetch_row_version(resultRelationDesc, tupleid, SnapshotAny, slot)) elog(ERROR, "failed to fetch deleted tuple for DELETE RETURNING"); } } rslot = ExecProcessReturning(resultRelInfo, CMD_DELETE, slot, NULL, context->planSlot); /* * Before releasing the target tuple again, make sure rslot has a * local copy of any pass-by-reference values. */ ExecMaterializeSlot(rslot); ExecClearTuple(slot); return rslot; } return NULL; } /* * ExecUpdatePrologue -- subroutine for ExecUpdate * * Prepare executor state for UPDATE. This includes running BEFORE ROW * triggers. We return false if one of them makes the update a no-op; * otherwise, return true. */ static bool ExecUpdatePrologue(ModifyTableContext *context, ResultRelInfo *resultRelInfo, ItemPointer tupleid, HeapTuple oldtuple, TupleTableSlot *slot, TM_Result *result) { Relation resultRelationDesc = resultRelInfo->ri_RelationDesc; if (result) *result = TM_Ok; ExecMaterializeSlot(slot); /* * Open the table's indexes, if we have not done so already, so that we * can add new index entries for the updated tuple. */ if (resultRelationDesc->rd_rel->relhasindex && resultRelInfo->ri_IndexRelationDescs == NULL) ExecOpenIndices(resultRelInfo, false); /* BEFORE ROW UPDATE triggers */ if (resultRelInfo->ri_TrigDesc && resultRelInfo->ri_TrigDesc->trig_update_before_row) { /* Flush any pending inserts, so rows are visible to the triggers */ if (context->estate->es_insert_pending_result_relations != NIL) ExecPendingInserts(context->estate); return ExecBRUpdateTriggersCompat(context->estate, context->epqstate, resultRelInfo, tupleid, oldtuple, slot, result, &context->tmfd, context->mtstate->operation == CMD_MERGE); } return true; } /* * ExecUpdatePrepareSlot -- subroutine for ExecUpdateAct * * Apply the final modifications to the tuple slot before the update. * (This is split out because we also need it in the foreign-table code path.) */ static void ExecUpdatePrepareSlot(ResultRelInfo *resultRelInfo, TupleTableSlot *slot, EState *estate) { Relation resultRelationDesc = resultRelInfo->ri_RelationDesc; /* * Constraints and GENERATED expressions might reference the tableoid * column, so (re-)initialize tts_tableOid before evaluating them. */ slot->tts_tableOid = RelationGetRelid(resultRelationDesc); /* * Compute stored generated columns */ if (resultRelationDesc->rd_att->constr && resultRelationDesc->rd_att->constr->has_generated_stored) ExecComputeStoredGenerated(resultRelInfo, estate, slot, CMD_UPDATE); } /* * ExecUpdateAct -- subroutine for ExecUpdate * * Actually update the tuple, when operating on a plain table. If the * table is a partition, and the command was called referencing an ancestor * partitioned table, this routine migrates the resulting tuple to another * partition. * * The caller is in charge of keeping indexes current as necessary. The * caller is also in charge of doing EvalPlanQual if the tuple is found to * be concurrently updated. However, in case of a cross-partition update, * this routine does it. */ static TM_Result ExecUpdateAct(ModifyTableContext *context, ResultRelInfo *resultRelInfo, ItemPointer tupleid, HeapTuple oldtuple, TupleTableSlot *slot, bool canSetTag, UpdateContext *updateCxt) { EState *estate = context->estate; Relation resultRelationDesc = resultRelInfo->ri_RelationDesc; bool partition_constraint_failed; TM_Result result; updateCxt->crossPartUpdate = false; /* Fill in GENERATEd columns */ ExecUpdatePrepareSlot(resultRelInfo, slot, estate); /* ensure slot is independent, consider e.g. EPQ */ ExecMaterializeSlot(slot); /* * If partition constraint fails, this row might get moved to another * partition, in which case we should check the RLS CHECK policy just * before inserting into the new partition, rather than doing it here. * This is because a trigger on that partition might again change the row. * So skip the WCO checks if the partition constraint fails. */ partition_constraint_failed = resultRelationDesc->rd_rel->relispartition && !ExecPartitionCheck(resultRelInfo, slot, estate, false); /* Check any RLS UPDATE WITH CHECK policies */ if (!partition_constraint_failed && resultRelInfo->ri_WithCheckOptions != NIL) { /* * ExecWithCheckOptions() will skip any WCOs which are not of the kind * we are looking for at this point. */ ExecWithCheckOptions(WCO_RLS_UPDATE_CHECK, resultRelInfo, slot, estate); } /* * If a partition check failed, try to move the row into the right * partition. */ if (partition_constraint_failed) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot update partition key of hypertable"), errdetail("The partition constraint failed, and the row was not moved to another partition."), errhint("Use DELETE and INSERT to change the partition key."))); } /* * Check the constraints of the tuple. We've already checked the * partition constraint above; however, we must still ensure the tuple * passes all other constraints, so we will call ExecConstraints() and * have it validate all remaining checks. */ if (resultRelationDesc->rd_att->constr) ExecConstraints(resultRelInfo, slot, estate); /* * replace the heap tuple * * Note: if es_crosscheck_snapshot isn't InvalidSnapshot, we check that * the row to be updated is visible to that snapshot, and throw a * can't-serialize error if not. This is a special-case behavior needed * for referential integrity updates in transaction-snapshot mode * transactions. */ result = table_tuple_update(resultRelationDesc, tupleid, slot, estate->es_output_cid, estate->es_snapshot, estate->es_crosscheck_snapshot, true /* wait for commit */ , &context->tmfd, &updateCxt->lockmode, &updateCxt->updateIndexes); return result; } /* * ExecUpdateEpilogue -- subroutine for ExecUpdate * * Closing steps of updating a tuple. Must be called if ExecUpdateAct * returns indicating that the tuple was updated. */ static void ExecUpdateEpilogue(ModifyTableContext *context, UpdateContext *updateCxt, ResultRelInfo *resultRelInfo, ItemPointer tupleid, HeapTuple oldtuple, TupleTableSlot *slot) { ModifyTableState *mtstate = context->mtstate; List *recheckIndexes = NIL; /* insert index entries for tuple if necessary */ #if PG15 if (resultRelInfo->ri_NumIndices > 0 && updateCxt->updateIndexes) recheckIndexes = ExecInsertIndexTuples(resultRelInfo, slot, context->estate, true, false, NULL, NIL); #else if (resultRelInfo->ri_NumIndices > 0 && (updateCxt->updateIndexes != TU_None)) recheckIndexes = ExecInsertIndexTuples(resultRelInfo, slot, context->estate, true, false, NULL, NIL, (updateCxt->updateIndexes == TU_Summarizing)); #endif /* AFTER ROW UPDATE Triggers */ ExecARUpdateTriggers(context->estate, resultRelInfo, NULL, NULL, tupleid, oldtuple, slot, recheckIndexes, mtstate->operation == CMD_INSERT ? mtstate->mt_oc_transition_capture : mtstate->mt_transition_capture, false); list_free(recheckIndexes); /* * Check any WITH CHECK OPTION constraints from parent views. We are * required to do this after testing all constraints and uniqueness * violations per the SQL spec, so we do it after actually updating the * record in the heap and all indexes. * * ExecWithCheckOptions() will skip any WCOs which are not of the kind we * are looking for at this point. */ if (resultRelInfo->ri_WithCheckOptions != NIL) ExecWithCheckOptions(WCO_VIEW_CHECK, resultRelInfo, slot, context->estate); } /* ---------------------------------------------------------------- * ExecUpdate * * note: we can't run UPDATE queries with transactions * off because UPDATEs are actually INSERTs and our * scan will mistakenly loop forever, updating the tuple * it just inserted.. This should be fixed but until it * is, we don't want to get stuck in an infinite loop * which corrupts your database.. * * When updating a table, tupleid identifies the tuple to update and * oldtuple is NULL. When updating through a view INSTEAD OF trigger, * oldtuple is passed to the triggers and identifies what to update, and * tupleid is invalid. When updating a foreign table, tupleid is * invalid; the FDW has to figure out which row to update using data from * the planSlot. oldtuple is passed to foreign table triggers; it is * NULL when the foreign table has no relevant triggers. * * slot contains the new tuple value to be stored. * planSlot is the output of the ModifyTable's subplan; we use it * to access values from other input tables (for RETURNING), * row-ID junk columns, etc. * * Returns RETURNING result if any, otherwise NULL. On exit, if tupleid * had identified the tuple to update, it will identify the tuple * actually updated after EvalPlanQual. * ---------------------------------------------------------------- */ static TupleTableSlot * ExecUpdate(ModifyTableContext *context, ResultRelInfo *resultRelInfo, ItemPointer tupleid, HeapTuple oldtuple, TupleTableSlot *oldSlot, TupleTableSlot *slot, bool canSetTag) { EState *estate = context->estate; Relation resultRelationDesc = resultRelInfo->ri_RelationDesc; UpdateContext updateCxt = {0}; TM_Result result; /* * abort the operation if not running transactions */ if (IsBootstrapProcessingMode()) elog(ERROR, "cannot UPDATE during bootstrap"); /* * Prepare for the update. This includes BEFORE ROW triggers, so we're * done if it says we are. */ if (!ExecUpdatePrologue(context, resultRelInfo, tupleid, oldtuple, slot, NULL)) return NULL; /* INSTEAD OF ROW UPDATE Triggers */ if (resultRelInfo->ri_TrigDesc && resultRelInfo->ri_TrigDesc->trig_update_instead_row) { if (!ExecIRUpdateTriggers(estate, resultRelInfo, oldtuple, slot)) return NULL; /* "do nothing" */ } else if (resultRelInfo->ri_FdwRoutine) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("updating foreign tables not supported in hypertable context"))); } else { #if PG16_GE ItemPointerData lockedtid; #endif /* * If we generate a new candidate tuple after EvalPlanQual testing, we * must loop back here to try again. (We don't need to redo triggers, * however. If there are any BEFORE triggers then trigger.c will have * done table_tuple_lock to lock the correct tuple, so there's no need * to do them again.) */ redo_act: #if PG16_GE lockedtid = *tupleid; #endif result = ExecUpdateAct(context, resultRelInfo, tupleid, oldtuple, slot, canSetTag, &updateCxt); /* * If ExecUpdateAct reports that a cross-partition update was done, * then the RETURNING tuple (if any) has been projected and there's * nothing else for us to do. */ if (updateCxt.crossPartUpdate) return context->cpUpdateReturningSlot; switch (result) { case TM_SelfModified: /* * The target tuple was already updated or deleted by the * current command, or by a later command in the current * transaction. The former case is possible in a join UPDATE * where multiple tuples join to the same target tuple. This * is pretty questionable, but Postgres has always allowed it: * we just execute the first update action and ignore * additional update attempts. * * The latter case arises if the tuple is modified by a * command in a BEFORE trigger, or perhaps by a command in a * volatile function used in the query. In such situations we * should not ignore the update, but it is equally unsafe to * proceed. We don't want to discard the original UPDATE * while keeping the triggered actions based on it; and we * have no principled way to merge this update with the * previous ones. So throwing an error is the only safe * course. * * If a trigger actually intends this type of interaction, it * can re-execute the UPDATE (assuming it can figure out how) * and then return NULL to cancel the outer update. */ if (context->tmfd.cmax != estate->es_output_cid) ereport(ERROR, (errcode(ERRCODE_TRIGGERED_DATA_CHANGE_VIOLATION), errmsg("tuple to be updated was already modified by an operation triggered by the current command"), errhint("Consider using an AFTER trigger instead of a BEFORE trigger to propagate changes to other rows."))); /* Else, already updated by self; nothing to do */ return NULL; case TM_Ok: break; case TM_Updated: { TupleTableSlot *inputslot; TupleTableSlot *epqslot; TupleTableSlot *oldSlot; if (IsolationUsesXactSnapshot()) ereport(ERROR, (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("could not serialize access due to concurrent update"))); /* * Already know that we're going to need to do EPQ, so * fetch tuple directly into the right slot. */ inputslot = EvalPlanQualSlot(context->epqstate, resultRelationDesc, resultRelInfo->ri_RangeTableIndex); result = table_tuple_lock(resultRelationDesc, tupleid, estate->es_snapshot, inputslot, estate->es_output_cid, updateCxt.lockmode, LockWaitBlock, TUPLE_LOCK_FLAG_FIND_LAST_VERSION, &context->tmfd); switch (result) { case TM_Ok: Assert(context->tmfd.traversed); epqslot = EvalPlanQual(context->epqstate, resultRelationDesc, resultRelInfo->ri_RangeTableIndex, inputslot); if (TupIsNull(epqslot)) /* Tuple not passing quals anymore, exiting... */ return NULL; /* Make sure ri_oldTupleSlot is initialized. */ if (unlikely(!resultRelInfo->ri_projectNewInfoValid)) ExecInitUpdateProjection(context->mtstate, resultRelInfo); #if PG16_GE if (resultRelInfo->ri_needLockTagTuple) { UnlockTuple(resultRelationDesc, &lockedtid, InplaceUpdateTupleLock); LockTuple(resultRelationDesc, tupleid, InplaceUpdateTupleLock); } #endif /* Fetch the most recent version of old tuple. */ oldSlot = resultRelInfo->ri_oldTupleSlot; if (!table_tuple_fetch_row_version(resultRelationDesc, tupleid, SnapshotAny, oldSlot)) elog(ERROR, "failed to fetch tuple being updated"); slot = ExecGetUpdateNewTuple(resultRelInfo, epqslot, oldSlot); goto redo_act; case TM_Deleted: /* tuple already deleted; nothing to do */ return NULL; case TM_SelfModified: /* * This can be reached when following an update * chain from a tuple updated by another session, * reaching a tuple that was already updated in * this transaction. If previously modified by * this command, ignore the redundant update, * otherwise error out. * * See also TM_SelfModified response to * table_tuple_update() above. */ if (context->tmfd.cmax != estate->es_output_cid) ereport(ERROR, (errcode(ERRCODE_TRIGGERED_DATA_CHANGE_VIOLATION), errmsg("tuple to be updated was already modified by an operation triggered by the current command"), errhint("Consider using an AFTER trigger instead of a BEFORE trigger to propagate changes to other rows."))); return NULL; default: /* see table_tuple_lock call in ExecDelete() */ elog(ERROR, "unexpected table_tuple_lock status: %u", result); return NULL; } } break; case TM_Deleted: if (IsolationUsesXactSnapshot()) ereport(ERROR, (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("could not serialize access due to concurrent delete"))); /* tuple already deleted; nothing to do */ return NULL; default: elog(ERROR, "unrecognized table_tuple_update status: %u", result); return NULL; } } if (canSetTag) (estate->es_processed)++; ExecUpdateEpilogue(context, &updateCxt, resultRelInfo, tupleid, oldtuple, slot); if (context->ht_state->has_continuous_aggregate) { TupleTableSlot *invalidation_slot = NULL; bool should_free_old = false, should_free_new = false; if (!oldtuple) { invalidation_slot = table_slot_create(resultRelationDesc, NULL); table_tuple_fetch_row_version(resultRelationDesc, tupleid, SnapshotAny, invalidation_slot); oldtuple = ExecFetchSlotHeapTuple(invalidation_slot, false, &should_free_old); } HeapTuple newtuple = ExecFetchSlotHeapTuple(slot, false, &should_free_new); ts_cm_functions->continuous_agg_dml_invalidate(context->ht_state->ht->fd.id, resultRelInfo->ri_RelationDesc, oldtuple, newtuple, true); if (should_free_old) heap_freetuple(oldtuple); if (should_free_new) heap_freetuple(newtuple); if (invalidation_slot) ExecDropSingleTupleTableSlot(invalidation_slot); } /* Process RETURNING if present */ if (resultRelInfo->ri_projectReturning) return ExecProcessReturning(resultRelInfo, CMD_UPDATE, oldSlot, slot, context->planSlot); return NULL; } /* * ExecOnConflictUpdate --- execute UPDATE of INSERT ON CONFLICT DO UPDATE * * Try to lock tuple for update as part of speculative insertion. If * a qual originating from ON CONFLICT DO UPDATE is satisfied, update * (but still lock row, even though it may not satisfy estate's * snapshot). * * Returns true if we're done (with or without an update), or false if * the caller must retry the INSERT from scratch. */ static bool ExecOnConflictUpdate(ModifyTableContext *context, ResultRelInfo *resultRelInfo, ItemPointer conflictTid, TupleTableSlot *excludedSlot, bool canSetTag, TupleTableSlot **returning) { ModifyTableState *mtstate = context->mtstate; ExprContext *econtext = mtstate->ps.ps_ExprContext; Relation relation = resultRelInfo->ri_RelationDesc; ExprState *onConflictSetWhere = resultRelInfo->ri_onConflict->oc_WhereClause; TupleTableSlot *existing = resultRelInfo->ri_onConflict->oc_Existing; TM_FailureData tmfd; LockTupleMode lockmode; TM_Result test; Datum xminDatum; TransactionId xmin; bool isnull; /* * Parse analysis should have blocked ON CONFLICT for all system * relations, which includes these. There's no fundamental obstacle to * supporting this; we'd just need to handle LOCKTAG_TUPLE like the other * ExecUpdate() caller. */ #if PG16_GE Assert(!resultRelInfo->ri_needLockTagTuple); #endif /* Determine lock mode to use */ lockmode = ExecUpdateLockMode(context->estate, resultRelInfo); /* * Lock tuple for update. Don't follow updates when tuple cannot be * locked without doing so. A row locking conflict here means our * previous conclusion that the tuple is conclusively committed is not * true anymore. */ test = table_tuple_lock(relation, conflictTid, context->estate->es_snapshot, existing, context->estate->es_output_cid, lockmode, LockWaitBlock, 0, &tmfd); switch (test) { case TM_Ok: /* success! */ break; case TM_Invisible: /* * This can occur when a just inserted tuple is updated again in * the same command. E.g. because multiple rows with the same * conflicting key values are inserted. * * This is somewhat similar to the ExecUpdate() TM_SelfModified * case. We do not want to proceed because it would lead to the * same row being updated a second time in some unspecified order, * and in contrast to plain UPDATEs there's no historical behavior * to break. * * It is the user's responsibility to prevent this situation from * occurring. These problems are why the SQL standard similarly * specifies that for SQL MERGE, an exception must be raised in * the event of an attempt to update the same row twice. */ xminDatum = slot_getsysattr(existing, MinTransactionIdAttributeNumber, &isnull); Assert(!isnull); xmin = DatumGetTransactionId(xminDatum); if (TransactionIdIsCurrentTransactionId(xmin)) ereport(ERROR, (errcode(ERRCODE_CARDINALITY_VIOLATION), /* translator: %s is a SQL command name */ errmsg("%s command cannot affect row a second time", "ON CONFLICT DO UPDATE"), errhint("Ensure that no rows proposed for insertion within the same command have duplicate constrained values."))); /* This shouldn't happen */ elog(ERROR, "attempted to lock invisible tuple"); break; case TM_SelfModified: /* * This state should never be reached. As a dirty snapshot is used * to find conflicting tuples, speculative insertion wouldn't have * seen this row to conflict with. */ elog(ERROR, "unexpected self-updated tuple"); break; case TM_Updated: if (IsolationUsesXactSnapshot()) ereport(ERROR, (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("could not serialize access due to concurrent update"))); /* * As long as we don't support an UPDATE of INSERT ON CONFLICT for * a partitioned table we shouldn't reach to a case where tuple to * be lock is moved to another partition due to concurrent update * of the partition key. */ Assert(!ItemPointerIndicatesMovedPartitions(&tmfd.ctid)); /* * Tell caller to try again from the very start. * * It does not make sense to use the usual EvalPlanQual() style * loop here, as the new version of the row might not conflict * anymore, or the conflicting tuple has actually been deleted. */ ExecClearTuple(existing); return false; case TM_Deleted: if (IsolationUsesXactSnapshot()) ereport(ERROR, (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("could not serialize access due to concurrent delete"))); /* see TM_Updated case */ Assert(!ItemPointerIndicatesMovedPartitions(&tmfd.ctid)); ExecClearTuple(existing); return false; default: elog(ERROR, "unrecognized table_tuple_lock status: %u", test); } /* Success, the tuple is locked. */ /* * Verify that the tuple is visible to our MVCC snapshot if the current * isolation level mandates that. * * It's not sufficient to rely on the check within ExecUpdate() as e.g. * CONFLICT ... WHERE clause may prevent us from reaching that. * * This means we only ever continue when a new command in the current * transaction could see the row, even though in READ COMMITTED mode the * tuple will not be visible according to the current statement's * snapshot. This is in line with the way UPDATE deals with newer tuple * versions. */ ExecCheckTupleVisible(context->estate, relation, existing); /* * Make tuple and any needed join variables available to ExecQual and * ExecProject. The EXCLUDED tuple is installed in ecxt_innertuple, while * the target's existing tuple is installed in the scantuple. EXCLUDED * has been made to reference INNER_VAR in setrefs.c, but there is no * other redirection. */ econtext->ecxt_scantuple = existing; econtext->ecxt_innertuple = excludedSlot; econtext->ecxt_outertuple = NULL; if (!ExecQual(onConflictSetWhere, econtext)) { ExecClearTuple(existing); /* see return below */ InstrCountFiltered1(&mtstate->ps, 1); return true; /* done with the tuple */ } if (resultRelInfo->ri_WithCheckOptions != NIL) { /* * Check target's existing tuple against UPDATE-applicable USING * security barrier quals (if any), enforced here as RLS checks/WCOs. * * The rewriter creates UPDATE RLS checks/WCOs for UPDATE security * quals, and stores them as WCOs of "kind" WCO_RLS_CONFLICT_CHECK, * but that's almost the extent of its special handling for ON * CONFLICT DO UPDATE. * * The rewriter will also have associated UPDATE applicable straight * RLS checks/WCOs for the benefit of the ExecUpdate() call that * follows. INSERTs and UPDATEs naturally have mutually exclusive WCO * kinds, so there is no danger of spurious over-enforcement in the * INSERT or UPDATE path. */ ExecWithCheckOptions(WCO_RLS_CONFLICT_CHECK, resultRelInfo, existing, mtstate->ps.state); } /* Project the new tuple version */ ExecProject(resultRelInfo->ri_onConflict->oc_ProjInfo); /* * Note that it is possible that the target tuple has been modified in * this session, after the above table_tuple_lock. We choose to not error * out in that case, in line with ExecUpdate's treatment of similar cases. * This can happen if an UPDATE is triggered from within ExecQual(), * ExecWithCheckOptions() or ExecProject() above, e.g. by selecting from a * wCTE in the ON CONFLICT's SET. */ /* Execute UPDATE with projection */ *returning = ExecUpdate(context, resultRelInfo, conflictTid, NULL, existing, resultRelInfo->ri_onConflict->oc_ProjSlot, canSetTag); /* * Clear out existing tuple, as there might not be another conflict among * the next input rows. Don't want to hold resources till the end of the * query. First though, make sure that the returning slot has a local * copy of any pass-by-reference values. */ if (*returning != NULL) ExecMaterializeSlot(*returning); ExecClearTuple(existing); return true; } static void fireASTriggers(ModifyTableState *node); static void fireBSTriggers(ModifyTableState *node); static void checkDMLOnFrozenChunk(ResultRelInfo *resultRelInfo); /* ---------------------------------------------------------------- * ExecModifyTable * * Perform table modifications as required, and return RETURNING results * if needed. * ---------------------------------------------------------------- * * modified version of ExecModifyTable from executor/nodeModifyTable.c */ TupleTableSlot * ExecModifyTable(CustomScanState *cs_node, PlanState *pstate) { ModifyHypertableState *ht_state = (ModifyHypertableState *) cs_node; ModifyTableState *node = castNode(ModifyTableState, pstate); ModifyTableContext context; EState *estate = node->ps.state; CmdType operation = node->operation; ResultRelInfo *resultRelInfo; PlanState *subplanstate; TupleTableSlot *slot; TupleTableSlot *oldSlot; ItemPointer tupleid; ItemPointerData tuple_ctid; HeapTupleData oldtupdata; HeapTuple oldtuple; List *relinfos = NIL; ListCell *lc; ChunkTupleRouting *ctr = ht_state->ctr; CHECK_FOR_INTERRUPTS(); /* * This should NOT get called during EvalPlanQual; we should have passed a * subplan tree to EvalPlanQual, instead. Use a runtime test not just * Assert because this condition is easy to miss in testing. (Note: * although ModifyTable should not get executed within an EvalPlanQual * operation, we do have to allow it to be initialized and shut down in * case it is within a CTE subplan. Hence this test must be here, not in * ExecInitModifyTable.) */ if (estate->es_epq_active != NULL) elog(ERROR, "ModifyTable should not be called during EvalPlanQual"); /* * If we've already completed processing, don't try to do more. We need * this test because ExecPostprocessPlan might call us an extra time, and * our subplan's nodes aren't necessarily robust against being called * extra times. */ if (node->mt_done) return NULL; /* * On first call, fire BEFORE STATEMENT triggers before proceeding. */ if (node->fireBSTriggers) { fireBSTriggers(node); node->fireBSTriggers = false; } /* Preload local variables */ resultRelInfo = node->resultRelInfo + node->mt_lastResultIndex; subplanstate = outerPlanState(node); /* * Check for frozen chunk DML operation. * INSERTS are blocked in chunk tuple routing. */ if (operation != CMD_INSERT) checkDMLOnFrozenChunk(resultRelInfo); /* Set global context */ context.ht_state = ht_state; context.mtstate = node; context.epqstate = &node->mt_epqstate; context.estate = estate; /* * For UPDATE/DELETE on compressed hypertable, decompress chunks and * move rows to uncompressed chunks. */ if ((operation == CMD_DELETE || operation == CMD_UPDATE) && !ht_state->comp_chunks_processed) { /* Modify snapshot only if something got decompressed */ if (ts_cm_functions->decompress_target_segments && ts_cm_functions->decompress_target_segments(ht_state)) { ht_state->comp_chunks_processed = true; /* * save snapshot set during ExecutorStart(), since this is the same * snapshot used to SeqScan of uncompressed chunks */ ht_state->snapshot = estate->es_snapshot; CommandCounterIncrement(); /* use a static copy of current transaction snapshot * this needs to be a copy so we don't read trigger updates */ estate->es_snapshot = RegisterSnapshot(GetTransactionSnapshot()); /* mark rows visible */ estate->es_output_cid = GetCurrentCommandId(true); if (ts_guc_max_tuples_decompressed_per_dml > 0) { if (ht_state->tuples_decompressed > ts_guc_max_tuples_decompressed_per_dml) { ereport(ERROR, (errcode(ERRCODE_CONFIGURATION_LIMIT_EXCEEDED), errmsg("tuple decompression limit exceeded by operation"), errdetail("current limit: %d, tuples decompressed: %lld", ts_guc_max_tuples_decompressed_per_dml, (long long int) ht_state->tuples_decompressed), errhint("Consider increasing " "timescaledb.max_tuples_decompressed_per_dml_transaction or " "set to 0 (unlimited)."))); } } } /* Account for tuples deleted via batch DELETE in compressed chunks */ if (operation == CMD_DELETE && ht_state->tuples_deleted > 0) estate->es_processed += ht_state->tuples_deleted; } /* * Fetch rows from subplan, and execute the required table modification * for each row. */ for (;;) { Oid resultoid = InvalidOid; /* * Reset the per-output-tuple exprcontext. This is needed because * triggers expect to use that context as workspace. It's a bit ugly * to do this below the top level of the plan, however. We might need * to rethink this later. */ ResetPerTupleExprContext(estate); /* * Reset per-tuple memory context used for processing on conflict and * returning clauses, to free any expression evaluation storage * allocated in the previous cycle. */ if (pstate->ps_ExprContext) ResetExprContext(pstate->ps_ExprContext); #if PG17_GE /* * If there is a pending MERGE ... WHEN NOT MATCHED [BY TARGET] action * to execute, do so now --- see the comments in ExecMerge(). */ if (node->mt_merge_pending_not_matched != NULL) { context.planSlot = node->mt_merge_pending_not_matched; slot = ExecMergeNotMatched(&context, node->resultRelInfo, ctr, node->canSetTag); /* Clear the pending action */ node->mt_merge_pending_not_matched = NULL; /* * If we got a RETURNING result, return it to the caller. We'll * continue the work on next call. */ if (slot) return slot; continue; /* continue with the next tuple */ } #endif context.planSlot = ExecProcNode(subplanstate); /* No more tuples to process? */ if (TupIsNull(context.planSlot)) break; if (operation == CMD_INSERT || operation == CMD_MERGE) { TupleTableSlot *slot = context.planSlot; if (operation == CMD_MERGE) { /* * XXX do we need an additional support of NOT MATCHED BY SOURCE * for PG >= 17? See PostgreSQL commit 0294df2f1f84 */ #if PG17_GE List *actionStates = ctr->root_rri->ri_MergeActions[MERGE_WHEN_NOT_MATCHED_BY_TARGET]; #else List *actionStates = ctr->root_rri->ri_notMatchedMergeAction; #endif ListCell *l; foreach (l, actionStates) { MergeActionState *action = (MergeActionState *) lfirst(l); CmdType commandType = action->mas_action->commandType; if (commandType == CMD_INSERT) { action->mas_proj->pi_exprContext->ecxt_innertuple = slot; slot = ExecProject(action->mas_proj); break; } } } /* do tuple routing in short lived memory context */ MemoryContext oldctx = MemoryContextSwitchTo(estate->es_per_tuple_exprcontext->ecxt_per_tuple_memory); Point *point = ts_hyperspace_calculate_point(ctr->hypertable->space, slot); /* Find or create the insert state matching the point */ ctr->cis = ts_chunk_tuple_routing_find_chunk(ctr, point); bool update_counter = ctr->cis->onConflictAction == ONCONFLICT_UPDATE; ts_chunk_tuple_routing_decompress_for_insert(ctr->cis, ctr->root_rri, slot, ctr->estate, update_counter); MemoryContextSwitchTo(oldctx); /* ON CONFLICT DO NOTHING optimization for columnstore */ if (operation == CMD_INSERT && ctr->cis->skip_current_tuple) { ctr->cis->skip_current_tuple = false; if (node->ps.instrument) node->ps.instrument->ntuples2++; continue; } /* direct compress */ if (operation == CMD_INSERT && ht_state->columnstore_insert) { ctr->cis->columnstore_insert = true; /* Flush on chunk change */ if (ht_state->compressor && ht_state->compressor_relid != RelationGetRelid(ctr->cis->rel)) { ts_cm_functions->compressor_flush(ht_state->compressor, ht_state->bulk_writer); ts_cm_functions->compressor_free(ht_state->compressor, ht_state->bulk_writer); ht_state->compressor = NULL; ht_state->compressor_relid = InvalidOid; } if (!ht_state->compressor) { bool sort = ts_guc_enable_direct_compress_insert_sort_batches && !ts_guc_enable_direct_compress_insert_client_sorted; ht_state->compressor = ts_cm_functions->compressor_init(ctr->cis->rel, &ht_state->bulk_writer, sort, ts_guc_direct_compress_insert_tuple_sort_limit); ht_state->compressor_relid = RelationGetRelid(ctr->cis->rel); if (ht_state->has_continuous_aggregate) { ts_cm_functions->compressor_set_invalidation(ht_state->compressor, ctr->hypertable, RelationGetRelid(ctr->cis->rel)); } /* if client does not commit to global ordering, set chunk to unordered */ if (!ts_guc_enable_direct_compress_insert_client_sorted) { Chunk *chunk = ts_chunk_get_by_id(ctr->cis->chunk_id, true); if (!ts_chunk_is_unordered(chunk)) ts_chunk_set_unordered(chunk); } } /* * Compute generated stored columns before compressing. * The direct compress path skips ExecInsert, so generated * columns would otherwise remain NULL. */ Relation rel = ctr->cis->rel; if (rel->rd_att->constr && rel->rd_att->constr->has_generated_stored) { slot->tts_tableOid = RelationGetRelid(rel); ExecComputeStoredGenerated(ctr->root_rri, estate, slot, CMD_INSERT); } ts_cm_functions->compressor_add_slot(ht_state->compressor, ht_state->bulk_writer, slot); estate->es_processed++; continue; } /* * copy INSERT merge action list to result relation info of corresponding chunk * * XXX do we need an additional support of NOT MATCHED BY SOURCE * for PG >= 17? See PostgreSQL commit 0294df2f1f84 */ if (operation == CMD_MERGE) #if PG17_GE ctr->cis->result_relation_info->ri_MergeActions[MERGE_WHEN_NOT_MATCHED_BY_TARGET] = resultRelInfo->ri_MergeActions[MERGE_WHEN_NOT_MATCHED_BY_TARGET]; #else ctr->cis->result_relation_info->ri_notMatchedMergeAction = resultRelInfo->ri_notMatchedMergeAction; #endif } /* * When there are multiple result relations, each tuple contains a * junk column that gives the OID of the rel from which it came. * Extract it and select the correct result relation. */ if (AttributeNumberIsValid(node->mt_resultOidAttno)) { Datum datum; bool isNull; datum = ExecGetJunkAttribute(context.planSlot, node->mt_resultOidAttno, &isNull); if (isNull) { if (operation == CMD_MERGE) { EvalPlanQualSetSlot(&node->mt_epqstate, context.planSlot); slot = ExecMerge(&context, node->resultRelInfo, ctr, NULL, NULL, node->canSetTag); if (slot) return slot; continue; } elog(ERROR, "tableoid is NULL"); } resultoid = DatumGetObjectId(datum); /* If it's not the same as last time, we need to locate the rel */ if (resultoid != node->mt_lastResultOid) { resultRelInfo = ExecLookupResultRelByOid(node, resultoid, false, true); checkDMLOnFrozenChunk(resultRelInfo); } } /* * If resultRelInfo->ri_usesFdwDirectModify is true, all we need to do * here is compute the RETURNING expressions. */ if (resultRelInfo->ri_usesFdwDirectModify) { Assert(resultRelInfo->ri_projectReturning); /* * A scan slot containing the data that was actually inserted, * updated or deleted has already been made available to * ExecProcessReturning by IterateDirectModify, so no need to * provide it here. */ slot = ExecProcessReturning(resultRelInfo, operation, NULL, NULL, context.planSlot); return slot; } EvalPlanQualSetSlot(&node->mt_epqstate, context.planSlot); slot = context.planSlot; tupleid = NULL; oldtuple = NULL; /* * For UPDATE/DELETE, fetch the row identity info for the tuple to be * updated/deleted. For a heap relation, that's a TID; otherwise we * may have a wholerow junk attr that carries the old tuple in toto. * Keep this in step with the part of ExecInitModifyTable that sets up * ri_RowIdAttNo. */ if (operation == CMD_UPDATE || operation == CMD_DELETE || operation == CMD_MERGE) { char relkind; Datum datum; bool isNull; relkind = resultRelInfo->ri_RelationDesc->rd_rel->relkind; /* Since this is a hypertable relkind should be RELKIND_RELATION for a local * chunk or RELKIND_FOREIGN_TABLE for a chunk that is a foreign table * (OSM chunks) */ Assert(relkind == RELKIND_RELATION || relkind == RELKIND_FOREIGN_TABLE); if (relkind == RELKIND_RELATION || relkind == RELKIND_MATVIEW || relkind == RELKIND_PARTITIONED_TABLE) { /* ri_RowIdAttNo refers to a ctid attribute */ Assert(AttributeNumberIsValid(resultRelInfo->ri_RowIdAttNo)); datum = ExecGetJunkAttribute(slot, resultRelInfo->ri_RowIdAttNo, &isNull); /* * For commands other than MERGE, any tuples having a null row * identifier are errors. For MERGE, we may need to handle * them as WHEN NOT MATCHED clauses if any, so do that. * * Note that we use the node's toplevel resultRelInfo, not any * specific partition's. */ if (isNull) { if (operation == CMD_MERGE) { EvalPlanQualSetSlot(&node->mt_epqstate, context.planSlot); slot = ExecMerge(&context, node->resultRelInfo, ctr, NULL, NULL, node->canSetTag); if (slot) return slot; continue; } elog(ERROR, "ctid is NULL"); } tupleid = (ItemPointer) DatumGetPointer(datum); tuple_ctid = *tupleid; /* be sure we don't free ctid!! */ tupleid = &tuple_ctid; } /* * Use the wholerow attribute, when available, to reconstruct the * old relation tuple. The old tuple serves one or both of two * purposes: 1) it serves as the OLD tuple for row triggers, 2) it * provides values for any unchanged columns for the NEW tuple of * an UPDATE, because the subplan does not produce all the columns * of the target table. * * Note that the wholerow attribute does not carry system columns, * so foreign table triggers miss seeing those, except that we * know enough here to set t_tableOid. Quite separately from * this, the FDW may fetch its own junk attrs to identify the row. * * Other relevant relkinds, currently limited to views, always * have a wholerow attribute. */ else if (AttributeNumberIsValid(resultRelInfo->ri_RowIdAttNo)) { datum = ExecGetJunkAttribute(slot, resultRelInfo->ri_RowIdAttNo, &isNull); #if PG17_GE if (isNull) { if (operation == CMD_MERGE) { EvalPlanQualSetSlot(&node->mt_epqstate, context.planSlot); slot = ExecMerge(&context, node->resultRelInfo, ctr, NULL, NULL, node->canSetTag); if (slot) return slot; continue; } elog(ERROR, "wholerow is NULL"); } #else /* shouldn't ever get a null result... */ if (isNull) elog(ERROR, "wholerow is NULL"); #endif oldtupdata.t_data = DatumGetHeapTupleHeader(datum); oldtupdata.t_len = HeapTupleHeaderGetDatumLength(oldtupdata.t_data); ItemPointerSetInvalid(&(oldtupdata.t_self)); /* Historically, view triggers see invalid t_tableOid. */ oldtupdata.t_tableOid = (relkind == RELKIND_VIEW) ? InvalidOid : RelationGetRelid(resultRelInfo->ri_RelationDesc); oldtuple = &oldtupdata; } else { /* Only foreign tables are allowed to omit a row-ID attr */ Assert(relkind == RELKIND_FOREIGN_TABLE); } } switch (operation) { case CMD_INSERT: /* Initialize projection info if first time for this table */ if (unlikely(!resultRelInfo->ri_projectNewInfoValid)) ExecInitInsertProjection(node, resultRelInfo); slot = ExecGetInsertNewTuple(resultRelInfo, context.planSlot); slot = ExecInsert(&context, resultRelInfo, ctr, slot, node->canSetTag); break; case CMD_UPDATE: /* Initialize projection info if first time for this table */ if (unlikely(!resultRelInfo->ri_projectNewInfoValid)) ExecInitUpdateProjection(node, resultRelInfo); /* * Make the new tuple by combining plan's output tuple with * the old tuple being updated. */ oldSlot = resultRelInfo->ri_oldTupleSlot; if (oldtuple != NULL) { /* Use the wholerow junk attr as the old tuple. */ ExecForceStoreHeapTuple(oldtuple, oldSlot, false); } else { /* Fetch the most recent version of old tuple. */ Relation relation = resultRelInfo->ri_RelationDesc; Assert(tupleid != NULL); if (!table_tuple_fetch_row_version(relation, tupleid, SnapshotAny, oldSlot)) elog(ERROR, "failed to fetch tuple being updated"); } slot = ExecGetUpdateNewTuple(resultRelInfo, context.planSlot, oldSlot); context.relaction = NULL; /* Now apply the update. */ slot = ExecUpdate(&context, resultRelInfo, tupleid, oldtuple, oldSlot, slot, node->canSetTag); break; case CMD_DELETE: slot = ExecDelete(&context, resultRelInfo, tupleid, oldtuple, true, false, node->canSetTag, NULL, NULL, NULL); break; case CMD_MERGE: slot = ExecMerge(&context, resultRelInfo, ctr, tupleid, oldtuple, node->canSetTag); break; default: elog(ERROR, "unknown operation"); break; } /* * If we got a RETURNING result, return it to caller. We'll continue * the work on next call. */ if (slot) return slot; } /* * Insert remaining tuples for batch insert. */ relinfos = estate->es_opened_result_relations; if (ht_state->comp_chunks_processed) { UnregisterSnapshot(estate->es_snapshot); estate->es_snapshot = ht_state->snapshot; ht_state->comp_chunks_processed = false; } foreach (lc, relinfos) { resultRelInfo = lfirst(lc); if (resultRelInfo->ri_NumSlots > 0) ExecBatchInsert(node, resultRelInfo, resultRelInfo->ri_Slots, resultRelInfo->ri_PlanSlots, resultRelInfo->ri_NumSlots, estate, node->canSetTag); } /* * We're done, but fire AFTER STATEMENT triggers before exiting. */ fireASTriggers(node); node->mt_done = true; return NULL; } /* * Process BEFORE EACH STATEMENT triggers * * copied verbatim from executor/nodeModifyTable.c */ static void fireBSTriggers(ModifyTableState *node) { ModifyTable *plan = (ModifyTable *) node->ps.plan; ResultRelInfo *resultRelInfo = node->rootResultRelInfo; switch (node->operation) { case CMD_INSERT: ExecBSInsertTriggers(node->ps.state, resultRelInfo); if (plan->onConflictAction == ONCONFLICT_UPDATE) ExecBSUpdateTriggers(node->ps.state, resultRelInfo); break; case CMD_UPDATE: ExecBSUpdateTriggers(node->ps.state, resultRelInfo); break; case CMD_DELETE: ExecBSDeleteTriggers(node->ps.state, resultRelInfo); break; case CMD_MERGE: if (node->mt_merge_subcommands & MERGE_INSERT) ExecBSInsertTriggers(node->ps.state, resultRelInfo); if (node->mt_merge_subcommands & MERGE_UPDATE) ExecBSUpdateTriggers(node->ps.state, resultRelInfo); if (node->mt_merge_subcommands & MERGE_DELETE) ExecBSDeleteTriggers(node->ps.state, resultRelInfo); break; default: elog(ERROR, "unknown operation"); break; } } /* * Process AFTER EACH STATEMENT triggers * * copied verbatim from executor/nodeModifyTable.c */ static void fireASTriggers(ModifyTableState *node) { ModifyTable *plan = (ModifyTable *) node->ps.plan; ResultRelInfo *resultRelInfo = node->rootResultRelInfo; switch (node->operation) { case CMD_INSERT: if (plan->onConflictAction == ONCONFLICT_UPDATE) ExecASUpdateTriggers(node->ps.state, resultRelInfo, node->mt_oc_transition_capture); ExecASInsertTriggers(node->ps.state, resultRelInfo, node->mt_transition_capture); break; case CMD_UPDATE: ExecASUpdateTriggers(node->ps.state, resultRelInfo, node->mt_transition_capture); break; case CMD_DELETE: ExecASDeleteTriggers(node->ps.state, resultRelInfo, node->mt_transition_capture); break; case CMD_MERGE: if (node->mt_merge_subcommands & MERGE_INSERT) ExecASInsertTriggers(node->ps.state, resultRelInfo, node->mt_transition_capture); if (node->mt_merge_subcommands & MERGE_UPDATE) ExecASUpdateTriggers(node->ps.state, resultRelInfo, node->mt_transition_capture); if (node->mt_merge_subcommands & MERGE_DELETE) ExecASDeleteTriggers(node->ps.state, resultRelInfo, node->mt_transition_capture); break; default: elog(ERROR, "unknown operation"); break; } } static void checkDMLOnFrozenChunk(ResultRelInfo *resultRelInfo) { Chunk *chunk = ts_chunk_get_by_relid(resultRelInfo->ri_RelationDesc->rd_id, false); if (chunk && ts_chunk_is_frozen(chunk)) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("cannot update/delete rows from chunk \"%s\" as it is frozen", get_rel_name(resultRelInfo->ri_RelationDesc->rd_id)))); } } /* * Check and execute the first qualifying MATCHED action. The current target * tuple is identified by tupleid. * * We start from the first WHEN MATCHED action and check if the WHEN quals * pass, if any. If the WHEN quals for the first action do not pass, we check * the second, then the third and so on. If we reach to the end, no action is * taken and we return true, indicating that no further action is required * for this tuple. * * If we do find a qualifying action, then we attempt to execute the action. * * If the tuple is concurrently updated, EvalPlanQual is run with the updated * tuple to recheck the join quals. Note that the additional quals associated * with individual actions are evaluated by this routine via ExecQual, while * EvalPlanQual checks for the join quals. If EvalPlanQual tells us that the * updated tuple still passes the join quals, then we restart from the first * action to look for a qualifying action. Otherwise, we return false -- * meaning that a NOT MATCHED action must now be executed for the current * source tuple. */ TupleTableSlot * ExecMergeMatched(ModifyTableContext *context, ResultRelInfo *resultRelInfo, ItemPointer tupleid, HeapTuple oldtuple, bool canSetTag, bool *matched) { ModifyTableState *mtstate = context->mtstate; TupleTableSlot *newslot = NULL; EState *estate = context->estate; ExprContext *econtext = mtstate->ps.ps_ExprContext; bool isNull; EPQState *epqstate = &mtstate->mt_epqstate; ListCell *l; TupleTableSlot *rslot = NULL; Assert(*matched == true); /* * If there are no WHEN MATCHED actions, we are done. */ #if PG17_GE if (resultRelInfo->ri_MergeActions[MERGE_WHEN_MATCHED] == NIL) return NULL; #else if (resultRelInfo->ri_matchedMergeAction == NIL) { *matched = true; return NULL; } #endif /* * Make tuple and any needed join variables available to ExecQual and * ExecProject. The target's existing tuple is installed in the * scantuple. Again, this target relation's slot is required only in * the case of a MATCHED tuple and UPDATE/DELETE actions. */ econtext->ecxt_scantuple = resultRelInfo->ri_oldTupleSlot; econtext->ecxt_innertuple = context->planSlot; econtext->ecxt_outertuple = NULL; lmerge_matched:; /* * This routine is only invoked for matched rows, and we must have * found the tupleid of the target row in that case; fetch that * tuple. * * We use SnapshotAny for this because we might get called again * after EvalPlanQual returns us a new tuple, which may not be * visible to our MVCC snapshot. */ #if PG17_GE if (oldtuple != NULL) ExecForceStoreHeapTuple(oldtuple, resultRelInfo->ri_oldTupleSlot, false); else #endif if (!table_tuple_fetch_row_version(resultRelInfo->ri_RelationDesc, tupleid, SnapshotAny, resultRelInfo->ri_oldTupleSlot)) elog(ERROR, "failed to fetch the target tuple"); #if PG17_GE foreach (l, resultRelInfo->ri_MergeActions[MERGE_WHEN_MATCHED]) { #else foreach (l, resultRelInfo->ri_matchedMergeAction) { #endif MergeActionState *relaction = (MergeActionState *) lfirst(l); CmdType commandType = relaction->mas_action->commandType; TM_Result result; UpdateContext updateCxt = { 0 }; /* * Test condition, if any. * * In the absence of any condition, we perform the action * unconditionally (no need to check separately since * ExecQual() will return true if there are no conditions to * evaluate). */ if (!ExecQual(relaction->mas_whenqual, econtext)) continue; /* * Check if the existing target tuple meets the USING checks * of UPDATE/DELETE RLS policies. If those checks fail, we * throw an error. * * The WITH CHECK quals are applied in ExecUpdate() and hence * we need not do anything special to handle them. * * NOTE: We must do this after WHEN quals are evaluated, so * that we check policies only when they matter. */ if (resultRelInfo->ri_WithCheckOptions) { ExecWithCheckOptions(commandType == CMD_UPDATE ? WCO_RLS_MERGE_UPDATE_CHECK : WCO_RLS_MERGE_DELETE_CHECK, resultRelInfo, resultRelInfo->ri_oldTupleSlot, context->mtstate->ps.state); } /* Perform stated action */ switch (commandType) { case CMD_UPDATE: /* * Project the output tuple, and use that to update * the table. We don't need to filter out junk * attributes, because the UPDATE action's targetlist * doesn't have any. */ newslot = ExecProject(relaction->mas_proj); #if PG17_GE mtstate->mt_merge_action = relaction; #else context->relaction = relaction; #endif context->cpUpdateRetrySlot = NULL; if (!ExecUpdatePrologue(context, resultRelInfo, tupleid, NULL, newslot, &result)) { #if PG16_LT result = TM_Ok; #else if (result == TM_Ok) return NULL; #endif /* if not TM_OK, it is concurrent update/delete */ break; } ExecUpdatePrepareSlot(resultRelInfo, newslot, context->estate); result = ExecUpdateAct(context, resultRelInfo, tupleid, NULL, newslot, mtstate->canSetTag, &updateCxt); if (result == TM_Ok) { ExecUpdateEpilogue(context, &updateCxt, resultRelInfo, tupleid, NULL, newslot); mtstate->mt_merge_updated += 1; } break; case CMD_DELETE: #if PG17_GE mtstate->mt_merge_action = relaction; #else context->relaction = relaction; #endif if (!ExecDeletePrologue(context, resultRelInfo, tupleid, NULL, NULL, &result)) { #if PG16_LT result = TM_Ok; #else if (result == TM_Ok) return NULL; /* "do nothing" */ /* if not TM_OK, it is concurrent update/delete */ #endif break; } result = ExecDeleteAct(context, resultRelInfo, tupleid, false); if (result == TM_Ok) { ExecDeleteEpilogue(context, resultRelInfo, tupleid, NULL, false); mtstate->mt_merge_deleted = 1; } break; case CMD_NOTHING: /* Doing nothing is always OK */ result = TM_Ok; break; default: elog(ERROR, "unknown action in MERGE WHEN MATCHED clause"); } switch (result) { case TM_Ok: /* all good; perform final actions */ if (canSetTag) (estate->es_processed)++; break; case TM_SelfModified: if (context->tmfd.cmax != estate->es_output_cid) ereport(ERROR, (errcode(ERRCODE_TRIGGERED_DATA_CHANGE_VIOLATION), errmsg("tuple to be updated or deleted was already modified by an " "operation triggered by the current command"), errhint("Consider using an AFTER trigger instead of a BEFORE trigger " "to propagate changes to other rows."))); if (TransactionIdIsCurrentTransactionId(context->tmfd.xmax)) ereport(ERROR, (errcode(ERRCODE_CARDINALITY_VIOLATION), /* translator: %s is a SQL command name */ errmsg("%s command cannot affect row a second time", "MERGE"), errhint("Ensure that not more than one source row matches any one " "target row."))); /* This shouldn't happen */ elog(ERROR, "attempted to update or delete invisible tuple"); break; case TM_Deleted: if (IsolationUsesXactSnapshot()) ereport(ERROR, (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("could not serialize access due to concurrent delete"))); /* * If the tuple was already deleted, return to let * caller handle it under NOT MATCHED clauses. */ *matched = false; return NULL; case TM_Updated: { Relation resultRelationDesc; TupleTableSlot *epqslot, *inputslot; LockTupleMode lockmode; /* * The target tuple was concurrently updated * by some other transaction. */ /* * If cpUpdateRetrySlot is set, * ExecCrossPartitionUpdate() must have * detected that the tuple was concurrently * updated, so we restart the search for an * appropriate WHEN MATCHED clause to process * the updated tuple. * * In this case, ExecDelete() would already * have performed EvalPlanQual() on the * latest version of the tuple, which in turn * would already have been loaded into * ri_oldTupleSlot, so no need to do either * of those things. * * XXX why do we not check the WHEN NOT * MATCHED list in this case? */ if (!TupIsNull(context->cpUpdateRetrySlot)) goto lmerge_matched; /* * Otherwise, we run the EvalPlanQual() with * the new version of the tuple. If * EvalPlanQual() does not return a tuple, * then we switch to the NOT MATCHED list of * actions. If it does return a tuple and the * join qual is still satisfied, then we just * need to recheck the MATCHED actions, * starting from the top, and execute the * first qualifying action. */ resultRelationDesc = resultRelInfo->ri_RelationDesc; lockmode = ExecUpdateLockMode(estate, resultRelInfo); inputslot = EvalPlanQualSlot(epqstate, resultRelationDesc, resultRelInfo->ri_RangeTableIndex); result = table_tuple_lock(resultRelationDesc, tupleid, estate->es_snapshot, inputslot, estate->es_output_cid, lockmode, LockWaitBlock, TUPLE_LOCK_FLAG_FIND_LAST_VERSION, &context->tmfd); switch (result) { case TM_Ok: // TODO: update this to match PG17 epqslot = EvalPlanQual(epqstate, resultRelationDesc, resultRelInfo->ri_RangeTableIndex, inputslot); /* * If we got no tuple, or the tuple * we get has a NULL ctid, go back to * caller: this one is not a MATCHED * tuple anymore, so they can retry * with NOT MATCHED actions. */ if (TupIsNull(epqslot)) return false; (void) ExecGetJunkAttribute(epqslot, resultRelInfo->ri_RowIdAttNo, &isNull); if (isNull) return false; /* * When a tuple was updated and * migrated to another partition * concurrently, the current MERGE * implementation can't follow. * There's probably a better way to * handle this case, but it'd require * recognizing the relation to which * the tuple moved, and setting our * current resultRelInfo to that. */ if (ItemPointerIndicatesMovedPartitions(&context->tmfd.ctid)) ereport(ERROR, (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("tuple to be deleted was already moved to another " "partition due to concurrent update"))); /* * A non-NULL ctid means that we are * still dealing with MATCHED case. * Restart the loop so that we apply * all the MATCHED rules again, to * ensure that the first qualifying * WHEN MATCHED action is executed. * * Update tupleid to that of the new * tuple, for the refetch we do at * the top. */ Ensure(tupleid != NULL, "matched tupleid during merge cannot be null"); ItemPointerCopy(&context->tmfd.ctid, tupleid); goto lmerge_matched; case TM_Deleted: /* * tuple already deleted; tell caller * to run NOT MATCHED actions */ *matched = false; return NULL; case TM_SelfModified: /* * This can be reached when following * an update chain from a tuple * updated by another session, * reaching a tuple that was already * updated in this transaction. If * previously modified by this * command, ignore the redundant * update, otherwise error out. * * See also response to * TM_SelfModified in * ht_ExecUpdate(). */ if (context->tmfd.cmax != estate->es_output_cid) ereport(ERROR, (errcode(ERRCODE_TRIGGERED_DATA_CHANGE_VIOLATION), errmsg("tuple to be updated or deleted was already modified " "by an operation triggered by the current command"), errhint("Consider using an AFTER trigger instead of a BEFORE " "trigger to propagate changes to other rows."))); if (TransactionIdIsCurrentTransactionId(context->tmfd.xmax)) ereport(ERROR, (errcode(ERRCODE_CARDINALITY_VIOLATION), /* translator: %s is a SQL command name */ errmsg("%s command cannot affect row a second time", "MERGE"), errhint("Ensure that not more than one source row matches any " "one target row."))); /* This shouldn't happen */ elog(ERROR, "attempted to update or delete invisible tuple"); return NULL; default: /* * see table_tuple_lock call in * ht_ExecDelete() */ elog(ERROR, "unexpected table_tuple_lock status: %u", result); return NULL; } } case TM_Invisible: case TM_WouldBlock: case TM_BeingModified: /* these should not occur */ elog(ERROR, "unexpected tuple operation result: %d", result); break; } #if PG17_GE /* Process RETURNING if present */ if (resultRelInfo->ri_projectReturning) { switch (commandType) { case CMD_UPDATE: /* Variable newslot should be set for CMD_UPDATE above */ Assert(newslot != NULL); rslot = ExecProcessReturning(resultRelInfo, CMD_UPDATE, resultRelInfo->ri_oldTupleSlot, newslot, context->planSlot); break; case CMD_DELETE: rslot = ExecProcessReturning(resultRelInfo, CMD_DELETE, resultRelInfo->ri_oldTupleSlot, NULL, context->planSlot); break; case CMD_NOTHING: break; default: elog(ERROR, "unrecognized commandType: %d", (int) commandType); } } #endif /* * We've activated one of the WHEN clauses, so we don't * search further. This is required behaviour, not an * optimization. */ break; } /* * Successfully executed an action or no qualifying action was found. */ return rslot; } /* * Execute the first qualifying NOT MATCHED action. */ static TupleTableSlot * ExecMergeNotMatched(ModifyTableContext *context, ResultRelInfo *resultRelInfo, ChunkTupleRouting *ctr, bool canSetTag) { ModifyTableState *mtstate = context->mtstate; ExprContext *econtext = mtstate->ps.ps_ExprContext; List *actionStates = NIL; ListCell *l; TupleTableSlot *rslot = NULL; /* * For INSERT actions, the root relation's merge action is OK since * the INSERT's targetlist and the WHEN conditions can only refer to * the source relation and hence it does not matter which result * relation we work with. * * XXX does this mean that we can avoid creating copies of * actionStates on partitioned tables, for not-matched actions? * * XXX do we need an additional support of NOT MATCHED BY SOURCE * for PG >= 17? See PostgreSQL commit 0294df2f1f84 */ #if PG17_GE actionStates = ctr->cis->result_relation_info->ri_MergeActions[MERGE_WHEN_NOT_MATCHED_BY_TARGET]; #else actionStates = ctr->cis->result_relation_info->ri_notMatchedMergeAction; #endif /* * Make source tuple available to ExecQual and ExecProject. We don't * need the target tuple, since the WHEN quals and targetlist can't * refer to the target columns. */ econtext->ecxt_scantuple = NULL; econtext->ecxt_innertuple = context->planSlot; econtext->ecxt_outertuple = NULL; foreach (l, actionStates) { MergeActionState *action = (MergeActionState *) lfirst(l); CmdType commandType = action->mas_action->commandType; TupleTableSlot *newslot; /* * Test condition, if any. * * In the absence of any condition, we perform the action * unconditionally (no need to check separately since * ExecQual() will return true if there are no conditions to * evaluate). */ if (!ExecQual(action->mas_whenqual, econtext)) continue; /* Perform stated action */ switch (commandType) { case CMD_INSERT: /* * Project the tuple. In case of a partitioned * table, the projection was already built to use the * root's descriptor, so we don't need to map the * tuple here. */ newslot = ExecProject(action->mas_proj); #if PG17_GE mtstate->mt_merge_action = action; #else context->relaction = action; #endif if (ctr->has_dropped_attrs) { AttrMap *map; TupleDesc parenttupdesc, chunktupdesc; TupleTableSlot *chunk_slot = NULL; parenttupdesc = RelationGetDescr(resultRelInfo->ri_RelationDesc); chunktupdesc = RelationGetDescr(ctr->cis->result_relation_info->ri_RelationDesc); /* map from parent to chunk */ #if PG16_LT map = build_attrmap_by_name_if_req(parenttupdesc, chunktupdesc); #else map = build_attrmap_by_name_if_req(parenttupdesc, chunktupdesc, false); #endif if (map != NULL) chunk_slot = execute_attr_map_slot(map, newslot, MakeSingleTupleTableSlot(chunktupdesc, &TTSOpsVirtual)); rslot = ExecInsert(context, resultRelInfo, ctr, (chunk_slot ? chunk_slot : newslot), canSetTag); if (chunk_slot) ExecDropSingleTupleTableSlot(chunk_slot); } else rslot = ExecInsert(context, resultRelInfo, ctr, newslot, canSetTag); mtstate->mt_merge_inserted = 1; break; case CMD_NOTHING: /* Do nothing */ break; default: elog(ERROR, "unknown action in MERGE WHEN NOT MATCHED clause"); } /* * We've activated one of the WHEN clauses, so we don't * search further. This is required behaviour, not an * optimization. */ break; } return rslot; } /* * Perform MERGE. */ TupleTableSlot * ExecMerge(ModifyTableContext *context, ResultRelInfo *resultRelInfo, ChunkTupleRouting *ctr, ItemPointer tupleid, HeapTuple oldtuple, bool canSetTag) { bool matched; TupleTableSlot *rslot = NULL; /*----- * If we are dealing with a WHEN MATCHED case (tupleid is valid), we * execute the first action for which the additional WHEN MATCHED AND * quals pass. If an action without quals is found, that action is * executed. * * Similarly, if we are dealing with WHEN NOT MATCHED case, we look at * the given WHEN NOT MATCHED actions in sequence until one passes. * * Things get interesting in case of concurrent update/delete of the * target tuple. Such concurrent update/delete is detected while we are * executing a WHEN MATCHED action. * * A concurrent update can: * * 1. modify the target tuple so that it no longer satisfies the * additional quals attached to the current WHEN MATCHED action * * In this case, we are still dealing with a WHEN MATCHED case. * We recheck the list of WHEN MATCHED actions from the start and * choose the first one that satisfies the new target tuple. * * 2. modify the target tuple so that the join quals no longer pass and * hence the source tuple no longer has a match. * * In this case, the source tuple no longer matches the target tuple, * so we now instead find a qualifying WHEN NOT MATCHED action to * execute. * * XXX Hmmm, what if the updated tuple would now match one that was * considered NOT MATCHED so far? * * A concurrent delete changes a WHEN MATCHED case to WHEN NOT MATCHED. * * ExecMergeMatched takes care of following the update chain and * re-finding the qualifying WHEN MATCHED action, as long as the updated * target tuple still satisfies the join quals, i.e., it remains a WHEN * MATCHED case. If the tuple gets deleted or the join quals fail, it * returns and we try ExecMergeNotMatched. Given that ExecMergeMatched * always make progress by following the update chain and we never switch * from ExecMergeNotMatched to ExecMergeMatched, there is no risk of a * livelock. */ #if PG17_GE matched = tupleid != NULL || oldtuple != NULL; #else matched = tupleid != NULL; #endif if (matched) rslot = ExecMergeMatched(context, resultRelInfo, tupleid, oldtuple, canSetTag, &matched); /* * Either we were dealing with a NOT MATCHED tuple or * ExecMergeMatched() returned "false", indicating the previously * MATCHED tuple no longer matches. */ if (!matched) { #if PG17_GE if (rslot == NULL) rslot = ExecMergeNotMatched(context, resultRelInfo, ctr, canSetTag); else context->mtstate->mt_merge_pending_not_matched = context->planSlot; #else (void) ExecMergeNotMatched(context, resultRelInfo, ctr, canSetTag); #endif } return rslot; } ================================================ FILE: src/nodes/vector_agg.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once /* * This file defines the node name of Vector Aggregation custom node, to be * used in the Apache part of the Timescale extension. The node itself is in the * the TSL part. */ #define VECTOR_AGG_NODE_NAME "VectorAgg" ================================================ FILE: src/osm_callbacks.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include "osm_callbacks.h" #include <fmgr.h> #define OSM_CALLBACKS "osm_callbacks" #define OSM_CALLBACKS_VAR_NAME "osm_callbacks_versioned" static OsmCallbacks_Versioned * ts_get_osm_callbacks(void) { OsmCallbacks_Versioned **ptr = (OsmCallbacks_Versioned **) find_rendezvous_variable(OSM_CALLBACKS_VAR_NAME); return *ptr; } /* This interface and version of the struct will be removed once we have a new version of OSM on all * instances */ static OsmCallbacks * ts_get_osm_callbacks_old(void) { OsmCallbacks **ptr = (OsmCallbacks **) find_rendezvous_variable(OSM_CALLBACKS); return *ptr; } chunk_insert_check_hook_type ts_get_osm_chunk_insert_hook() { OsmCallbacks_Versioned *ptr = ts_get_osm_callbacks(); if (ptr) { if (ptr->version_num == 1) return ptr->chunk_insert_check_hook; } else { OsmCallbacks *ptr_old = ts_get_osm_callbacks_old(); if (ptr_old) return ptr_old->chunk_insert_check_hook; } return NULL; } hypertable_drop_hook_type ts_get_osm_hypertable_drop_hook() { OsmCallbacks_Versioned *ptr = ts_get_osm_callbacks(); if (ptr) { if (ptr->version_num == 1) return ptr->hypertable_drop_hook; } else { OsmCallbacks *ptr_old = ts_get_osm_callbacks_old(); if (ptr_old) return ptr_old->hypertable_drop_hook; } return NULL; } hypertable_drop_chunks_hook_type ts_get_osm_hypertable_drop_chunks_hook() { OsmCallbacks_Versioned *ptr = ts_get_osm_callbacks(); if (ptr && ptr->version_num == 1) return ptr->hypertable_drop_chunks_hook; return NULL; } ================================================ FILE: src/osm_callbacks.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <catalog/objectaddress.h> /* range_start and range_end are in PG internal timestamp format. */ typedef int (*chunk_insert_check_hook_type)(Oid ht_oid, int64 range_start, int64 range_end); typedef void (*hypertable_drop_hook_type)(const char *schema_name, const char *table_name); typedef List *(*hypertable_drop_chunks_hook_type)(Oid osm_chunk_oid, const char *hypertable_schema_name, const char *hypertable_name, int64 range_start, int64 range_end); /* * Object Storage Manager callbacks. * * chunk_insert_check_hook - checks whether the specified range is managed by OSM * hypertable_drop_hook - used for OSM catalog cleanups */ /* This struct is retained for backward compatibility. We'll remove this in one * of the upcoming releases */ typedef struct { chunk_insert_check_hook_type chunk_insert_check_hook; hypertable_drop_hook_type hypertable_drop_hook; } OsmCallbacks; typedef struct { int64 version_num; chunk_insert_check_hook_type chunk_insert_check_hook; hypertable_drop_hook_type hypertable_drop_hook; hypertable_drop_chunks_hook_type hypertable_drop_chunks_hook; } OsmCallbacks_Versioned; extern chunk_insert_check_hook_type ts_get_osm_chunk_insert_hook(void); extern hypertable_drop_hook_type ts_get_osm_hypertable_drop_hook(void); extern hypertable_drop_chunks_hook_type ts_get_osm_hypertable_drop_chunks_hook(void); ================================================ FILE: src/partition_chunk.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/attmap.h> #include <access/toast_compression.h> #include <catalog/heap.h> #include <catalog/pg_constraint.h> #include <commands/tablecmds.h> #include <executor/executor.h> #include <nodes/makefuncs.h> #include <nodes/parsenodes.h> #include <rewrite/rewriteManip.h> #include <utils/partcache.h> #include "chunk.h" #include "extension.h" #include "guc.h" #include "hypercube.h" #include "hypertable.h" #include "partition_chunk.h" void _executor_init(void); void _executor_fini(void); static ExecutorEnd_hook_type prev_executor_end_hook = NULL; /* * Cache and the memory context to store recently created chunks to be attached * as partitions. */ static HTAB *PartChunkCache = NULL; static MemoryContext PartChunkCacheCxt = NULL; /* * Transaction callback to clean up the partition chunk cache on abort. * Memory context is deleted by the portal context cleanup. Just nullify the * pointers here. */ static void partcache_xact_callback(XactEvent event, void *arg) { switch (event) { case XACT_EVENT_ABORT: case XACT_EVENT_PARALLEL_ABORT: PartChunkCache = NULL; PartChunkCacheCxt = NULL; break; default: /* do nothing? */ break; } } /* * Insert a chunk into the partition cache. */ void ts_partition_cache_insert_chunk(const Hypertable *ht, Oid chunk_relid) { PartChunkCacheEntry *entry; bool found; if (PartChunkCache == NULL) { if (PartChunkCacheCxt == NULL) PartChunkCacheCxt = AllocSetContextCreate(PortalContext, "partition chunk cache", ALLOCSET_DEFAULT_SIZES); HASHCTL ctl; ctl.hcxt = AllocSetContextCreate(PortalContext, "partition chunk cache", ALLOCSET_DEFAULT_SIZES); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(PartChunkCacheEntry); PartChunkCache = hash_create("partition chunk cache", 256, /* start small, grows automatically */ &ctl, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); RegisterXactCallback(partcache_xact_callback, NULL); } entry = hash_search(PartChunkCache, &ht->main_table_relid, HASH_ENTER, &found); if (!found) entry->chunk_oids = NIL; MemoryContext oldctx = MemoryContextSwitchTo(PartChunkCacheCxt); entry->chunk_oids = lappend_oid(entry->chunk_oids, chunk_relid); MemoryContextSwitchTo(oldctx); } /* * Get a partition cache entry by hypertable relid. */ PartChunkCacheEntry * ts_partition_cache_get_by_hypertable(Oid ht_relid) { PartChunkCacheEntry *entry; if (PartChunkCache == NULL) return NULL; entry = (PartChunkCacheEntry *) hash_search(PartChunkCache, &ht_relid, HASH_FIND, NULL); return entry; } /* * Destroy the partition chunk cache. */ void ts_partition_cache_destroy(void) { if (PartChunkCache != NULL) { hash_destroy(PartChunkCache); PartChunkCache = NULL; PartChunkCacheCxt = NULL; } } /* * Fill the attribute and constraint lists by copying from the parent hypertable attributes. * Partition chunk's attributes are derived from the hypertable's attributes including storage, * compression, generation expressions, and default values. * * The constraints list is filled with the CHECK and NOT NULL constraints on attributes. * * This code is adapted from MergeAttributes() in tablecmds.c. */ void ts_partition_chunk_prepare_attributes(Oid ht_relid, List **attlist, List **constraints) { Relation rel = table_open(ht_relid, AccessShareLock); TupleDesc tupleDesc = RelationGetDescr(rel); TupleConstr *constr = tupleDesc->constr; AttrMap *newattmap = make_attrmap(tupleDesc->natts); List *inherited_defaults = NIL; List *cols_with_defaults = NIL; int child_attno = 0; for (int parent_attno = 1; parent_attno <= tupleDesc->natts; parent_attno++) { Form_pg_attribute attribute = TupleDescAttr(tupleDesc, parent_attno - 1); char *attributeName = NameStr(attribute->attname); ColumnDef *newdef; /* * Ignore dropped columns in the parent. */ if (attribute->attisdropped) continue; /* leave newattmap->attnums entry as zero */ /* * Create new column definition */ newdef = makeColumnDef(attributeName, attribute->atttypid, attribute->atttypmod, attribute->attcollation); newdef->type = T_ColumnDef; newdef->is_not_null = attribute->attnotnull; newdef->storage = attribute->attstorage; newdef->generated = attribute->attgenerated; if (CompressionMethodIsValid(attribute->attcompression)) newdef->compression = pstrdup(GetCompressionMethodName(attribute->attcompression)); newdef->inhcount = 0; newdef->is_local = false; newattmap->attnums[parent_attno - 1] = ++child_attno; /* * Locate default/generation expression if any */ if (attribute->atthasdef) { Node *this_default = NULL; /* Find default in constraint structure */ if (constr != NULL) { AttrDefault *attrdef = constr->defval; for (int i = 0; i < constr->num_defval; i++) { if (attrdef[i].adnum == parent_attno) { this_default = stringToNode(attrdef[i].adbin); break; } } } if (this_default == NULL) elog(ERROR, "default expression not found for attribute %d of relation \"%s\"", parent_attno, RelationGetRelationName(rel)); /* * If it's a GENERATED default, it might contain Vars that * need to be mapped to the inherited column(s)' new numbers. * We can't do that till newattmap is ready, so just remember * all the inherited default expressions for the moment. */ inherited_defaults = lappend(inherited_defaults, this_default); cols_with_defaults = lappend(cols_with_defaults, newdef); } *attlist = lappend(*attlist, newdef); } /* * Now process any inherited default expressions, adjusting attnos * using the completed newattmap map. */ ListCell *lc1, *lc2; forboth (lc1, inherited_defaults, lc2, cols_with_defaults) { Node *this_default = (Node *) lfirst(lc1); ColumnDef *def = (ColumnDef *) lfirst(lc2); bool found_whole_row; /* Adjust Vars to match new table's column numbering */ this_default = map_variable_attnos(this_default, 1, 0, newattmap, InvalidOid, &found_whole_row); /* * For the moment we have to reject whole-row variables. We could * convert them, if we knew the new table's rowtype OID, but that * hasn't been assigned yet. (A variable could only appear in a * generation expression, so the error message is correct.) */ if (found_whole_row) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot convert whole-row table reference"), errdetail("Generation expression for column \"%s\" contains a whole-row " "reference to table \"%s\".", def->colname, RelationGetRelationName(rel)))); Assert(def->raw_default == NULL); def->cooked_default = this_default; } /* * Now copy the CHECK constraints of this parent, adjusting attnos * using the completed newattmap map. */ if (constr && constr->num_check > 0) { for (int i = 0; i < constr->num_check; i++) { Node *expr; bool found_whole_row; /* ignore if the constraint is non-inheritable */ if (constr->check[i].ccnoinherit) continue; /* Adjust Vars to match new table's column numbering */ expr = map_variable_attnos(stringToNode(constr->check[i].ccbin), 1, 0, newattmap, InvalidOid, &found_whole_row); /* * For the moment we have to reject whole-row variables. We * could convert them, if we knew the new table's rowtype OID, * but that hasn't been assigned yet. */ if (found_whole_row) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot convert whole-row table reference"))); Constraint *c = makeNode(Constraint); c->type = T_Constraint; c->contype = CONSTR_CHECK; c->conname = pstrdup(constr->check[i].ccname); c->skip_validation = !constr->check[i].ccvalid; c->initially_valid = true; c->location = -1; c->is_no_inherit = constr->check[i].ccnoinherit; c->raw_expr = NULL; c->cooked_expr = nodeToString(expr); *constraints = lappend(*constraints, c); #if PG18_GE c->is_enforced = constr->check[i].ccenforced; #endif } } #if PG18_GE /* A row is added into pg_constraints for each NOT NULL constraint since PG18 */ List *notnulls = RelationGetNotNullConstraints(ht_relid, false, false); foreach_ptr(Constraint, nn, notnulls) { Assert(nn->contype == CONSTR_NOTNULL); *constraints = lappend(*constraints, nn); } #endif free_attrmap(newattmap); table_close(rel, NoLock); } /* * Attach a standalone chunk to a partitioned hypertable as a partition. */ static void partition_chunk_attach(const Hypertable *ht, const Chunk *chunk) { /* Currently only single-dimensional partitioned hypertables are supported */ Assert(chunk->cube->num_slices == 1); const Dimension *dim = ts_hyperspace_get_dimension_by_id(ht->space, chunk->cube->slices[0]->fd.dimension_id); Oid dimtype = ts_dimension_get_partition_type(dim); A_Const prd_lower = { .type = T_A_Const, .val.sval = { .type = T_String, .sval = ts_internal_to_time_string(chunk->cube->slices[0]->fd.range_start, dimtype), }, .location = -1 }; A_Const prd_upper = { .type = T_A_Const, .val.sval = { .type = T_String, .sval = ts_internal_to_time_string(chunk->cube->slices[0]->fd.range_end, dimtype), }, .location = -1 }; PartitionBoundSpec pbspec = { .type = T_PartitionBoundSpec, .is_default = false, .location = -1, .strategy = PARTITION_STRATEGY_RANGE, .lowerdatums = list_make1(&prd_lower), .upperdatums = list_make1(&prd_upper), }; PartitionCmd partcmd = { .type = T_PartitionCmd, .name = makeRangeVar((char *) NameStr(chunk->fd.schema_name), (char *) NameStr(chunk->fd.table_name), 0), .bound = &pbspec, .concurrent = false, }; AlterTableCmd altercmd = { .type = T_AlterTableCmd, .subtype = AT_AttachPartition, .def = (Node *) &partcmd, .missing_ok = false, }; AlterTableStmt alterstmt = { .type = T_AlterTableStmt, .cmds = list_make1(&altercmd), .missing_ok = false, .objtype = OBJECT_TABLE, .relation = makeRangeVar((char *) NameStr(ht->fd.schema_name), (char *) NameStr(ht->fd.table_name), 0), }; LOCKMODE lockmode = AlterTableGetLockLevel(alterstmt.cmds); AlterTableUtilityContext atcontext = { .relid = AlterTableLookupRelation(&alterstmt, lockmode), }; AlterTable(&alterstmt, lockmode, &atcontext); } /* * ExecutoreEnd hook to attach cached partition chunks to their hypertables. */ static void ts_executor_end_hook(QueryDesc *queryDesc) { ListCell *lc; if (prev_executor_end_hook) prev_executor_end_hook(queryDesc); else standard_ExecutorEnd(queryDesc); /* * Chunks cannot be created as a partition or attached as partition until * this point since Postgres does not allow such operations when there is * an open reference to the parent table. ModifyTable node opens the parent * table and it only gets closed in ExecEndPlan. */ if (queryDesc->operation == CMD_INSERT && PartChunkCache != NULL && ts_extension_is_loaded()) { Cache *hcache = ts_hypertable_cache_pin(); HASH_SEQ_STATUS status; PartChunkCacheEntry *entry; hash_seq_init(&status, PartChunkCache); while ((entry = hash_seq_search(&status)) != NULL) { foreach (lc, entry->chunk_oids) { Hypertable *ht = ts_hypertable_cache_get_entry(hcache, entry->ht_relid, CACHE_FLAG_MISSING_OK); if (ht) partition_chunk_attach(ht, ts_chunk_get_by_relid(lfirst_oid(lc), true)); } } ts_cache_release(&hcache); ts_partition_cache_destroy(); } } void _executor_init(void) { prev_executor_end_hook = ExecutorEnd_hook; ExecutorEnd_hook = ts_executor_end_hook; } void _executor_fini(void) { ExecutorEnd_hook = prev_executor_end_hook; } ================================================ FILE: src/partition_chunk.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include "export.h" #include "guc.h" #define is_partitioning_allowed(relid) \ (ts_guc_enable_partitioned_hypertables && (get_rel_relkind(relid) == RELKIND_PARTITIONED_TABLE)) /* * Cache entry for chunks to be attached as partitions */ typedef struct PartChunkCacheEntry { Oid ht_relid; List *chunk_oids; } PartChunkCacheEntry; extern void ts_partition_cache_insert_chunk(const Hypertable *ht, Oid chunk_relid); extern PartChunkCacheEntry *ts_partition_cache_get_by_hypertable(Oid ht_relid); extern void ts_partition_cache_destroy(void); extern void ts_partition_chunk_prepare_attributes(Oid ht_relid, List **attlist, List **constraints); ================================================ FILE: src/partitioning.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/hash.h> #include <access/htup_details.h> #include <catalog/namespace.h> #include <catalog/pg_type.h> #include <miscadmin.h> #include <nodes/makefuncs.h> #include <nodes/pg_list.h> #include <parser/parse_coerce.h> #include <utils/acl.h> #include <utils/builtins.h> #include <utils/cash.h> #include <utils/catcache.h> #include <utils/date.h> #include <utils/inet.h> #include <utils/jsonb.h> #include <utils/lsyscache.h> #include <utils/memutils.h> #include <utils/numeric.h> #include <utils/rangetypes.h> #include <utils/syscache.h> #include <utils/timestamp.h> #include "compat/compat.h" #include "partitioning.h" #include "ts_catalog/catalog.h" #include "utils.h" #define IS_VALID_CLOSED_PARTITIONING_FUNC(proform, argtype) \ ((proform)->prorettype == INT4OID && ((proform)->provolatile == PROVOLATILE_IMMUTABLE) && \ (proform)->pronargs == 1 && \ ((proform)->proargtypes.values[0] == (argtype) || \ (proform)->proargtypes.values[0] == ANYELEMENTOID)) #define IS_VALID_OPEN_PARTITIONING_FUNC(proform, argtype) \ (IS_VALID_OPEN_DIM_TYPE((proform)->prorettype) && \ ((proform)->provolatile == PROVOLATILE_IMMUTABLE) && (proform)->pronargs == 1 && \ ((proform)->proargtypes.values[0] == (argtype) || \ (proform)->proargtypes.values[0] == ANYELEMENTOID)) #define IS_VALID_PARTITIONING_FUNC(proform, dimtype, argtype) \ (((dimtype) == DIMENSION_TYPE_OPEN) ? IS_VALID_OPEN_PARTITIONING_FUNC(proform, argtype) : \ IS_VALID_CLOSED_PARTITIONING_FUNC(proform, argtype)) static bool closed_dim_partitioning_func_filter(Form_pg_proc form, void *arg) { Oid *argtype = arg; return IS_VALID_CLOSED_PARTITIONING_FUNC(form, *argtype); } static bool open_dim_partitioning_func_filter(Form_pg_proc form, void *arg) { Oid *argtype = arg; return IS_VALID_OPEN_PARTITIONING_FUNC(form, *argtype); } bool ts_partitioning_func_is_valid(regproc funcoid, DimensionType dimtype, Oid argtype) { HeapTuple tuple; bool isvalid; AclResult aclresult; tuple = SearchSysCache1(PROCOID, ObjectIdGetDatum(funcoid)); if (!HeapTupleIsValid(tuple)) elog(ERROR, "cache lookup failed for function %u", funcoid); aclresult = object_aclcheck(ProcedureRelationId, funcoid, GetUserId(), ACL_EXECUTE); if (aclresult != ACLCHECK_OK) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permission denied for function %s", get_func_name(funcoid)))); isvalid = IS_VALID_PARTITIONING_FUNC((Form_pg_proc) GETSTRUCT(tuple), dimtype, argtype); ReleaseSysCache(tuple); return isvalid; } Oid ts_partitioning_func_get_closed_default(void) { Oid argtype = ANYELEMENTOID; return ts_lookup_proc_filtered(DEFAULT_PARTITIONING_FUNC_SCHEMA, DEFAULT_PARTITIONING_FUNC_NAME, NULL, closed_dim_partitioning_func_filter, &argtype); } static bool ts_partitioning_func_is_closed_default(const char *schema, const char *funcname) { Assert(schema != NULL && funcname != NULL); return strcmp(DEFAULT_PARTITIONING_FUNC_SCHEMA, schema) == 0 && strcmp(DEFAULT_PARTITIONING_FUNC_NAME, funcname) == 0; } /* * Resolve the partitioning function set for a hypertable. */ static void partitioning_func_set_func_fmgr(PartitioningFunc *pf, Oid argtype, DimensionType dimtype) { Oid funcoid; proc_filter filter = dimtype == DIMENSION_TYPE_CLOSED ? closed_dim_partitioning_func_filter : open_dim_partitioning_func_filter; if (dimtype != DIMENSION_TYPE_CLOSED && dimtype != DIMENSION_TYPE_OPEN) elog(ERROR, "invalid dimension type %u", dimtype); funcoid = ts_lookup_proc_filtered(NameStr(pf->schema), NameStr(pf->name), &pf->rettype, filter, &argtype); if (!OidIsValid(funcoid)) { if (dimtype == DIMENSION_TYPE_CLOSED) ereport(ERROR, (errmsg("invalid partitioning function"), errhint("A partitioning function for a closed (space) dimension " "must be IMMUTABLE and have the signature (anyelement) -> integer"))); else ereport(ERROR, (errmsg("invalid partitioning function"), errhint("A partitioning function for a open (time) dimension " "must be IMMUTABLE, take one argument, and return a supported time " "type"))); } fmgr_info_cxt(funcoid, &pf->func_fmgr, CurrentMemoryContext); } static Oid find_text_coercion_func(Oid type) { Oid funcid; bool is_varlena; CoercionPathType cpt; /* * First look for an explicit cast type. Needed since the output of for * example character(20) not the same as character(20)::text */ cpt = find_coercion_pathway(TEXTOID, type, COERCION_EXPLICIT, &funcid); if (cpt != COERCION_PATH_FUNC) getTypeOutputInfo(type, &funcid, &is_varlena); return funcid; } #define TYPECACHE_HASH_FLAGS (TYPECACHE_HASH_PROC | TYPECACHE_HASH_PROC_FINFO) PartitioningInfo * ts_partitioning_info_create(const char *schema, const char *partfunc, const char *partcol, DimensionType dimtype, Oid relid) { PartitioningInfo *pinfo; Oid columntype, varcollid, funccollid = InvalidOid; Var *var; FuncExpr *expr; if (schema == NULL || partfunc == NULL || partcol == NULL) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("partitioning function information cannot be null"))); pinfo = palloc0(sizeof(PartitioningInfo)); namestrcpy(&pinfo->partfunc.name, partfunc); namestrcpy(&pinfo->column, partcol); pinfo->column_attnum = get_attnum(relid, NameStr(pinfo->column)); pinfo->dimtype = dimtype; /* handle the case that the attribute has been dropped */ if (pinfo->column_attnum == InvalidAttrNumber) return NULL; namestrcpy(&pinfo->partfunc.schema, schema); /* Lookup the type cache entry to access the hash function for the type */ columntype = get_atttype(relid, pinfo->column_attnum); if (dimtype == DIMENSION_TYPE_CLOSED) { TypeCacheEntry *tce = lookup_type_cache(columntype, TYPECACHE_HASH_FLAGS); if (!OidIsValid(tce->hash_proc) && ts_partitioning_func_is_closed_default(schema, partfunc)) elog(ERROR, "could not find hash function for type %s", format_type_be(columntype)); } partitioning_func_set_func_fmgr(&pinfo->partfunc, columntype, dimtype); /* * Prepare a function expression for this function. The partition hash * function needs this to be able to resolve the type of the value to be * hashed. */ varcollid = get_typcollation(columntype); var = makeVar(1, pinfo->column_attnum, columntype, -1, varcollid, 0); expr = makeFuncExpr(pinfo->partfunc.func_fmgr.fn_oid, pinfo->partfunc.rettype, list_make1(var), funccollid, varcollid, COERCE_EXPLICIT_CALL); fmgr_info_set_expr((Node *) expr, &pinfo->partfunc.func_fmgr); return pinfo; } /* * Apply a dimension's partitioning function to a value. * * We need to avoid FunctionCall1(), because we'd like to customize the error * message in case of NULL return values. */ TSDLLEXPORT Datum ts_partitioning_func_apply(PartitioningInfo *pinfo, Oid collation, Datum value) { LOCAL_FCINFO(fcinfo, 1); Datum result; InitFunctionCallInfoData(*fcinfo, &pinfo->partfunc.func_fmgr, 1, collation, NULL, NULL); FC_SET_ARG(fcinfo, 0, value); result = FunctionCallInvoke(fcinfo); if (fcinfo->isnull) { ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("partitioning function \"%s.%s\" returned NULL", NameStr(pinfo->partfunc.schema), NameStr(pinfo->partfunc.name)))); } return result; } /* * Helper function to find the right partition value from a tuple, * for space partitioned hypertables. Since attributes in tuple can * be of different order when compared to physical table columns order, * we pass partition_col_idx which points to correct space partitioned * column in the given tuple. */ TSDLLEXPORT Datum ts_partitioning_func_apply_slot(PartitioningInfo *pinfo, TupleTableSlot *slot, bool *isnull) { Datum value; bool null; Oid collation; value = slot_getattr(slot, pinfo->column_attnum, &null); if (NULL != isnull) *isnull = null; if (null) return 0; collation = TupleDescAttr(slot->tts_tupleDescriptor, AttrNumberGetAttrOffset(pinfo->column_attnum)) ->attcollation; return ts_partitioning_func_apply(pinfo, collation, value); } /* * Resolve the type of the argument passed to a function. * * The type is resolved from the function expression in the function call info. */ static Oid resolve_function_argtype(FunctionCallInfo fcinfo) { FuncExpr *fe; Node *node; Oid argtype; /* Get the function expression from the call info */ fe = (FuncExpr *) fcinfo->flinfo->fn_expr; if (NULL == fe || !IsA(fe, FuncExpr)) elog(ERROR, "no function expression set when invoking partitioning function"); if (list_length(fe->args) != 1) elog(ERROR, "unexpected number of arguments in function expression"); node = linitial(fe->args); switch (nodeTag(node)) { case T_Var: argtype = castNode(Var, node)->vartype; break; case T_Const: argtype = castNode(Const, node)->consttype; break; case T_CoerceViaIO: argtype = castNode(CoerceViaIO, node)->resulttype; break; case T_FuncExpr: /* Argument is function, so our input is its result type */ argtype = castNode(FuncExpr, node)->funcresulttype; break; case T_Param: argtype = castNode(Param, node)->paramtype; break; default: elog(ERROR, "unsupported expression argument node type: %s", ts_get_node_name(node)); } return argtype; } /* * Partitioning function cache. * * Holds type information to avoid repeated lookups. The cache is allocated on a * child memory context of the context that created the associated FmgrInfo * struct. For partitioning functions invoked on the insert path, this is * typically the Hypertable cache's memory context. Hence, the type cache lives * for the duration of the hypertable cache and can be reused across multiple * invocations of the partitioning function, even across transactions. * * If the partitioning function is invoked outside the insert path, the FmgrInfo * and its memory context has a lifetime corresponding to that invocation. */ typedef struct PartFuncCache { Oid argtype; Oid coerce_funcid; TypeCacheEntry *tce; } PartFuncCache; static PartFuncCache * part_func_cache_create(Oid argtype, TypeCacheEntry *tce, Oid coerce_funcid, MemoryContext mcxt) { PartFuncCache *pfc; pfc = MemoryContextAlloc(mcxt, sizeof(PartFuncCache)); pfc->argtype = argtype; pfc->tce = tce; pfc->coerce_funcid = coerce_funcid; return pfc; } /* _timescaledb_catalog.ts_get_partition_for_key(key anyelement) RETURNS INT */ TSDLLEXPORT Datum ts_get_partition_for_key(PG_FUNCTION_ARGS); TS_FUNCTION_INFO_V1(ts_get_partition_for_key); /* * Partition hash function that first converts all inputs to text before * hashing. */ Datum ts_get_partition_for_key(PG_FUNCTION_ARGS) { Datum arg = PG_GETARG_DATUM(0); PartFuncCache *pfc = fcinfo->flinfo->fn_extra; struct varlena *data; uint32 hash_u; int32 res; if (PG_NARGS() != 1) elog(ERROR, "unexpected number of arguments to partitioning function"); if (NULL == pfc) { Oid funcid = InvalidOid; Oid argtype = resolve_function_argtype(fcinfo); if (argtype != TEXTOID) { /* Not TEXT input -> need to convert to text */ funcid = find_text_coercion_func(argtype); if (!OidIsValid(funcid)) elog(ERROR, "could not coerce type %u to text", argtype); } pfc = part_func_cache_create(argtype, NULL, funcid, fcinfo->flinfo->fn_mcxt); fcinfo->flinfo->fn_extra = pfc; } if (pfc->argtype != TEXTOID) { arg = OidFunctionCall1(pfc->coerce_funcid, arg); arg = CStringGetTextDatum(DatumGetCString(arg)); } data = DatumGetTextPP(arg); hash_u = DatumGetUInt32(hash_any((unsigned char *) VARDATA_ANY(data), VARSIZE_ANY_EXHDR(data))); res = (int32) (hash_u & 0x7fffffff); /* Only positive numbers */ PG_FREE_IF_COPY(data, 0); PG_RETURN_INT32(res); } TSDLLEXPORT Datum ts_get_partition_hash(PG_FUNCTION_ARGS); TS_FUNCTION_INFO_V1(ts_get_partition_hash); /* * Compute a partition hash value for any input type. * * ts_get_partition_hash() takes a single argument of anyelement type. We compute * the hash based on the argument type information that we expect to find in the * function expression in the function call context. If no such expression * exists, or the type cannot be resolved from the expression, the function * throws an error. */ Datum ts_get_partition_hash(PG_FUNCTION_ARGS) { Datum arg = PG_GETARG_DATUM(0); PartFuncCache *pfc = fcinfo->flinfo->fn_extra; Datum hash; int32 res; Oid collation; if (PG_NARGS() != 1) elog(ERROR, "unexpected number of arguments to partitioning function"); if (NULL == pfc) { Oid argtype = resolve_function_argtype(fcinfo); TypeCacheEntry *tce = lookup_type_cache(argtype, TYPECACHE_HASH_FLAGS); pfc = part_func_cache_create(argtype, tce, InvalidOid, fcinfo->flinfo->fn_mcxt); fcinfo->flinfo->fn_extra = pfc; } if (!OidIsValid(pfc->tce->hash_proc)) elog(ERROR, "could not find hash function for type %u", pfc->argtype); /* use the supplied collation, if it exists, otherwise use the default for * the type */ collation = PG_GET_COLLATION(); if (!OidIsValid(collation)) collation = pfc->tce->typcollation; hash = FunctionCall1Coll(&pfc->tce->hash_proc_finfo, collation, arg); /* Only positive numbers */ res = (int32) (DatumGetUInt32(hash) & 0x7fffffff); PG_RETURN_INT32(res); } ================================================ FILE: src/partitioning.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #define KEYSPACE_PT_NO_PARTITIONING -1 #include <postgres.h> #include <access/attnum.h> #include <access/htup_details.h> #include <fmgr.h> #include <utils/typcache.h> #include "dimension.h" #include "ts_catalog/catalog.h" #define OPEN_START_TIME -1 #define OPEN_END_TIME PG_INT64_MAX #define DEFAULT_PARTITIONING_FUNC_SCHEMA FUNCTIONS_SCHEMA_NAME #define DEFAULT_PARTITIONING_FUNC_NAME "get_partition_hash" typedef struct PartitioningFunc { NameData schema; NameData name; Oid rettype; /* * Function manager info to call the partitioning function on the * partitioning column's text representation. */ FmgrInfo func_fmgr; } PartitioningFunc; typedef struct PartitioningInfo { NameData column; AttrNumber column_attnum; DimensionType dimtype; PartitioningFunc partfunc; } PartitioningInfo; extern Oid ts_partitioning_func_get_closed_default(void); extern bool ts_partitioning_func_is_valid(regproc funcoid, DimensionType dimtype, Oid argtype); extern PartitioningInfo *ts_partitioning_info_create(const char *schema, const char *partfunc, const char *partcol, DimensionType dimtype, Oid relid); extern TSDLLEXPORT Datum ts_partitioning_func_apply(PartitioningInfo *pinfo, Oid collation, Datum value); /* NOTE: assume the tuple belongs to the root table, use ts_partitioning_func_apply for chunk tuples */ extern TSDLLEXPORT Datum ts_partitioning_func_apply_slot(PartitioningInfo *pinfo, TupleTableSlot *slot, bool *isnull); ================================================ FILE: src/planner/CMakeLists.txt ================================================ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/planner.c ${CMAKE_CURRENT_SOURCE_DIR}/agg_bookend.c ${CMAKE_CURRENT_SOURCE_DIR}/constify_now.c ${CMAKE_CURRENT_SOURCE_DIR}/constraint_cleanup.c ${CMAKE_CURRENT_SOURCE_DIR}/expand_hypertable.c ${CMAKE_CURRENT_SOURCE_DIR}/space_constraint.c) target_sources(${PROJECT_NAME} PRIVATE ${SOURCES}) ================================================ FILE: src/planner/agg_bookend.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ /* * Optimization for FIRST/LAST aggregate functions. * * This module tries to replace FIRST/LAST aggregate functions by subqueries * of the form * (SELECT value FROM tab * WHERE sort IS NOT NULL AND existing-quals * ORDER BY sort ASC/DESC * LIMIT 1) * Given a suitable index on sort column, this can be much faster than the * generic scan-all-the-rows aggregation plan. We can handle multiple * FIRST/LAST aggregates by generating multiple subqueries, and their * orderings can be different. However, if the query also contains some * other aggregates (eg. MIN/MAX), we will skip optimization since we can't * optimize across different aggregate functions. * * Most of the code is borrowed from: * src/backend/optimizer/plan/planagg.c * * */ /* * This file contains source code that was copied and/or modified from * the PostgreSQL database, which is licensed under the open-source * PostgreSQL License. Please see the NOTICE at the top level * directory for a copy of the PostgreSQL License. */ #include <postgres.h> #include <access/htup_details.h> #include <access/stratnum.h> #include <catalog/namespace.h> #include <catalog/pg_aggregate.h> #include <catalog/pg_proc.h> #include <catalog/pg_type.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <optimizer/cost.h> #include <optimizer/optimizer.h> #include <optimizer/pathnode.h> #include <optimizer/paths.h> #include <optimizer/planmain.h> #include <optimizer/subselect.h> #include <optimizer/tlist.h> #include <parser/parse_clause.h> #include <parser/parse_func.h> #include <parser/parsetree.h> #include <rewrite/rewriteManip.h> #include <utils/builtins.h> #include <utils/lsyscache.h> #include <utils/regproc.h> #include <utils/syscache.h> #include <utils/typcache.h> #include "compat/compat.h" #include "extension.h" #include "func_cache.h" #include "planner.h" #include "utils.h" typedef struct FirstLastAggInfo { MinMaxAggInfo *m_agg_info; /* reusing MinMaxAggInfo to avoid code * duplication */ Expr *sort; /* Expression to use for ORDER BY */ } FirstLastAggInfo; typedef struct MutatorContext { MinMaxAggPath *mm_path; } MutatorContext; static bool find_first_last_aggs_walker(Node *node, List **context); static bool build_first_last_path(PlannerInfo *root, FirstLastAggInfo *fl_info, Oid eqop, Oid sortop, bool reverse_sort, bool nulls_first); static void first_last_qp_callback(PlannerInfo *root, void *extra); static Node *mutate_aggref_node(Node *node, MutatorContext *context); static void replace_aggref_in_tlist(MinMaxAggPath *minmaxagg_path); /* * mutate_aggref_node * * Mutator function used by recursive `expression_tree_mutator` * to replace Aggref node with Param node */ Node * mutate_aggref_node(Node *node, MutatorContext *context) { if (node == NULL) return NULL; if (IsA(node, Aggref)) { Aggref *aggref = (Aggref *) node; /* See if the Aggref should be replaced by a Param */ if (context->mm_path != NULL && list_length(aggref->args) == 2) { TargetEntry *curTarget = (TargetEntry *) linitial(aggref->args); ListCell *cell; foreach (cell, context->mm_path->mmaggregates) { MinMaxAggInfo *mminfo = (MinMaxAggInfo *) lfirst(cell); if (mminfo->aggfnoid == aggref->aggfnoid && equal(mminfo->target, curTarget->expr)) return (Node *) copyObject(mminfo->param); } } } return expression_tree_mutator(node, mutate_aggref_node, (void *) context); } /* * replace_aggref_in_tlist * * If MinMaxAggPath is chosen, instead of running aggregate * function we will execute subquery that we've generated. Since we * use subquery we need to replace target list Aggref node with Param * node. Param node passes output value from the subquery. * */ void replace_aggref_in_tlist(MinMaxAggPath *minmaxagg_path) { MutatorContext context; context.mm_path = minmaxagg_path; ((Path *) minmaxagg_path)->pathtarget->exprs = (List *) mutate_aggref_node((Node *) ((Path *) minmaxagg_path)->pathtarget->exprs, (void *) &context); } static StrategyNumber get_func_strategy(Oid func_oid) { /* Ensure function cache is initialized */ if (!OidIsValid(ts_first_func_oid) || !OidIsValid(ts_last_func_oid)) ts_func_cache_init(); if (func_oid == ts_first_func_oid) return BTLessStrategyNumber; if (func_oid == ts_last_func_oid) return BTGreaterStrategyNumber; return InvalidStrategy; } static bool is_first_last_node(Node *node, List **context) { if (node == NULL) return false; if (IsA(node, Aggref)) { Aggref *aggref = (Aggref *) node; if (aggref->aggfnoid == ts_first_func_oid || aggref->aggfnoid == ts_last_func_oid) return true; } return expression_tree_walker(node, is_first_last_node, (void *) context); } static bool contains_first_last_node(List *sortClause, List *targetList) { List *exprs = get_sortgrouplist_exprs(sortClause, targetList); ListCell *cell; List *context = NIL; foreach (cell, exprs) { Node *expr = lfirst(cell); bool found = is_first_last_node(expr, &context); if (found) return true; } return false; } /* * preprocess_first_last_aggregates - preprocess FIRST/LAST aggregates * * Check to see whether the query contains FIRST/LAST aggregate functions that * might be optimizable via index scans. If it does, and all the aggregates * are potentially optimizable, then create a MinMaxAggPath(reusing MinMax path implementation)\ * and add it to the (UPPERREL_GROUP_AGG, NULL) upperrel. * * This method is called from create_upper_paths_hook in the UPPERREL_GROUP_AGG stage. * * Note: we are passed the preprocessed targetlist separately, because it's * not necessarily equal to root->parse->targetList. * * Most of the code is borrowed from: preprocess_minmax_aggregates (planagg.c). Few * major differences: * - generate FirstLastAggInfo that wraps MinMaxAggInfo * - generate subquery (path) for FIRST/LAST (we reuse MinMaxAggPath) * - replace Aggref node with Param node * - reject ORDER BY on FIRST/LAST */ void ts_preprocess_first_last_aggregates(PlannerInfo *root, List *tlist) { Query *parse = root->parse; FromExpr *jtnode; RangeTblRef *rtr; RangeTblEntry *rte; List *first_last_aggs; RelOptInfo *grouped_rel; ListCell *lc; List *mm_agg_list; MinMaxAggPath *minmaxagg_path; /* minmax_aggs list should be empty at this point */ Assert(root->minmax_aggs == NIL); /* Nothing to do if query has no aggregates */ if (!parse->hasAggs) return; Assert(!parse->setOperations); /* shouldn't get here if a setop */ Assert(parse->rowMarks == NIL); /* nor if FOR UPDATE */ /* * Reject unoptimizable cases. * * We don't handle the case when agg function is in ORDER BY. The reason * being is that we replace Aggref node before sort keys are being * generated. * * We don't handle GROUP BY or windowing, because our current * implementations of grouping require looking at all the rows anyway, and * so there's not much point in optimizing FIRST/LAST. */ if (parse->groupClause || list_length(parse->groupingSets) > 1 || parse->hasWindowFuncs || contains_first_last_node(parse->sortClause, tlist)) return; /* * Reject if query contains any CTEs; there's no way to build an indexscan * on one so we couldn't succeed here. (If the CTEs are unreferenced, * that's not true, but it doesn't seem worth expending cycles to check.) */ if (parse->cteList) return; /* * We also restrict the query to reference exactly one table, since join * conditions can't be handled reasonably. (We could perhaps handle a * query containing cartesian-product joins, but it hardly seems worth the * trouble.) However, the single table could be buried in several levels * of FromExpr due to subqueries. Note the "single" table could be an * inheritance parent, too, including the case of a UNION ALL subquery * that's been flattened to an appendrel. */ jtnode = parse->jointree; while (IsA(jtnode, FromExpr)) { if (list_length(jtnode->fromlist) != 1) return; jtnode = linitial(jtnode->fromlist); } if (!IsA(jtnode, RangeTblRef)) return; rtr = (RangeTblRef *) jtnode; rte = planner_rt_fetch(rtr->rtindex, root); if (rte->rtekind == RTE_RELATION) /* ordinary relation, ok */; else if (rte->rtekind == RTE_SUBQUERY && rte->inh) /* flattened UNION ALL subquery, ok */; else return; /* * Scan the tlist and HAVING qual to find all the aggregates and verify * all are FIRST/LAST aggregates. Stop as soon as we find one that isn't. */ first_last_aggs = NIL; if (find_first_last_aggs_walker((Node *) tlist, &first_last_aggs)) return; if (find_first_last_aggs_walker(parse->havingQual, &first_last_aggs)) return; /* * OK, there is at least the possibility of performing the optimization. * Build an access path for each aggregate. If any of the aggregates * prove to be non-indexable, give up; there is no point in optimizing * just some of them. */ foreach (lc, first_last_aggs) { FirstLastAggInfo *fl_info = (FirstLastAggInfo *) lfirst(lc); MinMaxAggInfo *mminfo = fl_info->m_agg_info; Oid eqop; bool reverse; /* * We'll need the equality operator that goes with the aggregate's * ordering operator. */ eqop = get_equality_op_for_ordering_op(mminfo->aggsortop, &reverse); if (!OidIsValid(eqop)) /* shouldn't happen */ elog(ERROR, "could not find equality operator for ordering operator %u", mminfo->aggsortop); /* * We can use either an ordering that gives NULLS FIRST or one that * gives NULLS LAST; furthermore there's unlikely to be much * performance difference between them, so it doesn't seem worth * costing out both ways if we get a hit on the first one. NULLS * FIRST is more likely to be available if the operator is a * reverse-sort operator, so try that first if reverse. */ if (build_first_last_path(root, fl_info, eqop, mminfo->aggsortop, reverse, reverse)) continue; if (build_first_last_path(root, fl_info, eqop, mminfo->aggsortop, reverse, !reverse)) continue; /* No indexable path for this aggregate, so fail */ return; } /* * OK, we can do the query this way. We are using MinMaxAggPath to store * First/Last Agg path since the logic is almost the same. MinMaxAggPath * is used later on by planner so by reusing it we don't need to re-invent * planner. * * Prepare to create a MinMaxAggPath node. * * First, create an output Param node for each agg. (If we end up not * using the MinMaxAggPath, we'll waste a PARAM_EXEC slot for each agg, * which is not worth worrying about. We can't wait till create_plan time * to decide whether to make the Param, unfortunately.) */ mm_agg_list = NIL; foreach (lc, first_last_aggs) { FirstLastAggInfo *fl_info = (FirstLastAggInfo *) lfirst(lc); MinMaxAggInfo *mminfo = fl_info->m_agg_info; mminfo->param = SS_make_initplan_output_param(root, exprType((Node *) mminfo->target), -1, exprCollation((Node *) mminfo->target)); mm_agg_list = lcons(mminfo, mm_agg_list); } /* * Create a MinMaxAggPath node with the appropriate estimated costs and * other needed data, and add it to the UPPERREL_GROUP_AGG upperrel, where * it will compete against the standard aggregate implementation. (It * will likely always win, but we need not assume that here.) * * Note: grouping_planner won't have created this upperrel yet, but it's * fine for us to create it first. We will not have inserted the correct * consider_parallel value in it, but MinMaxAggPath paths are currently * never parallel-safe anyway, so that doesn't matter. Likewise, it * doesn't matter that we haven't filled FDW-related fields in the rel. */ grouped_rel = fetch_upper_rel(root, UPPERREL_GROUP_AGG, NULL); minmaxagg_path = create_minmaxagg_path(root, grouped_rel, create_pathtarget(root, tlist), mm_agg_list, (List *) parse->havingQual); /* Let's replace Aggref node since we will use subquery we've generated */ replace_aggref_in_tlist(minmaxagg_path); add_path(grouped_rel, (Path *) minmaxagg_path); } /* * find_first_last_aggs_walker * Recursively scan the Aggref nodes in an expression tree, and check * that each one is a FIRST/LAST aggregate. If so, build a list of the * distinct aggregate calls in the tree. * * Returns TRUE if a non-FIRST/LAST aggregate is found, FALSE otherwise. * (This seemingly-backward definition is used because expression_tree_walker * aborts the scan on TRUE return, which is what we want.) * * Found aggregates are added to the list at *context; it's up to the caller * to initialize the list to NIL. * * This does not descend into subqueries, and so should be used only after * reduction of sublinks to subplans. There mustn't be outer-aggregate * references either. * * Major differences from find_minmax_aggs_walker (planagg.c): * - only allow Aggref with two arguments * - wrap agg info in FirstLastAggInfo */ static bool find_first_last_aggs_walker(Node *node, List **context) { if (node == NULL) return false; if (IsA(node, Aggref)) { Aggref *aggref = (Aggref *) node; Oid aggsortop; TargetEntry *value; TargetEntry *sort; MinMaxAggInfo *mminfo; ListCell *l; FirstLastAggInfo *fl_info; Oid sort_oid; TypeCacheEntry *sort_tce; StrategyNumber func_strategy; Assert(aggref->agglevelsup == 0); if (list_length(aggref->args) != 2) return true; /* it couldn't be first/last */ /* * ORDER BY is usually irrelevant for FIRST/LAST, but it can change * the outcome if the aggsortop's operator class recognizes * non-identical values as equal. For example, 4.0 and 4.00 are equal * according to numeric_ops, yet distinguishable. If FIRST() receives * more than one value equal to 4.0 and no value less than 4.0, it is * unspecified which of those equal values FIRST() returns. An ORDER * BY expression that differs for each of those equal values of the * argument expression makes the result predictable once again. This * is a niche requirement, and we do not implement it with subquery * paths. In any case, this test lets us reject ordered-set aggregates * quickly. */ if (aggref->aggorder != NIL) return true; /* note: we do not care if DISTINCT is mentioned ... */ /* * We might implement the optimization when a FILTER clause is present * by adding the filter to the quals of the generated subquery. For * now, just punt. */ if (aggref->aggfilter != NULL) return true; /* We sort by second argument (eg. time) */ sort_oid = lsecond_oid(aggref->aggargtypes); func_strategy = get_func_strategy(aggref->aggfnoid); if (func_strategy == InvalidStrategy) return true; /* not first/last aggregate */ sort_tce = lookup_type_cache(sort_oid, TYPECACHE_BTREE_OPFAMILY); aggsortop = get_opfamily_member(sort_tce->btree_opf, sort_oid, sort_oid, func_strategy); if (!OidIsValid(aggsortop)) elog(ERROR, "Cannot resolve sort operator for function \"%s\" and type \"%s\"", format_procedure(aggref->aggfnoid), format_type_be(sort_oid)); /* Used in projection */ value = (TargetEntry *) linitial(aggref->args); /* Used in ORDER BY */ sort = (TargetEntry *) lsecond(aggref->args); if (contain_mutable_functions((Node *) sort->expr)) return true; /* not potentially indexable */ if (type_is_rowtype(exprType((Node *) sort->expr))) return true; /* IS NOT NULL would have weird semantics */ /* * Check whether it's already in the list, and add it if not. */ foreach (l, *context) { mminfo = (MinMaxAggInfo *) lfirst(l); if (mminfo->aggfnoid == aggref->aggfnoid && equal(mminfo->target, value->expr)) return false; } mminfo = makeNode(MinMaxAggInfo); mminfo->aggfnoid = aggref->aggfnoid; mminfo->aggsortop = aggsortop; mminfo->target = value->expr; mminfo->subroot = NULL; mminfo->path = NULL; mminfo->pathcost = 0; mminfo->param = NULL; fl_info = palloc(sizeof(FirstLastAggInfo)); fl_info->m_agg_info = mminfo; fl_info->sort = sort->expr; *context = lappend(*context, fl_info); /* * We need not recurse into the argument, since it can't contain any * aggregates. */ return false; } Assert(!IsA(node, SubLink)); return expression_tree_walker(node, find_first_last_aggs_walker, (void *) context); } /* * build_first_last_path * Given a FIRST/LAST aggregate, try to build an indexscan Path it can be * optimized with. * We will generate subquery with value and sort target, where we * SELECT value and we ORDER BY sort. * * If successful, stash the best path in *mminfo and return TRUE. * Otherwise, return FALSE. * * Major differences when compared to build_minmax_path(planagg.c): * - generates different subquery * - works with two target entries (value and sortby) * - resets EquivalenceClass(es) * */ static bool build_first_last_path(PlannerInfo *root, FirstLastAggInfo *fl_info, Oid eqop, Oid sortop, bool reverse_sort, bool nulls_first) { PlannerInfo *subroot; Query *parse; TargetEntry *value_target; TargetEntry *sort_target; List *tlist; NullTest *ntest; SortGroupClause *sortcl; RelOptInfo *final_rel; Path *sorted_path; Cost path_cost; double path_fraction; MinMaxAggInfo *mminfo; ListCell *lc; /* * We are going to construct what is effectively a sub-SELECT query, so * clone the current query level's state and adjust it to make it look * like a subquery. Any outer references will now be one level higher * than before. (This means that when we are done, there will be no Vars * of level 1, which is why the subquery can become an initplan.) */ subroot = (PlannerInfo *) palloc(sizeof(PlannerInfo)); memcpy(subroot, root, sizeof(PlannerInfo)); subroot->query_level++; subroot->parent_root = root; /* reset subplan-related stuff */ subroot->plan_params = NIL; subroot->outer_params = NULL; subroot->init_plans = NIL; /* reset EquivalenceClass since we will create it later on */ subroot->eq_classes = NIL; subroot->parse = parse = copyObject(root->parse); IncrementVarSublevelsUp((Node *) parse, 1, 1); #if PG16_GE /* Reset placeholdersFrozen: https://github.com/postgres/postgres/commit/b3ff6c74 */ subroot->placeholdersFrozen = false; #endif /* append_rel_list might contain outer Vars? */ subroot->append_rel_list = copyObject(root->append_rel_list); IncrementVarSublevelsUp((Node *) subroot->append_rel_list, 1, 1); /* There shouldn't be any OJ info to translate, as yet */ Assert(subroot->join_info_list == NIL); /* and we haven't made equivalence classes, either */ Assert(subroot->eq_classes == NIL); /* and we haven't created PlaceHolderInfos, either */ Assert(subroot->placeholder_list == NIL); mminfo = fl_info->m_agg_info; /*---------- * Generate modified query of the form * (SELECT value FROM tab * WHERE sort IS NOT NULL AND existing-quals * ORDER BY sort ASC/DESC * LIMIT 1) *---------- */ /* * Value and sort target entries but sort target is eliminated later on * from target list */ value_target = makeTargetEntry(copyObject(mminfo->target), (AttrNumber) 1, pstrdup("value"), false); sort_target = makeTargetEntry(copyObject(fl_info->sort), (AttrNumber) 2, pstrdup("sort"), true); tlist = list_make2(value_target, sort_target); subroot->processed_tlist = parse->targetList = tlist; /* No HAVING, no DISTINCT, no aggregates anymore */ parse->havingQual = NULL; subroot->hasHavingQual = false; parse->distinctClause = NIL; parse->hasDistinctOn = false; parse->hasAggs = false; /* * Build "sort IS NOT NULL" expression. Note that target can still be NULL. * We don't need it if the order is NULLS LAST. */ if (nulls_first) { ntest = makeNode(NullTest); ntest->nulltesttype = IS_NOT_NULL; ntest->arg = copyObject(fl_info->sort); /* we checked it wasn't a rowtype in find_minmax_aggs_walker */ ntest->argisrow = false; ntest->location = -1; /* User might have had that in WHERE already */ if (!list_member((List *) parse->jointree->quals, ntest)) parse->jointree->quals = (Node *) lcons(ntest, (List *) parse->jointree->quals); } /* Build suitable ORDER BY clause */ sortcl = makeNode(SortGroupClause); sortcl->tleSortGroupRef = assignSortGroupRef(sort_target, tlist); sortcl->eqop = eqop; sortcl->sortop = sortop; sortcl->nulls_first = nulls_first; sortcl->hashable = false; /* no need to make this accurate */ #if PG18_GE /* Track sort direction in SortGroupClause * https://github.com/postgres/postgres/commit/0d2aa4d4 */ sortcl->reverse_sort = reverse_sort; #endif parse->sortClause = list_make1(sortcl); /* set up expressions for LIMIT 1 */ parse->limitOffset = NULL; parse->limitCount = (Node *) makeConst(INT8OID, -1, InvalidOid, sizeof(int64), Int64GetDatum(1), false, FLOAT8PASSBYVAL); /* * Generate the best paths for this query, telling query_planner that we * have LIMIT 1. */ subroot->tuple_fraction = 1.0; subroot->limit_tuples = 1.0; /* min/max optimizations usually happen before * inheritance-relations are expanded, and thus query_planner will * try to expand our hypertables if they are marked as * inheritance-relations. Since we do not want this, we must mark * hypertables as non-inheritance now. */ foreach (lc, subroot->parse->rtable) { RangeTblEntry *rte = lfirst_node(RangeTblEntry, lc); if (ts_rte_is_hypertable(rte)) { ListCell *prev = NULL; ListCell *next = list_head(subroot->append_rel_list); Assert(rte->inh); rte->inh = false; /* query planner gets confused when entries in the * append_rel_list refer to entries in the relarray that * don't exist. Since we need to expand hypertables in the * subquery, all of the chunk entries will be invalid in * this manner, so we remove them from the list. */ /* Performance Enhancement: This can be made non-quadratic by: * 1) Loop once over all RTEs, storing the relid of any RTE that is a hypertable in * a bitset and setting its 'inh' flag to false 2) Loop over the append_rel_list, * removing any AppendRelInfo that has a parent relid which is in the previously * created bitset (i.e., is a hypertable) */ while (next != NULL) { AppendRelInfo *app = lfirst(next); if (app->parent_reloid == rte->relid) { subroot->append_rel_list = list_delete_cell(subroot->append_rel_list, next); next = prev != NULL ? lnext(subroot->append_rel_list, next) : list_head(subroot->append_rel_list); } else { prev = next; next = lnext(subroot->append_rel_list, next); } } } } final_rel = query_planner(subroot, first_last_qp_callback, NULL); /* we need to disable inheritance so the chunks are re-expanded correctly in the subroot */ foreach (lc, root->parse->rtable) { RangeTblEntry *rte = lfirst_node(RangeTblEntry, lc); if (ts_rte_is_hypertable(rte)) rte->inh = true; } /* * Since we didn't go through subquery_planner() to handle the subquery, * we have to do some of the same cleanup it would do, in particular cope * with params and initplans used within this subquery. (This won't * matter if we end up not using the subplan.) */ SS_identify_outer_params(subroot); SS_charge_for_initplans(subroot, final_rel); /* * Get the best presorted path, that being the one that's cheapest for * fetching just one row. If there's no such path, fail. */ if (final_rel->rows > 1.0) path_fraction = 1.0 / final_rel->rows; else path_fraction = 1.0; sorted_path = get_cheapest_fractional_path_for_pathkeys(final_rel->pathlist, subroot->query_pathkeys, NULL, path_fraction); if (!sorted_path) return false; /* * The path might not return exactly what we want, so fix that. (We * assume that this won't change any conclusions about which was the * cheapest path.) */ sorted_path = apply_projection_to_path(subroot, final_rel, sorted_path, create_pathtarget(subroot, tlist)); /* * Determine cost to get just the first row of the presorted path. * * Note: cost calculation here should match * compare_fractional_path_costs(). */ path_cost = sorted_path->startup_cost + path_fraction * (sorted_path->total_cost - sorted_path->startup_cost); /* Save state for further processing */ mminfo->subroot = subroot; mminfo->path = sorted_path; mminfo->pathcost = path_cost; return true; } /* * Compute query_pathkeys and other pathkeys during query_planner() */ static void first_last_qp_callback(PlannerInfo *root, void *extra) { root->group_pathkeys = NIL; root->window_pathkeys = NIL; root->distinct_pathkeys = NIL; root->sort_pathkeys = make_pathkeys_for_sortclauses(root, root->parse->sortClause, root->parse->targetList); root->query_pathkeys = root->sort_pathkeys; } ================================================ FILE: src/planner/constify_now.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <datatype/timestamp.h> #include <nodes/makefuncs.h> #include <optimizer/optimizer.h> #include <utils/fmgroids.h> #include "cache.h" #include "dimension.h" #include "hypertable.h" #include "hypertable_cache.h" #include "planner.h" /* * This implements an optimization to allow now() expression to be * used during plan time chunk exclusions. Since now() is stable it * would not normally be considered for plan time chunk exclusion. * To enable this behaviour we convert `column > now()` expressions * into `column > const AND column > now()`. Assuming that times * always moves forward this is safe even for prepared statements. * * We consider the following expressions valid for this optimization: * - Var > now() * - Var >= now() * - Var > now() - Interval * - Var > now() + Interval * - Var >= now() - Interval * - Var >= now() + Interval * * Interval needs to be Const in those expressions. */ static const Dimension * get_hypertable_dimension(Oid relid, int flags) { Hypertable *ht = ts_planner_get_hypertable(relid, flags); if (!ht) return NULL; return hyperspace_get_open_dimension(ht->space, 0); } bool is_valid_now_func(Node *node) { if (IsA(node, FuncExpr) && castNode(FuncExpr, node)->funcid == F_NOW) return true; if (IsA(node, SQLValueFunction) && castNode(SQLValueFunction, node)->type == SVFOP_CURRENT_TIMESTAMP) return true; return false; } static bool is_valid_now_expr(OpExpr *op, List *rtable) { int flags = CACHE_FLAG_MISSING_OK | CACHE_FLAG_NOCREATE; /* Var > or Var >= */ if ((op->opfuncid != F_TIMESTAMPTZ_GT && op->opfuncid != F_TIMESTAMPTZ_GE) || !IsA(linitial(op->args), Var)) return false; /* * Check that the constraint is actually on a partitioning * column. We only check for match on first open dimension * because that will be the time column. */ Var *var = linitial_node(Var, op->args); if (var->varlevelsup != 0) return false; Assert((int) var->varno <= list_length(rtable)); RangeTblEntry *rte = list_nth(rtable, var->varno - 1); /* * If this query on a view we might have a subquery here * and need to peek into the subquery range table to check * if the constraints are on a hypertable. */ if (rte->rtekind == RTE_SUBQUERY) { /* * Unfortunately the mechanism used to warm up the * hypertable cache does not apply to hypertables * referenced indirectly eg through VIEWs. So we * have to do the lookup for this hypertable without * CACHE_FLAG_NOCREATE flag. */ flags = CACHE_FLAG_MISSING_OK; TargetEntry *tle = list_nth(rte->subquery->targetList, var->varattno - 1); if (!IsA(tle->expr, Var)) return false; var = castNode(Var, tle->expr); if (var->varlevelsup != 0) return false; #if PG18_GE /* PG18 introduced RTEs for group clauses so * we can use rtable to look up GROUP BY expressions. * * https://github.com/postgres/postgres/commit/247dea89 */ RangeTblEntry *group_rte = list_nth(rte->subquery->rtable, var->varno - 1); if (group_rte->rtekind == RTE_GROUP) { Assert(var->varattno > 0); Expr *node = list_nth(group_rte->groupexprs, var->varattno - 1); if (!IsA(node, Var)) return false; var = castNode(Var, node); Assert(var->varno > 0); if (var->varlevelsup != 0) return false; } #endif rte = list_nth(rte->subquery->rtable, var->varno - 1); } const Dimension *dim = get_hypertable_dimension(rte->relid, flags); if (!dim || dim->fd.column_type != TIMESTAMPTZOID || dim->column_attno != var->varattno) return false; /* Var > now() or Var >= now() */ if (is_valid_now_func(lsecond(op->args))) return true; if (!IsA(lsecond(op->args), OpExpr)) return false; /* Var >|>= now() +|- Const */ OpExpr *op_inner = lsecond_node(OpExpr, op->args); if ((op_inner->opfuncid != F_TIMESTAMPTZ_MI_INTERVAL && op_inner->opfuncid != F_TIMESTAMPTZ_PL_INTERVAL) || !is_valid_now_func(linitial(op_inner->args)) || !IsA(lsecond(op_inner->args), Const)) return false; /* * The consttype check should not be necessary since the * operators we whitelist above already mandates it. */ Const *c = lsecond_node(Const, op_inner->args); Assert(c->consttype == INTERVALOID); if (c->constisnull || c->consttype != INTERVALOID) return false; return true; } static Const * make_now_const() { return makeConst(TIMESTAMPTZOID, -1, InvalidOid, sizeof(TimestampTz), #ifdef TS_DEBUG ts_get_mock_time_or_current_time(), #else TimestampTzGetDatum(GetCurrentTransactionStartTimestamp()), #endif false, FLOAT8PASSBYVAL); } /* returns a copy of the expression with the now() call constified */ /* * op will be OpExpr with Var > now() - Expr */ static OpExpr * constify_now_expr(PlannerInfo *root, OpExpr *op) { op = copyObject(op); op->location = PLANNER_LOCATION_MAGIC; if (is_valid_now_func(lsecond(op->args))) { lsecond(op->args) = make_now_const(); return op; } else { OpExpr *op_inner = lsecond_node(OpExpr, op->args); Const *const_offset = lsecond_node(Const, op_inner->args); Assert(const_offset->consttype == INTERVALOID); Interval *offset = DatumGetIntervalP(const_offset->constvalue); /* * Sanity check that this is a supported expression. We should never * end here if it isn't since this is checked in is_valid_now_expr. */ Assert(is_valid_now_func(linitial(op_inner->args))); Const *now = make_now_const(); linitial(op_inner->args) = now; /* * If the interval has a day component then the calculation needs * to take into account daylight saving time switches and thereby a * day would not always be exactly 24 hours. We mitigate this by * adding a safety buffer to account for these dst switches when * dealing with intervals with day component. These calculations * will be repeated with exact values during execution. * Since dst switches seem to range between -1 and 2 hours we set * the safety buffer to 4 hours. * When dealing with Intervals with month component timezone changes * can result in multiple day differences in the outcome of these * calculations due to different month lengths. When dealing with * months we add a 7 day safety buffer. * For all these calculations it is fine if we exclude less chunks * than strictly required for the operation, additional exclusion * with exact values will happen in the executor. But under no * circumstances must we exclude too much cause there would be * no way for the executor to get those chunks back. */ if (offset->day != 0 || offset->month != 0) { TimestampTz now_value = DatumGetTimestampTz(now->constvalue); if (offset->month != 0) now_value -= 7 * USECS_PER_DAY; if (offset->day != 0) now_value -= 4 * USECS_PER_HOUR; now->constvalue = TimestampTzGetDatum(now_value); } /* * Normally estimate_expression_value is not safe to use during planning * since it also evaluates stable expressions. Since we only allow a * very limited subset of expressions for this optimization it is safe * for those expressions we allowed earlier. * estimate_expression_value should always be able to completely constify * the expression due to the restrictions we impose on the expressions * supported. */ lsecond(op->args) = estimate_expression_value(root, (Node *) op_inner); Assert(IsA(lsecond(op->args), Const)); op->location = PLANNER_LOCATION_MAGIC; return op; } } Node * ts_constify_now(PlannerInfo *root, List *rtable, Node *node) { Assert(node); switch (nodeTag(node)) { case T_OpExpr: if (is_valid_now_expr(castNode(OpExpr, node), rtable)) { List *args = list_make2(copyObject(node), constify_now_expr(root, castNode(OpExpr, node))); return (Node *) makeBoolExpr(AND_EXPR, args, -1); } break; case T_BoolExpr: { List *additions = NIL; ListCell *lc; BoolExpr *be = castNode(BoolExpr, node); /* We only look for top-level AND */ if (be->boolop != AND_EXPR) return node; foreach (lc, be->args) { additions = lappend(additions, ts_constify_now(root, rtable, (Node *) lfirst(lc))); } if (additions) be->args = additions; break; } default: break; } return node; } ================================================ FILE: src/planner/constraint_cleanup.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <nodes/pathnodes.h> #include "planner.h" /* * This code deals with removing the intermediate constraints * we added before planning to improve chunk exclusion. */ static bool restrictinfo_is_marked(RestrictInfo *ri) { switch (nodeTag(ri->clause)) { case T_OpExpr: return castNode(OpExpr, ri->clause)->location == PLANNER_LOCATION_MAGIC; case T_ScalarArrayOpExpr: return castNode(ScalarArrayOpExpr, ri->clause)->location == PLANNER_LOCATION_MAGIC; default: break; } return false; } /* * Remove marked constraints from RestrictInfo clause. */ static List * restrictinfo_cleanup(List *restrictinfos, bool *pfiltered) { List *filtered_ri = NIL; ListCell *lc; bool filtered = false; if (!restrictinfos) return NULL; foreach (lc, restrictinfos) { RestrictInfo *ri = (RestrictInfo *) lfirst(lc); if (restrictinfo_is_marked(ri)) { filtered = true; continue; } filtered_ri = lappend(filtered_ri, ri); } if (pfiltered) *pfiltered = filtered; return filtered ? filtered_ri : restrictinfos; } /* * Remove marked constraints from IndexPath. */ static void indexpath_cleanup(IndexPath *path) { ListCell *lc; List *filtered_ic = NIL; path->indexinfo->indrestrictinfo = restrictinfo_cleanup(path->indexinfo->indrestrictinfo, NULL); foreach (lc, path->indexclauses) { IndexClause *iclause = lfirst_node(IndexClause, lc); if (restrictinfo_is_marked(iclause->rinfo)) continue; filtered_ic = lappend(filtered_ic, iclause); } path->indexclauses = filtered_ic; } void ts_planner_constraint_cleanup(PlannerInfo *root, RelOptInfo *rel) { ListCell *lc; bool filtered = false; if (rel->baserestrictinfo) rel->baserestrictinfo = restrictinfo_cleanup(rel->baserestrictinfo, &filtered); /* * If we added constraints those will be present in baserestrictinfo. * If we did not remove anything from baserestrictinfo in the step * above we can skip looking in the paths. */ if (filtered) { /* * For seqscan cleaning up baserestrictinfo is enough but for * BitmapHeapPath and IndexPath we need some extra steps. */ foreach (lc, rel->pathlist) { switch (nodeTag(lfirst(lc))) { case T_BitmapHeapPath: { BitmapHeapPath *path = lfirst_node(BitmapHeapPath, lc); if (IsA(path->bitmapqual, IndexPath)) indexpath_cleanup(castNode(IndexPath, path->bitmapqual)); break; } case T_IndexPath: { indexpath_cleanup(castNode(IndexPath, lfirst(lc))); break; } default: break; } } } } ================================================ FILE: src/planner/expand_hypertable.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ /* This planner optimization reduces planning times when a hypertable has many chunks. * It does this by expanding hypertable chunks manually, eliding the `expand_inherited_tables` * logic used by PG. * * Slow planning time were previously seen because `expand_inherited_tables` expands all chunks of * a hypertable, without regard to constraints present in the query. Then, `get_relation_info` is * called on all chunks before constraint exclusion. Getting the statistics on many chunks ends * up being expensive because RelationGetNumberOfBlocks has to open the file for each relation. * This gets even worse under high concurrency. * * This logic solves this by expanding only the chunks needed to fulfil the query instead of all * chunks. In effect, it moves chunk exclusion up in the planning process. But, we actually don't * use constraint exclusion here, but rather a variant of range exclusion implemented by * HypertableRestrictInfo. * */ #include <postgres.h> #include <catalog/pg_constraint.h> #include <catalog/pg_inherits.h> #include <catalog/pg_namespace.h> #include <catalog/pg_type.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <nodes/plannodes.h> #include <optimizer/cost.h> #include <optimizer/optimizer.h> #include <optimizer/pathnode.h> #include <optimizer/planmain.h> #include <optimizer/planner.h> #include <optimizer/prep.h> #include <optimizer/restrictinfo.h> #include <optimizer/tlist.h> #include <parser/parse_func.h> #include <parser/parse_type.h> #include <parser/parsetree.h> #include <partitioning/partbounds.h> #include <utils/builtins.h> #include <utils/date.h> #include <utils/errcodes.h> #include <utils/fmgroids.h> #include <utils/fmgrprotos.h> #include <utils/lsyscache.h> #include <utils/syscache.h> #include <utils/timestamp.h> #include <utils/typcache.h> #include <utils/uuid.h> #include "compat/compat.h" #include "annotations.h" #include "chunk.h" #include "cross_module_fn.h" #include "guc.h" #include "hypercube.h" #include "hypertable.h" #include "hypertable_restrict_info.h" #include "import/planner.h" #include "nodes/chunk_append/chunk_append.h" #include "planner.h" #include "time_utils.h" #include "uuid.h" typedef struct CollectQualCtx { PlannerInfo *root; RelOptInfo *rel; List *restrictions; List *propagate_conditions; List *all_quals; int join_level; } CollectQualCtx; static void propagate_join_quals(PlannerInfo *root, RelOptInfo *rel, CollectQualCtx *ctx); static bool is_time_bucket_function(Expr *node) { if (IsA(node, FuncExpr) && strncmp(get_func_name(castNode(FuncExpr, node)->funcid), "time_bucket", NAMEDATALEN) == 0) return true; return false; } static void ts_add_append_rel_infos(PlannerInfo *root, List *appinfos) { ListCell *lc; root->append_rel_list = list_concat(root->append_rel_list, appinfos); /* root->append_rel_array is required to be able to hold all the * additional entries by previous call to expand_planner_arrays */ Assert(root->append_rel_array); foreach (lc, appinfos) { AppendRelInfo *appinfo = lfirst_node(AppendRelInfo, lc); int child_relid = appinfo->child_relid; Assert(child_relid < root->simple_rel_array_size); root->append_rel_array[child_relid] = appinfo; } } /* * Pre-check to determine if an expression is eligible for constification. * A more thorough check is in constify_timestamptz_op_interval. */ static bool is_timestamptz_op_interval(Expr *expr) { OpExpr *op; Const *c1, *c2; if (!IsA(expr, OpExpr)) return false; op = castNode(OpExpr, expr); if (op->opresulttype != TIMESTAMPTZOID || op->args->length != 2 || !IsA(linitial(op->args), Const) || !IsA(llast(op->args), Const)) return false; c1 = linitial_node(Const, op->args); c2 = llast_node(Const, op->args); return (c1->consttype == TIMESTAMPTZOID && c2->consttype == INTERVALOID) || (c1->consttype == INTERVALOID && c2->consttype == TIMESTAMPTZOID); } static Const * integral_timeval_to_const(int64 value, Oid type) { bool typbyval = get_typbyval(type); switch (type) { case INT2OID: return makeConst(type, -1, InvalidOid, 2, Int16GetDatum(value), false, typbyval); case INT4OID: return makeConst(type, -1, InvalidOid, 4, Int32GetDatum(value), false, typbyval); case INT8OID: return makeConst(type, -1, InvalidOid, 8, Int64GetDatum(value), false, typbyval); case DATEOID: return makeConst(type, -1, InvalidOid, sizeof(DateADT), DateADTGetDatum(value), false, typbyval); case TIMESTAMPOID: return makeConst(type, -1, InvalidOid, sizeof(Timestamp), TimestampGetDatum(value), false, typbyval); case TIMESTAMPTZOID: return makeConst(type, -1, InvalidOid, sizeof(TimestampTz), TimestampTzGetDatum(value), false, typbyval); case UUIDOID: { /* * UUIDv7 doesn't support timestamps smaller than the UNIX epoch. However, caggs often * refresh from "beginning of time" so we need to restrict the lower boundary value to * the UNIX epoch. */ if (value <= UNIX_EPOCH_AS_TIMESTAMP) value = UNIX_EPOCH_AS_TIMESTAMP; pg_uuid_t *uuid = ts_create_uuid_v7_from_timestamptz((TimestampTz) value, true); return makeConst(type, -1, InvalidOid, UUID_LEN, UUIDPGetDatum(uuid), false, typbyval); } default: elog(ERROR, "unsupported datatype in %s: %s", __func__, format_type_be(type)); pg_unreachable(); } } static int64 const_to_integral_timeval(const Const *cnst) { Assert(!cnst->constisnull); switch (cnst->consttype) { case INT2OID: return (int64) (DatumGetInt16(cnst->constvalue)); case INT4OID: return (int64) (DatumGetInt32(cnst->constvalue)); case INT8OID: return DatumGetInt64(cnst->constvalue); case DATEOID: return DatumGetDateADT(cnst->constvalue); case TIMESTAMPOID: return DatumGetTimestamp(cnst->constvalue); case TIMESTAMPTZOID: return DatumGetTimestampTz(cnst->constvalue); case UUIDOID: { /* * While it is possible to extract the timestamp from a UUID, there is currently no use * case where this function is used since the UUID-based time_bucket() function returns * a timestamptz. Thus, any value compared to is also a timestamptz and not a UUID. * */ TS_FALLTHROUGH; } default: elog(ERROR, "unsupported datatype in %s: %s", __func__, format_type_be(cnst->consttype)); pg_unreachable(); } } /* * Constify expressions of the following form in WHERE clause: * * column OP timestamptz - interval * column OP timestamptz + interval * column OP interval + timestamptz * * Iff interval has no month component. * * Since the operators for timestamptz OP interval are marked * as stable they will not be constified during planning. * However, intervals without a month component can be safely * constified during planning as the result of those calculations * do not depend on the timezone setting. */ static OpExpr * constify_timestamptz_op_interval(PlannerInfo *root, OpExpr *constraint) { Expr *left, *right; OpExpr *op; bool var_on_left = false; Interval *interval; Const *c_ts, *c_int; Datum constified; PGFunction opfunc; Oid ts_pl_int, ts_mi_int, int_pl_ts; /* checked in caller already so only asserting */ Assert(constraint->args->length == 2); left = linitial(constraint->args); right = llast(constraint->args); if (IsA(left, Var) && IsA(right, OpExpr)) { op = castNode(OpExpr, right); var_on_left = true; } else if (IsA(left, OpExpr) && IsA(right, Var)) { op = castNode(OpExpr, left); } else return constraint; ts_pl_int = ts_get_operator("+", PG_CATALOG_NAMESPACE, TIMESTAMPTZOID, INTERVALOID); ts_mi_int = ts_get_operator("-", PG_CATALOG_NAMESPACE, TIMESTAMPTZOID, INTERVALOID); int_pl_ts = ts_get_operator("+", PG_CATALOG_NAMESPACE, INTERVALOID, TIMESTAMPTZOID); if (op->opno == ts_pl_int) { /* TIMESTAMPTZ + INTERVAL */ opfunc = timestamptz_pl_interval; c_ts = linitial_node(Const, op->args); c_int = llast_node(Const, op->args); } else if (op->opno == ts_mi_int) { /* TIMESTAMPTZ - INTERVAL */ opfunc = timestamptz_mi_interval; c_ts = linitial_node(Const, op->args); c_int = llast_node(Const, op->args); } else if (op->opno == int_pl_ts) { /* INTERVAL + TIMESTAMPTZ */ opfunc = timestamptz_pl_interval; c_int = linitial_node(Const, op->args); c_ts = llast_node(Const, op->args); } else return constraint; /* * arg types should match operator and were checked in precheck * so only asserting here */ Assert(c_ts->consttype == TIMESTAMPTZOID); Assert(c_int->consttype == INTERVALOID); if (c_ts->constisnull || c_int->constisnull) return constraint; interval = DatumGetIntervalP(c_int->constvalue); /* * constification is only safe when the interval has no month component * because month length is variable and calculation depends on local timezone */ if (interval->month != 0) return constraint; constified = DirectFunctionCall2(opfunc, c_ts->constvalue, c_int->constvalue); /* * Since constifying intervals with day component does depend on the timezone * this can lead to different results around daylight saving time switches. * So we add a safety buffer when the interval has day components to counteract. */ if (interval->day != 0) { bool add; TimestampTz constified_tstz = DatumGetTimestampTz(constified); switch (constraint->opfuncid) { case F_TIMESTAMPTZ_LE: case F_TIMESTAMPTZ_LT: add = true; break; case F_TIMESTAMPTZ_GE: case F_TIMESTAMPTZ_GT: add = false; break; default: return constraint; } /* * If Var is on wrong side reverse the direction. */ if (!var_on_left) add = !add; /* * The safety buffer is chosen to be 4 hours because daylight saving time * changes seem to be in the range between -1 and 2 hours. */ if (add) constified_tstz += 4 * USECS_PER_HOUR; else constified_tstz -= 4 * USECS_PER_HOUR; constified = TimestampTzGetDatum(constified_tstz); } c_ts = copyObject(c_ts); c_ts->constvalue = constified; if (var_on_left) right = (Expr *) c_ts; else left = (Expr *) c_ts; return (OpExpr *) make_opclause(constraint->opno, constraint->opresulttype, constraint->opretset, left, right, constraint->opcollid, constraint->inputcollid); } typedef struct TimeBucketInfo { Oid rettype; /* Type of the return value */ Const *width; /* Bucket width */ Node *timeval; /* Bucket "time" value */ Oid timetype; /* Type of the time value */ uint16 numargs; /* Number of bucket function arguments */ } TimeBucketInfo; /* * Representation of a parse time bucket Qual: * * <time_bucket() OP value> */ typedef struct TimeBucketQual { TimeBucketInfo tb; int strategy; Const *value; } TimeBucketQual; /* * Parse an expression of form <time_bucket(width, column) OP value> and extract the important * components into a TimeBucketQual struct. * * Returns false if the expression does not fit the expected format. */ static bool extract_time_bucket_qual(Expr *node, TimeBucketQual *tbqual) { if (!IsA(node, OpExpr)) return false; OpExpr *op = castNode(OpExpr, node); if (list_length((op)->args) != 2) return false; Expr *left = linitial((op)->args); Expr *right = lsecond((op)->args); FuncExpr *time_bucket; MemSet(tbqual, 0, sizeof(TimeBucketQual)); Oid opno = InvalidOid; if (IsA(left, FuncExpr) && IsA(right, Const)) { time_bucket = castNode(FuncExpr, left); tbqual->value = castNode(Const, right); opno = op->opno; } else if (IsA(right, FuncExpr) && IsA(left, Const)) { time_bucket = castNode(FuncExpr, right); tbqual->value = castNode(Const, left); opno = get_commutator(op->opno); } else { return false; } if (!is_time_bucket_function((Expr *) time_bucket) || tbqual->value->constisnull) return false; Const *width = linitial(time_bucket->args); /* Get the time/partitioning column argument */ Node *timearg = lsecond(time_bucket->args); if (!IsA(width, Const) || width->constisnull) return false; tbqual->tb.numargs = list_length(time_bucket->args); tbqual->tb.width = width; tbqual->tb.timeval = timearg; tbqual->tb.timetype = exprType(timearg); tbqual->tb.rettype = exprType((Node *) time_bucket); /* 3 or more args should have Const 3rd arg */ if (list_length(time_bucket->args) > 2 && !IsA(lthird(time_bucket->args), Const)) return false; /* 5 args variants should have Const 4th and 5th arg */ if (list_length(time_bucket->args) == 5 && (!IsA(lfourth(time_bucket->args), Const) || !IsA(lfifth(time_bucket->args), Const))) return false; Assert(list_length(time_bucket->args) == 2 || list_length(time_bucket->args) == 3 || list_length(time_bucket->args) == 5); TypeCacheEntry *tce = lookup_type_cache(tbqual->tb.rettype, TYPECACHE_BTREE_OPFAMILY); tbqual->strategy = get_op_opfamily_strategy(opno, tce->btree_opf); return true; } /* * Convert at time_bucket() width argument (typically Interval or integer) to a microseconds * integer. Also check that the width (interval) doesn't overflow the time value. */ static bool time_bucket_width_to_integral(const Const *width, Oid bucket_type, int64 integral_value, int64 *integral_width) { switch (width->consttype) { case INT2OID: case INT4OID: case INT8OID: /* We can support the offset variants of time_bucket as the * amount of shifting they do is never bigger than the bucketing * width. */ *integral_width = const_to_integral_timeval(width); if (integral_value >= ts_time_get_max(bucket_type) - *integral_width) return false; break; case INTERVALOID: { Interval *interval = DatumGetIntervalP(width->constvalue); /* * Optimization can't be applied when interval has month component. */ if (interval->month != 0) return false; if (bucket_type == DATEOID) { /* bail out if interval->time can't be exactly represented as a double */ if (interval->time >= 0x3FFFFFFFFFFFFFLL) return false; *integral_width = interval->day + ceil((double) interval->time / (double) USECS_PER_DAY); if (integral_value >= (TS_DATE_END - *integral_width)) return false; } else if (bucket_type == TIMESTAMPOID || bucket_type == TIMESTAMPTZOID) { /* * If width interval has day component we merge it with time component */ *integral_width = interval->time; if (interval->day != 0) { /* * if our transformed restriction would overflow we skip adding it */ if (interval->time >= TS_TIMESTAMP_END - interval->day * USECS_PER_DAY) return false; *integral_width += interval->day * USECS_PER_DAY; } if (integral_value >= (TS_TIMESTAMP_END - *integral_width)) return false; } else { return false; } break; } default: return false; } return true; } /* * Transform time_bucket calls of the following form in WHERE clause: * * time_bucket(width, column) OP value * * Since time_bucket always returns the lower bound of the bucket * for lower bound comparisons the width is not relevant and the * following transformation can be applied: * * time_bucket(width, column) > value * column > value * * Example with values: * * time_bucket(10, column) > 109 * column > 109 * * For upper bound comparisons width needs to be taken into account * and we need to extend the upper bound by width to capture all * possible values. * * time_bucket(width, column) < value * column < value + width * * Example with values: * * time_bucket(10, column) < 100 * column < 100 + 10 * * Expressions with value on the left side will be switched around * when building the expression for RestrictInfo. * * If the transformation cannot be applied, returns NULL. */ Expr * ts_transform_time_bucket_comparison(Expr *node) { TimeBucketQual tbqual; if (!extract_time_bucket_qual(node, &tbqual)) return NULL; /* * The qual is an expression <time_bucket OP value> or <value OP time_bucket>. Convert the value * to integral time format. */ int64 integral_value = const_to_integral_timeval(tbqual.value); Const *newvalue = NULL; /* * We strip the time_bucket() from the expression, leaving the input "time" argument. Depending * on the comparison OP, the value might need adjustment. Then the value is converted to the * input/column type for time_bucket(). In most cases, the value's original type and the bucket * input type is the same (e.g. TIMESTAMPTZ), but in some cases they differ. For example, it is * possible to compare an int8 bucket function with an int4 value. In the case of UUID bucket, * the bucket function's input type (UUID) is different from the output type (TIMESTAMPTZ), so * the timestamp value needs to be converted to a boundary UUID. */ switch (tbqual.strategy) { case BTGreaterStrategyNumber: case BTGreaterEqualStrategyNumber: /* * Since time_bucket will always shift the input to the left this * transformation is always safe even in the presence of offset variants. * * Handle expressions of form: * * - column > value * - column >= value */ newvalue = integral_timeval_to_const(integral_value, tbqual.tb.timetype); break; case BTLessStrategyNumber: case BTLessEqualStrategyNumber: { /* * Handle expressions of form: * * - column < value + width * - column <= value + width * */ int64 integral_width = 0; if (!time_bucket_width_to_integral(tbqual.tb.width, tbqual.tb.rettype, integral_value, &integral_width)) return NULL; if (tbqual.strategy == BTLessStrategyNumber && tbqual.tb.numargs == 2 && integral_value % integral_width == 0) newvalue = integral_timeval_to_const(integral_value, tbqual.tb.timetype); else newvalue = integral_timeval_to_const(integral_value + integral_width, tbqual.tb.timetype); break; } default: return NULL; } Assert(newvalue != NULL); /* Create a new "unwrapped" OpExpr using the time_bucket() input/column type */ TypeCacheEntry *tce = lookup_type_cache(tbqual.tb.timetype, TYPECACHE_BTREE_OPFAMILY); Oid opno = get_opfamily_member(tce->btree_opf, tce->btree_opintype, tce->btree_opintype, tbqual.strategy); OpExpr *op = (OpExpr *) copyObject(node); op->args = list_make2(tbqual.tb.timeval, newvalue); op->opno = opno; /* The operator might have changed, so reset the function ID */ op->opfuncid = InvalidOid; return &op->xpr; } /* * Since baserestrictinfo is not yet set by the planner, we have to derive * it ourselves. It's safe for us to miss some restrict info clauses (this * will just result in more chunks being included) so this does not need * to be as comprehensive as the PG native derivation. This is inspired * by the derivation in `deconstruct_recurse` in PG * * TODO: as of 2025, the baserestrictinfo and joininfo is already set when the * hypertable expansion code is called, so this does duplicate work. If any bugs * are found in this code, it should be switched to use the RelOptInfos and * equivalence classes instead of the parse tree. The chunk exclusion code for * the non-join clauses was already changed to use the former. */ static Node * process_quals(Node *quals, CollectQualCtx *ctx, bool is_outer_join) { ListCell *lc; ListCell *prev pg_attribute_unused() = NULL; List *additional_quals = NIL; for (lc = list_head((List *) quals); lc != NULL; prev = lc, lc = lnext((List *) quals, lc)) { Expr *qual = lfirst(lc); Relids relids = pull_varnos(ctx->root, (Node *) qual); int num_rels = bms_num_members(relids); /* stop processing if not for current rel */ if (num_rels != 1 || !bms_is_member(ctx->rel->relid, relids)) continue; if (IsA(qual, OpExpr) && list_length(castNode(OpExpr, qual)->args) == 2) { OpExpr *op = castNode(OpExpr, qual); Expr *left = linitial(op->args); Expr *right = lsecond(op->args); if ((IsA(left, Var) && is_timestamptz_op_interval(right)) || (IsA(right, Var) && is_timestamptz_op_interval(left))) { /* * check for constraints with TIMESTAMPTZ OP INTERVAL calculations */ qual = (Expr *) constify_timestamptz_op_interval(ctx->root, op); } else { /* * check for time_bucket comparisons * time_bucket(Const, time_colum) > Const */ Expr *transformed = ts_transform_time_bucket_comparison(qual); if (transformed != NULL) { /* * if we could transform the expression we add it to the list of * quals so it can be used as an index condition */ additional_quals = lappend(additional_quals, transformed); /* * Also use the transformed qual for chunk exclusion. */ qual = transformed; } } } /* Do not include this restriction if this is an outer join. Including * the restriction would exclude chunks and thus rows of the outer * relation when it should show all rows */ if (!is_outer_join) ctx->restrictions = lappend(ctx->restrictions, make_simple_restrictinfo(ctx->root, qual)); } return (Node *) list_concat((List *) quals, additional_quals); } static Node * timebucket_annotate(Node *quals, CollectQualCtx *ctx) { ListCell *lc; List *additional_quals = NIL; foreach (lc, castNode(List, quals)) { Expr *qual = lfirst(lc); Relids relids = pull_varnos(ctx->root, (Node *) qual); int num_rels = bms_num_members(relids); /* stop processing if not for current rel */ if (num_rels != 1 || !bms_is_member(ctx->rel->relid, relids)) continue; /* * check for time_bucket comparisons * time_bucket(Const, time_colum) > Const */ Expr *transformed = ts_transform_time_bucket_comparison(qual); if (transformed != NULL) { /* * if we could transform the expression we add it to the list of * quals so it can be used as an index condition */ additional_quals = lappend(additional_quals, transformed); /* * Also use the transformed qual for chunk exclusion. */ qual = transformed; } ctx->restrictions = lappend(ctx->restrictions, make_simple_restrictinfo(ctx->root, qual)); } return (Node *) list_concat((List *) quals, additional_quals); } /* * collect JOIN information * * This function adds information to the CollectQualCtx * * propagate_conditions * * This list contains toplevel or INNER JOIN equality conditions. * This list is used for propagating quals to the other side of * a JOIN. */ static void collect_join_quals(Node *quals, CollectQualCtx *ctx, bool can_propagate) { ListCell *lc; foreach (lc, (List *) quals) { Expr *qual = lfirst(lc); Relids relids = pull_varnos(ctx->root, (Node *) qual); int num_rels = bms_num_members(relids); /* * collect quals to propagate to join relations */ if (num_rels == 1 && can_propagate && IsA(qual, OpExpr) && list_length(castNode(OpExpr, qual)->args) == 2) ctx->all_quals = lappend(ctx->all_quals, qual); if (!bms_is_member(ctx->rel->relid, relids)) continue; /* collect equality JOIN conditions for current rel */ if (num_rels == 2 && IsA(qual, OpExpr) && list_length(castNode(OpExpr, qual)->args) == 2) { OpExpr *op = castNode(OpExpr, qual); Expr *left = linitial(op->args); Expr *right = lsecond(op->args); if (IsA(left, Var) && IsA(right, Var)) { Var *ht_var = castNode(Var, (Index) castNode(Var, left)->varno == ctx->rel->relid ? left : right); TypeCacheEntry *tce = lookup_type_cache(ht_var->vartype, TYPECACHE_EQ_OPR); if (op->opno == tce->eq_opr) { if (can_propagate) ctx->propagate_conditions = lappend(ctx->propagate_conditions, op); } } continue; } } } static bool collect_quals_walker(Node *node, CollectQualCtx *ctx) { if (node == NULL) return false; if (IsA(node, FromExpr)) { FromExpr *f = castNode(FromExpr, node); f->quals = process_quals(f->quals, ctx, false); /* if this is a nested join we don't propagate join quals */ collect_join_quals(f->quals, ctx, ctx->join_level == 0); } else if (IsA(node, JoinExpr)) { JoinExpr *j = castNode(JoinExpr, node); j->quals = process_quals(j->quals, ctx, IS_OUTER_JOIN(j->jointype)); collect_join_quals(j->quals, ctx, ctx->join_level == 0 && !IS_OUTER_JOIN(j->jointype)); if (IS_OUTER_JOIN(j->jointype)) { ctx->join_level++; bool result = expression_tree_walker(node, collect_quals_walker, ctx); ctx->join_level--; return result; } } return expression_tree_walker(node, collect_quals_walker, ctx); } static int chunk_cmp_chunk_reloid(const void *c1, const void *c2) { Oid lhs = (*(Chunk **) c1)->table_id; Oid rhs = (*(Chunk **) c2)->table_id; if (lhs < rhs) return -1; if (lhs > rhs) return 1; return 0; } static Chunk ** find_children_chunks(HypertableRestrictInfo *hri, Hypertable *ht, bool include_osm, unsigned int *num_chunks) { /* * Unlike find_all_inheritors we do not include parent because if there * are restrictions the parent table cannot fulfill them and since we do * have a trigger blocking inserts on the parent table it cannot contain * any rows. */ Chunk **chunks = ts_hypertable_restrict_info_get_chunks(hri, ht, include_osm, num_chunks); /* * Sort the chunks by oid ascending to roughly match the order provided * by find_inheritance_children. This is mostly needed to avoid test * reference changes. */ qsort((void *) chunks, *num_chunks, sizeof(Chunk *), chunk_cmp_chunk_reloid); return chunks; } static bool should_order_append(PlannerInfo *root, RelOptInfo *rel, Hypertable *ht, int *order_attno, bool *reverse) { /* check if optimizations are enabled */ if (!ts_guc_enable_optimizations || !ts_guc_enable_ordered_append || !ts_guc_enable_chunk_append) return false; /* * only do this optimization for hypertables with 1 dimension and queries * with an ORDER BY clause */ if (root->parse->sortClause == NIL) return false; return ts_ordered_append_should_optimize(root, rel, ht, order_attno, reverse); } /* * Some time conditions are not directly applicable for the chunk exclusion, but * imply a simpler time comparison condition which can be used for hypertable * expansion. Return a list of any simplified restrictions we could build for * the restrictions in the given list. */ static List * get_simplified_restrictions(PlannerInfo *root, List *restrictions) { List *simplified_restrictions = NIL; ListCell *lc; foreach (lc, restrictions) { RestrictInfo *ri = castNode(RestrictInfo, lfirst(lc)); Expr *qual = ri->clause; if (IsA(qual, OpExpr) && list_length(castNode(OpExpr, qual)->args) == 2) { OpExpr *op = castNode(OpExpr, qual); Expr *left = linitial(op->args); Expr *right = lsecond(op->args); if ((IsA(left, Var) && is_timestamptz_op_interval(right)) || (IsA(right, Var) && is_timestamptz_op_interval(left))) { /* * Check for constraints with TIMESTAMPTZ OP INTERVAL calculations. */ Expr *transformed = (Expr *) constify_timestamptz_op_interval(root, op); if (transformed != (Expr *) op) { RestrictInfo *ri_copy = copyObject(ri); ri_copy->clause = transformed; simplified_restrictions = lappend(simplified_restrictions, ri_copy); } } else { /* * check for time_bucket comparisons * time_bucket(Const, time_colum) > Const */ Expr *transformed = ts_transform_time_bucket_comparison(qual); if (transformed != NULL) { /* * Also use the transformed qual for chunk exclusion. */ RestrictInfo *ri_copy = copyObject(ri); ri_copy->clause = transformed; simplified_restrictions = lappend(simplified_restrictions, ri_copy); } } } } return simplified_restrictions; } /** * Get chunks from restrict info. * * If appends are returned in order appends_ordered on rel->fdw_private is set to true. * To make verifying pathkeys easier in set_rel_pathlist the hypertable attno of the column * ordered by is stored in rel->fdw_private. * If the hypertable uses space partitioning the nested oids are stored in nested_oids * on rel->fdw_private when appends are ordered. */ static Chunk ** get_chunks(PlannerInfo *root, RelOptInfo *rel, Hypertable *ht, bool include_osm, unsigned int *num_chunks, HypertableRestrictInfo **hri_out) { bool reverse; int order_attno; HypertableRestrictInfo *hri = ts_hypertable_restrict_info_create(rel, ht); /* * This is where the magic happens: use our HypertableRestrictInfo * infrastructure to deduce the appropriate chunks using our range * exclusion. */ ts_hypertable_restrict_info_add(hri, root, rel->baserestrictinfo); List *simplified_restrictions = get_simplified_restrictions(root, rel->baserestrictinfo); ts_hypertable_restrict_info_add(hri, root, simplified_restrictions); /* Limit to hypertables without multiple dimensions for now */ if (hri->num_base_restrictions >= 1 && hri->num_dimensions == 1 && ht->space->num_dimensions == 1) { *hri_out = hri; } /* * If fdw_private has not been setup by caller there is no point checking * for ordered append as we can't pass the required metadata in fdw_private * to signal that this is safe to transform in ordered append plan in * set_rel_pathlist. */ if (rel->fdw_private != NULL && should_order_append(root, rel, ht, &order_attno, &reverse)) { TimescaleDBPrivate *priv = ts_get_private_reloptinfo(rel); List **nested_oids = NULL; priv->appends_ordered = true; priv->order_attno = order_attno; /* * for space partitioning we need extra information about the * time slices of the chunks */ if (ht->space->num_dimensions > 1) nested_oids = &priv->nested_oids; return ts_hypertable_restrict_info_get_chunks_ordered(hri, ht, include_osm, NULL, reverse, nested_oids, num_chunks); } return find_children_chunks(hri, ht, include_osm, num_chunks); } static bool timebucket_annotate_walker(Node *node, CollectQualCtx *ctx) { if (node == NULL) return false; if (IsA(node, FromExpr)) { FromExpr *f = castNode(FromExpr, node); f->quals = timebucket_annotate(f->quals, ctx); } else if (IsA(node, JoinExpr)) { JoinExpr *j = castNode(JoinExpr, node); j->quals = timebucket_annotate(j->quals, ctx); } return expression_tree_walker(node, timebucket_annotate_walker, ctx); } void ts_plan_expand_timebucket_annotate(PlannerInfo *root, RelOptInfo *rel) { CollectQualCtx ctx = { .root = root, .rel = rel, .restrictions = NIL, .all_quals = NIL, .propagate_conditions = NIL, }; /* Walk the tree and find restrictions or chunk exclusion functions */ timebucket_annotate_walker((Node *) root->parse->jointree, &ctx); if (ctx.propagate_conditions != NIL) propagate_join_quals(root, rel, &ctx); } /* * Build a list of baserestrictinfo with any Var OP Const constraints on the primary * dimension removed. */ static List * filter_baserestrictions(Hypertable *ht, List *base_restrictions) { AttrNumber dim_attno = ht->space->dimensions[0].column_attno; List *filtered_restrictions = NIL; ListCell *lc; foreach (lc, base_restrictions) { RestrictInfo *ri = castNode(RestrictInfo, lfirst(lc)); Expr *qual = ri->clause; if (IsA(qual, OpExpr)) { OpExpr *op = castNode(OpExpr, qual); Node *left = strip_implicit_coercions(linitial(op->args)); Node *right = strip_implicit_coercions(lsecond(op->args)); if ((IsA(left, Var) && IsA(right, Const) && castNode(Var, left)->varattno == dim_attno) || (IsA(right, Var) && IsA(left, Const) && castNode(Var, right)->varattno == dim_attno)) { /* only consider simple column to constant comparisons */ continue; } } filtered_restrictions = lappend(filtered_restrictions, ri); } return filtered_restrictions; } /* * Returns true if the given chunk is fully included by the restrictions * on the primary dimension. */ static bool chunk_fully_covered(HypertableRestrictInfo *hri, Chunk *chunk) { DimensionRestrictInfoOpen *dri = (DimensionRestrictInfoOpen *) hri->dimension_restriction[0]; Ensure(dri->base.dimension->type == DIMENSION_TYPE_OPEN, "primary dimension must be open"); Ensure(hri->num_base_restrictions > 0, "must have base restrictions"); if (IS_OSM_CHUNK(chunk) || (dri->lower_strategy == InvalidStrategy && dri->upper_strategy == InvalidStrategy) || (chunk->cube->slices[0]->fd.range_start == TS_TIME_NOBEGIN || chunk->cube->slices[0]->fd.range_end == TS_TIME_NOEND)) return false; /* * DimensionRetrictInfo strategy should only be one BTGreaterStrategyNumber * or BTGreaterEqualStrategyNumber on the lower boundary and * BTLessStrategyNumber or BTLessEqualStrategyNumber on the upper boundary. * * BTEqualStrategyNumber gets changed to BTGreaterEqualStrategyNumber * on lower boundary and BTLessEqualStrategyNumber on upper boundary. */ if (dri->lower_strategy != InvalidStrategy) { switch (dri->lower_strategy) { case BTGreaterStrategyNumber: if (chunk->cube->slices[0]->fd.range_start <= dri->lower_bound) return false; break; case BTGreaterEqualStrategyNumber: if (chunk->cube->slices[0]->fd.range_start < dri->lower_bound) return false; break; default: /* Should never happen */ elog(ERROR, "unexpected dimension restrictinfo strategy: %d", dri->upper_strategy); } } if (dri->upper_strategy != InvalidStrategy) { switch (dri->upper_strategy) { case BTLessStrategyNumber: if (chunk->cube->slices[0]->fd.range_end > dri->upper_bound) return false; break; case BTLessEqualStrategyNumber: if (chunk->cube->slices[0]->fd.range_end - 1 > dri->upper_bound) return false; break; default: /* Should never happen */ elog(ERROR, "unexpected dimension restrictinfo strategy: %d", dri->upper_strategy); } } return true; } /* Inspired by expand_inherited_rtentry but expands * a hypertable chunks into an append relation. */ void ts_plan_expand_hypertable_chunks(Hypertable *ht, PlannerInfo *root, RelOptInfo *rel, bool include_osm) { RangeTblEntry *rte = rt_fetch(rel->relid, root->parse->rtable); Oid parent_oid = rte->relid; Relation oldrelation; Query *parse = root->parse; Index rti = rel->relid; List *appinfos = NIL; CollectQualCtx ctx = { .root = root, .rel = rel, .restrictions = NIL, .all_quals = NIL, .propagate_conditions = NIL, .join_level = 0, }; Index first_chunk_index = 0; /* double check our permissions are valid */ Assert(rti != (Index) parse->resultRelation); /* Walk the tree and find restrictions */ collect_quals_walker((Node *) root->parse->jointree, &ctx); /* check join_level bookkeeping is balanced */ Assert(ctx.join_level == 0); if (ctx.propagate_conditions != NIL) propagate_join_quals(root, rel, &ctx); Chunk **chunks = NULL; unsigned int num_chunks = 0; HypertableRestrictInfo *hri = NULL; chunks = get_chunks(root, rel, ht, include_osm, &num_chunks, &hri); /* Can have zero chunks. */ Assert(num_chunks == 0 || chunks != NULL); /* nothing to do here if we have no chunks */ if (!num_chunks) return; /* * Handle PlanRowMark for FOR UPDATE/SHARE and FK constraint enforcement. * This replicates expand_inherited_rtentry() in inherit.c. */ PlanRowMark *oldrc = get_plan_rowmark(root->rowMarks, rti); bool old_isParent = false; int old_allMarkTypes = 0; if (oldrc) { old_isParent = oldrc->isParent; oldrc->isParent = true; old_allMarkTypes = oldrc->allMarkTypes; } for (unsigned int i = 0; i < num_chunks; i++) { /* * Add the information about chunks to the baserel info cache for * classify_relation(). */ ts_add_baserel_cache_entry_for_chunk(chunks[i]->table_id, ht); } oldrelation = table_open(parent_oid, NoLock); /* * the simple_*_array structures have already been set, we need to add the * children to them. */ expand_planner_arrays(root, num_chunks); for (unsigned int i = 0; i < num_chunks; i++) { Chunk *chunk = chunks[i]; Oid child_oid = chunk->table_id; Relation newrelation; RangeTblEntry *childrte; Index child_rtindex; AppendRelInfo *appinfo; LOCKMODE chunk_lock = rte->rellockmode; /* Open rel if needed */ Assert(child_oid != parent_oid); newrelation = table_open(child_oid, chunk_lock); /* chunks cannot be temp tables */ Assert(!RELATION_IS_OTHER_TEMP(newrelation)); /* * Build an RTE for the child, and attach to query's rangetable list. * We copy most fields of the parent's RTE, but replace relation OID * and relkind, and set inh = false. Also, set requiredPerms to zero * since all required permissions checks are done on the original RTE. * Likewise, set the child's securityQuals to empty, because we only * want to apply the parent's RLS conditions regardless of what RLS * properties individual children may have. (This is an intentional * choice to make inherited RLS work like regular permissions checks.) * The parent securityQuals will be propagated to children along with * other base restriction clauses, so we don't need to do it here. */ childrte = copyObject(rte); childrte->relid = child_oid; childrte->relkind = newrelation->rd_rel->relkind; childrte->inh = false; /* clear the magic bit */ childrte->ctename = NULL; #if PG16_LT childrte->requiredPerms = 0; #else /* Since PG16, the permission info is maintained separately. Unlink * the old perminfo from the RTE to disable permission checking. */ childrte->perminfoindex = 0; #endif childrte->securityQuals = NIL; parse->rtable = lappend(parse->rtable, childrte); child_rtindex = list_length(parse->rtable); if (first_chunk_index == 0) first_chunk_index = child_rtindex; root->simple_rte_array[child_rtindex] = childrte; Assert(root->simple_rel_array[child_rtindex] == NULL); appinfo = makeNode(AppendRelInfo); appinfo->parent_relid = rti; appinfo->child_relid = child_rtindex; appinfo->parent_reltype = oldrelation->rd_rel->reltype; appinfo->child_reltype = newrelation->rd_rel->reltype; ts_make_inh_translation_list(oldrelation, newrelation, child_rtindex, appinfo); appinfo->parent_reloid = parent_oid; appinfos = lappend(appinfos, appinfo); /* * Create child PlanRowMark if parent has one. This replicates * expand_single_inheritance_child() in inherit.c. */ if (oldrc) { PlanRowMark *childrc = makeNode(PlanRowMark); childrc->rti = child_rtindex; childrc->prti = oldrc->rti; childrc->rowmarkId = oldrc->rowmarkId; childrc->markType = select_rowmark_type(childrte, oldrc->strength); childrc->allMarkTypes = (1 << childrc->markType); childrc->strength = oldrc->strength; childrc->waitPolicy = oldrc->waitPolicy; childrc->isParent = false; /* chunks are never partitioned */ oldrc->allMarkTypes |= childrc->allMarkTypes; root->rowMarks = lappend(root->rowMarks, childrc); } /* Close child relations, but keep locks */ if (child_oid != parent_oid) table_close(newrelation, NoLock); } table_close(oldrelation, NoLock); /* * Add required junk columns for row marks. This replicates the logic * after the expansion loop in expand_inherited_rtentry() in inherit.c. */ if (oldrc) { int new_allMarkTypes = oldrc->allMarkTypes; Var *var; TargetEntry *tle; char resname[32]; List *newvars = NIL; /* * TID junk var: only needed if parent had only ROW_MARK_COPY but children * added non-COPY marks. This can only happen if the parent is a foreign * table with regular table children. Since hypertable parents are always * regular tables, preprocess_targetlist() (preptlist.c) already adds TID * for the parent before expansion, so this path is unreachable. */ Ensure(!(new_allMarkTypes & ~(1 << ROW_MARK_COPY) && !(old_allMarkTypes & ~(1 << ROW_MARK_COPY))), "unexpected: TID junk var needed for hypertable (parent should always be regular " "table)"); /* Add whole-row junk Var if needed, unless we had it already */ if ((new_allMarkTypes & (1 << ROW_MARK_COPY)) && !(old_allMarkTypes & (1 << ROW_MARK_COPY))) { var = makeWholeRowVar(planner_rt_fetch(oldrc->rti, root), oldrc->rti, 0, false); snprintf(resname, sizeof(resname), "wholerow%u", oldrc->rowmarkId); tle = makeTargetEntry((Expr *) var, list_length(root->processed_tlist) + 1, pstrdup(resname), true); root->processed_tlist = lappend(root->processed_tlist, tle); newvars = lappend(newvars, var); } /* Add tableoid junk Var, unless we had it already */ if (!old_isParent) { var = makeVar(oldrc->rti, TableOidAttributeNumber, OIDOID, -1, InvalidOid, 0); snprintf(resname, sizeof(resname), "tableoid%u", oldrc->rowmarkId); tle = makeTargetEntry((Expr *) var, list_length(root->processed_tlist) + 1, pstrdup(resname), true); root->processed_tlist = lappend(root->processed_tlist, tle); newvars = lappend(newvars, var); } /* * Add the newly added Vars to parent's reltarget. We needn't worry * about the children's reltargets, they'll be made later. */ add_vars_to_targetlist_compat(root, newvars, bms_make_singleton(0)); } ts_add_append_rel_infos(root, appinfos); /* PostgreSQL will not set up the child rels for use, due to the games * we're playing with inheritance, so we must do it ourselves. * build_simple_rel will look things up in the append_rel_array, so we can * only use it after that array has been set up. */ List *base_restrictions = rel->baserestrictinfo; List *filtered_restrictions = NIL; bool try_restriction_filtering = ts_guc_enable_qual_filtering && hri && ht->space->num_dimensions == 1; if (try_restriction_filtering) { filtered_restrictions = filter_baserestrictions(ht, base_restrictions); /* Dont try filtering if all restrictions remain after filtering */ if (list_length(base_restrictions) == list_length(filtered_restrictions)) try_restriction_filtering = false; } for (unsigned int i = 0; i < num_chunks; i++) { bool can_clear_restrictinfo = false; Index child_rtindex = first_chunk_index + i; Chunk *chunk = chunks[i]; if (try_restriction_filtering) { can_clear_restrictinfo = chunk_fully_covered(hri, chunk); } /* build_simple_rel will copy baserestrictinfo to the child rel and * do the necessary attribute mapping. If we can determine that the chunk * is fully covered by the primary dimension restriction we can remove * primary dimension restrictions from baserestrictinfo. */ if (can_clear_restrictinfo) rel->baserestrictinfo = filtered_restrictions; /* build_simple_rel will add the child to the relarray */ RelOptInfo *child_rel = build_simple_rel(root, child_rtindex, rel); if (can_clear_restrictinfo) rel->baserestrictinfo = base_restrictions; /* * Can't touch fdw_private for OSM chunks, it might be managed by the * OSM extension, or, in the tests, by postgres_fdw. */ if (!IS_OSM_CHUNK(chunk)) { Assert(chunk->table_id == root->simple_rte_array[child_rtindex]->relid); ts_get_private_reloptinfo(child_rel)->cached_chunk_struct = chunk; } } } static bool restrictinfo_has_qual(List *restrictions, OpExpr *qual) { ListCell *lc_ri; foreach (lc_ri, restrictions) { if (equal(castNode(RestrictInfo, lfirst(lc_ri))->clause, (Expr *) qual)) return true; } return false; } void propagate_join_quals(PlannerInfo *root, RelOptInfo *rel, CollectQualCtx *ctx) { ListCell *lc; if (!ts_guc_enable_qual_propagation) return; /* propagate join constraints */ foreach (lc, ctx->propagate_conditions) { ListCell *lc_qual; OpExpr *op = lfirst(lc); Var *rel_var, *other_var; /* * propagate_conditions only has OpExpr with 2 Var as arguments * this is enforced in process_quals */ Assert(IsA(op, OpExpr) && list_length(castNode(OpExpr, op)->args) == 2); Assert(IsA(linitial(op->args), Var) && IsA(lsecond(op->args), Var)); /* * check this join condition refers to current hypertable * our Var might be on either side of the expression */ if ((Index) linitial_node(Var, op->args)->varno == rel->relid) { rel_var = linitial_node(Var, op->args); other_var = lsecond_node(Var, op->args); } else if ((Index) lsecond_node(Var, op->args)->varno == rel->relid) { rel_var = lsecond_node(Var, op->args); other_var = linitial_node(Var, op->args); } else continue; foreach (lc_qual, ctx->all_quals) { OpExpr *qual = lfirst(lc_qual); Expr *left = linitial(qual->args); Expr *right = lsecond(qual->args); OpExpr *propagated; /* * check this is Var OP Expr / Expr OP Var * Var needs to reference the relid of the JOIN condition and * Expr must not contain volatile functions */ if (IsA(left, Var) && castNode(Var, left)->varno == other_var->varno && castNode(Var, left)->varattno == other_var->varattno && !IsA(right, Var) && !contain_volatile_functions((Node *) right)) { propagated = copyObject(qual); propagated->args = list_make2(rel_var, lsecond(propagated->args)); } else if (IsA(right, Var) && castNode(Var, right)->varno == other_var->varno && castNode(Var, right)->varattno == other_var->varattno && !IsA(left, Var) && !contain_volatile_functions((Node *) left)) { propagated = copyObject(qual); propagated->args = list_make2(linitial(propagated->args), rel_var); } else continue; /* * check if this is a new qual */ if (restrictinfo_has_qual(ctx->restrictions, propagated)) continue; Relids relids = pull_varnos(ctx->root, (Node *) propagated); RestrictInfo *restrictinfo; restrictinfo = make_restrictinfo_compat(root, (Expr *) propagated, true, false, false, false, false, ctx->root->qual_security_level, relids, NULL, NULL, NULL); ctx->restrictions = lappend(ctx->restrictions, restrictinfo); /* * since hypertable expansion happens later, the propagated * constraints will not be pushed down to the actual scans but stay * as join filter. So we add them either as join filter or to * baserestrictinfo depending on whether they reference only * the currently processed relation or multiple relations. */ if (bms_num_members(relids) == 1 && bms_is_member(rel->relid, relids)) { if (!restrictinfo_has_qual(rel->baserestrictinfo, propagated)) rel->baserestrictinfo = lappend(rel->baserestrictinfo, restrictinfo); } else { root->parse->jointree->quals = (Node *) lappend((List *) root->parse->jointree->quals, propagated); } } } } ================================================ FILE: src/planner/planner.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/tsmapi.h> #include <access/xact.h> #include <catalog/namespace.h> #include <commands/extension.h> #include <executor/nodeAgg.h> #include <miscadmin.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <nodes/parsenodes.h> #include <nodes/plannodes.h> #include <optimizer/appendinfo.h> #include <optimizer/clauses.h> #include <optimizer/optimizer.h> #include <optimizer/pathnode.h> #include <optimizer/paths.h> #include <optimizer/plancat.h> #include <optimizer/planner.h> #include <optimizer/restrictinfo.h> #include <optimizer/tlist.h> #include <parser/parse_param.h> #include <parser/parse_relation.h> #include <parser/parsetree.h> #include <utils/elog.h> #include <utils/fmgroids.h> #include <utils/guc.h> #include <utils/lsyscache.h> #include <utils/memutils.h> #include <utils/selfuncs.h> #include <utils/timestamp.h> #include <math.h> #include "annotations.h" #include "chunk.h" #include "cross_module_fn.h" #include "debug_assert.h" #include "dimension.h" #include "dimension_slice.h" #include "dimension_vector.h" #include "extension.h" #include "func_cache.h" #include "guc.h" #include "hypertable.h" #include "hypertable_cache.h" #include "import/allpaths.h" #include "license_guc.h" #include "nodes/chunk_append/chunk_append.h" #include "nodes/constraint_aware_append/constraint_aware_append.h" #include "nodes/modify_hypertable.h" #include "partitioning.h" #include "planner/planner.h" #include "sort_transform.h" #include "utils.h" #include "compat/compat.h" #include <common/hashfn.h> #ifdef USE_TELEMETRY #include "telemetry/functions.h" #endif /* define parameters necessary to generate the baserel info hash table interface */ typedef struct BaserelInfoEntry { Oid reloid; Hypertable *ht; uint32 status; /* hash status */ } BaserelInfoEntry; #define SH_PREFIX BaserelInfo #define SH_ELEMENT_TYPE BaserelInfoEntry #define SH_KEY_TYPE Oid #define SH_KEY reloid #define SH_EQUAL(tb, a, b) ((a) == (b)) #define SH_HASH_KEY(tb, key) murmurhash32(key) #define SH_SCOPE static #define SH_DECLARE #define SH_DEFINE // We don't need most of the generated functions and there is no way to not // generate them. #ifdef __GNUC__ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-function" #endif // Generate the baserel info hash table functions. #include "lib/simplehash.h" #ifdef __GNUC__ #pragma GCC diagnostic pop #endif void _planner_init(void); void _planner_fini(void); static planner_hook_type prev_planner_hook; static set_rel_pathlist_hook_type prev_set_rel_pathlist_hook; static get_relation_info_hook_type prev_get_relation_info_hook; static create_upper_paths_hook_type prev_create_upper_paths_hook; static void cagg_reorder_groupby_clause(RangeTblEntry *subq_rte, Index rtno, List *outer_sortcl, List *outer_tlist); /* * We mark range table entries (RTEs) in a query with TS_CTE_EXPAND if we'd like * to control table expansion ourselves. We exploit the ctename for this purpose * since it is not used for regular (base) relations. * * Note that we cannot use this mark as a general way to identify hypertable * RTEs. Child RTEs, for instance, will inherit this value from the parent RTE * during expansion. While we can prevent this happening in our custom table * expansion, we also have to account for the case when our custom expansion * is turned off with a GUC. */ static const char *TS_CTE_EXPAND = "ts_expand"; static const char *TS_FK_EXPAND = "ts_fk_expand"; /* * A simplehash hash table that records the chunks and their corresponding * hypertables, and also the plain baserels. We use it to tell whether a * relation is a hypertable chunk, inside the classify_relation function. * It is valid inside the scope of timescaledb_planner(). * That function can be called recursively, e.g. when we evaluate a SQL function, * and this cache is initialized only at the top-level call. */ static struct BaserelInfo_hash *ts_baserel_info = NULL; /* * Add information about a chunk to the baserel info cache. Used to cache the * chunk info at the plan time chunk exclusion. */ void ts_add_baserel_cache_entry_for_chunk(Oid chunk_reloid, Hypertable *hypertable) { Assert(hypertable != NULL); Assert(ts_baserel_info != NULL); bool found = false; BaserelInfoEntry *entry = BaserelInfo_insert(ts_baserel_info, chunk_reloid, &found); if (found) { /* Already cached. */ Assert(entry->ht != NULL); return; } Assert(ts_chunk_get_hypertable_id_by_reloid(chunk_reloid) == hypertable->fd.id); /* Fill the cache entry. */ entry->ht = hypertable; } static void rte_mark_for_expansion(RangeTblEntry *rte) { Assert(rte->rtekind == RTE_RELATION); Assert(rte->ctename == NULL); rte->ctename = (char *) TS_CTE_EXPAND; /* * Do not mark partitioned hypertables for inheritance, as Postgres * is supposed to expand them. */ if (rte->relkind != RELKIND_PARTITIONED_TABLE) rte->inh = false; } static void rte_mark_for_fk_expansion(RangeTblEntry *rte) { Assert(rte->rtekind == RTE_RELATION); Assert(rte->ctename == NULL); rte->ctename = (char *) TS_FK_EXPAND; /* * If this is for an FK lookup query inherit should be false * initially for hypertables. */ Assert(!rte->inh); } bool ts_rte_is_marked_for_expansion(const RangeTblEntry *rte) { if (NULL == rte->ctename) return false; if (rte->ctename == TS_CTE_EXPAND || rte->ctename == TS_FK_EXPAND) return true; return strcmp(rte->ctename, TS_CTE_EXPAND) == 0; } /* * Planner-global hypertable cache. * * Each invocation of the planner (and our hooks) should reference the same * cache object. Since we warm the cache when pre-processing the query (prior to * invoking the planner), we'd like to ensure that we use the same cache object * throughout the planning of that query so that we can trust that the cache * holds the objects it was warmed with. Since the planner can be invoked * recursively, we also need to stack and pop cache objects. */ static List *planner_hcaches = NIL; static Cache * planner_hcache_push(void) { Cache *hcache = ts_hypertable_cache_pin(); planner_hcaches = lcons(hcache, planner_hcaches); return hcache; } static void planner_hcache_pop(bool release) { Cache *hcache; Assert(list_length(planner_hcaches) > 0); hcache = linitial(planner_hcaches); planner_hcaches = list_delete_first(planner_hcaches); if (release) { ts_cache_release(&hcache); /* If we pop a stack and discover a new hypertable cache, the basrel * cache can contain invalid entries, so we reset it. */ if (planner_hcaches != NIL && hcache != linitial(planner_hcaches)) BaserelInfo_reset(ts_baserel_info); } } static bool planner_hcache_exists(void) { return planner_hcaches != NIL; } static Cache * planner_hcache_get(void) { if (planner_hcaches == NIL) return NULL; return (Cache *) linitial(planner_hcaches); } /* * Get the Hypertable corresponding to the given relid. * * This function gets a hypertable from a pre-warmed hypertable cache. If * noresolve is specified (true), then it will do a cache-only lookup (i.e., it * will not try to scan metadata for a new entry to put in the cache). This * allows fast lookups during planning to also determine if something is _not_ a * hypertable. */ Hypertable * ts_planner_get_hypertable(const Oid relid, const unsigned int flags) { Cache *cache = planner_hcache_get(); if (NULL == cache) return NULL; return ts_hypertable_cache_get_entry(cache, relid, flags); } bool ts_rte_is_hypertable(const RangeTblEntry *rte) { Hypertable *ht = ts_planner_get_hypertable(rte->relid, CACHE_FLAG_CHECK); return ht != NULL; } #define IS_UPDL_CMD(parse) \ ((parse)->commandType == CMD_UPDATE || (parse)->commandType == CMD_DELETE) typedef struct { Query *rootquery; Query *current_query; PlannerInfo *root; } PreprocessQueryContext; static void preprocess_fk_checks(Query *query, Cache *hcache, PreprocessQueryContext *context); void replace_now_mock_walker(PlannerInfo *root, Node *clause, Oid funcid) { /* whenever we encounter a FuncExpr with now(), replace it with the supplied funcid */ switch (nodeTag(clause)) { case T_FuncExpr: { if (is_valid_now_func(clause)) { FuncExpr *fe = castNode(FuncExpr, clause); fe->funcid = funcid; return; } break; } case T_OpExpr: { ListCell *lc; OpExpr *oe = castNode(OpExpr, clause); foreach (lc, oe->args) { replace_now_mock_walker(root, (Node *) lfirst(lc), funcid); } break; } case T_BoolExpr: { ListCell *lc; BoolExpr *be = castNode(BoolExpr, clause); foreach (lc, be->args) { replace_now_mock_walker(root, (Node *) lfirst(lc), funcid); } break; } default: return; } } /* * Preprocess the query tree, including, e.g., subqueries. * * Preprocessing includes: * * 1. Identifying all range table entries (RTEs) that reference * hypertables. This will also warm the hypertable cache for faster lookup * of both hypertables (cache hit) and non-hypertables (cache miss), * without having to scan the metadata in either case. * * 2. Turning off inheritance for hypertable RTEs that we expand ourselves. * * 3. Reordering of GROUP BY clauses for continuous aggregates. * * 4. Constifying now() expressions for primary time dimension. */ static bool preprocess_query(Node *node, PreprocessQueryContext *context) { if (node == NULL) return false; if (IsA(node, FromExpr) && ts_guc_enable_optimizations) { FromExpr *from = castNode(FromExpr, node); if (from->quals) { if (ts_guc_enable_now_constify) { from->quals = ts_constify_now(context->root, context->current_query->rtable, from->quals); #ifdef TS_DEBUG /* * only replace if GUC is also set. This is used for testing purposes only, * so no need to change the output for other tests in DEBUG builds */ if (ts_current_timestamp_mock != NULL && strlen(ts_current_timestamp_mock) != 0) { Oid funcid_mock; const char *funcname = "ts_now_mock()"; funcid_mock = DatumGetObjectId( DirectFunctionCall1(regprocedurein, CStringGetDatum(funcname))); replace_now_mock_walker(context->root, from->quals, funcid_mock); } #endif } /* * We only amend space constraints for UPDATE/DELETE and SELECT FOR UPDATE * as for normal SELECT we use our own hypertable expansion which can handle * constraints on hashed space dimensions without further help. */ if (context->current_query->commandType != CMD_SELECT || context->current_query->rowMarks != NIL) { from->quals = ts_add_space_constraints(context->root, context->current_query->rtable, from->quals); } } } else if (IsA(node, Query)) { Query *query = castNode(Query, node); Query *prev_query; Cache *hcache = planner_hcache_get(); ListCell *lc; Index rti = 1; bool ret; if (ts_guc_enable_foreign_key_propagation) { preprocess_fk_checks(query, hcache, context); } foreach (lc, query->rtable) { RangeTblEntry *rte = lfirst_node(RangeTblEntry, lc); Hypertable *ht; switch (rte->rtekind) { case RTE_SUBQUERY: if (ts_guc_enable_optimizations && ts_guc_enable_cagg_reorder_groupby && query->commandType == CMD_SELECT) { /* applicable to selects on continuous aggregates */ List *outer_tlist = query->targetList; List *outer_sortcl = query->sortClause; cagg_reorder_groupby_clause(rte, rti, outer_sortcl, outer_tlist); } break; case RTE_RELATION: /* This lookup will warm the cache with all hypertables in the query */ ht = ts_hypertable_cache_get_entry(hcache, rte->relid, CACHE_FLAG_MISSING_OK); if (ht) { /* Mark hypertable RTEs we'd like to expand ourselves */ if (ts_guc_enable_optimizations && ts_guc_enable_constraint_exclusion && !IS_UPDL_CMD(context->rootquery) && query->resultRelation == 0 && rte->inh) rte_mark_for_expansion(rte); if (TS_HYPERTABLE_HAS_COMPRESSION_TABLE(ht)) { int compr_htid = ht->fd.compressed_hypertable_id; /* Also warm the cache with the compressed * companion hypertable */ ts_hypertable_cache_get_entry_by_id(hcache, compr_htid); } } else { /* To properly keep track of SELECT FROM ONLY <chunk> we * have to mark the rte here because postgres will set * rte->inh to false (when it detects the chunk has no * children which is true for all our chunks) before it * reaches set_rel_pathlist hook. But chunks from queries * like SELECT .. FROM ONLY <chunk> has rte->inh set to * false and other chunks have rte->inh set to true. * We want to distinguish between the two cases here by * marking the chunk when rte->inh is true. */ Chunk *chunk = ts_chunk_get_by_relid_locked(rte->relid, NoLock, NULL, false); if (chunk && rte->inh) rte_mark_for_expansion(rte); } break; default: break; } rti++; } prev_query = context->current_query; context->current_query = query; ret = query_tree_walker(query, preprocess_query, context, 0); context->current_query = prev_query; return ret; } return expression_tree_walker(node, preprocess_query, context); } /* * Detect FOREIGN KEY lookup queries and mark the RTE for expansion. * Unfortunately postgres will create lookup queries for foreign keys * with `ONLY` preventing hypertable expansion. Only for declarative * partitioned tables the queries will be created without `ONLY`. * We try to detect these queries here and undo the `ONLY` flag for * these specific queries. * * The implementation of this on the postgres side can be found in * src/backend/utils/adt/ri_triggers.c */ static void preprocess_fk_checks(Query *query, Cache *hcache, PreprocessQueryContext *context) { /* * RI_FKey_cascade_del * * DELETE FROM [ONLY] <fktable> WHERE $1 = fkatt1 [AND ...] */ if (query->commandType == CMD_DELETE && list_length(query->rtable) == 1 && query->jointree->quals && IsA(query->jointree->quals, OpExpr) && (context->root->glob->boundParams || query_contains_extern_params(query))) { RangeTblEntry *rte = linitial_node(RangeTblEntry, query->rtable); if (!rte->inh && rte->rtekind == RTE_RELATION) { Hypertable *ht = ts_hypertable_cache_get_entry(hcache, rte->relid, CACHE_FLAG_MISSING_OK); if (ht) { rte->inh = true; } } } /* * RI_FKey_cascade_upd * * UPDATE [ONLY] <fktable> SET fkatt1 = $1 [, ...] * WHERE $n = fkatt1 [AND ...] */ if (query->commandType == CMD_UPDATE && list_length(query->rtable) == 1 && query->jointree->quals && IsA(query->jointree->quals, OpExpr) && (context->root->glob->boundParams || query_contains_extern_params(query))) { RangeTblEntry *rte = linitial_node(RangeTblEntry, query->rtable); if (!rte->inh && rte->rtekind == RTE_RELATION) { Hypertable *ht = ts_hypertable_cache_get_entry(hcache, rte->relid, CACHE_FLAG_MISSING_OK); if (ht) { rte->inh = true; } } } /* * RI_FKey_check * * The RI_FKey_check query string built is * SELECT 1 FROM [ONLY] <pktable> x WHERE pkatt1 = $1 [AND ...] * FOR KEY SHARE OF x */ if (query->commandType == CMD_SELECT && query->hasForUpdate && list_length(query->rtable) == 1 && (context->root->glob->boundParams || query_contains_extern_params(query))) { RangeTblEntry *rte = linitial_node(RangeTblEntry, query->rtable); if (!rte->inh && rte->rtekind == RTE_RELATION && rte->rellockmode == RowShareLock && list_length(query->jointree->fromlist) == 1 && query->jointree->quals && strcmp(rte->eref->aliasname, "x") == 0) { Hypertable *ht = ts_hypertable_cache_get_entry(hcache, rte->relid, CACHE_FLAG_MISSING_OK); if (ht) { rte_mark_for_fk_expansion(rte); if (TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(ht)) query->rowMarks = NIL; } } } /* * RI_Initial_Check query * * The RI_Initial_Check query string built is: * SELECT fk.keycols FROM [ONLY] relname fk * LEFT OUTER JOIN [ONLY] pkrelname pk * ON (pk.pkkeycol1=fk.keycol1 [AND ...]) * WHERE pk.pkkeycol1 IS NULL AND * For MATCH SIMPLE: * (fk.keycol1 IS NOT NULL [AND ...]) * For MATCH FULL: * (fk.keycol1 IS NOT NULL [OR ...]) */ if (query->commandType == CMD_SELECT && list_length(query->rtable) == 3) { RangeTblEntry *rte1 = linitial_node(RangeTblEntry, query->rtable); RangeTblEntry *rte2 = lsecond_node(RangeTblEntry, query->rtable); if (!rte1->inh && !rte2->inh && rte1->rtekind == RTE_RELATION && rte2->rtekind == RTE_RELATION && strcmp(rte1->eref->aliasname, "fk") == 0 && strcmp(rte2->eref->aliasname, "pk") == 0) { if (ts_hypertable_cache_get_entry(hcache, rte1->relid, CACHE_FLAG_MISSING_OK)) { rte_mark_for_fk_expansion(rte1); } if (ts_hypertable_cache_get_entry(hcache, rte2->relid, CACHE_FLAG_MISSING_OK)) { rte_mark_for_fk_expansion(rte2); } } } } static PlannedStmt * timescaledb_planner(Query *parse, const char *query_string, int cursor_opts, ParamListInfo bound_params) { PlannedStmt *stmt; ListCell *lc; /* * Volatile is needed because these are the local variables that are * modified between setjmp/longjmp calls. */ volatile bool reset_baserel_info = false; /* * If we are in an aborted transaction, reject all queries. * While this state will not happen during normal operation it * can happen when executing plpgsql procedures. */ if (IsAbortedTransactionBlockState()) ereport(ERROR, (errcode(ERRCODE_IN_FAILED_SQL_TRANSACTION), errmsg("current transaction is aborted, " "commands ignored until end of transaction block"))); planner_hcache_push(); if (ts_baserel_info == NULL) { /* * The calls to timescaledb_planner can be recursive (e.g. when * evaluating an immutable SQL function at planning time). We want to * create and destroy the per-query baserel info table only at the * top-level call, hence this flag. */ reset_baserel_info = true; /* * This is a per-query cache, so we create it in the current memory * context for the top-level call of this function, which hopefully * should exist for the duration of the query. Message or portal * memory contexts could also be suitable, but they don't exist for * SPI calls. */ ts_baserel_info = BaserelInfo_create(CurrentMemoryContext, /* nelements = */ 1, /* private_data = */ NULL); } PG_TRY(); { PreprocessQueryContext context = { 0 }; PlannerGlobal glob = { .boundParams = bound_params, }; PlannerInfo root = { .glob = &glob, }; context.root = &root; context.rootquery = parse; context.current_query = parse; if (ts_extension_is_loaded_and_not_upgrading()) { #ifdef USE_TELEMETRY ts_telemetry_function_info_gather(parse); #endif /* * Preprocess the hypertables in the query and warm up the caches. */ preprocess_query((Node *) parse, &context); if (ts_guc_enable_optimizations) ts_cm_functions->preprocess_query_tsl(parse, &cursor_opts); } if (prev_planner_hook != NULL) /* Call any earlier hooks */ stmt = (prev_planner_hook) (parse, query_string, cursor_opts, bound_params); else /* Call the standard planner */ stmt = standard_planner(parse, query_string, cursor_opts, bound_params); if (ts_extension_is_loaded_and_not_upgrading()) { /* * Our top-level HypertableInsert plan node that wraps ModifyTable needs * to have a final target list that is the same as the ModifyTable plan * node, and we only have access to its final target list after * set_plan_references() (setrefs.c) has run at the end of * standard_planner. Therefore, we fixup the final target list for * HypertableInsert here. */ ts_modify_hypertable_fixup_tlist(stmt->planTree); foreach (lc, stmt->subplans) { Plan *subplan = (Plan *) lfirst(lc); if (subplan) ts_modify_hypertable_fixup_tlist(subplan); } ts_cm_functions->tsl_postprocess_plan(stmt); } if (reset_baserel_info) { Assert(ts_baserel_info != NULL); BaserelInfo_destroy(ts_baserel_info); ts_baserel_info = NULL; } } PG_CATCH(); { if (reset_baserel_info) { Assert(ts_baserel_info != NULL); BaserelInfo_destroy(ts_baserel_info); ts_baserel_info = NULL; } /* Pop the cache, but do not release since caches are auto-released on * error */ planner_hcache_pop(false); PG_RE_THROW(); } PG_END_TRY(); planner_hcache_pop(true); return stmt; } static RangeTblEntry * get_parent_rte(const PlannerInfo *root, Index rti) { ListCell *lc; /* Fast path when arrays are setup */ if (root->append_rel_array != NULL && root->append_rel_array[rti] != NULL) { AppendRelInfo *appinfo = root->append_rel_array[rti]; return planner_rt_fetch(appinfo->parent_relid, root); } foreach (lc, root->append_rel_list) { AppendRelInfo *appinfo = lfirst_node(AppendRelInfo, lc); if (appinfo->child_relid == rti) return planner_rt_fetch(appinfo->parent_relid, root); } return NULL; } /* * Fetch cached baserel entry. If it does not exists, create an entry for this * relid. * If this relid corresponds to a chunk, cache additional chunk * related metadata: like chunk_status and pointer to hypertable entry. * It is okay to cache a pointer to the hypertable, since this cache is * confined to the lifetime of the query and not used across queries. * If the parent reolid is known, the caller can specify it to avoid the costly * lookup. Otherwise pass InvalidOid. */ static BaserelInfoEntry * get_or_add_baserel_from_cache(Oid chunk_reloid, Oid parent_reloid) { Hypertable *ht = NULL; /* First, check if this reloid is in cache. */ bool found = false; BaserelInfoEntry *entry = BaserelInfo_insert(ts_baserel_info, chunk_reloid, &found); if (found) { return entry; } if (OidIsValid(parent_reloid)) { ht = ts_planner_get_hypertable(parent_reloid, CACHE_FLAG_CHECK); #ifdef USE_ASSERT_CHECKING /* Sanity check on the caller-specified hypertable reloid. */ int32 parent_hypertable_id = ts_chunk_get_hypertable_id_by_reloid(chunk_reloid); if (parent_hypertable_id != INVALID_HYPERTABLE_ID) { Assert(ts_hypertable_id_to_relid(parent_hypertable_id, false) == parent_reloid); if (ht != NULL) { Assert(ht->fd.id == parent_hypertable_id); } } #endif } else { /* Hypertable reloid not specified by the caller, look it up by * an expensive metadata scan. */ int32 hypertable_id = ts_chunk_get_hypertable_id_by_reloid(chunk_reloid); if (hypertable_id != INVALID_HYPERTABLE_ID) { /* Hypertable reloid not specified by the caller, look it up. */ parent_reloid = ts_hypertable_id_to_relid(hypertable_id, /* return_invalid */ false); ht = ts_planner_get_hypertable(parent_reloid, CACHE_FLAG_NONE); Assert(ht != NULL); Assert(ht->fd.id == hypertable_id); } } /* Cache the result. */ entry->ht = ht; return entry; } /* * Classify a planned relation. * * This makes use of cache warming that happened during Query preprocessing in * the first planner hook. */ TsRelType ts_classify_relation(const PlannerInfo *root, const RelOptInfo *rel, Hypertable **ht) { Assert(ht != NULL); *ht = NULL; if (rel->reloptkind != RELOPT_BASEREL && rel->reloptkind != RELOPT_OTHER_MEMBER_REL) { return TS_REL_OTHER; } RangeTblEntry *rte = planner_rt_fetch(rel->relid, root); if (rte->relkind == RELKIND_FOREIGN_TABLE) { /* * OSM chunk or other foreign chunk. We can't even access the * fdw_private for it, because it's a foreign chunk managed by a * different extension. Try to ignore it as much as possible. */ return TS_REL_OTHER; } if (!OidIsValid(rte->relid)) { return TS_REL_OTHER; } if (rel->reloptkind == RELOPT_BASEREL) { /* * To correctly classify relations in subqueries we cannot call * ts_planner_get_hypertable with CACHE_FLAG_CHECK which includes * CACHE_FLAG_NOCREATE flag because the rel might not be in cache yet. */ *ht = ts_planner_get_hypertable(rte->relid, CACHE_FLAG_MISSING_OK); if (*ht != NULL) { return TS_REL_HYPERTABLE; } /* * This is either a chunk seen as a standalone table, a compressed chunk * table, or a non-chunk baserel. We need a costly chunk metadata scan * to distinguish between them, so we cache the result of this lookup to * avoid doing it repeatedly. */ BaserelInfoEntry *entry = get_or_add_baserel_from_cache(rte->relid, InvalidOid); *ht = entry->ht; if (*ht) { /* * Note that this works in a slightly weird way for compressed * chunks expanded from a normal hypertable, always saying that they * are standalone. In practice we filter them out by also checking * that the respective hypertable is not an internal compression * hypertable. */ return TS_REL_CHUNK_STANDALONE; } return TS_REL_OTHER; } Assert(rel->reloptkind == RELOPT_OTHER_MEMBER_REL); RangeTblEntry *parent_rte = get_parent_rte(root, rel->relid); /* * An entry of reloptkind RELOPT_OTHER_MEMBER_REL might still * be a hypertable or a chunk here if it was pulled up from a * subquery as happens with UNION ALL for example. So we have to * check for that to properly detect that pattern. */ if (parent_rte->rtekind == RTE_SUBQUERY) { *ht = ts_planner_get_hypertable(rte->relid, rte->inh ? CACHE_FLAG_MISSING_OK : CACHE_FLAG_CHECK); if (*ht) return TS_REL_HYPERTABLE; /* * This is either a chunk seen as a standalone table or a non-chunk baserel. * We need a costly chunk metadata scan to distinguish between them, so we * cache the result of this lookup to avoid doing it repeatedly. */ BaserelInfoEntry *entry = get_or_add_baserel_from_cache(rte->relid, InvalidOid); *ht = entry->ht; if (*ht) return TS_REL_CHUNK_STANDALONE; return TS_REL_OTHER; } if (parent_rte->relid == rte->relid) { /* * A PostgreSQL table expansion peculiarity -- "self child", the root * table that is expanded as a child of itself. This happens when our * expansion code is turned off. */ *ht = ts_planner_get_hypertable(rte->relid, CACHE_FLAG_CHECK); return *ht != NULL ? TS_REL_HYPERTABLE_CHILD : TS_REL_OTHER; } /* * Either an other baserel or a chunk seen when expanding the hypertable. * Use the baserel cache to determine what it is. */ BaserelInfoEntry *entry = get_or_add_baserel_from_cache(rte->relid, parent_rte->relid); *ht = entry->ht; if (*ht) { return TS_REL_CHUNK_CHILD; } return TS_REL_OTHER; } static inline bool should_chunk_append(Hypertable *ht, PlannerInfo *root, RelOptInfo *rel, Path *path, bool ordered, int order_attno) { if (path->param_info != NULL && ordered) { /* * Ordered ChunkAppend might create MergeAppend path for individual * chunks when we have space partitioning or partial chunks. MergeAppend * paths cannot be parameterized. Refuse to use parameterized ordered * ChunkAppend altogether, because the more precise conditions are * difficult to check. */ return false; } if ( /* * We only support chunk exclusion on UPDATE/DELETE when no JOIN is involved on PG14+. */ ((root->parse->commandType == CMD_DELETE || root->parse->commandType == CMD_UPDATE) && bms_num_members(root->all_baserels) > 1) || !ts_guc_enable_chunk_append) return false; switch (nodeTag(path)) { case T_AppendPath: /* * If there are clauses that have mutable functions, or clauses that reference * Params this Path might benefit from startup or runtime exclusion */ { AppendPath *append = castNode(AppendPath, path); ListCell *lc; /* Don't create ChunkAppend with no children */ if (list_length(append->subpaths) == 0) return false; foreach (lc, rel->baserestrictinfo) { RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); if (contain_mutable_functions((Node *) rinfo->clause) || ts_contains_external_param((Node *) rinfo->clause) || ts_contains_join_param((Node *) rinfo->clause)) return true; } return false; break; } case T_MergeAppendPath: /* * Can we do ordered append */ { MergeAppendPath *merge = castNode(MergeAppendPath, path); PathKey *pk; ListCell *lc; if (!ordered || path->pathkeys == NIL || list_length(merge->subpaths) == 0) return false; /* * Do not try to do ordered append if the OSM chunk range is noncontiguous */ if (ht && ts_chunk_get_osm_chunk_id(ht->fd.id) != INVALID_CHUNK_ID) { if (ts_flags_are_set_32(ht->fd.status, HYPERTABLE_STATUS_OSM_CHUNK_NONCONTIGUOUS)) return false; } /* * If we only have 1 child node there is no need for the * ordered append optimization. We might still benefit from * a ChunkAppend node here due to runtime chunk exclusion * when we have non-immutable constraints. */ if (list_length(merge->subpaths) == 1) { foreach (lc, rel->baserestrictinfo) { RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc); if (contain_mutable_functions((Node *) rinfo->clause) || ts_contains_external_param((Node *) rinfo->clause) || ts_contains_join_param((Node *) rinfo->clause)) return true; } return false; } pk = linitial_node(PathKey, path->pathkeys); /* * Check PathKey is compatible with Ordered Append ordering * we created when expanding hypertable. * Even though ordered is true on the RelOptInfo we have to * double check that current Path fulfills requirements for * Ordered Append transformation because the RelOptInfo may * be used for multiple Paths. */ Expr *em_expr = ts_find_em_expr_for_rel(pk->pk_eclass, rel); /* * If this is a join the ordering information might not be * for the current rel and have no EquivalenceMember. */ if (!em_expr) return false; if (IsA(em_expr, Var) && castNode(Var, em_expr)->varattno == order_attno) return true; if (IsA(em_expr, FuncExpr) && list_length(path->pathkeys) == 1) { FuncExpr *func = castNode(FuncExpr, em_expr); FuncInfo *info = ts_func_cache_get_bucketing_func(func->funcid); Expr *transformed; if (info && info->sort_transform) { transformed = info->sort_transform(func); if (IsA(transformed, Var) && castNode(Var, transformed)->varattno == order_attno) return true; } } return false; break; } default: return false; } } static inline bool should_constraint_aware_append(PlannerInfo *root, Hypertable *ht, Path *path) { /* Constraint-aware append currently expects children that scans a real * "relation" (e.g., not an "upper" relation). */ if (root->parse->commandType != CMD_SELECT) return false; return ts_constraint_aware_append_possible(path); } static bool rte_should_expand(const RangeTblEntry *rte) { bool is_hypertable = ts_rte_is_hypertable(rte); return is_hypertable && !rte->inh && ts_rte_is_marked_for_expansion(rte) && rte->relkind != RELKIND_PARTITIONED_TABLE; } static void expand_hypertables(PlannerInfo *root, RelOptInfo *rel, Index rti, RangeTblEntry *rte) { bool set_pathlist_for_current_rel = false; double total_pages; bool reenabled_inheritance = false; for (int i = 1; i < root->simple_rel_array_size; i++) { RangeTblEntry *in_rte = root->simple_rte_array[i]; #if PG18_GE /* RTE could be removed due to self-join * elimination optimization. * * https://github.com/postgres/postgres/commit/5f6f95 */ if (!in_rte) continue; #endif if (rte_should_expand(in_rte) && root->simple_rel_array[i]) { RelOptInfo *in_rel = root->simple_rel_array[i]; Hypertable *ht = ts_planner_get_hypertable(in_rte->relid, CACHE_FLAG_NOCREATE); Assert(ht != NULL && in_rel != NULL); ts_plan_expand_hypertable_chunks(ht, root, in_rel, in_rte->ctename != TS_FK_EXPAND); in_rte->inh = true; reenabled_inheritance = true; /* Redo set_rel_consider_parallel, as results of the call may no longer be valid * here (due to adding more tables to the set of tables under consideration here). * This is especially true if dealing with foreign data wrappers. */ /* * An entry of reloptkind RELOPT_OTHER_MEMBER_REL might still * be a hypertable here if it was pulled up from a subquery * as happens with UNION ALL for example. */ if (in_rel->reloptkind == RELOPT_BASEREL || in_rel->reloptkind == RELOPT_OTHER_MEMBER_REL) { Assert(in_rte->relkind == RELKIND_RELATION); ts_set_rel_size(root, in_rel, i, in_rte); } /* if we're activating inheritance during a hypertable's pathlist * creation then we're past the point at which postgres will add * paths for the children, and we have to do it ourselves. We delay * the actual setting of the pathlists until after this loop, * because set_append_rel_pathlist will eventually call this hook again. */ if (in_rte == rte) { Assert(rti == (Index) i); set_pathlist_for_current_rel = true; } } } if (!reenabled_inheritance) return; total_pages = 0; for (int i = 1; i < root->simple_rel_array_size; i++) { RelOptInfo *brel = root->simple_rel_array[i]; if (brel == NULL) continue; Assert(brel->relid == (Index) i); /* sanity check on array */ if (IS_DUMMY_REL(brel)) continue; if (IS_SIMPLE_REL(brel)) total_pages += (double) brel->pages; } root->total_table_pages = total_pages; if (set_pathlist_for_current_rel) { rel->pathlist = NIL; rel->partial_pathlist = NIL; ts_set_append_rel_pathlist(root, rel, rti, rte); } } static void apply_optimizations(PlannerInfo *root, TsRelType reltype, RelOptInfo *rel, RangeTblEntry *rte, Hypertable *ht) { if (!ts_guc_enable_optimizations) return; switch (reltype) { case TS_REL_HYPERTABLE_CHILD: /* empty table so nothing to optimize */ break; case TS_REL_CHUNK_STANDALONE: case TS_REL_CHUNK_CHILD: { /* * Since the sort optimization adds new paths to the rel it has * to happen before any optimizations that replace pathlist. */ List *transformed_query_pathkeys = ts_sort_transform_get_pathkeys(root, rel, rte, ht); if (transformed_query_pathkeys != NIL) { List *orig_query_pathkeys = root->query_pathkeys; root->query_pathkeys = transformed_query_pathkeys; /* Create index paths with transformed pathkeys */ create_index_paths(root, rel); /* * Call the TSL hooks with the transformed pathkeys as well, so * that the decompression paths also use this optimization. */ if (ts_cm_functions->set_rel_pathlist_query != NULL) ts_cm_functions->set_rel_pathlist_query(root, rel, rel->relid, rte, ht); root->query_pathkeys = orig_query_pathkeys; /* * change returned paths to use original pathkeys. have to go through * all paths since create_index_paths might have modified existing * pathkey. Always safe to do transform since ordering of * transformed_query_pathkey implements ordering of * orig_query_pathkeys. */ ts_sort_transform_replace_pathkeys(rel->pathlist, transformed_query_pathkeys, orig_query_pathkeys); } else { if (ts_cm_functions->set_rel_pathlist_query != NULL) ts_cm_functions->set_rel_pathlist_query(root, rel, rel->relid, rte, ht); } break; } default: break; } if (reltype == TS_REL_HYPERTABLE && (root->parse->commandType == CMD_SELECT || root->parse->commandType == CMD_DELETE || root->parse->commandType == CMD_UPDATE)) { TimescaleDBPrivate *private = ts_get_private_reloptinfo(rel); bool ordered = private->appends_ordered; int order_attno = private->order_attno; List *nested_oids = private->nested_oids; ListCell *lc; Assert(ht != NULL); foreach (lc, rel->pathlist) { Path **pathptr = (Path **) &lfirst(lc); switch (nodeTag(*pathptr)) { case T_AppendPath: case T_MergeAppendPath: if (should_chunk_append(ht, root, rel, *pathptr, ordered, order_attno)) *pathptr = ts_chunk_append_path_create(root, rel, ht, *pathptr, false, ordered, nested_oids); else if (should_constraint_aware_append(root, ht, *pathptr)) *pathptr = ts_constraint_aware_append_path_create(root, *pathptr); break; default: break; } } foreach (lc, rel->partial_pathlist) { Path **pathptr = (Path **) &lfirst(lc); switch (nodeTag(*pathptr)) { case T_AppendPath: case T_MergeAppendPath: if (should_chunk_append(ht, root, rel, *pathptr, false, 0)) *pathptr = ts_chunk_append_path_create(root, rel, ht, *pathptr, true, false, NIL); else if (should_constraint_aware_append(root, ht, *pathptr)) *pathptr = ts_constraint_aware_append_path_create(root, *pathptr); break; default: break; } } } } static bool valid_hook_call(void) { return ts_extension_is_loaded_and_not_upgrading() && planner_hcache_exists(); } static bool dml_involves_hypertable(PlannerInfo *root, Hypertable *ht, Index rti) { Index result_rti = root->parse->resultRelation; RangeTblEntry *result_rte = planner_rt_fetch(result_rti, root); return result_rti == rti || ht->main_table_relid == result_rte->relid; } static void timescaledb_set_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, Index rti, RangeTblEntry *rte) { TsRelType reltype; Hypertable *ht; /* * Quick exit if this is a relation we're not interested in. * * If the rtekind is a named tuple store, it is a named tuple store *for* * the relation rte->relid (e.g., a transition table for a trigger), but * not the relation itself. */ if (!valid_hook_call() || rte->rtekind == RTE_NAMEDTUPLESTORE || !OidIsValid(rte->relid) || IS_DUMMY_REL(rel)) { if (prev_set_rel_pathlist_hook != NULL) (*prev_set_rel_pathlist_hook)(root, rel, rti, rte); return; } reltype = ts_classify_relation(root, rel, &ht); /* Check for unexpanded hypertable */ if (!rte->inh && ts_rte_is_marked_for_expansion(rte)) expand_hypertables(root, rel, rti, rte); if (ts_guc_enable_optimizations) ts_planner_constraint_cleanup(root, rel); /* Call other extensions. Do it after table expansion. */ if (prev_set_rel_pathlist_hook != NULL) (*prev_set_rel_pathlist_hook)(root, rel, rti, rte); switch (reltype) { case TS_REL_HYPERTABLE_CHILD: if (ts_guc_enable_optimizations && IS_UPDL_CMD(root->parse)) ts_planner_constraint_cleanup(root, rel); break; case TS_REL_CHUNK_STANDALONE: case TS_REL_CHUNK_CHILD: /* Check for UPDATE/DELETE/MERGE (DML) on compressed chunks */ if (IS_UPDL_CMD(root->parse) && dml_involves_hypertable(root, ht, rti)) { if (ts_cm_functions->set_rel_pathlist_dml != NULL) ts_cm_functions->set_rel_pathlist_dml(root, rel, rti, rte, ht); break; } /* * For MERGE command if there is an UPDATE or DELETE action, then * do not allow this to succeed on compressed chunks */ if (root->parse->commandType == CMD_MERGE && dml_involves_hypertable(root, ht, rti)) { ListCell *ml; foreach (ml, root->parse->mergeActionList) { MergeAction *action = (MergeAction *) lfirst(ml); if (action->commandType == CMD_UPDATE || action->commandType == CMD_DELETE) { if (ts_cm_functions->set_rel_pathlist_dml != NULL) ts_cm_functions->set_rel_pathlist_dml(root, rel, rti, rte, ht); } } break; } TS_FALLTHROUGH; default: /* * Set the indexlist for a hypertable parent to NIL since we * should not try to do any index scans on hypertable parents, * similar to how it works for partitioned tables. * * This can happen when building a merge join path and computing * cost for it. See get_actual_variable_range(). * * This has to be after the hypertable is expanded, since the * indexlist is used during hypertable expansion. */ if (reltype == TS_REL_HYPERTABLE) rel->indexlist = NIL; apply_optimizations(root, reltype, rel, rte, ht); break; } } /* This hook is meant to editorialize about the information the planner gets * about a relation. We use it to attach our own metadata to hypertable and * chunk relations that we need during planning. We also expand hypertables * here. */ static void timescaledb_get_relation_info_hook(PlannerInfo *root, Oid relation_objectid, bool inhparent, RelOptInfo *rel) { if (prev_get_relation_info_hook != NULL) prev_get_relation_info_hook(root, relation_objectid, inhparent, rel); if (!valid_hook_call()) return; RangeTblEntry *rte = planner_rt_fetch(rel->relid, root); Query *query = root->parse; Hypertable *ht; const TsRelType type = ts_classify_relation(root, rel, &ht); AclMode requiredPerms = 0; #if PG16_LT requiredPerms = rte->requiredPerms; #else if (rte->perminfoindex > 0) { RTEPermissionInfo *perminfo = getRTEPermissionInfo(query->rteperminfos, rte); requiredPerms = perminfo->requiredPerms; } #endif switch (type) { case TS_REL_HYPERTABLE: { /* Mark hypertable RTEs we'd like to expand ourselves. * Hypertables inside inlineable functions don't get marked during the query * preprocessing step. Therefore we do an extra try here. However, we need to * be careful for UPDATE/DELETE as Postgres (in at least version 12) plans them * in a complicated way (see planner.c:inheritance_planner). First, it runs the * UPDATE/DELETE through the planner as a simulated SELECT. It uses the results * of this fake planning to adapt its own UPDATE/DELETE plan. Then it's planned * a second time as a real UPDATE/DELETE, but with requiredPerms set to 0, as it * assumes permission checking has been done already during the first planner call. * We don't want to touch the UPDATE/DELETEs, so we need to check all the regular * conditions here that are checked during preprocess_query, as well as the * condition that requiredPerms is not requiring UPDATE/DELETE on this rel. */ if (ts_guc_enable_optimizations && ts_guc_enable_constraint_exclusion && inhparent && rte->ctename == NULL && !IS_UPDL_CMD(query) && query->resultRelation == 0 && (requiredPerms & (ACL_UPDATE | ACL_DELETE)) == 0) { rte_mark_for_expansion(rte); } ts_create_private_reloptinfo(rel); ts_plan_expand_timebucket_annotate(root, rel); break; } case TS_REL_CHUNK_STANDALONE: case TS_REL_CHUNK_CHILD: ts_create_private_reloptinfo(rel); /* * We don't want to plan index scans on empty uncompressed tables of * fully compressed chunks. It takes a lot of time, and these tables * are empty anyway. Just reset the indexlist in this case. For * uncompressed or partially compressed chunks, the uncompressed * tables are not empty, so we plan the index scans as usual. * * Normally the index list is reset in ts_set_append_rel_pathlist(), * based on the Chunk struct cached by our hypertable expansion, but * in cases when these functions don't run, we have to do it here. */ const bool use_columnar_scan = ts_guc_enable_columnarscan && TS_HYPERTABLE_HAS_COMPRESSION_TABLE(ht); const bool is_standalone_chunk = (type == TS_REL_CHUNK_STANDALONE) && !TS_HYPERTABLE_IS_INTERNAL_COMPRESSION_TABLE(ht); const bool is_child_chunk_in_update = (type == TS_REL_CHUNK_CHILD) && IS_UPDL_CMD(query); if (use_columnar_scan && (is_standalone_chunk || is_child_chunk_in_update)) { const Chunk *chunk = ts_planner_chunk_fetch(root, rel); if (!ts_chunk_is_partial(chunk) && ts_chunk_is_compressed(chunk)) { rel->indexlist = NIL; } } break; case TS_REL_HYPERTABLE_CHILD: /* When postgres expands an inheritance tree it also adds the * parent hypertable as child relation. Since for a hypertable the * parent will never have any data we can mark this relation as * dummy relation so it gets ignored in later steps. This is only * relevant for code paths that use the postgres inheritance code * as we don't include the hypertable as child when expanding the * hypertable ourself. */ if (IS_UPDL_CMD(root->parse)) mark_dummy_rel(rel); break; case TS_REL_OTHER: break; } } static bool join_involves_hypertable(const PlannerInfo *root, const RelOptInfo *rel) { int relid = -1; while ((relid = bms_next_member(rel->relids, relid)) >= 0) { const RangeTblEntry *rte = planner_rt_fetch(relid, root); if (rte != NULL) /* This might give a false positive for chunks in case of PostgreSQL * expansion since the ctename is copied from the parent hypertable * to the chunk */ return ts_rte_is_marked_for_expansion(rte); } return false; } static bool involves_hypertable(PlannerInfo *root, RelOptInfo *rel) { if (rel->reloptkind == RELOPT_JOINREL) return join_involves_hypertable(root, rel); Hypertable *ht; return ts_classify_relation(root, rel, &ht) == TS_REL_HYPERTABLE; } /* * Replace ModifyTablePath paths on hypertables. * * From the ModifyTable description: "Each ModifyTable node contains * a list of one or more subplans, much like an Append node. There * is one subplan per result relation." * * The subplans produce the tuples for INSERT, while the result relation is the * table we'd like to insert into. * * Conceptually, the plan modification looks like this: * * Original plan: * * ^ * | * [ ModifyTable ] -> resultRelation * ^ * | Tuple * | * [ subplan ] * * * Modified plan: * * [ ModifyHypertable ] * ^ * | * [ ModifyTable ] -> resultRelation * ^ * | Tuple * | * [ subplan ] * */ static List * replace_modify_hypertable_paths(PlannerInfo *root, List *pathlist, RelOptInfo *input_rel) { List *new_pathlist = NIL; ListCell *lc; foreach (lc, pathlist) { Path *path = lfirst(lc); if (IsA(path, ModifyTablePath)) { ModifyTablePath *mt = castNode(ModifyTablePath, path); RangeTblEntry *rte = planner_rt_fetch(mt->nominalRelation, root); Hypertable *ht = ts_planner_get_hypertable(rte->relid, CACHE_FLAG_CHECK); /* Direct INSERT into internal compressed hypertable is not supported. * Compressed chunks have no dimensions so we could not do tuple routing. * Additionally internal compressed hypertable has no columns so you * couldn't even insert any actual data. */ if (ht && ht->fd.compression_state == HypertableInternalCompressionTable) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("direct insert into internal compressed hypertable is not " "supported"))); /* Check for DML on chunk directly */ if (!ht) { Chunk *chunk = ts_chunk_get_by_relid(rte->relid, false); if (!chunk) { /* Not a hypertable or chunk, continue */ new_pathlist = lappend(new_pathlist, path); continue; } ht = ts_hypertable_get_by_id(chunk->fd.hypertable_id); if (ht->fd.compression_state == HypertableInternalCompressionTable) { /* * For operations on internal compressed chunks we block modifications * if the chunk belongs to a frozen chunk. * Direct modifications of uncompressed chunks is intercepted by chunk * tuple routing. * In all other cases of direct modification of chunks we dont interfere * and do not add a ModifyHypertable node. */ Chunk *uncompressed = ts_chunk_get_compressed_chunk_parent(chunk); if (ts_chunk_is_frozen(uncompressed)) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("cannot modify compressed chunk belonging to a frozen " "chunk"))); new_pathlist = lappend(new_pathlist, path); continue; } } switch (mt->operation) { case CMD_INSERT: case CMD_UPDATE: case CMD_DELETE: { path = ts_modify_hypertable_path_create(root, mt, input_rel); break; } case CMD_MERGE: { List *firstMergeActionList = linitial(mt->mergeActionLists); ListCell *l; /* * Iterate over merge action to check if there is an INSERT sql. * If so, then add ModifyHypertable node. */ foreach (l, firstMergeActionList) { MergeAction *action = (MergeAction *) lfirst(l); if (action->commandType == CMD_INSERT) { path = ts_modify_hypertable_path_create(root, mt, input_rel); break; } } break; } default: break; } } new_pathlist = lappend(new_pathlist, path); } return new_pathlist; } static void timescaledb_create_upper_paths_hook(PlannerInfo *root, UpperRelationKind stage, RelOptInfo *input_rel, RelOptInfo *output_rel, void *extra) { Query *parse = root->parse; TsRelType reltype = TS_REL_OTHER; Hypertable *ht = NULL; if (prev_create_upper_paths_hook != NULL) prev_create_upper_paths_hook(root, stage, input_rel, output_rel, extra); if (!ts_extension_is_loaded_and_not_upgrading()) return; if (input_rel != NULL) reltype = ts_classify_relation(root, input_rel, &ht); if (output_rel != NULL) { /* Modify for INSERTs on a hypertable */ if (output_rel->pathlist != NIL) output_rel->pathlist = replace_modify_hypertable_paths(root, output_rel->pathlist, input_rel); } if (stage == UPPERREL_GROUP_AGG && output_rel != NULL && ts_guc_enable_optimizations && input_rel != NULL && !IS_DUMMY_REL(input_rel) && involves_hypertable(root, input_rel)) { if (parse->hasAggs) ts_preprocess_first_last_aggregates(root, root->processed_tlist); } if (ts_cm_functions->create_upper_paths_hook != NULL) ts_cm_functions ->create_upper_paths_hook(root, stage, input_rel, output_rel, reltype, ht, extra); } static bool contains_join_param_walker(Node *node, void *context) { if (node == NULL) { return false; } if (IsA(node, Param) && castNode(Param, node)->paramkind == PARAM_EXEC) return true; return expression_tree_walker(node, contains_join_param_walker, context); } bool ts_contains_join_param(Node *node) { return contains_join_param_walker(node, NULL); } static bool contains_external_param_walker(Node *node, void *context) { if (node == NULL) { return false; } if (IsA(node, Param) && castNode(Param, node)->paramkind == PARAM_EXTERN) return true; return expression_tree_walker(node, contains_external_param_walker, context); } bool ts_contains_external_param(Node *node) { return contains_external_param_walker(node, NULL); } static List * fill_missing_groupclause(List *new_groupclause, List *orig_groupclause) { if (new_groupclause != NIL) { ListCell *gl; foreach (gl, orig_groupclause) { SortGroupClause *gc = lfirst_node(SortGroupClause, gl); if (list_member_ptr(new_groupclause, gc)) continue; /* already in list */ new_groupclause = lappend(new_groupclause, gc); } } return new_groupclause; } static bool check_cagg_view_rte(RangeTblEntry *rte) { ContinuousAgg *cagg = NULL; ListCell *rtlc; bool found = false; Query *viewq = rte->subquery; Assert(rte->rtekind == RTE_SUBQUERY); if (list_length(viewq->rtable) != 3) /* a view has 3 entries */ { return false; } /* should cache this information for cont. aggregates */ foreach (rtlc, viewq->rtable) { RangeTblEntry *rte = lfirst_node(RangeTblEntry, rtlc); if (!OidIsValid(rte->relid)) break; cagg = ts_continuous_agg_find_by_relid(rte->relid); if (cagg != NULL) found = true; } return found; } /* Note that it modifies the passed in Query * select * from (select a, b, max(c), min(d) from ... group by a, b) order by b; * is transformed as * SELECT * from (select a, b, max(c), min(d) from .. * group by B desc, A <------ note the change in order here * ) * order by b desc; * we transform only if order by is a subset of group-by * transformation is applicable only to continuous aggregates * Parameters: * subq_rte - rte for subquery (inner query that will be modified) * outer_sortcl -- outer query's sort clause * outer_tlist - outer query's target list */ static void cagg_reorder_groupby_clause(RangeTblEntry *subq_rte, Index rtno, List *outer_sortcl, List *outer_tlist) { bool not_found = true; Query *subq; ListCell *lc; Assert(subq_rte->rtekind == RTE_SUBQUERY); subq = subq_rte->subquery; if (outer_sortcl && subq->groupClause && subq->sortClause == NIL && check_cagg_view_rte(subq_rte)) { List *new_groupclause = NIL; /* we are going to modify this. so make a copy and use it if we replace */ List *subq_groupclause_copy = copyObject(subq->groupClause); foreach (lc, outer_sortcl) { SortGroupClause *outer_sc = (SortGroupClause *) lfirst(lc); TargetEntry *outer_tle = get_sortgroupclause_tle(outer_sc, outer_tlist); not_found = true; if (IsA(outer_tle->expr, Var) && ((Index) ((Var *) outer_tle->expr)->varno == rtno)) { int outer_attno = ((Var *) outer_tle->expr)->varattno; TargetEntry *subq_tle = list_nth(subq->targetList, outer_attno - 1); if (subq_tle->ressortgroupref > 0) { /* get group clause corresponding to this */ SortGroupClause *subq_gclause = get_sortgroupref_clause(subq_tle->ressortgroupref, subq_groupclause_copy); subq_gclause->sortop = outer_sc->sortop; subq_gclause->nulls_first = outer_sc->nulls_first; #if PG18_GE /* Track sort direction in SortGroupClause * https://github.com/postgres/postgres/commit/0d2aa4d4 */ subq_gclause->reverse_sort = outer_sc->reverse_sort; #endif Assert(subq_gclause->eqop == outer_sc->eqop); new_groupclause = lappend(new_groupclause, subq_gclause); not_found = false; } } if (not_found) break; } /* all order by found in group by clause */ if (new_groupclause != NIL && not_found == false) { /* use new groupby clause for this subquery/view */ subq->groupClause = fill_missing_groupclause(new_groupclause, subq_groupclause_copy); } } } void _planner_init(void) { prev_planner_hook = planner_hook; planner_hook = timescaledb_planner; prev_set_rel_pathlist_hook = set_rel_pathlist_hook; set_rel_pathlist_hook = timescaledb_set_rel_pathlist; prev_get_relation_info_hook = get_relation_info_hook; get_relation_info_hook = timescaledb_get_relation_info_hook; prev_create_upper_paths_hook = create_upper_paths_hook; create_upper_paths_hook = timescaledb_create_upper_paths_hook; } void _planner_fini(void) { planner_hook = prev_planner_hook; set_rel_pathlist_hook = prev_set_rel_pathlist_hook; get_relation_info_hook = prev_get_relation_info_hook; create_upper_paths_hook = prev_create_upper_paths_hook; } ================================================ FILE: src/planner/planner.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/parsenodes.h> #include <nodes/pathnodes.h> #include <nodes/pg_list.h> #include <parser/parsetree.h> #include "chunk.h" #include "export.h" #include "guc.h" #include "hypertable.h" #include <storage/lockdefs.h> /* * Constraints created during planning to improve chunk exclusion * will be marked with this value as location so they can be easily * identified and removed when they are no longer needed. * Removal happens in timescaledb_set_rel_pathlist hook. */ #define PLANNER_LOCATION_MAGIC -29811 typedef struct Chunk Chunk; typedef struct Hypertable Hypertable; typedef struct TimescaleDBPrivate { bool appends_ordered; /* attno of the time dimension in the parent table if appends are ordered */ int order_attno; List *nested_oids; List *chunk_oids; /* Cached chunk data for the chunk relinfo. */ Chunk *cached_chunk_struct; /* Cached equivalence members for compressed chunks. List of (EC, EM) Lists. */ List *compressed_ec_em_pairs; } TimescaleDBPrivate; extern TSDLLEXPORT bool ts_rte_is_hypertable(const RangeTblEntry *rte); extern TSDLLEXPORT bool ts_rte_is_marked_for_expansion(const RangeTblEntry *rte); extern TSDLLEXPORT bool ts_contains_external_param(Node *node); extern TSDLLEXPORT bool ts_contains_join_param(Node *node); static inline TimescaleDBPrivate * ts_create_private_reloptinfo(RelOptInfo *rel) { Assert(rel->fdw_private == NULL); rel->fdw_private = palloc0(sizeof(TimescaleDBPrivate)); return rel->fdw_private; } static inline TimescaleDBPrivate * ts_get_private_reloptinfo(RelOptInfo *rel) { /* If rel->fdw_private is not set up here it means the rel got missclassified * and did not get expanded by our code but by postgres native code. * This is not a problem by itself, but probably an oversight on our part. */ Assert(rel->fdw_private); return rel->fdw_private ? rel->fdw_private : ts_create_private_reloptinfo(rel); } /* * TsRelType provides consistent classification of planned relations across * planner hooks. */ typedef enum TsRelType { TS_REL_HYPERTABLE, /* A hypertable with no parent */ TS_REL_CHUNK_STANDALONE, /* Chunk with no parent (i.e., it's part of the * plan as a standalone table. For example, * querying the chunk directly and not via the * parent hypertable). */ TS_REL_HYPERTABLE_CHILD, /* Self child. With PostgreSQL's table expansion, * the root table is expanded as a child of * itself. This happens when our expansion code * is turned off. */ TS_REL_CHUNK_CHILD, /* Chunk with parent and the result of table * expansion. */ TS_REL_OTHER, /* Anything which is none of the above */ } TsRelType; extern TSDLLEXPORT Hypertable *ts_planner_get_hypertable(const Oid relid, const unsigned int flags); extern void ts_preprocess_first_last_aggregates(PlannerInfo *root, List *tlist); extern void ts_plan_expand_hypertable_chunks(Hypertable *ht, PlannerInfo *root, RelOptInfo *rel, bool include_osm); extern void ts_plan_expand_timebucket_annotate(PlannerInfo *root, RelOptInfo *rel); extern Expr *ts_transform_time_bucket_comparison(Expr *); extern Node *ts_constify_now(PlannerInfo *root, List *rtable, Node *node); extern void ts_planner_constraint_cleanup(PlannerInfo *root, RelOptInfo *rel); extern Node *ts_add_space_constraints(PlannerInfo *root, List *rtable, Node *node); extern TSDLLEXPORT void ts_add_baserel_cache_entry_for_chunk(Oid chunk_reloid, Hypertable *hypertable); TsRelType TSDLLEXPORT ts_classify_relation(const PlannerInfo *root, const RelOptInfo *rel, Hypertable **ht); /* * Chunk-equivalent of planner_rt_fetch(), but returns the corresponding chunk * instead of range table entry. * * Returns NULL if this rel is not a chunk. * * This cache should be pre-warmed by hypertable expansion, but it * doesn't run in the following cases: * * 1. if it was a direct query on the chunk; * * 2. if it is not a SELECT QUERY. */ static inline const Chunk * ts_planner_chunk_fetch(const PlannerInfo *root, RelOptInfo *rel) { TimescaleDBPrivate *rel_private; /* The rel can only be a chunk if it is part of a hypertable expansion * (RELOPT_OTHER_MEMBER_REL) or a directly query on the chunk * (RELOPT_BASEREL) */ if (rel->reloptkind != RELOPT_OTHER_MEMBER_REL && rel->reloptkind != RELOPT_BASEREL) return NULL; /* The rel_private entry should have been created as part of classifying * the relation in timescaledb_get_relation_info_hook(). Therefore, * ts_get_private_reloptinfo() asserts that it is already set but falls * back to creating rel_private in release builds for safety. */ rel_private = ts_get_private_reloptinfo(rel); if (NULL == rel_private->cached_chunk_struct) { RangeTblEntry *rte = planner_rt_fetch(rel->relid, root); /* * Get the chunk and cache it. Do not use a slice tuple lock because * that will assign a transaction ID, which is not necessary for * queries. */ rel_private->cached_chunk_struct = ts_chunk_get_by_relid_locked(rte->relid, AccessShareLock, NULL, /* fail_if_not_found = */ true); } return rel_private->cached_chunk_struct; } ================================================ FILE: src/planner/space_constraint.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <datatype/timestamp.h> #include <nodes/makefuncs.h> #include <nodes/pg_list.h> #include <optimizer/optimizer.h> #include <parser/parse_func.h> #include <utils/fmgroids.h> #include <utils/typcache.h> #include "cache.h" #include "dimension.h" #include "hypertable.h" #include "hypertable_cache.h" #include "partitioning.h" #include "planner.h" /* * Returns space dimension for a specific column. Returns NULL * if the column is not a space dimension. */ static Dimension * get_space_dimension(Oid relid, AttrNumber varattno) { Hypertable *ht = ts_planner_get_hypertable(relid, CACHE_FLAG_CHECK); if (!ht) return NULL; for (uint16 i = 0; i < ht->space->num_dimensions; i++) { Dimension *dim = &ht->space->dimensions[i]; if (dim->type == DIMENSION_TYPE_CLOSED && dim->column_attno == varattno) { return dim; } } return NULL; } /* * Check if this operator is compatible with the constraints on * the space dimension. This is the equality operator between * left and right in the btree operator family. */ bool ts_is_equality_operator(Oid opno, Oid left, Oid right) { TypeCacheEntry *tce; if (left == right) { /* * When left and right match lookup_type_cache can * directly return the equality operator saving us * one roundtrip. */ tce = lookup_type_cache(left, TYPECACHE_EQ_OPR); return tce && opno == tce->eq_opr; } else { /* * The left and right type might not match when comparing * different integer types eg comparing int2 or int8 * columns with integer literals which default to int4. */ tce = lookup_type_cache(left, TYPECACHE_BTREE_OPFAMILY); if (!tce) return false; Oid eqop = get_opfamily_member(tce->btree_opf, left, right, BTEqualStrategyNumber); return opno == eqop; } } /* * Valid constraints are: Var = Const * Var has to refer to a space partitioning column */ static bool is_valid_space_constraint(OpExpr *op, List *rtable) { Assert(IsA(op, OpExpr)); if (!IsA(linitial(op->args), Var) || !IsA(lsecond(op->args), Const)) return false; Var *var = linitial_node(Var, op->args); if (var->varlevelsup != 0) return false; Const *value = lsecond_node(Const, op->args); if (!ts_is_equality_operator(op->opno, var->vartype, value->consttype)) return false; /* * Check that the constraint is actually on a partitioning column. */ Assert((int) var->varno <= list_length(rtable)); RangeTblEntry *rte = list_nth(rtable, var->varno - 1); Dimension *dim = get_space_dimension(rte->relid, var->varattno); if (!dim) return false; return true; } /* * Valid constraints are: * Var = ANY(ARRAY[Const,Const]) * Var IN (Const,Const) * Var has to refer to a space partitioning column */ static bool is_valid_scalar_space_constraint(ScalarArrayOpExpr *op, List *rtable) { Assert(IsA(op, ScalarArrayOpExpr)); if (!IsA(linitial(op->args), Var) || !IsA(lsecond(op->args), ArrayExpr)) return false; Var *var = linitial_node(Var, op->args); ArrayExpr *arr = castNode(ArrayExpr, lsecond(op->args)); if (arr->multidims || !op->useOr || var->varlevelsup != 0) return false; if (!ts_is_equality_operator(op->opno, var->vartype, arr->element_typeid)) return false; /* * Check that the constraint is actually on a partitioning column. */ Assert((int) var->varno <= list_length(rtable)); RangeTblEntry *rte = list_nth(rtable, var->varno - 1); Dimension *dim = get_space_dimension(rte->relid, var->varattno); if (!dim) return false; ListCell *lc; foreach (lc, arr->elements) { switch (nodeTag(lfirst(lc))) { case T_Const: break; case T_FuncExpr: { FuncExpr *element = lfirst_node(FuncExpr, lc); if (element->funcformat != COERCE_IMPLICIT_CAST || !IsA(linitial(element->args), Const)) return false; break; } default: return false; break; } } return true; } static FuncExpr * make_partfunc_call(Oid funcid, Oid rettype, List *args, Oid inputcollid) { /* build FuncExpr to use in eval_const_expressions */ return makeFuncExpr(funcid /* funcid */, rettype /* rettype */, args /* args */, InvalidOid /* funccollid */, inputcollid /* inputcollid */, COERCE_EXPLICIT_CALL /* fformat */); } /* * Transform a constraint like: device_id = 1 * into * ((device_id = 1) AND (_timescaledb_functions.get_partition_hash(device_id) = 242423622)) */ static OpExpr * transform_space_constraint(PlannerInfo *root, List *rtable, OpExpr *op) { Var *var = linitial_node(Var, op->args); Const *value = lsecond_node(Const, op->args); Const *part_value; RangeTblEntry *rte = list_nth(rtable, var->varno - 1); Dimension *dim = get_space_dimension(rte->relid, var->varattno); Oid rettype = dim->partitioning->partfunc.rettype; TypeCacheEntry *tce = lookup_type_cache(rettype, TYPECACHE_EQ_OPR); /* build FuncExpr to use in eval_const_expressions */ FuncExpr *partcall = make_partfunc_call(dim->partitioning->partfunc.func_fmgr.fn_oid, rettype, list_make1(value), var->varcollid); /* * We should always be able to constify here */ part_value = castNode(Const, eval_const_expressions(root, (Node *) partcall)); /* build FuncExpr with column reference to use in constraint */ partcall->args = list_make1(copyObject(var)); OpExpr *ret = (OpExpr *) make_opclause(tce->eq_opr /* opno */, BOOLOID /*opresulttype */, false /* opretset */, (Expr *) partcall /* left */, (Expr *) part_value /* right */, InvalidOid /* opcollid */, InvalidOid /* inputcollid */); ret->location = PLANNER_LOCATION_MAGIC; return ret; } /* * Transforms a constraint like: s1 = ANY ('{s1_2,s1_2}'::text[]) * into * ((s1 = ANY ('{s1_2,s1_2}'::text[])) AND (_timescaledb_functions.get_partition_hash(s1) = ANY * ('{1583420735,1583420735}'::integer[]))) */ static ScalarArrayOpExpr * transform_scalar_space_constraint(PlannerInfo *root, List *rtable, ScalarArrayOpExpr *op) { Var *var = linitial_node(Var, op->args); RangeTblEntry *rte = list_nth(rtable, var->varno - 1); Dimension *dim = get_space_dimension(rte->relid, var->varattno); Oid rettype = dim->partitioning->partfunc.rettype; TypeCacheEntry *tce = lookup_type_cache(rettype, TYPECACHE_EQ_OPR); List *part_values = NIL; ListCell *lc; /* build FuncExpr to use in eval_const_expressions */ FuncExpr *partcall = make_partfunc_call(dim->partitioning->partfunc.func_fmgr.fn_oid, rettype, NIL, var->varcollid); foreach (lc, lsecond_node(ArrayExpr, op->args)->elements) { Assert(IsA(lfirst(lc), Const) || (IsA(lfirst(lc), FuncExpr) && lfirst_node(FuncExpr, lc)->funcformat == COERCE_IMPLICIT_CAST)); /* * We can skip NULL here as elements are ORed and partitioning dimensions * have NOT NULL constraint. */ if (IsA(lfirst(lc), Const) && lfirst_node(Const, lc)->constisnull) continue; List *args = list_make1(lfirst(lc)); partcall->args = args; part_values = lappend(part_values, castNode(Const, eval_const_expressions(root, (Node *) partcall))); } /* build FuncExpr with column reference to use in constraint */ partcall->args = list_make1(copyObject(var)); ArrayExpr *arr2 = makeNode(ArrayExpr); arr2->array_collid = InvalidOid; arr2->array_typeid = get_array_type(rettype); arr2->element_typeid = rettype; arr2->multidims = false; arr2->location = -1; arr2->elements = part_values; ScalarArrayOpExpr *op2 = makeNode(ScalarArrayOpExpr); op2->opno = tce->eq_opr; op2->args = list_make2(partcall, arr2); op2->inputcollid = InvalidOid; op2->useOr = true; op2->location = PLANNER_LOCATION_MAGIC; return op2; } /* * Transform constraints for hash-based partitioning columns to make * them usable by postgres constraint exclusion. * * If we have an equality condition on a space partitioning column, we add * a corresponding condition on get_partition_hash on this column. These * conditions match the constraints on chunks, so postgres' constraint * exclusion is able to use them and exclude the chunks. * */ Node * ts_add_space_constraints(PlannerInfo *root, List *rtable, Node *node) { Assert(node); switch (nodeTag(node)) { case T_ScalarArrayOpExpr: { if (is_valid_scalar_space_constraint(castNode(ScalarArrayOpExpr, node), rtable)) { List *args = list_make2(node, transform_scalar_space_constraint(root, rtable, castNode(ScalarArrayOpExpr, node))); return (Node *) makeBoolExpr(AND_EXPR, args, -1); } break; } case T_OpExpr: if (is_valid_space_constraint(castNode(OpExpr, node), rtable)) { List *args = list_make2(node, transform_space_constraint(root, rtable, castNode(OpExpr, node))); return (Node *) makeBoolExpr(AND_EXPR, args, -1); } break; case T_BoolExpr: { ListCell *lc; BoolExpr *be = castNode(BoolExpr, node); if (be->boolop == AND_EXPR) { List *additions = NIL; /* * If this is a top-level AND we can just append our transformed constraints * to the list of ANDed expressions. */ foreach (lc, be->args) { switch (nodeTag(lfirst(lc))) { case T_OpExpr: { OpExpr *op = lfirst_node(OpExpr, lc); if (is_valid_space_constraint(op, rtable)) additions = lappend(additions, transform_space_constraint(root, rtable, op)); break; } case T_ScalarArrayOpExpr: { ScalarArrayOpExpr *op = lfirst_node(ScalarArrayOpExpr, lc); if (is_valid_scalar_space_constraint(op, rtable)) additions = lappend(additions, transform_scalar_space_constraint(root, rtable, op)); break; } default: break; } } if (additions) be->args = list_concat(be->args, additions); } break; } default: break; } return node; } ================================================ FILE: src/process_utility.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/htup_details.h> #include <access/xact.h> #include <catalog/heap.h> #include <catalog/index.h> #include <catalog/namespace.h> #include <catalog/objectaddress.h> #include <catalog/pg_am.h> #include <catalog/pg_authid.h> #include <catalog/pg_class_d.h> #include <catalog/pg_constraint.h> #include <catalog/pg_inherits.h> #include <catalog/pg_trigger.h> #include <commands/alter.h> #include <commands/cluster.h> #include <commands/copy.h> #include <commands/defrem.h> #include <commands/event_trigger.h> #include <commands/prepare.h> #include <commands/tablecmds.h> #include <commands/tablespace.h> #include <commands/trigger.h> #include <commands/user.h> #include <commands/vacuum.h> #include <executor/spi.h> #include <miscadmin.h> #include <nodes/lockoptions.h> #include <nodes/makefuncs.h> #include <nodes/nodes.h> #include <nodes/parsenodes.h> #include <optimizer/optimizer.h> #include <parser/parse_expr.h> #include <parser/parse_relation.h> #include <parser/parse_type.h> #include <parser/parse_utilcmd.h> #include <storage/lmgr.h> #include <storage/lockdefs.h> #include <tcop/utility.h> #include <utils/acl.h> #include <utils/builtins.h> #include <utils/elog.h> #include <utils/guc.h> #include <utils/inval.h> #include <utils/lsyscache.h> #include <utils/palloc.h> #include <utils/regproc.h> #include <utils/rel.h> #include <utils/ruleutils.h> #include <utils/snapmgr.h> #include <utils/syscache.h> #include "compat/compat.h" #include "annotations.h" #include "chunk.h" #include "chunk_index.h" #include "copy.h" #include "cross_module_fn.h" #include "debug_assert.h" #include "debug_point.h" #include "dimension_vector.h" #include "errors.h" #include "event_trigger.h" #include "export.h" #include "extension.h" #include "extension_constants.h" #include "foreign_key.h" #include "hypercube.h" #include "hypertable.h" #include "hypertable_cache.h" #include "indexing.h" #include "license_guc.h" #include "partition_chunk.h" #include "partitioning.h" #include "process_utility.h" #include "scan_iterator.h" #include "time_utils.h" #include "trigger.h" #include "ts_catalog/array_utils.h" #include "ts_catalog/catalog.h" #include "ts_catalog/chunk_column_stats.h" #include "ts_catalog/chunk_rewrite.h" #include "ts_catalog/compression_settings.h" #include "ts_catalog/continuous_agg.h" #include "ts_catalog/continuous_aggs_watermark.h" #include "tss_callbacks.h" #include "utils.h" #include "with_clause/alter_table_with_clause.h" #include "with_clause/create_materialized_view_with_clause.h" #include "with_clause/create_table_with_clause.h" #include "with_clause/with_clause_parser.h" #ifdef USE_TELEMETRY #include "telemetry/functions.h" #endif void _process_utility_init(void); void _process_utility_fini(void); static ProcessUtility_hook_type prev_ProcessUtility_hook; static bool expect_chunk_modification = false; static ProcessUtilityContext last_process_utility_context = PROCESS_UTILITY_TOPLEVEL; static void check_no_timescale_options(AlterTableCmd *cmd, Oid reloid); static DDLResult process_altertable_set_options(AlterTableCmd *cmd, Hypertable *ht); static DDLResult process_altertable_reset_options(AlterTableCmd *cmd, Hypertable *ht); static void ts_bgw_job_update_owner(Relation rel, HeapTuple tuple, TupleDesc tupledesc, Oid newrole_oid); /* Call the default ProcessUtility and handle PostgreSQL version differences */ static void prev_ProcessUtility(ProcessUtilityArgs *args) { ProcessUtility_hook_type hook = prev_ProcessUtility_hook ? prev_ProcessUtility_hook : standard_ProcessUtility; hook(args->pstmt, args->query_string, args->readonly_tree, args->context, args->params, args->queryEnv, args->dest, args->completion_tag); /* * Reset the last_process_utility_context value that is saved at the * entrance of the TS ProcessUtility hook and can be used for transaction * checks inside refresh_cagg and other procedures. */ ts_process_utility_context_reset(); } static void check_chunk_alter_table_operation_allowed(Oid relid, AlterTableStmt *stmt) { const Chunk *chunk; if (expect_chunk_modification) return; chunk = ts_chunk_get_by_relid(relid, false /* fail_if_not_found */); if (chunk != NULL) { bool all_allowed = true; ListCell *lc; /* only allow if all commands are allowed */ foreach (lc, stmt->cmds) { AlterTableCmd *cmd = (AlterTableCmd *) lfirst(lc); switch (cmd->subtype) { case AT_SetOptions: case AT_ResetOptions: case AT_SetRelOptions: case AT_ResetRelOptions: case AT_ReplaceRelOptions: case AT_SetStatistics: case AT_SetStorage: case AT_DropCluster: case AT_ClusterOn: case AT_EnableRowSecurity: case AT_DisableRowSecurity: case AT_SetTableSpace: case AT_ReAddStatistics: case AT_SetCompression: case AT_SetAccessMethod: case AT_SetLogged: case AT_SetUnLogged: /* allowed on chunks */ break; case AT_AddConstraint: { /* if this is an OSM chunk, block the operation */ if (chunk->fd.osm_chunk) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("operation not supported on OSM chunk tables"))); } break; } case AT_DropConstraint: { /* if this is an OSM chunk, block the operation */ if (chunk->fd.osm_chunk) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("operation not supported on OSM chunk tables"))); } ChunkConstraints *ccs = ts_chunk_constraint_scan_by_chunk_id(chunk->fd.id, 10, CurrentMemoryContext); Assert(cmd->name); for (int i = 0; i < ccs->num_constraints; i++) { ChunkConstraint *cc = &ccs->constraints[i]; if (namestrcmp(&cc->fd.constraint_name, cmd->name) == 0) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot drop inherited constraint"), errhint("Drop the constraint on the hypertable instead."))); } break; } default: /* disable by default */ all_allowed = false; break; } } if (!all_allowed) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("operation not supported on chunk tables"))); } } /* we block some ALTER commands on continuous aggregate materialization tables */ static void check_continuous_agg_alter_table_allowed(Hypertable *ht, AlterTableStmt *stmt) { ListCell *lc; ContinuousAggHypertableStatus status = ts_continuous_agg_hypertable_status(ht->fd.id); if ((status & HypertableIsMaterialization) == 0) return; /* only allow if all commands are allowed */ foreach (lc, stmt->cmds) { AlterTableCmd *cmd = (AlterTableCmd *) lfirst(lc); switch (cmd->subtype) { case AT_AddIndex: case AT_ReAddIndex: case AT_SetRelOptions: case AT_ReplicaIdentity: /* allowed on materialization tables */ continue; default: /* disable by default */ ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("operation not supported on materialization tables"))); break; } } } /* check if hypertable has compressed chunks */ static bool ts_hypertable_has_compressed_chunks(const Hypertable *ht) { if (!TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(ht)) return false; return ts_chunk_exists_with_compression(ht->fd.id); } static void check_alter_table_allowed_on_ht_with_compression(Hypertable *ht, AlterTableStmt *stmt) { ListCell *lc; if (!TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(ht)) return; /* only allow if all commands are allowed */ foreach (lc, stmt->cmds) { AlterTableCmd *cmd = (AlterTableCmd *) lfirst(lc); switch (cmd->subtype) { /* * ALLOWED: * * This is a whitelist of allowed commands. */ case AT_AddIndex: case AT_AddConstraint: case AT_ReAddIndex: case AT_ResetRelOptions: case AT_ReplaceRelOptions: case AT_SetRelOptions: case AT_ClusterOn: case AT_DropCluster: case AT_ChangeOwner: /* this is passed down in `process_altertable_change_owner` */ case AT_SetTableSpace: /* this is passed down in `process_altertable_set_tablespace_end` */ case AT_SetStatistics: /* should this be pushed down in some way? */ case AT_AddColumn: /* this is passed down */ case AT_ColumnDefault: /* this is passed down */ case AT_DropColumn: /* this is passed down */ case AT_DropConstraint: /* this is passed down */ case AT_ReplicaIdentity: case AT_ReAddStatistics: case AT_SetCompression: case AT_DropNotNull: case AT_SetNotNull: case AT_SetAccessMethod: case AT_SetLogged: case AT_SetUnLogged: continue; /* * BLOCKED: * * List things that we want to explicitly block for documentation purposes * But also block everything else as well. */ case AT_AlterColumnType: /* Allow AT_AlterColumnType when no compressed chunks exist */ if (!ts_hypertable_has_compressed_chunks(ht)) continue; ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("operation not supported on hypertables with compressed chunks"))); break; case AT_EnableRowSecurity: case AT_DisableRowSecurity: case AT_ForceRowSecurity: case AT_NoForceRowSecurity: default: ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("operation not supported on hypertables that have columnstore " "enabled"))); break; } } } static void check_altertable_add_column_for_compressed(ParseState *parse_state, Hypertable *ht, ColumnDef *col) { if (col->constraints) { bool has_default = false; bool has_notnull = col->is_not_null; ListCell *lc; foreach (lc, col->constraints) { Constraint *constraint = lfirst_node(Constraint, lc); switch (constraint->contype) { /* * These will fail in combination with ADD COLUMN because this will * be a single column constraint and we require all partitioning * columns to be part if the unique/primary key constraint. */ case CONSTR_PRIMARY: case CONSTR_UNIQUE: break; /* * We can safelly ignore NULL constraints because it does nothing * and according to Postgres docs is useless and exist just for * compatibility with other database systems * https://www.postgresql.org/docs/current/ddl-constraints.html#id-1.5.4.6.6 */ case CONSTR_NULL: continue; case CONSTR_NOTNULL: has_notnull = true; continue; /* * check constraints are validated at end of alter table command * in validate_check_constraint */ case CONSTR_CHECK: continue; case CONSTR_DEFAULT: { /* * Since default expressions might trigger a table rewrite we * only allow Const here for now. */ Oid typeoid; int32 typmod; typenameTypeIdAndMod(parse_state, col->typeName, &typeoid, &typmod); Node *transformed = cookDefault(parse_state, constraint->raw_expr, typeoid, typmod, col->colname, col->generated); Node *constified = eval_const_expressions(NULL, transformed); if (!IsA(constified, Const)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot add column with non-constant default expression " "to a hypertable that has columnstore enabled"))); } has_default = true; continue; } default: ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot add column with constraints " "to a hypertable that has columnstore enabled"))); break; } } /* require a default for columns added with NOT NULL */ if (has_notnull && !has_default) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot add column with NOT NULL constraint without default " "to a hypertable that has columnstore enabled"))); } } if (col->is_not_null || col->identitySequence != NULL) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot add column with constraints to a hypertable that has " "columnstore enabled"))); } /* not possible to get non-null value here this is set when * ALTER TABLE ALTER COLUMN ... SET TYPE < > USING ... * but check anyway. */ Assert(col->raw_default == NULL && col->cooked_default == NULL); } static void relation_not_only(RangeVar *rv) { if (!rv->inh) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("ONLY option not supported on hypertable operations"))); } static bool check_table_in_rangevar_list(List *rvlist, Name schema_name, Name table_name) { ListCell *l; foreach (l, rvlist) { RangeVar *rvar = lfirst_node(RangeVar, l); if (strcmp(rvar->relname, NameStr(*table_name)) == 0 && strcmp(rvar->schemaname, NameStr(*schema_name)) == 0) return true; } return false; } static void add_chunk_oid(Hypertable *ht, Oid chunk_relid, void *vargs) { ProcessUtilityArgs *args = vargs; GrantStmt *stmt = castNode(GrantStmt, args->parsetree); Chunk *chunk = ts_chunk_get_by_relid(chunk_relid, true); /* * If chunk is in the same schema as the hypertable it could already be part of * the objects list in the case of "GRANT ALL IN SCHEMA" for example */ if (!check_table_in_rangevar_list(stmt->objects, &chunk->fd.schema_name, &chunk->fd.table_name)) { RangeVar *rv = makeRangeVar(NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name), -1); stmt->objects = lappend(stmt->objects, rv); } } static DDLResult process_drop_schema_start(DropStmt *stmt) { /* * An error will be raised when we start dropping the functions used by a * background worker, so there is no point in doing anything here. */ if (stmt->behavior == DROP_RESTRICT) return DDL_CONTINUE; /* * Here we are relying on that if we fail to drop one of the * procedures/functions, this transaction will be rolled back so these * changes will not be committed. */ ScanIterator iterator = ts_scan_iterator_create(BGW_JOB, RowExclusiveLock, CurrentMemoryContext); ts_scanner_foreach(&iterator) { ListCell *cell; TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); bool schema_isnull, job_id_isnull; int32 job_id = DatumGetInt32(slot_getattr(ti->slot, Anum_bgw_job_id, &job_id_isnull)); Name proc_schema = DatumGetName(slot_getattr(ti->slot, Anum_bgw_job_proc_schema, &schema_isnull)); Ensure(!job_id_isnull, "corrupt job entry: job id is null"); Ensure(!schema_isnull, "corrupt job entry: schema for job %d is null", job_id); foreach (cell, stmt->objects) { String *object = lfirst_node(String, cell); if (namestrcmp(proc_schema, strVal(object)) == 0) { CatalogSecurityContext sec_ctx; Assert(stmt->behavior == DROP_CASCADE); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ereport(NOTICE, errmsg("drop cascades to job %d", job_id)); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); ts_catalog_restore_user(&sec_ctx); } } } return DDL_CONTINUE; } /* * Start of dropping a procedure. * * We can abort the drop here by throwing an error. */ static void process_drop_procedure_start(DropStmt *stmt) { ScanIterator iterator = ts_scan_iterator_create(BGW_JOB, RowExclusiveLock, CurrentMemoryContext); ts_scanner_foreach(&iterator) { ListCell *cell; TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); bool schema_isnull, name_isnull, job_id_isnull; Name proc_schema = DatumGetName(slot_getattr(ti->slot, Anum_bgw_job_proc_schema, &schema_isnull)); Name proc_name = DatumGetName(slot_getattr(ti->slot, Anum_bgw_job_proc_name, &name_isnull)); int32 job_id = DatumGetInt32(slot_getattr(ti->slot, Anum_bgw_job_id, &job_id_isnull)); Ensure(!job_id_isnull, "corrupt job entry: job id was null"); Ensure(!schema_isnull, "corrupt job entry: schema for job %d was null", job_id); Ensure(!name_isnull, "corrupt job entry: name for job %d was null", job_id); TS_DEBUG_LOG("looking at job %d with %s.%s", job_id, NameStr(*proc_schema), NameStr(*proc_name)); foreach (cell, stmt->objects) { ObjectWithArgs *object = castNode(ObjectWithArgs, lfirst(cell)); RangeVar *rel = makeRangeVarFromNameList(object->objname); if (namestrcmp(proc_schema, rel->schemaname) == 0 && namestrcmp(proc_name, rel->relname) == 0) { Assert(stmt->removeType == OBJECT_PROCEDURE || stmt->removeType == OBJECT_FUNCTION); if (stmt->behavior == DROP_RESTRICT) { ereport(ERROR, errcode(ERRCODE_DEPENDENT_OBJECTS_STILL_EXIST), errmsg("cannot drop %s because background job %d depends on it", NameListToString(object->objname), job_id), errhint("Use delete_job() to drop the job first.")); } else { CatalogSecurityContext sec_ctx; ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ereport(NOTICE, errmsg("drop cascades to job %d", job_id)); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); ts_catalog_restore_user(&sec_ctx); } } } } } static void replace_attr_if_changed(AttrNumber attno, const char *newvalue, Name name_buf, Datum *values, bool *replace) { if (newvalue) { const Name orig_value = DatumGetName(values[AttrNumberGetAttrOffset(attno)]); if (namestrcmp(orig_value, newvalue) != 0) { namestrcpy(name_buf, newvalue); values[AttrNumberGetAttrOffset(attno)] = NameGetDatum(name_buf); replace[AttrNumberGetAttrOffset(attno)] = true; } } } /* * Update the schema or name of a procedure in the jobs tuple. */ static void ts_bgw_job_update_proc(Relation rel, HeapTuple tuple, TupleDesc tupledesc, const char *newschema, const char *newname) { bool isnull[Natts_bgw_job]; Datum values[Natts_bgw_job]; bool replace[Natts_bgw_job] = { false }; /* Allocated here to make sure that they are alive at the call of * heap_modify_tuple */ NameData proc_name_buf; NameData proc_schema_buf; heap_deform_tuple(tuple, tupledesc, values, isnull); replace_attr_if_changed(Anum_bgw_job_proc_name, newname, &proc_name_buf, values, replace); replace_attr_if_changed(Anum_bgw_job_proc_schema, newschema, &proc_schema_buf, values, replace); HeapTuple new_tuple = heap_modify_tuple(tuple, tupledesc, values, isnull, replace); ts_catalog_update(rel, new_tuple); heap_freetuple(new_tuple); } static void ts_bgw_job_rename_schema_name(const char *old_schema_name, const char *new_schema_name) { ScanIterator iterator = ts_scan_iterator_create(BGW_JOB, RowExclusiveLock, CurrentMemoryContext); ts_scanner_foreach(&iterator) { bool should_free, curr_schema_isnull, curr_name_isnull; TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); Name curr_proc_schema = DatumGetName(slot_getattr(ti->slot, Anum_bgw_job_proc_schema, &curr_schema_isnull)); Name curr_proc_name = DatumGetName(slot_getattr(ti->slot, Anum_bgw_job_proc_name, &curr_name_isnull)); if (!curr_schema_isnull && namestrcmp(curr_proc_schema, old_schema_name) == 0) { HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); ts_bgw_job_update_proc(ti->scanrel, tuple, ts_scanner_get_tupledesc(ti), new_schema_name, NameStr(*curr_proc_name)); if (should_free) heap_freetuple(tuple); } } } static DDLResult ts_bgw_job_rename_proc(ObjectAddress address, const char *newschema, const char *newname) { ScanIterator iterator = ts_scan_iterator_create(BGW_JOB, RowExclusiveLock, CurrentMemoryContext); ts_scanner_foreach(&iterator) { bool should_free, curr_schema_isnull, curr_name_isnull; TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); Name curr_proc_schema = DatumGetName(slot_getattr(ti->slot, Anum_bgw_job_proc_schema, &curr_schema_isnull)); Name curr_proc_name = DatumGetName(slot_getattr(ti->slot, Anum_bgw_job_proc_name, &curr_name_isnull)); const char *old_proc_schema = get_namespace_name(get_func_namespace(address.objectId)); const char *old_proc_name = get_func_name(address.objectId); if (!curr_schema_isnull && !curr_name_isnull && namestrcmp(curr_proc_name, old_proc_name) == 0 && namestrcmp(curr_proc_schema, old_proc_schema) == 0) { HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); ts_bgw_job_update_proc(ti->scanrel, tuple, ts_scanner_get_tupledesc(ti), newschema, newname); if (should_free) heap_freetuple(tuple); } } return DDL_CONTINUE; } static void process_alterprocedureschema(ProcessUtilityArgs *args) { AlterObjectSchemaStmt *stmt = (AlterObjectSchemaStmt *) args->parsetree; Relation relation; Assert(stmt->objectType == OBJECT_PROCEDURE || stmt->objectType == OBJECT_FUNCTION); ObjectAddress address = get_object_address(stmt->objectType, stmt->object, &relation, AccessExclusiveLock, false); ts_bgw_job_rename_proc(address, stmt->newschema, NULL); } /* We use this for both materialized views and views. */ static void process_alterviewschema(ProcessUtilityArgs *args) { AlterObjectSchemaStmt *stmt = (AlterObjectSchemaStmt *) args->parsetree; Oid relid; char *schema; char *name; Assert(stmt->objectType == OBJECT_MATVIEW || stmt->objectType == OBJECT_VIEW); if (NULL == stmt->relation) return; relid = RangeVarGetRelid(stmt->relation, NoLock, true); if (!OidIsValid(relid)) return; schema = get_namespace_name(get_rel_namespace(relid)); name = get_rel_name(relid); ts_continuous_agg_rename_view(schema, name, stmt->newschema, name, &stmt->objectType); } static void process_altertableschema(ProcessUtilityArgs *args) { AlterObjectSchemaStmt *alterstmt = (AlterObjectSchemaStmt *) args->parsetree; Oid relid; Cache *hcache; Hypertable *ht; Assert(alterstmt->objectType == OBJECT_TABLE); if (NULL == alterstmt->relation) return; relid = RangeVarGetRelid(alterstmt->relation, NoLock, true); if (!OidIsValid(relid)) return; ht = ts_hypertable_cache_get_cache_and_entry(relid, CACHE_FLAG_MISSING_OK, &hcache); if (ht == NULL) { ContinuousAgg *cagg = ts_continuous_agg_find_by_relid(relid); if (cagg) { alterstmt->objectType = OBJECT_MATVIEW; process_alterviewschema(args); ts_cache_release(&hcache); return; } Chunk *chunk = ts_chunk_get_by_relid(relid, false); if (NULL != chunk) ts_chunk_set_schema(chunk, alterstmt->newschema); } else { ts_hypertable_set_schema(ht, alterstmt->newschema); } ts_cache_release(&hcache); } /* Change the schema of a hypertable or a chunk */ static DDLResult process_alterobjectschema(ProcessUtilityArgs *args) { AlterObjectSchemaStmt *alterstmt = (AlterObjectSchemaStmt *) args->parsetree; switch (alterstmt->objectType) { case OBJECT_TABLE: process_altertableschema(args); break; case OBJECT_MATVIEW: case OBJECT_VIEW: process_alterviewschema(args); break; case OBJECT_PROCEDURE: case OBJECT_FUNCTION: process_alterprocedureschema(args); break; default: break; } return DDL_CONTINUE; } static DDLResult process_copy(ProcessUtilityArgs *args) { CopyStmt *stmt = (CopyStmt *) args->parsetree; ts_begin_tss_store_callback(); /* * Needed to add the appropriate number of tuples to the completion tag */ uint64 processed; Hypertable *ht = NULL; Cache *hcache = NULL; Oid relid; if (stmt->relation) { relid = RangeVarGetRelid(stmt->relation, NoLock, true); if (!OidIsValid(relid)) return DDL_CONTINUE; ht = ts_hypertable_cache_get_cache_and_entry(relid, CACHE_FLAG_MISSING_OK, &hcache); if (!ht) { Chunk *chunk = ts_chunk_get_by_relid(relid, false); /* target is neither hypertable nor chunk so let postgres handle it */ if (!chunk) { ts_cache_release(&hcache); return DDL_CONTINUE; } ht = ts_hypertable_get_by_id(chunk->fd.hypertable_id); if (ht->fd.compression_state == HypertableInternalCompressionTable) { /* * For operations on internal compressed chunks we block modifications * if the chunk belongs to a frozen chunk otherwise let postgres handle it. * Uncompressed frozen chunks are intercepted as part of tuple routing. */ Chunk *uncompressed = ts_chunk_get_compressed_chunk_parent(chunk); if (ts_chunk_is_frozen(uncompressed)) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("cannot COPY into chunk belonging to a frozen " "chunk"))); ts_cache_release(&hcache); return DDL_CONTINUE; } } } /* We only copy for COPY FROM (which copies into a hypertable). Since * hypertable data are in the hypertable chunks and no data would be * copied, we skip the copy for COPY TO, but print an informative * message. */ if (!stmt->is_from || !stmt->relation) { if (ht && stmt->relation) ereport(NOTICE, (errmsg("hypertable data are in the chunks, no data will be copied"), errdetail("Data for hypertables are stored in the chunks of a hypertable so " "COPY TO of a hypertable will not copy any data."), errhint("Use \"COPY (SELECT * FROM <hypertable>) TO ...\" to copy all data in " "hypertable, or copy each chunk individually."))); if (hcache) ts_cache_release(&hcache); return DDL_CONTINUE; } PreventCommandIfReadOnly("COPY FROM"); /* Performs acl check in here inside `copy_security_check` */ timescaledb_DoCopy(stmt, args->query_string, &processed, ht); args->completion_tag->commandTag = CMDTAG_COPY; args->completion_tag->nprocessed = processed; ts_cache_release(&hcache); ts_end_tss_store_callback(args->query_string, args->pstmt->stmt_location, args->pstmt->stmt_len, args->pstmt->queryId, args->completion_tag->nprocessed); return DDL_DONE; } typedef void (*process_chunk_t)(Hypertable *ht, Oid chunk_relid, void *arg); typedef void (*mt_process_chunk_t)(int32 hypertable_id, Oid chunk_relid, void *arg); /* * Applies a function to each chunk of a hypertable. * * Returns the number of processed chunks, or -1 if the table was not a * hypertable. */ static int foreach_chunk(Hypertable *ht, process_chunk_t process_chunk, void *arg) { List *chunks; ListCell *lc; int n = 0; if (NULL == ht) return -1; chunks = find_inheritance_children(ht->main_table_relid, NoLock); foreach (lc, chunks) { process_chunk(ht, lfirst_oid(lc), arg); n++; } return n; } /* * Applies a function to each compressed internal chunk of a hypertable. * * Returns the number of processed chunks, or -1 if the table was not a * hypertable. */ static int foreach_compressed_chunk(Hypertable *ht, process_chunk_t process_chunk, void *arg) { List *chunks; ListCell *lc; int n = 0; if (!ht || !ht->fd.compressed_hypertable_id) return -1; chunks = ts_chunk_get_by_hypertable_id(ht->fd.compressed_hypertable_id); foreach (lc, chunks) { Chunk *chunk = lfirst(lc); process_chunk(ht, chunk->table_id, arg); n++; } return n; } static int foreach_chunk_multitransaction(Oid relid, MemoryContext mctx, mt_process_chunk_t process_chunk, void *arg) { Cache *hcache; Hypertable *ht; int32 hypertable_id; List *chunks; ListCell *lc; int num_chunks = -1; StartTransactionCommand(); MemoryContextSwitchTo(mctx); LockRelationOid(relid, AccessShareLock); ht = ts_hypertable_cache_get_cache_and_entry(relid, CACHE_FLAG_MISSING_OK, &hcache); if (NULL == ht) { ts_cache_release(&hcache); CommitTransactionCommand(); return -1; } hypertable_id = ht->fd.id; chunks = find_inheritance_children(ht->main_table_relid, NoLock); ts_cache_release(&hcache); CommitTransactionCommand(); num_chunks = list_length(chunks); foreach (lc, chunks) { process_chunk(hypertable_id, lfirst_oid(lc), arg); } list_free(chunks); return num_chunks; } typedef struct VacuumCtx { bool is_vacuumfull; VacuumRelation *ht_vacuum_rel; List *chunk_rels; List *rebuild_columnstore_chunk_oids; } VacuumCtx; static bool chunk_has_missing_attrs(Chunk *chunk) { bool has_missing_attrs = false; Relation ht_rel = relation_open(chunk->hypertable_relid, AccessShareLock); Relation chunk_rel = relation_open(chunk->table_id, AccessShareLock); TupleDesc tupdesc = RelationGetDescr(chunk_rel); for (int i = 0; i < tupdesc->natts; i++) { Form_pg_attribute att = TupleDescAttr(tupdesc, i); if (att->atthasmissing) { has_missing_attrs = true; break; } } relation_close(chunk_rel, AccessShareLock); relation_close(ht_rel, AccessShareLock); return has_missing_attrs; } static void register_chunk_for_rebuild_if_needed(Oid chunk_relid, VacuumCtx *ctx) { /* Only VACUUM FULL does a complete table rewrite */ if (!ctx->is_vacuumfull) return; Chunk *chunk = ts_chunk_get_by_relid(chunk_relid, false); if (!chunk) return; /* * VACUUM FULL will materialize missing attributes. Propagate * these changes to columnstore data by rebuilding it. */ if (chunk_has_missing_attrs(chunk)) { /* * Frozen chunks are not allowed to add columns with defaults */ Ensure(!ts_chunk_is_frozen(chunk), "chunk \"%s.%s\" was altered unsafely after it was frozen", NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name)); ctx->rebuild_columnstore_chunk_oids = lappend_oid(ctx->rebuild_columnstore_chunk_oids, chunk_relid); } } /* Adds a chunk to the list of tables to be vacuumed */ static void add_chunk_to_vacuum(Hypertable *ht, Oid chunk_relid, void *arg) { VacuumCtx *ctx = (VacuumCtx *) arg; Chunk *chunk = ts_chunk_get_by_relid(chunk_relid, true); VacuumRelation *chunk_vacuum_rel; RangeVar *chunk_range_var; chunk_range_var = copyObject(ctx->ht_vacuum_rel->relation); chunk_range_var->relname = NameStr(chunk->fd.table_name); chunk_range_var->schemaname = NameStr(chunk->fd.schema_name); chunk_vacuum_rel = makeVacuumRelation(chunk_range_var, chunk_relid, ctx->ht_vacuum_rel->va_cols); ctx->chunk_rels = lappend(ctx->chunk_rels, chunk_vacuum_rel); /* If we have a compressed chunk make sure to analyze it as well */ if (chunk->fd.compressed_chunk_id != INVALID_CHUNK_ID) { Chunk *comp_chunk = ts_chunk_get_by_id(chunk->fd.compressed_chunk_id, false); /* Compressed chunk might be missing due to concurrent operations */ if (comp_chunk) { chunk_vacuum_rel = makeVacuumRelation(NULL, comp_chunk->table_id, NIL); ctx->chunk_rels = lappend(ctx->chunk_rels, chunk_vacuum_rel); } } register_chunk_for_rebuild_if_needed(chunk_relid, ctx); } /* * Construct a list of VacuumRelations for all vacuumable rels in * the current database. This is similar to the PostgresQL get_all_vacuum_rels * from vacuum.c. */ static List * ts_get_all_vacuum_rels(bool is_vacuumcmd, VacuumCtx *ctx) { List *vacrels = NIL; Relation pgclass; TableScanDesc scan; HeapTuple tuple; Cache *hcache = ts_hypertable_cache_pin(); pgclass = table_open(RelationRelationId, AccessShareLock); scan = table_beginscan_catalog(pgclass, 0, NULL); while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) { Form_pg_class classform = (Form_pg_class) GETSTRUCT(tuple); Oid relid; relid = classform->oid; /* check permissions of relation */ if (!vacuum_is_permitted_for_relation_compat(relid, classform, is_vacuumcmd ? VACOPT_VACUUM : VACOPT_ANALYZE)) continue; /* * We include partitioned tables here; depending on which operation is * to be performed, caller will decide whether to process or ignore * them. */ if (classform->relkind != RELKIND_RELATION && classform->relkind != RELKIND_MATVIEW && classform->relkind != RELKIND_PARTITIONED_TABLE) continue; /* * Build VacuumRelation(s) specifying the table OIDs to be processed. * We omit a RangeVar since it wouldn't be appropriate to complain * about failure to open one of these relations later. */ vacrels = lappend(vacrels, makeVacuumRelation(NULL, relid, NIL)); register_chunk_for_rebuild_if_needed(relid, ctx); } table_endscan(scan); table_close(pgclass, AccessShareLock); ts_cache_release(&hcache); return vacrels; } /* Vacuums/Analyzes a hypertable and all of it's chunks */ static DDLResult process_vacuum(ProcessUtilityArgs *args) { VacuumStmt *stmt = (VacuumStmt *) args->parsetree; bool is_toplevel = (args->context == PROCESS_UTILITY_TOPLEVEL); VacuumCtx ctx = { .is_vacuumfull = false, .ht_vacuum_rel = NULL, .chunk_rels = NIL, .rebuild_columnstore_chunk_oids = NIL, }; ListCell *lc; Hypertable *ht; List *vacuum_rels = NIL; bool is_vacuumcmd; /* save original VacuumRelation list */ List *saved_stmt_rels = stmt->rels; is_vacuumcmd = stmt->is_vacuumcmd; /* Look for new option ONLY_DATABASE_STATS and FULL */ foreach (lc, stmt->options) { DefElem *opt = (DefElem *) lfirst(lc); #if PG16_GE /* if "only_database_stats" is defined then don't execute our custom code and return to * the postgres execution for the proper validations */ if (is_vacuumcmd && strcmp(opt->defname, "only_database_stats") == 0) return DDL_CONTINUE; #endif if (strcmp(opt->defname, "full") == 0) ctx.is_vacuumfull = defGetBoolean(opt); } if (stmt->rels == NIL) vacuum_rels = ts_get_all_vacuum_rels(is_vacuumcmd, &ctx); else { Cache *hcache = ts_hypertable_cache_pin(); foreach (lc, stmt->rels) { VacuumRelation *vacuum_rel = lfirst_node(VacuumRelation, lc); Oid table_relid = vacuum_rel->oid; if (!OidIsValid(table_relid) && vacuum_rel->relation != NULL) table_relid = RangeVarGetRelid(vacuum_rel->relation, NoLock, true); if (OidIsValid(table_relid)) { ht = ts_hypertable_cache_get_entry(hcache, table_relid, CACHE_FLAG_MISSING_OK); if (ht) { ctx.ht_vacuum_rel = vacuum_rel; foreach_chunk(ht, add_chunk_to_vacuum, &ctx); } } vacuum_rels = lappend(vacuum_rels, vacuum_rel); } ts_cache_release(&hcache); } stmt->rels = list_concat(ctx.chunk_rels, vacuum_rels); /* The list of rels to vacuum could be empty if we are only vacuuming a * tiered hypertable with no local chunks. In that case, we don't want to vacuum locally. */ if (list_length(stmt->rels) > 0) { PreventCommandDuringRecovery(is_vacuumcmd ? "VACUUM" : "ANALYZE"); if (list_length(ctx.rebuild_columnstore_chunk_oids) > 0) { /* there are chunks that need rebuilding */ ListCell *lc; foreach (lc, ctx.rebuild_columnstore_chunk_oids) { Oid chunk_relid = lfirst_oid(lc); (void) DirectFunctionCall1(ts_cm_functions->rebuild_columnstore, ObjectIdGetDatum(chunk_relid)); } } /* ACL permission checks inside vacuum_rel and analyze_rel called by this ExecVacuum */ ExecVacuum(args->parse_state, stmt, is_toplevel); } /* Restore original list. stmt->rels which has references to VacuumRelation list is freed up, however VacuumStmt is not cleaned up because of which there is a crash. */ stmt->rels = saved_stmt_rels; return DDL_DONE; } static void process_truncate_chunk(Hypertable *ht, Oid chunk_relid, void *arg) { TruncateStmt *stmt = arg; ObjectAddress objaddr = { .classId = RelationRelationId, .objectId = chunk_relid, }; performDeletion(&objaddr, stmt->behavior, 0); } static bool relation_should_recurse(RangeVar *rv) { return rv->inh; } /* handle forwarding TRUNCATEs to the chunks of a hypertable */ static void handle_truncate_hypertable(ProcessUtilityArgs *args, TruncateStmt *stmt, Hypertable *ht) { /* Delete the metadata */ ts_chunk_delete_by_hypertable_id(ht->fd.id); /* Drop the chunk tables */ foreach_chunk(ht, process_truncate_chunk, stmt); } /* * Truncate a hypertable. */ static DDLResult process_truncate(ProcessUtilityArgs *args) { TruncateStmt *stmt = (TruncateStmt *) args->parsetree; Cache *hcache = ts_hypertable_cache_pin(); ListCell *cell; List *hypertables = NIL, *mat_hypertables = NIL; List *relations = NIL; bool list_changed = false; MemoryContext oldctx, parsetreectx = GetMemoryChunkContext(args->parsetree); /* For all hypertables, we drop the now empty chunks. We also propagate the * TRUNCATE call to the compressed version of the hypertable, if it exists. */ foreach (cell, stmt->relations) { RangeVar *rv = lfirst(cell); Oid relid; bool list_append = false; if (!rv) continue; /* Grab AccessExclusiveLock, same as regular TRUNCATE processing grabs * below. We just do it preemptively here. */ relid = RangeVarGetRelid(rv, AccessExclusiveLock, true); if (!OidIsValid(relid)) { /* We should add invalid relations to the list to raise error on the * standard_ProcessUtility when we're trying to TRUNCATE a nonexistent relation */ list_append = true; } else { switch (get_rel_relkind(relid)) { case RELKIND_VIEW: { ContinuousAgg *cagg = ts_continuous_agg_find_by_relid(relid); if (cagg) { Hypertable *mat_ht, *raw_ht; if (!relation_should_recurse(rv)) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot truncate only a continuous aggregate"))); mat_ht = ts_hypertable_get_by_id(cagg->data.mat_hypertable_id); Assert(mat_ht != NULL); /* Create list item into the same context of the list */ oldctx = MemoryContextSwitchTo(parsetreectx); rv = makeRangeVar(NameStr(mat_ht->fd.schema_name), NameStr(mat_ht->fd.table_name), -1); MemoryContextSwitchTo(oldctx); /* Invalidate the entire continuous aggregate since it no * longer has any data */ raw_ht = ts_hypertable_get_by_id(cagg->data.raw_hypertable_id); Assert(raw_ht != NULL); ts_cm_functions->continuous_agg_invalidate_mat_ht(raw_ht, mat_ht, TS_TIME_NOBEGIN, TS_TIME_NOEND); /* Additionally, this cagg's materialization hypertable could be the * underlying hypertable for other caggs defined on top of it, in that case * we must update the hypertable invalidation log */ ContinuousAggHypertableStatus agg_status; agg_status = ts_continuous_agg_hypertable_status(mat_ht->fd.id); if (agg_status & HypertableIsRawTable) ts_cm_functions->continuous_agg_invalidate_raw_ht(mat_ht, TS_TIME_NOBEGIN, TS_TIME_NOEND); /* mark list as changed because we'll add the materialization hypertable */ list_changed = true; /* list of materialization hypertables to reset the watermark */ mat_hypertables = lappend(mat_hypertables, mat_ht); /* include the materialization hypertable to the list to be handled by the * proper hypertable and chunk truncate code-path later */ hypertables = lappend(hypertables, mat_ht); } list_append = true; break; } case RELKIND_RELATION: /* TRUNCATE for foreign tables not implemented yet. This will raise an error. */ case RELKIND_FOREIGN_TABLE: { list_append = true; Hypertable *ht = ts_hypertable_cache_get_entry(hcache, relid, CACHE_FLAG_MISSING_OK); if (ht) { ContinuousAggHypertableStatus agg_status; agg_status = ts_continuous_agg_hypertable_status(ht->fd.id); if ((agg_status & HypertableIsMaterialization) != 0) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot TRUNCATE a hypertable underlying a continuous " "aggregate"), errhint("TRUNCATE the continuous aggregate instead."))); if (agg_status == HypertableIsRawTable) { /* The truncation invalidates all associated continuous aggregates */ ts_cm_functions->continuous_agg_invalidate_raw_ht(ht, TS_TIME_NOBEGIN, TS_TIME_NOEND); } if (!relation_should_recurse(rv)) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("cannot truncate only a hypertable"), errhint("Do not specify the ONLY keyword, or use truncate" " only on the chunks directly."))); hypertables = lappend(hypertables, ht); break; } Chunk *chunk = ts_chunk_get_by_relid(relid, false); if (chunk != NULL) { /* this is a chunk */ ht = ts_hypertable_cache_get_entry(hcache, chunk->hypertable_relid, CACHE_FLAG_NONE); /* * Block direct TRUNCATE on frozen chunk. */ if (ts_chunk_is_frozen(chunk)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot TRUNCATE frozen chunk \"%s\"", get_rel_name(relid)), errhint("Unfreeze the chunk to TRUNCATE it."))); Assert(ht != NULL); /* If the hypertable has continuous aggregates, then invalidate * the truncated region. */ if (ts_continuous_agg_hypertable_status(ht->fd.id) == HypertableIsRawTable) ts_continuous_agg_invalidate_chunk(ht, chunk); /* Truncate the compressed chunk too */ if (chunk->fd.compressed_chunk_id != INVALID_CHUNK_ID) { Chunk *compressed_chunk = ts_chunk_get_by_id(chunk->fd.compressed_chunk_id, false); if (compressed_chunk != NULL) { /* Create list item into the same context of the list. */ oldctx = MemoryContextSwitchTo(parsetreectx); rv = makeRangeVar(NameStr(compressed_chunk->fd.schema_name), NameStr(compressed_chunk->fd.table_name), -1); MemoryContextSwitchTo(oldctx); list_changed = true; } } /* if the chunk has statistics enabled on it then reset them */ ts_chunk_column_stats_reset_by_chunk_id(chunk->fd.id); } break; } default: /* * Do nothing for other relation types. This is mostly to * placate the static analyzers. */ break; } } /* Append the relation to the list in the same parse tree memory context */ if (list_append) { MemoryContext oldctx = MemoryContextSwitchTo(parsetreectx); relations = lappend(relations, rv); MemoryContextSwitchTo(oldctx); } } /* Update relations list just when changed to include only tables * that hold data. */ if (list_changed) stmt->relations = relations; if (stmt->relations != NIL) { /* Call standard PostgreSQL handler for remaining tables */ prev_ProcessUtility(args); } /* For all hypertables, we drop the now empty chunks */ foreach (cell, hypertables) { Hypertable *ht = lfirst(cell); Assert(ht != NULL); handle_truncate_hypertable(args, stmt, ht); /* propagate to the compressed hypertable */ if (TS_HYPERTABLE_HAS_COMPRESSION_TABLE(ht)) { Hypertable *compressed_ht = ts_hypertable_cache_get_entry_by_id(hcache, ht->fd.compressed_hypertable_id); TruncateStmt compressed_stmt = *stmt; compressed_stmt.relations = list_make1(makeRangeVar(NameStr(compressed_ht->fd.schema_name), NameStr(compressed_ht->fd.table_name), -1)); /* TRUNCATE the compressed hypertable */ ExecuteTruncate(&compressed_stmt); handle_truncate_hypertable(args, stmt, compressed_ht); } } /* For all materialization hypertables, reset the watermark */ foreach (cell, mat_hypertables) { Hypertable *mat_ht = lfirst(cell); Assert(mat_ht != NULL); /* Force update the watermark */ bool isnull; int64 watermark = ts_hypertable_get_open_dim_max_value(mat_ht, 0, &isnull); ts_cagg_watermark_update(mat_ht, watermark, isnull, true); } ts_cache_release(&hcache); return DDL_DONE; } static void process_drop_table_chunk(Hypertable *ht, Oid chunk_relid, void *arg) { DropStmt *stmt = arg; ObjectAddress objaddr = { .classId = RelationRelationId, .objectId = chunk_relid, }; ts_compression_settings_delete(chunk_relid); ts_chunk_rewrite_delete(chunk_relid, false); performDeletion(&objaddr, stmt->behavior, 0); } /* Block drop compressed chunks directly and drop corresponding compressed chunks if * cascade is on. */ static void process_drop_chunk(ProcessUtilityArgs *args, DropStmt *stmt) { ListCell *lc; Cache *hcache = ts_hypertable_cache_pin(); foreach (lc, stmt->objects) { List *object = lfirst(lc); RangeVar *relation = makeRangeVarFromNameList(object); ScanTupLock slice_lock = { .lockmode = LockTupleExclusive, .waitpolicy = LockWaitBlock, .lockflags = TUPLE_LOCK_FLAG_FIND_LAST_VERSION, }; Chunk *chunk; if (NULL == relation) continue; chunk = ts_chunk_get_by_name_with_memory_context(relation->schemaname, relation->relname, AccessExclusiveLock, &slice_lock, CurrentMemoryContext, false); if (chunk != NULL) { Hypertable *ht; if (ts_chunk_contains_compressed_data(chunk)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("dropping columnstore chunks not supported"), errhint("Please drop the corresponding chunk on the rowstore hypertable " "instead."))); /* if cascade is enabled, delete the compressed chunk with cascade too. Otherwise * it would be blocked if there are dependent objects */ if (stmt->behavior == DROP_CASCADE && chunk->fd.compressed_chunk_id != INVALID_CHUNK_ID) { Chunk *compressed_chunk = ts_chunk_get_by_id_with_slice_lock(chunk->fd.compressed_chunk_id, AccessExclusiveLock, &slice_lock, false); /* The chunk may have been delete by a CASCADE */ if (compressed_chunk != NULL) ts_chunk_drop(compressed_chunk, stmt->behavior, DEBUG1); } ht = ts_hypertable_cache_get_entry(hcache, chunk->hypertable_relid, CACHE_FLAG_NONE); Assert(ht != NULL); /* If the hypertable has continuous aggregates, then invalidate * the dropped region. */ if (ts_continuous_agg_hypertable_status(ht->fd.id) == HypertableIsRawTable) ts_continuous_agg_invalidate_chunk(ht, chunk); } } ts_cache_release(&hcache); } /* * We need to drop hypertable chunks and associated compressed hypertables * when dropping hypertables to maintain correct semantics wrt CASCADE modifiers. * Also block dropping compressed hypertables directly. */ static DDLResult process_drop_hypertable(ProcessUtilityArgs *args, DropStmt *stmt) { Cache *hcache = ts_hypertable_cache_pin(); ListCell *lc; DDLResult result = DDL_CONTINUE; foreach (lc, stmt->objects) { List *object = lfirst(lc); RangeVar *relation = makeRangeVarFromNameList(object); Oid relid; if (NULL == relation) continue; relid = RangeVarGetRelid(relation, NoLock, true); if (OidIsValid(relid)) { Hypertable *ht; ht = ts_hypertable_cache_get_entry(hcache, relid, CACHE_FLAG_MISSING_OK); if (ht) { if (TS_HYPERTABLE_IS_INTERNAL_COMPRESSION_TABLE(ht)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("dropping columnstore hypertables not supported"), errhint("Please drop the corresponding rowstore hypertable " "instead."))); /* * We need to drop hypertable chunks before the hypertable to avoid the need * to CASCADE such drops; */ foreach_chunk(ht, process_drop_table_chunk, stmt); /* The usual path for deleting an associated compressed hypertable uses * DROP_RESTRICT But if we are using DROP_CASCADE we should propagate that down to * the compressed hypertable. */ if (stmt->behavior == DROP_CASCADE && TS_HYPERTABLE_HAS_COMPRESSION_TABLE(ht)) { Hypertable *compressed_hypertable = ts_hypertable_get_by_id(ht->fd.compressed_hypertable_id); List *chunks = ts_chunk_get_by_hypertable_id(ht->fd.compressed_hypertable_id); foreach (lc, chunks) { Chunk *chunk = lfirst(lc); if (OidIsValid(chunk->table_id)) { ObjectAddress chunk_addr = (ObjectAddress){ .classId = RelationRelationId, .objectId = chunk->table_id, }; /* Drop the postgres table */ performDeletion(&chunk_addr, stmt->behavior, 0); } } ts_hypertable_drop(compressed_hypertable, DROP_CASCADE); } } result = DDL_DONE; } } ts_cache_release(&hcache); return result; } /* * We need to ensure that DROP INDEX uses only one hypertable per query, * otherwise query string might not be reusable for execution on a * data node. */ static void process_drop_hypertable_index(ProcessUtilityArgs *args, DropStmt *stmt) { Cache *hcache = ts_hypertable_cache_pin(); ListCell *lc; foreach (lc, stmt->objects) { List *object = lfirst(lc); RangeVar *relation = makeRangeVarFromNameList(object); Oid ht_relid, index_relid; Hypertable *ht; if (NULL == relation) continue; index_relid = RangeVarGetRelid(relation, NoLock, true); if (!OidIsValid(index_relid)) continue; ht_relid = IndexGetRelation(index_relid, true); if (!OidIsValid(ht_relid)) continue; ht = ts_hypertable_cache_get_entry(hcache, ht_relid, CACHE_FLAG_MISSING_OK); if (ht) { List *chunk_indexes = ts_chunk_index_get_mappings(ht, index_relid); ListCell *lc_index; foreach (lc_index, chunk_indexes) { ChunkIndexMapping *mapping = lfirst(lc_index); Oid chunk_relid = mapping->indexoid; char *schema_name = get_namespace_name(get_rel_namespace(chunk_relid)); char *index_name = get_rel_name(chunk_relid); stmt->objects = lappend(stmt->objects, list_make2(makeString(schema_name), makeString(index_name))); } } } ts_cache_release(&hcache); } /* Note that DROP TABLESPACE does not have a hook in event triggers so cannot go * through process_ddl_sql_drop */ static DDLResult process_drop_tablespace(ProcessUtilityArgs *args) { DropTableSpaceStmt *stmt = (DropTableSpaceStmt *) args->parsetree; int count = ts_tablespace_count_attached(stmt->tablespacename); if (count > 0) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("tablespace \"%s\" is still attached to %d hypertables", stmt->tablespacename, count), errhint("Detach the tablespace from all hypertables before removing it."))); return DDL_CONTINUE; } static void process_grant_add_by_rel(GrantStmt *stmt, RangeVar *relation) { stmt->objects = lappend(stmt->objects, relation); } /* * If it is a "GRANT/REVOKE ON ALL TABLES IN SCHEMA" operation then we need to check if * the rangevar was already added when we added all objects inside the SCHEMA * * This could get a little expensive for schemas containing a lot of objects.. */ static void process_grant_add_by_name(GrantStmt *stmt, bool was_schema_op, Name schema_name, Name table_name) { bool already_added = false; if (was_schema_op) already_added = check_table_in_rangevar_list(stmt->objects, schema_name, table_name); if (!already_added) process_grant_add_by_rel(stmt, makeRangeVar(NameStr(*schema_name), NameStr(*table_name), -1)); } static void process_relations_in_namespace(GrantStmt *stmt, Name schema_name, Oid namespaceId, char relkind) { ScanKeyData key[2]; Relation rel; TableScanDesc scan; HeapTuple tuple; ScanKeyInit(&key[0], Anum_pg_class_relnamespace, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(namespaceId)); ScanKeyInit(&key[1], Anum_pg_class_relkind, BTEqualStrategyNumber, F_CHAREQ, CharGetDatum(relkind)); rel = table_open(RelationRelationId, AccessShareLock); scan = table_beginscan_catalog(rel, 2, key); while ((tuple = heap_getnext(scan, ForwardScanDirection)) != NULL) { Name relname = palloc(NAMEDATALEN); namestrcpy(relname, NameStr(((Form_pg_class) GETSTRUCT(tuple))->relname)); /* these are being added for the first time into this list */ process_grant_add_by_name(stmt, false, schema_name, relname); } table_endscan(scan); table_close(rel, AccessShareLock); } /* * For "GRANT ALL ON ALL TABLES IN SCHEMA" GrantStmt, the targtype field is ACL_TARGET_ALL_IN_SCHEMA * whereas in regular "GRANT ON TABLE table_name", the targtype field is ACL_TARGET_OBJECT. In the * latter case the objects list contains a list of relation range vars whereas in the former it is * the list of schema names. * * To make things work we change the targtype field from ACL_TARGET_ALL_IN_SCHEMA to * ACL_TARGET_OBJECT and then create a new list of rangevars of all relation type entities in it and * assign it to the "stmt->objects" field. * */ static void process_grant_add_by_schema(GrantStmt *stmt) { ListCell *cell; List *nspnames = stmt->objects; /* * We will be adding rangevars to the "stmt->objects" field in the loop below. So * we track the nspnames separately above and NIL out the objects list */ stmt->objects = NIL; foreach (cell, nspnames) { char *nspname = strVal(lfirst(cell)); Oid namespaceId = LookupExplicitNamespace(nspname, false); Name schema; schema = (Name) palloc(NAMEDATALEN); namestrcpy(schema, nspname); /* Inspired from objectsInSchemaToOids PG function */ process_relations_in_namespace(stmt, schema, namespaceId, RELKIND_RELATION); process_relations_in_namespace(stmt, schema, namespaceId, RELKIND_VIEW); process_relations_in_namespace(stmt, schema, namespaceId, RELKIND_MATVIEW); process_relations_in_namespace(stmt, schema, namespaceId, RELKIND_FOREIGN_TABLE); process_relations_in_namespace(stmt, schema, namespaceId, RELKIND_PARTITIONED_TABLE); } /* change targtype to ACL_TARGET_OBJECT now */ stmt->targtype = ACL_TARGET_OBJECT; } /* * Handle GRANT / REVOKE. * * A revoke is a GrantStmt with 'is_grant' set to false. */ static DDLResult process_grant_and_revoke(ProcessUtilityArgs *args) { GrantStmt *stmt = (GrantStmt *) args->parsetree; DDLResult result = DDL_CONTINUE; /* We let the calling function handle anything that is not * ACL_TARGET_OBJECT or ACL_TARGET_ALL_IN_SCHEMA */ if (stmt->targtype != ACL_TARGET_OBJECT && stmt->targtype != ACL_TARGET_ALL_IN_SCHEMA) return DDL_CONTINUE; switch (stmt->objtype) { case OBJECT_TABLESPACE: /* * If we are granting on a tablespace, we need to apply the REVOKE * first to be able to check remaining permissions. */ prev_ProcessUtility(args); ts_tablespace_validate_revoke(stmt); result = DDL_DONE; break; case OBJECT_TABLE: /* * Collect the hypertables in the grant statement. We only need to * consider those when sending grants to other data nodes. */ { Cache *hcache; ListCell *cell; List *saved_schema_objects = NIL; bool was_schema_op = false; /* * If it's a GRANT/REVOKE ALL IN SCHEMA then we need to collect all * objects in this schema and convert this into an ACL_TARGET_OBJECT * entry with its objects field pointing to rangevars */ if (stmt->targtype == ACL_TARGET_ALL_IN_SCHEMA) { saved_schema_objects = stmt->objects; process_grant_add_by_schema(stmt); was_schema_op = true; } hcache = ts_hypertable_cache_pin(); /* First process all continuous aggregates in the list and add * the associated hypertables and views to the list of objects * to process */ foreach (cell, stmt->objects) { RangeVar *relation = lfirst_node(RangeVar, cell); ContinuousAgg *const cagg = ts_continuous_agg_find_by_rv(relation); if (cagg) { Hypertable *mat_hypertable = ts_hypertable_get_by_id(cagg->data.mat_hypertable_id); process_grant_add_by_name(stmt, was_schema_op, &mat_hypertable->fd.schema_name, &mat_hypertable->fd.table_name); process_grant_add_by_name(stmt, was_schema_op, &cagg->data.direct_view_schema, &cagg->data.direct_view_name); process_grant_add_by_name(stmt, was_schema_op, &cagg->data.partial_view_schema, &cagg->data.partial_view_name); } /* * If this is a hypertable and it has a compressed * hypertable associated with it, add it to the list of * hypertables to process. */ Hypertable *hypertable = ts_hypertable_cache_get_entry_rv(hcache, relation); if (hypertable && TS_HYPERTABLE_HAS_COMPRESSION_TABLE(hypertable)) { Hypertable *compressed_hypertable = ts_hypertable_get_by_id(hypertable->fd.compressed_hypertable_id); Assert(compressed_hypertable); process_grant_add_by_name(stmt, was_schema_op, &compressed_hypertable->fd.schema_name, &compressed_hypertable->fd.table_name); List *chunks = ts_chunk_get_by_hypertable_id(hypertable->fd.compressed_hypertable_id); ListCell *cell; foreach (cell, chunks) { Chunk *chunk = lfirst(cell); process_grant_add_by_name(stmt, was_schema_op, &chunk->fd.schema_name, &chunk->fd.table_name); } } } /* Process all hypertables, including those added in the loop above */ foreach (cell, stmt->objects) { RangeVar *relation = lfirst_node(RangeVar, cell); Hypertable *ht = ts_hypertable_cache_get_entry_rv(hcache, relation); if (ht) { foreach_chunk(ht, add_chunk_oid, args); } } ts_cache_release(&hcache); result = DDL_DONE; if (stmt->objects != NIL) prev_ProcessUtility(args); /* Restore ALL IN SCHEMA command type and it's objects */ if (was_schema_op) { stmt->targtype = ACL_TARGET_ALL_IN_SCHEMA; stmt->objects = saved_schema_objects; } break; } default: break; } return result; } static DDLResult process_grant_and_revoke_role(ProcessUtilityArgs *args) { GrantRoleStmt *stmt = (GrantRoleStmt *) args->parsetree; /* * Need to apply the REVOKE first to be able to check remaining * permissions */ prev_ProcessUtility(args); /* We only care about revokes and setting privileges on a specific object */ if (stmt->is_grant) return DDL_DONE; ts_tablespace_validate_revoke_role(stmt); return DDL_DONE; } static void process_drop_view_start(ProcessUtilityArgs *args, DropStmt *stmt) { ListCell *cell; foreach (cell, stmt->objects) { List *const object = lfirst(cell); RangeVar *const rv = makeRangeVarFromNameList(object); ContinuousAgg *const cagg = ts_continuous_agg_find_by_rv(rv); if (cagg) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot drop continuous aggregate using DROP VIEW"), errhint("Use DROP MATERIALIZED VIEW to drop a continuous aggregate."))); } } static void process_drop_continuous_aggregates(ProcessUtilityArgs *args, DropStmt *stmt) { ListCell *lc; int caggs_count = 0; foreach (lc, stmt->objects) { List *const object = lfirst(lc); RangeVar *const rv = makeRangeVarFromNameList(object); ContinuousAgg *const cagg = ts_continuous_agg_find_by_rv(rv); if (cagg) { /* If there is at least one cagg, the drop should be treated as a * DROP VIEW. */ stmt->removeType = OBJECT_VIEW; ++caggs_count; } } /* We check that there were only continuous aggregates or that there were no continuous aggregates. Otherwise, we have a mixture of tables and views and are looking for views only.*/ if (caggs_count > 0 && caggs_count < list_length(stmt->objects)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("mixing continuous aggregates and other objects not allowed"), errhint("Drop continuous aggregates and other objects in separate statements."))); } static bool fetch_role_info(RoleSpec *rolespec, Oid *roleid) { /* Special role specifiers should not be present when dropping a role, * but if they are, we just ignore them */ if (rolespec->roletype != ROLESPEC_CSTRING) return false; /* Fetch the heap tuple from system table. If heaptuple is not valid it * means we did not find a role. We ignore it since the real execution * will handle this. */ HeapTuple tuple = SearchSysCache1(AUTHNAME, PointerGetDatum(rolespec->rolename)); if (!HeapTupleIsValid(tuple)) return false; Form_pg_authid roleform = (Form_pg_authid) GETSTRUCT(tuple); *roleid = roleform->oid; ReleaseSysCache(tuple); return true; } static DDLResult process_drop_role(ProcessUtilityArgs *args) { DropRoleStmt *stmt = (DropRoleStmt *) args->parsetree; ListCell *cell; foreach (cell, stmt->roles) { RoleSpec *rolespec = lfirst(cell); Oid roleid; if (!fetch_role_info(rolespec, &roleid)) continue; ScanIterator iterator = ts_scan_iterator_create(BGW_JOB, AccessShareLock, CurrentMemoryContext); ts_scanner_foreach(&iterator) { bool isnull; TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); Datum value = slot_getattr(ti->slot, Anum_bgw_job_owner, &isnull); if (!isnull && DatumGetObjectId(value) == roleid) { Datum value = slot_getattr(ti->slot, Anum_bgw_job_id, &isnull); Ensure(!isnull, "job id was null"); ereport(ERROR, (errcode(ERRCODE_DEPENDENT_OBJECTS_STILL_EXIST), errmsg("role \"%s\" cannot be dropped because some objects depend on it", rolespec->rolename), errdetail("owner of job %d", DatumGetInt32(value)))); } } } return DDL_CONTINUE; } static DDLResult process_drop_start(ProcessUtilityArgs *args) { DropStmt *stmt = (DropStmt *) args->parsetree; switch (stmt->removeType) { case OBJECT_TABLE: process_drop_hypertable(args, stmt); TS_FALLTHROUGH; case OBJECT_FOREIGN_TABLE: /* Chunks can be either normal tables, or foreign tables in the case of a tiered * hypertable */ process_drop_chunk(args, stmt); break; case OBJECT_INDEX: process_drop_hypertable_index(args, stmt); break; case OBJECT_MATVIEW: process_drop_continuous_aggregates(args, stmt); break; case OBJECT_VIEW: process_drop_view_start(args, stmt); break; case OBJECT_PROCEDURE: case OBJECT_FUNCTION: process_drop_procedure_start(stmt); break; case OBJECT_SCHEMA: process_drop_schema_start(stmt); break; default: break; } return DDL_CONTINUE; } static void reindex_chunk(Hypertable *ht, Oid chunk_relid, void *arg) { ProcessUtilityArgs *args = arg; ReindexStmt *stmt = (ReindexStmt *) args->parsetree; Chunk *chunk = ts_chunk_get_by_relid(chunk_relid, true); switch (stmt->kind) { case REINDEX_OBJECT_TABLE: stmt->relation->relname = NameStr(chunk->fd.table_name); stmt->relation->schemaname = NameStr(chunk->fd.schema_name); ExecReindex(NULL, stmt, false); break; case REINDEX_OBJECT_INDEX: /* Not supported, a.t.m. See note in process_reindex(). */ break; default: break; } } /* * Reindex a hypertable and all its chunks. Currently works only for REINDEX * TABLE. */ static DDLResult process_reindex(ProcessUtilityArgs *args) { ReindexStmt *stmt = (ReindexStmt *) args->parsetree; Oid relid; Cache *hcache; Hypertable *ht; DDLResult result = DDL_CONTINUE; if (NULL == stmt->relation) /* Not a case we are interested in */ return DDL_CONTINUE; relid = RangeVarGetRelid(stmt->relation, NoLock, true); if (!OidIsValid(relid)) return DDL_CONTINUE; hcache = ts_hypertable_cache_pin(); switch (stmt->kind) { case REINDEX_OBJECT_TABLE: ht = ts_hypertable_cache_get_entry(hcache, relid, CACHE_FLAG_MISSING_OK); if (NULL != ht) { PreventCommandDuringRecovery("REINDEX"); ts_hypertable_permissions_check_by_id(ht->fd.id); if (get_reindex_options(stmt) & REINDEXOPT_CONCURRENTLY) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("concurrent index creation on hypertables is not supported"))); if (foreach_chunk(ht, reindex_chunk, args) >= 0) result = DDL_DONE; } break; case REINDEX_OBJECT_INDEX: ht = ts_hypertable_cache_get_entry(hcache, IndexGetRelation(relid, true), CACHE_FLAG_MISSING_OK); if (NULL != ht) { ts_hypertable_permissions_check_by_id(ht->fd.id); /* * Recursing to chunks is currently not supported. Need to * look up all chunk indexes that corresponds to the * hypertable's index. */ ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("reindexing of a specific index on a hypertable is unsupported"), errhint( "As a workaround, it is possible to run REINDEX TABLE to reindex all " "indexes on a hypertable, including all indexes on chunks."))); } break; default: break; } ts_cache_release(&hcache); return result; } static void process_rename_view(Oid relid, RenameStmt *stmt) { char *schema = get_namespace_name(get_rel_namespace(relid)); char *name = get_rel_name(relid); ts_continuous_agg_rename_view(schema, name, schema, stmt->newname, &stmt->renameType); } /* * Rename a hypertable, chunk or continuous aggregate. */ static void process_rename_table(ProcessUtilityArgs *args, Cache *hcache, Oid relid, RenameStmt *stmt) { Hypertable *ht = ts_hypertable_cache_get_entry(hcache, relid, CACHE_FLAG_MISSING_OK); if (NULL == ht) { ContinuousAgg *cagg = ts_continuous_agg_find_by_relid(relid); if (cagg) { stmt->renameType = OBJECT_MATVIEW; process_rename_view(relid, stmt); return; } Chunk *chunk = ts_chunk_get_by_relid(relid, false); if (NULL != chunk) ts_chunk_set_name(chunk, stmt->newname); } else { ts_hypertable_set_name(ht, stmt->newname); } } static DDLResult process_rename_column(ProcessUtilityArgs *args, Cache *hcache, Oid relid, RenameStmt *stmt) { Hypertable *ht = ts_hypertable_cache_get_entry(hcache, relid, CACHE_FLAG_MISSING_OK); Dimension *dim; bool is_cagg = false; if (!ht) { Chunk *chunk = ts_chunk_get_by_relid(relid, false); if (chunk) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot rename column \"%s\" of hypertable chunk \"%s\"", stmt->subname, get_rel_name(relid)), errhint("Rename the hypertable column instead."))); /* This was not a hypertable and not a chunk, but it could be a * continuous aggregate. * * If this is a continuous aggregate, the rename should be done on the * materialized table. Since the partial view and the direct view are * not referencing the materialized table, we need to handle it here, * and in addition, the dimension table contains the column name, we * need to update the name there. */ ContinuousAgg *cagg = ts_continuous_agg_find_by_relid(relid); if (cagg) { is_cagg = true; RenameStmt *direct_view_stmt = castNode(RenameStmt, copyObject(stmt)); direct_view_stmt->relation = makeRangeVar(NameStr(cagg->data.direct_view_schema), NameStr(cagg->data.direct_view_name), -1); ExecRenameStmt(direct_view_stmt); RenameStmt *partial_view_stmt = castNode(RenameStmt, copyObject(stmt)); partial_view_stmt->relation = makeRangeVar(NameStr(cagg->data.partial_view_schema), NameStr(cagg->data.partial_view_name), -1); ExecRenameStmt(partial_view_stmt); /* Fetch the main table and it's relid and use that for the * processing below. This is necessary to rebuild the view based * on the table with the renamed columns. */ ht = ts_hypertable_get_by_id(cagg->data.mat_hypertable_id); relid = ht->main_table_relid; RenameStmt *mat_hypertable_stmt = castNode(RenameStmt, copyObject(stmt)); mat_hypertable_stmt->relation = makeRangeVar(NameStr(ht->fd.schema_name), NameStr(ht->fd.table_name), -1); ExecRenameStmt(mat_hypertable_stmt); /* * Also rename the user view column now so that * cagg_rename_view_columns() can update stored query trees for * all views (including the user view) in a single pass. We * return DDL_DONE below to skip PostgreSQL's standard rename * which would otherwise fail on the already-renamed column. * We need CommandCounterIncrement here so that * cagg_rename_view_columns() sees the new column names when it * opens the relations and reads their TupleDescs. */ ExecRenameStmt(stmt); CommandCounterIncrement(); } } else { /* Block renaming columns on the materialization table of a continuous * agg, but only if this was an explicit request for rename on a * materialization table. */ if ((ts_continuous_agg_hypertable_status(ht->fd.id) & HypertableIsMaterialization) != 0) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("renaming columns on materialization tables is not supported"), errdetail("Column \"%s\" in materialization table \"%s\".", stmt->subname, get_rel_name(relid)), errhint("Rename the column on the continuous aggregate instead."))); } /* * If there were a hypertable or a continuous aggregate, we need to rename * the dimension that we used as well as rebuilding the view. Otherwise, * we don't do anything. * * If it's not a dimension then we also need to check if column statistics * have been enabled on this column. * */ if (ht) { /* The column rename needs to be processed before the compression settings updated * because the composite bloom filter renaming need to have the old column names * and this comes from the compression settings. */ if (ts_cm_functions->process_rename_cmd) ts_cm_functions->process_rename_cmd(relid, hcache, stmt); /* The compression settings update can only proceed after the columns are renamed */ ts_compression_settings_rename_column_cascade(ht->main_table_relid, stmt->subname, stmt->newname); dim = ts_hyperspace_get_mutable_dimension_by_name(ht->space, DIMENSION_TYPE_ANY, stmt->subname); if (dim) ts_dimension_set_name(dim, stmt->newname); else { Form_chunk_column_stats form = ts_chunk_column_stats_lookup(ht->fd.id, INVALID_CHUNK_ID, stmt->subname); if (form != NULL) { ts_chunk_column_stats_set_name(form, stmt->newname); /* refresh the ht entry to accommodate this rename */ if (ht->range_space) pfree(ht->range_space); ht->range_space = ts_chunk_column_stats_range_space_scan(ht->fd.id, ht->main_table_relid, ts_cache_memory_ctx(hcache)); } } } /* * For continuous aggregates we renamed the user view column above via * ExecRenameStmt, so tell the caller to skip PostgreSQL's standard * rename which would fail on the already-renamed column. */ return is_cagg ? DDL_DONE : DDL_CONTINUE; } static void process_rename_index(ProcessUtilityArgs *args, Cache *hcache, Oid relid, RenameStmt *stmt) { Oid tablerelid = IndexGetRelation(relid, true); Hypertable *ht; if (!OidIsValid(tablerelid)) return; ht = ts_hypertable_cache_get_entry(hcache, tablerelid, CACHE_FLAG_MISSING_OK); if (ht) { ts_chunk_index_rename(ht, relid, stmt->newname); } } /* Visit all internal catalog tables with a schema column to check for applicable rename */ static void process_rename_schema(RenameStmt *stmt) { int i = 0; /* Block any renames of our internal schemas */ for (i = 0; i < NUM_TIMESCALEDB_SCHEMAS; i++) { if (strncmp(stmt->subname, ts_extension_schema_names[i], NAMEDATALEN) == 0) { ereport(ERROR, (errcode(ERRCODE_TS_OPERATION_NOT_SUPPORTED), errmsg("cannot rename schemas used by the TimescaleDB extension"))); return; } } ts_bgw_job_rename_schema_name(stmt->subname, stmt->newname); ts_chunks_rename_schema_name(stmt->subname, stmt->newname); ts_dimensions_rename_schema_name(stmt->subname, stmt->newname); ts_hypertables_rename_schema_name(stmt->subname, stmt->newname); ts_continuous_agg_rename_schema_name(stmt->subname, stmt->newname); } static void process_rename_procedure(ProcessUtilityArgs *args) { RenameStmt *stmt = (RenameStmt *) args->parsetree; Relation relation; ObjectAddress address = get_object_address(stmt->renameType, stmt->object, &relation, AccessExclusiveLock, false); ts_bgw_job_rename_proc(address, NULL, stmt->newname); } static void rename_hypertable_constraint(Hypertable *ht, Oid chunk_relid, void *arg) { RenameStmt *stmt = (RenameStmt *) arg; Chunk *chunk = ts_chunk_get_by_relid(chunk_relid, true); ts_chunk_constraint_rename_hypertable_constraint(chunk->fd.id, stmt->subname, stmt->newname); } static void alter_hypertable_constraint(Hypertable *ht, Oid chunk_relid, void *arg) { AlterTableCmd *cmd = (AlterTableCmd *) arg; char *hypertable_constraint_name; #if PG18_LT Constraint *cmd_constraint; Assert(IsA(cmd->def, Constraint)); cmd_constraint = (Constraint *) cmd->def; #else /* PG18 adds ATAlterConstraint struct which is used * instead of Constraint struct * * https://github.com/postgres/postgres/commit/80d7f990 */ ATAlterConstraint *cmd_constraint; Assert(IsA(cmd->def, ATAlterConstraint)); cmd_constraint = (ATAlterConstraint *) cmd->def; #endif hypertable_constraint_name = cmd_constraint->conname; cmd_constraint->conname = ts_chunk_constraint_get_name_from_hypertable_constraint(chunk_relid, hypertable_constraint_name); AlterTableInternal(chunk_relid, list_make1(cmd), false); /* Restore for next iteration */ cmd_constraint->conname = hypertable_constraint_name; } static void validate_hypertable_constraint(Hypertable *ht, Oid chunk_relid, void *arg) { AlterTableCmd *cmd = (AlterTableCmd *) arg; AlterTableCmd *chunk_cmd = copyObject(cmd); chunk_cmd->name = ts_chunk_constraint_get_name_from_hypertable_constraint(chunk_relid, cmd->name); if (chunk_cmd->name == NULL) return; /* do not pass down the VALIDATE RECURSE subtype */ chunk_cmd->subtype = AT_ValidateConstraint; AlterTableInternal(chunk_relid, list_make1(chunk_cmd), false); } static void rename_hypertable_trigger(Hypertable *ht, Oid chunk_relid, void *arg) { RenameStmt *stmt = copyObject(castNode(RenameStmt, arg)); Chunk *chunk = ts_chunk_get_by_relid(chunk_relid, true); stmt->relation = makeRangeVar(NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name), 0); renametrig(stmt); } static void process_rename_constraint_or_trigger(ProcessUtilityArgs *args, Cache *hcache, Oid relid, RenameStmt *stmt) { Hypertable *ht; ht = ts_hypertable_cache_get_entry(hcache, relid, CACHE_FLAG_MISSING_OK); Assert(stmt->relation != NULL); if (NULL != ht) { relation_not_only(stmt->relation); if (stmt->renameType == OBJECT_TABCONSTRAINT) foreach_chunk(ht, rename_hypertable_constraint, stmt); else if (stmt->renameType == OBJECT_TRIGGER) foreach_chunk(ht, rename_hypertable_trigger, stmt); } else if (stmt->renameType == OBJECT_TABCONSTRAINT) { Chunk *chunk = ts_chunk_get_by_relid(relid, false); if (NULL != chunk) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("renaming constraints on chunks is not supported"))); } } static DDLResult process_rename(ProcessUtilityArgs *args) { RenameStmt *stmt = (RenameStmt *) args->parsetree; Oid relid = InvalidOid; Cache *hcache; /* Only get the relid if it exists for this stmt */ if (NULL != stmt->relation) { relid = RangeVarGetRelid(stmt->relation, NoLock, true); if (!OidIsValid(relid)) return DDL_CONTINUE; } hcache = ts_hypertable_cache_pin(); DDLResult result = DDL_CONTINUE; switch (stmt->renameType) { case OBJECT_TABLE: process_rename_table(args, hcache, relid, stmt); break; case OBJECT_COLUMN: result = process_rename_column(args, hcache, relid, stmt); break; case OBJECT_INDEX: process_rename_index(args, hcache, relid, stmt); break; case OBJECT_TABCONSTRAINT: case OBJECT_TRIGGER: process_rename_constraint_or_trigger(args, hcache, relid, stmt); break; case OBJECT_MATVIEW: case OBJECT_VIEW: process_rename_view(relid, stmt); break; case OBJECT_SCHEMA: process_rename_schema(stmt); break; case OBJECT_PROCEDURE: case OBJECT_FUNCTION: process_rename_procedure(args); break; default: break; } ts_cache_release(&hcache); return result; } static void process_altertable_change_owner_chunk(Hypertable *ht, Oid chunk_relid, void *arg) { AlterTableCmd *cmd = arg; Oid roleid = get_rolespec_oid(cmd->newowner, false); ATExecChangeOwner(chunk_relid, roleid, false, AccessExclusiveLock); } static void process_altertable_change_owner_bgw_jobs(int32 hypertable_id, Oid newrole_oid) { ScanIterator iterator = ts_scan_iterator_create(BGW_JOB, RowExclusiveLock, CurrentMemoryContext); ts_scanner_foreach(&iterator) { bool should_free, isnull; TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); Datum htid = slot_getattr(ti->slot, Anum_bgw_job_hypertable_id, &isnull); if (!isnull && DatumGetInt32(htid) == hypertable_id) { HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); ts_bgw_job_update_owner(ti->scanrel, tuple, ts_scanner_get_tupledesc(ti), newrole_oid); if (should_free) heap_freetuple(tuple); } } } static void process_altertable_change_owner(Hypertable *ht, AlterTableCmd *cmd) { Oid newrole_oid = get_rolespec_oid(cmd->newowner, false); Assert(IsA(cmd->newowner, RoleSpec)); process_altertable_change_owner_bgw_jobs(ht->fd.id, newrole_oid); foreach_chunk(ht, process_altertable_change_owner_chunk, cmd); if (TS_HYPERTABLE_HAS_COMPRESSION_TABLE(ht)) { Hypertable *compressed_hypertable = ts_hypertable_get_by_id(ht->fd.compressed_hypertable_id); AlterTableInternal(compressed_hypertable->main_table_relid, list_make1(cmd), false); ListCell *lc; List *chunks = ts_chunk_get_by_hypertable_id(ht->fd.compressed_hypertable_id); foreach (lc, chunks) { Chunk *chunk = lfirst(lc); AlterTableInternal(chunk->table_id, list_make1(cmd), false); } process_altertable_change_owner(compressed_hypertable, cmd); } } typedef struct ChunkConstraintInfo { const AlterTableCmd *cmd; const char *constraint_name; Oid hypertable_constraint_oid; } ChunkConstraintInfo; /* * Unique constraints are validated by postgres during creation * but the postgres process does not cover data present in compressed * chunks or data split between compressed and uncompressed chunks. * When adding unique constraints to chunks with compressed data we * have to check for constraint violation ourself. */ static void validate_index_constraints(Chunk *chunk, const IndexStmt *stmt) { if ((stmt->primary || stmt->unique) && ts_chunk_is_compressed(chunk)) { StringInfoData command; Oid nspcid = get_rel_namespace(chunk->table_id); ListCell *lc; List *dpcontext = deparse_context_for(get_rel_name(chunk->table_id), chunk->table_id); initStringInfo(&command); appendStringInfo(&command, "SELECT EXISTS(SELECT FROM %s.%s", quote_identifier(get_namespace_name(nspcid)), quote_identifier(get_rel_name(chunk->table_id))); /* * Before PG15 NULLs were always considered distinct, with * PG15 the behaviour became configurable. */ if (!stmt->nulls_not_distinct) { int i = 0; appendStringInfo(&command, " WHERE "); foreach (lc, stmt->indexParams) { i++; IndexElem *elem = lfirst_node(IndexElem, lc); appendStringInfo(&command, "%s IS NOT NULL", elem->name ? quote_identifier(elem->name) : deparse_expression(elem->expr, dpcontext, false, false)); if (i < list_length(stmt->indexParams)) appendStringInfo(&command, " AND "); } Assert(i > 0); } appendStringInfo(&command, " GROUP BY "); int j = 0; foreach (lc, stmt->indexParams) { j++; IndexElem *elem = lfirst_node(IndexElem, lc); appendStringInfo(&command, "%s", elem->name ? quote_identifier(elem->name) : deparse_expression(elem->expr, dpcontext, false, false)); if (j < list_length(stmt->indexParams)) appendStringInfo(&command, ","); } Assert(j > 0); appendStringInfo(&command, " HAVING count(*) > 1"); appendStringInfo(&command, ")"); if (SPI_connect() != SPI_OK_CONNECT) elog(ERROR, "could not connect to SPI"); /* Lock down search_path */ int save_nestlevel = NewGUCNestLevel(); RestrictSearchPath(); int res = SPI_execute(command.data, true /* read_only */, 0 /*count*/); if (res < 0) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), (errmsg("could not verify unique constraint on \"%s\"", get_rel_name(chunk->table_id))))); bool isnull; Datum has_conflicts = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull); Assert(!isnull); if (isnull || DatumGetBool(has_conflicts)) ereport(ERROR, (errcode(ERRCODE_UNIQUE_VIOLATION), (errmsg("duplicate key value violates unique constraint")))); /* Restore search_path */ AtEOXact_GUC(false, save_nestlevel); res = SPI_finish(); if (res != SPI_OK_FINISH) elog(ERROR, "SPI_finish failed: %s", SPI_result_code_string(res)); } } static void validate_check_constraint(Chunk *chunk, Constraint *con) { if (ts_chunk_is_compressed(chunk)) { StringInfoData command; Oid nspcid = get_rel_namespace(chunk->table_id); ParseState *pstate = make_parsestate(NULL); Relation rel = table_open(chunk->table_id, AccessExclusiveLock); ParseNamespaceItem *nsitem = addRangeTableEntryForRelation(pstate, rel, AccessShareLock, NULL, false, true); addNSItemToQuery(pstate, nsitem, true, true, true); List *context = deparse_context_for(get_rel_name(chunk->table_id), chunk->table_id); Node *tf = transformExpr(pstate, con->raw_expr, EXPR_KIND_CHECK_CONSTRAINT); char *deparsed = deparse_expression(tf, context, false, false); initStringInfo(&command); appendStringInfo(&command, "SELECT EXISTS(SELECT FROM %s.%s WHERE NOT (%s))", quote_identifier(get_namespace_name(nspcid)), quote_identifier(RelationGetRelationName(rel)), deparsed); if (SPI_connect() != SPI_OK_CONNECT) elog(ERROR, "could not connect to SPI"); /* Lock down search_path */ int save_nestlevel = NewGUCNestLevel(); RestrictSearchPath(); int res = SPI_execute(command.data, true /* read_only */, 0 /*count*/); if (res < 0) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), (errmsg("could not verify check constraint on \"%s\"", get_rel_name(chunk->table_id))))); bool isnull; Datum has_conflicts = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull); Assert(!isnull); if (isnull || DatumGetBool(has_conflicts)) ereport(ERROR, (errcode(ERRCODE_CHECK_VIOLATION), errmsg("check constraint \"%s\" of relation \"%s\" is violated by some row", con->conname, RelationGetRelationName(rel)), errtableconstraint(rel, con->conname))); table_close(rel, NoLock); /* Restore search_path */ AtEOXact_GUC(false, save_nestlevel); res = SPI_finish(); if (res != SPI_OK_FINISH) elog(ERROR, "SPI_finish failed: %s", SPI_result_code_string(res)); } } static void process_add_constraint_chunk(Hypertable *ht, Oid chunk_relid, void *arg) { const ChunkConstraintInfo *info = arg; Chunk *chunk = ts_chunk_get_by_relid(chunk_relid, true); switch (info->cmd->subtype) { case AT_AddIndex: if (ts_chunk_is_compressed(chunk)) validate_index_constraints(chunk, castNode(IndexStmt, info->cmd->def)); break; case AT_AddConstraint: #if PG16_LT case AT_AddConstraintRecurse: #endif { Constraint *con = castNode(Constraint, info->cmd->def); switch (con->contype) { /* * Unique and primary key constraints are checked as part of * creation of the index enforcing it so nothing to do here. */ case CONSTR_UNIQUE: case CONSTR_PRIMARY: /* * Foreign key constraints are checked by postgres since * the check happens through SPI and we adjust those queries * to include compressed data. */ case CONSTR_FOREIGN: break; #if PG18_GE /* NULL and NOT NULL constraints have been added to * pg_constraints in PG18, we can safely ignore them at end * just like at beginning. * * https://github.com/postgres/postgres/commit/b0e96f31 */ case CONSTR_NULL: case CONSTR_NOTNULL: break; #endif case CONSTR_CHECK: { validate_check_constraint(chunk, con); break; } default: if (ts_chunk_is_compressed(chunk)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "operation not supported on hypertables that have columnstore " "data"), errhint("Convert the data to rowstore before retrying the " "operation."))); } break; /* Other AT commands might not be allowed on compressed chunks, but * they are checked at hypertable level in that case */ } default: break; } ts_chunk_constraint_create_on_chunk(ht, chunk, info->hypertable_constraint_oid); } static void process_altertable_add_constraint(Hypertable *ht, const AlterTableCmd *cmd, const char *constraint_name) { ChunkConstraintInfo info = { .cmd = cmd, .constraint_name = constraint_name, .hypertable_constraint_oid = get_relation_constraint_oid(ht->main_table_relid, constraint_name, false), }; foreach_chunk(ht, process_add_constraint_chunk, &info); } static void process_altertable_alter_constraint_end(Hypertable *ht, AlterTableCmd *cmd) { foreach_chunk(ht, alter_hypertable_constraint, cmd); } static void process_altertable_validate_constraint_end(Hypertable *ht, AlterTableCmd *cmd) { foreach_chunk(ht, validate_hypertable_constraint, cmd); } /* * Validate that SET NOT NULL is ok for this chunk. */ static void validate_set_not_null(Hypertable *ht, Oid chunk_relid, void *arg) { Chunk *chunk = ts_chunk_get_by_relid(chunk_relid, true); if (ts_chunk_is_compressed(chunk)) { StringInfoData command; AlterTableCmd *cmd = (AlterTableCmd *) arg; const CompressionSettings *settings = ts_compression_settings_get(chunk->table_id); Oid nspcid = get_rel_namespace(settings->fd.compress_relid); initStringInfo(&command); appendStringInfo(&command, "SELECT EXISTS(SELECT FROM %s.%s WHERE ", quote_identifier(get_namespace_name(nspcid)), quote_identifier(get_rel_name(settings->fd.compress_relid))); if (ts_array_is_member(settings->fd.segmentby, cmd->name)) { /* For segmentby we can check directly whether NULLS are present */ appendStringInfo(&command, "%s IS NULL", quote_identifier(cmd->name)); } else { /* For other columns we need to check whether we have a DEFAULT in which * case NULL as column value would be fine. * */ HeapTuple atttuple = SearchSysCacheAttName(ht->main_table_relid, cmd->name); Form_pg_attribute attform = ((Form_pg_attribute) GETSTRUCT(atttuple)); if (attform->atthasdef) appendStringInfo(&command, "%s IS NOT NULL AND " "_timescaledb_functions.compressed_data_has_nulls(%s)", quote_identifier(cmd->name), quote_identifier(cmd->name)); else appendStringInfo(&command, "%s IS NULL OR " "_timescaledb_functions.compressed_data_has_nulls(%s)", quote_identifier(cmd->name), quote_identifier(cmd->name)); ReleaseSysCache(atttuple); } appendStringInfo(&command, ")"); if (SPI_connect() != SPI_OK_CONNECT) elog(ERROR, "could not connect to SPI"); int res = SPI_execute(command.data, true /* read_only */, 0 /*count*/); if (res < 0) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), (errmsg("could not verify presence of NULL values on \"%s\"", get_rel_name(chunk_relid))))); bool isnull; Datum has_nulls = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull); if (isnull || DatumGetBool(has_nulls)) ereport(ERROR, (errcode(ERRCODE_NOT_NULL_VIOLATION), (errmsg("column \"%s\" of relation \"%s\" contains null values", cmd->name, get_rel_name(chunk_relid))))); res = SPI_finish(); if (res != SPI_OK_FINISH) elog(ERROR, "SPI_finish failed: %s", SPI_result_code_string(res)); } } /* * This function checks that we are not dropping NOT NULL from partitioning columns and * that no NULL data is present when adding NOT NULL constraint. */ static void process_altertable_alter_not_null(Hypertable *ht, AlterTableCmd *cmd) { if (cmd->subtype == AT_SetNotNull) foreach_chunk(ht, validate_set_not_null, cmd); if (cmd->subtype == AT_DropNotNull) { for (int i = 0; i < ht->space->num_dimensions; i++) { Dimension *dim = &ht->space->dimensions[i]; if (IS_OPEN_DIMENSION(dim) && strncmp(NameStr(dim->fd.column_name), cmd->name, NAMEDATALEN) == 0) ereport(ERROR, (errcode(ERRCODE_TS_OPERATION_NOT_SUPPORTED), errmsg("cannot drop not-null constraint from a time-partitioned column"))); } } } static void process_altertable_drop_column(Hypertable *ht, AlterTableCmd *cmd) { int i; bool dropped; for (i = 0; i < ht->space->num_dimensions; i++) { Dimension *dim = &ht->space->dimensions[i]; if (namestrcmp(&dim->fd.column_name, cmd->name) == 0) ereport(ERROR, (errcode(ERRCODE_INVALID_TABLE_DEFINITION), errmsg("cannot drop column named in partition key"), errdetail("Cannot drop column that is a hypertable partitioning (space or " "time) dimension."))); } /* Delete dimension range entries on this column, if any. */ ts_chunk_column_stats_drop(ht, cmd->name, &dropped); } /* * Verify that a constraint is supported on a hypertable. */ static void verify_constraint_hypertable(Hypertable *ht, Node *constr_node) { ConstrType contype; const char *indexname; List *keys; if (IsA(constr_node, Constraint)) { Constraint *constr = (Constraint *) constr_node; contype = constr->contype; keys = (contype == CONSTR_EXCLUSION) ? constr->exclusions : constr->keys; indexname = constr->indexname; if (contype == CONSTR_FOREIGN) { Oid confrelid = ts_hypertable_relid(constr->pktable); if (OidIsValid(confrelid) && !is_partitioning_allowed(ht->main_table_relid)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("hypertables cannot be used as foreign key references of " "hypertables"))); } /* NO INHERIT constraints do not really make sense on a hypertable */ if (constr->is_no_inherit) ereport(ERROR, (errcode(ERRCODE_INVALID_TABLE_DEFINITION), errmsg("cannot have NO INHERIT constraints on hypertable \"%s\"", get_rel_name(ht->main_table_relid)))); } else if (IsA(constr_node, IndexStmt)) { IndexStmt *stmt = (IndexStmt *) constr_node; contype = stmt->primary ? CONSTR_PRIMARY : CONSTR_UNIQUE; keys = stmt->indexParams; indexname = stmt->idxname; } else { elog(ERROR, "unexpected constraint type"); return; } switch (contype) { case CONSTR_FOREIGN: break; case CONSTR_UNIQUE: case CONSTR_PRIMARY: /* * If this constraints is created using an existing index we need * not re-verify it's columns */ if (indexname != NULL) return; ts_indexing_verify_columns(ht->space, keys); break; case CONSTR_EXCLUSION: ts_indexing_verify_columns(ht->space, keys); break; default: break; } } static void verify_constraint(RangeVar *relation, Constraint *constr) { Cache *hcache = ts_hypertable_cache_pin(); Hypertable *ht = ts_hypertable_cache_get_entry_rv(hcache, relation); if (ht) verify_constraint_hypertable(ht, (Node *) constr); ts_cache_release(&hcache); } static void verify_constraint_list(RangeVar *relation, List *constraint_list) { ListCell *lc; foreach (lc, constraint_list) { Constraint *constraint = lfirst(lc); verify_constraint(relation, constraint); } } typedef struct HypertableIndexOptions { /* * true if we should run one transaction per chunk, otherwise use one * transaction for all the chunks */ bool multitransaction; int n_ht_atts; /* Concurrency testing options. */ #ifdef DEBUG /* * If barrier_table is a valid Oid we try to acquire a lock on it at the * start of each chunks sub-transaction. */ Oid barrier_table; /* * if max_chunks >= 0 we'll create indices on at most max_chunks, and * leave the table marked as Invalid when the command ends. */ int32 max_chunks; #endif } HypertableIndexOptions; typedef struct CreateIndexInfo { IndexStmt *stmt; ObjectAddress obj; Oid main_table_relid; HypertableIndexOptions extended_options; MemoryContext mctx; } CreateIndexInfo; /* * Create index on a chunk. * * A chunk index is created based on the original IndexStmt that created the * "parent" index on the hypertable. */ static void process_index_chunk(Hypertable *ht, Oid chunk_relid, void *arg) { CreateIndexInfo *info = (CreateIndexInfo *) arg; Relation hypertable_index_rel; Relation chunk_rel; IndexInfo *indexinfo; Chunk *chunk; chunk = ts_chunk_get_by_relid(chunk_relid, true); if (IS_OSM_CHUNK(chunk)) /*cannot create index on foreign OSM chunk */ { ereport(NOTICE, (errmsg("skipping index creation for tiered data"))); return; } validate_index_constraints(chunk, info->stmt); chunk_rel = table_open(chunk_relid, ShareLock); hypertable_index_rel = index_open(info->obj.objectId, AccessShareLock); indexinfo = BuildIndexInfo(hypertable_index_rel); if (chunk_index_columns_changed(info->extended_options.n_ht_atts, RelationGetDescr(chunk_rel))) ts_adjust_indexinfo_attnos(indexinfo, info->main_table_relid, chunk_rel); ts_chunk_index_create_from_adjusted_index_info(ht->fd.id, hypertable_index_rel, chunk->fd.id, chunk_rel, indexinfo); index_close(hypertable_index_rel, NoLock); table_close(chunk_rel, NoLock); } static void process_index_chunk_multitransaction(int32 hypertable_id, Oid chunk_relid, void *arg) { CreateIndexInfo *info = (CreateIndexInfo *) arg; CatalogSecurityContext sec_ctx; Chunk *chunk; Relation hypertable_index_rel; Relation chunk_rel; IndexInfo *indexinfo; Assert(info->extended_options.multitransaction); /* Start a new transaction for each relation. */ StartTransactionCommand(); PushActiveSnapshot(GetTransactionSnapshot()); #ifdef DEBUG if (info->extended_options.max_chunks == 0) { PopActiveSnapshot(); CommitTransactionCommand(); return; } /* * if max_chunks is < 0 then we're indexing all the chunks, if it's >= 0 * then we're only indexing some of the chunks, and leaving the root index * marked as invalid */ if (info->extended_options.max_chunks > 0) info->extended_options.max_chunks -= 1; if (OidIsValid(info->extended_options.barrier_table)) { /* * For isolation tests, and debugging, it's useful to be able to * pause CREATE INDEX immediately before it starts working on chunks. * We acquire and immediately release a lock on a barrier table to do * this. */ Relation barrier = relation_open(info->extended_options.barrier_table, AccessExclusiveLock); relation_close(barrier, AccessExclusiveLock); } #endif /* * Change user since chunks are typically located in an internal schema * and chunk indexes require metadata changes. In the single-transaction * case, we do this once for the entire table. */ ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); /* * Hold a lock on the hypertable index, and the chunk to prevent * from being altered. Since we use the same relids across transactions, * there is a potential issue if the id gets reassigned between one * sub-transaction and the next. CLUSTER has a similar issue. * * We grab a ShareLock on the chunk, because that's what CREATE INDEX * does. For the hypertable's index, we are ok using the weaker * AccessShareLock, since we only need to prevent the index itself from * being ALTERed or DROPped during this part of index creation. */ chunk_rel = table_open(chunk_relid, ShareLock); chunk = ts_chunk_get_by_relid(chunk_relid, true); /* * Validation happens when creating the hypertable's index, which goes * through the usual DefineIndex mechanism. */ if (!IS_OSM_CHUNK(chunk)) /*cannot create index on foreign OSM chunk */ { hypertable_index_rel = index_open(info->obj.objectId, AccessShareLock); indexinfo = BuildIndexInfo(hypertable_index_rel); if (chunk_index_columns_changed(info->extended_options.n_ht_atts, RelationGetDescr(chunk_rel))) ts_adjust_indexinfo_attnos(indexinfo, info->main_table_relid, chunk_rel); ts_chunk_index_create_from_adjusted_index_info(hypertable_id, hypertable_index_rel, chunk->fd.id, chunk_rel, indexinfo); index_close(hypertable_index_rel, NoLock); } else { ereport(NOTICE, (errmsg("skipping index creation for tiered data"))); } validate_index_constraints(chunk, info->stmt); table_close(chunk_rel, NoLock); ts_catalog_restore_user(&sec_ctx); PopActiveSnapshot(); CommitTransactionCommand(); } typedef enum HypertableIndexFlags { HypertableIndexFlagMultiTransaction = 0, #ifdef DEBUG HypertableIndexFlagBarrierTable, HypertableIndexFlagMaxChunks, #endif } HypertableIndexFlags; static const WithClauseDefinition index_with_clauses[] = { [HypertableIndexFlagMultiTransaction] = {.arg_names = {"transaction_per_chunk", NULL}, .type_id = BOOLOID,}, #ifdef DEBUG [HypertableIndexFlagBarrierTable] = {.arg_names = {"barrier_table", NULL}, .type_id = REGCLASSOID,}, [HypertableIndexFlagMaxChunks] = {.arg_names = {"max_chunks", NULL}, .type_id = INT4OID, .default_val = (Datum)-1}, #endif }; static bool multitransaction_create_index_mark_valid(CreateIndexInfo info) { #ifdef DEBUG return info.extended_options.max_chunks < 0; #else return true; #endif } /* * Create an index on a hypertable * * We override CREATE INDEX on hypertables in order to ensure that the index is * created on all of the hypertable's chunks, and to ensure that locks on all * of said chunks are acquired at the correct time. */ static DDLResult process_index_start(ProcessUtilityArgs *args) { IndexStmt *stmt = (IndexStmt *) args->parsetree; Cache *hcache; Hypertable *ht; List *postgres_options = NIL; List *hypertable_options = NIL; WithClauseResult *parsed_with_clauses; CreateIndexInfo info = { .stmt = stmt, #ifdef DEBUG .extended_options = {0, .max_chunks = -1,}, #endif }; ObjectAddress root_table_index; Relation main_table_relation; TupleDesc main_table_desc; Relation main_table_index_relation; LockRelId main_table_index_lock_relid; int sec_ctx; Oid uid = InvalidOid, saved_uid = InvalidOid; ContinuousAgg *cagg = NULL; Assert(IsA(stmt, IndexStmt)); /* * PG11 adds some cases where the relation is not there, namely on * declaratively partitioned tables, with partitioned indexes: * https://github.com/postgres/postgres/commit/8b08f7d4820fd7a8ef6152a9dd8c6e3cb01e5f99 * we don't deal with them so we will just return immediately */ if (NULL == stmt->relation) return DDL_CONTINUE; hcache = ts_hypertable_cache_pin(); ht = ts_hypertable_cache_get_entry_rv(hcache, stmt->relation); if (!ht) { /* Check if the relation is a Continuous Aggregate */ cagg = ts_continuous_agg_find_by_rv(stmt->relation); if (cagg) { ht = ts_hypertable_get_by_id(cagg->data.mat_hypertable_id); } if (!ht) { ts_cache_release(&hcache); return DDL_CONTINUE; } if (stmt->unique) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("continuous aggregates do not support UNIQUE indexes"))); /* Make the RangeVar for the underlying materialization hypertable */ stmt->relation = makeRangeVar(NameStr(ht->fd.schema_name), NameStr(ht->fd.table_name), -1); } ts_hypertable_permissions_check_by_id(ht->fd.id); ts_with_clause_filter(stmt->options, &hypertable_options, NULL, &postgres_options); stmt->options = postgres_options; parsed_with_clauses = ts_with_clauses_parse(hypertable_options, index_with_clauses, TS_ARRAY_LEN(index_with_clauses)); info.extended_options.multitransaction = DatumGetBool(parsed_with_clauses[HypertableIndexFlagMultiTransaction].parsed); #ifdef DEBUG info.extended_options.max_chunks = DatumGetInt32(parsed_with_clauses[HypertableIndexFlagMaxChunks].parsed); info.extended_options.barrier_table = DatumGetObjectId(parsed_with_clauses[HypertableIndexFlagBarrierTable].parsed); #endif /* Make sure this index is allowed */ if (stmt->concurrent) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("hypertables do not support concurrent " "index creation"))); if (info.extended_options.multitransaction && (stmt->unique || stmt->primary || stmt->isconstraint)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "cannot use timescaledb.transaction_per_chunk with UNIQUE or PRIMARY KEY"))); ts_indexing_verify_index(ht->space, stmt); if (info.extended_options.multitransaction) PreventInTransactionBlock(true, "CREATE INDEX ... WITH (timescaledb.transaction_per_chunk)"); if (cagg) { /* * If this is an index creation for cagg, then we need to switch user as the current * user might not have permissions on the internal schema where cagg index will be * created. * Need to restore user soon after this step. */ ts_cagg_permissions_check(ht->main_table_relid, GetUserId()); SWITCH_TO_TS_USER(NameStr(cagg->data.direct_view_schema), uid, saved_uid, sec_ctx); } /* CREATE INDEX on the root table of the hypertable */ root_table_index = ts_indexing_root_table_create_index(stmt, args->query_string, info.extended_options.multitransaction); if (cagg) RESTORE_USER(uid, saved_uid, sec_ctx); /* root_table_index will have 0 objectId if the index already exists * and if_not_exists is true. In that case there is nothing else * to do here. */ if (!OidIsValid(root_table_index.objectId) && stmt->if_not_exists) { ts_cache_release(&hcache); return DDL_DONE; } Assert(OidIsValid(root_table_index.objectId)); /* support ONLY ON clause, index on root table already created */ if (!stmt->relation->inh) { ts_cache_release(&hcache); return DDL_DONE; } info.obj.objectId = root_table_index.objectId; /* collect information required for per chunk index creation */ main_table_relation = table_open(ht->main_table_relid, AccessShareLock); main_table_desc = RelationGetDescr(main_table_relation); main_table_index_relation = index_open(info.obj.objectId, AccessShareLock); main_table_index_lock_relid = main_table_index_relation->rd_lockInfo.lockRelId; info.extended_options.n_ht_atts = main_table_desc->natts; info.main_table_relid = ht->main_table_relid; index_close(main_table_index_relation, NoLock); table_close(main_table_relation, NoLock); /* create chunk indexes using the same transaction for all the chunks */ if (!info.extended_options.multitransaction) { CatalogSecurityContext sec_ctx; /* * Change user since chunk's are typically located in an internal * schema and chunk indexes require metadata changes. In the * multi-transaction case, we do this once per chunk. */ ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); /* Recurse to each chunk and create a corresponding index. */ foreach_chunk(ht, process_index_chunk, &info); ts_catalog_restore_user(&sec_ctx); ts_cache_release(&hcache); return DDL_DONE; } /* create chunk indexes using a separate transaction for each chunk */ /* * Lock the index for the remainder of the command. Since we're using * multiple transactions for index creation, a regular * transaction-level lock won't prevent the index from being * concurrently ALTERed or DELETEd. Instead, we grab a session level * lock on the index, which we'll release when the command is * finished. (This is the same strategy postgres uses in CREATE INDEX * CONCURRENTLY) */ LockRelationIdForSession(&main_table_index_lock_relid, AccessShareLock); /* * mark the hypertable's index as invalid until all the chunk indexes * are created. This allows us to determine if the CREATE INDEX * completed successfully or not */ ts_indexing_mark_as_invalid(info.obj.objectId); CacheInvalidateRelcacheByRelid(info.main_table_relid); CacheInvalidateRelcacheByRelid(info.obj.objectId); ts_cache_release(&hcache); /* we need a long-lived context in which to store the list of chunks since the per-transaction * context will get freed at the end of each transaction. Fortunately we're within just such a * context now; the PortalContext. */ info.mctx = CurrentMemoryContext; PopActiveSnapshot(); CommitTransactionCommand(); foreach_chunk_multitransaction(info.main_table_relid, info.mctx, process_index_chunk_multitransaction, &info); StartTransactionCommand(); PushActiveSnapshot(GetTransactionSnapshot()); MemoryContextSwitchTo(info.mctx); if (multitransaction_create_index_mark_valid(info)) { /* we're done, the index is now valid */ ts_indexing_mark_as_valid(info.obj.objectId); CacheInvalidateRelcacheByRelid(info.main_table_relid); CacheInvalidateRelcacheByRelid(info.obj.objectId); } PopActiveSnapshot(); CommitTransactionCommand(); StartTransactionCommand(); UnlockRelationIdForSession(&main_table_index_lock_relid, AccessShareLock); DEBUG_WAITPOINT("process_index_start_indexing_done"); return DDL_DONE; } static int chunk_index_mappings_cmp(const void *p1, const void *p2) { const ChunkIndexMapping *lhs = *((ChunkIndexMapping *const *) p1); const ChunkIndexMapping *rhs = *((ChunkIndexMapping *const *) p2); if (lhs->chunkoid < rhs->chunkoid) return -1; if (lhs->chunkoid > rhs->chunkoid) return 1; return 0; } /* * Cluster a hypertable. * * The functionality to cluster all chunks of a hypertable is based on the * regular cluster function's mode to cluster multiple tables. Since clustering * involves taking exclusive locks on all tables for extensive periods of time, * each subtable is clustered in its own transaction. This will release all * locks on subtables once they are done. */ static DDLResult process_cluster_start(ProcessUtilityArgs *args) { ClusterStmt *stmt = (ClusterStmt *) args->parsetree; Cache *hcache; Hypertable *ht; DDLResult result = DDL_CONTINUE; Assert(IsA(stmt, ClusterStmt)); /* If this is a re-cluster on all tables, there is nothing we need to do */ if (NULL == stmt->relation) return DDL_CONTINUE; hcache = ts_hypertable_cache_pin(); ht = ts_hypertable_cache_get_entry_rv(hcache, stmt->relation); if (NULL != ht) { bool is_top_level = (args->context == PROCESS_UTILITY_TOPLEVEL); Oid index_relid; Relation index_rel; List *chunk_indexes; ListCell *lc; MemoryContext old, mcxt; LockRelId cluster_index_lockid; ChunkIndexMapping **mappings = NULL; int i; ts_hypertable_permissions_check_by_id(ht->fd.id); /* * If CLUSTER is run inside a user transaction block; we bail out or * otherwise we'd be holding locks way too long. */ PreventInTransactionBlock(is_top_level, "CLUSTER"); if (NULL == stmt->indexname) { index_relid = ts_indexing_find_clustered_index(ht->main_table_relid); if (!OidIsValid(index_relid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("there is no previously clustered index for table \"%s\"", get_rel_name(ht->main_table_relid)))); } else index_relid = get_relname_relid(stmt->indexname, get_rel_namespace(ht->main_table_relid)); if (!OidIsValid(index_relid)) { /* Let regular process utility handle */ ts_cache_release(&hcache); return DDL_CONTINUE; } /* * DROP INDEX locks the table then the index, to prevent deadlocks we * lock them in the same order. The main table lock will be released * when the current transaction commits, and never taken again. We * will use the index relation to grab a session lock on the index, * which we will hold throughout CLUSTER */ LockRelationOid(ht->main_table_relid, AccessShareLock); index_rel = index_open(index_relid, AccessShareLock); cluster_index_lockid = index_rel->rd_lockInfo.lockRelId; index_close(index_rel, NoLock); /* * mark the main table as clustered, even though it has no data, so * future calls to CLUSTER don't need to pass in the index */ ts_chunk_index_mark_clustered(ht->main_table_relid, index_relid); /* we will keep holding this lock throughout CLUSTER */ LockRelationIdForSession(&cluster_index_lockid, AccessShareLock); /* * The list of chunks and their indexes need to be on a memory context * that will survive moving to a new transaction for each chunk */ mcxt = AllocSetContextCreate(PortalContext, "Hypertable cluster", ALLOCSET_DEFAULT_SIZES); /* * Get a list of chunks and indexes that correspond to the * hypertable's index */ old = MemoryContextSwitchTo(mcxt); chunk_indexes = ts_chunk_index_get_mappings(ht, index_relid); if (list_length(chunk_indexes) > 0) { /* Sort the mappings on chunk OID. This makes the verbose output more * predictable in tests, but isn't strictly necessary. We could also do * it only for "verbose" output, but this doesn't seem worth it as the * cost of sorting is quickly amortized over the actual work to cluster * the chunks. */ mappings = (ChunkIndexMapping **) palloc(sizeof(ChunkIndexMapping *) * list_length(chunk_indexes)); i = 0; foreach (lc, chunk_indexes) mappings[i++] = lfirst(lc); qsort((void *) mappings, list_length(chunk_indexes), sizeof(ChunkIndexMapping *), chunk_index_mappings_cmp); } MemoryContextSwitchTo(old); hcache->release_on_commit = false; /* Commit to get out of starting transaction */ PopActiveSnapshot(); CommitTransactionCommand(); for (i = 0; i < list_length(chunk_indexes); i++) { ChunkIndexMapping *cim = mappings[i]; /* Start a new transaction for each relation. */ StartTransactionCommand(); /* functions in indexes may want a snapshot set */ PushActiveSnapshot(GetTransactionSnapshot()); /* * We must mark each chunk index as clustered before calling * cluster_rel() because it expects indexes that need to be * rechecked (due to new transaction) to already have that mark * set */ ts_chunk_index_mark_clustered(cim->chunkoid, cim->indexoid); /* Do the job. */ /* * Since we keep OIDs between transactions, there is a potential * issue if an OID gets reassigned between two subtransactions */ #if PG18_LT cluster_rel(cim->chunkoid, cim->indexoid, get_cluster_options(stmt)); #else Relation rel = table_open(cim->chunkoid, AccessExclusiveLock); cluster_rel(rel, cim->indexoid, get_cluster_options(stmt)); #endif PopActiveSnapshot(); CommitTransactionCommand(); } hcache->release_on_commit = true; /* Start a new transaction for the cleanup work. */ StartTransactionCommand(); /* Clean up working storage */ MemoryContextDelete(mcxt); UnlockRelationIdForSession(&cluster_index_lockid, AccessShareLock); result = DDL_DONE; } ts_cache_release(&hcache); return result; } typedef struct CreateTableInfo { bool hypertable; WithClauseResult *with_clauses; } CreateTableInfo; static CreateTableInfo create_table_info = { 0 }; /* * Scan the table for a suitable default partitioning column. * * The default partitioning column is the first timestamp column * * Caller is expected to have appropriate lock on the table. */ static char * get_default_partition_column(Oid relid) { Relation rel; TupleDesc tupdesc; int i; char *column_name = NULL; rel = relation_open(relid, NoLock); tupdesc = RelationGetDescr(rel); for (i = 0; i < tupdesc->natts; i++) { Form_pg_attribute att = TupleDescAttr(tupdesc, i); if (att->attisdropped) continue; if (att->atttypid == TIMESTAMPOID || att->atttypid == TIMESTAMPTZOID) { column_name = pstrdup(NameStr(att->attname)); break; } } relation_close(rel, NoLock); return column_name; } /* * Process create table statements. * * NOTE that this function should be called after parse analysis (in an end DDL * trigger or by running parse analysis manually). */ static void process_create_table_end(Node *parsetree) { CreateStmt *stmt = (CreateStmt *) parsetree; ListCell *lc; verify_constraint_list(stmt->relation, stmt->constraints); /* * Only after parse analysis does tableElts contain only ColumnDefs. So, * if we capture this in processUtility, we should be prepared to have * constraint nodes and TableLikeClauses intermixed */ foreach (lc, stmt->tableElts) { ColumnDef *coldef; switch (nodeTag(lfirst(lc))) { case T_ColumnDef: coldef = lfirst(lc); verify_constraint_list(stmt->relation, coldef->constraints); break; case T_Constraint: /* * There should be no Constraints in the list after parse * analysis, but this case is included anyway for completeness */ verify_constraint(stmt->relation, lfirst(lc)); break; case T_TableLikeClause: /* Some as above case */ break; default: break; } } if (create_table_info.hypertable) { Oid table_relid = RangeVarGetRelid(stmt->relation, NoLock, true); char *time_column = NULL; if (stmt->partspec != NULL) { time_column = ((PartitionElem *) linitial(stmt->partspec->partParams))->name; } else if (create_table_info.with_clauses[CreateTableFlagTimeColumn].is_default) { time_column = get_default_partition_column(table_relid); if (time_column) ereport(NOTICE, (errmsg("using column \"%s\" as partitioning column", time_column), errhint("Use \"timescaledb.partition_column\" to specify a different " "column to use as " "partitioning column."))); else ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("partition column could not be determined"), errhint( "Use \"timescaledb.partition_column\" to specify the column to use as " "partitioning column."))); } else time_column = TextDatumGetCString( create_table_info.with_clauses[CreateTableFlagTimeColumn].parsed); NameData time_column_name; NameData associated_schema_name; NameData associated_table_prefix; namestrcpy(&time_column_name, time_column); uint32 flags = 0; bool has_associated_schema = false; bool has_associated_table_prefix = false; if (get_attnum(table_relid, time_column) == InvalidAttrNumber) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("column \"%s\" does not exist", time_column))); Oid interval_type = InvalidOid; Datum interval = UnassignedDatum; if (!create_table_info.with_clauses[CreateTableFlagChunkTimeInterval].is_default) { AttrNumber time_attno = get_attnum(table_relid, time_column); Oid time_type = get_atttype(table_relid, time_attno); interval = ts_create_table_parse_chunk_time_interval(create_table_info.with_clauses [CreateTableFlagChunkTimeInterval], time_type, &interval_type); } if (!create_table_info.with_clauses[CreateTableFlagCreateDefaultIndexes].is_default) { if (!DatumGetBool( create_table_info.with_clauses[CreateTableFlagCreateDefaultIndexes].parsed)) flags |= HYPERTABLE_CREATE_DISABLE_DEFAULT_INDEXES; } if (!create_table_info.with_clauses[CreateTableFlagAssociatedSchema].is_default) { has_associated_schema = true; namestrcpy(&associated_schema_name, TextDatumGetCString( create_table_info.with_clauses[CreateTableFlagAssociatedSchema].parsed)); } if (!create_table_info.with_clauses[CreateTableFlagAssociatedTablePrefix].is_default) { has_associated_table_prefix = true; namestrcpy(&associated_table_prefix, TextDatumGetCString( create_table_info.with_clauses[CreateTableFlagAssociatedTablePrefix] .parsed)); } DimensionInfo *open_dim_info = ts_dimension_info_create_open(table_relid, &time_column_name, /* column name */ interval, /* interval */ interval_type, /* interval type */ InvalidOid /* partitioning func */ ); ChunkSizingInfo *csi = ts_chunk_sizing_info_get_default_disabled(table_relid); csi->colname = time_column; CatalogSecurityContext sec_ctx; ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); int32 ht_id = ts_catalog_table_next_seq_id(ts_catalog_get(), HYPERTABLE); ts_catalog_restore_user(&sec_ctx); if (ts_hypertable_create_from_info(table_relid, ht_id, flags, /* flags */ open_dim_info, /* open_dim_info */ NULL, /* closed_dim_info */ has_associated_schema ? &associated_schema_name : NULL, /* associated_schema_name */ has_associated_table_prefix ? &associated_table_prefix : NULL, /* associated_table_prefix */ csi)) { bool enable_columnstore; if (ts_license_is_apache() && create_table_info.with_clauses[CreateTableFlagColumnstore].is_default) enable_columnstore = false; else enable_columnstore = DatumGetBool(create_table_info.with_clauses[CreateTableFlagColumnstore].parsed); if (enable_columnstore) { Hypertable *ht = ts_hypertable_get_by_id(ht_id); ts_cm_functions->columnstore_setup(ht, create_table_info.with_clauses); } } } } static inline const char * typename_get_unqual_name(TypeName *tn) { return strVal(llast(tn->names)); } static void process_alter_column_type_start(ParseState *pstate, Hypertable *ht, AlterTableCmd *cmd) { int i; /* check if it's a partitioning column */ for (i = 0; i < ht->space->num_dimensions; i++) { Dimension *dim = &ht->space->dimensions[i]; if (IS_CLOSED_DIMENSION(dim) && strncmp(NameStr(dim->fd.column_name), cmd->name, NAMEDATALEN) == 0) ereport(ERROR, (errcode(ERRCODE_TS_OPERATION_NOT_SUPPORTED), errmsg("cannot change the type of a hash-partitioned column"))); if (dim->partitioning != NULL && strncmp(NameStr(dim->fd.column_name), cmd->name, NAMEDATALEN) == 0) ereport(ERROR, (errcode(ERRCODE_TS_OPERATION_NOT_SUPPORTED), errmsg("cannot change the type of a column with a custom partitioning " "function"))); } /* * Check if column has statistics enabled on it, if yes we need to check that the * new type is a permissible type. */ Form_chunk_column_stats form = ts_chunk_column_stats_lookup(ht->fd.id, INVALID_CHUNK_ID, cmd->name); if (form != NULL) { ColumnDef *def = (ColumnDef *) cmd->def; TypeName *typeName = def->typeName; Oid newtypid = typenameTypeId(pstate, typeName); /* * We only support a subset of types for ranges right now. If it's a * supported type and ranges have been calculated then we should * still be able to use them for chunk exclusion. * * So, we only do a basic check for compatible new data type. Nothing * else needs to be done. */ switch (newtypid) { case INT2OID: case INT4OID: case INT8OID: case TIMESTAMPOID: case TIMESTAMPTZOID: case DATEOID: break; default: ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("data type \"%s\" unsupported for statistics calculation", format_type_be(newtypid)), errhint("Integer-like, timestamp-like data types supported currently." " Disable the stats using disable_column_stats function" " before changing the type"))); } } } static void process_alter_column_type_end(Hypertable *ht, AlterTableCmd *cmd) { ColumnDef *coldef = (ColumnDef *) cmd->def; Oid new_type = TypenameGetTypid(typename_get_unqual_name(coldef->typeName)); Dimension *dim = ts_hyperspace_get_mutable_dimension_by_name(ht->space, DIMENSION_TYPE_ANY, cmd->name); if (NULL == dim) return; ts_dimension_set_type(dim, new_type); ts_process_utility_set_expect_chunk_modification(true); ts_chunk_recreate_all_constraints_for_dimension(ht, dim->fd.id); ts_process_utility_set_expect_chunk_modification(false); } static void process_altertable_clusteron_end(Hypertable *ht, AlterTableCmd *cmd) { Oid index_relid = ts_get_relation_relid(NameStr(ht->fd.schema_name), cmd->name, true); /* If this is part of changing the type of a column that is used in a clustered index * the above lookup might fail. But in this case we don't need to mark the index clustered * as postgres takes care of that already */ if (!OidIsValid(index_relid)) return; List *chunk_indexes = ts_chunk_index_get_mappings(ht, index_relid); ListCell *lc; foreach (lc, chunk_indexes) { ChunkIndexMapping *cim = lfirst(lc); ts_chunk_index_mark_clustered(cim->chunkoid, cim->indexoid); } } /* * Generic function to recurse ALTER TABLE commands to chunks. * * Call with foreach_chunk(). */ static void process_altertable_chunk(Hypertable *ht, Oid chunk_relid, void *arg) { AlterTableCmd *cmd = arg; /* Don't propagate ALTER TABLE SET to foreign tables */ if (get_rel_relkind(chunk_relid) == RELKIND_FOREIGN_TABLE && (cmd->subtype == AT_SetOptions || cmd->subtype == AT_ResetOptions || cmd->subtype == AT_SetRelOptions || cmd->subtype == AT_ReplaceRelOptions || cmd->subtype == AT_ResetRelOptions)) return; AlterTableInternal(chunk_relid, list_make1(cmd), false); } static void process_altertable_chunk_replica_identity(Hypertable *ht, Oid chunk_relid, void *arg) { AlterTableCmd *cmd = castNode(AlterTableCmd, copyObject(arg)); ReplicaIdentityStmt *stmt = castNode(ReplicaIdentityStmt, cmd->def); char relkind = get_rel_relkind(chunk_relid); /* If this is not a local chunk (e.g., it is foreign table representing a * data node or OSM chunk), then we don't set replica identity locally */ if (relkind != RELKIND_RELATION) return; if (stmt->identity_type == REPLICA_IDENTITY_INDEX) { Chunk *chunk = ts_chunk_get_by_relid(chunk_relid, true); Oid hyper_schema_oid = get_rel_namespace(ht->main_table_relid); Oid hyper_index_oid = get_relname_relid(stmt->name, hyper_schema_oid); Assert(OidIsValid(hyper_index_oid)); Relation chunk_rel = table_open(chunk_relid, AccessExclusiveLock); Oid chunk_index_relid = ts_chunk_index_get_by_hypertable_indexrelid(chunk_rel, hyper_index_oid); table_close(chunk_rel, NoLock); if (!OidIsValid(chunk_index_relid)) elog(ERROR, "chunk \"%s.%s\" has no index corresponding to hypertable index \"%s\"", NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name), stmt->name); stmt->name = get_rel_name(chunk_index_relid); } AlterTableInternal(chunk_relid, list_make1(cmd), false); } static void process_altertable_replica_identity(Hypertable *ht, AlterTableCmd *cmd) { ReplicaIdentityStmt *stmt = castNode(ReplicaIdentityStmt, cmd->def); if (stmt->identity_type == REPLICA_IDENTITY_INDEX) { Oid hyper_schema_oid = get_rel_namespace(ht->main_table_relid); Oid hyper_index_oid; hyper_index_oid = get_relname_relid(stmt->name, hyper_schema_oid); if (!OidIsValid(hyper_index_oid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("index \"%s\" for table \"%s.%s\" does not exist", stmt->name, NameStr(ht->fd.schema_name), NameStr(ht->fd.table_name)))); } foreach_chunk(ht, process_altertable_chunk_replica_identity, cmd); } static void process_altertable_set_tablespace_end(Hypertable *ht, AlterTableCmd *cmd) { NameData tspc_name; Tablespaces *tspcs; namestrcpy(&tspc_name, cmd->name); tspcs = ts_tablespace_scan(ht->fd.id); if (tspcs->num_tablespaces > 1) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("cannot set new tablespace when multiple tablespaces are attached to " "hypertable \"%s\"", get_rel_name(ht->main_table_relid)), errhint("Detach tablespaces before altering the hypertable."))); if (tspcs->num_tablespaces == 1) { Assert(ts_hypertable_has_tablespace(ht, tspcs->tablespaces[0].tablespace_oid)); ts_tablespace_delete(ht->fd.id, NameStr(tspcs->tablespaces[0].fd.tablespace_name), tspcs->tablespaces[0].tablespace_oid); } ts_tablespace_attach_internal(&tspc_name, ht->main_table_relid, true); foreach_chunk(ht, process_altertable_chunk, cmd); if (TS_HYPERTABLE_HAS_COMPRESSION_TABLE(ht)) { Hypertable *compressed_hypertable = ts_hypertable_get_by_id(ht->fd.compressed_hypertable_id); AlterTableInternal(compressed_hypertable->main_table_relid, list_make1(cmd), false); List *chunks = ts_chunk_get_by_hypertable_id(ht->fd.compressed_hypertable_id); ListCell *lc; foreach (lc, chunks) { Chunk *chunk = lfirst(lc); AlterTableInternal(chunk->table_id, list_make1(cmd), false); } process_altertable_set_tablespace_end(compressed_hypertable, cmd); } } static void process_altertable_end_index(Node *parsetree, CollectedCommand *cmd) { AlterTableStmt *stmt = (AlterTableStmt *) parsetree; Oid indexrelid = AlterTableLookupRelation(stmt, NoLock); Oid tablerelid = IndexGetRelation(indexrelid, false); Cache *hcache; Hypertable *ht; if (!OidIsValid(tablerelid)) return; ht = ts_hypertable_cache_get_cache_and_entry(tablerelid, CACHE_FLAG_MISSING_OK, &hcache); if (NULL != ht) { ListCell *lc; foreach (lc, stmt->cmds) { AlterTableCmd *cmd = (AlterTableCmd *) lfirst(lc); switch (cmd->subtype) { case AT_SetTableSpace: ts_chunk_index_set_tablespace(ht, indexrelid, cmd->name); break; default: break; } } } ts_cache_release(&hcache); } static void process_altertable_chunk_propagate_to_compressed(AlterTableCmd *cmd, Oid relid) { Chunk *chunk = ts_chunk_get_by_relid(relid, false); if (chunk == NULL) return; if (ts_chunk_contains_compressed_data(chunk)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("changing tablespace of columnstore chunk is not supported"), errhint("Please use the corresponding chunk on the rowstore hypertable " "instead."))); /* set tablespace for compressed chunk */ if (chunk->fd.compressed_chunk_id != INVALID_CHUNK_ID) { Chunk *compressed_chunk = ts_chunk_get_by_id(chunk->fd.compressed_chunk_id, true); AlterTableInternal(compressed_chunk->table_id, list_make1(cmd), false); } } static DDLResult process_altertable_start_table(ProcessUtilityArgs *args) { AlterTableStmt *stmt = (AlterTableStmt *) args->parsetree; Oid reloid = AlterTableLookupRelation(stmt, NoLock); Cache *hcache; Hypertable *ht; ListCell *lc; if (!OidIsValid(reloid)) return DDL_CONTINUE; check_chunk_alter_table_operation_allowed(reloid, stmt); ht = ts_hypertable_cache_get_cache_and_entry(reloid, CACHE_FLAG_MISSING_OK, &hcache); if (ht != NULL) { ts_hypertable_permissions_check_by_id(ht->fd.id); check_continuous_agg_alter_table_allowed(ht, stmt); check_alter_table_allowed_on_ht_with_compression(ht, stmt); if (!stmt->relation->inh) { /* only allow ALTER TABLE ... SET (option) with ONLY */ foreach (lc, stmt->cmds) { AlterTableCmd *cmd = (AlterTableCmd *) lfirst(lc); switch (cmd->subtype) { case AT_SetRelOptions: case AT_ResetRelOptions: case AT_ReplaceRelOptions: case AT_SetOptions: case AT_ResetOptions: continue; default: relation_not_only(stmt->relation); break; } } } } foreach (lc, stmt->cmds) { AlterTableCmd *cmd = (AlterTableCmd *) lfirst(lc); switch (cmd->subtype) { case AT_AddIndex: { IndexStmt *istmt = (IndexStmt *) cmd->def; Assert(IsA(cmd->def, IndexStmt)); if (NULL != ht && istmt->isconstraint) verify_constraint_hypertable(ht, cmd->def); } break; case AT_SetNotNull: case AT_DropNotNull: if (ht) process_altertable_alter_not_null(ht, cmd); break; case AT_AddColumn: #if PG16_LT case AT_AddColumnRecurse: #endif { ColumnDef *col; ListCell *constraint_lc; Assert(IsA(cmd->def, ColumnDef)); col = (ColumnDef *) cmd->def; if (ht && TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(ht)) check_altertable_add_column_for_compressed(args->parse_state, ht, col); if (ht) foreach (constraint_lc, col->constraints) verify_constraint_hypertable(ht, lfirst(constraint_lc)); break; } case AT_DropColumn: #if PG16_LT case AT_DropColumnRecurse: #endif if (ht) process_altertable_drop_column(ht, cmd); break; case AT_AddConstraint: #if PG16_LT case AT_AddConstraintRecurse: #endif Assert(IsA(cmd->def, Constraint)); if (ht) verify_constraint_hypertable(ht, cmd->def); break; case AT_AlterColumnType: Assert(IsA(cmd->def, ColumnDef)); if (ht) process_alter_column_type_start(args->parse_state, ht, cmd); break; case AT_AttachPartition: { RangeVar *relation; PartitionCmd *partstmt; partstmt = (PartitionCmd *) cmd->def; relation = partstmt->name; Assert(relation); if (OidIsValid(ts_hypertable_relid(relation))) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("hypertables do not support native " "postgres partitioning"))); } break; } case AT_SetRelOptions: { if (ht != NULL) { EventTriggerAlterTableStart(args->parsetree); /* If we dealt with the option, we remove it from the * list. We do not set the result variable since there * could be other options that are not dealt with. */ if (process_altertable_set_options(cmd, ht) == DDL_DONE) stmt->cmds = foreach_delete_current(stmt->cmds, lc); } else { check_no_timescale_options(cmd, reloid); } break; } case AT_ResetRelOptions: case AT_ReplaceRelOptions: if (ht) { process_altertable_reset_options(cmd, ht); } else { check_no_timescale_options(cmd, reloid); } break; case AT_SetTableSpace: case AT_SetLogged: case AT_SetUnLogged: if (!ht) process_altertable_chunk_propagate_to_compressed(cmd, reloid); break; default: break; } } ts_cache_release(&hcache); /* If there are any commands remaining in the list, we need to deal with * them. Otherwise, we just skip the rest. */ return (list_length(stmt->cmds) > 0) ? DDL_CONTINUE : DDL_DONE; } static void continuous_agg_with_clause_perm_check(ContinuousAgg *cagg, Oid view_relid) { Oid ownerid = ts_rel_get_owner(view_relid); if (!has_privs_of_role(GetUserId(), ownerid)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("must be owner of continuous aggregate \"%s\"", get_rel_name(view_relid)))); } static List * process_altercontinuousagg_set_with(ContinuousAgg *cagg, Oid view_relid, const List *defelems) { WithClauseResult *parse_results; List *pg_options = NIL, *other_namespace_options = NIL, *cagg_options = NIL; continuous_agg_with_clause_perm_check(cagg, view_relid); ts_with_clause_filter(defelems, &cagg_options, &other_namespace_options, &pg_options); if (list_length(pg_options) > 0) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("only timescaledb parameters allowed in WITH clause for continuous " "aggregate"))); if (list_length(cagg_options) > 0) { parse_results = ts_create_materialized_view_with_clause_parse(cagg_options); ts_cm_functions->continuous_agg_update_options(cagg, parse_results); } if (list_length(other_namespace_options) > 0) { return other_namespace_options; } else return NIL; } /* Run an alter table command on a relation */ static void alter_table_by_relation(RangeVar *relation, AlterTableCmd *cmd) { const Oid relid = RangeVarGetRelid(relation, NoLock, true); AlterTableInternal(relid, list_make1(cmd), false); } /* Run an alter table command on a relation given by name */ static void alter_table_by_name(Name schema_name, Name table_name, AlterTableCmd *cmd) { alter_table_by_relation(makeRangeVar(NameStr(*schema_name), NameStr(*table_name), -1), cmd); } /* Alter a hypertable and do some extra processing */ static void alter_hypertable_by_id(int32 hypertable_id, AlterTableStmt *stmt, AlterTableCmd *cmd, void (*extra)(Hypertable *, AlterTableCmd *)) { Cache *hcache = ts_hypertable_cache_pin(); Hypertable *ht = ts_hypertable_cache_get_entry_by_id(hcache, hypertable_id); Assert(ht); /* Broken continuous aggregate */ ts_hypertable_permissions_check_by_id(ht->fd.id); check_alter_table_allowed_on_ht_with_compression(ht, stmt); relation_not_only(stmt->relation); AlterTableInternal(ht->main_table_relid, list_make1(cmd), false); (*extra)(ht, cmd); ts_cache_release(&hcache); } static DDLResult process_altertable_start_matview(ProcessUtilityArgs *args) { AlterTableStmt *stmt = (AlterTableStmt *) args->parsetree; const Oid view_relid = RangeVarGetRelid(stmt->relation, NoLock, true); ContinuousAgg *cagg; ListCell *lc; DDLResult ddl_res = DDL_DONE; if (!OidIsValid(view_relid)) return DDL_CONTINUE; cagg = ts_continuous_agg_find_by_relid(view_relid); if (cagg == NULL) return DDL_CONTINUE; continuous_agg_with_clause_perm_check(cagg, view_relid); foreach (lc, stmt->cmds) { AlterTableCmd *cmd = (AlterTableCmd *) lfirst(lc); switch (cmd->subtype) { case AT_SetRelOptions: if (!IsA(cmd->def, List)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("expected set options to contain a list"))); List *other_namespace_opt = process_altercontinuousagg_set_with(cagg, view_relid, (List *) cmd->def); /* pass on SET options to other extensions like timescaledb-lake. only if * there are additional PG related ones, we error out */ if (other_namespace_opt != NIL) { cmd->def = (Node *) other_namespace_opt; ddl_res = DDL_CONTINUE; } break; case AT_ChangeOwner: alter_table_by_relation(stmt->relation, cmd); alter_table_by_name(&cagg->data.partial_view_schema, &cagg->data.partial_view_name, cmd); alter_table_by_name(&cagg->data.direct_view_schema, &cagg->data.direct_view_name, cmd); alter_hypertable_by_id(cagg->data.mat_hypertable_id, stmt, cmd, process_altertable_change_owner); break; case AT_SetTableSpace: alter_hypertable_by_id(cagg->data.mat_hypertable_id, stmt, cmd, process_altertable_set_tablespace_end); break; default: ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot alter only SET options of a continuous " "aggregate"))); } } return ddl_res; } static DDLResult process_altertable_start_view(ProcessUtilityArgs *args) { AlterTableStmt *stmt = (AlterTableStmt *) args->parsetree; Oid relid = AlterTableLookupRelation(stmt, NoLock); ContinuousAgg *cagg; ContinuousAggViewType vtyp; const char *view_name; const char *view_schema; /* Check if this is a materialized view and give error if it is. */ cagg = ts_continuous_agg_find_by_relid(relid); if (cagg) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot alter continuous aggregate using ALTER VIEW"), errhint("Use ALTER MATERIALIZED VIEW to alter a continuous aggregate."))); /* Check if this is an internal view of a continuous aggregate and give * error if attempts are made to alter them. */ view_name = get_rel_name(relid); view_schema = get_namespace_name(get_rel_namespace(relid)); cagg = ts_continuous_agg_find_by_view_name(view_schema, view_name, ContinuousAggAnyView); if (cagg == NULL) return DDL_CONTINUE; vtyp = ts_continuous_agg_view_type(&cagg->data, view_schema, view_name); if (vtyp == ContinuousAggPartialView || vtyp == ContinuousAggDirectView) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot alter the internal view of a continuous aggregate"))); return DDL_DONE; } static DDLResult process_altertable_start(ProcessUtilityArgs *args) { AlterTableStmt *stmt = (AlterTableStmt *) args->parsetree; switch (stmt->objtype) { case OBJECT_TABLE: return process_altertable_start_table(args); case OBJECT_MATVIEW: return process_altertable_start_matview(args); case OBJECT_VIEW: return process_altertable_start_view(args); default: return DDL_CONTINUE; } } static void process_altertable_end_subcmd(Hypertable *ht, Node *parsetree, ObjectAddress *obj) { AlterTableCmd *cmd = (AlterTableCmd *) parsetree; Assert(IsA(parsetree, AlterTableCmd)); switch (cmd->subtype) { case AT_ChangeOwner: process_altertable_change_owner(ht, cmd); break; case AT_AddIndexConstraint: ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("hypertables do not support adding a constraint " "using an existing index"))); break; case AT_AddIndex: { IndexStmt *stmt = (IndexStmt *) cmd->def; const char *idxname = stmt->idxname; Assert(IsA(cmd->def, IndexStmt)); Assert(stmt->isconstraint); if (idxname == NULL) idxname = get_rel_name(obj->objectId); process_altertable_add_constraint(ht, cmd, idxname); } break; case AT_AddConstraint: #if PG16_LT case AT_AddConstraintRecurse: #endif { Constraint *stmt = (Constraint *) cmd->def; const char *conname = stmt->conname; Assert(IsA(cmd->def, Constraint)); if (conname == NULL) conname = get_rel_name(obj->objectId); /* * Implicit constraints (e.g., those created by PRIMARY KEY or UNIQUE * constraints) have already been processed when the index was created. * These will have no objectId in the ObjectAddress passed to this * function and no conname. */ if (conname) process_altertable_add_constraint(ht, cmd, conname); } break; case AT_AlterColumnType: Assert(IsA(cmd->def, ColumnDef)); process_alter_column_type_end(ht, cmd); break; case AT_EnableTrig: case AT_EnableAlwaysTrig: case AT_EnableReplicaTrig: case AT_DisableTrig: case AT_EnableTrigAll: case AT_DisableTrigAll: case AT_EnableTrigUser: case AT_DisableTrigUser: ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("hypertables do not support " "enabling or disabling triggers."))); /* Break here to silence compiler */ break; case AT_ClusterOn: process_altertable_clusteron_end(ht, cmd); break; case AT_ReplicaIdentity: process_altertable_replica_identity(ht, cmd); break; case AT_EnableRule: case AT_EnableAlwaysRule: case AT_EnableReplicaRule: case AT_DisableRule: /* should never actually get here but just in case */ ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("hypertables do not support rules"))); /* Break here to silence compiler */ break; case AT_AlterConstraint: process_altertable_alter_constraint_end(ht, cmd); break; case AT_ValidateConstraint: #if PG16_LT case AT_ValidateConstraintRecurse: #endif process_altertable_validate_constraint_end(ht, cmd); break; case AT_SetLogged: case AT_SetUnLogged: foreach_chunk(ht, process_altertable_chunk, cmd); foreach_compressed_chunk(ht, process_altertable_chunk, cmd); break; case AT_DropCluster: case AT_SetNotNull: case AT_DropNotNull: case AT_SetRelOptions: case AT_ResetRelOptions: case AT_ReplaceRelOptions: case AT_DropOids: case AT_SetOptions: case AT_ResetOptions: case AT_ReAddStatistics: case AT_SetCompression: foreach_chunk(ht, process_altertable_chunk, cmd); break; case AT_SetTableSpace: process_altertable_set_tablespace_end(ht, cmd); break; case AT_AddInherit: case AT_DropInherit: ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("hypertables do not support inheritance"))); case AT_SetStatistics: case AT_SetStorage: case AT_ColumnDefault: case AT_CookedColumnDefault: case AT_AddOf: case AT_DropOf: case AT_AddIdentity: case AT_SetIdentity: case AT_DropIdentity: /* all of the above are handled by default recursion */ break; case AT_EnableRowSecurity: case AT_DisableRowSecurity: case AT_ForceRowSecurity: case AT_NoForceRowSecurity: /* RLS commands should not recurse to chunks */ break; case AT_ReAddConstraint: case AT_ReAddIndex: /* * all of the above are internal commands that are hit in tests * and correctly handled */ break; case AT_AddColumn: #if PG16_LT case AT_AddColumnRecurse: #endif /* this is handled for compressed hypertables by tsl code */ break; case AT_DropColumn: #if PG16_LT case AT_DropColumnRecurse: #endif case AT_DropExpression: /* * adding and dropping columns handled in * process_altertable_start_table */ break; case AT_DropConstraint: #if PG16_LT case AT_DropConstraintRecurse: #endif /* drop constraints handled by process_ddl_sql_drop */ break; case AT_ReAddComment: /* internal command never hit in our test * code, so don't know how to handle */ case AT_AddColumnToView: /* only used with views */ case AT_AlterColumnGenericOptions: /* only used with foreign tables */ case AT_GenericOptions: /* only used with foreign tables */ case AT_ReAddDomainConstraint: /* We should handle this in future, * new subset of constraints in PG11 * currently not hit in test code */ case AT_AttachPartition: /* handled in * process_altertable_start_table but also * here as failsafe */ case AT_DetachPartition: case AT_DetachPartitionFinalize: ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("operation not supported on hypertables %d", cmd->subtype))); break; default: break; } if (ts_cm_functions->process_altertable_cmd) ts_cm_functions->process_altertable_cmd(ht, cmd); } static void process_altertable_end_subcmds(Hypertable *ht, List *cmds) { ListCell *lc; foreach (lc, cmds) { CollectedATSubcmd *cmd = lfirst(lc); process_altertable_end_subcmd(ht, cmd->parsetree, &cmd->address); } } static void process_altertable_end_table(Node *parsetree, CollectedCommand *cmd) { AlterTableStmt *stmt = (AlterTableStmt *) parsetree; Oid relid; Cache *hcache; Hypertable *ht; Assert(IsA(stmt, AlterTableStmt)); relid = RangeVarGetRelid(stmt->relation, NoLock, true); if (!OidIsValid(relid)) return; ht = ts_hypertable_cache_get_cache_and_entry(relid, CACHE_FLAG_MISSING_OK, &hcache); if (ht) { switch (cmd->type) { case SCT_Simple: process_altertable_end_subcmd(ht, linitial(stmt->cmds), &cmd->d.simple.secondaryObject); break; case SCT_AlterTable: process_altertable_end_subcmds(ht, cmd->d.alterTable.subcmds); break; default: break; } } /* * Check any ALTER TABLE command is adding a FOREIGN KEY constraint * referencing a hypertable. */ if (cmd->type == SCT_AlterTable) { AlterTableStmt *stmt = castNode(AlterTableStmt, parsetree); ListCell *lc; foreach (lc, stmt->cmds) { AlterTableCmd *subcmd = (AlterTableCmd *) lfirst(lc); if (subcmd->subtype != AT_AddConstraint || castNode(Constraint, subcmd->def)->contype != CONSTR_FOREIGN) continue; Constraint *c = castNode(Constraint, subcmd->def); Oid confrelid = RangeVarGetRelid(c->pktable, AccessShareLock, true); Hypertable *pk = ts_hypertable_cache_get_entry(hcache, confrelid, CACHE_FLAG_MISSING_OK); if (pk) { if (ht && !is_partitioning_allowed(ht->main_table_relid)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("hypertables cannot be used as foreign key references of " "hypertables"))); ts_fk_propagate(relid, pk); } } } ts_cache_release(&hcache); } static void process_altertable_end(Node *parsetree, CollectedCommand *cmd) { AlterTableStmt *stmt = (AlterTableStmt *) parsetree; switch (stmt->objtype) { case OBJECT_TABLE: process_altertable_end_table(parsetree, cmd); break; case OBJECT_INDEX: process_altertable_end_index(parsetree, cmd); break; default: break; } } static DDLResult process_create_trigger_start(ProcessUtilityArgs *args) { CreateTrigStmt *stmt = (CreateTrigStmt *) args->parsetree; Cache *hcache; Hypertable *ht; ObjectAddress PG_USED_FOR_ASSERTS_ONLY address; Oid relid = RangeVarGetRelid(stmt->relation, NoLock, true); int16 tgtype; TRIGGER_CLEAR_TYPE(tgtype); if (stmt->row) TRIGGER_SETT_ROW(tgtype); tgtype |= stmt->timing; tgtype |= stmt->events; hcache = ts_hypertable_cache_pin(); ht = ts_hypertable_cache_get_entry(hcache, relid, CACHE_FLAG_MISSING_OK); if (ht == NULL) { ts_cache_release(&hcache); /* check if it's a cagg. We don't support triggers on them yet */ if (ts_continuous_agg_find_by_relid(relid) != NULL) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("triggers are not supported on continuous aggregate"))); if (stmt->transitionRels) if (ts_chunk_get_by_relid(relid, false) != NULL) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("triggers with transition tables are not supported on " "hypertable chunks"))); return DDL_CONTINUE; } /* * We do not support ROW triggers with transition tables on hypertables * since these are not supported on inheritance children, and we use * inheritance for our chunks (it is actually not supported for * declarative partition tables either). */ if (stmt->transitionRels && TRIGGER_FOR_ROW(tgtype)) { ts_cache_release(&hcache); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("ROW triggers with transition tables are not supported on hypertables"))); } /* * We currently don't support delete triggers with transition tables on * compressed tables because deleting a complete segment will not build * a transition table for the delete. */ if (stmt->transitionRels && TRIGGER_FOR_DELETE(tgtype) && TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(ht)) { ts_cache_release(&hcache); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("DELETE triggers with transition tables not supported"))); } /* * If it is not a ROW trigger, we do not need to create the ROW triggers * on the chunks, so we can return early. */ if (!stmt->row) { ts_cache_release(&hcache); return DDL_CONTINUE; } address = ts_hypertable_create_trigger(ht, stmt, args->query_string); Assert(OidIsValid(address.objectId)); ts_cache_release(&hcache); return DDL_DONE; } static DDLResult process_create_rule_start(ProcessUtilityArgs *args) { RuleStmt *stmt = (RuleStmt *) args->parsetree; if (!OidIsValid(ts_hypertable_relid(stmt->relation))) return DDL_CONTINUE; ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("hypertables do not support rules"))); return DDL_CONTINUE; } /* * Update the owner of a background job given by a heap tuple. * * Note that there is no check for correct privileges here and this is the * responsibility of the caller. */ static void ts_bgw_job_update_owner(Relation rel, HeapTuple tuple, TupleDesc tupledesc, Oid newrole_oid) { bool isnull[Natts_bgw_job]; Datum values[Natts_bgw_job]; bool replace[Natts_bgw_job] = { false }; HeapTuple new_tuple; heap_deform_tuple(tuple, tupledesc, values, isnull); if (DatumGetObjectId(values[AttrNumberGetAttrOffset(Anum_bgw_job_owner)]) != newrole_oid) { values[AttrNumberGetAttrOffset(Anum_bgw_job_owner)] = Int32GetDatum(newrole_oid); replace[AttrNumberGetAttrOffset(Anum_bgw_job_owner)] = true; new_tuple = heap_modify_tuple(tuple, tupledesc, values, isnull, replace); ts_catalog_update(rel, new_tuple); heap_freetuple(new_tuple); } } static DDLResult process_reassign_owned_start(ProcessUtilityArgs *args) { ReassignOwnedStmt *stmt = (ReassignOwnedStmt *) args->parsetree; List *role_ids = roleSpecsToIds(stmt->roles); ScanIterator iterator = ts_scan_iterator_create(BGW_JOB, RowExclusiveLock, CurrentMemoryContext); ts_scanner_foreach(&iterator) { bool should_free, isnull; TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); Datum value = slot_getattr(ti->slot, Anum_bgw_job_owner, &isnull); if (!isnull && list_member_oid(role_ids, DatumGetObjectId(value))) { Oid newrole_oid = get_rolespec_oid(stmt->newrole, false); HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); /* We do not need to check privileges here since ReassignOwnedObjects() will check * the privileges and error out if they are not correct. */ ts_bgw_job_update_owner(ti->scanrel, tuple, ts_scanner_get_tupledesc(ti), newrole_oid); if (should_free) heap_freetuple(tuple); } } return DDL_CONTINUE; } static void check_no_timescale_options(AlterTableCmd *cmd, Oid reloid) { List *pg_options = NIL, *compress_options = NIL; Ensure(IsA(cmd->def, List), "wrong node type used as ALTER TABLE command definition"); List *inpdef = (List *) cmd->def; ts_with_clause_filter(inpdef, &compress_options, NULL, &pg_options); if (compress_options != NIL) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("timescaledb table options can only be specified for hypertables"), errdetail("%s is not a hypertable", get_rel_name(reloid)))); } } /* ALTER TABLE <name> SET ( timescaledb.compress, ...) */ static DDLResult process_altertable_set_options(AlterTableCmd *cmd, Hypertable *ht) { List *pg_options = NIL, *tsdb_options = NIL; WithClauseResult *parse_results = NULL; /* split postgres and timescaledb options */ ts_with_clause_filter(castNode(List, cmd->def), &tsdb_options, NULL, &pg_options); if (!tsdb_options) return DDL_CONTINUE; parse_results = ts_alter_table_with_clause_parse(tsdb_options); if (ht && !parse_results[AlterTableFlagChunkTimeInterval].is_default) { Dimension *dim = ts_hyperspace_get_mutable_dimension(ht->space, DIMENSION_TYPE_OPEN, 0); Ensure(dim, "hypertable without open dimension"); Oid time_type = get_atttype(dim->main_table_relid, dim->column_attno); Oid interval_type = InvalidOid; Datum interval = ts_create_table_parse_chunk_time_interval(parse_results [AlterTableFlagChunkTimeInterval], time_type, &interval_type); int64 chunk_interval = ts_interval_value_to_internal(interval, interval_type); ts_dimension_set_chunk_interval(dim, chunk_interval); } if (!parse_results[AlterTableFlagColumnstore].is_default || !parse_results[AlterTableFlagOrderBy].is_default || !parse_results[AlterTableFlagSegmentBy].is_default || !parse_results[AlterTableFlagCompressChunkTimeInterval].is_default || !parse_results[AlterTableFlagIndex].is_default) ts_cm_functions->process_compress_table(ht, parse_results); cmd->def = (Node *) pg_options; return cmd->def ? DDL_CONTINUE : DDL_DONE; } static DDLResult process_altertable_reset_options(AlterTableCmd *cmd, Hypertable *ht) { List *pg_options = NIL, *tsdb_options = NIL; WithClauseResult *parse_results = NULL; /* split postgres and timescaledb options */ ts_with_clause_filter(castNode(List, cmd->def), &tsdb_options, NULL, &pg_options); if (!tsdb_options) return DDL_CONTINUE; parse_results = ts_alter_table_reset_with_clause_parse(tsdb_options); if (parse_results[AlterTableFlagOrderBy].is_default && parse_results[AlterTableFlagSegmentBy].is_default && parse_results[AlterTableFlagIndex].is_default) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("only columnstore options segmentby and orderby can be reset"))); } CompressionSettings *settings = ts_compression_settings_get(ht->main_table_relid); if (!settings) { return DDL_CONTINUE; } if (!parse_results[AlterTableFlagSegmentBy].is_default) { settings->fd.segmentby = NULL; } if (!parse_results[AlterTableFlagOrderBy].is_default) { settings->fd.index = ts_remove_orderby_sparse_index(settings); settings->fd.orderby = NULL; settings->fd.orderby_desc = NULL; settings->fd.orderby_nullsfirst = NULL; } if (!parse_results[AlterTableFlagIndex].is_default) { settings->fd.index = NULL; ts_add_orderby_sparse_index(settings); } ts_compression_settings_update(settings); return DDL_CONTINUE; } static DDLResult process_viewstmt(ProcessUtilityArgs *args) { ViewStmt *stmt = castNode(ViewStmt, args->parsetree); List *pg_options = NIL; List *cagg_options = NIL; /* Check if user is passing continuous aggregate parameters and print a * useful error message if that is the case. */ ts_with_clause_filter(stmt->options, &cagg_options, NULL, &pg_options); if (cagg_options) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create continuous aggregate with CREATE VIEW"), errhint("Use CREATE MATERIALIZED VIEW to create a continuous aggregate."))); return DDL_CONTINUE; } static DDLResult process_create_table_as(ProcessUtilityArgs *args) { CreateTableAsStmt *stmt = castNode(CreateTableAsStmt, args->parsetree); WithClauseResult *parse_results = NULL; bool is_cagg = false; List *pg_options = NIL, *cagg_options = NIL, *other_namespace_options = NIL; if (stmt->objtype == OBJECT_MATVIEW) { /* Check for creation of continuous aggregate */ ts_with_clause_filter(stmt->into->options, &cagg_options, &other_namespace_options, &pg_options); if (cagg_options) { parse_results = ts_create_materialized_view_with_clause_parse(cagg_options); is_cagg = DatumGetBool(parse_results[CreateMaterializedViewFlagContinuous].parsed); } if (!is_cagg) return DDL_CONTINUE; if (pg_options != NIL) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported combination of storage parameters"), errdetail("A continuous aggregate does not support standard storage " "parameters."), errhint("Use only parameters with the \"timescaledb.\" prefix when " "creating a continuous aggregate."))); if (other_namespace_options) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("non \"timescaledb\" namespace options can be set only via ALTER"))); } if (!stmt->into->skipData) PreventInTransactionBlock(args->context == PROCESS_UTILITY_TOPLEVEL, "CREATE MATERIALIZED VIEW ... WITH DATA"); return ts_cm_functions->process_cagg_viewstmt(args->parsetree, args->query_string, args->pstmt, parse_results); } return DDL_CONTINUE; } /* * Get the default partition column from the column definitions. The default * partition column is the first column of type TIMESTAMP/TZ. */ static char * get_default_partition_column_by_definitions(List *definitions) { char *column_name = NULL; ListCell *lc, *lc2; foreach (lc, definitions) { Node *node = lfirst(lc); if (!IsA(node, ColumnDef)) continue; ColumnDef *coldef = castNode(ColumnDef, node); if (coldef->typeName->names == NIL) continue; foreach (lc2, coldef->typeName->names) { char *typename = strVal(lfirst(lc2)); if (strstr(typename, "timestamp") || strstr(typename, "timestamptz")) { column_name = pstrdup(coldef->colname); break; } } } return column_name; } static DDLResult process_create_stmt(ProcessUtilityArgs *args) { CreateStmt *stmt = castNode(CreateStmt, args->parsetree); List *pg_options = NIL, *hypertable_options = NIL; ts_with_clause_filter(stmt->options, &hypertable_options, NULL, &pg_options); stmt->options = pg_options; create_table_info.hypertable = false; create_table_info.with_clauses = NULL; /* * We can only convert the table into a hypertable after postgres has created * the initial table so we store the information passed in the WITH clause * and do some initial sanity check and do the actual work of creating the hypertable * in process_create_table_end. */ if (hypertable_options) { create_table_info.with_clauses = ts_create_table_with_clause_parse(hypertable_options); create_table_info.hypertable = DatumGetBool(create_table_info.with_clauses[CreateTableFlagHypertable].parsed); if (!create_table_info.hypertable) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("timescaledb options requires hypertable option"), errhint("Use \"timescaledb.hypertable\" to enable creating a hypertable."))); if (ts_guc_enable_partitioned_hypertables) { if (stmt->partspec == NULL) { /* * For partitioned hypertables, we need to decide the default partition column here * instead of process_create_table_end() as opposed to regular hypertable case. This * is because in partitioned hypertable case, we need to set the PartitionSpec in * CreateStmt. */ char *time_column = NULL; if (create_table_info.with_clauses[CreateTableFlagTimeColumn].is_default) { time_column = get_default_partition_column_by_definitions(stmt->tableElts); if (time_column) ereport(NOTICE, (errmsg("using column \"%s\" as partitioning column", time_column), errhint( "Use \"timescaledb.partition_column\" to specify a different " "column to use as " "partitioning column."))); else ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("partition column could not be determined"), errhint("Use \"timescaledb.partition_column\" to specify the " "column to use as " "partitioning column."))); } else time_column = TextDatumGetCString( create_table_info.with_clauses[CreateTableFlagTimeColumn].parsed); PartitionElem *pelem = makeNode(PartitionElem); pelem->name = time_column; pelem->location = -1; PartitionSpec *partspec = makeNode(PartitionSpec); #if PG16_LT partspec->strategy = pstrdup("range"); #else partspec->strategy = PARTITION_STRATEGY_RANGE; #endif partspec->partParams = list_make1(pelem); partspec->location = -1; stmt->partspec = partspec; } else { /* User has specified PARTITION BY clause */ #if PG16_LT if (strcmp(stmt->partspec->strategy, "range") != 0) #else if (stmt->partspec->strategy != PARTITION_STRATEGY_RANGE) #endif { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("only RANGE partitioning is supported for partitioned " "hypertables"))); } if (list_length(stmt->partspec->partParams) != 1) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("only single column partitioning is supported for partitioned " "hypertables"))); } if (!create_table_info.with_clauses[CreateTableFlagTimeColumn].is_default) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot specify both PARTITION BY and " "timescaledb.partition_column"))); } } } } return DDL_CONTINUE; } static DDLResult process_refresh_mat_view_start(ProcessUtilityArgs *args) { RefreshMatViewStmt *stmt = castNode(RefreshMatViewStmt, args->parsetree); Oid view_relid = RangeVarGetRelid(stmt->relation, NoLock, true); const ContinuousAgg *cagg; if (!OidIsValid(view_relid)) return DDL_CONTINUE; cagg = ts_continuous_agg_find_by_relid(view_relid); if (cagg) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("operation not supported on continuous aggregate"), errdetail("A continuous aggregate does not support REFRESH MATERIALIZED VIEW."), errhint("Use \"refresh_continuous_aggregate\" or set up a policy to refresh the " "continuous aggregate."))); return DDL_CONTINUE; } static DDLResult preprocess_execute(ProcessUtilityArgs *args) { #ifdef USE_TELEMETRY ListCell *lc; ExecuteStmt *stmt = (ExecuteStmt *) args->parsetree; PreparedStatement *entry = FetchPreparedStatement(stmt->name, false); if (!entry) return DDL_CONTINUE; foreach (lc, entry->plansource->query_list) { Query *query = lfirst_node(Query, lc); ts_telemetry_function_info_gather(query); } #endif return DDL_CONTINUE; } /* * Handle DDL commands before they have been processed by PostgreSQL. */ static DDLResult process_ddl_command_start(ProcessUtilityArgs *args) { bool check_read_only = true; ts_process_utility_handler_t handler; switch (nodeTag(args->parsetree)) { case T_AlterObjectSchemaStmt: handler = process_alterobjectschema; break; case T_TruncateStmt: handler = process_truncate; break; case T_AlterTableStmt: handler = process_altertable_start; break; case T_RenameStmt: handler = process_rename; break; case T_IndexStmt: handler = process_index_start; break; case T_CreateTrigStmt: handler = process_create_trigger_start; break; case T_RuleStmt: handler = process_create_rule_start; break; case T_ReassignOwnedStmt: handler = process_reassign_owned_start; break; case T_DropStmt: /* * Drop associated metadata/chunks but also continue on to drop * the main table. Because chunks are deleted before the main * table is dropped, the drop respects CASCADE in the expected * way. */ handler = process_drop_start; break; case T_DropRoleStmt: handler = process_drop_role; break; case T_DropTableSpaceStmt: handler = process_drop_tablespace; break; case T_GrantStmt: handler = process_grant_and_revoke; break; case T_GrantRoleStmt: handler = process_grant_and_revoke_role; break; case T_CopyStmt: check_read_only = false; handler = process_copy; break; case T_VacuumStmt: handler = process_vacuum; break; case T_ReindexStmt: handler = process_reindex; break; case T_ClusterStmt: handler = process_cluster_start; break; case T_ViewStmt: handler = process_viewstmt; break; case T_RefreshMatViewStmt: handler = process_refresh_mat_view_start; break; case T_CreateTableAsStmt: handler = process_create_table_as; break; case T_CreateStmt: handler = process_create_stmt; break; case T_ExecuteStmt: check_read_only = false; handler = preprocess_execute; break; default: handler = NULL; break; } if (handler == NULL) return DDL_CONTINUE; if (check_read_only) PreventCommandIfReadOnly(CreateCommandName(args->parsetree)); return handler(args); } /* * Handle DDL commands after they've been processed by PostgreSQL. */ static void process_ddl_command_end(CollectedCommand *cmd) { switch (nodeTag(cmd->parsetree)) { case T_CreateStmt: process_create_table_end(cmd->parsetree); break; case T_AlterTableStmt: process_altertable_end(cmd->parsetree, cmd); break; default: break; } } static void process_drop_constraint_on_chunk(Hypertable *ht, Oid chunk_relid, void *arg) { const char *hypertable_constraint_name = arg; Chunk *chunk = ts_chunk_get_by_relid(chunk_relid, true); /* drop both metadata and table; sql_drop won't be called recursively */ ts_chunk_constraint_delete_by_hypertable_constraint_name(chunk->fd.id, hypertable_constraint_name); } static void process_drop_table_constraint(EventTriggerDropObject *obj) { EventTriggerDropTableConstraint *constraint = (EventTriggerDropTableConstraint *) obj; Hypertable *ht; Assert(obj->type == EVENT_TRIGGER_DROP_TABLE_CONSTRAINT); /* do not use relids because underlying table could be gone */ ht = ts_hypertable_get_by_name(constraint->schema, constraint->table); if (ht != NULL) { CatalogSecurityContext sec_ctx; ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); /* Recurse to each chunk and drop the corresponding constraint */ foreach_chunk(ht, process_drop_constraint_on_chunk, (void *) constraint->constraint_name); ts_catalog_restore_user(&sec_ctx); } else { /* Cannot get the full chunk here because it's table might be dropped */ int32 chunk_id; bool found = ts_chunk_get_id(constraint->schema, constraint->table, &chunk_id, true); if (found) ts_chunk_constraint_delete_by_constraint_name(chunk_id, constraint->constraint_name); } } static void process_drop_table(EventTriggerDropObject *obj) { EventTriggerDropRelation *table = (EventTriggerDropRelation *) obj; Assert(obj->type == EVENT_TRIGGER_DROP_TABLE || obj->type == EVENT_TRIGGER_DROP_FOREIGN_TABLE); ts_chunk_delete_by_relid_and_relname(table->relid, table->schema, table->name, DROP_RESTRICT); ts_hypertable_delete_by_name(table->schema, table->name); /* * Normally, dependent catalogs (like compression settings) are cleaned up * when deleting the hypertable or chunk. However, in some cases, e.g., * when a hypertable delete cascades to chunks, the chunk relids cannot be * resolved from the schema and name because the chunk relations are * already dropped by PostgreSQL when the "drop eventtrigger" is * called. Therefore, also try to delete dependent catalog entries here * since the eventtrigger gives us the relid of dropped objects. */ ts_compression_settings_delete_any(table->relid); ts_chunk_rewrite_delete(table->relid, false); } static void process_sql_drop_schema(EventTriggerDropObject *obj) { EventTriggerDropSchema *schema = (EventTriggerDropSchema *) obj; int count; Assert(obj->type == EVENT_TRIGGER_DROP_SCHEMA); if (strcmp(schema->schema, INTERNAL_SCHEMA_NAME) == 0) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot drop the internal schema for extension \"%s\"", EXTENSION_NAME), errhint("Use DROP EXTENSION to remove the extension and the schema."))); /* * Check for any remaining hypertables that use the schema as its * associated schema. For matches, we reset their associated schema to the * INTERNAL schema */ count = ts_hypertable_reset_associated_schema_name(schema->schema); if (count > 0) ereport(NOTICE, (errmsg("the chunk storage schema changed to \"%s\" for %d hypertable%c", INTERNAL_SCHEMA_NAME, count, (count > 1) ? 's' : '\0'))); } static void process_drop_trigger(EventTriggerDropObject *obj) { EventTriggerDropTrigger *trigger_event = (EventTriggerDropTrigger *) obj; Hypertable *ht; Assert(obj->type == EVENT_TRIGGER_DROP_TRIGGER); /* do not use relids because underlying table could be gone */ ht = ts_hypertable_get_by_name(trigger_event->schema, trigger_event->table); if (ht != NULL) { /* Recurse to each chunk and drop the corresponding trigger */ ts_hypertable_drop_trigger(ht->main_table_relid, trigger_event->trigger_name); } } static void process_drop_view(EventTriggerDropView *dropped_view) { ts_continuous_agg_drop(dropped_view->schema, dropped_view->view_name); } static void process_ddl_sql_drop(EventTriggerDropObject *obj) { switch (obj->type) { case EVENT_TRIGGER_DROP_TABLE_CONSTRAINT: process_drop_table_constraint(obj); break; case EVENT_TRIGGER_DROP_TABLE: process_drop_table(obj); break; case EVENT_TRIGGER_DROP_SCHEMA: process_sql_drop_schema(obj); break; case EVENT_TRIGGER_DROP_TRIGGER: process_drop_trigger(obj); break; case EVENT_TRIGGER_DROP_VIEW: process_drop_view((EventTriggerDropView *) obj); break; case EVENT_TRIGGER_DROP_FOREIGN_TABLE: case EVENT_TRIGGER_DROP_FOREIGN_SERVER: case EVENT_TRIGGER_DROP_INDEX: break; } } /* * ProcessUtility hook for DDL commands that have not yet been processed by * PostgreSQL. */ static void timescaledb_ddl_command_start(PlannedStmt *pstmt, const char *query_string, bool readonly_tree, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *completion_tag) { last_process_utility_context = context; ProcessUtilityArgs args = { .query_string = query_string, .context = context, .params = params, .readonly_tree = readonly_tree, .dest = dest, .completion_tag = completion_tag, .pstmt = pstmt, .parsetree = pstmt->utilityStmt, .queryEnv = queryEnv, .parse_state = make_parsestate(NULL) }; bool altering_timescaledb = false; DDLResult result; args.parse_state->p_sourcetext = query_string; if (IsA(args.parsetree, AlterExtensionStmt)) { AlterExtensionStmt *stmt = (AlterExtensionStmt *) args.parsetree; altering_timescaledb = (strcmp(stmt->extname, EXTENSION_NAME) == 0); } /* * We don't want to load the extension if we just got the command to alter * it. */ if (altering_timescaledb || !ts_extension_is_loaded_and_not_upgrading()) { prev_ProcessUtility(&args); return; } /* * Since we might alter the parsetree and strip timescaledb options * before passing it to Postgres, we need to make a copy of the original * statement in case it is cached. */ args.pstmt = copyObject(pstmt); args.parsetree = args.pstmt->utilityStmt; result = process_ddl_command_start(&args); if (result == DDL_CONTINUE) prev_ProcessUtility(&args); } static void process_ddl_event_command_end(EventTriggerData *trigdata) { ListCell *lc; /* Inhibit collecting new commands while in the trigger */ EventTriggerInhibitCommandCollection(); switch (nodeTag(trigdata->parsetree)) { case T_AlterTableStmt: case T_CreateTrigStmt: case T_CreateStmt: case T_IndexStmt: foreach (lc, ts_event_trigger_ddl_commands()) process_ddl_command_end(lfirst(lc)); break; default: break; } EventTriggerUndoInhibitCommandCollection(); } static void process_ddl_event_sql_drop(EventTriggerData *trigdata) { ListCell *lc; List *dropped_objects = ts_event_trigger_dropped_objects(); foreach (lc, dropped_objects) process_ddl_sql_drop(lfirst(lc)); } TS_FUNCTION_INFO_V1(ts_timescaledb_process_ddl_event); /* * Event trigger hook for DDL commands that have already been handled by * PostgreSQL (i.e., "ddl_command_end" and "sql_drop" events). */ Datum ts_timescaledb_process_ddl_event(PG_FUNCTION_ARGS) { EventTriggerData *trigdata = (EventTriggerData *) fcinfo->context; if (!CALLED_AS_EVENT_TRIGGER(fcinfo)) elog(ERROR, "not fired by event trigger manager"); if (!ts_extension_is_loaded_and_not_upgrading()) PG_RETURN_NULL(); if (strcmp("ddl_command_end", trigdata->event) == 0) process_ddl_event_command_end(trigdata); else if (strcmp("sql_drop", trigdata->event) == 0) process_ddl_event_sql_drop(trigdata); PG_RETURN_NULL(); } extern void ts_process_utility_set_expect_chunk_modification(bool expect) { expect_chunk_modification = expect; } bool ts_process_utility_is_top_level(void) { return last_process_utility_context == PROCESS_UTILITY_TOPLEVEL; } bool ts_process_utility_is_context_nonatomic(void) { ProcessUtilityContext context = last_process_utility_context; return context == PROCESS_UTILITY_TOPLEVEL || context == PROCESS_UTILITY_QUERY_NONATOMIC; } void ts_process_utility_context_reset(void) { last_process_utility_context = PROCESS_UTILITY_TOPLEVEL; } static void process_utility_xact_abort(XactEvent event, void *arg) { switch (event) { case XACT_EVENT_ABORT: case XACT_EVENT_PARALLEL_ABORT: /* * Reset the expect_chunk_modification flag because it this is an * internal safety flag that is set to true only temporarily * during chunk operations. It should never remain true across * transactions. */ expect_chunk_modification = false; break; default: break; } } static void process_utility_subxact_abort(SubXactEvent event, SubTransactionId mySubid, SubTransactionId parentSubid, void *arg) { switch (event) { case SUBXACT_EVENT_ABORT_SUB: /* see note in process_utility_xact_abort */ expect_chunk_modification = false; break; default: break; } } void _process_utility_init(void) { prev_ProcessUtility_hook = ProcessUtility_hook; ProcessUtility_hook = timescaledb_ddl_command_start; RegisterXactCallback(process_utility_xact_abort, NULL); RegisterSubXactCallback(process_utility_subxact_abort, NULL); } void _process_utility_fini(void) { ProcessUtility_hook = prev_ProcessUtility_hook; UnregisterXactCallback(process_utility_xact_abort, NULL); UnregisterSubXactCallback(process_utility_subxact_abort, NULL); } ================================================ FILE: src/process_utility.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/plannodes.h> #include <parser/parse_node.h> #include <tcop/utility.h> #include "cache.h" typedef struct ProcessUtilityArgs { Cache *hcache; PlannedStmt *pstmt; QueryEnvironment *queryEnv; ParseState *parse_state; Node *parsetree; const char *query_string; ProcessUtilityContext context; ParamListInfo params; DestReceiver *dest; QueryCompletion *completion_tag; bool readonly_tree; } ProcessUtilityArgs; typedef enum { DDL_CONTINUE, DDL_DONE } DDLResult; typedef DDLResult (*ts_process_utility_handler_t)(ProcessUtilityArgs *args); extern void ts_process_utility_set_expect_chunk_modification(bool expect); /* * Procedures that use multiple transactions cannot be run in a transaction * block (from a function, from dynamic SQL) or in a subtransaction (from a * procedure block with an EXCEPTION clause). Such procedures use * PreventInTransactionBlock function to check whether they can be run. * * Though currently such checks are incomplete, because * PreventInTransactionBlock requires isTopLevel argument to throw a * consistent error when the call originates from a function. This * isTopLevel flag (that is a bit poorly named - see below) is not readily * available inside C procedures. The source of truth for it - * ProcessUtilityContext parameter is passed to ProcessUtility hooks, but * is not included with the function calls. There is an undocumented * SPI_inside_nonatomic_context function, that would have been sufficient * for isTopLevel flag, but it currently returns false when SPI connection * is absent (that is a valid scenario when C procedures are called from * top-lelev SQL instead of PLPG procedures or DO blocks) so it cannot be * used. * * To work around this the value of ProcessUtilityContext parameter is * saved when TS ProcessUtility hook is entered and can be accessed from * C procedures using new ts_process_utility_is_context_nonatomic function. * The result is called "non-atomic" instead of "top-level" because the way * how isTopLevel flag is determined from the ProcessUtilityContext value * in standard_ProcessUtility is insufficient for C procedures - it * excludes PROCESS_UTILITY_QUERY_NONATOMIC value (used when called from * PLPG procedure without an EXCEPTION clause) that is a valid use case for * C procedures with transactions. See details in the description of * ExecuteCallStmt function. * * It is expected that calls to C procedures are done with CALL and always * pass though the ProcessUtility hook. The ProcessUtilityContext * parameter is set to PROCESS_UTILITY_TOPLEVEL value by default. In * unlikely case when a C procedure is called without passing through * ProcessUtility hook and the call is done in atomic context, then * PreventInTransactionBlock checks will pass, but SPI_commit will fail * when checking that all current active snapshots are portal-owned * snapshots (the same behaviour that was observed before this change). * In atomic context there will be an additional snapshot set in * _SPI_execute_plan, see the snapshot handling invariants description * in that function. */ extern TSDLLEXPORT bool ts_process_utility_is_context_nonatomic(void); /* * Check if we are at top level. */ extern TSDLLEXPORT bool ts_process_utility_is_top_level(void); /* * Currently in TS ProcessUtility hook the saved ProcessUtilityContext * value is reset back to PROCESS_UTILITY_TOPLEVEL on normal exit but * is NOT reset in case of ereport exit. C procedures can call this * function to reset the saved value before doing the checks that can * result in ereport exit. */ extern TSDLLEXPORT void ts_process_utility_context_reset(void); ================================================ FILE: src/scan_iterator.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include "scan_iterator.h" TSDLLEXPORT void ts_scan_iterator_set_index(ScanIterator *iterator, CatalogTable table, int indexid) { iterator->ctx.index = catalog_get_index(ts_catalog_get(), table, indexid); } void ts_scan_iterator_end(ScanIterator *iterator) { ts_scanner_end_scan(&iterator->ctx); } void ts_scan_iterator_close(ScanIterator *iterator) { /* Ending a scan is a no-op if already ended */ ts_scanner_end_scan(&iterator->ctx); ts_scanner_close(&iterator->ctx); } TSDLLEXPORT void ts_scan_iterator_scan_key_init(ScanIterator *iterator, AttrNumber attributeNumber, StrategyNumber strategy, RegProcedure procedure, Datum argument) { MemoryContext oldmcxt; Assert(iterator->ctx.scankey == NULL || iterator->ctx.scankey == iterator->scankey); iterator->ctx.scankey = iterator->scankey; if (iterator->ctx.nkeys >= EMBEDDED_SCAN_KEY_SIZE) elog(ERROR, "cannot scan more than %d keys", EMBEDDED_SCAN_KEY_SIZE); /* * For rescans, when the scan key is reinitialized during the scan, make * sure the scan key is initialized on the long-lived scankey memory * context. */ oldmcxt = MemoryContextSwitchTo(iterator->ctx.internal.scan_mcxt); ScanKeyInit(&iterator->scankey[iterator->ctx.nkeys++], attributeNumber, strategy, procedure, argument); MemoryContextSwitchTo(oldmcxt); } TSDLLEXPORT void ts_scan_iterator_rescan(ScanIterator *iterator) { ts_scanner_rescan(&iterator->ctx, NULL); } ================================================ FILE: src/scan_iterator.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <utils/palloc.h> #include "scanner.h" #include "ts_catalog/catalog.h" #define EMBEDDED_SCAN_KEY_SIZE 5 typedef struct ScanIterator { ScannerCtx ctx; TupleInfo *tinfo; ScanKeyData scankey[EMBEDDED_SCAN_KEY_SIZE]; } ScanIterator; #define ts_scan_iterator_create(catalog_table_id, lock_mode, mctx) \ (ScanIterator) \ { \ .ctx = { \ .internal = { \ .ended = true, \ .scan_mcxt = CurrentMemoryContext, \ }, \ .table = catalog_get_table_id(ts_catalog_get(), catalog_table_id), \ .nkeys = 0, \ .scandirection = ForwardScanDirection, \ .lockmode = lock_mode, \ .result_mctx = mctx, \ .flags = SCANNER_F_NOFLAGS, \ }, \ } #define ts_scan_iterator_create_with_catalog_snapshot(catalog_table_id, lock_mode, mctx) \ (ScanIterator) \ { \ .ctx = { \ .internal = { \ .ended = true, \ .scan_mcxt = CurrentMemoryContext, \ }, \ .table = catalog_get_table_id(ts_catalog_get(), catalog_table_id), \ .nkeys = 0, \ .scandirection = ForwardScanDirection, \ .lockmode = lock_mode, \ .result_mctx = mctx, \ .flags = SCANNER_F_NOFLAGS, \ .use_catalog_snapshot = true, \ }, \ } static inline TupleInfo * ts_scan_iterator_tuple_info(const ScanIterator *iterator) { return iterator->tinfo; } static inline TupleTableSlot * ts_scan_iterator_slot(const ScanIterator *iterator) { return iterator->tinfo->slot; } static inline HeapTuple ts_scan_iterator_fetch_heap_tuple(const ScanIterator *iterator, bool materialize, bool *should_free) { return ts_scanner_fetch_heap_tuple(iterator->tinfo, materialize, should_free); } static inline TupleDesc ts_scan_iterator_tupledesc(const ScanIterator *iterator) { return ts_scanner_get_tupledesc(iterator->tinfo); } static inline MemoryContext ts_scan_iterator_get_result_memory_context(const ScanIterator *iterator) { return iterator->ctx.result_mctx; } static inline void * ts_scan_iterator_alloc_result(const ScanIterator *iterator, Size size) { return MemoryContextAllocZero(iterator->ctx.result_mctx, size); } static inline void ts_scan_iterator_start_scan(ScanIterator *iterator) { ts_scanner_start_scan(&(iterator)->ctx); } static inline TupleInfo * ts_scan_iterator_next(ScanIterator *iterator) { iterator->tinfo = ts_scanner_next(&(iterator)->ctx); return iterator->tinfo; } static inline void ts_scan_iterator_scan_key_reset(ScanIterator *iterator) { iterator->ctx.nkeys = 0; } static inline bool ts_scan_iterator_is_started(ScanIterator *iterator) { return iterator->ctx.internal.started; } void TSDLLEXPORT ts_scan_iterator_set_index(ScanIterator *iterator, CatalogTable table, int indexid); void TSDLLEXPORT ts_scan_iterator_end(ScanIterator *iterator); void TSDLLEXPORT ts_scan_iterator_close(ScanIterator *iterator); void TSDLLEXPORT ts_scan_iterator_scan_key_init(ScanIterator *iterator, AttrNumber attributeNumber, StrategyNumber strategy, RegProcedure procedure, Datum argument); /* * Reset the scan to use a new scan key. * * Note that the scan key should typically be reinitialized before a rescan. */ void TSDLLEXPORT ts_scan_iterator_rescan(ScanIterator *iterator); static inline void ts_scan_iterator_start_or_restart_scan(ScanIterator *iterator) { if (ts_scan_iterator_is_started(iterator)) ts_scan_iterator_rescan(iterator); else ts_scan_iterator_start_scan(iterator); } /* You must use `ts_scan_iterator_close` if terminating this loop early */ #define ts_scanner_foreach(scan_iterator) \ for (ts_scan_iterator_start_scan((scan_iterator)); \ ts_scan_iterator_next(scan_iterator) != NULL;) ================================================ FILE: src/scanner.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/htup_details.h> #include <access/relscan.h> #include <access/xact.h> #include <executor/tuptable.h> #include <storage/bufmgr.h> #include <storage/lmgr.h> #include <storage/procarray.h> #include <utils/palloc.h> #include <utils/rel.h> #include <utils/snapmgr.h> #include "scanner.h" enum ScannerType { ScannerTypeTable, ScannerTypeIndex, }; /* * Scanner can implement both index and heap scans in a single interface. */ typedef struct Scanner { Relation (*openscan)(ScannerCtx *ctx); ScanDesc (*beginscan)(ScannerCtx *ctx); bool (*getnext)(ScannerCtx *ctx); void (*rescan)(ScannerCtx *ctx); void (*endscan)(ScannerCtx *ctx); void (*closescan)(ScannerCtx *ctx); } Scanner; /* Functions implementing heap scans */ static Relation table_scanner_open(ScannerCtx *ctx) { ctx->tablerel = table_open(ctx->table, ctx->lockmode); return ctx->tablerel; } static ScanDesc table_scanner_beginscan(ScannerCtx *ctx) { ctx->internal.scan.table_scan = table_beginscan(ctx->tablerel, ctx->snapshot, ctx->nkeys, ctx->scankey); return ctx->internal.scan; } static bool table_scanner_getnext(ScannerCtx *ctx) { bool success = table_scan_getnextslot(ctx->internal.scan.table_scan, ForwardScanDirection, ctx->internal.tinfo.slot); return success; } static void table_scanner_rescan(ScannerCtx *ctx) { table_rescan(ctx->internal.scan.table_scan, ctx->scankey); } static void table_scanner_endscan(ScannerCtx *ctx) { table_endscan(ctx->internal.scan.table_scan); } static void table_scanner_close(ScannerCtx *ctx) { LOCKMODE lockmode = (ctx->flags & SCANNER_F_KEEPLOCK) ? NoLock : ctx->lockmode; table_close(ctx->tablerel, lockmode); } /* Functions implementing index scans */ static Relation index_scanner_open(ScannerCtx *ctx) { ctx->tablerel = table_open(ctx->table, ctx->lockmode); ctx->indexrel = index_open(ctx->index, ctx->lockmode); return ctx->indexrel; } static ScanDesc index_scanner_beginscan(ScannerCtx *ctx) { InternalScannerCtx *ictx = &ctx->internal; ictx->scan.index_scan = index_beginscan_compat(ctx->tablerel, ctx->indexrel, ctx->snapshot, NULL, ctx->nkeys, ctx->norderbys); ictx->scan.index_scan->xs_want_itup = ctx->want_itup; index_rescan(ictx->scan.index_scan, ctx->scankey, ctx->nkeys, NULL, ctx->norderbys); return ictx->scan; } static bool index_scanner_getnext(ScannerCtx *ctx) { InternalScannerCtx *ictx = &ctx->internal; bool success; success = index_getnext_slot(ictx->scan.index_scan, ctx->scandirection, ictx->tinfo.slot); ictx->tinfo.ituple = ictx->scan.index_scan->xs_itup; ictx->tinfo.ituple_desc = ictx->scan.index_scan->xs_itupdesc; return success; } static void index_scanner_rescan(ScannerCtx *ctx) { index_rescan(ctx->internal.scan.index_scan, ctx->scankey, ctx->nkeys, NULL, ctx->norderbys); } static void index_scanner_endscan(ScannerCtx *ctx) { index_endscan(ctx->internal.scan.index_scan); } static void index_scanner_close(ScannerCtx *ctx) { LOCKMODE lockmode = (ctx->flags & SCANNER_F_KEEPLOCK) ? NoLock : ctx->lockmode; index_close(ctx->indexrel, ctx->lockmode); table_close(ctx->tablerel, lockmode); } /* * Two scanners by type: heap and index scanners. */ static Scanner scanners[] = { [ScannerTypeTable] = { .openscan = table_scanner_open, .beginscan = table_scanner_beginscan, .getnext = table_scanner_getnext, .rescan = table_scanner_rescan, .endscan = table_scanner_endscan, .closescan = table_scanner_close, }, [ScannerTypeIndex] = { .openscan = index_scanner_open, .beginscan = index_scanner_beginscan, .getnext = index_scanner_getnext, .rescan = index_scanner_rescan, .endscan = index_scanner_endscan, .closescan = index_scanner_close, } }; static inline Scanner * scanner_ctx_get_scanner(ScannerCtx *ctx) { if (OidIsValid(ctx->index)) return &scanners[ScannerTypeIndex]; else return &scanners[ScannerTypeTable]; } TSDLLEXPORT void ts_scanner_rescan(ScannerCtx *ctx, const ScanKey scankey) { Scanner *scanner = scanner_ctx_get_scanner(ctx); MemoryContext oldmcxt; /* If scankey is NULL, the existing scan key was already updated or the * old should be reused */ if (NULL != scankey) memcpy(ctx->scankey, scankey, sizeof(*ctx->scankey)); oldmcxt = MemoryContextSwitchTo(ctx->internal.scan_mcxt); scanner->rescan(ctx); MemoryContextSwitchTo(oldmcxt); } static void prepare_scan(ScannerCtx *ctx) { ctx->internal.ended = false; ctx->internal.registered_snapshot = false; if (ctx->internal.scan_mcxt == NULL) ctx->internal.scan_mcxt = CurrentMemoryContext; if (ctx->snapshot == NULL) { /* * We use SnapshotSelf by default, for historical reasons mostly, but * we probably want to move to an MVCC snapshot as the default. The * difference is that a Self snapshot is an "instant" snapshot and can * see its own changes. More importantly, however, unlike an MVCC * snapshot, a Self snapshot is not subject to the strictness of * SERIALIZABLE isolation mode. * * This is important in case of, e.g., concurrent chunk creation by * two transactions; we'd like a transaction to use a new chunk as * soon as the creating transaction commits, so that there aren't * multiple transactions creating the same chunk and all but one fails * with a conflict. However, under SERIALIZABLE mode a transaction is * only allowed to read data from transactions that were committed * prior to transaction start. This means that two or more * transactions that create the same chunk must have all but the first * committed transaction fail. * * Therefore, we probably want to exempt internal bookkeeping metadata * from full SERIALIZABLE semantics (at least in the case of chunk * creation), or otherwise the INSERT behavior will be different for * hypertables compared to regular tables under SERIALIZABLE * mode. */ MemoryContext oldmcxt = MemoryContextSwitchTo(ctx->internal.scan_mcxt); if (ctx->use_catalog_snapshot) { ctx->snapshot = RegisterSnapshot(GetCatalogSnapshot(ctx->table)); } else { ctx->snapshot = RegisterSnapshot(GetSnapshotData(SnapshotSelf)); /* * Invalidate the PG catalog snapshot to ensure it is refreshed and * up-to-date with the snapshot used to scan TimescaleDB * metadata. Since TimescaleDB metadata is often joined with PG * catalog data (e.g., calling get_relname_relid() to fill in chunk * table Oid), this avoids any potential problems where the different * snapshots used to scan TimescaleDB metadata and PG catalog metadata * aren't in sync. * * Ideally, a catalog snapshot would be used to scan TimescaleDB * metadata, but that will change the behavior of chunk creation in * SERIALIZED mode, as described above. */ InvalidateCatalogSnapshot(); } ctx->internal.registered_snapshot = true; MemoryContextSwitchTo(oldmcxt); } } TSDLLEXPORT Relation ts_scanner_open(ScannerCtx *ctx) { Scanner *scanner = scanner_ctx_get_scanner(ctx); MemoryContext oldmcxt; Relation rel; Assert(NULL == ctx->tablerel); prepare_scan(ctx); Assert(ctx->internal.scan_mcxt != NULL); oldmcxt = MemoryContextSwitchTo(ctx->internal.scan_mcxt); rel = scanner->openscan(ctx); MemoryContextSwitchTo(oldmcxt); return rel; } /* * Start either a heap or index scan depending on the information in the * ScannerCtx. ScannerCtx must be setup by caller with the proper information * for the scan, including filters and callbacks for found tuples. */ TSDLLEXPORT void ts_scanner_start_scan(ScannerCtx *ctx) { InternalScannerCtx *ictx = &ctx->internal; Scanner *scanner; TupleDesc tuple_desc; MemoryContext oldmcxt; if (ictx->started) { Assert(!ictx->ended); Assert(ctx->tablerel); Assert(OidIsValid(ctx->table)); return; } if (ctx->tablerel == NULL) { Assert(NULL == ctx->indexrel); ts_scanner_open(ctx); } else { /* * Relations already opened by caller: Only need to prepare the scan * and set relation Oids so that the scanner knows which scanner * implementation to use. Respect the auto-closing behavior set by the * user, which is to auto close if unspecified. */ prepare_scan(ctx); ctx->table = RelationGetRelid(ctx->tablerel); if (NULL != ctx->indexrel) ctx->index = RelationGetRelid(ctx->indexrel); } Assert(ctx->internal.scan_mcxt != NULL); oldmcxt = MemoryContextSwitchTo(ctx->internal.scan_mcxt); scanner = scanner_ctx_get_scanner(ctx); scanner->beginscan(ctx); tuple_desc = RelationGetDescr(ctx->tablerel); ictx->tinfo.scanrel = ctx->tablerel; ictx->tinfo.mctx = ctx->result_mctx == NULL ? CurrentMemoryContext : ctx->result_mctx; ictx->tinfo.slot = MakeSingleTupleTableSlot(tuple_desc, table_slot_callbacks(ctx->tablerel)); MemoryContextSwitchTo(oldmcxt); /* Call pre-scan handler, if any. */ if (ctx->prescan != NULL) ctx->prescan(ctx->data); ictx->started = true; } static inline bool ts_scanner_limit_reached(ScannerCtx *ctx) { return ctx->limit > 0 && ctx->internal.tinfo.count >= ctx->limit; } static void scanner_cleanup(ScannerCtx *ctx) { InternalScannerCtx *ictx = &ctx->internal; if (ictx->registered_snapshot) { UnregisterSnapshot(ctx->snapshot); ctx->snapshot = NULL; } if (NULL != ictx->tinfo.slot) { ExecDropSingleTupleTableSlot(ictx->tinfo.slot); ictx->tinfo.slot = NULL; } if (NULL != ictx->scan_mcxt) { ictx->scan_mcxt = NULL; } } TSDLLEXPORT void ts_scanner_end_scan(ScannerCtx *ctx) { InternalScannerCtx *ictx = &ctx->internal; Scanner *scanner = scanner_ctx_get_scanner(ctx); MemoryContext oldmcxt; if (ictx->ended) return; /* Call post-scan handler, if any. */ if (ctx->postscan != NULL) ctx->postscan(ictx->tinfo.count, ctx->data); oldmcxt = MemoryContextSwitchTo(ctx->internal.scan_mcxt); scanner->endscan(ctx); MemoryContextSwitchTo(oldmcxt); scanner_cleanup(ctx); ictx->ended = true; ictx->started = false; } TSDLLEXPORT void ts_scanner_close(ScannerCtx *ctx) { Scanner *scanner = scanner_ctx_get_scanner(ctx); Assert(ctx->internal.ended); if (NULL != ctx->tablerel) { scanner->closescan(ctx); ctx->tablerel = NULL; ctx->indexrel = NULL; } } TSDLLEXPORT TupleInfo * ts_scanner_next(ScannerCtx *ctx) { InternalScannerCtx *ictx = &ctx->internal; Scanner *scanner = scanner_ctx_get_scanner(ctx); bool is_valid = false; if (!ts_scanner_limit_reached(ctx)) { MemoryContext oldmcxt = MemoryContextSwitchTo(ctx->internal.scan_mcxt); is_valid = scanner->getnext(ctx); MemoryContextSwitchTo(oldmcxt); } while (is_valid) { if (ctx->filter == NULL || ctx->filter(&ictx->tinfo, ctx->data) == SCAN_INCLUDE) { ictx->tinfo.count++; if (ctx->tuplock) { TupleTableSlot *slot = ictx->tinfo.slot; Assert(ctx->snapshot); ictx->tinfo.lockresult = table_tuple_lock(ctx->tablerel, &(slot->tts_tid), ctx->snapshot, slot, GetCurrentCommandId(false), ctx->tuplock->lockmode, ctx->tuplock->waitpolicy, ctx->tuplock->lockflags, &ictx->tinfo.lockfd); } /* stop at a valid tuple */ return &ictx->tinfo; } if (ts_scanner_limit_reached(ctx)) is_valid = false; else { MemoryContext oldmcxt = MemoryContextSwitchTo(ctx->internal.scan_mcxt); is_valid = scanner->getnext(ctx); MemoryContextSwitchTo(oldmcxt); } } if (!(ctx->flags & SCANNER_F_NOEND)) ts_scanner_end_scan(ctx); if (!(ctx->flags & SCANNER_F_NOEND_AND_NOCLOSE)) ts_scanner_close(ctx); return NULL; } /* * Perform either a heap or index scan depending on the information in the * ScannerCtx. ScannerCtx must be setup by caller with the proper information * for the scan, including filters and callbacks for found tuples. * * Return the number of tuples that were found. */ TSDLLEXPORT int ts_scanner_scan(ScannerCtx *ctx) { TupleInfo *tinfo; MemSet(&ctx->internal, 0, sizeof(ctx->internal)); for (ts_scanner_start_scan(ctx); (tinfo = ts_scanner_next(ctx));) { if (ctx->tuple_found != NULL) { ScanTupleResult scan_result = ctx->tuple_found(tinfo, ctx->data); /* Call tuple_found handler. Abort the scan if the handler wants us to */ if (scan_result == SCAN_DONE) { if (!(ctx->flags & SCANNER_F_NOEND)) ts_scanner_end_scan(ctx); if (!(ctx->flags & SCANNER_F_NOEND_AND_NOCLOSE)) ts_scanner_close(ctx); break; } else if (scan_result == SCAN_RESTART_WITH_NEW_SNAPSHOT) { ts_scanner_end_scan(ctx); ctx->internal.tinfo.count = 0; ctx->snapshot = RegisterSnapshot(GetLatestSnapshot()); ts_scanner_start_scan(ctx); /* Since we register the snapshot manually above, * we need to mark it as registered in the scanner but only after we * start the scan since the scanner resets this flag and sets it only * if the snapshot gets registered during scan preparation phase. */ ctx->internal.registered_snapshot = true; } } } return ctx->internal.tinfo.count; } TSDLLEXPORT bool ts_scanner_scan_one(ScannerCtx *ctx, bool fail_if_not_found, const char *item_type) { /* Since this function ignores the custom limit, we assume that no custom limit is set by the * caller. */ Assert(ctx->limit == 0); /* We are interested in a maximum of two tuples to determine whether the addressed tuple is * unique. */ ctx->limit = 2; int num_found = ts_scanner_scan(ctx); switch (num_found) { case 0: if (fail_if_not_found) { elog(ERROR, "%s not found", item_type); } return false; case 1: return true; default: elog(ERROR, "more than one %s found", item_type); return false; } } ItemPointer ts_scanner_get_tuple_tid(TupleInfo *ti) { return &ti->slot->tts_tid; } HeapTuple ts_scanner_fetch_heap_tuple(const TupleInfo *ti, bool materialize, bool *should_free) { return ExecFetchSlotHeapTuple(ti->slot, materialize, should_free); } TupleDesc ts_scanner_get_tupledesc(const TupleInfo *ti) { return ti->slot->tts_tupleDescriptor; } ================================================ FILE: src/scanner.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <access/genam.h> #include <access/heapam.h> #include <nodes/lockoptions.h> #include <utils/fmgroids.h> #include "compat/compat.h" #include "utils.h" typedef struct ScanTupLock { LockTupleMode lockmode; LockWaitPolicy waitpolicy; unsigned int lockflags; } ScanTupLock; /* Tuple information passed on to handlers when scanning for tuples. */ typedef struct TupleInfo { Relation scanrel; TupleTableSlot *slot; /* return index tuple if it was requested -- only for index scans */ IndexTuple ituple; TupleDesc ituple_desc; /* * If the user requested a tuple lock, the result of the lock is passed on * in lockresult. */ TM_Result lockresult; /* Failure data in case of failed tuple lock */ TM_FailureData lockfd; int count; /* * The memory context (optionally) set initially in the ScannerCtx. This * can be used to allocate data on in the tuple handle function. */ MemoryContext mctx; } TupleInfo; typedef enum ScanTupleResult { SCAN_DONE, SCAN_CONTINUE, SCAN_RESTART_WITH_NEW_SNAPSHOT } ScanTupleResult; typedef enum ScanFilterResult { SCAN_EXCLUDE, SCAN_INCLUDE } ScanFilterResult; typedef ScanTupleResult (*tuple_found_func)(TupleInfo *ti, void *data); typedef ScanFilterResult (*tuple_filter_func)(const TupleInfo *ti, void *data); typedef void (*postscan_func)(int num_tuples, void *data); typedef union ScanDesc { IndexScanDesc index_scan; TableScanDesc table_scan; } ScanDesc; typedef enum ScannerFlags { SCANNER_F_NOFLAGS = 0x00, SCANNER_F_KEEPLOCK = 0x01, SCANNER_F_NOEND = 0x02, SCANNER_F_NOEND_AND_NOCLOSE = 0x04 | SCANNER_F_NOEND, } ScannerFlags; /* * InternalScannerCtx is used for internal state during scanning and shouldn't * be initialized or touched by the user. */ typedef struct InternalScannerCtx { TupleInfo tinfo; ScanDesc scan; /* * PG scan functions must be called on a memory context that lives * throughout the entire scan. Use the scan_mcxt to ensure that * functions aren't called on, e.g., a per-tuple context. */ MemoryContext scan_mcxt; bool registered_snapshot; bool started; bool ended; } InternalScannerCtx; typedef struct ScannerCtx { InternalScannerCtx internal; /* Fields below this line can be initialized by the user */ Oid table; Oid index; Relation tablerel; Relation indexrel; ScanKey scankey; int flags; int nkeys, norderbys, limit; /* Limit on number of tuples to return. 0 or * less means no limit */ bool want_itup; LOCKMODE lockmode; MemoryContext result_mctx; /* The memory context to allocate the result * on */ const ScanTupLock *tuplock; ScanDirection scandirection; Snapshot snapshot; /* Snapshot requested by the caller. Set automatically * when NULL */ bool use_catalog_snapshot; /* If true, use a catalog snapshot to scan data, unless snapshot is already set */ void *data; /* User-provided data passed on to filter() * and tuple_found() */ /* * Optional handler called before a scan starts, but relation locks are * acquired. */ void (*prescan)(void *data); /* * Optional handler called after a scan finishes and before relation locks * are released. Passes on the number of tuples found. */ void (*postscan)(int num_tuples, void *data); /* * Optional handler to filter tuples. Should return SCAN_INCLUDE for * tuples that should be passed on to tuple_found, or SCAN_EXCLUDE * otherwise. */ ScanFilterResult (*filter)(const TupleInfo *ti, void *data); /* * Handler for found tuples. Should return SCAN_CONTINUE to continue the * scan or SCAN_DONE to finish without scanning further tuples. */ ScanTupleResult (*tuple_found)(TupleInfo *ti, void *data); } ScannerCtx; /* Performs an index scan or heap scan and returns the number of matching * tuples. */ extern TSDLLEXPORT Relation ts_scanner_open(ScannerCtx *ctx); extern TSDLLEXPORT void ts_scanner_close(ScannerCtx *ctx); extern TSDLLEXPORT int ts_scanner_scan(ScannerCtx *ctx); extern TSDLLEXPORT bool ts_scanner_scan_one(ScannerCtx *ctx, bool fail_if_not_found, const char *item_type); extern TSDLLEXPORT void ts_scanner_start_scan(ScannerCtx *ctx); extern TSDLLEXPORT void ts_scanner_end_scan(ScannerCtx *ctx); extern TSDLLEXPORT void ts_scanner_rescan(ScannerCtx *ctx, const ScanKey scankey); extern TSDLLEXPORT TupleInfo *ts_scanner_next(ScannerCtx *ctx); extern TSDLLEXPORT ItemPointer ts_scanner_get_tuple_tid(TupleInfo *ti); extern TSDLLEXPORT HeapTuple ts_scanner_fetch_heap_tuple(const TupleInfo *ti, bool materialize, bool *should_free); extern TSDLLEXPORT TupleDesc ts_scanner_get_tupledesc(const TupleInfo *ti); ================================================ FILE: src/sort_transform.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_type.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <nodes/plannodes.h> #include <optimizer/paths.h> #include <optimizer/planner.h> #include <parser/parsetree.h> #include <utils/fmgroids.h> #include <utils/guc.h> #include <utils/lsyscache.h> #include "compat/compat.h" #include "cross_module_fn.h" #include "func_cache.h" #include "hypertable.h" #include "import/allpaths.h" #include "sort_transform.h" /* This optimizations allows GROUP BY clauses that transform time in * order-preserving ways to use indexes on the time field. It works * by transforming sorting clauses from their more complex versions * to simplified ones that can use the plain index, if the transform * is order preserving. * * For example, an ordering on date_trunc('minute', time) can be transformed * to an ordering on time. */ static Expr * transform_timestamp_cast(FuncExpr *func) { /* * transform cast from timestamptz to timestamp * * timestamp(var) => var * * proof: timestamp(time1) >= timestamp(time2) iff time1 > time2 * */ Expr *first; if (list_length(func->args) != 1) return (Expr *) func; first = ts_sort_transform_expr(linitial(func->args)); if (!IsA(first, Var)) return (Expr *) func; return (Expr *) copyObject(first); } static Expr * transform_timestamptz_cast(FuncExpr *func) { /* * Transform cast from date to timestamptz, or timestamp to timestamptz, * or abstime to timestamptz Handles only single-argument versions of the * cast to avoid explicit timezone specifiers * * * timestamptz(var) => var * * proof: timestamptz(time1) >= timestamptz(time2) iff time1 > time2 * */ Expr *first; if (list_length(func->args) != 1) return (Expr *) func; first = ts_sort_transform_expr(linitial(func->args)); if (!IsA(first, Var)) return (Expr *) func; return (Expr *) copyObject(first); } static inline Expr * transform_time_op_const_interval(OpExpr *op) { /* * optimize timestamp(tz) +/- const interval * * Sort of ts + 1 minute fulfilled by sort of ts */ if (list_length(op->args) == 2 && IsA(lsecond(op->args), Const)) { Oid left = exprType((Node *) linitial(op->args)); Oid right = exprType((Node *) lsecond(op->args)); if ((left == TIMESTAMPOID && right == INTERVALOID) || (left == TIMESTAMPTZOID && right == INTERVALOID) || (left == DATEOID && right == INTERVALOID)) { Interval *interval = DatumGetIntervalP((lsecond_node(Const, op->args))->constvalue); if (interval->month != 0 || interval->day != 0) return (Expr *) op; char *name = get_opname(op->opno); if (strncmp(name, "-", NAMEDATALEN) == 0 || strncmp(name, "+", NAMEDATALEN) == 0) { Expr *first = ts_sort_transform_expr((Expr *) linitial(op->args)); if (IsA(first, Var)) return copyObject(first); } } } return (Expr *) op; } static inline Expr * transform_int_op_const(OpExpr *op) { /* * Optimize int op const (or const op int), whenever possible. e.g. sort * of some_int + const fulfilled by sort of some_int same for the * following operator: + - / * * * Note that / is not commutative and const / var does NOT work (namely it * reverses sort order, which we don't handle yet) */ if (list_length(op->args) == 2 && (IsA(lsecond(op->args), Const) || IsA(linitial(op->args), Const))) { Oid left = exprType((Node *) linitial(op->args)); Oid right = exprType((Node *) lsecond(op->args)); if ((left == INT8OID && right == INT8OID) || (left == INT4OID && right == INT4OID) || (left == INT2OID && right == INT2OID)) { char *name = get_opname(op->opno); if (name[1] == '\0') { switch (name[0]) { case '-': case '+': case '*': /* commutative cases */ if (IsA(linitial(op->args), Const)) { Expr *nonconst = ts_sort_transform_expr((Expr *) lsecond(op->args)); if (IsA(nonconst, Var)) return copyObject(nonconst); } else { Expr *nonconst = ts_sort_transform_expr((Expr *) linitial(op->args)); if (IsA(nonconst, Var)) return copyObject(nonconst); } break; case '/': /* only if second arg is const */ if (IsA(lsecond(op->args), Const)) { Expr *nonconst = ts_sort_transform_expr((Expr *) linitial(op->args)); if (IsA(nonconst, Var)) return copyObject(nonconst); } break; default: /* * Do nothing for unknown operators. The explicit empty * branch is to placate the static analyzers. */ break; } } } } return (Expr *) op; } /* sort_transforms_expr returns a simplified sort expression in a form * more common for indexes. Must return same data type & collation too. * * Sort transforms have the following correctness condition: * Any ordering provided by the returned expression is a valid * ordering under the original expression. The reverse need not * be true to apply the transformation to the last member of pathkeys * but it would need to be true to apply the transformation to * arbitrary members of pathkeys. * * Namely if orig_expr(X) > orig_expr(Y) then * new_expr(X) > new_expr(Y). * * Note that if orig_expr(X) = orig_expr(Y) then * the ordering under new_expr is unconstrained. * */ Expr * ts_sort_transform_expr(Expr *orig_expr) { if (IsA(orig_expr, FuncExpr)) { FuncExpr *func = (FuncExpr *) orig_expr; FuncInfo *finfo = ts_func_cache_get_bucketing_func(func->funcid); if (NULL != finfo) { if (NULL == finfo->sort_transform) return orig_expr; return finfo->sort_transform(func); } /* Functions of one argument that convert something to timestamp(tz). */ if (func->funcid == F_TIMESTAMP_DATE || func->funcid == F_TIMESTAMP_TIMESTAMPTZ) { return transform_timestamp_cast(func); } if (func->funcid == F_TIMESTAMPTZ_DATE || func->funcid == F_TIMESTAMPTZ_TIMESTAMP) { return transform_timestamptz_cast(func); } } if (IsA(orig_expr, OpExpr)) { OpExpr *op = (OpExpr *) orig_expr; Oid type_first = exprType((Node *) linitial(op->args)); if (type_first == TIMESTAMPOID || type_first == TIMESTAMPTZOID || type_first == DATEOID) { return transform_time_op_const_interval(op); } if (type_first == INT2OID || type_first == INT4OID || type_first == INT8OID) { return transform_int_op_const(op); } } return orig_expr; } /* sort_transform_ec creates a new EquivalenceClass with transformed * expressions if any of the members of the original EC can be transformed for the sort. */ static EquivalenceClass * sort_transform_ec(PlannerInfo *root, EquivalenceClass *orig, Relids child_relids) { EquivalenceClass *newec = NULL; bool propagate_to_children = false; /* check all members, adding only transformable members to new ec */ EquivalenceMember *ec_mem; #if PG18_GE /* Use specialized iterator to include child ems. * * https://github.com/postgres/postgres/commit/d69d45a5 */ EquivalenceMemberIterator it; setup_eclass_member_iterator(&it, orig, child_relids); while ((ec_mem = eclass_member_iterator_next(&it)) != NULL) { #else ListCell *lc_member; foreach (lc_member, orig->ec_members) { ec_mem = (EquivalenceMember *) lfirst(lc_member); #endif Expr *transformed_expr = ts_sort_transform_expr(ec_mem->em_expr); if (transformed_expr != ec_mem->em_expr) { EquivalenceMember *em; Oid type_oid = exprType((Node *) transformed_expr); List *opfamilies = list_copy(orig->ec_opfamilies); #if PG16_LT /* * if the transform already exists for even one member, assume * exists for all */ EquivalenceClass *exist = get_eclass_for_sort_expr(root, transformed_expr, ec_mem->em_nullable_relids, opfamilies, type_oid, orig->ec_collation, orig->ec_sortref, ec_mem->em_relids, false); #else EquivalenceClass *exist = get_eclass_for_sort_expr(root, transformed_expr, opfamilies, type_oid, orig->ec_collation, orig->ec_sortref, ec_mem->em_relids, false); #endif if (exist != NULL) { return exist; } em = makeNode(EquivalenceMember); em->em_expr = transformed_expr; em->em_relids = bms_copy(ec_mem->em_relids); #if PG16_LT em->em_nullable_relids = bms_copy(ec_mem->em_nullable_relids); #else em->em_parent = ec_mem->em_parent; #endif em->em_is_const = ec_mem->em_is_const; em->em_is_child = ec_mem->em_is_child; em->em_datatype = type_oid; if (newec == NULL) { /* lazy create the ec. */ newec = makeNode(EquivalenceClass); newec->ec_opfamilies = opfamilies; newec->ec_collation = orig->ec_collation; newec->ec_members = NIL; #if PG18_GE newec->ec_childmembers = NULL; newec->ec_childmembers_size = 0; #endif newec->ec_sources = list_copy(orig->ec_sources); newec->ec_derives_list = list_copy(orig->ec_derives_list); newec->ec_relids = bms_copy(orig->ec_relids); newec->ec_has_const = orig->ec_has_const; /* Even if the original EC has volatile (it has time_bucket_gapfill) * this ordering is purely on the time column, so it is non-volatile * and should be propagated to the children. */ newec->ec_has_volatile = false; #if PG16_LT newec->ec_below_outer_join = orig->ec_below_outer_join; #endif newec->ec_broken = orig->ec_broken; newec->ec_sortref = orig->ec_sortref; newec->ec_merged = orig->ec_merged; /* Volatile ECs only ever have one member, that of the root, * so if the original EC was volatile, we need to propagate the * new EC to the children ourselves. */ propagate_to_children = orig->ec_has_volatile; /* Even though time_bucket_gapfill is marked as VOLATILE to * prevent the planner from removing the call, it's still safe * to use values from child tables in lieu of the output of the * root table. Among other things, this allows us to use the * sort-order from the child tables for the output. */ orig->ec_has_volatile = false; } #if PG18_LT newec->ec_members = lappend(newec->ec_members, em); #else /* Update the child member lists when adding child ems. * * https://github.com/postgres/postgres/commit/d69d45a5 */ if (em->em_is_child) ts_add_child_eq_member(root, newec, em, it.current_relid); else newec->ec_members = lappend(newec->ec_members, em); #endif int i = -1; while ((i = bms_next_member(em->em_relids, i)) >= 0) { RelOptInfo *rel = root->simple_rel_array[i]; rel->eclass_indexes = bms_add_member(rel->eclass_indexes, list_length(root->eq_classes)); } } } /* if any transforms were found return new ec */ if (newec != NULL) { root->eq_classes = lappend(root->eq_classes, newec); if (propagate_to_children) { Bitmapset *parents = bms_copy(newec->ec_relids); ListCell *lc; int parent; bms_get_singleton_member(parents, &parent); foreach (lc, root->append_rel_list) { AppendRelInfo *appinfo = lfirst_node(AppendRelInfo, lc); if (appinfo->parent_relid == (Index) parent) { RelOptInfo *parent_rel = root->simple_rel_array[appinfo->parent_relid]; RelOptInfo *child_rel = root->simple_rel_array[appinfo->child_relid]; add_child_rel_equivalences(root, appinfo, parent_rel, child_rel); } } } return newec; } return NULL; } /* * This optimization transforms between equivalent sort operations to try * to find useful indexes. * * For example: an ORDER BY date_trunc('minute', time) can be implemented by * an ordering of time. */ List * ts_sort_transform_get_pathkeys(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte, Hypertable *ht) { /* * We attack this problem in three steps: * * 1) Create a pathkey for the transformed (simplified) sort. * * 2) Use the transformed pathkey to find new useful index paths. * * 3) Transform the pathkey of the new paths back into the original form * to make this transparent to upper levels in the planner. * */ ListCell *lc; List *transformed_query_pathkeys = NIL; PathKey *last_pk; PathKey *new_pk; EquivalenceClass *transformed; /* * nothing to do for empty pathkeys */ if (root->query_pathkeys == NIL) return NIL; /* * These sort transformations are only safe for single member ORDER BY * clauses or as last member of the ORDER BY clause. * Using it for other ORDER BY clauses will result in wrong ordering. */ last_pk = llast(root->query_pathkeys); /* * We can only transform the original pathkey if it references our hypertable. * If it references another one, we might be able to successfully transform * it to a join EC that references both hypertables, but when we replace it * back, we'll get into an incorrect state where the pathkey for the scan * references only a different hypertable and doesn't have an EC member for * ours. */ int desired_ec_relid = rel->relid; if (rel->reloptkind == RELOPT_OTHER_MEMBER_REL) { /* * The EC relids contain only inheritance parents, not individual * children. */ AppendRelInfo *appinfo = root->append_rel_array[rel->relid]; desired_ec_relid = appinfo->parent_relid; } EquivalenceClass *last_pk_eclass = last_pk->pk_eclass; if (!bms_is_member(desired_ec_relid, last_pk_eclass->ec_relids)) { return NIL; } Relids child_relids = NULL; #if PG18_GE /* In PG18, iterating over child ems requires you to * use child relids with a special iterator. Here we gather * them by collecting them from childmembers array. * * https://github.com/postgres/postgres/commit/d69d45a5 */ for (int i = 0; i < last_pk_eclass->ec_childmembers_size; i++) { if (list_length(last_pk_eclass->ec_childmembers[i]) > 0) { child_relids = bms_add_member(child_relids, i); } } #endif /* * Try to apply the transformation. */ transformed = sort_transform_ec(root, last_pk_eclass, child_relids); if (transformed == NULL) return NIL; new_pk = make_canonical_pathkey(root, transformed, last_pk->pk_opfamily, last_pk->pk_cmptype, last_pk->pk_nulls_first); /* * create complete transformed pathkeys */ foreach (lc, root->query_pathkeys) { if (lfirst(lc) != last_pk) transformed_query_pathkeys = lappend(transformed_query_pathkeys, lfirst(lc)); else transformed_query_pathkeys = lappend(transformed_query_pathkeys, new_pk); } return transformed_query_pathkeys; } /* * After we have created new paths with transformed pathkeys, replace them back * with the original pathkeys. */ void ts_sort_transform_replace_pathkeys(void *node, List *transformed_pathkeys, List *original_pathkeys) { if (node == NULL) { return; } if (IsA(node, List)) { List *list = castNode(List, node); ListCell *lc; foreach (lc, list) { ts_sort_transform_replace_pathkeys(lfirst(lc), transformed_pathkeys, original_pathkeys); } return; } Path *path = (Path *) node; if (compare_pathkeys(path->pathkeys, transformed_pathkeys) == PATHKEYS_EQUAL) { path->pathkeys = original_pathkeys; } if (IsA(path, CustomPath)) { CustomPath *custom = castNode(CustomPath, path); ts_sort_transform_replace_pathkeys(custom->custom_paths, transformed_pathkeys, original_pathkeys); /* Need to handle tsl-specific custom path types */ if (ts_cm_functions->sort_transform_replace_pathkeys != NULL) { ts_cm_functions->sort_transform_replace_pathkeys(path, transformed_pathkeys, original_pathkeys); } } else if (IsA(path, MergeAppendPath)) { MergeAppendPath *append = castNode(MergeAppendPath, path); ts_sort_transform_replace_pathkeys(append->subpaths, transformed_pathkeys, original_pathkeys); } else if (IsA(path, AppendPath)) { AppendPath *append = castNode(AppendPath, path); ts_sort_transform_replace_pathkeys(append->subpaths, transformed_pathkeys, original_pathkeys); } else if (IsA(path, ProjectionPath)) { ProjectionPath *projection = castNode(ProjectionPath, path); ts_sort_transform_replace_pathkeys(projection->subpath, transformed_pathkeys, original_pathkeys); } } ================================================ FILE: src/sort_transform.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include "hypertable.h" #include "import/planner.h" extern Expr *ts_sort_transform_expr(Expr *expr); extern List *ts_sort_transform_get_pathkeys(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte, Hypertable *ht); extern void ts_sort_transform_replace_pathkeys(void *node, List *transformed_pathkeys, List *original_pathkeys); ================================================ FILE: src/subspace_store.README.md ================================================ A subspace store allows you to save data associated with a multidimensional-subspace. We use this to cache per-chunk values, such as `chunk`s or `chunk_insert_state`. Subspaces are defined conceptually via a Hypercube (that is a collection of slices -- one for each dimension). Thus, a subspace is a "rectangular" cutout in a multidimensional space. that is given a hypertable with (ts Timestamp, i int) with intervals of one hour and 2 hash-partitions the subspaces could be: ``` 00:00 01:00 02:00 03:00 ---|-------|-------|-------|-------|--- - - 1 | | | | | ---|-------|-------|-------|-------|--- - - 2 | | | | | ---|-------|-------|-------|-------|--- - - 3 | | | | | ---|-------|-------|-------|-------|--- - - ``` Each subspace is cached in a tree structure, with each level of the tree corresponding to a dimension the hypertable is partitioned on, with the first level always being an open (time) dimension time dimension. Thus, the aforementioned hypertable will have the following tree: ``` SubspaceStore | V SubspaceStoreInternalNode (time) | (.vector) V | o | ... | ... | ... | | V DimensionSlice (00:00 - 01:00) | V SubspaceStoreInternalNode (dim 1) | V . . . | V ChunkInsertState (or other leaf object) ``` Each `SubspaceStoreInternalNode` has a field `descendants` storing a count of the number of leaf objects for that subtree, which we used to ensure `SubspaceStore`s don't grow beyond their maximum size. Currently our strategy when adding to a full `SubspaceStore` is to evict the all elements referenced from the first entry of the top-most vector. The assumption is that the topmost vector indexes based on time, and that the first element stores state for those chunks with the earliest time. If we usually perform operations in time-order, these are the elements least likely to be reused. This eviction-strategy is the only reason that the first level of a `SubspaceStore` is always a open (time) dimension. ================================================ FILE: src/subspace_store.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <utils/memutils.h> #include "dimension.h" #include "dimension_slice.h" #include "dimension_vector.h" #include "hypercube.h" #include "subspace_store.h" /* * In terms of datastructures, the subspace store is actually a tree. At the * root of a tree is a DimensionVec representing the different DimensionSlices * for the first dimension. Each of the DimensionSlices of the * first dimension point to a DimensionVec of the second dimension. This recurses * for the N dimensions. The leaf DimensionSlice points to the data being stored. * * */ typedef struct SubspaceStoreInternalNode { DimensionVec *vector; uint16 descendants; bool last_internal_node; } SubspaceStoreInternalNode; typedef struct SubspaceStore { MemoryContext mcxt; uint16 num_dimensions; /* limit growth of store by limiting number of slices in first dimension, 0 for no limit */ uint16 max_items; SubspaceStoreInternalNode *origin; /* origin of the tree */ } SubspaceStore; static inline SubspaceStoreInternalNode * subspace_store_internal_node_create(bool last_internal_node) { SubspaceStoreInternalNode *node = palloc(sizeof(SubspaceStoreInternalNode)); node->vector = ts_dimension_vec_create(DIMENSION_VEC_DEFAULT_SIZE); node->descendants = 0; node->last_internal_node = last_internal_node; return node; } static inline void subspace_store_internal_node_free(void *node) { ts_dimension_vec_free(((SubspaceStoreInternalNode *) node)->vector); pfree(node); } static size_t subspace_store_internal_node_descendants(SubspaceStoreInternalNode *node, int index) { const DimensionSlice *slice = ts_dimension_vec_get(node->vector, index); if (slice == NULL) return 0; if (node->last_internal_node) return 1; return ((SubspaceStoreInternalNode *) slice->storage)->descendants; } SubspaceStore * ts_subspace_store_init(const Hyperspace *space, MemoryContext mcxt, int16 max_items) { MemoryContext old = MemoryContextSwitchTo(mcxt); SubspaceStore *sst = palloc(sizeof(SubspaceStore)); sst->origin = subspace_store_internal_node_create(space->num_dimensions == 1); sst->num_dimensions = space->num_dimensions; /* max_items = 0 is treated as unlimited */ sst->max_items = max_items; sst->mcxt = mcxt; MemoryContextSwitchTo(old); return sst; } void ts_subspace_store_add(SubspaceStore *subspace_store, const Hypercube *hypercube, void *object, void (*object_free)(void *)) { SubspaceStoreInternalNode *node = subspace_store->origin; DimensionSlice *last = NULL; MemoryContext old = MemoryContextSwitchTo(subspace_store->mcxt); int i; Assert(hypercube->num_slices == subspace_store->num_dimensions); for (i = 0; i < hypercube->num_slices; i++) { const DimensionSlice *target = hypercube->slices[i]; DimensionSlice *match; Assert(target->storage == NULL); if (node == NULL) { /* * We should have one internal node per dimension in the * hypertable. If we don't have one for the current dimension, * create one now. (There will always be one for time) */ Assert(last != NULL); last->storage = subspace_store_internal_node_create(i == (hypercube->num_slices - 1)); last->storage_free = subspace_store_internal_node_free; node = last->storage; } Assert(subspace_store->max_items == 0 || node->descendants <= (size_t) subspace_store->max_items); /* * We only call this function on a cache miss, so number of leaves * will definitely increase see `Assert(last != NULL && last->storage * == NULL);` at bottom. */ node->descendants += 1; Assert(0 == node->vector->num_slices || node->vector->slices[0]->fd.dimension_id == target->fd.dimension_id); /* Do we have enough space to store the object? */ if (subspace_store->max_items > 0 && node->descendants > subspace_store->max_items) { /* * Always delete the slice corresponding to the earliest time * range. In the normal case that inserts are performed in * time-order this is the one least likely to be reused. (Note * that we made sure that the first dimension is a time dimension * when creating the subspace_store). If out-of-order inserts are * become significant we may wish to change this to something more * sophisticated like LRU. */ size_t items_removed = subspace_store_internal_node_descendants(node, i); /* * descendants at the root is inclusive of the descendants at the * children, so if we have an overflow it must be in the time dim */ Assert(i == 0); Assert(subspace_store->max_items + 1 == node->descendants); ts_dimension_vec_remove_slice(&node->vector, i); /* * Note we would have to do this to ancestors if this was not the * root. */ node->descendants -= items_removed; } match = ts_dimension_vec_find_slice(node->vector, target->fd.range_start); /* Do we have a slot in this vector for the new object? */ if (match == NULL) { DimensionSlice *copy; /* * create a new copy of the range this slice covers, to store the * object in */ copy = ts_dimension_slice_copy(target); ts_dimension_vec_add_slice_sort(&node->vector, copy); match = copy; } Assert(subspace_store->max_items == 0 || node->descendants <= (size_t) subspace_store->max_items); last = match; /* internal slices point to the next SubspaceStoreInternalNode */ node = last->storage; } Assert(last != NULL && last->storage == NULL); last->storage = object; /* at the end we store the object */ last->storage_free = object_free; MemoryContextSwitchTo(old); } void * ts_subspace_store_get(const SubspaceStore *subspace_store, const Point *target) { int i; DimensionVec *vec = subspace_store->origin->vector; DimensionSlice *match = NULL; Assert(target->cardinality == subspace_store->num_dimensions); /* The internal compressed hypertable has no dimensions as * chunks are created explicitly by compress_chunk and linked * to the source chunk. */ if (subspace_store->num_dimensions == 0) return NULL; for (i = 0; i < target->cardinality; i++) { match = ts_dimension_vec_find_slice(vec, target->coordinates[i]); if (NULL == match) return NULL; vec = ((SubspaceStoreInternalNode *) match->storage)->vector; } Assert(match != NULL); return match->storage; } void ts_subspace_store_free(SubspaceStore *subspace_store) { subspace_store_internal_node_free(subspace_store->origin); pfree(subspace_store); } MemoryContext ts_subspace_store_mcxt(const SubspaceStore *subspace_store) { return subspace_store->mcxt; } ================================================ FILE: src/subspace_store.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include "dimension.h" /* A subspace store allows you to save data associated with * a multidimensional-subspace. Subspaces are defined conceptually * via a Hypercube (that is a collection of slices -- one for each dimension). * Thus, a subspace is a "rectangular" cutout in a multidimensional space. */ typedef struct Hypercube Hypercube; typedef struct Point Point; typedef struct SubspaceStore SubspaceStore; extern SubspaceStore *ts_subspace_store_init(const Hyperspace *space, MemoryContext mcxt, int16 max_items); /* Store an object associate with the subspace represented by a hypercube */ extern void ts_subspace_store_add(SubspaceStore *subspace_store, const Hypercube *hypercube, void *object, void (*object_free)(void *)); /* Get the object stored for the subspace that a point is in. * Return the object stored or NULL if this subspace is not in the store. */ extern void *ts_subspace_store_get(const SubspaceStore *subspace_store, const Point *target); extern void ts_subspace_store_free(SubspaceStore *subspace_store); extern MemoryContext ts_subspace_store_mcxt(const SubspaceStore *subspace_store); ================================================ FILE: src/telemetry/CMakeLists.txt ================================================ # Add all *.c to sources in upperlevel directory set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/functions.c ${CMAKE_CURRENT_SOURCE_DIR}/replication.c ${CMAKE_CURRENT_SOURCE_DIR}/stats.c ${CMAKE_CURRENT_SOURCE_DIR}/telemetry_metadata.c ${CMAKE_CURRENT_SOURCE_DIR}/telemetry.c) target_sources(${PROJECT_NAME} PRIVATE ${SOURCES}) ================================================ FILE: src/telemetry/functions.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <fmgr.h> #include <access/genam.h> #include <access/htup_details.h> #include <access/table.h> #include <catalog/indexing.h> #include <catalog/pg_depend.h> #include <catalog/pg_extension.h> #include <catalog/pg_proc.h> #include <commands/extension.h> #include <nodes/nodeFuncs.h> #include <port/atomics.h> #include <storage/lwlock.h> #include <utils/fmgroids.h> #include <utils/hsearch.h> #include <utils/regproc.h> #include "functions.h" #include "guc.h" #include "loader/function_telemetry.h" static bool skip_telemetry = false; static LWLock *function_counts_lock = NULL; static HTAB *function_counts; /*************************** * Telemetry draining code * ***************************/ typedef struct AllowedFnHashEntry { Oid fn; } AllowedFnHashEntry; // Get a HTAB of AllowedFnHashEntrys containing all and only those functions // that are within visible_extensions. This function should be equivalent to // the SQL // SELECT objid // FROM pg_catalog.pg_depend, pg_catalog.pg_extension extension // WHERE refclassid = 'pg_catalog.pg_extension'::pg_catalog.regclass // AND refobjid = extension.oid // AND deptype = 'e' // AND extname IN ('timescaledb','promscale','timescaledb_toolkit') // AND classid = 'pg_catalog.pg_proc'::regclass; static HTAB * allowed_extension_functions(const char **visible_extensions, int num_visible_extensions) { HASHCTL hash_info = { .keysize = sizeof(Oid), .entrysize = sizeof(AllowedFnHashEntry), .hcxt = CurrentMemoryContext, }; HTAB *allowed_fns = hash_create("fn telemetry allowed_functions", 1000, &hash_info, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); Relation depRel = table_open(DependRelationId, AccessShareLock); Oid *visible_extension_ids = palloc(num_visible_extensions * sizeof(Oid)); // get the Oid for each of the visible extensions for (int i = 0; i < num_visible_extensions; i++) visible_extension_ids[i] = get_extension_oid(visible_extensions[i], true); // go through the objects owned by each visible extension, and store the // ones that are functions in the set. for (int i = 0; i < num_visible_extensions; i++) { HeapTuple tup; ScanKeyData key[2]; SysScanDesc scan; Oid extension_id = visible_extension_ids[i]; if (!OidIsValid(extension_id)) continue; // Look in the (referenced object class, referenced object) index for // the allowed extensions. ScanKeyInit(&key[0], Anum_pg_depend_refclassid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(ExtensionRelationId)); ScanKeyInit(&key[1], Anum_pg_depend_refobjid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(extension_id)); scan = systable_beginscan(depRel, DependReferenceIndexId, true, NULL, 2, key); while (HeapTupleIsValid(tup = systable_getnext(scan))) { Form_pg_depend deprec = (Form_pg_depend) GETSTRUCT(tup); // Filter for those objects that have an extension dependencies // exist in pg_proc, those are the functions that live in the extension if (deprec->deptype == 'e' && deprec->classid == ProcedureRelationId) { AllowedFnHashEntry *entry = hash_search(allowed_fns, &deprec->objid, HASH_ENTER, NULL); entry->fn = deprec->objid; } } systable_endscan(scan); } table_close(depRel, AccessShareLock); return allowed_fns; } static fn_telemetry_entry_vec * read_shared_map() { HASH_SEQ_STATUS hash_seq; long i; long num_entries = hash_get_num_entries(function_counts); fn_telemetry_entry_vec *entries = fn_telemetry_entry_vec_create(CurrentMemoryContext, num_entries); LWLockAcquire(function_counts_lock, LW_SHARED); hash_seq_init(&hash_seq, function_counts); // limit to num_entries so we don't hold the lock for a realloc for (i = 0; i < num_entries; i++) { FnTelemetryEntry entry; FnTelemetryHashEntry *hash_entry = hash_seq_search(&hash_seq); if (!hash_entry) break; entry.fn = hash_entry->key; /* * We never remove entries here, merely set their counts to 0. At * steady-state we expect the functions used by most workloads to be * effectively constant, so by keeping the hashmap entries allocated we * reduce contention during the telemetry-gathering stage. If memory * usage become an issue we can delete based off some heuristic, eg. if * the count starts out as 0. */ entry.count = pg_atomic_read_u64(&hash_entry->count); if (entry.count != 0) fn_telemetry_entry_vec_append(entries, entry); } if (i == num_entries) hash_seq_term(&hash_seq); LWLockRelease(function_counts_lock); return entries; } /* * Read the function telemetry shared-memory hashmap for telemetry send. * * This function gathers (function_id, count) pairs from the shared hashmap, * and filters the set for the functions we're allowed to send back. * * In general, we should never send telemetry information about any functions * except for core functions and those is a specified list of extensions * (when originally written, the set of related_extensions along with * timescaledb itself), so this function is designed to make it difficult to do * so. * * @param visible_extensions list of extensions whose functions should be * returned * @param num_visible_extensions length of visible_extensions * @return vector of FnTelemetryEntry containing (function_id, count)s for the * functions in visible_extensions. * */ fn_telemetry_entry_vec * ts_function_telemetry_read(const char **visible_extensions, int num_visible_extensions) { fn_telemetry_entry_vec *entries_to_send; fn_telemetry_entry_vec *all_entries; HTAB *allowed_ext_fns; if (function_counts == NULL) { FnTelemetryRendezvous **rendezvous = (FnTelemetryRendezvous **) find_rendezvous_variable(RENDEZVOUS_FUNCTION_TELEMENTRY); if (*rendezvous == NULL) return NULL; function_counts = (*rendezvous)->function_counts; function_counts_lock = (*rendezvous)->lock; } all_entries = read_shared_map(); entries_to_send = fn_telemetry_entry_vec_create(CurrentMemoryContext, all_entries->num_elements); allowed_ext_fns = allowed_extension_functions(visible_extensions, num_visible_extensions); for (uint32 i = 0; i < all_entries->num_elements; i++) { FnTelemetryEntry *entry = fn_telemetry_entry_vec_at(all_entries, i); bool is_builtin = entry->fn >= 1 && entry->fn <= 9999; bool is_visible = is_builtin || hash_search(allowed_ext_fns, &entry->fn, HASH_FIND, NULL); if (is_visible) fn_telemetry_entry_vec_append(entries_to_send, *entry); } return entries_to_send; } /* * Reset the counts in the function telemetry shared-memory hashmap. * * This function resets the shared function counts after we send back telemetry * in preparation for the next recording cycle. Note that there is no way to * atomically read and reset the counts in the shared hashmap, so writes that * occur between sending the old counts and resetting for the next cycle will be * lost. Since this this telemetry is only ever an approximation of reality, we * believe this loss is acceptable considering that the alternatives are * resetting the counts whenever the telemetry is read (potentially even more * lossy), or holding the lock for the entire telemetry send (to long a * contention window). */ void ts_function_telemetry_reset_counts() { HASH_SEQ_STATUS hash_seq; if (!function_counts) return; LWLockAcquire(function_counts_lock, LW_SHARED); hash_seq_init(&hash_seq, function_counts); // limit to num_entries so we don't hold the lock for a realloc while (true) { FnTelemetryHashEntry *hash_entry = hash_seq_search(&hash_seq); if (!hash_entry) break; /* * We never remove entries here, merely set their counts to 0. At * steady-state we expect the functions used by most workloads to be * effectively constant, so by keeping the hashmap entries allocated we * reduce contention during the telemetry-gathering stage. If memory * usage become an issue we can delete based off some heuristic, eg. if * the count starts out as 0, though we will have to use a stronger * lock. */ pg_atomic_write_u64(&hash_entry->count, 0); } LWLockRelease(function_counts_lock); } /**************************** * Telemetry gathering code * ****************************/ static bool function_telemetry_increment(Oid func_id, HTAB **local_counts) { FnTelemetryEntry *entry; bool found; // if this is the first function we've seen initialize local_counts if (!*local_counts) { HASHCTL hash_info = { .keysize = sizeof(Oid), .entrysize = sizeof(FnTelemetryEntry), .hcxt = CurrentMemoryContext, }; *local_counts = hash_create("fn telemetry local function hash", 10, &hash_info, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); } entry = hash_search(*local_counts, &func_id, HASH_ENTER, &found); if (!found) entry->count = 0; entry->count += 1; return true; } static bool function_gather_checker(Oid func_id, void *context) { function_telemetry_increment(func_id, (HTAB **) context); return false; } static bool function_gather_walker(Node *node, void *context) { bool end_early; if (node == NULL) return false; end_early = check_functions_in_node(node, function_gather_checker, context); if (end_early) return true; if (IsA(node, Query)) { /* Recurse into subselects */ return query_tree_walker((Query *) node, function_gather_walker, context, 0); } return expression_tree_walker(node, function_gather_walker, context); } static HTAB * record_function_counts(Query *query) { HTAB *query_function_counts = NULL; query_tree_walker(query, function_gather_walker, (void *) &query_function_counts, 0); return query_function_counts; } /* * Store a map of (function_oid, count) into shared memory so it can be seen by * the telemetry worker. This insertion works in two phases: * 1. Under a SHARED lock, we increment the counts of all those functions that * are already present in the map, using atomic fetch-add to prevent races. * 2. Under an EXCLUSIVE lock, we insert entries for all those functions that * were not already in the map. * At steady state we expect that vast majority the time all the functions a * query uses will already be in the shared map, so this strategy should * minimize contention between queries. * * @param query_function_counts A hashtable of FnTelemetryEntry storing * function usage counts */ static void store_function_counts_in_shared_mem(HTAB *query_function_counts) { HASH_SEQ_STATUS hash_seq; FnTelemetryEntry *local_entry = NULL; fn_telemetry_entry_vec missing_entries; fn_telemetry_entry_vec_init(&missing_entries, CurrentMemoryContext, 0); /* * Increment the counts of any functions already in the table under a * shared lock; the atomicity of increments will handle concurrency. */ LWLockAcquire(function_counts_lock, LW_SHARED); hash_seq_init(&hash_seq, query_function_counts); while ((local_entry = hash_seq_search(&hash_seq))) { FnTelemetryHashEntry *shared_entry = hash_search(function_counts, &local_entry->fn, HASH_FIND, NULL); if (shared_entry) pg_atomic_fetch_add_u64(&shared_entry->count, local_entry->count); else fn_telemetry_entry_vec_append(&missing_entries, *local_entry); } LWLockRelease(function_counts_lock); /* * If any functions did not have an entries create them under an * exclusive lock */ if (missing_entries.num_elements > 0) { LWLockAcquire(function_counts_lock, LW_EXCLUSIVE); for (uint32 i = 0; i < missing_entries.num_elements; i++) { bool found = false; FnTelemetryEntry *missing_entry = fn_telemetry_entry_vec_at(&missing_entries, i); FnTelemetryHashEntry *shared_entry = hash_search(function_counts, &missing_entry->fn, HASH_ENTER_NULL, &found); if (!shared_entry) break; if (found) pg_atomic_fetch_add_u64(&shared_entry->count, missing_entry->count); else pg_atomic_init_u64(&shared_entry->count, missing_entry->count); } LWLockRelease(function_counts_lock); } } /* * Gather function usage telemetry for a query. * * This function walks a query looking for function Oids, counts their * occurrence, and stores the (function_id, count) set into the shared-memory * function telemetry hashtable for later processing by the telemetry background * worker. */ void ts_telemetry_function_info_gather(Query *query) { HTAB *query_function_counts; if (skip_telemetry || !ts_function_telemetry_on()) return; // At the first time through initialize the shared state if (function_counts == NULL) { FnTelemetryRendezvous **rendezvous = (FnTelemetryRendezvous **) find_rendezvous_variable(RENDEZVOUS_FUNCTION_TELEMENTRY); if (*rendezvous == NULL) { skip_telemetry = true; return; } function_counts = (*rendezvous)->function_counts; function_counts_lock = (*rendezvous)->lock; } query_function_counts = record_function_counts(query); if (query_function_counts) store_function_counts_in_shared_mem(query_function_counts); } ================================================ FILE: src/telemetry/functions.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> typedef struct FnTelemetryEntry { Oid fn; uint64 count; } FnTelemetryEntry; #define VEC_PREFIX fn_telemetry_entry #define VEC_ELEMENT_TYPE FnTelemetryEntry #define VEC_DECLARE 1 #define VEC_DEFINE 1 #define VEC_SCOPE static inline #include <adts/vec.h> extern void ts_telemetry_function_info_gather(Query *query); extern fn_telemetry_entry_vec *ts_function_telemetry_read(const char **visible_extensions, int num_visible_extensions); extern void ts_function_telemetry_reset_counts(void); ================================================ FILE: src/telemetry/replication.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <executor/spi.h> #include <utils/guc.h> #include "replication.h" ReplicationInfo ts_telemetry_replication_info_gather(void) { int res; bool isnull; Datum data; ReplicationInfo info = { .got_num_wal_senders = false, .got_is_wal_receiver = false, }; if (SPI_connect() != SPI_OK_CONNECT) return info; /* Lock down search_path */ int save_nestlevel = NewGUCNestLevel(); RestrictSearchPath(); res = SPI_execute("SELECT cast(count(pid) as int) from pg_catalog.pg_stat_get_wal_senders() " "WHERE pid is not null", true, /* read_only */ 0 /*count*/ ); if (res >= 0) { data = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull); info.num_wal_senders = DatumGetInt32(data); info.got_num_wal_senders = true; } /* use count() > 0 in case they start having pg_stat_get_wal_receiver() * return no rows when the DB isn't a replica */ res = SPI_execute("SELECT count(pid) > 0 from pg_catalog.pg_stat_get_wal_receiver() WHERE pid " "is not null", true, /* read_only */ 0 /*count*/ ); if (res >= 0) { data = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull); info.is_wal_receiver = DatumGetBool(data); info.got_is_wal_receiver = true; } res = SPI_finish(); if (res != SPI_OK_FINISH) elog(ERROR, "SPI_finish failed: %s", SPI_result_code_string(res)); /* Restore search_path */ AtEOXact_GUC(false, save_nestlevel); return info; } ================================================ FILE: src/telemetry/replication.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include "utils.h" typedef struct ReplicationInfo { bool got_num_wal_senders; int32 num_wal_senders; bool got_is_wal_receiver; bool is_wal_receiver; } ReplicationInfo; extern ReplicationInfo ts_telemetry_replication_info_gather(void); ================================================ FILE: src/telemetry/stats.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/genam.h> #include <access/htup_details.h> #include <access/table.h> #include <access/tableam.h> #include <catalog/indexing.h> #include <catalog/namespace.h> #include <catalog/pg_class.h> #include <catalog/pg_namespace.h> #include <fmgr.h> #include <storage/lmgr.h> #include <utils/builtins.h> #include <utils/snapmgr.h> #include <utils/syscache.h> #include "chunk.h" #include "debug_point.h" #include "extension.h" #include "hypertable_cache.h" #include "stats.h" #include "ts_catalog/catalog.h" #include "ts_catalog/continuous_agg.h" #include "utils.h" typedef struct StatsContext { TelemetryStats *stats; Snapshot snapshot; } StatsContext; /* * Determine the type of a hypertable. */ static StatsRelType classify_hypertable(const Hypertable *ht) { if (TS_HYPERTABLE_IS_INTERNAL_COMPRESSION_TABLE(ht)) { /* * This is an internal compression table, but could be for a * regular hypertable, or for an internal materialized * hypertable (cagg). The latter case is currently not handled */ return RELTYPE_COMPRESSION_HYPERTABLE; } else { /* * Not dealing with an internal compression hypertable, but * could be a materialized hypertable (cagg). */ return RELTYPE_HYPERTABLE; } } static StatsRelType classify_chunk(Cache *htcache, const Hypertable **ht, const Chunk *chunk) { StatsRelType parent_reltype; Assert(NULL != chunk); /* Classify the chunk's parent */ *ht = ts_hypertable_cache_get_entry(htcache, chunk->hypertable_relid, CACHE_FLAG_MISSING_OK); Assert(NULL != *ht); parent_reltype = classify_hypertable(*ht); /* Classify the chunk's parent */ switch (parent_reltype) { case RELTYPE_HYPERTABLE: return RELTYPE_CHUNK; case RELTYPE_MATERIALIZED_HYPERTABLE: return RELTYPE_MATERIALIZED_CHUNK; case RELTYPE_COMPRESSION_HYPERTABLE: return RELTYPE_COMPRESSION_CHUNK; default: /* Shouldn't really get here */ return RELTYPE_OTHER; } } static StatsRelType classify_table(const Form_pg_class class, Cache *htcache, const Hypertable **ht, const Chunk **chunk) { Assert(class->relkind == RELKIND_RELATION); if (class->relispartition) return RELTYPE_PARTITION; /* Check if it is a hypertable */ *ht = ts_hypertable_cache_get_entry(htcache, class->oid, CACHE_FLAG_MISSING_OK); if (*ht) return classify_hypertable(*ht); /* Check if it is a chunk */ *chunk = ts_chunk_get_by_relid(class->oid, false); if (NULL != *chunk) return classify_chunk(htcache, ht, *chunk); return RELTYPE_TABLE; } static StatsRelType classify_partitioned_table(const Form_pg_class class) { Assert(class->relkind == RELKIND_PARTITIONED_TABLE); /* * If the partitioned table itself is a partition, then it is a partition * in a multi-dimensional partitioned table. Treat it as a partition so * that only "root" tables are counted as partitioned tables. */ if (class->relispartition) return RELTYPE_PARTITION; return RELTYPE_PARTITIONED_TABLE; } static StatsRelType classify_foreign_table(Cache *htcache, Oid relid, const Hypertable **ht, const Chunk **chunk) { *chunk = ts_chunk_get_by_relid(relid, false); if (*chunk) return classify_chunk(htcache, ht, *chunk); /* * Currently don't care about non-chunk foreign tables, so classify as * "other". */ return RELTYPE_OTHER; } static StatsRelType classify_view(const Form_pg_class class, Cache *htcache, const ContinuousAgg **cagg) { const Catalog *catalog = ts_catalog_get(); if (class->relnamespace == catalog->extension_schema_id[TS_INTERNAL_SCHEMA]) return RELTYPE_OTHER; *cagg = ts_continuous_agg_find_by_relid(class->oid); if (*cagg) return RELTYPE_CONTINUOUS_AGG; return RELTYPE_VIEW; } static StatsRelType classify_relation(const Form_pg_class class, Cache *htcache, const Hypertable **ht, const Chunk **chunk, const ContinuousAgg **cagg) { *chunk = NULL; *ht = NULL; *cagg = NULL; switch (class->relkind) { case RELKIND_RELATION: return classify_table(class, htcache, ht, chunk); case RELKIND_PARTITIONED_TABLE: return classify_partitioned_table(class); case RELKIND_FOREIGN_TABLE: return classify_foreign_table(htcache, class->oid, ht, chunk); case RELKIND_MATVIEW: return RELTYPE_MATVIEW; case RELKIND_VIEW: return classify_view(class, htcache, cagg); default: return RELTYPE_OTHER; } } static void add_storage(StorageStats *stats, Form_pg_class class) { RelationSize relsize; relsize = ts_relation_size_impl(class->oid); stats->relsize.total_size += relsize.total_size; stats->relsize.heap_size += relsize.heap_size; stats->relsize.toast_size += relsize.toast_size; stats->relsize.index_size += relsize.index_size; } static void process_relation(BaseStats *stats, Form_pg_class class) { stats->relcount++; /* * As of PG14, pg_class.reltuples is set to -1 when the row count is * unknown. Make sure we only add the count when the information is * available. */ if (class->reltuples > 0) stats->reltuples += class->reltuples; if (RELKIND_HAS_STORAGE(class->relkind)) add_storage((StorageStats *) stats, class); } static void process_hypertable(HyperStats *hyp, Form_pg_class class, const Hypertable *ht) { process_relation(&hyp->storage.base, class); if (TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(ht)) hyp->compressed_hypertable_count++; } static void process_continuous_agg(CaggStats *cs, Form_pg_class class, const ContinuousAgg *cagg) { const Hypertable *mat_ht = ts_hypertable_get_by_id(cagg->data.mat_hypertable_id); Assert(cagg); process_relation(&cs->hyp.storage.base, class); if (TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(mat_ht)) cs->hyp.compressed_hypertable_count++; if (!cagg->data.materialized_only) cs->uses_real_time_aggregation_count++; cs->finalized++; if (cagg->data.parent_mat_hypertable_id != INVALID_HYPERTABLE_ID) cs->nested++; } static void process_partition(HyperStats *stats, Form_pg_class class, bool ischunk) { stats->child_count++; /* * Note that reltuples should be correct even for compressed chunks, since * we "freeze" those stats when a chunk is compressed, and for foreign * table chunks, since we import those stats from data nodes. * * Also, as of PG14, the parent tables include the cumulative stats for * all children, so no need to count the partitions separately since the * sum will be in the root. */ if (ischunk && class->reltuples > 0) { stats->storage.base.reltuples += class->reltuples; } add_storage(&stats->storage, class); } /* * Add a chunk's stats to the parent table. */ static void add_chunk_stats(HyperStats *stats, Form_pg_class class, const Chunk *chunk, const Form_compression_chunk_size fd_compr) { process_partition(stats, class, true); if (ts_chunk_is_compressed(chunk)) stats->compressed_chunk_count++; if (fd_compr) { stats->compressed_heap_size += fd_compr->compressed_heap_size; stats->compressed_indexes_size += fd_compr->compressed_index_size; stats->compressed_toast_size += fd_compr->compressed_toast_size; stats->uncompressed_heap_size += fd_compr->uncompressed_heap_size; stats->uncompressed_indexes_size += fd_compr->uncompressed_index_size; stats->uncompressed_toast_size += fd_compr->uncompressed_toast_size; stats->uncompressed_row_count += fd_compr->numrows_pre_compression; stats->compressed_row_count += fd_compr->numrows_post_compression; stats->compressed_row_frozen_immediately_count += fd_compr->numrows_frozen_immediately; /* Also add compressed sizes to total number for entire table */ stats->storage.relsize.heap_size += fd_compr->compressed_heap_size; stats->storage.relsize.toast_size += fd_compr->compressed_toast_size; stats->storage.relsize.index_size += fd_compr->compressed_index_size; } } static bool get_chunk_compression_stats(StatsContext *statsctx, const Chunk *chunk, Form_compression_chunk_size compr_stats) { TupleInfo *ti; ScanIterator it; bool found = false; if (!ts_chunk_is_compressed(chunk)) return false; it = ts_scan_iterator_create(COMPRESSION_CHUNK_SIZE, AccessShareLock, CurrentMemoryContext); ts_scan_iterator_set_index(&it, COMPRESSION_CHUNK_SIZE, COMPRESSION_CHUNK_SIZE_PKEY); it.ctx.snapshot = statsctx->snapshot; ts_scan_iterator_scan_key_reset(&it); ts_scan_iterator_scan_key_init(&it, Anum_compression_chunk_size_pkey_chunk_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(chunk->fd.id)); ts_scan_iterator_start_or_restart_scan(&it); ti = ts_scan_iterator_next(&it); if (ti) { Form_compression_chunk_size fd; bool should_free; HeapTuple tuple = ts_scan_iterator_fetch_heap_tuple(&it, false, &should_free); fd = (Form_compression_chunk_size) GETSTRUCT(tuple); memcpy(compr_stats, fd, sizeof(*fd)); if (should_free) heap_freetuple(tuple); found = true; } ts_scan_iterator_close(&it); return found; } /* * Process a relation identified as being a chunk. * * The chunk could be part of a * * - Hypertable * - Distributed hypertable * - Distributed hypertable member * - Materialized hypertable (cagg) chunk * - Internal compression table for hypertable * - Internal compression table for materialized hypertable (cagg) * * Note that we want to count regular chunks and compressed chunks as part of * the same hypertable, although they are children of different tables * internally. The same applies to chunks that belong to a continuous * aggregate, although in that case there is actually a two-level indirection: * The main cagg view is the user-facing relation we'd like to collect stats * for, while its chunks are actually stored in a materialized hypertable, * and, in a second tier, in a compressed hypertable. */ static void process_chunk(StatsContext *statsctx, StatsRelType chunk_reltype, Form_pg_class class, const Chunk *chunk) { TelemetryStats *stats = statsctx->stats; FormData_compression_chunk_size comp_stats_data; Form_compression_chunk_size compr_stats = NULL; Assert(chunk); /* * Ignore compression chunks since we have a separate metadata table with * stats for them */ if (chunk_reltype == RELTYPE_COMPRESSION_CHUNK) return; if (get_chunk_compression_stats(statsctx, chunk, &comp_stats_data)) compr_stats = &comp_stats_data; switch (chunk_reltype) { case RELTYPE_CHUNK: add_chunk_stats(&stats->hypertables, class, chunk, compr_stats); break; case RELTYPE_MATERIALIZED_CHUNK: add_chunk_stats(&stats->continuous_aggs.hyp, class, chunk, compr_stats); break; default: pg_unreachable(); break; } } static bool is_pg_schema(Oid namespaceid) { static Oid information_schema_oid = InvalidOid; if (namespaceid == PG_CATALOG_NAMESPACE || namespaceid == PG_TOAST_NAMESPACE) return true; if (!OidIsValid(information_schema_oid)) information_schema_oid = get_namespace_oid("information_schema", false); return namespaceid == information_schema_oid; } static bool is_ts_schema(const Catalog *catalog, Oid namespaceid) { int i; for (i = 0; i < _TS_MAX_SCHEMA; i++) { if (namespaceid != catalog->extension_schema_id[TS_INTERNAL_SCHEMA] && namespaceid == catalog->extension_schema_id[i]) return true; } return false; } static bool should_ignore_relation(const Catalog *catalog, Form_pg_class class) { return (is_pg_schema(class->relnamespace) || isAnyTempNamespace(class->relnamespace) || is_ts_schema(catalog, class->relnamespace) || ts_is_catalog_table(class->oid)); } /* * Scan the entire pg_class catalog table for all relations. For each * relation, classify it and gather stats based on the classification. */ void ts_telemetry_stats_gather(TelemetryStats *stats) { const Catalog *catalog = ts_catalog_get(); Relation rel; SysScanDesc scan; Cache *htcache = ts_hypertable_cache_pin(); MemoryContext oldmcxt, relmcxt; StatsContext statsctx = { .stats = stats, .snapshot = GetActiveSnapshot(), }; MemSet(stats, 0, sizeof(*stats)); rel = table_open(RelationRelationId, AccessShareLock); scan = systable_beginscan(rel, ClassOidIndexId, false, NULL, 0, NULL); relmcxt = AllocSetContextCreate(CurrentMemoryContext, "RelationStats", ALLOCSET_DEFAULT_SIZES); while (true) { HeapTuple tup; Form_pg_class class; StatsRelType reltype; const Chunk *chunk = NULL; const Hypertable *ht = NULL; const ContinuousAgg *cagg = NULL; tup = systable_getnext(scan); if (!HeapTupleIsValid(tup)) break; class = (Form_pg_class) GETSTRUCT(tup); if (should_ignore_relation(catalog, class)) continue; /* Lock the relation to ensure it does not disappear while we process * it */ LockRelationOid(class->oid, AccessShareLock); /* Now that the lock is acquired, ensure the relation still * exists. Otherwise, ignore the relation and release the useless * lock. */ if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(class->oid))) { UnlockRelationOid(class->oid, AccessShareLock); continue; } /* * Use temporary per-relation memory context to not accumulate cruft * during processing of pg_class. */ oldmcxt = MemoryContextSwitchTo(relmcxt); MemoryContextReset(relmcxt); reltype = classify_relation(class, htcache, &ht, &chunk, &cagg); DEBUG_WAITPOINT("telemetry_classify_relation"); switch (reltype) { case RELTYPE_HYPERTABLE: Assert(NULL != ht); process_hypertable(&stats->hypertables, class, ht); break; case RELTYPE_TABLE: process_relation(&stats->tables.base, class); break; case RELTYPE_PARTITIONED_TABLE: process_relation(&stats->partitioned_tables.storage.base, class); break; case RELTYPE_CHUNK: case RELTYPE_COMPRESSION_CHUNK: case RELTYPE_MATERIALIZED_CHUNK: Assert(NULL != chunk); process_chunk(&statsctx, reltype, class, chunk); break; case RELTYPE_PARTITION: process_partition(&stats->partitioned_tables, class, false); break; case RELTYPE_VIEW: /* Filter internal cagg views */ if (class->relnamespace != catalog->extension_schema_id[TS_INTERNAL_SCHEMA]) process_relation(&stats->views, class); break; case RELTYPE_MATVIEW: process_relation(&stats->materialized_views.base, class); break; case RELTYPE_CONTINUOUS_AGG: Assert(NULL != cagg); process_continuous_agg(&stats->continuous_aggs, class, cagg); break; /* No stats collected for types below */ case RELTYPE_COMPRESSION_HYPERTABLE: case RELTYPE_MATERIALIZED_HYPERTABLE: case RELTYPE_OTHER: break; } UnlockRelationOid(class->oid, AccessShareLock); MemoryContextSwitchTo(oldmcxt); } systable_endscan(scan); table_close(rel, AccessShareLock); ts_cache_release(&htcache); MemoryContextDelete(relmcxt); } ================================================ FILE: src/telemetry/stats.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include "utils.h" typedef enum StatsRelType { RELTYPE_HYPERTABLE, RELTYPE_MATERIALIZED_HYPERTABLE, RELTYPE_COMPRESSION_HYPERTABLE, RELTYPE_CONTINUOUS_AGG, RELTYPE_TABLE, RELTYPE_PARTITIONED_TABLE, RELTYPE_PARTITION, RELTYPE_VIEW, RELTYPE_MATVIEW, RELTYPE_CHUNK, RELTYPE_COMPRESSION_CHUNK, RELTYPE_MATERIALIZED_CHUNK, RELTYPE_OTHER, } StatsRelType; typedef enum StatsType { STATS_TYPE_BASE, STATS_TYPE_STORAGE, STATS_TYPE_HYPER, STATS_TYPE_CAGG, } StatsType; typedef struct BaseStats { int64 relcount; int64 reltuples; } BaseStats; typedef struct StorageStats { BaseStats base; RelationSize relsize; } StorageStats; typedef struct HyperStats { StorageStats storage; int64 replicated_hypertable_count; int64 child_count; int64 replica_chunk_count; /* only includes "additional" replica chunks */ int64 compressed_chunk_count; int64 compressed_hypertable_count; int64 compressed_size; int64 compressed_heap_size; int64 compressed_indexes_size; int64 compressed_toast_size; int64 compressed_row_count; int64 compressed_row_frozen_immediately_count; int64 uncompressed_heap_size; int64 uncompressed_indexes_size; int64 uncompressed_toast_size; int64 uncompressed_row_count; } HyperStats; typedef struct CaggStats { HyperStats hyp; /* "hyper" as field name leads to name conflict on Windows compiler */ int64 uses_real_time_aggregation_count; int64 finalized; int64 nested; } CaggStats; typedef struct TelemetryStats { HyperStats hypertables; HyperStats partitioned_tables; StorageStats tables; StorageStats materialized_views; CaggStats continuous_aggs; BaseStats views; } TelemetryStats; typedef struct TelemetryJobStats { int64 total_runs; int64 total_successes; int64 total_failures; int64 total_crashes; int32 max_consecutive_failures; int32 max_consecutive_crashes; Interval *total_duration; Interval *total_duration_failures; } TelemetryJobStats; extern void ts_telemetry_stats_gather(TelemetryStats *stats); ================================================ FILE: src/telemetry/telemetry.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <catalog/pg_collation.h> #include <commands/extension.h> #include <fmgr.h> #include <miscadmin.h> #include <storage/ipc.h> #include <utils/builtins.h> #include <utils/json.h> #include <utils/jsonb.h> #include <utils/regproc.h> #include <utils/snapmgr.h> #include "compat/compat.h" #include "bgw_policy/policy.h" #include "config.h" #include "extension.h" #include "functions.h" #include "guc.h" #include "hypertable.h" #include "jsonb_utils.h" #include "license_guc.h" #include "net/http.h" #include "replication.h" #include "stats.h" #include "telemetry.h" #include "telemetry_metadata.h" #include "ts_catalog/compression_chunk_size.h" #include "ts_catalog/metadata.h" #include "version.h" #include "cross_module_fn.h" #include <executor/spi.h> #define TS_TELEMETRY_VERSION 2 #define TS_VERSION_JSON_FIELD "current_timescaledb_version" #define TS_IS_UPTODATE_JSON_FIELD "is_up_to_date" /* HTTP request details */ #define MAX_REQUEST_SIZE 4096 #define REQ_TELEMETRY_VERSION "telemetry_version" #define REQ_DB_UUID "db_uuid" #define REQ_EXPORTED_DB_UUID "exported_db_uuid" #define REQ_INSTALL_TIME "installed_time" #define REQ_INSTALL_METHOD "install_method" #define REQ_OS "os_name" #define REQ_OS_VERSION "os_version" #define REQ_OS_RELEASE "os_release" #define REQ_OS_VERSION_PRETTY "os_name_pretty" #define REQ_PS_VERSION "postgresql_version" #define REQ_TS_VERSION "timescaledb_version" #define REQ_BUILD_OS "build_os_name" #define REQ_BUILD_OS_VERSION "build_os_version" #define REQ_BUILD_ARCHITECTURE_BIT_SIZE "build_architecture_bit_size" #define REQ_BUILD_ARCHITECTURE "build_architecture" #define REQ_DATA_VOLUME "data_volume" #define REQ_NUM_POLICY_CAGG_FIXED "num_continuous_aggs_policies_fixed" #define REQ_NUM_POLICY_COMPRESSION_FIXED "num_compression_policies_fixed" #define REQ_NUM_POLICY_REORDER_FIXED "num_reorder_policies_fixed" #define REQ_NUM_POLICY_RETENTION_FIXED "num_retention_policies_fixed" #define REQ_NUM_USER_DEFINED_ACTIONS_FIXED "num_user_defined_actions_fixed" #define REQ_NUM_POLICY_CAGG "num_continuous_aggs_policies" #define REQ_NUM_POLICY_COMPRESSION "num_compression_policies" #define REQ_NUM_POLICY_REORDER "num_reorder_policies" #define REQ_NUM_POLICY_RETENTION "num_retention_policies" #define REQ_NUM_USER_DEFINED_ACTIONS "num_user_defined_actions" #define REQ_RELATED_EXTENSIONS "related_extensions" #define REQ_METADATA "db_metadata" #define REQ_TELEMETRY_EVENT "db_telemetry_events" #define REQ_LICENSE_EDITION_APACHE "apache_only" #define REQ_LICENSE_EDITION_COMMUNITY "community" #define REQ_TS_LAST_TUNE_TIME "last_tuned_time" #define REQ_TS_LAST_TUNE_VERSION "last_tuned_version" #define REQ_INSTANCE_METADATA "instance_metadata" #define REQ_TS_TELEMETRY_CLOUD "cloud" #define REQ_NUM_WAL_SENDERS "num_wal_senders" #define REQ_IS_WAL_RECEIVER "is_wal_receiver" #define PG_PROMETHEUS "pg_prometheus" #define PG_VECTOR "vector" #define TS_AI "ai" #define TS_VECTORSCALE "vectorscale" #define PROMSCALE "promscale" #define POSTGIS "postgis" #define TIMESCALE_ANALYTICS "timescale_analytics" #define TIMESCALEDB_TOOLKIT "timescaledb_toolkit" #define REQ_JOB_STATS_BY_JOB_TYPE "stats_by_job_type" #define REQ_NUM_ERR_BY_SQLERRCODE "errors_by_sqlerrcode" static const char *related_extensions[] = { PG_PROMETHEUS, PROMSCALE, POSTGIS, TIMESCALE_ANALYTICS, TIMESCALEDB_TOOLKIT, PG_VECTOR, TS_AI, TS_VECTORSCALE, }; /* This function counts background worker jobs by type. */ static BgwJobTypeCount bgw_job_type_counts() { ListCell *lc; List *jobs = ts_bgw_job_get_all(sizeof(BgwJob), CurrentMemoryContext); BgwJobTypeCount counts = { 0 }; foreach (lc, jobs) { BgwJob *job = lfirst(lc); if (namestrcmp(&job->fd.proc_schema, FUNCTIONS_SCHEMA_NAME) == 0) { if (namestrcmp(&job->fd.proc_name, "policy_refresh_continuous_aggregate") == 0) { if (job->fd.fixed_schedule) counts.policy_cagg_fixed++; else counts.policy_cagg++; } else if (namestrcmp(&job->fd.proc_name, "policy_compression") == 0) { if (job->fd.fixed_schedule) counts.policy_compression_fixed++; else counts.policy_compression++; } else if (namestrcmp(&job->fd.proc_name, "policy_reorder") == 0) { if (job->fd.fixed_schedule) counts.policy_reorder_fixed++; else counts.policy_reorder++; } else if (namestrcmp(&job->fd.proc_name, "policy_retention") == 0) { if (job->fd.fixed_schedule) counts.policy_retention_fixed++; else counts.policy_retention++; } else if (namestrcmp(&job->fd.proc_name, "policy_telemetry") == 0) counts.policy_telemetry++; } else { if (job->fd.fixed_schedule) counts.user_defined_action_fixed++; else counts.user_defined_action++; } } return counts; } static bool char_in_valid_version_digits(const char c) { switch (c) { case '.': case '-': return true; default: return false; } } /* * Makes sure the server version string is less than MAX_VERSION_STR_LEN * chars, and all digits are "valid". Valid chars are either * alphanumeric or in the array valid_version_digits above. * * Returns false if either of these conditions are false. */ bool ts_validate_server_version(const char *json, VersionResult *result) { Datum version = DirectFunctionCall2(json_object_field_text, CStringGetTextDatum(json), PointerGetDatum(cstring_to_text(TS_VERSION_JSON_FIELD))); memset(result, 0, sizeof(VersionResult)); result->versionstr = text_to_cstring(DatumGetTextPP(version)); if (result->versionstr == NULL) { result->errhint = "no version string in response"; return false; } if (strlen(result->versionstr) > MAX_VERSION_STR_LEN) { result->errhint = "version string is too long"; return false; } for (size_t i = 0; i < strlen(result->versionstr); i++) { if (!isalpha(result->versionstr[i]) && !isdigit(result->versionstr[i]) && !char_in_valid_version_digits(result->versionstr[i])) { result->errhint = "version string has invalid characters"; return false; } } return true; } /* * Parse the JSON response from the TS endpoint. There should be a field * called "current_timescaledb_version". Check this against the local * version, and notify the user if it is behind. */ void ts_check_version_response(const char *json) { VersionResult result; bool is_uptodate = DatumGetBool( DirectFunctionCall2Coll(texteq, C_COLLATION_OID, DirectFunctionCall2Coll(json_object_field_text, C_COLLATION_OID, CStringGetTextDatum(json), PointerGetDatum(cstring_to_text( TS_IS_UPTODATE_JSON_FIELD))), PointerGetDatum(cstring_to_text("true")))); if (is_uptodate) elog(NOTICE, "the \"%s\" extension is up-to-date", EXTENSION_NAME); else { if (!ts_validate_server_version(json, &result)) { elog(NOTICE, "server did not return a valid TimescaleDB version: %s", result.errhint); return; } ereport(LOG, (errmsg("the \"%s\" extension is not up-to-date", EXTENSION_NAME), errhint("The most up-to-date version is %s, the installed version is %s.", result.versionstr, TIMESCALEDB_VERSION_MOD))); } } static int32 get_architecture_bit_size() { return BUILD_POINTER_BYTES * 8; } static void add_job_counts(JsonbParseState *state) { BgwJobTypeCount counts = bgw_job_type_counts(); ts_jsonb_add_int32(state, REQ_NUM_POLICY_CAGG, counts.policy_cagg); ts_jsonb_add_int32(state, REQ_NUM_POLICY_CAGG_FIXED, counts.policy_cagg_fixed); ts_jsonb_add_int32(state, REQ_NUM_POLICY_COMPRESSION, counts.policy_compression); ts_jsonb_add_int32(state, REQ_NUM_POLICY_COMPRESSION_FIXED, counts.policy_compression_fixed); ts_jsonb_add_int32(state, REQ_NUM_POLICY_REORDER, counts.policy_reorder); ts_jsonb_add_int32(state, REQ_NUM_POLICY_REORDER_FIXED, counts.policy_reorder_fixed); ts_jsonb_add_int32(state, REQ_NUM_POLICY_RETENTION, counts.policy_retention); ts_jsonb_add_int32(state, REQ_NUM_POLICY_RETENTION_FIXED, counts.policy_retention_fixed); ts_jsonb_add_int32(state, REQ_NUM_USER_DEFINED_ACTIONS, counts.user_defined_action); ts_jsonb_add_int32(state, REQ_NUM_USER_DEFINED_ACTIONS_FIXED, counts.user_defined_action_fixed); } static JsonbValue * add_errors_by_sqlerrcode_internal(JsonbParseState *parse_state, const char *job_type, Jsonb *sqlerrs_jsonb) { JsonbIterator *it; JsonbIteratorToken type; JsonbValue val; JsonbValue *ret; JsonbValue key = { .type = jbvString, .val.string.val = pstrdup(job_type), .val.string.len = strlen(job_type), }; ret = pushJsonbValue(&parse_state, WJB_KEY, &key); ret = pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); /* we don't expect nested values here */ it = JsonbIteratorInit(&sqlerrs_jsonb->root); type = JsonbIteratorNext(&it, &val, true /*skip_nested*/); if (type != WJB_BEGIN_OBJECT) elog(ERROR, "invalid JSON format"); while ((type = JsonbIteratorNext(&it, &val, true))) { const char *errcode; int64 errcnt; if (type == WJB_END_OBJECT) break; else if (type == WJB_KEY) { errcode = pnstrdup(val.val.string.val, val.val.string.len); /* get the corresponding value for this key */ type = JsonbIteratorNext(&it, &val, true); if (type != WJB_VALUE) elog(ERROR, "unexpected jsonb type"); errcnt = DatumGetInt64(DirectFunctionCall1(numeric_int8, NumericGetDatum(val.val.numeric))); ts_jsonb_add_int64(parse_state, errcode, errcnt); } else elog(ERROR, "unexpected jsonb type"); } ret = pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); return ret; } /* this function queries the database through SPI and gets back a set of records that look like (job_type TEXT, jsonb_object_agg JSONB). For example, (user_defined_action, {"P0001": 2, "42883": 5}) (we are expecting about 6 rows depending on how we write the query and if we exclude any jobs) Then for each returned row adds a new kv pair to the jsonb, which looks like "job_type": {"errtype1": errcnt1, ...} */ static void add_errors_by_sqlerrcode(JsonbParseState *parse_state) { int res; StringInfoData command; MemoryContext orig_context = CurrentMemoryContext; const char *command_string = "SELECT " "job_type, jsonb_object_agg(sqlerrcode, count) " "FROM" "(" " SELECT (" " CASE " " WHEN proc_schema = \'_timescaledb_functions\'" " AND proc_name ~ " "\'^policy_(retention|compression|reorder|refresh_continuous_" "aggregate|telemetry|job_error_retention)$\' " " THEN proc_name " " ELSE \'user_defined_action\'" " END" " ) as job_type, " " sqlerrcode, " " pg_catalog.COUNT(*) " " FROM " " timescaledb_information.job_errors " " WHERE sqlerrcode IS NOT NULL " " GROUP BY job_type, sqlerrcode " " ORDER BY job_type" ") q " "GROUP BY q.job_type"; if (SPI_connect() != SPI_OK_CONNECT) elog(ERROR, "could not connect to SPI"); /* Lock down search_path */ int save_nestlevel = NewGUCNestLevel(); RestrictSearchPath(); initStringInfo(&command); appendStringInfoString(&command, command_string); res = SPI_execute(command.data, true /*read only*/, 0 /* count */); if (res < 0) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), (errmsg("could not get errors by sqlerrcode and job type")))); /* we expect about 6 rows returned, each row is a record (TEXT, JSONB) */ for (uint64 i = 0; i < SPI_processed; i++) { Datum record_jobtype, record_jsonb; bool isnull_jobtype, isnull_jsonb; record_jobtype = SPI_getbinval(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 1, &isnull_jobtype); if (isnull_jobtype) elog(ERROR, "null job type returned"); record_jsonb = SPI_getbinval(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 2, &isnull_jsonb); /* this jsonb looks like {"P0001": 32, "42883": 6} */ Jsonb *sqlerrs_jsonb = isnull_jsonb ? NULL : DatumGetJsonbP(record_jsonb); if (sqlerrs_jsonb == NULL) continue; /* the jsonb object cannot be created in the SPI context or it will be lost */ MemoryContext spi_context = MemoryContextSwitchTo(orig_context); add_errors_by_sqlerrcode_internal(parse_state, TextDatumGetCString(record_jobtype), sqlerrs_jsonb); MemoryContextSwitchTo(spi_context); } /* Restore search_path */ AtEOXact_GUC(false, save_nestlevel); res = SPI_finish(); Assert(res == SPI_OK_FINISH); } static JsonbValue * add_job_stats_internal(JsonbParseState *state, const char *job_type, TelemetryJobStats *stats) { JsonbValue key = { .type = jbvString, .val.string.val = pstrdup(job_type), .val.string.len = strlen(job_type), }; pushJsonbValue(&state, WJB_KEY, &key); pushJsonbValue(&state, WJB_BEGIN_OBJECT, NULL); ts_jsonb_add_int64(state, "total_runs", stats->total_runs); ts_jsonb_add_int64(state, "total_successes", stats->total_successes); ts_jsonb_add_int64(state, "total_failures", stats->total_failures); ts_jsonb_add_int64(state, "total_crashes", stats->total_crashes); ts_jsonb_add_int32(state, "max_consecutive_failures", stats->max_consecutive_failures); ts_jsonb_add_int32(state, "max_consecutive_crashes", stats->max_consecutive_crashes); ts_jsonb_add_interval(state, "total_duration", stats->total_duration); ts_jsonb_add_interval(state, "total_duration_failures", stats->total_duration_failures); return pushJsonbValue(&state, WJB_END_OBJECT, NULL); } static void add_job_stats_by_job_type(JsonbParseState *parse_state) { StringInfoData command; int res; MemoryContext orig_context = CurrentMemoryContext; SPITupleTable *tuptable = NULL; const char *command_string = "SELECT (" " CASE " " WHEN j.proc_schema = \'_timescaledb_functions\' AND j.proc_name ~ " "\'^policy_(retention|compression|reorder|refresh_continuous_aggregate|telemetry|job_stat_" "history_retention)$\' " " THEN j.proc_name::TEXT " " ELSE \'user_defined_action\' " " END" ") AS job_type, " " SUM(total_runs)::BIGINT AS total_runs, " " SUM(total_successes)::BIGINT AS total_successes, " " SUM(total_failures)::BIGINT AS total_failures, " " SUM(total_crashes)::BIGINT AS total_crashes, " " SUM(total_duration) AS total_duration, " " SUM(total_duration_failures) AS total_duration_failures, " " MAX(consecutive_failures) AS max_consecutive_failures, " " MAX(consecutive_crashes) AS max_consecutive_crashes " "FROM " " _timescaledb_internal.bgw_job_stat s " " JOIN _timescaledb_catalog.bgw_job j on j.id = s.job_id " "GROUP BY job_type " "ORDER BY job_type"; if (SPI_connect() != SPI_OK_CONNECT) elog(ERROR, "could not connect to SPI"); /* Lock down search_path */ int save_nestlevel = NewGUCNestLevel(); RestrictSearchPath(); initStringInfo(&command); appendStringInfoString(&command, command_string); res = SPI_execute(command.data, true /* read_only */, 0 /*count*/); if (res < 0) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), (errmsg("could not get job statistics by job type")))); /* * a row returned looks like this: * (job_type, total_runs, total_successes, total_failures, total_crashes, total_duration, * total_duration_failures, max_consec_fails, max_consec_crashes) * ("policy_telemetry", 12, 10, 1, 1, 00:00:11, 00:00:01, 1, 1) */ for (uint64 i = 0; i < SPI_processed; i++) { tuptable = SPI_tuptable; TupleDesc tupdesc = tuptable->tupdesc; Datum jobtype_datum; Datum total_runs, total_successes, total_failures, total_crashes; Datum total_duration, total_duration_failures, max_consec_crashes, max_consec_fails; bool isnull_jobtype, isnull_runs, isnull_successes, isnull_failures, isnull_crashes; bool isnull_duration, isnull_duration_failures, isnull_consec_crashes, isnull_consec_fails; jobtype_datum = SPI_getbinval(SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 1, &isnull_jobtype); if (isnull_jobtype) elog(ERROR, "null job type returned"); total_runs = SPI_getbinval(tuptable->vals[i], tupdesc, 2, &isnull_runs); total_successes = SPI_getbinval(tuptable->vals[i], tupdesc, 3, &isnull_successes); total_failures = SPI_getbinval(tuptable->vals[i], tupdesc, 4, &isnull_failures); total_crashes = SPI_getbinval(tuptable->vals[i], tupdesc, 5, &isnull_crashes); total_duration = SPI_getbinval(tuptable->vals[i], tupdesc, 6, &isnull_duration); total_duration_failures = SPI_getbinval(tuptable->vals[i], tupdesc, 7, &isnull_duration_failures); max_consec_fails = SPI_getbinval(tuptable->vals[i], tupdesc, 8, &isnull_consec_fails); max_consec_crashes = SPI_getbinval(tuptable->vals[i], tupdesc, 9, &isnull_consec_crashes); if (isnull_jobtype || isnull_runs || isnull_successes || isnull_failures || isnull_crashes || isnull_duration || isnull_consec_crashes || isnull_consec_fails) { elog(ERROR, "null record field returned"); } MemoryContext spi_context = MemoryContextSwitchTo(orig_context); TelemetryJobStats stats = { .total_runs = DatumGetInt64(total_runs), .total_successes = DatumGetInt64(total_successes), .total_failures = DatumGetInt64(total_failures), .total_crashes = DatumGetInt64(total_crashes), .max_consecutive_failures = DatumGetInt32(max_consec_fails), .max_consecutive_crashes = DatumGetInt32(max_consec_crashes), .total_duration = DatumGetIntervalP(total_duration), .total_duration_failures = DatumGetIntervalP(total_duration_failures) }; add_job_stats_internal(parse_state, TextDatumGetCString(jobtype_datum), &stats); MemoryContextSwitchTo(spi_context); } /* Restore search_path */ AtEOXact_GUC(false, save_nestlevel); res = SPI_finish(); Assert(res == SPI_OK_FINISH); } static int64 get_database_size() { return DatumGetInt64(DirectFunctionCall1(pg_database_size_oid, ObjectIdGetDatum(MyDatabaseId))); } static void add_related_extensions(JsonbParseState *state) { pushJsonbValue(&state, WJB_BEGIN_OBJECT, NULL); for (size_t i = 0; i < sizeof(related_extensions) / sizeof(char *); i++) { const char *ext = related_extensions[i]; ts_jsonb_add_bool(state, ext, OidIsValid(get_extension_oid(ext, true))); } pushJsonbValue(&state, WJB_END_OBJECT, NULL); } static char * get_pgversion_string() { StringInfoData buf; int major, patch; /* * We have to read the server version from GUC and not use any of * the macros. By using any of the macros we would get the version * the extension is compiled against instead of the version actually * running. */ char *server_version_num_guc = GetConfigOptionByName("server_version_num", NULL, false); long server_version_num = strtol(server_version_num_guc, NULL, 10); major = server_version_num / 10000; patch = server_version_num % 100; Assert(major >= PG_MAJOR_MIN); initStringInfo(&buf); appendStringInfo(&buf, "%d.%d", major, patch); return buf.data; } #define ISO8601_FORMAT "YYYY-MM-DD\"T\"HH24:MI:SSOF" static char * format_iso8601(Datum value) { return TextDatumGetCString( DirectFunctionCall2(timestamptz_to_char, value, CStringGetTextDatum(ISO8601_FORMAT))); } #define REQ_RELKIND_COUNT "num_relations" #define REQ_RELKIND_RELTUPLES "num_reltuples" #define REQ_RELKIND_HEAP_SIZE "heap_size" #define REQ_RELKIND_TOAST_SIZE "toast_size" #define REQ_RELKIND_INDEXES_SIZE "indexes_size" #define REQ_RELKIND_CHILDREN "num_children" #define REQ_RELKIND_REPLICA_CHUNKS "num_replica_chunks" #define REQ_RELKIND_COMPRESSED_CHUNKS "num_compressed_chunks" #define REQ_RELKIND_COMPRESSED_HYPERTABLES "num_compressed_hypertables" #define REQ_RELKIND_COMPRESSED_CAGGS "num_compressed_caggs" #define REQ_RELKIND_UNCOMPRESSED_HEAP_SIZE "uncompressed_heap_size" #define REQ_RELKIND_UNCOMPRESSED_TOAST_SIZE "uncompressed_toast_size" #define REQ_RELKIND_UNCOMPRESSED_INDEXES_SIZE "uncompressed_indexes_size" #define REQ_RELKIND_UNCOMPRESSED_ROWCOUNT "uncompressed_row_count" #define REQ_RELKIND_COMPRESSED_HEAP_SIZE "compressed_heap_size" #define REQ_RELKIND_COMPRESSED_TOAST_SIZE "compressed_toast_size" #define REQ_RELKIND_COMPRESSED_INDEXES_SIZE "compressed_indexes_size" #define REQ_RELKIND_COMPRESSED_ROWCOUNT "compressed_row_count" #define REQ_RELKIND_COMPRESSED_ROWCOUNT_FROZEN_IMMEDIATELY "compressed_row_count_frozen_immediately" #define REQ_RELKIND_CAGG_USES_REAL_TIME_AGGREGATION_COUNT "num_caggs_using_real_time_aggregation" #define REQ_RELKIND_CAGG_FINALIZED "num_caggs_finalized" #define REQ_RELKIND_CAGG_NESTED "num_caggs_nested" static JsonbValue * add_compression_stats_object(JsonbParseState *parse_state, StatsRelType reltype, const HyperStats *hs) { JsonbValue name = { .type = jbvString, .val.string.val = pstrdup("compression"), .val.string.len = strlen("compression"), }; pushJsonbValue(&parse_state, WJB_KEY, &name); pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); ts_jsonb_add_int64(parse_state, REQ_RELKIND_COMPRESSED_CHUNKS, hs->compressed_chunk_count); if (reltype == RELTYPE_CONTINUOUS_AGG) ts_jsonb_add_int64(parse_state, REQ_RELKIND_COMPRESSED_CAGGS, hs->compressed_hypertable_count); else ts_jsonb_add_int64(parse_state, REQ_RELKIND_COMPRESSED_HYPERTABLES, hs->compressed_hypertable_count); ts_jsonb_add_int64(parse_state, REQ_RELKIND_COMPRESSED_ROWCOUNT, hs->compressed_row_count); ts_jsonb_add_int64(parse_state, REQ_RELKIND_COMPRESSED_HEAP_SIZE, hs->compressed_heap_size); ts_jsonb_add_int64(parse_state, REQ_RELKIND_COMPRESSED_TOAST_SIZE, hs->compressed_toast_size); ts_jsonb_add_int64(parse_state, REQ_RELKIND_COMPRESSED_INDEXES_SIZE, hs->compressed_indexes_size); ts_jsonb_add_int64(parse_state, REQ_RELKIND_COMPRESSED_ROWCOUNT_FROZEN_IMMEDIATELY, hs->compressed_row_frozen_immediately_count); ts_jsonb_add_int64(parse_state, REQ_RELKIND_UNCOMPRESSED_ROWCOUNT, hs->uncompressed_row_count); ts_jsonb_add_int64(parse_state, REQ_RELKIND_UNCOMPRESSED_HEAP_SIZE, hs->uncompressed_heap_size); ts_jsonb_add_int64(parse_state, REQ_RELKIND_UNCOMPRESSED_TOAST_SIZE, hs->uncompressed_toast_size); ts_jsonb_add_int64(parse_state, REQ_RELKIND_UNCOMPRESSED_INDEXES_SIZE, hs->uncompressed_indexes_size); return pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); } static JsonbValue * add_relkind_stats_object(JsonbParseState *parse_state, const char *relkindname, const BaseStats *stats, StatsRelType reltype, StatsType statstype) { JsonbValue name = { .type = jbvString, .val.string.val = pstrdup(relkindname), .val.string.len = strlen(relkindname), }; pushJsonbValue(&parse_state, WJB_KEY, &name); pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); ts_jsonb_add_int64(parse_state, REQ_RELKIND_COUNT, stats->relcount); if (statstype >= STATS_TYPE_STORAGE) { const StorageStats *ss = (const StorageStats *) stats; ts_jsonb_add_int64(parse_state, REQ_RELKIND_RELTUPLES, stats->reltuples); ts_jsonb_add_int64(parse_state, REQ_RELKIND_HEAP_SIZE, ss->relsize.heap_size); ts_jsonb_add_int64(parse_state, REQ_RELKIND_TOAST_SIZE, ss->relsize.toast_size); ts_jsonb_add_int64(parse_state, REQ_RELKIND_INDEXES_SIZE, ss->relsize.index_size); } if (statstype >= STATS_TYPE_HYPER) { const HyperStats *hs = (const HyperStats *) stats; ts_jsonb_add_int64(parse_state, REQ_RELKIND_CHILDREN, hs->child_count); if (reltype != RELTYPE_PARTITIONED_TABLE) add_compression_stats_object(parse_state, reltype, hs); } if (statstype == STATS_TYPE_CAGG) { const CaggStats *cs = (const CaggStats *) stats; ts_jsonb_add_int64(parse_state, REQ_RELKIND_CAGG_USES_REAL_TIME_AGGREGATION_COUNT, cs->uses_real_time_aggregation_count); ts_jsonb_add_int64(parse_state, REQ_RELKIND_CAGG_FINALIZED, cs->finalized); ts_jsonb_add_int64(parse_state, REQ_RELKIND_CAGG_NESTED, cs->nested); } return pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); } static void add_function_call_telemetry(JsonbParseState *state) { fn_telemetry_entry_vec *functions; const char *visible_extensions[(sizeof(related_extensions) / sizeof(char *)) + 1]; if (!ts_function_telemetry_on()) { JsonbValue value = { .type = jbvNull, }; pushJsonbValue(&state, WJB_VALUE, &value); return; } visible_extensions[0] = EXTENSION_NAME; for (size_t i = 1; i < sizeof(visible_extensions) / sizeof(char *); i++) visible_extensions[i] = related_extensions[i - 1]; functions = ts_function_telemetry_read(visible_extensions, sizeof(visible_extensions) / sizeof(char *)); pushJsonbValue(&state, WJB_BEGIN_OBJECT, NULL); if (functions) { for (uint32 i = 0; i < functions->num_elements; i++) { FnTelemetryEntry *entry = fn_telemetry_entry_vec_at(functions, i); char *proc_sig = format_procedure_qualified(entry->fn); ts_jsonb_add_int64(state, proc_sig, entry->count); } } pushJsonbValue(&state, WJB_END_OBJECT, NULL); } static void add_replication_telemetry(JsonbParseState *state) { ReplicationInfo info = ts_telemetry_replication_info_gather(); if (info.got_num_wal_senders) ts_jsonb_add_int32(state, REQ_NUM_WAL_SENDERS, info.num_wal_senders); if (info.got_is_wal_receiver) ts_jsonb_add_bool(state, REQ_IS_WAL_RECEIVER, info.is_wal_receiver); } #define REQ_RELS "relations" #define REQ_RELS_TABLES "tables" #define REQ_RELS_PARTITIONED_TABLES "partitioned_tables" #define REQ_RELS_MATVIEWS "materialized_views" #define REQ_RELS_VIEWS "views" #define REQ_RELS_HYPERTABLES "hypertables" #define REQ_RELS_CONTINUOUS_AGGS "continuous_aggregates" #define REQ_FUNCTIONS_USED "functions_used" #define REQ_REPLICATION "replication" #define REQ_ACCESS_METHODS "access_methods" /* * Add the result of a query as a sub-object to the JSONB. * * Each row from the query generates a separate object keyed by one of the * columns. Each row will be represented as an object and stored under the * "key" column. For example, with this query: * * select amname as name, * sum(relpages) as pages, * count(*) as instances * from pg_class join pg_am on relam = pg_am.oid * group by pg_am.oid; * * might generate the object * * { * "brin" : { * "instances" : 44, * "pages" : 432 * }, * "btree" : { * "instances" : 99, * "pages" : 1234 * } * } */ static void add_query_result_dict(JsonbParseState *state, const char *query) { MemoryContext orig_context = CurrentMemoryContext; int res; if (SPI_connect() != SPI_OK_CONNECT) elog(ERROR, "could not connect to SPI"); /* Lock down search_path */ int save_nestlevel = NewGUCNestLevel(); RestrictSearchPath(); res = SPI_execute(query, true, 0); Ensure(res >= 0, "could not execute query"); MemoryContext spi_context = MemoryContextSwitchTo(orig_context); (void) pushJsonbValue(&state, WJB_BEGIN_OBJECT, NULL); for (uint64 r = 0; r < SPI_processed; r++) { char *key_string = SPI_getvalue(SPI_tuptable->vals[r], SPI_tuptable->tupdesc, 1); JsonbValue key = { .type = jbvString, .val.string.val = pstrdup(key_string), .val.string.len = strlen(key_string), }; (void) pushJsonbValue(&state, WJB_KEY, &key); (void) pushJsonbValue(&state, WJB_BEGIN_OBJECT, NULL); for (int c = 1; c < SPI_tuptable->tupdesc->natts; ++c) { bool isnull; Datum val_datum = SPI_getbinval(SPI_tuptable->vals[r], SPI_tuptable->tupdesc, c + 1, &isnull); if (!isnull) { char *key_string = SPI_fname(SPI_tuptable->tupdesc, c + 1); JsonbValue value; ts_jsonb_set_value_by_type(&value, SPI_gettypeid(SPI_tuptable->tupdesc, c + 1), val_datum); ts_jsonb_add_value(state, key_string, &value); } } pushJsonbValue(&state, WJB_END_OBJECT, NULL); } /* Restore search_path */ AtEOXact_GUC(false, save_nestlevel); MemoryContextSwitchTo(spi_context); res = SPI_finish(); Assert(res == SPI_OK_FINISH); (void) pushJsonbValue(&state, WJB_END_OBJECT, NULL); } static Jsonb * build_telemetry_report() { JsonbParseState *parse_state = NULL; JsonbValue key; JsonbValue *result; TelemetryStats relstats; VersionOSInfo osinfo; pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); ts_jsonb_add_int32(parse_state, REQ_TELEMETRY_VERSION, TS_TELEMETRY_VERSION); ts_jsonb_add_str(parse_state, REQ_DB_UUID, DatumGetCString(DirectFunctionCall1(uuid_out, ts_metadata_get_uuid()))); ts_jsonb_add_str(parse_state, REQ_EXPORTED_DB_UUID, DatumGetCString( DirectFunctionCall1(uuid_out, ts_metadata_get_exported_uuid()))); ts_jsonb_add_str(parse_state, REQ_INSTALL_TIME, format_iso8601(ts_metadata_get_install_timestamp())); ts_jsonb_add_str(parse_state, REQ_INSTALL_METHOD, TIMESCALEDB_INSTALL_METHOD); if (ts_version_get_os_info(&osinfo)) { ts_jsonb_add_str(parse_state, REQ_OS, osinfo.sysname); ts_jsonb_add_str(parse_state, REQ_OS_VERSION, osinfo.version); ts_jsonb_add_str(parse_state, REQ_OS_RELEASE, osinfo.release); if (osinfo.has_pretty_version) ts_jsonb_add_str(parse_state, REQ_OS_VERSION_PRETTY, osinfo.pretty_version); } else ts_jsonb_add_str(parse_state, REQ_OS, "Unknown"); ts_jsonb_add_str(parse_state, REQ_PS_VERSION, get_pgversion_string()); ts_jsonb_add_str(parse_state, REQ_TS_VERSION, TIMESCALEDB_VERSION_MOD); ts_jsonb_add_str(parse_state, REQ_BUILD_OS, BUILD_OS_NAME); ts_jsonb_add_str(parse_state, REQ_BUILD_OS_VERSION, BUILD_OS_VERSION); ts_jsonb_add_str(parse_state, REQ_BUILD_ARCHITECTURE, BUILD_PROCESSOR); ts_jsonb_add_int32(parse_state, REQ_BUILD_ARCHITECTURE_BIT_SIZE, get_architecture_bit_size()); ts_jsonb_add_int64(parse_state, REQ_DATA_VOLUME, get_database_size()); /* add job execution stats */ key.type = jbvString; key.val.string.val = REQ_NUM_ERR_BY_SQLERRCODE; key.val.string.len = strlen(REQ_NUM_ERR_BY_SQLERRCODE); pushJsonbValue(&parse_state, WJB_KEY, &key); pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); add_errors_by_sqlerrcode(parse_state); pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); key.type = jbvString; key.val.string.val = REQ_JOB_STATS_BY_JOB_TYPE; key.val.string.len = strlen(REQ_JOB_STATS_BY_JOB_TYPE); pushJsonbValue(&parse_state, WJB_KEY, &key); pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); add_job_stats_by_job_type(parse_state); pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); /* Add relation stats */ ts_telemetry_stats_gather(&relstats); key.type = jbvString; key.val.string.val = REQ_RELS; key.val.string.len = strlen(REQ_RELS); pushJsonbValue(&parse_state, WJB_KEY, &key); pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); add_relkind_stats_object(parse_state, REQ_RELS_TABLES, &relstats.tables.base, RELTYPE_TABLE, STATS_TYPE_STORAGE); add_relkind_stats_object(parse_state, REQ_RELS_PARTITIONED_TABLES, &relstats.partitioned_tables.storage.base, RELTYPE_PARTITIONED_TABLE, STATS_TYPE_HYPER); add_relkind_stats_object(parse_state, REQ_RELS_MATVIEWS, &relstats.materialized_views.base, RELTYPE_MATVIEW, STATS_TYPE_STORAGE); add_relkind_stats_object(parse_state, REQ_RELS_VIEWS, &relstats.views, RELTYPE_VIEW, STATS_TYPE_BASE); add_relkind_stats_object(parse_state, REQ_RELS_HYPERTABLES, &relstats.hypertables.storage.base, RELTYPE_HYPERTABLE, STATS_TYPE_HYPER); add_relkind_stats_object(parse_state, REQ_RELS_CONTINUOUS_AGGS, &relstats.continuous_aggs.hyp.storage.base, RELTYPE_CONTINUOUS_AGG, STATS_TYPE_CAGG); pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); add_job_counts(parse_state); /* Add related extensions, which is a nested JSON */ key.type = jbvString; key.val.string.val = REQ_RELATED_EXTENSIONS; key.val.string.len = strlen(REQ_RELATED_EXTENSIONS); pushJsonbValue(&parse_state, WJB_KEY, &key); add_related_extensions(parse_state); /* license */ key.type = jbvString; key.val.string.val = REQ_LICENSE_INFO; key.val.string.len = strlen(REQ_LICENSE_INFO); pushJsonbValue(&parse_state, WJB_KEY, &key); pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); if (ts_license_is_apache()) ts_jsonb_add_str(parse_state, REQ_LICENSE_EDITION, REQ_LICENSE_EDITION_APACHE); else ts_jsonb_add_str(parse_state, REQ_LICENSE_EDITION, REQ_LICENSE_EDITION_COMMUNITY); pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); /* add tuned info, which is optional */ char *last_tune_time = GetConfigOptionByName("timescaledb.last_tune_time", NULL, true); if (last_tune_time != NULL) ts_jsonb_add_str(parse_state, REQ_TS_LAST_TUNE_TIME, last_tune_time); char *last_tune_version = GetConfigOptionByName("timescaledb.last_tune_version", NULL, true); if (last_tune_version != NULL) ts_jsonb_add_str(parse_state, REQ_TS_LAST_TUNE_VERSION, last_tune_version); /* add cloud to telemetry when set */ if (ts_telemetry_cloud != NULL) { key.type = jbvString; key.val.string.val = REQ_INSTANCE_METADATA; key.val.string.len = strlen(REQ_INSTANCE_METADATA); pushJsonbValue(&parse_state, WJB_KEY, &key); pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); ts_jsonb_add_str(parse_state, REQ_TS_TELEMETRY_CLOUD, ts_telemetry_cloud); pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); } /* Add additional content from metadata */ key.type = jbvString; key.val.string.val = REQ_METADATA; key.val.string.len = strlen(REQ_METADATA); pushJsonbValue(&parse_state, WJB_KEY, &key); pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); ts_telemetry_metadata_add_values(parse_state); pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); /* Add telemetry events */ key.type = jbvString; key.val.string.val = REQ_TELEMETRY_EVENT; key.val.string.len = strlen(REQ_TELEMETRY_EVENT); pushJsonbValue(&parse_state, WJB_KEY, &key); ts_telemetry_events_add(parse_state); /* Add function call telemetry */ key.type = jbvString; key.val.string.val = REQ_FUNCTIONS_USED; key.val.string.len = strlen(REQ_FUNCTIONS_USED); pushJsonbValue(&parse_state, WJB_KEY, &key); add_function_call_telemetry(parse_state); /* Add replication object */ key.type = jbvString; key.val.string.val = REQ_REPLICATION; key.val.string.len = strlen(REQ_REPLICATION); pushJsonbValue(&parse_state, WJB_KEY, &key); pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); add_replication_telemetry(parse_state); pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); key.type = jbvString; key.val.string.val = REQ_ACCESS_METHODS; key.val.string.len = strlen(REQ_ACCESS_METHODS); (void) pushJsonbValue(&parse_state, WJB_KEY, &key); add_query_result_dict(parse_state, "SELECT amname AS name, sum(relpages) AS pages, count(*) AS " "instances FROM pg_class JOIN pg_am ON relam = pg_am.oid " "GROUP BY amname"); /* end of telemetry object */ result = pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); return JsonbValueToJsonb(result); } HttpRequest * ts_build_version_request(const char *host, const char *path) { HttpRequest *req; Jsonb *json = build_telemetry_report(); /* Fill in HTTP request */ req = ts_http_request_create(HTTP_POST); ts_http_request_set_uri(req, path); ts_http_request_set_version(req, HTTP_VERSION_10); ts_http_request_set_header(req, HTTP_HOST, host); ts_http_request_set_body_jsonb(req, json); return req; } static ConnectionType connection_type(const char *service) { if (strcmp("http", service) == 0) return CONNECTION_PLAIN; else if (strcmp("https", service) == 0) return CONNECTION_SSL; ereport(NOTICE, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("scheme \"%s\" not supported for telemetry", service))); return _CONNECTION_MAX; } Connection * ts_telemetry_connect(const char *host, const char *service) { Connection *conn = ts_connection_create(connection_type(service)); if (conn) { int ret = ts_connection_connect(conn, host, service, 0); if (ret < 0) { const char *errstr = ts_connection_get_and_clear_error(conn); ts_connection_destroy(conn); conn = NULL; ereport(NOTICE, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("telemetry could not connect to \"%s\"", host), errdetail("%s", errstr))); } } return conn; } bool ts_telemetry_main_wrapper() { return ts_telemetry_main(TELEMETRY_HOST, TELEMETRY_PATH, TELEMETRY_SCHEME); } bool ts_telemetry_main(const char *host, const char *path, const char *service) { HttpError err; Connection *conn; HttpRequest *req; HttpResponseState *rsp; /* Declared volatile to suppress the incorrect -Wclobbered warning. */ volatile bool started = false; bool snapshot_set = false; const char *volatile json = NULL; if (!ts_telemetry_on()) return false; if (!IsTransactionOrTransactionBlock()) { started = true; StartTransactionCommand(); } conn = ts_telemetry_connect(host, service); if (conn == NULL) goto cleanup; if (!ActiveSnapshotSet()) { /* Need a valid snapshot to build telemetry information */ PushActiveSnapshot(GetTransactionSnapshot()); snapshot_set = true; } req = ts_build_version_request(host, path); if (snapshot_set) PopActiveSnapshot(); rsp = ts_http_response_state_create(); err = ts_http_send_and_recv(conn, req, rsp); ts_http_request_destroy(req); ts_connection_destroy(conn); if (err != HTTP_ERROR_NONE) { elog(NOTICE, "telemetry error: %s", ts_http_strerror(err)); goto cleanup; } if (!ts_http_response_state_valid_status(rsp)) { elog(NOTICE, "telemetry got unexpected HTTP response status: %d", ts_http_response_state_status_code(rsp)); goto cleanup; } ts_function_telemetry_reset_counts(); ts_telemetry_event_truncate(); /* * Do the version-check. Response is the body of a well-formed HTTP * response, since otherwise the previous line will throw an error. */ PG_TRY(); { json = ts_http_response_state_body_start(rsp); ts_check_version_response(json); } PG_CATCH(); { /* If the response is malformed, ts_check_version_response() will * throw an error, so we capture the error here and print debugging * information. */ FlushErrorState(); ereport(NOTICE, (errcode(ERRCODE_DATA_EXCEPTION), errmsg("malformed telemetry response body"), errdetail("host=%s, service=%s, path=%s: %s", host, service, path, json ? json : "<EMPTY>"))); /* Do not throw an error in this case, there is really nothing wrong with the system. It's only telemetry that is having problems, so we just wrap this up and exit. */ if (started) AbortCurrentTransaction(); return false; } PG_END_TRY(); ts_http_response_state_destroy(rsp); if (started) CommitTransactionCommand(); return true; cleanup: if (started) AbortCurrentTransaction(); return false; } TS_FUNCTION_INFO_V1(ts_telemetry_get_report_jsonb); Datum ts_telemetry_get_report_jsonb(PG_FUNCTION_ARGS) { Jsonb *jb = build_telemetry_report(); ts_function_telemetry_reset_counts(); PG_RETURN_JSONB_P(jb); } ================================================ FILE: src/telemetry/telemetry.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <fmgr.h> #include <utils/builtins.h> #include "compat/compat.h" #include "net/conn.h" #include "net/http.h" #include "utils.h" #include "version.h" #define REQ_LICENSE_INFO "license" #define REQ_LICENSE_EDITION "edition" #define TELEMETRY_SCHEME "https" #define TELEMETRY_HOST "telemetry.timescale.com" #define TELEMETRY_PATH "/v1/metrics" #define MAX_VERSION_STR_LEN 128 typedef struct BgwJobTypeCount { int32 policy_cagg; int32 policy_cagg_fixed; int32 policy_compression; int32 policy_compression_fixed; int32 policy_reorder; int32 policy_reorder_fixed; int32 policy_retention; int32 policy_retention_fixed; int32 policy_telemetry; int32 user_defined_action; int32 user_defined_action_fixed; } BgwJobTypeCount; typedef struct VersionResult { const char *versionstr; const char *errhint; } VersionResult; extern HttpRequest *ts_build_version_request(const char *host, const char *path); extern Connection *ts_telemetry_connect(const char *host, const char *service); extern bool ts_validate_server_version(const char *json, VersionResult *result); extern void ts_check_version_response(const char *json); /* * This function is intended as the main function for a BGW. * Its job is to send metrics and fetch the most up-to-date version of * Timescale via HTTPS. */ extern bool ts_telemetry_main(const char *host, const char *path, const char *service); extern TSDLLEXPORT bool ts_telemetry_main_wrapper(void); extern TSDLLEXPORT Datum ts_telemetry_get_report_jsonb(PG_FUNCTION_ARGS); ================================================ FILE: src/telemetry/telemetry_metadata.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_type.h> #include <commands/tablecmds.h> #include <utils/builtins.h> #include <utils/jsonb.h> #include <utils/timestamp.h> #include "jsonb_utils.h" #include "scan_iterator.h" #include "telemetry/telemetry_metadata.h" #include "ts_catalog/catalog.h" #include "ts_catalog/metadata.h" #include "uuid.h" void ts_telemetry_event_truncate(void) { RangeVar rv = { .schemaname = CATALOG_SCHEMA_NAME, .relname = TELEMETRY_EVENT_TABLE_NAME, }; ExecuteTruncate(&(TruncateStmt){ .type = T_TruncateStmt, .relations = list_make1(&rv), .behavior = DROP_RESTRICT, }); } void ts_telemetry_events_add(JsonbParseState *state) { ScanIterator iterator = ts_scan_iterator_create(TELEMETRY_EVENT, AccessShareLock, CurrentMemoryContext); pushJsonbValue(&state, WJB_BEGIN_ARRAY, NULL); ts_scanner_foreach(&iterator) { TupleInfo *ti = iterator.tinfo; TupleDesc tupdesc = ti->slot->tts_tupleDescriptor; bool created_isnull, tag_isnull, value_isnull; Datum created = slot_getattr(ti->slot, Anum_telemetry_event_created, &created_isnull); Datum tag = slot_getattr(ti->slot, Anum_telemetry_event_tag, &tag_isnull); Datum body = slot_getattr(ti->slot, Anum_telemetry_event_body, &value_isnull); pushJsonbValue(&state, WJB_BEGIN_OBJECT, NULL); if (!created_isnull) ts_jsonb_add_str(state, NameStr( TupleDescAttr(tupdesc, Anum_telemetry_event_created - 1)->attname), DatumGetCString(DirectFunctionCall1(timestamptz_out, created))); if (!tag_isnull) ts_jsonb_add_str(state, NameStr(TupleDescAttr(tupdesc, Anum_telemetry_event_tag - 1)->attname), pstrdup(NameStr(*DatumGetName(tag)))); if (!value_isnull) { JsonbValue jsonb_value; JsonbToJsonbValue(DatumGetJsonbPCopy(body), &jsonb_value); ts_jsonb_add_value(state, NameStr( TupleDescAttr(tupdesc, Anum_telemetry_event_body - 1)->attname), &jsonb_value); } pushJsonbValue(&state, WJB_END_OBJECT, NULL); } pushJsonbValue(&state, WJB_END_ARRAY, NULL); } /* * add all entries from _timescaledb_catalog.metadata */ void ts_telemetry_metadata_add_values(JsonbParseState *state) { Datum key, value; bool key_isnull, value_isnull, include_entry; ScanIterator iterator = ts_scan_iterator_create(METADATA, AccessShareLock, CurrentMemoryContext); iterator.ctx.index = catalog_get_index(ts_catalog_get(), METADATA, METADATA_PKEY_IDX); ts_scanner_foreach(&iterator) { TupleInfo *ti = iterator.tinfo; key = slot_getattr(ti->slot, Anum_metadata_key, &key_isnull); include_entry = !key_isnull && DatumGetBool(slot_getattr(ti->slot, Anum_metadata_include_in_telemetry, &key_isnull)); if (include_entry) { Name key_name = DatumGetName(key); /* skip keys included as toplevel items */ if (namestrcmp(key_name, METADATA_UUID_KEY_NAME) != 0 && namestrcmp(key_name, METADATA_EXPORTED_UUID_KEY_NAME) != 0 && namestrcmp(key_name, METADATA_TIMESTAMP_KEY_NAME) != 0) { value = slot_getattr(ti->slot, Anum_metadata_value, &value_isnull); if (!value_isnull) ts_jsonb_add_str(state, pstrdup(NameStr(*key_name)), pstrdup(TextDatumGetCString(value))); } } } } ================================================ FILE: src/telemetry/telemetry_metadata.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <utils/jsonb.h> #include <export.h> extern void ts_telemetry_metadata_add_values(JsonbParseState *state); extern void ts_telemetry_events_add(JsonbParseState *state); extern void ts_telemetry_event_truncate(void); ================================================ FILE: src/time_bucket.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_type.h> #include <fmgr.h> #include <utils/builtins.h> #include <utils/date.h> #include <utils/datetime.h> #include <utils/elog.h> #include <utils/fmgrprotos.h> #include <utils/timestamp.h> #include <utils/uuid.h> #include "export.h" #include "time_bucket.h" #include "utils.h" #include "uuid.h" #define TIME_BUCKET(period, timestamp, offset, min, max, result) \ do \ { \ if ((period) <= 0) \ ereport(ERROR, \ (errcode(ERRCODE_INVALID_PARAMETER_VALUE), \ errmsg("period must be greater than 0"))); \ if ((offset) != 0) \ { \ /* We need to ensure that the timestamp is in range _after_ the */ \ /* offset is applied: when the offset is positive we need to make */ \ /* sure the resultant time is at least min, and when negative that */ \ /* it is less than the max. */ \ (offset) = (offset) % (period); \ if (((offset) > 0 && (timestamp) < (min) + (offset)) || \ ((offset) < 0 && (timestamp) > (max) + (offset))) \ ereport(ERROR, \ (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), \ errmsg("timestamp out of range"))); \ (timestamp) -= (offset); \ } \ (result) = ((timestamp) / (period)) * (period); \ if ((timestamp) < 0 && (timestamp) % (period)) \ { \ if ((result) < (min) + (period)) \ ereport(ERROR, \ (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), \ errmsg("timestamp out of range"))); \ else \ (result) = (result) - (period); \ } \ (result) += (offset); \ } while (0) TS_FUNCTION_INFO_V1(ts_int16_bucket); TSDLLEXPORT Datum ts_int16_bucket(PG_FUNCTION_ARGS) { int16 result; int16 period = PG_GETARG_INT16(0); int16 timestamp = PG_GETARG_INT16(1); int16 offset = PG_NARGS() > 2 ? PG_GETARG_INT16(2) : 0; TIME_BUCKET(period, timestamp, offset, PG_INT16_MIN, PG_INT16_MAX, result); PG_RETURN_INT16(result); } TS_FUNCTION_INFO_V1(ts_int32_bucket); TSDLLEXPORT Datum ts_int32_bucket(PG_FUNCTION_ARGS) { int32 result; int32 period = PG_GETARG_INT32(0); int32 timestamp = PG_GETARG_INT32(1); int32 offset = PG_NARGS() > 2 ? PG_GETARG_INT32(2) : 0; TIME_BUCKET(period, timestamp, offset, PG_INT32_MIN, PG_INT32_MAX, result); PG_RETURN_INT32(result); } TS_FUNCTION_INFO_V1(ts_int64_bucket); TSDLLEXPORT Datum ts_int64_bucket(PG_FUNCTION_ARGS) { int64 result; int64 period = PG_GETARG_INT64(0); int64 timestamp = PG_GETARG_INT64(1); int64 offset = PG_NARGS() > 2 ? PG_GETARG_INT64(2) : 0; TIME_BUCKET(period, timestamp, offset, PG_INT64_MIN, PG_INT64_MAX, result); PG_RETURN_INT64(result); } #define JAN_3_2000 (2 * USECS_PER_DAY) /* * The default origin is Monday 2000-01-03. We don't use PG epoch since it starts on a saturday. * This makes time-buckets by a week more intuitive and aligns it with date_trunc. Since month * bucketing ignores the day component this makes origin for month buckets 2000-01-01. */ #define DEFAULT_ORIGIN (JAN_3_2000) #define TIME_BUCKET_TS(period, timestamp, result, shift) \ do \ { \ if ((period) <= 0) \ ereport(ERROR, \ (errcode(ERRCODE_INVALID_PARAMETER_VALUE), \ errmsg("period must be greater than 0"))); \ /* shift = shift % period, but use TMODULO */ \ TMODULO(shift, result, period); \ \ if (((shift) > 0 && (timestamp) < DT_NOBEGIN + (shift)) || \ ((shift) < 0 && (timestamp) > DT_NOEND + (shift))) \ ereport(ERROR, \ (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), \ errmsg("timestamp out of range"))); \ (timestamp) -= (shift); \ \ /* result = (timestamp / period) * period */ \ TMODULO(timestamp, result, period); \ if ((timestamp) < 0) \ { \ /* \ * need to subtract another period if remainder < 0 this only happens \ * if timestamp is negative to begin with and there is a remainder \ * after division. Need to subtract another period since division \ * truncates toward 0 in C99. \ */ \ (result) = ((result) * (period)) - (period); \ } \ else \ (result) *= (period); \ \ (result) += (shift); \ } while (0) static void validate_month_bucket(Interval *interval) { /* * Bucketing by a month and non-month cannot be mixed. */ Assert(interval->month); if (interval->day || interval->time) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("month intervals cannot have day or time component"))); } } /* * To bucket by month we get the year and month of a date and convert * that to the nth month since origin. This allows us to treat month * bucketing similar to int bucketing. During this process we ignore * the day component and therefore only support bucketing by full months. */ static DateADT bucket_month(int32 period, DateADT date, DateADT origin) { int32 year, month, day; int32 result; j2date(date + POSTGRES_EPOCH_JDATE, &year, &month, &day); int32 timestamp = (year * 12) + month - 1; j2date(origin + POSTGRES_EPOCH_JDATE, &year, &month, &day); int32 offset = (year * 12) + month - 1; TIME_BUCKET(period, timestamp, offset, PG_INT32_MIN, PG_INT32_MAX, result); year = result / 12; month = result % 12; day = 1; return date2j(year, month + 1, day) - POSTGRES_EPOCH_JDATE; } /* Returns the period in the same representation as Postgres Timestamps. * Note that this is not our internal representation (microseconds). * Always returns an exact value.*/ static inline int64 get_interval_period_timestamp_units(Interval *interval) { if (interval->month != 0) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("interval defined in terms of month, year, century etc. not supported"))); } return interval->time + (interval->day * USECS_PER_DAY); } TS_FUNCTION_INFO_V1(ts_timestamp_bucket); TSDLLEXPORT Datum ts_timestamp_bucket(PG_FUNCTION_ARGS) { Interval *interval = PG_GETARG_INTERVAL_P(0); Timestamp timestamp = PG_GETARG_TIMESTAMP(1); /* * USE NARGS and not IS_NULL to differentiate a NULL argument from a call * with 2 parameters */ Timestamp origin = (PG_NARGS() > 2 ? PG_GETARG_TIMESTAMP(2) : DEFAULT_ORIGIN); Timestamp result; if (TIMESTAMP_NOT_FINITE(timestamp)) PG_RETURN_TIMESTAMP(timestamp); if (interval->month) { DateADT origin_date = 0; validate_month_bucket(interval); DateADT date = DatumGetDateADT(DirectFunctionCall1(timestamp_date, PG_GETARG_DATUM(1))); if (origin != DEFAULT_ORIGIN) origin_date = DatumGetDateADT(DirectFunctionCall1(timestamp_date, TimestampGetDatum(origin))); date = bucket_month(interval->month, date, origin_date); PG_RETURN_DATUM(DirectFunctionCall1(date_timestamp, DateADTGetDatum(date))); } else { int64 period = get_interval_period_timestamp_units(interval); TIME_BUCKET_TS(period, timestamp, result, origin); PG_RETURN_TIMESTAMP(result); } } TS_FUNCTION_INFO_V1(ts_timestamp_offset_bucket); TSDLLEXPORT Datum ts_timestamp_offset_bucket(PG_FUNCTION_ARGS) { Datum period = PG_GETARG_DATUM(0); Datum timestamp = PG_GETARG_DATUM(1); if (TIMESTAMP_NOT_FINITE(DatumGetTimestamp(timestamp))) PG_RETURN_DATUM(timestamp); /* Apply offset. */ timestamp = DirectFunctionCall2(timestamp_mi_interval, timestamp, PG_GETARG_DATUM(2)); timestamp = DirectFunctionCall2(ts_timestamp_bucket, period, timestamp); /* Remove offset. */ timestamp = DirectFunctionCall2(timestamp_pl_interval, timestamp, PG_GETARG_DATUM(2)); PG_RETURN_DATUM(timestamp); } TS_FUNCTION_INFO_V1(ts_timestamptz_bucket); TSDLLEXPORT Datum ts_timestamptz_bucket(PG_FUNCTION_ARGS) { Interval *interval = PG_GETARG_INTERVAL_P(0); TimestampTz timestamp = PG_GETARG_TIMESTAMPTZ(1); /* * USE NARGS and not IS_NULL to differentiate a NULL argument from a call * with 2 parameters */ TimestampTz origin = (PG_NARGS() > 2 ? PG_GETARG_TIMESTAMPTZ(2) : DEFAULT_ORIGIN); TimestampTz result; if (TIMESTAMP_NOT_FINITE(timestamp)) PG_RETURN_TIMESTAMPTZ(timestamp); if (interval->month) { DateADT origin_date = 0; validate_month_bucket(interval); DateADT date = DatumGetDateADT(DirectFunctionCall1(timestamp_date, PG_GETARG_DATUM(1))); if (origin != DEFAULT_ORIGIN) origin_date = DatumGetDateADT(DirectFunctionCall1(timestamp_date, TimestampTzGetDatum(origin))); date = bucket_month(interval->month, date, origin_date); PG_RETURN_DATUM(DirectFunctionCall1(date_timestamp, DateADTGetDatum(date))); } else { int64 period = get_interval_period_timestamp_units(interval); TIME_BUCKET_TS(period, timestamp, result, origin); PG_RETURN_TIMESTAMPTZ(result); } } TS_FUNCTION_INFO_V1(ts_timestamptz_offset_bucket); TSDLLEXPORT Datum ts_timestamptz_offset_bucket(PG_FUNCTION_ARGS) { Datum period = PG_GETARG_DATUM(0); Datum timestamp = PG_GETARG_DATUM(1); if (TIMESTAMP_NOT_FINITE(DatumGetTimestampTz(timestamp))) PG_RETURN_DATUM(timestamp); /* Apply offset. */ timestamp = DirectFunctionCall2(timestamptz_mi_interval, timestamp, PG_GETARG_DATUM(2)); timestamp = DirectFunctionCall2(ts_timestamptz_bucket, period, timestamp); /* Remove offset. */ timestamp = DirectFunctionCall2(timestamptz_pl_interval, timestamp, PG_GETARG_DATUM(2)); PG_RETURN_DATUM(timestamp); } TS_FUNCTION_INFO_V1(ts_timestamptz_timezone_bucket); /* * time_bucket(bucket_width INTERVAL, ts TIMESTAMPTZ, timezone TEXT, origin TIMESTAMPTZ DEFAULT * NULL, "offset" INTERVAL DEFAULT NULL) RETURNS TIMESTAMPTZ */ TSDLLEXPORT Datum ts_timestamptz_timezone_bucket(PG_FUNCTION_ARGS) { Datum period = PG_GETARG_DATUM(0); Datum timestamp = PG_GETARG_DATUM(1); Datum tzname = PG_GETARG_DATUM(2); /* * When called from SQL we will always have 5 args because default values * will be filled in for missing arguments. When called from C with * DirectFunctionCall number of arguments might be less than 5. */ bool have_origin = PG_NARGS() > 3 && !PG_ARGISNULL(3); bool have_offset = PG_NARGS() > 4 && !PG_ARGISNULL(4); /* * We need to check for NULL arguments here because the function cannot be * defined STRICT due to the optional arguments. */ if (PG_ARGISNULL(0) || PG_ARGISNULL(1) || PG_ARGISNULL(2)) PG_RETURN_NULL(); /* * Apply offset in UTC space to avoid DST issues (issue #7059). * If we applied the offset after converting to local time, we could * create non-existent times during DST transitions. */ if (have_offset) { timestamp = DirectFunctionCall2(timestamptz_mi_interval, timestamp, PG_GETARG_DATUM(4)); } /* Convert to local timestamp according to timezone */ Datum local_ts = DirectFunctionCall2(timestamptz_zone, tzname, timestamp); if (have_origin) { Datum origin = DirectFunctionCall2(timestamptz_zone, tzname, PG_GETARG_DATUM(3)); local_ts = DirectFunctionCall3(ts_timestamp_bucket, period, local_ts, origin); } else { local_ts = DirectFunctionCall2(ts_timestamp_bucket, period, local_ts); } /* Convert back to timestamptz */ Datum result = DirectFunctionCall2(timestamp_zone, tzname, local_ts); /* * During DST fall-back, the same local time maps to two different UTC * times. PostgreSQL's timestamp_zone picks the later (standard time) * interpretation. If the original timestamp was in daylight time, the * bucket could start AFTER the timestamp. Fix by subtracting periods * until the bucket is at or before the timestamp (issue #9136). */ while (DatumGetTimestampTz(result) > DatumGetTimestampTz(timestamp)) { result = DirectFunctionCall2(timestamptz_mi_interval, result, period); } if (have_offset) { /* Remove offset in UTC space. */ result = DirectFunctionCall2(timestamptz_pl_interval, result, PG_GETARG_DATUM(4)); } PG_RETURN_DATUM(result); } static inline void check_period_is_daily(int64 period) { int64 day = USECS_PER_DAY; if (period < day) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("interval must not have sub-day precision"))); } if (period % day != 0) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("interval must be a multiple of a day"))); } } TS_FUNCTION_INFO_V1(ts_date_bucket); TSDLLEXPORT Datum ts_date_bucket(PG_FUNCTION_ARGS) { Interval *interval = PG_GETARG_INTERVAL_P(0); DateADT date = PG_GETARG_DATEADT(1); DateADT origin = 0; Timestamp origin_ts = DEFAULT_ORIGIN; Timestamp timestamp, result; if (DATE_NOT_FINITE(date)) PG_RETURN_DATEADT(date); /* convert to timestamp (NOT tz), bucket, convert back to date */ timestamp = DatumGetTimestamp(DirectFunctionCall1(date_timestamp, PG_GETARG_DATUM(1))); Assert(!TIMESTAMP_NOT_FINITE(timestamp)); if (PG_NARGS() > 2) { origin = PG_GETARG_DATEADT(2); if (!interval->month) origin_ts = DatumGetTimestamp(DirectFunctionCall1(date_timestamp, DateADTGetDatum(origin))); } if (interval->month) { validate_month_bucket(interval); date = bucket_month(interval->month, date, origin); PG_RETURN_DATEADT(date); } else { int64 period = get_interval_period_timestamp_units(interval); /* check the period aligns on a date */ check_period_is_daily(period); TIME_BUCKET_TS(period, timestamp, result, origin_ts); PG_RETURN_DATUM(DirectFunctionCall1(timestamp_date, TimestampGetDatum(result))); } } TS_FUNCTION_INFO_V1(ts_date_offset_bucket); TSDLLEXPORT Datum ts_date_offset_bucket(PG_FUNCTION_ARGS) { Datum period = PG_GETARG_DATUM(0); Datum date = PG_GETARG_DATUM(1); if (DATE_NOT_FINITE(DatumGetDateADT(date))) PG_RETURN_DATUM(date); /* Apply offset. */ Datum time = DirectFunctionCall2(date_mi_interval, date, PG_GETARG_DATUM(2)); date = DirectFunctionCall1(timestamp_date, time); date = DirectFunctionCall2(ts_date_bucket, period, date); /* Remove offset. */ time = DirectFunctionCall2(date_pl_interval, date, PG_GETARG_DATUM(2)); date = DirectFunctionCall1(timestamp_date, time); PG_RETURN_DATUM(date); } static const char * uuid_to_str(const pg_uuid_t *uuid) { return DatumGetCString(DirectFunctionCall1(uuid_out, UUIDPGetDatum(uuid))); } TS_FUNCTION_INFO_V1(ts_uuid_bucket); Datum ts_uuid_bucket(PG_FUNCTION_ARGS) { pg_uuid_t *uuid = PG_GETARG_UUID_P(1); Datum origin = (PG_NARGS() > 2 ? PG_GETARG_DATUM(2) : TimestampTzGetDatum(DEFAULT_ORIGIN)); TimestampTz timestamp; if (!ts_uuid_v7_extract_timestamptz(uuid, ×tamp, false)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("not a version 7 UUID: %s", uuid_to_str(uuid)))); PG_RETURN_DATUM(DirectFunctionCall3(ts_timestamptz_bucket, PG_GETARG_DATUM(0), TimestampTzGetDatum(timestamp), origin)); } TS_FUNCTION_INFO_V1(ts_uuid_offset_bucket); TSDLLEXPORT Datum ts_uuid_offset_bucket(PG_FUNCTION_ARGS) { pg_uuid_t *uuid = PG_GETARG_UUID_P(1); TimestampTz timestamp; if (!ts_uuid_v7_extract_timestamptz(uuid, ×tamp, false)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("not a version 7 UUID: %s", uuid_to_str(uuid)))); LOCAL_FCINFO(fcinfo_local, 3); Datum result; InitFunctionCallInfoData(*fcinfo_local, NULL, 3, InvalidOid, NULL, NULL); fcinfo_local->args[0].value = PG_GETARG_DATUM(0); /* Period */ fcinfo_local->args[0].isnull = PG_ARGISNULL(0); fcinfo_local->args[1].value = TimestampTzGetDatum(timestamp); fcinfo_local->args[1].isnull = PG_ARGISNULL(1); fcinfo_local->args[2].value = PG_GETARG_DATUM(2); fcinfo_local->args[2].isnull = PG_ARGISNULL(2); result = ts_timestamptz_offset_bucket(fcinfo_local); /* Timestamp offset bucket does not return NULL */ Assert(!fcinfo_local->isnull); return result; } TS_FUNCTION_INFO_V1(ts_uuid_timezone_bucket); /* * time_bucket(bucket_width INTERVAL, ts uuid, timezone TEXT, origin uuid DEFAULT * NULL, "offset" INTERVAL DEFAULT NULL) RETURNS TIMESTAMPTZ */ TSDLLEXPORT Datum ts_uuid_timezone_bucket(PG_FUNCTION_ARGS) { /* * We need to check for NULL arguments here because the function cannot be * defined STRICT due to the optional arguments. */ if (PG_ARGISNULL(0) || PG_ARGISNULL(1) || PG_ARGISNULL(2)) PG_RETURN_NULL(); pg_uuid_t *uuid = PG_GETARG_UUID_P(1); TimestampTz timestamp; if (!ts_uuid_v7_extract_timestamptz(uuid, ×tamp, false)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("not a version 7 UUID: %s", uuid_to_str(uuid)))); LOCAL_FCINFO(fcinfo_local, 4); Datum result; InitFunctionCallInfoData(*fcinfo_local, NULL, 4, InvalidOid, NULL, NULL); fcinfo_local->args[0].value = PG_GETARG_DATUM(0); /* Period */ fcinfo_local->args[0].isnull = PG_ARGISNULL(0); fcinfo_local->args[1].value = TimestampTzGetDatum(timestamp); fcinfo_local->args[1].isnull = PG_ARGISNULL(1); fcinfo_local->args[2].value = PG_GETARG_DATUM(2); fcinfo_local->args[2].isnull = PG_ARGISNULL(2); fcinfo_local->args[3].value = PG_GETARG_DATUM(3); fcinfo_local->args[3].isnull = PG_ARGISNULL(3); result = ts_timestamptz_timezone_bucket(fcinfo_local); /* The only case where the timestamp bucket function returns NULL is when passed NULL input * arguments, but we already checked for that above. */ Assert(!fcinfo_local->isnull); return result; } TSDLLEXPORT int64 ts_time_bucket_by_type(int64 interval, int64 timestamp, Oid timestamp_type) { NullableDatum null_datum = INIT_NULL_DATUM; return ts_time_bucket_by_type_extended(interval, timestamp, timestamp_type, null_datum, null_datum); } /* when working with time_buckets stored in our catalog, we may not know ahead of time which * bucketing function to use, this function dynamically dispatches to the correct time_bucket_<foo> * based on an inputted timestamp_type */ TSDLLEXPORT int64 ts_time_bucket_by_type_extended(int64 interval, int64 timestamp, Oid timestamp_type, NullableDatum offset, NullableDatum origin) { /* Defined offset and origin in one function is not supported */ Assert(offset.isnull == true || origin.isnull == true); Datum timestamp_in_time_type = ts_internal_to_time_value(timestamp, timestamp_type); Datum interval_in_interval_type; Datum time_bucketed; Datum (*bucket_function)(PG_FUNCTION_ARGS); switch (timestamp_type) { case INT2OID: interval_in_interval_type = ts_internal_to_interval_value(interval, timestamp_type); bucket_function = ts_int16_bucket; break; case INT4OID: interval_in_interval_type = ts_internal_to_interval_value(interval, timestamp_type); bucket_function = ts_int32_bucket; break; case INT8OID: interval_in_interval_type = ts_internal_to_interval_value(interval, timestamp_type); bucket_function = ts_int64_bucket; break; case TIMESTAMPOID: interval_in_interval_type = ts_internal_to_interval_value(interval, INTERVALOID); if (offset.isnull) bucket_function = ts_timestamp_bucket; /* handles also origin */ else bucket_function = ts_timestamp_offset_bucket; break; case TIMESTAMPTZOID: interval_in_interval_type = ts_internal_to_interval_value(interval, INTERVALOID); if (offset.isnull) bucket_function = ts_timestamptz_bucket; /* handles also origin */ else bucket_function = ts_timestamptz_offset_bucket; break; case DATEOID: interval_in_interval_type = ts_internal_to_interval_value(interval, INTERVALOID); if (offset.isnull) bucket_function = ts_date_bucket; /* handles also origin */ else bucket_function = ts_date_offset_bucket; break; default: elog(ERROR, "invalid time_bucket type \"%s\"", format_type_be(timestamp_type)); } if (!offset.isnull) { time_bucketed = DirectFunctionCall3(bucket_function, interval_in_interval_type, timestamp_in_time_type, offset.value); } else if (!origin.isnull) { time_bucketed = DirectFunctionCall3(bucket_function, interval_in_interval_type, timestamp_in_time_type, origin.value); } else { time_bucketed = DirectFunctionCall2(bucket_function, interval_in_interval_type, timestamp_in_time_type); } return ts_time_value_to_internal(time_bucketed, timestamp_type); } ================================================ FILE: src/time_bucket.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <fmgr.h> #include "export.h" extern TSDLLEXPORT Datum ts_int16_bucket(PG_FUNCTION_ARGS); extern TSDLLEXPORT Datum ts_int32_bucket(PG_FUNCTION_ARGS); extern TSDLLEXPORT Datum ts_int64_bucket(PG_FUNCTION_ARGS); extern TSDLLEXPORT Datum ts_date_bucket(PG_FUNCTION_ARGS); extern TSDLLEXPORT Datum ts_timestamp_bucket(PG_FUNCTION_ARGS); extern TSDLLEXPORT Datum ts_timestamp_offset_bucket(PG_FUNCTION_ARGS); extern TSDLLEXPORT Datum ts_timestamptz_bucket(PG_FUNCTION_ARGS); extern TSDLLEXPORT Datum ts_timestamptz_timezone_bucket(PG_FUNCTION_ARGS); extern TSDLLEXPORT int64 ts_time_bucket_by_type(int64 interval, int64 timestamp, Oid type); extern TSDLLEXPORT int64 ts_time_bucket_by_type_extended(int64 interval, int64 timestamp, Oid type, NullableDatum offset, NullableDatum origin); ================================================ FILE: src/time_utils.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <catalog/pg_type.h> #include <fmgr.h> #include <parser/parse_coerce.h> #include <utils/builtins.h> #include <utils/date.h> #include <utils/fmgrprotos.h> #include <utils/rangetypes.h> #include <utils/timestamp.h> #include <utils/uuid.h> #include "guc.h" #include "time_utils.h" #include "utils.h" #include "uuid.h" TS_FUNCTION_INFO_V1(ts_make_range_from_internal_time); TS_FUNCTION_INFO_V1(ts_get_internal_time_min); TS_FUNCTION_INFO_V1(ts_get_internal_time_max); /* * Subtract an interval from the current time and return the result as a Datum * of the specified time type. * * In debug mode, uses mock time if configured for testing purposes. */ TSDLLEXPORT Datum ts_subtract_interval_from_now(const Interval *interval, Oid timetype) { #ifdef TS_DEBUG Datum res = ts_get_mock_time_or_current_time(); #else Datum res = TimestampTzGetDatum(GetCurrentTransactionStartTimestamp()); #endif switch (timetype) { case TIMESTAMPOID: res = DirectFunctionCall1(timestamptz_timestamp, res); return DirectFunctionCall2(timestamp_mi_interval, res, IntervalPGetDatum(interval)); case TIMESTAMPTZOID: return DirectFunctionCall2(timestamptz_mi_interval, res, IntervalPGetDatum(interval)); case DATEOID: res = DirectFunctionCall1(timestamptz_timestamp, res); res = DirectFunctionCall2(timestamp_mi_interval, res, IntervalPGetDatum(interval)); return DirectFunctionCall1(timestamp_date, res); case UUIDOID: { /* * For UUIDv7-partitioned hypertables, compute (now - interval) and convert * to a UUIDv7 boundary value suitable for range comparisons. */ res = DirectFunctionCall2(timestamptz_mi_interval, res, IntervalPGetDatum(interval)); TimestampTz boundary_ts = DatumGetTimestampTz(res); pg_uuid_t *uuid = ts_create_uuid_v7_from_unixtime_us(boundary_ts, true, true); return UUIDPGetDatum(uuid); } default: ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unsupported time type %s", format_type_be(timetype)))); pg_unreachable(); } } Datum ts_time_datum_convert_arg(Datum arg, Oid *argtype, Oid timetype) { Oid type = *argtype; if (!OidIsValid(type) || type == UNKNOWNOID) { Oid infuncid = InvalidOid; Oid typeioparam; type = timetype; getTypeInputInfo(type, &infuncid, &typeioparam); switch (get_func_nargs(infuncid)) { case 1: /* Functions that take one input argument, e.g., the Date function */ arg = OidFunctionCall1(infuncid, arg); break; case 3: /* Timestamp functions take three input arguments */ arg = OidFunctionCall3(infuncid, arg, ObjectIdGetDatum(InvalidOid), Int32GetDatum(-1)); break; default: ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid time argument"), errhint("Time argument requires an explicit cast."))); } *argtype = type; } return arg; } /* * Get the internal time value from a pseudo-type function argument. * * API functions that take supported time types as arguments often use a * pseudo-type parameter to represent these. For instance, the "any" * pseudo-type is often used to represent any of these supported types. * * The downside of "any", however, is that it lacks type information and often * forces users to add explicit type casts. For instance, with the following * API call * * drop_chunk('conditions', '2020-10-01'); * * the argument type will be UNKNOWNOID. And the user would have to add * an explicit type cast: * * drop_chunks('conditions', '2020-10-01'::date); * * However, we can handle the UNKNOWNOID case since we have the time type * information in internal metadata (e.g., the time column type of a * hypertable) and we can try to convert the argument to that type. * * Thus, there are two cases: * * 1. An explicit cast was done --> the type is given in argtype. * 2. No cast was done --> we try to convert the argument to the known time * type. * * If an unsupported type is given, or the typeless argument has a nonsensical * string, then there will be an error raised. */ int64 ts_time_value_from_arg(Datum arg, Oid argtype, Oid timetype, bool need_now_func) { /* If no explicit cast was done by the user, try to convert the argument * to the time type. */ arg = ts_time_datum_convert_arg(arg, &argtype, timetype); if (IS_INTEGER_TYPE(timetype) && (argtype == INTERVALOID || IS_TIMESTAMP_TYPE(argtype))) { if (need_now_func) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid time argument type \"%s\"", format_type_be(argtype)), errhint("Try casting the argument to \"%s\".", format_type_be(timetype)))); /* * The argument type is INTERVAL or TIMESTAMP-like for INTEGER column; this signifies that * chunks are retained based on chunk creation time. Chunk creation time is represented * as TIMESTAMPTZ, the input argument should be typecast to TIMESTAMPTZ. */ if (argtype == INTERVALOID) arg = ts_subtract_interval_from_now(DatumGetIntervalP(arg), TIMESTAMPTZOID); return DatumGetInt64(arg); } if (argtype == INTERVALOID) { arg = ts_subtract_interval_from_now(DatumGetIntervalP(arg), timetype); argtype = timetype; } else if (argtype != timetype && !can_coerce_type(1, &argtype, &timetype, COERCION_IMPLICIT)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid time argument type \"%s\"", format_type_be(argtype)), errhint("Try casting the argument to \"%s\".", format_type_be(timetype)))); return ts_time_value_to_internal(arg, argtype); } /* * Try to coerce a type to a supported time type. * * To support custom time types in hypertables, we need to know the type's * boundaries in order to, e.g., construct dimensional chunk constraints. The * custom time type will inherit the valid time range of the supported time * type it can be casted to. * * Currently, we only support custom time types that are binary compatible * with bigint and then it also inherits the valid time range of a bigint. */ static Oid coerce_to_time_type(Oid type) { if (ts_type_is_int8_binary_compatible(type)) return INT8OID; ereport(ERROR, errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unsupported time type \"%s\"", DatumGetPointer(DirectFunctionCall1(regtypeout, type)))); pg_unreachable(); } /* * Get the min time datum for a time type. * * Note that the min is not the same the actual "min" of the underlying * storage for date and timestamps. */ Datum ts_time_datum_get_min(Oid timetype) { switch (timetype) { case DATEOID: return DateADTGetDatum(TS_DATE_MIN); case TIMESTAMPOID: return TimestampGetDatum(TS_TIMESTAMP_MIN); case TIMESTAMPTZOID: return TimestampTzGetDatum(TS_TIMESTAMP_MIN); case INT2OID: return Int16GetDatum(PG_INT16_MIN); case INT4OID: return Int32GetDatum(PG_INT32_MIN); case INT8OID: return Int64GetDatum(PG_INT64_MIN); case UUIDOID: return Int64GetDatum(TS_TIME_UUID_MIN); default: break; } return ts_time_datum_get_min(coerce_to_time_type(timetype)); } /* * Get the end time datum for a time type. * * Note that the end is not the same as "max" (hence not named max). The end * is exclusive for date and timestamps, and might not be the same as the max * value for the underlying storage type (e.g., TIMESTAMP_END is before * PG_INT64_MAX). Instead, the max value for dates and timestamps represent * -Infinity and +Infinity. */ Datum ts_time_datum_get_end(Oid timetype) { switch (timetype) { case DATEOID: return DateADTGetDatum(TS_DATE_END); case TIMESTAMPOID: return TimestampGetDatum(TS_TIMESTAMP_END); case TIMESTAMPTZOID: return TimestampTzGetDatum(TS_TIMESTAMP_END); case INT2OID: case INT4OID: case INT8OID: case UUIDOID: elog(ERROR, "END is not defined for \"%s\"", format_type_be(timetype)); break; default: break; } return ts_time_datum_get_end(coerce_to_time_type(timetype)); } Datum ts_time_datum_get_max(Oid timetype) { switch (timetype) { case DATEOID: return DateADTGetDatum(TS_DATE_END - 1); case TIMESTAMPOID: return TimestampGetDatum(TS_TIMESTAMP_END - 1); case TIMESTAMPTZOID: return TimestampTzGetDatum(TS_TIMESTAMP_END - 1); case INT2OID: return Int16GetDatum(PG_INT16_MAX); case INT4OID: return Int32GetDatum(PG_INT32_MAX); case INT8OID: return Int64GetDatum(PG_INT64_MAX); case UUIDOID: return Int64GetDatum(TS_TIME_UUID_MAX); default: break; } return ts_time_datum_get_max(coerce_to_time_type(timetype)); } Datum ts_time_datum_get_nobegin(Oid timetype) { switch (timetype) { case DATEOID: return DateADTGetDatum(DATEVAL_NOBEGIN); case TIMESTAMPOID: return TimestampGetDatum(DT_NOBEGIN); case TIMESTAMPTZOID: return TimestampTzGetDatum(DT_NOBEGIN); case INT2OID: case INT4OID: case INT8OID: case UUIDOID: elog(ERROR, "NOBEGIN is not defined for \"%s\"", format_type_be(timetype)); break; default: break; } return ts_time_datum_get_nobegin(coerce_to_time_type(timetype)); } Datum ts_time_datum_get_nobegin_or_min(Oid timetype) { if (IS_TIMESTAMP_TYPE(timetype)) return ts_time_datum_get_nobegin(timetype); return ts_time_datum_get_min(timetype); } Datum ts_time_datum_get_noend(Oid timetype) { switch (timetype) { case DATEOID: return DateADTGetDatum(DATEVAL_NOEND); case TIMESTAMPOID: return TimestampGetDatum(DT_NOEND); case TIMESTAMPTZOID: return TimestampTzGetDatum(DT_NOEND); case INT2OID: case INT4OID: case INT8OID: case UUIDOID: elog(ERROR, "NOEND is not defined for \"%s\"", format_type_be(timetype)); break; default: break; } return ts_time_datum_get_noend(coerce_to_time_type(timetype)); } /* * Get the min for a time type in internal time. */ int64 ts_time_get_min(Oid timetype) { switch (timetype) { case DATEOID: return TS_DATE_INTERNAL_MIN; case TIMESTAMPOID: return TS_TIMESTAMP_INTERNAL_MIN; case TIMESTAMPTZOID: return TS_TIMESTAMP_INTERNAL_MIN; case INT2OID: return PG_INT16_MIN; case INT4OID: return PG_INT32_MIN; case INT8OID: return PG_INT64_MIN; case UUIDOID: return TS_TIME_UUID_MIN; default: break; } return ts_time_get_min(coerce_to_time_type(timetype)); } /* * Get the max for a time type in internal time. */ int64 ts_time_get_max(Oid timetype) { switch (timetype) { case DATEOID: return TS_DATE_INTERNAL_END - 1; case TIMESTAMPOID: return TS_TIMESTAMP_INTERNAL_END - 1; case TIMESTAMPTZOID: return TS_TIMESTAMP_INTERNAL_END - 1; case INT2OID: return PG_INT16_MAX; case INT4OID: return PG_INT32_MAX; case INT8OID: return PG_INT64_MAX; case UUIDOID: return TS_TIME_UUID_MAX; default: break; } return ts_time_get_max(coerce_to_time_type(timetype)); } /* * Get the end value time for a time type in internal time. * * The end is not a valid time value (it is exclusive). */ int64 ts_time_get_end(Oid timetype) { switch (timetype) { case DATEOID: return TS_DATE_INTERNAL_END; case TIMESTAMPOID: return TS_TIMESTAMP_INTERNAL_END; case TIMESTAMPTZOID: return TS_TIMESTAMP_INTERNAL_END; case INT2OID: case INT4OID: case INT8OID: case UUIDOID: elog(ERROR, "END is not defined for \"%s\"", format_type_be(timetype)); break; default: break; } return ts_time_get_end(coerce_to_time_type(timetype)); } /* * Return the end (exclusive) or fall back to max. * * Integer time types have no definition for END, so we fall back to max. */ int64 ts_time_get_end_or_max(Oid timetype) { if (IS_TIMESTAMP_TYPE(timetype)) return ts_time_get_end(timetype); return ts_time_get_max(timetype); } int64 ts_time_get_nobegin(Oid timetype) { switch (timetype) { case DATEOID: case TIMESTAMPOID: case TIMESTAMPTZOID: return TS_TIME_NOBEGIN; case INT2OID: case INT4OID: case INT8OID: case UUIDOID: elog(ERROR, "-Infinity not defined for \"%s\"", format_type_be(timetype)); break; default: break; } return ts_time_get_nobegin(coerce_to_time_type(timetype)); } int64 ts_time_get_nobegin_or_min(Oid timetype) { if (IS_TIMESTAMP_TYPE(timetype)) return ts_time_get_nobegin(timetype); return ts_time_get_min(timetype); } int64 ts_time_get_noend(Oid timetype) { switch (timetype) { case DATEOID: case TIMESTAMPOID: case TIMESTAMPTZOID: return TS_TIME_NOEND; case INT2OID: case INT4OID: case INT8OID: case UUIDOID: elog(ERROR, "+Infinity not defined for \"%s\"", format_type_be(timetype)); break; default: break; } return ts_time_get_noend(coerce_to_time_type(timetype)); } int64 ts_time_get_noend_or_max(Oid timetype) { if (IS_TIMESTAMP_TYPE(timetype)) return ts_time_get_noend(timetype); return ts_time_get_max(timetype); } /* * Add an interval to a time value in a saturating way. * * In contrast to, e.g., PG's timestamp_pl_interval, this function adds an * interval in a saturating way without throwing an error in case of * overflow. Instead it clamps to max for integer types end NOEND for date and * timestamp types. */ int64 ts_time_saturating_add(int64 timeval, int64 interval, Oid timetype) { if (timeval > 0 && interval > 0 && timeval > (ts_time_get_max(timetype) - interval)) return ts_time_get_noend_or_max(timetype); if (timeval < 0 && interval < 0 && timeval < (ts_time_get_min(timetype) - interval)) return ts_time_get_nobegin_or_min(timetype); return timeval + interval; } /* * Subtract an interval from a time value in a saturating way. * * In contrast to, e.g., PG's timestamp_mi_interval, this function subtracts * an interval in a saturating way without throwing an error in case of * overflow. Instead, it clamps to min for integer types and NOBEGIN for date * and timestamp types. */ int64 ts_time_saturating_sub(int64 timeval, int64 interval, Oid timetype) { if (timeval < 0 && interval > 0 && timeval < (ts_time_get_min(timetype) + interval)) return ts_time_get_nobegin_or_min(timetype); if (timeval > 0 && interval < 0 && timeval > (ts_time_get_max(timetype) + interval)) return ts_time_get_noend_or_max(timetype); return timeval - interval; } int64 ts_subtract_integer_from_now_saturating(Oid now_func, int64 interval, Oid timetype) { Datum now = OidFunctionCall0(now_func); int64 time_min = ts_time_get_min(timetype); int64 time_max = ts_time_get_max(timetype); int64 nowval, res; Assert(IS_INTEGER_TYPE(timetype)); switch (timetype) { case INT2OID: { nowval = DatumGetInt16(now); break; } case INT4OID: { nowval = DatumGetInt32(now); break; } case INT8OID: { nowval = DatumGetInt64(now); break; } default: elog(ERROR, "unsupported integer time type \"%s\"", format_type_be(timetype)); } if (nowval > 0 && interval < 0 && nowval > time_max + interval) res = time_max; else if (nowval < 0 && interval > 0 && nowval < time_min + interval) res = time_min; else res = nowval - interval; return res; } #ifdef TS_DEBUG /* return mock time for testing */ Datum ts_get_mock_time_or_current_time(void) { Datum res; if (ts_current_timestamp_mock != NULL && strlen(ts_current_timestamp_mock) != 0) { res = DirectFunctionCall3(timestamptz_in, CStringGetDatum(ts_current_timestamp_mock), 0, Int32GetDatum(-1)); return res; } res = TimestampTzGetDatum(GetCurrentTransactionStartTimestamp()); return res; } #endif TS_FUNCTION_INFO_V1(ts_now_mock); /* return mock time for testing */ Datum ts_now_mock(PG_FUNCTION_ARGS) { Datum res; #ifdef TS_DEBUG if (ts_current_timestamp_mock != NULL && strlen(ts_current_timestamp_mock) != 0) { res = DirectFunctionCall3(timestamptz_in, CStringGetDatum(ts_current_timestamp_mock), 0, Int32GetDatum(-1)); return res; } #endif res = TimestampTzGetDatum(GetCurrentTimestamp()); return res; } Datum ts_make_range_from_internal_time(PG_FUNCTION_ARGS) { Oid rngtypid = get_fn_expr_rettype(fcinfo->flinfo); TypeCacheEntry *typcache = range_get_typcache(fcinfo, rngtypid); #if PG16_GE Node *escontext = fcinfo->context; #endif RangeBound lower = { .val = PG_ARGISNULL(1) ? 0 : PG_GETARG_DATUM(1), .infinite = PG_ARGISNULL(1), .inclusive = true, .lower = true, }; RangeBound upper = { .val = PG_ARGISNULL(2) ? 0 : PG_GETARG_DATUM(2), .infinite = PG_ARGISNULL(2), .inclusive = false, .lower = false, }; /* Need to check the types of the lower and upper values. They should * match the returned range. */ PG_RETURN_RANGE_P(make_range_compat(typcache, &lower, &upper, false, escontext)); } Datum ts_get_internal_time_min(PG_FUNCTION_ARGS) { PG_RETURN_INT64(ts_time_get_min(PG_GETARG_OID(0))); } Datum ts_get_internal_time_max(PG_FUNCTION_ARGS) { PG_RETURN_INT64(ts_time_get_max(PG_GETARG_OID(0))); } ================================================ FILE: src/time_utils.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <datatype/timestamp.h> #include "export.h" /* TimescaleDB-specific ranges for valid timestamps and dates: */ #define TS_EPOCH_DIFF (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) #define TS_EPOCH_DIFF_MICROSECONDS (TS_EPOCH_DIFF * USECS_PER_DAY) /* For Timestamps, we need to be able to go from UNIX epoch to POSTGRES epoch * and thus add the difference between the two epochs. This will constrain the * max supported timestamp by the same amount. */ #define TS_TIMESTAMP_MIN MIN_TIMESTAMP #define TS_TIMESTAMP_MAX (TS_TIMESTAMP_END - 1) #define TS_TIMESTAMP_END (END_TIMESTAMP - TS_EPOCH_DIFF_MICROSECONDS) #define TS_TIMESTAMP_INTERNAL_MIN (TS_TIMESTAMP_MIN + TS_EPOCH_DIFF_MICROSECONDS) #define TS_TIMESTAMP_INTERNAL_MAX (TS_TIMESTAMP_INTERNAL_END - 1) #define TS_TIMESTAMP_INTERNAL_END (TS_TIMESTAMP_END + TS_EPOCH_DIFF_MICROSECONDS) /* For Dates, we're limited by the timestamp range (since we internally first * convert dates to timestamps). Naturally the TimescaleDB-specific timestamp * limits apply as well. */ #define TS_DATE_MIN (DATETIME_MIN_JULIAN - POSTGRES_EPOCH_JDATE) #define TS_DATE_MAX (TS_DATE_END - 1) #define TS_DATE_END (TIMESTAMP_END_JULIAN - POSTGRES_EPOCH_JDATE - TS_EPOCH_DIFF) #define TS_DATE_INTERNAL_MIN (TS_TIMESTAMP_MIN + TS_EPOCH_DIFF_MICROSECONDS) #define TS_DATE_INTERNAL_MAX (TS_DATE_INTERNAL_END - 1) #define TS_DATE_INTERNAL_END (TS_TIMESTAMP_END + TS_EPOCH_DIFF_MICROSECONDS) /* * -Infinity and +Infinity in internal (Unix) time. */ #define TS_TIME_NOBEGIN (PG_INT64_MIN) #define TS_TIME_NOEND (PG_INT64_MAX) /* * A UUIDv7 timestamp is 6 bytes milliseconds in Unix epoch (unsigned). * * Since RFC9562 specifies the timestamp as unsigned, the minimum value is * 0. Further, the sub-millisecond part cannot be used as time since the bits * are optional and it is not possible to know if they are random or represent * a time fraction. Therefore, the max value is limited to the milliseconds. */ #define TS_TIME_UUID_MS_MIN (0x000000000000) #define TS_TIME_UUID_MIN (0x000000000000 * 1000) /* microseconds */ #define TS_TIME_UUID_MS_MAX (0xFFFFFFFFFFFF) #define TS_TIME_UUID_MAX (TS_TIME_UUID_MS_MAX * 1000) /* microseconds */ #define IS_INTEGER_TYPE(type) (type == INT2OID || type == INT4OID || type == INT8OID) #define IS_TIMESTAMP_TYPE(type) (type == TIMESTAMPOID || type == TIMESTAMPTZOID || type == DATEOID) #define IS_UUID_TYPE(type) (type == UUIDOID) #define IS_VALID_TIME_TYPE(type) \ (IS_INTEGER_TYPE(type) || IS_TIMESTAMP_TYPE(type) || IS_UUID_TYPE(type)) #define TS_TIME_DATUM_IS_MIN(timeval, type) (timeval == ts_time_datum_get_min(type)) #define TS_TIME_DATUM_IS_MAX(timeval, type) (timeval == ts_time_datum_get_max(type)) #define TS_TIME_DATUM_IS_END(timeval, type) \ (IS_TIMESTAMP_TYPE(type) && timeval == ts_time_datum_get_end(type))) #define TS_TIME_DATUM_IS_NOBEGIN(timeval, type) \ (IS_TIMESTAMP_TYPE(type) && (timeval == ts_time_datum_get_nobegin(type))) #define TS_TIME_DATUM_IS_NOEND(timeval, type) \ (IS_TIMESTAMP_TYPE(type) && (timeval == ts_time_datum_get_noend(type))) #define TS_TIME_DATUM_NOT_FINITE(timeval, type) \ (IS_INTEGER_TYPE(type) || TS_TIME_DATUM_IS_NOBEGIN(timeval, type) || \ TS_TIME_DATUM_IS_NOEND(timeval, type)) #define TS_TIME_IS_MIN(timeval, type) (timeval == ts_time_get_min(type)) #define TS_TIME_IS_MAX(timeval, type) (timeval == ts_time_get_max(type)) #define TS_TIME_IS_END(timeval, type) (IS_TIMESTAMP_TYPE(type) && timeval == ts_time_get_end(type)) #define TS_TIME_IS_NOBEGIN(timeval, type) \ (IS_TIMESTAMP_TYPE(type) && timeval == ts_time_get_nobegin(type)) #define TS_TIME_IS_NOEND(timeval, type) \ (IS_TIMESTAMP_TYPE(type) && timeval == ts_time_get_noend(type)) #define TS_TIME_NOT_FINITE(timeval, type) \ (IS_INTEGER_TYPE(type) || TS_TIME_IS_NOBEGIN(timeval, type) || TS_TIME_IS_NOEND(timeval, type)) extern TSDLLEXPORT int64 ts_time_value_from_arg(Datum arg, Oid argtype, Oid timetype, bool need_now_func); extern TSDLLEXPORT Datum ts_time_datum_convert_arg(Datum arg, Oid *argtype, Oid timetype); extern TSDLLEXPORT Datum ts_time_datum_get_min(Oid timetype); extern TSDLLEXPORT Datum ts_time_datum_get_max(Oid timetype); extern TSDLLEXPORT Datum ts_time_datum_get_end(Oid timetype); extern TSDLLEXPORT Datum ts_time_datum_get_nobegin(Oid timetype); extern TSDLLEXPORT Datum ts_time_datum_get_nobegin_or_min(Oid timetype); extern TSDLLEXPORT Datum ts_time_datum_get_noend(Oid timetype); extern TSDLLEXPORT int64 ts_time_get_min(Oid timetype); extern TSDLLEXPORT int64 ts_time_get_max(Oid timetype); extern TSDLLEXPORT int64 ts_time_get_end(Oid timetype); extern TSDLLEXPORT int64 ts_time_get_end_or_max(Oid timetype); extern TSDLLEXPORT int64 ts_time_get_nobegin(Oid timetype); extern TSDLLEXPORT int64 ts_time_get_nobegin_or_min(Oid timetype); extern TSDLLEXPORT int64 ts_time_get_noend(Oid timetype); extern TSDLLEXPORT int64 ts_time_get_noend_or_max(Oid timetype); extern TSDLLEXPORT int64 ts_time_saturating_add(int64 timeval, int64 interval, Oid timetype); extern TSDLLEXPORT int64 ts_time_saturating_sub(int64 timeval, int64 interval, Oid timetype); extern TSDLLEXPORT int64 ts_subtract_integer_from_now_saturating(Oid now_func, int64 interval, Oid timetype); extern TSDLLEXPORT Datum ts_subtract_interval_from_now(const Interval *interval, Oid timetype); #ifdef TS_DEBUG extern TSDLLEXPORT Datum ts_get_mock_time_or_current_time(void); #endif bool is_valid_now_func(Node *node); ================================================ FILE: src/timezones.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include "timezones.h" #include <access/xact.h> #include <datatype/timestamp.h> #include <pgtime.h> #include <port.h> #include <utils/timestamp.h> /* Checks if the given TZ name is valid. */ bool ts_is_valid_timezone_name(const char *tz_name) { pg_tz *tz; int tzoff; struct pg_tm tm; fsec_t fsec; const char *abbrev; bool found = false; TimestampTz now = GetCurrentTransactionStartTimestamp(); pg_tzenum *tzenum = pg_tzenumerate_start(); while (true) { tz = pg_tzenumerate_next(tzenum); if (!tz) break; /* * Convert now() to time in this TZ and skip if conversion fails. * This check is the same that pg_timezone_names() does. */ if (timestamp2tm(now, &tzoff, &tm, &fsec, &abbrev, tz) != 0) continue; if ((!strcmp(tz_name, pg_get_timezone_name(tz))) || (abbrev && !strcmp(tz_name, abbrev))) { found = true; break; } } pg_tzenumerate_end(tzenum); return found; } ================================================ FILE: src/timezones.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include "export.h" extern TSDLLEXPORT bool ts_is_valid_timezone_name(const char *tz_name); ================================================ FILE: src/trigger.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <commands/trigger.h> #include <miscadmin.h> #include <parser/analyze.h> #include <tcop/tcopprot.h> #include <utils/builtins.h> #include <utils/lsyscache.h> #include <utils/rel.h> #include <utils/syscache.h> #include "trigger.h" /* * Replicate a trigger on a chunk. * * Given a trigger OID (e.g., a Hypertable trigger), create the equivalent * trigger on a chunk. * * Note: it is assumed that this function is called under a user that has * permissions to modify the chunk since CreateTrigger() performs permissions * checks. */ void ts_trigger_create_on_chunk(Oid trigger_oid, const char *chunk_schema_name, const char *chunk_table_name) { Datum datum_def = DirectFunctionCall1(pg_get_triggerdef, ObjectIdGetDatum(trigger_oid)); const char *def = TextDatumGetCString(datum_def); List *deparsed_list; Node *deparsed_node; CreateTrigStmt *stmt; deparsed_list = pg_parse_query(def); Assert(list_length(deparsed_list) == 1); deparsed_node = linitial(deparsed_list); do { RawStmt *rawstmt = (RawStmt *) deparsed_node; ParseState *pstate = make_parsestate(NULL); Query *query; Assert(IsA(deparsed_node, RawStmt)); pstate->p_sourcetext = def; query = transformTopLevelStmt(pstate, rawstmt); free_parsestate(pstate); stmt = (CreateTrigStmt *) query->utilityStmt; } while (0); Assert(IsA(stmt, CreateTrigStmt)); stmt->relation->relname = (char *) chunk_table_name; stmt->relation->schemaname = (char *) chunk_schema_name; /* Using OR REPLACE option introduced on Postgres 14 */ stmt->replace = true; CreateTrigger(stmt, def, InvalidOid, InvalidOid, InvalidOid, InvalidOid, InvalidOid, InvalidOid, NULL, false, false); CommandCounterIncrement(); /* needed to prevent pg_class being updated * twice */ } typedef bool (*trigger_handler)(const Trigger *trigger, void *arg); static inline void for_each_trigger(Oid relid, trigger_handler on_trigger, void *arg) { Relation rel; rel = table_open(relid, AccessShareLock); if (rel->trigdesc != NULL) { int i; /* * The TriggerDesc from rel->trigdesc seems to be modified during * iterations of the loop and sometimes gets reallocated so we * access trigdesc only through rel->trigdesc. */ for (i = 0; i < rel->trigdesc->numtriggers; i++) { Trigger *trigger = &rel->trigdesc->triggers[i]; if (!on_trigger(trigger, arg)) break; } } table_close(rel, AccessShareLock); } static bool create_trigger_handler(const Trigger *trigger, void *arg) { const Chunk *chunk = arg; if ((TRIGGER_USES_TRANSITION_TABLE(trigger->tgoldtable) || TRIGGER_USES_TRANSITION_TABLE(trigger->tgnewtable)) && TRIGGER_FOR_ROW(trigger->tgtype)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("ROW triggers with transition tables are not supported on hypertable " "chunks"))); if (trigger && TRIGGER_FOR_ROW(trigger->tgtype) && !trigger->tgisinternal) ts_trigger_create_on_chunk(trigger->tgoid, NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name)); return true; } /* * Create all hypertable triggers on a new chunk. * * Since chunk creation typically happens automatically on hypertable INSERT, we * need to execute the trigger creation under the role of the hypertable owner. * This is due to the use of CreateTrigger(), which does permissions checks. The * user role inserting might have INSERT permissions, but not TRIGGER * permissions (needed to create triggers on a table). * * We assume that the owner of the Hypertable is also the owner of the new * chunk. */ void ts_trigger_create_all_on_chunk(const Chunk *chunk) { int sec_ctx; Oid saved_uid; Oid owner; /* We do not create triggers on foreign table chunks */ if (chunk->relkind == RELKIND_FOREIGN_TABLE) return; Assert(chunk->relkind == RELKIND_RELATION); owner = ts_rel_get_owner(chunk->hypertable_relid); GetUserIdAndSecContext(&saved_uid, &sec_ctx); if (saved_uid != owner) SetUserIdAndSecContext(owner, sec_ctx | SECURITY_LOCAL_USERID_CHANGE); for_each_trigger(chunk->hypertable_relid, create_trigger_handler, (Chunk *) chunk); if (saved_uid != owner) SetUserIdAndSecContext(saved_uid, sec_ctx); } static bool check_for_transition_table(const Trigger *trigger, void *arg) { if ((TRIGGER_USES_TRANSITION_TABLE(trigger->tgnewtable) || TRIGGER_USES_TRANSITION_TABLE(trigger->tgoldtable)) && TRIGGER_FOR_ROW(trigger->tgtype)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("ROW triggers with transition tables are not supported on hypertables"))); return false; } return true; } void ts_check_unsupported_triggers(Oid relid) { for_each_trigger(relid, check_for_transition_table, NULL); } ================================================ FILE: src/trigger.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include "chunk.h" #include "hypertable.h" #include <catalog/pg_trigger.h> extern void ts_trigger_create_on_chunk(Oid trigger_oid, const char *chunk_schema_name, const char *chunk_table_name); extern TSDLLEXPORT void ts_trigger_create_all_on_chunk(const Chunk *chunk); extern void ts_check_unsupported_triggers(Oid relid); ================================================ FILE: src/ts_catalog/CMakeLists.txt ================================================ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/array_utils.c ${CMAKE_CURRENT_SOURCE_DIR}/catalog.c ${CMAKE_CURRENT_SOURCE_DIR}/chunk_column_stats.c ${CMAKE_CURRENT_SOURCE_DIR}/chunk_rewrite.c ${CMAKE_CURRENT_SOURCE_DIR}/compression_chunk_size.c ${CMAKE_CURRENT_SOURCE_DIR}/compression_settings.c ${CMAKE_CURRENT_SOURCE_DIR}/continuous_agg.c ${CMAKE_CURRENT_SOURCE_DIR}/continuous_aggs_watermark.c ${CMAKE_CURRENT_SOURCE_DIR}/metadata.c ${CMAKE_CURRENT_SOURCE_DIR}/tablespace.c) target_sources(${PROJECT_NAME} PRIVATE ${SOURCES}) ================================================ FILE: src/ts_catalog/array_utils.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_collation.h> #include <catalog/pg_type.h> #include <fmgr.h> #include <utils/array.h> #include <utils/builtins.h> #include <utils/fmgroids.h> #include "compat/compat.h" #include "array_utils.h" #include "debug_assert.h" /* * Array helper function for internal catalog arrays. * These are not suitable for arbitrary dimension * arrays but only for 1-dimensional arrays as we use * them in our catalog. */ extern TSDLLEXPORT int ts_array_length(ArrayType *arr) { if (!arr || ARR_NDIM(arr) == 0) return 0; Assert(ARR_NDIM(arr) == 1); return ARR_DIMS(arr)[0]; } extern TSDLLEXPORT bool ts_array_equal(ArrayType *left, ArrayType *right) { /* Quick exit if both are NULL or point to same thing. */ if (left == right) return true; if (left == NULL || right == NULL) return false; if (ARR_NDIM(left) == 0 || ARR_NDIM(right) == 0) { if (ARR_NDIM(left) == 0 && ARR_NDIM(right) == 0) return true; else return false; } Assert(left != NULL && right != NULL && ARR_NDIM(left) == 1 && ARR_NDIM(right) == 1); Datum result = OidFunctionCall2Coll(F_ARRAY_EQ, DEFAULT_COLLATION_OID, PointerGetDatum(left), PointerGetDatum(right)); return DatumGetBool(result); } extern TSDLLEXPORT bool ts_array_is_member(ArrayType *arr, const char *name) { bool ret = false; Datum datum; bool null; if (!arr || ARR_NDIM(arr) == 0) return ret; Assert(ARR_NDIM(arr) == 1); Assert(arr->elemtype == TEXTOID); Assert(name); ArrayIterator it = array_create_iterator(arr, 0, NULL); while (array_iterate(it, &datum, &null)) { Assert(!null); /* * Our internal catalog arrays should either be NULL or * have non-NULL members. During normal operation it should * never have NULL members. If we have NULL members either * the catalog is corrupted or some catalog tampering has * happened. */ Ensure(!null, "array element was NULL"); if (strncmp(TextDatumGetCString(datum), name, NAMEDATALEN) == 0) { ret = true; break; } } array_free_iterator(it); return ret; } extern TSDLLEXPORT void ts_array_append_stringinfo(ArrayType *arr, StringInfo info) { bool first = true; Datum datum; bool null; if (!arr) return; Assert(ARR_NDIM(arr) <= 1); Assert(arr->elemtype == TEXTOID); ArrayIterator it = array_create_iterator(arr, 0, NULL); while (array_iterate(it, &datum, &null)) { Assert(!null); /* * Our internal catalog arrays should either be NULL or * have non-NULL members. During normal operation it should * never have NULL members. If we have NULL members either * the catalog is corrupted or some catalog tampering has * happened. */ Ensure(!null, "array element was NULL"); if (!first) appendStringInfoString(info, ", "); else first = false; appendStringInfo(info, "%s", TextDatumGetCString(datum)); } array_free_iterator(it); } extern TSDLLEXPORT int ts_array_position(ArrayType *arr, const char *name) { int pos = 0; Datum datum; bool found = false; bool null; if (!arr || ARR_NDIM(arr) == 0) return pos; Assert(ARR_NDIM(arr) == 1); Assert(arr->elemtype == TEXTOID); ArrayIterator it = array_create_iterator(arr, 0, NULL); while (array_iterate(it, &datum, &null)) { pos++; /* * Our internal catalog arrays should either be NULL or * have non-NULL members. During normal operation it should * never have NULL members. If we have NULL members either * the catalog is corrupted or some catalog tampering has * happened. */ Ensure(!null, "array element was NULL"); if (strncmp(TextDatumGetCString(datum), name, NAMEDATALEN) == 0) { found = true; break; } } array_free_iterator(it); return found ? pos : 0; } extern TSDLLEXPORT ArrayType * ts_array_replace_text(ArrayType *arr, const char *old, const char *new) { if (!arr || ARR_NDIM(arr) == 0) return NULL; Assert(ARR_NDIM(arr) == 1); Assert(arr->elemtype == TEXTOID); Datum datum; bool null; int pos = 1; ArrayIterator it = array_create_iterator(arr, 0, NULL); while (array_iterate(it, &datum, &null)) { /* * Our internal catalog arrays should either be NULL or * have non-NULL members. During normal operation it should * never have NULL members. If we have NULL members either * the catalog is corrupted or some catalog tampering has * happened. */ Ensure(!null, "array element was NULL"); if (strncmp(TextDatumGetCString(datum), old, NAMEDATALEN) == 0) { datum = array_set_element(PointerGetDatum(arr), 1, &pos, CStringGetTextDatum(new), false, -1, -1, false, TYPALIGN_INT); arr = DatumGetArrayTypeP(datum); } pos++; } array_free_iterator(it); return arr; } extern TSDLLEXPORT bool ts_array_get_element_bool(ArrayType *arr, int position) { Assert(arr); Assert(ARR_NDIM(arr) == 1); Assert(arr->elemtype == BOOLOID); bool isnull; Datum value = array_get_element(PointerGetDatum(arr), 1, &position, -1, 1, true, 'c', &isnull); Ensure(!isnull, "invalid array position"); return DatumGetBool(value); } extern TSDLLEXPORT const char * ts_array_get_element_text(ArrayType *arr, int position) { Assert(arr); Assert(ARR_NDIM(arr) == 1); Assert(arr->elemtype == TEXTOID); bool isnull; Datum value = array_get_element(PointerGetDatum(arr), 1, &position, -1, -1, false, 'i', &isnull); Ensure(!isnull, "invalid array position"); return TextDatumGetCString(value); } extern TSDLLEXPORT ArrayType * ts_array_add_element_text(ArrayType *arr, const char *value) { if (!arr && value == NULL) { /* return empty array */ return construct_array(NULL, 0, TEXTOID, -1, false, TYPALIGN_INT); } Datum val = CStringGetTextDatum(value); if (!arr) { return construct_array(&val, 1, TEXTOID, -1, false, TYPALIGN_INT); } else { Assert(ARR_NDIM(arr) == 1); Assert(arr->elemtype == TEXTOID); Datum d = PointerGetDatum(arr); int position = ts_array_length(arr); Assert(position); position++; d = array_set_element(d, 1, &position, val, false, -1, -1, false, TYPALIGN_INT); return DatumGetArrayTypeP(d); } } extern TSDLLEXPORT ArrayType * ts_array_add_element_bool(ArrayType *arr, bool value) { if (!arr) { Datum val = BoolGetDatum(value); return construct_array(&val, 1, BOOLOID, 1, true, TYPALIGN_CHAR); } else { Assert(ARR_NDIM(arr) == 1); Assert(arr->elemtype == BOOLOID); Datum d = PointerGetDatum(arr); int position = ts_array_length(arr); Assert(position); position++; d = array_set_element(d, 1, &position, BoolGetDatum(value), false, -1, 1, true, TYPALIGN_CHAR); return DatumGetArrayTypeP(d); } } ================================================ FILE: src/ts_catalog/array_utils.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <lib/stringinfo.h> #include <utils/array.h> #include "export.h" /* * Array helper function for internal catalog arrays. * These are not suitable for arbitrary dimension * arrays but only for 1-dimensional arrays as we use * them in our catalog. */ extern TSDLLEXPORT int ts_array_length(ArrayType *arr); extern TSDLLEXPORT bool ts_array_equal(ArrayType *left, ArrayType *right); extern TSDLLEXPORT bool ts_array_is_member(ArrayType *arr, const char *name); extern TSDLLEXPORT void ts_array_append_stringinfo(ArrayType *arr, StringInfo info); extern TSDLLEXPORT int ts_array_position(ArrayType *arr, const char *name); extern TSDLLEXPORT bool ts_array_get_element_bool(ArrayType *arr, int position); extern TSDLLEXPORT const char *ts_array_get_element_text(ArrayType *arr, int position); extern TSDLLEXPORT ArrayType *ts_array_add_element_bool(ArrayType *arr, bool value); extern TSDLLEXPORT ArrayType *ts_array_add_element_text(ArrayType *arr, const char *value); extern TSDLLEXPORT ArrayType *ts_array_replace_text(ArrayType *arr, const char *old, const char *new); ================================================ FILE: src/ts_catalog/catalog.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/htup_details.h> #include <access/xact.h> #include <catalog/indexing.h> #include <catalog/namespace.h> #include <catalog/pg_namespace.h> #include <commands/dbcommands.h> #include <commands/sequence.h> #include <miscadmin.h> #include <utils/builtins.h> #include <utils/inval.h> #include <utils/lsyscache.h> #include <utils/regproc.h> #include <utils/syscache.h> #include "compat/compat.h" #include "cache_invalidate.h" #include "extension.h" #include "ts_catalog/catalog.h" #include "utils.h" static const TableInfoDef catalog_table_names[_MAX_CATALOG_TABLES + 1] = { [HYPERTABLE] = { .schema_name = CATALOG_SCHEMA_NAME, .table_name = HYPERTABLE_TABLE_NAME, }, [DIMENSION] = { .schema_name = CATALOG_SCHEMA_NAME, .table_name = DIMENSION_TABLE_NAME, }, [DIMENSION_SLICE] = { .schema_name = CATALOG_SCHEMA_NAME, .table_name = DIMENSION_SLICE_TABLE_NAME, }, [CHUNK] = { .schema_name = CATALOG_SCHEMA_NAME, .table_name = CHUNK_TABLE_NAME, }, [CHUNK_CONSTRAINT] = { .schema_name = CATALOG_SCHEMA_NAME, .table_name = CHUNK_CONSTRAINT_TABLE_NAME, }, [CHUNK_REWRITE] = { .schema_name = CATALOG_SCHEMA_NAME, .table_name = CHUNK_REWRITE_TABLE_NAME, }, [TABLESPACE] = { .schema_name = CATALOG_SCHEMA_NAME, .table_name = TABLESPACE_TABLE_NAME, }, [BGW_JOB] = { .schema_name = CATALOG_SCHEMA_NAME, .table_name = BGW_JOB_TABLE_NAME, }, [BGW_JOB_STAT] = { .schema_name = INTERNAL_SCHEMA_NAME, .table_name = BGW_JOB_STAT_TABLE_NAME, }, [BGW_JOB_STAT_HISTORY] = { .schema_name = INTERNAL_SCHEMA_NAME, .table_name = BGW_JOB_STAT_HISTORY_TABLE_NAME, }, [METADATA] = { .schema_name = CATALOG_SCHEMA_NAME, .table_name = METADATA_TABLE_NAME, }, [BGW_POLICY_CHUNK_STATS] = { .schema_name = INTERNAL_SCHEMA_NAME, .table_name = BGW_POLICY_CHUNK_STATS_TABLE_NAME, }, [CONTINUOUS_AGG] = { .schema_name = CATALOG_SCHEMA_NAME, .table_name = CONTINUOUS_AGG_TABLE_NAME, }, [CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG] = { .schema_name = CATALOG_SCHEMA_NAME, .table_name = CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG_TABLE_NAME, }, [CONTINUOUS_AGGS_INVALIDATION_THRESHOLD] = { .schema_name = CATALOG_SCHEMA_NAME, .table_name = CONTINUOUS_AGGS_INVALIDATION_THRESHOLD_TABLE_NAME, }, [CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG] = { .schema_name = CATALOG_SCHEMA_NAME, .table_name = CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG_TABLE_NAME, }, [CONTINUOUS_AGGS_MATERIALIZATION_RANGES] = { .schema_name = CATALOG_SCHEMA_NAME, .table_name = CONTINUOUS_AGGS_MATERIALIZATION_RANGES_TABLE_NAME, }, [COMPRESSION_SETTINGS] = { .schema_name = CATALOG_SCHEMA_NAME, .table_name = COMPRESSION_SETTINGS_TABLE_NAME, }, [COMPRESSION_CHUNK_SIZE] = { .schema_name = CATALOG_SCHEMA_NAME, .table_name = COMPRESSION_CHUNK_SIZE_TABLE_NAME, }, [CONTINUOUS_AGGS_BUCKET_FUNCTION] = { .schema_name = CATALOG_SCHEMA_NAME, .table_name = CONTINUOUS_AGGS_BUCKET_FUNCTION_TABLE_NAME, }, [CONTINUOUS_AGGS_WATERMARK] = { .schema_name = CATALOG_SCHEMA_NAME, .table_name = CONTINUOUS_AGGS_WATERMARK_TABLE_NAME, }, [TELEMETRY_EVENT] = { .schema_name = CATALOG_SCHEMA_NAME, .table_name = TELEMETRY_EVENT_TABLE_NAME, }, [CHUNK_COLUMN_STATS] = { .schema_name = CATALOG_SCHEMA_NAME, .table_name = CHUNK_COLUMN_STATS_TABLE_NAME, }, [_MAX_CATALOG_TABLES] = { .schema_name = "invalid schema", .table_name = "invalid table", } }; static const TableIndexDef catalog_table_index_definitions[_MAX_CATALOG_TABLES] = { [HYPERTABLE] = { .length = _MAX_HYPERTABLE_INDEX, .names = (char *[]) { [HYPERTABLE_ID_INDEX] = "hypertable_pkey", [HYPERTABLE_NAME_INDEX] = "hypertable_table_name_schema_name_key", }, }, [DIMENSION] = { .length = _MAX_DIMENSION_INDEX, .names = (char *[]) { [DIMENSION_ID_IDX] = "dimension_pkey", [DIMENSION_HYPERTABLE_ID_COLUMN_NAME_IDX] = "dimension_hypertable_id_column_name_key", }, }, [DIMENSION_SLICE] = { .length = _MAX_DIMENSION_SLICE_INDEX, .names = (char *[]) { [DIMENSION_SLICE_ID_IDX] = "dimension_slice_pkey", [DIMENSION_SLICE_DIMENSION_ID_RANGE_START_RANGE_END_IDX] = "dimension_slice_dimension_id_range_start_range_end_key", }, }, [CHUNK_COLUMN_STATS] = { .length = _MAX_CHUNK_COLUMN_STATS_INDEX, .names = (char *[]) { [CHUNK_COLUMN_STATS_ID_IDX] = "chunk_column_stats_pkey", [CHUNK_COLUMN_STATS_HT_ID_CHUNK_ID_COLUMN_NAME_IDX] = "chunk_column_stats_ht_id_chunk_id_colname_key", }, }, [CHUNK] = { .length = _MAX_CHUNK_INDEX, .names = (char *[]) { [CHUNK_ID_INDEX] = "chunk_pkey", [CHUNK_HYPERTABLE_ID_INDEX] = "chunk_hypertable_id_idx", [CHUNK_SCHEMA_NAME_INDEX] = "chunk_schema_name_table_name_key", [CHUNK_COMPRESSED_CHUNK_ID_INDEX] = "chunk_compressed_chunk_id_idx", [CHUNK_OSM_CHUNK_INDEX] = "chunk_osm_chunk_idx", [CHUNK_HYPERTABLE_ID_CREATION_TIME_INDEX] = "chunk_hypertable_id_creation_time_idx", }, }, [CHUNK_CONSTRAINT] = { .length = _MAX_CHUNK_CONSTRAINT_INDEX, .names = (char *[]) { [CHUNK_CONSTRAINT_CHUNK_ID_CONSTRAINT_NAME_IDX] = "chunk_constraint_chunk_id_constraint_name_key", [CHUNK_CONSTRAINT_DIMENSION_SLICE_ID_IDX] = "chunk_constraint_dimension_slice_id_idx", }, }, [CHUNK_REWRITE] = { .length = _MAX_CHUNK_REWRITE_INDEX, .names = (char *[]) { [CHUNK_REWRITE_IDX] = "chunk_rewrite_key", }, }, [TABLESPACE] = { .length = _MAX_TABLESPACE_INDEX, .names = (char *[]) { [TABLESPACE_PKEY_IDX] = "tablespace_pkey", [TABLESPACE_HYPERTABLE_ID_TABLESPACE_NAME_IDX] = "tablespace_hypertable_id_tablespace_name_key", }, }, [BGW_JOB] = { .length = _MAX_BGW_JOB_INDEX, .names = (char *[]) { [BGW_JOB_PKEY_IDX] = "bgw_job_pkey", [BGW_JOB_PROC_HYPERTABLE_ID_IDX] = "bgw_job_proc_hypertable_id_idx", }, }, [BGW_JOB_STAT] = { .length = _MAX_BGW_JOB_STAT_INDEX, .names = (char *[]) { [BGW_JOB_STAT_PKEY_IDX] = "bgw_job_stat_pkey", }, }, [BGW_JOB_STAT_HISTORY] = { .length = _MAX_BGW_JOB_STAT_HISTORY_INDEX, .names = (char *[]) { [BGW_JOB_STAT_HISTORY_PKEY_IDX] = "bgw_job_stat_history_pkey", }, }, [METADATA] = { .length = _MAX_METADATA_INDEX, .names = (char *[]) { [METADATA_PKEY_IDX] = "metadata_pkey", }, }, [BGW_POLICY_CHUNK_STATS] = { .length = _MAX_BGW_POLICY_CHUNK_STATS_INDEX, .names = (char *[]) { [BGW_POLICY_CHUNK_STATS_JOB_ID_CHUNK_ID_IDX] = "bgw_policy_chunk_stats_job_id_chunk_id_key", } }, [CONTINUOUS_AGG] = { .length = _MAX_CONTINUOUS_AGG_INDEX, .names = (char *[]) { [CONTINUOUS_AGG_PARTIAL_VIEW_SCHEMA_PARTIAL_VIEW_NAME_KEY] = "continuous_agg_partial_view_schema_partial_view_name_key", [CONTINUOUS_AGG_PKEY] = TS_CAGG_CATALOG_IDX, [CONTINUOUS_AGG_USER_VIEW_SCHEMA_USER_VIEW_NAME_KEY] = "continuous_agg_user_view_schema_user_view_name_key", [CONTINUOUS_AGG_RAW_HYPERTABLE_ID_IDX] = "continuous_agg_raw_hypertable_id_idx" }, }, [CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG] = { .length = _MAX_CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG_INDEX, .names = (char *[]) { [CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG_IDX] = "continuous_aggs_hypertable_invalidation_log_idx", }, }, [CONTINUOUS_AGGS_INVALIDATION_THRESHOLD] = { .length = _MAX_CONTINUOUS_AGGS_INVALIDATION_THRESHOLD_INDEX, .names = (char *[]) { [CONTINUOUS_AGGS_INVALIDATION_THRESHOLD_PKEY] = "continuous_aggs_invalidation_threshold_pkey", }, }, [CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG] = { .length = _MAX_CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG_INDEX, .names = (char *[]) { [CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG_IDX] = "continuous_aggs_materialization_invalidation_log_idx", }, }, [CONTINUOUS_AGGS_MATERIALIZATION_RANGES] = { .length = _MAX_CONTINUOUS_AGGS_MATERIALIZATION_RANGES_INDEX, .names = (char *[]) { [CONTINUOUS_AGGS_MATERIALIZATION_RANGES_IDX] = "continuous_aggs_materialization_ranges_idx", }, }, [CONTINUOUS_AGGS_WATERMARK] = { .length = _MAX_CONTINUOUS_AGGS_WATERMARK_INDEX, .names = (char *[]) { [CONTINUOUS_AGGS_WATERMARK_PKEY] = "continuous_aggs_watermark_pkey", }, }, [COMPRESSION_SETTINGS] = { .length = _MAX_COMPRESSION_SETTINGS_INDEX, .names = (char *[]) { [COMPRESSION_SETTINGS_PKEY] = "compression_settings_pkey", [COMPRESSION_SETTINGS_COMPRESS_RELID_IDX] = "compression_settings_compress_relid_idx", }, }, [COMPRESSION_CHUNK_SIZE] = { .length = _MAX_COMPRESSION_CHUNK_SIZE_INDEX, .names = (char *[]) { [COMPRESSION_CHUNK_SIZE_PKEY] = "compression_chunk_size_pkey", }, }, [CONTINUOUS_AGGS_BUCKET_FUNCTION] = { .length = _MAX_CONTINUOUS_AGGS_BUCKET_FUNCTION_INDEX, .names = (char *[]) { [CONTINUOUS_AGGS_BUCKET_FUNCTION_PKEY_IDX] = "continuous_aggs_bucket_function_pkey", }, } }; static const char *catalog_table_serial_id_names[_MAX_CATALOG_TABLES] = { [HYPERTABLE] = CATALOG_SCHEMA_NAME ".hypertable_id_seq", [DIMENSION] = CATALOG_SCHEMA_NAME ".dimension_id_seq", [DIMENSION_SLICE] = CATALOG_SCHEMA_NAME ".dimension_slice_id_seq", [CHUNK] = CATALOG_SCHEMA_NAME ".chunk_id_seq", [CHUNK_CONSTRAINT] = CATALOG_SCHEMA_NAME ".chunk_constraint_name", [CHUNK_REWRITE] = NULL, [TABLESPACE] = CATALOG_SCHEMA_NAME ".tablespace_id_seq", [BGW_JOB] = CATALOG_SCHEMA_NAME ".bgw_job_id_seq", [BGW_JOB_STAT] = NULL, [BGW_JOB_STAT_HISTORY] = INTERNAL_SCHEMA_NAME ".bgw_job_stat_history_id_seq", [CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG] = NULL, [CONTINUOUS_AGGS_INVALIDATION_THRESHOLD] = NULL, [CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG] = NULL, [COMPRESSION_SETTINGS] = NULL, [COMPRESSION_CHUNK_SIZE] = NULL, [CHUNK_COLUMN_STATS] = CATALOG_SCHEMA_NAME ".chunk_column_stats_id_seq", }; typedef struct InternalFunctionDef { char *name; int args; } InternalFunctionDef; static const InternalFunctionDef internal_function_definitions[_MAX_INTERNAL_FUNCTIONS] = { [DDL_ADD_CHUNK_CONSTRAINT] = { .name = "chunk_constraint_add_table_constraint", .args = 1, }, [DDL_CONSTRAINT_CLONE] = { .name = "constraint_clone", .args = 2, }, }; /* Names for proxy tables used for cache invalidation. Must match names in * sql/cache.sql */ static const char *cache_proxy_table_names[_MAX_CACHE_TYPES] = { [CACHE_TYPE_HYPERTABLE] = "cache_inval_hypertable", [CACHE_TYPE_BGW_JOB] = "cache_inval_bgw_job", [CACHE_TYPE_EXTENSION] = "cache_inval_extension", }; /* Catalog information for the current database. */ static Catalog s_catalog = { .initialized = false, }; static CatalogDatabaseInfo database_info = { .database_id = InvalidOid, }; static bool catalog_is_valid(Catalog *catalog) { return catalog != NULL && catalog->initialized; } /* * Get the user ID of the catalog owner. */ static Oid catalog_owner(void) { HeapTuple tuple; Oid owner_oid; Oid nsp_oid = get_namespace_oid(CATALOG_SCHEMA_NAME, false); tuple = SearchSysCache1(NAMESPACEOID, ObjectIdGetDatum(nsp_oid)); if (!HeapTupleIsValid(tuple)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_SCHEMA), errmsg("schema with OID %u does not exist", nsp_oid))); owner_oid = ((Form_pg_namespace) GETSTRUCT(tuple))->nspowner; ReleaseSysCache(tuple); return owner_oid; } static const char * catalog_table_name(CatalogTable table) { return catalog_table_names[table].table_name; } static void catalog_database_info_init(CatalogDatabaseInfo *info) { info->database_id = MyDatabaseId; namestrcpy(&info->database_name, get_database_name(MyDatabaseId)); info->schema_id = get_namespace_oid(CATALOG_SCHEMA_NAME, false); info->owner_uid = catalog_owner(); if (!OidIsValid(info->schema_id)) elog(ERROR, "OID lookup failed for schema \"%s\"", CATALOG_SCHEMA_NAME); } TSDLLEXPORT CatalogDatabaseInfo * ts_catalog_database_info_get() { if (!ts_extension_is_loaded()) elog(ERROR, "tried calling catalog_database_info_get when extension isn't loaded"); if (!OidIsValid(database_info.database_id)) { if (!IsTransactionState()) elog(ERROR, "cannot initialize catalog_database_info outside of a transaction"); memset(&database_info, 0, sizeof(CatalogDatabaseInfo)); catalog_database_info_init(&database_info); } return &database_info; } /* * The rest of the arguments are used to populate the first arg. */ void ts_catalog_table_info_init(CatalogTableInfo *tables_info, int max_tables, const TableInfoDef *table_ary, const TableIndexDef *index_ary, const char **serial_id_ary) { int i; for (i = 0; i < max_tables; i++) { Oid id; const char *sequence_name; Size number_indexes, j; id = ts_get_relation_relid((char *) table_ary[i].schema_name, (char *) table_ary[i].table_name, false); if (!OidIsValid(id)) elog(ERROR, "OID lookup failed for table \"%s.%s\"", table_ary[i].schema_name, table_ary[i].table_name); tables_info[i].id = id; number_indexes = index_ary[i].length; Assert(number_indexes <= _MAX_TABLE_INDEXES); for (j = 0; j < number_indexes; j++) { id = ts_get_relation_relid(table_ary[i].schema_name, index_ary[i].names[j], true); if (!OidIsValid(id)) elog(ERROR, "OID lookup failed for table index \"%s\"", index_ary[i].names[j]); tables_info[i].index_ids[j] = id; } tables_info[i].name = table_ary[i].table_name; tables_info[i].schema_name = table_ary[i].schema_name; sequence_name = serial_id_ary[i]; if (NULL != sequence_name) { RangeVar *sequence; #if PG16_LT sequence = makeRangeVarFromNameList(stringToQualifiedNameList(sequence_name)); #else sequence = makeRangeVarFromNameList(stringToQualifiedNameList(sequence_name, NULL)); #endif tables_info[i].serial_relid = RangeVarGetRelid(sequence, NoLock, false); } else tables_info[i].serial_relid = InvalidOid; } } TSDLLEXPORT Catalog * ts_catalog_get(void) { int i; if (!OidIsValid(MyDatabaseId)) elog(ERROR, "invalid database ID"); if (!ts_extension_is_loaded()) elog(ERROR, "tried calling catalog_get when extension isn't loaded"); if (s_catalog.initialized || !IsTransactionState()) return &s_catalog; memset(&s_catalog, 0, sizeof(Catalog)); ts_catalog_table_info_init(s_catalog.tables, _MAX_CATALOG_TABLES, catalog_table_names, catalog_table_index_definitions, catalog_table_serial_id_names); for (i = 0; i < _TS_MAX_SCHEMA; i++) s_catalog.extension_schema_id[i] = get_namespace_oid(ts_extension_schema_names[i], false); for (i = 0; i < _MAX_CACHE_TYPES; i++) s_catalog.caches[i].inval_proxy_id = get_relname_relid(cache_proxy_table_names[i], s_catalog.extension_schema_id[TS_CACHE_SCHEMA]); ts_cache_invalidate_set_proxy_tables(s_catalog.caches[CACHE_TYPE_HYPERTABLE].inval_proxy_id, s_catalog.caches[CACHE_TYPE_BGW_JOB].inval_proxy_id); for (i = 0; i < _MAX_INTERNAL_FUNCTIONS; i++) { InternalFunctionDef def = internal_function_definitions[i]; FuncCandidateList funclist = FuncnameGetCandidates(list_make2(makeString(FUNCTIONS_SCHEMA_NAME), makeString(def.name)), def.args, NULL, false, false, /* include_out_arguments */ false, false); if (funclist == NULL || funclist->next) elog(ERROR, "OID lookup failed for the function \"%s\" with %d args", def.name, def.args); s_catalog.functions[i].function_id = funclist->oid; } s_catalog.initialized = true; return &s_catalog; } void ts_catalog_reset(void) { s_catalog.initialized = false; database_info.database_id = InvalidOid; ts_cache_invalidate_set_proxy_tables(InvalidOid, InvalidOid); } static CatalogTable catalog_get_table(Catalog *catalog, Oid relid) { unsigned int i; if (!catalog_is_valid(catalog)) { const char *schema_name = get_namespace_name(get_rel_namespace(relid)); const char *relname = get_rel_name(relid); for (i = 0; i < _MAX_CATALOG_TABLES; i++) if (strcmp(catalog_table_names[i].schema_name, schema_name) == 0 && strcmp(catalog_table_name(i), relname) == 0) return (CatalogTable) i; return INVALID_CATALOG_TABLE; } for (i = 0; i < _MAX_CATALOG_TABLES; i++) if (catalog->tables[i].id == relid) return (CatalogTable) i; return INVALID_CATALOG_TABLE; } bool ts_is_catalog_table(Oid relid) { return catalog_get_table(ts_catalog_get(), relid) != INVALID_CATALOG_TABLE; } /* * Get the next serial ID for a catalog table, if one exists for the given table. */ TSDLLEXPORT int64 ts_catalog_table_next_seq_id(const Catalog *catalog, CatalogTable table) { Oid relid = catalog->tables[table].serial_relid; if (!OidIsValid(relid)) elog(ERROR, "no serial ID column for table \"%s.%s\"", catalog_table_names[table].schema_name, catalog_table_name(table)); return DatumGetInt64(DirectFunctionCall1(nextval_oid, ObjectIdGetDatum(relid))); } Oid ts_catalog_get_cache_proxy_id(Catalog *catalog, CacheType type) { if (!catalog_is_valid(catalog)) { /* * The catalog can be invalid during upgrade scripts. Try a non-cached * relation lookup, but we need to be in a transaction for * get_namespace_oid() to work. */ if (!IsTransactionState()) return InvalidOid; return ts_get_relation_relid(CACHE_SCHEMA_NAME, (char *) cache_proxy_table_names[type], true); } return catalog->caches[type].inval_proxy_id; } /* * Become the user that owns the catalog schema. * * This might be necessary for users that do operations that require changes to * the catalog. * * The caller should pass a CatalogSecurityContext where the current security * context will be saved. The original security context can later be restored * with ts_catalog_restore_user(). */ TSDLLEXPORT bool ts_catalog_database_info_become_owner(CatalogDatabaseInfo *database_info, CatalogSecurityContext *sec_ctx) { GetUserIdAndSecContext(&sec_ctx->saved_uid, &sec_ctx->saved_security_context); if (sec_ctx->saved_uid != database_info->owner_uid) { SetUserIdAndSecContext(database_info->owner_uid, sec_ctx->saved_security_context | SECURITY_LOCAL_USERID_CHANGE); return true; } return false; } /* * Restore the security context of the original user after becoming the catalog * owner. The user should pass the original CatalogSecurityContext that was used * with ts_catalog_database_info_become_owner(). */ TSDLLEXPORT void ts_catalog_restore_user(CatalogSecurityContext *sec_ctx) { SetUserIdAndSecContext(sec_ctx->saved_uid, sec_ctx->saved_security_context); } /* * Insert a new row into a catalog table. */ void ts_catalog_insert_only(Relation rel, HeapTuple tuple) { CatalogTupleInsert(rel, tuple); ts_catalog_invalidate_cache(RelationGetRelid(rel), CMD_INSERT); } void ts_catalog_insert(Relation rel, HeapTuple tuple) { ts_catalog_insert_only(rel, tuple); /* Make changes visible */ CommandCounterIncrement(); } /* * Insert a new row into a catalog table. */ TSDLLEXPORT void ts_catalog_insert_values(Relation rel, TupleDesc tupdesc, Datum *values, bool *nulls) { HeapTuple tuple = heap_form_tuple(tupdesc, values, nulls); ts_catalog_insert(rel, tuple); heap_freetuple(tuple); } TSDLLEXPORT void ts_catalog_insert_datums(Relation rel, TupleDesc tupdesc, NullableDatum *datums) { HeapTuple tuple = ts_heap_form_tuple(tupdesc, datums); ts_catalog_insert(rel, tuple); heap_freetuple(tuple); } void ts_catalog_update_tid_only(Relation rel, ItemPointer tid, HeapTuple tuple) { CatalogTupleUpdate(rel, tid, tuple); ts_catalog_invalidate_cache(RelationGetRelid(rel), CMD_UPDATE); } void ts_catalog_update_tid(Relation rel, ItemPointer tid, HeapTuple tuple) { ts_catalog_update_tid_only(rel, tid, tuple); /* Make changes visible */ CommandCounterIncrement(); } TSDLLEXPORT void ts_catalog_update(Relation rel, HeapTuple tuple) { ts_catalog_update_tid(rel, &tuple->t_self, tuple); } void ts_catalog_delete_tid_only(Relation rel, ItemPointer tid) { CatalogTupleDelete(rel, tid); ts_catalog_invalidate_cache(RelationGetRelid(rel), CMD_DELETE); } void ts_catalog_delete_tid(Relation rel, ItemPointer tid) { ts_catalog_delete_tid_only(rel, tid); CommandCounterIncrement(); } /* * Invalidate TimescaleDB catalog caches. * * This function should be called whenever a TimescaleDB catalog table changes * in a way that might invalidate associated caches. It is currently called in * two distinct ways: * * 1. If a catalog table changes via the catalog API in catalog.c * 2. Via a trigger if a SQL INSERT/UPDATE/DELETE occurs on a catalog table * * Since triggers (2) require full parsing, planning and execution of SQL * statements, they aren't supported for simple catalog updates via (1) in * native code and are therefore discouraged. Ideally, catalog updates should * happen consistently via method (1) in the future, obviating the need for * triggers on catalog tables that cause side effects. * * The invalidation event is signaled to other backends (processes) via the * relcache invalidation mechanism on a dummy relation (table). * * Parameters: The OID of the catalog table that changed, and the operation * involved (e.g., INSERT, UPDATE, DELETE). */ void ts_catalog_invalidate_cache(Oid catalog_relid, CmdType operation) { Catalog *catalog = ts_catalog_get(); CatalogTable table = catalog_get_table(catalog, catalog_relid); Oid relid; switch (table) { case CHUNK: case CHUNK_CONSTRAINT: case DIMENSION_SLICE: if (operation == CMD_UPDATE || operation == CMD_DELETE) { relid = ts_catalog_get_cache_proxy_id(catalog, CACHE_TYPE_HYPERTABLE); CacheInvalidateRelcacheByRelid(relid); } break; case HYPERTABLE: case DIMENSION: case CONTINUOUS_AGG: case CHUNK_COLUMN_STATS: relid = ts_catalog_get_cache_proxy_id(catalog, CACHE_TYPE_HYPERTABLE); CacheInvalidateRelcacheByRelid(relid); break; case BGW_JOB: relid = ts_catalog_get_cache_proxy_id(catalog, CACHE_TYPE_BGW_JOB); CacheInvalidateRelcacheByRelid(relid); break; default: break; } } /* Scanner helper functions specifically for the catalog tables */ TSDLLEXPORT bool ts_catalog_scan_one(CatalogTable table, int indexid, ScanKeyData *scankey, int num_keys, tuple_found_func tuple_found, LOCKMODE lockmode, char *table_name, void *data) { Catalog *catalog = ts_catalog_get(); ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, table), .index = catalog_get_index(catalog, table, indexid), .nkeys = num_keys, .scankey = scankey, .tuple_found = tuple_found, .data = data, .lockmode = lockmode, .scandirection = ForwardScanDirection, }; return ts_scanner_scan_one(&scanctx, false, table_name); } TSDLLEXPORT void ts_catalog_scan_all(CatalogTable table, int indexid, ScanKeyData *scankey, int num_keys, tuple_found_func tuple_found, LOCKMODE lockmode, void *data) { Catalog *catalog = ts_catalog_get(); ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, table), .index = catalog_get_index(catalog, table, indexid), .nkeys = num_keys, .scankey = scankey, .tuple_found = tuple_found, .data = data, .lockmode = lockmode, .scandirection = ForwardScanDirection, }; ts_scanner_scan(&scanctx); } /* * Copied verbatim from postgres source CatalogIndexInsert which is static * in postgres source code. * We need to have this function available because we do not want to use * simple_heap_insert which is used by CatalogTupleInsert which would * prevent using bulk inserts. */ extern TSDLLEXPORT void ts_catalog_index_insert(ResultRelInfo *indstate, HeapTuple heapTuple) { int i; int numIndexes; RelationPtr relationDescs; Relation heapRelation; TupleTableSlot *slot; IndexInfo **indexInfoArray; Datum values[INDEX_MAX_KEYS]; bool isnull[INDEX_MAX_KEYS]; /* * HOT update does not require index inserts. But with asserts enabled we * want to check that it'd be legal to currently insert into the * table/index. */ #ifndef USE_ASSERT_CHECKING if (HeapTupleIsHeapOnly(heapTuple)) return; #endif /* * Get information from the state structure. Fall out if nothing to do. */ numIndexes = indstate->ri_NumIndices; if (numIndexes == 0) return; relationDescs = indstate->ri_IndexRelationDescs; indexInfoArray = indstate->ri_IndexRelationInfo; heapRelation = indstate->ri_RelationDesc; /* Need a slot to hold the tuple being examined */ slot = MakeSingleTupleTableSlot(RelationGetDescr(heapRelation), &TTSOpsHeapTuple); ExecStoreHeapTuple(heapTuple, slot, false); /* * for each index, form and insert the index tuple */ for (i = 0; i < numIndexes; i++) { IndexInfo *indexInfo; Relation index; indexInfo = indexInfoArray[i]; index = relationDescs[i]; /* If the index is marked as read-only, ignore it */ if (!indexInfo->ii_ReadyForInserts) continue; /* * Expressional and partial indexes on system catalogs are not * supported, nor exclusion constraints, nor deferred uniqueness */ Assert(indexInfo->ii_Expressions == NIL); Assert(indexInfo->ii_Predicate == NIL); Assert(indexInfo->ii_ExclusionOps == NULL); Assert(index->rd_index->indimmediate); Assert(indexInfo->ii_NumIndexKeyAttrs != 0); /* see earlier check above */ #ifdef USE_ASSERT_CHECKING if (HeapTupleIsHeapOnly(heapTuple)) { Assert(!ReindexIsProcessingIndex(RelationGetRelid(index))); continue; } #endif /* USE_ASSERT_CHECKING */ /* * FormIndexDatum fills in its values and isnull parameters with the * appropriate values for the column(s) of the index. */ FormIndexDatum(indexInfo, slot, NULL, /* no expression eval to do */ values, isnull); /* * The index AM does the rest. */ index_insert(index, /* index relation */ values, /* array of index Datums */ isnull, /* is-null flags */ &(heapTuple->t_self), /* tid of heap tuple */ heapRelation, index->rd_index->indisunique ? UNIQUE_CHECK_YES : UNIQUE_CHECK_NO, false, indexInfo); } ExecDropSingleTupleTableSlot(slot); } ================================================ FILE: src/ts_catalog/catalog.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <access/heapam.h> #include <nodes/nodes.h> #include <utils/jsonb.h> #include <utils/rel.h> #include "export.h" #include "extension_constants.h" #include "scanner.h" /* * TimescaleDB catalog. * * The TimescaleDB catalog contains schema metadata for hypertables, among other * things. The metadata is stored in regular tables. This header file contains * definitions for those tables and should match any table declarations in * sql/pre_install/tables.sql. * * A source file that includes this header has access to a catalog object, * which contains cached information about catalog tables, such as relation * OIDs. * * Generally, definitions and naming should roughly follow how things are done * in Postgres internally. */ typedef enum CatalogTable { HYPERTABLE = 0, DIMENSION, DIMENSION_SLICE, CHUNK, CHUNK_CONSTRAINT, CHUNK_REWRITE, TABLESPACE, BGW_JOB, BGW_JOB_STAT, BGW_JOB_STAT_HISTORY, METADATA, BGW_POLICY_CHUNK_STATS, CONTINUOUS_AGG, CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG, CONTINUOUS_AGGS_INVALIDATION_THRESHOLD, CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG, CONTINUOUS_AGGS_MATERIALIZATION_RANGES, COMPRESSION_SETTINGS, COMPRESSION_CHUNK_SIZE, CONTINUOUS_AGGS_BUCKET_FUNCTION, CONTINUOUS_AGGS_WATERMARK, TELEMETRY_EVENT, CHUNK_COLUMN_STATS, /* Don't forget updating catalog.c when adding new tables! */ _MAX_CATALOG_TABLES, } CatalogTable; typedef struct TableInfoDef { const char *schema_name; const char *table_name; } TableInfoDef; typedef struct TableIndexDef { int length; char **names; } TableIndexDef; #define TS_CAGG_CATALOG_IDX "continuous_agg_pkey" #define INVALID_CATALOG_TABLE _MAX_CATALOG_TABLES #define INVALID_INDEXID -1 #define CATALOG_INTERNAL_FUNC(catalog, func) (catalog->functions[func].function_id) #define CatalogInternalCall1(func, datum1) \ OidFunctionCall1(CATALOG_INTERNAL_FUNC(ts_catalog_get(), func), datum1) #define CatalogInternalCall2(func, datum1, datum2) \ OidFunctionCall2(CATALOG_INTERNAL_FUNC(ts_catalog_get(), func), datum1, datum2) #define CatalogInternalCall3(func, datum1, datum2, datum3) \ OidFunctionCall3(CATALOG_INTERNAL_FUNC(ts_catalog_get(), func), datum1, datum2, datum3) #define CatalogInternalCall4(func, datum1, datum2, datum3, datum4) \ OidFunctionCall4(CATALOG_INTERNAL_FUNC(ts_catalog_get(), func), datum1, datum2, datum3, datum4) typedef enum InternalFunction { DDL_ADD_CHUNK_CONSTRAINT, DDL_CONSTRAINT_CLONE, _MAX_INTERNAL_FUNCTIONS, } InternalFunction; /****************************** * * Hypertable table definitions * ******************************/ #define HYPERTABLE_TABLE_NAME "hypertable" /* Hypertable table attribute numbers */ enum Anum_hypertable { Anum_hypertable_id = 1, Anum_hypertable_schema_name, Anum_hypertable_table_name, Anum_hypertable_associated_schema_name, Anum_hypertable_associated_table_prefix, Anum_hypertable_num_dimensions, Anum_hypertable_chunk_sizing_func_schema, Anum_hypertable_chunk_sizing_func_name, Anum_hypertable_chunk_target_size, Anum_hypertable_compression_state, Anum_hypertable_compressed_hypertable_id, Anum_hypertable_status, _Anum_hypertable_max, }; #define Natts_hypertable (_Anum_hypertable_max - 1) typedef struct FormData_hypertable { int32 id; NameData schema_name; NameData table_name; NameData associated_schema_name; NameData associated_table_prefix; int16 num_dimensions; NameData chunk_sizing_func_schema; NameData chunk_sizing_func_name; int64 chunk_target_size; int16 compression_state; int32 compressed_hypertable_id; int32 status; } FormData_hypertable; typedef FormData_hypertable *Form_hypertable; /* Hypertable primary index attribute numbers */ enum Anum_hypertable_pkey_idx { Anum_hypertable_pkey_idx_id = 1, _Anum_hypertable_pkey_max, }; #define Natts_hypertable_pkey_idx (_Anum_hypertable_pkey_max - 1) /* Hypertable name (schema,table) index attribute numbers */ enum Anum_hypertable_name_idx { Anum_hypertable_name_idx_table = 1, Anum_hypertable_name_idx_schema, _Anum_hypertable_name_max, }; #define Natts_hypertable_name_idx (_Anum_hypertable_name_max - 1) enum { HYPERTABLE_ID_INDEX = 0, HYPERTABLE_NAME_INDEX, _MAX_HYPERTABLE_INDEX, }; /****************************** * * Dimension table definitions * ******************************/ #define DIMENSION_TABLE_NAME "dimension" enum Anum_dimension { Anum_dimension_id = 1, Anum_dimension_hypertable_id, Anum_dimension_column_name, Anum_dimension_column_type, Anum_dimension_aligned, Anum_dimension_num_slices, Anum_dimension_partitioning_func_schema, Anum_dimension_partitioning_func, Anum_dimension_interval_length, Anum_dimension_compress_interval_length, Anum_dimension_integer_now_func_schema, Anum_dimension_integer_now_func, _Anum_dimension_max, }; #define Natts_dimension (_Anum_dimension_max - 1) typedef struct FormData_dimension { int32 id; int32 hypertable_id; NameData column_name; Oid column_type; bool aligned; /* closed (space) columns */ int16 num_slices; NameData partitioning_func_schema; NameData partitioning_func; /* open (time) columns */ int64 interval_length; int64 compress_interval_length; NameData integer_now_func_schema; NameData integer_now_func; } FormData_dimension; typedef FormData_dimension *Form_dimension; enum Anum_dimension_id_idx { Anum_dimension_id_idx_id = 1, _Anum_dimension_id_idx_max, }; #define Natts_dimension_id_idx (_Anum_dimension_id_idx_max - 1) enum Anum_dimension_hypertable_id_column_name_idx { Anum_dimension_hypertable_id_column_name_idx_hypertable_id = 1, Anum_dimension_hypertable_id_column_name_idx_column_name, _Anum_dimension_hypertable_id_idx_max, }; #define Natts_dimension_hypertable_id_idx (_Anum_dimension_hypertable_id_idx_max - 1) enum { DIMENSION_ID_IDX = 0, DIMENSION_HYPERTABLE_ID_COLUMN_NAME_IDX, _MAX_DIMENSION_INDEX, }; /****************************** * * Dimension slice table definitions * ******************************/ #define DIMENSION_SLICE_TABLE_NAME "dimension_slice" enum Anum_dimension_slice { Anum_dimension_slice_id = 1, Anum_dimension_slice_dimension_id, Anum_dimension_slice_range_start, Anum_dimension_slice_range_end, _Anum_dimension_slice_max, }; #define Natts_dimension_slice (_Anum_dimension_slice_max - 1) typedef struct FormData_dimension_slice { int32 id; int32 dimension_id; int64 range_start; int64 range_end; } FormData_dimension_slice; typedef FormData_dimension_slice *Form_dimension_slice; enum Anum_dimension_slice_id_idx { Anum_dimension_slice_id_idx_id = 1, _Anum_dimension_slice_id_idx_max, }; #define Natts_dimension_slice_id_idx (_Anum_dimension_slice_id_idx_max - 1) enum Anum_dimension_slice_dimension_id_range_start_range_end_idx { Anum_dimension_slice_dimension_id_range_start_range_end_idx_dimension_id = 1, Anum_dimension_slice_dimension_id_range_start_range_end_idx_range_start, Anum_dimension_slice_dimension_id_range_start_range_end_idx_range_end, _Anum_dimension_slice_dimension_id_range_start_range_end_idx_max, }; #define Natts_dimension_slice_dimension_id_range_start_range_end_idx \ (_Anum_dimension_slice_dimension_id_range_start_range_end_idx_max - 1) enum { DIMENSION_SLICE_ID_IDX = 0, DIMENSION_SLICE_DIMENSION_ID_RANGE_START_RANGE_END_IDX, _MAX_DIMENSION_SLICE_INDEX, }; /****************************** * * Dimension range table definitions * ******************************/ #define CHUNK_COLUMN_STATS_TABLE_NAME "chunk_column_stats" enum Anum_chunk_column_stats { Anum_chunk_column_stats_id = 1, Anum_chunk_column_stats_hypertable_id, Anum_chunk_column_stats_chunk_id, Anum_chunk_column_stats_column_name, Anum_chunk_column_stats_range_start, Anum_chunk_column_stats_range_end, Anum_chunk_column_stats_valid, _Anum_chunk_column_stats_max, }; #define Natts_chunk_column_stats (_Anum_chunk_column_stats_max - 1) typedef struct FormData_chunk_column_stats { int32 id; int32 hypertable_id; int32 chunk_id; NameData column_name; int64 range_start; int64 range_end; bool valid; } FormData_chunk_column_stats; typedef FormData_chunk_column_stats *Form_chunk_column_stats; enum Anum_chunk_column_stats_id_idx { Anum_chunk_column_stats_id_idx_id = 1, _Anum_chunk_column_stats_id_idx_max, }; #define Natts_chunk_column_stats_id_idx (_Anum_chunk_column_stats_id_idx_max - 1) enum Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx { Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_hypertable_id = 1, Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_chunk_id, Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_column_name, Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_range_start, Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_range_end, _Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_max, }; #define Natts_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx \ (_Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_max - 1) enum { CHUNK_COLUMN_STATS_ID_IDX = 0, CHUNK_COLUMN_STATS_HT_ID_CHUNK_ID_COLUMN_NAME_IDX, _MAX_CHUNK_COLUMN_STATS_INDEX, }; /************************* * * Chunk table definitions * *************************/ #define CHUNK_TABLE_NAME "chunk" enum Anum_chunk { Anum_chunk_id = 1, Anum_chunk_hypertable_id, Anum_chunk_schema_name, Anum_chunk_table_name, Anum_chunk_compressed_chunk_id, Anum_chunk_status, Anum_chunk_osm_chunk, Anum_chunk_creation_time, _Anum_chunk_max, }; #define Natts_chunk (_Anum_chunk_max - 1) typedef struct FormData_chunk { int32 id; int32 hypertable_id; NameData schema_name; NameData table_name; int32 compressed_chunk_id; int32 status; bool osm_chunk; TimestampTz creation_time; } FormData_chunk; typedef FormData_chunk *Form_chunk; enum { CHUNK_ID_INDEX = 0, CHUNK_HYPERTABLE_ID_INDEX, CHUNK_SCHEMA_NAME_INDEX, CHUNK_COMPRESSED_CHUNK_ID_INDEX, CHUNK_OSM_CHUNK_INDEX, CHUNK_HYPERTABLE_ID_CREATION_TIME_INDEX, _MAX_CHUNK_INDEX, }; enum Anum_chunk_idx { Anum_chunk_idx_id = 1, }; enum Anum_chunk_hypertable_id_idx { Anum_chunk_hypertable_id_idx_hypertable_id = 1, }; enum Anum_chunk_compressed_chunk_id_idx { Anum_chunk_compressed_chunk_id_idx_compressed_chunk_id = 1, }; enum Anum_chunk_schema_name_idx { Anum_chunk_schema_name_idx_schema_name = 1, Anum_chunk_schema_name_idx_table_name, }; enum Anum_chunk_osm_chunk_idx { Anum_chunk_osm_chunk_idx_osm_chunk = 1, Anum_chunk_osm_chunk_idx_hypertable_id, }; enum Anum_chunk_hypertable_id_creation_time_idx { Anum_chunk_hypertable_id_creation_time_idx_hypertable_id = 1, Anum_chunk_hypertable_id_creation_time_idx_creation_time, }; /************************************ * * Chunk constraint table definitions * ************************************/ #define CHUNK_CONSTRAINT_TABLE_NAME "chunk_constraint" enum Anum_chunk_constraint { Anum_chunk_constraint_chunk_id = 1, Anum_chunk_constraint_dimension_slice_id, Anum_chunk_constraint_constraint_name, Anum_chunk_constraint_hypertable_constraint_name, _Anum_chunk_constraint_max, }; #define Natts_chunk_constraint (_Anum_chunk_constraint_max - 1) /* Do Not use GET_STRUCT with FormData_chunk_constraint. It contains NULLS */ typedef struct FormData_chunk_constraint { int32 chunk_id; int32 dimension_slice_id; NameData constraint_name; NameData hypertable_constraint_name; } FormData_chunk_constraint; typedef FormData_chunk_constraint *Form_chunk_constraint; enum { CHUNK_CONSTRAINT_CHUNK_ID_CONSTRAINT_NAME_IDX = 0, CHUNK_CONSTRAINT_DIMENSION_SLICE_ID_IDX, _MAX_CHUNK_CONSTRAINT_INDEX, }; enum Anum_chunk_constraint_dimension_slice_id_idx { Anum_chunk_constraint_dimension_slice_id_idx_dimension_slice_id = 1, _Anum_chunk_constraint_dimension_slice_id_idx_max, }; enum Anum_chunk_constraint_chunk_id_constraint_name_idx { Anum_chunk_constraint_chunk_id_constraint_name_idx_chunk_id = 1, Anum_chunk_constraint_chunk_id_constraint_name_idx_constraint_name, _Anum_chunk_constraint_chunk_id_constraint_name_idx_max, }; /************************************ * * Tablespace table definitions * ************************************/ #define TABLESPACE_TABLE_NAME "tablespace" enum Anum_tablespace { Anum_tablespace_id = 1, Anum_tablespace_hypertable_id, Anum_tablespace_tablespace_name, _Anum_tablespace_max, }; #define Natts_tablespace (_Anum_tablespace_max - 1) typedef struct FormData_tablespace { int32 id; int32 hypertable_id; NameData tablespace_name; } FormData_tablespace; typedef FormData_tablespace *Form_tablespace; enum { TABLESPACE_PKEY_IDX = 0, TABLESPACE_HYPERTABLE_ID_TABLESPACE_NAME_IDX, _MAX_TABLESPACE_INDEX, }; enum Anum_tablespace_pkey_idx { Anum_tablespace_pkey_idx_tablespace_id = 1, _Anum_tablespace_pkey_idx_max, }; typedef struct FormData_tablespace_pkey_idx { int32 tablespace_id; } FormData_tablespace_pkey_idx; enum Anum_tablespace_hypertable_id_tablespace_name_idx { Anum_tablespace_hypertable_id_tablespace_name_idx_hypertable_id = 1, Anum_tablespace_hypertable_id_tablespace_name_idx_tablespace_name, _Anum_tablespace_hypertable_id_tablespace_name_idx_max, }; typedef struct FormData_tablespace_hypertable_id_tablespace_name_idx { int32 hypertable_id; NameData tablespace_name; } FormData_tablespace_hypertable_id_tablespace_name_idx; /************************************ * * bgw_job table definitions * ************************************/ #define BGW_JOB_TABLE_NAME "bgw_job" enum Anum_bgw_job { Anum_bgw_job_id = 1, Anum_bgw_job_application_name, Anum_bgw_job_schedule_interval, Anum_bgw_job_max_runtime, Anum_bgw_job_max_retries, Anum_bgw_job_retry_period, Anum_bgw_job_proc_schema, Anum_bgw_job_proc_name, Anum_bgw_job_owner, Anum_bgw_job_scheduled, Anum_bgw_job_fixed_schedule, Anum_bgw_job_initial_start, Anum_bgw_job_hypertable_id, Anum_bgw_job_config, Anum_bgw_job_check_schema, Anum_bgw_job_check_name, Anum_bgw_job_timezone, _Anum_bgw_job_max, }; #define Natts_bgw_job (_Anum_bgw_job_max - 1) /* fixed_schedule needs to come before the varlen fields for GETSTRUCT to work */ typedef struct FormData_bgw_job { int32 id; NameData application_name; Interval schedule_interval; Interval max_runtime; int32 max_retries; Interval retry_period; NameData proc_schema; NameData proc_name; Oid owner; bool scheduled; bool fixed_schedule; TimestampTz initial_start; int32 hypertable_id; Jsonb *config; NameData check_schema; NameData check_name; text *timezone; } FormData_bgw_job; typedef FormData_bgw_job *Form_bgw_job; enum { BGW_JOB_PKEY_IDX = 0, BGW_JOB_PROC_HYPERTABLE_ID_IDX, _MAX_BGW_JOB_INDEX, }; enum Anum_bgw_job_pkey_idx { Anum_bgw_job_pkey_idx_id = 1, _Anum_bgw_job_pkey_idx_max, }; #define Natts_bjw_job_pkey_idx (_Anum_bgw_job_pkey_idx_max - 1) enum Anum_bgw_job_proc_hypertable_id_idx { Anum_bgw_job_proc_hypertable_id_idx_proc_schema = 1, Anum_bgw_job_proc_hypertable_id_idx_proc_name, Anum_bgw_job_proc_hypertable_id_idx_hypertable_id, _Anum_bgw_job_proc_hypertable_id_idx_max, }; #define Natts_bgw_job_proc_hypertable_id_idx (_Anum_bgw_job_proc_hypertable_id_idx_max - 1) /************************************ * * bgw_job_stat table definitions * ************************************/ #define BGW_JOB_STAT_TABLE_NAME "bgw_job_stat" enum Anum_bgw_job_stat { Anum_bgw_job_stat_job_id = 1, Anum_bgw_job_stat_last_start, Anum_bgw_job_stat_last_finish, Anum_bgw_job_stat_next_start, Anum_bgw_job_stat_last_successful_finish, Anum_bgw_job_stat_last_run_success, Anum_bgw_job_stat_total_runs, Anum_bgw_job_stat_total_duration, Anum_bgw_job_stat_total_duration_failures, Anum_bgw_job_stat_total_success, Anum_bgw_job_stat_total_failures, Anum_bgw_job_stat_total_crashes, Anum_bgw_job_stat_consecutive_failures, Anum_bgw_job_stat_consecutive_crashes, Anum_bgw_job_stat_flags, _Anum_bgw_job_stat_max, }; #define Natts_bgw_job_stat (_Anum_bgw_job_stat_max - 1) typedef struct FormData_bgw_job_stat { int32 id; TimestampTz last_start; TimestampTz last_finish; TimestampTz next_start; TimestampTz last_successful_finish; bool last_run_success; int64 total_runs; Interval total_duration; Interval total_duration_failures; int64 total_success; int64 total_failures; int64 total_crashes; int32 consecutive_failures; int32 consecutive_crashes; int32 flags; } FormData_bgw_job_stat; typedef FormData_bgw_job_stat *Form_bgw_job_stat; enum { BGW_JOB_STAT_PKEY_IDX = 0, _MAX_BGW_JOB_STAT_INDEX, }; enum Anum_bgw_job_stat_pkey_idx { Anum_bgw_job_stat_pkey_idx_job_id = 1, _Anum_bgw_job_stat_pkey_idx_max, }; #define Natts_bjw_job_stat_pkey_idx (_Anum_bgw_job_stat_pkey_idx_max - 1) #define BGW_JOB_STAT_HISTORY_TABLE_NAME "bgw_job_stat_history" enum Anum_bgw_job_stat_history { Anum_bgw_job_stat_history_id = 1, Anum_bgw_job_stat_history_job_id, Anum_bgw_job_stat_history_pid, Anum_bgw_job_stat_history_execution_start, Anum_bgw_job_stat_history_execution_finish, Anum_bgw_job_stat_history_succeeded, Anum_bgw_job_stat_history_data, _Anum_bgw_job_stat_history_max, }; #define Natts_bgw_job_stat_history (_Anum_bgw_job_stat_history_max - 1) typedef struct FormData_bgw_job_stat_history { int64 id; int32 job_id; int32 pid; TimestampTz execution_start; TimestampTz execution_finish; bool succeeded; Jsonb *data; } FormData_bgw_job_stat_history; typedef FormData_bgw_job_stat_history *Form_bgw_job_stat_history; enum { BGW_JOB_STAT_HISTORY_PKEY_IDX = 0, _MAX_BGW_JOB_STAT_HISTORY_INDEX, }; enum Anum_bgw_job_stat_history_pkey_idx { Anum_bgw_job_stat_history_pkey_idx_id = 1, _Anum_bgw_job_stat_history_pkey_idx_max, }; #define Natts_bjw_job_stat_history_pkey_idx (_Anum_bgw_job_stat_history_pkey_idx_max - 1) /****************************** * * metadata table definitions * ******************************/ #define METADATA_TABLE_NAME "metadata" enum Anum_metadata { Anum_metadata_key = 1, Anum_metadata_value, Anum_metadata_include_in_telemetry, _Anum_metadata_max, }; #define Natts_metadata (_Anum_metadata_max - 1) typedef struct FormData_metadata { NameData key; text *value; } FormData_metadata; typedef FormData_metadata *Form_metadata; /* metadata primary index attribute numbers */ enum Anum_metadata_pkey_idx { Anum_metadata_pkey_idx_id = 1, _Anum_metadata_pkey_max, }; #define Natts_metadata_pkey_idx (_Anum_metadata_pkey_max - 1) enum { METADATA_PKEY_IDX = 0, _MAX_METADATA_INDEX, }; /* * telemetry_event table definition */ #define TELEMETRY_EVENT_TABLE_NAME "telemetry_event" enum Anum_telemetry_event { Anum_telemetry_event_created = 1, Anum_telemetry_event_tag, Anum_telemetry_event_body, _Anum_telemetry_event_max, }; #define Natts_telemetry_event_max (_Anum_telemetry_event_max - 1) /****** BGW_POLICY_CHUNK_STATS TABLE definitions */ #define BGW_POLICY_CHUNK_STATS_TABLE_NAME "bgw_policy_chunk_stats" enum Anum_bgw_policy_chunk_stats { Anum_bgw_policy_chunk_stats_job_id = 1, Anum_bgw_policy_chunk_stats_chunk_id, Anum_bgw_policy_chunk_stats_num_times_job_run, Anum_bgw_policy_chunk_stats_last_time_job_run, _Anum_bgw_policy_chunk_stats_max, }; #define Natts_bgw_policy_chunk_stats (_Anum_bgw_policy_chunk_stats_max - 1) typedef struct FormData_bgw_policy_chunk_stats { int32 job_id; int32 chunk_id; int32 num_times_job_run; TimestampTz last_time_job_run; } FormData_bgw_policy_chunk_stats; typedef FormData_bgw_policy_chunk_stats *Form_bgw_job_chunk_stats; enum { BGW_POLICY_CHUNK_STATS_JOB_ID_CHUNK_ID_IDX = 0, _MAX_BGW_POLICY_CHUNK_STATS_INDEX, }; enum Anum_bgw_policy_chunk_stats_job_id_chunk_id_idx { Anum_bgw_policy_chunk_stats_job_id_chunk_id_idx_job_id = 1, Anum_bgw_policy_chunk_stats_job_id_chunk_id_idx_chunk_id, _Anum_bgw_policy_chunk_stats_job_id_chunk_id_idx_max, }; typedef struct FormData_bgw_policy_chunk_stats_job_id_chunk_id_idx { int32 job_id; int32 chunk_id; } FormData_bgw_policy_chunk_stats_job_id_chunk_id_idx; /****************************************** * * continuous_agg table definitions * ******************************************/ #define CONTINUOUS_AGG_TABLE_NAME "continuous_agg" typedef enum Anum_continuous_agg { Anum_continuous_agg_mat_hypertable_id = 1, Anum_continuous_agg_raw_hypertable_id, Anum_continuous_agg_parent_mat_hypertable_id, Anum_continuous_agg_user_view_schema, Anum_continuous_agg_user_view_name, Anum_continuous_agg_partial_view_schema, Anum_continuous_agg_partial_view_name, Anum_continuous_agg_direct_view_schema, Anum_continuous_agg_direct_view_name, Anum_continuous_agg_materialize_only, _Anum_continuous_agg_max, } Anum_continuous_agg; #define Natts_continuous_agg (_Anum_continuous_agg_max - 1) typedef struct FormData_continuous_agg { int32 mat_hypertable_id; int32 raw_hypertable_id; int32 parent_mat_hypertable_id; /* Nested Continuous Aggregate */ NameData user_view_schema; NameData user_view_name; NameData partial_view_schema; NameData partial_view_name; NameData direct_view_schema; NameData direct_view_name; bool materialized_only; } FormData_continuous_agg; typedef FormData_continuous_agg *Form_continuous_agg; enum { CONTINUOUS_AGG_PARTIAL_VIEW_SCHEMA_PARTIAL_VIEW_NAME_KEY = 0, CONTINUOUS_AGG_PKEY, CONTINUOUS_AGG_USER_VIEW_SCHEMA_USER_VIEW_NAME_KEY, CONTINUOUS_AGG_RAW_HYPERTABLE_ID_IDX, _MAX_CONTINUOUS_AGG_INDEX, }; typedef enum Anum_continuous_agg_partial_view_schema_partial_view_name_key { Anum_continuous_agg_partial_view_schema_partial_view_name_key_partial_view_schema = 1, Anum_continuous_agg_partial_view_schema_partial_view_name_key_partial_view_name, _Anum_continuous_agg_partial_view_schema_partial_view_name_key_max, } Anum_continuous_agg_partial_view_schema_partial_view_name_key; #define Natts_continuous_agg_partial_view_schema_partial_view_name_key \ (_Anum_continuous_agg_partial_view_schema_partial_view_name_key_max - 1) typedef enum Anum_continuous_agg_pkey { Anum_continuous_agg_pkey_mat_hypertable_id = 1, _Anum_continuous_agg_pkey_max, } Anum_continuous_agg_pkey; #define Natts_continuous_agg_pkey (_Anum_continuous_agg_pkey_max - 1) typedef enum Anum_continuous_agg_user_view_schema_user_view_name_key { Anum_continuous_agg_user_view_schema_user_view_name_key_user_view_schema = 1, Anum_continuous_agg_user_view_schema_user_view_name_key_user_view_name, _Anum_continuous_agg_user_view_schema_user_view_name_key_max, } Anum_continuous_agg_user_view_schema_user_view_name_key; #define Natts_continuous_agg_user_view_schema_user_view_name_key \ (_Anum_continuous_agg_user_view_schema_user_view_name_key_max - 1) typedef enum Anum_continuous_agg_raw_hypertable_id_idx { Anum_continuous_agg_raw_hypertable_id_idx_raw_hypertable_id = 1, _Anum_continuous_agg_raw_hypertable_id_idx_max, } Anum_continuous_agg_raw_hypertable_id_idx; #define Natts_continuous_agg_raw_hypertable_id_idx \ (_Anum_continuous_agg_raw_hypertable_id_idx_max - 1) /*** continuous_aggs_bucket_function table definitions ***/ #define CONTINUOUS_AGGS_BUCKET_FUNCTION_TABLE_NAME "continuous_aggs_bucket_function" typedef enum Anum_continuous_aggs_bucket_function { Anum_continuous_aggs_bucket_function_mat_hypertable_id = 1, Anum_continuous_aggs_bucket_function_function, Anum_continuous_aggs_bucket_function_bucket_width, Anum_continuous_aggs_bucket_function_bucket_origin, Anum_continuous_aggs_bucket_function_bucket_offset, Anum_continuous_aggs_bucket_function_bucket_timezone, Anum_continuous_aggs_bucket_function_bucket_fixed_width, _Anum_continuous_aggs_bucket_function_max, } Anum_continuous_aggs_bucket_function; #define Natts_continuous_aggs_bucket_function (_Anum_continuous_aggs_bucket_function_max - 1) enum { CONTINUOUS_AGGS_BUCKET_FUNCTION_PKEY_IDX = 0, _MAX_CONTINUOUS_AGGS_BUCKET_FUNCTION_INDEX, }; typedef enum Anum_continuous_aggs_bucket_function_pkey { Anum_continuous_aggs_bucket_function_pkey_mat_hypertable_id = 1, _Anum_continuous_aggs_bucket_function_pkey_max, } Anum_continuous_aggs_bucket_function_pkey; #define Natts_continuous_aggs_bucket_function_pkey \ (_Anum_continuous_aggs_bucket_function_pkey_max - 1) /* * CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG_TABLE definitions * * The definition of CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_PLUGIN_NAME is * generated from config.h.in and can be found in the generated file. */ #define CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG_TABLE_NAME \ "continuous_aggs_hypertable_invalidation_log" typedef enum Anum_continuous_aggs_hypertable_invalidation_log { Anum_continuous_aggs_hypertable_invalidation_log_hypertable_id = 1, Anum_continuous_aggs_hypertable_invalidation_log_lowest_modified_value, Anum_continuous_aggs_hypertable_invalidation_log_greatest_modified_value, _Anum_continuous_aggs_hypertable_invalidation_log_max, } Anum_continuous_aggs_hypertable_invalidation_log; #define Natts_continuous_aggs_hypertable_invalidation_log \ (_Anum_continuous_aggs_hypertable_invalidation_log_max - 1) typedef struct FormData_continuous_aggs_hypertable_invalidation_log { int32 hypertable_id; int64 lowest_modified_value; int64 greatest_modified_value; } FormData_continuous_aggs_hypertable_invalidation_log; typedef FormData_continuous_aggs_hypertable_invalidation_log *Form_continuous_aggs_hypertable_invalidation_log; enum { CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG_IDX = 0, _MAX_CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG_INDEX, }; typedef enum Anum_continuous_aggs_hypertable_invalidation_log_idx { Anum_continuous_aggs_hypertable_invalidation_log_idx_hypertable_id = 1, Anum_continuous_aggs_hypertable_invalidation_log_idx_lowest_modified_value, _Anum_continuous_aggs_hypertable_invalidation_log_idx_max, } Anum_continuous_aggs_hypertable_invalidation_log_idx; #define Natts_continuous_aggs_hypertable_invalidation_log_idx \ (_Anum_continuous_aggs_hypertable_invalidation_log_idx_max - 1) /****** CONTINUOUS_AGGS_INVALIDATION_THRESHOLD_TABLE definitions*/ #define CONTINUOUS_AGGS_INVALIDATION_THRESHOLD_TABLE_NAME "continuous_aggs_invalidation_threshold" typedef enum Anum_continuous_aggs_invalidation_threshold { Anum_continuous_aggs_invalidation_threshold_hypertable_id = 1, Anum_continuous_aggs_invalidation_threshold_watermark, _Anum_continuous_aggs_invalidation_threshold_max, } Anum_continuous_aggs_invalidation_threshold; #define Natts_continuous_aggs_invalidation_threshold \ (_Anum_continuous_aggs_invalidation_threshold_max - 1) typedef struct FormData_continuous_aggs_invalidation_threshold { int32 hypertable_id; int64 watermark; } FormData_continuous_aggs_invalidation_threshold; typedef FormData_continuous_aggs_invalidation_threshold *Form_continuous_aggs_invalidation_threshold; enum { CONTINUOUS_AGGS_INVALIDATION_THRESHOLD_PKEY = 0, _MAX_CONTINUOUS_AGGS_INVALIDATION_THRESHOLD_INDEX, }; typedef enum Anum_continuous_aggs_invalidation_threshold_pkey { Anum_continuous_aggs_invalidation_threshold_pkey_hypertable_id = 1, _Anum_continuous_aggs_invalidation_threshold_pkey_max, } Anum_continuous_aggs_invalidation_threshold_pkey; #define Natts_continuous_aggs_invalidation_threshold_pkey \ (_Anum_continuous_aggs_invalidation_threshold_pkey_max - 1) /****** CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG_TABLE definitions*/ #define CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG_TABLE_NAME \ "continuous_aggs_materialization_invalidation_log" typedef enum Anum_continuous_aggs_materialization_invalidation_log { Anum_continuous_aggs_materialization_invalidation_log_materialization_id = 1, Anum_continuous_aggs_materialization_invalidation_log_lowest_modified_value, Anum_continuous_aggs_materialization_invalidation_log_greatest_modified_value, _Anum_continuous_aggs_materialization_invalidation_log_max, } Anum_continuous_aggs_materialization_invalidation_log; #define Natts_continuous_aggs_materialization_invalidation_log \ (_Anum_continuous_aggs_materialization_invalidation_log_max - 1) typedef struct FormData_continuous_aggs_materialization_invalidation_log { int32 materialization_id; int64 lowest_modified_value; int64 greatest_modified_value; } FormData_continuous_aggs_materialization_invalidation_log; typedef FormData_continuous_aggs_materialization_invalidation_log *Form_continuous_aggs_materialization_invalidation_log; enum { CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG_IDX = 0, _MAX_CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG_INDEX, }; typedef enum Anum_continuous_aggs_materialization_invalidation_log_idx { Anum_continuous_aggs_materialization_invalidation_log_idx_materialization_id = 1, Anum_continuous_aggs_materialization_invalidation_log_idx_lowest_modified_value, _Anum_continuous_aggs_materialization_invalidation_log_idx_max, } Anum_continuous_aggs_materialization_invalidation_log_idx; #define Natts_continuous_aggs_materialization_invalidation_log_idx \ (_Anum_continuous_aggs_materialization_invalidation_log_idx_max - 1) /****** CONTINUOUS_AGGS_MATERIALIZATION_RANGES_TABLE definitions*/ #define CONTINUOUS_AGGS_MATERIALIZATION_RANGES_TABLE_NAME "continuous_aggs_materialization_ranges" typedef enum Anum_continuous_aggs_materialization_ranges { Anum_continuous_aggs_materialization_ranges_materialization_id = 1, Anum_continuous_aggs_materialization_ranges_lowest_modified_value, Anum_continuous_aggs_materialization_ranges_greatest_modified_value, _Anum_continuous_aggs_materialization_ranges_max, } Anum_continuous_aggs_materialization_ranges; #define Natts_continuous_aggs_materialization_ranges \ (_Anum_continuous_aggs_materialization_ranges_max - 1) typedef struct FormData_continuous_aggs_materialization_ranges { int32 materialization_id; int64 lowest_modified_value; int64 greatest_modified_value; } FormData_continuous_aggs_materialization_ranges; typedef FormData_continuous_aggs_materialization_ranges *Form_continuous_aggs_materialization_ranges; enum { CONTINUOUS_AGGS_MATERIALIZATION_RANGES_IDX = 0, _MAX_CONTINUOUS_AGGS_MATERIALIZATION_RANGES_INDEX, }; typedef enum Anum_continuous_aggs_materialization_ranges_idx { Anum_continuous_aggs_materialization_ranges_idx_materialization_id = 1, Anum_continuous_aggs_materialization_ranges_idx_lowest_modified_value, _Anum_continuous_aggs_materialization_ranges_idx_max, } Anum_continuous_aggs_materialization_ranges_idx; #define Natts_continuous_aggs_materialization_ranges_idx \ (_Anum_continuous_aggs_materialization_ranges_idx_max - 1) /****** CONTINUOUS_AGGS_WATERMARK_TABLE definitions*/ #define CONTINUOUS_AGGS_WATERMARK_TABLE_NAME "continuous_aggs_watermark" typedef enum Anum_continuous_aggs_watermark { Anum_continuous_aggs_watermark_mat_hypertable_id = 1, Anum_continuous_aggs_watermark_watermark, _Anum_continuous_aggs_watermark_max, } Anum_continuous_aggs_watermark; #define Natts_continuous_aggs_watermark (_Anum_continuous_aggs_watermark_max - 1) typedef struct FormData_continuous_aggs_watermark { int32 mat_hypertable_id; int64 watermark; } FormData_continuous_aggs_watermark; typedef FormData_continuous_aggs_watermark *Form_continuous_aggs_watermark; enum { CONTINUOUS_AGGS_WATERMARK_PKEY = 0, _MAX_CONTINUOUS_AGGS_WATERMARK_INDEX, }; typedef enum Anum_continuous_aggs_watermark_pkey { Anum_continuous_aggs_watermark_pkey_mat_hypertable_id = 1, _Anum_continuous_aggs_watermark_pkey_max, } Anum_continuous_aggs_watermark_pkey; #define Natts_continuous_aggs_watermark_pkey (_Anum_continuous_aggs_watermark_pkey_max - 1) #define COMPRESSION_SETTINGS_TABLE_NAME "compression_settings" typedef enum Anum_compression_settings { Anum_compression_settings_relid = 1, Anum_compression_settings_compress_relid, Anum_compression_settings_segmentby, Anum_compression_settings_orderby, Anum_compression_settings_orderby_desc, Anum_compression_settings_orderby_nullsfirst, Anum_compression_settings_index, _Anum_compression_settings_max, } Anum_compression_settings; #define Natts_compression_settings (_Anum_compression_settings_max - 1) typedef struct FormData_compression_settings { Oid relid; Oid compress_relid; ArrayType *segmentby; ArrayType *orderby; ArrayType *orderby_desc; ArrayType *orderby_nullsfirst; Jsonb *index; } FormData_compression_settings; typedef FormData_compression_settings *Form_compression_settings; enum { COMPRESSION_SETTINGS_PKEY = 0, COMPRESSION_SETTINGS_COMPRESS_RELID_IDX, _MAX_COMPRESSION_SETTINGS_INDEX, }; typedef enum Anum_compression_settings_pkey { Anum_compression_settings_pkey_relid = 1, _Anum_compression_settings_pkey_max, } Anum_compression_settings_pkey; #define Natts_compression_chunk_size_pkey (_Anum_compression_chunk_size_pkey_max - 1) typedef enum Anum_compression_settings_compress_relid_idx { Anum_compression_settings_compress_relid_idx_relid = 1, _Anum_compression_settings_compress_relid_idx_max, } Anum_compression_settings_compress_relid_idx; #define Natts_compression_settings_compress_relid_idx \ (_Anum_compression_settings_compress_relid_idx_max - 1) #define COMPRESSION_CHUNK_SIZE_TABLE_NAME "compression_chunk_size" typedef enum Anum_compression_chunk_size { Anum_compression_chunk_size_chunk_id = 1, Anum_compression_chunk_size_compressed_chunk_id, Anum_compression_chunk_size_uncompressed_heap_size, Anum_compression_chunk_size_uncompressed_toast_size, Anum_compression_chunk_size_uncompressed_index_size, Anum_compression_chunk_size_compressed_heap_size, Anum_compression_chunk_size_compressed_toast_size, Anum_compression_chunk_size_compressed_index_size, Anum_compression_chunk_size_numrows_pre_compression, Anum_compression_chunk_size_numrows_post_compression, Anum_compression_chunk_size_numrows_frozen_immediately, _Anum_compression_chunk_size_max, } Anum_compression_chunk_size; #define Natts_compression_chunk_size (_Anum_compression_chunk_size_max - 1) typedef struct FormData_compression_chunk_size { int32 chunk_id; int32 compressed_chunk_id; int64 uncompressed_heap_size; int64 uncompressed_toast_size; int64 uncompressed_index_size; int64 compressed_heap_size; int64 compressed_toast_size; int64 compressed_index_size; int64 numrows_pre_compression; int64 numrows_post_compression; int64 numrows_frozen_immediately; } FormData_compression_chunk_size; typedef FormData_compression_chunk_size *Form_compression_chunk_size; enum { COMPRESSION_CHUNK_SIZE_PKEY = 0, _MAX_COMPRESSION_CHUNK_SIZE_INDEX, }; typedef enum Anum_compression_chunk_size_pkey { Anum_compression_chunk_size_pkey_chunk_id = 1, _Anum_compression_chunk_size_pkey_max, } Anum_compression_chunk_size_pkey; #define Natts_compression_chunk_size_pkey (_Anum_compression_chunk_size_pkey_max - 1) #define CHUNK_REWRITE_TABLE_NAME "chunk_rewrite" typedef enum Anum_chunk_rewrite { Anum_chunk_rewrite_chunk_relid = 1, Anum_chunk_rewrite_new_relid, _Anum_chunk_rewrite_max, } Anum_chunk_rewrite; #define Natts_chunk_rewrite (_Anum_chunk_rewrite_max - 1) typedef struct FormData_chunk_rewrite { Oid chunk_relid; Oid new_relid; } FormData_chunk_rewrite; typedef FormData_chunk_rewrite *Form_chunk_rewrite; enum { CHUNK_REWRITE_IDX = 0, _MAX_CHUNK_REWRITE_INDEX, }; typedef enum Anum_chunk_rewrite_pkey { Anum_chunk_rewrite_key_chunk_relid = 1, _Anum_chunk_rewrite_key_max, } Anum_chunk_rewrite_pkey; #define Natts_chunk_rewrite_key (_Anum_chunk_rewrite_key_max - 1) /* * The maximum number of indexes a catalog table can have. * This needs to be bumped in case of new catalog tables that have more indexes. */ #define _MAX_TABLE_INDEXES 6 typedef enum CacheType { CACHE_TYPE_HYPERTABLE, CACHE_TYPE_BGW_JOB, CACHE_TYPE_EXTENSION, _MAX_CACHE_TYPES } CacheType; typedef struct CatalogTableInfo { const char *schema_name; const char *name; Oid id; Oid serial_relid; Oid index_ids[_MAX_TABLE_INDEXES]; } CatalogTableInfo; typedef struct CatalogDatabaseInfo { NameData database_name; Oid database_id; Oid schema_id; Oid owner_uid; } CatalogDatabaseInfo; typedef struct Catalog { CatalogTableInfo tables[_MAX_CATALOG_TABLES]; Oid extension_schema_id[_TS_MAX_SCHEMA]; struct { Oid inval_proxy_id; } caches[_MAX_CACHE_TYPES]; struct { Oid function_id; } functions[_MAX_INTERNAL_FUNCTIONS]; bool initialized; } Catalog; typedef struct CatalogSecurityContext { Oid saved_uid; int saved_security_context; } CatalogSecurityContext; #define HYPERTABLE_STATUS_DEFAULT 0 /* flag set when hypertable has an attached OSM chunk */ #define HYPERTABLE_STATUS_OSM 1 /* * Currently, the time slice range metadata is updated in * the timescaledb catalog with the min and max of the range managed by OSM. * However, this range has to be contiguous in order to * update our catalog with its min and max value. If it is not contiguous, * then we cannot store the min and max in our catalog because tuple routing * will not work properly with gaps in the range. * When attempting to insert into one of the gaps, which do not in fact contain * tiered data, we error out because this is perceived as an attempt to insert * into tiered chunks, which are immutable. * When the range is noncontiguous, we store [INT64_MAX - 1, INT64_MAX) and set * this flag. * This flag also serves to allow or block the ordered append optimization. When * the range covered by OSM is contiguous, then it is possible to do ordered * append. */ #define HYPERTABLE_STATUS_OSM_CHUNK_NONCONTIGUOUS 2 extern void ts_catalog_table_info_init(CatalogTableInfo *tables, int max_table, const TableInfoDef *table_ary, const TableIndexDef *index_ary, const char **serial_id_ary); extern TSDLLEXPORT CatalogDatabaseInfo *ts_catalog_database_info_get(void); extern TSDLLEXPORT Catalog *ts_catalog_get(void); extern void ts_catalog_reset(void); extern bool ts_is_catalog_table(Oid relid); /* Functions should operate on a passed-in Catalog struct */ static inline Oid catalog_get_table_id(Catalog *catalog, CatalogTable tableid) { return catalog->tables[tableid].id; } static inline Oid catalog_get_index(Catalog *catalog, CatalogTable tableid, int indexid) { return (indexid == INVALID_INDEXID) ? InvalidOid : catalog->tables[tableid].index_ids[indexid]; } extern TSDLLEXPORT int64 ts_catalog_table_next_seq_id(const Catalog *catalog, CatalogTable table); extern Oid ts_catalog_get_cache_proxy_id(Catalog *catalog, CacheType type); /* Functions that modify the actual catalog table on disk */ extern TSDLLEXPORT bool ts_catalog_database_info_become_owner(CatalogDatabaseInfo *database_info, CatalogSecurityContext *sec_ctx); extern TSDLLEXPORT void ts_catalog_restore_user(CatalogSecurityContext *sec_ctx); extern TSDLLEXPORT void ts_catalog_insert_only(Relation rel, HeapTuple tuple); extern TSDLLEXPORT void ts_catalog_insert(Relation rel, HeapTuple tuple); extern TSDLLEXPORT void ts_catalog_insert_values(Relation rel, TupleDesc tupdesc, Datum *values, bool *nulls); extern TSDLLEXPORT void ts_catalog_insert_datums(Relation rel, TupleDesc tupdesc, NullableDatum *datums); extern TSDLLEXPORT void ts_catalog_update_tid_only(Relation rel, ItemPointer tid, HeapTuple tuple); extern TSDLLEXPORT void ts_catalog_update_tid(Relation rel, ItemPointer tid, HeapTuple tuple); extern TSDLLEXPORT void ts_catalog_update(Relation rel, HeapTuple tuple); extern TSDLLEXPORT void ts_catalog_delete_tid_only(Relation rel, ItemPointer tid); extern TSDLLEXPORT void ts_catalog_delete_tid(Relation rel, ItemPointer tid); extern TSDLLEXPORT void ts_catalog_invalidate_cache(Oid catalog_relid, CmdType operation); extern TSDLLEXPORT void ts_catalog_index_insert(ResultRelInfo *indstate, HeapTuple heapTuple); bool TSDLLEXPORT ts_catalog_scan_one(CatalogTable table, int indexid, ScanKeyData *scankey, int num_keys, tuple_found_func tuple_found, LOCKMODE lockmode, char *policy_type, void *data); void TSDLLEXPORT ts_catalog_scan_all(CatalogTable table, int indexid, ScanKeyData *scankey, int num_keys, tuple_found_func tuple_found, LOCKMODE lockmode, void *data); ================================================ FILE: src/ts_catalog/chunk_column_stats.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/attnum.h> #include <access/htup.h> #include <access/htup_details.h> #include <access/skey.h> #include <access/stratnum.h> #include <access/tupdesc.h> #include <catalog/pg_collation.h> #include <executor/spi.h> #include <executor/tuptable.h> #include <funcapi.h> #include <nodes/makefuncs.h> #include <optimizer/optimizer.h> #include <parser/parse_coerce.h> #include <parser/parse_collate.h> #include <parser/parse_expr.h> #include <parser/parse_relation.h> #include <rewrite/rewriteManip.h> #include <storage/lmgr.h> #include <storage/lockdefs.h> #include <utils/datum.h> #include <utils/syscache.h> #include "chunk.h" #include "chunk_column_stats.h" #include "dimension_slice.h" #include "guc.h" #include "ts_catalog/catalog.h" /* * Enable chunk column stats attributes */ enum Anum_enable_chunk_column_stats { Anum_enable_chunk_column_stats_id = 1, Anum_enable_chunk_column_stats_enabled, _Anum_enable_chunk_column_stats_max, }; #define Natts_enable_chunk_column_stats (_Anum_enable_chunk_column_stats_max - 1) TS_FUNCTION_INFO_V1(ts_chunk_column_stats_enable); /* * Disable chunk column stats attributes */ enum Anum_disable_chunk_column_stats { Anum_disable_chunk_column_stats_hypertable_id = 1, Anum_disable_chunk_column_stats_column_name, Anum_disable_chunk_column_stats_disabled, _Anum_disable_chunk_column_stats_max, }; #define Natts_disable_chunk_column_stats (_Anum_disable_chunk_column_stats_max - 1) TS_FUNCTION_INFO_V1(ts_chunk_column_stats_disable); /* * Create a datum to be returned by ts_chunk_column_stats_enable DDL function */ static Datum chunk_column_stats_enable_datum(FunctionCallInfo fcinfo, int32 id, bool enabled) { TupleDesc tupdesc; HeapTuple tuple; Datum values[Natts_enable_chunk_column_stats]; bool nulls[Natts_enable_chunk_column_stats] = { false }; if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in " "context that cannot accept type record"))); tupdesc = BlessTupleDesc(tupdesc); Assert(tupdesc->natts == Natts_enable_chunk_column_stats); values[AttrNumberGetAttrOffset(Anum_enable_chunk_column_stats_id)] = Int32GetDatum(id); values[AttrNumberGetAttrOffset(Anum_enable_chunk_column_stats_enabled)] = BoolGetDatum(enabled); tuple = heap_form_tuple(tupdesc, values, nulls); return HeapTupleGetDatum(tuple); } /* * Create a datum to be returned by ts_chunk_column_stats_disable DDL function */ static Datum chunk_column_stats_disable_datum(FunctionCallInfo fcinfo, int32 hypertable_id, Name colname, bool disabled) { TupleDesc tupdesc; HeapTuple tuple; Datum values[Natts_disable_chunk_column_stats]; bool nulls[Natts_disable_chunk_column_stats] = { false }; if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in " "context that cannot accept type record"))); tupdesc = BlessTupleDesc(tupdesc); Assert(tupdesc->natts == Natts_disable_chunk_column_stats); values[AttrNumberGetAttrOffset(Anum_disable_chunk_column_stats_hypertable_id)] = Int32GetDatum(hypertable_id); values[AttrNumberGetAttrOffset(Anum_disable_chunk_column_stats_column_name)] = NameGetDatum(colname); values[AttrNumberGetAttrOffset(Anum_disable_chunk_column_stats_disabled)] = BoolGetDatum(disabled); tuple = heap_form_tuple(tupdesc, values, nulls); return HeapTupleGetDatum(tuple); } static int32 chunk_column_stats_insert_relation(const Relation rel, Form_chunk_column_stats info) { TupleDesc desc = RelationGetDescr(rel); Datum values[Natts_chunk_column_stats] = { 0 }; bool nulls[Natts_chunk_column_stats] = { false }; CatalogSecurityContext sec_ctx; ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); info->id = ts_catalog_table_next_seq_id(ts_catalog_get(), CHUNK_COLUMN_STATS); values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_id)] = Int32GetDatum(info->id); values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_hypertable_id)] = Int32GetDatum(info->hypertable_id); values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_chunk_id)] = Int32GetDatum(info->chunk_id); values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_column_name)] = NameGetDatum(&info->column_name); values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_range_start)] = Int64GetDatum(info->range_start); values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_range_end)] = Int64GetDatum(info->range_end); values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_valid)] = BoolGetDatum(info->valid); if (info->chunk_id == INVALID_CHUNK_ID) nulls[AttrNumberGetAttrOffset(Anum_chunk_column_stats_chunk_id)] = true; ts_catalog_insert_values(rel, desc, values, nulls); ts_catalog_restore_user(&sec_ctx); return info->id; } static int32 chunk_column_stats_insert(Form_chunk_column_stats info) { Catalog *catalog = ts_catalog_get(); Relation rel; int32 ccol_stats_id; rel = table_open(catalog_get_table_id(catalog, CHUNK_COLUMN_STATS), RowExclusiveLock); ccol_stats_id = chunk_column_stats_insert_relation(rel, info); table_close(rel, RowExclusiveLock); return ccol_stats_id; } static ScanTupleResult chunk_column_stats_tuple_update(TupleInfo *ti, void *data) { bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); FormData_chunk_column_stats *fd = (FormData_chunk_column_stats *) data; Datum values[Natts_chunk_column_stats] = { 0 }; bool isnull[Natts_chunk_column_stats] = { 0 }; bool doReplace[Natts_chunk_column_stats] = { 0 }; values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_range_start)] = Int64GetDatum(fd->range_start); doReplace[AttrNumberGetAttrOffset(Anum_chunk_column_stats_range_start)] = true; values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_range_end)] = Int64GetDatum(fd->range_end); doReplace[AttrNumberGetAttrOffset(Anum_chunk_column_stats_range_end)] = true; values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_valid)] = BoolGetDatum(fd->valid); doReplace[AttrNumberGetAttrOffset(Anum_chunk_column_stats_valid)] = true; HeapTuple new_tuple = heap_modify_tuple(tuple, ts_scanner_get_tupledesc(ti), values, isnull, doReplace); ts_catalog_update(ti->scanrel, new_tuple); heap_freetuple(new_tuple); if (should_free) heap_freetuple(tuple); return SCAN_DONE; } static int chunk_column_stats_scan_internal(ScanKeyData *scankey, int nkeys, tuple_found_func tuple_found, void *data, int limit, int dimension_index, LOCKMODE lockmode, MemoryContext mctx) { Catalog *catalog = ts_catalog_get(); ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, CHUNK_COLUMN_STATS), .index = catalog_get_index(catalog, CHUNK_COLUMN_STATS, dimension_index), .nkeys = nkeys, .limit = limit, .scankey = scankey, .data = data, .tuple_found = tuple_found, .lockmode = lockmode, .scandirection = ForwardScanDirection, .result_mctx = mctx, }; return ts_scanner_scan(&scanctx); } int ts_chunk_column_stats_update_by_id(int32 chunk_column_stats_id, FormData_chunk_column_stats *fd_range) { ScanKeyData scankey[1]; ScanKeyInit(&scankey[0], Anum_chunk_column_stats_id_idx_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(chunk_column_stats_id)); return chunk_column_stats_scan_internal(scankey, 1, chunk_column_stats_tuple_update, fd_range, 1, CHUNK_COLUMN_STATS_ID_IDX, RowExclusiveLock, CurrentMemoryContext); } static void ts_chunk_column_stats_validate(Form_chunk_column_stats info, const Oid hypertable_relid, bool if_not_exists) { HeapTuple tuple; Datum datum; bool isnull; Oid column_type; /* Check that the column exists and has not been dropped */ tuple = SearchSysCacheAttName(hypertable_relid, NameStr(info->column_name)); if (!HeapTupleIsValid(tuple)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("column \"%s\" does not exist", NameStr(info->column_name)))); datum = SysCacheGetAttr(ATTNAME, tuple, Anum_pg_attribute_atttypid, &isnull); Assert(!isnull); column_type = DatumGetObjectId(datum); ReleaseSysCache(tuple); /* we only support a subset of data types for range calculations right now */ switch (column_type) { case INT2OID: case INT4OID: case INT8OID: case TIMESTAMPOID: case TIMESTAMPTZOID: case DATEOID: break; default: ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("data type \"%s\" unsupported for range calculation", format_type_be(column_type)), errhint("Integer-like, timestamp-like data types supported currently"))); } } /* * Track min/max range for a given column in a hypertable */ static Datum ts_chunk_column_stats_add_internal(FunctionCallInfo fcinfo, Oid table_relid, Name colname, bool if_not_exists) { Hypertable *ht; Cache *hcache; Datum retval = 0; int32 ccol_stats_id = 0; FormData_chunk_column_stats fd = { 0 }; Form_chunk_column_stats form; bool enabled = true; ts_hypertable_permissions_check(table_relid, GetUserId()); namestrcpy(&fd.column_name, NameStr(*colname)); LockRelationOid(table_relid, AccessShareLock); ts_chunk_column_stats_validate(&fd, table_relid, if_not_exists); ht = ts_hypertable_cache_get_cache_and_entry(table_relid, CACHE_FLAG_NONE, &hcache); /* * Add an entry in the _timescaledb_catalog.chunk_column_stats table. We add * a special entry in the catalog which contains the hypertable_id, the colname, * an invalid id (for the chunk) and PG_INT64_MAX, PG_INT64_MIN as range values * to indicate that ranges should be calculated for this column for chunks. * * We have a uniqueness check on ht_id, colname, chunk_id * * Check if the entry already exists, first. */ form = ts_chunk_column_stats_lookup(ht->fd.id, INVALID_CHUNK_ID, NameStr(*colname)); if (form != NULL) { if (!if_not_exists) { ereport(ERROR, (errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("already enabled for column \"%s\"", NameStr(*colname)))); } else { ereport(NOTICE, (errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("already enabled for column \"%s\", skipping", NameStr(*colname)))); /* return the existing id */ ccol_stats_id = form->id; /* we still return true since it's already enabled */ enabled = true; goto do_return; } } fd.hypertable_id = ht->fd.id; fd.chunk_id = INVALID_CHUNK_ID; fd.range_start = PG_INT64_MIN; fd.range_end = PG_INT64_MAX; fd.valid = true; ccol_stats_id = chunk_column_stats_insert(&fd); /* refresh the ht entry to accommodate this new chunk_column_stats entry */ if (ht->range_space) pfree(ht->range_space); ht->range_space = ts_chunk_column_stats_range_space_scan(ht->fd.id, ht->main_table_relid, ts_cache_memory_ctx(hcache)); /* * If the hypertable has chunks, to make it compatible * we add artificial min/max range entries which will cover -inf / inf * range for all these existing chunks. * * TODO: Maybe have a future version which calculates actual ranges for * compressed chunks in this function itself? Or have an option to this * function which specifies if we should calculate ranges for compressed * chunks. */ if (ts_hypertable_has_chunks(ht->main_table_relid, AccessShareLock)) { ListCell *lc; List *chunk_id_list = ts_chunk_get_chunk_ids_by_hypertable_id(ht->fd.id); Catalog *catalog = ts_catalog_get(); Relation rel; rel = table_open(catalog_get_table_id(catalog, CHUNK_COLUMN_STATS), RowExclusiveLock); foreach (lc, chunk_id_list) { /* other fields are set appropriately in fd above. Only change chunk_id */ fd.chunk_id = lfirst_int(lc); chunk_column_stats_insert_relation(rel, &fd); } table_close(rel, RowExclusiveLock); } do_return: /* return the id of the main entry for this dimension range */ fd.id = ccol_stats_id; retval = chunk_column_stats_enable_datum(fcinfo, fd.id, enabled); ts_cache_release(&hcache); PG_RETURN_DATUM(retval); } /* * Add min/max range tracking for a column in a hypertable. * * Arguments: * 0. Relation ID of table * 1. Column name * 2. IF NOT EXISTS option (bool) */ Datum ts_chunk_column_stats_enable(PG_FUNCTION_ARGS) { Oid hypertable_relid; NameData colname; bool if_not_exists; TS_PREVENT_FUNC_IF_READ_ONLY(); if (!ts_guc_enable_chunk_skipping) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("chunk skipping functionality disabled, " "enable it by first setting timescaledb.enable_chunk_skipping to on"))); if (PG_ARGISNULL(0)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("hypertable cannot be NULL"))); hypertable_relid = PG_GETARG_OID(0); if (PG_ARGISNULL(1)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("column name cannot be NULL"))); namestrcpy(&colname, NameStr(*PG_GETARG_NAME(1))); if_not_exists = PG_ARGISNULL(2) ? false : PG_GETARG_BOOL(2); return ts_chunk_column_stats_add_internal(fcinfo, hypertable_relid, &colname, if_not_exists); } /* * Remove min/max range tracking for a column in a hypertable. * * Arguments: * 0. Relation ID of hypertable * 1. Column name * 2. IF NOT EXISTS option (bool) */ Datum ts_chunk_column_stats_disable(PG_FUNCTION_ARGS) { Oid hypertable_relid; NameData colname; bool if_not_exists; Hypertable *ht; Cache *hcache; Datum retval = 0; int delete_count = 0; TS_PREVENT_FUNC_IF_READ_ONLY(); if (!ts_guc_enable_chunk_skipping) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("chunk skipping functionality disabled, " "enable it by first setting timescaledb.enable_chunk_skipping to on"))); if (PG_ARGISNULL(0)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("hypertable cannot be NULL"))); hypertable_relid = PG_GETARG_OID(0); if (PG_ARGISNULL(1)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("column name cannot be NULL"))); namestrcpy(&colname, NameStr(*PG_GETARG_NAME(1))); if_not_exists = PG_ARGISNULL(2) ? false : PG_GETARG_BOOL(2); ts_hypertable_permissions_check(hypertable_relid, GetUserId()); LockRelationOid(hypertable_relid, ShareUpdateExclusiveLock); ht = ts_hypertable_cache_get_cache_and_entry(hypertable_relid, CACHE_FLAG_NONE, &hcache); /* * Remove entries from _timescaledb_catalog.chunk_column_stats table. * * There's a special entry in the catalog which contains the hypertable_id, the colname, * an invalid id (for the chunk) and PG_INT64_MAX, PG_INT64_MIN as range values * to indicate that ranges should be calculated for this column for chunks. * * Check if the entry already exists, first. */ if (ts_chunk_column_stats_lookup(ht->fd.id, INVALID_CHUNK_ID, NameStr(colname)) == NULL) { if (!if_not_exists) { ereport(ERROR, (errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("statistics not enabled for column \"%s\"", NameStr(colname)))); } else { ereport(NOTICE, (errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("statistics not enabled for column \"%s\", skipping", NameStr(colname)))); goto do_return; } } /* Delete all entries matching this hypertable_id and column_name. */ delete_count = ts_chunk_column_stats_delete_by_ht_colname(ht->fd.id, NameStr(colname)); /* refresh the ht entry to accommodate this deleted chunk_column_stats entry */ if (ht->range_space) pfree(ht->range_space); ht->range_space = ts_chunk_column_stats_range_space_scan(ht->fd.id, ht->main_table_relid, ts_cache_memory_ctx(hcache)); do_return: retval = chunk_column_stats_disable_datum(fcinfo, ht->fd.id, &colname, delete_count > 0); ts_cache_release(&hcache); PG_RETURN_DATUM(retval); } /* * Dimension range entries are similar to OPEN DIMENSION entries. So, most of * the default fields are similar to them. */ Dimension * ts_chunk_column_stats_fill_dummy_dimension(FormData_chunk_column_stats *r, Oid main_table_relid) { Dimension *d = palloc0(sizeof(Dimension)); d->fd.id = r->id; d->fd.hypertable_id = r->hypertable_id; d->fd.aligned = true; namestrcpy(&d->fd.column_name, NameStr(r->column_name)); d->fd.interval_length = 1; /* a dummy interval length for the dummy dimension */ /* similar to open dimensions except that we don't participate in partitioning */ d->type = DIMENSION_TYPE_STATS; d->column_attno = get_attnum(main_table_relid, NameStr(d->fd.column_name)); d->main_table_relid = main_table_relid; /* rest of the fields are zeroed out */ return d; } /* * Create a CHECK constraint for a min/max range chunk_column_stats entry */ static Constraint * create_col_stats_check_constraint(const Form_chunk_column_stats info, Oid main_table_relid, Oid chunk_relid, const char *name) { Constraint *constr = NULL; Node *rangedef; ColumnRef *colref; List *compexprs = NIL; Oid col_type; int attno; if (info->range_start == PG_INT64_MIN && info->range_end == PG_INT64_MAX) return NULL; colref = makeNode(ColumnRef); colref->fields = list_make1(makeString(pstrdup(NameStr(info->column_name)))); colref->location = -1; /* * Get the column type for later converting the internal format * to string. * * Get the attribute number in the HT for this column, and map to the chunk */ attno = get_attnum(main_table_relid, NameStr(info->column_name)); attno = ts_map_attno(main_table_relid, chunk_relid, attno); col_type = get_atttype(main_table_relid, attno); rangedef = (Node *) colref; /* Elide range constraint for +INF or -INF */ if (info->range_start != PG_INT64_MIN) { A_Const *start_const = makeNode(A_Const); memcpy(&start_const->val, makeString(ts_internal_to_time_string(info->range_start, col_type)), sizeof(start_const->val)); start_const->location = -1; A_Expr *ge_expr = makeSimpleA_Expr(AEXPR_OP, ">=", rangedef, (Node *) start_const, -1); compexprs = lappend(compexprs, ge_expr); } if (info->range_end != PG_INT64_MAX) { A_Const *end_const = makeNode(A_Const); memcpy(&end_const->val, makeString(ts_internal_to_time_string(info->range_end, col_type)), sizeof(end_const->val)); end_const->location = -1; A_Expr *lt_expr = makeSimpleA_Expr(AEXPR_OP, "<", rangedef, (Node *) end_const, -1); compexprs = lappend(compexprs, lt_expr); } constr = makeNode(Constraint); constr->contype = CONSTR_CHECK; constr->conname = name ? pstrdup(name) : NULL; constr->deferrable = false; constr->skip_validation = true; constr->initially_valid = true; Assert(list_length(compexprs) >= 1); if (list_length(compexprs) == 2) constr->raw_expr = (Node *) makeBoolExpr(AND_EXPR, compexprs, -1); else if (list_length(compexprs) == 1) constr->raw_expr = linitial(compexprs); return constr; } /* * Fill in the form for chunk_column_stats. * * Note that it is necessary to deform the tuple since it is not possible to * use GETSTRUCT when chunk_id can be NULL. */ static void fill_form_from_slot(TupleTableSlot *slot, Form_chunk_column_stats form) { bool should_free; HeapTuple tuple = ExecFetchSlotHeapTuple(slot, false, &should_free); Datum values[_Anum_chunk_column_stats_max]; bool nulls[_Anum_chunk_column_stats_max]; heap_deform_tuple(tuple, slot->tts_tupleDescriptor, values, nulls); form->id = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_id)]); form->hypertable_id = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_hypertable_id)]); if (nulls[AttrNumberGetAttrOffset(Anum_chunk_column_stats_chunk_id)]) form->chunk_id = INVALID_CHUNK_ID; else form->chunk_id = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_chunk_id)]); namestrcpy(&form->column_name, NameStr(*DatumGetName( values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_column_name)]))); form->range_end = DatumGetInt64(values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_range_end)]); form->range_start = DatumGetInt64(values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_range_start)]); form->valid = DatumGetBool(values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_valid)]); if (should_free) heap_freetuple(tuple); } static ScanTupleResult chunk_column_stats_tuple_found(TupleInfo *ti, void *data) { ChunkRangeSpace *rs = data; Form_chunk_column_stats d = &rs->range_cols[rs->num_range_cols++]; fill_form_from_slot(ti->slot, d); return SCAN_CONTINUE; } ChunkRangeSpace * ts_chunk_column_stats_range_space_scan(int32 hypertable_id, Oid ht_reloid, MemoryContext mctx) { /* We won't have more entries than the number of columns in the HT */ int num_range_cols = ts_get_relnatts(ht_reloid); ChunkRangeSpace *range_space = MemoryContextAllocZero(mctx, CHUNKRANGESPACE_SIZE(num_range_cols)); ScanKeyData scankey[2]; range_space->hypertable_id = hypertable_id; range_space->capacity = num_range_cols; range_space->num_range_cols = 0; /* Perform an index scan on hypertable_id, invalid chunk_id. */ ScanKeyInit( &scankey[0], Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); ScanKeyEntryInitialize( &scankey[1], SK_ISNULL | SK_SEARCHNULL, Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_chunk_id, BTEqualStrategyNumber, InvalidOid, InvalidOid, InvalidOid, Int32GetDatum(INVALID_CHUNK_ID)); chunk_column_stats_scan_internal(scankey, 2, chunk_column_stats_tuple_found, range_space, 0, CHUNK_COLUMN_STATS_HT_ID_CHUNK_ID_COLUMN_NAME_IDX, AccessShareLock, mctx); if (range_space->num_range_cols == 0) { pfree(range_space); return NULL; } return range_space; } static ScanTupleResult form_range_tuple_found(TupleInfo *ti, void *data) { Form_chunk_column_stats rg = data; fill_form_from_slot(ti->slot, rg); return SCAN_DONE; } Form_chunk_column_stats ts_chunk_column_stats_lookup(int32 hypertable_id, int32 chunk_id, const char *col_name) { ScanKeyData scankey[3]; Form_chunk_column_stats form_range = palloc0(sizeof(FormData_chunk_column_stats)); form_range->chunk_id = INVALID_CHUNK_ID; /* for clarity */ /* Perform an index scan on hypertable_id, chunk_id, col_name. */ ScanKeyInit( &scankey[0], Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); if (chunk_id == INVALID_CHUNK_ID) { ScanKeyEntryInitialize( &scankey[1], SK_ISNULL | SK_SEARCHNULL, Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_chunk_id, BTEqualStrategyNumber, InvalidOid, InvalidOid, InvalidOid, Int32GetDatum(chunk_id)); } else { ScanKeyInit( &scankey[1], Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_chunk_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(chunk_id)); } ScanKeyInit( &scankey[2], Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_column_name, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(col_name)); chunk_column_stats_scan_internal(scankey, 3, form_range_tuple_found, form_range, 1, CHUNK_COLUMN_STATS_HT_ID_CHUNK_ID_COLUMN_NAME_IDX, AccessShareLock, CurrentMemoryContext); if (strlen(NameStr(form_range->column_name)) == 0) { pfree(form_range); return NULL; } return form_range; } static bool chunk_get_minmax(const Chunk *chunk, Oid col_type, const char *col_name, Datum *minmax) { StringInfoData command; int res; /* Lock down search_path */ int save_nestlevel = NewGUCNestLevel(); RestrictSearchPath(); initStringInfo(&command); appendStringInfo(&command, "SELECT pg_catalog.min(%s), pg_catalog.max(%s) FROM %s.%s", quote_identifier(col_name), quote_identifier(col_name), quote_identifier(NameStr(chunk->fd.schema_name)), quote_identifier(NameStr(chunk->fd.table_name))); /* * SPI_connect will switch MemoryContext so we need to keep track * of caller context as we need to copy the values into caller * context. */ MemoryContext caller = CurrentMemoryContext; if (SPI_connect() != SPI_OK_CONNECT) elog(ERROR, "could not connect to SPI"); res = SPI_execute(command.data, true /* read_only */, 0 /*count*/); if (res < 0) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), (errmsg("could not get the min/max values for column \"%s\" of chunk \"%s.%s\"", col_name, chunk->fd.schema_name.data, chunk->fd.table_name.data)))); pfree(command.data); Datum min, max; bool isnull_min = false, isnull_max = false; min = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull_min); max = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 2, &isnull_max); Assert(SPI_gettypeid(SPI_tuptable->tupdesc, 1) == col_type); Assert(SPI_gettypeid(SPI_tuptable->tupdesc, 2) == col_type); bool found = !isnull_min && !isnull_max; if (found) { bool typbyval; int16 typlen; get_typlenbyval(col_type, &typlen, &typbyval); /* Copy the values into caller context */ MemoryContext spi = MemoryContextSwitchTo(caller); minmax[0] = datumCopy(min, typbyval, typlen); minmax[1] = datumCopy(max, typbyval, typlen); MemoryContextSwitchTo(spi); } /* Restore search_path */ AtEOXact_GUC(false, save_nestlevel); res = SPI_finish(); if (res != SPI_OK_FINISH) elog(ERROR, "SPI_finish failed: %s", SPI_result_code_string(res)); return found; } /* * Update column dimension ranges in the catalog for the * provided chunk (it's assumed that the chunk is locked * appropriately). * * Calculate actual ranges for the given chunk for the columns * insert these entries. This allows for the * chunk to be picked up when queries use these columns in * WHERE clauses with these ranges. * * Returns the number of column entries that have been added or * updated. */ int ts_chunk_column_stats_calculate(const Hypertable *ht, const Chunk *chunk) { Size i = 0; ChunkRangeSpace *rs = ht->range_space; MemoryContext work_mcxt, orig_mcxt; /* Quick check. Bail out early if none */ if (rs == NULL) return i; work_mcxt = AllocSetContextCreate(CurrentMemoryContext, "dimension-range-work", ALLOCSET_DEFAULT_SIZES); orig_mcxt = MemoryContextSwitchTo(work_mcxt); for (int range_index = 0; range_index < rs->num_range_cols; range_index++) { Datum minmax[2]; AttrNumber attno; char *col_name = NameStr(rs->range_cols[range_index].column_name); Oid col_type; attno = get_attnum(ht->main_table_relid, col_name); attno = ts_map_attno(ht->main_table_relid, chunk->table_id, attno); col_type = get_atttype(chunk->table_id, attno); /* calculate the min/max range for this column on this chunk */ if (chunk_get_minmax(chunk, col_type, col_name, minmax)) { Form_chunk_column_stats range; int64 min = ts_time_value_to_internal(minmax[0], col_type); int64 max = ts_time_value_to_internal(minmax[1], col_type); /* The end value is exclusive to the range, so incr by 1 */ if (max != DIMENSION_SLICE_MAXVALUE) { max++; /* Again, check overflow */ max = REMAP_LAST_COORDINATE(max); } /* * Check if an entry exists for this ht, chunk_id, colname combo. If it exists * and it's not -inf/+inf then it's probably a case of re-computation of the * ranges. In such a case, we compare the stored range_start and range_end entries * and compare with the min/max calculated. * * if min < range_start, then new_range_start = min * if max > range_end, then new_range_end = max * * We need to update the existing entry with changes in the range. * Also, in case of updates, the entry might be marked "invalid" so it needs to be * made "valid" again as well. */ range = ts_chunk_column_stats_lookup(ht->fd.id, chunk->fd.id, col_name); /* Add a new entry if none exists */ if (range == NULL) { FormData_chunk_column_stats fd = { 0 }; fd.hypertable_id = ht->fd.id; fd.chunk_id = chunk->fd.id; namestrcpy(&fd.column_name, col_name); fd.range_start = min; fd.range_end = max; fd.valid = true; chunk_column_stats_insert(&fd); i++; } /* update case */ else if (range->range_start != min || range->range_end != max || !range->valid) { range->range_start = min; range->range_end = max; range->valid = true; ts_chunk_column_stats_update_by_id(range->id, range); i++; } } else ereport(WARNING, errmsg("unable to calculate min/max values for column ranges")); } MemoryContextSwitchTo(orig_mcxt); MemoryContextDelete(work_mcxt); return i; } /* * Insert column dimension ranges in the catalog for the * provided chunk (it's assumed that the chunk is locked * appropriately). * * We insert -inf/+inf entries for the given chunk which means * default selection till the actual ranges get calculated later. * * Returns the number of column entries that have been inserted. */ int ts_chunk_column_stats_insert(const Hypertable *ht, const Chunk *chunk) { Size range_index = 0; ChunkRangeSpace *rs = ht->range_space; MemoryContext work_mcxt, orig_mcxt; /* Quick check. Bail out early if none */ if (rs == NULL) return range_index; work_mcxt = AllocSetContextCreate(CurrentMemoryContext, "dimension-range-work", ALLOCSET_DEFAULT_SIZES); orig_mcxt = MemoryContextSwitchTo(work_mcxt); for (range_index = 0; range_index < rs->num_range_cols; range_index++) { AttrNumber attno; char *col_name = NameStr(rs->range_cols[range_index].column_name); FormData_chunk_column_stats fd = { 0 }; /* Get the attribute number in the HT for this column, and map to the chunk */ attno = get_attnum(ht->main_table_relid, col_name); attno = ts_map_attno(ht->main_table_relid, chunk->table_id, attno); /* insert an entry for this ht_id, chunk_id for this col_name with -inf/+inf range */ fd.hypertable_id = ht->fd.id; fd.chunk_id = chunk->fd.id; namestrcpy(&fd.column_name, col_name); fd.range_start = PG_INT64_MIN; fd.range_end = PG_INT64_MAX; fd.valid = true; chunk_column_stats_insert(&fd); } MemoryContextSwitchTo(orig_mcxt); MemoryContextDelete(work_mcxt); return range_index; } /* * Check if there is a min/max range tracking on this column which is being dropped. * Need to delete the entries from _timescaledb_catalog.chunk_column_stats table in that case. */ void ts_chunk_column_stats_drop(const Hypertable *ht, const char *col_name, bool *dropped) { /* delete all entries belonging to this HT and pointing to this column */ *dropped = (ts_chunk_column_stats_delete_by_ht_colname(ht->fd.id, col_name) > 0); } static ScanTupleResult chunk_column_stats_tuple_delete(TupleInfo *ti, void *data) { CatalogSecurityContext sec_ctx; int *count = data; /* delete catalog entry */ ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); ts_catalog_restore_user(&sec_ctx); *count = *count + 1; return SCAN_CONTINUE; } int ts_chunk_column_stats_delete_by_ht_colname(int32 hypertable_id, const char *col_name) { ScanKeyData scankey[2]; int count = 0; /* Perform an index scan on hypertable_id, col_name. */ ScanKeyInit( &scankey[0], Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); ScanKeyInit( &scankey[1], Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_column_name, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum((col_name))); chunk_column_stats_scan_internal(scankey, 2, chunk_column_stats_tuple_delete, &count, 0, CHUNK_COLUMN_STATS_HT_ID_CHUNK_ID_COLUMN_NAME_IDX, RowExclusiveLock, CurrentMemoryContext); return count; } int ts_chunk_column_stats_delete_by_chunk_id(int32 chunk_id) { ScanKeyData scankey[1]; int count = 0; Assert(chunk_id != INVALID_CHUNK_ID); /* Perform an index scan on chunk_id. */ ScanKeyInit( &scankey[0], Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_chunk_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(chunk_id)); chunk_column_stats_scan_internal(scankey, 1, chunk_column_stats_tuple_delete, &count, 0, CHUNK_COLUMN_STATS_HT_ID_CHUNK_ID_COLUMN_NAME_IDX, RowExclusiveLock, CurrentMemoryContext); return count; } int ts_chunk_column_stats_reset_by_chunk_id(int32 chunk_id) { ScanKeyData scankey[1]; FormData_chunk_column_stats fd = { 0 }; /* reset the range to min and max for all entries belonging to this chunk */ fd.range_start = PG_INT64_MIN; fd.range_end = PG_INT64_MAX; fd.valid = true; Assert(chunk_id != INVALID_CHUNK_ID); /* Perform an index scan on chunk_id. */ ScanKeyInit( &scankey[0], Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_chunk_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(chunk_id)); return chunk_column_stats_scan_internal(scankey, 1, chunk_column_stats_tuple_update, &fd, 0, CHUNK_COLUMN_STATS_HT_ID_CHUNK_ID_COLUMN_NAME_IDX, RowExclusiveLock, CurrentMemoryContext); } int ts_chunk_column_stats_delete_by_hypertable_id(int32 hypertable_id) { ScanKeyData scankey[1]; int count = 0; /* Perform an index scan on hypertable_id. */ ScanKeyInit( &scankey[0], Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); chunk_column_stats_scan_internal(scankey, 1, chunk_column_stats_tuple_delete, &count, 0, CHUNK_COLUMN_STATS_HT_ID_CHUNK_ID_COLUMN_NAME_IDX, RowExclusiveLock, CurrentMemoryContext); return count; } /* * For min/max ranges we are interested in the occurrence of a value which * possibly lies in multiple entries from _timescaledb_catalog.chunk_column_stats. * * The check for enclosure needs to be run as a FILTER on top of all the matching * entries for the hypertable, column combo. So, we only use ht_id, col_name for * the scan below. */ static int chunk_column_stats_scan_iterator_set(ScanIterator *it, int32 hypertable_id, const char *col_name) { Catalog *catalog = ts_catalog_get(); it->ctx.index = catalog_get_index(catalog, CHUNK_COLUMN_STATS, CHUNK_COLUMN_STATS_HT_ID_CHUNK_ID_COLUMN_NAME_IDX); ts_scan_iterator_scan_key_reset(it); ts_scan_iterator_scan_key_init( it, Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); ts_scan_iterator_scan_key_init( it, Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_column_name, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum((col_name))); it->ctx.scandirection = ForwardScanDirection; return it->ctx.nkeys; } /* * We need to get all chunks matching the hypertable ID and the column name. * For each chunk obtained, we need to run a FILTER using the strategies * and the lower/upper bound values provided. * * The EXPLAIN plan is basically like below: * * Index Scan using chunk_column_stats_ht_id_chunk_id_colname_range_start_end_key * on _timescaledb_catalog.chunk_column_stats * Output: chunk_id * Index Cond: ((chunk_column_stats.hypertable_id = :ht_id) AND * (chunk_column_stats.column_name = ':colname')) * Filter: ((chunk_column_stats.range_end BTREE_OP lower_bound/upper_bound) OR * (chunk_column_stats.range_start BTREE_OP lower_bound/upper_bound)) * * The strategies and lower_bound/upper_bound values get assigned in * dimension_restrict_info_range_add function. * * We need to run the "Filter" above ourselves because there's no other PG mechanism for OR * types of checks like these. */ List * ts_chunk_column_stats_get_chunk_ids_by_scan(DimensionRestrictInfo *dri) { ScanIterator it; List *chunkids = NIL; DimensionRestrictInfoOpen *open; Assert(dri && dri->dimension->type == DIMENSION_TYPE_STATS); /* setup the scanner */ it = ts_scan_iterator_create(CHUNK_COLUMN_STATS, AccessShareLock, CurrentMemoryContext); it.ctx.flags |= SCANNER_F_NOEND_AND_NOCLOSE; it.ctx.tuplock = NULL; open = (DimensionRestrictInfoOpen *) dri; /* * We need to get all chunks matching the hypertable ID and the column name. */ chunk_column_stats_scan_iterator_set(&it, dri->dimension->fd.hypertable_id, NameStr(dri->dimension->fd.column_name)); /* * For each chunk obtained, we need to run a FILTER using the strategies * and the lower/upper bound values provided. */ ts_scan_iterator_start_or_restart_scan(&it); ts_scanner_foreach(&it) { FormData_chunk_column_stats fd; bool matched = false; bool chunk_id_isnull; chunk_id_isnull = slot_attisnull( it.tinfo->slot, Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_chunk_id); /* * We have an entry with INVALID_CHUNK_ID which will match all cases due to * -INF/+INF range entries for it. Ignore that. */ if (chunk_id_isnull) goto done; fill_form_from_slot(it.tinfo->slot, &fd); /* * If an entry is marked "invalid" then it means that the ranges cannot be relied * on. So, we assume the worse case and include this chunk for the scan. * * (Entry is typically marked "invalid" when a compressed chunk becomes partial * due to DML in it.) * * Also, if we have a valid chunnk with -inf/+inf entries then it matches all * queries */ if (!fd.valid || (fd.range_start == PG_INT64_MIN && fd.range_end == PG_INT64_MAX)) { matched = true; goto done; } /* * All data is in int8 format so we do regular comparisons. Also, it's an OR * check so prepare to short circuit if one evaluates to true. * * No real way to know if checking range_start or range_end first will be more * effective. So let's start with range_end checks first. */ switch (open->upper_strategy) { case BTLessEqualStrategyNumber: /* e.g: id <= 90 */ { matched = fd.range_start <= open->upper_bound; } break; case BTLessStrategyNumber: /* e.g: id < 90 */ { matched = fd.range_start < open->upper_bound; } break; default: open->upper_strategy = InvalidStrategy; break; } if (open->upper_strategy != InvalidStrategy && !matched) goto done; /* range_end checks didn't match, check for range_start now */ switch (open->lower_strategy) { case BTGreaterEqualStrategyNumber: { /* range_end is exclusive */ matched = (fd.range_end - 1) >= open->lower_bound; } break; case BTGreaterStrategyNumber: { /* range_end is exclusive */ matched = (fd.range_end - 1) > open->lower_bound; } break; default: /* unsupported strategy */ break; } done: if (matched) chunkids = lappend_int(chunkids, fd.chunk_id); } ts_scan_iterator_close(&it); return chunkids; } /* * Update all entries for this ht_id, old_colname to point to the new_colname */ int ts_chunk_column_stats_set_name(FormData_chunk_column_stats *in_fd, char *new_colname) { ScanIterator it; NameData new_column_name; int count = 0; namestrcpy(&new_column_name, new_colname); /* setup the scanner */ it = ts_scan_iterator_create(CHUNK_COLUMN_STATS, AccessShareLock, CurrentMemoryContext); it.ctx.flags |= SCANNER_F_NOEND_AND_NOCLOSE; it.ctx.tuplock = NULL; /* * We need to get all chunks matching the hypertable ID and the column name. */ chunk_column_stats_scan_iterator_set(&it, in_fd->hypertable_id, NameStr(in_fd->column_name)); /* * For each entry obtained, we need to update the column_name to point to the * new_colname */ ts_scan_iterator_start_or_restart_scan(&it); ts_scanner_foreach(&it) { Datum values[Natts_chunk_column_stats] = { 0 }; bool isnull[Natts_chunk_column_stats] = { false }; bool doReplace[Natts_chunk_column_stats] = { 0 }; bool should_free; TupleInfo *ti = ts_scan_iterator_tuple_info(&it); HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_column_name)] = NameGetDatum(&new_column_name); doReplace[AttrNumberGetAttrOffset(Anum_chunk_column_stats_column_name)] = true; HeapTuple new_tuple = heap_modify_tuple(tuple, ts_scanner_get_tupledesc(ti), values, isnull, doReplace); ts_catalog_update(ti->scanrel, new_tuple); heap_freetuple(new_tuple); if (should_free) heap_freetuple(tuple); count++; } ts_scan_iterator_close(&it); return count; } static ScanTupleResult invalidate_range_tuple_found(TupleInfo *ti, void *data) { bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); bool valid = false; Datum values[Natts_chunk_column_stats] = { 0 }; bool isnull[Natts_chunk_column_stats] = { 0 }; bool doReplace[Natts_chunk_column_stats] = { 0 }; values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_valid)] = BoolGetDatum(valid); doReplace[AttrNumberGetAttrOffset(Anum_chunk_column_stats_valid)] = true; HeapTuple new_tuple = heap_modify_tuple(tuple, ts_scanner_get_tupledesc(ti), values, isnull, doReplace); ts_catalog_update(ti->scanrel, new_tuple); heap_freetuple(new_tuple); if (should_free) heap_freetuple(tuple); return SCAN_CONTINUE; } /* * Mark all entries for a given chunk_id as "invalid" */ void ts_chunk_column_stats_set_invalid(int32 hypertable_id, int32 chunk_id) { ScanKeyData scankey[2]; Assert(chunk_id != INVALID_CHUNK_ID); /* Perform an index scan on hypertable_id, chunk_id. */ ScanKeyInit( &scankey[0], Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); ScanKeyInit( &scankey[1], Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_chunk_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(chunk_id)); chunk_column_stats_scan_internal(scankey, 2, invalidate_range_tuple_found, NULL, 0, CHUNK_COLUMN_STATS_HT_ID_CHUNK_ID_COLUMN_NAME_IDX, RowExclusiveLock, CurrentMemoryContext); } typedef struct CheckList { Oid chunk_relid; Oid main_table_relid; List *cclist; } CheckList; static ScanTupleResult construct_check_constraint_range_tuple(TupleInfo *ti, void *data) { bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); FormData_chunk_column_stats fd; Constraint *constr; CheckList *checklist = data; fill_form_from_slot(ti->slot, &fd); constr = create_col_stats_check_constraint(&fd, checklist->main_table_relid, checklist->chunk_relid, NULL); if (constr) checklist->cclist = lappend(checklist->cclist, constr); if (should_free) heap_freetuple(tuple); return SCAN_CONTINUE; } /* * Given an input relationObjectId, check that it's a chunk and if yes, check that it * has min/max ranges on it and return a list of constructed check constraint * entries for each such entry. */ List * ts_chunk_column_stats_construct_check_constraints(Relation relation, Oid reloid, Index varno) { FormData_chunk fd; CheckList clist = { 0 }; ListCell *lc; ScanKeyData scankey[2]; ParseState *pstate = NULL; List *result = NIL; /* check if it's not a chunk and return early in that case */ if (!ts_chunk_simple_scan_by_reloid(reloid, &fd, true)) return NIL; clist.chunk_relid = reloid; clist.main_table_relid = ts_hypertable_id_to_relid(fd.hypertable_id, false); Assert(fd.id != INVALID_CHUNK_ID); /* Perform an index scan on hypertable_id, chunk_id. */ ScanKeyInit( &scankey[0], Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(fd.hypertable_id)); ScanKeyInit( &scankey[1], Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_chunk_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(fd.id)); chunk_column_stats_scan_internal(scankey, 2, construct_check_constraint_range_tuple, &clist, 0, CHUNK_COLUMN_STATS_HT_ID_CHUNK_ID_COLUMN_NAME_IDX, RowExclusiveLock, CurrentMemoryContext); if (clist.cclist) { pstate = make_parsestate(NULL); /* The overall query should be holding an appropriate lock already on this relation */ ParseNamespaceItem *nsitem = addRangeTableEntryForRelation(pstate, relation, AccessShareLock, NULL, false, false); addNSItemToQuery(pstate, nsitem, true, true, true); } foreach (lc, clist.cclist) { Node *expr; Constraint *constr = lfirst(lc); /* Transform raw parsetree to executable expression. */ expr = transformExpr(pstate, constr->raw_expr, EXPR_KIND_CHECK_CONSTRAINT); /* Make sure it yields a boolean result. */ expr = coerce_to_boolean(pstate, expr, "CHECK"); /* Take care of collations. */ assign_expr_collations(pstate, expr); expr = eval_const_expressions(NULL, expr); expr = (Node *) canonicalize_qual((Expr *) expr, true); /* Fix Vars to have the desired varno */ if (varno != 1) ChangeVarNodes(expr, 1, varno, 0); /* * Finally, convert to implicit-AND format (that is, a List) and * append the resulting item(s) to our output list. */ result = list_concat(result, make_ands_implicit((Expr *) expr)); } return result; } ================================================ FILE: src/ts_catalog/chunk_column_stats.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include "chunk.h" #include "export.h" #include "hypertable_restrict_info.h" /* * A rangespace tracks all columns that need min/max value ranges calculation * for chunks from a given hypertable */ typedef struct ChunkRangeSpace { int32 hypertable_id; uint16 capacity; uint16 num_range_cols; FormData_chunk_column_stats range_cols[FLEXIBLE_ARRAY_MEMBER]; } ChunkRangeSpace; #define CHUNKRANGESPACE_SIZE(num_columns) \ (sizeof(ChunkRangeSpace) + (sizeof(NameData) * (num_columns))) extern ChunkRangeSpace *ts_chunk_column_stats_range_space_scan(int32 hypertable_id, Oid ht_reloid, MemoryContext mctx); extern int ts_chunk_column_stats_update_by_id(int32 chunk_column_stats_id, FormData_chunk_column_stats *fd_range); extern Form_chunk_column_stats ts_chunk_column_stats_lookup(int32 hypertable_id, int32 chunk_id, const char *col_name); extern TSDLLEXPORT int ts_chunk_column_stats_calculate(const Hypertable *ht, const Chunk *chunk); extern int ts_chunk_column_stats_insert(const Hypertable *ht, const Chunk *chunk); extern void ts_chunk_column_stats_drop(const Hypertable *ht, const char *col_name, bool *dropped); extern int ts_chunk_column_stats_delete_by_ht_colname(int32 hypertable_id, const char *col_name); extern TSDLLEXPORT int ts_chunk_column_stats_delete_by_chunk_id(int32 chunk_id); extern TSDLLEXPORT int ts_chunk_column_stats_reset_by_chunk_id(int32 chunk_id); extern int ts_chunk_column_stats_delete_by_hypertable_id(int32 hypertable_id); extern Dimension *ts_chunk_column_stats_fill_dummy_dimension(FormData_chunk_column_stats *r, Oid main_table_relid); extern List *ts_chunk_column_stats_get_chunk_ids_by_scan(DimensionRestrictInfo *dri); extern void ts_chunk_column_stats_set_invalid(int32 hypertable_id, int32 chunk_id); extern int ts_chunk_column_stats_set_name(FormData_chunk_column_stats *in_fd, char *new_colname); extern List *ts_chunk_column_stats_construct_check_constraints(Relation relation, Oid reloid, Index varno); ================================================ FILE: src/ts_catalog/chunk_rewrite.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/attnum.h> #include <access/htup.h> #include <access/htup_details.h> #include <access/stratnum.h> #include <access/tableam.h> #include <catalog/dependency.h> #include <catalog/objectaddress.h> #include <catalog/pg_class_d.h> #include <executor/tuptable.h> #include <miscadmin.h> #include <nodes/lockoptions.h> #include <nodes/parsenodes.h> #include <storage/itemptr.h> #include <storage/lmgr.h> #include <storage/lockdefs.h> #include <utils/acl.h> #include <utils/lsyscache.h> #include <utils/syscache.h> #include "chunk_rewrite.h" #include "scan_iterator.h" #include "ts_catalog/catalog.h" /* * chunk_rewrite: * * This catalog table tracks pending rewrite operations for chunks. It is used when merging chunks * in concurrent mode to track temporarily written relations/heaps that might orphaned in case * of a failed rewrite. For example, a multi-transactional chunk merge could fail in the second * transaction and leave behind orphaned rewritten heaps it created in the first transaction. * * Each entry in the catalog is a mapping from a current relation to its new heap. For merges, this * is a many-to-one relation for each merge. * * Future operations, like concurrent split, might also use this catalog table. */ static HeapTuple chunk_rewrite_make_tuple(Oid chunk_relid, Oid new_relid, TupleDesc desc) { Datum values[Natts_chunk_rewrite]; bool nulls[Natts_chunk_rewrite] = { false }; memset(values, 0, sizeof(Datum) * Natts_chunk_rewrite); values[AttrNumberGetAttrOffset(Anum_chunk_rewrite_chunk_relid)] = ObjectIdGetDatum(chunk_relid); values[AttrNumberGetAttrOffset(Anum_chunk_rewrite_new_relid)] = ObjectIdGetDatum(new_relid); return heap_form_tuple(desc, values, nulls); } /* * Add an entry to the chunk_rewrite table. */ void ts_chunk_rewrite_add(Oid chunk_relid, Oid new_relid) { Catalog *catalog = ts_catalog_get(); Oid cat_relid = catalog_get_table_id(catalog, CHUNK_REWRITE); HeapTuple new_tuple; CatalogSecurityContext sec_ctx; Relation catrel; catrel = table_open(cat_relid, RowExclusiveLock); new_tuple = chunk_rewrite_make_tuple(chunk_relid, new_relid, catrel->rd_att); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_insert_only(catrel, new_tuple); ts_catalog_restore_user(&sec_ctx); heap_freetuple(new_tuple); table_close(catrel, NoLock); } /* * Look up an entry based on the original chunk_id. The entry is locked FOR UPDATE. */ bool ts_chunk_rewrite_get_with_lock(Oid chunk_relid, Form_chunk_rewrite form, ItemPointer tid) { Catalog *catalog = ts_catalog_get(); ScanIterator it; ScanTupLock tuplock = { .waitpolicy = LockWaitBlock, .lockmode = LockTupleExclusive, }; bool found = false; it = ts_scan_iterator_create(CHUNK_REWRITE, RowShareLock, CurrentMemoryContext); it.ctx.tuplock = &tuplock; it.ctx.flags = SCANNER_F_KEEPLOCK; it.ctx.index = catalog_get_index(catalog, CHUNK_REWRITE, CHUNK_REWRITE_IDX); ts_scan_iterator_scan_key_init(&it, Anum_chunk_rewrite_key_chunk_relid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(chunk_relid)); ts_scanner_foreach(&it) { TupleInfo *ti = ts_scan_iterator_tuple_info(&it); switch (ti->lockresult) { case TM_Ok: found = true; if (tid) ItemPointerCopy(&ti->slot->tts_tid, tid); if (form) { bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); memcpy(form, GETSTRUCT(tuple), sizeof(FormData_chunk_rewrite)); if (should_free) heap_freetuple(tuple); } break; case TM_Deleted: ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("chunk merge state deleted by concurrent transaction"))); break; default: ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("unable to lock chunk rewrite catalog tuple, lock result is %d for " "chunk (%u)", ti->lockresult, chunk_relid))); break; } } ts_scan_iterator_close(&it); return found; } /* * Delete an entry from the chunk_rewrite table based on TID. */ void ts_chunk_rewrite_delete_by_tid(const ItemPointer tid) { Catalog *catalog = ts_catalog_get(); Oid cat_relid = catalog_get_table_id(catalog, CHUNK_REWRITE); CatalogSecurityContext sec_ctx; Relation catrel; catrel = table_open(cat_relid, RowExclusiveLock); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_delete_tid_only(catrel, tid); ts_catalog_restore_user(&sec_ctx); table_close(catrel, NoLock); } /* * Delete an entry from the chunk_rewrite table and drop the orphaned heap (if it exists). * * The delete result indicates whether both the entry and the orphaned heap was dropped, * or if only the entry was deleted (in case the heap was already dropped), or a failure occurred. * * If the "conditional" parameter is specified, the entry will only be deleted if the referenced * heap relation can be immediately locked without waiting. This is useful in order to skip entries * that are locked by ongoing merges. */ ChunkRewriteDeleteResult ts_chunk_rewrite_delete(Oid chunk_relid, bool conditional) { ItemPointerData tid; FormData_chunk_rewrite form; ChunkRewriteDeleteResult result; if (!ts_chunk_rewrite_get_with_lock(chunk_relid, &form, &tid)) return ChunkRewriteEntryDoesNotExist; if (conditional) { if (!ConditionalLockRelationOid(form.new_relid, AccessExclusiveLock)) return ChunkRewriteOngoing; } /* * Check if the new heap still exists by trying to get a lock. */ Relation newrel = try_table_open(form.new_relid, AccessExclusiveLock); if (newrel) { ObjectAddress tableaddr; /* New heap still exists, so delete it */ table_close(newrel, NoLock); ObjectAddressSet(tableaddr, RelationRelationId, form.new_relid); performDeletion(&tableaddr, DROP_RESTRICT, PERFORM_DELETION_INTERNAL); result = ChunkRewriteEntryDeletedAndTableDropped; } else { result = ChunkRewriteEntryDeleted; } ts_chunk_rewrite_delete_by_tid(&tid); return result; } TS_FUNCTION_INFO_V1(ts_chunk_rewrite_cleanup); /* * Clean up failed chunk rewrites. * * This function cleans up all non-ongoing chunk rewrites listed in the chunk_rewrite catalog, * including any "orphaned" heaps referenced in the table. * */ Datum ts_chunk_rewrite_cleanup(PG_FUNCTION_ARGS) { ScanIterator it; ObjectAddresses *objaddrs = new_object_addresses(); unsigned int cleanup_count = 0; unsigned int skipped_count = 0; Oid userid = GetUserId(); CatalogSecurityContext sec_ctx; ScanTupLock tuplock = { .lockmode = LockTupleExclusive, .waitpolicy = LockWaitSkip, .lockflags = TUPLE_LOCK_FLAG_FIND_LAST_VERSION, }; it = ts_scan_iterator_create(CHUNK_REWRITE, RowShareLock, CurrentMemoryContext); it.ctx.tuplock = &tuplock; ts_scanner_foreach(&it) { TupleInfo *ti = ts_scan_iterator_tuple_info(&it); bool isnull = false; bool entry_cleaned = false; Datum chunk_relid_dat = slot_getattr(ti->slot, Anum_chunk_rewrite_chunk_relid, &isnull); Assert(!isnull); Datum new_relid_dat = slot_getattr(ti->slot, Anum_chunk_rewrite_new_relid, &isnull); Assert(!isnull); if (ti->lockresult == TM_Ok) { Oid chunk_relid = DatumGetObjectId(chunk_relid_dat); Oid new_relid = DatumGetObjectId(new_relid_dat); /* * A concurrent merge might be in progress, so try to lock the "new" * relation and only delete it if the lock can be acquired * immediately. If a lock cannot be acquired, a merge is probably * ongoing and it might still complete successfully. */ if (ConditionalLockRelationOid(new_relid, AccessExclusiveLock)) { ObjectAddress new_objaddr = { .objectId = DatumGetObjectId(new_relid), .classId = RelationRelationId, }; /* * Check that the merge relation still exists and get its owner. */ HeapTuple tuple; Oid ownerid = InvalidOid; tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(chunk_relid)); if (HeapTupleIsValid(tuple)) ownerid = ((Form_pg_class) GETSTRUCT(tuple))->relowner; ReleaseSysCache(tuple); /* * Only clean an entry if the user has the privileges of the owner of the relation. * Also clean up if the relation doesn't exist anymore (relowner is InvalidOid). */ if (!OidIsValid(ownerid) || has_privs_of_role(userid, ownerid)) { /* * Check that the relation still exists. If it does, add to delete objects. * Otherwise release lock. */ if (SearchSysCacheExists1(RELOID, ObjectIdGetDatum(new_relid))) add_exact_object_address(&new_objaddr, objaddrs); else UnlockRelationOid(new_relid, AccessExclusiveLock); ItemPointer tid = ts_scanner_get_tuple_tid(ti); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_delete_tid_only(it.ctx.tablerel, tid); ts_catalog_restore_user(&sec_ctx); entry_cleaned = true; } } } if (entry_cleaned) { cleanup_count++; } else { Oid chunk_relid = DatumGetObjectId(chunk_relid_dat); elog(DEBUG1, "chunk merge in progress for \"%s\", skipping", get_rel_name(chunk_relid)); skipped_count++; } } ts_scan_iterator_close(&it); performMultipleDeletions(objaddrs, DROP_RESTRICT, PERFORM_DELETION_INTERNAL); elog(NOTICE, "cleaned up %u orphaned rewrite relations, skipped %u", cleanup_count, skipped_count); PG_RETURN_VOID(); } ================================================ FILE: src/ts_catalog/chunk_rewrite.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include "ts_catalog/catalog.h" extern TSDLLEXPORT void ts_chunk_rewrite_add(Oid chunk_relid, Oid new_relid); extern TSDLLEXPORT bool ts_chunk_rewrite_get_with_lock(Oid chunk_relid, Form_chunk_rewrite form, ItemPointer tid); extern TSDLLEXPORT void ts_chunk_rewrite_delete_by_tid(const ItemPointer tid); typedef enum ChunkRewriteDeleteResult { ChunkRewriteOngoing, ChunkRewriteEntryDeleted, ChunkRewriteEntryDeletedAndTableDropped, ChunkRewriteEntryDoesNotExist, } ChunkRewriteDeleteResult; extern TSDLLEXPORT ChunkRewriteDeleteResult ts_chunk_rewrite_delete(Oid chunk_relid, bool conditional); ================================================ FILE: src/ts_catalog/compression_chunk_size.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/htup_details.h> #include <executor/tuptable.h> #include "export.h" #include "scan_iterator.h" #include "scanner.h" #include "ts_catalog/catalog.h" #include "ts_catalog/compression_chunk_size.h" static void init_scan_by_uncompressed_chunk_id(ScanIterator *iterator, int32 uncompressed_chunk_id) { iterator->ctx.index = catalog_get_index(ts_catalog_get(), COMPRESSION_CHUNK_SIZE, COMPRESSION_CHUNK_SIZE_PKEY); ts_scan_iterator_scan_key_init(iterator, Anum_compression_chunk_size_pkey_chunk_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(uncompressed_chunk_id)); } TSDLLEXPORT int ts_compression_chunk_size_delete(int32 uncompressed_chunk_id) { ScanIterator iterator = ts_scan_iterator_create(COMPRESSION_CHUNK_SIZE, RowExclusiveLock, CurrentMemoryContext); int count = 0; init_scan_by_uncompressed_chunk_id(&iterator, uncompressed_chunk_id); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); ts_catalog_delete_tid_only(ti->scanrel, ts_scanner_get_tuple_tid(ti)); count++; } /* Make catalog changes visible */ if (count > 0) CommandCounterIncrement(); return count; } TSDLLEXPORT bool ts_compression_chunk_size_get(int32 chunk_id, Form_compression_chunk_size form) { ScanIterator iterator = ts_scan_iterator_create(COMPRESSION_CHUNK_SIZE, AccessExclusiveLock, CurrentMemoryContext); bool found = false; Assert(form != NULL); init_scan_by_uncompressed_chunk_id(&iterator, chunk_id); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); memcpy(form, GETSTRUCT(tuple), sizeof(*form)); found = true; Assert(form->chunk_id == chunk_id); if (should_free) heap_freetuple(tuple); break; } ts_scan_iterator_close(&iterator); return found; } TSDLLEXPORT bool ts_compression_chunk_size_update(int32 chunk_id, Form_compression_chunk_size form) { ScanIterator iterator = ts_scan_iterator_create(COMPRESSION_CHUNK_SIZE, RowExclusiveLock, CurrentMemoryContext); bool found = false; CatalogSecurityContext sec_ctx; Assert(form != NULL); init_scan_by_uncompressed_chunk_id(&iterator, chunk_id); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); HeapTuple copy = heap_copytuple(tuple); Form_compression_chunk_size tupform = (Form_compression_chunk_size) GETSTRUCT(copy); /* Don't update chunk IDs so copy from existing tuple */ form->chunk_id = tupform->chunk_id; form->compressed_chunk_id = tupform->compressed_chunk_id; memcpy(tupform, form, sizeof(FormData_compression_chunk_size)); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_update_tid_only(ti->scanrel, ts_scanner_get_tuple_tid(ti), copy); ts_catalog_restore_user(&sec_ctx); found = true; heap_freetuple(copy); if (should_free) heap_freetuple(tuple); break; } ts_scan_iterator_close(&iterator); return found; } ================================================ FILE: src/ts_catalog/compression_chunk_size.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <compat/compat.h> #include <postgres.h> #include <ts_catalog/catalog.h> extern TSDLLEXPORT int ts_compression_chunk_size_delete(int32 uncompressed_chunk_id); extern TSDLLEXPORT bool ts_compression_chunk_size_get(int32 chunk_id, Form_compression_chunk_size form); extern TSDLLEXPORT bool ts_compression_chunk_size_update(int32 chunk_id, Form_compression_chunk_size form); ================================================ FILE: src/ts_catalog/compression_settings.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_inherits.h> #include <parser/parse_func.h> #include <utils/builtins.h> #include "foreach_ptr.h" #include "jsonb_utils.h" #include "scan_iterator.h" #include "scanner.h" #include "ts_catalog/array_utils.h" #include "ts_catalog/catalog.h" #include "ts_catalog/compression_settings.h" #include <common/md5.h> #include <utils/palloc.h> TSDLLEXPORT const char *ts_sparse_index_type_names[] = { "bloom", "minmax" }; TSDLLEXPORT const char *ts_sparse_index_source_names[] = { "config", "default", "orderby" }; TSDLLEXPORT const char *ts_sparse_index_common_keys[] = { "type", "column", "source", NULL }; static ScanTupleResult compression_settings_tuple_update(TupleInfo *ti, void *data); static HeapTuple compression_settings_formdata_make_tuple(const FormData_compression_settings *fd, TupleDesc desc); static Bitmapset *resolve_columns_to_attnos(List *column_names, Oid relid); /* * Compare two compression settings for equality */ bool ts_compression_settings_equal(const CompressionSettings *left, const CompressionSettings *right) { return ts_array_equal(left->fd.segmentby, right->fd.segmentby) && ts_array_equal(left->fd.orderby, right->fd.orderby) && ts_array_equal(left->fd.orderby_desc, right->fd.orderby_desc) && ts_array_equal(left->fd.orderby_nullsfirst, right->fd.orderby_nullsfirst) && ts_jsonb_equal(left->fd.index, right->fd.index); } /* * Compare two compression settings for equality while ignoring default values. * * This essentially means that any NULL * values should be considered a match because they represent default * values which are determined at chunk level. * * This also means first argument needs to be the hypertable because chunks * cannot have implicit defaults. */ bool ts_compression_settings_equal_with_defaults(const CompressionSettings *ht, const CompressionSettings *chunk) { Assert(!OidIsValid(ht->fd.compress_relid)); return (ht->fd.segmentby == NULL || ts_array_equal(ht->fd.segmentby, chunk->fd.segmentby)) && (ht->fd.orderby == NULL || ts_array_equal(ht->fd.orderby, chunk->fd.orderby)) && (ht->fd.orderby_desc == NULL || ts_array_equal(ht->fd.orderby_desc, chunk->fd.orderby_desc)) && (ht->fd.orderby_nullsfirst == NULL || ts_array_equal(ht->fd.orderby_nullsfirst, chunk->fd.orderby_nullsfirst)) && (ht->fd.index == NULL || ts_jsonb_equal(ht->fd.index, chunk->fd.index)); } CompressionSettings * ts_compression_settings_materialize(const CompressionSettings *src, Oid relid, Oid compress_relid) { CompressionSettings *dst = ts_compression_settings_create(relid, compress_relid, src->fd.segmentby, src->fd.orderby, src->fd.orderby_desc, src->fd.orderby_nullsfirst, src->fd.index); return dst; } CompressionSettings * ts_compression_settings_create(Oid relid, Oid compress_relid, ArrayType *segmentby, ArrayType *orderby, ArrayType *orderby_desc, ArrayType *orderby_nullsfirst, Jsonb *sparse_index) { Catalog *catalog = ts_catalog_get(); CatalogSecurityContext sec_ctx; Relation rel; FormData_compression_settings fd; Assert(OidIsValid(relid)); /* * The default compression settings will always have orderby settings but the user may have * chosen to overwrite it. For both cases all 3 orderby arrays must either have the same number * of entries or be all NULL. */ Assert((orderby && orderby_desc && orderby_nullsfirst) || (!orderby && !orderby_desc && !orderby_nullsfirst)); fd.relid = relid; fd.compress_relid = compress_relid; fd.segmentby = segmentby; fd.orderby = orderby; fd.orderby_desc = orderby_desc; fd.orderby_nullsfirst = orderby_nullsfirst; fd.index = sparse_index; rel = table_open(catalog_get_table_id(catalog, COMPRESSION_SETTINGS), RowExclusiveLock); HeapTuple new_tuple = compression_settings_formdata_make_tuple(&fd, RelationGetDescr(rel)); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_insert(rel, new_tuple); ts_catalog_restore_user(&sec_ctx); heap_freetuple(new_tuple); table_close(rel, RowExclusiveLock); return ts_compression_settings_get(relid); } static void compression_settings_fill_from_tuple(CompressionSettings *settings, TupleInfo *ti) { FormData_compression_settings *fd = &settings->fd; Datum values[Natts_compression_settings]; bool nulls[Natts_compression_settings]; bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); heap_deform_tuple(tuple, ts_scanner_get_tupledesc(ti), values, nulls); MemoryContext old = MemoryContextSwitchTo(ti->mctx); fd->relid = DatumGetObjectId(values[AttrNumberGetAttrOffset(Anum_compression_settings_relid)]); if (nulls[AttrNumberGetAttrOffset(Anum_compression_settings_compress_relid)]) fd->compress_relid = InvalidOid; else fd->compress_relid = DatumGetObjectId( values[AttrNumberGetAttrOffset(Anum_compression_settings_compress_relid)]); if (nulls[AttrNumberGetAttrOffset(Anum_compression_settings_segmentby)]) fd->segmentby = NULL; else fd->segmentby = DatumGetArrayTypePCopy( values[AttrNumberGetAttrOffset(Anum_compression_settings_segmentby)]); if (nulls[AttrNumberGetAttrOffset(Anum_compression_settings_orderby)]) fd->orderby = NULL; else fd->orderby = DatumGetArrayTypePCopy( values[AttrNumberGetAttrOffset(Anum_compression_settings_orderby)]); if (nulls[AttrNumberGetAttrOffset(Anum_compression_settings_orderby_desc)]) fd->orderby_desc = NULL; else fd->orderby_desc = DatumGetArrayTypePCopy( values[AttrNumberGetAttrOffset(Anum_compression_settings_orderby_desc)]); if (nulls[AttrNumberGetAttrOffset(Anum_compression_settings_orderby_nullsfirst)]) fd->orderby_nullsfirst = NULL; else fd->orderby_nullsfirst = DatumGetArrayTypePCopy( values[AttrNumberGetAttrOffset(Anum_compression_settings_orderby_nullsfirst)]); if (nulls[AttrNumberGetAttrOffset(Anum_compression_settings_index)]) fd->index = NULL; else fd->index = DatumGetJsonbPCopy(values[AttrNumberGetAttrOffset(Anum_compression_settings_index)]); MemoryContextSwitchTo(old); if (should_free) heap_freetuple(tuple); } static void compression_settings_iterator_init(ScanIterator *iterator, Oid relid, bool by_compress_relid) { int indexid = by_compress_relid ? COMPRESSION_SETTINGS_COMPRESS_RELID_IDX : COMPRESSION_SETTINGS_PKEY; iterator->ctx.index = catalog_get_index(ts_catalog_get(), COMPRESSION_SETTINGS, indexid); ts_scan_iterator_scan_key_init(iterator, by_compress_relid ? Anum_compression_settings_compress_relid_idx_relid : Anum_compression_settings_pkey_relid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); } /* * Get compression settings for a relation. * * When 'by_compress_relid' is false, the 'relid' refers to the "main" * relation being compressed. When it is true the 'relid' refers to the * relation containing the associated compressed data. */ static CompressionSettings * compression_settings_get(Oid relid, bool by_compress_relid) { CompressionSettings *settings = NULL; ScanIterator iterator = ts_scan_iterator_create(COMPRESSION_SETTINGS, AccessShareLock, CurrentMemoryContext); compression_settings_iterator_init(&iterator, relid, by_compress_relid); ts_scanner_start_scan(&iterator.ctx); TupleInfo *ti = ts_scanner_next(&iterator.ctx); if (!ti) return NULL; settings = palloc0(sizeof(CompressionSettings)); compression_settings_fill_from_tuple(settings, ti); ts_scan_iterator_close(&iterator); return settings; } /* * Get the compression settings for the relation referred to by 'relid'. */ TSDLLEXPORT CompressionSettings * ts_compression_settings_get(Oid relid) { return compression_settings_get(relid, false); } /* * Get the compression settings for a relation given its associated compressed * relation. * * Ideally, settings should only be looked up by "primary key", i.e., the * non-compressed chunk's 'relid', and in that case this function wouldn't be * needed. It might be possible to remove this function in the future. */ TSDLLEXPORT CompressionSettings * ts_compression_settings_get_by_compress_relid(Oid relid) { CompressionSettings *settings = compression_settings_get(relid, true); Ensure(settings, "compression settings not found for %s", get_rel_name(relid)); return settings; } /* * Delete compression settings for a relation. * * When 'by_compress_relid' is false, the 'relid' refers to the "main" * relation being compressed. When it is true the 'relid' refers to the * relation containing the associated compressed data. */ static bool compression_settings_delete(Oid relid, bool by_compress_relid) { if (!OidIsValid(relid)) return false; int count = 0; ScanIterator iterator = ts_scan_iterator_create(COMPRESSION_SETTINGS, RowExclusiveLock, CurrentMemoryContext); compression_settings_iterator_init(&iterator, relid, by_compress_relid); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); count++; } return count > 0; } /* * Delete entries matching the non-compressed relation. */ TSDLLEXPORT bool ts_compression_settings_delete(Oid relid) { return compression_settings_delete(relid, false); } /* * Delete entries matching a compressed relation. */ TSDLLEXPORT bool ts_compression_settings_delete_by_compress_relid(Oid relid) { return compression_settings_delete(relid, true); } /* * Delete entries matching either the primary key (non-compressed relation) or * the secondary key (compressed relation). */ TSDLLEXPORT bool ts_compression_settings_delete_any(Oid relid) { if (!ts_compression_settings_delete(relid)) return ts_compression_settings_delete_by_compress_relid(relid); return true; } static void compression_settings_rename_column(CompressionSettings *settings, const char *old, const char *new) { Jsonb *replacejsonb = NULL; bool replaced = false; settings->fd.segmentby = ts_array_replace_text(settings->fd.segmentby, old, new); settings->fd.orderby = ts_array_replace_text(settings->fd.orderby, old, new); replacejsonb = settings->fd.index; if (replacejsonb) { SparseIndexSettings *parsed_settings = ts_convert_to_sparse_index_settings(replacejsonb); if (parsed_settings) { foreach_ptr(SparseIndexSettingsObject, obj, parsed_settings->objects) { Assert(obj != NULL); if (!obj) continue; foreach_ptr(SparseIndexSettingsPair, pair, obj->pairs) { Assert(pair != NULL); if (!pair) continue; ListCell *value_cell = NULL; foreach (value_cell, pair->values) { const char *value = (const char *) lfirst(value_cell); Assert(value != NULL); if (!value) continue; if (strcmp(value, old) == 0) { value_cell->ptr_value = ts_sparse_index_settings_pstrdup(parsed_settings, new); replaced = true; } } } } if (replaced) { replacejsonb = ts_convert_from_sparse_index_settings(parsed_settings); } ts_free_sparse_index_settings(parsed_settings); } } settings->fd.index = replaced ? replacejsonb : settings->fd.index; ts_compression_settings_update(settings); } TSDLLEXPORT void ts_compression_settings_rename_column_cascade(Oid parent_relid, const char *old, const char *new) { CompressionSettings *settings = ts_compression_settings_get(parent_relid); if (settings) compression_settings_rename_column(settings, old, new); List *children = find_inheritance_children(parent_relid, NoLock); ListCell *lc; foreach (lc, children) { Oid relid = lfirst_oid(lc); settings = ts_compression_settings_get(relid); if (settings) compression_settings_rename_column(settings, old, new); } } TSDLLEXPORT int ts_compression_settings_update(CompressionSettings *settings) { Catalog *catalog = ts_catalog_get(); FormData_compression_settings *fd = &settings->fd; ScanKeyData scankey[1]; if (settings->fd.orderby && (settings->fd.segmentby || settings->fd.index)) { Datum datum; bool isnull; ArrayIterator it = array_create_iterator(settings->fd.orderby, 0, NULL); while (array_iterate(it, &datum, &isnull)) { if (settings->fd.segmentby && ts_array_is_member(settings->fd.segmentby, TextDatumGetCString(datum))) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("cannot use column \"%s\" for both ordering and segmenting", TextDatumGetCString(datum)), errhint("Use separate columns for the timescaledb.compress_orderby and" " timescaledb.compress_segmentby options."))); } if (settings->fd.index && ts_contains_sparse_index_config(settings, TextDatumGetCString(datum), ts_sparse_index_type_names [_SparseIndexTypeEnumBloom], /* skip_column_arrays = */ true)) { /* disallow single column bloom index on orderby columns, composite bloom is * allowed, that is why we set 'skip_column_arrays' to true */ ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("the orderby column \"%s\" cannot have a bloom sparse index ", TextDatumGetCString(datum)), errdetail("For orderby columns, a minmax sparse index is added " "automatically and cannot have bloom sparse index."))); } } } if (settings->fd.index && settings->fd.segmentby) { Datum datum; bool isnull; ArrayIterator it = array_create_iterator(settings->fd.segmentby, 0, NULL); while (array_iterate(it, &datum, &isnull)) { for (int i = 0; i < _SparseIndexTypeEnumMax; i++) { if (ts_contains_sparse_index_config(settings, TextDatumGetCString(datum), ts_sparse_index_type_names[i], /* skip_column_arrays = */ false)) { /* segmentby columns cannot have sparse indexes of any type, including composite * bloom that is why we set 'skip_column_arrays' to false, which will look * inside column name arrays, so composite bloom filters are checked too */ ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("the segmentby column \"%s\" can not have sparse " "indexes", TextDatumGetCString(datum)))); } } } } /* * The default compression settings will always have orderby settings but the user may have * chosen to overwrite it. For both cases all 3 orderby arrays must either have the same number * of entries or be all NULL. */ Assert( (settings->fd.orderby && settings->fd.orderby_desc && settings->fd.orderby_nullsfirst) || (!settings->fd.orderby && !settings->fd.orderby_desc && !settings->fd.orderby_nullsfirst)); ScanKeyInit(&scankey[0], Anum_compression_settings_pkey_relid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(fd->relid)); ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, COMPRESSION_SETTINGS), .index = catalog_get_index(catalog, COMPRESSION_SETTINGS, COMPRESSION_SETTINGS_PKEY), .nkeys = 1, .scankey = scankey, .data = settings, .tuple_found = compression_settings_tuple_update, .lockmode = RowExclusiveLock, .scandirection = ForwardScanDirection, }; return ts_scanner_scan(&scanctx); } static HeapTuple compression_settings_formdata_make_tuple(const FormData_compression_settings *fd, TupleDesc desc) { Datum values[Natts_compression_settings] = { 0 }; bool nulls[Natts_compression_settings] = { false }; values[AttrNumberGetAttrOffset(Anum_compression_settings_relid)] = ObjectIdGetDatum(fd->relid); if (OidIsValid(fd->compress_relid)) values[AttrNumberGetAttrOffset(Anum_compression_settings_compress_relid)] = ObjectIdGetDatum(fd->compress_relid); else nulls[AttrNumberGetAttrOffset(Anum_compression_settings_compress_relid)] = true; if (fd->segmentby) values[AttrNumberGetAttrOffset(Anum_compression_settings_segmentby)] = PointerGetDatum(fd->segmentby); else nulls[AttrNumberGetAttrOffset(Anum_compression_settings_segmentby)] = true; if (fd->orderby) values[AttrNumberGetAttrOffset(Anum_compression_settings_orderby)] = PointerGetDatum(fd->orderby); else nulls[AttrNumberGetAttrOffset(Anum_compression_settings_orderby)] = true; if (fd->orderby_desc) values[AttrNumberGetAttrOffset(Anum_compression_settings_orderby_desc)] = PointerGetDatum(fd->orderby_desc); else nulls[AttrNumberGetAttrOffset(Anum_compression_settings_orderby_desc)] = true; if (fd->orderby_nullsfirst) values[AttrNumberGetAttrOffset(Anum_compression_settings_orderby_nullsfirst)] = PointerGetDatum(fd->orderby_nullsfirst); else nulls[AttrNumberGetAttrOffset(Anum_compression_settings_orderby_nullsfirst)] = true; if (fd->index) values[AttrNumberGetAttrOffset(Anum_compression_settings_index)] = JsonbPGetDatum(fd->index); else nulls[AttrNumberGetAttrOffset(Anum_compression_settings_index)] = true; return heap_form_tuple(desc, values, nulls); } static ScanTupleResult compression_settings_tuple_update(TupleInfo *ti, void *data) { CompressionSettings *settings = data; HeapTuple new_tuple; CatalogSecurityContext sec_ctx; new_tuple = compression_settings_formdata_make_tuple(&settings->fd, ts_scanner_get_tupledesc(ti)); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_update_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti), new_tuple); ts_catalog_restore_user(&sec_ctx); heap_freetuple(new_tuple); return SCAN_DONE; } void ts_convert_sparse_index_config_to_jsonb(JsonbParseState *parse_state, SparseIndexConfigBase *config) { MinmaxIndexColumnConfig *minmax_config = NULL; BloomFilterConfig *bloom_config = NULL; pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); ts_jsonb_add_str(parse_state, ts_sparse_index_common_keys[SparseIndexKeyType], ts_sparse_index_type_names[config->type]); /* type */ switch (config->type) { case _SparseIndexTypeEnumMinmax: minmax_config = (MinmaxIndexColumnConfig *) config; ts_jsonb_add_str(parse_state, ts_sparse_index_common_keys[SparseIndexKeyCol], minmax_config->col); /* column */ break; case _SparseIndexTypeEnumBloom: bloom_config = (BloomFilterConfig *) config; if (bloom_config->num_columns > 1) { /* add the column names as an array */ const char *column_names[MAX_BLOOM_FILTER_COLUMNS] = { 0 }; for (int i = 0; i < bloom_config->num_columns; i++) { column_names[i] = bloom_config->columns[i].name; } ts_jsonb_add_str_array(parse_state, ts_sparse_index_common_keys[SparseIndexKeyCol], column_names, bloom_config->num_columns); } else { ts_jsonb_add_str(parse_state, ts_sparse_index_common_keys[SparseIndexKeyCol], bloom_config->columns[0].name); /* column */ } break; default: elog(ERROR, "invalid sparse index type: %d", config->type); }; ts_jsonb_add_str(parse_state, ts_sparse_index_common_keys[SparseIndexKeySource], ts_sparse_index_source_names[config->source]); /* source */ pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); } bool ts_contains_sparse_index_config(CompressionSettings *settings, const char *attname, const char *sparse_index_type, bool skip_column_arrays) { bool result = false; if (settings == NULL || settings->fd.index == NULL || attname == NULL) return false; SparseIndexSettings *parsed = ts_convert_to_sparse_index_settings(settings->fd.index); if (parsed == NULL) { return false; } foreach_ptr(SparseIndexSettingsObject, obj, parsed->objects) { const char *index_type = NULL; bool attname_found = false; foreach_ptr(SparseIndexSettingsPair, pair, obj->pairs) { if (strcmp(pair->key, ts_sparse_index_common_keys[SparseIndexKeyCol]) == 0) { if (skip_column_arrays && list_length(pair->values) > 1) { continue; } foreach_ptr(const char, value, pair->values) { if (strcmp(value, attname) == 0) { attname_found = true; break; } } } if (strcmp(pair->key, ts_sparse_index_common_keys[SparseIndexKeyType]) == 0) { index_type = (const char *) lfirst(list_head(pair->values)); if (strcmp(index_type, sparse_index_type) != 0) { break; } else if (attname_found) { result = true; break; } } if (attname_found && index_type != NULL) { result = strcmp(index_type, sparse_index_type) == 0; break; } } } ts_free_sparse_index_settings(parsed); return result; } /* adds orderby sparse index settings into fd.index */ Jsonb * ts_add_orderby_sparse_index(CompressionSettings *settings) { Datum datum; bool isnull; JsonbParseState *parse_state = NULL; JsonbIterator *it_json; JsonbValue v; JsonbIteratorToken r; Jsonb *sparse_index = settings->fd.index; /* nothing to do if no orderby columns */ if (!settings->fd.orderby) { return sparse_index; } pushJsonbValue(&parse_state, WJB_BEGIN_ARRAY, NULL); /* add existing sparse index */ if (settings->fd.index) { it_json = JsonbIteratorInit(&sparse_index->root); JsonbIteratorNext(&it_json, &v, false); /* WJB_BEGIN_ARRAY */ while ((r = JsonbIteratorNext(&it_json, &v, true)) != WJB_END_ARRAY) { Assert(r == WJB_ELEM); pushJsonbValue(&parse_state, r, &v); } } /* add orderby sparse settings */ ArrayIterator it = array_create_iterator(settings->fd.orderby, 0, NULL); while (array_iterate(it, &datum, &isnull)) { /* * check if sparse index for column already exists * Validation is done by ts_compression_settings_update */ if (settings->fd.index && ts_jsonb_has_key_value_str_field(settings->fd.index, ts_sparse_index_common_keys[SparseIndexKeyCol], TextDatumGetCString(datum))) { continue; } MinmaxIndexColumnConfig config; config.base.type = _SparseIndexTypeEnumMinmax; config.col = TextDatumGetCString(datum); config.base.source = _SparseIndexSourceEnumOrderby; ts_convert_sparse_index_config_to_jsonb(parse_state, (SparseIndexConfigBase *) &config); } return JsonbValueToJsonb(pushJsonbValue(&parse_state, WJB_END_ARRAY, NULL)); } /* removed orderby sparse index settings from fd.index */ Jsonb * ts_remove_orderby_sparse_index(CompressionSettings *settings) { JsonbParseState *parse_state = NULL; JsonbContainer *container; JsonbIterator *it_json; JsonbIteratorToken r; JsonbValue v; Jsonb *sparse_index = settings->fd.index; bool removed = false; bool has_object = false; const char *key_name_source = ts_sparse_index_common_keys[SparseIndexKeySource]; const char *value_name_orderby = ts_sparse_index_source_names[_SparseIndexSourceEnumOrderby]; /* nothing to do if no orderby columns */ if (!settings->fd.orderby || !sparse_index) { return sparse_index; } pushJsonbValue(&parse_state, WJB_BEGIN_ARRAY, NULL); it_json = JsonbIteratorInit(&sparse_index->root); JsonbIteratorNext(&it_json, &v, false); /* WJB_BEGIN_ARRAY */ while ((r = JsonbIteratorNext(&it_json, &v, true)) != WJB_END_ARRAY) { Ensure(r == WJB_ELEM && v.type == jbvBinary && JsonContainerIsObject(v.val.binary.data), "sparse index format must be an array of objects"); container = v.val.binary.data; JsonbValue value; getKeyJsonValueFromContainer(container, key_name_source, strlen(key_name_source), &value); if (value.type == jbvString && ((int) strlen(value_name_orderby) == value.val.string.len) && strncmp(value_name_orderby, value.val.string.val, value.val.string.len) == 0) { removed = true; continue; } has_object = true; pushJsonbValue(&parse_state, r, &v); } /* this is a possible edge case, log it just in case */ if (!removed) elog(LOG, "orderby settings existed, but no orderby sparse index was removed"); return has_object ? JsonbValueToJsonb(pushJsonbValue(&parse_state, WJB_END_ARRAY, NULL)) : NULL; } int ts_qsort_attrnumber_cmp(const void *a, const void *b) { SparseIndexColumn *col_a = (SparseIndexColumn *) a; SparseIndexColumn *col_b = (SparseIndexColumn *) b; return ((int) (col_a->attnum)) - ((int) (col_b->attnum)); } SparseIndexSettings * ts_convert_to_sparse_index_settings(Jsonb *jsonb) { enum ParseState { PARSE_STATE_INIT, PARSE_STATE_KEY, PARSE_STATE_VALUE, PARSE_STATE_ARRAY_ENTRIES }; MemoryContext tmp_context, new_context; enum ParseState state = PARSE_STATE_INIT; JsonbValue jsonb_value; JsonbIterator *it; JsonbIteratorToken r; int num_arrays = 0; Assert(jsonb != NULL); if (jsonb == NULL) return NULL; if (JB_ROOT_IS_SCALAR(jsonb)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot convert scalar to SparseIndexSettings"))); if (JB_ROOT_COUNT(jsonb) == 0) return NULL; it = JsonbIteratorInit(&jsonb->root); new_context = AllocSetContextCreate(CurrentMemoryContext, "SparseIndexSettings", ALLOCSET_DEFAULT_MINSIZE, ALLOCSET_DEFAULT_INITSIZE, ALLOCSET_DEFAULT_MAXSIZE); SparseIndexSettings *parsed_settings = MemoryContextAllocZero(new_context, sizeof(SparseIndexSettings)); parsed_settings->objects = NIL; parsed_settings->context = new_context; while ((r = JsonbIteratorNext(&it, &jsonb_value, false)) != WJB_DONE) { switch (state) { case PARSE_STATE_INIT: if (r == WJB_END_OBJECT || r == WJB_END_ARRAY) { /* Ignore*/ } else if (r == WJB_BEGIN_OBJECT) { SparseIndexSettingsObject *current_object = NULL; /* If the previous object has no pairs, reuse its space for the new object */ if (list_length(parsed_settings->objects) > 0) { current_object = llast(parsed_settings->objects); if (list_length(current_object->pairs) > 0) { /* We can't reuse the previous object, so we need to create a new one */ current_object = NULL; } } if (current_object == NULL) { /* Create a new object */ tmp_context = MemoryContextSwitchTo(parsed_settings->context); current_object = palloc0(sizeof(SparseIndexSettingsObject)); parsed_settings->objects = lappend(parsed_settings->objects, current_object); MemoryContextSwitchTo(tmp_context); } state = PARSE_STATE_KEY; } else if (r == WJB_KEY) { Ensure(jsonb_value.type == jbvString, "Jsonb value is of type \"%s\", but expected of type string, in state " "INIT", JsonbTypeName(&jsonb_value)); Ensure(list_length(parsed_settings->objects) > 0, "Jsonb value has a key, but no object has been started, in state INIT"); SparseIndexSettingsObject *current_object = llast(parsed_settings->objects); Assert(current_object != NULL); tmp_context = MemoryContextSwitchTo(parsed_settings->context); char *tmp_str = pnstrdup(jsonb_value.val.string.val, jsonb_value.val.string.len); SparseIndexSettingsPair *current_pair = palloc0(sizeof(SparseIndexSettingsPair)); current_object->pairs = lappend(current_object->pairs, current_pair); current_pair->key = tmp_str; current_pair->values = NIL; MemoryContextSwitchTo(tmp_context); state = PARSE_STATE_VALUE; } else if (r == WJB_BEGIN_ARRAY) { num_arrays++; /* We can ignore one begin array, but more than one is not allowed as we don't * support nested arrays */ if (num_arrays > 1) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Jsonb value is of type \"%s\", but expected of type begin " "object or end object, in state INIT", JsonbTypeName(&jsonb_value)))); } else { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Jsonb value is of type \"%s\", but expected of type begin " "object or end object, in state INIT", JsonbTypeName(&jsonb_value)))); } break; case PARSE_STATE_KEY: if (r == WJB_KEY) { Ensure(jsonb_value.type == jbvString, "Jsonb value is of type \"%s\", but expected of type string, in state " "KEY", JsonbTypeName(&jsonb_value)); SparseIndexSettingsObject *current_object = llast(parsed_settings->objects); Assert(current_object != NULL); tmp_context = MemoryContextSwitchTo(parsed_settings->context); char *tmp_str = pnstrdup(jsonb_value.val.string.val, jsonb_value.val.string.len); SparseIndexSettingsPair *current_pair = palloc0(sizeof(SparseIndexSettingsPair)); current_object->pairs = lappend(current_object->pairs, current_pair); current_pair->key = tmp_str; current_pair->values = NIL; MemoryContextSwitchTo(tmp_context); state = PARSE_STATE_VALUE; } else if (r == WJB_END_OBJECT) { state = PARSE_STATE_INIT; } else { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Jsonb value is of type \"%s\", but expected of type key or " "end object, in state KEY", JsonbTypeName(&jsonb_value)))); } break; case PARSE_STATE_VALUE: if (r == WJB_VALUE) { SparseIndexSettingsObject *current_object = llast(parsed_settings->objects); Assert(current_object != NULL); Ensure(jsonb_value.type == jbvString, "Jsonb value is of type \"%s\", but expected of type string, in state " "VALUE", JsonbTypeName(&jsonb_value)); tmp_context = MemoryContextSwitchTo(parsed_settings->context); char *tmp_str = pnstrdup(jsonb_value.val.string.val, jsonb_value.val.string.len); SparseIndexSettingsPair *current_pair = llast(current_object->pairs); /* The pair should have been created in the KEY state, but no values should have * been added yet */ Assert(current_pair != NULL); Assert(current_pair->values == NIL); current_pair->values = lappend(current_pair->values, tmp_str); MemoryContextSwitchTo(tmp_context); state = PARSE_STATE_KEY; } else if (r == WJB_BEGIN_ARRAY) { #ifdef USE_ASSERT_CHECKING SparseIndexSettingsObject *current_object = llast(parsed_settings->objects); Assert(current_object != NULL); /* The pair list should have been created in the key state */ Assert(list_length(current_object->pairs) > 0); SparseIndexSettingsPair *current_pair = llast(current_object->pairs); Assert(current_pair != NULL); /* The values list should be empty when we arrive to the array start */ Assert(current_pair->values == NIL); #endif state = PARSE_STATE_ARRAY_ENTRIES; } else { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Jsonb value is of type \"%s\", but expected of type value or " "array, in state VALUE", JsonbTypeName(&jsonb_value)))); } break; case PARSE_STATE_ARRAY_ENTRIES: if (r == WJB_ELEM) { SparseIndexSettingsObject *current_object = llast(parsed_settings->objects); Assert(current_object != NULL); SparseIndexSettingsPair *current_pair = llast(current_object->pairs); Assert(current_pair != NULL); Ensure(jsonb_value.type == jbvString, "Jsonb value is of type \"%s\", but expected of type string, in state " "ARRAY_ENTRIES", JsonbTypeName(&jsonb_value)); tmp_context = MemoryContextSwitchTo(parsed_settings->context); char *tmp_str = pnstrdup(jsonb_value.val.string.val, jsonb_value.val.string.len); current_pair->values = lappend(current_pair->values, tmp_str); MemoryContextSwitchTo(tmp_context); /* state remains ARRAY_ENTRIES */ } else if (r == WJB_END_ARRAY) { state = PARSE_STATE_INIT; } else { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Jsonb value is of type \"%s\", but expected of type elem or " "end array, in state ARRAY_ENTRIES", JsonbTypeName(&jsonb_value)))); } break; } } /* If the last object has no pairs, remove it */ if (list_length(parsed_settings->objects) > 0) { SparseIndexSettingsObject *current_object = llast(parsed_settings->objects); Assert(current_object != NULL); if (list_length(current_object->pairs) == 0) { parsed_settings->objects = list_delete_last(parsed_settings->objects); } } /* If there are no objects, free the parsed settings */ if (list_length(parsed_settings->objects) == 0) { ts_free_sparse_index_settings(parsed_settings); parsed_settings = NULL; } return parsed_settings; } Jsonb * ts_convert_from_sparse_index_settings(SparseIndexSettings *settings) { JsonbParseState *parse_state = NULL; Assert(settings != NULL); if (settings == NULL) return NULL; if (list_length(settings->objects) == 0) return NULL; pushJsonbValue(&parse_state, WJB_BEGIN_ARRAY, NULL); foreach_ptr(SparseIndexSettingsObject, obj, settings->objects) { pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); Assert(list_length(obj->pairs) > 0); foreach_ptr(SparseIndexSettingsPair, pair, obj->pairs) { Assert(pair->key != NULL); Assert(list_length(pair->values) > 0); JsonbValue key = { .type = jbvString, .val = { .string = { .val = pair->key, .len = strlen(pair->key) } } }; pushJsonbValue(&parse_state, WJB_KEY, &key); if (list_length(pair->values) == 1) { const char *value = (const char *) lfirst(list_head(pair->values)); Assert(value != NULL); int len = strlen(value); Assert(len > 0); JsonbValue value_jsonb = { .type = jbvString, .val = { .string = { .val = pstrdup(value), .len = len } } }; pushJsonbValue(&parse_state, WJB_VALUE, &value_jsonb); } else { pushJsonbValue(&parse_state, WJB_BEGIN_ARRAY, NULL); foreach_ptr(const char, value, pair->values) { Assert(value != NULL); int len = strlen(value); Assert(len > 0); JsonbValue value_jsonb = { .type = jbvString, .val = { .string = { .val = pstrdup(value), .len = len } } }; pushJsonbValue(&parse_state, WJB_ELEM, &value_jsonb); } pushJsonbValue(&parse_state, WJB_END_ARRAY, NULL); } } pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); } return JsonbValueToJsonb(pushJsonbValue(&parse_state, WJB_END_ARRAY, NULL)); } void ts_free_sparse_index_settings(SparseIndexSettings *settings) { if (settings == NULL) return; if (settings->context != NULL) MemoryContextDelete(settings->context); } const char * ts_sparse_index_settings_to_cstring(const SparseIndexSettings *settings) { StringInfoData buf; int i = 0, j = 0, k = 0; initStringInfo(&buf); appendStringInfo(&buf, "["); foreach_ptr(SparseIndexSettingsObject, obj, settings->objects) { if (i > 0) appendStringInfo(&buf, ", "); appendStringInfo(&buf, "{"); j = 0; foreach_ptr(SparseIndexSettingsPair, pair, obj->pairs) { if (j > 0) appendStringInfo(&buf, ", "); escape_json(&buf, pair->key); appendStringInfo(&buf, ": "); if (list_length(pair->values) == 1) { escape_json(&buf, (const char *) lfirst(list_head(pair->values))); } else { k = 0; appendStringInfo(&buf, "["); foreach_ptr(const char, value, pair->values) { if (k > 0) appendStringInfo(&buf, ", "); escape_json(&buf, value); k++; } appendStringInfo(&buf, "]"); } j++; } appendStringInfo(&buf, "}"); i++; } appendStringInfo(&buf, "]"); return buf.data; } char * ts_sparse_index_settings_pstrdup(SparseIndexSettings *settings, const char *str) { Assert(settings != NULL); Assert(str != NULL); Assert(settings->context != NULL); MemoryContext old_context = MemoryContextSwitchTo(settings->context); char *new_str = pstrdup(str); MemoryContextSwitchTo(old_context); return new_str; } /* returns a list of PerColumnCompressionSettings objects */ List * ts_get_per_column_compression_settings(const SparseIndexSettings *settings) { if (settings == NULL) return NIL; List *result_settings = NIL; int obj_id = 0; foreach_ptr(SparseIndexSettingsObject, obj, settings->objects) { const char *index_type = NULL; SparseIndexSettingsPair *column_names_pair = NULL; Assert(obj != NULL); foreach_ptr(SparseIndexSettingsPair, pair, obj->pairs) { if (strcmp(pair->key, ts_sparse_index_common_keys[SparseIndexKeyType]) == 0) { Assert(list_length(pair->values) > 0); index_type = (const char *) lfirst(list_head(pair->values)); } else if (strcmp(pair->key, ts_sparse_index_common_keys[SparseIndexKeyCol]) == 0) { column_names_pair = pair; } } /* we may have an empty object, that is the default */ if (index_type != NULL && column_names_pair != NULL) { /* find the column names and iterate over them */ int num_columns = list_length(column_names_pair->values); foreach_ptr(const char, column_name, column_names_pair->values) { Assert(column_name != NULL); PerColumnCompressionSettings *per_column_setting = NULL; /* check if the column name is already in the list */ foreach_ptr(PerColumnCompressionSettings, tmp, result_settings) { if (strcmp(tmp->column_name, column_name) == 0) { per_column_setting = tmp; break; } } /* if no object is found, create a new one */ if (per_column_setting == NULL) { per_column_setting = palloc0(sizeof(PerColumnCompressionSettings)); per_column_setting->column_name = column_name; per_column_setting->minmax_obj_id = -1; per_column_setting->single_bloom_obj_id = -1; per_column_setting->composite_bloom_index_obj_ids = NULL; result_settings = lappend(result_settings, per_column_setting); } if (strcmp(index_type, ts_sparse_index_type_names[_SparseIndexTypeEnumMinmax]) == 0) { Assert(num_columns == 1); per_column_setting->minmax_obj_id = obj_id; } else if (strcmp(index_type, ts_sparse_index_type_names[_SparseIndexTypeEnumBloom]) == 0) { if (num_columns == 1) { per_column_setting->single_bloom_obj_id = obj_id; } else if (num_columns > 1) { if (per_column_setting->composite_bloom_index_obj_ids == NULL) { per_column_setting->composite_bloom_index_obj_ids = bms_make_singleton(obj_id); } else { per_column_setting->composite_bloom_index_obj_ids = bms_add_member(per_column_setting->composite_bloom_index_obj_ids, obj_id); } } } } } obj_id++; } return result_settings; } PerColumnCompressionSettings * ts_get_per_column_compression_settings_by_column_name(List *per_column_settings, const char *column_name) { Assert(column_name != NULL); ListCell *per_column_setting_cell = NULL; foreach (per_column_setting_cell, per_column_settings) { PerColumnCompressionSettings *tmp = (PerColumnCompressionSettings *) lfirst(per_column_setting_cell); if (strcmp(tmp->column_name, column_name) == 0) { return tmp; } } return NULL; } List * ts_get_column_names_from_parsed_object(SparseIndexSettingsObject *obj) { Assert(obj != NULL); foreach_ptr(SparseIndexSettingsPair, pair, obj->pairs) { if (strcmp(pair->key, ts_sparse_index_common_keys[SparseIndexKeyCol] /* "column" */) == 0) { return pair->values; } } return NULL; } static Bitmapset * resolve_columns_to_attnos(List *column_names, Oid relid) { Assert(column_names != NULL); Assert(OidIsValid(relid)); Bitmapset *result = NULL; ListCell *name_cell = NULL; foreach (name_cell, column_names) { const char *name = (const char *) lfirst(name_cell); AttrNumber attno = get_attnum(relid, name); if (AttributeNumberIsValid(attno)) { result = bms_add_member(result, attno); } else { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("column \"%s\" of relation \"%ld\" does not exist", name, (long) relid))); } } return result; } /* * Resolve the column names in the parsed settings to attribute numbers for the given relation * and return a list of bitmapsets corresponding to each object in the parsed settings. */ TsBmsList ts_resolve_columns_to_attnos_from_parsed_settings(SparseIndexSettings *settings, Oid relid) { Assert(settings != NULL); Assert(OidIsValid(relid)); TsBmsList result = NIL; foreach_ptr(SparseIndexSettingsObject, obj, settings->objects) { List *column_names = ts_get_column_names_from_parsed_object(obj); Bitmapset *attnos = resolve_columns_to_attnos(column_names, relid); result = lappend(result, attnos); } return result; } List * ts_get_values_by_key_from_parsed_object(SparseIndexSettingsObject *obj, const char *key) { Assert(obj != NULL); Assert(key != NULL); List *result = NIL; foreach_ptr(SparseIndexSettingsPair, pair, obj->pairs) { if (strcmp(pair->key, key) == 0) { result = pair->values; return result; } } return NIL; } ================================================ FILE: src/ts_catalog/compression_settings.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <catalog/pg_type.h> #include "bmslist_utils.h" #include "ts_catalog/catalog.h" typedef struct CompressionSettings { FormData_compression_settings fd; } CompressionSettings; typedef enum SparseIndexTypeEnum { _SparseIndexTypeEnumBloom = 0, _SparseIndexTypeEnumMinmax, _SparseIndexTypeEnumMax } SparseIndexTypeEnum; typedef enum SparseIndexSourceEnum { _SparseIndexSourceEnumConfig = 0, _SparseIndexSourceEnumDefault, _SparseIndexSourceEnumOrderby, _SparseIndexSourceEnumMax } SparseIndexSourceEnum; typedef enum SparseIndexConfigKeys { SparseIndexKeyType = 0, SparseIndexKeyCol, SparseIndexKeySource, SparseIndexKeyCustom } SparseIndexConfigKeys; extern TSDLLEXPORT const char *ts_sparse_index_type_names[]; extern TSDLLEXPORT const char *ts_sparse_index_source_names[]; extern TSDLLEXPORT const char *ts_sparse_index_common_keys[]; typedef struct SparseIndexConfigBase { SparseIndexTypeEnum type; SparseIndexSourceEnum source; } SparseIndexConfigBase; typedef struct MinmaxIndexColumnConfig { SparseIndexConfigBase base; const char *col; } MinmaxIndexColumnConfig; typedef struct SparseIndexColumn { /* composite bloom indexes will have multiple SparseIndexColumn entries and * they will be sorted by the attribute number */ AttrNumber attnum; const char *name; Oid type; } SparseIndexColumn; #define MAX_BLOOM_FILTER_COLUMNS 8 typedef struct BloomFilterConfig { SparseIndexConfigBase base; int num_columns; SparseIndexColumn *columns; } BloomFilterConfig; /* * The SparseIndexSettings structure is used to parse the compression * settings from the JSONB structure. * With this we can turn the stored JSONB into this structure, modify it and * turn it back into JSONB and we can avoid the messy and error prone JSONB * manipulation. * * The structure is a list of objects, each object is a list of pairs, each * pair is a key and a list of values. This allows us to store and manipulate * JSONB structures like this: * * [ * {"type": "bloom", "column": "big1", "source": "config"}, * {"type": "bloom", "column": ["value", "big1", "big2"], "source": "config"}, * {"type": "bloom", "column": ["o", "big2"], "source": "config"}, * {"type": "minmax", "column": "ts", "source": "orderby"} * ] * * Notice that the "column" key can have a string or an array of strings as value. */ typedef struct SparseIndexSettingsPair { char *key; List *values; /* List of strings */ } SparseIndexSettingsPair; typedef struct SparseIndexSettingsObject { List *pairs; /* List of SparseIndexSettingsPair */ } SparseIndexSettingsObject; typedef struct SparseIndexSettings { MemoryContext context; List *objects; /* List of SparseIndexSettingsObject */ } SparseIndexSettings; typedef struct PerColumnCompressionSettings { const char *column_name; /* the index of the minmax index object that the column participates in, -1 if not present */ int minmax_obj_id; /* the index of the single bloom index object that the column participates in, -1 if not present */ int single_bloom_obj_id; /* the object ids of the composite bloom index objects that the column participates in */ Bitmapset *composite_bloom_index_obj_ids; } PerColumnCompressionSettings; TSDLLEXPORT int ts_qsort_attrnumber_cmp(const void *a, const void *b); TSDLLEXPORT CompressionSettings * ts_compression_settings_create(Oid relid, Oid compress_relid, ArrayType *segmentby, ArrayType *orderby, ArrayType *orderby_desc, ArrayType *orderby_nullsfirst, Jsonb *sparse_index); TSDLLEXPORT CompressionSettings *ts_compression_settings_get(Oid relid); TSDLLEXPORT CompressionSettings *ts_compression_settings_get_by_compress_relid(Oid relid); TSDLLEXPORT CompressionSettings *ts_compression_settings_materialize(const CompressionSettings *src, Oid relid, Oid compress_relid); TSDLLEXPORT bool ts_compression_settings_delete(Oid relid); TSDLLEXPORT bool ts_compression_settings_delete_by_compress_relid(Oid relid); TSDLLEXPORT bool ts_compression_settings_delete_any(Oid relid); TSDLLEXPORT bool ts_compression_settings_equal(const CompressionSettings *left, const CompressionSettings *right); TSDLLEXPORT bool ts_compression_settings_equal_with_defaults(const CompressionSettings *ht, const CompressionSettings *chunk); TSDLLEXPORT int ts_compression_settings_update(CompressionSettings *settings); TSDLLEXPORT void ts_compression_settings_rename_column_cascade(Oid parent_relid, const char *old, const char *new); TSDLLEXPORT void ts_convert_sparse_index_config_to_jsonb(JsonbParseState *parse_state, SparseIndexConfigBase *config); TSDLLEXPORT bool ts_contains_sparse_index_config(CompressionSettings *settings, const char *attname, const char *sparse_index_type, bool skip_column_arrays); TSDLLEXPORT Jsonb *ts_add_orderby_sparse_index(CompressionSettings *settings); TSDLLEXPORT Jsonb *ts_remove_orderby_sparse_index(CompressionSettings *settings); extern TSDLLEXPORT SparseIndexSettings *ts_convert_to_sparse_index_settings(Jsonb *jsonb); extern TSDLLEXPORT Jsonb *ts_convert_from_sparse_index_settings(SparseIndexSettings *settings); extern TSDLLEXPORT void ts_free_sparse_index_settings(SparseIndexSettings *settings); extern TSDLLEXPORT const char * ts_sparse_index_settings_to_cstring(const SparseIndexSettings *settings); extern TSDLLEXPORT char *ts_sparse_index_settings_pstrdup(SparseIndexSettings *settings, const char *str); extern TSDLLEXPORT List * ts_get_per_column_compression_settings(const SparseIndexSettings *settings); extern TSDLLEXPORT PerColumnCompressionSettings * ts_get_per_column_compression_settings_by_column_name(List *per_column_settings, const char *column_name); extern TSDLLEXPORT List *ts_get_column_names_from_parsed_object(SparseIndexSettingsObject *obj); extern TSDLLEXPORT TsBmsList ts_resolve_columns_to_attnos_from_parsed_settings(SparseIndexSettings *settings, Oid relid); extern TSDLLEXPORT List *ts_get_values_by_key_from_parsed_object(SparseIndexSettingsObject *obj, const char *key); ================================================ FILE: src/ts_catalog/continuous_agg.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ /* * This file handles commands on continuous aggs that should be allowed in * apache only mode. Right now this consists mostly of drop commands */ #include <postgres.h> #include <access/htup_details.h> #include <catalog/dependency.h> #include <catalog/namespace.h> #include <catalog/pg_trigger.h> #include <commands/trigger.h> #include <executor/spi.h> #include <fmgr.h> #include <lib/stringinfo.h> #include <nodes/makefuncs.h> #include <replication/slot.h> #include <storage/lmgr.h> #include <utils/acl.h> #include <utils/builtins.h> #include <utils/date.h> #include <utils/lsyscache.h> #include <utils/timestamp.h> #include "compat/compat.h" #include "bgw/job.h" #include "cross_module_fn.h" #include "errors.h" #include "func_cache.h" #include "hypercube.h" #include "hypertable.h" #include "hypertable_cache.h" #include "scan_iterator.h" #include "time_bucket.h" #include "time_utils.h" #include "ts_catalog/catalog.h" #include "ts_catalog/compression_settings.h" #include "ts_catalog/continuous_agg.h" #include "ts_catalog/continuous_aggs_watermark.h" #include "utils.h" #include "with_clause/alter_table_with_clause.h" #define BUCKET_FUNCTION_SERIALIZE_VERSION 1 #define CHECK_NAME_MATCH(name1, name2) (namestrcmp(name1, name2) == 0) static void init_scan_by_mat_hypertable_id(ScanIterator *iterator, const int32 mat_hypertable_id) { iterator->ctx.index = catalog_get_index(ts_catalog_get(), CONTINUOUS_AGG, CONTINUOUS_AGG_PKEY); ts_scan_iterator_scan_key_init(iterator, Anum_continuous_agg_pkey_mat_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(mat_hypertable_id)); } static void init_scan_cagg_bucket_function_by_mat_hypertable_id(ScanIterator *iterator, const int32 mat_hypertable_id) { iterator->ctx.index = catalog_get_index(ts_catalog_get(), CONTINUOUS_AGGS_BUCKET_FUNCTION, CONTINUOUS_AGGS_BUCKET_FUNCTION_PKEY_IDX); ts_scan_iterator_scan_key_init(iterator, Anum_continuous_aggs_bucket_function_pkey_mat_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(mat_hypertable_id)); } static void init_scan_by_raw_hypertable_id(ScanIterator *iterator, const int32 raw_hypertable_id) { iterator->ctx.index = catalog_get_index(ts_catalog_get(), CONTINUOUS_AGG, CONTINUOUS_AGG_RAW_HYPERTABLE_ID_IDX); ts_scan_iterator_scan_key_init(iterator, Anum_continuous_agg_raw_hypertable_id_idx_raw_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(raw_hypertable_id)); } static void init_invalidation_threshold_scan_by_hypertable_id(ScanIterator *iterator, const int32 raw_hypertable_id) { iterator->ctx.index = catalog_get_index(ts_catalog_get(), CONTINUOUS_AGGS_INVALIDATION_THRESHOLD, CONTINUOUS_AGGS_INVALIDATION_THRESHOLD_PKEY); ts_scan_iterator_scan_key_init(iterator, Anum_continuous_aggs_invalidation_threshold_pkey_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(raw_hypertable_id)); } static void init_hypertable_invalidation_log_scan_by_hypertable_id(ScanIterator *iterator, const int32 raw_hypertable_id) { iterator->ctx.index = catalog_get_index(ts_catalog_get(), CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG, CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG_IDX); ts_scan_iterator_scan_key_init( iterator, Anum_continuous_aggs_hypertable_invalidation_log_idx_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(raw_hypertable_id)); } static void init_materialization_invalidation_log_scan_by_materialization_id(ScanIterator *iterator, const int32 materialization_id) { iterator->ctx.index = catalog_get_index(ts_catalog_get(), CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG, CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG_IDX); ts_scan_iterator_scan_key_init( iterator, Anum_continuous_aggs_materialization_invalidation_log_idx_materialization_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(materialization_id)); } static void init_materialization_ranges_scan_by_materialization_id(ScanIterator *iterator, const int32 materialization_id) { iterator->ctx.index = catalog_get_index(ts_catalog_get(), CONTINUOUS_AGGS_MATERIALIZATION_RANGES, CONTINUOUS_AGGS_MATERIALIZATION_RANGES_IDX); ts_scan_iterator_scan_key_init(iterator, Anum_continuous_aggs_materialization_ranges_materialization_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(materialization_id)); } static int32 number_of_continuous_aggs_attached(int32 raw_hypertable_id) { ScanIterator iterator = ts_scan_iterator_create(CONTINUOUS_AGG, AccessShareLock, CurrentMemoryContext); int32 count = 0; init_scan_by_raw_hypertable_id(&iterator, raw_hypertable_id); ts_scanner_foreach(&iterator) { count++; } return count; } static void invalidation_threshold_delete(int32 raw_hypertable_id) { ScanIterator iterator = ts_scan_iterator_create(CONTINUOUS_AGGS_INVALIDATION_THRESHOLD, RowExclusiveLock, CurrentMemoryContext); init_invalidation_threshold_scan_by_hypertable_id(&iterator, raw_hypertable_id); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); } } static void cagg_bucket_function_delete(int32 mat_hypertable_id) { ScanIterator iterator = ts_scan_iterator_create(CONTINUOUS_AGGS_BUCKET_FUNCTION, RowExclusiveLock, CurrentMemoryContext); init_scan_cagg_bucket_function_by_mat_hypertable_id(&iterator, mat_hypertable_id); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); } } static void hypertable_invalidation_log_delete(int32 raw_hypertable_id) { ScanIterator iterator = ts_scan_iterator_create(CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG, RowExclusiveLock, CurrentMemoryContext); init_hypertable_invalidation_log_scan_by_hypertable_id(&iterator, raw_hypertable_id); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); } } void ts_get_invalidation_replication_slot_name(char *slotname, Size szslot) { snprintf(slotname, szslot, "ts_%u_cagg", MyDatabaseId); } static void ts_materialization_invalidation_log_delete(int32 mat_hypertable_id) { ScanIterator iterator = ts_scan_iterator_create(CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG, RowExclusiveLock, CurrentMemoryContext); elog(DEBUG1, "materialization log delete for hypertable %d", mat_hypertable_id); init_materialization_invalidation_log_scan_by_materialization_id(&iterator, mat_hypertable_id); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); } } static void ts_materialization_ranges_delete(int32 mat_hypertable_id) { ScanIterator iterator = ts_scan_iterator_create(CONTINUOUS_AGGS_MATERIALIZATION_RANGES, RowExclusiveLock, CurrentMemoryContext); elog(DEBUG1, "materialization log delete for hypertable %d", mat_hypertable_id); init_materialization_ranges_scan_by_materialization_id(&iterator, mat_hypertable_id); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); } } static HeapTuple continuous_agg_formdata_make_tuple(const FormData_continuous_agg *fd, TupleDesc desc) { Datum values[Natts_continuous_agg]; bool nulls[Natts_continuous_agg] = { false }; memset(values, 0, sizeof(Datum) * Natts_continuous_agg); values[AttrNumberGetAttrOffset(Anum_continuous_agg_mat_hypertable_id)] = Int32GetDatum(fd->mat_hypertable_id); values[AttrNumberGetAttrOffset(Anum_continuous_agg_raw_hypertable_id)] = Int32GetDatum(fd->raw_hypertable_id); if (fd->parent_mat_hypertable_id == INVALID_HYPERTABLE_ID) nulls[AttrNumberGetAttrOffset(Anum_continuous_agg_parent_mat_hypertable_id)] = true; else { values[AttrNumberGetAttrOffset(Anum_continuous_agg_parent_mat_hypertable_id)] = Int32GetDatum(fd->parent_mat_hypertable_id); } values[AttrNumberGetAttrOffset(Anum_continuous_agg_user_view_schema)] = NameGetDatum(&fd->user_view_schema); values[AttrNumberGetAttrOffset(Anum_continuous_agg_user_view_name)] = NameGetDatum(&fd->user_view_name); values[AttrNumberGetAttrOffset(Anum_continuous_agg_partial_view_schema)] = NameGetDatum(&fd->partial_view_schema); values[AttrNumberGetAttrOffset(Anum_continuous_agg_partial_view_name)] = NameGetDatum(&fd->partial_view_name); values[AttrNumberGetAttrOffset(Anum_continuous_agg_direct_view_schema)] = NameGetDatum(&fd->direct_view_schema); values[AttrNumberGetAttrOffset(Anum_continuous_agg_direct_view_name)] = NameGetDatum(&fd->direct_view_name); values[AttrNumberGetAttrOffset(Anum_continuous_agg_materialize_only)] = BoolGetDatum(fd->materialized_only); return heap_form_tuple(desc, values, nulls); } static void continuous_agg_formdata_fill(FormData_continuous_agg *fd, const TupleInfo *ti) { bool should_free; HeapTuple tuple; Datum values[Natts_continuous_agg]; bool nulls[Natts_continuous_agg] = { false }; tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); heap_deform_tuple(tuple, ts_scanner_get_tupledesc(ti), values, nulls); fd->mat_hypertable_id = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_continuous_agg_mat_hypertable_id)]); fd->raw_hypertable_id = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_continuous_agg_raw_hypertable_id)]); if (nulls[AttrNumberGetAttrOffset(Anum_continuous_agg_parent_mat_hypertable_id)]) fd->parent_mat_hypertable_id = INVALID_HYPERTABLE_ID; else fd->parent_mat_hypertable_id = DatumGetInt32( values[AttrNumberGetAttrOffset(Anum_continuous_agg_parent_mat_hypertable_id)]); namestrcpy(&fd->user_view_schema, DatumGetCString( values[AttrNumberGetAttrOffset(Anum_continuous_agg_user_view_schema)])); namestrcpy(&fd->user_view_name, DatumGetCString( values[AttrNumberGetAttrOffset(Anum_continuous_agg_user_view_name)])); namestrcpy(&fd->partial_view_schema, DatumGetCString( values[AttrNumberGetAttrOffset(Anum_continuous_agg_partial_view_schema)])); namestrcpy(&fd->partial_view_name, DatumGetCString( values[AttrNumberGetAttrOffset(Anum_continuous_agg_partial_view_name)])); namestrcpy(&fd->direct_view_schema, DatumGetCString( values[AttrNumberGetAttrOffset(Anum_continuous_agg_direct_view_schema)])); namestrcpy(&fd->direct_view_name, DatumGetCString( values[AttrNumberGetAttrOffset(Anum_continuous_agg_direct_view_name)])); fd->materialized_only = DatumGetBool(values[AttrNumberGetAttrOffset(Anum_continuous_agg_materialize_only)]); if (should_free) heap_freetuple(tuple); } /* * Fill the fields of a integer based bucketing function */ static void cagg_fill_bucket_function_integer_based(ContinuousAggBucketFunction *bf, bool *isnull, Datum *values) { /* Bucket width */ Assert(!isnull[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_width)]); const char *bucket_width_str = TextDatumGetCString( values[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_width)]); Assert(strlen(bucket_width_str) > 0); bf->bucket_integer_width = pg_strtoint64(bucket_width_str); /* Bucket origin cannot be used with integer based buckets */ Assert(isnull[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_origin)] == true); /* Bucket offset */ if (!isnull[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_offset)]) { const char *offset_str = TextDatumGetCString( values[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_offset)]); bf->bucket_integer_offset = pg_strtoint64(offset_str); } /* Timezones cannot be used with integer based buckets */ Assert(isnull[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_timezone)] == true); } /* * Fill the fields of a time based bucketing function */ static void cagg_fill_bucket_function_time_based(ContinuousAggBucketFunction *bf, bool *isnull, Datum *values) { /* * bucket_width * * The value is stored as TEXT since we have to store the interval value of time * buckets and also the number value of integer based buckets. */ Assert(!isnull[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_width)]); const char *bucket_width_str = TextDatumGetCString( values[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_width)]); Assert(strlen(bucket_width_str) > 0); bf->bucket_time_width = DatumGetIntervalP( DirectFunctionCall3(interval_in, CStringGetDatum(bucket_width_str), InvalidOid, -1)); /* Bucket origin */ if (!isnull[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_origin)]) { const char *origin_str = TextDatumGetCString( values[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_origin)]); bf->bucket_time_origin = DatumGetTimestamp(DirectFunctionCall3(timestamptz_in, CStringGetDatum(origin_str), ObjectIdGetDatum(InvalidOid), Int32GetDatum(-1))); } else { TIMESTAMP_NOBEGIN(bf->bucket_time_origin); } /* Bucket offset */ if (!isnull[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_offset)]) { const char *offset_str = TextDatumGetCString( values[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_offset)]); bf->bucket_time_offset = DatumGetIntervalP( DirectFunctionCall3(interval_in, CStringGetDatum(offset_str), InvalidOid, -1)); } /* Bucket timezone */ if (!isnull[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_timezone)]) { bf->bucket_time_timezone = TextDatumGetCString( values[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_timezone)]); } } static void continuous_agg_fill_bucket_function(int32 mat_hypertable_id, ContinuousAggBucketFunction *bf) { ScanIterator iterator; int count = 0; iterator = ts_scan_iterator_create(CONTINUOUS_AGGS_BUCKET_FUNCTION, AccessShareLock, CurrentMemoryContext); init_scan_cagg_bucket_function_by_mat_hypertable_id(&iterator, mat_hypertable_id); ts_scanner_foreach(&iterator) { Datum values[Natts_continuous_aggs_bucket_function]; bool isnull[Natts_continuous_aggs_bucket_function]; bool should_free; HeapTuple tuple = ts_scan_iterator_fetch_heap_tuple(&iterator, false, &should_free); /* * Our usual GETSTRUCT() approach doesn't work when TEXT fields are involved, * thus a more robust approach with heap_deform_tuple() is used here. */ heap_deform_tuple(tuple, ts_scan_iterator_tupledesc(&iterator), values, isnull); /* Bucket function */ Assert(!isnull[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_function)]); const char *bucket_function_str = TextDatumGetCString( values[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_function)]); bf->bucket_function = DatumGetObjectId( DirectFunctionCall1(regprocedurein, CStringGetDatum(bucket_function_str))); bf->bucket_time_based = ts_continuous_agg_bucket_on_interval(bf->bucket_function); if (bf->bucket_time_based) { cagg_fill_bucket_function_time_based(bf, isnull, values); } else { cagg_fill_bucket_function_integer_based(bf, isnull, values); } /* Bucket fixed width */ Assert(!isnull[AttrNumberGetAttrOffset( Anum_continuous_aggs_bucket_function_bucket_fixed_width)]); bf->bucket_fixed_interval = DatumGetBool(values[AttrNumberGetAttrOffset( Anum_continuous_aggs_bucket_function_bucket_fixed_width)]); count++; if (should_free) heap_freetuple(tuple); } /* * This function should never be called unless we know that the corresponding * cagg exists and uses a variable-sized bucket. There should be exactly one * entry in .continuous_aggs_bucket_function catalog table for such a cagg. */ if (count != 1) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("invalid or missing information about the bucketing function for cagg"), errdetail("%d", mat_hypertable_id))); } } static void continuous_agg_init(ContinuousAgg *cagg, const Form_continuous_agg fd) { Oid nspid = get_namespace_oid(NameStr(fd->user_view_schema), false); Hypertable *cagg_ht = ts_hypertable_get_by_id(fd->mat_hypertable_id); if (!cagg_ht) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("continuous aggregate hypertable with ID %d does not exist", fd->mat_hypertable_id))); const Dimension *time_dim; time_dim = hyperspace_get_open_dimension(cagg_ht->space, 0); if (!time_dim) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("continuous aggregate hypertable with ID %d has no open dimension", fd->mat_hypertable_id))); cagg->partition_type = ts_dimension_get_partition_type(time_dim); cagg->relid = get_relname_relid(NameStr(fd->user_view_name), nspid); memcpy(&cagg->data, fd, sizeof(cagg->data)); Assert(OidIsValid(cagg->relid)); Assert(OidIsValid(cagg->partition_type)); cagg->bucket_function = palloc0(sizeof(ContinuousAggBucketFunction)); continuous_agg_fill_bucket_function(cagg->data.mat_hypertable_id, cagg->bucket_function); } TSDLLEXPORT ContinuousAggInfo ts_continuous_agg_get_all_caggs_info(int32 raw_hypertable_id) { ContinuousAggInfo all_caggs_info; List *caggs = ts_continuous_aggs_find_by_raw_table_id(raw_hypertable_id); ListCell *lc; all_caggs_info.mat_hypertable_ids = NIL; all_caggs_info.bucket_functions = NIL; Assert(list_length(caggs) > 0); foreach (lc, caggs) { ContinuousAgg *cagg = lfirst(lc); all_caggs_info.bucket_functions = lappend(all_caggs_info.bucket_functions, cagg->bucket_function); all_caggs_info.mat_hypertable_ids = lappend_int(all_caggs_info.mat_hypertable_ids, cagg->data.mat_hypertable_id); } return all_caggs_info; } TSDLLEXPORT ContinuousAggHypertableStatus ts_continuous_agg_hypertable_status(int32 hypertable_id) { ScanIterator iterator = ts_scan_iterator_create(CONTINUOUS_AGG, AccessShareLock, CurrentMemoryContext); ContinuousAggHypertableStatus status = HypertableIsNotContinuousAgg; ts_scanner_foreach(&iterator) { FormData_continuous_agg data; TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); continuous_agg_formdata_fill(&data, ti); if (data.raw_hypertable_id == hypertable_id) status |= HypertableIsRawTable; if (data.mat_hypertable_id == hypertable_id) status |= HypertableIsMaterialization; if (status == HypertableIsMaterializationAndRaw) { ts_scan_iterator_close(&iterator); return status; } } return status; } TSDLLEXPORT List * ts_continuous_aggs_find_by_raw_table_id(int32 raw_hypertable_id) { List *continuous_aggs = NIL; ScanIterator iterator = ts_scan_iterator_create(CONTINUOUS_AGG, AccessShareLock, CurrentMemoryContext); init_scan_by_raw_hypertable_id(&iterator, raw_hypertable_id); ts_scanner_foreach(&iterator) { ContinuousAgg *ca; FormData_continuous_agg data; MemoryContext oldmctx; TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); continuous_agg_formdata_fill(&data, ti); oldmctx = MemoryContextSwitchTo(ts_scan_iterator_get_result_memory_context(&iterator)); ca = palloc0(sizeof(*ca)); continuous_agg_init(ca, &data); continuous_aggs = lappend(continuous_aggs, ca); MemoryContextSwitchTo(oldmctx); } return continuous_aggs; } /* Find a continuous aggregate by the materialized hypertable id */ ContinuousAgg * ts_continuous_agg_find_by_mat_hypertable_id(int32 mat_hypertable_id, bool missing_ok) { ContinuousAgg *ca = NULL; ScanIterator iterator = ts_scan_iterator_create(CONTINUOUS_AGG, RowExclusiveLock, CurrentMemoryContext); init_scan_by_mat_hypertable_id(&iterator, mat_hypertable_id); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); FormData_continuous_agg form; continuous_agg_formdata_fill(&form, ti); /* Note that this scan can only match at most once, so we assert on * `ca` here. */ Assert(ca == NULL); ca = ts_scan_iterator_alloc_result(&iterator, sizeof(*ca)); continuous_agg_init(ca, &form); Assert(ca && ca->data.mat_hypertable_id == mat_hypertable_id); } ts_scan_iterator_close(&iterator); if (ca == NULL && !missing_ok) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid materialized hypertable ID: %d", mat_hypertable_id))); } return ca; } static bool continuous_agg_find_by_name(const char *schema, const char *name, ContinuousAggViewType type, FormData_continuous_agg *fd) { ScanIterator iterator; AttrNumber view_name_attrnum = 0; AttrNumber schema_name_attrnum = 0; int count = 0; Assert(schema); Assert(name); switch (type) { case ContinuousAggUserView: schema_name_attrnum = Anum_continuous_agg_user_view_schema; view_name_attrnum = Anum_continuous_agg_user_view_name; break; case ContinuousAggPartialView: schema_name_attrnum = Anum_continuous_agg_partial_view_schema; view_name_attrnum = Anum_continuous_agg_partial_view_name; break; case ContinuousAggDirectView: schema_name_attrnum = Anum_continuous_agg_direct_view_schema; view_name_attrnum = Anum_continuous_agg_direct_view_name; break; case ContinuousAggAnyView: break; } iterator = ts_scan_iterator_create(CONTINUOUS_AGG, AccessShareLock, CurrentMemoryContext); if (type != ContinuousAggAnyView) { ts_scan_iterator_scan_key_init(&iterator, schema_name_attrnum, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(schema)); ts_scan_iterator_scan_key_init(&iterator, view_name_attrnum, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(name)); } ts_scanner_foreach(&iterator) { ContinuousAggViewType vtype = type; TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); FormData_continuous_agg data; continuous_agg_formdata_fill(&data, ti); if (vtype == ContinuousAggAnyView) vtype = ts_continuous_agg_view_type(&data, schema, name); if (vtype != ContinuousAggAnyView) { memcpy(fd, &data, sizeof(*fd)); count++; } } Assert(count <= 1); return count == 1; } ContinuousAgg * ts_continuous_agg_find_by_view_name(const char *schema, const char *name, ContinuousAggViewType type) { FormData_continuous_agg fd; ContinuousAgg *ca; if (!continuous_agg_find_by_name(schema, name, type, &fd)) return NULL; ca = palloc0(sizeof(ContinuousAgg)); continuous_agg_init(ca, &fd); return ca; } ContinuousAgg * ts_continuous_agg_find_userview_name(const char *schema, const char *name) { return ts_continuous_agg_find_by_view_name(schema, name, ContinuousAggUserView); } /* * Find a continuous agg object by the main relid. * * The relid is the user-facing object ID that represents the continuous * aggregate (i.e., the query view's ID). */ ContinuousAgg * ts_continuous_agg_find_by_relid(Oid relid) { const char *relname = get_rel_name(relid); const char *schemaname = get_namespace_name(get_rel_namespace(relid)); if (NULL == relname || NULL == schemaname) return NULL; return ts_continuous_agg_find_userview_name(schemaname, relname); } /* * Find a continuous aggregate by range var. */ ContinuousAgg * ts_continuous_agg_find_by_rv(const RangeVar *rv) { Oid relid; if (rv == NULL) return NULL; relid = RangeVarGetRelid(rv, NoLock, true); if (!OidIsValid(relid)) return NULL; return ts_continuous_agg_find_by_relid(relid); } static ObjectAddress get_and_lock_rel_by_name(const Name schema, const Name name, LOCKMODE mode) { ObjectAddress addr; Oid relid = InvalidOid; Oid nspid = get_namespace_oid(NameStr(*schema), true); if (OidIsValid(nspid)) { relid = get_relname_relid(NameStr(*name), nspid); if (OidIsValid(relid)) LockRelationOid(relid, mode); } ObjectAddressSet(addr, RelationRelationId, relid); return addr; } static ObjectAddress get_and_lock_rel_by_hypertable_id(int32 hypertable_id, LOCKMODE mode) { ObjectAddress addr; Oid relid = ts_hypertable_id_to_relid(hypertable_id, true); if (OidIsValid(relid)) LockRelationOid(relid, mode); ObjectAddressSet(addr, RelationRelationId, relid); return addr; } /* * Drops continuous aggs and all related objects. * * This function is intended to be run by event trigger during CASCADE, * which implies that most of the dependent objects potentially could be * dropped including associated schema. * * These objects are: * * - user view itself * - continuous agg catalog entry * - partial view * - materialization hypertable * - trigger on the raw hypertable (hypertable specified in the user view) * - copy of the user view query (AKA the direct view) * * NOTE: The order in which the objects are dropped should be EXACTLY the * same as in materialize.c * * drop_user_view indicates whether to drop the user view. * (should be false if called as part of the drop-user-view callback) */ static void drop_continuous_agg(FormData_continuous_agg *cadata, bool drop_user_view) { Catalog *catalog; ScanIterator iterator; ObjectAddress user_view = { 0 }; ObjectAddress partial_view = { 0 }; ObjectAddress direct_view = { 0 }; ObjectAddress raw_hypertable = { 0 }; ObjectAddress mat_hypertable = { 0 }; bool raw_hypertable_has_other_caggs; /* Delete the job before taking locks as it kills long-running jobs * which we would otherwise wait on */ List *jobs = ts_bgw_job_find_by_hypertable_id(cadata->mat_hypertable_id); ListCell *lc; foreach (lc, jobs) { BgwJob *job = lfirst(lc); ts_bgw_job_delete_by_id(job->fd.id); } /* * Lock objects. * * Following objects might be already dropped in case of CASCADE * drop including the associated schema object. * * NOTE: the lock order matters, see tsl/src/materialization.c. * Perform all locking upfront. * * AccessExclusiveLock is needed to drop triggers and also prevent * concurrent DML commands. * * It is needed also in the case that we are using WAL-based invalidation * collection since we want to serialize create and drop of continuous * aggregates. */ if (drop_user_view) user_view = get_and_lock_rel_by_name(&cadata->user_view_schema, &cadata->user_view_name, AccessExclusiveLock); raw_hypertable = get_and_lock_rel_by_hypertable_id(cadata->raw_hypertable_id, AccessExclusiveLock); mat_hypertable = get_and_lock_rel_by_hypertable_id(cadata->mat_hypertable_id, AccessExclusiveLock); /* Lock catalogs */ catalog = ts_catalog_get(); LockRelationOid(catalog_get_table_id(catalog, BGW_JOB), RowExclusiveLock); LockRelationOid(catalog_get_table_id(catalog, CONTINUOUS_AGG), RowExclusiveLock); LockRelationOid(catalog_get_table_id(catalog, CONTINUOUS_AGGS_WATERMARK), RowExclusiveLock); raw_hypertable_has_other_caggs = OidIsValid(raw_hypertable.objectId) && number_of_continuous_aggs_attached(cadata->raw_hypertable_id) > 1; if (!raw_hypertable_has_other_caggs) { LockRelationOid(catalog_get_table_id(catalog, CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG), RowExclusiveLock); LockRelationOid(catalog_get_table_id(catalog, CONTINUOUS_AGGS_INVALIDATION_THRESHOLD), RowExclusiveLock); } /* * Following objects might be already dropped in case of CASCADE * drop including the associated schema object. */ partial_view = get_and_lock_rel_by_name(&cadata->partial_view_schema, &cadata->partial_view_name, AccessExclusiveLock); direct_view = get_and_lock_rel_by_name(&cadata->direct_view_schema, &cadata->direct_view_name, AccessExclusiveLock); /* Delete catalog entry */ iterator = ts_scan_iterator_create(CONTINUOUS_AGG, RowExclusiveLock, CurrentMemoryContext); init_scan_by_mat_hypertable_id(&iterator, cadata->mat_hypertable_id); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); FormData_continuous_agg form; continuous_agg_formdata_fill(&form, ti); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); /* Delete all related rows */ if (!raw_hypertable_has_other_caggs) { hypertable_invalidation_log_delete(form.raw_hypertable_id); } ts_materialization_invalidation_log_delete(form.mat_hypertable_id); ts_materialization_ranges_delete(form.mat_hypertable_id); if (!raw_hypertable_has_other_caggs) { invalidation_threshold_delete(form.raw_hypertable_id); } /* Delete watermark */ ts_cagg_watermark_delete_by_mat_hypertable_id(form.mat_hypertable_id); } cagg_bucket_function_delete(cadata->mat_hypertable_id); /* Perform actual deletions now */ if (OidIsValid(user_view.objectId)) performDeletion(&user_view, DROP_RESTRICT, 0); if (OidIsValid(mat_hypertable.objectId)) { performDeletion(&mat_hypertable, DROP_CASCADE, 0); ts_compression_settings_delete(mat_hypertable.objectId); ts_hypertable_delete_by_id(cadata->mat_hypertable_id); } if (OidIsValid(partial_view.objectId)) performDeletion(&partial_view, DROP_RESTRICT, 0); if (OidIsValid(direct_view.objectId)) performDeletion(&direct_view, DROP_RESTRICT, 0); } /* * This is a called when a hypertable gets dropped. * * If the hypertable is a raw hypertable for a continuous agg, * drop the continuous agg. * * If the hypertable is a materialization hypertable, error out * and force the user to drop the continuous agg instead. */ void ts_continuous_agg_drop_hypertable_callback(int32 hypertable_id) { ScanIterator iterator = ts_scan_iterator_create(CONTINUOUS_AGG, AccessShareLock, CurrentMemoryContext); ts_scanner_foreach(&iterator) { FormData_continuous_agg data; TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); continuous_agg_formdata_fill(&data, ti); if (data.raw_hypertable_id == hypertable_id) drop_continuous_agg(&data, true); if (data.mat_hypertable_id == hypertable_id) ereport(ERROR, (errcode(ERRCODE_DEPENDENT_OBJECTS_STILL_EXIST), errmsg("cannot drop the materialized table because it is required by a " "continuous aggregate"))); } } /* Block dropping the partial and direct view if the continuous aggregate still exists */ static void drop_internal_view(const FormData_continuous_agg *fd) { ScanIterator iterator = ts_scan_iterator_create(CONTINUOUS_AGG, AccessShareLock, CurrentMemoryContext); int count = 0; init_scan_by_mat_hypertable_id(&iterator, fd->mat_hypertable_id); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); count++; } if (count > 0) ereport(ERROR, (errcode(ERRCODE_DEPENDENT_OBJECTS_STILL_EXIST), errmsg( "cannot drop the partial/direct view because it is required by a continuous " "aggregate"))); } /* This gets called when a view gets dropped. */ static void continuous_agg_drop_view_callback(FormData_continuous_agg *fd, const char *schema, const char *name) { ContinuousAggViewType vtyp; vtyp = ts_continuous_agg_view_type(fd, schema, name); switch (vtyp) { case ContinuousAggUserView: drop_continuous_agg(fd, false /* The user view has already been dropped */); break; case ContinuousAggPartialView: case ContinuousAggDirectView: drop_internal_view(fd); break; default: elog(ERROR, "unknown continuous aggregate view type"); } } bool ts_continuous_agg_drop(const char *view_schema, const char *view_name) { FormData_continuous_agg fd; bool found = continuous_agg_find_by_name(view_schema, view_name, ContinuousAggAnyView, &fd); if (found) continuous_agg_drop_view_callback(&fd, view_schema, view_name); return found; } static inline bool ts_continuous_agg_is_user_view_schema(FormData_continuous_agg *data, const char *schema) { return CHECK_NAME_MATCH(&data->user_view_schema, schema); } static inline bool ts_continuous_agg_is_partial_view_schema(FormData_continuous_agg *data, const char *schema) { return CHECK_NAME_MATCH(&data->partial_view_schema, schema); } static inline bool ts_continuous_agg_is_direct_view_schema(FormData_continuous_agg *data, const char *schema) { return CHECK_NAME_MATCH(&data->direct_view_schema, schema); } ContinuousAggViewType ts_continuous_agg_view_type(FormData_continuous_agg *data, const char *schema, const char *name) { if (CHECK_NAME_MATCH(&data->user_view_schema, schema) && CHECK_NAME_MATCH(&data->user_view_name, name)) return ContinuousAggUserView; else if (CHECK_NAME_MATCH(&data->partial_view_schema, schema) && CHECK_NAME_MATCH(&data->partial_view_name, name)) return ContinuousAggPartialView; else if (CHECK_NAME_MATCH(&data->direct_view_schema, schema) && CHECK_NAME_MATCH(&data->direct_view_name, name)) return ContinuousAggDirectView; else return ContinuousAggAnyView; } typedef struct CaggRenameCtx { const char *old_schema; const char *old_name; const char *new_schema; const char *new_name; ObjectType *object_type; void (*process_rename)(FormData_continuous_agg *form, bool *do_update, void *data); } CaggRenameCtx; static void continuous_agg_rename_process_rename_schema(FormData_continuous_agg *form, bool *do_update, void *data) { CaggRenameCtx *ctx = (CaggRenameCtx *) data; if (ts_continuous_agg_is_user_view_schema(form, ctx->old_schema)) { namestrcpy(&form->user_view_schema, ctx->new_schema); *do_update = true; } if (ts_continuous_agg_is_partial_view_schema(form, ctx->old_schema)) { namestrcpy(&form->partial_view_schema, ctx->new_schema); *do_update = true; } if (ts_continuous_agg_is_direct_view_schema(form, ctx->old_schema)) { namestrcpy(&form->direct_view_schema, ctx->new_schema); *do_update = true; } } static void continuous_agg_rename_process_rename_view(FormData_continuous_agg *form, bool *do_update, void *data) { CaggRenameCtx *ctx = (CaggRenameCtx *) data; ContinuousAggViewType vtyp; vtyp = ts_continuous_agg_view_type(form, ctx->old_schema, ctx->old_name); switch (vtyp) { case ContinuousAggUserView: { if (*ctx->object_type == OBJECT_VIEW) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot alter continuous aggregate using ALTER VIEW"), errhint("Use ALTER MATERIALIZED VIEW to alter a continuous aggregate."))); Assert(*ctx->object_type == OBJECT_MATVIEW); *ctx->object_type = OBJECT_VIEW; namestrcpy(&form->user_view_schema, ctx->new_schema); namestrcpy(&form->user_view_name, ctx->new_name); *do_update = true; break; } case ContinuousAggPartialView: { namestrcpy(&form->partial_view_schema, ctx->new_schema); namestrcpy(&form->partial_view_name, ctx->new_name); *do_update = true; break; } case ContinuousAggDirectView: { namestrcpy(&form->direct_view_schema, ctx->new_schema); namestrcpy(&form->direct_view_name, ctx->new_name); *do_update = true; break; } default: break; } } static ScanTupleResult continuous_agg_rename(TupleInfo *ti, void *data) { CaggRenameCtx *ctx = (CaggRenameCtx *) data; FormData_continuous_agg form; bool do_update = false; CatalogSecurityContext sec_ctx; continuous_agg_formdata_fill(&form, ti); ctx->process_rename(&form, &do_update, (void *) ctx); if (do_update) { HeapTuple new_tuple = continuous_agg_formdata_make_tuple(&form, ts_scanner_get_tupledesc(ti)); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_update_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti), new_tuple); ts_catalog_restore_user(&sec_ctx); heap_freetuple(new_tuple); } return SCAN_CONTINUE; } void ts_continuous_agg_rename_schema_name(const char *old_schema, const char *new_schema) { CaggRenameCtx cagg_rename_ctx = { .old_schema = old_schema, .new_schema = new_schema, .process_rename = continuous_agg_rename_process_rename_schema, }; Catalog *catalog = ts_catalog_get(); ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, CONTINUOUS_AGG), .index = InvalidOid, .tuple_found = continuous_agg_rename, .data = &cagg_rename_ctx, .lockmode = RowExclusiveLock, .scandirection = ForwardScanDirection, }; ts_scanner_scan(&scanctx); } void ts_continuous_agg_rename_view(const char *old_schema, const char *old_name, const char *new_schema, const char *new_name, ObjectType *object_type) { CaggRenameCtx cagg_rename_ctx = { .old_schema = old_schema, .old_name = old_name, .new_schema = new_schema, .new_name = new_name, .object_type = object_type, .process_rename = continuous_agg_rename_process_rename_view, }; Catalog *catalog = ts_catalog_get(); ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, CONTINUOUS_AGG), .index = InvalidOid, .tuple_found = continuous_agg_rename, .data = &cagg_rename_ctx, .lockmode = RowExclusiveLock, .scandirection = ForwardScanDirection, }; ts_scanner_scan(&scanctx); } static int32 find_raw_hypertable_for_materialization(int32 mat_hypertable_id) { PG_USED_FOR_ASSERTS_ONLY short count = 0; int32 htid = INVALID_HYPERTABLE_ID; ScanIterator iterator = ts_scan_iterator_create(CONTINUOUS_AGG, RowExclusiveLock, CurrentMemoryContext); init_scan_by_mat_hypertable_id(&iterator, mat_hypertable_id); ts_scanner_foreach(&iterator) { bool isnull; Datum datum = slot_getattr(ts_scan_iterator_slot(&iterator), Anum_continuous_agg_raw_hypertable_id, &isnull); Assert(!isnull); htid = DatumGetInt32(datum); count++; } Assert(count <= 1); ts_scan_iterator_close(&iterator); return htid; } /* Continuous aggregate materialization hypertables inherit integer_now func * from the raw hypertable (unless it was explicitly reset for cont. aggregate. * Walk the materialization hypertable ->raw hypertable tree till * we find a hypertable that has integer_now_func set. */ TSDLLEXPORT const Dimension * ts_continuous_agg_find_integer_now_func_by_materialization_id(int32 mat_htid) { int32 raw_htid = mat_htid; const Dimension *par_dim = NULL; while (raw_htid != INVALID_HYPERTABLE_ID) { Hypertable *raw_ht = ts_hypertable_get_by_id(raw_htid); const Dimension *open_dim = hyperspace_get_open_dimension(raw_ht->space, 0); if (strlen(NameStr(open_dim->fd.integer_now_func)) != 0 && strlen(NameStr(open_dim->fd.integer_now_func_schema)) != 0) { par_dim = open_dim; break; } mat_htid = raw_htid; raw_htid = find_raw_hypertable_for_materialization(mat_htid); } return par_dim; } TSDLLEXPORT void ts_continuous_agg_invalidate_chunk(Hypertable *ht, Chunk *chunk) { int64 start = ts_chunk_primary_dimension_start(chunk); int64 end = ts_chunk_primary_dimension_end(chunk); Assert(hyperspace_get_open_dimension(ht->space, 0)->fd.id == chunk->cube->slices[0]->fd.dimension_id); ts_cm_functions->continuous_agg_invalidate_raw_ht(ht, start, end); } /* Determines if a bucket is using integer or an interval partitioning */ bool ts_continuous_agg_bucket_on_interval(Oid bucket_function) { Assert(OidIsValid(bucket_function)); FuncInfo *func_info = ts_func_cache_get(bucket_function); Ensure(func_info != NULL, "unable to get function info for Oid %d", bucket_function); /* The function has to be a currently allowed function or one of the deprecated bucketing * functions */ Assert(func_info->allowed_in_cagg_definition); Oid first_bucket_arg = func_info->arg_types[0]; return first_bucket_arg == INTERVALOID; } /* * Calls the desired time bucket function depending on the arguments * (i.e., whether it has timezone and offset/origin). * This is a common procedure used by ts_compute_* below. */ static Datum generic_time_bucket(const ContinuousAggBucketFunction *bf, Datum timestamp) { FuncInfo *func_info = ts_func_cache_get_bucketing_func(bf->bucket_function); Ensure(func_info != NULL, "unable to get bucket function for Oid %d", bf->bucket_function); bool has_offset = (bf->bucket_time_offset != NULL); if (bf->bucket_time_timezone != NULL) { /* * Use LOCAL_FCINFO to call ts_timestamptz_timezone_bucket with all * 5 arguments, including origin and offset. */ LOCAL_FCINFO(fcinfo, 5); InitFunctionCallInfoData(*fcinfo, NULL, 5, InvalidOid, NULL, NULL); fcinfo->args[0] = (NullableDatum){ .value = IntervalPGetDatum(bf->bucket_time_width), .isnull = false }; fcinfo->args[1] = (NullableDatum){ .value = timestamp, .isnull = false }; fcinfo->args[2] = (NullableDatum){ .value = CStringGetTextDatum(bf->bucket_time_timezone), .isnull = false }; if (TIMESTAMP_NOT_FINITE(bf->bucket_time_origin)) fcinfo->args[3] = (NullableDatum){ .value = (Datum) 0, .isnull = true }; else fcinfo->args[3] = (NullableDatum){ .value = TimestampTzGetDatum(bf->bucket_time_origin), .isnull = false }; if (has_offset) fcinfo->args[4] = (NullableDatum){ .value = IntervalPGetDatum(bf->bucket_time_offset), .isnull = false }; else fcinfo->args[4] = (NullableDatum){ .value = (Datum) 0, .isnull = true }; return ts_timestamptz_timezone_bucket(fcinfo); } if (has_offset) { return DirectFunctionCall3(ts_timestamp_offset_bucket, IntervalPGetDatum(bf->bucket_time_width), timestamp, IntervalPGetDatum(bf->bucket_time_offset)); } if (TIMESTAMP_NOT_FINITE(bf->bucket_time_origin)) { /* using default origin */ return DirectFunctionCall2(ts_timestamp_bucket, IntervalPGetDatum(bf->bucket_time_width), timestamp); } else { /* custom origin specified */ return DirectFunctionCall3(ts_timestamp_bucket, IntervalPGetDatum(bf->bucket_time_width), timestamp, TimestampTzGetDatum(bf->bucket_time_origin)); } } /* * Adds one bf->bucket_size interval to the timestamp. This is a common * procedure used by ts_compute_* below. * * If bf->bucket_time_timezone is specified, the math happens in this timezone. * Otherwise, it happens in UTC. */ static Datum generic_add_interval(const ContinuousAggBucketFunction *bf, Datum timestamp) { Datum tzname = 0; bool has_timezone = (bf->bucket_time_timezone != NULL); if (has_timezone) { /* * Convert 'timestamp' to TIMESTAMP at given timezone. * The code is equal to 'timestamptz AT TIME ZONE tzname'. */ tzname = CStringGetTextDatum(bf->bucket_time_timezone); timestamp = DirectFunctionCall2(timestamptz_zone, tzname, timestamp); } timestamp = DirectFunctionCall2(timestamp_pl_interval, timestamp, IntervalPGetDatum(bf->bucket_time_width)); if (has_timezone) { Assert(tzname != 0); timestamp = DirectFunctionCall2(timestamp_zone, tzname, timestamp); } return timestamp; } /* * Computes inscribed refresh_window for variable-sized buckets. * * The algorithm is simple: * * end = time_bucket(bucket_size, end) * * if(start != time_bucket(bucket_size, start)) * start = time_bucket(bucket_size, start) + interval bucket_size * */ void ts_compute_inscribed_bucketed_refresh_window_variable(int64 *start, int64 *end, const ContinuousAggBucketFunction *bf) { Datum start_old, end_old, start_aligned, end_aliged; /* * It's OK to use TIMESTAMPOID here. Variable-sized buckets can be used * only for dates, timestamps and timestamptz's. For all these types our * internal time representation is microseconds relative the UNIX epoch. * So the results will be correct regardless of the actual type used in * the CAGG. For more details see ts_internal_to_time_value() implementation. */ start_old = ts_internal_to_time_value(*start, TIMESTAMPOID); end_old = ts_internal_to_time_value(*end, TIMESTAMPOID); start_aligned = generic_time_bucket(bf, start_old); end_aliged = generic_time_bucket(bf, end_old); if (DatumGetTimestamp(start_aligned) != DatumGetTimestamp(start_old)) { start_aligned = generic_add_interval(bf, start_aligned); } *start = ts_time_value_to_internal(start_aligned, TIMESTAMPOID); *end = ts_time_value_to_internal(end_aliged, TIMESTAMPOID); } /* * Computes circumscribed refresh_window for variable-sized buckets. * * The algorithm is simple: * * start = time_bucket(bucket_size, start) * * if(end != time_bucket(bucket_size, end)) * end = time_bucket(bucket_size, end) + interval bucket_size */ void ts_compute_circumscribed_bucketed_refresh_window_variable(int64 *start, int64 *end, const ContinuousAggBucketFunction *bf) { Datum start_old, end_old, start_new, end_new; /* * It's OK to use TIMESTAMPOID here. * See the comment in ts_compute_inscribed_bucketed_refresh_window_variable() */ start_old = ts_internal_to_time_value(*start, TIMESTAMPOID); end_old = ts_internal_to_time_value(*end, TIMESTAMPOID); start_new = generic_time_bucket(bf, start_old); end_new = generic_time_bucket(bf, end_old); /* Add interval to expand to next bucket if: * 1. end wasn't at a bucket boundary (end moved during bucketing), OR * 2. we have a single-point at a bucket boundary (start == end after bucketing) */ if (DatumGetTimestamp(end_new) != DatumGetTimestamp(end_old) || DatumGetTimestamp(start_new) == DatumGetTimestamp(end_new)) { end_new = generic_add_interval(bf, end_new); } *start = ts_time_value_to_internal(start_new, TIMESTAMPOID); *end = ts_time_value_to_internal(end_new, TIMESTAMPOID); } /* * Calculates the beginning of the next bucket. * * The algorithm is just: * * val = time_bucket(bucket_size, val) + interval bucket_size */ int64 ts_compute_beginning_of_the_next_bucket_variable(int64 timeval, const ContinuousAggBucketFunction *bf) { Datum val_new; Datum val_old; /* * It's OK to use TIMESTAMPOID here. * See the comment in ts_compute_inscribed_bucketed_refresh_window_variable() */ val_old = ts_internal_to_time_value(timeval, TIMESTAMPOID); val_new = generic_time_bucket(bf, val_old); val_new = generic_add_interval(bf, val_new); return ts_time_value_to_internal(val_new, TIMESTAMPOID); } Oid ts_cagg_permissions_check(Oid cagg_oid, Oid userid) { Oid ownerid = ts_rel_get_owner(cagg_oid); if (!has_privs_of_role(userid, ownerid)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("must be owner of continuous aggregate \"%s\"", get_rel_name(cagg_oid)))); return ownerid; } Query * ts_continuous_agg_get_query(ContinuousAgg *cagg) { Oid cagg_view_oid; Relation cagg_view_rel; RuleLock *cagg_view_rules; RewriteRule *rule; Query *cagg_view_query; cagg_view_oid = ts_get_relation_relid(NameStr(cagg->data.partial_view_schema), NameStr(cagg->data.partial_view_name), false); cagg_view_rel = table_open(cagg_view_oid, AccessShareLock); cagg_view_rules = cagg_view_rel->rd_rules; Assert(cagg_view_rules && cagg_view_rules->numLocks == 1); rule = cagg_view_rules->rules[0]; if (rule->event != CMD_SELECT) ereport(ERROR, (errcode(ERRCODE_TS_UNEXPECTED), errmsg("unexpected rule event for view"))); cagg_view_query = (Query *) copyObject(linitial(rule->actions)); table_close(cagg_view_rel, NoLock); return cagg_view_query; } /* * Get the width of a fixed size bucket */ int64 ts_continuous_agg_fixed_bucket_width(const ContinuousAggBucketFunction *bucket_function) { Assert(bucket_function->bucket_fixed_interval == true); if (bucket_function->bucket_time_based) { Interval *interval = bucket_function->bucket_time_width; Assert(interval->month == 0); return interval->time + (interval->day * USECS_PER_DAY); } else { return bucket_function->bucket_integer_width; } } /* * Get the width of a bucket */ int64 ts_continuous_agg_bucket_width(const ContinuousAggBucketFunction *bucket_function) { int64 bucket_width; if (bucket_function->bucket_fixed_interval == false) { /* * There are several cases of variable-sized buckets: * 1. Monthly buckets * 2. Buckets with timezones * 3. Cases 1 and 2 at the same time * * For months we simply take 30 days like on interval_to_int64 and * multiply this number by the number of months in the bucket. This * reduces the task to days/hours/minutes scenario. * * Days/hours/minutes case is handled the same way as for fixed-sized * buckets. The refresh window at least two buckets in size is adequate * for such corner cases as DST. */ /* bucket_function should always be specified for variable-sized buckets */ Assert(bucket_function != NULL); /* ... and bucket_function->bucket_time_width too */ Assert(bucket_function->bucket_time_width != NULL); /* Make a temporary copy of bucket_width */ Interval interval = *bucket_function->bucket_time_width; interval.day += 30 * interval.month; interval.month = 0; bucket_width = ts_interval_value_to_internal(IntervalPGetDatum(&interval), INTERVALOID); } else { bucket_width = ts_continuous_agg_fixed_bucket_width(bucket_function); } return bucket_width; } ================================================ FILE: src/ts_catalog/continuous_agg.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <catalog/pg_type.h> #include <nodes/parsenodes.h> #include "chunk.h" #include "ts_catalog/catalog.h" #include "compat/compat.h" #include "with_clause/with_clause_parser.h" #define TS_INVALIDATION_SLOT_NAME_MAX (32) /*switch to ts user for _timescaledb_internal access */ #define SWITCH_TO_TS_USER(schemaname, newuid, saved_uid, saved_secctx) \ do \ { \ if ((schemaname) && \ strncmp(schemaname, INTERNAL_SCHEMA_NAME, strlen(INTERNAL_SCHEMA_NAME)) == 0) \ (newuid) = ts_catalog_database_info_get()->owner_uid; \ else \ (newuid) = InvalidOid; \ if (OidIsValid((newuid))) \ { \ GetUserIdAndSecContext(&(saved_uid), &(saved_secctx)); \ SetUserIdAndSecContext(uid, (saved_secctx) | SECURITY_LOCAL_USERID_CHANGE); \ } \ } while (0) #define RESTORE_USER(newuid, saved_uid, saved_secctx) \ do \ { \ if (OidIsValid((newuid))) \ SetUserIdAndSecContext(saved_uid, saved_secctx); \ } while (0); typedef enum ContinuousAggViewType { ContinuousAggUserView = 0, ContinuousAggPartialView, ContinuousAggDirectView, ContinuousAggAnyView } ContinuousAggViewType; typedef enum ContinuousAggInvalidateUsing { ContinuousAggInvalidateUsingDefault = 0, ContinuousAggInvalidateUsingTrigger, ContinuousAggInvalidateUsingWal, } ContinuousAggInvalidateUsing; /* * Information about the bucketing function. */ typedef struct ContinuousAggBucketFunction { /* Oid of the bucketing function. In the catalog table, the regprocedure is used. This ensures * that the Oid is mapped to a string when a backup is taken and the string is converted back to * the Oid when the backup is restored. This way, we can use an Oid in the catalog table even * when a backup is restored and the Oid may have changed. However, the dependency management in * PostgreSQL does not track the Oid. If the function is dropped and a new one is created, the * Oid changes and this value points to a non-existing Oid. This can not happen in real-world * situations since PostgreSQL protects the bucket_function from deletion until the CAgg is * defined. */ Oid bucket_function; Oid bucket_width_type; /* type of bucket_width */ /* Is the interval of the bucket fixed? */ bool bucket_fixed_interval; /* Is the bucket defined on a time datatype ?*/ bool bucket_time_based; /* * Fields that are used for time based buckets */ Interval *bucket_time_width; /* * Custom origin value stored as UTC timestamp. * If not specified, stores infinity. */ TimestampTz bucket_time_origin; /* * Bucket offset. Note that we don't support * both offset and origin at the same time */ Interval *bucket_time_offset; char *bucket_time_timezone; /* * Fields that are used on integer based buckets */ int64 bucket_integer_width; int64 bucket_integer_offset; } ContinuousAggBucketFunction; typedef struct ContinuousAgg { FormData_continuous_agg data; /* Info about the time bucketing function */ ContinuousAggBucketFunction *bucket_function; /* Relid of the user-facing view */ Oid relid; /* Type of the primary partitioning dimension */ Oid partition_type; } ContinuousAgg; static inline bool ContinuousAggIsHierarchical(const ContinuousAgg *cagg) { return (cagg->data.parent_mat_hypertable_id != INVALID_HYPERTABLE_ID); } typedef enum ContinuousAggHypertableStatus { HypertableIsNotContinuousAgg = 0, HypertableIsMaterialization = 1, HypertableIsRawTable = 2, HypertableIsMaterializationAndRaw = HypertableIsMaterialization | HypertableIsRawTable, } ContinuousAggHypertableStatus; typedef struct ContinuousAggInfo { /* (int32) elements */ List *mat_hypertable_ids; /* (const ContinuousAggBucketFunction *) elements; stores NULL for fixed buckets */ List *bucket_functions; } ContinuousAggInfo; typedef struct ContinuousAggPolicyOffset { Datum value; Oid type; bool isnull; const char *name; } ContinuousAggPolicyOffset; extern TSDLLEXPORT Oid ts_cagg_permissions_check(Oid cagg_oid, Oid userid); extern TSDLLEXPORT ContinuousAggInfo ts_continuous_agg_get_all_caggs_info(int32 raw_hypertable_id); extern TSDLLEXPORT ContinuousAgg * ts_continuous_agg_find_by_mat_hypertable_id(int32 mat_hypertable_id, bool missing_ok); extern TSDLLEXPORT ContinuousAggHypertableStatus ts_continuous_agg_hypertable_status(int32 hypertable_id); extern TSDLLEXPORT List *ts_continuous_aggs_find_by_raw_table_id(int32 raw_hypertable_id); extern TSDLLEXPORT ContinuousAgg *ts_continuous_agg_find_by_view_name(const char *schema, const char *name, ContinuousAggViewType type); extern TSDLLEXPORT ContinuousAgg *ts_continuous_agg_find_by_relid(Oid relid); extern TSDLLEXPORT ContinuousAgg *ts_continuous_agg_find_by_rv(const RangeVar *rv); extern bool ts_continuous_agg_drop(const char *view_schema, const char *view_name); extern void ts_continuous_agg_drop_hypertable_callback(int32 hypertable_id); extern TSDLLEXPORT ContinuousAggViewType ts_continuous_agg_view_type(FormData_continuous_agg *data, const char *schema, const char *name); extern TSDLLEXPORT void ts_continuous_agg_rename_schema_name(const char *old_schema, const char *new_schema); extern TSDLLEXPORT void ts_continuous_agg_rename_view(const char *old_schema, const char *old_name, const char *new_schema, const char *new_name, ObjectType *object_type); extern TSDLLEXPORT const Dimension * ts_continuous_agg_find_integer_now_func_by_materialization_id(int32 mat_htid); extern ContinuousAgg *ts_continuous_agg_find_userview_name(const char *schema, const char *name); extern TSDLLEXPORT void ts_continuous_agg_invalidate_chunk(Hypertable *ht, Chunk *chunk); extern TSDLLEXPORT bool ts_continuous_agg_bucket_on_interval(Oid bucket_function); extern TSDLLEXPORT void ts_compute_inscribed_bucketed_refresh_window_variable(int64 *start, int64 *end, const ContinuousAggBucketFunction *bf); extern TSDLLEXPORT void ts_compute_circumscribed_bucketed_refresh_window_variable(int64 *start, int64 *end, const ContinuousAggBucketFunction *bf); extern TSDLLEXPORT int64 ts_compute_beginning_of_the_next_bucket_variable( int64 timeval, const ContinuousAggBucketFunction *bf); extern TSDLLEXPORT Query *ts_continuous_agg_get_query(ContinuousAgg *cagg); extern TSDLLEXPORT int64 ts_continuous_agg_fixed_bucket_width(const ContinuousAggBucketFunction *bucket_function); extern TSDLLEXPORT int64 ts_continuous_agg_bucket_width(const ContinuousAggBucketFunction *bucket_function); extern TSDLLEXPORT void ts_get_invalidation_replication_slot_name(char *slotname, Size szslot); ================================================ FILE: src/ts_catalog/continuous_aggs_watermark.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ /* * This file handles continuous aggs watermark functions. */ #include <postgres.h> #include <access/xact.h> #include <fmgr.h> #include <miscadmin.h> #include <utils/acl.h> #include <utils/inval.h> #include <utils/snapmgr.h> #include "debug_point.h" #include "guc.h" #include "hypertable.h" #include "ts_catalog/continuous_agg.h" #include "ts_catalog/continuous_aggs_watermark.h" static void cagg_watermark_init_scan_by_mat_hypertable_id(ScanIterator *iterator, const int32 mat_hypertable_id) { iterator->ctx.index = catalog_get_index(ts_catalog_get(), CONTINUOUS_AGGS_WATERMARK, CONTINUOUS_AGGS_WATERMARK_PKEY); ts_scan_iterator_scan_key_init(iterator, Anum_continuous_aggs_watermark_pkey_mat_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(mat_hypertable_id)); } int64 ts_cagg_watermark_get(int32 hypertable_id) { PG_USED_FOR_ASSERTS_ONLY short count = 0; Datum watermark = UnassignedDatum; bool value_isnull = true; ScanIterator iterator = ts_scan_iterator_create(CONTINUOUS_AGGS_WATERMARK, AccessShareLock, CurrentMemoryContext); /* * The watermark of a CAGG has to be fetched by using the transaction snapshot. * * By default, the ts_scanner uses the SnapshotSelf to perform a scan. However, reading the * watermark must be done using the transaction snapshot in order to ensure that the view on the * watermark and the materialized part of the CAGG match. */ iterator.ctx.snapshot = RegisterSnapshot(GetTransactionSnapshot()); Assert(iterator.ctx.snapshot != NULL); cagg_watermark_init_scan_by_mat_hypertable_id(&iterator, hypertable_id); ts_scanner_foreach(&iterator) { watermark = slot_getattr(ts_scan_iterator_slot(&iterator), Anum_continuous_aggs_watermark_watermark, &value_isnull); count++; } Assert(count <= 1); UnregisterSnapshot(iterator.ctx.snapshot); ts_scan_iterator_close(&iterator); if (value_isnull) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("watermark not defined for continuous aggregate: %d", hypertable_id))); /* Log the read watermark, needed for MVCC tap tests */ ereport(DEBUG5, (errcode(ERRCODE_SUCCESSFUL_COMPLETION), errmsg("watermark for continuous aggregate, '%d' is: " INT64_FORMAT, hypertable_id, DatumGetInt64(watermark)))); return DatumGetInt64(watermark); } TS_FUNCTION_INFO_V1(ts_continuous_agg_watermark); /* * Get the watermark for a real-time aggregation query on a continuous * aggregate. * * The watermark determines where the materialization ends for a continuous * aggregate. It is used by real-time aggregation as the threshold between the * materialized data and real-time data in the UNION query. * * The watermark is stored into `_timescaledb_catalog.continuous_aggs_watermark` * catalog table by the `refresh_continuous_agregate` procedure. It is defined * as the end of the last (highest) bucket in the materialized hypertable of a * continuous aggregate. * * The materialized hypertable ID is given as input argument. */ Datum ts_continuous_agg_watermark(PG_FUNCTION_ARGS) { const int32 mat_hypertable_id = PG_GETARG_INT32(0); ContinuousAgg *cagg; AclResult aclresult; cagg = ts_continuous_agg_find_by_mat_hypertable_id(mat_hypertable_id, false); /* * Preemptive permission check to ensure the function complains about lack * of permissions on the cagg rather than the materialized hypertable */ aclresult = pg_class_aclcheck(cagg->relid, GetUserId(), ACL_SELECT); aclcheck_error(aclresult, OBJECT_MATVIEW, get_rel_name(cagg->relid)); int64 watermark = ts_cagg_watermark_get(cagg->data.mat_hypertable_id); PG_RETURN_INT64(watermark); } static int64 cagg_compute_watermark(ContinuousAgg *cagg, int64 watermark, bool isnull) { if (isnull) { watermark = ts_time_get_min(cagg->partition_type); } else { /* * The materialized hypertable is already bucketed, which means the * max is the start of the last bucket. Add one bucket to move to the * point where the materialized data ends. */ if (cagg->bucket_function->bucket_fixed_interval == false) { /* * Since `value` is already bucketed, `bucketed = true` flag can * be added to ts_compute_beginning_of_the_next_bucket_variable() as * an optimization, if necessary. */ watermark = ts_compute_beginning_of_the_next_bucket_variable(watermark, cagg->bucket_function); } else { watermark = ts_time_saturating_add(watermark, ts_continuous_agg_fixed_bucket_width(cagg->bucket_function), cagg->partition_type); } } return watermark; } TS_FUNCTION_INFO_V1(ts_continuous_agg_watermark_materialized); /* * Get the materialized watermark for a real-time aggregation query on a * continuous aggregate. * * The difference between this function and `ts_continuous_agg_watermark` is * that this one get the max open dimension of the materialization hypertable * instead of get the stored value in the catalog table. */ Datum ts_continuous_agg_watermark_materialized(PG_FUNCTION_ARGS) { const int32 mat_hypertable_id = PG_GETARG_INT32(0); ContinuousAgg *cagg; AclResult aclresult; bool isnull; Hypertable *ht; int64 watermark; cagg = ts_continuous_agg_find_by_mat_hypertable_id(mat_hypertable_id, false); /* * Preemptive permission check to ensure the function complains about lack * of permissions on the cagg rather than the materialized hypertable */ aclresult = pg_class_aclcheck(cagg->relid, GetUserId(), ACL_SELECT); aclcheck_error(aclresult, OBJECT_MATVIEW, get_rel_name(cagg->relid)); ht = ts_hypertable_get_by_id(cagg->data.mat_hypertable_id); watermark = ts_hypertable_get_open_dim_max_value(ht, 0, &isnull); watermark = cagg_compute_watermark(cagg, watermark, isnull); PG_RETURN_INT64(watermark); } TSDLLEXPORT void ts_cagg_watermark_insert(Hypertable *mat_ht, int64 watermark, bool watermark_isnull) { Catalog *catalog = ts_catalog_get(); Relation rel = table_open(catalog_get_table_id(catalog, CONTINUOUS_AGGS_WATERMARK), RowExclusiveLock); TupleDesc desc = RelationGetDescr(rel); Datum values[Natts_continuous_aggs_watermark]; bool nulls[Natts_continuous_aggs_watermark] = { false }; CatalogSecurityContext sec_ctx; /* if trying to insert a NULL watermark then get the MIN value for the time dimension */ if (watermark_isnull) { const Dimension *dim = hyperspace_get_open_dimension(mat_ht->space, 0); if (NULL == dim) elog(ERROR, "invalid open dimension index %d", 0); watermark = ts_time_get_min(ts_dimension_get_partition_type(dim)); } values[AttrNumberGetAttrOffset(Anum_continuous_aggs_watermark_mat_hypertable_id)] = Int32GetDatum(mat_ht->fd.id); values[AttrNumberGetAttrOffset(Anum_continuous_aggs_watermark_watermark)] = Int64GetDatum(watermark); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_insert_values(rel, desc, values, nulls); ts_catalog_restore_user(&sec_ctx); table_close(rel, NoLock); } typedef struct WatermarkUpdate { int64 watermark; bool force_update; bool invalidate_rel_cache; Oid ht_relid; } WatermarkUpdate; static ScanTupleResult cagg_watermark_update_scan_internal(TupleInfo *ti, void *data) { WatermarkUpdate *watermark_update = data; bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); Form_continuous_aggs_watermark form = (Form_continuous_aggs_watermark) GETSTRUCT(tuple); /* If the tuple was modified concurrently, retry the operation and use a new snapshot * to see the updated tuple. */ if (ti->lockresult == TM_Updated) return SCAN_RESTART_WITH_NEW_SNAPSHOT; Ensure(ti->lockresult == TM_Ok, "unable to lock watermark tuple for cagg %d (lock result %d)", watermark_update->ht_relid, ti->lockresult); if (watermark_update->watermark > form->watermark || watermark_update->force_update) { HeapTuple new_tuple = heap_copytuple(tuple); form = (Form_continuous_aggs_watermark) GETSTRUCT(new_tuple); form->watermark = watermark_update->watermark; ts_catalog_update(ti->scanrel, new_tuple); heap_freetuple(new_tuple); /* * During query planning, the values of the watermark function are constified using the * constify_cagg_watermark() function. However, this function's value changes when we update * the Cagg (the volatility of the function is STABLE not IMMUTABLE). To ensure that caches, * such as the query plan cache, are properly evicted, we send an invalidation message for * the hypertable. */ if (watermark_update->invalidate_rel_cache) { DEBUG_WAITPOINT("cagg_watermark_update_internal_before_refresh"); CacheInvalidateRelcacheByRelid(watermark_update->ht_relid); } } else { elog(DEBUG1, "hypertable %d existing watermark >= new watermark " INT64_FORMAT " " INT64_FORMAT, form->mat_hypertable_id, form->watermark, watermark_update->watermark); watermark_update->watermark = form->watermark; } if (should_free) heap_freetuple(tuple); return SCAN_DONE; } static void cagg_watermark_update_internal(int32 mat_hypertable_id, Oid ht_relid, int64 new_watermark, bool force_update, bool invalidate_rel_cache) { WatermarkUpdate data = { .watermark = new_watermark, .force_update = force_update, .invalidate_rel_cache = invalidate_rel_cache, .ht_relid = ht_relid }; ScanIterator iterator = ts_scan_iterator_create(CONTINUOUS_AGGS_WATERMARK, RowExclusiveLock, CurrentMemoryContext); cagg_watermark_init_scan_by_mat_hypertable_id(&iterator, mat_hypertable_id); iterator.ctx.tuple_found = cagg_watermark_update_scan_internal; iterator.ctx.data = &data; iterator.ctx.snapshot = RegisterSnapshot(GetLatestSnapshot()); ScanTupLock scantuplock = { .waitpolicy = LockWaitBlock, .lockmode = LockTupleExclusive, .lockflags = TUPLE_LOCK_FLAG_FIND_LAST_VERSION, }; iterator.ctx.tuplock = &scantuplock; iterator.ctx.flags = SCANNER_F_KEEPLOCK; bool watermark_updated = ts_scanner_scan_one(&iterator.ctx, false, "continuous aggregate watermark"); UnregisterSnapshot(iterator.ctx.snapshot); if (!watermark_updated) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("watermark not defined for continuous aggregate: %d", mat_hypertable_id))); } } TSDLLEXPORT void ts_cagg_watermark_update(Hypertable *mat_ht, int64 watermark, bool watermark_isnull, bool force_update) { ContinuousAgg *cagg = ts_continuous_agg_find_by_mat_hypertable_id(mat_ht->fd.id, false); /* If we have a real-time CAgg, it uses a watermark function. So, we have to invalidate the rel * cache to force a replanning of prepared statements. See cagg_watermark_update_internal for * more information. If the GUC enable_cagg_watermark_constify=false then it's not necessary * to invalidate relation cache. */ bool invalidate_rel_cache = !cagg->data.materialized_only && ts_guc_enable_cagg_watermark_constify; watermark = cagg_compute_watermark(cagg, watermark, watermark_isnull); cagg_watermark_update_internal(mat_ht->fd.id, mat_ht->main_table_relid, watermark, force_update, invalidate_rel_cache); } TSDLLEXPORT void ts_cagg_watermark_delete_by_mat_hypertable_id(int32 mat_hypertable_id) { ScanIterator iterator = ts_scan_iterator_create(CONTINUOUS_AGGS_WATERMARK, RowExclusiveLock, CurrentMemoryContext); cagg_watermark_init_scan_by_mat_hypertable_id(&iterator, mat_hypertable_id); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); } ts_scan_iterator_close(&iterator); } ================================================ FILE: src/ts_catalog/continuous_aggs_watermark.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include "export.h" #include "hypertable.h" extern TSDLLEXPORT void ts_cagg_watermark_delete_by_mat_hypertable_id(int32 mat_hypertable_id); extern TSDLLEXPORT void ts_cagg_watermark_insert(Hypertable *mat_ht, int64 watermark, bool watermark_isnull); extern TSDLLEXPORT void ts_cagg_watermark_update(Hypertable *mat_ht, int64 watermark, bool watermark_isnull, bool force_update); extern TSDLLEXPORT int64 ts_cagg_watermark_get(int32 hypertable_id); ================================================ FILE: src/ts_catalog/metadata.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <unistd.h> #include <postgres.h> #include <access/htup_details.h> #include <catalog/pg_type.h> #include <fmgr.h> #include <stdlib.h> #include <utils/builtins.h> #include <utils/datum.h> #include <utils/fmgroids.h> #include <utils/lsyscache.h> #include <utils/uuid.h> #include "scanner.h" #include "ts_catalog/catalog.h" #include "ts_catalog/metadata.h" #include "uuid.h" #include "compat/compat.h" #define TYPE_ERROR(inout, typeid) \ elog(ERROR, "ts_metadata: no %s function for type %u", inout, typeid); static Datum convert_type_to_text(Datum value, Oid from_type) { bool is_varlena; Oid outfunc; getTypeOutputInfo(from_type, &outfunc, &is_varlena); if (!OidIsValid(outfunc)) TYPE_ERROR("output", from_type); return DirectFunctionCall1(textin, OidFunctionCall1(outfunc, value)); } static Datum convert_text_to_type(Datum value, Oid to_type) { Oid value_in; Oid value_ioparam; getTypeInputInfo(to_type, &value_in, &value_ioparam); if (!OidIsValid(value_in)) TYPE_ERROR("input", to_type); value = OidFunctionCall3(value_in, CStringGetDatum(TextDatumGetCString(value)), ObjectIdGetDatum(InvalidOid), Int32GetDatum(-1)); return value; } typedef struct DatumValue { /* * This form is not used for anything. It is here to reference the type so * that pgindent works. It can be removed from this struct in case we * actually use the form type in code */ FormData_metadata *form; Datum value; Oid typeid; bool isnull; } DatumValue; static ScanTupleResult metadata_tuple_get_value(TupleInfo *ti, void *data) { DatumValue *dv = data; dv->value = slot_getattr(ti->slot, Anum_metadata_value, &dv->isnull); if (!dv->isnull) dv->value = convert_text_to_type(dv->value, dv->typeid); return SCAN_DONE; } static Datum metadata_get_value_internal(const char *key, Oid value_type, bool *isnull, LOCKMODE lockmode) { ScanKeyData scankey[1]; DatumValue dv = { .typeid = value_type, .isnull = true, }; Catalog *catalog = ts_catalog_get(); ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, METADATA), .index = catalog_get_index(catalog, METADATA, METADATA_PKEY_IDX), .nkeys = 1, .scankey = scankey, .tuple_found = metadata_tuple_get_value, .data = &dv, .lockmode = lockmode, .scandirection = ForwardScanDirection, }; ScanKeyInit(&scankey[0], Anum_metadata_key, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(key)); ts_scanner_scan(&scanctx); if (NULL != isnull) *isnull = dv.isnull; return dv.value; } Datum ts_metadata_get_value(const char *metadata_key, Oid value_type, bool *isnull) { return metadata_get_value_internal(metadata_key, value_type, isnull, AccessShareLock); } /* * Insert a row into the metadata table. Acquires a lock in * SHARE ROW EXCLUSIVE mode to conflict with itself, and then verifies that * the desired metadata KV pair still does not exist. Otherwise, exits * without inserting to avoid underlying database error on PK conflict. * Returns the value of the key; this is either the requested insert value or * the existing value if nothing was inserted. */ Datum ts_metadata_insert(const char *metadata_key, Datum metadata_value, Oid type, bool include_in_telemetry) { Datum existing_value; Datum values[Natts_metadata]; bool nulls[Natts_metadata] = { false }; bool isnull = false; Catalog *catalog = ts_catalog_get(); Relation rel; NameData key_data; rel = table_open(catalog_get_table_id(catalog, METADATA), ShareRowExclusiveLock); /* Check for row existence while we have the lock */ existing_value = metadata_get_value_internal(metadata_key, type, &isnull, ShareRowExclusiveLock); if (!isnull) { table_close(rel, ShareRowExclusiveLock); return existing_value; } /* We have to copy the key here because heap_form_tuple will copy NAMEDATALEN * into the tuple instead of checking length. */ namestrcpy(&key_data, metadata_key); /* Insert into the catalog table for persistence */ values[AttrNumberGetAttrOffset(Anum_metadata_key)] = NameGetDatum(&key_data); values[AttrNumberGetAttrOffset(Anum_metadata_value)] = convert_type_to_text(metadata_value, type); values[AttrNumberGetAttrOffset(Anum_metadata_include_in_telemetry)] = BoolGetDatum(include_in_telemetry); ts_catalog_insert_values(rel, RelationGetDescr(rel), values, nulls); table_close(rel, ShareRowExclusiveLock); return metadata_value; } static ScanTupleResult metadata_tuple_delete(TupleInfo *ti, void *data) { ts_catalog_delete_tid(ti->scanrel, ts_scanner_get_tuple_tid(ti)); return SCAN_CONTINUE; } void ts_metadata_drop(const char *metadata_key) { ScanKeyData scankey[1]; Catalog *catalog = ts_catalog_get(); ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, METADATA), .index = catalog_get_index(catalog, METADATA, METADATA_PKEY_IDX), .nkeys = 1, .scankey = scankey, .tuple_found = metadata_tuple_delete, .data = NULL, .lockmode = RowExclusiveLock, .scandirection = ForwardScanDirection, }; ScanKeyInit(&scankey[0], Anum_metadata_key, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(metadata_key)); ts_scanner_scan(&scanctx); } static Datum get_uuid_by_key(const char *key) { bool isnull; Datum uuid; uuid = ts_metadata_get_value(key, UUIDOID, &isnull); if (isnull) uuid = ts_metadata_insert(key, UUIDPGetDatum(ts_uuid_create()), UUIDOID, true); return uuid; } Datum ts_metadata_get_uuid(void) { return get_uuid_by_key(METADATA_UUID_KEY_NAME); } Datum ts_metadata_get_exported_uuid(void) { return get_uuid_by_key(METADATA_EXPORTED_UUID_KEY_NAME); } Datum ts_metadata_get_install_timestamp(void) { bool isnull; Datum timestamp; timestamp = ts_metadata_get_value(METADATA_TIMESTAMP_KEY_NAME, TIMESTAMPTZOID, &isnull); if (isnull) timestamp = ts_metadata_insert(METADATA_TIMESTAMP_KEY_NAME, TimestampTzGetDatum(GetCurrentTimestamp()), TIMESTAMPTZOID, true); return timestamp; } ================================================ FILE: src/ts_catalog/metadata.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include "export.h" #define METADATA_UUID_KEY_NAME "uuid" #define METADATA_EXPORTED_UUID_KEY_NAME "exported_uuid" #define METADATA_TIMESTAMP_KEY_NAME "install_timestamp" extern TSDLLEXPORT Datum ts_metadata_get_value(const char *metadata_key, Oid value_type, bool *isnull); extern TSDLLEXPORT Datum ts_metadata_insert(const char *metadata_key, Datum metadata_value, Oid value_type, bool include_in_telemetry); extern TSDLLEXPORT void ts_metadata_drop(const char *metadata_key); extern TSDLLEXPORT Datum ts_metadata_get_uuid(void); extern Datum ts_metadata_get_exported_uuid(void); extern Datum ts_metadata_get_install_timestamp(void); ================================================ FILE: src/ts_catalog/tablespace.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <catalog/pg_tablespace_d.h> #include <commands/tablecmds.h> #include <commands/tablespace.h> #include <fmgr.h> #include <funcapi.h> #include <miscadmin.h> #include <utils/acl.h> #include <utils/builtins.h> #include <utils/fmgroids.h> #include <utils/lsyscache.h> #include <utils/spccache.h> #include "compat/compat.h" #include "errors.h" #include "hypertable_cache.h" #include "scanner.h" #include "ts_catalog/catalog.h" #include "ts_catalog/tablespace.h" #include "utils.h" #define TABLESPACE_DEFAULT_CAPACITY 4 static Tablespaces * tablespaces_alloc(int capacity) { Tablespaces *tspcs; tspcs = palloc(sizeof(Tablespaces)); tspcs->capacity = capacity; tspcs->num_tablespaces = 0; tspcs->tablespaces = palloc(sizeof(Tablespace) * tspcs->capacity); return tspcs; } Tablespace * ts_tablespaces_add(Tablespaces *tablespaces, const FormData_tablespace *form, Oid tspc_oid) { Tablespace *tspc; if (tablespaces->num_tablespaces >= tablespaces->capacity) { tablespaces->capacity += TABLESPACE_DEFAULT_CAPACITY; Assert(tablespaces->tablespaces); /* repalloc() does not work with NULL argument */ tablespaces->tablespaces = repalloc(tablespaces->tablespaces, sizeof(Tablespace) * tablespaces->capacity); } tspc = &tablespaces->tablespaces[tablespaces->num_tablespaces++]; memcpy(&tspc->fd, form, sizeof(FormData_tablespace)); tspc->tablespace_oid = tspc_oid; return tspc; } bool ts_tablespaces_contain(const Tablespaces *tablespaces, Oid tspc_oid) { int i; for (i = 0; i < tablespaces->num_tablespaces; i++) if (tspc_oid == tablespaces->tablespaces[i].tablespace_oid) return true; return false; } static ScanTupleResult tablespace_tuple_found(TupleInfo *ti, void *data) { Tablespaces *tspcs = data; bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); FormData_tablespace *form = (FormData_tablespace *) GETSTRUCT(tuple); Oid tspcoid = get_tablespace_oid(NameStr(form->tablespace_name), true); if (NULL != tspcs) ts_tablespaces_add(tspcs, form, tspcoid); if (should_free) heap_freetuple(tuple); return SCAN_CONTINUE; } static int tablespace_scan_internal(int indexid, ScanKeyData *scankey, int nkeys, tuple_found_func tuple_found, tuple_filter_func tuple_filter, void *data, int limit, LOCKMODE lockmode) { Catalog *catalog = ts_catalog_get(); ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, TABLESPACE), .index = catalog_get_index(catalog, TABLESPACE, indexid), .nkeys = nkeys, .scankey = scankey, .tuple_found = tuple_found, .filter = tuple_filter, .data = data, .limit = limit, .lockmode = lockmode, .scandirection = ForwardScanDirection, }; return ts_scanner_scan(&scanctx); } Tablespaces * ts_tablespace_scan(int32 hypertable_id) { Tablespaces *tspcs = tablespaces_alloc(TABLESPACE_DEFAULT_CAPACITY); ScanKeyData scankey[1]; ScanKeyInit(&scankey[0], Anum_tablespace_hypertable_id_tablespace_name_idx_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); tablespace_scan_internal(TABLESPACE_HYPERTABLE_ID_TABLESPACE_NAME_IDX, scankey, 1, tablespace_tuple_found, NULL, tspcs, 0, AccessShareLock); return tspcs; } typedef struct TablespaceScanInfo { CatalogDatabaseInfo *database_info; Cache *hcache; Oid userid; int num_filtered; int stopcount; List *hypertables; /* Hypertables affected, where applicable */ void *data; } TablespaceScanInfo; static int tablespace_scan_by_name(const char *tspcname, tuple_found_func tuple_found, void *data) { ScanKeyData scankey[1]; int nkeys = 0; if (NULL != tspcname) ScanKeyInit(&scankey[nkeys++], Anum_tablespace_tablespace_name, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(tspcname)); return tablespace_scan_internal(INVALID_INDEXID, scankey, nkeys, tuple_found, NULL, data, 0, AccessShareLock); } int ts_tablespace_count_attached(const char *tspcname) { return tablespace_scan_by_name(tspcname, NULL, NULL); } static void tablespace_validate_revoke_internal(const char *tspcname, tuple_found_func tuple_found, void *stmt) { TablespaceScanInfo info = { .database_info = ts_catalog_database_info_get(), .hcache = ts_hypertable_cache_pin(), .data = stmt, }; tablespace_scan_by_name(tspcname, tuple_found, &info); ts_cache_release(&info.hcache); } static void validate_revoke_create(Oid tspcoid, Oid role, Oid relid) { AclResult aclresult = object_aclcheck(TableSpaceRelationId, tspcoid, role, ACL_CREATE); if (aclresult != ACLCHECK_OK) ereport(ERROR, (errcode(ERRCODE_DEPENDENT_OBJECTS_STILL_EXIST), errmsg("cannot revoke privilege while tablespace \"%s\" is attached to hypertable " "\"%s\"", get_tablespace_name(tspcoid), get_rel_name(relid)), errhint("Detach the tablespace before revoking the privilege on it."))); } /* * Verify that the REVOKE of permissions on a tablespace does not make it * impossible to use the tablespace for new chunks. * * This check should be done after the REVOKE has been applied. */ static ScanTupleResult revoke_tuple_found(TupleInfo *ti, void *data) { TablespaceScanInfo *info = data; GrantStmt *stmt = info->data; ListCell *lc_role; bool isnull; Datum hyper_id; Datum tablespace_name; Oid tspcoid; Hypertable *ht; Oid relowner; hyper_id = slot_getattr(ti->slot, Anum_tablespace_hypertable_id, &isnull); Assert(!isnull); tablespace_name = slot_getattr(ti->slot, Anum_tablespace_tablespace_name, &isnull); Assert(!isnull); tspcoid = get_tablespace_oid(NameStr(*DatumGetName(tablespace_name)), false); ht = ts_hypertable_cache_get_entry_by_id(info->hcache, DatumGetInt32(hyper_id)); relowner = ts_rel_get_owner(ht->main_table_relid); foreach (lc_role, stmt->grantees) { RoleSpec *role = lfirst(lc_role); Oid roleoid = get_role_oid_or_public(role->rolename); /* Check if this is a role we're interested in */ if (!OidIsValid(roleoid)) continue; /* * A revoke on a tablespace can only be for 'CREATE' (or ALL), so no * need to check which privilege is revoked. */ validate_revoke_create(tspcoid, relowner, ht->main_table_relid); } return SCAN_CONTINUE; } void ts_tablespace_validate_revoke(GrantStmt *stmt) { tablespace_validate_revoke_internal(strVal(linitial(stmt->objects)), revoke_tuple_found, stmt); } /* * Verify that the REVOKE of a role on a tablespace does not make it impossible * to use the tablespace for new chunks. * * This check should be done after the REVOKE has been applied. */ static ScanTupleResult revoke_role_tuple_found(TupleInfo *ti, void *data) { TablespaceScanInfo *info = data; GrantRoleStmt *stmt = info->data; bool isnull; Datum hyper_id; Datum tablespace_name; Oid tspcoid; Hypertable *ht; Oid relowner; ListCell *lc_role; hyper_id = slot_getattr(ti->slot, Anum_tablespace_hypertable_id, &isnull); Assert(!isnull); tablespace_name = slot_getattr(ti->slot, Anum_tablespace_tablespace_name, &isnull); Assert(!isnull); tspcoid = get_tablespace_oid(NameStr(*DatumGetName(tablespace_name)), false); ht = ts_hypertable_cache_get_entry_by_id(info->hcache, DatumGetInt32(hyper_id)); relowner = ts_rel_get_owner(ht->main_table_relid); foreach (lc_role, stmt->grantee_roles) { RoleSpec *rolespec = lfirst(lc_role); Oid grantee = get_rolespec_oid(rolespec, true); /* Only interested in revokes on table owners */ if (grantee != relowner) continue; /* * No need to check which role that was revoked since we are only * interested in the resulting permissions for the table owner. A * table owner could have CREATE on the tablespace from multiple * roles, so revoking one of those roles might not mean the owner no * longer has CREATE on the tablespace. */ validate_revoke_create(tspcoid, relowner, ht->main_table_relid); } return SCAN_CONTINUE; } void ts_tablespace_validate_revoke_role(GrantRoleStmt *stmt) { tablespace_validate_revoke_internal(NULL, revoke_role_tuple_found, stmt); } static int32 tablespace_insert_relation(Relation rel, int32 hypertable_id, const char *tspcname) { TupleDesc desc = RelationGetDescr(rel); Datum values[Natts_tablespace]; bool nulls[Natts_tablespace] = { false }; int32 id; memset(values, 0, sizeof(values)); id = ts_catalog_table_next_seq_id(ts_catalog_get(), TABLESPACE); values[AttrNumberGetAttrOffset(Anum_tablespace_id)] = Int32GetDatum(id); values[AttrNumberGetAttrOffset(Anum_tablespace_hypertable_id)] = Int32GetDatum(hypertable_id); values[AttrNumberGetAttrOffset(Anum_tablespace_tablespace_name)] = DirectFunctionCall1(namein, CStringGetDatum(tspcname)); ts_catalog_insert_values(rel, desc, values, nulls); return id; } static int32 tablespace_insert(int32 hypertable_id, const char *tspcname) { Catalog *catalog = ts_catalog_get(); Relation rel; int32 id; rel = table_open(catalog_get_table_id(catalog, TABLESPACE), RowExclusiveLock); id = tablespace_insert_relation(rel, hypertable_id, tspcname); table_close(rel, RowExclusiveLock); return id; } static ScanTupleResult tablespace_tuple_delete(TupleInfo *ti, void *data) { TablespaceScanInfo *info = data; bool should_free; CatalogSecurityContext sec_ctx; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); FormData_tablespace *form = (FormData_tablespace *) GETSTRUCT(tuple); ts_catalog_database_info_become_owner(info->database_info, &sec_ctx); ts_catalog_delete_tid_only(ti->scanrel, ts_scanner_get_tuple_tid(ti)); ts_catalog_restore_user(&sec_ctx); info->hypertables = lappend_int(info->hypertables, form->hypertable_id); if (should_free) heap_freetuple(tuple); return (info->stopcount == 0 || ti->count < info->stopcount) ? SCAN_CONTINUE : SCAN_DONE; } int ts_tablespace_delete(int32 hypertable_id, const char *tspcname, Oid tspcoid) { ScanKeyData scankey[2]; TablespaceScanInfo info = { .database_info = ts_catalog_database_info_get(), .stopcount = (NULL != tspcname), }; int num_deleted, nkeys = 0; ScanKeyInit(&scankey[nkeys++], Anum_tablespace_hypertable_id_tablespace_name_idx_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); if (NULL != tspcname) ScanKeyInit(&scankey[nkeys++], Anum_tablespace_hypertable_id_tablespace_name_idx_tablespace_name, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(tspcname)); num_deleted = tablespace_scan_internal(TABLESPACE_HYPERTABLE_ID_TABLESPACE_NAME_IDX, scankey, nkeys, tablespace_tuple_delete, NULL, &info, 0, RowExclusiveLock); if (num_deleted > 0) CommandCounterIncrement(); return num_deleted; } static ScanFilterResult tablespace_tuple_owner_filter(const TupleInfo *ti, void *data) { TablespaceScanInfo *info = data; bool isnull; Datum hyper_id; Hypertable *ht; ScanFilterResult result; hyper_id = slot_getattr(ti->slot, Anum_tablespace_hypertable_id, &isnull); Assert(!isnull); ht = ts_hypertable_cache_get_entry_by_id(info->hcache, DatumGetInt32(hyper_id)); Assert(NULL != ht); if (ts_hypertable_has_privs_of(ht->main_table_relid, info->userid)) result = SCAN_INCLUDE; else { result = SCAN_EXCLUDE; info->num_filtered++; } return result; } /* * Detach a tablespace from all hypertables it is attached to. * * Output parameters: * - `hypertables`: the list of hypertables that the tablespace was removed from. * * Returns: * integer giving the number of tablespaces deleted. */ static int tablespace_delete_from_all(const char *tspcname, Oid userid, List **hypertables) { ScanKeyData scankey[1]; TablespaceScanInfo info = { .database_info = ts_catalog_database_info_get(), .hcache = ts_hypertable_cache_pin(), .userid = userid, }; int num_deleted; ScanKeyInit(&scankey[0], Anum_tablespace_tablespace_name, BTEqualStrategyNumber, F_NAMEEQ, CStringGetDatum(tspcname)); num_deleted = tablespace_scan_internal(INVALID_INDEXID, scankey, 1, tablespace_tuple_delete, tablespace_tuple_owner_filter, &info, 0, RowExclusiveLock); ts_cache_release(&info.hcache); if (num_deleted > 0) CommandCounterIncrement(); if (info.num_filtered > 0) ereport(NOTICE, (errmsg("tablespace \"%s\" remains attached to %d hypertable(s) due to lack of " "permissions", tspcname, info.num_filtered))); *hypertables = info.hypertables; return num_deleted; } TS_FUNCTION_INFO_V1(ts_tablespace_attach); Datum ts_tablespace_attach(PG_FUNCTION_ARGS) { Name tspcname = PG_ARGISNULL(0) ? NULL : PG_GETARG_NAME(0); Oid hypertable_oid = PG_ARGISNULL(1) ? InvalidOid : PG_GETARG_OID(1); bool if_not_attached = PG_ARGISNULL(2) ? false : PG_GETARG_BOOL(2); Relation rel; TS_PREVENT_FUNC_IF_READ_ONLY(); if (PG_NARGS() < 2 || PG_NARGS() > 3) elog(ERROR, "invalid number of arguments"); ts_tablespace_attach_internal(tspcname, hypertable_oid, if_not_attached); /* If the hypertable did not have a tablespace assigned, we set one */ rel = relation_open(hypertable_oid, AccessShareLock); if (!OidIsValid(rel->rd_rel->reltablespace)) { AlterTableCmd *const cmd = makeNode(AlterTableCmd); cmd->subtype = AT_SetTableSpace; cmd->name = NameStr(*tspcname); ts_alter_table_with_event_trigger(hypertable_oid, fcinfo->context, list_make1(cmd), false); } relation_close(rel, AccessShareLock); PG_RETURN_VOID(); } void ts_tablespace_attach_internal(Name tspcname, Oid hypertable_oid, bool if_not_attached) { Cache *hcache; Hypertable *ht; Oid tspc_oid; Oid ownerid; AclResult aclresult; CatalogSecurityContext sec_ctx; if (NULL == tspcname) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid tablespace name"))); if (!OidIsValid(hypertable_oid)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid hypertable"))); tspc_oid = get_tablespace_oid(NameStr(*tspcname), true); if (!OidIsValid(tspc_oid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("tablespace \"%s\" does not exist", NameStr(*tspcname)), errhint("The tablespace needs to be created" " before attaching it to a hypertable."))); ownerid = ts_hypertable_permissions_check(hypertable_oid, GetUserId()); /* * Only check permissions on tablespace if it is not the database default. * In usual case users can create tables in their database which will use * the default tablespace of the database. This condition makes sure they * can also always move a table from another tablespace to the default of * their own database. Related to this issue in postgres core: * https://www.postgresql.org/message-id/52DC8AEA.7090507%402ndquadrant.com * Which was handled in a similar way. (See * tablecmds.c::ATPrepSetTableSpace) */ if (tspc_oid != MyDatabaseTableSpace) { /* * Note that we check against the table owner rather than the current * user here, since we're not actually creating a table using this * tablespace at this point */ aclresult = object_aclcheck(TableSpaceRelationId, tspc_oid, ownerid, ACL_CREATE); if (aclresult != ACLCHECK_OK) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permission denied for tablespace \"%s\" by table owner \"%s\"", NameStr(*tspcname), GetUserNameFromId(ownerid, true)))); } ht = ts_hypertable_cache_get_cache_and_entry(hypertable_oid, CACHE_FLAG_NONE, &hcache); if (ts_hypertable_has_tablespace(ht, tspc_oid)) { if (if_not_attached) ereport(NOTICE, (errcode(ERRCODE_TS_TABLESPACE_ALREADY_ATTACHED), errmsg("tablespace \"%s\" is already attached to hypertable \"%s\", skipping", NameStr(*tspcname), get_rel_name(hypertable_oid)))); else ereport(ERROR, (errcode(ERRCODE_TS_TABLESPACE_ALREADY_ATTACHED), errmsg("tablespace \"%s\" is already attached to hypertable \"%s\"", NameStr(*tspcname), get_rel_name(hypertable_oid)))); } else { ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); tablespace_insert(ht->fd.id, NameStr(*tspcname)); ts_catalog_restore_user(&sec_ctx); } ts_cache_release(&hcache); } static int tablespace_detach_one(Oid hypertable_oid, const char *tspcname, Oid tspcoid, bool if_attached) { Cache *hcache; Hypertable *ht; int ret = 0; ts_hypertable_permissions_check(hypertable_oid, GetUserId()); ht = ts_hypertable_cache_get_cache_and_entry(hypertable_oid, CACHE_FLAG_NONE, &hcache); if (ts_hypertable_has_tablespace(ht, tspcoid)) ret = ts_tablespace_delete(ht->fd.id, tspcname, tspcoid); else if (if_attached) ereport(NOTICE, (errcode(ERRCODE_TS_TABLESPACE_NOT_ATTACHED), errmsg("tablespace \"%s\" is not attached to hypertable \"%s\", skipping", tspcname, get_rel_name(hypertable_oid)))); else ereport(ERROR, (errcode(ERRCODE_TS_TABLESPACE_NOT_ATTACHED), errmsg("tablespace \"%s\" is not attached to hypertable \"%s\"", tspcname, get_rel_name(hypertable_oid)))); ts_cache_release(&hcache); return ret; } static int tablespace_detach_all(Oid hypertable_oid) { Cache *hcache; Hypertable *ht; int ret; ts_hypertable_permissions_check(hypertable_oid, GetUserId()); ht = ts_hypertable_cache_get_cache_and_entry(hypertable_oid, CACHE_FLAG_NONE, &hcache); ret = ts_tablespace_delete(ht->fd.id, NULL, InvalidOid); ts_cache_release(&hcache); return ret; } static void detach_tablespace_from_hypertable_if_set(Node *detach_cmd, Oid hypertable_oid, Oid tspcoid) { Relation rel; Assert(OidIsValid(hypertable_oid) && OidIsValid(tspcoid)); rel = relation_open(hypertable_oid, AccessShareLock); if (OidIsValid(rel->rd_rel->reltablespace) && rel->rd_rel->reltablespace == tspcoid) { AlterTableCmd *const cmd = makeNode(AlterTableCmd); cmd->subtype = AT_SetTableSpace; cmd->name = "pg_default"; ts_alter_table_with_event_trigger(hypertable_oid, detach_cmd, list_make1(cmd), false); } relation_close(rel, AccessShareLock); } TS_FUNCTION_INFO_V1(ts_tablespace_detach); Datum ts_tablespace_detach(PG_FUNCTION_ARGS) { Name tspcname = PG_ARGISNULL(0) ? NULL : PG_GETARG_NAME(0); Oid hypertable_oid = PG_ARGISNULL(1) ? InvalidOid : PG_GETARG_OID(1); bool if_attached = PG_ARGISNULL(2) ? false : PG_GETARG_BOOL(2); Oid tspcoid; int ret; TS_PREVENT_FUNC_IF_READ_ONLY(); if (PG_NARGS() < 1 || PG_NARGS() > 3) elog(ERROR, "invalid number of arguments"); if (NULL == tspcname) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid tablespace name"))); if (!PG_ARGISNULL(1) && !OidIsValid(hypertable_oid)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid hypertable"))); tspcoid = get_tablespace_oid(NameStr(*tspcname), true); if (!OidIsValid(tspcoid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("tablespace \"%s\" does not exist", NameStr(*tspcname)))); if (OidIsValid(hypertable_oid)) { ret = tablespace_detach_one(hypertable_oid, NameStr(*tspcname), tspcoid, if_attached); detach_tablespace_from_hypertable_if_set(fcinfo->context, hypertable_oid, tspcoid); } else { List *hypertables = NIL; ListCell *cell; ret = tablespace_delete_from_all(NameStr(*tspcname), GetUserId(), &hypertables); foreach (cell, hypertables) { const int32 hypertable_id = lfirst_int(cell); detach_tablespace_from_hypertable_if_set(fcinfo->context, ts_hypertable_id_to_relid(hypertable_id, false), tspcoid); } } PG_RETURN_INT32(ret); } TS_FUNCTION_INFO_V1(ts_tablespace_detach_all_from_hypertable); Datum ts_tablespace_detach_all_from_hypertable(PG_FUNCTION_ARGS) { const Oid hypertable_relid = PG_GETARG_OID(0); int32 result; AlterTableCmd *const cmd = makeNode(AlterTableCmd); cmd->subtype = AT_SetTableSpace; cmd->name = "pg_default"; TS_PREVENT_FUNC_IF_READ_ONLY(); if (PG_NARGS() != 1) elog(ERROR, "invalid number of arguments"); if (PG_ARGISNULL(0)) elog(ERROR, "invalid argument"); result = tablespace_detach_all(hypertable_relid); ts_alter_table_with_event_trigger(hypertable_relid, fcinfo->context, list_make1(cmd), false); PG_RETURN_INT32(result); } TS_FUNCTION_INFO_V1(ts_tablespace_show); Datum ts_tablespace_show(PG_FUNCTION_ARGS) { FuncCallContext *funcctx; Oid hypertable_oid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); Cache *hcache; Hypertable *ht; Tablespaces *tspcs; if (SRF_IS_FIRSTCALL()) { MemoryContext oldcontext; if (!OidIsValid(hypertable_oid)) elog(ERROR, "invalid argument"); funcctx = SRF_FIRSTCALL_INIT(); oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); funcctx->user_fctx = ts_hypertable_cache_pin(); MemoryContextSwitchTo(oldcontext); } funcctx = SRF_PERCALL_SETUP(); hcache = funcctx->user_fctx; ht = ts_hypertable_cache_get_entry(hcache, hypertable_oid, CACHE_FLAG_NONE); tspcs = ts_tablespace_scan(ht->fd.id); if (NULL != tspcs && funcctx->call_cntr < (uint64) tspcs->num_tablespaces) { Oid tablespace_oid = tspcs->tablespaces[funcctx->call_cntr].tablespace_oid; const char *tablespace_name = get_tablespace_name(tablespace_oid); Datum name; Assert(tablespace_name != NULL); name = DirectFunctionCall1(namein, CStringGetDatum(tablespace_name)); SRF_RETURN_NEXT(funcctx, name); } else { ts_cache_release(&hcache); SRF_RETURN_DONE(funcctx); } } ================================================ FILE: src/ts_catalog/tablespace.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/parsenodes.h> #include "ts_catalog/catalog.h" typedef struct Tablespace { FormData_tablespace fd; Oid tablespace_oid; } Tablespace; typedef struct Tablespaces { int capacity; int num_tablespaces; Tablespace *tablespaces; } Tablespaces; extern Tablespace *ts_tablespaces_add(Tablespaces *tablespaces, const FormData_tablespace *form, Oid tspc_oid); extern bool ts_tablespaces_contain(const Tablespaces *tablespaces, Oid tspc_oid); extern Tablespaces *ts_tablespace_scan(int32 hypertable_id); extern TSDLLEXPORT void ts_tablespace_attach_internal(Name tspcname, Oid hypertable_oid, bool if_not_attached); extern int ts_tablespace_delete(int32 hypertable_id, const char *tspcname, Oid tspcoid); extern int ts_tablespace_count_attached(const char *tspcname); extern void ts_tablespace_validate_revoke(GrantStmt *stmt); extern void ts_tablespace_validate_revoke_role(GrantRoleStmt *stmt); ================================================ FILE: src/tss_callbacks.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ /* * Currently we finish the execution of some process utility statements * and don't execute other hooks in the chain. * * Because that reason neither ts_stat_statements and pg_stat_statements * are able to track some utility statements, for example COPY ... FROM. * * To be able to track it on ts_stat_statements here we introduce some * callbacks in order to hook pgss_store from TimescaleDB and store * information about the execution of those statements. * * Hooking ts_stat_statements from TimescaleDB is controlled by a new GUC * named `enable_tss_callbacks`. */ #include <postgres.h> #include <executor/instrument.h> #include <fmgr.h> #include <utils/elog.h> #include "guc.h" #include "tss_callbacks.h" static instr_time tss_callback_start_time; static BufferUsage tss_callback_start_bufusage; static WalUsage tss_callback_start_walusage; static TSSCallbacks * ts_get_tss_callbacks(void) { TSSCallbacks **ptr = (TSSCallbacks **) find_rendezvous_variable(TSS_CALLBACKS_VAR_NAME); return *ptr; } static tss_store_hook_type ts_get_tss_store_hook(void) { TSSCallbacks *ptr = ts_get_tss_callbacks(); if (ptr && ptr->version_num == TSS_CALLBACKS_VERSION) return ptr->tss_store_hook; return NULL; } bool ts_is_tss_enabled(void) { if (ts_guc_enable_tss_callbacks) { TSSCallbacks *ptr = ts_get_tss_callbacks(); if (ptr) { if (ptr->version_num != TSS_CALLBACKS_VERSION) { ereport(WARNING, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("version mismatch between timescaledb and ts_stat_statements " "callbacks"), errdetail("Callbacks versions: TimescaleDB (%d) and ts_stat_statements " "(%d)", TSS_CALLBACKS_VERSION, ptr->version_num))); return false; } return ptr->tss_enabled_hook_type(0); /* consider top level statement */ } } return false; } void ts_begin_tss_store_callback(void) { if (!ts_is_tss_enabled()) return; tss_callback_start_bufusage = pgBufferUsage; tss_callback_start_walusage = pgWalUsage; INSTR_TIME_SET_CURRENT(tss_callback_start_time); } void ts_end_tss_store_callback(const char *query, int query_location, int query_len, uint64 query_id, uint64 rows) { instr_time duration; BufferUsage bufusage; WalUsage walusage; tss_store_hook_type hook; if (!ts_is_tss_enabled()) return; hook = ts_get_tss_store_hook(); if (!hook) return; INSTR_TIME_SET_CURRENT(duration); INSTR_TIME_SUBTRACT(duration, tss_callback_start_time); /* calc differences of buffer counters. */ memset(&bufusage, 0, sizeof(BufferUsage)); BufferUsageAccumDiff(&bufusage, &pgBufferUsage, &tss_callback_start_bufusage); /* calc differences of WAL counters. */ memset(&walusage, 0, sizeof(WalUsage)); WalUsageAccumDiff(&walusage, &pgWalUsage, &tss_callback_start_walusage); hook(query, query_location, query_len, query_id, INSTR_TIME_GET_MICROSEC(duration), rows, &bufusage, &walusage); } ================================================ FILE: src/tss_callbacks.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #define TSS_CALLBACKS_VAR_NAME "tss_callbacks" #define TSS_CALLBACKS_VERSION 1 /* ts_stat_statements -> pgss_store */ typedef void (*tss_store_hook_type)(const char *query, int query_location, int query_len, uint64 query_id, uint64 total_time, uint64 rows, const BufferUsage *bufusage, const WalUsage *walusage); /* ts_stat_statements -> pgss_enabled */ typedef bool (*tss_enabled_hook_type)(int level); /* ts_stat_statements callbacks */ typedef struct TSSCallbacks { uint32_t version_num; tss_store_hook_type tss_store_hook; tss_enabled_hook_type tss_enabled_hook_type; } TSSCallbacks; extern bool ts_is_tss_enabled(void); extern void ts_begin_tss_store_callback(void); extern void ts_end_tss_store_callback(const char *query, int query_location, int query_len, uint64 query_id, uint64 rows); ================================================ FILE: src/utils.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/genam.h> #include <access/heapam.h> #include <access/htup.h> #include <access/htup_details.h> #include <access/reloptions.h> #include <access/xact.h> #include <catalog/indexing.h> #include <catalog/namespace.h> #include <catalog/objectaccess.h> #include <catalog/pg_am.h> #include <catalog/pg_cast.h> #include <catalog/pg_inherits.h> #include <catalog/pg_operator.h> #include <catalog/pg_type.h> #include <commands/event_trigger.h> #include <commands/tablecmds.h> #include <fmgr.h> #include <funcapi.h> #include <nodes/makefuncs.h> #include <parser/parse_coerce.h> #include <parser/parse_func.h> #include <parser/scansup.h> #include <storage/lockdefs.h> #include <utils/acl.h> #include <utils/builtins.h> #include <utils/catcache.h> #include <utils/date.h> #include <utils/elog.h> #include <utils/fmgroids.h> #include <utils/fmgrprotos.h> #include <utils/lsyscache.h> #include <utils/relcache.h> #include <utils/snapmgr.h> #include <utils/syscache.h> #include <utils/timestamp.h> #include <utils/uuid.h> #include "compat/compat.h" #include "chunk.h" #include "cross_module_fn.h" #include "debug_point.h" #include "hypertable_cache.h" #include "jsonb_utils.h" #include "time_utils.h" #include "utils.h" #include "uuid.h" typedef struct { const char *name; AclMode value; } priv_map; TS_FUNCTION_INFO_V1(ts_pg_timestamp_to_unix_microseconds); TS_FUNCTION_INFO_V1(ts_makeaclitem); /* * Convert a Postgres TIMESTAMP to BIGINT microseconds relative the UNIX epoch. */ Datum ts_pg_timestamp_to_unix_microseconds(PG_FUNCTION_ARGS) { TimestampTz timestamp = PG_GETARG_TIMESTAMPTZ(0); if (TIMESTAMP_IS_NOBEGIN(timestamp)) PG_RETURN_INT64(TS_TIME_NOBEGIN); if (TIMESTAMP_IS_NOEND(timestamp)) PG_RETURN_INT64(TS_TIME_NOEND); if (timestamp < TS_TIMESTAMP_MIN) ereport(ERROR, (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), errmsg("timestamp out of range"))); if (timestamp >= TS_TIMESTAMP_END) ereport(ERROR, (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), errmsg("timestamp out of range"))); PG_RETURN_INT64(timestamp + TS_EPOCH_DIFF_MICROSECONDS); } TS_FUNCTION_INFO_V1(ts_pg_unix_microseconds_to_timestamp); TS_FUNCTION_INFO_V1(ts_pg_unix_microseconds_to_timestamp_without_timezone); TS_FUNCTION_INFO_V1(ts_pg_unix_microseconds_to_date); /* * Convert BIGINT microseconds relative the UNIX epoch to a Postgres TIMESTAMP. */ Datum ts_pg_unix_microseconds_to_timestamp(PG_FUNCTION_ARGS) { int64 microseconds = PG_GETARG_INT64(0); if (TS_TIME_IS_NOBEGIN(microseconds, TIMESTAMPTZOID)) PG_RETURN_DATUM(ts_time_datum_get_nobegin(TIMESTAMPTZOID)); if (TS_TIME_IS_NOEND(microseconds, TIMESTAMPTZOID)) PG_RETURN_DATUM(ts_time_datum_get_noend(TIMESTAMPTZOID)); /* * Test that the UNIX us timestamp is within bounds. Note that an int64 at * UNIX epoch and microsecond precision cannot represent the upper limit * of the supported date range (Julian end date), so INT64_MAX-1 is the * natural upper bound for this function. */ if (microseconds < TS_TIMESTAMP_INTERNAL_MIN) ereport(ERROR, (errcode(ERRCODE_DATETIME_VALUE_OUT_OF_RANGE), errmsg("timestamp out of range"))); PG_RETURN_TIMESTAMPTZ(microseconds - TS_EPOCH_DIFF_MICROSECONDS); } Datum ts_pg_unix_microseconds_to_date(PG_FUNCTION_ARGS) { int64 microseconds = PG_GETARG_INT64(0); Datum res; if (TS_TIME_IS_NOBEGIN(microseconds, DATEOID)) PG_RETURN_DATUM(ts_time_datum_get_nobegin(DATEOID)); if (TS_TIME_IS_NOEND(microseconds, DATEOID)) PG_RETURN_DATUM(ts_time_datum_get_noend(DATEOID)); res = DirectFunctionCall1(ts_pg_unix_microseconds_to_timestamp, Int64GetDatum(microseconds)); res = DirectFunctionCall1(timestamp_date, res); PG_RETURN_DATUM(res); } static int64 ts_integer_to_internal(Datum time_val, Oid type_oid); /* Convert valid timescale time column type to internal representation */ TSDLLEXPORT int64 ts_time_value_to_internal(Datum time_val, Oid type_oid) { Datum res, tz; /* Handle custom time types. We currently only support binary coercible * types */ if (!IS_VALID_TIME_TYPE(type_oid)) { if (ts_type_is_int8_binary_compatible(type_oid)) return DatumGetInt64(time_val); elog(ERROR, "unknown time type \"%s\"", format_type_be(type_oid)); } if (IS_INTEGER_TYPE(type_oid) || IS_UUID_TYPE(type_oid)) { /* Integer time types have no distinction between min, max and * infinity. We don't want min and max to be turned into infinity for * these types so check for those values first. */ if (TS_TIME_DATUM_IS_MIN(time_val, type_oid)) return ts_time_get_min(type_oid); if (TS_TIME_DATUM_IS_MAX(time_val, type_oid)) return ts_time_get_max(type_oid); } if (TS_TIME_DATUM_IS_NOBEGIN(time_val, type_oid)) return ts_time_get_nobegin(type_oid); if (TS_TIME_DATUM_IS_NOEND(time_val, type_oid)) return ts_time_get_noend(type_oid); switch (type_oid) { case INT8OID: case INT4OID: case INT2OID: return ts_integer_to_internal(time_val, type_oid); case TIMESTAMPOID: /* * for timestamps, ignore timezones, make believe the timestamp is * at UTC */ res = DirectFunctionCall1(ts_pg_timestamp_to_unix_microseconds, time_val); return DatumGetInt64(res); case TIMESTAMPTZOID: res = DirectFunctionCall1(ts_pg_timestamp_to_unix_microseconds, time_val); return DatumGetInt64(res); case DATEOID: tz = DirectFunctionCall1(date_timestamp, time_val); res = DirectFunctionCall1(ts_pg_timestamp_to_unix_microseconds, tz); return DatumGetInt64(res); case UUIDOID: { uint64 unixtime_ms = 0; /* * Extract the unix timestamp from the UUID. Note that we cannot * use the (optional) sub-milliseconds part because there is no * way to know whether it represents time or is random. * * If the UUID is not v7, error out. */ if (!ts_uuid_v7_extract_unixtime(DatumGetUUIDP(time_val), &unixtime_ms, NULL)) { ereport(ERROR, errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("%s is not a version 7 UUID", DatumGetCString(DirectFunctionCall1(uuid_out, time_val))), errdetail( "UUID \"time\" partitioning columns only support version 7 UUIDs.")); } /* Convert to microseconds */ return unixtime_ms * 1000; } default: elog(ERROR, "unknown time type \"%s\"", format_type_be(type_oid)); return -1; } } TSDLLEXPORT int64 ts_interval_value_to_internal(Datum time_val, Oid type_oid) { switch (type_oid) { case INT8OID: case INT4OID: case INT2OID: return ts_integer_to_internal(time_val, type_oid); case INTERVALOID: { Interval *interval = DatumGetIntervalP(time_val); if (interval->month != 0) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("months and years not supported"), errdetail("An interval must be defined as a fixed duration (such as " "weeks, days, hours, minutes, seconds, etc.)."))); return interval->time + (interval->day * USECS_PER_DAY); } default: elog(ERROR, "unknown interval type \"%s\"", format_type_be(type_oid)); return -1; } } static int64 ts_integer_to_internal(Datum time_val, Oid type_oid) { switch (type_oid) { case INT8OID: return DatumGetInt64(time_val); case INT4OID: return (int64) DatumGetInt32(time_val); case INT2OID: return (int64) DatumGetInt16(time_val); default: elog(ERROR, "unknown interval type \"%s\"", format_type_be(type_oid)); return -1; } } int64 ts_time_value_to_internal_or_infinite(Datum time_val, Oid type_oid) { switch (type_oid) { case TIMESTAMPOID: { Timestamp ts = DatumGetTimestamp(time_val); if (TIMESTAMP_NOT_FINITE(ts)) { if (TIMESTAMP_IS_NOBEGIN(ts)) { return PG_INT64_MIN; } else { return PG_INT64_MAX; } } /* * Timestamp is valid in PostgreSQL but exceeds TimescaleDB's * supported range (TS_TIMESTAMP_END < END_TIMESTAMP due to the * Unix epoch shift). Treat as +infinity to avoid errors during * chunk exclusion. No equivalent check is needed on the lower * bound since TS_TIMESTAMP_MIN == MIN_TIMESTAMP. */ if (ts >= TS_TIMESTAMP_END) return PG_INT64_MAX; return ts_time_value_to_internal(time_val, type_oid); } case TIMESTAMPTZOID: { TimestampTz ts = DatumGetTimestampTz(time_val); if (TIMESTAMP_NOT_FINITE(ts)) { if (TIMESTAMP_IS_NOBEGIN(ts)) { return PG_INT64_MIN; } else { return PG_INT64_MAX; } } /* See comment in TIMESTAMPOID case above. */ if (ts >= TS_TIMESTAMP_END) return PG_INT64_MAX; return ts_time_value_to_internal(time_val, type_oid); } case DATEOID: { DateADT d = DatumGetDateADT(time_val); if (DATE_NOT_FINITE(d)) { if (DATE_IS_NOBEGIN(d)) { return PG_INT64_MIN; } else { return PG_INT64_MAX; } } /* See comment in TIMESTAMPOID case above. */ if (d >= TS_DATE_END) return PG_INT64_MAX; return ts_time_value_to_internal(time_val, type_oid); } default: return ts_time_value_to_internal(time_val, type_oid); } } TS_FUNCTION_INFO_V1(ts_time_to_internal); Datum ts_time_to_internal(PG_FUNCTION_ARGS) { Datum time = PG_GETARG_DATUM(0); Oid time_type = get_fn_expr_argtype(fcinfo->flinfo, 0); int64 res = ts_time_value_to_internal(time, time_type); PG_RETURN_INT64(res); } static Datum ts_integer_to_internal_value(int64 value, Oid type); /* * convert int64 to Datum according to type * internally we store all times as int64 in the * same format postgres does */ TSDLLEXPORT Datum ts_internal_to_time_value(int64 value, Oid type) { if (TS_TIME_IS_NOBEGIN(value, type)) return ts_time_datum_get_nobegin(type); if (TS_TIME_IS_NOEND(value, type)) return ts_time_datum_get_noend(type); switch (type) { case INT2OID: case INT4OID: case INT8OID: return ts_integer_to_internal_value(value, type); case TIMESTAMPOID: case TIMESTAMPTZOID: /* we continue ts_time_value_to_internal's incorrect handling of TIMESTAMPs for * compatibility */ return DirectFunctionCall1(ts_pg_unix_microseconds_to_timestamp, Int64GetDatum(value)); case DATEOID: return DirectFunctionCall1(ts_pg_unix_microseconds_to_date, Int64GetDatum(value)); case UUIDOID: { /* * Convert the internal unixtime in ms to a UUID with the * non-timestamp bits set to zero. We do not set the version * either, because for ranges we only care about the prefix in * order to divide the whole UUID space into a set of slices. */ const pg_uuid_t *uuid = ts_create_uuid_v7_from_unixtime_us(value, true, false); return UUIDPGetDatum(uuid); } default: if (ts_type_is_int8_binary_compatible(type)) return Int64GetDatum(value); elog(ERROR, "unknown time type \"%s\" in ts_internal_to_time_value", format_type_be(type)); pg_unreachable(); } } TSDLLEXPORT int64 ts_internal_to_time_int64(int64 value, Oid type) { if (TS_TIME_IS_NOBEGIN(value, type)) return ts_time_datum_get_nobegin(type); if (TS_TIME_IS_NOEND(value, type)) return ts_time_datum_get_noend(type); switch (type) { case INT2OID: case INT4OID: case INT8OID: return value; case TIMESTAMPOID: case TIMESTAMPTZOID: case UUIDOID: /* we continue ts_time_value_to_internal's incorrect handling of TIMESTAMPs for * compatibility */ return DatumGetInt64( DirectFunctionCall1(ts_pg_unix_microseconds_to_timestamp, Int64GetDatum(value))); case DATEOID: return DatumGetInt64( DirectFunctionCall1(ts_pg_unix_microseconds_to_date, Int64GetDatum(value))); default: elog(ERROR, "unknown time type \"%s\" in ts_internal_to_time_value", format_type_be(type)); pg_unreachable(); } } TSDLLEXPORT char * ts_datum_to_string(Datum value, Oid type) { Oid typoutputfunc; bool typIsVarlena; FmgrInfo typoutputinfo; getTypeOutputInfo(type, &typoutputfunc, &typIsVarlena); fmgr_info(typoutputfunc, &typoutputinfo); return OutputFunctionCall(&typoutputinfo, value); } TSDLLEXPORT char * ts_internal_to_time_string(int64 value, Oid type) { Datum time_datum = ts_internal_to_time_value(value, type); return ts_datum_to_string(time_datum, type); } TS_FUNCTION_INFO_V1(ts_pg_unix_microseconds_to_interval); Datum ts_pg_unix_microseconds_to_interval(PG_FUNCTION_ARGS) { int64 microseconds = PG_GETARG_INT64(0); Interval *interval = palloc0(sizeof(*interval)); interval->day = microseconds / USECS_PER_DAY; interval->time = microseconds % USECS_PER_DAY; PG_RETURN_INTERVAL_P(interval); } TSDLLEXPORT Datum ts_internal_to_interval_value(int64 value, Oid type) { switch (type) { case INT2OID: case INT4OID: case INT8OID: return ts_integer_to_internal_value(value, type); case INTERVALOID: return DirectFunctionCall1(ts_pg_unix_microseconds_to_interval, Int64GetDatum(value)); default: elog(ERROR, "unknown time type \"%s\" in ts_internal_to_interval_value", format_type_be(type)); pg_unreachable(); } } static Datum ts_integer_to_internal_value(int64 value, Oid type) { switch (type) { case INT2OID: return Int16GetDatum(value); case INT4OID: return Int32GetDatum(value); case INT8OID: return Int64GetDatum(value); default: elog(ERROR, "unknown time type \"%s\" in ts_internal_to_time_value", format_type_be(type)); pg_unreachable(); } } /* Returns approximate period in microseconds */ int64 ts_get_interval_period_approx(Interval *interval) { return interval->time + ((((int64) interval->month * DAYS_PER_MONTH) + interval->day) * USECS_PER_DAY); } #define DAYS_PER_WEEK 7 #define DAYS_PER_QUARTER 89 #define YEARS_PER_DECADE 10 #define YEARS_PER_CENTURY 100 #define YEARS_PER_MILLENNIUM 1000 /* Returns approximate period in microseconds */ int64 ts_date_trunc_interval_period_approx(text *units) { int decode_type, val; char *lowunits = downcase_truncate_identifier(VARDATA_ANY(units), VARSIZE_ANY_EXHDR(units), false); decode_type = DecodeUnits(0, lowunits, &val); if (decode_type != UNITS) return -1; switch (val) { case DTK_WEEK: return DAYS_PER_WEEK * USECS_PER_DAY; case DTK_MILLENNIUM: return YEARS_PER_MILLENNIUM * DAYS_PER_YEAR * USECS_PER_DAY; case DTK_CENTURY: return YEARS_PER_CENTURY * DAYS_PER_YEAR * USECS_PER_DAY; case DTK_DECADE: return YEARS_PER_DECADE * DAYS_PER_YEAR * USECS_PER_DAY; case DTK_YEAR: return 1 * DAYS_PER_YEAR * USECS_PER_DAY; case DTK_QUARTER: return DAYS_PER_QUARTER * USECS_PER_DAY; case DTK_MONTH: return DAYS_PER_MONTH * USECS_PER_DAY; case DTK_DAY: return USECS_PER_DAY; case DTK_HOUR: return USECS_PER_HOUR; case DTK_MINUTE: return USECS_PER_MINUTE; case DTK_SECOND: return USECS_PER_SEC; case DTK_MILLISEC: return USECS_PER_SEC / 1000; case DTK_MICROSEC: return 1; default: ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("timestamp units \"%s\" not supported", lowunits))); } return -1; } Oid ts_inheritance_parent_relid(Oid relid) { Relation catalog; SysScanDesc scan; ScanKeyData skey; Oid parent = InvalidOid; HeapTuple tuple; catalog = table_open(InheritsRelationId, AccessShareLock); ScanKeyInit(&skey, Anum_pg_inherits_inhrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); scan = systable_beginscan(catalog, InheritsRelidSeqnoIndexId, true, NULL, 1, &skey); tuple = systable_getnext(scan); if (HeapTupleIsValid(tuple)) parent = ((Form_pg_inherits) GETSTRUCT(tuple))->inhparent; systable_endscan(scan); table_close(catalog, AccessShareLock); return parent; } TSDLLEXPORT bool ts_type_is_int8_binary_compatible(Oid sourcetype) { HeapTuple tuple; Form_pg_cast castForm; bool result; tuple = SearchSysCache2(CASTSOURCETARGET, ObjectIdGetDatum(sourcetype), ObjectIdGetDatum(INT8OID)); if (!HeapTupleIsValid(tuple)) return false; /* no cast */ castForm = (Form_pg_cast) GETSTRUCT(tuple); result = castForm->castmethod == COERCION_METHOD_BINARY; ReleaseSysCache(tuple); return result; } /* * Create a fresh struct pointer that will contain copied contents of the tuple. * Note that this function uses GETSTRUCT, which will not work correctly for tuple types * that might have variable lengths. * Also note that the function assumes no NULLs in the tuple. */ static void * ts_create_struct_from_tuple(HeapTuple tuple, MemoryContext mctx, size_t alloc_size, size_t copy_size) { void *struct_ptr = MemoryContextAllocZero(mctx, alloc_size); /* * Make sure the function is not used when the tuple contains NULLs. * Also compare the aligned sizes in the assert. */ Assert(copy_size == MAXALIGN(tuple->t_len - tuple->t_data->t_hoff)); memcpy(struct_ptr, GETSTRUCT(tuple), copy_size); return struct_ptr; } void * ts_create_struct_from_slot(TupleTableSlot *slot, MemoryContext mctx, size_t alloc_size, size_t copy_size) { bool should_free; HeapTuple tuple = ExecFetchSlotHeapTuple(slot, false, &should_free); void *result = ts_create_struct_from_tuple(tuple, mctx, alloc_size, copy_size); if (should_free) heap_freetuple(tuple); return result; } bool ts_function_types_equal(Oid left[], Oid right[], int nargs) { int arg_index; for (arg_index = 0; arg_index < nargs; arg_index++) { if (left[arg_index] != right[arg_index]) return false; } return true; } Oid ts_get_function_oid(const char *funcname, const char *schema_name, int nargs, Oid arg_types[]) { List *qualified_funcname = list_make2(makeString(pstrdup(schema_name)), makeString(pstrdup(funcname))); FuncCandidateList func_candidates; func_candidates = FuncnameGetCandidates(qualified_funcname, nargs, NIL, false, false, /* include_out_arguments */ false, false); while (func_candidates != NULL) { if (func_candidates->nargs == nargs && ts_function_types_equal(func_candidates->args, arg_types, nargs)) return func_candidates->oid; func_candidates = func_candidates->next; } elog(ERROR, "failed to find function %s with %d args in schema \"%s\"", funcname, nargs, schema_name); return InvalidOid; } /* * Find a partitioning function with a given schema and name. * * The caller can optionally pass a filter function and a type of the argument * that the partitioning function should take. */ Oid ts_lookup_proc_filtered(const char *schema, const char *funcname, Oid *rettype, proc_filter filter, void *filter_arg) { Oid namespace_oid = LookupExplicitNamespace(schema, false); regproc func = InvalidOid; CatCList *catlist; int i; /* * We could use SearchSysCache3 to get by (name, args, namespace), but * that would not allow us to check for functions that take either * ANYELEMENTOID or a dimension-specific in the same search. */ catlist = SearchSysCacheList1(PROCNAMEARGSNSP, CStringGetDatum(funcname)); for (i = 0; i < catlist->n_members; i++) { HeapTuple proctup = &catlist->members[i]->tuple; Form_pg_proc procform = (Form_pg_proc) GETSTRUCT(proctup); if (procform->pronamespace == namespace_oid && (filter == NULL || filter(procform, filter_arg))) { if (rettype) *rettype = procform->prorettype; func = procform->oid; break; } } ReleaseSysCacheList(catlist); return func; } /* * ts_get_operator * * finds an operator given an exact specification (name, namespace, * left and right type IDs). */ Oid ts_get_operator(const char *name, Oid namespace, Oid left, Oid right) { HeapTuple tup; Oid opoid = InvalidOid; tup = SearchSysCache4(OPERNAMENSP, PointerGetDatum(name), ObjectIdGetDatum(left), ObjectIdGetDatum(right), ObjectIdGetDatum(namespace)); if (HeapTupleIsValid(tup)) { Form_pg_operator oprform = (Form_pg_operator) GETSTRUCT(tup); opoid = oprform->oid; ReleaseSysCache(tup); } return opoid; } /* * ts_get_cast_func * * returns Oid of functions that implements cast from source to target */ Oid ts_get_cast_func(Oid source, Oid target) { Oid result = InvalidOid; HeapTuple casttup; casttup = SearchSysCache2(CASTSOURCETARGET, ObjectIdGetDatum(source), ObjectIdGetDatum(target)); if (HeapTupleIsValid(casttup)) { Form_pg_cast castform = (Form_pg_cast) GETSTRUCT(casttup); result = castform->castfunc; ReleaseSysCache(casttup); } return result; } AppendRelInfo * ts_get_appendrelinfo(PlannerInfo *root, Index rti, bool missing_ok) { ListCell *lc; /* use append_rel_array if it has been setup */ if (root->append_rel_array) { if (root->append_rel_array[rti]) return root->append_rel_array[rti]; if (!missing_ok) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("no appendrelinfo found for index %d", rti))); return NULL; } foreach (lc, root->append_rel_list) { AppendRelInfo *appinfo = lfirst(lc); if (appinfo->child_relid == rti) return appinfo; } if (!missing_ok) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("no appendrelinfo found for index %d", rti))); return NULL; } /* * Find an equivalence class member expression, all of whose Vars, come from * the indicated relation. * * This function has been copied from find_em_expr_for_rel in * contrib/postgres_fdw/postgres_fdw.c in postgres source. * This function was moved to postgres main in PG13 but was removed * again in PG15. So we use our own implementation for PG15+. */ EquivalenceMember * ts_find_em_for_rel(EquivalenceClass *ec, RelOptInfo *rel) { EquivalenceMember *em; #if PG18_GE /* Use specialized iterator to include child ems. * * https://github.com/postgres/postgres/commit/d69d45a5 */ EquivalenceMemberIterator it; setup_eclass_member_iterator(&it, ec, bms_make_singleton(rel->relid)); while ((em = eclass_member_iterator_next(&it)) != NULL) { #else ListCell *lc_em; foreach (lc_em, ec->ec_members) { em = lfirst(lc_em); #endif if (bms_is_subset(em->em_relids, rel->relids) && !bms_is_empty(em->em_relids)) { /* * If there is more than one equivalence member whose Vars are * taken entirely from this relation, we'll be content to choose * any one of those. */ return em; } } /* We didn't find any suitable equivalence class member */ return NULL; } Expr * ts_find_em_expr_for_rel(EquivalenceClass *ec, RelOptInfo *rel) { EquivalenceMember *em = ts_find_em_for_rel(ec, rel); return em ? em->em_expr : NULL; } bool ts_has_row_security(Oid relid) { HeapTuple tuple; Form_pg_class classform; bool relrowsecurity; bool relforcerowsecurity; /* Fetch relation's relrowsecurity and relforcerowsecurity flags */ tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); if (!HeapTupleIsValid(tuple)) elog(ERROR, "cache lookup failed for relid %u", relid); classform = (Form_pg_class) GETSTRUCT(tuple); relrowsecurity = classform->relrowsecurity; relforcerowsecurity = classform->relforcerowsecurity; ReleaseSysCache(tuple); return (relrowsecurity || relforcerowsecurity); } List * ts_get_reloptions(Oid relid) { HeapTuple tuple; Datum datum; bool isnull; List *options = NIL; Assert(OidIsValid(relid)); tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); if (!HeapTupleIsValid(tuple)) elog(ERROR, "cache lookup failed for relation %u", relid); datum = SysCacheGetAttr(RELOID, tuple, Anum_pg_class_reloptions, &isnull); if (!isnull && PointerIsValid(DatumGetPointer(datum))) options = untransformRelOptions(datum); ReleaseSysCache(tuple); return options; } /* * Get the integer_now function for a dimension */ Oid ts_get_integer_now_func(const Dimension *open_dim, bool fail_if_not_found) { Oid rettype; Oid now_func = InvalidOid; Oid argtypes[] = { 0 }; rettype = ts_dimension_get_partition_type(open_dim); Assert(IS_INTEGER_TYPE(rettype)); if (strlen(NameStr(open_dim->fd.integer_now_func)) == 0 && strlen(NameStr(open_dim->fd.integer_now_func_schema)) == 0) { if (fail_if_not_found) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_FUNCTION), (errmsg("integer_now function not set")))); else return now_func; } List *name = list_make2(makeString((char *) NameStr(open_dim->fd.integer_now_func_schema)), makeString((char *) NameStr(open_dim->fd.integer_now_func))); now_func = LookupFuncName(name, 0, argtypes, false); if (get_func_rettype(now_func) != rettype) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_FUNCTION), (errmsg("invalid integer_now function"), errhint("return type of function does not match dimension type")))); return now_func; } /* subtract passed in interval from the now. * Arguments: * now_func : function used to compute now. * interval : integer value * Returns: * now_func() - interval */ int64 ts_sub_integer_from_now(int64 interval, Oid time_dim_type, Oid now_func) { Datum now; int64 res; Assert(IS_INTEGER_TYPE(time_dim_type)); now = OidFunctionCall0(now_func); switch (time_dim_type) { case INT2OID: res = DatumGetInt16(now) - interval; if (res < PG_INT16_MIN || res > PG_INT16_MAX) ereport(ERROR, (errcode(ERRCODE_INTERVAL_FIELD_OVERFLOW), errmsg("integer time overflow"))); return res; case INT4OID: res = DatumGetInt32(now) - interval; if (res < PG_INT32_MIN || res > PG_INT32_MAX) ereport(ERROR, (errcode(ERRCODE_INTERVAL_FIELD_OVERFLOW), errmsg("integer time overflow"))); return res; case INT8OID: { bool overflow = pg_sub_s64_overflow(DatumGetInt64(now), interval, &res); if (overflow) { ereport(ERROR, (errcode(ERRCODE_INTERVAL_FIELD_OVERFLOW), errmsg("integer time overflow"))); } return res; } default: pg_unreachable(); } } TS_FUNCTION_INFO_V1(ts_subtract_integer_from_now); Datum ts_subtract_integer_from_now(PG_FUNCTION_ARGS) { Oid ht_relid = PG_GETARG_OID(0); int64 lag = PG_GETARG_INT64(1); Cache *hcache; Hypertable *ht = ts_hypertable_cache_get_cache_and_entry(ht_relid, CACHE_FLAG_NONE, &hcache); const Dimension *dim = hyperspace_get_open_dimension(ht->space, 0); if (!dim) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("hypertable has no open partitioning dimension"))); } Oid partitioning_type = ts_dimension_get_partition_type(dim); if (!IS_INTEGER_TYPE(partitioning_type)) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("hypertable has no integer partitioning dimension"))); } Oid now_func = ts_get_integer_now_func(dim, true); if (!OidIsValid(now_func)) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("could not find valid integer_now function for hypertable"))); } int64 res = ts_sub_integer_from_now(lag, partitioning_type, now_func); ts_cache_release(&hcache); return Int64GetDatum(res); } TS_FUNCTION_INFO_V1(ts_relation_size); Datum ts_relation_size(PG_FUNCTION_ARGS) { Oid relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); RelationSize relsize = { 0 }; TupleDesc tupdesc; HeapTuple tuple; Datum values[4] = { 0 }; bool nulls[4] = { false }; /* Build a tuple descriptor for our result type */ if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context " "that cannot accept type record"))); if (!OidIsValid(relid)) PG_RETURN_NULL(); relsize = ts_relation_size_impl(relid); tupdesc = BlessTupleDesc(tupdesc); values[0] = Int64GetDatum(relsize.total_size); values[1] = Int64GetDatum(relsize.heap_size); values[2] = Int64GetDatum(relsize.index_size); values[3] = Int64GetDatum(relsize.toast_size); tuple = heap_form_tuple(tupdesc, values, nulls); return HeapTupleGetDatum(tuple); } RelationSize ts_relation_size_impl(Oid relid) { RelationSize relsize = { 0 }; Datum reloid = ObjectIdGetDatum(relid); Relation rel; DEBUG_WAITPOINT("relation_size_before_lock"); /* Open relation earlier to keep a lock during all function calls */ rel = try_relation_open(relid, AccessShareLock); if (!rel) return relsize; /* Get to total relation size to be our calculation base */ relsize.total_size = DatumGetInt64(DirectFunctionCall1(pg_total_relation_size, reloid)); /* Get the indexes size of the relation (don't consider TOAST indexes) */ relsize.index_size = DatumGetInt64(DirectFunctionCall1(pg_indexes_size, reloid)); /* If exists an associated TOAST calculate the total size (including indexes) */ if (OidIsValid(rel->rd_rel->reltoastrelid)) relsize.toast_size = DatumGetInt64(DirectFunctionCall1(pg_total_relation_size, ObjectIdGetDatum(rel->rd_rel->reltoastrelid))); else relsize.toast_size = 0; relation_close(rel, AccessShareLock); /* Calculate the HEAP size based on the total size and indexes plus toast */ relsize.heap_size = relsize.total_size - (relsize.index_size + relsize.toast_size); return relsize; } /* * Try to get cached size for a provided relation across all forks. The * size is returned in terms of number of blocks. * * The function calls the underlying smgrnblocks if there is no cached * data. That call populates the cache for subsequent invocations. This * cached data gets removed asynchronously by PG relcache invalidations * and then the refresh/cache cycle repeats till the next invalidation. */ static int64 ts_try_relation_cached_size(Relation rel, bool verbose) { BlockNumber result = InvalidBlockNumber, nblocks = 0; ForkNumber forkNum; bool cached = true; if (!RELKIND_HAS_STORAGE(rel->rd_rel->relkind)) return (int64) nblocks; /* Get heap size, including FSM and VM */ for (forkNum = 0; forkNum <= MAX_FORKNUM; forkNum++) { result = RelationGetSmgr(rel)->smgr_cached_nblocks[forkNum]; if (result != InvalidBlockNumber) { nblocks += result; } else { if (smgrexists(RelationGetSmgr(rel), forkNum)) { cached = false; nblocks += smgrnblocks(RelationGetSmgr(rel), forkNum); } } } if (verbose) ereport(DEBUG2, (errmsg("%s for %s", cached ? "Cached size used" : "Fetching actual size", RelationGetRelationName(rel)))); /* convert the size into bytes and return */ return (int64) nblocks * BLCKSZ; } static RelationSize ts_relation_approximate_size_impl(Oid relid) { RelationSize relsize = { 0 }; Relation rel; DEBUG_WAITPOINT("relation_approximate_size_before_lock"); /* Open relation earlier to keep a lock during all function calls */ rel = try_relation_open(relid, AccessShareLock); if (!rel) return relsize; /* Get the main heap size */ relsize.heap_size = ts_try_relation_cached_size(rel, false); /* Get the size of the relation's indexes */ if (rel->rd_rel->relhasindex) { List *index_oids = RelationGetIndexList(rel); ListCell *cell; foreach (cell, index_oids) { Oid idxOid = lfirst_oid(cell); Relation idxRel; idxRel = relation_open(idxOid, AccessShareLock); relsize.index_size += ts_try_relation_cached_size(idxRel, false); relation_close(idxRel, AccessShareLock); } } /* If there's an associated TOAST table, calculate the total size (including its indexes) */ if (OidIsValid(rel->rd_rel->reltoastrelid)) { Relation toastRel; List *index_oids; ListCell *cell; toastRel = relation_open(rel->rd_rel->reltoastrelid, AccessShareLock); relsize.toast_size = ts_try_relation_cached_size(toastRel, false); /* Get the indexes size of the TOAST relation */ index_oids = RelationGetIndexList(toastRel); foreach (cell, index_oids) { Oid idxOid = lfirst_oid(cell); Relation idxRel; idxRel = relation_open(idxOid, AccessShareLock); relsize.toast_size += ts_try_relation_cached_size(idxRel, false); relation_close(idxRel, AccessShareLock); } relation_close(toastRel, AccessShareLock); } relation_close(rel, AccessShareLock); /* Add up the total size based on the heap size, indexes and toast */ relsize.total_size = relsize.heap_size + relsize.index_size + relsize.toast_size; return relsize; } TS_FUNCTION_INFO_V1(ts_relation_approximate_size); Datum ts_relation_approximate_size(PG_FUNCTION_ARGS) { Oid relid = PG_GETARG_OID(0); RelationSize relsize = { 0 }; TupleDesc tupdesc; HeapTuple tuple; Datum values[4] = { 0 }; bool nulls[4] = { false }; /* Build a tuple descriptor for our result type */ if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context " "that cannot accept type record"))); /* check if object exists, return NULL otherwise */ if (get_rel_name(relid) == NULL) PG_RETURN_NULL(); relsize = ts_relation_approximate_size_impl(relid); tupdesc = BlessTupleDesc(tupdesc); values[0] = Int64GetDatum(relsize.total_size); values[1] = Int64GetDatum(relsize.heap_size); values[2] = Int64GetDatum(relsize.index_size); values[3] = Int64GetDatum(relsize.toast_size); tuple = heap_form_tuple(tupdesc, values, nulls); return HeapTupleGetDatum(tuple); } static void init_scan_by_hypertable_id(ScanIterator *iterator, int32 hypertable_id) { iterator->ctx.index = catalog_get_index(ts_catalog_get(), CHUNK, CHUNK_HYPERTABLE_ID_INDEX); ts_scan_iterator_scan_key_init(iterator, Anum_chunk_hypertable_id_idx_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); } #define ADD_RELATIONSIZE(total, rel) \ do \ { \ (total).heap_size += (rel).heap_size; \ (total).toast_size += (rel).toast_size; \ (total).index_size += (rel).index_size; \ (total).total_size += (rel).total_size; \ } while (0) TS_FUNCTION_INFO_V1(ts_hypertable_approximate_size); Datum ts_hypertable_approximate_size(PG_FUNCTION_ARGS) { Oid relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); RelationSize total_relsize = { 0 }; TupleDesc tupdesc; HeapTuple tuple; Datum values[4] = { 0 }; bool nulls[4] = { false }; Cache *hcache; Hypertable *ht; ScanIterator iterator = ts_scan_iterator_create(CHUNK, RowExclusiveLock, CurrentMemoryContext); /* Build a tuple descriptor for our result type */ if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context " "that cannot accept type record"))); if (!OidIsValid(relid)) PG_RETURN_NULL(); /* go ahead only if this is a hypertable or a CAgg */ hcache = ts_hypertable_cache_pin(); ht = ts_resolve_hypertable_from_table_or_cagg(hcache, relid, true); if (ht == NULL) { ts_cache_release(&hcache); PG_RETURN_NULL(); } /* get the main hypertable size */ total_relsize = ts_relation_approximate_size_impl(relid); iterator = ts_scan_iterator_create(CHUNK, RowExclusiveLock, CurrentMemoryContext); init_scan_by_hypertable_id(&iterator, ht->fd.id); ts_scanner_foreach(&iterator) { bool isnull, is_osm_chunk; TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); Datum id = slot_getattr(ti->slot, Anum_chunk_id, &isnull); Datum comp_id = DatumGetInt32(slot_getattr(ti->slot, Anum_chunk_id, &isnull)); int32 chunk_id, compressed_chunk_id; Oid chunk_relid, compressed_chunk_relid; RelationSize chunk_relsize, compressed_chunk_relsize; if (isnull) continue; chunk_id = DatumGetInt32(id); /* avoid if it's an OSM chunk */ is_osm_chunk = slot_getattr(ti->slot, Anum_chunk_osm_chunk, &isnull); Assert(!isnull); if (is_osm_chunk) continue; chunk_relid = ts_chunk_get_relid(chunk_id, false); chunk_relsize = ts_relation_approximate_size_impl(chunk_relid); /* add this chunk's size to the total size */ ADD_RELATIONSIZE(total_relsize, chunk_relsize); /* check if the chunk has a compressed counterpart and add if yes */ comp_id = slot_getattr(ti->slot, Anum_chunk_compressed_chunk_id, &isnull); if (isnull) continue; compressed_chunk_id = DatumGetInt32(comp_id); compressed_chunk_relid = ts_chunk_get_relid(compressed_chunk_id, false); compressed_chunk_relsize = ts_relation_approximate_size_impl(compressed_chunk_relid); /* add this compressed chunk's size to the total size */ ADD_RELATIONSIZE(total_relsize, compressed_chunk_relsize); } ts_scan_iterator_close(&iterator); tupdesc = BlessTupleDesc(tupdesc); values[0] = Int64GetDatum(total_relsize.heap_size); values[1] = Int64GetDatum(total_relsize.index_size); values[2] = Int64GetDatum(total_relsize.toast_size); values[3] = Int64GetDatum(total_relsize.total_size); tuple = heap_form_tuple(tupdesc, values, nulls); ts_cache_release(&hcache); return HeapTupleGetDatum(tuple); } #define STR_VALUE(str) #str #define NODE_CASE(name) \ case T_##name: \ return STR_VALUE(name) /* * Return a string with the name of the node. * */ const char * ts_get_node_name(Node *node) { /* tags are defined in nodes/nodes.h postgres source */ switch (nodeTag(node)) { /* * primitive nodes (primnodes.h) */ NODE_CASE(Alias); NODE_CASE(RangeVar); NODE_CASE(TableFunc); NODE_CASE(IntoClause); NODE_CASE(Var); NODE_CASE(Const); NODE_CASE(Param); NODE_CASE(Aggref); NODE_CASE(GroupingFunc); NODE_CASE(WindowFunc); NODE_CASE(SubscriptingRef); NODE_CASE(FuncExpr); NODE_CASE(NamedArgExpr); NODE_CASE(OpExpr); NODE_CASE(DistinctExpr); NODE_CASE(NullIfExpr); NODE_CASE(ScalarArrayOpExpr); NODE_CASE(BoolExpr); NODE_CASE(SubLink); NODE_CASE(SubPlan); NODE_CASE(AlternativeSubPlan); NODE_CASE(FieldSelect); NODE_CASE(FieldStore); NODE_CASE(RelabelType); NODE_CASE(CoerceViaIO); NODE_CASE(ArrayCoerceExpr); NODE_CASE(ConvertRowtypeExpr); NODE_CASE(CollateExpr); NODE_CASE(CaseExpr); NODE_CASE(CaseWhen); NODE_CASE(CaseTestExpr); NODE_CASE(ArrayExpr); NODE_CASE(RowExpr); NODE_CASE(RowCompareExpr); NODE_CASE(CoalesceExpr); NODE_CASE(MinMaxExpr); NODE_CASE(SQLValueFunction); NODE_CASE(XmlExpr); NODE_CASE(NullTest); NODE_CASE(BooleanTest); NODE_CASE(CoerceToDomain); NODE_CASE(CoerceToDomainValue); NODE_CASE(SetToDefault); NODE_CASE(CurrentOfExpr); NODE_CASE(NextValueExpr); NODE_CASE(InferenceElem); NODE_CASE(TargetEntry); NODE_CASE(RangeTblRef); NODE_CASE(JoinExpr); NODE_CASE(FromExpr); NODE_CASE(OnConflictExpr); /* * plan nodes (plannodes.h) */ #if PG16_LT NODE_CASE(Plan); NODE_CASE(Scan); NODE_CASE(Join); #endif NODE_CASE(Result); NODE_CASE(ProjectSet); NODE_CASE(ModifyTable); NODE_CASE(Append); NODE_CASE(MergeAppend); NODE_CASE(RecursiveUnion); NODE_CASE(BitmapAnd); NODE_CASE(BitmapOr); NODE_CASE(SeqScan); NODE_CASE(SampleScan); NODE_CASE(IndexScan); NODE_CASE(IndexOnlyScan); NODE_CASE(BitmapIndexScan); NODE_CASE(BitmapHeapScan); NODE_CASE(TidScan); NODE_CASE(SubqueryScan); NODE_CASE(FunctionScan); NODE_CASE(ValuesScan); NODE_CASE(TableFuncScan); NODE_CASE(CteScan); NODE_CASE(NamedTuplestoreScan); NODE_CASE(WorkTableScan); NODE_CASE(ForeignScan); NODE_CASE(CustomScan); NODE_CASE(NestLoop); NODE_CASE(MergeJoin); NODE_CASE(HashJoin); NODE_CASE(Material); NODE_CASE(Sort); NODE_CASE(Group); NODE_CASE(Agg); NODE_CASE(WindowAgg); NODE_CASE(Unique); NODE_CASE(Gather); NODE_CASE(GatherMerge); NODE_CASE(Hash); NODE_CASE(SetOp); NODE_CASE(LockRows); NODE_CASE(Limit); /* * planner nodes (pathnodes.h) */ NODE_CASE(IndexPath); NODE_CASE(BitmapHeapPath); NODE_CASE(BitmapAndPath); NODE_CASE(BitmapOrPath); NODE_CASE(TidPath); NODE_CASE(SubqueryScanPath); NODE_CASE(ForeignPath); NODE_CASE(NestPath); NODE_CASE(MergePath); NODE_CASE(HashPath); NODE_CASE(AppendPath); NODE_CASE(MergeAppendPath); NODE_CASE(GroupResultPath); NODE_CASE(MaterialPath); NODE_CASE(UniquePath); NODE_CASE(GatherPath); NODE_CASE(GatherMergePath); NODE_CASE(ProjectionPath); NODE_CASE(ProjectSetPath); NODE_CASE(SortPath); NODE_CASE(GroupPath); NODE_CASE(UpperUniquePath); NODE_CASE(AggPath); NODE_CASE(GroupingSetsPath); NODE_CASE(MinMaxAggPath); NODE_CASE(WindowAggPath); NODE_CASE(SetOpPath); NODE_CASE(RecursiveUnionPath); NODE_CASE(LockRowsPath); NODE_CASE(ModifyTablePath); NODE_CASE(LimitPath); case T_Path: switch (castNode(Path, node)->pathtype) { NODE_CASE(SeqScan); NODE_CASE(SampleScan); NODE_CASE(SubqueryScan); NODE_CASE(FunctionScan); NODE_CASE(TableFuncScan); NODE_CASE(ValuesScan); NODE_CASE(CteScan); NODE_CASE(WorkTableScan); default: return psprintf("Path (%d)", castNode(Path, node)->pathtype); } case T_CustomPath: return psprintf("CustomPath (%s)", castNode(CustomPath, node)->methods->CustomName); default: return psprintf("Node (%d)", nodeTag(node)); } } /* * Implementation marked unused in PostgreSQL lsyscache.c */ int ts_get_relnatts(Oid relid) { HeapTuple tp; Form_pg_class reltup; int result; tp = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); if (!HeapTupleIsValid(tp)) return InvalidAttrNumber; reltup = (Form_pg_class) GETSTRUCT(tp); result = reltup->relnatts; ReleaseSysCache(tp); return result; } /* * Wrap AlterTableInternal() for event trigger handling. * * AlterTableInternal can be called as a utility command, which is common in a * SQL function that alters a table in some form when called in the form * SELECT <cmd> INTO <table>. This is transformed into a process utility * command (CREATE TABLE AS), which expects an event trigger context to be * set up. * * The "cmd" parameter can be set to a higher-level command that caused the * alter table to occur. If "cmd" is set to NULL, the "cmds" list will be used * instead. */ void ts_alter_table_with_event_trigger(Oid relid, Node *cmd, List *cmds, bool recurse) { if (cmd == NULL) cmd = (Node *) cmds; EventTriggerAlterTableStart(cmd); AlterTableInternal(relid, cmds, recurse); EventTriggerAlterTableEnd(); } void ts_copy_relation_acl(const Oid source_relid, const Oid target_relid, const Oid owner_id) { HeapTuple source_tuple; bool is_null; Datum acl_datum; Relation class_rel; /* We open it here since there is no point in trying to update the tuples * if we cannot open the Relation catalog table */ class_rel = table_open(RelationRelationId, RowExclusiveLock); source_tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(source_relid)); Assert(HeapTupleIsValid(source_tuple)); /* We only bother about setting the ACL if the source relation ACL is * non-null */ acl_datum = SysCacheGetAttr(RELOID, source_tuple, Anum_pg_class_relacl, &is_null); if (!is_null) { HeapTuple target_tuple, newtuple; Datum new_val[Natts_pg_class] = { 0 }; bool new_null[Natts_pg_class] = { false }; bool new_repl[Natts_pg_class] = { false }; Acl *acl = DatumGetAclP(acl_datum); new_repl[AttrNumberGetAttrOffset(Anum_pg_class_relacl)] = true; new_val[AttrNumberGetAttrOffset(Anum_pg_class_relacl)] = PointerGetDatum(acl); /* * ts_copy_relation_acl() is typically used to copy ACLs from the hypertable * to a newly created chunk. The creation is done via DefineRelation(), * which takes an AccessExclusiveLock and should be enough to handle any * inplace update issues. */ AssertSufficientPgClassUpdateLockHeld(target_relid); /* Find the tuple for the target in `pg_class` */ target_tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(target_relid)); Assert(HeapTupleIsValid(target_tuple)); /* Update the relacl for the target tuple to use the acl from the source */ newtuple = heap_modify_tuple(target_tuple, RelationGetDescr(class_rel), new_val, new_null, new_repl); CatalogTupleUpdate(class_rel, &newtuple->t_self, newtuple); /* We need to update the shared dependencies as well to indicate that * the target is dependent on any roles that the source is * dependent on. */ Oid *newmembers; int nnewmembers = aclmembers(acl, &newmembers); /* The list of old members is intentionally empty since we are using * updateAclDependencies to set the ACL for the target. We can use NULL * because getOidListDiff, which is called from updateAclDependencies, * can handle that. */ updateAclDependencies(RelationRelationId, target_relid, 0, owner_id, 0, NULL, nnewmembers, newmembers); heap_freetuple(newtuple); ReleaseSysCache(target_tuple); } ReleaseSysCache(source_tuple); table_close(class_rel, RowExclusiveLock); } /* * Map attno from source relation to target relation by column name */ AttrNumber ts_map_attno(Oid src_rel, Oid dst_rel, AttrNumber attno) { char *attname = get_attname(src_rel, attno, false); AttrNumber dst_attno = get_attnum(dst_rel, attname); /* * For any chunk mappings we do this should never happen. */ if (dst_attno == InvalidAttrNumber) elog(ERROR, "could not map attribute number from relation \"%s\" to \"%s\" for column \"%s\"", get_rel_name(src_rel), get_rel_name(dst_rel), attname); pfree(attname); return dst_attno; } bool ts_relation_has_tuples(Relation rel) { TableScanDesc scandesc = table_beginscan(rel, GetActiveSnapshot(), 0, NULL); TupleTableSlot *slot = MakeSingleTupleTableSlot(RelationGetDescr(rel), table_slot_callbacks(rel)); bool hastuples = table_scan_getnextslot(scandesc, ForwardScanDirection, slot); table_endscan(scandesc); ExecDropSingleTupleTableSlot(slot); return hastuples; } bool ts_table_has_tuples(Oid table_relid, LOCKMODE lockmode) { Relation rel = table_open(table_relid, lockmode); bool hastuples = ts_relation_has_tuples(rel); table_close(rel, lockmode); return hastuples; } /* * This is copied from PostgreSQL 16.0 since versions before 16.0 does not * support lists for privileges. */ static AclMode ts_convert_any_priv_string(text *priv_type_text, const priv_map *privileges) { AclMode result = 0; char *priv_type = text_to_cstring(priv_type_text); char *chunk; char *next_chunk; /* We rely on priv_type being a private, modifiable string */ for (chunk = priv_type; chunk; chunk = next_chunk) { int chunk_len; const priv_map *this_priv; /* Split string at commas */ next_chunk = strchr(chunk, ','); if (next_chunk) *next_chunk++ = '\0'; /* Drop leading/trailing whitespace in this chunk */ while (*chunk && isspace((unsigned char) *chunk)) chunk++; chunk_len = strlen(chunk); while (chunk_len > 0 && isspace((unsigned char) chunk[chunk_len - 1])) chunk_len--; chunk[chunk_len] = '\0'; /* Match to the privileges list */ for (this_priv = privileges; this_priv->name; this_priv++) { if (pg_strcasecmp(this_priv->name, chunk) == 0) { result |= this_priv->value; break; } } if (!this_priv->name) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unrecognized privilege type: \"%s\"", chunk))); } pfree(priv_type); return result; } /* * This is copied from PostgreSQL 16.0 since versions before 16.0 does not * support lists for privileges but we need that. */ Datum ts_makeaclitem(PG_FUNCTION_ARGS) { Oid grantee = PG_GETARG_OID(0); Oid grantor = PG_GETARG_OID(1); text *privtext = PG_GETARG_TEXT_PP(2); bool goption = PG_GETARG_BOOL(3); AclItem *result; AclMode priv; static const priv_map any_priv_map[] = { { "SELECT", ACL_SELECT }, { "INSERT", ACL_INSERT }, { "UPDATE", ACL_UPDATE }, { "DELETE", ACL_DELETE }, { "TRUNCATE", ACL_TRUNCATE }, { "REFERENCES", ACL_REFERENCES }, { "TRIGGER", ACL_TRIGGER }, { "EXECUTE", ACL_EXECUTE }, { "USAGE", ACL_USAGE }, { "CREATE", ACL_CREATE }, { "TEMP", ACL_CREATE_TEMP }, { "TEMPORARY", ACL_CREATE_TEMP }, { "CONNECT", ACL_CONNECT }, #if PG16_GE { "SET", ACL_SET }, { "ALTER SYSTEM", ACL_ALTER_SYSTEM }, #endif #if PG17_GE { "MAINTAIN", ACL_MAINTAIN }, #endif { "RULE", 0 }, /* ignore old RULE privileges */ { NULL, 0 } }; priv = ts_convert_any_priv_string(privtext, any_priv_map); result = (AclItem *) palloc(sizeof(AclItem)); result->ai_grantee = grantee; result->ai_grantor = grantor; ACLITEM_SET_PRIVS_GOPTIONS(*result, priv, (goption ? priv : ACL_NO_RIGHTS)); PG_RETURN_ACLITEM_P(result); } /* * heap_form_tuple using NullableDatum array instead of two arrays for * values and nulls */ HeapTuple ts_heap_form_tuple(TupleDesc tupleDescriptor, NullableDatum *datums) { int numElements = tupleDescriptor->natts; Datum *values = palloc0(sizeof(Datum) * numElements); bool *nulls = palloc0(sizeof(bool) * numElements); for (int i = 0; i < numElements; i++) { values[i] = datums[i].value; nulls[i] = datums[i].isnull; } return heap_form_tuple(tupleDescriptor, values, nulls); } /* * To not introduce shared object dependencies on functions in extension update * scripts we use this stub function as placeholder whenever we need to reference * c functions in the update scripts. */ TS_FUNCTION_INFO_V1(ts_update_placeholder); Datum ts_update_placeholder(PG_FUNCTION_ARGS) { elog(ERROR, "this stub function is used only as placeholder during extension updates"); PG_RETURN_NULL(); } /* * Get relation information from the syscache in one call. * * Returns relid and relkind. All are non-optional. */ void ts_get_rel_info_by_name(const char *relnamespace, const char *relname, Oid *relid, char *relkind) { HeapTuple tuple; Form_pg_class cform; Oid namespaceoid = get_namespace_oid(relnamespace, false); tuple = SearchSysCache2(RELNAMENSP, PointerGetDatum(relname), ObjectIdGetDatum(namespaceoid)); if (!HeapTupleIsValid(tuple)) elog(ERROR, "cache lookup failed for relation %s.%s", relnamespace, relname); cform = (Form_pg_class) GETSTRUCT(tuple); *relid = cform->oid; *relkind = cform->relkind; ReleaseSysCache(tuple); } Oid ts_get_rel_am(Oid relid) { HeapTuple tuple; Form_pg_class cform; Oid amoid; tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); if (!HeapTupleIsValid(tuple)) elog(ERROR, "cache lookup failed for relation %u", relid); cform = (Form_pg_class) GETSTRUCT(tuple); amoid = cform->relam; ReleaseSysCache(tuple); return amoid; } /* * Set reloption for relation. * * Most of the code is from ATExecSetRelOptions() in tablecmds.c since that * function is static and we also need to do a slightly different job. */ static void relation_set_reloption_impl(Relation rel, List *options, LOCKMODE lockmode) { Datum repl_val[Natts_pg_class] = { 0 }; bool repl_null[Natts_pg_class] = { false }; bool repl_repl[Natts_pg_class] = { false }; bool isnull; Assert(rel->rd_rel->relkind == RELKIND_RELATION || rel->rd_rel->relkind == RELKIND_TOASTVALUE); if (options == NIL) return; /* nothing to do */ TS_DEBUG_LOG("setting reloptions for %s", RelationGetRelationName(rel)); Relation pgclass = table_open(RelationRelationId, RowExclusiveLock); Oid relid = RelationGetRelid(rel); HeapTuple tuple = SearchSysCacheLockedCopy1(RELOID, ObjectIdGetDatum(relid)); if (!HeapTupleIsValid(tuple)) elog(ERROR, "cache lookup failed for relation %u", relid); #ifdef SYSCACHE_TUPLE_LOCK_NEEDED ItemPointerData otid = tuple->t_self; #endif /* Get the old reloptions */ Datum datum = SysCacheGetAttr(RELOID, tuple, Anum_pg_class_reloptions, &isnull); /* Generate new proposed reloptions (text array) */ Datum newOptions = transformRelOptions(isnull ? UnassignedDatum : datum, options, NULL, NULL, false, false); (void) heap_reloptions(rel->rd_rel->relkind, newOptions, true); if (newOptions) repl_val[AttrNumberGetAttrOffset(Anum_pg_class_reloptions)] = newOptions; else repl_null[AttrNumberGetAttrOffset(Anum_pg_class_reloptions)] = true; repl_repl[AttrNumberGetAttrOffset(Anum_pg_class_reloptions)] = true; HeapTuple newtuple = heap_modify_tuple(tuple, RelationGetDescr(pgclass), repl_val, repl_null, repl_repl); CatalogTupleUpdate(pgclass, &newtuple->t_self, newtuple); /* Not sure if we need this one, but keeping it as a precaution */ InvokeObjectPostAlterHook(RelationRelationId, RelationGetRelid(rel), 0); UnlockSysCacheTuple(pgclass, &otid); heap_freetuple(newtuple); heap_freetuple(tuple); table_close(pgclass, RowExclusiveLock); } /* * Set value of reloptions for given relation. * * This will also set the reloption for the relations' associated relations, * in this case the TOAST table. It is based on ATExecSetRelOptions but we * split out the code to set the reloptions rather than duplicating it. * * The lockmode is needed for taking a correct lock on the toast table for the * already locked relation. It is only used for * * rel: Relation to add reloptions to. * defList: List of DefElem for the new definitions. * lockmode: the mode that the actual tables are locked in. */ void ts_relation_set_reloption(Relation rel, List *options, LOCKMODE lockmode) { Assert(RelationIsValid(rel)); relation_set_reloption_impl(rel, options, lockmode); if (OidIsValid(rel->rd_rel->reltoastrelid)) { Relation toastrel = table_open(rel->rd_rel->reltoastrelid, lockmode); relation_set_reloption_impl(toastrel, options, lockmode); table_close(toastrel, NoLock); } } /* this function fills in a jsonb with the non-null fields of the error data and also includes the proc name and schema in the jsonb we include these here to avoid adding these fields to the table */ Jsonb * ts_errdata_to_jsonb(ErrorData *edata, Name proc_schema, Name proc_name) { JsonbParseState *parse_state = NULL; pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); if (edata->sqlerrcode) ts_jsonb_add_str(parse_state, "sqlerrcode", unpack_sql_state(edata->sqlerrcode)); if (edata->message) ts_jsonb_add_str(parse_state, "message", edata->message); if (edata->detail) ts_jsonb_add_str(parse_state, "detail", edata->detail); if (edata->hint) ts_jsonb_add_str(parse_state, "hint", edata->hint); if (edata->filename) ts_jsonb_add_str(parse_state, "filename", edata->filename); if (edata->lineno) ts_jsonb_add_int32(parse_state, "lineno", edata->lineno); if (edata->funcname) ts_jsonb_add_str(parse_state, "funcname", edata->funcname); if (edata->domain) ts_jsonb_add_str(parse_state, "domain", edata->domain); if (edata->context_domain) ts_jsonb_add_str(parse_state, "context_domain", edata->context_domain); if (edata->context) ts_jsonb_add_str(parse_state, "context", edata->context); if (edata->schema_name) ts_jsonb_add_str(parse_state, "schema_name", edata->schema_name); if (edata->table_name) ts_jsonb_add_str(parse_state, "table_name", edata->table_name); if (edata->column_name) ts_jsonb_add_str(parse_state, "column_name", edata->column_name); if (edata->datatype_name) ts_jsonb_add_str(parse_state, "datatype_name", edata->datatype_name); if (edata->constraint_name) ts_jsonb_add_str(parse_state, "constraint_name", edata->constraint_name); if (edata->internalquery) ts_jsonb_add_str(parse_state, "internalquery", edata->internalquery); if (edata->detail_log) ts_jsonb_add_str(parse_state, "detail_log", edata->detail_log); if (strlen(NameStr(*proc_schema)) > 0) ts_jsonb_add_str(parse_state, "proc_schema", NameStr(*proc_schema)); if (strlen(NameStr(*proc_name)) > 0) ts_jsonb_add_str(parse_state, "proc_name", NameStr(*proc_name)); /* we add the schema qualified name here as well*/ JsonbValue *result = pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); return JsonbValueToJsonb(result); } char * ts_get_attr_expr(Relation rel, AttrNumber attno) { TupleConstr *constr = rel->rd_att->constr; char *expr = NULL; for (int i = 0; i < constr->num_defval; i++) { if (constr->defval[i].adnum == attno) { expr = TextDatumGetCString( DirectFunctionCall2(pg_get_expr, CStringGetTextDatum(constr->defval[i].adbin), ObjectIdGetDatum(RelationGetRelid(rel)))); break; } } return expr; } char * ts_list_to_string(List *list, append_cell_func append) { StringInfoData info; ListCell *lc; initStringInfo(&info); foreach (lc, list) { if (!lnext(list, lc)) appendStringInfoString(&info, "and "); append(&info, lc); if (lnext(list, lc)) { if (list_length(list) > 2) appendStringInfoChar(&info, ','); appendStringInfoChar(&info, ' '); } } return info.data; } ================================================ FILE: src/utils.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <access/htup_details.h> #include <access/tupdesc.h> #include <catalog/namespace.h> #include <catalog/pg_proc.h> #include <common/int.h> #include <debug_assert.h> #include <foreign/foreign.h> #include <nodes/extensible.h> #include <nodes/pathnodes.h> #include <optimizer/paths.h> #include <utils/builtins.h> #include <utils/datetime.h> #include <utils/jsonb.h> #include "compat/compat.h" #include "process_utility.h" /* * Macro for debug messages that should *only* be present in debug builds but * which should be removed in release builds. This is typically used for * debug builds for development purposes. * * Note that some debug messages might be relevant to deploy in release build * for debugging production systems. This macro is *not* for those cases. */ #ifdef TS_DEBUG #define TS_DEBUG_LOG(FMT, ...) elog(DEBUG2, "%s - " FMT, __func__, ##__VA_ARGS__) #else #define TS_DEBUG_LOG(FMT, ...) #endif #define UnassignedDatum (Datum) 0 static inline int64 interval_to_usec(const Interval *interval) { return (interval->month * DAYS_PER_MONTH * USECS_PER_DAY) + (interval->day * USECS_PER_DAY) + interval->time; } /* * Get the function name in a PG_FUNCTION. * * The function name is resolved from the function Oid in the functioncall * data. However, this information is not present in case of a direct function * call, so fall back to the C-function name. */ #define TS_FUNCNAME() \ (psprintf("%s()", fcinfo->flinfo ? get_func_name(FC_FN_OID(fcinfo)) : __func__)) #define TS_PREVENT_FUNC_IF_READ_ONLY() PreventCommandIfReadOnly(TS_FUNCNAME()) #define TS_PREVENT_IN_TRANSACTION_BLOCK(CMD) \ do \ { \ bool _isTopLevel = ts_process_utility_is_top_level(); \ /* Reset context before calling PreventInTransactionBlock in case it aborts. */ \ ts_process_utility_context_reset(); \ PreventInTransactionBlock(_isTopLevel, (CMD)); \ } while (0) #define MAX(x, y) ((x) > (y) ? x : y) #define MIN(x, y) ((x) < (y) ? x : y) static inline bool contains_volatile_functions_checker(Oid func_id, void *context) { return (func_volatile(func_id) == PROVOLATILE_VOLATILE); } /* find the length of a statically sized array */ #define TS_ARRAY_LEN(array) (sizeof(array) / sizeof(*array)) extern TSDLLEXPORT bool ts_type_is_int8_binary_compatible(Oid sourcetype); typedef bool (*proc_filter)(Form_pg_proc form, void *arg); /* * Convert a column value into the internal time representation. * cannot store a timestamp earlier than MIN_TIMESTAMP, or greater than * END_TIMESTAMP - TS_EPOCH_DIFF_MICROSECONDS * nor dates that cannot be translated to timestamps * Will throw an error for that, or other conversion issues. */ extern TSDLLEXPORT int64 ts_time_value_to_internal(Datum time_val, Oid type); extern int64 ts_time_value_to_internal_or_infinite(Datum time_val, Oid type_oid); extern TSDLLEXPORT int64 ts_interval_value_to_internal(Datum time_val, Oid type_oid); /* * Convert a column from the internal time representation into the specified type */ extern TSDLLEXPORT Datum ts_internal_to_time_value(int64 value, Oid type); extern TSDLLEXPORT int64 ts_internal_to_time_int64(int64 value, Oid type); extern TSDLLEXPORT Datum ts_internal_to_interval_value(int64 value, Oid type); extern TSDLLEXPORT char *ts_datum_to_string(Datum value, Oid type); extern TSDLLEXPORT char *ts_internal_to_time_string(int64 value, Oid type); /* * Return the period in microseconds of the first argument to date_trunc. * This is approximate -- to be used for planning; */ extern int64 ts_date_trunc_interval_period_approx(text *units); /* * Return the interval period in microseconds. * This is approximate -- to be used for planning; */ extern TSDLLEXPORT int64 ts_get_interval_period_approx(Interval *interval); extern TSDLLEXPORT Oid ts_inheritance_parent_relid(Oid relid); extern Oid ts_lookup_proc_filtered(const char *schema, const char *funcname, Oid *rettype, proc_filter filter, void *filter_arg); extern Oid ts_get_operator(const char *name, Oid namespace, Oid left, Oid right); extern bool ts_function_types_equal(Oid left[], Oid right[], int nargs); extern TSDLLEXPORT Oid ts_get_function_oid(const char *funcname, const char *schema_name, int nargs, Oid arg_types[]); extern TSDLLEXPORT Oid ts_get_cast_func(Oid source, Oid target); typedef struct Dimension Dimension; extern TSDLLEXPORT Oid ts_get_integer_now_func(const Dimension *open_dim, bool fail_if_not_found); extern TSDLLEXPORT int64 ts_sub_integer_from_now(int64 interval, Oid time_dim_type, Oid now_func); extern TSDLLEXPORT void *ts_create_struct_from_slot(TupleTableSlot *slot, MemoryContext mctx, size_t alloc_size, size_t copy_size); extern TSDLLEXPORT AppendRelInfo *ts_get_appendrelinfo(PlannerInfo *root, Index rti, bool missing_ok); extern TSDLLEXPORT Expr *ts_find_em_expr_for_rel(EquivalenceClass *ec, RelOptInfo *rel); extern TSDLLEXPORT EquivalenceMember *ts_find_em_for_rel(EquivalenceClass *ec, RelOptInfo *rel); extern TSDLLEXPORT bool ts_has_row_security(Oid relid); extern TSDLLEXPORT List *ts_get_reloptions(Oid relid); #define STRUCT_FROM_SLOT(slot, mctx, to_type, form_type) \ (to_type *) ts_create_struct_from_slot(slot, mctx, sizeof(to_type), sizeof(form_type)); /* note PG10 has_superclass but PG96 does not so use this */ #define is_inheritance_child(relid) (OidIsValid(ts_inheritance_parent_relid((relid)))) #define is_inheritance_parent(relid) \ (find_inheritance_children(table_relid, AccessShareLock) != NIL) #define is_inheritance_table(relid) (is_inheritance_child(relid) || is_inheritance_parent(relid)) #define INIT_NULL_DATUM \ { \ .value = 0, .isnull = true \ } static inline Datum ts_fetch_att(const void *T, bool attbyval, int attlen) { /* Length should be set to something sensible, otherwise an error will be * raised by fetch_att, so we assert this here to get a stack for * violations. */ Assert(!attbyval || (attlen > 0 && attlen <= 8)); return fetch_att(T, attbyval, attlen); } static inline int64 int64_min(int64 a, int64 b) { if (a <= b) return a; return b; } static inline int64 int64_saturating_add(int64 a, int64 b) { int64 result; bool overflowed = pg_add_s64_overflow(a, b, &result); if (overflowed) result = a < 0 ? PG_INT64_MIN : PG_INT64_MAX; return result; } static inline int64 int64_saturating_sub(int64 a, int64 b) { int64 result; bool overflowed = pg_sub_s64_overflow(a, b, &result); if (overflowed) result = b < 0 ? PG_INT64_MAX : PG_INT64_MIN; return result; } static inline bool ts_flags_are_set_32(uint32 bitmap, uint32 flags) { return (bitmap & flags) == flags; } static inline pg_nodiscard uint32 ts_set_flags_32(uint32 bitmap, uint32 flags) { return bitmap | flags; } static inline uint32 ts_clear_flags_32(uint32 bitmap, uint32 flags) { return bitmap & ~flags; } /** * Try to register a custom scan method. * * When registering a custom scan node, it might be called multiple times when * different databases have different versions of the extension installed, so * this function can be used to try to register a custom scan method but not * fail if it has already been registered. */ static inline void TryRegisterCustomScanMethods(const CustomScanMethods *methods) { if (!GetCustomScanMethods(methods->CustomName, true)) RegisterCustomScanMethods(methods); } typedef struct RelationSize { int64 total_size; int64 heap_size; int64 toast_size; int64 index_size; } RelationSize; extern TSDLLEXPORT RelationSize ts_relation_size_impl(Oid relid); extern TSDLLEXPORT const char *ts_get_node_name(Node *node); extern TSDLLEXPORT int ts_get_relnatts(Oid relid); extern TSDLLEXPORT void ts_alter_table_with_event_trigger(Oid relid, Node *cmd, List *cmds, bool recurse); extern TSDLLEXPORT void ts_copy_relation_acl(const Oid source_relid, const Oid target_relid, const Oid owner_id); extern TSDLLEXPORT bool ts_relation_has_tuples(Relation rel); extern TSDLLEXPORT bool ts_table_has_tuples(Oid table_relid, LOCKMODE lockmode); extern TSDLLEXPORT AttrNumber ts_map_attno(Oid src_rel, Oid dst_rel, AttrNumber attno); /* * Return Oid for a schema-qualified relation. */ static inline Oid ts_get_relation_relid(char const *schema_name, char const *relation_name, bool return_invalid) { Oid schema_oid = get_namespace_oid(schema_name, true); if (OidIsValid(schema_oid)) { Oid rel_oid = get_relname_relid(relation_name, schema_oid); if (!return_invalid) Ensure(OidIsValid(rel_oid), "relation \"%s.%s\" not found", schema_name, relation_name); return rel_oid; } else { if (!return_invalid) Ensure(OidIsValid(schema_oid), "schema \"%s\" not found (during lookup of relation \"%s.%s\")", schema_name, schema_name, relation_name); return InvalidOid; } } struct Hypertable; void replace_now_mock_walker(PlannerInfo *root, Node *clause, Oid funcid); extern TSDLLEXPORT HeapTuple ts_heap_form_tuple(TupleDesc tupleDescriptor, NullableDatum *datums); static inline void ts_datum_set_text_from_cstring(const AttrNumber attno, NullableDatum *datums, const char *value) { if (value != NULL) { datums[AttrNumberGetAttrOffset(attno)].value = PointerGetDatum(cstring_to_text(value)); datums[AttrNumberGetAttrOffset(attno)].isnull = false; } else datums[AttrNumberGetAttrOffset(attno)].isnull = true; } static inline void ts_datum_set_bool(const AttrNumber attno, NullableDatum *datums, const bool value, const bool isnull) { if (!isnull) datums[AttrNumberGetAttrOffset(attno)].value = BoolGetDatum(value); datums[AttrNumberGetAttrOffset(attno)].isnull = isnull; } static inline void ts_datum_set_int32(const AttrNumber attno, NullableDatum *datums, const int32 value, const bool isnull) { if (!isnull) datums[AttrNumberGetAttrOffset(attno)].value = Int32GetDatum(value); datums[AttrNumberGetAttrOffset(attno)].isnull = isnull; } static inline void ts_datum_set_int64(const AttrNumber attno, NullableDatum *datums, const int64 value, const bool isnull) { if (!isnull) datums[AttrNumberGetAttrOffset(attno)].value = Int64GetDatum(value); datums[AttrNumberGetAttrOffset(attno)].isnull = isnull; } static inline void ts_datum_set_timestamptz(const AttrNumber attno, NullableDatum *datums, const TimestampTz value, const bool isnull) { if (!isnull) datums[AttrNumberGetAttrOffset(attno)].value = TimestampTzGetDatum(value); datums[AttrNumberGetAttrOffset(attno)].isnull = isnull; } static inline void ts_datum_set_jsonb(const AttrNumber attno, NullableDatum *datums, const Jsonb *value) { if (value != NULL) { datums[AttrNumberGetAttrOffset(attno)].value = JsonbPGetDatum(value); datums[AttrNumberGetAttrOffset(attno)].isnull = false; } else datums[AttrNumberGetAttrOffset(attno)].isnull = true; } static inline void ts_datum_set_objectid(const AttrNumber attno, NullableDatum *datums, const Oid value) { if (OidIsValid(value)) { datums[AttrNumberGetAttrOffset(attno)].value = ObjectIdGetDatum(value); datums[AttrNumberGetAttrOffset(attno)].isnull = false; } else datums[AttrNumberGetAttrOffset(attno)].isnull = true; } typedef void (*append_cell_func)(StringInfo, ListCell *); extern TSDLLEXPORT void ts_get_rel_info_by_name(const char *relnamespace, const char *relname, Oid *relid, char *relkind); extern TSDLLEXPORT Oid ts_get_rel_am(Oid relid); extern TSDLLEXPORT void ts_relation_set_reloption(Relation rel, List *options, LOCKMODE lockmode); extern TSDLLEXPORT Jsonb *ts_errdata_to_jsonb(ErrorData *edata, Name proc_schema, Name proc_name); extern TSDLLEXPORT char *ts_get_attr_expr(Relation rel, AttrNumber attno); extern TSDLLEXPORT char *ts_list_to_string(List *list, append_cell_func append); ================================================ FILE: src/uuid.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <fmgr.h> #include <port/pg_bswap.h> #include <utils/timestamp.h> #include <utils/uuid.h> #include "compat/compat.h" #include "uuid.h" /* * Generates a v4 UUID. Based on function pg_random_uuid() in the pgcrypto contrib module. * * Note that clib on Mac has a uuid_generate() function, so we call this ts_uuid_create(). */ pg_uuid_t * ts_uuid_create(void) { /* * PG9.6 doesn't expose the internals of pg_uuid_t, so we just treat it as * a byte array */ unsigned char *gen_uuid = palloc0(UUID_LEN); bool rand_success = false; rand_success = pg_backend_random((char *) gen_uuid, UUID_LEN); /* * If pg_backend_random() cannot find sources of randomness, then we use * the current timestamp as a "random source". * Timestamps are 8 bytes, so we copy this into bytes 9-16 of the UUID. * If we see all 0s in bytes 0-8 (other than version + * variant), we know * that there is something wrong with the RNG on this instance. */ if (!rand_success) { TimestampTz ts = GetCurrentTimestamp(); memcpy(&gen_uuid[8], &ts, sizeof(TimestampTz)); } gen_uuid[6] = (gen_uuid[6] & 0x0f) | 0x40; /* "version" field */ gen_uuid[8] = (gen_uuid[8] & 0x3f) | 0x80; /* "variant" field */ return (pg_uuid_t *) gen_uuid; } TS_FUNCTION_INFO_V1(ts_uuid_generate); Datum ts_uuid_generate(PG_FUNCTION_ARGS) { PG_RETURN_UUID_P(ts_uuid_create()); } /* * Create a UUIDv7 from a unix timestamp in microseconds. * * Optionally produce a boundary UUID with all otherwise random bits set to * zero that can be used in range queries. The version can also be set to zero * in order to produce partition ranges that excludes the UUID version. */ pg_uuid_t * ts_create_uuid_v7_from_unixtime_us(int64 unixtime_us, bool boundary, bool set_version) { pg_uuid_t *uuid; uint64_t timestamp_be = pg_hton64((unixtime_us / 1000) << 16); if (boundary) { uuid = (pg_uuid_t *) palloc0(UUID_LEN); } else { uuid = (pg_uuid_t *) palloc(UUID_LEN); pg_backend_random(&((char *) uuid)[8], UUID_LEN - 8); } /* Fill the first 48 bits with the timestamp */ memcpy(uuid->data, ×tamp_be, 6); /* The microseconds part of the timestamp, scaled to 12 bits, same as in PG18 */ uint32 ts_micros = (unixtime_us % 1000) * (1 << 12) / 1000; /* * Sub milliseconds timestamps are optional. We store the microseconds part in the * rand_a field as described in the UUID v7 specification. Following the PG18 logic * here. */ uuid->data[6] = (unsigned char) (ts_micros >> 8); uuid->data[7] = (unsigned char) ts_micros; if (set_version) { /* Set version 7 (0111) in bits 6-7 of byte 6, keep random bits 0-5 */ uuid->data[6] = (uuid->data[6] & 0x0F) | 0x70; /* Set variant (10) in bits 4-5 of byte 8, keep random bits 0-3 and 6-7 */ uuid->data[8] = (uuid->data[8] & 0x3F) | 0x80; } return uuid; } pg_uuid_t * ts_create_uuid_v7_from_timestamptz(TimestampTz ts, bool boundary) { int64 epoch_diff_us = ((int64) (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * USECS_PER_DAY); int64 unixtime_us = ts + epoch_diff_us; return ts_create_uuid_v7_from_unixtime_us(unixtime_us, boundary, true); } TS_FUNCTION_INFO_V1(ts_uuid_generate_v7); Datum ts_uuid_generate_v7(PG_FUNCTION_ARGS) { PG_RETURN_UUID_P(ts_create_uuid_v7_from_timestamptz(GetCurrentTimestamp(), false)); } TS_FUNCTION_INFO_V1(ts_uuid_v7_from_timestamptz); Datum ts_uuid_v7_from_timestamptz(PG_FUNCTION_ARGS) { TimestampTz timestamp = PG_GETARG_TIMESTAMPTZ(0); PG_RETURN_UUID_P(ts_create_uuid_v7_from_timestamptz(timestamp, false)); } TS_FUNCTION_INFO_V1(ts_uuid_v7_from_timestamptz_boundary); Datum ts_uuid_v7_from_timestamptz_boundary(PG_FUNCTION_ARGS) { TimestampTz timestamp = PG_GETARG_TIMESTAMPTZ(0); PG_RETURN_UUID_P(ts_create_uuid_v7_from_timestamptz(timestamp, true)); } #define UUID_VARIANT(uuid) ((uuid)->data[8] & 0xc0) #define IS_RFC9562_VARIANT(uuid) (UUID_VARIANT(uuid) == 0x80) #define UUID_VERSION(uuid) (((uuid)->data[6] & 0xf0) >> 4) /* * Extract the millisecond Unix epoch timestamp from the UUIDv7, with optional * extra sub-millisecond fraction in microseconds. */ bool ts_uuid_v7_extract_unixtime(const pg_uuid_t *uuid, uint64 *unixtime_ms, uint16 *extra_us) { bool is_uuidv7 = false; /* Check that the variant field corresponds to RFC9562 */ if (IS_RFC9562_VARIANT(uuid)) { /* Get the version from the UUID */ is_uuidv7 = (UUID_VERSION(uuid) == 7); } /* Big endian timestamp in milliseconds from Unix Epoch */ uint64 timestamp_be = 0; memcpy(×tamp_be, uuid->data, 6); /* The timestamp is now milliseconds from Unix Epoch (1970-01-01)*/ *unixtime_ms = (pg_ntoh64(timestamp_be)) >> 16; if (extra_us) { /* Optionally, get the sub ms part as microseconds, reversing the scaling */ *extra_us = ((((uuid->data[6] & 0xF) << 8) | uuid->data[7]) + 1) * 1000 / (1 << 12); } return is_uuidv7; } bool ts_uuid_v7_extract_timestamptz(const pg_uuid_t *uuid, TimestampTz *timestamp, bool with_micros) { uint64 unixtime_millis = 0; uint16 extra_micros = 0; if (!ts_uuid_v7_extract_unixtime(uuid, &unixtime_millis, &extra_micros)) return false; /* Milliseconds timestamp from PG Epoch (2000-01-01) */ const uint64 epoch_diff = POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE; uint64 timestamp_millis = (unixtime_millis - (epoch_diff * SECS_PER_DAY) * 1000ULL); /* Convert to microseconds */ *timestamp = timestamp_millis * 1000; /* Add extra microseconds if requested */ if (with_micros) *timestamp += extra_micros; return true; } TS_FUNCTION_INFO_V1(ts_timestamptz_from_uuid_v7); Datum ts_timestamptz_from_uuid_v7(PG_FUNCTION_ARGS) { pg_uuid_t *uuid = PG_GETARG_UUID_P(0); TimestampTz ts = 0; if (!ts_uuid_v7_extract_timestamptz(uuid, &ts, false)) PG_RETURN_NULL(); PG_RETURN_TIMESTAMPTZ(ts); } TS_FUNCTION_INFO_V1(ts_timestamptz_from_uuid_v7_with_microseconds); Datum ts_timestamptz_from_uuid_v7_with_microseconds(PG_FUNCTION_ARGS) { pg_uuid_t *uuid = PG_GETARG_UUID_P(0); TimestampTz ts = 0; if (!ts_uuid_v7_extract_timestamptz(uuid, &ts, true)) PG_RETURN_NULL(); PG_RETURN_TIMESTAMPTZ(ts); } TS_FUNCTION_INFO_V1(ts_uuid_version); Datum ts_uuid_version(PG_FUNCTION_ARGS) { pg_uuid_t *uuid = PG_GETARG_UUID_P(0); int version; /* Check that the variant field corresponds to RFC9562 */ if (!IS_RFC9562_VARIANT(uuid)) PG_RETURN_NULL(); version = UUID_VERSION(uuid); /* Get the version from the UUID */ PG_RETURN_INT32(version); } ================================================ FILE: src/uuid.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <fmgr.h> #include <utils/uuid.h> #define UNIX_EPOCH_AS_TIMESTAMP (0 - ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * USECS_PER_DAY)) extern pg_uuid_t *ts_uuid_create(void); extern pg_uuid_t *ts_create_uuid_v7_from_unixtime_us(int64 unixtime_us, bool boundary, bool set_version); extern TSDLLEXPORT pg_uuid_t *ts_create_uuid_v7_from_timestamptz(TimestampTz ts, bool boundary); extern bool ts_uuid_v7_extract_unixtime(const pg_uuid_t *uuid, uint64 *unixtime_ms, uint16 *extra_us); extern bool ts_uuid_v7_extract_timestamptz(const pg_uuid_t *uuid, TimestampTz *timestamp, bool with_micros); extern TSDLLEXPORT Datum ts_timestamptz_from_uuid_v7(PG_FUNCTION_ARGS); ================================================ FILE: src/version.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <string.h> #include <postgres.h> #include <access/htup_details.h> #include <fmgr.h> #include <funcapi.h> #include <storage/fd.h> #include <utils/builtins.h> #include <utils/timestamp.h> #include "compat/compat.h" #include "annotations.h" #include "config.h" #include "gitcommit.h" #include "version.h" /* Export the strings to that we can read them using strings(1). We add a * prefix so that we can easily find it using grep(1). We only bother about * generating them if the relevant symbol is defined. */ #ifdef EXT_GIT_COMMIT_HASH static const char commit_hash[] TS_USED = "commit-hash:" EXT_GIT_COMMIT_HASH; #endif #ifdef EXT_GIT_COMMIT_TAG static const char commit_tag[] TS_USED = "commit-tag:" EXT_GIT_COMMIT_TAG; #endif #ifdef EXT_GIT_COMMIT_TIME static const char commit_time[] TS_USED = "commit-time:" EXT_GIT_COMMIT_TIME; #endif TS_FUNCTION_INFO_V1(ts_get_git_commit); /* Return git commit information defined in header file gitcommit.h. We * support that some of the fields are defined and will only show the fields * that are defined. If no fields are defined, we throw an error notifying the * user that there is no git information available at all. */ #if defined(EXT_GIT_COMMIT_HASH) || defined(EXT_GIT_COMMIT_TAG) || defined(EXT_GIT_COMMIT_TIME) Datum ts_get_git_commit(PG_FUNCTION_ARGS) { TupleDesc tupdesc; HeapTuple tuple; Datum values[3] = { 0 }; bool nulls[3] = { false }; /* Build a tuple descriptor for our result type */ if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context " "that cannot accept type record"))); tupdesc = BlessTupleDesc(tupdesc); #ifdef EXT_GIT_COMMIT_TAG values[0] = CStringGetTextDatum(EXT_GIT_COMMIT_TAG); #else nulls[0] = true; #endif #ifdef EXT_GIT_COMMIT_HASH values[1] = CStringGetTextDatum(EXT_GIT_COMMIT_HASH); #else nulls[1] = true; #endif #ifdef EXT_GIT_COMMIT_TIME values[2] = DirectFunctionCall3(timestamptz_in, CStringGetDatum(EXT_GIT_COMMIT_TIME), Int32GetDatum(-1), Int32GetDatum(-1)); #else nulls[2] = true; #endif tuple = heap_form_tuple(tupdesc, values, nulls); return HeapTupleGetDatum(tuple); } #else Datum ts_get_git_commit(PG_FUNCTION_ARGS) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("extension not built with any Git commit information"))); } #endif #ifdef WIN32 #include <Windows.h> bool ts_version_get_os_info(VersionOSInfo *info) { DWORD bufsize; void *buffer; VS_FIXEDFILEINFO *vinfo = NULL; UINT vinfo_len = 0; memset(info, 0, sizeof(VersionOSInfo)); bufsize = GetFileVersionInfoSizeA(TEXT("kernel32.dll"), NULL); if (bufsize == 0) return false; buffer = palloc(bufsize); if (!GetFileVersionInfoA(TEXT("kernel32.dll"), 0, bufsize, buffer)) goto error; if (!VerQueryValueA(buffer, TEXT("\\"), &vinfo, &vinfo_len)) goto error; snprintf(info->sysname, VERSION_INFO_LEN - 1, "Windows"); snprintf(info->version, VERSION_INFO_LEN - 1, "%u", HIWORD(vinfo->dwProductVersionMS)); snprintf(info->release, VERSION_INFO_LEN - 1, "%u", LOWORD(vinfo->dwProductVersionMS)); pfree(buffer); return true; error: pfree(buffer); return false; } #elif defined(UNIX) #include <sys/utsname.h> #define OS_RELEASE_FILE "/etc/os-release" #define MAX_READ_LEN 1024 #define NAME_FIELD "PRETTY_NAME=\"" static bool get_pretty_version(char *pretty_version) { FILE *version_file; char *contents = palloc(MAX_READ_LEN); size_t bytes_read; bool got_pretty_version = false; int i; memset(pretty_version, '\0', VERSION_INFO_LEN); /* we cannot use pg_read_file because it doesn't allow absolute paths */ version_file = AllocateFile(OS_RELEASE_FILE, PG_BINARY_R); if (version_file == NULL) return false; fseeko(version_file, 0, SEEK_SET); bytes_read = fread(contents, 1, (size_t) MAX_READ_LEN, version_file); if (bytes_read <= 0) goto cleanup; if (bytes_read < MAX_READ_LEN) contents[bytes_read] = '\0'; else contents[MAX_READ_LEN - 1] = '\0'; contents = strstr(contents, NAME_FIELD); if (contents == NULL) goto cleanup; contents += sizeof(NAME_FIELD) - 1; for (i = 0; i < (VERSION_INFO_LEN - 1); i++) { char c = contents[i]; if (c == '\0' || c == '\n' || c == '\r' || c == '"') break; pretty_version[i] = c; } got_pretty_version = true; cleanup: FreeFile(version_file); return got_pretty_version; } bool ts_version_get_os_info(VersionOSInfo *info) { /* Get the OS name */ struct utsname os_info; uname(&os_info); memset(info, 0, sizeof(VersionOSInfo)); strncpy(info->sysname, os_info.sysname, VERSION_INFO_LEN - 1); strncpy(info->version, os_info.version, VERSION_INFO_LEN - 1); strncpy(info->release, os_info.release, VERSION_INFO_LEN - 1); info->has_pretty_version = get_pretty_version(info->pretty_version); return true; } #else bool ts_version_get_os_info(VersionOSInfo *info) { memset(info, 0, sizeof(VersionOSInfo)); return false; } #endif /* WIN32 */ TS_FUNCTION_INFO_V1(ts_get_os_info); Datum ts_get_os_info(PG_FUNCTION_ARGS) { TupleDesc tupdesc; Datum values[4]; bool nulls[4] = { false }; HeapTuple tuple; VersionOSInfo info; if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context " "that cannot accept type record"))); if (ts_version_get_os_info(&info)) { values[0] = CStringGetTextDatum(info.sysname); values[1] = CStringGetTextDatum(info.version); values[2] = CStringGetTextDatum(info.release); if (info.has_pretty_version) values[3] = CStringGetTextDatum(info.pretty_version); else nulls[3] = true; } else memset(nulls, true, sizeof(nulls)); tuple = heap_form_tuple(tupdesc, values, nulls); return HeapTupleGetDatum(tuple); } ================================================ FILE: src/version.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #define VERSION_INFO_LEN 128 typedef struct VersionOSInfo { char sysname[VERSION_INFO_LEN]; char version[VERSION_INFO_LEN]; char release[VERSION_INFO_LEN]; char pretty_version[VERSION_INFO_LEN]; bool has_pretty_version; } VersionOSInfo; extern bool ts_version_get_os_info(VersionOSInfo *info); ================================================ FILE: src/with_clause/CMakeLists.txt ================================================ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/alter_table_with_clause.c ${CMAKE_CURRENT_SOURCE_DIR}/create_table_with_clause.c ${CMAKE_CURRENT_SOURCE_DIR}/create_materialized_view_with_clause.c ${CMAKE_CURRENT_SOURCE_DIR}/with_clause_parser.c) target_sources(${PROJECT_NAME} PRIVATE ${SOURCES}) ================================================ FILE: src/with_clause/alter_table_with_clause.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/htup_details.h> #include <catalog/dependency.h> #include <catalog/namespace.h> #include <catalog/pg_trigger.h> #include <catalog/pg_type.h> #include <commands/trigger.h> #include <fmgr.h> #include <parser/parser.h> #include <port.h> #include <storage/lmgr.h> #include <utils/builtins.h> #include <utils/elog.h> #include <utils/lsyscache.h> #include <utils/typcache.h> #include "compat/compat.h" #include "bmslist_utils.h" #include "cross_module_fn.h" #include "debug_assert.h" #include "guc.h" #include "jsonb_utils.h" #include "ts_catalog/array_utils.h" #include "ts_catalog/compression_settings.h" #include "alter_table_with_clause.h" static const WithClauseDefinition alter_table_with_clause_def[] = { [AlterTableFlagChunkTimeInterval] = { .arg_names = {"chunk_interval", NULL}, .type_id = TEXTOID, }, [AlterTableFlagColumnstore] = { .arg_names = {"compress", "columnstore", "enable_columnstore", NULL}, .type_id = BOOLOID, .default_val = (Datum)false, }, [AlterTableFlagSegmentBy] = { .arg_names = {"compress_segmentby", "segmentby", "segment_by", NULL}, .type_id = TEXTOID, }, [AlterTableFlagOrderBy] = { .arg_names = {"compress_orderby", "orderby", "order_by", NULL}, .type_id = TEXTOID, }, [AlterTableFlagCompressChunkTimeInterval] = { .arg_names = {"compress_chunk_interval", "compress_chunk_time_interval", NULL}, .type_id = INTERVALOID, }, [AlterTableFlagIndex] = { .arg_names = {"compress_index", "compress_sparse_index", "index", "sparse_index", NULL}, .type_id = TEXTOID, }, }; static const WithClauseDefinition sparse_index_with_clause_def[] = { [_SparseIndexTypeEnumBloom] = { .arg_names = {"compress_bloom", "bloom", NULL}, .type_id = TEXTOID, }, [_SparseIndexTypeEnumMinmax] = { .arg_names = {"compress_minmax", "minmax", "compress_min_max", "min_max", NULL}, .type_id = TEXTOID, }, }; WithClauseResult * ts_alter_table_with_clause_parse(const List *defelems) { return ts_with_clauses_parse(defelems, alter_table_with_clause_def, TS_ARRAY_LEN(alter_table_with_clause_def)); } WithClauseResult * ts_alter_table_reset_with_clause_parse(const List *defelems) { return ts_with_clauses_parse_reset(defelems, alter_table_with_clause_def, TS_ARRAY_LEN(alter_table_with_clause_def)); } static inline void throw_segment_by_error(char *segment_by) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("unable to parse segmenting option \"%s\"", segment_by), errhint("The option timescaledb.compress_segmentby must" " be a set of columns separated by commas."))); } static bool select_stmt_as_expected(SelectStmt *stmt) { /* The only parts of the select stmt that are allowed to be set are the order by or group by. * Check that no other fields are set */ if (stmt->distinctClause != NIL || stmt->intoClause != NULL || stmt->targetList != NIL || stmt->whereClause != NULL || stmt->havingClause != NULL || stmt->windowClause != NIL || stmt->valuesLists != NULL || stmt->limitOffset != NULL || stmt->limitCount != NULL || stmt->lockingClause != NIL || stmt->withClause != NULL || stmt->op != 0 || stmt->all != false || stmt->larg != NULL || stmt->rarg != NULL) return false; return true; } static ArrayType * parse_segment_collist(char *inpstr, Hypertable *hypertable) { StringInfoData buf; List *parsed; ListCell *lc; SelectStmt *select; RawStmt *raw; /* segmentby can have empty array */ if (strlen(inpstr) == 0) return ts_array_add_element_text(NULL, NULL); initStringInfo(&buf); /* parse the segment by list exactly how you would a group by */ appendStringInfo(&buf, "SELECT FROM %s.%s GROUP BY %s", quote_identifier(NameStr(hypertable->fd.schema_name)), quote_identifier(NameStr(hypertable->fd.table_name)), inpstr); const MemoryContext oldcontext = CurrentMemoryContext; PG_TRY(); { parsed = raw_parser(buf.data, RAW_PARSE_DEFAULT); } PG_CATCH(); { /* We do this fandango to avoid exhausting the error stack if we get * anything else but a syntax error, for example, an out of memory * error. */ ErrorData *edata; MemoryContextSwitchTo(oldcontext); edata = CopyErrorData(); FlushErrorState(); if (edata->sqlerrcode == ERRCODE_SYNTAX_ERROR) { edata->cursorpos = edata->internalpos = 0; edata->detail = edata->message; edata->message = psprintf("unable to parse segmenting option \"%s\"", inpstr); edata->hint = psprintf("The option timescaledb.compress_segmentby must be a set of " "columns separated by commas."); } ReThrowError(edata); } PG_END_TRY(); if (list_length(parsed) != 1) throw_segment_by_error(inpstr); if (!IsA(linitial(parsed), RawStmt)) throw_segment_by_error(inpstr); raw = linitial(parsed); if (!IsA(raw->stmt, SelectStmt)) throw_segment_by_error(inpstr); select = (SelectStmt *) raw->stmt; if (!select_stmt_as_expected(select)) throw_segment_by_error(inpstr); if (select->sortClause != NIL) throw_segment_by_error(inpstr); ArrayType *segmentby = NULL; foreach (lc, select->groupClause) { if (!IsA(lfirst(lc), ColumnRef)) throw_segment_by_error(inpstr); ColumnRef *cf = lfirst(lc); if (list_length(cf->fields) != 1) throw_segment_by_error(inpstr); if (!IsA(linitial(cf->fields), String)) throw_segment_by_error(inpstr); char *colname = strVal(linitial(cf->fields)); AttrNumber col_attno = get_attnum(hypertable->main_table_relid, colname); if (col_attno == InvalidAttrNumber) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("column \"%s\" does not exist", colname), errhint("The timescaledb.compress_segmentby option must reference a valid " "column."))); } /* get normalized column name */ colname = get_attname(hypertable->main_table_relid, col_attno, false); /* check if segmentby columns are distinct. */ if (ts_array_is_member(segmentby, colname)) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("duplicate column name \"%s\"", colname), errhint("The timescaledb.compress_segmentby option must reference distinct " "column."))); segmentby = ts_array_add_element_text(segmentby, pstrdup(colname)); } return segmentby; } static inline void throw_order_by_error(char *order_by) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("unable to parse ordering option \"%s\"", order_by), errhint("The timescaledb.compress_orderby option must be a set of column" " names with sort options, separated by commas." " It is the same format as an ORDER BY clause."))); } /* compress_orderby is parsed same as order by in select queries */ OrderBySettings ts_compress_parse_order_collist(char *inpstr, Hypertable *hypertable) { StringInfoData buf; List *parsed; ListCell *lc; SelectStmt *select; RawStmt *raw; OrderBySettings settings = { 0 }; if (strlen(inpstr) == 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("ordering column can not be empty"), errhint("timescaledb.compress_orderby option must reference a valid " "column or be removed to use default settings."))); initStringInfo(&buf); /* parse the segment by list exactly how you would a order by by */ appendStringInfo(&buf, "SELECT FROM %s.%s ORDER BY %s", quote_identifier(NameStr(hypertable->fd.schema_name)), quote_identifier(NameStr(hypertable->fd.table_name)), inpstr); const MemoryContext oldcontext = CurrentMemoryContext; PG_TRY(); { parsed = raw_parser(buf.data, RAW_PARSE_DEFAULT); } PG_CATCH(); { /* We do this fandango to avoid exhausting the error stack if we get * anything else but a syntax error, for example, an out of memory * error. */ ErrorData *edata; MemoryContextSwitchTo(oldcontext); edata = CopyErrorData(); FlushErrorState(); if (edata->sqlerrcode == ERRCODE_SYNTAX_ERROR) { edata->cursorpos = edata->internalpos = 0; edata->detail = edata->message; edata->message = psprintf("unable to parse ordering option \"%s\"", inpstr); edata->hint = psprintf("The timescaledb.compress_orderby option must be a set of column" " names with sort options, separated by commas." " It is the same format as an ORDER BY clause."); } ReThrowError(edata); } PG_END_TRY(); if (list_length(parsed) != 1) throw_order_by_error(inpstr); if (!IsA(linitial(parsed), RawStmt)) throw_order_by_error(inpstr); raw = linitial(parsed); if (!IsA(raw->stmt, SelectStmt)) throw_order_by_error(inpstr); select = (SelectStmt *) raw->stmt; if (!select_stmt_as_expected(select)) throw_order_by_error(inpstr); if (select->groupClause != NIL) throw_order_by_error(inpstr); foreach (lc, select->sortClause) { SortBy *sort_by; ColumnRef *cf; CompressedParsedCol *col = (CompressedParsedCol *) palloc(sizeof(*col)); bool desc, nullsfirst; if (!IsA(lfirst(lc), SortBy)) throw_order_by_error(inpstr); sort_by = lfirst(lc); if (!IsA(sort_by->node, ColumnRef)) throw_order_by_error(inpstr); cf = (ColumnRef *) sort_by->node; if (list_length(cf->fields) != 1) throw_order_by_error(inpstr); if (!IsA(linitial(cf->fields), String)) throw_order_by_error(inpstr); namestrcpy(&col->colname, strVal(linitial(cf->fields))); char *colname = strVal(linitial(cf->fields)); AttrNumber col_attno = get_attnum(hypertable->main_table_relid, colname); if (col_attno == InvalidAttrNumber) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("column \"%s\" does not exist", NameStr(col->colname)), errhint("The timescaledb.compress_orderby option must reference a valid " "column."))); Oid col_type = get_atttype(hypertable->main_table_relid, col_attno); TypeCacheEntry *type = lookup_type_cache(col_type, TYPECACHE_LT_OPR); if (!OidIsValid(type->lt_opr)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_FUNCTION), errmsg("invalid ordering column type %s", format_type_be(col_type)), errdetail("Could not identify a less-than operator for the type."))); /* get normalized column name */ colname = get_attname(hypertable->main_table_relid, col_attno, false); /* check if orderby columns are distinct. */ if (ts_array_is_member(settings.orderby, colname)) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("duplicate column name \"%s\"", colname), errhint("The timescaledb.compress_orderby option must reference distinct " "column."))); if (sort_by->sortby_dir != SORTBY_ASC && sort_by->sortby_dir != SORTBY_DESC && sort_by->sortby_dir != SORTBY_DEFAULT) throw_order_by_error(inpstr); desc = sort_by->sortby_dir == SORTBY_DESC; if (sort_by->sortby_nulls == SORTBY_NULLS_DEFAULT) { /* default null ordering is LAST for ASC, FIRST for DESC */ nullsfirst = desc; } else { nullsfirst = sort_by->sortby_nulls == SORTBY_NULLS_FIRST; } settings.orderby = ts_array_add_element_text(settings.orderby, pstrdup(colname)); settings.orderby_desc = ts_array_add_element_bool(settings.orderby_desc, desc); settings.orderby_nullsfirst = ts_array_add_element_bool(settings.orderby_nullsfirst, nullsfirst); } Ensure(settings.orderby, "orderby setting is NULL after parsing"); return settings; } static inline void throw_sparse_index_error(char *sparse_index) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("unable to parse sparse index option \"%s\"", sparse_index))); } static SparseIndexTypeEnum sparse_index_type_with_clause_parse(const char *parse, const WithClauseDefinition *args, int nargs) { Assert((int) _SparseIndexTypeEnumMax == nargs); int i; for (i = 0; i < nargs; i++) { for (int j = 0; args[i].arg_names[j] != NULL; ++j) { if (pg_strcasecmp(parse, args[i].arg_names[j]) == 0) { return (SparseIndexTypeEnum) i; } } } ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unrecognized sparse index type \"%s\"", parse))); return _SparseIndexTypeEnumMax; } static SparseIndexColumn parse_sparse_index_column(Hypertable *hypertable, FuncCall *sparse_index_details, int index, SparseIndexTypeEnum type) { SparseIndexColumn column; Assert(index >= 0); Assert(list_length(sparse_index_details->args) > index); if (index >= list_length(sparse_index_details->args) || index >= MAX_BLOOM_FILTER_COLUMNS) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("sparse index %s has too many columns", ts_sparse_index_type_names[type]))); Node *arg = list_nth(sparse_index_details->args, index); if (!IsA(arg, ColumnRef)) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("sparse index column reference must reference a valid column name"))); ColumnRef *cf = (ColumnRef *) arg; if (list_length(cf->fields) != 1 || !IsA(linitial(cf->fields), String)) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("invalid sparse index column reference syntax"), errdetail( "Wildcard or qualified references like '*' or 'table.col' are not allowed."))); column.name = strVal(linitial(cf->fields)); column.attnum = get_attnum(hypertable->main_table_relid, column.name); if (column.attnum == InvalidAttrNumber) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("column \"%s\" does not exist", column.name), errhint("The sparse index %s option must reference a valid " "column.", ts_sparse_index_type_names[type]))); } /* get normalized column name */ column.name = get_attname(hypertable->main_table_relid, column.attnum, false); column.type = get_atttype(hypertable->main_table_relid, column.attnum); return column; } static const char * column_name_list_as_string(BloomFilterConfig *config) { StringInfoData buf; initStringInfo(&buf); appendStringInfo(&buf, "("); for (int i = 0; i < config->num_columns; i++) { appendStringInfo(&buf, "'%s'", config->columns[i].name); if (i < config->num_columns - 1) appendStringInfo(&buf, ","); } appendStringInfo(&buf, ")"); return buf.data; } /* parses the individual sparse index config entities. being called once for each sparse index * config entity in the list. */ static void parse_sparse_index_config(JsonbParseState *parse_state, FuncCall *sparse_index_details, Hypertable *hypertable, TsBmsList *sparse_index_columns) { TypeCacheEntry *type_cache; MinmaxIndexColumnConfig minmax_config; BloomFilterConfig bloom_config; SparseIndexConfigBase config; SparseIndexConfigBase *config_ptr = &config; SparseIndexColumn first_column; config.type = sparse_index_type_with_clause_parse(NameListToString(sparse_index_details->funcname), sparse_index_with_clause_def, TS_ARRAY_LEN(sparse_index_with_clause_def)); config.source = _SparseIndexSourceEnumConfig; int num_columns = list_length(sparse_index_details->args); if (num_columns != 1) { if (num_columns > 1 && config.type == _SparseIndexTypeEnumBloom) { /* This will be enabled once all composite bloom index functionality is rolled out */ if (!ts_guc_enable_composite_bloom_indexes) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("composite bloom indexes are disabled"), errhint("Set timescaledb.enable_composite_bloom_indexes = true"))); if (num_columns > MAX_BLOOM_FILTER_COLUMNS) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("bloom index has too many columns: %d > max %d", num_columns, MAX_BLOOM_FILTER_COLUMNS))); } else { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("minmax index can only have one column"))); } } /* parse the first column separately because we only need one for minmax */ first_column = parse_sparse_index_column(hypertable, sparse_index_details, 0, config.type); Bitmapset *attnums_bitmap = bms_make_singleton(first_column.attnum); /* extract custom sparse index type config */ switch (config.type) { case _SparseIndexTypeEnumBloom: if (!ts_guc_enable_sparse_index_bloom) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Creating bloom sparse index is disabled"), errhint("Either set \"enable_sparse_index_bloom\" to true or remove the " "bloom filter indexes from \"sparse_index\" configuration of the " "hypertable."))); } bloom_config.base = config; config_ptr = (SparseIndexConfigBase *) &bloom_config; bloom_config.num_columns = num_columns; bloom_config.columns = palloc(num_columns * sizeof(SparseIndexColumn)); bloom_config.columns[0] = first_column; for (int i = 1; i < num_columns; i++) { bloom_config.columns[i] = parse_sparse_index_column(hypertable, sparse_index_details, i, config.type); attnums_bitmap = bms_add_member(attnums_bitmap, bloom_config.columns[i].attnum); if (bms_num_members(attnums_bitmap) <= i) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("duplicate column name ('%s') in composite bloom index " "configuration: %s", bloom_config.columns[i].name, column_name_list_as_string(&bloom_config)), errhint( "The sparse index option must reference distinct column set."))); } if (ts_bmslist_contains_set(*sparse_index_columns, attnums_bitmap)) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("duplicate sparse index configuration %s", column_name_list_as_string(&bloom_config)), errhint("The sparse index option must reference distinct column set."))); } *sparse_index_columns = ts_bmslist_add_set(*sparse_index_columns, attnums_bitmap); for (int i = 0; i < num_columns; i++) { /* * The column type must be hashable. For some types we use our own hash functions * which have better characteristics. */ FmgrInfo *finfo = NULL; if (ts_cm_functions->bloom1_get_hash_function(bloom_config.columns[i].type, &finfo) == NULL) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_FUNCTION), errmsg("invalid bloom filter column type %s", format_type_be(bloom_config.columns[i].type)), errdetail("Could not identify a hashing function for the type."))); } /* the convention is that the column names are sorted by attribute number */ qsort(bloom_config.columns, num_columns, sizeof(SparseIndexColumn), ts_qsort_attrnumber_cmp); break; case _SparseIndexTypeEnumMinmax: if (ts_bmslist_contains_set(*sparse_index_columns, attnums_bitmap)) { ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("duplicate column name \"%s\"", first_column.name), errhint("The sparse index option must reference distinct " "column."))); } *sparse_index_columns = ts_bmslist_add_set(*sparse_index_columns, attnums_bitmap); type_cache = lookup_type_cache(first_column.type, TYPECACHE_LT_OPR); /* * a comparison operator is required for min max operations */ if (!OidIsValid(type_cache->lt_opr)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_FUNCTION), errmsg("invalid minmax column type %s", format_type_be(first_column.type)), errdetail("Could not identify a less-than operator for the type."))); minmax_config.base = config; config_ptr = (SparseIndexConfigBase *) &minmax_config; minmax_config.col = first_column.name; break; default: ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("Invalid sparse index type"))); } ts_convert_sparse_index_config_to_jsonb(parse_state, config_ptr); } static Jsonb * parse_sparse_index_config_list(char *inpstr, Hypertable *hypertable) { StringInfoData buf; List *parsed; ListCell *lc; SelectStmt *select; RawStmt *raw; JsonbParseState *parse_state = NULL; /* sparse index can have empty input. Return [{"source":"config"}] jsonb */ if (strlen(inpstr) == 0) { pushJsonbValue(&parse_state, WJB_BEGIN_ARRAY, NULL); pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); ts_jsonb_add_str(parse_state, ts_sparse_index_common_keys[SparseIndexKeySource], ts_sparse_index_source_names[_SparseIndexSourceEnumConfig]); /* source */ JsonbValueToJsonb(pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL)); return JsonbValueToJsonb(pushJsonbValue(&parse_state, WJB_END_ARRAY, NULL)); } initStringInfo(&buf); /* parse the sparse index list exactly how you would targetlist */ appendStringInfo(&buf, "SELECT %s", inpstr); const MemoryContext oldcontext = CurrentMemoryContext; PG_TRY(); { parsed = raw_parser(buf.data, RAW_PARSE_DEFAULT); } PG_CATCH(); { /* We do this fandango to avoid exhausting the error stack if we get * anything else but a syntax error, for example, an out of memory * error. */ ErrorData *edata; MemoryContextSwitchTo(oldcontext); edata = CopyErrorData(); FlushErrorState(); if (edata->sqlerrcode == ERRCODE_SYNTAX_ERROR) { edata->cursorpos = edata->internalpos = 0; edata->detail = edata->message; edata->message = psprintf("unable to parse sparse index option \"%s\"", inpstr); } ReThrowError(edata); } PG_END_TRY(); if (list_length(parsed) != 1) throw_sparse_index_error(inpstr); if (!IsA(linitial(parsed), RawStmt)) throw_sparse_index_error(inpstr); raw = linitial(parsed); if (!IsA(raw->stmt, SelectStmt)) throw_sparse_index_error(inpstr); select = (SelectStmt *) raw->stmt; if (select->targetList == NULL) throw_sparse_index_error(inpstr); /* json format will be * [{"type": "bloom", "source":"config", "column": "u"}, * {"type": "minmax","source":"config", "column": "ts"}, * {"type": "bloom", "source":"config", "column": ["age", "gender"]}] */ pushJsonbValue(&parse_state, WJB_BEGIN_ARRAY, NULL); TsBmsList sparse_index_columns = ts_bmslist_create(); foreach (lc, select->targetList) { ResTarget *target = lfirst_node(ResTarget, lc); if (!IsA(target->val, FuncCall)) throw_sparse_index_error(inpstr); FuncCall *fc = (FuncCall *) target->val; parse_sparse_index_config(parse_state, fc, hypertable, &sparse_index_columns); } ts_bmslist_free(sparse_index_columns); return JsonbValueToJsonb(pushJsonbValue(&parse_state, WJB_END_ARRAY, NULL)); } /* returns List of CompressedParsedCol * compress_segmentby = `col1,col2,col3` */ ArrayType * ts_compress_hypertable_parse_segment_by(WithClauseResult segmentby, Hypertable *hypertable) { if (!segmentby.is_default) { return parse_segment_collist(TextDatumGetCString(segmentby.parsed), hypertable); } else return NULL; } /* returns List of CompressedParsedCol * E.g. timescaledb.compress_orderby = 'col1 asc nulls first,col2 desc,col3' */ OrderBySettings ts_compress_hypertable_parse_order_by(WithClauseResult orderby, Hypertable *hypertable) { Ensure(!orderby.is_default, "with clause is not default"); return ts_compress_parse_order_collist(TextDatumGetCString(orderby.parsed), hypertable); } /* returns List of CompressedParsedCol * E.g. timescaledb.compress_orderby = 'col1 asc nulls first,col2 desc,col3' */ Interval * ts_compress_hypertable_parse_chunk_time_interval(WithClauseResult *parsed_options, Hypertable *hypertable) { if (parsed_options[AlterTableFlagCompressChunkTimeInterval].is_default == false) { Datum textarg = parsed_options[AlterTableFlagCompressChunkTimeInterval].parsed; return DatumGetIntervalP(textarg); } else return NULL; } /* returns List of CompressedParsedCol * compress_minmax = `col1,col2,col3` */ Jsonb * ts_compress_hypertable_parse_index(WithClauseResult index, Hypertable *hypertable) { if (!index.is_default) { return parse_sparse_index_config_list(TextDatumGetCString(index.parsed), hypertable); } else return NULL; } ================================================ FILE: src/with_clause/alter_table_with_clause.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <catalog/pg_type.h> #include "chunk.h" #include "ts_catalog/catalog.h" #include "with_clause_parser.h" typedef enum AlterTableFlags { AlterTableFlagChunkTimeInterval = 0, AlterTableFlagColumnstore, AlterTableFlagSegmentBy, AlterTableFlagOrderBy, AlterTableFlagCompressChunkTimeInterval, AlterTableFlagIndex, AlterTableFlagsMax } AlterTableFlags; typedef struct { NameData colname; bool nullsfirst; bool desc; } CompressedParsedCol; typedef struct { ArrayType *orderby; ArrayType *orderby_desc; ArrayType *orderby_nullsfirst; } OrderBySettings; extern TSDLLEXPORT WithClauseResult *ts_alter_table_with_clause_parse(const List *defelems); extern TSDLLEXPORT WithClauseResult *ts_alter_table_reset_with_clause_parse(const List *defelems); extern TSDLLEXPORT ArrayType *ts_compress_hypertable_parse_segment_by(WithClauseResult segmentby, Hypertable *hypertable); extern TSDLLEXPORT OrderBySettings ts_compress_hypertable_parse_order_by(WithClauseResult orderby, Hypertable *hypertable); extern TSDLLEXPORT Interval * ts_compress_hypertable_parse_chunk_time_interval(WithClauseResult *parsed_options, Hypertable *hypertable); extern TSDLLEXPORT OrderBySettings ts_compress_parse_order_collist(char *inpstr, Hypertable *hypertable); extern TSDLLEXPORT Jsonb *ts_compress_hypertable_parse_index(WithClauseResult index, Hypertable *hypertable); ================================================ FILE: src/with_clause/create_materialized_view_with_clause.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_type.h> #include <fmgr.h> #include <nodes/makefuncs.h> #include "compat/compat.h" #include "alter_table_with_clause.h" #include "create_materialized_view_with_clause.h" #include "cross_module_fn.h" #include "ts_catalog/continuous_agg.h" #include "with_clause_parser.h" static const WithClauseDefinition continuous_aggregate_with_clause_def[] = { [CreateMaterializedViewFlagContinuous] = { .arg_names = {"continuous", NULL}, .type_id = BOOLOID, .default_val = (Datum)false, }, [CreateMaterializedViewFlagCreateGroupIndexes] = { .arg_names = {"create_group_indexes", NULL}, .type_id = BOOLOID, .default_val = (Datum)true, }, [CreateMaterializedViewFlagMaterializedOnly] = { .arg_names = {"materialized_only", NULL}, .type_id = BOOLOID, .default_val = (Datum)true, }, [CreateMaterializedViewFlagColumnstore] = { .arg_names = {"columnstore", "enable_columnstore", "compress", NULL}, .type_id = BOOLOID, }, [CreateMaterializedViewFlagChunkTimeInterval] = { .arg_names = {"chunk_interval", NULL}, .type_id = INTERVALOID, }, [CreateMaterializedViewFlagSegmentBy] = { .arg_names = {"segmentby", "segment_by", "compress_segmentby", NULL}, .type_id = TEXTOID, }, [CreateMaterializedViewFlagOrderBy] = { .arg_names = {"orderby", "order_by", "compress_orderby", NULL}, .type_id = TEXTOID, }, [CreateMaterializedViewFlagCompressChunkTimeInterval] = { .arg_names = {"compress_chunk_interval", "compress_chunk_time_interval", NULL}, .type_id = INTERVALOID, }, }; WithClauseResult * ts_create_materialized_view_with_clause_parse(const List *defelems) { return ts_with_clauses_parse(defelems, continuous_aggregate_with_clause_def, TS_ARRAY_LEN(continuous_aggregate_with_clause_def)); } List * ts_continuous_agg_get_compression_defelems(const WithClauseResult *with_clauses) { List *ret = NIL; for (int i = 0; i < AlterTableFlagsMax; i++) { int option_index = 0; switch (i) { case AlterTableFlagChunkTimeInterval: case AlterTableFlagIndex: continue; break; case AlterTableFlagColumnstore: option_index = CreateMaterializedViewFlagColumnstore; break; case AlterTableFlagSegmentBy: option_index = CreateMaterializedViewFlagSegmentBy; break; case AlterTableFlagOrderBy: option_index = CreateMaterializedViewFlagOrderBy; break; case AlterTableFlagCompressChunkTimeInterval: option_index = CreateMaterializedViewFlagCompressChunkTimeInterval; break; default: elog(ERROR, "Unhandled compression option"); break; } const WithClauseResult *input = &with_clauses[option_index]; WithClauseDefinition def = continuous_aggregate_with_clause_def[option_index]; if (!input->is_default) { Node *value = (Node *) makeString(ts_with_clause_result_deparse_value(input)); DefElem *elem = makeDefElemExtended(EXTENSION_NAMESPACE, (char *) def.arg_names[0], value, DEFELEM_UNSPEC, -1); ret = lappend(ret, elem); } } return ret; } ================================================ FILE: src/with_clause/create_materialized_view_with_clause.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include "with_clause_parser.h" typedef enum CreateMaterializedViewFlags { CreateMaterializedViewFlagContinuous = 0, CreateMaterializedViewFlagCreateGroupIndexes, CreateMaterializedViewFlagMaterializedOnly, CreateMaterializedViewFlagColumnstore, CreateMaterializedViewFlagChunkTimeInterval, CreateMaterializedViewFlagSegmentBy, CreateMaterializedViewFlagOrderBy, CreateMaterializedViewFlagCompressChunkTimeInterval, } CreateMaterializedViewFlags; extern TSDLLEXPORT WithClauseResult * ts_create_materialized_view_with_clause_parse(const List *defelems); extern TSDLLEXPORT List * ts_continuous_agg_get_compression_defelems(const WithClauseResult *with_clauses); ================================================ FILE: src/with_clause/create_table_with_clause.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_type.h> #include <fmgr.h> #include "compat/compat.h" #include "create_table_with_clause.h" #include "with_clause_parser.h" static const WithClauseDefinition create_table_with_clauses_def[] = { [CreateTableFlagHypertable] = {.arg_names = {"hypertable", NULL}, .type_id = BOOLOID,}, [CreateTableFlagColumnstore] = {.arg_names = {"columnstore", "enable_columnstore", "compress", NULL}, .type_id = BOOLOID, .default_val = (Datum)true,}, [CreateTableFlagTimeColumn] = {.arg_names = {"partition_column", "partitioning_column", NULL}, .type_id = TEXTOID,}, [CreateTableFlagChunkTimeInterval] = {.arg_names = {"chunk_interval", NULL}, .type_id = TEXTOID,}, [CreateTableFlagCreateDefaultIndexes] = {.arg_names = {"create_default_indexes", NULL}, .type_id = BOOLOID, .default_val = (Datum) true,}, [CreateTableFlagAssociatedSchema] = {.arg_names = {"associated_schema", NULL}, .type_id = TEXTOID,}, [CreateTableFlagAssociatedTablePrefix] = {.arg_names = {"associated_table_prefix", NULL}, .type_id = TEXTOID,}, [CreateTableFlagSegmentBy] = { .arg_names = {"segmentby", "segment_by", "compress_segmentby", NULL}, .type_id = TEXTOID,}, [CreateTableFlagOrderBy] = { .arg_names = {"orderby", "order_by", "compress_orderby", NULL}, .type_id = TEXTOID,}, [CreateTableFlagIndex] = { .arg_names = {"compress_index", "compress_sparse_index", "index", "sparse_index", NULL}, .type_id = TEXTOID,}, }; WithClauseResult * ts_create_table_with_clause_parse(const List *defelems) { return ts_with_clauses_parse(defelems, create_table_with_clauses_def, TS_ARRAY_LEN(create_table_with_clauses_def)); } Datum ts_create_table_parse_chunk_time_interval(WithClauseResult option, Oid column_type, Oid *interval_type) { if (option.is_default == false) { Datum textarg = option.parsed; switch (column_type) { case INT2OID: { *interval_type = INT2OID; return DirectFunctionCall1(int2in, CStringGetDatum(TextDatumGetCString(textarg))); } case INT4OID: { *interval_type = INT4OID; return DirectFunctionCall1(int4in, CStringGetDatum(TextDatumGetCString(textarg))); } case INT8OID: { *interval_type = INT8OID; return DirectFunctionCall1(int8in, CStringGetDatum(TextDatumGetCString(textarg))); } case TIMESTAMPOID: case TIMESTAMPTZOID: case DATEOID: case UUIDOID: { *interval_type = INTERVALOID; return DirectFunctionCall3(interval_in, CStringGetDatum(TextDatumGetCString(textarg)), InvalidOid, -1); } default: Ensure(false, "unexpected column type %s when setting chunk interval", format_type_be(column_type)); } } *interval_type = InvalidOid; return UnassignedDatum; } ================================================ FILE: src/with_clause/create_table_with_clause.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include "with_clause_parser.h" typedef enum CreateTableFlags { CreateTableFlagHypertable = 0, CreateTableFlagColumnstore, CreateTableFlagTimeColumn, CreateTableFlagChunkTimeInterval, CreateTableFlagCreateDefaultIndexes, CreateTableFlagAssociatedSchema, CreateTableFlagAssociatedTablePrefix, CreateTableFlagOrderBy, CreateTableFlagSegmentBy, CreateTableFlagIndex } CreateTableFlags; WithClauseResult *ts_create_table_with_clause_parse(const List *defelems); Datum ts_create_table_parse_chunk_time_interval(WithClauseResult option, Oid column_type, Oid *interval_type); ================================================ FILE: src/with_clause/with_clause_parser.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <fmgr.h> #include <access/htup_details.h> #include <catalog/pg_type.h> #include <commands/defrem.h> #include <nodes/parsenodes.h> #include <utils/builtins.h> #include <utils/lsyscache.h> #include <utils/syscache.h> #include "debug_assert.h" #include "extension_constants.h" #include "with_clause_parser.h" /* * Filter a list of DefElem based on a namespace. * This function will iterate through DefElem and output up to two lists: * within_namespace: every element within the namespace * not_within_namespace: all the other elements * * That is, given a with clause like: * WITH (foo.foo_para, bar.bar_param, baz.baz_param) * * ts_with_clause_filter(elems, "foo", in, not_in) will have * in = foo.foo_para * not_in = bar.bar_param, baz.baz_param */ void ts_with_clause_filter(const List *def_elems, List **within_namespace, List **other_namespace, List **not_within_namespace) { ListCell *cell; foreach (cell, def_elems) { DefElem *def = (DefElem *) lfirst(cell); if (def->defnamespace != NULL && (pg_strcasecmp(def->defnamespace, EXTENSION_NAMESPACE) == 0 || pg_strcasecmp(def->defnamespace, EXTENSION_NAMESPACE_ALIAS) == 0)) { if (within_namespace != NULL) *within_namespace = lappend(*within_namespace, def); } else if (def->defnamespace != NULL && other_namespace != NULL) { *other_namespace = lappend(*other_namespace, def); } else if (not_within_namespace != NULL) { *not_within_namespace = lappend(*not_within_namespace, def); } } } static Datum parse_arg(WithClauseDefinition arg, DefElem *def); static char * ts_with_clause_definition_names(const WithClauseDefinition *args, Size nargs) { StringInfoData buf; Size i; initStringInfo(&buf); for (i = 0; i < nargs; i++) { if (i > 0) appendStringInfoString(&buf, ", "); appendStringInfoString(&buf, args[i].arg_names[0]); } return buf.data; } /* * Deserialize and apply the values in a WITH clause based on the on_arg table. * * This function will go through every element in def_elems and search for a * corresponding argument in args, if one is found it will attempt to deserialize * the argument, using that table elements deserialize function, then apply it * to state. * * This is used to turn the list into a form more useful for our internal * functions */ WithClauseResult * ts_with_clauses_parse(const List *def_elems, const WithClauseDefinition *args, Size nargs) { ListCell *cell; WithClauseResult *results = palloc0(sizeof(*results) * nargs); Size i; for (i = 0; i < nargs; i++) { results[i].definition = &args[i]; results[i].parsed = args[i].default_val; results[i].is_default = true; } foreach (cell, def_elems) { DefElem *def = (DefElem *) lfirst(cell); bool argument_recognized = false; for (i = 0; i < nargs; i++) { for (int j = 0; args[i].arg_names[j] != NULL; ++j) { if (pg_strcasecmp(def->defname, args[i].arg_names[j]) == 0) { argument_recognized = true; if (!results[i].is_default) ereport(ERROR, (errcode(ERRCODE_AMBIGUOUS_PARAMETER), errmsg("duplicate parameter \"%s.%s\"", def->defnamespace, def->defname))); results[i].parsed = parse_arg(args[i], def); results[i].is_default = false; break; } } } if (!argument_recognized) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unrecognized parameter \"%s.%s\"", def->defnamespace, def->defname), errhint("Valid timescaledb parameters are: %s", ts_with_clause_definition_names(args, nargs)))); } return results; } /* * This function handles parsing of WITH clauses for ALTER TABLE RESET. * Unlike ts_with_clauses_parse, it does not parse any option values, * as RESET clauses only include option names without associated values. */ WithClauseResult * ts_with_clauses_parse_reset(const List *def_elems, const WithClauseDefinition *args, Size nargs) { ListCell *cell; WithClauseResult *results = palloc0(sizeof(*results) * nargs); Size i; for (i = 0; i < nargs; i++) { results[i].definition = &args[i]; results[i].parsed = args[i].default_val; results[i].is_default = true; } foreach (cell, def_elems) { DefElem *def = (DefElem *) lfirst(cell); bool argument_recognized = false; for (i = 0; i < nargs; i++) { for (int j = 0; args[i].arg_names[j] != NULL; ++j) { if (pg_strcasecmp(def->defname, args[i].arg_names[j]) == 0) { argument_recognized = true; if (!results[i].is_default) ereport(ERROR, (errcode(ERRCODE_AMBIGUOUS_PARAMETER), errmsg("duplicate parameter \"%s.%s\"", def->defnamespace, def->defname))); results[i].is_default = false; break; } } } if (!argument_recognized) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unrecognized parameter \"%s.%s\"", def->defnamespace, def->defname), errhint("Valid timescaledb parameters are: %s", ts_with_clause_definition_names(args, nargs)))); } return results; } extern TSDLLEXPORT char * ts_with_clause_result_deparse_value(const WithClauseResult *result) { Ensure(OidIsValid(result->definition->type_id), "argument \"%d\" has invalid OID", result->definition->type_id); return ts_datum_to_string(result->parsed, result->definition->type_id); } static Datum parse_arg(WithClauseDefinition arg, DefElem *def) { char *value; Datum val; Oid in_fn; Oid typIOParam; if (!OidIsValid(arg.type_id)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_PARAMETER), errmsg("argument \"%s.%s\" not implemented", def->defnamespace, def->defname))); if (def->arg != NULL) value = defGetString(def); else if (arg.type_id == BOOLOID) /* for booleans, postgres defines the option timescale.foo to be the same as * timescaledb.foo='true' so if no value is found set it to "true" here */ value = "true"; else ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("parameter \"%s.%s\" must have a value", def->defnamespace, def->defname))); getTypeInputInfo(arg.type_id, &in_fn, &typIOParam); Assert(OidIsValid(in_fn)); /* * We could use InputFunctionCallSafe() here but this is just supported * for PG16 and later, so we opt for checking if the failure is what we * expected and re-throwing the error otherwise. */ PG_TRY(); { val = OidInputFunctionCall(in_fn, value, typIOParam, -1); } PG_CATCH(); { const int sqlerrcode = geterrcode(); /* * We can deal with the Data Exception category and in the Syntax * Error or Access Rule Violation category, but if the error is an * insufficient resources category, for example, an out of memory * error, we should just re-throw it. * * Errors in other categories are unlikely, but we cannot do anything * with them anyway, so just re-throw them as well. */ if (ERRCODE_TO_CATEGORY(sqlerrcode) != ERRCODE_DATA_EXCEPTION && ERRCODE_TO_CATEGORY(sqlerrcode) != ERRCODE_SYNTAX_ERROR_OR_ACCESS_RULE_VIOLATION) { PG_RE_THROW(); } FlushErrorState(); /* We are currently using the ErrorContext, but since we are going to * raise an error later, there is no reason to switch memory context * nor restore the resource owner here. */ Form_pg_type typetup; HeapTuple tup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(arg.type_id)); if (!HeapTupleIsValid(tup)) elog(ERROR, "cache lookup failed for type of %s.%s '%u'", def->defnamespace, def->defname, arg.type_id); typetup = (Form_pg_type) GETSTRUCT(tup); ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid value for %s.%s '%s'", def->defnamespace, def->defname, value), errhint("%s.%s must be a valid %s", def->defnamespace, def->defname, NameStr(typetup->typname)))); } PG_END_TRY(); return val; } ================================================ FILE: src/with_clause/with_clause_parser.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/parsenodes.h> #include <utils.h> #include "compat/compat.h" typedef struct WithClauseDefinition { /* Alternative names for the parameters. The first one is the "main" one * when it comes to printouts.*/ const char *arg_names[5]; Oid type_id; Datum default_val; } WithClauseDefinition; typedef struct WithClauseResult { const WithClauseDefinition *definition; bool is_default; Datum parsed; } WithClauseResult; extern TSDLLEXPORT void ts_with_clause_filter(const List *def_elems, List **within_namespace, List **other_namespace, List **not_within_namespace); extern TSDLLEXPORT WithClauseResult * ts_with_clauses_parse(const List *def_elems, const WithClauseDefinition *args, Size nargs); extern TSDLLEXPORT WithClauseResult * ts_with_clauses_parse_reset(const List *def_elems, const WithClauseDefinition *args, Size nargs); extern TSDLLEXPORT char *ts_with_clause_result_deparse_value(const WithClauseResult *result); ================================================ FILE: test/.gitignore ================================================ results/ dump/ regression.diffs regression.out unit/testoutputs.tmp ================================================ FILE: test/CMakeLists.txt ================================================ set(PRIMARY_TEST_DIR ${CMAKE_CURRENT_LIST_DIR}) set(PRIMARY_TEST_DIR ${CMAKE_CURRENT_LIST_DIR} PARENT_SCOPE) set(_local_install_checks) set(_install_checks) # Testing support include(test-defs.cmake) # No checks for REGRESS_CHECKS needed here since all the checks are done in the # parent CMakeLists.txt. if(PG_REGRESS) message(STATUS "Using pg_regress ${PG_REGRESS}") add_custom_target( regresscheck COMMAND ${CMAKE_COMMAND} -E env ${PG_REGRESS_ENV} TEST_PGPORT=${TEST_PGPORT_TEMP_INSTANCE} TEST_SCHEDULE=${TEST_SCHEDULE} TEST_TIMEOUT=${TEST_TIMEOUT} ${CMAKE_CURRENT_SOURCE_DIR}/pg_regress.sh ${PG_REGRESS_OPTS_BASE} ${PG_REGRESS_OPTS_EXTRA} ${PG_REGRESS_OPTS_INOUT} ${PG_REGRESS_OPTS_TEMP_INSTANCE} --temp-config=${TEST_OUTPUT_DIR}/postgresql.conf USES_TERMINAL) add_custom_target( regresscheck-rerun COMMAND ${PRIMARY_TEST_DIR}/ci_rerun.sh regresscheck USES_TERMINAL) add_custom_target( regresschecklocal COMMAND ${CMAKE_COMMAND} -E env ${PG_REGRESS_ENV} TEST_PGPORT=${TEST_PGPORT_LOCAL} TEST_TIMEOUT=${TEST_TIMEOUT} ${CMAKE_CURRENT_SOURCE_DIR}/pg_regress.sh ${PG_REGRESS_OPTS_BASE} ${PG_REGRESS_OPTS_EXTRA} ${PG_REGRESS_OPTS_INOUT} ${PG_REGRESS_OPTS_LOCAL_INSTANCE} USES_TERMINAL) list(APPEND _local_install_checks regresschecklocal) list(APPEND _install_checks regresscheck) elseif(REQUIRE_ALL_TESTS) message( FATAL_ERROR "All tests were required but 'pg_regress' could not be found") endif() if(PG_ISOLATION_REGRESS) message(STATUS "Using pg_isolation_regress ${PG_ISOLATION_REGRESS}") add_custom_target( isolationcheck COMMAND ${CMAKE_COMMAND} -E env ${PG_ISOLATION_REGRESS_ENV} SPECS_DIR=${CMAKE_CURRENT_SOURCE_DIR}/isolation/specs TEST_PGPORT=${TEST_PGPORT_TEMP_INSTANCE} ${CMAKE_CURRENT_SOURCE_DIR}/pg_regress.sh ${PG_REGRESS_OPTS_BASE} ${PG_ISOLATION_REGRESS_OPTS_EXTRA} ${PG_ISOLATION_REGRESS_OPTS_INOUT} ${PG_REGRESS_OPTS_TEMP_INSTANCE} --temp-config=${TEST_OUTPUT_DIR}/postgresql.conf USES_TERMINAL) add_custom_target( isolationcheck-rerun COMMAND ${PRIMARY_TEST_DIR}/ci_rerun.sh isolationcheck USES_TERMINAL) add_custom_target( isolationchecklocal COMMAND ${CMAKE_COMMAND} -E env ${PG_ISOLATION_REGRESS_ENV} SPECS_DIR=${CMAKE_CURRENT_SOURCE_DIR}/isolation/specs TEST_PGPORT=${TEST_PGPORT_LOCAL} ${CMAKE_CURRENT_SOURCE_DIR}/pg_regress.sh ${PG_REGRESS_OPTS_BASE} ${PG_ISOLATION_REGRESS_OPTS_EXTRA} ${PG_ISOLATION_REGRESS_OPTS_INOUT} ${PG_REGRESS_OPTS_LOCAL_INSTANCE} USES_TERMINAL) list(APPEND _local_install_checks isolationchecklocal) list(APPEND _install_checks isolationcheck) elseif(REQUIRE_ALL_TESTS) message( FATAL_ERROR "All tests were required but 'pg_isolation_regress' could not be found") endif() if(TAP_CHECKS) add_custom_target( provecheck COMMAND rm -rf ${CMAKE_CURRENT_BINARY_DIR}/tmp_check COMMAND CONFDIR=${CMAKE_BINARY_DIR}/test PATH="${PG_BINDIR}:$ENV{PATH}" PG_REGRESS=${PG_REGRESS} SRC_DIR=${PG_SOURCE_DIR} CM_SRC_DIR=${CMAKE_SOURCE_DIR} PG_LIBDIR=${PG_LIBDIR} PG_PKGLIBDIR=${PG_PKGLIBDIR} PG_VERSION_MAJOR=${PG_VERSION_MAJOR} ${PRIMARY_TEST_DIR}/pg_prove.sh USES_TERMINAL) list(APPEND _install_checks provecheck) elseif(REQUIRE_ALL_TESTS) message( FATAL_ERROR "All tests were required but TAP_CHECKS was off (see previous messages why)" ) endif() # We add the installcheck target even when _install_checks is empty as tsl code # might add dependencies to it even when regress checks are disabled. add_custom_target(installcheck DEPENDS ${_install_checks}) # Define a post test hook that is invoked after the installcheck target # finishes. One can use add_dependencies on post hook target to run other # targets after tests complete. This is used, e.g., by code coverage. add_custom_target(installcheck-post-hook COMMENT "Post test hook") add_custom_command( TARGET installcheck POST_BUILD COMMAND cmake --build ${CMAKE_CURRENT_BINARY_DIR} --target installcheck-post-hook) add_custom_target( installcheck-rerun COMMAND ${PRIMARY_TEST_DIR}/ci_rerun.sh installcheck USES_TERMINAL) # installchecklocal tests against an existing postgres instance add_custom_target(installchecklocal DEPENDS ${_local_install_checks}) add_subdirectory(sql) add_subdirectory(isolation) add_subdirectory(t) if(PG_SOURCE_DIR) add_subdirectory(pgtest) endif(PG_SOURCE_DIR) if(CMAKE_BUILD_TYPE MATCHES Debug) add_subdirectory(src) endif(CMAKE_BUILD_TYPE MATCHES Debug) add_subdirectory(perl) ================================================ FILE: test/README.md ================================================ # Testing TimescaleDB - [Regression tests](pgtest/README.md) - [Perl-based TAP tests](perl/README.md) - [Background Worker Test Infrastructure](src/bgw/README.md) ================================================ FILE: test/ci_rerun.sh ================================================ #!/bin/bash subcommand=$1 out_files="regression.out" case "$subcommand" in installcheck) out_files=$(find .. -name "regression.out") ;; regresscheck-shared) out_files="shared/regression.out" ;; isolationcheck|isolationcheck-t) out_files="isolation/regression.out" ;; *) out_files="regression.out" ;; esac failed=$(grep -h -P "^not ok|FAILED" $out_files | sed -r -e 's!^not ok [0-9]+ +[+-] ([a-z0-9_-]+) .*$!\1!' -e 's!^(test| ) ([a-z0-9_-]+) +... FAILED.*$!\2!') failed="${failed//$'\n'/ }" make -k $subcommand TESTS="$failed" ================================================ FILE: test/expected/agg_bookends-15.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set TEST_BASE_NAME agg_bookends SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') as "TEST_LOAD_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') as "TEST_QUERY_NAME", format('%s/results/%s_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_OPTIMIZED", format('%s/results/%s_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_UNOPTIMIZED" \gset SELECT format('\! diff -u --label "Unoptimized result" --label "Optimized result" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') as "DIFF_CMD" \gset \set PREFIX 'EXPLAIN (analyze, buffers off, costs off, timing off, summary off)' \ir :TEST_LOAD_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE btest(time timestamp NOT NULL, time_alt timestamp, gp INTEGER, temp float, strid TEXT DEFAULT 'testing'); SELECT schema_name, table_name, created FROM create_hypertable('btest', 'time'); psql:include/agg_bookends_load.sql:6: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices psql:include/agg_bookends_load.sql:6: WARNING: column type "timestamp without time zone" used for "time_alt" does not follow best practices schema_name | table_name | created -------------+------------+--------- public | btest | t INSERT INTO btest VALUES('2017-01-20T09:00:01', '2017-01-20T10:00:00', 1, 22.5); INSERT INTO btest VALUES('2017-01-20T09:00:21', '2017-01-20T09:00:59', 1, 21.2); INSERT INTO btest VALUES('2017-01-20T09:00:47', '2017-01-20T09:00:58', 1, 25.1); INSERT INTO btest VALUES('2017-01-20T09:00:02', '2017-01-20T09:00:57', 2, 35.5); INSERT INTO btest VALUES('2017-01-20T09:00:21', '2017-01-20T09:00:56', 2, 30.2); --TOASTED; INSERT INTO btest VALUES('2017-01-20T09:00:43', '2017-01-20T09:01:55', 2, 20.1, repeat('xyz', 1000000) ); CREATE TABLE btest_numeric (time timestamp NOT NULL, quantity numeric); SELECT schema_name, table_name, created FROM create_hypertable('btest_numeric', 'time'); psql:include/agg_bookends_load.sql:16: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices schema_name | table_name | created -------------+---------------+--------- public | btest_numeric | t \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be only output of results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations')) v(setting); setting | value ----------------------------------+------- timescaledb.enable_optimizations | on :PREFIX SELECT time, gp, temp FROM btest ORDER BY time; --- QUERY PLAN --- Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) :PREFIX SELECT first(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) :PREFIX SELECT last(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT first(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT gp, last(temp, time) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT gp, first(temp, time) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) --check whole row :PREFIX SELECT gp, first(btest, time) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) --check toasted col :PREFIX SELECT gp, left(last(strid, time), 10) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT gp, last(temp, strid) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT gp, last(strid, temp) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) BEGIN; --check null value as last element INSERT INTO btest VALUES('2018-01-20T09:00:43', '2017-01-20T09:00:55', 2, NULL); :PREFIX SELECT last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) Index Cond: ("time" IS NOT NULL) --check non-null element "overrides" NULL because it comes after. INSERT INTO btest VALUES('2019-01-20T09:00:43', '2018-01-20T09:00:55', 2, 30.5); :PREFIX SELECT last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) Index Cond: ("time" IS NOT NULL) --check null cmp element is skipped INSERT INTO btest VALUES('2018-01-20T09:00:43', NULL, 2, 32.3); :PREFIX SELECT last(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=9.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -- fist returns NULL value :PREFIX SELECT first(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=9.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -- test first return non NULL value INSERT INTO btest VALUES('2016-01-20T09:00:00', '2016-01-20T09:00:00', 2, 36.5); :PREFIX SELECT first(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=10.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) --check non null cmp element insert after null cmp INSERT INTO btest VALUES('2020-01-20T09:00:43', '2020-01-20T09:00:43', 2, 35.3); :PREFIX SELECT last(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) :PREFIX SELECT first(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) --cmp nulls should be ignored and not present in groups :PREFIX SELECT gp, last(temp, time_alt) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: btest.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: btest.gp -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) --Previously, some bugs were found with NULLS and numeric types, so test that INSERT INTO btest_numeric VALUES ('2019-01-20T09:00:43', NULL); :PREFIX SELECT last(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Index Scan using _hyper_2_6_chunk_btest_numeric_time_idx on _hyper_2_6_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) --check non-null element "overrides" NULL because it comes after. INSERT INTO btest_numeric VALUES('2020-01-20T09:00:43', 30.5); :PREFIX SELECT last(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest_numeric (actual rows=1.00 loops=1) Order: btest_numeric."time" DESC -> Index Scan using _hyper_2_7_chunk_btest_numeric_time_idx on _hyper_2_7_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_2_6_chunk_btest_numeric_time_idx on _hyper_2_6_chunk (never executed) Index Cond: ("time" IS NOT NULL) -- do index scan for last :PREFIX SELECT last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) Index Cond: ("time" IS NOT NULL) -- do index scan for first :PREFIX SELECT first(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" -> Index Scan Backward using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) -> Index Scan Backward using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan Backward using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) -> Index Scan Backward using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (never executed) -- can't do index scan when ordering on non-index column :PREFIX SELECT first(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) -- do index scan for subquery :PREFIX SELECT * FROM (SELECT last(temp, time) FROM btest) last; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) Index Cond: ("time" IS NOT NULL) -- can't do index scan when using group by :PREFIX SELECT last(temp, time) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: btest.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: btest.gp -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) -- do index scan when agg function is used in CTE subquery :PREFIX WITH last_temp AS (SELECT last(temp, time) FROM btest) SELECT * from last_temp; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) Index Cond: ("time" IS NOT NULL) -- do index scan when using both FIRST and LAST aggregate functions :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $1) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) Index Cond: ("time" IS NOT NULL) InitPlan 2 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest btest_1 (actual rows=1.00 loops=1) Order: btest_1."time" -> Index Scan Backward using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk _hyper_1_4_chunk_1 (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk _hyper_1_3_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk _hyper_1_5_chunk_1 (never executed) -- verify results when using both FIRST and LAST :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $1) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) Index Cond: ("time" IS NOT NULL) InitPlan 2 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest btest_1 (actual rows=1.00 loops=1) Order: btest_1."time" -> Index Scan Backward using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk _hyper_1_4_chunk_1 (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk _hyper_1_3_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk _hyper_1_5_chunk_1 (never executed) -- do index scan when using WHERE :PREFIX SELECT last(temp, time) FROM btest WHERE time <= '2017-01-20T09:00:02'; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) Index Cond: (("time" IS NOT NULL) AND ("time" <= 'Fri Jan 20 09:00:02 2017'::timestamp without time zone)) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) Index Cond: ("time" IS NOT NULL) -- can't do index scan for MAX and LAST combined (MinMax optimization fails when having different aggregate functions) :PREFIX SELECT max(time), last(temp, time) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) -- can't do index scan when using FIRST/LAST in ORDER BY :PREFIX SELECT last(temp, time) FROM btest ORDER BY last(temp, time); --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: (last(btest.temp, btest."time")) Sort Method: quicksort -> Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) -- do index scan :PREFIX SELECT last(temp, time) FROM btest WHERE temp < 30; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=0.00 loops=1) Index Cond: ("time" IS NOT NULL) Filter: (temp < '30'::double precision) Rows Removed by Filter: 1 -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (actual rows=0.00 loops=1) Index Cond: ("time" IS NOT NULL) Filter: (temp < '30'::double precision) Rows Removed by Filter: 1 -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (actual rows=0.00 loops=1) Index Cond: ("time" IS NOT NULL) Filter: (temp < '30'::double precision) Rows Removed by Filter: 2 -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) Filter: (temp < '30'::double precision) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) Index Cond: ("time" IS NOT NULL) Filter: (temp < '30'::double precision) -- SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- do index scan :PREFIX SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" -> Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Fri Jan 20 09:00:47 2017'::timestamp without time zone) -> Index Scan Backward using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan Backward using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) -> Index Scan Backward using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (never executed) -- can't do index scan when using WINDOW function :PREFIX SELECT gp, last(temp, time) OVER (PARTITION BY gp) AS last FROM btest; --- QUERY PLAN --- WindowAgg (actual rows=11.00 loops=1) -> Sort (actual rows=11.00 loops=1) Sort Key: btest.gp Sort Method: quicksort -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) -- test constants :PREFIX SELECT first(100, 100) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Result (actual rows=1.00 loops=1) -> Append (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (never executed) -> Seq Scan on _hyper_1_3_chunk (never executed) -> Seq Scan on _hyper_1_4_chunk (never executed) -> Seq Scan on _hyper_1_5_chunk (never executed) -- create an index so we can test optimization CREATE INDEX btest_time_alt_idx ON btest(time_alt); :PREFIX SELECT last(temp, time_alt) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Merge Append (actual rows=1.00 loops=1) Sort Key: btest.time_alt DESC -> Index Scan Backward using _hyper_1_1_chunk_btest_time_alt_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) Index Cond: (time_alt IS NOT NULL) -> Index Scan Backward using _hyper_1_2_chunk_btest_time_alt_idx on _hyper_1_2_chunk (actual rows=1.00 loops=1) Index Cond: (time_alt IS NOT NULL) -> Index Scan Backward using _hyper_1_3_chunk_btest_time_alt_idx on _hyper_1_3_chunk (actual rows=1.00 loops=1) Index Cond: (time_alt IS NOT NULL) -> Index Scan Backward using _hyper_1_4_chunk_btest_time_alt_idx on _hyper_1_4_chunk (actual rows=1.00 loops=1) Index Cond: (time_alt IS NOT NULL) -> Index Scan Backward using _hyper_1_5_chunk_btest_time_alt_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) Index Cond: (time_alt IS NOT NULL) --test nested FIRST/LAST - should optimize :PREFIX SELECT abs(last(temp, time)) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) Index Cond: ("time" IS NOT NULL) -- test nested FIRST/LAST in ORDER BY - no optimization possible :PREFIX SELECT abs(last(temp, time)) FROM btest ORDER BY abs(last(temp,time)); --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: (abs(last(btest.temp, btest."time"))) Sort Method: quicksort -> Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) ROLLBACK; -- Test with NULL numeric values BEGIN; TRUNCATE btest_numeric; -- Empty table :PREFIX SELECT first(btest_numeric, time) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Result (actual rows=0.00 loops=1) One-Time Filter: false :PREFIX SELECT last(btest_numeric, time) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Result (actual rows=0.00 loops=1) One-Time Filter: false -- Only NULL values INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_2_8_chunk_btest_numeric_time_idx on _hyper_2_8_chunk (actual rows=1.00 loops=1) :PREFIX SELECT last(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Index Scan using _hyper_2_8_chunk_btest_numeric_time_idx on _hyper_2_8_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) :PREFIX SELECT first(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Seq Scan on _hyper_2_8_chunk (actual rows=2.00 loops=1) :PREFIX SELECT last(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Seq Scan on _hyper_2_8_chunk (actual rows=2.00 loops=1) -- NULL values followed by non-NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); :PREFIX SELECT first(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest_numeric (actual rows=1.00 loops=1) Order: btest_numeric."time" -> Index Scan Backward using _hyper_2_8_chunk_btest_numeric_time_idx on _hyper_2_8_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_2_9_chunk_btest_numeric_time_idx on _hyper_2_9_chunk (never executed) :PREFIX SELECT last(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest_numeric (actual rows=1.00 loops=1) Order: btest_numeric."time" DESC -> Index Scan using _hyper_2_9_chunk_btest_numeric_time_idx on _hyper_2_9_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_2_8_chunk_btest_numeric_time_idx on _hyper_2_8_chunk (never executed) Index Cond: ("time" IS NOT NULL) :PREFIX SELECT first(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_2_8_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_2_9_chunk (actual rows=2.00 loops=1) :PREFIX SELECT last(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_2_8_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_2_9_chunk (actual rows=2.00 loops=1) TRUNCATE btest_numeric; -- non-NULL values followed by NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest_numeric (actual rows=1.00 loops=1) Order: btest_numeric."time" -> Index Scan Backward using _hyper_2_11_chunk_btest_numeric_time_idx on _hyper_2_11_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_2_10_chunk_btest_numeric_time_idx on _hyper_2_10_chunk (never executed) :PREFIX SELECT last(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest_numeric (actual rows=1.00 loops=1) Order: btest_numeric."time" DESC -> Index Scan using _hyper_2_10_chunk_btest_numeric_time_idx on _hyper_2_10_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_2_11_chunk_btest_numeric_time_idx on _hyper_2_11_chunk (never executed) Index Cond: ("time" IS NOT NULL) :PREFIX SELECT first(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_2_10_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_2_11_chunk (actual rows=2.00 loops=1) :PREFIX SELECT last(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_2_10_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_2_11_chunk (actual rows=2.00 loops=1) ROLLBACK; -- we want test results as part of the output too to make sure we produce correct output \set PREFIX '' \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be only output of results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations')) v(setting); setting | value ----------------------------------+------- timescaledb.enable_optimizations | on :PREFIX SELECT time, gp, temp FROM btest ORDER BY time; time | gp | temp --------------------------+----+------ Fri Jan 20 09:00:01 2017 | 1 | 22.5 Fri Jan 20 09:00:02 2017 | 2 | 35.5 Fri Jan 20 09:00:21 2017 | 1 | 21.2 Fri Jan 20 09:00:21 2017 | 2 | 30.2 Fri Jan 20 09:00:43 2017 | 2 | 20.1 Fri Jan 20 09:00:47 2017 | 1 | 25.1 :PREFIX SELECT last(temp, time) FROM btest; last ------ 25.1 :PREFIX SELECT first(temp, time) FROM btest; first ------- 22.5 :PREFIX SELECT last(temp, time_alt) FROM btest; last ------ 22.5 :PREFIX SELECT first(temp, time_alt) FROM btest; first ------- 30.2 :PREFIX SELECT gp, last(temp, time) FROM btest GROUP BY gp ORDER BY gp; gp | last ----+------ 1 | 25.1 2 | 20.1 :PREFIX SELECT gp, first(temp, time) FROM btest GROUP BY gp ORDER BY gp; gp | first ----+------- 1 | 22.5 2 | 35.5 --check whole row :PREFIX SELECT gp, first(btest, time) FROM btest GROUP BY gp ORDER BY gp; gp | first ----+------------------------------------------------------------------------ 1 | ("Fri Jan 20 09:00:01 2017","Fri Jan 20 10:00:00 2017",1,22.5,testing) 2 | ("Fri Jan 20 09:00:02 2017","Fri Jan 20 09:00:57 2017",2,35.5,testing) --check toasted col :PREFIX SELECT gp, left(last(strid, time), 10) FROM btest GROUP BY gp ORDER BY gp; gp | left ----+------------ 1 | testing 2 | xyzxyzxyzx :PREFIX SELECT gp, last(temp, strid) FROM btest GROUP BY gp ORDER BY gp; gp | last ----+------ 1 | 22.5 2 | 20.1 :PREFIX SELECT gp, last(strid, temp) FROM btest GROUP BY gp ORDER BY gp; gp | last ----+--------- 1 | testing 2 | testing BEGIN; --check null value as last element INSERT INTO btest VALUES('2018-01-20T09:00:43', '2017-01-20T09:00:55', 2, NULL); :PREFIX SELECT last(temp, time) FROM btest; last ------ --check non-null element "overrides" NULL because it comes after. INSERT INTO btest VALUES('2019-01-20T09:00:43', '2018-01-20T09:00:55', 2, 30.5); :PREFIX SELECT last(temp, time) FROM btest; last ------ 30.5 --check null cmp element is skipped INSERT INTO btest VALUES('2018-01-20T09:00:43', NULL, 2, 32.3); :PREFIX SELECT last(temp, time_alt) FROM btest; last ------ 30.5 -- fist returns NULL value :PREFIX SELECT first(temp, time_alt) FROM btest; first ------- -- test first return non NULL value INSERT INTO btest VALUES('2016-01-20T09:00:00', '2016-01-20T09:00:00', 2, 36.5); :PREFIX SELECT first(temp, time_alt) FROM btest; first ------- 36.5 --check non null cmp element insert after null cmp INSERT INTO btest VALUES('2020-01-20T09:00:43', '2020-01-20T09:00:43', 2, 35.3); :PREFIX SELECT last(temp, time_alt) FROM btest; last ------ 35.3 :PREFIX SELECT first(temp, time_alt) FROM btest; first ------- 36.5 --cmp nulls should be ignored and not present in groups :PREFIX SELECT gp, last(temp, time_alt) FROM btest GROUP BY gp ORDER BY gp; gp | last ----+------ 1 | 22.5 2 | 35.3 --Previously, some bugs were found with NULLS and numeric types, so test that INSERT INTO btest_numeric VALUES ('2019-01-20T09:00:43', NULL); :PREFIX SELECT last(quantity, time) FROM btest_numeric; last ------ --check non-null element "overrides" NULL because it comes after. INSERT INTO btest_numeric VALUES('2020-01-20T09:00:43', 30.5); :PREFIX SELECT last(quantity, time) FROM btest_numeric; last ------ 30.5 -- do index scan for last :PREFIX SELECT last(temp, time) FROM btest; last ------ 35.3 -- do index scan for first :PREFIX SELECT first(temp, time) FROM btest; first ------- 36.5 -- can't do index scan when ordering on non-index column :PREFIX SELECT first(temp, time_alt) FROM btest; first ------- 36.5 -- do index scan for subquery :PREFIX SELECT * FROM (SELECT last(temp, time) FROM btest) last; last ------ 35.3 -- can't do index scan when using group by :PREFIX SELECT last(temp, time) FROM btest GROUP BY gp ORDER BY gp; last ------ 25.1 35.3 -- do index scan when agg function is used in CTE subquery :PREFIX WITH last_temp AS (SELECT last(temp, time) FROM btest) SELECT * from last_temp; last ------ 35.3 -- do index scan when using both FIRST and LAST aggregate functions :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; first | last -------+------ 36.5 | 35.3 -- verify results when using both FIRST and LAST :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; first | last -------+------ 36.5 | 35.3 -- do index scan when using WHERE :PREFIX SELECT last(temp, time) FROM btest WHERE time <= '2017-01-20T09:00:02'; last ------ 35.5 -- can't do index scan for MAX and LAST combined (MinMax optimization fails when having different aggregate functions) :PREFIX SELECT max(time), last(temp, time) FROM btest; max | last --------------------------+------ Mon Jan 20 09:00:43 2020 | 35.3 -- can't do index scan when using FIRST/LAST in ORDER BY :PREFIX SELECT last(temp, time) FROM btest ORDER BY last(temp, time); last ------ 35.3 -- do index scan :PREFIX SELECT last(temp, time) FROM btest WHERE temp < 30; last ------ 25.1 -- SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- do index scan :PREFIX SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; first ------- 25.1 -- can't do index scan when using WINDOW function :PREFIX SELECT gp, last(temp, time) OVER (PARTITION BY gp) AS last FROM btest; gp | last ----+------ 1 | 25.1 1 | 25.1 1 | 25.1 2 | 35.3 2 | 35.3 2 | 35.3 2 | 35.3 2 | 35.3 2 | 35.3 2 | 35.3 2 | 35.3 -- test constants :PREFIX SELECT first(100, 100) FROM btest; first ------- 100 -- create an index so we can test optimization CREATE INDEX btest_time_alt_idx ON btest(time_alt); :PREFIX SELECT last(temp, time_alt) FROM btest; last ------ 35.3 --test nested FIRST/LAST - should optimize :PREFIX SELECT abs(last(temp, time)) FROM btest; abs ------ 35.3 -- test nested FIRST/LAST in ORDER BY - no optimization possible :PREFIX SELECT abs(last(temp, time)) FROM btest ORDER BY abs(last(temp,time)); abs ------ 35.3 ROLLBACK; -- Test with NULL numeric values BEGIN; TRUNCATE btest_numeric; -- Empty table :PREFIX SELECT first(btest_numeric, time) FROM btest_numeric; first ------- :PREFIX SELECT last(btest_numeric, time) FROM btest_numeric; last ------ -- Only NULL values INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; first ------- :PREFIX SELECT last(quantity, time) FROM btest_numeric; last ------ :PREFIX SELECT first(time, quantity) FROM btest_numeric; first ------- :PREFIX SELECT last(time, quantity) FROM btest_numeric; last ------ -- NULL values followed by non-NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); :PREFIX SELECT first(quantity, time) FROM btest_numeric; first ------- :PREFIX SELECT last(quantity, time) FROM btest_numeric; last ------ 1 :PREFIX SELECT first(time, quantity) FROM btest_numeric; first -------------------------- Sun Jan 20 09:00:43 2019 :PREFIX SELECT last(time, quantity) FROM btest_numeric; last -------------------------- Sun Jan 20 09:00:43 2019 TRUNCATE btest_numeric; -- non-NULL values followed by NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; first ------- :PREFIX SELECT last(quantity, time) FROM btest_numeric; last ------ 1 :PREFIX SELECT first(time, quantity) FROM btest_numeric; first -------------------------- Sun Jan 20 09:00:43 2019 :PREFIX SELECT last(time, quantity) FROM btest_numeric; last -------------------------- Sun Jan 20 09:00:43 2019 ROLLBACK; -- diff results with optimizations disabled and enabled \o :TEST_RESULTS_UNOPTIMIZED SET timescaledb.enable_optimizations TO false; \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be only output of results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations')) v(setting); :PREFIX SELECT time, gp, temp FROM btest ORDER BY time; :PREFIX SELECT last(temp, time) FROM btest; :PREFIX SELECT first(temp, time) FROM btest; :PREFIX SELECT last(temp, time_alt) FROM btest; :PREFIX SELECT first(temp, time_alt) FROM btest; :PREFIX SELECT gp, last(temp, time) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, first(temp, time) FROM btest GROUP BY gp ORDER BY gp; --check whole row :PREFIX SELECT gp, first(btest, time) FROM btest GROUP BY gp ORDER BY gp; --check toasted col :PREFIX SELECT gp, left(last(strid, time), 10) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, last(temp, strid) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, last(strid, temp) FROM btest GROUP BY gp ORDER BY gp; BEGIN; --check null value as last element INSERT INTO btest VALUES('2018-01-20T09:00:43', '2017-01-20T09:00:55', 2, NULL); :PREFIX SELECT last(temp, time) FROM btest; --check non-null element "overrides" NULL because it comes after. INSERT INTO btest VALUES('2019-01-20T09:00:43', '2018-01-20T09:00:55', 2, 30.5); :PREFIX SELECT last(temp, time) FROM btest; --check null cmp element is skipped INSERT INTO btest VALUES('2018-01-20T09:00:43', NULL, 2, 32.3); :PREFIX SELECT last(temp, time_alt) FROM btest; -- fist returns NULL value :PREFIX SELECT first(temp, time_alt) FROM btest; -- test first return non NULL value INSERT INTO btest VALUES('2016-01-20T09:00:00', '2016-01-20T09:00:00', 2, 36.5); :PREFIX SELECT first(temp, time_alt) FROM btest; --check non null cmp element insert after null cmp INSERT INTO btest VALUES('2020-01-20T09:00:43', '2020-01-20T09:00:43', 2, 35.3); :PREFIX SELECT last(temp, time_alt) FROM btest; :PREFIX SELECT first(temp, time_alt) FROM btest; --cmp nulls should be ignored and not present in groups :PREFIX SELECT gp, last(temp, time_alt) FROM btest GROUP BY gp ORDER BY gp; --Previously, some bugs were found with NULLS and numeric types, so test that INSERT INTO btest_numeric VALUES ('2019-01-20T09:00:43', NULL); :PREFIX SELECT last(quantity, time) FROM btest_numeric; --check non-null element "overrides" NULL because it comes after. INSERT INTO btest_numeric VALUES('2020-01-20T09:00:43', 30.5); :PREFIX SELECT last(quantity, time) FROM btest_numeric; -- do index scan for last :PREFIX SELECT last(temp, time) FROM btest; -- do index scan for first :PREFIX SELECT first(temp, time) FROM btest; -- can't do index scan when ordering on non-index column :PREFIX SELECT first(temp, time_alt) FROM btest; -- do index scan for subquery :PREFIX SELECT * FROM (SELECT last(temp, time) FROM btest) last; -- can't do index scan when using group by :PREFIX SELECT last(temp, time) FROM btest GROUP BY gp ORDER BY gp; -- do index scan when agg function is used in CTE subquery :PREFIX WITH last_temp AS (SELECT last(temp, time) FROM btest) SELECT * from last_temp; -- do index scan when using both FIRST and LAST aggregate functions :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; -- verify results when using both FIRST and LAST :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; -- do index scan when using WHERE :PREFIX SELECT last(temp, time) FROM btest WHERE time <= '2017-01-20T09:00:02'; -- can't do index scan for MAX and LAST combined (MinMax optimization fails when having different aggregate functions) :PREFIX SELECT max(time), last(temp, time) FROM btest; -- can't do index scan when using FIRST/LAST in ORDER BY :PREFIX SELECT last(temp, time) FROM btest ORDER BY last(temp, time); -- do index scan :PREFIX SELECT last(temp, time) FROM btest WHERE temp < 30; -- SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- do index scan :PREFIX SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- can't do index scan when using WINDOW function :PREFIX SELECT gp, last(temp, time) OVER (PARTITION BY gp) AS last FROM btest; -- test constants :PREFIX SELECT first(100, 100) FROM btest; -- create an index so we can test optimization CREATE INDEX btest_time_alt_idx ON btest(time_alt); :PREFIX SELECT last(temp, time_alt) FROM btest; --test nested FIRST/LAST - should optimize :PREFIX SELECT abs(last(temp, time)) FROM btest; -- test nested FIRST/LAST in ORDER BY - no optimization possible :PREFIX SELECT abs(last(temp, time)) FROM btest ORDER BY abs(last(temp,time)); ROLLBACK; -- Test with NULL numeric values BEGIN; TRUNCATE btest_numeric; -- Empty table :PREFIX SELECT first(btest_numeric, time) FROM btest_numeric; :PREFIX SELECT last(btest_numeric, time) FROM btest_numeric; -- Only NULL values INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; -- NULL values followed by non-NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; TRUNCATE btest_numeric; -- non-NULL values followed by NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; ROLLBACK; \o \o :TEST_RESULTS_OPTIMIZED SET timescaledb.enable_optimizations TO true; \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be only output of results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations')) v(setting); :PREFIX SELECT time, gp, temp FROM btest ORDER BY time; :PREFIX SELECT last(temp, time) FROM btest; :PREFIX SELECT first(temp, time) FROM btest; :PREFIX SELECT last(temp, time_alt) FROM btest; :PREFIX SELECT first(temp, time_alt) FROM btest; :PREFIX SELECT gp, last(temp, time) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, first(temp, time) FROM btest GROUP BY gp ORDER BY gp; --check whole row :PREFIX SELECT gp, first(btest, time) FROM btest GROUP BY gp ORDER BY gp; --check toasted col :PREFIX SELECT gp, left(last(strid, time), 10) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, last(temp, strid) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, last(strid, temp) FROM btest GROUP BY gp ORDER BY gp; BEGIN; --check null value as last element INSERT INTO btest VALUES('2018-01-20T09:00:43', '2017-01-20T09:00:55', 2, NULL); :PREFIX SELECT last(temp, time) FROM btest; --check non-null element "overrides" NULL because it comes after. INSERT INTO btest VALUES('2019-01-20T09:00:43', '2018-01-20T09:00:55', 2, 30.5); :PREFIX SELECT last(temp, time) FROM btest; --check null cmp element is skipped INSERT INTO btest VALUES('2018-01-20T09:00:43', NULL, 2, 32.3); :PREFIX SELECT last(temp, time_alt) FROM btest; -- fist returns NULL value :PREFIX SELECT first(temp, time_alt) FROM btest; -- test first return non NULL value INSERT INTO btest VALUES('2016-01-20T09:00:00', '2016-01-20T09:00:00', 2, 36.5); :PREFIX SELECT first(temp, time_alt) FROM btest; --check non null cmp element insert after null cmp INSERT INTO btest VALUES('2020-01-20T09:00:43', '2020-01-20T09:00:43', 2, 35.3); :PREFIX SELECT last(temp, time_alt) FROM btest; :PREFIX SELECT first(temp, time_alt) FROM btest; --cmp nulls should be ignored and not present in groups :PREFIX SELECT gp, last(temp, time_alt) FROM btest GROUP BY gp ORDER BY gp; --Previously, some bugs were found with NULLS and numeric types, so test that INSERT INTO btest_numeric VALUES ('2019-01-20T09:00:43', NULL); :PREFIX SELECT last(quantity, time) FROM btest_numeric; --check non-null element "overrides" NULL because it comes after. INSERT INTO btest_numeric VALUES('2020-01-20T09:00:43', 30.5); :PREFIX SELECT last(quantity, time) FROM btest_numeric; -- do index scan for last :PREFIX SELECT last(temp, time) FROM btest; -- do index scan for first :PREFIX SELECT first(temp, time) FROM btest; -- can't do index scan when ordering on non-index column :PREFIX SELECT first(temp, time_alt) FROM btest; -- do index scan for subquery :PREFIX SELECT * FROM (SELECT last(temp, time) FROM btest) last; -- can't do index scan when using group by :PREFIX SELECT last(temp, time) FROM btest GROUP BY gp ORDER BY gp; -- do index scan when agg function is used in CTE subquery :PREFIX WITH last_temp AS (SELECT last(temp, time) FROM btest) SELECT * from last_temp; -- do index scan when using both FIRST and LAST aggregate functions :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; -- verify results when using both FIRST and LAST :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; -- do index scan when using WHERE :PREFIX SELECT last(temp, time) FROM btest WHERE time <= '2017-01-20T09:00:02'; -- can't do index scan for MAX and LAST combined (MinMax optimization fails when having different aggregate functions) :PREFIX SELECT max(time), last(temp, time) FROM btest; -- can't do index scan when using FIRST/LAST in ORDER BY :PREFIX SELECT last(temp, time) FROM btest ORDER BY last(temp, time); -- do index scan :PREFIX SELECT last(temp, time) FROM btest WHERE temp < 30; -- SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- do index scan :PREFIX SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- can't do index scan when using WINDOW function :PREFIX SELECT gp, last(temp, time) OVER (PARTITION BY gp) AS last FROM btest; -- test constants :PREFIX SELECT first(100, 100) FROM btest; -- create an index so we can test optimization CREATE INDEX btest_time_alt_idx ON btest(time_alt); :PREFIX SELECT last(temp, time_alt) FROM btest; --test nested FIRST/LAST - should optimize :PREFIX SELECT abs(last(temp, time)) FROM btest; -- test nested FIRST/LAST in ORDER BY - no optimization possible :PREFIX SELECT abs(last(temp, time)) FROM btest ORDER BY abs(last(temp,time)); ROLLBACK; -- Test with NULL numeric values BEGIN; TRUNCATE btest_numeric; -- Empty table :PREFIX SELECT first(btest_numeric, time) FROM btest_numeric; :PREFIX SELECT last(btest_numeric, time) FROM btest_numeric; -- Only NULL values INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; -- NULL values followed by non-NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; TRUNCATE btest_numeric; -- non-NULL values followed by NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; ROLLBACK; \o :DIFF_CMD --- Unoptimized result +++ Optimized result @@ -1,6 +1,6 @@ setting | value ----------------------------------+------- - timescaledb.enable_optimizations | off + timescaledb.enable_optimizations | on time | gp | temp -- Test partial aggregation CREATE TABLE partial_aggregation (time timestamptz NOT NULL, quantity numeric, longvalue text); SELECT schema_name, table_name, created FROM create_hypertable('partial_aggregation', 'time'); schema_name | table_name | created -------------+---------------------+--------- public | partial_aggregation | t INSERT INTO partial_aggregation VALUES('2018-01-20T09:00:43', NULL, NULL); INSERT INTO partial_aggregation VALUES('2018-01-20T09:00:44', NULL, NULL); INSERT INTO partial_aggregation VALUES('2019-01-20T09:00:43', 1, 'hello'); INSERT INTO partial_aggregation VALUES('2019-01-20T09:00:44', 2, 'world'); INSERT INTO partial_aggregation VALUES('2020-01-20T09:00:43', 3.1, 'some1'); INSERT INTO partial_aggregation VALUES('2020-01-20T09:00:44', 3.2, 'more1'); INSERT INTO partial_aggregation VALUES('2021-01-20T09:00:43', 3.3, 'some2'); INSERT INTO partial_aggregation VALUES('2021-01-20T09:00:44', 3.4, 'more2'); INSERT INTO partial_aggregation VALUES('2022-01-20T09:00:43', 4, 'word1'); INSERT INTO partial_aggregation VALUES('2022-01-20T09:00:44', 5, 'word2'); INSERT INTO partial_aggregation VALUES('2023-01-20T09:00:43', 6, 'word3'); INSERT INTO partial_aggregation VALUES('2023-01-20T09:00:44', 7, 'word4'); -- Use enable_partitionwise_aggregate to create partial aggregates per chunk SET enable_partitionwise_aggregate = ON; SELECT format('SELECT %3$s, %1$s FROM partial_aggregation WHERE %2$s GROUP BY %3$s ORDER BY 1, 2;', function, condition, grouping) FROM unnest(array[ 'first(time, quantity), last(time, quantity)', 'last(longvalue, quantity)', 'last(quantity, longvalue)', 'last(quantity, time)', 'last(time, longvalue)']) AS function, unnest(array[ 'true', $$time < '2021-01-01'$$, 'quantity is null', 'quantity is not null', 'quantity >= 4']) AS condition, unnest(array[ '777::text' /* dummy grouping column */, 'longvalue', 'quantity', $$time_bucket('1 year', time)$$, $$time_bucket('3 year', time)$$]) AS grouping \gexec SELECT 777::text, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE true GROUP BY 777::text ORDER BY 1, 2; text | first | last ------+------------------------------+------------------------------ 777 | Sun Jan 20 09:00:43 2019 PST | Fri Jan 20 09:00:44 2023 PST SELECT 777::text, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY 777::text ORDER BY 1, 2; text | first | last ------+------------------------------+------------------------------ 777 | Sun Jan 20 09:00:43 2019 PST | Mon Jan 20 09:00:44 2020 PST SELECT 777::text, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY 777::text ORDER BY 1, 2; text | first | last ------+-------+------ 777 | | SELECT 777::text, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY 777::text ORDER BY 1, 2; text | first | last ------+------------------------------+------------------------------ 777 | Sun Jan 20 09:00:43 2019 PST | Fri Jan 20 09:00:44 2023 PST SELECT 777::text, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY 777::text ORDER BY 1, 2; text | first | last ------+------------------------------+------------------------------ 777 | Thu Jan 20 09:00:43 2022 PST | Fri Jan 20 09:00:44 2023 PST SELECT 777::text, last(longvalue, quantity) FROM partial_aggregation WHERE true GROUP BY 777::text ORDER BY 1, 2; text | last ------+------- 777 | word4 SELECT 777::text, last(longvalue, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY 777::text ORDER BY 1, 2; text | last ------+------- 777 | more1 SELECT 777::text, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | SELECT 777::text, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------- 777 | word4 SELECT 777::text, last(longvalue, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY 777::text ORDER BY 1, 2; text | last ------+------- 777 | word4 SELECT 777::text, last(quantity, longvalue) FROM partial_aggregation WHERE true GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 2 SELECT 777::text, last(quantity, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 2 SELECT 777::text, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | SELECT 777::text, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 2 SELECT 777::text, last(quantity, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 7 SELECT 777::text, last(quantity, time) FROM partial_aggregation WHERE true GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 7 SELECT 777::text, last(quantity, time) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 3.2 SELECT 777::text, last(quantity, time) FROM partial_aggregation WHERE quantity is null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | SELECT 777::text, last(quantity, time) FROM partial_aggregation WHERE quantity is not null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 7 SELECT 777::text, last(quantity, time) FROM partial_aggregation WHERE quantity >= 4 GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 7 SELECT 777::text, last(time, longvalue) FROM partial_aggregation WHERE true GROUP BY 777::text ORDER BY 1, 2; text | last ------+------------------------------ 777 | Sun Jan 20 09:00:44 2019 PST SELECT 777::text, last(time, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY 777::text ORDER BY 1, 2; text | last ------+------------------------------ 777 | Sun Jan 20 09:00:44 2019 PST SELECT 777::text, last(time, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | SELECT 777::text, last(time, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------------------------------ 777 | Sun Jan 20 09:00:44 2019 PST SELECT 777::text, last(time, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY 777::text ORDER BY 1, 2; text | last ------+------------------------------ 777 | Fri Jan 20 09:00:44 2023 PST SELECT longvalue, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE true GROUP BY longvalue ORDER BY 1, 2; longvalue | first | last -----------+------------------------------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST more2 | Wed Jan 20 09:00:44 2021 PST | Wed Jan 20 09:00:44 2021 PST some1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST some2 | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:43 2021 PST word1 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST world | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST | | SELECT longvalue, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY longvalue ORDER BY 1, 2; longvalue | first | last -----------+------------------------------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST some1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST world | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST | | SELECT longvalue, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY longvalue ORDER BY 1, 2; longvalue | first | last -----------+-------+------ | | SELECT longvalue, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY longvalue ORDER BY 1, 2; longvalue | first | last -----------+------------------------------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST more2 | Wed Jan 20 09:00:44 2021 PST | Wed Jan 20 09:00:44 2021 PST some1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST some2 | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:43 2021 PST word1 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST world | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST SELECT longvalue, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY longvalue ORDER BY 1, 2; longvalue | first | last -----------+------------------------------+------------------------------ word1 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT longvalue, last(longvalue, quantity) FROM partial_aggregation WHERE true GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------- hello | hello more1 | more1 more2 | more2 some1 | some1 some2 | some2 word1 | word1 word2 | word2 word3 | word3 word4 | word4 world | world | SELECT longvalue, last(longvalue, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------- hello | hello more1 | more1 some1 | some1 world | world | SELECT longvalue, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ | SELECT longvalue, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------- hello | hello more1 | more1 more2 | more2 some1 | some1 some2 | some2 word1 | word1 word2 | word2 word3 | word3 word4 | word4 world | world SELECT longvalue, last(longvalue, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------- word1 | word1 word2 | word2 word3 | word3 word4 | word4 SELECT longvalue, last(quantity, longvalue) FROM partial_aggregation WHERE true GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 more2 | 3.4 some1 | 3.1 some2 | 3.3 word1 | 4 word2 | 5 word3 | 6 word4 | 7 world | 2 | SELECT longvalue, last(quantity, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 some1 | 3.1 world | 2 | SELECT longvalue, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ | SELECT longvalue, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 more2 | 3.4 some1 | 3.1 some2 | 3.3 word1 | 4 word2 | 5 word3 | 6 word4 | 7 world | 2 SELECT longvalue, last(quantity, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ word1 | 4 word2 | 5 word3 | 6 word4 | 7 SELECT longvalue, last(quantity, time) FROM partial_aggregation WHERE true GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 more2 | 3.4 some1 | 3.1 some2 | 3.3 word1 | 4 word2 | 5 word3 | 6 word4 | 7 world | 2 | SELECT longvalue, last(quantity, time) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 some1 | 3.1 world | 2 | SELECT longvalue, last(quantity, time) FROM partial_aggregation WHERE quantity is null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ | SELECT longvalue, last(quantity, time) FROM partial_aggregation WHERE quantity is not null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 more2 | 3.4 some1 | 3.1 some2 | 3.3 word1 | 4 word2 | 5 word3 | 6 word4 | 7 world | 2 SELECT longvalue, last(quantity, time) FROM partial_aggregation WHERE quantity >= 4 GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ word1 | 4 word2 | 5 word3 | 6 word4 | 7 SELECT longvalue, last(time, longvalue) FROM partial_aggregation WHERE true GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST more2 | Wed Jan 20 09:00:44 2021 PST some1 | Mon Jan 20 09:00:43 2020 PST some2 | Wed Jan 20 09:00:43 2021 PST word1 | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST world | Sun Jan 20 09:00:44 2019 PST | SELECT longvalue, last(time, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST some1 | Mon Jan 20 09:00:43 2020 PST world | Sun Jan 20 09:00:44 2019 PST | SELECT longvalue, last(time, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ | SELECT longvalue, last(time, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST more2 | Wed Jan 20 09:00:44 2021 PST some1 | Mon Jan 20 09:00:43 2020 PST some2 | Wed Jan 20 09:00:43 2021 PST word1 | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST world | Sun Jan 20 09:00:44 2019 PST SELECT longvalue, last(time, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------------------------------ word1 | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST SELECT quantity, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE true GROUP BY quantity ORDER BY 1, 2; quantity | first | last ----------+------------------------------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST 3.3 | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:43 2021 PST 3.4 | Wed Jan 20 09:00:44 2021 PST | Wed Jan 20 09:00:44 2021 PST 4 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST | | SELECT quantity, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY quantity ORDER BY 1, 2; quantity | first | last ----------+------------------------------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST | | SELECT quantity, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY quantity ORDER BY 1, 2; quantity | first | last ----------+-------+------ | | SELECT quantity, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY quantity ORDER BY 1, 2; quantity | first | last ----------+------------------------------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST 3.3 | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:43 2021 PST 3.4 | Wed Jan 20 09:00:44 2021 PST | Wed Jan 20 09:00:44 2021 PST 4 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT quantity, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY quantity ORDER BY 1, 2; quantity | first | last ----------+------------------------------+------------------------------ 4 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT quantity, last(longvalue, quantity) FROM partial_aggregation WHERE true GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------- 1 | hello 2 | world 3.1 | some1 3.2 | more1 3.3 | some2 3.4 | more2 4 | word1 5 | word2 6 | word3 7 | word4 | SELECT quantity, last(longvalue, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------- 1 | hello 2 | world 3.1 | some1 3.2 | more1 | SELECT quantity, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ | SELECT quantity, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------- 1 | hello 2 | world 3.1 | some1 3.2 | more1 3.3 | some2 3.4 | more2 4 | word1 5 | word2 6 | word3 7 | word4 SELECT quantity, last(longvalue, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------- 4 | word1 5 | word2 6 | word3 7 | word4 SELECT quantity, last(quantity, longvalue) FROM partial_aggregation WHERE true GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 3.3 | 3.3 3.4 | 3.4 4 | 4 5 | 5 6 | 6 7 | 7 | SELECT quantity, last(quantity, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 | SELECT quantity, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ | SELECT quantity, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 3.3 | 3.3 3.4 | 3.4 4 | 4 5 | 5 6 | 6 7 | 7 SELECT quantity, last(quantity, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 4 | 4 5 | 5 6 | 6 7 | 7 SELECT quantity, last(quantity, time) FROM partial_aggregation WHERE true GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 3.3 | 3.3 3.4 | 3.4 4 | 4 5 | 5 6 | 6 7 | 7 | SELECT quantity, last(quantity, time) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 | SELECT quantity, last(quantity, time) FROM partial_aggregation WHERE quantity is null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ | SELECT quantity, last(quantity, time) FROM partial_aggregation WHERE quantity is not null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 3.3 | 3.3 3.4 | 3.4 4 | 4 5 | 5 6 | 6 7 | 7 SELECT quantity, last(quantity, time) FROM partial_aggregation WHERE quantity >= 4 GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 4 | 4 5 | 5 6 | 6 7 | 7 SELECT quantity, last(time, longvalue) FROM partial_aggregation WHERE true GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST 3.3 | Wed Jan 20 09:00:43 2021 PST 3.4 | Wed Jan 20 09:00:44 2021 PST 4 | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST | SELECT quantity, last(time, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST | SELECT quantity, last(time, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ | SELECT quantity, last(time, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST 3.3 | Wed Jan 20 09:00:43 2021 PST 3.4 | Wed Jan 20 09:00:44 2021 PST 4 | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST SELECT quantity, last(time, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------------------------------ 4 | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE true GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | | Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:44 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:44 2021 PST Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | | Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:44 2020 PST SELECT time_bucket('1 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+-------+------ Sun Dec 31 16:00:00 2017 PST | | SELECT time_bucket('1 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:44 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:44 2021 PST Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE true GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | world Tue Dec 31 16:00:00 2019 PST | more1 Thu Dec 31 16:00:00 2020 PST | more2 Fri Dec 31 16:00:00 2021 PST | word2 Sat Dec 31 16:00:00 2022 PST | word4 SELECT time_bucket('1 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | world Tue Dec 31 16:00:00 2019 PST | more1 SELECT time_bucket('1 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('1 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Mon Dec 31 16:00:00 2018 PST | world Tue Dec 31 16:00:00 2019 PST | more1 Thu Dec 31 16:00:00 2020 PST | more2 Fri Dec 31 16:00:00 2021 PST | word2 Sat Dec 31 16:00:00 2022 PST | word4 SELECT time_bucket('1 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Fri Dec 31 16:00:00 2021 PST | word2 Sat Dec 31 16:00:00 2022 PST | word4 SELECT time_bucket('1 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE true GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.1 Thu Dec 31 16:00:00 2020 PST | 3.3 Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.1 SELECT time_bucket('1 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('1 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.1 Thu Dec 31 16:00:00 2020 PST | 3.3 Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(quantity, time) FROM partial_aggregation WHERE true GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.2 Thu Dec 31 16:00:00 2020 PST | 3.4 Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(quantity, time) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.2 SELECT time_bucket('1 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('1 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.2 Thu Dec 31 16:00:00 2020 PST | 3.4 Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(time, longvalue) FROM partial_aggregation WHERE true GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), last(time, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST SELECT time_bucket('1 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('1 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE true GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:43 2019 PST | Mon Jan 20 09:00:44 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:43 2019 PST | Mon Jan 20 09:00:44 2020 PST SELECT time_bucket('3 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+-------+------ Sun Dec 31 16:00:00 2017 PST | | SELECT time_bucket('3 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:43 2019 PST | Mon Jan 20 09:00:44 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Thu Dec 31 16:00:00 2020 PST | Thu Jan 20 09:00:43 2022 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE true GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Sun Dec 31 16:00:00 2017 PST | more1 Thu Dec 31 16:00:00 2020 PST | word4 SELECT time_bucket('3 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Sun Dec 31 16:00:00 2017 PST | more1 SELECT time_bucket('3 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('3 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Sun Dec 31 16:00:00 2017 PST | more1 Thu Dec 31 16:00:00 2020 PST | word4 SELECT time_bucket('3 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Thu Dec 31 16:00:00 2020 PST | word4 SELECT time_bucket('3 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE true GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 2 Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 2 SELECT time_bucket('3 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('3 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 2 Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(quantity, time) FROM partial_aggregation WHERE true GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 3.2 Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(quantity, time) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 3.2 SELECT time_bucket('3 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('3 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 3.2 Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(time, longvalue) FROM partial_aggregation WHERE true GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:44 2019 PST Thu Dec 31 16:00:00 2020 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), last(time, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:44 2019 PST SELECT time_bucket('3 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('3 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:44 2019 PST Thu Dec 31 16:00:00 2020 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Thu Dec 31 16:00:00 2020 PST | Fri Jan 20 09:00:44 2023 PST SET enable_partitionwise_aggregate = OFF; ================================================ FILE: test/expected/agg_bookends-16.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set TEST_BASE_NAME agg_bookends SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') as "TEST_LOAD_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') as "TEST_QUERY_NAME", format('%s/results/%s_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_OPTIMIZED", format('%s/results/%s_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_UNOPTIMIZED" \gset SELECT format('\! diff -u --label "Unoptimized result" --label "Optimized result" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') as "DIFF_CMD" \gset \set PREFIX 'EXPLAIN (analyze, buffers off, costs off, timing off, summary off)' \ir :TEST_LOAD_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE btest(time timestamp NOT NULL, time_alt timestamp, gp INTEGER, temp float, strid TEXT DEFAULT 'testing'); SELECT schema_name, table_name, created FROM create_hypertable('btest', 'time'); psql:include/agg_bookends_load.sql:6: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices psql:include/agg_bookends_load.sql:6: WARNING: column type "timestamp without time zone" used for "time_alt" does not follow best practices schema_name | table_name | created -------------+------------+--------- public | btest | t INSERT INTO btest VALUES('2017-01-20T09:00:01', '2017-01-20T10:00:00', 1, 22.5); INSERT INTO btest VALUES('2017-01-20T09:00:21', '2017-01-20T09:00:59', 1, 21.2); INSERT INTO btest VALUES('2017-01-20T09:00:47', '2017-01-20T09:00:58', 1, 25.1); INSERT INTO btest VALUES('2017-01-20T09:00:02', '2017-01-20T09:00:57', 2, 35.5); INSERT INTO btest VALUES('2017-01-20T09:00:21', '2017-01-20T09:00:56', 2, 30.2); --TOASTED; INSERT INTO btest VALUES('2017-01-20T09:00:43', '2017-01-20T09:01:55', 2, 20.1, repeat('xyz', 1000000) ); CREATE TABLE btest_numeric (time timestamp NOT NULL, quantity numeric); SELECT schema_name, table_name, created FROM create_hypertable('btest_numeric', 'time'); psql:include/agg_bookends_load.sql:16: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices schema_name | table_name | created -------------+---------------+--------- public | btest_numeric | t \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be only output of results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations')) v(setting); setting | value ----------------------------------+------- timescaledb.enable_optimizations | on :PREFIX SELECT time, gp, temp FROM btest ORDER BY time; --- QUERY PLAN --- Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) :PREFIX SELECT first(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) :PREFIX SELECT last(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT first(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT gp, last(temp, time) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT gp, first(temp, time) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) --check whole row :PREFIX SELECT gp, first(btest, time) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) --check toasted col :PREFIX SELECT gp, left(last(strid, time), 10) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT gp, last(temp, strid) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT gp, last(strid, temp) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) BEGIN; --check null value as last element INSERT INTO btest VALUES('2018-01-20T09:00:43', '2017-01-20T09:00:55', 2, NULL); :PREFIX SELECT last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) Index Cond: ("time" IS NOT NULL) --check non-null element "overrides" NULL because it comes after. INSERT INTO btest VALUES('2019-01-20T09:00:43', '2018-01-20T09:00:55', 2, 30.5); :PREFIX SELECT last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) Index Cond: ("time" IS NOT NULL) --check null cmp element is skipped INSERT INTO btest VALUES('2018-01-20T09:00:43', NULL, 2, 32.3); :PREFIX SELECT last(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=9.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -- fist returns NULL value :PREFIX SELECT first(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=9.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -- test first return non NULL value INSERT INTO btest VALUES('2016-01-20T09:00:00', '2016-01-20T09:00:00', 2, 36.5); :PREFIX SELECT first(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=10.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) --check non null cmp element insert after null cmp INSERT INTO btest VALUES('2020-01-20T09:00:43', '2020-01-20T09:00:43', 2, 35.3); :PREFIX SELECT last(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) :PREFIX SELECT first(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) --cmp nulls should be ignored and not present in groups :PREFIX SELECT gp, last(temp, time_alt) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: btest.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: btest.gp -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) --Previously, some bugs were found with NULLS and numeric types, so test that INSERT INTO btest_numeric VALUES ('2019-01-20T09:00:43', NULL); :PREFIX SELECT last(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Index Scan using _hyper_2_6_chunk_btest_numeric_time_idx on _hyper_2_6_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) --check non-null element "overrides" NULL because it comes after. INSERT INTO btest_numeric VALUES('2020-01-20T09:00:43', 30.5); :PREFIX SELECT last(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest_numeric (actual rows=1.00 loops=1) Order: btest_numeric."time" DESC -> Index Scan using _hyper_2_7_chunk_btest_numeric_time_idx on _hyper_2_7_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_2_6_chunk_btest_numeric_time_idx on _hyper_2_6_chunk (never executed) Index Cond: ("time" IS NOT NULL) -- do index scan for last :PREFIX SELECT last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) Index Cond: ("time" IS NOT NULL) -- do index scan for first :PREFIX SELECT first(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" -> Index Scan Backward using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) -> Index Scan Backward using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan Backward using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) -> Index Scan Backward using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (never executed) -- can't do index scan when ordering on non-index column :PREFIX SELECT first(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) -- do index scan for subquery :PREFIX SELECT * FROM (SELECT last(temp, time) FROM btest) last; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) Index Cond: ("time" IS NOT NULL) -- can't do index scan when using group by :PREFIX SELECT last(temp, time) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: btest.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: btest.gp -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) -- do index scan when agg function is used in CTE subquery :PREFIX WITH last_temp AS (SELECT last(temp, time) FROM btest) SELECT * from last_temp; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) Index Cond: ("time" IS NOT NULL) -- do index scan when using both FIRST and LAST aggregate functions :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $1) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) Index Cond: ("time" IS NOT NULL) InitPlan 2 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest btest_1 (actual rows=1.00 loops=1) Order: btest_1."time" -> Index Scan Backward using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk _hyper_1_4_chunk_1 (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk _hyper_1_3_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk _hyper_1_5_chunk_1 (never executed) -- verify results when using both FIRST and LAST :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $1) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) Index Cond: ("time" IS NOT NULL) InitPlan 2 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest btest_1 (actual rows=1.00 loops=1) Order: btest_1."time" -> Index Scan Backward using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk _hyper_1_4_chunk_1 (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk _hyper_1_3_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk _hyper_1_5_chunk_1 (never executed) -- do index scan when using WHERE :PREFIX SELECT last(temp, time) FROM btest WHERE time <= '2017-01-20T09:00:02'; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) Index Cond: (("time" IS NOT NULL) AND ("time" <= 'Fri Jan 20 09:00:02 2017'::timestamp without time zone)) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) Index Cond: ("time" IS NOT NULL) -- can't do index scan for MAX and LAST combined (MinMax optimization fails when having different aggregate functions) :PREFIX SELECT max(time), last(temp, time) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) -- can't do index scan when using FIRST/LAST in ORDER BY :PREFIX SELECT last(temp, time) FROM btest ORDER BY last(temp, time); --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: (last(btest.temp, btest."time")) Sort Method: quicksort -> Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) -- do index scan :PREFIX SELECT last(temp, time) FROM btest WHERE temp < 30; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=0.00 loops=1) Index Cond: ("time" IS NOT NULL) Filter: (temp < '30'::double precision) Rows Removed by Filter: 1 -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (actual rows=0.00 loops=1) Index Cond: ("time" IS NOT NULL) Filter: (temp < '30'::double precision) Rows Removed by Filter: 1 -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (actual rows=0.00 loops=1) Index Cond: ("time" IS NOT NULL) Filter: (temp < '30'::double precision) Rows Removed by Filter: 2 -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) Filter: (temp < '30'::double precision) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) Index Cond: ("time" IS NOT NULL) Filter: (temp < '30'::double precision) -- SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- do index scan :PREFIX SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" -> Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Fri Jan 20 09:00:47 2017'::timestamp without time zone) -> Index Scan Backward using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan Backward using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) -> Index Scan Backward using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (never executed) -- can't do index scan when using WINDOW function :PREFIX SELECT gp, last(temp, time) OVER (PARTITION BY gp) AS last FROM btest; --- QUERY PLAN --- WindowAgg (actual rows=11.00 loops=1) -> Sort (actual rows=11.00 loops=1) Sort Key: btest.gp Sort Method: quicksort -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) -- test constants :PREFIX SELECT first(100, 100) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Result (actual rows=1.00 loops=1) -> Append (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (never executed) -> Seq Scan on _hyper_1_3_chunk (never executed) -> Seq Scan on _hyper_1_4_chunk (never executed) -> Seq Scan on _hyper_1_5_chunk (never executed) -- create an index so we can test optimization CREATE INDEX btest_time_alt_idx ON btest(time_alt); :PREFIX SELECT last(temp, time_alt) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Merge Append (actual rows=1.00 loops=1) Sort Key: btest.time_alt DESC -> Index Scan Backward using _hyper_1_1_chunk_btest_time_alt_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) Index Cond: (time_alt IS NOT NULL) -> Index Scan Backward using _hyper_1_2_chunk_btest_time_alt_idx on _hyper_1_2_chunk (actual rows=1.00 loops=1) Index Cond: (time_alt IS NOT NULL) -> Index Scan Backward using _hyper_1_3_chunk_btest_time_alt_idx on _hyper_1_3_chunk (actual rows=1.00 loops=1) Index Cond: (time_alt IS NOT NULL) -> Index Scan Backward using _hyper_1_4_chunk_btest_time_alt_idx on _hyper_1_4_chunk (actual rows=1.00 loops=1) Index Cond: (time_alt IS NOT NULL) -> Index Scan Backward using _hyper_1_5_chunk_btest_time_alt_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) Index Cond: (time_alt IS NOT NULL) --test nested FIRST/LAST - should optimize :PREFIX SELECT abs(last(temp, time)) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) Index Cond: ("time" IS NOT NULL) -- test nested FIRST/LAST in ORDER BY - no optimization possible :PREFIX SELECT abs(last(temp, time)) FROM btest ORDER BY abs(last(temp,time)); --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: (abs(last(btest.temp, btest."time"))) Sort Method: quicksort -> Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) ROLLBACK; -- Test with NULL numeric values BEGIN; TRUNCATE btest_numeric; -- Empty table :PREFIX SELECT first(btest_numeric, time) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Result (actual rows=0.00 loops=1) One-Time Filter: false :PREFIX SELECT last(btest_numeric, time) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Result (actual rows=0.00 loops=1) One-Time Filter: false -- Only NULL values INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_2_8_chunk_btest_numeric_time_idx on _hyper_2_8_chunk (actual rows=1.00 loops=1) :PREFIX SELECT last(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Index Scan using _hyper_2_8_chunk_btest_numeric_time_idx on _hyper_2_8_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) :PREFIX SELECT first(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Seq Scan on _hyper_2_8_chunk (actual rows=2.00 loops=1) :PREFIX SELECT last(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Seq Scan on _hyper_2_8_chunk (actual rows=2.00 loops=1) -- NULL values followed by non-NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); :PREFIX SELECT first(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest_numeric (actual rows=1.00 loops=1) Order: btest_numeric."time" -> Index Scan Backward using _hyper_2_8_chunk_btest_numeric_time_idx on _hyper_2_8_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_2_9_chunk_btest_numeric_time_idx on _hyper_2_9_chunk (never executed) :PREFIX SELECT last(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest_numeric (actual rows=1.00 loops=1) Order: btest_numeric."time" DESC -> Index Scan using _hyper_2_9_chunk_btest_numeric_time_idx on _hyper_2_9_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_2_8_chunk_btest_numeric_time_idx on _hyper_2_8_chunk (never executed) Index Cond: ("time" IS NOT NULL) :PREFIX SELECT first(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_2_8_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_2_9_chunk (actual rows=2.00 loops=1) :PREFIX SELECT last(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_2_8_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_2_9_chunk (actual rows=2.00 loops=1) TRUNCATE btest_numeric; -- non-NULL values followed by NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest_numeric (actual rows=1.00 loops=1) Order: btest_numeric."time" -> Index Scan Backward using _hyper_2_11_chunk_btest_numeric_time_idx on _hyper_2_11_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_2_10_chunk_btest_numeric_time_idx on _hyper_2_10_chunk (never executed) :PREFIX SELECT last(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest_numeric (actual rows=1.00 loops=1) Order: btest_numeric."time" DESC -> Index Scan using _hyper_2_10_chunk_btest_numeric_time_idx on _hyper_2_10_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_2_11_chunk_btest_numeric_time_idx on _hyper_2_11_chunk (never executed) Index Cond: ("time" IS NOT NULL) :PREFIX SELECT first(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_2_10_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_2_11_chunk (actual rows=2.00 loops=1) :PREFIX SELECT last(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_2_10_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_2_11_chunk (actual rows=2.00 loops=1) ROLLBACK; -- we want test results as part of the output too to make sure we produce correct output \set PREFIX '' \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be only output of results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations')) v(setting); setting | value ----------------------------------+------- timescaledb.enable_optimizations | on :PREFIX SELECT time, gp, temp FROM btest ORDER BY time; time | gp | temp --------------------------+----+------ Fri Jan 20 09:00:01 2017 | 1 | 22.5 Fri Jan 20 09:00:02 2017 | 2 | 35.5 Fri Jan 20 09:00:21 2017 | 1 | 21.2 Fri Jan 20 09:00:21 2017 | 2 | 30.2 Fri Jan 20 09:00:43 2017 | 2 | 20.1 Fri Jan 20 09:00:47 2017 | 1 | 25.1 :PREFIX SELECT last(temp, time) FROM btest; last ------ 25.1 :PREFIX SELECT first(temp, time) FROM btest; first ------- 22.5 :PREFIX SELECT last(temp, time_alt) FROM btest; last ------ 22.5 :PREFIX SELECT first(temp, time_alt) FROM btest; first ------- 30.2 :PREFIX SELECT gp, last(temp, time) FROM btest GROUP BY gp ORDER BY gp; gp | last ----+------ 1 | 25.1 2 | 20.1 :PREFIX SELECT gp, first(temp, time) FROM btest GROUP BY gp ORDER BY gp; gp | first ----+------- 1 | 22.5 2 | 35.5 --check whole row :PREFIX SELECT gp, first(btest, time) FROM btest GROUP BY gp ORDER BY gp; gp | first ----+------------------------------------------------------------------------ 1 | ("Fri Jan 20 09:00:01 2017","Fri Jan 20 10:00:00 2017",1,22.5,testing) 2 | ("Fri Jan 20 09:00:02 2017","Fri Jan 20 09:00:57 2017",2,35.5,testing) --check toasted col :PREFIX SELECT gp, left(last(strid, time), 10) FROM btest GROUP BY gp ORDER BY gp; gp | left ----+------------ 1 | testing 2 | xyzxyzxyzx :PREFIX SELECT gp, last(temp, strid) FROM btest GROUP BY gp ORDER BY gp; gp | last ----+------ 1 | 22.5 2 | 20.1 :PREFIX SELECT gp, last(strid, temp) FROM btest GROUP BY gp ORDER BY gp; gp | last ----+--------- 1 | testing 2 | testing BEGIN; --check null value as last element INSERT INTO btest VALUES('2018-01-20T09:00:43', '2017-01-20T09:00:55', 2, NULL); :PREFIX SELECT last(temp, time) FROM btest; last ------ --check non-null element "overrides" NULL because it comes after. INSERT INTO btest VALUES('2019-01-20T09:00:43', '2018-01-20T09:00:55', 2, 30.5); :PREFIX SELECT last(temp, time) FROM btest; last ------ 30.5 --check null cmp element is skipped INSERT INTO btest VALUES('2018-01-20T09:00:43', NULL, 2, 32.3); :PREFIX SELECT last(temp, time_alt) FROM btest; last ------ 30.5 -- fist returns NULL value :PREFIX SELECT first(temp, time_alt) FROM btest; first ------- -- test first return non NULL value INSERT INTO btest VALUES('2016-01-20T09:00:00', '2016-01-20T09:00:00', 2, 36.5); :PREFIX SELECT first(temp, time_alt) FROM btest; first ------- 36.5 --check non null cmp element insert after null cmp INSERT INTO btest VALUES('2020-01-20T09:00:43', '2020-01-20T09:00:43', 2, 35.3); :PREFIX SELECT last(temp, time_alt) FROM btest; last ------ 35.3 :PREFIX SELECT first(temp, time_alt) FROM btest; first ------- 36.5 --cmp nulls should be ignored and not present in groups :PREFIX SELECT gp, last(temp, time_alt) FROM btest GROUP BY gp ORDER BY gp; gp | last ----+------ 1 | 22.5 2 | 35.3 --Previously, some bugs were found with NULLS and numeric types, so test that INSERT INTO btest_numeric VALUES ('2019-01-20T09:00:43', NULL); :PREFIX SELECT last(quantity, time) FROM btest_numeric; last ------ --check non-null element "overrides" NULL because it comes after. INSERT INTO btest_numeric VALUES('2020-01-20T09:00:43', 30.5); :PREFIX SELECT last(quantity, time) FROM btest_numeric; last ------ 30.5 -- do index scan for last :PREFIX SELECT last(temp, time) FROM btest; last ------ 35.3 -- do index scan for first :PREFIX SELECT first(temp, time) FROM btest; first ------- 36.5 -- can't do index scan when ordering on non-index column :PREFIX SELECT first(temp, time_alt) FROM btest; first ------- 36.5 -- do index scan for subquery :PREFIX SELECT * FROM (SELECT last(temp, time) FROM btest) last; last ------ 35.3 -- can't do index scan when using group by :PREFIX SELECT last(temp, time) FROM btest GROUP BY gp ORDER BY gp; last ------ 25.1 35.3 -- do index scan when agg function is used in CTE subquery :PREFIX WITH last_temp AS (SELECT last(temp, time) FROM btest) SELECT * from last_temp; last ------ 35.3 -- do index scan when using both FIRST and LAST aggregate functions :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; first | last -------+------ 36.5 | 35.3 -- verify results when using both FIRST and LAST :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; first | last -------+------ 36.5 | 35.3 -- do index scan when using WHERE :PREFIX SELECT last(temp, time) FROM btest WHERE time <= '2017-01-20T09:00:02'; last ------ 35.5 -- can't do index scan for MAX and LAST combined (MinMax optimization fails when having different aggregate functions) :PREFIX SELECT max(time), last(temp, time) FROM btest; max | last --------------------------+------ Mon Jan 20 09:00:43 2020 | 35.3 -- can't do index scan when using FIRST/LAST in ORDER BY :PREFIX SELECT last(temp, time) FROM btest ORDER BY last(temp, time); last ------ 35.3 -- do index scan :PREFIX SELECT last(temp, time) FROM btest WHERE temp < 30; last ------ 25.1 -- SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- do index scan :PREFIX SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; first ------- 25.1 -- can't do index scan when using WINDOW function :PREFIX SELECT gp, last(temp, time) OVER (PARTITION BY gp) AS last FROM btest; gp | last ----+------ 1 | 25.1 1 | 25.1 1 | 25.1 2 | 35.3 2 | 35.3 2 | 35.3 2 | 35.3 2 | 35.3 2 | 35.3 2 | 35.3 2 | 35.3 -- test constants :PREFIX SELECT first(100, 100) FROM btest; first ------- 100 -- create an index so we can test optimization CREATE INDEX btest_time_alt_idx ON btest(time_alt); :PREFIX SELECT last(temp, time_alt) FROM btest; last ------ 35.3 --test nested FIRST/LAST - should optimize :PREFIX SELECT abs(last(temp, time)) FROM btest; abs ------ 35.3 -- test nested FIRST/LAST in ORDER BY - no optimization possible :PREFIX SELECT abs(last(temp, time)) FROM btest ORDER BY abs(last(temp,time)); abs ------ 35.3 ROLLBACK; -- Test with NULL numeric values BEGIN; TRUNCATE btest_numeric; -- Empty table :PREFIX SELECT first(btest_numeric, time) FROM btest_numeric; first ------- :PREFIX SELECT last(btest_numeric, time) FROM btest_numeric; last ------ -- Only NULL values INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; first ------- :PREFIX SELECT last(quantity, time) FROM btest_numeric; last ------ :PREFIX SELECT first(time, quantity) FROM btest_numeric; first ------- :PREFIX SELECT last(time, quantity) FROM btest_numeric; last ------ -- NULL values followed by non-NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); :PREFIX SELECT first(quantity, time) FROM btest_numeric; first ------- :PREFIX SELECT last(quantity, time) FROM btest_numeric; last ------ 1 :PREFIX SELECT first(time, quantity) FROM btest_numeric; first -------------------------- Sun Jan 20 09:00:43 2019 :PREFIX SELECT last(time, quantity) FROM btest_numeric; last -------------------------- Sun Jan 20 09:00:43 2019 TRUNCATE btest_numeric; -- non-NULL values followed by NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; first ------- :PREFIX SELECT last(quantity, time) FROM btest_numeric; last ------ 1 :PREFIX SELECT first(time, quantity) FROM btest_numeric; first -------------------------- Sun Jan 20 09:00:43 2019 :PREFIX SELECT last(time, quantity) FROM btest_numeric; last -------------------------- Sun Jan 20 09:00:43 2019 ROLLBACK; -- diff results with optimizations disabled and enabled \o :TEST_RESULTS_UNOPTIMIZED SET timescaledb.enable_optimizations TO false; \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be only output of results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations')) v(setting); :PREFIX SELECT time, gp, temp FROM btest ORDER BY time; :PREFIX SELECT last(temp, time) FROM btest; :PREFIX SELECT first(temp, time) FROM btest; :PREFIX SELECT last(temp, time_alt) FROM btest; :PREFIX SELECT first(temp, time_alt) FROM btest; :PREFIX SELECT gp, last(temp, time) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, first(temp, time) FROM btest GROUP BY gp ORDER BY gp; --check whole row :PREFIX SELECT gp, first(btest, time) FROM btest GROUP BY gp ORDER BY gp; --check toasted col :PREFIX SELECT gp, left(last(strid, time), 10) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, last(temp, strid) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, last(strid, temp) FROM btest GROUP BY gp ORDER BY gp; BEGIN; --check null value as last element INSERT INTO btest VALUES('2018-01-20T09:00:43', '2017-01-20T09:00:55', 2, NULL); :PREFIX SELECT last(temp, time) FROM btest; --check non-null element "overrides" NULL because it comes after. INSERT INTO btest VALUES('2019-01-20T09:00:43', '2018-01-20T09:00:55', 2, 30.5); :PREFIX SELECT last(temp, time) FROM btest; --check null cmp element is skipped INSERT INTO btest VALUES('2018-01-20T09:00:43', NULL, 2, 32.3); :PREFIX SELECT last(temp, time_alt) FROM btest; -- fist returns NULL value :PREFIX SELECT first(temp, time_alt) FROM btest; -- test first return non NULL value INSERT INTO btest VALUES('2016-01-20T09:00:00', '2016-01-20T09:00:00', 2, 36.5); :PREFIX SELECT first(temp, time_alt) FROM btest; --check non null cmp element insert after null cmp INSERT INTO btest VALUES('2020-01-20T09:00:43', '2020-01-20T09:00:43', 2, 35.3); :PREFIX SELECT last(temp, time_alt) FROM btest; :PREFIX SELECT first(temp, time_alt) FROM btest; --cmp nulls should be ignored and not present in groups :PREFIX SELECT gp, last(temp, time_alt) FROM btest GROUP BY gp ORDER BY gp; --Previously, some bugs were found with NULLS and numeric types, so test that INSERT INTO btest_numeric VALUES ('2019-01-20T09:00:43', NULL); :PREFIX SELECT last(quantity, time) FROM btest_numeric; --check non-null element "overrides" NULL because it comes after. INSERT INTO btest_numeric VALUES('2020-01-20T09:00:43', 30.5); :PREFIX SELECT last(quantity, time) FROM btest_numeric; -- do index scan for last :PREFIX SELECT last(temp, time) FROM btest; -- do index scan for first :PREFIX SELECT first(temp, time) FROM btest; -- can't do index scan when ordering on non-index column :PREFIX SELECT first(temp, time_alt) FROM btest; -- do index scan for subquery :PREFIX SELECT * FROM (SELECT last(temp, time) FROM btest) last; -- can't do index scan when using group by :PREFIX SELECT last(temp, time) FROM btest GROUP BY gp ORDER BY gp; -- do index scan when agg function is used in CTE subquery :PREFIX WITH last_temp AS (SELECT last(temp, time) FROM btest) SELECT * from last_temp; -- do index scan when using both FIRST and LAST aggregate functions :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; -- verify results when using both FIRST and LAST :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; -- do index scan when using WHERE :PREFIX SELECT last(temp, time) FROM btest WHERE time <= '2017-01-20T09:00:02'; -- can't do index scan for MAX and LAST combined (MinMax optimization fails when having different aggregate functions) :PREFIX SELECT max(time), last(temp, time) FROM btest; -- can't do index scan when using FIRST/LAST in ORDER BY :PREFIX SELECT last(temp, time) FROM btest ORDER BY last(temp, time); -- do index scan :PREFIX SELECT last(temp, time) FROM btest WHERE temp < 30; -- SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- do index scan :PREFIX SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- can't do index scan when using WINDOW function :PREFIX SELECT gp, last(temp, time) OVER (PARTITION BY gp) AS last FROM btest; -- test constants :PREFIX SELECT first(100, 100) FROM btest; -- create an index so we can test optimization CREATE INDEX btest_time_alt_idx ON btest(time_alt); :PREFIX SELECT last(temp, time_alt) FROM btest; --test nested FIRST/LAST - should optimize :PREFIX SELECT abs(last(temp, time)) FROM btest; -- test nested FIRST/LAST in ORDER BY - no optimization possible :PREFIX SELECT abs(last(temp, time)) FROM btest ORDER BY abs(last(temp,time)); ROLLBACK; -- Test with NULL numeric values BEGIN; TRUNCATE btest_numeric; -- Empty table :PREFIX SELECT first(btest_numeric, time) FROM btest_numeric; :PREFIX SELECT last(btest_numeric, time) FROM btest_numeric; -- Only NULL values INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; -- NULL values followed by non-NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; TRUNCATE btest_numeric; -- non-NULL values followed by NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; ROLLBACK; \o \o :TEST_RESULTS_OPTIMIZED SET timescaledb.enable_optimizations TO true; \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be only output of results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations')) v(setting); :PREFIX SELECT time, gp, temp FROM btest ORDER BY time; :PREFIX SELECT last(temp, time) FROM btest; :PREFIX SELECT first(temp, time) FROM btest; :PREFIX SELECT last(temp, time_alt) FROM btest; :PREFIX SELECT first(temp, time_alt) FROM btest; :PREFIX SELECT gp, last(temp, time) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, first(temp, time) FROM btest GROUP BY gp ORDER BY gp; --check whole row :PREFIX SELECT gp, first(btest, time) FROM btest GROUP BY gp ORDER BY gp; --check toasted col :PREFIX SELECT gp, left(last(strid, time), 10) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, last(temp, strid) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, last(strid, temp) FROM btest GROUP BY gp ORDER BY gp; BEGIN; --check null value as last element INSERT INTO btest VALUES('2018-01-20T09:00:43', '2017-01-20T09:00:55', 2, NULL); :PREFIX SELECT last(temp, time) FROM btest; --check non-null element "overrides" NULL because it comes after. INSERT INTO btest VALUES('2019-01-20T09:00:43', '2018-01-20T09:00:55', 2, 30.5); :PREFIX SELECT last(temp, time) FROM btest; --check null cmp element is skipped INSERT INTO btest VALUES('2018-01-20T09:00:43', NULL, 2, 32.3); :PREFIX SELECT last(temp, time_alt) FROM btest; -- fist returns NULL value :PREFIX SELECT first(temp, time_alt) FROM btest; -- test first return non NULL value INSERT INTO btest VALUES('2016-01-20T09:00:00', '2016-01-20T09:00:00', 2, 36.5); :PREFIX SELECT first(temp, time_alt) FROM btest; --check non null cmp element insert after null cmp INSERT INTO btest VALUES('2020-01-20T09:00:43', '2020-01-20T09:00:43', 2, 35.3); :PREFIX SELECT last(temp, time_alt) FROM btest; :PREFIX SELECT first(temp, time_alt) FROM btest; --cmp nulls should be ignored and not present in groups :PREFIX SELECT gp, last(temp, time_alt) FROM btest GROUP BY gp ORDER BY gp; --Previously, some bugs were found with NULLS and numeric types, so test that INSERT INTO btest_numeric VALUES ('2019-01-20T09:00:43', NULL); :PREFIX SELECT last(quantity, time) FROM btest_numeric; --check non-null element "overrides" NULL because it comes after. INSERT INTO btest_numeric VALUES('2020-01-20T09:00:43', 30.5); :PREFIX SELECT last(quantity, time) FROM btest_numeric; -- do index scan for last :PREFIX SELECT last(temp, time) FROM btest; -- do index scan for first :PREFIX SELECT first(temp, time) FROM btest; -- can't do index scan when ordering on non-index column :PREFIX SELECT first(temp, time_alt) FROM btest; -- do index scan for subquery :PREFIX SELECT * FROM (SELECT last(temp, time) FROM btest) last; -- can't do index scan when using group by :PREFIX SELECT last(temp, time) FROM btest GROUP BY gp ORDER BY gp; -- do index scan when agg function is used in CTE subquery :PREFIX WITH last_temp AS (SELECT last(temp, time) FROM btest) SELECT * from last_temp; -- do index scan when using both FIRST and LAST aggregate functions :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; -- verify results when using both FIRST and LAST :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; -- do index scan when using WHERE :PREFIX SELECT last(temp, time) FROM btest WHERE time <= '2017-01-20T09:00:02'; -- can't do index scan for MAX and LAST combined (MinMax optimization fails when having different aggregate functions) :PREFIX SELECT max(time), last(temp, time) FROM btest; -- can't do index scan when using FIRST/LAST in ORDER BY :PREFIX SELECT last(temp, time) FROM btest ORDER BY last(temp, time); -- do index scan :PREFIX SELECT last(temp, time) FROM btest WHERE temp < 30; -- SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- do index scan :PREFIX SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- can't do index scan when using WINDOW function :PREFIX SELECT gp, last(temp, time) OVER (PARTITION BY gp) AS last FROM btest; -- test constants :PREFIX SELECT first(100, 100) FROM btest; -- create an index so we can test optimization CREATE INDEX btest_time_alt_idx ON btest(time_alt); :PREFIX SELECT last(temp, time_alt) FROM btest; --test nested FIRST/LAST - should optimize :PREFIX SELECT abs(last(temp, time)) FROM btest; -- test nested FIRST/LAST in ORDER BY - no optimization possible :PREFIX SELECT abs(last(temp, time)) FROM btest ORDER BY abs(last(temp,time)); ROLLBACK; -- Test with NULL numeric values BEGIN; TRUNCATE btest_numeric; -- Empty table :PREFIX SELECT first(btest_numeric, time) FROM btest_numeric; :PREFIX SELECT last(btest_numeric, time) FROM btest_numeric; -- Only NULL values INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; -- NULL values followed by non-NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; TRUNCATE btest_numeric; -- non-NULL values followed by NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; ROLLBACK; \o :DIFF_CMD --- Unoptimized result +++ Optimized result @@ -1,6 +1,6 @@ setting | value ----------------------------------+------- - timescaledb.enable_optimizations | off + timescaledb.enable_optimizations | on time | gp | temp -- Test partial aggregation CREATE TABLE partial_aggregation (time timestamptz NOT NULL, quantity numeric, longvalue text); SELECT schema_name, table_name, created FROM create_hypertable('partial_aggregation', 'time'); schema_name | table_name | created -------------+---------------------+--------- public | partial_aggregation | t INSERT INTO partial_aggregation VALUES('2018-01-20T09:00:43', NULL, NULL); INSERT INTO partial_aggregation VALUES('2018-01-20T09:00:44', NULL, NULL); INSERT INTO partial_aggregation VALUES('2019-01-20T09:00:43', 1, 'hello'); INSERT INTO partial_aggregation VALUES('2019-01-20T09:00:44', 2, 'world'); INSERT INTO partial_aggregation VALUES('2020-01-20T09:00:43', 3.1, 'some1'); INSERT INTO partial_aggregation VALUES('2020-01-20T09:00:44', 3.2, 'more1'); INSERT INTO partial_aggregation VALUES('2021-01-20T09:00:43', 3.3, 'some2'); INSERT INTO partial_aggregation VALUES('2021-01-20T09:00:44', 3.4, 'more2'); INSERT INTO partial_aggregation VALUES('2022-01-20T09:00:43', 4, 'word1'); INSERT INTO partial_aggregation VALUES('2022-01-20T09:00:44', 5, 'word2'); INSERT INTO partial_aggregation VALUES('2023-01-20T09:00:43', 6, 'word3'); INSERT INTO partial_aggregation VALUES('2023-01-20T09:00:44', 7, 'word4'); -- Use enable_partitionwise_aggregate to create partial aggregates per chunk SET enable_partitionwise_aggregate = ON; SELECT format('SELECT %3$s, %1$s FROM partial_aggregation WHERE %2$s GROUP BY %3$s ORDER BY 1, 2;', function, condition, grouping) FROM unnest(array[ 'first(time, quantity), last(time, quantity)', 'last(longvalue, quantity)', 'last(quantity, longvalue)', 'last(quantity, time)', 'last(time, longvalue)']) AS function, unnest(array[ 'true', $$time < '2021-01-01'$$, 'quantity is null', 'quantity is not null', 'quantity >= 4']) AS condition, unnest(array[ '777::text' /* dummy grouping column */, 'longvalue', 'quantity', $$time_bucket('1 year', time)$$, $$time_bucket('3 year', time)$$]) AS grouping \gexec SELECT 777::text, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE true GROUP BY 777::text ORDER BY 1, 2; text | first | last ------+------------------------------+------------------------------ 777 | Sun Jan 20 09:00:43 2019 PST | Fri Jan 20 09:00:44 2023 PST SELECT 777::text, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY 777::text ORDER BY 1, 2; text | first | last ------+------------------------------+------------------------------ 777 | Sun Jan 20 09:00:43 2019 PST | Mon Jan 20 09:00:44 2020 PST SELECT 777::text, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY 777::text ORDER BY 1, 2; text | first | last ------+-------+------ 777 | | SELECT 777::text, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY 777::text ORDER BY 1, 2; text | first | last ------+------------------------------+------------------------------ 777 | Sun Jan 20 09:00:43 2019 PST | Fri Jan 20 09:00:44 2023 PST SELECT 777::text, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY 777::text ORDER BY 1, 2; text | first | last ------+------------------------------+------------------------------ 777 | Thu Jan 20 09:00:43 2022 PST | Fri Jan 20 09:00:44 2023 PST SELECT 777::text, last(longvalue, quantity) FROM partial_aggregation WHERE true GROUP BY 777::text ORDER BY 1, 2; text | last ------+------- 777 | word4 SELECT 777::text, last(longvalue, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY 777::text ORDER BY 1, 2; text | last ------+------- 777 | more1 SELECT 777::text, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | SELECT 777::text, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------- 777 | word4 SELECT 777::text, last(longvalue, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY 777::text ORDER BY 1, 2; text | last ------+------- 777 | word4 SELECT 777::text, last(quantity, longvalue) FROM partial_aggregation WHERE true GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 2 SELECT 777::text, last(quantity, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 2 SELECT 777::text, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | SELECT 777::text, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 2 SELECT 777::text, last(quantity, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 7 SELECT 777::text, last(quantity, time) FROM partial_aggregation WHERE true GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 7 SELECT 777::text, last(quantity, time) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 3.2 SELECT 777::text, last(quantity, time) FROM partial_aggregation WHERE quantity is null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | SELECT 777::text, last(quantity, time) FROM partial_aggregation WHERE quantity is not null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 7 SELECT 777::text, last(quantity, time) FROM partial_aggregation WHERE quantity >= 4 GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 7 SELECT 777::text, last(time, longvalue) FROM partial_aggregation WHERE true GROUP BY 777::text ORDER BY 1, 2; text | last ------+------------------------------ 777 | Sun Jan 20 09:00:44 2019 PST SELECT 777::text, last(time, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY 777::text ORDER BY 1, 2; text | last ------+------------------------------ 777 | Sun Jan 20 09:00:44 2019 PST SELECT 777::text, last(time, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | SELECT 777::text, last(time, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------------------------------ 777 | Sun Jan 20 09:00:44 2019 PST SELECT 777::text, last(time, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY 777::text ORDER BY 1, 2; text | last ------+------------------------------ 777 | Fri Jan 20 09:00:44 2023 PST SELECT longvalue, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE true GROUP BY longvalue ORDER BY 1, 2; longvalue | first | last -----------+------------------------------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST more2 | Wed Jan 20 09:00:44 2021 PST | Wed Jan 20 09:00:44 2021 PST some1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST some2 | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:43 2021 PST word1 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST world | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST | | SELECT longvalue, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY longvalue ORDER BY 1, 2; longvalue | first | last -----------+------------------------------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST some1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST world | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST | | SELECT longvalue, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY longvalue ORDER BY 1, 2; longvalue | first | last -----------+-------+------ | | SELECT longvalue, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY longvalue ORDER BY 1, 2; longvalue | first | last -----------+------------------------------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST more2 | Wed Jan 20 09:00:44 2021 PST | Wed Jan 20 09:00:44 2021 PST some1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST some2 | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:43 2021 PST word1 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST world | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST SELECT longvalue, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY longvalue ORDER BY 1, 2; longvalue | first | last -----------+------------------------------+------------------------------ word1 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT longvalue, last(longvalue, quantity) FROM partial_aggregation WHERE true GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------- hello | hello more1 | more1 more2 | more2 some1 | some1 some2 | some2 word1 | word1 word2 | word2 word3 | word3 word4 | word4 world | world | SELECT longvalue, last(longvalue, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------- hello | hello more1 | more1 some1 | some1 world | world | SELECT longvalue, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ | SELECT longvalue, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------- hello | hello more1 | more1 more2 | more2 some1 | some1 some2 | some2 word1 | word1 word2 | word2 word3 | word3 word4 | word4 world | world SELECT longvalue, last(longvalue, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------- word1 | word1 word2 | word2 word3 | word3 word4 | word4 SELECT longvalue, last(quantity, longvalue) FROM partial_aggregation WHERE true GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 more2 | 3.4 some1 | 3.1 some2 | 3.3 word1 | 4 word2 | 5 word3 | 6 word4 | 7 world | 2 | SELECT longvalue, last(quantity, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 some1 | 3.1 world | 2 | SELECT longvalue, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ | SELECT longvalue, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 more2 | 3.4 some1 | 3.1 some2 | 3.3 word1 | 4 word2 | 5 word3 | 6 word4 | 7 world | 2 SELECT longvalue, last(quantity, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ word1 | 4 word2 | 5 word3 | 6 word4 | 7 SELECT longvalue, last(quantity, time) FROM partial_aggregation WHERE true GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 more2 | 3.4 some1 | 3.1 some2 | 3.3 word1 | 4 word2 | 5 word3 | 6 word4 | 7 world | 2 | SELECT longvalue, last(quantity, time) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 some1 | 3.1 world | 2 | SELECT longvalue, last(quantity, time) FROM partial_aggregation WHERE quantity is null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ | SELECT longvalue, last(quantity, time) FROM partial_aggregation WHERE quantity is not null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 more2 | 3.4 some1 | 3.1 some2 | 3.3 word1 | 4 word2 | 5 word3 | 6 word4 | 7 world | 2 SELECT longvalue, last(quantity, time) FROM partial_aggregation WHERE quantity >= 4 GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ word1 | 4 word2 | 5 word3 | 6 word4 | 7 SELECT longvalue, last(time, longvalue) FROM partial_aggregation WHERE true GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST more2 | Wed Jan 20 09:00:44 2021 PST some1 | Mon Jan 20 09:00:43 2020 PST some2 | Wed Jan 20 09:00:43 2021 PST word1 | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST world | Sun Jan 20 09:00:44 2019 PST | SELECT longvalue, last(time, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST some1 | Mon Jan 20 09:00:43 2020 PST world | Sun Jan 20 09:00:44 2019 PST | SELECT longvalue, last(time, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ | SELECT longvalue, last(time, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST more2 | Wed Jan 20 09:00:44 2021 PST some1 | Mon Jan 20 09:00:43 2020 PST some2 | Wed Jan 20 09:00:43 2021 PST word1 | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST world | Sun Jan 20 09:00:44 2019 PST SELECT longvalue, last(time, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------------------------------ word1 | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST SELECT quantity, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE true GROUP BY quantity ORDER BY 1, 2; quantity | first | last ----------+------------------------------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST 3.3 | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:43 2021 PST 3.4 | Wed Jan 20 09:00:44 2021 PST | Wed Jan 20 09:00:44 2021 PST 4 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST | | SELECT quantity, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY quantity ORDER BY 1, 2; quantity | first | last ----------+------------------------------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST | | SELECT quantity, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY quantity ORDER BY 1, 2; quantity | first | last ----------+-------+------ | | SELECT quantity, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY quantity ORDER BY 1, 2; quantity | first | last ----------+------------------------------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST 3.3 | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:43 2021 PST 3.4 | Wed Jan 20 09:00:44 2021 PST | Wed Jan 20 09:00:44 2021 PST 4 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT quantity, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY quantity ORDER BY 1, 2; quantity | first | last ----------+------------------------------+------------------------------ 4 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT quantity, last(longvalue, quantity) FROM partial_aggregation WHERE true GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------- 1 | hello 2 | world 3.1 | some1 3.2 | more1 3.3 | some2 3.4 | more2 4 | word1 5 | word2 6 | word3 7 | word4 | SELECT quantity, last(longvalue, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------- 1 | hello 2 | world 3.1 | some1 3.2 | more1 | SELECT quantity, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ | SELECT quantity, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------- 1 | hello 2 | world 3.1 | some1 3.2 | more1 3.3 | some2 3.4 | more2 4 | word1 5 | word2 6 | word3 7 | word4 SELECT quantity, last(longvalue, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------- 4 | word1 5 | word2 6 | word3 7 | word4 SELECT quantity, last(quantity, longvalue) FROM partial_aggregation WHERE true GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 3.3 | 3.3 3.4 | 3.4 4 | 4 5 | 5 6 | 6 7 | 7 | SELECT quantity, last(quantity, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 | SELECT quantity, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ | SELECT quantity, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 3.3 | 3.3 3.4 | 3.4 4 | 4 5 | 5 6 | 6 7 | 7 SELECT quantity, last(quantity, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 4 | 4 5 | 5 6 | 6 7 | 7 SELECT quantity, last(quantity, time) FROM partial_aggregation WHERE true GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 3.3 | 3.3 3.4 | 3.4 4 | 4 5 | 5 6 | 6 7 | 7 | SELECT quantity, last(quantity, time) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 | SELECT quantity, last(quantity, time) FROM partial_aggregation WHERE quantity is null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ | SELECT quantity, last(quantity, time) FROM partial_aggregation WHERE quantity is not null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 3.3 | 3.3 3.4 | 3.4 4 | 4 5 | 5 6 | 6 7 | 7 SELECT quantity, last(quantity, time) FROM partial_aggregation WHERE quantity >= 4 GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 4 | 4 5 | 5 6 | 6 7 | 7 SELECT quantity, last(time, longvalue) FROM partial_aggregation WHERE true GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST 3.3 | Wed Jan 20 09:00:43 2021 PST 3.4 | Wed Jan 20 09:00:44 2021 PST 4 | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST | SELECT quantity, last(time, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST | SELECT quantity, last(time, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ | SELECT quantity, last(time, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST 3.3 | Wed Jan 20 09:00:43 2021 PST 3.4 | Wed Jan 20 09:00:44 2021 PST 4 | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST SELECT quantity, last(time, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------------------------------ 4 | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE true GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | | Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:44 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:44 2021 PST Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | | Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:44 2020 PST SELECT time_bucket('1 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+-------+------ Sun Dec 31 16:00:00 2017 PST | | SELECT time_bucket('1 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:44 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:44 2021 PST Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE true GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | world Tue Dec 31 16:00:00 2019 PST | more1 Thu Dec 31 16:00:00 2020 PST | more2 Fri Dec 31 16:00:00 2021 PST | word2 Sat Dec 31 16:00:00 2022 PST | word4 SELECT time_bucket('1 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | world Tue Dec 31 16:00:00 2019 PST | more1 SELECT time_bucket('1 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('1 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Mon Dec 31 16:00:00 2018 PST | world Tue Dec 31 16:00:00 2019 PST | more1 Thu Dec 31 16:00:00 2020 PST | more2 Fri Dec 31 16:00:00 2021 PST | word2 Sat Dec 31 16:00:00 2022 PST | word4 SELECT time_bucket('1 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Fri Dec 31 16:00:00 2021 PST | word2 Sat Dec 31 16:00:00 2022 PST | word4 SELECT time_bucket('1 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE true GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.1 Thu Dec 31 16:00:00 2020 PST | 3.3 Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.1 SELECT time_bucket('1 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('1 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.1 Thu Dec 31 16:00:00 2020 PST | 3.3 Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(quantity, time) FROM partial_aggregation WHERE true GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.2 Thu Dec 31 16:00:00 2020 PST | 3.4 Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(quantity, time) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.2 SELECT time_bucket('1 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('1 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.2 Thu Dec 31 16:00:00 2020 PST | 3.4 Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(time, longvalue) FROM partial_aggregation WHERE true GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), last(time, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST SELECT time_bucket('1 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('1 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE true GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:43 2019 PST | Mon Jan 20 09:00:44 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:43 2019 PST | Mon Jan 20 09:00:44 2020 PST SELECT time_bucket('3 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+-------+------ Sun Dec 31 16:00:00 2017 PST | | SELECT time_bucket('3 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:43 2019 PST | Mon Jan 20 09:00:44 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Thu Dec 31 16:00:00 2020 PST | Thu Jan 20 09:00:43 2022 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE true GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Sun Dec 31 16:00:00 2017 PST | more1 Thu Dec 31 16:00:00 2020 PST | word4 SELECT time_bucket('3 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Sun Dec 31 16:00:00 2017 PST | more1 SELECT time_bucket('3 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('3 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Sun Dec 31 16:00:00 2017 PST | more1 Thu Dec 31 16:00:00 2020 PST | word4 SELECT time_bucket('3 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Thu Dec 31 16:00:00 2020 PST | word4 SELECT time_bucket('3 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE true GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 2 Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 2 SELECT time_bucket('3 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('3 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 2 Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(quantity, time) FROM partial_aggregation WHERE true GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 3.2 Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(quantity, time) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 3.2 SELECT time_bucket('3 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('3 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 3.2 Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(time, longvalue) FROM partial_aggregation WHERE true GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:44 2019 PST Thu Dec 31 16:00:00 2020 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), last(time, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:44 2019 PST SELECT time_bucket('3 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('3 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:44 2019 PST Thu Dec 31 16:00:00 2020 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Thu Dec 31 16:00:00 2020 PST | Fri Jan 20 09:00:44 2023 PST SET enable_partitionwise_aggregate = OFF; ================================================ FILE: test/expected/agg_bookends-17.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set TEST_BASE_NAME agg_bookends SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') as "TEST_LOAD_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') as "TEST_QUERY_NAME", format('%s/results/%s_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_OPTIMIZED", format('%s/results/%s_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_UNOPTIMIZED" \gset SELECT format('\! diff -u --label "Unoptimized result" --label "Optimized result" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') as "DIFF_CMD" \gset \set PREFIX 'EXPLAIN (analyze, buffers off, costs off, timing off, summary off)' \ir :TEST_LOAD_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE btest(time timestamp NOT NULL, time_alt timestamp, gp INTEGER, temp float, strid TEXT DEFAULT 'testing'); SELECT schema_name, table_name, created FROM create_hypertable('btest', 'time'); psql:include/agg_bookends_load.sql:6: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices psql:include/agg_bookends_load.sql:6: WARNING: column type "timestamp without time zone" used for "time_alt" does not follow best practices schema_name | table_name | created -------------+------------+--------- public | btest | t INSERT INTO btest VALUES('2017-01-20T09:00:01', '2017-01-20T10:00:00', 1, 22.5); INSERT INTO btest VALUES('2017-01-20T09:00:21', '2017-01-20T09:00:59', 1, 21.2); INSERT INTO btest VALUES('2017-01-20T09:00:47', '2017-01-20T09:00:58', 1, 25.1); INSERT INTO btest VALUES('2017-01-20T09:00:02', '2017-01-20T09:00:57', 2, 35.5); INSERT INTO btest VALUES('2017-01-20T09:00:21', '2017-01-20T09:00:56', 2, 30.2); --TOASTED; INSERT INTO btest VALUES('2017-01-20T09:00:43', '2017-01-20T09:01:55', 2, 20.1, repeat('xyz', 1000000) ); CREATE TABLE btest_numeric (time timestamp NOT NULL, quantity numeric); SELECT schema_name, table_name, created FROM create_hypertable('btest_numeric', 'time'); psql:include/agg_bookends_load.sql:16: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices schema_name | table_name | created -------------+---------------+--------- public | btest_numeric | t \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be only output of results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations')) v(setting); setting | value ----------------------------------+------- timescaledb.enable_optimizations | on :PREFIX SELECT time, gp, temp FROM btest ORDER BY time; --- QUERY PLAN --- Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) :PREFIX SELECT first(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) :PREFIX SELECT last(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT first(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT gp, last(temp, time) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT gp, first(temp, time) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) --check whole row :PREFIX SELECT gp, first(btest, time) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) --check toasted col :PREFIX SELECT gp, left(last(strid, time), 10) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT gp, last(temp, strid) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT gp, last(strid, temp) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) BEGIN; --check null value as last element INSERT INTO btest VALUES('2018-01-20T09:00:43', '2017-01-20T09:00:55', 2, NULL); :PREFIX SELECT last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) --check non-null element "overrides" NULL because it comes after. INSERT INTO btest VALUES('2019-01-20T09:00:43', '2018-01-20T09:00:55', 2, 30.5); :PREFIX SELECT last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) --check null cmp element is skipped INSERT INTO btest VALUES('2018-01-20T09:00:43', NULL, 2, 32.3); :PREFIX SELECT last(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=9.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -- fist returns NULL value :PREFIX SELECT first(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=9.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -- test first return non NULL value INSERT INTO btest VALUES('2016-01-20T09:00:00', '2016-01-20T09:00:00', 2, 36.5); :PREFIX SELECT first(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=10.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) --check non null cmp element insert after null cmp INSERT INTO btest VALUES('2020-01-20T09:00:43', '2020-01-20T09:00:43', 2, 35.3); :PREFIX SELECT last(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) :PREFIX SELECT first(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) --cmp nulls should be ignored and not present in groups :PREFIX SELECT gp, last(temp, time_alt) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: btest.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: btest.gp -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) --Previously, some bugs were found with NULLS and numeric types, so test that INSERT INTO btest_numeric VALUES ('2019-01-20T09:00:43', NULL); :PREFIX SELECT last(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Index Scan using _hyper_2_6_chunk_btest_numeric_time_idx on _hyper_2_6_chunk (actual rows=1.00 loops=1) --check non-null element "overrides" NULL because it comes after. INSERT INTO btest_numeric VALUES('2020-01-20T09:00:43', 30.5); :PREFIX SELECT last(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest_numeric (actual rows=1.00 loops=1) Order: btest_numeric."time" DESC -> Index Scan using _hyper_2_7_chunk_btest_numeric_time_idx on _hyper_2_7_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_2_6_chunk_btest_numeric_time_idx on _hyper_2_6_chunk (never executed) -- do index scan for last :PREFIX SELECT last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) -- do index scan for first :PREFIX SELECT first(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" -> Index Scan Backward using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) -> Index Scan Backward using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan Backward using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) -> Index Scan Backward using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (never executed) -- can't do index scan when ordering on non-index column :PREFIX SELECT first(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) -- do index scan for subquery :PREFIX SELECT * FROM (SELECT last(temp, time) FROM btest) last; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) -- can't do index scan when using group by :PREFIX SELECT last(temp, time) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: btest.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: btest.gp -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) -- do index scan when agg function is used in CTE subquery :PREFIX WITH last_temp AS (SELECT last(temp, time) FROM btest) SELECT * from last_temp; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) -- do index scan when using both FIRST and LAST aggregate functions :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) InitPlan 2 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest btest_1 (actual rows=1.00 loops=1) Order: btest_1."time" -> Index Scan Backward using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk _hyper_1_4_chunk_1 (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk _hyper_1_3_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk _hyper_1_5_chunk_1 (never executed) -- verify results when using both FIRST and LAST :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) InitPlan 2 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest btest_1 (actual rows=1.00 loops=1) Order: btest_1."time" -> Index Scan Backward using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk _hyper_1_4_chunk_1 (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk _hyper_1_3_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk _hyper_1_5_chunk_1 (never executed) -- do index scan when using WHERE :PREFIX SELECT last(temp, time) FROM btest WHERE time <= '2017-01-20T09:00:02'; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) Index Cond: ("time" <= 'Fri Jan 20 09:00:02 2017'::timestamp without time zone) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) -- can't do index scan for MAX and LAST combined (MinMax optimization fails when having different aggregate functions) :PREFIX SELECT max(time), last(temp, time) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) -- can't do index scan when using FIRST/LAST in ORDER BY :PREFIX SELECT last(temp, time) FROM btest ORDER BY last(temp, time); --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: (last(btest.temp, btest."time")) Sort Method: quicksort -> Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) -- do index scan :PREFIX SELECT last(temp, time) FROM btest WHERE temp < 30; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=0.00 loops=1) Filter: (temp < '30'::double precision) Rows Removed by Filter: 1 -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (actual rows=0.00 loops=1) Filter: (temp < '30'::double precision) Rows Removed by Filter: 1 -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (actual rows=0.00 loops=1) Filter: (temp < '30'::double precision) Rows Removed by Filter: 2 -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) Filter: (temp < '30'::double precision) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) Filter: (temp < '30'::double precision) -- SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- do index scan :PREFIX SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" -> Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Fri Jan 20 09:00:47 2017'::timestamp without time zone) -> Index Scan Backward using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan Backward using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) -> Index Scan Backward using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (never executed) -- can't do index scan when using WINDOW function :PREFIX SELECT gp, last(temp, time) OVER (PARTITION BY gp) AS last FROM btest; --- QUERY PLAN --- WindowAgg (actual rows=11.00 loops=1) -> Sort (actual rows=11.00 loops=1) Sort Key: btest.gp Sort Method: quicksort -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) -- test constants :PREFIX SELECT first(100, 100) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Result (actual rows=1.00 loops=1) -> Append (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (never executed) -> Seq Scan on _hyper_1_3_chunk (never executed) -> Seq Scan on _hyper_1_4_chunk (never executed) -> Seq Scan on _hyper_1_5_chunk (never executed) -- create an index so we can test optimization CREATE INDEX btest_time_alt_idx ON btest(time_alt); :PREFIX SELECT last(temp, time_alt) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Merge Append (actual rows=1.00 loops=1) Sort Key: btest.time_alt DESC -> Index Scan Backward using _hyper_1_1_chunk_btest_time_alt_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) Index Cond: (time_alt IS NOT NULL) -> Index Scan Backward using _hyper_1_2_chunk_btest_time_alt_idx on _hyper_1_2_chunk (actual rows=1.00 loops=1) Index Cond: (time_alt IS NOT NULL) -> Index Scan Backward using _hyper_1_3_chunk_btest_time_alt_idx on _hyper_1_3_chunk (actual rows=1.00 loops=1) Index Cond: (time_alt IS NOT NULL) -> Index Scan Backward using _hyper_1_4_chunk_btest_time_alt_idx on _hyper_1_4_chunk (actual rows=1.00 loops=1) Index Cond: (time_alt IS NOT NULL) -> Index Scan Backward using _hyper_1_5_chunk_btest_time_alt_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) Index Cond: (time_alt IS NOT NULL) --test nested FIRST/LAST - should optimize :PREFIX SELECT abs(last(temp, time)) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) -- test nested FIRST/LAST in ORDER BY - no optimization possible :PREFIX SELECT abs(last(temp, time)) FROM btest ORDER BY abs(last(temp,time)); --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: (abs(last(btest.temp, btest."time"))) Sort Method: quicksort -> Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) ROLLBACK; -- Test with NULL numeric values BEGIN; TRUNCATE btest_numeric; -- Empty table :PREFIX SELECT first(btest_numeric, time) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Result (actual rows=0.00 loops=1) One-Time Filter: false :PREFIX SELECT last(btest_numeric, time) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Result (actual rows=0.00 loops=1) One-Time Filter: false -- Only NULL values INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_2_8_chunk_btest_numeric_time_idx on _hyper_2_8_chunk (actual rows=1.00 loops=1) :PREFIX SELECT last(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Index Scan using _hyper_2_8_chunk_btest_numeric_time_idx on _hyper_2_8_chunk (actual rows=1.00 loops=1) :PREFIX SELECT first(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Seq Scan on _hyper_2_8_chunk (actual rows=2.00 loops=1) :PREFIX SELECT last(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Seq Scan on _hyper_2_8_chunk (actual rows=2.00 loops=1) -- NULL values followed by non-NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); :PREFIX SELECT first(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest_numeric (actual rows=1.00 loops=1) Order: btest_numeric."time" -> Index Scan Backward using _hyper_2_8_chunk_btest_numeric_time_idx on _hyper_2_8_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_2_9_chunk_btest_numeric_time_idx on _hyper_2_9_chunk (never executed) :PREFIX SELECT last(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest_numeric (actual rows=1.00 loops=1) Order: btest_numeric."time" DESC -> Index Scan using _hyper_2_9_chunk_btest_numeric_time_idx on _hyper_2_9_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_2_8_chunk_btest_numeric_time_idx on _hyper_2_8_chunk (never executed) :PREFIX SELECT first(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_2_8_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_2_9_chunk (actual rows=2.00 loops=1) :PREFIX SELECT last(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_2_8_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_2_9_chunk (actual rows=2.00 loops=1) TRUNCATE btest_numeric; -- non-NULL values followed by NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest_numeric (actual rows=1.00 loops=1) Order: btest_numeric."time" -> Index Scan Backward using _hyper_2_11_chunk_btest_numeric_time_idx on _hyper_2_11_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_2_10_chunk_btest_numeric_time_idx on _hyper_2_10_chunk (never executed) :PREFIX SELECT last(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest_numeric (actual rows=1.00 loops=1) Order: btest_numeric."time" DESC -> Index Scan using _hyper_2_10_chunk_btest_numeric_time_idx on _hyper_2_10_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_2_11_chunk_btest_numeric_time_idx on _hyper_2_11_chunk (never executed) :PREFIX SELECT first(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_2_10_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_2_11_chunk (actual rows=2.00 loops=1) :PREFIX SELECT last(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_2_10_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_2_11_chunk (actual rows=2.00 loops=1) ROLLBACK; -- we want test results as part of the output too to make sure we produce correct output \set PREFIX '' \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be only output of results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations')) v(setting); setting | value ----------------------------------+------- timescaledb.enable_optimizations | on :PREFIX SELECT time, gp, temp FROM btest ORDER BY time; time | gp | temp --------------------------+----+------ Fri Jan 20 09:00:01 2017 | 1 | 22.5 Fri Jan 20 09:00:02 2017 | 2 | 35.5 Fri Jan 20 09:00:21 2017 | 1 | 21.2 Fri Jan 20 09:00:21 2017 | 2 | 30.2 Fri Jan 20 09:00:43 2017 | 2 | 20.1 Fri Jan 20 09:00:47 2017 | 1 | 25.1 :PREFIX SELECT last(temp, time) FROM btest; last ------ 25.1 :PREFIX SELECT first(temp, time) FROM btest; first ------- 22.5 :PREFIX SELECT last(temp, time_alt) FROM btest; last ------ 22.5 :PREFIX SELECT first(temp, time_alt) FROM btest; first ------- 30.2 :PREFIX SELECT gp, last(temp, time) FROM btest GROUP BY gp ORDER BY gp; gp | last ----+------ 1 | 25.1 2 | 20.1 :PREFIX SELECT gp, first(temp, time) FROM btest GROUP BY gp ORDER BY gp; gp | first ----+------- 1 | 22.5 2 | 35.5 --check whole row :PREFIX SELECT gp, first(btest, time) FROM btest GROUP BY gp ORDER BY gp; gp | first ----+------------------------------------------------------------------------ 1 | ("Fri Jan 20 09:00:01 2017","Fri Jan 20 10:00:00 2017",1,22.5,testing) 2 | ("Fri Jan 20 09:00:02 2017","Fri Jan 20 09:00:57 2017",2,35.5,testing) --check toasted col :PREFIX SELECT gp, left(last(strid, time), 10) FROM btest GROUP BY gp ORDER BY gp; gp | left ----+------------ 1 | testing 2 | xyzxyzxyzx :PREFIX SELECT gp, last(temp, strid) FROM btest GROUP BY gp ORDER BY gp; gp | last ----+------ 1 | 22.5 2 | 20.1 :PREFIX SELECT gp, last(strid, temp) FROM btest GROUP BY gp ORDER BY gp; gp | last ----+--------- 1 | testing 2 | testing BEGIN; --check null value as last element INSERT INTO btest VALUES('2018-01-20T09:00:43', '2017-01-20T09:00:55', 2, NULL); :PREFIX SELECT last(temp, time) FROM btest; last ------ --check non-null element "overrides" NULL because it comes after. INSERT INTO btest VALUES('2019-01-20T09:00:43', '2018-01-20T09:00:55', 2, 30.5); :PREFIX SELECT last(temp, time) FROM btest; last ------ 30.5 --check null cmp element is skipped INSERT INTO btest VALUES('2018-01-20T09:00:43', NULL, 2, 32.3); :PREFIX SELECT last(temp, time_alt) FROM btest; last ------ 30.5 -- fist returns NULL value :PREFIX SELECT first(temp, time_alt) FROM btest; first ------- -- test first return non NULL value INSERT INTO btest VALUES('2016-01-20T09:00:00', '2016-01-20T09:00:00', 2, 36.5); :PREFIX SELECT first(temp, time_alt) FROM btest; first ------- 36.5 --check non null cmp element insert after null cmp INSERT INTO btest VALUES('2020-01-20T09:00:43', '2020-01-20T09:00:43', 2, 35.3); :PREFIX SELECT last(temp, time_alt) FROM btest; last ------ 35.3 :PREFIX SELECT first(temp, time_alt) FROM btest; first ------- 36.5 --cmp nulls should be ignored and not present in groups :PREFIX SELECT gp, last(temp, time_alt) FROM btest GROUP BY gp ORDER BY gp; gp | last ----+------ 1 | 22.5 2 | 35.3 --Previously, some bugs were found with NULLS and numeric types, so test that INSERT INTO btest_numeric VALUES ('2019-01-20T09:00:43', NULL); :PREFIX SELECT last(quantity, time) FROM btest_numeric; last ------ --check non-null element "overrides" NULL because it comes after. INSERT INTO btest_numeric VALUES('2020-01-20T09:00:43', 30.5); :PREFIX SELECT last(quantity, time) FROM btest_numeric; last ------ 30.5 -- do index scan for last :PREFIX SELECT last(temp, time) FROM btest; last ------ 35.3 -- do index scan for first :PREFIX SELECT first(temp, time) FROM btest; first ------- 36.5 -- can't do index scan when ordering on non-index column :PREFIX SELECT first(temp, time_alt) FROM btest; first ------- 36.5 -- do index scan for subquery :PREFIX SELECT * FROM (SELECT last(temp, time) FROM btest) last; last ------ 35.3 -- can't do index scan when using group by :PREFIX SELECT last(temp, time) FROM btest GROUP BY gp ORDER BY gp; last ------ 25.1 35.3 -- do index scan when agg function is used in CTE subquery :PREFIX WITH last_temp AS (SELECT last(temp, time) FROM btest) SELECT * from last_temp; last ------ 35.3 -- do index scan when using both FIRST and LAST aggregate functions :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; first | last -------+------ 36.5 | 35.3 -- verify results when using both FIRST and LAST :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; first | last -------+------ 36.5 | 35.3 -- do index scan when using WHERE :PREFIX SELECT last(temp, time) FROM btest WHERE time <= '2017-01-20T09:00:02'; last ------ 35.5 -- can't do index scan for MAX and LAST combined (MinMax optimization fails when having different aggregate functions) :PREFIX SELECT max(time), last(temp, time) FROM btest; max | last --------------------------+------ Mon Jan 20 09:00:43 2020 | 35.3 -- can't do index scan when using FIRST/LAST in ORDER BY :PREFIX SELECT last(temp, time) FROM btest ORDER BY last(temp, time); last ------ 35.3 -- do index scan :PREFIX SELECT last(temp, time) FROM btest WHERE temp < 30; last ------ 25.1 -- SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- do index scan :PREFIX SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; first ------- 25.1 -- can't do index scan when using WINDOW function :PREFIX SELECT gp, last(temp, time) OVER (PARTITION BY gp) AS last FROM btest; gp | last ----+------ 1 | 25.1 1 | 25.1 1 | 25.1 2 | 35.3 2 | 35.3 2 | 35.3 2 | 35.3 2 | 35.3 2 | 35.3 2 | 35.3 2 | 35.3 -- test constants :PREFIX SELECT first(100, 100) FROM btest; first ------- 100 -- create an index so we can test optimization CREATE INDEX btest_time_alt_idx ON btest(time_alt); :PREFIX SELECT last(temp, time_alt) FROM btest; last ------ 35.3 --test nested FIRST/LAST - should optimize :PREFIX SELECT abs(last(temp, time)) FROM btest; abs ------ 35.3 -- test nested FIRST/LAST in ORDER BY - no optimization possible :PREFIX SELECT abs(last(temp, time)) FROM btest ORDER BY abs(last(temp,time)); abs ------ 35.3 ROLLBACK; -- Test with NULL numeric values BEGIN; TRUNCATE btest_numeric; -- Empty table :PREFIX SELECT first(btest_numeric, time) FROM btest_numeric; first ------- :PREFIX SELECT last(btest_numeric, time) FROM btest_numeric; last ------ -- Only NULL values INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; first ------- :PREFIX SELECT last(quantity, time) FROM btest_numeric; last ------ :PREFIX SELECT first(time, quantity) FROM btest_numeric; first ------- :PREFIX SELECT last(time, quantity) FROM btest_numeric; last ------ -- NULL values followed by non-NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); :PREFIX SELECT first(quantity, time) FROM btest_numeric; first ------- :PREFIX SELECT last(quantity, time) FROM btest_numeric; last ------ 1 :PREFIX SELECT first(time, quantity) FROM btest_numeric; first -------------------------- Sun Jan 20 09:00:43 2019 :PREFIX SELECT last(time, quantity) FROM btest_numeric; last -------------------------- Sun Jan 20 09:00:43 2019 TRUNCATE btest_numeric; -- non-NULL values followed by NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; first ------- :PREFIX SELECT last(quantity, time) FROM btest_numeric; last ------ 1 :PREFIX SELECT first(time, quantity) FROM btest_numeric; first -------------------------- Sun Jan 20 09:00:43 2019 :PREFIX SELECT last(time, quantity) FROM btest_numeric; last -------------------------- Sun Jan 20 09:00:43 2019 ROLLBACK; -- diff results with optimizations disabled and enabled \o :TEST_RESULTS_UNOPTIMIZED SET timescaledb.enable_optimizations TO false; \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be only output of results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations')) v(setting); :PREFIX SELECT time, gp, temp FROM btest ORDER BY time; :PREFIX SELECT last(temp, time) FROM btest; :PREFIX SELECT first(temp, time) FROM btest; :PREFIX SELECT last(temp, time_alt) FROM btest; :PREFIX SELECT first(temp, time_alt) FROM btest; :PREFIX SELECT gp, last(temp, time) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, first(temp, time) FROM btest GROUP BY gp ORDER BY gp; --check whole row :PREFIX SELECT gp, first(btest, time) FROM btest GROUP BY gp ORDER BY gp; --check toasted col :PREFIX SELECT gp, left(last(strid, time), 10) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, last(temp, strid) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, last(strid, temp) FROM btest GROUP BY gp ORDER BY gp; BEGIN; --check null value as last element INSERT INTO btest VALUES('2018-01-20T09:00:43', '2017-01-20T09:00:55', 2, NULL); :PREFIX SELECT last(temp, time) FROM btest; --check non-null element "overrides" NULL because it comes after. INSERT INTO btest VALUES('2019-01-20T09:00:43', '2018-01-20T09:00:55', 2, 30.5); :PREFIX SELECT last(temp, time) FROM btest; --check null cmp element is skipped INSERT INTO btest VALUES('2018-01-20T09:00:43', NULL, 2, 32.3); :PREFIX SELECT last(temp, time_alt) FROM btest; -- fist returns NULL value :PREFIX SELECT first(temp, time_alt) FROM btest; -- test first return non NULL value INSERT INTO btest VALUES('2016-01-20T09:00:00', '2016-01-20T09:00:00', 2, 36.5); :PREFIX SELECT first(temp, time_alt) FROM btest; --check non null cmp element insert after null cmp INSERT INTO btest VALUES('2020-01-20T09:00:43', '2020-01-20T09:00:43', 2, 35.3); :PREFIX SELECT last(temp, time_alt) FROM btest; :PREFIX SELECT first(temp, time_alt) FROM btest; --cmp nulls should be ignored and not present in groups :PREFIX SELECT gp, last(temp, time_alt) FROM btest GROUP BY gp ORDER BY gp; --Previously, some bugs were found with NULLS and numeric types, so test that INSERT INTO btest_numeric VALUES ('2019-01-20T09:00:43', NULL); :PREFIX SELECT last(quantity, time) FROM btest_numeric; --check non-null element "overrides" NULL because it comes after. INSERT INTO btest_numeric VALUES('2020-01-20T09:00:43', 30.5); :PREFIX SELECT last(quantity, time) FROM btest_numeric; -- do index scan for last :PREFIX SELECT last(temp, time) FROM btest; -- do index scan for first :PREFIX SELECT first(temp, time) FROM btest; -- can't do index scan when ordering on non-index column :PREFIX SELECT first(temp, time_alt) FROM btest; -- do index scan for subquery :PREFIX SELECT * FROM (SELECT last(temp, time) FROM btest) last; -- can't do index scan when using group by :PREFIX SELECT last(temp, time) FROM btest GROUP BY gp ORDER BY gp; -- do index scan when agg function is used in CTE subquery :PREFIX WITH last_temp AS (SELECT last(temp, time) FROM btest) SELECT * from last_temp; -- do index scan when using both FIRST and LAST aggregate functions :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; -- verify results when using both FIRST and LAST :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; -- do index scan when using WHERE :PREFIX SELECT last(temp, time) FROM btest WHERE time <= '2017-01-20T09:00:02'; -- can't do index scan for MAX and LAST combined (MinMax optimization fails when having different aggregate functions) :PREFIX SELECT max(time), last(temp, time) FROM btest; -- can't do index scan when using FIRST/LAST in ORDER BY :PREFIX SELECT last(temp, time) FROM btest ORDER BY last(temp, time); -- do index scan :PREFIX SELECT last(temp, time) FROM btest WHERE temp < 30; -- SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- do index scan :PREFIX SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- can't do index scan when using WINDOW function :PREFIX SELECT gp, last(temp, time) OVER (PARTITION BY gp) AS last FROM btest; -- test constants :PREFIX SELECT first(100, 100) FROM btest; -- create an index so we can test optimization CREATE INDEX btest_time_alt_idx ON btest(time_alt); :PREFIX SELECT last(temp, time_alt) FROM btest; --test nested FIRST/LAST - should optimize :PREFIX SELECT abs(last(temp, time)) FROM btest; -- test nested FIRST/LAST in ORDER BY - no optimization possible :PREFIX SELECT abs(last(temp, time)) FROM btest ORDER BY abs(last(temp,time)); ROLLBACK; -- Test with NULL numeric values BEGIN; TRUNCATE btest_numeric; -- Empty table :PREFIX SELECT first(btest_numeric, time) FROM btest_numeric; :PREFIX SELECT last(btest_numeric, time) FROM btest_numeric; -- Only NULL values INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; -- NULL values followed by non-NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; TRUNCATE btest_numeric; -- non-NULL values followed by NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; ROLLBACK; \o \o :TEST_RESULTS_OPTIMIZED SET timescaledb.enable_optimizations TO true; \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be only output of results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations')) v(setting); :PREFIX SELECT time, gp, temp FROM btest ORDER BY time; :PREFIX SELECT last(temp, time) FROM btest; :PREFIX SELECT first(temp, time) FROM btest; :PREFIX SELECT last(temp, time_alt) FROM btest; :PREFIX SELECT first(temp, time_alt) FROM btest; :PREFIX SELECT gp, last(temp, time) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, first(temp, time) FROM btest GROUP BY gp ORDER BY gp; --check whole row :PREFIX SELECT gp, first(btest, time) FROM btest GROUP BY gp ORDER BY gp; --check toasted col :PREFIX SELECT gp, left(last(strid, time), 10) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, last(temp, strid) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, last(strid, temp) FROM btest GROUP BY gp ORDER BY gp; BEGIN; --check null value as last element INSERT INTO btest VALUES('2018-01-20T09:00:43', '2017-01-20T09:00:55', 2, NULL); :PREFIX SELECT last(temp, time) FROM btest; --check non-null element "overrides" NULL because it comes after. INSERT INTO btest VALUES('2019-01-20T09:00:43', '2018-01-20T09:00:55', 2, 30.5); :PREFIX SELECT last(temp, time) FROM btest; --check null cmp element is skipped INSERT INTO btest VALUES('2018-01-20T09:00:43', NULL, 2, 32.3); :PREFIX SELECT last(temp, time_alt) FROM btest; -- fist returns NULL value :PREFIX SELECT first(temp, time_alt) FROM btest; -- test first return non NULL value INSERT INTO btest VALUES('2016-01-20T09:00:00', '2016-01-20T09:00:00', 2, 36.5); :PREFIX SELECT first(temp, time_alt) FROM btest; --check non null cmp element insert after null cmp INSERT INTO btest VALUES('2020-01-20T09:00:43', '2020-01-20T09:00:43', 2, 35.3); :PREFIX SELECT last(temp, time_alt) FROM btest; :PREFIX SELECT first(temp, time_alt) FROM btest; --cmp nulls should be ignored and not present in groups :PREFIX SELECT gp, last(temp, time_alt) FROM btest GROUP BY gp ORDER BY gp; --Previously, some bugs were found with NULLS and numeric types, so test that INSERT INTO btest_numeric VALUES ('2019-01-20T09:00:43', NULL); :PREFIX SELECT last(quantity, time) FROM btest_numeric; --check non-null element "overrides" NULL because it comes after. INSERT INTO btest_numeric VALUES('2020-01-20T09:00:43', 30.5); :PREFIX SELECT last(quantity, time) FROM btest_numeric; -- do index scan for last :PREFIX SELECT last(temp, time) FROM btest; -- do index scan for first :PREFIX SELECT first(temp, time) FROM btest; -- can't do index scan when ordering on non-index column :PREFIX SELECT first(temp, time_alt) FROM btest; -- do index scan for subquery :PREFIX SELECT * FROM (SELECT last(temp, time) FROM btest) last; -- can't do index scan when using group by :PREFIX SELECT last(temp, time) FROM btest GROUP BY gp ORDER BY gp; -- do index scan when agg function is used in CTE subquery :PREFIX WITH last_temp AS (SELECT last(temp, time) FROM btest) SELECT * from last_temp; -- do index scan when using both FIRST and LAST aggregate functions :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; -- verify results when using both FIRST and LAST :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; -- do index scan when using WHERE :PREFIX SELECT last(temp, time) FROM btest WHERE time <= '2017-01-20T09:00:02'; -- can't do index scan for MAX and LAST combined (MinMax optimization fails when having different aggregate functions) :PREFIX SELECT max(time), last(temp, time) FROM btest; -- can't do index scan when using FIRST/LAST in ORDER BY :PREFIX SELECT last(temp, time) FROM btest ORDER BY last(temp, time); -- do index scan :PREFIX SELECT last(temp, time) FROM btest WHERE temp < 30; -- SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- do index scan :PREFIX SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- can't do index scan when using WINDOW function :PREFIX SELECT gp, last(temp, time) OVER (PARTITION BY gp) AS last FROM btest; -- test constants :PREFIX SELECT first(100, 100) FROM btest; -- create an index so we can test optimization CREATE INDEX btest_time_alt_idx ON btest(time_alt); :PREFIX SELECT last(temp, time_alt) FROM btest; --test nested FIRST/LAST - should optimize :PREFIX SELECT abs(last(temp, time)) FROM btest; -- test nested FIRST/LAST in ORDER BY - no optimization possible :PREFIX SELECT abs(last(temp, time)) FROM btest ORDER BY abs(last(temp,time)); ROLLBACK; -- Test with NULL numeric values BEGIN; TRUNCATE btest_numeric; -- Empty table :PREFIX SELECT first(btest_numeric, time) FROM btest_numeric; :PREFIX SELECT last(btest_numeric, time) FROM btest_numeric; -- Only NULL values INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; -- NULL values followed by non-NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; TRUNCATE btest_numeric; -- non-NULL values followed by NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; ROLLBACK; \o :DIFF_CMD --- Unoptimized result +++ Optimized result @@ -1,6 +1,6 @@ setting | value ----------------------------------+------- - timescaledb.enable_optimizations | off + timescaledb.enable_optimizations | on time | gp | temp -- Test partial aggregation CREATE TABLE partial_aggregation (time timestamptz NOT NULL, quantity numeric, longvalue text); SELECT schema_name, table_name, created FROM create_hypertable('partial_aggregation', 'time'); schema_name | table_name | created -------------+---------------------+--------- public | partial_aggregation | t INSERT INTO partial_aggregation VALUES('2018-01-20T09:00:43', NULL, NULL); INSERT INTO partial_aggregation VALUES('2018-01-20T09:00:44', NULL, NULL); INSERT INTO partial_aggregation VALUES('2019-01-20T09:00:43', 1, 'hello'); INSERT INTO partial_aggregation VALUES('2019-01-20T09:00:44', 2, 'world'); INSERT INTO partial_aggregation VALUES('2020-01-20T09:00:43', 3.1, 'some1'); INSERT INTO partial_aggregation VALUES('2020-01-20T09:00:44', 3.2, 'more1'); INSERT INTO partial_aggregation VALUES('2021-01-20T09:00:43', 3.3, 'some2'); INSERT INTO partial_aggregation VALUES('2021-01-20T09:00:44', 3.4, 'more2'); INSERT INTO partial_aggregation VALUES('2022-01-20T09:00:43', 4, 'word1'); INSERT INTO partial_aggregation VALUES('2022-01-20T09:00:44', 5, 'word2'); INSERT INTO partial_aggregation VALUES('2023-01-20T09:00:43', 6, 'word3'); INSERT INTO partial_aggregation VALUES('2023-01-20T09:00:44', 7, 'word4'); -- Use enable_partitionwise_aggregate to create partial aggregates per chunk SET enable_partitionwise_aggregate = ON; SELECT format('SELECT %3$s, %1$s FROM partial_aggregation WHERE %2$s GROUP BY %3$s ORDER BY 1, 2;', function, condition, grouping) FROM unnest(array[ 'first(time, quantity), last(time, quantity)', 'last(longvalue, quantity)', 'last(quantity, longvalue)', 'last(quantity, time)', 'last(time, longvalue)']) AS function, unnest(array[ 'true', $$time < '2021-01-01'$$, 'quantity is null', 'quantity is not null', 'quantity >= 4']) AS condition, unnest(array[ '777::text' /* dummy grouping column */, 'longvalue', 'quantity', $$time_bucket('1 year', time)$$, $$time_bucket('3 year', time)$$]) AS grouping \gexec SELECT 777::text, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE true GROUP BY 777::text ORDER BY 1, 2; text | first | last ------+------------------------------+------------------------------ 777 | Sun Jan 20 09:00:43 2019 PST | Fri Jan 20 09:00:44 2023 PST SELECT 777::text, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY 777::text ORDER BY 1, 2; text | first | last ------+------------------------------+------------------------------ 777 | Sun Jan 20 09:00:43 2019 PST | Mon Jan 20 09:00:44 2020 PST SELECT 777::text, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY 777::text ORDER BY 1, 2; text | first | last ------+-------+------ 777 | | SELECT 777::text, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY 777::text ORDER BY 1, 2; text | first | last ------+------------------------------+------------------------------ 777 | Sun Jan 20 09:00:43 2019 PST | Fri Jan 20 09:00:44 2023 PST SELECT 777::text, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY 777::text ORDER BY 1, 2; text | first | last ------+------------------------------+------------------------------ 777 | Thu Jan 20 09:00:43 2022 PST | Fri Jan 20 09:00:44 2023 PST SELECT 777::text, last(longvalue, quantity) FROM partial_aggregation WHERE true GROUP BY 777::text ORDER BY 1, 2; text | last ------+------- 777 | word4 SELECT 777::text, last(longvalue, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY 777::text ORDER BY 1, 2; text | last ------+------- 777 | more1 SELECT 777::text, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | SELECT 777::text, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------- 777 | word4 SELECT 777::text, last(longvalue, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY 777::text ORDER BY 1, 2; text | last ------+------- 777 | word4 SELECT 777::text, last(quantity, longvalue) FROM partial_aggregation WHERE true GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 2 SELECT 777::text, last(quantity, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 2 SELECT 777::text, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | SELECT 777::text, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 2 SELECT 777::text, last(quantity, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 7 SELECT 777::text, last(quantity, time) FROM partial_aggregation WHERE true GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 7 SELECT 777::text, last(quantity, time) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 3.2 SELECT 777::text, last(quantity, time) FROM partial_aggregation WHERE quantity is null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | SELECT 777::text, last(quantity, time) FROM partial_aggregation WHERE quantity is not null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 7 SELECT 777::text, last(quantity, time) FROM partial_aggregation WHERE quantity >= 4 GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 7 SELECT 777::text, last(time, longvalue) FROM partial_aggregation WHERE true GROUP BY 777::text ORDER BY 1, 2; text | last ------+------------------------------ 777 | Sun Jan 20 09:00:44 2019 PST SELECT 777::text, last(time, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY 777::text ORDER BY 1, 2; text | last ------+------------------------------ 777 | Sun Jan 20 09:00:44 2019 PST SELECT 777::text, last(time, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | SELECT 777::text, last(time, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------------------------------ 777 | Sun Jan 20 09:00:44 2019 PST SELECT 777::text, last(time, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY 777::text ORDER BY 1, 2; text | last ------+------------------------------ 777 | Fri Jan 20 09:00:44 2023 PST SELECT longvalue, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE true GROUP BY longvalue ORDER BY 1, 2; longvalue | first | last -----------+------------------------------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST more2 | Wed Jan 20 09:00:44 2021 PST | Wed Jan 20 09:00:44 2021 PST some1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST some2 | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:43 2021 PST word1 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST world | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST | | SELECT longvalue, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY longvalue ORDER BY 1, 2; longvalue | first | last -----------+------------------------------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST some1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST world | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST | | SELECT longvalue, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY longvalue ORDER BY 1, 2; longvalue | first | last -----------+-------+------ | | SELECT longvalue, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY longvalue ORDER BY 1, 2; longvalue | first | last -----------+------------------------------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST more2 | Wed Jan 20 09:00:44 2021 PST | Wed Jan 20 09:00:44 2021 PST some1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST some2 | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:43 2021 PST word1 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST world | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST SELECT longvalue, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY longvalue ORDER BY 1, 2; longvalue | first | last -----------+------------------------------+------------------------------ word1 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT longvalue, last(longvalue, quantity) FROM partial_aggregation WHERE true GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------- hello | hello more1 | more1 more2 | more2 some1 | some1 some2 | some2 word1 | word1 word2 | word2 word3 | word3 word4 | word4 world | world | SELECT longvalue, last(longvalue, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------- hello | hello more1 | more1 some1 | some1 world | world | SELECT longvalue, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ | SELECT longvalue, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------- hello | hello more1 | more1 more2 | more2 some1 | some1 some2 | some2 word1 | word1 word2 | word2 word3 | word3 word4 | word4 world | world SELECT longvalue, last(longvalue, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------- word1 | word1 word2 | word2 word3 | word3 word4 | word4 SELECT longvalue, last(quantity, longvalue) FROM partial_aggregation WHERE true GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 more2 | 3.4 some1 | 3.1 some2 | 3.3 word1 | 4 word2 | 5 word3 | 6 word4 | 7 world | 2 | SELECT longvalue, last(quantity, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 some1 | 3.1 world | 2 | SELECT longvalue, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ | SELECT longvalue, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 more2 | 3.4 some1 | 3.1 some2 | 3.3 word1 | 4 word2 | 5 word3 | 6 word4 | 7 world | 2 SELECT longvalue, last(quantity, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ word1 | 4 word2 | 5 word3 | 6 word4 | 7 SELECT longvalue, last(quantity, time) FROM partial_aggregation WHERE true GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 more2 | 3.4 some1 | 3.1 some2 | 3.3 word1 | 4 word2 | 5 word3 | 6 word4 | 7 world | 2 | SELECT longvalue, last(quantity, time) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 some1 | 3.1 world | 2 | SELECT longvalue, last(quantity, time) FROM partial_aggregation WHERE quantity is null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ | SELECT longvalue, last(quantity, time) FROM partial_aggregation WHERE quantity is not null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 more2 | 3.4 some1 | 3.1 some2 | 3.3 word1 | 4 word2 | 5 word3 | 6 word4 | 7 world | 2 SELECT longvalue, last(quantity, time) FROM partial_aggregation WHERE quantity >= 4 GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ word1 | 4 word2 | 5 word3 | 6 word4 | 7 SELECT longvalue, last(time, longvalue) FROM partial_aggregation WHERE true GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST more2 | Wed Jan 20 09:00:44 2021 PST some1 | Mon Jan 20 09:00:43 2020 PST some2 | Wed Jan 20 09:00:43 2021 PST word1 | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST world | Sun Jan 20 09:00:44 2019 PST | SELECT longvalue, last(time, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST some1 | Mon Jan 20 09:00:43 2020 PST world | Sun Jan 20 09:00:44 2019 PST | SELECT longvalue, last(time, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ | SELECT longvalue, last(time, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST more2 | Wed Jan 20 09:00:44 2021 PST some1 | Mon Jan 20 09:00:43 2020 PST some2 | Wed Jan 20 09:00:43 2021 PST word1 | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST world | Sun Jan 20 09:00:44 2019 PST SELECT longvalue, last(time, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------------------------------ word1 | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST SELECT quantity, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE true GROUP BY quantity ORDER BY 1, 2; quantity | first | last ----------+------------------------------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST 3.3 | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:43 2021 PST 3.4 | Wed Jan 20 09:00:44 2021 PST | Wed Jan 20 09:00:44 2021 PST 4 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST | | SELECT quantity, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY quantity ORDER BY 1, 2; quantity | first | last ----------+------------------------------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST | | SELECT quantity, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY quantity ORDER BY 1, 2; quantity | first | last ----------+-------+------ | | SELECT quantity, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY quantity ORDER BY 1, 2; quantity | first | last ----------+------------------------------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST 3.3 | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:43 2021 PST 3.4 | Wed Jan 20 09:00:44 2021 PST | Wed Jan 20 09:00:44 2021 PST 4 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT quantity, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY quantity ORDER BY 1, 2; quantity | first | last ----------+------------------------------+------------------------------ 4 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT quantity, last(longvalue, quantity) FROM partial_aggregation WHERE true GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------- 1 | hello 2 | world 3.1 | some1 3.2 | more1 3.3 | some2 3.4 | more2 4 | word1 5 | word2 6 | word3 7 | word4 | SELECT quantity, last(longvalue, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------- 1 | hello 2 | world 3.1 | some1 3.2 | more1 | SELECT quantity, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ | SELECT quantity, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------- 1 | hello 2 | world 3.1 | some1 3.2 | more1 3.3 | some2 3.4 | more2 4 | word1 5 | word2 6 | word3 7 | word4 SELECT quantity, last(longvalue, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------- 4 | word1 5 | word2 6 | word3 7 | word4 SELECT quantity, last(quantity, longvalue) FROM partial_aggregation WHERE true GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 3.3 | 3.3 3.4 | 3.4 4 | 4 5 | 5 6 | 6 7 | 7 | SELECT quantity, last(quantity, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 | SELECT quantity, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ | SELECT quantity, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 3.3 | 3.3 3.4 | 3.4 4 | 4 5 | 5 6 | 6 7 | 7 SELECT quantity, last(quantity, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 4 | 4 5 | 5 6 | 6 7 | 7 SELECT quantity, last(quantity, time) FROM partial_aggregation WHERE true GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 3.3 | 3.3 3.4 | 3.4 4 | 4 5 | 5 6 | 6 7 | 7 | SELECT quantity, last(quantity, time) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 | SELECT quantity, last(quantity, time) FROM partial_aggregation WHERE quantity is null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ | SELECT quantity, last(quantity, time) FROM partial_aggregation WHERE quantity is not null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 3.3 | 3.3 3.4 | 3.4 4 | 4 5 | 5 6 | 6 7 | 7 SELECT quantity, last(quantity, time) FROM partial_aggregation WHERE quantity >= 4 GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 4 | 4 5 | 5 6 | 6 7 | 7 SELECT quantity, last(time, longvalue) FROM partial_aggregation WHERE true GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST 3.3 | Wed Jan 20 09:00:43 2021 PST 3.4 | Wed Jan 20 09:00:44 2021 PST 4 | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST | SELECT quantity, last(time, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST | SELECT quantity, last(time, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ | SELECT quantity, last(time, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST 3.3 | Wed Jan 20 09:00:43 2021 PST 3.4 | Wed Jan 20 09:00:44 2021 PST 4 | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST SELECT quantity, last(time, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------------------------------ 4 | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE true GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | | Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:44 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:44 2021 PST Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | | Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:44 2020 PST SELECT time_bucket('1 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+-------+------ Sun Dec 31 16:00:00 2017 PST | | SELECT time_bucket('1 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:44 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:44 2021 PST Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE true GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | world Tue Dec 31 16:00:00 2019 PST | more1 Thu Dec 31 16:00:00 2020 PST | more2 Fri Dec 31 16:00:00 2021 PST | word2 Sat Dec 31 16:00:00 2022 PST | word4 SELECT time_bucket('1 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | world Tue Dec 31 16:00:00 2019 PST | more1 SELECT time_bucket('1 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('1 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Mon Dec 31 16:00:00 2018 PST | world Tue Dec 31 16:00:00 2019 PST | more1 Thu Dec 31 16:00:00 2020 PST | more2 Fri Dec 31 16:00:00 2021 PST | word2 Sat Dec 31 16:00:00 2022 PST | word4 SELECT time_bucket('1 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Fri Dec 31 16:00:00 2021 PST | word2 Sat Dec 31 16:00:00 2022 PST | word4 SELECT time_bucket('1 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE true GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.1 Thu Dec 31 16:00:00 2020 PST | 3.3 Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.1 SELECT time_bucket('1 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('1 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.1 Thu Dec 31 16:00:00 2020 PST | 3.3 Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(quantity, time) FROM partial_aggregation WHERE true GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.2 Thu Dec 31 16:00:00 2020 PST | 3.4 Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(quantity, time) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.2 SELECT time_bucket('1 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('1 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.2 Thu Dec 31 16:00:00 2020 PST | 3.4 Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(time, longvalue) FROM partial_aggregation WHERE true GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), last(time, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST SELECT time_bucket('1 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('1 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE true GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:43 2019 PST | Mon Jan 20 09:00:44 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:43 2019 PST | Mon Jan 20 09:00:44 2020 PST SELECT time_bucket('3 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+-------+------ Sun Dec 31 16:00:00 2017 PST | | SELECT time_bucket('3 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:43 2019 PST | Mon Jan 20 09:00:44 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Thu Dec 31 16:00:00 2020 PST | Thu Jan 20 09:00:43 2022 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE true GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Sun Dec 31 16:00:00 2017 PST | more1 Thu Dec 31 16:00:00 2020 PST | word4 SELECT time_bucket('3 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Sun Dec 31 16:00:00 2017 PST | more1 SELECT time_bucket('3 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('3 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Sun Dec 31 16:00:00 2017 PST | more1 Thu Dec 31 16:00:00 2020 PST | word4 SELECT time_bucket('3 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Thu Dec 31 16:00:00 2020 PST | word4 SELECT time_bucket('3 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE true GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 2 Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 2 SELECT time_bucket('3 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('3 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 2 Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(quantity, time) FROM partial_aggregation WHERE true GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 3.2 Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(quantity, time) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 3.2 SELECT time_bucket('3 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('3 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 3.2 Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(time, longvalue) FROM partial_aggregation WHERE true GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:44 2019 PST Thu Dec 31 16:00:00 2020 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), last(time, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:44 2019 PST SELECT time_bucket('3 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('3 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:44 2019 PST Thu Dec 31 16:00:00 2020 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Thu Dec 31 16:00:00 2020 PST | Fri Jan 20 09:00:44 2023 PST SET enable_partitionwise_aggregate = OFF; ================================================ FILE: test/expected/agg_bookends-18.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set TEST_BASE_NAME agg_bookends SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') as "TEST_LOAD_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') as "TEST_QUERY_NAME", format('%s/results/%s_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_OPTIMIZED", format('%s/results/%s_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_UNOPTIMIZED" \gset SELECT format('\! diff -u --label "Unoptimized result" --label "Optimized result" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') as "DIFF_CMD" \gset \set PREFIX 'EXPLAIN (analyze, buffers off, costs off, timing off, summary off)' \ir :TEST_LOAD_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE btest(time timestamp NOT NULL, time_alt timestamp, gp INTEGER, temp float, strid TEXT DEFAULT 'testing'); SELECT schema_name, table_name, created FROM create_hypertable('btest', 'time'); psql:include/agg_bookends_load.sql:6: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices psql:include/agg_bookends_load.sql:6: WARNING: column type "timestamp without time zone" used for "time_alt" does not follow best practices schema_name | table_name | created -------------+------------+--------- public | btest | t INSERT INTO btest VALUES('2017-01-20T09:00:01', '2017-01-20T10:00:00', 1, 22.5); INSERT INTO btest VALUES('2017-01-20T09:00:21', '2017-01-20T09:00:59', 1, 21.2); INSERT INTO btest VALUES('2017-01-20T09:00:47', '2017-01-20T09:00:58', 1, 25.1); INSERT INTO btest VALUES('2017-01-20T09:00:02', '2017-01-20T09:00:57', 2, 35.5); INSERT INTO btest VALUES('2017-01-20T09:00:21', '2017-01-20T09:00:56', 2, 30.2); --TOASTED; INSERT INTO btest VALUES('2017-01-20T09:00:43', '2017-01-20T09:01:55', 2, 20.1, repeat('xyz', 1000000) ); CREATE TABLE btest_numeric (time timestamp NOT NULL, quantity numeric); SELECT schema_name, table_name, created FROM create_hypertable('btest_numeric', 'time'); psql:include/agg_bookends_load.sql:16: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices schema_name | table_name | created -------------+---------------+--------- public | btest_numeric | t \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be only output of results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations')) v(setting); setting | value ----------------------------------+------- timescaledb.enable_optimizations | on :PREFIX SELECT time, gp, temp FROM btest ORDER BY time; --- QUERY PLAN --- Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) :PREFIX SELECT first(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) :PREFIX SELECT last(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT first(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT gp, last(temp, time) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT gp, first(temp, time) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) --check whole row :PREFIX SELECT gp, first(btest, time) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) --check toasted col :PREFIX SELECT gp, left(last(strid, time), 10) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT gp, last(temp, strid) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) :PREFIX SELECT gp, last(strid, temp) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: _hyper_1_1_chunk.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: _hyper_1_1_chunk.gp -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) BEGIN; --check null value as last element INSERT INTO btest VALUES('2018-01-20T09:00:43', '2017-01-20T09:00:55', 2, NULL); :PREFIX SELECT last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) --check non-null element "overrides" NULL because it comes after. INSERT INTO btest VALUES('2019-01-20T09:00:43', '2018-01-20T09:00:55', 2, 30.5); :PREFIX SELECT last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) --check null cmp element is skipped INSERT INTO btest VALUES('2018-01-20T09:00:43', NULL, 2, 32.3); :PREFIX SELECT last(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=9.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -- fist returns NULL value :PREFIX SELECT first(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=9.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -- test first return non NULL value INSERT INTO btest VALUES('2016-01-20T09:00:00', '2016-01-20T09:00:00', 2, 36.5); :PREFIX SELECT first(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=10.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) --check non null cmp element insert after null cmp INSERT INTO btest VALUES('2020-01-20T09:00:43', '2020-01-20T09:00:43', 2, 35.3); :PREFIX SELECT last(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) :PREFIX SELECT first(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) --cmp nulls should be ignored and not present in groups :PREFIX SELECT gp, last(temp, time_alt) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: btest.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: btest.gp -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) --Previously, some bugs were found with NULLS and numeric types, so test that INSERT INTO btest_numeric VALUES ('2019-01-20T09:00:43', NULL); :PREFIX SELECT last(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Index Scan using _hyper_2_6_chunk_btest_numeric_time_idx on _hyper_2_6_chunk (actual rows=1.00 loops=1) --check non-null element "overrides" NULL because it comes after. INSERT INTO btest_numeric VALUES('2020-01-20T09:00:43', 30.5); :PREFIX SELECT last(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest_numeric (actual rows=1.00 loops=1) Order: btest_numeric."time" DESC -> Index Scan using _hyper_2_7_chunk_btest_numeric_time_idx on _hyper_2_7_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_2_6_chunk_btest_numeric_time_idx on _hyper_2_6_chunk (never executed) -- do index scan for last :PREFIX SELECT last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) -- do index scan for first :PREFIX SELECT first(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" -> Index Scan Backward using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) -> Index Scan Backward using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan Backward using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) -> Index Scan Backward using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (never executed) -- can't do index scan when ordering on non-index column :PREFIX SELECT first(temp, time_alt) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) -- do index scan for subquery :PREFIX SELECT * FROM (SELECT last(temp, time) FROM btest) last; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) -- can't do index scan when using group by :PREFIX SELECT last(temp, time) FROM btest GROUP BY gp ORDER BY gp; --- QUERY PLAN --- Sort (actual rows=2.00 loops=1) Sort Key: btest.gp Sort Method: quicksort -> HashAggregate (actual rows=2.00 loops=1) Group Key: btest.gp -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) -- do index scan when agg function is used in CTE subquery :PREFIX WITH last_temp AS (SELECT last(temp, time) FROM btest) SELECT * from last_temp; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) -- do index scan when using both FIRST and LAST aggregate functions :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) InitPlan 2 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest btest_1 (actual rows=1.00 loops=1) Order: btest_1."time" -> Index Scan Backward using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk _hyper_1_4_chunk_1 (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk _hyper_1_3_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk _hyper_1_5_chunk_1 (never executed) -- verify results when using both FIRST and LAST :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) InitPlan 2 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest btest_1 (actual rows=1.00 loops=1) Order: btest_1."time" -> Index Scan Backward using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk _hyper_1_4_chunk_1 (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk _hyper_1_3_chunk_1 (never executed) -> Index Scan Backward using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk _hyper_1_5_chunk_1 (never executed) -- do index scan when using WHERE :PREFIX SELECT last(temp, time) FROM btest WHERE time <= '2017-01-20T09:00:02'; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) Index Cond: ("time" <= 'Fri Jan 20 09:00:02 2017'::timestamp without time zone) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) -- can't do index scan for MAX and LAST combined (MinMax optimization fails when having different aggregate functions) :PREFIX SELECT max(time), last(temp, time) FROM btest; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) -- can't do index scan when using FIRST/LAST in ORDER BY :PREFIX SELECT last(temp, time) FROM btest ORDER BY last(temp, time); --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: (last(btest.temp, btest."time")) Sort Method: quicksort -> Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) -- do index scan :PREFIX SELECT last(temp, time) FROM btest WHERE temp < 30; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=0.00 loops=1) Filter: (temp < '30'::double precision) Rows Removed by Filter: 1 -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (actual rows=0.00 loops=1) Filter: (temp < '30'::double precision) Rows Removed by Filter: 1 -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (actual rows=0.00 loops=1) Filter: (temp < '30'::double precision) Rows Removed by Filter: 2 -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) Filter: (temp < '30'::double precision) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) Filter: (temp < '30'::double precision) -- SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- do index scan :PREFIX SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" -> Index Scan Backward using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Fri Jan 20 09:00:47 2017'::timestamp without time zone) -> Index Scan Backward using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan Backward using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) -> Index Scan Backward using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (never executed) -- can't do index scan when using WINDOW function :PREFIX SELECT gp, last(temp, time) OVER (PARTITION BY gp) AS last FROM btest; --- QUERY PLAN --- WindowAgg (actual rows=11.00 loops=1) -> Sort (actual rows=11.00 loops=1) Sort Key: btest.gp Sort Method: quicksort -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) -- test constants :PREFIX SELECT first(100, 100) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Result (actual rows=1.00 loops=1) -> Append (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (never executed) -> Seq Scan on _hyper_1_3_chunk (never executed) -> Seq Scan on _hyper_1_4_chunk (never executed) -> Seq Scan on _hyper_1_5_chunk (never executed) -- create an index so we can test optimization CREATE INDEX btest_time_alt_idx ON btest(time_alt); :PREFIX SELECT last(temp, time_alt) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Merge Append (actual rows=1.00 loops=1) Sort Key: btest.time_alt DESC -> Index Scan Backward using _hyper_1_1_chunk_btest_time_alt_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) Index Cond: (time_alt IS NOT NULL) -> Index Scan Backward using _hyper_1_2_chunk_btest_time_alt_idx on _hyper_1_2_chunk (actual rows=1.00 loops=1) Index Cond: (time_alt IS NOT NULL) -> Index Scan Backward using _hyper_1_3_chunk_btest_time_alt_idx on _hyper_1_3_chunk (actual rows=1.00 loops=1) Index Cond: (time_alt IS NOT NULL) -> Index Scan Backward using _hyper_1_4_chunk_btest_time_alt_idx on _hyper_1_4_chunk (actual rows=1.00 loops=1) Index Cond: (time_alt IS NOT NULL) -> Index Scan Backward using _hyper_1_5_chunk_btest_time_alt_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) Index Cond: (time_alt IS NOT NULL) --test nested FIRST/LAST - should optimize :PREFIX SELECT abs(last(temp, time)) FROM btest; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest (actual rows=1.00 loops=1) Order: btest."time" DESC -> Index Scan using _hyper_1_5_chunk_btest_time_idx on _hyper_1_5_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_3_chunk_btest_time_idx on _hyper_1_3_chunk (never executed) -> Index Scan using _hyper_1_2_chunk_btest_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan using _hyper_1_1_chunk_btest_time_idx on _hyper_1_1_chunk (never executed) -> Index Scan using _hyper_1_4_chunk_btest_time_idx on _hyper_1_4_chunk (never executed) -- test nested FIRST/LAST in ORDER BY - no optimization possible :PREFIX SELECT abs(last(temp, time)) FROM btest ORDER BY abs(last(temp,time)); --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: (abs(last(btest.temp, btest."time"))) Sort Method: quicksort -> Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=11.00 loops=1) -> Seq Scan on _hyper_1_1_chunk (actual rows=6.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_4_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_5_chunk (actual rows=1.00 loops=1) ROLLBACK; -- Test with NULL numeric values BEGIN; TRUNCATE btest_numeric; -- Empty table :PREFIX SELECT first(btest_numeric, time) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Result (actual rows=0.00 loops=1) One-Time Filter: false :PREFIX SELECT last(btest_numeric, time) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Result (actual rows=0.00 loops=1) One-Time Filter: false -- Only NULL values INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_2_8_chunk_btest_numeric_time_idx on _hyper_2_8_chunk (actual rows=1.00 loops=1) :PREFIX SELECT last(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Index Scan using _hyper_2_8_chunk_btest_numeric_time_idx on _hyper_2_8_chunk (actual rows=1.00 loops=1) :PREFIX SELECT first(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Seq Scan on _hyper_2_8_chunk (actual rows=2.00 loops=1) :PREFIX SELECT last(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Seq Scan on _hyper_2_8_chunk (actual rows=2.00 loops=1) -- NULL values followed by non-NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); :PREFIX SELECT first(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest_numeric (actual rows=1.00 loops=1) Order: btest_numeric."time" -> Index Scan Backward using _hyper_2_8_chunk_btest_numeric_time_idx on _hyper_2_8_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_2_9_chunk_btest_numeric_time_idx on _hyper_2_9_chunk (never executed) :PREFIX SELECT last(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest_numeric (actual rows=1.00 loops=1) Order: btest_numeric."time" DESC -> Index Scan using _hyper_2_9_chunk_btest_numeric_time_idx on _hyper_2_9_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_2_8_chunk_btest_numeric_time_idx on _hyper_2_8_chunk (never executed) :PREFIX SELECT first(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_2_8_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_2_9_chunk (actual rows=2.00 loops=1) :PREFIX SELECT last(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_2_8_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_2_9_chunk (actual rows=2.00 loops=1) TRUNCATE btest_numeric; -- non-NULL values followed by NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest_numeric (actual rows=1.00 loops=1) Order: btest_numeric."time" -> Index Scan Backward using _hyper_2_11_chunk_btest_numeric_time_idx on _hyper_2_11_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_2_10_chunk_btest_numeric_time_idx on _hyper_2_10_chunk (never executed) :PREFIX SELECT last(quantity, time) FROM btest_numeric; --- QUERY PLAN --- Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on btest_numeric (actual rows=1.00 loops=1) Order: btest_numeric."time" DESC -> Index Scan using _hyper_2_10_chunk_btest_numeric_time_idx on _hyper_2_10_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_2_11_chunk_btest_numeric_time_idx on _hyper_2_11_chunk (never executed) :PREFIX SELECT first(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_2_10_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_2_11_chunk (actual rows=2.00 loops=1) :PREFIX SELECT last(time, quantity) FROM btest_numeric; --- QUERY PLAN --- Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_2_10_chunk (actual rows=2.00 loops=1) -> Seq Scan on _hyper_2_11_chunk (actual rows=2.00 loops=1) ROLLBACK; -- we want test results as part of the output too to make sure we produce correct output \set PREFIX '' \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be only output of results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations')) v(setting); setting | value ----------------------------------+------- timescaledb.enable_optimizations | on :PREFIX SELECT time, gp, temp FROM btest ORDER BY time; time | gp | temp --------------------------+----+------ Fri Jan 20 09:00:01 2017 | 1 | 22.5 Fri Jan 20 09:00:02 2017 | 2 | 35.5 Fri Jan 20 09:00:21 2017 | 1 | 21.2 Fri Jan 20 09:00:21 2017 | 2 | 30.2 Fri Jan 20 09:00:43 2017 | 2 | 20.1 Fri Jan 20 09:00:47 2017 | 1 | 25.1 :PREFIX SELECT last(temp, time) FROM btest; last ------ 25.1 :PREFIX SELECT first(temp, time) FROM btest; first ------- 22.5 :PREFIX SELECT last(temp, time_alt) FROM btest; last ------ 22.5 :PREFIX SELECT first(temp, time_alt) FROM btest; first ------- 30.2 :PREFIX SELECT gp, last(temp, time) FROM btest GROUP BY gp ORDER BY gp; gp | last ----+------ 1 | 25.1 2 | 20.1 :PREFIX SELECT gp, first(temp, time) FROM btest GROUP BY gp ORDER BY gp; gp | first ----+------- 1 | 22.5 2 | 35.5 --check whole row :PREFIX SELECT gp, first(btest, time) FROM btest GROUP BY gp ORDER BY gp; gp | first ----+------------------------------------------------------------------------ 1 | ("Fri Jan 20 09:00:01 2017","Fri Jan 20 10:00:00 2017",1,22.5,testing) 2 | ("Fri Jan 20 09:00:02 2017","Fri Jan 20 09:00:57 2017",2,35.5,testing) --check toasted col :PREFIX SELECT gp, left(last(strid, time), 10) FROM btest GROUP BY gp ORDER BY gp; gp | left ----+------------ 1 | testing 2 | xyzxyzxyzx :PREFIX SELECT gp, last(temp, strid) FROM btest GROUP BY gp ORDER BY gp; gp | last ----+------ 1 | 22.5 2 | 20.1 :PREFIX SELECT gp, last(strid, temp) FROM btest GROUP BY gp ORDER BY gp; gp | last ----+--------- 1 | testing 2 | testing BEGIN; --check null value as last element INSERT INTO btest VALUES('2018-01-20T09:00:43', '2017-01-20T09:00:55', 2, NULL); :PREFIX SELECT last(temp, time) FROM btest; last ------ --check non-null element "overrides" NULL because it comes after. INSERT INTO btest VALUES('2019-01-20T09:00:43', '2018-01-20T09:00:55', 2, 30.5); :PREFIX SELECT last(temp, time) FROM btest; last ------ 30.5 --check null cmp element is skipped INSERT INTO btest VALUES('2018-01-20T09:00:43', NULL, 2, 32.3); :PREFIX SELECT last(temp, time_alt) FROM btest; last ------ 30.5 -- fist returns NULL value :PREFIX SELECT first(temp, time_alt) FROM btest; first ------- -- test first return non NULL value INSERT INTO btest VALUES('2016-01-20T09:00:00', '2016-01-20T09:00:00', 2, 36.5); :PREFIX SELECT first(temp, time_alt) FROM btest; first ------- 36.5 --check non null cmp element insert after null cmp INSERT INTO btest VALUES('2020-01-20T09:00:43', '2020-01-20T09:00:43', 2, 35.3); :PREFIX SELECT last(temp, time_alt) FROM btest; last ------ 35.3 :PREFIX SELECT first(temp, time_alt) FROM btest; first ------- 36.5 --cmp nulls should be ignored and not present in groups :PREFIX SELECT gp, last(temp, time_alt) FROM btest GROUP BY gp ORDER BY gp; gp | last ----+------ 1 | 22.5 2 | 35.3 --Previously, some bugs were found with NULLS and numeric types, so test that INSERT INTO btest_numeric VALUES ('2019-01-20T09:00:43', NULL); :PREFIX SELECT last(quantity, time) FROM btest_numeric; last ------ --check non-null element "overrides" NULL because it comes after. INSERT INTO btest_numeric VALUES('2020-01-20T09:00:43', 30.5); :PREFIX SELECT last(quantity, time) FROM btest_numeric; last ------ 30.5 -- do index scan for last :PREFIX SELECT last(temp, time) FROM btest; last ------ 35.3 -- do index scan for first :PREFIX SELECT first(temp, time) FROM btest; first ------- 36.5 -- can't do index scan when ordering on non-index column :PREFIX SELECT first(temp, time_alt) FROM btest; first ------- 36.5 -- do index scan for subquery :PREFIX SELECT * FROM (SELECT last(temp, time) FROM btest) last; last ------ 35.3 -- can't do index scan when using group by :PREFIX SELECT last(temp, time) FROM btest GROUP BY gp ORDER BY gp; last ------ 25.1 35.3 -- do index scan when agg function is used in CTE subquery :PREFIX WITH last_temp AS (SELECT last(temp, time) FROM btest) SELECT * from last_temp; last ------ 35.3 -- do index scan when using both FIRST and LAST aggregate functions :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; first | last -------+------ 36.5 | 35.3 -- verify results when using both FIRST and LAST :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; first | last -------+------ 36.5 | 35.3 -- do index scan when using WHERE :PREFIX SELECT last(temp, time) FROM btest WHERE time <= '2017-01-20T09:00:02'; last ------ 35.5 -- can't do index scan for MAX and LAST combined (MinMax optimization fails when having different aggregate functions) :PREFIX SELECT max(time), last(temp, time) FROM btest; max | last --------------------------+------ Mon Jan 20 09:00:43 2020 | 35.3 -- can't do index scan when using FIRST/LAST in ORDER BY :PREFIX SELECT last(temp, time) FROM btest ORDER BY last(temp, time); last ------ 35.3 -- do index scan :PREFIX SELECT last(temp, time) FROM btest WHERE temp < 30; last ------ 25.1 -- SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- do index scan :PREFIX SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; first ------- 25.1 -- can't do index scan when using WINDOW function :PREFIX SELECT gp, last(temp, time) OVER (PARTITION BY gp) AS last FROM btest; gp | last ----+------ 1 | 25.1 1 | 25.1 1 | 25.1 2 | 35.3 2 | 35.3 2 | 35.3 2 | 35.3 2 | 35.3 2 | 35.3 2 | 35.3 2 | 35.3 -- test constants :PREFIX SELECT first(100, 100) FROM btest; first ------- 100 -- create an index so we can test optimization CREATE INDEX btest_time_alt_idx ON btest(time_alt); :PREFIX SELECT last(temp, time_alt) FROM btest; last ------ 35.3 --test nested FIRST/LAST - should optimize :PREFIX SELECT abs(last(temp, time)) FROM btest; abs ------ 35.3 -- test nested FIRST/LAST in ORDER BY - no optimization possible :PREFIX SELECT abs(last(temp, time)) FROM btest ORDER BY abs(last(temp,time)); abs ------ 35.3 ROLLBACK; -- Test with NULL numeric values BEGIN; TRUNCATE btest_numeric; -- Empty table :PREFIX SELECT first(btest_numeric, time) FROM btest_numeric; first ------- :PREFIX SELECT last(btest_numeric, time) FROM btest_numeric; last ------ -- Only NULL values INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; first ------- :PREFIX SELECT last(quantity, time) FROM btest_numeric; last ------ :PREFIX SELECT first(time, quantity) FROM btest_numeric; first ------- :PREFIX SELECT last(time, quantity) FROM btest_numeric; last ------ -- NULL values followed by non-NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); :PREFIX SELECT first(quantity, time) FROM btest_numeric; first ------- :PREFIX SELECT last(quantity, time) FROM btest_numeric; last ------ 1 :PREFIX SELECT first(time, quantity) FROM btest_numeric; first -------------------------- Sun Jan 20 09:00:43 2019 :PREFIX SELECT last(time, quantity) FROM btest_numeric; last -------------------------- Sun Jan 20 09:00:43 2019 TRUNCATE btest_numeric; -- non-NULL values followed by NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; first ------- :PREFIX SELECT last(quantity, time) FROM btest_numeric; last ------ 1 :PREFIX SELECT first(time, quantity) FROM btest_numeric; first -------------------------- Sun Jan 20 09:00:43 2019 :PREFIX SELECT last(time, quantity) FROM btest_numeric; last -------------------------- Sun Jan 20 09:00:43 2019 ROLLBACK; -- diff results with optimizations disabled and enabled \o :TEST_RESULTS_UNOPTIMIZED SET timescaledb.enable_optimizations TO false; \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be only output of results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations')) v(setting); :PREFIX SELECT time, gp, temp FROM btest ORDER BY time; :PREFIX SELECT last(temp, time) FROM btest; :PREFIX SELECT first(temp, time) FROM btest; :PREFIX SELECT last(temp, time_alt) FROM btest; :PREFIX SELECT first(temp, time_alt) FROM btest; :PREFIX SELECT gp, last(temp, time) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, first(temp, time) FROM btest GROUP BY gp ORDER BY gp; --check whole row :PREFIX SELECT gp, first(btest, time) FROM btest GROUP BY gp ORDER BY gp; --check toasted col :PREFIX SELECT gp, left(last(strid, time), 10) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, last(temp, strid) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, last(strid, temp) FROM btest GROUP BY gp ORDER BY gp; BEGIN; --check null value as last element INSERT INTO btest VALUES('2018-01-20T09:00:43', '2017-01-20T09:00:55', 2, NULL); :PREFIX SELECT last(temp, time) FROM btest; --check non-null element "overrides" NULL because it comes after. INSERT INTO btest VALUES('2019-01-20T09:00:43', '2018-01-20T09:00:55', 2, 30.5); :PREFIX SELECT last(temp, time) FROM btest; --check null cmp element is skipped INSERT INTO btest VALUES('2018-01-20T09:00:43', NULL, 2, 32.3); :PREFIX SELECT last(temp, time_alt) FROM btest; -- fist returns NULL value :PREFIX SELECT first(temp, time_alt) FROM btest; -- test first return non NULL value INSERT INTO btest VALUES('2016-01-20T09:00:00', '2016-01-20T09:00:00', 2, 36.5); :PREFIX SELECT first(temp, time_alt) FROM btest; --check non null cmp element insert after null cmp INSERT INTO btest VALUES('2020-01-20T09:00:43', '2020-01-20T09:00:43', 2, 35.3); :PREFIX SELECT last(temp, time_alt) FROM btest; :PREFIX SELECT first(temp, time_alt) FROM btest; --cmp nulls should be ignored and not present in groups :PREFIX SELECT gp, last(temp, time_alt) FROM btest GROUP BY gp ORDER BY gp; --Previously, some bugs were found with NULLS and numeric types, so test that INSERT INTO btest_numeric VALUES ('2019-01-20T09:00:43', NULL); :PREFIX SELECT last(quantity, time) FROM btest_numeric; --check non-null element "overrides" NULL because it comes after. INSERT INTO btest_numeric VALUES('2020-01-20T09:00:43', 30.5); :PREFIX SELECT last(quantity, time) FROM btest_numeric; -- do index scan for last :PREFIX SELECT last(temp, time) FROM btest; -- do index scan for first :PREFIX SELECT first(temp, time) FROM btest; -- can't do index scan when ordering on non-index column :PREFIX SELECT first(temp, time_alt) FROM btest; -- do index scan for subquery :PREFIX SELECT * FROM (SELECT last(temp, time) FROM btest) last; -- can't do index scan when using group by :PREFIX SELECT last(temp, time) FROM btest GROUP BY gp ORDER BY gp; -- do index scan when agg function is used in CTE subquery :PREFIX WITH last_temp AS (SELECT last(temp, time) FROM btest) SELECT * from last_temp; -- do index scan when using both FIRST and LAST aggregate functions :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; -- verify results when using both FIRST and LAST :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; -- do index scan when using WHERE :PREFIX SELECT last(temp, time) FROM btest WHERE time <= '2017-01-20T09:00:02'; -- can't do index scan for MAX and LAST combined (MinMax optimization fails when having different aggregate functions) :PREFIX SELECT max(time), last(temp, time) FROM btest; -- can't do index scan when using FIRST/LAST in ORDER BY :PREFIX SELECT last(temp, time) FROM btest ORDER BY last(temp, time); -- do index scan :PREFIX SELECT last(temp, time) FROM btest WHERE temp < 30; -- SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- do index scan :PREFIX SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- can't do index scan when using WINDOW function :PREFIX SELECT gp, last(temp, time) OVER (PARTITION BY gp) AS last FROM btest; -- test constants :PREFIX SELECT first(100, 100) FROM btest; -- create an index so we can test optimization CREATE INDEX btest_time_alt_idx ON btest(time_alt); :PREFIX SELECT last(temp, time_alt) FROM btest; --test nested FIRST/LAST - should optimize :PREFIX SELECT abs(last(temp, time)) FROM btest; -- test nested FIRST/LAST in ORDER BY - no optimization possible :PREFIX SELECT abs(last(temp, time)) FROM btest ORDER BY abs(last(temp,time)); ROLLBACK; -- Test with NULL numeric values BEGIN; TRUNCATE btest_numeric; -- Empty table :PREFIX SELECT first(btest_numeric, time) FROM btest_numeric; :PREFIX SELECT last(btest_numeric, time) FROM btest_numeric; -- Only NULL values INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; -- NULL values followed by non-NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; TRUNCATE btest_numeric; -- non-NULL values followed by NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; ROLLBACK; \o \o :TEST_RESULTS_OPTIMIZED SET timescaledb.enable_optimizations TO true; \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be only output of results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations')) v(setting); :PREFIX SELECT time, gp, temp FROM btest ORDER BY time; :PREFIX SELECT last(temp, time) FROM btest; :PREFIX SELECT first(temp, time) FROM btest; :PREFIX SELECT last(temp, time_alt) FROM btest; :PREFIX SELECT first(temp, time_alt) FROM btest; :PREFIX SELECT gp, last(temp, time) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, first(temp, time) FROM btest GROUP BY gp ORDER BY gp; --check whole row :PREFIX SELECT gp, first(btest, time) FROM btest GROUP BY gp ORDER BY gp; --check toasted col :PREFIX SELECT gp, left(last(strid, time), 10) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, last(temp, strid) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, last(strid, temp) FROM btest GROUP BY gp ORDER BY gp; BEGIN; --check null value as last element INSERT INTO btest VALUES('2018-01-20T09:00:43', '2017-01-20T09:00:55', 2, NULL); :PREFIX SELECT last(temp, time) FROM btest; --check non-null element "overrides" NULL because it comes after. INSERT INTO btest VALUES('2019-01-20T09:00:43', '2018-01-20T09:00:55', 2, 30.5); :PREFIX SELECT last(temp, time) FROM btest; --check null cmp element is skipped INSERT INTO btest VALUES('2018-01-20T09:00:43', NULL, 2, 32.3); :PREFIX SELECT last(temp, time_alt) FROM btest; -- fist returns NULL value :PREFIX SELECT first(temp, time_alt) FROM btest; -- test first return non NULL value INSERT INTO btest VALUES('2016-01-20T09:00:00', '2016-01-20T09:00:00', 2, 36.5); :PREFIX SELECT first(temp, time_alt) FROM btest; --check non null cmp element insert after null cmp INSERT INTO btest VALUES('2020-01-20T09:00:43', '2020-01-20T09:00:43', 2, 35.3); :PREFIX SELECT last(temp, time_alt) FROM btest; :PREFIX SELECT first(temp, time_alt) FROM btest; --cmp nulls should be ignored and not present in groups :PREFIX SELECT gp, last(temp, time_alt) FROM btest GROUP BY gp ORDER BY gp; --Previously, some bugs were found with NULLS and numeric types, so test that INSERT INTO btest_numeric VALUES ('2019-01-20T09:00:43', NULL); :PREFIX SELECT last(quantity, time) FROM btest_numeric; --check non-null element "overrides" NULL because it comes after. INSERT INTO btest_numeric VALUES('2020-01-20T09:00:43', 30.5); :PREFIX SELECT last(quantity, time) FROM btest_numeric; -- do index scan for last :PREFIX SELECT last(temp, time) FROM btest; -- do index scan for first :PREFIX SELECT first(temp, time) FROM btest; -- can't do index scan when ordering on non-index column :PREFIX SELECT first(temp, time_alt) FROM btest; -- do index scan for subquery :PREFIX SELECT * FROM (SELECT last(temp, time) FROM btest) last; -- can't do index scan when using group by :PREFIX SELECT last(temp, time) FROM btest GROUP BY gp ORDER BY gp; -- do index scan when agg function is used in CTE subquery :PREFIX WITH last_temp AS (SELECT last(temp, time) FROM btest) SELECT * from last_temp; -- do index scan when using both FIRST and LAST aggregate functions :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; -- verify results when using both FIRST and LAST :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; -- do index scan when using WHERE :PREFIX SELECT last(temp, time) FROM btest WHERE time <= '2017-01-20T09:00:02'; -- can't do index scan for MAX and LAST combined (MinMax optimization fails when having different aggregate functions) :PREFIX SELECT max(time), last(temp, time) FROM btest; -- can't do index scan when using FIRST/LAST in ORDER BY :PREFIX SELECT last(temp, time) FROM btest ORDER BY last(temp, time); -- do index scan :PREFIX SELECT last(temp, time) FROM btest WHERE temp < 30; -- SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- do index scan :PREFIX SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- can't do index scan when using WINDOW function :PREFIX SELECT gp, last(temp, time) OVER (PARTITION BY gp) AS last FROM btest; -- test constants :PREFIX SELECT first(100, 100) FROM btest; -- create an index so we can test optimization CREATE INDEX btest_time_alt_idx ON btest(time_alt); :PREFIX SELECT last(temp, time_alt) FROM btest; --test nested FIRST/LAST - should optimize :PREFIX SELECT abs(last(temp, time)) FROM btest; -- test nested FIRST/LAST in ORDER BY - no optimization possible :PREFIX SELECT abs(last(temp, time)) FROM btest ORDER BY abs(last(temp,time)); ROLLBACK; -- Test with NULL numeric values BEGIN; TRUNCATE btest_numeric; -- Empty table :PREFIX SELECT first(btest_numeric, time) FROM btest_numeric; :PREFIX SELECT last(btest_numeric, time) FROM btest_numeric; -- Only NULL values INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; -- NULL values followed by non-NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; TRUNCATE btest_numeric; -- non-NULL values followed by NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; ROLLBACK; \o :DIFF_CMD --- Unoptimized result +++ Optimized result @@ -1,6 +1,6 @@ setting | value ----------------------------------+------- - timescaledb.enable_optimizations | off + timescaledb.enable_optimizations | on time | gp | temp -- Test partial aggregation CREATE TABLE partial_aggregation (time timestamptz NOT NULL, quantity numeric, longvalue text); SELECT schema_name, table_name, created FROM create_hypertable('partial_aggregation', 'time'); schema_name | table_name | created -------------+---------------------+--------- public | partial_aggregation | t INSERT INTO partial_aggregation VALUES('2018-01-20T09:00:43', NULL, NULL); INSERT INTO partial_aggregation VALUES('2018-01-20T09:00:44', NULL, NULL); INSERT INTO partial_aggregation VALUES('2019-01-20T09:00:43', 1, 'hello'); INSERT INTO partial_aggregation VALUES('2019-01-20T09:00:44', 2, 'world'); INSERT INTO partial_aggregation VALUES('2020-01-20T09:00:43', 3.1, 'some1'); INSERT INTO partial_aggregation VALUES('2020-01-20T09:00:44', 3.2, 'more1'); INSERT INTO partial_aggregation VALUES('2021-01-20T09:00:43', 3.3, 'some2'); INSERT INTO partial_aggregation VALUES('2021-01-20T09:00:44', 3.4, 'more2'); INSERT INTO partial_aggregation VALUES('2022-01-20T09:00:43', 4, 'word1'); INSERT INTO partial_aggregation VALUES('2022-01-20T09:00:44', 5, 'word2'); INSERT INTO partial_aggregation VALUES('2023-01-20T09:00:43', 6, 'word3'); INSERT INTO partial_aggregation VALUES('2023-01-20T09:00:44', 7, 'word4'); -- Use enable_partitionwise_aggregate to create partial aggregates per chunk SET enable_partitionwise_aggregate = ON; SELECT format('SELECT %3$s, %1$s FROM partial_aggregation WHERE %2$s GROUP BY %3$s ORDER BY 1, 2;', function, condition, grouping) FROM unnest(array[ 'first(time, quantity), last(time, quantity)', 'last(longvalue, quantity)', 'last(quantity, longvalue)', 'last(quantity, time)', 'last(time, longvalue)']) AS function, unnest(array[ 'true', $$time < '2021-01-01'$$, 'quantity is null', 'quantity is not null', 'quantity >= 4']) AS condition, unnest(array[ '777::text' /* dummy grouping column */, 'longvalue', 'quantity', $$time_bucket('1 year', time)$$, $$time_bucket('3 year', time)$$]) AS grouping \gexec SELECT 777::text, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE true GROUP BY 777::text ORDER BY 1, 2; text | first | last ------+------------------------------+------------------------------ 777 | Sun Jan 20 09:00:43 2019 PST | Fri Jan 20 09:00:44 2023 PST SELECT 777::text, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY 777::text ORDER BY 1, 2; text | first | last ------+------------------------------+------------------------------ 777 | Sun Jan 20 09:00:43 2019 PST | Mon Jan 20 09:00:44 2020 PST SELECT 777::text, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY 777::text ORDER BY 1, 2; text | first | last ------+-------+------ 777 | | SELECT 777::text, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY 777::text ORDER BY 1, 2; text | first | last ------+------------------------------+------------------------------ 777 | Sun Jan 20 09:00:43 2019 PST | Fri Jan 20 09:00:44 2023 PST SELECT 777::text, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY 777::text ORDER BY 1, 2; text | first | last ------+------------------------------+------------------------------ 777 | Thu Jan 20 09:00:43 2022 PST | Fri Jan 20 09:00:44 2023 PST SELECT 777::text, last(longvalue, quantity) FROM partial_aggregation WHERE true GROUP BY 777::text ORDER BY 1, 2; text | last ------+------- 777 | word4 SELECT 777::text, last(longvalue, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY 777::text ORDER BY 1, 2; text | last ------+------- 777 | more1 SELECT 777::text, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | SELECT 777::text, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------- 777 | word4 SELECT 777::text, last(longvalue, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY 777::text ORDER BY 1, 2; text | last ------+------- 777 | word4 SELECT 777::text, last(quantity, longvalue) FROM partial_aggregation WHERE true GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 2 SELECT 777::text, last(quantity, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 2 SELECT 777::text, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | SELECT 777::text, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 2 SELECT 777::text, last(quantity, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 7 SELECT 777::text, last(quantity, time) FROM partial_aggregation WHERE true GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 7 SELECT 777::text, last(quantity, time) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 3.2 SELECT 777::text, last(quantity, time) FROM partial_aggregation WHERE quantity is null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | SELECT 777::text, last(quantity, time) FROM partial_aggregation WHERE quantity is not null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 7 SELECT 777::text, last(quantity, time) FROM partial_aggregation WHERE quantity >= 4 GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | 7 SELECT 777::text, last(time, longvalue) FROM partial_aggregation WHERE true GROUP BY 777::text ORDER BY 1, 2; text | last ------+------------------------------ 777 | Sun Jan 20 09:00:44 2019 PST SELECT 777::text, last(time, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY 777::text ORDER BY 1, 2; text | last ------+------------------------------ 777 | Sun Jan 20 09:00:44 2019 PST SELECT 777::text, last(time, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------ 777 | SELECT 777::text, last(time, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY 777::text ORDER BY 1, 2; text | last ------+------------------------------ 777 | Sun Jan 20 09:00:44 2019 PST SELECT 777::text, last(time, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY 777::text ORDER BY 1, 2; text | last ------+------------------------------ 777 | Fri Jan 20 09:00:44 2023 PST SELECT longvalue, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE true GROUP BY longvalue ORDER BY 1, 2; longvalue | first | last -----------+------------------------------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST more2 | Wed Jan 20 09:00:44 2021 PST | Wed Jan 20 09:00:44 2021 PST some1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST some2 | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:43 2021 PST word1 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST world | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST | | SELECT longvalue, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY longvalue ORDER BY 1, 2; longvalue | first | last -----------+------------------------------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST some1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST world | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST | | SELECT longvalue, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY longvalue ORDER BY 1, 2; longvalue | first | last -----------+-------+------ | | SELECT longvalue, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY longvalue ORDER BY 1, 2; longvalue | first | last -----------+------------------------------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST more2 | Wed Jan 20 09:00:44 2021 PST | Wed Jan 20 09:00:44 2021 PST some1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST some2 | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:43 2021 PST word1 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST world | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST SELECT longvalue, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY longvalue ORDER BY 1, 2; longvalue | first | last -----------+------------------------------+------------------------------ word1 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT longvalue, last(longvalue, quantity) FROM partial_aggregation WHERE true GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------- hello | hello more1 | more1 more2 | more2 some1 | some1 some2 | some2 word1 | word1 word2 | word2 word3 | word3 word4 | word4 world | world | SELECT longvalue, last(longvalue, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------- hello | hello more1 | more1 some1 | some1 world | world | SELECT longvalue, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ | SELECT longvalue, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------- hello | hello more1 | more1 more2 | more2 some1 | some1 some2 | some2 word1 | word1 word2 | word2 word3 | word3 word4 | word4 world | world SELECT longvalue, last(longvalue, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------- word1 | word1 word2 | word2 word3 | word3 word4 | word4 SELECT longvalue, last(quantity, longvalue) FROM partial_aggregation WHERE true GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 more2 | 3.4 some1 | 3.1 some2 | 3.3 word1 | 4 word2 | 5 word3 | 6 word4 | 7 world | 2 | SELECT longvalue, last(quantity, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 some1 | 3.1 world | 2 | SELECT longvalue, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ | SELECT longvalue, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 more2 | 3.4 some1 | 3.1 some2 | 3.3 word1 | 4 word2 | 5 word3 | 6 word4 | 7 world | 2 SELECT longvalue, last(quantity, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ word1 | 4 word2 | 5 word3 | 6 word4 | 7 SELECT longvalue, last(quantity, time) FROM partial_aggregation WHERE true GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 more2 | 3.4 some1 | 3.1 some2 | 3.3 word1 | 4 word2 | 5 word3 | 6 word4 | 7 world | 2 | SELECT longvalue, last(quantity, time) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 some1 | 3.1 world | 2 | SELECT longvalue, last(quantity, time) FROM partial_aggregation WHERE quantity is null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ | SELECT longvalue, last(quantity, time) FROM partial_aggregation WHERE quantity is not null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ hello | 1 more1 | 3.2 more2 | 3.4 some1 | 3.1 some2 | 3.3 word1 | 4 word2 | 5 word3 | 6 word4 | 7 world | 2 SELECT longvalue, last(quantity, time) FROM partial_aggregation WHERE quantity >= 4 GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ word1 | 4 word2 | 5 word3 | 6 word4 | 7 SELECT longvalue, last(time, longvalue) FROM partial_aggregation WHERE true GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST more2 | Wed Jan 20 09:00:44 2021 PST some1 | Mon Jan 20 09:00:43 2020 PST some2 | Wed Jan 20 09:00:43 2021 PST word1 | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST world | Sun Jan 20 09:00:44 2019 PST | SELECT longvalue, last(time, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST some1 | Mon Jan 20 09:00:43 2020 PST world | Sun Jan 20 09:00:44 2019 PST | SELECT longvalue, last(time, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------ | SELECT longvalue, last(time, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------------------------------ hello | Sun Jan 20 09:00:43 2019 PST more1 | Mon Jan 20 09:00:44 2020 PST more2 | Wed Jan 20 09:00:44 2021 PST some1 | Mon Jan 20 09:00:43 2020 PST some2 | Wed Jan 20 09:00:43 2021 PST word1 | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST world | Sun Jan 20 09:00:44 2019 PST SELECT longvalue, last(time, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY longvalue ORDER BY 1, 2; longvalue | last -----------+------------------------------ word1 | Thu Jan 20 09:00:43 2022 PST word2 | Thu Jan 20 09:00:44 2022 PST word3 | Fri Jan 20 09:00:43 2023 PST word4 | Fri Jan 20 09:00:44 2023 PST SELECT quantity, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE true GROUP BY quantity ORDER BY 1, 2; quantity | first | last ----------+------------------------------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST 3.3 | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:43 2021 PST 3.4 | Wed Jan 20 09:00:44 2021 PST | Wed Jan 20 09:00:44 2021 PST 4 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST | | SELECT quantity, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY quantity ORDER BY 1, 2; quantity | first | last ----------+------------------------------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST | | SELECT quantity, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY quantity ORDER BY 1, 2; quantity | first | last ----------+-------+------ | | SELECT quantity, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY quantity ORDER BY 1, 2; quantity | first | last ----------+------------------------------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST | Mon Jan 20 09:00:44 2020 PST 3.3 | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:43 2021 PST 3.4 | Wed Jan 20 09:00:44 2021 PST | Wed Jan 20 09:00:44 2021 PST 4 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT quantity, first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY quantity ORDER BY 1, 2; quantity | first | last ----------+------------------------------+------------------------------ 4 | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT quantity, last(longvalue, quantity) FROM partial_aggregation WHERE true GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------- 1 | hello 2 | world 3.1 | some1 3.2 | more1 3.3 | some2 3.4 | more2 4 | word1 5 | word2 6 | word3 7 | word4 | SELECT quantity, last(longvalue, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------- 1 | hello 2 | world 3.1 | some1 3.2 | more1 | SELECT quantity, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ | SELECT quantity, last(longvalue, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------- 1 | hello 2 | world 3.1 | some1 3.2 | more1 3.3 | some2 3.4 | more2 4 | word1 5 | word2 6 | word3 7 | word4 SELECT quantity, last(longvalue, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------- 4 | word1 5 | word2 6 | word3 7 | word4 SELECT quantity, last(quantity, longvalue) FROM partial_aggregation WHERE true GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 3.3 | 3.3 3.4 | 3.4 4 | 4 5 | 5 6 | 6 7 | 7 | SELECT quantity, last(quantity, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 | SELECT quantity, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ | SELECT quantity, last(quantity, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 3.3 | 3.3 3.4 | 3.4 4 | 4 5 | 5 6 | 6 7 | 7 SELECT quantity, last(quantity, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 4 | 4 5 | 5 6 | 6 7 | 7 SELECT quantity, last(quantity, time) FROM partial_aggregation WHERE true GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 3.3 | 3.3 3.4 | 3.4 4 | 4 5 | 5 6 | 6 7 | 7 | SELECT quantity, last(quantity, time) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 | SELECT quantity, last(quantity, time) FROM partial_aggregation WHERE quantity is null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ | SELECT quantity, last(quantity, time) FROM partial_aggregation WHERE quantity is not null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 1 | 1 2 | 2 3.1 | 3.1 3.2 | 3.2 3.3 | 3.3 3.4 | 3.4 4 | 4 5 | 5 6 | 6 7 | 7 SELECT quantity, last(quantity, time) FROM partial_aggregation WHERE quantity >= 4 GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ 4 | 4 5 | 5 6 | 6 7 | 7 SELECT quantity, last(time, longvalue) FROM partial_aggregation WHERE true GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST 3.3 | Wed Jan 20 09:00:43 2021 PST 3.4 | Wed Jan 20 09:00:44 2021 PST 4 | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST | SELECT quantity, last(time, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST | SELECT quantity, last(time, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------ | SELECT quantity, last(time, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------------------------------ 1 | Sun Jan 20 09:00:43 2019 PST 2 | Sun Jan 20 09:00:44 2019 PST 3.1 | Mon Jan 20 09:00:43 2020 PST 3.2 | Mon Jan 20 09:00:44 2020 PST 3.3 | Wed Jan 20 09:00:43 2021 PST 3.4 | Wed Jan 20 09:00:44 2021 PST 4 | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST SELECT quantity, last(time, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY quantity ORDER BY 1, 2; quantity | last ----------+------------------------------ 4 | Thu Jan 20 09:00:43 2022 PST 5 | Thu Jan 20 09:00:44 2022 PST 6 | Fri Jan 20 09:00:43 2023 PST 7 | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE true GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | | Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:44 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:44 2021 PST Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | | Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:44 2020 PST SELECT time_bucket('1 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+-------+------ Sun Dec 31 16:00:00 2017 PST | | SELECT time_bucket('1 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:43 2019 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST | Mon Jan 20 09:00:44 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST | Wed Jan 20 09:00:44 2021 PST Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:43 2022 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:43 2023 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE true GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | world Tue Dec 31 16:00:00 2019 PST | more1 Thu Dec 31 16:00:00 2020 PST | more2 Fri Dec 31 16:00:00 2021 PST | word2 Sat Dec 31 16:00:00 2022 PST | word4 SELECT time_bucket('1 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | world Tue Dec 31 16:00:00 2019 PST | more1 SELECT time_bucket('1 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('1 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Mon Dec 31 16:00:00 2018 PST | world Tue Dec 31 16:00:00 2019 PST | more1 Thu Dec 31 16:00:00 2020 PST | more2 Fri Dec 31 16:00:00 2021 PST | word2 Sat Dec 31 16:00:00 2022 PST | word4 SELECT time_bucket('1 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Fri Dec 31 16:00:00 2021 PST | word2 Sat Dec 31 16:00:00 2022 PST | word4 SELECT time_bucket('1 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE true GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.1 Thu Dec 31 16:00:00 2020 PST | 3.3 Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.1 SELECT time_bucket('1 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('1 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.1 Thu Dec 31 16:00:00 2020 PST | 3.3 Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(quantity, time) FROM partial_aggregation WHERE true GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.2 Thu Dec 31 16:00:00 2020 PST | 3.4 Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(quantity, time) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.2 SELECT time_bucket('1 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('1 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Mon Dec 31 16:00:00 2018 PST | 2 Tue Dec 31 16:00:00 2019 PST | 3.2 Thu Dec 31 16:00:00 2020 PST | 3.4 Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Fri Dec 31 16:00:00 2021 PST | 5 Sat Dec 31 16:00:00 2022 PST | 7 SELECT time_bucket('1 year', time), last(time, longvalue) FROM partial_aggregation WHERE true GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), last(time, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST SELECT time_bucket('1 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('1 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Mon Dec 31 16:00:00 2018 PST | Sun Jan 20 09:00:44 2019 PST Tue Dec 31 16:00:00 2019 PST | Mon Jan 20 09:00:43 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('1 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('1 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Fri Dec 31 16:00:00 2021 PST | Thu Jan 20 09:00:44 2022 PST Sat Dec 31 16:00:00 2022 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE true GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:43 2019 PST | Mon Jan 20 09:00:44 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:43 2019 PST | Mon Jan 20 09:00:44 2020 PST SELECT time_bucket('3 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+-------+------ Sun Dec 31 16:00:00 2017 PST | | SELECT time_bucket('3 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:43 2019 PST | Mon Jan 20 09:00:44 2020 PST Thu Dec 31 16:00:00 2020 PST | Wed Jan 20 09:00:43 2021 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), first(time, quantity), last(time, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | first | last ------------------------------+------------------------------+------------------------------ Thu Dec 31 16:00:00 2020 PST | Thu Jan 20 09:00:43 2022 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE true GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Sun Dec 31 16:00:00 2017 PST | more1 Thu Dec 31 16:00:00 2020 PST | word4 SELECT time_bucket('3 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Sun Dec 31 16:00:00 2017 PST | more1 SELECT time_bucket('3 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('3 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Sun Dec 31 16:00:00 2017 PST | more1 Thu Dec 31 16:00:00 2020 PST | word4 SELECT time_bucket('3 year', time), last(longvalue, quantity) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------- Thu Dec 31 16:00:00 2020 PST | word4 SELECT time_bucket('3 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE true GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 2 Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 2 SELECT time_bucket('3 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('3 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 2 Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(quantity, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(quantity, time) FROM partial_aggregation WHERE true GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 3.2 Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(quantity, time) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 3.2 SELECT time_bucket('3 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('3 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | 3.2 Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(quantity, time) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Thu Dec 31 16:00:00 2020 PST | 7 SELECT time_bucket('3 year', time), last(time, longvalue) FROM partial_aggregation WHERE true GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:44 2019 PST Thu Dec 31 16:00:00 2020 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), last(time, longvalue) FROM partial_aggregation WHERE time < '2021-01-01' GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:44 2019 PST SELECT time_bucket('3 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity is null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------ Sun Dec 31 16:00:00 2017 PST | SELECT time_bucket('3 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity is not null GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Sun Dec 31 16:00:00 2017 PST | Sun Jan 20 09:00:44 2019 PST Thu Dec 31 16:00:00 2020 PST | Fri Jan 20 09:00:44 2023 PST SELECT time_bucket('3 year', time), last(time, longvalue) FROM partial_aggregation WHERE quantity >= 4 GROUP BY time_bucket('3 year', time) ORDER BY 1, 2; time_bucket | last ------------------------------+------------------------------ Thu Dec 31 16:00:00 2020 PST | Fri Jan 20 09:00:44 2023 PST SET enable_partitionwise_aggregate = OFF; ================================================ FILE: test/expected/alter.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Set this variable to avoid using a hard-coded path each time query -- results are compared \set QUERY_RESULT_TEST_EQUAL_RELPATH 'include/query_result_test_equal.sql' -- DROP a table's column before making it a hypertable CREATE TABLE alter_before(id serial, time timestamp, temp float, colorid integer, notes text, notes_2 text); ALTER TABLE alter_before DROP COLUMN id; ALTER TABLE alter_before ALTER COLUMN temp SET (n_distinct = 10); ALTER TABLE alter_before ALTER COLUMN colorid SET (n_distinct = 11); ALTER TABLE alter_before ALTER COLUMN colorid RESET (n_distinct); ALTER TABLE alter_before ALTER COLUMN temp SET STATISTICS 100; ALTER TABLE alter_before ALTER COLUMN notes SET STORAGE EXTERNAL; SELECT create_hypertable('alter_before', 'time', chunk_time_interval => 2628000000000); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable --------------------------- (1,public,alter_before,t) -- Test error hint for invalid timescaledb options on ALTER TABLE \set ON_ERROR_STOP 0 -- Invalid timescaledb option should show hint with valid options \set VERBOSITY default ALTER TABLE alter_before SET (tsdb.invalid_option = true); ERROR: unrecognized parameter "tsdb.invalid_option" HINT: Valid timescaledb parameters are: chunk_interval, compress, compress_segmentby, compress_orderby, compress_chunk_interval, compress_index ALTER TABLE alter_before SET (timescaledb.nonexistent = false); ERROR: unrecognized parameter "timescaledb.nonexistent" HINT: Valid timescaledb parameters are: chunk_interval, compress, compress_segmentby, compress_orderby, compress_chunk_interval, compress_index \set ON_ERROR_STOP 1 \set VERBOSITY terse INSERT INTO alter_before VALUES ('2017-03-22T09:18:22', 23.5, 1); SELECT * FROM alter_before; time | temp | colorid | notes | notes_2 --------------------------+------+---------+-------+--------- Wed Mar 22 09:18:22 2017 | 23.5 | 1 | | -- Show that deleted column is marked as dropped and that attnums are -- now different for the root table and the chunk -- PG17 made attstattarget NULLABLE and changed the default from -1 to NULL SELECT c.relname, a.attname, a.attnum, a.attoptions, CASE WHEN a.attstattarget = -1 OR (a.attisdropped AND a.attstattarget = 0) THEN NULL ELSE a.attstattarget END attstattarget, a.attstorage FROM pg_attribute a, pg_class c WHERE a.attrelid = c.oid AND (c.relname LIKE '_hyper_1%_chunk' OR c.relname = 'alter_before') AND a.attnum > 0 ORDER BY c.relname, a.attnum; relname | attname | attnum | attoptions | attstattarget | attstorage ------------------+------------------------------+--------+-----------------+---------------+------------ _hyper_1_1_chunk | time | 1 | | | p _hyper_1_1_chunk | temp | 2 | {n_distinct=10} | 100 | p _hyper_1_1_chunk | colorid | 3 | | | p _hyper_1_1_chunk | notes | 4 | | | e _hyper_1_1_chunk | notes_2 | 5 | | | x alter_before | ........pg.dropped.1........ | 1 | | | p alter_before | time | 2 | | | p alter_before | temp | 3 | {n_distinct=10} | 100 | p alter_before | colorid | 4 | | | p alter_before | notes | 5 | | | e alter_before | notes_2 | 6 | | | x -- DROP a table's column after making it a hypertable and having data CREATE TABLE alter_after(id serial, time timestamp, temp float, colorid integer, notes text, notes_2 text); SELECT create_hypertable('alter_after', 'time', chunk_time_interval => 2628000000000); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------- (2,public,alter_after,t) -- Create first chunk INSERT INTO alter_after (time, temp, colorid) VALUES ('2017-03-22T09:18:22', 23.5, 1); ALTER TABLE alter_after DROP COLUMN id; ALTER TABLE alter_after ALTER COLUMN temp SET (n_distinct = 10); ALTER TABLE alter_after ALTER COLUMN colorid SET (n_distinct = 11); ALTER TABLE alter_after ALTER COLUMN colorid RESET (n_distinct); ALTER TABLE alter_after ALTER COLUMN colorid SET STATISTICS 101; ALTER TABLE alter_after ALTER COLUMN notes_2 SET STORAGE EXTERNAL; -- Creating new chunks after dropping a column should work just fine INSERT INTO alter_after VALUES ('2017-03-22T09:18:23', 21.5, 1), ('2017-05-22T09:18:22', 36.2, 2), ('2017-05-22T09:18:23', 15.2, 2); -- Make sure tuple conversion also works with COPY \COPY alter_after FROM 'data/alter.tsv' NULL AS ''; -- Data should look OK SELECT * FROM alter_after; time | temp | colorid | notes | notes_2 --------------------------+------+---------+-------+--------- Wed Mar 22 09:18:22 2017 | 23.5 | 1 | | Wed Mar 22 09:18:23 2017 | 21.5 | 1 | | Mon May 22 09:18:22 2017 | 36.2 | 2 | | Mon May 22 09:18:23 2017 | 15.2 | 2 | | Tue Aug 22 09:19:22 2017 | 21.4 | 3 | nr1 | n2r1 Wed Aug 23 09:20:17 2017 | 31.5 | 2 | nr2 | n2r2 -- Show that attnums are different for chunks created after DROP -- column SELECT c.relname, a.attname, a.attnum FROM pg_attribute a, pg_class c WHERE a.attrelid = c.oid AND (c.relname LIKE '_hyper_2%_chunk' OR c.relname = 'alter_after') AND a.attnum > 0 ORDER BY c.relname, a.attnum; relname | attname | attnum ------------------+------------------------------+-------- _hyper_2_2_chunk | ........pg.dropped.1........ | 1 _hyper_2_2_chunk | time | 2 _hyper_2_2_chunk | temp | 3 _hyper_2_2_chunk | colorid | 4 _hyper_2_2_chunk | notes | 5 _hyper_2_2_chunk | notes_2 | 6 _hyper_2_3_chunk | time | 1 _hyper_2_3_chunk | temp | 2 _hyper_2_3_chunk | colorid | 3 _hyper_2_3_chunk | notes | 4 _hyper_2_3_chunk | notes_2 | 5 _hyper_2_4_chunk | time | 1 _hyper_2_4_chunk | temp | 2 _hyper_2_4_chunk | colorid | 3 _hyper_2_4_chunk | notes | 4 _hyper_2_4_chunk | notes_2 | 5 alter_after | ........pg.dropped.1........ | 1 alter_after | time | 2 alter_after | temp | 3 alter_after | colorid | 4 alter_after | notes | 5 alter_after | notes_2 | 6 -- Add an ID column again ALTER TABLE alter_after ADD COLUMN id serial; INSERT INTO alter_after (time, temp, colorid) VALUES ('2017-08-22T09:19:14', 12.5, 3); --test thing that we are allowed to do on chunks ALTER TABLE _timescaledb_internal._hyper_2_3_chunk ALTER COLUMN temp RESET (n_distinct); ALTER TABLE _timescaledb_internal._hyper_2_4_chunk ALTER COLUMN temp SET (n_distinct = 20); ALTER TABLE _timescaledb_internal._hyper_2_4_chunk ALTER COLUMN temp SET STATISTICS 201; ALTER TABLE _timescaledb_internal._hyper_2_4_chunk ALTER COLUMN notes SET STORAGE EXTERNAL; -- PG17 made attstattarget NULLABLE and changed the default from -1 to NULL SELECT c.relname, a.attname, a.attnum, a.attoptions, CASE WHEN a.attstattarget = -1 OR (a.attisdropped AND a.attstattarget = 0) THEN NULL ELSE a.attstattarget END attstattarget, a.attstorage FROM pg_attribute a, pg_class c WHERE a.attrelid = c.oid AND (c.relname LIKE '_hyper_2%_chunk' OR c.relname = 'alter_after') AND a.attnum > 0 ORDER BY c.relname, a.attnum; relname | attname | attnum | attoptions | attstattarget | attstorage ------------------+------------------------------+--------+-----------------+---------------+------------ _hyper_2_2_chunk | ........pg.dropped.1........ | 1 | | | p _hyper_2_2_chunk | time | 2 | | | p _hyper_2_2_chunk | temp | 3 | {n_distinct=10} | | p _hyper_2_2_chunk | colorid | 4 | | 101 | p _hyper_2_2_chunk | notes | 5 | | | x _hyper_2_2_chunk | notes_2 | 6 | | | e _hyper_2_2_chunk | id | 7 | | | p _hyper_2_3_chunk | time | 1 | | | p _hyper_2_3_chunk | temp | 2 | | | p _hyper_2_3_chunk | colorid | 3 | | 101 | p _hyper_2_3_chunk | notes | 4 | | | x _hyper_2_3_chunk | notes_2 | 5 | | | e _hyper_2_3_chunk | id | 6 | | | p _hyper_2_4_chunk | time | 1 | | | p _hyper_2_4_chunk | temp | 2 | {n_distinct=20} | 201 | p _hyper_2_4_chunk | colorid | 3 | | 101 | p _hyper_2_4_chunk | notes | 4 | | | e _hyper_2_4_chunk | notes_2 | 5 | | | e _hyper_2_4_chunk | id | 6 | | | p alter_after | ........pg.dropped.1........ | 1 | | | p alter_after | time | 2 | | | p alter_after | temp | 3 | {n_distinct=10} | | p alter_after | colorid | 4 | | 101 | p alter_after | notes | 5 | | | x alter_after | notes_2 | 6 | | | e alter_after | id | 7 | | | p SELECT * FROM alter_after; time | temp | colorid | notes | notes_2 | id --------------------------+------+---------+-------+---------+---- Wed Mar 22 09:18:22 2017 | 23.5 | 1 | | | 1 Wed Mar 22 09:18:23 2017 | 21.5 | 1 | | | 2 Mon May 22 09:18:22 2017 | 36.2 | 2 | | | 3 Mon May 22 09:18:23 2017 | 15.2 | 2 | | | 4 Tue Aug 22 09:19:22 2017 | 21.4 | 3 | nr1 | n2r1 | 5 Wed Aug 23 09:20:17 2017 | 31.5 | 2 | nr2 | n2r2 | 6 Tue Aug 22 09:19:14 2017 | 12.5 | 3 | | | 7 -- test setting reloptions ALTER TABLE _timescaledb_internal._hyper_2_3_chunk SET (parallel_workers=2); ALTER TABLE _timescaledb_internal._hyper_2_4_chunk SET (parallel_workers=4); ALTER TABLE _timescaledb_internal._hyper_2_4_chunk RESET (parallel_workers); SELECT relname, reloptions FROM pg_class WHERE relname IN ('_hyper_2_3_chunk','_hyper_2_4_chunk'); relname | reloptions ------------------+---------------------- _hyper_2_3_chunk | {parallel_workers=2} _hyper_2_4_chunk | -- Need superuser to ALTER chunks in _timescaledb_internal schema \c :TEST_DBNAME :ROLE_SUPERUSER SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk WHERE id = 2; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-----------------------+------------------+---------------------+--------+----------- 2 | 2 | _timescaledb_internal | _hyper_2_2_chunk | | 0 | f -- Rename chunk ALTER TABLE _timescaledb_internal._hyper_2_2_chunk RENAME TO new_chunk_name; SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk WHERE id = 2; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-----------------------+----------------+---------------------+--------+----------- 2 | 2 | _timescaledb_internal | new_chunk_name | | 0 | f -- Set schema ALTER TABLE _timescaledb_internal.new_chunk_name SET SCHEMA public; SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk WHERE id = 2; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-------------+----------------+---------------------+--------+----------- 2 | 2 | public | new_chunk_name | | 0 | f -- Test that we cannot rename chunk columns \set ON_ERROR_STOP 0 ALTER TABLE public.new_chunk_name RENAME COLUMN time TO newtime; ERROR: cannot rename column "time" of hypertable chunk "new_chunk_name" \set ON_ERROR_STOP 1 -- Test that we can set tablespace of a hypertable \c :TEST_DBNAME :ROLE_SUPERUSER SET client_min_messages = ERROR; DROP TABLESPACE IF EXISTS tablespace1; DROP TABLESPACE IF EXISTS tablespace2; SET client_min_messages = NOTICE; --test hypertable with tables space CREATE TABLESPACE tablespace1 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE1_PATH; CREATE TABLESPACE tablespace2 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE2_PATH; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- Test that we can directly change chunk tablespace ALTER TABLE public.new_chunk_name SET TABLESPACE tablespace1; SELECT tablespace FROM pg_tables WHERE tablename = 'new_chunk_name'; tablespace ------------- tablespace1 -- drop all tables to make checking the tests below easier DROP TABLE alter_before; DROP TABLE alter_after; -- should return 0 rows SELECT tablename, tablespace FROM pg_tables WHERE tablename = 'hyper_in_space' OR tablename LIKE '\_hyper\__\__\_chunk' ORDER BY tablename; tablename | tablespace -----------+------------ CREATE TABLE hyper_in_space(time bigint, temp float, device int); SELECT create_hypertable('hyper_in_space', 'time', 'device', 4, chunk_time_interval=>1); create_hypertable ----------------------------- (3,public,hyper_in_space,t) INSERT INTO hyper_in_space(time, temp, device) VALUES (1, 20, 1); INSERT INTO hyper_in_space(time, temp, device) VALUES (3, 21, 2); INSERT INTO hyper_in_space(time, temp, device) VALUES (5, 23, 1); SELECT tablename FROM pg_tables WHERE tablespace = 'tablespace1' ORDER BY tablename; tablename ----------- SET default_tablespace = tablespace1; -- should be inserted in tablespace1 which is now default INSERT INTO hyper_in_space(time, temp, device) VALUES (11, 24, 3); SELECT tablename, tablespace FROM pg_tables WHERE tablename = 'hyper_in_space' OR tablename LIKE '\_hyper\__\__\_chunk' ORDER BY tablename; tablename | tablespace ------------------+------------- _hyper_3_5_chunk | _hyper_3_6_chunk | _hyper_3_7_chunk | _hyper_3_8_chunk | tablespace1 hyper_in_space | SET default_tablespace TO DEFAULT; ALTER TABLE hyper_in_space SET TABLESPACE tablespace1; SELECT tablename FROM pg_tables WHERE tablespace = 'tablespace1' ORDER BY tablename; tablename ------------------ _hyper_3_5_chunk _hyper_3_6_chunk _hyper_3_7_chunk _hyper_3_8_chunk hyper_in_space -- should be inserted in an existing chunk in the new tablespace, -- no new chunks INSERT INTO hyper_in_space(time, temp, device) VALUES (5, 27, 1); -- the new chunk should be create in the new tablespace INSERT INTO hyper_in_space(time, temp, device) VALUES (8, 24, 2); SELECT tablename, tablespace FROM pg_tables WHERE tablename = 'hyper_in_space' OR tablename LIKE '\_hyper\__\__\_chunk' ORDER BY tablename; tablename | tablespace ------------------+------------- _hyper_3_5_chunk | tablespace1 _hyper_3_6_chunk | tablespace1 _hyper_3_7_chunk | tablespace1 _hyper_3_8_chunk | tablespace1 _hyper_3_9_chunk | tablespace1 hyper_in_space | tablespace1 -- should not fail (unlike attach_tablespace) ALTER TABLE hyper_in_space SET TABLESPACE tablespace1; \set ON_ERROR_STOP 0 -- not an empty tablespace DROP TABLESPACE tablespace1; ERROR: tablespace "tablespace1" is still attached to 1 hypertables \set ON_ERROR_STOP 1 -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'hyper_in_space\', 22)::NAME' \set QUERY2 'SELECT drop_chunks(\'hyper_in_space\', 22)::NAME' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 5 | 5 SELECT tablename, tablespace FROM pg_tables WHERE tablespace = 'tablespace1' ORDER BY tablename; tablename | tablespace ----------------+------------- hyper_in_space | tablespace1 \set ON_ERROR_STOP 0 -- should not be able to drop tablespace if a hypertable depends on it -- even when there are no chunks DROP TABLESPACE tablespace1; ERROR: tablespace "tablespace1" is still attached to 1 hypertables \set ON_ERROR_STOP 1 DROP TABLE hyper_in_space; CREATE TABLE hyper_in_space(time bigint, temp float, device int) TABLESPACE tablespace1; SELECT create_hypertable('hyper_in_space', 'time', 'device', 4, chunk_time_interval=>1); create_hypertable ----------------------------- (4,public,hyper_in_space,t) INSERT INTO hyper_in_space(time, temp, device) VALUES (1, 20, 1); INSERT INTO hyper_in_space(time, temp, device) VALUES (3, 21, 2); INSERT INTO hyper_in_space(time, temp, device) VALUES (5, 23, 1); SELECT tablename, tablespace FROM pg_tables WHERE tablename = 'hyper_in_space' OR tablename ~ '_hyper_\d+_\d+_chunk' ORDER BY tablename; tablename | tablespace -------------------+------------- _hyper_4_10_chunk | tablespace1 _hyper_4_11_chunk | tablespace1 _hyper_4_12_chunk | tablespace1 hyper_in_space | tablespace1 SELECT attach_tablespace('tablespace2', 'hyper_in_space'); attach_tablespace ------------------- \set ON_ERROR_STOP 0 -- should fail as >1 tablespaces are attached ALTER TABLE hyper_in_space SET TABLESPACE tablespace1; ERROR: cannot set new tablespace when multiple tablespaces are attached to hypertable "hyper_in_space" \set ON_ERROR_STOP 1 SELECT detach_tablespace('tablespace2', 'hyper_in_space'); detach_tablespace ------------------- 1 SELECT * FROM _timescaledb_catalog.tablespace; id | hypertable_id | tablespace_name ----+---------------+----------------- 3 | 4 | tablespace1 -- make sure when using ALTER TABLE, table spaces are not accumulated -- as in case of attach_tablespace -- should have one result SELECT * FROM _timescaledb_catalog.tablespace; id | hypertable_id | tablespace_name ----+---------------+----------------- 3 | 4 | tablespace1 ALTER TABLE hyper_in_space SET TABLESPACE tablespace2; -- should have one result SELECT * FROM _timescaledb_catalog.tablespace; id | hypertable_id | tablespace_name ----+---------------+----------------- 5 | 4 | tablespace2 ALTER TABLE hyper_in_space SET TABLESPACE tablespace1; -- should have one result, (same as the first in the block) SELECT * FROM _timescaledb_catalog.tablespace; id | hypertable_id | tablespace_name ----+---------------+----------------- 6 | 4 | tablespace1 SELECT tablename, tablespace FROM pg_tables WHERE tablename = 'hyper_in_space' OR tablename ~ '_hyper_\d+_\d+_chunk' ORDER BY tablename; tablename | tablespace -------------------+------------- _hyper_4_10_chunk | tablespace1 _hyper_4_11_chunk | tablespace1 _hyper_4_12_chunk | tablespace1 hyper_in_space | tablespace1 -- attach tb2 <-> ALTER SET tb1 <-> detach tb1 should work SELECT detach_tablespace('tablespace1', 'hyper_in_space'); detach_tablespace ------------------- 1 INSERT INTO hyper_in_space(time, temp, device) VALUES (5, 23, 1); INSERT INTO hyper_in_space(time, temp, device) VALUES (7, 23, 1); -- Since we have detached tablespace1 the new chunk should not be -- placed there. SELECT tablename, tablespace FROM pg_tables WHERE tablename = 'hyper_in_space' OR tablename ~ '_hyper_\d+_\d+_chunk' ORDER BY tablename; tablename | tablespace -------------------+------------- _hyper_4_10_chunk | tablespace1 _hyper_4_11_chunk | tablespace1 _hyper_4_12_chunk | tablespace1 _hyper_4_13_chunk | hyper_in_space | SELECT * FROM _timescaledb_catalog.tablespace; id | hypertable_id | tablespace_name ----+---------------+----------------- -- tablespace functions should handle the default tablespace just as they do others SELECT attach_tablespace('pg_default', 'hyper_in_space'); attach_tablespace ------------------- SELECT attach_tablespace('tablespace2', 'hyper_in_space'); attach_tablespace ------------------- SELECT tablename, tablespace FROM pg_tables WHERE tablename = 'hyper_in_space' OR tablename ~ '_hyper_\d+_\d+_chunk' ORDER BY tablename; tablename | tablespace -------------------+------------- _hyper_4_10_chunk | tablespace1 _hyper_4_11_chunk | tablespace1 _hyper_4_12_chunk | tablespace1 _hyper_4_13_chunk | hyper_in_space | tablespace2 SELECT * FROM _timescaledb_catalog.tablespace; id | hypertable_id | tablespace_name ----+---------------+----------------- 7 | 4 | pg_default 8 | 4 | tablespace2 INSERT INTO hyper_in_space(time, temp, device) VALUES (12, 22, 1); INSERT INTO hyper_in_space(time, temp, device) VALUES (13, 23, 3); SELECT tablename, tablespace FROM pg_tables WHERE tablename = 'hyper_in_space' OR tablename ~ '_hyper_\d+_\d+_chunk' ORDER BY tablename; tablename | tablespace -------------------+------------- _hyper_4_10_chunk | tablespace1 _hyper_4_11_chunk | tablespace1 _hyper_4_12_chunk | tablespace1 _hyper_4_13_chunk | _hyper_4_14_chunk | _hyper_4_15_chunk | tablespace2 hyper_in_space | tablespace2 SELECT detach_tablespace('pg_default', 'hyper_in_space'); detach_tablespace ------------------- 1 ALTER TABLE hyper_in_space SET TABLESPACE pg_default; SELECT tablename, tablespace FROM pg_tables WHERE tablename = 'hyper_in_space' OR tablename ~ '_hyper_\d+_\d+_chunk' ORDER BY tablename; tablename | tablespace -------------------+------------ _hyper_4_10_chunk | _hyper_4_11_chunk | _hyper_4_12_chunk | _hyper_4_13_chunk | _hyper_4_14_chunk | _hyper_4_15_chunk | hyper_in_space | SELECT detach_tablespace('pg_default', 'hyper_in_space'); detach_tablespace ------------------- 1 DROP TABLE hyper_in_space; -- test altering tablespace on index, issue #903 CREATE TABLE series( time timestamptz not null, device int, value float, CONSTRAINT series_pk PRIMARY KEY (time, device) USING INDEX TABLESPACE tablespace1); SELECT create_hypertable('series', 'time', create_default_indexes => FALSE); create_hypertable --------------------- (5,public,series,t) INSERT INTO series VALUES ('2019-04-21 10:12', 1, 1.01); CREATE INDEX series_value ON series (value, time) TABLESPACE tablespace2; SELECT schemaname, tablename, indexname, tablespace FROM pg_indexes WHERE indexname LIKE '%series%' ORDER BY indexname; schemaname | tablename | indexname | tablespace -----------------------+-------------------+--------------------------------+------------- _timescaledb_internal | _hyper_5_16_chunk | 16_1_series_pk | tablespace1 _timescaledb_internal | _hyper_5_16_chunk | _hyper_5_16_chunk_series_value | tablespace2 public | series | series_pk | tablespace1 public | series | series_value | tablespace2 ALTER INDEX series_pk SET TABLESPACE tablespace2; CREATE INDEX ON series (time) TABLESPACE tablespace1; ALTER INDEX series_value SET TABLESPACE pg_default; INSERT INTO series VALUES ('2019-04-29 10:12', 2, 1.31); SELECT schemaname, tablename, indexname, tablespace FROM pg_indexes WHERE indexname LIKE '%series%' ORDER BY indexname; schemaname | tablename | indexname | tablespace -----------------------+-------------------+-----------------------------------+------------- _timescaledb_internal | _hyper_5_16_chunk | 16_1_series_pk | tablespace2 _timescaledb_internal | _hyper_5_17_chunk | 17_2_series_pk | tablespace2 _timescaledb_internal | _hyper_5_16_chunk | _hyper_5_16_chunk_series_time_idx | tablespace1 _timescaledb_internal | _hyper_5_16_chunk | _hyper_5_16_chunk_series_value | _timescaledb_internal | _hyper_5_17_chunk | _hyper_5_17_chunk_series_time_idx | tablespace1 _timescaledb_internal | _hyper_5_17_chunk | _hyper_5_17_chunk_series_value | public | series | series_pk | tablespace2 public | series | series_time_idx | tablespace1 public | series | series_value | DROP TABLE series; DROP TABLESPACE tablespace1; DROP TABLESPACE tablespace2; -- Make sure we handle ALTER SCHEMA RENAME for hypertable schemas \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA IF NOT EXISTS original_name; CREATE TABLE original_name.my_table ( date timestamp with time zone NOT NULL, quantity double precision ); SELECT create_hypertable('original_name.my_table','date'); create_hypertable ------------------------------ (6,original_name,my_table,t) INSERT INTO original_name.my_table (date, quantity) VALUES ('2018-07-04T21:00:00+00:00', 8); ALTER SCHEMA original_name RENAME TO new_name; DROP TABLE new_name.my_table; DROP SCHEMA new_name; -- Now make sure schema is renamed for multiple hypertables, but not hypertables not in the schema CREATE SCHEMA IF NOT EXISTS original_name; CREATE TABLE original_name.my_table ( date timestamp with time zone NOT NULL, quantity double precision ); CREATE TABLE original_name.my_table2 ( date timestamp with time zone NOT NULL, quantity double precision ); CREATE TABLE regular_table ( date timestamp with time zone NOT NULL, quantity double precision ); SELECT create_hypertable('original_name.my_table','date'); create_hypertable ------------------------------ (7,original_name,my_table,t) SELECT create_hypertable('original_name.my_table2','date'); create_hypertable ------------------------------- (8,original_name,my_table2,t) SELECT create_hypertable('regular_table','date'); create_hypertable ---------------------------- (9,public,regular_table,t) INSERT INTO original_name.my_table (date, quantity) VALUES ('2018-07-04T21:00:00+00:00', 8); INSERT INTO original_name.my_table2 (date, quantity) VALUES ('2018-07-04T21:00:00+00:00', 8); INSERT INTO regular_table (date, quantity) VALUES ('2018-07-04T21:00:00+00:00', 8); ALTER SCHEMA original_name RENAME TO new_name; DROP TABLE new_name.my_table; DROP TABLE new_name.my_table2; DROP TABLE regular_table; DROP SCHEMA new_name; -- These tables should also drop when we drop the whole schema CREATE SCHEMA IF NOT EXISTS original_name; CREATE TABLE original_name.my_table ( date timestamp with time zone NOT NULL, quantity double precision ); CREATE TABLE original_name.my_table2 ( date timestamp with time zone NOT NULL, quantity double precision ); SELECT create_hypertable('original_name.my_table','date'); create_hypertable ------------------------------- (10,original_name,my_table,t) SELECT create_hypertable('original_name.my_table2','date'); create_hypertable -------------------------------- (11,original_name,my_table2,t) INSERT INTO original_name.my_table (date, quantity) VALUES ('2018-07-04T21:00:00+00:00', 8); INSERT INTO original_name.my_table2 (date, quantity) VALUES ('2018-07-04T21:00:00+00:00', 8); ALTER SCHEMA original_name RENAME TO new_name; DROP SCHEMA new_name CASCADE; NOTICE: drop cascades to 4 other objects SELECT * FROM test.relation WHERE schema = 'new_name'; schema | name | type | owner --------+------+------+------- -- Make sure we can't rename internal schemas \set ON_ERROR_STOP 0 ALTER SCHEMA _timescaledb_internal RENAME TO my_new_schema_name; ERROR: cannot rename schemas used by the TimescaleDB extension ALTER SCHEMA _timescaledb_catalog RENAME TO my_new_schema_name; ERROR: cannot rename schemas used by the TimescaleDB extension ALTER SCHEMA _timescaledb_cache RENAME TO my_new_schema_name; ERROR: cannot rename schemas used by the TimescaleDB extension \set ON_ERROR_STOP 1 -- Make sure we can rename associated schemas CREATE TABLE my_table ( date timestamp with time zone NOT NULL, quantity double precision ); SELECT create_hypertable('my_table','date', associated_schema_name => 'my_associated_schema'); create_hypertable ------------------------ (12,public,my_table,t) INSERT INTO my_table (date, quantity) VALUES ('2018-07-04T21:00:00+00:00', 8); ALTER SCHEMA my_associated_schema RENAME TO new_associated_schema; INSERT INTO my_table (date, quantity) VALUES ('2018-08-10T23:00:00+00:00', 20); -- Make sure the schema name is changed in both catalog tables SELECT * from _timescaledb_catalog.hypertable; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 12 | public | my_table | new_associated_schema | _hyper_12 | 1 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk from _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-----------------------+--------------------+---------------------+--------+----------- 24 | 12 | new_associated_schema | _hyper_12_24_chunk | | 0 | f 25 | 12 | new_associated_schema | _hyper_12_25_chunk | | 0 | f DROP TABLE my_table; -- test renaming unique constraints/indexes CREATE TABLE t_hypertable ( id INTEGER NOT NULL, time TIMESTAMPTZ NOT NULL, value FLOAT NOT NULL CHECK (value > 0), UNIQUE(id, time)); SELECT create_hypertable('t_hypertable', 'time'); create_hypertable ---------------------------- (13,public,t_hypertable,t) INSERT INTO t_hypertable AS h VALUES ( 1, '2020-01-01 00:00:00', 3.2) ON CONFLICT (id, time) DO UPDATE SET value = h.value + EXCLUDED.value; INSERT INTO t_hypertable AS h VALUES ( 1, '2021-01-01 00:00:00', 3.2) ON CONFLICT (id, time) DO UPDATE SET value = h.value + EXCLUDED.value; BEGIN; ALTER INDEX t_hypertable_id_time_key RENAME TO t_new_constraint; -- chunk_constraint should have updated constraint names SELECT hypertable_constraint_name, constraint_name from _timescaledb_catalog.chunk_constraint WHERE hypertable_constraint_name = 't_new_constraint' ORDER BY 1,2; hypertable_constraint_name | constraint_name ----------------------------+------------------------------------- t_new_constraint | _hyper_13_26_chunk_t_new_constraint t_new_constraint | _hyper_13_27_chunk_t_new_constraint INSERT INTO t_hypertable AS h VALUES ( 1, '2020-01-01 00:01:00', 3.2) ON CONFLICT (id, time) DO UPDATE SET value = h.value + EXCLUDED.value; ROLLBACK; BEGIN; ALTER TABLE t_hypertable RENAME CONSTRAINT t_hypertable_id_time_key TO t_new_constraint; -- chunk_constraint should have updated constraint names SELECT hypertable_constraint_name, constraint_name from _timescaledb_catalog.chunk_constraint WHERE hypertable_constraint_name = 't_new_constraint' ORDER BY 1,2; hypertable_constraint_name | constraint_name ----------------------------+----------------------- t_new_constraint | 26_5_t_new_constraint t_new_constraint | 27_6_t_new_constraint INSERT INTO t_hypertable AS h VALUES ( 1, '2020-01-01 00:01:00', 3.2) ON CONFLICT (id, time) DO UPDATE SET value = h.value + EXCLUDED.value; ROLLBACK; -- predicate reconstruction when attnos are different in hypertable and chunk CREATE TABLE p_hypertable (a integer not null, b integer, c integer); SELECT create_hypertable('p_hypertable', 'a', chunk_time_interval => int '3'); create_hypertable ---------------------------- (14,public,p_hypertable,t) BEGIN; ALTER TABLE p_hypertable DROP COLUMN b, ADD COLUMN d boolean; CREATE INDEX idx_ht ON p_hypertable(a, c) WHERE d = FALSE; END; INSERT INTO p_hypertable(a, c, d) VALUES (1, 1, FALSE); \d _timescaledb_internal._hyper_14_28_chunk Table "_timescaledb_internal._hyper_14_28_chunk" Column | Type | Collation | Nullable | Default --------+---------+-----------+----------+--------- a | integer | | not null | c | integer | | | d | boolean | | | Indexes: "_hyper_14_28_chunk_idx_ht" btree (a, c) WHERE NOT d "_hyper_14_28_chunk_p_hypertable_a_idx" btree (a DESC) Check constraints: "constraint_34" CHECK (a >= 0 AND a < 3) Inherits: p_hypertable DROP TABLE p_hypertable; -- check none of our hooks interact badly with normal alter view handling CREATE VIEW v1 AS SELECT random(); \set ON_ERROR_STOP 0 -- should error with unrecognized parameter ALTER VIEW v1 SET (autovacuum_enabled = false); ERROR: unrecognized parameter "autovacuum_enabled" \set ON_ERROR_STOP 1 -- issue 4474 -- test hypertable with non-default statistics target -- and chunk creation triggered by non-owner CREATE ROLE role_4474; CREATE TABLE i4474(time timestamptz NOT NULL); SELECT table_name FROM public.create_hypertable( 'i4474', 'time'); table_name ------------ i4474 GRANT SELECT, INSERT on i4474 TO role_4474; -- create chunk as owner INSERT INTO i4474 SELECT '2020-01-01'; -- set statistics ALTER TABLE i4474 ALTER COLUMN time SET statistics 10; -- create chunk as non-owner SET ROLE role_4474; INSERT INTO i4474 SELECT '2021-01-01'; RESET ROLE; DROP TABLE i4474 CASCADE; DROP ROLE role_4474; -- verify that setting replica identity works and chunks inherit the -- root table's setting CREATE TABLE replid(time timestamptz, value int); SELECT create_hypertable('replid', 'time', chunk_time_interval => interval '1 day', create_default_indexes => false); create_hypertable ---------------------- (16,public,replid,t) -- replica identity set to default SELECT relreplident FROM pg_class WHERE relname = 'replid'; relreplident -------------- d INSERT INTO replid VALUES ('2023-01-01', 1); -- the new chunk should have the same replica identity setting SELECT relname, relreplident FROM show_chunks('replid') ch INNER JOIN pg_class c ON (ch = c.oid) ORDER BY relname; relname | relreplident --------------------+-------------- _hyper_16_31_chunk | d -- test change to replica identity full ALTER TABLE replid REPLICA IDENTITY FULL; SELECT relname, relreplident FROM pg_class WHERE relname = 'replid' ORDER BY relname; relname | relreplident ---------+-------------- replid | f -- the chunk's setting should also change to FULL SELECT relname, relreplident FROM show_chunks('replid') ch INNER JOIN pg_class c ON (ch = c.oid) ORDER BY relname; relname | relreplident --------------------+-------------- _hyper_16_31_chunk | f -- change to replica identity index CREATE UNIQUE INDEX time_key ON replid (time); ALTER TABLE replid REPLICA IDENTITY USING INDEX time_key; SELECT relname, relreplident FROM pg_class WHERE relname = 'replid' ORDER BY relname; relname | relreplident ---------+-------------- replid | i SELECT relname, relreplident FROM show_chunks('replid') ch INNER JOIN pg_class c ON (ch = c.oid) ORDER BY relname; relname | relreplident --------------------+-------------- _hyper_16_31_chunk | i SELECT indexrelid::regclass::text AS index_name FROM show_chunks('replid') chid INNER JOIN pg_index i ON (i.indrelid = chid) AND indisreplident=true ORDER BY index_name; index_name --------------------------------------------------- _timescaledb_internal._hyper_16_31_chunk_time_key INSERT INTO replid VALUES ('2023-01-02', 2); -- the new chunk will also have replica identity "index" SELECT relname, relreplident FROM show_chunks('replid') ch INNER JOIN pg_class c ON (ch = c.oid) ORDER BY relname; relname | relreplident --------------------+-------------- _hyper_16_31_chunk | i _hyper_16_32_chunk | i SELECT indexrelid::regclass::text AS index_name FROM show_chunks('replid') chid INNER JOIN pg_index i ON (i.indrelid = chid) AND indisreplident=true ORDER BY index_name; index_name --------------------------------------------------- _timescaledb_internal._hyper_16_31_chunk_time_key _timescaledb_internal._hyper_16_32_chunk_time_key -- drop the replica identity index and create a new chunk. The new -- chunk should have replica identity "NOTHING" since this is the -- behavior of replica identity index when the index is dropped. DROP INDEX time_key; INSERT INTO replid VALUES ('2023-01-03', 3); -- no indexes left SELECT relname, relreplident FROM show_chunks('replid') ch INNER JOIN pg_class c ON (ch = c.oid) ORDER BY relname; relname | relreplident --------------------+-------------- _hyper_16_31_chunk | i _hyper_16_32_chunk | i _hyper_16_33_chunk | n SELECT indexrelid::regclass::text AS index_name FROM show_chunks('replid') chid INNER JOIN pg_index i ON (i.indrelid = chid) AND indisreplident=true ORDER BY index_name; index_name ------------ -- recreate the unique index after drop and insert to create a new chunk. -- This is a regression test for a bug where rd_replidindex was stale -- after relcache invalidation from chunk index creation, leading to -- "could not open relation with OID 0" error. CREATE UNIQUE INDEX time_key ON replid (time); INSERT INTO replid VALUES ('2023-01-04', 4); SELECT relname, relreplident FROM show_chunks('replid') ch INNER JOIN pg_class c ON (ch = c.oid) ORDER BY relname; relname | relreplident --------------------+-------------- _hyper_16_31_chunk | i _hyper_16_32_chunk | i _hyper_16_33_chunk | n _hyper_16_34_chunk | n -- Alter replica identity directly on a chunk is not supported SELECT ch AS chunk_name FROM show_chunks('replid') ch ORDER BY chunk_name LIMIT 1 \gset \set ON_ERROR_STOP 0 ALTER TABLE :chunk_name REPLICA IDENTITY FULL; ERROR: operation not supported on chunk tables \set ON_ERROR_STOP 1 SELECT relname, relreplident FROM show_chunks('replid') ch INNER JOIN pg_class c ON (ch = c.oid) ORDER BY relname; relname | relreplident --------------------+-------------- _hyper_16_31_chunk | i _hyper_16_32_chunk | i _hyper_16_33_chunk | n _hyper_16_34_chunk | n -- test implicit constraints gh issue #9132 CREATE TABLE i9132(time timestamptz) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column INSERT INTO i9132 VALUES ('2024-01-01'), ('2024-02-02'); ALTER TABLE i9132 ADD COLUMN id serial, ADD CONSTRAINT implicit_pk PRIMARY KEY (id, time); ================================================ FILE: test/expected/alternate_users.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \ir include/insert_single.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."one_Partition" ( "timeCustom" BIGINT NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."one_Partition" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; CREATE INDEX ON PUBLIC."one_Partition" ("timeCustom" DESC NULLS LAST, series_0) WHERE series_0 IS NOT NULL; CREATE INDEX ON PUBLIC."one_Partition" ("timeCustom" DESC NULLS LAST, series_1) WHERE series_1 IS NOT NULL; CREATE INDEX ON PUBLIC."one_Partition" ("timeCustom" DESC NULLS LAST, series_2) WHERE series_2 IS NOT NULL; CREATE INDEX ON PUBLIC."one_Partition" ("timeCustom" DESC NULLS LAST, series_bool) WHERE series_bool IS NOT NULL; \c :DBNAME :ROLE_SUPERUSER CREATE SCHEMA "one_Partition" AUTHORIZATION :ROLE_DEFAULT_PERM_USER; \c :DBNAME :ROLE_DEFAULT_PERM_USER; SELECT * FROM create_hypertable('"public"."one_Partition"', 'timeCustom', associated_schema_name=>'one_Partition', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+---------------+--------- 1 | public | one_Partition | t --output command tags \set QUIET off BEGIN; BEGIN \COPY "one_Partition" FROM 'data/ds1_dev1_1.tsv' NULL AS ''; COPY 7 COMMIT; COMMIT INSERT INTO "one_Partition"("timeCustom", device_id, series_0, series_1) VALUES (1257987600000000000, 'dev1', 1.5, 1), (1257987600000000000, 'dev1', 1.5, 2), (1257894000000000000, 'dev2', 1.5, 1), (1257894002000000000, 'dev1', 2.5, 3); INSERT 0 4 INSERT INTO "one_Partition"("timeCustom", device_id, series_0, series_1) VALUES (1257894000000000000, 'dev2', 1.5, 2); INSERT 0 1 \set QUIET on \c :TEST_DBNAME :ROLE_SUPERUSER -- make sure tablespace1 exists -- since there is no CREATE TABLESPACE IF EXISTS we drop with if exists and recreate SET client_min_messages TO error; DROP TABLESPACE IF EXISTS tablespace1; RESET client_min_messages; CREATE TABLESPACE tablespace1 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE1_PATH; --needed for ddl ops: CREATE SCHEMA IF NOT EXISTS "customSchema" AUTHORIZATION :ROLE_DEFAULT_PERM_USER_2; --needed for ROLE_DEFAULT_PERM_USER_2 to write to the 'one_Partition' schema which --is owned by ROLE_DEFAULT_PERM_USER GRANT CREATE ON SCHEMA "one_Partition" TO :ROLE_DEFAULT_PERM_USER_2; --test creating and using schema as non-superuser \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 SELECT * FROM test.relation WHERE schema='public' ORDER BY schema, name; schema | name | type | owner --------+---------------+-------+------------------- public | one_Partition | table | default_perm_user \set ON_ERROR_STOP 0 SELECT * FROM "one_Partition"; ERROR: permission denied for table one_Partition SELECT set_chunk_time_interval('"one_Partition"', 1::bigint); ERROR: must be owner of hypertable "one_Partition" select add_dimension('"one_Partition"', 'device_id', 2); ERROR: must be owner of hypertable "one_Partition" select attach_tablespace('tablespace1', '"one_Partition"'); ERROR: must be owner of hypertable "one_Partition" \set ON_ERROR_STOP 1 CREATE TABLE "1dim"(time timestamp, temp float); SELECT create_hypertable('"1dim"', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ------------------- (2,public,1dim,t) INSERT INTO "1dim" VALUES('2017-01-20T09:00:01', 22.5); INSERT INTO "1dim" VALUES('2017-01-20T09:00:21', 21.2); INSERT INTO "1dim" VALUES('2017-01-20T09:00:47', 25.1); SELECT * FROM "1dim"; time | temp --------------------------+------ Fri Jan 20 09:00:01 2017 | 22.5 Fri Jan 20 09:00:21 2017 | 21.2 Fri Jan 20 09:00:47 2017 | 25.1 \ir include/ddl_ops_1.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."Hypertable_1" ( time BIGINT NOT NULL, "Device_id" TEXT NOT NULL, temp_c int NOT NULL DEFAULT -1, humidity numeric NULL DEFAULT 0, sensor_1 NUMERIC NULL DEFAULT 1, sensor_2 NUMERIC NOT NULL DEFAULT 1, sensor_3 NUMERIC NOT NULL DEFAULT 1, sensor_4 NUMERIC NOT NULL DEFAULT 1 ); CREATE INDEX ON PUBLIC."Hypertable_1" (time, "Device_id"); CREATE TABLE "customSchema"."Hypertable_1" ( time BIGINT NOT NULL, "Device_id" TEXT NOT NULL, temp_c int NOT NULL DEFAULT -1, humidity numeric NULL DEFAULT 0, sensor_1 NUMERIC NULL DEFAULT 1, sensor_2 NUMERIC NOT NULL DEFAULT 1, sensor_3 NUMERIC NOT NULL DEFAULT 1, sensor_4 NUMERIC NOT NULL DEFAULT 1 ); CREATE INDEX ON "customSchema"."Hypertable_1" (time, "Device_id"); SELECT * FROM create_hypertable('"public"."Hypertable_1"', 'time', 'Device_id', 1, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+--------------+--------- 3 | public | Hypertable_1 | t SELECT * FROM create_hypertable('"customSchema"."Hypertable_1"', 'time', NULL, 1, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+--------------+--------------+--------- 4 | customSchema | Hypertable_1 | t SELECT * FROM _timescaledb_catalog.hypertable; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+--------------+---------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 1 | public | one_Partition | one_Partition | _hyper_1 | 1 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 2 | public | 1dim | _timescaledb_internal | _hyper_2 | 1 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 3 | public | Hypertable_1 | _timescaledb_internal | _hyper_3 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 4 | customSchema | Hypertable_1 | _timescaledb_internal | _hyper_4 | 1 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 CREATE INDEX ON PUBLIC."Hypertable_1" (time, "temp_c"); CREATE INDEX "ind_humidity" ON PUBLIC."Hypertable_1" (time, "humidity"); CREATE INDEX "ind_sensor_1" ON PUBLIC."Hypertable_1" (time, "sensor_1"); INSERT INTO PUBLIC."Hypertable_1"(time, "Device_id", temp_c, humidity, sensor_1, sensor_2, sensor_3, sensor_4) VALUES(1257894000000000000, 'dev1', 30, 70, 1, 2, 3, 100); CREATE UNIQUE INDEX "Unique1" ON PUBLIC."Hypertable_1" (time, "Device_id"); CREATE UNIQUE INDEX "Unique1" ON "customSchema"."Hypertable_1" (time); INSERT INTO "customSchema"."Hypertable_1"(time, "Device_id", temp_c, humidity, sensor_1, sensor_2, sensor_3, sensor_4) VALUES(1257894000000000000, 'dev1', 30, 70, 1, 2, 3, 100); INSERT INTO "customSchema"."Hypertable_1"(time, "Device_id", temp_c, humidity, sensor_1, sensor_2, sensor_3, sensor_4) VALUES(1257894000000000001, 'dev1', 30, 70, 1, 2, 3, 100); SELECT * FROM test.show_indexesp('%.%'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------+-----------------------------------------------------------------------------+--------------------------+------+--------+---------+-----------+------------ "one_Partition"._hyper_1_1_chunk | "one_Partition"."_hyper_1_1_chunk_one_Partition_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | "one_Partition"._hyper_1_1_chunk | "one_Partition"."_hyper_1_1_chunk_one_Partition_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | "one_Partition"._hyper_1_1_chunk | "one_Partition"."_hyper_1_1_chunk_one_Partition_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | "one_Partition"._hyper_1_1_chunk | "one_Partition"."_hyper_1_1_chunk_one_Partition_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | "one_Partition"._hyper_1_1_chunk | "one_Partition"."_hyper_1_1_chunk_one_Partition_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | "one_Partition"._hyper_1_1_chunk | "one_Partition"."_hyper_1_1_chunk_one_Partition_timeCustom_idx" | {timeCustom} | | f | f | f | "one_Partition"._hyper_1_2_chunk | "one_Partition"."_hyper_1_2_chunk_one_Partition_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | "one_Partition"._hyper_1_2_chunk | "one_Partition"."_hyper_1_2_chunk_one_Partition_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | "one_Partition"._hyper_1_2_chunk | "one_Partition"."_hyper_1_2_chunk_one_Partition_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | "one_Partition"._hyper_1_2_chunk | "one_Partition"."_hyper_1_2_chunk_one_Partition_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | "one_Partition"._hyper_1_2_chunk | "one_Partition"."_hyper_1_2_chunk_one_Partition_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | "one_Partition"._hyper_1_2_chunk | "one_Partition"."_hyper_1_2_chunk_one_Partition_timeCustom_idx" | {timeCustom} | | f | f | f | "one_Partition"._hyper_1_3_chunk | "one_Partition"."_hyper_1_3_chunk_one_Partition_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | "one_Partition"._hyper_1_3_chunk | "one_Partition"."_hyper_1_3_chunk_one_Partition_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | "one_Partition"._hyper_1_3_chunk | "one_Partition"."_hyper_1_3_chunk_one_Partition_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | "one_Partition"._hyper_1_3_chunk | "one_Partition"."_hyper_1_3_chunk_one_Partition_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | "one_Partition"._hyper_1_3_chunk | "one_Partition"."_hyper_1_3_chunk_one_Partition_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | "one_Partition"._hyper_1_3_chunk | "one_Partition"."_hyper_1_3_chunk_one_Partition_timeCustom_idx" | {timeCustom} | | f | f | f | "customSchema"."Hypertable_1" | "customSchema"."Hypertable_1_time_Device_id_idx" | {time,Device_id} | | f | f | f | "customSchema"."Hypertable_1" | "customSchema"."Hypertable_1_time_idx" | {time} | | f | f | f | "customSchema"."Hypertable_1" | "customSchema"."Unique1" | {time} | | t | f | f | SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper_%'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+--------------------------------------------------------------------------+------------------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_2_4_chunk | _timescaledb_internal._hyper_2_4_chunk_1dim_time_idx | {time} | | f | f | f | _timescaledb_internal._hyper_3_5_chunk | _timescaledb_internal."_hyper_3_5_chunk_Hypertable_1_time_Device_id_idx" | {time,Device_id} | | f | f | f | _timescaledb_internal._hyper_3_5_chunk | _timescaledb_internal."_hyper_3_5_chunk_Hypertable_1_time_idx" | {time} | | f | f | f | _timescaledb_internal._hyper_3_5_chunk | _timescaledb_internal."_hyper_3_5_chunk_Hypertable_1_Device_id_time_idx" | {Device_id,time} | | f | f | f | _timescaledb_internal._hyper_3_5_chunk | _timescaledb_internal."_hyper_3_5_chunk_Hypertable_1_time_temp_c_idx" | {time,temp_c} | | f | f | f | _timescaledb_internal._hyper_3_5_chunk | _timescaledb_internal._hyper_3_5_chunk_ind_humidity | {time,humidity} | | f | f | f | _timescaledb_internal._hyper_3_5_chunk | _timescaledb_internal._hyper_3_5_chunk_ind_sensor_1 | {time,sensor_1} | | f | f | f | _timescaledb_internal._hyper_3_5_chunk | _timescaledb_internal."_hyper_3_5_chunk_Unique1" | {time,Device_id} | | t | f | f | _timescaledb_internal._hyper_4_6_chunk | _timescaledb_internal."_hyper_4_6_chunk_Hypertable_1_time_Device_id_idx" | {time,Device_id} | | f | f | f | _timescaledb_internal._hyper_4_6_chunk | _timescaledb_internal."_hyper_4_6_chunk_Hypertable_1_time_idx" | {time} | | f | f | f | _timescaledb_internal._hyper_4_6_chunk | _timescaledb_internal."_hyper_4_6_chunk_Unique1" | {time} | | t | f | f | --expect error cases \set ON_ERROR_STOP 0 INSERT INTO "customSchema"."Hypertable_1"(time, "Device_id", temp_c, humidity, sensor_1, sensor_2, sensor_3, sensor_4) VALUES(1257894000000000000, 'dev1', 31, 71, 72, 4, 1, 102); psql:include/ddl_ops_1.sql:57: ERROR: duplicate key value violates unique constraint "_hyper_4_6_chunk_Unique1" CREATE UNIQUE INDEX "Unique2" ON PUBLIC."Hypertable_1" ("Device_id"); psql:include/ddl_ops_1.sql:58: ERROR: cannot create a unique index without the column "time" (used in partitioning) CREATE UNIQUE INDEX "Unique2" ON PUBLIC."Hypertable_1" (time); psql:include/ddl_ops_1.sql:59: ERROR: cannot create a unique index without the column "Device_id" (used in partitioning) CREATE UNIQUE INDEX "Unique2" ON PUBLIC."Hypertable_1" (sensor_1); psql:include/ddl_ops_1.sql:60: ERROR: cannot create a unique index without the column "time" (used in partitioning) UPDATE ONLY PUBLIC."Hypertable_1" SET time = 0 WHERE TRUE; DELETE FROM ONLY PUBLIC."Hypertable_1" WHERE "Device_id" = 'dev1'; \set ON_ERROR_STOP 1 CREATE TABLE my_ht (time BIGINT, val integer); SELECT * FROM create_hypertable('my_ht', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 5 | public | my_ht | t ALTER TABLE my_ht ADD COLUMN val2 integer; SELECT * FROM test.show_columns('my_ht'); Column | Type | NotNull --------+---------+--------- time | bigint | t val | integer | f val2 | integer | f -- Should error when adding again \set ON_ERROR_STOP 0 ALTER TABLE my_ht ADD COLUMN val2 integer; psql:include/ddl_ops_1.sql:73: ERROR: column "val2" of relation "my_ht" already exists \set ON_ERROR_STOP 1 -- Should create ALTER TABLE my_ht ADD COLUMN IF NOT EXISTS val3 integer; SELECT * FROM test.show_columns('my_ht'); Column | Type | NotNull --------+---------+--------- time | bigint | t val | integer | f val2 | integer | f val3 | integer | f -- Should skip and not error ALTER TABLE my_ht ADD COLUMN IF NOT EXISTS val3 integer; psql:include/ddl_ops_1.sql:81: NOTICE: column "val3" of relation "my_ht" already exists, skipping SELECT * FROM test.show_columns('my_ht'); Column | Type | NotNull --------+---------+--------- time | bigint | t val | integer | f val2 | integer | f val3 | integer | f -- Should drop ALTER TABLE my_ht DROP COLUMN IF EXISTS val3; SELECT * FROM test.show_columns('my_ht'); Column | Type | NotNull --------+---------+--------- time | bigint | t val | integer | f val2 | integer | f -- Should skip and not error ALTER TABLE my_ht DROP COLUMN IF EXISTS val3; psql:include/ddl_ops_1.sql:89: NOTICE: column "val3" of relation "my_ht" does not exist, skipping SELECT * FROM test.show_columns('my_ht'); Column | Type | NotNull --------+---------+--------- time | bigint | t val | integer | f val2 | integer | f --Test default index creation on create_hypertable(). --Make sure that we do not duplicate indexes that already exists -- --No existing indexes: both time and space-time indexes created BEGIN; CREATE TABLE PUBLIC."Hypertable_1_with_default_index_enabled" ( "Time" BIGINT NOT NULL, "Device_id" TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 ); SELECT * FROM create_hypertable('"public"."Hypertable_1_with_default_index_enabled"', 'Time', 'Device_id', 1, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+-----------------------------------------+--------- 6 | public | Hypertable_1_with_default_index_enabled | t SELECT * FROM test.show_indexes('"Hypertable_1_with_default_index_enabled"'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace --------------------------------------------------------------+------------------+------+--------+---------+-----------+------------ "Hypertable_1_with_default_index_enabled_Device_id_Time_idx" | {Device_id,Time} | | f | f | f | "Hypertable_1_with_default_index_enabled_Time_idx" | {Time} | | f | f | f | ROLLBACK; --Space index exists: only time index created BEGIN; CREATE TABLE PUBLIC."Hypertable_1_with_default_index_enabled" ( "Time" BIGINT NOT NULL, "Device_id" TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 ); CREATE INDEX ON PUBLIC."Hypertable_1_with_default_index_enabled" ("Device_id", "Time" DESC); SELECT * FROM create_hypertable('"public"."Hypertable_1_with_default_index_enabled"', 'Time', 'Device_id', 1, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+-----------------------------------------+--------- 7 | public | Hypertable_1_with_default_index_enabled | t SELECT * FROM test.show_indexes('"Hypertable_1_with_default_index_enabled"'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace --------------------------------------------------------------+------------------+------+--------+---------+-----------+------------ "Hypertable_1_with_default_index_enabled_Device_id_Time_idx" | {Device_id,Time} | | f | f | f | "Hypertable_1_with_default_index_enabled_Time_idx" | {Time} | | f | f | f | ROLLBACK; --Time index exists, only partition index created BEGIN; CREATE TABLE PUBLIC."Hypertable_1_with_default_index_enabled" ( "Time" BIGINT NOT NULL, "Device_id" TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 ); CREATE INDEX ON PUBLIC."Hypertable_1_with_default_index_enabled" ("Time" DESC); SELECT * FROM create_hypertable('"public"."Hypertable_1_with_default_index_enabled"', 'Time', 'Device_id', 1, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+-----------------------------------------+--------- 8 | public | Hypertable_1_with_default_index_enabled | t SELECT * FROM test.show_indexes('"Hypertable_1_with_default_index_enabled"'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace --------------------------------------------------------------+------------------+------+--------+---------+-----------+------------ "Hypertable_1_with_default_index_enabled_Device_id_Time_idx" | {Device_id,Time} | | f | f | f | "Hypertable_1_with_default_index_enabled_Time_idx" | {Time} | | f | f | f | ROLLBACK; --No space partitioning, only time index created BEGIN; CREATE TABLE PUBLIC."Hypertable_1_with_default_index_enabled" ( "Time" BIGINT NOT NULL, "Device_id" TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 ); SELECT * FROM create_hypertable('"public"."Hypertable_1_with_default_index_enabled"', 'Time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+-----------------------------------------+--------- 9 | public | Hypertable_1_with_default_index_enabled | t SELECT * FROM test.show_indexes('"Hypertable_1_with_default_index_enabled"'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------------------+---------+------+--------+---------+-----------+------------ "Hypertable_1_with_default_index_enabled_Time_idx" | {Time} | | f | f | f | ROLLBACK; --Disable index creation: no default indexes created BEGIN; CREATE TABLE PUBLIC."Hypertable_1_with_default_index_enabled" ( "Time" BIGINT NOT NULL, "Device_id" TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 ); SELECT * FROM create_hypertable('"public"."Hypertable_1_with_default_index_enabled"', 'Time', 'Device_id', 1, create_default_indexes=>FALSE, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+-----------------------------------------+--------- 10 | public | Hypertable_1_with_default_index_enabled | t SELECT * FROM test.show_indexes('"Hypertable_1_with_default_index_enabled"'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace -------+---------+------+--------+---------+-----------+------------ ROLLBACK; \ir include/ddl_ops_2.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. ALTER TABLE PUBLIC."Hypertable_1" ADD COLUMN temp_f INTEGER NOT NULL DEFAULT 31; ALTER TABLE PUBLIC."Hypertable_1" DROP COLUMN temp_c; ALTER TABLE PUBLIC."Hypertable_1" DROP COLUMN sensor_4; ALTER TABLE PUBLIC."Hypertable_1" ALTER COLUMN humidity SET DEFAULT 100; ALTER TABLE PUBLIC."Hypertable_1" ALTER COLUMN sensor_1 DROP DEFAULT; ALTER TABLE PUBLIC."Hypertable_1" ALTER COLUMN sensor_2 SET DEFAULT NULL; ALTER TABLE PUBLIC."Hypertable_1" ALTER COLUMN sensor_1 SET NOT NULL; ALTER TABLE PUBLIC."Hypertable_1" ALTER COLUMN sensor_2 DROP NOT NULL; ALTER TABLE PUBLIC."Hypertable_1" RENAME COLUMN sensor_2 TO sensor_2_renamed; ALTER TABLE PUBLIC."Hypertable_1" RENAME COLUMN sensor_3 TO sensor_3_renamed; DROP INDEX "ind_sensor_1"; CREATE OR REPLACE FUNCTION empty_trigger_func() RETURNS TRIGGER LANGUAGE PLPGSQL AS $BODY$ BEGIN END $BODY$; CREATE TRIGGER test_trigger BEFORE UPDATE OR DELETE ON PUBLIC."Hypertable_1" FOR EACH STATEMENT EXECUTE FUNCTION empty_trigger_func(); ALTER TABLE PUBLIC."Hypertable_1" ALTER COLUMN sensor_2_renamed SET DATA TYPE int; ALTER INDEX "ind_humidity" RENAME TO "ind_humdity2"; -- Change should be reflected here SELECT * FROM test.show_indexesp('%.%'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------+-----------------------------------------------------------------------------+--------------------------+------+--------+---------+-----------+------------ "one_Partition"._hyper_1_1_chunk | "one_Partition"."_hyper_1_1_chunk_one_Partition_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | "one_Partition"._hyper_1_1_chunk | "one_Partition"."_hyper_1_1_chunk_one_Partition_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | "one_Partition"._hyper_1_1_chunk | "one_Partition"."_hyper_1_1_chunk_one_Partition_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | "one_Partition"._hyper_1_1_chunk | "one_Partition"."_hyper_1_1_chunk_one_Partition_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | "one_Partition"._hyper_1_1_chunk | "one_Partition"."_hyper_1_1_chunk_one_Partition_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | "one_Partition"._hyper_1_1_chunk | "one_Partition"."_hyper_1_1_chunk_one_Partition_timeCustom_idx" | {timeCustom} | | f | f | f | "one_Partition"._hyper_1_2_chunk | "one_Partition"."_hyper_1_2_chunk_one_Partition_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | "one_Partition"._hyper_1_2_chunk | "one_Partition"."_hyper_1_2_chunk_one_Partition_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | "one_Partition"._hyper_1_2_chunk | "one_Partition"."_hyper_1_2_chunk_one_Partition_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | "one_Partition"._hyper_1_2_chunk | "one_Partition"."_hyper_1_2_chunk_one_Partition_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | "one_Partition"._hyper_1_2_chunk | "one_Partition"."_hyper_1_2_chunk_one_Partition_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | "one_Partition"._hyper_1_2_chunk | "one_Partition"."_hyper_1_2_chunk_one_Partition_timeCustom_idx" | {timeCustom} | | f | f | f | "one_Partition"._hyper_1_3_chunk | "one_Partition"."_hyper_1_3_chunk_one_Partition_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | "one_Partition"._hyper_1_3_chunk | "one_Partition"."_hyper_1_3_chunk_one_Partition_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | "one_Partition"._hyper_1_3_chunk | "one_Partition"."_hyper_1_3_chunk_one_Partition_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | "one_Partition"._hyper_1_3_chunk | "one_Partition"."_hyper_1_3_chunk_one_Partition_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | "one_Partition"._hyper_1_3_chunk | "one_Partition"."_hyper_1_3_chunk_one_Partition_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | "one_Partition"._hyper_1_3_chunk | "one_Partition"."_hyper_1_3_chunk_one_Partition_timeCustom_idx" | {timeCustom} | | f | f | f | "customSchema"."Hypertable_1" | "customSchema"."Hypertable_1_time_Device_id_idx" | {time,Device_id} | | f | f | f | "customSchema"."Hypertable_1" | "customSchema"."Hypertable_1_time_idx" | {time} | | f | f | f | "customSchema"."Hypertable_1" | "customSchema"."Unique1" | {time} | | t | f | f | SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+--------------------------------------------------------------------------+------------------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_2_4_chunk | _timescaledb_internal._hyper_2_4_chunk_1dim_time_idx | {time} | | f | f | f | _timescaledb_internal._hyper_3_5_chunk | _timescaledb_internal."_hyper_3_5_chunk_Hypertable_1_time_Device_id_idx" | {time,Device_id} | | f | f | f | _timescaledb_internal._hyper_3_5_chunk | _timescaledb_internal."_hyper_3_5_chunk_Hypertable_1_time_idx" | {time} | | f | f | f | _timescaledb_internal._hyper_3_5_chunk | _timescaledb_internal."_hyper_3_5_chunk_Hypertable_1_Device_id_time_idx" | {Device_id,time} | | f | f | f | _timescaledb_internal._hyper_3_5_chunk | _timescaledb_internal._hyper_3_5_chunk_ind_humdity2 | {time,humidity} | | f | f | f | _timescaledb_internal._hyper_3_5_chunk | _timescaledb_internal."_hyper_3_5_chunk_Unique1" | {time,Device_id} | | t | f | f | _timescaledb_internal._hyper_4_6_chunk | _timescaledb_internal."_hyper_4_6_chunk_Hypertable_1_time_Device_id_idx" | {time,Device_id} | | f | f | f | _timescaledb_internal._hyper_4_6_chunk | _timescaledb_internal."_hyper_4_6_chunk_Hypertable_1_time_idx" | {time} | | f | f | f | _timescaledb_internal._hyper_4_6_chunk | _timescaledb_internal."_hyper_4_6_chunk_Unique1" | {time} | | t | f | f | --create column with same name as previously renamed one ALTER TABLE PUBLIC."Hypertable_1" ADD COLUMN sensor_3 BIGINT NOT NULL DEFAULT 131; --create column with same name as previously dropped one ALTER TABLE PUBLIC."Hypertable_1" ADD COLUMN sensor_4 BIGINT NOT NULL DEFAULT 131; --test proper denials for all security definer functions: \c :TEST_DBNAME :ROLE_SUPERUSER CREATE TABLE plain_table_su (time timestamp, temp float); CREATE TABLE hypertable_su (time timestamp, temp float); SELECT create_hypertable('hypertable_su', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ----------------------------- (11,public,hypertable_su,t) CREATE INDEX "ind_1" ON hypertable_su (time); INSERT INTO hypertable_su VALUES('2017-01-20T09:00:01', 22.5); \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 --all of the following should produce errors \set ON_ERROR_STOP 0 SELECT create_hypertable('plain_table_su', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices ERROR: must be owner of hypertable "plain_table_su" CREATE INDEX ON plain_table_su (time, temp); ERROR: must be owner of table plain_table_su CREATE INDEX ON hypertable_su (time, temp); ERROR: must be owner of hypertable "hypertable_su" DROP INDEX "ind_1"; ERROR: must be owner of index ind_1 ALTER INDEX "ind_1" RENAME TO "ind_2"; ERROR: must be owner of index ind_1 \set ON_ERROR_STOP 1 --test that I can't do anything to a non-owned hypertable. \set ON_ERROR_STOP 0 CREATE INDEX ON hypertable_su (time, temp); ERROR: must be owner of hypertable "hypertable_su" SELECT * FROM hypertable_su; ERROR: permission denied for table hypertable_su INSERT INTO hypertable_su VALUES('2017-01-20T09:00:01', 22.5); ERROR: permission denied for table hypertable_su ALTER TABLE hypertable_su ADD COLUMN val2 integer; ERROR: must be owner of table hypertable_su \set ON_ERROR_STOP 1 --grant read permissions \c :TEST_DBNAME :ROLE_SUPERUSER GRANT SELECT ON hypertable_su TO :ROLE_DEFAULT_PERM_USER_2; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 SELECT * FROM hypertable_su; time | temp --------------------------+------ Fri Jan 20 09:00:01 2017 | 22.5 \set ON_ERROR_STOP 0 CREATE INDEX ON hypertable_su (time, temp); ERROR: must be owner of hypertable "hypertable_su" INSERT INTO hypertable_su VALUES('2017-01-20T09:00:01', 22.5); ERROR: permission denied for table hypertable_su ALTER TABLE hypertable_su ADD COLUMN val2 integer; ERROR: must be owner of table hypertable_su \set ON_ERROR_STOP 1 --grant read, insert permissions \c :TEST_DBNAME :ROLE_SUPERUSER GRANT SELECT, INSERT ON hypertable_su TO :ROLE_DEFAULT_PERM_USER_2; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 INSERT INTO hypertable_su VALUES('2017-01-20T09:00:01', 22.5); SELECT * FROM hypertable_su; time | temp --------------------------+------ Fri Jan 20 09:00:01 2017 | 22.5 Fri Jan 20 09:00:01 2017 | 22.5 \set ON_ERROR_STOP 0 CREATE INDEX ON hypertable_su (time, temp); ERROR: must be owner of hypertable "hypertable_su" ALTER TABLE hypertable_su ADD COLUMN val2 integer; ERROR: must be owner of table hypertable_su \set ON_ERROR_STOP 1 --change owner \c :TEST_DBNAME :ROLE_SUPERUSER ALTER TABLE hypertable_su OWNER TO :ROLE_DEFAULT_PERM_USER_2; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 INSERT INTO hypertable_su VALUES('2017-01-20T09:00:01', 22.5); SELECT * FROM hypertable_su; time | temp --------------------------+------ Fri Jan 20 09:00:01 2017 | 22.5 Fri Jan 20 09:00:01 2017 | 22.5 Fri Jan 20 09:00:01 2017 | 22.5 CREATE INDEX ON hypertable_su (time, temp); ALTER TABLE hypertable_su ADD COLUMN val2 integer; ================================================ FILE: test/expected/append-15.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set TEST_BASE_NAME append SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') as "TEST_LOAD_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') as "TEST_QUERY_NAME", format('%s/results/%s_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_OPTIMIZED", format('%s/results/%s_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_UNOPTIMIZED" \gset SELECT format('\! diff -u --label "Unoptimized results" --label "Optimized results" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') as "DIFF_CMD" \gset SET timescaledb.enable_now_constify TO false; -- disable memoize node to avoid flaky results SET enable_memoize TO 'off'; -- disable index only scans to avoid some flaky results SET enable_indexonlyscan TO FALSE; \set PREFIX 'EXPLAIN (analyze, buffers off, costs off, timing off, summary off)' \ir :TEST_LOAD_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- create a now() function for repeatable testing that always returns -- the same timestamp. It needs to be marked STABLE CREATE OR REPLACE FUNCTION now_s() RETURNS timestamptz LANGUAGE PLPGSQL STABLE PARALLEL SAFE AS $BODY$ BEGIN RAISE NOTICE 'Stable function now_s() called!'; RETURN '2017-08-22T10:00:00'::timestamptz; END; $BODY$; CREATE OR REPLACE FUNCTION now_i() RETURNS timestamptz LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RAISE NOTICE 'Immutable function now_i() called!'; RETURN '2017-08-22T10:00:00'::timestamptz; END; $BODY$; CREATE OR REPLACE FUNCTION now_v() RETURNS timestamptz LANGUAGE PLPGSQL VOLATILE AS $BODY$ BEGIN RAISE NOTICE 'Volatile function now_v() called!'; RETURN '2017-08-22T10:00:00'::timestamptz; END; $BODY$; CREATE OR REPLACE PROCEDURE force_parallel(on_or_off bool) LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('server_version_num')::int < 160000 THEN IF on_or_off THEN set force_parallel_mode = 'on'; ELSE set force_parallel_mode = 'off'; END IF; ELSE IF on_or_off THEN set debug_parallel_query = 'on'; ELSE set debug_parallel_query = 'off'; END IF; END IF; END; $$; CREATE TABLE append_test(time timestamptz, temp float, colorid integer, attr jsonb); SELECT create_hypertable('append_test', 'time', chunk_time_interval => 2628000000000); create_hypertable -------------------------- (1,public,append_test,t) -- create three chunks INSERT INTO append_test VALUES ('2017-03-22T09:18:22', 23.5, 1, '{"a": 1, "b": 2}'), ('2017-03-22T09:18:23', 21.5, 1, '{"a": 1, "b": 2}'), ('2017-05-22T09:18:22', 36.2, 2, '{"c": 3, "b": 2}'), ('2017-05-22T09:18:23', 15.2, 2, '{"c": 3}'), ('2017-08-22T09:18:22', 34.1, 3, '{"c": 4}'); VACUUM (ANALYZE) append_test; -- Create another hypertable to join with CREATE TABLE join_test(time timestamptz, temp float, colorid integer); SELECT create_hypertable('join_test', 'time', chunk_time_interval => 2628000000000); create_hypertable ------------------------ (2,public,join_test,t) INSERT INTO join_test VALUES ('2017-01-22T09:18:22', 15.2, 1), ('2017-02-22T09:18:22', 24.5, 2), ('2017-08-22T09:18:22', 23.1, 3); VACUUM (ANALYZE) join_test; -- Create another table to join with which is not a hypertable. CREATE TABLE join_test_plain(time timestamptz, temp float, colorid integer, attr jsonb); INSERT INTO join_test_plain VALUES ('2017-01-22T09:18:22', 15.2, 1, '{"a": 1}'), ('2017-02-22T09:18:22', 24.5, 2, '{"b": 2}'), ('2017-08-22T09:18:22', 23.1, 3, '{"c": 3}'); VACUUM (ANALYZE) join_test_plain; -- create hypertable with DATE time dimension CREATE TABLE metrics_date(time DATE NOT NULL); SELECT create_hypertable('metrics_date','time'); create_hypertable --------------------------- (3,public,metrics_date,t) INSERT INTO metrics_date SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval); VACUUM (ANALYZE) metrics_date; -- create hypertable with TIMESTAMP time dimension CREATE TABLE metrics_timestamp(time TIMESTAMP NOT NULL); SELECT create_hypertable('metrics_timestamp','time'); psql:include/append_load.sql:91: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------------- (4,public,metrics_timestamp,t) INSERT INTO metrics_timestamp SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval); VACUUM (ANALYZE) metrics_timestamp; -- create hypertable with TIMESTAMPTZ time dimension CREATE TABLE metrics_timestamptz(time TIMESTAMPTZ NOT NULL, device_id INT NOT NULL); CREATE INDEX ON metrics_timestamptz(device_id,time); SELECT create_hypertable('metrics_timestamptz','time'); create_hypertable ---------------------------------- (5,public,metrics_timestamptz,t) INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval), 1; INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval), 2; INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval), 3; VACUUM (ANALYZE) metrics_timestamptz; -- create space partitioned hypertable CREATE TABLE metrics_space(time timestamptz NOT NULL, device_id int NOT NULL, v1 float, v2 float, v3 text); SELECT create_hypertable('metrics_space','time','device_id',3); create_hypertable ---------------------------- (6,public,metrics_space,t) INSERT INTO metrics_space SELECT time, device_id, device_id + 0.25, device_id + 0.75, device_id FROM generate_series('2000-01-01'::timestamptz, '2000-01-14'::timestamptz, '5m'::interval) g1(time), generate_series(1,10,1) g2(device_id) ORDER BY time, device_id; VACUUM (ANALYZE) metrics_space; -- test ChunkAppend projection #2661 CREATE TABLE i2661 ( machine_id int4 NOT NULL, "name" varchar(255) NOT NULL, "timestamp" timestamptz NOT NULL, "first" float4 NULL ); SELECT create_hypertable('i2661', 'timestamp'); psql:include/append_load.sql:123: WARNING: column type "character varying" used for "name" does not follow best practices create_hypertable -------------------- (7,public,i2661,t) INSERT INTO i2661 SELECT 1, 'speed', generate_series('2019-12-31 00:00:00', '2020-01-10 00:00:00', '2m'::interval), 0; VACUUM (ANALYZE) i2661; \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be the only output of the results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations'),('timescaledb.enable_chunk_append')) v(setting); setting | value ----------------------------------+------- timescaledb.enable_optimizations | on timescaledb.enable_chunk_append | on -- query should exclude all chunks with optimization on :PREFIX SELECT * FROM append_test WHERE time > now_s() + '1 month' ORDER BY time DESC; psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Sort (actual rows=0.00 loops=1) Sort Key: append_test."time" DESC Sort Method: quicksort -> Custom Scan (ChunkAppend) on append_test (actual rows=0.00 loops=1) Chunks excluded during startup: 3 --query should exclude all chunks and be a MergeAppend :PREFIX SELECT * FROM append_test WHERE time > now_s() + '1 month' ORDER BY time DESC limit 1; psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Limit (actual rows=0.00 loops=1) -> Custom Scan (ChunkAppend) on append_test (actual rows=0.00 loops=1) Order: append_test."time" DESC Chunks excluded during startup: 3 -- when optimized, the plan should be a constraint-aware append and -- cover only one chunk. It should be a backward index scan due to -- descending index on time. Should also skip the main table, since it -- cannot hold tuples :PREFIX SELECT * FROM append_test WHERE time > now_s() - interval '2 months'; psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Custom Scan (ChunkAppend) on append_test (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 2 mons'::interval)) -- adding ORDER BY and LIMIT should turn the plan into an optimized -- ordered append plan :PREFIX SELECT * FROM append_test WHERE time > now_s() - interval '2 months' ORDER BY time LIMIT 3; psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Sort (actual rows=1.00 loops=1) Sort Key: append_test."time" Sort Method: quicksort -> Custom Scan (ChunkAppend) on append_test (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 2 mons'::interval)) -- no optimized plan for queries with restrictions that can be -- constified at planning time. Regular planning-time constraint -- exclusion should occur. :PREFIX SELECT * FROM append_test WHERE time > now_i() - interval '2 months' ORDER BY time; psql:include/append_query.sql:37: NOTICE: Immutable function now_i() called! --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: append_test."time" Sort Method: quicksort -> Custom Scan (ChunkAppend) on append_test (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > ('Tue Aug 22 10:00:00 2017 PDT'::timestamp with time zone - '@ 2 mons'::interval)) -- currently, we cannot distinguish between stable and volatile -- functions as far as applying our modified plan. However, volatile -- function should not be pre-evaluated to constants, so no chunk -- exclusion should occur. :PREFIX SELECT * FROM append_test WHERE time > now_v() - interval '2 months' ORDER BY time; psql:include/append_query.sql:45: NOTICE: Volatile function now_v() called! psql:include/append_query.sql:45: NOTICE: Volatile function now_v() called! psql:include/append_query.sql:45: NOTICE: Volatile function now_v() called! psql:include/append_query.sql:45: NOTICE: Volatile function now_v() called! psql:include/append_query.sql:45: NOTICE: Volatile function now_v() called! --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: append_test."time" Sort Method: quicksort -> Custom Scan (ChunkAppend) on append_test (actual rows=1.00 loops=1) Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk (actual rows=0.00 loops=1) Filter: ("time" > (now_v() - '@ 2 mons'::interval)) Rows Removed by Filter: 2 -> Seq Scan on _hyper_1_2_chunk (actual rows=0.00 loops=1) Filter: ("time" > (now_v() - '@ 2 mons'::interval)) Rows Removed by Filter: 2 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_v() - '@ 2 mons'::interval)) -- prepared statement output should be the same regardless of -- optimizations PREPARE query_opt AS SELECT * FROM append_test WHERE time > now_s() - interval '2 months' ORDER BY time; :PREFIX EXECUTE query_opt; psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: append_test."time" Sort Method: quicksort -> Custom Scan (ChunkAppend) on append_test (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 2 mons'::interval)) DEALLOCATE query_opt; -- aggregates should produce same output :PREFIX SELECT date_trunc('year', time) t, avg(temp) FROM append_test WHERE time > now_s() - interval '4 months' GROUP BY t ORDER BY t DESC; psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! --- QUERY PLAN --- GroupAggregate (actual rows=1.00 loops=1) Group Key: (date_trunc('year'::text, append_test."time")) -> Sort (actual rows=3.00 loops=1) Sort Key: (date_trunc('year'::text, append_test."time")) DESC Sort Method: quicksort -> Result (actual rows=3.00 loops=1) -> Custom Scan (ChunkAppend) on append_test (actual rows=3.00 loops=1) Chunks excluded during startup: 1 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 4 mons'::interval)) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) Filter: ("time" > (now_s() - '@ 4 mons'::interval)) -- querying outside the time range should return nothing. This tests -- that ConstraintAwareAppend can handle the case when an Append node -- is turned into a Result node due to no children :PREFIX SELECT date_trunc('year', time) t, avg(temp) FROM append_test WHERE time < '2016-03-22' AND date_part('dow', time) between 1 and 5 GROUP BY t ORDER BY t DESC; --- QUERY PLAN --- GroupAggregate (actual rows=0.00 loops=1) Group Key: (date_trunc('year'::text, "time")) -> Sort (actual rows=0.00 loops=1) Sort Key: (date_trunc('year'::text, "time")) DESC Sort Method: quicksort -> Result (actual rows=0.00 loops=1) One-Time Filter: false -- a parameterized query can safely constify params, so won't be -- optimized by constraint-aware append since regular constraint -- exclusion works just fine PREPARE query_param AS SELECT * FROM append_test WHERE time > $1 ORDER BY time; :PREFIX EXECUTE query_param(now_s() - interval '2 months'); psql:include/append_query.sql:82: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: _hyper_1_3_chunk."time" Sort Method: quicksort -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) DEALLOCATE query_param; --test with cte :PREFIX WITH data AS ( SELECT time_bucket(INTERVAL '30 day', TIME) AS btime, AVG(temp) AS VALUE FROM append_test WHERE TIME > now_s() - INTERVAL '400 day' AND colorid > 0 GROUP BY btime ), period AS ( SELECT time_bucket(INTERVAL '30 day', TIME) AS btime FROM GENERATE_SERIES('2017-03-22T01:01:01', '2017-08-23T01:01:01', INTERVAL '30 day') TIME ) SELECT period.btime, VALUE FROM period LEFT JOIN DATA USING (btime) ORDER BY period.btime; psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Sort (actual rows=6.00 loops=1) Sort Key: (time_bucket('@ 30 days'::interval, "time"."time")) Sort Method: quicksort -> Hash Left Join (actual rows=6.00 loops=1) Hash Cond: (time_bucket('@ 30 days'::interval, "time"."time") = data.btime) -> Function Scan on generate_series "time" (actual rows=6.00 loops=1) -> Hash (actual rows=3.00 loops=1) -> Subquery Scan on data (actual rows=3.00 loops=1) -> HashAggregate (actual rows=3.00 loops=1) Group Key: time_bucket('@ 30 days'::interval, append_test."time") -> Result (actual rows=5.00 loops=1) -> Custom Scan (ChunkAppend) on append_test (actual rows=5.00 loops=1) Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk (actual rows=2.00 loops=1) Filter: ((colorid > 0) AND ("time" > (now_s() - '@ 400 days'::interval))) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) Filter: ((colorid > 0) AND ("time" > (now_s() - '@ 400 days'::interval))) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ((colorid > 0) AND ("time" > (now_s() - '@ 400 days'::interval))) WITH data AS ( SELECT time_bucket(INTERVAL '30 day', TIME) AS btime, AVG(temp) AS VALUE FROM append_test WHERE TIME > now_s() - INTERVAL '400 day' AND colorid > 0 GROUP BY btime ), period AS ( SELECT time_bucket(INTERVAL '30 day', TIME) AS btime FROM GENERATE_SERIES('2017-03-22T01:01:01', '2017-08-23T01:01:01', INTERVAL '30 day') TIME ) SELECT period.btime, VALUE FROM period LEFT JOIN DATA USING (btime) ORDER BY period.btime; psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! btime | value ------------------------------+------- Fri Mar 03 16:00:00 2017 PST | 22.5 Sun Apr 02 17:00:00 2017 PDT | Tue May 02 17:00:00 2017 PDT | 25.7 Thu Jun 01 17:00:00 2017 PDT | Sat Jul 01 17:00:00 2017 PDT | Mon Jul 31 17:00:00 2017 PDT | 34.1 -- force nested loop join with no materialization. This tests that the -- inner ConstraintAwareScan supports resetting its scan for every -- iteration of the outer relation loop set enable_hashjoin = 'off'; set enable_mergejoin = 'off'; set enable_material = 'off'; :PREFIX SELECT * FROM append_test a INNER JOIN join_test j ON (a.colorid = j.colorid) WHERE a.time > now_s() - interval '3 hours' AND j.time > now_s() - interval '3 hours'; psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Nested Loop (actual rows=1.00 loops=1) Join Filter: (a.colorid = j.colorid) -> Custom Scan (ChunkAppend) on append_test a (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_1_3_chunk a_1 (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 3 hours'::interval)) -> Custom Scan (ChunkAppend) on join_test j (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_2_6_chunk j_1 (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 3 hours'::interval)) reset enable_hashjoin; reset enable_mergejoin; reset enable_material; -- test constraint_exclusion with date time dimension and DATE/TIMESTAMP/TIMESTAMPTZ constraints -- the queries should all have 3 chunks :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -- test Const OP Var -- the queries should all have 3 chunks :PREFIX SELECT * FROM metrics_date WHERE '2000-01-15'::date < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_date WHERE '2000-01-15'::timestamp < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_date WHERE '2000-01-15'::timestamptz < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -- test 2 constraints -- the queries should all have 2 chunks :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::date AND time < '2000-01-21'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=1440.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::timestamp AND time < '2000-01-21'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=1440.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::timestamptz AND time < '2000-01-21'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=1440.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -- test constraint_exclusion with timestamp time dimension and DATE/TIMESTAMP/TIMESTAMPTZ constraints -- the queries should all have 3 chunks :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -- test Const OP Var -- the queries should all have 3 chunks :PREFIX SELECT * FROM metrics_timestamp WHERE '2000-01-15'::date < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_timestamp WHERE '2000-01-15'::timestamp < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_timestamp WHERE '2000-01-15'::timestamptz < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -- test 2 constraints -- the queries should all have 2 chunks :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::date AND time < '2000-01-21'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=1727.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::timestamp AND time < '2000-01-21'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=1727.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::timestamptz AND time < '2000-01-21'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=1727.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -- test constraint_exclusion with timestamptz time dimension and DATE/TIMESTAMP/TIMESTAMPTZ constraints -- the queries should all have 3 chunks :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) -- test Const OP Var -- the queries should all have 3 chunks :PREFIX SELECT time FROM metrics_timestamptz WHERE '2000-01-15'::date < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) :PREFIX SELECT time FROM metrics_timestamptz WHERE '2000-01-15'::timestamp < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT time FROM metrics_timestamptz WHERE '2000-01-15'::timestamptz < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) -- test 2 constraints -- the queries should all have 2 chunks :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::date AND time < '2000-01-21'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=5181.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::timestamp AND time < '2000-01-21'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=5181.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::timestamptz AND time < '2000-01-21'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=5181.00 loops=1) Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -- test constraint_exclusion with space partitioning and DATE/TIMESTAMP/TIMESTAMPTZ constraints -- exclusion for constraints with non-matching datatypes not working for space partitioning atm :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -- test Const OP Var -- exclusion for constraints with non-matching datatypes not working for space partitioning atm :PREFIX SELECT time FROM metrics_space WHERE '2000-01-10'::date < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) :PREFIX SELECT time FROM metrics_space WHERE '2000-01-10'::timestamp < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT time FROM metrics_space WHERE '2000-01-10'::timestamptz < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -- test 2 constraints -- exclusion for constraints with non-matching datatypes not working for space partitioning atm :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::date AND time < '2000-01-15'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamp AND time < '2000-01-15'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND time < '2000-01-15'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -- test filtering on space partition :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND device_id = 1 ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=1152.00 loops=1) Order: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_device_id_time_idx on _hyper_6_25_chunk (actual rows=767.00 loops=1) Index Cond: ((device_id = 1) AND ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_device_id_time_idx on _hyper_6_28_chunk (actual rows=385.00 loops=1) Index Cond: ((device_id = 1) AND ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND device_id IN (1,2) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=2304.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=1534.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=767.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = ANY ('{1,2}'::integer[])) Rows Removed by Filter: 2301 -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=767.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = ANY ('{1,2}'::integer[])) Rows Removed by Filter: 2301 -> Merge Append (actual rows=770.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=385.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = ANY ('{1,2}'::integer[])) Rows Removed by Filter: 1155 -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=385.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = ANY ('{1,2}'::integer[])) Rows Removed by Filter: 1155 :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND device_id IN (VALUES(1)) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=1152.00 loops=1) Order: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_device_id_time_idx on _hyper_6_25_chunk (actual rows=767.00 loops=1) Index Cond: ((device_id = 1) AND ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_device_id_time_idx on _hyper_6_28_chunk (actual rows=385.00 loops=1) Index Cond: ((device_id = 1) AND ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND v3 IN (VALUES('1')) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=1152.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=767.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=767.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 2301 -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 3068 -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 1534 -> Merge Append (actual rows=385.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=385.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 1155 -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 1540 -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 770 :PREFIX SELECT * FROM metrics_space WHERE time = (VALUES ('2019-12-24' at time zone 'UTC')) AND v3 NOT IN (VALUES ('1')); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=0.00 loops=1) Chunks excluded during startup: 0 Chunks excluded during runtime: 9 InitPlan 1 (returns $0) -> Result (actual rows=1.00 loops=1) -> Index Scan using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (never executed) Index Cond: ("time" = $0) Filter: (NOT (hashed SubPlan 2)) SubPlan 2 -> Result (never executed) -> Index Scan using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (never executed) Index Cond: ("time" = $0) Filter: (NOT (hashed SubPlan 2)) -> Index Scan using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (never executed) Index Cond: ("time" = $0) Filter: (NOT (hashed SubPlan 2)) -> Index Scan using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (never executed) Index Cond: ("time" = $0) Filter: (NOT (hashed SubPlan 2)) -> Index Scan using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (never executed) Index Cond: ("time" = $0) Filter: (NOT (hashed SubPlan 2)) -> Index Scan using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (never executed) Index Cond: ("time" = $0) Filter: (NOT (hashed SubPlan 2)) -> Index Scan using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (never executed) Index Cond: ("time" = $0) Filter: (NOT (hashed SubPlan 2)) -> Index Scan using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (never executed) Index Cond: ("time" = $0) Filter: (NOT (hashed SubPlan 2)) -> Index Scan using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (never executed) Index Cond: ("time" = $0) Filter: (NOT (hashed SubPlan 2)) -- test CURRENT_DATE -- should be 0 chunks :PREFIX SELECT time FROM metrics_date WHERE time > CURRENT_DATE ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=0.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamp WHERE time > CURRENT_DATE ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=0.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamptz WHERE time > CURRENT_DATE ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=0.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_space WHERE time > CURRENT_DATE ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=0.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -- test CURRENT_TIMESTAMP -- should be 0 chunks :PREFIX SELECT time FROM metrics_date WHERE time > CURRENT_TIMESTAMP ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=0.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamp WHERE time > CURRENT_TIMESTAMP ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=0.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamptz WHERE time > CURRENT_TIMESTAMP ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=0.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_space WHERE time > CURRENT_TIMESTAMP ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=0.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -- test now() -- should be 0 chunks :PREFIX SELECT time FROM metrics_date WHERE time > now() ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=0.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamp WHERE time > now() ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=0.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamptz WHERE time > now() ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=0.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_space WHERE time > now() ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=0.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -- query with tablesample and planner exclusion :PREFIX SELECT * FROM metrics_date TABLESAMPLE BERNOULLI(5) REPEATABLE(0) WHERE time > '2000-01-15' ORDER BY time DESC; --- QUERY PLAN --- Sort (actual rows=217.00 loops=1) Sort Key: metrics_date."time" DESC Sort Method: quicksort -> Append (actual rows=217.00 loops=1) -> Sample Scan on _hyper_3_11_chunk (actual rows=72.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) -> Sample Scan on _hyper_3_10_chunk (actual rows=94.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) -> Sample Scan on _hyper_3_9_chunk (actual rows=51.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > '01-15-2000'::date) Rows Removed by Filter: 43 -- query with tablesample and startup exclusion :PREFIX SELECT * FROM metrics_date TABLESAMPLE BERNOULLI(5) REPEATABLE(0) WHERE time > '2000-01-15'::text::date ORDER BY time DESC; --- QUERY PLAN --- Sort (actual rows=217.00 loops=1) Sort Key: metrics_date."time" DESC Sort Method: quicksort -> Custom Scan (ChunkAppend) on metrics_date (actual rows=217.00 loops=1) Chunks excluded during startup: 2 -> Sample Scan on _hyper_3_11_chunk (actual rows=72.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > ('2000-01-15'::cstring)::date) -> Sample Scan on _hyper_3_10_chunk (actual rows=94.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > ('2000-01-15'::cstring)::date) -> Sample Scan on _hyper_3_9_chunk (actual rows=51.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > ('2000-01-15'::cstring)::date) Rows Removed by Filter: 43 -- query with tablesample, space partitioning and planner exclusion :PREFIX SELECT * FROM metrics_space TABLESAMPLE BERNOULLI(5) REPEATABLE(0) WHERE time > '2000-01-10'::timestamptz ORDER BY time DESC, device_id; --- QUERY PLAN --- Sort (actual rows=522.00 loops=1) Sort Key: metrics_space."time" DESC, metrics_space.device_id Sort Method: quicksort -> Append (actual rows=522.00 loops=1) -> Sample Scan on _hyper_6_30_chunk (actual rows=35.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Sample Scan on _hyper_6_29_chunk (actual rows=61.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Sample Scan on _hyper_6_28_chunk (actual rows=61.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Sample Scan on _hyper_6_27_chunk (actual rows=65.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Rows Removed by Filter: 113 -> Sample Scan on _hyper_6_26_chunk (actual rows=150.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Rows Removed by Filter: 218 -> Sample Scan on _hyper_6_25_chunk (actual rows=150.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Rows Removed by Filter: 218 -- test runtime exclusion -- test runtime exclusion with LATERAL and 2 hypertables :PREFIX SELECT m1.time, m2.time FROM metrics_timestamptz m1 LEFT JOIN LATERAL(SELECT time FROM metrics_timestamptz m2 WHERE m1.time = m2.time LIMIT 1) m2 ON true ORDER BY m1.time; --- QUERY PLAN --- Nested Loop Left Join (actual rows=26787.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=26787.00 loops=1) Order: m1."time" -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m1_1 (actual rows=4032.00 loops=1) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_2 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m1_3 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m1_4 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m1_5 (actual rows=4611.00 loops=1) -> Limit (actual rows=1.00 loops=26787) -> Custom Scan (ChunkAppend) on metrics_timestamptz m2 (actual rows=1.00 loops=26787) Chunks excluded during runtime: 4 -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m2_1 (actual rows=1.00 loops=4032) Index Cond: ("time" = m1."time") -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m2_2 (actual rows=1.00 loops=6048) Index Cond: ("time" = m1."time") -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m2_3 (actual rows=1.00 loops=6048) Index Cond: ("time" = m1."time") -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m2_4 (actual rows=1.00 loops=6048) Index Cond: ("time" = m1."time") -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m2_5 (actual rows=1.00 loops=4611) Index Cond: ("time" = m1."time") -- test runtime exclusion and startup exclusions :PREFIX SELECT m1.time, m2.time FROM metrics_timestamptz m1 LEFT JOIN LATERAL(SELECT time FROM metrics_timestamptz m2 WHERE m1.time = m2.time AND m2.time < '2000-01-10'::text::timestamptz LIMIT 1) m2 ON true ORDER BY m1.time; --- QUERY PLAN --- Nested Loop Left Join (actual rows=26787.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=26787.00 loops=1) Order: m1."time" -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m1_1 (actual rows=4032.00 loops=1) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_2 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m1_3 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m1_4 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m1_5 (actual rows=4611.00 loops=1) -> Limit (actual rows=0.00 loops=26787) -> Custom Scan (ChunkAppend) on metrics_timestamptz m2 (actual rows=0.00 loops=26787) Chunks excluded during startup: 3 Chunks excluded during runtime: 1 -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m2_1 (actual rows=1.00 loops=4032) Index Cond: (("time" < ('2000-01-10'::cstring)::timestamp with time zone) AND ("time" = m1."time")) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m2_2 (actual rows=1.00 loops=6048) Index Cond: (("time" < ('2000-01-10'::cstring)::timestamp with time zone) AND ("time" = m1."time")) -- test runtime exclusion does not activate for constraints on non-partitioning columns -- should not use runtime exclusion :PREFIX SELECT * FROM append_test a LEFT JOIN LATERAL(SELECT * FROM join_test j WHERE a.colorid = j.colorid ORDER BY time DESC LIMIT 1) j ON true ORDER BY a.time LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Nested Loop Left Join (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on append_test a (actual rows=1.00 loops=1) Order: a."time" -> Index Scan Backward using _hyper_1_1_chunk_append_test_time_idx on _hyper_1_1_chunk a_1 (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_2_chunk_append_test_time_idx on _hyper_1_2_chunk a_2 (never executed) -> Index Scan Backward using _hyper_1_3_chunk_append_test_time_idx on _hyper_1_3_chunk a_3 (never executed) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on join_test j (actual rows=1.00 loops=1) Order: j."time" DESC Hypertables excluded during runtime: 0 -> Index Scan using _hyper_2_6_chunk_join_test_time_idx on _hyper_2_6_chunk j_1 (actual rows=0.00 loops=1) Filter: (a.colorid = colorid) Rows Removed by Filter: 1 -> Index Scan using _hyper_2_5_chunk_join_test_time_idx on _hyper_2_5_chunk j_2 (actual rows=0.00 loops=1) Filter: (a.colorid = colorid) Rows Removed by Filter: 1 -> Index Scan using _hyper_2_4_chunk_join_test_time_idx on _hyper_2_4_chunk j_3 (actual rows=1.00 loops=1) Filter: (a.colorid = colorid) -- test runtime exclusion with LATERAL and generate_series :PREFIX SELECT g.time FROM generate_series('2000-01-01'::timestamptz, '2000-02-01'::timestamptz, '1d'::interval) g(time) LEFT JOIN LATERAL(SELECT time FROM metrics_timestamptz m WHERE m.time=g.time LIMIT 1) m ON true; --- QUERY PLAN --- Nested Loop Left Join (actual rows=32.00 loops=1) -> Function Scan on generate_series g (actual rows=32.00 loops=1) -> Limit (actual rows=1.00 loops=32) -> Result (actual rows=1.00 loops=32) -> Custom Scan (ChunkAppend) on metrics_timestamptz m (actual rows=1.00 loops=32) Chunks excluded during runtime: 4 -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m_1 (actual rows=1.00 loops=5) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m_2 (actual rows=1.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m_3 (actual rows=1.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m_4 (actual rows=1.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m_5 (actual rows=1.00 loops=6) Index Cond: ("time" = g."time") :PREFIX SELECT * FROM generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval) AS g(time) INNER JOIN LATERAL (SELECT time FROM metrics_timestamptz m WHERE time=g.time) m ON true; --- QUERY PLAN --- Hash Join (actual rows=96.00 loops=1) Hash Cond: (g."time" = m."time") -> Function Scan on generate_series g (actual rows=32.00 loops=1) -> Hash (actual rows=26787.00 loops=1) -> Append (actual rows=26787.00 loops=1) -> Seq Scan on _hyper_5_17_chunk m_1 (actual rows=4032.00 loops=1) -> Seq Scan on _hyper_5_18_chunk m_2 (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_19_chunk m_3 (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_20_chunk m_4 (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_21_chunk m_5 (actual rows=4611.00 loops=1) :PREFIX SELECT * FROM generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval) AS g(time) INNER JOIN LATERAL (SELECT time FROM metrics_timestamptz m WHERE time=g.time ORDER BY time) m ON true; --- QUERY PLAN --- Nested Loop (actual rows=96.00 loops=1) -> Function Scan on generate_series g (actual rows=32.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m (actual rows=3.00 loops=32) Chunks excluded during runtime: 4 -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m_1 (actual rows=3.00 loops=5) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m_2 (actual rows=3.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m_3 (actual rows=3.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m_4 (actual rows=3.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m_5 (actual rows=3.00 loops=6) Index Cond: ("time" = g."time") :PREFIX SELECT * FROM generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval) AS g(time) INNER JOIN LATERAL (SELECT time FROM metrics_timestamptz m WHERE time>g.time + '1 day' ORDER BY time LIMIT 1) m ON true; --- QUERY PLAN --- Nested Loop (actual rows=30.00 loops=1) -> Function Scan on generate_series g (actual rows=32.00 loops=1) -> Limit (actual rows=1.00 loops=32) -> Custom Scan (ChunkAppend) on metrics_timestamptz m (actual rows=1.00 loops=32) Order: m."time" Chunks excluded during startup: 0 Chunks excluded during runtime: 2 -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m_1 (actual rows=1.00 loops=4) Index Cond: ("time" > (g."time" + '@ 1 day'::interval)) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m_2 (actual rows=1.00 loops=7) Index Cond: ("time" > (g."time" + '@ 1 day'::interval)) -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m_3 (actual rows=1.00 loops=7) Index Cond: ("time" > (g."time" + '@ 1 day'::interval)) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m_4 (actual rows=1.00 loops=7) Index Cond: ("time" > (g."time" + '@ 1 day'::interval)) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m_5 (actual rows=1.00 loops=7) Index Cond: ("time" > (g."time" + '@ 1 day'::interval)) -- test runtime exclusion with subquery :PREFIX SELECT m1.time FROM metrics_timestamptz m1 WHERE m1.time=(SELECT max(time) FROM metrics_timestamptz); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=3.00 loops=1) Chunks excluded during runtime: 4 InitPlan 2 (returns $1) -> Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=1.00 loops=1) Order: metrics_timestamptz."time" DESC -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m1_1 (never executed) Index Cond: ("time" = $1) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_2 (never executed) Index Cond: ("time" = $1) -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m1_3 (never executed) Index Cond: ("time" = $1) -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m1_4 (never executed) Index Cond: ("time" = $1) -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m1_5 (actual rows=3.00 loops=1) Index Cond: ("time" = $1) -- test runtime exclusion with correlated subquery :PREFIX SELECT m1.time, (SELECT m2.time FROM metrics_timestamptz m2 WHERE m2.time < m1.time ORDER BY m2.time DESC LIMIT 1) FROM metrics_timestamptz m1 WHERE m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Result (actual rows=7776.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=7776.00 loops=1) Order: m1."time" -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m1_1 (actual rows=4032.00 loops=1) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_2 (actual rows=3744.00 loops=1) Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) SubPlan 1 -> Limit (actual rows=1.00 loops=7776) -> Custom Scan (ChunkAppend) on metrics_timestamptz m2 (actual rows=1.00 loops=7776) Order: m2."time" DESC Chunks excluded during runtime: 3 -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m2_1 (never executed) Index Cond: ("time" < m1."time") -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m2_2 (never executed) Index Cond: ("time" < m1."time") -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m2_3 (never executed) Index Cond: ("time" < m1."time") -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m2_4 (actual rows=1.00 loops=3741) Index Cond: ("time" < m1."time") -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m2_5 (actual rows=1.00 loops=4035) Index Cond: ("time" < m1."time") -- test EXISTS :PREFIX SELECT m1.time FROM metrics_timestamptz m1 WHERE EXISTS(SELECT 1 FROM metrics_timestamptz m2 WHERE m1.time < m2.time) ORDER BY m1.time DESC limit 1000; --- QUERY PLAN --- Limit (actual rows=1000.00 loops=1) -> Nested Loop Semi Join (actual rows=1000.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=1003.00 loops=1) Order: m1."time" DESC -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m1_1 (actual rows=1003.00 loops=1) -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m1_2 (never executed) -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m1_3 (never executed) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_4 (never executed) -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m1_5 (never executed) -> Append (actual rows=1.00 loops=1003) -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m2_1 (actual rows=0.00 loops=1003) Index Cond: ("time" > m1."time") -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m2_2 (actual rows=0.00 loops=1003) Index Cond: ("time" > m1."time") -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m2_3 (actual rows=0.00 loops=1003) Index Cond: ("time" > m1."time") -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m2_4 (actual rows=0.00 loops=1003) Index Cond: ("time" > m1."time") -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m2_5 (actual rows=1.00 loops=1003) Index Cond: ("time" > m1."time") -- test constraint exclusion for subqueries with append -- should include 2 chunks :PREFIX SELECT time FROM (SELECT time FROM metrics_timestamptz WHERE time < '2000-01-10'::text::timestamptz ORDER BY time) m; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=7776.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk (actual rows=4032.00 loops=1) Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk (actual rows=3744.00 loops=1) Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -- test constraint exclusion for subqueries with mergeappend -- should include 2 chunks :PREFIX SELECT device_id, time FROM (SELECT device_id, time FROM metrics_timestamptz WHERE time < '2000-01-10'::text::timestamptz ORDER BY device_id, time) m; --- QUERY PLAN --- Custom Scan (ConstraintAwareAppend) (actual rows=7776.00 loops=1) Hypertable: metrics_timestamptz Chunks excluded during startup: 3 -> Merge Append (actual rows=7776.00 loops=1) Sort Key: metrics_timestamptz.device_id, metrics_timestamptz."time" -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_17_chunk (actual rows=4032.00 loops=1) Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_18_chunk (actual rows=3744.00 loops=1) Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -- test LIMIT pushdown -- no aggregates/window functions/SRF should pushdown limit :PREFIX SELECT FROM metrics_timestamptz ORDER BY time LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=1.00 loops=1) Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk (never executed) -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (never executed) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (never executed) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (never executed) -- aggregates should prevent pushdown :PREFIX SELECT count(*) FROM metrics_timestamptz LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=26787.00 loops=1) -> Seq Scan on _hyper_5_17_chunk (actual rows=4032.00 loops=1) -> Seq Scan on _hyper_5_18_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_19_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_20_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_21_chunk (actual rows=4611.00 loops=1) :PREFIX SELECT count(*) FROM metrics_space LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=37450.00 loops=1) -> Seq Scan on _hyper_6_22_chunk (actual rows=5376.00 loops=1) -> Seq Scan on _hyper_6_23_chunk (actual rows=5376.00 loops=1) -> Seq Scan on _hyper_6_24_chunk (actual rows=2688.00 loops=1) -> Seq Scan on _hyper_6_25_chunk (actual rows=8064.00 loops=1) -> Seq Scan on _hyper_6_26_chunk (actual rows=8064.00 loops=1) -> Seq Scan on _hyper_6_27_chunk (actual rows=4032.00 loops=1) -> Seq Scan on _hyper_6_28_chunk (actual rows=1540.00 loops=1) -> Seq Scan on _hyper_6_29_chunk (actual rows=1540.00 loops=1) -> Seq Scan on _hyper_6_30_chunk (actual rows=770.00 loops=1) -- HAVING should prevent pushdown :PREFIX SELECT 1 FROM metrics_timestamptz HAVING count(*) > 1 LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Aggregate (actual rows=1.00 loops=1) Filter: (count(*) > 1) -> Append (actual rows=26787.00 loops=1) -> Seq Scan on _hyper_5_17_chunk (actual rows=4032.00 loops=1) -> Seq Scan on _hyper_5_18_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_19_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_20_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_21_chunk (actual rows=4611.00 loops=1) :PREFIX SELECT 1 FROM metrics_space HAVING count(*) > 1 LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Aggregate (actual rows=1.00 loops=1) Filter: (count(*) > 1) -> Append (actual rows=37450.00 loops=1) -> Seq Scan on _hyper_6_22_chunk (actual rows=5376.00 loops=1) -> Seq Scan on _hyper_6_23_chunk (actual rows=5376.00 loops=1) -> Seq Scan on _hyper_6_24_chunk (actual rows=2688.00 loops=1) -> Seq Scan on _hyper_6_25_chunk (actual rows=8064.00 loops=1) -> Seq Scan on _hyper_6_26_chunk (actual rows=8064.00 loops=1) -> Seq Scan on _hyper_6_27_chunk (actual rows=4032.00 loops=1) -> Seq Scan on _hyper_6_28_chunk (actual rows=1540.00 loops=1) -> Seq Scan on _hyper_6_29_chunk (actual rows=1540.00 loops=1) -> Seq Scan on _hyper_6_30_chunk (actual rows=770.00 loops=1) -- DISTINCT should prevent pushdown SET enable_hashagg TO false; :PREFIX SELECT DISTINCT device_id FROM metrics_timestamptz ORDER BY device_id LIMIT 3; --- QUERY PLAN --- Limit (actual rows=3.00 loops=1) -> Unique (actual rows=3.00 loops=1) -> Merge Append (actual rows=17859.00 loops=1) Sort Key: metrics_timestamptz.device_id -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_17_chunk (actual rows=2689.00 loops=1) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_18_chunk (actual rows=4033.00 loops=1) -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_19_chunk (actual rows=4033.00 loops=1) -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_20_chunk (actual rows=4033.00 loops=1) -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_21_chunk (actual rows=3075.00 loops=1) :PREFIX SELECT DISTINCT device_id FROM metrics_space ORDER BY device_id LIMIT 3; --- QUERY PLAN --- Limit (actual rows=3.00 loops=1) -> Unique (actual rows=3.00 loops=1) -> Merge Append (actual rows=7491.00 loops=1) Sort Key: metrics_space.device_id -> Index Scan using _hyper_6_22_chunk_metrics_space_device_id_time_idx on _hyper_6_22_chunk (actual rows=1345.00 loops=1) -> Index Scan using _hyper_6_23_chunk_metrics_space_device_id_time_idx on _hyper_6_23_chunk (actual rows=1345.00 loops=1) -> Index Scan using _hyper_6_24_chunk_metrics_space_device_id_time_idx on _hyper_6_24_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_6_25_chunk_metrics_space_device_id_time_idx on _hyper_6_25_chunk (actual rows=2017.00 loops=1) -> Index Scan using _hyper_6_26_chunk_metrics_space_device_id_time_idx on _hyper_6_26_chunk (actual rows=2017.00 loops=1) -> Index Scan using _hyper_6_27_chunk_metrics_space_device_id_time_idx on _hyper_6_27_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_6_28_chunk_metrics_space_device_id_time_idx on _hyper_6_28_chunk (actual rows=386.00 loops=1) -> Index Scan using _hyper_6_29_chunk_metrics_space_device_id_time_idx on _hyper_6_29_chunk (actual rows=386.00 loops=1) -> Index Scan using _hyper_6_30_chunk_metrics_space_device_id_time_idx on _hyper_6_30_chunk (actual rows=1.00 loops=1) RESET enable_hashagg; -- JOINs should prevent pushdown -- when LIMIT gets pushed to a Sort node it will switch to top-N heapsort -- if more tuples then LIMIT are requested this will trigger an error -- to trigger this we need a Sort node that is below ChunkAppend CREATE TABLE join_limit (time timestamptz, device_id int); SELECT table_name FROM create_hypertable('join_limit','time',create_default_indexes:=false); table_name ------------ join_limit CREATE INDEX ON join_limit(time,device_id); INSERT INTO join_limit SELECT time, device_id FROM generate_series('2000-01-01'::timestamptz,'2000-01-21','30m') g1(time), generate_series(1,10,1) g2(device_id) ORDER BY time, device_id; VACUUM (ANALYZE) join_limit; -- get 2nd chunk oid SELECT tableoid AS "CHUNK_OID" FROM join_limit WHERE time > '2000-01-07' ORDER BY time LIMIT 1 \gset --get index name for 2nd chunk SELECT indexrelid::regclass AS "INDEX_NAME" FROM pg_index WHERE indrelid = :CHUNK_OID \gset DROP INDEX :INDEX_NAME; :PREFIX SELECT * FROM metrics_timestamptz m1 INNER JOIN join_limit m2 ON m1.time = m2.time AND m1.device_id=m2.device_id WHERE m1.time > '2000-01-07' ORDER BY m1.time, m1.device_id LIMIT 3; --- QUERY PLAN --- Limit (actual rows=3.00 loops=1) -> Merge Join (actual rows=3.00 loops=1) Merge Cond: (m2."time" = m1."time") Join Filter: (m1.device_id = m2.device_id) Rows Removed by Join Filter: 4 -> Custom Scan (ChunkAppend) on join_limit m2 (actual rows=3.00 loops=1) Order: m2."time", m2.device_id -> Sort (actual rows=3.00 loops=1) Sort Key: m2_1."time", m2_1.device_id Sort Method: quicksort -> Seq Scan on _hyper_8_35_chunk m2_1 (actual rows=2710.00 loops=1) Filter: ("time" > 'Fri Jan 07 00:00:00 2000 PST'::timestamp with time zone) Rows Removed by Filter: 650 -> Index Scan using _hyper_8_36_chunk_join_limit_time_device_id_idx on _hyper_8_36_chunk m2_2 (never executed) -> Index Scan using _hyper_8_37_chunk_join_limit_time_device_id_idx on _hyper_8_37_chunk m2_3 (never executed) -> Materialize (actual rows=22.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=19.00 loops=1) Order: m1."time" -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_1 (actual rows=19.00 loops=1) Index Cond: ("time" > 'Fri Jan 07 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m1_2 (never executed) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m1_3 (never executed) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m1_4 (never executed) DROP TABLE join_limit; -- test ChunkAppend projection #2661 :PREFIX SELECT ts.timestamp, ht.timestamp FROM ( SELECT generate_series( to_timestamp(FLOOR(EXTRACT (EPOCH FROM '2020-01-01T00:01:00Z'::timestamp) / 300) * 300) AT TIME ZONE 'UTC', '2020-01-01T01:00:00Z', '5 minutes'::interval ) AS timestamp ) ts LEFT JOIN i2661 ht ON (FLOOR(EXTRACT (EPOCH FROM ht."timestamp") / 300) * 300 = EXTRACT (EPOCH FROM ts.timestamp)) AND ht.timestamp > '2019-12-30T00:00:00Z'::timestamp ORDER BY ts.timestamp, ht.timestamp; --- QUERY PLAN --- Sort (actual rows=33.00 loops=1) Sort Key: ts."timestamp", ht."timestamp" Sort Method: quicksort -> Merge Left Join (actual rows=33.00 loops=1) Merge Cond: ((EXTRACT(epoch FROM ts."timestamp")) = ((floor((EXTRACT(epoch FROM ht."timestamp") / '300'::numeric)) * '300'::numeric))) -> Sort (actual rows=13.00 loops=1) Sort Key: (EXTRACT(epoch FROM ts."timestamp")) Sort Method: quicksort -> Subquery Scan on ts (actual rows=13.00 loops=1) -> ProjectSet (actual rows=13.00 loops=1) -> Result (actual rows=1.00 loops=1) -> Sort (actual rows=514.00 loops=1) Sort Key: ((floor((EXTRACT(epoch FROM ht."timestamp") / '300'::numeric)) * '300'::numeric)) Sort Method: quicksort -> Result (actual rows=7201.00 loops=1) -> Custom Scan (ChunkAppend) on i2661 ht (actual rows=7201.00 loops=1) Chunks excluded during startup: 0 -> Seq Scan on _hyper_7_31_chunk ht_1 (actual rows=1200.00 loops=1) Filter: ("timestamp" > 'Mon Dec 30 00:00:00 2019'::timestamp without time zone) -> Seq Scan on _hyper_7_32_chunk ht_2 (actual rows=5040.00 loops=1) Filter: ("timestamp" > 'Mon Dec 30 00:00:00 2019'::timestamp without time zone) -> Seq Scan on _hyper_7_33_chunk ht_3 (actual rows=961.00 loops=1) Filter: ("timestamp" > 'Mon Dec 30 00:00:00 2019'::timestamp without time zone) -- #3030 test chunkappend keeps pathkeys when subpath is append -- on PG11 this will not use ChunkAppend but MergeAppend SET enable_seqscan TO FALSE; CREATE TABLE i3030(time timestamptz NOT NULL, a int, b int); SELECT table_name FROM create_hypertable('i3030', 'time', create_default_indexes=>false); table_name ------------ i3030 CREATE INDEX ON i3030(a,time); INSERT INTO i3030 (time,a) SELECT time, a FROM generate_series('2000-01-01'::timestamptz,'2000-01-01 3:00:00'::timestamptz,'1min'::interval) time, generate_series(1,30) a; VACUUM (ANALYZE) i3030; :PREFIX SELECT * FROM i3030 where time BETWEEN '2000-01-01'::text::timestamptz AND '2000-01-03'::text::timestamptz ORDER BY a,time LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on i3030 (actual rows=1.00 loops=1) Order: i3030.a, i3030."time" Chunks excluded during startup: 0 -> Index Scan using _hyper_9_38_chunk_i3030_a_time_idx on _hyper_9_38_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= ('2000-01-01'::cstring)::timestamp with time zone) AND ("time" <= ('2000-01-03'::cstring)::timestamp with time zone)) DROP TABLE i3030; RESET enable_seqscan; --parent runtime exclusion tests: --optimization works with ANY (array) :PREFIX SELECT * FROM append_test a WHERE a.attr @> ANY((SELECT coalesce(array_agg(attr), array[]::jsonb[]) FROM join_test_plain WHERE temp > 100)::jsonb[]); --- QUERY PLAN --- Custom Scan (ChunkAppend) on append_test a (actual rows=0.00 loops=1) Hypertables excluded during runtime: 1 InitPlan 1 (returns $0) -> Aggregate (actual rows=1.00 loops=1) -> Seq Scan on join_test_plain (actual rows=0.00 loops=1) Filter: (temp > '100'::double precision) Rows Removed by Filter: 3 -> Seq Scan on _hyper_1_1_chunk a_1 (never executed) Filter: (attr @> ANY ($0)) -> Seq Scan on _hyper_1_2_chunk a_2 (never executed) Filter: (attr @> ANY ($0)) -> Seq Scan on _hyper_1_3_chunk a_3 (never executed) Filter: (attr @> ANY ($0)) --optimization does not work for ANY subquery (does not force an initplan) :PREFIX SELECT * FROM append_test a WHERE a.attr @> ANY((SELECT attr FROM join_test_plain WHERE temp > 100)); --- QUERY PLAN --- Nested Loop Semi Join (actual rows=0.00 loops=1) Join Filter: (a.attr @> join_test_plain.attr) -> Append (actual rows=5.00 loops=1) -> Seq Scan on _hyper_1_1_chunk a_1 (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_2_chunk a_2 (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk a_3 (actual rows=1.00 loops=1) -> Materialize (actual rows=0.00 loops=5) -> Seq Scan on join_test_plain (actual rows=0.00 loops=1) Filter: (temp > '100'::double precision) Rows Removed by Filter: 3 --works on any strict operator without ANY :PREFIX SELECT * FROM append_test a WHERE a.attr @> (SELECT attr FROM join_test_plain WHERE temp > 100 limit 1); --- QUERY PLAN --- Custom Scan (ChunkAppend) on append_test a (actual rows=0.00 loops=1) Hypertables excluded during runtime: 1 InitPlan 1 (returns $0) -> Limit (actual rows=0.00 loops=1) -> Seq Scan on join_test_plain (actual rows=0.00 loops=1) Filter: (temp > '100'::double precision) Rows Removed by Filter: 3 -> Seq Scan on _hyper_1_1_chunk a_1 (never executed) Filter: (attr @> $0) -> Seq Scan on _hyper_1_2_chunk a_2 (never executed) Filter: (attr @> $0) -> Seq Scan on _hyper_1_3_chunk a_3 (never executed) Filter: (attr @> $0) --optimization works with function calls CREATE OR REPLACE FUNCTION select_tag(_min_temp int) RETURNS jsonb[] LANGUAGE sql STABLE PARALLEL SAFE AS $function$ SELECT coalesce(array_agg(attr), array[]::jsonb[]) FROM join_test_plain WHERE temp > _min_temp $function$; :PREFIX SELECT * FROM append_test a WHERE a.attr @> ANY((SELECT select_tag(100))::jsonb[]); --- QUERY PLAN --- Custom Scan (ChunkAppend) on append_test a (actual rows=0.00 loops=1) Hypertables excluded during runtime: 1 InitPlan 1 (returns $0) -> Result (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_1_chunk a_1 (never executed) Filter: (attr @> ANY ($0)) -> Seq Scan on _hyper_1_2_chunk a_2 (never executed) Filter: (attr @> ANY ($0)) -> Seq Scan on _hyper_1_3_chunk a_3 (never executed) Filter: (attr @> ANY ($0)) --optimization does not work when result is null :PREFIX SELECT * FROM append_test a WHERE a.attr @> ANY((SELECT array_agg(attr) FROM join_test_plain WHERE temp > 100)::jsonb[]); --- QUERY PLAN --- Custom Scan (ChunkAppend) on append_test a (actual rows=0.00 loops=1) Hypertables excluded during runtime: 0 InitPlan 1 (returns $0) -> Aggregate (actual rows=1.00 loops=1) -> Seq Scan on join_test_plain (actual rows=0.00 loops=1) Filter: (temp > '100'::double precision) Rows Removed by Filter: 3 -> Seq Scan on _hyper_1_1_chunk a_1 (actual rows=0.00 loops=1) Filter: (attr @> ANY ($0)) Rows Removed by Filter: 2 -> Seq Scan on _hyper_1_2_chunk a_2 (actual rows=0.00 loops=1) Filter: (attr @> ANY ($0)) Rows Removed by Filter: 2 -> Seq Scan on _hyper_1_3_chunk a_3 (actual rows=0.00 loops=1) Filter: (attr @> ANY ($0)) Rows Removed by Filter: 1 -- Test that ConstraintAwareAppend properly locks relations in -- parallel query mode set timescaledb.enable_chunk_append=false; call force_parallel(true); :PREFIX select time, avg(temp), colorid from append_test where time > now_s() - interval '3 months 20 days' group by time, colorid; psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Gather (actual rows=3.00 loops=1) Workers Planned: 1 Workers Launched: 1 Single Copy: true -> HashAggregate (actual rows=3.00 loops=1) Group Key: append_test."time", append_test.colorid -> Custom Scan (ConstraintAwareAppend) (actual rows=3.00 loops=1) Hypertable: append_test Chunks excluded during startup: 1 -> Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) Filter: ("time" > (now_s() - '@ 3 mons 20 days'::interval)) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 3 mons 20 days'::interval)) reset timescaledb.enable_chunk_append; reset max_parallel_workers_per_gather; call force_parallel(false); --generate the results into two different files \set ECHO errors --- Unoptimized results +++ Optimized results @@ -1,6 +1,6 @@ setting | value ----------------------------------+------- - timescaledb.enable_optimizations | off + timescaledb.enable_optimizations | on timescaledb.enable_chunk_append | on --- Unoptimized results +++ Optimized results @@ -1,7 +1,7 @@ setting | value ----------------------------------+------- - timescaledb.enable_optimizations | off - timescaledb.enable_chunk_append | on + timescaledb.enable_optimizations | on + timescaledb.enable_chunk_append | off time | temp | colorid | attr ================================================ FILE: test/expected/append-16.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set TEST_BASE_NAME append SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') as "TEST_LOAD_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') as "TEST_QUERY_NAME", format('%s/results/%s_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_OPTIMIZED", format('%s/results/%s_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_UNOPTIMIZED" \gset SELECT format('\! diff -u --label "Unoptimized results" --label "Optimized results" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') as "DIFF_CMD" \gset SET timescaledb.enable_now_constify TO false; -- disable memoize node to avoid flaky results SET enable_memoize TO 'off'; -- disable index only scans to avoid some flaky results SET enable_indexonlyscan TO FALSE; \set PREFIX 'EXPLAIN (analyze, buffers off, costs off, timing off, summary off)' \ir :TEST_LOAD_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- create a now() function for repeatable testing that always returns -- the same timestamp. It needs to be marked STABLE CREATE OR REPLACE FUNCTION now_s() RETURNS timestamptz LANGUAGE PLPGSQL STABLE PARALLEL SAFE AS $BODY$ BEGIN RAISE NOTICE 'Stable function now_s() called!'; RETURN '2017-08-22T10:00:00'::timestamptz; END; $BODY$; CREATE OR REPLACE FUNCTION now_i() RETURNS timestamptz LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RAISE NOTICE 'Immutable function now_i() called!'; RETURN '2017-08-22T10:00:00'::timestamptz; END; $BODY$; CREATE OR REPLACE FUNCTION now_v() RETURNS timestamptz LANGUAGE PLPGSQL VOLATILE AS $BODY$ BEGIN RAISE NOTICE 'Volatile function now_v() called!'; RETURN '2017-08-22T10:00:00'::timestamptz; END; $BODY$; CREATE OR REPLACE PROCEDURE force_parallel(on_or_off bool) LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('server_version_num')::int < 160000 THEN IF on_or_off THEN set force_parallel_mode = 'on'; ELSE set force_parallel_mode = 'off'; END IF; ELSE IF on_or_off THEN set debug_parallel_query = 'on'; ELSE set debug_parallel_query = 'off'; END IF; END IF; END; $$; CREATE TABLE append_test(time timestamptz, temp float, colorid integer, attr jsonb); SELECT create_hypertable('append_test', 'time', chunk_time_interval => 2628000000000); create_hypertable -------------------------- (1,public,append_test,t) -- create three chunks INSERT INTO append_test VALUES ('2017-03-22T09:18:22', 23.5, 1, '{"a": 1, "b": 2}'), ('2017-03-22T09:18:23', 21.5, 1, '{"a": 1, "b": 2}'), ('2017-05-22T09:18:22', 36.2, 2, '{"c": 3, "b": 2}'), ('2017-05-22T09:18:23', 15.2, 2, '{"c": 3}'), ('2017-08-22T09:18:22', 34.1, 3, '{"c": 4}'); VACUUM (ANALYZE) append_test; -- Create another hypertable to join with CREATE TABLE join_test(time timestamptz, temp float, colorid integer); SELECT create_hypertable('join_test', 'time', chunk_time_interval => 2628000000000); create_hypertable ------------------------ (2,public,join_test,t) INSERT INTO join_test VALUES ('2017-01-22T09:18:22', 15.2, 1), ('2017-02-22T09:18:22', 24.5, 2), ('2017-08-22T09:18:22', 23.1, 3); VACUUM (ANALYZE) join_test; -- Create another table to join with which is not a hypertable. CREATE TABLE join_test_plain(time timestamptz, temp float, colorid integer, attr jsonb); INSERT INTO join_test_plain VALUES ('2017-01-22T09:18:22', 15.2, 1, '{"a": 1}'), ('2017-02-22T09:18:22', 24.5, 2, '{"b": 2}'), ('2017-08-22T09:18:22', 23.1, 3, '{"c": 3}'); VACUUM (ANALYZE) join_test_plain; -- create hypertable with DATE time dimension CREATE TABLE metrics_date(time DATE NOT NULL); SELECT create_hypertable('metrics_date','time'); create_hypertable --------------------------- (3,public,metrics_date,t) INSERT INTO metrics_date SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval); VACUUM (ANALYZE) metrics_date; -- create hypertable with TIMESTAMP time dimension CREATE TABLE metrics_timestamp(time TIMESTAMP NOT NULL); SELECT create_hypertable('metrics_timestamp','time'); psql:include/append_load.sql:91: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------------- (4,public,metrics_timestamp,t) INSERT INTO metrics_timestamp SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval); VACUUM (ANALYZE) metrics_timestamp; -- create hypertable with TIMESTAMPTZ time dimension CREATE TABLE metrics_timestamptz(time TIMESTAMPTZ NOT NULL, device_id INT NOT NULL); CREATE INDEX ON metrics_timestamptz(device_id,time); SELECT create_hypertable('metrics_timestamptz','time'); create_hypertable ---------------------------------- (5,public,metrics_timestamptz,t) INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval), 1; INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval), 2; INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval), 3; VACUUM (ANALYZE) metrics_timestamptz; -- create space partitioned hypertable CREATE TABLE metrics_space(time timestamptz NOT NULL, device_id int NOT NULL, v1 float, v2 float, v3 text); SELECT create_hypertable('metrics_space','time','device_id',3); create_hypertable ---------------------------- (6,public,metrics_space,t) INSERT INTO metrics_space SELECT time, device_id, device_id + 0.25, device_id + 0.75, device_id FROM generate_series('2000-01-01'::timestamptz, '2000-01-14'::timestamptz, '5m'::interval) g1(time), generate_series(1,10,1) g2(device_id) ORDER BY time, device_id; VACUUM (ANALYZE) metrics_space; -- test ChunkAppend projection #2661 CREATE TABLE i2661 ( machine_id int4 NOT NULL, "name" varchar(255) NOT NULL, "timestamp" timestamptz NOT NULL, "first" float4 NULL ); SELECT create_hypertable('i2661', 'timestamp'); psql:include/append_load.sql:123: WARNING: column type "character varying" used for "name" does not follow best practices create_hypertable -------------------- (7,public,i2661,t) INSERT INTO i2661 SELECT 1, 'speed', generate_series('2019-12-31 00:00:00', '2020-01-10 00:00:00', '2m'::interval), 0; VACUUM (ANALYZE) i2661; \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be the only output of the results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations'),('timescaledb.enable_chunk_append')) v(setting); setting | value ----------------------------------+------- timescaledb.enable_optimizations | on timescaledb.enable_chunk_append | on -- query should exclude all chunks with optimization on :PREFIX SELECT * FROM append_test WHERE time > now_s() + '1 month' ORDER BY time DESC; psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Sort (actual rows=0.00 loops=1) Sort Key: append_test."time" DESC Sort Method: quicksort -> Custom Scan (ChunkAppend) on append_test (actual rows=0.00 loops=1) Chunks excluded during startup: 3 --query should exclude all chunks and be a MergeAppend :PREFIX SELECT * FROM append_test WHERE time > now_s() + '1 month' ORDER BY time DESC limit 1; psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Limit (actual rows=0.00 loops=1) -> Custom Scan (ChunkAppend) on append_test (actual rows=0.00 loops=1) Order: append_test."time" DESC Chunks excluded during startup: 3 -- when optimized, the plan should be a constraint-aware append and -- cover only one chunk. It should be a backward index scan due to -- descending index on time. Should also skip the main table, since it -- cannot hold tuples :PREFIX SELECT * FROM append_test WHERE time > now_s() - interval '2 months'; psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Custom Scan (ChunkAppend) on append_test (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 2 mons'::interval)) -- adding ORDER BY and LIMIT should turn the plan into an optimized -- ordered append plan :PREFIX SELECT * FROM append_test WHERE time > now_s() - interval '2 months' ORDER BY time LIMIT 3; psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Sort (actual rows=1.00 loops=1) Sort Key: append_test."time" Sort Method: quicksort -> Custom Scan (ChunkAppend) on append_test (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 2 mons'::interval)) -- no optimized plan for queries with restrictions that can be -- constified at planning time. Regular planning-time constraint -- exclusion should occur. :PREFIX SELECT * FROM append_test WHERE time > now_i() - interval '2 months' ORDER BY time; psql:include/append_query.sql:37: NOTICE: Immutable function now_i() called! --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: append_test."time" Sort Method: quicksort -> Custom Scan (ChunkAppend) on append_test (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > ('Tue Aug 22 10:00:00 2017 PDT'::timestamp with time zone - '@ 2 mons'::interval)) -- currently, we cannot distinguish between stable and volatile -- functions as far as applying our modified plan. However, volatile -- function should not be pre-evaluated to constants, so no chunk -- exclusion should occur. :PREFIX SELECT * FROM append_test WHERE time > now_v() - interval '2 months' ORDER BY time; psql:include/append_query.sql:45: NOTICE: Volatile function now_v() called! psql:include/append_query.sql:45: NOTICE: Volatile function now_v() called! psql:include/append_query.sql:45: NOTICE: Volatile function now_v() called! psql:include/append_query.sql:45: NOTICE: Volatile function now_v() called! psql:include/append_query.sql:45: NOTICE: Volatile function now_v() called! --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: append_test."time" Sort Method: quicksort -> Custom Scan (ChunkAppend) on append_test (actual rows=1.00 loops=1) Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk (actual rows=0.00 loops=1) Filter: ("time" > (now_v() - '@ 2 mons'::interval)) Rows Removed by Filter: 2 -> Seq Scan on _hyper_1_2_chunk (actual rows=0.00 loops=1) Filter: ("time" > (now_v() - '@ 2 mons'::interval)) Rows Removed by Filter: 2 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_v() - '@ 2 mons'::interval)) -- prepared statement output should be the same regardless of -- optimizations PREPARE query_opt AS SELECT * FROM append_test WHERE time > now_s() - interval '2 months' ORDER BY time; :PREFIX EXECUTE query_opt; psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: append_test."time" Sort Method: quicksort -> Custom Scan (ChunkAppend) on append_test (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 2 mons'::interval)) DEALLOCATE query_opt; -- aggregates should produce same output :PREFIX SELECT date_trunc('year', time) t, avg(temp) FROM append_test WHERE time > now_s() - interval '4 months' GROUP BY t ORDER BY t DESC; psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! --- QUERY PLAN --- GroupAggregate (actual rows=1.00 loops=1) Group Key: (date_trunc('year'::text, append_test."time")) -> Sort (actual rows=3.00 loops=1) Sort Key: (date_trunc('year'::text, append_test."time")) DESC Sort Method: quicksort -> Result (actual rows=3.00 loops=1) -> Custom Scan (ChunkAppend) on append_test (actual rows=3.00 loops=1) Chunks excluded during startup: 1 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 4 mons'::interval)) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) Filter: ("time" > (now_s() - '@ 4 mons'::interval)) -- querying outside the time range should return nothing. This tests -- that ConstraintAwareAppend can handle the case when an Append node -- is turned into a Result node due to no children :PREFIX SELECT date_trunc('year', time) t, avg(temp) FROM append_test WHERE time < '2016-03-22' AND date_part('dow', time) between 1 and 5 GROUP BY t ORDER BY t DESC; --- QUERY PLAN --- GroupAggregate (actual rows=0.00 loops=1) Group Key: (date_trunc('year'::text, "time")) -> Sort (actual rows=0.00 loops=1) Sort Key: (date_trunc('year'::text, "time")) DESC Sort Method: quicksort -> Result (actual rows=0.00 loops=1) One-Time Filter: false -- a parameterized query can safely constify params, so won't be -- optimized by constraint-aware append since regular constraint -- exclusion works just fine PREPARE query_param AS SELECT * FROM append_test WHERE time > $1 ORDER BY time; :PREFIX EXECUTE query_param(now_s() - interval '2 months'); psql:include/append_query.sql:82: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: _hyper_1_3_chunk."time" Sort Method: quicksort -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) DEALLOCATE query_param; --test with cte :PREFIX WITH data AS ( SELECT time_bucket(INTERVAL '30 day', TIME) AS btime, AVG(temp) AS VALUE FROM append_test WHERE TIME > now_s() - INTERVAL '400 day' AND colorid > 0 GROUP BY btime ), period AS ( SELECT time_bucket(INTERVAL '30 day', TIME) AS btime FROM GENERATE_SERIES('2017-03-22T01:01:01', '2017-08-23T01:01:01', INTERVAL '30 day') TIME ) SELECT period.btime, VALUE FROM period LEFT JOIN DATA USING (btime) ORDER BY period.btime; psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Sort (actual rows=6.00 loops=1) Sort Key: (time_bucket('@ 30 days'::interval, "time"."time")) Sort Method: quicksort -> Hash Left Join (actual rows=6.00 loops=1) Hash Cond: (time_bucket('@ 30 days'::interval, "time"."time") = data.btime) -> Function Scan on generate_series "time" (actual rows=6.00 loops=1) -> Hash (actual rows=3.00 loops=1) -> Subquery Scan on data (actual rows=3.00 loops=1) -> HashAggregate (actual rows=3.00 loops=1) Group Key: time_bucket('@ 30 days'::interval, append_test."time") -> Result (actual rows=5.00 loops=1) -> Custom Scan (ChunkAppend) on append_test (actual rows=5.00 loops=1) Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk (actual rows=2.00 loops=1) Filter: ((colorid > 0) AND ("time" > (now_s() - '@ 400 days'::interval))) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) Filter: ((colorid > 0) AND ("time" > (now_s() - '@ 400 days'::interval))) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ((colorid > 0) AND ("time" > (now_s() - '@ 400 days'::interval))) WITH data AS ( SELECT time_bucket(INTERVAL '30 day', TIME) AS btime, AVG(temp) AS VALUE FROM append_test WHERE TIME > now_s() - INTERVAL '400 day' AND colorid > 0 GROUP BY btime ), period AS ( SELECT time_bucket(INTERVAL '30 day', TIME) AS btime FROM GENERATE_SERIES('2017-03-22T01:01:01', '2017-08-23T01:01:01', INTERVAL '30 day') TIME ) SELECT period.btime, VALUE FROM period LEFT JOIN DATA USING (btime) ORDER BY period.btime; psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! btime | value ------------------------------+------- Fri Mar 03 16:00:00 2017 PST | 22.5 Sun Apr 02 17:00:00 2017 PDT | Tue May 02 17:00:00 2017 PDT | 25.7 Thu Jun 01 17:00:00 2017 PDT | Sat Jul 01 17:00:00 2017 PDT | Mon Jul 31 17:00:00 2017 PDT | 34.1 -- force nested loop join with no materialization. This tests that the -- inner ConstraintAwareScan supports resetting its scan for every -- iteration of the outer relation loop set enable_hashjoin = 'off'; set enable_mergejoin = 'off'; set enable_material = 'off'; :PREFIX SELECT * FROM append_test a INNER JOIN join_test j ON (a.colorid = j.colorid) WHERE a.time > now_s() - interval '3 hours' AND j.time > now_s() - interval '3 hours'; psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Nested Loop (actual rows=1.00 loops=1) Join Filter: (a.colorid = j.colorid) -> Custom Scan (ChunkAppend) on append_test a (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_1_3_chunk a_1 (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 3 hours'::interval)) -> Custom Scan (ChunkAppend) on join_test j (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_2_6_chunk j_1 (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 3 hours'::interval)) reset enable_hashjoin; reset enable_mergejoin; reset enable_material; -- test constraint_exclusion with date time dimension and DATE/TIMESTAMP/TIMESTAMPTZ constraints -- the queries should all have 3 chunks :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -- test Const OP Var -- the queries should all have 3 chunks :PREFIX SELECT * FROM metrics_date WHERE '2000-01-15'::date < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_date WHERE '2000-01-15'::timestamp < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_date WHERE '2000-01-15'::timestamptz < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -- test 2 constraints -- the queries should all have 2 chunks :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::date AND time < '2000-01-21'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=1440.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::timestamp AND time < '2000-01-21'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=1440.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::timestamptz AND time < '2000-01-21'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=1440.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -- test constraint_exclusion with timestamp time dimension and DATE/TIMESTAMP/TIMESTAMPTZ constraints -- the queries should all have 3 chunks :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -- test Const OP Var -- the queries should all have 3 chunks :PREFIX SELECT * FROM metrics_timestamp WHERE '2000-01-15'::date < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_timestamp WHERE '2000-01-15'::timestamp < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_timestamp WHERE '2000-01-15'::timestamptz < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -- test 2 constraints -- the queries should all have 2 chunks :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::date AND time < '2000-01-21'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=1727.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::timestamp AND time < '2000-01-21'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=1727.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::timestamptz AND time < '2000-01-21'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=1727.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -- test constraint_exclusion with timestamptz time dimension and DATE/TIMESTAMP/TIMESTAMPTZ constraints -- the queries should all have 3 chunks :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) -- test Const OP Var -- the queries should all have 3 chunks :PREFIX SELECT time FROM metrics_timestamptz WHERE '2000-01-15'::date < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) :PREFIX SELECT time FROM metrics_timestamptz WHERE '2000-01-15'::timestamp < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT time FROM metrics_timestamptz WHERE '2000-01-15'::timestamptz < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) -- test 2 constraints -- the queries should all have 2 chunks :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::date AND time < '2000-01-21'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=5181.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::timestamp AND time < '2000-01-21'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=5181.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::timestamptz AND time < '2000-01-21'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=5181.00 loops=1) Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -- test constraint_exclusion with space partitioning and DATE/TIMESTAMP/TIMESTAMPTZ constraints -- exclusion for constraints with non-matching datatypes not working for space partitioning atm :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -- test Const OP Var -- exclusion for constraints with non-matching datatypes not working for space partitioning atm :PREFIX SELECT time FROM metrics_space WHERE '2000-01-10'::date < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) :PREFIX SELECT time FROM metrics_space WHERE '2000-01-10'::timestamp < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT time FROM metrics_space WHERE '2000-01-10'::timestamptz < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -- test 2 constraints -- exclusion for constraints with non-matching datatypes not working for space partitioning atm :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::date AND time < '2000-01-15'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamp AND time < '2000-01-15'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND time < '2000-01-15'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -- test filtering on space partition :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND device_id = 1 ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=1152.00 loops=1) Order: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_device_id_time_idx on _hyper_6_25_chunk (actual rows=767.00 loops=1) Index Cond: ((device_id = 1) AND ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_device_id_time_idx on _hyper_6_28_chunk (actual rows=385.00 loops=1) Index Cond: ((device_id = 1) AND ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND device_id IN (1,2) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=2304.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=1534.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=767.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = ANY ('{1,2}'::integer[])) Rows Removed by Filter: 2301 -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=767.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = ANY ('{1,2}'::integer[])) Rows Removed by Filter: 2301 -> Merge Append (actual rows=770.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=385.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = ANY ('{1,2}'::integer[])) Rows Removed by Filter: 1155 -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=385.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = ANY ('{1,2}'::integer[])) Rows Removed by Filter: 1155 :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND device_id IN (VALUES(1)) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=1152.00 loops=1) Order: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_device_id_time_idx on _hyper_6_25_chunk (actual rows=767.00 loops=1) Index Cond: ((device_id = 1) AND ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_device_id_time_idx on _hyper_6_28_chunk (actual rows=385.00 loops=1) Index Cond: ((device_id = 1) AND ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND v3 IN (VALUES('1')) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=1152.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=767.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=767.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 2301 -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 3068 -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 1534 -> Merge Append (actual rows=385.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=385.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 1155 -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 1540 -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 770 :PREFIX SELECT * FROM metrics_space WHERE time = (VALUES ('2019-12-24' at time zone 'UTC')) AND v3 NOT IN (VALUES ('1')); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=0.00 loops=1) Chunks excluded during startup: 0 Chunks excluded during runtime: 9 InitPlan 1 (returns $0) -> Result (actual rows=1.00 loops=1) -> Index Scan using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (never executed) Index Cond: ("time" = $0) Filter: (NOT (hashed SubPlan 2)) SubPlan 2 -> Result (never executed) -> Index Scan using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (never executed) Index Cond: ("time" = $0) Filter: (NOT (hashed SubPlan 2)) -> Index Scan using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (never executed) Index Cond: ("time" = $0) Filter: (NOT (hashed SubPlan 2)) -> Index Scan using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (never executed) Index Cond: ("time" = $0) Filter: (NOT (hashed SubPlan 2)) -> Index Scan using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (never executed) Index Cond: ("time" = $0) Filter: (NOT (hashed SubPlan 2)) -> Index Scan using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (never executed) Index Cond: ("time" = $0) Filter: (NOT (hashed SubPlan 2)) -> Index Scan using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (never executed) Index Cond: ("time" = $0) Filter: (NOT (hashed SubPlan 2)) -> Index Scan using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (never executed) Index Cond: ("time" = $0) Filter: (NOT (hashed SubPlan 2)) -> Index Scan using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (never executed) Index Cond: ("time" = $0) Filter: (NOT (hashed SubPlan 2)) -- test CURRENT_DATE -- should be 0 chunks :PREFIX SELECT time FROM metrics_date WHERE time > CURRENT_DATE ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=0.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamp WHERE time > CURRENT_DATE ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=0.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamptz WHERE time > CURRENT_DATE ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=0.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_space WHERE time > CURRENT_DATE ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=0.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -- test CURRENT_TIMESTAMP -- should be 0 chunks :PREFIX SELECT time FROM metrics_date WHERE time > CURRENT_TIMESTAMP ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=0.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamp WHERE time > CURRENT_TIMESTAMP ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=0.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamptz WHERE time > CURRENT_TIMESTAMP ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=0.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_space WHERE time > CURRENT_TIMESTAMP ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=0.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -- test now() -- should be 0 chunks :PREFIX SELECT time FROM metrics_date WHERE time > now() ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=0.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamp WHERE time > now() ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=0.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamptz WHERE time > now() ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=0.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_space WHERE time > now() ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=0.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -- query with tablesample and planner exclusion :PREFIX SELECT * FROM metrics_date TABLESAMPLE BERNOULLI(5) REPEATABLE(0) WHERE time > '2000-01-15' ORDER BY time DESC; --- QUERY PLAN --- Sort (actual rows=217.00 loops=1) Sort Key: metrics_date."time" DESC Sort Method: quicksort -> Append (actual rows=217.00 loops=1) -> Sample Scan on _hyper_3_11_chunk (actual rows=72.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) -> Sample Scan on _hyper_3_10_chunk (actual rows=94.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) -> Sample Scan on _hyper_3_9_chunk (actual rows=51.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > '01-15-2000'::date) Rows Removed by Filter: 43 -- query with tablesample and startup exclusion :PREFIX SELECT * FROM metrics_date TABLESAMPLE BERNOULLI(5) REPEATABLE(0) WHERE time > '2000-01-15'::text::date ORDER BY time DESC; --- QUERY PLAN --- Sort (actual rows=217.00 loops=1) Sort Key: metrics_date."time" DESC Sort Method: quicksort -> Custom Scan (ChunkAppend) on metrics_date (actual rows=217.00 loops=1) Chunks excluded during startup: 2 -> Sample Scan on _hyper_3_11_chunk (actual rows=72.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > ('2000-01-15'::cstring)::date) -> Sample Scan on _hyper_3_10_chunk (actual rows=94.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > ('2000-01-15'::cstring)::date) -> Sample Scan on _hyper_3_9_chunk (actual rows=51.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > ('2000-01-15'::cstring)::date) Rows Removed by Filter: 43 -- query with tablesample, space partitioning and planner exclusion :PREFIX SELECT * FROM metrics_space TABLESAMPLE BERNOULLI(5) REPEATABLE(0) WHERE time > '2000-01-10'::timestamptz ORDER BY time DESC, device_id; --- QUERY PLAN --- Sort (actual rows=522.00 loops=1) Sort Key: metrics_space."time" DESC, metrics_space.device_id Sort Method: quicksort -> Append (actual rows=522.00 loops=1) -> Sample Scan on _hyper_6_30_chunk (actual rows=35.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Sample Scan on _hyper_6_29_chunk (actual rows=61.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Sample Scan on _hyper_6_28_chunk (actual rows=61.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Sample Scan on _hyper_6_27_chunk (actual rows=65.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Rows Removed by Filter: 113 -> Sample Scan on _hyper_6_26_chunk (actual rows=150.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Rows Removed by Filter: 218 -> Sample Scan on _hyper_6_25_chunk (actual rows=150.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Rows Removed by Filter: 218 -- test runtime exclusion -- test runtime exclusion with LATERAL and 2 hypertables :PREFIX SELECT m1.time, m2.time FROM metrics_timestamptz m1 LEFT JOIN LATERAL(SELECT time FROM metrics_timestamptz m2 WHERE m1.time = m2.time LIMIT 1) m2 ON true ORDER BY m1.time; --- QUERY PLAN --- Nested Loop Left Join (actual rows=26787.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=26787.00 loops=1) Order: m1."time" -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m1_1 (actual rows=4032.00 loops=1) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_2 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m1_3 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m1_4 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m1_5 (actual rows=4611.00 loops=1) -> Limit (actual rows=1.00 loops=26787) -> Custom Scan (ChunkAppend) on metrics_timestamptz m2 (actual rows=1.00 loops=26787) Chunks excluded during runtime: 4 -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m2_1 (actual rows=1.00 loops=4032) Index Cond: ("time" = m1."time") -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m2_2 (actual rows=1.00 loops=6048) Index Cond: ("time" = m1."time") -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m2_3 (actual rows=1.00 loops=6048) Index Cond: ("time" = m1."time") -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m2_4 (actual rows=1.00 loops=6048) Index Cond: ("time" = m1."time") -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m2_5 (actual rows=1.00 loops=4611) Index Cond: ("time" = m1."time") -- test runtime exclusion and startup exclusions :PREFIX SELECT m1.time, m2.time FROM metrics_timestamptz m1 LEFT JOIN LATERAL(SELECT time FROM metrics_timestamptz m2 WHERE m1.time = m2.time AND m2.time < '2000-01-10'::text::timestamptz LIMIT 1) m2 ON true ORDER BY m1.time; --- QUERY PLAN --- Nested Loop Left Join (actual rows=26787.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=26787.00 loops=1) Order: m1."time" -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m1_1 (actual rows=4032.00 loops=1) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_2 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m1_3 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m1_4 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m1_5 (actual rows=4611.00 loops=1) -> Limit (actual rows=0.00 loops=26787) -> Custom Scan (ChunkAppend) on metrics_timestamptz m2 (actual rows=0.00 loops=26787) Chunks excluded during startup: 3 Chunks excluded during runtime: 1 -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m2_1 (actual rows=1.00 loops=4032) Index Cond: (("time" < ('2000-01-10'::cstring)::timestamp with time zone) AND ("time" = m1."time")) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m2_2 (actual rows=1.00 loops=6048) Index Cond: (("time" < ('2000-01-10'::cstring)::timestamp with time zone) AND ("time" = m1."time")) -- test runtime exclusion does not activate for constraints on non-partitioning columns -- should not use runtime exclusion :PREFIX SELECT * FROM append_test a LEFT JOIN LATERAL(SELECT * FROM join_test j WHERE a.colorid = j.colorid ORDER BY time DESC LIMIT 1) j ON true ORDER BY a.time LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Nested Loop Left Join (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on append_test a (actual rows=1.00 loops=1) Order: a."time" -> Index Scan Backward using _hyper_1_1_chunk_append_test_time_idx on _hyper_1_1_chunk a_1 (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_2_chunk_append_test_time_idx on _hyper_1_2_chunk a_2 (never executed) -> Index Scan Backward using _hyper_1_3_chunk_append_test_time_idx on _hyper_1_3_chunk a_3 (never executed) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on join_test j (actual rows=1.00 loops=1) Order: j."time" DESC Hypertables excluded during runtime: 0 -> Index Scan using _hyper_2_6_chunk_join_test_time_idx on _hyper_2_6_chunk j_1 (actual rows=0.00 loops=1) Filter: (a.colorid = colorid) Rows Removed by Filter: 1 -> Index Scan using _hyper_2_5_chunk_join_test_time_idx on _hyper_2_5_chunk j_2 (actual rows=0.00 loops=1) Filter: (a.colorid = colorid) Rows Removed by Filter: 1 -> Index Scan using _hyper_2_4_chunk_join_test_time_idx on _hyper_2_4_chunk j_3 (actual rows=1.00 loops=1) Filter: (a.colorid = colorid) -- test runtime exclusion with LATERAL and generate_series :PREFIX SELECT g.time FROM generate_series('2000-01-01'::timestamptz, '2000-02-01'::timestamptz, '1d'::interval) g(time) LEFT JOIN LATERAL(SELECT time FROM metrics_timestamptz m WHERE m.time=g.time LIMIT 1) m ON true; --- QUERY PLAN --- Nested Loop Left Join (actual rows=32.00 loops=1) -> Function Scan on generate_series g (actual rows=32.00 loops=1) -> Limit (actual rows=1.00 loops=32) -> Result (actual rows=1.00 loops=32) -> Custom Scan (ChunkAppend) on metrics_timestamptz m (actual rows=1.00 loops=32) Chunks excluded during runtime: 4 -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m_1 (actual rows=1.00 loops=5) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m_2 (actual rows=1.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m_3 (actual rows=1.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m_4 (actual rows=1.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m_5 (actual rows=1.00 loops=6) Index Cond: ("time" = g."time") :PREFIX SELECT * FROM generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval) AS g(time) INNER JOIN LATERAL (SELECT time FROM metrics_timestamptz m WHERE time=g.time) m ON true; --- QUERY PLAN --- Hash Join (actual rows=96.00 loops=1) Hash Cond: (g."time" = m."time") -> Function Scan on generate_series g (actual rows=32.00 loops=1) -> Hash (actual rows=26787.00 loops=1) -> Append (actual rows=26787.00 loops=1) -> Seq Scan on _hyper_5_17_chunk m_1 (actual rows=4032.00 loops=1) -> Seq Scan on _hyper_5_18_chunk m_2 (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_19_chunk m_3 (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_20_chunk m_4 (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_21_chunk m_5 (actual rows=4611.00 loops=1) :PREFIX SELECT * FROM generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval) AS g(time) INNER JOIN LATERAL (SELECT time FROM metrics_timestamptz m WHERE time=g.time ORDER BY time) m ON true; --- QUERY PLAN --- Nested Loop (actual rows=96.00 loops=1) -> Function Scan on generate_series g (actual rows=32.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m (actual rows=3.00 loops=32) Chunks excluded during runtime: 4 -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m_1 (actual rows=3.00 loops=5) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m_2 (actual rows=3.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m_3 (actual rows=3.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m_4 (actual rows=3.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m_5 (actual rows=3.00 loops=6) Index Cond: ("time" = g."time") :PREFIX SELECT * FROM generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval) AS g(time) INNER JOIN LATERAL (SELECT time FROM metrics_timestamptz m WHERE time>g.time + '1 day' ORDER BY time LIMIT 1) m ON true; --- QUERY PLAN --- Nested Loop (actual rows=30.00 loops=1) -> Function Scan on generate_series g (actual rows=32.00 loops=1) -> Limit (actual rows=1.00 loops=32) -> Custom Scan (ChunkAppend) on metrics_timestamptz m (actual rows=1.00 loops=32) Order: m."time" Chunks excluded during startup: 0 Chunks excluded during runtime: 2 -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m_1 (actual rows=1.00 loops=4) Index Cond: ("time" > (g."time" + '@ 1 day'::interval)) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m_2 (actual rows=1.00 loops=7) Index Cond: ("time" > (g."time" + '@ 1 day'::interval)) -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m_3 (actual rows=1.00 loops=7) Index Cond: ("time" > (g."time" + '@ 1 day'::interval)) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m_4 (actual rows=1.00 loops=7) Index Cond: ("time" > (g."time" + '@ 1 day'::interval)) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m_5 (actual rows=1.00 loops=7) Index Cond: ("time" > (g."time" + '@ 1 day'::interval)) -- test runtime exclusion with subquery :PREFIX SELECT m1.time FROM metrics_timestamptz m1 WHERE m1.time=(SELECT max(time) FROM metrics_timestamptz); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=3.00 loops=1) Chunks excluded during runtime: 4 InitPlan 2 (returns $1) -> Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=1.00 loops=1) Order: metrics_timestamptz."time" DESC -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=1.00 loops=1) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk (never executed) Index Cond: ("time" IS NOT NULL) -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m1_1 (never executed) Index Cond: ("time" = $1) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_2 (never executed) Index Cond: ("time" = $1) -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m1_3 (never executed) Index Cond: ("time" = $1) -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m1_4 (never executed) Index Cond: ("time" = $1) -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m1_5 (actual rows=3.00 loops=1) Index Cond: ("time" = $1) -- test runtime exclusion with correlated subquery :PREFIX SELECT m1.time, (SELECT m2.time FROM metrics_timestamptz m2 WHERE m2.time < m1.time ORDER BY m2.time DESC LIMIT 1) FROM metrics_timestamptz m1 WHERE m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Result (actual rows=7776.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=7776.00 loops=1) Order: m1."time" -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m1_1 (actual rows=4032.00 loops=1) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_2 (actual rows=3744.00 loops=1) Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) SubPlan 1 -> Limit (actual rows=1.00 loops=7776) -> Custom Scan (ChunkAppend) on metrics_timestamptz m2 (actual rows=1.00 loops=7776) Order: m2."time" DESC Chunks excluded during runtime: 3 -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m2_1 (never executed) Index Cond: ("time" < m1."time") -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m2_2 (never executed) Index Cond: ("time" < m1."time") -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m2_3 (never executed) Index Cond: ("time" < m1."time") -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m2_4 (actual rows=1.00 loops=3741) Index Cond: ("time" < m1."time") -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m2_5 (actual rows=1.00 loops=4035) Index Cond: ("time" < m1."time") -- test EXISTS :PREFIX SELECT m1.time FROM metrics_timestamptz m1 WHERE EXISTS(SELECT 1 FROM metrics_timestamptz m2 WHERE m1.time < m2.time) ORDER BY m1.time DESC limit 1000; --- QUERY PLAN --- Limit (actual rows=1000.00 loops=1) -> Nested Loop Semi Join (actual rows=1000.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=1003.00 loops=1) Order: m1."time" DESC -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m1_1 (actual rows=1003.00 loops=1) -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m1_2 (never executed) -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m1_3 (never executed) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_4 (never executed) -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m1_5 (never executed) -> Append (actual rows=1.00 loops=1003) -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m2_1 (actual rows=0.00 loops=1003) Index Cond: ("time" > m1."time") -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m2_2 (actual rows=0.00 loops=1003) Index Cond: ("time" > m1."time") -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m2_3 (actual rows=0.00 loops=1003) Index Cond: ("time" > m1."time") -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m2_4 (actual rows=0.00 loops=1003) Index Cond: ("time" > m1."time") -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m2_5 (actual rows=1.00 loops=1003) Index Cond: ("time" > m1."time") -- test constraint exclusion for subqueries with append -- should include 2 chunks :PREFIX SELECT time FROM (SELECT time FROM metrics_timestamptz WHERE time < '2000-01-10'::text::timestamptz ORDER BY time) m; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=7776.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk (actual rows=4032.00 loops=1) Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk (actual rows=3744.00 loops=1) Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -- test constraint exclusion for subqueries with mergeappend -- should include 2 chunks :PREFIX SELECT device_id, time FROM (SELECT device_id, time FROM metrics_timestamptz WHERE time < '2000-01-10'::text::timestamptz ORDER BY device_id, time) m; --- QUERY PLAN --- Custom Scan (ConstraintAwareAppend) (actual rows=7776.00 loops=1) Hypertable: metrics_timestamptz Chunks excluded during startup: 3 -> Merge Append (actual rows=7776.00 loops=1) Sort Key: metrics_timestamptz.device_id, metrics_timestamptz."time" -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_17_chunk (actual rows=4032.00 loops=1) Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_18_chunk (actual rows=3744.00 loops=1) Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -- test LIMIT pushdown -- no aggregates/window functions/SRF should pushdown limit :PREFIX SELECT FROM metrics_timestamptz ORDER BY time LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=1.00 loops=1) Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk (never executed) -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (never executed) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (never executed) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (never executed) -- aggregates should prevent pushdown :PREFIX SELECT count(*) FROM metrics_timestamptz LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=26787.00 loops=1) -> Seq Scan on _hyper_5_17_chunk (actual rows=4032.00 loops=1) -> Seq Scan on _hyper_5_18_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_19_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_20_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_21_chunk (actual rows=4611.00 loops=1) :PREFIX SELECT count(*) FROM metrics_space LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=37450.00 loops=1) -> Seq Scan on _hyper_6_22_chunk (actual rows=5376.00 loops=1) -> Seq Scan on _hyper_6_23_chunk (actual rows=5376.00 loops=1) -> Seq Scan on _hyper_6_24_chunk (actual rows=2688.00 loops=1) -> Seq Scan on _hyper_6_25_chunk (actual rows=8064.00 loops=1) -> Seq Scan on _hyper_6_26_chunk (actual rows=8064.00 loops=1) -> Seq Scan on _hyper_6_27_chunk (actual rows=4032.00 loops=1) -> Seq Scan on _hyper_6_28_chunk (actual rows=1540.00 loops=1) -> Seq Scan on _hyper_6_29_chunk (actual rows=1540.00 loops=1) -> Seq Scan on _hyper_6_30_chunk (actual rows=770.00 loops=1) -- HAVING should prevent pushdown :PREFIX SELECT 1 FROM metrics_timestamptz HAVING count(*) > 1 LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Aggregate (actual rows=1.00 loops=1) Filter: (count(*) > 1) -> Append (actual rows=26787.00 loops=1) -> Seq Scan on _hyper_5_17_chunk (actual rows=4032.00 loops=1) -> Seq Scan on _hyper_5_18_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_19_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_20_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_21_chunk (actual rows=4611.00 loops=1) :PREFIX SELECT 1 FROM metrics_space HAVING count(*) > 1 LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Aggregate (actual rows=1.00 loops=1) Filter: (count(*) > 1) -> Append (actual rows=37450.00 loops=1) -> Seq Scan on _hyper_6_22_chunk (actual rows=5376.00 loops=1) -> Seq Scan on _hyper_6_23_chunk (actual rows=5376.00 loops=1) -> Seq Scan on _hyper_6_24_chunk (actual rows=2688.00 loops=1) -> Seq Scan on _hyper_6_25_chunk (actual rows=8064.00 loops=1) -> Seq Scan on _hyper_6_26_chunk (actual rows=8064.00 loops=1) -> Seq Scan on _hyper_6_27_chunk (actual rows=4032.00 loops=1) -> Seq Scan on _hyper_6_28_chunk (actual rows=1540.00 loops=1) -> Seq Scan on _hyper_6_29_chunk (actual rows=1540.00 loops=1) -> Seq Scan on _hyper_6_30_chunk (actual rows=770.00 loops=1) -- DISTINCT should prevent pushdown SET enable_hashagg TO false; :PREFIX SELECT DISTINCT device_id FROM metrics_timestamptz ORDER BY device_id LIMIT 3; --- QUERY PLAN --- Limit (actual rows=3.00 loops=1) -> Unique (actual rows=3.00 loops=1) -> Merge Append (actual rows=17859.00 loops=1) Sort Key: metrics_timestamptz.device_id -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_17_chunk (actual rows=2689.00 loops=1) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_18_chunk (actual rows=4033.00 loops=1) -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_19_chunk (actual rows=4033.00 loops=1) -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_20_chunk (actual rows=4033.00 loops=1) -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_21_chunk (actual rows=3075.00 loops=1) :PREFIX SELECT DISTINCT device_id FROM metrics_space ORDER BY device_id LIMIT 3; --- QUERY PLAN --- Limit (actual rows=3.00 loops=1) -> Unique (actual rows=3.00 loops=1) -> Merge Append (actual rows=7491.00 loops=1) Sort Key: metrics_space.device_id -> Index Scan using _hyper_6_22_chunk_metrics_space_device_id_time_idx on _hyper_6_22_chunk (actual rows=1345.00 loops=1) -> Index Scan using _hyper_6_23_chunk_metrics_space_device_id_time_idx on _hyper_6_23_chunk (actual rows=1345.00 loops=1) -> Index Scan using _hyper_6_24_chunk_metrics_space_device_id_time_idx on _hyper_6_24_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_6_25_chunk_metrics_space_device_id_time_idx on _hyper_6_25_chunk (actual rows=2017.00 loops=1) -> Index Scan using _hyper_6_26_chunk_metrics_space_device_id_time_idx on _hyper_6_26_chunk (actual rows=2017.00 loops=1) -> Index Scan using _hyper_6_27_chunk_metrics_space_device_id_time_idx on _hyper_6_27_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_6_28_chunk_metrics_space_device_id_time_idx on _hyper_6_28_chunk (actual rows=386.00 loops=1) -> Index Scan using _hyper_6_29_chunk_metrics_space_device_id_time_idx on _hyper_6_29_chunk (actual rows=386.00 loops=1) -> Index Scan using _hyper_6_30_chunk_metrics_space_device_id_time_idx on _hyper_6_30_chunk (actual rows=1.00 loops=1) RESET enable_hashagg; -- JOINs should prevent pushdown -- when LIMIT gets pushed to a Sort node it will switch to top-N heapsort -- if more tuples then LIMIT are requested this will trigger an error -- to trigger this we need a Sort node that is below ChunkAppend CREATE TABLE join_limit (time timestamptz, device_id int); SELECT table_name FROM create_hypertable('join_limit','time',create_default_indexes:=false); table_name ------------ join_limit CREATE INDEX ON join_limit(time,device_id); INSERT INTO join_limit SELECT time, device_id FROM generate_series('2000-01-01'::timestamptz,'2000-01-21','30m') g1(time), generate_series(1,10,1) g2(device_id) ORDER BY time, device_id; VACUUM (ANALYZE) join_limit; -- get 2nd chunk oid SELECT tableoid AS "CHUNK_OID" FROM join_limit WHERE time > '2000-01-07' ORDER BY time LIMIT 1 \gset --get index name for 2nd chunk SELECT indexrelid::regclass AS "INDEX_NAME" FROM pg_index WHERE indrelid = :CHUNK_OID \gset DROP INDEX :INDEX_NAME; :PREFIX SELECT * FROM metrics_timestamptz m1 INNER JOIN join_limit m2 ON m1.time = m2.time AND m1.device_id=m2.device_id WHERE m1.time > '2000-01-07' ORDER BY m1.time, m1.device_id LIMIT 3; --- QUERY PLAN --- Limit (actual rows=3.00 loops=1) -> Merge Join (actual rows=3.00 loops=1) Merge Cond: (m2."time" = m1."time") Join Filter: (m2.device_id = m1.device_id) Rows Removed by Join Filter: 4 -> Custom Scan (ChunkAppend) on join_limit m2 (actual rows=3.00 loops=1) Order: m2."time", m2.device_id -> Sort (actual rows=3.00 loops=1) Sort Key: m2_1."time", m2_1.device_id Sort Method: quicksort -> Seq Scan on _hyper_8_35_chunk m2_1 (actual rows=2710.00 loops=1) Filter: ("time" > 'Fri Jan 07 00:00:00 2000 PST'::timestamp with time zone) Rows Removed by Filter: 650 -> Index Scan using _hyper_8_36_chunk_join_limit_time_device_id_idx on _hyper_8_36_chunk m2_2 (never executed) -> Index Scan using _hyper_8_37_chunk_join_limit_time_device_id_idx on _hyper_8_37_chunk m2_3 (never executed) -> Materialize (actual rows=22.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=19.00 loops=1) Order: m1."time" -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_1 (actual rows=19.00 loops=1) Index Cond: ("time" > 'Fri Jan 07 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m1_2 (never executed) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m1_3 (never executed) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m1_4 (never executed) DROP TABLE join_limit; -- test ChunkAppend projection #2661 :PREFIX SELECT ts.timestamp, ht.timestamp FROM ( SELECT generate_series( to_timestamp(FLOOR(EXTRACT (EPOCH FROM '2020-01-01T00:01:00Z'::timestamp) / 300) * 300) AT TIME ZONE 'UTC', '2020-01-01T01:00:00Z', '5 minutes'::interval ) AS timestamp ) ts LEFT JOIN i2661 ht ON (FLOOR(EXTRACT (EPOCH FROM ht."timestamp") / 300) * 300 = EXTRACT (EPOCH FROM ts.timestamp)) AND ht.timestamp > '2019-12-30T00:00:00Z'::timestamp ORDER BY ts.timestamp, ht.timestamp; --- QUERY PLAN --- Sort (actual rows=33.00 loops=1) Sort Key: ts."timestamp", ht."timestamp" Sort Method: quicksort -> Merge Left Join (actual rows=33.00 loops=1) Merge Cond: ((EXTRACT(epoch FROM ts."timestamp")) = ((floor((EXTRACT(epoch FROM ht."timestamp") / '300'::numeric)) * '300'::numeric))) -> Sort (actual rows=13.00 loops=1) Sort Key: (EXTRACT(epoch FROM ts."timestamp")) Sort Method: quicksort -> Subquery Scan on ts (actual rows=13.00 loops=1) -> ProjectSet (actual rows=13.00 loops=1) -> Result (actual rows=1.00 loops=1) -> Sort (actual rows=514.00 loops=1) Sort Key: ((floor((EXTRACT(epoch FROM ht."timestamp") / '300'::numeric)) * '300'::numeric)) Sort Method: quicksort -> Result (actual rows=7201.00 loops=1) -> Custom Scan (ChunkAppend) on i2661 ht (actual rows=7201.00 loops=1) Chunks excluded during startup: 0 -> Seq Scan on _hyper_7_31_chunk ht_1 (actual rows=1200.00 loops=1) Filter: ("timestamp" > 'Mon Dec 30 00:00:00 2019'::timestamp without time zone) -> Seq Scan on _hyper_7_32_chunk ht_2 (actual rows=5040.00 loops=1) Filter: ("timestamp" > 'Mon Dec 30 00:00:00 2019'::timestamp without time zone) -> Seq Scan on _hyper_7_33_chunk ht_3 (actual rows=961.00 loops=1) Filter: ("timestamp" > 'Mon Dec 30 00:00:00 2019'::timestamp without time zone) -- #3030 test chunkappend keeps pathkeys when subpath is append -- on PG11 this will not use ChunkAppend but MergeAppend SET enable_seqscan TO FALSE; CREATE TABLE i3030(time timestamptz NOT NULL, a int, b int); SELECT table_name FROM create_hypertable('i3030', 'time', create_default_indexes=>false); table_name ------------ i3030 CREATE INDEX ON i3030(a,time); INSERT INTO i3030 (time,a) SELECT time, a FROM generate_series('2000-01-01'::timestamptz,'2000-01-01 3:00:00'::timestamptz,'1min'::interval) time, generate_series(1,30) a; VACUUM (ANALYZE) i3030; :PREFIX SELECT * FROM i3030 where time BETWEEN '2000-01-01'::text::timestamptz AND '2000-01-03'::text::timestamptz ORDER BY a,time LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on i3030 (actual rows=1.00 loops=1) Order: i3030.a, i3030."time" Chunks excluded during startup: 0 -> Index Scan using _hyper_9_38_chunk_i3030_a_time_idx on _hyper_9_38_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= ('2000-01-01'::cstring)::timestamp with time zone) AND ("time" <= ('2000-01-03'::cstring)::timestamp with time zone)) DROP TABLE i3030; RESET enable_seqscan; --parent runtime exclusion tests: --optimization works with ANY (array) :PREFIX SELECT * FROM append_test a WHERE a.attr @> ANY((SELECT coalesce(array_agg(attr), array[]::jsonb[]) FROM join_test_plain WHERE temp > 100)::jsonb[]); --- QUERY PLAN --- Custom Scan (ChunkAppend) on append_test a (actual rows=0.00 loops=1) Hypertables excluded during runtime: 1 InitPlan 1 (returns $0) -> Aggregate (actual rows=1.00 loops=1) -> Seq Scan on join_test_plain (actual rows=0.00 loops=1) Filter: (temp > '100'::double precision) Rows Removed by Filter: 3 -> Seq Scan on _hyper_1_1_chunk a_1 (never executed) Filter: (attr @> ANY ($0)) -> Seq Scan on _hyper_1_2_chunk a_2 (never executed) Filter: (attr @> ANY ($0)) -> Seq Scan on _hyper_1_3_chunk a_3 (never executed) Filter: (attr @> ANY ($0)) --optimization does not work for ANY subquery (does not force an initplan) :PREFIX SELECT * FROM append_test a WHERE a.attr @> ANY((SELECT attr FROM join_test_plain WHERE temp > 100)); --- QUERY PLAN --- Nested Loop Semi Join (actual rows=0.00 loops=1) Join Filter: (a.attr @> join_test_plain.attr) -> Append (actual rows=5.00 loops=1) -> Seq Scan on _hyper_1_1_chunk a_1 (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_2_chunk a_2 (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk a_3 (actual rows=1.00 loops=1) -> Materialize (actual rows=0.00 loops=5) -> Seq Scan on join_test_plain (actual rows=0.00 loops=1) Filter: (temp > '100'::double precision) Rows Removed by Filter: 3 --works on any strict operator without ANY :PREFIX SELECT * FROM append_test a WHERE a.attr @> (SELECT attr FROM join_test_plain WHERE temp > 100 limit 1); --- QUERY PLAN --- Custom Scan (ChunkAppend) on append_test a (actual rows=0.00 loops=1) Hypertables excluded during runtime: 1 InitPlan 1 (returns $0) -> Limit (actual rows=0.00 loops=1) -> Seq Scan on join_test_plain (actual rows=0.00 loops=1) Filter: (temp > '100'::double precision) Rows Removed by Filter: 3 -> Seq Scan on _hyper_1_1_chunk a_1 (never executed) Filter: (attr @> $0) -> Seq Scan on _hyper_1_2_chunk a_2 (never executed) Filter: (attr @> $0) -> Seq Scan on _hyper_1_3_chunk a_3 (never executed) Filter: (attr @> $0) --optimization works with function calls CREATE OR REPLACE FUNCTION select_tag(_min_temp int) RETURNS jsonb[] LANGUAGE sql STABLE PARALLEL SAFE AS $function$ SELECT coalesce(array_agg(attr), array[]::jsonb[]) FROM join_test_plain WHERE temp > _min_temp $function$; :PREFIX SELECT * FROM append_test a WHERE a.attr @> ANY((SELECT select_tag(100))::jsonb[]); --- QUERY PLAN --- Custom Scan (ChunkAppend) on append_test a (actual rows=0.00 loops=1) Hypertables excluded during runtime: 1 InitPlan 1 (returns $0) -> Result (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_1_chunk a_1 (never executed) Filter: (attr @> ANY ($0)) -> Seq Scan on _hyper_1_2_chunk a_2 (never executed) Filter: (attr @> ANY ($0)) -> Seq Scan on _hyper_1_3_chunk a_3 (never executed) Filter: (attr @> ANY ($0)) --optimization does not work when result is null :PREFIX SELECT * FROM append_test a WHERE a.attr @> ANY((SELECT array_agg(attr) FROM join_test_plain WHERE temp > 100)::jsonb[]); --- QUERY PLAN --- Custom Scan (ChunkAppend) on append_test a (actual rows=0.00 loops=1) Hypertables excluded during runtime: 0 InitPlan 1 (returns $0) -> Aggregate (actual rows=1.00 loops=1) -> Seq Scan on join_test_plain (actual rows=0.00 loops=1) Filter: (temp > '100'::double precision) Rows Removed by Filter: 3 -> Seq Scan on _hyper_1_1_chunk a_1 (actual rows=0.00 loops=1) Filter: (attr @> ANY ($0)) Rows Removed by Filter: 2 -> Seq Scan on _hyper_1_2_chunk a_2 (actual rows=0.00 loops=1) Filter: (attr @> ANY ($0)) Rows Removed by Filter: 2 -> Seq Scan on _hyper_1_3_chunk a_3 (actual rows=0.00 loops=1) Filter: (attr @> ANY ($0)) Rows Removed by Filter: 1 -- Test that ConstraintAwareAppend properly locks relations in -- parallel query mode set timescaledb.enable_chunk_append=false; call force_parallel(true); :PREFIX select time, avg(temp), colorid from append_test where time > now_s() - interval '3 months 20 days' group by time, colorid; psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Gather (actual rows=3.00 loops=1) Workers Planned: 1 Workers Launched: 1 Single Copy: true -> HashAggregate (actual rows=3.00 loops=1) Group Key: append_test."time", append_test.colorid -> Custom Scan (ConstraintAwareAppend) (actual rows=3.00 loops=1) Hypertable: append_test Chunks excluded during startup: 1 -> Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) Filter: ("time" > (now_s() - '@ 3 mons 20 days'::interval)) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 3 mons 20 days'::interval)) reset timescaledb.enable_chunk_append; reset max_parallel_workers_per_gather; call force_parallel(false); --generate the results into two different files \set ECHO errors --- Unoptimized results +++ Optimized results @@ -1,6 +1,6 @@ setting | value ----------------------------------+------- - timescaledb.enable_optimizations | off + timescaledb.enable_optimizations | on timescaledb.enable_chunk_append | on --- Unoptimized results +++ Optimized results @@ -1,7 +1,7 @@ setting | value ----------------------------------+------- - timescaledb.enable_optimizations | off - timescaledb.enable_chunk_append | on + timescaledb.enable_optimizations | on + timescaledb.enable_chunk_append | off time | temp | colorid | attr ================================================ FILE: test/expected/append-17.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set TEST_BASE_NAME append SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') as "TEST_LOAD_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') as "TEST_QUERY_NAME", format('%s/results/%s_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_OPTIMIZED", format('%s/results/%s_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_UNOPTIMIZED" \gset SELECT format('\! diff -u --label "Unoptimized results" --label "Optimized results" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') as "DIFF_CMD" \gset SET timescaledb.enable_now_constify TO false; -- disable memoize node to avoid flaky results SET enable_memoize TO 'off'; -- disable index only scans to avoid some flaky results SET enable_indexonlyscan TO FALSE; \set PREFIX 'EXPLAIN (analyze, buffers off, costs off, timing off, summary off)' \ir :TEST_LOAD_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- create a now() function for repeatable testing that always returns -- the same timestamp. It needs to be marked STABLE CREATE OR REPLACE FUNCTION now_s() RETURNS timestamptz LANGUAGE PLPGSQL STABLE PARALLEL SAFE AS $BODY$ BEGIN RAISE NOTICE 'Stable function now_s() called!'; RETURN '2017-08-22T10:00:00'::timestamptz; END; $BODY$; CREATE OR REPLACE FUNCTION now_i() RETURNS timestamptz LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RAISE NOTICE 'Immutable function now_i() called!'; RETURN '2017-08-22T10:00:00'::timestamptz; END; $BODY$; CREATE OR REPLACE FUNCTION now_v() RETURNS timestamptz LANGUAGE PLPGSQL VOLATILE AS $BODY$ BEGIN RAISE NOTICE 'Volatile function now_v() called!'; RETURN '2017-08-22T10:00:00'::timestamptz; END; $BODY$; CREATE OR REPLACE PROCEDURE force_parallel(on_or_off bool) LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('server_version_num')::int < 160000 THEN IF on_or_off THEN set force_parallel_mode = 'on'; ELSE set force_parallel_mode = 'off'; END IF; ELSE IF on_or_off THEN set debug_parallel_query = 'on'; ELSE set debug_parallel_query = 'off'; END IF; END IF; END; $$; CREATE TABLE append_test(time timestamptz, temp float, colorid integer, attr jsonb); SELECT create_hypertable('append_test', 'time', chunk_time_interval => 2628000000000); create_hypertable -------------------------- (1,public,append_test,t) -- create three chunks INSERT INTO append_test VALUES ('2017-03-22T09:18:22', 23.5, 1, '{"a": 1, "b": 2}'), ('2017-03-22T09:18:23', 21.5, 1, '{"a": 1, "b": 2}'), ('2017-05-22T09:18:22', 36.2, 2, '{"c": 3, "b": 2}'), ('2017-05-22T09:18:23', 15.2, 2, '{"c": 3}'), ('2017-08-22T09:18:22', 34.1, 3, '{"c": 4}'); VACUUM (ANALYZE) append_test; -- Create another hypertable to join with CREATE TABLE join_test(time timestamptz, temp float, colorid integer); SELECT create_hypertable('join_test', 'time', chunk_time_interval => 2628000000000); create_hypertable ------------------------ (2,public,join_test,t) INSERT INTO join_test VALUES ('2017-01-22T09:18:22', 15.2, 1), ('2017-02-22T09:18:22', 24.5, 2), ('2017-08-22T09:18:22', 23.1, 3); VACUUM (ANALYZE) join_test; -- Create another table to join with which is not a hypertable. CREATE TABLE join_test_plain(time timestamptz, temp float, colorid integer, attr jsonb); INSERT INTO join_test_plain VALUES ('2017-01-22T09:18:22', 15.2, 1, '{"a": 1}'), ('2017-02-22T09:18:22', 24.5, 2, '{"b": 2}'), ('2017-08-22T09:18:22', 23.1, 3, '{"c": 3}'); VACUUM (ANALYZE) join_test_plain; -- create hypertable with DATE time dimension CREATE TABLE metrics_date(time DATE NOT NULL); SELECT create_hypertable('metrics_date','time'); create_hypertable --------------------------- (3,public,metrics_date,t) INSERT INTO metrics_date SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval); VACUUM (ANALYZE) metrics_date; -- create hypertable with TIMESTAMP time dimension CREATE TABLE metrics_timestamp(time TIMESTAMP NOT NULL); SELECT create_hypertable('metrics_timestamp','time'); psql:include/append_load.sql:91: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------------- (4,public,metrics_timestamp,t) INSERT INTO metrics_timestamp SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval); VACUUM (ANALYZE) metrics_timestamp; -- create hypertable with TIMESTAMPTZ time dimension CREATE TABLE metrics_timestamptz(time TIMESTAMPTZ NOT NULL, device_id INT NOT NULL); CREATE INDEX ON metrics_timestamptz(device_id,time); SELECT create_hypertable('metrics_timestamptz','time'); create_hypertable ---------------------------------- (5,public,metrics_timestamptz,t) INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval), 1; INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval), 2; INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval), 3; VACUUM (ANALYZE) metrics_timestamptz; -- create space partitioned hypertable CREATE TABLE metrics_space(time timestamptz NOT NULL, device_id int NOT NULL, v1 float, v2 float, v3 text); SELECT create_hypertable('metrics_space','time','device_id',3); create_hypertable ---------------------------- (6,public,metrics_space,t) INSERT INTO metrics_space SELECT time, device_id, device_id + 0.25, device_id + 0.75, device_id FROM generate_series('2000-01-01'::timestamptz, '2000-01-14'::timestamptz, '5m'::interval) g1(time), generate_series(1,10,1) g2(device_id) ORDER BY time, device_id; VACUUM (ANALYZE) metrics_space; -- test ChunkAppend projection #2661 CREATE TABLE i2661 ( machine_id int4 NOT NULL, "name" varchar(255) NOT NULL, "timestamp" timestamptz NOT NULL, "first" float4 NULL ); SELECT create_hypertable('i2661', 'timestamp'); psql:include/append_load.sql:123: WARNING: column type "character varying" used for "name" does not follow best practices create_hypertable -------------------- (7,public,i2661,t) INSERT INTO i2661 SELECT 1, 'speed', generate_series('2019-12-31 00:00:00', '2020-01-10 00:00:00', '2m'::interval), 0; VACUUM (ANALYZE) i2661; \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be the only output of the results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations'),('timescaledb.enable_chunk_append')) v(setting); setting | value ----------------------------------+------- timescaledb.enable_optimizations | on timescaledb.enable_chunk_append | on -- query should exclude all chunks with optimization on :PREFIX SELECT * FROM append_test WHERE time > now_s() + '1 month' ORDER BY time DESC; psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Sort (actual rows=0.00 loops=1) Sort Key: append_test."time" DESC Sort Method: quicksort -> Custom Scan (ChunkAppend) on append_test (actual rows=0.00 loops=1) Chunks excluded during startup: 3 --query should exclude all chunks and be a MergeAppend :PREFIX SELECT * FROM append_test WHERE time > now_s() + '1 month' ORDER BY time DESC limit 1; psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Limit (actual rows=0.00 loops=1) -> Custom Scan (ChunkAppend) on append_test (actual rows=0.00 loops=1) Order: append_test."time" DESC Chunks excluded during startup: 3 -- when optimized, the plan should be a constraint-aware append and -- cover only one chunk. It should be a backward index scan due to -- descending index on time. Should also skip the main table, since it -- cannot hold tuples :PREFIX SELECT * FROM append_test WHERE time > now_s() - interval '2 months'; psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Custom Scan (ChunkAppend) on append_test (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 2 mons'::interval)) -- adding ORDER BY and LIMIT should turn the plan into an optimized -- ordered append plan :PREFIX SELECT * FROM append_test WHERE time > now_s() - interval '2 months' ORDER BY time LIMIT 3; psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Sort (actual rows=1.00 loops=1) Sort Key: append_test."time" Sort Method: quicksort -> Custom Scan (ChunkAppend) on append_test (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 2 mons'::interval)) -- no optimized plan for queries with restrictions that can be -- constified at planning time. Regular planning-time constraint -- exclusion should occur. :PREFIX SELECT * FROM append_test WHERE time > now_i() - interval '2 months' ORDER BY time; psql:include/append_query.sql:37: NOTICE: Immutable function now_i() called! --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: append_test."time" Sort Method: quicksort -> Custom Scan (ChunkAppend) on append_test (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > ('Tue Aug 22 10:00:00 2017 PDT'::timestamp with time zone - '@ 2 mons'::interval)) -- currently, we cannot distinguish between stable and volatile -- functions as far as applying our modified plan. However, volatile -- function should not be pre-evaluated to constants, so no chunk -- exclusion should occur. :PREFIX SELECT * FROM append_test WHERE time > now_v() - interval '2 months' ORDER BY time; psql:include/append_query.sql:45: NOTICE: Volatile function now_v() called! psql:include/append_query.sql:45: NOTICE: Volatile function now_v() called! psql:include/append_query.sql:45: NOTICE: Volatile function now_v() called! psql:include/append_query.sql:45: NOTICE: Volatile function now_v() called! psql:include/append_query.sql:45: NOTICE: Volatile function now_v() called! --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: append_test."time" Sort Method: quicksort -> Custom Scan (ChunkAppend) on append_test (actual rows=1.00 loops=1) Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk (actual rows=0.00 loops=1) Filter: ("time" > (now_v() - '@ 2 mons'::interval)) Rows Removed by Filter: 2 -> Seq Scan on _hyper_1_2_chunk (actual rows=0.00 loops=1) Filter: ("time" > (now_v() - '@ 2 mons'::interval)) Rows Removed by Filter: 2 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_v() - '@ 2 mons'::interval)) -- prepared statement output should be the same regardless of -- optimizations PREPARE query_opt AS SELECT * FROM append_test WHERE time > now_s() - interval '2 months' ORDER BY time; :PREFIX EXECUTE query_opt; psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: append_test."time" Sort Method: quicksort -> Custom Scan (ChunkAppend) on append_test (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 2 mons'::interval)) DEALLOCATE query_opt; -- aggregates should produce same output :PREFIX SELECT date_trunc('year', time) t, avg(temp) FROM append_test WHERE time > now_s() - interval '4 months' GROUP BY t ORDER BY t DESC; psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! --- QUERY PLAN --- GroupAggregate (actual rows=1.00 loops=1) Group Key: (date_trunc('year'::text, append_test."time")) -> Sort (actual rows=3.00 loops=1) Sort Key: (date_trunc('year'::text, append_test."time")) DESC Sort Method: quicksort -> Result (actual rows=3.00 loops=1) -> Custom Scan (ChunkAppend) on append_test (actual rows=3.00 loops=1) Chunks excluded during startup: 1 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 4 mons'::interval)) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) Filter: ("time" > (now_s() - '@ 4 mons'::interval)) -- querying outside the time range should return nothing. This tests -- that ConstraintAwareAppend can handle the case when an Append node -- is turned into a Result node due to no children :PREFIX SELECT date_trunc('year', time) t, avg(temp) FROM append_test WHERE time < '2016-03-22' AND date_part('dow', time) between 1 and 5 GROUP BY t ORDER BY t DESC; --- QUERY PLAN --- GroupAggregate (actual rows=0.00 loops=1) Group Key: (date_trunc('year'::text, "time")) -> Sort (actual rows=0.00 loops=1) Sort Key: (date_trunc('year'::text, "time")) DESC Sort Method: quicksort -> Result (actual rows=0.00 loops=1) One-Time Filter: false -- a parameterized query can safely constify params, so won't be -- optimized by constraint-aware append since regular constraint -- exclusion works just fine PREPARE query_param AS SELECT * FROM append_test WHERE time > $1 ORDER BY time; :PREFIX EXECUTE query_param(now_s() - interval '2 months'); psql:include/append_query.sql:82: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: _hyper_1_3_chunk."time" Sort Method: quicksort -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) DEALLOCATE query_param; --test with cte :PREFIX WITH data AS ( SELECT time_bucket(INTERVAL '30 day', TIME) AS btime, AVG(temp) AS VALUE FROM append_test WHERE TIME > now_s() - INTERVAL '400 day' AND colorid > 0 GROUP BY btime ), period AS ( SELECT time_bucket(INTERVAL '30 day', TIME) AS btime FROM GENERATE_SERIES('2017-03-22T01:01:01', '2017-08-23T01:01:01', INTERVAL '30 day') TIME ) SELECT period.btime, VALUE FROM period LEFT JOIN DATA USING (btime) ORDER BY period.btime; psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Sort (actual rows=6.00 loops=1) Sort Key: (time_bucket('@ 30 days'::interval, "time"."time")) Sort Method: quicksort -> Hash Left Join (actual rows=6.00 loops=1) Hash Cond: (time_bucket('@ 30 days'::interval, "time"."time") = data.btime) -> Function Scan on generate_series "time" (actual rows=6.00 loops=1) -> Hash (actual rows=3.00 loops=1) -> Subquery Scan on data (actual rows=3.00 loops=1) -> HashAggregate (actual rows=3.00 loops=1) Group Key: time_bucket('@ 30 days'::interval, append_test."time") -> Result (actual rows=5.00 loops=1) -> Custom Scan (ChunkAppend) on append_test (actual rows=5.00 loops=1) Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk (actual rows=2.00 loops=1) Filter: ((colorid > 0) AND ("time" > (now_s() - '@ 400 days'::interval))) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) Filter: ((colorid > 0) AND ("time" > (now_s() - '@ 400 days'::interval))) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ((colorid > 0) AND ("time" > (now_s() - '@ 400 days'::interval))) WITH data AS ( SELECT time_bucket(INTERVAL '30 day', TIME) AS btime, AVG(temp) AS VALUE FROM append_test WHERE TIME > now_s() - INTERVAL '400 day' AND colorid > 0 GROUP BY btime ), period AS ( SELECT time_bucket(INTERVAL '30 day', TIME) AS btime FROM GENERATE_SERIES('2017-03-22T01:01:01', '2017-08-23T01:01:01', INTERVAL '30 day') TIME ) SELECT period.btime, VALUE FROM period LEFT JOIN DATA USING (btime) ORDER BY period.btime; psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! btime | value ------------------------------+------- Fri Mar 03 16:00:00 2017 PST | 22.5 Sun Apr 02 17:00:00 2017 PDT | Tue May 02 17:00:00 2017 PDT | 25.7 Thu Jun 01 17:00:00 2017 PDT | Sat Jul 01 17:00:00 2017 PDT | Mon Jul 31 17:00:00 2017 PDT | 34.1 -- force nested loop join with no materialization. This tests that the -- inner ConstraintAwareScan supports resetting its scan for every -- iteration of the outer relation loop set enable_hashjoin = 'off'; set enable_mergejoin = 'off'; set enable_material = 'off'; :PREFIX SELECT * FROM append_test a INNER JOIN join_test j ON (a.colorid = j.colorid) WHERE a.time > now_s() - interval '3 hours' AND j.time > now_s() - interval '3 hours'; psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Nested Loop (actual rows=1.00 loops=1) Join Filter: (a.colorid = j.colorid) -> Custom Scan (ChunkAppend) on append_test a (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_1_3_chunk a_1 (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 3 hours'::interval)) -> Custom Scan (ChunkAppend) on join_test j (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_2_6_chunk j_1 (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 3 hours'::interval)) reset enable_hashjoin; reset enable_mergejoin; reset enable_material; -- test constraint_exclusion with date time dimension and DATE/TIMESTAMP/TIMESTAMPTZ constraints -- the queries should all have 3 chunks :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -- test Const OP Var -- the queries should all have 3 chunks :PREFIX SELECT * FROM metrics_date WHERE '2000-01-15'::date < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_date WHERE '2000-01-15'::timestamp < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_date WHERE '2000-01-15'::timestamptz < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -- test 2 constraints -- the queries should all have 2 chunks :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::date AND time < '2000-01-21'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=1440.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::timestamp AND time < '2000-01-21'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=1440.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::timestamptz AND time < '2000-01-21'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=1440.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -- test constraint_exclusion with timestamp time dimension and DATE/TIMESTAMP/TIMESTAMPTZ constraints -- the queries should all have 3 chunks :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -- test Const OP Var -- the queries should all have 3 chunks :PREFIX SELECT * FROM metrics_timestamp WHERE '2000-01-15'::date < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_timestamp WHERE '2000-01-15'::timestamp < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_timestamp WHERE '2000-01-15'::timestamptz < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -- test 2 constraints -- the queries should all have 2 chunks :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::date AND time < '2000-01-21'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=1727.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::timestamp AND time < '2000-01-21'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=1727.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::timestamptz AND time < '2000-01-21'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=1727.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -- test constraint_exclusion with timestamptz time dimension and DATE/TIMESTAMP/TIMESTAMPTZ constraints -- the queries should all have 3 chunks :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) -- test Const OP Var -- the queries should all have 3 chunks :PREFIX SELECT time FROM metrics_timestamptz WHERE '2000-01-15'::date < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) :PREFIX SELECT time FROM metrics_timestamptz WHERE '2000-01-15'::timestamp < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT time FROM metrics_timestamptz WHERE '2000-01-15'::timestamptz < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) -- test 2 constraints -- the queries should all have 2 chunks :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::date AND time < '2000-01-21'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=5181.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::timestamp AND time < '2000-01-21'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=5181.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::timestamptz AND time < '2000-01-21'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=5181.00 loops=1) Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -- test constraint_exclusion with space partitioning and DATE/TIMESTAMP/TIMESTAMPTZ constraints -- exclusion for constraints with non-matching datatypes not working for space partitioning atm :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -- test Const OP Var -- exclusion for constraints with non-matching datatypes not working for space partitioning atm :PREFIX SELECT time FROM metrics_space WHERE '2000-01-10'::date < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) :PREFIX SELECT time FROM metrics_space WHERE '2000-01-10'::timestamp < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT time FROM metrics_space WHERE '2000-01-10'::timestamptz < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -- test 2 constraints -- exclusion for constraints with non-matching datatypes not working for space partitioning atm :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::date AND time < '2000-01-15'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamp AND time < '2000-01-15'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND time < '2000-01-15'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -- test filtering on space partition :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND device_id = 1 ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=1152.00 loops=1) Order: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_device_id_time_idx on _hyper_6_25_chunk (actual rows=767.00 loops=1) Index Cond: ((device_id = 1) AND ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_device_id_time_idx on _hyper_6_28_chunk (actual rows=385.00 loops=1) Index Cond: ((device_id = 1) AND ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND device_id IN (1,2) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=2304.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=1534.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=767.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = ANY ('{1,2}'::integer[])) Rows Removed by Filter: 2301 -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=767.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = ANY ('{1,2}'::integer[])) Rows Removed by Filter: 2301 -> Merge Append (actual rows=770.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=385.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = ANY ('{1,2}'::integer[])) Rows Removed by Filter: 1155 -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=385.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = ANY ('{1,2}'::integer[])) Rows Removed by Filter: 1155 :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND device_id IN (VALUES(1)) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=1152.00 loops=1) Order: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_device_id_time_idx on _hyper_6_25_chunk (actual rows=767.00 loops=1) Index Cond: ((device_id = 1) AND ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_device_id_time_idx on _hyper_6_28_chunk (actual rows=385.00 loops=1) Index Cond: ((device_id = 1) AND ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND v3 IN (VALUES('1')) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=1152.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=767.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=767.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 2301 -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 3068 -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 1534 -> Merge Append (actual rows=385.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=385.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 1155 -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 1540 -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 770 :PREFIX SELECT * FROM metrics_space WHERE time = (VALUES ('2019-12-24' at time zone 'UTC')) AND v3 NOT IN (VALUES ('1')); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=0.00 loops=1) Chunks excluded during startup: 0 Chunks excluded during runtime: 9 InitPlan 1 -> Result (actual rows=1.00 loops=1) -> Index Scan using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (never executed) Index Cond: ("time" = (InitPlan 1).col1) Filter: (NOT (ANY (v3 = (hashed SubPlan 2).col1))) SubPlan 2 -> Result (never executed) -> Index Scan using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (never executed) Index Cond: ("time" = (InitPlan 1).col1) Filter: (NOT (ANY (v3 = (hashed SubPlan 2).col1))) -> Index Scan using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (never executed) Index Cond: ("time" = (InitPlan 1).col1) Filter: (NOT (ANY (v3 = (hashed SubPlan 2).col1))) -> Index Scan using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (never executed) Index Cond: ("time" = (InitPlan 1).col1) Filter: (NOT (ANY (v3 = (hashed SubPlan 2).col1))) -> Index Scan using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (never executed) Index Cond: ("time" = (InitPlan 1).col1) Filter: (NOT (ANY (v3 = (hashed SubPlan 2).col1))) -> Index Scan using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (never executed) Index Cond: ("time" = (InitPlan 1).col1) Filter: (NOT (ANY (v3 = (hashed SubPlan 2).col1))) -> Index Scan using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (never executed) Index Cond: ("time" = (InitPlan 1).col1) Filter: (NOT (ANY (v3 = (hashed SubPlan 2).col1))) -> Index Scan using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (never executed) Index Cond: ("time" = (InitPlan 1).col1) Filter: (NOT (ANY (v3 = (hashed SubPlan 2).col1))) -> Index Scan using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (never executed) Index Cond: ("time" = (InitPlan 1).col1) Filter: (NOT (ANY (v3 = (hashed SubPlan 2).col1))) -- test CURRENT_DATE -- should be 0 chunks :PREFIX SELECT time FROM metrics_date WHERE time > CURRENT_DATE ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=0.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamp WHERE time > CURRENT_DATE ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=0.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamptz WHERE time > CURRENT_DATE ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=0.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_space WHERE time > CURRENT_DATE ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=0.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -- test CURRENT_TIMESTAMP -- should be 0 chunks :PREFIX SELECT time FROM metrics_date WHERE time > CURRENT_TIMESTAMP ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=0.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamp WHERE time > CURRENT_TIMESTAMP ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=0.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamptz WHERE time > CURRENT_TIMESTAMP ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=0.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_space WHERE time > CURRENT_TIMESTAMP ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=0.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -- test now() -- should be 0 chunks :PREFIX SELECT time FROM metrics_date WHERE time > now() ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=0.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamp WHERE time > now() ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=0.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamptz WHERE time > now() ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=0.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_space WHERE time > now() ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=0.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -- query with tablesample and planner exclusion :PREFIX SELECT * FROM metrics_date TABLESAMPLE BERNOULLI(5) REPEATABLE(0) WHERE time > '2000-01-15' ORDER BY time DESC; --- QUERY PLAN --- Sort (actual rows=217.00 loops=1) Sort Key: metrics_date."time" DESC Sort Method: quicksort -> Append (actual rows=217.00 loops=1) -> Sample Scan on _hyper_3_11_chunk (actual rows=72.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) -> Sample Scan on _hyper_3_10_chunk (actual rows=94.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) -> Sample Scan on _hyper_3_9_chunk (actual rows=51.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > '01-15-2000'::date) Rows Removed by Filter: 43 -- query with tablesample and startup exclusion :PREFIX SELECT * FROM metrics_date TABLESAMPLE BERNOULLI(5) REPEATABLE(0) WHERE time > '2000-01-15'::text::date ORDER BY time DESC; --- QUERY PLAN --- Sort (actual rows=217.00 loops=1) Sort Key: metrics_date."time" DESC Sort Method: quicksort -> Custom Scan (ChunkAppend) on metrics_date (actual rows=217.00 loops=1) Chunks excluded during startup: 2 -> Sample Scan on _hyper_3_11_chunk (actual rows=72.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > ('2000-01-15'::cstring)::date) -> Sample Scan on _hyper_3_10_chunk (actual rows=94.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > ('2000-01-15'::cstring)::date) -> Sample Scan on _hyper_3_9_chunk (actual rows=51.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > ('2000-01-15'::cstring)::date) Rows Removed by Filter: 43 -- query with tablesample, space partitioning and planner exclusion :PREFIX SELECT * FROM metrics_space TABLESAMPLE BERNOULLI(5) REPEATABLE(0) WHERE time > '2000-01-10'::timestamptz ORDER BY time DESC, device_id; --- QUERY PLAN --- Sort (actual rows=522.00 loops=1) Sort Key: metrics_space."time" DESC, metrics_space.device_id Sort Method: quicksort -> Append (actual rows=522.00 loops=1) -> Sample Scan on _hyper_6_30_chunk (actual rows=35.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Sample Scan on _hyper_6_29_chunk (actual rows=61.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Sample Scan on _hyper_6_28_chunk (actual rows=61.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Sample Scan on _hyper_6_27_chunk (actual rows=65.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Rows Removed by Filter: 113 -> Sample Scan on _hyper_6_26_chunk (actual rows=150.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Rows Removed by Filter: 218 -> Sample Scan on _hyper_6_25_chunk (actual rows=150.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Rows Removed by Filter: 218 -- test runtime exclusion -- test runtime exclusion with LATERAL and 2 hypertables :PREFIX SELECT m1.time, m2.time FROM metrics_timestamptz m1 LEFT JOIN LATERAL(SELECT time FROM metrics_timestamptz m2 WHERE m1.time = m2.time LIMIT 1) m2 ON true ORDER BY m1.time; --- QUERY PLAN --- Nested Loop Left Join (actual rows=26787.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=26787.00 loops=1) Order: m1."time" -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m1_1 (actual rows=4032.00 loops=1) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_2 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m1_3 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m1_4 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m1_5 (actual rows=4611.00 loops=1) -> Limit (actual rows=1.00 loops=26787) -> Custom Scan (ChunkAppend) on metrics_timestamptz m2 (actual rows=1.00 loops=26787) Chunks excluded during runtime: 4 -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m2_1 (actual rows=1.00 loops=4032) Index Cond: ("time" = m1."time") -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m2_2 (actual rows=1.00 loops=6048) Index Cond: ("time" = m1."time") -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m2_3 (actual rows=1.00 loops=6048) Index Cond: ("time" = m1."time") -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m2_4 (actual rows=1.00 loops=6048) Index Cond: ("time" = m1."time") -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m2_5 (actual rows=1.00 loops=4611) Index Cond: ("time" = m1."time") -- test runtime exclusion and startup exclusions :PREFIX SELECT m1.time, m2.time FROM metrics_timestamptz m1 LEFT JOIN LATERAL(SELECT time FROM metrics_timestamptz m2 WHERE m1.time = m2.time AND m2.time < '2000-01-10'::text::timestamptz LIMIT 1) m2 ON true ORDER BY m1.time; --- QUERY PLAN --- Nested Loop Left Join (actual rows=26787.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=26787.00 loops=1) Order: m1."time" -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m1_1 (actual rows=4032.00 loops=1) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_2 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m1_3 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m1_4 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m1_5 (actual rows=4611.00 loops=1) -> Limit (actual rows=0.00 loops=26787) -> Custom Scan (ChunkAppend) on metrics_timestamptz m2 (actual rows=0.00 loops=26787) Chunks excluded during startup: 3 Chunks excluded during runtime: 1 -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m2_1 (actual rows=1.00 loops=4032) Index Cond: (("time" < ('2000-01-10'::cstring)::timestamp with time zone) AND ("time" = m1."time")) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m2_2 (actual rows=1.00 loops=6048) Index Cond: (("time" < ('2000-01-10'::cstring)::timestamp with time zone) AND ("time" = m1."time")) -- test runtime exclusion does not activate for constraints on non-partitioning columns -- should not use runtime exclusion :PREFIX SELECT * FROM append_test a LEFT JOIN LATERAL(SELECT * FROM join_test j WHERE a.colorid = j.colorid ORDER BY time DESC LIMIT 1) j ON true ORDER BY a.time LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Nested Loop Left Join (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on append_test a (actual rows=1.00 loops=1) Order: a."time" -> Index Scan Backward using _hyper_1_1_chunk_append_test_time_idx on _hyper_1_1_chunk a_1 (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_2_chunk_append_test_time_idx on _hyper_1_2_chunk a_2 (never executed) -> Index Scan Backward using _hyper_1_3_chunk_append_test_time_idx on _hyper_1_3_chunk a_3 (never executed) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on join_test j (actual rows=1.00 loops=1) Order: j."time" DESC Hypertables excluded during runtime: 0 -> Index Scan using _hyper_2_6_chunk_join_test_time_idx on _hyper_2_6_chunk j_1 (actual rows=0.00 loops=1) Filter: (a.colorid = colorid) Rows Removed by Filter: 1 -> Index Scan using _hyper_2_5_chunk_join_test_time_idx on _hyper_2_5_chunk j_2 (actual rows=0.00 loops=1) Filter: (a.colorid = colorid) Rows Removed by Filter: 1 -> Index Scan using _hyper_2_4_chunk_join_test_time_idx on _hyper_2_4_chunk j_3 (actual rows=1.00 loops=1) Filter: (a.colorid = colorid) -- test runtime exclusion with LATERAL and generate_series :PREFIX SELECT g.time FROM generate_series('2000-01-01'::timestamptz, '2000-02-01'::timestamptz, '1d'::interval) g(time) LEFT JOIN LATERAL(SELECT time FROM metrics_timestamptz m WHERE m.time=g.time LIMIT 1) m ON true; --- QUERY PLAN --- Nested Loop Left Join (actual rows=32.00 loops=1) -> Function Scan on generate_series g (actual rows=32.00 loops=1) -> Limit (actual rows=1.00 loops=32) -> Result (actual rows=1.00 loops=32) -> Custom Scan (ChunkAppend) on metrics_timestamptz m (actual rows=1.00 loops=32) Chunks excluded during runtime: 4 -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m_1 (actual rows=1.00 loops=5) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m_2 (actual rows=1.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m_3 (actual rows=1.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m_4 (actual rows=1.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m_5 (actual rows=1.00 loops=6) Index Cond: ("time" = g."time") :PREFIX SELECT * FROM generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval) AS g(time) INNER JOIN LATERAL (SELECT time FROM metrics_timestamptz m WHERE time=g.time) m ON true; --- QUERY PLAN --- Hash Join (actual rows=96.00 loops=1) Hash Cond: (g."time" = m."time") -> Function Scan on generate_series g (actual rows=32.00 loops=1) -> Hash (actual rows=26787.00 loops=1) -> Append (actual rows=26787.00 loops=1) -> Seq Scan on _hyper_5_17_chunk m_1 (actual rows=4032.00 loops=1) -> Seq Scan on _hyper_5_18_chunk m_2 (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_19_chunk m_3 (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_20_chunk m_4 (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_21_chunk m_5 (actual rows=4611.00 loops=1) :PREFIX SELECT * FROM generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval) AS g(time) INNER JOIN LATERAL (SELECT time FROM metrics_timestamptz m WHERE time=g.time ORDER BY time) m ON true; --- QUERY PLAN --- Nested Loop (actual rows=96.00 loops=1) -> Function Scan on generate_series g (actual rows=32.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m (actual rows=3.00 loops=32) Chunks excluded during runtime: 4 -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m_1 (actual rows=3.00 loops=5) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m_2 (actual rows=3.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m_3 (actual rows=3.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m_4 (actual rows=3.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m_5 (actual rows=3.00 loops=6) Index Cond: ("time" = g."time") :PREFIX SELECT * FROM generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval) AS g(time) INNER JOIN LATERAL (SELECT time FROM metrics_timestamptz m WHERE time>g.time + '1 day' ORDER BY time LIMIT 1) m ON true; --- QUERY PLAN --- Nested Loop (actual rows=30.00 loops=1) -> Function Scan on generate_series g (actual rows=32.00 loops=1) -> Limit (actual rows=1.00 loops=32) -> Custom Scan (ChunkAppend) on metrics_timestamptz m (actual rows=1.00 loops=32) Order: m."time" Chunks excluded during startup: 0 Chunks excluded during runtime: 2 -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m_1 (actual rows=1.00 loops=4) Index Cond: ("time" > (g."time" + '@ 1 day'::interval)) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m_2 (actual rows=1.00 loops=7) Index Cond: ("time" > (g."time" + '@ 1 day'::interval)) -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m_3 (actual rows=1.00 loops=7) Index Cond: ("time" > (g."time" + '@ 1 day'::interval)) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m_4 (actual rows=1.00 loops=7) Index Cond: ("time" > (g."time" + '@ 1 day'::interval)) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m_5 (actual rows=1.00 loops=7) Index Cond: ("time" > (g."time" + '@ 1 day'::interval)) -- test runtime exclusion with subquery :PREFIX SELECT m1.time FROM metrics_timestamptz m1 WHERE m1.time=(SELECT max(time) FROM metrics_timestamptz); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=3.00 loops=1) Chunks excluded during runtime: 4 InitPlan 2 -> Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=1.00 loops=1) Order: metrics_timestamptz."time" DESC -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (never executed) -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (never executed) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk (never executed) -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk (never executed) -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m1_1 (never executed) Index Cond: ("time" = (InitPlan 2).col1) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_2 (never executed) Index Cond: ("time" = (InitPlan 2).col1) -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m1_3 (never executed) Index Cond: ("time" = (InitPlan 2).col1) -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m1_4 (never executed) Index Cond: ("time" = (InitPlan 2).col1) -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m1_5 (actual rows=3.00 loops=1) Index Cond: ("time" = (InitPlan 2).col1) -- test runtime exclusion with correlated subquery :PREFIX SELECT m1.time, (SELECT m2.time FROM metrics_timestamptz m2 WHERE m2.time < m1.time ORDER BY m2.time DESC LIMIT 1) FROM metrics_timestamptz m1 WHERE m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Result (actual rows=7776.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=7776.00 loops=1) Order: m1."time" -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m1_1 (actual rows=4032.00 loops=1) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_2 (actual rows=3744.00 loops=1) Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) SubPlan 1 -> Limit (actual rows=1.00 loops=7776) -> Custom Scan (ChunkAppend) on metrics_timestamptz m2 (actual rows=1.00 loops=7776) Order: m2."time" DESC Chunks excluded during runtime: 3 -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m2_1 (never executed) Index Cond: ("time" < m1."time") -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m2_2 (never executed) Index Cond: ("time" < m1."time") -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m2_3 (never executed) Index Cond: ("time" < m1."time") -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m2_4 (actual rows=1.00 loops=3741) Index Cond: ("time" < m1."time") -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m2_5 (actual rows=1.00 loops=4035) Index Cond: ("time" < m1."time") -- test EXISTS :PREFIX SELECT m1.time FROM metrics_timestamptz m1 WHERE EXISTS(SELECT 1 FROM metrics_timestamptz m2 WHERE m1.time < m2.time) ORDER BY m1.time DESC limit 1000; --- QUERY PLAN --- Limit (actual rows=1000.00 loops=1) -> Nested Loop Semi Join (actual rows=1000.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=1003.00 loops=1) Order: m1."time" DESC -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m1_1 (actual rows=1003.00 loops=1) -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m1_2 (never executed) -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m1_3 (never executed) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_4 (never executed) -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m1_5 (never executed) -> Append (actual rows=1.00 loops=1003) -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m2_1 (actual rows=0.00 loops=1003) Index Cond: ("time" > m1."time") -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m2_2 (actual rows=0.00 loops=1003) Index Cond: ("time" > m1."time") -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m2_3 (actual rows=0.00 loops=1003) Index Cond: ("time" > m1."time") -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m2_4 (actual rows=0.00 loops=1003) Index Cond: ("time" > m1."time") -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m2_5 (actual rows=1.00 loops=1003) Index Cond: ("time" > m1."time") -- test constraint exclusion for subqueries with append -- should include 2 chunks :PREFIX SELECT time FROM (SELECT time FROM metrics_timestamptz WHERE time < '2000-01-10'::text::timestamptz ORDER BY time) m; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=7776.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk (actual rows=4032.00 loops=1) Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk (actual rows=3744.00 loops=1) Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -- test constraint exclusion for subqueries with mergeappend -- should include 2 chunks :PREFIX SELECT device_id, time FROM (SELECT device_id, time FROM metrics_timestamptz WHERE time < '2000-01-10'::text::timestamptz ORDER BY device_id, time) m; --- QUERY PLAN --- Custom Scan (ConstraintAwareAppend) (actual rows=7776.00 loops=1) Hypertable: metrics_timestamptz Chunks excluded during startup: 3 -> Merge Append (actual rows=7776.00 loops=1) Sort Key: metrics_timestamptz.device_id, metrics_timestamptz."time" -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_17_chunk (actual rows=4032.00 loops=1) Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_18_chunk (actual rows=3744.00 loops=1) Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -- test LIMIT pushdown -- no aggregates/window functions/SRF should pushdown limit :PREFIX SELECT FROM metrics_timestamptz ORDER BY time LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=1.00 loops=1) Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk (never executed) -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (never executed) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (never executed) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (never executed) -- aggregates should prevent pushdown :PREFIX SELECT count(*) FROM metrics_timestamptz LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=26787.00 loops=1) -> Seq Scan on _hyper_5_17_chunk (actual rows=4032.00 loops=1) -> Seq Scan on _hyper_5_18_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_19_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_20_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_21_chunk (actual rows=4611.00 loops=1) :PREFIX SELECT count(*) FROM metrics_space LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=37450.00 loops=1) -> Seq Scan on _hyper_6_22_chunk (actual rows=5376.00 loops=1) -> Seq Scan on _hyper_6_23_chunk (actual rows=5376.00 loops=1) -> Seq Scan on _hyper_6_24_chunk (actual rows=2688.00 loops=1) -> Seq Scan on _hyper_6_25_chunk (actual rows=8064.00 loops=1) -> Seq Scan on _hyper_6_26_chunk (actual rows=8064.00 loops=1) -> Seq Scan on _hyper_6_27_chunk (actual rows=4032.00 loops=1) -> Seq Scan on _hyper_6_28_chunk (actual rows=1540.00 loops=1) -> Seq Scan on _hyper_6_29_chunk (actual rows=1540.00 loops=1) -> Seq Scan on _hyper_6_30_chunk (actual rows=770.00 loops=1) -- HAVING should prevent pushdown :PREFIX SELECT 1 FROM metrics_timestamptz HAVING count(*) > 1 LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Aggregate (actual rows=1.00 loops=1) Filter: (count(*) > 1) -> Append (actual rows=26787.00 loops=1) -> Seq Scan on _hyper_5_17_chunk (actual rows=4032.00 loops=1) -> Seq Scan on _hyper_5_18_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_19_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_20_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_21_chunk (actual rows=4611.00 loops=1) :PREFIX SELECT 1 FROM metrics_space HAVING count(*) > 1 LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Aggregate (actual rows=1.00 loops=1) Filter: (count(*) > 1) -> Append (actual rows=37450.00 loops=1) -> Seq Scan on _hyper_6_22_chunk (actual rows=5376.00 loops=1) -> Seq Scan on _hyper_6_23_chunk (actual rows=5376.00 loops=1) -> Seq Scan on _hyper_6_24_chunk (actual rows=2688.00 loops=1) -> Seq Scan on _hyper_6_25_chunk (actual rows=8064.00 loops=1) -> Seq Scan on _hyper_6_26_chunk (actual rows=8064.00 loops=1) -> Seq Scan on _hyper_6_27_chunk (actual rows=4032.00 loops=1) -> Seq Scan on _hyper_6_28_chunk (actual rows=1540.00 loops=1) -> Seq Scan on _hyper_6_29_chunk (actual rows=1540.00 loops=1) -> Seq Scan on _hyper_6_30_chunk (actual rows=770.00 loops=1) -- DISTINCT should prevent pushdown SET enable_hashagg TO false; :PREFIX SELECT DISTINCT device_id FROM metrics_timestamptz ORDER BY device_id LIMIT 3; --- QUERY PLAN --- Limit (actual rows=3.00 loops=1) -> Unique (actual rows=3.00 loops=1) -> Merge Append (actual rows=17859.00 loops=1) Sort Key: metrics_timestamptz.device_id -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_17_chunk (actual rows=2689.00 loops=1) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_18_chunk (actual rows=4033.00 loops=1) -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_19_chunk (actual rows=4033.00 loops=1) -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_20_chunk (actual rows=4033.00 loops=1) -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_21_chunk (actual rows=3075.00 loops=1) :PREFIX SELECT DISTINCT device_id FROM metrics_space ORDER BY device_id LIMIT 3; --- QUERY PLAN --- Limit (actual rows=3.00 loops=1) -> Unique (actual rows=3.00 loops=1) -> Merge Append (actual rows=7491.00 loops=1) Sort Key: metrics_space.device_id -> Index Scan using _hyper_6_22_chunk_metrics_space_device_id_time_idx on _hyper_6_22_chunk (actual rows=1345.00 loops=1) -> Index Scan using _hyper_6_23_chunk_metrics_space_device_id_time_idx on _hyper_6_23_chunk (actual rows=1345.00 loops=1) -> Index Scan using _hyper_6_24_chunk_metrics_space_device_id_time_idx on _hyper_6_24_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_6_25_chunk_metrics_space_device_id_time_idx on _hyper_6_25_chunk (actual rows=2017.00 loops=1) -> Index Scan using _hyper_6_26_chunk_metrics_space_device_id_time_idx on _hyper_6_26_chunk (actual rows=2017.00 loops=1) -> Index Scan using _hyper_6_27_chunk_metrics_space_device_id_time_idx on _hyper_6_27_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_6_28_chunk_metrics_space_device_id_time_idx on _hyper_6_28_chunk (actual rows=386.00 loops=1) -> Index Scan using _hyper_6_29_chunk_metrics_space_device_id_time_idx on _hyper_6_29_chunk (actual rows=386.00 loops=1) -> Index Scan using _hyper_6_30_chunk_metrics_space_device_id_time_idx on _hyper_6_30_chunk (actual rows=1.00 loops=1) RESET enable_hashagg; -- JOINs should prevent pushdown -- when LIMIT gets pushed to a Sort node it will switch to top-N heapsort -- if more tuples then LIMIT are requested this will trigger an error -- to trigger this we need a Sort node that is below ChunkAppend CREATE TABLE join_limit (time timestamptz, device_id int); SELECT table_name FROM create_hypertable('join_limit','time',create_default_indexes:=false); table_name ------------ join_limit CREATE INDEX ON join_limit(time,device_id); INSERT INTO join_limit SELECT time, device_id FROM generate_series('2000-01-01'::timestamptz,'2000-01-21','30m') g1(time), generate_series(1,10,1) g2(device_id) ORDER BY time, device_id; VACUUM (ANALYZE) join_limit; -- get 2nd chunk oid SELECT tableoid AS "CHUNK_OID" FROM join_limit WHERE time > '2000-01-07' ORDER BY time LIMIT 1 \gset --get index name for 2nd chunk SELECT indexrelid::regclass AS "INDEX_NAME" FROM pg_index WHERE indrelid = :CHUNK_OID \gset DROP INDEX :INDEX_NAME; :PREFIX SELECT * FROM metrics_timestamptz m1 INNER JOIN join_limit m2 ON m1.time = m2.time AND m1.device_id=m2.device_id WHERE m1.time > '2000-01-07' ORDER BY m1.time, m1.device_id LIMIT 3; --- QUERY PLAN --- Limit (actual rows=3.00 loops=1) -> Merge Join (actual rows=3.00 loops=1) Merge Cond: (m2."time" = m1."time") Join Filter: (m2.device_id = m1.device_id) Rows Removed by Join Filter: 4 -> Custom Scan (ChunkAppend) on join_limit m2 (actual rows=3.00 loops=1) Order: m2."time", m2.device_id -> Sort (actual rows=3.00 loops=1) Sort Key: m2_1."time", m2_1.device_id Sort Method: quicksort -> Seq Scan on _hyper_8_35_chunk m2_1 (actual rows=2710.00 loops=1) Filter: ("time" > 'Fri Jan 07 00:00:00 2000 PST'::timestamp with time zone) Rows Removed by Filter: 650 -> Index Scan using _hyper_8_36_chunk_join_limit_time_device_id_idx on _hyper_8_36_chunk m2_2 (never executed) -> Index Scan using _hyper_8_37_chunk_join_limit_time_device_id_idx on _hyper_8_37_chunk m2_3 (never executed) -> Materialize (actual rows=22.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=19.00 loops=1) Order: m1."time" -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_1 (actual rows=19.00 loops=1) Index Cond: ("time" > 'Fri Jan 07 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m1_2 (never executed) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m1_3 (never executed) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m1_4 (never executed) DROP TABLE join_limit; -- test ChunkAppend projection #2661 :PREFIX SELECT ts.timestamp, ht.timestamp FROM ( SELECT generate_series( to_timestamp(FLOOR(EXTRACT (EPOCH FROM '2020-01-01T00:01:00Z'::timestamp) / 300) * 300) AT TIME ZONE 'UTC', '2020-01-01T01:00:00Z', '5 minutes'::interval ) AS timestamp ) ts LEFT JOIN i2661 ht ON (FLOOR(EXTRACT (EPOCH FROM ht."timestamp") / 300) * 300 = EXTRACT (EPOCH FROM ts.timestamp)) AND ht.timestamp > '2019-12-30T00:00:00Z'::timestamp ORDER BY ts.timestamp, ht.timestamp; --- QUERY PLAN --- Sort (actual rows=33.00 loops=1) Sort Key: ts."timestamp", ht."timestamp" Sort Method: quicksort -> Merge Left Join (actual rows=33.00 loops=1) Merge Cond: ((EXTRACT(epoch FROM ts."timestamp")) = ((floor((EXTRACT(epoch FROM ht."timestamp") / '300'::numeric)) * '300'::numeric))) -> Sort (actual rows=13.00 loops=1) Sort Key: (EXTRACT(epoch FROM ts."timestamp")) Sort Method: quicksort -> Subquery Scan on ts (actual rows=13.00 loops=1) -> ProjectSet (actual rows=13.00 loops=1) -> Result (actual rows=1.00 loops=1) -> Sort (actual rows=514.00 loops=1) Sort Key: ((floor((EXTRACT(epoch FROM ht."timestamp") / '300'::numeric)) * '300'::numeric)) Sort Method: quicksort -> Result (actual rows=7201.00 loops=1) -> Custom Scan (ChunkAppend) on i2661 ht (actual rows=7201.00 loops=1) Chunks excluded during startup: 0 -> Seq Scan on _hyper_7_31_chunk ht_1 (actual rows=1200.00 loops=1) Filter: ("timestamp" > 'Mon Dec 30 00:00:00 2019'::timestamp without time zone) -> Seq Scan on _hyper_7_32_chunk ht_2 (actual rows=5040.00 loops=1) Filter: ("timestamp" > 'Mon Dec 30 00:00:00 2019'::timestamp without time zone) -> Seq Scan on _hyper_7_33_chunk ht_3 (actual rows=961.00 loops=1) Filter: ("timestamp" > 'Mon Dec 30 00:00:00 2019'::timestamp without time zone) -- #3030 test chunkappend keeps pathkeys when subpath is append -- on PG11 this will not use ChunkAppend but MergeAppend SET enable_seqscan TO FALSE; CREATE TABLE i3030(time timestamptz NOT NULL, a int, b int); SELECT table_name FROM create_hypertable('i3030', 'time', create_default_indexes=>false); table_name ------------ i3030 CREATE INDEX ON i3030(a,time); INSERT INTO i3030 (time,a) SELECT time, a FROM generate_series('2000-01-01'::timestamptz,'2000-01-01 3:00:00'::timestamptz,'1min'::interval) time, generate_series(1,30) a; VACUUM (ANALYZE) i3030; :PREFIX SELECT * FROM i3030 where time BETWEEN '2000-01-01'::text::timestamptz AND '2000-01-03'::text::timestamptz ORDER BY a,time LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on i3030 (actual rows=1.00 loops=1) Order: i3030.a, i3030."time" Chunks excluded during startup: 0 -> Index Scan using _hyper_9_38_chunk_i3030_a_time_idx on _hyper_9_38_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= ('2000-01-01'::cstring)::timestamp with time zone) AND ("time" <= ('2000-01-03'::cstring)::timestamp with time zone)) DROP TABLE i3030; RESET enable_seqscan; --parent runtime exclusion tests: --optimization works with ANY (array) :PREFIX SELECT * FROM append_test a WHERE a.attr @> ANY((SELECT coalesce(array_agg(attr), array[]::jsonb[]) FROM join_test_plain WHERE temp > 100)::jsonb[]); --- QUERY PLAN --- Custom Scan (ChunkAppend) on append_test a (actual rows=0.00 loops=1) Hypertables excluded during runtime: 1 InitPlan 1 -> Aggregate (actual rows=1.00 loops=1) -> Seq Scan on join_test_plain (actual rows=0.00 loops=1) Filter: (temp > '100'::double precision) Rows Removed by Filter: 3 -> Seq Scan on _hyper_1_1_chunk a_1 (never executed) Filter: (attr @> ANY ((InitPlan 1).col1)) -> Seq Scan on _hyper_1_2_chunk a_2 (never executed) Filter: (attr @> ANY ((InitPlan 1).col1)) -> Seq Scan on _hyper_1_3_chunk a_3 (never executed) Filter: (attr @> ANY ((InitPlan 1).col1)) --optimization does not work for ANY subquery (does not force an initplan) :PREFIX SELECT * FROM append_test a WHERE a.attr @> ANY((SELECT attr FROM join_test_plain WHERE temp > 100)); --- QUERY PLAN --- Nested Loop Semi Join (actual rows=0.00 loops=1) Join Filter: (a.attr @> join_test_plain.attr) -> Append (actual rows=5.00 loops=1) -> Seq Scan on _hyper_1_1_chunk a_1 (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_2_chunk a_2 (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk a_3 (actual rows=1.00 loops=1) -> Materialize (actual rows=0.00 loops=5) -> Seq Scan on join_test_plain (actual rows=0.00 loops=1) Filter: (temp > '100'::double precision) Rows Removed by Filter: 3 --works on any strict operator without ANY :PREFIX SELECT * FROM append_test a WHERE a.attr @> (SELECT attr FROM join_test_plain WHERE temp > 100 limit 1); --- QUERY PLAN --- Custom Scan (ChunkAppend) on append_test a (actual rows=0.00 loops=1) Hypertables excluded during runtime: 1 InitPlan 1 -> Limit (actual rows=0.00 loops=1) -> Seq Scan on join_test_plain (actual rows=0.00 loops=1) Filter: (temp > '100'::double precision) Rows Removed by Filter: 3 -> Seq Scan on _hyper_1_1_chunk a_1 (never executed) Filter: (attr @> (InitPlan 1).col1) -> Seq Scan on _hyper_1_2_chunk a_2 (never executed) Filter: (attr @> (InitPlan 1).col1) -> Seq Scan on _hyper_1_3_chunk a_3 (never executed) Filter: (attr @> (InitPlan 1).col1) --optimization works with function calls CREATE OR REPLACE FUNCTION select_tag(_min_temp int) RETURNS jsonb[] LANGUAGE sql STABLE PARALLEL SAFE AS $function$ SELECT coalesce(array_agg(attr), array[]::jsonb[]) FROM join_test_plain WHERE temp > _min_temp $function$; :PREFIX SELECT * FROM append_test a WHERE a.attr @> ANY((SELECT select_tag(100))::jsonb[]); --- QUERY PLAN --- Custom Scan (ChunkAppend) on append_test a (actual rows=0.00 loops=1) Hypertables excluded during runtime: 1 InitPlan 1 -> Result (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_1_chunk a_1 (never executed) Filter: (attr @> ANY ((InitPlan 1).col1)) -> Seq Scan on _hyper_1_2_chunk a_2 (never executed) Filter: (attr @> ANY ((InitPlan 1).col1)) -> Seq Scan on _hyper_1_3_chunk a_3 (never executed) Filter: (attr @> ANY ((InitPlan 1).col1)) --optimization does not work when result is null :PREFIX SELECT * FROM append_test a WHERE a.attr @> ANY((SELECT array_agg(attr) FROM join_test_plain WHERE temp > 100)::jsonb[]); --- QUERY PLAN --- Custom Scan (ChunkAppend) on append_test a (actual rows=0.00 loops=1) Hypertables excluded during runtime: 0 InitPlan 1 -> Aggregate (actual rows=1.00 loops=1) -> Seq Scan on join_test_plain (actual rows=0.00 loops=1) Filter: (temp > '100'::double precision) Rows Removed by Filter: 3 -> Seq Scan on _hyper_1_1_chunk a_1 (actual rows=0.00 loops=1) Filter: (attr @> ANY ((InitPlan 1).col1)) Rows Removed by Filter: 2 -> Seq Scan on _hyper_1_2_chunk a_2 (actual rows=0.00 loops=1) Filter: (attr @> ANY ((InitPlan 1).col1)) Rows Removed by Filter: 2 -> Seq Scan on _hyper_1_3_chunk a_3 (actual rows=0.00 loops=1) Filter: (attr @> ANY ((InitPlan 1).col1)) Rows Removed by Filter: 1 -- Test that ConstraintAwareAppend properly locks relations in -- parallel query mode set timescaledb.enable_chunk_append=false; call force_parallel(true); :PREFIX select time, avg(temp), colorid from append_test where time > now_s() - interval '3 months 20 days' group by time, colorid; psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Gather (actual rows=3.00 loops=1) Workers Planned: 1 Workers Launched: 1 Single Copy: true -> HashAggregate (actual rows=3.00 loops=1) Group Key: append_test."time", append_test.colorid -> Custom Scan (ConstraintAwareAppend) (actual rows=3.00 loops=1) Hypertable: append_test Chunks excluded during startup: 1 -> Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) Filter: ("time" > (now_s() - '@ 3 mons 20 days'::interval)) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 3 mons 20 days'::interval)) reset timescaledb.enable_chunk_append; reset max_parallel_workers_per_gather; call force_parallel(false); --generate the results into two different files \set ECHO errors --- Unoptimized results +++ Optimized results @@ -1,6 +1,6 @@ setting | value ----------------------------------+------- - timescaledb.enable_optimizations | off + timescaledb.enable_optimizations | on timescaledb.enable_chunk_append | on --- Unoptimized results +++ Optimized results @@ -1,7 +1,7 @@ setting | value ----------------------------------+------- - timescaledb.enable_optimizations | off - timescaledb.enable_chunk_append | on + timescaledb.enable_optimizations | on + timescaledb.enable_chunk_append | off time | temp | colorid | attr ================================================ FILE: test/expected/append-18.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set TEST_BASE_NAME append SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') as "TEST_LOAD_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') as "TEST_QUERY_NAME", format('%s/results/%s_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_OPTIMIZED", format('%s/results/%s_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_UNOPTIMIZED" \gset SELECT format('\! diff -u --label "Unoptimized results" --label "Optimized results" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') as "DIFF_CMD" \gset SET timescaledb.enable_now_constify TO false; -- disable memoize node to avoid flaky results SET enable_memoize TO 'off'; -- disable index only scans to avoid some flaky results SET enable_indexonlyscan TO FALSE; \set PREFIX 'EXPLAIN (analyze, buffers off, costs off, timing off, summary off)' \ir :TEST_LOAD_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- create a now() function for repeatable testing that always returns -- the same timestamp. It needs to be marked STABLE CREATE OR REPLACE FUNCTION now_s() RETURNS timestamptz LANGUAGE PLPGSQL STABLE PARALLEL SAFE AS $BODY$ BEGIN RAISE NOTICE 'Stable function now_s() called!'; RETURN '2017-08-22T10:00:00'::timestamptz; END; $BODY$; CREATE OR REPLACE FUNCTION now_i() RETURNS timestamptz LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RAISE NOTICE 'Immutable function now_i() called!'; RETURN '2017-08-22T10:00:00'::timestamptz; END; $BODY$; CREATE OR REPLACE FUNCTION now_v() RETURNS timestamptz LANGUAGE PLPGSQL VOLATILE AS $BODY$ BEGIN RAISE NOTICE 'Volatile function now_v() called!'; RETURN '2017-08-22T10:00:00'::timestamptz; END; $BODY$; CREATE OR REPLACE PROCEDURE force_parallel(on_or_off bool) LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('server_version_num')::int < 160000 THEN IF on_or_off THEN set force_parallel_mode = 'on'; ELSE set force_parallel_mode = 'off'; END IF; ELSE IF on_or_off THEN set debug_parallel_query = 'on'; ELSE set debug_parallel_query = 'off'; END IF; END IF; END; $$; CREATE TABLE append_test(time timestamptz, temp float, colorid integer, attr jsonb); SELECT create_hypertable('append_test', 'time', chunk_time_interval => 2628000000000); create_hypertable -------------------------- (1,public,append_test,t) -- create three chunks INSERT INTO append_test VALUES ('2017-03-22T09:18:22', 23.5, 1, '{"a": 1, "b": 2}'), ('2017-03-22T09:18:23', 21.5, 1, '{"a": 1, "b": 2}'), ('2017-05-22T09:18:22', 36.2, 2, '{"c": 3, "b": 2}'), ('2017-05-22T09:18:23', 15.2, 2, '{"c": 3}'), ('2017-08-22T09:18:22', 34.1, 3, '{"c": 4}'); VACUUM (ANALYZE) append_test; -- Create another hypertable to join with CREATE TABLE join_test(time timestamptz, temp float, colorid integer); SELECT create_hypertable('join_test', 'time', chunk_time_interval => 2628000000000); create_hypertable ------------------------ (2,public,join_test,t) INSERT INTO join_test VALUES ('2017-01-22T09:18:22', 15.2, 1), ('2017-02-22T09:18:22', 24.5, 2), ('2017-08-22T09:18:22', 23.1, 3); VACUUM (ANALYZE) join_test; -- Create another table to join with which is not a hypertable. CREATE TABLE join_test_plain(time timestamptz, temp float, colorid integer, attr jsonb); INSERT INTO join_test_plain VALUES ('2017-01-22T09:18:22', 15.2, 1, '{"a": 1}'), ('2017-02-22T09:18:22', 24.5, 2, '{"b": 2}'), ('2017-08-22T09:18:22', 23.1, 3, '{"c": 3}'); VACUUM (ANALYZE) join_test_plain; -- create hypertable with DATE time dimension CREATE TABLE metrics_date(time DATE NOT NULL); SELECT create_hypertable('metrics_date','time'); create_hypertable --------------------------- (3,public,metrics_date,t) INSERT INTO metrics_date SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval); VACUUM (ANALYZE) metrics_date; -- create hypertable with TIMESTAMP time dimension CREATE TABLE metrics_timestamp(time TIMESTAMP NOT NULL); SELECT create_hypertable('metrics_timestamp','time'); psql:include/append_load.sql:91: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------------- (4,public,metrics_timestamp,t) INSERT INTO metrics_timestamp SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval); VACUUM (ANALYZE) metrics_timestamp; -- create hypertable with TIMESTAMPTZ time dimension CREATE TABLE metrics_timestamptz(time TIMESTAMPTZ NOT NULL, device_id INT NOT NULL); CREATE INDEX ON metrics_timestamptz(device_id,time); SELECT create_hypertable('metrics_timestamptz','time'); create_hypertable ---------------------------------- (5,public,metrics_timestamptz,t) INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval), 1; INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval), 2; INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval), 3; VACUUM (ANALYZE) metrics_timestamptz; -- create space partitioned hypertable CREATE TABLE metrics_space(time timestamptz NOT NULL, device_id int NOT NULL, v1 float, v2 float, v3 text); SELECT create_hypertable('metrics_space','time','device_id',3); create_hypertable ---------------------------- (6,public,metrics_space,t) INSERT INTO metrics_space SELECT time, device_id, device_id + 0.25, device_id + 0.75, device_id FROM generate_series('2000-01-01'::timestamptz, '2000-01-14'::timestamptz, '5m'::interval) g1(time), generate_series(1,10,1) g2(device_id) ORDER BY time, device_id; VACUUM (ANALYZE) metrics_space; -- test ChunkAppend projection #2661 CREATE TABLE i2661 ( machine_id int4 NOT NULL, "name" varchar(255) NOT NULL, "timestamp" timestamptz NOT NULL, "first" float4 NULL ); SELECT create_hypertable('i2661', 'timestamp'); psql:include/append_load.sql:123: WARNING: column type "character varying" used for "name" does not follow best practices create_hypertable -------------------- (7,public,i2661,t) INSERT INTO i2661 SELECT 1, 'speed', generate_series('2019-12-31 00:00:00', '2020-01-10 00:00:00', '2m'::interval), 0; VACUUM (ANALYZE) i2661; \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be the only output of the results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations'),('timescaledb.enable_chunk_append')) v(setting); setting | value ----------------------------------+------- timescaledb.enable_optimizations | on timescaledb.enable_chunk_append | on -- query should exclude all chunks with optimization on :PREFIX SELECT * FROM append_test WHERE time > now_s() + '1 month' ORDER BY time DESC; psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! psql:include/append_query.sql:12: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Sort (actual rows=0.00 loops=1) Sort Key: append_test."time" DESC Sort Method: quicksort -> Custom Scan (ChunkAppend) on append_test (actual rows=0.00 loops=1) Chunks excluded during startup: 3 --query should exclude all chunks and be a MergeAppend :PREFIX SELECT * FROM append_test WHERE time > now_s() + '1 month' ORDER BY time DESC limit 1; psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! psql:include/append_query.sql:17: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Limit (actual rows=0.00 loops=1) -> Custom Scan (ChunkAppend) on append_test (actual rows=0.00 loops=1) Order: append_test."time" DESC Chunks excluded during startup: 3 -- when optimized, the plan should be a constraint-aware append and -- cover only one chunk. It should be a backward index scan due to -- descending index on time. Should also skip the main table, since it -- cannot hold tuples :PREFIX SELECT * FROM append_test WHERE time > now_s() - interval '2 months'; psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! psql:include/append_query.sql:24: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Custom Scan (ChunkAppend) on append_test (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 2 mons'::interval)) -- adding ORDER BY and LIMIT should turn the plan into an optimized -- ordered append plan :PREFIX SELECT * FROM append_test WHERE time > now_s() - interval '2 months' ORDER BY time LIMIT 3; psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! psql:include/append_query.sql:30: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Sort (actual rows=1.00 loops=1) Sort Key: append_test."time" Sort Method: quicksort -> Custom Scan (ChunkAppend) on append_test (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 2 mons'::interval)) -- no optimized plan for queries with restrictions that can be -- constified at planning time. Regular planning-time constraint -- exclusion should occur. :PREFIX SELECT * FROM append_test WHERE time > now_i() - interval '2 months' ORDER BY time; psql:include/append_query.sql:37: NOTICE: Immutable function now_i() called! --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: append_test."time" Sort Method: quicksort -> Custom Scan (ChunkAppend) on append_test (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > ('Tue Aug 22 10:00:00 2017 PDT'::timestamp with time zone - '@ 2 mons'::interval)) -- currently, we cannot distinguish between stable and volatile -- functions as far as applying our modified plan. However, volatile -- function should not be pre-evaluated to constants, so no chunk -- exclusion should occur. :PREFIX SELECT * FROM append_test WHERE time > now_v() - interval '2 months' ORDER BY time; psql:include/append_query.sql:45: NOTICE: Volatile function now_v() called! psql:include/append_query.sql:45: NOTICE: Volatile function now_v() called! psql:include/append_query.sql:45: NOTICE: Volatile function now_v() called! psql:include/append_query.sql:45: NOTICE: Volatile function now_v() called! psql:include/append_query.sql:45: NOTICE: Volatile function now_v() called! --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: append_test."time" Sort Method: quicksort -> Custom Scan (ChunkAppend) on append_test (actual rows=1.00 loops=1) Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk (actual rows=0.00 loops=1) Filter: ("time" > (now_v() - '@ 2 mons'::interval)) Rows Removed by Filter: 2 -> Seq Scan on _hyper_1_2_chunk (actual rows=0.00 loops=1) Filter: ("time" > (now_v() - '@ 2 mons'::interval)) Rows Removed by Filter: 2 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_v() - '@ 2 mons'::interval)) -- prepared statement output should be the same regardless of -- optimizations PREPARE query_opt AS SELECT * FROM append_test WHERE time > now_s() - interval '2 months' ORDER BY time; :PREFIX EXECUTE query_opt; psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! psql:include/append_query.sql:53: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: append_test."time" Sort Method: quicksort -> Custom Scan (ChunkAppend) on append_test (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 2 mons'::interval)) DEALLOCATE query_opt; -- aggregates should produce same output :PREFIX SELECT date_trunc('year', time) t, avg(temp) FROM append_test WHERE time > now_s() - interval '4 months' GROUP BY t ORDER BY t DESC; psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! psql:include/append_query.sql:62: NOTICE: Stable function now_s() called! --- QUERY PLAN --- GroupAggregate (actual rows=1.00 loops=1) Group Key: (date_trunc('year'::text, append_test."time")) -> Sort (actual rows=3.00 loops=1) Sort Key: (date_trunc('year'::text, append_test."time")) DESC Sort Method: quicksort -> Result (actual rows=3.00 loops=1) -> Custom Scan (ChunkAppend) on append_test (actual rows=3.00 loops=1) Chunks excluded during startup: 1 -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 4 mons'::interval)) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) Filter: ("time" > (now_s() - '@ 4 mons'::interval)) -- querying outside the time range should return nothing. This tests -- that ConstraintAwareAppend can handle the case when an Append node -- is turned into a Result node due to no children :PREFIX SELECT date_trunc('year', time) t, avg(temp) FROM append_test WHERE time < '2016-03-22' AND date_part('dow', time) between 1 and 5 GROUP BY t ORDER BY t DESC; --- QUERY PLAN --- GroupAggregate (actual rows=0.00 loops=1) Group Key: (date_trunc('year'::text, "time")) -> Sort (actual rows=0.00 loops=1) Sort Key: (date_trunc('year'::text, "time")) DESC Sort Method: quicksort -> Result (actual rows=0.00 loops=1) One-Time Filter: false -- a parameterized query can safely constify params, so won't be -- optimized by constraint-aware append since regular constraint -- exclusion works just fine PREPARE query_param AS SELECT * FROM append_test WHERE time > $1 ORDER BY time; :PREFIX EXECUTE query_param(now_s() - interval '2 months'); psql:include/append_query.sql:82: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Sort (actual rows=1.00 loops=1) Sort Key: _hyper_1_3_chunk."time" Sort Method: quicksort -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) DEALLOCATE query_param; --test with cte :PREFIX WITH data AS ( SELECT time_bucket(INTERVAL '30 day', TIME) AS btime, AVG(temp) AS VALUE FROM append_test WHERE TIME > now_s() - INTERVAL '400 day' AND colorid > 0 GROUP BY btime ), period AS ( SELECT time_bucket(INTERVAL '30 day', TIME) AS btime FROM GENERATE_SERIES('2017-03-22T01:01:01', '2017-08-23T01:01:01', INTERVAL '30 day') TIME ) SELECT period.btime, VALUE FROM period LEFT JOIN DATA USING (btime) ORDER BY period.btime; psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! psql:include/append_query.sql:102: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Sort (actual rows=6.00 loops=1) Sort Key: (time_bucket('@ 30 days'::interval, "time"."time")) Sort Method: quicksort -> Hash Left Join (actual rows=6.00 loops=1) Hash Cond: (time_bucket('@ 30 days'::interval, "time"."time") = data.btime) -> Function Scan on generate_series "time" (actual rows=6.00 loops=1) -> Hash (actual rows=3.00 loops=1) -> Subquery Scan on data (actual rows=3.00 loops=1) -> HashAggregate (actual rows=3.00 loops=1) Group Key: time_bucket('@ 30 days'::interval, append_test."time") -> Result (actual rows=5.00 loops=1) -> Custom Scan (ChunkAppend) on append_test (actual rows=5.00 loops=1) Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk (actual rows=2.00 loops=1) Filter: ((colorid > 0) AND ("time" > (now_s() - '@ 400 days'::interval))) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) Filter: ((colorid > 0) AND ("time" > (now_s() - '@ 400 days'::interval))) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ((colorid > 0) AND ("time" > (now_s() - '@ 400 days'::interval))) WITH data AS ( SELECT time_bucket(INTERVAL '30 day', TIME) AS btime, AVG(temp) AS VALUE FROM append_test WHERE TIME > now_s() - INTERVAL '400 day' AND colorid > 0 GROUP BY btime ), period AS ( SELECT time_bucket(INTERVAL '30 day', TIME) AS btime FROM GENERATE_SERIES('2017-03-22T01:01:01', '2017-08-23T01:01:01', INTERVAL '30 day') TIME ) SELECT period.btime, VALUE FROM period LEFT JOIN DATA USING (btime) ORDER BY period.btime; psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! psql:include/append_query.sql:119: NOTICE: Stable function now_s() called! btime | value ------------------------------+------- Fri Mar 03 16:00:00 2017 PST | 22.5 Sun Apr 02 17:00:00 2017 PDT | Tue May 02 17:00:00 2017 PDT | 25.7 Thu Jun 01 17:00:00 2017 PDT | Sat Jul 01 17:00:00 2017 PDT | Mon Jul 31 17:00:00 2017 PDT | 34.1 -- force nested loop join with no materialization. This tests that the -- inner ConstraintAwareScan supports resetting its scan for every -- iteration of the outer relation loop set enable_hashjoin = 'off'; set enable_mergejoin = 'off'; set enable_material = 'off'; :PREFIX SELECT * FROM append_test a INNER JOIN join_test j ON (a.colorid = j.colorid) WHERE a.time > now_s() - interval '3 hours' AND j.time > now_s() - interval '3 hours'; psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! psql:include/append_query.sql:130: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Nested Loop (actual rows=1.00 loops=1) Join Filter: (a.colorid = j.colorid) -> Custom Scan (ChunkAppend) on append_test a (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_1_3_chunk a_1 (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 3 hours'::interval)) -> Custom Scan (ChunkAppend) on join_test j (actual rows=1.00 loops=1) Chunks excluded during startup: 2 -> Seq Scan on _hyper_2_6_chunk j_1 (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 3 hours'::interval)) reset enable_hashjoin; reset enable_mergejoin; reset enable_material; -- test constraint_exclusion with date time dimension and DATE/TIMESTAMP/TIMESTAMPTZ constraints -- the queries should all have 3 chunks :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -- test Const OP Var -- the queries should all have 3 chunks :PREFIX SELECT * FROM metrics_date WHERE '2000-01-15'::date < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_date WHERE '2000-01-15'::timestamp < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_date WHERE '2000-01-15'::timestamptz < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=4609.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=2016.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_3_11_chunk_metrics_date_time_idx on _hyper_3_11_chunk (actual rows=1441.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -- test 2 constraints -- the queries should all have 2 chunks :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::date AND time < '2000-01-21'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=1440.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::timestamp AND time < '2000-01-21'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=1440.00 loops=1) Order: metrics_date."time" -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::timestamptz AND time < '2000-01-21'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=1440.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_3_9_chunk_metrics_date_time_idx on _hyper_3_9_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_3_10_chunk_metrics_date_time_idx on _hyper_3_10_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -- test constraint_exclusion with timestamp time dimension and DATE/TIMESTAMP/TIMESTAMPTZ constraints -- the queries should all have 3 chunks :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -- test Const OP Var -- the queries should all have 3 chunks :PREFIX SELECT * FROM metrics_timestamp WHERE '2000-01-15'::date < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_timestamp WHERE '2000-01-15'::timestamp < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) :PREFIX SELECT * FROM metrics_timestamp WHERE '2000-01-15'::timestamptz < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=4896.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=2016.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_4_16_chunk_metrics_timestamp_time_idx on _hyper_4_16_chunk (actual rows=1441.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -- test 2 constraints -- the queries should all have 2 chunks :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::date AND time < '2000-01-21'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=1727.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::timestamp AND time < '2000-01-21'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=1727.00 loops=1) Order: metrics_timestamp."time" -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::timestamptz AND time < '2000-01-21'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=1727.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_4_14_chunk_metrics_timestamp_time_idx on _hyper_4_14_chunk (actual rows=1439.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_4_15_chunk_metrics_timestamp_time_idx on _hyper_4_15_chunk (actual rows=288.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -- test constraint_exclusion with timestamptz time dimension and DATE/TIMESTAMP/TIMESTAMPTZ constraints -- the queries should all have 3 chunks :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) -- test Const OP Var -- the queries should all have 3 chunks :PREFIX SELECT time FROM metrics_timestamptz WHERE '2000-01-15'::date < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) Index Cond: ("time" > '01-15-2000'::date) :PREFIX SELECT time FROM metrics_timestamptz WHERE '2000-01-15'::timestamp < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 2 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT time FROM metrics_timestamptz WHERE '2000-01-15'::timestamptz < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=14688.00 loops=1) Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: ("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=4611.00 loops=1) -- test 2 constraints -- the queries should all have 2 chunks :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::date AND time < '2000-01-21'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=5181.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > '01-15-2000'::date) AND ("time" < '01-21-2000'::date)) :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::timestamp AND time < '2000-01-21'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=5181.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000'::timestamp without time zone)) :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::timestamptz AND time < '2000-01-21'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=5181.00 loops=1) Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (actual rows=4029.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (actual rows=1152.00 loops=1) Index Cond: (("time" > 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 21 00:00:00 2000 PST'::timestamp with time zone)) -- test constraint_exclusion with space partitioning and DATE/TIMESTAMP/TIMESTAMPTZ constraints -- exclusion for constraints with non-matching datatypes not working for space partitioning atm :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -- test Const OP Var -- exclusion for constraints with non-matching datatypes not working for space partitioning atm :PREFIX SELECT time FROM metrics_space WHERE '2000-01-10'::date < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > '01-10-2000'::date) :PREFIX SELECT time FROM metrics_space WHERE '2000-01-10'::timestamp < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT time FROM metrics_space WHERE '2000-01-10'::timestamptz < time ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -- test 2 constraints -- exclusion for constraints with non-matching datatypes not working for space partitioning atm :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::date AND time < '2000-01-15'::date ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: (("time" > '01-10-2000'::date) AND ("time" < '01-15-2000'::date)) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamp AND time < '2000-01-15'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000'::timestamp without time zone)) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND time < '2000-01-15'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=11520.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=7670.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=3068.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=1534.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -> Merge Append (actual rows=3850.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=1540.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=770.00 loops=1) Index Cond: (("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Sat Jan 15 00:00:00 2000 PST'::timestamp with time zone)) -- test filtering on space partition :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND device_id = 1 ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=1152.00 loops=1) Order: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_device_id_time_idx on _hyper_6_25_chunk (actual rows=767.00 loops=1) Index Cond: ((device_id = 1) AND ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_device_id_time_idx on _hyper_6_28_chunk (actual rows=385.00 loops=1) Index Cond: ((device_id = 1) AND ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND device_id IN (1,2) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=2304.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=1534.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=767.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = ANY ('{1,2}'::integer[])) Rows Removed by Filter: 2301 -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=767.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = ANY ('{1,2}'::integer[])) Rows Removed by Filter: 2301 -> Merge Append (actual rows=770.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=385.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = ANY ('{1,2}'::integer[])) Rows Removed by Filter: 1155 -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=385.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = ANY ('{1,2}'::integer[])) Rows Removed by Filter: 1155 :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND device_id IN (VALUES(1)) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=1152.00 loops=1) Order: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_device_id_time_idx on _hyper_6_25_chunk (actual rows=767.00 loops=1) Index Cond: ((device_id = 1) AND ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_device_id_time_idx on _hyper_6_28_chunk (actual rows=385.00 loops=1) Index Cond: ((device_id = 1) AND ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND v3 IN (VALUES('1')) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=1152.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=767.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=767.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 2301 -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 3068 -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 1534 -> Merge Append (actual rows=385.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=385.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 1155 -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 1540 -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (v3 = '1'::text) Rows Removed by Filter: 770 :PREFIX SELECT * FROM metrics_space WHERE time = (VALUES ('2019-12-24' at time zone 'UTC')) AND v3 NOT IN (VALUES ('1')); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=0.00 loops=1) Chunks excluded during startup: 0 Chunks excluded during runtime: 9 InitPlan 1 -> Result (actual rows=1.00 loops=1) -> Index Scan using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (never executed) Index Cond: ("time" = (InitPlan 1).col1) Filter: (NOT (ANY (v3 = (hashed SubPlan 2).col1))) SubPlan 2 -> Result (never executed) -> Index Scan using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (never executed) Index Cond: ("time" = (InitPlan 1).col1) Filter: (NOT (ANY (v3 = (hashed SubPlan 2).col1))) -> Index Scan using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (never executed) Index Cond: ("time" = (InitPlan 1).col1) Filter: (NOT (ANY (v3 = (hashed SubPlan 2).col1))) -> Index Scan using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (never executed) Index Cond: ("time" = (InitPlan 1).col1) Filter: (NOT (ANY (v3 = (hashed SubPlan 2).col1))) -> Index Scan using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (never executed) Index Cond: ("time" = (InitPlan 1).col1) Filter: (NOT (ANY (v3 = (hashed SubPlan 2).col1))) -> Index Scan using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (never executed) Index Cond: ("time" = (InitPlan 1).col1) Filter: (NOT (ANY (v3 = (hashed SubPlan 2).col1))) -> Index Scan using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (never executed) Index Cond: ("time" = (InitPlan 1).col1) Filter: (NOT (ANY (v3 = (hashed SubPlan 2).col1))) -> Index Scan using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (never executed) Index Cond: ("time" = (InitPlan 1).col1) Filter: (NOT (ANY (v3 = (hashed SubPlan 2).col1))) -> Index Scan using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (never executed) Index Cond: ("time" = (InitPlan 1).col1) Filter: (NOT (ANY (v3 = (hashed SubPlan 2).col1))) -- test CURRENT_DATE -- should be 0 chunks :PREFIX SELECT time FROM metrics_date WHERE time > CURRENT_DATE ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=0.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamp WHERE time > CURRENT_DATE ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=0.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamptz WHERE time > CURRENT_DATE ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=0.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_space WHERE time > CURRENT_DATE ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=0.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_DATE) -- test CURRENT_TIMESTAMP -- should be 0 chunks :PREFIX SELECT time FROM metrics_date WHERE time > CURRENT_TIMESTAMP ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=0.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamp WHERE time > CURRENT_TIMESTAMP ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=0.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamptz WHERE time > CURRENT_TIMESTAMP ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=0.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_space WHERE time > CURRENT_TIMESTAMP ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=0.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > CURRENT_TIMESTAMP) -- test now() -- should be 0 chunks :PREFIX SELECT time FROM metrics_date WHERE time > now() ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date (actual rows=0.00 loops=1) Order: metrics_date."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamp WHERE time > now() ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp (actual rows=0.00 loops=1) Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_timestamptz WHERE time > now() ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=0.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX SELECT time FROM metrics_space WHERE time > now() ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_space (actual rows=0.00 loops=1) Order: metrics_space."time" -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_22_chunk_metrics_space_time_idx on _hyper_6_22_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_23_chunk_metrics_space_time_idx on _hyper_6_23_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_24_chunk_metrics_space_time_idx on _hyper_6_24_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_25_chunk_metrics_space_time_idx on _hyper_6_25_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_26_chunk_metrics_space_time_idx on _hyper_6_26_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_27_chunk_metrics_space_time_idx on _hyper_6_27_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Merge Append (actual rows=0.00 loops=1) Sort Key: metrics_space."time" -> Index Scan Backward using _hyper_6_28_chunk_metrics_space_time_idx on _hyper_6_28_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_29_chunk_metrics_space_time_idx on _hyper_6_29_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -> Index Scan Backward using _hyper_6_30_chunk_metrics_space_time_idx on _hyper_6_30_chunk (actual rows=0.00 loops=1) Index Cond: ("time" > now()) -- query with tablesample and planner exclusion :PREFIX SELECT * FROM metrics_date TABLESAMPLE BERNOULLI(5) REPEATABLE(0) WHERE time > '2000-01-15' ORDER BY time DESC; --- QUERY PLAN --- Sort (actual rows=217.00 loops=1) Sort Key: metrics_date."time" DESC Sort Method: quicksort -> Append (actual rows=217.00 loops=1) -> Sample Scan on _hyper_3_11_chunk (actual rows=72.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) -> Sample Scan on _hyper_3_10_chunk (actual rows=94.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) -> Sample Scan on _hyper_3_9_chunk (actual rows=51.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > '01-15-2000'::date) Rows Removed by Filter: 43 -- query with tablesample and startup exclusion :PREFIX SELECT * FROM metrics_date TABLESAMPLE BERNOULLI(5) REPEATABLE(0) WHERE time > '2000-01-15'::text::date ORDER BY time DESC; --- QUERY PLAN --- Sort (actual rows=217.00 loops=1) Sort Key: metrics_date."time" DESC Sort Method: quicksort -> Custom Scan (ChunkAppend) on metrics_date (actual rows=217.00 loops=1) Chunks excluded during startup: 2 -> Sample Scan on _hyper_3_11_chunk (actual rows=72.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > ('2000-01-15'::cstring)::date) -> Sample Scan on _hyper_3_10_chunk (actual rows=94.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > ('2000-01-15'::cstring)::date) -> Sample Scan on _hyper_3_9_chunk (actual rows=51.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > ('2000-01-15'::cstring)::date) Rows Removed by Filter: 43 -- query with tablesample, space partitioning and planner exclusion :PREFIX SELECT * FROM metrics_space TABLESAMPLE BERNOULLI(5) REPEATABLE(0) WHERE time > '2000-01-10'::timestamptz ORDER BY time DESC, device_id; --- QUERY PLAN --- Sort (actual rows=522.00 loops=1) Sort Key: metrics_space."time" DESC, metrics_space.device_id Sort Method: quicksort -> Append (actual rows=522.00 loops=1) -> Sample Scan on _hyper_6_30_chunk (actual rows=35.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Sample Scan on _hyper_6_29_chunk (actual rows=61.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Sample Scan on _hyper_6_28_chunk (actual rows=61.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Sample Scan on _hyper_6_27_chunk (actual rows=65.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Rows Removed by Filter: 113 -> Sample Scan on _hyper_6_26_chunk (actual rows=150.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Rows Removed by Filter: 218 -> Sample Scan on _hyper_6_25_chunk (actual rows=150.00 loops=1) Sampling: bernoulli ('5'::real) REPEATABLE ('0'::double precision) Filter: ("time" > 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Rows Removed by Filter: 218 -- test runtime exclusion -- test runtime exclusion with LATERAL and 2 hypertables :PREFIX SELECT m1.time, m2.time FROM metrics_timestamptz m1 LEFT JOIN LATERAL(SELECT time FROM metrics_timestamptz m2 WHERE m1.time = m2.time LIMIT 1) m2 ON true ORDER BY m1.time; --- QUERY PLAN --- Nested Loop Left Join (actual rows=26787.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=26787.00 loops=1) Order: m1."time" -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m1_1 (actual rows=4032.00 loops=1) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_2 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m1_3 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m1_4 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m1_5 (actual rows=4611.00 loops=1) -> Limit (actual rows=1.00 loops=26787) -> Custom Scan (ChunkAppend) on metrics_timestamptz m2 (actual rows=1.00 loops=26787) Chunks excluded during runtime: 4 -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m2_1 (actual rows=1.00 loops=4032) Index Cond: ("time" = m1."time") -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m2_2 (actual rows=1.00 loops=6048) Index Cond: ("time" = m1."time") -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m2_3 (actual rows=1.00 loops=6048) Index Cond: ("time" = m1."time") -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m2_4 (actual rows=1.00 loops=6048) Index Cond: ("time" = m1."time") -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m2_5 (actual rows=1.00 loops=4611) Index Cond: ("time" = m1."time") -- test runtime exclusion and startup exclusions :PREFIX SELECT m1.time, m2.time FROM metrics_timestamptz m1 LEFT JOIN LATERAL(SELECT time FROM metrics_timestamptz m2 WHERE m1.time = m2.time AND m2.time < '2000-01-10'::text::timestamptz LIMIT 1) m2 ON true ORDER BY m1.time; --- QUERY PLAN --- Nested Loop Left Join (actual rows=26787.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=26787.00 loops=1) Order: m1."time" -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m1_1 (actual rows=4032.00 loops=1) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_2 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m1_3 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m1_4 (actual rows=6048.00 loops=1) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m1_5 (actual rows=4611.00 loops=1) -> Limit (actual rows=0.29 loops=26787) -> Custom Scan (ChunkAppend) on metrics_timestamptz m2 (actual rows=0.29 loops=26787) Chunks excluded during startup: 3 Chunks excluded during runtime: 1 -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m2_1 (actual rows=1.00 loops=4032) Index Cond: (("time" < ('2000-01-10'::cstring)::timestamp with time zone) AND ("time" = m1."time")) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m2_2 (actual rows=0.62 loops=6048) Index Cond: (("time" < ('2000-01-10'::cstring)::timestamp with time zone) AND ("time" = m1."time")) -- test runtime exclusion does not activate for constraints on non-partitioning columns -- should not use runtime exclusion :PREFIX SELECT * FROM append_test a LEFT JOIN LATERAL(SELECT * FROM join_test j WHERE a.colorid = j.colorid ORDER BY time DESC LIMIT 1) j ON true ORDER BY a.time LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Nested Loop Left Join (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on append_test a (actual rows=1.00 loops=1) Order: a."time" -> Index Scan Backward using _hyper_1_1_chunk_append_test_time_idx on _hyper_1_1_chunk a_1 (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_2_chunk_append_test_time_idx on _hyper_1_2_chunk a_2 (never executed) -> Index Scan Backward using _hyper_1_3_chunk_append_test_time_idx on _hyper_1_3_chunk a_3 (never executed) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on join_test j (actual rows=1.00 loops=1) Order: j."time" DESC Hypertables excluded during runtime: 0 -> Index Scan using _hyper_2_6_chunk_join_test_time_idx on _hyper_2_6_chunk j_1 (actual rows=0.00 loops=1) Filter: (a.colorid = colorid) Rows Removed by Filter: 1 -> Index Scan using _hyper_2_5_chunk_join_test_time_idx on _hyper_2_5_chunk j_2 (actual rows=0.00 loops=1) Filter: (a.colorid = colorid) Rows Removed by Filter: 1 -> Index Scan using _hyper_2_4_chunk_join_test_time_idx on _hyper_2_4_chunk j_3 (actual rows=1.00 loops=1) Filter: (a.colorid = colorid) -- test runtime exclusion with LATERAL and generate_series :PREFIX SELECT g.time FROM generate_series('2000-01-01'::timestamptz, '2000-02-01'::timestamptz, '1d'::interval) g(time) LEFT JOIN LATERAL(SELECT time FROM metrics_timestamptz m WHERE m.time=g.time LIMIT 1) m ON true; --- QUERY PLAN --- Nested Loop Left Join (actual rows=32.00 loops=1) -> Function Scan on generate_series g (actual rows=32.00 loops=1) -> Limit (actual rows=1.00 loops=32) -> Result (actual rows=1.00 loops=32) -> Custom Scan (ChunkAppend) on metrics_timestamptz m (actual rows=1.00 loops=32) Chunks excluded during runtime: 4 -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m_1 (actual rows=1.00 loops=5) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m_2 (actual rows=1.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m_3 (actual rows=1.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m_4 (actual rows=1.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m_5 (actual rows=1.00 loops=6) Index Cond: ("time" = g."time") :PREFIX SELECT * FROM generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval) AS g(time) INNER JOIN LATERAL (SELECT time FROM metrics_timestamptz m WHERE time=g.time) m ON true; --- QUERY PLAN --- Nested Loop (actual rows=96.00 loops=1) -> Function Scan on generate_series g (actual rows=32.00 loops=1) -> Append (actual rows=3.00 loops=32) -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m_1 (actual rows=0.47 loops=32) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m_2 (actual rows=0.66 loops=32) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m_3 (actual rows=0.66 loops=32) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m_4 (actual rows=0.66 loops=32) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m_5 (actual rows=0.56 loops=32) Index Cond: ("time" = g."time") :PREFIX SELECT * FROM generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval) AS g(time) INNER JOIN LATERAL (SELECT time FROM metrics_timestamptz m WHERE time=g.time ORDER BY time) m ON true; --- QUERY PLAN --- Nested Loop (actual rows=96.00 loops=1) -> Function Scan on generate_series g (actual rows=32.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m (actual rows=3.00 loops=32) Chunks excluded during runtime: 4 -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m_1 (actual rows=3.00 loops=5) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m_2 (actual rows=3.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m_3 (actual rows=3.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m_4 (actual rows=3.00 loops=7) Index Cond: ("time" = g."time") -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m_5 (actual rows=3.00 loops=6) Index Cond: ("time" = g."time") :PREFIX SELECT * FROM generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval) AS g(time) INNER JOIN LATERAL (SELECT time FROM metrics_timestamptz m WHERE time>g.time + '1 day' ORDER BY time LIMIT 1) m ON true; --- QUERY PLAN --- Nested Loop (actual rows=30.00 loops=1) -> Function Scan on generate_series g (actual rows=32.00 loops=1) -> Limit (actual rows=0.94 loops=32) -> Custom Scan (ChunkAppend) on metrics_timestamptz m (actual rows=0.94 loops=32) Order: m."time" Chunks excluded during startup: 0 Chunks excluded during runtime: 2 -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m_1 (actual rows=1.00 loops=4) Index Cond: ("time" > (g."time" + '@ 1 day'::interval)) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m_2 (actual rows=1.00 loops=7) Index Cond: ("time" > (g."time" + '@ 1 day'::interval)) -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m_3 (actual rows=1.00 loops=7) Index Cond: ("time" > (g."time" + '@ 1 day'::interval)) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m_4 (actual rows=1.00 loops=7) Index Cond: ("time" > (g."time" + '@ 1 day'::interval)) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m_5 (actual rows=0.71 loops=7) Index Cond: ("time" > (g."time" + '@ 1 day'::interval)) -- test runtime exclusion with subquery :PREFIX SELECT m1.time FROM metrics_timestamptz m1 WHERE m1.time=(SELECT max(time) FROM metrics_timestamptz); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=3.00 loops=1) Chunks excluded during runtime: 4 InitPlan 2 -> Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=1.00 loops=1) Order: metrics_timestamptz."time" DESC -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (never executed) -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (never executed) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk (never executed) -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk (never executed) -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m1_1 (never executed) Index Cond: ("time" = (InitPlan 2).col1) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_2 (never executed) Index Cond: ("time" = (InitPlan 2).col1) -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m1_3 (never executed) Index Cond: ("time" = (InitPlan 2).col1) -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m1_4 (never executed) Index Cond: ("time" = (InitPlan 2).col1) -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m1_5 (actual rows=3.00 loops=1) Index Cond: ("time" = (InitPlan 2).col1) -- test runtime exclusion with correlated subquery :PREFIX SELECT m1.time, (SELECT m2.time FROM metrics_timestamptz m2 WHERE m2.time < m1.time ORDER BY m2.time DESC LIMIT 1) FROM metrics_timestamptz m1 WHERE m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Result (actual rows=7776.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=7776.00 loops=1) Order: m1."time" -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m1_1 (actual rows=4032.00 loops=1) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_2 (actual rows=3744.00 loops=1) Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) SubPlan 1 -> Limit (actual rows=1.00 loops=7776) -> Custom Scan (ChunkAppend) on metrics_timestamptz m2 (actual rows=1.00 loops=7776) Order: m2."time" DESC Chunks excluded during runtime: 3 -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m2_1 (never executed) Index Cond: ("time" < m1."time") -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m2_2 (never executed) Index Cond: ("time" < m1."time") -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m2_3 (never executed) Index Cond: ("time" < m1."time") -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m2_4 (actual rows=1.00 loops=3741) Index Cond: ("time" < m1."time") -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m2_5 (actual rows=1.00 loops=4035) Index Cond: ("time" < m1."time") -- test EXISTS :PREFIX SELECT m1.time FROM metrics_timestamptz m1 WHERE EXISTS(SELECT 1 FROM metrics_timestamptz m2 WHERE m1.time < m2.time) ORDER BY m1.time DESC limit 1000; --- QUERY PLAN --- Limit (actual rows=1000.00 loops=1) -> Nested Loop Semi Join (actual rows=1000.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=1003.00 loops=1) Order: m1."time" DESC -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m1_1 (actual rows=1003.00 loops=1) -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m1_2 (never executed) -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m1_3 (never executed) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_4 (never executed) -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m1_5 (never executed) -> Append (actual rows=1.00 loops=1003) -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk m2_1 (actual rows=0.00 loops=1003) Index Cond: ("time" > m1."time") -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m2_2 (actual rows=0.00 loops=1003) Index Cond: ("time" > m1."time") -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m2_3 (actual rows=0.00 loops=1003) Index Cond: ("time" > m1."time") -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m2_4 (actual rows=0.00 loops=1003) Index Cond: ("time" > m1."time") -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m2_5 (actual rows=1.00 loops=1003) Index Cond: ("time" > m1."time") -- test constraint exclusion for subqueries with append -- should include 2 chunks :PREFIX SELECT time FROM (SELECT time FROM metrics_timestamptz WHERE time < '2000-01-10'::text::timestamptz ORDER BY time) m; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=7776.00 loops=1) Order: metrics_timestamptz."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk (actual rows=4032.00 loops=1) Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk (actual rows=3744.00 loops=1) Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -- test constraint exclusion for subqueries with mergeappend -- should include 2 chunks :PREFIX SELECT device_id, time FROM (SELECT device_id, time FROM metrics_timestamptz WHERE time < '2000-01-10'::text::timestamptz ORDER BY device_id, time) m; --- QUERY PLAN --- Custom Scan (ConstraintAwareAppend) (actual rows=7776.00 loops=1) Hypertable: metrics_timestamptz Chunks excluded during startup: 3 -> Merge Append (actual rows=7776.00 loops=1) Sort Key: metrics_timestamptz.device_id, metrics_timestamptz."time" -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_17_chunk (actual rows=4032.00 loops=1) Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_18_chunk (actual rows=3744.00 loops=1) Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -- test LIMIT pushdown -- no aggregates/window functions/SRF should pushdown limit :PREFIX SELECT FROM metrics_timestamptz ORDER BY time LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz (actual rows=1.00 loops=1) Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_5_17_chunk_metrics_timestamptz_time_idx on _hyper_5_17_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk (never executed) -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk (never executed) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk (never executed) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk (never executed) -- aggregates should prevent pushdown :PREFIX SELECT count(*) FROM metrics_timestamptz LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=26787.00 loops=1) -> Seq Scan on _hyper_5_17_chunk (actual rows=4032.00 loops=1) -> Seq Scan on _hyper_5_18_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_19_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_20_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_21_chunk (actual rows=4611.00 loops=1) :PREFIX SELECT count(*) FROM metrics_space LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Aggregate (actual rows=1.00 loops=1) -> Append (actual rows=37450.00 loops=1) -> Seq Scan on _hyper_6_22_chunk (actual rows=5376.00 loops=1) -> Seq Scan on _hyper_6_23_chunk (actual rows=5376.00 loops=1) -> Seq Scan on _hyper_6_24_chunk (actual rows=2688.00 loops=1) -> Seq Scan on _hyper_6_25_chunk (actual rows=8064.00 loops=1) -> Seq Scan on _hyper_6_26_chunk (actual rows=8064.00 loops=1) -> Seq Scan on _hyper_6_27_chunk (actual rows=4032.00 loops=1) -> Seq Scan on _hyper_6_28_chunk (actual rows=1540.00 loops=1) -> Seq Scan on _hyper_6_29_chunk (actual rows=1540.00 loops=1) -> Seq Scan on _hyper_6_30_chunk (actual rows=770.00 loops=1) -- HAVING should prevent pushdown :PREFIX SELECT 1 FROM metrics_timestamptz HAVING count(*) > 1 LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Aggregate (actual rows=1.00 loops=1) Filter: (count(*) > 1) -> Append (actual rows=26787.00 loops=1) -> Seq Scan on _hyper_5_17_chunk (actual rows=4032.00 loops=1) -> Seq Scan on _hyper_5_18_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_19_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_20_chunk (actual rows=6048.00 loops=1) -> Seq Scan on _hyper_5_21_chunk (actual rows=4611.00 loops=1) :PREFIX SELECT 1 FROM metrics_space HAVING count(*) > 1 LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Aggregate (actual rows=1.00 loops=1) Filter: (count(*) > 1) -> Append (actual rows=37450.00 loops=1) -> Seq Scan on _hyper_6_22_chunk (actual rows=5376.00 loops=1) -> Seq Scan on _hyper_6_23_chunk (actual rows=5376.00 loops=1) -> Seq Scan on _hyper_6_24_chunk (actual rows=2688.00 loops=1) -> Seq Scan on _hyper_6_25_chunk (actual rows=8064.00 loops=1) -> Seq Scan on _hyper_6_26_chunk (actual rows=8064.00 loops=1) -> Seq Scan on _hyper_6_27_chunk (actual rows=4032.00 loops=1) -> Seq Scan on _hyper_6_28_chunk (actual rows=1540.00 loops=1) -> Seq Scan on _hyper_6_29_chunk (actual rows=1540.00 loops=1) -> Seq Scan on _hyper_6_30_chunk (actual rows=770.00 loops=1) -- DISTINCT should prevent pushdown SET enable_hashagg TO false; :PREFIX SELECT DISTINCT device_id FROM metrics_timestamptz ORDER BY device_id LIMIT 3; --- QUERY PLAN --- Limit (actual rows=3.00 loops=1) -> Unique (actual rows=3.00 loops=1) -> Merge Append (actual rows=17859.00 loops=1) Sort Key: metrics_timestamptz.device_id -> Index Scan using _hyper_5_17_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_17_chunk (actual rows=2689.00 loops=1) -> Index Scan using _hyper_5_18_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_18_chunk (actual rows=4033.00 loops=1) -> Index Scan using _hyper_5_19_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_19_chunk (actual rows=4033.00 loops=1) -> Index Scan using _hyper_5_20_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_20_chunk (actual rows=4033.00 loops=1) -> Index Scan using _hyper_5_21_chunk_metrics_timestamptz_device_id_time_idx on _hyper_5_21_chunk (actual rows=3075.00 loops=1) :PREFIX SELECT DISTINCT device_id FROM metrics_space ORDER BY device_id LIMIT 3; --- QUERY PLAN --- Limit (actual rows=3.00 loops=1) -> Unique (actual rows=3.00 loops=1) -> Merge Append (actual rows=7491.00 loops=1) Sort Key: metrics_space.device_id -> Index Scan using _hyper_6_22_chunk_metrics_space_device_id_time_idx on _hyper_6_22_chunk (actual rows=1345.00 loops=1) -> Index Scan using _hyper_6_23_chunk_metrics_space_device_id_time_idx on _hyper_6_23_chunk (actual rows=1345.00 loops=1) -> Index Scan using _hyper_6_24_chunk_metrics_space_device_id_time_idx on _hyper_6_24_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_6_25_chunk_metrics_space_device_id_time_idx on _hyper_6_25_chunk (actual rows=2017.00 loops=1) -> Index Scan using _hyper_6_26_chunk_metrics_space_device_id_time_idx on _hyper_6_26_chunk (actual rows=2017.00 loops=1) -> Index Scan using _hyper_6_27_chunk_metrics_space_device_id_time_idx on _hyper_6_27_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_6_28_chunk_metrics_space_device_id_time_idx on _hyper_6_28_chunk (actual rows=386.00 loops=1) -> Index Scan using _hyper_6_29_chunk_metrics_space_device_id_time_idx on _hyper_6_29_chunk (actual rows=386.00 loops=1) -> Index Scan using _hyper_6_30_chunk_metrics_space_device_id_time_idx on _hyper_6_30_chunk (actual rows=1.00 loops=1) RESET enable_hashagg; -- JOINs should prevent pushdown -- when LIMIT gets pushed to a Sort node it will switch to top-N heapsort -- if more tuples then LIMIT are requested this will trigger an error -- to trigger this we need a Sort node that is below ChunkAppend CREATE TABLE join_limit (time timestamptz, device_id int); SELECT table_name FROM create_hypertable('join_limit','time',create_default_indexes:=false); table_name ------------ join_limit CREATE INDEX ON join_limit(time,device_id); INSERT INTO join_limit SELECT time, device_id FROM generate_series('2000-01-01'::timestamptz,'2000-01-21','30m') g1(time), generate_series(1,10,1) g2(device_id) ORDER BY time, device_id; VACUUM (ANALYZE) join_limit; -- get 2nd chunk oid SELECT tableoid AS "CHUNK_OID" FROM join_limit WHERE time > '2000-01-07' ORDER BY time LIMIT 1 \gset --get index name for 2nd chunk SELECT indexrelid::regclass AS "INDEX_NAME" FROM pg_index WHERE indrelid = :CHUNK_OID \gset DROP INDEX :INDEX_NAME; :PREFIX SELECT * FROM metrics_timestamptz m1 INNER JOIN join_limit m2 ON m1.time = m2.time AND m1.device_id=m2.device_id WHERE m1.time > '2000-01-07' ORDER BY m1.time, m1.device_id LIMIT 3; --- QUERY PLAN --- Limit (actual rows=3.00 loops=1) -> Merge Join (actual rows=3.00 loops=1) Merge Cond: (m2."time" = m1."time") Join Filter: (m2.device_id = m1.device_id) Rows Removed by Join Filter: 4 -> Custom Scan (ChunkAppend) on join_limit m2 (actual rows=3.00 loops=1) Order: m2."time", m2.device_id -> Sort (actual rows=3.00 loops=1) Sort Key: m2_1."time", m2_1.device_id Sort Method: quicksort -> Seq Scan on _hyper_8_35_chunk m2_1 (actual rows=2710.00 loops=1) Filter: ("time" > 'Fri Jan 07 00:00:00 2000 PST'::timestamp with time zone) Rows Removed by Filter: 650 -> Index Scan using _hyper_8_36_chunk_join_limit_time_device_id_idx on _hyper_8_36_chunk m2_2 (never executed) -> Index Scan using _hyper_8_37_chunk_join_limit_time_device_id_idx on _hyper_8_37_chunk m2_3 (never executed) -> Materialize (actual rows=22.00 loops=1) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 (actual rows=19.00 loops=1) Order: m1."time" -> Index Scan Backward using _hyper_5_18_chunk_metrics_timestamptz_time_idx on _hyper_5_18_chunk m1_1 (actual rows=19.00 loops=1) Index Cond: ("time" > 'Fri Jan 07 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_5_19_chunk_metrics_timestamptz_time_idx on _hyper_5_19_chunk m1_2 (never executed) -> Index Scan Backward using _hyper_5_20_chunk_metrics_timestamptz_time_idx on _hyper_5_20_chunk m1_3 (never executed) -> Index Scan Backward using _hyper_5_21_chunk_metrics_timestamptz_time_idx on _hyper_5_21_chunk m1_4 (never executed) DROP TABLE join_limit; -- test ChunkAppend projection #2661 :PREFIX SELECT ts.timestamp, ht.timestamp FROM ( SELECT generate_series( to_timestamp(FLOOR(EXTRACT (EPOCH FROM '2020-01-01T00:01:00Z'::timestamp) / 300) * 300) AT TIME ZONE 'UTC', '2020-01-01T01:00:00Z', '5 minutes'::interval ) AS timestamp ) ts LEFT JOIN i2661 ht ON (FLOOR(EXTRACT (EPOCH FROM ht."timestamp") / 300) * 300 = EXTRACT (EPOCH FROM ts.timestamp)) AND ht.timestamp > '2019-12-30T00:00:00Z'::timestamp ORDER BY ts.timestamp, ht.timestamp; --- QUERY PLAN --- Sort (actual rows=33.00 loops=1) Sort Key: (generate_series('Wed Jan 01 00:00:00 2020'::timestamp without time zone, 'Wed Jan 01 01:00:00 2020'::timestamp without time zone, '@ 5 mins'::interval)), ht."timestamp" Sort Method: quicksort -> Hash Right Join (actual rows=33.00 loops=1) Hash Cond: ((floor((EXTRACT(epoch FROM ht."timestamp") / '300'::numeric)) * '300'::numeric) = EXTRACT(epoch FROM (generate_series('Wed Jan 01 00:00:00 2020'::timestamp without time zone, 'Wed Jan 01 01:00:00 2020'::timestamp without time zone, '@ 5 mins'::interval)))) -> Custom Scan (ChunkAppend) on i2661 ht (actual rows=7201.00 loops=1) Chunks excluded during startup: 0 -> Seq Scan on _hyper_7_31_chunk ht_1 (actual rows=1200.00 loops=1) Filter: ("timestamp" > 'Mon Dec 30 00:00:00 2019'::timestamp without time zone) -> Seq Scan on _hyper_7_32_chunk ht_2 (actual rows=5040.00 loops=1) Filter: ("timestamp" > 'Mon Dec 30 00:00:00 2019'::timestamp without time zone) -> Seq Scan on _hyper_7_33_chunk ht_3 (actual rows=961.00 loops=1) Filter: ("timestamp" > 'Mon Dec 30 00:00:00 2019'::timestamp without time zone) -> Hash (actual rows=13.00 loops=1) -> ProjectSet (actual rows=13.00 loops=1) -> Result (actual rows=1.00 loops=1) -- #3030 test chunkappend keeps pathkeys when subpath is append -- on PG11 this will not use ChunkAppend but MergeAppend SET enable_seqscan TO FALSE; CREATE TABLE i3030(time timestamptz NOT NULL, a int, b int); SELECT table_name FROM create_hypertable('i3030', 'time', create_default_indexes=>false); table_name ------------ i3030 CREATE INDEX ON i3030(a,time); INSERT INTO i3030 (time,a) SELECT time, a FROM generate_series('2000-01-01'::timestamptz,'2000-01-01 3:00:00'::timestamptz,'1min'::interval) time, generate_series(1,30) a; VACUUM (ANALYZE) i3030; :PREFIX SELECT * FROM i3030 where time BETWEEN '2000-01-01'::text::timestamptz AND '2000-01-03'::text::timestamptz ORDER BY a,time LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on i3030 (actual rows=1.00 loops=1) Order: i3030.a, i3030."time" Chunks excluded during startup: 0 -> Index Scan using _hyper_9_38_chunk_i3030_a_time_idx on _hyper_9_38_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= ('2000-01-01'::cstring)::timestamp with time zone) AND ("time" <= ('2000-01-03'::cstring)::timestamp with time zone)) DROP TABLE i3030; RESET enable_seqscan; --parent runtime exclusion tests: --optimization works with ANY (array) :PREFIX SELECT * FROM append_test a WHERE a.attr @> ANY((SELECT coalesce(array_agg(attr), array[]::jsonb[]) FROM join_test_plain WHERE temp > 100)::jsonb[]); --- QUERY PLAN --- Custom Scan (ChunkAppend) on append_test a (actual rows=0.00 loops=1) Hypertables excluded during runtime: 1 InitPlan 1 -> Aggregate (actual rows=1.00 loops=1) -> Seq Scan on join_test_plain (actual rows=0.00 loops=1) Filter: (temp > '100'::double precision) Rows Removed by Filter: 3 -> Seq Scan on _hyper_1_1_chunk a_1 (never executed) Filter: (attr @> ANY ((InitPlan 1).col1)) -> Seq Scan on _hyper_1_2_chunk a_2 (never executed) Filter: (attr @> ANY ((InitPlan 1).col1)) -> Seq Scan on _hyper_1_3_chunk a_3 (never executed) Filter: (attr @> ANY ((InitPlan 1).col1)) --optimization does not work for ANY subquery (does not force an initplan) :PREFIX SELECT * FROM append_test a WHERE a.attr @> ANY((SELECT attr FROM join_test_plain WHERE temp > 100)); --- QUERY PLAN --- Nested Loop Semi Join (actual rows=0.00 loops=1) Join Filter: (a.attr @> join_test_plain.attr) -> Append (actual rows=5.00 loops=1) -> Seq Scan on _hyper_1_1_chunk a_1 (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_2_chunk a_2 (actual rows=2.00 loops=1) -> Seq Scan on _hyper_1_3_chunk a_3 (actual rows=1.00 loops=1) -> Materialize (actual rows=0.00 loops=5) -> Seq Scan on join_test_plain (actual rows=0.00 loops=1) Filter: (temp > '100'::double precision) Rows Removed by Filter: 3 --works on any strict operator without ANY :PREFIX SELECT * FROM append_test a WHERE a.attr @> (SELECT attr FROM join_test_plain WHERE temp > 100 limit 1); --- QUERY PLAN --- Custom Scan (ChunkAppend) on append_test a (actual rows=0.00 loops=1) Hypertables excluded during runtime: 1 InitPlan 1 -> Limit (actual rows=0.00 loops=1) -> Seq Scan on join_test_plain (actual rows=0.00 loops=1) Filter: (temp > '100'::double precision) Rows Removed by Filter: 3 -> Seq Scan on _hyper_1_1_chunk a_1 (never executed) Filter: (attr @> (InitPlan 1).col1) -> Seq Scan on _hyper_1_2_chunk a_2 (never executed) Filter: (attr @> (InitPlan 1).col1) -> Seq Scan on _hyper_1_3_chunk a_3 (never executed) Filter: (attr @> (InitPlan 1).col1) --optimization works with function calls CREATE OR REPLACE FUNCTION select_tag(_min_temp int) RETURNS jsonb[] LANGUAGE sql STABLE PARALLEL SAFE AS $function$ SELECT coalesce(array_agg(attr), array[]::jsonb[]) FROM join_test_plain WHERE temp > _min_temp $function$; :PREFIX SELECT * FROM append_test a WHERE a.attr @> ANY((SELECT select_tag(100))::jsonb[]); --- QUERY PLAN --- Custom Scan (ChunkAppend) on append_test a (actual rows=0.00 loops=1) Hypertables excluded during runtime: 1 InitPlan 1 -> Result (actual rows=1.00 loops=1) -> Seq Scan on _hyper_1_1_chunk a_1 (never executed) Filter: (attr @> ANY ((InitPlan 1).col1)) -> Seq Scan on _hyper_1_2_chunk a_2 (never executed) Filter: (attr @> ANY ((InitPlan 1).col1)) -> Seq Scan on _hyper_1_3_chunk a_3 (never executed) Filter: (attr @> ANY ((InitPlan 1).col1)) --optimization does not work when result is null :PREFIX SELECT * FROM append_test a WHERE a.attr @> ANY((SELECT array_agg(attr) FROM join_test_plain WHERE temp > 100)::jsonb[]); --- QUERY PLAN --- Custom Scan (ChunkAppend) on append_test a (actual rows=0.00 loops=1) Hypertables excluded during runtime: 0 InitPlan 1 -> Aggregate (actual rows=1.00 loops=1) -> Seq Scan on join_test_plain (actual rows=0.00 loops=1) Filter: (temp > '100'::double precision) Rows Removed by Filter: 3 -> Seq Scan on _hyper_1_1_chunk a_1 (actual rows=0.00 loops=1) Filter: (attr @> ANY ((InitPlan 1).col1)) Rows Removed by Filter: 2 -> Seq Scan on _hyper_1_2_chunk a_2 (actual rows=0.00 loops=1) Filter: (attr @> ANY ((InitPlan 1).col1)) Rows Removed by Filter: 2 -> Seq Scan on _hyper_1_3_chunk a_3 (actual rows=0.00 loops=1) Filter: (attr @> ANY ((InitPlan 1).col1)) Rows Removed by Filter: 1 -- Test that ConstraintAwareAppend properly locks relations in -- parallel query mode set timescaledb.enable_chunk_append=false; call force_parallel(true); :PREFIX select time, avg(temp), colorid from append_test where time > now_s() - interval '3 months 20 days' group by time, colorid; psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! psql:include/append_query.sql:415: NOTICE: Stable function now_s() called! --- QUERY PLAN --- Gather (actual rows=3.00 loops=1) Workers Planned: 1 Workers Launched: 1 Single Copy: true -> HashAggregate (actual rows=3.00 loops=1) Group Key: append_test."time", append_test.colorid -> Custom Scan (ConstraintAwareAppend) (actual rows=3.00 loops=1) Hypertable: append_test Chunks excluded during startup: 1 -> Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_1_2_chunk (actual rows=2.00 loops=1) Filter: ("time" > (now_s() - '@ 3 mons 20 days'::interval)) -> Seq Scan on _hyper_1_3_chunk (actual rows=1.00 loops=1) Filter: ("time" > (now_s() - '@ 3 mons 20 days'::interval)) reset timescaledb.enable_chunk_append; reset max_parallel_workers_per_gather; call force_parallel(false); --generate the results into two different files \set ECHO errors --- Unoptimized results +++ Optimized results @@ -1,6 +1,6 @@ setting | value ----------------------------------+------- - timescaledb.enable_optimizations | off + timescaledb.enable_optimizations | on timescaledb.enable_chunk_append | on --- Unoptimized results +++ Optimized results @@ -1,7 +1,7 @@ setting | value ----------------------------------+------- - timescaledb.enable_optimizations | off - timescaledb.enable_chunk_append | on + timescaledb.enable_optimizations | on + timescaledb.enable_chunk_append | off time | temp | colorid | attr ================================================ FILE: test/expected/baserel_cache.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test that the baserel cache is not clobbered if there's an error -- in a SQL function. CREATE TABLE valid_ids ( id UUID PRIMARY KEY ); CREATE FUNCTION DEFAULT_UUID(TEXT DEFAULT '') RETURNS UUID AS $$ BEGIN RETURN COALESCE($1, '')::UUID; EXCEPTION WHEN invalid_text_representation THEN RETURN '00000000-0000-0000-0000-000000000000'; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE FUNCTION KNOWN_ID(UUID, TEXT) RETURNS UUID AS $$ SELECT COALESCE( (SELECT id FROM valid_ids WHERE id = $1), DEFAULT_UUID() ); $$ LANGUAGE SQL; SELECT KNOWN_ID(NULL, ''), KNOWN_ID(NULL, ''); known_id | known_id --------------------------------------+-------------------------------------- 00000000-0000-0000-0000-000000000000 | 00000000-0000-0000-0000-000000000000 ================================================ FILE: test/expected/bgw_launcher.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set TEST_DBNAME_2 :TEST_DBNAME _2 \c :TEST_DBNAME :ROLE_SUPERUSER -- start bgw since they are stopped for tests by default SELECT _timescaledb_functions.start_background_workers(); start_background_workers -------------------------- t CREATE DATABASE :TEST_DBNAME_2; \c :TEST_DBNAME_2 :ROLE_SUPERUSER \ir include/bgw_launcher_utils.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Note on testing: need a couple wrappers that pg_sleep in a loop to wait for changes -- to appear in pg_stat_activity. -- Further Note: PG 9.6 changed what appeared in pg_stat_activity, so the launcher doesn't actually show up. -- we can still test its interactions with its children, but can't test some of the things specific to the launcher. -- So we've added some bits about the version number as needed. CREATE VIEW worker_counts as SELECT count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Launcher') as launcher, count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME') as single_scheduler, count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2') as single_2_scheduler, count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = 'template1') as template1_scheduler FROM pg_stat_activity; CREATE FUNCTION wait_worker_counts(launcher_ct INTEGER, scheduler1_ct INTEGER, scheduler2_ct INTEGER, template1_ct INTEGER) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE r INTEGER; BEGIN FOR i in 1..10 LOOP SELECT COUNT(*) from worker_counts where launcher = launcher_ct AND single_scheduler = scheduler1_ct AND single_2_scheduler = scheduler2_ct and template1_scheduler = template1_ct into r; if(r < 1) THEN PERFORM pg_sleep(0.1); PERFORM pg_stat_clear_snapshot(); ELSE --We have the correct counts! RETURN TRUE; END IF; END LOOP; RETURN FALSE; END $BODY$; CREATE FUNCTION wait_for_bgw_scheduler(_datname NAME, _count INT DEFAULT 1, _ticks INT DEFAULT 10) RETURNS TEXT LANGUAGE PLPGSQL AS $BODY$ DECLARE r INTEGER; BEGIN FOR i in 1.._ticks LOOP SELECT count(*) FROM pg_stat_activity WHERE application_name = 'TimescaleDB Background Worker Scheduler' AND datname = _datname INTO r; IF(r <> _count) THEN PERFORM pg_sleep(0.1); PERFORM pg_stat_clear_snapshot(); ELSE RETURN 'BGW Scheduler found.'; END IF; END LOOP; RETURN 'BGW Scheduler NOT found.'; END $BODY$; CREATE PROCEDURE kill_database_backends(_datname NAME) LANGUAGE PLPGSQL AS $BODY$ DECLARE r INTEGER; BEGIN FOR i in 1..100 LOOP SELECT count(pg_terminate_backend(pg_stat_activity.pid)) FROM pg_stat_activity WHERE datname = _datname AND pg_stat_activity.pid <> pg_backend_pid() INTO r; IF(r = 0) THEN RETURN; END IF; PERFORM pg_sleep(0.1); PERFORM pg_stat_clear_snapshot(); END LOOP; RAISE 'Failed to terminate backends'; END $BODY$; -- When we've connected to test db 2, we should be able to see the cluster launcher -- and the scheduler for test db in pg_stat_activity -- but test db 2 shouldn't have a scheduler because ext not created yet SELECT wait_worker_counts(1,1,0,0); wait_worker_counts -------------------- t -- Now create the extension in test db 2 SET client_min_messages = ERROR; CREATE EXTENSION timescaledb CASCADE; RESET client_min_messages; SELECT wait_worker_counts(1,1,1,0); wait_worker_counts -------------------- t DROP DATABASE :TEST_DBNAME WITH (FORCE); -- Now the db_scheduler for test db should have disappeared SELECT wait_worker_counts(1,0,1,0); wait_worker_counts -------------------- t -- Now let's restart the scheduler in test db 2 and make sure our backend_start changed SELECT backend_start as orig_backend_start FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2' \gset -- We'll do this in a txn so that we can see that the worker locks on our txn before continuing BEGIN; SELECT _timescaledb_functions.restart_background_workers(); restart_background_workers ---------------------------- t SELECT wait_worker_counts(1,0,1,0); wait_worker_counts -------------------- t SELECT (backend_start > :'orig_backend_start'::timestamptz) backend_start_changed, (wait_event = 'virtualxid') wait_event_changed FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2'; backend_start_changed | wait_event_changed -----------------------+-------------------- t | t COMMIT; SELECT wait_worker_counts(1,0,1,0); wait_worker_counts -------------------- t SELECT (wait_event IS DISTINCT FROM 'virtualxid') wait_event_changed FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2'; wait_event_changed -------------------- t -- Test stop SELECT _timescaledb_functions.stop_background_workers(); stop_background_workers ------------------------- t SELECT wait_worker_counts(1,0,0,0); wait_worker_counts -------------------- t -- Make sure it doesn't break if we stop twice in a row SELECT _timescaledb_functions.stop_background_workers(); stop_background_workers ------------------------- t SELECT wait_worker_counts(1,0,0,0); wait_worker_counts -------------------- t -- test start SELECT _timescaledb_functions.start_background_workers(); start_background_workers -------------------------- t SELECT wait_worker_counts(1,0,1,0); wait_worker_counts -------------------- t -- make sure start is idempotent SELECT backend_start as orig_backend_start FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2' \gset -- Since we're doing idempotency tests, we're also going to exercise our queue and start 20 times SELECT _timescaledb_functions.start_background_workers() as start_background_workers, * FROM generate_series(1,20); start_background_workers | generate_series --------------------------+----------------- t | 1 t | 2 t | 3 t | 4 t | 5 t | 6 t | 7 t | 8 t | 9 t | 10 t | 11 t | 12 t | 13 t | 14 t | 15 t | 16 t | 17 t | 18 t | 19 t | 20 -- Here we're waiting to see if something shows up in pg_stat_activity, -- so we have to condition our loop in the opposite way. We'll only wait -- half a second in total as well so that tests don't take too long. CREATE FUNCTION wait_equals(TIMESTAMPTZ, TEXT) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE r BOOLEAN; BEGIN FOR i in 1..5 LOOP SELECT (backend_start = $1::timestamptz) backend_start_unchanged FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = $2 into r; if(r) THEN PERFORM pg_sleep(0.1); PERFORM pg_stat_clear_snapshot(); ELSE RETURN FALSE; END IF; END LOOP; RETURN TRUE; END $BODY$; select wait_equals(:'orig_backend_start', :'TEST_DBNAME_2'); wait_equals ------------- t -- Make sure restart starts a worker even if it is stopped SELECT _timescaledb_functions.stop_background_workers(); stop_background_workers ------------------------- t SELECT wait_worker_counts(1,0,0,0); wait_worker_counts -------------------- t SELECT _timescaledb_functions.restart_background_workers(); restart_background_workers ---------------------------- t SELECT wait_worker_counts(1,0,1,0); wait_worker_counts -------------------- t -- Make sure drop extension statement restarts the worker and on rollback it keeps running -- Now let's restart the scheduler and make sure our backend_start changed SELECT backend_start as orig_backend_start FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2' \gset BEGIN; DROP EXTENSION timescaledb; SELECT wait_worker_counts(1,0,1,0); wait_worker_counts -------------------- t ROLLBACK; CREATE FUNCTION wait_greater(TIMESTAMPTZ,TEXT) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE r BOOLEAN; BEGIN FOR i in 1..10 LOOP SELECT (backend_start > $1::timestamptz) backend_start_changed FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = $2 into r; if(NOT r) THEN PERFORM pg_sleep(0.1); PERFORM pg_stat_clear_snapshot(); ELSE RETURN TRUE; END IF; END LOOP; RETURN FALSE; END $BODY$; SELECT wait_greater(:'orig_backend_start',:'TEST_DBNAME_2'); wait_greater -------------- t -- Make sure canceling the launcher backend causes a restart of schedulers SELECT backend_start as orig_backend_start FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2' \gset SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Launcher'; pg_cancel_backend ------------------- t SELECT wait_worker_counts(1,0,1,0); wait_worker_counts -------------------- t SELECT wait_greater(:'orig_backend_start', :'TEST_DBNAME_2'); wait_greater -------------- t -- Make sure running pre_restore function stops background workers SELECT timescaledb_pre_restore(); timescaledb_pre_restore ------------------------- t SELECT wait_worker_counts(1,0,0,0); wait_worker_counts -------------------- t -- Make sure a restart with restoring on first starts the background worker BEGIN; SELECT _timescaledb_functions.restart_background_workers(); restart_background_workers ---------------------------- t SELECT wait_worker_counts(1,0,1,0); wait_worker_counts -------------------- t COMMIT; -- Then the worker dies when it sees that restoring is on after the txn commits SELECT wait_worker_counts(1,0,0,0); wait_worker_counts -------------------- t --And post_restore starts them BEGIN; SELECT timescaledb_post_restore(); timescaledb_post_restore -------------------------- t SELECT wait_worker_counts(1,0,1,0); wait_worker_counts -------------------- t COMMIT; -- And they stay started SELECT wait_worker_counts(1,0,1,0); wait_worker_counts -------------------- t -- Make sure dropping the extension means that the scheduler is stopped BEGIN; DROP EXTENSION timescaledb; COMMIT; SELECT wait_worker_counts(1,0,0,0); wait_worker_counts -------------------- t -- Test that background workers are stopped with DROP OWNED ALTER ROLE :ROLE_DEFAULT_PERM_USER WITH SUPERUSER; \c :TEST_DBNAME_2 :ROLE_DEFAULT_PERM_USER SET client_min_messages = ERROR; CREATE EXTENSION timescaledb CASCADE; RESET client_min_messages; -- Make sure there is 1 launcher and 1 bgw in test db 2 SELECT wait_worker_counts(launcher_ct=>1, scheduler1_ct=> 0, scheduler2_ct=>1, template1_ct=>0); wait_worker_counts -------------------- t -- drop a non-owner of the extension results in no change to worker counts DROP OWNED BY :ROLE_DEFAULT_PERM_USER_2; SELECT wait_worker_counts(launcher_ct=>1, scheduler1_ct=> 0, scheduler2_ct=>1, template1_ct=>0); wait_worker_counts -------------------- t -- drop of owner of extension results in extension drop and a stop to the bgw DROP OWNED BY :ROLE_DEFAULT_PERM_USER; -- The worker in test db 2 is dead. Note that 0s are respected SELECT wait_worker_counts(launcher_ct=>1, scheduler1_ct=>0, scheduler2_ct=>0, template1_ct=>0); wait_worker_counts -------------------- t \c :TEST_DBNAME_2 :ROLE_SUPERUSER ALTER ROLE :ROLE_DEFAULT_PERM_USER WITH NOSUPERUSER; -- Connect to the template1 database \c template1 \ir include/bgw_launcher_utils.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Note on testing: need a couple wrappers that pg_sleep in a loop to wait for changes -- to appear in pg_stat_activity. -- Further Note: PG 9.6 changed what appeared in pg_stat_activity, so the launcher doesn't actually show up. -- we can still test its interactions with its children, but can't test some of the things specific to the launcher. -- So we've added some bits about the version number as needed. CREATE VIEW worker_counts as SELECT count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Launcher') as launcher, count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME') as single_scheduler, count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2') as single_2_scheduler, count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = 'template1') as template1_scheduler FROM pg_stat_activity; CREATE FUNCTION wait_worker_counts(launcher_ct INTEGER, scheduler1_ct INTEGER, scheduler2_ct INTEGER, template1_ct INTEGER) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE r INTEGER; BEGIN FOR i in 1..10 LOOP SELECT COUNT(*) from worker_counts where launcher = launcher_ct AND single_scheduler = scheduler1_ct AND single_2_scheduler = scheduler2_ct and template1_scheduler = template1_ct into r; if(r < 1) THEN PERFORM pg_sleep(0.1); PERFORM pg_stat_clear_snapshot(); ELSE --We have the correct counts! RETURN TRUE; END IF; END LOOP; RETURN FALSE; END $BODY$; CREATE FUNCTION wait_for_bgw_scheduler(_datname NAME, _count INT DEFAULT 1, _ticks INT DEFAULT 10) RETURNS TEXT LANGUAGE PLPGSQL AS $BODY$ DECLARE r INTEGER; BEGIN FOR i in 1.._ticks LOOP SELECT count(*) FROM pg_stat_activity WHERE application_name = 'TimescaleDB Background Worker Scheduler' AND datname = _datname INTO r; IF(r <> _count) THEN PERFORM pg_sleep(0.1); PERFORM pg_stat_clear_snapshot(); ELSE RETURN 'BGW Scheduler found.'; END IF; END LOOP; RETURN 'BGW Scheduler NOT found.'; END $BODY$; CREATE PROCEDURE kill_database_backends(_datname NAME) LANGUAGE PLPGSQL AS $BODY$ DECLARE r INTEGER; BEGIN FOR i in 1..100 LOOP SELECT count(pg_terminate_backend(pg_stat_activity.pid)) FROM pg_stat_activity WHERE datname = _datname AND pg_stat_activity.pid <> pg_backend_pid() INTO r; IF(r = 0) THEN RETURN; END IF; PERFORM pg_sleep(0.1); PERFORM pg_stat_clear_snapshot(); END LOOP; RAISE 'Failed to terminate backends'; END $BODY$; BEGIN; -- Then create extension there in a txn and make sure we see a scheduler start SET client_min_messages = ERROR; CREATE EXTENSION timescaledb CASCADE; RESET client_min_messages; SELECT wait_worker_counts(1,0,0,1); wait_worker_counts -------------------- t COMMIT; -- End our transaction and it should immediately exit because it's a template database. SELECT wait_worker_counts(1,0,0,0); wait_worker_counts -------------------- t \c :TEST_DBNAME_2 -- Now try creating a DB from a template with the extension already installed. -- Make sure we see a scheduler start. CREATE DATABASE :TEST_DBNAME; SELECT wait_worker_counts(1,1,0,0); wait_worker_counts -------------------- t DROP DATABASE :TEST_DBNAME WITH (FORCE); -- Now make sure that there's no race between create database and create extension. -- Although to be honest, this race probably wouldn't manifest in this test. \c template1 DROP EXTENSION timescaledb; \c :TEST_DBNAME_2 CREATE DATABASE :TEST_DBNAME; \c :TEST_DBNAME SET client_min_messages = ERROR; CREATE EXTENSION timescaledb; RESET client_min_messages; \c :TEST_DBNAME_2 SELECT wait_worker_counts(1,1,0,0); wait_worker_counts -------------------- t -- test rename database CREATE DATABASE db_rename_test; \c db_rename_test :ROLE_SUPERUSER SET client_min_messages=error; CREATE EXTENSION timescaledb; \c :TEST_DBNAME_2 :ROLE_SUPERUSER SELECT wait_for_bgw_scheduler('db_rename_test'); wait_for_bgw_scheduler ------------------------ BGW Scheduler found. ALTER DATABASE db_rename_test RENAME TO db_rename_test2; WARNING: you need to manually restart any running background workers after this command DROP DATABASE db_rename_test2 WITH (FORCE); -- test create database with timescaledb database as template SELECT wait_for_bgw_scheduler(:'TEST_DBNAME'); wait_for_bgw_scheduler ------------------------ BGW Scheduler found. CREATE DATABASE db_from_template WITH TEMPLATE :TEST_DBNAME; SELECT wait_for_bgw_scheduler(:'TEST_DBNAME'); wait_for_bgw_scheduler ------------------------ BGW Scheduler found. DROP DATABASE db_from_template WITH (FORCE); -- test alter database set tablespace SET client_min_messages TO error; DROP TABLESPACE IF EXISTS tablespace1; RESET client_min_messages; CREATE TABLESPACE tablespace1 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE1_PATH; -- Stop background worker before we change the tablespace of the database (otherwise, the database might be used) SELECT wait_for_bgw_scheduler(:'TEST_DBNAME'); wait_for_bgw_scheduler ------------------------ BGW Scheduler found. -- Connect to TEST_DBNAME (_timescaledb_functions.stop_background_workers() is not available in TEST_DBNAME_2) \c :TEST_DBNAME :ROLE_SUPERUSER SELECT _timescaledb_functions.stop_background_workers(); stop_background_workers ------------------------- t SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Launcher'; pg_terminate_backend ---------------------- t \c :TEST_DBNAME_2 :ROLE_SUPERUSER -- make sure nobody is using it REVOKE CONNECT ON DATABASE :TEST_DBNAME FROM public; CALL kill_database_backends(:'TEST_DBNAME'); -- Change tablespace ALTER DATABASE :TEST_DBNAME SET TABLESPACE tablespace1; WARNING: you may need to manually restart any running background workers after this command -- tear down test and clean up additional database \c :TEST_DBNAME :ROLE_SUPERUSER SELECT _timescaledb_functions.stop_background_workers() \gset REVOKE CONNECT ON DATABASE :TEST_DBNAME_2 FROM public; CALL kill_database_backends(:'TEST_DBNAME_2'); SELECT * FROM pg_stat_activity WHERE datname = :'TEST_DBNAME_2'; datid | datname | pid | leader_pid | usesysid | usename | application_name | client_addr | client_hostname | client_port | backend_start | xact_start | query_start | state_change | wait_event_type | wait_event | state | backend_xid | backend_xmin | query_id | query | backend_type -------+---------+-----+------------+----------+---------+------------------+-------------+-----------------+-------------+---------------+------------+-------------+--------------+-----------------+------------+-------+-------------+--------------+----------+-------+-------------- DROP DATABASE :TEST_DBNAME_2 WITH (force); -- Clean up the template database, removing our test utilities etc \c template1 \ir include/bgw_launcher_utils_cleanup.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. DROP FUNCTION wait_worker_counts(integer, integer, integer, integer); DROP VIEW worker_counts; DROP FUNCTION wait_for_bgw_scheduler(name,int,int); DROP PROCEDURE kill_database_backends(name); ================================================ FILE: test/expected/c_unit_tests.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION test.time_to_internal_conversion() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_time_to_internal_conversion' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION test.interval_to_internal_conversion() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_interval_to_internal_conversion' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION test.adts() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_adts' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION test.time_utils() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_time_utils' LANGUAGE C; CREATE OR REPLACE FUNCTION test.bmslist_utils() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_bmslist_utils' LANGUAGE C; CREATE OR REPLACE FUNCTION test.jsonb_utils() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_jsonb_utils' LANGUAGE C; CREATE OR REPLACE FUNCTION test.compression_settings() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_compression_settings' LANGUAGE C; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT test.time_to_internal_conversion(); time_to_internal_conversion ----------------------------- SELECT test.interval_to_internal_conversion(); interval_to_internal_conversion --------------------------------- SELECT test.adts(); adts ------ SELECT test.time_utils(); time_utils ------------ SELECT test.bmslist_utils(); bmslist_utils --------------- SELECT test.jsonb_utils(); jsonb_utils ------------- SELECT test.compression_settings(); compression_settings ---------------------- ================================================ FILE: test/expected/catalog_corruption.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER --- Test handling of missing dimension slices CREATE TABLE dim_test(time TIMESTAMPTZ, device int); SELECT create_hypertable('dim_test', 'time', chunk_time_interval => INTERVAL '1 day'); create_hypertable ----------------------- (1,public,dim_test,t) -- Create two chunks INSERT INTO dim_test values('2000-01-01 00:00:00', 1); INSERT INTO dim_test values('2020-01-01 00:00:00', 1); SELECT id AS dim_slice_id FROM _timescaledb_catalog.dimension_slice ORDER BY id DESC LIMIT 1 \gset -- Delete the dimension slice for the second chunk DELETE FROM _timescaledb_catalog.chunk_constraint WHERE dimension_slice_id = :dim_slice_id; \set ON_ERROR_STOP 0 -- Select data SELECT * FROM dim_test; ERROR: chunk _hyper_1_2_chunk has no dimension slices -- Select data using ordered append SELECT * FROM dim_test ORDER BY time; ERROR: chunk _hyper_1_2_chunk has no dimension slices \set ON_ERROR_STOP 1 DROP TABLE dim_test; ================================================ FILE: test/expected/chunk_adaptive.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- test error handling _timescaledb_functions.calculate_chunk_interval \set ON_ERROR_STOP 0 SELECT _timescaledb_functions.calculate_chunk_interval(0,0,-0); ERROR: could not find a matching hypertable for dimension 0 SELECT _timescaledb_functions.calculate_chunk_interval(1,0,-1); ERROR: chunk_target_size must be positive \set ON_ERROR_STOP 1 -- Valid chunk sizing function for testing CREATE OR REPLACE FUNCTION calculate_chunk_interval( dimension_id INTEGER, dimension_coord BIGINT, chunk_target_size BIGINT ) RETURNS BIGINT LANGUAGE PLPGSQL AS $BODY$ DECLARE BEGIN RETURN -1; END $BODY$; -- Chunk sizing function with bad signature CREATE OR REPLACE FUNCTION bad_calculate_chunk_interval( dimension_id INTEGER ) RETURNS BIGINT LANGUAGE PLPGSQL AS $BODY$ DECLARE BEGIN RETURN -1; END $BODY$; -- Set a fixed memory cache size to make tests determinstic -- (independent of available machine memory) SELECT * FROM test.set_memory_cache_size('2GB'); set_memory_cache_size ----------------------- 2147483648 -- test NULL handling \set ON_ERROR_STOP 0 SELECT * FROM set_adaptive_chunking(NULL,NULL); ERROR: invalid hypertable: cannot be NULL \set ON_ERROR_STOP 1 CREATE TABLE test_adaptive(time timestamptz, temp float, location int); \set ON_ERROR_STOP 0 -- Bad signature of sizing func should fail SELECT create_hypertable('test_adaptive', 'time', chunk_target_size => '1MB', chunk_sizing_func => 'bad_calculate_chunk_interval'); ERROR: invalid function signature \set ON_ERROR_STOP 1 -- Setting sizing func with correct signature should work SELECT create_hypertable('test_adaptive', 'time', chunk_target_size => '1MB', chunk_sizing_func => 'calculate_chunk_interval'); WARNING: target chunk size for adaptive chunking is less than 10 MB NOTICE: adaptive chunking is a BETA feature and is not recommended for production deployments create_hypertable ---------------------------- (1,public,test_adaptive,t) DROP TABLE test_adaptive; CREATE TABLE test_adaptive(time timestamptz, temp float, location int); -- Size but no explicit func should use default func SELECT create_hypertable('test_adaptive', 'time', chunk_target_size => '1MB', create_default_indexes => true); WARNING: target chunk size for adaptive chunking is less than 10 MB NOTICE: adaptive chunking is a BETA feature and is not recommended for production deployments create_hypertable ---------------------------- (2,public,test_adaptive,t) SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; table_name | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size ---------------+--------------------------+--------------------------+------------------- test_adaptive | _timescaledb_functions | calculate_chunk_interval | 1048576 -- Check that adaptive chunking sets a 1 day default chunk time -- interval => 86400000000 microseconds SELECT * FROM _timescaledb_catalog.dimension; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+--------------------------+---------+------------+--------------------------+-------------------+-----------------+--------------------------+-------------------------+------------------ 2 | 2 | time | timestamp with time zone | t | | | | 86400000000 | | | -- Change the target size SELECT * FROM set_adaptive_chunking('test_adaptive', '2MB'); WARNING: target chunk size for adaptive chunking is less than 10 MB chunk_sizing_func | chunk_target_size -------------------------------------------------+------------------- _timescaledb_functions.calculate_chunk_interval | 2097152 SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; table_name | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size ---------------+--------------------------+--------------------------+------------------- test_adaptive | _timescaledb_functions | calculate_chunk_interval | 2097152 \set ON_ERROR_STOP 0 -- Setting NULL func should fail SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB', NULL); ERROR: invalid chunk sizing function \set ON_ERROR_STOP 1 -- Setting NULL size disables adaptive chunking SELECT * FROM set_adaptive_chunking('test_adaptive', NULL); chunk_sizing_func | chunk_target_size -------------------------------------------------+------------------- _timescaledb_functions.calculate_chunk_interval | 0 SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; table_name | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size ---------------+--------------------------+--------------------------+------------------- test_adaptive | _timescaledb_functions | calculate_chunk_interval | 0 SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB'); WARNING: target chunk size for adaptive chunking is less than 10 MB chunk_sizing_func | chunk_target_size -------------------------------------------------+------------------- _timescaledb_functions.calculate_chunk_interval | 1048576 -- Setting size to 'off' should also disable SELECT * FROM set_adaptive_chunking('test_adaptive', 'off'); chunk_sizing_func | chunk_target_size -------------------------------------------------+------------------- _timescaledb_functions.calculate_chunk_interval | 0 SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; table_name | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size ---------------+--------------------------+--------------------------+------------------- test_adaptive | _timescaledb_functions | calculate_chunk_interval | 0 -- Setting 0 size should also disable SELECT * FROM set_adaptive_chunking('test_adaptive', '0MB'); chunk_sizing_func | chunk_target_size -------------------------------------------------+------------------- _timescaledb_functions.calculate_chunk_interval | 0 SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; table_name | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size ---------------+--------------------------+--------------------------+------------------- test_adaptive | _timescaledb_functions | calculate_chunk_interval | 0 SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB'); WARNING: target chunk size for adaptive chunking is less than 10 MB chunk_sizing_func | chunk_target_size -------------------------------------------------+------------------- _timescaledb_functions.calculate_chunk_interval | 1048576 -- No warning about small target size if > 10MB SELECT * FROM set_adaptive_chunking('test_adaptive', '11MB'); chunk_sizing_func | chunk_target_size -------------------------------------------------+------------------- _timescaledb_functions.calculate_chunk_interval | 11534336 -- Setting size to 'estimate' should also estimate size SELECT * FROM set_adaptive_chunking('test_adaptive', 'estimate'); chunk_sizing_func | chunk_target_size -------------------------------------------------+------------------- _timescaledb_functions.calculate_chunk_interval | 1932735283 SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; table_name | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size ---------------+--------------------------+--------------------------+------------------- test_adaptive | _timescaledb_functions | calculate_chunk_interval | 1932735283 -- Use a lower memory setting to test that the calculated chunk_target_size is reduced SELECT * FROM test.set_memory_cache_size('512MB'); set_memory_cache_size ----------------------- 536870912 SELECT * FROM set_adaptive_chunking('test_adaptive', 'estimate'); chunk_sizing_func | chunk_target_size -------------------------------------------------+------------------- _timescaledb_functions.calculate_chunk_interval | 483183820 SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; table_name | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size ---------------+--------------------------+--------------------------+------------------- test_adaptive | _timescaledb_functions | calculate_chunk_interval | 483183820 -- Reset memory settings SELECT * FROM test.set_memory_cache_size('2GB'); set_memory_cache_size ----------------------- 2147483648 -- Set a reasonable test value SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB'); WARNING: target chunk size for adaptive chunking is less than 10 MB chunk_sizing_func | chunk_target_size -------------------------------------------------+------------------- _timescaledb_functions.calculate_chunk_interval | 1048576 -- Show the interval length before and after adaptation SELECT id, hypertable_id, interval_length FROM _timescaledb_catalog.dimension; id | hypertable_id | interval_length ----+---------------+----------------- 2 | 2 | 86400000000 -- Generate data to create chunks. We use the hash of the time value -- to get determinstic location IDs so that we always spread these -- values the same way across space partitions INSERT INTO test_adaptive SELECT time, random() * 35, _timescaledb_functions.get_partition_hash(time) FROM generate_series('2017-03-07T18:18:03+00'::timestamptz - interval '175 days', '2017-03-07T18:18:03+00'::timestamptz, '2 minutes') as time; SELECT chunk_name, primary_dimension, range_start, range_end FROM timescaledb_information.chunks WHERE hypertable_name = 'test_adaptive' ORDER BY chunk_name; chunk_name | primary_dimension | range_start | range_end -------------------+-------------------+-------------------------------------+------------------------------------- _hyper_2_10_chunk | time | Fri Sep 23 22:08:15.728855 2016 PDT | Sat Oct 01 13:16:09.024252 2016 PDT _hyper_2_11_chunk | time | Sat Oct 01 13:16:09.024252 2016 PDT | Fri Oct 14 03:19:44.231212 2016 PDT _hyper_2_12_chunk | time | Fri Oct 14 03:19:44.231212 2016 PDT | Wed Oct 26 19:20:54.4938 2016 PDT _hyper_2_13_chunk | time | Wed Oct 26 19:20:54.4938 2016 PDT | Fri Nov 04 04:03:56.248528 2016 PDT _hyper_2_14_chunk | time | Fri Nov 04 04:03:56.248528 2016 PDT | Fri Nov 18 21:58:20.411232 2016 PST _hyper_2_15_chunk | time | Fri Nov 18 21:58:20.411232 2016 PST | Sat Dec 03 16:52:44.573936 2016 PST _hyper_2_16_chunk | time | Sat Dec 03 16:52:44.573936 2016 PST | Sun Dec 18 11:47:08.73664 2016 PST _hyper_2_17_chunk | time | Sun Dec 18 11:47:08.73664 2016 PST | Mon Jan 02 06:41:32.899344 2017 PST _hyper_2_18_chunk | time | Mon Jan 02 06:41:32.899344 2017 PST | Tue Jan 17 01:35:57.062048 2017 PST _hyper_2_19_chunk | time | Tue Jan 17 01:35:57.062048 2017 PST | Tue Jan 31 20:30:21.224752 2017 PST _hyper_2_1_chunk | time | Mon Sep 12 17:00:00 2016 PDT | Tue Sep 13 17:00:00 2016 PDT _hyper_2_20_chunk | time | Tue Jan 31 20:30:21.224752 2017 PST | Wed Feb 15 15:24:45.387456 2017 PST _hyper_2_21_chunk | time | Wed Feb 15 15:24:45.387456 2017 PST | Thu Mar 02 10:19:09.55016 2017 PST _hyper_2_22_chunk | time | Thu Mar 02 10:19:09.55016 2017 PST | Fri Mar 17 06:13:33.712864 2017 PDT _hyper_2_2_chunk | time | Tue Sep 13 17:00:00 2016 PDT | Wed Sep 14 17:00:00 2016 PDT _hyper_2_3_chunk | time | Wed Sep 14 17:00:00 2016 PDT | Thu Sep 15 17:00:00 2016 PDT _hyper_2_4_chunk | time | Thu Sep 15 17:00:00 2016 PDT | Fri Sep 16 15:02:54.2208 2016 PDT _hyper_2_5_chunk | time | Fri Sep 16 15:02:54.2208 2016 PDT | Sun Sep 18 03:12:14.342144 2016 PDT _hyper_2_6_chunk | time | Sun Sep 18 03:12:14.342144 2016 PDT | Mon Sep 19 15:21:34.463488 2016 PDT _hyper_2_7_chunk | time | Mon Sep 19 15:21:34.463488 2016 PDT | Wed Sep 21 03:30:54.584832 2016 PDT _hyper_2_8_chunk | time | Wed Sep 21 03:30:54.584832 2016 PDT | Thu Sep 22 03:45:14.901568 2016 PDT _hyper_2_9_chunk | time | Thu Sep 22 03:45:14.901568 2016 PDT | Fri Sep 23 22:08:15.728855 2016 PDT -- Do same thing without an index on the time column. This affects -- both the calculation of fill-factor of the chunk and its size CREATE TABLE test_adaptive_no_index(time timestamptz, temp float, location int); -- Size but no explicit func should use default func -- No default indexes should warn and use heap scan for min and max SELECT create_hypertable('test_adaptive_no_index', 'time', chunk_target_size => '1MB', create_default_indexes => false); WARNING: target chunk size for adaptive chunking is less than 10 MB WARNING: no index on "time" found for adaptive chunking on hypertable "test_adaptive_no_index" NOTICE: adaptive chunking is a BETA feature and is not recommended for production deployments create_hypertable ------------------------------------- (3,public,test_adaptive_no_index,t) SELECT id, hypertable_id, interval_length FROM _timescaledb_catalog.dimension; id | hypertable_id | interval_length ----+---------------+----------------- 2 | 2 | 1277664162704 3 | 3 | 86400000000 INSERT INTO test_adaptive_no_index SELECT time, random() * 35, _timescaledb_functions.get_partition_hash(time) FROM generate_series('2017-03-07T18:18:03+00'::timestamptz - interval '175 days', '2017-03-07T18:18:03+00'::timestamptz, '2 minutes') as time; WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_23_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_23_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_24_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_23_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_24_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_25_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_24_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_25_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_26_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_25_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_26_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_27_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_26_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_27_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_28_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_27_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_28_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_29_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_28_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_29_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_30_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_29_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_30_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_31_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_30_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_31_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_32_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_31_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_32_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_33_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_32_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_33_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_34_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_33_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_34_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_35_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_34_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_35_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_36_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_35_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_36_chunk" WARNING: no index on "time" found for adaptive chunking on chunk "_hyper_3_37_chunk" SELECT chunk_name, primary_dimension, range_start, range_end FROM timescaledb_information.chunks WHERE hypertable_name = 'test_adaptive_no_index' ORDER BY chunk_name; chunk_name | primary_dimension | range_start | range_end -------------------+-------------------+-------------------------------------+------------------------------------- _hyper_3_23_chunk | time | Mon Sep 12 17:00:00 2016 PDT | Tue Sep 13 17:00:00 2016 PDT _hyper_3_24_chunk | time | Tue Sep 13 17:00:00 2016 PDT | Wed Sep 14 17:00:00 2016 PDT _hyper_3_25_chunk | time | Wed Sep 14 17:00:00 2016 PDT | Thu Sep 15 17:00:00 2016 PDT _hyper_3_26_chunk | time | Thu Sep 15 17:00:00 2016 PDT | Sun Sep 18 02:18:45.310968 2016 PDT _hyper_3_27_chunk | time | Sun Sep 18 02:18:45.310968 2016 PDT | Sun Sep 18 06:20:21.359312 2016 PDT _hyper_3_28_chunk | time | Sun Sep 18 06:20:21.359312 2016 PDT | Wed Sep 21 08:25:00.957966 2016 PDT _hyper_3_29_chunk | time | Wed Sep 21 08:25:00.957966 2016 PDT | Thu Sep 22 03:26:42.599807 2016 PDT _hyper_3_30_chunk | time | Thu Sep 22 03:26:42.599807 2016 PDT | Sun Sep 25 18:03:30.59359 2016 PDT _hyper_3_31_chunk | time | Sun Sep 25 18:03:30.59359 2016 PDT | Sat Oct 08 05:32:02.75732 2016 PDT _hyper_3_32_chunk | time | Sat Oct 08 05:32:02.75732 2016 PDT | Mon Oct 31 07:33:42.652938 2016 PDT _hyper_3_33_chunk | time | Mon Oct 31 07:33:42.652938 2016 PDT | Wed Nov 23 08:35:22.548556 2016 PST _hyper_3_34_chunk | time | Wed Nov 23 08:35:22.548556 2016 PST | Thu Dec 15 09:48:28.1888 2016 PST _hyper_3_35_chunk | time | Thu Dec 15 09:48:28.1888 2016 PST | Wed Jan 11 04:57:38.357845 2017 PST _hyper_3_36_chunk | time | Wed Jan 11 04:57:38.357845 2017 PST | Tue Feb 07 00:06:48.52689 2017 PST _hyper_3_37_chunk | time | Tue Feb 07 00:06:48.52689 2017 PST | Sun Mar 05 19:15:58.695935 2017 PST _hyper_3_38_chunk | time | Sun Mar 05 19:15:58.695935 2017 PST | Sat Apr 01 15:25:08.86498 2017 PDT -- Test added to check that the correct index (i.e. time index) is being used -- to find the min and max. Previously a bug selected the first index listed, -- which in this case is location rather than time and therefore could return -- the wrong min and max if items at the start and end of the index did not have -- the correct min and max timestamps. -- -- In this test, we create chunks with a lot of locations with only one reading -- that is at the beginning of the time frame, and then one location in the middle -- of the range that has two readings, one that is the same as the others and one -- that is larger. The algorithm should use these two readings for min & max; however, -- if it's broken (as it was before), it would choose just the reading that is common -- to all the locations. CREATE TABLE test_adaptive_correct_index(time timestamptz, temp float, location int); SELECT create_hypertable('test_adaptive_correct_index', 'time', chunk_target_size => '100MB', chunk_time_interval => 86400000000, create_default_indexes => false); WARNING: no index on "time" found for adaptive chunking on hypertable "test_adaptive_correct_index" NOTICE: adaptive chunking is a BETA feature and is not recommended for production deployments create_hypertable ------------------------------------------ (4,public,test_adaptive_correct_index,t) CREATE INDEX ON test_adaptive_correct_index(location); CREATE INDEX ON test_adaptive_correct_index(time DESC); -- First chunk INSERT INTO test_adaptive_correct_index SELECT '2018-01-01T00:00:00+00'::timestamptz, val, val + 1 FROM generate_series(1, 1000) as val; INSERT INTO test_adaptive_correct_index SELECT time, 0.0, '1500' FROM generate_series('2018-01-01T00:00:00+00'::timestamptz, '2018-01-01T20:00:00+00'::timestamptz, '10 hours') as time; INSERT INTO test_adaptive_correct_index SELECT '2018-01-01T00:00:00+00'::timestamptz, val, val + 1 FROM generate_series(2001, 3000) as val; -- Second chunk INSERT INTO test_adaptive_correct_index SELECT '2018-01-02T00:00:00+00'::timestamptz, val, val + 1 FROM generate_series(1, 1000) as val; INSERT INTO test_adaptive_correct_index SELECT time, 0.0, '1500' FROM generate_series('2018-01-02T00:00:00+00'::timestamptz, '2018-01-02T20:00:00+00'::timestamptz, '10 hours') as time; INSERT INTO test_adaptive_correct_index SELECT '2018-01-02T00:00:00+00'::timestamptz, val, val + 1 FROM generate_series(2001, 3000) as val; -- Third chunk INSERT INTO test_adaptive_correct_index SELECT '2018-01-03T00:00:00+00'::timestamptz, val, val + 1 FROM generate_series(1, 1000) as val; INSERT INTO test_adaptive_correct_index SELECT time, 0.0, '1500' FROM generate_series('2018-01-03T00:00:00+00'::timestamptz, '2018-01-03T20:00:00+00'::timestamptz, '10 hours') as time; INSERT INTO test_adaptive_correct_index SELECT '2018-01-03T00:00:00+00'::timestamptz, val, val + 1 FROM generate_series(2001, 3000) as val; -- This should be the start of the fourth chunk INSERT INTO test_adaptive_correct_index SELECT '2018-01-04T00:00:00+00'::timestamptz, val, val + 1 FROM generate_series(1, 1000) as val; INSERT INTO test_adaptive_correct_index SELECT time, 0.0, '1500' FROM generate_series('2018-01-04T00:00:00+00'::timestamptz, '2018-01-04T20:00:00+00'::timestamptz, '10 hours') as time; INSERT INTO test_adaptive_correct_index SELECT '2018-01-04T00:00:00+00'::timestamptz, val, val + 1 FROM generate_series(2001, 3000) as val; -- If working correctly, this goes in the 4th chunk, otherwise its a separate 5th chunk INSERT INTO test_adaptive_correct_index SELECT '2018-01-05T00:00:00+00'::timestamptz, val, val + 1 FROM generate_series(1, 1000) as val; INSERT INTO test_adaptive_correct_index SELECT time, 0.0, '1500' FROM generate_series('2018-01-05T00:00:00+00'::timestamptz, '2018-01-05T20:00:00+00'::timestamptz, '10 hours') as time; INSERT INTO test_adaptive_correct_index SELECT '2018-01-05T00:00:00+00'::timestamptz, val, val + 1 FROM generate_series(2001, 3000) as val; -- This should show 4 chunks, rather than 5 SELECT count(*) FROM timescaledb_information.chunks WHERE hypertable_name = 'test_adaptive_correct_index'; count ------- 4 -- The interval_length should no longer be 86400000000 for our hypertable, so 3rd column so be true. -- Note: the exact interval_length is non-deterministic, so we can't use its actual value for tests SELECT id, hypertable_id, interval_length > 86400000000 FROM _timescaledb_catalog.dimension; id | hypertable_id | ?column? ----+---------------+---------- 2 | 2 | t 3 | 3 | t 4 | 4 | t -- Drop because it's size and estimated chunk_interval is non-deterministic so -- we don't want to make other tests flaky. DROP TABLE test_adaptive_correct_index; -- Test with space partitioning. This might affect the estimation -- since there are more chunks in the same time interval and space -- chunks might be unevenly filled. CREATE TABLE test_adaptive_space(time timestamptz, temp float, location int); SELECT create_hypertable('test_adaptive_space', 'time', 'location', 2, chunk_target_size => '1MB', create_default_indexes => true); WARNING: target chunk size for adaptive chunking is less than 10 MB NOTICE: adaptive chunking is a BETA feature and is not recommended for production deployments create_hypertable ---------------------------------- (5,public,test_adaptive_space,t) SELECT id, hypertable_id, interval_length FROM _timescaledb_catalog.dimension; id | hypertable_id | interval_length ----+---------------+----------------- 2 | 2 | 1277664162704 3 | 3 | 2315350169045 5 | 5 | 86400000000 6 | 5 | INSERT INTO test_adaptive_space SELECT time, random() * 35, _timescaledb_functions.get_partition_hash(time) FROM generate_series('2017-03-07T18:18:03+00'::timestamptz - interval '175 days', '2017-03-07T18:18:03+00'::timestamptz, '2 minutes') as time; \x SELECT chunk_name, range_start, range_end FROM timescaledb_information.chunks WHERE hypertable_name = 'test_adaptive_space' ORDER BY chunk_name; -[ RECORD 1 ]------------------------------------ chunk_name | _hyper_5_43_chunk range_start | Mon Sep 12 17:00:00 2016 PDT range_end | Tue Sep 13 17:00:00 2016 PDT -[ RECORD 2 ]------------------------------------ chunk_name | _hyper_5_44_chunk range_start | Mon Sep 12 17:00:00 2016 PDT range_end | Tue Sep 13 17:00:00 2016 PDT -[ RECORD 3 ]------------------------------------ chunk_name | _hyper_5_45_chunk range_start | Tue Sep 13 17:00:00 2016 PDT range_end | Wed Sep 14 17:00:00 2016 PDT -[ RECORD 4 ]------------------------------------ chunk_name | _hyper_5_46_chunk range_start | Tue Sep 13 17:00:00 2016 PDT range_end | Wed Sep 14 17:00:00 2016 PDT -[ RECORD 5 ]------------------------------------ chunk_name | _hyper_5_47_chunk range_start | Wed Sep 14 17:00:00 2016 PDT range_end | Thu Sep 15 11:47:51.47376 2016 PDT -[ RECORD 6 ]------------------------------------ chunk_name | _hyper_5_48_chunk range_start | Wed Sep 14 17:00:00 2016 PDT range_end | Thu Sep 15 11:47:51.47376 2016 PDT -[ RECORD 7 ]------------------------------------ chunk_name | _hyper_5_49_chunk range_start | Thu Sep 15 11:47:51.47376 2016 PDT range_end | Sat Sep 17 02:40:49.182352 2016 PDT -[ RECORD 8 ]------------------------------------ chunk_name | _hyper_5_50_chunk range_start | Thu Sep 15 11:47:51.47376 2016 PDT range_end | Sat Sep 17 02:40:49.182352 2016 PDT -[ RECORD 9 ]------------------------------------ chunk_name | _hyper_5_51_chunk range_start | Sat Sep 17 02:40:49.182352 2016 PDT range_end | Sun Sep 18 17:33:46.890944 2016 PDT -[ RECORD 10 ]----------------------------------- chunk_name | _hyper_5_52_chunk range_start | Sat Sep 17 02:40:49.182352 2016 PDT range_end | Sun Sep 18 17:33:46.890944 2016 PDT -[ RECORD 11 ]----------------------------------- chunk_name | _hyper_5_53_chunk range_start | Sun Sep 18 17:33:46.890944 2016 PDT range_end | Sun Sep 18 20:35:55.67676 2016 PDT -[ RECORD 12 ]----------------------------------- chunk_name | _hyper_5_54_chunk range_start | Sun Sep 18 17:33:46.890944 2016 PDT range_end | Sun Sep 18 20:35:55.67676 2016 PDT -[ RECORD 13 ]----------------------------------- chunk_name | _hyper_5_55_chunk range_start | Sun Sep 18 20:35:55.67676 2016 PDT range_end | Tue Sep 20 18:46:40.16883 2016 PDT -[ RECORD 14 ]----------------------------------- chunk_name | _hyper_5_56_chunk range_start | Sun Sep 18 20:35:55.67676 2016 PDT range_end | Tue Sep 20 18:46:40.16883 2016 PDT -[ RECORD 15 ]----------------------------------- chunk_name | _hyper_5_57_chunk range_start | Tue Sep 20 18:46:40.16883 2016 PDT range_end | Sun Oct 02 16:44:29.071032 2016 PDT -[ RECORD 16 ]----------------------------------- chunk_name | _hyper_5_58_chunk range_start | Tue Sep 20 18:46:40.16883 2016 PDT range_end | Sun Oct 02 16:44:29.071032 2016 PDT -[ RECORD 17 ]----------------------------------- chunk_name | _hyper_5_59_chunk range_start | Sun Oct 02 16:44:29.071032 2016 PDT range_end | Tue Oct 11 00:37:03.738979 2016 PDT -[ RECORD 18 ]----------------------------------- chunk_name | _hyper_5_60_chunk range_start | Sun Oct 02 16:44:29.071032 2016 PDT range_end | Tue Oct 11 00:37:03.738979 2016 PDT -[ RECORD 19 ]----------------------------------- chunk_name | _hyper_5_61_chunk range_start | Tue Oct 11 00:37:03.738979 2016 PDT range_end | Thu Oct 27 03:05:25.740618 2016 PDT -[ RECORD 20 ]----------------------------------- chunk_name | _hyper_5_62_chunk range_start | Tue Oct 11 00:37:03.738979 2016 PDT range_end | Thu Oct 27 03:05:25.740618 2016 PDT -[ RECORD 21 ]----------------------------------- chunk_name | _hyper_5_63_chunk range_start | Thu Oct 27 03:05:25.740618 2016 PDT range_end | Sun Nov 13 12:38:49.541703 2016 PST -[ RECORD 22 ]----------------------------------- chunk_name | _hyper_5_64_chunk range_start | Thu Oct 27 03:05:25.740618 2016 PDT range_end | Sun Nov 13 12:38:49.541703 2016 PST -[ RECORD 23 ]----------------------------------- chunk_name | _hyper_5_65_chunk range_start | Sun Nov 13 12:38:49.541703 2016 PST range_end | Fri Dec 02 17:45:40.237036 2016 PST -[ RECORD 24 ]----------------------------------- chunk_name | _hyper_5_66_chunk range_start | Sun Nov 13 12:38:49.541703 2016 PST range_end | Fri Dec 02 17:45:40.237036 2016 PST -[ RECORD 25 ]----------------------------------- chunk_name | _hyper_5_67_chunk range_start | Fri Dec 02 17:45:40.237036 2016 PST range_end | Wed Dec 21 22:52:30.932369 2016 PST -[ RECORD 26 ]----------------------------------- chunk_name | _hyper_5_68_chunk range_start | Fri Dec 02 17:45:40.237036 2016 PST range_end | Wed Dec 21 22:52:30.932369 2016 PST -[ RECORD 27 ]----------------------------------- chunk_name | _hyper_5_69_chunk range_start | Wed Dec 21 22:52:30.932369 2016 PST range_end | Tue Jan 10 03:59:21.627702 2017 PST -[ RECORD 28 ]----------------------------------- chunk_name | _hyper_5_70_chunk range_start | Wed Dec 21 22:52:30.932369 2016 PST range_end | Tue Jan 10 03:59:21.627702 2017 PST -[ RECORD 29 ]----------------------------------- chunk_name | _hyper_5_71_chunk range_start | Tue Jan 10 03:59:21.627702 2017 PST range_end | Sun Jan 29 09:06:12.323035 2017 PST -[ RECORD 30 ]----------------------------------- chunk_name | _hyper_5_72_chunk range_start | Tue Jan 10 03:59:21.627702 2017 PST range_end | Sun Jan 29 09:06:12.323035 2017 PST -[ RECORD 31 ]----------------------------------- chunk_name | _hyper_5_73_chunk range_start | Sun Jan 29 09:06:12.323035 2017 PST range_end | Fri Feb 17 14:13:03.018368 2017 PST -[ RECORD 32 ]----------------------------------- chunk_name | _hyper_5_74_chunk range_start | Sun Jan 29 09:06:12.323035 2017 PST range_end | Fri Feb 17 14:13:03.018368 2017 PST -[ RECORD 33 ]----------------------------------- chunk_name | _hyper_5_75_chunk range_start | Fri Feb 17 14:13:03.018368 2017 PST range_end | Wed Mar 08 19:19:53.713701 2017 PST -[ RECORD 34 ]----------------------------------- chunk_name | _hyper_5_76_chunk range_start | Fri Feb 17 14:13:03.018368 2017 PST range_end | Wed Mar 08 19:19:53.713701 2017 PST SELECT * FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_adaptive_space' ORDER BY dimension_number; -[ RECORD 1 ]-----+---------------------------------------- hypertable_schema | public hypertable_name | test_adaptive_space dimension_number | 1 column_name | time column_type | timestamp with time zone dimension_type | Time time_interval | @ 19 days 5 hours 6 mins 50.695333 secs integer_interval | integer_now_func | num_partitions | -[ RECORD 2 ]-----+---------------------------------------- hypertable_schema | public hypertable_name | test_adaptive_space dimension_number | 2 column_name | location column_type | integer dimension_type | Space time_interval | integer_interval | integer_now_func | num_partitions | 2 \x SELECT * FROM chunks_detailed_size('test_adaptive_space') ORDER BY chunk_name; chunk_schema | chunk_name | table_bytes | index_bytes | toast_bytes | total_bytes | node_name -----------------------+-------------------+-------------+-------------+-------------+-------------+----------- _timescaledb_internal | _hyper_5_43_chunk | 8192 | 32768 | 0 | 40960 | _timescaledb_internal | _hyper_5_44_chunk | 8192 | 32768 | 0 | 40960 | _timescaledb_internal | _hyper_5_45_chunk | 49152 | 57344 | 0 | 106496 | _timescaledb_internal | _hyper_5_46_chunk | 49152 | 57344 | 0 | 106496 | _timescaledb_internal | _hyper_5_47_chunk | 40960 | 49152 | 0 | 90112 | _timescaledb_internal | _hyper_5_48_chunk | 40960 | 32768 | 0 | 73728 | _timescaledb_internal | _hyper_5_49_chunk | 57344 | 81920 | 0 | 139264 | _timescaledb_internal | _hyper_5_50_chunk | 57344 | 81920 | 0 | 139264 | _timescaledb_internal | _hyper_5_51_chunk | 57344 | 81920 | 0 | 139264 | _timescaledb_internal | _hyper_5_52_chunk | 57344 | 81920 | 0 | 139264 | _timescaledb_internal | _hyper_5_53_chunk | 8192 | 32768 | 0 | 40960 | _timescaledb_internal | _hyper_5_54_chunk | 8192 | 32768 | 0 | 40960 | _timescaledb_internal | _hyper_5_55_chunk | 65536 | 106496 | 0 | 172032 | _timescaledb_internal | _hyper_5_56_chunk | 65536 | 98304 | 0 | 163840 | _timescaledb_internal | _hyper_5_57_chunk | 253952 | 360448 | 0 | 614400 | _timescaledb_internal | _hyper_5_58_chunk | 253952 | 368640 | 0 | 622592 | _timescaledb_internal | _hyper_5_59_chunk | 180224 | 303104 | 0 | 483328 | _timescaledb_internal | _hyper_5_60_chunk | 188416 | 303104 | 0 | 491520 | _timescaledb_internal | _hyper_5_61_chunk | 327680 | 540672 | 0 | 868352 | _timescaledb_internal | _hyper_5_62_chunk | 327680 | 532480 | 0 | 860160 | _timescaledb_internal | _hyper_5_63_chunk | 360448 | 581632 | 0 | 942080 | _timescaledb_internal | _hyper_5_64_chunk | 352256 | 589824 | 0 | 942080 | _timescaledb_internal | _hyper_5_65_chunk | 385024 | 598016 | 0 | 983040 | _timescaledb_internal | _hyper_5_66_chunk | 393216 | 614400 | 0 | 1007616 | _timescaledb_internal | _hyper_5_67_chunk | 385024 | 598016 | 0 | 983040 | _timescaledb_internal | _hyper_5_68_chunk | 393216 | 598016 | 0 | 991232 | _timescaledb_internal | _hyper_5_69_chunk | 393216 | 622592 | 0 | 1015808 | _timescaledb_internal | _hyper_5_70_chunk | 385024 | 606208 | 0 | 991232 | _timescaledb_internal | _hyper_5_71_chunk | 385024 | 614400 | 0 | 999424 | _timescaledb_internal | _hyper_5_72_chunk | 393216 | 622592 | 0 | 1015808 | _timescaledb_internal | _hyper_5_73_chunk | 393216 | 614400 | 0 | 1007616 | _timescaledb_internal | _hyper_5_74_chunk | 385024 | 614400 | 0 | 999424 | _timescaledb_internal | _hyper_5_75_chunk | 360448 | 581632 | 0 | 942080 | _timescaledb_internal | _hyper_5_76_chunk | 368640 | 598016 | 0 | 966656 | SELECT id, hypertable_id, interval_length FROM _timescaledb_catalog.dimension; id | hypertable_id | interval_length ----+---------------+----------------- 2 | 2 | 1277664162704 3 | 3 | 2315350169045 6 | 5 | 5 | 5 | 1660010695333 -- A previous version stopped working as soon as hypertable_id stopped being -- equal to dimension_id (i.e., there was a hypertable with more than 1 dimension). -- This test comes after test_adaptive_space, which has 2 dimensions, and makes -- sure that it still works. CREATE TABLE test_adaptive_after_multiple_dims(time timestamptz, temp float, location int); SELECT create_hypertable('test_adaptive_after_multiple_dims', 'time', chunk_target_size => '100MB', create_default_indexes => true); NOTICE: adaptive chunking is a BETA feature and is not recommended for production deployments create_hypertable ------------------------------------------------ (6,public,test_adaptive_after_multiple_dims,t) INSERT INTO test_adaptive_after_multiple_dims VALUES('2018-01-01T00:00:00+00'::timestamptz, 0.0, 5); \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 \set ON_ERROR_STOP 0 SELECT * FROM set_adaptive_chunking('test_adaptive', '2MB'); ERROR: must be owner of hypertable "test_adaptive" \set ON_ERROR_STOP 1 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- Now make sure renaming schema gets propagated to the func_schema DROP TABLE test_adaptive; \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA IF NOT EXISTS my_chunk_func_schema; CREATE OR REPLACE FUNCTION my_chunk_func_schema.calculate_chunk_interval( dimension_id INTEGER, dimension_coord BIGINT, chunk_target_size BIGINT ) RETURNS BIGINT LANGUAGE PLPGSQL AS $BODY$ DECLARE BEGIN RETURN 2; END $BODY$; CREATE TABLE test_adaptive(time timestamptz, temp float, location int); SELECT create_hypertable('test_adaptive', 'time', chunk_target_size => '1MB', chunk_sizing_func => 'my_chunk_func_schema.calculate_chunk_interval'); WARNING: target chunk size for adaptive chunking is less than 10 MB NOTICE: adaptive chunking is a BETA feature and is not recommended for production deployments create_hypertable ---------------------------- (7,public,test_adaptive,t) ALTER SCHEMA my_chunk_func_schema RENAME TO new_chunk_func_schema; INSERT INTO test_adaptive VALUES (now(), 1.0, 1); ================================================ FILE: test/expected/chunk_merge.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION _timescaledb_internal.test_merge_chunks_across_dimension(chunk REGCLASS, merge_chunk REGCLASS, dimension_id INTEGER) RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_merge_chunks_across_dimension' LANGUAGE C VOLATILE; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE test1 ("Time" timestamptz, i integer, value integer); SELECT table_name FROM Create_hypertable('test1', 'Time', chunk_time_interval=> INTERVAL '1 hour'); NOTICE: adding not-null constraint to column "Time" table_name ------------ test1 (1 row) SELECT table_name FROM add_dimension('test1', 'i', chunk_time_interval=> 1); NOTICE: adding not-null constraint to column "i" table_name ------------ test1 (1 row) -- This creates chunks 1 - 3 on first hypertable. INSERT INTO test1 SELECT t, 1, 1.0 FROM generate_series('2018-03-02 1:00'::TIMESTAMPTZ, '2018-03-02 3:00', '1 minute') t; -- This creates chunks 4 - 6 on first hypertable. INSERT INTO test1 SELECT t, 2, 1.0 FROM generate_series('2018-03-02 1:00'::TIMESTAMPTZ, '2018-03-02 3:00', '1 minute') t; CREATE TABLE test2 ("Time" timestamptz, i integer, value integer); SELECT table_name FROM Create_hypertable('test2', 'Time', chunk_time_interval=> INTERVAL '1 hour'); NOTICE: adding not-null constraint to column "Time" table_name ------------ test2 (1 row) -- This creates chunks 7 - 9 on second hypertable. INSERT INTO test2 SELECT t, 1, 1.0 FROM generate_series('2018-03-02 1:00'::TIMESTAMPTZ, '2018-03-02 3:00', '1 minute') t; SELECT * FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk ----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | | f | 0 | f 2 | 1 | _timescaledb_internal | _hyper_1_2_chunk | | f | 0 | f 3 | 1 | _timescaledb_internal | _hyper_1_3_chunk | | f | 0 | f 4 | 1 | _timescaledb_internal | _hyper_1_4_chunk | | f | 0 | f 5 | 1 | _timescaledb_internal | _hyper_1_5_chunk | | f | 0 | f 6 | 1 | _timescaledb_internal | _hyper_1_6_chunk | | f | 0 | f 7 | 2 | _timescaledb_internal | _hyper_2_7_chunk | | f | 0 | f 8 | 2 | _timescaledb_internal | _hyper_2_8_chunk | | f | 0 | f 9 | 2 | _timescaledb_internal | _hyper_2_9_chunk | | f | 0 | f (9 rows) \set ON_ERROR_STOP 0 -- Cannot merge chunks from different hypertables SELECT _timescaledb_internal.test_merge_chunks_across_dimension('_timescaledb_internal._hyper_1_1_chunk','_timescaledb_internal._hyper_2_7_chunk', 1); ERROR: cannot merge chunks from different hypertables -- Cannot merge non-adjacent chunks SELECT _timescaledb_internal.test_merge_chunks_across_dimension('_timescaledb_internal._hyper_1_1_chunk','_timescaledb_internal._hyper_1_3_chunk', 1); ERROR: cannot merge non-adjacent chunks over supplied dimension -- Cannot merge same chunk to itself (its not adjacent to itself). SELECT _timescaledb_internal.test_merge_chunks_across_dimension('_timescaledb_internal._hyper_1_1_chunk','_timescaledb_internal._hyper_1_1_chunk', 1); ERROR: cannot merge non-adjacent chunks over supplied dimension -- Cannot merge chunks on with different partitioning schemas. SELECT _timescaledb_internal.test_merge_chunks_across_dimension('_timescaledb_internal._hyper_1_1_chunk','_timescaledb_internal._hyper_1_4_chunk', 1); ERROR: cannot merge chunks with different partitioning schemas -- Cannot merge chunks on with non-existant dimension slice. -- NOTE: we are merging the same chunk just so they have the exact same partitioning schema and we don't hit the previous test error. SELECT _timescaledb_internal.test_merge_chunks_across_dimension('_timescaledb_internal._hyper_1_1_chunk','_timescaledb_internal._hyper_1_1_chunk', 999); ERROR: cannot find slice for merging dimension \set ON_ERROR_STOP 1 -- Merge on open (time) dimension. SELECT _timescaledb_internal.test_merge_chunks_across_dimension('_timescaledb_internal._hyper_1_5_chunk','_timescaledb_internal._hyper_1_6_chunk', 1); test_merge_chunks_across_dimension ------------------------------------ (1 row) -- Merge on close dimension. SELECT _timescaledb_internal.test_merge_chunks_across_dimension('_timescaledb_internal._hyper_1_1_chunk','_timescaledb_internal._hyper_1_4_chunk', 2); test_merge_chunks_across_dimension ------------------------------------ (1 row) SELECT * FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | dropped | status | osm_chunk ----+---------------+-----------------------+------------------+---------------------+---------+--------+----------- 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | | f | 0 | f 2 | 1 | _timescaledb_internal | _hyper_1_2_chunk | | f | 0 | f 3 | 1 | _timescaledb_internal | _hyper_1_3_chunk | | f | 0 | f 5 | 1 | _timescaledb_internal | _hyper_1_5_chunk | | f | 0 | f 7 | 2 | _timescaledb_internal | _hyper_2_7_chunk | | f | 0 | f 8 | 2 | _timescaledb_internal | _hyper_2_8_chunk | | f | 0 | f 9 | 2 | _timescaledb_internal | _hyper_2_9_chunk | | f | 0 | f (7 rows) ================================================ FILE: test/expected/chunk_publication.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test automatic addition of chunks to publications -- Publications require superuser privileges \c :TEST_DBNAME :ROLE_SUPERUSER SET client_min_messages = WARNING; SET timescaledb.enable_chunk_auto_publication = true; -- Test 1: Basic single publication CREATE TABLE test_hypertable (time timestamptz NOT NULL, device_id int, value float, extra text); SELECT create_hypertable('test_hypertable', 'time', chunk_time_interval => interval '1 day'); create_hypertable ------------------------------ (1,public,test_hypertable,t) -- Insert to create first chunk INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1'); -- Create publication and add hypertable CREATE PUBLICATION test_pub FOR TABLE test_hypertable; -- Verify initial state (1 chunk) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+------------------+------------------------------+----------- _timescaledb_internal | _hyper_1_1_chunk | {time,device_id,value,extra} | public | test_hypertable | {time,device_id,value,extra} | -- Insert to create 2 more chunks (total 3 chunks) INSERT INTO test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0, 'data2'); INSERT INTO test_hypertable VALUES ('2024-01-03 00:00:00+00', 3, 3.0, 'data3'); -- Verify state (3 chunks) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+------------------+------------------------------+----------- _timescaledb_internal | _hyper_1_1_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_1_2_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_1_3_chunk | {time,device_id,value,extra} | public | test_hypertable | {time,device_id,value,extra} | -- Insert to create 5 more chunks (total 8 chunks) INSERT INTO test_hypertable VALUES ('2024-01-04 00:00:00+00', 4, 4.0, 'data4'); INSERT INTO test_hypertable VALUES ('2024-01-05 00:00:00+00', 5, 5.0, 'data5'); INSERT INTO test_hypertable VALUES ('2024-01-06 00:00:00+00', 6, 6.0, 'data6'); INSERT INTO test_hypertable VALUES ('2024-01-07 00:00:00+00', 7, 7.0, 'data7'); INSERT INTO test_hypertable VALUES ('2024-01-08 00:00:00+00', 8, 8.0, 'data8'); -- Verify final state (8 chunks) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+------------------+------------------------------+----------- _timescaledb_internal | _hyper_1_1_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_1_2_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_1_3_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_1_4_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_1_5_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_1_6_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_1_7_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_1_8_chunk | {time,device_id,value,extra} | public | test_hypertable | {time,device_id,value,extra} | -- Verify chunk removal via DROP TABLE SELECT chunk_schema || '.' || chunk_name as "CHUNK_TO_DROP" FROM timescaledb_information.chunks WHERE hypertable_name = 'test_hypertable' ORDER BY chunk_schema, chunk_name LIMIT 1 \gset -- Verify chunk removal via DROP TABLE DROP TABLE :CHUNK_TO_DROP; -- Verify chunk was removed from publication (7 chunks remaining) SELECT chunk_schema, chunk_name FROM timescaledb_information.chunks WHERE hypertable_name = 'test_hypertable' ORDER BY chunk_schema, chunk_name; chunk_schema | chunk_name -----------------------+------------------ _timescaledb_internal | _hyper_1_2_chunk _timescaledb_internal | _hyper_1_3_chunk _timescaledb_internal | _hyper_1_4_chunk _timescaledb_internal | _hyper_1_5_chunk _timescaledb_internal | _hyper_1_6_chunk _timescaledb_internal | _hyper_1_7_chunk _timescaledb_internal | _hyper_1_8_chunk SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+------------------+------------------------------+----------- _timescaledb_internal | _hyper_1_2_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_1_3_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_1_4_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_1_5_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_1_6_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_1_7_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_1_8_chunk | {time,device_id,value,extra} | public | test_hypertable | {time,device_id,value,extra} | -- Verify chunk removal via drop_chunks() SELECT drop_chunks('test_hypertable', older_than => '2024-01-07 00:00:00+00'::timestamptz); drop_chunks ---------------------------------------- _timescaledb_internal._hyper_1_2_chunk _timescaledb_internal._hyper_1_3_chunk _timescaledb_internal._hyper_1_4_chunk _timescaledb_internal._hyper_1_5_chunk _timescaledb_internal._hyper_1_6_chunk -- Verify dropped chunks were removed from publication (2 chunks remaining: 2024-01-07 and 2024-01-08) SELECT chunk_schema, chunk_name FROM timescaledb_information.chunks WHERE hypertable_name = 'test_hypertable' ORDER BY chunk_schema, chunk_name; chunk_schema | chunk_name -----------------------+------------------ _timescaledb_internal | _hyper_1_7_chunk _timescaledb_internal | _hyper_1_8_chunk SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+------------------+------------------------------+----------- _timescaledb_internal | _hyper_1_7_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_1_8_chunk | {time,device_id,value,extra} | public | test_hypertable | {time,device_id,value,extra} | -- Verify chunk removal via TRUNCATE TRUNCATE TABLE test_hypertable; -- Verify all chunks were removed from publication (0 chunks remaining) SELECT chunk_schema, chunk_name FROM timescaledb_information.chunks WHERE hypertable_name = 'test_hypertable' ORDER BY chunk_schema, chunk_name; chunk_schema | chunk_name --------------+------------ SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter ------------+-----------------+------------------------------+----------- public | test_hypertable | {time,device_id,value,extra} | -- Cleanup DROP PUBLICATION test_pub CASCADE; DROP TABLE test_hypertable CASCADE; -- Test 2: Multiple publications CREATE TABLE test_hypertable (time timestamptz NOT NULL, device_id int, value float, extra text); SELECT create_hypertable('test_hypertable', 'time', chunk_time_interval => interval '1 day'); create_hypertable ------------------------------ (2,public,test_hypertable,t) -- Insert to create first chunk INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1'); -- Create pub1 and add hypertable CREATE PUBLICATION test_pub1 FOR TABLE test_hypertable; -- Verify (1 chunk in pub1) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub1' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+------------------+------------------------------+----------- _timescaledb_internal | _hyper_2_9_chunk | {time,device_id,value,extra} | public | test_hypertable | {time,device_id,value,extra} | -- Insert to create 2 more chunks (total 3 chunks) INSERT INTO test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0, 'data2'); INSERT INTO test_hypertable VALUES ('2024-01-03 00:00:00+00', 3, 3.0, 'data3'); -- Verify (3 chunks in pub1) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub1' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+-------------------+------------------------------+----------- _timescaledb_internal | _hyper_2_10_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_2_11_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_2_9_chunk | {time,device_id,value,extra} | public | test_hypertable | {time,device_id,value,extra} | -- Create pub2 and add hypertable CREATE PUBLICATION test_pub2 FOR TABLE test_hypertable; -- Verify (3 chunks in pub1, 3 chunks in pub2) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub1' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+-------------------+------------------------------+----------- _timescaledb_internal | _hyper_2_10_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_2_11_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_2_9_chunk | {time,device_id,value,extra} | public | test_hypertable | {time,device_id,value,extra} | SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub2' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+-------------------+------------------------------+----------- _timescaledb_internal | _hyper_2_10_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_2_11_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_2_9_chunk | {time,device_id,value,extra} | public | test_hypertable | {time,device_id,value,extra} | -- Insert to create 5 more chunks (total 8 chunks) INSERT INTO test_hypertable VALUES ('2024-01-04 00:00:00+00', 4, 4.0, 'data4'); INSERT INTO test_hypertable VALUES ('2024-01-05 00:00:00+00', 5, 5.0, 'data5'); INSERT INTO test_hypertable VALUES ('2024-01-06 00:00:00+00', 6, 6.0, 'data6'); INSERT INTO test_hypertable VALUES ('2024-01-07 00:00:00+00', 7, 7.0, 'data7'); INSERT INTO test_hypertable VALUES ('2024-01-08 00:00:00+00', 8, 8.0, 'data8'); -- Verify (8 chunks in pub1, 8 chunks in pub2) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub1' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+-------------------+------------------------------+----------- _timescaledb_internal | _hyper_2_10_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_2_11_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_2_12_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_2_13_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_2_14_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_2_15_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_2_16_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_2_9_chunk | {time,device_id,value,extra} | public | test_hypertable | {time,device_id,value,extra} | SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub2' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+-------------------+------------------------------+----------- _timescaledb_internal | _hyper_2_10_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_2_11_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_2_12_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_2_13_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_2_14_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_2_15_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_2_16_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_2_9_chunk | {time,device_id,value,extra} | public | test_hypertable | {time,device_id,value,extra} | -- Cleanup DROP PUBLICATION test_pub1 CASCADE; DROP PUBLICATION test_pub2 CASCADE; DROP TABLE test_hypertable CASCADE; -- Test 3: Row filtering (WHERE clause with multiple conditions) CREATE TABLE test_hypertable (time timestamptz NOT NULL, device_id int, value float, extra text); SELECT create_hypertable('test_hypertable', 'time', chunk_time_interval => interval '1 day'); create_hypertable ------------------------------ (3,public,test_hypertable,t) -- Insert to create first chunk INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1'); -- Create publication with row filter (multiple conditions) CREATE PUBLICATION test_pub_row_filter FOR TABLE test_hypertable WHERE (device_id > 10 AND value > 1000); -- Verify initial state (1 chunk with row filter) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_row_filter' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+-------------------+------------------------------+----------------------------------------------------------- _timescaledb_internal | _hyper_3_17_chunk | {time,device_id,value,extra} | ((device_id > 10) AND (value > (1000)::double precision)) public | test_hypertable | {time,device_id,value,extra} | ((device_id > 10) AND (value > (1000)::double precision)) -- Insert to create 2 more chunks (total 3 chunks) INSERT INTO test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0, 'data2'); INSERT INTO test_hypertable VALUES ('2024-01-03 00:00:00+00', 3, 3.0, 'data3'); -- Verify state (3 chunks with row filters) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_row_filter' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+-------------------+------------------------------+----------------------------------------------------------- _timescaledb_internal | _hyper_3_17_chunk | {time,device_id,value,extra} | ((device_id > 10) AND (value > (1000)::double precision)) _timescaledb_internal | _hyper_3_18_chunk | {time,device_id,value,extra} | ((device_id > 10) AND (value > (1000)::double precision)) _timescaledb_internal | _hyper_3_19_chunk | {time,device_id,value,extra} | ((device_id > 10) AND (value > (1000)::double precision)) public | test_hypertable | {time,device_id,value,extra} | ((device_id > 10) AND (value > (1000)::double precision)) -- Insert to create 5 more chunks (total 8 chunks) INSERT INTO test_hypertable VALUES ('2024-01-04 00:00:00+00', 4, 4.0, 'data4'); INSERT INTO test_hypertable VALUES ('2024-01-05 00:00:00+00', 5, 5.0, 'data5'); INSERT INTO test_hypertable VALUES ('2024-01-06 00:00:00+00', 6, 6.0, 'data6'); INSERT INTO test_hypertable VALUES ('2024-01-07 00:00:00+00', 7, 7.0, 'data7'); INSERT INTO test_hypertable VALUES ('2024-01-08 00:00:00+00', 8, 8.0, 'data8'); -- Verify final state (8 chunks with row filters) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_row_filter' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+-------------------+------------------------------+----------------------------------------------------------- _timescaledb_internal | _hyper_3_17_chunk | {time,device_id,value,extra} | ((device_id > 10) AND (value > (1000)::double precision)) _timescaledb_internal | _hyper_3_18_chunk | {time,device_id,value,extra} | ((device_id > 10) AND (value > (1000)::double precision)) _timescaledb_internal | _hyper_3_19_chunk | {time,device_id,value,extra} | ((device_id > 10) AND (value > (1000)::double precision)) _timescaledb_internal | _hyper_3_20_chunk | {time,device_id,value,extra} | ((device_id > 10) AND (value > (1000)::double precision)) _timescaledb_internal | _hyper_3_21_chunk | {time,device_id,value,extra} | ((device_id > 10) AND (value > (1000)::double precision)) _timescaledb_internal | _hyper_3_22_chunk | {time,device_id,value,extra} | ((device_id > 10) AND (value > (1000)::double precision)) _timescaledb_internal | _hyper_3_23_chunk | {time,device_id,value,extra} | ((device_id > 10) AND (value > (1000)::double precision)) _timescaledb_internal | _hyper_3_24_chunk | {time,device_id,value,extra} | ((device_id > 10) AND (value > (1000)::double precision)) public | test_hypertable | {time,device_id,value,extra} | ((device_id > 10) AND (value > (1000)::double precision)) -- Cleanup DROP PUBLICATION test_pub_row_filter CASCADE; DROP TABLE test_hypertable CASCADE; -- Test 4: Column filtering CREATE TABLE test_hypertable (time timestamptz NOT NULL, device_id int, value float, extra text); SELECT create_hypertable('test_hypertable', 'time', chunk_time_interval => interval '1 day'); create_hypertable ------------------------------ (4,public,test_hypertable,t) -- Insert to create first chunk INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1'); -- Create publication with column filter CREATE PUBLICATION test_pub_col_filter FOR TABLE test_hypertable (time, device_id); -- Verify initial state (1 chunk with column filter) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_col_filter' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+-------------------+------------------+----------- _timescaledb_internal | _hyper_4_25_chunk | {time,device_id} | public | test_hypertable | {time,device_id} | -- Insert to create 2 more chunks (total 3 chunks) INSERT INTO test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0, 'data2'); INSERT INTO test_hypertable VALUES ('2024-01-03 00:00:00+00', 3, 3.0, 'data3'); -- Verify state (3 chunks with column filters) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_col_filter' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+-------------------+------------------+----------- _timescaledb_internal | _hyper_4_25_chunk | {time,device_id} | _timescaledb_internal | _hyper_4_26_chunk | {time,device_id} | _timescaledb_internal | _hyper_4_27_chunk | {time,device_id} | public | test_hypertable | {time,device_id} | -- Insert to create 5 more chunks (total 8 chunks) INSERT INTO test_hypertable VALUES ('2024-01-04 00:00:00+00', 4, 4.0, 'data4'); INSERT INTO test_hypertable VALUES ('2024-01-05 00:00:00+00', 5, 5.0, 'data5'); INSERT INTO test_hypertable VALUES ('2024-01-06 00:00:00+00', 6, 6.0, 'data6'); INSERT INTO test_hypertable VALUES ('2024-01-07 00:00:00+00', 7, 7.0, 'data7'); INSERT INTO test_hypertable VALUES ('2024-01-08 00:00:00+00', 8, 8.0, 'data8'); -- Verify final state (8 chunks with column filters) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_col_filter' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+-------------------+------------------+----------- _timescaledb_internal | _hyper_4_25_chunk | {time,device_id} | _timescaledb_internal | _hyper_4_26_chunk | {time,device_id} | _timescaledb_internal | _hyper_4_27_chunk | {time,device_id} | _timescaledb_internal | _hyper_4_28_chunk | {time,device_id} | _timescaledb_internal | _hyper_4_29_chunk | {time,device_id} | _timescaledb_internal | _hyper_4_30_chunk | {time,device_id} | _timescaledb_internal | _hyper_4_31_chunk | {time,device_id} | _timescaledb_internal | _hyper_4_32_chunk | {time,device_id} | public | test_hypertable | {time,device_id} | -- Cleanup DROP PUBLICATION test_pub_col_filter CASCADE; DROP TABLE test_hypertable CASCADE; -- Test 5: Combined row + column filtering CREATE TABLE test_hypertable (time timestamptz NOT NULL, device_id int, value float, extra text); SELECT create_hypertable('test_hypertable', 'time', chunk_time_interval => interval '1 day'); create_hypertable ------------------------------ (5,public,test_hypertable,t) -- Insert to create first chunk INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1'); -- Create publication with both row and column filters CREATE PUBLICATION test_pub_combined FOR TABLE test_hypertable (time, device_id) WHERE (device_id > 10); -- Verify initial state (1 chunk with both filters) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_combined' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+-------------------+------------------+------------------ _timescaledb_internal | _hyper_5_33_chunk | {time,device_id} | (device_id > 10) public | test_hypertable | {time,device_id} | (device_id > 10) -- Insert to create 2 more chunks (total 3 chunks) INSERT INTO test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0, 'data2'); INSERT INTO test_hypertable VALUES ('2024-01-03 00:00:00+00', 3, 3.0, 'data3'); -- Verify state (3 chunks with both filters) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_combined' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+-------------------+------------------+------------------ _timescaledb_internal | _hyper_5_33_chunk | {time,device_id} | (device_id > 10) _timescaledb_internal | _hyper_5_34_chunk | {time,device_id} | (device_id > 10) _timescaledb_internal | _hyper_5_35_chunk | {time,device_id} | (device_id > 10) public | test_hypertable | {time,device_id} | (device_id > 10) -- Insert to create 5 more chunks (total 8 chunks) INSERT INTO test_hypertable VALUES ('2024-01-04 00:00:00+00', 4, 4.0, 'data4'); INSERT INTO test_hypertable VALUES ('2024-01-05 00:00:00+00', 5, 5.0, 'data5'); INSERT INTO test_hypertable VALUES ('2024-01-06 00:00:00+00', 6, 6.0, 'data6'); INSERT INTO test_hypertable VALUES ('2024-01-07 00:00:00+00', 7, 7.0, 'data7'); INSERT INTO test_hypertable VALUES ('2024-01-08 00:00:00+00', 8, 8.0, 'data8'); -- Verify final state (8 chunks with both filters) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_combined' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+-------------------+------------------+------------------ _timescaledb_internal | _hyper_5_33_chunk | {time,device_id} | (device_id > 10) _timescaledb_internal | _hyper_5_34_chunk | {time,device_id} | (device_id > 10) _timescaledb_internal | _hyper_5_35_chunk | {time,device_id} | (device_id > 10) _timescaledb_internal | _hyper_5_36_chunk | {time,device_id} | (device_id > 10) _timescaledb_internal | _hyper_5_37_chunk | {time,device_id} | (device_id > 10) _timescaledb_internal | _hyper_5_38_chunk | {time,device_id} | (device_id > 10) _timescaledb_internal | _hyper_5_39_chunk | {time,device_id} | (device_id > 10) _timescaledb_internal | _hyper_5_40_chunk | {time,device_id} | (device_id > 10) public | test_hypertable | {time,device_id} | (device_id > 10) -- Cleanup DROP PUBLICATION test_pub_combined CASCADE; DROP TABLE test_hypertable CASCADE; -- Test 6: FOR ALL TABLES publication CREATE TABLE test_hypertable (time timestamptz NOT NULL, device_id int, value float, extra text); SELECT create_hypertable('test_hypertable', 'time', chunk_time_interval => interval '1 day'); create_hypertable ------------------------------ (6,public,test_hypertable,t) -- Insert to create first chunk INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1'); -- Create FOR ALL TABLES publication CREATE PUBLICATION test_pub_all_tables FOR ALL TABLES; -- Verify initial state (1 chunk) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_all_tables' AND tablename LIKE '%test_hypertable%' OR tablename LIKE '_hyper_%' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+-------------------+------------------------------+----------- _timescaledb_internal | _hyper_6_41_chunk | {time,device_id,value,extra} | public | test_hypertable | {time,device_id,value,extra} | -- Insert to create 2 more chunks (total 3 chunks) INSERT INTO test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0, 'data2'); INSERT INTO test_hypertable VALUES ('2024-01-03 00:00:00+00', 3, 3.0, 'data3'); -- Verify state (3 chunks) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_all_tables' AND tablename LIKE '%test_hypertable%' OR tablename LIKE '_hyper_%' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+-------------------+------------------------------+----------- _timescaledb_internal | _hyper_6_41_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_6_42_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_6_43_chunk | {time,device_id,value,extra} | public | test_hypertable | {time,device_id,value,extra} | -- Insert to create 5 more chunks (total 8 chunks) INSERT INTO test_hypertable VALUES ('2024-01-04 00:00:00+00', 4, 4.0, 'data4'); INSERT INTO test_hypertable VALUES ('2024-01-05 00:00:00+00', 5, 5.0, 'data5'); INSERT INTO test_hypertable VALUES ('2024-01-06 00:00:00+00', 6, 6.0, 'data6'); INSERT INTO test_hypertable VALUES ('2024-01-07 00:00:00+00', 7, 7.0, 'data7'); INSERT INTO test_hypertable VALUES ('2024-01-08 00:00:00+00', 8, 8.0, 'data8'); -- Verify final state (8 chunks) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_all_tables' AND tablename LIKE '%test_hypertable%' OR tablename LIKE '_hyper_%' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+-------------------+------------------------------+----------- _timescaledb_internal | _hyper_6_41_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_6_42_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_6_43_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_6_44_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_6_45_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_6_46_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_6_47_chunk | {time,device_id,value,extra} | _timescaledb_internal | _hyper_6_48_chunk | {time,device_id,value,extra} | public | test_hypertable | {time,device_id,value,extra} | -- Cleanup DROP PUBLICATION test_pub_all_tables CASCADE; DROP TABLE test_hypertable CASCADE; -- Test 7: Edge case - Hypertable not in any publication CREATE TABLE test_hypertable (time timestamptz NOT NULL, device_id int, value float, extra text); SELECT create_hypertable('test_hypertable', 'time', chunk_time_interval => interval '1 day'); create_hypertable ------------------------------ (7,public,test_hypertable,t) -- Insert to create 8 chunks without any publication INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1'); INSERT INTO test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0, 'data2'); INSERT INTO test_hypertable VALUES ('2024-01-03 00:00:00+00', 3, 3.0, 'data3'); INSERT INTO test_hypertable VALUES ('2024-01-04 00:00:00+00', 4, 4.0, 'data4'); INSERT INTO test_hypertable VALUES ('2024-01-05 00:00:00+00', 5, 5.0, 'data5'); INSERT INTO test_hypertable VALUES ('2024-01-06 00:00:00+00', 6, 6.0, 'data6'); INSERT INTO test_hypertable VALUES ('2024-01-07 00:00:00+00', 7, 7.0, 'data7'); INSERT INTO test_hypertable VALUES ('2024-01-08 00:00:00+00', 8, 8.0, 'data8'); -- Verify chunks were created successfully SELECT COUNT(*) as chunks_created FROM timescaledb_information.chunks WHERE hypertable_name = 'test_hypertable'; chunks_created ---------------- 8 -- Cleanup DROP TABLE test_hypertable CASCADE; -- Test 8: Edge case - Publication dropped before chunk creation CREATE TABLE test_hypertable (time timestamptz NOT NULL, device_id int, value float, extra text); SELECT create_hypertable('test_hypertable', 'time', chunk_time_interval => interval '1 day'); create_hypertable ------------------------------ (8,public,test_hypertable,t) -- Insert to create first chunk INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1'); -- Create publication and add hypertable CREATE PUBLICATION test_pub FOR TABLE test_hypertable; -- Verify (1 chunk in publication) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub' ORDER BY schemaname, tablename; schemaname | tablename | attnames | rowfilter -----------------------+-------------------+------------------------------+----------- _timescaledb_internal | _hyper_8_57_chunk | {time,device_id,value,extra} | public | test_hypertable | {time,device_id,value,extra} | -- Drop the publication DROP PUBLICATION test_pub; -- Insert to create 2 more chunks (total 3 chunks) -- Should succeed with WARNING, not error INSERT INTO test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0, 'data2'); INSERT INTO test_hypertable VALUES ('2024-01-03 00:00:00+00', 3, 3.0, 'data3'); -- Verify chunks were created successfully despite missing publication SELECT COUNT(*) as chunks_after_pub_drop FROM timescaledb_information.chunks WHERE hypertable_name = 'test_hypertable'; chunks_after_pub_drop ----------------------- 3 -- Cleanup DROP TABLE test_hypertable CASCADE; -- Test 9: GUC control of chunk publication CREATE TABLE test_hypertable (time timestamptz NOT NULL, device_id int, value float, extra text); SELECT create_hypertable('test_hypertable', 'time', chunk_time_interval => interval '1 day'); create_hypertable ------------------------------ (9,public,test_hypertable,t) -- Insert to create first chunk INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1'); -- Create publication CREATE PUBLICATION test_pub_guc FOR TABLE test_hypertable; -- Verify initial state (1 chunk) SELECT schemaname, tablename FROM pg_publication_tables WHERE pubname = 'test_pub_guc' ORDER BY schemaname, tablename; schemaname | tablename -----------------------+------------------- _timescaledb_internal | _hyper_9_60_chunk public | test_hypertable -- Test Part 1: GUC enabled - chunks should be added to publication automatically -- Insert to create a new chunk - should be added to publication automatically INSERT INTO test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0, 'data2'); -- Verify (2 chunks in publication) SELECT schemaname, tablename FROM pg_publication_tables WHERE pubname = 'test_pub_guc' ORDER BY schemaname, tablename; schemaname | tablename -----------------------+------------------- _timescaledb_internal | _hyper_9_60_chunk _timescaledb_internal | _hyper_9_61_chunk public | test_hypertable -- Test Part 2: Disable the GUC and create another chunk SET timescaledb.enable_chunk_auto_publication = false; -- Insert to create a new chunk - should NOT be added to publication INSERT INTO test_hypertable VALUES ('2024-01-03 00:00:00+00', 3, 3.0, 'data3'); -- Verify (still 2 chunks in publication, chunk 3 should not be there) SELECT schemaname, tablename FROM pg_publication_tables WHERE pubname = 'test_pub_guc' ORDER BY schemaname, tablename; schemaname | tablename -----------------------+------------------- _timescaledb_internal | _hyper_9_60_chunk _timescaledb_internal | _hyper_9_61_chunk public | test_hypertable -- Verify that chunk 3 exists but is not in the publication SELECT chunk_schema, chunk_name FROM timescaledb_information.chunks WHERE hypertable_name = 'test_hypertable'; chunk_schema | chunk_name -----------------------+------------------- _timescaledb_internal | _hyper_9_60_chunk _timescaledb_internal | _hyper_9_61_chunk _timescaledb_internal | _hyper_9_62_chunk -- Test Part 3: Re-enable the GUC and create another chunk SET timescaledb.enable_chunk_auto_publication = true; -- Insert to create a new chunk - should be added to publication again INSERT INTO test_hypertable VALUES ('2024-01-04 00:00:00+00', 4, 4.0, 'data4'); -- Verify (3 chunks in publication: chunk 1, 2, and 4; chunk 3 still missing) SELECT schemaname, tablename FROM pg_publication_tables WHERE pubname = 'test_pub_guc' ORDER BY schemaname, tablename; schemaname | tablename -----------------------+------------------- _timescaledb_internal | _hyper_9_60_chunk _timescaledb_internal | _hyper_9_61_chunk _timescaledb_internal | _hyper_9_63_chunk public | test_hypertable SELECT chunk_schema, chunk_name FROM timescaledb_information.chunks WHERE hypertable_name = 'test_hypertable'; chunk_schema | chunk_name -----------------------+------------------- _timescaledb_internal | _hyper_9_60_chunk _timescaledb_internal | _hyper_9_61_chunk _timescaledb_internal | _hyper_9_62_chunk _timescaledb_internal | _hyper_9_63_chunk -- Cleanup DROP PUBLICATION test_pub_guc CASCADE; DROP TABLE test_hypertable CASCADE; RESET client_min_messages; ================================================ FILE: test/expected/chunk_utils.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Set this variable to avoid using a hard-coded path each time query -- results are compared \set QUERY_RESULT_TEST_EQUAL_RELPATH 'include/query_result_test_equal.sql' CREATE OR REPLACE FUNCTION dimension_get_time( hypertable_id INT ) RETURNS _timescaledb_catalog.dimension LANGUAGE SQL STABLE AS $BODY$ SELECT * FROM _timescaledb_catalog.dimension d WHERE d.hypertable_id = dimension_get_time.hypertable_id AND d.interval_length IS NOT NULL $BODY$; CREATE TABLE PUBLIC.drop_chunk_test1(time bigint, temp float8, device_id text); CREATE TABLE PUBLIC.drop_chunk_test2(time bigint, temp float8, device_id text); CREATE TABLE PUBLIC.drop_chunk_test3(time bigint, temp float8, device_id text); CREATE INDEX ON drop_chunk_test1(time DESC); -- show_chunks() without specifying a table is not allowed \set ON_ERROR_STOP 0 SELECT show_chunks(NULL); ERROR: invalid hypertable or continuous aggregate \set ON_ERROR_STOP 1 SELECT create_hypertable('public.drop_chunk_test1', 'time', chunk_time_interval => 1, create_default_indexes=>false); create_hypertable ------------------------------- (1,public,drop_chunk_test1,t) SELECT create_hypertable('public.drop_chunk_test2', 'time', chunk_time_interval => 1, create_default_indexes=>false); create_hypertable ------------------------------- (2,public,drop_chunk_test2,t) SELECT create_hypertable('public.drop_chunk_test3', 'time', chunk_time_interval => 1, create_default_indexes=>false); create_hypertable ------------------------------- (3,public,drop_chunk_test3,t) -- Add space dimensions to ensure chunks share dimension slices SELECT add_dimension('public.drop_chunk_test1', 'device_id', 2); add_dimension ----------------------------------------- (4,public,drop_chunk_test1,device_id,t) SELECT add_dimension('public.drop_chunk_test2', 'device_id', 2); add_dimension ----------------------------------------- (5,public,drop_chunk_test2,device_id,t) SELECT add_dimension('public.drop_chunk_test3', 'device_id', 2); add_dimension ----------------------------------------- (6,public,drop_chunk_test3,device_id,t) SELECT c.id AS chunk_id, c.hypertable_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN dimension_get_time(h.id) time_dimension ON(true) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = time_dimension.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.schema_name = 'public' AND (h.table_name = 'drop_chunk_test1' OR h.table_name = 'drop_chunk_test2') ORDER BY c.id; chunk_id | hypertable_id | chunk_schema | chunk_table | range_start | range_end ----------+---------------+--------------+-------------+-------------+----------- SELECT * FROM test.relation WHERE schema = '_timescaledb_internal' AND name LIKE '\_hyper%'; schema | name | type | owner --------+------+------+------- SELECT _timescaledb_functions.get_partition_for_key('dev1'::text); get_partition_for_key ----------------------- 1129986420 SELECT _timescaledb_functions.get_partition_for_key('dev7'::varchar(5)); get_partition_for_key ----------------------- 449729092 INSERT INTO PUBLIC.drop_chunk_test1 VALUES(1, 1.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test1 VALUES(2, 2.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test1 VALUES(3, 3.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test1 VALUES(4, 4.0, 'dev7'); INSERT INTO PUBLIC.drop_chunk_test1 VALUES(5, 5.0, 'dev7'); INSERT INTO PUBLIC.drop_chunk_test1 VALUES(6, 6.0, 'dev7'); INSERT INTO PUBLIC.drop_chunk_test2 VALUES(1, 1.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test2 VALUES(2, 2.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test2 VALUES(3, 3.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test2 VALUES(4, 4.0, 'dev7'); INSERT INTO PUBLIC.drop_chunk_test2 VALUES(5, 5.0, 'dev7'); INSERT INTO PUBLIC.drop_chunk_test2 VALUES(6, 6.0, 'dev7'); INSERT INTO PUBLIC.drop_chunk_test3 VALUES(1, 1.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test3 VALUES(2, 2.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test3 VALUES(3, 3.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test3 VALUES(4, 4.0, 'dev7'); INSERT INTO PUBLIC.drop_chunk_test3 VALUES(5, 5.0, 'dev7'); INSERT INTO PUBLIC.drop_chunk_test3 VALUES(6, 6.0, 'dev7'); SELECT c.id AS chunk_id, c.hypertable_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN dimension_get_time(h.id) time_dimension ON(true) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = time_dimension.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.schema_name = 'public' AND (h.table_name = 'drop_chunk_test1' OR h.table_name = 'drop_chunk_test2') ORDER BY c.id; chunk_id | hypertable_id | chunk_schema | chunk_table | range_start | range_end ----------+---------------+-----------------------+-------------------+-------------+----------- 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | 1 | 2 2 | 1 | _timescaledb_internal | _hyper_1_2_chunk | 2 | 3 3 | 1 | _timescaledb_internal | _hyper_1_3_chunk | 3 | 4 4 | 1 | _timescaledb_internal | _hyper_1_4_chunk | 4 | 5 5 | 1 | _timescaledb_internal | _hyper_1_5_chunk | 5 | 6 6 | 1 | _timescaledb_internal | _hyper_1_6_chunk | 6 | 7 7 | 2 | _timescaledb_internal | _hyper_2_7_chunk | 1 | 2 8 | 2 | _timescaledb_internal | _hyper_2_8_chunk | 2 | 3 9 | 2 | _timescaledb_internal | _hyper_2_9_chunk | 3 | 4 10 | 2 | _timescaledb_internal | _hyper_2_10_chunk | 4 | 5 11 | 2 | _timescaledb_internal | _hyper_2_11_chunk | 5 | 6 12 | 2 | _timescaledb_internal | _hyper_2_12_chunk | 6 | 7 SELECT * FROM test.relation WHERE schema = '_timescaledb_internal' AND name LIKE '\_hyper%'; schema | name | type | owner -----------------------+-------------------+-------+------------------- _timescaledb_internal | _hyper_1_1_chunk | table | default_perm_user _timescaledb_internal | _hyper_1_2_chunk | table | default_perm_user _timescaledb_internal | _hyper_1_3_chunk | table | default_perm_user _timescaledb_internal | _hyper_1_4_chunk | table | default_perm_user _timescaledb_internal | _hyper_1_5_chunk | table | default_perm_user _timescaledb_internal | _hyper_1_6_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_10_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_11_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_12_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_7_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_8_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_9_chunk | table | default_perm_user _timescaledb_internal | _hyper_3_13_chunk | table | default_perm_user _timescaledb_internal | _hyper_3_14_chunk | table | default_perm_user _timescaledb_internal | _hyper_3_15_chunk | table | default_perm_user _timescaledb_internal | _hyper_3_16_chunk | table | default_perm_user _timescaledb_internal | _hyper_3_17_chunk | table | default_perm_user _timescaledb_internal | _hyper_3_18_chunk | table | default_perm_user -- next two calls of show_chunks should give same set of chunks as above when combined SELECT show_chunks('drop_chunk_test1'); show_chunks ---------------------------------------- _timescaledb_internal._hyper_1_1_chunk _timescaledb_internal._hyper_1_2_chunk _timescaledb_internal._hyper_1_3_chunk _timescaledb_internal._hyper_1_4_chunk _timescaledb_internal._hyper_1_5_chunk _timescaledb_internal._hyper_1_6_chunk SELECT * FROM show_chunks('drop_chunk_test2'); show_chunks ----------------------------------------- _timescaledb_internal._hyper_2_7_chunk _timescaledb_internal._hyper_2_8_chunk _timescaledb_internal._hyper_2_9_chunk _timescaledb_internal._hyper_2_10_chunk _timescaledb_internal._hyper_2_11_chunk _timescaledb_internal._hyper_2_12_chunk CREATE VIEW dependent_view AS SELECT * FROM _timescaledb_internal._hyper_1_1_chunk; \set ON_ERROR_STOP 0 SELECT drop_chunks('drop_chunk_test1'); ERROR: invalid time range for dropping chunks SELECT drop_chunks('drop_chunk_test1', older_than => 2); ERROR: cannot drop table _timescaledb_internal._hyper_1_1_chunk because other objects depend on it SELECT drop_chunks('drop_chunk_test1', older_than => NULL::interval); ERROR: invalid time range for dropping chunks SELECT drop_chunks('drop_chunk_test1', older_than => NULL::int); ERROR: invalid time range for dropping chunks DROP VIEW dependent_view; -- should error because of wrong relative order of time constraints SELECT show_chunks('drop_chunk_test1', older_than=>3, newer_than=>4); ERROR: invalid time range -- Should error because NULL was used for the table name. SELECT drop_chunks(NULL, older_than => 3); ERROR: invalid hypertable or continuous aggregate -- should error because there is no relation with that OID. SELECT drop_chunks(3533, older_than => 3); ERROR: invalid hypertable or continuous aggregate \set ON_ERROR_STOP 1 -- show created constraints and dimension slices for each chunk SELECT c.table_name, cc.constraint_name, ds.id AS dimension_slice_id, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (c.id = cc.chunk_id) FULL OUTER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.id = cc.dimension_slice_id) ORDER BY c.id; table_name | constraint_name | dimension_slice_id | range_start | range_end -------------------+-----------------+--------------------+----------------------+--------------------- _hyper_1_1_chunk | constraint_1 | 1 | 1 | 2 _hyper_1_1_chunk | constraint_2 | 2 | 1073741823 | 9223372036854775807 _hyper_1_2_chunk | constraint_3 | 3 | 2 | 3 _hyper_1_2_chunk | constraint_2 | 2 | 1073741823 | 9223372036854775807 _hyper_1_3_chunk | constraint_4 | 4 | 3 | 4 _hyper_1_3_chunk | constraint_2 | 2 | 1073741823 | 9223372036854775807 _hyper_1_4_chunk | constraint_5 | 5 | 4 | 5 _hyper_1_4_chunk | constraint_6 | 6 | -9223372036854775808 | 1073741823 _hyper_1_5_chunk | constraint_7 | 7 | 5 | 6 _hyper_1_5_chunk | constraint_6 | 6 | -9223372036854775808 | 1073741823 _hyper_1_6_chunk | constraint_8 | 8 | 6 | 7 _hyper_1_6_chunk | constraint_6 | 6 | -9223372036854775808 | 1073741823 _hyper_2_7_chunk | constraint_9 | 9 | 1 | 2 _hyper_2_7_chunk | constraint_10 | 10 | 1073741823 | 9223372036854775807 _hyper_2_8_chunk | constraint_11 | 11 | 2 | 3 _hyper_2_8_chunk | constraint_10 | 10 | 1073741823 | 9223372036854775807 _hyper_2_9_chunk | constraint_12 | 12 | 3 | 4 _hyper_2_9_chunk | constraint_10 | 10 | 1073741823 | 9223372036854775807 _hyper_2_10_chunk | constraint_13 | 13 | 4 | 5 _hyper_2_10_chunk | constraint_14 | 14 | -9223372036854775808 | 1073741823 _hyper_2_11_chunk | constraint_15 | 15 | 5 | 6 _hyper_2_11_chunk | constraint_14 | 14 | -9223372036854775808 | 1073741823 _hyper_2_12_chunk | constraint_16 | 16 | 6 | 7 _hyper_2_12_chunk | constraint_14 | 14 | -9223372036854775808 | 1073741823 _hyper_3_13_chunk | constraint_17 | 17 | 1 | 2 _hyper_3_13_chunk | constraint_18 | 18 | 1073741823 | 9223372036854775807 _hyper_3_14_chunk | constraint_19 | 19 | 2 | 3 _hyper_3_14_chunk | constraint_18 | 18 | 1073741823 | 9223372036854775807 _hyper_3_15_chunk | constraint_20 | 20 | 3 | 4 _hyper_3_15_chunk | constraint_18 | 18 | 1073741823 | 9223372036854775807 _hyper_3_16_chunk | constraint_21 | 21 | 4 | 5 _hyper_3_16_chunk | constraint_22 | 22 | -9223372036854775808 | 1073741823 _hyper_3_17_chunk | constraint_23 | 23 | 5 | 6 _hyper_3_17_chunk | constraint_22 | 22 | -9223372036854775808 | 1073741823 _hyper_3_18_chunk | constraint_24 | 24 | 6 | 7 _hyper_3_18_chunk | constraint_22 | 22 | -9223372036854775808 | 1073741823 SELECT * FROM _timescaledb_catalog.dimension_slice ORDER BY id; id | dimension_id | range_start | range_end ----+--------------+----------------------+--------------------- 1 | 1 | 1 | 2 2 | 4 | 1073741823 | 9223372036854775807 3 | 1 | 2 | 3 4 | 1 | 3 | 4 5 | 1 | 4 | 5 6 | 4 | -9223372036854775808 | 1073741823 7 | 1 | 5 | 6 8 | 1 | 6 | 7 9 | 2 | 1 | 2 10 | 5 | 1073741823 | 9223372036854775807 11 | 2 | 2 | 3 12 | 2 | 3 | 4 13 | 2 | 4 | 5 14 | 5 | -9223372036854775808 | 1073741823 15 | 2 | 5 | 6 16 | 2 | 6 | 7 17 | 3 | 1 | 2 18 | 6 | 1073741823 | 9223372036854775807 19 | 3 | 2 | 3 20 | 3 | 3 | 4 21 | 3 | 4 | 5 22 | 6 | -9223372036854775808 | 1073741823 23 | 3 | 5 | 6 24 | 3 | 6 | 7 -- Test that truncating chunks works SELECT count(*) FROM _timescaledb_internal._hyper_2_7_chunk; count ------- 1 TRUNCATE TABLE _timescaledb_internal._hyper_2_7_chunk; SELECT count(*) FROM _timescaledb_internal._hyper_2_7_chunk; count ------- 0 -- Drop one chunk "manually" and verify that dimension slices and -- constraints are cleaned up. Each chunk has two constraints and two -- dimension slices. Both constraints should be deleted, but only one -- slice should be deleted since the space-dimension slice is shared -- with other chunks in the same hypertable DROP TABLE _timescaledb_internal._hyper_2_7_chunk; -- Two constraints deleted compared to above SELECT c.table_name, cc.constraint_name, ds.id AS dimension_slice_id, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (c.id = cc.chunk_id) FULL OUTER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.id = cc.dimension_slice_id) ORDER BY c.id; table_name | constraint_name | dimension_slice_id | range_start | range_end -------------------+-----------------+--------------------+----------------------+--------------------- _hyper_1_1_chunk | constraint_1 | 1 | 1 | 2 _hyper_1_1_chunk | constraint_2 | 2 | 1073741823 | 9223372036854775807 _hyper_1_2_chunk | constraint_3 | 3 | 2 | 3 _hyper_1_2_chunk | constraint_2 | 2 | 1073741823 | 9223372036854775807 _hyper_1_3_chunk | constraint_4 | 4 | 3 | 4 _hyper_1_3_chunk | constraint_2 | 2 | 1073741823 | 9223372036854775807 _hyper_1_4_chunk | constraint_5 | 5 | 4 | 5 _hyper_1_4_chunk | constraint_6 | 6 | -9223372036854775808 | 1073741823 _hyper_1_5_chunk | constraint_7 | 7 | 5 | 6 _hyper_1_5_chunk | constraint_6 | 6 | -9223372036854775808 | 1073741823 _hyper_1_6_chunk | constraint_8 | 8 | 6 | 7 _hyper_1_6_chunk | constraint_6 | 6 | -9223372036854775808 | 1073741823 _hyper_2_8_chunk | constraint_11 | 11 | 2 | 3 _hyper_2_8_chunk | constraint_10 | 10 | 1073741823 | 9223372036854775807 _hyper_2_9_chunk | constraint_12 | 12 | 3 | 4 _hyper_2_9_chunk | constraint_10 | 10 | 1073741823 | 9223372036854775807 _hyper_2_10_chunk | constraint_13 | 13 | 4 | 5 _hyper_2_10_chunk | constraint_14 | 14 | -9223372036854775808 | 1073741823 _hyper_2_11_chunk | constraint_15 | 15 | 5 | 6 _hyper_2_11_chunk | constraint_14 | 14 | -9223372036854775808 | 1073741823 _hyper_2_12_chunk | constraint_16 | 16 | 6 | 7 _hyper_2_12_chunk | constraint_14 | 14 | -9223372036854775808 | 1073741823 _hyper_3_13_chunk | constraint_17 | 17 | 1 | 2 _hyper_3_13_chunk | constraint_18 | 18 | 1073741823 | 9223372036854775807 _hyper_3_14_chunk | constraint_19 | 19 | 2 | 3 _hyper_3_14_chunk | constraint_18 | 18 | 1073741823 | 9223372036854775807 _hyper_3_15_chunk | constraint_20 | 20 | 3 | 4 _hyper_3_15_chunk | constraint_18 | 18 | 1073741823 | 9223372036854775807 _hyper_3_16_chunk | constraint_21 | 21 | 4 | 5 _hyper_3_16_chunk | constraint_22 | 22 | -9223372036854775808 | 1073741823 _hyper_3_17_chunk | constraint_23 | 23 | 5 | 6 _hyper_3_17_chunk | constraint_22 | 22 | -9223372036854775808 | 1073741823 _hyper_3_18_chunk | constraint_24 | 24 | 6 | 7 _hyper_3_18_chunk | constraint_22 | 22 | -9223372036854775808 | 1073741823 -- Only one dimension slice deleted SELECT * FROM _timescaledb_catalog.dimension_slice ORDER BY id; id | dimension_id | range_start | range_end ----+--------------+----------------------+--------------------- 1 | 1 | 1 | 2 2 | 4 | 1073741823 | 9223372036854775807 3 | 1 | 2 | 3 4 | 1 | 3 | 4 5 | 1 | 4 | 5 6 | 4 | -9223372036854775808 | 1073741823 7 | 1 | 5 | 6 8 | 1 | 6 | 7 10 | 5 | 1073741823 | 9223372036854775807 11 | 2 | 2 | 3 12 | 2 | 3 | 4 13 | 2 | 4 | 5 14 | 5 | -9223372036854775808 | 1073741823 15 | 2 | 5 | 6 16 | 2 | 6 | 7 17 | 3 | 1 | 2 18 | 6 | 1073741823 | 9223372036854775807 19 | 3 | 2 | 3 20 | 3 | 3 | 4 21 | 3 | 4 | 5 22 | 6 | -9223372036854775808 | 1073741823 23 | 3 | 5 | 6 24 | 3 | 6 | 7 -- We drop all chunks older than timestamp 2 in all hypertable. This -- is added only to avoid making the diff for this commit larger than -- necessary and make reviews easier. SELECT drop_chunks(format('%1$I.%2$I', schema_name, table_name)::regclass, older_than => 2) FROM _timescaledb_catalog.hypertable; drop_chunks ----------------------------------------- _timescaledb_internal._hyper_1_1_chunk _timescaledb_internal._hyper_3_13_chunk SELECT c.table_name, cc.constraint_name, ds.id AS dimension_slice_id, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (c.id = cc.chunk_id) FULL OUTER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.id = cc.dimension_slice_id) ORDER BY c.id; table_name | constraint_name | dimension_slice_id | range_start | range_end -------------------+-----------------+--------------------+----------------------+--------------------- _hyper_1_2_chunk | constraint_3 | 3 | 2 | 3 _hyper_1_2_chunk | constraint_2 | 2 | 1073741823 | 9223372036854775807 _hyper_1_3_chunk | constraint_4 | 4 | 3 | 4 _hyper_1_3_chunk | constraint_2 | 2 | 1073741823 | 9223372036854775807 _hyper_1_4_chunk | constraint_5 | 5 | 4 | 5 _hyper_1_4_chunk | constraint_6 | 6 | -9223372036854775808 | 1073741823 _hyper_1_5_chunk | constraint_7 | 7 | 5 | 6 _hyper_1_5_chunk | constraint_6 | 6 | -9223372036854775808 | 1073741823 _hyper_1_6_chunk | constraint_8 | 8 | 6 | 7 _hyper_1_6_chunk | constraint_6 | 6 | -9223372036854775808 | 1073741823 _hyper_2_8_chunk | constraint_11 | 11 | 2 | 3 _hyper_2_8_chunk | constraint_10 | 10 | 1073741823 | 9223372036854775807 _hyper_2_9_chunk | constraint_12 | 12 | 3 | 4 _hyper_2_9_chunk | constraint_10 | 10 | 1073741823 | 9223372036854775807 _hyper_2_10_chunk | constraint_13 | 13 | 4 | 5 _hyper_2_10_chunk | constraint_14 | 14 | -9223372036854775808 | 1073741823 _hyper_2_11_chunk | constraint_15 | 15 | 5 | 6 _hyper_2_11_chunk | constraint_14 | 14 | -9223372036854775808 | 1073741823 _hyper_2_12_chunk | constraint_16 | 16 | 6 | 7 _hyper_2_12_chunk | constraint_14 | 14 | -9223372036854775808 | 1073741823 _hyper_3_14_chunk | constraint_19 | 19 | 2 | 3 _hyper_3_14_chunk | constraint_18 | 18 | 1073741823 | 9223372036854775807 _hyper_3_15_chunk | constraint_20 | 20 | 3 | 4 _hyper_3_15_chunk | constraint_18 | 18 | 1073741823 | 9223372036854775807 _hyper_3_16_chunk | constraint_21 | 21 | 4 | 5 _hyper_3_16_chunk | constraint_22 | 22 | -9223372036854775808 | 1073741823 _hyper_3_17_chunk | constraint_23 | 23 | 5 | 6 _hyper_3_17_chunk | constraint_22 | 22 | -9223372036854775808 | 1073741823 _hyper_3_18_chunk | constraint_24 | 24 | 6 | 7 _hyper_3_18_chunk | constraint_22 | 22 | -9223372036854775808 | 1073741823 SELECT * FROM _timescaledb_catalog.dimension_slice ORDER BY id; id | dimension_id | range_start | range_end ----+--------------+----------------------+--------------------- 2 | 4 | 1073741823 | 9223372036854775807 3 | 1 | 2 | 3 4 | 1 | 3 | 4 5 | 1 | 4 | 5 6 | 4 | -9223372036854775808 | 1073741823 7 | 1 | 5 | 6 8 | 1 | 6 | 7 10 | 5 | 1073741823 | 9223372036854775807 11 | 2 | 2 | 3 12 | 2 | 3 | 4 13 | 2 | 4 | 5 14 | 5 | -9223372036854775808 | 1073741823 15 | 2 | 5 | 6 16 | 2 | 6 | 7 18 | 6 | 1073741823 | 9223372036854775807 19 | 3 | 2 | 3 20 | 3 | 3 | 4 21 | 3 | 4 | 5 22 | 6 | -9223372036854775808 | 1073741823 23 | 3 | 5 | 6 24 | 3 | 6 | 7 SELECT c.id AS chunk_id, c.hypertable_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN dimension_get_time(h.id) time_dimension ON(true) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = time_dimension.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.schema_name = 'public' AND (h.table_name = 'drop_chunk_test1' OR h.table_name = 'drop_chunk_test2') ORDER BY c.id; chunk_id | hypertable_id | chunk_schema | chunk_table | range_start | range_end ----------+---------------+-----------------------+-------------------+-------------+----------- 2 | 1 | _timescaledb_internal | _hyper_1_2_chunk | 2 | 3 3 | 1 | _timescaledb_internal | _hyper_1_3_chunk | 3 | 4 4 | 1 | _timescaledb_internal | _hyper_1_4_chunk | 4 | 5 5 | 1 | _timescaledb_internal | _hyper_1_5_chunk | 5 | 6 6 | 1 | _timescaledb_internal | _hyper_1_6_chunk | 6 | 7 8 | 2 | _timescaledb_internal | _hyper_2_8_chunk | 2 | 3 9 | 2 | _timescaledb_internal | _hyper_2_9_chunk | 3 | 4 10 | 2 | _timescaledb_internal | _hyper_2_10_chunk | 4 | 5 11 | 2 | _timescaledb_internal | _hyper_2_11_chunk | 5 | 6 12 | 2 | _timescaledb_internal | _hyper_2_12_chunk | 6 | 7 -- next two calls of show_chunks should give same set of chunks as above when combined SELECT show_chunks('drop_chunk_test1'); show_chunks ---------------------------------------- _timescaledb_internal._hyper_1_2_chunk _timescaledb_internal._hyper_1_3_chunk _timescaledb_internal._hyper_1_4_chunk _timescaledb_internal._hyper_1_5_chunk _timescaledb_internal._hyper_1_6_chunk SELECT * FROM show_chunks('drop_chunk_test2'); show_chunks ----------------------------------------- _timescaledb_internal._hyper_2_8_chunk _timescaledb_internal._hyper_2_9_chunk _timescaledb_internal._hyper_2_10_chunk _timescaledb_internal._hyper_2_11_chunk _timescaledb_internal._hyper_2_12_chunk SELECT * FROM test.relation WHERE schema = '_timescaledb_internal' AND name LIKE '\_hyper%'; schema | name | type | owner -----------------------+-------------------+-------+------------------- _timescaledb_internal | _hyper_1_2_chunk | table | default_perm_user _timescaledb_internal | _hyper_1_3_chunk | table | default_perm_user _timescaledb_internal | _hyper_1_4_chunk | table | default_perm_user _timescaledb_internal | _hyper_1_5_chunk | table | default_perm_user _timescaledb_internal | _hyper_1_6_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_10_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_11_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_12_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_8_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_9_chunk | table | default_perm_user _timescaledb_internal | _hyper_3_14_chunk | table | default_perm_user _timescaledb_internal | _hyper_3_15_chunk | table | default_perm_user _timescaledb_internal | _hyper_3_16_chunk | table | default_perm_user _timescaledb_internal | _hyper_3_17_chunk | table | default_perm_user _timescaledb_internal | _hyper_3_18_chunk | table | default_perm_user -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test1\', older_than => 3)::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test1\', older_than => 3)::NAME' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 1 | 1 SELECT c.id AS chunk_id, c.hypertable_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN dimension_get_time(h.id) time_dimension ON(true) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = time_dimension.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.schema_name = 'public' AND (h.table_name = 'drop_chunk_test1' OR h.table_name = 'drop_chunk_test2') ORDER BY c.id; chunk_id | hypertable_id | chunk_schema | chunk_table | range_start | range_end ----------+---------------+-----------------------+-------------------+-------------+----------- 3 | 1 | _timescaledb_internal | _hyper_1_3_chunk | 3 | 4 4 | 1 | _timescaledb_internal | _hyper_1_4_chunk | 4 | 5 5 | 1 | _timescaledb_internal | _hyper_1_5_chunk | 5 | 6 6 | 1 | _timescaledb_internal | _hyper_1_6_chunk | 6 | 7 8 | 2 | _timescaledb_internal | _hyper_2_8_chunk | 2 | 3 9 | 2 | _timescaledb_internal | _hyper_2_9_chunk | 3 | 4 10 | 2 | _timescaledb_internal | _hyper_2_10_chunk | 4 | 5 11 | 2 | _timescaledb_internal | _hyper_2_11_chunk | 5 | 6 12 | 2 | _timescaledb_internal | _hyper_2_12_chunk | 6 | 7 SELECT * FROM test.relation WHERE schema = '_timescaledb_internal' AND name LIKE '\_hyper%'; schema | name | type | owner -----------------------+-------------------+-------+------------------- _timescaledb_internal | _hyper_1_3_chunk | table | default_perm_user _timescaledb_internal | _hyper_1_4_chunk | table | default_perm_user _timescaledb_internal | _hyper_1_5_chunk | table | default_perm_user _timescaledb_internal | _hyper_1_6_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_10_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_11_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_12_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_8_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_9_chunk | table | default_perm_user _timescaledb_internal | _hyper_3_14_chunk | table | default_perm_user _timescaledb_internal | _hyper_3_15_chunk | table | default_perm_user _timescaledb_internal | _hyper_3_16_chunk | table | default_perm_user _timescaledb_internal | _hyper_3_17_chunk | table | default_perm_user _timescaledb_internal | _hyper_3_18_chunk | table | default_perm_user -- next two calls of show_chunks should give same set of chunks as above when combined SELECT show_chunks('drop_chunk_test1'); show_chunks ---------------------------------------- _timescaledb_internal._hyper_1_3_chunk _timescaledb_internal._hyper_1_4_chunk _timescaledb_internal._hyper_1_5_chunk _timescaledb_internal._hyper_1_6_chunk SELECT * FROM show_chunks('drop_chunk_test2'); show_chunks ----------------------------------------- _timescaledb_internal._hyper_2_8_chunk _timescaledb_internal._hyper_2_9_chunk _timescaledb_internal._hyper_2_10_chunk _timescaledb_internal._hyper_2_11_chunk _timescaledb_internal._hyper_2_12_chunk -- 2,147,483,647 is the largest int so this tests that BIGINTs work -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test3\', older_than => 2147483648)::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test3\', older_than => 2147483648)::NAME' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 5 | 5 SELECT c.id AS chunk_id, c.hypertable_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN dimension_get_time(h.id) time_dimension ON(true) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = time_dimension.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.schema_name = 'public' AND (h.table_name = 'drop_chunk_test1' OR h.table_name = 'drop_chunk_test2' OR h.table_name = 'drop_chunk_test3') ORDER BY c.id; chunk_id | hypertable_id | chunk_schema | chunk_table | range_start | range_end ----------+---------------+-----------------------+-------------------+-------------+----------- 3 | 1 | _timescaledb_internal | _hyper_1_3_chunk | 3 | 4 4 | 1 | _timescaledb_internal | _hyper_1_4_chunk | 4 | 5 5 | 1 | _timescaledb_internal | _hyper_1_5_chunk | 5 | 6 6 | 1 | _timescaledb_internal | _hyper_1_6_chunk | 6 | 7 8 | 2 | _timescaledb_internal | _hyper_2_8_chunk | 2 | 3 9 | 2 | _timescaledb_internal | _hyper_2_9_chunk | 3 | 4 10 | 2 | _timescaledb_internal | _hyper_2_10_chunk | 4 | 5 11 | 2 | _timescaledb_internal | _hyper_2_11_chunk | 5 | 6 12 | 2 | _timescaledb_internal | _hyper_2_12_chunk | 6 | 7 SELECT * FROM test.relation WHERE schema = '_timescaledb_internal' AND name LIKE '\_hyper%'; schema | name | type | owner -----------------------+-------------------+-------+------------------- _timescaledb_internal | _hyper_1_3_chunk | table | default_perm_user _timescaledb_internal | _hyper_1_4_chunk | table | default_perm_user _timescaledb_internal | _hyper_1_5_chunk | table | default_perm_user _timescaledb_internal | _hyper_1_6_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_10_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_11_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_12_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_8_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_9_chunk | table | default_perm_user \set ON_ERROR_STOP 0 -- should error because no hypertable SELECT drop_chunks('drop_chunk_test4', older_than => 5); ERROR: relation "drop_chunk_test4" does not exist at character 20 SELECT show_chunks('drop_chunk_test4'); ERROR: relation "drop_chunk_test4" does not exist at character 20 SELECT show_chunks('drop_chunk_test4', 5); ERROR: relation "drop_chunk_test4" does not exist at character 20 \set ON_ERROR_STOP 1 DROP TABLE _timescaledb_internal._hyper_1_6_chunk; SELECT c.id AS chunk_id, c.hypertable_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN dimension_get_time(h.id) time_dimension ON(true) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = time_dimension.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.schema_name = 'public' AND (h.table_name = 'drop_chunk_test1' OR h.table_name = 'drop_chunk_test2') ORDER BY c.id; chunk_id | hypertable_id | chunk_schema | chunk_table | range_start | range_end ----------+---------------+-----------------------+-------------------+-------------+----------- 3 | 1 | _timescaledb_internal | _hyper_1_3_chunk | 3 | 4 4 | 1 | _timescaledb_internal | _hyper_1_4_chunk | 4 | 5 5 | 1 | _timescaledb_internal | _hyper_1_5_chunk | 5 | 6 8 | 2 | _timescaledb_internal | _hyper_2_8_chunk | 2 | 3 9 | 2 | _timescaledb_internal | _hyper_2_9_chunk | 3 | 4 10 | 2 | _timescaledb_internal | _hyper_2_10_chunk | 4 | 5 11 | 2 | _timescaledb_internal | _hyper_2_11_chunk | 5 | 6 12 | 2 | _timescaledb_internal | _hyper_2_12_chunk | 6 | 7 SELECT * FROM test.relation WHERE schema = '_timescaledb_internal' AND name LIKE '\_hyper%'; schema | name | type | owner -----------------------+-------------------+-------+------------------- _timescaledb_internal | _hyper_1_3_chunk | table | default_perm_user _timescaledb_internal | _hyper_1_4_chunk | table | default_perm_user _timescaledb_internal | _hyper_1_5_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_10_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_11_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_12_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_8_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_9_chunk | table | default_perm_user -- newer_than tests -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test1\', newer_than=>5)::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test1\', newer_than=>5, verbose => true)::NAME' \set ECHO errors psql:include/query_result_test_equal.sql:16: INFO: dropping chunk _timescaledb_internal._hyper_1_5_chunk Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 1 | 1 SELECT c.id AS chunk_id, c.hypertable_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN dimension_get_time(h.id) time_dimension ON(true) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = time_dimension.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.schema_name = 'public' AND (h.table_name = 'drop_chunk_test1') ORDER BY c.id; chunk_id | hypertable_id | chunk_schema | chunk_table | range_start | range_end ----------+---------------+-----------------------+------------------+-------------+----------- 3 | 1 | _timescaledb_internal | _hyper_1_3_chunk | 3 | 4 4 | 1 | _timescaledb_internal | _hyper_1_4_chunk | 4 | 5 SELECT show_chunks('drop_chunk_test1'); show_chunks ---------------------------------------- _timescaledb_internal._hyper_1_3_chunk _timescaledb_internal._hyper_1_4_chunk SELECT * FROM test.relation WHERE schema = '_timescaledb_internal' AND name LIKE '_hyper%'; schema | name | type | owner -----------------------+-------------------+-------+------------------- _timescaledb_internal | _hyper_1_3_chunk | table | default_perm_user _timescaledb_internal | _hyper_1_4_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_10_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_11_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_12_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_8_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_9_chunk | table | default_perm_user -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test1\', older_than=>4, newer_than=>3)::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test1\', older_than=>4, newer_than=>3)::NAME' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 1 | 1 SELECT c.id AS chunk_id, c.hypertable_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN dimension_get_time(h.id) time_dimension ON(true) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = time_dimension.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.schema_name = 'public' AND (h.table_name = 'drop_chunk_test1') ORDER BY c.id; chunk_id | hypertable_id | chunk_schema | chunk_table | range_start | range_end ----------+---------------+-----------------------+------------------+-------------+----------- 4 | 1 | _timescaledb_internal | _hyper_1_4_chunk | 4 | 5 -- the call of show_chunks should give same set of chunks as above SELECT show_chunks('drop_chunk_test1'); show_chunks ---------------------------------------- _timescaledb_internal._hyper_1_4_chunk SELECT c.id AS chunk_id, c.hypertable_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN dimension_get_time(h.id) time_dimension ON(true) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = time_dimension.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.schema_name = 'public' ORDER BY c.id; chunk_id | hypertable_id | chunk_schema | chunk_table | range_start | range_end ----------+---------------+-----------------------+-------------------+-------------+----------- 4 | 1 | _timescaledb_internal | _hyper_1_4_chunk | 4 | 5 8 | 2 | _timescaledb_internal | _hyper_2_8_chunk | 2 | 3 9 | 2 | _timescaledb_internal | _hyper_2_9_chunk | 3 | 4 10 | 2 | _timescaledb_internal | _hyper_2_10_chunk | 4 | 5 11 | 2 | _timescaledb_internal | _hyper_2_11_chunk | 5 | 6 12 | 2 | _timescaledb_internal | _hyper_2_12_chunk | 6 | 7 -- We support show/drop chunks using timestamps/interval even with integer partitioning -- the chunk creation time gets used for these. But we need to use "created_before, created_after" -- for these \set ON_ERROR_STOP 0 SELECT show_chunks('drop_chunk_test3', older_than => now()); ERROR: invalid time argument type "timestamp with time zone" SELECT show_chunks('drop_chunk_test2', older_than => now()); ERROR: invalid time argument type "timestamp with time zone" SELECT show_chunks('drop_chunk_test1', newer_than => INTERVAL '15 minutes'); ERROR: invalid time argument type "interval" SELECT show_chunks('drop_chunk_test1', older_than => now(), newer_than => INTERVAL '15 minutes'); ERROR: invalid time argument type "timestamp with time zone" SELECT drop_chunks('drop_chunk_test1', older_than => now()); ERROR: invalid time argument type "timestamp with time zone" -- mix of older_than/newer_than and created_after/created_before doesn't work SELECT show_chunks('drop_chunk_test1', older_than => now(), created_after => INTERVAL '15 minutes'); ERROR: invalid time argument type "timestamp with time zone" SELECT show_chunks('drop_chunk_test1', created_before => now(), newer_than => INTERVAL '15 minutes'); ERROR: invalid time argument type "interval" \set ON_ERROR_STOP 1 SELECT show_chunks('drop_chunk_test3', created_before => now() + INTERVAL '1 hour'); show_chunks ------------- SELECT show_chunks('drop_chunk_test2', created_before => now() + INTERVAL '1 hour'); show_chunks ----------------------------------------- _timescaledb_internal._hyper_2_8_chunk _timescaledb_internal._hyper_2_9_chunk _timescaledb_internal._hyper_2_10_chunk _timescaledb_internal._hyper_2_11_chunk _timescaledb_internal._hyper_2_12_chunk SELECT show_chunks('drop_chunk_test1', created_after => INTERVAL '15 minutes'); show_chunks ---------------------------------------- _timescaledb_internal._hyper_1_4_chunk SELECT show_chunks('drop_chunk_test1', created_before => now() + INTERVAL '1 hour', created_after => INTERVAL '1 hour'); show_chunks ---------------------------------------- _timescaledb_internal._hyper_1_4_chunk SELECT drop_chunks('drop_chunk_test1', created_before => now() + INTERVAL '1 hour'); drop_chunks ---------------------------------------- _timescaledb_internal._hyper_1_4_chunk SELECT drop_chunks(format('%1$I.%2$I', schema_name, table_name)::regclass, older_than => 5, newer_than => 4) FROM _timescaledb_catalog.hypertable WHERE schema_name = 'public'; drop_chunks ----------------------------------------- _timescaledb_internal._hyper_2_10_chunk CREATE TABLE PUBLIC.drop_chunk_test_ts(time timestamp, temp float8, device_id text); SELECT create_hypertable('public.drop_chunk_test_ts', 'time', chunk_time_interval => interval '1 minute', create_default_indexes=>false); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable --------------------------------- (4,public,drop_chunk_test_ts,t) CREATE TABLE PUBLIC.drop_chunk_test_tstz(time timestamptz, temp float8, device_id text); SELECT create_hypertable('public.drop_chunk_test_tstz', 'time', chunk_time_interval => interval '1 minute', create_default_indexes=>false); create_hypertable ----------------------------------- (5,public,drop_chunk_test_tstz,t) SET timezone = '+1'; INSERT INTO PUBLIC.drop_chunk_test_ts VALUES(now()-INTERVAL '5 minutes', 1.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test_ts VALUES(now()+INTERVAL '5 minutes', 1.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test_tstz VALUES(now()-INTERVAL '5 minutes', 1.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test_tstz VALUES(now()+INTERVAL '5 minutes', 1.0, 'dev1'); SELECT * FROM test.show_subtables('drop_chunk_test_ts'); Child | Tablespace -----------------------------------------+------------ _timescaledb_internal._hyper_4_19_chunk | _timescaledb_internal._hyper_4_20_chunk | SELECT * FROM test.show_subtables('drop_chunk_test_tstz'); Child | Tablespace -----------------------------------------+------------ _timescaledb_internal._hyper_5_21_chunk | _timescaledb_internal._hyper_5_22_chunk | BEGIN; SELECT show_chunks('drop_chunk_test_ts'); show_chunks ----------------------------------------- _timescaledb_internal._hyper_4_19_chunk _timescaledb_internal._hyper_4_20_chunk SELECT show_chunks('drop_chunk_test_ts', now()::timestamp-interval '1 minute'); show_chunks ----------------------------------------- _timescaledb_internal._hyper_4_19_chunk -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test_ts\', newer_than => interval \'1 minute\')::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_ts\', newer_than => interval \'1 minute\')::NAME' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 1 | 1 \set QUERY1 'SELECT show_chunks(\'drop_chunk_test_ts\', older_than => interval \'6 minute\')::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_ts\', older_than => interval \'6 minute\')::NAME' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 0 | 0 SELECT * FROM test.show_subtables('drop_chunk_test_ts'); Child | Tablespace -----------------------------------------+------------ _timescaledb_internal._hyper_4_19_chunk | \set QUERY1 'SELECT show_chunks(\'drop_chunk_test_ts\', older_than => interval \'1 minute\')::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_ts\', interval \'1 minute\')::NAME' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 1 | 1 SELECT * FROM test.show_subtables('drop_chunk_test_ts'); Child | Tablespace -------+------------ SELECT show_chunks('drop_chunk_test_tstz'); show_chunks ----------------------------------------- _timescaledb_internal._hyper_5_21_chunk _timescaledb_internal._hyper_5_22_chunk SELECT show_chunks('drop_chunk_test_tstz', older_than => now() - interval '1 minute', newer_than => now() - interval '6 minute'); show_chunks ----------------------------------------- _timescaledb_internal._hyper_5_21_chunk SELECT show_chunks('drop_chunk_test_tstz', newer_than => now() - interval '1 minute'); show_chunks ----------------------------------------- _timescaledb_internal._hyper_5_22_chunk SELECT show_chunks('drop_chunk_test_tstz', older_than => now() - interval '1 minute'); show_chunks ----------------------------------------- _timescaledb_internal._hyper_5_21_chunk \set QUERY1 'SELECT show_chunks(older_than => interval \'1 minute\', relation => \'drop_chunk_test_tstz\')::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_tstz\', interval \'1 minute\')::NAME' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 1 | 1 SELECT * FROM test.show_subtables('drop_chunk_test_tstz'); Child | Tablespace -----------------------------------------+------------ _timescaledb_internal._hyper_5_22_chunk | ROLLBACK; BEGIN; -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test_ts\', newer_than => interval \'6 minute\')::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_ts\', newer_than => interval \'6 minute\')::NAME' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 2 | 2 SELECT * FROM test.show_subtables('drop_chunk_test_ts'); Child | Tablespace -------+------------ ROLLBACK; BEGIN; -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test_ts\', older_than => interval \'1 minute\')::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_ts\', older_than => interval \'1 minute\')::NAME' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 1 | 1 SELECT * FROM test.show_subtables('drop_chunk_test_ts'); Child | Tablespace -----------------------------------------+------------ _timescaledb_internal._hyper_4_20_chunk | \set QUERY1 'SELECT show_chunks(\'drop_chunk_test_tstz\', older_than => interval \'1 minute\')::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_tstz\', older_than => interval \'1 minute\')::NAME' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 1 | 1 SELECT * FROM test.show_subtables('drop_chunk_test_tstz'); Child | Tablespace -----------------------------------------+------------ _timescaledb_internal._hyper_5_22_chunk | ROLLBACK; BEGIN; -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test_ts\', older_than => now()::timestamp-interval \'1 minute\')::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_ts\', older_than => now()::timestamp-interval \'1 minute\')::NAME' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 1 | 1 SELECT * FROM test.show_subtables('drop_chunk_test_ts'); Child | Tablespace -----------------------------------------+------------ _timescaledb_internal._hyper_4_20_chunk | \set QUERY1 'SELECT show_chunks(\'drop_chunk_test_tstz\', older_than => now()-interval \'1 minute\')::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_tstz\', older_than => now()-interval \'1 minute\')::NAME' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 1 | 1 SELECT * FROM test.show_subtables('drop_chunk_test_tstz'); Child | Tablespace -----------------------------------------+------------ _timescaledb_internal._hyper_5_22_chunk | ROLLBACK; SELECT * FROM test.relation WHERE schema = '_timescaledb_internal' AND name LIKE '\_hyper%'; schema | name | type | owner -----------------------+-------------------+-------+------------------- _timescaledb_internal | _hyper_2_11_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_12_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_8_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_9_chunk | table | default_perm_user _timescaledb_internal | _hyper_4_19_chunk | table | default_perm_user _timescaledb_internal | _hyper_4_20_chunk | table | default_perm_user _timescaledb_internal | _hyper_5_21_chunk | table | default_perm_user _timescaledb_internal | _hyper_5_22_chunk | table | default_perm_user \set ON_ERROR_STOP 0 SELECT drop_chunks(interval '1 minute'); ERROR: function drop_chunks(interval) does not exist at character 8 SELECT drop_chunks('drop_chunk_test_ts', (now()-interval '1 minute')); ERROR: invalid time argument type "timestamp with time zone" SELECT drop_chunks('drop_chunk_test3', verbose => true); ERROR: invalid time range for dropping chunks SELECT drop_chunks('drop_chunk_test3', interval '1 minute'); ERROR: invalid time argument type "interval" \set ON_ERROR_STOP 1 -- Interval boundary for INTEGER type columns. It uses chunk creation -- time to identify the affected chunks. SELECT drop_chunks('drop_chunk_test3', created_after => interval '1 minute'); drop_chunks ------------- SELECT * FROM test.relation WHERE schema = '_timescaledb_internal' AND name LIKE '\_hyper%'; schema | name | type | owner -----------------------+-------------------+-------+------------------- _timescaledb_internal | _hyper_2_11_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_12_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_8_chunk | table | default_perm_user _timescaledb_internal | _hyper_2_9_chunk | table | default_perm_user _timescaledb_internal | _hyper_4_19_chunk | table | default_perm_user _timescaledb_internal | _hyper_4_20_chunk | table | default_perm_user _timescaledb_internal | _hyper_5_21_chunk | table | default_perm_user _timescaledb_internal | _hyper_5_22_chunk | table | default_perm_user CREATE TABLE PUBLIC.drop_chunk_test_date(time date, temp float8, device_id text); SELECT create_hypertable('public.drop_chunk_test_date', 'time', chunk_time_interval => interval '1 day', create_default_indexes=>false); create_hypertable ----------------------------------- (6,public,drop_chunk_test_date,t) SET timezone = '+100'; INSERT INTO PUBLIC.drop_chunk_test_date VALUES(now()-INTERVAL '2 day', 1.0, 'dev1'); BEGIN; -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test_date\', older_than => interval \'1 day\')::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_date\', older_than => interval \'1 day\')::NAME' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 1 | 1 SELECT * FROM test.show_subtables('drop_chunk_test_date'); Child | Tablespace -------+------------ ROLLBACK; BEGIN; -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test_date\', older_than => (now()-interval \'1 day\')::date)::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_date\', older_than => (now()-interval \'1 day\')::date)::NAME' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 1 | 1 SELECT * FROM test.show_subtables('drop_chunk_test_date'); Child | Tablespace -------+------------ ROLLBACK; SET timezone TO '-5'; CREATE TABLE chunk_id_from_relid_test(time bigint, temp float8, device_id int); SELECT hypertable_id FROM create_hypertable('chunk_id_from_relid_test', 'time', chunk_time_interval => 10) \gset INSERT INTO chunk_id_from_relid_test VALUES (0, 1.1, 0), (0, 1.3, 11), (12, 2.0, 0), (12, 0.1, 11); SELECT _timescaledb_functions.chunk_id_from_relid(tableoid) FROM chunk_id_from_relid_test; chunk_id_from_relid --------------------- 24 24 25 25 DROP TABLE chunk_id_from_relid_test; CREATE TABLE chunk_id_from_relid_test(time bigint, temp float8, device_id int); SELECT hypertable_id FROM create_hypertable('chunk_id_from_relid_test', 'time', chunk_time_interval => 10, partitioning_column => 'device_id', number_partitions => 3) \gset INSERT INTO chunk_id_from_relid_test VALUES (0, 1.1, 2), (0, 1.3, 11), (12, 2.0, 2), (12, 0.1, 11); SELECT _timescaledb_functions.chunk_id_from_relid(tableoid) FROM chunk_id_from_relid_test; chunk_id_from_relid --------------------- 26 27 28 29 \set ON_ERROR_STOP 0 SELECT _timescaledb_functions.chunk_id_from_relid('pg_type'::regclass); ERROR: chunk not found SELECT _timescaledb_functions.chunk_id_from_relid('chunk_id_from_relid_test'::regclass); ERROR: chunk not found -- test drop/show_chunks on custom partition types CREATE FUNCTION extract_time(a jsonb) RETURNS TIMESTAMPTZ LANGUAGE SQL AS $$ SELECT (a->>'time')::TIMESTAMPTZ $$ IMMUTABLE; CREATE TABLE test_weird_type(a jsonb); SELECT create_hypertable('test_weird_type', 'a', time_partitioning_func=>'extract_time'::regproc, chunk_time_interval=>'2 hours'::interval); create_hypertable ------------------------------ (9,public,test_weird_type,t) INSERT INTO test_weird_type VALUES ('{"time":"2019/06/06 1:00+0"}'), ('{"time":"2019/06/06 5:00+0"}'); SELECT * FROM test.show_subtables('test_weird_type'); Child | Tablespace -----------------------------------------+------------ _timescaledb_internal._hyper_9_30_chunk | _timescaledb_internal._hyper_9_31_chunk | SELECT show_chunks('test_weird_type', older_than=>'2019/06/06 4:00+0'::TIMESTAMPTZ); show_chunks ----------------------------------------- _timescaledb_internal._hyper_9_30_chunk SELECT show_chunks('test_weird_type', older_than=>'2019/06/06 10:00+0'::TIMESTAMPTZ); show_chunks ----------------------------------------- _timescaledb_internal._hyper_9_30_chunk _timescaledb_internal._hyper_9_31_chunk -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'test_weird_type\', older_than => \'2019/06/06 5:00+0\'::TIMESTAMPTZ)::NAME' \set QUERY2 'SELECT drop_chunks(\'test_weird_type\', older_than => \'2019/06/06 5:00+0\'::TIMESTAMPTZ)::NAME' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 1 | 1 SELECT * FROM test.show_subtables('test_weird_type'); Child | Tablespace -----------------------------------------+------------ _timescaledb_internal._hyper_9_31_chunk | SELECT show_chunks('test_weird_type', older_than=>'2019/06/06 4:00+0'::TIMESTAMPTZ); show_chunks ------------- SELECT show_chunks('test_weird_type', older_than=>'2019/06/06 10:00+0'::TIMESTAMPTZ); show_chunks ----------------------------------------- _timescaledb_internal._hyper_9_31_chunk -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'test_weird_type\', older_than => \'2019/06/06 6:00+0\'::TIMESTAMPTZ)::NAME' \set QUERY2 'SELECT drop_chunks(\'test_weird_type\', older_than => \'2019/06/06 6:00+0\'::TIMESTAMPTZ)::NAME' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 1 | 1 SELECT * FROM test.show_subtables('test_weird_type'); Child | Tablespace -------+------------ SELECT show_chunks('test_weird_type', older_than=>'2019/06/06 10:00+0'::TIMESTAMPTZ); show_chunks ------------- DROP TABLE test_weird_type; CREATE FUNCTION extract_int_time(a jsonb) RETURNS BIGINT LANGUAGE SQL AS $$ SELECT (a->>'time')::BIGINT $$ IMMUTABLE; CREATE TABLE test_weird_type_i(a jsonb); SELECT create_hypertable('test_weird_type_i', 'a', time_partitioning_func=>'extract_int_time'::regproc, chunk_time_interval=>5); create_hypertable --------------------------------- (10,public,test_weird_type_i,t) INSERT INTO test_weird_type_i VALUES ('{"time":"0"}'), ('{"time":"5"}'); SELECT * FROM test.show_subtables('test_weird_type_i'); Child | Tablespace ------------------------------------------+------------ _timescaledb_internal._hyper_10_32_chunk | _timescaledb_internal._hyper_10_33_chunk | SELECT show_chunks('test_weird_type_i', older_than=>5); show_chunks ------------------------------------------ _timescaledb_internal._hyper_10_32_chunk SELECT show_chunks('test_weird_type_i', older_than=>10); show_chunks ------------------------------------------ _timescaledb_internal._hyper_10_32_chunk _timescaledb_internal._hyper_10_33_chunk -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'test_weird_type_i\', older_than=>5)::NAME' \set QUERY2 'SELECT drop_chunks(\'test_weird_type_i\', older_than => 5)::NAME' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 1 | 1 SELECT * FROM test.show_subtables('test_weird_type_i'); Child | Tablespace ------------------------------------------+------------ _timescaledb_internal._hyper_10_33_chunk | SELECT show_chunks('test_weird_type_i', older_than=>5); show_chunks ------------- SELECT show_chunks('test_weird_type_i', older_than=>10); show_chunks ------------------------------------------ _timescaledb_internal._hyper_10_33_chunk -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'test_weird_type_i\', older_than=>10)::NAME' \set QUERY2 'SELECT drop_chunks(\'test_weird_type_i\', older_than => 10)::NAME' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 1 | 1 SELECT * FROM test.show_subtables('test_weird_type_i'); Child | Tablespace -------+------------ SELECT show_chunks('test_weird_type_i', older_than=>10); show_chunks ------------- DROP TABLE test_weird_type_i CASCADE; \c :TEST_DBNAME :ROLE_SUPERUSER ALTER TABLE drop_chunk_test2 OWNER TO :ROLE_DEFAULT_PERM_USER_2; --drop chunks 3 will have a chunk we a dependent object (a view) --we create the dependent object now INSERT INTO PUBLIC.drop_chunk_test3 VALUES(1, 1.0, 'dev1'); SELECT c.schema_name as chunk_schema, c.table_name as chunk_table FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) WHERE h.schema_name = 'public' AND h.table_name = 'drop_chunk_test3' ORDER BY c.id \gset create view dependent_view as SELECT * FROM :"chunk_schema".:"chunk_table"; create view dependent_view2 as SELECT * FROM :"chunk_schema".:"chunk_table"; ALTER TABLE drop_chunk_test3 OWNER TO :ROLE_DEFAULT_PERM_USER_2; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 \set ON_ERROR_STOP 0 SELECT drop_chunks('drop_chunk_test1', older_than=>4, newer_than=>3); ERROR: must be owner of hypertable "drop_chunk_test1" --works with modified owner tables -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test2\', older_than=>4, newer_than=>3)::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test2\', older_than=>4, newer_than=>3)::NAME' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 1 | 1 \set VERBOSITY default --this fails because there are dependent objects SELECT drop_chunks('drop_chunk_test3', older_than=>100); ERROR: cannot drop table _timescaledb_internal._hyper_3_34_chunk because other objects depend on it DETAIL: view dependent_view depends on table _timescaledb_internal._hyper_3_34_chunk view dependent_view2 depends on table _timescaledb_internal._hyper_3_34_chunk HINT: Use DROP ... to drop the dependent objects. \set VERBOSITY terse \c :TEST_DBNAME :ROLE_SUPERUSER DROP VIEW dependent_view; DROP VIEW dependent_view2; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 \set ON_ERROR_STOP 1 --drop chunks from hypertable with same name in different schema -- order of schema in search_path matters -- \c :TEST_DBNAME :ROLE_SUPERUSER drop table chunk_id_from_relid_test; drop table drop_chunk_test1; drop table drop_chunk_test2; drop table drop_chunk_test3; CREATE SCHEMA try_schema; CREATE SCHEMA test1; CREATE SCHEMA test2; CREATE SCHEMA test3; GRANT CREATE ON SCHEMA try_schema, test1, test2, test3 TO :ROLE_DEFAULT_PERM_USER; GRANT USAGE ON SCHEMA try_schema, test1, test2, test3 TO :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE TABLE try_schema.drop_chunk_test_date(time date, temp float8, device_id text); SELECT create_hypertable('try_schema.drop_chunk_test_date', 'time', chunk_time_interval => interval '1 day', create_default_indexes=>false); create_hypertable ---------------------------------------- (11,try_schema,drop_chunk_test_date,t) INSERT INTO public.drop_chunk_test_date VALUES( '2020-01-10', 100, 'hello'); INSERT INTO try_schema.drop_chunk_test_date VALUES( '2020-01-10', 100, 'hello'); set search_path to try_schema, test1, test2, test3, public; SELECT show_chunks('public.drop_chunk_test_date', older_than=>'1 day'::interval); show_chunks ----------------------------------------- _timescaledb_internal._hyper_6_35_chunk SELECT show_chunks('try_schema.drop_chunk_test_date', older_than=>'1 day'::interval); show_chunks ------------------------------------------ _timescaledb_internal._hyper_11_36_chunk SELECT drop_chunks('drop_chunk_test_date', older_than=> '1 day'::interval); drop_chunks ------------------------------------------ _timescaledb_internal._hyper_11_36_chunk -- test drop chunks across two tables within the same schema CREATE TABLE test1.hyper1 (time bigint, temp float); CREATE TABLE test1.hyper2 (time bigint, temp float); SELECT create_hypertable('test1.hyper1', 'time', chunk_time_interval => 10); create_hypertable --------------------- (12,test1,hyper1,t) SELECT create_hypertable('test1.hyper2', 'time', chunk_time_interval => 10); create_hypertable --------------------- (13,test1,hyper2,t) INSERT INTO test1.hyper1 VALUES (10, 0.5); INSERT INTO test1.hyper2 VALUES (10, 0.7); SELECT show_chunks('test1.hyper1'); show_chunks ------------------------------------------ _timescaledb_internal._hyper_12_37_chunk SELECT show_chunks('test1.hyper2'); show_chunks ------------------------------------------ _timescaledb_internal._hyper_13_38_chunk -- test drop chunks for given table name across all schemas CREATE TABLE test2.hyperx (time bigint, temp float); CREATE TABLE test3.hyperx (time bigint, temp float); SELECT create_hypertable('test2.hyperx', 'time', chunk_time_interval => 10); create_hypertable --------------------- (14,test2,hyperx,t) SELECT create_hypertable('test3.hyperx', 'time', chunk_time_interval => 10); create_hypertable --------------------- (15,test3,hyperx,t) INSERT INTO test2.hyperx VALUES (10, 0.5); INSERT INTO test3.hyperx VALUES (10, 0.7); SELECT show_chunks('test2.hyperx'); show_chunks ------------------------------------------ _timescaledb_internal._hyper_14_39_chunk SELECT show_chunks('test3.hyperx'); show_chunks ------------------------------------------ _timescaledb_internal._hyper_15_40_chunk -- This will only drop from one of the tables since the one that is -- first in the search path will hide the other one. SELECT drop_chunks('hyperx', older_than => 100); drop_chunks ------------------------------------------ _timescaledb_internal._hyper_14_39_chunk SELECT show_chunks('test2.hyperx'); show_chunks ------------- SELECT show_chunks('test3.hyperx'); show_chunks ------------------------------------------ _timescaledb_internal._hyper_15_40_chunk -- Check CTAS behavior when internal ALTER TABLE gets fired CREATE TABLE PUBLIC.drop_chunk_test4(time bigint, temp float8, device_id text); CREATE TABLE drop_chunks_table_id AS SELECT hypertable_id FROM create_hypertable('public.drop_chunk_test4', 'time', chunk_time_interval => 1); -- TEST for internal api that drops a single chunk -- this drops the table and removes entry from the catalog. -- does not affect any materialized cagg data INSERT INTO test1.hyper1 VALUES (20, 0.5); SELECT chunk_schema as "CHSCHEMA", chunk_name as "CHNAME" FROM timescaledb_information.chunks WHERE hypertable_name = 'hyper1' and hypertable_schema = 'test1' ORDER BY chunk_name ; CHSCHEMA | CHNAME -----------------------+-------------------- _timescaledb_internal | _hyper_12_37_chunk _timescaledb_internal | _hyper_12_41_chunk --drop one of the chunks SELECT chunk_schema || '.' || chunk_name as "CHNAME" FROM timescaledb_information.chunks WHERE hypertable_name = 'hyper1' and hypertable_schema = 'test1' ORDER BY chunk_name LIMIT 1 \gset SELECT _timescaledb_functions.drop_chunk(:'CHNAME'); drop_chunk ------------ t SELECT chunk_schema as "CHSCHEMA", chunk_name as "CHNAME" FROM timescaledb_information.chunks WHERE hypertable_name = 'hyper1' and hypertable_schema = 'test1' ORDER BY chunk_name ; CHSCHEMA | CHNAME -----------------------+-------------------- _timescaledb_internal | _hyper_12_41_chunk -- "created_before/after" can be used with time partitioning in drop/show chunks SELECT show_chunks('drop_chunk_test_tstz', created_before => now() - INTERVAL '1 hour'); show_chunks ------------- SELECT drop_chunks('drop_chunk_test_tstz', created_before => now() + INTERVAL '1 hour'); drop_chunks ----------------------------------------- _timescaledb_internal._hyper_5_21_chunk _timescaledb_internal._hyper_5_22_chunk SELECT show_chunks('drop_chunk_test_ts'); show_chunks ----------------------------------------- _timescaledb_internal._hyper_4_19_chunk _timescaledb_internal._hyper_4_20_chunk -- "created_before/after" accept timestamptz even though partitioning col is just -- timestamp SELECT show_chunks('drop_chunk_test_ts', created_after => now() - INTERVAL '1 hour', created_before => now()); show_chunks ----------------------------------------- _timescaledb_internal._hyper_4_19_chunk _timescaledb_internal._hyper_4_20_chunk SELECT drop_chunks('drop_chunk_test_ts', created_after => INTERVAL '1 hour', created_before => now()); drop_chunks ----------------------------------------- _timescaledb_internal._hyper_4_19_chunk _timescaledb_internal._hyper_4_20_chunk -- Test views on top of hypertables CREATE TABLE view_test (project_id INT, ts TIMESTAMPTZ NOT NULL); SELECT create_hypertable('view_test', by_range('ts', INTERVAL '1 day')); create_hypertable ------------------- (17,t) -- exactly one partition per project_id SELECT * FROM add_dimension('view_test', 'project_id', chunk_time_interval => 1); -- exactly one partition per project; works for *integer* types dimension_id | schema_name | table_name | column_name | created --------------+-------------+------------+-------------+--------- 22 | try_schema | view_test | project_id | t INSERT INTO view_test (project_id, ts) SELECT g % 25 + 1 AS project_id, i.ts + (g * interval '1 week') / i.total AS ts FROM (SELECT timestamptz '2024-01-01 00:00:00+0', 600) i(ts, total), generate_series(1, i.total) g; -- Create a view on top of this hypertable CREATE VIEW test_view_part_few AS SELECT project_id, ts FROM view_test WHERE project_id = ANY (ARRAY[5, 10, 15]); -- Complicated query on a view involving a range check and a sort SELECT * FROM test_view_part_few WHERE ts BETWEEN '2024-01-04 00:00:00+00'AND '2024-01-05 00:00:00' ORDER BY ts LIMIT 1000; project_id | ts ------------+------------------------------ 10 | Wed Jan 03 16:31:12 2024 PST 15 | Wed Jan 03 17:55:12 2024 PST 5 | Wed Jan 03 22:07:12 2024 PST 10 | Wed Jan 03 23:31:12 2024 PST 15 | Thu Jan 04 00:55:12 2024 PST 5 | Thu Jan 04 05:07:12 2024 PST 10 | Thu Jan 04 06:31:12 2024 PST 15 | Thu Jan 04 07:55:12 2024 PST 5 | Thu Jan 04 12:07:12 2024 PST 10 | Thu Jan 04 13:31:12 2024 PST 15 | Thu Jan 04 14:55:12 2024 PST 5 | Thu Jan 04 19:07:12 2024 PST 10 | Thu Jan 04 20:31:12 2024 PST 15 | Thu Jan 04 21:55:12 2024 PST -- Test chunk_status_text function CREATE TABLE chunk_status_test(time timestamptz) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.columnstore=false); INSERT INTO chunk_status_test VALUES ('2025-01-01'),('2025-02-01'),('2025-03-01'); SELECT _timescaledb_functions.chunk_status_text(i) FROM generate_series(0,15) i; chunk_status_text --------------------------------------- {} {COMPRESSED} {UNORDERED} {COMPRESSED,UNORDERED} {FROZEN} {COMPRESSED,FROZEN} {UNORDERED,FROZEN} {COMPRESSED,UNORDERED,FROZEN} {PARTIAL} {COMPRESSED,PARTIAL} {UNORDERED,PARTIAL} {COMPRESSED,UNORDERED,PARTIAL} {FROZEN,PARTIAL} {COMPRESSED,FROZEN,PARTIAL} {UNORDERED,FROZEN,PARTIAL} {COMPRESSED,UNORDERED,FROZEN,PARTIAL} SELECT chunk, _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('chunk_status_test') chunk; chunk | chunk_status_text -------------------------------------------+------------------- _timescaledb_internal._hyper_18_218_chunk | {} _timescaledb_internal._hyper_18_219_chunk | {} _timescaledb_internal._hyper_18_220_chunk | {} SELECT _timescaledb_functions.chunk_status_text(NULL::int); chunk_status_text ------------------- SELECT _timescaledb_functions.chunk_status_text(NULL::regclass); chunk_status_text ------------------- \set ON_ERROR_STOP 0 SELECT _timescaledb_functions.chunk_status_text(-1); ERROR: invalid chunk status -1 SELECT _timescaledb_functions.chunk_status_text(16); ERROR: invalid chunk status 16 SELECT _timescaledb_functions.chunk_status_text(1000); ERROR: invalid chunk status 1000 SELECT _timescaledb_functions.chunk_status_text(0::regclass); ERROR: invalid Oid SELECT _timescaledb_functions.chunk_status_text('pg_class'::regclass); ERROR: chunk not found \set ON_ERROR_STOP 1 -- Test that function exists and returns an array type SELECT pg_typeof(_timescaledb_functions.chunk_status_text(0)); pg_typeof ----------- text[] ================================================ FILE: test/expected/chunks.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \unset ECHO ================================================ FILE: test/expected/cluster.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE cluster_test(time timestamptz, temp float, location int); SELECT create_hypertable('cluster_test', 'time', chunk_time_interval => interval '1 day'); create_hypertable --------------------------- (1,public,cluster_test,t) -- Show default indexes SELECT * FROM test.show_indexes('cluster_test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace -----------------------+---------+------+--------+---------+-----------+------------ cluster_test_time_idx | {time} | | f | f | f | -- Create two chunks INSERT INTO cluster_test VALUES ('2017-01-20T09:00:01', 23.4, 1), ('2017-01-21T09:00:01', 21.3, 2); -- Run cluster CLUSTER VERBOSE cluster_test USING cluster_test_time_idx; INFO: clustering "_timescaledb_internal._hyper_1_1_chunk" using index scan on "_hyper_1_1_chunk_cluster_test_time_idx" INFO: clustering "_timescaledb_internal._hyper_1_2_chunk" using index scan on "_hyper_1_2_chunk_cluster_test_time_idx" -- Create a third chunk INSERT INTO cluster_test VALUES ('2017-01-22T09:00:01', 19.5, 3); -- Show clustered indexes SELECT indexrelid::regclass, indisclustered FROM pg_index WHERE indisclustered = true ORDER BY 1; indexrelid | indisclustered --------------------------------------------------------------+---------------- cluster_test_time_idx | t _timescaledb_internal._hyper_1_1_chunk_cluster_test_time_idx | t _timescaledb_internal._hyper_1_2_chunk_cluster_test_time_idx | t -- Reorder just our table CLUSTER VERBOSE cluster_test; INFO: clustering "_timescaledb_internal._hyper_1_1_chunk" using sequential scan and sort INFO: clustering "_timescaledb_internal._hyper_1_2_chunk" using sequential scan and sort INFO: clustering "_timescaledb_internal._hyper_1_3_chunk" using index scan on "_hyper_1_3_chunk_cluster_test_time_idx" -- Show clustered indexes, including new chunk SELECT indexrelid::regclass, indisclustered FROM pg_index WHERE indisclustered = true ORDER BY 1; indexrelid | indisclustered --------------------------------------------------------------+---------------- cluster_test_time_idx | t _timescaledb_internal._hyper_1_1_chunk_cluster_test_time_idx | t _timescaledb_internal._hyper_1_2_chunk_cluster_test_time_idx | t _timescaledb_internal._hyper_1_3_chunk_cluster_test_time_idx | t -- Reorder all tables (although will only be our test table) CLUSTER VERBOSE; INFO: clustering "public.cluster_test" using sequential scan and sort INFO: clustering "_timescaledb_internal._hyper_1_1_chunk" using sequential scan and sort INFO: clustering "_timescaledb_internal._hyper_1_2_chunk" using sequential scan and sort INFO: clustering "_timescaledb_internal._hyper_1_3_chunk" using sequential scan and sort -- Change the clustered index CREATE INDEX ON cluster_test (time, location); CLUSTER VERBOSE cluster_test using cluster_test_time_location_idx; INFO: clustering "_timescaledb_internal._hyper_1_1_chunk" using sequential scan and sort INFO: clustering "_timescaledb_internal._hyper_1_2_chunk" using sequential scan and sort INFO: clustering "_timescaledb_internal._hyper_1_3_chunk" using sequential scan and sort -- Show updated clustered indexes SELECT indexrelid::regclass, indisclustered FROM pg_index WHERE indisclustered = true ORDER BY 1; indexrelid | indisclustered -----------------------------------------------------------------------+---------------- cluster_test_time_location_idx | t _timescaledb_internal._hyper_1_1_chunk_cluster_test_time_location_idx | t _timescaledb_internal._hyper_1_2_chunk_cluster_test_time_location_idx | t _timescaledb_internal._hyper_1_3_chunk_cluster_test_time_location_idx | t --check the setting of cluster indexes on hypertables and chunks ALTER TABLE cluster_test CLUSTER ON cluster_test_time_idx; SELECT indexrelid::regclass, indisclustered FROM pg_index WHERE indisclustered = true ORDER BY 1,2; indexrelid | indisclustered --------------------------------------------------------------+---------------- cluster_test_time_idx | t _timescaledb_internal._hyper_1_1_chunk_cluster_test_time_idx | t _timescaledb_internal._hyper_1_2_chunk_cluster_test_time_idx | t _timescaledb_internal._hyper_1_3_chunk_cluster_test_time_idx | t CLUSTER VERBOSE cluster_test; INFO: clustering "_timescaledb_internal._hyper_1_1_chunk" using sequential scan and sort INFO: clustering "_timescaledb_internal._hyper_1_2_chunk" using sequential scan and sort INFO: clustering "_timescaledb_internal._hyper_1_3_chunk" using sequential scan and sort ALTER TABLE cluster_test SET WITHOUT CLUSTER; SELECT indexrelid::regclass, indisclustered FROM pg_index WHERE indisclustered = true ORDER BY 1,2; indexrelid | indisclustered ------------+---------------- \set ON_ERROR_STOP 0 CLUSTER VERBOSE cluster_test; ERROR: there is no previously clustered index for table "cluster_test" \set ON_ERROR_STOP 1 ALTER TABLE _timescaledb_internal._hyper_1_1_chunk CLUSTER ON _hyper_1_1_chunk_cluster_test_time_idx; SELECT indexrelid::regclass, indisclustered FROM pg_index WHERE indisclustered = true ORDER BY 1,2; indexrelid | indisclustered --------------------------------------------------------------+---------------- _timescaledb_internal._hyper_1_1_chunk_cluster_test_time_idx | t CLUSTER VERBOSE _timescaledb_internal._hyper_1_1_chunk; INFO: clustering "_timescaledb_internal._hyper_1_1_chunk" using sequential scan and sort ALTER TABLE _timescaledb_internal._hyper_1_1_chunk SET WITHOUT CLUSTER; SELECT indexrelid::regclass, indisclustered FROM pg_index WHERE indisclustered = true ORDER BY 1,2; indexrelid | indisclustered ------------+---------------- \set ON_ERROR_STOP 0 CLUSTER VERBOSE _timescaledb_internal._hyper_1_1_chunk; ERROR: there is no previously clustered index for table "_hyper_1_1_chunk" \set ON_ERROR_STOP 1 -- test alter column type on hypertable with clustering CREATE TABLE cluster_alter(time timestamp, id text, val int); CREATE INDEX idstuff ON cluster_alter USING btree (id ASC NULLS LAST, time); SELECT table_name FROM create_hypertable('cluster_alter', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices table_name --------------- cluster_alter INSERT INTO cluster_alter VALUES('2020-01-01', '123', 1); CLUSTER cluster_alter using idstuff; --attempt the alter table ALTER TABLE cluster_alter ALTER COLUMN id TYPE int USING id::int; CLUSTER cluster_alter; CLUSTER cluster_alter using idstuff; ================================================ FILE: test/expected/constraint.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE hyper ( time BIGINT NOT NULL, device_id TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10) ); SELECT * FROM create_hypertable('hyper', 'time', chunk_time_interval => 10); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | hyper | t --check and not-null constraints are inherited through regular inheritance. \set ON_ERROR_STOP 0 INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 9); ERROR: new row for relation "_hyper_1_1_chunk" violates check constraint "hyper_sensor_1_check" INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987700000000000, NULL, 11); ERROR: null value in column "device_id" of relation "_hyper_1_2_chunk" violates not-null constraint ALTER TABLE hyper ALTER COLUMN time DROP NOT NULL; ERROR: cannot drop not-null constraint from a time-partitioned column ALTER TABLE ONLY hyper ALTER COLUMN sensor_1 SET NOT NULL; ERROR: ONLY option not supported on hypertable operations ALTER TABLE ONLY hyper ALTER COLUMN device_id DROP NOT NULL; ERROR: ONLY option not supported on hypertable operations \set ON_ERROR_STOP 1 INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); ALTER TABLE hyper ALTER COLUMN device_id DROP NOT NULL; INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987700000000000, NULL, 11); --make sure validate works \set ON_ERROR_STOP 0 ALTER TABLE hyper ADD CONSTRAINT bad_check_const CHECK (sensor_1 > 100); ERROR: check constraint "bad_check_const" of relation "_hyper_1_3_chunk" is violated by some row \set ON_ERROR_STOP 1 ALTER TABLE hyper ADD CONSTRAINT bad_check_const CHECK (sensor_1 > 100) NOT VALID; \set ON_ERROR_STOP 0 ALTER TABLE hyper VALIDATE CONSTRAINT bad_check_const; ERROR: check constraint "bad_check_const" of relation "_hyper_1_3_chunk" is violated by some row \set ON_ERROR_STOP 1 ----------------------- UNIQUE CONSTRAINTS ------------------ CREATE TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name ( time BIGINT NOT NULL UNIQUE, device_id TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10) ); SELECT * FROM create_hypertable('hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name', 'time', chunk_time_interval => 10); hypertable_id | schema_name | table_name | created ---------------+-------------+-----------------------------------------------------------------+--------- 2 | public | hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name | t INSERT INTO hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); INSERT INTO hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name(time, device_id,sensor_1) VALUES (1257987800000000000, 'dev2', 11); \set ON_ERROR_STOP 0 INSERT INTO hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); ERROR: duplicate key value violates unique constraint "4_1_hyper_unique_with_looooooooooooooooooooooooooooooooooo_time" \set ON_ERROR_STOP 1 -- Show constraints on main tables SELECT * FROM _timescaledb_catalog.chunk_constraint; chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----------+--------------------+-----------------------------------------------------------------+----------------------------------------------------------------- 3 | 3 | constraint_3 | 4 | 4 | constraint_4 | 4 | | 4_1_hyper_unique_with_looooooooooooooooooooooooooooooooooo_time | hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key 5 | 5 | constraint_5 | 5 | | 5_2_hyper_unique_with_looooooooooooooooooooooooooooooooooo_time | hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key SELECT * FROM test.show_constraints('hyper'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated ----------------------+------+------------+-------+-----------------------------+------------+----------+----------- bad_check_const | c | {sensor_1} | - | (sensor_1 > (100)::numeric) | f | f | f hyper_sensor_1_check | c | {sensor_1} | - | (sensor_1 > (10)::numeric) | f | f | t SELECT * FROM test.show_constraints('hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated -----------------------------------------------------------------+------+------------+-----------------------------------------------------------------+----------------------------+------------+----------+----------- hyper_unique_with_looooooooooooooooooooooooooooo_sensor_1_check | c | {sensor_1} | - | (sensor_1 > (10)::numeric) | f | f | t hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key | u | {time} | hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key | | f | f | t --should have unique constraint not just unique index SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_2_4_chunk'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated -----------------------------------------------------------------+------+------------+-----------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+------------+----------+----------- 4_1_hyper_unique_with_looooooooooooooooooooooooooooooooooo_time | u | {time} | _timescaledb_internal."4_1_hyper_unique_with_looooooooooooooooooooooooooooooooooo_time" | | f | f | t constraint_4 | c | {time} | - | (("time" >= '1257987700000000000'::bigint) AND ("time" < '1257987700000000010'::bigint)) | f | f | t hyper_unique_with_looooooooooooooooooooooooooooo_sensor_1_check | c | {sensor_1} | - | (sensor_1 > (10)::numeric) | f | f | t ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name DROP CONSTRAINT hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key; -- The constraint should have been removed from the chunk as well SELECT * FROM _timescaledb_catalog.chunk_constraint; chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----------+--------------------+-----------------+---------------------------- 3 | 3 | constraint_3 | 4 | 4 | constraint_4 | 5 | 5 | constraint_5 | SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_2_4_chunk'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated -----------------------------------------------------------------+------+------------+-------+------------------------------------------------------------------------------------------+------------+----------+----------- constraint_4 | c | {time} | - | (("time" >= '1257987700000000000'::bigint) AND ("time" < '1257987700000000010'::bigint)) | f | f | t hyper_unique_with_looooooooooooooooooooooooooooo_sensor_1_check | c | {sensor_1} | - | (sensor_1 > (10)::numeric) | f | f | t --uniqueness not enforced INSERT INTO hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev3', 11); --shouldn't be able to create constraint \set ON_ERROR_STOP 0 ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name ADD CONSTRAINT hyper_unique_time_key UNIQUE (time); ERROR: could not create unique index "4_3_hyper_unique_time_key" \set ON_ERROR_STOP 1 DELETE FROM hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name WHERE device_id = 'dev3'; -- Try multi-alter table statement with a constraint without a name ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name ADD CHECK (time > 0), ADD UNIQUE (time) DEFERRABLE INITIALLY DEFERRED; \set ON_ERROR_STOP 0 BEGIN; --testing deferred checking. The following row has an error, which will not appear until the commit INSERT INTO hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev3', 11); SELECT 1; ?column? ---------- 1 COMMIT; ERROR: duplicate key value violates unique constraint "4_4_hyper_unique_with_looooooooooooooooooooooooooooooooooo_time" \set ON_ERROR_STOP 1 SELECT * FROM _timescaledb_catalog.chunk_constraint; chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----------+--------------------+-----------------------------------------------------------------+----------------------------------------------------------------- 3 | 3 | constraint_3 | 4 | 4 | constraint_4 | 5 | 5 | constraint_5 | 4 | | 4_4_hyper_unique_with_looooooooooooooooooooooooooooooooooo_time | hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key 5 | | 5_5_hyper_unique_with_looooooooooooooooooooooooooooooooooo_time | hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key SELECT * FROM test.show_constraints('hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated -----------------------------------------------------------------+------+------------+-----------------------------------------------------------------+----------------------------+------------+----------+----------- hyper_unique_with_looooooooooooooooooooooooooooo_sensor_1_check | c | {sensor_1} | - | (sensor_1 > (10)::numeric) | f | f | t hyper_unique_with_looooooooooooooooooooooooooooooooo_time_check | c | {time} | - | ("time" > 0) | f | f | t hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key | u | {time} | hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key | | t | t | t SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_2_4_chunk'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated -----------------------------------------------------------------+------+------------+-----------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+------------+----------+----------- 4_4_hyper_unique_with_looooooooooooooooooooooooooooooooooo_time | u | {time} | _timescaledb_internal."4_4_hyper_unique_with_looooooooooooooooooooooooooooooooooo_time" | | t | t | t constraint_4 | c | {time} | - | (("time" >= '1257987700000000000'::bigint) AND ("time" < '1257987700000000010'::bigint)) | f | f | t hyper_unique_with_looooooooooooooooooooooooooooo_sensor_1_check | c | {sensor_1} | - | (sensor_1 > (10)::numeric) | f | f | t hyper_unique_with_looooooooooooooooooooooooooooooooo_time_check | c | {time} | - | ("time" > 0) | f | f | t ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name DROP CONSTRAINT hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key, DROP CONSTRAINT hyper_unique_with_looooooooooooooooooooooooooooooooo_time_check; SELECT * FROM _timescaledb_catalog.chunk_constraint; chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----------+--------------------+-----------------+---------------------------- 3 | 3 | constraint_3 | 4 | 4 | constraint_4 | 5 | 5 | constraint_5 | SELECT * FROM test.show_constraints('hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated -----------------------------------------------------------------+------+------------+-------+----------------------------+------------+----------+----------- hyper_unique_with_looooooooooooooooooooooooooooo_sensor_1_check | c | {sensor_1} | - | (sensor_1 > (10)::numeric) | f | f | t SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_2_4_chunk'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated -----------------------------------------------------------------+------+------------+-------+------------------------------------------------------------------------------------------+------------+----------+----------- constraint_4 | c | {time} | - | (("time" >= '1257987700000000000'::bigint) AND ("time" < '1257987700000000010'::bigint)) | f | f | t hyper_unique_with_looooooooooooooooooooooooooooo_sensor_1_check | c | {sensor_1} | - | (sensor_1 > (10)::numeric) | f | f | t CREATE UNIQUE INDEX hyper_unique_with_looooooooooooooooooooooooooooooooo_time_idx ON hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name (time); \set ON_ERROR_STOP 0 -- Try adding constraint using existing index ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name ADD CONSTRAINT hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key UNIQUE USING INDEX hyper_unique_with_looooooooooooooooooooooooooooooooo_time_idx; NOTICE: ALTER TABLE / ADD CONSTRAINT USING INDEX will rename index "hyper_unique_with_looooooooooooooooooooooooooooooooo_time_idx" to "hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key" ERROR: hypertables do not support adding a constraint using an existing index \set ON_ERROR_STOP 1 DROP INDEX hyper_unique_with_looooooooooooooooooooooooooooooooo_time_idx; --now can create ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name ADD CONSTRAINT hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key UNIQUE (time); SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_2_4_chunk'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated -----------------------------------------------------------------+------+------------+-----------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------+------------+----------+----------- 4_6_hyper_unique_with_looooooooooooooooooooooooooooooooooo_time | u | {time} | _timescaledb_internal."4_6_hyper_unique_with_looooooooooooooooooooooooooooooooooo_time" | | f | f | t constraint_4 | c | {time} | - | (("time" >= '1257987700000000000'::bigint) AND ("time" < '1257987700000000010'::bigint)) | f | f | t hyper_unique_with_looooooooooooooooooooooooooooo_sensor_1_check | c | {sensor_1} | - | (sensor_1 > (10)::numeric) | f | f | t --test adding constraint with same name to different table -- should fail \set ON_ERROR_STOP 0 ALTER TABLE hyper ADD CONSTRAINT hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key UNIQUE (time); ERROR: relation "hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key" already exists \set ON_ERROR_STOP 1 --uniquness violation fails \set ON_ERROR_STOP 0 INSERT INTO hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); ERROR: duplicate key value violates unique constraint "4_6_hyper_unique_with_looooooooooooooooooooooooooooooooooo_time" \set ON_ERROR_STOP 1 --cannot create unique constraint on non-partition column \set ON_ERROR_STOP 0 ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name ADD CONSTRAINT hyper_unique_invalid UNIQUE (device_id); ERROR: cannot create a unique index without the column "time" (used in partitioning) ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name ADD COLUMN new_device_id int UNIQUE; ERROR: cannot create a unique index without the column "time" (used in partitioning) ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name DROP COLUMN device_id, ADD COLUMN new_device_id int UNIQUE; ERROR: cannot create a unique index without the column "time" (used in partitioning) \set ON_ERROR_STOP 1 ----------------------- RENAME CONSTRAINT ------------------ ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name RENAME CONSTRAINT hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key TO new_name; ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name * RENAME CONSTRAINT new_name TO new_name2; ALTER TABLE IF EXISTS hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name RENAME CONSTRAINT hyper_unique_with_looooooooooooooooooooooooooooo_sensor_1_check TO check_2; SELECT * FROM test.show_constraints('hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated ------------+------+------------+-----------+----------------------------+------------+----------+----------- check_2 | c | {sensor_1} | - | (sensor_1 > (10)::numeric) | f | f | t new_name2 | u | {time} | new_name2 | | f | f | t SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_2_4_chunk'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated ----------------+------+------------+----------------------------------------+------------------------------------------------------------------------------------------+------------+----------+----------- 4_10_new_name2 | u | {time} | _timescaledb_internal."4_10_new_name2" | | f | f | t check_2 | c | {sensor_1} | - | (sensor_1 > (10)::numeric) | f | f | t constraint_4 | c | {time} | - | (("time" >= '1257987700000000000'::bigint) AND ("time" < '1257987700000000010'::bigint)) | f | f | t SELECT * FROM _timescaledb_catalog.chunk_constraint; chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----------+--------------------+-----------------+---------------------------- 3 | 3 | constraint_3 | 4 | 4 | constraint_4 | 5 | 5 | constraint_5 | 4 | | 4_10_new_name2 | new_name2 5 | | 5_11_new_name2 | new_name2 \set ON_ERROR_STOP 0 ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name RENAME CONSTRAINT new_name TO new_name2; ERROR: constraint "new_name" for table "hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name" does not exist ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name RENAME CONSTRAINT new_name2 TO check_2; ERROR: constraint "check_2" for relation "hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name" already exists ALTER TABLE ONLY hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name RENAME CONSTRAINT new_name2 TO new_name; ERROR: ONLY option not supported on hypertable operations ALTER TABLE _timescaledb_internal._hyper_2_4_chunk RENAME CONSTRAINT "4_10_new_name2" TO new_name; ERROR: renaming constraints on chunks is not supported \set ON_ERROR_STOP 1 ----------------------- PRIMARY KEY ------------------ CREATE TABLE hyper_pk ( time BIGINT NOT NULL PRIMARY KEY, device_id TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10) ); SELECT * FROM create_hypertable('hyper_pk', 'time', chunk_time_interval => 10); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 3 | public | hyper_pk | t INSERT INTO hyper_pk(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); \set ON_ERROR_STOP 0 INSERT INTO hyper_pk(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); ERROR: duplicate key value violates unique constraint "6_14_hyper_pk_pkey" \set ON_ERROR_STOP 1 --should have unique constraint not just unique index SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_3_6_chunk'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated -------------------------+------+------------+--------------------------------------------+------------------------------------------------------------------------------------------+------------+----------+----------- 6_14_hyper_pk_pkey | p | {time} | _timescaledb_internal."6_14_hyper_pk_pkey" | | f | f | t constraint_6 | c | {time} | - | (("time" >= '1257987700000000000'::bigint) AND ("time" < '1257987700000000010'::bigint)) | f | f | t hyper_pk_sensor_1_check | c | {sensor_1} | - | (sensor_1 > (10)::numeric) | f | f | t ALTER TABLE hyper_pk DROP CONSTRAINT hyper_pk_pkey; SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_3_6_chunk'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated -------------------------+------+------------+-------+------------------------------------------------------------------------------------------+------------+----------+----------- constraint_6 | c | {time} | - | (("time" >= '1257987700000000000'::bigint) AND ("time" < '1257987700000000010'::bigint)) | f | f | t hyper_pk_sensor_1_check | c | {sensor_1} | - | (sensor_1 > (10)::numeric) | f | f | t --uniqueness not enforced INSERT INTO hyper_pk(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev3', 11); --shouldn't be able to create pk \set ON_ERROR_STOP 0 ALTER TABLE hyper_pk ADD CONSTRAINT hyper_pk_pkey PRIMARY KEY (time); ERROR: could not create unique index "6_15_hyper_pk_pkey" ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name ADD COLUMN new_device_id int PRIMARY KEY; ERROR: cannot create a unique index without the column "time" (used in partitioning) \set ON_ERROR_STOP 1 DELETE FROM hyper_pk WHERE device_id = 'dev3'; --cannot create pk constraint on non-partition column \set ON_ERROR_STOP 0 ALTER TABLE hyper_pk ADD CONSTRAINT hyper_pk_invalid PRIMARY KEY (device_id); ERROR: cannot create a unique index without the column "time" (used in partitioning) \set ON_ERROR_STOP 1 --now can create ALTER TABLE hyper_pk ADD CONSTRAINT hyper_pk_pkey PRIMARY KEY (time) DEFERRABLE INITIALLY DEFERRED; SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_3_6_chunk'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated -------------------------+------+------------+--------------------------------------------+------------------------------------------------------------------------------------------+------------+----------+----------- 6_16_hyper_pk_pkey | p | {time} | _timescaledb_internal."6_16_hyper_pk_pkey" | | t | t | t constraint_6 | c | {time} | - | (("time" >= '1257987700000000000'::bigint) AND ("time" < '1257987700000000010'::bigint)) | f | f | t hyper_pk_sensor_1_check | c | {sensor_1} | - | (sensor_1 > (10)::numeric) | f | f | t --test adding constraint with same name to different table -- should fail \set ON_ERROR_STOP 0 ALTER TABLE hyper ADD CONSTRAINT hyper_pk_pkey UNIQUE (time); ERROR: relation "hyper_pk_pkey" already exists \set ON_ERROR_STOP 1 --uniquness violation fails \set ON_ERROR_STOP 0 BEGIN; --error here deferred until commit INSERT INTO hyper_pk(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); SELECT 1; ?column? ---------- 1 COMMIT; ERROR: duplicate key value violates unique constraint "6_16_hyper_pk_pkey" \set ON_ERROR_STOP 1 ----------------------- FOREIGN KEY ------------------ CREATE TABLE devices( device_id TEXT NOT NULL, PRIMARY KEY (device_id) ); CREATE TABLE hyper_fk ( time BIGINT NOT NULL PRIMARY KEY, device_id TEXT NOT NULL REFERENCES devices(device_id), sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10) ); SELECT * FROM create_hypertable('hyper_fk', 'time', chunk_time_interval => 10); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 4 | public | hyper_fk | t --fail fk constraint \set ON_ERROR_STOP 0 INSERT INTO hyper_fk(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); ERROR: insert or update on table "_hyper_4_7_chunk" violates foreign key constraint "7_17_hyper_fk_device_id_fkey" \set ON_ERROR_STOP 1 INSERT INTO devices VALUES ('dev2'); INSERT INTO hyper_fk(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); --delete should fail \set ON_ERROR_STOP 0 DELETE FROM devices; ERROR: update or delete on table "devices" violates foreign key constraint "hyper_fk_device_id_fkey" on table "hyper_fk" \set ON_ERROR_STOP 1 ALTER TABLE hyper_fk DROP CONSTRAINT hyper_fk_device_id_fkey; --should now be able to add non-fk rows INSERT INTO hyper_fk(time, device_id,sensor_1) VALUES (1257987700000000001, 'dev3', 11); --can't add fk because of dev3 row \set ON_ERROR_STOP 0 ALTER TABLE hyper_fk ADD CONSTRAINT hyper_fk_device_id_fkey FOREIGN KEY (device_id) REFERENCES devices(device_id); ERROR: insert or update on table "hyper_fk" violates foreign key constraint "hyper_fk_device_id_fkey" \set ON_ERROR_STOP 1 --but can add a NOT VALID one ALTER TABLE hyper_fk ADD CONSTRAINT hyper_fk_device_id_fkey FOREIGN KEY (device_id) REFERENCES devices(device_id) NOT VALID; --which will fail when validated \set ON_ERROR_STOP 0 ALTER TABLE hyper_fk VALIDATE CONSTRAINT hyper_fk_device_id_fkey; ERROR: insert or update on table "hyper_fk" violates foreign key constraint "hyper_fk_device_id_fkey" \set ON_ERROR_STOP 1 ALTER TABLE hyper_fk DROP CONSTRAINT hyper_fk_device_id_fkey; DELETE FROM hyper_fk WHERE device_id = 'dev3'; ALTER TABLE hyper_fk ADD CONSTRAINT hyper_fk_device_id_fkey FOREIGN KEY (device_id) REFERENCES devices(device_id); \set ON_ERROR_STOP 0 INSERT INTO hyper_fk(time, device_id,sensor_1) VALUES (1257987700000000002, 'dev3', 11); ERROR: insert or update on table "_hyper_4_8_chunk" violates foreign key constraint "8_22_hyper_fk_device_id_fkey" \set ON_ERROR_STOP 1 SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_4_8_chunk'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated ------------------------------+------+-------------+--------------------------------------------+------------------------------------------------------------------------------------------+------------+----------+----------- 8_20_hyper_fk_pkey | p | {time} | _timescaledb_internal."8_20_hyper_fk_pkey" | | f | f | t 8_22_hyper_fk_device_id_fkey | f | {device_id} | devices_pkey | | f | f | t constraint_8 | c | {time} | - | (("time" >= '1257987700000000000'::bigint) AND ("time" < '1257987700000000010'::bigint)) | f | f | t hyper_fk_sensor_1_check | c | {sensor_1} | - | (sensor_1 > (10)::numeric) | f | f | t SELECT * FROM _timescaledb_catalog.chunk_constraint; chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----------+--------------------+------------------------------+---------------------------- 3 | 3 | constraint_3 | 4 | 4 | constraint_4 | 5 | 5 | constraint_5 | 4 | | 4_10_new_name2 | new_name2 5 | | 5_11_new_name2 | new_name2 6 | 6 | constraint_6 | 6 | | 6_16_hyper_pk_pkey | hyper_pk_pkey 8 | 8 | constraint_8 | 8 | | 8_20_hyper_fk_pkey | hyper_fk_pkey 8 | | 8_22_hyper_fk_device_id_fkey | hyper_fk_device_id_fkey --test CASCADE drop behavior DROP TABLE devices CASCADE; NOTICE: drop cascades to 2 other objects SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_4_8_chunk'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated -------------------------+------+------------+--------------------------------------------+------------------------------------------------------------------------------------------+------------+----------+----------- 8_20_hyper_fk_pkey | p | {time} | _timescaledb_internal."8_20_hyper_fk_pkey" | | f | f | t constraint_8 | c | {time} | - | (("time" >= '1257987700000000000'::bigint) AND ("time" < '1257987700000000010'::bigint)) | f | f | t hyper_fk_sensor_1_check | c | {sensor_1} | - | (sensor_1 > (10)::numeric) | f | f | t SELECT * FROM _timescaledb_catalog.chunk_constraint; chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----------+--------------------+--------------------+---------------------------- 3 | 3 | constraint_3 | 4 | 4 | constraint_4 | 5 | 5 | constraint_5 | 4 | | 4_10_new_name2 | new_name2 5 | | 5_11_new_name2 | new_name2 6 | 6 | constraint_6 | 6 | | 6_16_hyper_pk_pkey | hyper_pk_pkey 8 | 8 | constraint_8 | 8 | | 8_20_hyper_fk_pkey | hyper_fk_pkey --the fk went away. INSERT INTO hyper_fk(time, device_id,sensor_1) VALUES (1257987700000000002, 'dev3', 11); CREATE TABLE devices( device_id TEXT NOT NULL, PRIMARY KEY (device_id) ); INSERT INTO devices VALUES ('dev2'), ('dev3'); ALTER TABLE hyper_fk ADD CONSTRAINT hyper_fk_device_id_fkey FOREIGN KEY (device_id) REFERENCES devices(device_id) DEFERRABLE INITIALLY DEFERRED; \set ON_ERROR_STOP 0 BEGIN; --error deferred until commmit INSERT INTO hyper_fk(time, device_id,sensor_1) VALUES (1257987700000000003, 'dev4', 11); SELECT 1; ?column? ---------- 1 COMMIT; ERROR: insert or update on table "_hyper_4_8_chunk" violates foreign key constraint "8_23_hyper_fk_device_id_fkey" \set ON_ERROR_STOP 1 ALTER TABLE hyper_fk ALTER CONSTRAINT hyper_fk_device_id_fkey NOT DEFERRABLE; \set ON_ERROR_STOP 0 BEGIN; --error detected right away INSERT INTO hyper_fk(time, device_id,sensor_1) VALUES (1257987700000000003, 'dev4', 11); ERROR: insert or update on table "_hyper_4_8_chunk" violates foreign key constraint "8_23_hyper_fk_device_id_fkey" SELECT 1; ERROR: current transaction is aborted, commands ignored until end of transaction block COMMIT; \set ON_ERROR_STOP 1 --this tests that there are no extra chunk_constraints left on hyper_fk TRUNCATE hyper_fk; ----------------------- FOREIGN KEY INTO A HYPERTABLE ------------------ --FOREIGN KEY references into a hypertable are currently broken. --The referencing table will never find the corresponding row in the hypertable --since it will only search the parent. Thus any insert will result in an ERROR --Block such foreign keys or fix. (Hard to block on create table so punting for now) CREATE TABLE hyper_for_ref ( time BIGINT NOT NULL PRIMARY KEY, device_id TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10) ); SELECT * FROM create_hypertable('hyper_for_ref', 'time', chunk_time_interval => 10); hypertable_id | schema_name | table_name | created ---------------+-------------+---------------+--------- 5 | public | hyper_for_ref | t \set ON_ERROR_STOP 0 CREATE TABLE referrer ( time BIGINT NOT NULL REFERENCES hyper_for_ref(time) ); \set ON_ERROR_STOP 1 CREATE TABLE referrer2 ( time BIGINT NOT NULL ); \set ON_ERROR_STOP 0 ALTER TABLE referrer2 ADD CONSTRAINT hyper_fk_device_id_fkey FOREIGN KEY (time) REFERENCES hyper_for_ref(time); \set ON_ERROR_STOP 1 -- github issue 8082: FK referencing hypertable with composite unique index -- fails on first insert because chunk indexes are created after FK propagation CREATE TABLE messages_ref ( time_received TIMESTAMPTZ NOT NULL, message_id BIGSERIAL, message_type SMALLINT NOT NULL ); SELECT create_hypertable('messages_ref', by_range('time_received')); create_hypertable ------------------- (6,t) CREATE UNIQUE INDEX ON messages_ref(time_received, message_id); -- Create FK referencing the hypertable BEFORE any data exists CREATE TABLE contents_ref ( content_id BIGSERIAL, time_received TIMESTAMPTZ NOT NULL, message_id BIGINT NOT NULL, content CHAR(10), FOREIGN KEY (time_received, message_id) REFERENCES messages_ref(time_received, message_id) ON DELETE CASCADE ); -- This insert creates a new chunk. Previously it would fail with -- "index for constraint not found on chunk" because FK propagation -- happened before chunk indexes were created. INSERT INTO messages_ref (time_received, message_type) VALUES ('2025-05-05 14:56:58.000 UTC', 2); INSERT INTO contents_ref (message_id, time_received, content) VALUES (CURRVAL('messages_ref_message_id_seq'), '2025-05-05 14:56:58.000 UTC', 'HEJ'); -- Insert into a second chunk INSERT INTO messages_ref (time_received, message_type) VALUES ('2025-06-05 14:57:58.000 UTC', 3); INSERT INTO contents_ref (message_id, time_received, content) VALUES (CURRVAL('messages_ref_message_id_seq'), '2025-06-05 14:57:58.000 UTC', 'HEJ2'); -- Verify data SELECT message_type FROM messages_ref ORDER BY time_received; message_type -------------- 2 3 SELECT content FROM contents_ref ORDER BY time_received; content ------------ HEJ HEJ2 -- Verify FK enforcement \set ON_ERROR_STOP 0 INSERT INTO contents_ref (message_id, time_received, content) VALUES (9999, '2025-05-05 14:56:58.000 UTC', 'FAIL'); ERROR: insert or update on table "contents_ref" violates foreign key constraint "contents_ref_time_received_message_id_fkey" \set ON_ERROR_STOP 1 -- Verify cascade delete DELETE FROM messages_ref WHERE message_type = 2; SELECT content FROM contents_ref ORDER BY time_received; content ------------ HEJ2 DROP TABLE contents_ref; DROP TABLE messages_ref; ----------------------- EXCLUSION CONSTRAINT ------------------ CREATE TABLE hyper_ex ( time BIGINT, device_id TEXT NOT NULL REFERENCES devices(device_id), sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10), canceled boolean DEFAULT false, EXCLUDE USING btree ( time WITH =, device_id WITH = ) WHERE (not canceled) ); SELECT * FROM create_hypertable('hyper_ex', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 7 | public | hyper_ex | t INSERT INTO hyper_ex(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); \set ON_ERROR_STOP 0 INSERT INTO hyper_ex(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 12); ERROR: conflicting key value violates exclusion constraint "11_25_hyper_ex_time_device_id_excl" \set ON_ERROR_STOP 1 ALTER TABLE hyper_ex DROP CONSTRAINT hyper_ex_time_device_id_excl; --can now add INSERT INTO hyper_ex(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 12); --cannot add because of conflicts \set ON_ERROR_STOP 0 ALTER TABLE hyper_ex ADD CONSTRAINT hyper_ex_time_device_id_excl EXCLUDE USING btree ( time WITH =, device_id WITH = ) WHERE (not canceled) ; ERROR: could not create exclusion constraint "11_26_hyper_ex_time_device_id_excl" \set ON_ERROR_STOP 1 DELETE FROM hyper_ex WHERE sensor_1 = 12; ALTER TABLE hyper_ex ADD CONSTRAINT hyper_ex_time_device_id_excl EXCLUDE USING btree ( time WITH =, device_id WITH = ) WHERE (not canceled) DEFERRABLE INITIALLY DEFERRED ; \set ON_ERROR_STOP 0 BEGIN; --error deferred til commit INSERT INTO hyper_ex(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 12); SELECT 1; ?column? ---------- 1 COMMIT; ERROR: conflicting key value violates exclusion constraint "11_27_hyper_ex_time_device_id_excl" \set ON_ERROR_STOP 1 --cannot add exclusion constraint without partition key. CREATE TABLE hyper_ex_invalid ( time BIGINT, device_id TEXT NOT NULL REFERENCES devices(device_id), sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10), canceled boolean DEFAULT false, EXCLUDE USING btree ( device_id WITH = ) WHERE (not canceled) ); \set ON_ERROR_STOP 0 SELECT * FROM create_hypertable('hyper_ex_invalid', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); ERROR: cannot create a unique index without the column "time" (used in partitioning) \set ON_ERROR_STOP 1 --- NO INHERIT constraints (not allowed) ---- CREATE TABLE hyper_noinherit ( time BIGINT, sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 0) NO INHERIT ); SELECT * FROM test.show_constraints('hyper_noinherit'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated --------------------------------+------+------------+-------+---------------------------+------------+----------+----------- hyper_noinherit_sensor_1_check | c | {sensor_1} | - | (sensor_1 > (0)::numeric) | f | f | t \set ON_ERROR_STOP 0 SELECT * FROM create_hypertable('hyper_noinherit', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); ERROR: cannot have NO INHERIT constraints on hypertable "hyper_noinherit" \set ON_ERROR_STOP 1 CREATE TABLE hyper_noinherit_alter ( time BIGINT, sensor_1 NUMERIC NULL DEFAULT 1 ); SELECT * FROM create_hypertable('hyper_noinherit_alter', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+-----------------------+--------- 9 | public | hyper_noinherit_alter | t \set ON_ERROR_STOP 0 ALTER TABLE hyper_noinherit_alter ADD CONSTRAINT check_noinherit CHECK (sensor_1 > 0) NO INHERIT; ERROR: cannot have NO INHERIT constraints on hypertable "hyper_noinherit_alter" -- CREATE TABLE WITH DEFERRED CONSTRAINTS -- CREATE TABLE hyper_unique_deferred ( time BIGINT UNIQUE DEFERRABLE INITIALLY DEFERRED, device_id TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10) ); SELECT * FROM create_hypertable('hyper_unique_deferred', 'time', chunk_time_interval => 10); hypertable_id | schema_name | table_name | created ---------------+-------------+-----------------------+--------- 10 | public | hyper_unique_deferred | t INSERT INTO hyper_unique_deferred(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); \set ON_ERROR_STOP 0 BEGIN; --error here deferred until commit INSERT INTO hyper_unique_deferred(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); SELECT 1; ?column? ---------- 1 COMMIT; ERROR: duplicate key value violates unique constraint "12_28_hyper_unique_deferred_time_key" \set ON_ERROR_STOP 1 --test deferred on create table CREATE TABLE hyper_pk_deferred ( time BIGINT NOT NULL PRIMARY KEY DEFERRABLE INITIALLY DEFERRED, device_id TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10) ); SELECT * FROM create_hypertable('hyper_pk_deferred', 'time', chunk_time_interval => 10); hypertable_id | schema_name | table_name | created ---------------+-------------+-------------------+--------- 11 | public | hyper_pk_deferred | t INSERT INTO hyper_pk_deferred(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); \set ON_ERROR_STOP 0 BEGIN; --error here deferred until commit INSERT INTO hyper_pk_deferred(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); SELECT 1; ?column? ---------- 1 COMMIT; ERROR: duplicate key value violates unique constraint "13_29_hyper_pk_deferred_pkey" \set ON_ERROR_STOP 1 --test that deferred works on create table too CREATE TABLE hyper_fk_deferred ( time BIGINT NOT NULL PRIMARY KEY, device_id TEXT NOT NULL REFERENCES devices(device_id) DEFERRABLE INITIALLY DEFERRED, sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10) ); SELECT * FROM create_hypertable('hyper_fk_deferred', 'time', chunk_time_interval => 10); hypertable_id | schema_name | table_name | created ---------------+-------------+-------------------+--------- 12 | public | hyper_fk_deferred | t \set ON_ERROR_STOP 0 BEGIN; --error deferred until commmit INSERT INTO hyper_fk_deferred(time, device_id,sensor_1) VALUES (1257987700000000003, 'dev4', 11); SELECT 1; ?column? ---------- 1 COMMIT; ERROR: insert or update on table "_hyper_12_14_chunk" violates foreign key constraint "14_30_hyper_fk_deferred_device_id_fkey" \set ON_ERROR_STOP 1 CREATE TABLE hyper_ex_deferred ( time BIGINT, device_id TEXT NOT NULL REFERENCES devices(device_id), sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10), canceled boolean DEFAULT false, EXCLUDE USING btree ( time WITH =, device_id WITH = ) WHERE (not canceled) DEFERRABLE INITIALLY DEFERRED ); SELECT * FROM create_hypertable('hyper_ex_deferred', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+-------------------+--------- 13 | public | hyper_ex_deferred | t INSERT INTO hyper_ex_deferred(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 12); \set ON_ERROR_STOP 0 BEGIN; --error deferred til commit INSERT INTO hyper_ex_deferred(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 12); SELECT 1; ?column? ---------- 1 COMMIT; ERROR: conflicting key value violates exclusion constraint "15_33_hyper_ex_deferred_time_device_id_excl" \set ON_ERROR_STOP 1 -- Make sure renaming schemas won't break dropping constraints \c :TEST_DBNAME :ROLE_SUPERUSER CREATE TABLE hyper_unique ( time BIGINT NOT NULL UNIQUE, device_id TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10) ); SELECT * FROM create_hypertable('hyper_unique', 'time', chunk_time_interval => 10, associated_schema_name => 'my_associated_schema'); hypertable_id | schema_name | table_name | created ---------------+-------------+--------------+--------- 14 | public | hyper_unique | t INSERT INTO hyper_unique(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); ALTER SCHEMA my_associated_schema RENAME TO new_associated_schema; ALTER TABLE hyper_unique DROP CONSTRAINT hyper_unique_time_key; -- test for constraint validation crash, see #1183 CREATE TABLE test_validate(time timestamp NOT NULL, a TEXT, b TEXT); SELECT * FROM create_hypertable('test_validate', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices hypertable_id | schema_name | table_name | created ---------------+-------------+---------------+--------- 15 | public | test_validate | t INSERT INTO test_validate values(now(), 'a', 'b'); ALTER TABLE test_validate ADD COLUMN c TEXT, ADD CONSTRAINT c_not_null CHECK (c IS NOT NULL) NOT VALID; UPDATE test_validate SET c = ''; ALTER TABLE test_validate VALIDATE CONSTRAINT c_not_null; DROP TABLE test_validate; -- test for hypertables constraints both using index tablespaces and not See #2604 SET client_min_messages = ERROR; DROP TABLESPACE IF EXISTS tablespace1; SET client_min_messages = NOTICE; CREATE TABLESPACE tablespace1 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE1_PATH; CREATE TABLE fk_tbl ( id int, CONSTRAINT pkfk PRIMARY KEY (id) USING INDEX TABLESPACE tablespace1); CREATE TABLE tbl ( fk_id int, id int, time timestamp, CONSTRAINT pk PRIMARY KEY (time, id) USING INDEX TABLESPACE tablespace1 DEFERRABLE INITIALLY DEFERRED); SELECT create_hypertable('tbl', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ------------------- (16,public,tbl,t) ALTER TABLE tbl ADD CONSTRAINT fk_con FOREIGN KEY (fk_id) REFERENCES fk_tbl(id) ON UPDATE SET NULL ON DELETE SET NULL; INSERT INTO fk_tbl VALUES(1); INSERT INTO tbl VALUES ( 1, 1, now() ); DROP TABLE tbl; DROP TABLE fk_tbl; DROP TABLESPACE IF EXISTS tablespace1; ================================================ FILE: test/expected/copy.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \o /dev/null \ir include/insert_two_partitions.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."two_Partitions" ( "timeCustom" BIGINT NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."two_Partitions" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_0) WHERE series_0 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_1) WHERE series_1 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_2) WHERE series_2 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_bool) WHERE series_bool IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, device_id); SELECT * FROM create_hypertable('"public"."two_Partitions"'::regclass, 'timeCustom'::name, 'device_id'::name, associated_schema_name=>'_timescaledb_internal'::text, number_partitions => 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); \set QUIET off BEGIN; \COPY public."two_Partitions" FROM 'data/ds1_dev1_1.tsv' NULL AS ''; COMMIT; INSERT INTO public."two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257987600000000000, 'dev1', 1.5, 1), (1257987600000000000, 'dev1', 1.5, 2), (1257894000000000000, 'dev2', 1.5, 1), (1257894002000000000, 'dev1', 2.5, 3); INSERT INTO "two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257894000000000000, 'dev2', 1.5, 2); \set QUIET on \o --old chunks COPY "two_Partitions"("timeCustom", device_id, series_0, series_1) FROM STDIN DELIMITER ','; \copy "two_Partitions"("timeCustom", device_id, series_0, series_1) FROM STDIN DELIMITER ','; --new chunks COPY "two_Partitions"("timeCustom", device_id, series_0, series_1) FROM STDIN DELIMITER ','; \copy "two_Partitions"("timeCustom", device_id, series_0, series_1) FROM STDIN DELIMITER ','; COPY (SELECT * FROM "two_Partitions" ORDER BY "timeCustom", device_id, series_0, series_1) TO STDOUT; 1257894000000000000 dev1 1.5 1 2 t 1257894000000000000 dev1 1.5 2 \N \N 1257894000000000000 dev2 1.5 1 \N \N 1257894000000000000 dev2 1.5 2 \N \N 1257894000000000000 dev3 1.5 2 \N \N 1257894000000000000 dev3 1.5 2 \N \N 1257894000000001000 dev1 2.5 3 \N \N 1257894001000000000 dev1 3.5 4 \N \N 1257894002000000000 dev1 2.5 3 \N \N 1257894002000000000 dev1 5.5 6 \N t 1257894002000000000 dev1 5.5 7 \N f 1257897600000000000 dev1 4.5 5 \N f 1257987600000000000 dev1 1.5 1 \N \N 1257987600000000000 dev1 1.5 2 \N \N 2257894000000000000 dev3 1.5 2 \N \N 2257894000000000000 dev3 1.5 2 \N \N ---test hypertable with FK CREATE TABLE "meta" ("id" serial PRIMARY KEY); CREATE TABLE "hyper" ( "meta_id" integer NOT NULL REFERENCES meta(id), "time" bigint NOT NULL, "value" double precision NOT NULL ); SELECT create_hypertable('hyper', 'time', chunk_time_interval => 100); create_hypertable -------------------- (2,public,hyper,t) INSERT INTO "meta" ("id") values (1); \copy hyper (time, meta_id, value) FROM STDIN DELIMITER ','; COPY hyper (time, meta_id, value) FROM STDIN DELIMITER ','; \set ON_ERROR_STOP 0 \copy hyper (time, meta_id, value) FROM STDIN DELIMITER ','; ERROR: insert or update on table "_hyper_2_6_chunk" violates foreign key constraint "6_1_hyper_meta_id_fkey" COPY hyper (time, meta_id, value) FROM STDIN DELIMITER ','; ERROR: insert or update on table "_hyper_2_6_chunk" violates foreign key constraint "6_1_hyper_meta_id_fkey" \set ON_ERROR_STOP 1 COPY (SELECT * FROM hyper ORDER BY time, meta_id) TO STDOUT; 1 1 1 1 2 1 --test that copy works with a low setting for max_open_chunks_per_insert set timescaledb.max_open_chunks_per_insert = 1; CREATE TABLE "hyper2" ( "time" bigint NOT NULL, "value" double precision NOT NULL ); SELECT create_hypertable('hyper2', 'time', chunk_time_interval => 10); create_hypertable --------------------- (3,public,hyper2,t) \copy hyper2 from data/copy_data.csv with csv header ; -- test copy with blocking trigger CREATE FUNCTION gt_10() RETURNS trigger AS $func$ BEGIN IF NEW."time" < 11 THEN RETURN NULL; END IF; RETURN NEW; END $func$ LANGUAGE plpgsql; CREATE TABLE "trigger_test" ( "time" bigint NOT NULL, "value" double precision NOT NULL ); SELECT create_hypertable('trigger_test', 'time', chunk_time_interval => 10); create_hypertable --------------------------- (4,public,trigger_test,t) CREATE TRIGGER check_time BEFORE INSERT ON trigger_test FOR EACH ROW EXECUTE FUNCTION gt_10(); \copy trigger_test from data/copy_data.csv with csv header ; SELECT * FROM trigger_test ORDER BY time; time | value ------+-------------------- 11 | 0.795640022493899 12 | 0.631451691035181 13 | 0.0958626130595803 14 | 0.929304684977978 15 | 0.524866581428796 16 | 0.919249163009226 17 | 0.878917074296623 18 | 0.68551931809634 19 | 0.594833800103515 20 | 0.819584367796779 21 | 0.474171321373433 22 | 0.938535195309669 23 | 0.333933369256556 24 | 0.274582070298493 25 | 0.602348630782217 -- Test that if we copy from stdin to a hypertable and violate a null -- constraint, it does not crash and generate an appropriate error -- message. CREATE TABLE test(a INT NOT NULL, b TIMESTAMPTZ); SELECT create_hypertable('test', 'b'); create_hypertable ------------------- (5,public,test,t) \set ON_ERROR_STOP 0 COPY TEST (a,b) FROM STDIN (delimiter ',', null 'N'); ERROR: null value in column "a" of relation "_hyper_5_13_chunk" violates not-null constraint \c :TEST_DBNAME :ROLE_SUPERUSER SET client_min_messages TO NOTICE; -- Do a basic test of COPY with a wrong PROGRAM COPY hyper FROM PROGRAM 'error'; ERROR: program "error" failed \set ON_ERROR_STOP 1 ---------------------------------------------------------------- -- Testing COPY TO. ---------------------------------------------------------------- -- COPY TO using a hypertable will not copy any tuples, but should -- show a notice. COPY hyper TO STDOUT DELIMITER ','; NOTICE: hypertable data are in the chunks, no data will be copied -- COPY TO using a query should display all the tuples and not show a -- notice. COPY (SELECT * FROM hyper) TO STDOUT DELIMITER ','; 1,1,1 1,2,1 ---------------------------------------------------------------- -- Testing multi-buffer optimization. ---------------------------------------------------------------- CREATE TABLE "hyper_copy" ( "time" bigint NOT NULL, "value" double precision NOT NULL ); SELECT create_hypertable('hyper_copy', 'time', chunk_time_interval => 2); create_hypertable ------------------------- (6,public,hyper_copy,t) -- First copy call with default client_min_messages, to get rid of the -- building index "_hyper_XXX_chunk_hyper_copy_time_idx" on table "_hyper_XXX_chunk" serially -- messages \copy hyper_copy FROM data/copy_data.csv WITH csv header; SET client_min_messages TO DEBUG1; \copy hyper_copy FROM data/copy_data.csv WITH csv header; DEBUG: Using optimized multi-buffer copy operation (TS_CIM_MULTI_CONDITIONAL). SELECT count(*) FROM hyper_copy; count ------- 50 -- Limit number of open chunks SET timescaledb.max_open_chunks_per_insert = 1; \copy hyper_copy FROM data/copy_data.csv WITH csv header; DEBUG: Using optimized multi-buffer copy operation (TS_CIM_MULTI_CONDITIONAL). SELECT count(*) FROM hyper_copy; count ------- 75 -- Before trigger disable the multi-buffer optimization CREATE OR REPLACE FUNCTION empty_test_trigger() RETURNS TRIGGER LANGUAGE PLPGSQL AS $BODY$ BEGIN IF TG_OP = 'DELETE' THEN RETURN OLD; END IF; RETURN NEW; END $BODY$; -- Before trigger (CIM_SINGLE should be used) CREATE TRIGGER hyper_copy_trigger_insert_before BEFORE INSERT ON hyper_copy FOR EACH ROW EXECUTE FUNCTION empty_test_trigger(); \copy hyper_copy FROM data/copy_data.csv WITH csv header; DEBUG: Using normal unbuffered copy operation (TS_CIM_SINGLE) because triggers are defined on the destination table. SELECT count(*) FROM hyper_copy; count ------- 100 -- Suppress 'DEBUG: EventTriggerInvoke XXXX' messages RESET client_min_messages; DROP TRIGGER hyper_copy_trigger_insert_before ON hyper_copy; SET client_min_messages TO DEBUG1; -- After trigger (CIM_MULTI_CONDITIONAL should be used) CREATE TRIGGER hyper_copy_trigger_insert_after AFTER INSERT ON hyper_copy FOR EACH ROW EXECUTE FUNCTION empty_test_trigger(); \copy hyper_copy FROM data/copy_data.csv WITH csv header; DEBUG: Using optimized multi-buffer copy operation (TS_CIM_MULTI_CONDITIONAL). SELECT count(*) FROM hyper_copy; count ------- 125 -- Insert data into the chunks in random order COPY hyper_copy FROM STDIN DELIMITER ',' NULL AS 'null'; DEBUG: Using optimized multi-buffer copy operation (TS_CIM_MULTI_CONDITIONAL). SELECT count(*) FROM hyper_copy; count ------- 154 RESET client_min_messages; RESET timescaledb.max_open_chunks_per_insert; ---------------------------------------------------------------- -- Testing multi-buffer optimization -- (no index on destination hypertable). ---------------------------------------------------------------- CREATE TABLE "hyper_copy_noindex" ( "time" bigint NOT NULL, "value" double precision NOT NULL ); SELECT create_hypertable('hyper_copy_noindex', 'time', chunk_time_interval => 10, create_default_indexes => false); create_hypertable --------------------------------- (7,public,hyper_copy_noindex,t) -- No trigger \copy hyper_copy_noindex FROM data/copy_data.csv WITH csv header; SET client_min_messages TO DEBUG1; \copy hyper_copy_noindex FROM data/copy_data.csv WITH csv header; DEBUG: Using optimized multi-buffer copy operation (TS_CIM_MULTI_CONDITIONAL). RESET client_min_messages; SELECT count(*) FROM hyper_copy_noindex; count ------- 50 -- Before trigger (CIM_SINGLE should be used) CREATE TRIGGER hyper_copy_trigger_insert_before BEFORE INSERT ON hyper_copy_noindex FOR EACH ROW EXECUTE FUNCTION empty_test_trigger(); \copy hyper_copy_noindex FROM data/copy_data.csv WITH csv header; SET client_min_messages TO DEBUG1; \copy hyper_copy_noindex FROM data/copy_data.csv WITH csv header; DEBUG: Using normal unbuffered copy operation (TS_CIM_SINGLE) because triggers are defined on the destination table. RESET client_min_messages; SELECT count(*) FROM hyper_copy_noindex; count ------- 100 -- After trigger (CIM_MULTI_CONDITIONAL should be used) DROP TRIGGER hyper_copy_trigger_insert_before ON hyper_copy_noindex; CREATE TRIGGER hyper_copy_trigger_insert_after AFTER INSERT ON hyper_copy_noindex FOR EACH ROW EXECUTE FUNCTION empty_test_trigger(); \copy hyper_copy_noindex FROM data/copy_data.csv WITH csv header; SET client_min_messages TO DEBUG1; \copy hyper_copy_noindex FROM data/copy_data.csv WITH csv header; DEBUG: Using optimized multi-buffer copy operation (TS_CIM_MULTI_CONDITIONAL). RESET client_min_messages; SELECT count(*) FROM hyper_copy_noindex; count ------- 150 ---------------------------------------------------------------- -- Testing multi-buffer optimization -- (more chunks than MAX_PARTITION_BUFFERS). ---------------------------------------------------------------- CREATE TABLE "hyper_copy_large" ( "time" timestamp NOT NULL, "value" double precision NOT NULL ); -- Genate data that will create more than 32 (MAX_PARTITION_BUFFERS) -- chunks on the 10 second chunk_time_interval partitioned hypertable. INSERT INTO hyper_copy_large SELECT time, random() AS value FROM generate_series('2022-01-01', '2022-01-31', INTERVAL '1 hour') AS g1(time) ORDER BY time; SELECT COUNT(*) FROM hyper_copy_large; count ------- 721 -- Migrate data to chunks by using copy SELECT create_hypertable('hyper_copy_large', 'time', chunk_time_interval => INTERVAL '1 hour', migrate_data => 'true'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices NOTICE: migrating data to chunks create_hypertable ------------------------------- (8,public,hyper_copy_large,t) SELECT COUNT(*) FROM hyper_copy_large; count ------- 721 ---------------------------------------------------------------- -- Testing multi-buffer optimization -- (triggers on chunks). ---------------------------------------------------------------- CREATE TABLE "table_with_chunk_trigger" ( "time" bigint NOT NULL, "value" double precision NOT NULL ); -- This trigger counts the already inserted tuples in -- the table table_with_chunk_trigger. CREATE OR REPLACE FUNCTION count_test_chunk_trigger() RETURNS TRIGGER LANGUAGE PLPGSQL AS $BODY$ DECLARE cnt INTEGER; BEGIN SELECT count(*) FROM table_with_chunk_trigger INTO cnt; RAISE WARNING 'Trigger counted % tuples in table table_with_chunk_trigger', cnt; IF TG_OP = 'DELETE' THEN RETURN OLD; END IF; RETURN NEW; END $BODY$; -- Create hypertable and chunks SELECT create_hypertable('table_with_chunk_trigger', 'time', chunk_time_interval => 1); create_hypertable --------------------------------------- (9,public,table_with_chunk_trigger,t) -- Insert data to create all missing chunks \copy table_with_chunk_trigger from data/copy_data.csv with csv header; SELECT count(*) FROM table_with_chunk_trigger; count ------- 25 -- Chunk 1: 1-2, Chunk 2: 2-3, Chunk 3: 3-4, Chunk 4: 4-5 SELECT chunk_schema, chunk_name FROM timescaledb_information.chunks WHERE hypertable_name = 'table_with_chunk_trigger' AND range_end_integer=5 \gset -- Create before trigger on the 4th chunk CREATE TRIGGER table_with_chunk_trigger_before_trigger BEFORE INSERT ON :chunk_schema.:chunk_name FOR EACH ROW EXECUTE FUNCTION count_test_chunk_trigger(); -- Insert data -- 25 tuples are already imported. The trigger is executed before tuples -- are copied into the 4th chunk. So, the trigger should report 25+3 = 28 -- This test requires that the multi-insert buffers of the other chunks -- are flushed before the trigger is executed. SET client_min_messages TO DEBUG1; \copy table_with_chunk_trigger FROM data/copy_data.csv WITH csv header; DEBUG: Using optimized multi-buffer copy operation (TS_CIM_MULTI_CONDITIONAL). WARNING: Trigger counted 28 tuples in table table_with_chunk_trigger RESET client_min_messages; SELECT count(*) FROM table_with_chunk_trigger; count ------- 50 DROP TRIGGER table_with_chunk_trigger_before_trigger ON :chunk_schema.:chunk_name; -- Create after trigger CREATE TRIGGER table_with_chunk_trigger_after_trigger AFTER INSERT ON :chunk_schema.:chunk_name FOR EACH ROW EXECUTE FUNCTION count_test_chunk_trigger(); -- Insert data -- 50 tuples are already imported. The trigger is executed after all -- tuples are imported. So, the trigger should report 50+25 = 75 SET client_min_messages TO DEBUG1; \copy table_with_chunk_trigger FROM data/copy_data.csv WITH csv header; DEBUG: Using optimized multi-buffer copy operation (TS_CIM_MULTI_CONDITIONAL). WARNING: Trigger counted 75 tuples in table table_with_chunk_trigger RESET client_min_messages; SELECT count(*) FROM table_with_chunk_trigger; count ------- 75 -- Hypertable with after row trigger and no index DROP TABLE table_with_chunk_trigger; CREATE TABLE "table_with_chunk_trigger" ( "time" bigint NOT NULL, "value" double precision NOT NULL ); -- Create hypertable and chunks SELECT create_hypertable('table_with_chunk_trigger', 'time', chunk_time_interval => 1, create_default_indexes => false); create_hypertable ---------------------------------------- (10,public,table_with_chunk_trigger,t) -- Insert data to create all missing chunks \copy table_with_chunk_trigger from data/copy_data.csv with csv header; SELECT count(*) FROM table_with_chunk_trigger; count ------- 25 -- Chunk 1: 1-2, Chunk 2: 2-3, Chunk 3: 3-4, Chunk 4: 4-5 SELECT chunk_schema, chunk_name FROM timescaledb_information.chunks WHERE hypertable_name = 'table_with_chunk_trigger' AND range_end_integer=5 \gset -- Create after trigger CREATE TRIGGER table_with_chunk_trigger_after_trigger AFTER INSERT ON :chunk_schema.:chunk_name FOR EACH ROW EXECUTE FUNCTION count_test_chunk_trigger(); \copy table_with_chunk_trigger from data/copy_data.csv with csv header; WARNING: Trigger counted 50 tuples in table table_with_chunk_trigger SELECT count(*) FROM table_with_chunk_trigger; count ------- 50 ---------------------------------------------------------------- -- Testing multi-buffer optimization -- (Hypertable without before insert trigger) ---------------------------------------------------------------- CREATE TABLE "table_without_bf_trigger" ( "time" bigint NOT NULL, "value" double precision NOT NULL ); SELECT create_hypertable('table_without_bf_trigger', 'time', chunk_time_interval => 1); create_hypertable ---------------------------------------- (11,public,table_without_bf_trigger,t) \copy table_without_bf_trigger from data/copy_data.csv with csv header; SET client_min_messages TO DEBUG1; \copy table_without_bf_trigger from data/copy_data.csv with csv header; DEBUG: Using optimized multi-buffer copy operation (TS_CIM_MULTI_CONDITIONAL). RESET client_min_messages; SELECT count(*) FROM table_without_bf_trigger; count ------- 50 -- After trigger (CIM_MULTI_CONDITIONAL should be used) CREATE TRIGGER table_with_chunk_trigger_after_trigger AFTER INSERT ON table_without_bf_trigger FOR EACH ROW EXECUTE FUNCTION empty_test_trigger(); SET client_min_messages TO DEBUG1; \copy table_without_bf_trigger from data/copy_data.csv with csv header; DEBUG: Using optimized multi-buffer copy operation (TS_CIM_MULTI_CONDITIONAL). RESET client_min_messages; SELECT count(*) FROM table_without_bf_trigger; count ------- 75 ---------------------------------------------------------------- -- Testing multi-buffer optimization -- (Chunks with different layouts) ---------------------------------------------------------------- -- Time is not the first attribute of the hypertable CREATE TABLE "table_with_layout_change" ( "value1" real NOT NULL DEFAULT 1, "value2" smallint DEFAULT NULL, "value3" bigint DEFAULT NULL, "time" bigint NOT NULL, "value4" double precision NOT NULL DEFAULT 4, "value5" double precision NOT NULL DEFAULT 5 ); SELECT create_hypertable('table_with_layout_change', 'time', chunk_time_interval => 1); create_hypertable ---------------------------------------- (12,public,table_with_layout_change,t) -- Chunk 1 (time = 1) COPY table_with_layout_change FROM STDIN DELIMITER ',' NULL AS 'null'; SELECT * FROM table_with_layout_change; value1 | value2 | value3 | time | value4 | value5 --------+--------+--------+------+--------+-------- 100 | 200 | 300 | 1 | 400 | 500 -- Drop the first attribute ALTER TABLE table_with_layout_change DROP COLUMN value1; SELECT * FROM table_with_layout_change; value2 | value3 | time | value4 | value5 --------+--------+------+--------+-------- 200 | 300 | 1 | 400 | 500 -- COPY into existing chunk (time = 1) COPY table_with_layout_change FROM STDIN DELIMITER ',' NULL AS 'null'; -- Create new chunk (time = 2) COPY table_with_layout_change FROM STDIN DELIMITER ',' NULL AS 'null'; SELECT * FROM table_with_layout_change ORDER BY time, value2, value3, value4, value5; value2 | value3 | time | value4 | value5 --------+--------+------+--------+-------- 200 | 300 | 1 | 400 | 500 201 | 301 | 1 | 401 | 501 202 | 302 | 2 | 402 | 502 -- Create new chunk (time = 2), insert in different order COPY table_with_layout_change (time, value5, value4, value3, value2) FROM STDIN DELIMITER ',' NULL AS 'null'; COPY table_with_layout_change (value5, value4, value3, value2, time) FROM STDIN DELIMITER ',' NULL AS 'null'; COPY table_with_layout_change (value5, value4, value3, time, value2) FROM STDIN DELIMITER ',' NULL AS 'null'; SELECT * FROM table_with_layout_change ORDER BY time, value2, value3, value4, value5; value2 | value3 | time | value4 | value5 --------+--------+------+--------+-------- 200 | 300 | 1 | 400 | 500 201 | 301 | 1 | 401 | 501 202 | 302 | 2 | 402 | 502 203 | 303 | 2 | 403 | 503 204 | 304 | 2 | 404 | 504 205 | 305 | 2 | 405 | 505 -- Drop the last attribute and add a new one ALTER TABLE table_with_layout_change DROP COLUMN value5; ALTER TABLE table_with_layout_change ADD COLUMN value6 double precision NOT NULL default 600; SELECT * FROM table_with_layout_change ORDER BY time, value2, value3, value4, value6; value2 | value3 | time | value4 | value6 --------+--------+------+--------+-------- 200 | 300 | 1 | 400 | 600 201 | 301 | 1 | 401 | 600 202 | 302 | 2 | 402 | 600 203 | 303 | 2 | 403 | 600 204 | 304 | 2 | 404 | 600 205 | 305 | 2 | 405 | 600 -- COPY in first chunk (time = 1) COPY table_with_layout_change (time, value2, value3, value4, value6) FROM STDIN DELIMITER ',' NULL AS 'null'; -- COPY in second chunk (time = 2) COPY table_with_layout_change (time, value2, value3, value4, value6) FROM STDIN DELIMITER ',' NULL AS 'null'; -- COPY in new chunk (time = 3) COPY table_with_layout_change (time, value2, value3, value4, value6) FROM STDIN DELIMITER ',' NULL AS 'null'; -- COPY in all chunks, different attribute order COPY table_with_layout_change (value3, value4, time, value6, value2) FROM STDIN DELIMITER ',' NULL AS 'null'; SELECT * FROM table_with_layout_change ORDER BY time, value2, value3, value4, value6; value2 | value3 | time | value4 | value6 --------+--------+------+--------+-------- 200 | 300 | 1 | 400 | 600 201 | 301 | 1 | 401 | 600 206 | 306 | 1 | 406 | 606 211 | 311 | 1 | 411 | 611 202 | 302 | 2 | 402 | 600 203 | 303 | 2 | 403 | 600 204 | 304 | 2 | 404 | 600 205 | 305 | 2 | 405 | 600 207 | 307 | 2 | 407 | 607 210 | 310 | 2 | 410 | 610 208 | 308 | 3 | 408 | 608 209 | 309 | 3 | 409 | 609 -- Drop first column ALTER TABLE table_with_layout_change DROP COLUMN value2; SELECT * FROM table_with_layout_change ORDER BY time, value3, value4, value6; value3 | time | value4 | value6 --------+------+--------+-------- 300 | 1 | 400 | 600 301 | 1 | 401 | 600 306 | 1 | 406 | 606 311 | 1 | 411 | 611 302 | 2 | 402 | 600 303 | 2 | 403 | 600 304 | 2 | 404 | 600 305 | 2 | 405 | 600 307 | 2 | 407 | 607 310 | 2 | 410 | 610 308 | 3 | 408 | 608 309 | 3 | 409 | 609 -- COPY in all exiting chunks and create a new one (time 4) COPY table_with_layout_change (value3, value4, time, value6) FROM STDIN DELIMITER ',' NULL AS 'null'; SELECT * FROM table_with_layout_change ORDER BY time, value3, value4, value6; value3 | time | value4 | value6 --------+------+--------+-------- 300 | 1 | 400 | 600 301 | 1 | 401 | 600 306 | 1 | 406 | 606 311 | 1 | 411 | 611 315 | 1 | 415 | 615 302 | 2 | 402 | 600 303 | 2 | 403 | 600 304 | 2 | 404 | 600 305 | 2 | 405 | 600 307 | 2 | 407 | 607 310 | 2 | 410 | 610 313 | 2 | 413 | 613 308 | 3 | 408 | 608 309 | 3 | 409 | 609 312 | 3 | 412 | 612 314 | 4 | 414 | 614 -- Drop the last two columns ALTER TABLE table_with_layout_change DROP COLUMN value4; ALTER TABLE table_with_layout_change DROP COLUMN value6; -- COPY in all exiting chunks and create a new one (time 5) COPY table_with_layout_change (value3, time) FROM STDIN DELIMITER ',' NULL AS 'null'; SELECT * FROM table_with_layout_change ORDER BY time, value3; value3 | time --------+------ 300 | 1 301 | 1 306 | 1 311 | 1 315 | 1 317 | 1 302 | 2 303 | 2 304 | 2 305 | 2 307 | 2 310 | 2 313 | 2 316 | 2 308 | 3 309 | 3 312 | 3 318 | 3 314 | 4 320 | 4 319 | 5 -- Drop the last of the initial attributes and add a new one ALTER TABLE table_with_layout_change DROP COLUMN value3; ALTER TABLE table_with_layout_change ADD COLUMN value7 double precision NOT NULL default 700; -- COPY in all exiting chunks and create a new one (time 6) COPY table_with_layout_change (value7, time) FROM STDIN DELIMITER ',' NULL AS 'null'; SELECT * FROM table_with_layout_change ORDER BY time, value7; time | value7 ------+-------- 1 | 700 1 | 700 1 | 700 1 | 700 1 | 700 1 | 700 1 | 722 2 | 700 2 | 700 2 | 700 2 | 700 2 | 700 2 | 700 2 | 700 2 | 700 2 | 721 3 | 700 3 | 700 3 | 700 3 | 700 3 | 723 4 | 700 4 | 700 4 | 726 5 | 700 5 | 724 6 | 725 -- verify check constraints work CREATE TABLE test_check(a INT, b TIMESTAMPTZ); ALTER TABLE test_check ADD CONSTRAINT c1 CHECK (a > 7); SELECT table_name FROM create_hypertable('test_check', 'b'); table_name ------------ test_check COPY test_check(a,b) FROM STDIN (delimiter ',', null 'N'); \set ON_ERROR_STOP 0 COPY test_check(a,b) FROM STDIN (delimiter ',', null 'N'); ERROR: new row for relation "_hyper_13_832_chunk" violates check constraint "c1" \set ON_ERROR_STOP 1 ================================================ FILE: test/expected/copy_memory_usage.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test that transaction memory usage with COPY doesn't grow. -- We need memory usage in PortalContext after the completion of the query, so -- we'll have to log it from a trigger that runs after the query is completed. \c :TEST_DBNAME :ROLE_SUPERUSER; create table uk_price_paid(price integer, "date" date, postcode1 text, postcode2 text, type smallint, is_new bool, duration smallint, addr1 text, addr2 text, street text, locality text, town text, district text, country text, category smallint); -- Aim to about 100 partitions, the data is from 1995 to 2022. select create_hypertable('uk_price_paid', 'date', chunk_time_interval => interval '90 day'); create_hypertable ---------------------------- (1,public,uk_price_paid,t) -- This is where we log the memory usage. create table portal_memory_log(id serial, bytes bigint); -- Returns the amount of memory currently allocated in a given -- memory context. Only works for PortalContext, and doesn't work for PG 12. create or replace function ts_debug_allocated_bytes(text) returns bigint as :MODULE_PATHNAME, 'ts_debug_allocated_bytes' language c strict volatile; -- Log current memory usage into the log table. create function log_memory() returns trigger as $$ begin insert into portal_memory_log values (default, ts_debug_allocated_bytes('PortalContext')); return new; end; $$ language plpgsql; -- Prepare version dependent TopTransactionContext total memory usage query. -- Using prepared statements to avoid contaminating memory usage numbers. -- PG18 removed parent column so we have to use path to get TopTransactionContext child entries. -- https://github.com/postgres/postgres/commit/f0d11275 create or replace function prepare_transaction_total_memory_usage_stmt() returns void language plpgsql as $$ begin if current_setting('server_version_num')::int < 180000 then prepare total_stmt as select sum(total_bytes) from pg_backend_memory_contexts where parent = 'TopTransactionContext'; else prepare total_stmt as select sum(m.total_bytes) from pg_backend_memory_contexts m inner join pg_backend_memory_contexts p on (m.path[m.level-1] = p.path[p.level]) where p.name = 'TopTransactionContext'; end if; end; $$; -- Add a trigger that runs after completion of each INSERT/COPY and logs the -- current memory usage. create trigger check_update after insert on uk_price_paid for each statement execute function log_memory(); -- Memory leaks often happen on cache invalidation, so make sure they are -- invalidated often and independently (at co-prime periods). set timescaledb.max_open_chunks_per_insert = 2; set timescaledb.max_cached_chunks_per_hypertable = 3; -- Try increasingly larger data sets by concatenating the same file multiple -- times. \copy uk_price_paid from program 'bash -c "cat <(zcat < data/prices-10k-random-1.tsv.gz)"'; \copy uk_price_paid from program 'bash -c "cat <(zcat < data/prices-10k-random-1.tsv.gz) <(zcat < data/prices-10k-random-1.tsv.gz)"'; \copy uk_price_paid from program 'bash -c "cat <(zcat < data/prices-10k-random-1.tsv.gz) <(zcat < data/prices-10k-random-1.tsv.gz) <(zcat < data/prices-10k-random-1.tsv.gz)"'; \copy uk_price_paid from program 'bash -c "cat <(zcat < data/prices-10k-random-1.tsv.gz) <(zcat < data/prices-10k-random-1.tsv.gz) <(zcat < data/prices-10k-random-1.tsv.gz) <(zcat < data/prices-10k-random-1.tsv.gz)"'; \copy uk_price_paid from program 'bash -c "cat <(zcat < data/prices-10k-random-1.tsv.gz) <(zcat < data/prices-10k-random-1.tsv.gz) <(zcat < data/prices-10k-random-1.tsv.gz) <(zcat < data/prices-10k-random-1.tsv.gz) <(zcat < data/prices-10k-random-1.tsv.gz)"'; select count(*) from portal_memory_log; count ------- 5 -- Check that the memory doesn't increase with file size by using linear regression. select * from portal_memory_log where ( select regr_slope(bytes, id - 1) / regr_intercept(bytes, id - 1)::float > 0.05 from portal_memory_log ); id | bytes ----+------- -- Test plpgsql leaks CREATE TABLE test_ht(tm timestamptz, val float8); SELECT * FROM create_hypertable('test_ht', 'tm'); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 2 | public | test_ht | t -- Use a plpgsql function to insert into the hypertable CREATE OR REPLACE FUNCTION to_double(_in text, INOUT _out double precision) LANGUAGE plpgsql IMMUTABLE parallel safe AS $$ BEGIN SELECT CAST(_in AS double precision) INTO _out; EXCEPTION WHEN others THEN --do nothing: _out already carries default END; $$; -- TopTransactionContext usage needs to remain the same after every insert -- There was a leak earlier in the child CurTransactionContext SELECT prepare_transaction_total_memory_usage_stmt(); prepare_transaction_total_memory_usage_stmt --------------------------------------------- BEGIN; INSERT INTO test_ht VALUES ('1980-01-01 00:00:00-00', to_double('23.11', 0)); EXECUTE total_stmt; sum ------- 16384 INSERT INTO test_ht VALUES ('1980-02-01 00:00:00-00', to_double('24.11', 0)); EXECUTE total_stmt; sum ------- 16384 COMMIT; ================================================ FILE: test/expected/copy_where.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. ------- TEST 1: Restrictive copy from file CREATE TABLE "copy_golden" ( "time" bigint NOT NULL, "value" double precision NOT NULL ); \COPY copy_golden (time, value) FROM data/copy_data.csv WITH CSV HEADER SELECT * FROM copy_golden ORDER BY TIME; time | value ------+-------------------- 1 | 0.951734602451324 2 | 0.717823888640851 3 | 0.543408489786088 4 | 0.641131274402142 5 | 0.12689296528697 6 | 0.0126486560329795 7 | 0.213605496101081 8 | 0.132784110959619 9 | 0.381155731156468 10 | 0.284836102742702 11 | 0.795640022493899 12 | 0.631451691035181 13 | 0.0958626130595803 14 | 0.929304684977978 15 | 0.524866581428796 16 | 0.919249163009226 17 | 0.878917074296623 18 | 0.68551931809634 19 | 0.594833800103515 20 | 0.819584367796779 21 | 0.474171321373433 22 | 0.938535195309669 23 | 0.333933369256556 24 | 0.274582070298493 25 | 0.602348630782217 CREATE TABLE "copy_control" ( "time" bigint NOT NULL, "value" double precision NOT NULL ); \COPY copy_control (time, value) FROM data/copy_data.csv WITH CSV HEADER WHERE time > 10; SELECT * FROM copy_control ORDER BY TIME; time | value ------+-------------------- 11 | 0.795640022493899 12 | 0.631451691035181 13 | 0.0958626130595803 14 | 0.929304684977978 15 | 0.524866581428796 16 | 0.919249163009226 17 | 0.878917074296623 18 | 0.68551931809634 19 | 0.594833800103515 20 | 0.819584367796779 21 | 0.474171321373433 22 | 0.938535195309669 23 | 0.333933369256556 24 | 0.274582070298493 25 | 0.602348630782217 CREATE TABLE "copy_test" ( "time" bigint NOT NULL, "value" double precision NOT NULL ); SELECT create_hypertable('copy_test', 'time', chunk_time_interval => 10); create_hypertable ------------------------ (1,public,copy_test,t) \COPY copy_test (time, value) FROM data/copy_data.csv WITH CSV HEADER WHERE time > 10; SELECT * FROM copy_test ORDER BY TIME; time | value ------+-------------------- 11 | 0.795640022493899 12 | 0.631451691035181 13 | 0.0958626130595803 14 | 0.929304684977978 15 | 0.524866581428796 16 | 0.919249163009226 17 | 0.878917074296623 18 | 0.68551931809634 19 | 0.594833800103515 20 | 0.819584367796779 21 | 0.474171321373433 22 | 0.938535195309669 23 | 0.333933369256556 24 | 0.274582070298493 25 | 0.602348630782217 -- Verify attempting to use subqueries fails the same as non-hypertables \set ON_ERROR_STOP 0 \COPY copy_control (time, value) FROM data/copy_data.csv WITH CSV HEADER WHERE time IN (SELECT time FROM copy_golden); ERROR: cannot use subquery in COPY FROM WHERE condition at character 74 \COPY copy_test (time, value) FROM data/copy_data.csv WITH CSV HEADER WHERE time IN (SELECT time FROM copy_golden); ERROR: cannot use subquery in COPY FROM WHERE condition at character 71 \set ON_ERROR_STOP 1 DROP TABLE copy_golden; DROP TABLE copy_control; DROP TABLE copy_test; ================================================ FILE: test/expected/create_chunks.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- -- This test will create chunks in two dimenisions, time (x) and -- space (y), where the time dimension is aligned. The figure below -- shows the expected result. The chunk number in the figure -- indicates the creation order. -- -- + -- + -- + +-----+ +-----+ -- + | 2 | | 3 | -- + | +---+-+ | -- + +-----+ 5 |6+-----+ -- + | 1 +---+-+-----+ +---------+ -- + | | |4| 7 | | 8 | -- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+- -- 0 5 10 15 20 -- -- Partitioning: -- -- Chunk # | time | space -- 1 | 3 | 2 -- 4 | 1 | 3 -- 5 | 5 | 3 -- CREATE TABLE chunk_test(time integer, temp float8, tag integer, color integer); SELECT create_hypertable('chunk_test', 'time', 'tag', 2, chunk_time_interval => 3); create_hypertable ------------------------- (1,public,chunk_test,t) INSERT INTO chunk_test VALUES (4, 24.3, 1, 1); SELECT * FROM _timescaledb_catalog.dimension_slice; id | dimension_id | range_start | range_end ----+--------------+----------------------+------------ 1 | 1 | 3 | 6 2 | 2 | -9223372036854775808 | 1073741823 INSERT INTO chunk_test VALUES (4, 24.3, 2, 1); INSERT INTO chunk_test VALUES (10, 24.3, 2, 1); SELECT c.table_name AS chunk_name, d.id AS dimension_id, ds.id AS slice_id, range_start, range_end FROM _timescaledb_catalog.chunk c LEFT JOIN _timescaledb_catalog.chunk_constraint cc ON (c.id = cc.chunk_id) LEFT JOIN _timescaledb_catalog.dimension_slice ds ON (ds.id = cc.dimension_slice_id) LEFT JOIN _timescaledb_catalog.dimension d ON (d.id = ds.dimension_id) LEFT JOIN _timescaledb_catalog.hypertable h ON (d.hypertable_id = h.id) WHERE h.schema_name = 'public' AND h.table_name = 'chunk_test' ORDER BY c.id, d.id; chunk_name | dimension_id | slice_id | range_start | range_end ------------------+--------------+----------+----------------------+--------------------- _hyper_1_1_chunk | 1 | 1 | 3 | 6 _hyper_1_1_chunk | 2 | 2 | -9223372036854775808 | 1073741823 _hyper_1_2_chunk | 1 | 1 | 3 | 6 _hyper_1_2_chunk | 2 | 3 | 1073741823 | 9223372036854775807 _hyper_1_3_chunk | 1 | 4 | 9 | 12 _hyper_1_3_chunk | 2 | 3 | 1073741823 | 9223372036854775807 \c :TEST_DBNAME :ROLE_SUPERUSER SELECT set_number_partitions('chunk_test', 3); set_number_partitions ----------------------- \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT set_chunk_time_interval('chunk_test', 1::bigint); set_chunk_time_interval ------------------------- INSERT INTO chunk_test VALUES (8, 24.3, 11233, 1); SELECT set_chunk_time_interval('chunk_test', 5::bigint); set_chunk_time_interval ------------------------- SELECT * FROM _timescaledb_catalog.dimension; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+-------------+---------+------------+--------------------------+--------------------+-----------------+--------------------------+-------------------------+------------------ 2 | 1 | tag | integer | f | 3 | _timescaledb_functions | get_partition_hash | | | | 1 | 1 | time | integer | t | | | | 5 | | | INSERT INTO chunk_test VALUES (7, 24.3, 79669, 1); INSERT INTO chunk_test VALUES (8, 24.3, 79669, 1); INSERT INTO chunk_test VALUES (10, 24.3, 11233, 1); INSERT INTO chunk_test VALUES (16, 24.3, 11233, 1); SELECT c.table_name AS chunk_name, d.id AS dimension_id, ds.id AS slice_id, range_start, range_end FROM _timescaledb_catalog.chunk c LEFT JOIN _timescaledb_catalog.chunk_constraint cc ON (c.id = cc.chunk_id) LEFT JOIN _timescaledb_catalog.dimension_slice ds ON (ds.id = cc.dimension_slice_id) LEFT JOIN _timescaledb_catalog.dimension d ON (d.id = ds.dimension_id) LEFT JOIN _timescaledb_catalog.hypertable h ON (d.hypertable_id = h.id) WHERE h.schema_name = 'public' AND h.table_name = 'chunk_test' ORDER BY c.id, d.id; chunk_name | dimension_id | slice_id | range_start | range_end ------------------+--------------+----------+----------------------+--------------------- _hyper_1_1_chunk | 1 | 1 | 3 | 6 _hyper_1_1_chunk | 2 | 2 | -9223372036854775808 | 1073741823 _hyper_1_2_chunk | 1 | 1 | 3 | 6 _hyper_1_2_chunk | 2 | 3 | 1073741823 | 9223372036854775807 _hyper_1_3_chunk | 1 | 4 | 9 | 12 _hyper_1_3_chunk | 2 | 3 | 1073741823 | 9223372036854775807 _hyper_1_4_chunk | 1 | 5 | 8 | 9 _hyper_1_4_chunk | 2 | 6 | -9223372036854775808 | 715827882 _hyper_1_5_chunk | 1 | 7 | 6 | 8 _hyper_1_5_chunk | 2 | 8 | 715827882 | 1431655764 _hyper_1_6_chunk | 1 | 5 | 8 | 9 _hyper_1_6_chunk | 2 | 8 | 715827882 | 1431655764 _hyper_1_7_chunk | 1 | 4 | 9 | 12 _hyper_1_7_chunk | 2 | 6 | -9223372036854775808 | 715827882 _hyper_1_8_chunk | 1 | 9 | 15 | 20 _hyper_1_8_chunk | 2 | 6 | -9223372036854775808 | 715827882 --test the edges of an open partition -- INT_64_MAX and INT_64_MIN. CREATE TABLE chunk_test_ends(time bigint, temp float8, tag integer, color integer); SELECT create_hypertable('chunk_test_ends', 'time', chunk_time_interval => 5); create_hypertable ------------------------------ (2,public,chunk_test_ends,t) INSERT INTO chunk_test_ends VALUES ((-9223372036854775808)::bigint, 23.1, 11233, 1); INSERT INTO chunk_test_ends VALUES (9223372036854775807::bigint, 24.1, 11233, 1); --try to hit cache INSERT INTO chunk_test_ends VALUES (9223372036854775807::bigint, 24.2, 11233, 1); INSERT INTO chunk_test_ends VALUES (9223372036854775807::bigint, 24.3, 11233, 1), (9223372036854775807::bigint, 24.4, 11233, 1); INSERT INTO chunk_test_ends VALUES ((-9223372036854775808)::bigint, 23.2, 11233, 1); INSERT INTO chunk_test_ends VALUES ((-9223372036854775808)::bigint, 23.3, 11233, 1), ((-9223372036854775808)::bigint, 23.4, 11233, 1); SELECT * FROM chunk_test_ends ORDER BY time asc, tag, temp; time | temp | tag | color ----------------------+------+-------+------- -9223372036854775808 | 23.1 | 11233 | 1 -9223372036854775808 | 23.2 | 11233 | 1 -9223372036854775808 | 23.3 | 11233 | 1 -9223372036854775808 | 23.4 | 11233 | 1 9223372036854775807 | 24.1 | 11233 | 1 9223372036854775807 | 24.2 | 11233 | 1 9223372036854775807 | 24.3 | 11233 | 1 9223372036854775807 | 24.4 | 11233 | 1 --further tests of set_chunk_time_interval CREATE TABLE chunk_test2(time TIMESTAMPTZ, temp float8, tag integer, color integer); SELECT create_hypertable('chunk_test2', 'time', 'tag', 2, chunk_time_interval => 3); WARNING: unexpected interval: smaller than one second create_hypertable -------------------------- (3,public,chunk_test2,t) SELECT interval_length FROM _timescaledb_catalog.dimension d LEFT JOIN _timescaledb_catalog.hypertable h ON (d.hypertable_id = h.id) WHERE h.schema_name = 'public' AND h.table_name = 'chunk_test2' ORDER BY d.id; interval_length ----------------- 3 -- should work since time column is non-INT SELECT set_chunk_time_interval('chunk_test2', INTERVAL '1 minute'); set_chunk_time_interval ------------------------- SELECT interval_length FROM _timescaledb_catalog.dimension d LEFT JOIN _timescaledb_catalog.hypertable h ON (d.hypertable_id = h.id) WHERE h.schema_name = 'public' AND h.table_name = 'chunk_test2' ORDER BY d.id; interval_length ----------------- 60000000 -- should still work for non-INT time columns SELECT set_chunk_time_interval('chunk_test2', 1000000); set_chunk_time_interval ------------------------- SELECT interval_length FROM _timescaledb_catalog.dimension d LEFT JOIN _timescaledb_catalog.hypertable h ON (d.hypertable_id = h.id) WHERE h.schema_name = 'public' AND h.table_name = 'chunk_test2' ORDER BY d.id; interval_length ----------------- 1000000 \set ON_ERROR_STOP 0 select set_chunk_time_interval(NULL,NULL::interval); ERROR: hypertable cannot be NULL -- should fail since time column is an int SELECT set_chunk_time_interval('chunk_test', INTERVAL '1 minute'); ERROR: invalid interval type for integer dimension -- should fail since its not a valid way to represent time SELECT set_chunk_time_interval('chunk_test', 'foo'::TEXT); ERROR: invalid interval type for integer dimension SELECT set_chunk_time_interval('chunk_test', NULL::BIGINT); ERROR: invalid interval: an explicit interval must be specified SELECT set_chunk_time_interval('chunk_test2', NULL::BIGINT); ERROR: invalid interval: an explicit interval must be specified SELECT set_chunk_time_interval('chunk_test2', NULL::INTERVAL); ERROR: invalid interval: an explicit interval must be specified \set ON_ERROR_STOP 1 -- Issue https://github.com/timescale/timescaledb/issues/7406 CREATE TABLE test_ht (time TIMESTAMPTZ, v1 INTEGER); SELECT create_hypertable('test_ht', by_range('time', INTERVAL '1 hour')); create_hypertable ------------------- (4,t) CREATE TABLE test_tb (time TIMESTAMPTZ, v1 INTEGER); CREATE OR REPLACE FUNCTION test_tb_trg_insert() RETURNS TRIGGER AS $$ BEGIN INSERT INTO test_ht VALUES (NEW.time, NEW.v1); RETURN NEW; END; $$ LANGUAGE plpgsql; CREATE TRIGGER test_tb_trg_insert AFTER INSERT ON test_tb FOR EACH ROW EXECUTE FUNCTION test_tb_trg_insert(); -- Creating new chunk inside a trigger called by -- a DDL statement should not fail. CREATE TABLE test_output AS WITH inserted AS ( INSERT INTO test_tb VALUES (NOW(), 1), (NOW(), 2) RETURNING * ) SELECT * FROM inserted; -- Check the DEFAULT REPLICA IDENTITY of the chunks SELECT relname, relreplident FROM show_chunks('test_ht') ch JOIN pg_class c ON (ch = c.oid) ORDER BY relname; relname | relreplident -------------------+-------------- _hyper_4_11_chunk | d -- Clean up TRUNCATE test_ht, test_tb; DROP TABLE test_output; -- Change the DEFAULT REPLICA IDENTITY of the chunks ALTER TABLE test_ht REPLICA IDENTITY FULL; -- Internally we force new chunks have the same REPLICA IDENTITY -- as the parent table. CREATE TABLE test_output AS WITH inserted AS ( INSERT INTO test_tb VALUES (NOW(), 1), (NOW(), 2) RETURNING * ) SELECT * FROM inserted; -- Check current new REPLICA IDENTITY FULL in the chunks SELECT relname, relreplident FROM show_chunks('test_ht') ch JOIN pg_class c ON (ch = c.oid) ORDER BY relname; relname | relreplident -------------------+-------------- _hyper_4_12_chunk | f -- All tables should have the same number of rows SELECT count(*) FROM test_tb; count ------- 2 SELECT count(*) FROM test_ht; count ------- 2 SELECT count(*) FROM test_output; count ------- 2 -- test ALTER TABLE SET (tsdb.chunk_interval) on a hypertable CREATE TABLE t_with(time timestamptz not null, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 't_with'; time_interval --------------- @ 7 days ALTER TABLE t_with SET (tsdb.chunk_interval = '1 hour'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 't_with'; time_interval --------------- @ 1 hour ================================================ FILE: test/expected/create_hypertable.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER create schema test_schema AUTHORIZATION :ROLE_DEFAULT_PERM_USER; create schema chunk_schema AUTHORIZATION :ROLE_DEFAULT_PERM_USER_2; SET ROLE :ROLE_DEFAULT_PERM_USER; create table test_schema.test_table(time BIGINT, temp float8, device_id text, device_type text, location text, id int, id2 int); \set ON_ERROR_STOP 0 -- get_create_command should fail since hypertable isn't made yet SELECT * FROM _timescaledb_functions.get_create_command('test_table'); ERROR: hypertable "test_table" not found \set ON_ERROR_STOP 1 SELECT * FROM test.relation WHERE schema = 'test_schema'; schema | name | type | owner -------------+------------+-------+------------------- test_schema | test_table | table | default_perm_user \d _timescaledb_catalog.chunk Table "_timescaledb_catalog.chunk" Column | Type | Collation | Nullable | Default ---------------------+--------------------------+-----------+----------+-------------------------------------------------------- id | integer | | not null | nextval('_timescaledb_catalog.chunk_id_seq'::regclass) hypertable_id | integer | | not null | schema_name | name | | not null | table_name | name | | not null | compressed_chunk_id | integer | | | status | integer | | not null | 0 osm_chunk | boolean | | not null | false creation_time | timestamp with time zone | | not null | Indexes: "chunk_pkey" PRIMARY KEY, btree (id) "chunk_compressed_chunk_id_idx" btree (compressed_chunk_id) "chunk_hypertable_id_creation_time_idx" btree (hypertable_id, creation_time) "chunk_hypertable_id_idx" btree (hypertable_id) "chunk_osm_chunk_idx" btree (osm_chunk, hypertable_id) "chunk_schema_name_table_name_key" UNIQUE CONSTRAINT, btree (schema_name, table_name) Foreign-key constraints: "chunk_compressed_chunk_id_fkey" FOREIGN KEY (compressed_chunk_id) REFERENCES _timescaledb_catalog.chunk(id) "chunk_hypertable_id_fkey" FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable(id) Referenced by: TABLE "_timescaledb_internal.bgw_policy_chunk_stats" CONSTRAINT "bgw_policy_chunk_stats_chunk_id_fkey" FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE TABLE "_timescaledb_catalog.chunk_column_stats" CONSTRAINT "chunk_column_stats_chunk_id_fkey" FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id) TABLE "_timescaledb_catalog.chunk" CONSTRAINT "chunk_compressed_chunk_id_fkey" FOREIGN KEY (compressed_chunk_id) REFERENCES _timescaledb_catalog.chunk(id) TABLE "_timescaledb_catalog.chunk_constraint" CONSTRAINT "chunk_constraint_chunk_id_fkey" FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id) TABLE "_timescaledb_catalog.compression_chunk_size" CONSTRAINT "compression_chunk_size_chunk_id_fkey" FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE TABLE "_timescaledb_catalog.compression_chunk_size" CONSTRAINT "compression_chunk_size_compressed_chunk_id_fkey" FOREIGN KEY (compressed_chunk_id) REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE create table test_schema.test_table_no_not_null(time BIGINT, device_id text); \set ON_ERROR_STOP 0 -- Permission denied with unprivileged role SET ROLE :ROLE_DEFAULT_PERM_USER_2; select * from create_hypertable('test_schema.test_table_no_not_null', 'time', 'device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); ERROR: permission denied for schema test_schema at character 33 -- CREATE on schema is not enough SET ROLE :ROLE_DEFAULT_PERM_USER; GRANT ALL ON SCHEMA test_schema TO :ROLE_DEFAULT_PERM_USER_2; SET ROLE :ROLE_DEFAULT_PERM_USER_2; select * from create_hypertable('test_schema.test_table_no_not_null', 'time', 'device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); ERROR: must be owner of hypertable "test_table_no_not_null" \set ON_ERROR_STOP 1 -- Should work with when granted table owner role RESET ROLE; GRANT :ROLE_DEFAULT_PERM_USER TO :ROLE_DEFAULT_PERM_USER_2; SET ROLE :ROLE_DEFAULT_PERM_USER_2; select * from create_hypertable('test_schema.test_table_no_not_null', 'time', 'device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+------------------------+--------- 1 | test_schema | test_table_no_not_null | t \set ON_ERROR_STOP 0 insert into test_schema.test_table_no_not_null (device_id) VALUES('foo'); ERROR: NULL value in column "time" violates not-null constraint \set ON_ERROR_STOP 1 insert into test_schema.test_table_no_not_null (time, device_id) VALUES(1, 'foo'); RESET ROLE; SET ROLE :ROLE_DEFAULT_PERM_USER; \set ON_ERROR_STOP 0 -- No permissions on associated schema should fail select * from create_hypertable('test_schema.test_table', 'time', 'device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month'), associated_schema_name => 'chunk_schema'); ERROR: permissions denied: cannot create chunks in schema "chunk_schema" \set ON_ERROR_STOP 1 -- Granting permissions on chunk_schema should make things work RESET ROLE; GRANT CREATE ON SCHEMA chunk_schema TO :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; select * from create_hypertable('test_schema.test_table', 'time', 'device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month'), associated_schema_name => 'chunk_schema'); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 2 | test_schema | test_table | t -- Check that the insert block trigger exists SELECT * FROM test.show_triggers('test_schema.test_table'); Trigger | Type | Function ---------+------+---------- SELECT * FROM _timescaledb_functions.get_create_command('test_table'); get_create_command -------------------------------------------------------------------------------------------------------------------------------------------------- SELECT create_hypertable('test_schema.test_table', 'time', 'device_id', 2, chunk_time_interval => 2592000000000, create_default_indexes=>FALSE); --test adding one more closed dimension select add_dimension('test_schema.test_table', 'location', 4); add_dimension --------------------------------------- (5,test_schema,test_table,location,t) select * from _timescaledb_catalog.hypertable where table_name = 'test_table'; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 2 | test_schema | test_table | chunk_schema | _hyper_2 | 3 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 select * from _timescaledb_catalog.dimension; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+-------------+---------+------------+--------------------------+--------------------+-----------------+--------------------------+-------------------------+------------------ 1 | 1 | time | bigint | t | | | | 2592000000000 | | | 2 | 1 | device_id | text | f | 2 | _timescaledb_functions | get_partition_hash | | | | 3 | 2 | time | bigint | t | | | | 2592000000000 | | | 4 | 2 | device_id | text | f | 2 | _timescaledb_functions | get_partition_hash | | | | 5 | 2 | location | text | f | 4 | _timescaledb_functions | get_partition_hash | | | | --test that we can change the number of partitions and that 1 is allowed SELECT set_number_partitions('test_schema.test_table', 1, 'location'); set_number_partitions ----------------------- select * from _timescaledb_catalog.dimension WHERE column_name = 'location'; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+-------------+---------+------------+--------------------------+--------------------+-----------------+--------------------------+-------------------------+------------------ 5 | 2 | location | text | f | 1 | _timescaledb_functions | get_partition_hash | | | | SELECT set_number_partitions('test_schema.test_table', 2, 'location'); set_number_partitions ----------------------- select * from _timescaledb_catalog.dimension WHERE column_name = 'location'; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+-------------+---------+------------+--------------------------+--------------------+-----------------+--------------------------+-------------------------+------------------ 5 | 2 | location | text | f | 2 | _timescaledb_functions | get_partition_hash | | | | \set ON_ERROR_STOP 0 --must give an explicit dimension when there are multiple space dimensions SELECT set_number_partitions('test_schema.test_table', 3); ERROR: hypertable "test_table" has multiple space dimensions --too few SELECT set_number_partitions('test_schema.test_table', 0, 'location'); ERROR: invalid number of partitions: must be between 1 and 32767 -- Too many SELECT set_number_partitions('test_schema.test_table', 32768, 'location'); ERROR: invalid number of partitions: must be between 1 and 32767 -- get_create_command only works on tables w/ 1 or 2 dimensions SELECT * FROM _timescaledb_functions.get_create_command('test_table'); ERROR: get_create_command only supports hypertables with up to 2 dimensions \set ON_ERROR_STOP 1 --test adding one more open dimension select add_dimension('test_schema.test_table', 'id', chunk_time_interval => 1000); add_dimension --------------------------------- (6,test_schema,test_table,id,t) select * from _timescaledb_catalog.hypertable where table_name = 'test_table'; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 2 | test_schema | test_table | chunk_schema | _hyper_2 | 4 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 select * from _timescaledb_catalog.dimension; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+-------------+---------+------------+--------------------------+--------------------+-----------------+--------------------------+-------------------------+------------------ 1 | 1 | time | bigint | t | | | | 2592000000000 | | | 2 | 1 | device_id | text | f | 2 | _timescaledb_functions | get_partition_hash | | | | 3 | 2 | time | bigint | t | | | | 2592000000000 | | | 4 | 2 | device_id | text | f | 2 | _timescaledb_functions | get_partition_hash | | | | 5 | 2 | location | text | f | 2 | _timescaledb_functions | get_partition_hash | | | | 6 | 2 | id | integer | t | | | | 1000 | | | -- Test add_dimension: can use interval types for TIMESTAMPTZ columns CREATE TABLE dim_test_time(time TIMESTAMPTZ, time2 TIMESTAMPTZ, time3 BIGINT, temp float8, device int, location int); SELECT create_hypertable('dim_test_time', 'time'); create_hypertable ---------------------------- (3,public,dim_test_time,t) SELECT add_dimension('dim_test_time', 'time2', chunk_time_interval => INTERVAL '1 day'); add_dimension ---------------------------------- (8,public,dim_test_time,time2,t) -- Test add_dimension: only integral should work on BIGINT columns \set ON_ERROR_STOP 0 SELECT add_dimension('dim_test_time', 'time3', chunk_time_interval => INTERVAL '1 day'); ERROR: invalid interval type for bigint dimension -- string is not a valid type SELECT add_dimension('dim_test_time', 'time3', chunk_time_interval => 'foo'::TEXT); ERROR: invalid interval type for bigint dimension \set ON_ERROR_STOP 1 SELECT add_dimension('dim_test_time', 'time3', chunk_time_interval => 500); add_dimension ---------------------------------- (9,public,dim_test_time,time3,t) -- Test add_dimension: integrals should work on TIMESTAMPTZ columns CREATE TABLE dim_test_time2(time TIMESTAMPTZ, time2 TIMESTAMPTZ, temp float8, device int, location int); SELECT create_hypertable('dim_test_time2', 'time'); create_hypertable ----------------------------- (4,public,dim_test_time2,t) SELECT add_dimension('dim_test_time2', 'time2', chunk_time_interval => 500); WARNING: unexpected interval: smaller than one second add_dimension ------------------------------------ (11,public,dim_test_time2,time2,t) --adding a dimension twice should not fail with 'if_not_exists' SELECT add_dimension('dim_test_time2', 'time2', chunk_time_interval => 500, if_not_exists => true); NOTICE: column "time2" is already a dimension, skipping add_dimension ------------------------------------ (11,public,dim_test_time2,time2,f) \set ON_ERROR_STOP 0 --adding on a non-hypertable CREATE TABLE not_hypertable(time TIMESTAMPTZ, temp float8, device int, location int); SELECT add_dimension('not_hypertable', 'time', chunk_time_interval => 500); ERROR: table "not_hypertable" is not a hypertable --adding a non-exist column SELECT add_dimension('test_schema.test_table', 'nope', 2); ERROR: column "nope" does not exist --adding the same dimension twice should fail select add_dimension('test_schema.test_table', 'location', 2); ERROR: column "location" is already a dimension --adding dimension with both number_partitions and chunk_time_interval should fail select add_dimension('test_schema.test_table', 'id2', number_partitions => 2, chunk_time_interval => 1000); ERROR: cannot specify both the number of partitions and an interval \set ON_ERROR_STOP 1 -- test adding a new dimension on a non-empty table CREATE TABLE dim_test(time TIMESTAMPTZ, device int); SELECT create_hypertable('dim_test', 'time', chunk_time_interval => INTERVAL '1 day'); create_hypertable ----------------------- (5,public,dim_test,t) CREATE VIEW dim_test_slices AS SELECT c.id AS chunk_id, c.hypertable_id, ds.dimension_id, cc.dimension_slice_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN _timescaledb_catalog.dimension td ON (h.id = td.hypertable_id) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = td.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.table_name = 'dim_test' ORDER BY c.id, ds.dimension_id; INSERT INTO dim_test VALUES ('2004-10-10 00:00:00+00', 1); INSERT INTO dim_test VALUES ('2004-10-20 00:00:00+00', 2); SELECT * FROM dim_test_slices; chunk_id | hypertable_id | dimension_id | dimension_slice_id | chunk_schema | chunk_table | range_start | range_end ----------+---------------+--------------+--------------------+-----------------------+------------------+------------------+------------------ 2 | 5 | 12 | 3 | _timescaledb_internal | _hyper_5_2_chunk | 1097366400000000 | 1097452800000000 3 | 5 | 12 | 4 | _timescaledb_internal | _hyper_5_3_chunk | 1098230400000000 | 1098316800000000 SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_5_2_chunk'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated --------------+------+---------+-------+------------------------------------------------------------------------------------------------------------------------------------------------+------------+----------+----------- constraint_3 | c | {time} | - | (("time" >= 'Sat Oct 09 17:00:00 2004 PDT'::timestamp with time zone) AND ("time" < 'Sun Oct 10 17:00:00 2004 PDT'::timestamp with time zone)) | f | f | t SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_5_3_chunk'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated --------------+------+---------+-------+------------------------------------------------------------------------------------------------------------------------------------------------+------------+----------+----------- constraint_4 | c | {time} | - | (("time" >= 'Tue Oct 19 17:00:00 2004 PDT'::timestamp with time zone) AND ("time" < 'Wed Oct 20 17:00:00 2004 PDT'::timestamp with time zone)) | f | f | t -- add dimension to the existing chunks by adding -inf/inf dimension slices SELECT add_dimension('dim_test', 'device', 2); add_dimension ------------------------------- (13,public,dim_test,device,t) SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_5_2_chunk'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated --------------+------+---------+-------+------------------------------------------------------------------------------------------------------------------------------------------------+------------+----------+----------- constraint_3 | c | {time} | - | (("time" >= 'Sat Oct 09 17:00:00 2004 PDT'::timestamp with time zone) AND ("time" < 'Sun Oct 10 17:00:00 2004 PDT'::timestamp with time zone)) | f | f | t SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_5_3_chunk'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated --------------+------+---------+-------+------------------------------------------------------------------------------------------------------------------------------------------------+------------+----------+----------- constraint_4 | c | {time} | - | (("time" >= 'Tue Oct 19 17:00:00 2004 PDT'::timestamp with time zone) AND ("time" < 'Wed Oct 20 17:00:00 2004 PDT'::timestamp with time zone)) | f | f | t SELECT * FROM dim_test_slices; chunk_id | hypertable_id | dimension_id | dimension_slice_id | chunk_schema | chunk_table | range_start | range_end ----------+---------------+--------------+--------------------+-----------------------+------------------+----------------------+--------------------- 2 | 5 | 12 | 3 | _timescaledb_internal | _hyper_5_2_chunk | 1097366400000000 | 1097452800000000 2 | 5 | 13 | 5 | _timescaledb_internal | _hyper_5_2_chunk | -9223372036854775808 | 9223372036854775807 3 | 5 | 12 | 4 | _timescaledb_internal | _hyper_5_3_chunk | 1098230400000000 | 1098316800000000 3 | 5 | 13 | 5 | _timescaledb_internal | _hyper_5_3_chunk | -9223372036854775808 | 9223372036854775807 -- newer chunks have proper dimension slices range INSERT INTO dim_test VALUES ('2004-10-30 00:00:00+00', 3); SELECT * FROM dim_test_slices; chunk_id | hypertable_id | dimension_id | dimension_slice_id | chunk_schema | chunk_table | range_start | range_end ----------+---------------+--------------+--------------------+-----------------------+------------------+----------------------+--------------------- 2 | 5 | 12 | 3 | _timescaledb_internal | _hyper_5_2_chunk | 1097366400000000 | 1097452800000000 2 | 5 | 13 | 5 | _timescaledb_internal | _hyper_5_2_chunk | -9223372036854775808 | 9223372036854775807 3 | 5 | 12 | 4 | _timescaledb_internal | _hyper_5_3_chunk | 1098230400000000 | 1098316800000000 3 | 5 | 13 | 5 | _timescaledb_internal | _hyper_5_3_chunk | -9223372036854775808 | 9223372036854775807 4 | 5 | 12 | 6 | _timescaledb_internal | _hyper_5_4_chunk | 1099094400000000 | 1099180800000000 4 | 5 | 13 | 7 | _timescaledb_internal | _hyper_5_4_chunk | 1073741823 | 9223372036854775807 SELECT * FROM dim_test ORDER BY time; time | device ------------------------------+-------- Sat Oct 09 17:00:00 2004 PDT | 1 Tue Oct 19 17:00:00 2004 PDT | 2 Fri Oct 29 17:00:00 2004 PDT | 3 DROP VIEW dim_test_slices; DROP TABLE dim_test; -- test add_dimension() with existing data on table with space partitioning CREATE TABLE dim_test(time TIMESTAMPTZ, device int, data int); SELECT create_hypertable('dim_test', 'time', 'device', 2, chunk_time_interval => INTERVAL '1 day'); create_hypertable ----------------------- (6,public,dim_test,t) CREATE VIEW dim_test_slices AS SELECT c.id AS chunk_id, c.hypertable_id, ds.dimension_id, cc.dimension_slice_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN _timescaledb_catalog.dimension td ON (h.id = td.hypertable_id) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = td.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.table_name = 'dim_test' ORDER BY c.id, ds.dimension_id; INSERT INTO dim_test VALUES ('2004-10-10 00:00:00+00', 1, 3); INSERT INTO dim_test VALUES ('2004-10-20 00:00:00+00', 2, 2); SELECT * FROM dim_test_slices; chunk_id | hypertable_id | dimension_id | dimension_slice_id | chunk_schema | chunk_table | range_start | range_end ----------+---------------+--------------+--------------------+-----------------------+------------------+----------------------+--------------------- 5 | 6 | 14 | 8 | _timescaledb_internal | _hyper_6_5_chunk | 1097366400000000 | 1097452800000000 5 | 6 | 15 | 9 | _timescaledb_internal | _hyper_6_5_chunk | -9223372036854775808 | 1073741823 6 | 6 | 14 | 10 | _timescaledb_internal | _hyper_6_6_chunk | 1098230400000000 | 1098316800000000 6 | 6 | 15 | 11 | _timescaledb_internal | _hyper_6_6_chunk | 1073741823 | 9223372036854775807 -- new dimension slice will cover full range on existing chunks SELECT add_dimension('dim_test', 'data', 1); add_dimension ----------------------------- (16,public,dim_test,data,t) SELECT * FROM dim_test_slices; chunk_id | hypertable_id | dimension_id | dimension_slice_id | chunk_schema | chunk_table | range_start | range_end ----------+---------------+--------------+--------------------+-----------------------+------------------+----------------------+--------------------- 5 | 6 | 14 | 8 | _timescaledb_internal | _hyper_6_5_chunk | 1097366400000000 | 1097452800000000 5 | 6 | 15 | 9 | _timescaledb_internal | _hyper_6_5_chunk | -9223372036854775808 | 1073741823 5 | 6 | 16 | 12 | _timescaledb_internal | _hyper_6_5_chunk | -9223372036854775808 | 9223372036854775807 6 | 6 | 14 | 10 | _timescaledb_internal | _hyper_6_6_chunk | 1098230400000000 | 1098316800000000 6 | 6 | 15 | 11 | _timescaledb_internal | _hyper_6_6_chunk | 1073741823 | 9223372036854775807 6 | 6 | 16 | 12 | _timescaledb_internal | _hyper_6_6_chunk | -9223372036854775808 | 9223372036854775807 INSERT INTO dim_test VALUES ('2004-10-30 00:00:00+00', 3, 1); SELECT * FROM dim_test_slices; chunk_id | hypertable_id | dimension_id | dimension_slice_id | chunk_schema | chunk_table | range_start | range_end ----------+---------------+--------------+--------------------+-----------------------+------------------+----------------------+--------------------- 5 | 6 | 14 | 8 | _timescaledb_internal | _hyper_6_5_chunk | 1097366400000000 | 1097452800000000 5 | 6 | 15 | 9 | _timescaledb_internal | _hyper_6_5_chunk | -9223372036854775808 | 1073741823 5 | 6 | 16 | 12 | _timescaledb_internal | _hyper_6_5_chunk | -9223372036854775808 | 9223372036854775807 6 | 6 | 14 | 10 | _timescaledb_internal | _hyper_6_6_chunk | 1098230400000000 | 1098316800000000 6 | 6 | 15 | 11 | _timescaledb_internal | _hyper_6_6_chunk | 1073741823 | 9223372036854775807 6 | 6 | 16 | 12 | _timescaledb_internal | _hyper_6_6_chunk | -9223372036854775808 | 9223372036854775807 7 | 6 | 14 | 13 | _timescaledb_internal | _hyper_6_7_chunk | 1099094400000000 | 1099180800000000 7 | 6 | 15 | 11 | _timescaledb_internal | _hyper_6_7_chunk | 1073741823 | 9223372036854775807 7 | 6 | 16 | 12 | _timescaledb_internal | _hyper_6_7_chunk | -9223372036854775808 | 9223372036854775807 SELECT * FROM dim_test ORDER BY time; time | device | data ------------------------------+--------+------ Sat Oct 09 17:00:00 2004 PDT | 1 | 3 Tue Oct 19 17:00:00 2004 PDT | 2 | 2 Fri Oct 29 17:00:00 2004 PDT | 3 | 1 DROP VIEW dim_test_slices; DROP TABLE dim_test; -- should not fail on non-empty table with 'if_not_exists' in case the dimension exists select add_dimension('test_schema.test_table', 'location', 2, if_not_exists => true); NOTICE: column "location" is already a dimension, skipping add_dimension --------------------------------------- (5,test_schema,test_table,location,f) --test partitioning in only time dimension create table test_schema.test_1dim(time timestamp, temp float); select create_hypertable('test_schema.test_1dim', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ----------------------------- (7,test_schema,test_1dim,t) SELECT * FROM _timescaledb_functions.get_create_command('test_1dim'); get_create_command -------------------------------------------------------------------------------------------------------------------------------- SELECT create_hypertable('test_schema.test_1dim', 'time', chunk_time_interval => 604800000000, create_default_indexes=>FALSE); SELECT * FROM test.relation WHERE schema = 'test_schema'; schema | name | type | owner -------------+------------------------+-------+------------------- test_schema | test_1dim | table | default_perm_user test_schema | test_table | table | default_perm_user test_schema | test_table_no_not_null | table | default_perm_user select create_hypertable('test_schema.test_1dim', 'time', if_not_exists => true); NOTICE: table "test_1dim" is already a hypertable, skipping create_hypertable ----------------------------- (7,test_schema,test_1dim,f) -- Should error when creating again without if_not_exists set to true \set ON_ERROR_STOP 0 select create_hypertable('test_schema.test_1dim', 'time'); ERROR: table "test_1dim" is already a hypertable \set ON_ERROR_STOP 1 -- if_not_exist should also work with data in the hypertable insert into test_schema.test_1dim VALUES ('2004-10-19 10:23:54+02', 1.0); select create_hypertable('test_schema.test_1dim', 'time', if_not_exists => true); NOTICE: table "test_1dim" is already a hypertable, skipping create_hypertable ----------------------------- (7,test_schema,test_1dim,f) -- Should error when creating again without if_not_exists set to true \set ON_ERROR_STOP 0 select create_hypertable('test_schema.test_1dim', 'time'); ERROR: table "test_1dim" is already a hypertable \set ON_ERROR_STOP 1 -- Test partitioning functions CREATE OR REPLACE FUNCTION invalid_partfunc(source integer) RETURNS INTEGER LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RETURN NULL; END $BODY$; CREATE OR REPLACE FUNCTION time_partfunc(source text) RETURNS TIMESTAMPTZ LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RETURN timezone('UTC', to_timestamp(source)); END $BODY$; CREATE TABLE test_schema.test_invalid_func(time timestamptz, temp float8, device text); \set ON_ERROR_STOP 0 -- should fail due to invalid signature SELECT create_hypertable('test_schema.test_invalid_func', 'time', 'device', 2, partitioning_func => 'invalid_partfunc'); ERROR: invalid partitioning function SELECT create_hypertable('test_schema.test_invalid_func', 'time'); create_hypertable ------------------------------------- (8,test_schema,test_invalid_func,t) -- should also fail due to invalid signature SELECT add_dimension('test_schema.test_invalid_func', 'device', 2, partitioning_func => 'invalid_partfunc'); ERROR: invalid partitioning function \set ON_ERROR_STOP 1 -- Test open-dimension function CREATE TABLE test_schema.open_dim_part_func(time text, temp float8, device text, event_time text); \set ON_ERROR_STOP 0 -- should fail due to invalid signature SELECT create_hypertable('test_schema.open_dim_part_func', 'time', time_partitioning_func => 'invalid_partfunc'); ERROR: invalid partitioning function \set ON_ERROR_STOP 1 SELECT create_hypertable('test_schema.open_dim_part_func', 'time', time_partitioning_func => 'time_partfunc'); create_hypertable -------------------------------------- (9,test_schema,open_dim_part_func,t) \set ON_ERROR_STOP 0 -- should fail due to invalid signature SELECT add_dimension('test_schema.open_dim_part_func', 'event_time', chunk_time_interval => interval '1 day', partitioning_func => 'invalid_partfunc'); ERROR: invalid partitioning function \set ON_ERROR_STOP 1 SELECT add_dimension('test_schema.open_dim_part_func', 'event_time', chunk_time_interval => interval '1 day', partitioning_func => 'time_partfunc'); add_dimension -------------------------------------------------- (20,test_schema,open_dim_part_func,event_time,t) --test data migration create table test_schema.test_migrate(time timestamp, temp float); insert into test_schema.test_migrate VALUES ('2004-10-19 10:23:54+02', 1.0), ('2004-12-19 10:23:54+02', 2.0); select * from only test_schema.test_migrate; time | temp --------------------------+------ Tue Oct 19 10:23:54 2004 | 1 Sun Dec 19 10:23:54 2004 | 2 \set ON_ERROR_STOP 0 --should fail without migrate_data => true select create_hypertable('test_schema.test_migrate', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices ERROR: table "test_migrate" is not empty \set ON_ERROR_STOP 1 select create_hypertable('test_schema.test_migrate', 'time', migrate_data => true); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices NOTICE: migrating data to chunks create_hypertable --------------------------------- (10,test_schema,test_migrate,t) --there should be two new chunks select * from _timescaledb_catalog.hypertable where table_name = 'test_migrate'; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+--------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 10 | test_schema | test_migrate | _timescaledb_internal | _hyper_10 | 1 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 select id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk from _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-----------------------+--------------------+---------------------+--------+----------- 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | | 0 | f 8 | 7 | _timescaledb_internal | _hyper_7_8_chunk | | 0 | f 9 | 10 | _timescaledb_internal | _hyper_10_9_chunk | | 0 | f 10 | 10 | _timescaledb_internal | _hyper_10_10_chunk | | 0 | f select * from test_schema.test_migrate; time | temp --------------------------+------ Tue Oct 19 10:23:54 2004 | 1 Sun Dec 19 10:23:54 2004 | 2 --main table should now be empty select * from only test_schema.test_migrate; time | temp ------+------ select * from only _timescaledb_internal._hyper_10_9_chunk; time | temp --------------------------+------ Tue Oct 19 10:23:54 2004 | 1 select * from only _timescaledb_internal._hyper_10_10_chunk; time | temp --------------------------+------ Sun Dec 19 10:23:54 2004 | 2 create table test_schema.test_migrate_empty(time timestamp, temp float); select create_hypertable('test_schema.test_migrate_empty', 'time', migrate_data => true); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable --------------------------------------- (11,test_schema,test_migrate_empty,t) CREATE TYPE test_type AS (time timestamp, temp float); CREATE TABLE test_table_of_type OF test_type; SELECT create_hypertable('test_table_of_type', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ---------------------------------- (12,public,test_table_of_type,t) INSERT INTO test_table_of_type VALUES ('2004-10-19 10:23:54+02', 1.0), ('2004-12-19 10:23:54+02', 2.0); \set ON_ERROR_STOP 0 DROP TYPE test_type; ERROR: cannot drop type test_type because other objects depend on it \set ON_ERROR_STOP 1 DROP TYPE test_type CASCADE; NOTICE: drop cascades to 3 other objects CREATE TABLE test_table_of_type (time timestamp, temp float); SELECT create_hypertable('test_table_of_type', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ---------------------------------- (13,public,test_table_of_type,t) INSERT INTO test_table_of_type VALUES ('2004-10-19 10:23:54+02', 1.0), ('2004-12-19 10:23:54+02', 2.0); CREATE TYPE test_type AS (time timestamp, temp float); ALTER TABLE test_table_of_type OF test_type; \set ON_ERROR_STOP 0 DROP TYPE test_type; ERROR: cannot drop type test_type because other objects depend on it \set ON_ERROR_STOP 1 BEGIN; DROP TYPE test_type CASCADE; NOTICE: drop cascades to 3 other objects ROLLBACK; ALTER TABLE test_table_of_type NOT OF; DROP TYPE test_type; -- Reset GRANTS \c :TEST_DBNAME :ROLE_SUPERUSER REVOKE :ROLE_DEFAULT_PERM_USER FROM :ROLE_DEFAULT_PERM_USER_2; -- Test custom partitioning functions CREATE OR REPLACE FUNCTION partfunc_not_immutable(source anyelement) RETURNS INTEGER LANGUAGE PLPGSQL AS $BODY$ BEGIN RETURN _timescaledb_functions.get_partition_hash(source); END $BODY$; CREATE OR REPLACE FUNCTION partfunc_bad_return_type(source anyelement) RETURNS BIGINT LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RETURN _timescaledb_functions.get_partition_hash(source); END $BODY$; CREATE OR REPLACE FUNCTION partfunc_bad_arg_type(source text) RETURNS INTEGER LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RETURN _timescaledb_functions.get_partition_hash(source); END $BODY$; CREATE OR REPLACE FUNCTION partfunc_bad_multi_arg(source anyelement, extra_arg integer) RETURNS INTEGER LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RETURN _timescaledb_functions.get_partition_hash(source); END $BODY$; CREATE OR REPLACE FUNCTION partfunc_valid(source anyelement) RETURNS INTEGER LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RETURN _timescaledb_functions.get_partition_hash(source); END $BODY$; create table test_schema.test_partfunc(time timestamptz, temp float, device int); -- Test that create_hypertable fails due to invalid partitioning function \set ON_ERROR_STOP 0 select create_hypertable('test_schema.test_partfunc', 'time', 'device', 2, partitioning_func => 'partfunc_not_immutable'); ERROR: invalid partitioning function select create_hypertable('test_schema.test_partfunc', 'time', 'device', 2, partitioning_func => 'partfunc_bad_return_type'); ERROR: invalid partitioning function select create_hypertable('test_schema.test_partfunc', 'time', 'device', 2, partitioning_func => 'partfunc_bad_arg_type'); ERROR: invalid partitioning function select create_hypertable('test_schema.test_partfunc', 'time', 'device', 2, partitioning_func => 'partfunc_bad_multi_arg'); ERROR: invalid partitioning function \set ON_ERROR_STOP 1 -- Test that add_dimension fails due to invalid partitioning function select create_hypertable('test_schema.test_partfunc', 'time'); create_hypertable ---------------------------------- (14,test_schema,test_partfunc,t) \set ON_ERROR_STOP 0 select add_dimension('test_schema.test_partfunc', 'device', 2, partitioning_func => 'partfunc_not_immutable'); ERROR: invalid partitioning function select add_dimension('test_schema.test_partfunc', 'device', 2, partitioning_func => 'partfunc_bad_return_type'); ERROR: invalid partitioning function select add_dimension('test_schema.test_partfunc', 'device', 2, partitioning_func => 'partfunc_bad_arg_type'); ERROR: invalid partitioning function select add_dimension('test_schema.test_partfunc', 'device', 2, partitioning_func => 'partfunc_bad_multi_arg'); ERROR: invalid partitioning function \set ON_ERROR_STOP 1 -- A valid function should work: select add_dimension('test_schema.test_partfunc', 'device', 2, partitioning_func => 'partfunc_valid'); add_dimension ----------------------------------------- (26,test_schema,test_partfunc,device,t) -- check get_create_command produces valid command CREATE TABLE test_schema.test_sql_cmd(time TIMESTAMPTZ, temp FLOAT8, device_id TEXT, device_type TEXT, location TEXT, id INT, id2 INT); SELECT create_hypertable('test_schema.test_sql_cmd','time'); create_hypertable --------------------------------- (15,test_schema,test_sql_cmd,t) SELECT * FROM _timescaledb_functions.get_create_command('test_sql_cmd'); get_create_command ----------------------------------------------------------------------------------------------------------------------------------- SELECT create_hypertable('test_schema.test_sql_cmd', 'time', chunk_time_interval => 604800000000, create_default_indexes=>FALSE); SELECT _timescaledb_functions.get_create_command('test_sql_cmd') AS create_cmd; \gset create_cmd ----------------------------------------------------------------------------------------------------------------------------------- SELECT create_hypertable('test_schema.test_sql_cmd', 'time', chunk_time_interval => 604800000000, create_default_indexes=>FALSE); DROP TABLE test_schema.test_sql_cmd CASCADE; CREATE TABLE test_schema.test_sql_cmd(time TIMESTAMPTZ, temp FLOAT8, device_id TEXT, device_type TEXT, location TEXT, id INT, id2 INT); SELECT test.execute_sql(:'create_cmd'); execute_sql ----------------------------------------------------------------------------------------------------------------------------------- SELECT create_hypertable('test_schema.test_sql_cmd', 'time', chunk_time_interval => 604800000000, create_default_indexes=>FALSE); \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE test_table_int(time bigint, junk int); SELECT hypertable_id AS "TEST_TABLE_INT_HYPERTABLE_ID" FROM create_hypertable('test_table_int', 'time', chunk_time_interval => 1) \gset \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA IF NOT EXISTS my_schema; create or replace function my_schema.dummy_now2() returns BIGINT LANGUAGE SQL IMMUTABLE as 'SELECT 1::BIGINT'; grant execute on ALL FUNCTIONS IN SCHEMA my_schema to public; create or replace function dummy_now3() returns BIGINT LANGUAGE SQL IMMUTABLE as 'SELECT 1::BIGINT'; grant execute on ALL FUNCTIONS IN SCHEMA my_schema to public; REVOKE execute ON function dummy_now3() FROM PUBLIC; CREATE SCHEMA IF NOT EXISTS my_user_schema; GRANT ALL ON SCHEMA my_user_schema to PUBLIC; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER create or replace function dummy_now() returns BIGINT LANGUAGE SQL IMMUTABLE as 'SELECT 1::BIGINT'; create or replace function my_user_schema.dummy_now4() returns BIGINT LANGUAGE SQL IMMUTABLE as 'SELECT 1::BIGINT'; select set_integer_now_func('test_table_int', 'dummy_now'); set_integer_now_func ---------------------- select * from _timescaledb_catalog.dimension WHERE hypertable_id = :TEST_TABLE_INT_HYPERTABLE_ID; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+-------------+---------+------------+--------------------------+-------------------+-----------------+--------------------------+-------------------------+------------------ 29 | 17 | time | bigint | t | | | | 1 | | public | dummy_now -- show chunks works with "created_before" and errors out with time used in "older_than" SELECT SHOW_CHUNKS('test_table_int', older_than => 10); show_chunks ------------- SELECT SHOW_CHUNKS('test_table_int', created_before => now()); show_chunks ------------- \set ON_ERROR_STOP 0 SELECT SHOW_CHUNKS('test_table_int', older_than => now()); ERROR: invalid time argument type "timestamp with time zone" select set_integer_now_func('test_table_int', 'dummy_now'); ERROR: custom time function already set for hypertable "test_table_int" select set_integer_now_func('test_table_int', 'my_schema.dummy_now2', replace_if_exists => TRUE); ERROR: permission denied for schema my_schema at character 47 select set_integer_now_func('test_table_int', 'dummy_now3', replace_if_exists => TRUE); ERROR: permission denied for function dummy_now3 -- test invalid oid as the integer_now_func select set_integer_now_func('test_table_int', 1, replace_if_exists => TRUE); ERROR: cache lookup failed for function 1 \set ON_ERROR_STOP select set_integer_now_func('test_table_int', 'my_user_schema.dummy_now4', replace_if_exists => TRUE); set_integer_now_func ---------------------- \c :TEST_DBNAME :ROLE_SUPERUSER ALTER SCHEMA my_user_schema RENAME TO my_new_schema; select * from _timescaledb_catalog.dimension WHERE hypertable_id = :TEST_TABLE_INT_HYPERTABLE_ID; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+-------------+---------+------------+--------------------------+-------------------+-----------------+--------------------------+-------------------------+------------------ 29 | 17 | time | bigint | t | | | | 1 | | my_new_schema | dummy_now4 -- github issue #4650 CREATE TABLE sample_table ( cpu double precision null, time TIMESTAMP WITH TIME ZONE NOT NULL, sensor_id INTEGER NOT NULL, name varchar(100) default 'this is a default string value', UNIQUE(sensor_id, time) ); ALTER TABLE sample_table DROP COLUMN name; -- below creation should not report any warnings. SELECT * FROM create_hypertable('sample_table', 'time'); hypertable_id | schema_name | table_name | created ---------------+-------------+--------------+--------- 18 | public | sample_table | t -- cleanup DROP TABLE sample_table CASCADE; -- github issue 4684 -- test PARTITION BY HASH CREATE TABLE regular( id INT NOT NULL, dev INT NOT NULL, value INT, CONSTRAINT cstr_regular_pky PRIMARY KEY (id) ) PARTITION BY HASH (id); DO $$ BEGIN FOR i IN 1..2 LOOP EXECUTE format(' CREATE TABLE %I PARTITION OF regular FOR VALUES WITH (MODULUS 2, REMAINDER %s)', 'regular_' || i, i - 1 ); END LOOP; END; $$; INSERT INTO regular SELECT generate_series(1,1000), 44,55; CREATE TABLE timescale ( ts TIMESTAMP WITH TIME ZONE NOT NULL, id INT NOT NULL, dev INT NOT NULL, FOREIGN KEY (id) REFERENCES regular(id) ON DELETE CASCADE ); SELECT create_hypertable( relation => 'timescale', time_column_name => 'ts' ); create_hypertable ------------------------- (19,public,timescale,t) -- creates chunk1 INSERT INTO timescale SELECT now(), generate_series(1,200), 43; -- creates chunk2 INSERT INTO timescale SELECT now() + interval '20' day, generate_series(1,200), 43; -- creates chunk3 INSERT INTO timescale SELECT now() + interval '40' day, generate_series(1,200), 43; -- show chunks SELECT SHOW_CHUNKS('timescale'); show_chunks ------------------------------------------ _timescaledb_internal._hyper_19_15_chunk _timescaledb_internal._hyper_19_16_chunk _timescaledb_internal._hyper_19_17_chunk \set ON_ERROR_STOP 0 -- record goes into chunk1 violating FK constraint as value 1001 is not present in regular table INSERT INTO timescale SELECT now(), 1001, 43; ERROR: insert or update on table "_hyper_19_15_chunk" violates foreign key constraint "15_1_timescale_id_fkey" -- record goes into chunk2 violating FK constraint as value 1002 is not present in regular table INSERT INTO timescale SELECT now() + interval '20' day, 1002, 43; ERROR: insert or update on table "_hyper_19_16_chunk" violates foreign key constraint "16_2_timescale_id_fkey" -- record goes into chunk3 violating FK constraint as value 1003 is not present in regular table INSERT INTO timescale SELECT now() + interval '40' day, 1003, 43; ERROR: insert or update on table "_hyper_19_17_chunk" violates foreign key constraint "17_3_timescale_id_fkey" \set ON_ERROR_STOP 1 -- cleanup DROP TABLE regular cascade; NOTICE: drop cascades to 4 other objects DROP TABLE timescale cascade; -- test PARTITION BY RANGE CREATE TABLE regular( id INT NOT NULL, dev INT NOT NULL, value INT, CONSTRAINT cstr_regular_pky PRIMARY KEY (id) ) PARTITION BY RANGE (id); CREATE TABLE regular_1_500 PARTITION OF regular FOR VALUES FROM (1) TO (500); CREATE TABLE regular_500_1000 PARTITION OF regular FOR VALUES FROM (500) TO (801); INSERT INTO regular SELECT generate_series(1,800), 44,55; CREATE TABLE timescale ( ts TIMESTAMP WITH TIME ZONE NOT NULL, id INT NOT NULL, dev INT NOT NULL, FOREIGN KEY (id) REFERENCES regular(id) ON DELETE CASCADE ); SELECT create_hypertable( relation => 'timescale', time_column_name => 'ts' ); create_hypertable ------------------------- (20,public,timescale,t) -- creates chunk1 INSERT INTO timescale SELECT now(), generate_series(1,200), 43; -- creates chunk2 INSERT INTO timescale SELECT now() + interval '20' day, generate_series(200,400), 43; -- creates chunk3 INSERT INTO timescale SELECT now() + interval '40' day, generate_series(400,600), 43; -- show chunks SELECT SHOW_CHUNKS('timescale'); show_chunks ------------------------------------------ _timescaledb_internal._hyper_20_18_chunk _timescaledb_internal._hyper_20_19_chunk _timescaledb_internal._hyper_20_20_chunk \set ON_ERROR_STOP 0 -- FK constraint violation as value 801 is not present in regular table INSERT INTO timescale SELECT now(), 801, 43; ERROR: insert or update on table "_hyper_20_18_chunk" violates foreign key constraint "18_4_timescale_id_fkey" -- FK constraint violation as value 902 is not present in regular table INSERT INTO timescale SELECT now() + interval '20' day, 902, 43; ERROR: insert or update on table "_hyper_20_19_chunk" violates foreign key constraint "19_5_timescale_id_fkey" -- FK constraint violation as value 1003 is not present in regular table INSERT INTO timescale SELECT now() + interval '40' day, 1003, 43; ERROR: insert or update on table "_hyper_20_20_chunk" violates foreign key constraint "20_6_timescale_id_fkey" \set ON_ERROR_STOP 1 -- cleanup DROP TABLE regular cascade; NOTICE: drop cascades to 4 other objects DROP TABLE timescale cascade; -- test PARTITION BY LIST CREATE TABLE regular( id INT NOT NULL, dev INT NOT NULL, value INT, CONSTRAINT cstr_regular_pky PRIMARY KEY (id) ) PARTITION BY LIST (id); CREATE TABLE regular_1_2_3_4 PARTITION OF regular FOR VALUES IN (1,2,3,4); CREATE TABLE regular_5_6_7_8 PARTITION OF regular FOR VALUES IN (5,6,7,8); INSERT INTO regular SELECT generate_series(1,8), 44,55; CREATE TABLE timescale ( ts TIMESTAMP WITH TIME ZONE NOT NULL, id INT NOT NULL, dev INT NOT NULL, FOREIGN KEY (id) REFERENCES regular(id) ON DELETE CASCADE ); SELECT create_hypertable( relation => 'timescale', time_column_name => 'ts' ); create_hypertable ------------------------- (21,public,timescale,t) insert into timescale values (now(), 1,2); insert into timescale values (now(), 2,2); insert into timescale values (now(), 3,2); insert into timescale values (now(), 4,2); insert into timescale values (now(), 5,2); insert into timescale values (now(), 6,2); insert into timescale values (now(), 7,2); insert into timescale values (now(), 8,2); \set ON_ERROR_STOP 0 -- FK constraint violation as value 9 is not present in regular table insert into timescale values (now(), 9,2); ERROR: insert or update on table "_hyper_21_21_chunk" violates foreign key constraint "21_7_timescale_id_fkey" -- FK constraint violation as value 10 is not present in regular table insert into timescale values (now(), 10,2); ERROR: insert or update on table "_hyper_21_21_chunk" violates foreign key constraint "21_7_timescale_id_fkey" -- FK constraint violation as value 111 is not present in regular table insert into timescale values (now(), 111,2); ERROR: insert or update on table "_hyper_21_21_chunk" violates foreign key constraint "21_7_timescale_id_fkey" \set ON_ERROR_STOP 1 -- cleanup DROP TABLE regular cascade; NOTICE: drop cascades to 2 other objects DROP TABLE timescale cascade; -- github issue 4872 -- If subplan of ChunkAppend is TidRangeScan, then SELECT on -- hypertable fails with error "invalid child of chunk append: Node (26)" create table tidrangescan_test ( time timestamp with time zone, some_column bigint ); select create_hypertable('tidrangescan_test', 'time'); create_hypertable --------------------------------- (22,public,tidrangescan_test,t) insert into tidrangescan_test (time, some_column) values ('2023-02-12 00:00:00+02:40', 1); insert into tidrangescan_test (time, some_column) values ('2023-02-12 00:00:10+02:40', 2); insert into tidrangescan_test (time, some_column) values ('2023-02-12 00:00:20+02:40', 3); -- Below query will generate plan as -- Custom Scan (ChunkAppend) -- -> Tid Range Scan -- However when traversing ChunkAppend node, Tid Range Scan node is not -- recognised as a valid child node of ChunkAppend which causes error -- "invalid child of chunk append: Node (26)" when below query is executed select * from tidrangescan_test where time > '2023-02-12 00:00:00+02:40'::timestamp with time zone - interval '5 years' and ctid < '(1,1)'::tid ORDER BY time; time | some_column ------------------------------+------------- Sat Feb 11 13:20:00 2023 PST | 1 Sat Feb 11 13:20:10 2023 PST | 2 Sat Feb 11 13:20:20 2023 PST | 3 drop table tidrangescan_test; \set VERBOSITY default set client_min_messages = WARNING; -- test creating a hypertable from table referenced by a foreign key fails with -- error "cannot have FOREIGN KEY constraints to hypertable". create table test_schema.fk_parent(time timestamptz, id int, unique(time, id)); create table test_schema.fk_child( time timestamptz, id int, foreign key (time, id) references test_schema.fk_parent(time, id) ); select create_hypertable ('test_schema.fk_child', 'time'); create_hypertable ----------------------------- (23,test_schema,fk_child,t) \set ON_ERROR_STOP 0 select create_hypertable ('test_schema.fk_parent', 'time'); ERROR: cannot have FOREIGN KEY constraints to hypertable "fk_parent" HINT: Remove all FOREIGN KEY constraints to table "fk_parent" before making it a hypertable. \set ON_ERROR_STOP 1 -- create default indexes on chunks when migrating data CREATE TABLE test(time TIMESTAMPTZ, val BIGINT); CREATE INDEX test_val_idx ON test(val); INSERT INTO test VALUES('2024-01-01 00:00:00-03', 500); SELECT FROM create_hypertable('test', 'time', migrate_data=>TRUE); -- -- should return ALL indexes for hypertable and chunk SELECT * FROM test.show_indexes('test') ORDER BY 1; Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ---------------+---------+------+--------+---------+-----------+------------ test_val_idx | {val} | | f | f | f | test_time_idx | {time} | | f | f | f | SELECT * FROM show_chunks('test') ch, LATERAL test.show_indexes(ch) ORDER BY 1, 2; ch | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ------------------------------------------+--------------------------------------------------------+---------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_24_23_chunk | _timescaledb_internal._hyper_24_23_chunk_test_val_idx | {val} | | f | f | f | _timescaledb_internal._hyper_24_23_chunk | _timescaledb_internal._hyper_24_23_chunk_test_time_idx | {time} | | f | f | f | DROP TABLE test; -- don't create default indexes on chunks when migrating data CREATE TABLE test(time TIMESTAMPTZ, val BIGINT); CREATE INDEX test_val_idx ON test(val); INSERT INTO test VALUES('2024-01-01 00:00:00-03', 500); SELECT FROM create_hypertable('test', 'time', create_default_indexes => FALSE, migrate_data=>TRUE); -- -- should NOT return default indexes for hypertable and chunk -- only user indexes should be returned SELECT * FROM test.show_indexes('test') ORDER BY 1; Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace --------------+---------+------+--------+---------+-----------+------------ test_val_idx | {val} | | f | f | f | SELECT * FROM show_chunks('test') ch, LATERAL test.show_indexes(ch) ORDER BY 1, 2; ch | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ------------------------------------------+-------------------------------------------------------+---------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_25_24_chunk | _timescaledb_internal._hyper_25_24_chunk_test_val_idx | {val} | | f | f | f | DROP TABLE test; -- test creating a hypertable with a primary key where the partitioning column is not part of the primary key CREATE TABLE test_schema.partition_not_pk (id INT NOT NULL, device_id INT NOT NULL, time TIMESTAMPTZ NOT NULL, a TEXT NOT NULL, PRIMARY KEY (id)); \set ON_ERROR_STOP 0 select create_hypertable ('test_schema.partition_not_pk', 'time'); ERROR: cannot create a unique index without the column "time" (used in partitioning) HINT: If you're creating a hypertable on a table with a primary key, ensure the partitioning column is part of the primary or composite key. \set ON_ERROR_STOP 1 DROP TABLE test_schema.partition_not_pk; -- test creating a hypertable with a composite key where the partitioning column is not part of the composite key CREATE TABLE test_schema.partition_not_pk (id INT NOT NULL, device_id INT NOT NULL, time TIMESTAMPTZ NOT NULL, a TEXT NOT NULL, PRIMARY KEY (id, device_id)); \set ON_ERROR_STOP 0 select create_hypertable ('test_schema.partition_not_pk', 'time'); ERROR: cannot create a unique index without the column "time" (used in partitioning) HINT: If you're creating a hypertable on a table with a primary key, ensure the partitioning column is part of the primary or composite key. \set ON_ERROR_STOP 1 DROP TABLE test_schema.partition_not_pk; -- test hypertable is not created for a table that is a part of a publication explicitly SET client_min_messages = ERROR; CREATE TABLE test (timestamp TIMESTAMPTZ NOT NULL); CREATE PUBLICATION publication_test; ALTER PUBLICATION publication_test ADD TABLE test; \set ON_ERROR_STOP 0 SELECT create_hypertable('test', 'timestamp'); ERROR: cannot create hypertable for table "test" because it is part of a publication \set ON_ERROR_STOP 1 INSERT INTO test (timestamp) values (now()); ALTER PUBLICATION publication_test DROP TABLE test; DROP PUBLICATION publication_test; DROP TABLE test; CREATE TABLE test (timestamp TIMESTAMPTZ NOT NULL); CREATE PUBLICATION publication_test1; CREATE PUBLICATION publication_test2; ALTER PUBLICATION publication_test1 ADD TABLE test; ALTER PUBLICATION publication_test2 ADD TABLE test; \set ON_ERROR_STOP 0 SELECT create_hypertable('test', 'timestamp'); ERROR: cannot create hypertable for table "test" because it is part of a publication \set ON_ERROR_STOP 1 INSERT INTO test (timestamp) values (now()); ALTER PUBLICATION publication_test1 DROP TABLE test; ALTER PUBLICATION publication_test2 DROP TABLE test; DROP PUBLICATION publication_test1; DROP PUBLICATION publication_test2; DROP TABLE test; -- test hypertable is not created for a table that is a part of a publication implicitly CREATE PUBLICATION publication_test FOR ALL tables; CREATE TABLE test (timestamp TIMESTAMPTZ NOT NULL); \set ON_ERROR_STOP 0 SELECT create_hypertable('test', 'timestamp'); ERROR: cannot create hypertable for table "test" because it is part of a publication \set ON_ERROR_STOP 1 DROP PUBLICATION publication_test; DROP TABLE test; CREATE TABLE test (timestamp TIMESTAMPTZ NOT NULL); CREATE PUBLICATION publication_test FOR ALL tables; \set ON_ERROR_STOP 0 SELECT create_hypertable('test', 'timestamp'); ERROR: cannot create hypertable for table "test" because it is part of a publication \set ON_ERROR_STOP 1 DROP PUBLICATION publication_test; DROP TABLE test; RESET client_min_messages; -- Test default_chunk_time_interval GUC -- Tests that the GUC correctly sets the default chunk interval for hypertables -- with different time column types (timestamp, timestamptz, date) and integer types. -- Show initial state (should be NULL meaning legacy defaults) SHOW timescaledb.default_chunk_time_interval; timescaledb.default_chunk_time_interval ----------------------------------------- -- No default_chunk_time_interval set (NULL) - uses legacy defaults CREATE TABLE test_no_guc_timestamptz(time TIMESTAMPTZ NOT NULL, val INT); SELECT create_hypertable('test_no_guc_timestamptz', 'time'); create_hypertable --------------------------------------- (28,public,test_no_guc_timestamptz,t) SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_no_guc_timestamptz'; time_interval --------------- @ 7 days DROP TABLE test_no_guc_timestamptz; CREATE TABLE test_no_guc_timestamp(time TIMESTAMP NOT NULL, val INT); SELECT create_hypertable('test_no_guc_timestamp', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices HINT: Use datatype TIMESTAMPTZ instead. create_hypertable ------------------------------------- (29,public,test_no_guc_timestamp,t) SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_no_guc_timestamp'; time_interval --------------- @ 7 days DROP TABLE test_no_guc_timestamp; CREATE TABLE test_no_guc_date(time DATE NOT NULL, val INT); SELECT create_hypertable('test_no_guc_date', 'time'); create_hypertable -------------------------------- (30,public,test_no_guc_date,t) SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_no_guc_date'; time_interval --------------- @ 7 days DROP TABLE test_no_guc_date; CREATE TABLE test_no_guc_uuid(time UUID NOT NULL, val INT); SELECT create_hypertable('test_no_guc_uuid', 'time'); create_hypertable -------------------------------- (31,public,test_no_guc_uuid,t) SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_no_guc_uuid'; time_interval --------------- @ 7 days DROP TABLE test_no_guc_uuid; -- Set default_chunk_time_interval to '1 week' and create hypertables SET timescaledb.default_chunk_time_interval = '1 week'; SHOW timescaledb.default_chunk_time_interval; timescaledb.default_chunk_time_interval ----------------------------------------- 1 week CREATE TABLE test_guc_timestamptz(time TIMESTAMPTZ NOT NULL, val INT); SELECT create_hypertable('test_guc_timestamptz', 'time'); create_hypertable ------------------------------------ (32,public,test_guc_timestamptz,t) SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_guc_timestamptz'; time_interval --------------- @ 7 days DROP TABLE test_guc_timestamptz; CREATE TABLE test_guc_timestamp(time TIMESTAMP NOT NULL, val INT); SELECT create_hypertable('test_guc_timestamp', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices HINT: Use datatype TIMESTAMPTZ instead. create_hypertable ---------------------------------- (33,public,test_guc_timestamp,t) SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_guc_timestamp'; time_interval --------------- @ 7 days DROP TABLE test_guc_timestamp; CREATE TABLE test_guc_date(time DATE NOT NULL, val INT); SELECT create_hypertable('test_guc_date', 'time'); create_hypertable ----------------------------- (34,public,test_guc_date,t) SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_guc_date'; time_interval --------------- @ 7 days DROP TABLE test_guc_date; CREATE TABLE test_guc_uuid(time UUID NOT NULL, val INT); SELECT create_hypertable('test_guc_uuid', 'time'); create_hypertable ----------------------------- (35,public,test_guc_uuid,t) SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_guc_uuid'; time_interval --------------- @ 7 days DROP TABLE test_guc_uuid; -- Set default_chunk_time_interval to '1 day' SET timescaledb.default_chunk_time_interval = '1 day'; SHOW timescaledb.default_chunk_time_interval; timescaledb.default_chunk_time_interval ----------------------------------------- 1 day CREATE TABLE test_guc_1day_timestamptz(time TIMESTAMPTZ NOT NULL, val INT); SELECT create_hypertable('test_guc_1day_timestamptz', 'time'); create_hypertable ----------------------------------------- (36,public,test_guc_1day_timestamptz,t) SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_guc_1day_timestamptz'; time_interval --------------- @ 1 day DROP TABLE test_guc_1day_timestamptz; -- Set default_chunk_time_interval to '1 month' SET timescaledb.default_chunk_time_interval = '1 month'; SHOW timescaledb.default_chunk_time_interval; timescaledb.default_chunk_time_interval ----------------------------------------- 1 month CREATE TABLE test_guc_1month_timestamptz(time TIMESTAMPTZ NOT NULL, val INT); SELECT create_hypertable('test_guc_1month_timestamptz', 'time'); create_hypertable ------------------------------------------- (37,public,test_guc_1month_timestamptz,t) SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_guc_1month_timestamptz'; time_interval --------------- @ 30 days DROP TABLE test_guc_1month_timestamptz; -- Integer partition types have their own defaults and do not use the GUC SET timescaledb.default_chunk_time_interval = '1 week'; CREATE TABLE test_guc_bigint(time BIGINT NOT NULL, val INT); SELECT create_hypertable('test_guc_bigint', 'time'); create_hypertable ------------------------------- (38,public,test_guc_bigint,t) SELECT integer_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_guc_bigint'; integer_interval ------------------ 1000000 DROP TABLE test_guc_bigint; CREATE TABLE test_guc_int(time INT NOT NULL, val INT); SELECT create_hypertable('test_guc_int', 'time'); create_hypertable ---------------------------- (39,public,test_guc_int,t) SELECT integer_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_guc_int'; integer_interval ------------------ 100000 DROP TABLE test_guc_int; CREATE TABLE test_guc_smallint(time SMALLINT NOT NULL, val INT); SELECT create_hypertable('test_guc_smallint', 'time'); create_hypertable --------------------------------- (40,public,test_guc_smallint,t) SELECT integer_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_guc_smallint'; integer_interval ------------------ 10000 DROP TABLE test_guc_smallint; -- Explicit chunk_time_interval should override the GUC SET timescaledb.default_chunk_time_interval = '1 week'; CREATE TABLE test_override_timestamptz(time TIMESTAMPTZ NOT NULL, val INT); SELECT create_hypertable('test_override_timestamptz', 'time', chunk_time_interval => INTERVAL '2 days'); create_hypertable ----------------------------------------- (41,public,test_override_timestamptz,t) SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_override_timestamptz'; time_interval --------------- @ 2 days DROP TABLE test_override_timestamptz; -- Reset GUC to NULL (legacy behavior) RESET timescaledb.default_chunk_time_interval; SHOW timescaledb.default_chunk_time_interval; timescaledb.default_chunk_time_interval ----------------------------------------- CREATE TABLE test_reset_timestamptz(time TIMESTAMPTZ NOT NULL, val INT); SELECT create_hypertable('test_reset_timestamptz', 'time'); create_hypertable -------------------------------------- (42,public,test_reset_timestamptz,t) SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_reset_timestamptz'; time_interval --------------- @ 7 days DROP TABLE test_reset_timestamptz; -- Invalid interval should fail \set ON_ERROR_STOP 0 SET timescaledb.default_chunk_time_interval = 'not_an_interval'; ERROR: invalid input syntax for type interval: "not_an_interval" SET timescaledb.default_chunk_time_interval = '123abc'; ERROR: invalid input syntax for type interval: "123abc" \set ON_ERROR_STOP 1 -- add_dimension does not use the GUC, keeps legacy behavior requiring explicit interval RESET timescaledb.default_chunk_time_interval; SET timescaledb.default_chunk_time_interval = '3 days'; CREATE TABLE test_add_dim(time TIMESTAMPTZ NOT NULL, time2 TIMESTAMPTZ NOT NULL, val INT); SELECT create_hypertable('test_add_dim', 'time'); create_hypertable ---------------------------- (43,public,test_add_dim,t) -- First dimension uses GUC SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_add_dim'; time_interval --------------- @ 3 days -- add_dimension without explicit interval should fail \set ON_ERROR_STOP 0 SELECT add_dimension('test_add_dim', 'time2'); ERROR: must specify either the number of partitions or an interval \set ON_ERROR_STOP 1 -- add_dimension with explicit interval works SELECT add_dimension('test_add_dim', 'time2', chunk_time_interval => INTERVAL '1 day'); add_dimension ---------------------------------- (56,public,test_add_dim,time2,t) SELECT column_name, time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_add_dim' ORDER BY dimension_number; column_name | time_interval -------------+--------------- time | @ 3 days time2 | @ 1 day DROP TABLE test_add_dim; -- Session-level GUC changes RESET timescaledb.default_chunk_time_interval; SET timescaledb.default_chunk_time_interval = '5 days'; CREATE TABLE test_session_1(time TIMESTAMPTZ NOT NULL, val INT); SELECT create_hypertable('test_session_1', 'time'); create_hypertable ------------------------------ (44,public,test_session_1,t) SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_session_1'; time_interval --------------- @ 5 days -- Change GUC mid-session SET timescaledb.default_chunk_time_interval = '10 days'; CREATE TABLE test_session_2(time TIMESTAMPTZ NOT NULL, val INT); SELECT create_hypertable('test_session_2', 'time'); create_hypertable ------------------------------ (45,public,test_session_2,t) SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_session_2'; time_interval --------------- @ 10 days DROP TABLE test_session_1; DROP TABLE test_session_2; -- Transaction-level GUC with SET LOCAL RESET timescaledb.default_chunk_time_interval; SET timescaledb.default_chunk_time_interval = '1 week'; BEGIN; SET LOCAL timescaledb.default_chunk_time_interval = '2 weeks'; CREATE TABLE test_local_guc(time TIMESTAMPTZ NOT NULL, val INT); SELECT create_hypertable('test_local_guc', 'time'); create_hypertable ------------------------------ (46,public,test_local_guc,t) SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_local_guc'; time_interval --------------- @ 14 days COMMIT; -- After commit, GUC should be back to session level (1 week) SHOW timescaledb.default_chunk_time_interval; timescaledb.default_chunk_time_interval ----------------------------------------- 1 week DROP TABLE test_local_guc; -- UUID partition type with various intervals RESET timescaledb.default_chunk_time_interval; SET timescaledb.default_chunk_time_interval = '1 day'; CREATE TABLE test_uuid_1day(time UUID NOT NULL, val INT); SELECT create_hypertable('test_uuid_1day', 'time'); create_hypertable ------------------------------ (47,public,test_uuid_1day,t) SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_uuid_1day'; time_interval --------------- @ 1 day DROP TABLE test_uuid_1day; SET timescaledb.default_chunk_time_interval = '1 hour'; CREATE TABLE test_uuid_1hour(time UUID NOT NULL, val INT); SELECT create_hypertable('test_uuid_1hour', 'time'); create_hypertable ------------------------------- (48,public,test_uuid_1hour,t) SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_uuid_1hour'; time_interval --------------- @ 1 hour DROP TABLE test_uuid_1hour; -- UUID with explicit override should work SET timescaledb.default_chunk_time_interval = '1 week'; CREATE TABLE test_uuid_override(time UUID NOT NULL, val INT); SELECT create_hypertable('test_uuid_override', 'time', chunk_time_interval => INTERVAL '12 hours'); create_hypertable ---------------------------------- (49,public,test_uuid_override,t) SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_uuid_override'; time_interval --------------- @ 12 hours DROP TABLE test_uuid_override; -- UUID with no GUC set (legacy default) RESET timescaledb.default_chunk_time_interval; CREATE TABLE test_uuid_legacy(time UUID NOT NULL, val INT); SELECT create_hypertable('test_uuid_legacy', 'time'); create_hypertable -------------------------------- (50,public,test_uuid_legacy,t) SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_uuid_legacy'; time_interval --------------- @ 7 days DROP TABLE test_uuid_legacy; -- Cleanup RESET timescaledb.default_chunk_time_interval; -- Check that we produce the proper error code for nonexistent tables. select create_hypertable(77777777, 'nonexistent'); ERROR: relation with oid 77777777 not found ================================================ FILE: test/expected/create_table.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test that we can verify constraints on regular tables CREATE TABLE test_hyper_pk(time TIMESTAMPTZ PRIMARY KEY, temp FLOAT, device INT); CREATE TABLE test_pk(device INT PRIMARY KEY); CREATE TABLE test_like(LIKE test_pk); SELECT create_hypertable('test_hyper_pk', 'time'); create_hypertable ---------------------------- (1,public,test_hyper_pk,t) \set ON_ERROR_STOP 0 -- Foreign key constraints that reference hypertables are currently unsupported CREATE TABLE test_fk(time TIMESTAMPTZ REFERENCES test_hyper_pk(time)); \set ON_ERROR_STOP 1 CREATE TABLE test_delete(time timestamp with time zone PRIMARY KEY, temp float); SELECT create_hypertable('test_delete', 'time'); create_hypertable -------------------------- (2,public,test_delete,t) INSERT INTO test_delete VALUES('2017-01-20T09:00:01', 22.5); INSERT INTO test_delete VALUES('2017-01-20T09:00:21', 21.2); INSERT INTO test_delete VALUES('2017-01-20T09:00:47', 25.1); INSERT INTO test_delete VALUES('2020-01-20T09:00:47', 25.1); INSERT INTO test_delete VALUES('2021-01-20T09:00:47', 25.1); SELECT * FROM test_delete WHERE temp = 25.1 ORDER BY time; time | temp ------------------------------+------ Fri Jan 20 09:00:47 2017 PST | 25.1 Mon Jan 20 09:00:47 2020 PST | 25.1 Wed Jan 20 09:00:47 2021 PST | 25.1 CREATE OR replace FUNCTION test_delete_row_count() RETURNS void AS $$ DECLARE v_cnt numeric; BEGIN v_cnt := 0; DELETE FROM test_delete WHERE temp = 25.1; GET DIAGNOSTICS v_cnt = ROW_COUNT; IF v_cnt != 3 THEN RAISE EXCEPTION 'unexpected result'; END IF; END; $$ LANGUAGE plpgsql; SELECT test_delete_row_count(); test_delete_row_count ----------------------- ================================================ FILE: test/expected/create_table_with.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- our user needs permission to create schema for the schema tests \c :TEST_DBNAME :ROLE_SUPERUSER GRANT CREATE ON DATABASE :TEST_DBNAME TO :ROLE_DEFAULT_PERM_USER; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- create table with non-tsdb option should not be affected CREATE TABLE t1(time timestamptz, device text, value float) WITH (autovacuum_enabled); DROP TABLE t1; -- test error cases \set ON_ERROR_STOP 0 \set VERBOSITY default CREATE TABLE t2(time float, device text, value float) WITH (tsdb.hypertable); ERROR: partition column could not be determined HINT: Use "timescaledb.partition_column" to specify the column to use as partitioning column. CREATE TABLE t2(time float, device text, value float) WITH (timescaledb.hypertable); ERROR: partition column could not be determined HINT: Use "timescaledb.partition_column" to specify the column to use as partitioning column. CREATE TABLE t2(time timestamptz, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column=NULL); ERROR: column "null" does not exist CREATE TABLE t2(time timestamptz, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column=''); ERROR: column "" does not exist CREATE TABLE t2(time timestamptz, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='foo'); ERROR: column "foo" does not exist CREATE TABLE t2(time timestamptz, device text, value float) WITH (tsdb.partition_column='time'); ERROR: timescaledb options requires hypertable option HINT: Use "timescaledb.hypertable" to enable creating a hypertable. CREATE TABLE t2(time timestamptz, device text, value float) WITH (timescaledb.partition_column='time'); ERROR: timescaledb options requires hypertable option HINT: Use "timescaledb.hypertable" to enable creating a hypertable. CREATE TABLE t2(time timestamptz , device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval='foo'); ERROR: invalid input syntax for type interval: "foo" CREATE TABLE t2(time int2 NOT NULL, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval='3 months'); ERROR: invalid input syntax for type smallint: "3 months" CREATE TABLE t2(time timestamptz, device text, value float) WITH (tsdb.create_default_indexes='time'); ERROR: invalid value for tsdb.create_default_indexes 'time' HINT: tsdb.create_default_indexes must be a valid bool CREATE TABLE t2(time timestamptz, device text, value float) WITH (tsdb.create_default_indexes=2); ERROR: invalid value for tsdb.create_default_indexes '2' HINT: tsdb.create_default_indexes must be a valid bool CREATE TABLE t2(time timestamptz, device text, value float) WITH (tsdb.create_default_indexes=-1); ERROR: invalid value for tsdb.create_default_indexes '-1' HINT: tsdb.create_default_indexes must be a valid bool CREATE TABLE t2(time timestamptz NOT NULL, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.columnstore=true); ERROR: functionality not supported under the current "apache" license. Learn more at https://tsdb.co/pdbir1r3 HINT: To access all features and the best time-series experience, try out Timescale Cloud. CREATE TABLE t2(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore,tsdb.hypertable,tsdb.partition_column='time'); ERROR: functionality not supported under the current "apache" license. Learn more at https://tsdb.co/pdbir1r3 HINT: To access all features and the best time-series experience, try out Timescale Cloud. -- Test error hint for invalid timescaledb options during CREATE TABLE CREATE TABLE t2(time timestamptz, device text, value float) WITH (tsdb.invalid_option = true); ERROR: unrecognized parameter "tsdb.invalid_option" HINT: Valid timescaledb parameters are: hypertable, columnstore, partition_column, chunk_interval, create_default_indexes, associated_schema, associated_table_prefix, orderby, segmentby, compress_index CREATE TABLE t2(time timestamptz, device text, value float) WITH (timescaledb.nonexistent_param = false); ERROR: unrecognized parameter "timescaledb.nonexistent_param" HINT: Valid timescaledb parameters are: hypertable, columnstore, partition_column, chunk_interval, create_default_indexes, associated_schema, associated_table_prefix, orderby, segmentby, compress_index \set ON_ERROR_STOP 1 \set VERBOSITY terse BEGIN; CREATE TABLE t3(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time'); CREATE TABLE t4(time timestamp, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,timescaledb.partition_column='time'); CREATE TABLE t5(time date, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time',autovacuum_enabled); CREATE TABLE t6(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,timescaledb.hypertable,tsdb.partition_column='time'); CREATE TABLE t7(time timestamptz, device text, value float) WITH (timescaledb.hypertable,tsdb.partition_column='time'); SELECT hypertable_name FROM timescaledb_information.hypertables ORDER BY 1; hypertable_name ----------------- t3 t4 t5 t6 t7 ROLLBACK; -- IF NOT EXISTS BEGIN; CREATE TABLE IF NOT EXISTS t7(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time'); CREATE TABLE IF NOT EXISTS t7(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time'); NOTICE: relation "t7" already exists, skipping CREATE TABLE IF NOT EXISTS t7(time timestamptz NOT NULL, device text, value float); NOTICE: relation "t7" already exists, skipping SELECT hypertable_name FROM timescaledb_information.hypertables ORDER BY 1; hypertable_name ----------------- t7 ROLLBACK; -- table won't be converted to hypertable unless it is in the initial CREATE TABLE BEGIN; CREATE TABLE IF NOT EXISTS t8(time timestamptz NOT NULL, device text, value float); CREATE TABLE IF NOT EXISTS t8(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time'); NOTICE: relation "t8" already exists, skipping CREATE TABLE IF NOT EXISTS t8(time timestamptz NOT NULL, device text, value float); NOTICE: relation "t8" already exists, skipping SELECT hypertable_name FROM timescaledb_information.hypertables ORDER BY 1; hypertable_name ----------------- ROLLBACK; -- chunk_interval BEGIN; CREATE TABLE IF NOT EXISTS t9(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval='8weeks'); SELECT hypertable_name, column_name, column_type, time_interval FROM timescaledb_information.dimensions; hypertable_name | column_name | column_type | time_interval -----------------+-------------+--------------------------+--------------- t9 | time | timestamp with time zone | @ 56 days ROLLBACK; BEGIN; CREATE TABLE IF NOT EXISTS t9(time timestamp NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval='23 days'); SELECT hypertable_name, column_name, column_type, time_interval FROM timescaledb_information.dimensions; hypertable_name | column_name | column_type | time_interval -----------------+-------------+-----------------------------+--------------- t9 | time | timestamp without time zone | @ 23 days ROLLBACK; BEGIN; CREATE TABLE IF NOT EXISTS t9(time date NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval='3 months'); SELECT hypertable_name, column_name, column_type, time_interval FROM timescaledb_information.dimensions; hypertable_name | column_name | column_type | time_interval -----------------+-------------+-------------+--------------- t9 | time | date | @ 90 days ROLLBACK; BEGIN; CREATE TABLE IF NOT EXISTS t9(time int2 NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval=12); SELECT hypertable_name, column_name, column_type, integer_interval FROM timescaledb_information.dimensions; hypertable_name | column_name | column_type | integer_interval -----------------+-------------+-------------+------------------ t9 | time | smallint | 12 ROLLBACK; BEGIN; CREATE TABLE IF NOT EXISTS t9(time int4 NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval=3453); SELECT hypertable_name, column_name, column_type, integer_interval FROM timescaledb_information.dimensions; hypertable_name | column_name | column_type | integer_interval -----------------+-------------+-------------+------------------ t9 | time | integer | 3453 ROLLBACK; BEGIN; CREATE TABLE IF NOT EXISTS t9(time int8 NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval=32768); SELECT hypertable_name, column_name, column_type, integer_interval FROM timescaledb_information.dimensions; hypertable_name | column_name | column_type | integer_interval -----------------+-------------+-------------+------------------ t9 | time | bigint | 32768 ROLLBACK; -- create_default_indexes BEGIN; CREATE TABLE t10(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time'); SELECT indexrelid::regclass from pg_index where indrelid='t10'::regclass ORDER BY indexrelid::regclass::text; indexrelid -------------- t10_time_idx ROLLBACK; BEGIN; CREATE TABLE t10(time timestamptz NOT NULL PRIMARY KEY, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time'); SELECT indexrelid::regclass from pg_index where indrelid='t10'::regclass ORDER BY indexrelid::regclass::text; indexrelid ------------ t10_pkey ROLLBACK; BEGIN; CREATE TABLE t10(time timestamptz NOT NULL UNIQUE, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time'); SELECT indexrelid::regclass from pg_index where indrelid='t10'::regclass ORDER BY indexrelid::regclass::text; indexrelid -------------- t10_time_key ROLLBACK; BEGIN; CREATE TABLE t10(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time',tsdb.create_default_indexes=true); SELECT indexrelid::regclass from pg_index where indrelid='t10'::regclass ORDER BY indexrelid::regclass::text; indexrelid -------------- t10_time_idx ROLLBACK; BEGIN; CREATE TABLE t10(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time',tsdb.create_default_indexes=false); SELECT indexrelid::regclass from pg_index where indrelid='t10'::regclass ORDER BY indexrelid::regclass::text; indexrelid ------------ ROLLBACK; -- associated_schema BEGIN; CREATE TABLE t11(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time'); SELECT associated_schema_name FROM _timescaledb_catalog.hypertable WHERE table_name = 't11'; associated_schema_name ------------------------ _timescaledb_internal ROLLBACK; BEGIN; CREATE TABLE t11(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time', tsdb.associated_schema='abc'); SELECT associated_schema_name FROM _timescaledb_catalog.hypertable WHERE table_name = 't11'; associated_schema_name ------------------------ abc INSERT INTO t11 SELECT '2025-01-01', 'd1', 0.1; SELECT relname from pg_class where relnamespace = 'abc'::regnamespace ORDER BY 1; relname -------------------------------- _hyper_21_1_chunk _hyper_21_1_chunk_t11_time_idx ROLLBACK; BEGIN; CREATE SCHEMA abc2; CREATE TABLE t11(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time', tsdb.associated_schema='abc2'); SELECT associated_schema_name FROM _timescaledb_catalog.hypertable WHERE table_name = 't11'; associated_schema_name ------------------------ abc2 INSERT INTO t11 SELECT '2025-01-01', 'd1', 0.1; SELECT relname from pg_class where relnamespace = 'abc2'::regnamespace ORDER BY 1; relname -------------------------------- _hyper_22_2_chunk _hyper_22_2_chunk_t11_time_idx ROLLBACK; -- associated_table_prefix BEGIN; CREATE TABLE t12(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time'); SELECT associated_table_prefix FROM _timescaledb_catalog.hypertable WHERE table_name = 't12'; associated_table_prefix ------------------------- _hyper_23 ROLLBACK; BEGIN; CREATE TABLE t12(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time', tsdb.associated_schema='abc', tsdb.associated_table_prefix='tbl_prefix'); SELECT associated_table_prefix FROM _timescaledb_catalog.hypertable WHERE table_name = 't12'; associated_table_prefix ------------------------- tbl_prefix INSERT INTO t12 SELECT '2025-01-01', 'd1', 0.1; SELECT relname from pg_class where relnamespace = 'abc'::regnamespace ORDER BY 1; relname --------------------------------- tbl_prefix_3_chunk tbl_prefix_3_chunk_t12_time_idx ROLLBACK; -- default partition column BEGIN; CREATE TABLE t13(time timestamptz, device text, value float) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column CREATE TABLE t14("TiMe" timestamptz, device text, value float) WITH (tsdb.hypertable); NOTICE: using column "TiMe" as partitioning column SELECT hypertable_name, column_name FROM timescaledb_information.dimensions WHERE hypertable_name IN ('t13','t14') ORDER BY 1; hypertable_name | column_name -----------------+------------- t13 | time t14 | TiMe ROLLBACK; -- Test default_chunk_time_interval GUC interaction with CREATE TABLE WITH -- Tests that the GUC correctly sets the default chunk interval when using -- CREATE TABLE WITH syntax instead of create_hypertable(). -- GUC set to '1 week' should be used by CREATE TABLE WITH BEGIN; SET timescaledb.default_chunk_time_interval = '1 week'; CREATE TABLE t_guc_week(time timestamptz NOT NULL, device text, value float) WITH (tsdb.hypertable, tsdb.partition_column='time'); SELECT hypertable_name, time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 't_guc_week'; hypertable_name | time_interval -----------------+--------------- t_guc_week | @ 7 days ROLLBACK; -- GUC set to '1 day' with different time types BEGIN; SET timescaledb.default_chunk_time_interval = '1 day'; CREATE TABLE t_guc_timestamptz(time timestamptz NOT NULL, device text, value float) WITH (tsdb.hypertable, tsdb.partition_column='time'); CREATE TABLE t_guc_timestamp(time timestamp NOT NULL, device text, value float) WITH (tsdb.hypertable, tsdb.partition_column='time'); CREATE TABLE t_guc_date(time date NOT NULL, device text, value float) WITH (tsdb.hypertable, tsdb.partition_column='time'); SELECT hypertable_name, time_interval FROM timescaledb_information.dimensions WHERE hypertable_name LIKE 't_guc_%' ORDER BY hypertable_name; hypertable_name | time_interval -------------------+--------------- t_guc_date | @ 1 day t_guc_timestamp | @ 1 day t_guc_timestamptz | @ 1 day ROLLBACK; -- Explicit tsdb.chunk_interval should override the GUC BEGIN; SET timescaledb.default_chunk_time_interval = '1 week'; CREATE TABLE t_guc_override(time timestamptz NOT NULL, device text, value float) WITH (tsdb.hypertable, tsdb.partition_column='time', tsdb.chunk_interval='2 days'); SELECT hypertable_name, time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 't_guc_override'; hypertable_name | time_interval -----------------+--------------- t_guc_override | @ 2 days ROLLBACK; -- Integer partition types have their own default and do not use the GUC BEGIN; SET timescaledb.default_chunk_time_interval = '1 week'; CREATE TABLE t_guc_int(time int8 NOT NULL, device text, value float) WITH (tsdb.hypertable, tsdb.partition_column='time'); SELECT hypertable_name, integer_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 't_guc_int'; hypertable_name | integer_interval -----------------+------------------ t_guc_int | 1000000 ROLLBACK; -- No GUC set (NULL) should use legacy defaults BEGIN; RESET timescaledb.default_chunk_time_interval; CREATE TABLE t_no_guc(time timestamptz NOT NULL, device text, value float) WITH (tsdb.hypertable, tsdb.partition_column='time'); SELECT hypertable_name, time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 't_no_guc'; hypertable_name | time_interval -----------------+--------------- t_no_guc | @ 7 days ROLLBACK; -- GUC with UUID partition type BEGIN; SET timescaledb.default_chunk_time_interval = '12 hours'; CREATE TABLE t_guc_uuid(time uuid NOT NULL, device text, value float) WITH (tsdb.hypertable, tsdb.partition_column='time'); SELECT hypertable_name, time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 't_guc_uuid'; hypertable_name | time_interval -----------------+--------------- t_guc_uuid | @ 12 hours ROLLBACK; -- Cleanup RESET timescaledb.default_chunk_time_interval; ================================================ FILE: test/expected/cursor.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE cursor_test(time timestamptz, device_id int, temp float); SELECT create_hypertable('cursor_test','time'); create_hypertable -------------------------- (1,public,cursor_test,t) INSERT INTO cursor_test SELECT '2000-01-01',1,0.5; INSERT INTO cursor_test SELECT '2001-01-01',1,0.5; INSERT INTO cursor_test SELECT '2002-01-01',1,0.5; \set ON_ERROR_STOP 0 BEGIN; DECLARE c1 SCROLL CURSOR FOR SELECT * FROM cursor_test; FETCH NEXT FROM c1; time | device_id | temp ------------------------------+-----------+------ Sat Jan 01 00:00:00 2000 PST | 1 | 0.5 -- this will produce an error on PG < 14 because PostgreSQL checks -- for the existence of a scan node with the relation id for every relation -- used in the update plan in the plan of the cursor. UPDATE cursor_test SET temp = 0.7 WHERE CURRENT OF c1; COMMIT; -- test cursor with no chunks left after runtime exclusion BEGIN; DECLARE c1 SCROLL CURSOR FOR SELECT * FROM cursor_test WHERE time > now(); UPDATE cursor_test SET temp = 0.7 WHERE CURRENT OF c1; ERROR: cursor "c1" is not a simply updatable scan of table "_hyper_1_1_chunk" COMMIT; -- test cursor with no chunks left after planning exclusion BEGIN; DECLARE c1 SCROLL CURSOR FOR SELECT * FROM cursor_test WHERE time > '2010-01-01'; UPDATE cursor_test SET temp = 0.7 WHERE CURRENT OF c1; ERROR: cursor "c1" is not a simply updatable scan of table "_hyper_1_1_chunk" COMMIT; \set ON_ERROR_STOP 1 SET timescaledb.enable_constraint_exclusion TO off; BEGIN; DECLARE c1 SCROLL CURSOR FOR SELECT * FROM cursor_test; FETCH NEXT FROM c1; time | device_id | temp ------------------------------+-----------+------ Sat Jan 01 00:00:00 2000 PST | 1 | 0.7 UPDATE cursor_test SET temp = 0.7 WHERE CURRENT OF c1; COMMIT; ================================================ FILE: test/expected/custom_type.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER SET client_min_messages TO WARNING; CREATE OR REPLACE FUNCTION customtype_in(cstring) RETURNS customtype AS 'timestamptz_in' LANGUAGE internal IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION customtype_out(customtype) RETURNS cstring AS 'timestamptz_out' LANGUAGE internal IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION customtype_recv(internal) RETURNS customtype AS 'timestamptz_recv' LANGUAGE internal IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION customtype_send(customtype) RETURNS bytea AS 'timestamptz_send' LANGUAGE internal IMMUTABLE STRICT; SET client_min_messages TO DEFAULT; CREATE TYPE customtype ( INPUT = customtype_in, OUTPUT = customtype_out, RECEIVE = customtype_recv, SEND = customtype_send, LIKE = TIMESTAMPTZ ); CREATE CAST (customtype AS bigint) WITHOUT FUNCTION AS ASSIGNMENT; CREATE CAST (bigint AS customtype) WITHOUT FUNCTION AS IMPLICIT; CREATE CAST (customtype AS timestamptz) WITHOUT FUNCTION AS ASSIGNMENT; CREATE CAST (timestamptz AS customtype) WITHOUT FUNCTION AS ASSIGNMENT; CREATE OR REPLACE FUNCTION customtype_lt(customtype, customtype) RETURNS bool AS 'timestamp_lt' LANGUAGE internal IMMUTABLE STRICT; CREATE OPERATOR < ( LEFTARG = customtype, RIGHTARG = customtype, PROCEDURE = customtype_lt, COMMUTATOR = >, NEGATOR = >=, RESTRICT = scalarltsel, JOIN = scalarltjoinsel ); CREATE OR REPLACE FUNCTION customtype_ge(customtype, customtype) RETURNS bool AS 'timestamp_ge' LANGUAGE internal IMMUTABLE STRICT; CREATE OPERATOR >= ( LEFTARG = customtype, RIGHTARG = customtype, PROCEDURE = customtype_ge, COMMUTATOR = <=, NEGATOR = <, RESTRICT = scalargtsel, JOIN = scalargtjoinsel ); \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE customtype_test(time_custom customtype, val int); \set ON_ERROR_STOP 0 -- Using interval type for chunk time interval should fail with custom time type SELECT create_hypertable('customtype_test', 'time_custom', chunk_time_interval => INTERVAL '1 day', create_default_indexes=>false); ERROR: invalid interval type for customtype dimension \set ON_ERROR_STOP 1 SELECT create_hypertable('customtype_test', 'time_custom', chunk_time_interval => 10e6::bigint, create_default_indexes=>false); create_hypertable ------------------------------ (1,public,customtype_test,t) INSERT INTO customtype_test VALUES ('2001-01-01 01:02:03'::customtype, 10); INSERT INTO customtype_test VALUES ('2001-01-01 01:02:03'::customtype, 10); INSERT INTO customtype_test VALUES ('2001-01-01 01:02:03'::customtype, 10); EXPLAIN (buffers off, costs off) SELECT * FROM customtype_test; --- QUERY PLAN --- Seq Scan on _hyper_1_1_chunk INSERT INTO customtype_test VALUES ('2001-01-01 01:02:23'::customtype, 11); EXPLAIN (buffers off, costs off) SELECT * FROM customtype_test; --- QUERY PLAN --- Append -> Seq Scan on _hyper_1_1_chunk -> Seq Scan on _hyper_1_2_chunk SELECT * FROM customtype_test; time_custom | val ------------------------------+----- Mon Jan 01 01:02:03 2001 PST | 10 Mon Jan 01 01:02:03 2001 PST | 10 Mon Jan 01 01:02:03 2001 PST | 10 Mon Jan 01 01:02:23 2001 PST | 11 ================================================ FILE: test/expected/ddl.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA IF NOT EXISTS "customSchema" AUTHORIZATION :ROLE_DEFAULT_PERM_USER; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER \ir include/ddl_ops_1.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."Hypertable_1" ( time BIGINT NOT NULL, "Device_id" TEXT NOT NULL, temp_c int NOT NULL DEFAULT -1, humidity numeric NULL DEFAULT 0, sensor_1 NUMERIC NULL DEFAULT 1, sensor_2 NUMERIC NOT NULL DEFAULT 1, sensor_3 NUMERIC NOT NULL DEFAULT 1, sensor_4 NUMERIC NOT NULL DEFAULT 1 ); CREATE INDEX ON PUBLIC."Hypertable_1" (time, "Device_id"); CREATE TABLE "customSchema"."Hypertable_1" ( time BIGINT NOT NULL, "Device_id" TEXT NOT NULL, temp_c int NOT NULL DEFAULT -1, humidity numeric NULL DEFAULT 0, sensor_1 NUMERIC NULL DEFAULT 1, sensor_2 NUMERIC NOT NULL DEFAULT 1, sensor_3 NUMERIC NOT NULL DEFAULT 1, sensor_4 NUMERIC NOT NULL DEFAULT 1 ); CREATE INDEX ON "customSchema"."Hypertable_1" (time, "Device_id"); SELECT * FROM create_hypertable('"public"."Hypertable_1"', 'time', 'Device_id', 1, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+--------------+--------- 1 | public | Hypertable_1 | t SELECT * FROM create_hypertable('"customSchema"."Hypertable_1"', 'time', NULL, 1, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+--------------+--------------+--------- 2 | customSchema | Hypertable_1 | t SELECT * FROM _timescaledb_catalog.hypertable; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+--------------+--------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 1 | public | Hypertable_1 | _timescaledb_internal | _hyper_1 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 2 | customSchema | Hypertable_1 | _timescaledb_internal | _hyper_2 | 1 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 CREATE INDEX ON PUBLIC."Hypertable_1" (time, "temp_c"); CREATE INDEX "ind_humidity" ON PUBLIC."Hypertable_1" (time, "humidity"); CREATE INDEX "ind_sensor_1" ON PUBLIC."Hypertable_1" (time, "sensor_1"); INSERT INTO PUBLIC."Hypertable_1"(time, "Device_id", temp_c, humidity, sensor_1, sensor_2, sensor_3, sensor_4) VALUES(1257894000000000000, 'dev1', 30, 70, 1, 2, 3, 100); CREATE UNIQUE INDEX "Unique1" ON PUBLIC."Hypertable_1" (time, "Device_id"); CREATE UNIQUE INDEX "Unique1" ON "customSchema"."Hypertable_1" (time); INSERT INTO "customSchema"."Hypertable_1"(time, "Device_id", temp_c, humidity, sensor_1, sensor_2, sensor_3, sensor_4) VALUES(1257894000000000000, 'dev1', 30, 70, 1, 2, 3, 100); INSERT INTO "customSchema"."Hypertable_1"(time, "Device_id", temp_c, humidity, sensor_1, sensor_2, sensor_3, sensor_4) VALUES(1257894000000000001, 'dev1', 30, 70, 1, 2, 3, 100); SELECT * FROM test.show_indexesp('%.%'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace -------------------------------+--------------------------------------------------+------------------+------+--------+---------+-----------+------------ "customSchema"."Hypertable_1" | "customSchema"."Hypertable_1_time_Device_id_idx" | {time,Device_id} | | f | f | f | "customSchema"."Hypertable_1" | "customSchema"."Hypertable_1_time_idx" | {time} | | f | f | f | "customSchema"."Hypertable_1" | "customSchema"."Unique1" | {time} | | t | f | f | SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper_%'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+--------------------------------------------------------------------------+------------------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_Hypertable_1_time_Device_id_idx" | {time,Device_id} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_Hypertable_1_time_idx" | {time} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_Hypertable_1_Device_id_time_idx" | {Device_id,time} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_Hypertable_1_time_temp_c_idx" | {time,temp_c} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_ind_humidity | {time,humidity} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_ind_sensor_1 | {time,sensor_1} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_Unique1" | {time,Device_id} | | t | f | f | _timescaledb_internal._hyper_2_2_chunk | _timescaledb_internal."_hyper_2_2_chunk_Hypertable_1_time_Device_id_idx" | {time,Device_id} | | f | f | f | _timescaledb_internal._hyper_2_2_chunk | _timescaledb_internal."_hyper_2_2_chunk_Hypertable_1_time_idx" | {time} | | f | f | f | _timescaledb_internal._hyper_2_2_chunk | _timescaledb_internal."_hyper_2_2_chunk_Unique1" | {time} | | t | f | f | --expect error cases \set ON_ERROR_STOP 0 INSERT INTO "customSchema"."Hypertable_1"(time, "Device_id", temp_c, humidity, sensor_1, sensor_2, sensor_3, sensor_4) VALUES(1257894000000000000, 'dev1', 31, 71, 72, 4, 1, 102); psql:include/ddl_ops_1.sql:57: ERROR: duplicate key value violates unique constraint "_hyper_2_2_chunk_Unique1" CREATE UNIQUE INDEX "Unique2" ON PUBLIC."Hypertable_1" ("Device_id"); psql:include/ddl_ops_1.sql:58: ERROR: cannot create a unique index without the column "time" (used in partitioning) CREATE UNIQUE INDEX "Unique2" ON PUBLIC."Hypertable_1" (time); psql:include/ddl_ops_1.sql:59: ERROR: cannot create a unique index without the column "Device_id" (used in partitioning) CREATE UNIQUE INDEX "Unique2" ON PUBLIC."Hypertable_1" (sensor_1); psql:include/ddl_ops_1.sql:60: ERROR: cannot create a unique index without the column "time" (used in partitioning) UPDATE ONLY PUBLIC."Hypertable_1" SET time = 0 WHERE TRUE; DELETE FROM ONLY PUBLIC."Hypertable_1" WHERE "Device_id" = 'dev1'; \set ON_ERROR_STOP 1 CREATE TABLE my_ht (time BIGINT, val integer); SELECT * FROM create_hypertable('my_ht', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 3 | public | my_ht | t ALTER TABLE my_ht ADD COLUMN val2 integer; SELECT * FROM test.show_columns('my_ht'); Column | Type | NotNull --------+---------+--------- time | bigint | t val | integer | f val2 | integer | f -- Should error when adding again \set ON_ERROR_STOP 0 ALTER TABLE my_ht ADD COLUMN val2 integer; psql:include/ddl_ops_1.sql:73: ERROR: column "val2" of relation "my_ht" already exists \set ON_ERROR_STOP 1 -- Should create ALTER TABLE my_ht ADD COLUMN IF NOT EXISTS val3 integer; SELECT * FROM test.show_columns('my_ht'); Column | Type | NotNull --------+---------+--------- time | bigint | t val | integer | f val2 | integer | f val3 | integer | f -- Should skip and not error ALTER TABLE my_ht ADD COLUMN IF NOT EXISTS val3 integer; psql:include/ddl_ops_1.sql:81: NOTICE: column "val3" of relation "my_ht" already exists, skipping SELECT * FROM test.show_columns('my_ht'); Column | Type | NotNull --------+---------+--------- time | bigint | t val | integer | f val2 | integer | f val3 | integer | f -- Should drop ALTER TABLE my_ht DROP COLUMN IF EXISTS val3; SELECT * FROM test.show_columns('my_ht'); Column | Type | NotNull --------+---------+--------- time | bigint | t val | integer | f val2 | integer | f -- Should skip and not error ALTER TABLE my_ht DROP COLUMN IF EXISTS val3; psql:include/ddl_ops_1.sql:89: NOTICE: column "val3" of relation "my_ht" does not exist, skipping SELECT * FROM test.show_columns('my_ht'); Column | Type | NotNull --------+---------+--------- time | bigint | t val | integer | f val2 | integer | f --Test default index creation on create_hypertable(). --Make sure that we do not duplicate indexes that already exists -- --No existing indexes: both time and space-time indexes created BEGIN; CREATE TABLE PUBLIC."Hypertable_1_with_default_index_enabled" ( "Time" BIGINT NOT NULL, "Device_id" TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 ); SELECT * FROM create_hypertable('"public"."Hypertable_1_with_default_index_enabled"', 'Time', 'Device_id', 1, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+-----------------------------------------+--------- 4 | public | Hypertable_1_with_default_index_enabled | t SELECT * FROM test.show_indexes('"Hypertable_1_with_default_index_enabled"'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace --------------------------------------------------------------+------------------+------+--------+---------+-----------+------------ "Hypertable_1_with_default_index_enabled_Device_id_Time_idx" | {Device_id,Time} | | f | f | f | "Hypertable_1_with_default_index_enabled_Time_idx" | {Time} | | f | f | f | ROLLBACK; --Space index exists: only time index created BEGIN; CREATE TABLE PUBLIC."Hypertable_1_with_default_index_enabled" ( "Time" BIGINT NOT NULL, "Device_id" TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 ); CREATE INDEX ON PUBLIC."Hypertable_1_with_default_index_enabled" ("Device_id", "Time" DESC); SELECT * FROM create_hypertable('"public"."Hypertable_1_with_default_index_enabled"', 'Time', 'Device_id', 1, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+-----------------------------------------+--------- 5 | public | Hypertable_1_with_default_index_enabled | t SELECT * FROM test.show_indexes('"Hypertable_1_with_default_index_enabled"'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace --------------------------------------------------------------+------------------+------+--------+---------+-----------+------------ "Hypertable_1_with_default_index_enabled_Device_id_Time_idx" | {Device_id,Time} | | f | f | f | "Hypertable_1_with_default_index_enabled_Time_idx" | {Time} | | f | f | f | ROLLBACK; --Time index exists, only partition index created BEGIN; CREATE TABLE PUBLIC."Hypertable_1_with_default_index_enabled" ( "Time" BIGINT NOT NULL, "Device_id" TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 ); CREATE INDEX ON PUBLIC."Hypertable_1_with_default_index_enabled" ("Time" DESC); SELECT * FROM create_hypertable('"public"."Hypertable_1_with_default_index_enabled"', 'Time', 'Device_id', 1, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+-----------------------------------------+--------- 6 | public | Hypertable_1_with_default_index_enabled | t SELECT * FROM test.show_indexes('"Hypertable_1_with_default_index_enabled"'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace --------------------------------------------------------------+------------------+------+--------+---------+-----------+------------ "Hypertable_1_with_default_index_enabled_Device_id_Time_idx" | {Device_id,Time} | | f | f | f | "Hypertable_1_with_default_index_enabled_Time_idx" | {Time} | | f | f | f | ROLLBACK; --No space partitioning, only time index created BEGIN; CREATE TABLE PUBLIC."Hypertable_1_with_default_index_enabled" ( "Time" BIGINT NOT NULL, "Device_id" TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 ); SELECT * FROM create_hypertable('"public"."Hypertable_1_with_default_index_enabled"', 'Time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+-----------------------------------------+--------- 7 | public | Hypertable_1_with_default_index_enabled | t SELECT * FROM test.show_indexes('"Hypertable_1_with_default_index_enabled"'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------------------+---------+------+--------+---------+-----------+------------ "Hypertable_1_with_default_index_enabled_Time_idx" | {Time} | | f | f | f | ROLLBACK; --Disable index creation: no default indexes created BEGIN; CREATE TABLE PUBLIC."Hypertable_1_with_default_index_enabled" ( "Time" BIGINT NOT NULL, "Device_id" TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 ); SELECT * FROM create_hypertable('"public"."Hypertable_1_with_default_index_enabled"', 'Time', 'Device_id', 1, create_default_indexes=>FALSE, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+-----------------------------------------+--------- 8 | public | Hypertable_1_with_default_index_enabled | t SELECT * FROM test.show_indexes('"Hypertable_1_with_default_index_enabled"'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace -------+---------+------+--------+---------+-----------+------------ ROLLBACK; SELECT * FROM PUBLIC."Hypertable_1"; time | Device_id | temp_c | humidity | sensor_1 | sensor_2 | sensor_3 | sensor_4 ---------------------+-----------+--------+----------+----------+----------+----------+---------- 1257894000000000000 | dev1 | 30 | 70 | 1 | 2 | 3 | 100 SELECT * FROM ONLY PUBLIC."Hypertable_1"; time | Device_id | temp_c | humidity | sensor_1 | sensor_2 | sensor_3 | sensor_4 ------+-----------+--------+----------+----------+----------+----------+---------- EXPLAIN (buffers off, costs off) SELECT * FROM ONLY PUBLIC."Hypertable_1"; --- QUERY PLAN --- Seq Scan on "Hypertable_1" SELECT * FROM test.show_columns('PUBLIC."Hypertable_1"'); Column | Type | NotNull -----------+---------+--------- time | bigint | t Device_id | text | t temp_c | integer | t humidity | numeric | f sensor_1 | numeric | f sensor_2 | numeric | t sensor_3 | numeric | t sensor_4 | numeric | t SELECT * FROM test.show_columns('_timescaledb_internal._hyper_1_1_chunk'); Column | Type | NotNull -----------+---------+--------- time | bigint | t Device_id | text | t temp_c | integer | t humidity | numeric | f sensor_1 | numeric | f sensor_2 | numeric | t sensor_3 | numeric | t sensor_4 | numeric | t \ir include/ddl_ops_2.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. ALTER TABLE PUBLIC."Hypertable_1" ADD COLUMN temp_f INTEGER NOT NULL DEFAULT 31; ALTER TABLE PUBLIC."Hypertable_1" DROP COLUMN temp_c; ALTER TABLE PUBLIC."Hypertable_1" DROP COLUMN sensor_4; ALTER TABLE PUBLIC."Hypertable_1" ALTER COLUMN humidity SET DEFAULT 100; ALTER TABLE PUBLIC."Hypertable_1" ALTER COLUMN sensor_1 DROP DEFAULT; ALTER TABLE PUBLIC."Hypertable_1" ALTER COLUMN sensor_2 SET DEFAULT NULL; ALTER TABLE PUBLIC."Hypertable_1" ALTER COLUMN sensor_1 SET NOT NULL; ALTER TABLE PUBLIC."Hypertable_1" ALTER COLUMN sensor_2 DROP NOT NULL; ALTER TABLE PUBLIC."Hypertable_1" RENAME COLUMN sensor_2 TO sensor_2_renamed; ALTER TABLE PUBLIC."Hypertable_1" RENAME COLUMN sensor_3 TO sensor_3_renamed; DROP INDEX "ind_sensor_1"; CREATE OR REPLACE FUNCTION empty_trigger_func() RETURNS TRIGGER LANGUAGE PLPGSQL AS $BODY$ BEGIN END $BODY$; CREATE TRIGGER test_trigger BEFORE UPDATE OR DELETE ON PUBLIC."Hypertable_1" FOR EACH STATEMENT EXECUTE FUNCTION empty_trigger_func(); ALTER TABLE PUBLIC."Hypertable_1" ALTER COLUMN sensor_2_renamed SET DATA TYPE int; ALTER INDEX "ind_humidity" RENAME TO "ind_humdity2"; -- Change should be reflected here SELECT * FROM test.show_indexesp('%.%'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace -------------------------------+--------------------------------------------------+------------------+------+--------+---------+-----------+------------ "customSchema"."Hypertable_1" | "customSchema"."Hypertable_1_time_Device_id_idx" | {time,Device_id} | | f | f | f | "customSchema"."Hypertable_1" | "customSchema"."Hypertable_1_time_idx" | {time} | | f | f | f | "customSchema"."Hypertable_1" | "customSchema"."Unique1" | {time} | | t | f | f | SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+--------------------------------------------------------------------------+------------------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_Hypertable_1_time_Device_id_idx" | {time,Device_id} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_Hypertable_1_time_idx" | {time} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_Hypertable_1_Device_id_time_idx" | {Device_id,time} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_ind_humdity2 | {time,humidity} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_Unique1" | {time,Device_id} | | t | f | f | _timescaledb_internal._hyper_2_2_chunk | _timescaledb_internal."_hyper_2_2_chunk_Hypertable_1_time_Device_id_idx" | {time,Device_id} | | f | f | f | _timescaledb_internal._hyper_2_2_chunk | _timescaledb_internal."_hyper_2_2_chunk_Hypertable_1_time_idx" | {time} | | f | f | f | _timescaledb_internal._hyper_2_2_chunk | _timescaledb_internal."_hyper_2_2_chunk_Unique1" | {time} | | t | f | f | --create column with same name as previously renamed one ALTER TABLE PUBLIC."Hypertable_1" ADD COLUMN sensor_3 BIGINT NOT NULL DEFAULT 131; --create column with same name as previously dropped one ALTER TABLE PUBLIC."Hypertable_1" ADD COLUMN sensor_4 BIGINT NOT NULL DEFAULT 131; SELECT * FROM test.show_columns('PUBLIC."Hypertable_1"'); Column | Type | NotNull ------------------+---------+--------- time | bigint | t Device_id | text | t humidity | numeric | f sensor_1 | numeric | t sensor_2_renamed | integer | f sensor_3_renamed | numeric | t temp_f | integer | t sensor_3 | bigint | t sensor_4 | bigint | t SELECT * FROM test.show_columns('_timescaledb_internal._hyper_1_1_chunk'); Column | Type | NotNull ------------------+---------+--------- time | bigint | t Device_id | text | t humidity | numeric | f sensor_1 | numeric | t sensor_2_renamed | integer | f sensor_3_renamed | numeric | t temp_f | integer | t sensor_3 | bigint | t sensor_4 | bigint | t SELECT * FROM PUBLIC."Hypertable_1"; time | Device_id | humidity | sensor_1 | sensor_2_renamed | sensor_3_renamed | temp_f | sensor_3 | sensor_4 ---------------------+-----------+----------+----------+------------------+------------------+--------+----------+---------- 1257894000000000000 | dev1 | 70 | 1 | 2 | 3 | 31 | 131 | 131 -- alter column tests CREATE TABLE alter_test(time timestamptz, temp float, color varchar(10)); -- create hypertable with two chunks SELECT create_hypertable('alter_test', 'time', 'color', 2, chunk_time_interval => 2628000000000); WARNING: column type "character varying" used for "color" does not follow best practices create_hypertable ------------------------- (9,public,alter_test,t) INSERT INTO alter_test VALUES ('2017-01-20T09:00:01', 17.5, 'blue'), ('2017-01-21T09:00:01', 19.1, 'yellow'), ('2017-04-20T09:00:01', 89.5, 'green'), ('2017-04-21T09:00:01', 17.1, 'black'); SELECT * FROM test.show_columns('alter_test'); Column | Type | NotNull --------+--------------------------+--------- time | timestamp with time zone | t temp | double precision | f color | character varying | f SELECT * FROM test.show_columnsp('_timescaledb_internal._hyper_9_%chunk'); Relation | Kind | Column | Column type | NotNull ----------------------------------------+------+--------+--------------------------+--------- _timescaledb_internal._hyper_9_3_chunk | r | time | timestamp with time zone | t _timescaledb_internal._hyper_9_3_chunk | r | temp | double precision | f _timescaledb_internal._hyper_9_3_chunk | r | color | character varying | f _timescaledb_internal._hyper_9_4_chunk | r | time | timestamp with time zone | t _timescaledb_internal._hyper_9_4_chunk | r | temp | double precision | f _timescaledb_internal._hyper_9_4_chunk | r | color | character varying | f _timescaledb_internal._hyper_9_5_chunk | r | time | timestamp with time zone | t _timescaledb_internal._hyper_9_5_chunk | r | temp | double precision | f _timescaledb_internal._hyper_9_5_chunk | r | color | character varying | f -- show the column name and type of the partitioning dimension in the -- metadata table SELECT * FROM _timescaledb_catalog.dimension WHERE hypertable_id = 9; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+--------------------------+---------+------------+--------------------------+--------------------+-----------------+--------------------------+-------------------------+------------------ 15 | 9 | color | character varying | f | 2 | _timescaledb_functions | get_partition_hash | | | | 14 | 9 | time | timestamp with time zone | t | | | | 2628000000000 | | | EXPLAIN (buffers off, costs off) SELECT * FROM alter_test WHERE time > '2017-05-20T10:00:01'; --- QUERY PLAN --- Append -> Index Scan using _hyper_9_4_chunk_alter_test_time_idx on _hyper_9_4_chunk Index Cond: ("time" > 'Sat May 20 10:00:01 2017 PDT'::timestamp with time zone) -> Index Scan using _hyper_9_5_chunk_alter_test_time_idx on _hyper_9_5_chunk Index Cond: ("time" > 'Sat May 20 10:00:01 2017 PDT'::timestamp with time zone) -- rename column and change its type ALTER TABLE alter_test RENAME COLUMN time TO time_us; --converting timestamptz->timestamp should happen under UTC SET timezone = 'UTC'; ALTER TABLE alter_test ALTER COLUMN time_us TYPE timestamp; RESET timezone; ALTER TABLE alter_test RENAME COLUMN color TO colorname; \set ON_ERROR_STOP 0 -- Changing types on hash-partitioned columns is not safe for some -- types and is therefore blocked. ALTER TABLE alter_test ALTER COLUMN colorname TYPE text; ERROR: cannot change the type of a hash-partitioned column \set ON_ERROR_STOP 1 SELECT * FROM test.show_columns('alter_test'); Column | Type | NotNull -----------+-----------------------------+--------- time_us | timestamp without time zone | t temp | double precision | f colorname | character varying | f SELECT * FROM test.show_columnsp('_timescaledb_internal._hyper_9_%chunk'); Relation | Kind | Column | Column type | NotNull ----------------------------------------+------+-----------+-----------------------------+--------- _timescaledb_internal._hyper_9_3_chunk | r | time_us | timestamp without time zone | t _timescaledb_internal._hyper_9_3_chunk | r | temp | double precision | f _timescaledb_internal._hyper_9_3_chunk | r | colorname | character varying | f _timescaledb_internal._hyper_9_4_chunk | r | time_us | timestamp without time zone | t _timescaledb_internal._hyper_9_4_chunk | r | temp | double precision | f _timescaledb_internal._hyper_9_4_chunk | r | colorname | character varying | f _timescaledb_internal._hyper_9_5_chunk | r | time_us | timestamp without time zone | t _timescaledb_internal._hyper_9_5_chunk | r | temp | double precision | f _timescaledb_internal._hyper_9_5_chunk | r | colorname | character varying | f -- show that the metadata has been updated SELECT * FROM _timescaledb_catalog.dimension WHERE hypertable_id = 9; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+-----------------------------+---------+------------+--------------------------+--------------------+-----------------+--------------------------+-------------------------+------------------ 15 | 9 | colorname | character varying | f | 2 | _timescaledb_functions | get_partition_hash | | | | 14 | 9 | time_us | timestamp without time zone | t | | | | 2628000000000 | | | -- constraint exclusion should still work with updated column EXPLAIN (buffers off, costs off) SELECT * FROM alter_test WHERE time_us > '2017-05-20T10:00:01'; --- QUERY PLAN --- Append -> Seq Scan on _hyper_9_4_chunk Filter: (time_us > 'Sat May 20 10:00:01 2017'::timestamp without time zone) -> Seq Scan on _hyper_9_5_chunk Filter: (time_us > 'Sat May 20 10:00:01 2017'::timestamp without time zone) \set ON_ERROR_STOP 0 -- verify that we cannot change the column type to something incompatible ALTER TABLE alter_test ALTER COLUMN colorname TYPE varchar(3); ERROR: cannot change the type of a hash-partitioned column -- conversion that messes up partitioning fails ALTER TABLE alter_test ALTER COLUMN time_us TYPE timestamptz USING time_us::timestamptz+INTERVAL '1 year'; ERROR: check constraint "constraint_4" of relation "_hyper_9_3_chunk" is violated by some row -- dropping column that messes up partiitoning fails ALTER TABLE alter_test DROP COLUMN colorname; ERROR: cannot drop column named in partition key --ONLY blocked ALTER TABLE ONLY alter_test RENAME COLUMN colorname TO colorname2; ERROR: inherited column "colorname" must be renamed in child tables too ALTER TABLE ONLY alter_test ALTER COLUMN colorname TYPE varchar(10); ERROR: ONLY option not supported on hypertable operations \set ON_ERROR_STOP 1 CREATE TABLE alter_test_bigint(time bigint, temp float); SELECT create_hypertable('alter_test_bigint', 'time', chunk_time_interval => 2628000000000); create_hypertable --------------------------------- (10,public,alter_test_bigint,t) \set ON_ERROR_STOP 0 -- Changing type of time dimension to a non-supported type -- shall not be allowed ALTER TABLE alter_test_bigint ALTER COLUMN time TYPE TEXT; ERROR: cannot change data type of hypertable column "time" from bigint to text -- dropping open time dimension shall not be allowed. ALTER TABLE alter_test_bigint DROP COLUMN time; ERROR: cannot drop column named in partition key \set ON_ERROR_STOP 1 -- test expression index creation where physical layout of chunks differs from hypertable CREATE TABLE i2504(time timestamp NOT NULL, a int, b int, c int, d int); select create_hypertable('i2504', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable --------------------- (11,public,i2504,t) INSERT INTO i2504 VALUES (now(), 1, 2, 3, 4); ALTER TABLE i2504 DROP COLUMN b; INSERT INTO i2504(time, a, c, d) VALUES (now() - interval '1 year', 1, 2, 3), (now() - interval '2 years', 1, 2, 3); CREATE INDEX idx2 ON i2504(a,d) WHERE c IS NOT NULL; DROP INDEX idx2; CREATE INDEX idx2 ON i2504(a,d) WITH (timescaledb.transaction_per_chunk) WHERE c IS NOT NULL; -- Make sure custom composite types are supported as dimensions CREATE TYPE TUPLE as (val1 int4, val2 int4); CREATE TABLE part_custom_dim (time TIMESTAMPTZ, combo TUPLE, device TEXT); \set ON_ERROR_STOP 0 -- should fail on PG < 14 because no partitioning function supplied and the given custom type -- has no default hash function -- on PG14 custom types are hashable SELECT create_hypertable('part_custom_dim', 'time', 'combo', 4); create_hypertable ------------------------------- (12,public,part_custom_dim,t) \set ON_ERROR_STOP 1 -- immutable functions with sub-transaction (issue #4489) CREATE FUNCTION i4489(value TEXT DEFAULT '') RETURNS INTEGER AS $$ BEGIN RETURN value::INTEGER; EXCEPTION WHEN invalid_text_representation THEN RETURN 0; END; $$ LANGUAGE PLPGSQL IMMUTABLE; -- should return 1 (one) in both cases SELECT i4489('1'), i4489('1'); i4489 | i4489 -------+------- 1 | 1 -- should return 0 (zero) in all cases handled by the exception SELECT i4489(), i4489(); i4489 | i4489 -------+------- 0 | 0 SELECT i4489('a'), i4489('a'); i4489 | i4489 -------+------- 0 | 0 -- test ALTER TABLE ONLY for hypertables CREATE TABLE at_test(time timestamptz) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column -- adding column only on the parent table should be blocked \set ON_ERROR_STOP 0 ALTER TABLE ONLY at_test ADD COLUMN value INT; ERROR: ONLY option not supported on hypertable operations \set ON_ERROR_STOP 1 ALTER TABLE ONLY at_test SET (autovacuum_enabled = false); ALTER TABLE ONLY at_test RESET (autovacuum_enabled); -- test again after creating some chunks INSERT INTO at_test VALUES ('2025-01-01'); INSERT INTO at_test VALUES ('2025-02-01'); ALTER TABLE ONLY at_test SET (autovacuum_enabled = false); ALTER TABLE ONLY at_test RESET (autovacuum_enabled); -- test DDL inside function CREATE OR REPLACE FUNCTION ddl_function() RETURNS VOID LANGUAGE PLPGSQL AS $$ BEGIN DROP TABLE IF EXISTS func_table; CREATE TABLE func_table(time timestamptz) WITH (tsdb.hypertable); END $$; SELECT ddl_function(); NOTICE: table "func_table" does not exist, skipping NOTICE: using column "time" as partitioning column ddl_function -------------- SELECT hypertable_name from timescaledb_information.hypertables WHERE hypertable_name='func_table'; hypertable_name ----------------- func_table SELECT ddl_function(); NOTICE: using column "time" as partitioning column ddl_function -------------- SELECT hypertable_name from timescaledb_information.hypertables WHERE hypertable_name='func_table'; hypertable_name ----------------- func_table ================================================ FILE: test/expected/ddl_errors.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."Hypertable_1" ( time BIGINT NOT NULL, "Device_id" TEXT NOT NULL, temp_c int NOT NULL DEFAULT -1 ); CREATE INDEX ON PUBLIC."Hypertable_1" (time, "Device_id"); -- Default integer interval is supported as part of -- hypertable generalization, verify additional secnarios \set ON_ERROR_STOP 0 SELECT * FROM create_hypertable(NULL, NULL); ERROR: relation cannot be NULL SELECT * FROM create_hypertable('"public"."Hypertable_1"', NULL); ERROR: partition column cannot be NULL -- space dimensions require explicit number of partitions SELECT * FROM create_hypertable('"public"."Hypertable_1"', 'time', 'Device_id', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); ERROR: invalid number of partitions for dimension "Device_id" SELECT * FROM create_hypertable('"public"."Hypertable_1_mispelled"', 'time', 'Device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); ERROR: relation "public.Hypertable_1_mispelled" does not exist at character 33 SELECT * FROM create_hypertable('"public"."Hypertable_1"', 'time_mispelled', 'Device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); ERROR: column "time_mispelled" does not exist SELECT * FROM create_hypertable('"public"."Hypertable_1"', 'Device_id', 'Device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); ERROR: invalid type for dimension "Device_id" SELECT * FROM create_hypertable('"public"."Hypertable_1"', 'time', 'Device_id_mispelled', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); ERROR: column "Device_id_mispelled" does not exist INSERT INTO PUBLIC."Hypertable_1" VALUES(1,'dev_1', 3); SELECT * FROM create_hypertable('"public"."Hypertable_1"', 'time', 'Device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); ERROR: table "Hypertable_1" is not empty DELETE FROM PUBLIC."Hypertable_1" ; \set ON_ERROR_STOP 1 SELECT * FROM create_hypertable('"public"."Hypertable_1"', 'time', 'Device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+--------------+--------- 1 | public | Hypertable_1 | t \set ON_ERROR_STOP 0 SELECT * FROM create_hypertable('"public"."Hypertable_1"', 'time', 'Device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); ERROR: table "Hypertable_1" is already a hypertable \set ON_ERROR_STOP 1 INSERT INTO "Hypertable_1" VALUES (0, 1, 0); \set ON_ERROR_STOP 0 ALTER TABLE _timescaledb_internal._hyper_1_1_chunk ALTER COLUMN temp_c DROP NOT NULL; ERROR: operation not supported on chunk tables \set ON_ERROR_STOP 1 CREATE TABLE PUBLIC."Parent" ( time BIGINT NOT NULL, "Device_id" TEXT NOT NULL, temp_c int NOT NULL DEFAULT -1 ); \set ON_ERROR_STOP 0 ALTER TABLE "Hypertable_1" INHERIT "Parent"; ERROR: hypertables do not support inheritance ALTER TABLE _timescaledb_internal._hyper_1_1_chunk INHERIT "Parent"; ERROR: operation not supported on chunk tables ALTER TABLE _timescaledb_internal._hyper_1_1_chunk NO INHERIT "Parent"; ERROR: operation not supported on chunk tables \set ON_ERROR_STOP 1 CREATE TABLE PUBLIC."Child" () INHERITS ("Parent"); \set ON_ERROR_STOP 0 SELECT * FROM create_hypertable('"public"."Parent"', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); ERROR: table "Parent" is already partitioned SELECT * FROM create_hypertable('"public"."Child"', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); ERROR: table "Child" is already partitioned \set ON_ERROR_STOP 1 \set ON_ERROR_STOP 0 CREATE TEMPORARY TABLE temp_table (time timestamptz) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column ERROR: table "temp_table" cannot be temporary \set ON_ERROR_STOP 1 CREATE TEMP TABLE "Hypertable_temp" ( time BIGINT NOT NULL, "Device_id" TEXT NOT NULL, temp_c int NOT NULL DEFAULT -1 ); \set ON_ERROR_STOP 0 SELECT * FROM create_hypertable('"Hypertable_temp"', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); ERROR: table "Hypertable_temp" cannot be temporary ALTER TABLE "Hypertable_1" SET UNLOGGED; \set ON_ERROR_STOP 1 ALTER TABLE "Hypertable_1" SET LOGGED; CREATE TABLE PUBLIC."Hypertable_1_rule" ( time BIGINT NOT NULL, "Device_id" TEXT NOT NULL, temp_c int NOT NULL DEFAULT -1 ); CREATE RULE notify_me AS ON UPDATE TO "Hypertable_1_rule" DO ALSO NOTIFY "Hypertable_1_rule"; \set ON_ERROR_STOP 0 SELECT * FROM create_hypertable('"public"."Hypertable_1_rule"', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); ERROR: hypertables do not support rules \set ON_ERROR_STOP 1 ALTER TABLE "Hypertable_1_rule" DISABLE RULE notify_me; \set ON_ERROR_STOP 0 SELECT * FROM create_hypertable('"public"."Hypertable_1_rule"', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); ERROR: hypertables do not support rules \set ON_ERROR_STOP 1 DROP RULE notify_me ON "Hypertable_1_rule"; SELECT * FROM create_hypertable('"public"."Hypertable_1_rule"', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+-------------------+--------- 3 | public | Hypertable_1_rule | t \set ON_ERROR_STOP 0 CREATE RULE notify_me AS ON UPDATE TO "Hypertable_1_rule" DO ALSO NOTIFY "Hypertable_1_rule"; ERROR: hypertables do not support rules \set ON_ERROR_STOP 1 \set ON_ERROR_STOP 0 SELECT add_dimension(NULL,NULL); ERROR: hypertable cannot be NULL \set ON_ERROR_STOP 1 \set ON_ERROR_STOP 0 SELECT attach_tablespace(NULL,NULL); ERROR: invalid tablespace name \set ON_ERROR_STOP 1 \set ON_ERROR_STOP 0 select set_number_partitions(NULL,NULL); ERROR: hypertable cannot be NULL \set ON_ERROR_STOP 1 ================================================ FILE: test/expected/ddl_extra.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE OR REPLACE FUNCTION show_columns_ext(rel regclass) RETURNS TABLE("Column" name, "Type" text, "NotNull" boolean, "Compression" text) LANGUAGE SQL STABLE AS $BODY$ SELECT a.attname, format_type(t.oid, t.typtypmod), a.attnotnull, (CASE WHEN a.attcompression = 'l' THEN 'lz4' WHEN a.attcompression = 'p' THEN 'pglz' ELSE '' END) FROM pg_attribute a, pg_type t WHERE a.attrelid = rel AND a.atttypid = t.oid AND a.attnum >= 0 ORDER BY a.attnum; $BODY$; CREATE TABLE conditions ( time TIMESTAMP NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL ); SELECT create_hypertable('conditions', 'time', chunk_time_interval := '1 day'::interval); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ------------------------- (1,public,conditions,t) INSERT INTO conditions SELECT generate_series('2021-10-10 00:00'::timestamp, '2021-10-11 00:00'::timestamp, '1 day'), 'POR', 55, 75; CREATE VIEW t AS SELECT 'conditions'::regclass AS r UNION ALL SELECT * FROM show_chunks('conditions'); SELECT * FROM t, LATERAL show_columns_ext(r) WHERE "Column" = 'location' ORDER BY 1, 2; r | Column | Type | NotNull | Compression ----------------------------------------+----------+------+---------+------------- conditions | location | text | t | _timescaledb_internal._hyper_1_1_chunk | location | text | t | _timescaledb_internal._hyper_1_2_chunk | location | text | t | ALTER TABLE conditions ALTER COLUMN location SET COMPRESSION pglz; SELECT * FROM t, LATERAL show_columns_ext(r) WHERE "Column" = 'location' ORDER BY 1, 2; r | Column | Type | NotNull | Compression ----------------------------------------+----------+------+---------+------------- conditions | location | text | t | pglz _timescaledb_internal._hyper_1_1_chunk | location | text | t | pglz _timescaledb_internal._hyper_1_2_chunk | location | text | t | pglz INSERT INTO conditions VALUES ('2021-10-12 00:00'::timestamp, 'BRA', 66, 77); SELECT * FROM t, LATERAL show_columns_ext(r) WHERE "Column" = 'location' ORDER BY 1, 2; r | Column | Type | NotNull | Compression ----------------------------------------+----------+------+---------+------------- conditions | location | text | t | pglz _timescaledb_internal._hyper_1_1_chunk | location | text | t | pglz _timescaledb_internal._hyper_1_2_chunk | location | text | t | pglz _timescaledb_internal._hyper_1_3_chunk | location | text | t | pglz ALTER TABLE conditions ALTER COLUMN location SET COMPRESSION default; SELECT * FROM t, LATERAL show_columns_ext(r) WHERE "Column" = 'location' ORDER BY 1, 2; r | Column | Type | NotNull | Compression ----------------------------------------+----------+------+---------+------------- conditions | location | text | t | _timescaledb_internal._hyper_1_1_chunk | location | text | t | _timescaledb_internal._hyper_1_2_chunk | location | text | t | _timescaledb_internal._hyper_1_3_chunk | location | text | t | \set ON_ERROR_STOP 0 -- failing test because compression is not allowed in "non-TOASTable" datatypes ALTER TABLE conditions ALTER COLUMN temperature SET COMPRESSION pglz; ERROR: column data type double precision does not support compression SELECT * FROM t, LATERAL show_columns_ext(r) WHERE "Column" = 'temperature' ORDER BY 1, 2; r | Column | Type | NotNull | Compression ----------------------------------------+-------------+------------------+---------+------------- conditions | temperature | double precision | f | _timescaledb_internal._hyper_1_1_chunk | temperature | double precision | f | _timescaledb_internal._hyper_1_2_chunk | temperature | double precision | f | _timescaledb_internal._hyper_1_3_chunk | temperature | double precision | f | ================================================ FILE: test/expected/debug_utils.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT _timescaledb_functions.extension_state(); extension_state ----------------- created RESET ROLE; DO $$ DECLARE module text; BEGIN SELECT probin INTO module FROM pg_proc WHERE proname = 'extension_state' AND pronamespace = '_timescaledb_functions'::regnamespace; EXECUTE format('CREATE FUNCTION extension_state() RETURNS TEXT AS ''%s'', ''ts_extension_get_state'' LANGUAGE C', module); END $$; DROP EXTENSION timescaledb; SELECT * FROM extension_state(); extension_state ----------------- unknown \c CREATE EXTENSION timescaledb; SELECT * FROM extension_state(); extension_state ----------------- created ================================================ FILE: test/expected/delete.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \o /dev/null \ir include/insert_two_partitions.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."two_Partitions" ( "timeCustom" BIGINT NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."two_Partitions" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_0) WHERE series_0 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_1) WHERE series_1 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_2) WHERE series_2 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_bool) WHERE series_bool IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, device_id); SELECT * FROM create_hypertable('"public"."two_Partitions"'::regclass, 'timeCustom'::name, 'device_id'::name, associated_schema_name=>'_timescaledb_internal'::text, number_partitions => 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); \set QUIET off BEGIN; \COPY public."two_Partitions" FROM 'data/ds1_dev1_1.tsv' NULL AS ''; COMMIT; INSERT INTO public."two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257987600000000000, 'dev1', 1.5, 1), (1257987600000000000, 'dev1', 1.5, 2), (1257894000000000000, 'dev2', 1.5, 1), (1257894002000000000, 'dev1', 2.5, 3); INSERT INTO "two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257894000000000000, 'dev2', 1.5, 2); \set QUIET on \o SELECT * FROM "two_Partitions" ORDER BY "timeCustom", device_id, series_0, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257894000000000000 | dev1 | 1.5 | 1 | 2 | t 1257894000000000000 | dev1 | 1.5 | 2 | | 1257894000000000000 | dev2 | 1.5 | 1 | | 1257894000000000000 | dev2 | 1.5 | 2 | | 1257894000000001000 | dev1 | 2.5 | 3 | | 1257894001000000000 | dev1 | 3.5 | 4 | | 1257894002000000000 | dev1 | 2.5 | 3 | | 1257894002000000000 | dev1 | 5.5 | 6 | | t 1257894002000000000 | dev1 | 5.5 | 7 | | f 1257897600000000000 | dev1 | 4.5 | 5 | | f 1257987600000000000 | dev1 | 1.5 | 1 | | 1257987600000000000 | dev1 | 1.5 | 2 | | DELETE FROM "two_Partitions" WHERE series_0 = 1.5; DELETE FROM "two_Partitions" WHERE series_0 = 100; SELECT * FROM "two_Partitions" ORDER BY "timeCustom", device_id, series_0, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257894000000001000 | dev1 | 2.5 | 3 | | 1257894001000000000 | dev1 | 3.5 | 4 | | 1257894002000000000 | dev1 | 2.5 | 3 | | 1257894002000000000 | dev1 | 5.5 | 6 | | t 1257894002000000000 | dev1 | 5.5 | 7 | | f 1257897600000000000 | dev1 | 4.5 | 5 | | f -- Make sure DELETE isn't optimized if it includes Append plans -- Need to turn of nestloop to make append appear the same on PG96 and PG10 set enable_nestloop = 'off'; CREATE OR REPLACE FUNCTION series_val() RETURNS integer LANGUAGE PLPGSQL STABLE AS $BODY$ BEGIN RETURN 5; END; $BODY$; -- ConstraintAwareAppend applied for SELECT EXPLAIN (buffers off, costs off) SELECT FROM "two_Partitions" WHERE series_1 IN (SELECT series_1 FROM "two_Partitions" WHERE series_1 > series_val()); --- QUERY PLAN --- Hash Join Hash Cond: ("two_Partitions".series_1 = "two_Partitions_1".series_1) -> Custom Scan (ChunkAppend) on "two_Partitions" Chunks excluded during startup: 0 -> Index Only Scan using "_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" on _hyper_1_1_chunk Index Cond: (series_1 > (series_val())::double precision) -> Index Only Scan using "_hyper_1_2_chunk_two_Partitions_timeCustom_series_1_idx" on _hyper_1_2_chunk Index Cond: (series_1 > (series_val())::double precision) -> Index Only Scan using "_hyper_1_3_chunk_two_Partitions_timeCustom_series_1_idx" on _hyper_1_3_chunk Index Cond: (series_1 > (series_val())::double precision) -> Index Only Scan using "_hyper_1_4_chunk_two_Partitions_timeCustom_series_1_idx" on _hyper_1_4_chunk Index Cond: (series_1 > (series_val())::double precision) -> Hash -> HashAggregate Group Key: "two_Partitions_1".series_1 -> Custom Scan (ChunkAppend) on "two_Partitions" "two_Partitions_1" Chunks excluded during startup: 0 -> Index Only Scan using "_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" on _hyper_1_1_chunk _hyper_1_1_chunk_1 Index Cond: (series_1 > (series_val())::double precision) -> Index Only Scan using "_hyper_1_2_chunk_two_Partitions_timeCustom_series_1_idx" on _hyper_1_2_chunk _hyper_1_2_chunk_1 Index Cond: (series_1 > (series_val())::double precision) -> Index Only Scan using "_hyper_1_3_chunk_two_Partitions_timeCustom_series_1_idx" on _hyper_1_3_chunk _hyper_1_3_chunk_1 Index Cond: (series_1 > (series_val())::double precision) -> Index Only Scan using "_hyper_1_4_chunk_two_Partitions_timeCustom_series_1_idx" on _hyper_1_4_chunk _hyper_1_4_chunk_1 Index Cond: (series_1 > (series_val())::double precision) -- ConstraintAwareAppend NOT applied for DELETE EXPLAIN (buffers off, costs off) DELETE FROM "two_Partitions" WHERE series_1 IN (SELECT series_1 FROM "two_Partitions" WHERE series_1 > series_val()); --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Delete on "two_Partitions" Delete on _hyper_1_1_chunk "two_Partitions_2" Delete on _hyper_1_2_chunk "two_Partitions_3" Delete on _hyper_1_3_chunk "two_Partitions_4" Delete on _hyper_1_4_chunk "two_Partitions_5" -> Hash Join Hash Cond: ("two_Partitions".series_1 = "two_Partitions_1".series_1) -> Append -> Seq Scan on _hyper_1_1_chunk "two_Partitions_2" -> Seq Scan on _hyper_1_2_chunk "two_Partitions_3" -> Seq Scan on _hyper_1_3_chunk "two_Partitions_4" -> Seq Scan on _hyper_1_4_chunk "two_Partitions_5" -> Hash -> HashAggregate Group Key: "two_Partitions_1".series_1 -> Append -> Index Scan using "_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" on _hyper_1_1_chunk "two_Partitions_6" Index Cond: (series_1 > (series_val())::double precision) -> Index Scan using "_hyper_1_2_chunk_two_Partitions_timeCustom_series_1_idx" on _hyper_1_2_chunk "two_Partitions_7" Index Cond: (series_1 > (series_val())::double precision) -> Index Scan using "_hyper_1_3_chunk_two_Partitions_timeCustom_series_1_idx" on _hyper_1_3_chunk "two_Partitions_8" Index Cond: (series_1 > (series_val())::double precision) -> Index Scan using "_hyper_1_4_chunk_two_Partitions_timeCustom_series_1_idx" on _hyper_1_4_chunk "two_Partitions_9" Index Cond: (series_1 > (series_val())::double precision) SELECT * FROM "two_Partitions" ORDER BY "timeCustom", device_id, series_0, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257894000000001000 | dev1 | 2.5 | 3 | | 1257894001000000000 | dev1 | 3.5 | 4 | | 1257894002000000000 | dev1 | 2.5 | 3 | | 1257894002000000000 | dev1 | 5.5 | 6 | | t 1257894002000000000 | dev1 | 5.5 | 7 | | f 1257897600000000000 | dev1 | 4.5 | 5 | | f BEGIN; DELETE FROM "two_Partitions" WHERE series_1 IN (SELECT series_1 FROM "two_Partitions" WHERE series_1 > series_val()); SELECT * FROM "two_Partitions" ORDER BY "timeCustom", device_id, series_0, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257894000000001000 | dev1 | 2.5 | 3 | | 1257894001000000000 | dev1 | 3.5 | 4 | | 1257894002000000000 | dev1 | 2.5 | 3 | | 1257897600000000000 | dev1 | 4.5 | 5 | | f ROLLBACK; BEGIN; DELETE FROM "two_Partitions" WHERE series_1 IN (SELECT series_1 FROM "two_Partitions" WHERE series_1 > series_val()) RETURNING "timeCustom"; timeCustom --------------------- 1257894002000000000 1257894002000000000 SELECT * FROM "two_Partitions" ORDER BY "timeCustom", device_id, series_0, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257894000000001000 | dev1 | 2.5 | 3 | | 1257894001000000000 | dev1 | 3.5 | 4 | | 1257894002000000000 | dev1 | 2.5 | 3 | | 1257897600000000000 | dev1 | 4.5 | 5 | | f ROLLBACK; -- test update on chunks directly CREATE TABLE direct_delete(time timestamptz) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column INSERT INTO direct_delete VALUES ('2020-01-01'); SELECT show_chunks('direct_delete') AS "CHUNK" \gset --should have ModifyHyperable node EXPLAIN (costs off, timing off, summary off) DELETE FROM :CHUNK; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Delete on _hyper_2_5_chunk -> Seq Scan on _hyper_2_5_chunk EXPLAIN (costs off, timing off, summary off) DELETE FROM ONLY :CHUNK; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Delete on _hyper_2_5_chunk -> Seq Scan on _hyper_2_5_chunk -- DELETE should succeed BEGIN; DELETE FROM :CHUNK RETURNING *; time ------------------------------ Wed Jan 01 00:00:00 2020 PST ROLLBACK; BEGIN; DELETE FROM ONLY :CHUNK RETURNING *; time ------------------------------ Wed Jan 01 00:00:00 2020 PST ROLLBACK; -- Test that EXPLAIN VERBOSE on prepared statements does not corrupt cached plans. SET plan_cache_mode = 'force_generic_plan'; CREATE TABLE explain_verbose_ht( time timestamptz NOT NULL, device int, value float) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column INSERT INTO explain_verbose_ht SELECT t, 1, 0.1 FROM generate_series('2026-01-01'::timestamptz, '2026-01-08'::timestamptz, interval '6 hours') t; -- Verify the DELETE plan uses ChunkAppend EXPLAIN (costs off) DELETE FROM explain_verbose_ht WHERE time > '2025-01-01'::text::timestamptz; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Delete on explain_verbose_ht Delete on _hyper_3_6_chunk explain_verbose_ht_1 Delete on _hyper_3_7_chunk explain_verbose_ht_2 -> Custom Scan (ChunkAppend) on explain_verbose_ht Chunks excluded during startup: 0 -> Index Scan using _hyper_3_6_chunk_explain_verbose_ht_time_idx on _hyper_3_6_chunk explain_verbose_ht_1 Index Cond: ("time" > ('2025-01-01'::cstring)::timestamp with time zone) -> Index Scan using _hyper_3_7_chunk_explain_verbose_ht_time_idx on _hyper_3_7_chunk explain_verbose_ht_2 Index Cond: ("time" > ('2025-01-01'::cstring)::timestamp with time zone) PREPARE delete_ht AS DELETE FROM explain_verbose_ht WHERE time > '2025-01-01'::text::timestamptz AND device = 2; EXECUTE delete_ht; EXPLAIN (verbose, costs off) EXECUTE delete_ht; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Delete on public.explain_verbose_ht Delete on _timescaledb_internal._hyper_3_6_chunk explain_verbose_ht_1 Delete on _timescaledb_internal._hyper_3_7_chunk explain_verbose_ht_2 -> Custom Scan (ChunkAppend) on public.explain_verbose_ht Startup Exclusion: true Runtime Exclusion: false Chunks excluded during startup: 0 -> Index Scan using _hyper_3_6_chunk_explain_verbose_ht_time_idx on _timescaledb_internal._hyper_3_6_chunk explain_verbose_ht_1 Output: explain_verbose_ht_1.tableoid, explain_verbose_ht_1.ctid Index Cond: (explain_verbose_ht_1."time" > ('2025-01-01'::cstring)::timestamp with time zone) Filter: (explain_verbose_ht_1.device = 2) -> Index Scan using _hyper_3_7_chunk_explain_verbose_ht_time_idx on _timescaledb_internal._hyper_3_7_chunk explain_verbose_ht_2 Output: explain_verbose_ht_2.tableoid, explain_verbose_ht_2.ctid Index Cond: (explain_verbose_ht_2."time" > ('2025-01-01'::cstring)::timestamp with time zone) Filter: (explain_verbose_ht_2.device = 2) EXECUTE delete_ht; DEALLOCATE delete_ht; -- repeat test with explain analyze PREPARE delete_ht AS DELETE FROM explain_verbose_ht WHERE time > '2025-01-01'::text::timestamptz AND device = 2; EXECUTE delete_ht; EXPLAIN (verbose, analyze, buffers off, costs off, timing off, summary off) EXECUTE delete_ht; --- QUERY PLAN --- Custom Scan (ModifyHypertable) (actual rows=0.00 loops=1) -> Delete on public.explain_verbose_ht (actual rows=0.00 loops=1) Delete on _timescaledb_internal._hyper_3_6_chunk explain_verbose_ht_1 Delete on _timescaledb_internal._hyper_3_7_chunk explain_verbose_ht_2 -> Custom Scan (ChunkAppend) on public.explain_verbose_ht (actual rows=0.00 loops=1) Startup Exclusion: true Runtime Exclusion: false Chunks excluded during startup: 0 -> Index Scan using _hyper_3_6_chunk_explain_verbose_ht_time_idx on _timescaledb_internal._hyper_3_6_chunk explain_verbose_ht_1 (actual rows=0.00 loops=1) Output: explain_verbose_ht_1.tableoid, explain_verbose_ht_1.ctid Index Cond: (explain_verbose_ht_1."time" > ('2025-01-01'::cstring)::timestamp with time zone) Filter: (explain_verbose_ht_1.device = 2) Rows Removed by Filter: 27 -> Index Scan using _hyper_3_7_chunk_explain_verbose_ht_time_idx on _timescaledb_internal._hyper_3_7_chunk explain_verbose_ht_2 (actual rows=0.00 loops=1) Output: explain_verbose_ht_2.tableoid, explain_verbose_ht_2.ctid Index Cond: (explain_verbose_ht_2."time" > ('2025-01-01'::cstring)::timestamp with time zone) Filter: (explain_verbose_ht_2.device = 2) Rows Removed by Filter: 2 EXECUTE delete_ht; DEALLOCATE delete_ht; RESET plan_cache_mode; -- github issue #6790 -- test DELETE with WHERE EXISTS on hypertable CREATE TABLE i6790(time timestamptz NOT NULL, device int, value float) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column INSERT INTO i6790 SELECT t, 1, 0.1 FROM generate_series('2026-01-01'::timestamptz, '2026-01-03'::timestamptz, interval '12 hours') t; -- DELETE with simple EXISTS - creates gating Result node wrapping ChunkAppend DELETE FROM i6790 WHERE EXISTS (SELECT 1); -- all rows should be gone SELECT count(*) FROM i6790; count ------- 0 -- repopulate for next test INSERT INTO i6790 SELECT t, 1, 0.1 FROM generate_series('2026-01-01'::timestamptz, '2026-01-03'::timestamptz, interval '12 hours') t; -- DELETE with correlated EXISTS DELETE FROM i6790 WHERE EXISTS (SELECT 1 FROM i6790 g WHERE g.device = i6790.device); SELECT count(*) FROM i6790; count ------- 0 ================================================ FILE: test/expected/drop_extension.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE drop_test(time timestamp, temp float8, device text); SELECT create_hypertable('drop_test', 'time', 'device', 2); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ------------------------ (1,public,drop_test,t) SELECT * FROM _timescaledb_catalog.hypertable; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 1 | public | drop_test | _timescaledb_internal | _hyper_1 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 INSERT INTO drop_test VALUES('Mon Mar 20 09:17:00.936242 2017', 23.4, 'dev1'); SELECT * FROM drop_test; time | temp | device ---------------------------------+------+-------- Mon Mar 20 09:17:00.936242 2017 | 23.4 | dev1 \c :TEST_DBNAME :ROLE_SUPERUSER DROP EXTENSION timescaledb CASCADE; NOTICE: drop cascades to table _timescaledb_internal._hyper_1_1_chunk \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- Querying the original table should not return any rows since all of -- them actually existed in chunks that are now gone SELECT * FROM drop_test; time | temp | device ------+------+-------- \c :TEST_DBNAME :ROLE_SUPERUSER -- Recreate the extension SET client_min_messages=error; CREATE EXTENSION timescaledb; RESET client_min_messages; -- Test that calling twice generates proper error \set ON_ERROR_STOP 0 CREATE EXTENSION timescaledb; ERROR: extension "timescaledb" has already been loaded with another version \set ON_ERROR_STOP 1 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- CREATE twice with IF NOT EXISTS should be OK CREATE EXTENSION IF NOT EXISTS timescaledb; NOTICE: extension "timescaledb" already exists, skipping -- Make the table a hypertable again SELECT create_hypertable('drop_test', 'time', 'device', 2); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ------------------------ (1,public,drop_test,t) SELECT * FROM _timescaledb_catalog.hypertable; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 1 | public | drop_test | _timescaledb_internal | _hyper_1 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 INSERT INTO drop_test VALUES('Mon Mar 20 09:18:19.100462 2017', 22.1, 'dev1'); SELECT * FROM drop_test; time | temp | device ---------------------------------+------+-------- Mon Mar 20 09:18:19.100462 2017 | 22.1 | dev1 --test drops thru cascades of other objects \c :TEST_DBNAME :ROLE_SUPERUSER -- Stop background workers to prevent them from interfering with the drop public schema SELECT _timescaledb_functions.stop_background_workers(); stop_background_workers ------------------------- t SET client_min_messages TO ERROR; REVOKE CONNECT ON DATABASE :TEST_DBNAME FROM public; SELECT count(pg_terminate_backend(pg_stat_activity.pid)) AS TERMINATED FROM pg_stat_activity WHERE pg_stat_activity.datname = :'TEST_DBNAME' AND pg_stat_activity.pid <> pg_backend_pid() \gset RESET client_min_messages; -- drop the public schema and all its objects DROP SCHEMA public CASCADE; NOTICE: drop cascades to 3 other objects \dn List of schemas Name | Owner ------+------------ test | super_user -- Recreate the public schema and extension in the same session. -- This should work without requiring a reconnect (issue #5884). CREATE SCHEMA public; SET client_min_messages=error; CREATE EXTENSION timescaledb SCHEMA public; RESET client_min_messages; SELECT extname FROM pg_extension WHERE extname = 'timescaledb'; extname ------------- timescaledb -- Verify the extension is functional after re-creation CREATE TABLE drop_test2(time timestamptz, temp float8); SELECT create_hypertable('drop_test2', 'time'); create_hypertable ------------------------- (1,public,drop_test2,t) INSERT INTO drop_test2 VALUES('2024-01-01', 23.4); SELECT * FROM drop_test2; time | temp ------------------------------+------ Mon Jan 01 00:00:00 2024 PST | 23.4 DROP TABLE drop_test2; -- Test that dropping and recreating extension directly also works in the same session DROP EXTENSION timescaledb CASCADE; SET client_min_messages=error; CREATE EXTENSION timescaledb; RESET client_min_messages; SELECT extname FROM pg_extension WHERE extname = 'timescaledb'; extname ------------- timescaledb ================================================ FILE: test/expected/drop_hypertable.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SELECT * from _timescaledb_catalog.hypertable; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+------------+------------------------+-------------------------+----------------+--------------------------+------------------------+-------------------+-------------------+--------------------------+-------- SELECT * from _timescaledb_catalog.dimension; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+-------------+---------+------------+--------------------------+-------------------+-----------------+--------------------------+-------------------------+------------------ CREATE TABLE should_drop (time timestamp, temp float8); SELECT create_hypertable('should_drop', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------- (1,public,should_drop,t) CREATE TABLE hyper_with_dependencies (time timestamp, temp float8); SELECT create_hypertable('hyper_with_dependencies', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------------------- (2,public,hyper_with_dependencies,t) CREATE VIEW dependent_view AS SELECT * FROM hyper_with_dependencies; INSERT INTO hyper_with_dependencies VALUES (now(), 1.0); \set ON_ERROR_STOP 0 DROP TABLE hyper_with_dependencies; ERROR: cannot drop table hyper_with_dependencies because other objects depend on it \set ON_ERROR_STOP 1 DROP TABLE hyper_with_dependencies CASCADE; NOTICE: drop cascades to view dependent_view -- check that the view is dropped SELECT oid FROM pg_class WHERE relname = 'dependent_view'; oid ----- CREATE TABLE chunk_with_dependencies (time timestamp, temp float8); SELECT create_hypertable('chunk_with_dependencies', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------------------- (3,public,chunk_with_dependencies,t) INSERT INTO chunk_with_dependencies VALUES (now(), 1.0); CREATE VIEW dependent_view_chunk AS SELECT * FROM _timescaledb_internal._hyper_3_2_chunk; \set ON_ERROR_STOP 0 DROP TABLE chunk_with_dependencies; ERROR: cannot drop table _timescaledb_internal._hyper_3_2_chunk because other objects depend on it \set ON_ERROR_STOP 1 DROP TABLE chunk_with_dependencies CASCADE; NOTICE: drop cascades to view dependent_view_chunk -- check that the view is dropped SELECT oid FROM pg_class WHERE relname = 'dependent_view_chunk'; oid ----- -- Calling create hypertable again will increment hypertable ID -- although no new hypertable is created. Make sure we can handle this. SELECT create_hypertable('should_drop', 'time', if_not_exists => true); NOTICE: table "should_drop" is already a hypertable, skipping create_hypertable -------------------------- (1,public,should_drop,f) SELECT * from _timescaledb_catalog.hypertable; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+-------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 1 | public | should_drop | _timescaledb_internal | _hyper_1 | 1 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 SELECT * from _timescaledb_catalog.dimension; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+-----------------------------+---------+------------+--------------------------+-------------------+-----------------+--------------------------+-------------------------+------------------ 1 | 1 | time | timestamp without time zone | t | | | | 604800000000 | | | DROP TABLE should_drop; CREATE TABLE should_drop (time timestamp, temp float8); SELECT create_hypertable('should_drop', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------- (4,public,should_drop,t) INSERT INTO should_drop VALUES (now(), 1.0); SELECT * from _timescaledb_catalog.hypertable; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+-------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 4 | public | should_drop | _timescaledb_internal | _hyper_4 | 1 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 SELECT * from _timescaledb_catalog.dimension; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+-----------------------------+---------+------------+--------------------------+-------------------+-----------------+--------------------------+-------------------------+------------------ 4 | 4 | time | timestamp without time zone | t | | | | 604800000000 | | | -- test dropping multiple objects at once CREATE TABLE t1 (time timestamptz) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column INSERT INTO t1 VALUES ('2025-01-01'); CREATE TABLE t2 (time timestamptz) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column INSERT INTO t2 VALUES ('2025-01-01'); CREATE TABLE t3 (time timestamptz); DROP TABLE t1, t2, t3; ================================================ FILE: test/expected/drop_owned-15.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA hypertable_schema; GRANT ALL ON SCHEMA hypertable_schema TO :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE TABLE hypertable_schema.default_perm_user (time timestamptz, temp float, location int); SELECT create_hypertable('hypertable_schema.default_perm_user', 'time', 'location', 2); create_hypertable ------------------------------------------- (1,hypertable_schema,default_perm_user,t) INSERT INTO hypertable_schema.default_perm_user VALUES ('2001-01-01 01:01:01', 23.3, 1); RESET ROLE; CREATE TABLE hypertable_schema.superuser (time timestamptz, temp float, location int); SELECT create_hypertable('hypertable_schema.superuser', 'time', 'location', 2); create_hypertable ----------------------------------- (2,hypertable_schema,superuser,t) INSERT INTO hypertable_schema.superuser VALUES ('2001-01-01 01:01:01', 23.3, 1); SELECT * FROM _timescaledb_catalog.hypertable ORDER BY id; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------------+-------------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 1 | hypertable_schema | default_perm_user | _timescaledb_internal | _hyper_1 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 2 | hypertable_schema | superuser | _timescaledb_internal | _hyper_2 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-----------------------+------------------+---------------------+--------+----------- 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | | 0 | f 2 | 2 | _timescaledb_internal | _hyper_2_2_chunk | | 0 | f DROP OWNED BY :ROLE_DEFAULT_PERM_USER; SELECT * FROM _timescaledb_catalog.hypertable ORDER BY id; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------------+------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 2 | hypertable_schema | superuser | _timescaledb_internal | _hyper_2 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-----------------------+------------------+---------------------+--------+----------- 2 | 2 | _timescaledb_internal | _hyper_2_2_chunk | | 0 | f DROP TABLE hypertable_schema.superuser; --everything should be cleaned up SELECT * FROM _timescaledb_catalog.hypertable GROUP BY id; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+------------+------------------------+-------------------------+----------------+--------------------------+------------------------+-------------------+-------------------+--------------------------+-------- SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-------------+------------+---------------------+--------+----------- SELECT * FROM _timescaledb_catalog.dimension; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+-------------+---------+------------+--------------------------+-------------------+-----------------+--------------------------+-------------------------+------------------ SELECT * FROM _timescaledb_catalog.dimension_slice; id | dimension_id | range_start | range_end ----+--------------+-------------+----------- SELECT * FROM _timescaledb_catalog.chunk_constraint; chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----------+--------------------+-----------------+---------------------------- -- test drop owned in database without extension installed \c :TEST_DBNAME :ROLE_SUPERUSER CREATE database test_drop_owned; \c test_drop_owned DROP OWNED BY :ROLE_SUPERUSER; \c :TEST_DBNAME :ROLE_SUPERUSER DROP DATABASE test_drop_owned WITH (FORCE); -- Test that dependencies on roles are added to chunks when creating -- new chunks. If that is not done, DROP OWNED BY will not revoke the -- privilege on the chunk. CREATE TABLE sensor_data(time timestamptz not null, cpu double precision null); SELECT * FROM create_hypertable('sensor_data','time'); hypertable_id | schema_name | table_name | created ---------------+-------------+-------------+--------- 3 | public | sensor_data | t INSERT INTO sensor_data SELECT time, random() AS cpu FROM generate_series('2020-01-01'::timestamptz, '2020-01-24'::timestamptz, INTERVAL '10 minute') AS g1(time); \dp sensor_data Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+-------------+-------+-------------------+-------------------+---------- public | sensor_data | table | | | \dp _timescaledb_internal._hyper_3* Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------+-------------------+---------- _timescaledb_internal | _hyper_3_3_chunk | table | | | _timescaledb_internal | _hyper_3_4_chunk | table | | | _timescaledb_internal | _hyper_3_5_chunk | table | | | _timescaledb_internal | _hyper_3_6_chunk | table | | | _timescaledb_internal | _hyper_3_7_chunk | table | | | GRANT SELECT ON sensor_data TO :ROLE_DEFAULT_PERM_USER; \dp sensor_data Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+-------------+-------+--------------------------------+-------------------+---------- public | sensor_data | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | \dp _timescaledb_internal._hyper_3* Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_3_3_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_4_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_5_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_6_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_7_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | -- Insert more chunks after adding the user to the hypertable. These -- will now get the privileges of the hypertable. INSERT INTO sensor_data SELECT time, random() AS cpu FROM generate_series('2020-01-20'::timestamptz, '2020-02-05'::timestamptz, INTERVAL '10 minute') AS g1(time); \dp _timescaledb_internal._hyper_3* Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_3_3_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_4_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_5_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_6_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_7_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_8_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | -- This should revoke the privileges on both the hypertable and the chunks. DROP OWNED BY :ROLE_DEFAULT_PERM_USER; \dp sensor_data Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+-------------+-------+-------------------------------+-------------------+---------- public | sensor_data | table | super_user=arwdDxt/super_user | | \dp _timescaledb_internal._hyper_3* Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_3_3_chunk | table | super_user=arwdDxt/super_user | | _timescaledb_internal | _hyper_3_4_chunk | table | super_user=arwdDxt/super_user | | _timescaledb_internal | _hyper_3_5_chunk | table | super_user=arwdDxt/super_user | | _timescaledb_internal | _hyper_3_6_chunk | table | super_user=arwdDxt/super_user | | _timescaledb_internal | _hyper_3_7_chunk | table | super_user=arwdDxt/super_user | | _timescaledb_internal | _hyper_3_8_chunk | table | super_user=arwdDxt/super_user | | ================================================ FILE: test/expected/drop_owned-16.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA hypertable_schema; GRANT ALL ON SCHEMA hypertable_schema TO :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE TABLE hypertable_schema.default_perm_user (time timestamptz, temp float, location int); SELECT create_hypertable('hypertable_schema.default_perm_user', 'time', 'location', 2); create_hypertable ------------------------------------------- (1,hypertable_schema,default_perm_user,t) INSERT INTO hypertable_schema.default_perm_user VALUES ('2001-01-01 01:01:01', 23.3, 1); RESET ROLE; CREATE TABLE hypertable_schema.superuser (time timestamptz, temp float, location int); SELECT create_hypertable('hypertable_schema.superuser', 'time', 'location', 2); create_hypertable ----------------------------------- (2,hypertable_schema,superuser,t) INSERT INTO hypertable_schema.superuser VALUES ('2001-01-01 01:01:01', 23.3, 1); SELECT * FROM _timescaledb_catalog.hypertable ORDER BY id; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------------+-------------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 1 | hypertable_schema | default_perm_user | _timescaledb_internal | _hyper_1 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 2 | hypertable_schema | superuser | _timescaledb_internal | _hyper_2 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-----------------------+------------------+---------------------+--------+----------- 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | | 0 | f 2 | 2 | _timescaledb_internal | _hyper_2_2_chunk | | 0 | f DROP OWNED BY :ROLE_DEFAULT_PERM_USER; SELECT * FROM _timescaledb_catalog.hypertable ORDER BY id; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------------+------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 2 | hypertable_schema | superuser | _timescaledb_internal | _hyper_2 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-----------------------+------------------+---------------------+--------+----------- 2 | 2 | _timescaledb_internal | _hyper_2_2_chunk | | 0 | f DROP TABLE hypertable_schema.superuser; --everything should be cleaned up SELECT * FROM _timescaledb_catalog.hypertable GROUP BY id; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+------------+------------------------+-------------------------+----------------+--------------------------+------------------------+-------------------+-------------------+--------------------------+-------- SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-------------+------------+---------------------+--------+----------- SELECT * FROM _timescaledb_catalog.dimension; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+-------------+---------+------------+--------------------------+-------------------+-----------------+--------------------------+-------------------------+------------------ SELECT * FROM _timescaledb_catalog.dimension_slice; id | dimension_id | range_start | range_end ----+--------------+-------------+----------- SELECT * FROM _timescaledb_catalog.chunk_constraint; chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----------+--------------------+-----------------+---------------------------- -- test drop owned in database without extension installed \c :TEST_DBNAME :ROLE_SUPERUSER CREATE database test_drop_owned; \c test_drop_owned DROP OWNED BY :ROLE_SUPERUSER; \c :TEST_DBNAME :ROLE_SUPERUSER DROP DATABASE test_drop_owned WITH (FORCE); -- Test that dependencies on roles are added to chunks when creating -- new chunks. If that is not done, DROP OWNED BY will not revoke the -- privilege on the chunk. CREATE TABLE sensor_data(time timestamptz not null, cpu double precision null); SELECT * FROM create_hypertable('sensor_data','time'); hypertable_id | schema_name | table_name | created ---------------+-------------+-------------+--------- 3 | public | sensor_data | t INSERT INTO sensor_data SELECT time, random() AS cpu FROM generate_series('2020-01-01'::timestamptz, '2020-01-24'::timestamptz, INTERVAL '10 minute') AS g1(time); \dp sensor_data Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+-------------+-------+-------------------+-------------------+---------- public | sensor_data | table | | | \dp _timescaledb_internal._hyper_3* Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------+-------------------+---------- _timescaledb_internal | _hyper_3_3_chunk | table | | | _timescaledb_internal | _hyper_3_4_chunk | table | | | _timescaledb_internal | _hyper_3_5_chunk | table | | | _timescaledb_internal | _hyper_3_6_chunk | table | | | _timescaledb_internal | _hyper_3_7_chunk | table | | | GRANT SELECT ON sensor_data TO :ROLE_DEFAULT_PERM_USER; \dp sensor_data Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+-------------+-------+--------------------------------+-------------------+---------- public | sensor_data | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | \dp _timescaledb_internal._hyper_3* Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_3_3_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_4_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_5_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_6_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_7_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | -- Insert more chunks after adding the user to the hypertable. These -- will now get the privileges of the hypertable. INSERT INTO sensor_data SELECT time, random() AS cpu FROM generate_series('2020-01-20'::timestamptz, '2020-02-05'::timestamptz, INTERVAL '10 minute') AS g1(time); \dp _timescaledb_internal._hyper_3* Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_3_3_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_4_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_5_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_6_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_7_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_8_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user=r/super_user | | -- This should revoke the privileges on both the hypertable and the chunks. DROP OWNED BY :ROLE_DEFAULT_PERM_USER; \dp sensor_data Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+-------------+-------+-------------------------------+-------------------+---------- public | sensor_data | table | super_user=arwdDxt/super_user | | \dp _timescaledb_internal._hyper_3* Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_3_3_chunk | table | super_user=arwdDxt/super_user | | _timescaledb_internal | _hyper_3_4_chunk | table | super_user=arwdDxt/super_user | | _timescaledb_internal | _hyper_3_5_chunk | table | super_user=arwdDxt/super_user | | _timescaledb_internal | _hyper_3_6_chunk | table | super_user=arwdDxt/super_user | | _timescaledb_internal | _hyper_3_7_chunk | table | super_user=arwdDxt/super_user | | _timescaledb_internal | _hyper_3_8_chunk | table | super_user=arwdDxt/super_user | | ================================================ FILE: test/expected/drop_owned-17.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA hypertable_schema; GRANT ALL ON SCHEMA hypertable_schema TO :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE TABLE hypertable_schema.default_perm_user (time timestamptz, temp float, location int); SELECT create_hypertable('hypertable_schema.default_perm_user', 'time', 'location', 2); create_hypertable ------------------------------------------- (1,hypertable_schema,default_perm_user,t) INSERT INTO hypertable_schema.default_perm_user VALUES ('2001-01-01 01:01:01', 23.3, 1); RESET ROLE; CREATE TABLE hypertable_schema.superuser (time timestamptz, temp float, location int); SELECT create_hypertable('hypertable_schema.superuser', 'time', 'location', 2); create_hypertable ----------------------------------- (2,hypertable_schema,superuser,t) INSERT INTO hypertable_schema.superuser VALUES ('2001-01-01 01:01:01', 23.3, 1); SELECT * FROM _timescaledb_catalog.hypertable ORDER BY id; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------------+-------------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 1 | hypertable_schema | default_perm_user | _timescaledb_internal | _hyper_1 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 2 | hypertable_schema | superuser | _timescaledb_internal | _hyper_2 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-----------------------+------------------+---------------------+--------+----------- 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | | 0 | f 2 | 2 | _timescaledb_internal | _hyper_2_2_chunk | | 0 | f DROP OWNED BY :ROLE_DEFAULT_PERM_USER; SELECT * FROM _timescaledb_catalog.hypertable ORDER BY id; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------------+------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 2 | hypertable_schema | superuser | _timescaledb_internal | _hyper_2 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-----------------------+------------------+---------------------+--------+----------- 2 | 2 | _timescaledb_internal | _hyper_2_2_chunk | | 0 | f DROP TABLE hypertable_schema.superuser; --everything should be cleaned up SELECT * FROM _timescaledb_catalog.hypertable GROUP BY id; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+------------+------------------------+-------------------------+----------------+--------------------------+------------------------+-------------------+-------------------+--------------------------+-------- SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-------------+------------+---------------------+--------+----------- SELECT * FROM _timescaledb_catalog.dimension; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+-------------+---------+------------+--------------------------+-------------------+-----------------+--------------------------+-------------------------+------------------ SELECT * FROM _timescaledb_catalog.dimension_slice; id | dimension_id | range_start | range_end ----+--------------+-------------+----------- SELECT * FROM _timescaledb_catalog.chunk_constraint; chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----------+--------------------+-----------------+---------------------------- -- test drop owned in database without extension installed \c :TEST_DBNAME :ROLE_SUPERUSER CREATE database test_drop_owned; \c test_drop_owned DROP OWNED BY :ROLE_SUPERUSER; \c :TEST_DBNAME :ROLE_SUPERUSER DROP DATABASE test_drop_owned WITH (FORCE); -- Test that dependencies on roles are added to chunks when creating -- new chunks. If that is not done, DROP OWNED BY will not revoke the -- privilege on the chunk. CREATE TABLE sensor_data(time timestamptz not null, cpu double precision null); SELECT * FROM create_hypertable('sensor_data','time'); hypertable_id | schema_name | table_name | created ---------------+-------------+-------------+--------- 3 | public | sensor_data | t INSERT INTO sensor_data SELECT time, random() AS cpu FROM generate_series('2020-01-01'::timestamptz, '2020-01-24'::timestamptz, INTERVAL '10 minute') AS g1(time); \dp sensor_data Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+-------------+-------+-------------------+-------------------+---------- public | sensor_data | table | | | \dp _timescaledb_internal._hyper_3* Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------+-------------------+---------- _timescaledb_internal | _hyper_3_3_chunk | table | | | _timescaledb_internal | _hyper_3_4_chunk | table | | | _timescaledb_internal | _hyper_3_5_chunk | table | | | _timescaledb_internal | _hyper_3_6_chunk | table | | | _timescaledb_internal | _hyper_3_7_chunk | table | | | GRANT SELECT ON sensor_data TO :ROLE_DEFAULT_PERM_USER; \dp sensor_data Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+-------------+-------+--------------------------------+-------------------+---------- public | sensor_data | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | \dp _timescaledb_internal._hyper_3* Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_3_3_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_4_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_5_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_6_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_7_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | -- Insert more chunks after adding the user to the hypertable. These -- will now get the privileges of the hypertable. INSERT INTO sensor_data SELECT time, random() AS cpu FROM generate_series('2020-01-20'::timestamptz, '2020-02-05'::timestamptz, INTERVAL '10 minute') AS g1(time); \dp _timescaledb_internal._hyper_3* Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_3_3_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_4_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_5_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_6_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_7_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_8_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | -- This should revoke the privileges on both the hypertable and the chunks. DROP OWNED BY :ROLE_DEFAULT_PERM_USER; \dp sensor_data Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+-------------+-------+--------------------------------+-------------------+---------- public | sensor_data | table | super_user=arwdDxtm/super_user | | \dp _timescaledb_internal._hyper_3* Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_3_3_chunk | table | super_user=arwdDxtm/super_user | | _timescaledb_internal | _hyper_3_4_chunk | table | super_user=arwdDxtm/super_user | | _timescaledb_internal | _hyper_3_5_chunk | table | super_user=arwdDxtm/super_user | | _timescaledb_internal | _hyper_3_6_chunk | table | super_user=arwdDxtm/super_user | | _timescaledb_internal | _hyper_3_7_chunk | table | super_user=arwdDxtm/super_user | | _timescaledb_internal | _hyper_3_8_chunk | table | super_user=arwdDxtm/super_user | | ================================================ FILE: test/expected/drop_owned-18.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA hypertable_schema; GRANT ALL ON SCHEMA hypertable_schema TO :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE TABLE hypertable_schema.default_perm_user (time timestamptz, temp float, location int); SELECT create_hypertable('hypertable_schema.default_perm_user', 'time', 'location', 2); create_hypertable ------------------------------------------- (1,hypertable_schema,default_perm_user,t) INSERT INTO hypertable_schema.default_perm_user VALUES ('2001-01-01 01:01:01', 23.3, 1); RESET ROLE; CREATE TABLE hypertable_schema.superuser (time timestamptz, temp float, location int); SELECT create_hypertable('hypertable_schema.superuser', 'time', 'location', 2); create_hypertable ----------------------------------- (2,hypertable_schema,superuser,t) INSERT INTO hypertable_schema.superuser VALUES ('2001-01-01 01:01:01', 23.3, 1); SELECT * FROM _timescaledb_catalog.hypertable ORDER BY id; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------------+-------------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 1 | hypertable_schema | default_perm_user | _timescaledb_internal | _hyper_1 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 2 | hypertable_schema | superuser | _timescaledb_internal | _hyper_2 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-----------------------+------------------+---------------------+--------+----------- 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | | 0 | f 2 | 2 | _timescaledb_internal | _hyper_2_2_chunk | | 0 | f DROP OWNED BY :ROLE_DEFAULT_PERM_USER; SELECT * FROM _timescaledb_catalog.hypertable ORDER BY id; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------------+------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 2 | hypertable_schema | superuser | _timescaledb_internal | _hyper_2 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-----------------------+------------------+---------------------+--------+----------- 2 | 2 | _timescaledb_internal | _hyper_2_2_chunk | | 0 | f DROP TABLE hypertable_schema.superuser; --everything should be cleaned up SELECT * FROM _timescaledb_catalog.hypertable GROUP BY id; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+------------+------------------------+-------------------------+----------------+--------------------------+------------------------+-------------------+-------------------+--------------------------+-------- SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-------------+------------+---------------------+--------+----------- SELECT * FROM _timescaledb_catalog.dimension; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+-------------+---------+------------+--------------------------+-------------------+-----------------+--------------------------+-------------------------+------------------ SELECT * FROM _timescaledb_catalog.dimension_slice; id | dimension_id | range_start | range_end ----+--------------+-------------+----------- SELECT * FROM _timescaledb_catalog.chunk_constraint; chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----------+--------------------+-----------------+---------------------------- -- test drop owned in database without extension installed \c :TEST_DBNAME :ROLE_SUPERUSER CREATE database test_drop_owned; \c test_drop_owned DROP OWNED BY :ROLE_SUPERUSER; \c :TEST_DBNAME :ROLE_SUPERUSER DROP DATABASE test_drop_owned WITH (FORCE); -- Test that dependencies on roles are added to chunks when creating -- new chunks. If that is not done, DROP OWNED BY will not revoke the -- privilege on the chunk. CREATE TABLE sensor_data(time timestamptz not null, cpu double precision null); SELECT * FROM create_hypertable('sensor_data','time'); hypertable_id | schema_name | table_name | created ---------------+-------------+-------------+--------- 3 | public | sensor_data | t INSERT INTO sensor_data SELECT time, random() AS cpu FROM generate_series('2020-01-01'::timestamptz, '2020-01-24'::timestamptz, INTERVAL '10 minute') AS g1(time); \dp sensor_data Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+-------------+-------+-------------------+-------------------+---------- public | sensor_data | table | | | \dp _timescaledb_internal._hyper_3* Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------+-------------------+---------- _timescaledb_internal | _hyper_3_3_chunk | table | | | _timescaledb_internal | _hyper_3_4_chunk | table | | | _timescaledb_internal | _hyper_3_5_chunk | table | | | _timescaledb_internal | _hyper_3_6_chunk | table | | | _timescaledb_internal | _hyper_3_7_chunk | table | | | GRANT SELECT ON sensor_data TO :ROLE_DEFAULT_PERM_USER; \dp sensor_data Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+-------------+-------+--------------------------------+-------------------+---------- public | sensor_data | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | \dp _timescaledb_internal._hyper_3* Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_3_3_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_4_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_5_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_6_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_7_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | -- Insert more chunks after adding the user to the hypertable. These -- will now get the privileges of the hypertable. INSERT INTO sensor_data SELECT time, random() AS cpu FROM generate_series('2020-01-20'::timestamptz, '2020-02-05'::timestamptz, INTERVAL '10 minute') AS g1(time); \dp _timescaledb_internal._hyper_3* Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_3_3_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_4_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_5_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_6_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_7_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | _timescaledb_internal | _hyper_3_8_chunk | table | super_user=arwdDxtm/super_user+| | | | | default_perm_user=r/super_user | | -- This should revoke the privileges on both the hypertable and the chunks. DROP OWNED BY :ROLE_DEFAULT_PERM_USER; \dp sensor_data Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+-------------+-------+--------------------------------+-------------------+---------- public | sensor_data | table | super_user=arwdDxtm/super_user | | \dp _timescaledb_internal._hyper_3* Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_3_3_chunk | table | super_user=arwdDxtm/super_user | | _timescaledb_internal | _hyper_3_4_chunk | table | super_user=arwdDxtm/super_user | | _timescaledb_internal | _hyper_3_5_chunk | table | super_user=arwdDxtm/super_user | | _timescaledb_internal | _hyper_3_6_chunk | table | super_user=arwdDxtm/super_user | | _timescaledb_internal | _hyper_3_7_chunk | table | super_user=arwdDxtm/super_user | | _timescaledb_internal | _hyper_3_8_chunk | table | super_user=arwdDxtm/super_user | | ================================================ FILE: test/expected/drop_rename_hypertable.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \o /dev/null \ir include/insert_two_partitions.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."two_Partitions" ( "timeCustom" BIGINT NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."two_Partitions" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_0) WHERE series_0 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_1) WHERE series_1 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_2) WHERE series_2 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_bool) WHERE series_bool IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, device_id); SELECT * FROM create_hypertable('"public"."two_Partitions"'::regclass, 'timeCustom'::name, 'device_id'::name, associated_schema_name=>'_timescaledb_internal'::text, number_partitions => 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); \set QUIET off BEGIN; \COPY public."two_Partitions" FROM 'data/ds1_dev1_1.tsv' NULL AS ''; COMMIT; INSERT INTO public."two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257987600000000000, 'dev1', 1.5, 1), (1257987600000000000, 'dev1', 1.5, 2), (1257894000000000000, 'dev2', 1.5, 1), (1257894002000000000, 'dev1', 2.5, 3); INSERT INTO "two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257894000000000000, 'dev2', 1.5, 2); \set QUIET on \o SELECT * FROM test.show_columnsp('_timescaledb_internal.%_hyper%'); Relation | Kind | Column | Column type | NotNull ------------------------------------------------------------------------------------+------+-------------+------------------+--------- _timescaledb_internal._hyper_1_1_chunk | r | timeCustom | bigint | t _timescaledb_internal._hyper_1_1_chunk | r | device_id | text | t _timescaledb_internal._hyper_1_1_chunk | r | series_0 | double precision | f _timescaledb_internal._hyper_1_1_chunk | r | series_1 | double precision | f _timescaledb_internal._hyper_1_1_chunk | r | series_2 | double precision | f _timescaledb_internal._hyper_1_1_chunk | r | series_bool | boolean | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_device_id_timeCustom_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_device_id_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_device_id_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_device_id_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_0_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_0_idx" | i | series_0 | double precision | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" | i | series_1 | double precision | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_2_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_2_idx" | i | series_2 | double precision | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_bool_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_bool_idx" | i | series_bool | boolean | f _timescaledb_internal._hyper_1_2_chunk | r | timeCustom | bigint | t _timescaledb_internal._hyper_1_2_chunk | r | device_id | text | t _timescaledb_internal._hyper_1_2_chunk | r | series_0 | double precision | f _timescaledb_internal._hyper_1_2_chunk | r | series_1 | double precision | f _timescaledb_internal._hyper_1_2_chunk | r | series_2 | double precision | f _timescaledb_internal._hyper_1_2_chunk | r | series_bool | boolean | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_device_id_timeCustom_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_device_id_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_device_id_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_device_id_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_0_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_0_idx" | i | series_0 | double precision | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_1_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_1_idx" | i | series_1 | double precision | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_2_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_2_idx" | i | series_2 | double precision | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_bool_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_bool_idx" | i | series_bool | boolean | f _timescaledb_internal._hyper_1_3_chunk | r | timeCustom | bigint | t _timescaledb_internal._hyper_1_3_chunk | r | device_id | text | t _timescaledb_internal._hyper_1_3_chunk | r | series_0 | double precision | f _timescaledb_internal._hyper_1_3_chunk | r | series_1 | double precision | f _timescaledb_internal._hyper_1_3_chunk | r | series_2 | double precision | f _timescaledb_internal._hyper_1_3_chunk | r | series_bool | boolean | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_device_id_timeCustom_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_device_id_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_device_id_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_device_id_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_0_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_0_idx" | i | series_0 | double precision | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_1_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_1_idx" | i | series_1 | double precision | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_2_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_2_idx" | i | series_2 | double precision | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_bool_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_bool_idx" | i | series_bool | boolean | f _timescaledb_internal._hyper_1_4_chunk | r | timeCustom | bigint | t _timescaledb_internal._hyper_1_4_chunk | r | device_id | text | t _timescaledb_internal._hyper_1_4_chunk | r | series_0 | double precision | f _timescaledb_internal._hyper_1_4_chunk | r | series_1 | double precision | f _timescaledb_internal._hyper_1_4_chunk | r | series_2 | double precision | f _timescaledb_internal._hyper_1_4_chunk | r | series_bool | boolean | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_device_id_timeCustom_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_device_id_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_device_id_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_device_id_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_0_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_0_idx" | i | series_0 | double precision | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_1_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_1_idx" | i | series_1 | double precision | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_2_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_2_idx" | i | series_2 | double precision | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_bool_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_bool_idx" | i | series_bool | boolean | f -- Test that renaming hypertable works SELECT * FROM test.show_columns('_timescaledb_internal._hyper_1_1_chunk'); Column | Type | NotNull -------------+------------------+--------- timeCustom | bigint | t device_id | text | t series_0 | double precision | f series_1 | double precision | f series_2 | double precision | f series_bool | boolean | f ALTER TABLE "two_Partitions" RENAME TO "newname"; SELECT * FROM "newname"; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257894000000000000 | dev1 | 1.5 | 1 | 2 | t 1257894000000000000 | dev1 | 1.5 | 2 | | 1257894000000001000 | dev1 | 2.5 | 3 | | 1257894001000000000 | dev1 | 3.5 | 4 | | 1257894002000000000 | dev1 | 5.5 | 6 | | t 1257894002000000000 | dev1 | 5.5 | 7 | | f 1257894002000000000 | dev1 | 2.5 | 3 | | 1257897600000000000 | dev1 | 4.5 | 5 | | f 1257987600000000000 | dev1 | 1.5 | 1 | | 1257987600000000000 | dev1 | 1.5 | 2 | | 1257894000000000000 | dev2 | 1.5 | 1 | | 1257894000000000000 | dev2 | 1.5 | 2 | | SELECT * FROM _timescaledb_catalog.hypertable; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 1 | public | newname | _timescaledb_internal | _hyper_1 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA "newschema" AUTHORIZATION :ROLE_DEFAULT_PERM_USER; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER ALTER TABLE "newname" SET SCHEMA "newschema"; SELECT * FROM "newschema"."newname"; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257894000000000000 | dev1 | 1.5 | 1 | 2 | t 1257894000000000000 | dev1 | 1.5 | 2 | | 1257894000000001000 | dev1 | 2.5 | 3 | | 1257894001000000000 | dev1 | 3.5 | 4 | | 1257894002000000000 | dev1 | 5.5 | 6 | | t 1257894002000000000 | dev1 | 5.5 | 7 | | f 1257894002000000000 | dev1 | 2.5 | 3 | | 1257897600000000000 | dev1 | 4.5 | 5 | | f 1257987600000000000 | dev1 | 1.5 | 1 | | 1257987600000000000 | dev1 | 1.5 | 2 | | 1257894000000000000 | dev2 | 1.5 | 1 | | 1257894000000000000 | dev2 | 1.5 | 2 | | SELECT * FROM _timescaledb_catalog.hypertable; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 1 | newschema | newname | _timescaledb_internal | _hyper_1 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 DROP TABLE "newschema"."newname"; SELECT * FROM _timescaledb_catalog.hypertable; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+------------+------------------------+-------------------------+----------------+--------------------------+------------------------+-------------------+-------------------+--------------------------+-------- SELECT schema, name FROM test.relation WHERE schema IN ('public', '_timescaledb_catalog', '_timescaledb_internal'); schema | name -----------------------+-------------------------------------------------- _timescaledb_catalog | bgw_job _timescaledb_catalog | chunk _timescaledb_catalog | chunk_column_stats _timescaledb_catalog | chunk_constraint _timescaledb_catalog | chunk_rewrite _timescaledb_catalog | compression_algorithm _timescaledb_catalog | compression_chunk_size _timescaledb_catalog | compression_settings _timescaledb_catalog | continuous_agg _timescaledb_catalog | continuous_aggs_bucket_function _timescaledb_catalog | continuous_aggs_hypertable_invalidation_log _timescaledb_catalog | continuous_aggs_invalidation_threshold _timescaledb_catalog | continuous_aggs_materialization_invalidation_log _timescaledb_catalog | continuous_aggs_materialization_ranges _timescaledb_catalog | continuous_aggs_watermark _timescaledb_catalog | dimension _timescaledb_catalog | dimension_slice _timescaledb_catalog | hypertable _timescaledb_catalog | metadata _timescaledb_catalog | tablespace _timescaledb_catalog | telemetry_event _timescaledb_internal | bgw_job_stat _timescaledb_internal | bgw_job_stat_history _timescaledb_internal | bgw_policy_chunk_stats _timescaledb_internal | compressed_chunk_stats _timescaledb_internal | hypertable_chunk_local_size -- Test that renaming ordinary table works CREATE TABLE renametable (foo int); ALTER TABLE "renametable" RENAME TO "newname_none_ht"; SELECT * FROM "newname_none_ht"; foo ----- ================================================ FILE: test/expected/drop_schema.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA chunk_schema1; CREATE SCHEMA chunk_schema2; CREATE SCHEMA hypertable_schema; CREATE SCHEMA extra_schema; GRANT ALL ON SCHEMA hypertable_schema TO :ROLE_DEFAULT_PERM_USER; GRANT ALL ON SCHEMA chunk_schema1 TO :ROLE_DEFAULT_PERM_USER; GRANT ALL ON SCHEMA chunk_schema2 TO :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE TABLE hypertable_schema.test1 (time timestamptz, temp float, location int); CREATE TABLE hypertable_schema.test2 (time timestamptz, temp float, location int); --create two identical tables with their own chunk schemas SELECT create_hypertable('hypertable_schema.test1', 'time', 'location', 2, associated_schema_name => 'chunk_schema1'); create_hypertable ------------------------------- (1,hypertable_schema,test1,t) SELECT create_hypertable('hypertable_schema.test2', 'time', 'location', 2, associated_schema_name => 'chunk_schema2'); create_hypertable ------------------------------- (2,hypertable_schema,test2,t) INSERT INTO hypertable_schema.test1 VALUES ('2001-01-01 01:01:01', 23.3, 1); INSERT INTO hypertable_schema.test2 VALUES ('2001-01-01 01:01:01', 23.3, 1); SELECT * FROM _timescaledb_catalog.hypertable ORDER BY id; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------------+------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 1 | hypertable_schema | test1 | chunk_schema1 | _hyper_1 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 2 | hypertable_schema | test2 | chunk_schema2 | _hyper_2 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+---------------+------------------+---------------------+--------+----------- 1 | 1 | chunk_schema1 | _hyper_1_1_chunk | | 0 | f 2 | 2 | chunk_schema2 | _hyper_2_2_chunk | | 0 | f RESET ROLE; --drop the associated schema. We drop the extra schema to show we can --handle multi-schema drops DROP SCHEMA chunk_schema1, extra_schema CASCADE; NOTICE: drop cascades to table chunk_schema1._hyper_1_1_chunk NOTICE: the chunk storage schema changed to "_timescaledb_internal" for 1 hypertable SET ROLE :ROLE_DEFAULT_PERM_USER; --show that the metadata for the table using the dropped schema is --changed. The other table is not affected. SELECT * FROM _timescaledb_catalog.hypertable ORDER BY id; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------------+------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 1 | hypertable_schema | test1 | _timescaledb_internal | _hyper_1 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 2 | hypertable_schema | test2 | chunk_schema2 | _hyper_2 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+---------------+------------------+---------------------+--------+----------- 2 | 2 | chunk_schema2 | _hyper_2_2_chunk | | 0 | f --new chunk should be created in the internal associated schema INSERT INTO hypertable_schema.test1 VALUES ('2001-01-01 01:01:01', 23.3, 1); SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-----------------------+------------------+---------------------+--------+----------- 2 | 2 | chunk_schema2 | _hyper_2_2_chunk | | 0 | f 3 | 1 | _timescaledb_internal | _hyper_1_3_chunk | | 0 | f RESET ROLE; --dropping the internal schema should not work \set ON_ERROR_STOP 0 DROP SCHEMA _timescaledb_internal CASCADE; ERROR: cannot drop schema _timescaledb_internal because extension timescaledb requires it \set ON_ERROR_STOP 1 --dropping the hypertable schema should delete everything DROP SCHEMA hypertable_schema CASCADE; NOTICE: drop cascades to 4 other objects SET ROLE :ROLE_DEFAULT_PERM_USER; --everything should be cleaned up SELECT * FROM _timescaledb_catalog.hypertable GROUP BY id; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+------------+------------------------+-------------------------+----------------+--------------------------+------------------------+-------------------+-------------------+--------------------------+-------- SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-------------+------------+---------------------+--------+----------- SELECT * FROM _timescaledb_catalog.dimension; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+-------------+---------+------------+--------------------------+-------------------+-----------------+--------------------------+-------------------------+------------------ SELECT * FROM _timescaledb_catalog.dimension_slice; id | dimension_id | range_start | range_end ----+--------------+-------------+----------- SELECT * FROM _timescaledb_catalog.chunk_constraint; chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----------+--------------------+-----------------+---------------------------- ================================================ FILE: test/expected/dump_meta.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \ir include/insert_two_partitions.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."two_Partitions" ( "timeCustom" BIGINT NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."two_Partitions" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_0) WHERE series_0 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_1) WHERE series_1 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_2) WHERE series_2 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_bool) WHERE series_bool IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, device_id); SELECT * FROM create_hypertable('"public"."two_Partitions"'::regclass, 'timeCustom'::name, 'device_id'::name, associated_schema_name=>'_timescaledb_internal'::text, number_partitions => 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+----------------+--------- 1 | public | two_Partitions | t \set QUIET off BEGIN; BEGIN \COPY public."two_Partitions" FROM 'data/ds1_dev1_1.tsv' NULL AS ''; COPY 7 COMMIT; COMMIT INSERT INTO public."two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257987600000000000, 'dev1', 1.5, 1), (1257987600000000000, 'dev1', 1.5, 2), (1257894000000000000, 'dev2', 1.5, 1), (1257894002000000000, 'dev1', 2.5, 3); INSERT 0 4 INSERT INTO "two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257894000000000000, 'dev2', 1.5, 2); INSERT 0 1 \set QUIET on \ir ../../scripts/dump_meta_data.sql -- -- This file is licensed under the Apache License, see LICENSE-APACHE -- at the top level directory of the TimescaleDB distribution. -- This script will dump relevant meta data from internal TimescaleDB tables -- that can help our engineers trouble shoot. -- -- usage: -- psql [your connect flags] -d your_timescale_db < dump_meta_data.sql > dumpfile.txt \echo 'TimescaleDB meta data dump' TimescaleDB meta data dump </exclude_from_test> \echo 'List of tables' List of tables SELECT n.nspname as "Schema", c.relname as "Name", CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 't' THEN 'TOAST table' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'partitioned table' WHEN 'I' THEN 'partitioned index' END as "Type", pg_catalog.pg_get_userbyid(c.relowner) as "Owner" FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace LEFT JOIN pg_catalog.pg_am am ON am.oid = c.relam WHERE c.relkind IN ('r','p','') AND n.nspname <> 'pg_catalog' AND n.nspname !~ '^pg_toast' AND n.nspname <> 'information_schema' AND pg_catalog.pg_table_is_visible(c.oid) ORDER BY 1,2; Schema | Name | Type | Owner --------+----------------+-------+------------------- public | two_Partitions | table | default_perm_user \echo 'List of hypertables' List of hypertables SELECT * FROM _timescaledb_catalog.hypertable; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+----------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 1 | public | two_Partitions | _timescaledb_internal | _hyper_1 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 \echo 'List of chunk indexes' List of chunk indexes SELECT ch.id AS chunk_id, ci.indexrelid::regclass::text AS index_name, h.id AS hypertable_id FROM _timescaledb_catalog.hypertable h JOIN _timescaledb_catalog.chunk ch ON ch.hypertable_id = h.id JOIN pg_index ci ON ci.indrelid = format('%I.%I', ch.schema_name, ch.table_name)::regclass ORDER BY h.id, ch.id, ci.indrelid::regclass::text; chunk_id | index_name | hypertable_id ----------+------------------------------------------------------------------------------------+--------------- 1 | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_idx" | 1 1 | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_device_id_idx" | 1 1 | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_bool_idx" | 1 1 | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_2_idx" | 1 1 | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" | 1 1 | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_0_idx" | 1 1 | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_device_id_timeCustom_idx" | 1 2 | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_idx" | 1 2 | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_device_id_idx" | 1 2 | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_bool_idx" | 1 2 | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_2_idx" | 1 2 | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_1_idx" | 1 2 | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_0_idx" | 1 2 | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_device_id_timeCustom_idx" | 1 3 | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_idx" | 1 3 | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_device_id_idx" | 1 3 | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_bool_idx" | 1 3 | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_2_idx" | 1 3 | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_1_idx" | 1 3 | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_0_idx" | 1 3 | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_device_id_timeCustom_idx" | 1 4 | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_idx" | 1 4 | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_device_id_idx" | 1 4 | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_bool_idx" | 1 4 | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_2_idx" | 1 4 | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_1_idx" | 1 4 | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_0_idx" | 1 4 | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_device_id_timeCustom_idx" | 1 \echo 'Size of hypertables' Size of hypertables SELECT hypertable, table_bytes, index_bytes, toast_bytes, total_bytes FROM ( SELECT *, total_bytes-index_bytes-COALESCE(toast_bytes,0) AS table_bytes FROM ( SELECT pgc.oid::regclass::text as hypertable, sum(pg_total_relation_size('"' || c.schema_name || '"."' || c.table_name || '"'))::bigint as total_bytes, sum(pg_indexes_size('"' || c.schema_name || '"."' || c.table_name || '"'))::bigint AS index_bytes, sum(pg_total_relation_size(reltoastrelid))::bigint AS toast_bytes FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.chunk c, pg_class pgc, pg_namespace pns WHERE c.hypertable_id = h.id AND pgc.relname = h.table_name AND pns.oid = pgc.relnamespace AND pns.nspname = h.schema_name AND relkind = 'r' GROUP BY pgc.oid ) sub1 ) sub2; hypertable | table_bytes | index_bytes | toast_bytes | total_bytes ------------------+-------------+-------------+-------------+------------- "two_Partitions" | 32768 | 417792 | 32768 | 483328 \echo 'Chunk sizes:' Chunk sizes: SELECT chunk_id, chunk_table, partitioning_columns, partitioning_column_types, partitioning_hash_functions, ranges, table_bytes, index_bytes, toast_bytes, total_bytes FROM ( SELECT *, total_bytes-index_bytes-COALESCE(toast_bytes,0) AS table_bytes FROM ( SELECT c.id as chunk_id, '"' || c.schema_name || '"."' || c.table_name || '"' as chunk_table, pg_total_relation_size('"' || c.schema_name || '"."' || c.table_name || '"') AS total_bytes, pg_indexes_size('"' || c.schema_name || '"."' || c.table_name || '"') AS index_bytes, pg_total_relation_size(reltoastrelid) AS toast_bytes, array_agg(d.column_name ORDER BY d.interval_length, d.column_name ASC) as partitioning_columns, array_agg(d.column_type ORDER BY d.interval_length, d.column_name ASC) as partitioning_column_types, array_agg(d.partitioning_func_schema || '.' || d.partitioning_func ORDER BY d.interval_length, d.column_name ASC) as partitioning_hash_functions, array_agg('[' || _timescaledb_functions.range_value_to_pretty(range_start, column_type) || ',' || _timescaledb_functions.range_value_to_pretty(range_end, column_type) || ')' ORDER BY d.interval_length, d.column_name ASC) as ranges FROM _timescaledb_catalog.hypertable h, _timescaledb_catalog.chunk c, _timescaledb_catalog.chunk_constraint cc, _timescaledb_catalog.dimension d, _timescaledb_catalog.dimension_slice ds, pg_class pgc, pg_namespace pns WHERE pgc.relname = h.table_name AND pns.oid = pgc.relnamespace AND pns.nspname = h.schema_name AND relkind = 'r' AND c.hypertable_id = h.id AND c.id = cc.chunk_id AND cc.dimension_slice_id = ds.id AND ds.dimension_id = d.id GROUP BY c.id, pgc.reltoastrelid, pgc.oid ORDER BY c.id ) sub1 ) sub2; chunk_id | chunk_table | partitioning_columns | partitioning_column_types | partitioning_hash_functions | ranges | table_bytes | index_bytes | toast_bytes | total_bytes ----------+--------------------------------------------+------------------------+---------------------------+--------------------------------------------------+-------------------------------------------------------------------+-------------+-------------+-------------+------------- 1 | "_timescaledb_internal"."_hyper_1_1_chunk" | {timeCustom,device_id} | {bigint,text} | {NULL,_timescaledb_functions.get_partition_hash} | {"['1257892416000000000','1257895008000000000')","[1073741823,)"} | 8192 | 114688 | 8192 | 131072 2 | "_timescaledb_internal"."_hyper_1_2_chunk" | {timeCustom,device_id} | {bigint,text} | {NULL,_timescaledb_functions.get_partition_hash} | {"['1257897600000000000','1257900192000000000')","[1073741823,)"} | 8192 | 106496 | 8192 | 122880 3 | "_timescaledb_internal"."_hyper_1_3_chunk" | {timeCustom,device_id} | {bigint,text} | {NULL,_timescaledb_functions.get_partition_hash} | {"['1257985728000000000','1257988320000000000')","[1073741823,)"} | 8192 | 98304 | 8192 | 114688 4 | "_timescaledb_internal"."_hyper_1_4_chunk" | {timeCustom,device_id} | {bigint,text} | {NULL,_timescaledb_functions.get_partition_hash} | {"['1257892416000000000','1257895008000000000')","[,1073741823)"} | 8192 | 98304 | 8192 | 114688 \echo 'Hypertable index sizes' Hypertable index sizes SET search_path TO pg_catalog, pg_temp; SELECT i.indrelid::regclass AS hypertable, i.indexrelid::regclass AS index_name, public.hypertable_index_size(i.indexrelid::regclass) AS index_bytes FROM _timescaledb_catalog.hypertable h JOIN pg_index i ON i.indrelid = format('%I.%I',h.schema_name,h.table_name)::regclass ORDER BY i.indrelid::regclass::text, i.indexrelid::regclass::text; hypertable | index_name | index_bytes -------------------------+----------------------------------------------------+------------- public."two_Partitions" | public."two_Partitions_device_id_timeCustom_idx" | 73728 public."two_Partitions" | public."two_Partitions_timeCustom_device_id_idx" | 73728 public."two_Partitions" | public."two_Partitions_timeCustom_idx" | 73728 public."two_Partitions" | public."two_Partitions_timeCustom_series_0_idx" | 73728 public."two_Partitions" | public."two_Partitions_timeCustom_series_1_idx" | 73728 public."two_Partitions" | public."two_Partitions_timeCustom_series_2_idx" | 49152 public."two_Partitions" | public."two_Partitions_timeCustom_series_bool_idx" | 57344 RESET search_path; ================================================ FILE: test/expected/extension_scripts.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER DROP EXTENSION timescaledb; -- test that installation script errors when any of our internal schemas already exists \set ON_ERROR_STOP 0 CREATE SCHEMA _timescaledb_catalog; CREATE EXTENSION timescaledb; ERROR: schema "_timescaledb_catalog" already exists DROP SCHEMA _timescaledb_catalog; \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA _timescaledb_internal; CREATE EXTENSION timescaledb; ERROR: schema "_timescaledb_internal" already exists DROP SCHEMA _timescaledb_internal; \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA _timescaledb_cache; CREATE EXTENSION timescaledb; ERROR: schema "_timescaledb_cache" already exists DROP SCHEMA _timescaledb_cache; \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA timescaledb_experimental; CREATE EXTENSION timescaledb; ERROR: schema "timescaledb_experimental" already exists DROP SCHEMA timescaledb_experimental; \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA timescaledb_information; CREATE EXTENSION timescaledb; ERROR: schema "timescaledb_information" already exists DROP SCHEMA timescaledb_information; -- test that installation script errors when any of the function in public schema already exists -- we don't test every public function but just a few common ones \c :TEST_DBNAME :ROLE_SUPERUSER CREATE FUNCTION time_bucket(int,int) RETURNS int LANGUAGE SQL AS $$ SELECT 1::int; $$; CREATE EXTENSION timescaledb; ERROR: function "time_bucket" already exists with same argument types DROP FUNCTION time_bucket; \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION show_chunks(relation regclass, older_than "any" DEFAULT NULL, newer_than "any" DEFAULT NULL, created_before "any" DEFAULT NULL, created_after "any" DEFAULT NULL) RETURNS SETOF regclass language internal as 'pg_partition_ancestors'; CREATE EXTENSION timescaledb; ERROR: function "show_chunks" already exists with same argument types DROP FUNCTION show_chunks; -- Create a user that is not all-lowercase CREATE USER "FooBar" WITH SUPERUSER; \c :TEST_DBNAME "FooBar" SET client_min_messages TO error; CREATE EXTENSION timescaledb; DROP EXTENSION timescaledb; RESET client_min_messages; \c :TEST_DBNAME :ROLE_SUPERUSER DROP USER "FooBar"; SET client_min_messages TO ERROR; CREATE EXTENSION timescaledb; ================================================ FILE: test/expected/generated_as_identity.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE table test_gen ( id int generated by default AS IDENTITY primary key, payload text ); SELECT create_hypertable('test_gen', 'id', chunk_time_interval=>10); create_hypertable ----------------------- (1,public,test_gen,t) insert into test_gen (payload) select generate_series(1,15) returning *; id | payload ----+--------- 1 | 1 2 | 2 3 | 3 4 | 4 5 | 5 6 | 6 7 | 7 8 | 8 9 | 9 10 | 10 11 | 11 12 | 12 13 | 13 14 | 14 15 | 15 select * from test_gen; id | payload ----+--------- 1 | 1 2 | 2 3 | 3 4 | 4 5 | 5 6 | 6 7 | 7 8 | 8 9 | 9 10 | 10 11 | 11 12 | 12 13 | 13 14 | 14 15 | 15 \set ON_ERROR_STOP 0 insert into test_gen values('1', 'a'); ERROR: duplicate key value violates unique constraint "1_1_test_gen_pkey" \set ON_ERROR_STOP 1 ALTER TABLE test_gen ALTER COLUMN id DROP IDENTITY; \set ON_ERROR_STOP 0 insert into test_gen (payload) select generate_series(15,20) returning *; ERROR: NULL value in column "id" violates not-null constraint \set ON_ERROR_STOP 1 ALTER TABLE test_gen ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY; \set ON_ERROR_STOP 0 insert into test_gen (payload) select generate_series(15,20) returning *; ERROR: duplicate key value violates unique constraint "1_1_test_gen_pkey" \set ON_ERROR_STOP 1 ALTER TABLE test_gen ALTER COLUMN id SET GENERATED BY DEFAULT RESTART 100; insert into test_gen (payload) select generate_series(15,20) returning *; id | payload -----+--------- 100 | 15 101 | 16 102 | 17 103 | 18 104 | 19 105 | 20 select * from test_gen; id | payload -----+--------- 1 | 1 2 | 2 3 | 3 4 | 4 5 | 5 6 | 6 7 | 7 8 | 8 9 | 9 10 | 10 11 | 11 12 | 12 13 | 13 14 | 14 15 | 15 100 | 15 101 | 16 102 | 17 103 | 18 104 | 19 105 | 20 SELECT * FROM test.show_subtables('test_gen'); Child | Tablespace ----------------------------------------+------------ _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_3_chunk | ================================================ FILE: test/expected/grant_hypertable-15.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE TABLE conditions( time TIMESTAMPTZ NOT NULL, device INTEGER, temperature FLOAT ); -- Create a hypertable and show that it does not have any privileges SELECT * FROM create_hypertable('conditions', 'time', chunk_time_interval => '5 days'::interval); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | conditions | t INSERT INTO conditions SELECT time, (random()*30)::int, random()*80 - 40 FROM generate_series('2018-12-01 00:00'::timestamp, '2018-12-10 00:00'::timestamp, '1h') AS time; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------+-------------------+---------- public | conditions | table | | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | | | _timescaledb_internal | _hyper_1_2_chunk | table | | | _timescaledb_internal | _hyper_1_3_chunk | table | | | -- Add privileges and show that they propagate to the chunks GRANT SELECT, INSERT ON conditions TO PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | -- Create some more chunks and show that they also get the privileges. INSERT INTO conditions SELECT time, (random()*30)::int, random()*80 - 40 FROM generate_series('2018-12-10 00:00'::timestamp, '2018-12-20 00:00'::timestamp, '1h') AS time; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_4_chunk | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_5_chunk | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | -- Revoke one of the privileges and show that it propagate to the -- chunks. REVOKE INSERT ON conditions FROM PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_4_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_5_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | -- Add some more chunks and show that it inherits the grants from the -- hypertable. INSERT INTO conditions SELECT time, (random()*30)::int, random()*80 - 40 FROM generate_series('2018-12-20 00:00'::timestamp, '2018-12-30 00:00'::timestamp, '1h') AS time; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_4_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_5_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_6_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_7_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | -- Change grants of one chunk explicitly and check that it is possible \z _timescaledb_internal._hyper_1_1_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | GRANT UPDATE ON _timescaledb_internal._hyper_1_1_chunk TO PUBLIC; \z _timescaledb_internal._hyper_1_1_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxt/super_user+| | | | | =rw/super_user | | REVOKE SELECT ON _timescaledb_internal._hyper_1_1_chunk FROM PUBLIC; \z _timescaledb_internal._hyper_1_1_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxt/super_user+| | | | | =w/super_user | | -- Check that revoking a permission first on the chunk and then on the -- hypertable that was added through the hypertable (INSERT and -- SELECT, in this case) still do not copy permissions from the -- hypertable (so there should not be a select permission to public on -- the chunk but there should be one on the hypertable). GRANT INSERT ON conditions TO PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | \z _timescaledb_internal._hyper_1_2_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | REVOKE SELECT ON _timescaledb_internal._hyper_1_2_chunk FROM PUBLIC; REVOKE INSERT ON conditions FROM PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | \z _timescaledb_internal._hyper_1_2_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxt/super_user | | -- Check that granting permissions through hypertable does not remove -- separate grants on chunk. GRANT UPDATE ON _timescaledb_internal._hyper_1_3_chunk TO PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | \z _timescaledb_internal._hyper_1_3_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxt/super_user+| | | | | =rw/super_user | | GRANT INSERT ON conditions TO PUBLIC; REVOKE INSERT ON conditions FROM PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | \z _timescaledb_internal._hyper_1_3_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxt/super_user+| | | | | =rw/super_user | | -- Check that GRANT ALL IN SCHEMA adds privileges to the parent -- and also goes to chunks in another schema GRANT ALL ON ALL TABLES IN SCHEMA public TO :ROLE_DEFAULT_PERM_USER_2; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+----------------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+----------------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxt/super_user +| | | | | =w/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxt/super_user +| | | | | =rw/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | _timescaledb_internal | _hyper_1_4_chunk | table | super_user=arwdDxt/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | _timescaledb_internal | _hyper_1_5_chunk | table | super_user=arwdDxt/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | _timescaledb_internal | _hyper_1_6_chunk | table | super_user=arwdDxt/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | _timescaledb_internal | _hyper_1_7_chunk | table | super_user=arwdDxt/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | -- Check that REVOKE ALL IN SCHEMA removes privileges of the parent -- and also goes to chunks in another schema REVOKE ALL ON ALL TABLES IN SCHEMA public FROM :ROLE_DEFAULT_PERM_USER_2; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxt/super_user+| | | | | =w/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxt/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxt/super_user+| | | | | =rw/super_user | | _timescaledb_internal | _hyper_1_4_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_5_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_6_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_7_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | -- Create chunks in the same schema as the hypertable and check that -- they also get the same privileges as the hypertable CREATE TABLE measurements( time TIMESTAMPTZ NOT NULL, device INTEGER, temperature FLOAT ); -- Create a hypertable with chunks in the same schema SELECT * FROM create_hypertable('public.measurements', 'time', chunk_time_interval => '5 days'::interval, associated_schema_name => 'public'); hypertable_id | schema_name | table_name | created ---------------+-------------+--------------+--------- 2 | public | measurements | t INSERT INTO measurements SELECT time, (random()*30)::int, random()*80 - 40 FROM generate_series('2018-12-01 00:00'::timestamp, '2018-12-10 00:00'::timestamp, '1h') AS time; -- GRANT ALL and check privileges GRANT ALL ON ALL TABLES IN SCHEMA public TO :ROLE_DEFAULT_PERM_USER_2; \z measurements Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+--------------+-------+----------------------------------------+-------------------+---------- public | measurements | table | super_user=arwdDxt/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+----------------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | \z public.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+-------------------+-------+----------------------------------------+-------------------+---------- public | _hyper_2_10_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | public | _hyper_2_8_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | public | _hyper_2_9_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | -- REVOKE ALL and check privileges REVOKE ALL ON ALL TABLES IN SCHEMA public FROM :ROLE_DEFAULT_PERM_USER_2; \z measurements Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+--------------+-------+-------------------------------+-------------------+---------- public | measurements | table | super_user=arwdDxt/super_user | | \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | \z public.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+-------------------+-------+-------------------------------+-------------------+---------- public | _hyper_2_10_chunk | table | super_user=arwdDxt/super_user | | public | _hyper_2_8_chunk | table | super_user=arwdDxt/super_user | | public | _hyper_2_9_chunk | table | super_user=arwdDxt/super_user | | -- GRANT/REVOKE in an empty schema (Issue #4581) CREATE SCHEMA test_grant; GRANT ALL ON ALL TABLES IN SCHEMA test_grant TO :ROLE_DEFAULT_PERM_USER_2; REVOKE ALL ON ALL TABLES IN SCHEMA test_grant FROM :ROLE_DEFAULT_PERM_USER_2; ================================================ FILE: test/expected/grant_hypertable-16.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE TABLE conditions( time TIMESTAMPTZ NOT NULL, device INTEGER, temperature FLOAT ); -- Create a hypertable and show that it does not have any privileges SELECT * FROM create_hypertable('conditions', 'time', chunk_time_interval => '5 days'::interval); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | conditions | t INSERT INTO conditions SELECT time, (random()*30)::int, random()*80 - 40 FROM generate_series('2018-12-01 00:00'::timestamp, '2018-12-10 00:00'::timestamp, '1h') AS time; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------+-------------------+---------- public | conditions | table | | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | | | _timescaledb_internal | _hyper_1_2_chunk | table | | | _timescaledb_internal | _hyper_1_3_chunk | table | | | -- Add privileges and show that they propagate to the chunks GRANT SELECT, INSERT ON conditions TO PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | -- Create some more chunks and show that they also get the privileges. INSERT INTO conditions SELECT time, (random()*30)::int, random()*80 - 40 FROM generate_series('2018-12-10 00:00'::timestamp, '2018-12-20 00:00'::timestamp, '1h') AS time; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_4_chunk | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_5_chunk | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | -- Revoke one of the privileges and show that it propagate to the -- chunks. REVOKE INSERT ON conditions FROM PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_4_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_5_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | -- Add some more chunks and show that it inherits the grants from the -- hypertable. INSERT INTO conditions SELECT time, (random()*30)::int, random()*80 - 40 FROM generate_series('2018-12-20 00:00'::timestamp, '2018-12-30 00:00'::timestamp, '1h') AS time; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_4_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_5_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_6_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_7_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | -- Change grants of one chunk explicitly and check that it is possible \z _timescaledb_internal._hyper_1_1_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | GRANT UPDATE ON _timescaledb_internal._hyper_1_1_chunk TO PUBLIC; \z _timescaledb_internal._hyper_1_1_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxt/super_user+| | | | | =rw/super_user | | REVOKE SELECT ON _timescaledb_internal._hyper_1_1_chunk FROM PUBLIC; \z _timescaledb_internal._hyper_1_1_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxt/super_user+| | | | | =w/super_user | | -- Check that revoking a permission first on the chunk and then on the -- hypertable that was added through the hypertable (INSERT and -- SELECT, in this case) still do not copy permissions from the -- hypertable (so there should not be a select permission to public on -- the chunk but there should be one on the hypertable). GRANT INSERT ON conditions TO PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | \z _timescaledb_internal._hyper_1_2_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxt/super_user+| | | | | =ar/super_user | | REVOKE SELECT ON _timescaledb_internal._hyper_1_2_chunk FROM PUBLIC; REVOKE INSERT ON conditions FROM PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | \z _timescaledb_internal._hyper_1_2_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxt/super_user | | -- Check that granting permissions through hypertable does not remove -- separate grants on chunk. GRANT UPDATE ON _timescaledb_internal._hyper_1_3_chunk TO PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | \z _timescaledb_internal._hyper_1_3_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxt/super_user+| | | | | =rw/super_user | | GRANT INSERT ON conditions TO PUBLIC; REVOKE INSERT ON conditions FROM PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | \z _timescaledb_internal._hyper_1_3_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxt/super_user+| | | | | =rw/super_user | | -- Check that GRANT ALL IN SCHEMA adds privileges to the parent -- and also goes to chunks in another schema GRANT ALL ON ALL TABLES IN SCHEMA public TO :ROLE_DEFAULT_PERM_USER_2; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+----------------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+----------------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxt/super_user +| | | | | =w/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxt/super_user +| | | | | =rw/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | _timescaledb_internal | _hyper_1_4_chunk | table | super_user=arwdDxt/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | _timescaledb_internal | _hyper_1_5_chunk | table | super_user=arwdDxt/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | _timescaledb_internal | _hyper_1_6_chunk | table | super_user=arwdDxt/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | _timescaledb_internal | _hyper_1_7_chunk | table | super_user=arwdDxt/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | -- Check that REVOKE ALL IN SCHEMA removes privileges of the parent -- and also goes to chunks in another schema REVOKE ALL ON ALL TABLES IN SCHEMA public FROM :ROLE_DEFAULT_PERM_USER_2; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxt/super_user+| | | | | =w/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxt/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxt/super_user+| | | | | =rw/super_user | | _timescaledb_internal | _hyper_1_4_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_5_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_6_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_7_chunk | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | -- Create chunks in the same schema as the hypertable and check that -- they also get the same privileges as the hypertable CREATE TABLE measurements( time TIMESTAMPTZ NOT NULL, device INTEGER, temperature FLOAT ); -- Create a hypertable with chunks in the same schema SELECT * FROM create_hypertable('public.measurements', 'time', chunk_time_interval => '5 days'::interval, associated_schema_name => 'public'); hypertable_id | schema_name | table_name | created ---------------+-------------+--------------+--------- 2 | public | measurements | t INSERT INTO measurements SELECT time, (random()*30)::int, random()*80 - 40 FROM generate_series('2018-12-01 00:00'::timestamp, '2018-12-10 00:00'::timestamp, '1h') AS time; -- GRANT ALL and check privileges GRANT ALL ON ALL TABLES IN SCHEMA public TO :ROLE_DEFAULT_PERM_USER_2; \z measurements Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+--------------+-------+----------------------------------------+-------------------+---------- public | measurements | table | super_user=arwdDxt/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+----------------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | \z public.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+-------------------+-------+----------------------------------------+-------------------+---------- public | _hyper_2_10_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | public | _hyper_2_8_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | public | _hyper_2_9_chunk | table | super_user=arwdDxt/super_user +| | | | | default_perm_user_2=arwdDxt/super_user | | -- REVOKE ALL and check privileges REVOKE ALL ON ALL TABLES IN SCHEMA public FROM :ROLE_DEFAULT_PERM_USER_2; \z measurements Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+--------------+-------+-------------------------------+-------------------+---------- public | measurements | table | super_user=arwdDxt/super_user | | \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxt/super_user+| | | | | =r/super_user | | \z public.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+-------------------+-------+-------------------------------+-------------------+---------- public | _hyper_2_10_chunk | table | super_user=arwdDxt/super_user | | public | _hyper_2_8_chunk | table | super_user=arwdDxt/super_user | | public | _hyper_2_9_chunk | table | super_user=arwdDxt/super_user | | -- GRANT/REVOKE in an empty schema (Issue #4581) CREATE SCHEMA test_grant; GRANT ALL ON ALL TABLES IN SCHEMA test_grant TO :ROLE_DEFAULT_PERM_USER_2; REVOKE ALL ON ALL TABLES IN SCHEMA test_grant FROM :ROLE_DEFAULT_PERM_USER_2; ================================================ FILE: test/expected/grant_hypertable-17.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE TABLE conditions( time TIMESTAMPTZ NOT NULL, device INTEGER, temperature FLOAT ); -- Create a hypertable and show that it does not have any privileges SELECT * FROM create_hypertable('conditions', 'time', chunk_time_interval => '5 days'::interval); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | conditions | t INSERT INTO conditions SELECT time, (random()*30)::int, random()*80 - 40 FROM generate_series('2018-12-01 00:00'::timestamp, '2018-12-10 00:00'::timestamp, '1h') AS time; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------+-------------------+---------- public | conditions | table | | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | | | _timescaledb_internal | _hyper_1_2_chunk | table | | | _timescaledb_internal | _hyper_1_3_chunk | table | | | -- Add privileges and show that they propagate to the chunks GRANT SELECT, INSERT ON conditions TO PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+--------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | -- Create some more chunks and show that they also get the privileges. INSERT INTO conditions SELECT time, (random()*30)::int, random()*80 - 40 FROM generate_series('2018-12-10 00:00'::timestamp, '2018-12-20 00:00'::timestamp, '1h') AS time; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+--------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_4_chunk | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_5_chunk | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | -- Revoke one of the privileges and show that it propagate to the -- chunks. REVOKE INSERT ON conditions FROM PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+--------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_4_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_5_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | -- Add some more chunks and show that it inherits the grants from the -- hypertable. INSERT INTO conditions SELECT time, (random()*30)::int, random()*80 - 40 FROM generate_series('2018-12-20 00:00'::timestamp, '2018-12-30 00:00'::timestamp, '1h') AS time; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+--------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_4_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_5_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_6_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_7_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | -- Change grants of one chunk explicitly and check that it is possible \z _timescaledb_internal._hyper_1_1_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | GRANT UPDATE ON _timescaledb_internal._hyper_1_1_chunk TO PUBLIC; \z _timescaledb_internal._hyper_1_1_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxtm/super_user+| | | | | =rw/super_user | | REVOKE SELECT ON _timescaledb_internal._hyper_1_1_chunk FROM PUBLIC; \z _timescaledb_internal._hyper_1_1_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxtm/super_user+| | | | | =w/super_user | | -- Check that revoking a permission first on the chunk and then on the -- hypertable that was added through the hypertable (INSERT and -- SELECT, in this case) still do not copy permissions from the -- hypertable (so there should not be a select permission to public on -- the chunk but there should be one on the hypertable). GRANT INSERT ON conditions TO PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+--------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | \z _timescaledb_internal._hyper_1_2_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | REVOKE SELECT ON _timescaledb_internal._hyper_1_2_chunk FROM PUBLIC; REVOKE INSERT ON conditions FROM PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+--------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | \z _timescaledb_internal._hyper_1_2_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxtm/super_user | | -- Check that granting permissions through hypertable does not remove -- separate grants on chunk. GRANT UPDATE ON _timescaledb_internal._hyper_1_3_chunk TO PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+--------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | \z _timescaledb_internal._hyper_1_3_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxtm/super_user+| | | | | =rw/super_user | | GRANT INSERT ON conditions TO PUBLIC; REVOKE INSERT ON conditions FROM PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+--------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | \z _timescaledb_internal._hyper_1_3_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxtm/super_user+| | | | | =rw/super_user | | -- Check that GRANT ALL IN SCHEMA adds privileges to the parent -- and also goes to chunks in another schema GRANT ALL ON ALL TABLES IN SCHEMA public TO :ROLE_DEFAULT_PERM_USER_2; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-----------------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-----------------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxtm/super_user +| | | | | =w/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxtm/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxtm/super_user +| | | | | =rw/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | _timescaledb_internal | _hyper_1_4_chunk | table | super_user=arwdDxtm/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | _timescaledb_internal | _hyper_1_5_chunk | table | super_user=arwdDxtm/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | _timescaledb_internal | _hyper_1_6_chunk | table | super_user=arwdDxtm/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | _timescaledb_internal | _hyper_1_7_chunk | table | super_user=arwdDxtm/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | -- Check that REVOKE ALL IN SCHEMA removes privileges of the parent -- and also goes to chunks in another schema REVOKE ALL ON ALL TABLES IN SCHEMA public FROM :ROLE_DEFAULT_PERM_USER_2; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+--------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxtm/super_user+| | | | | =w/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxtm/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxtm/super_user+| | | | | =rw/super_user | | _timescaledb_internal | _hyper_1_4_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_5_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_6_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_7_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | -- Create chunks in the same schema as the hypertable and check that -- they also get the same privileges as the hypertable CREATE TABLE measurements( time TIMESTAMPTZ NOT NULL, device INTEGER, temperature FLOAT ); -- Create a hypertable with chunks in the same schema SELECT * FROM create_hypertable('public.measurements', 'time', chunk_time_interval => '5 days'::interval, associated_schema_name => 'public'); hypertable_id | schema_name | table_name | created ---------------+-------------+--------------+--------- 2 | public | measurements | t INSERT INTO measurements SELECT time, (random()*30)::int, random()*80 - 40 FROM generate_series('2018-12-01 00:00'::timestamp, '2018-12-10 00:00'::timestamp, '1h') AS time; -- GRANT ALL and check privileges GRANT ALL ON ALL TABLES IN SCHEMA public TO :ROLE_DEFAULT_PERM_USER_2; \z measurements Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+--------------+-------+-----------------------------------------+-------------------+---------- public | measurements | table | super_user=arwdDxtm/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-----------------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | \z public.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+-------------------+-------+-----------------------------------------+-------------------+---------- public | _hyper_2_10_chunk | table | super_user=arwdDxtm/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | public | _hyper_2_8_chunk | table | super_user=arwdDxtm/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | public | _hyper_2_9_chunk | table | super_user=arwdDxtm/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | -- REVOKE ALL and check privileges REVOKE ALL ON ALL TABLES IN SCHEMA public FROM :ROLE_DEFAULT_PERM_USER_2; \z measurements Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+--------------+-------+--------------------------------+-------------------+---------- public | measurements | table | super_user=arwdDxtm/super_user | | \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+--------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | \z public.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+-------------------+-------+--------------------------------+-------------------+---------- public | _hyper_2_10_chunk | table | super_user=arwdDxtm/super_user | | public | _hyper_2_8_chunk | table | super_user=arwdDxtm/super_user | | public | _hyper_2_9_chunk | table | super_user=arwdDxtm/super_user | | -- GRANT/REVOKE in an empty schema (Issue #4581) CREATE SCHEMA test_grant; GRANT ALL ON ALL TABLES IN SCHEMA test_grant TO :ROLE_DEFAULT_PERM_USER_2; REVOKE ALL ON ALL TABLES IN SCHEMA test_grant FROM :ROLE_DEFAULT_PERM_USER_2; ================================================ FILE: test/expected/grant_hypertable-18.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE TABLE conditions( time TIMESTAMPTZ NOT NULL, device INTEGER, temperature FLOAT ); -- Create a hypertable and show that it does not have any privileges SELECT * FROM create_hypertable('conditions', 'time', chunk_time_interval => '5 days'::interval); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | conditions | t INSERT INTO conditions SELECT time, (random()*30)::int, random()*80 - 40 FROM generate_series('2018-12-01 00:00'::timestamp, '2018-12-10 00:00'::timestamp, '1h') AS time; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-------------------+-------------------+---------- public | conditions | table | | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | | | _timescaledb_internal | _hyper_1_2_chunk | table | | | _timescaledb_internal | _hyper_1_3_chunk | table | | | -- Add privileges and show that they propagate to the chunks GRANT SELECT, INSERT ON conditions TO PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+--------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | -- Create some more chunks and show that they also get the privileges. INSERT INTO conditions SELECT time, (random()*30)::int, random()*80 - 40 FROM generate_series('2018-12-10 00:00'::timestamp, '2018-12-20 00:00'::timestamp, '1h') AS time; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+--------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_4_chunk | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | _timescaledb_internal | _hyper_1_5_chunk | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | -- Revoke one of the privileges and show that it propagate to the -- chunks. REVOKE INSERT ON conditions FROM PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+--------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_4_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_5_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | -- Add some more chunks and show that it inherits the grants from the -- hypertable. INSERT INTO conditions SELECT time, (random()*30)::int, random()*80 - 40 FROM generate_series('2018-12-20 00:00'::timestamp, '2018-12-30 00:00'::timestamp, '1h') AS time; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+--------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_4_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_5_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_6_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_7_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | -- Change grants of one chunk explicitly and check that it is possible \z _timescaledb_internal._hyper_1_1_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | GRANT UPDATE ON _timescaledb_internal._hyper_1_1_chunk TO PUBLIC; \z _timescaledb_internal._hyper_1_1_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxtm/super_user+| | | | | =rw/super_user | | REVOKE SELECT ON _timescaledb_internal._hyper_1_1_chunk FROM PUBLIC; \z _timescaledb_internal._hyper_1_1_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxtm/super_user+| | | | | =w/super_user | | -- Check that revoking a permission first on the chunk and then on the -- hypertable that was added through the hypertable (INSERT and -- SELECT, in this case) still do not copy permissions from the -- hypertable (so there should not be a select permission to public on -- the chunk but there should be one on the hypertable). GRANT INSERT ON conditions TO PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+--------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | \z _timescaledb_internal._hyper_1_2_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxtm/super_user+| | | | | =ar/super_user | | REVOKE SELECT ON _timescaledb_internal._hyper_1_2_chunk FROM PUBLIC; REVOKE INSERT ON conditions FROM PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+--------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | \z _timescaledb_internal._hyper_1_2_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxtm/super_user | | -- Check that granting permissions through hypertable does not remove -- separate grants on chunk. GRANT UPDATE ON _timescaledb_internal._hyper_1_3_chunk TO PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+--------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | \z _timescaledb_internal._hyper_1_3_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxtm/super_user+| | | | | =rw/super_user | | GRANT INSERT ON conditions TO PUBLIC; REVOKE INSERT ON conditions FROM PUBLIC; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+--------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | \z _timescaledb_internal._hyper_1_3_chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxtm/super_user+| | | | | =rw/super_user | | -- Check that GRANT ALL IN SCHEMA adds privileges to the parent -- and also goes to chunks in another schema GRANT ALL ON ALL TABLES IN SCHEMA public TO :ROLE_DEFAULT_PERM_USER_2; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-----------------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+-----------------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxtm/super_user +| | | | | =w/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxtm/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxtm/super_user +| | | | | =rw/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | _timescaledb_internal | _hyper_1_4_chunk | table | super_user=arwdDxtm/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | _timescaledb_internal | _hyper_1_5_chunk | table | super_user=arwdDxtm/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | _timescaledb_internal | _hyper_1_6_chunk | table | super_user=arwdDxtm/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | _timescaledb_internal | _hyper_1_7_chunk | table | super_user=arwdDxtm/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | -- Check that REVOKE ALL IN SCHEMA removes privileges of the parent -- and also goes to chunks in another schema REVOKE ALL ON ALL TABLES IN SCHEMA public FROM :ROLE_DEFAULT_PERM_USER_2; \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+--------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | \z _timescaledb_internal.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies -----------------------+------------------+-------+--------------------------------+-------------------+---------- _timescaledb_internal | _hyper_1_1_chunk | table | super_user=arwdDxtm/super_user+| | | | | =w/super_user | | _timescaledb_internal | _hyper_1_2_chunk | table | super_user=arwdDxtm/super_user | | _timescaledb_internal | _hyper_1_3_chunk | table | super_user=arwdDxtm/super_user+| | | | | =rw/super_user | | _timescaledb_internal | _hyper_1_4_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_5_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_6_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | _timescaledb_internal | _hyper_1_7_chunk | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | -- Create chunks in the same schema as the hypertable and check that -- they also get the same privileges as the hypertable CREATE TABLE measurements( time TIMESTAMPTZ NOT NULL, device INTEGER, temperature FLOAT ); -- Create a hypertable with chunks in the same schema SELECT * FROM create_hypertable('public.measurements', 'time', chunk_time_interval => '5 days'::interval, associated_schema_name => 'public'); hypertable_id | schema_name | table_name | created ---------------+-------------+--------------+--------- 2 | public | measurements | t INSERT INTO measurements SELECT time, (random()*30)::int, random()*80 - 40 FROM generate_series('2018-12-01 00:00'::timestamp, '2018-12-10 00:00'::timestamp, '1h') AS time; -- GRANT ALL and check privileges GRANT ALL ON ALL TABLES IN SCHEMA public TO :ROLE_DEFAULT_PERM_USER_2; \z measurements Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+--------------+-------+-----------------------------------------+-------------------+---------- public | measurements | table | super_user=arwdDxtm/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+-----------------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user +| | | | | =r/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | \z public.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+-------------------+-------+-----------------------------------------+-------------------+---------- public | _hyper_2_10_chunk | table | super_user=arwdDxtm/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | public | _hyper_2_8_chunk | table | super_user=arwdDxtm/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | public | _hyper_2_9_chunk | table | super_user=arwdDxtm/super_user +| | | | | default_perm_user_2=arwdDxtm/super_user | | -- REVOKE ALL and check privileges REVOKE ALL ON ALL TABLES IN SCHEMA public FROM :ROLE_DEFAULT_PERM_USER_2; \z measurements Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+--------------+-------+--------------------------------+-------------------+---------- public | measurements | table | super_user=arwdDxtm/super_user | | \z conditions Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+------------+-------+--------------------------------+-------------------+---------- public | conditions | table | super_user=arwdDxtm/super_user+| | | | | =r/super_user | | \z public.*chunk Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------+-------------------+-------+--------------------------------+-------------------+---------- public | _hyper_2_10_chunk | table | super_user=arwdDxtm/super_user | | public | _hyper_2_8_chunk | table | super_user=arwdDxtm/super_user | | public | _hyper_2_9_chunk | table | super_user=arwdDxtm/super_user | | -- GRANT/REVOKE in an empty schema (Issue #4581) CREATE SCHEMA test_grant; GRANT ALL ON ALL TABLES IN SCHEMA test_grant TO :ROLE_DEFAULT_PERM_USER_2; REVOKE ALL ON ALL TABLES IN SCHEMA test_grant FROM :ROLE_DEFAULT_PERM_USER_2; ================================================ FILE: test/expected/hash.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test hashing Const values. We should expect the same hash value for -- all integer types when values are compatible SELECT _timescaledb_functions.get_partition_hash(1::int); get_partition_hash -------------------- 242423622 SELECT _timescaledb_functions.get_partition_hash(1::bigint); get_partition_hash -------------------- 242423622 SELECT _timescaledb_functions.get_partition_hash(1::smallint); get_partition_hash -------------------- 242423622 SELECT _timescaledb_functions.get_partition_hash(true); get_partition_hash -------------------- 242423622 -- Floating point types should also hash the same for compatible values SELECT _timescaledb_functions.get_partition_hash(1.0::real); get_partition_hash -------------------- 376496956 SELECT _timescaledb_functions.get_partition_hash(1.0::double precision); get_partition_hash -------------------- 376496956 -- Float aliases SELECT _timescaledb_functions.get_partition_hash(1.0::float); get_partition_hash -------------------- 376496956 SELECT _timescaledb_functions.get_partition_hash(1.0::float4); get_partition_hash -------------------- 376496956 SELECT _timescaledb_functions.get_partition_hash(1.0::float8); get_partition_hash -------------------- 376496956 SELECT _timescaledb_functions.get_partition_hash(1.0::numeric); get_partition_hash -------------------- 1324868424 -- 'name' and '"char"' are internal PostgreSQL types, which are not -- intended for use by the general user. They are included here only -- for completeness -- https://www.postgresql.org/docs/10/static/datatype-character.html#datatype-character-special-table SELECT _timescaledb_functions.get_partition_hash('c'::name); get_partition_hash -------------------- 1903644986 SELECT _timescaledb_functions.get_partition_hash('c'::"char"); get_partition_hash -------------------- 203891234 -- String and character hashes should also have the same output for -- compatible values SELECT _timescaledb_functions.get_partition_hash('c'::char); get_partition_hash -------------------- 1903644986 SELECT _timescaledb_functions.get_partition_hash('c'::varchar(2)); get_partition_hash -------------------- 1903644986 SELECT _timescaledb_functions.get_partition_hash('c'::text); get_partition_hash -------------------- 1903644986 -- 'c' is 0x63 in ASCII SELECT _timescaledb_functions.get_partition_hash(E'\\x63'::bytea); get_partition_hash -------------------- 1903644986 -- Time and date types SELECT _timescaledb_functions.get_partition_hash(interval '1 day'); get_partition_hash -------------------- 93502988 SELECT _timescaledb_functions.get_partition_hash('2017-03-22T09:18:23'::timestamp); get_partition_hash -------------------- 307315039 SELECT _timescaledb_functions.get_partition_hash('2017-03-22T09:18:23'::timestamptz); get_partition_hash -------------------- 1195163597 SELECT _timescaledb_functions.get_partition_hash('2017-03-22'::date); get_partition_hash -------------------- 693590295 SELECT _timescaledb_functions.get_partition_hash('10:00:00'::time); get_partition_hash -------------------- 1380652790 SELECT _timescaledb_functions.get_partition_hash('10:00:00-1'::timetz); get_partition_hash -------------------- 769387140 -- Other types SELECT _timescaledb_functions.get_partition_hash(ARRAY[1,2,3]); get_partition_hash -------------------- 1822090118 SELECT _timescaledb_functions.get_partition_hash('08002b:010203'::macaddr); get_partition_hash -------------------- 294987870 SELECT _timescaledb_functions.get_partition_hash('192.168.100.128/25'::cidr); get_partition_hash -------------------- 1612896565 SELECT _timescaledb_functions.get_partition_hash('192.168.100.128'::inet); get_partition_hash -------------------- 1952516432 SELECT _timescaledb_functions.get_partition_hash('2001:4f8:3:ba:2e0:81ff:fe22:d1f1'::inet); get_partition_hash -------------------- 933321588 SELECT _timescaledb_functions.get_partition_hash('2001:4f8:3:ba:2e0:81ff:fe22:d1f1/128'::cidr); get_partition_hash -------------------- 933321588 SELECT _timescaledb_functions.get_partition_hash('{ "foo": "bar" }'::jsonb); get_partition_hash -------------------- 208840587 SELECT _timescaledb_functions.get_partition_hash('4b6a5eec-b344-11e7-abc4-cec278b6b50a'::uuid); get_partition_hash -------------------- 504202548 SELECT _timescaledb_functions.get_partition_hash(1::regclass); get_partition_hash -------------------- 242423622 SELECT _timescaledb_functions.get_partition_hash(int4range(10, 20)); get_partition_hash -------------------- 1202375768 SELECT _timescaledb_functions.get_partition_hash(int8range(10, 20)); get_partition_hash -------------------- 1202375768 SELECT _timescaledb_functions.get_partition_hash(numrange(10, 20)); get_partition_hash -------------------- 1083987536 SELECT _timescaledb_functions.get_partition_hash(tsrange('2017-03-22T09:18:23', '2017-03-23T09:18:23')); get_partition_hash -------------------- 2079608838 SELECT _timescaledb_functions.get_partition_hash(tstzrange('2017-03-22T09:18:23+01', '2017-03-23T09:18:23+00')); get_partition_hash -------------------- 1255083771 -- Test hashing Var values CREATE TABLE hash_test(id int, value text); INSERT INTO hash_test VALUES (1, 'test'); -- Test Vars SELECT _timescaledb_functions.get_partition_hash(id) FROM hash_test; get_partition_hash -------------------- 242423622 SELECT _timescaledb_functions.get_partition_hash(value) FROM hash_test; get_partition_hash -------------------- 1771415073 -- Test coerced value SELECT _timescaledb_functions.get_partition_hash(id::text) FROM hash_test; get_partition_hash -------------------- 1516350201 -- Test legacy function that converts values to text first SELECT _timescaledb_functions.get_partition_for_key('4b6a5eec-b344-11e7-abc4-cec278b6b50a'::text); get_partition_for_key ----------------------- 934882099 SELECT _timescaledb_functions.get_partition_for_key('4b6a5eec-b344-11e7-abc4-cec278b6b50a'::varchar); get_partition_for_key ----------------------- 934882099 SELECT _timescaledb_functions.get_partition_for_key(187); get_partition_for_key ----------------------- 1161071810 SELECT _timescaledb_functions.get_partition_for_key(187::bigint); get_partition_for_key ----------------------- 1161071810 SELECT _timescaledb_functions.get_partition_for_key(187::numeric); get_partition_for_key ----------------------- 1161071810 SELECT _timescaledb_functions.get_partition_for_key(187::double precision); get_partition_for_key ----------------------- 1161071810 SELECT _timescaledb_functions.get_partition_for_key(int4range(10, 20)); get_partition_for_key ----------------------- 505239042 SELECT _timescaledb_functions.get_partition_hash('08002b:010203'::macaddr); get_partition_hash -------------------- 294987870 -- Test inside IMMUTABLE function (Issue #4575) CREATE FUNCTION my_get_partition_hash(INTEGER) RETURNS INTEGER AS 'SELECT _timescaledb_functions.get_partition_hash($1);' LANGUAGE SQL IMMUTABLE; CREATE FUNCTION my_get_partition_for_key(INTEGER) RETURNS INTEGER AS 'SELECT _timescaledb_functions.get_partition_for_key($1);' LANGUAGE SQL IMMUTABLE; SELECT my_get_partition_hash(1); my_get_partition_hash ----------------------- 242423622 SELECT my_get_partition_for_key(1); my_get_partition_for_key -------------------------- 1516350201 ================================================ FILE: test/expected/histogram_test-15.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- table 1 CREATE TABLE "hitest1"(key real, val varchar(40)); -- insertions INSERT INTO "hitest1" VALUES(0, 'hi'); INSERT INTO "hitest1" VALUES(1, 'sup'); INSERT INTO "hitest1" VALUES(2, 'hello'); INSERT INTO "hitest1" VALUES(3, 'yo'); INSERT INTO "hitest1" VALUES(4, 'howdy'); INSERT INTO "hitest1" VALUES(5, 'hola'); INSERT INTO "hitest1" VALUES(6, 'ya'); INSERT INTO "hitest1" VALUES(1, 'sup'); INSERT INTO "hitest1" VALUES(2, 'hello'); INSERT INTO "hitest1" VALUES(1, 'sup'); -- table 2 CREATE TABLE "hitest2"(name varchar(30), score integer, qualify boolean); -- insertions INSERT INTO "hitest2" VALUES('Tom', 6, TRUE); INSERT INTO "hitest2" VALUES('Mary', 4, FALSE); INSERT INTO "hitest2" VALUES('Jaq', 3, FALSE); INSERT INTO "hitest2" VALUES('Jane', 10, TRUE); -- standard 2 bucket SELECT histogram(key, 0, 9, 2) FROM hitest1; histogram ----------- {0,8,2,0} -- standard multi-bucket SELECT histogram(key, 0, 9, 5) FROM hitest1; histogram ----------------- {0,4,3,2,1,0,0} -- standard 3 bucket SELECT val, histogram(key, 0, 7, 3) FROM hitest1 GROUP BY val ORDER BY val; val | histogram -------+------------- hello | {0,2,0,0,0} hi | {0,1,0,0,0} hola | {0,0,0,1,0} howdy | {0,0,1,0,0} sup | {0,3,0,0,0} ya | {0,0,0,1,0} yo | {0,0,1,0,0} -- standard element beneath lb SELECT histogram(key, 1, 7, 3) FROM hitest1; histogram ------------- {1,5,2,2,0} -- standard element above ub SELECT histogram(key, 0, 3, 3) FROM hitest1; histogram ------------- {0,1,3,2,4} -- standard element beneath and above lb and ub, respectively SELECT histogram(key, 1, 3, 2) FROM hitest1; histogram ----------- {1,3,2,4} -- standard 1 bucket SELECT histogram(key, 1, 3, 1) FROM hitest1; histogram ----------- {1,5,4} -- standard 2 bucket SELECT qualify, histogram(score, 0, 10, 2) FROM hitest2 GROUP BY qualify ORDER BY qualify; qualify | histogram ---------+----------- f | {0,2,0,0} t | {0,0,1,1} -- standard multi-bucket SELECT qualify, histogram(score, 0, 10, 5) FROM hitest2 GROUP BY qualify ORDER BY qualify; qualify | histogram ---------+----------------- f | {0,0,1,1,0,0,0} t | {0,0,0,0,1,0,1} -- check number of buckets is constant \set ON_ERROR_STOP 0 select histogram(i,10,90,case when i=1 then 1 else 1000000 end) FROM generate_series(1,100) i; ERROR: number of buckets must not change between calls \set ON_ERROR_STOP 1 CREATE TABLE weather ( time TIMESTAMPTZ NOT NULL, city TEXT, temperature FLOAT, PRIMARY KEY(time, city) ); -- There is a bug in width_bucket() causing a NaN as a result, so we -- check that it is not causing a crash in histogram(). SELECT * FROM create_hypertable('weather', 'time', 'city', 3); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | weather | t INSERT INTO weather VALUES ('2023-02-10 09:16:51.133584+00','city1',10.4), ('2023-02-10 11:16:51.611618+00','city1',10.3), ('2023-02-10 06:58:59.999999+00','city1',10.3), ('2023-02-10 01:58:59.999999+00','city1',10.3), ('2023-02-09 01:58:59.999999+00','city1',10.3), ('2023-02-10 08:58:59.999999+00','city1',10.3), ('2023-03-23 06:12:02.73765+00 ','city1', 9.7), ('2023-03-23 06:12:06.990998+00','city1',11.7); -- This will currently generate an error on PG15 and prior versions \set ON_ERROR_STOP 0 SELECT histogram(temperature, -1.79769e+308, 1.79769e+308,10) FROM weather GROUP BY city; ERROR: index -2147483648 from "width_bucket" out of range \set ON_ERROR_STOP 1 ================================================ FILE: test/expected/histogram_test-16.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- table 1 CREATE TABLE "hitest1"(key real, val varchar(40)); -- insertions INSERT INTO "hitest1" VALUES(0, 'hi'); INSERT INTO "hitest1" VALUES(1, 'sup'); INSERT INTO "hitest1" VALUES(2, 'hello'); INSERT INTO "hitest1" VALUES(3, 'yo'); INSERT INTO "hitest1" VALUES(4, 'howdy'); INSERT INTO "hitest1" VALUES(5, 'hola'); INSERT INTO "hitest1" VALUES(6, 'ya'); INSERT INTO "hitest1" VALUES(1, 'sup'); INSERT INTO "hitest1" VALUES(2, 'hello'); INSERT INTO "hitest1" VALUES(1, 'sup'); -- table 2 CREATE TABLE "hitest2"(name varchar(30), score integer, qualify boolean); -- insertions INSERT INTO "hitest2" VALUES('Tom', 6, TRUE); INSERT INTO "hitest2" VALUES('Mary', 4, FALSE); INSERT INTO "hitest2" VALUES('Jaq', 3, FALSE); INSERT INTO "hitest2" VALUES('Jane', 10, TRUE); -- standard 2 bucket SELECT histogram(key, 0, 9, 2) FROM hitest1; histogram ----------- {0,8,2,0} -- standard multi-bucket SELECT histogram(key, 0, 9, 5) FROM hitest1; histogram ----------------- {0,4,3,2,1,0,0} -- standard 3 bucket SELECT val, histogram(key, 0, 7, 3) FROM hitest1 GROUP BY val ORDER BY val; val | histogram -------+------------- hello | {0,2,0,0,0} hi | {0,1,0,0,0} hola | {0,0,0,1,0} howdy | {0,0,1,0,0} sup | {0,3,0,0,0} ya | {0,0,0,1,0} yo | {0,0,1,0,0} -- standard element beneath lb SELECT histogram(key, 1, 7, 3) FROM hitest1; histogram ------------- {1,5,2,2,0} -- standard element above ub SELECT histogram(key, 0, 3, 3) FROM hitest1; histogram ------------- {0,1,3,2,4} -- standard element beneath and above lb and ub, respectively SELECT histogram(key, 1, 3, 2) FROM hitest1; histogram ----------- {1,3,2,4} -- standard 1 bucket SELECT histogram(key, 1, 3, 1) FROM hitest1; histogram ----------- {1,5,4} -- standard 2 bucket SELECT qualify, histogram(score, 0, 10, 2) FROM hitest2 GROUP BY qualify ORDER BY qualify; qualify | histogram ---------+----------- f | {0,2,0,0} t | {0,0,1,1} -- standard multi-bucket SELECT qualify, histogram(score, 0, 10, 5) FROM hitest2 GROUP BY qualify ORDER BY qualify; qualify | histogram ---------+----------------- f | {0,0,1,1,0,0,0} t | {0,0,0,0,1,0,1} -- check number of buckets is constant \set ON_ERROR_STOP 0 select histogram(i,10,90,case when i=1 then 1 else 1000000 end) FROM generate_series(1,100) i; ERROR: number of buckets must not change between calls \set ON_ERROR_STOP 1 CREATE TABLE weather ( time TIMESTAMPTZ NOT NULL, city TEXT, temperature FLOAT, PRIMARY KEY(time, city) ); -- There is a bug in width_bucket() causing a NaN as a result, so we -- check that it is not causing a crash in histogram(). SELECT * FROM create_hypertable('weather', 'time', 'city', 3); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | weather | t INSERT INTO weather VALUES ('2023-02-10 09:16:51.133584+00','city1',10.4), ('2023-02-10 11:16:51.611618+00','city1',10.3), ('2023-02-10 06:58:59.999999+00','city1',10.3), ('2023-02-10 01:58:59.999999+00','city1',10.3), ('2023-02-09 01:58:59.999999+00','city1',10.3), ('2023-02-10 08:58:59.999999+00','city1',10.3), ('2023-03-23 06:12:02.73765+00 ','city1', 9.7), ('2023-03-23 06:12:06.990998+00','city1',11.7); -- This will currently generate an error on PG15 and prior versions \set ON_ERROR_STOP 0 SELECT histogram(temperature, -1.79769e+308, 1.79769e+308,10) FROM weather GROUP BY city; histogram --------------------------- {0,0,0,0,0,0,8,0,0,0,0,0} \set ON_ERROR_STOP 1 ================================================ FILE: test/expected/histogram_test-17.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- table 1 CREATE TABLE "hitest1"(key real, val varchar(40)); -- insertions INSERT INTO "hitest1" VALUES(0, 'hi'); INSERT INTO "hitest1" VALUES(1, 'sup'); INSERT INTO "hitest1" VALUES(2, 'hello'); INSERT INTO "hitest1" VALUES(3, 'yo'); INSERT INTO "hitest1" VALUES(4, 'howdy'); INSERT INTO "hitest1" VALUES(5, 'hola'); INSERT INTO "hitest1" VALUES(6, 'ya'); INSERT INTO "hitest1" VALUES(1, 'sup'); INSERT INTO "hitest1" VALUES(2, 'hello'); INSERT INTO "hitest1" VALUES(1, 'sup'); -- table 2 CREATE TABLE "hitest2"(name varchar(30), score integer, qualify boolean); -- insertions INSERT INTO "hitest2" VALUES('Tom', 6, TRUE); INSERT INTO "hitest2" VALUES('Mary', 4, FALSE); INSERT INTO "hitest2" VALUES('Jaq', 3, FALSE); INSERT INTO "hitest2" VALUES('Jane', 10, TRUE); -- standard 2 bucket SELECT histogram(key, 0, 9, 2) FROM hitest1; histogram ----------- {0,8,2,0} -- standard multi-bucket SELECT histogram(key, 0, 9, 5) FROM hitest1; histogram ----------------- {0,4,3,2,1,0,0} -- standard 3 bucket SELECT val, histogram(key, 0, 7, 3) FROM hitest1 GROUP BY val ORDER BY val; val | histogram -------+------------- hello | {0,2,0,0,0} hi | {0,1,0,0,0} hola | {0,0,0,1,0} howdy | {0,0,1,0,0} sup | {0,3,0,0,0} ya | {0,0,0,1,0} yo | {0,0,1,0,0} -- standard element beneath lb SELECT histogram(key, 1, 7, 3) FROM hitest1; histogram ------------- {1,5,2,2,0} -- standard element above ub SELECT histogram(key, 0, 3, 3) FROM hitest1; histogram ------------- {0,1,3,2,4} -- standard element beneath and above lb and ub, respectively SELECT histogram(key, 1, 3, 2) FROM hitest1; histogram ----------- {1,3,2,4} -- standard 1 bucket SELECT histogram(key, 1, 3, 1) FROM hitest1; histogram ----------- {1,5,4} -- standard 2 bucket SELECT qualify, histogram(score, 0, 10, 2) FROM hitest2 GROUP BY qualify ORDER BY qualify; qualify | histogram ---------+----------- f | {0,2,0,0} t | {0,0,1,1} -- standard multi-bucket SELECT qualify, histogram(score, 0, 10, 5) FROM hitest2 GROUP BY qualify ORDER BY qualify; qualify | histogram ---------+----------------- f | {0,0,1,1,0,0,0} t | {0,0,0,0,1,0,1} -- check number of buckets is constant \set ON_ERROR_STOP 0 select histogram(i,10,90,case when i=1 then 1 else 1000000 end) FROM generate_series(1,100) i; ERROR: number of buckets must not change between calls \set ON_ERROR_STOP 1 CREATE TABLE weather ( time TIMESTAMPTZ NOT NULL, city TEXT, temperature FLOAT, PRIMARY KEY(time, city) ); -- There is a bug in width_bucket() causing a NaN as a result, so we -- check that it is not causing a crash in histogram(). SELECT * FROM create_hypertable('weather', 'time', 'city', 3); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | weather | t INSERT INTO weather VALUES ('2023-02-10 09:16:51.133584+00','city1',10.4), ('2023-02-10 11:16:51.611618+00','city1',10.3), ('2023-02-10 06:58:59.999999+00','city1',10.3), ('2023-02-10 01:58:59.999999+00','city1',10.3), ('2023-02-09 01:58:59.999999+00','city1',10.3), ('2023-02-10 08:58:59.999999+00','city1',10.3), ('2023-03-23 06:12:02.73765+00 ','city1', 9.7), ('2023-03-23 06:12:06.990998+00','city1',11.7); -- This will currently generate an error on PG15 and prior versions \set ON_ERROR_STOP 0 SELECT histogram(temperature, -1.79769e+308, 1.79769e+308,10) FROM weather GROUP BY city; histogram --------------------------- {0,0,0,0,0,0,8,0,0,0,0,0} \set ON_ERROR_STOP 1 ================================================ FILE: test/expected/histogram_test-18.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- table 1 CREATE TABLE "hitest1"(key real, val varchar(40)); -- insertions INSERT INTO "hitest1" VALUES(0, 'hi'); INSERT INTO "hitest1" VALUES(1, 'sup'); INSERT INTO "hitest1" VALUES(2, 'hello'); INSERT INTO "hitest1" VALUES(3, 'yo'); INSERT INTO "hitest1" VALUES(4, 'howdy'); INSERT INTO "hitest1" VALUES(5, 'hola'); INSERT INTO "hitest1" VALUES(6, 'ya'); INSERT INTO "hitest1" VALUES(1, 'sup'); INSERT INTO "hitest1" VALUES(2, 'hello'); INSERT INTO "hitest1" VALUES(1, 'sup'); -- table 2 CREATE TABLE "hitest2"(name varchar(30), score integer, qualify boolean); -- insertions INSERT INTO "hitest2" VALUES('Tom', 6, TRUE); INSERT INTO "hitest2" VALUES('Mary', 4, FALSE); INSERT INTO "hitest2" VALUES('Jaq', 3, FALSE); INSERT INTO "hitest2" VALUES('Jane', 10, TRUE); -- standard 2 bucket SELECT histogram(key, 0, 9, 2) FROM hitest1; histogram ----------- {0,8,2,0} -- standard multi-bucket SELECT histogram(key, 0, 9, 5) FROM hitest1; histogram ----------------- {0,4,3,2,1,0,0} -- standard 3 bucket SELECT val, histogram(key, 0, 7, 3) FROM hitest1 GROUP BY val ORDER BY val; val | histogram -------+------------- hello | {0,2,0,0,0} hi | {0,1,0,0,0} hola | {0,0,0,1,0} howdy | {0,0,1,0,0} sup | {0,3,0,0,0} ya | {0,0,0,1,0} yo | {0,0,1,0,0} -- standard element beneath lb SELECT histogram(key, 1, 7, 3) FROM hitest1; histogram ------------- {1,5,2,2,0} -- standard element above ub SELECT histogram(key, 0, 3, 3) FROM hitest1; histogram ------------- {0,1,3,2,4} -- standard element beneath and above lb and ub, respectively SELECT histogram(key, 1, 3, 2) FROM hitest1; histogram ----------- {1,3,2,4} -- standard 1 bucket SELECT histogram(key, 1, 3, 1) FROM hitest1; histogram ----------- {1,5,4} -- standard 2 bucket SELECT qualify, histogram(score, 0, 10, 2) FROM hitest2 GROUP BY qualify ORDER BY qualify; qualify | histogram ---------+----------- f | {0,2,0,0} t | {0,0,1,1} -- standard multi-bucket SELECT qualify, histogram(score, 0, 10, 5) FROM hitest2 GROUP BY qualify ORDER BY qualify; qualify | histogram ---------+----------------- f | {0,0,1,1,0,0,0} t | {0,0,0,0,1,0,1} -- check number of buckets is constant \set ON_ERROR_STOP 0 select histogram(i,10,90,case when i=1 then 1 else 1000000 end) FROM generate_series(1,100) i; ERROR: number of buckets must not change between calls \set ON_ERROR_STOP 1 CREATE TABLE weather ( time TIMESTAMPTZ NOT NULL, city TEXT, temperature FLOAT, PRIMARY KEY(time, city) ); -- There is a bug in width_bucket() causing a NaN as a result, so we -- check that it is not causing a crash in histogram(). SELECT * FROM create_hypertable('weather', 'time', 'city', 3); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | weather | t INSERT INTO weather VALUES ('2023-02-10 09:16:51.133584+00','city1',10.4), ('2023-02-10 11:16:51.611618+00','city1',10.3), ('2023-02-10 06:58:59.999999+00','city1',10.3), ('2023-02-10 01:58:59.999999+00','city1',10.3), ('2023-02-09 01:58:59.999999+00','city1',10.3), ('2023-02-10 08:58:59.999999+00','city1',10.3), ('2023-03-23 06:12:02.73765+00 ','city1', 9.7), ('2023-03-23 06:12:06.990998+00','city1',11.7); -- This will currently generate an error on PG15 and prior versions \set ON_ERROR_STOP 0 SELECT histogram(temperature, -1.79769e+308, 1.79769e+308,10) FROM weather GROUP BY city; histogram --------------------------- {0,0,0,0,0,0,8,0,0,0,0,0} \set ON_ERROR_STOP 1 ================================================ FILE: test/expected/index.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE index_test(time timestamptz, temp float); SELECT create_hypertable('index_test', 'time'); create_hypertable ------------------------- (1,public,index_test,t) -- Default indexes created SELECT * FROM test.show_indexes('index_test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ---------------------+---------+------+--------+---------+-----------+------------ index_test_time_idx | {time} | | f | f | f | DROP TABLE index_test; CREATE TABLE index_test(time timestamptz, device integer, temp float); -- Create index before create_hypertable() CREATE UNIQUE INDEX index_test_time_idx ON index_test (time); \set ON_ERROR_STOP 0 -- Creating a hypertable from a table with an index that doesn't cover -- all partitioning columns should fail SELECT create_hypertable('index_test', 'time', 'device', 2); ERROR: cannot create a unique index without the column "device" (used in partitioning) \set ON_ERROR_STOP 1 -- Partitioning on only time should work SELECT create_hypertable('index_test', 'time'); create_hypertable ------------------------- (3,public,index_test,t) INSERT INTO index_test VALUES ('2017-01-20T09:00:01', 1, 17.5); -- Check that index is also created on chunk SELECT * FROM test.show_indexes('index_test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ---------------------+---------+------+--------+---------+-----------+------------ index_test_time_idx | {time} | | t | f | f | SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+------------------------------------------------------------+---------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_3_1_chunk | _timescaledb_internal._hyper_3_1_chunk_index_test_time_idx | {time} | | t | f | f | -- Create another chunk INSERT INTO index_test VALUES ('2017-05-20T09:00:01', 3, 17.5); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+------------------------------------------------------------+---------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_3_1_chunk | _timescaledb_internal._hyper_3_1_chunk_index_test_time_idx | {time} | | t | f | f | _timescaledb_internal._hyper_3_2_chunk | _timescaledb_internal._hyper_3_2_chunk_index_test_time_idx | {time} | | t | f | f | -- Delete the index on only one chunk DROP INDEX _timescaledb_internal._hyper_3_1_chunk_index_test_time_idx; SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+------------------------------------------------------------+---------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_3_2_chunk | _timescaledb_internal._hyper_3_2_chunk_index_test_time_idx | {time} | | t | f | f | -- Recreate table with new partitioning DROP TABLE index_test; CREATE TABLE index_test(id serial, time timestamptz, device integer, temp float); SELECT * FROM test.show_columns('index_test'); Column | Type | NotNull --------+--------------------------+--------- id | integer | t time | timestamp with time zone | f device | integer | f temp | double precision | f -- Test that we can handle difference in attnos across hypertable and -- chunks by dropping the ID column ALTER TABLE index_test DROP COLUMN id; SELECT * FROM test.show_columns('index_test'); Column | Type | NotNull --------+--------------------------+--------- time | timestamp with time zone | f device | integer | f temp | double precision | f -- No pre-existing UNIQUE index, so partitioning on two columns should work SELECT create_hypertable('index_test', 'time', 'device', 2); create_hypertable ------------------------- (4,public,index_test,t) INSERT INTO index_test VALUES ('2017-01-20T09:00:01', 1, 17.5); \set ON_ERROR_STOP 0 -- Create unique index without all partitioning columns should fail CREATE UNIQUE INDEX index_test_time_device_idx ON index_test (time); ERROR: cannot create a unique index without the column "device" (used in partitioning) \set ON_ERROR_STOP 1 CREATE UNIQUE INDEX index_test_time_device_idx ON index_test (time, device); -- Regular index need not cover all partitioning columns CREATE INDEX ON index_test (time, temp); -- Create another chunk INSERT INTO index_test VALUES ('2017-04-20T09:00:01', 1, 17.5); -- New index should have been recursed to chunks SELECT * FROM test.show_indexes('index_test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------+---------------+------+--------+---------+-----------+------------ index_test_device_time_idx | {device,time} | | f | f | f | index_test_time_device_idx | {time,device} | | t | f | f | index_test_time_idx | {time} | | f | f | f | index_test_time_temp_idx | {time,temp} | | f | f | f | SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+-------------------------------------------------------------------+---------------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_index_test_time_idx | {time} | | f | f | f | _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_index_test_time_device_idx | {time,device} | | t | f | f | _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_index_test_time_idx | {time} | | f | f | f | _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_index_test_time_device_idx | {time,device} | | t | f | f | _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | ALTER INDEX index_test_time_idx RENAME TO index_test_time_idx2; -- Metadata and index should have changed name SELECT * FROM test.show_indexes('index_test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------+---------------+------+--------+---------+-----------+------------ index_test_device_time_idx | {device,time} | | f | f | f | index_test_time_device_idx | {time,device} | | t | f | f | index_test_time_idx2 | {time} | | f | f | f | index_test_time_temp_idx | {time,temp} | | f | f | f | SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+-------------------------------------------------------------------+---------------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_index_test_time_idx2 | {time} | | f | f | f | _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_index_test_time_device_idx | {time,device} | | t | f | f | _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_index_test_time_idx2 | {time} | | f | f | f | _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_index_test_time_device_idx | {time,device} | | t | f | f | _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | DROP INDEX index_test_time_idx2; DROP INDEX index_test_time_device_idx; -- Index should have been dropped SELECT * FROM test.show_indexes('index_test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------+---------------+------+--------+---------+-----------+------------ index_test_device_time_idx | {device,time} | | f | f | f | index_test_time_temp_idx | {time,temp} | | f | f | f | SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+-------------------------------------------------------------------+---------------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | -- Create index with long name to see how this is handled on chunks CREATE INDEX a_hypertable_index_with_a_very_very_long_name_that_truncates ON index_test (time DESC, temp); CREATE INDEX a_hypertable_index_with_a_very_very_long_name_that_truncates_2 ON index_test (time DESC, temp DESC); SELECT * FROM test.show_indexes('index_test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------------------------------+---------------+------+--------+---------+-----------+------------ a_hypertable_index_with_a_very_very_long_name_that_truncates | {time,temp} | | f | f | f | a_hypertable_index_with_a_very_very_long_name_that_truncates_2 | {time,temp} | | f | f | f | index_test_device_time_idx | {device,time} | | f | f | f | index_test_time_temp_idx | {time,temp} | | f | f | f | SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+---------------------------------------------------------------------------------------+---------------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_a_hypertable_index_with_a_very_very_long_name_ | {time,temp} | | f | f | f | _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_a_hypertable_index_with_a_very_very_long_nam_1 | {time,temp} | | f | f | f | _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_a_hypertable_index_with_a_very_very_long_name_ | {time,temp} | | f | f | f | _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_a_hypertable_index_with_a_very_very_long_nam_1 | {time,temp} | | f | f | f | DROP INDEX a_hypertable_index_with_a_very_very_long_name_that_truncates; DROP INDEX a_hypertable_index_with_a_very_very_long_name_that_truncates_2; \set ON_ERROR_STOP 0 -- Create index CONCURRENTLY CREATE UNIQUE INDEX CONCURRENTLY index_test_time_device_idx ON index_test (time, device); ERROR: hypertables do not support concurrent index creation \set ON_ERROR_STOP 1 -- Test tablespaces. Chunk indexes should end up in same tablespace as -- main index. \c :TEST_DBNAME :ROLE_SUPERUSER SET client_min_messages = ERROR; DROP TABLESPACE IF EXISTS tablespace1; DROP TABLESPACE IF EXISTS tablespace2; SET client_min_messages = NOTICE; CREATE TABLESPACE tablespace1 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE1_PATH; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE INDEX index_test_time_idx ON index_test (time) TABLESPACE tablespace1; SELECT * FROM test.show_indexes('index_test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------+---------------+------+--------+---------+-----------+------------- index_test_device_time_idx | {device,time} | | f | f | f | index_test_time_idx | {time} | | f | f | f | tablespace1 index_test_time_temp_idx | {time,temp} | | f | f | f | SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+-------------------------------------------------------------------+---------------+------+--------+---------+-----------+------------- _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_index_test_time_idx | {time} | | f | f | f | tablespace1 _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_index_test_time_idx | {time} | | f | f | f | tablespace1 \c :TEST_DBNAME :ROLE_SUPERUSER CREATE TABLESPACE tablespace2 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE2_PATH; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER ALTER INDEX index_test_time_idx SET TABLESPACE tablespace2; SELECT * FROM test.show_indexes('index_test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------+---------------+------+--------+---------+-----------+------------- index_test_device_time_idx | {device,time} | | f | f | f | index_test_time_idx | {time} | | f | f | f | tablespace2 index_test_time_temp_idx | {time,temp} | | f | f | f | SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+-------------------------------------------------------------------+---------------+------+--------+---------+-----------+------------- _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_index_test_time_idx | {time} | | f | f | f | tablespace2 _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_index_test_time_idx | {time} | | f | f | f | tablespace2 -- Add constraint index ALTER TABLE index_test ADD UNIQUE (time, device); SELECT * FROM test.show_indexes('index_test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------+---------------+------+--------+---------+-----------+------------- index_test_device_time_idx | {device,time} | | f | f | f | index_test_time_device_key | {time,device} | | t | f | f | index_test_time_idx | {time} | | f | f | f | tablespace2 index_test_time_temp_idx | {time,temp} | | f | f | f | SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+-------------------------------------------------------------------+---------------+------+--------+---------+-----------+------------- _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal._hyper_4_3_chunk_index_test_time_idx | {time} | | f | f | f | tablespace2 _timescaledb_internal._hyper_4_3_chunk | _timescaledb_internal."3_1_index_test_time_device_key" | {time,device} | | t | f | f | _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal._hyper_4_4_chunk_index_test_time_idx | {time} | | f | f | f | tablespace2 _timescaledb_internal._hyper_4_4_chunk | _timescaledb_internal."4_2_index_test_time_device_key" | {time,device} | | t | f | f | -- Constraints are added to chunk_constraint table. SELECT * FROM _timescaledb_catalog.chunk_constraint; chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----------+--------------------+--------------------------------+---------------------------- 3 | 3 | constraint_3 | 3 | 4 | constraint_4 | 4 | 5 | constraint_5 | 4 | 4 | constraint_4 | 3 | | 3_1_index_test_time_device_key | index_test_time_device_key 4 | | 4_2_index_test_time_device_key | index_test_time_device_key DROP TABLE index_test; -- Create table in a tablespace CREATE TABLE index_test(time timestamptz, temp float, device int) TABLESPACE tablespace1; -- Default indexes should be in the table's tablespace SELECT create_hypertable('index_test', 'time'); create_hypertable ------------------------- (5,public,index_test,t) -- Explicitly defining an index tablespace should work and propagate -- to chunks CREATE INDEX ON index_test (time, device) TABLESPACE tablespace2; -- New indexes without explicit tablespaces should use the default -- tablespace CREATE INDEX ON index_test (device); -- Create chunk INSERT INTO index_test VALUES ('2017-01-20T09:00:01', 17.5); -- Check that the tablespaces of chunk indexes match the tablespace of -- the main index SELECT * FROM test.show_indexes('index_test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------+---------------+------+--------+---------+-----------+------------- index_test_device_idx | {device} | | f | f | f | index_test_time_device_idx | {time,device} | | f | f | f | tablespace2 index_test_time_idx | {time} | | f | f | f | tablespace1 SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+-------------------------------------------------------------------+---------------+------+--------+---------+-----------+------------- _timescaledb_internal._hyper_5_5_chunk | _timescaledb_internal._hyper_5_5_chunk_index_test_time_idx | {time} | | f | f | f | tablespace1 _timescaledb_internal._hyper_5_5_chunk | _timescaledb_internal._hyper_5_5_chunk_index_test_time_device_idx | {time,device} | | f | f | f | tablespace2 _timescaledb_internal._hyper_5_5_chunk | _timescaledb_internal._hyper_5_5_chunk_index_test_device_idx | {device} | | f | f | f | tablespace1 -- Creating a new index should propagate to existing chunks, including -- the given tablespace CREATE INDEX ON index_test (time, temp) TABLESPACE tablespace2; SELECT * FROM test.show_indexes('index_test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------+---------------+------+--------+---------+-----------+------------- index_test_device_idx | {device} | | f | f | f | index_test_time_device_idx | {time,device} | | f | f | f | tablespace2 index_test_time_idx | {time} | | f | f | f | tablespace1 index_test_time_temp_idx | {time,temp} | | f | f | f | tablespace2 SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+-------------------------------------------------------------------+---------------+------+--------+---------+-----------+------------- _timescaledb_internal._hyper_5_5_chunk | _timescaledb_internal._hyper_5_5_chunk_index_test_time_idx | {time} | | f | f | f | tablespace1 _timescaledb_internal._hyper_5_5_chunk | _timescaledb_internal._hyper_5_5_chunk_index_test_time_device_idx | {time,device} | | f | f | f | tablespace2 _timescaledb_internal._hyper_5_5_chunk | _timescaledb_internal._hyper_5_5_chunk_index_test_device_idx | {device} | | f | f | f | tablespace1 _timescaledb_internal._hyper_5_5_chunk | _timescaledb_internal._hyper_5_5_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | tablespace2 -- Cleanup DROP TABLE index_test CASCADE; \c :TEST_DBNAME :ROLE_SUPERUSER DROP TABLESPACE tablespace1; DROP TABLESPACE tablespace2; -- Test expression indexes CREATE TABLE index_expr_test(id serial, time timestamptz, temp float, meta jsonb); -- Screw up the attribute numbers ALTER TABLE index_expr_test DROP COLUMN id; CREATE INDEX ON index_expr_test ((meta ->> 'field')) ; INSERT INTO index_expr_test VALUES ('2017-01-20T09:00:01', 17.5, '{"field": "value1"}'); INSERT INTO index_expr_test VALUES ('2017-01-20T09:00:01', 17.5, '{"field": "value2"}'); EXPLAIN (verbose, buffers off, costs off) SELECT * FROM index_expr_test WHERE meta ->> 'field' = 'value1'; --- QUERY PLAN --- Index Scan using index_expr_test_expr_idx on public.index_expr_test Output: "time", temp, meta Index Cond: ((index_expr_test.meta ->> 'field'::text) = 'value1'::text) SELECT * FROM index_expr_test WHERE meta ->> 'field' = 'value1'; time | temp | meta ------------------------------+------+--------------------- Fri Jan 20 09:00:01 2017 PST | 17.5 | {"field": "value1"} -- Test INDEX DROP error for multiple objects CREATE TABLE index_test(time timestamptz, temp float); CREATE UNIQUE INDEX index_test_idx ON index_test (time, temp); SELECT create_hypertable('index_test', 'time'); create_hypertable ------------------------- (6,public,index_test,t) CREATE TABLE index_test_2(time timestamptz, temp float); CREATE UNIQUE INDEX index_test_2_idx ON index_test_2 (time, temp); \set ON_ERROR_STOP 0 DROP INDEX index_test_idx, index_test_2_idx; \set ON_ERROR_STOP 1 -- test expression index with dropped columns CREATE TABLE idx_expr_test(filler int, time timestamptz, meta text); SELECT table_name FROM create_hypertable('idx_expr_test', 'time'); table_name --------------- idx_expr_test ALTER TABLE idx_expr_test DROP COLUMN filler; CREATE INDEX tag_idx ON idx_expr_test(('foo'||meta)); INSERT INTO idx_expr_test(time, meta) VALUES ('2000-01-01', 'bar'); DROP TABLE idx_expr_test CASCADE; -- test multicolumn expression index with dropped columns CREATE TABLE idx_expr_test(filler int, time timestamptz, t1 text, t2 text, t3 text); SELECT table_name FROM create_hypertable('idx_expr_test', 'time'); table_name --------------- idx_expr_test ALTER TABLE idx_expr_test DROP COLUMN filler; CREATE INDEX tag_idx ON idx_expr_test((t1||t2||t3)); INSERT INTO idx_expr_test(time, t1, t2, t3) VALUES ('2000-01-01', 'foo', 'bar', 'baz'); DROP TABLE idx_expr_test CASCADE; -- test index with predicate and dropped columns CREATE TABLE idx_predicate_test(filler int, time timestamptz); SELECT table_name FROM create_hypertable('idx_predicate_test', 'time'); table_name -------------------- idx_predicate_test ALTER TABLE idx_predicate_test DROP COLUMN filler; ALTER TABLE idx_predicate_test ADD COLUMN b1 bool; CREATE INDEX idx_predicate_test_b1 ON idx_predicate_test(b1) WHERE b1=true; INSERT INTO idx_predicate_test VALUES ('2000-01-01',true); DROP TABLE idx_predicate_test; -- test index with table references CREATE TABLE idx_tableref_test(time timestamptz); SELECT table_name FROM create_hypertable('idx_tableref_test', 'time'); table_name ------------------- idx_tableref_test -- we use security definer to prevent function inlining CREATE OR REPLACE FUNCTION tableref_func(t idx_tableref_test) RETURNS timestamptz LANGUAGE SQL IMMUTABLE SECURITY DEFINER AS $f$ SELECT t.time; $f$; -- try creating index with no existing chunks CREATE INDEX tableref_idx ON idx_tableref_test(tableref_func(idx_tableref_test)); -- insert data to trigger chunk creation INSERT INTO idx_tableref_test SELECT '2000-01-01'; DROP INDEX tableref_idx; -- try creating index on hypertable with existing chunks CREATE INDEX tableref_idx ON idx_tableref_test(tableref_func(idx_tableref_test)); -- test index creation with if not exists CREATE TABLE idx_exists(time timestamptz NOT NULL); SELECT table_name FROM create_hypertable('idx_exists', 'time'); table_name ------------ idx_exists -- should be skipped since this index was already created by create_hypertable CREATE INDEX IF NOT EXISTS idx_exists_time_idx ON idx_exists(time DESC); NOTICE: relation "idx_exists_time_idx" already exists, skipping -- should create index CREATE INDEX IF NOT EXISTS idx_exists_time_asc_idx ON idx_exists(time ASC); -- should be skipped since it was created in previous command CREATE INDEX IF NOT EXISTS idx_exists_time_asc_idx ON idx_exists(time ASC); NOTICE: relation "idx_exists_time_asc_idx" already exists, skipping DROP INDEX idx_exists_time_asc_idx; INSERT INTO idx_exists VALUES ('2000-01-01'),('2001-01-01'); -- should create index CREATE INDEX IF NOT EXISTS idx_exists_time_asc_idx ON idx_exists(time ASC); -- should be skipped since it was created in previous command CREATE INDEX IF NOT EXISTS idx_exists_time_asc_idx ON idx_exists(time ASC); NOTICE: relation "idx_exists_time_asc_idx" already exists, skipping -- test reindex CREATE TABLE reindex_test(time timestamp, temp float, PRIMARY KEY(time, temp)); CREATE UNIQUE INDEX reindex_test_time_unique_idx ON reindex_test(time); -- create hypertable with three chunks SELECT create_hypertable('reindex_test', 'time', chunk_time_interval => 2628000000000); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ---------------------------- (12,public,reindex_test,t) INSERT INTO reindex_test VALUES ('2017-01-20T09:00:01', 17.5), ('2017-01-21T09:00:01', 19.1), ('2017-04-20T09:00:01', 89.5), ('2017-04-21T09:00:01', 17.1), ('2017-06-20T09:00:01', 18.5), ('2017-06-21T09:00:01', 11.0); SELECT * FROM test.show_columns('reindex_test'); Column | Type | NotNull --------+-----------------------------+--------- time | timestamp without time zone | t temp | double precision | t SELECT * FROM test.show_subtables('reindex_test'); Child | Tablespace ------------------------------------------+------------ _timescaledb_internal._hyper_12_12_chunk | _timescaledb_internal._hyper_12_13_chunk | _timescaledb_internal._hyper_12_14_chunk | _timescaledb_internal._hyper_12_15_chunk | _timescaledb_internal._hyper_12_16_chunk | -- show reindexing REINDEX (VERBOSE) TABLE reindex_test; INFO: index "12_3_reindex_test_pkey" was reindexed INFO: index "_hyper_12_12_chunk_reindex_test_time_unique_idx" was reindexed INFO: index "13_4_reindex_test_pkey" was reindexed INFO: index "_hyper_12_13_chunk_reindex_test_time_unique_idx" was reindexed INFO: index "14_5_reindex_test_pkey" was reindexed INFO: index "_hyper_12_14_chunk_reindex_test_time_unique_idx" was reindexed INFO: index "15_6_reindex_test_pkey" was reindexed INFO: index "_hyper_12_15_chunk_reindex_test_time_unique_idx" was reindexed INFO: index "16_7_reindex_test_pkey" was reindexed INFO: index "_hyper_12_16_chunk_reindex_test_time_unique_idx" was reindexed \set ON_ERROR_STOP 0 -- REINDEX TABLE CONCURRENTLY on hypertables is not supported REINDEX TABLE CONCURRENTLY reindex_test; ERROR: concurrent index creation on hypertables is not supported -- this one currently doesn't recurse to chunks and instead gives an -- error REINDEX (VERBOSE) INDEX reindex_test_time_unique_idx; ERROR: reindexing of a specific index on a hypertable is unsupported \set ON_ERROR_STOP 1 -- show reindexing on a normal table CREATE TABLE reindex_norm(time timestamp, temp float); CREATE UNIQUE INDEX reindex_norm_time_unique_idx ON reindex_norm(time); INSERT INTO reindex_norm VALUES ('2017-01-20T09:00:01', 17.5), ('2017-01-21T09:00:01', 19.1), ('2017-04-20T09:00:01', 89.5), ('2017-04-21T09:00:01', 17.1), ('2017-06-20T09:00:01', 18.5), ('2017-06-21T09:00:01', 11.0); REINDEX (VERBOSE) TABLE reindex_norm; INFO: index "reindex_norm_time_unique_idx" was reindexed REINDEX (VERBOSE) INDEX reindex_norm_time_unique_idx; INFO: index "reindex_norm_time_unique_idx" was reindexed SELECT * FROM test.show_constraintsp('_timescaledb_internal._hyper_12%'); Table | Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated ------------------------------------------+------------------------+------+-------------+------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------+------------+----------+----------- _timescaledb_internal._hyper_12_12_chunk | 12_3_reindex_test_pkey | p | {time,temp} | _timescaledb_internal."12_3_reindex_test_pkey" | | f | f | t _timescaledb_internal._hyper_12_12_chunk | constraint_13 | c | {time} | - | (("time" >= 'Thu Jan 19 10:00:00 2017'::timestamp without time zone) AND ("time" < 'Sat Feb 18 20:00:00 2017'::timestamp without time zone)) | f | f | t _timescaledb_internal._hyper_12_13_chunk | 13_4_reindex_test_pkey | p | {time,temp} | _timescaledb_internal."13_4_reindex_test_pkey" | | f | f | t _timescaledb_internal._hyper_12_13_chunk | constraint_14 | c | {time} | - | (("time" >= 'Tue Mar 21 06:00:00 2017'::timestamp without time zone) AND ("time" < 'Thu Apr 20 16:00:00 2017'::timestamp without time zone)) | f | f | t _timescaledb_internal._hyper_12_14_chunk | 14_5_reindex_test_pkey | p | {time,temp} | _timescaledb_internal."14_5_reindex_test_pkey" | | f | f | t _timescaledb_internal._hyper_12_14_chunk | constraint_15 | c | {time} | - | (("time" >= 'Thu Apr 20 16:00:00 2017'::timestamp without time zone) AND ("time" < 'Sun May 21 02:00:00 2017'::timestamp without time zone)) | f | f | t _timescaledb_internal._hyper_12_15_chunk | 15_6_reindex_test_pkey | p | {time,temp} | _timescaledb_internal."15_6_reindex_test_pkey" | | f | f | t _timescaledb_internal._hyper_12_15_chunk | constraint_16 | c | {time} | - | (("time" >= 'Sun May 21 02:00:00 2017'::timestamp without time zone) AND ("time" < 'Tue Jun 20 12:00:00 2017'::timestamp without time zone)) | f | f | t _timescaledb_internal._hyper_12_16_chunk | 16_7_reindex_test_pkey | p | {time,temp} | _timescaledb_internal."16_7_reindex_test_pkey" | | f | f | t _timescaledb_internal._hyper_12_16_chunk | constraint_17 | c | {time} | - | (("time" >= 'Tue Jun 20 12:00:00 2017'::timestamp without time zone) AND ("time" < 'Thu Jul 20 22:00:00 2017'::timestamp without time zone)) | f | f | t SELECT * FROM reindex_norm; time | temp --------------------------+------ Fri Jan 20 09:00:01 2017 | 17.5 Sat Jan 21 09:00:01 2017 | 19.1 Thu Apr 20 09:00:01 2017 | 89.5 Fri Apr 21 09:00:01 2017 | 17.1 Tue Jun 20 09:00:01 2017 | 18.5 Wed Jun 21 09:00:01 2017 | 11 CREATE TABLE ht_dropped(time timestamptz, d0 int, d1 int, c0 int, c1 int, c2 int); SELECT create_hypertable('ht_dropped','time'); create_hypertable -------------------------- (13,public,ht_dropped,t) INSERT INTO ht_dropped(time,c0,c1,c2) SELECT '2000-01-01',1,2,3; ALTER TABLE ht_dropped DROP COLUMN d0; INSERT INTO ht_dropped(time,c0,c1,c2) SELECT '2001-01-01',1,2,3; ALTER TABLE ht_dropped DROP COLUMN d1; INSERT INTO ht_dropped(time,c0,c1,c2) SELECT '2002-01-01',1,2,3; CREATE INDEX ON ht_dropped(c0,c1,c2) WHERE c1 IS NOT NULL; CREATE INDEX ON ht_dropped(c0,c1,c2) WITH(timescaledb.transaction_per_chunk) WHERE c2 IS NOT NULL; SELECT oid::TEXT AS "Chunk", i.* FROM (SELECT tableoid::REGCLASS FROM ht_dropped GROUP BY tableoid) ch (oid) LEFT JOIN LATERAL ( SELECT * FROM test.show_indexes (ch.oid)) i ON TRUE ORDER BY 1, 2; Chunk | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ------------------------------------------+-------------------------------------------------------------------+------------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_13_17_chunk | _timescaledb_internal._hyper_13_17_chunk_ht_dropped_time_idx | {time} | | f | f | f | _timescaledb_internal._hyper_13_17_chunk | _timescaledb_internal._hyper_13_17_chunk_ht_dropped_c0_c1_c2_idx | {c0,c1,c2} | | f | f | f | _timescaledb_internal._hyper_13_17_chunk | _timescaledb_internal._hyper_13_17_chunk_ht_dropped_c0_c1_c2_idx1 | {c0,c1,c2} | | f | f | f | _timescaledb_internal._hyper_13_18_chunk | _timescaledb_internal._hyper_13_18_chunk_ht_dropped_time_idx | {time} | | f | f | f | _timescaledb_internal._hyper_13_18_chunk | _timescaledb_internal._hyper_13_18_chunk_ht_dropped_c0_c1_c2_idx | {c0,c1,c2} | | f | f | f | _timescaledb_internal._hyper_13_18_chunk | _timescaledb_internal._hyper_13_18_chunk_ht_dropped_c0_c1_c2_idx1 | {c0,c1,c2} | | f | f | f | _timescaledb_internal._hyper_13_19_chunk | _timescaledb_internal._hyper_13_19_chunk_ht_dropped_time_idx | {time} | | f | f | f | _timescaledb_internal._hyper_13_19_chunk | _timescaledb_internal._hyper_13_19_chunk_ht_dropped_c0_c1_c2_idx | {c0,c1,c2} | | f | f | f | _timescaledb_internal._hyper_13_19_chunk | _timescaledb_internal._hyper_13_19_chunk_ht_dropped_c0_c1_c2_idx1 | {c0,c1,c2} | | f | f | f | -- #3056 check chunk index column name mapping CREATE TABLE i3056(c int, order_number int NOT NULL, date_created timestamptz NOT NULL); CREATE INDEX ON i3056(order_number) INCLUDE(order_number); CREATE INDEX ON i3056(date_created, (order_number % 5)) INCLUDE(order_number); SELECT table_name FROM create_hypertable('i3056', 'date_created'); table_name ------------ i3056 ALTER TABLE i3056 DROP COLUMN c; INSERT INTO i3056(order_number,date_created) VALUES (1, '2000-01-01'); -- #5908 test CREATE INDEX ON ONLY main table CREATE TABLE test(time timestamptz, temp float); SELECT create_hypertable('test', 'time'); create_hypertable -------------------- (15,public,test,t) INSERT INTO test (time,temp) VALUES (generate_series(TIMESTAMP '2019-08-01', TIMESTAMP '2019-08-10', INTERVAL '10 minutes'), ROUND(RANDOM()*10)::float); SELECT * FROM show_chunks('test'); show_chunks ------------------------------------------ _timescaledb_internal._hyper_15_21_chunk _timescaledb_internal._hyper_15_22_chunk SELECT * FROM test.show_indexes('_timescaledb_internal._hyper_15_21_chunk'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace --------------------------------------------------------+---------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_15_21_chunk_test_time_idx | {time} | | f | f | f | -- create index per chunk CREATE INDEX _hyper_15_21_chunk_test_temp_idx ON _timescaledb_internal._hyper_15_21_chunk(temp); SELECT * FROM test.show_indexes('_timescaledb_internal._hyper_15_21_chunk'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace --------------------------------------------------------+---------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_15_21_chunk_test_temp_idx | {temp} | | f | f | f | _timescaledb_internal._hyper_15_21_chunk_test_time_idx | {time} | | f | f | f | SELECT * FROM test.show_indexes('test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ---------------+---------+------+--------+---------+-----------+------------ test_time_idx | {time} | | f | f | f | -- create index only on main table CREATE INDEX test_temp_idx ON ONLY test (time); SELECT * FROM test.show_indexes('test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ---------------+---------+------+--------+---------+-----------+------------ test_temp_idx | {time} | | f | f | f | test_time_idx | {time} | | f | f | f | SELECT * FROM test.show_indexes('_timescaledb_internal._hyper_15_21_chunk'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace --------------------------------------------------------+---------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_15_21_chunk_test_temp_idx | {temp} | | f | f | f | _timescaledb_internal._hyper_15_21_chunk_test_time_idx | {time} | | f | f | f | ================================================ FILE: test/expected/information_views.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SELECT * FROM timescaledb_information.hypertables; hypertable_schema | hypertable_name | owner | num_dimensions | num_chunks | compression_enabled | tablespaces | primary_dimension | primary_dimension_type -------------------+-----------------+-------+----------------+------------+---------------------+-------------+-------------------+------------------------ -- create simple hypertable with 1 chunk CREATE TABLE ht1(time TIMESTAMPTZ NOT NULL); SELECT create_hypertable('ht1','time'); create_hypertable ------------------- (1,public,ht1,t) INSERT INTO ht1 SELECT '2000-01-01'::TIMESTAMPTZ; -- create simple hypertable with 1 chunk and toasted data CREATE TABLE ht2(time TIMESTAMPTZ NOT NULL, data TEXT); SELECT create_hypertable('ht2','time'); create_hypertable ------------------- (2,public,ht2,t) INSERT INTO ht2 SELECT '2000-01-01'::TIMESTAMPTZ, repeat('8k',4096); SELECT * FROM timescaledb_information.hypertables ORDER BY hypertable_schema, hypertable_name; hypertable_schema | hypertable_name | owner | num_dimensions | num_chunks | compression_enabled | tablespaces | primary_dimension | primary_dimension_type -------------------+-----------------+-------------------+----------------+------------+---------------------+-------------+-------------------+-------------------------- public | ht1 | default_perm_user | 1 | 1 | f | | time | timestamp with time zone public | ht2 | default_perm_user | 1 | 1 | f | | time | timestamp with time zone \c :TEST_DBNAME :ROLE_SUPERUSER -- create schema open and hypertable with 3 chunks CREATE SCHEMA open; GRANT USAGE ON SCHEMA open TO :ROLE_DEFAULT_PERM_USER; CREATE TABLE open.open_ht(time TIMESTAMPTZ NOT NULL); SELECT create_hypertable('open.open_ht','time'); create_hypertable -------------------- (3,open,open_ht,t) INSERT INTO open.open_ht SELECT '2000-01-01'::TIMESTAMPTZ; INSERT INTO open.open_ht SELECT '2001-01-01'::TIMESTAMPTZ; INSERT INTO open.open_ht SELECT '2002-01-01'::TIMESTAMPTZ; -- create schema closed and hypertable CREATE SCHEMA closed; CREATE TABLE closed.closed_ht(time TIMESTAMPTZ NOT NULL); SELECT create_hypertable('closed.closed_ht','time'); create_hypertable ------------------------ (4,closed,closed_ht,t) INSERT INTO closed.closed_ht SELECT '2000-01-01'::TIMESTAMPTZ; SELECT * FROM timescaledb_information.hypertables ORDER BY hypertable_schema, hypertable_name; hypertable_schema | hypertable_name | owner | num_dimensions | num_chunks | compression_enabled | tablespaces | primary_dimension | primary_dimension_type -------------------+-----------------+-------------------+----------------+------------+---------------------+-------------+-------------------+-------------------------- closed | closed_ht | super_user | 1 | 1 | f | | time | timestamp with time zone open | open_ht | super_user | 1 | 3 | f | | time | timestamp with time zone public | ht1 | default_perm_user | 1 | 1 | f | | time | timestamp with time zone public | ht2 | default_perm_user | 1 | 1 | f | | time | timestamp with time zone \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER \set ON_ERROR_STOP 0 \x SELECT * FROM timescaledb_information.hypertables ORDER BY hypertable_schema, hypertable_name; -[ RECORD 1 ]----------+------------------------- hypertable_schema | closed hypertable_name | closed_ht owner | super_user num_dimensions | 1 num_chunks | 1 compression_enabled | f tablespaces | primary_dimension | time primary_dimension_type | timestamp with time zone -[ RECORD 2 ]----------+------------------------- hypertable_schema | open hypertable_name | open_ht owner | super_user num_dimensions | 1 num_chunks | 3 compression_enabled | f tablespaces | primary_dimension | time primary_dimension_type | timestamp with time zone -[ RECORD 3 ]----------+------------------------- hypertable_schema | public hypertable_name | ht1 owner | default_perm_user num_dimensions | 1 num_chunks | 1 compression_enabled | f tablespaces | primary_dimension | time primary_dimension_type | timestamp with time zone -[ RECORD 4 ]----------+------------------------- hypertable_schema | public hypertable_name | ht2 owner | default_perm_user num_dimensions | 1 num_chunks | 1 compression_enabled | f tablespaces | primary_dimension | time primary_dimension_type | timestamp with time zone -- filter by schema SELECT * FROM timescaledb_information.hypertables WHERE hypertable_schema = 'closed' ORDER BY hypertable_schema, hypertable_name; -[ RECORD 1 ]----------+------------------------- hypertable_schema | closed hypertable_name | closed_ht owner | super_user num_dimensions | 1 num_chunks | 1 compression_enabled | f tablespaces | primary_dimension | time primary_dimension_type | timestamp with time zone -- filter by table name SELECT * FROM timescaledb_information.hypertables WHERE hypertable_name = 'ht1' ORDER BY hypertable_schema, hypertable_name; -[ RECORD 1 ]----------+------------------------- hypertable_schema | public hypertable_name | ht1 owner | default_perm_user num_dimensions | 1 num_chunks | 1 compression_enabled | f tablespaces | primary_dimension | time primary_dimension_type | timestamp with time zone -- filter by owner SELECT * FROM timescaledb_information.hypertables WHERE owner = 'super_user' ORDER BY hypertable_schema, hypertable_name; -[ RECORD 1 ]----------+------------------------- hypertable_schema | closed hypertable_name | closed_ht owner | super_user num_dimensions | 1 num_chunks | 1 compression_enabled | f tablespaces | primary_dimension | time primary_dimension_type | timestamp with time zone -[ RECORD 2 ]----------+------------------------- hypertable_schema | open hypertable_name | open_ht owner | super_user num_dimensions | 1 num_chunks | 3 compression_enabled | f tablespaces | primary_dimension | time primary_dimension_type | timestamp with time zone \x ---Add integer table -- CREATE TABLE test_table_int(time bigint, junk int); SELECT create_hypertable('test_table_int', 'time', chunk_time_interval => 10); create_hypertable ----------------------------- (5,public,test_table_int,t) CREATE OR REPLACE function table_int_now() returns BIGINT LANGUAGE SQL IMMUTABLE as 'SELECT 1::BIGINT'; SELECT set_integer_now_func('test_table_int', 'table_int_now'); set_integer_now_func ---------------------- INSERT into test_table_int SELECT generate_series( 1, 20), 100; \d timescaledb_information.chunks View "timescaledb_information.chunks" Column | Type | Collation | Nullable | Default ------------------------+--------------------------+-----------+----------+--------- hypertable_schema | name | | | hypertable_name | name | | | chunk_schema | name | | | chunk_name | name | | | primary_dimension | name | | | primary_dimension_type | regtype | | | range_start | timestamp with time zone | | | range_end | timestamp with time zone | | | range_start_integer | bigint | | | range_end_integer | bigint | | | is_compressed | boolean | | | chunk_tablespace | name | | | chunk_creation_time | timestamp with time zone | | | SELECT hypertable_schema, hypertable_name, chunk_schema, chunk_name, primary_dimension, primary_dimension_type, range_start, range_end, range_start_integer, range_end_integer, is_compressed, chunk_tablespace, data_nodes FROM timescaledb_information.chunks WHERE hypertable_name = 'ht1' ORDER BY chunk_name; ERROR: column "data_nodes" does not exist at character 294 SELECT hypertable_schema, hypertable_name, chunk_schema, chunk_name, primary_dimension, primary_dimension_type, range_start, range_end, range_start_integer, range_end_integer, is_compressed, chunk_tablespace, data_nodes FROM timescaledb_information.chunks WHERE hypertable_name = 'test_table_int' ORDER BY chunk_name; ERROR: column "data_nodes" does not exist at character 294 \x SELECT * FROM timescaledb_information.dimensions ORDER BY hypertable_name, dimension_number; -[ RECORD 1 ]-----+------------------------- hypertable_schema | closed hypertable_name | closed_ht dimension_number | 1 column_name | time column_type | timestamp with time zone dimension_type | Time time_interval | @ 7 days integer_interval | integer_now_func | num_partitions | -[ RECORD 2 ]-----+------------------------- hypertable_schema | public hypertable_name | ht1 dimension_number | 1 column_name | time column_type | timestamp with time zone dimension_type | Time time_interval | @ 7 days integer_interval | integer_now_func | num_partitions | -[ RECORD 3 ]-----+------------------------- hypertable_schema | public hypertable_name | ht2 dimension_number | 1 column_name | time column_type | timestamp with time zone dimension_type | Time time_interval | @ 7 days integer_interval | integer_now_func | num_partitions | -[ RECORD 4 ]-----+------------------------- hypertable_schema | open hypertable_name | open_ht dimension_number | 1 column_name | time column_type | timestamp with time zone dimension_type | Time time_interval | @ 7 days integer_interval | integer_now_func | num_partitions | -[ RECORD 5 ]-----+------------------------- hypertable_schema | public hypertable_name | test_table_int dimension_number | 1 column_name | time column_type | bigint dimension_type | Time time_interval | integer_interval | 10 integer_now_func | table_int_now num_partitions | \x ================================================ FILE: test/expected/insert-15.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SET enable_seqscan TO off; \ir include/insert_two_partitions.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."two_Partitions" ( "timeCustom" BIGINT NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."two_Partitions" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_0) WHERE series_0 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_1) WHERE series_1 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_2) WHERE series_2 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_bool) WHERE series_bool IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, device_id); SELECT * FROM create_hypertable('"public"."two_Partitions"'::regclass, 'timeCustom'::name, 'device_id'::name, associated_schema_name=>'_timescaledb_internal'::text, number_partitions => 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+----------------+--------- 1 | public | two_Partitions | t \set QUIET off BEGIN; BEGIN \COPY public."two_Partitions" FROM 'data/ds1_dev1_1.tsv' NULL AS ''; COPY 7 COMMIT; COMMIT INSERT INTO public."two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257987600000000000, 'dev1', 1.5, 1), (1257987600000000000, 'dev1', 1.5, 2), (1257894000000000000, 'dev2', 1.5, 1), (1257894002000000000, 'dev1', 2.5, 3); INSERT 0 4 INSERT INTO "two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257894000000000000, 'dev2', 1.5, 2); INSERT 0 1 \set QUIET on SELECT * FROM test.show_columnsp('_timescaledb_internal.%_hyper%'); Relation | Kind | Column | Column type | NotNull ------------------------------------------------------------------------------------+------+-------------+------------------+--------- _timescaledb_internal._hyper_1_1_chunk | r | timeCustom | bigint | t _timescaledb_internal._hyper_1_1_chunk | r | device_id | text | t _timescaledb_internal._hyper_1_1_chunk | r | series_0 | double precision | f _timescaledb_internal._hyper_1_1_chunk | r | series_1 | double precision | f _timescaledb_internal._hyper_1_1_chunk | r | series_2 | double precision | f _timescaledb_internal._hyper_1_1_chunk | r | series_bool | boolean | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_device_id_timeCustom_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_device_id_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_device_id_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_device_id_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_0_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_0_idx" | i | series_0 | double precision | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" | i | series_1 | double precision | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_2_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_2_idx" | i | series_2 | double precision | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_bool_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_bool_idx" | i | series_bool | boolean | f _timescaledb_internal._hyper_1_2_chunk | r | timeCustom | bigint | t _timescaledb_internal._hyper_1_2_chunk | r | device_id | text | t _timescaledb_internal._hyper_1_2_chunk | r | series_0 | double precision | f _timescaledb_internal._hyper_1_2_chunk | r | series_1 | double precision | f _timescaledb_internal._hyper_1_2_chunk | r | series_2 | double precision | f _timescaledb_internal._hyper_1_2_chunk | r | series_bool | boolean | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_device_id_timeCustom_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_device_id_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_device_id_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_device_id_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_0_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_0_idx" | i | series_0 | double precision | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_1_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_1_idx" | i | series_1 | double precision | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_2_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_2_idx" | i | series_2 | double precision | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_bool_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_bool_idx" | i | series_bool | boolean | f _timescaledb_internal._hyper_1_3_chunk | r | timeCustom | bigint | t _timescaledb_internal._hyper_1_3_chunk | r | device_id | text | t _timescaledb_internal._hyper_1_3_chunk | r | series_0 | double precision | f _timescaledb_internal._hyper_1_3_chunk | r | series_1 | double precision | f _timescaledb_internal._hyper_1_3_chunk | r | series_2 | double precision | f _timescaledb_internal._hyper_1_3_chunk | r | series_bool | boolean | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_device_id_timeCustom_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_device_id_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_device_id_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_device_id_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_0_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_0_idx" | i | series_0 | double precision | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_1_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_1_idx" | i | series_1 | double precision | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_2_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_2_idx" | i | series_2 | double precision | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_bool_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_bool_idx" | i | series_bool | boolean | f _timescaledb_internal._hyper_1_4_chunk | r | timeCustom | bigint | t _timescaledb_internal._hyper_1_4_chunk | r | device_id | text | t _timescaledb_internal._hyper_1_4_chunk | r | series_0 | double precision | f _timescaledb_internal._hyper_1_4_chunk | r | series_1 | double precision | f _timescaledb_internal._hyper_1_4_chunk | r | series_2 | double precision | f _timescaledb_internal._hyper_1_4_chunk | r | series_bool | boolean | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_device_id_timeCustom_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_device_id_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_device_id_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_device_id_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_0_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_0_idx" | i | series_0 | double precision | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_1_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_1_idx" | i | series_1 | double precision | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_2_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_2_idx" | i | series_2 | double precision | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_bool_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_bool_idx" | i | series_bool | boolean | f SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+------------------------------------------------------------------------------------+--------------------------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_device_id_idx" | {timeCustom,device_id} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_idx" | {timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_device_id_idx" | {timeCustom,device_id} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_idx" | {timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_device_id_idx" | {timeCustom,device_id} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_idx" | {timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_device_id_idx" | {timeCustom,device_id} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_idx" | {timeCustom} | | f | f | f | SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-----------------------+------------------+---------------------+--------+----------- 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | | 0 | f 2 | 1 | _timescaledb_internal | _hyper_1_2_chunk | | 0 | f 3 | 1 | _timescaledb_internal | _hyper_1_3_chunk | | 0 | f 4 | 1 | _timescaledb_internal | _hyper_1_4_chunk | | 0 | f SELECT * FROM "two_Partitions" ORDER BY "timeCustom", device_id, series_0, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257894000000000000 | dev1 | 1.5 | 1 | 2 | t 1257894000000000000 | dev1 | 1.5 | 2 | | 1257894000000000000 | dev2 | 1.5 | 1 | | 1257894000000000000 | dev2 | 1.5 | 2 | | 1257894000000001000 | dev1 | 2.5 | 3 | | 1257894001000000000 | dev1 | 3.5 | 4 | | 1257894002000000000 | dev1 | 2.5 | 3 | | 1257894002000000000 | dev1 | 5.5 | 6 | | t 1257894002000000000 | dev1 | 5.5 | 7 | | f 1257897600000000000 | dev1 | 4.5 | 5 | | f 1257987600000000000 | dev1 | 1.5 | 1 | | 1257987600000000000 | dev1 | 1.5 | 2 | | SELECT * FROM ONLY "two_Partitions"; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ------------+-----------+----------+----------+----------+------------- CREATE TABLE error_test(time timestamp, temp float8, device text NOT NULL); SELECT create_hypertable('error_test', 'time', 'device', 2); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ------------------------- (2,public,error_test,t) \set QUIET off INSERT INTO error_test VALUES ('Mon Mar 20 09:18:20.1 2017', 21.3, 'dev1'); INSERT 0 1 \set ON_ERROR_STOP 0 -- generate insert error INSERT INTO error_test VALUES ('Mon Mar 20 09:18:22.3 2017', 21.1, NULL); ERROR: null value in column "device" of relation "_hyper_2_6_chunk" violates not-null constraint \set ON_ERROR_STOP 1 INSERT INTO error_test VALUES ('Mon Mar 20 09:18:25.7 2017', 22.4, 'dev2'); INSERT 0 1 \set QUIET on SELECT * FROM error_test; time | temp | device ----------------------------+------+-------- Mon Mar 20 09:18:20.1 2017 | 21.3 | dev1 Mon Mar 20 09:18:25.7 2017 | 22.4 | dev2 --test character(9) partition keys since there were issues with padding causing partitioning errors CREATE TABLE tick_character ( symbol character(9) NOT NULL, mid REAL NOT NULL, spread REAL NOT NULL, time TIMESTAMPTZ NOT NULL ); SELECT create_hypertable ('tick_character', 'time', 'symbol', 2); create_hypertable ----------------------------- (3,public,tick_character,t) INSERT INTO tick_character ( symbol, mid, spread, time ) VALUES ( 'GBPJPY', 142.639000, 5.80, 'Mon Mar 20 09:18:22.3 2017') RETURNING time, symbol, mid; time | symbol | mid --------------------------------+-----------+--------- Mon Mar 20 09:18:22.3 2017 PDT | GBPJPY | 142.639 SELECT * FROM tick_character; symbol | mid | spread | time -----------+---------+--------+-------------------------------- GBPJPY | 142.639 | 5.8 | Mon Mar 20 09:18:22.3 2017 PDT CREATE TABLE date_col_test(time date, temp float8, device text NOT NULL); SELECT create_hypertable('date_col_test', 'time', 'device', 1000, chunk_time_interval => INTERVAL '1 Day'); create_hypertable ---------------------------- (4,public,date_col_test,t) INSERT INTO date_col_test VALUES ('2001-02-01', 98, 'dev1'), ('2001-03-02', 98, 'dev1'); SELECT * FROM date_col_test WHERE time > '2001-01-01'; time | temp | device ------------+------+-------- 02-01-2001 | 98 | dev1 03-02-2001 | 98 | dev1 -- Out-of-order insertion regression test. -- this used to trip an assert in subspace_store.c checking that -- max_open_chunks_per_insert was obeyed set timescaledb.max_open_chunks_per_insert=1; CREATE TABLE chunk_assert_fail(i bigint, j bigint); SELECT create_hypertable('chunk_assert_fail', 'i', 'j', 1000, chunk_time_interval=>1); create_hypertable -------------------------------- (5,public,chunk_assert_fail,t) insert into chunk_assert_fail values (1, 1), (1, 2), (2,1); select * from chunk_assert_fail; i | j ---+--- 1 | 1 1 | 2 2 | 1 CREATE TABLE one_space_test(time timestamp, temp float8, device text NOT NULL); SELECT create_hypertable('one_space_test', 'time', 'device', 1); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ----------------------------- (6,public,one_space_test,t) INSERT INTO one_space_test VALUES ('2001-01-01 01:01:01', 1.0, 'device'), ('2002-01-01 01:02:01', 1.0, 'device'); SELECT * FROM one_space_test; time | temp | device --------------------------+------+-------- Mon Jan 01 01:01:01 2001 | 1 | device Tue Jan 01 01:02:01 2002 | 1 | device --CTE & EXPLAIN ANALYZE TESTS WITH insert_cte as ( INSERT INTO one_space_test VALUES ('2001-01-01 01:02:01', 1.0, 'device') RETURNING *) SELECT * FROM insert_cte; time | temp | device --------------------------+------+-------- Mon Jan 01 01:02:01 2001 | 1 | device EXPLAIN (analyze, buffers off, costs off, timing off) --can't turn summary off in 9.6 so instead grep it away at end. WITH insert_cte as ( INSERT INTO one_space_test VALUES ('2001-01-01 01:03:01', 1.0, 'device') ) SELECT 1 \g | grep -v "Planning" | grep -v "Execution" --- QUERY PLAN --- Result (actual rows=1.00 loops=1) CTE insert_cte -> Custom Scan (ModifyHypertable) (actual rows=0.00 loops=1) -> Insert on one_space_test (actual rows=0.00 loops=1) -> Result (actual rows=1.00 loops=1) -- INSERTs can exclude chunks based on constraints EXPLAIN (buffers off, costs off) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on chunk_assert_fail -> Append -> Index Only Scan using _hyper_5_11_chunk_chunk_assert_fail_j_i_idx on _hyper_5_11_chunk -> Index Only Scan using _hyper_5_12_chunk_chunk_assert_fail_j_i_idx on _hyper_5_12_chunk -> Index Only Scan using _hyper_5_13_chunk_chunk_assert_fail_j_i_idx on _hyper_5_13_chunk EXPLAIN (buffers off, costs off) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail WHERE i < 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on chunk_assert_fail -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail WHERE i = 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on chunk_assert_fail -> Append -> Index Scan using _hyper_5_11_chunk_chunk_assert_fail_i_idx on _hyper_5_11_chunk Index Cond: (i = 1) -> Index Scan using _hyper_5_12_chunk_chunk_assert_fail_i_idx on _hyper_5_12_chunk Index Cond: (i = 1) EXPLAIN (buffers off, costs off) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail WHERE i > 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on chunk_assert_fail -> Index Scan using _hyper_5_13_chunk_chunk_assert_fail_i_idx on _hyper_5_13_chunk Index Cond: (i > 1) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail WHERE i > 1; EXPLAIN (buffers off, costs off) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time < 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on one_space_test -> Limit -> Append -> Index Scan using _hyper_6_14_chunk_one_space_test_time_idx on _hyper_6_14_chunk Index Cond: ("time" < 'infinity'::timestamp without time zone) -> Index Scan using _hyper_6_15_chunk_one_space_test_time_idx on _hyper_6_15_chunk Index Cond: ("time" < 'infinity'::timestamp without time zone) EXPLAIN (buffers off, costs off) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time >= 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on one_space_test -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time <= '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on one_space_test -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time > '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on one_space_test -> Limit -> Append -> Index Scan using _hyper_6_14_chunk_one_space_test_time_idx on _hyper_6_14_chunk Index Cond: ("time" > '-infinity'::timestamp without time zone) -> Index Scan using _hyper_6_15_chunk_one_space_test_time_idx on _hyper_6_15_chunk Index Cond: ("time" > '-infinity'::timestamp without time zone) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time < 'infinity' LIMIT 1; INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time >= 'infinity' LIMIT 1; INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time <= '-infinity' LIMIT 1; INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time > '-infinity' LIMIT 1; CREATE TABLE timestamp_inf(time TIMESTAMP); SELECT create_hypertable('timestamp_inf', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ---------------------------- (7,public,timestamp_inf,t) INSERT INTO timestamp_inf VALUES ('2018/01/02'), ('2019/01/02'); EXPLAIN (buffers off, costs off) INSERT INTO timestamp_inf SELECT * FROM timestamp_inf WHERE time < 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on timestamp_inf -> Limit -> Append -> Index Only Scan using _hyper_7_16_chunk_timestamp_inf_time_idx on _hyper_7_16_chunk -> Index Only Scan using _hyper_7_17_chunk_timestamp_inf_time_idx on _hyper_7_17_chunk EXPLAIN (buffers off, costs off) INSERT INTO timestamp_inf SELECT * FROM timestamp_inf WHERE time >= 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on timestamp_inf -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO timestamp_inf SELECT * FROM timestamp_inf WHERE time <= '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on timestamp_inf -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO timestamp_inf SELECT * FROM timestamp_inf WHERE time > '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on timestamp_inf -> Limit -> Append -> Index Only Scan using _hyper_7_16_chunk_timestamp_inf_time_idx on _hyper_7_16_chunk -> Index Only Scan using _hyper_7_17_chunk_timestamp_inf_time_idx on _hyper_7_17_chunk CREATE TABLE date_inf(time DATE); SELECT create_hypertable('date_inf', 'time'); create_hypertable ----------------------- (8,public,date_inf,t) INSERT INTO date_inf VALUES ('2018/01/02'), ('2019/01/02'); EXPLAIN (buffers off, costs off) INSERT INTO date_inf SELECT * FROM date_inf WHERE time < 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on date_inf -> Limit -> Append -> Index Only Scan using _hyper_8_18_chunk_date_inf_time_idx on _hyper_8_18_chunk -> Index Only Scan using _hyper_8_19_chunk_date_inf_time_idx on _hyper_8_19_chunk EXPLAIN (buffers off, costs off) INSERT INTO date_inf SELECT * FROM date_inf WHERE time >= 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on date_inf -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO date_inf SELECT * FROM date_inf WHERE time <= '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on date_inf -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO date_inf SELECT * FROM date_inf WHERE time > '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on date_inf -> Limit -> Append -> Index Only Scan using _hyper_8_18_chunk_date_inf_time_idx on _hyper_8_18_chunk -> Index Only Scan using _hyper_8_19_chunk_date_inf_time_idx on _hyper_8_19_chunk -- test INSERT with cached plans / plpgsql functions -- https://github.com/timescale/timescaledb/issues/1809 CREATE TABLE status_table(a int, b int, last_ts timestamptz, UNIQUE(a,b)); CREATE TABLE metrics(time timestamptz NOT NULL, value float); CREATE TABLE metrics2(time timestamptz NOT NULL, value float); SELECT (create_hypertable(t,'time')).table_name FROM (VALUES ('metrics'),('metrics2')) v(t); table_name ------------ metrics metrics2 INSERT INTO metrics VALUES ('2000-01-01',random()), ('2000-02-01',random()), ('2000-03-01',random()); CREATE OR REPLACE FUNCTION insert_test() RETURNS VOID LANGUAGE plpgsql AS $$ DECLARE r RECORD; BEGIN FOR r IN SELECT * FROM metrics LOOP WITH foo AS ( INSERT INTO metrics2 SELECT * FROM metrics RETURNING * ) INSERT INTO status_table (a,b, last_ts) VALUES (1,1, now()) ON CONFLICT (a,b) DO UPDATE SET last_ts=(SELECT max(time) FROM metrics); END LOOP; END; $$; SELECT insert_test(), insert_test(), insert_test(); insert_test | insert_test | insert_test -------------+-------------+------------- | | -- test Postgres crashes on INSERT ... SELECT ... WHERE NOT EXISTS with empty table -- https://github.com/timescale/timescaledb/issues/1883 CREATE TABLE readings ( toe TIMESTAMPTZ NOT NULL, sensor_id INT NOT NULL, value INT NOT NULL ); SELECT create_hypertable( 'readings', 'toe', chunk_time_interval => interval '1 day', if_not_exists => TRUE, migrate_data => TRUE ); create_hypertable ------------------------ (11,public,readings,t) EXPLAIN (buffers off, costs off) INSERT INTO readings SELECT '2020-05-09 10:34:35.296288+00', 1, 0 WHERE NOT EXISTS ( SELECT 1 FROM readings WHERE sensor_id = 1 AND toe = '2020-05-09 10:34:35.296288+00' ); --- QUERY PLAN --- Custom Scan (ModifyHypertable) InitPlan 1 (returns $0) -> Result One-Time Filter: false -> Insert on readings -> Result One-Time Filter: (NOT $0) INSERT INTO readings SELECT '2020-05-09 10:34:35.296288+00', 1, 0 WHERE NOT EXISTS ( SELECT 1 FROM readings WHERE sensor_id = 1 AND toe = '2020-05-09 10:34:35.296288+00' ); DROP TABLE readings; CREATE TABLE sample_table ( sequence INTEGER NOT NULL, time TIMESTAMP WITHOUT TIME ZONE NOT NULL, value NUMERIC NOT NULL, UNIQUE (sequence, time) ); SELECT * FROM create_hypertable('sample_table', 'time', chunk_time_interval => INTERVAL '1 day'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices hypertable_id | schema_name | table_name | created ---------------+-------------+--------------+--------- 12 | public | sample_table | t INSERT INTO sample_table (sequence,time,value) VALUES (7, generate_series(TIMESTAMP '2019-08-01', TIMESTAMP '2019-08-10', INTERVAL '10 minutes'), ROUND(RANDOM()*10)::int); \set ON_ERROR_STOP 0 INSERT INTO sample_table (sequence,time,value) VALUES (7, generate_series(TIMESTAMP '2019-07-21', TIMESTAMP '2019-08-01', INTERVAL '10 minutes'), ROUND(RANDOM()*10)::int); ERROR: duplicate key value violates unique constraint "27_1_sample_table_sequence_time_key" \set ON_ERROR_STOP 1 INSERT INTO sample_table (sequence,time,value) VALUES (7,generate_series(TIMESTAMP '2019-01-01', TIMESTAMP '2019-07-01', '10 minutes'), ROUND(RANDOM()*10)::int); DROP TABLE sample_table; -- test on conflict clause on columns with default value -- issue #3037 CREATE TABLE i3037(time timestamptz PRIMARY KEY); SELECT create_hypertable('i3037','time'); create_hypertable --------------------- (13,public,i3037,t) ALTER TABLE i3037 ADD COLUMN value float DEFAULT 0; INSERT INTO i3037 VALUES ('2000-01-01'); INSERT INTO i3037 VALUES ('2000-01-01') ON CONFLICT(time) DO UPDATE SET value = EXCLUDED.value; -- test inserting into chunks directly CREATE TABLE direct_insert(time timestamptz, meta text) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column INSERT INTO direct_insert VALUES ('2020-01-01'); SELECT show_chunks('direct_insert') AS "CHUNK" \gset --should have ModifyHyperable node EXPLAIN (costs off, timing off, summary off) INSERT INTO :CHUNK VALUES ('2020-01-01'); --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on _hyper_14_231_chunk -> Result -- correct time range should succeed INSERT INTO :CHUNK VALUES ('2020-01-01') RETURNING *; time | meta ------------------------------+------ Wed Jan 01 00:00:00 2020 PST | -- incorrect time range should fail \set ON_ERROR_STOP 0 INSERT INTO :CHUNK VALUES ('2020-01-01'), ('2021-01-01') RETURNING *; ERROR: new row for relation "_hyper_14_231_chunk" violates chunk constraint INSERT INTO :CHUNK VALUES ('2025-01-01') RETURNING *; ERROR: new row for relation "_hyper_14_231_chunk" violates chunk constraint \set ON_ERROR_STOP 1 -- test that triggers on chunks work CREATE FUNCTION test_trigger() RETURNS TRIGGER AS $$ BEGIN RAISE NOTICE 'trigger called'; NEW.meta = 'triggered'; RETURN NEW; END; $$ LANGUAGE plpgsql; CREATE TRIGGER test_trigger BEFORE INSERT ON direct_insert FOR EACH ROW EXECUTE PROCEDURE test_trigger(); INSERT INTO :CHUNK VALUES ('2020-01-01') RETURNING *; NOTICE: trigger called time | meta ------------------------------+----------- Wed Jan 01 00:00:00 2020 PST | triggered -- test upsert DELETE FROM direct_insert; ALTER TABLE direct_insert ADD CONSTRAINT direct_insert_pkey PRIMARY KEY (time); INSERT INTO :CHUNK VALUES ('2020-01-01') RETURNING *; NOTICE: trigger called time | meta ------------------------------+----------- Wed Jan 01 00:00:00 2020 PST | triggered -- DO NOTHING should succeed INSERT INTO :CHUNK VALUES ('2020-01-01') ON CONFLICT DO NOTHING RETURNING *; NOTICE: trigger called time | meta ------+------ INSERT INTO :CHUNK VALUES ('2020-01-01') ON CONFLICT (time) DO UPDATE SET meta = 'update' RETURNING *; NOTICE: trigger called time | meta ------------------------------+-------- Wed Jan 01 00:00:00 2020 PST | update \set ON_ERROR_STOP 0 -- conflict should fail INSERT INTO :CHUNK VALUES ('2020-01-01') RETURNING *; NOTICE: trigger called ERROR: duplicate key value violates unique constraint "231_205_direct_insert_pkey" INSERT INTO :CHUNK VALUES ('2020-01-01') ON CONFLICT (time) DO UPDATE SET time = '2000-01-01' RETURNING *; NOTICE: trigger called ERROR: new row for relation "_hyper_14_231_chunk" violates check constraint "constraint_237" INSERT INTO :CHUNK VALUES ('2020-01-01') ON CONFLICT (time) DO UPDATE SET time = EXCLUDED.time + '1 year' RETURNING *; NOTICE: trigger called ERROR: new row for relation "_hyper_14_231_chunk" violates check constraint "constraint_237" \set ON_ERROR_STOP 1 ================================================ FILE: test/expected/insert-16.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SET enable_seqscan TO off; \ir include/insert_two_partitions.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."two_Partitions" ( "timeCustom" BIGINT NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."two_Partitions" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_0) WHERE series_0 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_1) WHERE series_1 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_2) WHERE series_2 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_bool) WHERE series_bool IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, device_id); SELECT * FROM create_hypertable('"public"."two_Partitions"'::regclass, 'timeCustom'::name, 'device_id'::name, associated_schema_name=>'_timescaledb_internal'::text, number_partitions => 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+----------------+--------- 1 | public | two_Partitions | t \set QUIET off BEGIN; BEGIN \COPY public."two_Partitions" FROM 'data/ds1_dev1_1.tsv' NULL AS ''; COPY 7 COMMIT; COMMIT INSERT INTO public."two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257987600000000000, 'dev1', 1.5, 1), (1257987600000000000, 'dev1', 1.5, 2), (1257894000000000000, 'dev2', 1.5, 1), (1257894002000000000, 'dev1', 2.5, 3); INSERT 0 4 INSERT INTO "two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257894000000000000, 'dev2', 1.5, 2); INSERT 0 1 \set QUIET on SELECT * FROM test.show_columnsp('_timescaledb_internal.%_hyper%'); Relation | Kind | Column | Column type | NotNull ------------------------------------------------------------------------------------+------+-------------+------------------+--------- _timescaledb_internal._hyper_1_1_chunk | r | timeCustom | bigint | t _timescaledb_internal._hyper_1_1_chunk | r | device_id | text | t _timescaledb_internal._hyper_1_1_chunk | r | series_0 | double precision | f _timescaledb_internal._hyper_1_1_chunk | r | series_1 | double precision | f _timescaledb_internal._hyper_1_1_chunk | r | series_2 | double precision | f _timescaledb_internal._hyper_1_1_chunk | r | series_bool | boolean | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_device_id_timeCustom_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_device_id_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_device_id_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_device_id_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_0_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_0_idx" | i | series_0 | double precision | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" | i | series_1 | double precision | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_2_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_2_idx" | i | series_2 | double precision | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_bool_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_bool_idx" | i | series_bool | boolean | f _timescaledb_internal._hyper_1_2_chunk | r | timeCustom | bigint | t _timescaledb_internal._hyper_1_2_chunk | r | device_id | text | t _timescaledb_internal._hyper_1_2_chunk | r | series_0 | double precision | f _timescaledb_internal._hyper_1_2_chunk | r | series_1 | double precision | f _timescaledb_internal._hyper_1_2_chunk | r | series_2 | double precision | f _timescaledb_internal._hyper_1_2_chunk | r | series_bool | boolean | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_device_id_timeCustom_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_device_id_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_device_id_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_device_id_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_0_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_0_idx" | i | series_0 | double precision | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_1_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_1_idx" | i | series_1 | double precision | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_2_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_2_idx" | i | series_2 | double precision | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_bool_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_bool_idx" | i | series_bool | boolean | f _timescaledb_internal._hyper_1_3_chunk | r | timeCustom | bigint | t _timescaledb_internal._hyper_1_3_chunk | r | device_id | text | t _timescaledb_internal._hyper_1_3_chunk | r | series_0 | double precision | f _timescaledb_internal._hyper_1_3_chunk | r | series_1 | double precision | f _timescaledb_internal._hyper_1_3_chunk | r | series_2 | double precision | f _timescaledb_internal._hyper_1_3_chunk | r | series_bool | boolean | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_device_id_timeCustom_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_device_id_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_device_id_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_device_id_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_0_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_0_idx" | i | series_0 | double precision | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_1_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_1_idx" | i | series_1 | double precision | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_2_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_2_idx" | i | series_2 | double precision | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_bool_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_bool_idx" | i | series_bool | boolean | f _timescaledb_internal._hyper_1_4_chunk | r | timeCustom | bigint | t _timescaledb_internal._hyper_1_4_chunk | r | device_id | text | t _timescaledb_internal._hyper_1_4_chunk | r | series_0 | double precision | f _timescaledb_internal._hyper_1_4_chunk | r | series_1 | double precision | f _timescaledb_internal._hyper_1_4_chunk | r | series_2 | double precision | f _timescaledb_internal._hyper_1_4_chunk | r | series_bool | boolean | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_device_id_timeCustom_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_device_id_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_device_id_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_device_id_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_0_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_0_idx" | i | series_0 | double precision | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_1_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_1_idx" | i | series_1 | double precision | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_2_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_2_idx" | i | series_2 | double precision | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_bool_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_bool_idx" | i | series_bool | boolean | f SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+------------------------------------------------------------------------------------+--------------------------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_device_id_idx" | {timeCustom,device_id} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_idx" | {timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_device_id_idx" | {timeCustom,device_id} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_idx" | {timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_device_id_idx" | {timeCustom,device_id} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_idx" | {timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_device_id_idx" | {timeCustom,device_id} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_idx" | {timeCustom} | | f | f | f | SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-----------------------+------------------+---------------------+--------+----------- 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | | 0 | f 2 | 1 | _timescaledb_internal | _hyper_1_2_chunk | | 0 | f 3 | 1 | _timescaledb_internal | _hyper_1_3_chunk | | 0 | f 4 | 1 | _timescaledb_internal | _hyper_1_4_chunk | | 0 | f SELECT * FROM "two_Partitions" ORDER BY "timeCustom", device_id, series_0, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257894000000000000 | dev1 | 1.5 | 1 | 2 | t 1257894000000000000 | dev1 | 1.5 | 2 | | 1257894000000000000 | dev2 | 1.5 | 1 | | 1257894000000000000 | dev2 | 1.5 | 2 | | 1257894000000001000 | dev1 | 2.5 | 3 | | 1257894001000000000 | dev1 | 3.5 | 4 | | 1257894002000000000 | dev1 | 2.5 | 3 | | 1257894002000000000 | dev1 | 5.5 | 6 | | t 1257894002000000000 | dev1 | 5.5 | 7 | | f 1257897600000000000 | dev1 | 4.5 | 5 | | f 1257987600000000000 | dev1 | 1.5 | 1 | | 1257987600000000000 | dev1 | 1.5 | 2 | | SELECT * FROM ONLY "two_Partitions"; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ------------+-----------+----------+----------+----------+------------- CREATE TABLE error_test(time timestamp, temp float8, device text NOT NULL); SELECT create_hypertable('error_test', 'time', 'device', 2); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ------------------------- (2,public,error_test,t) \set QUIET off INSERT INTO error_test VALUES ('Mon Mar 20 09:18:20.1 2017', 21.3, 'dev1'); INSERT 0 1 \set ON_ERROR_STOP 0 -- generate insert error INSERT INTO error_test VALUES ('Mon Mar 20 09:18:22.3 2017', 21.1, NULL); ERROR: null value in column "device" of relation "_hyper_2_6_chunk" violates not-null constraint \set ON_ERROR_STOP 1 INSERT INTO error_test VALUES ('Mon Mar 20 09:18:25.7 2017', 22.4, 'dev2'); INSERT 0 1 \set QUIET on SELECT * FROM error_test; time | temp | device ----------------------------+------+-------- Mon Mar 20 09:18:20.1 2017 | 21.3 | dev1 Mon Mar 20 09:18:25.7 2017 | 22.4 | dev2 --test character(9) partition keys since there were issues with padding causing partitioning errors CREATE TABLE tick_character ( symbol character(9) NOT NULL, mid REAL NOT NULL, spread REAL NOT NULL, time TIMESTAMPTZ NOT NULL ); SELECT create_hypertable ('tick_character', 'time', 'symbol', 2); create_hypertable ----------------------------- (3,public,tick_character,t) INSERT INTO tick_character ( symbol, mid, spread, time ) VALUES ( 'GBPJPY', 142.639000, 5.80, 'Mon Mar 20 09:18:22.3 2017') RETURNING time, symbol, mid; time | symbol | mid --------------------------------+-----------+--------- Mon Mar 20 09:18:22.3 2017 PDT | GBPJPY | 142.639 SELECT * FROM tick_character; symbol | mid | spread | time -----------+---------+--------+-------------------------------- GBPJPY | 142.639 | 5.8 | Mon Mar 20 09:18:22.3 2017 PDT CREATE TABLE date_col_test(time date, temp float8, device text NOT NULL); SELECT create_hypertable('date_col_test', 'time', 'device', 1000, chunk_time_interval => INTERVAL '1 Day'); create_hypertable ---------------------------- (4,public,date_col_test,t) INSERT INTO date_col_test VALUES ('2001-02-01', 98, 'dev1'), ('2001-03-02', 98, 'dev1'); SELECT * FROM date_col_test WHERE time > '2001-01-01'; time | temp | device ------------+------+-------- 02-01-2001 | 98 | dev1 03-02-2001 | 98 | dev1 -- Out-of-order insertion regression test. -- this used to trip an assert in subspace_store.c checking that -- max_open_chunks_per_insert was obeyed set timescaledb.max_open_chunks_per_insert=1; CREATE TABLE chunk_assert_fail(i bigint, j bigint); SELECT create_hypertable('chunk_assert_fail', 'i', 'j', 1000, chunk_time_interval=>1); create_hypertable -------------------------------- (5,public,chunk_assert_fail,t) insert into chunk_assert_fail values (1, 1), (1, 2), (2,1); select * from chunk_assert_fail; i | j ---+--- 1 | 1 1 | 2 2 | 1 CREATE TABLE one_space_test(time timestamp, temp float8, device text NOT NULL); SELECT create_hypertable('one_space_test', 'time', 'device', 1); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ----------------------------- (6,public,one_space_test,t) INSERT INTO one_space_test VALUES ('2001-01-01 01:01:01', 1.0, 'device'), ('2002-01-01 01:02:01', 1.0, 'device'); SELECT * FROM one_space_test; time | temp | device --------------------------+------+-------- Mon Jan 01 01:01:01 2001 | 1 | device Tue Jan 01 01:02:01 2002 | 1 | device --CTE & EXPLAIN ANALYZE TESTS WITH insert_cte as ( INSERT INTO one_space_test VALUES ('2001-01-01 01:02:01', 1.0, 'device') RETURNING *) SELECT * FROM insert_cte; time | temp | device --------------------------+------+-------- Mon Jan 01 01:02:01 2001 | 1 | device EXPLAIN (analyze, buffers off, costs off, timing off) --can't turn summary off in 9.6 so instead grep it away at end. WITH insert_cte as ( INSERT INTO one_space_test VALUES ('2001-01-01 01:03:01', 1.0, 'device') ) SELECT 1 \g | grep -v "Planning" | grep -v "Execution" --- QUERY PLAN --- Result (actual rows=1.00 loops=1) CTE insert_cte -> Custom Scan (ModifyHypertable) (actual rows=0.00 loops=1) -> Insert on one_space_test (actual rows=0.00 loops=1) -> Result (actual rows=1.00 loops=1) -- INSERTs can exclude chunks based on constraints EXPLAIN (buffers off, costs off) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on chunk_assert_fail -> Append -> Index Only Scan using _hyper_5_11_chunk_chunk_assert_fail_j_i_idx on _hyper_5_11_chunk -> Index Only Scan using _hyper_5_12_chunk_chunk_assert_fail_j_i_idx on _hyper_5_12_chunk -> Index Only Scan using _hyper_5_13_chunk_chunk_assert_fail_j_i_idx on _hyper_5_13_chunk EXPLAIN (buffers off, costs off) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail WHERE i < 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on chunk_assert_fail -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail WHERE i = 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on chunk_assert_fail -> Append -> Index Scan using _hyper_5_11_chunk_chunk_assert_fail_i_idx on _hyper_5_11_chunk Index Cond: (i = 1) -> Index Scan using _hyper_5_12_chunk_chunk_assert_fail_i_idx on _hyper_5_12_chunk Index Cond: (i = 1) EXPLAIN (buffers off, costs off) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail WHERE i > 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on chunk_assert_fail -> Index Scan using _hyper_5_13_chunk_chunk_assert_fail_i_idx on _hyper_5_13_chunk Index Cond: (i > 1) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail WHERE i > 1; EXPLAIN (buffers off, costs off) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time < 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on one_space_test -> Limit -> Append -> Index Scan using _hyper_6_14_chunk_one_space_test_time_idx on _hyper_6_14_chunk Index Cond: ("time" < 'infinity'::timestamp without time zone) -> Index Scan using _hyper_6_15_chunk_one_space_test_time_idx on _hyper_6_15_chunk Index Cond: ("time" < 'infinity'::timestamp without time zone) EXPLAIN (buffers off, costs off) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time >= 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on one_space_test -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time <= '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on one_space_test -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time > '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on one_space_test -> Limit -> Append -> Index Scan using _hyper_6_14_chunk_one_space_test_time_idx on _hyper_6_14_chunk Index Cond: ("time" > '-infinity'::timestamp without time zone) -> Index Scan using _hyper_6_15_chunk_one_space_test_time_idx on _hyper_6_15_chunk Index Cond: ("time" > '-infinity'::timestamp without time zone) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time < 'infinity' LIMIT 1; INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time >= 'infinity' LIMIT 1; INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time <= '-infinity' LIMIT 1; INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time > '-infinity' LIMIT 1; CREATE TABLE timestamp_inf(time TIMESTAMP); SELECT create_hypertable('timestamp_inf', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ---------------------------- (7,public,timestamp_inf,t) INSERT INTO timestamp_inf VALUES ('2018/01/02'), ('2019/01/02'); EXPLAIN (buffers off, costs off) INSERT INTO timestamp_inf SELECT * FROM timestamp_inf WHERE time < 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on timestamp_inf -> Limit -> Append -> Index Only Scan using _hyper_7_16_chunk_timestamp_inf_time_idx on _hyper_7_16_chunk -> Index Only Scan using _hyper_7_17_chunk_timestamp_inf_time_idx on _hyper_7_17_chunk EXPLAIN (buffers off, costs off) INSERT INTO timestamp_inf SELECT * FROM timestamp_inf WHERE time >= 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on timestamp_inf -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO timestamp_inf SELECT * FROM timestamp_inf WHERE time <= '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on timestamp_inf -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO timestamp_inf SELECT * FROM timestamp_inf WHERE time > '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on timestamp_inf -> Limit -> Append -> Index Only Scan using _hyper_7_16_chunk_timestamp_inf_time_idx on _hyper_7_16_chunk -> Index Only Scan using _hyper_7_17_chunk_timestamp_inf_time_idx on _hyper_7_17_chunk CREATE TABLE date_inf(time DATE); SELECT create_hypertable('date_inf', 'time'); create_hypertable ----------------------- (8,public,date_inf,t) INSERT INTO date_inf VALUES ('2018/01/02'), ('2019/01/02'); EXPLAIN (buffers off, costs off) INSERT INTO date_inf SELECT * FROM date_inf WHERE time < 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on date_inf -> Limit -> Append -> Index Only Scan using _hyper_8_18_chunk_date_inf_time_idx on _hyper_8_18_chunk -> Index Only Scan using _hyper_8_19_chunk_date_inf_time_idx on _hyper_8_19_chunk EXPLAIN (buffers off, costs off) INSERT INTO date_inf SELECT * FROM date_inf WHERE time >= 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on date_inf -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO date_inf SELECT * FROM date_inf WHERE time <= '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on date_inf -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO date_inf SELECT * FROM date_inf WHERE time > '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on date_inf -> Limit -> Append -> Index Only Scan using _hyper_8_18_chunk_date_inf_time_idx on _hyper_8_18_chunk -> Index Only Scan using _hyper_8_19_chunk_date_inf_time_idx on _hyper_8_19_chunk -- test INSERT with cached plans / plpgsql functions -- https://github.com/timescale/timescaledb/issues/1809 CREATE TABLE status_table(a int, b int, last_ts timestamptz, UNIQUE(a,b)); CREATE TABLE metrics(time timestamptz NOT NULL, value float); CREATE TABLE metrics2(time timestamptz NOT NULL, value float); SELECT (create_hypertable(t,'time')).table_name FROM (VALUES ('metrics'),('metrics2')) v(t); table_name ------------ metrics metrics2 INSERT INTO metrics VALUES ('2000-01-01',random()), ('2000-02-01',random()), ('2000-03-01',random()); CREATE OR REPLACE FUNCTION insert_test() RETURNS VOID LANGUAGE plpgsql AS $$ DECLARE r RECORD; BEGIN FOR r IN SELECT * FROM metrics LOOP WITH foo AS ( INSERT INTO metrics2 SELECT * FROM metrics RETURNING * ) INSERT INTO status_table (a,b, last_ts) VALUES (1,1, now()) ON CONFLICT (a,b) DO UPDATE SET last_ts=(SELECT max(time) FROM metrics); END LOOP; END; $$; SELECT insert_test(), insert_test(), insert_test(); insert_test | insert_test | insert_test -------------+-------------+------------- | | -- test Postgres crashes on INSERT ... SELECT ... WHERE NOT EXISTS with empty table -- https://github.com/timescale/timescaledb/issues/1883 CREATE TABLE readings ( toe TIMESTAMPTZ NOT NULL, sensor_id INT NOT NULL, value INT NOT NULL ); SELECT create_hypertable( 'readings', 'toe', chunk_time_interval => interval '1 day', if_not_exists => TRUE, migrate_data => TRUE ); create_hypertable ------------------------ (11,public,readings,t) EXPLAIN (buffers off, costs off) INSERT INTO readings SELECT '2020-05-09 10:34:35.296288+00', 1, 0 WHERE NOT EXISTS ( SELECT 1 FROM readings WHERE sensor_id = 1 AND toe = '2020-05-09 10:34:35.296288+00' ); --- QUERY PLAN --- Custom Scan (ModifyHypertable) InitPlan 1 (returns $0) -> Result One-Time Filter: false -> Insert on readings -> Result One-Time Filter: (NOT $0) INSERT INTO readings SELECT '2020-05-09 10:34:35.296288+00', 1, 0 WHERE NOT EXISTS ( SELECT 1 FROM readings WHERE sensor_id = 1 AND toe = '2020-05-09 10:34:35.296288+00' ); DROP TABLE readings; CREATE TABLE sample_table ( sequence INTEGER NOT NULL, time TIMESTAMP WITHOUT TIME ZONE NOT NULL, value NUMERIC NOT NULL, UNIQUE (sequence, time) ); SELECT * FROM create_hypertable('sample_table', 'time', chunk_time_interval => INTERVAL '1 day'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices hypertable_id | schema_name | table_name | created ---------------+-------------+--------------+--------- 12 | public | sample_table | t INSERT INTO sample_table (sequence,time,value) VALUES (7, generate_series(TIMESTAMP '2019-08-01', TIMESTAMP '2019-08-10', INTERVAL '10 minutes'), ROUND(RANDOM()*10)::int); \set ON_ERROR_STOP 0 INSERT INTO sample_table (sequence,time,value) VALUES (7, generate_series(TIMESTAMP '2019-07-21', TIMESTAMP '2019-08-01', INTERVAL '10 minutes'), ROUND(RANDOM()*10)::int); ERROR: duplicate key value violates unique constraint "27_1_sample_table_sequence_time_key" \set ON_ERROR_STOP 1 INSERT INTO sample_table (sequence,time,value) VALUES (7,generate_series(TIMESTAMP '2019-01-01', TIMESTAMP '2019-07-01', '10 minutes'), ROUND(RANDOM()*10)::int); DROP TABLE sample_table; -- test on conflict clause on columns with default value -- issue #3037 CREATE TABLE i3037(time timestamptz PRIMARY KEY); SELECT create_hypertable('i3037','time'); create_hypertable --------------------- (13,public,i3037,t) ALTER TABLE i3037 ADD COLUMN value float DEFAULT 0; INSERT INTO i3037 VALUES ('2000-01-01'); INSERT INTO i3037 VALUES ('2000-01-01') ON CONFLICT(time) DO UPDATE SET value = EXCLUDED.value; -- test inserting into chunks directly CREATE TABLE direct_insert(time timestamptz, meta text) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column INSERT INTO direct_insert VALUES ('2020-01-01'); SELECT show_chunks('direct_insert') AS "CHUNK" \gset --should have ModifyHyperable node EXPLAIN (costs off, timing off, summary off) INSERT INTO :CHUNK VALUES ('2020-01-01'); --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on _hyper_14_231_chunk -> Result -- correct time range should succeed INSERT INTO :CHUNK VALUES ('2020-01-01') RETURNING *; time | meta ------------------------------+------ Wed Jan 01 00:00:00 2020 PST | -- incorrect time range should fail \set ON_ERROR_STOP 0 INSERT INTO :CHUNK VALUES ('2020-01-01'), ('2021-01-01') RETURNING *; ERROR: new row for relation "_hyper_14_231_chunk" violates chunk constraint INSERT INTO :CHUNK VALUES ('2025-01-01') RETURNING *; ERROR: new row for relation "_hyper_14_231_chunk" violates chunk constraint \set ON_ERROR_STOP 1 -- test that triggers on chunks work CREATE FUNCTION test_trigger() RETURNS TRIGGER AS $$ BEGIN RAISE NOTICE 'trigger called'; NEW.meta = 'triggered'; RETURN NEW; END; $$ LANGUAGE plpgsql; CREATE TRIGGER test_trigger BEFORE INSERT ON direct_insert FOR EACH ROW EXECUTE PROCEDURE test_trigger(); INSERT INTO :CHUNK VALUES ('2020-01-01') RETURNING *; NOTICE: trigger called time | meta ------------------------------+----------- Wed Jan 01 00:00:00 2020 PST | triggered -- test upsert DELETE FROM direct_insert; ALTER TABLE direct_insert ADD CONSTRAINT direct_insert_pkey PRIMARY KEY (time); INSERT INTO :CHUNK VALUES ('2020-01-01') RETURNING *; NOTICE: trigger called time | meta ------------------------------+----------- Wed Jan 01 00:00:00 2020 PST | triggered -- DO NOTHING should succeed INSERT INTO :CHUNK VALUES ('2020-01-01') ON CONFLICT DO NOTHING RETURNING *; NOTICE: trigger called time | meta ------+------ INSERT INTO :CHUNK VALUES ('2020-01-01') ON CONFLICT (time) DO UPDATE SET meta = 'update' RETURNING *; NOTICE: trigger called time | meta ------------------------------+-------- Wed Jan 01 00:00:00 2020 PST | update \set ON_ERROR_STOP 0 -- conflict should fail INSERT INTO :CHUNK VALUES ('2020-01-01') RETURNING *; NOTICE: trigger called ERROR: duplicate key value violates unique constraint "231_205_direct_insert_pkey" INSERT INTO :CHUNK VALUES ('2020-01-01') ON CONFLICT (time) DO UPDATE SET time = '2000-01-01' RETURNING *; NOTICE: trigger called ERROR: new row for relation "_hyper_14_231_chunk" violates check constraint "constraint_237" INSERT INTO :CHUNK VALUES ('2020-01-01') ON CONFLICT (time) DO UPDATE SET time = EXCLUDED.time + '1 year' RETURNING *; NOTICE: trigger called ERROR: new row for relation "_hyper_14_231_chunk" violates check constraint "constraint_237" \set ON_ERROR_STOP 1 ================================================ FILE: test/expected/insert-17.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SET enable_seqscan TO off; \ir include/insert_two_partitions.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."two_Partitions" ( "timeCustom" BIGINT NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."two_Partitions" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_0) WHERE series_0 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_1) WHERE series_1 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_2) WHERE series_2 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_bool) WHERE series_bool IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, device_id); SELECT * FROM create_hypertable('"public"."two_Partitions"'::regclass, 'timeCustom'::name, 'device_id'::name, associated_schema_name=>'_timescaledb_internal'::text, number_partitions => 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+----------------+--------- 1 | public | two_Partitions | t \set QUIET off BEGIN; BEGIN \COPY public."two_Partitions" FROM 'data/ds1_dev1_1.tsv' NULL AS ''; COPY 7 COMMIT; COMMIT INSERT INTO public."two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257987600000000000, 'dev1', 1.5, 1), (1257987600000000000, 'dev1', 1.5, 2), (1257894000000000000, 'dev2', 1.5, 1), (1257894002000000000, 'dev1', 2.5, 3); INSERT 0 4 INSERT INTO "two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257894000000000000, 'dev2', 1.5, 2); INSERT 0 1 \set QUIET on SELECT * FROM test.show_columnsp('_timescaledb_internal.%_hyper%'); Relation | Kind | Column | Column type | NotNull ------------------------------------------------------------------------------------+------+-------------+------------------+--------- _timescaledb_internal._hyper_1_1_chunk | r | timeCustom | bigint | t _timescaledb_internal._hyper_1_1_chunk | r | device_id | text | t _timescaledb_internal._hyper_1_1_chunk | r | series_0 | double precision | f _timescaledb_internal._hyper_1_1_chunk | r | series_1 | double precision | f _timescaledb_internal._hyper_1_1_chunk | r | series_2 | double precision | f _timescaledb_internal._hyper_1_1_chunk | r | series_bool | boolean | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_device_id_timeCustom_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_device_id_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_device_id_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_device_id_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_0_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_0_idx" | i | series_0 | double precision | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" | i | series_1 | double precision | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_2_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_2_idx" | i | series_2 | double precision | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_bool_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_bool_idx" | i | series_bool | boolean | f _timescaledb_internal._hyper_1_2_chunk | r | timeCustom | bigint | t _timescaledb_internal._hyper_1_2_chunk | r | device_id | text | t _timescaledb_internal._hyper_1_2_chunk | r | series_0 | double precision | f _timescaledb_internal._hyper_1_2_chunk | r | series_1 | double precision | f _timescaledb_internal._hyper_1_2_chunk | r | series_2 | double precision | f _timescaledb_internal._hyper_1_2_chunk | r | series_bool | boolean | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_device_id_timeCustom_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_device_id_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_device_id_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_device_id_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_0_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_0_idx" | i | series_0 | double precision | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_1_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_1_idx" | i | series_1 | double precision | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_2_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_2_idx" | i | series_2 | double precision | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_bool_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_bool_idx" | i | series_bool | boolean | f _timescaledb_internal._hyper_1_3_chunk | r | timeCustom | bigint | t _timescaledb_internal._hyper_1_3_chunk | r | device_id | text | t _timescaledb_internal._hyper_1_3_chunk | r | series_0 | double precision | f _timescaledb_internal._hyper_1_3_chunk | r | series_1 | double precision | f _timescaledb_internal._hyper_1_3_chunk | r | series_2 | double precision | f _timescaledb_internal._hyper_1_3_chunk | r | series_bool | boolean | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_device_id_timeCustom_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_device_id_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_device_id_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_device_id_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_0_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_0_idx" | i | series_0 | double precision | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_1_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_1_idx" | i | series_1 | double precision | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_2_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_2_idx" | i | series_2 | double precision | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_bool_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_bool_idx" | i | series_bool | boolean | f _timescaledb_internal._hyper_1_4_chunk | r | timeCustom | bigint | t _timescaledb_internal._hyper_1_4_chunk | r | device_id | text | t _timescaledb_internal._hyper_1_4_chunk | r | series_0 | double precision | f _timescaledb_internal._hyper_1_4_chunk | r | series_1 | double precision | f _timescaledb_internal._hyper_1_4_chunk | r | series_2 | double precision | f _timescaledb_internal._hyper_1_4_chunk | r | series_bool | boolean | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_device_id_timeCustom_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_device_id_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_device_id_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_device_id_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_0_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_0_idx" | i | series_0 | double precision | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_1_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_1_idx" | i | series_1 | double precision | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_2_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_2_idx" | i | series_2 | double precision | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_bool_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_bool_idx" | i | series_bool | boolean | f SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+------------------------------------------------------------------------------------+--------------------------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_device_id_idx" | {timeCustom,device_id} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_idx" | {timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_device_id_idx" | {timeCustom,device_id} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_idx" | {timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_device_id_idx" | {timeCustom,device_id} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_idx" | {timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_device_id_idx" | {timeCustom,device_id} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_idx" | {timeCustom} | | f | f | f | SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-----------------------+------------------+---------------------+--------+----------- 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | | 0 | f 2 | 1 | _timescaledb_internal | _hyper_1_2_chunk | | 0 | f 3 | 1 | _timescaledb_internal | _hyper_1_3_chunk | | 0 | f 4 | 1 | _timescaledb_internal | _hyper_1_4_chunk | | 0 | f SELECT * FROM "two_Partitions" ORDER BY "timeCustom", device_id, series_0, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257894000000000000 | dev1 | 1.5 | 1 | 2 | t 1257894000000000000 | dev1 | 1.5 | 2 | | 1257894000000000000 | dev2 | 1.5 | 1 | | 1257894000000000000 | dev2 | 1.5 | 2 | | 1257894000000001000 | dev1 | 2.5 | 3 | | 1257894001000000000 | dev1 | 3.5 | 4 | | 1257894002000000000 | dev1 | 2.5 | 3 | | 1257894002000000000 | dev1 | 5.5 | 6 | | t 1257894002000000000 | dev1 | 5.5 | 7 | | f 1257897600000000000 | dev1 | 4.5 | 5 | | f 1257987600000000000 | dev1 | 1.5 | 1 | | 1257987600000000000 | dev1 | 1.5 | 2 | | SELECT * FROM ONLY "two_Partitions"; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ------------+-----------+----------+----------+----------+------------- CREATE TABLE error_test(time timestamp, temp float8, device text NOT NULL); SELECT create_hypertable('error_test', 'time', 'device', 2); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ------------------------- (2,public,error_test,t) \set QUIET off INSERT INTO error_test VALUES ('Mon Mar 20 09:18:20.1 2017', 21.3, 'dev1'); INSERT 0 1 \set ON_ERROR_STOP 0 -- generate insert error INSERT INTO error_test VALUES ('Mon Mar 20 09:18:22.3 2017', 21.1, NULL); ERROR: null value in column "device" of relation "_hyper_2_6_chunk" violates not-null constraint \set ON_ERROR_STOP 1 INSERT INTO error_test VALUES ('Mon Mar 20 09:18:25.7 2017', 22.4, 'dev2'); INSERT 0 1 \set QUIET on SELECT * FROM error_test; time | temp | device ----------------------------+------+-------- Mon Mar 20 09:18:20.1 2017 | 21.3 | dev1 Mon Mar 20 09:18:25.7 2017 | 22.4 | dev2 --test character(9) partition keys since there were issues with padding causing partitioning errors CREATE TABLE tick_character ( symbol character(9) NOT NULL, mid REAL NOT NULL, spread REAL NOT NULL, time TIMESTAMPTZ NOT NULL ); SELECT create_hypertable ('tick_character', 'time', 'symbol', 2); create_hypertable ----------------------------- (3,public,tick_character,t) INSERT INTO tick_character ( symbol, mid, spread, time ) VALUES ( 'GBPJPY', 142.639000, 5.80, 'Mon Mar 20 09:18:22.3 2017') RETURNING time, symbol, mid; time | symbol | mid --------------------------------+-----------+--------- Mon Mar 20 09:18:22.3 2017 PDT | GBPJPY | 142.639 SELECT * FROM tick_character; symbol | mid | spread | time -----------+---------+--------+-------------------------------- GBPJPY | 142.639 | 5.8 | Mon Mar 20 09:18:22.3 2017 PDT CREATE TABLE date_col_test(time date, temp float8, device text NOT NULL); SELECT create_hypertable('date_col_test', 'time', 'device', 1000, chunk_time_interval => INTERVAL '1 Day'); create_hypertable ---------------------------- (4,public,date_col_test,t) INSERT INTO date_col_test VALUES ('2001-02-01', 98, 'dev1'), ('2001-03-02', 98, 'dev1'); SELECT * FROM date_col_test WHERE time > '2001-01-01'; time | temp | device ------------+------+-------- 02-01-2001 | 98 | dev1 03-02-2001 | 98 | dev1 -- Out-of-order insertion regression test. -- this used to trip an assert in subspace_store.c checking that -- max_open_chunks_per_insert was obeyed set timescaledb.max_open_chunks_per_insert=1; CREATE TABLE chunk_assert_fail(i bigint, j bigint); SELECT create_hypertable('chunk_assert_fail', 'i', 'j', 1000, chunk_time_interval=>1); create_hypertable -------------------------------- (5,public,chunk_assert_fail,t) insert into chunk_assert_fail values (1, 1), (1, 2), (2,1); select * from chunk_assert_fail; i | j ---+--- 1 | 1 1 | 2 2 | 1 CREATE TABLE one_space_test(time timestamp, temp float8, device text NOT NULL); SELECT create_hypertable('one_space_test', 'time', 'device', 1); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ----------------------------- (6,public,one_space_test,t) INSERT INTO one_space_test VALUES ('2001-01-01 01:01:01', 1.0, 'device'), ('2002-01-01 01:02:01', 1.0, 'device'); SELECT * FROM one_space_test; time | temp | device --------------------------+------+-------- Mon Jan 01 01:01:01 2001 | 1 | device Tue Jan 01 01:02:01 2002 | 1 | device --CTE & EXPLAIN ANALYZE TESTS WITH insert_cte as ( INSERT INTO one_space_test VALUES ('2001-01-01 01:02:01', 1.0, 'device') RETURNING *) SELECT * FROM insert_cte; time | temp | device --------------------------+------+-------- Mon Jan 01 01:02:01 2001 | 1 | device EXPLAIN (analyze, buffers off, costs off, timing off) --can't turn summary off in 9.6 so instead grep it away at end. WITH insert_cte as ( INSERT INTO one_space_test VALUES ('2001-01-01 01:03:01', 1.0, 'device') ) SELECT 1 \g | grep -v "Planning" | grep -v "Execution" --- QUERY PLAN --- Result (actual rows=1.00 loops=1) CTE insert_cte -> Custom Scan (ModifyHypertable) (actual rows=0.00 loops=1) -> Insert on one_space_test (actual rows=0.00 loops=1) -> Result (actual rows=1.00 loops=1) -- INSERTs can exclude chunks based on constraints EXPLAIN (buffers off, costs off) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on chunk_assert_fail -> Append -> Index Only Scan using _hyper_5_11_chunk_chunk_assert_fail_j_i_idx on _hyper_5_11_chunk -> Index Only Scan using _hyper_5_12_chunk_chunk_assert_fail_j_i_idx on _hyper_5_12_chunk -> Index Only Scan using _hyper_5_13_chunk_chunk_assert_fail_j_i_idx on _hyper_5_13_chunk EXPLAIN (buffers off, costs off) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail WHERE i < 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on chunk_assert_fail -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail WHERE i = 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on chunk_assert_fail -> Append -> Index Scan using _hyper_5_11_chunk_chunk_assert_fail_i_idx on _hyper_5_11_chunk Index Cond: (i = 1) -> Index Scan using _hyper_5_12_chunk_chunk_assert_fail_i_idx on _hyper_5_12_chunk Index Cond: (i = 1) EXPLAIN (buffers off, costs off) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail WHERE i > 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on chunk_assert_fail -> Index Scan using _hyper_5_13_chunk_chunk_assert_fail_i_idx on _hyper_5_13_chunk Index Cond: (i > 1) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail WHERE i > 1; EXPLAIN (buffers off, costs off) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time < 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on one_space_test -> Limit -> Append -> Index Scan using _hyper_6_14_chunk_one_space_test_time_idx on _hyper_6_14_chunk Index Cond: ("time" < 'infinity'::timestamp without time zone) -> Index Scan using _hyper_6_15_chunk_one_space_test_time_idx on _hyper_6_15_chunk Index Cond: ("time" < 'infinity'::timestamp without time zone) EXPLAIN (buffers off, costs off) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time >= 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on one_space_test -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time <= '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on one_space_test -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time > '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on one_space_test -> Limit -> Append -> Index Scan using _hyper_6_14_chunk_one_space_test_time_idx on _hyper_6_14_chunk Index Cond: ("time" > '-infinity'::timestamp without time zone) -> Index Scan using _hyper_6_15_chunk_one_space_test_time_idx on _hyper_6_15_chunk Index Cond: ("time" > '-infinity'::timestamp without time zone) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time < 'infinity' LIMIT 1; INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time >= 'infinity' LIMIT 1; INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time <= '-infinity' LIMIT 1; INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time > '-infinity' LIMIT 1; CREATE TABLE timestamp_inf(time TIMESTAMP); SELECT create_hypertable('timestamp_inf', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ---------------------------- (7,public,timestamp_inf,t) INSERT INTO timestamp_inf VALUES ('2018/01/02'), ('2019/01/02'); EXPLAIN (buffers off, costs off) INSERT INTO timestamp_inf SELECT * FROM timestamp_inf WHERE time < 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on timestamp_inf -> Limit -> Append -> Index Only Scan using _hyper_7_16_chunk_timestamp_inf_time_idx on _hyper_7_16_chunk -> Index Only Scan using _hyper_7_17_chunk_timestamp_inf_time_idx on _hyper_7_17_chunk EXPLAIN (buffers off, costs off) INSERT INTO timestamp_inf SELECT * FROM timestamp_inf WHERE time >= 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on timestamp_inf -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO timestamp_inf SELECT * FROM timestamp_inf WHERE time <= '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on timestamp_inf -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO timestamp_inf SELECT * FROM timestamp_inf WHERE time > '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on timestamp_inf -> Limit -> Append -> Index Only Scan using _hyper_7_16_chunk_timestamp_inf_time_idx on _hyper_7_16_chunk -> Index Only Scan using _hyper_7_17_chunk_timestamp_inf_time_idx on _hyper_7_17_chunk CREATE TABLE date_inf(time DATE); SELECT create_hypertable('date_inf', 'time'); create_hypertable ----------------------- (8,public,date_inf,t) INSERT INTO date_inf VALUES ('2018/01/02'), ('2019/01/02'); EXPLAIN (buffers off, costs off) INSERT INTO date_inf SELECT * FROM date_inf WHERE time < 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on date_inf -> Limit -> Append -> Index Only Scan using _hyper_8_18_chunk_date_inf_time_idx on _hyper_8_18_chunk -> Index Only Scan using _hyper_8_19_chunk_date_inf_time_idx on _hyper_8_19_chunk EXPLAIN (buffers off, costs off) INSERT INTO date_inf SELECT * FROM date_inf WHERE time >= 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on date_inf -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO date_inf SELECT * FROM date_inf WHERE time <= '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on date_inf -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO date_inf SELECT * FROM date_inf WHERE time > '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on date_inf -> Limit -> Append -> Index Only Scan using _hyper_8_18_chunk_date_inf_time_idx on _hyper_8_18_chunk -> Index Only Scan using _hyper_8_19_chunk_date_inf_time_idx on _hyper_8_19_chunk -- test INSERT with cached plans / plpgsql functions -- https://github.com/timescale/timescaledb/issues/1809 CREATE TABLE status_table(a int, b int, last_ts timestamptz, UNIQUE(a,b)); CREATE TABLE metrics(time timestamptz NOT NULL, value float); CREATE TABLE metrics2(time timestamptz NOT NULL, value float); SELECT (create_hypertable(t,'time')).table_name FROM (VALUES ('metrics'),('metrics2')) v(t); table_name ------------ metrics metrics2 INSERT INTO metrics VALUES ('2000-01-01',random()), ('2000-02-01',random()), ('2000-03-01',random()); CREATE OR REPLACE FUNCTION insert_test() RETURNS VOID LANGUAGE plpgsql AS $$ DECLARE r RECORD; BEGIN FOR r IN SELECT * FROM metrics LOOP WITH foo AS ( INSERT INTO metrics2 SELECT * FROM metrics RETURNING * ) INSERT INTO status_table (a,b, last_ts) VALUES (1,1, now()) ON CONFLICT (a,b) DO UPDATE SET last_ts=(SELECT max(time) FROM metrics); END LOOP; END; $$; SELECT insert_test(), insert_test(), insert_test(); insert_test | insert_test | insert_test -------------+-------------+------------- | | -- test Postgres crashes on INSERT ... SELECT ... WHERE NOT EXISTS with empty table -- https://github.com/timescale/timescaledb/issues/1883 CREATE TABLE readings ( toe TIMESTAMPTZ NOT NULL, sensor_id INT NOT NULL, value INT NOT NULL ); SELECT create_hypertable( 'readings', 'toe', chunk_time_interval => interval '1 day', if_not_exists => TRUE, migrate_data => TRUE ); create_hypertable ------------------------ (11,public,readings,t) EXPLAIN (buffers off, costs off) INSERT INTO readings SELECT '2020-05-09 10:34:35.296288+00', 1, 0 WHERE NOT EXISTS ( SELECT 1 FROM readings WHERE sensor_id = 1 AND toe = '2020-05-09 10:34:35.296288+00' ); --- QUERY PLAN --- Custom Scan (ModifyHypertable) InitPlan 1 -> Result One-Time Filter: false -> Insert on readings -> Result One-Time Filter: (NOT (InitPlan 1).col1) INSERT INTO readings SELECT '2020-05-09 10:34:35.296288+00', 1, 0 WHERE NOT EXISTS ( SELECT 1 FROM readings WHERE sensor_id = 1 AND toe = '2020-05-09 10:34:35.296288+00' ); DROP TABLE readings; CREATE TABLE sample_table ( sequence INTEGER NOT NULL, time TIMESTAMP WITHOUT TIME ZONE NOT NULL, value NUMERIC NOT NULL, UNIQUE (sequence, time) ); SELECT * FROM create_hypertable('sample_table', 'time', chunk_time_interval => INTERVAL '1 day'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices hypertable_id | schema_name | table_name | created ---------------+-------------+--------------+--------- 12 | public | sample_table | t INSERT INTO sample_table (sequence,time,value) VALUES (7, generate_series(TIMESTAMP '2019-08-01', TIMESTAMP '2019-08-10', INTERVAL '10 minutes'), ROUND(RANDOM()*10)::int); \set ON_ERROR_STOP 0 INSERT INTO sample_table (sequence,time,value) VALUES (7, generate_series(TIMESTAMP '2019-07-21', TIMESTAMP '2019-08-01', INTERVAL '10 minutes'), ROUND(RANDOM()*10)::int); ERROR: duplicate key value violates unique constraint "27_1_sample_table_sequence_time_key" \set ON_ERROR_STOP 1 INSERT INTO sample_table (sequence,time,value) VALUES (7,generate_series(TIMESTAMP '2019-01-01', TIMESTAMP '2019-07-01', '10 minutes'), ROUND(RANDOM()*10)::int); DROP TABLE sample_table; -- test on conflict clause on columns with default value -- issue #3037 CREATE TABLE i3037(time timestamptz PRIMARY KEY); SELECT create_hypertable('i3037','time'); create_hypertable --------------------- (13,public,i3037,t) ALTER TABLE i3037 ADD COLUMN value float DEFAULT 0; INSERT INTO i3037 VALUES ('2000-01-01'); INSERT INTO i3037 VALUES ('2000-01-01') ON CONFLICT(time) DO UPDATE SET value = EXCLUDED.value; -- test inserting into chunks directly CREATE TABLE direct_insert(time timestamptz, meta text) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column INSERT INTO direct_insert VALUES ('2020-01-01'); SELECT show_chunks('direct_insert') AS "CHUNK" \gset --should have ModifyHyperable node EXPLAIN (costs off, timing off, summary off) INSERT INTO :CHUNK VALUES ('2020-01-01'); --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on _hyper_14_231_chunk -> Result -- correct time range should succeed INSERT INTO :CHUNK VALUES ('2020-01-01') RETURNING *; time | meta ------------------------------+------ Wed Jan 01 00:00:00 2020 PST | -- incorrect time range should fail \set ON_ERROR_STOP 0 INSERT INTO :CHUNK VALUES ('2020-01-01'), ('2021-01-01') RETURNING *; ERROR: new row for relation "_hyper_14_231_chunk" violates chunk constraint INSERT INTO :CHUNK VALUES ('2025-01-01') RETURNING *; ERROR: new row for relation "_hyper_14_231_chunk" violates chunk constraint \set ON_ERROR_STOP 1 -- test that triggers on chunks work CREATE FUNCTION test_trigger() RETURNS TRIGGER AS $$ BEGIN RAISE NOTICE 'trigger called'; NEW.meta = 'triggered'; RETURN NEW; END; $$ LANGUAGE plpgsql; CREATE TRIGGER test_trigger BEFORE INSERT ON direct_insert FOR EACH ROW EXECUTE PROCEDURE test_trigger(); INSERT INTO :CHUNK VALUES ('2020-01-01') RETURNING *; NOTICE: trigger called time | meta ------------------------------+----------- Wed Jan 01 00:00:00 2020 PST | triggered -- test upsert DELETE FROM direct_insert; ALTER TABLE direct_insert ADD CONSTRAINT direct_insert_pkey PRIMARY KEY (time); INSERT INTO :CHUNK VALUES ('2020-01-01') RETURNING *; NOTICE: trigger called time | meta ------------------------------+----------- Wed Jan 01 00:00:00 2020 PST | triggered -- DO NOTHING should succeed INSERT INTO :CHUNK VALUES ('2020-01-01') ON CONFLICT DO NOTHING RETURNING *; NOTICE: trigger called time | meta ------+------ INSERT INTO :CHUNK VALUES ('2020-01-01') ON CONFLICT (time) DO UPDATE SET meta = 'update' RETURNING *; NOTICE: trigger called time | meta ------------------------------+-------- Wed Jan 01 00:00:00 2020 PST | update \set ON_ERROR_STOP 0 -- conflict should fail INSERT INTO :CHUNK VALUES ('2020-01-01') RETURNING *; NOTICE: trigger called ERROR: duplicate key value violates unique constraint "231_205_direct_insert_pkey" INSERT INTO :CHUNK VALUES ('2020-01-01') ON CONFLICT (time) DO UPDATE SET time = '2000-01-01' RETURNING *; NOTICE: trigger called ERROR: new row for relation "_hyper_14_231_chunk" violates check constraint "constraint_237" INSERT INTO :CHUNK VALUES ('2020-01-01') ON CONFLICT (time) DO UPDATE SET time = EXCLUDED.time + '1 year' RETURNING *; NOTICE: trigger called ERROR: new row for relation "_hyper_14_231_chunk" violates check constraint "constraint_237" \set ON_ERROR_STOP 1 ================================================ FILE: test/expected/insert-18.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SET enable_seqscan TO off; \ir include/insert_two_partitions.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."two_Partitions" ( "timeCustom" BIGINT NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."two_Partitions" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_0) WHERE series_0 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_1) WHERE series_1 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_2) WHERE series_2 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_bool) WHERE series_bool IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, device_id); SELECT * FROM create_hypertable('"public"."two_Partitions"'::regclass, 'timeCustom'::name, 'device_id'::name, associated_schema_name=>'_timescaledb_internal'::text, number_partitions => 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+----------------+--------- 1 | public | two_Partitions | t \set QUIET off BEGIN; BEGIN \COPY public."two_Partitions" FROM 'data/ds1_dev1_1.tsv' NULL AS ''; COPY 7 COMMIT; COMMIT INSERT INTO public."two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257987600000000000, 'dev1', 1.5, 1), (1257987600000000000, 'dev1', 1.5, 2), (1257894000000000000, 'dev2', 1.5, 1), (1257894002000000000, 'dev1', 2.5, 3); INSERT 0 4 INSERT INTO "two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257894000000000000, 'dev2', 1.5, 2); INSERT 0 1 \set QUIET on SELECT * FROM test.show_columnsp('_timescaledb_internal.%_hyper%'); Relation | Kind | Column | Column type | NotNull ------------------------------------------------------------------------------------+------+-------------+------------------+--------- _timescaledb_internal._hyper_1_1_chunk | r | timeCustom | bigint | t _timescaledb_internal._hyper_1_1_chunk | r | device_id | text | t _timescaledb_internal._hyper_1_1_chunk | r | series_0 | double precision | f _timescaledb_internal._hyper_1_1_chunk | r | series_1 | double precision | f _timescaledb_internal._hyper_1_1_chunk | r | series_2 | double precision | f _timescaledb_internal._hyper_1_1_chunk | r | series_bool | boolean | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_device_id_timeCustom_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_device_id_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_device_id_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_device_id_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_0_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_0_idx" | i | series_0 | double precision | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" | i | series_1 | double precision | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_2_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_2_idx" | i | series_2 | double precision | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_bool_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_bool_idx" | i | series_bool | boolean | f _timescaledb_internal._hyper_1_2_chunk | r | timeCustom | bigint | t _timescaledb_internal._hyper_1_2_chunk | r | device_id | text | t _timescaledb_internal._hyper_1_2_chunk | r | series_0 | double precision | f _timescaledb_internal._hyper_1_2_chunk | r | series_1 | double precision | f _timescaledb_internal._hyper_1_2_chunk | r | series_2 | double precision | f _timescaledb_internal._hyper_1_2_chunk | r | series_bool | boolean | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_device_id_timeCustom_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_device_id_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_device_id_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_device_id_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_0_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_0_idx" | i | series_0 | double precision | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_1_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_1_idx" | i | series_1 | double precision | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_2_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_2_idx" | i | series_2 | double precision | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_bool_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_bool_idx" | i | series_bool | boolean | f _timescaledb_internal._hyper_1_3_chunk | r | timeCustom | bigint | t _timescaledb_internal._hyper_1_3_chunk | r | device_id | text | t _timescaledb_internal._hyper_1_3_chunk | r | series_0 | double precision | f _timescaledb_internal._hyper_1_3_chunk | r | series_1 | double precision | f _timescaledb_internal._hyper_1_3_chunk | r | series_2 | double precision | f _timescaledb_internal._hyper_1_3_chunk | r | series_bool | boolean | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_device_id_timeCustom_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_device_id_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_device_id_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_device_id_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_0_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_0_idx" | i | series_0 | double precision | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_1_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_1_idx" | i | series_1 | double precision | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_2_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_2_idx" | i | series_2 | double precision | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_bool_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_bool_idx" | i | series_bool | boolean | f _timescaledb_internal._hyper_1_4_chunk | r | timeCustom | bigint | t _timescaledb_internal._hyper_1_4_chunk | r | device_id | text | t _timescaledb_internal._hyper_1_4_chunk | r | series_0 | double precision | f _timescaledb_internal._hyper_1_4_chunk | r | series_1 | double precision | f _timescaledb_internal._hyper_1_4_chunk | r | series_2 | double precision | f _timescaledb_internal._hyper_1_4_chunk | r | series_bool | boolean | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_device_id_timeCustom_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_device_id_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_device_id_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_device_id_idx" | i | device_id | text | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_0_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_0_idx" | i | series_0 | double precision | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_1_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_1_idx" | i | series_1 | double precision | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_2_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_2_idx" | i | series_2 | double precision | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_bool_idx" | i | timeCustom | bigint | f _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_bool_idx" | i | series_bool | boolean | f SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+------------------------------------------------------------------------------------+--------------------------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_device_id_idx" | {timeCustom,device_id} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_idx" | {timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_device_id_idx" | {timeCustom,device_id} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."_hyper_1_2_chunk_two_Partitions_timeCustom_idx" | {timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_device_id_idx" | {timeCustom,device_id} | | f | f | f | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal."_hyper_1_3_chunk_two_Partitions_timeCustom_idx" | {timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_device_id_idx" | {timeCustom,device_id} | | f | f | f | _timescaledb_internal._hyper_1_4_chunk | _timescaledb_internal."_hyper_1_4_chunk_two_Partitions_timeCustom_idx" | {timeCustom} | | f | f | f | SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-----------------------+------------------+---------------------+--------+----------- 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | | 0 | f 2 | 1 | _timescaledb_internal | _hyper_1_2_chunk | | 0 | f 3 | 1 | _timescaledb_internal | _hyper_1_3_chunk | | 0 | f 4 | 1 | _timescaledb_internal | _hyper_1_4_chunk | | 0 | f SELECT * FROM "two_Partitions" ORDER BY "timeCustom", device_id, series_0, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257894000000000000 | dev1 | 1.5 | 1 | 2 | t 1257894000000000000 | dev1 | 1.5 | 2 | | 1257894000000000000 | dev2 | 1.5 | 1 | | 1257894000000000000 | dev2 | 1.5 | 2 | | 1257894000000001000 | dev1 | 2.5 | 3 | | 1257894001000000000 | dev1 | 3.5 | 4 | | 1257894002000000000 | dev1 | 2.5 | 3 | | 1257894002000000000 | dev1 | 5.5 | 6 | | t 1257894002000000000 | dev1 | 5.5 | 7 | | f 1257897600000000000 | dev1 | 4.5 | 5 | | f 1257987600000000000 | dev1 | 1.5 | 1 | | 1257987600000000000 | dev1 | 1.5 | 2 | | SELECT * FROM ONLY "two_Partitions"; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ------------+-----------+----------+----------+----------+------------- CREATE TABLE error_test(time timestamp, temp float8, device text NOT NULL); SELECT create_hypertable('error_test', 'time', 'device', 2); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ------------------------- (2,public,error_test,t) \set QUIET off INSERT INTO error_test VALUES ('Mon Mar 20 09:18:20.1 2017', 21.3, 'dev1'); INSERT 0 1 \set ON_ERROR_STOP 0 -- generate insert error INSERT INTO error_test VALUES ('Mon Mar 20 09:18:22.3 2017', 21.1, NULL); ERROR: null value in column "device" of relation "_hyper_2_6_chunk" violates not-null constraint \set ON_ERROR_STOP 1 INSERT INTO error_test VALUES ('Mon Mar 20 09:18:25.7 2017', 22.4, 'dev2'); INSERT 0 1 \set QUIET on SELECT * FROM error_test; time | temp | device ----------------------------+------+-------- Mon Mar 20 09:18:20.1 2017 | 21.3 | dev1 Mon Mar 20 09:18:25.7 2017 | 22.4 | dev2 --test character(9) partition keys since there were issues with padding causing partitioning errors CREATE TABLE tick_character ( symbol character(9) NOT NULL, mid REAL NOT NULL, spread REAL NOT NULL, time TIMESTAMPTZ NOT NULL ); SELECT create_hypertable ('tick_character', 'time', 'symbol', 2); create_hypertable ----------------------------- (3,public,tick_character,t) INSERT INTO tick_character ( symbol, mid, spread, time ) VALUES ( 'GBPJPY', 142.639000, 5.80, 'Mon Mar 20 09:18:22.3 2017') RETURNING time, symbol, mid; time | symbol | mid --------------------------------+-----------+--------- Mon Mar 20 09:18:22.3 2017 PDT | GBPJPY | 142.639 SELECT * FROM tick_character; symbol | mid | spread | time -----------+---------+--------+-------------------------------- GBPJPY | 142.639 | 5.8 | Mon Mar 20 09:18:22.3 2017 PDT CREATE TABLE date_col_test(time date, temp float8, device text NOT NULL); SELECT create_hypertable('date_col_test', 'time', 'device', 1000, chunk_time_interval => INTERVAL '1 Day'); create_hypertable ---------------------------- (4,public,date_col_test,t) INSERT INTO date_col_test VALUES ('2001-02-01', 98, 'dev1'), ('2001-03-02', 98, 'dev1'); SELECT * FROM date_col_test WHERE time > '2001-01-01'; time | temp | device ------------+------+-------- 02-01-2001 | 98 | dev1 03-02-2001 | 98 | dev1 -- Out-of-order insertion regression test. -- this used to trip an assert in subspace_store.c checking that -- max_open_chunks_per_insert was obeyed set timescaledb.max_open_chunks_per_insert=1; CREATE TABLE chunk_assert_fail(i bigint, j bigint); SELECT create_hypertable('chunk_assert_fail', 'i', 'j', 1000, chunk_time_interval=>1); create_hypertable -------------------------------- (5,public,chunk_assert_fail,t) insert into chunk_assert_fail values (1, 1), (1, 2), (2,1); select * from chunk_assert_fail; i | j ---+--- 1 | 1 1 | 2 2 | 1 CREATE TABLE one_space_test(time timestamp, temp float8, device text NOT NULL); SELECT create_hypertable('one_space_test', 'time', 'device', 1); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ----------------------------- (6,public,one_space_test,t) INSERT INTO one_space_test VALUES ('2001-01-01 01:01:01', 1.0, 'device'), ('2002-01-01 01:02:01', 1.0, 'device'); SELECT * FROM one_space_test; time | temp | device --------------------------+------+-------- Mon Jan 01 01:01:01 2001 | 1 | device Tue Jan 01 01:02:01 2002 | 1 | device --CTE & EXPLAIN ANALYZE TESTS WITH insert_cte as ( INSERT INTO one_space_test VALUES ('2001-01-01 01:02:01', 1.0, 'device') RETURNING *) SELECT * FROM insert_cte; time | temp | device --------------------------+------+-------- Mon Jan 01 01:02:01 2001 | 1 | device EXPLAIN (analyze, buffers off, costs off, timing off) --can't turn summary off in 9.6 so instead grep it away at end. WITH insert_cte as ( INSERT INTO one_space_test VALUES ('2001-01-01 01:03:01', 1.0, 'device') ) SELECT 1 \g | grep -v "Planning" | grep -v "Execution" --- QUERY PLAN --- Result (actual rows=1.00 loops=1) CTE insert_cte -> Custom Scan (ModifyHypertable) (actual rows=0.00 loops=1) -> Insert on one_space_test (actual rows=0.00 loops=1) -> Result (actual rows=1.00 loops=1) -- INSERTs can exclude chunks based on constraints EXPLAIN (buffers off, costs off) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on chunk_assert_fail -> Append -> Index Only Scan using _hyper_5_11_chunk_chunk_assert_fail_j_i_idx on _hyper_5_11_chunk -> Index Only Scan using _hyper_5_12_chunk_chunk_assert_fail_j_i_idx on _hyper_5_12_chunk -> Index Only Scan using _hyper_5_13_chunk_chunk_assert_fail_j_i_idx on _hyper_5_13_chunk EXPLAIN (buffers off, costs off) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail WHERE i < 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on chunk_assert_fail -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail WHERE i = 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on chunk_assert_fail -> Append -> Index Scan using _hyper_5_11_chunk_chunk_assert_fail_i_idx on _hyper_5_11_chunk Index Cond: (i = 1) -> Index Scan using _hyper_5_12_chunk_chunk_assert_fail_i_idx on _hyper_5_12_chunk Index Cond: (i = 1) EXPLAIN (buffers off, costs off) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail WHERE i > 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on chunk_assert_fail -> Index Scan using _hyper_5_13_chunk_chunk_assert_fail_i_idx on _hyper_5_13_chunk Index Cond: (i > 1) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail WHERE i > 1; EXPLAIN (buffers off, costs off) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time < 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on one_space_test -> Limit -> Append -> Index Scan using _hyper_6_14_chunk_one_space_test_time_idx on _hyper_6_14_chunk Index Cond: ("time" < 'infinity'::timestamp without time zone) -> Index Scan using _hyper_6_15_chunk_one_space_test_time_idx on _hyper_6_15_chunk Index Cond: ("time" < 'infinity'::timestamp without time zone) EXPLAIN (buffers off, costs off) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time >= 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on one_space_test -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time <= '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on one_space_test -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time > '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on one_space_test -> Limit -> Append -> Index Scan using _hyper_6_14_chunk_one_space_test_time_idx on _hyper_6_14_chunk Index Cond: ("time" > '-infinity'::timestamp without time zone) -> Index Scan using _hyper_6_15_chunk_one_space_test_time_idx on _hyper_6_15_chunk Index Cond: ("time" > '-infinity'::timestamp without time zone) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time < 'infinity' LIMIT 1; INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time >= 'infinity' LIMIT 1; INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time <= '-infinity' LIMIT 1; INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time > '-infinity' LIMIT 1; CREATE TABLE timestamp_inf(time TIMESTAMP); SELECT create_hypertable('timestamp_inf', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ---------------------------- (7,public,timestamp_inf,t) INSERT INTO timestamp_inf VALUES ('2018/01/02'), ('2019/01/02'); EXPLAIN (buffers off, costs off) INSERT INTO timestamp_inf SELECT * FROM timestamp_inf WHERE time < 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on timestamp_inf -> Limit -> Append -> Index Only Scan using _hyper_7_16_chunk_timestamp_inf_time_idx on _hyper_7_16_chunk -> Index Only Scan using _hyper_7_17_chunk_timestamp_inf_time_idx on _hyper_7_17_chunk EXPLAIN (buffers off, costs off) INSERT INTO timestamp_inf SELECT * FROM timestamp_inf WHERE time >= 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on timestamp_inf -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO timestamp_inf SELECT * FROM timestamp_inf WHERE time <= '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on timestamp_inf -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO timestamp_inf SELECT * FROM timestamp_inf WHERE time > '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on timestamp_inf -> Limit -> Append -> Index Only Scan using _hyper_7_16_chunk_timestamp_inf_time_idx on _hyper_7_16_chunk -> Index Only Scan using _hyper_7_17_chunk_timestamp_inf_time_idx on _hyper_7_17_chunk CREATE TABLE date_inf(time DATE); SELECT create_hypertable('date_inf', 'time'); create_hypertable ----------------------- (8,public,date_inf,t) INSERT INTO date_inf VALUES ('2018/01/02'), ('2019/01/02'); EXPLAIN (buffers off, costs off) INSERT INTO date_inf SELECT * FROM date_inf WHERE time < 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on date_inf -> Limit -> Append -> Index Only Scan using _hyper_8_18_chunk_date_inf_time_idx on _hyper_8_18_chunk -> Index Only Scan using _hyper_8_19_chunk_date_inf_time_idx on _hyper_8_19_chunk EXPLAIN (buffers off, costs off) INSERT INTO date_inf SELECT * FROM date_inf WHERE time >= 'infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on date_inf -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO date_inf SELECT * FROM date_inf WHERE time <= '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on date_inf -> Limit -> Result One-Time Filter: false EXPLAIN (buffers off, costs off) INSERT INTO date_inf SELECT * FROM date_inf WHERE time > '-infinity' LIMIT 1; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on date_inf -> Limit -> Append -> Index Only Scan using _hyper_8_18_chunk_date_inf_time_idx on _hyper_8_18_chunk -> Index Only Scan using _hyper_8_19_chunk_date_inf_time_idx on _hyper_8_19_chunk -- test INSERT with cached plans / plpgsql functions -- https://github.com/timescale/timescaledb/issues/1809 CREATE TABLE status_table(a int, b int, last_ts timestamptz, UNIQUE(a,b)); CREATE TABLE metrics(time timestamptz NOT NULL, value float); CREATE TABLE metrics2(time timestamptz NOT NULL, value float); SELECT (create_hypertable(t,'time')).table_name FROM (VALUES ('metrics'),('metrics2')) v(t); table_name ------------ metrics metrics2 INSERT INTO metrics VALUES ('2000-01-01',random()), ('2000-02-01',random()), ('2000-03-01',random()); CREATE OR REPLACE FUNCTION insert_test() RETURNS VOID LANGUAGE plpgsql AS $$ DECLARE r RECORD; BEGIN FOR r IN SELECT * FROM metrics LOOP WITH foo AS ( INSERT INTO metrics2 SELECT * FROM metrics RETURNING * ) INSERT INTO status_table (a,b, last_ts) VALUES (1,1, now()) ON CONFLICT (a,b) DO UPDATE SET last_ts=(SELECT max(time) FROM metrics); END LOOP; END; $$; SELECT insert_test(), insert_test(), insert_test(); insert_test | insert_test | insert_test -------------+-------------+------------- | | -- test Postgres crashes on INSERT ... SELECT ... WHERE NOT EXISTS with empty table -- https://github.com/timescale/timescaledb/issues/1883 CREATE TABLE readings ( toe TIMESTAMPTZ NOT NULL, sensor_id INT NOT NULL, value INT NOT NULL ); SELECT create_hypertable( 'readings', 'toe', chunk_time_interval => interval '1 day', if_not_exists => TRUE, migrate_data => TRUE ); create_hypertable ------------------------ (11,public,readings,t) EXPLAIN (buffers off, costs off) INSERT INTO readings SELECT '2020-05-09 10:34:35.296288+00', 1, 0 WHERE NOT EXISTS ( SELECT 1 FROM readings WHERE sensor_id = 1 AND toe = '2020-05-09 10:34:35.296288+00' ); --- QUERY PLAN --- Custom Scan (ModifyHypertable) InitPlan 1 -> Result One-Time Filter: false -> Insert on readings -> Result One-Time Filter: (NOT (InitPlan 1).col1) INSERT INTO readings SELECT '2020-05-09 10:34:35.296288+00', 1, 0 WHERE NOT EXISTS ( SELECT 1 FROM readings WHERE sensor_id = 1 AND toe = '2020-05-09 10:34:35.296288+00' ); DROP TABLE readings; CREATE TABLE sample_table ( sequence INTEGER NOT NULL, time TIMESTAMP WITHOUT TIME ZONE NOT NULL, value NUMERIC NOT NULL, UNIQUE (sequence, time) ); SELECT * FROM create_hypertable('sample_table', 'time', chunk_time_interval => INTERVAL '1 day'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices hypertable_id | schema_name | table_name | created ---------------+-------------+--------------+--------- 12 | public | sample_table | t INSERT INTO sample_table (sequence,time,value) VALUES (7, generate_series(TIMESTAMP '2019-08-01', TIMESTAMP '2019-08-10', INTERVAL '10 minutes'), ROUND(RANDOM()*10)::int); \set ON_ERROR_STOP 0 INSERT INTO sample_table (sequence,time,value) VALUES (7, generate_series(TIMESTAMP '2019-07-21', TIMESTAMP '2019-08-01', INTERVAL '10 minutes'), ROUND(RANDOM()*10)::int); ERROR: duplicate key value violates unique constraint "27_1_sample_table_sequence_time_key" \set ON_ERROR_STOP 1 INSERT INTO sample_table (sequence,time,value) VALUES (7,generate_series(TIMESTAMP '2019-01-01', TIMESTAMP '2019-07-01', '10 minutes'), ROUND(RANDOM()*10)::int); DROP TABLE sample_table; -- test on conflict clause on columns with default value -- issue #3037 CREATE TABLE i3037(time timestamptz PRIMARY KEY); SELECT create_hypertable('i3037','time'); create_hypertable --------------------- (13,public,i3037,t) ALTER TABLE i3037 ADD COLUMN value float DEFAULT 0; INSERT INTO i3037 VALUES ('2000-01-01'); INSERT INTO i3037 VALUES ('2000-01-01') ON CONFLICT(time) DO UPDATE SET value = EXCLUDED.value; -- test inserting into chunks directly CREATE TABLE direct_insert(time timestamptz, meta text) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column INSERT INTO direct_insert VALUES ('2020-01-01'); SELECT show_chunks('direct_insert') AS "CHUNK" \gset --should have ModifyHyperable node EXPLAIN (costs off, timing off, summary off) INSERT INTO :CHUNK VALUES ('2020-01-01'); --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on _hyper_14_231_chunk -> Result -- correct time range should succeed INSERT INTO :CHUNK VALUES ('2020-01-01') RETURNING *; time | meta ------------------------------+------ Wed Jan 01 00:00:00 2020 PST | -- incorrect time range should fail \set ON_ERROR_STOP 0 INSERT INTO :CHUNK VALUES ('2020-01-01'), ('2021-01-01') RETURNING *; ERROR: new row for relation "_hyper_14_231_chunk" violates chunk constraint INSERT INTO :CHUNK VALUES ('2025-01-01') RETURNING *; ERROR: new row for relation "_hyper_14_231_chunk" violates chunk constraint \set ON_ERROR_STOP 1 -- test that triggers on chunks work CREATE FUNCTION test_trigger() RETURNS TRIGGER AS $$ BEGIN RAISE NOTICE 'trigger called'; NEW.meta = 'triggered'; RETURN NEW; END; $$ LANGUAGE plpgsql; CREATE TRIGGER test_trigger BEFORE INSERT ON direct_insert FOR EACH ROW EXECUTE PROCEDURE test_trigger(); INSERT INTO :CHUNK VALUES ('2020-01-01') RETURNING *; NOTICE: trigger called time | meta ------------------------------+----------- Wed Jan 01 00:00:00 2020 PST | triggered -- test upsert DELETE FROM direct_insert; ALTER TABLE direct_insert ADD CONSTRAINT direct_insert_pkey PRIMARY KEY (time); INSERT INTO :CHUNK VALUES ('2020-01-01') RETURNING *; NOTICE: trigger called time | meta ------------------------------+----------- Wed Jan 01 00:00:00 2020 PST | triggered -- DO NOTHING should succeed INSERT INTO :CHUNK VALUES ('2020-01-01') ON CONFLICT DO NOTHING RETURNING *; NOTICE: trigger called time | meta ------+------ INSERT INTO :CHUNK VALUES ('2020-01-01') ON CONFLICT (time) DO UPDATE SET meta = 'update' RETURNING *; NOTICE: trigger called time | meta ------------------------------+-------- Wed Jan 01 00:00:00 2020 PST | update \set ON_ERROR_STOP 0 -- conflict should fail INSERT INTO :CHUNK VALUES ('2020-01-01') RETURNING *; NOTICE: trigger called ERROR: duplicate key value violates unique constraint "231_205_direct_insert_pkey" INSERT INTO :CHUNK VALUES ('2020-01-01') ON CONFLICT (time) DO UPDATE SET time = '2000-01-01' RETURNING *; NOTICE: trigger called ERROR: new row for relation "_hyper_14_231_chunk" violates check constraint "constraint_237" INSERT INTO :CHUNK VALUES ('2020-01-01') ON CONFLICT (time) DO UPDATE SET time = EXCLUDED.time + '1 year' RETURNING *; NOTICE: trigger called ERROR: new row for relation "_hyper_14_231_chunk" violates check constraint "constraint_237" \set ON_ERROR_STOP 1 ================================================ FILE: test/expected/insert_many.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE many_partitions_test(time timestamp, temp float8, device text NOT NULL); SELECT create_hypertable('many_partitions_test', 'time', 'device', 1000); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ----------------------------------- (1,public,many_partitions_test,t) --NOTE: how much slower the first two queries are -- they are creating chunks INSERT INTO many_partitions_test SELECT to_timestamp(ser), ser, ser::text FROM generate_series(1,100) ser; INSERT INTO many_partitions_test SELECT to_timestamp(ser), ser, ser::text FROM generate_series(101,200) ser; INSERT INTO many_partitions_test SELECT to_timestamp(ser), ser, (ser-201)::text FROM generate_series(201,300) ser; SELECT * FROM many_partitions_test ORDER BY time DESC LIMIT 2; time | temp | device --------------------------+------+-------- Wed Dec 31 16:05:00 1969 | 300 | 99 Wed Dec 31 16:04:59 1969 | 299 | 98 SELECT count(*) FROM many_partitions_test; count ------- 300 CREATE TABLE many_partitions_test_1m (time timestamp, temp float8, device text NOT NULL); SELECT create_hypertable('many_partitions_test_1m', 'time', 'device', 1000); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------------------- (2,public,many_partitions_test_1m,t) EXPLAIN (verbose on, buffers off, costs off) INSERT INTO many_partitions_test_1m(time, temp, device) SELECT time_bucket('1 minute', time) AS period, avg(temp), device FROM many_partitions_test GROUP BY period, device; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on public.many_partitions_test_1m -> HashAggregate Output: (time_bucket('@ 1 min'::interval, many_partitions_test."time")), avg(many_partitions_test.temp), many_partitions_test.device Group Key: time_bucket('@ 1 min'::interval, many_partitions_test."time"), many_partitions_test.device -> Result Output: time_bucket('@ 1 min'::interval, many_partitions_test."time"), many_partitions_test.device, many_partitions_test.temp -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.temp, _hyper_1_1_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.temp, _hyper_1_2_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.temp, _hyper_1_3_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.temp, _hyper_1_4_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_5_chunk Output: _hyper_1_5_chunk."time", _hyper_1_5_chunk.temp, _hyper_1_5_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_6_chunk Output: _hyper_1_6_chunk."time", _hyper_1_6_chunk.temp, _hyper_1_6_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_7_chunk Output: _hyper_1_7_chunk."time", _hyper_1_7_chunk.temp, _hyper_1_7_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_8_chunk Output: _hyper_1_8_chunk."time", _hyper_1_8_chunk.temp, _hyper_1_8_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_9_chunk Output: _hyper_1_9_chunk."time", _hyper_1_9_chunk.temp, _hyper_1_9_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_10_chunk Output: _hyper_1_10_chunk."time", _hyper_1_10_chunk.temp, _hyper_1_10_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_11_chunk Output: _hyper_1_11_chunk."time", _hyper_1_11_chunk.temp, _hyper_1_11_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_12_chunk Output: _hyper_1_12_chunk."time", _hyper_1_12_chunk.temp, _hyper_1_12_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_13_chunk Output: _hyper_1_13_chunk."time", _hyper_1_13_chunk.temp, _hyper_1_13_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_14_chunk Output: _hyper_1_14_chunk."time", _hyper_1_14_chunk.temp, _hyper_1_14_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_15_chunk Output: _hyper_1_15_chunk."time", _hyper_1_15_chunk.temp, _hyper_1_15_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_16_chunk Output: _hyper_1_16_chunk."time", _hyper_1_16_chunk.temp, _hyper_1_16_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_17_chunk Output: _hyper_1_17_chunk."time", _hyper_1_17_chunk.temp, _hyper_1_17_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_18_chunk Output: _hyper_1_18_chunk."time", _hyper_1_18_chunk.temp, _hyper_1_18_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_19_chunk Output: _hyper_1_19_chunk."time", _hyper_1_19_chunk.temp, _hyper_1_19_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_20_chunk Output: _hyper_1_20_chunk."time", _hyper_1_20_chunk.temp, _hyper_1_20_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_21_chunk Output: _hyper_1_21_chunk."time", _hyper_1_21_chunk.temp, _hyper_1_21_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_22_chunk Output: _hyper_1_22_chunk."time", _hyper_1_22_chunk.temp, _hyper_1_22_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_23_chunk Output: _hyper_1_23_chunk."time", _hyper_1_23_chunk.temp, _hyper_1_23_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_24_chunk Output: _hyper_1_24_chunk."time", _hyper_1_24_chunk.temp, _hyper_1_24_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_25_chunk Output: _hyper_1_25_chunk."time", _hyper_1_25_chunk.temp, _hyper_1_25_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_26_chunk Output: _hyper_1_26_chunk."time", _hyper_1_26_chunk.temp, _hyper_1_26_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_27_chunk Output: _hyper_1_27_chunk."time", _hyper_1_27_chunk.temp, _hyper_1_27_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_28_chunk Output: _hyper_1_28_chunk."time", _hyper_1_28_chunk.temp, _hyper_1_28_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_29_chunk Output: _hyper_1_29_chunk."time", _hyper_1_29_chunk.temp, _hyper_1_29_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_30_chunk Output: _hyper_1_30_chunk."time", _hyper_1_30_chunk.temp, _hyper_1_30_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_31_chunk Output: _hyper_1_31_chunk."time", _hyper_1_31_chunk.temp, _hyper_1_31_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_32_chunk Output: _hyper_1_32_chunk."time", _hyper_1_32_chunk.temp, _hyper_1_32_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_33_chunk Output: _hyper_1_33_chunk."time", _hyper_1_33_chunk.temp, _hyper_1_33_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_34_chunk Output: _hyper_1_34_chunk."time", _hyper_1_34_chunk.temp, _hyper_1_34_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_35_chunk Output: _hyper_1_35_chunk."time", _hyper_1_35_chunk.temp, _hyper_1_35_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_36_chunk Output: _hyper_1_36_chunk."time", _hyper_1_36_chunk.temp, _hyper_1_36_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_37_chunk Output: _hyper_1_37_chunk."time", _hyper_1_37_chunk.temp, _hyper_1_37_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_38_chunk Output: _hyper_1_38_chunk."time", _hyper_1_38_chunk.temp, _hyper_1_38_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_39_chunk Output: _hyper_1_39_chunk."time", _hyper_1_39_chunk.temp, _hyper_1_39_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_40_chunk Output: _hyper_1_40_chunk."time", _hyper_1_40_chunk.temp, _hyper_1_40_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_41_chunk Output: _hyper_1_41_chunk."time", _hyper_1_41_chunk.temp, _hyper_1_41_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_42_chunk Output: _hyper_1_42_chunk."time", _hyper_1_42_chunk.temp, _hyper_1_42_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_43_chunk Output: _hyper_1_43_chunk."time", _hyper_1_43_chunk.temp, _hyper_1_43_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_44_chunk Output: _hyper_1_44_chunk."time", _hyper_1_44_chunk.temp, _hyper_1_44_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_45_chunk Output: _hyper_1_45_chunk."time", _hyper_1_45_chunk.temp, _hyper_1_45_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_46_chunk Output: _hyper_1_46_chunk."time", _hyper_1_46_chunk.temp, _hyper_1_46_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_47_chunk Output: _hyper_1_47_chunk."time", _hyper_1_47_chunk.temp, _hyper_1_47_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_48_chunk Output: _hyper_1_48_chunk."time", _hyper_1_48_chunk.temp, _hyper_1_48_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_49_chunk Output: _hyper_1_49_chunk."time", _hyper_1_49_chunk.temp, _hyper_1_49_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_50_chunk Output: _hyper_1_50_chunk."time", _hyper_1_50_chunk.temp, _hyper_1_50_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_51_chunk Output: _hyper_1_51_chunk."time", _hyper_1_51_chunk.temp, _hyper_1_51_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_52_chunk Output: _hyper_1_52_chunk."time", _hyper_1_52_chunk.temp, _hyper_1_52_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_53_chunk Output: _hyper_1_53_chunk."time", _hyper_1_53_chunk.temp, _hyper_1_53_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_54_chunk Output: _hyper_1_54_chunk."time", _hyper_1_54_chunk.temp, _hyper_1_54_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_55_chunk Output: _hyper_1_55_chunk."time", _hyper_1_55_chunk.temp, _hyper_1_55_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_56_chunk Output: _hyper_1_56_chunk."time", _hyper_1_56_chunk.temp, _hyper_1_56_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_57_chunk Output: _hyper_1_57_chunk."time", _hyper_1_57_chunk.temp, _hyper_1_57_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_58_chunk Output: _hyper_1_58_chunk."time", _hyper_1_58_chunk.temp, _hyper_1_58_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_59_chunk Output: _hyper_1_59_chunk."time", _hyper_1_59_chunk.temp, _hyper_1_59_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_60_chunk Output: _hyper_1_60_chunk."time", _hyper_1_60_chunk.temp, _hyper_1_60_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_61_chunk Output: _hyper_1_61_chunk."time", _hyper_1_61_chunk.temp, _hyper_1_61_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_62_chunk Output: _hyper_1_62_chunk."time", _hyper_1_62_chunk.temp, _hyper_1_62_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_63_chunk Output: _hyper_1_63_chunk."time", _hyper_1_63_chunk.temp, _hyper_1_63_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_64_chunk Output: _hyper_1_64_chunk."time", _hyper_1_64_chunk.temp, _hyper_1_64_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_65_chunk Output: _hyper_1_65_chunk."time", _hyper_1_65_chunk.temp, _hyper_1_65_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_66_chunk Output: _hyper_1_66_chunk."time", _hyper_1_66_chunk.temp, _hyper_1_66_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_67_chunk Output: _hyper_1_67_chunk."time", _hyper_1_67_chunk.temp, _hyper_1_67_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_68_chunk Output: _hyper_1_68_chunk."time", _hyper_1_68_chunk.temp, _hyper_1_68_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_69_chunk Output: _hyper_1_69_chunk."time", _hyper_1_69_chunk.temp, _hyper_1_69_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_70_chunk Output: _hyper_1_70_chunk."time", _hyper_1_70_chunk.temp, _hyper_1_70_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_71_chunk Output: _hyper_1_71_chunk."time", _hyper_1_71_chunk.temp, _hyper_1_71_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_72_chunk Output: _hyper_1_72_chunk."time", _hyper_1_72_chunk.temp, _hyper_1_72_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_73_chunk Output: _hyper_1_73_chunk."time", _hyper_1_73_chunk.temp, _hyper_1_73_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_74_chunk Output: _hyper_1_74_chunk."time", _hyper_1_74_chunk.temp, _hyper_1_74_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_75_chunk Output: _hyper_1_75_chunk."time", _hyper_1_75_chunk.temp, _hyper_1_75_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_76_chunk Output: _hyper_1_76_chunk."time", _hyper_1_76_chunk.temp, _hyper_1_76_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_77_chunk Output: _hyper_1_77_chunk."time", _hyper_1_77_chunk.temp, _hyper_1_77_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_78_chunk Output: _hyper_1_78_chunk."time", _hyper_1_78_chunk.temp, _hyper_1_78_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_79_chunk Output: _hyper_1_79_chunk."time", _hyper_1_79_chunk.temp, _hyper_1_79_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_80_chunk Output: _hyper_1_80_chunk."time", _hyper_1_80_chunk.temp, _hyper_1_80_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_81_chunk Output: _hyper_1_81_chunk."time", _hyper_1_81_chunk.temp, _hyper_1_81_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_82_chunk Output: _hyper_1_82_chunk."time", _hyper_1_82_chunk.temp, _hyper_1_82_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_83_chunk Output: _hyper_1_83_chunk."time", _hyper_1_83_chunk.temp, _hyper_1_83_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_84_chunk Output: _hyper_1_84_chunk."time", _hyper_1_84_chunk.temp, _hyper_1_84_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_85_chunk Output: _hyper_1_85_chunk."time", _hyper_1_85_chunk.temp, _hyper_1_85_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_86_chunk Output: _hyper_1_86_chunk."time", _hyper_1_86_chunk.temp, _hyper_1_86_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_87_chunk Output: _hyper_1_87_chunk."time", _hyper_1_87_chunk.temp, _hyper_1_87_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_88_chunk Output: _hyper_1_88_chunk."time", _hyper_1_88_chunk.temp, _hyper_1_88_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_89_chunk Output: _hyper_1_89_chunk."time", _hyper_1_89_chunk.temp, _hyper_1_89_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_90_chunk Output: _hyper_1_90_chunk."time", _hyper_1_90_chunk.temp, _hyper_1_90_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_91_chunk Output: _hyper_1_91_chunk."time", _hyper_1_91_chunk.temp, _hyper_1_91_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_92_chunk Output: _hyper_1_92_chunk."time", _hyper_1_92_chunk.temp, _hyper_1_92_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_93_chunk Output: _hyper_1_93_chunk."time", _hyper_1_93_chunk.temp, _hyper_1_93_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_94_chunk Output: _hyper_1_94_chunk."time", _hyper_1_94_chunk.temp, _hyper_1_94_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_95_chunk Output: _hyper_1_95_chunk."time", _hyper_1_95_chunk.temp, _hyper_1_95_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_96_chunk Output: _hyper_1_96_chunk."time", _hyper_1_96_chunk.temp, _hyper_1_96_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_97_chunk Output: _hyper_1_97_chunk."time", _hyper_1_97_chunk.temp, _hyper_1_97_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_98_chunk Output: _hyper_1_98_chunk."time", _hyper_1_98_chunk.temp, _hyper_1_98_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_99_chunk Output: _hyper_1_99_chunk."time", _hyper_1_99_chunk.temp, _hyper_1_99_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_100_chunk Output: _hyper_1_100_chunk."time", _hyper_1_100_chunk.temp, _hyper_1_100_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_101_chunk Output: _hyper_1_101_chunk."time", _hyper_1_101_chunk.temp, _hyper_1_101_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_102_chunk Output: _hyper_1_102_chunk."time", _hyper_1_102_chunk.temp, _hyper_1_102_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_103_chunk Output: _hyper_1_103_chunk."time", _hyper_1_103_chunk.temp, _hyper_1_103_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_104_chunk Output: _hyper_1_104_chunk."time", _hyper_1_104_chunk.temp, _hyper_1_104_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_105_chunk Output: _hyper_1_105_chunk."time", _hyper_1_105_chunk.temp, _hyper_1_105_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_106_chunk Output: _hyper_1_106_chunk."time", _hyper_1_106_chunk.temp, _hyper_1_106_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_107_chunk Output: _hyper_1_107_chunk."time", _hyper_1_107_chunk.temp, _hyper_1_107_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_108_chunk Output: _hyper_1_108_chunk."time", _hyper_1_108_chunk.temp, _hyper_1_108_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_109_chunk Output: _hyper_1_109_chunk."time", _hyper_1_109_chunk.temp, _hyper_1_109_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_110_chunk Output: _hyper_1_110_chunk."time", _hyper_1_110_chunk.temp, _hyper_1_110_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_111_chunk Output: _hyper_1_111_chunk."time", _hyper_1_111_chunk.temp, _hyper_1_111_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_112_chunk Output: _hyper_1_112_chunk."time", _hyper_1_112_chunk.temp, _hyper_1_112_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_113_chunk Output: _hyper_1_113_chunk."time", _hyper_1_113_chunk.temp, _hyper_1_113_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_114_chunk Output: _hyper_1_114_chunk."time", _hyper_1_114_chunk.temp, _hyper_1_114_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_115_chunk Output: _hyper_1_115_chunk."time", _hyper_1_115_chunk.temp, _hyper_1_115_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_116_chunk Output: _hyper_1_116_chunk."time", _hyper_1_116_chunk.temp, _hyper_1_116_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_117_chunk Output: _hyper_1_117_chunk."time", _hyper_1_117_chunk.temp, _hyper_1_117_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_118_chunk Output: _hyper_1_118_chunk."time", _hyper_1_118_chunk.temp, _hyper_1_118_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_119_chunk Output: _hyper_1_119_chunk."time", _hyper_1_119_chunk.temp, _hyper_1_119_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_120_chunk Output: _hyper_1_120_chunk."time", _hyper_1_120_chunk.temp, _hyper_1_120_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_121_chunk Output: _hyper_1_121_chunk."time", _hyper_1_121_chunk.temp, _hyper_1_121_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_122_chunk Output: _hyper_1_122_chunk."time", _hyper_1_122_chunk.temp, _hyper_1_122_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_123_chunk Output: _hyper_1_123_chunk."time", _hyper_1_123_chunk.temp, _hyper_1_123_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_124_chunk Output: _hyper_1_124_chunk."time", _hyper_1_124_chunk.temp, _hyper_1_124_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_125_chunk Output: _hyper_1_125_chunk."time", _hyper_1_125_chunk.temp, _hyper_1_125_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_126_chunk Output: _hyper_1_126_chunk."time", _hyper_1_126_chunk.temp, _hyper_1_126_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_127_chunk Output: _hyper_1_127_chunk."time", _hyper_1_127_chunk.temp, _hyper_1_127_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_128_chunk Output: _hyper_1_128_chunk."time", _hyper_1_128_chunk.temp, _hyper_1_128_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_129_chunk Output: _hyper_1_129_chunk."time", _hyper_1_129_chunk.temp, _hyper_1_129_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_130_chunk Output: _hyper_1_130_chunk."time", _hyper_1_130_chunk.temp, _hyper_1_130_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_131_chunk Output: _hyper_1_131_chunk."time", _hyper_1_131_chunk.temp, _hyper_1_131_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_132_chunk Output: _hyper_1_132_chunk."time", _hyper_1_132_chunk.temp, _hyper_1_132_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_133_chunk Output: _hyper_1_133_chunk."time", _hyper_1_133_chunk.temp, _hyper_1_133_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_134_chunk Output: _hyper_1_134_chunk."time", _hyper_1_134_chunk.temp, _hyper_1_134_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_135_chunk Output: _hyper_1_135_chunk."time", _hyper_1_135_chunk.temp, _hyper_1_135_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_136_chunk Output: _hyper_1_136_chunk."time", _hyper_1_136_chunk.temp, _hyper_1_136_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_137_chunk Output: _hyper_1_137_chunk."time", _hyper_1_137_chunk.temp, _hyper_1_137_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_138_chunk Output: _hyper_1_138_chunk."time", _hyper_1_138_chunk.temp, _hyper_1_138_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_139_chunk Output: _hyper_1_139_chunk."time", _hyper_1_139_chunk.temp, _hyper_1_139_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_140_chunk Output: _hyper_1_140_chunk."time", _hyper_1_140_chunk.temp, _hyper_1_140_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_141_chunk Output: _hyper_1_141_chunk."time", _hyper_1_141_chunk.temp, _hyper_1_141_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_142_chunk Output: _hyper_1_142_chunk."time", _hyper_1_142_chunk.temp, _hyper_1_142_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_143_chunk Output: _hyper_1_143_chunk."time", _hyper_1_143_chunk.temp, _hyper_1_143_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_144_chunk Output: _hyper_1_144_chunk."time", _hyper_1_144_chunk.temp, _hyper_1_144_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_145_chunk Output: _hyper_1_145_chunk."time", _hyper_1_145_chunk.temp, _hyper_1_145_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_146_chunk Output: _hyper_1_146_chunk."time", _hyper_1_146_chunk.temp, _hyper_1_146_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_147_chunk Output: _hyper_1_147_chunk."time", _hyper_1_147_chunk.temp, _hyper_1_147_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_148_chunk Output: _hyper_1_148_chunk."time", _hyper_1_148_chunk.temp, _hyper_1_148_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_149_chunk Output: _hyper_1_149_chunk."time", _hyper_1_149_chunk.temp, _hyper_1_149_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_150_chunk Output: _hyper_1_150_chunk."time", _hyper_1_150_chunk.temp, _hyper_1_150_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_151_chunk Output: _hyper_1_151_chunk."time", _hyper_1_151_chunk.temp, _hyper_1_151_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_152_chunk Output: _hyper_1_152_chunk."time", _hyper_1_152_chunk.temp, _hyper_1_152_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_153_chunk Output: _hyper_1_153_chunk."time", _hyper_1_153_chunk.temp, _hyper_1_153_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_154_chunk Output: _hyper_1_154_chunk."time", _hyper_1_154_chunk.temp, _hyper_1_154_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_155_chunk Output: _hyper_1_155_chunk."time", _hyper_1_155_chunk.temp, _hyper_1_155_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_156_chunk Output: _hyper_1_156_chunk."time", _hyper_1_156_chunk.temp, _hyper_1_156_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_157_chunk Output: _hyper_1_157_chunk."time", _hyper_1_157_chunk.temp, _hyper_1_157_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_158_chunk Output: _hyper_1_158_chunk."time", _hyper_1_158_chunk.temp, _hyper_1_158_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_159_chunk Output: _hyper_1_159_chunk."time", _hyper_1_159_chunk.temp, _hyper_1_159_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_160_chunk Output: _hyper_1_160_chunk."time", _hyper_1_160_chunk.temp, _hyper_1_160_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_161_chunk Output: _hyper_1_161_chunk."time", _hyper_1_161_chunk.temp, _hyper_1_161_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_162_chunk Output: _hyper_1_162_chunk."time", _hyper_1_162_chunk.temp, _hyper_1_162_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_163_chunk Output: _hyper_1_163_chunk."time", _hyper_1_163_chunk.temp, _hyper_1_163_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_164_chunk Output: _hyper_1_164_chunk."time", _hyper_1_164_chunk.temp, _hyper_1_164_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_165_chunk Output: _hyper_1_165_chunk."time", _hyper_1_165_chunk.temp, _hyper_1_165_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_166_chunk Output: _hyper_1_166_chunk."time", _hyper_1_166_chunk.temp, _hyper_1_166_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_167_chunk Output: _hyper_1_167_chunk."time", _hyper_1_167_chunk.temp, _hyper_1_167_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_168_chunk Output: _hyper_1_168_chunk."time", _hyper_1_168_chunk.temp, _hyper_1_168_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_169_chunk Output: _hyper_1_169_chunk."time", _hyper_1_169_chunk.temp, _hyper_1_169_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_170_chunk Output: _hyper_1_170_chunk."time", _hyper_1_170_chunk.temp, _hyper_1_170_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_171_chunk Output: _hyper_1_171_chunk."time", _hyper_1_171_chunk.temp, _hyper_1_171_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_172_chunk Output: _hyper_1_172_chunk."time", _hyper_1_172_chunk.temp, _hyper_1_172_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_173_chunk Output: _hyper_1_173_chunk."time", _hyper_1_173_chunk.temp, _hyper_1_173_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_174_chunk Output: _hyper_1_174_chunk."time", _hyper_1_174_chunk.temp, _hyper_1_174_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_175_chunk Output: _hyper_1_175_chunk."time", _hyper_1_175_chunk.temp, _hyper_1_175_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_176_chunk Output: _hyper_1_176_chunk."time", _hyper_1_176_chunk.temp, _hyper_1_176_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_177_chunk Output: _hyper_1_177_chunk."time", _hyper_1_177_chunk.temp, _hyper_1_177_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_178_chunk Output: _hyper_1_178_chunk."time", _hyper_1_178_chunk.temp, _hyper_1_178_chunk.device INSERT INTO many_partitions_test_1m(time, temp, device) SELECT time_bucket('1 minute', time) AS period, avg(temp), device FROM many_partitions_test GROUP BY period, device; SELECT * FROM many_partitions_test_1m ORDER BY time, device LIMIT 10; time | temp | device --------------------------+------+-------- Wed Dec 31 16:00:00 1969 | 1 | 1 Wed Dec 31 16:00:00 1969 | 10 | 10 Wed Dec 31 16:00:00 1969 | 11 | 11 Wed Dec 31 16:00:00 1969 | 12 | 12 Wed Dec 31 16:00:00 1969 | 13 | 13 Wed Dec 31 16:00:00 1969 | 14 | 14 Wed Dec 31 16:00:00 1969 | 15 | 15 Wed Dec 31 16:00:00 1969 | 16 | 16 Wed Dec 31 16:00:00 1969 | 17 | 17 Wed Dec 31 16:00:00 1969 | 18 | 18 ================================================ FILE: test/expected/insert_returning.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE standard_table ( standard_id integer PRIMARY KEY, name text not null ); CREATE TABLE hypertable ( time timestamptz not null, name text not null, standard_id integer not null ); select * from create_hypertable('hypertable', 'time'); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | hypertable | t INSERT INTO standard_table (standard_id, name) VALUES (1, 'standard_1'); INSERT INTO hypertable (time, name, standard_id) VALUES ('2021-01-01 01:01:01+00', 'hypertable_1', 1); INSERT INTO hypertable (time, name, standard_id) VALUES ('2022-02-02 02:02:02+00', 'hypertable_2', 1) RETURNING *, EXISTS ( SELECT * FROM standard_table WHERE standard_table.standard_id = hypertable.standard_id ); time | name | standard_id | exists ------------------------------+--------------+-------------+-------- Tue Feb 01 18:02:02 2022 PST | hypertable_2 | 1 | t INSERT INTO hypertable (time, name, standard_id) VALUES ('2023-03-03 03:03:03+00', 'hypertable_3', 1) RETURNING *, EXISTS ( SELECT * FROM standard_table WHERE standard_table.standard_id = hypertable.standard_id ); time | name | standard_id | exists ------------------------------+--------------+-------------+-------- Thu Mar 02 19:03:03 2023 PST | hypertable_3 | 1 | t INSERT INTO hypertable (time, name, standard_id) VALUES ('2024-04-04 04:04:04+00', 'hypertable_4', 2) RETURNING *, EXISTS ( SELECT * FROM standard_table WHERE standard_table.standard_id = hypertable.standard_id ); time | name | standard_id | exists ------------------------------+--------------+-------------+-------- Wed Apr 03 21:04:04 2024 PDT | hypertable_4 | 2 | f ================================================ FILE: test/expected/insert_returning_old_new.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test OLD/NEW references in RETURNING clause (PG18+ feature) CREATE TABLE ht_returning( time timestamptz NOT NULL, value int NOT NULL, UNIQUE(time) ) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column -- Insert initial rows INSERT INTO ht_returning(time, value) VALUES ('2024-01-01', 10); INSERT INTO ht_returning(time, value) VALUES ('2024-01-02', 20); -- Test 1: INSERT ON CONFLICT DO UPDATE with RETURNING OLD.col, NEW.col -- This should show the old value (10) and the new value (100) INSERT INTO ht_returning(time, value) VALUES ('2024-01-01', 100) ON CONFLICT (time) DO UPDATE SET value = EXCLUDED.value RETURNING OLD.value AS old_val, NEW.value AS new_val; old_val | new_val ---------+--------- 10 | 100 -- Test 2: INSERT ON CONFLICT DO UPDATE with RETURNING arithmetic on OLD/NEW INSERT INTO ht_returning(time, value) VALUES ('2024-01-02', 50) ON CONFLICT (time) DO UPDATE SET value = EXCLUDED.value RETURNING OLD.value AS old_val, NEW.value AS new_val, NEW.value - OLD.value AS diff; old_val | new_val | diff ---------+---------+------ 20 | 50 | 30 -- Test 3: Plain INSERT with RETURNING NEW.col (OLD should be NULL for fresh inserts) INSERT INTO ht_returning(time, value) VALUES ('2024-01-03', 30) RETURNING OLD.value AS old_val, NEW.value AS new_val; old_val | new_val ---------+--------- | 30 -- Test 4: MERGE with both UPDATE and INSERT paths, returning OLD/NEW values and merge_action() CREATE TABLE t1(time timestamptz NOT NULL, value int NOT NULL); INSERT INTO t1(time, value) VALUES ('2024-01-01', 5); -- Will trigger UPDATE (existing row) INSERT INTO t1(time, value) VALUES ('2024-01-05', 10); -- Will trigger INSERT (new row) MERGE INTO ht_returning AS t USING t1 AS s ON t.time = s.time WHEN MATCHED THEN UPDATE SET value = t.value + s.value WHEN NOT MATCHED THEN INSERT (time, value) VALUES (s.time, s.value) RETURNING NEW.time, NEW.value, t.value, OLD.value, s.value, merge_action(); time | value | value | value | value | merge_action ------------------------------+-------+-------+-------+-------+-------------- Mon Jan 01 00:00:00 2024 PST | 105 | 105 | 100 | 5 | UPDATE Fri Jan 05 00:00:00 2024 PST | 10 | 10 | | 10 | INSERT -- Verify final state SELECT * FROM ht_returning ORDER BY time; time | value ------------------------------+------- Mon Jan 01 00:00:00 2024 PST | 105 Tue Jan 02 00:00:00 2024 PST | 50 Wed Jan 03 00:00:00 2024 PST | 30 Fri Jan 05 00:00:00 2024 PST | 10 -- Cleanup DROP TABLE ht_returning; ================================================ FILE: test/expected/insert_single.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \ir include/insert_single.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."one_Partition" ( "timeCustom" BIGINT NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."one_Partition" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; CREATE INDEX ON PUBLIC."one_Partition" ("timeCustom" DESC NULLS LAST, series_0) WHERE series_0 IS NOT NULL; CREATE INDEX ON PUBLIC."one_Partition" ("timeCustom" DESC NULLS LAST, series_1) WHERE series_1 IS NOT NULL; CREATE INDEX ON PUBLIC."one_Partition" ("timeCustom" DESC NULLS LAST, series_2) WHERE series_2 IS NOT NULL; CREATE INDEX ON PUBLIC."one_Partition" ("timeCustom" DESC NULLS LAST, series_bool) WHERE series_bool IS NOT NULL; \c :DBNAME :ROLE_SUPERUSER CREATE SCHEMA "one_Partition" AUTHORIZATION :ROLE_DEFAULT_PERM_USER; \c :DBNAME :ROLE_DEFAULT_PERM_USER; SELECT * FROM create_hypertable('"public"."one_Partition"', 'timeCustom', associated_schema_name=>'one_Partition', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+---------------+--------- 1 | public | one_Partition | t --output command tags \set QUIET off BEGIN; BEGIN \COPY "one_Partition" FROM 'data/ds1_dev1_1.tsv' NULL AS ''; COPY 7 COMMIT; COMMIT INSERT INTO "one_Partition"("timeCustom", device_id, series_0, series_1) VALUES (1257987600000000000, 'dev1', 1.5, 1), (1257987600000000000, 'dev1', 1.5, 2), (1257894000000000000, 'dev2', 1.5, 1), (1257894002000000000, 'dev1', 2.5, 3); INSERT 0 4 INSERT INTO "one_Partition"("timeCustom", device_id, series_0, series_1) VALUES (1257894000000000000, 'dev2', 1.5, 2); INSERT 0 1 \set QUIET on SELECT * FROM test.show_columnsp('"one_Partition".%'); Relation | Kind | Column | Column type | NotNull -----------------------------------------------------------------------------+------+-------------+------------------+--------- "one_Partition"._hyper_1_1_chunk | r | timeCustom | bigint | t "one_Partition"._hyper_1_1_chunk | r | device_id | text | t "one_Partition"._hyper_1_1_chunk | r | series_0 | double precision | f "one_Partition"._hyper_1_1_chunk | r | series_1 | double precision | f "one_Partition"._hyper_1_1_chunk | r | series_2 | double precision | f "one_Partition"._hyper_1_1_chunk | r | series_bool | boolean | f "one_Partition"."_hyper_1_1_chunk_one_Partition_device_id_timeCustom_idx" | i | device_id | text | f "one_Partition"."_hyper_1_1_chunk_one_Partition_device_id_timeCustom_idx" | i | timeCustom | bigint | f "one_Partition"."_hyper_1_1_chunk_one_Partition_timeCustom_idx" | i | timeCustom | bigint | f "one_Partition"."_hyper_1_1_chunk_one_Partition_timeCustom_series_0_idx" | i | timeCustom | bigint | f "one_Partition"."_hyper_1_1_chunk_one_Partition_timeCustom_series_0_idx" | i | series_0 | double precision | f "one_Partition"."_hyper_1_1_chunk_one_Partition_timeCustom_series_1_idx" | i | timeCustom | bigint | f "one_Partition"."_hyper_1_1_chunk_one_Partition_timeCustom_series_1_idx" | i | series_1 | double precision | f "one_Partition"."_hyper_1_1_chunk_one_Partition_timeCustom_series_2_idx" | i | timeCustom | bigint | f "one_Partition"."_hyper_1_1_chunk_one_Partition_timeCustom_series_2_idx" | i | series_2 | double precision | f "one_Partition"."_hyper_1_1_chunk_one_Partition_timeCustom_series_bool_idx" | i | timeCustom | bigint | f "one_Partition"."_hyper_1_1_chunk_one_Partition_timeCustom_series_bool_idx" | i | series_bool | boolean | f "one_Partition"._hyper_1_2_chunk | r | timeCustom | bigint | t "one_Partition"._hyper_1_2_chunk | r | device_id | text | t "one_Partition"._hyper_1_2_chunk | r | series_0 | double precision | f "one_Partition"._hyper_1_2_chunk | r | series_1 | double precision | f "one_Partition"._hyper_1_2_chunk | r | series_2 | double precision | f "one_Partition"._hyper_1_2_chunk | r | series_bool | boolean | f "one_Partition"."_hyper_1_2_chunk_one_Partition_device_id_timeCustom_idx" | i | device_id | text | f "one_Partition"."_hyper_1_2_chunk_one_Partition_device_id_timeCustom_idx" | i | timeCustom | bigint | f "one_Partition"."_hyper_1_2_chunk_one_Partition_timeCustom_idx" | i | timeCustom | bigint | f "one_Partition"."_hyper_1_2_chunk_one_Partition_timeCustom_series_0_idx" | i | timeCustom | bigint | f "one_Partition"."_hyper_1_2_chunk_one_Partition_timeCustom_series_0_idx" | i | series_0 | double precision | f "one_Partition"."_hyper_1_2_chunk_one_Partition_timeCustom_series_1_idx" | i | timeCustom | bigint | f "one_Partition"."_hyper_1_2_chunk_one_Partition_timeCustom_series_1_idx" | i | series_1 | double precision | f "one_Partition"."_hyper_1_2_chunk_one_Partition_timeCustom_series_2_idx" | i | timeCustom | bigint | f "one_Partition"."_hyper_1_2_chunk_one_Partition_timeCustom_series_2_idx" | i | series_2 | double precision | f "one_Partition"."_hyper_1_2_chunk_one_Partition_timeCustom_series_bool_idx" | i | timeCustom | bigint | f "one_Partition"."_hyper_1_2_chunk_one_Partition_timeCustom_series_bool_idx" | i | series_bool | boolean | f "one_Partition"._hyper_1_3_chunk | r | timeCustom | bigint | t "one_Partition"._hyper_1_3_chunk | r | device_id | text | t "one_Partition"._hyper_1_3_chunk | r | series_0 | double precision | f "one_Partition"._hyper_1_3_chunk | r | series_1 | double precision | f "one_Partition"._hyper_1_3_chunk | r | series_2 | double precision | f "one_Partition"._hyper_1_3_chunk | r | series_bool | boolean | f "one_Partition"."_hyper_1_3_chunk_one_Partition_device_id_timeCustom_idx" | i | device_id | text | f "one_Partition"."_hyper_1_3_chunk_one_Partition_device_id_timeCustom_idx" | i | timeCustom | bigint | f "one_Partition"."_hyper_1_3_chunk_one_Partition_timeCustom_idx" | i | timeCustom | bigint | f "one_Partition"."_hyper_1_3_chunk_one_Partition_timeCustom_series_0_idx" | i | timeCustom | bigint | f "one_Partition"."_hyper_1_3_chunk_one_Partition_timeCustom_series_0_idx" | i | series_0 | double precision | f "one_Partition"."_hyper_1_3_chunk_one_Partition_timeCustom_series_1_idx" | i | timeCustom | bigint | f "one_Partition"."_hyper_1_3_chunk_one_Partition_timeCustom_series_1_idx" | i | series_1 | double precision | f "one_Partition"."_hyper_1_3_chunk_one_Partition_timeCustom_series_2_idx" | i | timeCustom | bigint | f "one_Partition"."_hyper_1_3_chunk_one_Partition_timeCustom_series_2_idx" | i | series_2 | double precision | f "one_Partition"."_hyper_1_3_chunk_one_Partition_timeCustom_series_bool_idx" | i | timeCustom | bigint | f "one_Partition"."_hyper_1_3_chunk_one_Partition_timeCustom_series_bool_idx" | i | series_bool | boolean | f SELECT * FROM "one_Partition" ORDER BY "timeCustom", device_id, series_0, series_1, series_2; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257894000000000000 | dev1 | 1.5 | 1 | 2 | t 1257894000000000000 | dev1 | 1.5 | 2 | | 1257894000000000000 | dev2 | 1.5 | 1 | | 1257894000000000000 | dev2 | 1.5 | 2 | | 1257894000000001000 | dev1 | 2.5 | 3 | | 1257894001000000000 | dev1 | 3.5 | 4 | | 1257894002000000000 | dev1 | 2.5 | 3 | | 1257894002000000000 | dev1 | 5.5 | 6 | | t 1257894002000000000 | dev1 | 5.5 | 7 | | f 1257897600000000000 | dev1 | 4.5 | 5 | | f 1257987600000000000 | dev1 | 1.5 | 1 | | 1257987600000000000 | dev1 | 1.5 | 2 | | --test that we can insert data into a 1-dimensional table (only time partitioning) CREATE TABLE "1dim"(time timestamp PRIMARY KEY, temp float); SELECT create_hypertable('"1dim"', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ------------------- (2,public,1dim,t) INSERT INTO "1dim" VALUES('2017-01-20T09:00:01', 22.5) RETURNING *; time | temp --------------------------+------ Fri Jan 20 09:00:01 2017 | 22.5 INSERT INTO "1dim" VALUES('2017-01-20T09:00:21', 21.2); INSERT INTO "1dim" VALUES('2017-01-20T09:00:47', 25.1); SELECT * FROM "1dim"; time | temp --------------------------+------ Fri Jan 20 09:00:01 2017 | 22.5 Fri Jan 20 09:00:21 2017 | 21.2 Fri Jan 20 09:00:47 2017 | 25.1 CREATE TABLE regular_table (time timestamp, temp float); INSERT INTO regular_table SELECT * FROM "1dim"; SELECT * FROM regular_table; time | temp --------------------------+------ Fri Jan 20 09:00:01 2017 | 22.5 Fri Jan 20 09:00:21 2017 | 21.2 Fri Jan 20 09:00:47 2017 | 25.1 TRUNCATE TABLE regular_table; INSERT INTO regular_table VALUES('2017-01-20T09:00:59', 29.2); INSERT INTO "1dim" SELECT * FROM regular_table; SELECT * FROM "1dim"; time | temp --------------------------+------ Fri Jan 20 09:00:01 2017 | 22.5 Fri Jan 20 09:00:21 2017 | 21.2 Fri Jan 20 09:00:47 2017 | 25.1 Fri Jan 20 09:00:59 2017 | 29.2 SELECT "1dim" FROM "1dim"; 1dim ----------------------------------- ("Fri Jan 20 09:00:01 2017",22.5) ("Fri Jan 20 09:00:21 2017",21.2) ("Fri Jan 20 09:00:47 2017",25.1) ("Fri Jan 20 09:00:59 2017",29.2) --test that we can insert pre-1970 dates CREATE TABLE "1dim_pre1970"(time timestamp PRIMARY KEY, temp float); SELECT create_hypertable('"1dim_pre1970"', 'time', chunk_time_interval=> INTERVAL '1 Month'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable --------------------------- (3,public,1dim_pre1970,t) INSERT INTO "1dim_pre1970" VALUES('1969-12-01T19:00:00', 21.2); INSERT INTO "1dim_pre1970" VALUES('1969-12-20T09:00:00', 25.1); INSERT INTO "1dim_pre1970" VALUES('1970-01-20T09:00:00', 26.6); INSERT INTO "1dim_pre1970" VALUES('1969-02-20T09:00:00', 29.9); --should show warning BEGIN; CREATE TABLE "1dim_usec_interval"(time timestamp PRIMARY KEY, temp float); SELECT create_hypertable('"1dim_usec_interval"', 'time', chunk_time_interval=> 10); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices WARNING: unexpected interval: smaller than one second create_hypertable --------------------------------- (4,public,1dim_usec_interval,t) INSERT INTO "1dim_usec_interval" VALUES('1969-12-01T19:00:00', 21.2); ROLLBACK; CREATE TABLE "1dim_usec_interval"(time timestamp PRIMARY KEY, temp float); SELECT create_hypertable('"1dim_usec_interval"', 'time', chunk_time_interval=> 1000000); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable --------------------------------- (5,public,1dim_usec_interval,t) INSERT INTO "1dim_usec_interval" VALUES('1969-12-01T19:00:00', 21.2); CREATE TABLE "1dim_neg"(time INTEGER, temp float); SELECT create_hypertable('"1dim_neg"', 'time', chunk_time_interval=>10); create_hypertable ----------------------- (6,public,1dim_neg,t) INSERT INTO "1dim_neg" VALUES (-20, 21.2); INSERT INTO "1dim_neg" VALUES (-19, 21.2); INSERT INTO "1dim_neg" VALUES (-1, 21.2); INSERT INTO "1dim_neg" VALUES (0, 21.2); INSERT INTO "1dim_neg" VALUES (1, 21.2); INSERT INTO "1dim_neg" VALUES (19, 21.2); INSERT INTO "1dim_neg" VALUES (20, 21.2); SELECT * FROM "1dim_pre1970"; time | temp --------------------------+------ Mon Dec 01 19:00:00 1969 | 21.2 Sat Dec 20 09:00:00 1969 | 25.1 Tue Jan 20 09:00:00 1970 | 26.6 Thu Feb 20 09:00:00 1969 | 29.9 SELECT * FROM "1dim_neg"; time | temp ------+------ -20 | 21.2 -19 | 21.2 -1 | 21.2 0 | 21.2 1 | 21.2 19 | 21.2 20 | 21.2 SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-----------------------+-------------------+---------------------+--------+----------- 1 | 1 | one_Partition | _hyper_1_1_chunk | | 0 | f 2 | 1 | one_Partition | _hyper_1_2_chunk | | 0 | f 3 | 1 | one_Partition | _hyper_1_3_chunk | | 0 | f 4 | 2 | _timescaledb_internal | _hyper_2_4_chunk | | 0 | f 5 | 3 | _timescaledb_internal | _hyper_3_5_chunk | | 0 | f 6 | 3 | _timescaledb_internal | _hyper_3_6_chunk | | 0 | f 7 | 3 | _timescaledb_internal | _hyper_3_7_chunk | | 0 | f 8 | 3 | _timescaledb_internal | _hyper_3_8_chunk | | 0 | f 10 | 5 | _timescaledb_internal | _hyper_5_10_chunk | | 0 | f 11 | 6 | _timescaledb_internal | _hyper_6_11_chunk | | 0 | f 12 | 6 | _timescaledb_internal | _hyper_6_12_chunk | | 0 | f 13 | 6 | _timescaledb_internal | _hyper_6_13_chunk | | 0 | f 14 | 6 | _timescaledb_internal | _hyper_6_14_chunk | | 0 | f 15 | 6 | _timescaledb_internal | _hyper_6_15_chunk | | 0 | f SELECT * FROM _timescaledb_catalog.dimension_slice; id | dimension_id | range_start | range_end ----+--------------+---------------------+--------------------- 1 | 1 | 1257892416000000000 | 1257895008000000000 2 | 1 | 1257897600000000000 | 1257900192000000000 3 | 1 | 1257985728000000000 | 1257988320000000000 4 | 2 | 1484784000000000 | 1485388800000000 5 | 3 | -5184000000000 | -2592000000000 6 | 3 | -2592000000000 | 0 7 | 3 | 0 | 2592000000000 8 | 3 | -28512000000000 | -25920000000000 10 | 5 | -2610000000000 | -2609999000000 11 | 6 | -20 | -10 12 | 6 | -10 | 0 13 | 6 | 0 | 10 14 | 6 | 10 | 20 15 | 6 | 20 | 30 -- Create a three-dimensional table CREATE TABLE "3dim" (time timestamp, temp float, device text, location text); SELECT create_hypertable('"3dim"', 'time', 'device', 2); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ------------------- (7,public,3dim,t) SELECT add_dimension('"3dim"', 'location', 2); add_dimension ---------------------------- (9,public,3dim,location,t) INSERT INTO "3dim" VALUES('2017-01-20T09:00:01', 22.5, 'blue', 'nyc'); INSERT INTO "3dim" VALUES('2017-01-20T09:00:21', 21.2, 'brown', 'sthlm'); INSERT INTO "3dim" VALUES('2017-01-20T09:00:47', 25.1, 'yellow', 'la'); --show the constraints on the three-dimensional chunk SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_7_16_chunk'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated ---------------+------+------------+-------+----------------------------------------------------------------------------------------------------------------------------------------------+------------+----------+----------- constraint_16 | c | {time} | - | (("time" >= 'Thu Jan 19 00:00:00 2017'::timestamp without time zone) AND ("time" < 'Thu Jan 26 00:00:00 2017'::timestamp without time zone)) | f | f | t constraint_17 | c | {device} | - | (_timescaledb_functions.get_partition_hash(device) < 1073741823) | f | f | t constraint_18 | c | {location} | - | (_timescaledb_functions.get_partition_hash(location) >= 1073741823) | f | f | t --queries should work in three dimensions SELECT * FROM "3dim"; time | temp | device | location --------------------------+------+--------+---------- Fri Jan 20 09:00:01 2017 | 22.5 | blue | nyc Fri Jan 20 09:00:47 2017 | 25.1 | yellow | la Fri Jan 20 09:00:21 2017 | 21.2 | brown | sthlm -- test that explain works EXPLAIN (BUFFERS FALSE, COSTS FALSE) INSERT INTO "3dim" VALUES('2017-01-21T09:00:01', 32.9, 'green', 'nyc'), ('2017-01-21T09:00:47', 27.3, 'purple', 'la') RETURNING *; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on "3dim" -> Values Scan on "*VALUES*" EXPLAIN (BUFFERS FALSE, COSTS FALSE) WITH "3dim_insert" AS ( INSERT INTO "3dim" VALUES('2017-01-21T09:01:44', 19.3, 'black', 'la') RETURNING time, temp ), regular_insert AS ( INSERT INTO regular_table VALUES('2017-01-21T10:00:51', 14.3) RETURNING time, temp ) INSERT INTO "1dim" (SELECT time, temp FROM "3dim_insert" UNION SELECT time, temp FROM regular_insert); --- QUERY PLAN --- Custom Scan (ModifyHypertable) CTE 3dim_insert -> Custom Scan (ModifyHypertable) -> Insert on "3dim" -> Result CTE regular_insert -> Insert on regular_table -> Result -> Insert on "1dim" -> Unique -> Sort Sort Key: "3dim_insert"."time", "3dim_insert".temp -> Append -> CTE Scan on "3dim_insert" -> CTE Scan on regular_insert -- test prepared statement INSERT PREPARE "1dim_plan" (timestamp, float) AS INSERT INTO "1dim" VALUES($1, $2) ON CONFLICT (time) DO NOTHING; EXECUTE "1dim_plan" ('2017-04-17 23:35', 31.4); EXECUTE "1dim_plan" ('2017-04-17 23:35', 32.6); -- test prepared statement with generic plan (forced when no parameters) PREPARE "1dim_plan_generic" AS INSERT INTO "1dim" VALUES('2017-05-18 17:24', 18.3); EXECUTE "1dim_plan_generic"; SELECT * FROM "1dim" ORDER BY time; time | temp --------------------------+------ Fri Jan 20 09:00:01 2017 | 22.5 Fri Jan 20 09:00:21 2017 | 21.2 Fri Jan 20 09:00:47 2017 | 25.1 Fri Jan 20 09:00:59 2017 | 29.2 Mon Apr 17 23:35:00 2017 | 31.4 Thu May 18 17:24:00 2017 | 18.3 SELECT * FROM "3dim" ORDER BY (time, device); time | temp | device | location --------------------------+------+--------+---------- Fri Jan 20 09:00:01 2017 | 22.5 | blue | nyc Fri Jan 20 09:00:21 2017 | 21.2 | brown | sthlm Fri Jan 20 09:00:47 2017 | 25.1 | yellow | la -- Test large intervals as default interval for integer is -- supported as part of hypertable generalization \set ON_ERROR_STOP 0 CREATE TABLE "inttime_err"(time INTEGER PRIMARY KEY, temp float); SELECT create_hypertable('"inttime_err"', 'time', chunk_time_interval=>2147483648); ERROR: invalid interval: must be between 1 and 2147483647 \set ON_ERROR_STOP 1 SELECT create_hypertable('"inttime_err"', 'time', chunk_time_interval=>2147483647); create_hypertable -------------------------- (8,public,inttime_err,t) -- Test large intervals as default interval is supported -- for integer types as part of hypertable generalization. \set ON_ERROR_STOP 0 CREATE TABLE "smallinttime_err"(time SMALLINT PRIMARY KEY, temp float); SELECT create_hypertable('"smallinttime_err"', 'time', chunk_time_interval=>32768); ERROR: invalid interval: must be between 1 and 32767 \set ON_ERROR_STOP 1 SELECT create_hypertable('"smallinttime_err"', 'time', chunk_time_interval=>32767); create_hypertable ------------------------------- (9,public,smallinttime_err,t) --make sure date inserts work even when the timezone changes the CREATE TABLE hyper_date(time date, temp float); SELECT create_hypertable('"hyper_date"', 'time'); create_hypertable -------------------------- (10,public,hyper_date,t) SET timezone=+1; INSERT INTO "hyper_date" VALUES('2011-01-26', 22.5); RESET timezone; --make sure timestamp inserts work even when the timezone changes the SET timezone = 'UTC'; CREATE TABLE "test_tz"(time timestamp PRIMARY KEY, temp float); SELECT create_hypertable('"test_tz"', 'time', chunk_time_interval=> INTERVAL '1 day'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ----------------------- (11,public,test_tz,t) INSERT INTO "test_tz" VALUES('2017-09-22 10:00:00', 21.2); INSERT INTO "test_tz" VALUES('2017-09-21 19:00:00', 21.2); SET timezone = 'US/central'; INSERT INTO "test_tz" VALUES('2017-09-21 19:01:00', 21.2); SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_10_20_chunk'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated ---------------+------+---------+-------+--------------------------------------------------------------------+------------+----------+----------- constraint_23 | c | {time} | - | (("time" >= '01-20-2011'::date) AND ("time" < '01-27-2011'::date)) | f | f | t SELECT * FROM test_tz; time | temp --------------------------+------ Fri Sep 22 10:00:00 2017 | 21.2 Thu Sep 21 19:00:00 2017 | 21.2 Thu Sep 21 19:01:00 2017 | 21.2 -- test various memory settings -- SET timescaledb.max_open_chunks_per_insert = 10; SET timescaledb.max_cached_chunks_per_hypertable = 10; CREATE TABLE "nondefault_mem_settings"(time timestamp PRIMARY KEY, temp float); SELECT create_hypertable('"nondefault_mem_settings"', 'time', chunk_time_interval=> INTERVAL '1 Month'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable --------------------------------------- (12,public,nondefault_mem_settings,t) INSERT INTO "nondefault_mem_settings" VALUES('2000-12-01T19:00:00', 21.2); INSERT INTO "nondefault_mem_settings" VALUES('2001-12-20T09:00:00', 25.1); --lowest possible SET timescaledb.max_open_chunks_per_insert = 1; SET timescaledb.max_cached_chunks_per_hypertable = 1; INSERT INTO "nondefault_mem_settings" VALUES ('2001-01-20T09:00:00', 26.6), ('2002-02-20T09:00:00', 27.9), ('2003-02-20T09:00:00', 28.9); INSERT INTO "nondefault_mem_settings" VALUES ('2001-03-20T09:00:00', 30.6), ('2002-03-20T09:00:00', 31.9), ('2003-03-20T09:00:00', 32.9); --warning about mismatched cache sizes SET timescaledb.max_open_chunks_per_insert = 100; WARNING: insert cache size is larger than hypertable chunk cache size SET timescaledb.max_cached_chunks_per_hypertable = 10; WARNING: insert cache size is larger than hypertable chunk cache size INSERT INTO "nondefault_mem_settings" VALUES ('2001-05-20T09:00:00', 36.6), ('2002-05-20T09:00:00', 37.9), ('2003-05-20T09:00:00', 38.9); --unlimited SET timescaledb.max_open_chunks_per_insert = 0; SET timescaledb.max_cached_chunks_per_hypertable = 0; INSERT INTO "nondefault_mem_settings" VALUES ('2001-04-20T09:00:00', 33.6), ('2002-04-20T09:00:00', 34.9), ('2003-04-20T09:00:00', 35.9); SELECT * FROM "nondefault_mem_settings"; time | temp --------------------------+------ Fri Dec 01 19:00:00 2000 | 21.2 Thu Dec 20 09:00:00 2001 | 25.1 Sat Jan 20 09:00:00 2001 | 26.6 Wed Feb 20 09:00:00 2002 | 27.9 Thu Feb 20 09:00:00 2003 | 28.9 Tue Mar 20 09:00:00 2001 | 30.6 Wed Mar 20 09:00:00 2002 | 31.9 Thu Mar 20 09:00:00 2003 | 32.9 Sun May 20 09:00:00 2001 | 36.6 Mon May 20 09:00:00 2002 | 37.9 Tue May 20 09:00:00 2003 | 38.9 Fri Apr 20 09:00:00 2001 | 33.6 Sat Apr 20 09:00:00 2002 | 34.9 Sun Apr 20 09:00:00 2003 | 35.9 --test rollback BEGIN; \set QUIET off CREATE TABLE "data_records" ("time" bigint NOT NULL, "value" integer CHECK (VALUE >= 0)); CREATE TABLE SELECT create_hypertable('data_records', 'time', chunk_time_interval => 2592000000); create_hypertable ---------------------------- (13,public,data_records,t) INSERT INTO "data_records" ("time", "value") VALUES (0, 1); INSERT 0 1 SAVEPOINT savepoint_1; SAVEPOINT INSERT INTO "data_records" ("time", "value") VALUES (1, 0); INSERT 0 1 ROLLBACK TO SAVEPOINT savepoint_1; ROLLBACK INSERT INTO "data_records" ("time", "value") VALUES (2, 1); INSERT 0 1 SAVEPOINT savepoint_2; SAVEPOINT \set ON_ERROR_STOP 0 INSERT INTO "data_records" ("time", "value") VALUES (3, -1); ERROR: new row for relation "_hyper_13_37_chunk" violates check constraint "data_records_value_check" \set ON_ERROR_STOP 1 ROLLBACK TO SAVEPOINT savepoint_2; ROLLBACK INSERT INTO "data_records" ("time", "value") VALUES (4, 1); INSERT 0 1 SAVEPOINT savepoint_3; SAVEPOINT INSERT INTO "data_records" ("time", "value") VALUES (5, 0); INSERT 0 1 ROLLBACK TO SAVEPOINT savepoint_3; ROLLBACK SELECT * FROM data_records; time | value ------+------- 0 | 1 2 | 1 4 | 1 \set QUIET on ROLLBACK; -- Test INSERT into hypertable with a generated column whose type is a -- domain with a NOT NULL constraint CREATE DOMAIN nn_int AS int CHECK (VALUE IS NOT NULL); CREATE TABLE generated_col_ht( time timestamptz NOT NULL, val int NOT NULL, doubled nn_int GENERATED ALWAYS AS (val * 2) STORED ); SELECT create_hypertable('generated_col_ht', 'time'); create_hypertable -------------------------------- (14,public,generated_col_ht,t) INSERT INTO generated_col_ht(time, val) VALUES ('2024-01-01', 5); INSERT INTO generated_col_ht(time, val) VALUES ('2024-01-02', 10) RETURNING *; time | val | doubled ------------------------------+-----+--------- Tue Jan 02 00:00:00 2024 CST | 10 | 20 SELECT * FROM generated_col_ht ORDER BY time; time | val | doubled ------------------------------+-----+--------- Mon Jan 01 00:00:00 2024 CST | 5 | 10 Tue Jan 02 00:00:00 2024 CST | 10 | 20 DROP TABLE generated_col_ht; DROP DOMAIN nn_int; ================================================ FILE: test/expected/lateral.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE regular_table(name text, junk text); CREATE TABLE ht(time timestamptz NOT NULL, location text); SELECT create_hypertable('ht', 'time'); create_hypertable ------------------- (1,public,ht,t) INSERT INTO ht(time) select timestamp 'epoch' + (i * interval '1 second') from generate_series(1, 100) as T(i); INSERT INTO regular_table values('name', 'junk'); SELECT * FROM regular_table ik LEFT JOIN LATERAL (select max(time::timestamptz) from ht s where ik.name='name' and s.time < now()) s on true; name | junk | max ------+------+------------------------------ name | junk | Thu Jan 01 00:01:40 1970 PST select * from regular_table ik LEFT JOIN LATERAL (select max(time::timestamptz) from ht s where ik.name='name' and s.time > now()) s on true; name | junk | max ------+------+----- name | junk | DROP TABLE regular_table; DROP TABLE ht; CREATE TABLE orders(id int, user_id int, time TIMESTAMPTZ NOT NULL); SELECT create_hypertable('orders', 'time'); create_hypertable --------------------- (2,public,orders,t) INSERT INTO orders values(1,1,timestamp 'epoch' + '1 second'); INSERT INTO orders values(2,1,timestamp 'epoch' + '2 second'); INSERT INTO orders values(3,1,timestamp 'epoch' + '3 second'); INSERT INTO orders values(4,2,timestamp 'epoch' + '4 second'); INSERT INTO orders values(5,1,timestamp 'epoch' + '5 second'); INSERT INTO orders values(6,3,timestamp 'epoch' + '6 second'); INSERT INTO orders values(7,1,timestamp 'epoch' + '7 second'); INSERT INTO orders values(8,4,timestamp 'epoch' + '8 second'); INSERT INTO orders values(9,2,timestamp 'epoch' + '9 second'); -- Need a LATERAL query with a reference to the upper-level table and -- with a restriction on time -- Upper-level table constraint should be a constant in order to trigger -- creation of a one-time filter in the planner SELECT user_id, first_order_time, max_time FROM (SELECT user_id, min(time) AS first_order_time FROM orders GROUP BY user_id) o1 LEFT JOIN LATERAL (SELECT max(time) AS max_time FROM orders WHERE o1.user_id = '2' AND time > now()) o2 ON true ORDER BY user_id, first_order_time, max_time; user_id | first_order_time | max_time ---------+------------------------------+---------- 1 | Thu Jan 01 00:00:01 1970 PST | 2 | Thu Jan 01 00:00:04 1970 PST | 3 | Thu Jan 01 00:00:06 1970 PST | 4 | Thu Jan 01 00:00:08 1970 PST | SELECT user_id, first_order_time, max_time FROM (SELECT user_id, min(time) AS first_order_time FROM orders GROUP BY user_id) o1 LEFT JOIN LATERAL (SELECT max(time) AS max_time FROM orders WHERE o1.user_id = '2' AND time < now()) o2 ON true ORDER BY user_id, first_order_time, max_time; user_id | first_order_time | max_time ---------+------------------------------+------------------------------ 1 | Thu Jan 01 00:00:01 1970 PST | 2 | Thu Jan 01 00:00:04 1970 PST | Thu Jan 01 00:00:09 1970 PST 3 | Thu Jan 01 00:00:06 1970 PST | 4 | Thu Jan 01 00:00:08 1970 PST | -- Nested LATERALs SELECT user_id, first_order_time, time1, min_time FROM (SELECT user_id, min(time) AS first_order_time FROM orders GROUP BY user_id) o1 LEFT JOIN LATERAL (SELECT user_id as o2user_id, time AS time1 FROM orders WHERE o1.user_id = '2' AND time < now()) o2 ON true LEFT JOIN LATERAL (SELECT min(time) as min_time FROM orders WHERE o2.o2user_id = '1' AND time < now()) o3 ON true ORDER BY user_id, first_order_time, time1, min_time; user_id | first_order_time | time1 | min_time ---------+------------------------------+------------------------------+------------------------------ 1 | Thu Jan 01 00:00:01 1970 PST | | 2 | Thu Jan 01 00:00:04 1970 PST | Thu Jan 01 00:00:01 1970 PST | Thu Jan 01 00:00:01 1970 PST 2 | Thu Jan 01 00:00:04 1970 PST | Thu Jan 01 00:00:02 1970 PST | Thu Jan 01 00:00:01 1970 PST 2 | Thu Jan 01 00:00:04 1970 PST | Thu Jan 01 00:00:03 1970 PST | Thu Jan 01 00:00:01 1970 PST 2 | Thu Jan 01 00:00:04 1970 PST | Thu Jan 01 00:00:04 1970 PST | 2 | Thu Jan 01 00:00:04 1970 PST | Thu Jan 01 00:00:05 1970 PST | Thu Jan 01 00:00:01 1970 PST 2 | Thu Jan 01 00:00:04 1970 PST | Thu Jan 01 00:00:06 1970 PST | 2 | Thu Jan 01 00:00:04 1970 PST | Thu Jan 01 00:00:07 1970 PST | Thu Jan 01 00:00:01 1970 PST 2 | Thu Jan 01 00:00:04 1970 PST | Thu Jan 01 00:00:08 1970 PST | 2 | Thu Jan 01 00:00:04 1970 PST | Thu Jan 01 00:00:09 1970 PST | 3 | Thu Jan 01 00:00:06 1970 PST | | 4 | Thu Jan 01 00:00:08 1970 PST | | -- Cleanup DROP TABLE orders; ---- OUTER JOIN tests --- --github issue 2500 CREATE TABLE t1_timescale (a int, b int); CREATE TABLE t2 (a int, b int); SELECT create_hypertable('t1_timescale', 'a', chunk_time_interval=>1000); create_hypertable --------------------------- (3,public,t1_timescale,t) INSERT into t2 values (3, 3), (15 , 15); INSERT into t1_timescale select generate_series(5, 25, 1), 77; UPDATE t1_timescale SET b = 15 WHERE a = 15; SELECT * FROM t1_timescale FULL OUTER JOIN t2 on t1_timescale.b=t2.b and t2.b between 10 and 20 ORDER BY 1, 2, 3, 4; a | b | a | b ----+----+----+---- 5 | 77 | | 6 | 77 | | 7 | 77 | | 8 | 77 | | 9 | 77 | | 10 | 77 | | 11 | 77 | | 12 | 77 | | 13 | 77 | | 14 | 77 | | 15 | 15 | 15 | 15 16 | 77 | | 17 | 77 | | 18 | 77 | | 19 | 77 | | 20 | 77 | | 21 | 77 | | 22 | 77 | | 23 | 77 | | 24 | 77 | | 25 | 77 | | | | 3 | 3 SELECT * FROM t1_timescale LEFT OUTER JOIN t2 on t1_timescale.b=t2.b and t2.b between 10 and 20 WHERE t1_timescale.a=5 ORDER BY 1, 2, 3, 4; a | b | a | b ---+----+---+--- 5 | 77 | | SELECT * FROM t1_timescale RIGHT JOIN t2 on t1_timescale.b=t2.b and t2.b between 10 and 20 ORDER BY 1, 2, 3, 4; a | b | a | b ----+----+----+---- 15 | 15 | 15 | 15 | | 3 | 3 SELECT * FROM t1_timescale RIGHT JOIN t2 on t1_timescale.b=t2.b and t2.b between 10 and 20 WHERE t1_timescale.a=5 ORDER BY 1, 2, 3, 4; a | b | a | b ---+---+---+--- SELECT * FROM t1_timescale LEFT OUTER JOIN t2 on t1_timescale.a=t2.a and t2.b between 10 and 20 WHERE t1_timescale.a IN ( 10, 15, 20, 25) ORDER BY 1, 2, 3, 4; a | b | a | b ----+----+----+---- 10 | 77 | | 15 | 15 | 15 | 15 20 | 77 | | 25 | 77 | | SELECT * FROM t1_timescale RIGHT OUTER JOIN t2 on t1_timescale.a=t2.a and t2.b between 10 and 20 ORDER BY 1, 2, 3, 4; a | b | a | b ----+----+----+---- 15 | 15 | 15 | 15 | | 3 | 3 ================================================ FILE: test/expected/license.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER \set ECHO queries SHOW timescaledb.license; timescaledb.license --------------------- apache SELECT _timescaledb_functions.tsl_loaded(); tsl_loaded ------------ f SET timescaledb.license='apache'; ERROR: invalid value for parameter "timescaledb.license": "apache" DETAIL: Cannot change a license in a running session. HINT: Change the license in the configuration file or server command line. SET timescaledb.license='timescale'; ERROR: invalid value for parameter "timescaledb.license": "timescale" DETAIL: Cannot change a license in a running session. HINT: Change the license in the configuration file or server command line. SET timescaledb.license='something_else'; ERROR: invalid value for parameter "timescaledb.license": "something_else" DETAIL: Unrecognized license type. HINT: Supported license types are 'timescale' or 'apache'. SELECT locf(1); ERROR: function "locf" is not supported under the current "apache" license HINT: Upgrade your license to 'timescale' to use this free community feature. SELECT interpolate(1); ERROR: function "interpolate" is not supported under the current "apache" license HINT: Upgrade your license to 'timescale' to use this free community feature. SELECT time_bucket_gapfill(1,1,1,1); ERROR: function "time_bucket_gapfill" is not supported under the current "apache" license HINT: Upgrade your license to 'timescale' to use this free community feature. CREATE OR REPLACE FUNCTION custom_func(jobid int, args jsonb) RETURNS VOID AS $$ DECLARE BEGIN END; $$ LANGUAGE plpgsql; SELECT add_job('custom_func','1h', config:='{"type":"function"}'::jsonb); ERROR: function "add_job" is not supported under the current "apache" license HINT: Upgrade your license to 'timescale' to use this free community feature. DROP FUNCTION custom_func; CREATE TABLE metrics(time timestamptz NOT NULL, value float); SELECT create_hypertable('metrics', 'time'); create_hypertable ---------------------- (1,public,metrics,t) ALTER TABLE metrics SET (timescaledb.compress); ERROR: functionality not supported under the current "apache" license. Learn more at https://tsdb.co/pdbir1r3 HINT: To access all features and the best time-series experience, try out Timescale Cloud. INSERT INTO metrics VALUES ('2022-01-01 00:00:00', 1), ('2022-01-01 01:00:00', 2), ('2022-01-01 02:00:00', 3); CREATE MATERIALIZED VIEW metrics_hourly WITH (timescaledb.continuous) AS SELECT time_bucket(INTERVAL '1 hour', time) AS bucket, AVG(value), MAX(value), MIN(value) FROM metrics GROUP BY bucket WITH NO DATA; ERROR: functionality not supported under the current "apache" license. Learn more at https://tsdb.co/pdbir1r3 HINT: To access all features and the best time-series experience, try out Timescale Cloud. CREATE MATERIALIZED VIEW metrics_hourly AS SELECT time_bucket(INTERVAL '1 hour', time) AS bucket, AVG(value), MAX(value), MIN(value) FROM metrics GROUP BY bucket; CALL refresh_continuous_aggregate('metrics_hourly', NULL, NULL); ERROR: function "refresh_continuous_aggregate" is not supported under the current "apache" license HINT: Upgrade your license to 'timescale' to use this free community feature. DROP MATERIALIZED VIEW metrics_hourly; DROP TABLE metrics; ================================================ FILE: test/expected/loader-oss.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set TEST_DBNAME_2 :TEST_DBNAME _2 \c :TEST_DBNAME :ROLE_SUPERUSER CREATE DATABASE :"TEST_DBNAME_2"; DROP EXTENSION timescaledb; --no extension SELECT * FROM test.extension; List of installed extensions Name | Version | Schema | Description ---------+---------+------------+------------------------------ plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (1 row) SELECT 1; ?column? ---------- 1 (1 row) \c :TEST_DBNAME :ROLE_SUPERUSER CREATE EXTENSION timescaledb VERSION 'mock-1'; WARNING: mock init "mock-1" SELECT 1; WARNING: mock post_analyze_hook "mock-1" ?column? ---------- 1 (1 row) SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-1" List of installed extensions Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-1 | public | Enables scalable inserts and complex queries for time-series data (2 rows) CREATE EXTENSION IF NOT EXISTS timescaledb VERSION 'mock-1'; WARNING: mock post_analyze_hook "mock-1" NOTICE: extension "timescaledb" already exists, skipping CREATE EXTENSION IF NOT EXISTS timescaledb VERSION 'mock-2'; WARNING: mock post_analyze_hook "mock-1" NOTICE: extension "timescaledb" already exists, skipping DROP EXTENSION timescaledb; WARNING: mock post_analyze_hook "mock-1" \set ON_ERROR_STOP 0 --test that we cannot accidentally load another library version CREATE EXTENSION IF NOT EXISTS timescaledb VERSION 'mock-2'; ERROR: extension "timescaledb" has already been loaded with another version \set ON_ERROR_STOP 1 \c :TEST_DBNAME :ROLE_SUPERUSER --no extension SELECT * FROM test.extension; List of installed extensions Name | Version | Schema | Description ---------+---------+------------+------------------------------ plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (1 row) SELECT 1; ?column? ---------- 1 (1 row) CREATE EXTENSION timescaledb VERSION 'mock-1'; WARNING: mock init "mock-1" --same backend as create extension; SELECT 1; WARNING: mock post_analyze_hook "mock-1" ?column? ---------- 1 (1 row) SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-1" List of installed extensions Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-1 | public | Enables scalable inserts and complex queries for time-series data (2 rows) --start new backend; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT 1; WARNING: mock init "mock-1" WARNING: mock post_analyze_hook "mock-1" ?column? ---------- 1 (1 row) SELECT 1; WARNING: mock post_analyze_hook "mock-1" ?column? ---------- 1 (1 row) --test fn call after load SELECT mock_function(); WARNING: mock post_analyze_hook "mock-1" WARNING: mock function call "mock-1" mock_function --------------- (1 row) SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-1" List of installed extensions Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-1 | public | Enables scalable inserts and complex queries for time-series data (2 rows) \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --test fn call as first command SELECT mock_function(); WARNING: mock init "mock-1" WARNING: mock post_analyze_hook "mock-1" WARNING: mock function call "mock-1" mock_function --------------- (1 row) --use guc to prevent loading \c :TEST_DBNAME :ROLE_SUPERUSER SET timescaledb.disable_load = 'on'; SELECT 1; ?column? ---------- 1 (1 row) SELECT 1; ?column? ---------- 1 (1 row) SET timescaledb.disable_load = 'off'; SELECT 1; WARNING: mock init "mock-1" WARNING: mock post_analyze_hook "mock-1" ?column? ---------- 1 (1 row) \set ON_ERROR_STOP 0 SET timescaledb.disable_load = 'not bool'; WARNING: mock post_analyze_hook "mock-1" ERROR: parameter "timescaledb.disable_load" requires a Boolean value \set ON_ERROR_STOP 1 \c :TEST_DBNAME :ROLE_SUPERUSER RESET ALL; WARNING: mock init "mock-1" WARNING: mock post_analyze_hook "mock-1" SELECT 1; WARNING: mock post_analyze_hook "mock-1" ?column? ---------- 1 (1 row) \c :TEST_DBNAME :ROLE_SUPERUSER SET timescaledb.disable_load TO DEFAULT; SELECT 1; WARNING: mock init "mock-1" WARNING: mock post_analyze_hook "mock-1" ?column? ---------- 1 (1 row) \c :TEST_DBNAME :ROLE_SUPERUSER RESET timescaledb.disable_load; SELECT 1; WARNING: mock init "mock-1" WARNING: mock post_analyze_hook "mock-1" ?column? ---------- 1 (1 row) \c :TEST_DBNAME :ROLE_SUPERUSER SET timescaledb.other = 'on'; WARNING: mock init "mock-1" WARNING: mock post_analyze_hook "mock-1" SELECT 1; WARNING: mock post_analyze_hook "mock-1" ?column? ---------- 1 (1 row) \set ON_ERROR_STOP 0 --cannot update extension after .so of previous version already loaded ALTER EXTENSION timescaledb UPDATE TO 'mock-2'; ERROR: extension "timescaledb" cannot be updated after the old version has already been loaded \set ON_ERROR_STOP 1 \c :TEST_DBNAME_2 :ROLE_SUPERUSER SELECT * FROM test.extension; List of installed extensions Name | Version | Schema | Description ---------+---------+------------+------------------------------ plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (1 row) CREATE EXTENSION timescaledb VERSION 'mock-1'; WARNING: mock init "mock-1" SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-1" List of installed extensions Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-1 | public | Enables scalable inserts and complex queries for time-series data (2 rows) --start a new backend to update \c :TEST_DBNAME_2 :ROLE_SUPERUSER ALTER EXTENSION timescaledb UPDATE TO 'mock-2'; WARNING: mock init "mock-2" SELECT 1; WARNING: mock post_analyze_hook "mock-2" ?column? ---------- 1 (1 row) SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-2" List of installed extensions Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-2 | public | Enables scalable inserts and complex queries for time-series data (2 rows) --drop extension DROP EXTENSION timescaledb; WARNING: mock post_analyze_hook "mock-2" SELECT 1; ?column? ---------- 1 (1 row) SELECT * FROM test.extension; List of installed extensions Name | Version | Schema | Description ---------+---------+------------+------------------------------ plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (1 row) \c :TEST_DBNAME_2 :ROLE_SUPERUSER CREATE EXTENSION timescaledb VERSION 'mock-2'; WARNING: mock init "mock-2" SELECT 1; WARNING: mock post_analyze_hook "mock-2" ?column? ---------- 1 (1 row) SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-2" List of installed extensions Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-2 | public | Enables scalable inserts and complex queries for time-series data (2 rows) -- test db 1 still has old version \c :TEST_DBNAME :ROLE_SUPERUSER SELECT 1; WARNING: mock init "mock-1" WARNING: mock post_analyze_hook "mock-1" ?column? ---------- 1 (1 row) SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-1" List of installed extensions Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-1 | public | Enables scalable inserts and complex queries for time-series data (2 rows) --try a broken upgrade \c :TEST_DBNAME_2 :ROLE_SUPERUSER SELECT * FROM test.extension; WARNING: mock init "mock-2" WARNING: mock post_analyze_hook "mock-2" List of installed extensions Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-2 | public | Enables scalable inserts and complex queries for time-series data (2 rows) \set ON_ERROR_STOP 0 ALTER EXTENSION timescaledb UPDATE TO 'mock-3'; ERROR: extension "timescaledb" cannot be updated after the old version has already been loaded \set ON_ERROR_STOP 1 --should still be on mock-2 SELECT 1; WARNING: mock post_analyze_hook "mock-2" ?column? ---------- 1 (1 row) SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-2" List of installed extensions Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-2 | public | Enables scalable inserts and complex queries for time-series data (2 rows) --drop extension DROP EXTENSION timescaledb; WARNING: mock post_analyze_hook "mock-2" SELECT 1; ?column? ---------- 1 (1 row) SELECT * FROM test.extension; List of installed extensions Name | Version | Schema | Description ---------+---------+------------+------------------------------ plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (1 row) --create extension anew, only upgrade was broken \c :TEST_DBNAME_2 :ROLE_SUPERUSER CREATE EXTENSION timescaledb VERSION 'mock-3'; WARNING: mock init "mock-3" SELECT 1; WARNING: mock post_analyze_hook "mock-3" ?column? ---------- 1 (1 row) SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-3" List of installed extensions Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-3 | public | Enables scalable inserts and complex queries for time-series data (2 rows) DROP EXTENSION timescaledb; WARNING: mock post_analyze_hook "mock-3" SELECT 1; ?column? ---------- 1 (1 row) --mismatched version errors \c :TEST_DBNAME_2 :ROLE_SUPERUSER --mock-4 has mismatched versions, so the .so load should be fatal SELECT format($$\! utils/test_fatal_command.sh %1$s "CREATE EXTENSION timescaledb VERSION 'mock-4'"$$, :'TEST_DBNAME_2') as command_to_run \gset :command_to_run FATAL: extension "timescaledb" version mismatch: shared library version mock-4-mismatch; SQL version mock-4 server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection to server was lost --mock-4 not installed. SELECT * FROM test.extension; List of installed extensions Name | Version | Schema | Description ---------+---------+------------+------------------------------ plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (1 row) \c :TEST_DBNAME_2 :ROLE_SUPERUSER --broken version and drop CREATE EXTENSION timescaledb VERSION 'mock-broken'; WARNING: mock init "mock-broken" \set ON_ERROR_STOP 0 --intentional broken version SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-broken" ERROR: mock broken "mock-broken" SELECT 1; WARNING: mock post_analyze_hook "mock-broken" ERROR: mock broken "mock-broken" SELECT 1; WARNING: mock post_analyze_hook "mock-broken" ERROR: mock broken "mock-broken" --cannot drop extension; already loaded broken version DROP EXTENSION timescaledb; WARNING: mock post_analyze_hook "mock-broken" ERROR: mock broken "mock-broken" \set ON_ERROR_STOP 1 \c :TEST_DBNAME_2 :ROLE_SUPERUSER --can drop extension now. Since drop first command. DROP EXTENSION timescaledb; SELECT * FROM test.extension; List of installed extensions Name | Version | Schema | Description ---------+---------+------------+------------------------------ plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (1 row) --broken version and update to fixed \c :TEST_DBNAME_2 :ROLE_SUPERUSER CREATE EXTENSION timescaledb VERSION 'mock-broken'; WARNING: mock init "mock-broken" \set ON_ERROR_STOP 0 --intentional broken version SELECT 1; WARNING: mock post_analyze_hook "mock-broken" ERROR: mock broken "mock-broken" --cannot update extension; already loaded bad version ALTER EXTENSION timescaledb UPDATE TO 'mock-5'; ERROR: extension "timescaledb" cannot be updated after the old version has already been loaded \set ON_ERROR_STOP 1 \c :TEST_DBNAME_2 :ROLE_SUPERUSER --can update extension now. ALTER EXTENSION timescaledb UPDATE TO 'mock-5'; WARNING: mock init "mock-5" SELECT 1; WARNING: mock post_analyze_hook "mock-5" ?column? ---------- 1 (1 row) SELECT mock_function(); WARNING: mock post_analyze_hook "mock-5" WARNING: mock function call "mock-5" mock_function --------------- (1 row) \c :TEST_DBNAME_2 :ROLE_SUPERUSER ALTER EXTENSION timescaledb UPDATE TO 'mock-6'; WARNING: mock init "mock-6" --The mock-5->mock_6 upgrade is intentionally broken. --The mock_function was never changed to point to mock-6 in the update script. --Thus mock_function is defined incorrectly to point to the mock-5.so --This will now be a FATAL error. SELECT format($$\! utils/test_fatal_command.sh %1$s "SELECT mock_function()"$$, :'TEST_DBNAME_2') as command_to_run \gset WARNING: mock post_analyze_hook "mock-6" :command_to_run WARNING: mock init "mock-6" WARNING: mock post_analyze_hook "mock-6" FATAL: extension "timescaledb" version mismatch: shared library version mock-5; SQL version mock-6 server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection to server was lost SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-6" List of installed extensions Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-6 | public | Enables scalable inserts and complex queries for time-series data (2 rows) --TEST: create extension when old .so already loaded \c :TEST_DBNAME :ROLE_SUPERUSER --force load of extension with (SELECT * FROM test.extension;) SELECT * FROM test.extension; WARNING: mock init "mock-1" WARNING: mock post_analyze_hook "mock-1" List of installed extensions Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-1 | public | Enables scalable inserts and complex queries for time-series data (2 rows) DROP EXTENSION timescaledb; WARNING: mock post_analyze_hook "mock-1" SELECT * FROM test.extension; List of installed extensions Name | Version | Schema | Description ---------+---------+------------+------------------------------ plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (1 row) \set ON_ERROR_STOP 0 CREATE EXTENSION timescaledb VERSION 'mock-2'; ERROR: extension "timescaledb" has already been loaded with another version \set ON_ERROR_STOP 1 SELECT * FROM test.extension; List of installed extensions Name | Version | Schema | Description ---------+---------+------------+------------------------------ plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language (1 row) --can create in a new session. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE EXTENSION timescaledb VERSION 'mock-2'; WARNING: mock init "mock-2" SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-2" List of installed extensions Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-2 | public | Enables scalable inserts and complex queries for time-series data (2 rows) --make sure parallel workers started after a 'DISCARD ALL' work CREATE TABLE test (i int, j double precision); WARNING: mock post_analyze_hook "mock-2" INSERT INTO test SELECT x, x+0.1 FROM generate_series(1,100) AS x; WARNING: mock post_analyze_hook "mock-2" DISCARD ALL; WARNING: mock post_analyze_hook "mock-2" SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false); WARNING: mock post_analyze_hook "mock-2" set_config ------------ on (1 row) SET max_parallel_workers_per_gather = 1; WARNING: mock post_analyze_hook "mock-2" SELECT count(*) FROM test; WARNING: mock post_analyze_hook "mock-2" WARNING: mock init "mock-2" count ------- 100 (1 row) -- clean up additional database \c :TEST_DBNAME :ROLE_SUPERUSER DROP DATABASE :"TEST_DBNAME_2" WITH (FORCE); WARNING: mock init "mock-2" WARNING: mock post_analyze_hook "mock-2" ================================================ FILE: test/expected/loader-tsl.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set TEST_DBNAME_2 :TEST_DBNAME _2 \c :TEST_DBNAME :ROLE_SUPERUSER CREATE DATABASE :"TEST_DBNAME_2"; DROP EXTENSION timescaledb; --no extension SELECT * FROM test.extension; Name | Version | Schema | Description ---------+---------+------------+------------------------------ plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language SELECT 1; ?column? ---------- 1 \c :TEST_DBNAME :ROLE_SUPERUSER CREATE EXTENSION timescaledb VERSION 'mock-1'; WARNING: mock init "mock-1" SELECT 1; WARNING: mock post_analyze_hook "mock-1" ?column? ---------- 1 SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-1" Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-1 | public | Enables scalable inserts and complex queries for time-series data CREATE EXTENSION IF NOT EXISTS timescaledb VERSION 'mock-1'; WARNING: mock post_analyze_hook "mock-1" NOTICE: extension "timescaledb" already exists, skipping CREATE EXTENSION IF NOT EXISTS timescaledb VERSION 'mock-2'; WARNING: mock post_analyze_hook "mock-1" NOTICE: extension "timescaledb" already exists, skipping DROP EXTENSION timescaledb; WARNING: mock post_analyze_hook "mock-1" \set ON_ERROR_STOP 0 --test that we cannot accidentally load another library version CREATE EXTENSION IF NOT EXISTS timescaledb VERSION 'mock-2'; ERROR: extension "timescaledb" has already been loaded with another version \set ON_ERROR_STOP 1 \c :TEST_DBNAME :ROLE_SUPERUSER --no extension SELECT * FROM test.extension; Name | Version | Schema | Description ---------+---------+------------+------------------------------ plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language SELECT 1; ?column? ---------- 1 CREATE EXTENSION timescaledb VERSION 'mock-1'; WARNING: mock init "mock-1" --same backend as create extension; SELECT 1; WARNING: mock post_analyze_hook "mock-1" ?column? ---------- 1 SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-1" Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-1 | public | Enables scalable inserts and complex queries for time-series data --start new backend; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT 1; WARNING: mock init "mock-1" WARNING: mock post_analyze_hook "mock-1" ?column? ---------- 1 SELECT 1; WARNING: mock post_analyze_hook "mock-1" ?column? ---------- 1 --test fn call after load SELECT mock_function(); WARNING: mock post_analyze_hook "mock-1" WARNING: mock function call "mock-1" mock_function --------------- SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-1" Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-1 | public | Enables scalable inserts and complex queries for time-series data \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --test fn call as first command SELECT mock_function(); WARNING: mock init "mock-1" WARNING: mock post_analyze_hook "mock-1" WARNING: mock function call "mock-1" mock_function --------------- --use guc to prevent loading \c :TEST_DBNAME :ROLE_SUPERUSER SET timescaledb.disable_load = 'on'; SELECT 1; ?column? ---------- 1 SELECT 1; ?column? ---------- 1 SET timescaledb.disable_load = 'off'; SELECT 1; WARNING: mock init "mock-1" WARNING: mock post_analyze_hook "mock-1" ?column? ---------- 1 \set ON_ERROR_STOP 0 SET timescaledb.disable_load = 'not bool'; WARNING: mock post_analyze_hook "mock-1" ERROR: parameter "timescaledb.disable_load" requires a Boolean value \set ON_ERROR_STOP 1 \c :TEST_DBNAME :ROLE_SUPERUSER RESET ALL; WARNING: mock init "mock-1" WARNING: mock post_analyze_hook "mock-1" SELECT 1; WARNING: mock post_analyze_hook "mock-1" ?column? ---------- 1 \c :TEST_DBNAME :ROLE_SUPERUSER SET timescaledb.disable_load TO DEFAULT; SELECT 1; WARNING: mock init "mock-1" WARNING: mock post_analyze_hook "mock-1" ?column? ---------- 1 \c :TEST_DBNAME :ROLE_SUPERUSER RESET timescaledb.disable_load; SELECT 1; WARNING: mock init "mock-1" WARNING: mock post_analyze_hook "mock-1" ?column? ---------- 1 \c :TEST_DBNAME :ROLE_SUPERUSER SET timescaledb.other = 'on'; WARNING: mock init "mock-1" WARNING: mock post_analyze_hook "mock-1" SELECT 1; WARNING: mock post_analyze_hook "mock-1" ?column? ---------- 1 \set ON_ERROR_STOP 0 --cannot update extension after .so of previous version already loaded ALTER EXTENSION timescaledb UPDATE TO 'mock-2'; ERROR: extension "timescaledb" cannot be updated after the old version has already been loaded \set ON_ERROR_STOP 1 \c :TEST_DBNAME_2 :ROLE_SUPERUSER SELECT * FROM test.extension; Name | Version | Schema | Description ---------+---------+------------+------------------------------ plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language CREATE EXTENSION timescaledb VERSION 'mock-1'; WARNING: mock init "mock-1" SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-1" Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-1 | public | Enables scalable inserts and complex queries for time-series data --start a new backend to update \c :TEST_DBNAME_2 :ROLE_SUPERUSER ALTER EXTENSION timescaledb UPDATE TO 'mock-2'; WARNING: mock init "mock-2" SELECT 1; WARNING: mock post_analyze_hook "mock-2" ?column? ---------- 1 SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-2" Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-2 | public | Enables scalable inserts and complex queries for time-series data --drop extension DROP EXTENSION timescaledb; WARNING: mock post_analyze_hook "mock-2" SELECT 1; ?column? ---------- 1 SELECT * FROM test.extension; Name | Version | Schema | Description ---------+---------+------------+------------------------------ plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language \c :TEST_DBNAME_2 :ROLE_SUPERUSER CREATE EXTENSION timescaledb VERSION 'mock-2'; WARNING: mock init "mock-2" SELECT 1; WARNING: mock post_analyze_hook "mock-2" ?column? ---------- 1 SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-2" Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-2 | public | Enables scalable inserts and complex queries for time-series data -- test db 1 still has old version \c :TEST_DBNAME :ROLE_SUPERUSER SELECT 1; WARNING: mock init "mock-1" WARNING: mock post_analyze_hook "mock-1" ?column? ---------- 1 SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-1" Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-1 | public | Enables scalable inserts and complex queries for time-series data --try a broken upgrade \c :TEST_DBNAME_2 :ROLE_SUPERUSER SELECT * FROM test.extension; WARNING: mock init "mock-2" WARNING: mock post_analyze_hook "mock-2" Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-2 | public | Enables scalable inserts and complex queries for time-series data \set ON_ERROR_STOP 0 ALTER EXTENSION timescaledb UPDATE TO 'mock-3'; ERROR: extension "timescaledb" cannot be updated after the old version has already been loaded \set ON_ERROR_STOP 1 --should still be on mock-2 SELECT 1; WARNING: mock post_analyze_hook "mock-2" ?column? ---------- 1 SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-2" Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-2 | public | Enables scalable inserts and complex queries for time-series data --drop extension DROP EXTENSION timescaledb; WARNING: mock post_analyze_hook "mock-2" SELECT 1; ?column? ---------- 1 SELECT * FROM test.extension; Name | Version | Schema | Description ---------+---------+------------+------------------------------ plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language --create extension anew, only upgrade was broken \c :TEST_DBNAME_2 :ROLE_SUPERUSER CREATE EXTENSION timescaledb VERSION 'mock-3'; WARNING: mock init "mock-3" SELECT 1; WARNING: mock post_analyze_hook "mock-3" ?column? ---------- 1 SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-3" Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-3 | public | Enables scalable inserts and complex queries for time-series data DROP EXTENSION timescaledb; WARNING: mock post_analyze_hook "mock-3" SELECT 1; ?column? ---------- 1 --mismatched version errors \c :TEST_DBNAME_2 :ROLE_SUPERUSER --mock-4 has mismatched versions, so the .so load should be fatal SELECT format($$\! utils/test_fatal_command.sh %1$s "CREATE EXTENSION timescaledb VERSION 'mock-4'"$$, :'TEST_DBNAME_2') as command_to_run \gset :command_to_run FATAL: extension "timescaledb" version mismatch: shared library version mock-4-mismatch; SQL version mock-4 server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection to server was lost --mock-4 not installed. SELECT * FROM test.extension; Name | Version | Schema | Description ---------+---------+------------+------------------------------ plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language \c :TEST_DBNAME_2 :ROLE_SUPERUSER --broken version and drop CREATE EXTENSION timescaledb VERSION 'mock-broken'; WARNING: mock init "mock-broken" \set ON_ERROR_STOP 0 --intentional broken version SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-broken" ERROR: mock broken "mock-broken" SELECT 1; WARNING: mock post_analyze_hook "mock-broken" ERROR: mock broken "mock-broken" SELECT 1; WARNING: mock post_analyze_hook "mock-broken" ERROR: mock broken "mock-broken" --cannot drop extension; already loaded broken version DROP EXTENSION timescaledb; WARNING: mock post_analyze_hook "mock-broken" ERROR: mock broken "mock-broken" \set ON_ERROR_STOP 1 \c :TEST_DBNAME_2 :ROLE_SUPERUSER --can drop extension now. Since drop first command. DROP EXTENSION timescaledb; SELECT * FROM test.extension; Name | Version | Schema | Description ---------+---------+------------+------------------------------ plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language --broken version and update to fixed \c :TEST_DBNAME_2 :ROLE_SUPERUSER CREATE EXTENSION timescaledb VERSION 'mock-broken'; WARNING: mock init "mock-broken" \set ON_ERROR_STOP 0 --intentional broken version SELECT 1; WARNING: mock post_analyze_hook "mock-broken" ERROR: mock broken "mock-broken" --cannot update extension; already loaded bad version ALTER EXTENSION timescaledb UPDATE TO 'mock-5'; ERROR: extension "timescaledb" cannot be updated after the old version has already been loaded \set ON_ERROR_STOP 1 \c :TEST_DBNAME_2 :ROLE_SUPERUSER --can update extension now. ALTER EXTENSION timescaledb UPDATE TO 'mock-5'; WARNING: mock init "mock-5" SELECT 1; WARNING: mock post_analyze_hook "mock-5" ?column? ---------- 1 SELECT mock_function(); WARNING: mock post_analyze_hook "mock-5" WARNING: mock function call "mock-5" mock_function --------------- \c :TEST_DBNAME_2 :ROLE_SUPERUSER ALTER EXTENSION timescaledb UPDATE TO 'mock-6'; WARNING: mock init "mock-6" --The mock-5->mock_6 upgrade is intentionally broken. --The mock_function was never changed to point to mock-6 in the update script. --Thus mock_function is defined incorrectly to point to the mock-5.so --This will now be a FATAL error. SELECT format($$\! utils/test_fatal_command.sh %1$s "SELECT mock_function()"$$, :'TEST_DBNAME_2') as command_to_run \gset WARNING: mock post_analyze_hook "mock-6" :command_to_run WARNING: mock init "mock-6" WARNING: mock post_analyze_hook "mock-6" FATAL: extension "timescaledb" version mismatch: shared library version mock-5; SQL version mock-6 server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request. connection to server was lost SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-6" Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-6 | public | Enables scalable inserts and complex queries for time-series data --TEST: create extension when old .so already loaded \c :TEST_DBNAME :ROLE_SUPERUSER SELECT * FROM test.extension; WARNING: mock init "mock-1" WARNING: mock post_analyze_hook "mock-1" Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-1 | public | Enables scalable inserts and complex queries for time-series data DROP EXTENSION timescaledb; WARNING: mock post_analyze_hook "mock-1" SELECT * FROM test.extension; Name | Version | Schema | Description ---------+---------+------------+------------------------------ plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language \set ON_ERROR_STOP 0 CREATE EXTENSION timescaledb VERSION 'mock-2'; ERROR: extension "timescaledb" has already been loaded with another version \set ON_ERROR_STOP 1 SELECT * FROM test.extension; Name | Version | Schema | Description ---------+---------+------------+------------------------------ plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language --can create in a new session. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE EXTENSION timescaledb VERSION 'mock-2'; WARNING: mock init "mock-2" SELECT * FROM test.extension; WARNING: mock post_analyze_hook "mock-2" Name | Version | Schema | Description -------------+---------+------------+------------------------------------------------------------------- plpgsql | 1.0 | pg_catalog | PL/pgSQL procedural language timescaledb | mock-2 | public | Enables scalable inserts and complex queries for time-series data --make sure parallel workers started after a 'DISCARD ALL' work CREATE TABLE test (i int, j double precision); WARNING: mock post_analyze_hook "mock-2" INSERT INTO test SELECT x, x+0.1 FROM generate_series(1,100) AS x; WARNING: mock post_analyze_hook "mock-2" DISCARD ALL; WARNING: mock post_analyze_hook "mock-2" SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false); WARNING: mock post_analyze_hook "mock-2" set_config ------------ on SET max_parallel_workers_per_gather = 1; WARNING: mock post_analyze_hook "mock-2" SELECT count(*) FROM test; WARNING: mock post_analyze_hook "mock-2" WARNING: mock init "mock-2" count ------- 100 CREATE EXTENSION timescaledb_osm VERSION 'mock-1'; WARNING: mock post_analyze_hook "mock-2" WARNING: mock post_analyze_hook "mock-2" WARNING: OSM-mock-1 _PG_init WARNING: got lwlock osm lock WARNING: mock post_analyze_hook "mock-2" WARNING: mock post_analyze_hook "mock-2" -- Test that OSM process utility hook works: it should see this DROP TABLE. DROP TABLE test; WARNING: mock post_analyze_hook "mock-2" NOTICE: OSM-mock-1 got DROP TABLE 'test' -- clean up additional database \c :TEST_DBNAME :ROLE_SUPERUSER DROP DATABASE :"TEST_DBNAME_2" WITH (FORCE); WARNING: mock init "mock-2" WARNING: mock post_analyze_hook "mock-2" ================================================ FILE: test/expected/merge.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER -- Create target table with location and temperature CREATE TABLE target ( time TIMESTAMPTZ NOT NULL, location SMALLINT NOT NULL, temperature DOUBLE PRECISION NULL, to_be_dropped text ); SELECT create_hypertable( 'target', 'time', chunk_time_interval => INTERVAL '5 seconds'); create_hypertable --------------------- (1,public,target,t) INSERT INTO target SELECT time, location, 14 as temperature FROM generate_series( '2021-01-01 00:00:00', '2021-01-01 00:00:09', INTERVAL '5 seconds' ) as time, generate_series(1,4) as location; -- This makes sure we have one column with attisdropped and one column -- with atthasmissing set to true. These two cases can cause problems -- with chunk dispatch execution when merging using a when-clause with -- inserts. Unfortunately they are hard to trigger, so this is not a -- definitive test. ALTER TABLE target DROP COLUMN to_be_dropped; ALTER TABLE target ADD COLUMN val text default 'string -'; -- Create source table with location and temperature CREATE TABLE source ( time TIMESTAMPTZ NOT NULL, location SMALLINT NOT NULL, temperature DOUBLE PRECISION NULL ); SELECT create_hypertable( 'source', 'time', chunk_time_interval => INTERVAL '5 seconds'); create_hypertable --------------------- (2,public,source,t) -- Generate data that overlaps with target table INSERT INTO source SELECT time, location, 80 as temperature FROM generate_series( '2021-01-01 00:00:05', '2021-01-01 00:00:14', INTERVAL '5 seconds' ) as time, generate_series(1,4) as location; -- Print table/rows/num of chunks select * from target order by time, location asc; time | location | temperature | val ------------------------------+----------+-------------+---------- Fri Jan 01 00:00:00 2021 PST | 1 | 14 | string - Fri Jan 01 00:00:00 2021 PST | 2 | 14 | string - Fri Jan 01 00:00:00 2021 PST | 3 | 14 | string - Fri Jan 01 00:00:00 2021 PST | 4 | 14 | string - Fri Jan 01 00:00:05 2021 PST | 1 | 14 | string - Fri Jan 01 00:00:05 2021 PST | 2 | 14 | string - Fri Jan 01 00:00:05 2021 PST | 3 | 14 | string - Fri Jan 01 00:00:05 2021 PST | 4 | 14 | string - select * from source order by time, location asc; time | location | temperature ------------------------------+----------+------------- Fri Jan 01 00:00:05 2021 PST | 1 | 80 Fri Jan 01 00:00:05 2021 PST | 2 | 80 Fri Jan 01 00:00:05 2021 PST | 3 | 80 Fri Jan 01 00:00:05 2021 PST | 4 | 80 Fri Jan 01 00:00:10 2021 PST | 1 | 80 Fri Jan 01 00:00:10 2021 PST | 2 | 80 Fri Jan 01 00:00:10 2021 PST | 3 | 80 Fri Jan 01 00:00:10 2021 PST | 4 | 80 -- CREATE normal PostgreSQL tables CREATE TABLE target_pg AS SELECT * FROM target; CREATE TABLE source_pg AS SELECT * FROM source; -- Merge UPDATE matched rows for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN UPDATE SET temperature = (t.temperature + s.temperature)/2, val = val || ' UPDATED BY MERGE'; -- Merge UPDATE matched rows for hypertables MERGE INTO target t USING source s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN UPDATE SET temperature = (t.temperature + s.temperature)/2, val = val || ' UPDATED BY MERGE'; -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; result -------- same -- Merge DELETE matched rows for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN DELETE; -- Merge DELETE matched rows for hypertables MERGE INTO target t USING source s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN DELETE; -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; result -------- same -- clean up tables DELETE FROM target_pg; DELETE FROM target; DELETE FROM source_pg; DELETE FROM source; INSERT INTO target SELECT time, location, 14 as temperature FROM generate_series( '2021-01-01 00:00:00', '2021-01-01 00:00:09', INTERVAL '5 seconds' ) as time, generate_series(1,4) as location; INSERT INTO source SELECT time, location, 80 as temperature FROM generate_series( '2021-01-01 00:00:05', '2021-01-01 00:00:14', INTERVAL '5 seconds' ) as time, generate_series(1,4) as location; INSERT INTO target_pg SELECT * FROM target; INSERT INTO source_pg SELECT * FROM source; -- Merge UPDATE matched rows and INSERT new row for unmatched rows for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN UPDATE SET temperature = (t.temperature + s.temperature)/2, val = val || ' UPDATED BY MERGE' WHEN NOT MATCHED THEN INSERT (time, location, temperature, val) VALUES (s.time, s.location, s.temperature, 'string - INSERTED BY MERGE'); -- Merge UPDATE matched rows and INSERT new row for unmatched rows for hypertables MERGE INTO target t USING source s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN UPDATE SET temperature = (t.temperature + s.temperature)/2, val = val || ' UPDATED BY MERGE' WHEN NOT MATCHED THEN INSERT (time, location, temperature, val) VALUES (s.time, s.location, s.temperature, 'string - INSERTED BY MERGE'); -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; result -------- same -- Merge INSERT with constant literals for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.location = 1234 WHEN NOT MATCHED THEN INSERT VALUES ('2021-11-01 00:00:05'::timestamp with time zone, 5, 210, 'string - INSERTED BY MERGE'); -- Merge INSERT with constant literals for hypertables MERGE INTO target t USING source s ON t.location = 1234 WHEN NOT MATCHED THEN INSERT VALUES ('2021-11-01 00:00:05'::timestamp with time zone, 5, 210, 'string - INSERTED BY MERGE'); -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; result -------- same -- Merge with INSERT/DELETE/UPDATE on PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.location = s.location WHEN MATCHED AND t.location = 560076 THEN UPDATE SET temperature = (t.temperature + s.temperature) * 2, val = val || ' UPDATED BY MERGE' WHEN MATCHED AND t.location = 560083 THEN DELETE WHEN NOT MATCHED THEN INSERT (time, location, temperature, val) VALUES (s.time, s.location, s.temperature, 'string - INSERTED BY MERGE'); -- Merge with INSERT/DELETE/UPDATE on hypertables MERGE INTO target t USING source s ON t.time = s.time AND t.location = s.location WHEN MATCHED AND t.location = 560076 THEN UPDATE SET temperature = (t.temperature + s.temperature) * 2, val = val || ' UPDATED BY MERGE' WHEN MATCHED AND t.location = 560083 THEN DELETE WHEN NOT MATCHED THEN INSERT (time, location, temperature, val) VALUES (s.time, s.location, s.temperature, 'string - INSERTED BY MERGE'); -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; result -------- same -- Merge with Subqueries on PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.location > (SELECT count(*) FROM source_pg) WHEN MATCHED AND t.temperature = 23 THEN UPDATE SET temperature = (SELECT count(*) FROM target_pg) * 2, val = val || ' UPDATED BY MERGE' WHEN MATCHED AND t.temperature = 47 THEN DELETE WHEN NOT MATCHED THEN INSERT (time, location, temperature, val) VALUES (s.time, s.location, s.temperature, 'SUBQUERY string - INSERTED BY MERGE'); -- Merge with Subqueries on hypertables MERGE INTO target t USING source s ON t.time = s.time AND t.location > (SELECT count(*) FROM source) WHEN MATCHED AND t.temperature = 23 THEN UPDATE SET temperature = (SELECT count(*) FROM target) * 2, val = val || ' UPDATED BY MERGE' WHEN MATCHED AND t.temperature = 47 THEN DELETE WHEN NOT MATCHED THEN INSERT (time, location, temperature, val) VALUES (s.time, s.location, s.temperature, 'SUBQUERY string - INSERTED BY MERGE'); -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; result -------- same -- clean up tables DELETE FROM target_pg; DELETE FROM target; DELETE FROM source_pg; DELETE FROM source; -- TEST with target as hypertable and source as normal PG table INSERT INTO target SELECT time, location, 14 as temperature FROM generate_series( '2021-01-01 00:00:00', '2021-01-01 00:00:09', INTERVAL '5 seconds' ) as time, generate_series(1,4) as location; INSERT INTO source SELECT time, location, 80 as temperature FROM generate_series( '2021-01-01 00:00:05', '2021-01-01 00:00:14', INTERVAL '5 seconds' ) as time, generate_series(1,4) as location; INSERT INTO target_pg SELECT * FROM target; INSERT INTO source_pg SELECT * FROM source; -- Merge UPDATE matched rows for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN UPDATE SET temperature = (t.temperature + s.temperature)/2, val = val || ' UPDATED BY MERGE'; -- Merge UPDATE with target as hypertables and source as normal PG tables MERGE INTO target t USING source_pg s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN UPDATE SET temperature = (t.temperature + s.temperature)/2, val = val || ' UPDATED BY MERGE'; -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; result -------- same -- Merge DELETE matched rows for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN DELETE; -- Merge DELETE with target as hypertables and source as normal PG tables MERGE INTO target t USING source_pg s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN DELETE; -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; result -------- same -- Merge INSERT with constant literals for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.location = 1234 WHEN NOT MATCHED THEN INSERT VALUES ('2021-11-01 00:00:05'::timestamp with time zone, 5, 210, 'string - INSERTED BY MERGE'); -- Merge INSERT with constant literals for target as hypertables and source as normal PG tables MERGE INTO target t USING source s ON t.location = 1234 WHEN NOT MATCHED THEN INSERT VALUES ('2021-11-01 00:00:05'::timestamp with time zone, 5, 210, 'string - INSERTED BY MERGE'); -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; result -------- same -- Merge with INSERT/DELETE/UPDATE on PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.location = s.location WHEN MATCHED AND t.temperature = 23 THEN UPDATE SET temperature = (t.temperature + s.temperature) * 2, val = val || ' UPDATED BY MERGE' WHEN MATCHED AND t.temperature = 47 THEN DELETE WHEN NOT MATCHED THEN INSERT (time, location, temperature, val) VALUES (s.time, s.location, s.temperature, 'string - INSERTED BY MERGE'); -- Merge with INSERT/DELETE/UPDATE on target as hypertables and source as normal PG tables MERGE INTO target t USING source s ON t.time = s.time AND t.location = s.location WHEN MATCHED AND t.temperature = 23 THEN UPDATE SET temperature = (t.temperature + s.temperature) * 2, val = val || ' UPDATED BY MERGE' WHEN MATCHED AND t.temperature = 47 THEN DELETE WHEN NOT MATCHED THEN INSERT (time, location, temperature, val) VALUES (s.time, s.location, s.temperature, 'string - INSERTED BY MERGE'); -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; result -------- same DROP TABLE target_pg CASCADE; DROP TABLE target CASCADE; DROP TABLE source_pg CASCADE; DROP TABLE source CASCADE; -- test MERGE with source being a PARTITION table CREATE TABLE source_pg( id INT NOT NULL, dev INT NOT NULL, value INT, CONSTRAINT cstr_source_pky PRIMARY KEY (id) ) PARTITION BY LIST (id); CREATE TABLE source_1_2_3_4 PARTITION OF source_pg FOR VALUES IN (1,2,3,4); CREATE TABLE source_5_6_7_8 PARTITION OF source_pg FOR VALUES IN (5,6,7,8); INSERT INTO source_pg SELECT generate_series(1,8), 44,55; CREATE TABLE target ( ts TIMESTAMP WITH TIME ZONE NOT NULL, id INT NOT NULL, dev INT NOT NULL, FOREIGN KEY (id) REFERENCES source_pg(id) ON DELETE CASCADE ); SELECT create_hypertable( relation => 'target', time_column_name => 'ts' ); create_hypertable --------------------- (3,public,target,t) insert into target values ('2023-01-12 00:00:05'::timestamp with time zone, 1,2); insert into target values ('2023-01-12 00:00:10'::timestamp with time zone, 2,2); insert into target values ('2023-01-12 00:00:15'::timestamp with time zone, 3,2); insert into target values ('2023-01-12 00:00:20'::timestamp with time zone, 4,2); insert into target values ('2023-01-14 00:00:25'::timestamp with time zone, 5,2); insert into target values ('2023-01-14 00:00:30'::timestamp with time zone, 6,2); insert into target values ('2023-01-14 00:00:35'::timestamp with time zone, 7,2); insert into target values ('2023-01-14 00:00:40'::timestamp with time zone, 8,2); CREATE TABLE target_pg AS SELECT * FROM target; -- Merge UPDATE matched rows for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.id = s.id WHEN MATCHED THEN UPDATE SET dev = (t.dev + s.dev)/2; -- Merge UPDATE matched rows for hypertables MERGE INTO target t USING source_pg s ON t.id = s.id WHEN MATCHED THEN UPDATE SET dev = (t.dev + s.dev)/2; -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; result -------- same -- Merge DELETE matched rows for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.id = s.id WHEN MATCHED THEN DELETE; -- Merge DELETE matched rows for hypertables MERGE INTO target t USING source_pg s ON t.id = s.id WHEN MATCHED THEN DELETE; -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; result -------- same -- clean up tables DROP TABLE target_pg CASCADE; DROP TABLE target CASCADE; DROP TABLE source_pg CASCADE; -- test MERGE with hypertables with time and space partitions CREATE TABLE target ( filler_1 int, filler_2 int, filler_3 int, time timestamptz NOT NULL, device_id int, device_id_peer int, v0 int, v1 float, v2 float, v3 float ); SELECT create_hypertable ('target', 'time', 'device_id', 5); create_hypertable --------------------- (4,public,target,t) SELECT add_dimension('target', 'device_id_peer', 5); add_dimension ------------------------------------ (6,public,target,device_id_peer,t) SELECT add_dimension('target', 'v2', 5); add_dimension ------------------------ (7,public,target,v2,t) INSERT INTO target (time, device_id, device_id_peer, v0, v1, v2, v3) SELECT time, device_id, 0, device_id + 1, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-01 0:00:00+0'::timestamptz, '2000-01-05 23:55:00+0', '20m') gtime (time), generate_series(1, 2, 1) gdevice (device_id); CREATE TABLE source ( filler_1 int, filler_2 int, filler_3 int, time timestamptz NOT NULL, device_id int ); SELECT create_hypertable ('source', 'time', 'device_id', 3); create_hypertable --------------------- (5,public,source,t) INSERT INTO source (time, device_id, filler_2, filler_3, filler_1) SELECT time, device_id, device_id + 134, device_id + 209, device_id + 0.50127 FROM generate_series('2000-01-01 0:00:00+0'::timestamptz, '2000-01-05 23:55:00+0', '20m') gtime (time), generate_series(1, 5, 1) gdevice (device_id); -- create PG tables to compare PG target and hypertable target tables CREATE table target_pg as SELECT * FROM target; CREATE table source_pg as SELECT * FROM source; -- Merge UDPATE matched rows for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.device_id = s.device_id WHEN MATCHED THEN UPDATE SET filler_2 = s.filler_1 + 100; -- Merge UDPATE matched rows for space partitioned hypertables MERGE INTO target t USING source s ON t.time = s.time AND t.device_id = s.device_id WHEN MATCHED THEN UPDATE SET filler_2 = s.filler_1 + 100; SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; result -------- same -- Merge DELETE matched rows for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.device_id = s.device_id WHEN MATCHED THEN DELETE; -- Merge DELETE matched rows for space partitioned hypertables MERGE INTO target t USING source s ON t.time = s.time AND t.device_id = s.device_id WHEN MATCHED THEN DELETE; SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; result -------- same -- Merge INSERT matched rows for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.device_id = s.device_id WHEN NOT MATCHED THEN INSERT (filler_1, filler_2, filler_3, time, device_id, device_id_peer, v0, v1, v2, v3) VALUES (s.filler_1, s.filler_2, s.filler_3, s.time, s.device_id, s.device_id + 10, 1,2,3,4); -- Merge INSERT matched rows for space partitioned hypertables MERGE INTO target t USING source s ON t.time = s.time AND t.device_id = s.device_id WHEN NOT MATCHED THEN INSERT (filler_1, filler_2, filler_3, time, device_id, device_id_peer, v0, v1, v2, v3) VALUES (s.filler_1, s.filler_2, s.filler_3, s.time, s.device_id, s.device_id + 10, 1,2,3,4); SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; result -------- same -- Merge with INSERT/DELETE/UPDATE on PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.device_id = s.device_id WHEN MATCHED AND t.device_id_peer = 2 THEN UPDATE SET filler_2 = s.filler_1 + s.filler_2 + s.filler_3 + 100 WHEN MATCHED AND t.device_id_peer = 7 THEN DELETE WHEN NOT MATCHED THEN INSERT (filler_1, filler_2, filler_3, time, device_id, device_id_peer, v0, v1, v2, v3) VALUES (s.filler_1, s.filler_2, s.filler_3, s.time, s.device_id, s.device_id + 10, 1,2,3,4); -- Merge with INSERT/DELETE/UPDATE on space partitioned hypertables MERGE INTO target t USING source s ON t.time = s.time AND t.device_id = s.device_id WHEN MATCHED AND t.device_id_peer = 2 THEN UPDATE SET filler_2 = s.filler_1 + s.filler_2 + s.filler_3 + 100 WHEN MATCHED AND t.device_id_peer = 7 THEN DELETE WHEN NOT MATCHED THEN INSERT (filler_1, filler_2, filler_3, time, device_id, device_id_peer, v0, v1, v2, v3) VALUES (s.filler_1, s.filler_2, s.filler_3, s.time, s.device_id, s.device_id + 10, 1,2,3,4); SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; result -------- same -- clean up tables DROP TABLE target_pg CASCADE; DROP TABLE target CASCADE; DROP TABLE source_pg CASCADE; DROP TABLE source CASCADE; -- TEST with parition column place after similar data type column CREATE TABLE target ( filler_1 int, filler_2 int, filler_3 int, time timestamptz NOT NULL, device_id int, device_id_peer int, v0 int, v1 float, v2 float, v3 float, partition_column TIMESTAMPTZ NOT NULL ); SELECT create_hypertable ('target', 'partition_column'); create_hypertable --------------------- (6,public,target,t) INSERT INTO target (time, device_id, device_id_peer, v0, v1, v2, v3, partition_column) SELECT time, device_id, 0, device_id + 1, device_id + 2, device_id + 0.5, NULL, time + interval '10m' FROM generate_series('2000-01-01 0:00:00+0'::timestamptz, '2000-01-05 23:55:00+0', '20m') gtime (time), generate_series(1, 2, 1) gdevice (device_id); CREATE TABLE source ( filler_1 int, filler_2 int, filler_3 int, time timestamptz NOT NULL, device_id int ); SELECT create_hypertable ('source', 'time', 'device_id', 3); create_hypertable --------------------- (7,public,source,t) INSERT INTO source (time, device_id, filler_2, filler_3, filler_1) SELECT time, device_id, device_id + 134, device_id + 209, device_id + 0.50127 FROM generate_series('2000-01-01 0:00:00+0'::timestamptz, '2000-01-05 23:55:00+0', '20m') gtime (time), generate_series(1, 5, 1) gdevice (device_id); -- create PG tables to compare PG target and hypertable target tables CREATE table target_pg as SELECT * FROM target; MERGE INTO target_pg t USING source s ON t.time = s.time AND t.device_id = s.device_id WHEN NOT MATCHED THEN INSERT (time, device_id, device_id_peer, v0, v1, v2, v3, partition_column) VALUES ('2010-01-06 05:30:00+05:30', 23, 2, 11, 22, 33, 44, '2023-01-06 05:33:00+05:30'); MERGE INTO target t USING source s ON t.time = s.time AND t.device_id = s.device_id WHEN NOT MATCHED THEN INSERT (time, device_id, device_id_peer, v0, v1, v2, v3, partition_column) VALUES ('2010-01-06 05:30:00+05:30', 23, 2, 11, 22, 33, 44, '2023-01-06 05:33:00+05:30'); SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; result -------- same MERGE INTO target_pg t USING source s ON t.time = s.time AND t.device_id = s.device_id WHEN MATCHED THEN DELETE; MERGE INTO target t USING source s ON t.time = s.time AND t.device_id = s.device_id WHEN MATCHED THEN DELETE; SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; result -------- same MERGE INTO target_pg t USING source s ON t.time = s.time AND t.device_id = s.device_id WHEN NOT MATCHED THEN INSERT (filler_1, filler_2, filler_3, time, device_id, device_id_peer, v0, v1, v2, v3) VALUES (s.filler_1, s.filler_2, s.filler_3, s.time, s.device_id, s.device_id + 10, 1,2,3,4); -- time dimension column is NULL, this will report an null constraint violation error \set ON_ERROR_STOP 0 MERGE INTO target t USING source s ON t.time = s.time AND t.device_id = s.device_id WHEN NOT MATCHED THEN INSERT (filler_1, filler_2, filler_3, time, device_id, device_id_peer, v0, v1, v2, v3) VALUES (s.filler_1, s.filler_2, s.filler_3, s.time, s.device_id, s.device_id + 10, 1,2,3,4); ERROR: NULL value in column "partition_column" violates not-null constraint \set ON_ERROR_STOP 1 DROP TABLE target CASCADE; DROP TABLE target_pg CASCADE; DROP TABLE source CASCADE; -- TEST with target table have CHECK constraints CREATE TABLE target ( time TIMESTAMPTZ NOT NULL, location SMALLINT NOT NULL, temperature DOUBLE PRECISION NULL CHECK (temperature > 10), val text default 'string -' ); SELECT create_hypertable( 'target', 'time', chunk_time_interval => INTERVAL '5 seconds'); create_hypertable --------------------- (8,public,target,t) INSERT INTO target SELECT time, location, 14 as temperature FROM generate_series( '2021-01-01 00:00:00', '2021-01-01 00:00:09', INTERVAL '5 seconds' ) as time, generate_series(1,4) as location; -- Create source table with location and temperature CREATE TABLE source ( time TIMESTAMPTZ NOT NULL, location SMALLINT NOT NULL, temperature DOUBLE PRECISION NULL ); -- Generate data that overlaps with target table INSERT INTO source SELECT time, location, 80 as temperature FROM generate_series( '2021-01-01 00:00:05', '2021-01-01 00:00:14', INTERVAL '5 seconds' ) as time, generate_series(1,4) as location; -- CREATE normal PostgreSQL tables CREATE TABLE target_pg AS SELECT * FROM target; -- Merge UPDATE/DELETE with DO NOTHING on pg tables MERGE INTO target_pg t USING source s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN DO NOTHING WHEN NOT MATCHED THEN DO NOTHING; -- Merge UPDATE/DELETE with DO NOTHING on hypertable MERGE INTO target t USING source s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN DO NOTHING WHEN NOT MATCHED THEN DO NOTHING; -- Error cases for Merge \set ON_ERROR_STOP 0 -- Merge UPDATE should fail with check constraint violation MERGE INTO target_pg t USING source s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN UPDATE SET temperature = 8, val = val || ' UPDATED BY MERGE'; -- Merge UPDATE should fail with check constraint violation MERGE INTO target t USING source s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN UPDATE SET temperature = 8, val = val || ' UPDATED BY MERGE'; ERROR: new row for relation "_hyper_8_24_chunk" violates check constraint "target_temperature_check" -- Merge error with unreachable WHEN clause on pg tables MERGE INTO target_pg t USING source s ON t.time = s.time AND t.location != s.location WHEN MATCHED THEN UPDATE SET temperature = 8, val = val || ' UPDATED BY MERGE' WHEN MATCHED AND t.time < now() THEN DELETE WHEN NOT MATCHED THEN DO NOTHING; ERROR: unreachable WHEN clause specified after unconditional WHEN clause -- Merge error with unreachable WHEN clause on hypertable MERGE INTO target t USING source s ON t.time = s.time AND t.location != s.location WHEN MATCHED THEN UPDATE SET temperature = 8, val = val || ' UPDATED BY MERGE' WHEN MATCHED AND t.time < now() THEN DELETE WHEN NOT MATCHED THEN DO NOTHING; ERROR: unreachable WHEN clause specified after unconditional WHEN clause -- Merge error with unknown action in MERGE WHEN MATCHED clause on pg tables MERGE INTO target_pg t USING source s ON t.time = s.time AND t.location != s.location WHEN MATCHED THEN SELECT 1; ERROR: syntax error at or near "SELECT" at character 105 -- Merge error with unknown action in MERGE WHEN MATCHED clause on hypertable MERGE INTO target t USING source s ON t.time = s.time AND t.location != s.location WHEN MATCHED THEN SELECT 1; ERROR: syntax error at or near "SELECT" at character 102 -- Merge error cannot affect row a second time on pg tables MERGE INTO target_pg t USING source s ON t.location = s.location WHEN MATCHED THEN UPDATE SET temperature = 28, val = val || ' UPDATED BY MERGE'; ERROR: MERGE command cannot affect row a second time -- Merge error cannot affect row a second time on hypertable MERGE INTO target t USING source s ON t.location = s.location WHEN MATCHED THEN UPDATE SET temperature = 28, val = val || ' UPDATED BY MERGE'; ERROR: MERGE command cannot affect row a second time \set ON_ERROR_STOP 1 DROP TABLE target CASCADE; DROP TABLE target_pg CASCADE; DROP TABLE source CASCADE; -- TEST for PERMISSIONS CREATE USER priv_user; CREATE USER non_priv_user; CREATE TABLE target ( value DOUBLE PRECISION NOT NULL, time TIMESTAMPTZ NOT NULL ); SELECT table_name FROM create_hypertable( 'target'::regclass, 'time'::name, chunk_time_interval=>interval '8 hours', create_default_indexes=> false); table_name ------------ target SELECT '2022-10-10 14:33:44.1234+05:30' as start_date \gset INSERT INTO target (value, time) SELECT 1,t from generate_series(:'start_date'::timestamptz, :'start_date'::timestamptz + interval '1 day', '5m') t cross join generate_series(1,3) s; CREATE TABLE source ( time TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP NOT NULL, value DOUBLE PRECISION NOT NULL ); SELECT table_name FROM create_hypertable( 'source'::regclass, 'time'::name, chunk_time_interval=>interval '6 hours', create_default_indexes=> false); table_name ------------ source ALTER TABLE target OWNER TO priv_user; ALTER TABLE source OWNER TO priv_user; GRANT SELECT ON source TO non_priv_user; SET SESSION AUTHORIZATION non_priv_user; \set ON_ERROR_STOP 0 -- non_priv_user does not have UPDATE privilege on target table MERGE INTO target USING source ON target.time = source.time WHEN MATCHED THEN UPDATE SET value = 0; ERROR: permission denied for table target -- non_priv_user does not have DELETE privilege on target table MERGE INTO target USING source ON target.time = source.time WHEN MATCHED THEN DELETE; ERROR: permission denied for table target -- non_priv_user does not have INSERT privilege on target table MERGE INTO target USING source ON target.time = source.time WHEN NOT MATCHED THEN INSERT VALUES (10, '2023-01-15 00:00:10'::timestamp with time zone); ERROR: permission denied for table target \set ON_ERROR_STOP 1 RESET SESSION AUTHORIZATION; DROP TABLE target; DROP TABLE source; DROP USER priv_user; DROP USER non_priv_user; ================================================ FILE: test/expected/metadata.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION _timescaledb_internal.test_uuid() RETURNS UUID AS :MODULE_PATHNAME, 'ts_test_uuid' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_exported_uuid() RETURNS UUID AS :MODULE_PATHNAME, 'ts_test_exported_uuid' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_install_timestamp() RETURNS TIMESTAMPTZ AS :MODULE_PATHNAME, 'ts_test_install_timestamp' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; INSERT INTO _timescaledb_catalog.metadata (key, value, include_in_telemetry) SELECT 'metadata_test', 'FOO', TRUE; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- uuid and install_timestamp should already be in the table before we generate SELECT COUNT(*) from _timescaledb_catalog.metadata; count ------- 4 SELECT _timescaledb_internal.test_uuid() as uuid_1 \gset SELECT _timescaledb_internal.test_exported_uuid() as uuid_ex_1 \gset SELECT _timescaledb_internal.test_install_timestamp() as timestamp_1 \gset -- Check that there is exactly 1 UUID row SELECT COUNT(*) from _timescaledb_catalog.metadata where key='uuid'; count ------- 1 -- Check that exported_uuid and timestamp are also generated SELECT COUNT(*) from _timescaledb_catalog.metadata where key='exported_uuid'; count ------- 1 SELECT COUNT(*) from _timescaledb_catalog.metadata where key='install_timestamp'; count ------- 1 -- Make sure that the UUID is idempotent SELECT _timescaledb_internal.test_uuid() = :'uuid_1' as uuids_equal; uuids_equal ------------- t SELECT _timescaledb_internal.test_uuid() = :'uuid_1' as uuids_equal; uuids_equal ------------- t -- Also make sure install_time and exported_uuid are idempotent SELECT _timescaledb_internal.test_exported_uuid() = :'uuid_ex_1' as exported_uuids_equal; exported_uuids_equal ---------------------- t SELECT _timescaledb_internal.test_exported_uuid() = :'uuid_ex_1' as exported_uuids_equal; exported_uuids_equal ---------------------- t SELECT _timescaledb_internal.test_install_timestamp() = :'timestamp_1' as timestamps_equal; timestamps_equal ------------------ t SELECT _timescaledb_internal.test_install_timestamp() = :'timestamp_1' as timestamps_equal; timestamps_equal ------------------ t -- Now make sure that only the exported_uuid is exported on pg_dump \c postgres :ROLE_SUPERUSER \setenv PGOPTIONS '--client-min-messages=warning' \! utils/pg_dump_aux_dump.sh dump/instmeta.sql ALTER DATABASE :TEST_DBNAME SET timescaledb.restoring='on'; -- Redirect to /dev/null to suppress NOTICE \! utils/pg_dump_aux_restore.sh dump/instmeta.sql ALTER DATABASE :TEST_DBNAME SET timescaledb.restoring='off'; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- Should have all 3 row, because pg_dump includes the insertion of uuid and timestamp. SELECT COUNT(*) FROM _timescaledb_catalog.metadata; count ------- 5 -- Verify that this is the old exported_uuid SELECT _timescaledb_internal.test_exported_uuid() = :'uuid_ex_1' as exported_uuids_equal; exported_uuids_equal ---------------------- t -- Verify that the uuid is new SELECT _timescaledb_internal.test_uuid() = :'uuid_1' as exported_uuids_diff; exported_uuids_diff --------------------- f -- Verify that the install_timestamp got restored SELECT _timescaledb_internal.test_install_timestamp() = :'timestamp_1' as timestamps_equal; timestamps_equal ------------------ t SELECT * FROM _timescaledb_catalog.metadata WHERE key = 'metadata_test'; key | value | include_in_telemetry ---------------+-------+---------------------- metadata_test | FOO | t -- check metadata version matches expected value SELECT x.extversion = m.value AS "version match" FROM pg_extension x JOIN _timescaledb_catalog.metadata m ON m.key='timescaledb_version' WHERE x.extname='timescaledb'; version match --------------- t -- test version check in post_restore \c :TEST_DBNAME :ROLE_SUPERUSER UPDATE _timescaledb_catalog.metadata SET value = '1.2.3' WHERE key = 'timescaledb_version'; \set ON_ERROR_STOP 0 -- set verbosity to sqlstate to suppress version dependant error message \set VERBOSITY sqlstate SELECT timescaledb_post_restore(); ERROR: P0001 \set ON_ERROR_STOP 1 UPDATE _timescaledb_catalog.metadata m SET value = x.extversion FROM pg_extension x WHERE m.key = 'timescaledb_version' AND x.extname='timescaledb'; SELECT timescaledb_post_restore(); timescaledb_post_restore -------------------------- t ================================================ FILE: test/expected/multi_transaction_index.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE index_test(id serial, time timestamptz, device integer, temp float); SELECT * FROM test.show_columns('index_test'); Column | Type | NotNull --------+--------------------------+--------- id | integer | t time | timestamp with time zone | f device | integer | f temp | double precision | f -- Test that we can handle difference in attnos across hypertable and -- chunks by dropping the ID column ALTER TABLE index_test DROP COLUMN id; SELECT * FROM test.show_columns('index_test'); Column | Type | NotNull --------+--------------------------+--------- time | timestamp with time zone | f device | integer | f temp | double precision | f -- No pre-existing UNIQUE index, so partitioning on two columns should work SELECT create_hypertable('index_test', 'time', 'device', 2); create_hypertable ------------------------- (1,public,index_test,t) INSERT INTO index_test VALUES ('2017-01-20T09:00:01', 1, 17.5); \set ON_ERROR_STOP 0 -- cannot create a UNIQUE index with transaction_per_chunk CREATE UNIQUE INDEX index_test_time_device_idx ON index_test (time) WITH (timescaledb.transaction_per_chunk); ERROR: cannot use timescaledb.transaction_per_chunk with UNIQUE or PRIMARY KEY CREATE UNIQUE INDEX index_test_time_device_idx ON index_test (time, device) WITH(timescaledb.transaction_per_chunk); ERROR: cannot use timescaledb.transaction_per_chunk with UNIQUE or PRIMARY KEY \set ON_ERROR_STOP 1 CREATE INDEX index_test_time_device_idx ON index_test (time, device) WITH (timescaledb.transaction_per_chunk); -- Regular index need not cover all partitioning columns CREATE INDEX ON index_test (time, temp) WITH (timescaledb.transaction_per_chunk); -- Create another chunk INSERT INTO index_test VALUES ('2017-04-20T09:00:01', 1, 17.5); -- New index should have been recursed to chunks SELECT * FROM test.show_indexes('index_test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------+---------------+------+--------+---------+-----------+------------ index_test_device_time_idx | {device,time} | | f | f | f | index_test_time_device_idx | {time,device} | | f | f | f | index_test_time_idx | {time} | | f | f | f | index_test_time_temp_idx | {time,temp} | | f | f | f | SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk') ORDER BY 1,2; Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+-------------------------------------------------------------------+---------------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_index_test_time_idx | {time} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_index_test_time_device_idx | {time,device} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_index_test_time_idx | {time} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_index_test_time_device_idx | {time,device} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | ALTER INDEX index_test_time_idx RENAME TO index_test_time_idx2; -- Metadata and index should have changed name SELECT * FROM test.show_indexes('index_test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------+---------------+------+--------+---------+-----------+------------ index_test_device_time_idx | {device,time} | | f | f | f | index_test_time_device_idx | {time,device} | | f | f | f | index_test_time_idx2 | {time} | | f | f | f | index_test_time_temp_idx | {time,temp} | | f | f | f | SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk') ORDER BY 1,2; Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+-------------------------------------------------------------------+---------------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_index_test_time_idx2 | {time} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_index_test_time_device_idx | {time,device} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_index_test_time_idx2 | {time} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_index_test_time_device_idx | {time,device} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | DROP INDEX index_test_time_idx2; DROP INDEX index_test_time_device_idx; -- Index should have been dropped SELECT * FROM test.show_indexes('index_test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------+---------------+------+--------+---------+-----------+------------ index_test_device_time_idx | {device,time} | | f | f | f | index_test_time_temp_idx | {time,temp} | | f | f | f | SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+-------------------------------------------------------------------+---------------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | -- Create index with long name to see how this is handled on chunks CREATE INDEX a_hypertable_index_with_a_very_very_long_name_that_truncates ON index_test (time ASC, temp DESC) WITH (timescaledb.transaction_per_chunk); CREATE INDEX a_hypertable_index_with_a_very_very_long_name_that_truncates_2 ON index_test (time DESC, temp ASC) WITH (timescaledb.transaction_per_chunk); SELECT * FROM test.show_indexes('index_test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------------------------------+---------------+------+--------+---------+-----------+------------ a_hypertable_index_with_a_very_very_long_name_that_truncates | {time,temp} | | f | f | f | a_hypertable_index_with_a_very_very_long_name_that_truncates_2 | {time,temp} | | f | f | f | index_test_device_time_idx | {device,time} | | f | f | f | index_test_time_temp_idx | {time,temp} | | f | f | f | SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+---------------------------------------------------------------------------------------+---------------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_a_hypertable_index_with_a_very_very_long_name_ | {time,temp} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_a_hypertable_index_with_a_very_very_long_nam_1 | {time,temp} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_a_hypertable_index_with_a_very_very_long_name_ | {time,temp} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_a_hypertable_index_with_a_very_very_long_nam_1 | {time,temp} | | f | f | f | DROP INDEX a_hypertable_index_with_a_very_very_long_name_that_truncates; DROP INDEX a_hypertable_index_with_a_very_very_long_name_that_truncates_2; SELECT * FROM test.show_indexes('index_test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------+---------------+------+--------+---------+-----------+------------ index_test_device_time_idx | {device,time} | | f | f | f | index_test_time_temp_idx | {time,temp} | | f | f | f | SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+-------------------------------------------------------------------+---------------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | SELECT * FROM test.show_indexes('index_test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------+---------------+------+--------+---------+-----------+------------ index_test_device_time_idx | {device,time} | | f | f | f | index_test_time_temp_idx | {time,temp} | | f | f | f | SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+-------------------------------------------------------------------+---------------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | -- Add constraint index ALTER TABLE index_test ADD UNIQUE (time, device); SELECT * FROM test.show_indexes('index_test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------+---------------+------+--------+---------+-----------+------------ index_test_device_time_idx | {device,time} | | f | f | f | index_test_time_device_key | {time,device} | | t | f | f | index_test_time_temp_idx | {time,temp} | | f | f | f | SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+-------------------------------------------------------------------+---------------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal."1_1_index_test_time_device_key" | {time,device} | | t | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_index_test_device_time_idx | {device,time} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_index_test_time_temp_idx | {time,temp} | | f | f | f | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal."2_2_index_test_time_device_key" | {time,device} | | t | f | f | -- Constraints are added to chunk_constraint table. SELECT * FROM _timescaledb_catalog.chunk_constraint; chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----------+--------------------+--------------------------------+---------------------------- 1 | 1 | constraint_1 | 1 | 2 | constraint_2 | 2 | 3 | constraint_3 | 2 | 2 | constraint_2 | 1 | | 1_1_index_test_time_device_key | index_test_time_device_key 2 | | 2_2_index_test_time_device_key | index_test_time_device_key DROP TABLE index_test; -- Test that indexes are planned correctly CREATE TABLE index_expr_test(id serial, time timestamptz, temp float, meta int); select create_hypertable('index_expr_test', 'time'); create_hypertable ------------------------------ (2,public,index_expr_test,t) -- Screw up the attribute numbers ALTER TABLE index_expr_test DROP COLUMN id; CREATE INDEX ON index_expr_test (meta) WITH (timescaledb.transaction_per_chunk); INSERT INTO index_expr_test VALUES ('2017-01-20T09:00:01', 17.5, 1); INSERT INTO index_expr_test VALUES ('2017-01-20T09:00:01', 17.5, 2); SET enable_seqscan TO false; SET enable_bitmapscan TO false; EXPLAIN (verbose, buffers off, costs off) SELECT * FROM index_expr_test WHERE meta = 1; --- QUERY PLAN --- Index Scan using _hyper_2_3_chunk_index_expr_test_meta_idx on _timescaledb_internal._hyper_2_3_chunk Output: _hyper_2_3_chunk."time", _hyper_2_3_chunk.temp, _hyper_2_3_chunk.meta Index Cond: (_hyper_2_3_chunk.meta = 1) SELECT * FROM index_expr_test WHERE meta = 1; time | temp | meta ------------------------------+------+------ Fri Jan 20 09:00:01 2017 PST | 17.5 | 1 SET enable_seqscan TO default; SET enable_bitmapscan TO default; \set ON_ERROR_STOP 0 -- cannot create a transaction_per_chunk index within a transaction block BEGIN; CREATE INDEX ON index_expr_test (temp) WITH (timescaledb.transaction_per_chunk); ERROR: CREATE INDEX ... WITH (timescaledb.transaction_per_chunk) cannot run inside a transaction block ROLLBACK; \set ON_ERROR_STOP 1 DROP TABLE index_expr_test CASCADE; CREATE TABLE partial_index_test(time INTEGER); SELECT create_hypertable('partial_index_test', 'time', chunk_time_interval => 1, create_default_indexes => false); create_hypertable --------------------------------- (3,public,partial_index_test,t) -- create 3 chunks INSERT INTO partial_index_test(time) SELECT generate_series(0, 2); select * from partial_index_test order by 1; time ------ 0 1 2 -- create indexes on only 1 of the chunks CREATE INDEX ON partial_index_test (time) WITH (timescaledb.transaction_per_chunk, timescaledb.max_chunks='1'); SELECT * FROM test.show_indexes('partial_index_test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace -----------------------------+---------+------+--------+---------+-----------+------------ partial_index_test_time_idx | {time} | | f | f | f | SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+--------------------------------------------------------------------+---------+------+--------+---------+-----------+------------ _timescaledb_internal._hyper_3_4_chunk | _timescaledb_internal._hyper_3_4_chunk_partial_index_test_time_idx | {time} | | f | f | f | -- regerssion test for bug fixed by PR #1008. -- this caused an assertion failure when a MergeAppend node contained unsorted children SET enable_bitmapscan TO false; EXPLAIN (verbose, buffers off, costs off) SELECT * FROM partial_index_test WHERE time < 2 ORDER BY time LIMIT 2; --- QUERY PLAN --- Limit Output: partial_index_test."time" -> Custom Scan (ChunkAppend) on public.partial_index_test Output: partial_index_test."time" Order: partial_index_test."time" Startup Exclusion: false Runtime Exclusion: false -> Index Only Scan using _hyper_3_4_chunk_partial_index_test_time_idx on _timescaledb_internal._hyper_3_4_chunk Output: _hyper_3_4_chunk."time" -> Sort Output: _hyper_3_5_chunk."time" Sort Key: _hyper_3_5_chunk."time" -> Seq Scan on _timescaledb_internal._hyper_3_5_chunk Output: _hyper_3_5_chunk."time" SELECT * FROM partial_index_test WHERE time < 2 ORDER BY time LIMIT 2; time ------ 0 1 -- we can drop the partially created index DROP INDEX partial_index_test_time_idx; SELECT * FROM test.show_indexes('partial_index_test'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace -------+---------+------+--------+---------+-----------+------------ SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace -------+-------+---------+------+--------+---------+-----------+------------ EXPLAIN (verbose, buffers off, costs off) SELECT * FROM partial_index_test WHERE time < 2 ORDER BY time LIMIT 2; --- QUERY PLAN --- Limit Output: partial_index_test."time" -> Sort Output: partial_index_test."time" Sort Key: partial_index_test."time" -> Append -> Seq Scan on _timescaledb_internal._hyper_3_4_chunk Output: _hyper_3_4_chunk."time" -> Seq Scan on _timescaledb_internal._hyper_3_5_chunk Output: _hyper_3_5_chunk."time" SELECT * FROM partial_index_test WHERE time < 2 ORDER BY time LIMIT 2; time ------ 0 1 SET enable_seqscan TO true; SET enable_bitmapscan TO true; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 \set ON_ERROR_STOP 0 CREATE INDEX ON partial_index_test (time) WITH (timescaledb.transaction_per_chunk, timescaledb.max_chunks='1'); ERROR: must be owner of hypertable "partial_index_test" \set ON_ERROR_STOP 1 ================================================ FILE: test/expected/net.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION _timescaledb_internal.test_http_parsing(int) RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_http_parsing' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_http_parsing_full() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_http_parsing_full' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_http_request_build() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_http_request_build' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_conn() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_conn' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT _timescaledb_internal.test_http_parsing(10000); test_http_parsing ------------------- SELECT _timescaledb_internal.test_http_parsing_full(); test_http_parsing_full ------------------------ SELECT _timescaledb_internal.test_http_request_build(); test_http_request_build ------------------------- SELECT _timescaledb_internal.test_conn(); test_conn ----------- ================================================ FILE: test/expected/null_exclusion-15.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. create table metrics(ts timestamp, id int, value float); select create_hypertable('metrics', 'ts'); WARNING: column type "timestamp without time zone" used for "ts" does not follow best practices create_hypertable ---------------------- (1,public,metrics,t) insert into metrics values ('2022-02-02 02:02:02', 2, 2.), ('2023-03-03 03:03:03', 3, 3.); analyze metrics; -- non-const condition explain (analyze, buffers off, costs off, summary off, timing off) select * from metrics where ts >= (select max(ts) from metrics); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics (actual rows=1.00 loops=1) Chunks excluded during runtime: 1 InitPlan 2 (returns $1) -> Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on metrics metrics_1 (actual rows=1.00 loops=1) Order: metrics_1.ts DESC -> Index Only Scan using _hyper_1_2_chunk_metrics_ts_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (actual rows=1.00 loops=1) Index Cond: (ts IS NOT NULL) -> Index Only Scan using _hyper_1_1_chunk_metrics_ts_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (never executed) Index Cond: (ts IS NOT NULL) -> Seq Scan on _hyper_1_1_chunk (never executed) Filter: (ts >= $1) -> Seq Scan on _hyper_1_2_chunk (actual rows=1.00 loops=1) Filter: (ts >= $1) -- two non-const conditions explain (analyze, buffers off, costs off, summary off, timing off) select * from metrics where ts >= (select max(ts) from metrics) and id = 1; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics (actual rows=0.00 loops=1) Chunks excluded during runtime: 1 InitPlan 2 (returns $1) -> Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on metrics metrics_1 (actual rows=1.00 loops=1) Order: metrics_1.ts DESC -> Index Only Scan using _hyper_1_2_chunk_metrics_ts_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (actual rows=1.00 loops=1) Index Cond: (ts IS NOT NULL) -> Index Only Scan using _hyper_1_1_chunk_metrics_ts_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (never executed) Index Cond: (ts IS NOT NULL) -> Seq Scan on _hyper_1_1_chunk (never executed) Filter: ((ts >= $1) AND (id = 1)) -> Seq Scan on _hyper_1_2_chunk (actual rows=0.00 loops=1) Filter: ((ts >= $1) AND (id = 1)) Rows Removed by Filter: 1 -- condition that becomes const null after evaluating the param explain (analyze, buffers off, costs off, summary off, timing off) select * from metrics where ts >= (select max(ts) from metrics where id = -1); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics (actual rows=0.00 loops=1) Chunks excluded during runtime: 2 InitPlan 2 (returns $1) -> Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=0.00 loops=1) -> Custom Scan (ChunkAppend) on metrics metrics_1 (actual rows=0.00 loops=1) Order: metrics_1.ts DESC -> Index Scan using _hyper_1_2_chunk_metrics_ts_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (actual rows=0.00 loops=1) Index Cond: (ts IS NOT NULL) Filter: (id = '-1'::integer) Rows Removed by Filter: 1 -> Index Scan using _hyper_1_1_chunk_metrics_ts_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (actual rows=0.00 loops=1) Index Cond: (ts IS NOT NULL) Filter: (id = '-1'::integer) Rows Removed by Filter: 1 -> Seq Scan on _hyper_1_1_chunk (never executed) Filter: (ts >= $1) -> Seq Scan on _hyper_1_2_chunk (never executed) Filter: (ts >= $1) -- const null condition and some other condition explain (analyze, buffers off, costs off, summary off, timing off) select * from metrics where ts >= (select max(ts) from metrics where id = -1) and id = 1; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics (actual rows=0.00 loops=1) Chunks excluded during runtime: 2 InitPlan 2 (returns $1) -> Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=0.00 loops=1) -> Custom Scan (ChunkAppend) on metrics metrics_1 (actual rows=0.00 loops=1) Order: metrics_1.ts DESC -> Index Scan using _hyper_1_2_chunk_metrics_ts_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (actual rows=0.00 loops=1) Index Cond: (ts IS NOT NULL) Filter: (id = '-1'::integer) Rows Removed by Filter: 1 -> Index Scan using _hyper_1_1_chunk_metrics_ts_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (actual rows=0.00 loops=1) Index Cond: (ts IS NOT NULL) Filter: (id = '-1'::integer) Rows Removed by Filter: 1 -> Seq Scan on _hyper_1_1_chunk (never executed) Filter: ((ts >= $1) AND (id = 1)) -> Seq Scan on _hyper_1_2_chunk (never executed) Filter: ((ts >= $1) AND (id = 1)) ================================================ FILE: test/expected/null_exclusion-16.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. create table metrics(ts timestamp, id int, value float); select create_hypertable('metrics', 'ts'); WARNING: column type "timestamp without time zone" used for "ts" does not follow best practices create_hypertable ---------------------- (1,public,metrics,t) insert into metrics values ('2022-02-02 02:02:02', 2, 2.), ('2023-03-03 03:03:03', 3, 3.); analyze metrics; -- non-const condition explain (analyze, buffers off, costs off, summary off, timing off) select * from metrics where ts >= (select max(ts) from metrics); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics (actual rows=1.00 loops=1) Chunks excluded during runtime: 1 InitPlan 2 (returns $1) -> Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on metrics metrics_1 (actual rows=1.00 loops=1) Order: metrics_1.ts DESC -> Index Only Scan using _hyper_1_2_chunk_metrics_ts_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (actual rows=1.00 loops=1) Index Cond: (ts IS NOT NULL) -> Index Only Scan using _hyper_1_1_chunk_metrics_ts_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (never executed) Index Cond: (ts IS NOT NULL) -> Seq Scan on _hyper_1_1_chunk (never executed) Filter: (ts >= $1) -> Seq Scan on _hyper_1_2_chunk (actual rows=1.00 loops=1) Filter: (ts >= $1) -- two non-const conditions explain (analyze, buffers off, costs off, summary off, timing off) select * from metrics where ts >= (select max(ts) from metrics) and id = 1; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics (actual rows=0.00 loops=1) Chunks excluded during runtime: 1 InitPlan 2 (returns $1) -> Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on metrics metrics_1 (actual rows=1.00 loops=1) Order: metrics_1.ts DESC -> Index Only Scan using _hyper_1_2_chunk_metrics_ts_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (actual rows=1.00 loops=1) Index Cond: (ts IS NOT NULL) -> Index Only Scan using _hyper_1_1_chunk_metrics_ts_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (never executed) Index Cond: (ts IS NOT NULL) -> Seq Scan on _hyper_1_1_chunk (never executed) Filter: ((ts >= $1) AND (id = 1)) -> Seq Scan on _hyper_1_2_chunk (actual rows=0.00 loops=1) Filter: ((ts >= $1) AND (id = 1)) Rows Removed by Filter: 1 -- condition that becomes const null after evaluating the param explain (analyze, buffers off, costs off, summary off, timing off) select * from metrics where ts >= (select max(ts) from metrics where id = -1); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics (actual rows=0.00 loops=1) Chunks excluded during runtime: 2 InitPlan 2 (returns $1) -> Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=0.00 loops=1) -> Custom Scan (ChunkAppend) on metrics metrics_1 (actual rows=0.00 loops=1) Order: metrics_1.ts DESC -> Index Scan using _hyper_1_2_chunk_metrics_ts_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (actual rows=0.00 loops=1) Index Cond: (ts IS NOT NULL) Filter: (id = '-1'::integer) Rows Removed by Filter: 1 -> Index Scan using _hyper_1_1_chunk_metrics_ts_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (actual rows=0.00 loops=1) Index Cond: (ts IS NOT NULL) Filter: (id = '-1'::integer) Rows Removed by Filter: 1 -> Seq Scan on _hyper_1_1_chunk (never executed) Filter: (ts >= $1) -> Seq Scan on _hyper_1_2_chunk (never executed) Filter: (ts >= $1) -- const null condition and some other condition explain (analyze, buffers off, costs off, summary off, timing off) select * from metrics where ts >= (select max(ts) from metrics where id = -1) and id = 1; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics (actual rows=0.00 loops=1) Chunks excluded during runtime: 2 InitPlan 2 (returns $1) -> Result (actual rows=1.00 loops=1) InitPlan 1 (returns $0) -> Limit (actual rows=0.00 loops=1) -> Custom Scan (ChunkAppend) on metrics metrics_1 (actual rows=0.00 loops=1) Order: metrics_1.ts DESC -> Index Scan using _hyper_1_2_chunk_metrics_ts_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (actual rows=0.00 loops=1) Index Cond: (ts IS NOT NULL) Filter: (id = '-1'::integer) Rows Removed by Filter: 1 -> Index Scan using _hyper_1_1_chunk_metrics_ts_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (actual rows=0.00 loops=1) Index Cond: (ts IS NOT NULL) Filter: (id = '-1'::integer) Rows Removed by Filter: 1 -> Seq Scan on _hyper_1_1_chunk (never executed) Filter: ((ts >= $1) AND (id = 1)) -> Seq Scan on _hyper_1_2_chunk (never executed) Filter: ((ts >= $1) AND (id = 1)) ================================================ FILE: test/expected/null_exclusion-17.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. create table metrics(ts timestamp, id int, value float); select create_hypertable('metrics', 'ts'); WARNING: column type "timestamp without time zone" used for "ts" does not follow best practices create_hypertable ---------------------- (1,public,metrics,t) insert into metrics values ('2022-02-02 02:02:02', 2, 2.), ('2023-03-03 03:03:03', 3, 3.); analyze metrics; -- non-const condition explain (analyze, buffers off, costs off, summary off, timing off) select * from metrics where ts >= (select max(ts) from metrics); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics (actual rows=1.00 loops=1) Chunks excluded during runtime: 1 InitPlan 2 -> Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on metrics metrics_1 (actual rows=1.00 loops=1) Order: metrics_1.ts DESC -> Index Only Scan using _hyper_1_2_chunk_metrics_ts_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (actual rows=1.00 loops=1) -> Index Only Scan using _hyper_1_1_chunk_metrics_ts_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (never executed) -> Seq Scan on _hyper_1_1_chunk (never executed) Filter: (ts >= (InitPlan 2).col1) -> Seq Scan on _hyper_1_2_chunk (actual rows=1.00 loops=1) Filter: (ts >= (InitPlan 2).col1) -- two non-const conditions explain (analyze, buffers off, costs off, summary off, timing off) select * from metrics where ts >= (select max(ts) from metrics) and id = 1; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics (actual rows=0.00 loops=1) Chunks excluded during runtime: 1 InitPlan 2 -> Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on metrics metrics_1 (actual rows=1.00 loops=1) Order: metrics_1.ts DESC -> Index Only Scan using _hyper_1_2_chunk_metrics_ts_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (actual rows=1.00 loops=1) -> Index Only Scan using _hyper_1_1_chunk_metrics_ts_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (never executed) -> Seq Scan on _hyper_1_1_chunk (never executed) Filter: ((ts >= (InitPlan 2).col1) AND (id = 1)) -> Seq Scan on _hyper_1_2_chunk (actual rows=0.00 loops=1) Filter: ((ts >= (InitPlan 2).col1) AND (id = 1)) Rows Removed by Filter: 1 -- condition that becomes const null after evaluating the param explain (analyze, buffers off, costs off, summary off, timing off) select * from metrics where ts >= (select max(ts) from metrics where id = -1); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics (actual rows=0.00 loops=1) Chunks excluded during runtime: 2 InitPlan 2 -> Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=0.00 loops=1) -> Custom Scan (ChunkAppend) on metrics metrics_1 (actual rows=0.00 loops=1) Order: metrics_1.ts DESC -> Index Scan using _hyper_1_2_chunk_metrics_ts_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (actual rows=0.00 loops=1) Filter: (id = '-1'::integer) Rows Removed by Filter: 1 -> Index Scan using _hyper_1_1_chunk_metrics_ts_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (actual rows=0.00 loops=1) Filter: (id = '-1'::integer) Rows Removed by Filter: 1 -> Seq Scan on _hyper_1_1_chunk (never executed) Filter: (ts >= (InitPlan 2).col1) -> Seq Scan on _hyper_1_2_chunk (never executed) Filter: (ts >= (InitPlan 2).col1) -- const null condition and some other condition explain (analyze, buffers off, costs off, summary off, timing off) select * from metrics where ts >= (select max(ts) from metrics where id = -1) and id = 1; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics (actual rows=0.00 loops=1) Chunks excluded during runtime: 2 InitPlan 2 -> Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=0.00 loops=1) -> Custom Scan (ChunkAppend) on metrics metrics_1 (actual rows=0.00 loops=1) Order: metrics_1.ts DESC -> Index Scan using _hyper_1_2_chunk_metrics_ts_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (actual rows=0.00 loops=1) Filter: (id = '-1'::integer) Rows Removed by Filter: 1 -> Index Scan using _hyper_1_1_chunk_metrics_ts_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (actual rows=0.00 loops=1) Filter: (id = '-1'::integer) Rows Removed by Filter: 1 -> Seq Scan on _hyper_1_1_chunk (never executed) Filter: ((ts >= (InitPlan 2).col1) AND (id = 1)) -> Seq Scan on _hyper_1_2_chunk (never executed) Filter: ((ts >= (InitPlan 2).col1) AND (id = 1)) ================================================ FILE: test/expected/null_exclusion-18.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. create table metrics(ts timestamp, id int, value float); select create_hypertable('metrics', 'ts'); WARNING: column type "timestamp without time zone" used for "ts" does not follow best practices create_hypertable ---------------------- (1,public,metrics,t) insert into metrics values ('2022-02-02 02:02:02', 2, 2.), ('2023-03-03 03:03:03', 3, 3.); analyze metrics; -- non-const condition explain (analyze, buffers off, costs off, summary off, timing off) select * from metrics where ts >= (select max(ts) from metrics); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics (actual rows=1.00 loops=1) Chunks excluded during runtime: 1 InitPlan 2 -> Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on metrics metrics_1 (actual rows=1.00 loops=1) Order: metrics_1.ts DESC -> Index Only Scan using _hyper_1_2_chunk_metrics_ts_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (actual rows=1.00 loops=1) -> Index Only Scan using _hyper_1_1_chunk_metrics_ts_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (never executed) -> Seq Scan on _hyper_1_1_chunk (never executed) Filter: (ts >= (InitPlan 2).col1) -> Seq Scan on _hyper_1_2_chunk (actual rows=1.00 loops=1) Filter: (ts >= (InitPlan 2).col1) -- two non-const conditions explain (analyze, buffers off, costs off, summary off, timing off) select * from metrics where ts >= (select max(ts) from metrics) and id = 1; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics (actual rows=0.00 loops=1) Chunks excluded during runtime: 1 InitPlan 2 -> Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on metrics metrics_1 (actual rows=1.00 loops=1) Order: metrics_1.ts DESC -> Index Only Scan using _hyper_1_2_chunk_metrics_ts_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (actual rows=1.00 loops=1) -> Index Only Scan using _hyper_1_1_chunk_metrics_ts_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (never executed) -> Seq Scan on _hyper_1_1_chunk (never executed) Filter: ((ts >= (InitPlan 2).col1) AND (id = 1)) -> Seq Scan on _hyper_1_2_chunk (actual rows=0.00 loops=1) Filter: ((ts >= (InitPlan 2).col1) AND (id = 1)) Rows Removed by Filter: 1 -- condition that becomes const null after evaluating the param explain (analyze, buffers off, costs off, summary off, timing off) select * from metrics where ts >= (select max(ts) from metrics where id = -1); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics (actual rows=0.00 loops=1) Chunks excluded during runtime: 2 InitPlan 2 -> Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=0.00 loops=1) -> Custom Scan (ChunkAppend) on metrics metrics_1 (actual rows=0.00 loops=1) Order: metrics_1.ts DESC -> Index Scan using _hyper_1_2_chunk_metrics_ts_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (actual rows=0.00 loops=1) Filter: (id = '-1'::integer) Rows Removed by Filter: 1 -> Index Scan using _hyper_1_1_chunk_metrics_ts_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (actual rows=0.00 loops=1) Filter: (id = '-1'::integer) Rows Removed by Filter: 1 -> Seq Scan on _hyper_1_1_chunk (never executed) Filter: (ts >= (InitPlan 2).col1) -> Seq Scan on _hyper_1_2_chunk (never executed) Filter: (ts >= (InitPlan 2).col1) -- const null condition and some other condition explain (analyze, buffers off, costs off, summary off, timing off) select * from metrics where ts >= (select max(ts) from metrics where id = -1) and id = 1; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics (actual rows=0.00 loops=1) Chunks excluded during runtime: 2 InitPlan 2 -> Result (actual rows=1.00 loops=1) InitPlan 1 -> Limit (actual rows=0.00 loops=1) -> Custom Scan (ChunkAppend) on metrics metrics_1 (actual rows=0.00 loops=1) Order: metrics_1.ts DESC -> Index Scan using _hyper_1_2_chunk_metrics_ts_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 (actual rows=0.00 loops=1) Filter: (id = '-1'::integer) Rows Removed by Filter: 1 -> Index Scan using _hyper_1_1_chunk_metrics_ts_idx on _hyper_1_1_chunk _hyper_1_1_chunk_1 (actual rows=0.00 loops=1) Filter: (id = '-1'::integer) Rows Removed by Filter: 1 -> Seq Scan on _hyper_1_1_chunk (never executed) Filter: ((ts >= (InitPlan 2).col1) AND (id = 1)) -> Seq Scan on _hyper_1_2_chunk (never executed) Filter: ((ts >= (InitPlan 2).col1) AND (id = 1)) ================================================ FILE: test/expected/parallel-15.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --parallel queries require big-ish tables so collect them all here --so that we need to generate queries only once. -- output with analyze is not stable because it depends on worker assignment \set PREFIX 'EXPLAIN (buffers off, costs off)' \set CHUNK1 _timescaledb_internal._hyper_1_1_chunk \set CHUNK2 _timescaledb_internal._hyper_1_2_chunk CREATE TABLE test (i int, j double precision, ts timestamp); SELECT create_hypertable('test','i',chunk_time_interval:=500000); WARNING: column type "timestamp without time zone" used for "ts" does not follow best practices create_hypertable ------------------- (1,public,test,t) INSERT INTO test SELECT x, x+0.1, _timescaledb_functions.to_timestamp(x*1000) FROM generate_series(0,1000000-1,10) AS x; ANALYZE test; ALTER TABLE :CHUNK1 SET (parallel_workers=2); ALTER TABLE :CHUNK2 SET (parallel_workers=2); SET work_mem TO '50MB'; SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false); set_config ------------ on SET max_parallel_workers_per_gather = 4; SET parallel_setup_cost TO 0; EXPLAIN (buffers off, costs off) SELECT first(i, j) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT first(i, j) FROM "test"; first ------- 0 EXPLAIN (buffers off, costs off) SELECT last(i, j) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT last(i, j) FROM "test"; last -------- 999990 EXPLAIN (buffers off, costs off) SELECT time_bucket('1 second', ts) sec, last(i, j) FROM "test" GROUP BY sec ORDER BY sec LIMIT 5; --- QUERY PLAN --- Limit -> Finalize GroupAggregate Group Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Gather Merge Workers Planned: 2 -> Partial GroupAggregate Group Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Sort Sort Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Result -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk -- test single copy parallel plan with parallel chunk append :PREFIX SELECT time_bucket('1 second', ts) sec, last(i, j) FROM "test" WHERE length(version()) > 0 GROUP BY sec ORDER BY sec LIMIT 5; --- QUERY PLAN --- Limit -> Finalize GroupAggregate Group Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Gather Merge Workers Planned: 2 -> Partial GroupAggregate Group Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Sort Sort Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Result -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk SELECT time_bucket('1 second', ts) sec, last(i, j) FROM "test" GROUP BY sec ORDER BY sec LIMIT 5; sec | last --------------------------+------ Wed Dec 31 16:00:00 1969 | 990 Wed Dec 31 16:00:01 1969 | 1990 Wed Dec 31 16:00:02 1969 | 2990 Wed Dec 31 16:00:03 1969 | 3990 Wed Dec 31 16:00:04 1969 | 4990 --test variants of histogram EXPLAIN (buffers off, costs off) SELECT histogram(i, 1, 1000000, 2) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT histogram(i, 1, 1000000, 2) FROM "test"; histogram ------------------- {1,50000,49999,0} EXPLAIN (buffers off, costs off) SELECT histogram(i, 1,1000001,10) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT histogram(i, 1, 1000001, 10) FROM "test"; histogram ------------------------------------------------------------------ {1,10000,10000,10000,10000,10000,10000,10000,10000,10000,9999,0} EXPLAIN (buffers off, costs off) SELECT histogram(i, 0,100000,5) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT histogram(i, 0, 100000, 5) FROM "test"; histogram ------------------------------------ {0,2000,2000,2000,2000,2000,90000} EXPLAIN (buffers off, costs off) SELECT histogram(i, 10,100000,5) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT histogram(i, 10, 100000, 5) FROM "test"; histogram ------------------------------------ {1,2000,2000,2000,2000,1999,90000} EXPLAIN (buffers off, costs off) SELECT histogram(NULL, 10,100000,5) FROM "test" WHERE i = coalesce(-1,j); --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk Filter: ((i)::double precision = '-1'::double precision) -> Parallel Seq Scan on _hyper_1_2_chunk Filter: ((i)::double precision = '-1'::double precision) SELECT histogram(NULL, 10,100000,5) FROM "test" WHERE i = coalesce(-1,j); histogram ----------- -- test parallel ChunkAppend :PREFIX SELECT i FROM "test" WHERE length(version()) > 0; --- QUERY PLAN --- Gather Workers Planned: 1 Single Copy: true -> Result One-Time Filter: (length(version()) > 0) -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Seq Scan on _hyper_1_1_chunk -> Result One-Time Filter: (length(version()) > 0) -> Seq Scan on _hyper_1_2_chunk :PREFIX SELECT count(*) FROM "test" WHERE i > 1 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk Filter: (i > 1) -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk SELECT count(*) FROM "test" WHERE i > 1 AND length(version()) > 0; count ------- 99999 -- test parallel ChunkAppend with only work done in the parallel workers SET parallel_leader_participation = off; SELECT count(*) FROM "test" WHERE i > 1 AND length(version()) > 0; count ------- 99999 RESET parallel_leader_participation; -- Test parallel chunk append is used (index scan is disabled to trigger a parallel chunk append) SET parallel_tuple_cost = 0; SET enable_indexscan = OFF; :PREFIX SELECT * FROM (SELECT * FROM "test" WHERE length(version()) > 0 ORDER BY I LIMIT 10) AS t1 LEFT JOIN (SELECT * FROM "test" WHERE i < 500000 ORDER BY I LIMIT 10) AS t2 ON (t1.i = t2.i) ORDER BY t1.i, t2.i; --- QUERY PLAN --- Sort Sort Key: test.i, _hyper_1_1_chunk_1.i -> Merge Left Join Merge Cond: (test.i = _hyper_1_1_chunk_1.i) -> Limit -> Gather Merge Workers Planned: 2 -> Sort Sort Key: test.i -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk -> Materialize -> Limit -> Gather Merge Workers Planned: 2 -> Sort Sort Key: _hyper_1_1_chunk_1.i -> Parallel Seq Scan on _hyper_1_1_chunk _hyper_1_1_chunk_1 SELECT * FROM (SELECT * FROM "test" WHERE length(version()) > 0 ORDER BY I LIMIT 10) AS t1 LEFT JOIN (SELECT * FROM "test" WHERE i < 500000 ORDER BY I LIMIT 10) AS t2 ON (t1.i = t2.i) ORDER BY t1.i, t2.i; i | j | ts | i | j | ts ----+------+-----------------------------+----+------+----------------------------- 0 | 0.1 | Wed Dec 31 16:00:00 1969 | 0 | 0.1 | Wed Dec 31 16:00:00 1969 10 | 10.1 | Wed Dec 31 16:00:00.01 1969 | 10 | 10.1 | Wed Dec 31 16:00:00.01 1969 20 | 20.1 | Wed Dec 31 16:00:00.02 1969 | 20 | 20.1 | Wed Dec 31 16:00:00.02 1969 30 | 30.1 | Wed Dec 31 16:00:00.03 1969 | 30 | 30.1 | Wed Dec 31 16:00:00.03 1969 40 | 40.1 | Wed Dec 31 16:00:00.04 1969 | 40 | 40.1 | Wed Dec 31 16:00:00.04 1969 50 | 50.1 | Wed Dec 31 16:00:00.05 1969 | 50 | 50.1 | Wed Dec 31 16:00:00.05 1969 60 | 60.1 | Wed Dec 31 16:00:00.06 1969 | 60 | 60.1 | Wed Dec 31 16:00:00.06 1969 70 | 70.1 | Wed Dec 31 16:00:00.07 1969 | 70 | 70.1 | Wed Dec 31 16:00:00.07 1969 80 | 80.1 | Wed Dec 31 16:00:00.08 1969 | 80 | 80.1 | Wed Dec 31 16:00:00.08 1969 90 | 90.1 | Wed Dec 31 16:00:00.09 1969 | 90 | 90.1 | Wed Dec 31 16:00:00.09 1969 SET enable_indexscan = ON; -- Test normal chunk append can be used in a parallel worker :PREFIX SELECT * FROM (SELECT * FROM "test" WHERE i >= 999000 ORDER BY i) AS t1 JOIN (SELECT * FROM "test" WHERE i >= 400000 ORDER BY i) AS t2 ON (TRUE) ORDER BY t1.i, t2.i LIMIT 10; --- QUERY PLAN --- Gather Workers Planned: 1 Single Copy: true -> Limit -> Incremental Sort Sort Key: _hyper_1_2_chunk.i, test.i Presorted Key: _hyper_1_2_chunk.i -> Nested Loop -> Index Scan Backward using _hyper_1_2_chunk_test_i_idx on _hyper_1_2_chunk Index Cond: (i >= 999000) -> Materialize -> Custom Scan (ChunkAppend) on test Order: test.i -> Index Scan Backward using _hyper_1_1_chunk_test_i_idx on _hyper_1_1_chunk Index Cond: (i >= 400000) -> Index Scan Backward using _hyper_1_2_chunk_test_i_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 SELECT * FROM (SELECT * FROM "test" WHERE i >= 999000 ORDER BY i) AS t1 JOIN (SELECT * FROM "test" WHERE i >= 400000 ORDER BY i) AS t2 ON (TRUE) ORDER BY t1.i, t2.i LIMIT 10; i | j | ts | i | j | ts --------+----------+--------------------------+--------+----------+----------------------------- 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400000 | 400000.1 | Wed Dec 31 16:06:40 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400010 | 400010.1 | Wed Dec 31 16:06:40.01 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400020 | 400020.1 | Wed Dec 31 16:06:40.02 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400030 | 400030.1 | Wed Dec 31 16:06:40.03 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400040 | 400040.1 | Wed Dec 31 16:06:40.04 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400050 | 400050.1 | Wed Dec 31 16:06:40.05 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400060 | 400060.1 | Wed Dec 31 16:06:40.06 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400070 | 400070.1 | Wed Dec 31 16:06:40.07 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400080 | 400080.1 | Wed Dec 31 16:06:40.08 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400090 | 400090.1 | Wed Dec 31 16:06:40.09 1969 -- Test parallel ChunkAppend reinit SET enable_material = off; SET min_parallel_table_scan_size = 0; SET min_parallel_index_scan_size = 0; SET enable_hashjoin = 'off'; SET enable_nestloop = 'off'; CREATE TABLE sensor_data( time timestamptz NOT NULL, sensor_id integer NOT NULL); SELECT FROM create_hypertable(relation=>'sensor_data', time_column_name=> 'time'); -- -- Sensors 1 and 2 INSERT INTO sensor_data SELECT time, sensor_id FROM generate_series('2000-01-01 00:00:30', '2022-01-01 00:00:30', INTERVAL '3 months') AS g1(time), generate_series(1, 2, 1) AS g2(sensor_id) ORDER BY time; -- Sensor 100 INSERT INTO sensor_data SELECT time, 100 as sensor_id FROM generate_series('2000-01-01 00:00:30', '2022-01-01 00:00:30', INTERVAL '1 year') AS g1(time) ORDER BY time; :PREFIX SELECT * FROM sensor_data AS s1 JOIN sensor_data AS s2 ON (TRUE) WHERE s1.time > '2020-01-01 00:00:30'::text::timestamptz AND s2.time > '2020-01-01 00:00:30' AND s2.time < '2021-01-01 00:00:30' AND s1.sensor_id > 50 ORDER BY s2.time, s1.time, s1.sensor_id, s2.sensor_id; --- QUERY PLAN --- Sort Sort Key: s2."time", s1."time", s1.sensor_id, s2.sensor_id -> Nested Loop -> Custom Scan (ChunkAppend) on sensor_data s2 Order: s2."time" -> Index Scan Backward using _hyper_2_83_chunk_sensor_data_time_idx on _hyper_2_83_chunk s2_1 Index Cond: (("time" > 'Wed Jan 01 00:00:30 2020 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 01 00:00:30 2021 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_2_84_chunk_sensor_data_time_idx on _hyper_2_84_chunk s2_2 -> Index Scan Backward using _hyper_2_85_chunk_sensor_data_time_idx on _hyper_2_85_chunk s2_3 -> Index Scan Backward using _hyper_2_86_chunk_sensor_data_time_idx on _hyper_2_86_chunk s2_4 -> Index Scan Backward using _hyper_2_87_chunk_sensor_data_time_idx on _hyper_2_87_chunk s2_5 Index Cond: (("time" > 'Wed Jan 01 00:00:30 2020 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 01 00:00:30 2021 PST'::timestamp with time zone)) -> Gather Workers Planned: 4 -> Parallel Custom Scan (ChunkAppend) on sensor_data s1 Chunks excluded during startup: 80 -> Parallel Bitmap Heap Scan on _hyper_2_83_chunk s1_1 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_83_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_84_chunk s1_2 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_84_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_85_chunk s1_3 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_85_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_86_chunk s1_4 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_86_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_87_chunk s1_5 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_87_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_88_chunk s1_6 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_88_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_89_chunk s1_7 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_89_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_90_chunk s1_8 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_90_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_91_chunk s1_9 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_91_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -- Check query result SELECT * FROM sensor_data AS s1 JOIN sensor_data AS s2 ON (TRUE) WHERE s1.time > '2020-01-01 00:00:30'::text::timestamptz AND s2.time > '2020-01-01 00:00:30' AND s2.time < '2021-01-01 00:00:30' AND s1.sensor_id > 50 ORDER BY s2.time, s1.time, s1.sensor_id, s2.sensor_id; time | sensor_id | time | sensor_id ------------------------------+-----------+------------------------------+----------- Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 -- Ensure the same result is produced if only the parallel workers have to produce them (i.e., the pstate is reinitialized properly) SET parallel_leader_participation = off; SELECT * FROM sensor_data AS s1 JOIN sensor_data AS s2 ON (TRUE) WHERE s1.time > '2020-01-01 00:00:30'::text::timestamptz AND s2.time > '2020-01-01 00:00:30' AND s2.time < '2021-01-01 00:00:30' AND s1.sensor_id > 50 ORDER BY s2.time, s1.time, s1.sensor_id, s2.sensor_id; time | sensor_id | time | sensor_id ------------------------------+-----------+------------------------------+----------- Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 RESET parallel_leader_participation; -- Ensure the same query result is produced by a sequencial query SET max_parallel_workers_per_gather TO 0; SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'off', false); set_config ------------ off SELECT * FROM sensor_data AS s1 JOIN sensor_data AS s2 ON (TRUE) WHERE s1.time > '2020-01-01 00:00:30'::text::timestamptz AND s2.time > '2020-01-01 00:00:30' AND s2.time < '2021-01-01 00:00:30' AND s1.sensor_id > 50 ORDER BY s2.time, s1.time, s1.sensor_id, s2.sensor_id; time | sensor_id | time | sensor_id ------------------------------+-----------+------------------------------+----------- Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 RESET enable_material; RESET min_parallel_table_scan_size; RESET min_parallel_index_scan_size; RESET enable_hashjoin; RESET enable_nestloop; RESET parallel_tuple_cost; SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false); set_config ------------ on -- test worker assignment -- first chunk should have 1 worker and second chunk should have 2 SET max_parallel_workers_per_gather TO 2; :PREFIX SELECT count(*) FROM "test" WHERE i >= 400000 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Index Only Scan using _hyper_1_1_chunk_test_i_idx on _hyper_1_1_chunk Index Cond: (i >= 400000) -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk SELECT count(*) FROM "test" WHERE i >= 400000 AND length(version()) > 0; count ------- 60000 -- test worker assignment -- first chunk should have 2 worker and second chunk should have 1 :PREFIX SELECT count(*) FROM "test" WHERE i < 600000 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Index Only Scan using _hyper_1_2_chunk_test_i_idx on _hyper_1_2_chunk Index Cond: (i < 600000) -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk SELECT count(*) FROM "test" WHERE i < 600000 AND length(version()) > 0; count ------- 60000 -- test ChunkAppend with # workers < # childs SET max_parallel_workers_per_gather TO 1; :PREFIX SELECT count(*) FROM "test" WHERE length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 1 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk SELECT count(*) FROM "test" WHERE length(version()) > 0; count -------- 100000 -- test ChunkAppend with # workers > # childs SET max_parallel_workers_per_gather TO 2; :PREFIX SELECT count(*) FROM "test" WHERE i >= 500000 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk SELECT count(*) FROM "test" WHERE i >= 500000 AND length(version()) > 0; count ------- 50000 RESET max_parallel_workers_per_gather; -- test partial and non-partial plans -- these will not be parallel on PG < 11 ALTER TABLE :CHUNK1 SET (parallel_workers=0); ALTER TABLE :CHUNK2 SET (parallel_workers=2); :PREFIX SELECT count(*) FROM "test" WHERE i > 400000 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Index Only Scan using _hyper_1_1_chunk_test_i_idx on _hyper_1_1_chunk Index Cond: (i > 400000) -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk ALTER TABLE :CHUNK1 SET (parallel_workers=2); ALTER TABLE :CHUNK2 SET (parallel_workers=0); :PREFIX SELECT count(*) FROM "test" WHERE i < 600000 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Index Only Scan using _hyper_1_2_chunk_test_i_idx on _hyper_1_2_chunk Index Cond: (i < 600000) -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk ALTER TABLE :CHUNK1 RESET (parallel_workers); ALTER TABLE :CHUNK2 RESET (parallel_workers); -- now() is not marked parallel safe in PostgreSQL < 12 so using now() -- in a query will prevent parallelism but CURRENT_TIMESTAMP and -- transaction_timestamp() are marked parallel safe :PREFIX SELECT i FROM "test" WHERE ts < CURRENT_TIMESTAMP; --- QUERY PLAN --- Gather Workers Planned: 1 Single Copy: true -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk Filter: (ts < CURRENT_TIMESTAMP) -> Seq Scan on _hyper_1_2_chunk Filter: (ts < CURRENT_TIMESTAMP) :PREFIX SELECT i FROM "test" WHERE ts < transaction_timestamp(); --- QUERY PLAN --- Gather Workers Planned: 1 Single Copy: true -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk Filter: (ts < transaction_timestamp()) -> Seq Scan on _hyper_1_2_chunk Filter: (ts < transaction_timestamp()) -- this won't be parallel query because now() is parallel restricted in PG < 12 :PREFIX SELECT i FROM "test" WHERE ts < now(); --- QUERY PLAN --- Gather Workers Planned: 1 Single Copy: true -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk Filter: (ts < now()) -> Seq Scan on _hyper_1_2_chunk Filter: (ts < now()) ================================================ FILE: test/expected/parallel-16.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --parallel queries require big-ish tables so collect them all here --so that we need to generate queries only once. -- output with analyze is not stable because it depends on worker assignment \set PREFIX 'EXPLAIN (buffers off, costs off)' \set CHUNK1 _timescaledb_internal._hyper_1_1_chunk \set CHUNK2 _timescaledb_internal._hyper_1_2_chunk CREATE TABLE test (i int, j double precision, ts timestamp); SELECT create_hypertable('test','i',chunk_time_interval:=500000); WARNING: column type "timestamp without time zone" used for "ts" does not follow best practices create_hypertable ------------------- (1,public,test,t) INSERT INTO test SELECT x, x+0.1, _timescaledb_functions.to_timestamp(x*1000) FROM generate_series(0,1000000-1,10) AS x; ANALYZE test; ALTER TABLE :CHUNK1 SET (parallel_workers=2); ALTER TABLE :CHUNK2 SET (parallel_workers=2); SET work_mem TO '50MB'; SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false); set_config ------------ on SET max_parallel_workers_per_gather = 4; SET parallel_setup_cost TO 0; EXPLAIN (buffers off, costs off) SELECT first(i, j) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT first(i, j) FROM "test"; first ------- 0 EXPLAIN (buffers off, costs off) SELECT last(i, j) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT last(i, j) FROM "test"; last -------- 999990 EXPLAIN (buffers off, costs off) SELECT time_bucket('1 second', ts) sec, last(i, j) FROM "test" GROUP BY sec ORDER BY sec LIMIT 5; --- QUERY PLAN --- Limit -> Finalize GroupAggregate Group Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Gather Merge Workers Planned: 2 -> Partial GroupAggregate Group Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Sort Sort Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Result -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk -- test single copy parallel plan with parallel chunk append :PREFIX SELECT time_bucket('1 second', ts) sec, last(i, j) FROM "test" WHERE length(version()) > 0 GROUP BY sec ORDER BY sec LIMIT 5; --- QUERY PLAN --- Limit -> Finalize GroupAggregate Group Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Gather Merge Workers Planned: 2 -> Partial GroupAggregate Group Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Sort Sort Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Result -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk SELECT time_bucket('1 second', ts) sec, last(i, j) FROM "test" GROUP BY sec ORDER BY sec LIMIT 5; sec | last --------------------------+------ Wed Dec 31 16:00:00 1969 | 990 Wed Dec 31 16:00:01 1969 | 1990 Wed Dec 31 16:00:02 1969 | 2990 Wed Dec 31 16:00:03 1969 | 3990 Wed Dec 31 16:00:04 1969 | 4990 --test variants of histogram EXPLAIN (buffers off, costs off) SELECT histogram(i, 1, 1000000, 2) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT histogram(i, 1, 1000000, 2) FROM "test"; histogram ------------------- {1,50000,49999,0} EXPLAIN (buffers off, costs off) SELECT histogram(i, 1,1000001,10) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT histogram(i, 1, 1000001, 10) FROM "test"; histogram ------------------------------------------------------------------ {1,10000,10000,10000,10000,10000,10000,10000,10000,10000,9999,0} EXPLAIN (buffers off, costs off) SELECT histogram(i, 0,100000,5) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT histogram(i, 0, 100000, 5) FROM "test"; histogram ------------------------------------ {0,2000,2000,2000,2000,2000,90000} EXPLAIN (buffers off, costs off) SELECT histogram(i, 10,100000,5) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT histogram(i, 10, 100000, 5) FROM "test"; histogram ------------------------------------ {1,2000,2000,2000,2000,1999,90000} EXPLAIN (buffers off, costs off) SELECT histogram(NULL, 10,100000,5) FROM "test" WHERE i = coalesce(-1,j); --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk Filter: ((i)::double precision = '-1'::double precision) -> Parallel Seq Scan on _hyper_1_2_chunk Filter: ((i)::double precision = '-1'::double precision) SELECT histogram(NULL, 10,100000,5) FROM "test" WHERE i = coalesce(-1,j); histogram ----------- -- test parallel ChunkAppend :PREFIX SELECT i FROM "test" WHERE length(version()) > 0; --- QUERY PLAN --- Gather Workers Planned: 1 Single Copy: true -> Result One-Time Filter: (length(version()) > 0) -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Seq Scan on _hyper_1_1_chunk -> Result One-Time Filter: (length(version()) > 0) -> Seq Scan on _hyper_1_2_chunk :PREFIX SELECT count(*) FROM "test" WHERE i > 1 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk Filter: (i > 1) -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk SELECT count(*) FROM "test" WHERE i > 1 AND length(version()) > 0; count ------- 99999 -- test parallel ChunkAppend with only work done in the parallel workers SET parallel_leader_participation = off; SELECT count(*) FROM "test" WHERE i > 1 AND length(version()) > 0; count ------- 99999 RESET parallel_leader_participation; -- Test parallel chunk append is used (index scan is disabled to trigger a parallel chunk append) SET parallel_tuple_cost = 0; SET enable_indexscan = OFF; :PREFIX SELECT * FROM (SELECT * FROM "test" WHERE length(version()) > 0 ORDER BY I LIMIT 10) AS t1 LEFT JOIN (SELECT * FROM "test" WHERE i < 500000 ORDER BY I LIMIT 10) AS t2 ON (t1.i = t2.i) ORDER BY t1.i, t2.i; --- QUERY PLAN --- Incremental Sort Sort Key: test.i, _hyper_1_1_chunk_1.i Presorted Key: test.i -> Merge Left Join Merge Cond: (test.i = _hyper_1_1_chunk_1.i) -> Limit -> Gather Merge Workers Planned: 2 -> Sort Sort Key: test.i -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk -> Materialize -> Limit -> Gather Merge Workers Planned: 2 -> Sort Sort Key: _hyper_1_1_chunk_1.i -> Parallel Seq Scan on _hyper_1_1_chunk _hyper_1_1_chunk_1 SELECT * FROM (SELECT * FROM "test" WHERE length(version()) > 0 ORDER BY I LIMIT 10) AS t1 LEFT JOIN (SELECT * FROM "test" WHERE i < 500000 ORDER BY I LIMIT 10) AS t2 ON (t1.i = t2.i) ORDER BY t1.i, t2.i; i | j | ts | i | j | ts ----+------+-----------------------------+----+------+----------------------------- 0 | 0.1 | Wed Dec 31 16:00:00 1969 | 0 | 0.1 | Wed Dec 31 16:00:00 1969 10 | 10.1 | Wed Dec 31 16:00:00.01 1969 | 10 | 10.1 | Wed Dec 31 16:00:00.01 1969 20 | 20.1 | Wed Dec 31 16:00:00.02 1969 | 20 | 20.1 | Wed Dec 31 16:00:00.02 1969 30 | 30.1 | Wed Dec 31 16:00:00.03 1969 | 30 | 30.1 | Wed Dec 31 16:00:00.03 1969 40 | 40.1 | Wed Dec 31 16:00:00.04 1969 | 40 | 40.1 | Wed Dec 31 16:00:00.04 1969 50 | 50.1 | Wed Dec 31 16:00:00.05 1969 | 50 | 50.1 | Wed Dec 31 16:00:00.05 1969 60 | 60.1 | Wed Dec 31 16:00:00.06 1969 | 60 | 60.1 | Wed Dec 31 16:00:00.06 1969 70 | 70.1 | Wed Dec 31 16:00:00.07 1969 | 70 | 70.1 | Wed Dec 31 16:00:00.07 1969 80 | 80.1 | Wed Dec 31 16:00:00.08 1969 | 80 | 80.1 | Wed Dec 31 16:00:00.08 1969 90 | 90.1 | Wed Dec 31 16:00:00.09 1969 | 90 | 90.1 | Wed Dec 31 16:00:00.09 1969 SET enable_indexscan = ON; -- Test normal chunk append can be used in a parallel worker :PREFIX SELECT * FROM (SELECT * FROM "test" WHERE i >= 999000 ORDER BY i) AS t1 JOIN (SELECT * FROM "test" WHERE i >= 400000 ORDER BY i) AS t2 ON (TRUE) ORDER BY t1.i, t2.i LIMIT 10; --- QUERY PLAN --- Gather Workers Planned: 1 Single Copy: true -> Limit -> Incremental Sort Sort Key: _hyper_1_2_chunk.i, test.i Presorted Key: _hyper_1_2_chunk.i -> Nested Loop -> Index Scan Backward using _hyper_1_2_chunk_test_i_idx on _hyper_1_2_chunk Index Cond: (i >= 999000) -> Materialize -> Custom Scan (ChunkAppend) on test Order: test.i -> Index Scan Backward using _hyper_1_1_chunk_test_i_idx on _hyper_1_1_chunk Index Cond: (i >= 400000) -> Index Scan Backward using _hyper_1_2_chunk_test_i_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 SELECT * FROM (SELECT * FROM "test" WHERE i >= 999000 ORDER BY i) AS t1 JOIN (SELECT * FROM "test" WHERE i >= 400000 ORDER BY i) AS t2 ON (TRUE) ORDER BY t1.i, t2.i LIMIT 10; i | j | ts | i | j | ts --------+----------+--------------------------+--------+----------+----------------------------- 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400000 | 400000.1 | Wed Dec 31 16:06:40 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400010 | 400010.1 | Wed Dec 31 16:06:40.01 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400020 | 400020.1 | Wed Dec 31 16:06:40.02 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400030 | 400030.1 | Wed Dec 31 16:06:40.03 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400040 | 400040.1 | Wed Dec 31 16:06:40.04 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400050 | 400050.1 | Wed Dec 31 16:06:40.05 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400060 | 400060.1 | Wed Dec 31 16:06:40.06 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400070 | 400070.1 | Wed Dec 31 16:06:40.07 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400080 | 400080.1 | Wed Dec 31 16:06:40.08 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400090 | 400090.1 | Wed Dec 31 16:06:40.09 1969 -- Test parallel ChunkAppend reinit SET enable_material = off; SET min_parallel_table_scan_size = 0; SET min_parallel_index_scan_size = 0; SET enable_hashjoin = 'off'; SET enable_nestloop = 'off'; CREATE TABLE sensor_data( time timestamptz NOT NULL, sensor_id integer NOT NULL); SELECT FROM create_hypertable(relation=>'sensor_data', time_column_name=> 'time'); -- -- Sensors 1 and 2 INSERT INTO sensor_data SELECT time, sensor_id FROM generate_series('2000-01-01 00:00:30', '2022-01-01 00:00:30', INTERVAL '3 months') AS g1(time), generate_series(1, 2, 1) AS g2(sensor_id) ORDER BY time; -- Sensor 100 INSERT INTO sensor_data SELECT time, 100 as sensor_id FROM generate_series('2000-01-01 00:00:30', '2022-01-01 00:00:30', INTERVAL '1 year') AS g1(time) ORDER BY time; :PREFIX SELECT * FROM sensor_data AS s1 JOIN sensor_data AS s2 ON (TRUE) WHERE s1.time > '2020-01-01 00:00:30'::text::timestamptz AND s2.time > '2020-01-01 00:00:30' AND s2.time < '2021-01-01 00:00:30' AND s1.sensor_id > 50 ORDER BY s2.time, s1.time, s1.sensor_id, s2.sensor_id; --- QUERY PLAN --- Incremental Sort Sort Key: s2."time", s1."time", s1.sensor_id, s2.sensor_id Presorted Key: s2."time" -> Nested Loop -> Custom Scan (ChunkAppend) on sensor_data s2 Order: s2."time" -> Index Scan Backward using _hyper_2_83_chunk_sensor_data_time_idx on _hyper_2_83_chunk s2_1 Index Cond: (("time" > 'Wed Jan 01 00:00:30 2020 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 01 00:00:30 2021 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_2_84_chunk_sensor_data_time_idx on _hyper_2_84_chunk s2_2 -> Index Scan Backward using _hyper_2_85_chunk_sensor_data_time_idx on _hyper_2_85_chunk s2_3 -> Index Scan Backward using _hyper_2_86_chunk_sensor_data_time_idx on _hyper_2_86_chunk s2_4 -> Index Scan Backward using _hyper_2_87_chunk_sensor_data_time_idx on _hyper_2_87_chunk s2_5 Index Cond: (("time" > 'Wed Jan 01 00:00:30 2020 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 01 00:00:30 2021 PST'::timestamp with time zone)) -> Gather Workers Planned: 4 -> Parallel Custom Scan (ChunkAppend) on sensor_data s1 Chunks excluded during startup: 80 -> Parallel Bitmap Heap Scan on _hyper_2_83_chunk s1_1 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_83_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_84_chunk s1_2 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_84_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_85_chunk s1_3 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_85_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_86_chunk s1_4 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_86_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_87_chunk s1_5 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_87_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_88_chunk s1_6 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_88_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_89_chunk s1_7 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_89_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_90_chunk s1_8 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_90_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_91_chunk s1_9 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_91_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -- Check query result SELECT * FROM sensor_data AS s1 JOIN sensor_data AS s2 ON (TRUE) WHERE s1.time > '2020-01-01 00:00:30'::text::timestamptz AND s2.time > '2020-01-01 00:00:30' AND s2.time < '2021-01-01 00:00:30' AND s1.sensor_id > 50 ORDER BY s2.time, s1.time, s1.sensor_id, s2.sensor_id; time | sensor_id | time | sensor_id ------------------------------+-----------+------------------------------+----------- Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 -- Ensure the same result is produced if only the parallel workers have to produce them (i.e., the pstate is reinitialized properly) SET parallel_leader_participation = off; SELECT * FROM sensor_data AS s1 JOIN sensor_data AS s2 ON (TRUE) WHERE s1.time > '2020-01-01 00:00:30'::text::timestamptz AND s2.time > '2020-01-01 00:00:30' AND s2.time < '2021-01-01 00:00:30' AND s1.sensor_id > 50 ORDER BY s2.time, s1.time, s1.sensor_id, s2.sensor_id; time | sensor_id | time | sensor_id ------------------------------+-----------+------------------------------+----------- Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 RESET parallel_leader_participation; -- Ensure the same query result is produced by a sequencial query SET max_parallel_workers_per_gather TO 0; SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'off', false); set_config ------------ off SELECT * FROM sensor_data AS s1 JOIN sensor_data AS s2 ON (TRUE) WHERE s1.time > '2020-01-01 00:00:30'::text::timestamptz AND s2.time > '2020-01-01 00:00:30' AND s2.time < '2021-01-01 00:00:30' AND s1.sensor_id > 50 ORDER BY s2.time, s1.time, s1.sensor_id, s2.sensor_id; time | sensor_id | time | sensor_id ------------------------------+-----------+------------------------------+----------- Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 RESET enable_material; RESET min_parallel_table_scan_size; RESET min_parallel_index_scan_size; RESET enable_hashjoin; RESET enable_nestloop; RESET parallel_tuple_cost; SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false); set_config ------------ on -- test worker assignment -- first chunk should have 1 worker and second chunk should have 2 SET max_parallel_workers_per_gather TO 2; :PREFIX SELECT count(*) FROM "test" WHERE i >= 400000 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Index Only Scan using _hyper_1_1_chunk_test_i_idx on _hyper_1_1_chunk Index Cond: (i >= 400000) -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk SELECT count(*) FROM "test" WHERE i >= 400000 AND length(version()) > 0; count ------- 60000 -- test worker assignment -- first chunk should have 2 worker and second chunk should have 1 :PREFIX SELECT count(*) FROM "test" WHERE i < 600000 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Index Only Scan using _hyper_1_2_chunk_test_i_idx on _hyper_1_2_chunk Index Cond: (i < 600000) -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk SELECT count(*) FROM "test" WHERE i < 600000 AND length(version()) > 0; count ------- 60000 -- test ChunkAppend with # workers < # childs SET max_parallel_workers_per_gather TO 1; :PREFIX SELECT count(*) FROM "test" WHERE length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 1 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk SELECT count(*) FROM "test" WHERE length(version()) > 0; count -------- 100000 -- test ChunkAppend with # workers > # childs SET max_parallel_workers_per_gather TO 2; :PREFIX SELECT count(*) FROM "test" WHERE i >= 500000 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk SELECT count(*) FROM "test" WHERE i >= 500000 AND length(version()) > 0; count ------- 50000 RESET max_parallel_workers_per_gather; -- test partial and non-partial plans -- these will not be parallel on PG < 11 ALTER TABLE :CHUNK1 SET (parallel_workers=0); ALTER TABLE :CHUNK2 SET (parallel_workers=2); :PREFIX SELECT count(*) FROM "test" WHERE i > 400000 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Index Only Scan using _hyper_1_1_chunk_test_i_idx on _hyper_1_1_chunk Index Cond: (i > 400000) -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk ALTER TABLE :CHUNK1 SET (parallel_workers=2); ALTER TABLE :CHUNK2 SET (parallel_workers=0); :PREFIX SELECT count(*) FROM "test" WHERE i < 600000 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Index Only Scan using _hyper_1_2_chunk_test_i_idx on _hyper_1_2_chunk Index Cond: (i < 600000) -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk ALTER TABLE :CHUNK1 RESET (parallel_workers); ALTER TABLE :CHUNK2 RESET (parallel_workers); -- now() is not marked parallel safe in PostgreSQL < 12 so using now() -- in a query will prevent parallelism but CURRENT_TIMESTAMP and -- transaction_timestamp() are marked parallel safe :PREFIX SELECT i FROM "test" WHERE ts < CURRENT_TIMESTAMP; --- QUERY PLAN --- Gather Workers Planned: 1 Single Copy: true -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk Filter: (ts < CURRENT_TIMESTAMP) -> Seq Scan on _hyper_1_2_chunk Filter: (ts < CURRENT_TIMESTAMP) :PREFIX SELECT i FROM "test" WHERE ts < transaction_timestamp(); --- QUERY PLAN --- Gather Workers Planned: 1 Single Copy: true -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk Filter: (ts < transaction_timestamp()) -> Seq Scan on _hyper_1_2_chunk Filter: (ts < transaction_timestamp()) -- this won't be parallel query because now() is parallel restricted in PG < 12 :PREFIX SELECT i FROM "test" WHERE ts < now(); --- QUERY PLAN --- Gather Workers Planned: 1 Single Copy: true -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk Filter: (ts < now()) -> Seq Scan on _hyper_1_2_chunk Filter: (ts < now()) ================================================ FILE: test/expected/parallel-17.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --parallel queries require big-ish tables so collect them all here --so that we need to generate queries only once. -- output with analyze is not stable because it depends on worker assignment \set PREFIX 'EXPLAIN (buffers off, costs off)' \set CHUNK1 _timescaledb_internal._hyper_1_1_chunk \set CHUNK2 _timescaledb_internal._hyper_1_2_chunk CREATE TABLE test (i int, j double precision, ts timestamp); SELECT create_hypertable('test','i',chunk_time_interval:=500000); WARNING: column type "timestamp without time zone" used for "ts" does not follow best practices create_hypertable ------------------- (1,public,test,t) INSERT INTO test SELECT x, x+0.1, _timescaledb_functions.to_timestamp(x*1000) FROM generate_series(0,1000000-1,10) AS x; ANALYZE test; ALTER TABLE :CHUNK1 SET (parallel_workers=2); ALTER TABLE :CHUNK2 SET (parallel_workers=2); SET work_mem TO '50MB'; SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false); set_config ------------ on SET max_parallel_workers_per_gather = 4; SET parallel_setup_cost TO 0; EXPLAIN (buffers off, costs off) SELECT first(i, j) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT first(i, j) FROM "test"; first ------- 0 EXPLAIN (buffers off, costs off) SELECT last(i, j) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT last(i, j) FROM "test"; last -------- 999990 EXPLAIN (buffers off, costs off) SELECT time_bucket('1 second', ts) sec, last(i, j) FROM "test" GROUP BY sec ORDER BY sec LIMIT 5; --- QUERY PLAN --- Limit -> Finalize GroupAggregate Group Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Gather Merge Workers Planned: 2 -> Partial GroupAggregate Group Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Sort Sort Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Result -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk -- test single copy parallel plan with parallel chunk append :PREFIX SELECT time_bucket('1 second', ts) sec, last(i, j) FROM "test" WHERE length(version()) > 0 GROUP BY sec ORDER BY sec LIMIT 5; --- QUERY PLAN --- Limit -> Finalize GroupAggregate Group Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Gather Merge Workers Planned: 2 -> Partial GroupAggregate Group Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Sort Sort Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Result -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk SELECT time_bucket('1 second', ts) sec, last(i, j) FROM "test" GROUP BY sec ORDER BY sec LIMIT 5; sec | last --------------------------+------ Wed Dec 31 16:00:00 1969 | 990 Wed Dec 31 16:00:01 1969 | 1990 Wed Dec 31 16:00:02 1969 | 2990 Wed Dec 31 16:00:03 1969 | 3990 Wed Dec 31 16:00:04 1969 | 4990 --test variants of histogram EXPLAIN (buffers off, costs off) SELECT histogram(i, 1, 1000000, 2) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT histogram(i, 1, 1000000, 2) FROM "test"; histogram ------------------- {1,50000,49999,0} EXPLAIN (buffers off, costs off) SELECT histogram(i, 1,1000001,10) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT histogram(i, 1, 1000001, 10) FROM "test"; histogram ------------------------------------------------------------------ {1,10000,10000,10000,10000,10000,10000,10000,10000,10000,9999,0} EXPLAIN (buffers off, costs off) SELECT histogram(i, 0,100000,5) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT histogram(i, 0, 100000, 5) FROM "test"; histogram ------------------------------------ {0,2000,2000,2000,2000,2000,90000} EXPLAIN (buffers off, costs off) SELECT histogram(i, 10,100000,5) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT histogram(i, 10, 100000, 5) FROM "test"; histogram ------------------------------------ {1,2000,2000,2000,2000,1999,90000} EXPLAIN (buffers off, costs off) SELECT histogram(NULL, 10,100000,5) FROM "test" WHERE i = coalesce(-1,j); --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk Filter: ((i)::double precision = '-1'::double precision) -> Parallel Seq Scan on _hyper_1_2_chunk Filter: ((i)::double precision = '-1'::double precision) SELECT histogram(NULL, 10,100000,5) FROM "test" WHERE i = coalesce(-1,j); histogram ----------- -- test parallel ChunkAppend :PREFIX SELECT i FROM "test" WHERE length(version()) > 0; --- QUERY PLAN --- Gather Workers Planned: 1 Single Copy: true -> Result One-Time Filter: (length(version()) > 0) -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Seq Scan on _hyper_1_1_chunk -> Result One-Time Filter: (length(version()) > 0) -> Seq Scan on _hyper_1_2_chunk :PREFIX SELECT count(*) FROM "test" WHERE i > 1 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk Filter: (i > 1) -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk SELECT count(*) FROM "test" WHERE i > 1 AND length(version()) > 0; count ------- 99999 -- test parallel ChunkAppend with only work done in the parallel workers SET parallel_leader_participation = off; SELECT count(*) FROM "test" WHERE i > 1 AND length(version()) > 0; count ------- 99999 RESET parallel_leader_participation; -- Test parallel chunk append is used (index scan is disabled to trigger a parallel chunk append) SET parallel_tuple_cost = 0; SET enable_indexscan = OFF; :PREFIX SELECT * FROM (SELECT * FROM "test" WHERE length(version()) > 0 ORDER BY I LIMIT 10) AS t1 LEFT JOIN (SELECT * FROM "test" WHERE i < 500000 ORDER BY I LIMIT 10) AS t2 ON (t1.i = t2.i) ORDER BY t1.i, t2.i; --- QUERY PLAN --- Incremental Sort Sort Key: test.i, _hyper_1_1_chunk_1.i Presorted Key: test.i -> Merge Left Join Merge Cond: (test.i = _hyper_1_1_chunk_1.i) -> Limit -> Gather Merge Workers Planned: 2 -> Sort Sort Key: test.i -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk -> Materialize -> Limit -> Gather Merge Workers Planned: 2 -> Sort Sort Key: _hyper_1_1_chunk_1.i -> Parallel Seq Scan on _hyper_1_1_chunk _hyper_1_1_chunk_1 SELECT * FROM (SELECT * FROM "test" WHERE length(version()) > 0 ORDER BY I LIMIT 10) AS t1 LEFT JOIN (SELECT * FROM "test" WHERE i < 500000 ORDER BY I LIMIT 10) AS t2 ON (t1.i = t2.i) ORDER BY t1.i, t2.i; i | j | ts | i | j | ts ----+------+-----------------------------+----+------+----------------------------- 0 | 0.1 | Wed Dec 31 16:00:00 1969 | 0 | 0.1 | Wed Dec 31 16:00:00 1969 10 | 10.1 | Wed Dec 31 16:00:00.01 1969 | 10 | 10.1 | Wed Dec 31 16:00:00.01 1969 20 | 20.1 | Wed Dec 31 16:00:00.02 1969 | 20 | 20.1 | Wed Dec 31 16:00:00.02 1969 30 | 30.1 | Wed Dec 31 16:00:00.03 1969 | 30 | 30.1 | Wed Dec 31 16:00:00.03 1969 40 | 40.1 | Wed Dec 31 16:00:00.04 1969 | 40 | 40.1 | Wed Dec 31 16:00:00.04 1969 50 | 50.1 | Wed Dec 31 16:00:00.05 1969 | 50 | 50.1 | Wed Dec 31 16:00:00.05 1969 60 | 60.1 | Wed Dec 31 16:00:00.06 1969 | 60 | 60.1 | Wed Dec 31 16:00:00.06 1969 70 | 70.1 | Wed Dec 31 16:00:00.07 1969 | 70 | 70.1 | Wed Dec 31 16:00:00.07 1969 80 | 80.1 | Wed Dec 31 16:00:00.08 1969 | 80 | 80.1 | Wed Dec 31 16:00:00.08 1969 90 | 90.1 | Wed Dec 31 16:00:00.09 1969 | 90 | 90.1 | Wed Dec 31 16:00:00.09 1969 SET enable_indexscan = ON; -- Test normal chunk append can be used in a parallel worker :PREFIX SELECT * FROM (SELECT * FROM "test" WHERE i >= 999000 ORDER BY i) AS t1 JOIN (SELECT * FROM "test" WHERE i >= 400000 ORDER BY i) AS t2 ON (TRUE) ORDER BY t1.i, t2.i LIMIT 10; --- QUERY PLAN --- Gather Workers Planned: 1 Single Copy: true -> Limit -> Incremental Sort Sort Key: _hyper_1_2_chunk.i, test.i Presorted Key: _hyper_1_2_chunk.i -> Nested Loop -> Index Scan Backward using _hyper_1_2_chunk_test_i_idx on _hyper_1_2_chunk Index Cond: (i >= 999000) -> Materialize -> Custom Scan (ChunkAppend) on test Order: test.i -> Index Scan Backward using _hyper_1_1_chunk_test_i_idx on _hyper_1_1_chunk Index Cond: (i >= 400000) -> Index Scan Backward using _hyper_1_2_chunk_test_i_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 SELECT * FROM (SELECT * FROM "test" WHERE i >= 999000 ORDER BY i) AS t1 JOIN (SELECT * FROM "test" WHERE i >= 400000 ORDER BY i) AS t2 ON (TRUE) ORDER BY t1.i, t2.i LIMIT 10; i | j | ts | i | j | ts --------+----------+--------------------------+--------+----------+----------------------------- 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400000 | 400000.1 | Wed Dec 31 16:06:40 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400010 | 400010.1 | Wed Dec 31 16:06:40.01 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400020 | 400020.1 | Wed Dec 31 16:06:40.02 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400030 | 400030.1 | Wed Dec 31 16:06:40.03 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400040 | 400040.1 | Wed Dec 31 16:06:40.04 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400050 | 400050.1 | Wed Dec 31 16:06:40.05 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400060 | 400060.1 | Wed Dec 31 16:06:40.06 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400070 | 400070.1 | Wed Dec 31 16:06:40.07 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400080 | 400080.1 | Wed Dec 31 16:06:40.08 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400090 | 400090.1 | Wed Dec 31 16:06:40.09 1969 -- Test parallel ChunkAppend reinit SET enable_material = off; SET min_parallel_table_scan_size = 0; SET min_parallel_index_scan_size = 0; SET enable_hashjoin = 'off'; SET enable_nestloop = 'off'; CREATE TABLE sensor_data( time timestamptz NOT NULL, sensor_id integer NOT NULL); SELECT FROM create_hypertable(relation=>'sensor_data', time_column_name=> 'time'); -- -- Sensors 1 and 2 INSERT INTO sensor_data SELECT time, sensor_id FROM generate_series('2000-01-01 00:00:30', '2022-01-01 00:00:30', INTERVAL '3 months') AS g1(time), generate_series(1, 2, 1) AS g2(sensor_id) ORDER BY time; -- Sensor 100 INSERT INTO sensor_data SELECT time, 100 as sensor_id FROM generate_series('2000-01-01 00:00:30', '2022-01-01 00:00:30', INTERVAL '1 year') AS g1(time) ORDER BY time; :PREFIX SELECT * FROM sensor_data AS s1 JOIN sensor_data AS s2 ON (TRUE) WHERE s1.time > '2020-01-01 00:00:30'::text::timestamptz AND s2.time > '2020-01-01 00:00:30' AND s2.time < '2021-01-01 00:00:30' AND s1.sensor_id > 50 ORDER BY s2.time, s1.time, s1.sensor_id, s2.sensor_id; --- QUERY PLAN --- Incremental Sort Sort Key: s2."time", s1."time", s1.sensor_id, s2.sensor_id Presorted Key: s2."time" -> Nested Loop -> Custom Scan (ChunkAppend) on sensor_data s2 Order: s2."time" -> Index Scan Backward using _hyper_2_83_chunk_sensor_data_time_idx on _hyper_2_83_chunk s2_1 Index Cond: (("time" > 'Wed Jan 01 00:00:30 2020 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 01 00:00:30 2021 PST'::timestamp with time zone)) -> Index Scan Backward using _hyper_2_84_chunk_sensor_data_time_idx on _hyper_2_84_chunk s2_2 -> Index Scan Backward using _hyper_2_85_chunk_sensor_data_time_idx on _hyper_2_85_chunk s2_3 -> Index Scan Backward using _hyper_2_86_chunk_sensor_data_time_idx on _hyper_2_86_chunk s2_4 -> Index Scan Backward using _hyper_2_87_chunk_sensor_data_time_idx on _hyper_2_87_chunk s2_5 Index Cond: (("time" > 'Wed Jan 01 00:00:30 2020 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 01 00:00:30 2021 PST'::timestamp with time zone)) -> Gather Workers Planned: 4 -> Parallel Custom Scan (ChunkAppend) on sensor_data s1 Chunks excluded during startup: 80 -> Parallel Bitmap Heap Scan on _hyper_2_83_chunk s1_1 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_83_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_84_chunk s1_2 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_84_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_85_chunk s1_3 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_85_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_86_chunk s1_4 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_86_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_87_chunk s1_5 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_87_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_88_chunk s1_6 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_88_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_89_chunk s1_7 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_89_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_90_chunk s1_8 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_90_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -> Parallel Bitmap Heap Scan on _hyper_2_91_chunk s1_9 Recheck Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Bitmap Index Scan on _hyper_2_91_chunk_sensor_data_time_idx Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) -- Check query result SELECT * FROM sensor_data AS s1 JOIN sensor_data AS s2 ON (TRUE) WHERE s1.time > '2020-01-01 00:00:30'::text::timestamptz AND s2.time > '2020-01-01 00:00:30' AND s2.time < '2021-01-01 00:00:30' AND s1.sensor_id > 50 ORDER BY s2.time, s1.time, s1.sensor_id, s2.sensor_id; time | sensor_id | time | sensor_id ------------------------------+-----------+------------------------------+----------- Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 -- Ensure the same result is produced if only the parallel workers have to produce them (i.e., the pstate is reinitialized properly) SET parallel_leader_participation = off; SELECT * FROM sensor_data AS s1 JOIN sensor_data AS s2 ON (TRUE) WHERE s1.time > '2020-01-01 00:00:30'::text::timestamptz AND s2.time > '2020-01-01 00:00:30' AND s2.time < '2021-01-01 00:00:30' AND s1.sensor_id > 50 ORDER BY s2.time, s1.time, s1.sensor_id, s2.sensor_id; time | sensor_id | time | sensor_id ------------------------------+-----------+------------------------------+----------- Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 RESET parallel_leader_participation; -- Ensure the same query result is produced by a sequencial query SET max_parallel_workers_per_gather TO 0; SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'off', false); set_config ------------ off SELECT * FROM sensor_data AS s1 JOIN sensor_data AS s2 ON (TRUE) WHERE s1.time > '2020-01-01 00:00:30'::text::timestamptz AND s2.time > '2020-01-01 00:00:30' AND s2.time < '2021-01-01 00:00:30' AND s1.sensor_id > 50 ORDER BY s2.time, s1.time, s1.sensor_id, s2.sensor_id; time | sensor_id | time | sensor_id ------------------------------+-----------+------------------------------+----------- Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 RESET enable_material; RESET min_parallel_table_scan_size; RESET min_parallel_index_scan_size; RESET enable_hashjoin; RESET enable_nestloop; RESET parallel_tuple_cost; SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false); set_config ------------ on -- test worker assignment -- first chunk should have 1 worker and second chunk should have 2 SET max_parallel_workers_per_gather TO 2; :PREFIX SELECT count(*) FROM "test" WHERE i >= 400000 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Index Only Scan using _hyper_1_1_chunk_test_i_idx on _hyper_1_1_chunk Index Cond: (i >= 400000) -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk SELECT count(*) FROM "test" WHERE i >= 400000 AND length(version()) > 0; count ------- 60000 -- test worker assignment -- first chunk should have 2 worker and second chunk should have 1 :PREFIX SELECT count(*) FROM "test" WHERE i < 600000 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Index Only Scan using _hyper_1_2_chunk_test_i_idx on _hyper_1_2_chunk Index Cond: (i < 600000) -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk SELECT count(*) FROM "test" WHERE i < 600000 AND length(version()) > 0; count ------- 60000 -- test ChunkAppend with # workers < # childs SET max_parallel_workers_per_gather TO 1; :PREFIX SELECT count(*) FROM "test" WHERE length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 1 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk SELECT count(*) FROM "test" WHERE length(version()) > 0; count -------- 100000 -- test ChunkAppend with # workers > # childs SET max_parallel_workers_per_gather TO 2; :PREFIX SELECT count(*) FROM "test" WHERE i >= 500000 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk SELECT count(*) FROM "test" WHERE i >= 500000 AND length(version()) > 0; count ------- 50000 RESET max_parallel_workers_per_gather; -- test partial and non-partial plans -- these will not be parallel on PG < 11 ALTER TABLE :CHUNK1 SET (parallel_workers=0); ALTER TABLE :CHUNK2 SET (parallel_workers=2); :PREFIX SELECT count(*) FROM "test" WHERE i > 400000 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Index Only Scan using _hyper_1_1_chunk_test_i_idx on _hyper_1_1_chunk Index Cond: (i > 400000) -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk ALTER TABLE :CHUNK1 SET (parallel_workers=2); ALTER TABLE :CHUNK2 SET (parallel_workers=0); :PREFIX SELECT count(*) FROM "test" WHERE i < 600000 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Index Only Scan using _hyper_1_2_chunk_test_i_idx on _hyper_1_2_chunk Index Cond: (i < 600000) -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk ALTER TABLE :CHUNK1 RESET (parallel_workers); ALTER TABLE :CHUNK2 RESET (parallel_workers); -- now() is not marked parallel safe in PostgreSQL < 12 so using now() -- in a query will prevent parallelism but CURRENT_TIMESTAMP and -- transaction_timestamp() are marked parallel safe :PREFIX SELECT i FROM "test" WHERE ts < CURRENT_TIMESTAMP; --- QUERY PLAN --- Gather Workers Planned: 1 Single Copy: true -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk Filter: (ts < CURRENT_TIMESTAMP) -> Seq Scan on _hyper_1_2_chunk Filter: (ts < CURRENT_TIMESTAMP) :PREFIX SELECT i FROM "test" WHERE ts < transaction_timestamp(); --- QUERY PLAN --- Gather Workers Planned: 1 Single Copy: true -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk Filter: (ts < transaction_timestamp()) -> Seq Scan on _hyper_1_2_chunk Filter: (ts < transaction_timestamp()) -- this won't be parallel query because now() is parallel restricted in PG < 12 :PREFIX SELECT i FROM "test" WHERE ts < now(); --- QUERY PLAN --- Gather Workers Planned: 1 Single Copy: true -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk Filter: (ts < now()) -> Seq Scan on _hyper_1_2_chunk Filter: (ts < now()) ================================================ FILE: test/expected/parallel-18.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --parallel queries require big-ish tables so collect them all here --so that we need to generate queries only once. -- output with analyze is not stable because it depends on worker assignment \set PREFIX 'EXPLAIN (buffers off, costs off)' \set CHUNK1 _timescaledb_internal._hyper_1_1_chunk \set CHUNK2 _timescaledb_internal._hyper_1_2_chunk CREATE TABLE test (i int, j double precision, ts timestamp); SELECT create_hypertable('test','i',chunk_time_interval:=500000); WARNING: column type "timestamp without time zone" used for "ts" does not follow best practices create_hypertable ------------------- (1,public,test,t) INSERT INTO test SELECT x, x+0.1, _timescaledb_functions.to_timestamp(x*1000) FROM generate_series(0,1000000-1,10) AS x; ANALYZE test; ALTER TABLE :CHUNK1 SET (parallel_workers=2); ALTER TABLE :CHUNK2 SET (parallel_workers=2); SET work_mem TO '50MB'; SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false); set_config ------------ on SET max_parallel_workers_per_gather = 4; SET parallel_setup_cost TO 0; EXPLAIN (buffers off, costs off) SELECT first(i, j) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT first(i, j) FROM "test"; first ------- 0 EXPLAIN (buffers off, costs off) SELECT last(i, j) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT last(i, j) FROM "test"; last -------- 999990 EXPLAIN (buffers off, costs off) SELECT time_bucket('1 second', ts) sec, last(i, j) FROM "test" GROUP BY sec ORDER BY sec LIMIT 5; --- QUERY PLAN --- Limit -> GroupAggregate Group Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Gather Merge Workers Planned: 2 -> Sort Sort Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Result -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk -- test single copy parallel plan with parallel chunk append :PREFIX SELECT time_bucket('1 second', ts) sec, last(i, j) FROM "test" WHERE length(version()) > 0 GROUP BY sec ORDER BY sec LIMIT 5; --- QUERY PLAN --- Limit -> GroupAggregate Group Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Gather Merge Workers Planned: 2 -> Sort Sort Key: (time_bucket('@ 1 sec'::interval, test.ts)) -> Result -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk SELECT time_bucket('1 second', ts) sec, last(i, j) FROM "test" GROUP BY sec ORDER BY sec LIMIT 5; sec | last --------------------------+------ Wed Dec 31 16:00:00 1969 | 990 Wed Dec 31 16:00:01 1969 | 1990 Wed Dec 31 16:00:02 1969 | 2990 Wed Dec 31 16:00:03 1969 | 3990 Wed Dec 31 16:00:04 1969 | 4990 --test variants of histogram EXPLAIN (buffers off, costs off) SELECT histogram(i, 1, 1000000, 2) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT histogram(i, 1, 1000000, 2) FROM "test"; histogram ------------------- {1,50000,49999,0} EXPLAIN (buffers off, costs off) SELECT histogram(i, 1,1000001,10) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT histogram(i, 1, 1000001, 10) FROM "test"; histogram ------------------------------------------------------------------ {1,10000,10000,10000,10000,10000,10000,10000,10000,10000,9999,0} EXPLAIN (buffers off, costs off) SELECT histogram(i, 0,100000,5) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT histogram(i, 0, 100000, 5) FROM "test"; histogram ------------------------------------ {0,2000,2000,2000,2000,2000,90000} EXPLAIN (buffers off, costs off) SELECT histogram(i, 10,100000,5) FROM "test"; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk -> Parallel Seq Scan on _hyper_1_2_chunk SELECT histogram(i, 10, 100000, 5) FROM "test"; histogram ------------------------------------ {1,2000,2000,2000,2000,1999,90000} EXPLAIN (buffers off, costs off) SELECT histogram(NULL, 10,100000,5) FROM "test" WHERE i = coalesce(-1,j); --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Parallel Append -> Parallel Seq Scan on _hyper_1_1_chunk Filter: ((i)::double precision = '-1'::double precision) -> Parallel Seq Scan on _hyper_1_2_chunk Filter: ((i)::double precision = '-1'::double precision) SELECT histogram(NULL, 10,100000,5) FROM "test" WHERE i = coalesce(-1,j); histogram ----------- -- test parallel ChunkAppend :PREFIX SELECT i FROM "test" WHERE length(version()) > 0; --- QUERY PLAN --- Gather Workers Planned: 1 Single Copy: true -> Result One-Time Filter: (length(version()) > 0) -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Seq Scan on _hyper_1_1_chunk -> Result One-Time Filter: (length(version()) > 0) -> Seq Scan on _hyper_1_2_chunk :PREFIX SELECT count(*) FROM "test" WHERE i > 1 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk Filter: (i > 1) -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk SELECT count(*) FROM "test" WHERE i > 1 AND length(version()) > 0; count ------- 99999 -- test parallel ChunkAppend with only work done in the parallel workers SET parallel_leader_participation = off; SELECT count(*) FROM "test" WHERE i > 1 AND length(version()) > 0; count ------- 99999 RESET parallel_leader_participation; -- Test parallel chunk append is used (index scan is disabled to trigger a parallel chunk append) SET parallel_tuple_cost = 0; SET enable_indexscan = OFF; :PREFIX SELECT * FROM (SELECT * FROM "test" WHERE length(version()) > 0 ORDER BY I LIMIT 10) AS t1 LEFT JOIN (SELECT * FROM "test" WHERE i < 500000 ORDER BY I LIMIT 10) AS t2 ON (t1.i = t2.i) ORDER BY t1.i, t2.i; --- QUERY PLAN --- Incremental Sort Sort Key: test.i, _hyper_1_1_chunk_1.i Presorted Key: test.i -> Merge Left Join Merge Cond: (test.i = _hyper_1_1_chunk_1.i) -> Limit -> Result One-Time Filter: (length(version()) > 0) -> Custom Scan (ChunkAppend) on test Order: test.i Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Index Scan Backward using _hyper_1_1_chunk_test_i_idx on _hyper_1_1_chunk -> Result One-Time Filter: (length(version()) > 0) -> Index Scan Backward using _hyper_1_2_chunk_test_i_idx on _hyper_1_2_chunk -> Materialize -> Limit -> Gather Merge Workers Planned: 2 -> Sort Sort Key: _hyper_1_1_chunk_1.i -> Parallel Seq Scan on _hyper_1_1_chunk _hyper_1_1_chunk_1 SELECT * FROM (SELECT * FROM "test" WHERE length(version()) > 0 ORDER BY I LIMIT 10) AS t1 LEFT JOIN (SELECT * FROM "test" WHERE i < 500000 ORDER BY I LIMIT 10) AS t2 ON (t1.i = t2.i) ORDER BY t1.i, t2.i; i | j | ts | i | j | ts ----+------+-----------------------------+----+------+----------------------------- 0 | 0.1 | Wed Dec 31 16:00:00 1969 | 0 | 0.1 | Wed Dec 31 16:00:00 1969 10 | 10.1 | Wed Dec 31 16:00:00.01 1969 | 10 | 10.1 | Wed Dec 31 16:00:00.01 1969 20 | 20.1 | Wed Dec 31 16:00:00.02 1969 | 20 | 20.1 | Wed Dec 31 16:00:00.02 1969 30 | 30.1 | Wed Dec 31 16:00:00.03 1969 | 30 | 30.1 | Wed Dec 31 16:00:00.03 1969 40 | 40.1 | Wed Dec 31 16:00:00.04 1969 | 40 | 40.1 | Wed Dec 31 16:00:00.04 1969 50 | 50.1 | Wed Dec 31 16:00:00.05 1969 | 50 | 50.1 | Wed Dec 31 16:00:00.05 1969 60 | 60.1 | Wed Dec 31 16:00:00.06 1969 | 60 | 60.1 | Wed Dec 31 16:00:00.06 1969 70 | 70.1 | Wed Dec 31 16:00:00.07 1969 | 70 | 70.1 | Wed Dec 31 16:00:00.07 1969 80 | 80.1 | Wed Dec 31 16:00:00.08 1969 | 80 | 80.1 | Wed Dec 31 16:00:00.08 1969 90 | 90.1 | Wed Dec 31 16:00:00.09 1969 | 90 | 90.1 | Wed Dec 31 16:00:00.09 1969 SET enable_indexscan = ON; -- Test normal chunk append can be used in a parallel worker :PREFIX SELECT * FROM (SELECT * FROM "test" WHERE i >= 999000 ORDER BY i) AS t1 JOIN (SELECT * FROM "test" WHERE i >= 400000 ORDER BY i) AS t2 ON (TRUE) ORDER BY t1.i, t2.i LIMIT 10; --- QUERY PLAN --- Gather Workers Planned: 1 Single Copy: true -> Limit -> Incremental Sort Sort Key: _hyper_1_2_chunk.i, test.i Presorted Key: _hyper_1_2_chunk.i -> Nested Loop -> Index Scan Backward using _hyper_1_2_chunk_test_i_idx on _hyper_1_2_chunk Index Cond: (i >= 999000) -> Materialize -> Custom Scan (ChunkAppend) on test Order: test.i -> Index Scan Backward using _hyper_1_1_chunk_test_i_idx on _hyper_1_1_chunk Index Cond: (i >= 400000) -> Index Scan Backward using _hyper_1_2_chunk_test_i_idx on _hyper_1_2_chunk _hyper_1_2_chunk_1 SELECT * FROM (SELECT * FROM "test" WHERE i >= 999000 ORDER BY i) AS t1 JOIN (SELECT * FROM "test" WHERE i >= 400000 ORDER BY i) AS t2 ON (TRUE) ORDER BY t1.i, t2.i LIMIT 10; i | j | ts | i | j | ts --------+----------+--------------------------+--------+----------+----------------------------- 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400000 | 400000.1 | Wed Dec 31 16:06:40 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400010 | 400010.1 | Wed Dec 31 16:06:40.01 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400020 | 400020.1 | Wed Dec 31 16:06:40.02 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400030 | 400030.1 | Wed Dec 31 16:06:40.03 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400040 | 400040.1 | Wed Dec 31 16:06:40.04 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400050 | 400050.1 | Wed Dec 31 16:06:40.05 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400060 | 400060.1 | Wed Dec 31 16:06:40.06 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400070 | 400070.1 | Wed Dec 31 16:06:40.07 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400080 | 400080.1 | Wed Dec 31 16:06:40.08 1969 999000 | 999000.1 | Wed Dec 31 16:16:39 1969 | 400090 | 400090.1 | Wed Dec 31 16:06:40.09 1969 -- Test parallel ChunkAppend reinit SET enable_material = off; SET min_parallel_table_scan_size = 0; SET min_parallel_index_scan_size = 0; SET enable_hashjoin = 'off'; SET enable_nestloop = 'off'; CREATE TABLE sensor_data( time timestamptz NOT NULL, sensor_id integer NOT NULL); SELECT FROM create_hypertable(relation=>'sensor_data', time_column_name=> 'time'); -- -- Sensors 1 and 2 INSERT INTO sensor_data SELECT time, sensor_id FROM generate_series('2000-01-01 00:00:30', '2022-01-01 00:00:30', INTERVAL '3 months') AS g1(time), generate_series(1, 2, 1) AS g2(sensor_id) ORDER BY time; -- Sensor 100 INSERT INTO sensor_data SELECT time, 100 as sensor_id FROM generate_series('2000-01-01 00:00:30', '2022-01-01 00:00:30', INTERVAL '1 year') AS g1(time) ORDER BY time; :PREFIX SELECT * FROM sensor_data AS s1 JOIN sensor_data AS s2 ON (TRUE) WHERE s1.time > '2020-01-01 00:00:30'::text::timestamptz AND s2.time > '2020-01-01 00:00:30' AND s2.time < '2021-01-01 00:00:30' AND s1.sensor_id > 50 ORDER BY s2.time, s1.time, s1.sensor_id, s2.sensor_id; --- QUERY PLAN --- Sort Sort Key: s2."time", s1."time", s1.sensor_id, s2.sensor_id -> Nested Loop -> Custom Scan (ChunkAppend) on sensor_data s1 Chunks excluded during startup: 80 -> Index Scan using _hyper_2_83_chunk_sensor_data_time_idx on _hyper_2_83_chunk s1_1 Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Index Scan using _hyper_2_84_chunk_sensor_data_time_idx on _hyper_2_84_chunk s1_2 Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Index Scan using _hyper_2_85_chunk_sensor_data_time_idx on _hyper_2_85_chunk s1_3 Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Index Scan using _hyper_2_86_chunk_sensor_data_time_idx on _hyper_2_86_chunk s1_4 Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Index Scan using _hyper_2_87_chunk_sensor_data_time_idx on _hyper_2_87_chunk s1_5 Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Index Scan using _hyper_2_88_chunk_sensor_data_time_idx on _hyper_2_88_chunk s1_6 Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Index Scan using _hyper_2_89_chunk_sensor_data_time_idx on _hyper_2_89_chunk s1_7 Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Index Scan using _hyper_2_90_chunk_sensor_data_time_idx on _hyper_2_90_chunk s1_8 Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Index Scan using _hyper_2_91_chunk_sensor_data_time_idx on _hyper_2_91_chunk s1_9 Index Cond: ("time" > ('2020-01-01 00:00:30'::cstring)::timestamp with time zone) Filter: (sensor_id > 50) -> Gather Workers Planned: 3 -> Parallel Append -> Parallel Index Scan Backward using _hyper_2_83_chunk_sensor_data_time_idx on _hyper_2_83_chunk s2_1 Index Cond: (("time" > 'Wed Jan 01 00:00:30 2020 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 01 00:00:30 2021 PST'::timestamp with time zone)) -> Parallel Index Scan Backward using _hyper_2_87_chunk_sensor_data_time_idx on _hyper_2_87_chunk s2_5 Index Cond: (("time" > 'Wed Jan 01 00:00:30 2020 PST'::timestamp with time zone) AND ("time" < 'Fri Jan 01 00:00:30 2021 PST'::timestamp with time zone)) -> Parallel Seq Scan on _hyper_2_84_chunk s2_2 -> Parallel Seq Scan on _hyper_2_85_chunk s2_3 -> Parallel Seq Scan on _hyper_2_86_chunk s2_4 -- Check query result SELECT * FROM sensor_data AS s1 JOIN sensor_data AS s2 ON (TRUE) WHERE s1.time > '2020-01-01 00:00:30'::text::timestamptz AND s2.time > '2020-01-01 00:00:30' AND s2.time < '2021-01-01 00:00:30' AND s1.sensor_id > 50 ORDER BY s2.time, s1.time, s1.sensor_id, s2.sensor_id; time | sensor_id | time | sensor_id ------------------------------+-----------+------------------------------+----------- Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 -- Ensure the same result is produced if only the parallel workers have to produce them (i.e., the pstate is reinitialized properly) SET parallel_leader_participation = off; SELECT * FROM sensor_data AS s1 JOIN sensor_data AS s2 ON (TRUE) WHERE s1.time > '2020-01-01 00:00:30'::text::timestamptz AND s2.time > '2020-01-01 00:00:30' AND s2.time < '2021-01-01 00:00:30' AND s1.sensor_id > 50 ORDER BY s2.time, s1.time, s1.sensor_id, s2.sensor_id; time | sensor_id | time | sensor_id ------------------------------+-----------+------------------------------+----------- Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 RESET parallel_leader_participation; -- Ensure the same query result is produced by a sequencial query SET max_parallel_workers_per_gather TO 0; SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'off', false); set_config ------------ off SELECT * FROM sensor_data AS s1 JOIN sensor_data AS s2 ON (TRUE) WHERE s1.time > '2020-01-01 00:00:30'::text::timestamptz AND s2.time > '2020-01-01 00:00:30' AND s2.time < '2021-01-01 00:00:30' AND s1.sensor_id > 50 ORDER BY s2.time, s1.time, s1.sensor_id, s2.sensor_id; time | sensor_id | time | sensor_id ------------------------------+-----------+------------------------------+----------- Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Apr 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Wed Jul 01 00:00:30 2020 PDT | 2 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Fri Jan 01 00:00:30 2021 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 1 Sat Jan 01 00:00:30 2022 PST | 100 | Thu Oct 01 00:00:30 2020 PDT | 2 RESET enable_material; RESET min_parallel_table_scan_size; RESET min_parallel_index_scan_size; RESET enable_hashjoin; RESET enable_nestloop; RESET parallel_tuple_cost; SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false); set_config ------------ on -- test worker assignment -- first chunk should have 1 worker and second chunk should have 2 SET max_parallel_workers_per_gather TO 2; :PREFIX SELECT count(*) FROM "test" WHERE i >= 400000 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Index Only Scan using _hyper_1_1_chunk_test_i_idx on _hyper_1_1_chunk Index Cond: (i >= 400000) -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk SELECT count(*) FROM "test" WHERE i >= 400000 AND length(version()) > 0; count ------- 60000 -- test worker assignment -- first chunk should have 2 worker and second chunk should have 1 :PREFIX SELECT count(*) FROM "test" WHERE i < 600000 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Index Only Scan using _hyper_1_2_chunk_test_i_idx on _hyper_1_2_chunk Index Cond: (i < 600000) -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk SELECT count(*) FROM "test" WHERE i < 600000 AND length(version()) > 0; count ------- 60000 -- test ChunkAppend with # workers < # childs SET max_parallel_workers_per_gather TO 1; :PREFIX SELECT count(*) FROM "test" WHERE length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 1 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk SELECT count(*) FROM "test" WHERE length(version()) > 0; count -------- 100000 -- test ChunkAppend with # workers > # childs SET max_parallel_workers_per_gather TO 2; :PREFIX SELECT count(*) FROM "test" WHERE i >= 500000 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk SELECT count(*) FROM "test" WHERE i >= 500000 AND length(version()) > 0; count ------- 50000 RESET max_parallel_workers_per_gather; -- test partial and non-partial plans -- these will not be parallel on PG < 11 ALTER TABLE :CHUNK1 SET (parallel_workers=0); ALTER TABLE :CHUNK2 SET (parallel_workers=2); :PREFIX SELECT count(*) FROM "test" WHERE i > 400000 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Index Only Scan using _hyper_1_1_chunk_test_i_idx on _hyper_1_1_chunk Index Cond: (i > 400000) -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_2_chunk ALTER TABLE :CHUNK1 SET (parallel_workers=2); ALTER TABLE :CHUNK2 SET (parallel_workers=0); :PREFIX SELECT count(*) FROM "test" WHERE i < 600000 AND length(version()) > 0; --- QUERY PLAN --- Finalize Aggregate -> Gather Workers Planned: 2 -> Partial Aggregate -> Result One-Time Filter: (length(version()) > 0) -> Parallel Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Result One-Time Filter: (length(version()) > 0) -> Index Only Scan using _hyper_1_2_chunk_test_i_idx on _hyper_1_2_chunk Index Cond: (i < 600000) -> Result One-Time Filter: (length(version()) > 0) -> Parallel Seq Scan on _hyper_1_1_chunk ALTER TABLE :CHUNK1 RESET (parallel_workers); ALTER TABLE :CHUNK2 RESET (parallel_workers); -- now() is not marked parallel safe in PostgreSQL < 12 so using now() -- in a query will prevent parallelism but CURRENT_TIMESTAMP and -- transaction_timestamp() are marked parallel safe :PREFIX SELECT i FROM "test" WHERE ts < CURRENT_TIMESTAMP; --- QUERY PLAN --- Gather Workers Planned: 1 Single Copy: true -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk Filter: (ts < CURRENT_TIMESTAMP) -> Seq Scan on _hyper_1_2_chunk Filter: (ts < CURRENT_TIMESTAMP) :PREFIX SELECT i FROM "test" WHERE ts < transaction_timestamp(); --- QUERY PLAN --- Gather Workers Planned: 1 Single Copy: true -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk Filter: (ts < transaction_timestamp()) -> Seq Scan on _hyper_1_2_chunk Filter: (ts < transaction_timestamp()) -- this won't be parallel query because now() is parallel restricted in PG < 12 :PREFIX SELECT i FROM "test" WHERE ts < now(); --- QUERY PLAN --- Gather Workers Planned: 1 Single Copy: true -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk Filter: (ts < now()) -> Seq Scan on _hyper_1_2_chunk Filter: (ts < now()) ================================================ FILE: test/expected/partition.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE part_legacy(time timestamptz, temp float, device int); SELECT create_hypertable('part_legacy', 'time', 'device', 2, partitioning_func => '_timescaledb_functions.get_partition_for_key'); create_hypertable -------------------------- (1,public,part_legacy,t) -- Show legacy partitioning function is used SELECT * FROM _timescaledb_catalog.dimension; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+--------------------------+---------+------------+--------------------------+-----------------------+-----------------+--------------------------+-------------------------+------------------ 1 | 1 | time | timestamp with time zone | t | | | | 604800000000 | | | 2 | 1 | device | integer | f | 2 | _timescaledb_functions | get_partition_for_key | | | | INSERT INTO part_legacy VALUES ('2017-03-22T09:18:23', 23.4, 1); INSERT INTO part_legacy VALUES ('2017-03-22T09:18:23', 23.4, 76); VACUUM part_legacy; -- Show two chunks and CHECK constraint with cast SELECT * FROM test.show_constraintsp('_timescaledb_internal._hyper_1_%_chunk'); Table | Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated ----------------------------------------+--------------+------+----------+-------+------------------------------------------------------------------------------------------------------------------------------------------------+------------+----------+----------- _timescaledb_internal._hyper_1_1_chunk | constraint_1 | c | {time} | - | (("time" >= 'Wed Mar 15 17:00:00 2017 PDT'::timestamp with time zone) AND ("time" < 'Wed Mar 22 17:00:00 2017 PDT'::timestamp with time zone)) | f | f | t _timescaledb_internal._hyper_1_1_chunk | constraint_2 | c | {device} | - | (_timescaledb_functions.get_partition_for_key(device) >= 1073741823) | f | f | t _timescaledb_internal._hyper_1_2_chunk | constraint_1 | c | {time} | - | (("time" >= 'Wed Mar 15 17:00:00 2017 PDT'::timestamp with time zone) AND ("time" < 'Wed Mar 22 17:00:00 2017 PDT'::timestamp with time zone)) | f | f | t _timescaledb_internal._hyper_1_2_chunk | constraint_3 | c | {device} | - | (_timescaledb_functions.get_partition_for_key(device) < 1073741823) | f | f | t -- Make sure constraint exclusion works on device column BEGIN; -- For plan stability between versions SET LOCAL enable_bitmapscan = false; SET LOCAL enable_indexscan = false; EXPLAIN (verbose, buffers off, costs off) SELECT * FROM part_legacy WHERE device = 1; --- QUERY PLAN --- Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.temp, _hyper_1_1_chunk.device Filter: (_hyper_1_1_chunk.device = 1) COMMIT; CREATE TABLE part_new(time timestamptz, temp float, device int); SELECT create_hypertable('part_new', 'time', 'device', 2); create_hypertable ----------------------- (2,public,part_new,t) SELECT * FROM _timescaledb_catalog.dimension; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+--------------------------+---------+------------+--------------------------+-----------------------+-----------------+--------------------------+-------------------------+------------------ 1 | 1 | time | timestamp with time zone | t | | | | 604800000000 | | | 2 | 1 | device | integer | f | 2 | _timescaledb_functions | get_partition_for_key | | | | 3 | 2 | time | timestamp with time zone | t | | | | 604800000000 | | | 4 | 2 | device | integer | f | 2 | _timescaledb_functions | get_partition_hash | | | | INSERT INTO part_new VALUES ('2017-03-22T09:18:23', 23.4, 1); INSERT INTO part_new VALUES ('2017-03-22T09:18:23', 23.4, 2); VACUUM part_new; -- Show two chunks and CHECK constraint without cast SELECT * FROM test.show_constraintsp('_timescaledb_internal._hyper_2_%_chunk'); Table | Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated ----------------------------------------+--------------+------+----------+-------+------------------------------------------------------------------------------------------------------------------------------------------------+------------+----------+----------- _timescaledb_internal._hyper_2_3_chunk | constraint_4 | c | {time} | - | (("time" >= 'Wed Mar 15 17:00:00 2017 PDT'::timestamp with time zone) AND ("time" < 'Wed Mar 22 17:00:00 2017 PDT'::timestamp with time zone)) | f | f | t _timescaledb_internal._hyper_2_3_chunk | constraint_5 | c | {device} | - | (_timescaledb_functions.get_partition_hash(device) < 1073741823) | f | f | t _timescaledb_internal._hyper_2_4_chunk | constraint_4 | c | {time} | - | (("time" >= 'Wed Mar 15 17:00:00 2017 PDT'::timestamp with time zone) AND ("time" < 'Wed Mar 22 17:00:00 2017 PDT'::timestamp with time zone)) | f | f | t _timescaledb_internal._hyper_2_4_chunk | constraint_6 | c | {device} | - | (_timescaledb_functions.get_partition_hash(device) >= 1073741823) | f | f | t -- Make sure constraint exclusion works on device column BEGIN; -- For plan stability between versions SET LOCAL enable_bitmapscan = false; SET LOCAL enable_indexscan = false; EXPLAIN (verbose, buffers off, costs off) SELECT * FROM part_new WHERE device = 1; --- QUERY PLAN --- Seq Scan on _timescaledb_internal._hyper_2_3_chunk Output: _hyper_2_3_chunk."time", _hyper_2_3_chunk.temp, _hyper_2_3_chunk.device Filter: (_hyper_2_3_chunk.device = 1) COMMIT; CREATE TABLE part_new_convert1(time timestamptz, temp float8, device int); SELECT create_hypertable('part_new_convert1', 'time', 'temp', 2); create_hypertable -------------------------------- (3,public,part_new_convert1,t) INSERT INTO part_new_convert1 VALUES ('2017-03-22T09:18:23', 1.0, 2); \set ON_ERROR_STOP 0 -- Changing the type of a hash-partitioned column should not be supported ALTER TABLE part_new_convert1 ALTER COLUMN temp TYPE numeric; ERROR: cannot change the type of a hash-partitioned column \set ON_ERROR_STOP 1 -- Should be able to change if not hash partitioned though ALTER TABLE part_new_convert1 ALTER COLUMN time TYPE timestamp; SELECT * FROM test.show_columnsp('_timescaledb_internal._hyper_3_%_chunk'); Relation | Kind | Column | Column type | NotNull ----------------------------------------+------+--------+-----------------------------+--------- _timescaledb_internal._hyper_3_5_chunk | r | time | timestamp without time zone | t _timescaledb_internal._hyper_3_5_chunk | r | temp | double precision | f _timescaledb_internal._hyper_3_5_chunk | r | device | integer | f CREATE TABLE part_add_dim(time timestamptz, temp float8, device int, location int); SELECT create_hypertable('part_add_dim', 'time', 'temp', 2); create_hypertable --------------------------- (4,public,part_add_dim,t) \set ON_ERROR_STOP 0 SELECT add_dimension('part_add_dim', 'location', 2, partitioning_func => 'bad_func'); ERROR: function "bad_func" does not exist at character 74 \set ON_ERROR_STOP 1 SELECT add_dimension('part_add_dim', 'location', 2, partitioning_func => '_timescaledb_functions.get_partition_for_key'); add_dimension ------------------------------------ (9,public,part_add_dim,location,t) SELECT * FROM _timescaledb_catalog.dimension; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+-----------------------------+---------+------------+--------------------------+-----------------------+-----------------+--------------------------+-------------------------+------------------ 1 | 1 | time | timestamp with time zone | t | | | | 604800000000 | | | 2 | 1 | device | integer | f | 2 | _timescaledb_functions | get_partition_for_key | | | | 3 | 2 | time | timestamp with time zone | t | | | | 604800000000 | | | 4 | 2 | device | integer | f | 2 | _timescaledb_functions | get_partition_hash | | | | 6 | 3 | temp | double precision | f | 2 | _timescaledb_functions | get_partition_hash | | | | 5 | 3 | time | timestamp without time zone | t | | | | 604800000000 | | | 7 | 4 | time | timestamp with time zone | t | | | | 604800000000 | | | 8 | 4 | temp | double precision | f | 2 | _timescaledb_functions | get_partition_hash | | | | 9 | 4 | location | integer | f | 2 | _timescaledb_functions | get_partition_for_key | | | | -- Test that we support custom SQL-based partitioning functions and -- that our native partitioning function handles function expressions -- as argument CREATE OR REPLACE FUNCTION custom_partfunc(source anyelement) RETURNS INTEGER LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ DECLARE retval INTEGER; BEGIN retval = _timescaledb_functions.get_partition_hash(substring(source::text FROM '[A-za-z0-9 ]+')); RAISE NOTICE 'hash value for % is %', source, retval; RETURN retval; END $BODY$; CREATE TABLE part_custom_func(time timestamptz, temp float8, device text); SELECT create_hypertable('part_custom_func', 'time', 'device', 2, partitioning_func => 'custom_partfunc'); create_hypertable ------------------------------- (5,public,part_custom_func,t) SELECT _timescaledb_functions.get_partition_hash(substring('dev1' FROM '[A-za-z0-9 ]+')); get_partition_hash -------------------- 1129986420 SELECT _timescaledb_functions.get_partition_hash('dev1'::text); get_partition_hash -------------------- 1129986420 SELECT _timescaledb_functions.get_partition_hash('dev7'::text); get_partition_hash -------------------- 449729092 INSERT INTO part_custom_func VALUES ('2017-03-22T09:18:23', 23.4, 'dev1'), ('2017-03-22T09:18:23', 23.4, 'dev7'); NOTICE: hash value for dev1 is 1129986420 NOTICE: hash value for dev1 is 1129986420 NOTICE: hash value for dev7 is 449729092 NOTICE: hash value for dev7 is 449729092 SELECT * FROM test.show_subtables('part_custom_func'); Child | Tablespace ----------------------------------------+------------ _timescaledb_internal._hyper_5_6_chunk | _timescaledb_internal._hyper_5_7_chunk | -- This first test is slightly trivial, but segfaulted in old versions CREATE TYPE simpl AS (val1 int4); CREATE OR REPLACE FUNCTION simpl_type_hash(ANYELEMENT) RETURNS int4 AS $$ SELECT $1.val1; $$ LANGUAGE SQL IMMUTABLE; CREATE TABLE simpl_partition ("timestamp" TIMESTAMPTZ, object simpl); SELECT create_hypertable( 'simpl_partition', 'timestamp', 'object', 1000, chunk_time_interval => interval '1 day', partitioning_func=>'simpl_type_hash'); create_hypertable ------------------------------ (6,public,simpl_partition,t) INSERT INTO simpl_partition VALUES ('2017-03-22T09:18:23', ROW(1)::simpl); SELECT * from simpl_partition; timestamp | object ------------------------------+-------- Wed Mar 22 09:18:23 2017 PDT | (1) -- Also test that the fix works when we have more chunks than allowed at once SET timescaledb.max_open_chunks_per_insert=1; INSERT INTO simpl_partition VALUES ('2017-03-22T10:18:23', ROW(0)::simpl), ('2017-03-22T10:18:23', ROW(1)::simpl), ('2017-03-22T10:18:23', ROW(2)::simpl), ('2017-03-22T10:18:23', ROW(3)::simpl), ('2017-03-22T10:18:23', ROW(4)::simpl), ('2017-03-22T10:18:23', ROW(5)::simpl); SET timescaledb.max_open_chunks_per_insert=default; SELECT * from simpl_partition; timestamp | object ------------------------------+-------- Wed Mar 22 09:18:23 2017 PDT | (1) Wed Mar 22 10:18:23 2017 PDT | (0) Wed Mar 22 10:18:23 2017 PDT | (1) Wed Mar 22 10:18:23 2017 PDT | (2) Wed Mar 22 10:18:23 2017 PDT | (3) Wed Mar 22 10:18:23 2017 PDT | (4) Wed Mar 22 10:18:23 2017 PDT | (5) -- Test that index creation is handled correctly. CREATE TABLE hyper_with_index(time timestamptz, temp float, device int); CREATE UNIQUE INDEX temp_index ON hyper_with_index(temp); \set ON_ERROR_STOP 0 SELECT create_hypertable('hyper_with_index', 'time'); ERROR: cannot create a unique index without the column "time" (used in partitioning) SELECT create_hypertable('hyper_with_index', 'time', 'device', 2); ERROR: cannot create a unique index without the column "time" (used in partitioning) SELECT create_hypertable('hyper_with_index', 'time', 'temp', 2); ERROR: cannot create a unique index without the column "time" (used in partitioning) \set ON_ERROR_STOP 1 DROP INDEX temp_index; CREATE UNIQUE INDEX time_index ON hyper_with_index(time); \set ON_ERROR_STOP 0 -- should error because device not in index SELECT create_hypertable('hyper_with_index', 'time', 'device', 4); ERROR: cannot create a unique index without the column "device" (used in partitioning) \set ON_ERROR_STOP 1 SELECT create_hypertable('hyper_with_index', 'time'); create_hypertable -------------------------------- (11,public,hyper_with_index,t) -- make sure user created index is used. -- not using \d or \d+ because output syntax differs -- between postgres 9 and postgres 10. SELECT indexname FROM pg_indexes WHERE tablename = 'hyper_with_index'; indexname ------------ time_index \set ON_ERROR_STOP 0 SELECT add_dimension('hyper_with_index', 'device', 4); ERROR: cannot create a unique index without the column "device" (used in partitioning) \set ON_ERROR_STOP 1 DROP INDEX time_index; CREATE UNIQUE INDEX time_space_index ON hyper_with_index(time, device); SELECT add_dimension('hyper_with_index', 'device', 4); add_dimension --------------------------------------- (23,public,hyper_with_index,device,t) CREATE TABLE hyper_with_primary(time TIMESTAMPTZ PRIMARY KEY, temp float, device int); \set ON_ERROR_STOP 0 SELECT create_hypertable('hyper_with_primary', 'time', 'device', 4); ERROR: cannot create a unique index without the column "device" (used in partitioning) \set ON_ERROR_STOP 1 SELECT create_hypertable('hyper_with_primary', 'time'); create_hypertable ---------------------------------- (13,public,hyper_with_primary,t) \set ON_ERROR_STOP 0 SELECT add_dimension('hyper_with_primary', 'device', 4); ERROR: cannot create a unique index without the column "device" (used in partitioning) \set ON_ERROR_STOP 1 -- NON-unique indexes can still be created CREATE INDEX temp_index ON hyper_with_index(temp); -- Make sure custom composite types are supported as dimensions CREATE TYPE TUPLE as (val1 int4, val2 int4); CREATE FUNCTION tuple_hash(value ANYELEMENT) RETURNS INT4 LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RAISE NOTICE 'custom hash value is: %', value.val1+value.val2; RETURN value.val1+value.val2; END $BODY$; CREATE TABLE part_custom_dim (time TIMESTAMPTZ, combo TUPLE, device TEXT); SELECT create_hypertable('part_custom_dim', 'time', 'combo', 4, partitioning_func=>'tuple_hash'); create_hypertable ------------------------------- (14,public,part_custom_dim,t) INSERT INTO part_custom_dim(time, combo) VALUES (now(), (1,2)); NOTICE: custom hash value is: 3 NOTICE: custom hash value is: 3 DROP TABLE part_custom_dim; -- Now make sure that renaming partitioning_func_schema will get updated properly \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA IF NOT EXISTS my_partitioning_schema; CREATE FUNCTION my_partitioning_schema.tuple_hash(value ANYELEMENT) RETURNS INT4 LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RAISE NOTICE 'custom hash value is: %', value.val1+value.val2; RETURN value.val1+value.val2; END $BODY$; CREATE TABLE part_custom_dim (time TIMESTAMPTZ, combo TUPLE, device TEXT); SELECT create_hypertable('part_custom_dim', 'time', 'combo', 4, partitioning_func=>'my_partitioning_schema.tuple_hash'); create_hypertable ------------------------------- (15,public,part_custom_dim,t) INSERT INTO part_custom_dim(time, combo) VALUES (now(), (1,2)); NOTICE: custom hash value is: 3 NOTICE: custom hash value is: 3 ALTER SCHEMA my_partitioning_schema RENAME TO new_partitioning_schema; -- Inserts should work even after we rename the schema INSERT INTO part_custom_dim(time, combo) VALUES (now(), (3,4)); NOTICE: custom hash value is: 7 NOTICE: custom hash value is: 7 -- Test partitioning function on an open (time) dimension CREATE OR REPLACE FUNCTION time_partfunc(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ DECLARE retval TIMESTAMPTZ; BEGIN retval := to_timestamp(unixtime); RAISE NOTICE 'time value for % is %', unixtime, timezone('UTC', retval); RETURN retval; END $BODY$; CREATE OR REPLACE FUNCTION time_partfunc_bad_parameters(unixtime float8, extra text) RETURNS TIMESTAMPTZ LANGUAGE SQL IMMUTABLE AS $BODY$ SELECT to_timestamp(unixtime); $BODY$; CREATE OR REPLACE FUNCTION time_partfunc_bad_return_type(unixtime float8) RETURNS FLOAT8 LANGUAGE SQL IMMUTABLE AS $BODY$ SELECT unixtime; $BODY$; CREATE TABLE part_time_func(time float8, temp float8, device text); \set ON_ERROR_STOP 0 -- Should fail due to invalid time column SELECT create_hypertable('part_time_func', 'time'); ERROR: invalid type for dimension "time" -- Should fail due to bad signature of time partitioning function SELECT create_hypertable('part_time_func', 'time', time_partitioning_func => 'time_partfunc_bad_parameters'); ERROR: invalid partitioning function SELECT create_hypertable('part_time_func', 'time', time_partitioning_func => 'time_partfunc_bad_return_type'); ERROR: invalid partitioning function \set ON_ERROR_STOP 1 -- Should work with time partitioning function that returns a valid time type SELECT create_hypertable('part_time_func', 'time', time_partitioning_func => 'time_partfunc'); create_hypertable ------------------------------ (16,public,part_time_func,t) INSERT INTO part_time_func VALUES (1530214157.134, 23.4, 'dev1'), (1533214157.8734, 22.3, 'dev7'); NOTICE: time value for 1530214157.134 is Thu Jun 28 19:29:17.134 2018 NOTICE: time value for 1530214157.134 is Thu Jun 28 19:29:17.134 2018 NOTICE: time value for 1530214157.134 is Thu Jun 28 19:29:17.134 2018 NOTICE: time value for 1530214157.134 is Thu Jun 28 19:29:17.134 2018 NOTICE: time value for 1533214157.8734 is Thu Aug 02 12:49:17.8734 2018 NOTICE: time value for 1533214157.8734 is Thu Aug 02 12:49:17.8734 2018 NOTICE: time value for 1533214157.8734 is Thu Aug 02 12:49:17.8734 2018 NOTICE: time value for 1533214157.8734 is Thu Aug 02 12:49:17.8734 2018 SELECT time, temp, device FROM part_time_func; time | temp | device -----------------+------+-------- 1530214157.134 | 23.4 | dev1 1533214157.8734 | 22.3 | dev7 SELECT time_partfunc(time) at time zone 'UTC', temp, device FROM part_time_func; NOTICE: time value for 1530214157.134 is Thu Jun 28 19:29:17.134 2018 NOTICE: time value for 1533214157.8734 is Thu Aug 02 12:49:17.8734 2018 timezone | temp | device -------------------------------+------+-------- Thu Jun 28 19:29:17.134 2018 | 23.4 | dev1 Thu Aug 02 12:49:17.8734 2018 | 22.3 | dev7 SELECT * FROM test.show_subtables('part_time_func'); Child | Tablespace ------------------------------------------+------------ _timescaledb_internal._hyper_16_11_chunk | _timescaledb_internal._hyper_16_12_chunk | SELECT (test.show_constraints("Child")).* FROM test.show_subtables('part_time_func'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated ---------------+------+---------+-------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------+----------+----------- constraint_18 | c | {time} | - | ((time_partfunc("time") >= 'Wed Jun 27 17:00:00 2018 PDT'::timestamp with time zone) AND (time_partfunc("time") < 'Wed Jul 04 17:00:00 2018 PDT'::timestamp with time zone)) | f | f | t constraint_19 | c | {time} | - | ((time_partfunc("time") >= 'Wed Aug 01 17:00:00 2018 PDT'::timestamp with time zone) AND (time_partfunc("time") < 'Wed Aug 08 17:00:00 2018 PDT'::timestamp with time zone)) | f | f | t SELECT (test.show_indexes("Child")).* FROM test.show_subtables('part_time_func'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ------------------------------------------------------------------+---------+---------------------+--------+---------+-----------+------------ _timescaledb_internal._hyper_16_11_chunk_part_time_func_expr_idx | {expr} | time_partfunc(expr) | f | f | f | _timescaledb_internal._hyper_16_12_chunk_part_time_func_expr_idx | {expr} | time_partfunc(expr) | f | f | f | -- Check that constraint exclusion works with time partitioning -- function (scan only one chunk) -- No exclusion EXPLAIN (verbose, buffers off, costs off) SELECT * FROM part_time_func; --- QUERY PLAN --- Append -> Seq Scan on _timescaledb_internal._hyper_16_11_chunk Output: _hyper_16_11_chunk."time", _hyper_16_11_chunk.temp, _hyper_16_11_chunk.device -> Seq Scan on _timescaledb_internal._hyper_16_12_chunk Output: _hyper_16_12_chunk."time", _hyper_16_12_chunk.temp, _hyper_16_12_chunk.device -- Exclude using the function on time EXPLAIN (verbose, buffers off, costs off) SELECT * FROM part_time_func WHERE time_partfunc(time) < '2018-07-01'; --- QUERY PLAN --- Index Scan using _hyper_16_11_chunk_part_time_func_expr_idx on _timescaledb_internal._hyper_16_11_chunk Output: _hyper_16_11_chunk."time", _hyper_16_11_chunk.temp, _hyper_16_11_chunk.device Index Cond: (time_partfunc(_hyper_16_11_chunk."time") < 'Sun Jul 01 00:00:00 2018 PDT'::timestamp with time zone) -- Exclude using the same date but as a UNIX timestamp. Won't do an -- index scan since the index is on the time function expression EXPLAIN (verbose, buffers off, costs off) SELECT * FROM part_time_func WHERE time < 1530403200.0; NOTICE: time value for 1530403200 is Sun Jul 01 00:00:00 2018 --- QUERY PLAN --- Seq Scan on _timescaledb_internal._hyper_16_11_chunk Output: _hyper_16_11_chunk."time", _hyper_16_11_chunk.temp, _hyper_16_11_chunk.device Filter: (_hyper_16_11_chunk."time" < '1530403200'::double precision) -- Check that inserts will fail if we use a time partitioning function -- that returns NULL CREATE OR REPLACE FUNCTION time_partfunc_null_ret(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RETURN NULL; END $BODY$; CREATE TABLE part_time_func_null_ret(time float8, temp float8, device text); SELECT create_hypertable('part_time_func_null_ret', 'time', time_partitioning_func => 'time_partfunc_null_ret'); create_hypertable --------------------------------------- (17,public,part_time_func_null_ret,t) \set ON_ERROR_STOP 0 INSERT INTO part_time_func_null_ret VALUES (1530214157.134, 23.4, 'dev1'), (1533214157.8734, 22.3, 'dev7'); ERROR: partitioning function "public.time_partfunc_null_ret" returned NULL \set ON_ERROR_STOP 1 ================================================ FILE: test/expected/partition_coercion.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test partition chunk exclusion with cross-type comparisons -- wrong result: text column + name literal CREATE TABLE hash_text(time timestamptz NOT NULL, device text); SELECT create_hypertable('hash_text', 'time'); create_hypertable ------------------------ (1,public,hash_text,t) SELECT add_dimension('hash_text', 'device', number_partitions => 3); add_dimension ------------------------------- (2,public,hash_text,device,t) INSERT INTO hash_text VALUES ('2000-01-01', 'abc'); SELECT count(*) FROM hash_text WHERE device = 'abc'::name; count ------- 1 DROP TABLE hash_text; -- x86_64 wrong result: int4 time column + large int8 literal + range query -- 4294967196::int8 truncates to -100 as signed int4 -- Chunk exclusion uses time < -100, excluding all positive-time chunks CREATE FUNCTION time_part_int4(val int4) RETURNS int4 AS $$ SELECT val $$ LANGUAGE SQL IMMUTABLE; CREATE TABLE time_int4(time int4 NOT NULL, v int); SELECT create_hypertable('time_int4', 'time', chunk_time_interval => 100, time_partitioning_func => 'time_part_int4'); create_hypertable ------------------------ (2,public,time_int4,t) INSERT INTO time_int4 VALUES (100, 1), (200, 2); -- Both rows satisfy time < 4294967196, but bug truncates to time < -100 SELECT count(*) FROM time_int4 WHERE time < 4294967196::int8; count ------- 2 DROP TABLE time_int4; DROP FUNCTION time_part_int4; -- i386 crash: int8 time column + int4 literal + custom partitioning -- On i386: SEGFAULT (DatumGetInt64 dereferences byval int4 as pointer) -- On x86_64: works by coincidence (both int4 and int8 are byval) CREATE FUNCTION time_part_int8(val int8) RETURNS int8 AS $$ SELECT val $$ LANGUAGE SQL IMMUTABLE; CREATE TABLE time_int8(time int8 NOT NULL, v int); SELECT create_hypertable('time_int8', 'time', chunk_time_interval => 10, time_partitioning_func => 'time_part_int8'); create_hypertable ------------------------ (3,public,time_int8,t) INSERT INTO time_int8 VALUES (1, 1), (11, 2), (21, 3); SELECT count(*) FROM time_int8 WHERE time = 1::int4; count ------- 1 DROP TABLE time_int8; DROP FUNCTION time_part_int8; -- Exact type match: text column + text literal (no coercion needed) CREATE TABLE hash_text_exact(time timestamptz NOT NULL, device text); SELECT create_hypertable('hash_text_exact', 'time'); create_hypertable ------------------------------ (4,public,hash_text_exact,t) SELECT add_dimension('hash_text_exact', 'device', number_partitions => 3); add_dimension ------------------------------------- (6,public,hash_text_exact,device,t) INSERT INTO hash_text_exact VALUES ('2000-01-01', 'abc'); SELECT count(*) FROM hash_text_exact WHERE device = 'abc'::text; count ------- 1 DROP TABLE hash_text_exact; -- Binary compatible types: text column + varchar literal -- PostgreSQL coerces varchar to text at parse time CREATE TABLE hash_text_varchar(time timestamptz NOT NULL, device text); SELECT create_hypertable('hash_text_varchar', 'time'); create_hypertable -------------------------------- (5,public,hash_text_varchar,t) SELECT add_dimension('hash_text_varchar', 'device', number_partitions => 3); add_dimension --------------------------------------- (8,public,hash_text_varchar,device,t) INSERT INTO hash_text_varchar VALUES ('2000-01-01', 'abc'); SELECT count(*) FROM hash_text_varchar WHERE device = 'abc'::varchar; count ------- 1 DROP TABLE hash_text_varchar; -- Array coercion: text column + name[] array (ScalarArrayOpExpr) -- Test both ANY (OR) and ALL (AND) semantics CREATE TABLE hash_text_array(time timestamptz NOT NULL, device text); SELECT create_hypertable('hash_text_array', 'time'); create_hypertable ------------------------------ (6,public,hash_text_array,t) SELECT add_dimension('hash_text_array', 'device', number_partitions => 3); add_dimension -------------------------------------- (10,public,hash_text_array,device,t) INSERT INTO hash_text_array VALUES ('2000-01-01', 'abc'), ('2000-01-01', 'def'), ('2000-01-01', 'ghi'); -- OR: match any element SELECT count(*) FROM hash_text_array WHERE device = ANY(ARRAY['abc', 'def']::name[]); count ------- 2 -- AND: match all elements (logically empty for different values, but exercises code path) SELECT count(*) FROM hash_text_array WHERE device = ALL(ARRAY['abc', 'def']::name[]); count ------- 0 -- AND: single element (equivalent to =) SELECT count(*) FROM hash_text_array WHERE device = ALL(ARRAY['abc']::name[]); count ------- 1 DROP TABLE hash_text_array; -- Time dimension with SAOP + type coercion (int4 column, int8 array) -- Note: open (time/range) dimensions can't use SAOP with multiple OR values -- for chunk exclusion, but AND with single effective bound works. -- These tests verify correct results and that AND cases use chunk exclusion. CREATE FUNCTION time_part_int4_saop(val int4) RETURNS int4 AS $$ SELECT val $$ LANGUAGE SQL IMMUTABLE; CREATE TABLE time_int4_saop(time int4 NOT NULL, v int); SELECT create_hypertable('time_int4_saop', 'time', chunk_time_interval => 100, time_partitioning_func => 'time_part_int4_saop'); create_hypertable ----------------------------- (7,public,time_int4_saop,t) INSERT INTO time_int4_saop VALUES (50, 1), (150, 2), (250, 3); -- AND: time < ALL(array) means time < min(array) = 100 -- Single effective bound, chunk exclusion should work SELECT count(*) FROM time_int4_saop WHERE time < ALL(ARRAY[100, 200]::int8[]); count ------- 1 -- AND: time > ALL(array) means time > max(array) = 200 SELECT count(*) FROM time_int4_saop WHERE time > ALL(ARRAY[100, 200]::int8[]); count ------- 1 -- OR cases: chunk exclusion not used (multiple OR values rejected), -- but results must still be correct SELECT count(*) FROM time_int4_saop WHERE time < ANY(ARRAY[100, 200]::int8[]); count ------- 2 SELECT count(*) FROM time_int4_saop WHERE time > ANY(ARRAY[100, 200]::int8[]); count ------- 2 DROP TABLE time_int4_saop; DROP FUNCTION time_part_int4_saop; -- Prepared statement with varchar parameter, text column -- Custom plan: coercion at plan time, chunk exclusion works -- Generic plan: no chunk exclusion (param unknown), but correct result CREATE TABLE hash_prep(time timestamptz NOT NULL, device text); SELECT create_hypertable('hash_prep', 'time'); create_hypertable ------------------------ (8,public,hash_prep,t) SELECT add_dimension('hash_prep', 'device', number_partitions => 3); add_dimension -------------------------------- (13,public,hash_prep,device,t) INSERT INTO hash_prep VALUES ('2000-01-01', 'abc'), ('2000-01-01', 'def'); PREPARE hash_q(varchar) AS SELECT count(*) FROM hash_prep WHERE device = $1; SET plan_cache_mode = force_custom_plan; EXECUTE hash_q('abc'); count ------- 1 EXECUTE hash_q('def'); count ------- 1 SET plan_cache_mode = force_generic_plan; EXECUTE hash_q('abc'); count ------- 1 EXECUTE hash_q('def'); count ------- 1 RESET plan_cache_mode; DEALLOCATE hash_q; DROP TABLE hash_prep; -- Multiple ANDed restrictions on closed dimension -- Use IN + = to exercise intersection: IN creates list, = intersects with it CREATE TABLE hash_multi(time timestamptz NOT NULL, device text); SELECT create_hypertable('hash_multi', 'time'); create_hypertable ------------------------- (9,public,hash_multi,t) SELECT add_dimension('hash_multi', 'device', number_partitions => 3); add_dimension --------------------------------- (15,public,hash_multi,device,t) INSERT INTO hash_multi VALUES ('2000-01-01', 'abc'), ('2000-01-01', 'def'); -- IN creates partition list [abc,def], then = intersects with [abc] => [abc] SELECT count(*) FROM hash_multi WHERE device IN ('abc', 'def') AND device = 'abc'; count ------- 1 DROP TABLE hash_multi; -- NULL array elements: exercises branch when iterating arrays with NULLs -- The NULL elements should be skipped, non-NULL elements should work CREATE TABLE hash_null_array(time timestamptz NOT NULL, device text); SELECT create_hypertable('hash_null_array', 'time'); create_hypertable ------------------------------- (10,public,hash_null_array,t) SELECT add_dimension('hash_null_array', 'device', number_partitions => 3); add_dimension -------------------------------------- (17,public,hash_null_array,device,t) INSERT INTO hash_null_array VALUES ('2000-01-01', 'abc'), ('2000-01-01', 'def'); -- Array with NULL elements (type coercion path) SELECT count(*) FROM hash_null_array WHERE device = ANY(ARRAY['abc', NULL, 'def']::name[]); count ------- 2 -- Array with NULL elements (no type coercion path) SELECT count(*) FROM hash_null_array WHERE device = ANY(ARRAY['abc', NULL, 'def']::text[]); count ------- 2 DROP TABLE hash_null_array; ================================================ FILE: test/expected/partitioned_hypertable.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test declarative partitioning for hypertables -- Enable declarative partitioning for all subsequent tests SET timescaledb.enable_partitioned_hypertables = true; -- Basic hypertable creation with TIMESTAMPTZ CREATE TABLE metrics( time TIMESTAMP WITH TIME ZONE, device TEXT, value FLOAT ) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); -- Create with TIMESTAMP CREATE TABLE metrics_ts( time TIMESTAMP NOT NULL, device TEXT, value FLOAT ) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); -- Create with DATE CREATE TABLE metrics_date( time DATE NOT NULL, device TEXT, value FLOAT ) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); -- Create with int CREATE TABLE metrics_int( time INT NOT NULL, device TEXT, value FLOAT ) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); -- Create with custom chunk_time_interval CREATE TABLE metrics_custom_interval( time TIMESTAMPTZ NOT NULL, device TEXT, value FLOAT ) WITH (timescaledb.hypertable, timescaledb.partition_column='time', timescaledb.chunk_interval='30 days'); -- Verify hypertables are actually created and partitioned SELECT hypertable_name FROM timescaledb_information.hypertables WHERE hypertable_name IN ('metrics', 'metrics_ts', 'metrics_date', 'metrics_int', 'metrics_custom_interval') ORDER BY hypertable_name; hypertable_name ------------------------- metrics metrics_custom_interval metrics_date metrics_int metrics_ts SELECT DISTINCT(relkind) = 'p' FROM pg_class WHERE relname IN ('metrics', 'metrics_ts', 'metrics_date', 'metrics_int', 'metrics_custom_interval'); ?column? ---------- t \set ON_ERROR_STOP 0 -- Try to create with invalid partition column type CREATE TABLE invalid(time TEXT, value FLOAT) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); ERROR: invalid type for dimension "time" CREATE TABLE invalid(time TEXT, value FLOAT) WITH (timescaledb.hypertable); ERROR: partition column could not be determined \set ON_ERROR_STOP 1 DROP TABLE IF EXISTS metrics_ts; DROP TABLE IF EXISTS metrics_date; DROP TABLE IF EXISTS metrics_int; DROP TABLE IF EXISTS metrics_custom_interval; DROP TABLE IF EXISTS invalid; NOTICE: table "invalid" does not exist, skipping -- Test PARTITION BY syntax CREATE TABLE metrics_partition_by( time TIMESTAMPTZ NOT NULL, device TEXT, value FLOAT ) PARTITION BY RANGE (time) WITH (timescaledb.hypertable); \set ON_ERROR_STOP 0 CREATE TABLE part_col_specified(time TIMESTAMPTZ, device TEXT) PARTITION BY RANGE (time) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); ERROR: cannot specify both PARTITION BY and timescaledb.partition_column CREATE TABLE multiple_part_key(time TIMESTAMPTZ, time2 TIMESTAMP, device TEXT) PARTITION BY RANGE (time, time2) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); ERROR: only single column partitioning is supported for partitioned hypertables CREATE TABLE bad_strategy(time TIMESTAMPTZ, device TEXT) PARTITION BY LIST (time) WITH (timescaledb.hypertable); ERROR: only RANGE partitioning is supported for partitioned hypertables \set ON_ERROR_STOP 1 DROP TABLE IF EXISTS metrics_partition_by; -- Insert Operations and Chunk Creation INSERT INTO metrics VALUES ('2025-01-15 00:00:00+00', 'device1', 11.0), ('2025-02-15 00:00:00+00', 'device2', 12.0), ('2025-03-15 00:00:00+00', 'device3', 13.0); SELECT count(*) FROM show_chunks('metrics'); count ------- 3 SELECT count(*) FROM metrics; count ------- 3 SELECT * FROM metrics ORDER BY time; time | device | value ------------------------------+---------+------- Tue Jan 14 16:00:00 2025 PST | device1 | 11 Fri Feb 14 16:00:00 2025 PST | device2 | 12 Fri Mar 14 17:00:00 2025 PDT | device3 | 13 -- Verify chunk was created and attached as partition SELECT count(*) FROM show_chunks('metrics'); count ------- 3 SELECT child.relname AS chunk, parent.relname AS hypertable FROM pg_inherits JOIN pg_class child ON inhrelid = child.oid JOIN pg_class parent ON inhparent = parent.oid WHERE parent.relname = 'metrics' LIMIT 1; chunk | hypertable ------------------+------------ _hyper_1_1_chunk | metrics -- Insert with CHECK constraint ALTER TABLE metrics ADD CONSTRAINT valcheck CHECK (value >= 0); -- Try inserting into existing and new chunk to violate CHECK constraint \set ON_ERROR_STOP 0 INSERT INTO metrics VALUES ('2025-03-15 00:00:00+00', 'device1', -10.0); ERROR: new row for relation "_hyper_1_3_chunk" violates check constraint "valcheck" INSERT INTO metrics VALUES ('2025-04-15 00:00:00+00', 'device1', -10.0); ERROR: new row for relation "_hyper_1_4_chunk" violates check constraint "valcheck" \set ON_ERROR_STOP 1 -- SELECT with WHERE on time (partition pruning) EXPLAIN (COSTS OFF) SELECT * FROM metrics WHERE time >= '2025-01-01' AND time < '2025-02-01'; --- QUERY PLAN --- Seq Scan on _hyper_1_1_chunk metrics Filter: (("time" >= 'Wed Jan 01 00:00:00 2025 PST'::timestamp with time zone) AND ("time" < 'Sat Feb 01 00:00:00 2025 PST'::timestamp with time zone)) -- FOREIGN KEY from hypertable to regular table CREATE TABLE ref_table(id INT PRIMARY KEY, name TEXT); INSERT INTO ref_table VALUES (1, 'ref1'), (2, 'ref2'); CREATE TABLE fk_table( time TIMESTAMPTZ NOT NULL, ref_id INT REFERENCES ref_table(id), value FLOAT ) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); INSERT INTO fk_table VALUES ('2025-11-01', 1, 10.0); \set ON_ERROR_STOP 0 INSERT INTO fk_table VALUES ('2025-11-01', 999, 20.0); ERROR: insert or update on table "_hyper_8_5_chunk" violates foreign key constraint "fk_table_ref_id_fkey" \set ON_ERROR_STOP 1 DROP TABLE ref_table CASCADE; NOTICE: drop cascades to constraint fk_table_ref_id_fkey on table fk_table DROP TABLE fk_table CASCADE; -- FOREIGN KEY from hypertable to hypertable CREATE TABLE ref_ht( time TIMESTAMPTZ NOT NULL , id INT, CONSTRAINT ref_ht_pkey PRIMARY KEY (time, id) ) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); INSERT INTO ref_ht VALUES ('2025-06-15 00:00:00+00', 1), ('2025-07-15 00:00:00+00', 2); CREATE TABLE fk_ht( time TIMESTAMPTZ NOT NULL, ref_time TIMESTAMPTZ, ref_id INT, value FLOAT, FOREIGN KEY (ref_time, ref_id) REFERENCES ref_ht(time, id) ) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); INSERT INTO fk_ht VALUES ('2025-08-15 00:00:00+00', '2025-06-15 00:00:00+00', 1, 31.0); \set ON_ERROR_STOP 0 INSERT INTO fk_ht VALUES ('2025-08-15 00:00:00+00', '2025-01-01 00:00:00+00', 999, 32.0); ERROR: insert or update on table "_hyper_10_8_chunk" violates foreign key constraint "fk_ht_ref_time_ref_id_fkey" \set ON_ERROR_STOP 1 DROP TABLE ref_ht CASCADE; NOTICE: drop cascades to constraint fk_ht_ref_time_ref_id_fkey on table fk_ht DROP TABLE fk_ht CASCADE; -- Test if foreign keys to hypertables not using declarative partitioning are still disallowed SET timescaledb.enable_partitioned_hypertables = false; CREATE TABLE ref_ht( time TIMESTAMPTZ NOT NULL , id INT, CONSTRAINT ref_ht_pkey PRIMARY KEY (time, id) ) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); INSERT INTO ref_ht VALUES ('2025-06-15 00:00:00+00', 1), ('2025-07-15 00:00:00+00', 2); SET timescaledb.enable_partitioned_hypertables = true; \set ON_ERROR_STOP 0 CREATE TABLE fk_ht( time TIMESTAMPTZ NOT NULL, ref_time TIMESTAMPTZ, ref_id INT, value FLOAT, FOREIGN KEY (ref_time, ref_id) REFERENCES ref_ht(time, id) ) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); ERROR: hypertables cannot be used as foreign key references of hypertables \set ON_ERROR_STOP 1 DROP TABLE ref_ht CASCADE; DROP TABLE IF EXISTS fk_ht CASCADE; NOTICE: table "fk_ht" does not exist, skipping -- Test partition wise joins CREATE TABLE metrics_pwj( time TIMESTAMPTZ NOT NULL, device TEXT, value FLOAT ) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); INSERT INTO metrics_pwj VALUES ('2025-01-15 00:00:00+00', 'device1', 11.0), ('2025-02-15 00:00:00+00', 'device2', 12.0), ('2025-03-15 00:00:00+00', 'device3', 13.0); SET enable_partitionwise_join = true; EXPLAIN (COSTS OFF) SELECT m1.device, m2.device FROM metrics AS m1 JOIN metrics_pwj AS m2 ON m1.time = m2.time; --- QUERY PLAN --- Append -> Nested Loop Join Filter: (m1_1."time" = m2_1."time") -> Seq Scan on _hyper_1_1_chunk m1_1 -> Seq Scan on _hyper_13_11_chunk m2_1 -> Nested Loop Join Filter: (m1_2."time" = m2_2."time") -> Seq Scan on _hyper_1_2_chunk m1_2 -> Seq Scan on _hyper_13_12_chunk m2_2 -> Nested Loop Join Filter: (m1_3."time" = m2_3."time") -> Seq Scan on _hyper_1_3_chunk m1_3 -> Seq Scan on _hyper_13_13_chunk m2_3 SET enable_partitionwise_join = false; -- Transaction - ROLLBACK BEGIN; INSERT INTO metrics VALUES ('2024-12-20', 'rollback_test', 42.0); SELECT count(*)=1 FROM metrics WHERE device = 'rollback_test'; ?column? ---------- t ROLLBACK; SELECT count(*)=0 FROM metrics WHERE device = 'rollback_test'; ?column? ---------- t -- Reset GUC SET timescaledb.enable_partitioned_hypertables = false; -- Cleanup DROP TABLE IF EXISTS metrics CASCADE; DROP TABLE IF EXISTS metrics_pwj CASCADE; ================================================ FILE: test/expected/partitioning.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Should expect an error when creating a hypertable from a partition \set ON_ERROR_STOP 0 CREATE TABLE partitioned_ht_create(time timestamptz, temp float, device int) PARTITION BY RANGE (time); SELECT create_hypertable('partitioned_ht_create', 'time'); ERROR: table "partitioned_ht_create" is already partitioned \set ON_ERROR_STOP 1 -- Should expect an error when attaching a hypertable to a partition \set ON_ERROR_STOP 0 CREATE TABLE partitioned_attachment_vanilla(time timestamptz, temp float, device int) PARTITION BY RANGE (time); CREATE TABLE attachment_hypertable(time timestamptz, temp float, device int); SELECT create_hypertable('attachment_hypertable', 'time'); create_hypertable ------------------------------------ (1,public,attachment_hypertable,t) ALTER TABLE partitioned_attachment_vanilla ATTACH PARTITION attachment_hypertable FOR VALUES FROM ('2016-07-01') TO ('2016-08-01'); ERROR: hypertables do not support native postgres partitioning \set ON_ERROR_STOP 1 -- Should not expect an error when attaching a normal table to a partition CREATE TABLE partitioned_vanilla(time timestamptz, temp float, device int) PARTITION BY RANGE (time); CREATE TABLE attachment_vanilla(time timestamptz, temp float, device int); ALTER TABLE partitioned_vanilla ATTACH PARTITION attachment_vanilla FOR VALUES FROM ('2016-07-01') TO ('2016-08-01'); ================================================ FILE: test/expected/partitionwise-15.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set PREFIX 'EXPLAIN (VERBOSE, BUFFERS OFF, COSTS OFF)' -- Create a two dimensional hypertable CREATE TABLE hyper (time timestamptz, device int, temp float); SELECT * FROM create_hypertable('hyper', 'time', 'device', 2); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | hyper | t -- Create a similar PostgreSQL partitioned table CREATE TABLE pg2dim (time timestamptz, device int, temp float) PARTITION BY HASH (device); CREATE TABLE pg2dim_h1 PARTITION OF pg2dim FOR VALUES WITH (MODULUS 2, REMAINDER 0) PARTITION BY RANGE(time); CREATE TABLE pg2dim_h2 PARTITION OF pg2dim FOR VALUES WITH (MODULUS 2, REMAINDER 1) PARTITION BY RANGE(time); CREATE TABLE pg2dim_h1_t1 PARTITION OF pg2dim_h1 FOR VALUES FROM ('2018-01-01 00:00') TO ('2018-09-01 00:00'); CREATE TABLE pg2dim_h1_t2 PARTITION OF pg2dim_h1 FOR VALUES FROM ('2018-09-01 00:00') TO ('2018-12-01 00:00'); CREATE TABLE pg2dim_h2_t1 PARTITION OF pg2dim_h2 FOR VALUES FROM ('2018-01-01 00:00') TO ('2018-09-01 00:00'); CREATE TABLE pg2dim_h2_t2 PARTITION OF pg2dim_h2 FOR VALUES FROM ('2018-09-01 00:00') TO ('2018-12-01 00:00'); -- Create a 1-dimensional partitioned table for comparison CREATE TABLE pg1dim (time timestamptz, device int, temp float) PARTITION BY HASH (device); CREATE TABLE pg1dim_h1 PARTITION OF pg1dim FOR VALUES WITH (MODULUS 2, REMAINDER 0); CREATE TABLE pg1dim_h2 PARTITION OF pg1dim FOR VALUES WITH (MODULUS 2, REMAINDER 1); INSERT INTO hyper VALUES ('2018-02-19 13:01', 1, 2.3), ('2018-02-19 13:02', 3, 3.1), ('2018-10-19 13:01', 1, 7.6), ('2018-10-19 13:02', 3, 9.0); INSERT INTO pg2dim VALUES ('2018-02-19 13:01', 1, 2.3), ('2018-02-19 13:02', 3, 3.1), ('2018-10-19 13:01', 1, 7.6), ('2018-10-19 13:02', 3, 9.0); INSERT INTO pg1dim VALUES ('2018-02-19 13:01', 1, 2.3), ('2018-02-19 13:02', 3, 3.1), ('2018-10-19 13:01', 1, 7.6), ('2018-10-19 13:02', 3, 9.0); SELECT * FROM test.show_subtables('hyper'); Child | Tablespace ----------------------------------------+------------ _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal._hyper_1_4_chunk | SELECT * FROM pg2dim_h1_t1; time | device | temp ------------------------------+--------+------ Mon Feb 19 13:01:00 2018 PST | 1 | 2.3 SELECT * FROM pg2dim_h1_t2; time | device | temp ------------------------------+--------+------ Fri Oct 19 13:01:00 2018 PDT | 1 | 7.6 SELECT * FROM pg2dim_h2_t1; time | device | temp ------------------------------+--------+------ Mon Feb 19 13:02:00 2018 PST | 3 | 3.1 SELECT * FROM pg2dim_h2_t2; time | device | temp ------------------------------+--------+------ Fri Oct 19 13:02:00 2018 PDT | 3 | 9 -- Compare partitionwise aggreate enabled/disabled. First run queries -- on PG partitioned tables for reference. -- All partition keys covered by GROUP BY SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT device, avg(temp) FROM pg1dim GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: pg1dim.device, (avg(pg1dim.temp)) Sort Key: pg1dim.device -> HashAggregate Output: pg1dim.device, avg(pg1dim.temp) Group Key: pg1dim.device -> Append -> Seq Scan on public.pg1dim_h1 pg1dim_1 Output: pg1dim_1.device, pg1dim_1.temp -> Seq Scan on public.pg1dim_h2 pg1dim_2 Output: pg1dim_2.device, pg1dim_2.temp SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT device, avg(temp) FROM pg1dim GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: pg1dim.device, (avg(pg1dim.temp)) Sort Key: pg1dim.device -> Append -> HashAggregate Output: pg1dim.device, avg(pg1dim.temp) Group Key: pg1dim.device -> Seq Scan on public.pg1dim_h1 pg1dim Output: pg1dim.device, pg1dim.temp -> HashAggregate Output: pg1dim_1.device, avg(pg1dim_1.temp) Group Key: pg1dim_1.device -> Seq Scan on public.pg1dim_h2 pg1dim_1 Output: pg1dim_1.device, pg1dim_1.temp -- All partition keys not covered by GROUP BY (partial partitionwise) SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT device, avg(temp) FROM pg2dim GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: pg2dim.device, (avg(pg2dim.temp)) Sort Key: pg2dim.device -> HashAggregate Output: pg2dim.device, avg(pg2dim.temp) Group Key: pg2dim.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2dim_1 Output: pg2dim_1.device, pg2dim_1.temp -> Seq Scan on public.pg2dim_h1_t2 pg2dim_2 Output: pg2dim_2.device, pg2dim_2.temp -> Seq Scan on public.pg2dim_h2_t1 pg2dim_3 Output: pg2dim_3.device, pg2dim_3.temp -> Seq Scan on public.pg2dim_h2_t2 pg2dim_4 Output: pg2dim_4.device, pg2dim_4.temp SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT device, avg(temp) FROM pg2dim GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: pg2dim.device, (avg(pg2dim.temp)) Sort Key: pg2dim.device -> Finalize HashAggregate Output: pg2dim.device, avg(pg2dim.temp) Group Key: pg2dim.device -> Append -> Partial HashAggregate Output: pg2dim.device, PARTIAL avg(pg2dim.temp) Group Key: pg2dim.device -> Seq Scan on public.pg2dim_h1_t1 pg2dim Output: pg2dim.device, pg2dim.temp -> Partial HashAggregate Output: pg2dim_1.device, PARTIAL avg(pg2dim_1.temp) Group Key: pg2dim_1.device -> Seq Scan on public.pg2dim_h1_t2 pg2dim_1 Output: pg2dim_1.device, pg2dim_1.temp -> Partial HashAggregate Output: pg2dim_2.device, PARTIAL avg(pg2dim_2.temp) Group Key: pg2dim_2.device -> Seq Scan on public.pg2dim_h2_t1 pg2dim_2 Output: pg2dim_2.device, pg2dim_2.temp -> Partial HashAggregate Output: pg2dim_3.device, PARTIAL avg(pg2dim_3.temp) Group Key: pg2dim_3.device -> Seq Scan on public.pg2dim_h2_t2 pg2dim_3 Output: pg2dim_3.device, pg2dim_3.temp -- All partition keys covered by GROUP BY (full partitionwise) SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT time, device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: pg2dim."time", pg2dim.device, (avg(pg2dim.temp)) Sort Key: pg2dim."time", pg2dim.device -> HashAggregate Output: pg2dim."time", pg2dim.device, avg(pg2dim.temp) Group Key: pg2dim."time", pg2dim.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2dim_1 Output: pg2dim_1."time", pg2dim_1.device, pg2dim_1.temp -> Seq Scan on public.pg2dim_h1_t2 pg2dim_2 Output: pg2dim_2."time", pg2dim_2.device, pg2dim_2.temp -> Seq Scan on public.pg2dim_h2_t1 pg2dim_3 Output: pg2dim_3."time", pg2dim_3.device, pg2dim_3.temp -> Seq Scan on public.pg2dim_h2_t2 pg2dim_4 Output: pg2dim_4."time", pg2dim_4.device, pg2dim_4.temp SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT time, device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: pg2dim."time", pg2dim.device, (avg(pg2dim.temp)) Sort Key: pg2dim."time", pg2dim.device -> Append -> HashAggregate Output: pg2dim."time", pg2dim.device, avg(pg2dim.temp) Group Key: pg2dim."time", pg2dim.device -> Seq Scan on public.pg2dim_h1_t1 pg2dim Output: pg2dim."time", pg2dim.device, pg2dim.temp -> HashAggregate Output: pg2dim_1."time", pg2dim_1.device, avg(pg2dim_1.temp) Group Key: pg2dim_1."time", pg2dim_1.device -> Seq Scan on public.pg2dim_h1_t2 pg2dim_1 Output: pg2dim_1."time", pg2dim_1.device, pg2dim_1.temp -> HashAggregate Output: pg2dim_2."time", pg2dim_2.device, avg(pg2dim_2.temp) Group Key: pg2dim_2."time", pg2dim_2.device -> Seq Scan on public.pg2dim_h2_t1 pg2dim_2 Output: pg2dim_2."time", pg2dim_2.device, pg2dim_2.temp -> HashAggregate Output: pg2dim_3."time", pg2dim_3.device, avg(pg2dim_3.temp) Group Key: pg2dim_3."time", pg2dim_3.device -> Seq Scan on public.pg2dim_h2_t2 pg2dim_3 Output: pg2dim_3."time", pg2dim_3.device, pg2dim_3.temp -- All partition keys not covered by GROUP BY because of date_trunc -- expression on time (partial partitionwise) SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, (avg(pg2dim.temp)) Sort Key: (date_trunc('month'::text, pg2dim."time")), pg2dim.device -> HashAggregate Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, avg(pg2dim.temp) Group Key: (date_trunc('month'::text, pg2dim."time")), pg2dim.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2dim_1 Output: date_trunc('month'::text, pg2dim_1."time"), pg2dim_1.device, pg2dim_1.temp -> Seq Scan on public.pg2dim_h1_t2 pg2dim_2 Output: date_trunc('month'::text, pg2dim_2."time"), pg2dim_2.device, pg2dim_2.temp -> Seq Scan on public.pg2dim_h2_t1 pg2dim_3 Output: date_trunc('month'::text, pg2dim_3."time"), pg2dim_3.device, pg2dim_3.temp -> Seq Scan on public.pg2dim_h2_t2 pg2dim_4 Output: date_trunc('month'::text, pg2dim_4."time"), pg2dim_4.device, pg2dim_4.temp SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, (avg(pg2dim.temp)) Sort Key: (date_trunc('month'::text, pg2dim."time")), pg2dim.device -> Finalize HashAggregate Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, avg(pg2dim.temp) Group Key: (date_trunc('month'::text, pg2dim."time")), pg2dim.device -> Append -> Partial HashAggregate Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, PARTIAL avg(pg2dim.temp) Group Key: date_trunc('month'::text, pg2dim."time"), pg2dim.device -> Seq Scan on public.pg2dim_h1_t1 pg2dim Output: date_trunc('month'::text, pg2dim."time"), pg2dim.device, pg2dim.temp -> Partial HashAggregate Output: (date_trunc('month'::text, pg2dim_1."time")), pg2dim_1.device, PARTIAL avg(pg2dim_1.temp) Group Key: date_trunc('month'::text, pg2dim_1."time"), pg2dim_1.device -> Seq Scan on public.pg2dim_h1_t2 pg2dim_1 Output: date_trunc('month'::text, pg2dim_1."time"), pg2dim_1.device, pg2dim_1.temp -> Partial HashAggregate Output: (date_trunc('month'::text, pg2dim_2."time")), pg2dim_2.device, PARTIAL avg(pg2dim_2.temp) Group Key: date_trunc('month'::text, pg2dim_2."time"), pg2dim_2.device -> Seq Scan on public.pg2dim_h2_t1 pg2dim_2 Output: date_trunc('month'::text, pg2dim_2."time"), pg2dim_2.device, pg2dim_2.temp -> Partial HashAggregate Output: (date_trunc('month'::text, pg2dim_3."time")), pg2dim_3.device, PARTIAL avg(pg2dim_3.temp) Group Key: date_trunc('month'::text, pg2dim_3."time"), pg2dim_3.device -> Seq Scan on public.pg2dim_h2_t2 pg2dim_3 Output: date_trunc('month'::text, pg2dim_3."time"), pg2dim_3.device, pg2dim_3.temp -- Now run on hypertable -- All partition keys not covered by GROUP BY (partial partitionwise) SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT device, avg(temp) FROM hyper GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: hyper.device, (avg(hyper.temp)) Sort Key: hyper.device -> HashAggregate Output: hyper.device, avg(hyper.temp) Group Key: hyper.device -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT device, avg(temp) FROM hyper GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: hyper.device, (avg(hyper.temp)) Sort Key: hyper.device -> HashAggregate Output: hyper.device, avg(hyper.temp) Group Key: hyper.device -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp -- All partition keys covered (full partitionwise) SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT time, device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: hyper."time", hyper.device, (avg(hyper.temp)) Sort Key: hyper."time", hyper.device -> HashAggregate Output: hyper."time", hyper.device, avg(hyper.temp) Group Key: hyper."time", hyper.device -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT time, device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: hyper."time", hyper.device, (avg(hyper.temp)) Sort Key: hyper."time", hyper.device -> HashAggregate Output: hyper."time", hyper.device, avg(hyper.temp) Group Key: hyper."time", hyper.device -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp -- Partial aggregation since date_trunc(time) is not a partition key SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (date_trunc('month'::text, hyper."time")), hyper.device, (avg(hyper.temp)) Sort Key: (date_trunc('month'::text, hyper."time")), hyper.device -> HashAggregate Output: (date_trunc('month'::text, hyper."time")), hyper.device, avg(hyper.temp) Group Key: date_trunc('month'::text, hyper."time"), hyper.device -> Result Output: date_trunc('month'::text, hyper."time"), hyper.device, hyper.temp -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp -- Partial aggregation pushdown is currently not supported for this query by -- the TSDB pushdown code since a projection is used in the path. SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (date_trunc('month'::text, hyper."time")), hyper.device, (avg(hyper.temp)) Sort Key: (date_trunc('month'::text, hyper."time")), hyper.device -> HashAggregate Output: (date_trunc('month'::text, hyper."time")), hyper.device, avg(hyper.temp) Group Key: date_trunc('month'::text, hyper."time"), hyper.device -> Result Output: date_trunc('month'::text, hyper."time"), hyper.device, hyper.temp -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp -- Also test time_bucket SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT time_bucket('1 month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device, (avg(hyper.temp)) Sort Key: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device -> HashAggregate Output: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device, avg(hyper.temp) Group Key: time_bucket('@ 1 mon'::interval, hyper."time"), hyper.device -> Result Output: time_bucket('@ 1 mon'::interval, hyper."time"), hyper.device, hyper.temp -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT time_bucket('1 month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device, (avg(hyper.temp)) Sort Key: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device -> HashAggregate Output: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device, avg(hyper.temp) Group Key: time_bucket('@ 1 mon'::interval, hyper."time"), hyper.device -> Result Output: time_bucket('@ 1 mon'::interval, hyper."time"), hyper.device, hyper.temp -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp -- Test partitionwise joins, mostly to see that we do not break -- anything CREATE TABLE hyper_meta (time timestamptz, device int, info text); SELECT * FROM create_hypertable('hyper_meta', 'time', 'device', 2); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 2 | public | hyper_meta | t INSERT INTO hyper_meta VALUES ('2018-02-19 13:01', 1, 'device_1'), ('2018-02-19 13:02', 3, 'device_3'); SET enable_partitionwise_join = 'off'; :PREFIX SELECT h.time, h.device, h.temp, hm.info FROM hyper h, hyper_meta hm WHERE h.device = hm.device; --- QUERY PLAN --- Merge Join Output: h."time", h.device, h.temp, hm.info Merge Cond: (hm.device = h.device) -> Merge Append Sort Key: hm.device -> Index Scan using _hyper_2_5_chunk_hyper_meta_device_time_idx on _timescaledb_internal._hyper_2_5_chunk hm_1 Output: hm_1.info, hm_1.device -> Index Scan using _hyper_2_6_chunk_hyper_meta_device_time_idx on _timescaledb_internal._hyper_2_6_chunk hm_2 Output: hm_2.info, hm_2.device -> Materialize Output: h."time", h.device, h.temp -> Merge Append Sort Key: h.device -> Index Scan using _hyper_1_1_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_1_chunk h_1 Output: h_1."time", h_1.device, h_1.temp -> Index Scan using _hyper_1_2_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_2_chunk h_2 Output: h_2."time", h_2.device, h_2.temp -> Index Scan using _hyper_1_3_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_3_chunk h_3 Output: h_3."time", h_3.device, h_3.temp -> Index Scan using _hyper_1_4_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_4_chunk h_4 Output: h_4."time", h_4.device, h_4.temp :PREFIX SELECT pg2.time, pg2.device, pg2.temp, pg1.temp FROM pg2dim pg2, pg1dim pg1 WHERE pg2.device = pg1.device; --- QUERY PLAN --- Merge Join Output: pg2."time", pg2.device, pg2.temp, pg1.temp Merge Cond: (pg1.device = pg2.device) -> Sort Output: pg1.temp, pg1.device Sort Key: pg1.device -> Append -> Seq Scan on public.pg1dim_h1 pg1_1 Output: pg1_1.temp, pg1_1.device -> Seq Scan on public.pg1dim_h2 pg1_2 Output: pg1_2.temp, pg1_2.device -> Sort Output: pg2."time", pg2.device, pg2.temp Sort Key: pg2.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2_1 Output: pg2_1."time", pg2_1.device, pg2_1.temp -> Seq Scan on public.pg2dim_h1_t2 pg2_2 Output: pg2_2."time", pg2_2.device, pg2_2.temp -> Seq Scan on public.pg2dim_h2_t1 pg2_3 Output: pg2_3."time", pg2_3.device, pg2_3.temp -> Seq Scan on public.pg2dim_h2_t2 pg2_4 Output: pg2_4."time", pg2_4.device, pg2_4.temp SET enable_partitionwise_join = 'on'; :PREFIX SELECT h.time, h.device, h.temp, hm.info FROM hyper h, hyper_meta hm WHERE h.device = hm.device; --- QUERY PLAN --- Merge Join Output: h."time", h.device, h.temp, hm.info Merge Cond: (hm.device = h.device) -> Merge Append Sort Key: hm.device -> Index Scan using _hyper_2_5_chunk_hyper_meta_device_time_idx on _timescaledb_internal._hyper_2_5_chunk hm_1 Output: hm_1.info, hm_1.device -> Index Scan using _hyper_2_6_chunk_hyper_meta_device_time_idx on _timescaledb_internal._hyper_2_6_chunk hm_2 Output: hm_2.info, hm_2.device -> Materialize Output: h."time", h.device, h.temp -> Merge Append Sort Key: h.device -> Index Scan using _hyper_1_1_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_1_chunk h_1 Output: h_1."time", h_1.device, h_1.temp -> Index Scan using _hyper_1_2_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_2_chunk h_2 Output: h_2."time", h_2.device, h_2.temp -> Index Scan using _hyper_1_3_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_3_chunk h_3 Output: h_3."time", h_3.device, h_3.temp -> Index Scan using _hyper_1_4_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_4_chunk h_4 Output: h_4."time", h_4.device, h_4.temp :PREFIX SELECT pg2.time, pg2.device, pg2.temp, pg1.temp FROM pg2dim pg2, pg1dim pg1 WHERE pg2.device = pg1.device; --- QUERY PLAN --- Append -> Merge Join Output: pg2_2."time", pg2_2.device, pg2_2.temp, pg1_1.temp Merge Cond: (pg1_1.device = pg2_2.device) -> Sort Output: pg1_1.temp, pg1_1.device Sort Key: pg1_1.device -> Seq Scan on public.pg1dim_h1 pg1_1 Output: pg1_1.temp, pg1_1.device -> Sort Output: pg2_2."time", pg2_2.device, pg2_2.temp Sort Key: pg2_2.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2_2 Output: pg2_2."time", pg2_2.device, pg2_2.temp -> Seq Scan on public.pg2dim_h1_t2 pg2_3 Output: pg2_3."time", pg2_3.device, pg2_3.temp -> Merge Join Output: pg2_5."time", pg2_5.device, pg2_5.temp, pg1_2.temp Merge Cond: (pg1_2.device = pg2_5.device) -> Sort Output: pg1_2.temp, pg1_2.device Sort Key: pg1_2.device -> Seq Scan on public.pg1dim_h2 pg1_2 Output: pg1_2.temp, pg1_2.device -> Sort Output: pg2_5."time", pg2_5.device, pg2_5.temp Sort Key: pg2_5.device -> Append -> Seq Scan on public.pg2dim_h2_t1 pg2_5 Output: pg2_5."time", pg2_5.device, pg2_5.temp -> Seq Scan on public.pg2dim_h2_t2 pg2_6 Output: pg2_6."time", pg2_6.device, pg2_6.temp -- Test hypertable with time partitioning function CREATE OR REPLACE FUNCTION time_func(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ DECLARE retval TIMESTAMPTZ; BEGIN retval := to_timestamp(unixtime); RETURN retval; END $BODY$; CREATE TABLE hyper_timepart (time float8, device int, temp float); SELECT * FROM create_hypertable('hyper_timepart', 'time', 'device', 2, time_partitioning_func => 'time_func'); hypertable_id | schema_name | table_name | created ---------------+-------------+----------------+--------- 3 | public | hyper_timepart | t -- Planner won't pick push-down aggs on table with time function -- unless a certain amount of data SELECT setseed(1); setseed --------- INSERT INTO hyper_timepart SELECT x, ceil(random() * 8), random() * 20 FROM generate_series(0,5000-1) AS x; -- All partition keys covered (full partitionwise) SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT time, device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; --- QUERY PLAN --- Limit Output: hyper_timepart."time", hyper_timepart.device, (avg(hyper_timepart.temp)) -> Sort Output: hyper_timepart."time", hyper_timepart.device, (avg(hyper_timepart.temp)) Sort Key: hyper_timepart."time", hyper_timepart.device -> HashAggregate Output: hyper_timepart."time", hyper_timepart.device, avg(hyper_timepart.temp) Group Key: hyper_timepart."time", hyper_timepart.device -> Append -> Seq Scan on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp :PREFIX SELECT time_func(time), device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; --- QUERY PLAN --- Limit Output: (time_func(hyper_timepart."time")), hyper_timepart.device, (avg(hyper_timepart.temp)) -> GroupAggregate Output: (time_func(hyper_timepart."time")), hyper_timepart.device, avg(hyper_timepart.temp) Group Key: (time_func(hyper_timepart."time")), hyper_timepart.device -> Incremental Sort Output: (time_func(hyper_timepart."time")), hyper_timepart.device, hyper_timepart.temp Sort Key: (time_func(hyper_timepart."time")), hyper_timepart.device Presorted Key: (time_func(hyper_timepart."time")) -> Result Output: (time_func(hyper_timepart."time")), hyper_timepart.device, hyper_timepart.temp -> Merge Append Sort Key: (time_func(hyper_timepart."time")) -> Index Scan Backward using _hyper_3_7_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp, time_func(_hyper_3_7_chunk."time") -> Index Scan Backward using _hyper_3_8_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp, time_func(_hyper_3_8_chunk."time") -- Grouping on original time column should be pushed-down SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT time, device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; --- QUERY PLAN --- Limit Output: hyper_timepart."time", hyper_timepart.device, (avg(hyper_timepart.temp)) -> Sort Output: hyper_timepart."time", hyper_timepart.device, (avg(hyper_timepart.temp)) Sort Key: hyper_timepart."time", hyper_timepart.device -> HashAggregate Output: hyper_timepart."time", hyper_timepart.device, avg(hyper_timepart.temp) Group Key: hyper_timepart."time", hyper_timepart.device -> Append -> Seq Scan on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp -- Applying the time partitioning function should also allow push-down -- on open dimensions :PREFIX SELECT time_func(time), device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; --- QUERY PLAN --- Limit Output: (time_func(hyper_timepart."time")), hyper_timepart.device, (avg(hyper_timepart.temp)) -> GroupAggregate Output: (time_func(hyper_timepart."time")), hyper_timepart.device, avg(hyper_timepart.temp) Group Key: (time_func(hyper_timepart."time")), hyper_timepart.device -> Incremental Sort Output: (time_func(hyper_timepart."time")), hyper_timepart.device, hyper_timepart.temp Sort Key: (time_func(hyper_timepart."time")), hyper_timepart.device Presorted Key: (time_func(hyper_timepart."time")) -> Result Output: (time_func(hyper_timepart."time")), hyper_timepart.device, hyper_timepart.temp -> Merge Append Sort Key: (time_func(hyper_timepart."time")) -> Index Scan Backward using _hyper_3_7_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp, time_func(_hyper_3_7_chunk."time") -> Index Scan Backward using _hyper_3_8_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp, time_func(_hyper_3_8_chunk."time") -- Partial aggregation pushdown is currently not supported for this query by -- the TSDB pushdown code since a projection is used in the path. :PREFIX SELECT time_func(time), _timescaledb_functions.get_partition_hash(device), avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; --- QUERY PLAN --- Limit Output: (time_func(hyper_timepart."time")), (_timescaledb_functions.get_partition_hash(hyper_timepart.device)), (avg(hyper_timepart.temp)) -> GroupAggregate Output: (time_func(hyper_timepart."time")), (_timescaledb_functions.get_partition_hash(hyper_timepart.device)), avg(hyper_timepart.temp) Group Key: (time_func(hyper_timepart."time")), (_timescaledb_functions.get_partition_hash(hyper_timepart.device)) -> Incremental Sort Output: (time_func(hyper_timepart."time")), (_timescaledb_functions.get_partition_hash(hyper_timepart.device)), hyper_timepart.temp Sort Key: (time_func(hyper_timepart."time")), (_timescaledb_functions.get_partition_hash(hyper_timepart.device)) Presorted Key: (time_func(hyper_timepart."time")) -> Result Output: (time_func(hyper_timepart."time")), _timescaledb_functions.get_partition_hash(hyper_timepart.device), hyper_timepart.temp -> Merge Append Sort Key: (time_func(hyper_timepart."time")) -> Index Scan Backward using _hyper_3_7_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp, time_func(_hyper_3_7_chunk."time") -> Index Scan Backward using _hyper_3_8_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp, time_func(_hyper_3_8_chunk."time") -- Test removal of redundant group key optimization in PG16 -- All lower versions include the redundant key on device column :PREFIX SELECT device, avg(temp) FROM hyper_timepart WHERE device = 1 GROUP BY 1 LIMIT 10; --- QUERY PLAN --- Limit Output: _hyper_3_8_chunk.device, (avg(_hyper_3_8_chunk.temp)) -> GroupAggregate Output: _hyper_3_8_chunk.device, avg(_hyper_3_8_chunk.temp) Group Key: _hyper_3_8_chunk.device -> Index Scan using _hyper_3_8_chunk_hyper_timepart_device_expr_idx on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp Index Cond: (_hyper_3_8_chunk.device = 1) ================================================ FILE: test/expected/partitionwise-16.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set PREFIX 'EXPLAIN (VERBOSE, BUFFERS OFF, COSTS OFF)' -- Create a two dimensional hypertable CREATE TABLE hyper (time timestamptz, device int, temp float); SELECT * FROM create_hypertable('hyper', 'time', 'device', 2); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | hyper | t -- Create a similar PostgreSQL partitioned table CREATE TABLE pg2dim (time timestamptz, device int, temp float) PARTITION BY HASH (device); CREATE TABLE pg2dim_h1 PARTITION OF pg2dim FOR VALUES WITH (MODULUS 2, REMAINDER 0) PARTITION BY RANGE(time); CREATE TABLE pg2dim_h2 PARTITION OF pg2dim FOR VALUES WITH (MODULUS 2, REMAINDER 1) PARTITION BY RANGE(time); CREATE TABLE pg2dim_h1_t1 PARTITION OF pg2dim_h1 FOR VALUES FROM ('2018-01-01 00:00') TO ('2018-09-01 00:00'); CREATE TABLE pg2dim_h1_t2 PARTITION OF pg2dim_h1 FOR VALUES FROM ('2018-09-01 00:00') TO ('2018-12-01 00:00'); CREATE TABLE pg2dim_h2_t1 PARTITION OF pg2dim_h2 FOR VALUES FROM ('2018-01-01 00:00') TO ('2018-09-01 00:00'); CREATE TABLE pg2dim_h2_t2 PARTITION OF pg2dim_h2 FOR VALUES FROM ('2018-09-01 00:00') TO ('2018-12-01 00:00'); -- Create a 1-dimensional partitioned table for comparison CREATE TABLE pg1dim (time timestamptz, device int, temp float) PARTITION BY HASH (device); CREATE TABLE pg1dim_h1 PARTITION OF pg1dim FOR VALUES WITH (MODULUS 2, REMAINDER 0); CREATE TABLE pg1dim_h2 PARTITION OF pg1dim FOR VALUES WITH (MODULUS 2, REMAINDER 1); INSERT INTO hyper VALUES ('2018-02-19 13:01', 1, 2.3), ('2018-02-19 13:02', 3, 3.1), ('2018-10-19 13:01', 1, 7.6), ('2018-10-19 13:02', 3, 9.0); INSERT INTO pg2dim VALUES ('2018-02-19 13:01', 1, 2.3), ('2018-02-19 13:02', 3, 3.1), ('2018-10-19 13:01', 1, 7.6), ('2018-10-19 13:02', 3, 9.0); INSERT INTO pg1dim VALUES ('2018-02-19 13:01', 1, 2.3), ('2018-02-19 13:02', 3, 3.1), ('2018-10-19 13:01', 1, 7.6), ('2018-10-19 13:02', 3, 9.0); SELECT * FROM test.show_subtables('hyper'); Child | Tablespace ----------------------------------------+------------ _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal._hyper_1_4_chunk | SELECT * FROM pg2dim_h1_t1; time | device | temp ------------------------------+--------+------ Mon Feb 19 13:01:00 2018 PST | 1 | 2.3 SELECT * FROM pg2dim_h1_t2; time | device | temp ------------------------------+--------+------ Fri Oct 19 13:01:00 2018 PDT | 1 | 7.6 SELECT * FROM pg2dim_h2_t1; time | device | temp ------------------------------+--------+------ Mon Feb 19 13:02:00 2018 PST | 3 | 3.1 SELECT * FROM pg2dim_h2_t2; time | device | temp ------------------------------+--------+------ Fri Oct 19 13:02:00 2018 PDT | 3 | 9 -- Compare partitionwise aggreate enabled/disabled. First run queries -- on PG partitioned tables for reference. -- All partition keys covered by GROUP BY SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT device, avg(temp) FROM pg1dim GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: pg1dim.device, (avg(pg1dim.temp)) Sort Key: pg1dim.device -> HashAggregate Output: pg1dim.device, avg(pg1dim.temp) Group Key: pg1dim.device -> Append -> Seq Scan on public.pg1dim_h1 pg1dim_1 Output: pg1dim_1.device, pg1dim_1.temp -> Seq Scan on public.pg1dim_h2 pg1dim_2 Output: pg1dim_2.device, pg1dim_2.temp SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT device, avg(temp) FROM pg1dim GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: pg1dim.device, (avg(pg1dim.temp)) Sort Key: pg1dim.device -> Append -> HashAggregate Output: pg1dim.device, avg(pg1dim.temp) Group Key: pg1dim.device -> Seq Scan on public.pg1dim_h1 pg1dim Output: pg1dim.device, pg1dim.temp -> HashAggregate Output: pg1dim_1.device, avg(pg1dim_1.temp) Group Key: pg1dim_1.device -> Seq Scan on public.pg1dim_h2 pg1dim_1 Output: pg1dim_1.device, pg1dim_1.temp -- All partition keys not covered by GROUP BY (partial partitionwise) SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT device, avg(temp) FROM pg2dim GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: pg2dim.device, (avg(pg2dim.temp)) Sort Key: pg2dim.device -> HashAggregate Output: pg2dim.device, avg(pg2dim.temp) Group Key: pg2dim.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2dim_1 Output: pg2dim_1.device, pg2dim_1.temp -> Seq Scan on public.pg2dim_h1_t2 pg2dim_2 Output: pg2dim_2.device, pg2dim_2.temp -> Seq Scan on public.pg2dim_h2_t1 pg2dim_3 Output: pg2dim_3.device, pg2dim_3.temp -> Seq Scan on public.pg2dim_h2_t2 pg2dim_4 Output: pg2dim_4.device, pg2dim_4.temp SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT device, avg(temp) FROM pg2dim GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: pg2dim.device, (avg(pg2dim.temp)) Sort Key: pg2dim.device -> Finalize HashAggregate Output: pg2dim.device, avg(pg2dim.temp) Group Key: pg2dim.device -> Append -> Partial HashAggregate Output: pg2dim.device, PARTIAL avg(pg2dim.temp) Group Key: pg2dim.device -> Seq Scan on public.pg2dim_h1_t1 pg2dim Output: pg2dim.device, pg2dim.temp -> Partial HashAggregate Output: pg2dim_1.device, PARTIAL avg(pg2dim_1.temp) Group Key: pg2dim_1.device -> Seq Scan on public.pg2dim_h1_t2 pg2dim_1 Output: pg2dim_1.device, pg2dim_1.temp -> Partial HashAggregate Output: pg2dim_2.device, PARTIAL avg(pg2dim_2.temp) Group Key: pg2dim_2.device -> Seq Scan on public.pg2dim_h2_t1 pg2dim_2 Output: pg2dim_2.device, pg2dim_2.temp -> Partial HashAggregate Output: pg2dim_3.device, PARTIAL avg(pg2dim_3.temp) Group Key: pg2dim_3.device -> Seq Scan on public.pg2dim_h2_t2 pg2dim_3 Output: pg2dim_3.device, pg2dim_3.temp -- All partition keys covered by GROUP BY (full partitionwise) SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT time, device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: pg2dim."time", pg2dim.device, (avg(pg2dim.temp)) Sort Key: pg2dim."time", pg2dim.device -> HashAggregate Output: pg2dim."time", pg2dim.device, avg(pg2dim.temp) Group Key: pg2dim."time", pg2dim.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2dim_1 Output: pg2dim_1."time", pg2dim_1.device, pg2dim_1.temp -> Seq Scan on public.pg2dim_h1_t2 pg2dim_2 Output: pg2dim_2."time", pg2dim_2.device, pg2dim_2.temp -> Seq Scan on public.pg2dim_h2_t1 pg2dim_3 Output: pg2dim_3."time", pg2dim_3.device, pg2dim_3.temp -> Seq Scan on public.pg2dim_h2_t2 pg2dim_4 Output: pg2dim_4."time", pg2dim_4.device, pg2dim_4.temp SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT time, device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: pg2dim."time", pg2dim.device, (avg(pg2dim.temp)) Sort Key: pg2dim."time", pg2dim.device -> Append -> HashAggregate Output: pg2dim."time", pg2dim.device, avg(pg2dim.temp) Group Key: pg2dim."time", pg2dim.device -> Seq Scan on public.pg2dim_h1_t1 pg2dim Output: pg2dim."time", pg2dim.device, pg2dim.temp -> HashAggregate Output: pg2dim_1."time", pg2dim_1.device, avg(pg2dim_1.temp) Group Key: pg2dim_1."time", pg2dim_1.device -> Seq Scan on public.pg2dim_h1_t2 pg2dim_1 Output: pg2dim_1."time", pg2dim_1.device, pg2dim_1.temp -> HashAggregate Output: pg2dim_2."time", pg2dim_2.device, avg(pg2dim_2.temp) Group Key: pg2dim_2."time", pg2dim_2.device -> Seq Scan on public.pg2dim_h2_t1 pg2dim_2 Output: pg2dim_2."time", pg2dim_2.device, pg2dim_2.temp -> HashAggregate Output: pg2dim_3."time", pg2dim_3.device, avg(pg2dim_3.temp) Group Key: pg2dim_3."time", pg2dim_3.device -> Seq Scan on public.pg2dim_h2_t2 pg2dim_3 Output: pg2dim_3."time", pg2dim_3.device, pg2dim_3.temp -- All partition keys not covered by GROUP BY because of date_trunc -- expression on time (partial partitionwise) SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, (avg(pg2dim.temp)) Sort Key: (date_trunc('month'::text, pg2dim."time")), pg2dim.device -> HashAggregate Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, avg(pg2dim.temp) Group Key: (date_trunc('month'::text, pg2dim."time")), pg2dim.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2dim_1 Output: date_trunc('month'::text, pg2dim_1."time"), pg2dim_1.device, pg2dim_1.temp -> Seq Scan on public.pg2dim_h1_t2 pg2dim_2 Output: date_trunc('month'::text, pg2dim_2."time"), pg2dim_2.device, pg2dim_2.temp -> Seq Scan on public.pg2dim_h2_t1 pg2dim_3 Output: date_trunc('month'::text, pg2dim_3."time"), pg2dim_3.device, pg2dim_3.temp -> Seq Scan on public.pg2dim_h2_t2 pg2dim_4 Output: date_trunc('month'::text, pg2dim_4."time"), pg2dim_4.device, pg2dim_4.temp SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, (avg(pg2dim.temp)) Sort Key: (date_trunc('month'::text, pg2dim."time")), pg2dim.device -> Finalize HashAggregate Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, avg(pg2dim.temp) Group Key: (date_trunc('month'::text, pg2dim."time")), pg2dim.device -> Append -> Partial HashAggregate Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, PARTIAL avg(pg2dim.temp) Group Key: date_trunc('month'::text, pg2dim."time"), pg2dim.device -> Seq Scan on public.pg2dim_h1_t1 pg2dim Output: date_trunc('month'::text, pg2dim."time"), pg2dim.device, pg2dim.temp -> Partial HashAggregate Output: (date_trunc('month'::text, pg2dim_1."time")), pg2dim_1.device, PARTIAL avg(pg2dim_1.temp) Group Key: date_trunc('month'::text, pg2dim_1."time"), pg2dim_1.device -> Seq Scan on public.pg2dim_h1_t2 pg2dim_1 Output: date_trunc('month'::text, pg2dim_1."time"), pg2dim_1.device, pg2dim_1.temp -> Partial HashAggregate Output: (date_trunc('month'::text, pg2dim_2."time")), pg2dim_2.device, PARTIAL avg(pg2dim_2.temp) Group Key: date_trunc('month'::text, pg2dim_2."time"), pg2dim_2.device -> Seq Scan on public.pg2dim_h2_t1 pg2dim_2 Output: date_trunc('month'::text, pg2dim_2."time"), pg2dim_2.device, pg2dim_2.temp -> Partial HashAggregate Output: (date_trunc('month'::text, pg2dim_3."time")), pg2dim_3.device, PARTIAL avg(pg2dim_3.temp) Group Key: date_trunc('month'::text, pg2dim_3."time"), pg2dim_3.device -> Seq Scan on public.pg2dim_h2_t2 pg2dim_3 Output: date_trunc('month'::text, pg2dim_3."time"), pg2dim_3.device, pg2dim_3.temp -- Now run on hypertable -- All partition keys not covered by GROUP BY (partial partitionwise) SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT device, avg(temp) FROM hyper GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: hyper.device, (avg(hyper.temp)) Sort Key: hyper.device -> HashAggregate Output: hyper.device, avg(hyper.temp) Group Key: hyper.device -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT device, avg(temp) FROM hyper GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: hyper.device, (avg(hyper.temp)) Sort Key: hyper.device -> HashAggregate Output: hyper.device, avg(hyper.temp) Group Key: hyper.device -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp -- All partition keys covered (full partitionwise) SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT time, device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: hyper."time", hyper.device, (avg(hyper.temp)) Sort Key: hyper."time", hyper.device -> HashAggregate Output: hyper."time", hyper.device, avg(hyper.temp) Group Key: hyper."time", hyper.device -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT time, device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: hyper."time", hyper.device, (avg(hyper.temp)) Sort Key: hyper."time", hyper.device -> HashAggregate Output: hyper."time", hyper.device, avg(hyper.temp) Group Key: hyper."time", hyper.device -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp -- Partial aggregation since date_trunc(time) is not a partition key SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (date_trunc('month'::text, hyper."time")), hyper.device, (avg(hyper.temp)) Sort Key: (date_trunc('month'::text, hyper."time")), hyper.device -> HashAggregate Output: (date_trunc('month'::text, hyper."time")), hyper.device, avg(hyper.temp) Group Key: date_trunc('month'::text, hyper."time"), hyper.device -> Result Output: date_trunc('month'::text, hyper."time"), hyper.device, hyper.temp -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp -- Partial aggregation pushdown is currently not supported for this query by -- the TSDB pushdown code since a projection is used in the path. SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (date_trunc('month'::text, hyper."time")), hyper.device, (avg(hyper.temp)) Sort Key: (date_trunc('month'::text, hyper."time")), hyper.device -> HashAggregate Output: (date_trunc('month'::text, hyper."time")), hyper.device, avg(hyper.temp) Group Key: date_trunc('month'::text, hyper."time"), hyper.device -> Result Output: date_trunc('month'::text, hyper."time"), hyper.device, hyper.temp -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp -- Also test time_bucket SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT time_bucket('1 month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device, (avg(hyper.temp)) Sort Key: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device -> HashAggregate Output: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device, avg(hyper.temp) Group Key: time_bucket('@ 1 mon'::interval, hyper."time"), hyper.device -> Result Output: time_bucket('@ 1 mon'::interval, hyper."time"), hyper.device, hyper.temp -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT time_bucket('1 month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device, (avg(hyper.temp)) Sort Key: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device -> HashAggregate Output: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device, avg(hyper.temp) Group Key: time_bucket('@ 1 mon'::interval, hyper."time"), hyper.device -> Result Output: time_bucket('@ 1 mon'::interval, hyper."time"), hyper.device, hyper.temp -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp -- Test partitionwise joins, mostly to see that we do not break -- anything CREATE TABLE hyper_meta (time timestamptz, device int, info text); SELECT * FROM create_hypertable('hyper_meta', 'time', 'device', 2); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 2 | public | hyper_meta | t INSERT INTO hyper_meta VALUES ('2018-02-19 13:01', 1, 'device_1'), ('2018-02-19 13:02', 3, 'device_3'); SET enable_partitionwise_join = 'off'; :PREFIX SELECT h.time, h.device, h.temp, hm.info FROM hyper h, hyper_meta hm WHERE h.device = hm.device; --- QUERY PLAN --- Merge Join Output: h."time", h.device, h.temp, hm.info Merge Cond: (hm.device = h.device) -> Merge Append Sort Key: hm.device -> Index Scan using _hyper_2_5_chunk_hyper_meta_device_time_idx on _timescaledb_internal._hyper_2_5_chunk hm_1 Output: hm_1.info, hm_1.device -> Index Scan using _hyper_2_6_chunk_hyper_meta_device_time_idx on _timescaledb_internal._hyper_2_6_chunk hm_2 Output: hm_2.info, hm_2.device -> Materialize Output: h."time", h.device, h.temp -> Merge Append Sort Key: h.device -> Index Scan using _hyper_1_1_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_1_chunk h_1 Output: h_1."time", h_1.device, h_1.temp -> Index Scan using _hyper_1_2_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_2_chunk h_2 Output: h_2."time", h_2.device, h_2.temp -> Index Scan using _hyper_1_3_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_3_chunk h_3 Output: h_3."time", h_3.device, h_3.temp -> Index Scan using _hyper_1_4_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_4_chunk h_4 Output: h_4."time", h_4.device, h_4.temp :PREFIX SELECT pg2.time, pg2.device, pg2.temp, pg1.temp FROM pg2dim pg2, pg1dim pg1 WHERE pg2.device = pg1.device; --- QUERY PLAN --- Merge Join Output: pg2."time", pg2.device, pg2.temp, pg1.temp Merge Cond: (pg1.device = pg2.device) -> Sort Output: pg1.temp, pg1.device Sort Key: pg1.device -> Append -> Seq Scan on public.pg1dim_h1 pg1_1 Output: pg1_1.temp, pg1_1.device -> Seq Scan on public.pg1dim_h2 pg1_2 Output: pg1_2.temp, pg1_2.device -> Sort Output: pg2."time", pg2.device, pg2.temp Sort Key: pg2.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2_1 Output: pg2_1."time", pg2_1.device, pg2_1.temp -> Seq Scan on public.pg2dim_h1_t2 pg2_2 Output: pg2_2."time", pg2_2.device, pg2_2.temp -> Seq Scan on public.pg2dim_h2_t1 pg2_3 Output: pg2_3."time", pg2_3.device, pg2_3.temp -> Seq Scan on public.pg2dim_h2_t2 pg2_4 Output: pg2_4."time", pg2_4.device, pg2_4.temp SET enable_partitionwise_join = 'on'; :PREFIX SELECT h.time, h.device, h.temp, hm.info FROM hyper h, hyper_meta hm WHERE h.device = hm.device; --- QUERY PLAN --- Merge Join Output: h."time", h.device, h.temp, hm.info Merge Cond: (hm.device = h.device) -> Merge Append Sort Key: hm.device -> Index Scan using _hyper_2_5_chunk_hyper_meta_device_time_idx on _timescaledb_internal._hyper_2_5_chunk hm_1 Output: hm_1.info, hm_1.device -> Index Scan using _hyper_2_6_chunk_hyper_meta_device_time_idx on _timescaledb_internal._hyper_2_6_chunk hm_2 Output: hm_2.info, hm_2.device -> Materialize Output: h."time", h.device, h.temp -> Merge Append Sort Key: h.device -> Index Scan using _hyper_1_1_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_1_chunk h_1 Output: h_1."time", h_1.device, h_1.temp -> Index Scan using _hyper_1_2_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_2_chunk h_2 Output: h_2."time", h_2.device, h_2.temp -> Index Scan using _hyper_1_3_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_3_chunk h_3 Output: h_3."time", h_3.device, h_3.temp -> Index Scan using _hyper_1_4_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_4_chunk h_4 Output: h_4."time", h_4.device, h_4.temp :PREFIX SELECT pg2.time, pg2.device, pg2.temp, pg1.temp FROM pg2dim pg2, pg1dim pg1 WHERE pg2.device = pg1.device; --- QUERY PLAN --- Append -> Merge Join Output: pg2_2."time", pg2_2.device, pg2_2.temp, pg1_1.temp Merge Cond: (pg1_1.device = pg2_2.device) -> Sort Output: pg1_1.temp, pg1_1.device Sort Key: pg1_1.device -> Seq Scan on public.pg1dim_h1 pg1_1 Output: pg1_1.temp, pg1_1.device -> Sort Output: pg2_2."time", pg2_2.device, pg2_2.temp Sort Key: pg2_2.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2_2 Output: pg2_2."time", pg2_2.device, pg2_2.temp -> Seq Scan on public.pg2dim_h1_t2 pg2_3 Output: pg2_3."time", pg2_3.device, pg2_3.temp -> Merge Join Output: pg2_5."time", pg2_5.device, pg2_5.temp, pg1_2.temp Merge Cond: (pg1_2.device = pg2_5.device) -> Sort Output: pg1_2.temp, pg1_2.device Sort Key: pg1_2.device -> Seq Scan on public.pg1dim_h2 pg1_2 Output: pg1_2.temp, pg1_2.device -> Sort Output: pg2_5."time", pg2_5.device, pg2_5.temp Sort Key: pg2_5.device -> Append -> Seq Scan on public.pg2dim_h2_t1 pg2_5 Output: pg2_5."time", pg2_5.device, pg2_5.temp -> Seq Scan on public.pg2dim_h2_t2 pg2_6 Output: pg2_6."time", pg2_6.device, pg2_6.temp -- Test hypertable with time partitioning function CREATE OR REPLACE FUNCTION time_func(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ DECLARE retval TIMESTAMPTZ; BEGIN retval := to_timestamp(unixtime); RETURN retval; END $BODY$; CREATE TABLE hyper_timepart (time float8, device int, temp float); SELECT * FROM create_hypertable('hyper_timepart', 'time', 'device', 2, time_partitioning_func => 'time_func'); hypertable_id | schema_name | table_name | created ---------------+-------------+----------------+--------- 3 | public | hyper_timepart | t -- Planner won't pick push-down aggs on table with time function -- unless a certain amount of data SELECT setseed(1); setseed --------- INSERT INTO hyper_timepart SELECT x, ceil(random() * 8), random() * 20 FROM generate_series(0,5000-1) AS x; -- All partition keys covered (full partitionwise) SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT time, device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; --- QUERY PLAN --- Limit Output: hyper_timepart."time", hyper_timepart.device, (avg(hyper_timepart.temp)) -> Sort Output: hyper_timepart."time", hyper_timepart.device, (avg(hyper_timepart.temp)) Sort Key: hyper_timepart."time", hyper_timepart.device -> HashAggregate Output: hyper_timepart."time", hyper_timepart.device, avg(hyper_timepart.temp) Group Key: hyper_timepart."time", hyper_timepart.device -> Append -> Seq Scan on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp :PREFIX SELECT time_func(time), device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; --- QUERY PLAN --- Limit Output: (time_func(hyper_timepart."time")), hyper_timepart.device, (avg(hyper_timepart.temp)) -> GroupAggregate Output: (time_func(hyper_timepart."time")), hyper_timepart.device, avg(hyper_timepart.temp) Group Key: (time_func(hyper_timepart."time")), hyper_timepart.device -> Incremental Sort Output: (time_func(hyper_timepart."time")), hyper_timepart.device, hyper_timepart.temp Sort Key: (time_func(hyper_timepart."time")), hyper_timepart.device Presorted Key: (time_func(hyper_timepart."time")) -> Result Output: (time_func(hyper_timepart."time")), hyper_timepart.device, hyper_timepart.temp -> Merge Append Sort Key: (time_func(hyper_timepart."time")) -> Index Scan Backward using _hyper_3_7_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp, time_func(_hyper_3_7_chunk."time") -> Index Scan Backward using _hyper_3_8_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp, time_func(_hyper_3_8_chunk."time") -- Grouping on original time column should be pushed-down SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT time, device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; --- QUERY PLAN --- Limit Output: hyper_timepart."time", hyper_timepart.device, (avg(hyper_timepart.temp)) -> Sort Output: hyper_timepart."time", hyper_timepart.device, (avg(hyper_timepart.temp)) Sort Key: hyper_timepart."time", hyper_timepart.device -> HashAggregate Output: hyper_timepart."time", hyper_timepart.device, avg(hyper_timepart.temp) Group Key: hyper_timepart."time", hyper_timepart.device -> Append -> Seq Scan on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp -- Applying the time partitioning function should also allow push-down -- on open dimensions :PREFIX SELECT time_func(time), device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; --- QUERY PLAN --- Limit Output: (time_func(hyper_timepart."time")), hyper_timepart.device, (avg(hyper_timepart.temp)) -> GroupAggregate Output: (time_func(hyper_timepart."time")), hyper_timepart.device, avg(hyper_timepart.temp) Group Key: (time_func(hyper_timepart."time")), hyper_timepart.device -> Incremental Sort Output: (time_func(hyper_timepart."time")), hyper_timepart.device, hyper_timepart.temp Sort Key: (time_func(hyper_timepart."time")), hyper_timepart.device Presorted Key: (time_func(hyper_timepart."time")) -> Result Output: (time_func(hyper_timepart."time")), hyper_timepart.device, hyper_timepart.temp -> Merge Append Sort Key: (time_func(hyper_timepart."time")) -> Index Scan Backward using _hyper_3_7_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp, time_func(_hyper_3_7_chunk."time") -> Index Scan Backward using _hyper_3_8_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp, time_func(_hyper_3_8_chunk."time") -- Partial aggregation pushdown is currently not supported for this query by -- the TSDB pushdown code since a projection is used in the path. :PREFIX SELECT time_func(time), _timescaledb_functions.get_partition_hash(device), avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; --- QUERY PLAN --- Limit Output: (time_func(hyper_timepart."time")), (_timescaledb_functions.get_partition_hash(hyper_timepart.device)), (avg(hyper_timepart.temp)) -> GroupAggregate Output: (time_func(hyper_timepart."time")), (_timescaledb_functions.get_partition_hash(hyper_timepart.device)), avg(hyper_timepart.temp) Group Key: (time_func(hyper_timepart."time")), (_timescaledb_functions.get_partition_hash(hyper_timepart.device)) -> Incremental Sort Output: (time_func(hyper_timepart."time")), (_timescaledb_functions.get_partition_hash(hyper_timepart.device)), hyper_timepart.temp Sort Key: (time_func(hyper_timepart."time")), (_timescaledb_functions.get_partition_hash(hyper_timepart.device)) Presorted Key: (time_func(hyper_timepart."time")) -> Result Output: (time_func(hyper_timepart."time")), _timescaledb_functions.get_partition_hash(hyper_timepart.device), hyper_timepart.temp -> Merge Append Sort Key: (time_func(hyper_timepart."time")) -> Index Scan Backward using _hyper_3_7_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp, time_func(_hyper_3_7_chunk."time") -> Index Scan Backward using _hyper_3_8_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp, time_func(_hyper_3_8_chunk."time") -- Test removal of redundant group key optimization in PG16 -- All lower versions include the redundant key on device column :PREFIX SELECT device, avg(temp) FROM hyper_timepart WHERE device = 1 GROUP BY 1 LIMIT 10; --- QUERY PLAN --- Limit Output: _hyper_3_8_chunk.device, (avg(_hyper_3_8_chunk.temp)) -> GroupAggregate Output: _hyper_3_8_chunk.device, avg(_hyper_3_8_chunk.temp) -> Index Scan using _hyper_3_8_chunk_hyper_timepart_device_expr_idx on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp Index Cond: (_hyper_3_8_chunk.device = 1) ================================================ FILE: test/expected/partitionwise-17.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set PREFIX 'EXPLAIN (VERBOSE, BUFFERS OFF, COSTS OFF)' -- Create a two dimensional hypertable CREATE TABLE hyper (time timestamptz, device int, temp float); SELECT * FROM create_hypertable('hyper', 'time', 'device', 2); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | hyper | t -- Create a similar PostgreSQL partitioned table CREATE TABLE pg2dim (time timestamptz, device int, temp float) PARTITION BY HASH (device); CREATE TABLE pg2dim_h1 PARTITION OF pg2dim FOR VALUES WITH (MODULUS 2, REMAINDER 0) PARTITION BY RANGE(time); CREATE TABLE pg2dim_h2 PARTITION OF pg2dim FOR VALUES WITH (MODULUS 2, REMAINDER 1) PARTITION BY RANGE(time); CREATE TABLE pg2dim_h1_t1 PARTITION OF pg2dim_h1 FOR VALUES FROM ('2018-01-01 00:00') TO ('2018-09-01 00:00'); CREATE TABLE pg2dim_h1_t2 PARTITION OF pg2dim_h1 FOR VALUES FROM ('2018-09-01 00:00') TO ('2018-12-01 00:00'); CREATE TABLE pg2dim_h2_t1 PARTITION OF pg2dim_h2 FOR VALUES FROM ('2018-01-01 00:00') TO ('2018-09-01 00:00'); CREATE TABLE pg2dim_h2_t2 PARTITION OF pg2dim_h2 FOR VALUES FROM ('2018-09-01 00:00') TO ('2018-12-01 00:00'); -- Create a 1-dimensional partitioned table for comparison CREATE TABLE pg1dim (time timestamptz, device int, temp float) PARTITION BY HASH (device); CREATE TABLE pg1dim_h1 PARTITION OF pg1dim FOR VALUES WITH (MODULUS 2, REMAINDER 0); CREATE TABLE pg1dim_h2 PARTITION OF pg1dim FOR VALUES WITH (MODULUS 2, REMAINDER 1); INSERT INTO hyper VALUES ('2018-02-19 13:01', 1, 2.3), ('2018-02-19 13:02', 3, 3.1), ('2018-10-19 13:01', 1, 7.6), ('2018-10-19 13:02', 3, 9.0); INSERT INTO pg2dim VALUES ('2018-02-19 13:01', 1, 2.3), ('2018-02-19 13:02', 3, 3.1), ('2018-10-19 13:01', 1, 7.6), ('2018-10-19 13:02', 3, 9.0); INSERT INTO pg1dim VALUES ('2018-02-19 13:01', 1, 2.3), ('2018-02-19 13:02', 3, 3.1), ('2018-10-19 13:01', 1, 7.6), ('2018-10-19 13:02', 3, 9.0); SELECT * FROM test.show_subtables('hyper'); Child | Tablespace ----------------------------------------+------------ _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal._hyper_1_4_chunk | SELECT * FROM pg2dim_h1_t1; time | device | temp ------------------------------+--------+------ Mon Feb 19 13:01:00 2018 PST | 1 | 2.3 SELECT * FROM pg2dim_h1_t2; time | device | temp ------------------------------+--------+------ Fri Oct 19 13:01:00 2018 PDT | 1 | 7.6 SELECT * FROM pg2dim_h2_t1; time | device | temp ------------------------------+--------+------ Mon Feb 19 13:02:00 2018 PST | 3 | 3.1 SELECT * FROM pg2dim_h2_t2; time | device | temp ------------------------------+--------+------ Fri Oct 19 13:02:00 2018 PDT | 3 | 9 -- Compare partitionwise aggreate enabled/disabled. First run queries -- on PG partitioned tables for reference. -- All partition keys covered by GROUP BY SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT device, avg(temp) FROM pg1dim GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: pg1dim.device, (avg(pg1dim.temp)) Sort Key: pg1dim.device -> HashAggregate Output: pg1dim.device, avg(pg1dim.temp) Group Key: pg1dim.device -> Append -> Seq Scan on public.pg1dim_h1 pg1dim_1 Output: pg1dim_1.device, pg1dim_1.temp -> Seq Scan on public.pg1dim_h2 pg1dim_2 Output: pg1dim_2.device, pg1dim_2.temp SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT device, avg(temp) FROM pg1dim GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: pg1dim.device, (avg(pg1dim.temp)) Sort Key: pg1dim.device -> Append -> HashAggregate Output: pg1dim.device, avg(pg1dim.temp) Group Key: pg1dim.device -> Seq Scan on public.pg1dim_h1 pg1dim Output: pg1dim.device, pg1dim.temp -> HashAggregate Output: pg1dim_1.device, avg(pg1dim_1.temp) Group Key: pg1dim_1.device -> Seq Scan on public.pg1dim_h2 pg1dim_1 Output: pg1dim_1.device, pg1dim_1.temp -- All partition keys not covered by GROUP BY (partial partitionwise) SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT device, avg(temp) FROM pg2dim GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: pg2dim.device, (avg(pg2dim.temp)) Sort Key: pg2dim.device -> HashAggregate Output: pg2dim.device, avg(pg2dim.temp) Group Key: pg2dim.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2dim_1 Output: pg2dim_1.device, pg2dim_1.temp -> Seq Scan on public.pg2dim_h1_t2 pg2dim_2 Output: pg2dim_2.device, pg2dim_2.temp -> Seq Scan on public.pg2dim_h2_t1 pg2dim_3 Output: pg2dim_3.device, pg2dim_3.temp -> Seq Scan on public.pg2dim_h2_t2 pg2dim_4 Output: pg2dim_4.device, pg2dim_4.temp SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT device, avg(temp) FROM pg2dim GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: pg2dim.device, (avg(pg2dim.temp)) Sort Key: pg2dim.device -> Finalize HashAggregate Output: pg2dim.device, avg(pg2dim.temp) Group Key: pg2dim.device -> Append -> Partial HashAggregate Output: pg2dim.device, PARTIAL avg(pg2dim.temp) Group Key: pg2dim.device -> Seq Scan on public.pg2dim_h1_t1 pg2dim Output: pg2dim.device, pg2dim.temp -> Partial HashAggregate Output: pg2dim_1.device, PARTIAL avg(pg2dim_1.temp) Group Key: pg2dim_1.device -> Seq Scan on public.pg2dim_h1_t2 pg2dim_1 Output: pg2dim_1.device, pg2dim_1.temp -> Partial HashAggregate Output: pg2dim_2.device, PARTIAL avg(pg2dim_2.temp) Group Key: pg2dim_2.device -> Seq Scan on public.pg2dim_h2_t1 pg2dim_2 Output: pg2dim_2.device, pg2dim_2.temp -> Partial HashAggregate Output: pg2dim_3.device, PARTIAL avg(pg2dim_3.temp) Group Key: pg2dim_3.device -> Seq Scan on public.pg2dim_h2_t2 pg2dim_3 Output: pg2dim_3.device, pg2dim_3.temp -- All partition keys covered by GROUP BY (full partitionwise) SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT time, device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: pg2dim."time", pg2dim.device, (avg(pg2dim.temp)) Sort Key: pg2dim."time", pg2dim.device -> HashAggregate Output: pg2dim."time", pg2dim.device, avg(pg2dim.temp) Group Key: pg2dim."time", pg2dim.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2dim_1 Output: pg2dim_1."time", pg2dim_1.device, pg2dim_1.temp -> Seq Scan on public.pg2dim_h1_t2 pg2dim_2 Output: pg2dim_2."time", pg2dim_2.device, pg2dim_2.temp -> Seq Scan on public.pg2dim_h2_t1 pg2dim_3 Output: pg2dim_3."time", pg2dim_3.device, pg2dim_3.temp -> Seq Scan on public.pg2dim_h2_t2 pg2dim_4 Output: pg2dim_4."time", pg2dim_4.device, pg2dim_4.temp SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT time, device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: pg2dim."time", pg2dim.device, (avg(pg2dim.temp)) Sort Key: pg2dim."time", pg2dim.device -> Append -> HashAggregate Output: pg2dim."time", pg2dim.device, avg(pg2dim.temp) Group Key: pg2dim."time", pg2dim.device -> Seq Scan on public.pg2dim_h1_t1 pg2dim Output: pg2dim."time", pg2dim.device, pg2dim.temp -> HashAggregate Output: pg2dim_1."time", pg2dim_1.device, avg(pg2dim_1.temp) Group Key: pg2dim_1."time", pg2dim_1.device -> Seq Scan on public.pg2dim_h1_t2 pg2dim_1 Output: pg2dim_1."time", pg2dim_1.device, pg2dim_1.temp -> HashAggregate Output: pg2dim_2."time", pg2dim_2.device, avg(pg2dim_2.temp) Group Key: pg2dim_2."time", pg2dim_2.device -> Seq Scan on public.pg2dim_h2_t1 pg2dim_2 Output: pg2dim_2."time", pg2dim_2.device, pg2dim_2.temp -> HashAggregate Output: pg2dim_3."time", pg2dim_3.device, avg(pg2dim_3.temp) Group Key: pg2dim_3."time", pg2dim_3.device -> Seq Scan on public.pg2dim_h2_t2 pg2dim_3 Output: pg2dim_3."time", pg2dim_3.device, pg2dim_3.temp -- All partition keys not covered by GROUP BY because of date_trunc -- expression on time (partial partitionwise) SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, (avg(pg2dim.temp)) Sort Key: (date_trunc('month'::text, pg2dim."time")), pg2dim.device -> HashAggregate Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, avg(pg2dim.temp) Group Key: (date_trunc('month'::text, pg2dim."time")), pg2dim.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2dim_1 Output: date_trunc('month'::text, pg2dim_1."time"), pg2dim_1.device, pg2dim_1.temp -> Seq Scan on public.pg2dim_h1_t2 pg2dim_2 Output: date_trunc('month'::text, pg2dim_2."time"), pg2dim_2.device, pg2dim_2.temp -> Seq Scan on public.pg2dim_h2_t1 pg2dim_3 Output: date_trunc('month'::text, pg2dim_3."time"), pg2dim_3.device, pg2dim_3.temp -> Seq Scan on public.pg2dim_h2_t2 pg2dim_4 Output: date_trunc('month'::text, pg2dim_4."time"), pg2dim_4.device, pg2dim_4.temp SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, (avg(pg2dim.temp)) Sort Key: (date_trunc('month'::text, pg2dim."time")), pg2dim.device -> Finalize HashAggregate Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, avg(pg2dim.temp) Group Key: (date_trunc('month'::text, pg2dim."time")), pg2dim.device -> Append -> Partial HashAggregate Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, PARTIAL avg(pg2dim.temp) Group Key: date_trunc('month'::text, pg2dim."time"), pg2dim.device -> Seq Scan on public.pg2dim_h1_t1 pg2dim Output: date_trunc('month'::text, pg2dim."time"), pg2dim.device, pg2dim.temp -> Partial HashAggregate Output: (date_trunc('month'::text, pg2dim_1."time")), pg2dim_1.device, PARTIAL avg(pg2dim_1.temp) Group Key: date_trunc('month'::text, pg2dim_1."time"), pg2dim_1.device -> Seq Scan on public.pg2dim_h1_t2 pg2dim_1 Output: date_trunc('month'::text, pg2dim_1."time"), pg2dim_1.device, pg2dim_1.temp -> Partial HashAggregate Output: (date_trunc('month'::text, pg2dim_2."time")), pg2dim_2.device, PARTIAL avg(pg2dim_2.temp) Group Key: date_trunc('month'::text, pg2dim_2."time"), pg2dim_2.device -> Seq Scan on public.pg2dim_h2_t1 pg2dim_2 Output: date_trunc('month'::text, pg2dim_2."time"), pg2dim_2.device, pg2dim_2.temp -> Partial HashAggregate Output: (date_trunc('month'::text, pg2dim_3."time")), pg2dim_3.device, PARTIAL avg(pg2dim_3.temp) Group Key: date_trunc('month'::text, pg2dim_3."time"), pg2dim_3.device -> Seq Scan on public.pg2dim_h2_t2 pg2dim_3 Output: date_trunc('month'::text, pg2dim_3."time"), pg2dim_3.device, pg2dim_3.temp -- Now run on hypertable -- All partition keys not covered by GROUP BY (partial partitionwise) SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT device, avg(temp) FROM hyper GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: hyper.device, (avg(hyper.temp)) Sort Key: hyper.device -> HashAggregate Output: hyper.device, avg(hyper.temp) Group Key: hyper.device -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT device, avg(temp) FROM hyper GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: hyper.device, (avg(hyper.temp)) Sort Key: hyper.device -> HashAggregate Output: hyper.device, avg(hyper.temp) Group Key: hyper.device -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp -- All partition keys covered (full partitionwise) SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT time, device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: hyper."time", hyper.device, (avg(hyper.temp)) Sort Key: hyper."time", hyper.device -> HashAggregate Output: hyper."time", hyper.device, avg(hyper.temp) Group Key: hyper."time", hyper.device -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT time, device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: hyper."time", hyper.device, (avg(hyper.temp)) Sort Key: hyper."time", hyper.device -> HashAggregate Output: hyper."time", hyper.device, avg(hyper.temp) Group Key: hyper."time", hyper.device -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp -- Partial aggregation since date_trunc(time) is not a partition key SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (date_trunc('month'::text, hyper."time")), hyper.device, (avg(hyper.temp)) Sort Key: (date_trunc('month'::text, hyper."time")), hyper.device -> HashAggregate Output: (date_trunc('month'::text, hyper."time")), hyper.device, avg(hyper.temp) Group Key: date_trunc('month'::text, hyper."time"), hyper.device -> Result Output: date_trunc('month'::text, hyper."time"), hyper.device, hyper.temp -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp -- Partial aggregation pushdown is currently not supported for this query by -- the TSDB pushdown code since a projection is used in the path. SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (date_trunc('month'::text, hyper."time")), hyper.device, (avg(hyper.temp)) Sort Key: (date_trunc('month'::text, hyper."time")), hyper.device -> HashAggregate Output: (date_trunc('month'::text, hyper."time")), hyper.device, avg(hyper.temp) Group Key: date_trunc('month'::text, hyper."time"), hyper.device -> Result Output: date_trunc('month'::text, hyper."time"), hyper.device, hyper.temp -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp -- Also test time_bucket SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT time_bucket('1 month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device, (avg(hyper.temp)) Sort Key: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device -> HashAggregate Output: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device, avg(hyper.temp) Group Key: time_bucket('@ 1 mon'::interval, hyper."time"), hyper.device -> Result Output: time_bucket('@ 1 mon'::interval, hyper."time"), hyper.device, hyper.temp -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT time_bucket('1 month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device, (avg(hyper.temp)) Sort Key: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device -> HashAggregate Output: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device, avg(hyper.temp) Group Key: time_bucket('@ 1 mon'::interval, hyper."time"), hyper.device -> Result Output: time_bucket('@ 1 mon'::interval, hyper."time"), hyper.device, hyper.temp -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp -- Test partitionwise joins, mostly to see that we do not break -- anything CREATE TABLE hyper_meta (time timestamptz, device int, info text); SELECT * FROM create_hypertable('hyper_meta', 'time', 'device', 2); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 2 | public | hyper_meta | t INSERT INTO hyper_meta VALUES ('2018-02-19 13:01', 1, 'device_1'), ('2018-02-19 13:02', 3, 'device_3'); SET enable_partitionwise_join = 'off'; :PREFIX SELECT h.time, h.device, h.temp, hm.info FROM hyper h, hyper_meta hm WHERE h.device = hm.device; --- QUERY PLAN --- Merge Join Output: h."time", h.device, h.temp, hm.info Merge Cond: (hm.device = h.device) -> Merge Append Sort Key: hm.device -> Index Scan using _hyper_2_5_chunk_hyper_meta_device_time_idx on _timescaledb_internal._hyper_2_5_chunk hm_1 Output: hm_1.info, hm_1.device -> Index Scan using _hyper_2_6_chunk_hyper_meta_device_time_idx on _timescaledb_internal._hyper_2_6_chunk hm_2 Output: hm_2.info, hm_2.device -> Materialize Output: h."time", h.device, h.temp -> Merge Append Sort Key: h.device -> Index Scan using _hyper_1_1_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_1_chunk h_1 Output: h_1."time", h_1.device, h_1.temp -> Index Scan using _hyper_1_2_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_2_chunk h_2 Output: h_2."time", h_2.device, h_2.temp -> Index Scan using _hyper_1_3_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_3_chunk h_3 Output: h_3."time", h_3.device, h_3.temp -> Index Scan using _hyper_1_4_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_4_chunk h_4 Output: h_4."time", h_4.device, h_4.temp :PREFIX SELECT pg2.time, pg2.device, pg2.temp, pg1.temp FROM pg2dim pg2, pg1dim pg1 WHERE pg2.device = pg1.device; --- QUERY PLAN --- Merge Join Output: pg2."time", pg2.device, pg2.temp, pg1.temp Merge Cond: (pg1.device = pg2.device) -> Sort Output: pg1.temp, pg1.device Sort Key: pg1.device -> Append -> Seq Scan on public.pg1dim_h1 pg1_1 Output: pg1_1.temp, pg1_1.device -> Seq Scan on public.pg1dim_h2 pg1_2 Output: pg1_2.temp, pg1_2.device -> Sort Output: pg2."time", pg2.device, pg2.temp Sort Key: pg2.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2_1 Output: pg2_1."time", pg2_1.device, pg2_1.temp -> Seq Scan on public.pg2dim_h1_t2 pg2_2 Output: pg2_2."time", pg2_2.device, pg2_2.temp -> Seq Scan on public.pg2dim_h2_t1 pg2_3 Output: pg2_3."time", pg2_3.device, pg2_3.temp -> Seq Scan on public.pg2dim_h2_t2 pg2_4 Output: pg2_4."time", pg2_4.device, pg2_4.temp SET enable_partitionwise_join = 'on'; :PREFIX SELECT h.time, h.device, h.temp, hm.info FROM hyper h, hyper_meta hm WHERE h.device = hm.device; --- QUERY PLAN --- Merge Join Output: h."time", h.device, h.temp, hm.info Merge Cond: (hm.device = h.device) -> Merge Append Sort Key: hm.device -> Index Scan using _hyper_2_5_chunk_hyper_meta_device_time_idx on _timescaledb_internal._hyper_2_5_chunk hm_1 Output: hm_1.info, hm_1.device -> Index Scan using _hyper_2_6_chunk_hyper_meta_device_time_idx on _timescaledb_internal._hyper_2_6_chunk hm_2 Output: hm_2.info, hm_2.device -> Materialize Output: h."time", h.device, h.temp -> Merge Append Sort Key: h.device -> Index Scan using _hyper_1_1_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_1_chunk h_1 Output: h_1."time", h_1.device, h_1.temp -> Index Scan using _hyper_1_2_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_2_chunk h_2 Output: h_2."time", h_2.device, h_2.temp -> Index Scan using _hyper_1_3_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_3_chunk h_3 Output: h_3."time", h_3.device, h_3.temp -> Index Scan using _hyper_1_4_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_4_chunk h_4 Output: h_4."time", h_4.device, h_4.temp :PREFIX SELECT pg2.time, pg2.device, pg2.temp, pg1.temp FROM pg2dim pg2, pg1dim pg1 WHERE pg2.device = pg1.device; --- QUERY PLAN --- Append -> Merge Join Output: pg2_2."time", pg2_2.device, pg2_2.temp, pg1_1.temp Merge Cond: (pg1_1.device = pg2_2.device) -> Sort Output: pg1_1.temp, pg1_1.device Sort Key: pg1_1.device -> Seq Scan on public.pg1dim_h1 pg1_1 Output: pg1_1.temp, pg1_1.device -> Sort Output: pg2_2."time", pg2_2.device, pg2_2.temp Sort Key: pg2_2.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2_2 Output: pg2_2."time", pg2_2.device, pg2_2.temp -> Seq Scan on public.pg2dim_h1_t2 pg2_3 Output: pg2_3."time", pg2_3.device, pg2_3.temp -> Merge Join Output: pg2_5."time", pg2_5.device, pg2_5.temp, pg1_2.temp Merge Cond: (pg1_2.device = pg2_5.device) -> Sort Output: pg1_2.temp, pg1_2.device Sort Key: pg1_2.device -> Seq Scan on public.pg1dim_h2 pg1_2 Output: pg1_2.temp, pg1_2.device -> Sort Output: pg2_5."time", pg2_5.device, pg2_5.temp Sort Key: pg2_5.device -> Append -> Seq Scan on public.pg2dim_h2_t1 pg2_5 Output: pg2_5."time", pg2_5.device, pg2_5.temp -> Seq Scan on public.pg2dim_h2_t2 pg2_6 Output: pg2_6."time", pg2_6.device, pg2_6.temp -- Test hypertable with time partitioning function CREATE OR REPLACE FUNCTION time_func(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ DECLARE retval TIMESTAMPTZ; BEGIN retval := to_timestamp(unixtime); RETURN retval; END $BODY$; CREATE TABLE hyper_timepart (time float8, device int, temp float); SELECT * FROM create_hypertable('hyper_timepart', 'time', 'device', 2, time_partitioning_func => 'time_func'); hypertable_id | schema_name | table_name | created ---------------+-------------+----------------+--------- 3 | public | hyper_timepart | t -- Planner won't pick push-down aggs on table with time function -- unless a certain amount of data SELECT setseed(1); setseed --------- INSERT INTO hyper_timepart SELECT x, ceil(random() * 8), random() * 20 FROM generate_series(0,5000-1) AS x; -- All partition keys covered (full partitionwise) SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT time, device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; --- QUERY PLAN --- Limit Output: hyper_timepart."time", hyper_timepart.device, (avg(hyper_timepart.temp)) -> Sort Output: hyper_timepart."time", hyper_timepart.device, (avg(hyper_timepart.temp)) Sort Key: hyper_timepart."time", hyper_timepart.device -> HashAggregate Output: hyper_timepart."time", hyper_timepart.device, avg(hyper_timepart.temp) Group Key: hyper_timepart."time", hyper_timepart.device -> Append -> Seq Scan on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp :PREFIX SELECT time_func(time), device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; --- QUERY PLAN --- Limit Output: (time_func(hyper_timepart."time")), hyper_timepart.device, (avg(hyper_timepart.temp)) -> GroupAggregate Output: (time_func(hyper_timepart."time")), hyper_timepart.device, avg(hyper_timepart.temp) Group Key: (time_func(hyper_timepart."time")), hyper_timepart.device -> Incremental Sort Output: (time_func(hyper_timepart."time")), hyper_timepart.device, hyper_timepart.temp Sort Key: (time_func(hyper_timepart."time")), hyper_timepart.device Presorted Key: (time_func(hyper_timepart."time")) -> Result Output: (time_func(hyper_timepart."time")), hyper_timepart.device, hyper_timepart.temp -> Merge Append Sort Key: (time_func(hyper_timepart."time")) -> Index Scan Backward using _hyper_3_7_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp, time_func(_hyper_3_7_chunk."time") -> Index Scan Backward using _hyper_3_8_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp, time_func(_hyper_3_8_chunk."time") -- Grouping on original time column should be pushed-down SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT time, device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; --- QUERY PLAN --- Limit Output: hyper_timepart."time", hyper_timepart.device, (avg(hyper_timepart.temp)) -> Sort Output: hyper_timepart."time", hyper_timepart.device, (avg(hyper_timepart.temp)) Sort Key: hyper_timepart."time", hyper_timepart.device -> HashAggregate Output: hyper_timepart."time", hyper_timepart.device, avg(hyper_timepart.temp) Group Key: hyper_timepart."time", hyper_timepart.device -> Append -> Seq Scan on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp -- Applying the time partitioning function should also allow push-down -- on open dimensions :PREFIX SELECT time_func(time), device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; --- QUERY PLAN --- Limit Output: (time_func(hyper_timepart."time")), hyper_timepart.device, (avg(hyper_timepart.temp)) -> GroupAggregate Output: (time_func(hyper_timepart."time")), hyper_timepart.device, avg(hyper_timepart.temp) Group Key: (time_func(hyper_timepart."time")), hyper_timepart.device -> Incremental Sort Output: (time_func(hyper_timepart."time")), hyper_timepart.device, hyper_timepart.temp Sort Key: (time_func(hyper_timepart."time")), hyper_timepart.device Presorted Key: (time_func(hyper_timepart."time")) -> Result Output: (time_func(hyper_timepart."time")), hyper_timepart.device, hyper_timepart.temp -> Merge Append Sort Key: (time_func(hyper_timepart."time")) -> Index Scan Backward using _hyper_3_7_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp, time_func(_hyper_3_7_chunk."time") -> Index Scan Backward using _hyper_3_8_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp, time_func(_hyper_3_8_chunk."time") -- Partial aggregation pushdown is currently not supported for this query by -- the TSDB pushdown code since a projection is used in the path. :PREFIX SELECT time_func(time), _timescaledb_functions.get_partition_hash(device), avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; --- QUERY PLAN --- Limit Output: (time_func(hyper_timepart."time")), (_timescaledb_functions.get_partition_hash(hyper_timepart.device)), (avg(hyper_timepart.temp)) -> GroupAggregate Output: (time_func(hyper_timepart."time")), (_timescaledb_functions.get_partition_hash(hyper_timepart.device)), avg(hyper_timepart.temp) Group Key: (time_func(hyper_timepart."time")), (_timescaledb_functions.get_partition_hash(hyper_timepart.device)) -> Incremental Sort Output: (time_func(hyper_timepart."time")), (_timescaledb_functions.get_partition_hash(hyper_timepart.device)), hyper_timepart.temp Sort Key: (time_func(hyper_timepart."time")), (_timescaledb_functions.get_partition_hash(hyper_timepart.device)) Presorted Key: (time_func(hyper_timepart."time")) -> Result Output: (time_func(hyper_timepart."time")), _timescaledb_functions.get_partition_hash(hyper_timepart.device), hyper_timepart.temp -> Merge Append Sort Key: (time_func(hyper_timepart."time")) -> Index Scan Backward using _hyper_3_7_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp, time_func(_hyper_3_7_chunk."time") -> Index Scan Backward using _hyper_3_8_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp, time_func(_hyper_3_8_chunk."time") -- Test removal of redundant group key optimization in PG16 -- All lower versions include the redundant key on device column :PREFIX SELECT device, avg(temp) FROM hyper_timepart WHERE device = 1 GROUP BY 1 LIMIT 10; --- QUERY PLAN --- Limit Output: _hyper_3_8_chunk.device, (avg(_hyper_3_8_chunk.temp)) -> GroupAggregate Output: _hyper_3_8_chunk.device, avg(_hyper_3_8_chunk.temp) -> Index Scan using _hyper_3_8_chunk_hyper_timepart_device_expr_idx on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp Index Cond: (_hyper_3_8_chunk.device = 1) ================================================ FILE: test/expected/partitionwise-18.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set PREFIX 'EXPLAIN (VERBOSE, BUFFERS OFF, COSTS OFF)' -- Create a two dimensional hypertable CREATE TABLE hyper (time timestamptz, device int, temp float); SELECT * FROM create_hypertable('hyper', 'time', 'device', 2); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | hyper | t -- Create a similar PostgreSQL partitioned table CREATE TABLE pg2dim (time timestamptz, device int, temp float) PARTITION BY HASH (device); CREATE TABLE pg2dim_h1 PARTITION OF pg2dim FOR VALUES WITH (MODULUS 2, REMAINDER 0) PARTITION BY RANGE(time); CREATE TABLE pg2dim_h2 PARTITION OF pg2dim FOR VALUES WITH (MODULUS 2, REMAINDER 1) PARTITION BY RANGE(time); CREATE TABLE pg2dim_h1_t1 PARTITION OF pg2dim_h1 FOR VALUES FROM ('2018-01-01 00:00') TO ('2018-09-01 00:00'); CREATE TABLE pg2dim_h1_t2 PARTITION OF pg2dim_h1 FOR VALUES FROM ('2018-09-01 00:00') TO ('2018-12-01 00:00'); CREATE TABLE pg2dim_h2_t1 PARTITION OF pg2dim_h2 FOR VALUES FROM ('2018-01-01 00:00') TO ('2018-09-01 00:00'); CREATE TABLE pg2dim_h2_t2 PARTITION OF pg2dim_h2 FOR VALUES FROM ('2018-09-01 00:00') TO ('2018-12-01 00:00'); -- Create a 1-dimensional partitioned table for comparison CREATE TABLE pg1dim (time timestamptz, device int, temp float) PARTITION BY HASH (device); CREATE TABLE pg1dim_h1 PARTITION OF pg1dim FOR VALUES WITH (MODULUS 2, REMAINDER 0); CREATE TABLE pg1dim_h2 PARTITION OF pg1dim FOR VALUES WITH (MODULUS 2, REMAINDER 1); INSERT INTO hyper VALUES ('2018-02-19 13:01', 1, 2.3), ('2018-02-19 13:02', 3, 3.1), ('2018-10-19 13:01', 1, 7.6), ('2018-10-19 13:02', 3, 9.0); INSERT INTO pg2dim VALUES ('2018-02-19 13:01', 1, 2.3), ('2018-02-19 13:02', 3, 3.1), ('2018-10-19 13:01', 1, 7.6), ('2018-10-19 13:02', 3, 9.0); INSERT INTO pg1dim VALUES ('2018-02-19 13:01', 1, 2.3), ('2018-02-19 13:02', 3, 3.1), ('2018-10-19 13:01', 1, 7.6), ('2018-10-19 13:02', 3, 9.0); SELECT * FROM test.show_subtables('hyper'); Child | Tablespace ----------------------------------------+------------ _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal._hyper_1_4_chunk | SELECT * FROM pg2dim_h1_t1; time | device | temp ------------------------------+--------+------ Mon Feb 19 13:01:00 2018 PST | 1 | 2.3 SELECT * FROM pg2dim_h1_t2; time | device | temp ------------------------------+--------+------ Fri Oct 19 13:01:00 2018 PDT | 1 | 7.6 SELECT * FROM pg2dim_h2_t1; time | device | temp ------------------------------+--------+------ Mon Feb 19 13:02:00 2018 PST | 3 | 3.1 SELECT * FROM pg2dim_h2_t2; time | device | temp ------------------------------+--------+------ Fri Oct 19 13:02:00 2018 PDT | 3 | 9 -- Compare partitionwise aggreate enabled/disabled. First run queries -- on PG partitioned tables for reference. -- All partition keys covered by GROUP BY SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT device, avg(temp) FROM pg1dim GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: pg1dim.device, (avg(pg1dim.temp)) Sort Key: pg1dim.device -> HashAggregate Output: pg1dim.device, avg(pg1dim.temp) Group Key: pg1dim.device -> Append -> Seq Scan on public.pg1dim_h1 pg1dim_1 Output: pg1dim_1.device, pg1dim_1.temp -> Seq Scan on public.pg1dim_h2 pg1dim_2 Output: pg1dim_2.device, pg1dim_2.temp SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT device, avg(temp) FROM pg1dim GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: pg1dim.device, (avg(pg1dim.temp)) Sort Key: pg1dim.device -> Append -> HashAggregate Output: pg1dim.device, avg(pg1dim.temp) Group Key: pg1dim.device -> Seq Scan on public.pg1dim_h1 pg1dim Output: pg1dim.device, pg1dim.temp -> HashAggregate Output: pg1dim_1.device, avg(pg1dim_1.temp) Group Key: pg1dim_1.device -> Seq Scan on public.pg1dim_h2 pg1dim_1 Output: pg1dim_1.device, pg1dim_1.temp -- All partition keys not covered by GROUP BY (partial partitionwise) SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT device, avg(temp) FROM pg2dim GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: pg2dim.device, (avg(pg2dim.temp)) Sort Key: pg2dim.device -> HashAggregate Output: pg2dim.device, avg(pg2dim.temp) Group Key: pg2dim.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2dim_1 Output: pg2dim_1.device, pg2dim_1.temp -> Seq Scan on public.pg2dim_h1_t2 pg2dim_2 Output: pg2dim_2.device, pg2dim_2.temp -> Seq Scan on public.pg2dim_h2_t1 pg2dim_3 Output: pg2dim_3.device, pg2dim_3.temp -> Seq Scan on public.pg2dim_h2_t2 pg2dim_4 Output: pg2dim_4.device, pg2dim_4.temp SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT device, avg(temp) FROM pg2dim GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: pg2dim.device, (avg(pg2dim.temp)) Sort Key: pg2dim.device -> Finalize HashAggregate Output: pg2dim.device, avg(pg2dim.temp) Group Key: pg2dim.device -> Append -> Partial HashAggregate Output: pg2dim.device, PARTIAL avg(pg2dim.temp) Group Key: pg2dim.device -> Seq Scan on public.pg2dim_h1_t1 pg2dim Output: pg2dim.device, pg2dim.temp -> Partial HashAggregate Output: pg2dim_1.device, PARTIAL avg(pg2dim_1.temp) Group Key: pg2dim_1.device -> Seq Scan on public.pg2dim_h1_t2 pg2dim_1 Output: pg2dim_1.device, pg2dim_1.temp -> Partial HashAggregate Output: pg2dim_2.device, PARTIAL avg(pg2dim_2.temp) Group Key: pg2dim_2.device -> Seq Scan on public.pg2dim_h2_t1 pg2dim_2 Output: pg2dim_2.device, pg2dim_2.temp -> Partial HashAggregate Output: pg2dim_3.device, PARTIAL avg(pg2dim_3.temp) Group Key: pg2dim_3.device -> Seq Scan on public.pg2dim_h2_t2 pg2dim_3 Output: pg2dim_3.device, pg2dim_3.temp -- All partition keys covered by GROUP BY (full partitionwise) SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT time, device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: pg2dim."time", pg2dim.device, (avg(pg2dim.temp)) Sort Key: pg2dim."time", pg2dim.device -> HashAggregate Output: pg2dim."time", pg2dim.device, avg(pg2dim.temp) Group Key: pg2dim."time", pg2dim.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2dim_1 Output: pg2dim_1."time", pg2dim_1.device, pg2dim_1.temp -> Seq Scan on public.pg2dim_h1_t2 pg2dim_2 Output: pg2dim_2."time", pg2dim_2.device, pg2dim_2.temp -> Seq Scan on public.pg2dim_h2_t1 pg2dim_3 Output: pg2dim_3."time", pg2dim_3.device, pg2dim_3.temp -> Seq Scan on public.pg2dim_h2_t2 pg2dim_4 Output: pg2dim_4."time", pg2dim_4.device, pg2dim_4.temp SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT time, device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: pg2dim."time", pg2dim.device, (avg(pg2dim.temp)) Sort Key: pg2dim."time", pg2dim.device -> Append -> HashAggregate Output: pg2dim."time", pg2dim.device, avg(pg2dim.temp) Group Key: pg2dim."time", pg2dim.device -> Seq Scan on public.pg2dim_h1_t1 pg2dim Output: pg2dim."time", pg2dim.device, pg2dim.temp -> HashAggregate Output: pg2dim_1."time", pg2dim_1.device, avg(pg2dim_1.temp) Group Key: pg2dim_1."time", pg2dim_1.device -> Seq Scan on public.pg2dim_h1_t2 pg2dim_1 Output: pg2dim_1."time", pg2dim_1.device, pg2dim_1.temp -> HashAggregate Output: pg2dim_2."time", pg2dim_2.device, avg(pg2dim_2.temp) Group Key: pg2dim_2."time", pg2dim_2.device -> Seq Scan on public.pg2dim_h2_t1 pg2dim_2 Output: pg2dim_2."time", pg2dim_2.device, pg2dim_2.temp -> HashAggregate Output: pg2dim_3."time", pg2dim_3.device, avg(pg2dim_3.temp) Group Key: pg2dim_3."time", pg2dim_3.device -> Seq Scan on public.pg2dim_h2_t2 pg2dim_3 Output: pg2dim_3."time", pg2dim_3.device, pg2dim_3.temp -- All partition keys not covered by GROUP BY because of date_trunc -- expression on time (partial partitionwise) SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, (avg(pg2dim.temp)) Sort Key: (date_trunc('month'::text, pg2dim."time")), pg2dim.device -> HashAggregate Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, avg(pg2dim.temp) Group Key: (date_trunc('month'::text, pg2dim."time")), pg2dim.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2dim_1 Output: date_trunc('month'::text, pg2dim_1."time"), pg2dim_1.device, pg2dim_1.temp -> Seq Scan on public.pg2dim_h1_t2 pg2dim_2 Output: date_trunc('month'::text, pg2dim_2."time"), pg2dim_2.device, pg2dim_2.temp -> Seq Scan on public.pg2dim_h2_t1 pg2dim_3 Output: date_trunc('month'::text, pg2dim_3."time"), pg2dim_3.device, pg2dim_3.temp -> Seq Scan on public.pg2dim_h2_t2 pg2dim_4 Output: date_trunc('month'::text, pg2dim_4."time"), pg2dim_4.device, pg2dim_4.temp SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, (avg(pg2dim.temp)) Sort Key: (date_trunc('month'::text, pg2dim."time")), pg2dim.device -> Finalize HashAggregate Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, avg(pg2dim.temp) Group Key: (date_trunc('month'::text, pg2dim."time")), pg2dim.device -> Append -> Partial HashAggregate Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, PARTIAL avg(pg2dim.temp) Group Key: date_trunc('month'::text, pg2dim."time"), pg2dim.device -> Seq Scan on public.pg2dim_h1_t1 pg2dim Output: date_trunc('month'::text, pg2dim."time"), pg2dim.device, pg2dim.temp -> Partial HashAggregate Output: (date_trunc('month'::text, pg2dim_1."time")), pg2dim_1.device, PARTIAL avg(pg2dim_1.temp) Group Key: date_trunc('month'::text, pg2dim_1."time"), pg2dim_1.device -> Seq Scan on public.pg2dim_h1_t2 pg2dim_1 Output: date_trunc('month'::text, pg2dim_1."time"), pg2dim_1.device, pg2dim_1.temp -> Partial HashAggregate Output: (date_trunc('month'::text, pg2dim_2."time")), pg2dim_2.device, PARTIAL avg(pg2dim_2.temp) Group Key: date_trunc('month'::text, pg2dim_2."time"), pg2dim_2.device -> Seq Scan on public.pg2dim_h2_t1 pg2dim_2 Output: date_trunc('month'::text, pg2dim_2."time"), pg2dim_2.device, pg2dim_2.temp -> Partial HashAggregate Output: (date_trunc('month'::text, pg2dim_3."time")), pg2dim_3.device, PARTIAL avg(pg2dim_3.temp) Group Key: date_trunc('month'::text, pg2dim_3."time"), pg2dim_3.device -> Seq Scan on public.pg2dim_h2_t2 pg2dim_3 Output: date_trunc('month'::text, pg2dim_3."time"), pg2dim_3.device, pg2dim_3.temp -- Now run on hypertable -- All partition keys not covered by GROUP BY (partial partitionwise) SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT device, avg(temp) FROM hyper GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: hyper.device, (avg(hyper.temp)) Sort Key: hyper.device -> HashAggregate Output: hyper.device, avg(hyper.temp) Group Key: hyper.device -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT device, avg(temp) FROM hyper GROUP BY 1 ORDER BY 1; --- QUERY PLAN --- Sort Output: hyper.device, (avg(hyper.temp)) Sort Key: hyper.device -> HashAggregate Output: hyper.device, avg(hyper.temp) Group Key: hyper.device -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp -- All partition keys covered (full partitionwise) SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT time, device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: hyper."time", hyper.device, (avg(hyper.temp)) Sort Key: hyper."time", hyper.device -> HashAggregate Output: hyper."time", hyper.device, avg(hyper.temp) Group Key: hyper."time", hyper.device -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT time, device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: hyper."time", hyper.device, (avg(hyper.temp)) Sort Key: hyper."time", hyper.device -> HashAggregate Output: hyper."time", hyper.device, avg(hyper.temp) Group Key: hyper."time", hyper.device -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp -- Partial aggregation since date_trunc(time) is not a partition key SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (date_trunc('month'::text, hyper."time")), hyper.device, (avg(hyper.temp)) Sort Key: (date_trunc('month'::text, hyper."time")), hyper.device -> HashAggregate Output: (date_trunc('month'::text, hyper."time")), hyper.device, avg(hyper.temp) Group Key: date_trunc('month'::text, hyper."time"), hyper.device -> Result Output: date_trunc('month'::text, hyper."time"), hyper.device, hyper.temp -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp -- Partial aggregation pushdown is currently not supported for this query by -- the TSDB pushdown code since a projection is used in the path. SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (date_trunc('month'::text, hyper."time")), hyper.device, (avg(hyper.temp)) Sort Key: (date_trunc('month'::text, hyper."time")), hyper.device -> HashAggregate Output: (date_trunc('month'::text, hyper."time")), hyper.device, avg(hyper.temp) Group Key: date_trunc('month'::text, hyper."time"), hyper.device -> Result Output: date_trunc('month'::text, hyper."time"), hyper.device, hyper.temp -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp -- Also test time_bucket SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT time_bucket('1 month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device, (avg(hyper.temp)) Sort Key: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device -> HashAggregate Output: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device, avg(hyper.temp) Group Key: time_bucket('@ 1 mon'::interval, hyper."time"), hyper.device -> Result Output: time_bucket('@ 1 mon'::interval, hyper."time"), hyper.device, hyper.temp -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT time_bucket('1 month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; --- QUERY PLAN --- Sort Output: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device, (avg(hyper.temp)) Sort Key: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device -> HashAggregate Output: (time_bucket('@ 1 mon'::interval, hyper."time")), hyper.device, avg(hyper.temp) Group Key: time_bucket('@ 1 mon'::interval, hyper."time"), hyper.device -> Result Output: time_bucket('@ 1 mon'::interval, hyper."time"), hyper.device, hyper.temp -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp -- Test partitionwise joins, mostly to see that we do not break -- anything CREATE TABLE hyper_meta (time timestamptz, device int, info text); SELECT * FROM create_hypertable('hyper_meta', 'time', 'device', 2); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 2 | public | hyper_meta | t INSERT INTO hyper_meta VALUES ('2018-02-19 13:01', 1, 'device_1'), ('2018-02-19 13:02', 3, 'device_3'); SET enable_partitionwise_join = 'off'; :PREFIX SELECT h.time, h.device, h.temp, hm.info FROM hyper h, hyper_meta hm WHERE h.device = hm.device; --- QUERY PLAN --- Merge Join Output: h."time", h.device, h.temp, hm.info Merge Cond: (hm.device = h.device) -> Merge Append Sort Key: hm.device -> Index Scan using _hyper_2_5_chunk_hyper_meta_device_time_idx on _timescaledb_internal._hyper_2_5_chunk hm_1 Output: hm_1.info, hm_1.device -> Index Scan using _hyper_2_6_chunk_hyper_meta_device_time_idx on _timescaledb_internal._hyper_2_6_chunk hm_2 Output: hm_2.info, hm_2.device -> Materialize Output: h."time", h.device, h.temp -> Merge Append Sort Key: h.device -> Index Scan using _hyper_1_1_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_1_chunk h_1 Output: h_1."time", h_1.device, h_1.temp -> Index Scan using _hyper_1_2_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_2_chunk h_2 Output: h_2."time", h_2.device, h_2.temp -> Index Scan using _hyper_1_3_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_3_chunk h_3 Output: h_3."time", h_3.device, h_3.temp -> Index Scan using _hyper_1_4_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_4_chunk h_4 Output: h_4."time", h_4.device, h_4.temp :PREFIX SELECT pg2.time, pg2.device, pg2.temp, pg1.temp FROM pg2dim pg2, pg1dim pg1 WHERE pg2.device = pg1.device; --- QUERY PLAN --- Merge Join Output: pg2."time", pg2.device, pg2.temp, pg1.temp Merge Cond: (pg1.device = pg2.device) -> Sort Output: pg1.temp, pg1.device Sort Key: pg1.device -> Append -> Seq Scan on public.pg1dim_h1 pg1_1 Output: pg1_1.temp, pg1_1.device -> Seq Scan on public.pg1dim_h2 pg1_2 Output: pg1_2.temp, pg1_2.device -> Sort Output: pg2."time", pg2.device, pg2.temp Sort Key: pg2.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2_1 Output: pg2_1."time", pg2_1.device, pg2_1.temp -> Seq Scan on public.pg2dim_h1_t2 pg2_2 Output: pg2_2."time", pg2_2.device, pg2_2.temp -> Seq Scan on public.pg2dim_h2_t1 pg2_3 Output: pg2_3."time", pg2_3.device, pg2_3.temp -> Seq Scan on public.pg2dim_h2_t2 pg2_4 Output: pg2_4."time", pg2_4.device, pg2_4.temp SET enable_partitionwise_join = 'on'; :PREFIX SELECT h.time, h.device, h.temp, hm.info FROM hyper h, hyper_meta hm WHERE h.device = hm.device; --- QUERY PLAN --- Merge Join Output: h."time", h.device, h.temp, hm.info Merge Cond: (hm.device = h.device) -> Merge Append Sort Key: hm.device -> Index Scan using _hyper_2_5_chunk_hyper_meta_device_time_idx on _timescaledb_internal._hyper_2_5_chunk hm_1 Output: hm_1.info, hm_1.device -> Index Scan using _hyper_2_6_chunk_hyper_meta_device_time_idx on _timescaledb_internal._hyper_2_6_chunk hm_2 Output: hm_2.info, hm_2.device -> Materialize Output: h."time", h.device, h.temp -> Merge Append Sort Key: h.device -> Index Scan using _hyper_1_1_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_1_chunk h_1 Output: h_1."time", h_1.device, h_1.temp -> Index Scan using _hyper_1_2_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_2_chunk h_2 Output: h_2."time", h_2.device, h_2.temp -> Index Scan using _hyper_1_3_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_3_chunk h_3 Output: h_3."time", h_3.device, h_3.temp -> Index Scan using _hyper_1_4_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_4_chunk h_4 Output: h_4."time", h_4.device, h_4.temp :PREFIX SELECT pg2.time, pg2.device, pg2.temp, pg1.temp FROM pg2dim pg2, pg1dim pg1 WHERE pg2.device = pg1.device; --- QUERY PLAN --- Append -> Merge Join Output: pg2_2."time", pg2_2.device, pg2_2.temp, pg1_1.temp Merge Cond: (pg1_1.device = pg2_2.device) -> Sort Output: pg1_1.temp, pg1_1.device Sort Key: pg1_1.device -> Seq Scan on public.pg1dim_h1 pg1_1 Output: pg1_1.temp, pg1_1.device -> Sort Output: pg2_2."time", pg2_2.device, pg2_2.temp Sort Key: pg2_2.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2_2 Output: pg2_2."time", pg2_2.device, pg2_2.temp -> Seq Scan on public.pg2dim_h1_t2 pg2_3 Output: pg2_3."time", pg2_3.device, pg2_3.temp -> Merge Join Output: pg2_5."time", pg2_5.device, pg2_5.temp, pg1_2.temp Merge Cond: (pg1_2.device = pg2_5.device) -> Sort Output: pg1_2.temp, pg1_2.device Sort Key: pg1_2.device -> Seq Scan on public.pg1dim_h2 pg1_2 Output: pg1_2.temp, pg1_2.device -> Sort Output: pg2_5."time", pg2_5.device, pg2_5.temp Sort Key: pg2_5.device -> Append -> Seq Scan on public.pg2dim_h2_t1 pg2_5 Output: pg2_5."time", pg2_5.device, pg2_5.temp -> Seq Scan on public.pg2dim_h2_t2 pg2_6 Output: pg2_6."time", pg2_6.device, pg2_6.temp -- Test hypertable with time partitioning function CREATE OR REPLACE FUNCTION time_func(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ DECLARE retval TIMESTAMPTZ; BEGIN retval := to_timestamp(unixtime); RETURN retval; END $BODY$; CREATE TABLE hyper_timepart (time float8, device int, temp float); SELECT * FROM create_hypertable('hyper_timepart', 'time', 'device', 2, time_partitioning_func => 'time_func'); hypertable_id | schema_name | table_name | created ---------------+-------------+----------------+--------- 3 | public | hyper_timepart | t -- Planner won't pick push-down aggs on table with time function -- unless a certain amount of data SELECT setseed(1); setseed --------- INSERT INTO hyper_timepart SELECT x, ceil(random() * 8), random() * 20 FROM generate_series(0,5000-1) AS x; -- All partition keys covered (full partitionwise) SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT time, device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; --- QUERY PLAN --- Limit Output: hyper_timepart."time", hyper_timepart.device, (avg(hyper_timepart.temp)) -> Sort Output: hyper_timepart."time", hyper_timepart.device, (avg(hyper_timepart.temp)) Sort Key: hyper_timepart."time", hyper_timepart.device -> HashAggregate Output: hyper_timepart."time", hyper_timepart.device, avg(hyper_timepart.temp) Group Key: hyper_timepart."time", hyper_timepart.device -> Append -> Seq Scan on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp :PREFIX SELECT time_func(time), device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; --- QUERY PLAN --- Limit Output: (time_func(hyper_timepart."time")), hyper_timepart.device, (avg(hyper_timepart.temp)) -> GroupAggregate Output: (time_func(hyper_timepart."time")), hyper_timepart.device, avg(hyper_timepart.temp) Group Key: (time_func(hyper_timepart."time")), hyper_timepart.device -> Incremental Sort Output: (time_func(hyper_timepart."time")), hyper_timepart.device, hyper_timepart.temp Sort Key: (time_func(hyper_timepart."time")), hyper_timepart.device Presorted Key: (time_func(hyper_timepart."time")) -> Result Output: (time_func(hyper_timepart."time")), hyper_timepart.device, hyper_timepart.temp -> Merge Append Sort Key: (time_func(hyper_timepart."time")) -> Index Scan Backward using _hyper_3_7_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp, time_func(_hyper_3_7_chunk."time") -> Index Scan Backward using _hyper_3_8_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp, time_func(_hyper_3_8_chunk."time") -- Grouping on original time column should be pushed-down SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT time, device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; --- QUERY PLAN --- Limit Output: hyper_timepart."time", hyper_timepart.device, (avg(hyper_timepart.temp)) -> Sort Output: hyper_timepart."time", hyper_timepart.device, (avg(hyper_timepart.temp)) Sort Key: hyper_timepart."time", hyper_timepart.device -> HashAggregate Output: hyper_timepart."time", hyper_timepart.device, avg(hyper_timepart.temp) Group Key: hyper_timepart."time", hyper_timepart.device -> Append -> Seq Scan on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp -- Applying the time partitioning function should also allow push-down -- on open dimensions :PREFIX SELECT time_func(time), device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; --- QUERY PLAN --- Limit Output: (time_func(hyper_timepart."time")), hyper_timepart.device, (avg(hyper_timepart.temp)) -> GroupAggregate Output: (time_func(hyper_timepart."time")), hyper_timepart.device, avg(hyper_timepart.temp) Group Key: (time_func(hyper_timepart."time")), hyper_timepart.device -> Incremental Sort Output: (time_func(hyper_timepart."time")), hyper_timepart.device, hyper_timepart.temp Sort Key: (time_func(hyper_timepart."time")), hyper_timepart.device Presorted Key: (time_func(hyper_timepart."time")) -> Result Output: (time_func(hyper_timepart."time")), hyper_timepart.device, hyper_timepart.temp -> Merge Append Sort Key: (time_func(hyper_timepart."time")) -> Index Scan Backward using _hyper_3_7_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp, time_func(_hyper_3_7_chunk."time") -> Index Scan Backward using _hyper_3_8_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp, time_func(_hyper_3_8_chunk."time") -- Partial aggregation pushdown is currently not supported for this query by -- the TSDB pushdown code since a projection is used in the path. :PREFIX SELECT time_func(time), _timescaledb_functions.get_partition_hash(device), avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; --- QUERY PLAN --- Limit Output: (time_func(hyper_timepart."time")), (_timescaledb_functions.get_partition_hash(hyper_timepart.device)), (avg(hyper_timepart.temp)) -> GroupAggregate Output: (time_func(hyper_timepart."time")), (_timescaledb_functions.get_partition_hash(hyper_timepart.device)), avg(hyper_timepart.temp) Group Key: (time_func(hyper_timepart."time")), (_timescaledb_functions.get_partition_hash(hyper_timepart.device)) -> Incremental Sort Output: (time_func(hyper_timepart."time")), (_timescaledb_functions.get_partition_hash(hyper_timepart.device)), hyper_timepart.temp Sort Key: (time_func(hyper_timepart."time")), (_timescaledb_functions.get_partition_hash(hyper_timepart.device)) Presorted Key: (time_func(hyper_timepart."time")) -> Result Output: (time_func(hyper_timepart."time")), _timescaledb_functions.get_partition_hash(hyper_timepart.device), hyper_timepart.temp -> Merge Append Sort Key: (time_func(hyper_timepart."time")) -> Index Scan Backward using _hyper_3_7_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp, time_func(_hyper_3_7_chunk."time") -> Index Scan Backward using _hyper_3_8_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp, time_func(_hyper_3_8_chunk."time") -- Test removal of redundant group key optimization in PG16 -- All lower versions include the redundant key on device column :PREFIX SELECT device, avg(temp) FROM hyper_timepart WHERE device = 1 GROUP BY 1 LIMIT 10; --- QUERY PLAN --- Limit Output: _hyper_3_8_chunk.device, (avg(_hyper_3_8_chunk.temp)) -> GroupAggregate Output: _hyper_3_8_chunk.device, avg(_hyper_3_8_chunk.temp) -> Index Scan using _hyper_3_8_chunk_hyper_timepart_device_expr_idx on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp Index Cond: (_hyper_3_8_chunk.device = 1) ================================================ FILE: test/expected/partitionwise.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set PREFIX 'EXPLAIN (VERBOSE, COSTS OFF)' -- Create a two dimensional hypertable CREATE TABLE hyper (time timestamptz, device int, temp float); SELECT * FROM create_hypertable('hyper', 'time', 'device', 2); NOTICE: adding not-null constraint to column "time" hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | hyper | t (1 row) -- Create a similar PostgreSQL partitioned table CREATE TABLE pg2dim (time timestamptz, device int, temp float) PARTITION BY HASH (device); CREATE TABLE pg2dim_h1 PARTITION OF pg2dim FOR VALUES WITH (MODULUS 2, REMAINDER 0) PARTITION BY RANGE(time); CREATE TABLE pg2dim_h2 PARTITION OF pg2dim FOR VALUES WITH (MODULUS 2, REMAINDER 1) PARTITION BY RANGE(time); CREATE TABLE pg2dim_h1_t1 PARTITION OF pg2dim_h1 FOR VALUES FROM ('2018-01-01 00:00') TO ('2018-09-01 00:00'); CREATE TABLE pg2dim_h1_t2 PARTITION OF pg2dim_h1 FOR VALUES FROM ('2018-09-01 00:00') TO ('2018-12-01 00:00'); CREATE TABLE pg2dim_h2_t1 PARTITION OF pg2dim_h2 FOR VALUES FROM ('2018-01-01 00:00') TO ('2018-09-01 00:00'); CREATE TABLE pg2dim_h2_t2 PARTITION OF pg2dim_h2 FOR VALUES FROM ('2018-09-01 00:00') TO ('2018-12-01 00:00'); -- Create a 1-dimensional partitioned table for comparison CREATE TABLE pg1dim (time timestamptz, device int, temp float) PARTITION BY HASH (device); CREATE TABLE pg1dim_h1 PARTITION OF pg1dim FOR VALUES WITH (MODULUS 2, REMAINDER 0); CREATE TABLE pg1dim_h2 PARTITION OF pg1dim FOR VALUES WITH (MODULUS 2, REMAINDER 1); INSERT INTO hyper VALUES ('2018-02-19 13:01', 1, 2.3), ('2018-02-19 13:02', 3, 3.1), ('2018-10-19 13:01', 1, 7.6), ('2018-10-19 13:02', 3, 9.0); INSERT INTO pg2dim VALUES ('2018-02-19 13:01', 1, 2.3), ('2018-02-19 13:02', 3, 3.1), ('2018-10-19 13:01', 1, 7.6), ('2018-10-19 13:02', 3, 9.0); INSERT INTO pg1dim VALUES ('2018-02-19 13:01', 1, 2.3), ('2018-02-19 13:02', 3, 3.1), ('2018-10-19 13:01', 1, 7.6), ('2018-10-19 13:02', 3, 9.0); SELECT * FROM test.show_subtables('hyper'); Child | Tablespace ----------------------------------------+------------ _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal._hyper_1_4_chunk | (4 rows) SELECT * FROM pg2dim_h1_t1; time | device | temp ------------------------------+--------+------ Mon Feb 19 13:01:00 2018 PST | 1 | 2.3 (1 row) SELECT * FROM pg2dim_h1_t2; time | device | temp ------------------------------+--------+------ Fri Oct 19 13:01:00 2018 PDT | 1 | 7.6 (1 row) SELECT * FROM pg2dim_h2_t1; time | device | temp ------------------------------+--------+------ Mon Feb 19 13:02:00 2018 PST | 3 | 3.1 (1 row) SELECT * FROM pg2dim_h2_t2; time | device | temp ------------------------------+--------+------ Fri Oct 19 13:02:00 2018 PDT | 3 | 9 (1 row) -- Compare partitionwise aggreate enabled/disabled. First run queries -- on PG partitioned tables for reference. -- All partition keys covered by GROUP BY SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT device, avg(temp) FROM pg1dim GROUP BY 1 ORDER BY 1; QUERY PLAN ------------------------------------------------------------ Sort Output: pg1dim.device, (avg(pg1dim.temp)) Sort Key: pg1dim.device -> HashAggregate Output: pg1dim.device, avg(pg1dim.temp) Group Key: pg1dim.device -> Append -> Seq Scan on public.pg1dim_h1 pg1dim_1 Output: pg1dim_1.device, pg1dim_1.temp -> Seq Scan on public.pg1dim_h2 pg1dim_2 Output: pg1dim_2.device, pg1dim_2.temp (11 rows) SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT device, avg(temp) FROM pg1dim GROUP BY 1 ORDER BY 1; QUERY PLAN ------------------------------------------------------------ Sort Output: pg1dim.device, (avg(pg1dim.temp)) Sort Key: pg1dim.device -> Append -> HashAggregate Output: pg1dim.device, avg(pg1dim.temp) Group Key: pg1dim.device -> Seq Scan on public.pg1dim_h1 pg1dim Output: pg1dim.device, pg1dim.temp -> HashAggregate Output: pg1dim_1.device, avg(pg1dim_1.temp) Group Key: pg1dim_1.device -> Seq Scan on public.pg1dim_h2 pg1dim_1 Output: pg1dim_1.device, pg1dim_1.temp (14 rows) -- All partition keys not covered by GROUP BY (partial partitionwise) SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT device, avg(temp) FROM pg2dim GROUP BY 1 ORDER BY 1; QUERY PLAN ------------------------------------------------------------ Sort Output: pg2dim.device, (avg(pg2dim.temp)) Sort Key: pg2dim.device -> HashAggregate Output: pg2dim.device, avg(pg2dim.temp) Group Key: pg2dim.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2dim_1 Output: pg2dim_1.device, pg2dim_1.temp -> Seq Scan on public.pg2dim_h1_t2 pg2dim_2 Output: pg2dim_2.device, pg2dim_2.temp -> Seq Scan on public.pg2dim_h2_t1 pg2dim_3 Output: pg2dim_3.device, pg2dim_3.temp -> Seq Scan on public.pg2dim_h2_t2 pg2dim_4 Output: pg2dim_4.device, pg2dim_4.temp (15 rows) SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT device, avg(temp) FROM pg2dim GROUP BY 1 ORDER BY 1; QUERY PLAN ------------------------------------------------------------------------- Sort Output: pg2dim.device, (avg(pg2dim.temp)) Sort Key: pg2dim.device -> Finalize HashAggregate Output: pg2dim.device, avg(pg2dim.temp) Group Key: pg2dim.device -> Append -> Partial HashAggregate Output: pg2dim.device, PARTIAL avg(pg2dim.temp) Group Key: pg2dim.device -> Seq Scan on public.pg2dim_h1_t1 pg2dim Output: pg2dim.device, pg2dim.temp -> Partial HashAggregate Output: pg2dim_1.device, PARTIAL avg(pg2dim_1.temp) Group Key: pg2dim_1.device -> Seq Scan on public.pg2dim_h1_t2 pg2dim_1 Output: pg2dim_1.device, pg2dim_1.temp -> Partial HashAggregate Output: pg2dim_2.device, PARTIAL avg(pg2dim_2.temp) Group Key: pg2dim_2.device -> Seq Scan on public.pg2dim_h2_t1 pg2dim_2 Output: pg2dim_2.device, pg2dim_2.temp -> Partial HashAggregate Output: pg2dim_3.device, PARTIAL avg(pg2dim_3.temp) Group Key: pg2dim_3.device -> Seq Scan on public.pg2dim_h2_t2 pg2dim_3 Output: pg2dim_3.device, pg2dim_3.temp (27 rows) -- All partition keys covered by GROUP BY (full partitionwise) SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT time, device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; QUERY PLAN ----------------------------------------------------------------------------- Sort Output: pg2dim."time", pg2dim.device, (avg(pg2dim.temp)) Sort Key: pg2dim."time", pg2dim.device -> HashAggregate Output: pg2dim."time", pg2dim.device, avg(pg2dim.temp) Group Key: pg2dim."time", pg2dim.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2dim_1 Output: pg2dim_1."time", pg2dim_1.device, pg2dim_1.temp -> Seq Scan on public.pg2dim_h1_t2 pg2dim_2 Output: pg2dim_2."time", pg2dim_2.device, pg2dim_2.temp -> Seq Scan on public.pg2dim_h2_t1 pg2dim_3 Output: pg2dim_3."time", pg2dim_3.device, pg2dim_3.temp -> Seq Scan on public.pg2dim_h2_t2 pg2dim_4 Output: pg2dim_4."time", pg2dim_4.device, pg2dim_4.temp (15 rows) SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT time, device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; QUERY PLAN ----------------------------------------------------------------------------- Sort Output: pg2dim."time", pg2dim.device, (avg(pg2dim.temp)) Sort Key: pg2dim."time", pg2dim.device -> Append -> HashAggregate Output: pg2dim."time", pg2dim.device, avg(pg2dim.temp) Group Key: pg2dim."time", pg2dim.device -> Seq Scan on public.pg2dim_h1_t1 pg2dim Output: pg2dim."time", pg2dim.device, pg2dim.temp -> HashAggregate Output: pg2dim_1."time", pg2dim_1.device, avg(pg2dim_1.temp) Group Key: pg2dim_1."time", pg2dim_1.device -> Seq Scan on public.pg2dim_h1_t2 pg2dim_1 Output: pg2dim_1."time", pg2dim_1.device, pg2dim_1.temp -> HashAggregate Output: pg2dim_2."time", pg2dim_2.device, avg(pg2dim_2.temp) Group Key: pg2dim_2."time", pg2dim_2.device -> Seq Scan on public.pg2dim_h2_t1 pg2dim_2 Output: pg2dim_2."time", pg2dim_2.device, pg2dim_2.temp -> HashAggregate Output: pg2dim_3."time", pg2dim_3.device, avg(pg2dim_3.temp) Group Key: pg2dim_3."time", pg2dim_3.device -> Seq Scan on public.pg2dim_h2_t2 pg2dim_3 Output: pg2dim_3."time", pg2dim_3.device, pg2dim_3.temp (24 rows) -- All partition keys not covered by GROUP BY because of date_trunc -- expression on time (partial partitionwise) SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; QUERY PLAN -------------------------------------------------------------------------------------------------------- Sort Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, (avg(pg2dim.temp)) Sort Key: (date_trunc('month'::text, pg2dim."time")), pg2dim.device -> HashAggregate Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, avg(pg2dim.temp) Group Key: (date_trunc('month'::text, pg2dim."time")), pg2dim.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2dim_1 Output: date_trunc('month'::text, pg2dim_1."time"), pg2dim_1.device, pg2dim_1.temp -> Seq Scan on public.pg2dim_h1_t2 pg2dim_2 Output: date_trunc('month'::text, pg2dim_2."time"), pg2dim_2.device, pg2dim_2.temp -> Seq Scan on public.pg2dim_h2_t1 pg2dim_3 Output: date_trunc('month'::text, pg2dim_3."time"), pg2dim_3.device, pg2dim_3.temp -> Seq Scan on public.pg2dim_h2_t2 pg2dim_4 Output: date_trunc('month'::text, pg2dim_4."time"), pg2dim_4.device, pg2dim_4.temp (15 rows) SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; QUERY PLAN ----------------------------------------------------------------------------------------------------------------------- Sort Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, (avg(pg2dim.temp)) Sort Key: (date_trunc('month'::text, pg2dim."time")), pg2dim.device -> Finalize HashAggregate Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, avg(pg2dim.temp) Group Key: (date_trunc('month'::text, pg2dim."time")), pg2dim.device -> Append -> Partial HashAggregate Output: (date_trunc('month'::text, pg2dim."time")), pg2dim.device, PARTIAL avg(pg2dim.temp) Group Key: date_trunc('month'::text, pg2dim."time"), pg2dim.device -> Seq Scan on public.pg2dim_h1_t1 pg2dim Output: date_trunc('month'::text, pg2dim."time"), pg2dim.device, pg2dim.temp -> Partial HashAggregate Output: (date_trunc('month'::text, pg2dim_1."time")), pg2dim_1.device, PARTIAL avg(pg2dim_1.temp) Group Key: date_trunc('month'::text, pg2dim_1."time"), pg2dim_1.device -> Seq Scan on public.pg2dim_h1_t2 pg2dim_1 Output: date_trunc('month'::text, pg2dim_1."time"), pg2dim_1.device, pg2dim_1.temp -> Partial HashAggregate Output: (date_trunc('month'::text, pg2dim_2."time")), pg2dim_2.device, PARTIAL avg(pg2dim_2.temp) Group Key: date_trunc('month'::text, pg2dim_2."time"), pg2dim_2.device -> Seq Scan on public.pg2dim_h2_t1 pg2dim_2 Output: date_trunc('month'::text, pg2dim_2."time"), pg2dim_2.device, pg2dim_2.temp -> Partial HashAggregate Output: (date_trunc('month'::text, pg2dim_3."time")), pg2dim_3.device, PARTIAL avg(pg2dim_3.temp) Group Key: date_trunc('month'::text, pg2dim_3."time"), pg2dim_3.device -> Seq Scan on public.pg2dim_h2_t2 pg2dim_3 Output: date_trunc('month'::text, pg2dim_3."time"), pg2dim_3.device, pg2dim_3.temp (27 rows) -- Now run on hypertable -- All partition keys not covered by GROUP BY (partial partitionwise) SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT device, avg(temp) FROM hyper GROUP BY 1 ORDER BY 1; QUERY PLAN ---------------------------------------------------------------------------- Sort Output: _hyper_1_1_chunk.device, (avg(_hyper_1_1_chunk.temp)) Sort Key: _hyper_1_1_chunk.device -> HashAggregate Output: _hyper_1_1_chunk.device, avg(_hyper_1_1_chunk.temp) Group Key: _hyper_1_1_chunk.device -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp (15 rows) SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT device, avg(temp) FROM hyper GROUP BY 1 ORDER BY 1; QUERY PLAN ----------------------------------------------------------------------------------------- Sort Output: _hyper_1_1_chunk.device, (avg(_hyper_1_1_chunk.temp)) Sort Key: _hyper_1_1_chunk.device -> Finalize HashAggregate Output: _hyper_1_1_chunk.device, avg(_hyper_1_1_chunk.temp) Group Key: _hyper_1_1_chunk.device -> Append -> Partial HashAggregate Output: _hyper_1_1_chunk.device, PARTIAL avg(_hyper_1_1_chunk.temp) Group Key: _hyper_1_1_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Partial HashAggregate Output: _hyper_1_2_chunk.device, PARTIAL avg(_hyper_1_2_chunk.temp) Group Key: _hyper_1_2_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Partial HashAggregate Output: _hyper_1_3_chunk.device, PARTIAL avg(_hyper_1_3_chunk.temp) Group Key: _hyper_1_3_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Partial HashAggregate Output: _hyper_1_4_chunk.device, PARTIAL avg(_hyper_1_4_chunk.temp) Group Key: _hyper_1_4_chunk.device -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp (27 rows) -- All partition keys covered (full partitionwise) SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT time, device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; QUERY PLAN ----------------------------------------------------------------------------------------------------- Sort Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, (avg(_hyper_1_1_chunk.temp)) Sort Key: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device -> HashAggregate Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, avg(_hyper_1_1_chunk.temp) Group Key: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp (15 rows) SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT time, device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------- Finalize GroupAggregate Output: hyper."time", hyper.device, avg(hyper.temp) Group Key: hyper."time", hyper.device -> Sort Output: hyper."time", hyper.device, (PARTIAL avg(hyper.temp)) Sort Key: hyper."time", hyper.device -> Custom Scan (ChunkAppend) on public.hyper Output: hyper."time", hyper.device, (PARTIAL avg(hyper.temp)) Order: hyper."time" Startup Exclusion: false Runtime Exclusion: false -> Merge Append Sort Key: _hyper_1_1_chunk."time" -> Partial GroupAggregate Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, PARTIAL avg(_hyper_1_1_chunk.temp) Group Key: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device -> Sort Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp Sort Key: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device -> Index Scan Backward using _hyper_1_1_chunk_hyper_time_idx on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Partial GroupAggregate Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, PARTIAL avg(_hyper_1_2_chunk.temp) Group Key: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device -> Sort Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp Sort Key: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device -> Index Scan Backward using _hyper_1_2_chunk_hyper_time_idx on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Merge Append Sort Key: _hyper_1_3_chunk."time" -> Partial GroupAggregate Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, PARTIAL avg(_hyper_1_3_chunk.temp) Group Key: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device -> Sort Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp Sort Key: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device -> Index Scan Backward using _hyper_1_3_chunk_hyper_time_idx on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Partial GroupAggregate Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, PARTIAL avg(_hyper_1_4_chunk.temp) Group Key: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device -> Sort Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp Sort Key: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device -> Index Scan Backward using _hyper_1_4_chunk_hyper_time_idx on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp (47 rows) -- Partial aggregation since date_trunc(time) is not a partition key SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------- Sort Output: (date_trunc('month'::text, _hyper_1_1_chunk."time")), _hyper_1_1_chunk.device, (avg(_hyper_1_1_chunk.temp)) Sort Key: (date_trunc('month'::text, _hyper_1_1_chunk."time")), _hyper_1_1_chunk.device -> HashAggregate Output: (date_trunc('month'::text, _hyper_1_1_chunk."time")), _hyper_1_1_chunk.device, avg(_hyper_1_1_chunk.temp) Group Key: date_trunc('month'::text, _hyper_1_1_chunk."time"), _hyper_1_1_chunk.device -> Result Output: date_trunc('month'::text, _hyper_1_1_chunk."time"), _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp (17 rows) -- Partial aggregation pushdown is currently not supported for this query by -- the TSDB pushdown code since a projection is used in the path. SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; QUERY PLAN --------------------------------------------------------------------------------------------------------------------------- Sort Output: (date_trunc('month'::text, _hyper_1_1_chunk."time")), _hyper_1_1_chunk.device, (avg(_hyper_1_1_chunk.temp)) Sort Key: (date_trunc('month'::text, _hyper_1_1_chunk."time")), _hyper_1_1_chunk.device -> HashAggregate Output: (date_trunc('month'::text, _hyper_1_1_chunk."time")), _hyper_1_1_chunk.device, avg(_hyper_1_1_chunk.temp) Group Key: date_trunc('month'::text, _hyper_1_1_chunk."time"), _hyper_1_1_chunk.device -> Result Output: date_trunc('month'::text, _hyper_1_1_chunk."time"), _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp (17 rows) -- Also test time_bucket SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT time_bucket('1 month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------- Sort Output: (time_bucket('@ 1 mon'::interval, _hyper_1_1_chunk."time")), _hyper_1_1_chunk.device, (avg(_hyper_1_1_chunk.temp)) Sort Key: (time_bucket('@ 1 mon'::interval, _hyper_1_1_chunk."time")), _hyper_1_1_chunk.device -> HashAggregate Output: (time_bucket('@ 1 mon'::interval, _hyper_1_1_chunk."time")), _hyper_1_1_chunk.device, avg(_hyper_1_1_chunk.temp) Group Key: time_bucket('@ 1 mon'::interval, _hyper_1_1_chunk."time"), _hyper_1_1_chunk.device -> Result Output: time_bucket('@ 1 mon'::interval, _hyper_1_1_chunk."time"), _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp (17 rows) SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT time_bucket('1 month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------- Sort Output: (time_bucket('@ 1 mon'::interval, _hyper_1_1_chunk."time")), _hyper_1_1_chunk.device, (avg(_hyper_1_1_chunk.temp)) Sort Key: (time_bucket('@ 1 mon'::interval, _hyper_1_1_chunk."time")), _hyper_1_1_chunk.device -> HashAggregate Output: (time_bucket('@ 1 mon'::interval, _hyper_1_1_chunk."time")), _hyper_1_1_chunk.device, avg(_hyper_1_1_chunk.temp) Group Key: time_bucket('@ 1 mon'::interval, _hyper_1_1_chunk."time"), _hyper_1_1_chunk.device -> Result Output: time_bucket('@ 1 mon'::interval, _hyper_1_1_chunk."time"), _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."time", _hyper_1_1_chunk.device, _hyper_1_1_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."time", _hyper_1_2_chunk.device, _hyper_1_2_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."time", _hyper_1_3_chunk.device, _hyper_1_3_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."time", _hyper_1_4_chunk.device, _hyper_1_4_chunk.temp (17 rows) -- Test partitionwise joins, mostly to see that we do not break -- anything CREATE TABLE hyper_meta (time timestamptz, device int, info text); SELECT * FROM create_hypertable('hyper_meta', 'time', 'device', 2); NOTICE: adding not-null constraint to column "time" hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 2 | public | hyper_meta | t (1 row) INSERT INTO hyper_meta VALUES ('2018-02-19 13:01', 1, 'device_1'), ('2018-02-19 13:02', 3, 'device_3'); SET enable_partitionwise_join = 'off'; :PREFIX SELECT h.time, h.device, h.temp, hm.info FROM hyper h, hyper_meta hm WHERE h.device = hm.device; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------- Merge Join Output: h_1."time", h_1.device, h_1.temp, hm_1.info Merge Cond: (hm_1.device = h_1.device) -> Merge Append Sort Key: hm_1.device -> Index Scan using _hyper_2_5_chunk_hyper_meta_device_time_idx on _timescaledb_internal._hyper_2_5_chunk hm_1 Output: hm_1.info, hm_1.device -> Index Scan using _hyper_2_6_chunk_hyper_meta_device_time_idx on _timescaledb_internal._hyper_2_6_chunk hm_2 Output: hm_2.info, hm_2.device -> Materialize Output: h_1."time", h_1.device, h_1.temp -> Merge Append Sort Key: h_1.device -> Index Scan using _hyper_1_1_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_1_chunk h_1 Output: h_1."time", h_1.device, h_1.temp -> Index Scan using _hyper_1_2_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_2_chunk h_2 Output: h_2."time", h_2.device, h_2.temp -> Index Scan using _hyper_1_3_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_3_chunk h_3 Output: h_3."time", h_3.device, h_3.temp -> Index Scan using _hyper_1_4_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_4_chunk h_4 Output: h_4."time", h_4.device, h_4.temp (21 rows) :PREFIX SELECT pg2.time, pg2.device, pg2.temp, pg1.temp FROM pg2dim pg2, pg1dim pg1 WHERE pg2.device = pg1.device; QUERY PLAN -------------------------------------------------------------------- Merge Join Output: pg2."time", pg2.device, pg2.temp, pg1.temp Merge Cond: (pg1.device = pg2.device) -> Sort Output: pg1.temp, pg1.device Sort Key: pg1.device -> Append -> Seq Scan on public.pg1dim_h1 pg1_1 Output: pg1_1.temp, pg1_1.device -> Seq Scan on public.pg1dim_h2 pg1_2 Output: pg1_2.temp, pg1_2.device -> Sort Output: pg2."time", pg2.device, pg2.temp Sort Key: pg2.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2_1 Output: pg2_1."time", pg2_1.device, pg2_1.temp -> Seq Scan on public.pg2dim_h1_t2 pg2_2 Output: pg2_2."time", pg2_2.device, pg2_2.temp -> Seq Scan on public.pg2dim_h2_t1 pg2_3 Output: pg2_3."time", pg2_3.device, pg2_3.temp -> Seq Scan on public.pg2dim_h2_t2 pg2_4 Output: pg2_4."time", pg2_4.device, pg2_4.temp (23 rows) SET enable_partitionwise_join = 'on'; :PREFIX SELECT h.time, h.device, h.temp, hm.info FROM hyper h, hyper_meta hm WHERE h.device = hm.device; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------- Merge Join Output: h_1."time", h_1.device, h_1.temp, hm_1.info Merge Cond: (hm_1.device = h_1.device) -> Merge Append Sort Key: hm_1.device -> Index Scan using _hyper_2_5_chunk_hyper_meta_device_time_idx on _timescaledb_internal._hyper_2_5_chunk hm_1 Output: hm_1.info, hm_1.device -> Index Scan using _hyper_2_6_chunk_hyper_meta_device_time_idx on _timescaledb_internal._hyper_2_6_chunk hm_2 Output: hm_2.info, hm_2.device -> Materialize Output: h_1."time", h_1.device, h_1.temp -> Merge Append Sort Key: h_1.device -> Index Scan using _hyper_1_1_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_1_chunk h_1 Output: h_1."time", h_1.device, h_1.temp -> Index Scan using _hyper_1_2_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_2_chunk h_2 Output: h_2."time", h_2.device, h_2.temp -> Index Scan using _hyper_1_3_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_3_chunk h_3 Output: h_3."time", h_3.device, h_3.temp -> Index Scan using _hyper_1_4_chunk_hyper_device_time_idx on _timescaledb_internal._hyper_1_4_chunk h_4 Output: h_4."time", h_4.device, h_4.temp (21 rows) :PREFIX SELECT pg2.time, pg2.device, pg2.temp, pg1.temp FROM pg2dim pg2, pg1dim pg1 WHERE pg2.device = pg1.device; QUERY PLAN -------------------------------------------------------------------------- Append -> Merge Join Output: pg2_2."time", pg2_2.device, pg2_2.temp, pg1_1.temp Merge Cond: (pg1_1.device = pg2_2.device) -> Sort Output: pg1_1.temp, pg1_1.device Sort Key: pg1_1.device -> Seq Scan on public.pg1dim_h1 pg1_1 Output: pg1_1.temp, pg1_1.device -> Sort Output: pg2_2."time", pg2_2.device, pg2_2.temp Sort Key: pg2_2.device -> Append -> Seq Scan on public.pg2dim_h1_t1 pg2_2 Output: pg2_2."time", pg2_2.device, pg2_2.temp -> Seq Scan on public.pg2dim_h1_t2 pg2_3 Output: pg2_3."time", pg2_3.device, pg2_3.temp -> Merge Join Output: pg2_5."time", pg2_5.device, pg2_5.temp, pg1_2.temp Merge Cond: (pg1_2.device = pg2_5.device) -> Sort Output: pg1_2.temp, pg1_2.device Sort Key: pg1_2.device -> Seq Scan on public.pg1dim_h2 pg1_2 Output: pg1_2.temp, pg1_2.device -> Sort Output: pg2_5."time", pg2_5.device, pg2_5.temp Sort Key: pg2_5.device -> Append -> Seq Scan on public.pg2dim_h2_t1 pg2_5 Output: pg2_5."time", pg2_5.device, pg2_5.temp -> Seq Scan on public.pg2dim_h2_t2 pg2_6 Output: pg2_6."time", pg2_6.device, pg2_6.temp (33 rows) -- Test hypertable with time partitioning function CREATE OR REPLACE FUNCTION time_func(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ DECLARE retval TIMESTAMPTZ; BEGIN retval := to_timestamp(unixtime); RETURN retval; END $BODY$; CREATE TABLE hyper_timepart (time float8, device int, temp float); SELECT * FROM create_hypertable('hyper_timepart', 'time', 'device', 2, time_partitioning_func => 'time_func'); NOTICE: adding not-null constraint to column "time" hypertable_id | schema_name | table_name | created ---------------+-------------+----------------+--------- 3 | public | hyper_timepart | t (1 row) -- Planner won't pick push-down aggs on table with time function -- unless a certain amount of data SELECT setseed(1); setseed --------- (1 row) INSERT INTO hyper_timepart SELECT x, ceil(random() * 8), random() * 20 FROM generate_series(0,5000-1) AS x; -- All partition keys covered (full partitionwise) SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT time, device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; QUERY PLAN ----------------------------------------------------------------------------------------------------------- Limit Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, (avg(_hyper_3_7_chunk.temp)) -> Sort Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, (avg(_hyper_3_7_chunk.temp)) Sort Key: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device -> HashAggregate Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, avg(_hyper_3_7_chunk.temp) Group Key: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device -> Append -> Seq Scan on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp (13 rows) :PREFIX SELECT time_func(time), device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------- Limit Output: (time_func(_hyper_3_7_chunk."time")), _hyper_3_7_chunk.device, (avg(_hyper_3_7_chunk.temp)) -> GroupAggregate Output: (time_func(_hyper_3_7_chunk."time")), _hyper_3_7_chunk.device, avg(_hyper_3_7_chunk.temp) Group Key: (time_func(_hyper_3_7_chunk."time")), _hyper_3_7_chunk.device -> Incremental Sort Output: (time_func(_hyper_3_7_chunk."time")), _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp Sort Key: (time_func(_hyper_3_7_chunk."time")), _hyper_3_7_chunk.device Presorted Key: (time_func(_hyper_3_7_chunk."time")) -> Result Output: (time_func(_hyper_3_7_chunk."time")), _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp -> Merge Append Sort Key: (time_func(_hyper_3_7_chunk."time")) -> Index Scan Backward using _hyper_3_7_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp, time_func(_hyper_3_7_chunk."time") -> Index Scan Backward using _hyper_3_8_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp, time_func(_hyper_3_8_chunk."time") (17 rows) -- Grouping on original time column should be pushed-down SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT time, device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------ Limit Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, (avg(_hyper_3_7_chunk.temp)) -> Sort Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, (avg(_hyper_3_7_chunk.temp)) Sort Key: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device -> Finalize HashAggregate Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, avg(_hyper_3_7_chunk.temp) Group Key: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device -> Append -> Partial HashAggregate Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, PARTIAL avg(_hyper_3_7_chunk.temp) Group Key: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device -> Seq Scan on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp -> Partial HashAggregate Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, PARTIAL avg(_hyper_3_8_chunk.temp) Group Key: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device -> Seq Scan on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp (19 rows) -- Applying the time partitioning function should also allow push-down -- on open dimensions :PREFIX SELECT time_func(time), device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------- Limit Output: (time_func(_hyper_3_7_chunk."time")), _hyper_3_7_chunk.device, (avg(_hyper_3_7_chunk.temp)) -> GroupAggregate Output: (time_func(_hyper_3_7_chunk."time")), _hyper_3_7_chunk.device, avg(_hyper_3_7_chunk.temp) Group Key: (time_func(_hyper_3_7_chunk."time")), _hyper_3_7_chunk.device -> Incremental Sort Output: (time_func(_hyper_3_7_chunk."time")), _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp Sort Key: (time_func(_hyper_3_7_chunk."time")), _hyper_3_7_chunk.device Presorted Key: (time_func(_hyper_3_7_chunk."time")) -> Result Output: (time_func(_hyper_3_7_chunk."time")), _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp -> Merge Append Sort Key: (time_func(_hyper_3_7_chunk."time")) -> Index Scan Backward using _hyper_3_7_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp, time_func(_hyper_3_7_chunk."time") -> Index Scan Backward using _hyper_3_8_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp, time_func(_hyper_3_8_chunk."time") (17 rows) -- Partial aggregation pushdown is currently not supported for this query by -- the TSDB pushdown code since a projection is used in the path. :PREFIX SELECT time_func(time), _timescaledb_functions.get_partition_hash(device), avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------------------------------------- Limit Output: (time_func(_hyper_3_7_chunk."time")), (_timescaledb_functions.get_partition_hash(_hyper_3_7_chunk.device)), (avg(_hyper_3_7_chunk.temp)) -> GroupAggregate Output: (time_func(_hyper_3_7_chunk."time")), (_timescaledb_functions.get_partition_hash(_hyper_3_7_chunk.device)), avg(_hyper_3_7_chunk.temp) Group Key: (time_func(_hyper_3_7_chunk."time")), (_timescaledb_functions.get_partition_hash(_hyper_3_7_chunk.device)) -> Incremental Sort Output: (time_func(_hyper_3_7_chunk."time")), (_timescaledb_functions.get_partition_hash(_hyper_3_7_chunk.device)), _hyper_3_7_chunk.temp Sort Key: (time_func(_hyper_3_7_chunk."time")), (_timescaledb_functions.get_partition_hash(_hyper_3_7_chunk.device)) Presorted Key: (time_func(_hyper_3_7_chunk."time")) -> Result Output: (time_func(_hyper_3_7_chunk."time")), _timescaledb_functions.get_partition_hash(_hyper_3_7_chunk.device), _hyper_3_7_chunk.temp -> Merge Append Sort Key: (time_func(_hyper_3_7_chunk."time")) -> Index Scan Backward using _hyper_3_7_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_7_chunk Output: _hyper_3_7_chunk."time", _hyper_3_7_chunk.device, _hyper_3_7_chunk.temp, time_func(_hyper_3_7_chunk."time") -> Index Scan Backward using _hyper_3_8_chunk_hyper_timepart_expr_idx on _timescaledb_internal._hyper_3_8_chunk Output: _hyper_3_8_chunk."time", _hyper_3_8_chunk.device, _hyper_3_8_chunk.temp, time_func(_hyper_3_8_chunk."time") (17 rows) ================================================ FILE: test/expected/pg_dump.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set TEST_DBNAME_EXTRA :TEST_DBNAME _extra \o /dev/null \ir include/insert_two_partitions.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."two_Partitions" ( "timeCustom" BIGINT NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."two_Partitions" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_0) WHERE series_0 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_1) WHERE series_1 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_2) WHERE series_2 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_bool) WHERE series_bool IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, device_id); SELECT * FROM create_hypertable('"public"."two_Partitions"'::regclass, 'timeCustom'::name, 'device_id'::name, associated_schema_name=>'_timescaledb_internal'::text, number_partitions => 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); \set QUIET off BEGIN; \COPY public."two_Partitions" FROM 'data/ds1_dev1_1.tsv' NULL AS ''; COMMIT; INSERT INTO public."two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257987600000000000, 'dev1', 1.5, 1), (1257987600000000000, 'dev1', 1.5, 2), (1257894000000000000, 'dev2', 1.5, 1), (1257894002000000000, 'dev1', 2.5, 3); INSERT INTO "two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257894000000000000, 'dev2', 1.5, 2); \set QUIET on \o \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION bgw_wait(database TEXT, timeout INT, raise_error BOOLEAN DEFAULT TRUE) RETURNS VOID AS :MODULE_PATHNAME, 'ts_bgw_wait' LANGUAGE C VOLATILE; CREATE SCHEMA test_schema AUTHORIZATION :ROLE_DEFAULT_PERM_USER; \c :TEST_DBNAME ALTER TABLE PUBLIC."two_Partitions" SET SCHEMA "test_schema"; -- Test that we can restore constraints ALTER TABLE "test_schema"."two_Partitions" ADD CONSTRAINT timeCustom_device_id_series_2_key UNIQUE ("timeCustom", device_id, series_2); -- Test that we can restore triggers CREATE OR REPLACE FUNCTION test_trigger() RETURNS TRIGGER LANGUAGE PLPGSQL AS $BODY$ BEGIN RETURN NEW; END $BODY$; -- Test that a custom chunk sizing function is restored CREATE OR REPLACE FUNCTION custom_calculate_chunk_interval( dimension_id INTEGER, dimension_coord BIGINT, chunk_target_size BIGINT ) RETURNS BIGINT LANGUAGE PLPGSQL AS $BODY$ DECLARE BEGIN RETURN -1; END $BODY$; SELECT * FROM set_adaptive_chunking('"test_schema"."two_Partitions"', '1 MB', 'custom_calculate_chunk_interval'); WARNING: target chunk size for adaptive chunking is less than 10 MB chunk_sizing_func | chunk_target_size ---------------------------------+------------------- custom_calculate_chunk_interval | 1048576 -- Chunk sizing func set SELECT * FROM _timescaledb_catalog.hypertable; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+----------------+------------------------+-------------------------+----------------+--------------------------+---------------------------------+-------------------+-------------------+--------------------------+-------- 1 | test_schema | two_Partitions | _timescaledb_internal | _hyper_1 | 2 | public | custom_calculate_chunk_interval | 1048576 | 0 | | 0 SELECT proname, pronamespace, pronargs FROM pg_proc WHERE proname = 'custom_calculate_chunk_interval'; proname | pronamespace | pronargs ---------------------------------+--------------+---------- custom_calculate_chunk_interval | 2200 | 3 CREATE TRIGGER restore_trigger BEFORE INSERT ON "test_schema"."two_Partitions" FOR EACH ROW EXECUTE FUNCTION test_trigger(); -- Save the number of dependent objects so we can make sure we have the same number later SELECT count(*) as num_dependent_objects FROM pg_depend WHERE refclassid = 'pg_extension'::regclass AND refobjid = (SELECT oid FROM pg_extension WHERE extname = 'timescaledb') \gset SELECT * FROM test.show_columns('"test_schema"."two_Partitions"'); Column | Type | NotNull -------------+------------------+--------- timeCustom | bigint | t device_id | text | t series_0 | double precision | f series_1 | double precision | f series_2 | double precision | f series_bool | boolean | f SELECT * FROM test.show_columns('_timescaledb_internal._hyper_1_1_chunk'); Column | Type | NotNull -------------+------------------+--------- timeCustom | bigint | t device_id | text | t series_0 | double precision | f series_1 | double precision | f series_2 | double precision | f series_bool | boolean | f SELECT * FROM test.show_indexes('"test_schema"."two_Partitions"'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ---------------------------------------------------------+---------------------------------+------+--------+---------+-----------+------------ test_schema.timecustom_device_id_series_2_key | {timeCustom,device_id,series_2} | | t | f | f | test_schema."two_Partitions_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | test_schema."two_Partitions_timeCustom_device_id_idx" | {timeCustom,device_id} | | f | f | f | test_schema."two_Partitions_timeCustom_idx" | {timeCustom} | | f | f | f | test_schema."two_Partitions_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | test_schema."two_Partitions_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | test_schema."two_Partitions_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | test_schema."two_Partitions_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | SELECT * FROM test.show_indexes('_timescaledb_internal._hyper_1_1_chunk'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ------------------------------------------------------------------------------------+---------------------------------+------+--------+---------+-----------+------------ _timescaledb_internal."1_1_timecustom_device_id_series_2_key" | {timeCustom,device_id,series_2} | | t | f | f | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_device_id_idx" | {timeCustom,device_id} | | f | f | f | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_idx" | {timeCustom} | | f | f | f | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | SELECT * FROM test.show_constraints('"test_schema"."two_Partitions"'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated -----------------------------------+------+---------------------------------+-----------------------------------------------+------+------------+----------+----------- timecustom_device_id_series_2_key | u | {timeCustom,device_id,series_2} | test_schema.timecustom_device_id_series_2_key | | f | f | t SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_1_1_chunk'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated ---------------------------------------+------+---------------------------------+---------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------+----------+----------- 1_1_timecustom_device_id_series_2_key | u | {timeCustom,device_id,series_2} | _timescaledb_internal."1_1_timecustom_device_id_series_2_key" | | f | f | t constraint_1 | c | {timeCustom} | - | (("timeCustom" >= '1257892416000000000'::bigint) AND ("timeCustom" < '1257895008000000000'::bigint)) | f | f | t constraint_2 | c | {device_id} | - | (_timescaledb_functions.get_partition_hash(device_id) >= 1073741823) | f | f | t SELECT * FROM test.show_triggers('"test_schema"."two_Partitions"'); Trigger | Type | Function -----------------+------+-------------- restore_trigger | 7 | test_trigger SELECT * FROM test.show_triggers('_timescaledb_internal._hyper_1_1_chunk'); Trigger | Type | Function -----------------+------+-------------- restore_trigger | 7 | test_trigger SELECT * FROM "test_schema"."two_Partitions" ORDER BY "timeCustom", device_id, series_0, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257894000000000000 | dev1 | 1.5 | 1 | 2 | t 1257894000000000000 | dev1 | 1.5 | 2 | | 1257894000000000000 | dev2 | 1.5 | 1 | | 1257894000000000000 | dev2 | 1.5 | 2 | | 1257894000000001000 | dev1 | 2.5 | 3 | | 1257894001000000000 | dev1 | 3.5 | 4 | | 1257894002000000000 | dev1 | 2.5 | 3 | | 1257894002000000000 | dev1 | 5.5 | 6 | | t 1257894002000000000 | dev1 | 5.5 | 7 | | f 1257897600000000000 | dev1 | 4.5 | 5 | | f 1257987600000000000 | dev1 | 1.5 | 1 | | 1257987600000000000 | dev1 | 1.5 | 2 | | SELECT * FROM _timescaledb_internal._hyper_1_1_chunk ORDER BY "timeCustom", device_id, series_0, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257894000000000000 | dev1 | 1.5 | 1 | 2 | t 1257894000000000000 | dev1 | 1.5 | 2 | | 1257894000000001000 | dev1 | 2.5 | 3 | | 1257894001000000000 | dev1 | 3.5 | 4 | | 1257894002000000000 | dev1 | 2.5 | 3 | | 1257894002000000000 | dev1 | 5.5 | 6 | | t 1257894002000000000 | dev1 | 5.5 | 7 | | f SELECT * FROM _timescaledb_internal._hyper_1_2_chunk ORDER BY "timeCustom", device_id, series_0, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257897600000000000 | dev1 | 4.5 | 5 | | f -- Show all constraints SELECT * FROM _timescaledb_catalog.chunk_constraint; chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----------+--------------------+---------------------------------------+----------------------------------- 1 | 1 | constraint_1 | 1 | 2 | constraint_2 | 2 | 3 | constraint_3 | 2 | 2 | constraint_2 | 3 | 4 | constraint_4 | 3 | 2 | constraint_2 | 4 | 1 | constraint_1 | 4 | 5 | constraint_5 | 1 | | 1_1_timecustom_device_id_series_2_key | timecustom_device_id_series_2_key 2 | | 2_2_timecustom_device_id_series_2_key | timecustom_device_id_series_2_key 3 | | 3_3_timecustom_device_id_series_2_key | timecustom_device_id_series_2_key 4 | | 4_4_timecustom_device_id_series_2_key | timecustom_device_id_series_2_key INSERT INTO _timescaledb_catalog.metadata VALUES ('exported_uuid', 'original_uuid', true); INSERT INTO _timescaledb_catalog.metadata VALUES ('metadata_test', 'FOO', false); \c postgres :ROLE_SUPERUSER -- We shell out to a script in order to grab the correct hostname from the -- environmental variables that originally called this psql command. Sadly -- vars passed to psql do not work in \! commands so we can't do it that way. \! utils/pg_dump_aux_dump.sh dump/pg_dump.sql \c :TEST_DBNAME SET client_min_messages = ERROR; CREATE EXTENSION timescaledb CASCADE; --create a exported uuid before restoring (mocks telemetry running before restore) INSERT INTO _timescaledb_catalog.metadata VALUES ('exported_uuid', 'new_db_uuid', true); -- disable background jobs UPDATE _timescaledb_catalog.bgw_job SET scheduled = false; RESET client_min_messages; SELECT timescaledb_pre_restore(); timescaledb_pre_restore ------------------------- t SHOW timescaledb.restoring; timescaledb.restoring ----------------------- on -- reconnect and check GUC value in new session \c SHOW timescaledb.restoring; timescaledb.restoring ----------------------- on \! utils/pg_dump_aux_restore.sh dump/pg_dump.sql -- Now run our post-restore function. SELECT timescaledb_post_restore(); timescaledb_post_restore -------------------------- t SHOW timescaledb.restoring; timescaledb.restoring ----------------------- off -- timescaledb_post_restore restarts background worker so we have to stop them -- to make sure they dont interfere with this database being used as template below SELECT _timescaledb_functions.stop_background_workers(); stop_background_workers ------------------------- t --should be same as count above SELECT count(*) = :num_dependent_objects as dependent_objects_match FROM pg_depend WHERE refclassid = 'pg_extension'::regclass AND refobjid = (SELECT oid FROM pg_extension WHERE extname = 'timescaledb'); dependent_objects_match ------------------------- t --we should have the original uuid from the backed up db set as the exported_uuid SELECT value = 'original_uuid' FROM _timescaledb_catalog.metadata WHERE key='exported_uuid'; ?column? ---------- t SELECT count(*) = 1 FROM _timescaledb_catalog.metadata WHERE key LIKE 'exported%'; ?column? ---------- t --we should have the original value of metadata_test SELECT * FROM _timescaledb_catalog.metadata WHERE key='metadata_test'; key | value | include_in_telemetry ---------------+-------+---------------------- metadata_test | FOO | f --main table and chunk schemas should be the same SELECT * FROM test.show_columns('"test_schema"."two_Partitions"'); Column | Type | NotNull -------------+------------------+--------- timeCustom | bigint | t device_id | text | t series_0 | double precision | f series_1 | double precision | f series_2 | double precision | f series_bool | boolean | f SELECT * FROM test.show_columns('_timescaledb_internal._hyper_1_1_chunk'); Column | Type | NotNull -------------+------------------+--------- timeCustom | bigint | t device_id | text | t series_0 | double precision | f series_1 | double precision | f series_2 | double precision | f series_bool | boolean | f SELECT * FROM test.show_indexes('"test_schema"."two_Partitions"'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ---------------------------------------------------------+---------------------------------+------+--------+---------+-----------+------------ test_schema.timecustom_device_id_series_2_key | {timeCustom,device_id,series_2} | | t | f | f | test_schema."two_Partitions_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | test_schema."two_Partitions_timeCustom_device_id_idx" | {timeCustom,device_id} | | f | f | f | test_schema."two_Partitions_timeCustom_idx" | {timeCustom} | | f | f | f | test_schema."two_Partitions_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | test_schema."two_Partitions_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | test_schema."two_Partitions_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | test_schema."two_Partitions_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | SELECT * FROM test.show_indexes('_timescaledb_internal._hyper_1_1_chunk'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ------------------------------------------------------------------------------------+---------------------------------+------+--------+---------+-----------+------------ _timescaledb_internal."1_1_timecustom_device_id_series_2_key" | {timeCustom,device_id,series_2} | | t | f | f | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_device_id_timeCustom_idx" | {device_id,timeCustom} | | f | f | f | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_device_id_idx" | {timeCustom,device_id} | | f | f | f | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_idx" | {timeCustom} | | f | f | f | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_0_idx" | {timeCustom,series_0} | | f | f | f | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" | {timeCustom,series_1} | | f | f | f | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_2_idx" | {timeCustom,series_2} | | f | f | f | _timescaledb_internal."_hyper_1_1_chunk_two_Partitions_timeCustom_series_bool_idx" | {timeCustom,series_bool} | | f | f | f | SELECT * FROM test.show_constraints('"test_schema"."two_Partitions"'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated -----------------------------------+------+---------------------------------+-----------------------------------------------+------+------------+----------+----------- timecustom_device_id_series_2_key | u | {timeCustom,device_id,series_2} | test_schema.timecustom_device_id_series_2_key | | f | f | t SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_1_1_chunk'); Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated ---------------------------------------+------+---------------------------------+---------------------------------------------------------------+------------------------------------------------------------------------------------------------------+------------+----------+----------- 1_1_timecustom_device_id_series_2_key | u | {timeCustom,device_id,series_2} | _timescaledb_internal."1_1_timecustom_device_id_series_2_key" | | f | f | t constraint_1 | c | {timeCustom} | - | (("timeCustom" >= '1257892416000000000'::bigint) AND ("timeCustom" < '1257895008000000000'::bigint)) | f | f | t constraint_2 | c | {device_id} | - | (_timescaledb_functions.get_partition_hash(device_id) >= 1073741823) | f | f | t SELECT * FROM test.show_triggers('"test_schema"."two_Partitions"'); Trigger | Type | Function -----------------+------+-------------- restore_trigger | 7 | test_trigger SELECT * FROM test.show_triggers('_timescaledb_internal._hyper_1_1_chunk'); Trigger | Type | Function -----------------+------+-------------- restore_trigger | 7 | test_trigger --data should be the same SELECT * FROM "test_schema"."two_Partitions" ORDER BY "timeCustom", device_id, series_0, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257894000000000000 | dev1 | 1.5 | 1 | 2 | t 1257894000000000000 | dev1 | 1.5 | 2 | | 1257894000000000000 | dev2 | 1.5 | 1 | | 1257894000000000000 | dev2 | 1.5 | 2 | | 1257894000000001000 | dev1 | 2.5 | 3 | | 1257894001000000000 | dev1 | 3.5 | 4 | | 1257894002000000000 | dev1 | 2.5 | 3 | | 1257894002000000000 | dev1 | 5.5 | 6 | | t 1257894002000000000 | dev1 | 5.5 | 7 | | f 1257897600000000000 | dev1 | 4.5 | 5 | | f 1257987600000000000 | dev1 | 1.5 | 1 | | 1257987600000000000 | dev1 | 1.5 | 2 | | SELECT * FROM _timescaledb_internal._hyper_1_1_chunk ORDER BY "timeCustom", device_id, series_0, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257894000000000000 | dev1 | 1.5 | 1 | 2 | t 1257894000000000000 | dev1 | 1.5 | 2 | | 1257894000000001000 | dev1 | 2.5 | 3 | | 1257894001000000000 | dev1 | 3.5 | 4 | | 1257894002000000000 | dev1 | 2.5 | 3 | | 1257894002000000000 | dev1 | 5.5 | 6 | | t 1257894002000000000 | dev1 | 5.5 | 7 | | f SELECT * FROM _timescaledb_internal._hyper_1_2_chunk ORDER BY "timeCustom", device_id, series_0, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257897600000000000 | dev1 | 4.5 | 5 | | f SELECT * FROM _timescaledb_catalog.chunk_constraint; chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----------+--------------------+---------------------------------------+----------------------------------- 1 | 1 | constraint_1 | 1 | 2 | constraint_2 | 2 | 3 | constraint_3 | 2 | 2 | constraint_2 | 3 | 4 | constraint_4 | 3 | 2 | constraint_2 | 4 | 1 | constraint_1 | 4 | 5 | constraint_5 | 1 | | 1_1_timecustom_device_id_series_2_key | timecustom_device_id_series_2_key 2 | | 2_2_timecustom_device_id_series_2_key | timecustom_device_id_series_2_key 3 | | 3_3_timecustom_device_id_series_2_key | timecustom_device_id_series_2_key 4 | | 4_4_timecustom_device_id_series_2_key | timecustom_device_id_series_2_key --Chunk sizing function should have been restored SELECT * FROM _timescaledb_catalog.hypertable; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+----------------+------------------------+-------------------------+----------------+--------------------------+---------------------------------+-------------------+-------------------+--------------------------+-------- 1 | test_schema | two_Partitions | _timescaledb_internal | _hyper_1 | 2 | public | custom_calculate_chunk_interval | 1048576 | 0 | | 0 SELECT proname, pronamespace, pronargs FROM pg_proc WHERE proname = 'custom_calculate_chunk_interval'; proname | pronamespace | pronargs ---------------------------------+--------------+---------- custom_calculate_chunk_interval | 2200 | 3 --check simple ddl still works ALTER TABLE "test_schema"."two_Partitions" ADD COLUMN series_3 integer; INSERT INTO "test_schema"."two_Partitions"("timeCustom", device_id, series_0, series_1, series_3) VALUES (1357894000000000000, 'dev5', 1.5, 2, 4); SELECT * FROM ONLY "test_schema"."two_Partitions"; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool | series_3 ------------+-----------+----------+----------+----------+-------------+---------- --query for the extension tables/sequences that will not be dumped by pg_dump (should --be empty except for views and explicitly excluded tables) SELECT objid::regclass FROM pg_catalog.pg_depend WHERE refclassid = 'pg_catalog.pg_extension'::pg_catalog.regclass AND refobjid = (select oid from pg_extension where extname='timescaledb') AND deptype = 'e' AND classid='pg_catalog.pg_class'::pg_catalog.regclass AND objid NOT IN (select unnest(extconfig) from pg_extension where extname='timescaledb') ORDER BY objid::regclass::text COLLATE "C"; objid --------------------------------------------------------- _timescaledb_cache.cache_inval_bgw_job _timescaledb_cache.cache_inval_extension _timescaledb_cache.cache_inval_hypertable _timescaledb_catalog.chunk_rewrite _timescaledb_catalog.compression_algorithm _timescaledb_catalog.tablespace_id_seq _timescaledb_catalog.telemetry_event _timescaledb_config.bgw_job _timescaledb_internal.bgw_job_stat _timescaledb_internal.bgw_job_stat_history _timescaledb_internal.bgw_job_stat_history_id_seq _timescaledb_internal.bgw_policy_chunk_stats _timescaledb_internal.compressed_chunk_stats _timescaledb_internal.hypertable_chunk_local_size timescaledb_experimental.policies timescaledb_information.chunk_columnstore_settings timescaledb_information.chunk_compression_settings timescaledb_information.chunks timescaledb_information.compression_settings timescaledb_information.continuous_aggregates timescaledb_information.dimensions timescaledb_information.hypertable_columnstore_settings timescaledb_information.hypertable_compression_settings timescaledb_information.hypertables timescaledb_information.job_errors timescaledb_information.job_history timescaledb_information.job_stats timescaledb_information.jobs -- Make sure we can't run our restoring functions as a normal perm user as that would disable functionality for the whole db \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- Hides error messages in cases where error messages differ between Postgres versions create or replace function get_sqlstate(in_text TEXT) RETURNS TEXT AS $$ BEGIN BEGIN EXECUTE in_text; EXCEPTION WHEN others THEN GET STACKED DIAGNOSTICS in_text = RETURNED_SQLSTATE; END; RETURN in_text; END; $$ LANGUAGE PLPGSQL; SELECT get_sqlstate('SELECT timescaledb_pre_restore()'); get_sqlstate -------------- 42501 SELECT get_sqlstate('SELECT timescaledb_post_restore()'); get_sqlstate -------------- 42501 drop function get_sqlstate(TEXT); -- Check that the extension can be copied from an existing database -- without explicitly installing it. Stop background workers since we -- cannot have any backends connected to the database when cloning it. \c :TEST_DBNAME :ROLE_SUPERUSER SELECT timescaledb_pre_restore(); timescaledb_pre_restore ------------------------- t SELECT bgw_wait(:'TEST_DBNAME', 60, FALSE); bgw_wait ---------- -- Force other sessions connected to the TEST_DBNAME to be finished \c postgres :ROLE_SUPERUSER REVOKE CONNECT ON DATABASE :TEST_DBNAME FROM public; ALTER DATABASE :TEST_DBNAME allow_connections = off; SET client_min_messages TO ERROR; SELECT COUNT(pg_catalog.pg_terminate_backend(pid))>=0 FROM pg_stat_activity WHERE datname = ':TEST_DBNAME'; ?column? ---------- t RESET client_min_messages; CREATE DATABASE :TEST_DBNAME_EXTRA WITH TEMPLATE :TEST_DBNAME; ALTER DATABASE :TEST_DBNAME allow_connections = on; GRANT CONNECT ON DATABASE :TEST_DBNAME TO public; -- Connect to the database and do some basic stuff to check that the -- extension works. \c :TEST_DBNAME_EXTRA :ROLE_DEFAULT_PERM_USER CREATE TABLE test_tz(time timestamptz not null, temp float8, device text); SELECT create_hypertable('test_tz', 'time', 'device', 2); create_hypertable ---------------------- (2,public,test_tz,t) SELECT id, schema_name, table_name FROM _timescaledb_catalog.hypertable; id | schema_name | table_name ----+-------------+---------------- 1 | test_schema | two_Partitions 2 | public | test_tz INSERT INTO test_tz VALUES('Mon Mar 20 09:17:00.936242 2017', 23.4, 'dev1'); INSERT INTO test_tz VALUES('Mon Mar 20 09:27:00.936242 2017', 22, 'dev2'); INSERT INTO test_tz VALUES('Mon Mar 20 09:28:00.936242 2017', 21.2, 'dev1'); INSERT INTO test_tz VALUES('Mon Mar 20 09:37:00.936242 2017', 30, 'dev3'); SELECT * FROM test_tz ORDER BY time; time | temp | device -------------------------------------+------+-------- Mon Mar 20 09:17:00.936242 2017 PDT | 23.4 | dev1 Mon Mar 20 09:27:00.936242 2017 PDT | 22 | dev2 Mon Mar 20 09:28:00.936242 2017 PDT | 21.2 | dev1 Mon Mar 20 09:37:00.936242 2017 PDT | 30 | dev3 \c :TEST_DBNAME :ROLE_SUPERUSER -- make sure nobody is using it SET client_min_messages TO ERROR; REVOKE CONNECT ON DATABASE :TEST_DBNAME_EXTRA FROM public; SELECT count(pg_terminate_backend(pg_stat_activity.pid)) AS TERMINATED FROM pg_stat_activity WHERE pg_stat_activity.datname = :'TEST_DBNAME_EXTRA' AND pg_stat_activity.pid <> pg_backend_pid() \gset RESET client_min_messages; DROP DATABASE :TEST_DBNAME_EXTRA WITH (FORCE); ================================================ FILE: test/expected/pg_dump_unprivileged.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c template1 :ROLE_SUPERUSER SET client_min_messages TO ERROR; CREATE EXTENSION IF NOT EXISTS timescaledb; RESET client_min_messages; CREATE USER dump_unprivileged CREATEDB; \c template1 dump_unprivileged CREATE database dump_unprivileged; \! utils/pg_dump_unprivileged.sh Database dumped successfully \c dump_unprivileged :ROLE_SUPERUSER DROP EXTENSION timescaledb; GRANT ALL ON DATABASE dump_unprivileged TO dump_unprivileged; \c dump_unprivileged dump_unprivileged -- Create the timescale extension and table as underprivileged user CREATE EXTENSION timescaledb; CREATE TABLE t1 (a int); -- pg_dump currently fails when dumped \! utils/pg_dump_unprivileged.sh Database dumped successfully \c template1 :ROLE_SUPERUSER DROP EXTENSION timescaledb; DROP DATABASE dump_unprivileged WITH (FORCE); DROP USER dump_unprivileged; ================================================ FILE: test/expected/pg_join.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- this test suite is based on the postgres join tests -- -- the tests have been adjusted to work with hypertables -- statements that would generate an error have been commented out -- because errors don't play nicely with psql output redirection -- plan output has been disabled, because we are not interested in -- the actual plans produced but in the correctness of the results -- we need superuser because some of the tests modify statistics \c :TEST_DBNAME :ROLE_SUPERUSER \set TEST_BASE_NAME join SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') as "TEST_LOAD_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') as "TEST_QUERY_NAME", format('%s/results/%s_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_OPTIMIZED", format('%s/results/%s_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_UNOPTIMIZED" \gset SELECT format('\! diff -u --label "Unoptimized results" --label "Optimized results" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') as "DIFF_CMD" \gset set client_min_messages to warning; \ir :TEST_LOAD_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- these table definitions have been adjusted from -- table defintions of the postgres test suite CREATE TABLE INT2_TBL(f1 int2, ts timestamptz NOT NULL DEFAULT '2000-01-01'); SELECT table_name FROM create_hypertable('int2_tbl','ts'); table_name ------------ int2_tbl INSERT INTO INT2_TBL(f1) VALUES ('0 '); INSERT INTO INT2_TBL(f1) VALUES (' 1234 '); INSERT INTO INT2_TBL(f1) VALUES (' -1234'); -- largest and smallest values INSERT INTO INT2_TBL(f1) VALUES ('32767'); INSERT INTO INT2_TBL(f1) VALUES ('-32767'); CREATE TABLE INT4_TBL(f1 int4, ts timestamptz NOT NULL DEFAULT '2000-01-01'); SELECT table_name FROM create_hypertable('int4_tbl','ts'); table_name ------------ int4_tbl INSERT INTO INT4_TBL(f1) VALUES (' 0 '); INSERT INTO INT4_TBL(f1) VALUES ('123456 '); INSERT INTO INT4_TBL(f1) VALUES (' -123456'); -- largest and smallest values INSERT INTO INT4_TBL(f1) VALUES ('2147483647'); INSERT INTO INT4_TBL(f1) VALUES ('-2147483647'); CREATE TABLE INT8_TBL(q1 int8, q2 int8, ts timestamptz NOT NULL DEFAULT '2000-01-01'); SELECT table_name FROM create_hypertable('int8_tbl','ts'); table_name ------------ int8_tbl INSERT INTO INT8_TBL VALUES(' 123 ',' 456'); INSERT INTO INT8_TBL VALUES('123 ','4567890123456789'); INSERT INTO INT8_TBL VALUES('4567890123456789','123'); INSERT INTO INT8_TBL VALUES(+4567890123456789,'4567890123456789'); INSERT INTO INT8_TBL VALUES('+4567890123456789','-4567890123456789'); CREATE TABLE FLOAT8_TBL(f1 float8, ts timestamptz NOT NULL DEFAULT '2000-01-01'); SELECT table_name FROM create_hypertable('float8_tbl','ts'); table_name ------------ float8_tbl INSERT INTO FLOAT8_TBL(f1) VALUES (' 0.0 '); INSERT INTO FLOAT8_TBL(f1) VALUES ('1004.30 '); INSERT INTO FLOAT8_TBL(f1) VALUES (' -34.84'); INSERT INTO FLOAT8_TBL(f1) VALUES ('1.2345678901234e+200'); INSERT INTO FLOAT8_TBL(f1) VALUES ('1.2345678901234e-200'); CREATE TABLE TEXT_TBL (f1 text, ts timestamptz NOT NULL DEFAULT '2000-01-01'); SELECT table_name FROM create_hypertable('text_tbl','ts'); table_name ------------ text_tbl INSERT INTO TEXT_TBL VALUES ('doh!'); INSERT INTO TEXT_TBL VALUES ('hi de ho neighbor'); CREATE TABLE a (aa TEXT); CREATE TABLE b (bb TEXT) INHERITS (a); CREATE TABLE c (cc TEXT) INHERITS (a); CREATE TABLE d (dd TEXT) INHERITS (b,c,a); INSERT INTO a(aa) VALUES('aaa'); INSERT INTO a(aa) VALUES('aaaa'); INSERT INTO a(aa) VALUES('aaaaa'); INSERT INTO a(aa) VALUES('aaaaaa'); INSERT INTO a(aa) VALUES('aaaaaaa'); INSERT INTO a(aa) VALUES('aaaaaaaa'); INSERT INTO b(aa) VALUES('bbb'); INSERT INTO b(aa) VALUES('bbbb'); INSERT INTO b(aa) VALUES('bbbbb'); INSERT INTO b(aa) VALUES('bbbbbb'); INSERT INTO b(aa) VALUES('bbbbbbb'); INSERT INTO b(aa) VALUES('bbbbbbbb'); INSERT INTO c(aa) VALUES('ccc'); INSERT INTO c(aa) VALUES('cccc'); INSERT INTO c(aa) VALUES('ccccc'); INSERT INTO c(aa) VALUES('cccccc'); INSERT INTO c(aa) VALUES('ccccccc'); INSERT INTO c(aa) VALUES('cccccccc'); INSERT INTO d(aa) VALUES('ddd'); INSERT INTO d(aa) VALUES('dddd'); INSERT INTO d(aa) VALUES('ddddd'); INSERT INTO d(aa) VALUES('dddddd'); INSERT INTO d(aa) VALUES('ddddddd'); INSERT INTO d(aa) VALUES('dddddddd'); CREATE TABLE onek ( unique1 int4, unique2 int4, two int4, four int4, ten int4, twenty int4, hundred int4, thousand int4, twothousand int4, fivethous int4, tenthous int4, odd int4, even int4, stringu1 name, stringu2 name, string4 name ); SELECT table_name FROM create_hypertable('onek','unique2',chunk_time_interval:=1000); table_name ------------ onek \copy onek FROM 'data/onek.data' CREATE TABLE tenk1 ( unique1 int4, unique2 int4, two int4, four int4, ten int4, twenty int4, hundred int4, thousand int4, twothousand int4, fivethous int4, tenthous int4, odd int4, even int4, stringu1 name, stringu2 name, string4 name ); SELECT table_name FROM create_hypertable('tenk1','unique2',chunk_time_interval:=1000); table_name ------------ tenk1 \copy tenk1 FROM 'data/tenk.data' CREATE TABLE tenk2 ( unique1 int4, unique2 int4, two int4, four int4, ten int4, twenty int4, hundred int4, thousand int4, twothousand int4, fivethous int4, tenthous int4, odd int4, even int4, stringu1 name, stringu2 name, string4 name ); SELECT table_name FROM create_hypertable('tenk2','unique2',chunk_time_interval:=1000); table_name ------------ tenk2 INSERT INTO tenk2 SELECT * FROM tenk1; \set PREFIX '' \set ECHO errors --- Unoptimized results +++ Optimized results @@ -1,6 +1,6 @@ setting | value ----------------------------------+------- - timescaledb.enable_optimizations | off + timescaledb.enable_optimizations | on table_name ================================================ FILE: test/expected/plain.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Tests for plain PostgreSQL commands to ensure that they work while -- the TimescaleDB extension is loaded. This is a mix of statements -- added mostly as regression checks when bugs are discovered and -- fixed. CREATE TABLE regular_table(time timestamp, temp float8, tag text, color integer); -- Renaming indexes should work CREATE INDEX time_color_idx ON regular_table(time, color); ALTER INDEX time_color_idx RENAME TO time_color_idx2; ALTER TABLE regular_table ALTER COLUMN color TYPE bigint; SELECT * FROM test.show_columns('regular_table'); Column | Type | NotNull --------+-----------------------------+--------- time | timestamp without time zone | f temp | double precision | f tag | text | f color | bigint | f SELECT * FROM test.show_indexes('regular_table'); Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace -----------------+--------------+------+--------+---------+-----------+------------ time_color_idx2 | {time,color} | | f | f | f | -- Renaming types should work CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple'); ALTER TYPE rainbow RENAME TO colors; \dT+ List of data types Schema | Name | Internal name | Size | Elements | Owner | Access privileges | Description --------+--------+---------------+------+----------+-------------------+-------------------+------------- public | colors | colors | 4 | red +| default_perm_user | | | | | | orange +| | | | | | | yellow +| | | | | | | green +| | | | | | | blue +| | | | | | | purple | | | REINDEX TABLE regular_table; \c :TEST_DBNAME :ROLE_SUPERUSER REINDEX SCHEMA public; -- Not only simple statements should work CREATE TABLE a (aa TEXT); CREATE TABLE z (b TEXT, PRIMARY KEY(aa, b)) inherits (a); ================================================ FILE: test/expected/plan_expand_hypertable-15.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set PREFIX 'EXPLAIN (buffers off, costs off) ' \ir include/plan_expand_hypertable_load.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --single time dimension CREATE TABLE hyper ("time_broken" bigint NOT NULL, "value" integer); ALTER TABLE hyper DROP COLUMN time_broken, ADD COLUMN time BIGINT; SELECT create_hypertable('hyper', 'time', chunk_time_interval => 10); create_hypertable -------------------- (1,public,hyper,t) INSERT INTO hyper SELECT g, g FROM generate_series(0,1000) g; --insert a point with INT_MAX_64 INSERT INTO hyper (time, value) SELECT 9223372036854775807::bigint, 0; --time and space CREATE TABLE hyper_w_space ("time_broken" bigint NOT NULL, "device_id" text, "value" integer); ALTER TABLE hyper_w_space DROP COLUMN time_broken, ADD COLUMN time BIGINT; SELECT create_hypertable('hyper_w_space', 'time', 'device_id', 4, chunk_time_interval => 10); create_hypertable ---------------------------- (2,public,hyper_w_space,t) INSERT INTO hyper_w_space (time, device_id, value) SELECT g, 'dev' || g, g FROM generate_series(0,30) g; CREATE VIEW hyper_w_space_view AS (SELECT * FROM hyper_w_space); --with timestamp and space CREATE TABLE tag (id serial PRIMARY KEY, name text); CREATE TABLE hyper_ts ("time_broken" timestamptz NOT NULL, "device_id" text, tag_id INT REFERENCES tag(id), "value" integer); ALTER TABLE hyper_ts DROP COLUMN time_broken, ADD COLUMN time TIMESTAMPTZ; SELECT create_hypertable('hyper_ts', 'time', 'device_id', 2, chunk_time_interval => '10 seconds'::interval); create_hypertable ----------------------- (3,public,hyper_ts,t) INSERT INTO tag(name) SELECT 'tag'||g FROM generate_series(0,10) g; INSERT INTO hyper_ts (time, device_id, tag_id, value) SELECT to_timestamp(g), 'dev' || g, (random() /10)+1, g FROM generate_series(0,30) g; --one in the future INSERT INTO hyper_ts (time, device_id, tag_id, value) VALUES ('2100-01-01 02:03:04 PST', 'dev101', 1, 0); --time partitioning function CREATE OR REPLACE FUNCTION unix_to_timestamp(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT AS $BODY$ SELECT to_timestamp(unixtime); $BODY$; CREATE TABLE hyper_timefunc ("time" float8 NOT NULL, "device_id" text, "value" integer); SELECT create_hypertable('hyper_timefunc', 'time', 'device_id', 4, chunk_time_interval => 10, time_partitioning_func => 'unix_to_timestamp'); psql:include/plan_expand_hypertable_load.sql:57: WARNING: unexpected interval: smaller than one second create_hypertable ----------------------------- (4,public,hyper_timefunc,t) INSERT INTO hyper_timefunc (time, device_id, value) SELECT g, 'dev' || g, g FROM generate_series(0,30) g; CREATE TABLE metrics_timestamp(time timestamp); SELECT create_hypertable('metrics_timestamp','time'); psql:include/plan_expand_hypertable_load.sql:62: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------------- (5,public,metrics_timestamp,t) INSERT INTO metrics_timestamp SELECT generate_series('2000-01-01'::timestamp,'2000-02-01'::timestamp,'1d'::interval); CREATE TABLE metrics_timestamptz(time timestamptz, device_id int); SELECT create_hypertable('metrics_timestamptz','time'); create_hypertable ---------------------------------- (6,public,metrics_timestamptz,t) INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval), 1; INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval), 2; INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval), 3; --create a second table to test joins with CREATE TABLE metrics_timestamptz_2 (LIKE metrics_timestamptz); SELECT create_hypertable('metrics_timestamptz_2','time'); create_hypertable ------------------------------------ (7,public,metrics_timestamptz_2,t) INSERT INTO metrics_timestamptz_2 SELECT * FROM metrics_timestamptz; INSERT INTO metrics_timestamptz_2 VALUES ('2000-12-01'::timestamptz, 3); CREATE TABLE metrics_date(time date); SELECT create_hypertable('metrics_date','time'); create_hypertable --------------------------- (8,public,metrics_date,t) INSERT INTO metrics_date SELECT generate_series('2000-01-01'::date,'2000-02-01'::date,'1d'::interval); ANALYZE hyper; ANALYZE hyper_w_space; ANALYZE tag; ANALYZE hyper_ts; ANALYZE hyper_timefunc; -- create normal table for JOIN tests CREATE TABLE regular_timestamptz(time timestamptz); INSERT INTO regular_timestamptz SELECT generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval); \ir include/plan_expand_hypertable_query.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --we want to see how our logic excludes chunks --and not how much work constraint_exclusion does SET constraint_exclusion = 'off'; \qecho test upper bounds test upper bounds :PREFIX SELECT * FROM hyper WHERE time < 10 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_1_chunk.value -> Seq Scan on _hyper_1_1_chunk :PREFIX SELECT * FROM hyper WHERE time < 11 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk -> Seq Scan on _hyper_1_2_chunk Filter: ("time" < 11) :PREFIX SELECT * FROM hyper WHERE time = 10 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk.value -> Seq Scan on _hyper_1_2_chunk Filter: ("time" = 10) :PREFIX SELECT * FROM hyper WHERE 10 >= time ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk -> Seq Scan on _hyper_1_2_chunk Filter: (10 >= "time") \qecho test lower bounds test lower bounds :PREFIX SELECT * FROM hyper WHERE time >= 10 and time < 20 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk.value -> Seq Scan on _hyper_1_2_chunk :PREFIX SELECT * FROM hyper WHERE 10 < time and 20 >= time ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_2_chunk Filter: ((10 < "time") AND (20 >= "time")) -> Seq Scan on _hyper_1_3_chunk Filter: ((10 < "time") AND (20 >= "time")) :PREFIX SELECT * FROM hyper WHERE time >= 9 and time < 20 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 9) AND ("time" < 20)) -> Seq Scan on _hyper_1_2_chunk :PREFIX SELECT * FROM hyper WHERE time > 9 and time < 20 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk.value -> Seq Scan on _hyper_1_2_chunk \qecho test empty result test empty result :PREFIX SELECT * FROM hyper WHERE time < 0; --- QUERY PLAN --- Result One-Time Filter: false \qecho test expression evaluation test expression evaluation :PREFIX SELECT * FROM hyper WHERE time < (5*2)::smallint; --- QUERY PLAN --- Seq Scan on _hyper_1_1_chunk \qecho test logic at INT64_MAX test logic at INT64_MAX :PREFIX SELECT * FROM hyper WHERE time = 9223372036854775807::bigint ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_102_chunk.value -> Seq Scan on _hyper_1_102_chunk Filter: ("time" = '9223372036854775807'::bigint) :PREFIX SELECT * FROM hyper WHERE time = 9223372036854775806::bigint ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_102_chunk.value -> Seq Scan on _hyper_1_102_chunk Filter: ("time" = '9223372036854775806'::bigint) :PREFIX SELECT * FROM hyper WHERE time >= 9223372036854775807::bigint ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_102_chunk.value -> Seq Scan on _hyper_1_102_chunk Filter: ("time" >= '9223372036854775807'::bigint) :PREFIX SELECT * FROM hyper WHERE time > 9223372036854775807::bigint ORDER BY value; --- QUERY PLAN --- Sort Sort Key: value -> Result One-Time Filter: false :PREFIX SELECT * FROM hyper WHERE time > 9223372036854775806::bigint ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_102_chunk.value -> Seq Scan on _hyper_1_102_chunk Filter: ("time" > '9223372036854775806'::bigint) \qecho cte cte :PREFIX WITH cte AS( SELECT * FROM hyper WHERE time < 10 ) SELECT * FROM cte ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_1_chunk.value -> Seq Scan on _hyper_1_1_chunk \qecho subquery subquery :PREFIX SELECT 0 = ANY (SELECT value FROM hyper WHERE time < 10); --- QUERY PLAN --- Result SubPlan 1 -> Seq Scan on _hyper_1_1_chunk \qecho no space constraint no space constraint :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" < 10) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" < 10) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" < 10) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" < 10) \qecho valid space constraint valid space constraint :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 and device_id = 'dev5' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 and 'dev5' = device_id ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND ('dev5'::text = device_id)) :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 and 'dev'||(2+3) = device_id ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND ('dev5'::text = device_id)) \qecho only space constraint only space constraint :PREFIX SELECT * FROM hyper_w_space WHERE 'dev5' = device_id ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_106_chunk Filter: ('dev5'::text = device_id) -> Seq Scan on _hyper_2_109_chunk Filter: ('dev5'::text = device_id) -> Seq Scan on _hyper_2_111_chunk Filter: ('dev5'::text = device_id) \qecho unhandled space constraint unhandled space constraint :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 and device_id > 'dev5' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: (("time" < 10) AND (device_id > 'dev5'::text)) -> Seq Scan on _hyper_2_104_chunk Filter: (("time" < 10) AND (device_id > 'dev5'::text)) -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND (device_id > 'dev5'::text)) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id > 'dev5'::text)) \qecho use of OR - does not filter chunks use of OR - does not filter chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND (device_id = 'dev5' or device_id = 'dev6') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: (("time" < 10) AND ((device_id = 'dev5'::text) OR (device_id = 'dev6'::text))) -> Seq Scan on _hyper_2_104_chunk Filter: (("time" < 10) AND ((device_id = 'dev5'::text) OR (device_id = 'dev6'::text))) -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND ((device_id = 'dev5'::text) OR (device_id = 'dev6'::text))) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND ((device_id = 'dev5'::text) OR (device_id = 'dev6'::text))) \qecho cte cte :PREFIX WITH cte AS( SELECT * FROM hyper_w_space WHERE time < 10 and device_id = 'dev5' ) SELECT * FROM cte ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) \qecho subquery subquery :PREFIX SELECT 0 = ANY (SELECT value FROM hyper_w_space WHERE time < 10 and device_id = 'dev5'); --- QUERY PLAN --- Result SubPlan 1 -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) \qecho view view :PREFIX SELECT * FROM hyper_w_space_view WHERE time < 10 and device_id = 'dev5' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) \qecho IN statement - simple IN statement - simple :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id IN ('dev5') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) \qecho IN statement - two chunks IN statement - two chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id IN ('dev5','dev6') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev5,dev6}'::text[]))) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev5,dev6}'::text[]))) \qecho IN statement - one chunk IN statement - one chunk :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id IN ('dev4','dev5') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev4,dev5}'::text[]))) \qecho NOT IN - does not filter chunks NOT IN - does not filter chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id NOT IN ('dev5','dev6') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: (("time" < 10) AND (device_id <> ALL ('{dev5,dev6}'::text[]))) -> Seq Scan on _hyper_2_104_chunk Filter: (("time" < 10) AND (device_id <> ALL ('{dev5,dev6}'::text[]))) -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND (device_id <> ALL ('{dev5,dev6}'::text[]))) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id <> ALL ('{dev5,dev6}'::text[]))) \qecho IN statement with subquery - does not filter chunks IN statement with subquery - does not filter chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id IN (SELECT 'dev5'::text) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) \qecho ANY ANY :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id = ANY(ARRAY['dev5','dev6']) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev5,dev6}'::text[]))) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev5,dev6}'::text[]))) \qecho ANY with intersection ANY with intersection :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id = ANY(ARRAY['dev5','dev6']) AND device_id = ANY(ARRAY['dev6','dev7']) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_105_chunk.value -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev5,dev6}'::text[])) AND (device_id = ANY ('{dev6,dev7}'::text[]))) \qecho ANY without intersection shouldnt scan any chunks ANY without intersection shouldnt scan any chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id = ANY(ARRAY['dev5','dev6']) AND device_id = ANY(ARRAY['dev8','dev9']) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: value -> Result One-Time Filter: false \qecho ANY/IN/ALL only works for equals operator ANY/IN/ALL only works for equals operator :PREFIX SELECT * FROM hyper_w_space WHERE device_id < ANY(ARRAY['dev5','dev6']) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_104_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_105_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_106_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_107_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_108_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_109_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_110_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_111_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_112_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_113_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_114_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_115_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) \qecho ALL with equals and different values shouldnt scan any chunks ALL with equals and different values shouldnt scan any chunks :PREFIX SELECT * FROM hyper_w_space WHERE device_id = ALL(ARRAY['dev5','dev6']) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: value -> Result One-Time Filter: false \qecho Multi AND Multi AND :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND time < 100 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: (("time" < 10) AND ("time" < 100)) -> Seq Scan on _hyper_2_104_chunk Filter: (("time" < 10) AND ("time" < 100)) -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND ("time" < 100)) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND ("time" < 100)) \qecho Time dimension doesnt filter chunks when using non-equality IN/ANY with multiple arguments Time dimension doesnt filter chunks when using non-equality IN/ANY with multiple arguments :PREFIX SELECT * FROM hyper_w_space WHERE time < ANY(ARRAY[1,2]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_107_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_108_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_109_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_110_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_111_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_112_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_113_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_114_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_115_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) \qecho Time dimension chunk exclusion with IN/ANY equality uses bounding range Time dimension chunk exclusion with IN/ANY equality uses bounding range :PREFIX SELECT * FROM hyper WHERE time IN (5, 15) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_1_2_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) :PREFIX SELECT * FROM hyper WHERE time = ANY(ARRAY[5, 15, 25]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ("time" = ANY ('{5,15,25}'::integer[])) -> Seq Scan on _hyper_1_2_chunk Filter: ("time" = ANY ('{5,15,25}'::integer[])) -> Seq Scan on _hyper_1_3_chunk Filter: ("time" = ANY ('{5,15,25}'::integer[])) :PREFIX SELECT * FROM hyper WHERE time = ANY(ARRAY[25, 15, 5]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ("time" = ANY ('{25,15,5}'::integer[])) -> Seq Scan on _hyper_1_2_chunk Filter: ("time" = ANY ('{25,15,5}'::integer[])) -> Seq Scan on _hyper_1_3_chunk Filter: ("time" = ANY ('{25,15,5}'::integer[])) :PREFIX SELECT * FROM hyper_w_space WHERE time IN (5, 15) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_107_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_108_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_109_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_110_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) :PREFIX SELECT * FROM metrics_timestamp WHERE time IN ('2000-01-05'::timestamp, '2000-01-15'::timestamp) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000","Sat Jan 15 00:00:00 2000"}'::timestamp without time zone[])) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000","Sat Jan 15 00:00:00 2000"}'::timestamp without time zone[])) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000","Sat Jan 15 00:00:00 2000"}'::timestamp without time zone[])) :PREFIX SELECT * FROM metrics_timestamptz WHERE time IN ('2000-01-05'::timestamptz, '2000-01-15'::timestamptz) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000 PST","Sat Jan 15 00:00:00 2000 PST"}'::timestamp with time zone[])) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000 PST","Sat Jan 15 00:00:00 2000 PST"}'::timestamp with time zone[])) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000 PST","Sat Jan 15 00:00:00 2000 PST"}'::timestamp with time zone[])) :PREFIX SELECT * FROM metrics_date WHERE time IN ('2000-01-05'::date, '2000-01-15'::date) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date Order: metrics_date."time" -> Index Only Scan Backward using _hyper_8_171_chunk_metrics_date_time_idx on _hyper_8_171_chunk Index Cond: ("time" = ANY ('{01-05-2000,01-15-2000}'::date[])) -> Index Only Scan Backward using _hyper_8_172_chunk_metrics_date_time_idx on _hyper_8_172_chunk Index Cond: ("time" = ANY ('{01-05-2000,01-15-2000}'::date[])) -> Index Only Scan Backward using _hyper_8_173_chunk_metrics_date_time_idx on _hyper_8_173_chunk Index Cond: ("time" = ANY ('{01-05-2000,01-15-2000}'::date[])) \qecho cross-type IN/ANY: timestamp to timestamptz column does not use bounding range (stable cast) cross-type IN/ANY: timestamp to timestamptz column does not use bounding range (stable cast) :PREFIX SELECT * FROM metrics_timestamptz WHERE time IN ('2000-01-05'::timestamp, '2000-01-15'::timestamp) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: ("time" = ANY (ARRAY[('Wed Jan 05 00:00:00 2000'::timestamp without time zone)::timestamp with time zone, ('Sat Jan 15 00:00:00 2000'::timestamp without time zone)::timestamp with time zone])) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Index Cond: ("time" = ANY (ARRAY[('Wed Jan 05 00:00:00 2000'::timestamp without time zone)::timestamp with time zone, ('Sat Jan 15 00:00:00 2000'::timestamp without time zone)::timestamp with time zone])) \qecho Time dimension chunk filtering works for ANY with single argument Time dimension chunk filtering works for ANY with single argument :PREFIX SELECT * FROM hyper_w_space WHERE time < ANY(ARRAY[1]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" < ANY ('{1}'::integer[])) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" < ANY ('{1}'::integer[])) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" < ANY ('{1}'::integer[])) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" < ANY ('{1}'::integer[])) \qecho Time dimension chunk filtering works for ALL with single argument Time dimension chunk filtering works for ALL with single argument :PREFIX SELECT * FROM hyper_w_space WHERE time < ALL(ARRAY[1]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" < ALL ('{1}'::integer[])) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" < ALL ('{1}'::integer[])) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" < ALL ('{1}'::integer[])) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" < ALL ('{1}'::integer[])) \qecho Time dimension chunk filtering works for ALL with multiple arguments Time dimension chunk filtering works for ALL with multiple arguments :PREFIX SELECT * FROM hyper_w_space WHERE time < ALL(ARRAY[1,10,20,30]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" < ALL ('{1,10,20,30}'::integer[])) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" < ALL ('{1,10,20,30}'::integer[])) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" < ALL ('{1,10,20,30}'::integer[])) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" < ALL ('{1,10,20,30}'::integer[])) \qecho AND intersection using IN and EQUALS AND intersection using IN and EQUALS :PREFIX SELECT * FROM hyper_w_space WHERE device_id IN ('dev1','dev2') AND device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ((device_id = ANY ('{dev1,dev2}'::text[])) AND (device_id = 'dev1'::text)) -> Seq Scan on _hyper_2_110_chunk Filter: ((device_id = ANY ('{dev1,dev2}'::text[])) AND (device_id = 'dev1'::text)) -> Seq Scan on _hyper_2_114_chunk Filter: ((device_id = ANY ('{dev1,dev2}'::text[])) AND (device_id = 'dev1'::text)) \qecho AND with no intersection using IN and EQUALS AND with no intersection using IN and EQUALS :PREFIX SELECT * FROM hyper_w_space WHERE device_id IN ('dev1','dev2') AND device_id = 'dev3' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: value -> Result One-Time Filter: false \qecho timestamps timestamps \qecho these should work since they are immutable functions these should work since they are immutable functions :PREFIX SELECT * FROM hyper_ts WHERE time < 'Wed Dec 31 16:00:10 1969 PST'::timestamptz ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Append -> Seq Scan on _hyper_3_116_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_3_117_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) :PREFIX SELECT * FROM hyper_ts WHERE time < to_timestamp(10) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Append -> Seq Scan on _hyper_3_116_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_3_117_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) :PREFIX SELECT * FROM hyper_ts WHERE time < 'Wed Dec 31 16:00:10 1969'::timestamp AT TIME ZONE 'PST' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Append -> Seq Scan on _hyper_3_116_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_3_117_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) :PREFIX SELECT * FROM hyper_ts WHERE time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_3_116_chunk.value -> Seq Scan on _hyper_3_116_chunk Filter: (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text)) \qecho these should not work since uses stable functions; these should not work since uses stable functions; :PREFIX SELECT * FROM hyper_ts WHERE time < 'Wed Dec 31 16:00:10 1969'::timestamp ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Custom Scan (ChunkAppend) on hyper_ts Chunks excluded during startup: 6 -> Seq Scan on _hyper_3_116_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969'::timestamp without time zone) -> Seq Scan on _hyper_3_117_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969'::timestamp without time zone) :PREFIX SELECT * FROM hyper_ts WHERE time < ('Wed Dec 31 16:00:10 1969'::timestamp::timestamptz) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Custom Scan (ChunkAppend) on hyper_ts Chunks excluded during startup: 6 -> Seq Scan on _hyper_3_116_chunk Filter: ("time" < ('Wed Dec 31 16:00:10 1969'::timestamp without time zone)::timestamp with time zone) -> Seq Scan on _hyper_3_117_chunk Filter: ("time" < ('Wed Dec 31 16:00:10 1969'::timestamp without time zone)::timestamp with time zone) :PREFIX SELECT * FROM hyper_ts WHERE NOW() < time ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Custom Scan (ChunkAppend) on hyper_ts Chunks excluded during startup: 7 -> Seq Scan on _hyper_3_123_chunk Filter: (now() < "time") \qecho joins joins :PREFIX SELECT * FROM hyper_ts WHERE tag_id IN (SELECT id FROM tag WHERE tag.id=1) and time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_3_116_chunk.value -> Nested Loop Semi Join -> Seq Scan on _hyper_3_116_chunk Filter: (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text) AND (tag_id = 1)) -> Seq Scan on tag Filter: (id = 1) :PREFIX SELECT * FROM hyper_ts WHERE tag_id IN (SELECT id FROM tag WHERE tag.id=1) or (time < to_timestamp(10) and device_id = 'dev1') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Custom Scan (ChunkAppend) on hyper_ts -> Seq Scan on _hyper_3_116_chunk Filter: ((hashed SubPlan 1) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) SubPlan 1 -> Seq Scan on tag Filter: (id = 1) -> Seq Scan on _hyper_3_117_chunk Filter: ((hashed SubPlan 1) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_118_chunk Filter: ((hashed SubPlan 1) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_119_chunk Filter: ((hashed SubPlan 1) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_120_chunk Filter: ((hashed SubPlan 1) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_121_chunk Filter: ((hashed SubPlan 1) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_122_chunk Filter: ((hashed SubPlan 1) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_123_chunk Filter: ((hashed SubPlan 1) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) :PREFIX SELECT * FROM hyper_ts WHERE tag_id IN (SELECT id FROM tag WHERE tag.name='tag1') and time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_3_116_chunk.value -> Nested Loop Join Filter: (_hyper_3_116_chunk.tag_id = tag.id) -> Seq Scan on _hyper_3_116_chunk Filter: (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text)) -> Seq Scan on tag Filter: (name = 'tag1'::text) :PREFIX SELECT * FROM hyper_ts JOIN tag on (hyper_ts.tag_id = tag.id ) WHERE time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_3_116_chunk.value -> Merge Join Merge Cond: (tag.id = _hyper_3_116_chunk.tag_id) -> Index Scan using tag_pkey on tag -> Sort Sort Key: _hyper_3_116_chunk.tag_id -> Seq Scan on _hyper_3_116_chunk Filter: (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text)) :PREFIX SELECT * FROM hyper_ts JOIN tag on (hyper_ts.tag_id = tag.id ) WHERE tag.name = 'tag1' and time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_3_116_chunk.value -> Nested Loop Join Filter: (_hyper_3_116_chunk.tag_id = tag.id) -> Seq Scan on _hyper_3_116_chunk Filter: (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text)) -> Seq Scan on tag Filter: (name = 'tag1'::text) \qecho test constraint exclusion for constraints in ON clause of JOINs test constraint exclusion for constraints in ON clause of JOINs \qecho should exclude chunks on m1 and propagate qual to m2 because of INNER JOIN should exclude chunks on m1 and propagate qual to m2 because of INNER JOIN :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho should exclude chunks on m2 and propagate qual to m1 because of INNER JOIN should exclude chunks on m2 and propagate qual to m1 because of INNER JOIN :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho must not exclude on m1 must not exclude on m1 :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Left Join Merge Cond: (m1."time" = m2."time") Join Filter: (m1."time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 \qecho should exclude chunks on m2 should exclude chunks on m2 :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Left Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho should exclude chunks on m1 should exclude chunks on m1 :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Sort Sort Key: m1."time" -> Merge Right Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 \qecho must not exclude chunks on m2 must not exclude chunks on m2 :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time < '2000-01-10' ORDER BY m1.time, m2.time; --- QUERY PLAN --- Sort Sort Key: m1."time", m2."time" -> Merge Left Join Merge Cond: (m2."time" = m1."time") Join Filter: (m2."time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 \qecho time_bucket exclusion time_bucket exclusion :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) < 10::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: _hyper_1_1_chunk."time" -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time") < '10'::bigint) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) < 11::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time") < '11'::bigint) -> Seq Scan on _hyper_1_2_chunk Filter: (time_bucket('10'::bigint, "time") < '11'::bigint) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" < '21'::bigint) AND (time_bucket('10'::bigint, "time") < '11'::bigint)) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) <= 10::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time") <= '10'::bigint) -> Seq Scan on _hyper_1_2_chunk Filter: (time_bucket('10'::bigint, "time") <= '10'::bigint) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" <= '20'::bigint) AND (time_bucket('10'::bigint, "time") <= '10'::bigint)) :PREFIX SELECT * FROM hyper WHERE 10::bigint > time_bucket(10, time) ORDER BY time; --- QUERY PLAN --- Sort Sort Key: _hyper_1_1_chunk."time" -> Seq Scan on _hyper_1_1_chunk Filter: ('10'::bigint > time_bucket('10'::bigint, "time")) :PREFIX SELECT * FROM hyper WHERE 11::bigint > time_bucket(10, time) ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ('11'::bigint > time_bucket('10'::bigint, "time")) -> Seq Scan on _hyper_1_2_chunk Filter: ('11'::bigint > time_bucket('10'::bigint, "time")) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" < '21'::bigint) AND ('11'::bigint > time_bucket('10'::bigint, "time"))) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time, 5) < 10::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) < '10'::bigint) -> Seq Scan on _hyper_1_2_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) < '10'::bigint) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time, 5) < 11::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) < '11'::bigint) -> Seq Scan on _hyper_1_2_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) < '11'::bigint) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" < '21'::bigint) AND (time_bucket('10'::bigint, "time", '5'::bigint) < '11'::bigint)) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time, 5) <= 10::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) <= '10'::bigint) -> Seq Scan on _hyper_1_2_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) <= '10'::bigint) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" <= '20'::bigint) AND (time_bucket('10'::bigint, "time", '5'::bigint) <= '10'::bigint)) :PREFIX SELECT * FROM hyper WHERE 10::bigint > time_bucket(10, time, 5) ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ('10'::bigint > time_bucket('10'::bigint, "time", '5'::bigint)) -> Seq Scan on _hyper_1_2_chunk Filter: ('10'::bigint > time_bucket('10'::bigint, "time", '5'::bigint)) :PREFIX SELECT * FROM hyper WHERE 11::bigint > time_bucket(10, time, 5) ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ('11'::bigint > time_bucket('10'::bigint, "time", '5'::bigint)) -> Seq Scan on _hyper_1_2_chunk Filter: ('11'::bigint > time_bucket('10'::bigint, "time", '5'::bigint)) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" < '21'::bigint) AND ('11'::bigint > time_bucket('10'::bigint, "time", '5'::bigint))) \qecho timestamp time_bucket exclusion timestamp time_bucket exclusion SELECT count(DISTINCT tableoid) FROM metrics_timestamp; count ------- 5 :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time") < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time") < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time") <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time") <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time") > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time") > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time) >= '2000-01-15' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Index Cond: ("time" >= 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time") >= 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket('@ 7 days'::interval, "time") >= 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time") >= 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'3d'::interval) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'3d'::interval) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'3d'::interval) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'3d'::interval) >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'2000-01-10'::timestamp) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'2000-01-10'::timestamp) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'2000-01-10'::timestamp) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'2000-01-10'::timestamp) >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) \qecho timestamptz time_bucket exclusion timestamptz time_bucket exclusion SELECT count(DISTINCT tableoid) FROM metrics_timestamptz; count ------- 5 :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time") < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time") < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time") <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time") <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time") > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time") > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time) >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time") >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time") >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'3d'::interval) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'3d'::interval) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'3d'::interval) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'3d'::interval) >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'2000-01-10'::timestamptz) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'2000-01-10'::timestamptz) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'2000-01-10'::timestamptz) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'2000-01-10'::timestamptz) >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'Europe/Berlin') < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'Europe/Berlin') <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'Europe/Berlin') > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'Europe/Berlin') >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) \qecho test overflow behaviour of time_bucket exclusion test overflow behaviour of time_bucket exclusion :PREFIX SELECT * FROM hyper WHERE time > 950 AND time_bucket(10, time) < '9223372036854775807'::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_96_chunk Filter: (("time" > 950) AND (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint)) -> Seq Scan on _hyper_1_97_chunk Filter: (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint) -> Seq Scan on _hyper_1_98_chunk Filter: (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint) -> Seq Scan on _hyper_1_99_chunk Filter: (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint) -> Seq Scan on _hyper_1_100_chunk Filter: (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint) -> Seq Scan on _hyper_1_101_chunk Filter: (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint) -> Seq Scan on _hyper_1_102_chunk Filter: (("time" > 950) AND (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint)) \qecho test timestamp upper boundary test timestamp upper boundary \qecho there should be no transformation if we are out of the supported (TimescaleDB-specific) range there should be no transformation if we are out of the supported (TimescaleDB-specific) range :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('1d',time) < '294276-01-01'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) \qecho transformation would be out of range transformation would be out of range :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('1000d',time) < '294276-01-01'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) \qecho test timestamptz upper boundary test timestamptz upper boundary \qecho there should be no transformation if we are out of the supported (TimescaleDB-specific) range there should be no transformation if we are out of the supported (TimescaleDB-specific) range :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1d',time) < '294276-01-01'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) \qecho transformation would be out of range transformation would be out of range :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1000d',time) < '294276-01-01'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) \qecho time_bucket exclusion with run-time constants time_bucket exclusion with run-time constants -- These queries have a stable time_bucket expression, because the text::interval conversion is stable. PREPARE P1(text , text) AS SELECT * FROM metrics_timestamptz WHERE time_bucket($1::interval, time) > $2::timestamptz ORDER BY time; PREPARE P2(text , text) AS SELECT * FROM metrics_timestamp WHERE time_bucket($1::interval, time) > $2::timestamptz ORDER BY time; PREPARE P3(text , text) AS SELECT * FROM metrics_timestamptz WHERE time_bucket($1::interval, time) > $2::timestamp ORDER BY time; PREPARE P4(text , text) AS SELECT * FROM metrics_timestamp WHERE time_bucket($1::interval, time) > $2::timestamp ORDER BY time; -- These queries have an immutable time_bucket expression, because the parameter is passed as interval, and no conversion is involved. PREPARE P5(interval, text) AS SELECT * FROM metrics_timestamptz WHERE time_bucket($1::interval, time) > $2::timestamptz ORDER BY time; PREPARE P6(interval, text) AS SELECT * FROM metrics_timestamp WHERE time_bucket($1::interval, time) > $2::timestamptz ORDER BY time; PREPARE P7(interval, text) AS SELECT * FROM metrics_timestamptz WHERE time_bucket($1::interval, time) > $2::timestamp ORDER BY time; PREPARE P8(interval, text) AS SELECT * FROM metrics_timestamp WHERE time_bucket($1::interval, time) > $2::timestamp ORDER BY time; SET plan_cache_mode TO 'force_custom_plan'; :PREFIX EXECUTE P1('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P2('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P3('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P4('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P5('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P6('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P7('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P8('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P1('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) :PREFIX EXECUTE P2('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) :PREFIX EXECUTE P3('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) :PREFIX EXECUTE P4('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) :PREFIX EXECUTE P5('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) :PREFIX EXECUTE P6('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) :PREFIX EXECUTE P7('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) :PREFIX EXECUTE P8('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) SET plan_cache_mode TO 'force_generic_plan'; :PREFIX EXECUTE P1('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P2('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P3('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P4('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P5('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P6('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P7('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P8('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P1('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) :PREFIX EXECUTE P2('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) :PREFIX EXECUTE P3('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) :PREFIX EXECUTE P4('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) :PREFIX EXECUTE P5('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) :PREFIX EXECUTE P6('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) :PREFIX EXECUTE P7('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) :PREFIX EXECUTE P8('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) RESET plan_cache_mode; DEALLOCATE P1; DEALLOCATE P2; DEALLOCATE P3; DEALLOCATE P4; DEALLOCATE P5; DEALLOCATE P6; DEALLOCATE P7; DEALLOCATE P8; :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) > 10 AND time_bucket(10, time) < 100 ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_2_chunk Filter: (("time" > '10'::bigint) AND ("time" < '100'::bigint) AND (time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_3_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_4_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_5_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_6_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_7_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_8_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_9_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_10_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) > 10 AND time_bucket(10, time) < 20 ORDER BY time; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk."time" -> Seq Scan on _hyper_1_2_chunk Filter: (("time" > '10'::bigint) AND ("time" < '20'::bigint) AND (time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 20)) :PREFIX SELECT * FROM hyper WHERE time_bucket(1, time) > 11 AND time_bucket(1, time) < 19 ORDER BY time; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk."time" -> Seq Scan on _hyper_1_2_chunk Filter: (("time" > '11'::bigint) AND ("time" < '19'::bigint) AND (time_bucket('1'::bigint, "time") > 11) AND (time_bucket('1'::bigint, "time") < 19)) :PREFIX SELECT * FROM hyper WHERE 10 < time_bucket(10, time) AND 20 > time_bucket(10,time) ORDER BY time; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk."time" -> Seq Scan on _hyper_1_2_chunk Filter: (("time" > '10'::bigint) AND ("time" < '20'::bigint) AND (10 < time_bucket('10'::bigint, "time")) AND (20 > time_bucket('10'::bigint, "time"))) \qecho time_bucket exclusion with date time_bucket exclusion with date :PREFIX SELECT * FROM metrics_date WHERE time_bucket('1d',time) < '2000-01-03' ORDER BY time; --- QUERY PLAN --- Index Only Scan Backward using _hyper_8_171_chunk_metrics_date_time_idx on _hyper_8_171_chunk Index Cond: ("time" < '01-03-2000'::date) Filter: (time_bucket('@ 1 day'::interval, "time") < '01-03-2000'::date) :PREFIX SELECT * FROM metrics_date WHERE time_bucket('1d',time) >= '2000-01-03' AND time_bucket('1d',time) <= '2000-01-10' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date Order: metrics_date."time" -> Index Only Scan Backward using _hyper_8_171_chunk_metrics_date_time_idx on _hyper_8_171_chunk Index Cond: (("time" >= '01-03-2000'::date) AND ("time" <= '01-11-2000'::date)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= '01-03-2000'::date) AND (time_bucket('@ 1 day'::interval, "time") <= '01-10-2000'::date)) -> Index Only Scan Backward using _hyper_8_172_chunk_metrics_date_time_idx on _hyper_8_172_chunk Index Cond: (("time" >= '01-03-2000'::date) AND ("time" <= '01-11-2000'::date)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= '01-03-2000'::date) AND (time_bucket('@ 1 day'::interval, "time") <= '01-10-2000'::date)) \qecho time_bucket exclusion with timestamp time_bucket exclusion with timestamp :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('1d',time) < '2000-01-03' ORDER BY time; --- QUERY PLAN --- Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Index Cond: ("time" < 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 1 day'::interval, "time") < 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('1d',time) >= '2000-01-03' AND time_bucket('1d',time) <= '2000-01-10' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) AND ("time" <= 'Tue Jan 11 00:00:00 2000'::timestamp without time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) AND (time_bucket('@ 1 day'::interval, "time") <= 'Mon Jan 10 00:00:00 2000'::timestamp without time zone)) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) AND ("time" <= 'Tue Jan 11 00:00:00 2000'::timestamp without time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) AND (time_bucket('@ 1 day'::interval, "time") <= 'Mon Jan 10 00:00:00 2000'::timestamp without time zone)) \qecho time_bucket exclusion with timestamptz time_bucket exclusion with timestamptz :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('6h',time) < '2000-01-03' ORDER BY time; --- QUERY PLAN --- Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: ("time" < 'Mon Jan 03 06:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 6 hours'::interval, "time") < 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('6h',time) >= '2000-01-03' AND time_bucket('6h',time) <= '2000-01-10' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Mon Jan 10 06:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 6 hours'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 6 hours'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Mon Jan 10 06:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 6 hours'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 6 hours'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho time_bucket exclusion with timestamptz and day interval time_bucket exclusion with timestamptz and day interval :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1d',time) < '2000-01-03' ORDER BY time; --- QUERY PLAN --- Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: ("time" < 'Tue Jan 04 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 1 day'::interval, "time") < 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1d',time) >= '2000-01-03' AND time_bucket('1d',time) <= '2000-01-10' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Tue Jan 11 00:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 1 day'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Tue Jan 11 00:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 1 day'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1d',time) >= '2000-01-03' AND time_bucket('7d',time) <= '2000-01-10' ORDER BY time; --- QUERY PLAN --- Sort Sort Key: metrics_timestamptz."time" -> Append -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Mon Jan 17 00:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 7 days'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Seq Scan on _hyper_6_161_chunk Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 7 days'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Mon Jan 17 00:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 7 days'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho no transformation no transformation :PREFIX SELECT * FROM hyper WHERE time_bucket(10 + floor(random())::int, time) > 10 AND time_bucket(10 + floor(random())::int, time) < 100 AND time < 150 ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Custom Scan (ChunkAppend) on hyper Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_2_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_3_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_4_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_5_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_6_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_7_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_8_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_9_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_10_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_11_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_12_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_13_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_14_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_15_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) \qecho exclude chunks based on time column with partitioning function. This exclude chunks based on time column with partitioning function. This \qecho transparently applies the time partitioning function on the time transparently applies the time partitioning function on the time \qecho value to be able to exclude chunks (similar to a closed dimension). value to be able to exclude chunks (similar to a closed dimension). :PREFIX SELECT * FROM hyper_timefunc WHERE time < 4 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_timefunc.value -> Append -> Seq Scan on _hyper_4_124_chunk Filter: ("time" < '4'::double precision) -> Seq Scan on _hyper_4_125_chunk Filter: ("time" < '4'::double precision) -> Seq Scan on _hyper_4_126_chunk Filter: ("time" < '4'::double precision) -> Seq Scan on _hyper_4_127_chunk Filter: ("time" < '4'::double precision) \qecho excluding based on time expression is currently unoptimized excluding based on time expression is currently unoptimized :PREFIX SELECT * FROM hyper_timefunc WHERE unix_to_timestamp(time) < 'Wed Dec 31 16:00:04 1969 PST' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_timefunc.value -> Append -> Seq Scan on _hyper_4_124_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_125_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_126_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_127_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_128_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_129_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_130_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_131_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_132_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_133_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_134_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_135_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_136_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_137_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_138_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_139_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_140_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_141_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_142_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_143_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_144_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_145_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_146_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_147_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_148_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_149_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_150_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_151_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_152_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_153_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_154_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) \qecho test qual propagation for joins test qual propagation for joins RESET constraint_exclusion; \qecho nothing to propagate nothing to propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1, metrics_timestamptz_2 m2 WHERE m1.time = m2.time ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time ORDER BY m1.time; --- QUERY PLAN --- Merge Left Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time ORDER BY m1.time; --- QUERY PLAN --- Sort Sort Key: m1."time" -> Merge Right Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 \qecho OR constraints should not propagate OR constraints should not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' OR m1.time > '2001-01-01' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Filter: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) OR ("time" > 'Mon Jan 01 00:00:00 2001 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Filter: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) OR ("time" > 'Mon Jan 01 00:00:00 2001 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 \qecho test single constraint test single constraint \qecho constraint should be on both scans constraint should be on both scans \qecho these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column :PREFIX SELECT m1.time FROM metrics_timestamptz m1, metrics_timestamptz_2 m2 WHERE m1.time = m2.time AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Left Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho test 2 constraints on single relation test 2 constraints on single relation \qecho these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column :PREFIX SELECT m1.time FROM metrics_timestamptz m1, metrics_timestamptz_2 m2 WHERE m1.time = m2.time AND m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Nested Loop Left Join -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Append -> Index Only Scan using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 Index Cond: ("time" = m1."time") :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho test 2 constraints with 1 constraint on each relation test 2 constraints with 1 constraint on each relation \qecho these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column :PREFIX SELECT m1.time FROM metrics_timestamptz m1, metrics_timestamptz_2 m2 WHERE m1.time = m2.time AND m1.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) \qecho test constraints in ON clause of INNER JOIN test constraints in ON clause of INNER JOIN :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho test constraints in ON clause of LEFT JOIN test constraints in ON clause of LEFT JOIN \qecho must not propagate must not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Left Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho test constraints in ON clause of RIGHT JOIN test constraints in ON clause of RIGHT JOIN \qecho must not propagate must not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Gather Merge Workers Planned: 2 -> Sort Sort Key: m1."time" -> Parallel Hash Left Join Hash Cond: (m2."time" = m1."time") Join Filter: ((m2."time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND (m2."time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Parallel Append -> Parallel Seq Scan on _hyper_7_165_chunk m2_1 -> Parallel Seq Scan on _hyper_7_166_chunk m2_2 -> Parallel Seq Scan on _hyper_7_167_chunk m2_3 -> Parallel Seq Scan on _hyper_7_168_chunk m2_4 -> Parallel Seq Scan on _hyper_7_169_chunk m2_5 -> Parallel Seq Scan on _hyper_7_170_chunk m2_6 -> Parallel Hash -> Parallel Append -> Parallel Seq Scan on _hyper_6_160_chunk m1_1 -> Parallel Seq Scan on _hyper_6_161_chunk m1_2 -> Parallel Seq Scan on _hyper_6_162_chunk m1_3 -> Parallel Seq Scan on _hyper_6_163_chunk m1_4 -> Parallel Seq Scan on _hyper_6_164_chunk m1_5 \qecho test equality condition not in ON clause test equality condition not in ON clause :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON true WHERE m2.time = m1.time AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho test constraints not joined on test constraints not joined on \qecho device_id constraint must not propagate device_id constraint must not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON true WHERE m2.time = m1.time AND m2.time < '2000-01-10' AND m1.device_id = 1 ORDER BY m1.time; --- QUERY PLAN --- Sort Sort Key: m1."time" -> Nested Loop -> Append -> Seq Scan on _hyper_6_160_chunk m1_1 Filter: (device_id = 1) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 1) -> Append -> Index Only Scan using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" = m1."time") AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho test multiple join conditions test multiple join conditions \qecho device_id constraint should propagate device_id constraint should propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON true WHERE m2.time = m1.time AND m1.device_id = m2.device_id AND m2.time < '2000-01-10' AND m1.device_id = 1 ORDER BY m1.time; --- QUERY PLAN --- Sort Sort Key: m1."time" -> Nested Loop -> Append -> Seq Scan on _hyper_6_160_chunk m1_1 Filter: (device_id = 1) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 1) -> Append -> Index Scan using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: ("time" = m1."time") Filter: (device_id = 1) -> Index Scan using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" = m1."time") AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) Filter: (device_id = 1) \qecho test join with 3 tables test join with 3 tables :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time INNER JOIN metrics_timestamptz m3 ON m2.time=m3.time WHERE m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Nested Loop -> Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Append -> Index Only Scan using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m3_1 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m3_2 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m3_3 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m3_4 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m3_5 Index Cond: ("time" = m1."time") \qecho test non-Const constraints test non-Const constraints :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10'::text::timestamptz ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" Chunks excluded during startup: 3 -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" Chunks excluded during startup: 4 -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) \qecho test now() test now() :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < now() ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 Index Cond: ("time" < now()) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 Index Cond: ("time" < now()) \qecho test volatile function test volatile function \qecho should not propagate should not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < clock_timestamp() ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 Filter: ("time" < clock_timestamp()) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m2.time < clock_timestamp() ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m2."time" = m1."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 Filter: ("time" < clock_timestamp()) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 \qecho test JOINs with normal table test JOINs with normal table \qecho will not propagate because constraints are only added to hypertables will not propagate because constraints are only added to hypertables :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN regular_timestamptz m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Sort Sort Key: m2."time" -> Seq Scan on regular_timestamptz m2 \qecho test JOINs with normal table test JOINs with normal table :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN regular_timestamptz m2 ON m1.time = m2.time WHERE m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m2."time" = m1."time") -> Sort Sort Key: m2."time" -> Seq Scan on regular_timestamptz m2 Filter: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho test quals are not pushed into OUTER JOIN test quals are not pushed into OUTER JOIN CREATE TABLE outer_join_1 (id int, name text,time timestamptz NOT NULL DEFAULT '2000-01-01'); CREATE TABLE outer_join_2 (id int, name text,time timestamptz NOT NULL DEFAULT '2000-01-01'); SELECT (SELECT table_name FROM create_hypertable(tbl, 'time')) FROM (VALUES ('outer_join_1'),('outer_join_2')) v(tbl); table_name -------------- outer_join_1 outer_join_2 INSERT INTO outer_join_1 VALUES(1,'a'), (2,'b'); INSERT INTO outer_join_2 VALUES(1,'a'); :PREFIX SELECT one.id, two.name FROM outer_join_1 one LEFT OUTER JOIN outer_join_2 two ON one.id=two.id WHERE one.id=2; --- QUERY PLAN --- Nested Loop Left Join Join Filter: (one.id = two.id) -> Seq Scan on _hyper_9_176_chunk one Filter: (id = 2) -> Materialize -> Seq Scan on _hyper_10_177_chunk two Filter: (id = 2) :PREFIX SELECT one.id, two.name FROM outer_join_2 two RIGHT OUTER JOIN outer_join_1 one ON one.id=two.id WHERE one.id=2; --- QUERY PLAN --- Nested Loop Left Join Join Filter: (one.id = two.id) -> Seq Scan on _hyper_9_176_chunk one Filter: (id = 2) -> Materialize -> Seq Scan on _hyper_10_177_chunk two Filter: (id = 2) DROP TABLE outer_join_1; DROP TABLE outer_join_2; -- test UNION between regular table and hypertable SELECT time FROM regular_timestamptz UNION SELECT time FROM metrics_timestamptz ORDER BY 1; time ------------------------------ Sat Jan 01 00:00:00 2000 PST Sun Jan 02 00:00:00 2000 PST Mon Jan 03 00:00:00 2000 PST Tue Jan 04 00:00:00 2000 PST Wed Jan 05 00:00:00 2000 PST Thu Jan 06 00:00:00 2000 PST Fri Jan 07 00:00:00 2000 PST Sat Jan 08 00:00:00 2000 PST Sun Jan 09 00:00:00 2000 PST Mon Jan 10 00:00:00 2000 PST Tue Jan 11 00:00:00 2000 PST Wed Jan 12 00:00:00 2000 PST Thu Jan 13 00:00:00 2000 PST Fri Jan 14 00:00:00 2000 PST Sat Jan 15 00:00:00 2000 PST Sun Jan 16 00:00:00 2000 PST Mon Jan 17 00:00:00 2000 PST Tue Jan 18 00:00:00 2000 PST Wed Jan 19 00:00:00 2000 PST Thu Jan 20 00:00:00 2000 PST Fri Jan 21 00:00:00 2000 PST Sat Jan 22 00:00:00 2000 PST Sun Jan 23 00:00:00 2000 PST Mon Jan 24 00:00:00 2000 PST Tue Jan 25 00:00:00 2000 PST Wed Jan 26 00:00:00 2000 PST Thu Jan 27 00:00:00 2000 PST Fri Jan 28 00:00:00 2000 PST Sat Jan 29 00:00:00 2000 PST Sun Jan 30 00:00:00 2000 PST Mon Jan 31 00:00:00 2000 PST Tue Feb 01 00:00:00 2000 PST -- test UNION ALL between regular table and hypertable SELECT time FROM regular_timestamptz UNION ALL SELECT time FROM metrics_timestamptz ORDER BY 1; time ------------------------------ Sat Jan 01 00:00:00 2000 PST Sat Jan 01 00:00:00 2000 PST Sat Jan 01 00:00:00 2000 PST Sat Jan 01 00:00:00 2000 PST Sun Jan 02 00:00:00 2000 PST Sun Jan 02 00:00:00 2000 PST Sun Jan 02 00:00:00 2000 PST Sun Jan 02 00:00:00 2000 PST Mon Jan 03 00:00:00 2000 PST Mon Jan 03 00:00:00 2000 PST Mon Jan 03 00:00:00 2000 PST Mon Jan 03 00:00:00 2000 PST Tue Jan 04 00:00:00 2000 PST Tue Jan 04 00:00:00 2000 PST Tue Jan 04 00:00:00 2000 PST Tue Jan 04 00:00:00 2000 PST Wed Jan 05 00:00:00 2000 PST Wed Jan 05 00:00:00 2000 PST Wed Jan 05 00:00:00 2000 PST Wed Jan 05 00:00:00 2000 PST Thu Jan 06 00:00:00 2000 PST Thu Jan 06 00:00:00 2000 PST Thu Jan 06 00:00:00 2000 PST Thu Jan 06 00:00:00 2000 PST Fri Jan 07 00:00:00 2000 PST Fri Jan 07 00:00:00 2000 PST Fri Jan 07 00:00:00 2000 PST Fri Jan 07 00:00:00 2000 PST Sat Jan 08 00:00:00 2000 PST Sat Jan 08 00:00:00 2000 PST Sat Jan 08 00:00:00 2000 PST Sat Jan 08 00:00:00 2000 PST Sun Jan 09 00:00:00 2000 PST Sun Jan 09 00:00:00 2000 PST Sun Jan 09 00:00:00 2000 PST Sun Jan 09 00:00:00 2000 PST Mon Jan 10 00:00:00 2000 PST Mon Jan 10 00:00:00 2000 PST Mon Jan 10 00:00:00 2000 PST Mon Jan 10 00:00:00 2000 PST Tue Jan 11 00:00:00 2000 PST Tue Jan 11 00:00:00 2000 PST Tue Jan 11 00:00:00 2000 PST Tue Jan 11 00:00:00 2000 PST Wed Jan 12 00:00:00 2000 PST Wed Jan 12 00:00:00 2000 PST Wed Jan 12 00:00:00 2000 PST Wed Jan 12 00:00:00 2000 PST Thu Jan 13 00:00:00 2000 PST Thu Jan 13 00:00:00 2000 PST Thu Jan 13 00:00:00 2000 PST Thu Jan 13 00:00:00 2000 PST Fri Jan 14 00:00:00 2000 PST Fri Jan 14 00:00:00 2000 PST Fri Jan 14 00:00:00 2000 PST Fri Jan 14 00:00:00 2000 PST Sat Jan 15 00:00:00 2000 PST Sat Jan 15 00:00:00 2000 PST Sat Jan 15 00:00:00 2000 PST Sat Jan 15 00:00:00 2000 PST Sun Jan 16 00:00:00 2000 PST Sun Jan 16 00:00:00 2000 PST Sun Jan 16 00:00:00 2000 PST Sun Jan 16 00:00:00 2000 PST Mon Jan 17 00:00:00 2000 PST Mon Jan 17 00:00:00 2000 PST Mon Jan 17 00:00:00 2000 PST Mon Jan 17 00:00:00 2000 PST Tue Jan 18 00:00:00 2000 PST Tue Jan 18 00:00:00 2000 PST Tue Jan 18 00:00:00 2000 PST Tue Jan 18 00:00:00 2000 PST Wed Jan 19 00:00:00 2000 PST Wed Jan 19 00:00:00 2000 PST Wed Jan 19 00:00:00 2000 PST Wed Jan 19 00:00:00 2000 PST Thu Jan 20 00:00:00 2000 PST Thu Jan 20 00:00:00 2000 PST Thu Jan 20 00:00:00 2000 PST Thu Jan 20 00:00:00 2000 PST Fri Jan 21 00:00:00 2000 PST Fri Jan 21 00:00:00 2000 PST Fri Jan 21 00:00:00 2000 PST Fri Jan 21 00:00:00 2000 PST Sat Jan 22 00:00:00 2000 PST Sat Jan 22 00:00:00 2000 PST Sat Jan 22 00:00:00 2000 PST Sat Jan 22 00:00:00 2000 PST Sun Jan 23 00:00:00 2000 PST Sun Jan 23 00:00:00 2000 PST Sun Jan 23 00:00:00 2000 PST Sun Jan 23 00:00:00 2000 PST Mon Jan 24 00:00:00 2000 PST Mon Jan 24 00:00:00 2000 PST Mon Jan 24 00:00:00 2000 PST Mon Jan 24 00:00:00 2000 PST Tue Jan 25 00:00:00 2000 PST Tue Jan 25 00:00:00 2000 PST Tue Jan 25 00:00:00 2000 PST Tue Jan 25 00:00:00 2000 PST Wed Jan 26 00:00:00 2000 PST Wed Jan 26 00:00:00 2000 PST Wed Jan 26 00:00:00 2000 PST Wed Jan 26 00:00:00 2000 PST Thu Jan 27 00:00:00 2000 PST Thu Jan 27 00:00:00 2000 PST Thu Jan 27 00:00:00 2000 PST Thu Jan 27 00:00:00 2000 PST Fri Jan 28 00:00:00 2000 PST Fri Jan 28 00:00:00 2000 PST Fri Jan 28 00:00:00 2000 PST Fri Jan 28 00:00:00 2000 PST Sat Jan 29 00:00:00 2000 PST Sat Jan 29 00:00:00 2000 PST Sat Jan 29 00:00:00 2000 PST Sat Jan 29 00:00:00 2000 PST Sun Jan 30 00:00:00 2000 PST Sun Jan 30 00:00:00 2000 PST Sun Jan 30 00:00:00 2000 PST Sun Jan 30 00:00:00 2000 PST Mon Jan 31 00:00:00 2000 PST Mon Jan 31 00:00:00 2000 PST Mon Jan 31 00:00:00 2000 PST Mon Jan 31 00:00:00 2000 PST Tue Feb 01 00:00:00 2000 PST Tue Feb 01 00:00:00 2000 PST Tue Feb 01 00:00:00 2000 PST Tue Feb 01 00:00:00 2000 PST -- test nested join qual propagation :PREFIX SELECT * FROM ( SELECT o1_m1.time FROM metrics_timestamptz o1_m1 INNER JOIN metrics_timestamptz_2 o1_m2 ON true WHERE o1_m2.time = o1_m1.time AND o1_m1.device_id = o1_m2.device_id AND o1_m2.time < '2000-01-10' AND o1_m1.device_id = 1 ) o1 FULL OUTER JOIN ( SELECT o2_m1.time FROM metrics_timestamptz o2_m1 FULL OUTER JOIN metrics_timestamptz_2 o2_m2 ON true WHERE o2_m2.time = o2_m1.time AND o2_m1.device_id = o2_m2.device_id AND o2_m2.time > '2000-01-20' AND o2_m1.device_id = 2 ) o2 ON o1.time = o2.time ORDER BY 1,2; --- QUERY PLAN --- Sort Sort Key: o1_m1."time", o2_m1."time" -> Hash Full Join Hash Cond: (o2_m1."time" = o1_m1."time") -> Nested Loop -> Append -> Index Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk o2_m2_1 Index Cond: ("time" > 'Thu Jan 20 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 2) -> Seq Scan on _hyper_7_169_chunk o2_m2_2 Filter: (device_id = 2) -> Seq Scan on _hyper_7_170_chunk o2_m2_3 Filter: (device_id = 2) -> Append -> Index Scan using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk o2_m1_1 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk o2_m1_2 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk o2_m1_3 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk o2_m1_4 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk o2_m1_5 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Hash -> Nested Loop -> Append -> Seq Scan on _hyper_7_165_chunk o1_m2_1 Filter: (device_id = 1) -> Index Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk o1_m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 1) -> Append -> Index Scan using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk o1_m1_1 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk o1_m1_2 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk o1_m1_3 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk o1_m1_4 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk o1_m1_5 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) :PREFIX SELECT * FROM ( SELECT o1_m1.time FROM metrics_timestamptz o1_m1 INNER JOIN metrics_timestamptz_2 o1_m2 ON o1_m2.time = o1_m1.time AND o1_m1.device_id = o1_m2.device_id WHERE o1_m2.time < '2000-01-10' AND o1_m1.device_id = 1 ) o1 FULL OUTER JOIN ( SELECT o2_m1.time FROM metrics_timestamptz o2_m1 FULL OUTER JOIN metrics_timestamptz_2 o2_m2 ON o2_m2.time = o2_m1.time AND o2_m1.device_id = o2_m2.device_id WHERE o2_m2.time > '2000-01-20' AND o2_m1.device_id = 2 ) o2 ON o1.time = o2.time ORDER BY 1,2; --- QUERY PLAN --- Sort Sort Key: o1_m1."time", o2_m1."time" -> Hash Full Join Hash Cond: (o2_m1."time" = o1_m1."time") -> Nested Loop -> Append -> Index Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk o2_m2_1 Index Cond: ("time" > 'Thu Jan 20 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 2) -> Seq Scan on _hyper_7_169_chunk o2_m2_2 Filter: (device_id = 2) -> Seq Scan on _hyper_7_170_chunk o2_m2_3 Filter: (device_id = 2) -> Append -> Index Scan using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk o2_m1_1 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk o2_m1_2 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk o2_m1_3 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk o2_m1_4 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk o2_m1_5 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Hash -> Nested Loop -> Append -> Seq Scan on _hyper_7_165_chunk o1_m2_1 Filter: (device_id = 1) -> Index Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk o1_m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 1) -> Append -> Index Scan using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk o1_m1_1 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk o1_m1_2 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk o1_m1_3 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk o1_m1_4 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk o1_m1_5 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) \set ECHO errors RESET timescaledb.enable_optimizations; CREATE TABLE t(time timestamptz NOT NULL); SELECT table_name FROM create_hypertable('t','time'); table_name ------------ t INSERT INTO t VALUES ('2000-01-01'), ('2010-01-01'), ('2020-01-01'); EXPLAIN (buffers off, costs off) SELECT * FROM t t1 INNER JOIN t t2 ON t1.time = t2.time WHERE t1.time < timestamptz '2010-01-01'; --- QUERY PLAN --- Merge Join Merge Cond: (t1."time" = t2."time") -> Merge Append Sort Key: t1."time" -> Index Only Scan Backward using _hyper_15_182_chunk_t_time_idx on _hyper_15_182_chunk t1_1 -> Index Only Scan Backward using _hyper_15_183_chunk_t_time_idx on _hyper_15_183_chunk t1_2 Index Cond: ("time" < 'Fri Jan 01 00:00:00 2010 PST'::timestamp with time zone) -> Materialize -> Merge Append Sort Key: t2."time" -> Index Only Scan Backward using _hyper_15_182_chunk_t_time_idx on _hyper_15_182_chunk t2_1 -> Index Only Scan Backward using _hyper_15_183_chunk_t_time_idx on _hyper_15_183_chunk t2_2 Index Cond: ("time" < 'Fri Jan 01 00:00:00 2010 PST'::timestamp with time zone) SET timescaledb.enable_qual_propagation TO false; EXPLAIN (buffers off, costs off) SELECT * FROM t t1 INNER JOIN t t2 ON t1.time = t2.time WHERE t1.time < timestamptz '2010-01-01'; --- QUERY PLAN --- Merge Join Merge Cond: (t1."time" = t2."time") -> Merge Append Sort Key: t1."time" -> Index Only Scan Backward using _hyper_15_182_chunk_t_time_idx on _hyper_15_182_chunk t1_1 -> Index Only Scan Backward using _hyper_15_183_chunk_t_time_idx on _hyper_15_183_chunk t1_2 Index Cond: ("time" < 'Fri Jan 01 00:00:00 2010 PST'::timestamp with time zone) -> Materialize -> Merge Append Sort Key: t2."time" -> Index Only Scan Backward using _hyper_15_182_chunk_t_time_idx on _hyper_15_182_chunk t2_1 -> Index Only Scan Backward using _hyper_15_183_chunk_t_time_idx on _hyper_15_183_chunk t2_2 -> Index Only Scan Backward using _hyper_15_184_chunk_t_time_idx on _hyper_15_184_chunk t2_3 RESET timescaledb.enable_qual_propagation; CREATE TABLE test (a int, time timestamptz NOT NULL); SELECT table_name FROM create_hypertable('public.test', 'time'); table_name ------------ test INSERT INTO test SELECT i, '2020-04-01'::date-10-i from generate_series(1,20) i; CREATE OR REPLACE FUNCTION test_f(_ts timestamptz) RETURNS SETOF test LANGUAGE SQL STABLE PARALLEL SAFE AS $f$ SELECT DISTINCT ON (a) * FROM test WHERE time >= _ts ORDER BY a, time DESC $f$; EXPLAIN (buffers off, costs off) SELECT * FROM test_f(now()); --- QUERY PLAN --- Unique -> Sort Sort Key: test.a, test."time" DESC -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 4 EXPLAIN (buffers off, costs off) SELECT * FROM test_f(now()); --- QUERY PLAN --- Unique -> Sort Sort Key: test.a, test."time" DESC -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 4 CREATE TABLE t1 (a int, b int NOT NULL); SELECT create_hypertable('t1', 'b', chunk_time_interval=>10); create_hypertable ------------------- (17,public,t1,t) CREATE TABLE t2 (a int, b int NOT NULL); SELECT create_hypertable('t2', 'b', chunk_time_interval=>10); create_hypertable ------------------- (18,public,t2,t) CREATE OR REPLACE FUNCTION f_t1(_a int, _b int) RETURNS SETOF t1 LANGUAGE SQL STABLE PARALLEL SAFE AS $function$ SELECT DISTINCT ON (a) * FROM t1 WHERE a = _a and b = _b ORDER BY a, b DESC $function$ ; CREATE OR REPLACE FUNCTION f_t2(_a int, _b int) RETURNS SETOF t2 LANGUAGE sql STABLE PARALLEL SAFE AS $function$ SELECT DISTINCT ON (j.a) j.* FROM f_t1(_a, _b) sc, t2 j WHERE j.b = _b AND j.a = _a ORDER BY j.a, j.b DESC $function$ ; CREATE OR REPLACE FUNCTION f_t1_2(_b int) RETURNS SETOF t1 LANGUAGE SQL STABLE PARALLEL SAFE AS $function$ SELECT DISTINCT ON (j.a) jt.* FROM t1 j, f_t1(j.a, _b) jt $function$; EXPLAIN (buffers off, costs off) SELECT * FROM f_t1_2(10); --- QUERY PLAN --- Subquery Scan on f_t1_2 -> Unique -> Sort Sort Key: j.a -> Nested Loop -> Seq Scan on t1 j -> Unique -> Index Scan using t1_b_idx on t1 Index Cond: (b = 10) Filter: (a = j.a) EXPLAIN (buffers off, costs off) SELECT * FROM f_t1_2(10) sc, f_t2(sc.a, 10); --- QUERY PLAN --- Nested Loop -> Unique -> Sort Sort Key: j.a -> Nested Loop -> Seq Scan on t1 j -> Unique -> Index Scan using t1_b_idx on t1 Index Cond: (b = 10) Filter: (a = j.a) -> Unique -> Nested Loop -> Unique -> Index Scan using t1_b_idx on t1 t1_1 Index Cond: (b = 10) Filter: (a = t1.a) -> Index Scan using t2_b_idx on t2 j_1 Index Cond: (b = 10) Filter: (a = t1.a) CREATE TABLE metrics_int1(time int, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval=1); INSERT INTO metrics_int1 SELECT i, i::text, i FROM generate_series(3,7) i; SELECT tableoid::regclass, time FROM metrics_int1 ORDER BY time; tableoid | time -------------------------------------------+------ _timescaledb_internal._hyper_19_189_chunk | 3 _timescaledb_internal._hyper_19_190_chunk | 4 _timescaledb_internal._hyper_19_191_chunk | 5 _timescaledb_internal._hyper_19_192_chunk | 6 _timescaledb_internal._hyper_19_193_chunk | 7 EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time >= 2; --- QUERY PLAN --- Append (actual rows=5.00 loops=1) -> Seq Scan on _hyper_19_189_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_192_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_193_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time >= 1 AND time >= 2; --- QUERY PLAN --- Append (actual rows=5.00 loops=1) -> Seq Scan on _hyper_19_189_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_192_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_193_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time >= 4; --- QUERY PLAN --- Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_192_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_193_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time > 4 AND time < 6; --- QUERY PLAN --- Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time > 4 AND time <= 6; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_192_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time >= 4 AND time < 6; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time >= 4 AND time <= 6; --- QUERY PLAN --- Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_192_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time BETWEEN 4 AND 5; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time = 5; --- QUERY PLAN --- Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) SET TIMEZONE='UTC'; CREATE TABLE metrics_tstz(time timestamptz, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval='1day'); INSERT INTO metrics_tstz SELECT '2000-01-01'::timestamptz + format('%s day',i)::interval, i::text, i FROM generate_series(2,6) i; SELECT tableoid::regclass, time FROM metrics_tstz ORDER BY time; tableoid | time -------------------------------------------+------------------------------ _timescaledb_internal._hyper_20_194_chunk | Mon Jan 03 00:00:00 2000 UTC _timescaledb_internal._hyper_20_195_chunk | Tue Jan 04 00:00:00 2000 UTC _timescaledb_internal._hyper_20_196_chunk | Wed Jan 05 00:00:00 2000 UTC _timescaledb_internal._hyper_20_197_chunk | Thu Jan 06 00:00:00 2000 UTC _timescaledb_internal._hyper_20_198_chunk | Fri Jan 07 00:00:00 2000 UTC EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-02'; --- QUERY PLAN --- Append (actual rows=5.00 loops=1) -> Seq Scan on _hyper_20_194_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_197_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_198_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-01' AND time >= '2000-01-02'; --- QUERY PLAN --- Append (actual rows=5.00 loops=1) -> Seq Scan on _hyper_20_194_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_197_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_198_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04'; --- QUERY PLAN --- Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_197_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_198_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time > '2000-01-04' AND time < '2000-01-06'; --- QUERY PLAN --- Append (actual rows=1.00 loops=1) -> Index Scan using _hyper_20_195_chunk_metrics_tstz_time_idx on _hyper_20_195_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" < 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time > '2000-01-04' AND time <= '2000-01-06'; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Index Scan using _hyper_20_195_chunk_metrics_tstz_time_idx on _hyper_20_195_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_20_197_chunk_metrics_tstz_time_idx on _hyper_20_197_chunk (actual rows=1.00 loops=1) Index Cond: (("time" > 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time < '2000-01-06'; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time <= '2000-01-06'; --- QUERY PLAN --- Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_20_197_chunk_metrics_tstz_time_idx on _hyper_20_197_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time BETWEEN '2000-01-04' AND '2000-01-05'; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_20_196_chunk_metrics_tstz_time_idx on _hyper_20_196_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Wed Jan 05 00:00:00 2000 UTC'::timestamp with time zone)) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time = '2000-01-05'; --- QUERY PLAN --- Index Scan using _hyper_20_196_chunk_metrics_tstz_time_idx on _hyper_20_196_chunk (actual rows=1.00 loops=1) Index Cond: ("time" = 'Wed Jan 05 00:00:00 2000 UTC'::timestamp with time zone) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time <= '2000-01-06' AND device = '5'; --- QUERY PLAN --- Append (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=0.00 loops=1) Filter: (device = '5'::text) Rows Removed by Filter: 1 -> Seq Scan on _hyper_20_196_chunk (actual rows=0.00 loops=1) Filter: (device = '5'::text) Rows Removed by Filter: 1 -> Index Scan using _hyper_20_197_chunk_metrics_tstz_time_idx on _hyper_20_197_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) Filter: (device = '5'::text) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time <= '2000-01-06' AND device IS NOT NULL; --- QUERY PLAN --- Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) Filter: (device IS NOT NULL) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) Filter: (device IS NOT NULL) -> Index Scan using _hyper_20_197_chunk_metrics_tstz_time_idx on _hyper_20_197_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) Filter: (device IS NOT NULL) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time <= '2000-01-06' AND time < now(); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_tstz (actual rows=3.00 loops=1) Chunks excluded during startup: 0 -> Index Scan using _hyper_20_195_chunk_metrics_tstz_time_idx on _hyper_20_195_chunk (actual rows=1.00 loops=1) Index Cond: ("time" < now()) -> Index Scan using _hyper_20_196_chunk_metrics_tstz_time_idx on _hyper_20_196_chunk (actual rows=1.00 loops=1) Index Cond: ("time" < now()) -> Index Scan using _hyper_20_197_chunk_metrics_tstz_time_idx on _hyper_20_197_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" < now())) CREATE TABLE metrics_space(time timestamptz, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval='1day'); SELECT add_dimension('metrics_space', 'device', 4); add_dimension ------------------------------------ (25,public,metrics_space,device,t) INSERT INTO metrics_space SELECT '2000-01-01'::timestamptz + format('%s day',i)::interval, i::text, i FROM generate_series(2,6) i; EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_space WHERE time >= '2000-01-02'; --- QUERY PLAN --- Append (actual rows=5.00 loops=1) -> Index Scan using _hyper_21_199_chunk_metrics_space_time_idx on _hyper_21_199_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Sun Jan 02 00:00:00 2000 UTC'::timestamp with time zone) -> Index Scan using _hyper_21_200_chunk_metrics_space_time_idx on _hyper_21_200_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Sun Jan 02 00:00:00 2000 UTC'::timestamp with time zone) -> Index Scan using _hyper_21_201_chunk_metrics_space_time_idx on _hyper_21_201_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Sun Jan 02 00:00:00 2000 UTC'::timestamp with time zone) -> Index Scan using _hyper_21_202_chunk_metrics_space_time_idx on _hyper_21_202_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Sun Jan 02 00:00:00 2000 UTC'::timestamp with time zone) -> Index Scan using _hyper_21_203_chunk_metrics_space_time_idx on _hyper_21_203_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Sun Jan 02 00:00:00 2000 UTC'::timestamp with time zone) --TEST END-- ================================================ FILE: test/expected/plan_expand_hypertable-16.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set PREFIX 'EXPLAIN (buffers off, costs off) ' \ir include/plan_expand_hypertable_load.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --single time dimension CREATE TABLE hyper ("time_broken" bigint NOT NULL, "value" integer); ALTER TABLE hyper DROP COLUMN time_broken, ADD COLUMN time BIGINT; SELECT create_hypertable('hyper', 'time', chunk_time_interval => 10); create_hypertable -------------------- (1,public,hyper,t) INSERT INTO hyper SELECT g, g FROM generate_series(0,1000) g; --insert a point with INT_MAX_64 INSERT INTO hyper (time, value) SELECT 9223372036854775807::bigint, 0; --time and space CREATE TABLE hyper_w_space ("time_broken" bigint NOT NULL, "device_id" text, "value" integer); ALTER TABLE hyper_w_space DROP COLUMN time_broken, ADD COLUMN time BIGINT; SELECT create_hypertable('hyper_w_space', 'time', 'device_id', 4, chunk_time_interval => 10); create_hypertable ---------------------------- (2,public,hyper_w_space,t) INSERT INTO hyper_w_space (time, device_id, value) SELECT g, 'dev' || g, g FROM generate_series(0,30) g; CREATE VIEW hyper_w_space_view AS (SELECT * FROM hyper_w_space); --with timestamp and space CREATE TABLE tag (id serial PRIMARY KEY, name text); CREATE TABLE hyper_ts ("time_broken" timestamptz NOT NULL, "device_id" text, tag_id INT REFERENCES tag(id), "value" integer); ALTER TABLE hyper_ts DROP COLUMN time_broken, ADD COLUMN time TIMESTAMPTZ; SELECT create_hypertable('hyper_ts', 'time', 'device_id', 2, chunk_time_interval => '10 seconds'::interval); create_hypertable ----------------------- (3,public,hyper_ts,t) INSERT INTO tag(name) SELECT 'tag'||g FROM generate_series(0,10) g; INSERT INTO hyper_ts (time, device_id, tag_id, value) SELECT to_timestamp(g), 'dev' || g, (random() /10)+1, g FROM generate_series(0,30) g; --one in the future INSERT INTO hyper_ts (time, device_id, tag_id, value) VALUES ('2100-01-01 02:03:04 PST', 'dev101', 1, 0); --time partitioning function CREATE OR REPLACE FUNCTION unix_to_timestamp(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT AS $BODY$ SELECT to_timestamp(unixtime); $BODY$; CREATE TABLE hyper_timefunc ("time" float8 NOT NULL, "device_id" text, "value" integer); SELECT create_hypertable('hyper_timefunc', 'time', 'device_id', 4, chunk_time_interval => 10, time_partitioning_func => 'unix_to_timestamp'); psql:include/plan_expand_hypertable_load.sql:57: WARNING: unexpected interval: smaller than one second create_hypertable ----------------------------- (4,public,hyper_timefunc,t) INSERT INTO hyper_timefunc (time, device_id, value) SELECT g, 'dev' || g, g FROM generate_series(0,30) g; CREATE TABLE metrics_timestamp(time timestamp); SELECT create_hypertable('metrics_timestamp','time'); psql:include/plan_expand_hypertable_load.sql:62: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------------- (5,public,metrics_timestamp,t) INSERT INTO metrics_timestamp SELECT generate_series('2000-01-01'::timestamp,'2000-02-01'::timestamp,'1d'::interval); CREATE TABLE metrics_timestamptz(time timestamptz, device_id int); SELECT create_hypertable('metrics_timestamptz','time'); create_hypertable ---------------------------------- (6,public,metrics_timestamptz,t) INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval), 1; INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval), 2; INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval), 3; --create a second table to test joins with CREATE TABLE metrics_timestamptz_2 (LIKE metrics_timestamptz); SELECT create_hypertable('metrics_timestamptz_2','time'); create_hypertable ------------------------------------ (7,public,metrics_timestamptz_2,t) INSERT INTO metrics_timestamptz_2 SELECT * FROM metrics_timestamptz; INSERT INTO metrics_timestamptz_2 VALUES ('2000-12-01'::timestamptz, 3); CREATE TABLE metrics_date(time date); SELECT create_hypertable('metrics_date','time'); create_hypertable --------------------------- (8,public,metrics_date,t) INSERT INTO metrics_date SELECT generate_series('2000-01-01'::date,'2000-02-01'::date,'1d'::interval); ANALYZE hyper; ANALYZE hyper_w_space; ANALYZE tag; ANALYZE hyper_ts; ANALYZE hyper_timefunc; -- create normal table for JOIN tests CREATE TABLE regular_timestamptz(time timestamptz); INSERT INTO regular_timestamptz SELECT generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval); \ir include/plan_expand_hypertable_query.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --we want to see how our logic excludes chunks --and not how much work constraint_exclusion does SET constraint_exclusion = 'off'; \qecho test upper bounds test upper bounds :PREFIX SELECT * FROM hyper WHERE time < 10 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_1_chunk.value -> Seq Scan on _hyper_1_1_chunk :PREFIX SELECT * FROM hyper WHERE time < 11 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk -> Seq Scan on _hyper_1_2_chunk Filter: ("time" < 11) :PREFIX SELECT * FROM hyper WHERE time = 10 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk.value -> Seq Scan on _hyper_1_2_chunk Filter: ("time" = 10) :PREFIX SELECT * FROM hyper WHERE 10 >= time ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk -> Seq Scan on _hyper_1_2_chunk Filter: (10 >= "time") \qecho test lower bounds test lower bounds :PREFIX SELECT * FROM hyper WHERE time >= 10 and time < 20 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk.value -> Seq Scan on _hyper_1_2_chunk :PREFIX SELECT * FROM hyper WHERE 10 < time and 20 >= time ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_2_chunk Filter: ((10 < "time") AND (20 >= "time")) -> Seq Scan on _hyper_1_3_chunk Filter: ((10 < "time") AND (20 >= "time")) :PREFIX SELECT * FROM hyper WHERE time >= 9 and time < 20 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 9) AND ("time" < 20)) -> Seq Scan on _hyper_1_2_chunk :PREFIX SELECT * FROM hyper WHERE time > 9 and time < 20 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk.value -> Seq Scan on _hyper_1_2_chunk \qecho test empty result test empty result :PREFIX SELECT * FROM hyper WHERE time < 0; --- QUERY PLAN --- Result One-Time Filter: false \qecho test expression evaluation test expression evaluation :PREFIX SELECT * FROM hyper WHERE time < (5*2)::smallint; --- QUERY PLAN --- Seq Scan on _hyper_1_1_chunk \qecho test logic at INT64_MAX test logic at INT64_MAX :PREFIX SELECT * FROM hyper WHERE time = 9223372036854775807::bigint ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_102_chunk.value -> Seq Scan on _hyper_1_102_chunk Filter: ("time" = '9223372036854775807'::bigint) :PREFIX SELECT * FROM hyper WHERE time = 9223372036854775806::bigint ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_102_chunk.value -> Seq Scan on _hyper_1_102_chunk Filter: ("time" = '9223372036854775806'::bigint) :PREFIX SELECT * FROM hyper WHERE time >= 9223372036854775807::bigint ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_102_chunk.value -> Seq Scan on _hyper_1_102_chunk Filter: ("time" >= '9223372036854775807'::bigint) :PREFIX SELECT * FROM hyper WHERE time > 9223372036854775807::bigint ORDER BY value; --- QUERY PLAN --- Sort Sort Key: value -> Result One-Time Filter: false :PREFIX SELECT * FROM hyper WHERE time > 9223372036854775806::bigint ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_102_chunk.value -> Seq Scan on _hyper_1_102_chunk Filter: ("time" > '9223372036854775806'::bigint) \qecho cte cte :PREFIX WITH cte AS( SELECT * FROM hyper WHERE time < 10 ) SELECT * FROM cte ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_1_chunk.value -> Seq Scan on _hyper_1_1_chunk \qecho subquery subquery :PREFIX SELECT 0 = ANY (SELECT value FROM hyper WHERE time < 10); --- QUERY PLAN --- Result SubPlan 1 -> Seq Scan on _hyper_1_1_chunk \qecho no space constraint no space constraint :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" < 10) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" < 10) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" < 10) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" < 10) \qecho valid space constraint valid space constraint :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 and device_id = 'dev5' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 and 'dev5' = device_id ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND ('dev5'::text = device_id)) :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 and 'dev'||(2+3) = device_id ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND ('dev5'::text = device_id)) \qecho only space constraint only space constraint :PREFIX SELECT * FROM hyper_w_space WHERE 'dev5' = device_id ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_106_chunk Filter: ('dev5'::text = device_id) -> Seq Scan on _hyper_2_109_chunk Filter: ('dev5'::text = device_id) -> Seq Scan on _hyper_2_111_chunk Filter: ('dev5'::text = device_id) \qecho unhandled space constraint unhandled space constraint :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 and device_id > 'dev5' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: (("time" < 10) AND (device_id > 'dev5'::text)) -> Seq Scan on _hyper_2_104_chunk Filter: (("time" < 10) AND (device_id > 'dev5'::text)) -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND (device_id > 'dev5'::text)) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id > 'dev5'::text)) \qecho use of OR - does not filter chunks use of OR - does not filter chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND (device_id = 'dev5' or device_id = 'dev6') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: (("time" < 10) AND ((device_id = 'dev5'::text) OR (device_id = 'dev6'::text))) -> Seq Scan on _hyper_2_104_chunk Filter: (("time" < 10) AND ((device_id = 'dev5'::text) OR (device_id = 'dev6'::text))) -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND ((device_id = 'dev5'::text) OR (device_id = 'dev6'::text))) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND ((device_id = 'dev5'::text) OR (device_id = 'dev6'::text))) \qecho cte cte :PREFIX WITH cte AS( SELECT * FROM hyper_w_space WHERE time < 10 and device_id = 'dev5' ) SELECT * FROM cte ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) \qecho subquery subquery :PREFIX SELECT 0 = ANY (SELECT value FROM hyper_w_space WHERE time < 10 and device_id = 'dev5'); --- QUERY PLAN --- Result SubPlan 1 -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) \qecho view view :PREFIX SELECT * FROM hyper_w_space_view WHERE time < 10 and device_id = 'dev5' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) \qecho IN statement - simple IN statement - simple :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id IN ('dev5') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) \qecho IN statement - two chunks IN statement - two chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id IN ('dev5','dev6') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev5,dev6}'::text[]))) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev5,dev6}'::text[]))) \qecho IN statement - one chunk IN statement - one chunk :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id IN ('dev4','dev5') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev4,dev5}'::text[]))) \qecho NOT IN - does not filter chunks NOT IN - does not filter chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id NOT IN ('dev5','dev6') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: (("time" < 10) AND (device_id <> ALL ('{dev5,dev6}'::text[]))) -> Seq Scan on _hyper_2_104_chunk Filter: (("time" < 10) AND (device_id <> ALL ('{dev5,dev6}'::text[]))) -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND (device_id <> ALL ('{dev5,dev6}'::text[]))) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id <> ALL ('{dev5,dev6}'::text[]))) \qecho IN statement with subquery - does not filter chunks IN statement with subquery - does not filter chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id IN (SELECT 'dev5'::text) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) \qecho ANY ANY :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id = ANY(ARRAY['dev5','dev6']) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev5,dev6}'::text[]))) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev5,dev6}'::text[]))) \qecho ANY with intersection ANY with intersection :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id = ANY(ARRAY['dev5','dev6']) AND device_id = ANY(ARRAY['dev6','dev7']) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_105_chunk.value -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev5,dev6}'::text[])) AND (device_id = ANY ('{dev6,dev7}'::text[]))) \qecho ANY without intersection shouldnt scan any chunks ANY without intersection shouldnt scan any chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id = ANY(ARRAY['dev5','dev6']) AND device_id = ANY(ARRAY['dev8','dev9']) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: value -> Result One-Time Filter: false \qecho ANY/IN/ALL only works for equals operator ANY/IN/ALL only works for equals operator :PREFIX SELECT * FROM hyper_w_space WHERE device_id < ANY(ARRAY['dev5','dev6']) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_104_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_105_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_106_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_107_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_108_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_109_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_110_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_111_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_112_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_113_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_114_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_115_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) \qecho ALL with equals and different values shouldnt scan any chunks ALL with equals and different values shouldnt scan any chunks :PREFIX SELECT * FROM hyper_w_space WHERE device_id = ALL(ARRAY['dev5','dev6']) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: value -> Result One-Time Filter: false \qecho Multi AND Multi AND :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND time < 100 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: (("time" < 10) AND ("time" < 100)) -> Seq Scan on _hyper_2_104_chunk Filter: (("time" < 10) AND ("time" < 100)) -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND ("time" < 100)) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND ("time" < 100)) \qecho Time dimension doesnt filter chunks when using non-equality IN/ANY with multiple arguments Time dimension doesnt filter chunks when using non-equality IN/ANY with multiple arguments :PREFIX SELECT * FROM hyper_w_space WHERE time < ANY(ARRAY[1,2]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_107_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_108_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_109_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_110_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_111_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_112_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_113_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_114_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_115_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) \qecho Time dimension chunk exclusion with IN/ANY equality uses bounding range Time dimension chunk exclusion with IN/ANY equality uses bounding range :PREFIX SELECT * FROM hyper WHERE time IN (5, 15) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_1_2_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) :PREFIX SELECT * FROM hyper WHERE time = ANY(ARRAY[5, 15, 25]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ("time" = ANY ('{5,15,25}'::integer[])) -> Seq Scan on _hyper_1_2_chunk Filter: ("time" = ANY ('{5,15,25}'::integer[])) -> Seq Scan on _hyper_1_3_chunk Filter: ("time" = ANY ('{5,15,25}'::integer[])) :PREFIX SELECT * FROM hyper WHERE time = ANY(ARRAY[25, 15, 5]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ("time" = ANY ('{25,15,5}'::integer[])) -> Seq Scan on _hyper_1_2_chunk Filter: ("time" = ANY ('{25,15,5}'::integer[])) -> Seq Scan on _hyper_1_3_chunk Filter: ("time" = ANY ('{25,15,5}'::integer[])) :PREFIX SELECT * FROM hyper_w_space WHERE time IN (5, 15) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_107_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_108_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_109_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_110_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) :PREFIX SELECT * FROM metrics_timestamp WHERE time IN ('2000-01-05'::timestamp, '2000-01-15'::timestamp) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000","Sat Jan 15 00:00:00 2000"}'::timestamp without time zone[])) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000","Sat Jan 15 00:00:00 2000"}'::timestamp without time zone[])) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000","Sat Jan 15 00:00:00 2000"}'::timestamp without time zone[])) :PREFIX SELECT * FROM metrics_timestamptz WHERE time IN ('2000-01-05'::timestamptz, '2000-01-15'::timestamptz) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000 PST","Sat Jan 15 00:00:00 2000 PST"}'::timestamp with time zone[])) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000 PST","Sat Jan 15 00:00:00 2000 PST"}'::timestamp with time zone[])) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000 PST","Sat Jan 15 00:00:00 2000 PST"}'::timestamp with time zone[])) :PREFIX SELECT * FROM metrics_date WHERE time IN ('2000-01-05'::date, '2000-01-15'::date) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date Order: metrics_date."time" -> Index Only Scan Backward using _hyper_8_171_chunk_metrics_date_time_idx on _hyper_8_171_chunk Index Cond: ("time" = ANY ('{01-05-2000,01-15-2000}'::date[])) -> Index Only Scan Backward using _hyper_8_172_chunk_metrics_date_time_idx on _hyper_8_172_chunk Index Cond: ("time" = ANY ('{01-05-2000,01-15-2000}'::date[])) -> Index Only Scan Backward using _hyper_8_173_chunk_metrics_date_time_idx on _hyper_8_173_chunk Index Cond: ("time" = ANY ('{01-05-2000,01-15-2000}'::date[])) \qecho cross-type IN/ANY: timestamp to timestamptz column does not use bounding range (stable cast) cross-type IN/ANY: timestamp to timestamptz column does not use bounding range (stable cast) :PREFIX SELECT * FROM metrics_timestamptz WHERE time IN ('2000-01-05'::timestamp, '2000-01-15'::timestamp) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: ("time" = ANY (ARRAY[('Wed Jan 05 00:00:00 2000'::timestamp without time zone)::timestamp with time zone, ('Sat Jan 15 00:00:00 2000'::timestamp without time zone)::timestamp with time zone])) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Index Cond: ("time" = ANY (ARRAY[('Wed Jan 05 00:00:00 2000'::timestamp without time zone)::timestamp with time zone, ('Sat Jan 15 00:00:00 2000'::timestamp without time zone)::timestamp with time zone])) \qecho Time dimension chunk filtering works for ANY with single argument Time dimension chunk filtering works for ANY with single argument :PREFIX SELECT * FROM hyper_w_space WHERE time < ANY(ARRAY[1]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" < ANY ('{1}'::integer[])) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" < ANY ('{1}'::integer[])) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" < ANY ('{1}'::integer[])) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" < ANY ('{1}'::integer[])) \qecho Time dimension chunk filtering works for ALL with single argument Time dimension chunk filtering works for ALL with single argument :PREFIX SELECT * FROM hyper_w_space WHERE time < ALL(ARRAY[1]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" < ALL ('{1}'::integer[])) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" < ALL ('{1}'::integer[])) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" < ALL ('{1}'::integer[])) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" < ALL ('{1}'::integer[])) \qecho Time dimension chunk filtering works for ALL with multiple arguments Time dimension chunk filtering works for ALL with multiple arguments :PREFIX SELECT * FROM hyper_w_space WHERE time < ALL(ARRAY[1,10,20,30]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" < ALL ('{1,10,20,30}'::integer[])) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" < ALL ('{1,10,20,30}'::integer[])) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" < ALL ('{1,10,20,30}'::integer[])) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" < ALL ('{1,10,20,30}'::integer[])) \qecho AND intersection using IN and EQUALS AND intersection using IN and EQUALS :PREFIX SELECT * FROM hyper_w_space WHERE device_id IN ('dev1','dev2') AND device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ((device_id = ANY ('{dev1,dev2}'::text[])) AND (device_id = 'dev1'::text)) -> Seq Scan on _hyper_2_110_chunk Filter: ((device_id = ANY ('{dev1,dev2}'::text[])) AND (device_id = 'dev1'::text)) -> Seq Scan on _hyper_2_114_chunk Filter: ((device_id = ANY ('{dev1,dev2}'::text[])) AND (device_id = 'dev1'::text)) \qecho AND with no intersection using IN and EQUALS AND with no intersection using IN and EQUALS :PREFIX SELECT * FROM hyper_w_space WHERE device_id IN ('dev1','dev2') AND device_id = 'dev3' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: value -> Result One-Time Filter: false \qecho timestamps timestamps \qecho these should work since they are immutable functions these should work since they are immutable functions :PREFIX SELECT * FROM hyper_ts WHERE time < 'Wed Dec 31 16:00:10 1969 PST'::timestamptz ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Append -> Seq Scan on _hyper_3_116_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_3_117_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) :PREFIX SELECT * FROM hyper_ts WHERE time < to_timestamp(10) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Append -> Seq Scan on _hyper_3_116_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_3_117_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) :PREFIX SELECT * FROM hyper_ts WHERE time < 'Wed Dec 31 16:00:10 1969'::timestamp AT TIME ZONE 'PST' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Append -> Seq Scan on _hyper_3_116_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_3_117_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) :PREFIX SELECT * FROM hyper_ts WHERE time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_3_116_chunk.value -> Seq Scan on _hyper_3_116_chunk Filter: (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text)) \qecho these should not work since uses stable functions; these should not work since uses stable functions; :PREFIX SELECT * FROM hyper_ts WHERE time < 'Wed Dec 31 16:00:10 1969'::timestamp ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Custom Scan (ChunkAppend) on hyper_ts Chunks excluded during startup: 6 -> Seq Scan on _hyper_3_116_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969'::timestamp without time zone) -> Seq Scan on _hyper_3_117_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969'::timestamp without time zone) :PREFIX SELECT * FROM hyper_ts WHERE time < ('Wed Dec 31 16:00:10 1969'::timestamp::timestamptz) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Custom Scan (ChunkAppend) on hyper_ts Chunks excluded during startup: 6 -> Seq Scan on _hyper_3_116_chunk Filter: ("time" < ('Wed Dec 31 16:00:10 1969'::timestamp without time zone)::timestamp with time zone) -> Seq Scan on _hyper_3_117_chunk Filter: ("time" < ('Wed Dec 31 16:00:10 1969'::timestamp without time zone)::timestamp with time zone) :PREFIX SELECT * FROM hyper_ts WHERE NOW() < time ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Custom Scan (ChunkAppend) on hyper_ts Chunks excluded during startup: 7 -> Seq Scan on _hyper_3_123_chunk Filter: (now() < "time") \qecho joins joins :PREFIX SELECT * FROM hyper_ts WHERE tag_id IN (SELECT id FROM tag WHERE tag.id=1) and time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_3_116_chunk.value -> Nested Loop Semi Join -> Seq Scan on _hyper_3_116_chunk Filter: (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text) AND (tag_id = 1)) -> Seq Scan on tag Filter: (id = 1) :PREFIX SELECT * FROM hyper_ts WHERE tag_id IN (SELECT id FROM tag WHERE tag.id=1) or (time < to_timestamp(10) and device_id = 'dev1') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Custom Scan (ChunkAppend) on hyper_ts -> Seq Scan on _hyper_3_116_chunk Filter: ((hashed SubPlan 1) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) SubPlan 1 -> Seq Scan on tag Filter: (id = 1) -> Seq Scan on _hyper_3_117_chunk Filter: ((hashed SubPlan 1) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_118_chunk Filter: ((hashed SubPlan 1) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_119_chunk Filter: ((hashed SubPlan 1) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_120_chunk Filter: ((hashed SubPlan 1) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_121_chunk Filter: ((hashed SubPlan 1) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_122_chunk Filter: ((hashed SubPlan 1) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_123_chunk Filter: ((hashed SubPlan 1) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) :PREFIX SELECT * FROM hyper_ts WHERE tag_id IN (SELECT id FROM tag WHERE tag.name='tag1') and time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_3_116_chunk.value -> Nested Loop Join Filter: (_hyper_3_116_chunk.tag_id = tag.id) -> Seq Scan on _hyper_3_116_chunk Filter: (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text)) -> Seq Scan on tag Filter: (name = 'tag1'::text) :PREFIX SELECT * FROM hyper_ts JOIN tag on (hyper_ts.tag_id = tag.id ) WHERE time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_3_116_chunk.value -> Merge Join Merge Cond: (tag.id = _hyper_3_116_chunk.tag_id) -> Index Scan using tag_pkey on tag -> Sort Sort Key: _hyper_3_116_chunk.tag_id -> Seq Scan on _hyper_3_116_chunk Filter: (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text)) :PREFIX SELECT * FROM hyper_ts JOIN tag on (hyper_ts.tag_id = tag.id ) WHERE tag.name = 'tag1' and time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_3_116_chunk.value -> Nested Loop Join Filter: (tag.id = _hyper_3_116_chunk.tag_id) -> Seq Scan on _hyper_3_116_chunk Filter: (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text)) -> Seq Scan on tag Filter: (name = 'tag1'::text) \qecho test constraint exclusion for constraints in ON clause of JOINs test constraint exclusion for constraints in ON clause of JOINs \qecho should exclude chunks on m1 and propagate qual to m2 because of INNER JOIN should exclude chunks on m1 and propagate qual to m2 because of INNER JOIN :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho should exclude chunks on m2 and propagate qual to m1 because of INNER JOIN should exclude chunks on m2 and propagate qual to m1 because of INNER JOIN :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho must not exclude on m1 must not exclude on m1 :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Left Join Merge Cond: (m1."time" = m2."time") Join Filter: (m1."time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 \qecho should exclude chunks on m2 should exclude chunks on m2 :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Left Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho should exclude chunks on m1 should exclude chunks on m1 :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Sort Sort Key: m1."time" -> Merge Right Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 \qecho must not exclude chunks on m2 must not exclude chunks on m2 :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time < '2000-01-10' ORDER BY m1.time, m2.time; --- QUERY PLAN --- Sort Sort Key: m1."time", m2."time" -> Merge Left Join Merge Cond: (m2."time" = m1."time") Join Filter: (m2."time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 \qecho time_bucket exclusion time_bucket exclusion :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) < 10::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: _hyper_1_1_chunk."time" -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time") < '10'::bigint) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) < 11::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time") < '11'::bigint) -> Seq Scan on _hyper_1_2_chunk Filter: (time_bucket('10'::bigint, "time") < '11'::bigint) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" < '21'::bigint) AND (time_bucket('10'::bigint, "time") < '11'::bigint)) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) <= 10::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time") <= '10'::bigint) -> Seq Scan on _hyper_1_2_chunk Filter: (time_bucket('10'::bigint, "time") <= '10'::bigint) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" <= '20'::bigint) AND (time_bucket('10'::bigint, "time") <= '10'::bigint)) :PREFIX SELECT * FROM hyper WHERE 10::bigint > time_bucket(10, time) ORDER BY time; --- QUERY PLAN --- Sort Sort Key: _hyper_1_1_chunk."time" -> Seq Scan on _hyper_1_1_chunk Filter: ('10'::bigint > time_bucket('10'::bigint, "time")) :PREFIX SELECT * FROM hyper WHERE 11::bigint > time_bucket(10, time) ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ('11'::bigint > time_bucket('10'::bigint, "time")) -> Seq Scan on _hyper_1_2_chunk Filter: ('11'::bigint > time_bucket('10'::bigint, "time")) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" < '21'::bigint) AND ('11'::bigint > time_bucket('10'::bigint, "time"))) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time, 5) < 10::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) < '10'::bigint) -> Seq Scan on _hyper_1_2_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) < '10'::bigint) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time, 5) < 11::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) < '11'::bigint) -> Seq Scan on _hyper_1_2_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) < '11'::bigint) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" < '21'::bigint) AND (time_bucket('10'::bigint, "time", '5'::bigint) < '11'::bigint)) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time, 5) <= 10::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) <= '10'::bigint) -> Seq Scan on _hyper_1_2_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) <= '10'::bigint) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" <= '20'::bigint) AND (time_bucket('10'::bigint, "time", '5'::bigint) <= '10'::bigint)) :PREFIX SELECT * FROM hyper WHERE 10::bigint > time_bucket(10, time, 5) ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ('10'::bigint > time_bucket('10'::bigint, "time", '5'::bigint)) -> Seq Scan on _hyper_1_2_chunk Filter: ('10'::bigint > time_bucket('10'::bigint, "time", '5'::bigint)) :PREFIX SELECT * FROM hyper WHERE 11::bigint > time_bucket(10, time, 5) ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ('11'::bigint > time_bucket('10'::bigint, "time", '5'::bigint)) -> Seq Scan on _hyper_1_2_chunk Filter: ('11'::bigint > time_bucket('10'::bigint, "time", '5'::bigint)) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" < '21'::bigint) AND ('11'::bigint > time_bucket('10'::bigint, "time", '5'::bigint))) \qecho timestamp time_bucket exclusion timestamp time_bucket exclusion SELECT count(DISTINCT tableoid) FROM metrics_timestamp; count ------- 5 :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time") < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time") < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time") <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time") <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time") > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time") > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time) >= '2000-01-15' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Index Cond: ("time" >= 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time") >= 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket('@ 7 days'::interval, "time") >= 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time") >= 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'3d'::interval) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'3d'::interval) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'3d'::interval) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'3d'::interval) >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'2000-01-10'::timestamp) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'2000-01-10'::timestamp) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'2000-01-10'::timestamp) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'2000-01-10'::timestamp) >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) \qecho timestamptz time_bucket exclusion timestamptz time_bucket exclusion SELECT count(DISTINCT tableoid) FROM metrics_timestamptz; count ------- 5 :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time") < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time") < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time") <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time") <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time") > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time") > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time) >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time") >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time") >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'3d'::interval) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'3d'::interval) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'3d'::interval) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'3d'::interval) >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'2000-01-10'::timestamptz) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'2000-01-10'::timestamptz) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'2000-01-10'::timestamptz) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'2000-01-10'::timestamptz) >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'Europe/Berlin') < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'Europe/Berlin') <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'Europe/Berlin') > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'Europe/Berlin') >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) \qecho test overflow behaviour of time_bucket exclusion test overflow behaviour of time_bucket exclusion :PREFIX SELECT * FROM hyper WHERE time > 950 AND time_bucket(10, time) < '9223372036854775807'::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_96_chunk Filter: (("time" > 950) AND (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint)) -> Seq Scan on _hyper_1_97_chunk Filter: (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint) -> Seq Scan on _hyper_1_98_chunk Filter: (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint) -> Seq Scan on _hyper_1_99_chunk Filter: (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint) -> Seq Scan on _hyper_1_100_chunk Filter: (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint) -> Seq Scan on _hyper_1_101_chunk Filter: (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint) -> Seq Scan on _hyper_1_102_chunk Filter: (("time" > 950) AND (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint)) \qecho test timestamp upper boundary test timestamp upper boundary \qecho there should be no transformation if we are out of the supported (TimescaleDB-specific) range there should be no transformation if we are out of the supported (TimescaleDB-specific) range :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('1d',time) < '294276-01-01'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) \qecho transformation would be out of range transformation would be out of range :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('1000d',time) < '294276-01-01'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) \qecho test timestamptz upper boundary test timestamptz upper boundary \qecho there should be no transformation if we are out of the supported (TimescaleDB-specific) range there should be no transformation if we are out of the supported (TimescaleDB-specific) range :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1d',time) < '294276-01-01'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) \qecho transformation would be out of range transformation would be out of range :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1000d',time) < '294276-01-01'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) \qecho time_bucket exclusion with run-time constants time_bucket exclusion with run-time constants -- These queries have a stable time_bucket expression, because the text::interval conversion is stable. PREPARE P1(text , text) AS SELECT * FROM metrics_timestamptz WHERE time_bucket($1::interval, time) > $2::timestamptz ORDER BY time; PREPARE P2(text , text) AS SELECT * FROM metrics_timestamp WHERE time_bucket($1::interval, time) > $2::timestamptz ORDER BY time; PREPARE P3(text , text) AS SELECT * FROM metrics_timestamptz WHERE time_bucket($1::interval, time) > $2::timestamp ORDER BY time; PREPARE P4(text , text) AS SELECT * FROM metrics_timestamp WHERE time_bucket($1::interval, time) > $2::timestamp ORDER BY time; -- These queries have an immutable time_bucket expression, because the parameter is passed as interval, and no conversion is involved. PREPARE P5(interval, text) AS SELECT * FROM metrics_timestamptz WHERE time_bucket($1::interval, time) > $2::timestamptz ORDER BY time; PREPARE P6(interval, text) AS SELECT * FROM metrics_timestamp WHERE time_bucket($1::interval, time) > $2::timestamptz ORDER BY time; PREPARE P7(interval, text) AS SELECT * FROM metrics_timestamptz WHERE time_bucket($1::interval, time) > $2::timestamp ORDER BY time; PREPARE P8(interval, text) AS SELECT * FROM metrics_timestamp WHERE time_bucket($1::interval, time) > $2::timestamp ORDER BY time; SET plan_cache_mode TO 'force_custom_plan'; :PREFIX EXECUTE P1('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P2('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P3('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P4('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P5('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P6('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P7('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P8('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P1('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) :PREFIX EXECUTE P2('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) :PREFIX EXECUTE P3('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) :PREFIX EXECUTE P4('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) :PREFIX EXECUTE P5('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) :PREFIX EXECUTE P6('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) :PREFIX EXECUTE P7('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) :PREFIX EXECUTE P8('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) SET plan_cache_mode TO 'force_generic_plan'; :PREFIX EXECUTE P1('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P2('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P3('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P4('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P5('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P6('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P7('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P8('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P1('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) :PREFIX EXECUTE P2('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) :PREFIX EXECUTE P3('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) :PREFIX EXECUTE P4('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) :PREFIX EXECUTE P5('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) :PREFIX EXECUTE P6('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) :PREFIX EXECUTE P7('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) :PREFIX EXECUTE P8('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) RESET plan_cache_mode; DEALLOCATE P1; DEALLOCATE P2; DEALLOCATE P3; DEALLOCATE P4; DEALLOCATE P5; DEALLOCATE P6; DEALLOCATE P7; DEALLOCATE P8; :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) > 10 AND time_bucket(10, time) < 100 ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_2_chunk Filter: (("time" > '10'::bigint) AND ("time" < '100'::bigint) AND (time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_3_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_4_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_5_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_6_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_7_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_8_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_9_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_10_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) > 10 AND time_bucket(10, time) < 20 ORDER BY time; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk."time" -> Seq Scan on _hyper_1_2_chunk Filter: (("time" > '10'::bigint) AND ("time" < '20'::bigint) AND (time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 20)) :PREFIX SELECT * FROM hyper WHERE time_bucket(1, time) > 11 AND time_bucket(1, time) < 19 ORDER BY time; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk."time" -> Seq Scan on _hyper_1_2_chunk Filter: (("time" > '11'::bigint) AND ("time" < '19'::bigint) AND (time_bucket('1'::bigint, "time") > 11) AND (time_bucket('1'::bigint, "time") < 19)) :PREFIX SELECT * FROM hyper WHERE 10 < time_bucket(10, time) AND 20 > time_bucket(10,time) ORDER BY time; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk."time" -> Seq Scan on _hyper_1_2_chunk Filter: (("time" > '10'::bigint) AND ("time" < '20'::bigint) AND (10 < time_bucket('10'::bigint, "time")) AND (20 > time_bucket('10'::bigint, "time"))) \qecho time_bucket exclusion with date time_bucket exclusion with date :PREFIX SELECT * FROM metrics_date WHERE time_bucket('1d',time) < '2000-01-03' ORDER BY time; --- QUERY PLAN --- Index Only Scan Backward using _hyper_8_171_chunk_metrics_date_time_idx on _hyper_8_171_chunk Index Cond: ("time" < '01-03-2000'::date) Filter: (time_bucket('@ 1 day'::interval, "time") < '01-03-2000'::date) :PREFIX SELECT * FROM metrics_date WHERE time_bucket('1d',time) >= '2000-01-03' AND time_bucket('1d',time) <= '2000-01-10' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date Order: metrics_date."time" -> Index Only Scan Backward using _hyper_8_171_chunk_metrics_date_time_idx on _hyper_8_171_chunk Index Cond: (("time" >= '01-03-2000'::date) AND ("time" <= '01-11-2000'::date)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= '01-03-2000'::date) AND (time_bucket('@ 1 day'::interval, "time") <= '01-10-2000'::date)) -> Index Only Scan Backward using _hyper_8_172_chunk_metrics_date_time_idx on _hyper_8_172_chunk Index Cond: (("time" >= '01-03-2000'::date) AND ("time" <= '01-11-2000'::date)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= '01-03-2000'::date) AND (time_bucket('@ 1 day'::interval, "time") <= '01-10-2000'::date)) \qecho time_bucket exclusion with timestamp time_bucket exclusion with timestamp :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('1d',time) < '2000-01-03' ORDER BY time; --- QUERY PLAN --- Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Index Cond: ("time" < 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 1 day'::interval, "time") < 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('1d',time) >= '2000-01-03' AND time_bucket('1d',time) <= '2000-01-10' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) AND ("time" <= 'Tue Jan 11 00:00:00 2000'::timestamp without time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) AND (time_bucket('@ 1 day'::interval, "time") <= 'Mon Jan 10 00:00:00 2000'::timestamp without time zone)) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) AND ("time" <= 'Tue Jan 11 00:00:00 2000'::timestamp without time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) AND (time_bucket('@ 1 day'::interval, "time") <= 'Mon Jan 10 00:00:00 2000'::timestamp without time zone)) \qecho time_bucket exclusion with timestamptz time_bucket exclusion with timestamptz :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('6h',time) < '2000-01-03' ORDER BY time; --- QUERY PLAN --- Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: ("time" < 'Mon Jan 03 06:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 6 hours'::interval, "time") < 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('6h',time) >= '2000-01-03' AND time_bucket('6h',time) <= '2000-01-10' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Mon Jan 10 06:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 6 hours'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 6 hours'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Mon Jan 10 06:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 6 hours'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 6 hours'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho time_bucket exclusion with timestamptz and day interval time_bucket exclusion with timestamptz and day interval :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1d',time) < '2000-01-03' ORDER BY time; --- QUERY PLAN --- Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: ("time" < 'Tue Jan 04 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 1 day'::interval, "time") < 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1d',time) >= '2000-01-03' AND time_bucket('1d',time) <= '2000-01-10' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Tue Jan 11 00:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 1 day'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Tue Jan 11 00:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 1 day'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1d',time) >= '2000-01-03' AND time_bucket('7d',time) <= '2000-01-10' ORDER BY time; --- QUERY PLAN --- Sort Sort Key: metrics_timestamptz."time" -> Append -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Mon Jan 17 00:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 7 days'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Seq Scan on _hyper_6_161_chunk Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 7 days'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Mon Jan 17 00:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 7 days'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho no transformation no transformation :PREFIX SELECT * FROM hyper WHERE time_bucket(10 + floor(random())::int, time) > 10 AND time_bucket(10 + floor(random())::int, time) < 100 AND time < 150 ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Custom Scan (ChunkAppend) on hyper Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_2_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_3_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_4_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_5_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_6_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_7_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_8_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_9_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_10_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_11_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_12_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_13_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_14_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_15_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) \qecho exclude chunks based on time column with partitioning function. This exclude chunks based on time column with partitioning function. This \qecho transparently applies the time partitioning function on the time transparently applies the time partitioning function on the time \qecho value to be able to exclude chunks (similar to a closed dimension). value to be able to exclude chunks (similar to a closed dimension). :PREFIX SELECT * FROM hyper_timefunc WHERE time < 4 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_timefunc.value -> Append -> Seq Scan on _hyper_4_124_chunk Filter: ("time" < '4'::double precision) -> Seq Scan on _hyper_4_125_chunk Filter: ("time" < '4'::double precision) -> Seq Scan on _hyper_4_126_chunk Filter: ("time" < '4'::double precision) -> Seq Scan on _hyper_4_127_chunk Filter: ("time" < '4'::double precision) \qecho excluding based on time expression is currently unoptimized excluding based on time expression is currently unoptimized :PREFIX SELECT * FROM hyper_timefunc WHERE unix_to_timestamp(time) < 'Wed Dec 31 16:00:04 1969 PST' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_timefunc.value -> Append -> Seq Scan on _hyper_4_124_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_125_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_126_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_127_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_128_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_129_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_130_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_131_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_132_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_133_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_134_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_135_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_136_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_137_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_138_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_139_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_140_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_141_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_142_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_143_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_144_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_145_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_146_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_147_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_148_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_149_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_150_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_151_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_152_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_153_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_154_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) \qecho test qual propagation for joins test qual propagation for joins RESET constraint_exclusion; \qecho nothing to propagate nothing to propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1, metrics_timestamptz_2 m2 WHERE m1.time = m2.time ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time ORDER BY m1.time; --- QUERY PLAN --- Merge Left Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time ORDER BY m1.time; --- QUERY PLAN --- Sort Sort Key: m1."time" -> Merge Right Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 \qecho OR constraints should not propagate OR constraints should not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' OR m1.time > '2001-01-01' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Filter: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) OR ("time" > 'Mon Jan 01 00:00:00 2001 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Filter: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) OR ("time" > 'Mon Jan 01 00:00:00 2001 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 \qecho test single constraint test single constraint \qecho constraint should be on both scans constraint should be on both scans \qecho these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column :PREFIX SELECT m1.time FROM metrics_timestamptz m1, metrics_timestamptz_2 m2 WHERE m1.time = m2.time AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Left Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho test 2 constraints on single relation test 2 constraints on single relation \qecho these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column :PREFIX SELECT m1.time FROM metrics_timestamptz m1, metrics_timestamptz_2 m2 WHERE m1.time = m2.time AND m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Nested Loop Left Join -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Append -> Index Only Scan using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 Index Cond: ("time" = m1."time") :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho test 2 constraints with 1 constraint on each relation test 2 constraints with 1 constraint on each relation \qecho these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column :PREFIX SELECT m1.time FROM metrics_timestamptz m1, metrics_timestamptz_2 m2 WHERE m1.time = m2.time AND m1.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) \qecho test constraints in ON clause of INNER JOIN test constraints in ON clause of INNER JOIN :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho test constraints in ON clause of LEFT JOIN test constraints in ON clause of LEFT JOIN \qecho must not propagate must not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Left Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho test constraints in ON clause of RIGHT JOIN test constraints in ON clause of RIGHT JOIN \qecho must not propagate must not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Gather Merge Workers Planned: 2 -> Sort Sort Key: m1."time" -> Parallel Hash Left Join Hash Cond: (m2."time" = m1."time") Join Filter: ((m2."time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND (m2."time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Parallel Append -> Parallel Seq Scan on _hyper_7_165_chunk m2_1 -> Parallel Seq Scan on _hyper_7_166_chunk m2_2 -> Parallel Seq Scan on _hyper_7_167_chunk m2_3 -> Parallel Seq Scan on _hyper_7_168_chunk m2_4 -> Parallel Seq Scan on _hyper_7_169_chunk m2_5 -> Parallel Seq Scan on _hyper_7_170_chunk m2_6 -> Parallel Hash -> Parallel Append -> Parallel Seq Scan on _hyper_6_160_chunk m1_1 -> Parallel Seq Scan on _hyper_6_161_chunk m1_2 -> Parallel Seq Scan on _hyper_6_162_chunk m1_3 -> Parallel Seq Scan on _hyper_6_163_chunk m1_4 -> Parallel Seq Scan on _hyper_6_164_chunk m1_5 \qecho test equality condition not in ON clause test equality condition not in ON clause :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON true WHERE m2.time = m1.time AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho test constraints not joined on test constraints not joined on \qecho device_id constraint must not propagate device_id constraint must not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON true WHERE m2.time = m1.time AND m2.time < '2000-01-10' AND m1.device_id = 1 ORDER BY m1.time; --- QUERY PLAN --- Sort Sort Key: m1."time" -> Nested Loop -> Append -> Seq Scan on _hyper_6_160_chunk m1_1 Filter: (device_id = 1) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 1) -> Append -> Index Only Scan using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" = m1."time") AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho test multiple join conditions test multiple join conditions \qecho device_id constraint should propagate device_id constraint should propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON true WHERE m2.time = m1.time AND m1.device_id = m2.device_id AND m2.time < '2000-01-10' AND m1.device_id = 1 ORDER BY m1.time; --- QUERY PLAN --- Sort Sort Key: m1."time" -> Nested Loop -> Append -> Seq Scan on _hyper_6_160_chunk m1_1 Filter: (device_id = 1) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 1) -> Append -> Index Scan using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: ("time" = m1."time") Filter: (device_id = 1) -> Index Scan using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" = m1."time") AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) Filter: (device_id = 1) \qecho test join with 3 tables test join with 3 tables :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time INNER JOIN metrics_timestamptz m3 ON m2.time=m3.time WHERE m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Nested Loop -> Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Append -> Index Only Scan using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m3_1 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m3_2 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m3_3 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m3_4 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m3_5 Index Cond: ("time" = m1."time") \qecho test non-Const constraints test non-Const constraints :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10'::text::timestamptz ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" Chunks excluded during startup: 3 -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" Chunks excluded during startup: 4 -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) \qecho test now() test now() :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < now() ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 Index Cond: ("time" < now()) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 Index Cond: ("time" < now()) \qecho test volatile function test volatile function \qecho should not propagate should not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < clock_timestamp() ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 Filter: ("time" < clock_timestamp()) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m2.time < clock_timestamp() ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m2."time" = m1."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 Filter: ("time" < clock_timestamp()) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 \qecho test JOINs with normal table test JOINs with normal table \qecho will not propagate because constraints are only added to hypertables will not propagate because constraints are only added to hypertables :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN regular_timestamptz m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Sort Sort Key: m2."time" -> Seq Scan on regular_timestamptz m2 \qecho test JOINs with normal table test JOINs with normal table :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN regular_timestamptz m2 ON m1.time = m2.time WHERE m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m2."time" = m1."time") -> Sort Sort Key: m2."time" -> Seq Scan on regular_timestamptz m2 Filter: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho test quals are not pushed into OUTER JOIN test quals are not pushed into OUTER JOIN CREATE TABLE outer_join_1 (id int, name text,time timestamptz NOT NULL DEFAULT '2000-01-01'); CREATE TABLE outer_join_2 (id int, name text,time timestamptz NOT NULL DEFAULT '2000-01-01'); SELECT (SELECT table_name FROM create_hypertable(tbl, 'time')) FROM (VALUES ('outer_join_1'),('outer_join_2')) v(tbl); table_name -------------- outer_join_1 outer_join_2 INSERT INTO outer_join_1 VALUES(1,'a'), (2,'b'); INSERT INTO outer_join_2 VALUES(1,'a'); :PREFIX SELECT one.id, two.name FROM outer_join_1 one LEFT OUTER JOIN outer_join_2 two ON one.id=two.id WHERE one.id=2; --- QUERY PLAN --- Nested Loop Left Join -> Seq Scan on _hyper_9_176_chunk one Filter: (id = 2) -> Materialize -> Seq Scan on _hyper_10_177_chunk two Filter: (id = 2) :PREFIX SELECT one.id, two.name FROM outer_join_2 two RIGHT OUTER JOIN outer_join_1 one ON one.id=two.id WHERE one.id=2; --- QUERY PLAN --- Nested Loop Left Join -> Seq Scan on _hyper_9_176_chunk one Filter: (id = 2) -> Materialize -> Seq Scan on _hyper_10_177_chunk two Filter: (id = 2) DROP TABLE outer_join_1; DROP TABLE outer_join_2; -- test UNION between regular table and hypertable SELECT time FROM regular_timestamptz UNION SELECT time FROM metrics_timestamptz ORDER BY 1; time ------------------------------ Sat Jan 01 00:00:00 2000 PST Sun Jan 02 00:00:00 2000 PST Mon Jan 03 00:00:00 2000 PST Tue Jan 04 00:00:00 2000 PST Wed Jan 05 00:00:00 2000 PST Thu Jan 06 00:00:00 2000 PST Fri Jan 07 00:00:00 2000 PST Sat Jan 08 00:00:00 2000 PST Sun Jan 09 00:00:00 2000 PST Mon Jan 10 00:00:00 2000 PST Tue Jan 11 00:00:00 2000 PST Wed Jan 12 00:00:00 2000 PST Thu Jan 13 00:00:00 2000 PST Fri Jan 14 00:00:00 2000 PST Sat Jan 15 00:00:00 2000 PST Sun Jan 16 00:00:00 2000 PST Mon Jan 17 00:00:00 2000 PST Tue Jan 18 00:00:00 2000 PST Wed Jan 19 00:00:00 2000 PST Thu Jan 20 00:00:00 2000 PST Fri Jan 21 00:00:00 2000 PST Sat Jan 22 00:00:00 2000 PST Sun Jan 23 00:00:00 2000 PST Mon Jan 24 00:00:00 2000 PST Tue Jan 25 00:00:00 2000 PST Wed Jan 26 00:00:00 2000 PST Thu Jan 27 00:00:00 2000 PST Fri Jan 28 00:00:00 2000 PST Sat Jan 29 00:00:00 2000 PST Sun Jan 30 00:00:00 2000 PST Mon Jan 31 00:00:00 2000 PST Tue Feb 01 00:00:00 2000 PST -- test UNION ALL between regular table and hypertable SELECT time FROM regular_timestamptz UNION ALL SELECT time FROM metrics_timestamptz ORDER BY 1; time ------------------------------ Sat Jan 01 00:00:00 2000 PST Sat Jan 01 00:00:00 2000 PST Sat Jan 01 00:00:00 2000 PST Sat Jan 01 00:00:00 2000 PST Sun Jan 02 00:00:00 2000 PST Sun Jan 02 00:00:00 2000 PST Sun Jan 02 00:00:00 2000 PST Sun Jan 02 00:00:00 2000 PST Mon Jan 03 00:00:00 2000 PST Mon Jan 03 00:00:00 2000 PST Mon Jan 03 00:00:00 2000 PST Mon Jan 03 00:00:00 2000 PST Tue Jan 04 00:00:00 2000 PST Tue Jan 04 00:00:00 2000 PST Tue Jan 04 00:00:00 2000 PST Tue Jan 04 00:00:00 2000 PST Wed Jan 05 00:00:00 2000 PST Wed Jan 05 00:00:00 2000 PST Wed Jan 05 00:00:00 2000 PST Wed Jan 05 00:00:00 2000 PST Thu Jan 06 00:00:00 2000 PST Thu Jan 06 00:00:00 2000 PST Thu Jan 06 00:00:00 2000 PST Thu Jan 06 00:00:00 2000 PST Fri Jan 07 00:00:00 2000 PST Fri Jan 07 00:00:00 2000 PST Fri Jan 07 00:00:00 2000 PST Fri Jan 07 00:00:00 2000 PST Sat Jan 08 00:00:00 2000 PST Sat Jan 08 00:00:00 2000 PST Sat Jan 08 00:00:00 2000 PST Sat Jan 08 00:00:00 2000 PST Sun Jan 09 00:00:00 2000 PST Sun Jan 09 00:00:00 2000 PST Sun Jan 09 00:00:00 2000 PST Sun Jan 09 00:00:00 2000 PST Mon Jan 10 00:00:00 2000 PST Mon Jan 10 00:00:00 2000 PST Mon Jan 10 00:00:00 2000 PST Mon Jan 10 00:00:00 2000 PST Tue Jan 11 00:00:00 2000 PST Tue Jan 11 00:00:00 2000 PST Tue Jan 11 00:00:00 2000 PST Tue Jan 11 00:00:00 2000 PST Wed Jan 12 00:00:00 2000 PST Wed Jan 12 00:00:00 2000 PST Wed Jan 12 00:00:00 2000 PST Wed Jan 12 00:00:00 2000 PST Thu Jan 13 00:00:00 2000 PST Thu Jan 13 00:00:00 2000 PST Thu Jan 13 00:00:00 2000 PST Thu Jan 13 00:00:00 2000 PST Fri Jan 14 00:00:00 2000 PST Fri Jan 14 00:00:00 2000 PST Fri Jan 14 00:00:00 2000 PST Fri Jan 14 00:00:00 2000 PST Sat Jan 15 00:00:00 2000 PST Sat Jan 15 00:00:00 2000 PST Sat Jan 15 00:00:00 2000 PST Sat Jan 15 00:00:00 2000 PST Sun Jan 16 00:00:00 2000 PST Sun Jan 16 00:00:00 2000 PST Sun Jan 16 00:00:00 2000 PST Sun Jan 16 00:00:00 2000 PST Mon Jan 17 00:00:00 2000 PST Mon Jan 17 00:00:00 2000 PST Mon Jan 17 00:00:00 2000 PST Mon Jan 17 00:00:00 2000 PST Tue Jan 18 00:00:00 2000 PST Tue Jan 18 00:00:00 2000 PST Tue Jan 18 00:00:00 2000 PST Tue Jan 18 00:00:00 2000 PST Wed Jan 19 00:00:00 2000 PST Wed Jan 19 00:00:00 2000 PST Wed Jan 19 00:00:00 2000 PST Wed Jan 19 00:00:00 2000 PST Thu Jan 20 00:00:00 2000 PST Thu Jan 20 00:00:00 2000 PST Thu Jan 20 00:00:00 2000 PST Thu Jan 20 00:00:00 2000 PST Fri Jan 21 00:00:00 2000 PST Fri Jan 21 00:00:00 2000 PST Fri Jan 21 00:00:00 2000 PST Fri Jan 21 00:00:00 2000 PST Sat Jan 22 00:00:00 2000 PST Sat Jan 22 00:00:00 2000 PST Sat Jan 22 00:00:00 2000 PST Sat Jan 22 00:00:00 2000 PST Sun Jan 23 00:00:00 2000 PST Sun Jan 23 00:00:00 2000 PST Sun Jan 23 00:00:00 2000 PST Sun Jan 23 00:00:00 2000 PST Mon Jan 24 00:00:00 2000 PST Mon Jan 24 00:00:00 2000 PST Mon Jan 24 00:00:00 2000 PST Mon Jan 24 00:00:00 2000 PST Tue Jan 25 00:00:00 2000 PST Tue Jan 25 00:00:00 2000 PST Tue Jan 25 00:00:00 2000 PST Tue Jan 25 00:00:00 2000 PST Wed Jan 26 00:00:00 2000 PST Wed Jan 26 00:00:00 2000 PST Wed Jan 26 00:00:00 2000 PST Wed Jan 26 00:00:00 2000 PST Thu Jan 27 00:00:00 2000 PST Thu Jan 27 00:00:00 2000 PST Thu Jan 27 00:00:00 2000 PST Thu Jan 27 00:00:00 2000 PST Fri Jan 28 00:00:00 2000 PST Fri Jan 28 00:00:00 2000 PST Fri Jan 28 00:00:00 2000 PST Fri Jan 28 00:00:00 2000 PST Sat Jan 29 00:00:00 2000 PST Sat Jan 29 00:00:00 2000 PST Sat Jan 29 00:00:00 2000 PST Sat Jan 29 00:00:00 2000 PST Sun Jan 30 00:00:00 2000 PST Sun Jan 30 00:00:00 2000 PST Sun Jan 30 00:00:00 2000 PST Sun Jan 30 00:00:00 2000 PST Mon Jan 31 00:00:00 2000 PST Mon Jan 31 00:00:00 2000 PST Mon Jan 31 00:00:00 2000 PST Mon Jan 31 00:00:00 2000 PST Tue Feb 01 00:00:00 2000 PST Tue Feb 01 00:00:00 2000 PST Tue Feb 01 00:00:00 2000 PST Tue Feb 01 00:00:00 2000 PST -- test nested join qual propagation :PREFIX SELECT * FROM ( SELECT o1_m1.time FROM metrics_timestamptz o1_m1 INNER JOIN metrics_timestamptz_2 o1_m2 ON true WHERE o1_m2.time = o1_m1.time AND o1_m1.device_id = o1_m2.device_id AND o1_m2.time < '2000-01-10' AND o1_m1.device_id = 1 ) o1 FULL OUTER JOIN ( SELECT o2_m1.time FROM metrics_timestamptz o2_m1 FULL OUTER JOIN metrics_timestamptz_2 o2_m2 ON true WHERE o2_m2.time = o2_m1.time AND o2_m1.device_id = o2_m2.device_id AND o2_m2.time > '2000-01-20' AND o2_m1.device_id = 2 ) o2 ON o1.time = o2.time ORDER BY 1,2; --- QUERY PLAN --- Sort Sort Key: o1_m1."time", o2_m1."time" -> Hash Full Join Hash Cond: (o2_m1."time" = o1_m1."time") -> Nested Loop -> Append -> Index Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk o2_m2_1 Index Cond: ("time" > 'Thu Jan 20 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 2) -> Seq Scan on _hyper_7_169_chunk o2_m2_2 Filter: (device_id = 2) -> Seq Scan on _hyper_7_170_chunk o2_m2_3 Filter: (device_id = 2) -> Append -> Index Scan using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk o2_m1_1 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk o2_m1_2 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk o2_m1_3 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk o2_m1_4 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk o2_m1_5 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Hash -> Nested Loop -> Append -> Seq Scan on _hyper_7_165_chunk o1_m2_1 Filter: (device_id = 1) -> Index Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk o1_m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 1) -> Append -> Index Scan using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk o1_m1_1 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk o1_m1_2 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk o1_m1_3 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk o1_m1_4 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk o1_m1_5 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) :PREFIX SELECT * FROM ( SELECT o1_m1.time FROM metrics_timestamptz o1_m1 INNER JOIN metrics_timestamptz_2 o1_m2 ON o1_m2.time = o1_m1.time AND o1_m1.device_id = o1_m2.device_id WHERE o1_m2.time < '2000-01-10' AND o1_m1.device_id = 1 ) o1 FULL OUTER JOIN ( SELECT o2_m1.time FROM metrics_timestamptz o2_m1 FULL OUTER JOIN metrics_timestamptz_2 o2_m2 ON o2_m2.time = o2_m1.time AND o2_m1.device_id = o2_m2.device_id WHERE o2_m2.time > '2000-01-20' AND o2_m1.device_id = 2 ) o2 ON o1.time = o2.time ORDER BY 1,2; --- QUERY PLAN --- Sort Sort Key: o1_m1."time", o2_m1."time" -> Hash Full Join Hash Cond: (o2_m1."time" = o1_m1."time") -> Nested Loop -> Append -> Index Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk o2_m2_1 Index Cond: ("time" > 'Thu Jan 20 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 2) -> Seq Scan on _hyper_7_169_chunk o2_m2_2 Filter: (device_id = 2) -> Seq Scan on _hyper_7_170_chunk o2_m2_3 Filter: (device_id = 2) -> Append -> Index Scan using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk o2_m1_1 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk o2_m1_2 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk o2_m1_3 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk o2_m1_4 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk o2_m1_5 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Hash -> Nested Loop -> Append -> Seq Scan on _hyper_7_165_chunk o1_m2_1 Filter: (device_id = 1) -> Index Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk o1_m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 1) -> Append -> Index Scan using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk o1_m1_1 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk o1_m1_2 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk o1_m1_3 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk o1_m1_4 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk o1_m1_5 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) \set ECHO errors RESET timescaledb.enable_optimizations; CREATE TABLE t(time timestamptz NOT NULL); SELECT table_name FROM create_hypertable('t','time'); table_name ------------ t INSERT INTO t VALUES ('2000-01-01'), ('2010-01-01'), ('2020-01-01'); EXPLAIN (buffers off, costs off) SELECT * FROM t t1 INNER JOIN t t2 ON t1.time = t2.time WHERE t1.time < timestamptz '2010-01-01'; --- QUERY PLAN --- Merge Join Merge Cond: (t1."time" = t2."time") -> Merge Append Sort Key: t1."time" -> Index Only Scan Backward using _hyper_15_182_chunk_t_time_idx on _hyper_15_182_chunk t1_1 -> Index Only Scan Backward using _hyper_15_183_chunk_t_time_idx on _hyper_15_183_chunk t1_2 Index Cond: ("time" < 'Fri Jan 01 00:00:00 2010 PST'::timestamp with time zone) -> Materialize -> Merge Append Sort Key: t2."time" -> Index Only Scan Backward using _hyper_15_182_chunk_t_time_idx on _hyper_15_182_chunk t2_1 -> Index Only Scan Backward using _hyper_15_183_chunk_t_time_idx on _hyper_15_183_chunk t2_2 Index Cond: ("time" < 'Fri Jan 01 00:00:00 2010 PST'::timestamp with time zone) SET timescaledb.enable_qual_propagation TO false; EXPLAIN (buffers off, costs off) SELECT * FROM t t1 INNER JOIN t t2 ON t1.time = t2.time WHERE t1.time < timestamptz '2010-01-01'; --- QUERY PLAN --- Merge Join Merge Cond: (t1."time" = t2."time") -> Merge Append Sort Key: t1."time" -> Index Only Scan Backward using _hyper_15_182_chunk_t_time_idx on _hyper_15_182_chunk t1_1 -> Index Only Scan Backward using _hyper_15_183_chunk_t_time_idx on _hyper_15_183_chunk t1_2 Index Cond: ("time" < 'Fri Jan 01 00:00:00 2010 PST'::timestamp with time zone) -> Materialize -> Merge Append Sort Key: t2."time" -> Index Only Scan Backward using _hyper_15_182_chunk_t_time_idx on _hyper_15_182_chunk t2_1 -> Index Only Scan Backward using _hyper_15_183_chunk_t_time_idx on _hyper_15_183_chunk t2_2 -> Index Only Scan Backward using _hyper_15_184_chunk_t_time_idx on _hyper_15_184_chunk t2_3 RESET timescaledb.enable_qual_propagation; CREATE TABLE test (a int, time timestamptz NOT NULL); SELECT table_name FROM create_hypertable('public.test', 'time'); table_name ------------ test INSERT INTO test SELECT i, '2020-04-01'::date-10-i from generate_series(1,20) i; CREATE OR REPLACE FUNCTION test_f(_ts timestamptz) RETURNS SETOF test LANGUAGE SQL STABLE PARALLEL SAFE AS $f$ SELECT DISTINCT ON (a) * FROM test WHERE time >= _ts ORDER BY a, time DESC $f$; EXPLAIN (buffers off, costs off) SELECT * FROM test_f(now()); --- QUERY PLAN --- Unique -> Sort Sort Key: test.a, test."time" DESC -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 4 EXPLAIN (buffers off, costs off) SELECT * FROM test_f(now()); --- QUERY PLAN --- Unique -> Sort Sort Key: test.a, test."time" DESC -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 4 CREATE TABLE t1 (a int, b int NOT NULL); SELECT create_hypertable('t1', 'b', chunk_time_interval=>10); create_hypertable ------------------- (17,public,t1,t) CREATE TABLE t2 (a int, b int NOT NULL); SELECT create_hypertable('t2', 'b', chunk_time_interval=>10); create_hypertable ------------------- (18,public,t2,t) CREATE OR REPLACE FUNCTION f_t1(_a int, _b int) RETURNS SETOF t1 LANGUAGE SQL STABLE PARALLEL SAFE AS $function$ SELECT DISTINCT ON (a) * FROM t1 WHERE a = _a and b = _b ORDER BY a, b DESC $function$ ; CREATE OR REPLACE FUNCTION f_t2(_a int, _b int) RETURNS SETOF t2 LANGUAGE sql STABLE PARALLEL SAFE AS $function$ SELECT DISTINCT ON (j.a) j.* FROM f_t1(_a, _b) sc, t2 j WHERE j.b = _b AND j.a = _a ORDER BY j.a, j.b DESC $function$ ; CREATE OR REPLACE FUNCTION f_t1_2(_b int) RETURNS SETOF t1 LANGUAGE SQL STABLE PARALLEL SAFE AS $function$ SELECT DISTINCT ON (j.a) jt.* FROM t1 j, f_t1(j.a, _b) jt $function$; EXPLAIN (buffers off, costs off) SELECT * FROM f_t1_2(10); --- QUERY PLAN --- Subquery Scan on f_t1_2 -> Unique -> Sort Sort Key: j.a -> Nested Loop -> Seq Scan on t1 j -> Limit -> Index Scan using t1_b_idx on t1 Index Cond: (b = 10) Filter: (a = j.a) EXPLAIN (buffers off, costs off) SELECT * FROM f_t1_2(10) sc, f_t2(sc.a, 10); --- QUERY PLAN --- Nested Loop -> Unique -> Sort Sort Key: j.a -> Nested Loop -> Seq Scan on t1 j -> Limit -> Index Scan using t1_b_idx on t1 Index Cond: (b = 10) Filter: (a = j.a) -> Limit -> Nested Loop -> Limit -> Index Scan using t1_b_idx on t1 t1_1 Index Cond: (b = 10) Filter: (a = t1.a) -> Index Scan using t2_b_idx on t2 j_1 Index Cond: (b = 10) Filter: (a = t1.a) CREATE TABLE metrics_int1(time int, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval=1); INSERT INTO metrics_int1 SELECT i, i::text, i FROM generate_series(3,7) i; SELECT tableoid::regclass, time FROM metrics_int1 ORDER BY time; tableoid | time -------------------------------------------+------ _timescaledb_internal._hyper_19_189_chunk | 3 _timescaledb_internal._hyper_19_190_chunk | 4 _timescaledb_internal._hyper_19_191_chunk | 5 _timescaledb_internal._hyper_19_192_chunk | 6 _timescaledb_internal._hyper_19_193_chunk | 7 EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time >= 2; --- QUERY PLAN --- Append (actual rows=5.00 loops=1) -> Seq Scan on _hyper_19_189_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_192_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_193_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time >= 1 AND time >= 2; --- QUERY PLAN --- Append (actual rows=5.00 loops=1) -> Seq Scan on _hyper_19_189_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_192_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_193_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time >= 4; --- QUERY PLAN --- Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_192_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_193_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time > 4 AND time < 6; --- QUERY PLAN --- Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time > 4 AND time <= 6; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_192_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time >= 4 AND time < 6; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time >= 4 AND time <= 6; --- QUERY PLAN --- Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_192_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time BETWEEN 4 AND 5; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time = 5; --- QUERY PLAN --- Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) SET TIMEZONE='UTC'; CREATE TABLE metrics_tstz(time timestamptz, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval='1day'); INSERT INTO metrics_tstz SELECT '2000-01-01'::timestamptz + format('%s day',i)::interval, i::text, i FROM generate_series(2,6) i; SELECT tableoid::regclass, time FROM metrics_tstz ORDER BY time; tableoid | time -------------------------------------------+------------------------------ _timescaledb_internal._hyper_20_194_chunk | Mon Jan 03 00:00:00 2000 UTC _timescaledb_internal._hyper_20_195_chunk | Tue Jan 04 00:00:00 2000 UTC _timescaledb_internal._hyper_20_196_chunk | Wed Jan 05 00:00:00 2000 UTC _timescaledb_internal._hyper_20_197_chunk | Thu Jan 06 00:00:00 2000 UTC _timescaledb_internal._hyper_20_198_chunk | Fri Jan 07 00:00:00 2000 UTC EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-02'; --- QUERY PLAN --- Append (actual rows=5.00 loops=1) -> Seq Scan on _hyper_20_194_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_197_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_198_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-01' AND time >= '2000-01-02'; --- QUERY PLAN --- Append (actual rows=5.00 loops=1) -> Seq Scan on _hyper_20_194_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_197_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_198_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04'; --- QUERY PLAN --- Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_197_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_198_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time > '2000-01-04' AND time < '2000-01-06'; --- QUERY PLAN --- Append (actual rows=1.00 loops=1) -> Index Scan using _hyper_20_195_chunk_metrics_tstz_time_idx on _hyper_20_195_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" < 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time > '2000-01-04' AND time <= '2000-01-06'; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Index Scan using _hyper_20_195_chunk_metrics_tstz_time_idx on _hyper_20_195_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_20_197_chunk_metrics_tstz_time_idx on _hyper_20_197_chunk (actual rows=1.00 loops=1) Index Cond: (("time" > 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time < '2000-01-06'; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time <= '2000-01-06'; --- QUERY PLAN --- Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_20_197_chunk_metrics_tstz_time_idx on _hyper_20_197_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time BETWEEN '2000-01-04' AND '2000-01-05'; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_20_196_chunk_metrics_tstz_time_idx on _hyper_20_196_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Wed Jan 05 00:00:00 2000 UTC'::timestamp with time zone)) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time = '2000-01-05'; --- QUERY PLAN --- Index Scan using _hyper_20_196_chunk_metrics_tstz_time_idx on _hyper_20_196_chunk (actual rows=1.00 loops=1) Index Cond: ("time" = 'Wed Jan 05 00:00:00 2000 UTC'::timestamp with time zone) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time <= '2000-01-06' AND device = '5'; --- QUERY PLAN --- Append (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=0.00 loops=1) Filter: (device = '5'::text) Rows Removed by Filter: 1 -> Seq Scan on _hyper_20_196_chunk (actual rows=0.00 loops=1) Filter: (device = '5'::text) Rows Removed by Filter: 1 -> Index Scan using _hyper_20_197_chunk_metrics_tstz_time_idx on _hyper_20_197_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) Filter: (device = '5'::text) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time <= '2000-01-06' AND device IS NOT NULL; --- QUERY PLAN --- Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) Filter: (device IS NOT NULL) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) Filter: (device IS NOT NULL) -> Index Scan using _hyper_20_197_chunk_metrics_tstz_time_idx on _hyper_20_197_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) Filter: (device IS NOT NULL) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time <= '2000-01-06' AND time < now(); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_tstz (actual rows=3.00 loops=1) Chunks excluded during startup: 0 -> Index Scan using _hyper_20_195_chunk_metrics_tstz_time_idx on _hyper_20_195_chunk (actual rows=1.00 loops=1) Index Cond: ("time" < now()) -> Index Scan using _hyper_20_196_chunk_metrics_tstz_time_idx on _hyper_20_196_chunk (actual rows=1.00 loops=1) Index Cond: ("time" < now()) -> Index Scan using _hyper_20_197_chunk_metrics_tstz_time_idx on _hyper_20_197_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" < now())) CREATE TABLE metrics_space(time timestamptz, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval='1day'); SELECT add_dimension('metrics_space', 'device', 4); add_dimension ------------------------------------ (25,public,metrics_space,device,t) INSERT INTO metrics_space SELECT '2000-01-01'::timestamptz + format('%s day',i)::interval, i::text, i FROM generate_series(2,6) i; EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_space WHERE time >= '2000-01-02'; --- QUERY PLAN --- Append (actual rows=5.00 loops=1) -> Index Scan using _hyper_21_199_chunk_metrics_space_time_idx on _hyper_21_199_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Sun Jan 02 00:00:00 2000 UTC'::timestamp with time zone) -> Index Scan using _hyper_21_200_chunk_metrics_space_time_idx on _hyper_21_200_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Sun Jan 02 00:00:00 2000 UTC'::timestamp with time zone) -> Index Scan using _hyper_21_201_chunk_metrics_space_time_idx on _hyper_21_201_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Sun Jan 02 00:00:00 2000 UTC'::timestamp with time zone) -> Index Scan using _hyper_21_202_chunk_metrics_space_time_idx on _hyper_21_202_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Sun Jan 02 00:00:00 2000 UTC'::timestamp with time zone) -> Index Scan using _hyper_21_203_chunk_metrics_space_time_idx on _hyper_21_203_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Sun Jan 02 00:00:00 2000 UTC'::timestamp with time zone) --TEST END-- ================================================ FILE: test/expected/plan_expand_hypertable-17.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set PREFIX 'EXPLAIN (buffers off, costs off) ' \ir include/plan_expand_hypertable_load.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --single time dimension CREATE TABLE hyper ("time_broken" bigint NOT NULL, "value" integer); ALTER TABLE hyper DROP COLUMN time_broken, ADD COLUMN time BIGINT; SELECT create_hypertable('hyper', 'time', chunk_time_interval => 10); create_hypertable -------------------- (1,public,hyper,t) INSERT INTO hyper SELECT g, g FROM generate_series(0,1000) g; --insert a point with INT_MAX_64 INSERT INTO hyper (time, value) SELECT 9223372036854775807::bigint, 0; --time and space CREATE TABLE hyper_w_space ("time_broken" bigint NOT NULL, "device_id" text, "value" integer); ALTER TABLE hyper_w_space DROP COLUMN time_broken, ADD COLUMN time BIGINT; SELECT create_hypertable('hyper_w_space', 'time', 'device_id', 4, chunk_time_interval => 10); create_hypertable ---------------------------- (2,public,hyper_w_space,t) INSERT INTO hyper_w_space (time, device_id, value) SELECT g, 'dev' || g, g FROM generate_series(0,30) g; CREATE VIEW hyper_w_space_view AS (SELECT * FROM hyper_w_space); --with timestamp and space CREATE TABLE tag (id serial PRIMARY KEY, name text); CREATE TABLE hyper_ts ("time_broken" timestamptz NOT NULL, "device_id" text, tag_id INT REFERENCES tag(id), "value" integer); ALTER TABLE hyper_ts DROP COLUMN time_broken, ADD COLUMN time TIMESTAMPTZ; SELECT create_hypertable('hyper_ts', 'time', 'device_id', 2, chunk_time_interval => '10 seconds'::interval); create_hypertable ----------------------- (3,public,hyper_ts,t) INSERT INTO tag(name) SELECT 'tag'||g FROM generate_series(0,10) g; INSERT INTO hyper_ts (time, device_id, tag_id, value) SELECT to_timestamp(g), 'dev' || g, (random() /10)+1, g FROM generate_series(0,30) g; --one in the future INSERT INTO hyper_ts (time, device_id, tag_id, value) VALUES ('2100-01-01 02:03:04 PST', 'dev101', 1, 0); --time partitioning function CREATE OR REPLACE FUNCTION unix_to_timestamp(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT AS $BODY$ SELECT to_timestamp(unixtime); $BODY$; CREATE TABLE hyper_timefunc ("time" float8 NOT NULL, "device_id" text, "value" integer); SELECT create_hypertable('hyper_timefunc', 'time', 'device_id', 4, chunk_time_interval => 10, time_partitioning_func => 'unix_to_timestamp'); psql:include/plan_expand_hypertable_load.sql:57: WARNING: unexpected interval: smaller than one second create_hypertable ----------------------------- (4,public,hyper_timefunc,t) INSERT INTO hyper_timefunc (time, device_id, value) SELECT g, 'dev' || g, g FROM generate_series(0,30) g; CREATE TABLE metrics_timestamp(time timestamp); SELECT create_hypertable('metrics_timestamp','time'); psql:include/plan_expand_hypertable_load.sql:62: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------------- (5,public,metrics_timestamp,t) INSERT INTO metrics_timestamp SELECT generate_series('2000-01-01'::timestamp,'2000-02-01'::timestamp,'1d'::interval); CREATE TABLE metrics_timestamptz(time timestamptz, device_id int); SELECT create_hypertable('metrics_timestamptz','time'); create_hypertable ---------------------------------- (6,public,metrics_timestamptz,t) INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval), 1; INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval), 2; INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval), 3; --create a second table to test joins with CREATE TABLE metrics_timestamptz_2 (LIKE metrics_timestamptz); SELECT create_hypertable('metrics_timestamptz_2','time'); create_hypertable ------------------------------------ (7,public,metrics_timestamptz_2,t) INSERT INTO metrics_timestamptz_2 SELECT * FROM metrics_timestamptz; INSERT INTO metrics_timestamptz_2 VALUES ('2000-12-01'::timestamptz, 3); CREATE TABLE metrics_date(time date); SELECT create_hypertable('metrics_date','time'); create_hypertable --------------------------- (8,public,metrics_date,t) INSERT INTO metrics_date SELECT generate_series('2000-01-01'::date,'2000-02-01'::date,'1d'::interval); ANALYZE hyper; ANALYZE hyper_w_space; ANALYZE tag; ANALYZE hyper_ts; ANALYZE hyper_timefunc; -- create normal table for JOIN tests CREATE TABLE regular_timestamptz(time timestamptz); INSERT INTO regular_timestamptz SELECT generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval); \ir include/plan_expand_hypertable_query.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --we want to see how our logic excludes chunks --and not how much work constraint_exclusion does SET constraint_exclusion = 'off'; \qecho test upper bounds test upper bounds :PREFIX SELECT * FROM hyper WHERE time < 10 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_1_chunk.value -> Seq Scan on _hyper_1_1_chunk :PREFIX SELECT * FROM hyper WHERE time < 11 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk -> Seq Scan on _hyper_1_2_chunk Filter: ("time" < 11) :PREFIX SELECT * FROM hyper WHERE time = 10 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk.value -> Seq Scan on _hyper_1_2_chunk Filter: ("time" = 10) :PREFIX SELECT * FROM hyper WHERE 10 >= time ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk -> Seq Scan on _hyper_1_2_chunk Filter: (10 >= "time") \qecho test lower bounds test lower bounds :PREFIX SELECT * FROM hyper WHERE time >= 10 and time < 20 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk.value -> Seq Scan on _hyper_1_2_chunk :PREFIX SELECT * FROM hyper WHERE 10 < time and 20 >= time ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_2_chunk Filter: ((10 < "time") AND (20 >= "time")) -> Seq Scan on _hyper_1_3_chunk Filter: ((10 < "time") AND (20 >= "time")) :PREFIX SELECT * FROM hyper WHERE time >= 9 and time < 20 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 9) AND ("time" < 20)) -> Seq Scan on _hyper_1_2_chunk :PREFIX SELECT * FROM hyper WHERE time > 9 and time < 20 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk.value -> Seq Scan on _hyper_1_2_chunk \qecho test empty result test empty result :PREFIX SELECT * FROM hyper WHERE time < 0; --- QUERY PLAN --- Result One-Time Filter: false \qecho test expression evaluation test expression evaluation :PREFIX SELECT * FROM hyper WHERE time < (5*2)::smallint; --- QUERY PLAN --- Seq Scan on _hyper_1_1_chunk \qecho test logic at INT64_MAX test logic at INT64_MAX :PREFIX SELECT * FROM hyper WHERE time = 9223372036854775807::bigint ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_102_chunk.value -> Seq Scan on _hyper_1_102_chunk Filter: ("time" = '9223372036854775807'::bigint) :PREFIX SELECT * FROM hyper WHERE time = 9223372036854775806::bigint ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_102_chunk.value -> Seq Scan on _hyper_1_102_chunk Filter: ("time" = '9223372036854775806'::bigint) :PREFIX SELECT * FROM hyper WHERE time >= 9223372036854775807::bigint ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_102_chunk.value -> Seq Scan on _hyper_1_102_chunk Filter: ("time" >= '9223372036854775807'::bigint) :PREFIX SELECT * FROM hyper WHERE time > 9223372036854775807::bigint ORDER BY value; --- QUERY PLAN --- Sort Sort Key: value -> Result One-Time Filter: false :PREFIX SELECT * FROM hyper WHERE time > 9223372036854775806::bigint ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_102_chunk.value -> Seq Scan on _hyper_1_102_chunk Filter: ("time" > '9223372036854775806'::bigint) \qecho cte cte :PREFIX WITH cte AS( SELECT * FROM hyper WHERE time < 10 ) SELECT * FROM cte ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_1_chunk.value -> Seq Scan on _hyper_1_1_chunk \qecho subquery subquery :PREFIX SELECT 0 = ANY (SELECT value FROM hyper WHERE time < 10); --- QUERY PLAN --- Result SubPlan 1 -> Seq Scan on _hyper_1_1_chunk \qecho no space constraint no space constraint :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" < 10) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" < 10) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" < 10) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" < 10) \qecho valid space constraint valid space constraint :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 and device_id = 'dev5' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 and 'dev5' = device_id ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND ('dev5'::text = device_id)) :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 and 'dev'||(2+3) = device_id ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND ('dev5'::text = device_id)) \qecho only space constraint only space constraint :PREFIX SELECT * FROM hyper_w_space WHERE 'dev5' = device_id ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_106_chunk Filter: ('dev5'::text = device_id) -> Seq Scan on _hyper_2_109_chunk Filter: ('dev5'::text = device_id) -> Seq Scan on _hyper_2_111_chunk Filter: ('dev5'::text = device_id) \qecho unhandled space constraint unhandled space constraint :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 and device_id > 'dev5' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: (("time" < 10) AND (device_id > 'dev5'::text)) -> Seq Scan on _hyper_2_104_chunk Filter: (("time" < 10) AND (device_id > 'dev5'::text)) -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND (device_id > 'dev5'::text)) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id > 'dev5'::text)) \qecho use of OR - does not filter chunks use of OR - does not filter chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND (device_id = 'dev5' or device_id = 'dev6') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: (("time" < 10) AND ((device_id = 'dev5'::text) OR (device_id = 'dev6'::text))) -> Seq Scan on _hyper_2_104_chunk Filter: (("time" < 10) AND ((device_id = 'dev5'::text) OR (device_id = 'dev6'::text))) -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND ((device_id = 'dev5'::text) OR (device_id = 'dev6'::text))) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND ((device_id = 'dev5'::text) OR (device_id = 'dev6'::text))) \qecho cte cte :PREFIX WITH cte AS( SELECT * FROM hyper_w_space WHERE time < 10 and device_id = 'dev5' ) SELECT * FROM cte ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) \qecho subquery subquery :PREFIX SELECT 0 = ANY (SELECT value FROM hyper_w_space WHERE time < 10 and device_id = 'dev5'); --- QUERY PLAN --- Result SubPlan 1 -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) \qecho view view :PREFIX SELECT * FROM hyper_w_space_view WHERE time < 10 and device_id = 'dev5' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) \qecho IN statement - simple IN statement - simple :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id IN ('dev5') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) \qecho IN statement - two chunks IN statement - two chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id IN ('dev5','dev6') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev5,dev6}'::text[]))) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev5,dev6}'::text[]))) \qecho IN statement - one chunk IN statement - one chunk :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id IN ('dev4','dev5') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev4,dev5}'::text[]))) \qecho NOT IN - does not filter chunks NOT IN - does not filter chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id NOT IN ('dev5','dev6') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: (("time" < 10) AND (device_id <> ALL ('{dev5,dev6}'::text[]))) -> Seq Scan on _hyper_2_104_chunk Filter: (("time" < 10) AND (device_id <> ALL ('{dev5,dev6}'::text[]))) -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND (device_id <> ALL ('{dev5,dev6}'::text[]))) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id <> ALL ('{dev5,dev6}'::text[]))) \qecho IN statement with subquery - does not filter chunks IN statement with subquery - does not filter chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id IN (SELECT 'dev5'::text) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) \qecho ANY ANY :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id = ANY(ARRAY['dev5','dev6']) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev5,dev6}'::text[]))) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev5,dev6}'::text[]))) \qecho ANY with intersection ANY with intersection :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id = ANY(ARRAY['dev5','dev6']) AND device_id = ANY(ARRAY['dev6','dev7']) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_105_chunk.value -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev5,dev6}'::text[])) AND (device_id = ANY ('{dev6,dev7}'::text[]))) \qecho ANY without intersection shouldnt scan any chunks ANY without intersection shouldnt scan any chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id = ANY(ARRAY['dev5','dev6']) AND device_id = ANY(ARRAY['dev8','dev9']) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: value -> Result One-Time Filter: false \qecho ANY/IN/ALL only works for equals operator ANY/IN/ALL only works for equals operator :PREFIX SELECT * FROM hyper_w_space WHERE device_id < ANY(ARRAY['dev5','dev6']) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_104_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_105_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_106_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_107_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_108_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_109_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_110_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_111_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_112_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_113_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_114_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_115_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) \qecho ALL with equals and different values shouldnt scan any chunks ALL with equals and different values shouldnt scan any chunks :PREFIX SELECT * FROM hyper_w_space WHERE device_id = ALL(ARRAY['dev5','dev6']) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: value -> Result One-Time Filter: false \qecho Multi AND Multi AND :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND time < 100 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: (("time" < 10) AND ("time" < 100)) -> Seq Scan on _hyper_2_104_chunk Filter: (("time" < 10) AND ("time" < 100)) -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND ("time" < 100)) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND ("time" < 100)) \qecho Time dimension doesnt filter chunks when using non-equality IN/ANY with multiple arguments Time dimension doesnt filter chunks when using non-equality IN/ANY with multiple arguments :PREFIX SELECT * FROM hyper_w_space WHERE time < ANY(ARRAY[1,2]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_107_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_108_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_109_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_110_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_111_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_112_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_113_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_114_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_115_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) \qecho Time dimension chunk exclusion with IN/ANY equality uses bounding range Time dimension chunk exclusion with IN/ANY equality uses bounding range :PREFIX SELECT * FROM hyper WHERE time IN (5, 15) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_1_2_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) :PREFIX SELECT * FROM hyper WHERE time = ANY(ARRAY[5, 15, 25]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ("time" = ANY ('{5,15,25}'::integer[])) -> Seq Scan on _hyper_1_2_chunk Filter: ("time" = ANY ('{5,15,25}'::integer[])) -> Seq Scan on _hyper_1_3_chunk Filter: ("time" = ANY ('{5,15,25}'::integer[])) :PREFIX SELECT * FROM hyper WHERE time = ANY(ARRAY[25, 15, 5]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ("time" = ANY ('{25,15,5}'::integer[])) -> Seq Scan on _hyper_1_2_chunk Filter: ("time" = ANY ('{25,15,5}'::integer[])) -> Seq Scan on _hyper_1_3_chunk Filter: ("time" = ANY ('{25,15,5}'::integer[])) :PREFIX SELECT * FROM hyper_w_space WHERE time IN (5, 15) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_107_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_108_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_109_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_110_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) :PREFIX SELECT * FROM metrics_timestamp WHERE time IN ('2000-01-05'::timestamp, '2000-01-15'::timestamp) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000","Sat Jan 15 00:00:00 2000"}'::timestamp without time zone[])) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000","Sat Jan 15 00:00:00 2000"}'::timestamp without time zone[])) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000","Sat Jan 15 00:00:00 2000"}'::timestamp without time zone[])) :PREFIX SELECT * FROM metrics_timestamptz WHERE time IN ('2000-01-05'::timestamptz, '2000-01-15'::timestamptz) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000 PST","Sat Jan 15 00:00:00 2000 PST"}'::timestamp with time zone[])) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000 PST","Sat Jan 15 00:00:00 2000 PST"}'::timestamp with time zone[])) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000 PST","Sat Jan 15 00:00:00 2000 PST"}'::timestamp with time zone[])) :PREFIX SELECT * FROM metrics_date WHERE time IN ('2000-01-05'::date, '2000-01-15'::date) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date Order: metrics_date."time" -> Index Only Scan Backward using _hyper_8_171_chunk_metrics_date_time_idx on _hyper_8_171_chunk Index Cond: ("time" = ANY ('{01-05-2000,01-15-2000}'::date[])) -> Index Only Scan Backward using _hyper_8_172_chunk_metrics_date_time_idx on _hyper_8_172_chunk Index Cond: ("time" = ANY ('{01-05-2000,01-15-2000}'::date[])) -> Index Only Scan Backward using _hyper_8_173_chunk_metrics_date_time_idx on _hyper_8_173_chunk Index Cond: ("time" = ANY ('{01-05-2000,01-15-2000}'::date[])) \qecho cross-type IN/ANY: timestamp to timestamptz column does not use bounding range (stable cast) cross-type IN/ANY: timestamp to timestamptz column does not use bounding range (stable cast) :PREFIX SELECT * FROM metrics_timestamptz WHERE time IN ('2000-01-05'::timestamp, '2000-01-15'::timestamp) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: ("time" = ANY (ARRAY[('Wed Jan 05 00:00:00 2000'::timestamp without time zone)::timestamp with time zone, ('Sat Jan 15 00:00:00 2000'::timestamp without time zone)::timestamp with time zone])) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Index Cond: ("time" = ANY (ARRAY[('Wed Jan 05 00:00:00 2000'::timestamp without time zone)::timestamp with time zone, ('Sat Jan 15 00:00:00 2000'::timestamp without time zone)::timestamp with time zone])) \qecho Time dimension chunk filtering works for ANY with single argument Time dimension chunk filtering works for ANY with single argument :PREFIX SELECT * FROM hyper_w_space WHERE time < ANY(ARRAY[1]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" < ANY ('{1}'::integer[])) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" < ANY ('{1}'::integer[])) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" < ANY ('{1}'::integer[])) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" < ANY ('{1}'::integer[])) \qecho Time dimension chunk filtering works for ALL with single argument Time dimension chunk filtering works for ALL with single argument :PREFIX SELECT * FROM hyper_w_space WHERE time < ALL(ARRAY[1]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" < ALL ('{1}'::integer[])) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" < ALL ('{1}'::integer[])) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" < ALL ('{1}'::integer[])) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" < ALL ('{1}'::integer[])) \qecho Time dimension chunk filtering works for ALL with multiple arguments Time dimension chunk filtering works for ALL with multiple arguments :PREFIX SELECT * FROM hyper_w_space WHERE time < ALL(ARRAY[1,10,20,30]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" < ALL ('{1,10,20,30}'::integer[])) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" < ALL ('{1,10,20,30}'::integer[])) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" < ALL ('{1,10,20,30}'::integer[])) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" < ALL ('{1,10,20,30}'::integer[])) \qecho AND intersection using IN and EQUALS AND intersection using IN and EQUALS :PREFIX SELECT * FROM hyper_w_space WHERE device_id IN ('dev1','dev2') AND device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ((device_id = ANY ('{dev1,dev2}'::text[])) AND (device_id = 'dev1'::text)) -> Seq Scan on _hyper_2_110_chunk Filter: ((device_id = ANY ('{dev1,dev2}'::text[])) AND (device_id = 'dev1'::text)) -> Seq Scan on _hyper_2_114_chunk Filter: ((device_id = ANY ('{dev1,dev2}'::text[])) AND (device_id = 'dev1'::text)) \qecho AND with no intersection using IN and EQUALS AND with no intersection using IN and EQUALS :PREFIX SELECT * FROM hyper_w_space WHERE device_id IN ('dev1','dev2') AND device_id = 'dev3' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: value -> Result One-Time Filter: false \qecho timestamps timestamps \qecho these should work since they are immutable functions these should work since they are immutable functions :PREFIX SELECT * FROM hyper_ts WHERE time < 'Wed Dec 31 16:00:10 1969 PST'::timestamptz ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Append -> Seq Scan on _hyper_3_116_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_3_117_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) :PREFIX SELECT * FROM hyper_ts WHERE time < to_timestamp(10) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Append -> Seq Scan on _hyper_3_116_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_3_117_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) :PREFIX SELECT * FROM hyper_ts WHERE time < 'Wed Dec 31 16:00:10 1969'::timestamp AT TIME ZONE 'PST' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Append -> Seq Scan on _hyper_3_116_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_3_117_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) :PREFIX SELECT * FROM hyper_ts WHERE time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_3_116_chunk.value -> Seq Scan on _hyper_3_116_chunk Filter: (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text)) \qecho these should not work since uses stable functions; these should not work since uses stable functions; :PREFIX SELECT * FROM hyper_ts WHERE time < 'Wed Dec 31 16:00:10 1969'::timestamp ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Custom Scan (ChunkAppend) on hyper_ts Chunks excluded during startup: 6 -> Seq Scan on _hyper_3_116_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969'::timestamp without time zone) -> Seq Scan on _hyper_3_117_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969'::timestamp without time zone) :PREFIX SELECT * FROM hyper_ts WHERE time < ('Wed Dec 31 16:00:10 1969'::timestamp::timestamptz) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Custom Scan (ChunkAppend) on hyper_ts Chunks excluded during startup: 6 -> Seq Scan on _hyper_3_116_chunk Filter: ("time" < ('Wed Dec 31 16:00:10 1969'::timestamp without time zone)::timestamp with time zone) -> Seq Scan on _hyper_3_117_chunk Filter: ("time" < ('Wed Dec 31 16:00:10 1969'::timestamp without time zone)::timestamp with time zone) :PREFIX SELECT * FROM hyper_ts WHERE NOW() < time ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Custom Scan (ChunkAppend) on hyper_ts Chunks excluded during startup: 7 -> Seq Scan on _hyper_3_123_chunk Filter: (now() < "time") \qecho joins joins :PREFIX SELECT * FROM hyper_ts WHERE tag_id IN (SELECT id FROM tag WHERE tag.id=1) and time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_3_116_chunk.value -> Nested Loop Semi Join -> Seq Scan on _hyper_3_116_chunk Filter: (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text) AND (tag_id = 1)) -> Seq Scan on tag Filter: (id = 1) :PREFIX SELECT * FROM hyper_ts WHERE tag_id IN (SELECT id FROM tag WHERE tag.id=1) or (time < to_timestamp(10) and device_id = 'dev1') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Custom Scan (ChunkAppend) on hyper_ts -> Seq Scan on _hyper_3_116_chunk Filter: ((ANY (tag_id = (hashed SubPlan 1).col1)) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) SubPlan 1 -> Seq Scan on tag Filter: (id = 1) -> Seq Scan on _hyper_3_117_chunk Filter: ((ANY (tag_id = (hashed SubPlan 1).col1)) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_118_chunk Filter: ((ANY (tag_id = (hashed SubPlan 1).col1)) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_119_chunk Filter: ((ANY (tag_id = (hashed SubPlan 1).col1)) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_120_chunk Filter: ((ANY (tag_id = (hashed SubPlan 1).col1)) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_121_chunk Filter: ((ANY (tag_id = (hashed SubPlan 1).col1)) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_122_chunk Filter: ((ANY (tag_id = (hashed SubPlan 1).col1)) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_123_chunk Filter: ((ANY (tag_id = (hashed SubPlan 1).col1)) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) :PREFIX SELECT * FROM hyper_ts WHERE tag_id IN (SELECT id FROM tag WHERE tag.name='tag1') and time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_3_116_chunk.value -> Nested Loop Join Filter: (_hyper_3_116_chunk.tag_id = tag.id) -> Seq Scan on _hyper_3_116_chunk Filter: (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text)) -> Seq Scan on tag Filter: (name = 'tag1'::text) :PREFIX SELECT * FROM hyper_ts JOIN tag on (hyper_ts.tag_id = tag.id ) WHERE time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_3_116_chunk.value -> Merge Join Merge Cond: (tag.id = _hyper_3_116_chunk.tag_id) -> Index Scan using tag_pkey on tag -> Sort Sort Key: _hyper_3_116_chunk.tag_id -> Seq Scan on _hyper_3_116_chunk Filter: (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text)) :PREFIX SELECT * FROM hyper_ts JOIN tag on (hyper_ts.tag_id = tag.id ) WHERE tag.name = 'tag1' and time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_3_116_chunk.value -> Nested Loop Join Filter: (tag.id = _hyper_3_116_chunk.tag_id) -> Seq Scan on _hyper_3_116_chunk Filter: (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text)) -> Seq Scan on tag Filter: (name = 'tag1'::text) \qecho test constraint exclusion for constraints in ON clause of JOINs test constraint exclusion for constraints in ON clause of JOINs \qecho should exclude chunks on m1 and propagate qual to m2 because of INNER JOIN should exclude chunks on m1 and propagate qual to m2 because of INNER JOIN :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho should exclude chunks on m2 and propagate qual to m1 because of INNER JOIN should exclude chunks on m2 and propagate qual to m1 because of INNER JOIN :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho must not exclude on m1 must not exclude on m1 :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Left Join Merge Cond: (m1."time" = m2."time") Join Filter: (m1."time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 \qecho should exclude chunks on m2 should exclude chunks on m2 :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Left Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho should exclude chunks on m1 should exclude chunks on m1 :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Sort Sort Key: m1."time" -> Merge Right Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 \qecho must not exclude chunks on m2 must not exclude chunks on m2 :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time < '2000-01-10' ORDER BY m1.time, m2.time; --- QUERY PLAN --- Sort Sort Key: m1."time", m2."time" -> Merge Left Join Merge Cond: (m2."time" = m1."time") Join Filter: (m2."time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 \qecho time_bucket exclusion time_bucket exclusion :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) < 10::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: _hyper_1_1_chunk."time" -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time") < '10'::bigint) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) < 11::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time") < '11'::bigint) -> Seq Scan on _hyper_1_2_chunk Filter: (time_bucket('10'::bigint, "time") < '11'::bigint) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" < '21'::bigint) AND (time_bucket('10'::bigint, "time") < '11'::bigint)) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) <= 10::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time") <= '10'::bigint) -> Seq Scan on _hyper_1_2_chunk Filter: (time_bucket('10'::bigint, "time") <= '10'::bigint) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" <= '20'::bigint) AND (time_bucket('10'::bigint, "time") <= '10'::bigint)) :PREFIX SELECT * FROM hyper WHERE 10::bigint > time_bucket(10, time) ORDER BY time; --- QUERY PLAN --- Sort Sort Key: _hyper_1_1_chunk."time" -> Seq Scan on _hyper_1_1_chunk Filter: ('10'::bigint > time_bucket('10'::bigint, "time")) :PREFIX SELECT * FROM hyper WHERE 11::bigint > time_bucket(10, time) ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ('11'::bigint > time_bucket('10'::bigint, "time")) -> Seq Scan on _hyper_1_2_chunk Filter: ('11'::bigint > time_bucket('10'::bigint, "time")) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" < '21'::bigint) AND ('11'::bigint > time_bucket('10'::bigint, "time"))) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time, 5) < 10::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) < '10'::bigint) -> Seq Scan on _hyper_1_2_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) < '10'::bigint) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time, 5) < 11::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) < '11'::bigint) -> Seq Scan on _hyper_1_2_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) < '11'::bigint) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" < '21'::bigint) AND (time_bucket('10'::bigint, "time", '5'::bigint) < '11'::bigint)) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time, 5) <= 10::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) <= '10'::bigint) -> Seq Scan on _hyper_1_2_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) <= '10'::bigint) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" <= '20'::bigint) AND (time_bucket('10'::bigint, "time", '5'::bigint) <= '10'::bigint)) :PREFIX SELECT * FROM hyper WHERE 10::bigint > time_bucket(10, time, 5) ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ('10'::bigint > time_bucket('10'::bigint, "time", '5'::bigint)) -> Seq Scan on _hyper_1_2_chunk Filter: ('10'::bigint > time_bucket('10'::bigint, "time", '5'::bigint)) :PREFIX SELECT * FROM hyper WHERE 11::bigint > time_bucket(10, time, 5) ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ('11'::bigint > time_bucket('10'::bigint, "time", '5'::bigint)) -> Seq Scan on _hyper_1_2_chunk Filter: ('11'::bigint > time_bucket('10'::bigint, "time", '5'::bigint)) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" < '21'::bigint) AND ('11'::bigint > time_bucket('10'::bigint, "time", '5'::bigint))) \qecho timestamp time_bucket exclusion timestamp time_bucket exclusion SELECT count(DISTINCT tableoid) FROM metrics_timestamp; count ------- 5 :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time") < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time") < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time") <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time") <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time") > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time") > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time) >= '2000-01-15' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Index Cond: ("time" >= 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time") >= 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket('@ 7 days'::interval, "time") >= 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time") >= 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'3d'::interval) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'3d'::interval) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'3d'::interval) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'3d'::interval) >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'2000-01-10'::timestamp) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'2000-01-10'::timestamp) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'2000-01-10'::timestamp) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'2000-01-10'::timestamp) >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) \qecho timestamptz time_bucket exclusion timestamptz time_bucket exclusion SELECT count(DISTINCT tableoid) FROM metrics_timestamptz; count ------- 5 :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time") < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time") < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time") <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time") <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time") > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time") > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time) >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time") >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time") >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'3d'::interval) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'3d'::interval) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'3d'::interval) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'3d'::interval) >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'2000-01-10'::timestamptz) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'2000-01-10'::timestamptz) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'2000-01-10'::timestamptz) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'2000-01-10'::timestamptz) >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'Europe/Berlin') < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'Europe/Berlin') <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'Europe/Berlin') > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'Europe/Berlin') >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) \qecho test overflow behaviour of time_bucket exclusion test overflow behaviour of time_bucket exclusion :PREFIX SELECT * FROM hyper WHERE time > 950 AND time_bucket(10, time) < '9223372036854775807'::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_96_chunk Filter: (("time" > 950) AND (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint)) -> Seq Scan on _hyper_1_97_chunk Filter: (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint) -> Seq Scan on _hyper_1_98_chunk Filter: (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint) -> Seq Scan on _hyper_1_99_chunk Filter: (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint) -> Seq Scan on _hyper_1_100_chunk Filter: (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint) -> Seq Scan on _hyper_1_101_chunk Filter: (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint) -> Seq Scan on _hyper_1_102_chunk Filter: (("time" > 950) AND (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint)) \qecho test timestamp upper boundary test timestamp upper boundary \qecho there should be no transformation if we are out of the supported (TimescaleDB-specific) range there should be no transformation if we are out of the supported (TimescaleDB-specific) range :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('1d',time) < '294276-01-01'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) \qecho transformation would be out of range transformation would be out of range :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('1000d',time) < '294276-01-01'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) \qecho test timestamptz upper boundary test timestamptz upper boundary \qecho there should be no transformation if we are out of the supported (TimescaleDB-specific) range there should be no transformation if we are out of the supported (TimescaleDB-specific) range :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1d',time) < '294276-01-01'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) \qecho transformation would be out of range transformation would be out of range :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1000d',time) < '294276-01-01'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) \qecho time_bucket exclusion with run-time constants time_bucket exclusion with run-time constants -- These queries have a stable time_bucket expression, because the text::interval conversion is stable. PREPARE P1(text , text) AS SELECT * FROM metrics_timestamptz WHERE time_bucket($1::interval, time) > $2::timestamptz ORDER BY time; PREPARE P2(text , text) AS SELECT * FROM metrics_timestamp WHERE time_bucket($1::interval, time) > $2::timestamptz ORDER BY time; PREPARE P3(text , text) AS SELECT * FROM metrics_timestamptz WHERE time_bucket($1::interval, time) > $2::timestamp ORDER BY time; PREPARE P4(text , text) AS SELECT * FROM metrics_timestamp WHERE time_bucket($1::interval, time) > $2::timestamp ORDER BY time; -- These queries have an immutable time_bucket expression, because the parameter is passed as interval, and no conversion is involved. PREPARE P5(interval, text) AS SELECT * FROM metrics_timestamptz WHERE time_bucket($1::interval, time) > $2::timestamptz ORDER BY time; PREPARE P6(interval, text) AS SELECT * FROM metrics_timestamp WHERE time_bucket($1::interval, time) > $2::timestamptz ORDER BY time; PREPARE P7(interval, text) AS SELECT * FROM metrics_timestamptz WHERE time_bucket($1::interval, time) > $2::timestamp ORDER BY time; PREPARE P8(interval, text) AS SELECT * FROM metrics_timestamp WHERE time_bucket($1::interval, time) > $2::timestamp ORDER BY time; SET plan_cache_mode TO 'force_custom_plan'; :PREFIX EXECUTE P1('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P2('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P3('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P4('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P5('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P6('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P7('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P8('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P1('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) :PREFIX EXECUTE P2('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) :PREFIX EXECUTE P3('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) :PREFIX EXECUTE P4('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) :PREFIX EXECUTE P5('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) :PREFIX EXECUTE P6('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) :PREFIX EXECUTE P7('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) :PREFIX EXECUTE P8('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) SET plan_cache_mode TO 'force_generic_plan'; :PREFIX EXECUTE P1('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P2('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P3('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P4('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P5('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P6('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P7('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P8('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P1('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) :PREFIX EXECUTE P2('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) :PREFIX EXECUTE P3('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) :PREFIX EXECUTE P4('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) :PREFIX EXECUTE P5('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) :PREFIX EXECUTE P6('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) :PREFIX EXECUTE P7('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) :PREFIX EXECUTE P8('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) RESET plan_cache_mode; DEALLOCATE P1; DEALLOCATE P2; DEALLOCATE P3; DEALLOCATE P4; DEALLOCATE P5; DEALLOCATE P6; DEALLOCATE P7; DEALLOCATE P8; :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) > 10 AND time_bucket(10, time) < 100 ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_2_chunk Filter: (("time" > '10'::bigint) AND ("time" < '100'::bigint) AND (time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_3_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_4_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_5_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_6_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_7_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_8_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_9_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_10_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) > 10 AND time_bucket(10, time) < 20 ORDER BY time; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk."time" -> Seq Scan on _hyper_1_2_chunk Filter: (("time" > '10'::bigint) AND ("time" < '20'::bigint) AND (time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 20)) :PREFIX SELECT * FROM hyper WHERE time_bucket(1, time) > 11 AND time_bucket(1, time) < 19 ORDER BY time; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk."time" -> Seq Scan on _hyper_1_2_chunk Filter: (("time" > '11'::bigint) AND ("time" < '19'::bigint) AND (time_bucket('1'::bigint, "time") > 11) AND (time_bucket('1'::bigint, "time") < 19)) :PREFIX SELECT * FROM hyper WHERE 10 < time_bucket(10, time) AND 20 > time_bucket(10,time) ORDER BY time; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk."time" -> Seq Scan on _hyper_1_2_chunk Filter: (("time" > '10'::bigint) AND ("time" < '20'::bigint) AND (10 < time_bucket('10'::bigint, "time")) AND (20 > time_bucket('10'::bigint, "time"))) \qecho time_bucket exclusion with date time_bucket exclusion with date :PREFIX SELECT * FROM metrics_date WHERE time_bucket('1d',time) < '2000-01-03' ORDER BY time; --- QUERY PLAN --- Index Only Scan Backward using _hyper_8_171_chunk_metrics_date_time_idx on _hyper_8_171_chunk Index Cond: ("time" < '01-03-2000'::date) Filter: (time_bucket('@ 1 day'::interval, "time") < '01-03-2000'::date) :PREFIX SELECT * FROM metrics_date WHERE time_bucket('1d',time) >= '2000-01-03' AND time_bucket('1d',time) <= '2000-01-10' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date Order: metrics_date."time" -> Index Only Scan Backward using _hyper_8_171_chunk_metrics_date_time_idx on _hyper_8_171_chunk Index Cond: (("time" >= '01-03-2000'::date) AND ("time" <= '01-11-2000'::date)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= '01-03-2000'::date) AND (time_bucket('@ 1 day'::interval, "time") <= '01-10-2000'::date)) -> Index Only Scan Backward using _hyper_8_172_chunk_metrics_date_time_idx on _hyper_8_172_chunk Index Cond: (("time" >= '01-03-2000'::date) AND ("time" <= '01-11-2000'::date)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= '01-03-2000'::date) AND (time_bucket('@ 1 day'::interval, "time") <= '01-10-2000'::date)) \qecho time_bucket exclusion with timestamp time_bucket exclusion with timestamp :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('1d',time) < '2000-01-03' ORDER BY time; --- QUERY PLAN --- Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Index Cond: ("time" < 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 1 day'::interval, "time") < 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('1d',time) >= '2000-01-03' AND time_bucket('1d',time) <= '2000-01-10' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) AND ("time" <= 'Tue Jan 11 00:00:00 2000'::timestamp without time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) AND (time_bucket('@ 1 day'::interval, "time") <= 'Mon Jan 10 00:00:00 2000'::timestamp without time zone)) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) AND ("time" <= 'Tue Jan 11 00:00:00 2000'::timestamp without time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) AND (time_bucket('@ 1 day'::interval, "time") <= 'Mon Jan 10 00:00:00 2000'::timestamp without time zone)) \qecho time_bucket exclusion with timestamptz time_bucket exclusion with timestamptz :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('6h',time) < '2000-01-03' ORDER BY time; --- QUERY PLAN --- Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: ("time" < 'Mon Jan 03 06:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 6 hours'::interval, "time") < 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('6h',time) >= '2000-01-03' AND time_bucket('6h',time) <= '2000-01-10' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Mon Jan 10 06:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 6 hours'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 6 hours'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Mon Jan 10 06:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 6 hours'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 6 hours'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho time_bucket exclusion with timestamptz and day interval time_bucket exclusion with timestamptz and day interval :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1d',time) < '2000-01-03' ORDER BY time; --- QUERY PLAN --- Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: ("time" < 'Tue Jan 04 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 1 day'::interval, "time") < 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1d',time) >= '2000-01-03' AND time_bucket('1d',time) <= '2000-01-10' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Tue Jan 11 00:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 1 day'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Tue Jan 11 00:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 1 day'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1d',time) >= '2000-01-03' AND time_bucket('7d',time) <= '2000-01-10' ORDER BY time; --- QUERY PLAN --- Sort Sort Key: metrics_timestamptz."time" -> Append -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Mon Jan 17 00:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 7 days'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Seq Scan on _hyper_6_161_chunk Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 7 days'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Mon Jan 17 00:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 7 days'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho no transformation no transformation :PREFIX SELECT * FROM hyper WHERE time_bucket(10 + floor(random())::int, time) > 10 AND time_bucket(10 + floor(random())::int, time) < 100 AND time < 150 ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Custom Scan (ChunkAppend) on hyper Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_2_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_3_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_4_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_5_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_6_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_7_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_8_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_9_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_10_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_11_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_12_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_13_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_14_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_15_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) \qecho exclude chunks based on time column with partitioning function. This exclude chunks based on time column with partitioning function. This \qecho transparently applies the time partitioning function on the time transparently applies the time partitioning function on the time \qecho value to be able to exclude chunks (similar to a closed dimension). value to be able to exclude chunks (similar to a closed dimension). :PREFIX SELECT * FROM hyper_timefunc WHERE time < 4 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_timefunc.value -> Append -> Seq Scan on _hyper_4_124_chunk Filter: ("time" < '4'::double precision) -> Seq Scan on _hyper_4_125_chunk Filter: ("time" < '4'::double precision) -> Seq Scan on _hyper_4_126_chunk Filter: ("time" < '4'::double precision) -> Seq Scan on _hyper_4_127_chunk Filter: ("time" < '4'::double precision) \qecho excluding based on time expression is currently unoptimized excluding based on time expression is currently unoptimized :PREFIX SELECT * FROM hyper_timefunc WHERE unix_to_timestamp(time) < 'Wed Dec 31 16:00:04 1969 PST' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_timefunc.value -> Append -> Seq Scan on _hyper_4_124_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_125_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_126_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_127_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_128_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_129_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_130_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_131_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_132_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_133_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_134_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_135_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_136_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_137_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_138_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_139_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_140_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_141_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_142_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_143_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_144_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_145_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_146_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_147_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_148_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_149_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_150_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_151_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_152_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_153_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_154_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) \qecho test qual propagation for joins test qual propagation for joins RESET constraint_exclusion; \qecho nothing to propagate nothing to propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1, metrics_timestamptz_2 m2 WHERE m1.time = m2.time ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time ORDER BY m1.time; --- QUERY PLAN --- Merge Left Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time ORDER BY m1.time; --- QUERY PLAN --- Sort Sort Key: m1."time" -> Merge Right Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 \qecho OR constraints should not propagate OR constraints should not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' OR m1.time > '2001-01-01' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Filter: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) OR ("time" > 'Mon Jan 01 00:00:00 2001 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Filter: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) OR ("time" > 'Mon Jan 01 00:00:00 2001 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 \qecho test single constraint test single constraint \qecho constraint should be on both scans constraint should be on both scans \qecho these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column :PREFIX SELECT m1.time FROM metrics_timestamptz m1, metrics_timestamptz_2 m2 WHERE m1.time = m2.time AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Left Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho test 2 constraints on single relation test 2 constraints on single relation \qecho these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column :PREFIX SELECT m1.time FROM metrics_timestamptz m1, metrics_timestamptz_2 m2 WHERE m1.time = m2.time AND m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Nested Loop Left Join -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Append -> Index Only Scan using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 Index Cond: ("time" = m1."time") :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho test 2 constraints with 1 constraint on each relation test 2 constraints with 1 constraint on each relation \qecho these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column :PREFIX SELECT m1.time FROM metrics_timestamptz m1, metrics_timestamptz_2 m2 WHERE m1.time = m2.time AND m1.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) \qecho test constraints in ON clause of INNER JOIN test constraints in ON clause of INNER JOIN :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho test constraints in ON clause of LEFT JOIN test constraints in ON clause of LEFT JOIN \qecho must not propagate must not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Left Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho test constraints in ON clause of RIGHT JOIN test constraints in ON clause of RIGHT JOIN \qecho must not propagate must not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Gather Merge Workers Planned: 2 -> Sort Sort Key: m1."time" -> Parallel Hash Left Join Hash Cond: (m2."time" = m1."time") Join Filter: ((m2."time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND (m2."time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Parallel Append -> Parallel Seq Scan on _hyper_7_165_chunk m2_1 -> Parallel Seq Scan on _hyper_7_166_chunk m2_2 -> Parallel Seq Scan on _hyper_7_167_chunk m2_3 -> Parallel Seq Scan on _hyper_7_168_chunk m2_4 -> Parallel Seq Scan on _hyper_7_169_chunk m2_5 -> Parallel Seq Scan on _hyper_7_170_chunk m2_6 -> Parallel Hash -> Parallel Append -> Parallel Seq Scan on _hyper_6_160_chunk m1_1 -> Parallel Seq Scan on _hyper_6_161_chunk m1_2 -> Parallel Seq Scan on _hyper_6_162_chunk m1_3 -> Parallel Seq Scan on _hyper_6_163_chunk m1_4 -> Parallel Seq Scan on _hyper_6_164_chunk m1_5 \qecho test equality condition not in ON clause test equality condition not in ON clause :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON true WHERE m2.time = m1.time AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho test constraints not joined on test constraints not joined on \qecho device_id constraint must not propagate device_id constraint must not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON true WHERE m2.time = m1.time AND m2.time < '2000-01-10' AND m1.device_id = 1 ORDER BY m1.time; --- QUERY PLAN --- Sort Sort Key: m1."time" -> Nested Loop -> Append -> Seq Scan on _hyper_6_160_chunk m1_1 Filter: (device_id = 1) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 1) -> Append -> Index Only Scan using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" = m1."time") AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho test multiple join conditions test multiple join conditions \qecho device_id constraint should propagate device_id constraint should propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON true WHERE m2.time = m1.time AND m1.device_id = m2.device_id AND m2.time < '2000-01-10' AND m1.device_id = 1 ORDER BY m1.time; --- QUERY PLAN --- Sort Sort Key: m1."time" -> Nested Loop -> Append -> Seq Scan on _hyper_6_160_chunk m1_1 Filter: (device_id = 1) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 1) -> Append -> Index Scan using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: ("time" = m1."time") Filter: (device_id = 1) -> Index Scan using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" = m1."time") AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) Filter: (device_id = 1) \qecho test join with 3 tables test join with 3 tables :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time INNER JOIN metrics_timestamptz m3 ON m2.time=m3.time WHERE m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Nested Loop -> Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Append -> Index Only Scan using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m3_1 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m3_2 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m3_3 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m3_4 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m3_5 Index Cond: ("time" = m1."time") \qecho test non-Const constraints test non-Const constraints :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10'::text::timestamptz ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" Chunks excluded during startup: 3 -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" Chunks excluded during startup: 4 -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) \qecho test now() test now() :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < now() ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 Index Cond: ("time" < now()) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 Index Cond: ("time" < now()) \qecho test volatile function test volatile function \qecho should not propagate should not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < clock_timestamp() ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 Filter: ("time" < clock_timestamp()) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m2.time < clock_timestamp() ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m2."time" = m1."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 Filter: ("time" < clock_timestamp()) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 \qecho test JOINs with normal table test JOINs with normal table \qecho will not propagate because constraints are only added to hypertables will not propagate because constraints are only added to hypertables :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN regular_timestamptz m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Sort Sort Key: m2."time" -> Seq Scan on regular_timestamptz m2 \qecho test JOINs with normal table test JOINs with normal table :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN regular_timestamptz m2 ON m1.time = m2.time WHERE m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m2."time" = m1."time") -> Sort Sort Key: m2."time" -> Seq Scan on regular_timestamptz m2 Filter: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho test quals are not pushed into OUTER JOIN test quals are not pushed into OUTER JOIN CREATE TABLE outer_join_1 (id int, name text,time timestamptz NOT NULL DEFAULT '2000-01-01'); CREATE TABLE outer_join_2 (id int, name text,time timestamptz NOT NULL DEFAULT '2000-01-01'); SELECT (SELECT table_name FROM create_hypertable(tbl, 'time')) FROM (VALUES ('outer_join_1'),('outer_join_2')) v(tbl); table_name -------------- outer_join_1 outer_join_2 INSERT INTO outer_join_1 VALUES(1,'a'), (2,'b'); INSERT INTO outer_join_2 VALUES(1,'a'); :PREFIX SELECT one.id, two.name FROM outer_join_1 one LEFT OUTER JOIN outer_join_2 two ON one.id=two.id WHERE one.id=2; --- QUERY PLAN --- Nested Loop Left Join -> Seq Scan on _hyper_9_176_chunk one Filter: (id = 2) -> Materialize -> Seq Scan on _hyper_10_177_chunk two Filter: (id = 2) :PREFIX SELECT one.id, two.name FROM outer_join_2 two RIGHT OUTER JOIN outer_join_1 one ON one.id=two.id WHERE one.id=2; --- QUERY PLAN --- Nested Loop Left Join -> Seq Scan on _hyper_9_176_chunk one Filter: (id = 2) -> Materialize -> Seq Scan on _hyper_10_177_chunk two Filter: (id = 2) DROP TABLE outer_join_1; DROP TABLE outer_join_2; -- test UNION between regular table and hypertable SELECT time FROM regular_timestamptz UNION SELECT time FROM metrics_timestamptz ORDER BY 1; time ------------------------------ Sat Jan 01 00:00:00 2000 PST Sun Jan 02 00:00:00 2000 PST Mon Jan 03 00:00:00 2000 PST Tue Jan 04 00:00:00 2000 PST Wed Jan 05 00:00:00 2000 PST Thu Jan 06 00:00:00 2000 PST Fri Jan 07 00:00:00 2000 PST Sat Jan 08 00:00:00 2000 PST Sun Jan 09 00:00:00 2000 PST Mon Jan 10 00:00:00 2000 PST Tue Jan 11 00:00:00 2000 PST Wed Jan 12 00:00:00 2000 PST Thu Jan 13 00:00:00 2000 PST Fri Jan 14 00:00:00 2000 PST Sat Jan 15 00:00:00 2000 PST Sun Jan 16 00:00:00 2000 PST Mon Jan 17 00:00:00 2000 PST Tue Jan 18 00:00:00 2000 PST Wed Jan 19 00:00:00 2000 PST Thu Jan 20 00:00:00 2000 PST Fri Jan 21 00:00:00 2000 PST Sat Jan 22 00:00:00 2000 PST Sun Jan 23 00:00:00 2000 PST Mon Jan 24 00:00:00 2000 PST Tue Jan 25 00:00:00 2000 PST Wed Jan 26 00:00:00 2000 PST Thu Jan 27 00:00:00 2000 PST Fri Jan 28 00:00:00 2000 PST Sat Jan 29 00:00:00 2000 PST Sun Jan 30 00:00:00 2000 PST Mon Jan 31 00:00:00 2000 PST Tue Feb 01 00:00:00 2000 PST -- test UNION ALL between regular table and hypertable SELECT time FROM regular_timestamptz UNION ALL SELECT time FROM metrics_timestamptz ORDER BY 1; time ------------------------------ Sat Jan 01 00:00:00 2000 PST Sat Jan 01 00:00:00 2000 PST Sat Jan 01 00:00:00 2000 PST Sat Jan 01 00:00:00 2000 PST Sun Jan 02 00:00:00 2000 PST Sun Jan 02 00:00:00 2000 PST Sun Jan 02 00:00:00 2000 PST Sun Jan 02 00:00:00 2000 PST Mon Jan 03 00:00:00 2000 PST Mon Jan 03 00:00:00 2000 PST Mon Jan 03 00:00:00 2000 PST Mon Jan 03 00:00:00 2000 PST Tue Jan 04 00:00:00 2000 PST Tue Jan 04 00:00:00 2000 PST Tue Jan 04 00:00:00 2000 PST Tue Jan 04 00:00:00 2000 PST Wed Jan 05 00:00:00 2000 PST Wed Jan 05 00:00:00 2000 PST Wed Jan 05 00:00:00 2000 PST Wed Jan 05 00:00:00 2000 PST Thu Jan 06 00:00:00 2000 PST Thu Jan 06 00:00:00 2000 PST Thu Jan 06 00:00:00 2000 PST Thu Jan 06 00:00:00 2000 PST Fri Jan 07 00:00:00 2000 PST Fri Jan 07 00:00:00 2000 PST Fri Jan 07 00:00:00 2000 PST Fri Jan 07 00:00:00 2000 PST Sat Jan 08 00:00:00 2000 PST Sat Jan 08 00:00:00 2000 PST Sat Jan 08 00:00:00 2000 PST Sat Jan 08 00:00:00 2000 PST Sun Jan 09 00:00:00 2000 PST Sun Jan 09 00:00:00 2000 PST Sun Jan 09 00:00:00 2000 PST Sun Jan 09 00:00:00 2000 PST Mon Jan 10 00:00:00 2000 PST Mon Jan 10 00:00:00 2000 PST Mon Jan 10 00:00:00 2000 PST Mon Jan 10 00:00:00 2000 PST Tue Jan 11 00:00:00 2000 PST Tue Jan 11 00:00:00 2000 PST Tue Jan 11 00:00:00 2000 PST Tue Jan 11 00:00:00 2000 PST Wed Jan 12 00:00:00 2000 PST Wed Jan 12 00:00:00 2000 PST Wed Jan 12 00:00:00 2000 PST Wed Jan 12 00:00:00 2000 PST Thu Jan 13 00:00:00 2000 PST Thu Jan 13 00:00:00 2000 PST Thu Jan 13 00:00:00 2000 PST Thu Jan 13 00:00:00 2000 PST Fri Jan 14 00:00:00 2000 PST Fri Jan 14 00:00:00 2000 PST Fri Jan 14 00:00:00 2000 PST Fri Jan 14 00:00:00 2000 PST Sat Jan 15 00:00:00 2000 PST Sat Jan 15 00:00:00 2000 PST Sat Jan 15 00:00:00 2000 PST Sat Jan 15 00:00:00 2000 PST Sun Jan 16 00:00:00 2000 PST Sun Jan 16 00:00:00 2000 PST Sun Jan 16 00:00:00 2000 PST Sun Jan 16 00:00:00 2000 PST Mon Jan 17 00:00:00 2000 PST Mon Jan 17 00:00:00 2000 PST Mon Jan 17 00:00:00 2000 PST Mon Jan 17 00:00:00 2000 PST Tue Jan 18 00:00:00 2000 PST Tue Jan 18 00:00:00 2000 PST Tue Jan 18 00:00:00 2000 PST Tue Jan 18 00:00:00 2000 PST Wed Jan 19 00:00:00 2000 PST Wed Jan 19 00:00:00 2000 PST Wed Jan 19 00:00:00 2000 PST Wed Jan 19 00:00:00 2000 PST Thu Jan 20 00:00:00 2000 PST Thu Jan 20 00:00:00 2000 PST Thu Jan 20 00:00:00 2000 PST Thu Jan 20 00:00:00 2000 PST Fri Jan 21 00:00:00 2000 PST Fri Jan 21 00:00:00 2000 PST Fri Jan 21 00:00:00 2000 PST Fri Jan 21 00:00:00 2000 PST Sat Jan 22 00:00:00 2000 PST Sat Jan 22 00:00:00 2000 PST Sat Jan 22 00:00:00 2000 PST Sat Jan 22 00:00:00 2000 PST Sun Jan 23 00:00:00 2000 PST Sun Jan 23 00:00:00 2000 PST Sun Jan 23 00:00:00 2000 PST Sun Jan 23 00:00:00 2000 PST Mon Jan 24 00:00:00 2000 PST Mon Jan 24 00:00:00 2000 PST Mon Jan 24 00:00:00 2000 PST Mon Jan 24 00:00:00 2000 PST Tue Jan 25 00:00:00 2000 PST Tue Jan 25 00:00:00 2000 PST Tue Jan 25 00:00:00 2000 PST Tue Jan 25 00:00:00 2000 PST Wed Jan 26 00:00:00 2000 PST Wed Jan 26 00:00:00 2000 PST Wed Jan 26 00:00:00 2000 PST Wed Jan 26 00:00:00 2000 PST Thu Jan 27 00:00:00 2000 PST Thu Jan 27 00:00:00 2000 PST Thu Jan 27 00:00:00 2000 PST Thu Jan 27 00:00:00 2000 PST Fri Jan 28 00:00:00 2000 PST Fri Jan 28 00:00:00 2000 PST Fri Jan 28 00:00:00 2000 PST Fri Jan 28 00:00:00 2000 PST Sat Jan 29 00:00:00 2000 PST Sat Jan 29 00:00:00 2000 PST Sat Jan 29 00:00:00 2000 PST Sat Jan 29 00:00:00 2000 PST Sun Jan 30 00:00:00 2000 PST Sun Jan 30 00:00:00 2000 PST Sun Jan 30 00:00:00 2000 PST Sun Jan 30 00:00:00 2000 PST Mon Jan 31 00:00:00 2000 PST Mon Jan 31 00:00:00 2000 PST Mon Jan 31 00:00:00 2000 PST Mon Jan 31 00:00:00 2000 PST Tue Feb 01 00:00:00 2000 PST Tue Feb 01 00:00:00 2000 PST Tue Feb 01 00:00:00 2000 PST Tue Feb 01 00:00:00 2000 PST -- test nested join qual propagation :PREFIX SELECT * FROM ( SELECT o1_m1.time FROM metrics_timestamptz o1_m1 INNER JOIN metrics_timestamptz_2 o1_m2 ON true WHERE o1_m2.time = o1_m1.time AND o1_m1.device_id = o1_m2.device_id AND o1_m2.time < '2000-01-10' AND o1_m1.device_id = 1 ) o1 FULL OUTER JOIN ( SELECT o2_m1.time FROM metrics_timestamptz o2_m1 FULL OUTER JOIN metrics_timestamptz_2 o2_m2 ON true WHERE o2_m2.time = o2_m1.time AND o2_m1.device_id = o2_m2.device_id AND o2_m2.time > '2000-01-20' AND o2_m1.device_id = 2 ) o2 ON o1.time = o2.time ORDER BY 1,2; --- QUERY PLAN --- Sort Sort Key: o1_m1."time", o2_m1."time" -> Hash Full Join Hash Cond: (o2_m1."time" = o1_m1."time") -> Nested Loop -> Append -> Index Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk o2_m2_1 Index Cond: ("time" > 'Thu Jan 20 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 2) -> Seq Scan on _hyper_7_169_chunk o2_m2_2 Filter: (device_id = 2) -> Seq Scan on _hyper_7_170_chunk o2_m2_3 Filter: (device_id = 2) -> Append -> Index Scan using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk o2_m1_1 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk o2_m1_2 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk o2_m1_3 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk o2_m1_4 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk o2_m1_5 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Hash -> Nested Loop -> Append -> Seq Scan on _hyper_7_165_chunk o1_m2_1 Filter: (device_id = 1) -> Index Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk o1_m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 1) -> Append -> Index Scan using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk o1_m1_1 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk o1_m1_2 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk o1_m1_3 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk o1_m1_4 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk o1_m1_5 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) :PREFIX SELECT * FROM ( SELECT o1_m1.time FROM metrics_timestamptz o1_m1 INNER JOIN metrics_timestamptz_2 o1_m2 ON o1_m2.time = o1_m1.time AND o1_m1.device_id = o1_m2.device_id WHERE o1_m2.time < '2000-01-10' AND o1_m1.device_id = 1 ) o1 FULL OUTER JOIN ( SELECT o2_m1.time FROM metrics_timestamptz o2_m1 FULL OUTER JOIN metrics_timestamptz_2 o2_m2 ON o2_m2.time = o2_m1.time AND o2_m1.device_id = o2_m2.device_id WHERE o2_m2.time > '2000-01-20' AND o2_m1.device_id = 2 ) o2 ON o1.time = o2.time ORDER BY 1,2; --- QUERY PLAN --- Sort Sort Key: o1_m1."time", o2_m1."time" -> Hash Full Join Hash Cond: (o2_m1."time" = o1_m1."time") -> Nested Loop -> Append -> Index Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk o2_m2_1 Index Cond: ("time" > 'Thu Jan 20 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 2) -> Seq Scan on _hyper_7_169_chunk o2_m2_2 Filter: (device_id = 2) -> Seq Scan on _hyper_7_170_chunk o2_m2_3 Filter: (device_id = 2) -> Append -> Index Scan using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk o2_m1_1 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk o2_m1_2 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk o2_m1_3 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk o2_m1_4 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk o2_m1_5 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Hash -> Nested Loop -> Append -> Seq Scan on _hyper_7_165_chunk o1_m2_1 Filter: (device_id = 1) -> Index Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk o1_m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 1) -> Append -> Index Scan using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk o1_m1_1 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk o1_m1_2 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk o1_m1_3 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk o1_m1_4 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk o1_m1_5 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) \set ECHO errors RESET timescaledb.enable_optimizations; CREATE TABLE t(time timestamptz NOT NULL); SELECT table_name FROM create_hypertable('t','time'); table_name ------------ t INSERT INTO t VALUES ('2000-01-01'), ('2010-01-01'), ('2020-01-01'); EXPLAIN (buffers off, costs off) SELECT * FROM t t1 INNER JOIN t t2 ON t1.time = t2.time WHERE t1.time < timestamptz '2010-01-01'; --- QUERY PLAN --- Merge Join Merge Cond: (t1."time" = t2."time") -> Merge Append Sort Key: t1."time" -> Index Only Scan Backward using _hyper_15_182_chunk_t_time_idx on _hyper_15_182_chunk t1_1 -> Index Only Scan Backward using _hyper_15_183_chunk_t_time_idx on _hyper_15_183_chunk t1_2 Index Cond: ("time" < 'Fri Jan 01 00:00:00 2010 PST'::timestamp with time zone) -> Materialize -> Merge Append Sort Key: t2."time" -> Index Only Scan Backward using _hyper_15_182_chunk_t_time_idx on _hyper_15_182_chunk t2_1 -> Index Only Scan Backward using _hyper_15_183_chunk_t_time_idx on _hyper_15_183_chunk t2_2 Index Cond: ("time" < 'Fri Jan 01 00:00:00 2010 PST'::timestamp with time zone) SET timescaledb.enable_qual_propagation TO false; EXPLAIN (buffers off, costs off) SELECT * FROM t t1 INNER JOIN t t2 ON t1.time = t2.time WHERE t1.time < timestamptz '2010-01-01'; --- QUERY PLAN --- Merge Join Merge Cond: (t1."time" = t2."time") -> Merge Append Sort Key: t1."time" -> Index Only Scan Backward using _hyper_15_182_chunk_t_time_idx on _hyper_15_182_chunk t1_1 -> Index Only Scan Backward using _hyper_15_183_chunk_t_time_idx on _hyper_15_183_chunk t1_2 Index Cond: ("time" < 'Fri Jan 01 00:00:00 2010 PST'::timestamp with time zone) -> Materialize -> Merge Append Sort Key: t2."time" -> Index Only Scan Backward using _hyper_15_182_chunk_t_time_idx on _hyper_15_182_chunk t2_1 -> Index Only Scan Backward using _hyper_15_183_chunk_t_time_idx on _hyper_15_183_chunk t2_2 -> Index Only Scan Backward using _hyper_15_184_chunk_t_time_idx on _hyper_15_184_chunk t2_3 RESET timescaledb.enable_qual_propagation; CREATE TABLE test (a int, time timestamptz NOT NULL); SELECT table_name FROM create_hypertable('public.test', 'time'); table_name ------------ test INSERT INTO test SELECT i, '2020-04-01'::date-10-i from generate_series(1,20) i; CREATE OR REPLACE FUNCTION test_f(_ts timestamptz) RETURNS SETOF test LANGUAGE SQL STABLE PARALLEL SAFE AS $f$ SELECT DISTINCT ON (a) * FROM test WHERE time >= _ts ORDER BY a, time DESC $f$; EXPLAIN (buffers off, costs off) SELECT * FROM test_f(now()); --- QUERY PLAN --- Unique -> Sort Sort Key: test.a, test."time" DESC -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 4 EXPLAIN (buffers off, costs off) SELECT * FROM test_f(now()); --- QUERY PLAN --- Unique -> Sort Sort Key: test.a, test."time" DESC -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 4 CREATE TABLE t1 (a int, b int NOT NULL); SELECT create_hypertable('t1', 'b', chunk_time_interval=>10); create_hypertable ------------------- (17,public,t1,t) CREATE TABLE t2 (a int, b int NOT NULL); SELECT create_hypertable('t2', 'b', chunk_time_interval=>10); create_hypertable ------------------- (18,public,t2,t) CREATE OR REPLACE FUNCTION f_t1(_a int, _b int) RETURNS SETOF t1 LANGUAGE SQL STABLE PARALLEL SAFE AS $function$ SELECT DISTINCT ON (a) * FROM t1 WHERE a = _a and b = _b ORDER BY a, b DESC $function$ ; CREATE OR REPLACE FUNCTION f_t2(_a int, _b int) RETURNS SETOF t2 LANGUAGE sql STABLE PARALLEL SAFE AS $function$ SELECT DISTINCT ON (j.a) j.* FROM f_t1(_a, _b) sc, t2 j WHERE j.b = _b AND j.a = _a ORDER BY j.a, j.b DESC $function$ ; CREATE OR REPLACE FUNCTION f_t1_2(_b int) RETURNS SETOF t1 LANGUAGE SQL STABLE PARALLEL SAFE AS $function$ SELECT DISTINCT ON (j.a) jt.* FROM t1 j, f_t1(j.a, _b) jt $function$; EXPLAIN (buffers off, costs off) SELECT * FROM f_t1_2(10); --- QUERY PLAN --- Subquery Scan on f_t1_2 -> Unique -> Sort Sort Key: j.a -> Nested Loop -> Seq Scan on t1 j -> Limit -> Index Scan using t1_b_idx on t1 Index Cond: (b = 10) Filter: (a = j.a) EXPLAIN (buffers off, costs off) SELECT * FROM f_t1_2(10) sc, f_t2(sc.a, 10); --- QUERY PLAN --- Nested Loop -> Unique -> Sort Sort Key: j.a -> Nested Loop -> Seq Scan on t1 j -> Limit -> Index Scan using t1_b_idx on t1 Index Cond: (b = 10) Filter: (a = j.a) -> Limit -> Nested Loop -> Limit -> Index Scan using t1_b_idx on t1 t1_1 Index Cond: (b = 10) Filter: (a = t1.a) -> Index Scan using t2_b_idx on t2 j_1 Index Cond: (b = 10) Filter: (a = t1.a) CREATE TABLE metrics_int1(time int, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval=1); INSERT INTO metrics_int1 SELECT i, i::text, i FROM generate_series(3,7) i; SELECT tableoid::regclass, time FROM metrics_int1 ORDER BY time; tableoid | time -------------------------------------------+------ _timescaledb_internal._hyper_19_189_chunk | 3 _timescaledb_internal._hyper_19_190_chunk | 4 _timescaledb_internal._hyper_19_191_chunk | 5 _timescaledb_internal._hyper_19_192_chunk | 6 _timescaledb_internal._hyper_19_193_chunk | 7 EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time >= 2; --- QUERY PLAN --- Append (actual rows=5.00 loops=1) -> Seq Scan on _hyper_19_189_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_192_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_193_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time >= 1 AND time >= 2; --- QUERY PLAN --- Append (actual rows=5.00 loops=1) -> Seq Scan on _hyper_19_189_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_192_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_193_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time >= 4; --- QUERY PLAN --- Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_192_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_193_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time > 4 AND time < 6; --- QUERY PLAN --- Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time > 4 AND time <= 6; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_192_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time >= 4 AND time < 6; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time >= 4 AND time <= 6; --- QUERY PLAN --- Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_192_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time BETWEEN 4 AND 5; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time = 5; --- QUERY PLAN --- Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) SET TIMEZONE='UTC'; CREATE TABLE metrics_tstz(time timestamptz, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval='1day'); INSERT INTO metrics_tstz SELECT '2000-01-01'::timestamptz + format('%s day',i)::interval, i::text, i FROM generate_series(2,6) i; SELECT tableoid::regclass, time FROM metrics_tstz ORDER BY time; tableoid | time -------------------------------------------+------------------------------ _timescaledb_internal._hyper_20_194_chunk | Mon Jan 03 00:00:00 2000 UTC _timescaledb_internal._hyper_20_195_chunk | Tue Jan 04 00:00:00 2000 UTC _timescaledb_internal._hyper_20_196_chunk | Wed Jan 05 00:00:00 2000 UTC _timescaledb_internal._hyper_20_197_chunk | Thu Jan 06 00:00:00 2000 UTC _timescaledb_internal._hyper_20_198_chunk | Fri Jan 07 00:00:00 2000 UTC EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-02'; --- QUERY PLAN --- Append (actual rows=5.00 loops=1) -> Seq Scan on _hyper_20_194_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_197_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_198_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-01' AND time >= '2000-01-02'; --- QUERY PLAN --- Append (actual rows=5.00 loops=1) -> Seq Scan on _hyper_20_194_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_197_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_198_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04'; --- QUERY PLAN --- Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_197_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_198_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time > '2000-01-04' AND time < '2000-01-06'; --- QUERY PLAN --- Append (actual rows=1.00 loops=1) -> Index Scan using _hyper_20_195_chunk_metrics_tstz_time_idx on _hyper_20_195_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" < 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time > '2000-01-04' AND time <= '2000-01-06'; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Index Scan using _hyper_20_195_chunk_metrics_tstz_time_idx on _hyper_20_195_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_20_197_chunk_metrics_tstz_time_idx on _hyper_20_197_chunk (actual rows=1.00 loops=1) Index Cond: (("time" > 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time < '2000-01-06'; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time <= '2000-01-06'; --- QUERY PLAN --- Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_20_197_chunk_metrics_tstz_time_idx on _hyper_20_197_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time BETWEEN '2000-01-04' AND '2000-01-05'; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_20_196_chunk_metrics_tstz_time_idx on _hyper_20_196_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Wed Jan 05 00:00:00 2000 UTC'::timestamp with time zone)) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time = '2000-01-05'; --- QUERY PLAN --- Index Scan using _hyper_20_196_chunk_metrics_tstz_time_idx on _hyper_20_196_chunk (actual rows=1.00 loops=1) Index Cond: ("time" = 'Wed Jan 05 00:00:00 2000 UTC'::timestamp with time zone) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time <= '2000-01-06' AND device = '5'; --- QUERY PLAN --- Append (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=0.00 loops=1) Filter: (device = '5'::text) Rows Removed by Filter: 1 -> Seq Scan on _hyper_20_196_chunk (actual rows=0.00 loops=1) Filter: (device = '5'::text) Rows Removed by Filter: 1 -> Index Scan using _hyper_20_197_chunk_metrics_tstz_time_idx on _hyper_20_197_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) Filter: (device = '5'::text) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time <= '2000-01-06' AND device IS NOT NULL; --- QUERY PLAN --- Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) Filter: (device IS NOT NULL) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) Filter: (device IS NOT NULL) -> Index Scan using _hyper_20_197_chunk_metrics_tstz_time_idx on _hyper_20_197_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) Filter: (device IS NOT NULL) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time <= '2000-01-06' AND time < now(); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_tstz (actual rows=3.00 loops=1) Chunks excluded during startup: 0 -> Index Scan using _hyper_20_195_chunk_metrics_tstz_time_idx on _hyper_20_195_chunk (actual rows=1.00 loops=1) Index Cond: ("time" < now()) -> Index Scan using _hyper_20_196_chunk_metrics_tstz_time_idx on _hyper_20_196_chunk (actual rows=1.00 loops=1) Index Cond: ("time" < now()) -> Index Scan using _hyper_20_197_chunk_metrics_tstz_time_idx on _hyper_20_197_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" < now())) CREATE TABLE metrics_space(time timestamptz, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval='1day'); SELECT add_dimension('metrics_space', 'device', 4); add_dimension ------------------------------------ (25,public,metrics_space,device,t) INSERT INTO metrics_space SELECT '2000-01-01'::timestamptz + format('%s day',i)::interval, i::text, i FROM generate_series(2,6) i; EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_space WHERE time >= '2000-01-02'; --- QUERY PLAN --- Append (actual rows=5.00 loops=1) -> Index Scan using _hyper_21_199_chunk_metrics_space_time_idx on _hyper_21_199_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Sun Jan 02 00:00:00 2000 UTC'::timestamp with time zone) -> Index Scan using _hyper_21_200_chunk_metrics_space_time_idx on _hyper_21_200_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Sun Jan 02 00:00:00 2000 UTC'::timestamp with time zone) -> Index Scan using _hyper_21_201_chunk_metrics_space_time_idx on _hyper_21_201_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Sun Jan 02 00:00:00 2000 UTC'::timestamp with time zone) -> Index Scan using _hyper_21_202_chunk_metrics_space_time_idx on _hyper_21_202_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Sun Jan 02 00:00:00 2000 UTC'::timestamp with time zone) -> Index Scan using _hyper_21_203_chunk_metrics_space_time_idx on _hyper_21_203_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Sun Jan 02 00:00:00 2000 UTC'::timestamp with time zone) --TEST END-- ================================================ FILE: test/expected/plan_expand_hypertable-18.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set PREFIX 'EXPLAIN (buffers off, costs off) ' \ir include/plan_expand_hypertable_load.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --single time dimension CREATE TABLE hyper ("time_broken" bigint NOT NULL, "value" integer); ALTER TABLE hyper DROP COLUMN time_broken, ADD COLUMN time BIGINT; SELECT create_hypertable('hyper', 'time', chunk_time_interval => 10); create_hypertable -------------------- (1,public,hyper,t) INSERT INTO hyper SELECT g, g FROM generate_series(0,1000) g; --insert a point with INT_MAX_64 INSERT INTO hyper (time, value) SELECT 9223372036854775807::bigint, 0; --time and space CREATE TABLE hyper_w_space ("time_broken" bigint NOT NULL, "device_id" text, "value" integer); ALTER TABLE hyper_w_space DROP COLUMN time_broken, ADD COLUMN time BIGINT; SELECT create_hypertable('hyper_w_space', 'time', 'device_id', 4, chunk_time_interval => 10); create_hypertable ---------------------------- (2,public,hyper_w_space,t) INSERT INTO hyper_w_space (time, device_id, value) SELECT g, 'dev' || g, g FROM generate_series(0,30) g; CREATE VIEW hyper_w_space_view AS (SELECT * FROM hyper_w_space); --with timestamp and space CREATE TABLE tag (id serial PRIMARY KEY, name text); CREATE TABLE hyper_ts ("time_broken" timestamptz NOT NULL, "device_id" text, tag_id INT REFERENCES tag(id), "value" integer); ALTER TABLE hyper_ts DROP COLUMN time_broken, ADD COLUMN time TIMESTAMPTZ; SELECT create_hypertable('hyper_ts', 'time', 'device_id', 2, chunk_time_interval => '10 seconds'::interval); create_hypertable ----------------------- (3,public,hyper_ts,t) INSERT INTO tag(name) SELECT 'tag'||g FROM generate_series(0,10) g; INSERT INTO hyper_ts (time, device_id, tag_id, value) SELECT to_timestamp(g), 'dev' || g, (random() /10)+1, g FROM generate_series(0,30) g; --one in the future INSERT INTO hyper_ts (time, device_id, tag_id, value) VALUES ('2100-01-01 02:03:04 PST', 'dev101', 1, 0); --time partitioning function CREATE OR REPLACE FUNCTION unix_to_timestamp(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT AS $BODY$ SELECT to_timestamp(unixtime); $BODY$; CREATE TABLE hyper_timefunc ("time" float8 NOT NULL, "device_id" text, "value" integer); SELECT create_hypertable('hyper_timefunc', 'time', 'device_id', 4, chunk_time_interval => 10, time_partitioning_func => 'unix_to_timestamp'); psql:include/plan_expand_hypertable_load.sql:57: WARNING: unexpected interval: smaller than one second create_hypertable ----------------------------- (4,public,hyper_timefunc,t) INSERT INTO hyper_timefunc (time, device_id, value) SELECT g, 'dev' || g, g FROM generate_series(0,30) g; CREATE TABLE metrics_timestamp(time timestamp); SELECT create_hypertable('metrics_timestamp','time'); psql:include/plan_expand_hypertable_load.sql:62: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------------- (5,public,metrics_timestamp,t) INSERT INTO metrics_timestamp SELECT generate_series('2000-01-01'::timestamp,'2000-02-01'::timestamp,'1d'::interval); CREATE TABLE metrics_timestamptz(time timestamptz, device_id int); SELECT create_hypertable('metrics_timestamptz','time'); create_hypertable ---------------------------------- (6,public,metrics_timestamptz,t) INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval), 1; INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval), 2; INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval), 3; --create a second table to test joins with CREATE TABLE metrics_timestamptz_2 (LIKE metrics_timestamptz); SELECT create_hypertable('metrics_timestamptz_2','time'); create_hypertable ------------------------------------ (7,public,metrics_timestamptz_2,t) INSERT INTO metrics_timestamptz_2 SELECT * FROM metrics_timestamptz; INSERT INTO metrics_timestamptz_2 VALUES ('2000-12-01'::timestamptz, 3); CREATE TABLE metrics_date(time date); SELECT create_hypertable('metrics_date','time'); create_hypertable --------------------------- (8,public,metrics_date,t) INSERT INTO metrics_date SELECT generate_series('2000-01-01'::date,'2000-02-01'::date,'1d'::interval); ANALYZE hyper; ANALYZE hyper_w_space; ANALYZE tag; ANALYZE hyper_ts; ANALYZE hyper_timefunc; -- create normal table for JOIN tests CREATE TABLE regular_timestamptz(time timestamptz); INSERT INTO regular_timestamptz SELECT generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval); \ir include/plan_expand_hypertable_query.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --we want to see how our logic excludes chunks --and not how much work constraint_exclusion does SET constraint_exclusion = 'off'; \qecho test upper bounds test upper bounds :PREFIX SELECT * FROM hyper WHERE time < 10 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_1_chunk.value -> Seq Scan on _hyper_1_1_chunk :PREFIX SELECT * FROM hyper WHERE time < 11 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk -> Seq Scan on _hyper_1_2_chunk Filter: ("time" < 11) :PREFIX SELECT * FROM hyper WHERE time = 10 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk.value -> Seq Scan on _hyper_1_2_chunk Filter: ("time" = 10) :PREFIX SELECT * FROM hyper WHERE 10 >= time ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk -> Seq Scan on _hyper_1_2_chunk Filter: (10 >= "time") \qecho test lower bounds test lower bounds :PREFIX SELECT * FROM hyper WHERE time >= 10 and time < 20 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk.value -> Seq Scan on _hyper_1_2_chunk :PREFIX SELECT * FROM hyper WHERE 10 < time and 20 >= time ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_2_chunk Filter: ((10 < "time") AND (20 >= "time")) -> Seq Scan on _hyper_1_3_chunk Filter: ((10 < "time") AND (20 >= "time")) :PREFIX SELECT * FROM hyper WHERE time >= 9 and time < 20 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 9) AND ("time" < 20)) -> Seq Scan on _hyper_1_2_chunk :PREFIX SELECT * FROM hyper WHERE time > 9 and time < 20 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk.value -> Seq Scan on _hyper_1_2_chunk \qecho test empty result test empty result :PREFIX SELECT * FROM hyper WHERE time < 0; --- QUERY PLAN --- Result One-Time Filter: false \qecho test expression evaluation test expression evaluation :PREFIX SELECT * FROM hyper WHERE time < (5*2)::smallint; --- QUERY PLAN --- Seq Scan on _hyper_1_1_chunk \qecho test logic at INT64_MAX test logic at INT64_MAX :PREFIX SELECT * FROM hyper WHERE time = 9223372036854775807::bigint ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_102_chunk.value -> Seq Scan on _hyper_1_102_chunk Filter: ("time" = '9223372036854775807'::bigint) :PREFIX SELECT * FROM hyper WHERE time = 9223372036854775806::bigint ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_102_chunk.value -> Seq Scan on _hyper_1_102_chunk Filter: ("time" = '9223372036854775806'::bigint) :PREFIX SELECT * FROM hyper WHERE time >= 9223372036854775807::bigint ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_102_chunk.value -> Seq Scan on _hyper_1_102_chunk Filter: ("time" >= '9223372036854775807'::bigint) :PREFIX SELECT * FROM hyper WHERE time > 9223372036854775807::bigint ORDER BY value; --- QUERY PLAN --- Sort Sort Key: value -> Result One-Time Filter: false :PREFIX SELECT * FROM hyper WHERE time > 9223372036854775806::bigint ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_102_chunk.value -> Seq Scan on _hyper_1_102_chunk Filter: ("time" > '9223372036854775806'::bigint) \qecho cte cte :PREFIX WITH cte AS( SELECT * FROM hyper WHERE time < 10 ) SELECT * FROM cte ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_1_1_chunk.value -> Seq Scan on _hyper_1_1_chunk \qecho subquery subquery :PREFIX SELECT 0 = ANY (SELECT value FROM hyper WHERE time < 10); --- QUERY PLAN --- Result SubPlan 1 -> Seq Scan on _hyper_1_1_chunk \qecho no space constraint no space constraint :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" < 10) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" < 10) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" < 10) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" < 10) \qecho valid space constraint valid space constraint :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 and device_id = 'dev5' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 and 'dev5' = device_id ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND ('dev5'::text = device_id)) :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 and 'dev'||(2+3) = device_id ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND ('dev5'::text = device_id)) \qecho only space constraint only space constraint :PREFIX SELECT * FROM hyper_w_space WHERE 'dev5' = device_id ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_106_chunk Filter: ('dev5'::text = device_id) -> Seq Scan on _hyper_2_109_chunk Filter: ('dev5'::text = device_id) -> Seq Scan on _hyper_2_111_chunk Filter: ('dev5'::text = device_id) \qecho unhandled space constraint unhandled space constraint :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 and device_id > 'dev5' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: (("time" < 10) AND (device_id > 'dev5'::text)) -> Seq Scan on _hyper_2_104_chunk Filter: (("time" < 10) AND (device_id > 'dev5'::text)) -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND (device_id > 'dev5'::text)) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id > 'dev5'::text)) \qecho use of OR - does not filter chunks use of OR - does not filter chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND (device_id = 'dev5' or device_id = 'dev6') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: (("time" < 10) AND ((device_id = 'dev5'::text) OR (device_id = 'dev6'::text))) -> Seq Scan on _hyper_2_104_chunk Filter: (("time" < 10) AND ((device_id = 'dev5'::text) OR (device_id = 'dev6'::text))) -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND ((device_id = 'dev5'::text) OR (device_id = 'dev6'::text))) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND ((device_id = 'dev5'::text) OR (device_id = 'dev6'::text))) \qecho cte cte :PREFIX WITH cte AS( SELECT * FROM hyper_w_space WHERE time < 10 and device_id = 'dev5' ) SELECT * FROM cte ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) \qecho subquery subquery :PREFIX SELECT 0 = ANY (SELECT value FROM hyper_w_space WHERE time < 10 and device_id = 'dev5'); --- QUERY PLAN --- Result SubPlan 1 -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) \qecho view view :PREFIX SELECT * FROM hyper_w_space_view WHERE time < 10 and device_id = 'dev5' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) \qecho IN statement - simple IN statement - simple :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id IN ('dev5') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) \qecho IN statement - two chunks IN statement - two chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id IN ('dev5','dev6') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev5,dev6}'::text[]))) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev5,dev6}'::text[]))) \qecho IN statement - one chunk IN statement - one chunk :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id IN ('dev4','dev5') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev4,dev5}'::text[]))) \qecho NOT IN - does not filter chunks NOT IN - does not filter chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id NOT IN ('dev5','dev6') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: (("time" < 10) AND (device_id <> ALL ('{dev5,dev6}'::text[]))) -> Seq Scan on _hyper_2_104_chunk Filter: (("time" < 10) AND (device_id <> ALL ('{dev5,dev6}'::text[]))) -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND (device_id <> ALL ('{dev5,dev6}'::text[]))) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id <> ALL ('{dev5,dev6}'::text[]))) \qecho IN statement with subquery - does not filter chunks IN statement with subquery - does not filter chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id IN (SELECT 'dev5'::text) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_106_chunk.value -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = 'dev5'::text)) \qecho ANY ANY :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id = ANY(ARRAY['dev5','dev6']) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev5,dev6}'::text[]))) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev5,dev6}'::text[]))) \qecho ANY with intersection ANY with intersection :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id = ANY(ARRAY['dev5','dev6']) AND device_id = ANY(ARRAY['dev6','dev7']) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_2_105_chunk.value -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND (device_id = ANY ('{dev5,dev6}'::text[])) AND (device_id = ANY ('{dev6,dev7}'::text[]))) \qecho ANY without intersection shouldnt scan any chunks ANY without intersection shouldnt scan any chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id = ANY(ARRAY['dev5','dev6']) AND device_id = ANY(ARRAY['dev8','dev9']) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: value -> Result One-Time Filter: false \qecho ANY/IN/ALL only works for equals operator ANY/IN/ALL only works for equals operator :PREFIX SELECT * FROM hyper_w_space WHERE device_id < ANY(ARRAY['dev5','dev6']) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_104_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_105_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_106_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_107_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_108_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_109_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_110_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_111_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_112_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_113_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_114_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) -> Seq Scan on _hyper_2_115_chunk Filter: (device_id < ANY ('{dev5,dev6}'::text[])) \qecho ALL with equals and different values shouldnt scan any chunks ALL with equals and different values shouldnt scan any chunks :PREFIX SELECT * FROM hyper_w_space WHERE device_id = ALL(ARRAY['dev5','dev6']) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: value -> Result One-Time Filter: false \qecho Multi AND Multi AND :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND time < 100 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: (("time" < 10) AND ("time" < 100)) -> Seq Scan on _hyper_2_104_chunk Filter: (("time" < 10) AND ("time" < 100)) -> Seq Scan on _hyper_2_105_chunk Filter: (("time" < 10) AND ("time" < 100)) -> Seq Scan on _hyper_2_106_chunk Filter: (("time" < 10) AND ("time" < 100)) \qecho Time dimension doesnt filter chunks when using non-equality IN/ANY with multiple arguments Time dimension doesnt filter chunks when using non-equality IN/ANY with multiple arguments :PREFIX SELECT * FROM hyper_w_space WHERE time < ANY(ARRAY[1,2]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_107_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_108_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_109_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_110_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_111_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_112_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_113_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_114_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) -> Seq Scan on _hyper_2_115_chunk Filter: ("time" < ANY ('{1,2}'::integer[])) \qecho Time dimension chunk exclusion with IN/ANY equality uses bounding range Time dimension chunk exclusion with IN/ANY equality uses bounding range :PREFIX SELECT * FROM hyper WHERE time IN (5, 15) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_1_2_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) :PREFIX SELECT * FROM hyper WHERE time = ANY(ARRAY[5, 15, 25]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ("time" = ANY ('{5,15,25}'::integer[])) -> Seq Scan on _hyper_1_2_chunk Filter: ("time" = ANY ('{5,15,25}'::integer[])) -> Seq Scan on _hyper_1_3_chunk Filter: ("time" = ANY ('{5,15,25}'::integer[])) :PREFIX SELECT * FROM hyper WHERE time = ANY(ARRAY[25, 15, 5]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper.value -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ("time" = ANY ('{25,15,5}'::integer[])) -> Seq Scan on _hyper_1_2_chunk Filter: ("time" = ANY ('{25,15,5}'::integer[])) -> Seq Scan on _hyper_1_3_chunk Filter: ("time" = ANY ('{25,15,5}'::integer[])) :PREFIX SELECT * FROM hyper_w_space WHERE time IN (5, 15) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_107_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_108_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_109_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) -> Seq Scan on _hyper_2_110_chunk Filter: ("time" = ANY ('{5,15}'::bigint[])) :PREFIX SELECT * FROM metrics_timestamp WHERE time IN ('2000-01-05'::timestamp, '2000-01-15'::timestamp) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000","Sat Jan 15 00:00:00 2000"}'::timestamp without time zone[])) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000","Sat Jan 15 00:00:00 2000"}'::timestamp without time zone[])) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000","Sat Jan 15 00:00:00 2000"}'::timestamp without time zone[])) :PREFIX SELECT * FROM metrics_timestamptz WHERE time IN ('2000-01-05'::timestamptz, '2000-01-15'::timestamptz) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000 PST","Sat Jan 15 00:00:00 2000 PST"}'::timestamp with time zone[])) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000 PST","Sat Jan 15 00:00:00 2000 PST"}'::timestamp with time zone[])) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Index Cond: ("time" = ANY ('{"Wed Jan 05 00:00:00 2000 PST","Sat Jan 15 00:00:00 2000 PST"}'::timestamp with time zone[])) :PREFIX SELECT * FROM metrics_date WHERE time IN ('2000-01-05'::date, '2000-01-15'::date) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date Order: metrics_date."time" -> Index Only Scan Backward using _hyper_8_171_chunk_metrics_date_time_idx on _hyper_8_171_chunk Index Cond: ("time" = ANY ('{01-05-2000,01-15-2000}'::date[])) -> Index Only Scan Backward using _hyper_8_172_chunk_metrics_date_time_idx on _hyper_8_172_chunk Index Cond: ("time" = ANY ('{01-05-2000,01-15-2000}'::date[])) -> Index Only Scan Backward using _hyper_8_173_chunk_metrics_date_time_idx on _hyper_8_173_chunk Index Cond: ("time" = ANY ('{01-05-2000,01-15-2000}'::date[])) \qecho cross-type IN/ANY: timestamp to timestamptz column does not use bounding range (stable cast) cross-type IN/ANY: timestamp to timestamptz column does not use bounding range (stable cast) :PREFIX SELECT * FROM metrics_timestamptz WHERE time IN ('2000-01-05'::timestamp, '2000-01-15'::timestamp) ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 3 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: ("time" = ANY (ARRAY[('Wed Jan 05 00:00:00 2000'::timestamp without time zone)::timestamp with time zone, ('Sat Jan 15 00:00:00 2000'::timestamp without time zone)::timestamp with time zone])) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Index Cond: ("time" = ANY (ARRAY[('Wed Jan 05 00:00:00 2000'::timestamp without time zone)::timestamp with time zone, ('Sat Jan 15 00:00:00 2000'::timestamp without time zone)::timestamp with time zone])) \qecho Time dimension chunk filtering works for ANY with single argument Time dimension chunk filtering works for ANY with single argument :PREFIX SELECT * FROM hyper_w_space WHERE time < ANY(ARRAY[1]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" < ANY ('{1}'::integer[])) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" < ANY ('{1}'::integer[])) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" < ANY ('{1}'::integer[])) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" < ANY ('{1}'::integer[])) \qecho Time dimension chunk filtering works for ALL with single argument Time dimension chunk filtering works for ALL with single argument :PREFIX SELECT * FROM hyper_w_space WHERE time < ALL(ARRAY[1]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" < ALL ('{1}'::integer[])) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" < ALL ('{1}'::integer[])) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" < ALL ('{1}'::integer[])) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" < ALL ('{1}'::integer[])) \qecho Time dimension chunk filtering works for ALL with multiple arguments Time dimension chunk filtering works for ALL with multiple arguments :PREFIX SELECT * FROM hyper_w_space WHERE time < ALL(ARRAY[1,10,20,30]) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ("time" < ALL ('{1,10,20,30}'::integer[])) -> Seq Scan on _hyper_2_104_chunk Filter: ("time" < ALL ('{1,10,20,30}'::integer[])) -> Seq Scan on _hyper_2_105_chunk Filter: ("time" < ALL ('{1,10,20,30}'::integer[])) -> Seq Scan on _hyper_2_106_chunk Filter: ("time" < ALL ('{1,10,20,30}'::integer[])) \qecho AND intersection using IN and EQUALS AND intersection using IN and EQUALS :PREFIX SELECT * FROM hyper_w_space WHERE device_id IN ('dev1','dev2') AND device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_w_space.value -> Append -> Seq Scan on _hyper_2_103_chunk Filter: ((device_id = ANY ('{dev1,dev2}'::text[])) AND (device_id = 'dev1'::text)) -> Seq Scan on _hyper_2_110_chunk Filter: ((device_id = ANY ('{dev1,dev2}'::text[])) AND (device_id = 'dev1'::text)) -> Seq Scan on _hyper_2_114_chunk Filter: ((device_id = ANY ('{dev1,dev2}'::text[])) AND (device_id = 'dev1'::text)) \qecho AND with no intersection using IN and EQUALS AND with no intersection using IN and EQUALS :PREFIX SELECT * FROM hyper_w_space WHERE device_id IN ('dev1','dev2') AND device_id = 'dev3' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: value -> Result One-Time Filter: false \qecho timestamps timestamps \qecho these should work since they are immutable functions these should work since they are immutable functions :PREFIX SELECT * FROM hyper_ts WHERE time < 'Wed Dec 31 16:00:10 1969 PST'::timestamptz ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Append -> Seq Scan on _hyper_3_116_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_3_117_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) :PREFIX SELECT * FROM hyper_ts WHERE time < to_timestamp(10) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Append -> Seq Scan on _hyper_3_116_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_3_117_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) :PREFIX SELECT * FROM hyper_ts WHERE time < 'Wed Dec 31 16:00:10 1969'::timestamp AT TIME ZONE 'PST' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Append -> Seq Scan on _hyper_3_116_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_3_117_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) :PREFIX SELECT * FROM hyper_ts WHERE time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_3_116_chunk.value -> Seq Scan on _hyper_3_116_chunk Filter: (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text)) \qecho these should not work since uses stable functions; these should not work since uses stable functions; :PREFIX SELECT * FROM hyper_ts WHERE time < 'Wed Dec 31 16:00:10 1969'::timestamp ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Custom Scan (ChunkAppend) on hyper_ts Chunks excluded during startup: 6 -> Seq Scan on _hyper_3_116_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969'::timestamp without time zone) -> Seq Scan on _hyper_3_117_chunk Filter: ("time" < 'Wed Dec 31 16:00:10 1969'::timestamp without time zone) :PREFIX SELECT * FROM hyper_ts WHERE time < ('Wed Dec 31 16:00:10 1969'::timestamp::timestamptz) ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Custom Scan (ChunkAppend) on hyper_ts Chunks excluded during startup: 6 -> Seq Scan on _hyper_3_116_chunk Filter: ("time" < ('Wed Dec 31 16:00:10 1969'::timestamp without time zone)::timestamp with time zone) -> Seq Scan on _hyper_3_117_chunk Filter: ("time" < ('Wed Dec 31 16:00:10 1969'::timestamp without time zone)::timestamp with time zone) :PREFIX SELECT * FROM hyper_ts WHERE NOW() < time ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Custom Scan (ChunkAppend) on hyper_ts Chunks excluded during startup: 7 -> Seq Scan on _hyper_3_123_chunk Filter: (now() < "time") \qecho joins joins :PREFIX SELECT * FROM hyper_ts WHERE tag_id IN (SELECT id FROM tag WHERE tag.id=1) and time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_3_116_chunk.value -> Nested Loop Semi Join -> Seq Scan on _hyper_3_116_chunk Filter: (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text) AND (tag_id = 1)) -> Seq Scan on tag Filter: (id = 1) :PREFIX SELECT * FROM hyper_ts WHERE tag_id IN (SELECT id FROM tag WHERE tag.id=1) or (time < to_timestamp(10) and device_id = 'dev1') ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_ts.value -> Custom Scan (ChunkAppend) on hyper_ts -> Seq Scan on _hyper_3_116_chunk Filter: ((ANY (tag_id = (hashed SubPlan 1).col1)) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) SubPlan 1 -> Seq Scan on tag Filter: (id = 1) -> Seq Scan on _hyper_3_117_chunk Filter: ((ANY (tag_id = (hashed SubPlan 1).col1)) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_118_chunk Filter: ((ANY (tag_id = (hashed SubPlan 1).col1)) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_119_chunk Filter: ((ANY (tag_id = (hashed SubPlan 1).col1)) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_120_chunk Filter: ((ANY (tag_id = (hashed SubPlan 1).col1)) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_121_chunk Filter: ((ANY (tag_id = (hashed SubPlan 1).col1)) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_122_chunk Filter: ((ANY (tag_id = (hashed SubPlan 1).col1)) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) -> Seq Scan on _hyper_3_123_chunk Filter: ((ANY (tag_id = (hashed SubPlan 1).col1)) OR (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text))) :PREFIX SELECT * FROM hyper_ts WHERE tag_id IN (SELECT id FROM tag WHERE tag.name='tag1') and time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_3_116_chunk.value -> Nested Loop Join Filter: (_hyper_3_116_chunk.tag_id = tag.id) -> Seq Scan on _hyper_3_116_chunk Filter: (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text)) -> Seq Scan on tag Filter: (name = 'tag1'::text) :PREFIX SELECT * FROM hyper_ts JOIN tag on (hyper_ts.tag_id = tag.id ) WHERE time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_3_116_chunk.value -> Merge Join Merge Cond: (tag.id = _hyper_3_116_chunk.tag_id) -> Index Scan using tag_pkey on tag -> Sort Sort Key: _hyper_3_116_chunk.tag_id -> Seq Scan on _hyper_3_116_chunk Filter: (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text)) :PREFIX SELECT * FROM hyper_ts JOIN tag on (hyper_ts.tag_id = tag.id ) WHERE tag.name = 'tag1' and time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: _hyper_3_116_chunk.value -> Nested Loop Join Filter: (tag.id = _hyper_3_116_chunk.tag_id) -> Seq Scan on _hyper_3_116_chunk Filter: (("time" < 'Wed Dec 31 16:00:10 1969 PST'::timestamp with time zone) AND (device_id = 'dev1'::text)) -> Seq Scan on tag Filter: (name = 'tag1'::text) \qecho test constraint exclusion for constraints in ON clause of JOINs test constraint exclusion for constraints in ON clause of JOINs \qecho should exclude chunks on m1 and propagate qual to m2 because of INNER JOIN should exclude chunks on m1 and propagate qual to m2 because of INNER JOIN :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho should exclude chunks on m2 and propagate qual to m1 because of INNER JOIN should exclude chunks on m2 and propagate qual to m1 because of INNER JOIN :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho must not exclude on m1 must not exclude on m1 :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Left Join Merge Cond: (m1."time" = m2."time") Join Filter: (m1."time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 \qecho should exclude chunks on m2 should exclude chunks on m2 :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Left Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho should exclude chunks on m1 should exclude chunks on m1 :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Sort Sort Key: m1."time" -> Merge Right Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 \qecho must not exclude chunks on m2 must not exclude chunks on m2 :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time < '2000-01-10' ORDER BY m1.time, m2.time; --- QUERY PLAN --- Sort Sort Key: m1."time", m2."time" -> Merge Left Join Merge Cond: (m2."time" = m1."time") Join Filter: (m2."time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 \qecho time_bucket exclusion time_bucket exclusion :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) < 10::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: _hyper_1_1_chunk."time" -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time") < '10'::bigint) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) < 11::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time") < '11'::bigint) -> Seq Scan on _hyper_1_2_chunk Filter: (time_bucket('10'::bigint, "time") < '11'::bigint) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" < '21'::bigint) AND (time_bucket('10'::bigint, "time") < '11'::bigint)) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) <= 10::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time") <= '10'::bigint) -> Seq Scan on _hyper_1_2_chunk Filter: (time_bucket('10'::bigint, "time") <= '10'::bigint) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" <= '20'::bigint) AND (time_bucket('10'::bigint, "time") <= '10'::bigint)) :PREFIX SELECT * FROM hyper WHERE 10::bigint > time_bucket(10, time) ORDER BY time; --- QUERY PLAN --- Sort Sort Key: _hyper_1_1_chunk."time" -> Seq Scan on _hyper_1_1_chunk Filter: ('10'::bigint > time_bucket('10'::bigint, "time")) :PREFIX SELECT * FROM hyper WHERE 11::bigint > time_bucket(10, time) ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ('11'::bigint > time_bucket('10'::bigint, "time")) -> Seq Scan on _hyper_1_2_chunk Filter: ('11'::bigint > time_bucket('10'::bigint, "time")) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" < '21'::bigint) AND ('11'::bigint > time_bucket('10'::bigint, "time"))) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time, 5) < 10::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) < '10'::bigint) -> Seq Scan on _hyper_1_2_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) < '10'::bigint) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time, 5) < 11::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) < '11'::bigint) -> Seq Scan on _hyper_1_2_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) < '11'::bigint) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" < '21'::bigint) AND (time_bucket('10'::bigint, "time", '5'::bigint) < '11'::bigint)) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time, 5) <= 10::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) <= '10'::bigint) -> Seq Scan on _hyper_1_2_chunk Filter: (time_bucket('10'::bigint, "time", '5'::bigint) <= '10'::bigint) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" <= '20'::bigint) AND (time_bucket('10'::bigint, "time", '5'::bigint) <= '10'::bigint)) :PREFIX SELECT * FROM hyper WHERE 10::bigint > time_bucket(10, time, 5) ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ('10'::bigint > time_bucket('10'::bigint, "time", '5'::bigint)) -> Seq Scan on _hyper_1_2_chunk Filter: ('10'::bigint > time_bucket('10'::bigint, "time", '5'::bigint)) :PREFIX SELECT * FROM hyper WHERE 11::bigint > time_bucket(10, time, 5) ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_1_chunk Filter: ('11'::bigint > time_bucket('10'::bigint, "time", '5'::bigint)) -> Seq Scan on _hyper_1_2_chunk Filter: ('11'::bigint > time_bucket('10'::bigint, "time", '5'::bigint)) -> Seq Scan on _hyper_1_3_chunk Filter: (("time" < '21'::bigint) AND ('11'::bigint > time_bucket('10'::bigint, "time", '5'::bigint))) \qecho timestamp time_bucket exclusion timestamp time_bucket exclusion SELECT count(DISTINCT tableoid) FROM metrics_timestamp; count ------- 5 :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time") < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time") < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time") <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time") <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time") > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time") > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time) >= '2000-01-15' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Index Cond: ("time" >= 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time") >= 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket('@ 7 days'::interval, "time") >= 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time") >= 'Sat Jan 15 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'3d'::interval) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'3d'::interval) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'3d'::interval) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'3d'::interval) >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'2000-01-10'::timestamp) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) < 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'2000-01-10'::timestamp) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) <= 'Wed Jan 05 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'2000-01-10'::timestamp) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) > 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'2000-01-10'::timestamp) >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000'::timestamp without time zone) >= 'Tue Jan 25 00:00:00 2000'::timestamp without time zone) \qecho timestamptz time_bucket exclusion timestamptz time_bucket exclusion SELECT count(DISTINCT tableoid) FROM metrics_timestamptz; count ------- 5 :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time") < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time") < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time") <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time") <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time") > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time") > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time) >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time") >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time") >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'3d'::interval) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'3d'::interval) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'3d'::interval) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'3d'::interval) >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", '@ 3 days'::interval) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'2000-01-10'::timestamptz) < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'2000-01-10'::timestamptz) <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'2000-01-10'::timestamptz) > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'2000-01-10'::timestamptz) >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'Europe/Berlin') < '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" < 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) < 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'Europe/Berlin') <= '2000-01-05' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: ("time" <= 'Wed Jan 12 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) <= 'Wed Jan 05 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'Europe/Berlin') > '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) > 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'Europe/Berlin') >= '2000-01-25' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Index Cond: ("time" >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 7 days'::interval, "time", 'Europe/Berlin'::text, NULL::timestamp with time zone, NULL::interval) >= 'Tue Jan 25 00:00:00 2000 PST'::timestamp with time zone) \qecho test overflow behaviour of time_bucket exclusion test overflow behaviour of time_bucket exclusion :PREFIX SELECT * FROM hyper WHERE time > 950 AND time_bucket(10, time) < '9223372036854775807'::bigint ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_96_chunk Filter: (("time" > 950) AND (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint)) -> Seq Scan on _hyper_1_97_chunk Filter: (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint) -> Seq Scan on _hyper_1_98_chunk Filter: (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint) -> Seq Scan on _hyper_1_99_chunk Filter: (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint) -> Seq Scan on _hyper_1_100_chunk Filter: (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint) -> Seq Scan on _hyper_1_101_chunk Filter: (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint) -> Seq Scan on _hyper_1_102_chunk Filter: (("time" > 950) AND (time_bucket('10'::bigint, "time") < '9223372036854775807'::bigint)) \qecho test timestamp upper boundary test timestamp upper boundary \qecho there should be no transformation if we are out of the supported (TimescaleDB-specific) range there should be no transformation if we are out of the supported (TimescaleDB-specific) range :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('1d',time) < '294276-01-01'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) \qecho transformation would be out of range transformation would be out of range :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('1000d',time) < '294276-01-01'::timestamp ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276'::timestamp without time zone) \qecho test timestamptz upper boundary test timestamptz upper boundary \qecho there should be no transformation if we are out of the supported (TimescaleDB-specific) range there should be no transformation if we are out of the supported (TimescaleDB-specific) range :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1d',time) < '294276-01-01'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 1 day'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) \qecho transformation would be out of range transformation would be out of range :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1000d',time) < '294276-01-01'::timestamptz ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 1000 days'::interval, "time") < 'Sat Jan 01 00:00:00 294276 PST'::timestamp with time zone) \qecho time_bucket exclusion with run-time constants time_bucket exclusion with run-time constants -- These queries have a stable time_bucket expression, because the text::interval conversion is stable. PREPARE P1(text , text) AS SELECT * FROM metrics_timestamptz WHERE time_bucket($1::interval, time) > $2::timestamptz ORDER BY time; PREPARE P2(text , text) AS SELECT * FROM metrics_timestamp WHERE time_bucket($1::interval, time) > $2::timestamptz ORDER BY time; PREPARE P3(text , text) AS SELECT * FROM metrics_timestamptz WHERE time_bucket($1::interval, time) > $2::timestamp ORDER BY time; PREPARE P4(text , text) AS SELECT * FROM metrics_timestamp WHERE time_bucket($1::interval, time) > $2::timestamp ORDER BY time; -- These queries have an immutable time_bucket expression, because the parameter is passed as interval, and no conversion is involved. PREPARE P5(interval, text) AS SELECT * FROM metrics_timestamptz WHERE time_bucket($1::interval, time) > $2::timestamptz ORDER BY time; PREPARE P6(interval, text) AS SELECT * FROM metrics_timestamp WHERE time_bucket($1::interval, time) > $2::timestamptz ORDER BY time; PREPARE P7(interval, text) AS SELECT * FROM metrics_timestamptz WHERE time_bucket($1::interval, time) > $2::timestamp ORDER BY time; PREPARE P8(interval, text) AS SELECT * FROM metrics_timestamp WHERE time_bucket($1::interval, time) > $2::timestamp ORDER BY time; SET plan_cache_mode TO 'force_custom_plan'; :PREFIX EXECUTE P1('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P2('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P3('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P4('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P5('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P6('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P7('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P8('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P1('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) :PREFIX EXECUTE P2('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) :PREFIX EXECUTE P3('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) :PREFIX EXECUTE P4('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket(('60 mins'::cstring)::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) :PREFIX EXECUTE P5('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) :PREFIX EXECUTE P6('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp with time zone) :PREFIX EXECUTE P7('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) :PREFIX EXECUTE P8('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket('@ 1 hour'::interval, "time") > ('2000-01-01 UTC'::cstring)::timestamp without time zone) SET plan_cache_mode TO 'force_generic_plan'; :PREFIX EXECUTE P1('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P2('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P3('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P4('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P5('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P6('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P7('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P8('60 mins', '2024-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 5 :PREFIX EXECUTE P1('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) :PREFIX EXECUTE P2('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp with time zone) :PREFIX EXECUTE P3('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) :PREFIX EXECUTE P4('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket(($1)::interval, "time") > ($2)::timestamp without time zone) :PREFIX EXECUTE P5('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) :PREFIX EXECUTE P6('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp with time zone) :PREFIX EXECUTE P7('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" Chunks excluded during startup: 0 -> Index Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) :PREFIX EXECUTE P8('60 mins', '2000-01-01 UTC'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_157_chunk_metrics_timestamp_time_idx on _hyper_5_157_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_158_chunk_metrics_timestamp_time_idx on _hyper_5_158_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) -> Index Only Scan Backward using _hyper_5_159_chunk_metrics_timestamp_time_idx on _hyper_5_159_chunk Filter: (time_bucket($1, "time") > ($2)::timestamp without time zone) RESET plan_cache_mode; DEALLOCATE P1; DEALLOCATE P2; DEALLOCATE P3; DEALLOCATE P4; DEALLOCATE P5; DEALLOCATE P6; DEALLOCATE P7; DEALLOCATE P8; :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) > 10 AND time_bucket(10, time) < 100 ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Append -> Seq Scan on _hyper_1_2_chunk Filter: (("time" > '10'::bigint) AND ("time" < '100'::bigint) AND (time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_3_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_4_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_5_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_6_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_7_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_8_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_9_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) -> Seq Scan on _hyper_1_10_chunk Filter: ((time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 100)) :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) > 10 AND time_bucket(10, time) < 20 ORDER BY time; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk."time" -> Seq Scan on _hyper_1_2_chunk Filter: (("time" > '10'::bigint) AND ("time" < '20'::bigint) AND (time_bucket('10'::bigint, "time") > 10) AND (time_bucket('10'::bigint, "time") < 20)) :PREFIX SELECT * FROM hyper WHERE time_bucket(1, time) > 11 AND time_bucket(1, time) < 19 ORDER BY time; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk."time" -> Seq Scan on _hyper_1_2_chunk Filter: (("time" > '11'::bigint) AND ("time" < '19'::bigint) AND (time_bucket('1'::bigint, "time") > 11) AND (time_bucket('1'::bigint, "time") < 19)) :PREFIX SELECT * FROM hyper WHERE 10 < time_bucket(10, time) AND 20 > time_bucket(10,time) ORDER BY time; --- QUERY PLAN --- Sort Sort Key: _hyper_1_2_chunk."time" -> Seq Scan on _hyper_1_2_chunk Filter: (("time" > '10'::bigint) AND ("time" < '20'::bigint) AND (10 < time_bucket('10'::bigint, "time")) AND (20 > time_bucket('10'::bigint, "time"))) \qecho time_bucket exclusion with date time_bucket exclusion with date :PREFIX SELECT * FROM metrics_date WHERE time_bucket('1d',time) < '2000-01-03' ORDER BY time; --- QUERY PLAN --- Index Only Scan Backward using _hyper_8_171_chunk_metrics_date_time_idx on _hyper_8_171_chunk Index Cond: ("time" < '01-03-2000'::date) Filter: (time_bucket('@ 1 day'::interval, "time") < '01-03-2000'::date) :PREFIX SELECT * FROM metrics_date WHERE time_bucket('1d',time) >= '2000-01-03' AND time_bucket('1d',time) <= '2000-01-10' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_date Order: metrics_date."time" -> Index Only Scan Backward using _hyper_8_171_chunk_metrics_date_time_idx on _hyper_8_171_chunk Index Cond: (("time" >= '01-03-2000'::date) AND ("time" <= '01-11-2000'::date)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= '01-03-2000'::date) AND (time_bucket('@ 1 day'::interval, "time") <= '01-10-2000'::date)) -> Index Only Scan Backward using _hyper_8_172_chunk_metrics_date_time_idx on _hyper_8_172_chunk Index Cond: (("time" >= '01-03-2000'::date) AND ("time" <= '01-11-2000'::date)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= '01-03-2000'::date) AND (time_bucket('@ 1 day'::interval, "time") <= '01-10-2000'::date)) \qecho time_bucket exclusion with timestamp time_bucket exclusion with timestamp :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('1d',time) < '2000-01-03' ORDER BY time; --- QUERY PLAN --- Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Index Cond: ("time" < 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) Filter: (time_bucket('@ 1 day'::interval, "time") < 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('1d',time) >= '2000-01-03' AND time_bucket('1d',time) <= '2000-01-10' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamp Order: metrics_timestamp."time" -> Index Only Scan Backward using _hyper_5_155_chunk_metrics_timestamp_time_idx on _hyper_5_155_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) AND ("time" <= 'Tue Jan 11 00:00:00 2000'::timestamp without time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) AND (time_bucket('@ 1 day'::interval, "time") <= 'Mon Jan 10 00:00:00 2000'::timestamp without time zone)) -> Index Only Scan Backward using _hyper_5_156_chunk_metrics_timestamp_time_idx on _hyper_5_156_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) AND ("time" <= 'Tue Jan 11 00:00:00 2000'::timestamp without time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000'::timestamp without time zone) AND (time_bucket('@ 1 day'::interval, "time") <= 'Mon Jan 10 00:00:00 2000'::timestamp without time zone)) \qecho time_bucket exclusion with timestamptz time_bucket exclusion with timestamptz :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('6h',time) < '2000-01-03' ORDER BY time; --- QUERY PLAN --- Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: ("time" < 'Mon Jan 03 06:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 6 hours'::interval, "time") < 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('6h',time) >= '2000-01-03' AND time_bucket('6h',time) <= '2000-01-10' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Mon Jan 10 06:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 6 hours'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 6 hours'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Mon Jan 10 06:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 6 hours'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 6 hours'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho time_bucket exclusion with timestamptz and day interval time_bucket exclusion with timestamptz and day interval :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1d',time) < '2000-01-03' ORDER BY time; --- QUERY PLAN --- Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: ("time" < 'Tue Jan 04 00:00:00 2000 PST'::timestamp with time zone) Filter: (time_bucket('@ 1 day'::interval, "time") < 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1d',time) >= '2000-01-03' AND time_bucket('1d',time) <= '2000-01-10' ORDER BY time; --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_timestamptz Order: metrics_timestamptz."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Tue Jan 11 00:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 1 day'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Tue Jan 11 00:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 1 day'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1d',time) >= '2000-01-03' AND time_bucket('7d',time) <= '2000-01-10' ORDER BY time; --- QUERY PLAN --- Sort Sort Key: metrics_timestamptz."time" -> Append -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Mon Jan 17 00:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 7 days'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Seq Scan on _hyper_6_161_chunk Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 7 days'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk Index Cond: (("time" >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND ("time" <= 'Mon Jan 17 00:00:00 2000 PST'::timestamp with time zone)) Filter: ((time_bucket('@ 1 day'::interval, "time") >= 'Mon Jan 03 00:00:00 2000 PST'::timestamp with time zone) AND (time_bucket('@ 7 days'::interval, "time") <= 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho no transformation no transformation :PREFIX SELECT * FROM hyper WHERE time_bucket(10 + floor(random())::int, time) > 10 AND time_bucket(10 + floor(random())::int, time) < 100 AND time < 150 ORDER BY time; --- QUERY PLAN --- Sort Sort Key: hyper."time" -> Custom Scan (ChunkAppend) on hyper Chunks excluded during startup: 0 -> Seq Scan on _hyper_1_1_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_2_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_3_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_4_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_5_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_6_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_7_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_8_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_9_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_10_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_11_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_12_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_13_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_14_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) -> Seq Scan on _hyper_1_15_chunk Filter: ((time_bucket(((10 + (floor(random()))::integer))::bigint, "time") > 10) AND (time_bucket(((10 + (floor(random()))::integer))::bigint, "time") < 100)) \qecho exclude chunks based on time column with partitioning function. This exclude chunks based on time column with partitioning function. This \qecho transparently applies the time partitioning function on the time transparently applies the time partitioning function on the time \qecho value to be able to exclude chunks (similar to a closed dimension). value to be able to exclude chunks (similar to a closed dimension). :PREFIX SELECT * FROM hyper_timefunc WHERE time < 4 ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_timefunc.value -> Append -> Seq Scan on _hyper_4_124_chunk Filter: ("time" < '4'::double precision) -> Seq Scan on _hyper_4_125_chunk Filter: ("time" < '4'::double precision) -> Seq Scan on _hyper_4_126_chunk Filter: ("time" < '4'::double precision) -> Seq Scan on _hyper_4_127_chunk Filter: ("time" < '4'::double precision) \qecho excluding based on time expression is currently unoptimized excluding based on time expression is currently unoptimized :PREFIX SELECT * FROM hyper_timefunc WHERE unix_to_timestamp(time) < 'Wed Dec 31 16:00:04 1969 PST' ORDER BY value; --- QUERY PLAN --- Sort Sort Key: hyper_timefunc.value -> Append -> Seq Scan on _hyper_4_124_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_125_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_126_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_127_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_128_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_129_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_130_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_131_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_132_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_133_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_134_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_135_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_136_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_137_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_138_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_139_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_140_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_141_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_142_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_143_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_144_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_145_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_146_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_147_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_148_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_149_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_150_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_151_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_152_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_153_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) -> Seq Scan on _hyper_4_154_chunk Filter: (to_timestamp("time") < 'Wed Dec 31 16:00:04 1969 PST'::timestamp with time zone) \qecho test qual propagation for joins test qual propagation for joins RESET constraint_exclusion; \qecho nothing to propagate nothing to propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1, metrics_timestamptz_2 m2 WHERE m1.time = m2.time ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time ORDER BY m1.time; --- QUERY PLAN --- Merge Left Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time ORDER BY m1.time; --- QUERY PLAN --- Sort Sort Key: m1."time" -> Merge Right Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 \qecho OR constraints should not propagate OR constraints should not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' OR m1.time > '2001-01-01' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Filter: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) OR ("time" > 'Mon Jan 01 00:00:00 2001 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Filter: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) OR ("time" > 'Mon Jan 01 00:00:00 2001 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 \qecho test single constraint test single constraint \qecho constraint should be on both scans constraint should be on both scans \qecho these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column :PREFIX SELECT m1.time FROM metrics_timestamptz m1, metrics_timestamptz_2 m2 WHERE m1.time = m2.time AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Left Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho test 2 constraints on single relation test 2 constraints on single relation \qecho these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column :PREFIX SELECT m1.time FROM metrics_timestamptz m1, metrics_timestamptz_2 m2 WHERE m1.time = m2.time AND m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Nested Loop Left Join -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Append -> Index Only Scan using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 Index Cond: ("time" = m1."time") :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho test 2 constraints with 1 constraint on each relation test 2 constraints with 1 constraint on each relation \qecho these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column :PREFIX SELECT m1.time FROM metrics_timestamptz m1, metrics_timestamptz_2 m2 WHERE m1.time = m2.time AND m1.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) AND ("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone)) \qecho test constraints in ON clause of INNER JOIN test constraints in ON clause of INNER JOIN :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho test constraints in ON clause of LEFT JOIN test constraints in ON clause of LEFT JOIN \qecho must not propagate must not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Left Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho test constraints in ON clause of RIGHT JOIN test constraints in ON clause of RIGHT JOIN \qecho must not propagate must not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Gather Merge Workers Planned: 2 -> Sort Sort Key: m1."time" -> Parallel Hash Left Join Hash Cond: (m2."time" = m1."time") Join Filter: ((m2."time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND (m2."time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Parallel Append -> Parallel Seq Scan on _hyper_7_165_chunk m2_1 -> Parallel Seq Scan on _hyper_7_166_chunk m2_2 -> Parallel Seq Scan on _hyper_7_167_chunk m2_3 -> Parallel Seq Scan on _hyper_7_168_chunk m2_4 -> Parallel Seq Scan on _hyper_7_169_chunk m2_5 -> Parallel Seq Scan on _hyper_7_170_chunk m2_6 -> Parallel Hash -> Parallel Append -> Parallel Seq Scan on _hyper_6_160_chunk m1_1 -> Parallel Seq Scan on _hyper_6_161_chunk m1_2 -> Parallel Seq Scan on _hyper_6_162_chunk m1_3 -> Parallel Seq Scan on _hyper_6_163_chunk m1_4 -> Parallel Seq Scan on _hyper_6_164_chunk m1_5 \qecho test equality condition not in ON clause test equality condition not in ON clause :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON true WHERE m2.time = m1.time AND m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho test constraints not joined on test constraints not joined on \qecho device_id constraint must not propagate device_id constraint must not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON true WHERE m2.time = m1.time AND m2.time < '2000-01-10' AND m1.device_id = 1 ORDER BY m1.time; --- QUERY PLAN --- Sort Sort Key: m1."time" -> Nested Loop -> Append -> Seq Scan on _hyper_6_160_chunk m1_1 Filter: (device_id = 1) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 1) -> Append -> Index Only Scan using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" = m1."time") AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) \qecho test multiple join conditions test multiple join conditions \qecho device_id constraint should propagate device_id constraint should propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON true WHERE m2.time = m1.time AND m1.device_id = m2.device_id AND m2.time < '2000-01-10' AND m1.device_id = 1 ORDER BY m1.time; --- QUERY PLAN --- Sort Sort Key: m1."time" -> Nested Loop -> Append -> Seq Scan on _hyper_6_160_chunk m1_1 Filter: (device_id = 1) -> Index Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 1) -> Append -> Index Scan using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: ("time" = m1."time") Filter: (device_id = 1) -> Index Scan using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" = m1."time") AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) Filter: (device_id = 1) \qecho test join with 3 tables test join with 3 tables :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time INNER JOIN metrics_timestamptz m3 ON m2.time=m3.time WHERE m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Nested Loop -> Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: (("time" > 'Sat Jan 01 00:00:00 2000 PST'::timestamp with time zone) AND ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone)) -> Append -> Index Only Scan using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m3_1 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m3_2 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m3_3 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m3_4 Index Cond: ("time" = m1."time") -> Index Only Scan using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m3_5 Index Cond: ("time" = m1."time") \qecho test non-Const constraints test non-Const constraints :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10'::text::timestamptz ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" Chunks excluded during startup: 3 -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" Chunks excluded during startup: 4 -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < ('2000-01-10'::cstring)::timestamp with time zone) \qecho test now() test now() :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < now() ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 Index Cond: ("time" < now()) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 Index Cond: ("time" < now()) -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 Index Cond: ("time" < now()) \qecho test volatile function test volatile function \qecho should not propagate should not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < clock_timestamp() ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 Filter: ("time" < clock_timestamp()) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m2.time < clock_timestamp() ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m2."time" = m1."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz_2 m2 Order: m2."time" Chunks excluded during startup: 0 -> Index Only Scan Backward using _hyper_7_165_chunk_metrics_timestamptz_2_time_idx on _hyper_7_165_chunk m2_1 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk m2_2 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_7_167_chunk_metrics_timestamptz_2_time_idx on _hyper_7_167_chunk m2_3 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk m2_4 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_7_169_chunk_metrics_timestamptz_2_time_idx on _hyper_7_169_chunk m2_5 Filter: ("time" < clock_timestamp()) -> Index Only Scan Backward using _hyper_7_170_chunk_metrics_timestamptz_2_time_idx on _hyper_7_170_chunk m2_6 Filter: ("time" < clock_timestamp()) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 -> Index Only Scan Backward using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk m1_3 -> Index Only Scan Backward using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk m1_4 -> Index Only Scan Backward using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk m1_5 \qecho test JOINs with normal table test JOINs with normal table \qecho will not propagate because constraints are only added to hypertables will not propagate because constraints are only added to hypertables :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN regular_timestamptz m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m1."time" = m2."time") -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Sort Sort Key: m2."time" -> Seq Scan on regular_timestamptz m2 \qecho test JOINs with normal table test JOINs with normal table :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN regular_timestamptz m2 ON m1.time = m2.time WHERE m2.time < '2000-01-10' ORDER BY m1.time; --- QUERY PLAN --- Merge Join Merge Cond: (m2."time" = m1."time") -> Sort Sort Key: m2."time" -> Seq Scan on regular_timestamptz m2 Filter: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) -> Materialize -> Custom Scan (ChunkAppend) on metrics_timestamptz m1 Order: m1."time" -> Index Only Scan Backward using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk m1_1 -> Index Only Scan Backward using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk m1_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) \qecho test quals are not pushed into OUTER JOIN test quals are not pushed into OUTER JOIN CREATE TABLE outer_join_1 (id int, name text,time timestamptz NOT NULL DEFAULT '2000-01-01'); CREATE TABLE outer_join_2 (id int, name text,time timestamptz NOT NULL DEFAULT '2000-01-01'); SELECT (SELECT table_name FROM create_hypertable(tbl, 'time')) FROM (VALUES ('outer_join_1'),('outer_join_2')) v(tbl); table_name -------------- outer_join_1 outer_join_2 INSERT INTO outer_join_1 VALUES(1,'a'), (2,'b'); INSERT INTO outer_join_2 VALUES(1,'a'); :PREFIX SELECT one.id, two.name FROM outer_join_1 one LEFT OUTER JOIN outer_join_2 two ON one.id=two.id WHERE one.id=2; --- QUERY PLAN --- Nested Loop Left Join -> Seq Scan on _hyper_9_176_chunk one Filter: (id = 2) -> Materialize -> Seq Scan on _hyper_10_177_chunk two Filter: (id = 2) :PREFIX SELECT one.id, two.name FROM outer_join_2 two RIGHT OUTER JOIN outer_join_1 one ON one.id=two.id WHERE one.id=2; --- QUERY PLAN --- Nested Loop Left Join -> Seq Scan on _hyper_9_176_chunk one Filter: (id = 2) -> Materialize -> Seq Scan on _hyper_10_177_chunk two Filter: (id = 2) DROP TABLE outer_join_1; DROP TABLE outer_join_2; -- test UNION between regular table and hypertable SELECT time FROM regular_timestamptz UNION SELECT time FROM metrics_timestamptz ORDER BY 1; time ------------------------------ Sat Jan 01 00:00:00 2000 PST Sun Jan 02 00:00:00 2000 PST Mon Jan 03 00:00:00 2000 PST Tue Jan 04 00:00:00 2000 PST Wed Jan 05 00:00:00 2000 PST Thu Jan 06 00:00:00 2000 PST Fri Jan 07 00:00:00 2000 PST Sat Jan 08 00:00:00 2000 PST Sun Jan 09 00:00:00 2000 PST Mon Jan 10 00:00:00 2000 PST Tue Jan 11 00:00:00 2000 PST Wed Jan 12 00:00:00 2000 PST Thu Jan 13 00:00:00 2000 PST Fri Jan 14 00:00:00 2000 PST Sat Jan 15 00:00:00 2000 PST Sun Jan 16 00:00:00 2000 PST Mon Jan 17 00:00:00 2000 PST Tue Jan 18 00:00:00 2000 PST Wed Jan 19 00:00:00 2000 PST Thu Jan 20 00:00:00 2000 PST Fri Jan 21 00:00:00 2000 PST Sat Jan 22 00:00:00 2000 PST Sun Jan 23 00:00:00 2000 PST Mon Jan 24 00:00:00 2000 PST Tue Jan 25 00:00:00 2000 PST Wed Jan 26 00:00:00 2000 PST Thu Jan 27 00:00:00 2000 PST Fri Jan 28 00:00:00 2000 PST Sat Jan 29 00:00:00 2000 PST Sun Jan 30 00:00:00 2000 PST Mon Jan 31 00:00:00 2000 PST Tue Feb 01 00:00:00 2000 PST -- test UNION ALL between regular table and hypertable SELECT time FROM regular_timestamptz UNION ALL SELECT time FROM metrics_timestamptz ORDER BY 1; time ------------------------------ Sat Jan 01 00:00:00 2000 PST Sat Jan 01 00:00:00 2000 PST Sat Jan 01 00:00:00 2000 PST Sat Jan 01 00:00:00 2000 PST Sun Jan 02 00:00:00 2000 PST Sun Jan 02 00:00:00 2000 PST Sun Jan 02 00:00:00 2000 PST Sun Jan 02 00:00:00 2000 PST Mon Jan 03 00:00:00 2000 PST Mon Jan 03 00:00:00 2000 PST Mon Jan 03 00:00:00 2000 PST Mon Jan 03 00:00:00 2000 PST Tue Jan 04 00:00:00 2000 PST Tue Jan 04 00:00:00 2000 PST Tue Jan 04 00:00:00 2000 PST Tue Jan 04 00:00:00 2000 PST Wed Jan 05 00:00:00 2000 PST Wed Jan 05 00:00:00 2000 PST Wed Jan 05 00:00:00 2000 PST Wed Jan 05 00:00:00 2000 PST Thu Jan 06 00:00:00 2000 PST Thu Jan 06 00:00:00 2000 PST Thu Jan 06 00:00:00 2000 PST Thu Jan 06 00:00:00 2000 PST Fri Jan 07 00:00:00 2000 PST Fri Jan 07 00:00:00 2000 PST Fri Jan 07 00:00:00 2000 PST Fri Jan 07 00:00:00 2000 PST Sat Jan 08 00:00:00 2000 PST Sat Jan 08 00:00:00 2000 PST Sat Jan 08 00:00:00 2000 PST Sat Jan 08 00:00:00 2000 PST Sun Jan 09 00:00:00 2000 PST Sun Jan 09 00:00:00 2000 PST Sun Jan 09 00:00:00 2000 PST Sun Jan 09 00:00:00 2000 PST Mon Jan 10 00:00:00 2000 PST Mon Jan 10 00:00:00 2000 PST Mon Jan 10 00:00:00 2000 PST Mon Jan 10 00:00:00 2000 PST Tue Jan 11 00:00:00 2000 PST Tue Jan 11 00:00:00 2000 PST Tue Jan 11 00:00:00 2000 PST Tue Jan 11 00:00:00 2000 PST Wed Jan 12 00:00:00 2000 PST Wed Jan 12 00:00:00 2000 PST Wed Jan 12 00:00:00 2000 PST Wed Jan 12 00:00:00 2000 PST Thu Jan 13 00:00:00 2000 PST Thu Jan 13 00:00:00 2000 PST Thu Jan 13 00:00:00 2000 PST Thu Jan 13 00:00:00 2000 PST Fri Jan 14 00:00:00 2000 PST Fri Jan 14 00:00:00 2000 PST Fri Jan 14 00:00:00 2000 PST Fri Jan 14 00:00:00 2000 PST Sat Jan 15 00:00:00 2000 PST Sat Jan 15 00:00:00 2000 PST Sat Jan 15 00:00:00 2000 PST Sat Jan 15 00:00:00 2000 PST Sun Jan 16 00:00:00 2000 PST Sun Jan 16 00:00:00 2000 PST Sun Jan 16 00:00:00 2000 PST Sun Jan 16 00:00:00 2000 PST Mon Jan 17 00:00:00 2000 PST Mon Jan 17 00:00:00 2000 PST Mon Jan 17 00:00:00 2000 PST Mon Jan 17 00:00:00 2000 PST Tue Jan 18 00:00:00 2000 PST Tue Jan 18 00:00:00 2000 PST Tue Jan 18 00:00:00 2000 PST Tue Jan 18 00:00:00 2000 PST Wed Jan 19 00:00:00 2000 PST Wed Jan 19 00:00:00 2000 PST Wed Jan 19 00:00:00 2000 PST Wed Jan 19 00:00:00 2000 PST Thu Jan 20 00:00:00 2000 PST Thu Jan 20 00:00:00 2000 PST Thu Jan 20 00:00:00 2000 PST Thu Jan 20 00:00:00 2000 PST Fri Jan 21 00:00:00 2000 PST Fri Jan 21 00:00:00 2000 PST Fri Jan 21 00:00:00 2000 PST Fri Jan 21 00:00:00 2000 PST Sat Jan 22 00:00:00 2000 PST Sat Jan 22 00:00:00 2000 PST Sat Jan 22 00:00:00 2000 PST Sat Jan 22 00:00:00 2000 PST Sun Jan 23 00:00:00 2000 PST Sun Jan 23 00:00:00 2000 PST Sun Jan 23 00:00:00 2000 PST Sun Jan 23 00:00:00 2000 PST Mon Jan 24 00:00:00 2000 PST Mon Jan 24 00:00:00 2000 PST Mon Jan 24 00:00:00 2000 PST Mon Jan 24 00:00:00 2000 PST Tue Jan 25 00:00:00 2000 PST Tue Jan 25 00:00:00 2000 PST Tue Jan 25 00:00:00 2000 PST Tue Jan 25 00:00:00 2000 PST Wed Jan 26 00:00:00 2000 PST Wed Jan 26 00:00:00 2000 PST Wed Jan 26 00:00:00 2000 PST Wed Jan 26 00:00:00 2000 PST Thu Jan 27 00:00:00 2000 PST Thu Jan 27 00:00:00 2000 PST Thu Jan 27 00:00:00 2000 PST Thu Jan 27 00:00:00 2000 PST Fri Jan 28 00:00:00 2000 PST Fri Jan 28 00:00:00 2000 PST Fri Jan 28 00:00:00 2000 PST Fri Jan 28 00:00:00 2000 PST Sat Jan 29 00:00:00 2000 PST Sat Jan 29 00:00:00 2000 PST Sat Jan 29 00:00:00 2000 PST Sat Jan 29 00:00:00 2000 PST Sun Jan 30 00:00:00 2000 PST Sun Jan 30 00:00:00 2000 PST Sun Jan 30 00:00:00 2000 PST Sun Jan 30 00:00:00 2000 PST Mon Jan 31 00:00:00 2000 PST Mon Jan 31 00:00:00 2000 PST Mon Jan 31 00:00:00 2000 PST Mon Jan 31 00:00:00 2000 PST Tue Feb 01 00:00:00 2000 PST Tue Feb 01 00:00:00 2000 PST Tue Feb 01 00:00:00 2000 PST Tue Feb 01 00:00:00 2000 PST -- test nested join qual propagation :PREFIX SELECT * FROM ( SELECT o1_m1.time FROM metrics_timestamptz o1_m1 INNER JOIN metrics_timestamptz_2 o1_m2 ON true WHERE o1_m2.time = o1_m1.time AND o1_m1.device_id = o1_m2.device_id AND o1_m2.time < '2000-01-10' AND o1_m1.device_id = 1 ) o1 FULL OUTER JOIN ( SELECT o2_m1.time FROM metrics_timestamptz o2_m1 FULL OUTER JOIN metrics_timestamptz_2 o2_m2 ON true WHERE o2_m2.time = o2_m1.time AND o2_m1.device_id = o2_m2.device_id AND o2_m2.time > '2000-01-20' AND o2_m1.device_id = 2 ) o2 ON o1.time = o2.time ORDER BY 1,2; --- QUERY PLAN --- Sort Sort Key: o1_m1."time", o2_m1."time" -> Hash Full Join Hash Cond: (o2_m1."time" = o1_m1."time") -> Nested Loop -> Append -> Index Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk o2_m2_1 Index Cond: ("time" > 'Thu Jan 20 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 2) -> Seq Scan on _hyper_7_169_chunk o2_m2_2 Filter: (device_id = 2) -> Seq Scan on _hyper_7_170_chunk o2_m2_3 Filter: (device_id = 2) -> Append -> Index Scan using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk o2_m1_1 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk o2_m1_2 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk o2_m1_3 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk o2_m1_4 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk o2_m1_5 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Hash -> Nested Loop -> Append -> Seq Scan on _hyper_7_165_chunk o1_m2_1 Filter: (device_id = 1) -> Index Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk o1_m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 1) -> Append -> Index Scan using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk o1_m1_1 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk o1_m1_2 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk o1_m1_3 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk o1_m1_4 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk o1_m1_5 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) :PREFIX SELECT * FROM ( SELECT o1_m1.time FROM metrics_timestamptz o1_m1 INNER JOIN metrics_timestamptz_2 o1_m2 ON o1_m2.time = o1_m1.time AND o1_m1.device_id = o1_m2.device_id WHERE o1_m2.time < '2000-01-10' AND o1_m1.device_id = 1 ) o1 FULL OUTER JOIN ( SELECT o2_m1.time FROM metrics_timestamptz o2_m1 FULL OUTER JOIN metrics_timestamptz_2 o2_m2 ON o2_m2.time = o2_m1.time AND o2_m1.device_id = o2_m2.device_id WHERE o2_m2.time > '2000-01-20' AND o2_m1.device_id = 2 ) o2 ON o1.time = o2.time ORDER BY 1,2; --- QUERY PLAN --- Sort Sort Key: o1_m1."time", o2_m1."time" -> Hash Full Join Hash Cond: (o2_m1."time" = o1_m1."time") -> Nested Loop -> Append -> Index Scan Backward using _hyper_7_168_chunk_metrics_timestamptz_2_time_idx on _hyper_7_168_chunk o2_m2_1 Index Cond: ("time" > 'Thu Jan 20 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 2) -> Seq Scan on _hyper_7_169_chunk o2_m2_2 Filter: (device_id = 2) -> Seq Scan on _hyper_7_170_chunk o2_m2_3 Filter: (device_id = 2) -> Append -> Index Scan using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk o2_m1_1 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk o2_m1_2 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk o2_m1_3 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk o2_m1_4 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Index Scan using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk o2_m1_5 Index Cond: ("time" = o2_m2."time") Filter: (device_id = 2) -> Hash -> Nested Loop -> Append -> Seq Scan on _hyper_7_165_chunk o1_m2_1 Filter: (device_id = 1) -> Index Scan Backward using _hyper_7_166_chunk_metrics_timestamptz_2_time_idx on _hyper_7_166_chunk o1_m2_2 Index Cond: ("time" < 'Mon Jan 10 00:00:00 2000 PST'::timestamp with time zone) Filter: (device_id = 1) -> Append -> Index Scan using _hyper_6_160_chunk_metrics_timestamptz_time_idx on _hyper_6_160_chunk o1_m1_1 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_161_chunk_metrics_timestamptz_time_idx on _hyper_6_161_chunk o1_m1_2 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_162_chunk_metrics_timestamptz_time_idx on _hyper_6_162_chunk o1_m1_3 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_163_chunk_metrics_timestamptz_time_idx on _hyper_6_163_chunk o1_m1_4 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) -> Index Scan using _hyper_6_164_chunk_metrics_timestamptz_time_idx on _hyper_6_164_chunk o1_m1_5 Index Cond: ("time" = o1_m2."time") Filter: (device_id = 1) \set ECHO errors RESET timescaledb.enable_optimizations; CREATE TABLE t(time timestamptz NOT NULL); SELECT table_name FROM create_hypertable('t','time'); table_name ------------ t INSERT INTO t VALUES ('2000-01-01'), ('2010-01-01'), ('2020-01-01'); EXPLAIN (buffers off, costs off) SELECT * FROM t t1 INNER JOIN t t2 ON t1.time = t2.time WHERE t1.time < timestamptz '2010-01-01'; --- QUERY PLAN --- Merge Join Merge Cond: (t1."time" = t2."time") -> Merge Append Sort Key: t1."time" -> Index Only Scan Backward using _hyper_15_182_chunk_t_time_idx on _hyper_15_182_chunk t1_1 -> Index Only Scan Backward using _hyper_15_183_chunk_t_time_idx on _hyper_15_183_chunk t1_2 Index Cond: ("time" < 'Fri Jan 01 00:00:00 2010 PST'::timestamp with time zone) -> Materialize -> Merge Append Sort Key: t2."time" -> Index Only Scan Backward using _hyper_15_182_chunk_t_time_idx on _hyper_15_182_chunk t2_1 -> Index Only Scan Backward using _hyper_15_183_chunk_t_time_idx on _hyper_15_183_chunk t2_2 Index Cond: ("time" < 'Fri Jan 01 00:00:00 2010 PST'::timestamp with time zone) SET timescaledb.enable_qual_propagation TO false; EXPLAIN (buffers off, costs off) SELECT * FROM t t1 INNER JOIN t t2 ON t1.time = t2.time WHERE t1.time < timestamptz '2010-01-01'; --- QUERY PLAN --- Merge Join Merge Cond: (t1."time" = t2."time") -> Merge Append Sort Key: t1."time" -> Index Only Scan Backward using _hyper_15_182_chunk_t_time_idx on _hyper_15_182_chunk t1_1 -> Index Only Scan Backward using _hyper_15_183_chunk_t_time_idx on _hyper_15_183_chunk t1_2 Index Cond: ("time" < 'Fri Jan 01 00:00:00 2010 PST'::timestamp with time zone) -> Materialize -> Merge Append Sort Key: t2."time" -> Index Only Scan Backward using _hyper_15_182_chunk_t_time_idx on _hyper_15_182_chunk t2_1 -> Index Only Scan Backward using _hyper_15_183_chunk_t_time_idx on _hyper_15_183_chunk t2_2 -> Index Only Scan Backward using _hyper_15_184_chunk_t_time_idx on _hyper_15_184_chunk t2_3 RESET timescaledb.enable_qual_propagation; CREATE TABLE test (a int, time timestamptz NOT NULL); SELECT table_name FROM create_hypertable('public.test', 'time'); table_name ------------ test INSERT INTO test SELECT i, '2020-04-01'::date-10-i from generate_series(1,20) i; CREATE OR REPLACE FUNCTION test_f(_ts timestamptz) RETURNS SETOF test LANGUAGE SQL STABLE PARALLEL SAFE AS $f$ SELECT DISTINCT ON (a) * FROM test WHERE time >= _ts ORDER BY a, time DESC $f$; EXPLAIN (buffers off, costs off) SELECT * FROM test_f(now()); --- QUERY PLAN --- Unique -> Sort Sort Key: test.a, test."time" DESC -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 4 EXPLAIN (buffers off, costs off) SELECT * FROM test_f(now()); --- QUERY PLAN --- Unique -> Sort Sort Key: test.a, test."time" DESC -> Custom Scan (ChunkAppend) on test Chunks excluded during startup: 4 CREATE TABLE t1 (a int, b int NOT NULL); SELECT create_hypertable('t1', 'b', chunk_time_interval=>10); create_hypertable ------------------- (17,public,t1,t) CREATE TABLE t2 (a int, b int NOT NULL); SELECT create_hypertable('t2', 'b', chunk_time_interval=>10); create_hypertable ------------------- (18,public,t2,t) CREATE OR REPLACE FUNCTION f_t1(_a int, _b int) RETURNS SETOF t1 LANGUAGE SQL STABLE PARALLEL SAFE AS $function$ SELECT DISTINCT ON (a) * FROM t1 WHERE a = _a and b = _b ORDER BY a, b DESC $function$ ; CREATE OR REPLACE FUNCTION f_t2(_a int, _b int) RETURNS SETOF t2 LANGUAGE sql STABLE PARALLEL SAFE AS $function$ SELECT DISTINCT ON (j.a) j.* FROM f_t1(_a, _b) sc, t2 j WHERE j.b = _b AND j.a = _a ORDER BY j.a, j.b DESC $function$ ; CREATE OR REPLACE FUNCTION f_t1_2(_b int) RETURNS SETOF t1 LANGUAGE SQL STABLE PARALLEL SAFE AS $function$ SELECT DISTINCT ON (j.a) jt.* FROM t1 j, f_t1(j.a, _b) jt $function$; EXPLAIN (buffers off, costs off) SELECT * FROM f_t1_2(10); --- QUERY PLAN --- Subquery Scan on f_t1_2 -> Unique -> Sort Sort Key: j.a -> Nested Loop -> Seq Scan on t1 j -> Limit -> Index Scan using t1_b_idx on t1 Index Cond: (b = 10) Filter: (a = j.a) EXPLAIN (buffers off, costs off) SELECT * FROM f_t1_2(10) sc, f_t2(sc.a, 10); --- QUERY PLAN --- Nested Loop -> Unique -> Sort Sort Key: j.a -> Nested Loop -> Seq Scan on t1 j -> Limit -> Index Scan using t1_b_idx on t1 Index Cond: (b = 10) Filter: (a = j.a) -> Limit -> Nested Loop -> Limit -> Index Scan using t1_b_idx on t1 t1_1 Index Cond: (b = 10) Filter: (a = t1.a) -> Index Scan using t2_b_idx on t2 j_1 Index Cond: (b = 10) Filter: (a = t1.a) CREATE TABLE metrics_int1(time int, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval=1); INSERT INTO metrics_int1 SELECT i, i::text, i FROM generate_series(3,7) i; SELECT tableoid::regclass, time FROM metrics_int1 ORDER BY time; tableoid | time -------------------------------------------+------ _timescaledb_internal._hyper_19_189_chunk | 3 _timescaledb_internal._hyper_19_190_chunk | 4 _timescaledb_internal._hyper_19_191_chunk | 5 _timescaledb_internal._hyper_19_192_chunk | 6 _timescaledb_internal._hyper_19_193_chunk | 7 EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time >= 2; --- QUERY PLAN --- Append (actual rows=5.00 loops=1) -> Seq Scan on _hyper_19_189_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_192_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_193_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time >= 1 AND time >= 2; --- QUERY PLAN --- Append (actual rows=5.00 loops=1) -> Seq Scan on _hyper_19_189_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_192_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_193_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time >= 4; --- QUERY PLAN --- Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_192_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_193_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time > 4 AND time < 6; --- QUERY PLAN --- Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time > 4 AND time <= 6; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_192_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time >= 4 AND time < 6; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time >= 4 AND time <= 6; --- QUERY PLAN --- Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_192_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time BETWEEN 4 AND 5; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Seq Scan on _hyper_19_190_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_int1 WHERE time = 5; --- QUERY PLAN --- Seq Scan on _hyper_19_191_chunk (actual rows=1.00 loops=1) SET TIMEZONE='UTC'; CREATE TABLE metrics_tstz(time timestamptz, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval='1day'); INSERT INTO metrics_tstz SELECT '2000-01-01'::timestamptz + format('%s day',i)::interval, i::text, i FROM generate_series(2,6) i; SELECT tableoid::regclass, time FROM metrics_tstz ORDER BY time; tableoid | time -------------------------------------------+------------------------------ _timescaledb_internal._hyper_20_194_chunk | Mon Jan 03 00:00:00 2000 UTC _timescaledb_internal._hyper_20_195_chunk | Tue Jan 04 00:00:00 2000 UTC _timescaledb_internal._hyper_20_196_chunk | Wed Jan 05 00:00:00 2000 UTC _timescaledb_internal._hyper_20_197_chunk | Thu Jan 06 00:00:00 2000 UTC _timescaledb_internal._hyper_20_198_chunk | Fri Jan 07 00:00:00 2000 UTC EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-02'; --- QUERY PLAN --- Append (actual rows=5.00 loops=1) -> Seq Scan on _hyper_20_194_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_197_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_198_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-01' AND time >= '2000-01-02'; --- QUERY PLAN --- Append (actual rows=5.00 loops=1) -> Seq Scan on _hyper_20_194_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_197_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_198_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04'; --- QUERY PLAN --- Append (actual rows=4.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_197_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_198_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time > '2000-01-04' AND time < '2000-01-06'; --- QUERY PLAN --- Append (actual rows=1.00 loops=1) -> Index Scan using _hyper_20_195_chunk_metrics_tstz_time_idx on _hyper_20_195_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" < 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time > '2000-01-04' AND time <= '2000-01-06'; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Index Scan using _hyper_20_195_chunk_metrics_tstz_time_idx on _hyper_20_195_chunk (actual rows=0.00 loops=1) Index Cond: (("time" > 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_20_197_chunk_metrics_tstz_time_idx on _hyper_20_197_chunk (actual rows=1.00 loops=1) Index Cond: (("time" > 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time < '2000-01-06'; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time <= '2000-01-06'; --- QUERY PLAN --- Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_20_197_chunk_metrics_tstz_time_idx on _hyper_20_197_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time BETWEEN '2000-01-04' AND '2000-01-05'; --- QUERY PLAN --- Append (actual rows=2.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_20_196_chunk_metrics_tstz_time_idx on _hyper_20_196_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Wed Jan 05 00:00:00 2000 UTC'::timestamp with time zone)) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time = '2000-01-05'; --- QUERY PLAN --- Index Scan using _hyper_20_196_chunk_metrics_tstz_time_idx on _hyper_20_196_chunk (actual rows=1.00 loops=1) Index Cond: ("time" = 'Wed Jan 05 00:00:00 2000 UTC'::timestamp with time zone) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time <= '2000-01-06' AND device = '5'; --- QUERY PLAN --- Append (actual rows=1.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=0.00 loops=1) Filter: (device = '5'::text) Rows Removed by Filter: 1 -> Seq Scan on _hyper_20_196_chunk (actual rows=0.00 loops=1) Filter: (device = '5'::text) Rows Removed by Filter: 1 -> Index Scan using _hyper_20_197_chunk_metrics_tstz_time_idx on _hyper_20_197_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) Filter: (device = '5'::text) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time <= '2000-01-06' AND device IS NOT NULL; --- QUERY PLAN --- Append (actual rows=3.00 loops=1) -> Seq Scan on _hyper_20_195_chunk (actual rows=1.00 loops=1) Filter: (device IS NOT NULL) -> Seq Scan on _hyper_20_196_chunk (actual rows=1.00 loops=1) Filter: (device IS NOT NULL) -> Index Scan using _hyper_20_197_chunk_metrics_tstz_time_idx on _hyper_20_197_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone)) Filter: (device IS NOT NULL) EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time <= '2000-01-06' AND time < now(); --- QUERY PLAN --- Custom Scan (ChunkAppend) on metrics_tstz (actual rows=3.00 loops=1) Chunks excluded during startup: 0 -> Index Scan using _hyper_20_195_chunk_metrics_tstz_time_idx on _hyper_20_195_chunk (actual rows=1.00 loops=1) Index Cond: ("time" < now()) -> Index Scan using _hyper_20_196_chunk_metrics_tstz_time_idx on _hyper_20_196_chunk (actual rows=1.00 loops=1) Index Cond: ("time" < now()) -> Index Scan using _hyper_20_197_chunk_metrics_tstz_time_idx on _hyper_20_197_chunk (actual rows=1.00 loops=1) Index Cond: (("time" >= 'Tue Jan 04 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" <= 'Thu Jan 06 00:00:00 2000 UTC'::timestamp with time zone) AND ("time" < now())) CREATE TABLE metrics_space(time timestamptz, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval='1day'); SELECT add_dimension('metrics_space', 'device', 4); add_dimension ------------------------------------ (25,public,metrics_space,device,t) INSERT INTO metrics_space SELECT '2000-01-01'::timestamptz + format('%s day',i)::interval, i::text, i FROM generate_series(2,6) i; EXPLAIN (buffers off, costs off, timing off, summary off, analyze) SELECT * FROM metrics_space WHERE time >= '2000-01-02'; --- QUERY PLAN --- Append (actual rows=5.00 loops=1) -> Index Scan using _hyper_21_199_chunk_metrics_space_time_idx on _hyper_21_199_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Sun Jan 02 00:00:00 2000 UTC'::timestamp with time zone) -> Index Scan using _hyper_21_200_chunk_metrics_space_time_idx on _hyper_21_200_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Sun Jan 02 00:00:00 2000 UTC'::timestamp with time zone) -> Index Scan using _hyper_21_201_chunk_metrics_space_time_idx on _hyper_21_201_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Sun Jan 02 00:00:00 2000 UTC'::timestamp with time zone) -> Index Scan using _hyper_21_202_chunk_metrics_space_time_idx on _hyper_21_202_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Sun Jan 02 00:00:00 2000 UTC'::timestamp with time zone) -> Index Scan using _hyper_21_203_chunk_metrics_space_time_idx on _hyper_21_203_chunk (actual rows=1.00 loops=1) Index Cond: ("time" >= 'Sun Jan 02 00:00:00 2000 UTC'::timestamp with time zone) --TEST END-- ================================================ FILE: test/expected/plan_hashagg-15.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SET max_parallel_workers_per_gather TO 0; \set PREFIX 'EXPLAIN (buffers off, costs off) ' \ir include/plan_hashagg_load.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE metric (id SERIAL PRIMARY KEY, value INT); CREATE TABLE hyper(time TIMESTAMP NOT NULL, time_int BIGINT, time_broken DATE, metricid int, value double precision); CREATE TABLE regular(time TIMESTAMP NOT NULL, time_int BIGINT, time_date DATE, metricid int, value double precision); SELECT create_hypertable('hyper', 'time', chunk_time_interval => interval '20 day', create_default_indexes=>FALSE); psql:include/plan_hashagg_load.sql:9: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------- (1,public,hyper,t) ALTER TABLE hyper DROP COLUMN time_broken, ADD COLUMN time_date DATE; INSERT INTO metric(value) SELECT random()*100 FROM generate_series(0,10); INSERT INTO hyper SELECT t, EXTRACT(EPOCH FROM t), (EXTRACT(EPOCH FROM t)::int % 10)+1, 1.0, t::date FROM generate_series('2001-01-01', '2001-01-10', INTERVAL '1 second') t; INSERT INTO regular(time, time_int, time_date, metricid, value) SELECT t, EXTRACT(EPOCH FROM t), t::date, (EXTRACT(EPOCH FROM t)::int % 10) + 1, 1.0 FROM generate_series('2001-01-01', '2001-01-02', INTERVAL '1 second') t; --test some queries before analyze; EXPLAIN (buffers off, costs off) SELECT time_bucket('1 minute', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- Sort Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) DESC -> HashAggregate Group Key: time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time") -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) EXPLAIN (buffers off, costs off) SELECT date_trunc('minute', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- Sort Sort Key: (date_trunc('minute'::text, _hyper_1_1_chunk."time")) DESC -> HashAggregate Group Key: date_trunc('minute'::text, _hyper_1_1_chunk."time") -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) -- Test partitioning function on an open (time) dimension CREATE OR REPLACE FUNCTION unix_to_timestamp(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE SQL IMMUTABLE AS $BODY$ SELECT to_timestamp(unixtime); $BODY$; CREATE TABLE hyper_timefunc(time float8 NOT NULL, metricid int, VALUE double precision, time_date DATE); SELECT create_hypertable('hyper_timefunc', 'time', chunk_time_interval => interval '20 day', create_default_indexes=>FALSE, time_partitioning_func => 'unix_to_timestamp'); create_hypertable ----------------------------- (2,public,hyper_timefunc,t) INSERT INTO hyper_timefunc SELECT time_int, metricid, VALUE, time_date FROM hyper; ANALYZE metric; ANALYZE hyper; ANALYZE regular; ANALYZE hyper_timefunc; \ir include/plan_hashagg_query.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. :PREFIX SELECT time_bucket('1 minute', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) -> Sort Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT time_bucket('1 hour', time) AS MetricMinuteTs, metricid, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs, metricid ORDER BY MetricMinuteTs DESC, metricid; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('@ 1 hour'::interval, _hyper_1_1_chunk."time")), _hyper_1_1_chunk.metricid -> Sort Sort Key: (time_bucket('@ 1 hour'::interval, _hyper_1_1_chunk."time")) DESC, _hyper_1_1_chunk.metricid -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) --should be too many groups will not hashaggregate :PREFIX SELECT time_bucket('1 second', time) AS MetricMinuteTs, metricid, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs, metricid ORDER BY MetricMinuteTs DESC, metricid; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('@ 1 sec'::interval, _hyper_1_1_chunk."time")), _hyper_1_1_chunk.metricid -> Sort Sort Key: (time_bucket('@ 1 sec'::interval, _hyper_1_1_chunk."time")) DESC, _hyper_1_1_chunk.metricid -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT time_bucket('1 minute', time, INTERVAL '30 seconds') AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time", '@ 30 secs'::interval)) -> Sort Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time", '@ 30 secs'::interval)) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT time_bucket(60, time_int) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('60'::bigint, _hyper_1_1_chunk.time_int)) -> Sort Sort Key: (time_bucket('60'::bigint, _hyper_1_1_chunk.time_int)) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT time_bucket(60, time_int, 10) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('60'::bigint, _hyper_1_1_chunk.time_int, '10'::bigint)) -> Sort Sort Key: (time_bucket('60'::bigint, _hyper_1_1_chunk.time_int, '10'::bigint)) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT time_bucket('1 day', time_date) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- Sort Sort Key: (time_bucket('@ 1 day'::interval, _hyper_1_1_chunk.time_date)) DESC -> HashAggregate Group Key: time_bucket('@ 1 day'::interval, _hyper_1_1_chunk.time_date) -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT date_trunc('minute', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (date_trunc('minute'::text, _hyper_1_1_chunk."time")) -> Sort Sort Key: (date_trunc('minute'::text, _hyper_1_1_chunk."time")) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) \set ON_ERROR_STOP 0 --can't optimize invalid time unit :PREFIX SELECT date_trunc('invalid', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (date_trunc('invalid'::text, _hyper_1_1_chunk."time")) -> Sort Sort Key: (date_trunc('invalid'::text, _hyper_1_1_chunk."time")) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) \set ON_ERROR_STOP 1 :PREFIX SELECT date_trunc('day', time_date) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- Sort Sort Key: (date_trunc('day'::text, (_hyper_1_1_chunk.time_date)::timestamp with time zone)) DESC -> HashAggregate Group Key: date_trunc('day'::text, (_hyper_1_1_chunk.time_date)::timestamp with time zone) -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) --joins --with hypertable, optimize :PREFIX SELECT time_bucket(3600, time_int, 10) AS MetricMinuteTs, metric.value, AVG(hyper.value) as avg FROM hyper JOIN metric ON (hyper.metricid = metric.id) WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs, metric.id ORDER BY MetricMinuteTs DESC, metric.id; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('3600'::bigint, _hyper_1_1_chunk.time_int, '10'::bigint)), metric.id -> Sort Sort Key: (time_bucket('3600'::bigint, _hyper_1_1_chunk.time_int, '10'::bigint)) DESC, metric.id -> Hash Join Hash Cond: (_hyper_1_1_chunk.metricid = metric.id) -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) -> Hash -> Seq Scan on metric --no hypertable involved, no optimization :PREFIX SELECT time_bucket(3600, time_int, 10) AS MetricMinuteTs, metric.value, AVG(regular.value) as avg FROM regular JOIN metric ON (regular.metricid = metric.id) WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs, metric.id ORDER BY MetricMinuteTs DESC, metric.id; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('3600'::bigint, regular.time_int, '10'::bigint)), metric.id -> Sort Sort Key: (time_bucket('3600'::bigint, regular.time_int, '10'::bigint)) DESC, metric.id -> Nested Loop Join Filter: (regular.metricid = metric.id) -> Seq Scan on regular Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) -> Seq Scan on metric -- Try with time partitioning function. Currently not optimized for hash aggregates :PREFIX SELECT time_bucket('1 minute', unix_to_timestamp(time)) AS MetricMinuteTs, AVG(value) as avg FROM hyper_timefunc WHERE unix_to_timestamp(time) >= '2001-01-04T00:00:00' AND unix_to_timestamp(time) <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('@ 1 min'::interval, to_timestamp(_hyper_2_2_chunk."time"))) -> Sort Sort Key: (time_bucket('@ 1 min'::interval, to_timestamp(_hyper_2_2_chunk."time"))) DESC -> Result -> Seq Scan on _hyper_2_2_chunk Filter: ((to_timestamp("time") >= 'Thu Jan 04 00:00:00 2001 PST'::timestamp with time zone) AND (to_timestamp("time") <= 'Fri Jan 05 01:00:00 2001 PST'::timestamp with time zone)) \set ECHO none psql:include/plan_hashagg_query.sql:60: ERROR: unit "invalid" not recognized for type timestamp without time zone psql:include/plan_hashagg_query.sql:60: ERROR: unit "invalid" not recognized for type timestamp without time zone ================================================ FILE: test/expected/plan_hashagg-16.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SET max_parallel_workers_per_gather TO 0; \set PREFIX 'EXPLAIN (buffers off, costs off) ' \ir include/plan_hashagg_load.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE metric (id SERIAL PRIMARY KEY, value INT); CREATE TABLE hyper(time TIMESTAMP NOT NULL, time_int BIGINT, time_broken DATE, metricid int, value double precision); CREATE TABLE regular(time TIMESTAMP NOT NULL, time_int BIGINT, time_date DATE, metricid int, value double precision); SELECT create_hypertable('hyper', 'time', chunk_time_interval => interval '20 day', create_default_indexes=>FALSE); psql:include/plan_hashagg_load.sql:9: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------- (1,public,hyper,t) ALTER TABLE hyper DROP COLUMN time_broken, ADD COLUMN time_date DATE; INSERT INTO metric(value) SELECT random()*100 FROM generate_series(0,10); INSERT INTO hyper SELECT t, EXTRACT(EPOCH FROM t), (EXTRACT(EPOCH FROM t)::int % 10)+1, 1.0, t::date FROM generate_series('2001-01-01', '2001-01-10', INTERVAL '1 second') t; INSERT INTO regular(time, time_int, time_date, metricid, value) SELECT t, EXTRACT(EPOCH FROM t), t::date, (EXTRACT(EPOCH FROM t)::int % 10) + 1, 1.0 FROM generate_series('2001-01-01', '2001-01-02', INTERVAL '1 second') t; --test some queries before analyze; EXPLAIN (buffers off, costs off) SELECT time_bucket('1 minute', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- Sort Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) DESC -> HashAggregate Group Key: time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time") -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) EXPLAIN (buffers off, costs off) SELECT date_trunc('minute', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- Sort Sort Key: (date_trunc('minute'::text, _hyper_1_1_chunk."time")) DESC -> HashAggregate Group Key: date_trunc('minute'::text, _hyper_1_1_chunk."time") -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) -- Test partitioning function on an open (time) dimension CREATE OR REPLACE FUNCTION unix_to_timestamp(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE SQL IMMUTABLE AS $BODY$ SELECT to_timestamp(unixtime); $BODY$; CREATE TABLE hyper_timefunc(time float8 NOT NULL, metricid int, VALUE double precision, time_date DATE); SELECT create_hypertable('hyper_timefunc', 'time', chunk_time_interval => interval '20 day', create_default_indexes=>FALSE, time_partitioning_func => 'unix_to_timestamp'); create_hypertable ----------------------------- (2,public,hyper_timefunc,t) INSERT INTO hyper_timefunc SELECT time_int, metricid, VALUE, time_date FROM hyper; ANALYZE metric; ANALYZE hyper; ANALYZE regular; ANALYZE hyper_timefunc; \ir include/plan_hashagg_query.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. :PREFIX SELECT time_bucket('1 minute', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) -> Sort Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT time_bucket('1 hour', time) AS MetricMinuteTs, metricid, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs, metricid ORDER BY MetricMinuteTs DESC, metricid; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('@ 1 hour'::interval, _hyper_1_1_chunk."time")), _hyper_1_1_chunk.metricid -> Sort Sort Key: (time_bucket('@ 1 hour'::interval, _hyper_1_1_chunk."time")) DESC, _hyper_1_1_chunk.metricid -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) --should be too many groups will not hashaggregate :PREFIX SELECT time_bucket('1 second', time) AS MetricMinuteTs, metricid, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs, metricid ORDER BY MetricMinuteTs DESC, metricid; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('@ 1 sec'::interval, _hyper_1_1_chunk."time")), _hyper_1_1_chunk.metricid -> Sort Sort Key: (time_bucket('@ 1 sec'::interval, _hyper_1_1_chunk."time")) DESC, _hyper_1_1_chunk.metricid -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT time_bucket('1 minute', time, INTERVAL '30 seconds') AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time", '@ 30 secs'::interval)) -> Sort Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time", '@ 30 secs'::interval)) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT time_bucket(60, time_int) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('60'::bigint, _hyper_1_1_chunk.time_int)) -> Sort Sort Key: (time_bucket('60'::bigint, _hyper_1_1_chunk.time_int)) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT time_bucket(60, time_int, 10) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('60'::bigint, _hyper_1_1_chunk.time_int, '10'::bigint)) -> Sort Sort Key: (time_bucket('60'::bigint, _hyper_1_1_chunk.time_int, '10'::bigint)) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT time_bucket('1 day', time_date) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- Sort Sort Key: (time_bucket('@ 1 day'::interval, _hyper_1_1_chunk.time_date)) DESC -> HashAggregate Group Key: time_bucket('@ 1 day'::interval, _hyper_1_1_chunk.time_date) -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT date_trunc('minute', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (date_trunc('minute'::text, _hyper_1_1_chunk."time")) -> Sort Sort Key: (date_trunc('minute'::text, _hyper_1_1_chunk."time")) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) \set ON_ERROR_STOP 0 --can't optimize invalid time unit :PREFIX SELECT date_trunc('invalid', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (date_trunc('invalid'::text, _hyper_1_1_chunk."time")) -> Sort Sort Key: (date_trunc('invalid'::text, _hyper_1_1_chunk."time")) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) \set ON_ERROR_STOP 1 :PREFIX SELECT date_trunc('day', time_date) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- Sort Sort Key: (date_trunc('day'::text, (_hyper_1_1_chunk.time_date)::timestamp with time zone)) DESC -> HashAggregate Group Key: date_trunc('day'::text, (_hyper_1_1_chunk.time_date)::timestamp with time zone) -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) --joins --with hypertable, optimize :PREFIX SELECT time_bucket(3600, time_int, 10) AS MetricMinuteTs, metric.value, AVG(hyper.value) as avg FROM hyper JOIN metric ON (hyper.metricid = metric.id) WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs, metric.id ORDER BY MetricMinuteTs DESC, metric.id; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('3600'::bigint, _hyper_1_1_chunk.time_int, '10'::bigint)), metric.id -> Sort Sort Key: (time_bucket('3600'::bigint, _hyper_1_1_chunk.time_int, '10'::bigint)) DESC, metric.id -> Hash Join Hash Cond: (_hyper_1_1_chunk.metricid = metric.id) -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) -> Hash -> Seq Scan on metric --no hypertable involved, no optimization :PREFIX SELECT time_bucket(3600, time_int, 10) AS MetricMinuteTs, metric.value, AVG(regular.value) as avg FROM regular JOIN metric ON (regular.metricid = metric.id) WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs, metric.id ORDER BY MetricMinuteTs DESC, metric.id; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('3600'::bigint, regular.time_int, '10'::bigint)), metric.id -> Sort Sort Key: (time_bucket('3600'::bigint, regular.time_int, '10'::bigint)) DESC, metric.id -> Nested Loop Join Filter: (metric.id = regular.metricid) -> Seq Scan on regular Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) -> Seq Scan on metric -- Try with time partitioning function. Currently not optimized for hash aggregates :PREFIX SELECT time_bucket('1 minute', unix_to_timestamp(time)) AS MetricMinuteTs, AVG(value) as avg FROM hyper_timefunc WHERE unix_to_timestamp(time) >= '2001-01-04T00:00:00' AND unix_to_timestamp(time) <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('@ 1 min'::interval, to_timestamp(_hyper_2_2_chunk."time"))) -> Sort Sort Key: (time_bucket('@ 1 min'::interval, to_timestamp(_hyper_2_2_chunk."time"))) DESC -> Result -> Seq Scan on _hyper_2_2_chunk Filter: ((to_timestamp("time") >= 'Thu Jan 04 00:00:00 2001 PST'::timestamp with time zone) AND (to_timestamp("time") <= 'Fri Jan 05 01:00:00 2001 PST'::timestamp with time zone)) \set ECHO none psql:include/plan_hashagg_query.sql:60: ERROR: unit "invalid" not recognized for type timestamp without time zone psql:include/plan_hashagg_query.sql:60: ERROR: unit "invalid" not recognized for type timestamp without time zone ================================================ FILE: test/expected/plan_hashagg-17.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SET max_parallel_workers_per_gather TO 0; \set PREFIX 'EXPLAIN (buffers off, costs off) ' \ir include/plan_hashagg_load.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE metric (id SERIAL PRIMARY KEY, value INT); CREATE TABLE hyper(time TIMESTAMP NOT NULL, time_int BIGINT, time_broken DATE, metricid int, value double precision); CREATE TABLE regular(time TIMESTAMP NOT NULL, time_int BIGINT, time_date DATE, metricid int, value double precision); SELECT create_hypertable('hyper', 'time', chunk_time_interval => interval '20 day', create_default_indexes=>FALSE); psql:include/plan_hashagg_load.sql:9: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------- (1,public,hyper,t) ALTER TABLE hyper DROP COLUMN time_broken, ADD COLUMN time_date DATE; INSERT INTO metric(value) SELECT random()*100 FROM generate_series(0,10); INSERT INTO hyper SELECT t, EXTRACT(EPOCH FROM t), (EXTRACT(EPOCH FROM t)::int % 10)+1, 1.0, t::date FROM generate_series('2001-01-01', '2001-01-10', INTERVAL '1 second') t; INSERT INTO regular(time, time_int, time_date, metricid, value) SELECT t, EXTRACT(EPOCH FROM t), t::date, (EXTRACT(EPOCH FROM t)::int % 10) + 1, 1.0 FROM generate_series('2001-01-01', '2001-01-02', INTERVAL '1 second') t; --test some queries before analyze; EXPLAIN (buffers off, costs off) SELECT time_bucket('1 minute', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- Sort Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) DESC -> HashAggregate Group Key: time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time") -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) EXPLAIN (buffers off, costs off) SELECT date_trunc('minute', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- Sort Sort Key: (date_trunc('minute'::text, _hyper_1_1_chunk."time")) DESC -> HashAggregate Group Key: date_trunc('minute'::text, _hyper_1_1_chunk."time") -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) -- Test partitioning function on an open (time) dimension CREATE OR REPLACE FUNCTION unix_to_timestamp(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE SQL IMMUTABLE AS $BODY$ SELECT to_timestamp(unixtime); $BODY$; CREATE TABLE hyper_timefunc(time float8 NOT NULL, metricid int, VALUE double precision, time_date DATE); SELECT create_hypertable('hyper_timefunc', 'time', chunk_time_interval => interval '20 day', create_default_indexes=>FALSE, time_partitioning_func => 'unix_to_timestamp'); create_hypertable ----------------------------- (2,public,hyper_timefunc,t) INSERT INTO hyper_timefunc SELECT time_int, metricid, VALUE, time_date FROM hyper; ANALYZE metric; ANALYZE hyper; ANALYZE regular; ANALYZE hyper_timefunc; \ir include/plan_hashagg_query.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. :PREFIX SELECT time_bucket('1 minute', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) -> Sort Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT time_bucket('1 hour', time) AS MetricMinuteTs, metricid, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs, metricid ORDER BY MetricMinuteTs DESC, metricid; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('@ 1 hour'::interval, _hyper_1_1_chunk."time")), _hyper_1_1_chunk.metricid -> Sort Sort Key: (time_bucket('@ 1 hour'::interval, _hyper_1_1_chunk."time")) DESC, _hyper_1_1_chunk.metricid -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) --should be too many groups will not hashaggregate :PREFIX SELECT time_bucket('1 second', time) AS MetricMinuteTs, metricid, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs, metricid ORDER BY MetricMinuteTs DESC, metricid; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('@ 1 sec'::interval, _hyper_1_1_chunk."time")), _hyper_1_1_chunk.metricid -> Sort Sort Key: (time_bucket('@ 1 sec'::interval, _hyper_1_1_chunk."time")) DESC, _hyper_1_1_chunk.metricid -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT time_bucket('1 minute', time, INTERVAL '30 seconds') AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time", '@ 30 secs'::interval)) -> Sort Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time", '@ 30 secs'::interval)) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT time_bucket(60, time_int) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('60'::bigint, _hyper_1_1_chunk.time_int)) -> Sort Sort Key: (time_bucket('60'::bigint, _hyper_1_1_chunk.time_int)) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT time_bucket(60, time_int, 10) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('60'::bigint, _hyper_1_1_chunk.time_int, '10'::bigint)) -> Sort Sort Key: (time_bucket('60'::bigint, _hyper_1_1_chunk.time_int, '10'::bigint)) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT time_bucket('1 day', time_date) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- Sort Sort Key: (time_bucket('@ 1 day'::interval, _hyper_1_1_chunk.time_date)) DESC -> HashAggregate Group Key: time_bucket('@ 1 day'::interval, _hyper_1_1_chunk.time_date) -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT date_trunc('minute', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (date_trunc('minute'::text, _hyper_1_1_chunk."time")) -> Sort Sort Key: (date_trunc('minute'::text, _hyper_1_1_chunk."time")) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) \set ON_ERROR_STOP 0 --can't optimize invalid time unit :PREFIX SELECT date_trunc('invalid', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (date_trunc('invalid'::text, _hyper_1_1_chunk."time")) -> Sort Sort Key: (date_trunc('invalid'::text, _hyper_1_1_chunk."time")) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) \set ON_ERROR_STOP 1 :PREFIX SELECT date_trunc('day', time_date) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- Sort Sort Key: (date_trunc('day'::text, (_hyper_1_1_chunk.time_date)::timestamp with time zone)) DESC -> HashAggregate Group Key: date_trunc('day'::text, (_hyper_1_1_chunk.time_date)::timestamp with time zone) -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) --joins --with hypertable, optimize :PREFIX SELECT time_bucket(3600, time_int, 10) AS MetricMinuteTs, metric.value, AVG(hyper.value) as avg FROM hyper JOIN metric ON (hyper.metricid = metric.id) WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs, metric.id ORDER BY MetricMinuteTs DESC, metric.id; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('3600'::bigint, _hyper_1_1_chunk.time_int, '10'::bigint)), metric.id -> Sort Sort Key: (time_bucket('3600'::bigint, _hyper_1_1_chunk.time_int, '10'::bigint)) DESC, metric.id -> Hash Join Hash Cond: (_hyper_1_1_chunk.metricid = metric.id) -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) -> Hash -> Seq Scan on metric --no hypertable involved, no optimization :PREFIX SELECT time_bucket(3600, time_int, 10) AS MetricMinuteTs, metric.value, AVG(regular.value) as avg FROM regular JOIN metric ON (regular.metricid = metric.id) WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs, metric.id ORDER BY MetricMinuteTs DESC, metric.id; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('3600'::bigint, regular.time_int, '10'::bigint)), metric.id -> Sort Sort Key: (time_bucket('3600'::bigint, regular.time_int, '10'::bigint)) DESC, metric.id -> Nested Loop Join Filter: (metric.id = regular.metricid) -> Seq Scan on regular Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) -> Seq Scan on metric -- Try with time partitioning function. Currently not optimized for hash aggregates :PREFIX SELECT time_bucket('1 minute', unix_to_timestamp(time)) AS MetricMinuteTs, AVG(value) as avg FROM hyper_timefunc WHERE unix_to_timestamp(time) >= '2001-01-04T00:00:00' AND unix_to_timestamp(time) <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('@ 1 min'::interval, to_timestamp(_hyper_2_2_chunk."time"))) -> Sort Sort Key: (time_bucket('@ 1 min'::interval, to_timestamp(_hyper_2_2_chunk."time"))) DESC -> Result -> Seq Scan on _hyper_2_2_chunk Filter: ((to_timestamp("time") >= 'Thu Jan 04 00:00:00 2001 PST'::timestamp with time zone) AND (to_timestamp("time") <= 'Fri Jan 05 01:00:00 2001 PST'::timestamp with time zone)) \set ECHO none psql:include/plan_hashagg_query.sql:60: ERROR: unit "invalid" not recognized for type timestamp without time zone psql:include/plan_hashagg_query.sql:60: ERROR: unit "invalid" not recognized for type timestamp without time zone ================================================ FILE: test/expected/plan_hashagg-18.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SET max_parallel_workers_per_gather TO 0; \set PREFIX 'EXPLAIN (buffers off, costs off) ' \ir include/plan_hashagg_load.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE metric (id SERIAL PRIMARY KEY, value INT); CREATE TABLE hyper(time TIMESTAMP NOT NULL, time_int BIGINT, time_broken DATE, metricid int, value double precision); CREATE TABLE regular(time TIMESTAMP NOT NULL, time_int BIGINT, time_date DATE, metricid int, value double precision); SELECT create_hypertable('hyper', 'time', chunk_time_interval => interval '20 day', create_default_indexes=>FALSE); psql:include/plan_hashagg_load.sql:9: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------- (1,public,hyper,t) ALTER TABLE hyper DROP COLUMN time_broken, ADD COLUMN time_date DATE; INSERT INTO metric(value) SELECT random()*100 FROM generate_series(0,10); INSERT INTO hyper SELECT t, EXTRACT(EPOCH FROM t), (EXTRACT(EPOCH FROM t)::int % 10)+1, 1.0, t::date FROM generate_series('2001-01-01', '2001-01-10', INTERVAL '1 second') t; INSERT INTO regular(time, time_int, time_date, metricid, value) SELECT t, EXTRACT(EPOCH FROM t), t::date, (EXTRACT(EPOCH FROM t)::int % 10) + 1, 1.0 FROM generate_series('2001-01-01', '2001-01-02', INTERVAL '1 second') t; --test some queries before analyze; EXPLAIN (buffers off, costs off) SELECT time_bucket('1 minute', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- Sort Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) DESC -> HashAggregate Group Key: time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time") -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) EXPLAIN (buffers off, costs off) SELECT date_trunc('minute', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- Sort Sort Key: (date_trunc('minute'::text, _hyper_1_1_chunk."time")) DESC -> HashAggregate Group Key: date_trunc('minute'::text, _hyper_1_1_chunk."time") -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) -- Test partitioning function on an open (time) dimension CREATE OR REPLACE FUNCTION unix_to_timestamp(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE SQL IMMUTABLE AS $BODY$ SELECT to_timestamp(unixtime); $BODY$; CREATE TABLE hyper_timefunc(time float8 NOT NULL, metricid int, VALUE double precision, time_date DATE); SELECT create_hypertable('hyper_timefunc', 'time', chunk_time_interval => interval '20 day', create_default_indexes=>FALSE, time_partitioning_func => 'unix_to_timestamp'); create_hypertable ----------------------------- (2,public,hyper_timefunc,t) INSERT INTO hyper_timefunc SELECT time_int, metricid, VALUE, time_date FROM hyper; ANALYZE metric; ANALYZE hyper; ANALYZE regular; ANALYZE hyper_timefunc; \ir include/plan_hashagg_query.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. :PREFIX SELECT time_bucket('1 minute', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) -> Sort Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT time_bucket('1 hour', time) AS MetricMinuteTs, metricid, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs, metricid ORDER BY MetricMinuteTs DESC, metricid; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('@ 1 hour'::interval, _hyper_1_1_chunk."time")), _hyper_1_1_chunk.metricid -> Sort Sort Key: (time_bucket('@ 1 hour'::interval, _hyper_1_1_chunk."time")) DESC, _hyper_1_1_chunk.metricid -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) --should be too many groups will not hashaggregate :PREFIX SELECT time_bucket('1 second', time) AS MetricMinuteTs, metricid, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs, metricid ORDER BY MetricMinuteTs DESC, metricid; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('@ 1 sec'::interval, _hyper_1_1_chunk."time")), _hyper_1_1_chunk.metricid -> Sort Sort Key: (time_bucket('@ 1 sec'::interval, _hyper_1_1_chunk."time")) DESC, _hyper_1_1_chunk.metricid -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT time_bucket('1 minute', time, INTERVAL '30 seconds') AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time", '@ 30 secs'::interval)) -> Sort Sort Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time", '@ 30 secs'::interval)) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT time_bucket(60, time_int) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('60'::bigint, _hyper_1_1_chunk.time_int)) -> Sort Sort Key: (time_bucket('60'::bigint, _hyper_1_1_chunk.time_int)) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT time_bucket(60, time_int, 10) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('60'::bigint, _hyper_1_1_chunk.time_int, '10'::bigint)) -> Sort Sort Key: (time_bucket('60'::bigint, _hyper_1_1_chunk.time_int, '10'::bigint)) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT time_bucket('1 day', time_date) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- Sort Sort Key: (time_bucket('@ 1 day'::interval, _hyper_1_1_chunk.time_date)) DESC -> HashAggregate Group Key: time_bucket('@ 1 day'::interval, _hyper_1_1_chunk.time_date) -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) :PREFIX SELECT date_trunc('minute', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (date_trunc('minute'::text, _hyper_1_1_chunk."time")) -> Sort Sort Key: (date_trunc('minute'::text, _hyper_1_1_chunk."time")) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) \set ON_ERROR_STOP 0 --can't optimize invalid time unit :PREFIX SELECT date_trunc('invalid', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (date_trunc('invalid'::text, _hyper_1_1_chunk."time")) -> Sort Sort Key: (date_trunc('invalid'::text, _hyper_1_1_chunk."time")) DESC -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) \set ON_ERROR_STOP 1 :PREFIX SELECT date_trunc('day', time_date) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- Sort Sort Key: (date_trunc('day'::text, (_hyper_1_1_chunk.time_date)::timestamp with time zone)) DESC -> HashAggregate Group Key: date_trunc('day'::text, (_hyper_1_1_chunk.time_date)::timestamp with time zone) -> Result -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) --joins --with hypertable, optimize :PREFIX SELECT time_bucket(3600, time_int, 10) AS MetricMinuteTs, metric.value, AVG(hyper.value) as avg FROM hyper JOIN metric ON (hyper.metricid = metric.id) WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs, metric.id ORDER BY MetricMinuteTs DESC, metric.id; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('3600'::bigint, _hyper_1_1_chunk.time_int, '10'::bigint)), metric.id -> Sort Sort Key: (time_bucket('3600'::bigint, _hyper_1_1_chunk.time_int, '10'::bigint)) DESC, metric.id -> Hash Join Hash Cond: (_hyper_1_1_chunk.metricid = metric.id) -> Seq Scan on _hyper_1_1_chunk Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) -> Hash -> Seq Scan on metric --no hypertable involved, no optimization :PREFIX SELECT time_bucket(3600, time_int, 10) AS MetricMinuteTs, metric.value, AVG(regular.value) as avg FROM regular JOIN metric ON (regular.metricid = metric.id) WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs, metric.id ORDER BY MetricMinuteTs DESC, metric.id; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('3600'::bigint, regular.time_int, '10'::bigint)), metric.id -> Sort Sort Key: (time_bucket('3600'::bigint, regular.time_int, '10'::bigint)) DESC, metric.id -> Nested Loop Join Filter: (metric.id = regular.metricid) -> Seq Scan on regular Filter: (("time" >= 'Thu Jan 04 00:00:00 2001'::timestamp without time zone) AND ("time" <= 'Fri Jan 05 01:00:00 2001'::timestamp without time zone)) -> Seq Scan on metric -- Try with time partitioning function. Currently not optimized for hash aggregates :PREFIX SELECT time_bucket('1 minute', unix_to_timestamp(time)) AS MetricMinuteTs, AVG(value) as avg FROM hyper_timefunc WHERE unix_to_timestamp(time) >= '2001-01-04T00:00:00' AND unix_to_timestamp(time) <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --- QUERY PLAN --- GroupAggregate Group Key: (time_bucket('@ 1 min'::interval, to_timestamp(_hyper_2_2_chunk."time"))) -> Sort Sort Key: (time_bucket('@ 1 min'::interval, to_timestamp(_hyper_2_2_chunk."time"))) DESC -> Result -> Seq Scan on _hyper_2_2_chunk Filter: ((to_timestamp("time") >= 'Thu Jan 04 00:00:00 2001 PST'::timestamp with time zone) AND (to_timestamp("time") <= 'Fri Jan 05 01:00:00 2001 PST'::timestamp with time zone)) \set ECHO none psql:include/plan_hashagg_query.sql:60: ERROR: unit "invalid" not recognized for type timestamp without time zone psql:include/plan_hashagg_query.sql:60: ERROR: unit "invalid" not recognized for type timestamp without time zone ================================================ FILE: test/expected/plan_hypertable_inline.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- test hypertable classification when query is in an inlineable function \set PREFIX 'EXPLAIN (buffers off, costs off)' CREATE TABLE test (a int, b bigint NOT NULL); SELECT create_hypertable('public.test', 'b', chunk_time_interval=>10); create_hypertable ------------------- (1,public,test,t) INSERT INTO test SELECT i, i FROM generate_series(1, 20) i; CREATE OR REPLACE FUNCTION test_f(_ts bigint) RETURNS SETOF test LANGUAGE SQL STABLE as $f$ SELECT DISTINCT ON (a) * FROM test WHERE b >= _ts AND b <= _ts + 2 $f$; -- plans must be the same in both cases -- specifically, the first plan should not contain the parent hypertable -- as that is a sign the pruning was not done successfully :PREFIX SELECT * FROM test_f(5); --- QUERY PLAN --- Unique -> Sort Sort Key: _hyper_1_1_chunk.a -> Index Scan using _hyper_1_1_chunk_test_b_idx on _hyper_1_1_chunk Index Cond: ((b >= '5'::bigint) AND (b <= '7'::bigint)) :PREFIX SELECT DISTINCT ON (a) * FROM test WHERE b >= 5 AND b <= 5 + 2; --- QUERY PLAN --- Unique -> Sort Sort Key: _hyper_1_1_chunk.a -> Index Scan using _hyper_1_1_chunk_test_b_idx on _hyper_1_1_chunk Index Cond: ((b >= 5) AND (b <= 7)) -- test with FOR UPDATE CREATE OR REPLACE FUNCTION test_f(_ts bigint) RETURNS SETOF test LANGUAGE SQL STABLE as $f$ SELECT * FROM test WHERE b >= _ts AND b <= _ts + 2 FOR UPDATE $f$; -- pruning should not be done by TimescaleDb in this case -- specifically, the parent hypertable must exist in the output plan :PREFIX SELECT * FROM test_f(5); --- QUERY PLAN --- Subquery Scan on test_f -> LockRows -> Append -> Seq Scan on test test_1 Filter: ((b >= '5'::bigint) AND (b <= '7'::bigint)) -> Index Scan using _hyper_1_1_chunk_test_b_idx on _hyper_1_1_chunk test_2 Index Cond: ((b >= '5'::bigint) AND (b <= '7'::bigint)) :PREFIX SELECT * FROM test WHERE b >= 5 AND b <= 5 + 2 FOR UPDATE; --- QUERY PLAN --- LockRows -> Index Scan using _hyper_1_1_chunk_test_b_idx on _hyper_1_1_chunk Index Cond: ((b >= 5) AND (b <= 7)) -- test with CTE -- these cases are just to make sure we're everything is alright with -- the way we identify hypertables to prune chunks - we abuse ctename -- for this purpose. So double-check if we're not breaking plans -- with CTEs here. CREATE OR REPLACE FUNCTION test_f(_ts bigint) RETURNS SETOF test LANGUAGE SQL STABLE as $f$ WITH ct AS MATERIALIZED ( SELECT DISTINCT ON (a) * FROM test WHERE b >= _ts AND b <= _ts + 2 ) SELECT * FROM ct $f$; :PREFIX SELECT * FROM test_f(5); --- QUERY PLAN --- CTE Scan on ct CTE ct -> Unique -> Sort Sort Key: _hyper_1_1_chunk.a -> Index Scan using _hyper_1_1_chunk_test_b_idx on _hyper_1_1_chunk Index Cond: ((b >= '5'::bigint) AND (b <= '7'::bigint)) :PREFIX WITH ct AS MATERIALIZED ( SELECT DISTINCT ON (a) * FROM test WHERE b >= 5 AND b <= 5 + 2 ) SELECT * FROM ct; --- QUERY PLAN --- CTE Scan on ct CTE ct -> Unique -> Sort Sort Key: _hyper_1_1_chunk.a -> Index Scan using _hyper_1_1_chunk_test_b_idx on _hyper_1_1_chunk Index Cond: ((b >= 5) AND (b <= 7)) -- CTE within CTE :PREFIX WITH ct AS MATERIALIZED ( SELECT * FROM test_f(5) ) SELECT * FROM ct; --- QUERY PLAN --- CTE Scan on ct CTE ct -> CTE Scan on ct ct_1 CTE ct -> Unique -> Sort Sort Key: _hyper_1_1_chunk.a -> Index Scan using _hyper_1_1_chunk_test_b_idx on _hyper_1_1_chunk Index Cond: ((b >= '5'::bigint) AND (b <= '7'::bigint)) -- CTE within NO MATERIALIZED CTE :PREFIX WITH ct AS NOT MATERIALIZED ( SELECT * FROM test_f(5) ) SELECT * FROM ct; --- QUERY PLAN --- CTE Scan on ct CTE ct -> Unique -> Sort Sort Key: _hyper_1_1_chunk.a -> Index Scan using _hyper_1_1_chunk_test_b_idx on _hyper_1_1_chunk Index Cond: ((b >= '5'::bigint) AND (b <= '7'::bigint)) ================================================ FILE: test/expected/plan_ordered_append-15.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- we run these with analyze to confirm that nodes that are not -- needed to fulfill the limit are not executed -- unfortunately this doesn't work on PostgreSQL 9.6 which lacks -- the ability to turn off analyze timing summary so we run -- them without ANALYZE on PostgreSQL 9.6, but since LATERAL plans -- are different across versions we need version specific output -- here anyway. \set TEST_BASE_NAME plan_ordered_append SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') as "TEST_LOAD_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') as "TEST_QUERY_NAME", format('%s/results/%s_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_OPTIMIZED", format('%s/results/%s_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_UNOPTIMIZED" \gset SELECT format('\! diff -u --label "Unoptimized result" --label "Optimized result" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') as "DIFF_CMD" \gset \set PREFIX 'EXPLAIN (analyze, buffers off, costs off, timing off, summary off)' \set PREFIX_NO_ANALYZE 'EXPLAIN (buffers off, costs off)' \ir :TEST_LOAD_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- create a now() function for repeatable testing that always returns -- the same timestamp. It needs to be marked STABLE CREATE OR REPLACE FUNCTION now_s() RETURNS timestamptz LANGUAGE PLPGSQL STABLE AS $BODY$ BEGIN RETURN '2000-01-08T0:00:00+0'::timestamptz; END; $BODY$; CREATE TABLE devices(device_id INT PRIMARY KEY, name TEXT); INSERT INTO devices VALUES (1,'Device 1'), (2,'Device 2'), (3,'Device 3'); -- create a second table where we create chunks in reverse order CREATE TABLE ordered_append_reverse(time timestamptz NOT NULL, device_id INT, value float); SELECT create_hypertable('ordered_append_reverse','time'); create_hypertable ------------------------------------- (1,public,ordered_append_reverse,t) INSERT INTO ordered_append_reverse SELECT generate_series('2000-01-18'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 0.5; -- table where dimension column is last column CREATE TABLE IF NOT EXISTS dimension_last( id INT8 NOT NULL, device_id INT NOT NULL, name TEXT NOT NULL, time timestamptz NOT NULL ); SELECT create_hypertable('dimension_last', 'time', chunk_time_interval => interval '1day', if_not_exists => True); create_hypertable ----------------------------- (2,public,dimension_last,t) -- table with only dimension column CREATE TABLE IF NOT EXISTS dimension_only( time timestamptz NOT NULL ); SELECT create_hypertable('dimension_only', 'time', chunk_time_interval => interval '1day', if_not_exists => True); create_hypertable ----------------------------- (3,public,dimension_only,t) INSERT INTO dimension_last SELECT 1,1,'Device 1',generate_series('2000-01-01 0:00:00+0'::timestamptz,'2000-01-04 23:59:00+0'::timestamptz,'1m'::interval); INSERT INTO dimension_only VALUES ('2000-01-01'), ('2000-01-03'), ('2000-01-05'), ('2000-01-07'); ANALYZE devices; ANALYZE ordered_append_reverse; ANALYZE dimension_last; ANALYZE dimension_only; -- create hypertable with indexes not on all chunks CREATE TABLE ht_missing_indexes(time timestamptz NOT NULL, device_id int, value float); SELECT create_hypertable('ht_missing_indexes','time'); create_hypertable --------------------------------- (4,public,ht_missing_indexes,t) INSERT INTO ht_missing_indexes SELECT generate_series('2000-01-01'::timestamptz,'2000-01-18'::timestamptz,'1m'::interval), 1, 0.5; INSERT INTO ht_missing_indexes SELECT generate_series('2000-01-01'::timestamptz,'2000-01-18'::timestamptz,'1m'::interval), 2, 1.5; INSERT INTO ht_missing_indexes SELECT generate_series('2000-01-01'::timestamptz,'2000-01-18'::timestamptz,'1m'::interval), 3, 2.5; -- drop index from 2nd chunk of ht_missing_indexes SELECT format('%I.%I',i.schemaname,i.indexname) AS "INDEX_NAME" FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable ht ON c.hypertable_id = ht.id INNER JOIN pg_indexes i ON i.schemaname = c.schema_name AND i.tablename=c.table_name WHERE ht.table_name = 'ht_missing_indexes' ORDER BY c.id LIMIT 1 OFFSET 1 \gset DROP INDEX :INDEX_NAME; ANALYZE ht_missing_indexes; -- create hypertable with with dropped columns CREATE TABLE ht_dropped_columns(c1 int, c2 int, c3 int, c4 int, c5 int, time timestamptz NOT NULL, device_id int, value float); SELECT create_hypertable('ht_dropped_columns','time'); create_hypertable --------------------------------- (5,public,ht_dropped_columns,t) ALTER TABLE ht_dropped_columns DROP COLUMN c1; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-01'::timestamptz,'2000-01-02'::timestamptz,'1m'::interval), 1, 0.5; ALTER TABLE ht_dropped_columns DROP COLUMN c2; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-08'::timestamptz,'2000-01-09'::timestamptz,'1m'::interval), 1, 0.5; ALTER TABLE ht_dropped_columns DROP COLUMN c3; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-15'::timestamptz,'2000-01-16'::timestamptz,'1m'::interval), 1, 0.5; ALTER TABLE ht_dropped_columns DROP COLUMN c4; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-22'::timestamptz,'2000-01-23'::timestamptz,'1m'::interval), 1, 0.5; ALTER TABLE ht_dropped_columns DROP COLUMN c5; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-29'::timestamptz,'2000-01-30'::timestamptz,'1m'::interval), 1, 0.5; ANALYZE ht_dropped_columns; CREATE TABLE space2(time timestamptz NOT NULL, device_id int NOT NULL, tag_id int NOT NULL, value float); SELECT create_hypertable('space2','time','device_id',number_partitions:=3); create_hypertable --------------------- (6,public,space2,t) SELECT add_dimension('space2','tag_id',number_partitions:=3); add_dimension ---------------------------- (8,public,space2,tag_id,t) INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 1, 1.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 1, 2.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 3, 1, 3.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 2, 1.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 2, 2.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 3, 2, 3.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 3, 1.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 3, 2.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 3, 3, 3.5; ANALYZE space2; CREATE TABLE space3(time timestamptz NOT NULL, x int NOT NULL, y int NOT NULL, z int NOT NULL, value float); SELECT create_hypertable('space3','time','x',number_partitions:=2); create_hypertable --------------------- (7,public,space3,t) SELECT add_dimension('space3','y',number_partitions:=2); add_dimension ------------------------ (11,public,space3,y,t) SELECT add_dimension('space3','z',number_partitions:=2); add_dimension ------------------------ (12,public,space3,z,t) INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 1, 1, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 1, 2, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 2, 1, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 2, 2, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 1, 1, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 1, 2, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 2, 1, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 2, 2, 1.5; ANALYZE space3; CREATE TABLE sortopt_test(time timestamptz NOT NULL, device TEXT); SELECT create_hypertable('sortopt_test','time',create_default_indexes:=false); create_hypertable --------------------------- (8,public,sortopt_test,t) -- since alpine does not support locales we cant test collations in our ci -- CREATE COLLATION IF NOT EXISTS en_US(LOCALE='en_US.utf8'); -- CREATE INDEX time_device_utf8 ON sortopt_test(time, device COLLATE "en_US"); CREATE INDEX time_device_nullsfirst ON sortopt_test(time, device NULLS FIRST); CREATE INDEX time_device_nullslast ON sortopt_test(time, device DESC NULLS LAST); INSERT INTO sortopt_test SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 'Device 1'; ANALYZE sortopt_test; \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- print chunks ordered by time to ensure ordering we want SELECT ht.table_name AS hypertable, c.table_name AS chunk, ds.range_start FROM _timescaledb_catalog.chunk c INNER JOIN LATERAL(SELECT * FROM _timescaledb_catalog.chunk_constraint cc WHERE c.id = cc.chunk_id ORDER BY cc.dimension_slice_id LIMIT 1) cc ON true INNER JOIN _timescaledb_catalog.dimension_slice ds ON ds.id=cc.dimension_slice_id INNER JOIN _timescaledb_catalog.dimension d ON ds.dimension_id = d.id INNER JOIN _timescaledb_catalog.hypertable ht ON d.hypertable_id = ht.id ORDER BY ht.table_name, range_start, chunk; hypertable | chunk | range_start ------------------------+-------------------+---------------------- dimension_last | _hyper_2_4_chunk | 946684800000000 dimension_last | _hyper_2_5_chunk | 946771200000000 dimension_last | _hyper_2_6_chunk | 946857600000000 dimension_last | _hyper_2_7_chunk | 946944000000000 dimension_only | _hyper_3_8_chunk | 946684800000000 dimension_only | _hyper_3_9_chunk | 946857600000000 dimension_only | _hyper_3_10_chunk | 947030400000000 dimension_only | _hyper_3_11_chunk | 947203200000000 ht_dropped_columns | _hyper_5_15_chunk | 946512000000000 ht_dropped_columns | _hyper_5_16_chunk | 947116800000000 ht_dropped_columns | _hyper_5_17_chunk | 947721600000000 ht_dropped_columns | _hyper_5_18_chunk | 948326400000000 ht_dropped_columns | _hyper_5_19_chunk | 948931200000000 ht_missing_indexes | _hyper_4_12_chunk | 946512000000000 ht_missing_indexes | _hyper_4_13_chunk | 947116800000000 ht_missing_indexes | _hyper_4_14_chunk | 947721600000000 ordered_append_reverse | _hyper_1_3_chunk | 946512000000000 ordered_append_reverse | _hyper_1_2_chunk | 947116800000000 ordered_append_reverse | _hyper_1_1_chunk | 947721600000000 sortopt_test | _hyper_8_55_chunk | 946512000000000 sortopt_test | _hyper_8_54_chunk | 947116800000000 space2 | _hyper_6_21_chunk | -9223372036854775808 space2 | _hyper_6_23_chunk | -9223372036854775808 space2 | _hyper_6_25_chunk | -9223372036854775808 space2 | _hyper_6_27_chunk | -9223372036854775808 space2 | _hyper_6_33_chunk | -9223372036854775808 space2 | _hyper_6_29_chunk | 946512000000000 space2 | _hyper_6_31_chunk | 946512000000000 space2 | _hyper_6_35_chunk | 946512000000000 space2 | _hyper_6_37_chunk | 946512000000000 space2 | _hyper_6_20_chunk | 947116800000000 space2 | _hyper_6_22_chunk | 947116800000000 space2 | _hyper_6_24_chunk | 947116800000000 space2 | _hyper_6_26_chunk | 947116800000000 space2 | _hyper_6_28_chunk | 947116800000000 space2 | _hyper_6_30_chunk | 947116800000000 space2 | _hyper_6_32_chunk | 947116800000000 space2 | _hyper_6_34_chunk | 947116800000000 space2 | _hyper_6_36_chunk | 947116800000000 space3 | _hyper_7_39_chunk | -9223372036854775808 space3 | _hyper_7_41_chunk | -9223372036854775808 space3 | _hyper_7_43_chunk | -9223372036854775808 space3 | _hyper_7_45_chunk | -9223372036854775808 space3 | _hyper_7_47_chunk | -9223372036854775808 space3 | _hyper_7_49_chunk | -9223372036854775808 space3 | _hyper_7_51_chunk | -9223372036854775808 space3 | _hyper_7_53_chunk | 946512000000000 space3 | _hyper_7_38_chunk | 947116800000000 space3 | _hyper_7_40_chunk | 947116800000000 space3 | _hyper_7_42_chunk | 947116800000000 space3 | _hyper_7_44_chunk | 947116800000000 space3 | _hyper_7_46_chunk | 947116800000000 space3 | _hyper_7_48_chunk | 947116800000000 space3 | _hyper_7_50_chunk | 947116800000000 space3 | _hyper_7_52_chunk | 947116800000000 -- test ASC for reverse ordered chunks :PREFIX SELECT time, device_id, value FROM ordered_append_reverse ORDER BY time ASC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on ordered_append_reverse (actual rows=1.00 loops=1) Order: ordered_append_reverse."time" -> Index Scan Backward using _hyper_1_3_chunk_ordered_append_reverse_time_idx on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_2_chunk_ordered_append_reverse_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan Backward using _hyper_1_1_chunk_ordered_append_reverse_time_idx on _hyper_1_1_chunk (never executed) -- test DESC for reverse ordered chunks :PREFIX SELECT time, device_id, value FROM ordered_append_reverse ORDER BY time DESC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on ordered_append_reverse (actual rows=1.00 loops=1) Order: ordered_append_reverse."time" DESC -> Index Scan using _hyper_1_1_chunk_ordered_append_reverse_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_2_chunk_ordered_append_reverse_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan using _hyper_1_3_chunk_ordered_append_reverse_time_idx on _hyper_1_3_chunk (never executed) -- test query with ORDER BY time_bucket, device_id -- must not use ordered append :PREFIX SELECT time_bucket('1d',time), device_id, name FROM dimension_last ORDER BY time_bucket('1d',time), device_id LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Sort (actual rows=1.00 loops=1) Sort Key: (time_bucket('@ 1 day'::interval, dimension_last."time")), dimension_last.device_id Sort Method: top-N heapsort -> Result (actual rows=5760.00 loops=1) -> Append (actual rows=5760.00 loops=1) -> Seq Scan on _hyper_2_4_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_5_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_6_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_7_chunk (actual rows=1440.00 loops=1) -- test query with ORDER BY date_trunc, device_id -- must not use ordered append :PREFIX SELECT date_trunc('day',time), device_id, name FROM dimension_last ORDER BY 1,2 LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Sort (actual rows=1.00 loops=1) Sort Key: (date_trunc('day'::text, dimension_last."time")), dimension_last.device_id Sort Method: top-N heapsort -> Result (actual rows=5760.00 loops=1) -> Append (actual rows=5760.00 loops=1) -> Seq Scan on _hyper_2_4_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_5_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_6_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_7_chunk (actual rows=1440.00 loops=1) -- test with table with only dimension column :PREFIX SELECT * FROM dimension_only ORDER BY time DESC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on dimension_only (actual rows=1.00 loops=1) Order: dimension_only."time" DESC -> Index Only Scan using _hyper_3_11_chunk_dimension_only_time_idx on _hyper_3_11_chunk (actual rows=1.00 loops=1) -> Index Only Scan using _hyper_3_10_chunk_dimension_only_time_idx on _hyper_3_10_chunk (never executed) -> Index Only Scan using _hyper_3_9_chunk_dimension_only_time_idx on _hyper_3_9_chunk (never executed) -> Index Only Scan using _hyper_3_8_chunk_dimension_only_time_idx on _hyper_3_8_chunk (never executed) -- test LEFT JOIN against hypertable :PREFIX_NO_ANALYZE SELECT * FROM dimension_last LEFT JOIN dimension_only USING (time) ORDER BY dimension_last.time DESC LIMIT 2; --- QUERY PLAN --- Limit -> Nested Loop Left Join Join Filter: (dimension_last."time" = dimension_only."time") -> Custom Scan (ChunkAppend) on dimension_last Order: dimension_last."time" DESC -> Index Scan using _hyper_2_7_chunk_dimension_last_time_idx on _hyper_2_7_chunk -> Index Scan using _hyper_2_6_chunk_dimension_last_time_idx on _hyper_2_6_chunk -> Index Scan using _hyper_2_5_chunk_dimension_last_time_idx on _hyper_2_5_chunk -> Index Scan using _hyper_2_4_chunk_dimension_last_time_idx on _hyper_2_4_chunk -> Materialize -> Append -> Seq Scan on _hyper_3_11_chunk -> Seq Scan on _hyper_3_10_chunk -> Seq Scan on _hyper_3_9_chunk -> Seq Scan on _hyper_3_8_chunk -- test INNER JOIN against non-hypertable :PREFIX_NO_ANALYZE SELECT * FROM dimension_last INNER JOIN dimension_only USING (time) ORDER BY dimension_last.time DESC LIMIT 2; --- QUERY PLAN --- Limit -> Nested Loop -> Custom Scan (ChunkAppend) on dimension_only Order: dimension_only."time" DESC -> Index Only Scan using _hyper_3_11_chunk_dimension_only_time_idx on _hyper_3_11_chunk -> Index Only Scan using _hyper_3_10_chunk_dimension_only_time_idx on _hyper_3_10_chunk -> Index Only Scan using _hyper_3_9_chunk_dimension_only_time_idx on _hyper_3_9_chunk -> Index Only Scan using _hyper_3_8_chunk_dimension_only_time_idx on _hyper_3_8_chunk -> Append -> Index Scan using _hyper_2_7_chunk_dimension_last_time_idx on _hyper_2_7_chunk Index Cond: ("time" = dimension_only."time") -> Index Scan using _hyper_2_6_chunk_dimension_last_time_idx on _hyper_2_6_chunk Index Cond: ("time" = dimension_only."time") -> Index Scan using _hyper_2_5_chunk_dimension_last_time_idx on _hyper_2_5_chunk Index Cond: ("time" = dimension_only."time") -> Index Scan using _hyper_2_4_chunk_dimension_last_time_idx on _hyper_2_4_chunk Index Cond: ("time" = dimension_only."time") -- test join against non-hypertable :PREFIX SELECT * FROM dimension_last INNER JOIN devices USING(device_id) ORDER BY dimension_last.time DESC LIMIT 2; --- QUERY PLAN --- Limit (actual rows=2.00 loops=1) -> Nested Loop (actual rows=2.00 loops=1) Join Filter: (dimension_last.device_id = devices.device_id) -> Custom Scan (ChunkAppend) on dimension_last (actual rows=2.00 loops=1) Order: dimension_last."time" DESC -> Index Scan using _hyper_2_7_chunk_dimension_last_time_idx on _hyper_2_7_chunk (actual rows=2.00 loops=1) -> Index Scan using _hyper_2_6_chunk_dimension_last_time_idx on _hyper_2_6_chunk (never executed) -> Index Scan using _hyper_2_5_chunk_dimension_last_time_idx on _hyper_2_5_chunk (never executed) -> Index Scan using _hyper_2_4_chunk_dimension_last_time_idx on _hyper_2_4_chunk (never executed) -> Materialize (actual rows=1.00 loops=2) -> Seq Scan on devices (actual rows=1.00 loops=1) -- test hypertable with index missing on one chunk :PREFIX SELECT time, device_id, value FROM ht_missing_indexes ORDER BY time ASC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on ht_missing_indexes (actual rows=1.00 loops=1) Order: ht_missing_indexes."time" -> Index Scan Backward using _hyper_4_12_chunk_ht_missing_indexes_time_idx on _hyper_4_12_chunk (actual rows=1.00 loops=1) -> Sort (never executed) Sort Key: _hyper_4_13_chunk."time" -> Seq Scan on _hyper_4_13_chunk (never executed) -> Index Scan Backward using _hyper_4_14_chunk_ht_missing_indexes_time_idx on _hyper_4_14_chunk (never executed) -- test hypertable with index missing on one chunk -- and no data :PREFIX SELECT time, device_id, value FROM ht_missing_indexes WHERE device_id = 2 ORDER BY time DESC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on ht_missing_indexes (actual rows=1.00 loops=1) Order: ht_missing_indexes."time" DESC -> Index Scan using _hyper_4_14_chunk_ht_missing_indexes_time_idx on _hyper_4_14_chunk (actual rows=1.00 loops=1) Filter: (device_id = 2) Rows Removed by Filter: 1 -> Sort (never executed) Sort Key: _hyper_4_13_chunk."time" DESC -> Seq Scan on _hyper_4_13_chunk (never executed) Filter: (device_id = 2) -> Index Scan using _hyper_4_12_chunk_ht_missing_indexes_time_idx on _hyper_4_12_chunk (never executed) Filter: (device_id = 2) -- test hypertable with index missing on one chunk -- and no data :PREFIX SELECT time, device_id, value FROM ht_missing_indexes WHERE time > '2000-01-07' ORDER BY time LIMIT 10; --- QUERY PLAN --- Limit (actual rows=10.00 loops=1) -> Custom Scan (ChunkAppend) on ht_missing_indexes (actual rows=10.00 loops=1) Order: ht_missing_indexes."time" -> Sort (actual rows=10.00 loops=1) Sort Key: _hyper_4_13_chunk."time" Sort Method: top-N heapsort -> Seq Scan on _hyper_4_13_chunk (actual rows=24477.00 loops=1) Filter: ("time" > 'Fri Jan 07 00:00:00 2000 PST'::timestamp with time zone) Rows Removed by Filter: 5763 -> Index Scan Backward using _hyper_4_14_chunk_ht_missing_indexes_time_idx on _hyper_4_14_chunk (never executed) -- test hypertable with dropped columns :PREFIX SELECT time, device_id, value FROM ht_dropped_columns ORDER BY time ASC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on ht_dropped_columns (actual rows=1.00 loops=1) Order: ht_dropped_columns."time" -> Index Scan Backward using _hyper_5_15_chunk_ht_dropped_columns_time_idx on _hyper_5_15_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_5_16_chunk_ht_dropped_columns_time_idx on _hyper_5_16_chunk (never executed) -> Index Scan Backward using _hyper_5_17_chunk_ht_dropped_columns_time_idx on _hyper_5_17_chunk (never executed) -> Index Scan Backward using _hyper_5_18_chunk_ht_dropped_columns_time_idx on _hyper_5_18_chunk (never executed) -> Index Scan Backward using _hyper_5_19_chunk_ht_dropped_columns_time_idx on _hyper_5_19_chunk (never executed) -- test hypertable with dropped columns :PREFIX SELECT time, device_id, value FROM ht_dropped_columns WHERE device_id = 1 ORDER BY time DESC; --- QUERY PLAN --- Custom Scan (ChunkAppend) on ht_dropped_columns (actual rows=7205.00 loops=1) Order: ht_dropped_columns."time" DESC -> Index Scan using _hyper_5_19_chunk_ht_dropped_columns_time_idx on _hyper_5_19_chunk (actual rows=1441.00 loops=1) Filter: (device_id = 1) -> Index Scan using _hyper_5_18_chunk_ht_dropped_columns_time_idx on _hyper_5_18_chunk (actual rows=1441.00 loops=1) Filter: (device_id = 1) -> Index Scan using _hyper_5_17_chunk_ht_dropped_columns_time_idx on _hyper_5_17_chunk (actual rows=1441.00 loops=1) Filter: (device_id = 1) -> Index Scan using _hyper_5_16_chunk_ht_dropped_columns_time_idx on _hyper_5_16_chunk (actual rows=1441.00 loops=1) Filter: (device_id = 1) -> Index Scan using _hyper_5_15_chunk_ht_dropped_columns_time_idx on _hyper_5_15_chunk (actual rows=1441.00 loops=1) Filter: (device_id = 1) -- test hypertable with 2 space dimensions :PREFIX SELECT time, device_id, value FROM space2 ORDER BY time DESC; --- QUERY PLAN --- Custom Scan (ChunkAppend) on space2 (actual rows=116649.00 loops=1) Order: space2."time" DESC -> Merge Append (actual rows=56169.00 loops=1) Sort Key: space2."time" DESC -> Index Scan using _hyper_6_36_chunk_space2_time_idx on _hyper_6_36_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_34_chunk_space2_time_idx on _hyper_6_34_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_32_chunk_space2_time_idx on _hyper_6_32_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_30_chunk_space2_time_idx on _hyper_6_30_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_28_chunk_space2_time_idx on _hyper_6_28_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_26_chunk_space2_time_idx on _hyper_6_26_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_24_chunk_space2_time_idx on _hyper_6_24_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_22_chunk_space2_time_idx on _hyper_6_22_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_20_chunk_space2_time_idx on _hyper_6_20_chunk (actual rows=6241.00 loops=1) -> Merge Append (actual rows=60480.00 loops=1) Sort Key: space2."time" DESC -> Index Scan using _hyper_6_37_chunk_space2_time_idx on _hyper_6_37_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_35_chunk_space2_time_idx on _hyper_6_35_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_33_chunk_space2_time_idx on _hyper_6_33_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_31_chunk_space2_time_idx on _hyper_6_31_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_29_chunk_space2_time_idx on _hyper_6_29_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_27_chunk_space2_time_idx on _hyper_6_27_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_25_chunk_space2_time_idx on _hyper_6_25_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_23_chunk_space2_time_idx on _hyper_6_23_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_21_chunk_space2_time_idx on _hyper_6_21_chunk (actual rows=6720.00 loops=1) -- test hypertable with 3 space dimensions :PREFIX SELECT time FROM space3 ORDER BY time DESC; --- QUERY PLAN --- Custom Scan (ChunkAppend) on space3 (actual rows=103688.00 loops=1) Order: space3."time" DESC -> Merge Append (actual rows=49928.00 loops=1) Sort Key: space3."time" DESC -> Index Only Scan using _hyper_7_52_chunk_space3_time_idx on _hyper_7_52_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_50_chunk_space3_time_idx on _hyper_7_50_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_48_chunk_space3_time_idx on _hyper_7_48_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_46_chunk_space3_time_idx on _hyper_7_46_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_44_chunk_space3_time_idx on _hyper_7_44_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_42_chunk_space3_time_idx on _hyper_7_42_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_40_chunk_space3_time_idx on _hyper_7_40_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_38_chunk_space3_time_idx on _hyper_7_38_chunk (actual rows=6241.00 loops=1) -> Merge Append (actual rows=53760.00 loops=1) Sort Key: space3."time" DESC -> Index Only Scan using _hyper_7_53_chunk_space3_time_idx on _hyper_7_53_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_51_chunk_space3_time_idx on _hyper_7_51_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_49_chunk_space3_time_idx on _hyper_7_49_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_47_chunk_space3_time_idx on _hyper_7_47_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_45_chunk_space3_time_idx on _hyper_7_45_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_43_chunk_space3_time_idx on _hyper_7_43_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_41_chunk_space3_time_idx on _hyper_7_41_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_39_chunk_space3_time_idx on _hyper_7_39_chunk (actual rows=6720.00 loops=1) -- test COLLATION -- cant be tested in our ci because alpine doesnt support locales -- :PREFIX SELECT * FROM sortopt_test ORDER BY time, device COLLATE "en_US.utf8"; -- test NULLS FIRST :PREFIX SELECT * FROM sortopt_test ORDER BY time, device NULLS FIRST; --- QUERY PLAN --- Custom Scan (ChunkAppend) on sortopt_test (actual rows=12961.00 loops=1) Order: sortopt_test."time", sortopt_test.device NULLS FIRST -> Index Only Scan using _hyper_8_55_chunk_time_device_nullsfirst on _hyper_8_55_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_8_54_chunk_time_device_nullsfirst on _hyper_8_54_chunk (actual rows=6241.00 loops=1) -- test NULLS LAST :PREFIX SELECT * FROM sortopt_test ORDER BY time, device DESC NULLS LAST; --- QUERY PLAN --- Custom Scan (ChunkAppend) on sortopt_test (actual rows=12961.00 loops=1) Order: sortopt_test."time", sortopt_test.device DESC NULLS LAST -> Index Only Scan using _hyper_8_55_chunk_time_device_nullslast on _hyper_8_55_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_8_54_chunk_time_device_nullslast on _hyper_8_54_chunk (actual rows=6241.00 loops=1) --generate the results into two different files \set ECHO errors ================================================ FILE: test/expected/plan_ordered_append-16.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- we run these with analyze to confirm that nodes that are not -- needed to fulfill the limit are not executed -- unfortunately this doesn't work on PostgreSQL 9.6 which lacks -- the ability to turn off analyze timing summary so we run -- them without ANALYZE on PostgreSQL 9.6, but since LATERAL plans -- are different across versions we need version specific output -- here anyway. \set TEST_BASE_NAME plan_ordered_append SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') as "TEST_LOAD_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') as "TEST_QUERY_NAME", format('%s/results/%s_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_OPTIMIZED", format('%s/results/%s_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_UNOPTIMIZED" \gset SELECT format('\! diff -u --label "Unoptimized result" --label "Optimized result" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') as "DIFF_CMD" \gset \set PREFIX 'EXPLAIN (analyze, buffers off, costs off, timing off, summary off)' \set PREFIX_NO_ANALYZE 'EXPLAIN (buffers off, costs off)' \ir :TEST_LOAD_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- create a now() function for repeatable testing that always returns -- the same timestamp. It needs to be marked STABLE CREATE OR REPLACE FUNCTION now_s() RETURNS timestamptz LANGUAGE PLPGSQL STABLE AS $BODY$ BEGIN RETURN '2000-01-08T0:00:00+0'::timestamptz; END; $BODY$; CREATE TABLE devices(device_id INT PRIMARY KEY, name TEXT); INSERT INTO devices VALUES (1,'Device 1'), (2,'Device 2'), (3,'Device 3'); -- create a second table where we create chunks in reverse order CREATE TABLE ordered_append_reverse(time timestamptz NOT NULL, device_id INT, value float); SELECT create_hypertable('ordered_append_reverse','time'); create_hypertable ------------------------------------- (1,public,ordered_append_reverse,t) INSERT INTO ordered_append_reverse SELECT generate_series('2000-01-18'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 0.5; -- table where dimension column is last column CREATE TABLE IF NOT EXISTS dimension_last( id INT8 NOT NULL, device_id INT NOT NULL, name TEXT NOT NULL, time timestamptz NOT NULL ); SELECT create_hypertable('dimension_last', 'time', chunk_time_interval => interval '1day', if_not_exists => True); create_hypertable ----------------------------- (2,public,dimension_last,t) -- table with only dimension column CREATE TABLE IF NOT EXISTS dimension_only( time timestamptz NOT NULL ); SELECT create_hypertable('dimension_only', 'time', chunk_time_interval => interval '1day', if_not_exists => True); create_hypertable ----------------------------- (3,public,dimension_only,t) INSERT INTO dimension_last SELECT 1,1,'Device 1',generate_series('2000-01-01 0:00:00+0'::timestamptz,'2000-01-04 23:59:00+0'::timestamptz,'1m'::interval); INSERT INTO dimension_only VALUES ('2000-01-01'), ('2000-01-03'), ('2000-01-05'), ('2000-01-07'); ANALYZE devices; ANALYZE ordered_append_reverse; ANALYZE dimension_last; ANALYZE dimension_only; -- create hypertable with indexes not on all chunks CREATE TABLE ht_missing_indexes(time timestamptz NOT NULL, device_id int, value float); SELECT create_hypertable('ht_missing_indexes','time'); create_hypertable --------------------------------- (4,public,ht_missing_indexes,t) INSERT INTO ht_missing_indexes SELECT generate_series('2000-01-01'::timestamptz,'2000-01-18'::timestamptz,'1m'::interval), 1, 0.5; INSERT INTO ht_missing_indexes SELECT generate_series('2000-01-01'::timestamptz,'2000-01-18'::timestamptz,'1m'::interval), 2, 1.5; INSERT INTO ht_missing_indexes SELECT generate_series('2000-01-01'::timestamptz,'2000-01-18'::timestamptz,'1m'::interval), 3, 2.5; -- drop index from 2nd chunk of ht_missing_indexes SELECT format('%I.%I',i.schemaname,i.indexname) AS "INDEX_NAME" FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable ht ON c.hypertable_id = ht.id INNER JOIN pg_indexes i ON i.schemaname = c.schema_name AND i.tablename=c.table_name WHERE ht.table_name = 'ht_missing_indexes' ORDER BY c.id LIMIT 1 OFFSET 1 \gset DROP INDEX :INDEX_NAME; ANALYZE ht_missing_indexes; -- create hypertable with with dropped columns CREATE TABLE ht_dropped_columns(c1 int, c2 int, c3 int, c4 int, c5 int, time timestamptz NOT NULL, device_id int, value float); SELECT create_hypertable('ht_dropped_columns','time'); create_hypertable --------------------------------- (5,public,ht_dropped_columns,t) ALTER TABLE ht_dropped_columns DROP COLUMN c1; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-01'::timestamptz,'2000-01-02'::timestamptz,'1m'::interval), 1, 0.5; ALTER TABLE ht_dropped_columns DROP COLUMN c2; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-08'::timestamptz,'2000-01-09'::timestamptz,'1m'::interval), 1, 0.5; ALTER TABLE ht_dropped_columns DROP COLUMN c3; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-15'::timestamptz,'2000-01-16'::timestamptz,'1m'::interval), 1, 0.5; ALTER TABLE ht_dropped_columns DROP COLUMN c4; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-22'::timestamptz,'2000-01-23'::timestamptz,'1m'::interval), 1, 0.5; ALTER TABLE ht_dropped_columns DROP COLUMN c5; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-29'::timestamptz,'2000-01-30'::timestamptz,'1m'::interval), 1, 0.5; ANALYZE ht_dropped_columns; CREATE TABLE space2(time timestamptz NOT NULL, device_id int NOT NULL, tag_id int NOT NULL, value float); SELECT create_hypertable('space2','time','device_id',number_partitions:=3); create_hypertable --------------------- (6,public,space2,t) SELECT add_dimension('space2','tag_id',number_partitions:=3); add_dimension ---------------------------- (8,public,space2,tag_id,t) INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 1, 1.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 1, 2.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 3, 1, 3.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 2, 1.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 2, 2.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 3, 2, 3.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 3, 1.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 3, 2.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 3, 3, 3.5; ANALYZE space2; CREATE TABLE space3(time timestamptz NOT NULL, x int NOT NULL, y int NOT NULL, z int NOT NULL, value float); SELECT create_hypertable('space3','time','x',number_partitions:=2); create_hypertable --------------------- (7,public,space3,t) SELECT add_dimension('space3','y',number_partitions:=2); add_dimension ------------------------ (11,public,space3,y,t) SELECT add_dimension('space3','z',number_partitions:=2); add_dimension ------------------------ (12,public,space3,z,t) INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 1, 1, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 1, 2, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 2, 1, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 2, 2, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 1, 1, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 1, 2, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 2, 1, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 2, 2, 1.5; ANALYZE space3; CREATE TABLE sortopt_test(time timestamptz NOT NULL, device TEXT); SELECT create_hypertable('sortopt_test','time',create_default_indexes:=false); create_hypertable --------------------------- (8,public,sortopt_test,t) -- since alpine does not support locales we cant test collations in our ci -- CREATE COLLATION IF NOT EXISTS en_US(LOCALE='en_US.utf8'); -- CREATE INDEX time_device_utf8 ON sortopt_test(time, device COLLATE "en_US"); CREATE INDEX time_device_nullsfirst ON sortopt_test(time, device NULLS FIRST); CREATE INDEX time_device_nullslast ON sortopt_test(time, device DESC NULLS LAST); INSERT INTO sortopt_test SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 'Device 1'; ANALYZE sortopt_test; \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- print chunks ordered by time to ensure ordering we want SELECT ht.table_name AS hypertable, c.table_name AS chunk, ds.range_start FROM _timescaledb_catalog.chunk c INNER JOIN LATERAL(SELECT * FROM _timescaledb_catalog.chunk_constraint cc WHERE c.id = cc.chunk_id ORDER BY cc.dimension_slice_id LIMIT 1) cc ON true INNER JOIN _timescaledb_catalog.dimension_slice ds ON ds.id=cc.dimension_slice_id INNER JOIN _timescaledb_catalog.dimension d ON ds.dimension_id = d.id INNER JOIN _timescaledb_catalog.hypertable ht ON d.hypertable_id = ht.id ORDER BY ht.table_name, range_start, chunk; hypertable | chunk | range_start ------------------------+-------------------+---------------------- dimension_last | _hyper_2_4_chunk | 946684800000000 dimension_last | _hyper_2_5_chunk | 946771200000000 dimension_last | _hyper_2_6_chunk | 946857600000000 dimension_last | _hyper_2_7_chunk | 946944000000000 dimension_only | _hyper_3_8_chunk | 946684800000000 dimension_only | _hyper_3_9_chunk | 946857600000000 dimension_only | _hyper_3_10_chunk | 947030400000000 dimension_only | _hyper_3_11_chunk | 947203200000000 ht_dropped_columns | _hyper_5_15_chunk | 946512000000000 ht_dropped_columns | _hyper_5_16_chunk | 947116800000000 ht_dropped_columns | _hyper_5_17_chunk | 947721600000000 ht_dropped_columns | _hyper_5_18_chunk | 948326400000000 ht_dropped_columns | _hyper_5_19_chunk | 948931200000000 ht_missing_indexes | _hyper_4_12_chunk | 946512000000000 ht_missing_indexes | _hyper_4_13_chunk | 947116800000000 ht_missing_indexes | _hyper_4_14_chunk | 947721600000000 ordered_append_reverse | _hyper_1_3_chunk | 946512000000000 ordered_append_reverse | _hyper_1_2_chunk | 947116800000000 ordered_append_reverse | _hyper_1_1_chunk | 947721600000000 sortopt_test | _hyper_8_55_chunk | 946512000000000 sortopt_test | _hyper_8_54_chunk | 947116800000000 space2 | _hyper_6_21_chunk | -9223372036854775808 space2 | _hyper_6_23_chunk | -9223372036854775808 space2 | _hyper_6_25_chunk | -9223372036854775808 space2 | _hyper_6_27_chunk | -9223372036854775808 space2 | _hyper_6_33_chunk | -9223372036854775808 space2 | _hyper_6_29_chunk | 946512000000000 space2 | _hyper_6_31_chunk | 946512000000000 space2 | _hyper_6_35_chunk | 946512000000000 space2 | _hyper_6_37_chunk | 946512000000000 space2 | _hyper_6_20_chunk | 947116800000000 space2 | _hyper_6_22_chunk | 947116800000000 space2 | _hyper_6_24_chunk | 947116800000000 space2 | _hyper_6_26_chunk | 947116800000000 space2 | _hyper_6_28_chunk | 947116800000000 space2 | _hyper_6_30_chunk | 947116800000000 space2 | _hyper_6_32_chunk | 947116800000000 space2 | _hyper_6_34_chunk | 947116800000000 space2 | _hyper_6_36_chunk | 947116800000000 space3 | _hyper_7_39_chunk | -9223372036854775808 space3 | _hyper_7_41_chunk | -9223372036854775808 space3 | _hyper_7_43_chunk | -9223372036854775808 space3 | _hyper_7_45_chunk | -9223372036854775808 space3 | _hyper_7_47_chunk | -9223372036854775808 space3 | _hyper_7_49_chunk | -9223372036854775808 space3 | _hyper_7_51_chunk | -9223372036854775808 space3 | _hyper_7_53_chunk | 946512000000000 space3 | _hyper_7_38_chunk | 947116800000000 space3 | _hyper_7_40_chunk | 947116800000000 space3 | _hyper_7_42_chunk | 947116800000000 space3 | _hyper_7_44_chunk | 947116800000000 space3 | _hyper_7_46_chunk | 947116800000000 space3 | _hyper_7_48_chunk | 947116800000000 space3 | _hyper_7_50_chunk | 947116800000000 space3 | _hyper_7_52_chunk | 947116800000000 -- test ASC for reverse ordered chunks :PREFIX SELECT time, device_id, value FROM ordered_append_reverse ORDER BY time ASC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on ordered_append_reverse (actual rows=1.00 loops=1) Order: ordered_append_reverse."time" -> Index Scan Backward using _hyper_1_3_chunk_ordered_append_reverse_time_idx on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_2_chunk_ordered_append_reverse_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan Backward using _hyper_1_1_chunk_ordered_append_reverse_time_idx on _hyper_1_1_chunk (never executed) -- test DESC for reverse ordered chunks :PREFIX SELECT time, device_id, value FROM ordered_append_reverse ORDER BY time DESC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on ordered_append_reverse (actual rows=1.00 loops=1) Order: ordered_append_reverse."time" DESC -> Index Scan using _hyper_1_1_chunk_ordered_append_reverse_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_2_chunk_ordered_append_reverse_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan using _hyper_1_3_chunk_ordered_append_reverse_time_idx on _hyper_1_3_chunk (never executed) -- test query with ORDER BY time_bucket, device_id -- must not use ordered append :PREFIX SELECT time_bucket('1d',time), device_id, name FROM dimension_last ORDER BY time_bucket('1d',time), device_id LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Sort (actual rows=1.00 loops=1) Sort Key: (time_bucket('@ 1 day'::interval, dimension_last."time")), dimension_last.device_id Sort Method: top-N heapsort -> Result (actual rows=5760.00 loops=1) -> Append (actual rows=5760.00 loops=1) -> Seq Scan on _hyper_2_4_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_5_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_6_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_7_chunk (actual rows=1440.00 loops=1) -- test query with ORDER BY date_trunc, device_id -- must not use ordered append :PREFIX SELECT date_trunc('day',time), device_id, name FROM dimension_last ORDER BY 1,2 LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Sort (actual rows=1.00 loops=1) Sort Key: (date_trunc('day'::text, dimension_last."time")), dimension_last.device_id Sort Method: top-N heapsort -> Result (actual rows=5760.00 loops=1) -> Append (actual rows=5760.00 loops=1) -> Seq Scan on _hyper_2_4_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_5_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_6_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_7_chunk (actual rows=1440.00 loops=1) -- test with table with only dimension column :PREFIX SELECT * FROM dimension_only ORDER BY time DESC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on dimension_only (actual rows=1.00 loops=1) Order: dimension_only."time" DESC -> Index Only Scan using _hyper_3_11_chunk_dimension_only_time_idx on _hyper_3_11_chunk (actual rows=1.00 loops=1) -> Index Only Scan using _hyper_3_10_chunk_dimension_only_time_idx on _hyper_3_10_chunk (never executed) -> Index Only Scan using _hyper_3_9_chunk_dimension_only_time_idx on _hyper_3_9_chunk (never executed) -> Index Only Scan using _hyper_3_8_chunk_dimension_only_time_idx on _hyper_3_8_chunk (never executed) -- test LEFT JOIN against hypertable :PREFIX_NO_ANALYZE SELECT * FROM dimension_last LEFT JOIN dimension_only USING (time) ORDER BY dimension_last.time DESC LIMIT 2; --- QUERY PLAN --- Limit -> Nested Loop Left Join Join Filter: (dimension_last."time" = dimension_only."time") -> Custom Scan (ChunkAppend) on dimension_last Order: dimension_last."time" DESC -> Index Scan using _hyper_2_7_chunk_dimension_last_time_idx on _hyper_2_7_chunk -> Index Scan using _hyper_2_6_chunk_dimension_last_time_idx on _hyper_2_6_chunk -> Index Scan using _hyper_2_5_chunk_dimension_last_time_idx on _hyper_2_5_chunk -> Index Scan using _hyper_2_4_chunk_dimension_last_time_idx on _hyper_2_4_chunk -> Materialize -> Append -> Seq Scan on _hyper_3_11_chunk -> Seq Scan on _hyper_3_10_chunk -> Seq Scan on _hyper_3_9_chunk -> Seq Scan on _hyper_3_8_chunk -- test INNER JOIN against non-hypertable :PREFIX_NO_ANALYZE SELECT * FROM dimension_last INNER JOIN dimension_only USING (time) ORDER BY dimension_last.time DESC LIMIT 2; --- QUERY PLAN --- Limit -> Nested Loop -> Custom Scan (ChunkAppend) on dimension_only Order: dimension_only."time" DESC -> Index Only Scan using _hyper_3_11_chunk_dimension_only_time_idx on _hyper_3_11_chunk -> Index Only Scan using _hyper_3_10_chunk_dimension_only_time_idx on _hyper_3_10_chunk -> Index Only Scan using _hyper_3_9_chunk_dimension_only_time_idx on _hyper_3_9_chunk -> Index Only Scan using _hyper_3_8_chunk_dimension_only_time_idx on _hyper_3_8_chunk -> Append -> Index Scan using _hyper_2_7_chunk_dimension_last_time_idx on _hyper_2_7_chunk Index Cond: ("time" = dimension_only."time") -> Index Scan using _hyper_2_6_chunk_dimension_last_time_idx on _hyper_2_6_chunk Index Cond: ("time" = dimension_only."time") -> Index Scan using _hyper_2_5_chunk_dimension_last_time_idx on _hyper_2_5_chunk Index Cond: ("time" = dimension_only."time") -> Index Scan using _hyper_2_4_chunk_dimension_last_time_idx on _hyper_2_4_chunk Index Cond: ("time" = dimension_only."time") -- test join against non-hypertable :PREFIX SELECT * FROM dimension_last INNER JOIN devices USING(device_id) ORDER BY dimension_last.time DESC LIMIT 2; --- QUERY PLAN --- Limit (actual rows=2.00 loops=1) -> Nested Loop (actual rows=2.00 loops=1) Join Filter: (devices.device_id = dimension_last.device_id) -> Custom Scan (ChunkAppend) on dimension_last (actual rows=2.00 loops=1) Order: dimension_last."time" DESC -> Index Scan using _hyper_2_7_chunk_dimension_last_time_idx on _hyper_2_7_chunk (actual rows=2.00 loops=1) -> Index Scan using _hyper_2_6_chunk_dimension_last_time_idx on _hyper_2_6_chunk (never executed) -> Index Scan using _hyper_2_5_chunk_dimension_last_time_idx on _hyper_2_5_chunk (never executed) -> Index Scan using _hyper_2_4_chunk_dimension_last_time_idx on _hyper_2_4_chunk (never executed) -> Materialize (actual rows=1.00 loops=2) -> Seq Scan on devices (actual rows=1.00 loops=1) -- test hypertable with index missing on one chunk :PREFIX SELECT time, device_id, value FROM ht_missing_indexes ORDER BY time ASC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on ht_missing_indexes (actual rows=1.00 loops=1) Order: ht_missing_indexes."time" -> Index Scan Backward using _hyper_4_12_chunk_ht_missing_indexes_time_idx on _hyper_4_12_chunk (actual rows=1.00 loops=1) -> Sort (never executed) Sort Key: _hyper_4_13_chunk."time" -> Seq Scan on _hyper_4_13_chunk (never executed) -> Index Scan Backward using _hyper_4_14_chunk_ht_missing_indexes_time_idx on _hyper_4_14_chunk (never executed) -- test hypertable with index missing on one chunk -- and no data :PREFIX SELECT time, device_id, value FROM ht_missing_indexes WHERE device_id = 2 ORDER BY time DESC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on ht_missing_indexes (actual rows=1.00 loops=1) Order: ht_missing_indexes."time" DESC -> Index Scan using _hyper_4_14_chunk_ht_missing_indexes_time_idx on _hyper_4_14_chunk (actual rows=1.00 loops=1) Filter: (device_id = 2) Rows Removed by Filter: 1 -> Sort (never executed) Sort Key: _hyper_4_13_chunk."time" DESC -> Seq Scan on _hyper_4_13_chunk (never executed) Filter: (device_id = 2) -> Index Scan using _hyper_4_12_chunk_ht_missing_indexes_time_idx on _hyper_4_12_chunk (never executed) Filter: (device_id = 2) -- test hypertable with index missing on one chunk -- and no data :PREFIX SELECT time, device_id, value FROM ht_missing_indexes WHERE time > '2000-01-07' ORDER BY time LIMIT 10; --- QUERY PLAN --- Limit (actual rows=10.00 loops=1) -> Custom Scan (ChunkAppend) on ht_missing_indexes (actual rows=10.00 loops=1) Order: ht_missing_indexes."time" -> Sort (actual rows=10.00 loops=1) Sort Key: _hyper_4_13_chunk."time" Sort Method: top-N heapsort -> Seq Scan on _hyper_4_13_chunk (actual rows=24477.00 loops=1) Filter: ("time" > 'Fri Jan 07 00:00:00 2000 PST'::timestamp with time zone) Rows Removed by Filter: 5763 -> Index Scan Backward using _hyper_4_14_chunk_ht_missing_indexes_time_idx on _hyper_4_14_chunk (never executed) -- test hypertable with dropped columns :PREFIX SELECT time, device_id, value FROM ht_dropped_columns ORDER BY time ASC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on ht_dropped_columns (actual rows=1.00 loops=1) Order: ht_dropped_columns."time" -> Index Scan Backward using _hyper_5_15_chunk_ht_dropped_columns_time_idx on _hyper_5_15_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_5_16_chunk_ht_dropped_columns_time_idx on _hyper_5_16_chunk (never executed) -> Index Scan Backward using _hyper_5_17_chunk_ht_dropped_columns_time_idx on _hyper_5_17_chunk (never executed) -> Index Scan Backward using _hyper_5_18_chunk_ht_dropped_columns_time_idx on _hyper_5_18_chunk (never executed) -> Index Scan Backward using _hyper_5_19_chunk_ht_dropped_columns_time_idx on _hyper_5_19_chunk (never executed) -- test hypertable with dropped columns :PREFIX SELECT time, device_id, value FROM ht_dropped_columns WHERE device_id = 1 ORDER BY time DESC; --- QUERY PLAN --- Custom Scan (ChunkAppend) on ht_dropped_columns (actual rows=7205.00 loops=1) Order: ht_dropped_columns."time" DESC -> Index Scan using _hyper_5_19_chunk_ht_dropped_columns_time_idx on _hyper_5_19_chunk (actual rows=1441.00 loops=1) Filter: (device_id = 1) -> Index Scan using _hyper_5_18_chunk_ht_dropped_columns_time_idx on _hyper_5_18_chunk (actual rows=1441.00 loops=1) Filter: (device_id = 1) -> Index Scan using _hyper_5_17_chunk_ht_dropped_columns_time_idx on _hyper_5_17_chunk (actual rows=1441.00 loops=1) Filter: (device_id = 1) -> Index Scan using _hyper_5_16_chunk_ht_dropped_columns_time_idx on _hyper_5_16_chunk (actual rows=1441.00 loops=1) Filter: (device_id = 1) -> Index Scan using _hyper_5_15_chunk_ht_dropped_columns_time_idx on _hyper_5_15_chunk (actual rows=1441.00 loops=1) Filter: (device_id = 1) -- test hypertable with 2 space dimensions :PREFIX SELECT time, device_id, value FROM space2 ORDER BY time DESC; --- QUERY PLAN --- Custom Scan (ChunkAppend) on space2 (actual rows=116649.00 loops=1) Order: space2."time" DESC -> Merge Append (actual rows=56169.00 loops=1) Sort Key: space2."time" DESC -> Index Scan using _hyper_6_36_chunk_space2_time_idx on _hyper_6_36_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_34_chunk_space2_time_idx on _hyper_6_34_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_32_chunk_space2_time_idx on _hyper_6_32_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_30_chunk_space2_time_idx on _hyper_6_30_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_28_chunk_space2_time_idx on _hyper_6_28_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_26_chunk_space2_time_idx on _hyper_6_26_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_24_chunk_space2_time_idx on _hyper_6_24_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_22_chunk_space2_time_idx on _hyper_6_22_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_20_chunk_space2_time_idx on _hyper_6_20_chunk (actual rows=6241.00 loops=1) -> Merge Append (actual rows=60480.00 loops=1) Sort Key: space2."time" DESC -> Index Scan using _hyper_6_37_chunk_space2_time_idx on _hyper_6_37_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_35_chunk_space2_time_idx on _hyper_6_35_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_33_chunk_space2_time_idx on _hyper_6_33_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_31_chunk_space2_time_idx on _hyper_6_31_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_29_chunk_space2_time_idx on _hyper_6_29_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_27_chunk_space2_time_idx on _hyper_6_27_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_25_chunk_space2_time_idx on _hyper_6_25_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_23_chunk_space2_time_idx on _hyper_6_23_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_21_chunk_space2_time_idx on _hyper_6_21_chunk (actual rows=6720.00 loops=1) -- test hypertable with 3 space dimensions :PREFIX SELECT time FROM space3 ORDER BY time DESC; --- QUERY PLAN --- Custom Scan (ChunkAppend) on space3 (actual rows=103688.00 loops=1) Order: space3."time" DESC -> Merge Append (actual rows=49928.00 loops=1) Sort Key: space3."time" DESC -> Index Only Scan using _hyper_7_52_chunk_space3_time_idx on _hyper_7_52_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_50_chunk_space3_time_idx on _hyper_7_50_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_48_chunk_space3_time_idx on _hyper_7_48_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_46_chunk_space3_time_idx on _hyper_7_46_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_44_chunk_space3_time_idx on _hyper_7_44_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_42_chunk_space3_time_idx on _hyper_7_42_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_40_chunk_space3_time_idx on _hyper_7_40_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_38_chunk_space3_time_idx on _hyper_7_38_chunk (actual rows=6241.00 loops=1) -> Merge Append (actual rows=53760.00 loops=1) Sort Key: space3."time" DESC -> Index Only Scan using _hyper_7_53_chunk_space3_time_idx on _hyper_7_53_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_51_chunk_space3_time_idx on _hyper_7_51_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_49_chunk_space3_time_idx on _hyper_7_49_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_47_chunk_space3_time_idx on _hyper_7_47_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_45_chunk_space3_time_idx on _hyper_7_45_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_43_chunk_space3_time_idx on _hyper_7_43_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_41_chunk_space3_time_idx on _hyper_7_41_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_39_chunk_space3_time_idx on _hyper_7_39_chunk (actual rows=6720.00 loops=1) -- test COLLATION -- cant be tested in our ci because alpine doesnt support locales -- :PREFIX SELECT * FROM sortopt_test ORDER BY time, device COLLATE "en_US.utf8"; -- test NULLS FIRST :PREFIX SELECT * FROM sortopt_test ORDER BY time, device NULLS FIRST; --- QUERY PLAN --- Custom Scan (ChunkAppend) on sortopt_test (actual rows=12961.00 loops=1) Order: sortopt_test."time", sortopt_test.device NULLS FIRST -> Index Only Scan using _hyper_8_55_chunk_time_device_nullsfirst on _hyper_8_55_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_8_54_chunk_time_device_nullsfirst on _hyper_8_54_chunk (actual rows=6241.00 loops=1) -- test NULLS LAST :PREFIX SELECT * FROM sortopt_test ORDER BY time, device DESC NULLS LAST; --- QUERY PLAN --- Custom Scan (ChunkAppend) on sortopt_test (actual rows=12961.00 loops=1) Order: sortopt_test."time", sortopt_test.device DESC NULLS LAST -> Index Only Scan using _hyper_8_55_chunk_time_device_nullslast on _hyper_8_55_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_8_54_chunk_time_device_nullslast on _hyper_8_54_chunk (actual rows=6241.00 loops=1) --generate the results into two different files \set ECHO errors ================================================ FILE: test/expected/plan_ordered_append-17.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- we run these with analyze to confirm that nodes that are not -- needed to fulfill the limit are not executed -- unfortunately this doesn't work on PostgreSQL 9.6 which lacks -- the ability to turn off analyze timing summary so we run -- them without ANALYZE on PostgreSQL 9.6, but since LATERAL plans -- are different across versions we need version specific output -- here anyway. \set TEST_BASE_NAME plan_ordered_append SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') as "TEST_LOAD_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') as "TEST_QUERY_NAME", format('%s/results/%s_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_OPTIMIZED", format('%s/results/%s_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_UNOPTIMIZED" \gset SELECT format('\! diff -u --label "Unoptimized result" --label "Optimized result" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') as "DIFF_CMD" \gset \set PREFIX 'EXPLAIN (analyze, buffers off, costs off, timing off, summary off)' \set PREFIX_NO_ANALYZE 'EXPLAIN (buffers off, costs off)' \ir :TEST_LOAD_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- create a now() function for repeatable testing that always returns -- the same timestamp. It needs to be marked STABLE CREATE OR REPLACE FUNCTION now_s() RETURNS timestamptz LANGUAGE PLPGSQL STABLE AS $BODY$ BEGIN RETURN '2000-01-08T0:00:00+0'::timestamptz; END; $BODY$; CREATE TABLE devices(device_id INT PRIMARY KEY, name TEXT); INSERT INTO devices VALUES (1,'Device 1'), (2,'Device 2'), (3,'Device 3'); -- create a second table where we create chunks in reverse order CREATE TABLE ordered_append_reverse(time timestamptz NOT NULL, device_id INT, value float); SELECT create_hypertable('ordered_append_reverse','time'); create_hypertable ------------------------------------- (1,public,ordered_append_reverse,t) INSERT INTO ordered_append_reverse SELECT generate_series('2000-01-18'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 0.5; -- table where dimension column is last column CREATE TABLE IF NOT EXISTS dimension_last( id INT8 NOT NULL, device_id INT NOT NULL, name TEXT NOT NULL, time timestamptz NOT NULL ); SELECT create_hypertable('dimension_last', 'time', chunk_time_interval => interval '1day', if_not_exists => True); create_hypertable ----------------------------- (2,public,dimension_last,t) -- table with only dimension column CREATE TABLE IF NOT EXISTS dimension_only( time timestamptz NOT NULL ); SELECT create_hypertable('dimension_only', 'time', chunk_time_interval => interval '1day', if_not_exists => True); create_hypertable ----------------------------- (3,public,dimension_only,t) INSERT INTO dimension_last SELECT 1,1,'Device 1',generate_series('2000-01-01 0:00:00+0'::timestamptz,'2000-01-04 23:59:00+0'::timestamptz,'1m'::interval); INSERT INTO dimension_only VALUES ('2000-01-01'), ('2000-01-03'), ('2000-01-05'), ('2000-01-07'); ANALYZE devices; ANALYZE ordered_append_reverse; ANALYZE dimension_last; ANALYZE dimension_only; -- create hypertable with indexes not on all chunks CREATE TABLE ht_missing_indexes(time timestamptz NOT NULL, device_id int, value float); SELECT create_hypertable('ht_missing_indexes','time'); create_hypertable --------------------------------- (4,public,ht_missing_indexes,t) INSERT INTO ht_missing_indexes SELECT generate_series('2000-01-01'::timestamptz,'2000-01-18'::timestamptz,'1m'::interval), 1, 0.5; INSERT INTO ht_missing_indexes SELECT generate_series('2000-01-01'::timestamptz,'2000-01-18'::timestamptz,'1m'::interval), 2, 1.5; INSERT INTO ht_missing_indexes SELECT generate_series('2000-01-01'::timestamptz,'2000-01-18'::timestamptz,'1m'::interval), 3, 2.5; -- drop index from 2nd chunk of ht_missing_indexes SELECT format('%I.%I',i.schemaname,i.indexname) AS "INDEX_NAME" FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable ht ON c.hypertable_id = ht.id INNER JOIN pg_indexes i ON i.schemaname = c.schema_name AND i.tablename=c.table_name WHERE ht.table_name = 'ht_missing_indexes' ORDER BY c.id LIMIT 1 OFFSET 1 \gset DROP INDEX :INDEX_NAME; ANALYZE ht_missing_indexes; -- create hypertable with with dropped columns CREATE TABLE ht_dropped_columns(c1 int, c2 int, c3 int, c4 int, c5 int, time timestamptz NOT NULL, device_id int, value float); SELECT create_hypertable('ht_dropped_columns','time'); create_hypertable --------------------------------- (5,public,ht_dropped_columns,t) ALTER TABLE ht_dropped_columns DROP COLUMN c1; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-01'::timestamptz,'2000-01-02'::timestamptz,'1m'::interval), 1, 0.5; ALTER TABLE ht_dropped_columns DROP COLUMN c2; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-08'::timestamptz,'2000-01-09'::timestamptz,'1m'::interval), 1, 0.5; ALTER TABLE ht_dropped_columns DROP COLUMN c3; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-15'::timestamptz,'2000-01-16'::timestamptz,'1m'::interval), 1, 0.5; ALTER TABLE ht_dropped_columns DROP COLUMN c4; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-22'::timestamptz,'2000-01-23'::timestamptz,'1m'::interval), 1, 0.5; ALTER TABLE ht_dropped_columns DROP COLUMN c5; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-29'::timestamptz,'2000-01-30'::timestamptz,'1m'::interval), 1, 0.5; ANALYZE ht_dropped_columns; CREATE TABLE space2(time timestamptz NOT NULL, device_id int NOT NULL, tag_id int NOT NULL, value float); SELECT create_hypertable('space2','time','device_id',number_partitions:=3); create_hypertable --------------------- (6,public,space2,t) SELECT add_dimension('space2','tag_id',number_partitions:=3); add_dimension ---------------------------- (8,public,space2,tag_id,t) INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 1, 1.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 1, 2.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 3, 1, 3.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 2, 1.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 2, 2.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 3, 2, 3.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 3, 1.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 3, 2.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 3, 3, 3.5; ANALYZE space2; CREATE TABLE space3(time timestamptz NOT NULL, x int NOT NULL, y int NOT NULL, z int NOT NULL, value float); SELECT create_hypertable('space3','time','x',number_partitions:=2); create_hypertable --------------------- (7,public,space3,t) SELECT add_dimension('space3','y',number_partitions:=2); add_dimension ------------------------ (11,public,space3,y,t) SELECT add_dimension('space3','z',number_partitions:=2); add_dimension ------------------------ (12,public,space3,z,t) INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 1, 1, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 1, 2, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 2, 1, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 2, 2, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 1, 1, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 1, 2, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 2, 1, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 2, 2, 1.5; ANALYZE space3; CREATE TABLE sortopt_test(time timestamptz NOT NULL, device TEXT); SELECT create_hypertable('sortopt_test','time',create_default_indexes:=false); create_hypertable --------------------------- (8,public,sortopt_test,t) -- since alpine does not support locales we cant test collations in our ci -- CREATE COLLATION IF NOT EXISTS en_US(LOCALE='en_US.utf8'); -- CREATE INDEX time_device_utf8 ON sortopt_test(time, device COLLATE "en_US"); CREATE INDEX time_device_nullsfirst ON sortopt_test(time, device NULLS FIRST); CREATE INDEX time_device_nullslast ON sortopt_test(time, device DESC NULLS LAST); INSERT INTO sortopt_test SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 'Device 1'; ANALYZE sortopt_test; \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- print chunks ordered by time to ensure ordering we want SELECT ht.table_name AS hypertable, c.table_name AS chunk, ds.range_start FROM _timescaledb_catalog.chunk c INNER JOIN LATERAL(SELECT * FROM _timescaledb_catalog.chunk_constraint cc WHERE c.id = cc.chunk_id ORDER BY cc.dimension_slice_id LIMIT 1) cc ON true INNER JOIN _timescaledb_catalog.dimension_slice ds ON ds.id=cc.dimension_slice_id INNER JOIN _timescaledb_catalog.dimension d ON ds.dimension_id = d.id INNER JOIN _timescaledb_catalog.hypertable ht ON d.hypertable_id = ht.id ORDER BY ht.table_name, range_start, chunk; hypertable | chunk | range_start ------------------------+-------------------+---------------------- dimension_last | _hyper_2_4_chunk | 946684800000000 dimension_last | _hyper_2_5_chunk | 946771200000000 dimension_last | _hyper_2_6_chunk | 946857600000000 dimension_last | _hyper_2_7_chunk | 946944000000000 dimension_only | _hyper_3_8_chunk | 946684800000000 dimension_only | _hyper_3_9_chunk | 946857600000000 dimension_only | _hyper_3_10_chunk | 947030400000000 dimension_only | _hyper_3_11_chunk | 947203200000000 ht_dropped_columns | _hyper_5_15_chunk | 946512000000000 ht_dropped_columns | _hyper_5_16_chunk | 947116800000000 ht_dropped_columns | _hyper_5_17_chunk | 947721600000000 ht_dropped_columns | _hyper_5_18_chunk | 948326400000000 ht_dropped_columns | _hyper_5_19_chunk | 948931200000000 ht_missing_indexes | _hyper_4_12_chunk | 946512000000000 ht_missing_indexes | _hyper_4_13_chunk | 947116800000000 ht_missing_indexes | _hyper_4_14_chunk | 947721600000000 ordered_append_reverse | _hyper_1_3_chunk | 946512000000000 ordered_append_reverse | _hyper_1_2_chunk | 947116800000000 ordered_append_reverse | _hyper_1_1_chunk | 947721600000000 sortopt_test | _hyper_8_55_chunk | 946512000000000 sortopt_test | _hyper_8_54_chunk | 947116800000000 space2 | _hyper_6_21_chunk | -9223372036854775808 space2 | _hyper_6_23_chunk | -9223372036854775808 space2 | _hyper_6_25_chunk | -9223372036854775808 space2 | _hyper_6_27_chunk | -9223372036854775808 space2 | _hyper_6_33_chunk | -9223372036854775808 space2 | _hyper_6_29_chunk | 946512000000000 space2 | _hyper_6_31_chunk | 946512000000000 space2 | _hyper_6_35_chunk | 946512000000000 space2 | _hyper_6_37_chunk | 946512000000000 space2 | _hyper_6_20_chunk | 947116800000000 space2 | _hyper_6_22_chunk | 947116800000000 space2 | _hyper_6_24_chunk | 947116800000000 space2 | _hyper_6_26_chunk | 947116800000000 space2 | _hyper_6_28_chunk | 947116800000000 space2 | _hyper_6_30_chunk | 947116800000000 space2 | _hyper_6_32_chunk | 947116800000000 space2 | _hyper_6_34_chunk | 947116800000000 space2 | _hyper_6_36_chunk | 947116800000000 space3 | _hyper_7_39_chunk | -9223372036854775808 space3 | _hyper_7_41_chunk | -9223372036854775808 space3 | _hyper_7_43_chunk | -9223372036854775808 space3 | _hyper_7_45_chunk | -9223372036854775808 space3 | _hyper_7_47_chunk | -9223372036854775808 space3 | _hyper_7_49_chunk | -9223372036854775808 space3 | _hyper_7_51_chunk | -9223372036854775808 space3 | _hyper_7_53_chunk | 946512000000000 space3 | _hyper_7_38_chunk | 947116800000000 space3 | _hyper_7_40_chunk | 947116800000000 space3 | _hyper_7_42_chunk | 947116800000000 space3 | _hyper_7_44_chunk | 947116800000000 space3 | _hyper_7_46_chunk | 947116800000000 space3 | _hyper_7_48_chunk | 947116800000000 space3 | _hyper_7_50_chunk | 947116800000000 space3 | _hyper_7_52_chunk | 947116800000000 -- test ASC for reverse ordered chunks :PREFIX SELECT time, device_id, value FROM ordered_append_reverse ORDER BY time ASC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on ordered_append_reverse (actual rows=1.00 loops=1) Order: ordered_append_reverse."time" -> Index Scan Backward using _hyper_1_3_chunk_ordered_append_reverse_time_idx on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_2_chunk_ordered_append_reverse_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan Backward using _hyper_1_1_chunk_ordered_append_reverse_time_idx on _hyper_1_1_chunk (never executed) -- test DESC for reverse ordered chunks :PREFIX SELECT time, device_id, value FROM ordered_append_reverse ORDER BY time DESC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on ordered_append_reverse (actual rows=1.00 loops=1) Order: ordered_append_reverse."time" DESC -> Index Scan using _hyper_1_1_chunk_ordered_append_reverse_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_2_chunk_ordered_append_reverse_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan using _hyper_1_3_chunk_ordered_append_reverse_time_idx on _hyper_1_3_chunk (never executed) -- test query with ORDER BY time_bucket, device_id -- must not use ordered append :PREFIX SELECT time_bucket('1d',time), device_id, name FROM dimension_last ORDER BY time_bucket('1d',time), device_id LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Sort (actual rows=1.00 loops=1) Sort Key: (time_bucket('@ 1 day'::interval, dimension_last."time")), dimension_last.device_id Sort Method: top-N heapsort -> Result (actual rows=5760.00 loops=1) -> Append (actual rows=5760.00 loops=1) -> Seq Scan on _hyper_2_4_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_5_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_6_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_7_chunk (actual rows=1440.00 loops=1) -- test query with ORDER BY date_trunc, device_id -- must not use ordered append :PREFIX SELECT date_trunc('day',time), device_id, name FROM dimension_last ORDER BY 1,2 LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Sort (actual rows=1.00 loops=1) Sort Key: (date_trunc('day'::text, dimension_last."time")), dimension_last.device_id Sort Method: top-N heapsort -> Result (actual rows=5760.00 loops=1) -> Append (actual rows=5760.00 loops=1) -> Seq Scan on _hyper_2_4_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_5_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_6_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_7_chunk (actual rows=1440.00 loops=1) -- test with table with only dimension column :PREFIX SELECT * FROM dimension_only ORDER BY time DESC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on dimension_only (actual rows=1.00 loops=1) Order: dimension_only."time" DESC -> Index Only Scan using _hyper_3_11_chunk_dimension_only_time_idx on _hyper_3_11_chunk (actual rows=1.00 loops=1) -> Index Only Scan using _hyper_3_10_chunk_dimension_only_time_idx on _hyper_3_10_chunk (never executed) -> Index Only Scan using _hyper_3_9_chunk_dimension_only_time_idx on _hyper_3_9_chunk (never executed) -> Index Only Scan using _hyper_3_8_chunk_dimension_only_time_idx on _hyper_3_8_chunk (never executed) -- test LEFT JOIN against hypertable :PREFIX_NO_ANALYZE SELECT * FROM dimension_last LEFT JOIN dimension_only USING (time) ORDER BY dimension_last.time DESC LIMIT 2; --- QUERY PLAN --- Limit -> Nested Loop Left Join Join Filter: (dimension_last."time" = dimension_only."time") -> Custom Scan (ChunkAppend) on dimension_last Order: dimension_last."time" DESC -> Index Scan using _hyper_2_7_chunk_dimension_last_time_idx on _hyper_2_7_chunk -> Index Scan using _hyper_2_6_chunk_dimension_last_time_idx on _hyper_2_6_chunk -> Index Scan using _hyper_2_5_chunk_dimension_last_time_idx on _hyper_2_5_chunk -> Index Scan using _hyper_2_4_chunk_dimension_last_time_idx on _hyper_2_4_chunk -> Materialize -> Append -> Seq Scan on _hyper_3_11_chunk -> Seq Scan on _hyper_3_10_chunk -> Seq Scan on _hyper_3_9_chunk -> Seq Scan on _hyper_3_8_chunk -- test INNER JOIN against non-hypertable :PREFIX_NO_ANALYZE SELECT * FROM dimension_last INNER JOIN dimension_only USING (time) ORDER BY dimension_last.time DESC LIMIT 2; --- QUERY PLAN --- Limit -> Nested Loop -> Custom Scan (ChunkAppend) on dimension_only Order: dimension_only."time" DESC -> Index Only Scan using _hyper_3_11_chunk_dimension_only_time_idx on _hyper_3_11_chunk -> Index Only Scan using _hyper_3_10_chunk_dimension_only_time_idx on _hyper_3_10_chunk -> Index Only Scan using _hyper_3_9_chunk_dimension_only_time_idx on _hyper_3_9_chunk -> Index Only Scan using _hyper_3_8_chunk_dimension_only_time_idx on _hyper_3_8_chunk -> Append -> Index Scan using _hyper_2_7_chunk_dimension_last_time_idx on _hyper_2_7_chunk Index Cond: ("time" = dimension_only."time") -> Index Scan using _hyper_2_6_chunk_dimension_last_time_idx on _hyper_2_6_chunk Index Cond: ("time" = dimension_only."time") -> Index Scan using _hyper_2_5_chunk_dimension_last_time_idx on _hyper_2_5_chunk Index Cond: ("time" = dimension_only."time") -> Index Scan using _hyper_2_4_chunk_dimension_last_time_idx on _hyper_2_4_chunk Index Cond: ("time" = dimension_only."time") -- test join against non-hypertable :PREFIX SELECT * FROM dimension_last INNER JOIN devices USING(device_id) ORDER BY dimension_last.time DESC LIMIT 2; --- QUERY PLAN --- Limit (actual rows=2.00 loops=1) -> Nested Loop (actual rows=2.00 loops=1) Join Filter: (devices.device_id = dimension_last.device_id) -> Custom Scan (ChunkAppend) on dimension_last (actual rows=2.00 loops=1) Order: dimension_last."time" DESC -> Index Scan using _hyper_2_7_chunk_dimension_last_time_idx on _hyper_2_7_chunk (actual rows=2.00 loops=1) -> Index Scan using _hyper_2_6_chunk_dimension_last_time_idx on _hyper_2_6_chunk (never executed) -> Index Scan using _hyper_2_5_chunk_dimension_last_time_idx on _hyper_2_5_chunk (never executed) -> Index Scan using _hyper_2_4_chunk_dimension_last_time_idx on _hyper_2_4_chunk (never executed) -> Materialize (actual rows=1.00 loops=2) -> Seq Scan on devices (actual rows=1.00 loops=1) -- test hypertable with index missing on one chunk :PREFIX SELECT time, device_id, value FROM ht_missing_indexes ORDER BY time ASC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on ht_missing_indexes (actual rows=1.00 loops=1) Order: ht_missing_indexes."time" -> Index Scan Backward using _hyper_4_12_chunk_ht_missing_indexes_time_idx on _hyper_4_12_chunk (actual rows=1.00 loops=1) -> Sort (never executed) Sort Key: _hyper_4_13_chunk."time" -> Seq Scan on _hyper_4_13_chunk (never executed) -> Index Scan Backward using _hyper_4_14_chunk_ht_missing_indexes_time_idx on _hyper_4_14_chunk (never executed) -- test hypertable with index missing on one chunk -- and no data :PREFIX SELECT time, device_id, value FROM ht_missing_indexes WHERE device_id = 2 ORDER BY time DESC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on ht_missing_indexes (actual rows=1.00 loops=1) Order: ht_missing_indexes."time" DESC -> Index Scan using _hyper_4_14_chunk_ht_missing_indexes_time_idx on _hyper_4_14_chunk (actual rows=1.00 loops=1) Filter: (device_id = 2) Rows Removed by Filter: 1 -> Sort (never executed) Sort Key: _hyper_4_13_chunk."time" DESC -> Seq Scan on _hyper_4_13_chunk (never executed) Filter: (device_id = 2) -> Index Scan using _hyper_4_12_chunk_ht_missing_indexes_time_idx on _hyper_4_12_chunk (never executed) Filter: (device_id = 2) -- test hypertable with index missing on one chunk -- and no data :PREFIX SELECT time, device_id, value FROM ht_missing_indexes WHERE time > '2000-01-07' ORDER BY time LIMIT 10; --- QUERY PLAN --- Limit (actual rows=10.00 loops=1) -> Custom Scan (ChunkAppend) on ht_missing_indexes (actual rows=10.00 loops=1) Order: ht_missing_indexes."time" -> Sort (actual rows=10.00 loops=1) Sort Key: _hyper_4_13_chunk."time" Sort Method: top-N heapsort -> Seq Scan on _hyper_4_13_chunk (actual rows=24477.00 loops=1) Filter: ("time" > 'Fri Jan 07 00:00:00 2000 PST'::timestamp with time zone) Rows Removed by Filter: 5763 -> Index Scan Backward using _hyper_4_14_chunk_ht_missing_indexes_time_idx on _hyper_4_14_chunk (never executed) -- test hypertable with dropped columns :PREFIX SELECT time, device_id, value FROM ht_dropped_columns ORDER BY time ASC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on ht_dropped_columns (actual rows=1.00 loops=1) Order: ht_dropped_columns."time" -> Index Scan Backward using _hyper_5_15_chunk_ht_dropped_columns_time_idx on _hyper_5_15_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_5_16_chunk_ht_dropped_columns_time_idx on _hyper_5_16_chunk (never executed) -> Index Scan Backward using _hyper_5_17_chunk_ht_dropped_columns_time_idx on _hyper_5_17_chunk (never executed) -> Index Scan Backward using _hyper_5_18_chunk_ht_dropped_columns_time_idx on _hyper_5_18_chunk (never executed) -> Index Scan Backward using _hyper_5_19_chunk_ht_dropped_columns_time_idx on _hyper_5_19_chunk (never executed) -- test hypertable with dropped columns :PREFIX SELECT time, device_id, value FROM ht_dropped_columns WHERE device_id = 1 ORDER BY time DESC; --- QUERY PLAN --- Custom Scan (ChunkAppend) on ht_dropped_columns (actual rows=7205.00 loops=1) Order: ht_dropped_columns."time" DESC -> Index Scan using _hyper_5_19_chunk_ht_dropped_columns_time_idx on _hyper_5_19_chunk (actual rows=1441.00 loops=1) Filter: (device_id = 1) -> Index Scan using _hyper_5_18_chunk_ht_dropped_columns_time_idx on _hyper_5_18_chunk (actual rows=1441.00 loops=1) Filter: (device_id = 1) -> Index Scan using _hyper_5_17_chunk_ht_dropped_columns_time_idx on _hyper_5_17_chunk (actual rows=1441.00 loops=1) Filter: (device_id = 1) -> Index Scan using _hyper_5_16_chunk_ht_dropped_columns_time_idx on _hyper_5_16_chunk (actual rows=1441.00 loops=1) Filter: (device_id = 1) -> Index Scan using _hyper_5_15_chunk_ht_dropped_columns_time_idx on _hyper_5_15_chunk (actual rows=1441.00 loops=1) Filter: (device_id = 1) -- test hypertable with 2 space dimensions :PREFIX SELECT time, device_id, value FROM space2 ORDER BY time DESC; --- QUERY PLAN --- Custom Scan (ChunkAppend) on space2 (actual rows=116649.00 loops=1) Order: space2."time" DESC -> Merge Append (actual rows=56169.00 loops=1) Sort Key: space2."time" DESC -> Index Scan using _hyper_6_36_chunk_space2_time_idx on _hyper_6_36_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_34_chunk_space2_time_idx on _hyper_6_34_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_32_chunk_space2_time_idx on _hyper_6_32_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_30_chunk_space2_time_idx on _hyper_6_30_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_28_chunk_space2_time_idx on _hyper_6_28_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_26_chunk_space2_time_idx on _hyper_6_26_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_24_chunk_space2_time_idx on _hyper_6_24_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_22_chunk_space2_time_idx on _hyper_6_22_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_20_chunk_space2_time_idx on _hyper_6_20_chunk (actual rows=6241.00 loops=1) -> Merge Append (actual rows=60480.00 loops=1) Sort Key: space2."time" DESC -> Index Scan using _hyper_6_37_chunk_space2_time_idx on _hyper_6_37_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_35_chunk_space2_time_idx on _hyper_6_35_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_33_chunk_space2_time_idx on _hyper_6_33_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_31_chunk_space2_time_idx on _hyper_6_31_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_29_chunk_space2_time_idx on _hyper_6_29_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_27_chunk_space2_time_idx on _hyper_6_27_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_25_chunk_space2_time_idx on _hyper_6_25_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_23_chunk_space2_time_idx on _hyper_6_23_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_21_chunk_space2_time_idx on _hyper_6_21_chunk (actual rows=6720.00 loops=1) -- test hypertable with 3 space dimensions :PREFIX SELECT time FROM space3 ORDER BY time DESC; --- QUERY PLAN --- Custom Scan (ChunkAppend) on space3 (actual rows=103688.00 loops=1) Order: space3."time" DESC -> Merge Append (actual rows=49928.00 loops=1) Sort Key: space3."time" DESC -> Index Only Scan using _hyper_7_52_chunk_space3_time_idx on _hyper_7_52_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_50_chunk_space3_time_idx on _hyper_7_50_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_48_chunk_space3_time_idx on _hyper_7_48_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_46_chunk_space3_time_idx on _hyper_7_46_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_44_chunk_space3_time_idx on _hyper_7_44_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_42_chunk_space3_time_idx on _hyper_7_42_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_40_chunk_space3_time_idx on _hyper_7_40_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_38_chunk_space3_time_idx on _hyper_7_38_chunk (actual rows=6241.00 loops=1) -> Merge Append (actual rows=53760.00 loops=1) Sort Key: space3."time" DESC -> Index Only Scan using _hyper_7_53_chunk_space3_time_idx on _hyper_7_53_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_51_chunk_space3_time_idx on _hyper_7_51_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_49_chunk_space3_time_idx on _hyper_7_49_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_47_chunk_space3_time_idx on _hyper_7_47_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_45_chunk_space3_time_idx on _hyper_7_45_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_43_chunk_space3_time_idx on _hyper_7_43_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_41_chunk_space3_time_idx on _hyper_7_41_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_39_chunk_space3_time_idx on _hyper_7_39_chunk (actual rows=6720.00 loops=1) -- test COLLATION -- cant be tested in our ci because alpine doesnt support locales -- :PREFIX SELECT * FROM sortopt_test ORDER BY time, device COLLATE "en_US.utf8"; -- test NULLS FIRST :PREFIX SELECT * FROM sortopt_test ORDER BY time, device NULLS FIRST; --- QUERY PLAN --- Custom Scan (ChunkAppend) on sortopt_test (actual rows=12961.00 loops=1) Order: sortopt_test."time", sortopt_test.device NULLS FIRST -> Index Only Scan using _hyper_8_55_chunk_time_device_nullsfirst on _hyper_8_55_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_8_54_chunk_time_device_nullsfirst on _hyper_8_54_chunk (actual rows=6241.00 loops=1) -- test NULLS LAST :PREFIX SELECT * FROM sortopt_test ORDER BY time, device DESC NULLS LAST; --- QUERY PLAN --- Custom Scan (ChunkAppend) on sortopt_test (actual rows=12961.00 loops=1) Order: sortopt_test."time", sortopt_test.device DESC NULLS LAST -> Index Only Scan using _hyper_8_55_chunk_time_device_nullslast on _hyper_8_55_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_8_54_chunk_time_device_nullslast on _hyper_8_54_chunk (actual rows=6241.00 loops=1) --generate the results into two different files \set ECHO errors ================================================ FILE: test/expected/plan_ordered_append-18.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- we run these with analyze to confirm that nodes that are not -- needed to fulfill the limit are not executed -- unfortunately this doesn't work on PostgreSQL 9.6 which lacks -- the ability to turn off analyze timing summary so we run -- them without ANALYZE on PostgreSQL 9.6, but since LATERAL plans -- are different across versions we need version specific output -- here anyway. \set TEST_BASE_NAME plan_ordered_append SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') as "TEST_LOAD_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') as "TEST_QUERY_NAME", format('%s/results/%s_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_OPTIMIZED", format('%s/results/%s_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_UNOPTIMIZED" \gset SELECT format('\! diff -u --label "Unoptimized result" --label "Optimized result" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') as "DIFF_CMD" \gset \set PREFIX 'EXPLAIN (analyze, buffers off, costs off, timing off, summary off)' \set PREFIX_NO_ANALYZE 'EXPLAIN (buffers off, costs off)' \ir :TEST_LOAD_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- create a now() function for repeatable testing that always returns -- the same timestamp. It needs to be marked STABLE CREATE OR REPLACE FUNCTION now_s() RETURNS timestamptz LANGUAGE PLPGSQL STABLE AS $BODY$ BEGIN RETURN '2000-01-08T0:00:00+0'::timestamptz; END; $BODY$; CREATE TABLE devices(device_id INT PRIMARY KEY, name TEXT); INSERT INTO devices VALUES (1,'Device 1'), (2,'Device 2'), (3,'Device 3'); -- create a second table where we create chunks in reverse order CREATE TABLE ordered_append_reverse(time timestamptz NOT NULL, device_id INT, value float); SELECT create_hypertable('ordered_append_reverse','time'); create_hypertable ------------------------------------- (1,public,ordered_append_reverse,t) INSERT INTO ordered_append_reverse SELECT generate_series('2000-01-18'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 0.5; -- table where dimension column is last column CREATE TABLE IF NOT EXISTS dimension_last( id INT8 NOT NULL, device_id INT NOT NULL, name TEXT NOT NULL, time timestamptz NOT NULL ); SELECT create_hypertable('dimension_last', 'time', chunk_time_interval => interval '1day', if_not_exists => True); create_hypertable ----------------------------- (2,public,dimension_last,t) -- table with only dimension column CREATE TABLE IF NOT EXISTS dimension_only( time timestamptz NOT NULL ); SELECT create_hypertable('dimension_only', 'time', chunk_time_interval => interval '1day', if_not_exists => True); create_hypertable ----------------------------- (3,public,dimension_only,t) INSERT INTO dimension_last SELECT 1,1,'Device 1',generate_series('2000-01-01 0:00:00+0'::timestamptz,'2000-01-04 23:59:00+0'::timestamptz,'1m'::interval); INSERT INTO dimension_only VALUES ('2000-01-01'), ('2000-01-03'), ('2000-01-05'), ('2000-01-07'); ANALYZE devices; ANALYZE ordered_append_reverse; ANALYZE dimension_last; ANALYZE dimension_only; -- create hypertable with indexes not on all chunks CREATE TABLE ht_missing_indexes(time timestamptz NOT NULL, device_id int, value float); SELECT create_hypertable('ht_missing_indexes','time'); create_hypertable --------------------------------- (4,public,ht_missing_indexes,t) INSERT INTO ht_missing_indexes SELECT generate_series('2000-01-01'::timestamptz,'2000-01-18'::timestamptz,'1m'::interval), 1, 0.5; INSERT INTO ht_missing_indexes SELECT generate_series('2000-01-01'::timestamptz,'2000-01-18'::timestamptz,'1m'::interval), 2, 1.5; INSERT INTO ht_missing_indexes SELECT generate_series('2000-01-01'::timestamptz,'2000-01-18'::timestamptz,'1m'::interval), 3, 2.5; -- drop index from 2nd chunk of ht_missing_indexes SELECT format('%I.%I',i.schemaname,i.indexname) AS "INDEX_NAME" FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable ht ON c.hypertable_id = ht.id INNER JOIN pg_indexes i ON i.schemaname = c.schema_name AND i.tablename=c.table_name WHERE ht.table_name = 'ht_missing_indexes' ORDER BY c.id LIMIT 1 OFFSET 1 \gset DROP INDEX :INDEX_NAME; ANALYZE ht_missing_indexes; -- create hypertable with with dropped columns CREATE TABLE ht_dropped_columns(c1 int, c2 int, c3 int, c4 int, c5 int, time timestamptz NOT NULL, device_id int, value float); SELECT create_hypertable('ht_dropped_columns','time'); create_hypertable --------------------------------- (5,public,ht_dropped_columns,t) ALTER TABLE ht_dropped_columns DROP COLUMN c1; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-01'::timestamptz,'2000-01-02'::timestamptz,'1m'::interval), 1, 0.5; ALTER TABLE ht_dropped_columns DROP COLUMN c2; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-08'::timestamptz,'2000-01-09'::timestamptz,'1m'::interval), 1, 0.5; ALTER TABLE ht_dropped_columns DROP COLUMN c3; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-15'::timestamptz,'2000-01-16'::timestamptz,'1m'::interval), 1, 0.5; ALTER TABLE ht_dropped_columns DROP COLUMN c4; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-22'::timestamptz,'2000-01-23'::timestamptz,'1m'::interval), 1, 0.5; ALTER TABLE ht_dropped_columns DROP COLUMN c5; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-29'::timestamptz,'2000-01-30'::timestamptz,'1m'::interval), 1, 0.5; ANALYZE ht_dropped_columns; CREATE TABLE space2(time timestamptz NOT NULL, device_id int NOT NULL, tag_id int NOT NULL, value float); SELECT create_hypertable('space2','time','device_id',number_partitions:=3); create_hypertable --------------------- (6,public,space2,t) SELECT add_dimension('space2','tag_id',number_partitions:=3); add_dimension ---------------------------- (8,public,space2,tag_id,t) INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 1, 1.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 1, 2.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 3, 1, 3.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 2, 1.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 2, 2.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 3, 2, 3.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 3, 1.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 3, 2.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 3, 3, 3.5; ANALYZE space2; CREATE TABLE space3(time timestamptz NOT NULL, x int NOT NULL, y int NOT NULL, z int NOT NULL, value float); SELECT create_hypertable('space3','time','x',number_partitions:=2); create_hypertable --------------------- (7,public,space3,t) SELECT add_dimension('space3','y',number_partitions:=2); add_dimension ------------------------ (11,public,space3,y,t) SELECT add_dimension('space3','z',number_partitions:=2); add_dimension ------------------------ (12,public,space3,z,t) INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 1, 1, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 1, 2, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 2, 1, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 2, 2, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 1, 1, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 1, 2, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 2, 1, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 2, 2, 1.5; ANALYZE space3; CREATE TABLE sortopt_test(time timestamptz NOT NULL, device TEXT); SELECT create_hypertable('sortopt_test','time',create_default_indexes:=false); create_hypertable --------------------------- (8,public,sortopt_test,t) -- since alpine does not support locales we cant test collations in our ci -- CREATE COLLATION IF NOT EXISTS en_US(LOCALE='en_US.utf8'); -- CREATE INDEX time_device_utf8 ON sortopt_test(time, device COLLATE "en_US"); CREATE INDEX time_device_nullsfirst ON sortopt_test(time, device NULLS FIRST); CREATE INDEX time_device_nullslast ON sortopt_test(time, device DESC NULLS LAST); INSERT INTO sortopt_test SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 'Device 1'; ANALYZE sortopt_test; \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- print chunks ordered by time to ensure ordering we want SELECT ht.table_name AS hypertable, c.table_name AS chunk, ds.range_start FROM _timescaledb_catalog.chunk c INNER JOIN LATERAL(SELECT * FROM _timescaledb_catalog.chunk_constraint cc WHERE c.id = cc.chunk_id ORDER BY cc.dimension_slice_id LIMIT 1) cc ON true INNER JOIN _timescaledb_catalog.dimension_slice ds ON ds.id=cc.dimension_slice_id INNER JOIN _timescaledb_catalog.dimension d ON ds.dimension_id = d.id INNER JOIN _timescaledb_catalog.hypertable ht ON d.hypertable_id = ht.id ORDER BY ht.table_name, range_start, chunk; hypertable | chunk | range_start ------------------------+-------------------+---------------------- dimension_last | _hyper_2_4_chunk | 946684800000000 dimension_last | _hyper_2_5_chunk | 946771200000000 dimension_last | _hyper_2_6_chunk | 946857600000000 dimension_last | _hyper_2_7_chunk | 946944000000000 dimension_only | _hyper_3_8_chunk | 946684800000000 dimension_only | _hyper_3_9_chunk | 946857600000000 dimension_only | _hyper_3_10_chunk | 947030400000000 dimension_only | _hyper_3_11_chunk | 947203200000000 ht_dropped_columns | _hyper_5_15_chunk | 946512000000000 ht_dropped_columns | _hyper_5_16_chunk | 947116800000000 ht_dropped_columns | _hyper_5_17_chunk | 947721600000000 ht_dropped_columns | _hyper_5_18_chunk | 948326400000000 ht_dropped_columns | _hyper_5_19_chunk | 948931200000000 ht_missing_indexes | _hyper_4_12_chunk | 946512000000000 ht_missing_indexes | _hyper_4_13_chunk | 947116800000000 ht_missing_indexes | _hyper_4_14_chunk | 947721600000000 ordered_append_reverse | _hyper_1_3_chunk | 946512000000000 ordered_append_reverse | _hyper_1_2_chunk | 947116800000000 ordered_append_reverse | _hyper_1_1_chunk | 947721600000000 sortopt_test | _hyper_8_55_chunk | 946512000000000 sortopt_test | _hyper_8_54_chunk | 947116800000000 space2 | _hyper_6_21_chunk | -9223372036854775808 space2 | _hyper_6_23_chunk | -9223372036854775808 space2 | _hyper_6_25_chunk | -9223372036854775808 space2 | _hyper_6_27_chunk | -9223372036854775808 space2 | _hyper_6_33_chunk | -9223372036854775808 space2 | _hyper_6_29_chunk | 946512000000000 space2 | _hyper_6_31_chunk | 946512000000000 space2 | _hyper_6_35_chunk | 946512000000000 space2 | _hyper_6_37_chunk | 946512000000000 space2 | _hyper_6_20_chunk | 947116800000000 space2 | _hyper_6_22_chunk | 947116800000000 space2 | _hyper_6_24_chunk | 947116800000000 space2 | _hyper_6_26_chunk | 947116800000000 space2 | _hyper_6_28_chunk | 947116800000000 space2 | _hyper_6_30_chunk | 947116800000000 space2 | _hyper_6_32_chunk | 947116800000000 space2 | _hyper_6_34_chunk | 947116800000000 space2 | _hyper_6_36_chunk | 947116800000000 space3 | _hyper_7_39_chunk | -9223372036854775808 space3 | _hyper_7_41_chunk | -9223372036854775808 space3 | _hyper_7_43_chunk | -9223372036854775808 space3 | _hyper_7_45_chunk | -9223372036854775808 space3 | _hyper_7_47_chunk | -9223372036854775808 space3 | _hyper_7_49_chunk | -9223372036854775808 space3 | _hyper_7_51_chunk | -9223372036854775808 space3 | _hyper_7_53_chunk | 946512000000000 space3 | _hyper_7_38_chunk | 947116800000000 space3 | _hyper_7_40_chunk | 947116800000000 space3 | _hyper_7_42_chunk | 947116800000000 space3 | _hyper_7_44_chunk | 947116800000000 space3 | _hyper_7_46_chunk | 947116800000000 space3 | _hyper_7_48_chunk | 947116800000000 space3 | _hyper_7_50_chunk | 947116800000000 space3 | _hyper_7_52_chunk | 947116800000000 -- test ASC for reverse ordered chunks :PREFIX SELECT time, device_id, value FROM ordered_append_reverse ORDER BY time ASC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on ordered_append_reverse (actual rows=1.00 loops=1) Order: ordered_append_reverse."time" -> Index Scan Backward using _hyper_1_3_chunk_ordered_append_reverse_time_idx on _hyper_1_3_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_1_2_chunk_ordered_append_reverse_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan Backward using _hyper_1_1_chunk_ordered_append_reverse_time_idx on _hyper_1_1_chunk (never executed) -- test DESC for reverse ordered chunks :PREFIX SELECT time, device_id, value FROM ordered_append_reverse ORDER BY time DESC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on ordered_append_reverse (actual rows=1.00 loops=1) Order: ordered_append_reverse."time" DESC -> Index Scan using _hyper_1_1_chunk_ordered_append_reverse_time_idx on _hyper_1_1_chunk (actual rows=1.00 loops=1) -> Index Scan using _hyper_1_2_chunk_ordered_append_reverse_time_idx on _hyper_1_2_chunk (never executed) -> Index Scan using _hyper_1_3_chunk_ordered_append_reverse_time_idx on _hyper_1_3_chunk (never executed) -- test query with ORDER BY time_bucket, device_id -- must not use ordered append :PREFIX SELECT time_bucket('1d',time), device_id, name FROM dimension_last ORDER BY time_bucket('1d',time), device_id LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Sort (actual rows=1.00 loops=1) Sort Key: (time_bucket('@ 1 day'::interval, dimension_last."time")), dimension_last.device_id Sort Method: top-N heapsort -> Result (actual rows=5760.00 loops=1) -> Append (actual rows=5760.00 loops=1) -> Seq Scan on _hyper_2_4_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_5_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_6_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_7_chunk (actual rows=1440.00 loops=1) -- test query with ORDER BY date_trunc, device_id -- must not use ordered append :PREFIX SELECT date_trunc('day',time), device_id, name FROM dimension_last ORDER BY 1,2 LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Sort (actual rows=1.00 loops=1) Sort Key: (date_trunc('day'::text, dimension_last."time")), dimension_last.device_id Sort Method: top-N heapsort -> Result (actual rows=5760.00 loops=1) -> Append (actual rows=5760.00 loops=1) -> Seq Scan on _hyper_2_4_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_5_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_6_chunk (actual rows=1440.00 loops=1) -> Seq Scan on _hyper_2_7_chunk (actual rows=1440.00 loops=1) -- test with table with only dimension column :PREFIX SELECT * FROM dimension_only ORDER BY time DESC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on dimension_only (actual rows=1.00 loops=1) Order: dimension_only."time" DESC -> Index Only Scan using _hyper_3_11_chunk_dimension_only_time_idx on _hyper_3_11_chunk (actual rows=1.00 loops=1) -> Index Only Scan using _hyper_3_10_chunk_dimension_only_time_idx on _hyper_3_10_chunk (never executed) -> Index Only Scan using _hyper_3_9_chunk_dimension_only_time_idx on _hyper_3_9_chunk (never executed) -> Index Only Scan using _hyper_3_8_chunk_dimension_only_time_idx on _hyper_3_8_chunk (never executed) -- test LEFT JOIN against hypertable :PREFIX_NO_ANALYZE SELECT * FROM dimension_last LEFT JOIN dimension_only USING (time) ORDER BY dimension_last.time DESC LIMIT 2; --- QUERY PLAN --- Limit -> Nested Loop Left Join Join Filter: (dimension_last."time" = dimension_only."time") -> Custom Scan (ChunkAppend) on dimension_last Order: dimension_last."time" DESC -> Index Scan using _hyper_2_7_chunk_dimension_last_time_idx on _hyper_2_7_chunk -> Index Scan using _hyper_2_6_chunk_dimension_last_time_idx on _hyper_2_6_chunk -> Index Scan using _hyper_2_5_chunk_dimension_last_time_idx on _hyper_2_5_chunk -> Index Scan using _hyper_2_4_chunk_dimension_last_time_idx on _hyper_2_4_chunk -> Materialize -> Append -> Seq Scan on _hyper_3_11_chunk -> Seq Scan on _hyper_3_10_chunk -> Seq Scan on _hyper_3_9_chunk -> Seq Scan on _hyper_3_8_chunk -- test INNER JOIN against non-hypertable :PREFIX_NO_ANALYZE SELECT * FROM dimension_last INNER JOIN dimension_only USING (time) ORDER BY dimension_last.time DESC LIMIT 2; --- QUERY PLAN --- Limit -> Nested Loop -> Custom Scan (ChunkAppend) on dimension_only Order: dimension_only."time" DESC -> Index Only Scan using _hyper_3_11_chunk_dimension_only_time_idx on _hyper_3_11_chunk -> Index Only Scan using _hyper_3_10_chunk_dimension_only_time_idx on _hyper_3_10_chunk -> Index Only Scan using _hyper_3_9_chunk_dimension_only_time_idx on _hyper_3_9_chunk -> Index Only Scan using _hyper_3_8_chunk_dimension_only_time_idx on _hyper_3_8_chunk -> Append -> Index Scan using _hyper_2_7_chunk_dimension_last_time_idx on _hyper_2_7_chunk Index Cond: ("time" = dimension_only."time") -> Index Scan using _hyper_2_6_chunk_dimension_last_time_idx on _hyper_2_6_chunk Index Cond: ("time" = dimension_only."time") -> Index Scan using _hyper_2_5_chunk_dimension_last_time_idx on _hyper_2_5_chunk Index Cond: ("time" = dimension_only."time") -> Index Scan using _hyper_2_4_chunk_dimension_last_time_idx on _hyper_2_4_chunk Index Cond: ("time" = dimension_only."time") -- test join against non-hypertable :PREFIX SELECT * FROM dimension_last INNER JOIN devices USING(device_id) ORDER BY dimension_last.time DESC LIMIT 2; --- QUERY PLAN --- Limit (actual rows=2.00 loops=1) -> Nested Loop (actual rows=2.00 loops=1) Join Filter: (devices.device_id = dimension_last.device_id) -> Custom Scan (ChunkAppend) on dimension_last (actual rows=2.00 loops=1) Order: dimension_last."time" DESC -> Index Scan using _hyper_2_7_chunk_dimension_last_time_idx on _hyper_2_7_chunk (actual rows=2.00 loops=1) -> Index Scan using _hyper_2_6_chunk_dimension_last_time_idx on _hyper_2_6_chunk (never executed) -> Index Scan using _hyper_2_5_chunk_dimension_last_time_idx on _hyper_2_5_chunk (never executed) -> Index Scan using _hyper_2_4_chunk_dimension_last_time_idx on _hyper_2_4_chunk (never executed) -> Materialize (actual rows=1.00 loops=2) -> Seq Scan on devices (actual rows=1.00 loops=1) -- test hypertable with index missing on one chunk :PREFIX SELECT time, device_id, value FROM ht_missing_indexes ORDER BY time ASC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on ht_missing_indexes (actual rows=1.00 loops=1) Order: ht_missing_indexes."time" -> Index Scan Backward using _hyper_4_12_chunk_ht_missing_indexes_time_idx on _hyper_4_12_chunk (actual rows=1.00 loops=1) -> Sort (never executed) Sort Key: _hyper_4_13_chunk."time" -> Seq Scan on _hyper_4_13_chunk (never executed) -> Index Scan Backward using _hyper_4_14_chunk_ht_missing_indexes_time_idx on _hyper_4_14_chunk (never executed) -- test hypertable with index missing on one chunk -- and no data :PREFIX SELECT time, device_id, value FROM ht_missing_indexes WHERE device_id = 2 ORDER BY time DESC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on ht_missing_indexes (actual rows=1.00 loops=1) Order: ht_missing_indexes."time" DESC -> Index Scan using _hyper_4_14_chunk_ht_missing_indexes_time_idx on _hyper_4_14_chunk (actual rows=1.00 loops=1) Filter: (device_id = 2) Rows Removed by Filter: 1 -> Sort (never executed) Sort Key: _hyper_4_13_chunk."time" DESC -> Seq Scan on _hyper_4_13_chunk (never executed) Filter: (device_id = 2) -> Index Scan using _hyper_4_12_chunk_ht_missing_indexes_time_idx on _hyper_4_12_chunk (never executed) Filter: (device_id = 2) -- test hypertable with index missing on one chunk -- and no data :PREFIX SELECT time, device_id, value FROM ht_missing_indexes WHERE time > '2000-01-07' ORDER BY time LIMIT 10; --- QUERY PLAN --- Limit (actual rows=10.00 loops=1) -> Custom Scan (ChunkAppend) on ht_missing_indexes (actual rows=10.00 loops=1) Order: ht_missing_indexes."time" -> Sort (actual rows=10.00 loops=1) Sort Key: _hyper_4_13_chunk."time" Sort Method: top-N heapsort -> Seq Scan on _hyper_4_13_chunk (actual rows=24477.00 loops=1) Filter: ("time" > 'Fri Jan 07 00:00:00 2000 PST'::timestamp with time zone) Rows Removed by Filter: 5763 -> Index Scan Backward using _hyper_4_14_chunk_ht_missing_indexes_time_idx on _hyper_4_14_chunk (never executed) -- test hypertable with dropped columns :PREFIX SELECT time, device_id, value FROM ht_dropped_columns ORDER BY time ASC LIMIT 1; --- QUERY PLAN --- Limit (actual rows=1.00 loops=1) -> Custom Scan (ChunkAppend) on ht_dropped_columns (actual rows=1.00 loops=1) Order: ht_dropped_columns."time" -> Index Scan Backward using _hyper_5_15_chunk_ht_dropped_columns_time_idx on _hyper_5_15_chunk (actual rows=1.00 loops=1) -> Index Scan Backward using _hyper_5_16_chunk_ht_dropped_columns_time_idx on _hyper_5_16_chunk (never executed) -> Index Scan Backward using _hyper_5_17_chunk_ht_dropped_columns_time_idx on _hyper_5_17_chunk (never executed) -> Index Scan Backward using _hyper_5_18_chunk_ht_dropped_columns_time_idx on _hyper_5_18_chunk (never executed) -> Index Scan Backward using _hyper_5_19_chunk_ht_dropped_columns_time_idx on _hyper_5_19_chunk (never executed) -- test hypertable with dropped columns :PREFIX SELECT time, device_id, value FROM ht_dropped_columns WHERE device_id = 1 ORDER BY time DESC; --- QUERY PLAN --- Custom Scan (ChunkAppend) on ht_dropped_columns (actual rows=7205.00 loops=1) Order: ht_dropped_columns."time" DESC -> Index Scan using _hyper_5_19_chunk_ht_dropped_columns_time_idx on _hyper_5_19_chunk (actual rows=1441.00 loops=1) Filter: (device_id = 1) -> Index Scan using _hyper_5_18_chunk_ht_dropped_columns_time_idx on _hyper_5_18_chunk (actual rows=1441.00 loops=1) Filter: (device_id = 1) -> Index Scan using _hyper_5_17_chunk_ht_dropped_columns_time_idx on _hyper_5_17_chunk (actual rows=1441.00 loops=1) Filter: (device_id = 1) -> Index Scan using _hyper_5_16_chunk_ht_dropped_columns_time_idx on _hyper_5_16_chunk (actual rows=1441.00 loops=1) Filter: (device_id = 1) -> Index Scan using _hyper_5_15_chunk_ht_dropped_columns_time_idx on _hyper_5_15_chunk (actual rows=1441.00 loops=1) Filter: (device_id = 1) -- test hypertable with 2 space dimensions :PREFIX SELECT time, device_id, value FROM space2 ORDER BY time DESC; --- QUERY PLAN --- Custom Scan (ChunkAppend) on space2 (actual rows=116649.00 loops=1) Order: space2."time" DESC -> Merge Append (actual rows=56169.00 loops=1) Sort Key: space2."time" DESC -> Index Scan using _hyper_6_36_chunk_space2_time_idx on _hyper_6_36_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_34_chunk_space2_time_idx on _hyper_6_34_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_32_chunk_space2_time_idx on _hyper_6_32_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_30_chunk_space2_time_idx on _hyper_6_30_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_28_chunk_space2_time_idx on _hyper_6_28_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_26_chunk_space2_time_idx on _hyper_6_26_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_24_chunk_space2_time_idx on _hyper_6_24_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_22_chunk_space2_time_idx on _hyper_6_22_chunk (actual rows=6241.00 loops=1) -> Index Scan using _hyper_6_20_chunk_space2_time_idx on _hyper_6_20_chunk (actual rows=6241.00 loops=1) -> Merge Append (actual rows=60480.00 loops=1) Sort Key: space2."time" DESC -> Index Scan using _hyper_6_37_chunk_space2_time_idx on _hyper_6_37_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_35_chunk_space2_time_idx on _hyper_6_35_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_33_chunk_space2_time_idx on _hyper_6_33_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_31_chunk_space2_time_idx on _hyper_6_31_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_29_chunk_space2_time_idx on _hyper_6_29_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_27_chunk_space2_time_idx on _hyper_6_27_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_25_chunk_space2_time_idx on _hyper_6_25_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_23_chunk_space2_time_idx on _hyper_6_23_chunk (actual rows=6720.00 loops=1) -> Index Scan using _hyper_6_21_chunk_space2_time_idx on _hyper_6_21_chunk (actual rows=6720.00 loops=1) -- test hypertable with 3 space dimensions :PREFIX SELECT time FROM space3 ORDER BY time DESC; --- QUERY PLAN --- Custom Scan (ChunkAppend) on space3 (actual rows=103688.00 loops=1) Order: space3."time" DESC -> Merge Append (actual rows=49928.00 loops=1) Sort Key: space3."time" DESC -> Index Only Scan using _hyper_7_52_chunk_space3_time_idx on _hyper_7_52_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_50_chunk_space3_time_idx on _hyper_7_50_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_48_chunk_space3_time_idx on _hyper_7_48_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_46_chunk_space3_time_idx on _hyper_7_46_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_44_chunk_space3_time_idx on _hyper_7_44_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_42_chunk_space3_time_idx on _hyper_7_42_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_40_chunk_space3_time_idx on _hyper_7_40_chunk (actual rows=6241.00 loops=1) -> Index Only Scan using _hyper_7_38_chunk_space3_time_idx on _hyper_7_38_chunk (actual rows=6241.00 loops=1) -> Merge Append (actual rows=53760.00 loops=1) Sort Key: space3."time" DESC -> Index Only Scan using _hyper_7_53_chunk_space3_time_idx on _hyper_7_53_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_51_chunk_space3_time_idx on _hyper_7_51_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_49_chunk_space3_time_idx on _hyper_7_49_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_47_chunk_space3_time_idx on _hyper_7_47_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_45_chunk_space3_time_idx on _hyper_7_45_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_43_chunk_space3_time_idx on _hyper_7_43_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_41_chunk_space3_time_idx on _hyper_7_41_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_7_39_chunk_space3_time_idx on _hyper_7_39_chunk (actual rows=6720.00 loops=1) -- test COLLATION -- cant be tested in our ci because alpine doesnt support locales -- :PREFIX SELECT * FROM sortopt_test ORDER BY time, device COLLATE "en_US.utf8"; -- test NULLS FIRST :PREFIX SELECT * FROM sortopt_test ORDER BY time, device NULLS FIRST; --- QUERY PLAN --- Custom Scan (ChunkAppend) on sortopt_test (actual rows=12961.00 loops=1) Order: sortopt_test."time", sortopt_test.device NULLS FIRST -> Index Only Scan using _hyper_8_55_chunk_time_device_nullsfirst on _hyper_8_55_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_8_54_chunk_time_device_nullsfirst on _hyper_8_54_chunk (actual rows=6241.00 loops=1) -- test NULLS LAST :PREFIX SELECT * FROM sortopt_test ORDER BY time, device DESC NULLS LAST; --- QUERY PLAN --- Custom Scan (ChunkAppend) on sortopt_test (actual rows=12961.00 loops=1) Order: sortopt_test."time", sortopt_test.device DESC NULLS LAST -> Index Only Scan using _hyper_8_55_chunk_time_device_nullslast on _hyper_8_55_chunk (actual rows=6720.00 loops=1) -> Index Only Scan using _hyper_8_54_chunk_time_device_nullslast on _hyper_8_54_chunk (actual rows=6241.00 loops=1) --generate the results into two different files \set ECHO errors ================================================ FILE: test/expected/query.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set TEST_BASE_NAME query SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') as "TEST_LOAD_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') as "TEST_QUERY_NAME", format('%s/results/%s_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_OPTIMIZED", format('%s/results/%s_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_UNOPTIMIZED" \gset SELECT format('\! diff -u --label "Unoptimized result" --label "Optimized result" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') as "DIFF_CMD" \gset \set PREFIX 'EXPLAIN (buffers OFF, costs OFF)' \ir :TEST_LOAD_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC.hyper_1 ( time TIMESTAMP NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL ); CREATE INDEX "time_plain" ON PUBLIC.hyper_1 (time DESC, series_0); SELECT * FROM create_hypertable('"public"."hyper_1"'::regclass, 'time'::name, number_partitions => 1, create_default_indexes=>false); psql:include/query_load.sql:13: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | hyper_1 | t INSERT INTO hyper_1 SELECT to_timestamp(ser), ser, ser+10000, sqrt(ser::numeric) FROM generate_series(0,10000) ser; INSERT INTO hyper_1 SELECT to_timestamp(ser), ser, ser+10000, sqrt(ser::numeric) FROM generate_series(10001,20000) ser; CREATE TABLE PUBLIC.hyper_1_tz ( time TIMESTAMPTZ NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL ); CREATE INDEX "time_plain_tz" ON PUBLIC.hyper_1_tz (time DESC, series_0); SELECT * FROM create_hypertable('"public"."hyper_1_tz"'::regclass, 'time'::name, number_partitions => 1, create_default_indexes=>false); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 2 | public | hyper_1_tz | t INSERT INTO hyper_1_tz SELECT to_timestamp(ser), ser, ser+10000, sqrt(ser::numeric) FROM generate_series(0,10000) ser; INSERT INTO hyper_1_tz SELECT to_timestamp(ser), ser, ser+10000, sqrt(ser::numeric) FROM generate_series(10001,20000) ser; CREATE TABLE PUBLIC.hyper_1_int ( time int NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL ); CREATE INDEX "time_plain_int" ON PUBLIC.hyper_1_int (time DESC, series_0); SELECT * FROM create_hypertable('"public"."hyper_1_int"'::regclass, 'time'::name, number_partitions => 1, chunk_time_interval=>10000, create_default_indexes=>FALSE); hypertable_id | schema_name | table_name | created ---------------+-------------+-------------+--------- 3 | public | hyper_1_int | t INSERT INTO hyper_1_int SELECT ser, ser, ser+10000, sqrt(ser::numeric) FROM generate_series(0,10000) ser; INSERT INTO hyper_1_int SELECT ser, ser, ser+10000, sqrt(ser::numeric) FROM generate_series(10001,20000) ser; CREATE TABLE PUBLIC.hyper_1_date ( time date NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL ); CREATE INDEX "time_plain_date" ON PUBLIC.hyper_1_date (time DESC, series_0); SELECT * FROM create_hypertable('"public"."hyper_1_date"'::regclass, 'time'::name, number_partitions => 1, chunk_time_interval=>86400000000, create_default_indexes=>FALSE); hypertable_id | schema_name | table_name | created ---------------+-------------+--------------+--------- 4 | public | hyper_1_date | t INSERT INTO hyper_1_date SELECT to_timestamp(ser)::date, ser, ser+10000, sqrt(ser::numeric) FROM generate_series(0,10000) ser; INSERT INTO hyper_1_date SELECT to_timestamp(ser)::date, ser, ser+10000, sqrt(ser::numeric) FROM generate_series(10001,20000) ser; --below needed to create enough unique dates to trigger an index scan INSERT INTO hyper_1_date SELECT to_timestamp(ser*100)::date, ser, ser+10000, sqrt(ser::numeric) FROM generate_series(10001,20000) ser; CREATE TABLE PUBLIC.plain_table ( time TIMESTAMPTZ NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL ); CREATE INDEX "time_plain_plain_table" ON PUBLIC.plain_table (time DESC, series_0); INSERT INTO plain_table SELECT to_timestamp(ser), ser, ser+10000, sqrt(ser::numeric) FROM generate_series(0,10000) ser; INSERT INTO plain_table SELECT to_timestamp(ser), ser, ser+10000, sqrt(ser::numeric) FROM generate_series(10001,20000) ser; -- Table with a time partitioning function CREATE TABLE PUBLIC.hyper_timefunc ( time float8 NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL ); CREATE OR REPLACE FUNCTION unix_to_timestamp(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE SQL IMMUTABLE AS $BODY$ SELECT to_timestamp(unixtime); $BODY$; CREATE INDEX "time_plain_timefunc" ON PUBLIC.hyper_timefunc (to_timestamp(time) DESC, series_0); SELECT * FROM create_hypertable('"public"."hyper_timefunc"'::regclass, 'time'::name, number_partitions => 1, create_default_indexes=>false, time_partitioning_func => 'unix_to_timestamp'); hypertable_id | schema_name | table_name | created ---------------+-------------+----------------+--------- 5 | public | hyper_timefunc | t INSERT INTO hyper_timefunc SELECT ser, ser, ser+10000, sqrt(ser::numeric) FROM generate_series(0,10000) ser; INSERT INTO hyper_timefunc SELECT ser, ser, ser+10000, sqrt(ser::numeric) FROM generate_series(10001,20000) ser; ANALYZE plain_table; ANALYZE hyper_timefunc; ANALYZE hyper_1; ANALYZE hyper_1_tz; ANALYZE hyper_1_int; ANALYZE hyper_1_date; \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SHOW timescaledb.enable_optimizations; timescaledb.enable_optimizations ---------------------------------- on --non-aggregates use MergeAppend in both optimized and non-optimized :PREFIX SELECT * FROM hyper_1 ORDER BY "time" DESC limit 2; --- QUERY PLAN --- Limit -> Index Scan using _hyper_1_1_chunk_time_plain on _hyper_1_1_chunk :PREFIX SELECT * FROM hyper_timefunc ORDER BY unix_to_timestamp("time") DESC limit 2; --- QUERY PLAN --- Limit -> Index Scan using _hyper_5_19_chunk_time_plain_timefunc on _hyper_5_19_chunk --Aggregates use MergeAppend only in optimized :PREFIX SELECT date_trunc('minute', time) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1 GROUP BY t ORDER BY t DESC limit 2; --- QUERY PLAN --- Limit -> GroupAggregate Group Key: (date_trunc('minute'::text, _hyper_1_1_chunk."time")) -> Result -> Index Scan using _hyper_1_1_chunk_time_plain on _hyper_1_1_chunk :PREFIX SELECT date_trunc('minute', time) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1_date GROUP BY t ORDER BY t DESC limit 2; --- QUERY PLAN --- Limit -> GroupAggregate Group Key: (date_trunc('minute'::text, (hyper_1_date."time")::timestamp with time zone)) -> Result -> Merge Append Sort Key: (date_trunc('minute'::text, (hyper_1_date."time")::timestamp with time zone)) DESC -> Index Scan using _hyper_4_6_chunk_time_plain_date on _hyper_4_6_chunk -> Index Scan using _hyper_4_7_chunk_time_plain_date on _hyper_4_7_chunk -> Index Scan using _hyper_4_8_chunk_time_plain_date on _hyper_4_8_chunk -> Index Scan using _hyper_4_9_chunk_time_plain_date on _hyper_4_9_chunk -> Index Scan using _hyper_4_10_chunk_time_plain_date on _hyper_4_10_chunk -> Index Scan using _hyper_4_11_chunk_time_plain_date on _hyper_4_11_chunk -> Index Scan using _hyper_4_12_chunk_time_plain_date on _hyper_4_12_chunk -> Index Scan using _hyper_4_13_chunk_time_plain_date on _hyper_4_13_chunk -> Index Scan using _hyper_4_14_chunk_time_plain_date on _hyper_4_14_chunk -> Index Scan using _hyper_4_15_chunk_time_plain_date on _hyper_4_15_chunk -> Index Scan using _hyper_4_16_chunk_time_plain_date on _hyper_4_16_chunk -> Index Scan using _hyper_4_17_chunk_time_plain_date on _hyper_4_17_chunk -> Index Scan using _hyper_4_18_chunk_time_plain_date on _hyper_4_18_chunk --the minute and second results should be diff :PREFIX SELECT date_trunc('minute', time) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1 GROUP BY t ORDER BY t DESC limit 2; --- QUERY PLAN --- Limit -> GroupAggregate Group Key: (date_trunc('minute'::text, _hyper_1_1_chunk."time")) -> Result -> Index Scan using _hyper_1_1_chunk_time_plain on _hyper_1_1_chunk :PREFIX SELECT date_trunc('second', time) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1 GROUP BY t ORDER BY t DESC limit 2; --- QUERY PLAN --- Limit -> GroupAggregate Group Key: (date_trunc('second'::text, _hyper_1_1_chunk."time")) -> Result -> Index Scan using _hyper_1_1_chunk_time_plain on _hyper_1_1_chunk --test that when index on time used by constraint, still works correctly :PREFIX SELECT date_trunc('minute', time) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1 WHERE time < to_timestamp(900) GROUP BY t ORDER BY t DESC LIMIT 2; --- QUERY PLAN --- Limit -> GroupAggregate Group Key: (date_trunc('minute'::text, hyper_1."time")) -> Result -> Custom Scan (ChunkAppend) on hyper_1 Order: date_trunc('minute'::text, hyper_1."time") DESC Chunks excluded during startup: 0 -> Index Scan using _hyper_1_1_chunk_time_plain on _hyper_1_1_chunk Index Cond: ("time" < 'Wed Dec 31 16:15:00 1969 PST'::timestamp with time zone) --test on table with time partitioning function. Currently not --optimized to use index for ordering since the index is an expression --on time (e.g., timefunc(time)), and we currently don't handle that --case. :PREFIX SELECT date_trunc('minute', to_timestamp(time)) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_timefunc WHERE to_timestamp(time) < to_timestamp(900) GROUP BY t ORDER BY t DESC LIMIT 2; --- QUERY PLAN --- Limit -> Sort Sort Key: (date_trunc('minute'::text, to_timestamp(_hyper_5_19_chunk."time"))) DESC -> HashAggregate Group Key: date_trunc('minute'::text, to_timestamp(_hyper_5_19_chunk."time")) -> Result -> Index Scan using _hyper_5_19_chunk_time_plain_timefunc on _hyper_5_19_chunk Index Cond: (to_timestamp("time") < 'Wed Dec 31 16:15:00 1969 PST'::timestamp with time zone) BEGIN; --test that still works with an expression index on data_trunc. DROP INDEX "time_plain"; CREATE INDEX "time_trunc" ON PUBLIC.hyper_1 (date_trunc('minute', time)); ANALYZE hyper_1; :PREFIX SELECT date_trunc('minute', time) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1 GROUP BY t ORDER BY t DESC limit 2; --- QUERY PLAN --- Limit -> GroupAggregate Group Key: (date_trunc('minute'::text, _hyper_1_1_chunk."time")) -> Result -> Index Scan Backward using _hyper_1_1_chunk_time_trunc on _hyper_1_1_chunk --test that works with both indexes CREATE INDEX "time_plain" ON PUBLIC.hyper_1 (time DESC, series_0); ANALYZE hyper_1; :PREFIX SELECT date_trunc('minute', time) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1 GROUP BY t ORDER BY t DESC limit 2; --- QUERY PLAN --- Limit -> GroupAggregate Group Key: (date_trunc('minute'::text, _hyper_1_1_chunk."time")) -> Result -> Index Scan Backward using _hyper_1_1_chunk_time_trunc on _hyper_1_1_chunk :PREFIX SELECT time_bucket('1 minute', time) t, avg(series_0), min(series_1), trunc(avg(series_2)::numeric, 5) FROM hyper_1 GROUP BY t ORDER BY t DESC limit 2; --- QUERY PLAN --- Limit -> GroupAggregate Group Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time")) -> Result -> Index Scan using _hyper_1_1_chunk_time_plain on _hyper_1_1_chunk :PREFIX SELECT time_bucket('1 minute', time, INTERVAL '30 seconds') t, avg(series_0), min(series_1), trunc(avg(series_2)::numeric,5) FROM hyper_1 GROUP BY t ORDER BY t DESC limit 2; --- QUERY PLAN --- Limit -> GroupAggregate Group Key: (time_bucket('@ 1 min'::interval, _hyper_1_1_chunk."time", '@ 30 secs'::interval)) -> Result -> Index Scan using _hyper_1_1_chunk_time_plain on _hyper_1_1_chunk :PREFIX SELECT time_bucket('1 minute', time - INTERVAL '30 seconds') t, avg(series_0), min(series_1), trunc(avg(series_2)::numeric,5) FROM hyper_1 GROUP BY t ORDER BY t DESC limit 2; --- QUERY PLAN --- Limit -> GroupAggregate Group Key: (time_bucket('@ 1 min'::interval, (_hyper_1_1_chunk."time" - '@ 30 secs'::interval))) -> Result -> Index Scan using _hyper_1_1_chunk_time_plain on _hyper_1_1_chunk :PREFIX SELECT time_bucket('1 minute', time - INTERVAL '30 seconds') + INTERVAL '30 seconds' t, avg(series_0), min(series_1), trunc(avg(series_2)::numeric,5) FROM hyper_1 GROUP BY t ORDER BY t DESC limit 2; --- QUERY PLAN --- Limit -> GroupAggregate Group Key: ((time_bucket('@ 1 min'::interval, (_hyper_1_1_chunk."time" - '@ 30 secs'::interval)) + '@ 30 secs'::interval)) -> Result -> Index Scan using _hyper_1_1_chunk_time_plain on _hyper_1_1_chunk :PREFIX SELECT time_bucket('1 minute', time) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1_tz GROUP BY t ORDER BY t DESC limit 2; --- QUERY PLAN --- Limit -> GroupAggregate Group Key: (time_bucket('@ 1 min'::interval, _hyper_2_2_chunk."time")) -> Result -> Index Scan using _hyper_2_2_chunk_time_plain_tz on _hyper_2_2_chunk :PREFIX SELECT time_bucket('1 minute', time::timestamp) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1_tz GROUP BY t ORDER BY t DESC limit 2; --- QUERY PLAN --- Limit -> GroupAggregate Group Key: (time_bucket('@ 1 min'::interval, (_hyper_2_2_chunk."time")::timestamp without time zone)) -> Result -> Index Scan using _hyper_2_2_chunk_time_plain_tz on _hyper_2_2_chunk :PREFIX SELECT time_bucket(10, time) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1_int GROUP BY t ORDER BY t DESC limit 2; --- QUERY PLAN --- Limit -> GroupAggregate Group Key: (time_bucket(10, hyper_1_int."time")) -> Result -> Custom Scan (ChunkAppend) on hyper_1_int Order: time_bucket(10, hyper_1_int."time") DESC -> Index Scan using _hyper_3_5_chunk_time_plain_int on _hyper_3_5_chunk -> Index Scan using _hyper_3_4_chunk_time_plain_int on _hyper_3_4_chunk -> Index Scan using _hyper_3_3_chunk_time_plain_int on _hyper_3_3_chunk :PREFIX SELECT time_bucket(10, time, 2) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1_int GROUP BY t ORDER BY t DESC limit 2; --- QUERY PLAN --- Limit -> GroupAggregate Group Key: (time_bucket(10, hyper_1_int."time", 2)) -> Result -> Custom Scan (ChunkAppend) on hyper_1_int Order: time_bucket(10, hyper_1_int."time", 2) DESC -> Index Scan using _hyper_3_5_chunk_time_plain_int on _hyper_3_5_chunk -> Index Scan using _hyper_3_4_chunk_time_plain_int on _hyper_3_4_chunk -> Index Scan using _hyper_3_3_chunk_time_plain_int on _hyper_3_3_chunk ROLLBACK; -- sort order optimization should not be applied to non-hypertables :PREFIX SELECT date_trunc('minute', time) t, avg(series_0), min(series_1), avg(series_2) FROM plain_table WHERE time < to_timestamp(900) GROUP BY t ORDER BY t DESC LIMIT 2; --- QUERY PLAN --- Limit -> Sort Sort Key: (date_trunc('minute'::text, "time")) DESC -> HashAggregate Group Key: date_trunc('minute'::text, "time") -> Index Scan using time_plain_plain_table on plain_table Index Cond: ("time" < 'Wed Dec 31 16:15:00 1969 PST'::timestamp with time zone) --generate the results into two different files \set ECHO errors --- Unoptimized result +++ Optimized result @@ -1,6 +1,6 @@ timescaledb.enable_optimizations ---------------------------------- - off + on time | series_0 | series_1 | series_2 ?column? ---------- Done ================================================ FILE: test/expected/relocate_extension.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Set this variable to avoid using a hard-coded path each time query -- results are compared \set QUERY_RESULT_TEST_EQUAL_RELPATH 'include/query_result_test_equal.sql' \c postgres :ROLE_SUPERUSER DROP DATABASE :TEST_DBNAME WITH (FORCE); CREATE DATABASE :TEST_DBNAME; \c :TEST_DBNAME CREATE SCHEMA "testSchema0"; SET client_min_messages=error; CREATE EXTENSION IF NOT EXISTS timescaledb SCHEMA "testSchema0"; RESET client_min_messages; CREATE TABLE test_ts(time timestamp, temp float8, device text); CREATE TABLE test_tz(time timestamptz, temp float8, device text); CREATE TABLE test_dt(time date, temp float8, device text); SELECT "testSchema0".create_hypertable('test_ts', 'time', 'device', 2); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ---------------------- (1,public,test_ts,t) SELECT "testSchema0".create_hypertable('test_tz', 'time', 'device', 2); create_hypertable ---------------------- (2,public,test_tz,t) SELECT "testSchema0".create_hypertable('test_dt', 'time', 'device', 2); create_hypertable ---------------------- (3,public,test_dt,t) SELECT * FROM _timescaledb_catalog.hypertable; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 1 | public | test_ts | _timescaledb_internal | _hyper_1 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 2 | public | test_tz | _timescaledb_internal | _hyper_2 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 3 | public | test_dt | _timescaledb_internal | _hyper_3 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 INSERT INTO test_ts VALUES('Mon Mar 20 09:17:00.936242 2017', 23.4, 'dev1'); INSERT INTO test_ts VALUES('Mon Mar 20 09:27:00.936242 2017', 22, 'dev2'); INSERT INTO test_ts VALUES('Mon Mar 20 09:28:00.936242 2017', 21.2, 'dev1'); INSERT INTO test_ts VALUES('Mon Mar 20 09:37:00.936242 2017', 30, 'dev3'); SELECT * FROM test_ts ORDER BY time; time | temp | device ---------------------------------+------+-------- Mon Mar 20 09:17:00.936242 2017 | 23.4 | dev1 Mon Mar 20 09:27:00.936242 2017 | 22 | dev2 Mon Mar 20 09:28:00.936242 2017 | 21.2 | dev1 Mon Mar 20 09:37:00.936242 2017 | 30 | dev3 INSERT INTO test_tz VALUES('Mon Mar 20 09:17:00.936242 2017', 23.4, 'dev1'); INSERT INTO test_tz VALUES('Mon Mar 20 09:27:00.936242 2017', 22, 'dev2'); INSERT INTO test_tz VALUES('Mon Mar 20 09:28:00.936242 2017', 21.2, 'dev1'); INSERT INTO test_tz VALUES('Mon Mar 20 09:37:00.936242 2017', 30, 'dev3'); SELECT * FROM test_tz ORDER BY time; time | temp | device -------------------------------------+------+-------- Mon Mar 20 09:17:00.936242 2017 PDT | 23.4 | dev1 Mon Mar 20 09:27:00.936242 2017 PDT | 22 | dev2 Mon Mar 20 09:28:00.936242 2017 PDT | 21.2 | dev1 Mon Mar 20 09:37:00.936242 2017 PDT | 30 | dev3 INSERT INTO test_dt VALUES('Mon Mar 20 09:17:00.936242 2017', 23.4, 'dev1'); INSERT INTO test_dt VALUES('Mon Mar 21 09:27:00.936242 2017', 22, 'dev2'); INSERT INTO test_dt VALUES('Mon Mar 22 09:28:00.936242 2017', 21.2, 'dev1'); INSERT INTO test_dt VALUES('Mon Mar 23 09:37:00.936242 2017', 30, 'dev3'); SELECT * FROM test_dt ORDER BY time; time | temp | device ------------+------+-------- 03-20-2017 | 23.4 | dev1 03-21-2017 | 22 | dev2 03-22-2017 | 21.2 | dev1 03-23-2017 | 30 | dev3 -- testing time_bucket START SELECT AVG(temp) AS avg_tmp, "testSchema0".time_bucket('5 minutes', time, INTERVAL '1 minutes') AS ten_min FROM test_ts GROUP BY ten_min ORDER BY avg_tmp; avg_tmp | ten_min ---------+-------------------------- 21.6 | Mon Mar 20 09:26:00 2017 23.4 | Mon Mar 20 09:16:00 2017 30 | Mon Mar 20 09:36:00 2017 SELECT AVG(temp) AS avg_tmp, "testSchema0".time_bucket('5 minutes', time, INTERVAL '1 minutes') AS ten_min FROM test_tz GROUP BY ten_min ORDER BY avg_tmp; avg_tmp | ten_min ---------+------------------------------ 21.6 | Mon Mar 20 09:26:00 2017 PDT 23.4 | Mon Mar 20 09:16:00 2017 PDT 30 | Mon Mar 20 09:36:00 2017 PDT SELECT AVG(temp) AS avg_tmp, "testSchema0".time_bucket('1 day', time, INTERVAL '-0.5 day') AS ten_min FROM test_dt GROUP BY ten_min ORDER BY avg_tmp; avg_tmp | ten_min ---------+------------ 21.2 | 03-21-2017 22 | 03-20-2017 23.4 | 03-19-2017 30 | 03-22-2017 -- testing time_bucket END -- testing drop_chunks START -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT "testSchema0".show_chunks(older_than => \'2017-03-01\'::timestamp, relation => \'test_ts\')::REGCLASS::TEXT' \set QUERY2 'SELECT "testSchema0".drop_chunks(\'test_ts\', \'2017-03-01\'::timestamp)::TEXT' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 0 | 0 SELECT * FROM test_ts ORDER BY time; time | temp | device ---------------------------------+------+-------- Mon Mar 20 09:17:00.936242 2017 | 23.4 | dev1 Mon Mar 20 09:27:00.936242 2017 | 22 | dev2 Mon Mar 20 09:28:00.936242 2017 | 21.2 | dev1 Mon Mar 20 09:37:00.936242 2017 | 30 | dev3 \set QUERY1 'SELECT "testSchema0".show_chunks(older_than => interval \'1 minutes\', relation => \'test_tz\')::REGCLASS::TEXT' \set QUERY2 'SELECT "testSchema0".drop_chunks(\'test_tz\', interval \'1 minutes\')::TEXT' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 2 | 2 SELECT * FROM test_tz ORDER BY time; time | temp | device ------+------+-------- \set QUERY1 'SELECT "testSchema0".show_chunks(older_than => interval \'1 minutes\', relation => \'test_dt\')::REGCLASS::TEXT' \set QUERY2 'SELECT "testSchema0".drop_chunks(\'test_dt\', interval \'1 minutes\')::TEXT' \set ECHO errors Different Rows | Total Rows from Query 1 | Total Rows from Query 2 ----------------+-------------------------+------------------------- 0 | 3 | 3 SELECT * FROM test_dt ORDER BY time; time | temp | device ------+------+-------- -- testing drop_chunks END -- testing hypertable_detailed_size START SELECT * FROM "testSchema0".hypertable_detailed_size('test_ts'); table_bytes | index_bytes | toast_bytes | total_bytes | node_name -------------+-------------+-------------+-------------+----------- 16384 | 81920 | 24576 | 122880 | -- testing hypertable_detailed_size END SELECT * FROM "testSchema0".hypertable_index_size('test_ts_time_idx'); hypertable_index_size ----------------------- 40960 SELECT * FROM "testSchema0".hypertable_index_size('test_ts_device_time_idx'); hypertable_index_size ----------------------- 40960 CREATE SCHEMA "testSchema"; \set ON_ERROR_STOP 0 ALTER EXTENSION timescaledb SET SCHEMA "testSchema"; ERROR: extension "timescaledb" does not support SET SCHEMA \set ON_ERROR_STOP 1 ================================================ FILE: test/expected/reloptions.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE reloptions_test(time integer, temp float8, color integer) WITH (fillfactor=75, autovacuum_vacuum_threshold=100); SELECT create_hypertable('reloptions_test', 'time', chunk_time_interval => 3); create_hypertable ------------------------------ (1,public,reloptions_test,t) INSERT INTO reloptions_test VALUES (4, 24.3, 1), (9, 13.3, 2); -- Show that reloptions are inherited by chunks SELECT relname, reloptions FROM pg_class WHERE relname ~ '^_hyper.*' AND relkind = 'r'; relname | reloptions ------------------+------------------------------------------------- _hyper_1_1_chunk | {fillfactor=75,autovacuum_vacuum_threshold=100} _hyper_1_2_chunk | {fillfactor=75,autovacuum_vacuum_threshold=100} -- Alter reloptions. We support multiple options for the ALTER TABLE ALTER TABLE reloptions_test SET (fillfactor=80, parallel_workers=8); ALTER TABLE reloptions_test SET (fillfactor=80), SET (parallel_workers=8); SELECT relname, reloptions FROM pg_class WHERE relname ~ '^_hyper.*' AND relkind = 'r'; relname | reloptions ------------------+-------------------------------------------------------------------- _hyper_1_1_chunk | {autovacuum_vacuum_threshold=100,fillfactor=80,parallel_workers=8} _hyper_1_2_chunk | {autovacuum_vacuum_threshold=100,fillfactor=80,parallel_workers=8} ALTER TABLE reloptions_test RESET (fillfactor); SELECT relname, reloptions FROM pg_class WHERE relname ~ '^_hyper.*' AND relkind = 'r'; relname | reloptions ------------------+------------------------------------------------------ _hyper_1_1_chunk | {autovacuum_vacuum_threshold=100,parallel_workers=8} _hyper_1_2_chunk | {autovacuum_vacuum_threshold=100,parallel_workers=8} -- Test reloptions on a regular table CREATE TABLE reloptions_test2(time integer, temp float8, color integer); ALTER TABLE reloptions_test2 SET (fillfactor=80, parallel_workers=8); ALTER TABLE reloptions_test2 SET (fillfactor=80), SET (parallel_workers=8); DROP TABLE reloptions_test2; ================================================ FILE: test/expected/repair.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- We are testing different repair functions here to make sure that -- they work as expected. \c :TEST_DBNAME :ROLE_SUPERUSER \set TMP_USER :TEST_DBNAME _wizard CREATE USER :TMP_USER; CREATE USER "Random L User"; CREATE TABLE test_table_1(time timestamptz not null, temp float); SELECT create_hypertable('test_table_1', by_range('time', '1 day'::interval)); create_hypertable ------------------- (1,t) INSERT INTO test_table_1(time,temp) SELECT time, 100 * random() FROM generate_series( '2000-01-01'::timestamptz, '2000-01-05'::timestamptz, '1min'::interval ) time; CREATE TABLE test_table_2(time timestamptz not null, temp float); SELECT create_hypertable('test_table_2', by_range('time', '1 day'::interval)); create_hypertable ------------------- (2,t) INSERT INTO test_table_2(time,temp) SELECT time, 100 * random() FROM generate_series( '2000-01-01'::timestamptz, '2000-01-05'::timestamptz, '1min'::interval ) time; GRANT ALL ON test_table_1 TO :TMP_USER; GRANT ALL ON test_table_2 TO :TMP_USER; GRANT SELECT, INSERT ON test_table_1 TO "Random L User"; GRANT INSERT ON test_table_2 TO "Random L User"; -- Break the relacl of the table by deleting users DELETE FROM pg_authid WHERE rolname IN (:'TMP_USER', 'Random L User'); CREATE TABLE saved (LIKE pg_class); INSERT INTO saved SELECT * FROM pg_class; CALL _timescaledb_functions.repair_relation_acls(); -- The only thing we should see here are the relations we broke and -- the privileges we added for that user. No other relations should be -- touched. WITH lhs AS (SELECT oid, aclexplode(relacl) FROM pg_class), rhs AS (SELECT oid, aclexplode(relacl) FROM saved) SELECT rhs.oid::regclass FROM lhs FULL OUTER JOIN rhs ON row(lhs) = row(rhs) WHERE lhs.oid IS NULL AND rhs.oid IS NOT NULL GROUP BY rhs.oid; oid ----------------------------------------- test_table_1 _timescaledb_internal._hyper_1_1_chunk _timescaledb_internal._hyper_1_2_chunk _timescaledb_internal._hyper_1_3_chunk _timescaledb_internal._hyper_1_4_chunk _timescaledb_internal._hyper_1_5_chunk test_table_2 _timescaledb_internal._hyper_2_6_chunk _timescaledb_internal._hyper_2_7_chunk _timescaledb_internal._hyper_2_8_chunk _timescaledb_internal._hyper_2_9_chunk _timescaledb_internal._hyper_2_10_chunk DROP TABLE saved; DROP TABLE test_table_1; DROP TABLE test_table_2; ================================================ FILE: test/expected/rowsecurity-15.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- -- Test of Row-level security feature -- -- Clean up in case a prior regression run failed \c :TEST_DBNAME :ROLE_SUPERUSER \set ON_ERROR_STOP 0 \set VERBOSITY default SET timescaledb.enable_constraint_exclusion TO off; -- Suppress NOTICE messages when users/groups don't exist SET client_min_messages TO 'warning'; DROP USER IF EXISTS regress_rls_alice; DROP USER IF EXISTS regress_rls_bob; DROP USER IF EXISTS regress_rls_carol; DROP USER IF EXISTS regress_rls_dave; DROP USER IF EXISTS regress_rls_exempt_user; DROP ROLE IF EXISTS regress_rls_group1; DROP ROLE IF EXISTS regress_rls_group2; DROP SCHEMA IF EXISTS regress_rls_schema CASCADE; RESET client_min_messages; -- initial setup CREATE USER regress_rls_alice NOLOGIN; CREATE USER regress_rls_bob NOLOGIN; CREATE USER regress_rls_carol NOLOGIN; CREATE USER regress_rls_dave NOLOGIN; CREATE USER regress_rls_exempt_user BYPASSRLS NOLOGIN; CREATE ROLE regress_rls_group1 NOLOGIN; CREATE ROLE regress_rls_group2 NOLOGIN; GRANT regress_rls_group1 TO regress_rls_bob; GRANT regress_rls_group2 TO regress_rls_carol; CREATE SCHEMA regress_rls_schema; GRANT ALL ON SCHEMA regress_rls_schema to public; SET search_path = regress_rls_schema; -- setup of malicious function CREATE OR REPLACE FUNCTION f_leak(text) RETURNS bool COST 0.0000001 LANGUAGE plpgsql AS 'BEGIN RAISE NOTICE ''f_leak => %'', $1; RETURN true; END'; GRANT EXECUTE ON FUNCTION f_leak(text) TO public; -- BASIC Row-Level Security Scenario SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE uaccount ( pguser name primary key, seclv int ); GRANT SELECT ON uaccount TO public; INSERT INTO uaccount VALUES ('regress_rls_alice', 99), ('regress_rls_bob', 1), ('regress_rls_carol', 2), ('regress_rls_dave', 3); CREATE TABLE category ( cid int primary key, cname text ); GRANT ALL ON category TO public; INSERT INTO category VALUES (11, 'novel'), (22, 'science fiction'), (33, 'technology'), (44, 'manga'); CREATE TABLE document ( did int primary key, cid int references category(cid), dlevel int not null, dauthor name, dtitle text ); GRANT ALL ON document TO public; SELECT public.create_hypertable('document', 'did', chunk_time_interval=>2); create_hypertable ----------------------------------- (1,regress_rls_schema,document,t) INSERT INTO document VALUES ( 1, 11, 1, 'regress_rls_bob', 'my first novel'), ( 2, 11, 2, 'regress_rls_bob', 'my second novel'), ( 3, 22, 2, 'regress_rls_bob', 'my science fiction'), ( 4, 44, 1, 'regress_rls_bob', 'my first manga'), ( 5, 44, 2, 'regress_rls_bob', 'my second manga'), ( 6, 22, 1, 'regress_rls_carol', 'great science fiction'), ( 7, 33, 2, 'regress_rls_carol', 'great technology book'), ( 8, 44, 1, 'regress_rls_carol', 'great manga'), ( 9, 22, 1, 'regress_rls_dave', 'awesome science fiction'), (10, 33, 2, 'regress_rls_dave', 'awesome technology book'); ALTER TABLE document ENABLE ROW LEVEL SECURITY; -- user's security level must be higher than or equal to document's CREATE POLICY p1 ON document AS PERMISSIVE USING (dlevel <= (SELECT seclv FROM uaccount WHERE pguser = current_user)); -- try to create a policy of bogus type CREATE POLICY p1 ON document AS UGLY USING (dlevel <= (SELECT seclv FROM uaccount WHERE pguser = current_user)); ERROR: unrecognized row security option "ugly" LINE 1: CREATE POLICY p1 ON document AS UGLY ^ HINT: Only PERMISSIVE or RESTRICTIVE policies are supported currently. -- but Dave isn't allowed to anything at cid 50 or above -- this is to make sure that we sort the policies by name first -- when applying WITH CHECK, a later INSERT by Dave should fail due -- to p1r first CREATE POLICY p2r ON document AS RESTRICTIVE TO regress_rls_dave USING (cid <> 44 AND cid < 50); -- and Dave isn't allowed to see manga documents CREATE POLICY p1r ON document AS RESTRICTIVE TO regress_rls_dave USING (cid <> 44); \dp Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------------------+----------+-------+---------------------------------------------+-------------------+-------------------------------------------- regress_rls_schema | category | table | regress_rls_alice=arwdDxt/regress_rls_alice+| | | | | =arwdDxt/regress_rls_alice | | regress_rls_schema | document | table | regress_rls_alice=arwdDxt/regress_rls_alice+| | p1: + | | | =arwdDxt/regress_rls_alice | | (u): (dlevel <= ( SELECT uaccount.seclv + | | | | | FROM uaccount + | | | | | WHERE (uaccount.pguser = CURRENT_USER)))+ | | | | | p2r (RESTRICTIVE): + | | | | | (u): ((cid <> 44) AND (cid < 50)) + | | | | | to: regress_rls_dave + | | | | | p1r (RESTRICTIVE): + | | | | | (u): (cid <> 44) + | | | | | to: regress_rls_dave regress_rls_schema | uaccount | table | regress_rls_alice=arwdDxt/regress_rls_alice+| | | | | =r/regress_rls_alice | | \d document Table "regress_rls_schema.document" Column | Type | Collation | Nullable | Default ---------+---------+-----------+----------+--------- did | integer | | not null | cid | integer | | | dlevel | integer | | not null | dauthor | name | | | dtitle | text | | | Indexes: "document_pkey" PRIMARY KEY, btree (did) Foreign-key constraints: "document_cid_fkey" FOREIGN KEY (cid) REFERENCES category(cid) Policies: POLICY "p1" USING ((dlevel <= ( SELECT uaccount.seclv FROM uaccount WHERE (uaccount.pguser = CURRENT_USER)))) POLICY "p1r" AS RESTRICTIVE TO regress_rls_dave USING ((cid <> 44)) POLICY "p2r" AS RESTRICTIVE TO regress_rls_dave USING (((cid <> 44) AND (cid < 50))) Number of child tables: 6 (Use \d+ to list them.) SELECT * FROM pg_policies WHERE schemaname = 'regress_rls_schema' AND tablename = 'document' ORDER BY policyname; schemaname | tablename | policyname | permissive | roles | cmd | qual | with_check --------------------+-----------+------------+-------------+--------------------+-----+--------------------------------------------+------------ regress_rls_schema | document | p1 | PERMISSIVE | {public} | ALL | (dlevel <= ( SELECT uaccount.seclv +| | | | | | | FROM uaccount +| | | | | | | WHERE (uaccount.pguser = CURRENT_USER))) | regress_rls_schema | document | p1r | RESTRICTIVE | {regress_rls_dave} | ALL | (cid <> 44) | regress_rls_schema | document | p2r | RESTRICTIVE | {regress_rls_dave} | ALL | ((cid <> 44) AND (cid < 50)) | -- viewpoint from regress_rls_bob SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my first manga NOTICE: f_leak => great science fiction NOTICE: f_leak => great manga NOTICE: f_leak => awesome science fiction did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 4 | 44 | 1 | regress_rls_bob | my first manga 6 | 22 | 1 | regress_rls_carol | great science fiction 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my first manga NOTICE: f_leak => great science fiction NOTICE: f_leak => great manga NOTICE: f_leak => awesome science fiction cid | did | dlevel | dauthor | dtitle | cname -----+-----+--------+-------------------+-------------------------+----------------- 11 | 1 | 1 | regress_rls_bob | my first novel | novel 44 | 4 | 1 | regress_rls_bob | my first manga | manga 22 | 6 | 1 | regress_rls_carol | great science fiction | science fiction 44 | 8 | 1 | regress_rls_carol | great manga | manga 22 | 9 | 1 | regress_rls_dave | awesome science fiction | science fiction -- try a sampled version SELECT * FROM document TABLESAMPLE BERNOULLI(50) REPEATABLE(0) WHERE f_leak(dtitle) ORDER BY did; did | cid | dlevel | dauthor | dtitle -----+-----+--------+---------+-------- -- viewpoint from regress_rls_carol SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => my first manga NOTICE: f_leak => my second manga NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great manga NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => my first manga NOTICE: f_leak => my second manga NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great manga NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book cid | did | dlevel | dauthor | dtitle | cname -----+-----+--------+-------------------+-------------------------+----------------- 11 | 1 | 1 | regress_rls_bob | my first novel | novel 11 | 2 | 2 | regress_rls_bob | my second novel | novel 22 | 3 | 2 | regress_rls_bob | my science fiction | science fiction 44 | 4 | 1 | regress_rls_bob | my first manga | manga 44 | 5 | 2 | regress_rls_bob | my second manga | manga 22 | 6 | 1 | regress_rls_carol | great science fiction | science fiction 33 | 7 | 2 | regress_rls_carol | great technology book | technology 44 | 8 | 1 | regress_rls_carol | great manga | manga 22 | 9 | 1 | regress_rls_dave | awesome science fiction | science fiction 33 | 10 | 2 | regress_rls_dave | awesome technology book | technology -- try a sampled version SELECT * FROM document TABLESAMPLE BERNOULLI(50) REPEATABLE(0) WHERE f_leak(dtitle) ORDER BY did; did | cid | dlevel | dauthor | dtitle -----+-----+--------+---------+-------- EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on document document_1 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((dlevel <= $0) AND f_leak(dtitle)) EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle); --- QUERY PLAN --- Hash Join Hash Cond: (document.cid = category.cid) InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 -> Seq Scan on document document_1 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Hash -> Seq Scan on category -- viewpoint from regress_rls_dave SET SESSION AUTHORIZATION regress_rls_dave; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book cid | did | dlevel | dauthor | dtitle | cname -----+-----+--------+-------------------+-------------------------+----------------- 11 | 1 | 1 | regress_rls_bob | my first novel | novel 11 | 2 | 2 | regress_rls_bob | my second novel | novel 22 | 3 | 2 | regress_rls_bob | my science fiction | science fiction 22 | 6 | 1 | regress_rls_carol | great science fiction | science fiction 33 | 7 | 2 | regress_rls_carol | great technology book | technology 22 | 9 | 1 | regress_rls_dave | awesome science fiction | science fiction 33 | 10 | 2 | regress_rls_dave | awesome technology book | technology EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on document document_1 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle); --- QUERY PLAN --- Hash Join Hash Cond: (category.cid = document.cid) InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on category -> Hash -> Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 -> Seq Scan on document document_1 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -- 44 would technically fail for both p2r and p1r, but we should get an error -- back from p1r for this because it sorts first INSERT INTO document VALUES (100, 44, 1, 'regress_rls_dave', 'testing sorting of policies'); -- fail ERROR: new row violates row-level security policy "p1r" for table "document" -- Just to see a p2r error INSERT INTO document VALUES (100, 55, 1, 'regress_rls_dave', 'testing sorting of policies'); -- fail ERROR: new row violates row-level security policy "p2r" for table "document" -- only owner can change policies ALTER POLICY p1 ON document USING (true); --fail ERROR: must be owner of table document DROP POLICY p1 ON document; --fail ERROR: must be owner of relation document SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY p1 ON document USING (dauthor = current_user); -- viewpoint from regress_rls_bob again SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => my first manga NOTICE: f_leak => my second manga did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+-------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER by did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => my first manga NOTICE: f_leak => my second manga cid | did | dlevel | dauthor | dtitle | cname -----+-----+--------+-----------------+--------------------+----------------- 11 | 1 | 1 | regress_rls_bob | my first novel | novel 11 | 2 | 2 | regress_rls_bob | my second novel | novel 22 | 3 | 2 | regress_rls_bob | my science fiction | science fiction 44 | 4 | 1 | regress_rls_bob | my first manga | manga 44 | 5 | 2 | regress_rls_bob | my second manga | manga -- viewpoint from rls_regres_carol again SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great manga did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+----------------------- 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER by did; NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great manga cid | did | dlevel | dauthor | dtitle | cname -----+-----+--------+-------------------+-----------------------+----------------- 22 | 6 | 1 | regress_rls_carol | great science fiction | science fiction 33 | 7 | 2 | regress_rls_carol | great technology book | technology 44 | 8 | 1 | regress_rls_carol | great manga | manga EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 -> Seq Scan on document document_1 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle); --- QUERY PLAN --- Nested Loop -> Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 -> Seq Scan on document document_1 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Index Scan using category_pkey on category Index Cond: (cid = document.cid) -- interaction of FK/PK constraints SET SESSION AUTHORIZATION regress_rls_alice; CREATE POLICY p2 ON category USING (CASE WHEN current_user = 'regress_rls_bob' THEN cid IN (11, 33) WHEN current_user = 'regress_rls_carol' THEN cid IN (22, 44) ELSE false END); ALTER TABLE category ENABLE ROW LEVEL SECURITY; -- cannot delete PK referenced by invisible FK SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM document d FULL OUTER JOIN category c on d.cid = c.cid ORDER BY d.did, c.cid; did | cid | dlevel | dauthor | dtitle | cid | cname -----+-----+--------+-----------------+--------------------+-----+------------ 1 | 11 | 1 | regress_rls_bob | my first novel | 11 | novel 2 | 11 | 2 | regress_rls_bob | my second novel | 11 | novel 3 | 22 | 2 | regress_rls_bob | my science fiction | | 4 | 44 | 1 | regress_rls_bob | my first manga | | 5 | 44 | 2 | regress_rls_bob | my second manga | | | | | | | 33 | technology \set VERBOSITY sqlstate DELETE FROM category WHERE cid = 33; -- fails with FK violation ERROR: 23503 \set VERBOSITY default -- can insert FK referencing invisible PK SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM document d FULL OUTER JOIN category c on d.cid = c.cid ORDER BY d.did, c.cid; did | cid | dlevel | dauthor | dtitle | cid | cname -----+-----+--------+-------------------+-----------------------+-----+----------------- 6 | 22 | 1 | regress_rls_carol | great science fiction | 22 | science fiction 7 | 33 | 2 | regress_rls_carol | great technology book | | 8 | 44 | 1 | regress_rls_carol | great manga | 44 | manga INSERT INTO document VALUES (11, 33, 1, current_user, 'hoge'); -- UNIQUE or PRIMARY KEY constraint violation DOES reveal presence of row SET SESSION AUTHORIZATION regress_rls_bob; INSERT INTO document VALUES (8, 44, 1, 'regress_rls_bob', 'my third manga'); -- Must fail with unique violation, revealing presence of did we can't see ERROR: duplicate key value violates unique constraint "5_10_document_pkey" DETAIL: Key (did)=(8) already exists. SELECT * FROM document WHERE did = 8; -- and confirm we can't see it did | cid | dlevel | dauthor | dtitle -----+-----+--------+---------+-------- -- RLS policies are checked before constraints INSERT INTO document VALUES (8, 44, 1, 'regress_rls_carol', 'my third manga'); -- Should fail with RLS check violation, not duplicate key violation ERROR: new row violates row-level security policy for table "document" UPDATE document SET did = 8, dauthor = 'regress_rls_carol' WHERE did = 5; -- Should fail with RLS check violation, not duplicate key violation ERROR: new row violates row-level security policy for table "document" -- database superuser does bypass RLS policy when enabled RESET SESSION AUTHORIZATION; SET row_security TO ON; SELECT * FROM document; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book 11 | 33 | 1 | regress_rls_carol | hoge SELECT * FROM category; cid | cname -----+----------------- 11 | novel 22 | science fiction 33 | technology 44 | manga -- database superuser does bypass RLS policy when disabled RESET SESSION AUTHORIZATION; SET row_security TO OFF; SELECT * FROM document; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book 11 | 33 | 1 | regress_rls_carol | hoge SELECT * FROM category; cid | cname -----+----------------- 11 | novel 22 | science fiction 33 | technology 44 | manga -- database non-superuser with bypass privilege can bypass RLS policy when disabled SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; SELECT * FROM document; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book 11 | 33 | 1 | regress_rls_carol | hoge SELECT * FROM category; cid | cname -----+----------------- 11 | novel 22 | science fiction 33 | technology 44 | manga -- RLS policy does not apply to table owner when RLS enabled. SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; SELECT * FROM document; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book 11 | 33 | 1 | regress_rls_carol | hoge SELECT * FROM category; cid | cname -----+----------------- 11 | novel 22 | science fiction 33 | technology 44 | manga -- RLS policy does not apply to table owner when RLS disabled. SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO OFF; SELECT * FROM document; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book 11 | 33 | 1 | regress_rls_carol | hoge SELECT * FROM category; cid | cname -----+----------------- 11 | novel 22 | science fiction 33 | technology 44 | manga -- -- Table inheritance and RLS policy -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; CREATE TABLE t1 (a int, junk1 text, b text); ALTER TABLE t1 DROP COLUMN junk1; -- just a disturbing factor GRANT ALL ON t1 TO public; COPY t1 FROM stdin; CREATE TABLE t2 (c float) INHERITS (t1); GRANT ALL ON t2 TO public; COPY t2 FROM stdin; CREATE TABLE t3 (c text, b text, a int); ALTER TABLE t3 INHERIT t1; GRANT ALL ON t3 TO public; COPY t3(a,b,c) FROM stdin; CREATE POLICY p1 ON t1 FOR ALL TO PUBLIC USING (a % 2 = 0); -- be even number CREATE POLICY p2 ON t2 FOR ALL TO PUBLIC USING (a % 2 = 1); -- be odd number ALTER TABLE t1 ENABLE ROW LEVEL SECURITY; ALTER TABLE t2 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM t1; a | b ---+----- 2 | bbb 4 | dad 2 | bcd 4 | def 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: ((a % 2) = 0) -> Seq Scan on t2 t1_2 Filter: ((a % 2) = 0) -> Seq Scan on t3 t1_3 Filter: ((a % 2) = 0) SELECT * FROM t1 WHERE f_leak(b); NOTICE: f_leak => bbb NOTICE: f_leak => dad NOTICE: f_leak => bcd NOTICE: f_leak => def NOTICE: f_leak => yyy a | b ---+----- 2 | bbb 4 | dad 2 | bcd 4 | def 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -- reference to system column SELECT ctid, * FROM t1; ctid | a | b -------+---+----- (0,2) | 2 | bbb (0,4) | 4 | dad (0,2) | 2 | bcd (0,4) | 4 | def (0,2) | 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT *, t1 FROM t1; --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: ((a % 2) = 0) -> Seq Scan on t2 t1_2 Filter: ((a % 2) = 0) -> Seq Scan on t3 t1_3 Filter: ((a % 2) = 0) -- reference to whole-row reference SELECT *, t1 FROM t1; a | b | t1 ---+-----+--------- 2 | bbb | (2,bbb) 4 | dad | (4,dad) 2 | bcd | (2,bcd) 4 | def | (4,def) 2 | yyy | (2,yyy) EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT *, t1 FROM t1; --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: ((a % 2) = 0) -> Seq Scan on t2 t1_2 Filter: ((a % 2) = 0) -> Seq Scan on t3 t1_3 Filter: ((a % 2) = 0) -- for share/update lock SELECT * FROM t1 FOR SHARE; a | b ---+----- 2 | bbb 4 | dad 2 | bcd 4 | def 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 FOR SHARE; --- QUERY PLAN --- LockRows -> Append -> Seq Scan on t1 t1_1 Filter: ((a % 2) = 0) -> Seq Scan on t2 t1_2 Filter: ((a % 2) = 0) -> Seq Scan on t3 t1_3 Filter: ((a % 2) = 0) SELECT * FROM t1 WHERE f_leak(b) FOR SHARE; NOTICE: f_leak => bbb NOTICE: f_leak => dad NOTICE: f_leak => bcd NOTICE: f_leak => def NOTICE: f_leak => yyy a | b ---+----- 2 | bbb 4 | dad 2 | bcd 4 | def 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b) FOR SHARE; --- QUERY PLAN --- LockRows -> Append -> Seq Scan on t1 t1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -- union all query SELECT a, b, ctid FROM t2 UNION ALL SELECT a, b, ctid FROM t3; a | b | ctid ---+-----+------- 1 | abc | (0,1) 3 | cde | (0,3) 1 | xxx | (0,1) 2 | yyy | (0,2) 3 | zzz | (0,3) EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT a, b, ctid FROM t2 UNION ALL SELECT a, b, ctid FROM t3; --- QUERY PLAN --- Append -> Seq Scan on t2 Filter: ((a % 2) = 1) -> Seq Scan on t3 -- superuser is allowed to bypass RLS checks RESET SESSION AUTHORIZATION; SET row_security TO OFF; SELECT * FROM t1 WHERE f_leak(b); NOTICE: f_leak => aba NOTICE: f_leak => bbb NOTICE: f_leak => ccc NOTICE: f_leak => dad NOTICE: f_leak => abc NOTICE: f_leak => bcd NOTICE: f_leak => cde NOTICE: f_leak => def NOTICE: f_leak => xxx NOTICE: f_leak => yyy NOTICE: f_leak => zzz a | b ---+----- 1 | aba 2 | bbb 3 | ccc 4 | dad 1 | abc 2 | bcd 3 | cde 4 | def 1 | xxx 2 | yyy 3 | zzz EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: f_leak(b) -> Seq Scan on t2 t1_2 Filter: f_leak(b) -> Seq Scan on t3 t1_3 Filter: f_leak(b) -- non-superuser with bypass privilege can bypass RLS policy when disabled SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; SELECT * FROM t1 WHERE f_leak(b); NOTICE: f_leak => aba NOTICE: f_leak => bbb NOTICE: f_leak => ccc NOTICE: f_leak => dad NOTICE: f_leak => abc NOTICE: f_leak => bcd NOTICE: f_leak => cde NOTICE: f_leak => def NOTICE: f_leak => xxx NOTICE: f_leak => yyy NOTICE: f_leak => zzz a | b ---+----- 1 | aba 2 | bbb 3 | ccc 4 | dad 1 | abc 2 | bcd 3 | cde 4 | def 1 | xxx 2 | yyy 3 | zzz EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: f_leak(b) -> Seq Scan on t2 t1_2 Filter: f_leak(b) -> Seq Scan on t3 t1_3 Filter: f_leak(b) -- -- Hyper Tables -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE hyper_document ( did int, cid int, dlevel int not null, dauthor name, dtitle text ); GRANT ALL ON hyper_document TO public; SELECT public.create_hypertable('hyper_document', 'did', chunk_time_interval=>2); create_hypertable ----------------------------------------- (2,regress_rls_schema,hyper_document,t) INSERT INTO hyper_document VALUES ( 1, 11, 1, 'regress_rls_bob', 'my first novel'), ( 2, 11, 2, 'regress_rls_bob', 'my second novel'), ( 3, 99, 2, 'regress_rls_bob', 'my science textbook'), ( 4, 55, 1, 'regress_rls_bob', 'my first satire'), ( 5, 99, 2, 'regress_rls_bob', 'my history book'), ( 6, 11, 1, 'regress_rls_carol', 'great science fiction'), ( 7, 99, 2, 'regress_rls_carol', 'great technology book'), ( 8, 55, 2, 'regress_rls_carol', 'great satire'), ( 9, 11, 1, 'regress_rls_dave', 'awesome science fiction'), (10, 99, 2, 'regress_rls_dave', 'awesome technology book'); ALTER TABLE hyper_document ENABLE ROW LEVEL SECURITY; GRANT ALL ON _timescaledb_internal._hyper_2_9_chunk TO public; -- Create policy on parent -- user's security level must be higher than or equal to document's CREATE POLICY pp1 ON hyper_document AS PERMISSIVE USING (dlevel <= (SELECT seclv FROM uaccount WHERE pguser = current_user)); -- Dave is only allowed to see cid < 55 CREATE POLICY pp1r ON hyper_document AS RESTRICTIVE TO regress_rls_dave USING (cid < 55); \d+ hyper_document Table "regress_rls_schema.hyper_document" Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- did | integer | | not null | | plain | | cid | integer | | | | plain | | dlevel | integer | | not null | | plain | | dauthor | name | | | | plain | | dtitle | text | | | | extended | | Indexes: "hyper_document_did_idx" btree (did DESC) Policies: POLICY "pp1" USING ((dlevel <= ( SELECT uaccount.seclv FROM uaccount WHERE (uaccount.pguser = CURRENT_USER)))) POLICY "pp1r" AS RESTRICTIVE TO regress_rls_dave USING ((cid < 55)) Child tables: _timescaledb_internal._hyper_2_10_chunk, _timescaledb_internal._hyper_2_11_chunk, _timescaledb_internal._hyper_2_12_chunk, _timescaledb_internal._hyper_2_13_chunk, _timescaledb_internal._hyper_2_14_chunk, _timescaledb_internal._hyper_2_9_chunk SELECT * FROM pg_policies WHERE schemaname = 'regress_rls_schema' AND tablename like '%hyper_document%' ORDER BY policyname; schemaname | tablename | policyname | permissive | roles | cmd | qual | with_check --------------------+----------------+------------+-------------+--------------------+-----+--------------------------------------------+------------ regress_rls_schema | hyper_document | pp1 | PERMISSIVE | {public} | ALL | (dlevel <= ( SELECT uaccount.seclv +| | | | | | | FROM uaccount +| | | | | | | WHERE (uaccount.pguser = CURRENT_USER))) | regress_rls_schema | hyper_document | pp1r | RESTRICTIVE | {regress_rls_dave} | ALL | (cid < 55) | -- viewpoint from regress_rls_bob SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my first satire NOTICE: f_leak => great science fiction NOTICE: f_leak => awesome science fiction did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 4 | 55 | 1 | regress_rls_bob | my first satire 6 | 11 | 1 | regress_rls_carol | great science fiction 9 | 11 | 1 | regress_rls_dave | awesome science fiction EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on hyper_document hyper_document_1 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -- viewpoint from regress_rls_carol SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science textbook NOTICE: f_leak => my first satire NOTICE: f_leak => my history book NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great satire NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire 9 | 11 | 1 | regress_rls_dave | awesome science fiction 10 | 99 | 2 | regress_rls_dave | awesome technology book EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on hyper_document hyper_document_1 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -- viewpoint from regress_rls_dave SET SESSION AUTHORIZATION regress_rls_dave; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => great science fiction NOTICE: f_leak => awesome science fiction did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 6 | 11 | 1 | regress_rls_carol | great science fiction 9 | 11 | 1 | regress_rls_dave | awesome science fiction EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on hyper_document hyper_document_1 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -- pp1 ERROR INSERT INTO hyper_document VALUES (1, 11, 5, 'regress_rls_dave', 'testing pp1'); -- fail ERROR: new row violates row-level security policy for table "hyper_document" -- pp1r ERROR INSERT INTO hyper_document VALUES (1, 99, 1, 'regress_rls_dave', 'testing pp1r'); -- fail ERROR: new row violates row-level security policy "pp1r" for table "hyper_document" -- Show that RLS policy does not apply for direct inserts to children -- This should fail with RLS POLICY pp1r violation. INSERT INTO hyper_document VALUES (1, 55, 1, 'regress_rls_dave', 'testing RLS with hypertables'); -- fail ERROR: new row violates row-level security policy "pp1r" for table "hyper_document" -- But this should succeed. INSERT INTO _timescaledb_internal._hyper_2_9_chunk VALUES (1, 55, 1, 'regress_rls_dave', 'testing RLS with hypertables'); -- success -- We still cannot see the row using the parent SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => great science fiction NOTICE: f_leak => awesome science fiction did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 6 | 11 | 1 | regress_rls_carol | great science fiction 9 | 11 | 1 | regress_rls_dave | awesome science fiction -- But we can if we look directly SELECT * FROM _timescaledb_internal._hyper_2_9_chunk WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => my first novel NOTICE: f_leak => testing RLS with hypertables did | cid | dlevel | dauthor | dtitle -----+-----+--------+------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables -- Turn on RLS and create policy on child to show RLS is checked before constraints SET SESSION AUTHORIZATION regress_rls_alice; ALTER TABLE _timescaledb_internal._hyper_2_9_chunk ENABLE ROW LEVEL SECURITY; CREATE POLICY pp3 ON _timescaledb_internal._hyper_2_9_chunk AS RESTRICTIVE USING (cid < 55); -- This should fail with RLS violation now. SET SESSION AUTHORIZATION regress_rls_dave; INSERT INTO _timescaledb_internal._hyper_2_9_chunk VALUES (1, 55, 1, 'regress_rls_dave', 'testing RLS with hypertables - round 2'); -- fail ERROR: new row violates row-level security policy for table "_hyper_2_9_chunk" -- And now we cannot see directly into the partition either, due to RLS SELECT * FROM _timescaledb_internal._hyper_2_9_chunk WHERE f_leak(dtitle) ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+---------+-------- -- The parent looks same as before -- viewpoint from regress_rls_dave SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => great science fiction NOTICE: f_leak => awesome science fiction did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 6 | 11 | 1 | regress_rls_carol | great science fiction 9 | 11 | 1 | regress_rls_dave | awesome science fiction EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on hyper_document hyper_document_1 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -- viewpoint from regress_rls_carol SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => my first novel NOTICE: f_leak => testing RLS with hypertables NOTICE: f_leak => my second novel NOTICE: f_leak => my science textbook NOTICE: f_leak => my first satire NOTICE: f_leak => my history book NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great satire NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire 9 | 11 | 1 | regress_rls_dave | awesome science fiction 10 | 99 | 2 | regress_rls_dave | awesome technology book EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on hyper_document hyper_document_1 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -- only owner can change policies ALTER POLICY pp1 ON hyper_document USING (true); --fail ERROR: must be owner of table hyper_document DROP POLICY pp1 ON hyper_document; --fail ERROR: must be owner of relation hyper_document SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY pp1 ON hyper_document USING (dauthor = current_user); -- viewpoint from regress_rls_bob again SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science textbook NOTICE: f_leak => my first satire NOTICE: f_leak => my history book did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+--------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book -- viewpoint from rls_regres_carol again SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great satire did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+----------------------- 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 -> Seq Scan on hyper_document hyper_document_1 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -- database superuser does bypass RLS policy when enabled RESET SESSION AUTHORIZATION; SET row_security TO ON; SELECT * FROM hyper_document ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire 9 | 11 | 1 | regress_rls_dave | awesome science fiction 10 | 99 | 2 | regress_rls_dave | awesome technology book SELECT * FROM _timescaledb_internal._hyper_2_9_chunk ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables -- database non-superuser with bypass privilege can bypass RLS policy when disabled SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; SELECT * FROM hyper_document ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire 9 | 11 | 1 | regress_rls_dave | awesome science fiction 10 | 99 | 2 | regress_rls_dave | awesome technology book SELECT * FROM _timescaledb_internal._hyper_2_9_chunk ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables -- RLS policy does not apply to table owner when RLS enabled. SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; SELECT * FROM hyper_document ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire 9 | 11 | 1 | regress_rls_dave | awesome science fiction 10 | 99 | 2 | regress_rls_dave | awesome technology book SELECT * FROM _timescaledb_internal._hyper_2_9_chunk ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables -- When RLS disabled, other users get ERROR. SET SESSION AUTHORIZATION regress_rls_dave; SET row_security TO OFF; SELECT * FROM hyper_document ORDER BY did, cid; ERROR: query would be affected by row-level security policy for table "hyper_document" SELECT * FROM _timescaledb_internal._hyper_2_9_chunk ORDER BY did, cid; ERROR: query would be affected by row-level security policy for table "_hyper_2_9_chunk" -- Check behavior with a policy that uses a SubPlan not an InitPlan. SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; CREATE POLICY pp3 ON hyper_document AS RESTRICTIVE USING ((SELECT dlevel <= seclv FROM uaccount WHERE pguser = current_user)); SET SESSION AUTHORIZATION regress_rls_carol; INSERT INTO hyper_document VALUES (100, 11, 5, 'regress_rls_carol', 'testing pp3'); -- fail ERROR: new row violates row-level security policy "pp3" for table "hyper_document" ----- Dependencies ----- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; CREATE TABLE dependee (x integer, y integer); SELECT public.create_hypertable('dependee', 'x', chunk_time_interval=>2); create_hypertable ----------------------------------- (3,regress_rls_schema,dependee,t) CREATE TABLE dependent (x integer, y integer); SELECT public.create_hypertable('dependent', 'x', chunk_time_interval=>2); create_hypertable ------------------------------------ (4,regress_rls_schema,dependent,t) CREATE POLICY d1 ON dependent FOR ALL TO PUBLIC USING (x = (SELECT d.x FROM dependee d WHERE d.y = y)); DROP TABLE dependee; -- Should fail without CASCADE due to dependency on row security qual? ERROR: cannot drop table dependee because other objects depend on it DETAIL: policy d1 on table dependent depends on table dependee HINT: Use DROP ... CASCADE to drop the dependent objects too. DROP TABLE dependee CASCADE; NOTICE: drop cascades to policy d1 on table dependent EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM dependent; -- After drop, should be unqualified --- QUERY PLAN --- Seq Scan on dependent ----- RECURSION ---- -- -- Simple recursion -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE rec1 (x integer, y integer); SELECT public.create_hypertable('rec1', 'x', chunk_time_interval=>2); create_hypertable ------------------------------- (5,regress_rls_schema,rec1,t) CREATE POLICY r1 ON rec1 USING (x = (SELECT r.x FROM rec1 r WHERE y = r.y)); ALTER TABLE rec1 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rec1; -- fail, direct recursion ERROR: infinite recursion detected in policy for relation "rec1" -- -- Mutual recursion -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE rec2 (a integer, b integer); SELECT public.create_hypertable('rec2', 'x', chunk_time_interval=>2); ERROR: column "x" does not exist ALTER POLICY r1 ON rec1 USING (x = (SELECT a FROM rec2 WHERE b = y)); CREATE POLICY r2 ON rec2 USING (a = (SELECT x FROM rec1 WHERE y = b)); ALTER TABLE rec2 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rec1; -- fail, mutual recursion ERROR: infinite recursion detected in policy for relation "rec1" -- -- Mutual recursion via views -- SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW rec1v AS SELECT * FROM rec1; CREATE VIEW rec2v AS SELECT * FROM rec2; SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY r1 ON rec1 USING (x = (SELECT a FROM rec2v WHERE b = y)); ALTER POLICY r2 ON rec2 USING (a = (SELECT x FROM rec1v WHERE y = b)); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rec1; -- fail, mutual recursion via views ERROR: infinite recursion detected in policy for relation "rec1" -- -- Mutual recursion via .s.b views -- SET SESSION AUTHORIZATION regress_rls_bob; \set VERBOSITY terse \\ -- suppress cascade details DROP VIEW rec1v, rec2v CASCADE; NOTICE: drop cascades to 2 other objects \set VERBOSITY default CREATE VIEW rec1v WITH (security_barrier) AS SELECT * FROM rec1; CREATE VIEW rec2v WITH (security_barrier) AS SELECT * FROM rec2; SET SESSION AUTHORIZATION regress_rls_alice; CREATE POLICY r1 ON rec1 USING (x = (SELECT a FROM rec2v WHERE b = y)); CREATE POLICY r2 ON rec2 USING (a = (SELECT x FROM rec1v WHERE y = b)); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rec1; -- fail, mutual recursion via s.b. views ERROR: infinite recursion detected in policy for relation "rec1" -- -- recursive RLS and VIEWs in policy -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE s1 (a int, b text); SELECT public.create_hypertable('s1', 'a', chunk_time_interval=>2); create_hypertable ----------------------------- (6,regress_rls_schema,s1,t) INSERT INTO s1 (SELECT x, md5(x::text) FROM generate_series(-10,10) x); CREATE TABLE s2 (x int, y text); SELECT public.create_hypertable('s2', 'x', chunk_time_interval=>2); create_hypertable ----------------------------- (7,regress_rls_schema,s2,t) INSERT INTO s2 (SELECT x, md5(x::text) FROM generate_series(-6,6) x); GRANT SELECT ON s1, s2 TO regress_rls_bob; CREATE POLICY p1 ON s1 USING (a in (select x from s2 where y like '%2f%')); CREATE POLICY p2 ON s2 USING (x in (select a from s1 where b like '%22%')); CREATE POLICY p3 ON s1 FOR INSERT WITH CHECK (a = (SELECT a FROM s1)); ALTER TABLE s1 ENABLE ROW LEVEL SECURITY; ALTER TABLE s2 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW v2 AS SELECT * FROM s2 WHERE y like '%af%'; SELECT * FROM s1 WHERE f_leak(b); -- fail (infinite recursion) ERROR: infinite recursion detected in policy for relation "s1" INSERT INTO s1 VALUES (1, 'foo'); -- fail (infinite recursion) ERROR: infinite recursion detected in policy for relation "s1" SET SESSION AUTHORIZATION regress_rls_alice; DROP POLICY p3 on s1; ALTER POLICY p2 ON s2 USING (x % 2 = 0); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM s1 WHERE f_leak(b); -- OK NOTICE: f_leak => c81e728d9d4c2f636f067f89cc14862c NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c a | b ---+---------------------------------- 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM only s1 WHERE f_leak(b); --- QUERY PLAN --- Seq Scan on s1 Filter: ((hashed SubPlan 1) AND f_leak(b)) SubPlan 1 -> Append -> Seq Scan on s2 s2_1 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_27_chunk s2_2 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_28_chunk s2_3 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_29_chunk s2_4 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_30_chunk s2_5 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_31_chunk s2_6 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_32_chunk s2_7 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_33_chunk s2_8 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY p1 ON s1 USING (a in (select x from v2)); -- using VIEW in RLS policy SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM s1 WHERE f_leak(b); -- OK NOTICE: f_leak => 0267aaf632e87a63288a08331f22c7c3 NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc a | b ----+---------------------------------- -4 | 0267aaf632e87a63288a08331f22c7c3 6 | 1679091c5a880faf6fb5e6087eb1b2dc EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM s1 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on s1 Chunks excluded during startup: 0 -> Seq Scan on s1 s1_1 Filter: ((hashed SubPlan 1) AND f_leak(b)) SubPlan 1 -> Append -> Seq Scan on s2 s2_1 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_27_chunk s2_2 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_28_chunk s2_3 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_29_chunk s2_4 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_30_chunk s2_5 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_31_chunk s2_6 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_32_chunk s2_7 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_33_chunk s2_8 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_6_16_chunk s1_2 Filter: ((hashed SubPlan 1) AND f_leak(b)) -> Seq Scan on _hyper_6_17_chunk s1_3 Filter: ((hashed SubPlan 1) AND f_leak(b)) -> Seq Scan on _hyper_6_18_chunk s1_4 Filter: ((hashed SubPlan 1) AND f_leak(b)) -> Seq Scan on _hyper_6_19_chunk s1_5 Filter: ((hashed SubPlan 1) AND f_leak(b)) -> Seq Scan on _hyper_6_20_chunk s1_6 Filter: ((hashed SubPlan 1) AND f_leak(b)) -> Seq Scan on _hyper_6_21_chunk s1_7 Filter: ((hashed SubPlan 1) AND f_leak(b)) -> Seq Scan on _hyper_6_22_chunk s1_8 Filter: ((hashed SubPlan 1) AND f_leak(b)) -> Seq Scan on _hyper_6_23_chunk s1_9 Filter: ((hashed SubPlan 1) AND f_leak(b)) -> Seq Scan on _hyper_6_24_chunk s1_10 Filter: ((hashed SubPlan 1) AND f_leak(b)) -> Seq Scan on _hyper_6_25_chunk s1_11 Filter: ((hashed SubPlan 1) AND f_leak(b)) -> Seq Scan on _hyper_6_26_chunk s1_12 Filter: ((hashed SubPlan 1) AND f_leak(b)) SELECT (SELECT x FROM s1 LIMIT 1) xx, * FROM s2 WHERE y like '%28%'; xx | x | y ----+----+---------------------------------- -6 | -6 | 596a3d04481816330f07e4f97510c28f -4 | -4 | 0267aaf632e87a63288a08331f22c7c3 2 | 2 | c81e728d9d4c2f636f067f89cc14862c EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT (SELECT x FROM s1 LIMIT 1) xx, * FROM s2 WHERE y like '%28%'; --- QUERY PLAN --- Result -> Append -> Seq Scan on s2 s2_1 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_27_chunk s2_2 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_28_chunk s2_3 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_29_chunk s2_4 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_30_chunk s2_5 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_31_chunk s2_6 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_32_chunk s2_7 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_33_chunk s2_8 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) SubPlan 2 -> Limit -> Result -> Custom Scan (ChunkAppend) on s1 -> Seq Scan on s1 s1_1 Filter: (hashed SubPlan 1) SubPlan 1 -> Append -> Seq Scan on s2 s2_10 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_27_chunk s2_11 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_28_chunk s2_12 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_29_chunk s2_13 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_30_chunk s2_14 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_31_chunk s2_15 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_32_chunk s2_16 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_33_chunk s2_17 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_6_16_chunk s1_2 Filter: (hashed SubPlan 1) -> Seq Scan on _hyper_6_17_chunk s1_3 Filter: (hashed SubPlan 1) -> Seq Scan on _hyper_6_18_chunk s1_4 Filter: (hashed SubPlan 1) -> Seq Scan on _hyper_6_19_chunk s1_5 Filter: (hashed SubPlan 1) -> Seq Scan on _hyper_6_20_chunk s1_6 Filter: (hashed SubPlan 1) -> Seq Scan on _hyper_6_21_chunk s1_7 Filter: (hashed SubPlan 1) -> Seq Scan on _hyper_6_22_chunk s1_8 Filter: (hashed SubPlan 1) -> Seq Scan on _hyper_6_23_chunk s1_9 Filter: (hashed SubPlan 1) -> Seq Scan on _hyper_6_24_chunk s1_10 Filter: (hashed SubPlan 1) -> Seq Scan on _hyper_6_25_chunk s1_11 Filter: (hashed SubPlan 1) -> Seq Scan on _hyper_6_26_chunk s1_12 Filter: (hashed SubPlan 1) SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY p2 ON s2 USING (x in (select a from s1 where b like '%d2%')); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM s1 WHERE f_leak(b); -- fail (infinite recursion via view) ERROR: infinite recursion detected in policy for relation "s1" -- prepared statement with regress_rls_alice privilege PREPARE p1(int) AS SELECT * FROM t1 WHERE a <= $1; EXECUTE p1(2); a | b ---+----- 2 | bbb 2 | bcd 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE p1(2); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: ((a <= 2) AND ((a % 2) = 0)) -> Seq Scan on t2 t1_2 Filter: ((a <= 2) AND ((a % 2) = 0)) -> Seq Scan on t3 t1_3 Filter: ((a <= 2) AND ((a % 2) = 0)) -- superuser is allowed to bypass RLS checks RESET SESSION AUTHORIZATION; SET row_security TO OFF; SELECT * FROM t1 WHERE f_leak(b); NOTICE: f_leak => aba NOTICE: f_leak => bbb NOTICE: f_leak => ccc NOTICE: f_leak => dad NOTICE: f_leak => abc NOTICE: f_leak => bcd NOTICE: f_leak => cde NOTICE: f_leak => def NOTICE: f_leak => xxx NOTICE: f_leak => yyy NOTICE: f_leak => zzz a | b ---+----- 1 | aba 2 | bbb 3 | ccc 4 | dad 1 | abc 2 | bcd 3 | cde 4 | def 1 | xxx 2 | yyy 3 | zzz EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: f_leak(b) -> Seq Scan on t2 t1_2 Filter: f_leak(b) -> Seq Scan on t3 t1_3 Filter: f_leak(b) -- plan cache should be invalidated EXECUTE p1(2); a | b ---+----- 1 | aba 2 | bbb 1 | abc 2 | bcd 1 | xxx 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE p1(2); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: (a <= 2) -> Seq Scan on t2 t1_2 Filter: (a <= 2) -> Seq Scan on t3 t1_3 Filter: (a <= 2) PREPARE p2(int) AS SELECT * FROM t1 WHERE a = $1; EXECUTE p2(2); a | b ---+----- 2 | bbb 2 | bcd 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE p2(2); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: (a = 2) -> Seq Scan on t2 t1_2 Filter: (a = 2) -> Seq Scan on t3 t1_3 Filter: (a = 2) -- also, case when privilege switch from superuser SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; EXECUTE p2(2); a | b ---+----- 2 | bbb 2 | bcd 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE p2(2); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: ((a = 2) AND ((a % 2) = 0)) -> Seq Scan on t2 t1_2 Filter: ((a = 2) AND ((a % 2) = 0)) -> Seq Scan on t3 t1_3 Filter: ((a = 2) AND ((a % 2) = 0)) -- -- UPDATE / DELETE and Row-level security -- SET SESSION AUTHORIZATION regress_rls_bob; EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t1 SET b = b || b WHERE f_leak(b); --- QUERY PLAN --- Update on t1 Update on t1 t1_1 Update on t2 t1_2 Update on t3 t1_3 -> Result -> Append -> Seq Scan on t1 t1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: (((a % 2) = 0) AND f_leak(b)) UPDATE t1 SET b = b || b WHERE f_leak(b); NOTICE: f_leak => bbb NOTICE: f_leak => dad NOTICE: f_leak => bcd NOTICE: f_leak => def NOTICE: f_leak => yyy EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE only t1 SET b = b || '_updt' WHERE f_leak(b); --- QUERY PLAN --- Update on t1 -> Seq Scan on t1 Filter: (((a % 2) = 0) AND f_leak(b)) UPDATE only t1 SET b = b || '_updt' WHERE f_leak(b); NOTICE: f_leak => bbbbbb NOTICE: f_leak => daddad -- returning clause with system column UPDATE only t1 SET b = b WHERE f_leak(b) RETURNING ctid, *, t1; NOTICE: f_leak => bbbbbb_updt NOTICE: f_leak => daddad_updt ctid | a | b | t1 --------+---+-------------+----------------- (0,9) | 2 | bbbbbb_updt | (2,bbbbbb_updt) (0,10) | 4 | daddad_updt | (4,daddad_updt) UPDATE t1 SET b = b WHERE f_leak(b) RETURNING *; NOTICE: f_leak => bbbbbb_updt NOTICE: f_leak => daddad_updt NOTICE: f_leak => bcdbcd NOTICE: f_leak => defdef NOTICE: f_leak => yyyyyy a | b ---+------------- 2 | bbbbbb_updt 4 | daddad_updt 2 | bcdbcd 4 | defdef 2 | yyyyyy UPDATE t1 SET b = b WHERE f_leak(b) RETURNING ctid, *, t1; NOTICE: f_leak => bbbbbb_updt NOTICE: f_leak => daddad_updt NOTICE: f_leak => bcdbcd NOTICE: f_leak => defdef NOTICE: f_leak => yyyyyy ctid | a | b | t1 --------+---+-------------+----------------- (0,13) | 2 | bbbbbb_updt | (2,bbbbbb_updt) (0,14) | 4 | daddad_updt | (4,daddad_updt) (0,9) | 2 | bcdbcd | (2,bcdbcd) (0,10) | 4 | defdef | (4,defdef) (0,6) | 2 | yyyyyy | (2,yyyyyy) -- updates with from clause EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t2 SET b=t2.b FROM t3 WHERE t2.a = 3 and t3.a = 2 AND f_leak(t2.b) AND f_leak(t3.b); --- QUERY PLAN --- Update on t2 -> Nested Loop -> Seq Scan on t2 Filter: ((a = 3) AND ((a % 2) = 1) AND f_leak(b)) -> Seq Scan on t3 Filter: ((a = 2) AND f_leak(b)) UPDATE t2 SET b=t2.b FROM t3 WHERE t2.a = 3 and t3.a = 2 AND f_leak(t2.b) AND f_leak(t3.b); NOTICE: f_leak => cde NOTICE: f_leak => yyyyyy EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t1 SET b=t1.b FROM t2 WHERE t1.a = 3 and t2.a = 3 AND f_leak(t1.b) AND f_leak(t2.b); --- QUERY PLAN --- Update on t1 Update on t1 t1_1 Update on t2 t1_2 Update on t3 t1_3 -> Nested Loop -> Seq Scan on t2 Filter: ((a = 3) AND ((a % 2) = 1) AND f_leak(b)) -> Append -> Seq Scan on t1 t1_1 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) UPDATE t1 SET b=t1.b FROM t2 WHERE t1.a = 3 and t2.a = 3 AND f_leak(t1.b) AND f_leak(t2.b); NOTICE: f_leak => cde EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t2 SET b=t2.b FROM t1 WHERE t1.a = 3 and t2.a = 3 AND f_leak(t1.b) AND f_leak(t2.b); --- QUERY PLAN --- Update on t2 -> Nested Loop -> Seq Scan on t2 Filter: ((a = 3) AND ((a % 2) = 1) AND f_leak(b)) -> Append -> Seq Scan on t1 t1_1 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) UPDATE t2 SET b=t2.b FROM t1 WHERE t1.a = 3 and t2.a = 3 AND f_leak(t1.b) AND f_leak(t2.b); NOTICE: f_leak => cde -- updates with from clause self join EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t2 t2_1 SET b = t2_2.b FROM t2 t2_2 WHERE t2_1.a = 3 AND t2_2.a = t2_1.a AND t2_2.b = t2_1.b AND f_leak(t2_1.b) AND f_leak(t2_2.b) RETURNING *, t2_1, t2_2; --- QUERY PLAN --- Update on t2 t2_1 -> Nested Loop Join Filter: (t2_1.b = t2_2.b) -> Seq Scan on t2 t2_1 Filter: ((a = 3) AND ((a % 2) = 1) AND f_leak(b)) -> Seq Scan on t2 t2_2 Filter: ((a = 3) AND ((a % 2) = 1) AND f_leak(b)) UPDATE t2 t2_1 SET b = t2_2.b FROM t2 t2_2 WHERE t2_1.a = 3 AND t2_2.a = t2_1.a AND t2_2.b = t2_1.b AND f_leak(t2_1.b) AND f_leak(t2_2.b) RETURNING *, t2_1, t2_2; NOTICE: f_leak => cde NOTICE: f_leak => cde a | b | c | a | b | c | t2_1 | t2_2 ---+-----+-----+---+-----+-----+-------------+------------- 3 | cde | 3.3 | 3 | cde | 3.3 | (3,cde,3.3) | (3,cde,3.3) EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t1 t1_1 SET b = t1_2.b FROM t1 t1_2 WHERE t1_1.a = 4 AND t1_2.a = t1_1.a AND t1_2.b = t1_1.b AND f_leak(t1_1.b) AND f_leak(t1_2.b) RETURNING *, t1_1, t1_2; --- QUERY PLAN --- Update on t1 t1_1 Update on t1 t1_1_1 Update on t2 t1_1_2 Update on t3 t1_1_3 -> Nested Loop Join Filter: (t1_1.b = t1_2.b) -> Append -> Seq Scan on t1 t1_1_1 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_1_2 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_1_3 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) -> Materialize -> Append -> Seq Scan on t1 t1_2_1 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2_2 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_2_3 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) UPDATE t1 t1_1 SET b = t1_2.b FROM t1 t1_2 WHERE t1_1.a = 4 AND t1_2.a = t1_1.a AND t1_2.b = t1_1.b AND f_leak(t1_1.b) AND f_leak(t1_2.b) RETURNING *, t1_1, t1_2; NOTICE: f_leak => daddad_updt NOTICE: f_leak => daddad_updt NOTICE: f_leak => defdef NOTICE: f_leak => defdef a | b | a | b | t1_1 | t1_2 ---+-------------+---+-------------+-----------------+----------------- 4 | daddad_updt | 4 | daddad_updt | (4,daddad_updt) | (4,daddad_updt) 4 | defdef | 4 | defdef | (4,defdef) | (4,defdef) RESET SESSION AUTHORIZATION; SET row_security TO OFF; SELECT * FROM t1 ORDER BY a,b; a | b ---+------------- 1 | aba 1 | abc 1 | xxx 2 | bbbbbb_updt 2 | bcdbcd 2 | yyyyyy 3 | ccc 3 | cde 3 | zzz 4 | daddad_updt 4 | defdef SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; EXPLAIN (BUFFERS OFF, COSTS OFF) DELETE FROM only t1 WHERE f_leak(b); --- QUERY PLAN --- Delete on t1 -> Seq Scan on t1 Filter: (((a % 2) = 0) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) DELETE FROM t1 WHERE f_leak(b); --- QUERY PLAN --- Delete on t1 Delete on t1 t1_1 Delete on t2 t1_2 Delete on t3 t1_3 -> Append -> Seq Scan on t1 t1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: (((a % 2) = 0) AND f_leak(b)) DELETE FROM only t1 WHERE f_leak(b) RETURNING ctid, *, t1; NOTICE: f_leak => bbbbbb_updt NOTICE: f_leak => daddad_updt ctid | a | b | t1 --------+---+-------------+----------------- (0,13) | 2 | bbbbbb_updt | (2,bbbbbb_updt) (0,15) | 4 | daddad_updt | (4,daddad_updt) DELETE FROM t1 WHERE f_leak(b) RETURNING ctid, *, t1; NOTICE: f_leak => bcdbcd NOTICE: f_leak => defdef NOTICE: f_leak => yyyyyy ctid | a | b | t1 --------+---+--------+------------ (0,9) | 2 | bcdbcd | (2,bcdbcd) (0,13) | 4 | defdef | (4,defdef) (0,6) | 2 | yyyyyy | (2,yyyyyy) -- -- S.b. view on top of Row-level security -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE b1 (a int, b text); SELECT public.create_hypertable('b1', 'a', chunk_time_interval=>2); create_hypertable ----------------------------- (8,regress_rls_schema,b1,t) INSERT INTO b1 (SELECT x, md5(x::text) FROM generate_series(-10,10) x); CREATE POLICY p1 ON b1 USING (a % 2 = 0); ALTER TABLE b1 ENABLE ROW LEVEL SECURITY; GRANT ALL ON b1 TO regress_rls_bob; SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW bv1 WITH (security_barrier) AS SELECT * FROM b1 WHERE a > 0 WITH CHECK OPTION; GRANT ALL ON bv1 TO regress_rls_carol; SET SESSION AUTHORIZATION regress_rls_carol; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM bv1 WHERE f_leak(b); --- QUERY PLAN --- Subquery Scan on bv1 Filter: f_leak(bv1.b) -> Append -> Seq Scan on b1 b1_1 Filter: ((a > 0) AND ((a % 2) = 0)) -> Index Scan using _hyper_8_39_chunk_b1_a_idx on _hyper_8_39_chunk b1_2 Index Cond: (a > 0) Filter: ((a % 2) = 0) -> Index Scan using _hyper_8_40_chunk_b1_a_idx on _hyper_8_40_chunk b1_3 Index Cond: (a > 0) Filter: ((a % 2) = 0) -> Index Scan using _hyper_8_41_chunk_b1_a_idx on _hyper_8_41_chunk b1_4 Index Cond: (a > 0) Filter: ((a % 2) = 0) -> Index Scan using _hyper_8_42_chunk_b1_a_idx on _hyper_8_42_chunk b1_5 Index Cond: (a > 0) Filter: ((a % 2) = 0) -> Index Scan using _hyper_8_43_chunk_b1_a_idx on _hyper_8_43_chunk b1_6 Index Cond: (a > 0) Filter: ((a % 2) = 0) -> Index Scan using _hyper_8_44_chunk_b1_a_idx on _hyper_8_44_chunk b1_7 Index Cond: (a > 0) Filter: ((a % 2) = 0) SELECT * FROM bv1 WHERE f_leak(b); NOTICE: f_leak => c81e728d9d4c2f636f067f89cc14862c NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc NOTICE: f_leak => c9f0f895fb98ab9159f51fd0297e236d NOTICE: f_leak => d3d9446802a44259755d38e6d163e820 a | b ----+---------------------------------- 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 INSERT INTO bv1 VALUES (-1, 'xxx'); -- should fail view WCO ERROR: new row violates row-level security policy for table "b1" INSERT INTO bv1 VALUES (11, 'xxx'); -- should fail RLS check ERROR: new row violates row-level security policy for table "b1" INSERT INTO bv1 VALUES (12, 'xxx'); -- ok EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE bv1 SET b = 'yyy' WHERE a = 4 AND f_leak(b); --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Update on b1 Update on _hyper_8_41_chunk b1_1 -> Result -> Custom Scan (ChunkAppend) on b1 Chunks excluded during startup: 0 -> Index Scan using _hyper_8_41_chunk_b1_a_idx on _hyper_8_41_chunk b1_1 Index Cond: ((a > 0) AND (a = 4)) Filter: (((a % 2) = 0) AND f_leak(b)) UPDATE bv1 SET b = 'yyy' WHERE a = 4 AND f_leak(b); NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c EXPLAIN (BUFFERS OFF, COSTS OFF) DELETE FROM bv1 WHERE a = 6 AND f_leak(b); --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Delete on b1 Delete on _hyper_8_42_chunk b1_1 -> Custom Scan (ChunkAppend) on b1 Chunks excluded during startup: 0 -> Index Scan using _hyper_8_42_chunk_b1_a_idx on _hyper_8_42_chunk b1_1 Index Cond: ((a > 0) AND (a = 6)) Filter: (((a % 2) = 0) AND f_leak(b)) DELETE FROM bv1 WHERE a = 6 AND f_leak(b); NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM b1; a | b -----+---------------------------------- -10 | 1b0fd9efa5279c4203b7c70233f86dbf -9 | 252e691406782824eec43d7eadc3d256 -8 | a8d2ec85eaf98407310b72eb73dda247 -7 | 74687a12d3915d3c4d83f1af7b3683d5 -6 | 596a3d04481816330f07e4f97510c28f -5 | 47c1b025fa18ea96c33fbb6718688c0f -4 | 0267aaf632e87a63288a08331f22c7c3 -3 | b3149ecea4628efd23d2f86e5a723472 -2 | 5d7b9adcbe1c629ec722529dd12e5129 -1 | 6bb61e3b7bce0931da574d19d1d82c88 0 | cfcd208495d565ef66e7dff9f98764da 1 | c4ca4238a0b923820dcc509a6f75849b 2 | c81e728d9d4c2f636f067f89cc14862c 3 | eccbc87e4b5ce2fe28308fd9f2a7baf3 5 | e4da3b7fbbce2345d7772b0674a318d5 4 | yyy 7 | 8f14e45fceea167a5a36dedd4bea2543 8 | c9f0f895fb98ab9159f51fd0297e236d 9 | 45c48cce2e2d7fbdea1afc51c7c6ad26 10 | d3d9446802a44259755d38e6d163e820 12 | xxx -- -- INSERT ... ON CONFLICT DO UPDATE and Row-level security -- SET SESSION AUTHORIZATION regress_rls_alice; DROP POLICY p1 ON document; DROP POLICY p1r ON document; CREATE POLICY p1 ON document FOR SELECT USING (true); CREATE POLICY p2 ON document FOR INSERT WITH CHECK (dauthor = current_user); CREATE POLICY p3 ON document FOR UPDATE USING (cid = (SELECT cid from category WHERE cname = 'novel')) WITH CHECK (dauthor = current_user); SET SESSION AUTHORIZATION regress_rls_bob; -- Exists... SELECT * FROM document WHERE did = 2; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+----------------- 2 | 11 | 2 | regress_rls_bob | my second novel -- ...so violates actual WITH CHECK OPTION within UPDATE (not INSERT, since -- alternative UPDATE path happens to be taken): INSERT INTO document VALUES (2, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_carol', 'my first novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, dauthor = EXCLUDED.dauthor; ERROR: new row violates row-level security policy for table "document" -- Violates USING qual for UPDATE policy p3. -- -- UPDATE path is taken, but UPDATE fails purely because *existing* row to be -- updated is not a "novel"/cid 11 (row is not leaked, even though we have -- SELECT privileges sufficient to see the row in this instance): INSERT INTO document VALUES (33, 22, 1, 'regress_rls_bob', 'okay science fiction'); -- preparation for next statement INSERT INTO document VALUES (33, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'Some novel, replaces sci-fi') -- takes UPDATE path ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle; ERROR: new row violates row-level security policy (USING expression) for table "document" -- Fine (we UPDATE, since INSERT WCOs and UPDATE security barrier quals + WCOs -- not violated): INSERT INTO document VALUES (2, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'my first novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+---------------- 2 | 11 | 2 | regress_rls_bob | my first novel -- Fine (we INSERT, so "cid = 33" ("technology") isn't evaluated): INSERT INTO document VALUES (78, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'some technology novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33 RETURNING *; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+----------------------- 78 | 11 | 1 | regress_rls_bob | some technology novel -- Fine (same query, but we UPDATE, so "cid = 33", ("technology") is not the -- case in respect of *existing* tuple): INSERT INTO document VALUES (78, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'some technology novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33 RETURNING *; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+----------------------- 78 | 33 | 1 | regress_rls_bob | some technology novel -- Same query a third time, but now fails due to existing tuple finally not -- passing quals: INSERT INTO document VALUES (78, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'some technology novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33 RETURNING *; ERROR: new row violates row-level security policy (USING expression) for table "document" -- Don't fail just because INSERT doesn't satisfy WITH CHECK option that -- originated as a barrier/USING() qual from the UPDATE. Note that the UPDATE -- path *isn't* taken, and so UPDATE-related policy does not apply: INSERT INTO document VALUES (79, (SELECT cid from category WHERE cname = 'technology'), 1, 'regress_rls_bob', 'technology book, can only insert') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+---------------------------------- 79 | 33 | 1 | regress_rls_bob | technology book, can only insert -- But this time, the same statement fails, because the UPDATE path is taken, -- and updating the row just inserted falls afoul of security barrier qual -- (enforced as WCO) -- what we might have updated target tuple to is -- irrelevant, in fact. INSERT INTO document VALUES (79, (SELECT cid from category WHERE cname = 'technology'), 1, 'regress_rls_bob', 'technology book, can only insert') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; ERROR: new row violates row-level security policy (USING expression) for table "document" -- Test default USING qual enforced as WCO SET SESSION AUTHORIZATION regress_rls_alice; DROP POLICY p1 ON document; DROP POLICY p2 ON document; DROP POLICY p3 ON document; CREATE POLICY p3_with_default ON document FOR UPDATE USING (cid = (SELECT cid from category WHERE cname = 'novel')); SET SESSION AUTHORIZATION regress_rls_bob; -- Just because WCO-style enforcement of USING quals occurs with -- existing/target tuple does not mean that the implementation can be allowed -- to fail to also enforce this qual against the final tuple appended to -- relation (since in the absence of an explicit WCO, this is also interpreted -- as an UPDATE/ALL WCO in general). -- -- UPDATE path is taken here (fails due to existing tuple). Note that this is -- not reported as a "USING expression", because it's an RLS UPDATE check that originated as -- a USING qual for the purposes of RLS in general, as opposed to an explicit -- USING qual that is ordinarily a security barrier. We leave it up to the -- UPDATE to make this fail: INSERT INTO document VALUES (79, (SELECT cid from category WHERE cname = 'technology'), 1, 'regress_rls_bob', 'technology book, can only insert') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; ERROR: new row violates row-level security policy for table "document" -- UPDATE path is taken here. Existing tuple passes, since it's cid -- corresponds to "novel", but default USING qual is enforced against -- post-UPDATE tuple too (as always when updating with a policy that lacks an -- explicit WCO), and so this fails: INSERT INTO document VALUES (2, (SELECT cid from category WHERE cname = 'technology'), 1, 'regress_rls_bob', 'my first novel') ON CONFLICT (did) DO UPDATE SET cid = EXCLUDED.cid, dtitle = EXCLUDED.dtitle RETURNING *; ERROR: new row violates row-level security policy for table "document" SET SESSION AUTHORIZATION regress_rls_alice; DROP POLICY p3_with_default ON document; -- -- Test ALL policies with ON CONFLICT DO UPDATE (much the same as existing UPDATE -- tests) -- CREATE POLICY p3_with_all ON document FOR ALL USING (cid = (SELECT cid from category WHERE cname = 'novel')) WITH CHECK (dauthor = current_user); SET SESSION AUTHORIZATION regress_rls_bob; -- Fails, since ALL WCO is enforced in insert path: INSERT INTO document VALUES (80, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_carol', 'my first novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33; ERROR: new row violates row-level security policy for table "document" -- Fails, since ALL policy USING qual is enforced (existing, target tuple is in -- violation, since it has the "manga" cid): INSERT INTO document VALUES (4, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'my first novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle; ERROR: new row violates row-level security policy (USING expression) for table "document" -- Fails, since ALL WCO are enforced: INSERT INTO document VALUES (1, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'my first novel') ON CONFLICT (did) DO UPDATE SET dauthor = 'regress_rls_carol'; ERROR: new row violates row-level security policy for table "document" -- -- ROLE/GROUP -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE z1 (a int, b text); SELECT public.create_hypertable('z1', 'a', chunk_time_interval=>2); create_hypertable ----------------------------- (9,regress_rls_schema,z1,t) CREATE TABLE z2 (a int, b text); SELECT public.create_hypertable('z2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (10,regress_rls_schema,z2,t) GRANT SELECT ON z1,z2 TO regress_rls_group1, regress_rls_group2, regress_rls_bob, regress_rls_carol; INSERT INTO z1 VALUES (1, 'aba'), (2, 'bbb'), (3, 'ccc'), (4, 'dad'); CREATE POLICY p1 ON z1 TO regress_rls_group1 USING (a % 2 = 0); CREATE POLICY p2 ON z1 TO regress_rls_group2 USING (a % 2 = 1); ALTER TABLE z1 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM z1 WHERE f_leak(b); NOTICE: f_leak => bbb NOTICE: f_leak => dad a | b ---+----- 2 | bbb 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM z1 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) PREPARE plancache_test AS SELECT * FROM z1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) PREPARE plancache_test2 AS WITH q AS MATERIALIZED (SELECT * FROM z1 WHERE f_leak(b)) SELECT * FROM q,z2; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test2; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 PREPARE plancache_test4 AS WITH q AS (SELECT * FROM z1 WHERE f_leak(b)) SELECT * FROM q,z2; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test4; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 PREPARE plancache_test6 AS WITH q AS NOT MATERIALIZED (SELECT * FROM z1 WHERE f_leak(b)) SELECT * FROM q,z2; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test6; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 PREPARE plancache_test3 AS WITH q AS MATERIALIZED (SELECT * FROM z2) SELECT * FROM q,z1 WHERE f_leak(z1.b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test3; --- QUERY PLAN --- Nested Loop CTE q -> Seq Scan on z2 -> CTE Scan on q -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) PREPARE plancache_test5 AS WITH q AS (SELECT * FROM z2) SELECT * FROM q,z1 WHERE f_leak(z1.b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test5; --- QUERY PLAN --- Nested Loop -> Seq Scan on z2 -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) PREPARE plancache_test7 AS WITH q AS NOT MATERIALIZED (SELECT * FROM z2) SELECT * FROM q,z1 WHERE f_leak(z1.b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test7; --- QUERY PLAN --- Nested Loop -> Seq Scan on z2 -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) SET ROLE regress_rls_group1; SELECT * FROM z1 WHERE f_leak(b); NOTICE: f_leak => bbb NOTICE: f_leak => dad a | b ---+----- 2 | bbb 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM z1 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test2; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test4; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test3; --- QUERY PLAN --- Nested Loop CTE q -> Seq Scan on z2 -> CTE Scan on q -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test5; --- QUERY PLAN --- Nested Loop -> Seq Scan on z2 -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM z1 WHERE f_leak(b); NOTICE: f_leak => aba NOTICE: f_leak => ccc a | b ---+----- 1 | aba 3 | ccc EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM z1 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test2; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test4; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test3; --- QUERY PLAN --- Nested Loop CTE q -> Seq Scan on z2 -> CTE Scan on q -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test5; --- QUERY PLAN --- Nested Loop -> Seq Scan on z2 -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) SET ROLE regress_rls_group2; SELECT * FROM z1 WHERE f_leak(b); NOTICE: f_leak => aba NOTICE: f_leak => ccc a | b ---+----- 1 | aba 3 | ccc EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM z1 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test2; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test4; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test3; --- QUERY PLAN --- Nested Loop CTE q -> Seq Scan on z2 -> CTE Scan on q -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test5; --- QUERY PLAN --- Nested Loop -> Seq Scan on z2 -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) -- -- Views should follow policy for view owner. -- -- View and Table owner are the same. SET SESSION AUTHORIZATION regress_rls_alice; CREATE VIEW rls_view AS SELECT * FROM z1 WHERE f_leak(b); GRANT SELECT ON rls_view TO regress_rls_bob; -- Query as role that is not owner of view or table. Should return all records. SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rls_view; NOTICE: f_leak => aba NOTICE: f_leak => bbb NOTICE: f_leak => ccc NOTICE: f_leak => dad a | b ---+----- 1 | aba 2 | bbb 3 | ccc 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: f_leak(b) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: f_leak(b) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: f_leak(b) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: f_leak(b) -- Query as view/table owner. Should return all records. SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM rls_view; NOTICE: f_leak => aba NOTICE: f_leak => bbb NOTICE: f_leak => ccc NOTICE: f_leak => dad a | b ---+----- 1 | aba 2 | bbb 3 | ccc 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: f_leak(b) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: f_leak(b) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: f_leak(b) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: f_leak(b) DROP VIEW rls_view; -- View and Table owners are different. SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW rls_view AS SELECT * FROM z1 WHERE f_leak(b); GRANT SELECT ON rls_view TO regress_rls_alice; -- Query as role that is not owner of view but is owner of table. -- Should return records based on view owner policies. SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM rls_view; NOTICE: f_leak => bbb NOTICE: f_leak => dad a | b ---+----- 2 | bbb 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -- Query as role that is not owner of table but is owner of view. -- Should return records based on view owner policies. SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rls_view; NOTICE: f_leak => bbb NOTICE: f_leak => dad a | b ---+----- 2 | bbb 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -- Query as role that is not the owner of the table or view without permissions. SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM rls_view; --fail - permission denied. ERROR: permission denied for view rls_view EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --fail - permission denied. ERROR: permission denied for view rls_view -- Query as role that is not the owner of the table or view with permissions. SET SESSION AUTHORIZATION regress_rls_bob; GRANT SELECT ON rls_view TO regress_rls_carol; SELECT * FROM rls_view; NOTICE: f_leak => bbb NOTICE: f_leak => dad a | b ---+----- 2 | bbb 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) SET SESSION AUTHORIZATION regress_rls_bob; DROP VIEW rls_view; -- -- Command specific -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE x1 (a int, b text, c text); SELECT public.create_hypertable('x1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (11,regress_rls_schema,x1,t) GRANT ALL ON x1 TO PUBLIC; INSERT INTO x1 VALUES (1, 'abc', 'regress_rls_bob'), (2, 'bcd', 'regress_rls_bob'), (3, 'cde', 'regress_rls_carol'), (4, 'def', 'regress_rls_carol'), (5, 'efg', 'regress_rls_bob'), (6, 'fgh', 'regress_rls_bob'), (7, 'fgh', 'regress_rls_carol'), (8, 'fgh', 'regress_rls_carol'); CREATE POLICY p0 ON x1 FOR ALL USING (c = current_user); CREATE POLICY p1 ON x1 FOR SELECT USING (a % 2 = 0); CREATE POLICY p2 ON x1 FOR INSERT WITH CHECK (a % 2 = 1); CREATE POLICY p3 ON x1 FOR UPDATE USING (a % 2 = 0); CREATE POLICY p4 ON x1 FOR DELETE USING (a < 8); ALTER TABLE x1 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM x1 WHERE f_leak(b) ORDER BY a ASC; NOTICE: f_leak => abc NOTICE: f_leak => bcd NOTICE: f_leak => def NOTICE: f_leak => efg NOTICE: f_leak => fgh NOTICE: f_leak => fgh a | b | c ---+-----+------------------- 1 | abc | regress_rls_bob 2 | bcd | regress_rls_bob 4 | def | regress_rls_carol 5 | efg | regress_rls_bob 6 | fgh | regress_rls_bob 8 | fgh | regress_rls_carol UPDATE x1 SET b = b || '_updt' WHERE f_leak(b) RETURNING *; NOTICE: f_leak => abc NOTICE: f_leak => bcd NOTICE: f_leak => def NOTICE: f_leak => efg NOTICE: f_leak => fgh NOTICE: f_leak => fgh a | b | c ---+----------+------------------- 1 | abc_updt | regress_rls_bob 2 | bcd_updt | regress_rls_bob 4 | def_updt | regress_rls_carol 5 | efg_updt | regress_rls_bob 6 | fgh_updt | regress_rls_bob 8 | fgh_updt | regress_rls_carol SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM x1 WHERE f_leak(b) ORDER BY a ASC; NOTICE: f_leak => cde NOTICE: f_leak => bcd_updt NOTICE: f_leak => def_updt NOTICE: f_leak => fgh NOTICE: f_leak => fgh_updt NOTICE: f_leak => fgh_updt a | b | c ---+----------+------------------- 2 | bcd_updt | regress_rls_bob 3 | cde | regress_rls_carol 4 | def_updt | regress_rls_carol 6 | fgh_updt | regress_rls_bob 7 | fgh | regress_rls_carol 8 | fgh_updt | regress_rls_carol UPDATE x1 SET b = b || '_updt' WHERE f_leak(b) RETURNING *; NOTICE: f_leak => cde NOTICE: f_leak => bcd_updt NOTICE: f_leak => def_updt NOTICE: f_leak => fgh NOTICE: f_leak => fgh_updt NOTICE: f_leak => fgh_updt a | b | c ---+---------------+------------------- 3 | cde_updt | regress_rls_carol 2 | bcd_updt_updt | regress_rls_bob 4 | def_updt_updt | regress_rls_carol 7 | fgh_updt | regress_rls_carol 6 | fgh_updt_updt | regress_rls_bob 8 | fgh_updt_updt | regress_rls_carol DELETE FROM x1 WHERE f_leak(b) RETURNING *; NOTICE: f_leak => cde_updt NOTICE: f_leak => bcd_updt_updt NOTICE: f_leak => def_updt_updt NOTICE: f_leak => fgh_updt NOTICE: f_leak => fgh_updt_updt NOTICE: f_leak => fgh_updt_updt a | b | c ---+---------------+------------------- 3 | cde_updt | regress_rls_carol 2 | bcd_updt_updt | regress_rls_bob 4 | def_updt_updt | regress_rls_carol 7 | fgh_updt | regress_rls_carol 6 | fgh_updt_updt | regress_rls_bob 8 | fgh_updt_updt | regress_rls_carol -- -- Duplicate Policy Names -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE y1 (a int, b text); SELECT public.create_hypertable('y1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (12,regress_rls_schema,y1,t) INSERT INTO y1 VALUES(1,2); CREATE TABLE y2 (a int, b text); SELECT public.create_hypertable('y2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (13,regress_rls_schema,y2,t) GRANT ALL ON y1, y2 TO regress_rls_bob; CREATE POLICY p1 ON y1 FOR ALL USING (a % 2 = 0); CREATE POLICY p2 ON y1 FOR SELECT USING (a > 2); CREATE POLICY p1 ON y1 FOR SELECT USING (a % 2 = 1); --fail ERROR: policy "p1" for table "y1" already exists CREATE POLICY p1 ON y2 FOR ALL USING (a % 2 = 0); --OK ALTER TABLE y1 ENABLE ROW LEVEL SECURITY; ALTER TABLE y2 ENABLE ROW LEVEL SECURITY; -- -- Expression structure with SBV -- -- Create view as table owner. RLS should NOT be applied. SET SESSION AUTHORIZATION regress_rls_alice; CREATE VIEW rls_sbv WITH (security_barrier) AS SELECT * FROM y1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_sbv WHERE (a = 1); --- QUERY PLAN --- Custom Scan (ChunkAppend) on y1 Chunks excluded during startup: 0 -> Seq Scan on y1 y1_1 Filter: (f_leak(b) AND (a = 1)) -> Index Scan using _hyper_12_57_chunk_y1_a_idx on _hyper_12_57_chunk y1_2 Index Cond: (a = 1) Filter: f_leak(b) DROP VIEW rls_sbv; -- Create view as role that does not own table. RLS should be applied. SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW rls_sbv WITH (security_barrier) AS SELECT * FROM y1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_sbv WHERE (a = 1); --- QUERY PLAN --- Custom Scan (ChunkAppend) on y1 Chunks excluded during startup: 0 -> Seq Scan on y1 y1_1 Filter: ((a = 1) AND ((a > 2) OR ((a % 2) = 0)) AND f_leak(b)) -> Index Scan using _hyper_12_57_chunk_y1_a_idx on _hyper_12_57_chunk y1_2 Index Cond: (a = 1) Filter: (((a > 2) OR ((a % 2) = 0)) AND f_leak(b)) DROP VIEW rls_sbv; -- -- Expression structure -- SET SESSION AUTHORIZATION regress_rls_alice; INSERT INTO y2 (SELECT x, md5(x::text) FROM generate_series(0,20) x); CREATE POLICY p2 ON y2 USING (a % 3 = 0); CREATE POLICY p3 ON y2 USING (a % 4 = 0); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM y2 WHERE f_leak(b); NOTICE: f_leak => cfcd208495d565ef66e7dff9f98764da NOTICE: f_leak => c81e728d9d4c2f636f067f89cc14862c NOTICE: f_leak => eccbc87e4b5ce2fe28308fd9f2a7baf3 NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc NOTICE: f_leak => c9f0f895fb98ab9159f51fd0297e236d NOTICE: f_leak => 45c48cce2e2d7fbdea1afc51c7c6ad26 NOTICE: f_leak => d3d9446802a44259755d38e6d163e820 NOTICE: f_leak => c20ad4d76fe97759aa27a0c99bff6710 NOTICE: f_leak => aab3238922bcc25a6f606eb525ffdc56 NOTICE: f_leak => 9bf31c7ff062936a96d3c8bd1f8f2ff3 NOTICE: f_leak => c74d97b01eae257e44aa9d5bade97baf NOTICE: f_leak => 6f4922f45568161a8cdf4ad2299f6d23 NOTICE: f_leak => 98f13708210194c475687be6106a3b84 a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 3 | eccbc87e4b5ce2fe28308fd9f2a7baf3 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 9 | 45c48cce2e2d7fbdea1afc51c7c6ad26 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 15 | 9bf31c7ff062936a96d3c8bd1f8f2ff3 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM y2 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on y2 Chunks excluded during startup: 0 -> Seq Scan on y2 y2_1 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_58_chunk y2_2 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_59_chunk y2_3 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_60_chunk y2_4 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_61_chunk y2_5 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_62_chunk y2_6 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_63_chunk y2_7 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_64_chunk y2_8 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_65_chunk y2_9 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_66_chunk y2_10 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_67_chunk y2_11 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_68_chunk y2_12 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -- -- Qual push-down of leaky functions, when not referring to table -- SELECT * FROM y2 WHERE f_leak('abc'); NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 3 | eccbc87e4b5ce2fe28308fd9f2a7baf3 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 9 | 45c48cce2e2d7fbdea1afc51c7c6ad26 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 15 | 9bf31c7ff062936a96d3c8bd1f8f2ff3 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM y2 WHERE f_leak('abc'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on y2 Chunks excluded during startup: 0 -> Seq Scan on y2 y2_1 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_58_chunk y2_2 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_59_chunk y2_3 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_60_chunk y2_4 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_61_chunk y2_5 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_62_chunk y2_6 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_63_chunk y2_7 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_64_chunk y2_8 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_65_chunk y2_9 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_66_chunk y2_10 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_67_chunk y2_11 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_68_chunk y2_12 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) CREATE TABLE test_qual_pushdown ( abc text ); INSERT INTO test_qual_pushdown VALUES ('abc'),('def'); SELECT * FROM y2 JOIN test_qual_pushdown ON (b = abc) WHERE f_leak(abc); NOTICE: f_leak => abc NOTICE: f_leak => def a | b | abc ---+---+----- EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM y2 JOIN test_qual_pushdown ON (b = abc) WHERE f_leak(abc); --- QUERY PLAN --- Hash Join Hash Cond: (y2.b = test_qual_pushdown.abc) -> Append -> Seq Scan on y2 y2_1 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_58_chunk y2_2 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_59_chunk y2_3 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_60_chunk y2_4 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_61_chunk y2_5 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_62_chunk y2_6 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_63_chunk y2_7 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_64_chunk y2_8 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_65_chunk y2_9 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_66_chunk y2_10 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_67_chunk y2_11 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_68_chunk y2_12 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Hash -> Seq Scan on test_qual_pushdown Filter: f_leak(abc) SELECT * FROM y2 JOIN test_qual_pushdown ON (b = abc) WHERE f_leak(b); NOTICE: f_leak => cfcd208495d565ef66e7dff9f98764da NOTICE: f_leak => c81e728d9d4c2f636f067f89cc14862c NOTICE: f_leak => eccbc87e4b5ce2fe28308fd9f2a7baf3 NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc NOTICE: f_leak => c9f0f895fb98ab9159f51fd0297e236d NOTICE: f_leak => 45c48cce2e2d7fbdea1afc51c7c6ad26 NOTICE: f_leak => d3d9446802a44259755d38e6d163e820 NOTICE: f_leak => c20ad4d76fe97759aa27a0c99bff6710 NOTICE: f_leak => aab3238922bcc25a6f606eb525ffdc56 NOTICE: f_leak => 9bf31c7ff062936a96d3c8bd1f8f2ff3 NOTICE: f_leak => c74d97b01eae257e44aa9d5bade97baf NOTICE: f_leak => 6f4922f45568161a8cdf4ad2299f6d23 NOTICE: f_leak => 98f13708210194c475687be6106a3b84 a | b | abc ---+---+----- EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM y2 JOIN test_qual_pushdown ON (b = abc) WHERE f_leak(b); --- QUERY PLAN --- Hash Join Hash Cond: (test_qual_pushdown.abc = y2.b) -> Seq Scan on test_qual_pushdown -> Hash -> Custom Scan (ChunkAppend) on y2 Chunks excluded during startup: 0 -> Seq Scan on y2 y2_1 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_58_chunk y2_2 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_59_chunk y2_3 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_60_chunk y2_4 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_61_chunk y2_5 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_62_chunk y2_6 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_63_chunk y2_7 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_64_chunk y2_8 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_65_chunk y2_9 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_66_chunk y2_10 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_67_chunk y2_11 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_68_chunk y2_12 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) DROP TABLE test_qual_pushdown; -- -- Plancache invalidate on user change. -- RESET SESSION AUTHORIZATION; \set VERBOSITY terse \\ -- suppress cascade details DROP TABLE t1 CASCADE; NOTICE: drop cascades to 2 other objects \set VERBOSITY default CREATE TABLE t1 (a integer); SELECT public.create_hypertable('t1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (14,regress_rls_schema,t1,t) GRANT SELECT ON t1 TO regress_rls_bob, regress_rls_carol; CREATE POLICY p1 ON t1 TO regress_rls_bob USING ((a % 2) = 0); CREATE POLICY p2 ON t1 TO regress_rls_carol USING ((a % 4) = 0); ALTER TABLE t1 ENABLE ROW LEVEL SECURITY; -- Prepare as regress_rls_bob SET ROLE regress_rls_bob; PREPARE role_inval AS SELECT * FROM t1; -- Check plan EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE role_inval; --- QUERY PLAN --- Seq Scan on t1 Filter: ((a % 2) = 0) -- Change to regress_rls_carol SET ROLE regress_rls_carol; -- Check plan- should be different EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE role_inval; --- QUERY PLAN --- Seq Scan on t1 Filter: ((a % 4) = 0) -- Change back to regress_rls_bob SET ROLE regress_rls_bob; -- Check plan- should be back to original EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE role_inval; --- QUERY PLAN --- Seq Scan on t1 Filter: ((a % 2) = 0) -- -- CTE and RLS -- RESET SESSION AUTHORIZATION; DROP TABLE t1 CASCADE; CREATE TABLE t1 (a integer, b text); SELECT public.create_hypertable('t1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (15,regress_rls_schema,t1,t) CREATE POLICY p1 ON t1 USING (a % 2 = 0); ALTER TABLE t1 ENABLE ROW LEVEL SECURITY; GRANT ALL ON t1 TO regress_rls_bob; INSERT INTO t1 (SELECT x, md5(x::text) FROM generate_series(0,20) x); SET SESSION AUTHORIZATION regress_rls_bob; WITH cte1 AS (SELECT * FROM t1 WHERE f_leak(b)) SELECT * FROM cte1; NOTICE: f_leak => cfcd208495d565ef66e7dff9f98764da NOTICE: f_leak => c81e728d9d4c2f636f067f89cc14862c NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc NOTICE: f_leak => c9f0f895fb98ab9159f51fd0297e236d NOTICE: f_leak => d3d9446802a44259755d38e6d163e820 NOTICE: f_leak => c20ad4d76fe97759aa27a0c99bff6710 NOTICE: f_leak => aab3238922bcc25a6f606eb525ffdc56 NOTICE: f_leak => c74d97b01eae257e44aa9d5bade97baf NOTICE: f_leak => 6f4922f45568161a8cdf4ad2299f6d23 NOTICE: f_leak => 98f13708210194c475687be6106a3b84 a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 EXPLAIN (BUFFERS OFF, COSTS OFF) WITH cte1 AS (SELECT * FROM t1 WHERE f_leak(b)) SELECT * FROM cte1; --- QUERY PLAN --- CTE Scan on cte1 CTE cte1 -> Custom Scan (ChunkAppend) on t1 Chunks excluded during startup: 0 -> Seq Scan on t1 t1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_69_chunk t1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_70_chunk t1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_71_chunk t1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_72_chunk t1_5 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_73_chunk t1_6 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_74_chunk t1_7 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_75_chunk t1_8 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_76_chunk t1_9 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_77_chunk t1_10 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_78_chunk t1_11 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_79_chunk t1_12 Filter: (((a % 2) = 0) AND f_leak(b)) WITH cte1 AS (UPDATE t1 SET a = a + 1 RETURNING *) SELECT * FROM cte1; --fail ERROR: new row violates row-level security policy for table "t1" WITH cte1 AS (UPDATE t1 SET a = a RETURNING *) SELECT * FROM cte1; --ok a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 WITH cte1 AS (INSERT INTO t1 VALUES (21, 'Fail') RETURNING *) SELECT * FROM cte1; --fail ERROR: new row violates row-level security policy for table "t1" WITH cte1 AS (INSERT INTO t1 VALUES (20, 'Success') RETURNING *) SELECT * FROM cte1; --ok a | b ----+--------- 20 | Success -- -- Rename Policy -- RESET SESSION AUTHORIZATION; ALTER POLICY p1 ON t1 RENAME TO p1; --fail ERROR: policy "p1" for table "t1" already exists SELECT polname, relname FROM pg_policy pol JOIN pg_class pc ON (pc.oid = pol.polrelid) WHERE relname = 't1'; polname | relname ---------+--------- p1 | t1 ALTER POLICY p1 ON t1 RENAME TO p2; --ok SELECT polname, relname FROM pg_policy pol JOIN pg_class pc ON (pc.oid = pol.polrelid) WHERE relname = 't1'; polname | relname ---------+--------- p2 | t1 -- -- Check INSERT SELECT -- SET SESSION AUTHORIZATION regress_rls_bob; CREATE TABLE t2 (a integer, b text); SELECT public.create_hypertable('t2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (16,regress_rls_schema,t2,t) INSERT INTO t2 (SELECT * FROM t1); EXPLAIN (BUFFERS OFF, COSTS OFF) INSERT INTO t2 (SELECT * FROM t1); --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on t2 -> Append -> Seq Scan on t1 t1_1 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_69_chunk t1_2 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_70_chunk t1_3 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_71_chunk t1_4 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_72_chunk t1_5 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_73_chunk t1_6 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_74_chunk t1_7 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_75_chunk t1_8 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_76_chunk t1_9 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_77_chunk t1_10 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_78_chunk t1_11 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_79_chunk t1_12 Filter: ((a % 2) = 0) SELECT * FROM t2; a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 20 | Success EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t2; --- QUERY PLAN --- Append -> Seq Scan on t2 t2_1 -> Seq Scan on _hyper_16_80_chunk t2_2 -> Seq Scan on _hyper_16_81_chunk t2_3 -> Seq Scan on _hyper_16_82_chunk t2_4 -> Seq Scan on _hyper_16_83_chunk t2_5 -> Seq Scan on _hyper_16_84_chunk t2_6 -> Seq Scan on _hyper_16_85_chunk t2_7 -> Seq Scan on _hyper_16_86_chunk t2_8 -> Seq Scan on _hyper_16_87_chunk t2_9 -> Seq Scan on _hyper_16_88_chunk t2_10 -> Seq Scan on _hyper_16_89_chunk t2_11 -> Seq Scan on _hyper_16_90_chunk t2_12 CREATE TABLE t3 AS SELECT * FROM t1; SELECT public.create_hypertable('t2', 'a', chunk_time_interval=>2); ERROR: table "t2" is already a hypertable SELECT * FROM t3; a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 20 | Success SELECT * INTO t4 FROM t1; SELECT * FROM t4; a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 20 | Success -- -- RLS with JOIN -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE blog (id integer, author text, post text); SELECT public.create_hypertable('blog', 'id', chunk_time_interval=>2); create_hypertable -------------------------------- (17,regress_rls_schema,blog,t) CREATE TABLE comment (blog_id integer, message text); SELECT public.create_hypertable('comment', 'blog_id', chunk_time_interval=>2); create_hypertable ----------------------------------- (18,regress_rls_schema,comment,t) GRANT ALL ON blog, comment TO regress_rls_bob; CREATE POLICY blog_1 ON blog USING (id % 2 = 0); ALTER TABLE blog ENABLE ROW LEVEL SECURITY; INSERT INTO blog VALUES (1, 'alice', 'blog #1'), (2, 'bob', 'blog #1'), (3, 'alice', 'blog #2'), (4, 'alice', 'blog #3'), (5, 'john', 'blog #1'); INSERT INTO comment VALUES (1, 'cool blog'), (1, 'fun blog'), (3, 'crazy blog'), (5, 'what?'), (4, 'insane!'), (2, 'who did it?'); SET SESSION AUTHORIZATION regress_rls_bob; -- Check RLS JOIN with Non-RLS. SELECT id, author, message FROM blog JOIN comment ON id = blog_id; id | author | message ----+--------+------------- 2 | bob | who did it? 4 | alice | insane! -- Check Non-RLS JOIN with RLS. SELECT id, author, message FROM comment JOIN blog ON id = blog_id; id | author | message ----+--------+------------- 2 | bob | who did it? 4 | alice | insane! SET SESSION AUTHORIZATION regress_rls_alice; CREATE POLICY comment_1 ON comment USING (blog_id < 4); ALTER TABLE comment ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; -- Check RLS JOIN RLS SELECT id, author, message FROM blog JOIN comment ON id = blog_id; id | author | message ----+--------+------------- 2 | bob | who did it? SELECT id, author, message FROM comment JOIN blog ON id = blog_id; id | author | message ----+--------+------------- 2 | bob | who did it? SET SESSION AUTHORIZATION regress_rls_alice; DROP TABLE blog; DROP TABLE comment; -- -- Default Deny Policy -- RESET SESSION AUTHORIZATION; DROP POLICY p2 ON t1; ALTER TABLE t1 OWNER TO regress_rls_alice; -- Check that default deny does not apply to superuser. RESET SESSION AUTHORIZATION; SELECT * FROM t1; a | b ----+---------------------------------- 1 | c4ca4238a0b923820dcc509a6f75849b 0 | cfcd208495d565ef66e7dff9f98764da 3 | eccbc87e4b5ce2fe28308fd9f2a7baf3 2 | c81e728d9d4c2f636f067f89cc14862c 5 | e4da3b7fbbce2345d7772b0674a318d5 4 | a87ff679a2f3e71d9181a67b7542122c 7 | 8f14e45fceea167a5a36dedd4bea2543 6 | 1679091c5a880faf6fb5e6087eb1b2dc 9 | 45c48cce2e2d7fbdea1afc51c7c6ad26 8 | c9f0f895fb98ab9159f51fd0297e236d 11 | 6512bd43d9caa6e02c990b0a82652dca 10 | d3d9446802a44259755d38e6d163e820 13 | c51ce410c124a10e0db5e4b97fc2af39 12 | c20ad4d76fe97759aa27a0c99bff6710 15 | 9bf31c7ff062936a96d3c8bd1f8f2ff3 14 | aab3238922bcc25a6f606eb525ffdc56 17 | 70efdf2ec9b086079795c442636b55fb 16 | c74d97b01eae257e44aa9d5bade97baf 19 | 1f0e3dad99908345f7439f8ffabdffc4 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 20 | Success EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 -> Seq Scan on _hyper_15_69_chunk t1_2 -> Seq Scan on _hyper_15_70_chunk t1_3 -> Seq Scan on _hyper_15_71_chunk t1_4 -> Seq Scan on _hyper_15_72_chunk t1_5 -> Seq Scan on _hyper_15_73_chunk t1_6 -> Seq Scan on _hyper_15_74_chunk t1_7 -> Seq Scan on _hyper_15_75_chunk t1_8 -> Seq Scan on _hyper_15_76_chunk t1_9 -> Seq Scan on _hyper_15_77_chunk t1_10 -> Seq Scan on _hyper_15_78_chunk t1_11 -> Seq Scan on _hyper_15_79_chunk t1_12 -- Check that default deny does not apply to table owner. SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM t1; a | b ----+---------------------------------- 1 | c4ca4238a0b923820dcc509a6f75849b 0 | cfcd208495d565ef66e7dff9f98764da 3 | eccbc87e4b5ce2fe28308fd9f2a7baf3 2 | c81e728d9d4c2f636f067f89cc14862c 5 | e4da3b7fbbce2345d7772b0674a318d5 4 | a87ff679a2f3e71d9181a67b7542122c 7 | 8f14e45fceea167a5a36dedd4bea2543 6 | 1679091c5a880faf6fb5e6087eb1b2dc 9 | 45c48cce2e2d7fbdea1afc51c7c6ad26 8 | c9f0f895fb98ab9159f51fd0297e236d 11 | 6512bd43d9caa6e02c990b0a82652dca 10 | d3d9446802a44259755d38e6d163e820 13 | c51ce410c124a10e0db5e4b97fc2af39 12 | c20ad4d76fe97759aa27a0c99bff6710 15 | 9bf31c7ff062936a96d3c8bd1f8f2ff3 14 | aab3238922bcc25a6f606eb525ffdc56 17 | 70efdf2ec9b086079795c442636b55fb 16 | c74d97b01eae257e44aa9d5bade97baf 19 | 1f0e3dad99908345f7439f8ffabdffc4 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 20 | Success EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 -> Seq Scan on _hyper_15_69_chunk t1_2 -> Seq Scan on _hyper_15_70_chunk t1_3 -> Seq Scan on _hyper_15_71_chunk t1_4 -> Seq Scan on _hyper_15_72_chunk t1_5 -> Seq Scan on _hyper_15_73_chunk t1_6 -> Seq Scan on _hyper_15_74_chunk t1_7 -> Seq Scan on _hyper_15_75_chunk t1_8 -> Seq Scan on _hyper_15_76_chunk t1_9 -> Seq Scan on _hyper_15_77_chunk t1_10 -> Seq Scan on _hyper_15_78_chunk t1_11 -> Seq Scan on _hyper_15_79_chunk t1_12 -- Check that default deny applies to non-owner/non-superuser when RLS on. SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; SELECT * FROM t1; a | b ---+--- EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; --- QUERY PLAN --- Result One-Time Filter: false SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM t1; a | b ---+--- EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; --- QUERY PLAN --- Result One-Time Filter: false -- -- COPY TO/FROM -- RESET SESSION AUTHORIZATION; DROP TABLE copy_t CASCADE; ERROR: table "copy_t" does not exist CREATE TABLE copy_t (a integer, b text); SELECT public.create_hypertable('copy_t', 'a', chunk_time_interval=>2); create_hypertable ---------------------------------- (19,regress_rls_schema,copy_t,t) CREATE POLICY p1 ON copy_t USING (a % 2 = 0); ALTER TABLE copy_t ENABLE ROW LEVEL SECURITY; GRANT ALL ON copy_t TO regress_rls_bob, regress_rls_exempt_user; INSERT INTO copy_t (SELECT x, md5(x::text) FROM generate_series(0,10) x); -- Check COPY TO as Superuser/owner. RESET SESSION AUTHORIZATION; SET row_security TO OFF; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; 0,cfcd208495d565ef66e7dff9f98764da 1,c4ca4238a0b923820dcc509a6f75849b 2,c81e728d9d4c2f636f067f89cc14862c 3,eccbc87e4b5ce2fe28308fd9f2a7baf3 4,a87ff679a2f3e71d9181a67b7542122c 5,e4da3b7fbbce2345d7772b0674a318d5 6,1679091c5a880faf6fb5e6087eb1b2dc 7,8f14e45fceea167a5a36dedd4bea2543 8,c9f0f895fb98ab9159f51fd0297e236d 9,45c48cce2e2d7fbdea1afc51c7c6ad26 10,d3d9446802a44259755d38e6d163e820 SET row_security TO ON; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; 0,cfcd208495d565ef66e7dff9f98764da 1,c4ca4238a0b923820dcc509a6f75849b 2,c81e728d9d4c2f636f067f89cc14862c 3,eccbc87e4b5ce2fe28308fd9f2a7baf3 4,a87ff679a2f3e71d9181a67b7542122c 5,e4da3b7fbbce2345d7772b0674a318d5 6,1679091c5a880faf6fb5e6087eb1b2dc 7,8f14e45fceea167a5a36dedd4bea2543 8,c9f0f895fb98ab9159f51fd0297e236d 9,45c48cce2e2d7fbdea1afc51c7c6ad26 10,d3d9446802a44259755d38e6d163e820 -- Check COPY TO as user with permissions. SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO OFF; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --fail - would be affected by RLS ERROR: query would be affected by row-level security policy for table "copy_t" SET row_security TO ON; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --ok 0,cfcd208495d565ef66e7dff9f98764da 2,c81e728d9d4c2f636f067f89cc14862c 4,a87ff679a2f3e71d9181a67b7542122c 6,1679091c5a880faf6fb5e6087eb1b2dc 8,c9f0f895fb98ab9159f51fd0297e236d 10,d3d9446802a44259755d38e6d163e820 -- Check COPY TO as user with permissions and BYPASSRLS SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --ok 0,cfcd208495d565ef66e7dff9f98764da 1,c4ca4238a0b923820dcc509a6f75849b 2,c81e728d9d4c2f636f067f89cc14862c 3,eccbc87e4b5ce2fe28308fd9f2a7baf3 4,a87ff679a2f3e71d9181a67b7542122c 5,e4da3b7fbbce2345d7772b0674a318d5 6,1679091c5a880faf6fb5e6087eb1b2dc 7,8f14e45fceea167a5a36dedd4bea2543 8,c9f0f895fb98ab9159f51fd0297e236d 9,45c48cce2e2d7fbdea1afc51c7c6ad26 10,d3d9446802a44259755d38e6d163e820 SET row_security TO ON; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --ok 0,cfcd208495d565ef66e7dff9f98764da 1,c4ca4238a0b923820dcc509a6f75849b 2,c81e728d9d4c2f636f067f89cc14862c 3,eccbc87e4b5ce2fe28308fd9f2a7baf3 4,a87ff679a2f3e71d9181a67b7542122c 5,e4da3b7fbbce2345d7772b0674a318d5 6,1679091c5a880faf6fb5e6087eb1b2dc 7,8f14e45fceea167a5a36dedd4bea2543 8,c9f0f895fb98ab9159f51fd0297e236d 9,45c48cce2e2d7fbdea1afc51c7c6ad26 10,d3d9446802a44259755d38e6d163e820 -- Check COPY TO as user without permissions. SET row_security TO OFF; SET SESSION AUTHORIZATION regress_rls_carol; SET row_security TO OFF; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --fail - would be affected by RLS ERROR: query would be affected by row-level security policy for table "copy_t" SET row_security TO ON; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --fail - permission denied ERROR: permission denied for table copy_t -- Check COPY relation TO; keep it just one row to avoid reordering issues RESET SESSION AUTHORIZATION; SET row_security TO ON; CREATE TABLE copy_rel_to (a integer, b text); SELECT public.create_hypertable('copy_rel_to', 'a', chunk_time_interval=>2); create_hypertable --------------------------------------- (20,regress_rls_schema,copy_rel_to,t) CREATE POLICY p1 ON copy_rel_to USING (a % 2 = 0); ALTER TABLE copy_rel_to ENABLE ROW LEVEL SECURITY; GRANT ALL ON copy_rel_to TO regress_rls_bob, regress_rls_exempt_user; INSERT INTO copy_rel_to VALUES (1, md5('1')); -- Check COPY TO as Superuser/owner. RESET SESSION AUTHORIZATION; SET row_security TO OFF; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; 1,c4ca4238a0b923820dcc509a6f75849b SET row_security TO ON; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; 1,c4ca4238a0b923820dcc509a6f75849b -- Check COPY TO as user with permissions. SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO OFF; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --fail - would be affected by RLS ERROR: query would be affected by row-level security policy for table "copy_rel_to" SET row_security TO ON; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --ok -- Check COPY TO as user with permissions and BYPASSRLS SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --ok 1,c4ca4238a0b923820dcc509a6f75849b SET row_security TO ON; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --ok 1,c4ca4238a0b923820dcc509a6f75849b -- Check COPY TO as user without permissions. SET row_security TO OFF; SET SESSION AUTHORIZATION regress_rls_carol; SET row_security TO OFF; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --fail - permission denied ERROR: query would be affected by row-level security policy for table "copy_rel_to" SET row_security TO ON; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --fail - permission denied ERROR: permission denied for table copy_rel_to -- Check COPY FROM as Superuser/owner. RESET SESSION AUTHORIZATION; SET row_security TO OFF; COPY copy_t FROM STDIN; --ok SET row_security TO ON; COPY copy_t FROM STDIN; --ok -- Check COPY FROM as user with permissions. SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO OFF; COPY copy_t FROM STDIN; --fail - would be affected by RLS. ERROR: query would be affected by row-level security policy for table "copy_t" SET row_security TO ON; COPY copy_t FROM STDIN; --fail - COPY FROM not supported by RLS. ERROR: COPY FROM not supported with row-level security HINT: Use INSERT statements instead. -- Check COPY FROM as user with permissions and BYPASSRLS SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO ON; COPY copy_t FROM STDIN; --ok -- Check COPY FROM as user without permissions. SET SESSION AUTHORIZATION regress_rls_carol; SET row_security TO OFF; COPY copy_t FROM STDIN; --fail - permission denied. ERROR: permission denied for table copy_t SET row_security TO ON; COPY copy_t FROM STDIN; --fail - permission denied. ERROR: permission denied for table copy_t RESET SESSION AUTHORIZATION; DROP TABLE copy_t; DROP TABLE copy_rel_to CASCADE; -- Check WHERE CURRENT OF SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE current_check (currentid int, payload text, rlsuser text); SELECT public.create_hypertable('current_check', 'currentid', chunk_time_interval=>10); create_hypertable ----------------------------------------- (21,regress_rls_schema,current_check,t) GRANT ALL ON current_check TO PUBLIC; INSERT INTO current_check VALUES (1, 'abc', 'regress_rls_bob'), (2, 'bcd', 'regress_rls_bob'), (3, 'cde', 'regress_rls_bob'), (4, 'def', 'regress_rls_bob'); CREATE POLICY p1 ON current_check FOR SELECT USING (currentid % 2 = 0); CREATE POLICY p2 ON current_check FOR DELETE USING (currentid = 4 AND rlsuser = current_user); CREATE POLICY p3 ON current_check FOR UPDATE USING (currentid = 4) WITH CHECK (rlsuser = current_user); ALTER TABLE current_check ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; -- Can SELECT even rows SELECT * FROM current_check; currentid | payload | rlsuser -----------+---------+----------------- 2 | bcd | regress_rls_bob 4 | def | regress_rls_bob -- Cannot UPDATE row 2 UPDATE current_check SET payload = payload || '_new' WHERE currentid = 2 RETURNING *; currentid | payload | rlsuser -----------+---------+--------- BEGIN; -- WHERE CURRENT OF does not work with custom scan nodes -- so we have to disable chunk append here SET timescaledb.enable_chunk_append TO false; DECLARE current_check_cursor SCROLL CURSOR FOR SELECT * FROM current_check; -- Returns rows that can be seen according to SELECT policy, like plain SELECT -- above (even rows) FETCH ABSOLUTE 1 FROM current_check_cursor; currentid | payload | rlsuser -----------+---------+----------------- 2 | bcd | regress_rls_bob -- Still cannot UPDATE row 2 through cursor UPDATE current_check SET payload = payload || '_new' WHERE CURRENT OF current_check_cursor RETURNING *; currentid | payload | rlsuser -----------+---------+--------- -- Can update row 4 through cursor, which is the next visible row FETCH RELATIVE 1 FROM current_check_cursor; currentid | payload | rlsuser -----------+---------+----------------- 4 | def | regress_rls_bob UPDATE current_check SET payload = payload || '_new' WHERE CURRENT OF current_check_cursor RETURNING *; currentid | payload | rlsuser -----------+---------+----------------- 4 | def_new | regress_rls_bob SELECT * FROM current_check; currentid | payload | rlsuser -----------+---------+----------------- 2 | bcd | regress_rls_bob 4 | def_new | regress_rls_bob -- Plan should be a subquery TID scan EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE current_check SET payload = payload WHERE CURRENT OF current_check_cursor; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Update on current_check Update on _hyper_21_104_chunk current_check_1 -> Tid Scan on _hyper_21_104_chunk current_check_1 TID Cond: CURRENT OF current_check_cursor Filter: ((currentid = 4) AND ((currentid % 2) = 0)) -- Similarly can only delete row 4 FETCH ABSOLUTE 1 FROM current_check_cursor; currentid | payload | rlsuser -----------+---------+----------------- 2 | bcd | regress_rls_bob DELETE FROM current_check WHERE CURRENT OF current_check_cursor RETURNING *; currentid | payload | rlsuser -----------+---------+--------- FETCH RELATIVE 1 FROM current_check_cursor; currentid | payload | rlsuser -----------+---------+----------------- 4 | def | regress_rls_bob DELETE FROM current_check WHERE CURRENT OF current_check_cursor RETURNING *; currentid | payload | rlsuser -----------+---------+----------------- 4 | def_new | regress_rls_bob SELECT * FROM current_check; currentid | payload | rlsuser -----------+---------+----------------- 2 | bcd | regress_rls_bob RESET timescaledb.enable_chunk_append; COMMIT; -- -- check pg_stats view filtering -- SET row_security TO ON; SET SESSION AUTHORIZATION regress_rls_alice; ANALYZE current_check; -- Stats visible SELECT row_security_active('current_check'); row_security_active --------------------- f SELECT attname, most_common_vals FROM pg_stats WHERE tablename = 'current_check' ORDER BY 1; attname | most_common_vals -----------+------------------- currentid | payload | rlsuser | {regress_rls_bob} SET SESSION AUTHORIZATION regress_rls_bob; -- Stats not visible SELECT row_security_active('current_check'); row_security_active --------------------- t SELECT attname, most_common_vals FROM pg_stats WHERE tablename = 'current_check' ORDER BY 1; attname | most_common_vals ---------+------------------ -- -- Collation support -- BEGIN; CREATE TABLE coll_t (c) AS VALUES ('bar'::text); CREATE POLICY coll_p ON coll_t USING (c < ('foo'::text COLLATE "C")); ALTER TABLE coll_t ENABLE ROW LEVEL SECURITY; GRANT SELECT ON coll_t TO regress_rls_alice; SELECT (string_to_array(polqual, ':'))[7] AS inputcollid FROM pg_policy WHERE polrelid = 'coll_t'::regclass; inputcollid ------------------ inputcollid 950 SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM coll_t; c ----- bar ROLLBACK; -- -- Shared Object Dependencies -- RESET SESSION AUTHORIZATION; BEGIN; CREATE ROLE regress_rls_eve; CREATE ROLE regress_rls_frank; CREATE TABLE tbl1 (c) AS VALUES ('bar'::text); GRANT SELECT ON TABLE tbl1 TO regress_rls_eve; CREATE POLICY P ON tbl1 TO regress_rls_eve, regress_rls_frank USING (true); SELECT refclassid::regclass, deptype FROM pg_depend WHERE classid = 'pg_policy'::regclass AND refobjid = 'tbl1'::regclass; refclassid | deptype ------------+--------- pg_class | a SELECT refclassid::regclass, deptype FROM pg_shdepend WHERE classid = 'pg_policy'::regclass AND refobjid IN ('regress_rls_eve'::regrole, 'regress_rls_frank'::regrole); refclassid | deptype ------------+--------- pg_authid | r pg_authid | r SAVEPOINT q; DROP ROLE regress_rls_eve; --fails due to dependency on POLICY p ERROR: role "regress_rls_eve" cannot be dropped because some objects depend on it DETAIL: privileges for table tbl1 target of policy p on table tbl1 ROLLBACK TO q; ALTER POLICY p ON tbl1 TO regress_rls_frank USING (true); SAVEPOINT q; DROP ROLE regress_rls_eve; --fails due to dependency on GRANT SELECT ERROR: role "regress_rls_eve" cannot be dropped because some objects depend on it DETAIL: privileges for table tbl1 ROLLBACK TO q; REVOKE ALL ON TABLE tbl1 FROM regress_rls_eve; SAVEPOINT q; DROP ROLE regress_rls_eve; --succeeds ROLLBACK TO q; SAVEPOINT q; DROP ROLE regress_rls_frank; --fails due to dependency on POLICY p ERROR: role "regress_rls_frank" cannot be dropped because some objects depend on it DETAIL: target of policy p on table tbl1 ROLLBACK TO q; DROP POLICY p ON tbl1; SAVEPOINT q; DROP ROLE regress_rls_frank; -- succeeds ROLLBACK TO q; ROLLBACK; -- cleanup -- -- Converting table to view -- BEGIN; CREATE TABLE t (c int); SELECT public.create_hypertable('t', 'c', chunk_time_interval=>2); create_hypertable ----------------------------- (22,regress_rls_schema,t,t) CREATE POLICY p ON t USING (c % 2 = 1); ALTER TABLE t ENABLE ROW LEVEL SECURITY; SAVEPOINT q; CREATE RULE "_RETURN" AS ON SELECT TO t DO INSTEAD SELECT * FROM generate_series(1,5) t0(c); -- fails due to row level security enabled ERROR: hypertables do not support rules ROLLBACK TO q; ALTER TABLE t DISABLE ROW LEVEL SECURITY; SAVEPOINT q; CREATE RULE "_RETURN" AS ON SELECT TO t DO INSTEAD SELECT * FROM generate_series(1,5) t0(c); -- fails due to policy p on t ERROR: hypertables do not support rules ROLLBACK TO q; DROP POLICY p ON t; CREATE RULE "_RETURN" AS ON SELECT TO t DO INSTEAD SELECT * FROM generate_series(1,5) t0(c); -- succeeds ERROR: hypertables do not support rules ROLLBACK; -- -- Policy expression handling -- BEGIN; CREATE TABLE t (c) AS VALUES ('bar'::text); CREATE POLICY p ON t USING (max(c)); -- fails: aggregate functions are not allowed in policy expressions ERROR: aggregate functions are not allowed in policy expressions ROLLBACK; -- -- Non-target relations are only subject to SELECT policies -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE r1 (a int); SELECT public.create_hypertable('r1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (23,regress_rls_schema,r1,t) CREATE TABLE r2 (a int); SELECT public.create_hypertable('r2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (24,regress_rls_schema,r2,t) INSERT INTO r1 VALUES (10), (20); INSERT INTO r2 VALUES (10), (20); GRANT ALL ON r1, r2 TO regress_rls_bob; CREATE POLICY p1 ON r1 USING (true); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; CREATE POLICY p1 ON r2 FOR SELECT USING (true); CREATE POLICY p2 ON r2 FOR INSERT WITH CHECK (false); CREATE POLICY p3 ON r2 FOR UPDATE USING (false); CREATE POLICY p4 ON r2 FOR DELETE USING (false); ALTER TABLE r2 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM r1; a ---- 10 20 SELECT * FROM r2; a ---- 10 20 -- r2 is read-only INSERT INTO r2 VALUES (2); -- Not allowed ERROR: new row violates row-level security policy for table "r2" \pset tuples_only 1 UPDATE r2 SET a = 2 RETURNING *; -- Updates nothing DELETE FROM r2 RETURNING *; -- Deletes nothing \pset tuples_only 0 -- r2 can be used as a non-target relation in DML INSERT INTO r1 SELECT a + 1 FROM r2 RETURNING *; -- OK a ---- 11 21 UPDATE r1 SET a = r2.a + 2 FROM r2 WHERE r1.a = r2.a RETURNING *; -- OK ERROR: new row for relation "_hyper_23_105_chunk" violates check constraint "constraint_105" DELETE FROM r1 USING r2 WHERE r1.a = r2.a + 2 RETURNING *; -- OK a | a ---+--- SELECT * FROM r1; a ---- 10 11 20 21 SELECT * FROM r2; a ---- 10 20 SET SESSION AUTHORIZATION regress_rls_alice; DROP TABLE r1; DROP TABLE r2; -- -- FORCE ROW LEVEL SECURITY applies RLS to owners too -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security = on; CREATE TABLE r1 (a int); SELECT public.create_hypertable('r1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (25,regress_rls_schema,r1,t) INSERT INTO r1 VALUES (10), (20); CREATE POLICY p1 ON r1 USING (false); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- No error, but no rows TABLE r1; a --- -- RLS error INSERT INTO r1 VALUES (1); ERROR: new row violates row-level security policy for table "r1" -- No error (unable to see any rows to update) UPDATE r1 SET a = 1; TABLE r1; a --- -- No error (unable to see any rows to delete) DELETE FROM r1; TABLE r1; a --- SET row_security = off; -- these all fail, would be affected by RLS TABLE r1; ERROR: query would be affected by row-level security policy for table "r1" HINT: To disable the policy for the table's owner, use ALTER TABLE NO FORCE ROW LEVEL SECURITY. UPDATE r1 SET a = 1; ERROR: query would be affected by row-level security policy for table "r1" HINT: To disable the policy for the table's owner, use ALTER TABLE NO FORCE ROW LEVEL SECURITY. DELETE FROM r1; ERROR: query would be affected by row-level security policy for table "r1" HINT: To disable the policy for the table's owner, use ALTER TABLE NO FORCE ROW LEVEL SECURITY. DROP TABLE r1; -- -- FORCE ROW LEVEL SECURITY does not break RI -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security = on; CREATE TABLE r1 (a int PRIMARY KEY); -- r1 is not a hypertable since r1.a is referenced by r2 CREATE TABLE r2 (a int REFERENCES r1); SELECT public.create_hypertable('r2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (26,regress_rls_schema,r2,t) INSERT INTO r1 VALUES (10), (20); INSERT INTO r2 VALUES (10), (20); -- Create policies on r2 which prevent the -- owner from seeing any rows, but RI should -- still see them. CREATE POLICY p1 ON r2 USING (false); ALTER TABLE r2 ENABLE ROW LEVEL SECURITY; ALTER TABLE r2 FORCE ROW LEVEL SECURITY; -- Errors due to rows in r2 DELETE FROM r1; ERROR: update or delete on table "r1" violates foreign key constraint "r2_a_fkey" on table "r2" DETAIL: Key (a)=(10) is still referenced from table "r2". -- Reset r2 to no-RLS DROP POLICY p1 ON r2; ALTER TABLE r2 NO FORCE ROW LEVEL SECURITY; ALTER TABLE r2 DISABLE ROW LEVEL SECURITY; -- clean out r2 for INSERT test below DELETE FROM r2; -- Change r1 to not allow rows to be seen CREATE POLICY p1 ON r1 USING (false); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- No rows seen TABLE r1; a --- -- No error, RI still sees that row exists in r1 INSERT INTO r2 VALUES (10); DROP TABLE r2; DROP TABLE r1; -- Ensure cascaded DELETE works CREATE TABLE r1 (a int PRIMARY KEY); -- r1 is not a hypertable since r1.a is referenced by r2 CREATE TABLE r2 (a int REFERENCES r1 ON DELETE CASCADE); SELECT public.create_hypertable('r2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (27,regress_rls_schema,r2,t) INSERT INTO r1 VALUES (10), (20); INSERT INTO r2 VALUES (10), (20); -- Create policies on r2 which prevent the -- owner from seeing any rows, but RI should -- still see them. CREATE POLICY p1 ON r2 USING (false); ALTER TABLE r2 ENABLE ROW LEVEL SECURITY; ALTER TABLE r2 FORCE ROW LEVEL SECURITY; -- Deletes all records from both DELETE FROM r1; -- Remove FORCE from r2 ALTER TABLE r2 NO FORCE ROW LEVEL SECURITY; -- As owner, we now bypass RLS -- verify no rows in r2 now TABLE r2; a --- DROP TABLE r2; DROP TABLE r1; -- Ensure cascaded UPDATE works CREATE TABLE r1 (a int PRIMARY KEY); -- r1 is not a hypertable since r1.a is referenced by r2 CREATE TABLE r2 (a int REFERENCES r1 ON UPDATE CASCADE); SELECT public.create_hypertable('r2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (28,regress_rls_schema,r2,t) INSERT INTO r1 VALUES (10), (20); INSERT INTO r2 VALUES (10), (20); -- Create policies on r2 which prevent the -- owner from seeing any rows, but RI should -- still see them. CREATE POLICY p1 ON r2 USING (false); ALTER TABLE r2 ENABLE ROW LEVEL SECURITY; ALTER TABLE r2 FORCE ROW LEVEL SECURITY; -- Updates records in both (terse output to not print CONTEXT, which can be different). \set VERBOSITY terse UPDATE r1 SET a = a+5; ERROR: new row for relation "_hyper_28_117_chunk" violates check constraint "constraint_117" \set VERBOSITY default -- Remove FORCE from r2 ALTER TABLE r2 NO FORCE ROW LEVEL SECURITY; -- As owner, we now bypass RLS -- verify records in r2 updated TABLE r2; a ---- 10 20 DROP TABLE r2; DROP TABLE r1; -- -- Test INSERT+RETURNING applies SELECT policies as -- WithCheckOptions (meaning an error is thrown) -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security = on; CREATE TABLE r1 (a int); SELECT public.create_hypertable('r1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (29,regress_rls_schema,r1,t) CREATE POLICY p1 ON r1 FOR SELECT USING (false); CREATE POLICY p2 ON r1 FOR INSERT WITH CHECK (true); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- Works fine INSERT INTO r1 VALUES (10), (20); -- No error, but no rows TABLE r1; a --- SET row_security = off; -- fail, would be affected by RLS TABLE r1; ERROR: query would be affected by row-level security policy for table "r1" HINT: To disable the policy for the table's owner, use ALTER TABLE NO FORCE ROW LEVEL SECURITY. SET row_security = on; -- Error INSERT INTO r1 VALUES (10), (20) RETURNING *; ERROR: new row violates row-level security policy for table "r1" DROP TABLE r1; -- -- Test UPDATE+RETURNING applies SELECT policies as -- WithCheckOptions (meaning an error is thrown) -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security = on; CREATE TABLE r1 (a int PRIMARY KEY); SELECT public.create_hypertable('r1', 'a', chunk_time_interval=>100); create_hypertable ------------------------------ (30,regress_rls_schema,r1,t) CREATE POLICY p1 ON r1 FOR SELECT USING (a < 20); CREATE POLICY p2 ON r1 FOR UPDATE USING (a < 20) WITH CHECK (true); CREATE POLICY p3 ON r1 FOR INSERT WITH CHECK (true); INSERT INTO r1 VALUES (10); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- Works fine UPDATE r1 SET a = 30; -- Show updated rows ALTER TABLE r1 NO FORCE ROW LEVEL SECURITY; TABLE r1; a ---- 30 -- reset value in r1 for test with RETURNING UPDATE r1 SET a = 10; -- Verify row reset TABLE r1; a ---- 10 ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- Error UPDATE r1 SET a = 30 RETURNING *; ERROR: new row violates row-level security policy for table "r1" -- UPDATE path of INSERT ... ON CONFLICT DO UPDATE should also error out INSERT INTO r1 VALUES (10) ON CONFLICT (a) DO UPDATE SET a = 30 RETURNING *; ERROR: new row violates row-level security policy for table "r1" -- Should still error out without RETURNING (use of arbiter always requires -- SELECT permissions) INSERT INTO r1 VALUES (10) ON CONFLICT (a) DO UPDATE SET a = 30; ERROR: new row violates row-level security policy for table "r1" -- ON CONFLICT ON CONSTRAINT INSERT INTO r1 VALUES (10) ON CONFLICT ON CONSTRAINT r1_pkey DO UPDATE SET a = 30; ERROR: new row violates row-level security policy for table "r1" DROP TABLE r1; -- Check dependency handling RESET SESSION AUTHORIZATION; CREATE TABLE dep1 (c1 int); SELECT public.create_hypertable('dep1', 'c1', chunk_time_interval=>2); create_hypertable -------------------------------- (31,regress_rls_schema,dep1,t) CREATE TABLE dep2 (c1 int); SELECT public.create_hypertable('dep2', 'c1', chunk_time_interval=>2); create_hypertable -------------------------------- (32,regress_rls_schema,dep2,t) CREATE POLICY dep_p1 ON dep1 TO regress_rls_bob USING (c1 > (select max(dep2.c1) from dep2)); ALTER POLICY dep_p1 ON dep1 TO regress_rls_bob,regress_rls_carol; -- Should return one SELECT count(*) = 1 FROM pg_depend WHERE objid = (SELECT oid FROM pg_policy WHERE polname = 'dep_p1') AND refobjid = (SELECT oid FROM pg_class WHERE relname = 'dep2'); ?column? ---------- t ALTER POLICY dep_p1 ON dep1 USING (true); -- Should return one SELECT count(*) = 1 FROM pg_shdepend WHERE objid = (SELECT oid FROM pg_policy WHERE polname = 'dep_p1') AND refobjid = (SELECT oid FROM pg_authid WHERE rolname = 'regress_rls_bob'); ?column? ---------- t -- Should return one SELECT count(*) = 1 FROM pg_shdepend WHERE objid = (SELECT oid FROM pg_policy WHERE polname = 'dep_p1') AND refobjid = (SELECT oid FROM pg_authid WHERE rolname = 'regress_rls_carol'); ?column? ---------- t -- Should return zero SELECT count(*) = 0 FROM pg_depend WHERE objid = (SELECT oid FROM pg_policy WHERE polname = 'dep_p1') AND refobjid = (SELECT oid FROM pg_class WHERE relname = 'dep2'); ?column? ---------- t -- DROP OWNED BY testing RESET SESSION AUTHORIZATION; CREATE ROLE regress_rls_dob_role1; CREATE ROLE regress_rls_dob_role2; CREATE TABLE dob_t1 (c1 int); SELECT public.create_hypertable('dob_t1', 'c1', chunk_time_interval=>2); create_hypertable ---------------------------------- (33,regress_rls_schema,dob_t1,t) CREATE TABLE dob_t2 (c1 int) PARTITION BY RANGE (c1); CREATE POLICY p1 ON dob_t1 TO regress_rls_dob_role1 USING (true); DROP OWNED BY regress_rls_dob_role1; DROP POLICY p1 ON dob_t1; -- should fail, already gone ERROR: policy "p1" for table "dob_t1" does not exist CREATE POLICY p1 ON dob_t1 TO regress_rls_dob_role1,regress_rls_dob_role2 USING (true); DROP OWNED BY regress_rls_dob_role1; DROP POLICY p1 ON dob_t1; -- should succeed CREATE POLICY p1 ON dob_t2 TO regress_rls_dob_role1,regress_rls_dob_role2 USING (true); DROP OWNED BY regress_rls_dob_role1; DROP POLICY p1 ON dob_t2; -- should succeed DROP USER regress_rls_dob_role1; DROP USER regress_rls_dob_role2; -- -- Clean up objects -- RESET SESSION AUTHORIZATION; \set VERBOSITY terse \\ -- suppress cascade details DROP SCHEMA regress_rls_schema CASCADE; NOTICE: drop cascades to 116 other objects \set VERBOSITY default DROP USER regress_rls_alice; DROP USER regress_rls_bob; DROP USER regress_rls_carol; DROP USER regress_rls_dave; DROP USER regress_rls_exempt_user; DROP ROLE regress_rls_group1; DROP ROLE regress_rls_group2; -- Arrange to have a few policies left over, for testing -- pg_dump/pg_restore CREATE SCHEMA regress_rls_schema; CREATE TABLE rls_tbl (c1 int); SELECT public.create_hypertable('rls_tbl', 'c1', chunk_time_interval=>2); create_hypertable ----------------------------------- (34,regress_rls_schema,rls_tbl,t) ALTER TABLE rls_tbl ENABLE ROW LEVEL SECURITY; CREATE POLICY p1 ON rls_tbl USING (c1 > 5); CREATE POLICY p2 ON rls_tbl FOR SELECT USING (c1 <= 3); CREATE POLICY p3 ON rls_tbl FOR UPDATE USING (c1 <= 3) WITH CHECK (c1 > 5); CREATE POLICY p4 ON rls_tbl FOR DELETE USING (c1 <= 3); CREATE TABLE rls_tbl_force (c1 int); SELECT public.create_hypertable('rls_tbl_force', 'c1', chunk_time_interval=>2); create_hypertable ----------------------------------------- (35,regress_rls_schema,rls_tbl_force,t) ALTER TABLE rls_tbl_force ENABLE ROW LEVEL SECURITY; ALTER TABLE rls_tbl_force FORCE ROW LEVEL SECURITY; CREATE POLICY p1 ON rls_tbl_force USING (c1 = 5) WITH CHECK (c1 < 5); CREATE POLICY p2 ON rls_tbl_force FOR SELECT USING (c1 = 8); CREATE POLICY p3 ON rls_tbl_force FOR UPDATE USING (c1 = 8) WITH CHECK (c1 >= 5); CREATE POLICY p4 ON rls_tbl_force FOR DELETE USING (c1 = 8); ================================================ FILE: test/expected/rowsecurity-16.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- -- Test of Row-level security feature -- -- Clean up in case a prior regression run failed \c :TEST_DBNAME :ROLE_SUPERUSER \set ON_ERROR_STOP 0 \set VERBOSITY default SET timescaledb.enable_constraint_exclusion TO off; -- Suppress NOTICE messages when users/groups don't exist SET client_min_messages TO 'warning'; DROP USER IF EXISTS regress_rls_alice; DROP USER IF EXISTS regress_rls_bob; DROP USER IF EXISTS regress_rls_carol; DROP USER IF EXISTS regress_rls_dave; DROP USER IF EXISTS regress_rls_exempt_user; DROP ROLE IF EXISTS regress_rls_group1; DROP ROLE IF EXISTS regress_rls_group2; DROP SCHEMA IF EXISTS regress_rls_schema CASCADE; RESET client_min_messages; -- initial setup CREATE USER regress_rls_alice NOLOGIN; CREATE USER regress_rls_bob NOLOGIN; CREATE USER regress_rls_carol NOLOGIN; CREATE USER regress_rls_dave NOLOGIN; CREATE USER regress_rls_exempt_user BYPASSRLS NOLOGIN; CREATE ROLE regress_rls_group1 NOLOGIN; CREATE ROLE regress_rls_group2 NOLOGIN; GRANT regress_rls_group1 TO regress_rls_bob; GRANT regress_rls_group2 TO regress_rls_carol; CREATE SCHEMA regress_rls_schema; GRANT ALL ON SCHEMA regress_rls_schema to public; SET search_path = regress_rls_schema; -- setup of malicious function CREATE OR REPLACE FUNCTION f_leak(text) RETURNS bool COST 0.0000001 LANGUAGE plpgsql AS 'BEGIN RAISE NOTICE ''f_leak => %'', $1; RETURN true; END'; GRANT EXECUTE ON FUNCTION f_leak(text) TO public; -- BASIC Row-Level Security Scenario SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE uaccount ( pguser name primary key, seclv int ); GRANT SELECT ON uaccount TO public; INSERT INTO uaccount VALUES ('regress_rls_alice', 99), ('regress_rls_bob', 1), ('regress_rls_carol', 2), ('regress_rls_dave', 3); CREATE TABLE category ( cid int primary key, cname text ); GRANT ALL ON category TO public; INSERT INTO category VALUES (11, 'novel'), (22, 'science fiction'), (33, 'technology'), (44, 'manga'); CREATE TABLE document ( did int primary key, cid int references category(cid), dlevel int not null, dauthor name, dtitle text ); GRANT ALL ON document TO public; SELECT public.create_hypertable('document', 'did', chunk_time_interval=>2); create_hypertable ----------------------------------- (1,regress_rls_schema,document,t) INSERT INTO document VALUES ( 1, 11, 1, 'regress_rls_bob', 'my first novel'), ( 2, 11, 2, 'regress_rls_bob', 'my second novel'), ( 3, 22, 2, 'regress_rls_bob', 'my science fiction'), ( 4, 44, 1, 'regress_rls_bob', 'my first manga'), ( 5, 44, 2, 'regress_rls_bob', 'my second manga'), ( 6, 22, 1, 'regress_rls_carol', 'great science fiction'), ( 7, 33, 2, 'regress_rls_carol', 'great technology book'), ( 8, 44, 1, 'regress_rls_carol', 'great manga'), ( 9, 22, 1, 'regress_rls_dave', 'awesome science fiction'), (10, 33, 2, 'regress_rls_dave', 'awesome technology book'); ALTER TABLE document ENABLE ROW LEVEL SECURITY; -- user's security level must be higher than or equal to document's CREATE POLICY p1 ON document AS PERMISSIVE USING (dlevel <= (SELECT seclv FROM uaccount WHERE pguser = current_user)); -- try to create a policy of bogus type CREATE POLICY p1 ON document AS UGLY USING (dlevel <= (SELECT seclv FROM uaccount WHERE pguser = current_user)); ERROR: unrecognized row security option "ugly" LINE 1: CREATE POLICY p1 ON document AS UGLY ^ HINT: Only PERMISSIVE or RESTRICTIVE policies are supported currently. -- but Dave isn't allowed to anything at cid 50 or above -- this is to make sure that we sort the policies by name first -- when applying WITH CHECK, a later INSERT by Dave should fail due -- to p1r first CREATE POLICY p2r ON document AS RESTRICTIVE TO regress_rls_dave USING (cid <> 44 AND cid < 50); -- and Dave isn't allowed to see manga documents CREATE POLICY p1r ON document AS RESTRICTIVE TO regress_rls_dave USING (cid <> 44); \dp Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------------------+----------+-------+---------------------------------------------+-------------------+-------------------------------------------- regress_rls_schema | category | table | regress_rls_alice=arwdDxt/regress_rls_alice+| | | | | =arwdDxt/regress_rls_alice | | regress_rls_schema | document | table | regress_rls_alice=arwdDxt/regress_rls_alice+| | p1: + | | | =arwdDxt/regress_rls_alice | | (u): (dlevel <= ( SELECT uaccount.seclv + | | | | | FROM uaccount + | | | | | WHERE (uaccount.pguser = CURRENT_USER)))+ | | | | | p2r (RESTRICTIVE): + | | | | | (u): ((cid <> 44) AND (cid < 50)) + | | | | | to: regress_rls_dave + | | | | | p1r (RESTRICTIVE): + | | | | | (u): (cid <> 44) + | | | | | to: regress_rls_dave regress_rls_schema | uaccount | table | regress_rls_alice=arwdDxt/regress_rls_alice+| | | | | =r/regress_rls_alice | | \d document Table "regress_rls_schema.document" Column | Type | Collation | Nullable | Default ---------+---------+-----------+----------+--------- did | integer | | not null | cid | integer | | | dlevel | integer | | not null | dauthor | name | | | dtitle | text | | | Indexes: "document_pkey" PRIMARY KEY, btree (did) Foreign-key constraints: "document_cid_fkey" FOREIGN KEY (cid) REFERENCES category(cid) Policies: POLICY "p1" USING ((dlevel <= ( SELECT uaccount.seclv FROM uaccount WHERE (uaccount.pguser = CURRENT_USER)))) POLICY "p1r" AS RESTRICTIVE TO regress_rls_dave USING ((cid <> 44)) POLICY "p2r" AS RESTRICTIVE TO regress_rls_dave USING (((cid <> 44) AND (cid < 50))) Number of child tables: 6 (Use \d+ to list them.) SELECT * FROM pg_policies WHERE schemaname = 'regress_rls_schema' AND tablename = 'document' ORDER BY policyname; schemaname | tablename | policyname | permissive | roles | cmd | qual | with_check --------------------+-----------+------------+-------------+--------------------+-----+--------------------------------------------+------------ regress_rls_schema | document | p1 | PERMISSIVE | {public} | ALL | (dlevel <= ( SELECT uaccount.seclv +| | | | | | | FROM uaccount +| | | | | | | WHERE (uaccount.pguser = CURRENT_USER))) | regress_rls_schema | document | p1r | RESTRICTIVE | {regress_rls_dave} | ALL | (cid <> 44) | regress_rls_schema | document | p2r | RESTRICTIVE | {regress_rls_dave} | ALL | ((cid <> 44) AND (cid < 50)) | -- viewpoint from regress_rls_bob SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my first manga NOTICE: f_leak => great science fiction NOTICE: f_leak => great manga NOTICE: f_leak => awesome science fiction did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 4 | 44 | 1 | regress_rls_bob | my first manga 6 | 22 | 1 | regress_rls_carol | great science fiction 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my first manga NOTICE: f_leak => great science fiction NOTICE: f_leak => great manga NOTICE: f_leak => awesome science fiction cid | did | dlevel | dauthor | dtitle | cname -----+-----+--------+-------------------+-------------------------+----------------- 11 | 1 | 1 | regress_rls_bob | my first novel | novel 44 | 4 | 1 | regress_rls_bob | my first manga | manga 22 | 6 | 1 | regress_rls_carol | great science fiction | science fiction 44 | 8 | 1 | regress_rls_carol | great manga | manga 22 | 9 | 1 | regress_rls_dave | awesome science fiction | science fiction -- try a sampled version SELECT * FROM document TABLESAMPLE BERNOULLI(50) REPEATABLE(0) WHERE f_leak(dtitle) ORDER BY did; did | cid | dlevel | dauthor | dtitle -----+-----+--------+---------+-------- -- viewpoint from regress_rls_carol SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => my first manga NOTICE: f_leak => my second manga NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great manga NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => my first manga NOTICE: f_leak => my second manga NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great manga NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book cid | did | dlevel | dauthor | dtitle | cname -----+-----+--------+-------------------+-------------------------+----------------- 11 | 1 | 1 | regress_rls_bob | my first novel | novel 11 | 2 | 2 | regress_rls_bob | my second novel | novel 22 | 3 | 2 | regress_rls_bob | my science fiction | science fiction 44 | 4 | 1 | regress_rls_bob | my first manga | manga 44 | 5 | 2 | regress_rls_bob | my second manga | manga 22 | 6 | 1 | regress_rls_carol | great science fiction | science fiction 33 | 7 | 2 | regress_rls_carol | great technology book | technology 44 | 8 | 1 | regress_rls_carol | great manga | manga 22 | 9 | 1 | regress_rls_dave | awesome science fiction | science fiction 33 | 10 | 2 | regress_rls_dave | awesome technology book | technology -- try a sampled version SELECT * FROM document TABLESAMPLE BERNOULLI(50) REPEATABLE(0) WHERE f_leak(dtitle) ORDER BY did; did | cid | dlevel | dauthor | dtitle -----+-----+--------+---------+-------- EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on document document_1 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((dlevel <= $0) AND f_leak(dtitle)) EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle); --- QUERY PLAN --- Hash Join Hash Cond: (document.cid = category.cid) InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 -> Seq Scan on document document_1 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Hash -> Seq Scan on category -- viewpoint from regress_rls_dave SET SESSION AUTHORIZATION regress_rls_dave; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book cid | did | dlevel | dauthor | dtitle | cname -----+-----+--------+-------------------+-------------------------+----------------- 11 | 1 | 1 | regress_rls_bob | my first novel | novel 11 | 2 | 2 | regress_rls_bob | my second novel | novel 22 | 3 | 2 | regress_rls_bob | my science fiction | science fiction 22 | 6 | 1 | regress_rls_carol | great science fiction | science fiction 33 | 7 | 2 | regress_rls_carol | great technology book | technology 22 | 9 | 1 | regress_rls_dave | awesome science fiction | science fiction 33 | 10 | 2 | regress_rls_dave | awesome technology book | technology EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on document document_1 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle); --- QUERY PLAN --- Hash Join Hash Cond: (category.cid = document.cid) InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on category -> Hash -> Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 -> Seq Scan on document document_1 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= $0) AND f_leak(dtitle)) -- 44 would technically fail for both p2r and p1r, but we should get an error -- back from p1r for this because it sorts first INSERT INTO document VALUES (100, 44, 1, 'regress_rls_dave', 'testing sorting of policies'); -- fail ERROR: new row violates row-level security policy "p1r" for table "document" -- Just to see a p2r error INSERT INTO document VALUES (100, 55, 1, 'regress_rls_dave', 'testing sorting of policies'); -- fail ERROR: new row violates row-level security policy "p2r" for table "document" -- only owner can change policies ALTER POLICY p1 ON document USING (true); --fail ERROR: must be owner of table document DROP POLICY p1 ON document; --fail ERROR: must be owner of relation document SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY p1 ON document USING (dauthor = current_user); -- viewpoint from regress_rls_bob again SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => my first manga NOTICE: f_leak => my second manga did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+-------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER by did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => my first manga NOTICE: f_leak => my second manga cid | did | dlevel | dauthor | dtitle | cname -----+-----+--------+-----------------+--------------------+----------------- 11 | 1 | 1 | regress_rls_bob | my first novel | novel 11 | 2 | 2 | regress_rls_bob | my second novel | novel 22 | 3 | 2 | regress_rls_bob | my science fiction | science fiction 44 | 4 | 1 | regress_rls_bob | my first manga | manga 44 | 5 | 2 | regress_rls_bob | my second manga | manga -- viewpoint from rls_regres_carol again SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great manga did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+----------------------- 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER by did; NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great manga cid | did | dlevel | dauthor | dtitle | cname -----+-----+--------+-------------------+-----------------------+----------------- 22 | 6 | 1 | regress_rls_carol | great science fiction | science fiction 33 | 7 | 2 | regress_rls_carol | great technology book | technology 44 | 8 | 1 | regress_rls_carol | great manga | manga EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 -> Seq Scan on document document_1 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle); --- QUERY PLAN --- Nested Loop -> Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 -> Seq Scan on document document_1 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Index Scan using category_pkey on category Index Cond: (cid = document.cid) -- interaction of FK/PK constraints SET SESSION AUTHORIZATION regress_rls_alice; CREATE POLICY p2 ON category USING (CASE WHEN current_user = 'regress_rls_bob' THEN cid IN (11, 33) WHEN current_user = 'regress_rls_carol' THEN cid IN (22, 44) ELSE false END); ALTER TABLE category ENABLE ROW LEVEL SECURITY; -- cannot delete PK referenced by invisible FK SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM document d FULL OUTER JOIN category c on d.cid = c.cid ORDER BY d.did, c.cid; did | cid | dlevel | dauthor | dtitle | cid | cname -----+-----+--------+-----------------+--------------------+-----+------------ 1 | 11 | 1 | regress_rls_bob | my first novel | 11 | novel 2 | 11 | 2 | regress_rls_bob | my second novel | 11 | novel 3 | 22 | 2 | regress_rls_bob | my science fiction | | 4 | 44 | 1 | regress_rls_bob | my first manga | | 5 | 44 | 2 | regress_rls_bob | my second manga | | | | | | | 33 | technology \set VERBOSITY sqlstate DELETE FROM category WHERE cid = 33; -- fails with FK violation ERROR: 23503 \set VERBOSITY default -- can insert FK referencing invisible PK SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM document d FULL OUTER JOIN category c on d.cid = c.cid ORDER BY d.did, c.cid; did | cid | dlevel | dauthor | dtitle | cid | cname -----+-----+--------+-------------------+-----------------------+-----+----------------- 6 | 22 | 1 | regress_rls_carol | great science fiction | 22 | science fiction 7 | 33 | 2 | regress_rls_carol | great technology book | | 8 | 44 | 1 | regress_rls_carol | great manga | 44 | manga INSERT INTO document VALUES (11, 33, 1, current_user, 'hoge'); -- UNIQUE or PRIMARY KEY constraint violation DOES reveal presence of row SET SESSION AUTHORIZATION regress_rls_bob; INSERT INTO document VALUES (8, 44, 1, 'regress_rls_bob', 'my third manga'); -- Must fail with unique violation, revealing presence of did we can't see ERROR: duplicate key value violates unique constraint "5_10_document_pkey" DETAIL: Key (did)=(8) already exists. SELECT * FROM document WHERE did = 8; -- and confirm we can't see it did | cid | dlevel | dauthor | dtitle -----+-----+--------+---------+-------- -- RLS policies are checked before constraints INSERT INTO document VALUES (8, 44, 1, 'regress_rls_carol', 'my third manga'); -- Should fail with RLS check violation, not duplicate key violation ERROR: new row violates row-level security policy for table "document" UPDATE document SET did = 8, dauthor = 'regress_rls_carol' WHERE did = 5; -- Should fail with RLS check violation, not duplicate key violation ERROR: new row violates row-level security policy for table "document" -- database superuser does bypass RLS policy when enabled RESET SESSION AUTHORIZATION; SET row_security TO ON; SELECT * FROM document; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book 11 | 33 | 1 | regress_rls_carol | hoge SELECT * FROM category; cid | cname -----+----------------- 11 | novel 22 | science fiction 33 | technology 44 | manga -- database superuser does bypass RLS policy when disabled RESET SESSION AUTHORIZATION; SET row_security TO OFF; SELECT * FROM document; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book 11 | 33 | 1 | regress_rls_carol | hoge SELECT * FROM category; cid | cname -----+----------------- 11 | novel 22 | science fiction 33 | technology 44 | manga -- database non-superuser with bypass privilege can bypass RLS policy when disabled SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; SELECT * FROM document; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book 11 | 33 | 1 | regress_rls_carol | hoge SELECT * FROM category; cid | cname -----+----------------- 11 | novel 22 | science fiction 33 | technology 44 | manga -- RLS policy does not apply to table owner when RLS enabled. SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; SELECT * FROM document; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book 11 | 33 | 1 | regress_rls_carol | hoge SELECT * FROM category; cid | cname -----+----------------- 11 | novel 22 | science fiction 33 | technology 44 | manga -- RLS policy does not apply to table owner when RLS disabled. SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO OFF; SELECT * FROM document; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book 11 | 33 | 1 | regress_rls_carol | hoge SELECT * FROM category; cid | cname -----+----------------- 11 | novel 22 | science fiction 33 | technology 44 | manga -- -- Table inheritance and RLS policy -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; CREATE TABLE t1 (a int, junk1 text, b text); ALTER TABLE t1 DROP COLUMN junk1; -- just a disturbing factor GRANT ALL ON t1 TO public; COPY t1 FROM stdin; CREATE TABLE t2 (c float) INHERITS (t1); GRANT ALL ON t2 TO public; COPY t2 FROM stdin; CREATE TABLE t3 (c text, b text, a int); ALTER TABLE t3 INHERIT t1; GRANT ALL ON t3 TO public; COPY t3(a,b,c) FROM stdin; CREATE POLICY p1 ON t1 FOR ALL TO PUBLIC USING (a % 2 = 0); -- be even number CREATE POLICY p2 ON t2 FOR ALL TO PUBLIC USING (a % 2 = 1); -- be odd number ALTER TABLE t1 ENABLE ROW LEVEL SECURITY; ALTER TABLE t2 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM t1; a | b ---+----- 2 | bbb 4 | dad 2 | bcd 4 | def 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: ((a % 2) = 0) -> Seq Scan on t2 t1_2 Filter: ((a % 2) = 0) -> Seq Scan on t3 t1_3 Filter: ((a % 2) = 0) SELECT * FROM t1 WHERE f_leak(b); NOTICE: f_leak => bbb NOTICE: f_leak => dad NOTICE: f_leak => bcd NOTICE: f_leak => def NOTICE: f_leak => yyy a | b ---+----- 2 | bbb 4 | dad 2 | bcd 4 | def 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -- reference to system column SELECT ctid, * FROM t1; ctid | a | b -------+---+----- (0,2) | 2 | bbb (0,4) | 4 | dad (0,2) | 2 | bcd (0,4) | 4 | def (0,2) | 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT *, t1 FROM t1; --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: ((a % 2) = 0) -> Seq Scan on t2 t1_2 Filter: ((a % 2) = 0) -> Seq Scan on t3 t1_3 Filter: ((a % 2) = 0) -- reference to whole-row reference SELECT *, t1 FROM t1; a | b | t1 ---+-----+--------- 2 | bbb | (2,bbb) 4 | dad | (4,dad) 2 | bcd | (2,bcd) 4 | def | (4,def) 2 | yyy | (2,yyy) EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT *, t1 FROM t1; --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: ((a % 2) = 0) -> Seq Scan on t2 t1_2 Filter: ((a % 2) = 0) -> Seq Scan on t3 t1_3 Filter: ((a % 2) = 0) -- for share/update lock SELECT * FROM t1 FOR SHARE; a | b ---+----- 2 | bbb 4 | dad 2 | bcd 4 | def 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 FOR SHARE; --- QUERY PLAN --- LockRows -> Append -> Seq Scan on t1 t1_1 Filter: ((a % 2) = 0) -> Seq Scan on t2 t1_2 Filter: ((a % 2) = 0) -> Seq Scan on t3 t1_3 Filter: ((a % 2) = 0) SELECT * FROM t1 WHERE f_leak(b) FOR SHARE; NOTICE: f_leak => bbb NOTICE: f_leak => dad NOTICE: f_leak => bcd NOTICE: f_leak => def NOTICE: f_leak => yyy a | b ---+----- 2 | bbb 4 | dad 2 | bcd 4 | def 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b) FOR SHARE; --- QUERY PLAN --- LockRows -> Append -> Seq Scan on t1 t1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -- union all query SELECT a, b, ctid FROM t2 UNION ALL SELECT a, b, ctid FROM t3; a | b | ctid ---+-----+------- 1 | abc | (0,1) 3 | cde | (0,3) 1 | xxx | (0,1) 2 | yyy | (0,2) 3 | zzz | (0,3) EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT a, b, ctid FROM t2 UNION ALL SELECT a, b, ctid FROM t3; --- QUERY PLAN --- Append -> Seq Scan on t2 Filter: ((a % 2) = 1) -> Seq Scan on t3 -- superuser is allowed to bypass RLS checks RESET SESSION AUTHORIZATION; SET row_security TO OFF; SELECT * FROM t1 WHERE f_leak(b); NOTICE: f_leak => aba NOTICE: f_leak => bbb NOTICE: f_leak => ccc NOTICE: f_leak => dad NOTICE: f_leak => abc NOTICE: f_leak => bcd NOTICE: f_leak => cde NOTICE: f_leak => def NOTICE: f_leak => xxx NOTICE: f_leak => yyy NOTICE: f_leak => zzz a | b ---+----- 1 | aba 2 | bbb 3 | ccc 4 | dad 1 | abc 2 | bcd 3 | cde 4 | def 1 | xxx 2 | yyy 3 | zzz EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: f_leak(b) -> Seq Scan on t2 t1_2 Filter: f_leak(b) -> Seq Scan on t3 t1_3 Filter: f_leak(b) -- non-superuser with bypass privilege can bypass RLS policy when disabled SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; SELECT * FROM t1 WHERE f_leak(b); NOTICE: f_leak => aba NOTICE: f_leak => bbb NOTICE: f_leak => ccc NOTICE: f_leak => dad NOTICE: f_leak => abc NOTICE: f_leak => bcd NOTICE: f_leak => cde NOTICE: f_leak => def NOTICE: f_leak => xxx NOTICE: f_leak => yyy NOTICE: f_leak => zzz a | b ---+----- 1 | aba 2 | bbb 3 | ccc 4 | dad 1 | abc 2 | bcd 3 | cde 4 | def 1 | xxx 2 | yyy 3 | zzz EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: f_leak(b) -> Seq Scan on t2 t1_2 Filter: f_leak(b) -> Seq Scan on t3 t1_3 Filter: f_leak(b) -- -- Hyper Tables -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE hyper_document ( did int, cid int, dlevel int not null, dauthor name, dtitle text ); GRANT ALL ON hyper_document TO public; SELECT public.create_hypertable('hyper_document', 'did', chunk_time_interval=>2); create_hypertable ----------------------------------------- (2,regress_rls_schema,hyper_document,t) INSERT INTO hyper_document VALUES ( 1, 11, 1, 'regress_rls_bob', 'my first novel'), ( 2, 11, 2, 'regress_rls_bob', 'my second novel'), ( 3, 99, 2, 'regress_rls_bob', 'my science textbook'), ( 4, 55, 1, 'regress_rls_bob', 'my first satire'), ( 5, 99, 2, 'regress_rls_bob', 'my history book'), ( 6, 11, 1, 'regress_rls_carol', 'great science fiction'), ( 7, 99, 2, 'regress_rls_carol', 'great technology book'), ( 8, 55, 2, 'regress_rls_carol', 'great satire'), ( 9, 11, 1, 'regress_rls_dave', 'awesome science fiction'), (10, 99, 2, 'regress_rls_dave', 'awesome technology book'); ALTER TABLE hyper_document ENABLE ROW LEVEL SECURITY; GRANT ALL ON _timescaledb_internal._hyper_2_9_chunk TO public; -- Create policy on parent -- user's security level must be higher than or equal to document's CREATE POLICY pp1 ON hyper_document AS PERMISSIVE USING (dlevel <= (SELECT seclv FROM uaccount WHERE pguser = current_user)); -- Dave is only allowed to see cid < 55 CREATE POLICY pp1r ON hyper_document AS RESTRICTIVE TO regress_rls_dave USING (cid < 55); \d+ hyper_document Table "regress_rls_schema.hyper_document" Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- did | integer | | not null | | plain | | cid | integer | | | | plain | | dlevel | integer | | not null | | plain | | dauthor | name | | | | plain | | dtitle | text | | | | extended | | Indexes: "hyper_document_did_idx" btree (did DESC) Policies: POLICY "pp1" USING ((dlevel <= ( SELECT uaccount.seclv FROM uaccount WHERE (uaccount.pguser = CURRENT_USER)))) POLICY "pp1r" AS RESTRICTIVE TO regress_rls_dave USING ((cid < 55)) Child tables: _timescaledb_internal._hyper_2_10_chunk, _timescaledb_internal._hyper_2_11_chunk, _timescaledb_internal._hyper_2_12_chunk, _timescaledb_internal._hyper_2_13_chunk, _timescaledb_internal._hyper_2_14_chunk, _timescaledb_internal._hyper_2_9_chunk SELECT * FROM pg_policies WHERE schemaname = 'regress_rls_schema' AND tablename like '%hyper_document%' ORDER BY policyname; schemaname | tablename | policyname | permissive | roles | cmd | qual | with_check --------------------+----------------+------------+-------------+--------------------+-----+--------------------------------------------+------------ regress_rls_schema | hyper_document | pp1 | PERMISSIVE | {public} | ALL | (dlevel <= ( SELECT uaccount.seclv +| | | | | | | FROM uaccount +| | | | | | | WHERE (uaccount.pguser = CURRENT_USER))) | regress_rls_schema | hyper_document | pp1r | RESTRICTIVE | {regress_rls_dave} | ALL | (cid < 55) | -- viewpoint from regress_rls_bob SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my first satire NOTICE: f_leak => great science fiction NOTICE: f_leak => awesome science fiction did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 4 | 55 | 1 | regress_rls_bob | my first satire 6 | 11 | 1 | regress_rls_carol | great science fiction 9 | 11 | 1 | regress_rls_dave | awesome science fiction EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on hyper_document hyper_document_1 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -- viewpoint from regress_rls_carol SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science textbook NOTICE: f_leak => my first satire NOTICE: f_leak => my history book NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great satire NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire 9 | 11 | 1 | regress_rls_dave | awesome science fiction 10 | 99 | 2 | regress_rls_dave | awesome technology book EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on hyper_document hyper_document_1 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -- viewpoint from regress_rls_dave SET SESSION AUTHORIZATION regress_rls_dave; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => great science fiction NOTICE: f_leak => awesome science fiction did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 6 | 11 | 1 | regress_rls_carol | great science fiction 9 | 11 | 1 | regress_rls_dave | awesome science fiction EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on hyper_document hyper_document_1 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -- pp1 ERROR INSERT INTO hyper_document VALUES (1, 11, 5, 'regress_rls_dave', 'testing pp1'); -- fail ERROR: new row violates row-level security policy for table "hyper_document" -- pp1r ERROR INSERT INTO hyper_document VALUES (1, 99, 1, 'regress_rls_dave', 'testing pp1r'); -- fail ERROR: new row violates row-level security policy "pp1r" for table "hyper_document" -- Show that RLS policy does not apply for direct inserts to children -- This should fail with RLS POLICY pp1r violation. INSERT INTO hyper_document VALUES (1, 55, 1, 'regress_rls_dave', 'testing RLS with hypertables'); -- fail ERROR: new row violates row-level security policy "pp1r" for table "hyper_document" -- But this should succeed. INSERT INTO _timescaledb_internal._hyper_2_9_chunk VALUES (1, 55, 1, 'regress_rls_dave', 'testing RLS with hypertables'); -- success -- We still cannot see the row using the parent SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => great science fiction NOTICE: f_leak => awesome science fiction did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 6 | 11 | 1 | regress_rls_carol | great science fiction 9 | 11 | 1 | regress_rls_dave | awesome science fiction -- But we can if we look directly SELECT * FROM _timescaledb_internal._hyper_2_9_chunk WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => my first novel NOTICE: f_leak => testing RLS with hypertables did | cid | dlevel | dauthor | dtitle -----+-----+--------+------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables -- Turn on RLS and create policy on child to show RLS is checked before constraints SET SESSION AUTHORIZATION regress_rls_alice; ALTER TABLE _timescaledb_internal._hyper_2_9_chunk ENABLE ROW LEVEL SECURITY; CREATE POLICY pp3 ON _timescaledb_internal._hyper_2_9_chunk AS RESTRICTIVE USING (cid < 55); -- This should fail with RLS violation now. SET SESSION AUTHORIZATION regress_rls_dave; INSERT INTO _timescaledb_internal._hyper_2_9_chunk VALUES (1, 55, 1, 'regress_rls_dave', 'testing RLS with hypertables - round 2'); -- fail ERROR: new row violates row-level security policy for table "_hyper_2_9_chunk" -- And now we cannot see directly into the partition either, due to RLS SELECT * FROM _timescaledb_internal._hyper_2_9_chunk WHERE f_leak(dtitle) ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+---------+-------- -- The parent looks same as before -- viewpoint from regress_rls_dave SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => great science fiction NOTICE: f_leak => awesome science fiction did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 6 | 11 | 1 | regress_rls_carol | great science fiction 9 | 11 | 1 | regress_rls_dave | awesome science fiction EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on hyper_document hyper_document_1 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((cid < 55) AND (dlevel <= $0) AND f_leak(dtitle)) -- viewpoint from regress_rls_carol SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => my first novel NOTICE: f_leak => testing RLS with hypertables NOTICE: f_leak => my second novel NOTICE: f_leak => my science textbook NOTICE: f_leak => my first satire NOTICE: f_leak => my history book NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great satire NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire 9 | 11 | 1 | regress_rls_dave | awesome science fiction 10 | 99 | 2 | regress_rls_dave | awesome technology book EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 InitPlan 1 (returns $0) -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on hyper_document hyper_document_1 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((dlevel <= $0) AND f_leak(dtitle)) -- only owner can change policies ALTER POLICY pp1 ON hyper_document USING (true); --fail ERROR: must be owner of table hyper_document DROP POLICY pp1 ON hyper_document; --fail ERROR: must be owner of relation hyper_document SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY pp1 ON hyper_document USING (dauthor = current_user); -- viewpoint from regress_rls_bob again SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science textbook NOTICE: f_leak => my first satire NOTICE: f_leak => my history book did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+--------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book -- viewpoint from rls_regres_carol again SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great satire did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+----------------------- 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 -> Seq Scan on hyper_document hyper_document_1 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -- database superuser does bypass RLS policy when enabled RESET SESSION AUTHORIZATION; SET row_security TO ON; SELECT * FROM hyper_document ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire 9 | 11 | 1 | regress_rls_dave | awesome science fiction 10 | 99 | 2 | regress_rls_dave | awesome technology book SELECT * FROM _timescaledb_internal._hyper_2_9_chunk ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables -- database non-superuser with bypass privilege can bypass RLS policy when disabled SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; SELECT * FROM hyper_document ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire 9 | 11 | 1 | regress_rls_dave | awesome science fiction 10 | 99 | 2 | regress_rls_dave | awesome technology book SELECT * FROM _timescaledb_internal._hyper_2_9_chunk ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables -- RLS policy does not apply to table owner when RLS enabled. SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; SELECT * FROM hyper_document ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire 9 | 11 | 1 | regress_rls_dave | awesome science fiction 10 | 99 | 2 | regress_rls_dave | awesome technology book SELECT * FROM _timescaledb_internal._hyper_2_9_chunk ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables -- When RLS disabled, other users get ERROR. SET SESSION AUTHORIZATION regress_rls_dave; SET row_security TO OFF; SELECT * FROM hyper_document ORDER BY did, cid; ERROR: query would be affected by row-level security policy for table "hyper_document" SELECT * FROM _timescaledb_internal._hyper_2_9_chunk ORDER BY did, cid; ERROR: query would be affected by row-level security policy for table "_hyper_2_9_chunk" -- Check behavior with a policy that uses a SubPlan not an InitPlan. SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; CREATE POLICY pp3 ON hyper_document AS RESTRICTIVE USING ((SELECT dlevel <= seclv FROM uaccount WHERE pguser = current_user)); SET SESSION AUTHORIZATION regress_rls_carol; INSERT INTO hyper_document VALUES (100, 11, 5, 'regress_rls_carol', 'testing pp3'); -- fail ERROR: new row violates row-level security policy "pp3" for table "hyper_document" ----- Dependencies ----- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; CREATE TABLE dependee (x integer, y integer); SELECT public.create_hypertable('dependee', 'x', chunk_time_interval=>2); create_hypertable ----------------------------------- (3,regress_rls_schema,dependee,t) CREATE TABLE dependent (x integer, y integer); SELECT public.create_hypertable('dependent', 'x', chunk_time_interval=>2); create_hypertable ------------------------------------ (4,regress_rls_schema,dependent,t) CREATE POLICY d1 ON dependent FOR ALL TO PUBLIC USING (x = (SELECT d.x FROM dependee d WHERE d.y = y)); DROP TABLE dependee; -- Should fail without CASCADE due to dependency on row security qual? ERROR: cannot drop table dependee because other objects depend on it DETAIL: policy d1 on table dependent depends on table dependee HINT: Use DROP ... CASCADE to drop the dependent objects too. DROP TABLE dependee CASCADE; NOTICE: drop cascades to policy d1 on table dependent EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM dependent; -- After drop, should be unqualified --- QUERY PLAN --- Seq Scan on dependent ----- RECURSION ---- -- -- Simple recursion -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE rec1 (x integer, y integer); SELECT public.create_hypertable('rec1', 'x', chunk_time_interval=>2); create_hypertable ------------------------------- (5,regress_rls_schema,rec1,t) CREATE POLICY r1 ON rec1 USING (x = (SELECT r.x FROM rec1 r WHERE y = r.y)); ALTER TABLE rec1 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rec1; -- fail, direct recursion ERROR: infinite recursion detected in policy for relation "rec1" -- -- Mutual recursion -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE rec2 (a integer, b integer); SELECT public.create_hypertable('rec2', 'x', chunk_time_interval=>2); ERROR: column "x" does not exist ALTER POLICY r1 ON rec1 USING (x = (SELECT a FROM rec2 WHERE b = y)); CREATE POLICY r2 ON rec2 USING (a = (SELECT x FROM rec1 WHERE y = b)); ALTER TABLE rec2 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rec1; -- fail, mutual recursion ERROR: infinite recursion detected in policy for relation "rec1" -- -- Mutual recursion via views -- SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW rec1v AS SELECT * FROM rec1; CREATE VIEW rec2v AS SELECT * FROM rec2; SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY r1 ON rec1 USING (x = (SELECT a FROM rec2v WHERE b = y)); ALTER POLICY r2 ON rec2 USING (a = (SELECT x FROM rec1v WHERE y = b)); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rec1; -- fail, mutual recursion via views ERROR: infinite recursion detected in policy for relation "rec1" -- -- Mutual recursion via .s.b views -- SET SESSION AUTHORIZATION regress_rls_bob; \set VERBOSITY terse \\ -- suppress cascade details DROP VIEW rec1v, rec2v CASCADE; NOTICE: drop cascades to 2 other objects \set VERBOSITY default CREATE VIEW rec1v WITH (security_barrier) AS SELECT * FROM rec1; CREATE VIEW rec2v WITH (security_barrier) AS SELECT * FROM rec2; SET SESSION AUTHORIZATION regress_rls_alice; CREATE POLICY r1 ON rec1 USING (x = (SELECT a FROM rec2v WHERE b = y)); CREATE POLICY r2 ON rec2 USING (a = (SELECT x FROM rec1v WHERE y = b)); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rec1; -- fail, mutual recursion via s.b. views ERROR: infinite recursion detected in policy for relation "rec1" -- -- recursive RLS and VIEWs in policy -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE s1 (a int, b text); SELECT public.create_hypertable('s1', 'a', chunk_time_interval=>2); create_hypertable ----------------------------- (6,regress_rls_schema,s1,t) INSERT INTO s1 (SELECT x, md5(x::text) FROM generate_series(-10,10) x); CREATE TABLE s2 (x int, y text); SELECT public.create_hypertable('s2', 'x', chunk_time_interval=>2); create_hypertable ----------------------------- (7,regress_rls_schema,s2,t) INSERT INTO s2 (SELECT x, md5(x::text) FROM generate_series(-6,6) x); GRANT SELECT ON s1, s2 TO regress_rls_bob; CREATE POLICY p1 ON s1 USING (a in (select x from s2 where y like '%2f%')); CREATE POLICY p2 ON s2 USING (x in (select a from s1 where b like '%22%')); CREATE POLICY p3 ON s1 FOR INSERT WITH CHECK (a = (SELECT a FROM s1)); ALTER TABLE s1 ENABLE ROW LEVEL SECURITY; ALTER TABLE s2 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW v2 AS SELECT * FROM s2 WHERE y like '%af%'; SELECT * FROM s1 WHERE f_leak(b); -- fail (infinite recursion) ERROR: infinite recursion detected in policy for relation "s1" INSERT INTO s1 VALUES (1, 'foo'); -- fail (infinite recursion) ERROR: infinite recursion detected in policy for relation "s1" SET SESSION AUTHORIZATION regress_rls_alice; DROP POLICY p3 on s1; ALTER POLICY p2 ON s2 USING (x % 2 = 0); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM s1 WHERE f_leak(b); -- OK NOTICE: f_leak => c81e728d9d4c2f636f067f89cc14862c NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c a | b ---+---------------------------------- 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM only s1 WHERE f_leak(b); --- QUERY PLAN --- Seq Scan on s1 Filter: ((hashed SubPlan 1) AND f_leak(b)) SubPlan 1 -> Append -> Seq Scan on s2 s2_1 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_27_chunk s2_2 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_28_chunk s2_3 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_29_chunk s2_4 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_30_chunk s2_5 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_31_chunk s2_6 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_32_chunk s2_7 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_33_chunk s2_8 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY p1 ON s1 USING (a in (select x from v2)); -- using VIEW in RLS policy SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM s1 WHERE f_leak(b); -- OK NOTICE: f_leak => 0267aaf632e87a63288a08331f22c7c3 NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc a | b ----+---------------------------------- -4 | 0267aaf632e87a63288a08331f22c7c3 6 | 1679091c5a880faf6fb5e6087eb1b2dc EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM s1 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on s1 Chunks excluded during startup: 0 -> Seq Scan on s1 s1_1 Filter: ((hashed SubPlan 1) AND f_leak(b)) SubPlan 1 -> Append -> Seq Scan on s2 s2_1 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_27_chunk s2_2 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_28_chunk s2_3 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_29_chunk s2_4 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_30_chunk s2_5 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_31_chunk s2_6 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_32_chunk s2_7 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_33_chunk s2_8 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_6_16_chunk s1_2 Filter: ((hashed SubPlan 1) AND f_leak(b)) -> Seq Scan on _hyper_6_17_chunk s1_3 Filter: ((hashed SubPlan 1) AND f_leak(b)) -> Seq Scan on _hyper_6_18_chunk s1_4 Filter: ((hashed SubPlan 1) AND f_leak(b)) -> Seq Scan on _hyper_6_19_chunk s1_5 Filter: ((hashed SubPlan 1) AND f_leak(b)) -> Seq Scan on _hyper_6_20_chunk s1_6 Filter: ((hashed SubPlan 1) AND f_leak(b)) -> Seq Scan on _hyper_6_21_chunk s1_7 Filter: ((hashed SubPlan 1) AND f_leak(b)) -> Seq Scan on _hyper_6_22_chunk s1_8 Filter: ((hashed SubPlan 1) AND f_leak(b)) -> Seq Scan on _hyper_6_23_chunk s1_9 Filter: ((hashed SubPlan 1) AND f_leak(b)) -> Seq Scan on _hyper_6_24_chunk s1_10 Filter: ((hashed SubPlan 1) AND f_leak(b)) -> Seq Scan on _hyper_6_25_chunk s1_11 Filter: ((hashed SubPlan 1) AND f_leak(b)) -> Seq Scan on _hyper_6_26_chunk s1_12 Filter: ((hashed SubPlan 1) AND f_leak(b)) SELECT (SELECT x FROM s1 LIMIT 1) xx, * FROM s2 WHERE y like '%28%'; xx | x | y ----+----+---------------------------------- -6 | -6 | 596a3d04481816330f07e4f97510c28f -4 | -4 | 0267aaf632e87a63288a08331f22c7c3 2 | 2 | c81e728d9d4c2f636f067f89cc14862c EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT (SELECT x FROM s1 LIMIT 1) xx, * FROM s2 WHERE y like '%28%'; --- QUERY PLAN --- Result -> Append -> Seq Scan on s2 s2_1 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_27_chunk s2_2 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_28_chunk s2_3 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_29_chunk s2_4 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_30_chunk s2_5 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_31_chunk s2_6 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_32_chunk s2_7 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_33_chunk s2_8 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) SubPlan 2 -> Limit -> Result -> Custom Scan (ChunkAppend) on s1 -> Seq Scan on s1 s1_1 Filter: (hashed SubPlan 1) SubPlan 1 -> Append -> Seq Scan on s2 s2_10 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_27_chunk s2_11 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_28_chunk s2_12 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_29_chunk s2_13 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_30_chunk s2_14 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_31_chunk s2_15 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_32_chunk s2_16 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_33_chunk s2_17 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_6_16_chunk s1_2 Filter: (hashed SubPlan 1) -> Seq Scan on _hyper_6_17_chunk s1_3 Filter: (hashed SubPlan 1) -> Seq Scan on _hyper_6_18_chunk s1_4 Filter: (hashed SubPlan 1) -> Seq Scan on _hyper_6_19_chunk s1_5 Filter: (hashed SubPlan 1) -> Seq Scan on _hyper_6_20_chunk s1_6 Filter: (hashed SubPlan 1) -> Seq Scan on _hyper_6_21_chunk s1_7 Filter: (hashed SubPlan 1) -> Seq Scan on _hyper_6_22_chunk s1_8 Filter: (hashed SubPlan 1) -> Seq Scan on _hyper_6_23_chunk s1_9 Filter: (hashed SubPlan 1) -> Seq Scan on _hyper_6_24_chunk s1_10 Filter: (hashed SubPlan 1) -> Seq Scan on _hyper_6_25_chunk s1_11 Filter: (hashed SubPlan 1) -> Seq Scan on _hyper_6_26_chunk s1_12 Filter: (hashed SubPlan 1) SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY p2 ON s2 USING (x in (select a from s1 where b like '%d2%')); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM s1 WHERE f_leak(b); -- fail (infinite recursion via view) ERROR: infinite recursion detected in policy for relation "s1" -- prepared statement with regress_rls_alice privilege PREPARE p1(int) AS SELECT * FROM t1 WHERE a <= $1; EXECUTE p1(2); a | b ---+----- 2 | bbb 2 | bcd 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE p1(2); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: ((a <= 2) AND ((a % 2) = 0)) -> Seq Scan on t2 t1_2 Filter: ((a <= 2) AND ((a % 2) = 0)) -> Seq Scan on t3 t1_3 Filter: ((a <= 2) AND ((a % 2) = 0)) -- superuser is allowed to bypass RLS checks RESET SESSION AUTHORIZATION; SET row_security TO OFF; SELECT * FROM t1 WHERE f_leak(b); NOTICE: f_leak => aba NOTICE: f_leak => bbb NOTICE: f_leak => ccc NOTICE: f_leak => dad NOTICE: f_leak => abc NOTICE: f_leak => bcd NOTICE: f_leak => cde NOTICE: f_leak => def NOTICE: f_leak => xxx NOTICE: f_leak => yyy NOTICE: f_leak => zzz a | b ---+----- 1 | aba 2 | bbb 3 | ccc 4 | dad 1 | abc 2 | bcd 3 | cde 4 | def 1 | xxx 2 | yyy 3 | zzz EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: f_leak(b) -> Seq Scan on t2 t1_2 Filter: f_leak(b) -> Seq Scan on t3 t1_3 Filter: f_leak(b) -- plan cache should be invalidated EXECUTE p1(2); a | b ---+----- 1 | aba 2 | bbb 1 | abc 2 | bcd 1 | xxx 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE p1(2); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: (a <= 2) -> Seq Scan on t2 t1_2 Filter: (a <= 2) -> Seq Scan on t3 t1_3 Filter: (a <= 2) PREPARE p2(int) AS SELECT * FROM t1 WHERE a = $1; EXECUTE p2(2); a | b ---+----- 2 | bbb 2 | bcd 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE p2(2); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: (a = 2) -> Seq Scan on t2 t1_2 Filter: (a = 2) -> Seq Scan on t3 t1_3 Filter: (a = 2) -- also, case when privilege switch from superuser SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; EXECUTE p2(2); a | b ---+----- 2 | bbb 2 | bcd 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE p2(2); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: ((a = 2) AND ((a % 2) = 0)) -> Seq Scan on t2 t1_2 Filter: ((a = 2) AND ((a % 2) = 0)) -> Seq Scan on t3 t1_3 Filter: ((a = 2) AND ((a % 2) = 0)) -- -- UPDATE / DELETE and Row-level security -- SET SESSION AUTHORIZATION regress_rls_bob; EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t1 SET b = b || b WHERE f_leak(b); --- QUERY PLAN --- Update on t1 Update on t1 t1_1 Update on t2 t1_2 Update on t3 t1_3 -> Result -> Append -> Seq Scan on t1 t1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: (((a % 2) = 0) AND f_leak(b)) UPDATE t1 SET b = b || b WHERE f_leak(b); NOTICE: f_leak => bbb NOTICE: f_leak => dad NOTICE: f_leak => bcd NOTICE: f_leak => def NOTICE: f_leak => yyy EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE only t1 SET b = b || '_updt' WHERE f_leak(b); --- QUERY PLAN --- Update on t1 -> Seq Scan on t1 Filter: (((a % 2) = 0) AND f_leak(b)) UPDATE only t1 SET b = b || '_updt' WHERE f_leak(b); NOTICE: f_leak => bbbbbb NOTICE: f_leak => daddad -- returning clause with system column UPDATE only t1 SET b = b WHERE f_leak(b) RETURNING ctid, *, t1; NOTICE: f_leak => bbbbbb_updt NOTICE: f_leak => daddad_updt ctid | a | b | t1 --------+---+-------------+----------------- (0,9) | 2 | bbbbbb_updt | (2,bbbbbb_updt) (0,10) | 4 | daddad_updt | (4,daddad_updt) UPDATE t1 SET b = b WHERE f_leak(b) RETURNING *; NOTICE: f_leak => bbbbbb_updt NOTICE: f_leak => daddad_updt NOTICE: f_leak => bcdbcd NOTICE: f_leak => defdef NOTICE: f_leak => yyyyyy a | b ---+------------- 2 | bbbbbb_updt 4 | daddad_updt 2 | bcdbcd 4 | defdef 2 | yyyyyy UPDATE t1 SET b = b WHERE f_leak(b) RETURNING ctid, *, t1; NOTICE: f_leak => bbbbbb_updt NOTICE: f_leak => daddad_updt NOTICE: f_leak => bcdbcd NOTICE: f_leak => defdef NOTICE: f_leak => yyyyyy ctid | a | b | t1 --------+---+-------------+----------------- (0,13) | 2 | bbbbbb_updt | (2,bbbbbb_updt) (0,14) | 4 | daddad_updt | (4,daddad_updt) (0,9) | 2 | bcdbcd | (2,bcdbcd) (0,10) | 4 | defdef | (4,defdef) (0,6) | 2 | yyyyyy | (2,yyyyyy) -- updates with from clause EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t2 SET b=t2.b FROM t3 WHERE t2.a = 3 and t3.a = 2 AND f_leak(t2.b) AND f_leak(t3.b); --- QUERY PLAN --- Update on t2 -> Nested Loop -> Seq Scan on t2 Filter: ((a = 3) AND ((a % 2) = 1) AND f_leak(b)) -> Seq Scan on t3 Filter: ((a = 2) AND f_leak(b)) UPDATE t2 SET b=t2.b FROM t3 WHERE t2.a = 3 and t3.a = 2 AND f_leak(t2.b) AND f_leak(t3.b); NOTICE: f_leak => cde NOTICE: f_leak => yyyyyy EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t1 SET b=t1.b FROM t2 WHERE t1.a = 3 and t2.a = 3 AND f_leak(t1.b) AND f_leak(t2.b); --- QUERY PLAN --- Update on t1 Update on t1 t1_1 Update on t2 t1_2 Update on t3 t1_3 -> Nested Loop -> Seq Scan on t2 Filter: ((a = 3) AND ((a % 2) = 1) AND f_leak(b)) -> Append -> Seq Scan on t1 t1_1 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) UPDATE t1 SET b=t1.b FROM t2 WHERE t1.a = 3 and t2.a = 3 AND f_leak(t1.b) AND f_leak(t2.b); NOTICE: f_leak => cde EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t2 SET b=t2.b FROM t1 WHERE t1.a = 3 and t2.a = 3 AND f_leak(t1.b) AND f_leak(t2.b); --- QUERY PLAN --- Update on t2 -> Nested Loop -> Seq Scan on t2 Filter: ((a = 3) AND ((a % 2) = 1) AND f_leak(b)) -> Append -> Seq Scan on t1 t1_1 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) UPDATE t2 SET b=t2.b FROM t1 WHERE t1.a = 3 and t2.a = 3 AND f_leak(t1.b) AND f_leak(t2.b); NOTICE: f_leak => cde -- updates with from clause self join EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t2 t2_1 SET b = t2_2.b FROM t2 t2_2 WHERE t2_1.a = 3 AND t2_2.a = t2_1.a AND t2_2.b = t2_1.b AND f_leak(t2_1.b) AND f_leak(t2_2.b) RETURNING *, t2_1, t2_2; --- QUERY PLAN --- Update on t2 t2_1 -> Nested Loop Join Filter: (t2_1.b = t2_2.b) -> Seq Scan on t2 t2_1 Filter: ((a = 3) AND ((a % 2) = 1) AND f_leak(b)) -> Seq Scan on t2 t2_2 Filter: ((a = 3) AND ((a % 2) = 1) AND f_leak(b)) UPDATE t2 t2_1 SET b = t2_2.b FROM t2 t2_2 WHERE t2_1.a = 3 AND t2_2.a = t2_1.a AND t2_2.b = t2_1.b AND f_leak(t2_1.b) AND f_leak(t2_2.b) RETURNING *, t2_1, t2_2; NOTICE: f_leak => cde NOTICE: f_leak => cde a | b | c | a | b | c | t2_1 | t2_2 ---+-----+-----+---+-----+-----+-------------+------------- 3 | cde | 3.3 | 3 | cde | 3.3 | (3,cde,3.3) | (3,cde,3.3) EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t1 t1_1 SET b = t1_2.b FROM t1 t1_2 WHERE t1_1.a = 4 AND t1_2.a = t1_1.a AND t1_2.b = t1_1.b AND f_leak(t1_1.b) AND f_leak(t1_2.b) RETURNING *, t1_1, t1_2; --- QUERY PLAN --- Update on t1 t1_1 Update on t1 t1_1_1 Update on t2 t1_1_2 Update on t3 t1_1_3 -> Nested Loop Join Filter: (t1_1.b = t1_2.b) -> Append -> Seq Scan on t1 t1_1_1 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_1_2 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_1_3 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) -> Materialize -> Append -> Seq Scan on t1 t1_2_1 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2_2 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_2_3 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) UPDATE t1 t1_1 SET b = t1_2.b FROM t1 t1_2 WHERE t1_1.a = 4 AND t1_2.a = t1_1.a AND t1_2.b = t1_1.b AND f_leak(t1_1.b) AND f_leak(t1_2.b) RETURNING *, t1_1, t1_2; NOTICE: f_leak => daddad_updt NOTICE: f_leak => daddad_updt NOTICE: f_leak => defdef NOTICE: f_leak => defdef a | b | a | b | t1_1 | t1_2 ---+-------------+---+-------------+-----------------+----------------- 4 | daddad_updt | 4 | daddad_updt | (4,daddad_updt) | (4,daddad_updt) 4 | defdef | 4 | defdef | (4,defdef) | (4,defdef) RESET SESSION AUTHORIZATION; SET row_security TO OFF; SELECT * FROM t1 ORDER BY a,b; a | b ---+------------- 1 | aba 1 | abc 1 | xxx 2 | bbbbbb_updt 2 | bcdbcd 2 | yyyyyy 3 | ccc 3 | cde 3 | zzz 4 | daddad_updt 4 | defdef SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; EXPLAIN (BUFFERS OFF, COSTS OFF) DELETE FROM only t1 WHERE f_leak(b); --- QUERY PLAN --- Delete on t1 -> Seq Scan on t1 Filter: (((a % 2) = 0) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) DELETE FROM t1 WHERE f_leak(b); --- QUERY PLAN --- Delete on t1 Delete on t1 t1_1 Delete on t2 t1_2 Delete on t3 t1_3 -> Append -> Seq Scan on t1 t1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: (((a % 2) = 0) AND f_leak(b)) DELETE FROM only t1 WHERE f_leak(b) RETURNING ctid, *, t1; NOTICE: f_leak => bbbbbb_updt NOTICE: f_leak => daddad_updt ctid | a | b | t1 --------+---+-------------+----------------- (0,13) | 2 | bbbbbb_updt | (2,bbbbbb_updt) (0,15) | 4 | daddad_updt | (4,daddad_updt) DELETE FROM t1 WHERE f_leak(b) RETURNING ctid, *, t1; NOTICE: f_leak => bcdbcd NOTICE: f_leak => defdef NOTICE: f_leak => yyyyyy ctid | a | b | t1 --------+---+--------+------------ (0,9) | 2 | bcdbcd | (2,bcdbcd) (0,13) | 4 | defdef | (4,defdef) (0,6) | 2 | yyyyyy | (2,yyyyyy) -- -- S.b. view on top of Row-level security -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE b1 (a int, b text); SELECT public.create_hypertable('b1', 'a', chunk_time_interval=>2); create_hypertable ----------------------------- (8,regress_rls_schema,b1,t) INSERT INTO b1 (SELECT x, md5(x::text) FROM generate_series(-10,10) x); CREATE POLICY p1 ON b1 USING (a % 2 = 0); ALTER TABLE b1 ENABLE ROW LEVEL SECURITY; GRANT ALL ON b1 TO regress_rls_bob; SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW bv1 WITH (security_barrier) AS SELECT * FROM b1 WHERE a > 0 WITH CHECK OPTION; GRANT ALL ON bv1 TO regress_rls_carol; SET SESSION AUTHORIZATION regress_rls_carol; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM bv1 WHERE f_leak(b); --- QUERY PLAN --- Subquery Scan on bv1 Filter: f_leak(bv1.b) -> Append -> Seq Scan on b1 b1_1 Filter: ((a > 0) AND ((a % 2) = 0)) -> Index Scan using _hyper_8_39_chunk_b1_a_idx on _hyper_8_39_chunk b1_2 Index Cond: (a > 0) Filter: ((a % 2) = 0) -> Index Scan using _hyper_8_40_chunk_b1_a_idx on _hyper_8_40_chunk b1_3 Index Cond: (a > 0) Filter: ((a % 2) = 0) -> Index Scan using _hyper_8_41_chunk_b1_a_idx on _hyper_8_41_chunk b1_4 Index Cond: (a > 0) Filter: ((a % 2) = 0) -> Index Scan using _hyper_8_42_chunk_b1_a_idx on _hyper_8_42_chunk b1_5 Index Cond: (a > 0) Filter: ((a % 2) = 0) -> Index Scan using _hyper_8_43_chunk_b1_a_idx on _hyper_8_43_chunk b1_6 Index Cond: (a > 0) Filter: ((a % 2) = 0) -> Index Scan using _hyper_8_44_chunk_b1_a_idx on _hyper_8_44_chunk b1_7 Index Cond: (a > 0) Filter: ((a % 2) = 0) SELECT * FROM bv1 WHERE f_leak(b); NOTICE: f_leak => c81e728d9d4c2f636f067f89cc14862c NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc NOTICE: f_leak => c9f0f895fb98ab9159f51fd0297e236d NOTICE: f_leak => d3d9446802a44259755d38e6d163e820 a | b ----+---------------------------------- 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 INSERT INTO bv1 VALUES (-1, 'xxx'); -- should fail view WCO ERROR: new row violates row-level security policy for table "b1" INSERT INTO bv1 VALUES (11, 'xxx'); -- should fail RLS check ERROR: new row violates row-level security policy for table "b1" INSERT INTO bv1 VALUES (12, 'xxx'); -- ok EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE bv1 SET b = 'yyy' WHERE a = 4 AND f_leak(b); --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Update on b1 Update on _hyper_8_41_chunk b1_1 -> Result -> Custom Scan (ChunkAppend) on b1 Chunks excluded during startup: 0 -> Index Scan using _hyper_8_41_chunk_b1_a_idx on _hyper_8_41_chunk b1_1 Index Cond: ((a > 0) AND (a = 4)) Filter: (((a % 2) = 0) AND f_leak(b)) UPDATE bv1 SET b = 'yyy' WHERE a = 4 AND f_leak(b); NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c EXPLAIN (BUFFERS OFF, COSTS OFF) DELETE FROM bv1 WHERE a = 6 AND f_leak(b); --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Delete on b1 Delete on _hyper_8_42_chunk b1_1 -> Custom Scan (ChunkAppend) on b1 Chunks excluded during startup: 0 -> Index Scan using _hyper_8_42_chunk_b1_a_idx on _hyper_8_42_chunk b1_1 Index Cond: ((a > 0) AND (a = 6)) Filter: (((a % 2) = 0) AND f_leak(b)) DELETE FROM bv1 WHERE a = 6 AND f_leak(b); NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM b1; a | b -----+---------------------------------- -10 | 1b0fd9efa5279c4203b7c70233f86dbf -9 | 252e691406782824eec43d7eadc3d256 -8 | a8d2ec85eaf98407310b72eb73dda247 -7 | 74687a12d3915d3c4d83f1af7b3683d5 -6 | 596a3d04481816330f07e4f97510c28f -5 | 47c1b025fa18ea96c33fbb6718688c0f -4 | 0267aaf632e87a63288a08331f22c7c3 -3 | b3149ecea4628efd23d2f86e5a723472 -2 | 5d7b9adcbe1c629ec722529dd12e5129 -1 | 6bb61e3b7bce0931da574d19d1d82c88 0 | cfcd208495d565ef66e7dff9f98764da 1 | c4ca4238a0b923820dcc509a6f75849b 2 | c81e728d9d4c2f636f067f89cc14862c 3 | eccbc87e4b5ce2fe28308fd9f2a7baf3 5 | e4da3b7fbbce2345d7772b0674a318d5 4 | yyy 7 | 8f14e45fceea167a5a36dedd4bea2543 8 | c9f0f895fb98ab9159f51fd0297e236d 9 | 45c48cce2e2d7fbdea1afc51c7c6ad26 10 | d3d9446802a44259755d38e6d163e820 12 | xxx -- -- INSERT ... ON CONFLICT DO UPDATE and Row-level security -- SET SESSION AUTHORIZATION regress_rls_alice; DROP POLICY p1 ON document; DROP POLICY p1r ON document; CREATE POLICY p1 ON document FOR SELECT USING (true); CREATE POLICY p2 ON document FOR INSERT WITH CHECK (dauthor = current_user); CREATE POLICY p3 ON document FOR UPDATE USING (cid = (SELECT cid from category WHERE cname = 'novel')) WITH CHECK (dauthor = current_user); SET SESSION AUTHORIZATION regress_rls_bob; -- Exists... SELECT * FROM document WHERE did = 2; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+----------------- 2 | 11 | 2 | regress_rls_bob | my second novel -- ...so violates actual WITH CHECK OPTION within UPDATE (not INSERT, since -- alternative UPDATE path happens to be taken): INSERT INTO document VALUES (2, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_carol', 'my first novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, dauthor = EXCLUDED.dauthor; ERROR: new row violates row-level security policy for table "document" -- Violates USING qual for UPDATE policy p3. -- -- UPDATE path is taken, but UPDATE fails purely because *existing* row to be -- updated is not a "novel"/cid 11 (row is not leaked, even though we have -- SELECT privileges sufficient to see the row in this instance): INSERT INTO document VALUES (33, 22, 1, 'regress_rls_bob', 'okay science fiction'); -- preparation for next statement INSERT INTO document VALUES (33, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'Some novel, replaces sci-fi') -- takes UPDATE path ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle; ERROR: new row violates row-level security policy (USING expression) for table "document" -- Fine (we UPDATE, since INSERT WCOs and UPDATE security barrier quals + WCOs -- not violated): INSERT INTO document VALUES (2, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'my first novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+---------------- 2 | 11 | 2 | regress_rls_bob | my first novel -- Fine (we INSERT, so "cid = 33" ("technology") isn't evaluated): INSERT INTO document VALUES (78, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'some technology novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33 RETURNING *; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+----------------------- 78 | 11 | 1 | regress_rls_bob | some technology novel -- Fine (same query, but we UPDATE, so "cid = 33", ("technology") is not the -- case in respect of *existing* tuple): INSERT INTO document VALUES (78, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'some technology novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33 RETURNING *; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+----------------------- 78 | 33 | 1 | regress_rls_bob | some technology novel -- Same query a third time, but now fails due to existing tuple finally not -- passing quals: INSERT INTO document VALUES (78, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'some technology novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33 RETURNING *; ERROR: new row violates row-level security policy (USING expression) for table "document" -- Don't fail just because INSERT doesn't satisfy WITH CHECK option that -- originated as a barrier/USING() qual from the UPDATE. Note that the UPDATE -- path *isn't* taken, and so UPDATE-related policy does not apply: INSERT INTO document VALUES (79, (SELECT cid from category WHERE cname = 'technology'), 1, 'regress_rls_bob', 'technology book, can only insert') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+---------------------------------- 79 | 33 | 1 | regress_rls_bob | technology book, can only insert -- But this time, the same statement fails, because the UPDATE path is taken, -- and updating the row just inserted falls afoul of security barrier qual -- (enforced as WCO) -- what we might have updated target tuple to is -- irrelevant, in fact. INSERT INTO document VALUES (79, (SELECT cid from category WHERE cname = 'technology'), 1, 'regress_rls_bob', 'technology book, can only insert') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; ERROR: new row violates row-level security policy (USING expression) for table "document" -- Test default USING qual enforced as WCO SET SESSION AUTHORIZATION regress_rls_alice; DROP POLICY p1 ON document; DROP POLICY p2 ON document; DROP POLICY p3 ON document; CREATE POLICY p3_with_default ON document FOR UPDATE USING (cid = (SELECT cid from category WHERE cname = 'novel')); SET SESSION AUTHORIZATION regress_rls_bob; -- Just because WCO-style enforcement of USING quals occurs with -- existing/target tuple does not mean that the implementation can be allowed -- to fail to also enforce this qual against the final tuple appended to -- relation (since in the absence of an explicit WCO, this is also interpreted -- as an UPDATE/ALL WCO in general). -- -- UPDATE path is taken here (fails due to existing tuple). Note that this is -- not reported as a "USING expression", because it's an RLS UPDATE check that originated as -- a USING qual for the purposes of RLS in general, as opposed to an explicit -- USING qual that is ordinarily a security barrier. We leave it up to the -- UPDATE to make this fail: INSERT INTO document VALUES (79, (SELECT cid from category WHERE cname = 'technology'), 1, 'regress_rls_bob', 'technology book, can only insert') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; ERROR: new row violates row-level security policy for table "document" -- UPDATE path is taken here. Existing tuple passes, since it's cid -- corresponds to "novel", but default USING qual is enforced against -- post-UPDATE tuple too (as always when updating with a policy that lacks an -- explicit WCO), and so this fails: INSERT INTO document VALUES (2, (SELECT cid from category WHERE cname = 'technology'), 1, 'regress_rls_bob', 'my first novel') ON CONFLICT (did) DO UPDATE SET cid = EXCLUDED.cid, dtitle = EXCLUDED.dtitle RETURNING *; ERROR: new row violates row-level security policy for table "document" SET SESSION AUTHORIZATION regress_rls_alice; DROP POLICY p3_with_default ON document; -- -- Test ALL policies with ON CONFLICT DO UPDATE (much the same as existing UPDATE -- tests) -- CREATE POLICY p3_with_all ON document FOR ALL USING (cid = (SELECT cid from category WHERE cname = 'novel')) WITH CHECK (dauthor = current_user); SET SESSION AUTHORIZATION regress_rls_bob; -- Fails, since ALL WCO is enforced in insert path: INSERT INTO document VALUES (80, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_carol', 'my first novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33; ERROR: new row violates row-level security policy for table "document" -- Fails, since ALL policy USING qual is enforced (existing, target tuple is in -- violation, since it has the "manga" cid): INSERT INTO document VALUES (4, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'my first novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle; ERROR: new row violates row-level security policy (USING expression) for table "document" -- Fails, since ALL WCO are enforced: INSERT INTO document VALUES (1, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'my first novel') ON CONFLICT (did) DO UPDATE SET dauthor = 'regress_rls_carol'; ERROR: new row violates row-level security policy for table "document" -- -- ROLE/GROUP -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE z1 (a int, b text); SELECT public.create_hypertable('z1', 'a', chunk_time_interval=>2); create_hypertable ----------------------------- (9,regress_rls_schema,z1,t) CREATE TABLE z2 (a int, b text); SELECT public.create_hypertable('z2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (10,regress_rls_schema,z2,t) GRANT SELECT ON z1,z2 TO regress_rls_group1, regress_rls_group2, regress_rls_bob, regress_rls_carol; INSERT INTO z1 VALUES (1, 'aba'), (2, 'bbb'), (3, 'ccc'), (4, 'dad'); CREATE POLICY p1 ON z1 TO regress_rls_group1 USING (a % 2 = 0); CREATE POLICY p2 ON z1 TO regress_rls_group2 USING (a % 2 = 1); ALTER TABLE z1 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM z1 WHERE f_leak(b); NOTICE: f_leak => bbb NOTICE: f_leak => dad a | b ---+----- 2 | bbb 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM z1 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) PREPARE plancache_test AS SELECT * FROM z1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) PREPARE plancache_test2 AS WITH q AS MATERIALIZED (SELECT * FROM z1 WHERE f_leak(b)) SELECT * FROM q,z2; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test2; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 PREPARE plancache_test4 AS WITH q AS (SELECT * FROM z1 WHERE f_leak(b)) SELECT * FROM q,z2; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test4; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 PREPARE plancache_test6 AS WITH q AS NOT MATERIALIZED (SELECT * FROM z1 WHERE f_leak(b)) SELECT * FROM q,z2; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test6; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 PREPARE plancache_test3 AS WITH q AS MATERIALIZED (SELECT * FROM z2) SELECT * FROM q,z1 WHERE f_leak(z1.b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test3; --- QUERY PLAN --- Nested Loop CTE q -> Seq Scan on z2 -> CTE Scan on q -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) PREPARE plancache_test5 AS WITH q AS (SELECT * FROM z2) SELECT * FROM q,z1 WHERE f_leak(z1.b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test5; --- QUERY PLAN --- Nested Loop -> Seq Scan on z2 -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) PREPARE plancache_test7 AS WITH q AS NOT MATERIALIZED (SELECT * FROM z2) SELECT * FROM q,z1 WHERE f_leak(z1.b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test7; --- QUERY PLAN --- Nested Loop -> Seq Scan on z2 -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) SET ROLE regress_rls_group1; SELECT * FROM z1 WHERE f_leak(b); NOTICE: f_leak => bbb NOTICE: f_leak => dad a | b ---+----- 2 | bbb 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM z1 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test2; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test4; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test3; --- QUERY PLAN --- Nested Loop CTE q -> Seq Scan on z2 -> CTE Scan on q -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test5; --- QUERY PLAN --- Nested Loop -> Seq Scan on z2 -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM z1 WHERE f_leak(b); NOTICE: f_leak => aba NOTICE: f_leak => ccc a | b ---+----- 1 | aba 3 | ccc EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM z1 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test2; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test4; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test3; --- QUERY PLAN --- Nested Loop CTE q -> Seq Scan on z2 -> CTE Scan on q -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test5; --- QUERY PLAN --- Nested Loop -> Seq Scan on z2 -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) SET ROLE regress_rls_group2; SELECT * FROM z1 WHERE f_leak(b); NOTICE: f_leak => aba NOTICE: f_leak => ccc a | b ---+----- 1 | aba 3 | ccc EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM z1 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test2; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test4; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test3; --- QUERY PLAN --- Nested Loop CTE q -> Seq Scan on z2 -> CTE Scan on q -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test5; --- QUERY PLAN --- Nested Loop -> Seq Scan on z2 -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) -- -- Views should follow policy for view owner. -- -- View and Table owner are the same. SET SESSION AUTHORIZATION regress_rls_alice; CREATE VIEW rls_view AS SELECT * FROM z1 WHERE f_leak(b); GRANT SELECT ON rls_view TO regress_rls_bob; -- Query as role that is not owner of view or table. Should return all records. SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rls_view; NOTICE: f_leak => aba NOTICE: f_leak => bbb NOTICE: f_leak => ccc NOTICE: f_leak => dad a | b ---+----- 1 | aba 2 | bbb 3 | ccc 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: f_leak(b) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: f_leak(b) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: f_leak(b) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: f_leak(b) -- Query as view/table owner. Should return all records. SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM rls_view; NOTICE: f_leak => aba NOTICE: f_leak => bbb NOTICE: f_leak => ccc NOTICE: f_leak => dad a | b ---+----- 1 | aba 2 | bbb 3 | ccc 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: f_leak(b) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: f_leak(b) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: f_leak(b) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: f_leak(b) DROP VIEW rls_view; -- View and Table owners are different. SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW rls_view AS SELECT * FROM z1 WHERE f_leak(b); GRANT SELECT ON rls_view TO regress_rls_alice; -- Query as role that is not owner of view but is owner of table. -- Should return records based on view owner policies. SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM rls_view; NOTICE: f_leak => bbb NOTICE: f_leak => dad a | b ---+----- 2 | bbb 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -- Query as role that is not owner of table but is owner of view. -- Should return records based on view owner policies. SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rls_view; NOTICE: f_leak => bbb NOTICE: f_leak => dad a | b ---+----- 2 | bbb 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -- Query as role that is not the owner of the table or view without permissions. SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM rls_view; --fail - permission denied. ERROR: permission denied for view rls_view EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --fail - permission denied. ERROR: permission denied for view rls_view -- Query as role that is not the owner of the table or view with permissions. SET SESSION AUTHORIZATION regress_rls_bob; GRANT SELECT ON rls_view TO regress_rls_carol; SELECT * FROM rls_view; NOTICE: f_leak => bbb NOTICE: f_leak => dad a | b ---+----- 2 | bbb 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) SET SESSION AUTHORIZATION regress_rls_bob; DROP VIEW rls_view; -- -- Command specific -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE x1 (a int, b text, c text); SELECT public.create_hypertable('x1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (11,regress_rls_schema,x1,t) GRANT ALL ON x1 TO PUBLIC; INSERT INTO x1 VALUES (1, 'abc', 'regress_rls_bob'), (2, 'bcd', 'regress_rls_bob'), (3, 'cde', 'regress_rls_carol'), (4, 'def', 'regress_rls_carol'), (5, 'efg', 'regress_rls_bob'), (6, 'fgh', 'regress_rls_bob'), (7, 'fgh', 'regress_rls_carol'), (8, 'fgh', 'regress_rls_carol'); CREATE POLICY p0 ON x1 FOR ALL USING (c = current_user); CREATE POLICY p1 ON x1 FOR SELECT USING (a % 2 = 0); CREATE POLICY p2 ON x1 FOR INSERT WITH CHECK (a % 2 = 1); CREATE POLICY p3 ON x1 FOR UPDATE USING (a % 2 = 0); CREATE POLICY p4 ON x1 FOR DELETE USING (a < 8); ALTER TABLE x1 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM x1 WHERE f_leak(b) ORDER BY a ASC; NOTICE: f_leak => abc NOTICE: f_leak => bcd NOTICE: f_leak => def NOTICE: f_leak => efg NOTICE: f_leak => fgh NOTICE: f_leak => fgh a | b | c ---+-----+------------------- 1 | abc | regress_rls_bob 2 | bcd | regress_rls_bob 4 | def | regress_rls_carol 5 | efg | regress_rls_bob 6 | fgh | regress_rls_bob 8 | fgh | regress_rls_carol UPDATE x1 SET b = b || '_updt' WHERE f_leak(b) RETURNING *; NOTICE: f_leak => abc NOTICE: f_leak => bcd NOTICE: f_leak => def NOTICE: f_leak => efg NOTICE: f_leak => fgh NOTICE: f_leak => fgh a | b | c ---+----------+------------------- 1 | abc_updt | regress_rls_bob 2 | bcd_updt | regress_rls_bob 4 | def_updt | regress_rls_carol 5 | efg_updt | regress_rls_bob 6 | fgh_updt | regress_rls_bob 8 | fgh_updt | regress_rls_carol SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM x1 WHERE f_leak(b) ORDER BY a ASC; NOTICE: f_leak => cde NOTICE: f_leak => bcd_updt NOTICE: f_leak => def_updt NOTICE: f_leak => fgh NOTICE: f_leak => fgh_updt NOTICE: f_leak => fgh_updt a | b | c ---+----------+------------------- 2 | bcd_updt | regress_rls_bob 3 | cde | regress_rls_carol 4 | def_updt | regress_rls_carol 6 | fgh_updt | regress_rls_bob 7 | fgh | regress_rls_carol 8 | fgh_updt | regress_rls_carol UPDATE x1 SET b = b || '_updt' WHERE f_leak(b) RETURNING *; NOTICE: f_leak => cde NOTICE: f_leak => bcd_updt NOTICE: f_leak => def_updt NOTICE: f_leak => fgh NOTICE: f_leak => fgh_updt NOTICE: f_leak => fgh_updt a | b | c ---+---------------+------------------- 3 | cde_updt | regress_rls_carol 2 | bcd_updt_updt | regress_rls_bob 4 | def_updt_updt | regress_rls_carol 7 | fgh_updt | regress_rls_carol 6 | fgh_updt_updt | regress_rls_bob 8 | fgh_updt_updt | regress_rls_carol DELETE FROM x1 WHERE f_leak(b) RETURNING *; NOTICE: f_leak => cde_updt NOTICE: f_leak => bcd_updt_updt NOTICE: f_leak => def_updt_updt NOTICE: f_leak => fgh_updt NOTICE: f_leak => fgh_updt_updt NOTICE: f_leak => fgh_updt_updt a | b | c ---+---------------+------------------- 3 | cde_updt | regress_rls_carol 2 | bcd_updt_updt | regress_rls_bob 4 | def_updt_updt | regress_rls_carol 7 | fgh_updt | regress_rls_carol 6 | fgh_updt_updt | regress_rls_bob 8 | fgh_updt_updt | regress_rls_carol -- -- Duplicate Policy Names -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE y1 (a int, b text); SELECT public.create_hypertable('y1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (12,regress_rls_schema,y1,t) INSERT INTO y1 VALUES(1,2); CREATE TABLE y2 (a int, b text); SELECT public.create_hypertable('y2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (13,regress_rls_schema,y2,t) GRANT ALL ON y1, y2 TO regress_rls_bob; CREATE POLICY p1 ON y1 FOR ALL USING (a % 2 = 0); CREATE POLICY p2 ON y1 FOR SELECT USING (a > 2); CREATE POLICY p1 ON y1 FOR SELECT USING (a % 2 = 1); --fail ERROR: policy "p1" for table "y1" already exists CREATE POLICY p1 ON y2 FOR ALL USING (a % 2 = 0); --OK ALTER TABLE y1 ENABLE ROW LEVEL SECURITY; ALTER TABLE y2 ENABLE ROW LEVEL SECURITY; -- -- Expression structure with SBV -- -- Create view as table owner. RLS should NOT be applied. SET SESSION AUTHORIZATION regress_rls_alice; CREATE VIEW rls_sbv WITH (security_barrier) AS SELECT * FROM y1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_sbv WHERE (a = 1); --- QUERY PLAN --- Custom Scan (ChunkAppend) on y1 Chunks excluded during startup: 0 -> Seq Scan on y1 y1_1 Filter: (f_leak(b) AND (a = 1)) -> Index Scan using _hyper_12_57_chunk_y1_a_idx on _hyper_12_57_chunk y1_2 Index Cond: (a = 1) Filter: f_leak(b) DROP VIEW rls_sbv; -- Create view as role that does not own table. RLS should be applied. SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW rls_sbv WITH (security_barrier) AS SELECT * FROM y1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_sbv WHERE (a = 1); --- QUERY PLAN --- Custom Scan (ChunkAppend) on y1 Chunks excluded during startup: 0 -> Seq Scan on y1 y1_1 Filter: ((a = 1) AND ((a > 2) OR ((a % 2) = 0)) AND f_leak(b)) -> Index Scan using _hyper_12_57_chunk_y1_a_idx on _hyper_12_57_chunk y1_2 Index Cond: (a = 1) Filter: (((a > 2) OR ((a % 2) = 0)) AND f_leak(b)) DROP VIEW rls_sbv; -- -- Expression structure -- SET SESSION AUTHORIZATION regress_rls_alice; INSERT INTO y2 (SELECT x, md5(x::text) FROM generate_series(0,20) x); CREATE POLICY p2 ON y2 USING (a % 3 = 0); CREATE POLICY p3 ON y2 USING (a % 4 = 0); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM y2 WHERE f_leak(b); NOTICE: f_leak => cfcd208495d565ef66e7dff9f98764da NOTICE: f_leak => c81e728d9d4c2f636f067f89cc14862c NOTICE: f_leak => eccbc87e4b5ce2fe28308fd9f2a7baf3 NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc NOTICE: f_leak => c9f0f895fb98ab9159f51fd0297e236d NOTICE: f_leak => 45c48cce2e2d7fbdea1afc51c7c6ad26 NOTICE: f_leak => d3d9446802a44259755d38e6d163e820 NOTICE: f_leak => c20ad4d76fe97759aa27a0c99bff6710 NOTICE: f_leak => aab3238922bcc25a6f606eb525ffdc56 NOTICE: f_leak => 9bf31c7ff062936a96d3c8bd1f8f2ff3 NOTICE: f_leak => c74d97b01eae257e44aa9d5bade97baf NOTICE: f_leak => 6f4922f45568161a8cdf4ad2299f6d23 NOTICE: f_leak => 98f13708210194c475687be6106a3b84 a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 3 | eccbc87e4b5ce2fe28308fd9f2a7baf3 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 9 | 45c48cce2e2d7fbdea1afc51c7c6ad26 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 15 | 9bf31c7ff062936a96d3c8bd1f8f2ff3 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM y2 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on y2 Chunks excluded during startup: 0 -> Seq Scan on y2 y2_1 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_58_chunk y2_2 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_59_chunk y2_3 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_60_chunk y2_4 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_61_chunk y2_5 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_62_chunk y2_6 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_63_chunk y2_7 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_64_chunk y2_8 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_65_chunk y2_9 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_66_chunk y2_10 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_67_chunk y2_11 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_68_chunk y2_12 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -- -- Qual push-down of leaky functions, when not referring to table -- SELECT * FROM y2 WHERE f_leak('abc'); NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 3 | eccbc87e4b5ce2fe28308fd9f2a7baf3 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 9 | 45c48cce2e2d7fbdea1afc51c7c6ad26 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 15 | 9bf31c7ff062936a96d3c8bd1f8f2ff3 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM y2 WHERE f_leak('abc'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on y2 Chunks excluded during startup: 0 -> Seq Scan on y2 y2_1 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_58_chunk y2_2 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_59_chunk y2_3 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_60_chunk y2_4 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_61_chunk y2_5 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_62_chunk y2_6 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_63_chunk y2_7 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_64_chunk y2_8 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_65_chunk y2_9 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_66_chunk y2_10 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_67_chunk y2_11 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_68_chunk y2_12 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) CREATE TABLE test_qual_pushdown ( abc text ); INSERT INTO test_qual_pushdown VALUES ('abc'),('def'); SELECT * FROM y2 JOIN test_qual_pushdown ON (b = abc) WHERE f_leak(abc); NOTICE: f_leak => abc NOTICE: f_leak => def a | b | abc ---+---+----- EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM y2 JOIN test_qual_pushdown ON (b = abc) WHERE f_leak(abc); --- QUERY PLAN --- Hash Join Hash Cond: (y2.b = test_qual_pushdown.abc) -> Append -> Seq Scan on y2 y2_1 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_58_chunk y2_2 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_59_chunk y2_3 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_60_chunk y2_4 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_61_chunk y2_5 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_62_chunk y2_6 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_63_chunk y2_7 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_64_chunk y2_8 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_65_chunk y2_9 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_66_chunk y2_10 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_67_chunk y2_11 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_68_chunk y2_12 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Hash -> Seq Scan on test_qual_pushdown Filter: f_leak(abc) SELECT * FROM y2 JOIN test_qual_pushdown ON (b = abc) WHERE f_leak(b); NOTICE: f_leak => cfcd208495d565ef66e7dff9f98764da NOTICE: f_leak => c81e728d9d4c2f636f067f89cc14862c NOTICE: f_leak => eccbc87e4b5ce2fe28308fd9f2a7baf3 NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc NOTICE: f_leak => c9f0f895fb98ab9159f51fd0297e236d NOTICE: f_leak => 45c48cce2e2d7fbdea1afc51c7c6ad26 NOTICE: f_leak => d3d9446802a44259755d38e6d163e820 NOTICE: f_leak => c20ad4d76fe97759aa27a0c99bff6710 NOTICE: f_leak => aab3238922bcc25a6f606eb525ffdc56 NOTICE: f_leak => 9bf31c7ff062936a96d3c8bd1f8f2ff3 NOTICE: f_leak => c74d97b01eae257e44aa9d5bade97baf NOTICE: f_leak => 6f4922f45568161a8cdf4ad2299f6d23 NOTICE: f_leak => 98f13708210194c475687be6106a3b84 a | b | abc ---+---+----- EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM y2 JOIN test_qual_pushdown ON (b = abc) WHERE f_leak(b); --- QUERY PLAN --- Hash Join Hash Cond: (test_qual_pushdown.abc = y2.b) -> Seq Scan on test_qual_pushdown -> Hash -> Custom Scan (ChunkAppend) on y2 Chunks excluded during startup: 0 -> Seq Scan on y2 y2_1 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_58_chunk y2_2 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_59_chunk y2_3 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_60_chunk y2_4 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_61_chunk y2_5 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_62_chunk y2_6 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_63_chunk y2_7 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_64_chunk y2_8 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_65_chunk y2_9 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_66_chunk y2_10 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_67_chunk y2_11 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_68_chunk y2_12 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) DROP TABLE test_qual_pushdown; -- -- Plancache invalidate on user change. -- RESET SESSION AUTHORIZATION; \set VERBOSITY terse \\ -- suppress cascade details DROP TABLE t1 CASCADE; NOTICE: drop cascades to 2 other objects \set VERBOSITY default CREATE TABLE t1 (a integer); SELECT public.create_hypertable('t1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (14,regress_rls_schema,t1,t) GRANT SELECT ON t1 TO regress_rls_bob, regress_rls_carol; CREATE POLICY p1 ON t1 TO regress_rls_bob USING ((a % 2) = 0); CREATE POLICY p2 ON t1 TO regress_rls_carol USING ((a % 4) = 0); ALTER TABLE t1 ENABLE ROW LEVEL SECURITY; -- Prepare as regress_rls_bob SET ROLE regress_rls_bob; PREPARE role_inval AS SELECT * FROM t1; -- Check plan EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE role_inval; --- QUERY PLAN --- Seq Scan on t1 Filter: ((a % 2) = 0) -- Change to regress_rls_carol SET ROLE regress_rls_carol; -- Check plan- should be different EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE role_inval; --- QUERY PLAN --- Seq Scan on t1 Filter: ((a % 4) = 0) -- Change back to regress_rls_bob SET ROLE regress_rls_bob; -- Check plan- should be back to original EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE role_inval; --- QUERY PLAN --- Seq Scan on t1 Filter: ((a % 2) = 0) -- -- CTE and RLS -- RESET SESSION AUTHORIZATION; DROP TABLE t1 CASCADE; CREATE TABLE t1 (a integer, b text); SELECT public.create_hypertable('t1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (15,regress_rls_schema,t1,t) CREATE POLICY p1 ON t1 USING (a % 2 = 0); ALTER TABLE t1 ENABLE ROW LEVEL SECURITY; GRANT ALL ON t1 TO regress_rls_bob; INSERT INTO t1 (SELECT x, md5(x::text) FROM generate_series(0,20) x); SET SESSION AUTHORIZATION regress_rls_bob; WITH cte1 AS (SELECT * FROM t1 WHERE f_leak(b)) SELECT * FROM cte1; NOTICE: f_leak => cfcd208495d565ef66e7dff9f98764da NOTICE: f_leak => c81e728d9d4c2f636f067f89cc14862c NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc NOTICE: f_leak => c9f0f895fb98ab9159f51fd0297e236d NOTICE: f_leak => d3d9446802a44259755d38e6d163e820 NOTICE: f_leak => c20ad4d76fe97759aa27a0c99bff6710 NOTICE: f_leak => aab3238922bcc25a6f606eb525ffdc56 NOTICE: f_leak => c74d97b01eae257e44aa9d5bade97baf NOTICE: f_leak => 6f4922f45568161a8cdf4ad2299f6d23 NOTICE: f_leak => 98f13708210194c475687be6106a3b84 a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 EXPLAIN (BUFFERS OFF, COSTS OFF) WITH cte1 AS (SELECT * FROM t1 WHERE f_leak(b)) SELECT * FROM cte1; --- QUERY PLAN --- CTE Scan on cte1 CTE cte1 -> Custom Scan (ChunkAppend) on t1 Chunks excluded during startup: 0 -> Seq Scan on t1 t1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_69_chunk t1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_70_chunk t1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_71_chunk t1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_72_chunk t1_5 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_73_chunk t1_6 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_74_chunk t1_7 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_75_chunk t1_8 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_76_chunk t1_9 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_77_chunk t1_10 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_78_chunk t1_11 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_79_chunk t1_12 Filter: (((a % 2) = 0) AND f_leak(b)) WITH cte1 AS (UPDATE t1 SET a = a + 1 RETURNING *) SELECT * FROM cte1; --fail ERROR: new row violates row-level security policy for table "t1" WITH cte1 AS (UPDATE t1 SET a = a RETURNING *) SELECT * FROM cte1; --ok a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 WITH cte1 AS (INSERT INTO t1 VALUES (21, 'Fail') RETURNING *) SELECT * FROM cte1; --fail ERROR: new row violates row-level security policy for table "t1" WITH cte1 AS (INSERT INTO t1 VALUES (20, 'Success') RETURNING *) SELECT * FROM cte1; --ok a | b ----+--------- 20 | Success -- -- Rename Policy -- RESET SESSION AUTHORIZATION; ALTER POLICY p1 ON t1 RENAME TO p1; --fail ERROR: policy "p1" for table "t1" already exists SELECT polname, relname FROM pg_policy pol JOIN pg_class pc ON (pc.oid = pol.polrelid) WHERE relname = 't1'; polname | relname ---------+--------- p1 | t1 ALTER POLICY p1 ON t1 RENAME TO p2; --ok SELECT polname, relname FROM pg_policy pol JOIN pg_class pc ON (pc.oid = pol.polrelid) WHERE relname = 't1'; polname | relname ---------+--------- p2 | t1 -- -- Check INSERT SELECT -- SET SESSION AUTHORIZATION regress_rls_bob; CREATE TABLE t2 (a integer, b text); SELECT public.create_hypertable('t2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (16,regress_rls_schema,t2,t) INSERT INTO t2 (SELECT * FROM t1); EXPLAIN (BUFFERS OFF, COSTS OFF) INSERT INTO t2 (SELECT * FROM t1); --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on t2 -> Append -> Seq Scan on t1 t1_1 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_69_chunk t1_2 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_70_chunk t1_3 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_71_chunk t1_4 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_72_chunk t1_5 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_73_chunk t1_6 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_74_chunk t1_7 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_75_chunk t1_8 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_76_chunk t1_9 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_77_chunk t1_10 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_78_chunk t1_11 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_79_chunk t1_12 Filter: ((a % 2) = 0) SELECT * FROM t2; a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 20 | Success EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t2; --- QUERY PLAN --- Append -> Seq Scan on t2 t2_1 -> Seq Scan on _hyper_16_80_chunk t2_2 -> Seq Scan on _hyper_16_81_chunk t2_3 -> Seq Scan on _hyper_16_82_chunk t2_4 -> Seq Scan on _hyper_16_83_chunk t2_5 -> Seq Scan on _hyper_16_84_chunk t2_6 -> Seq Scan on _hyper_16_85_chunk t2_7 -> Seq Scan on _hyper_16_86_chunk t2_8 -> Seq Scan on _hyper_16_87_chunk t2_9 -> Seq Scan on _hyper_16_88_chunk t2_10 -> Seq Scan on _hyper_16_89_chunk t2_11 -> Seq Scan on _hyper_16_90_chunk t2_12 CREATE TABLE t3 AS SELECT * FROM t1; SELECT public.create_hypertable('t2', 'a', chunk_time_interval=>2); ERROR: table "t2" is already a hypertable SELECT * FROM t3; a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 20 | Success SELECT * INTO t4 FROM t1; SELECT * FROM t4; a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 20 | Success -- -- RLS with JOIN -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE blog (id integer, author text, post text); SELECT public.create_hypertable('blog', 'id', chunk_time_interval=>2); create_hypertable -------------------------------- (17,regress_rls_schema,blog,t) CREATE TABLE comment (blog_id integer, message text); SELECT public.create_hypertable('comment', 'blog_id', chunk_time_interval=>2); create_hypertable ----------------------------------- (18,regress_rls_schema,comment,t) GRANT ALL ON blog, comment TO regress_rls_bob; CREATE POLICY blog_1 ON blog USING (id % 2 = 0); ALTER TABLE blog ENABLE ROW LEVEL SECURITY; INSERT INTO blog VALUES (1, 'alice', 'blog #1'), (2, 'bob', 'blog #1'), (3, 'alice', 'blog #2'), (4, 'alice', 'blog #3'), (5, 'john', 'blog #1'); INSERT INTO comment VALUES (1, 'cool blog'), (1, 'fun blog'), (3, 'crazy blog'), (5, 'what?'), (4, 'insane!'), (2, 'who did it?'); SET SESSION AUTHORIZATION regress_rls_bob; -- Check RLS JOIN with Non-RLS. SELECT id, author, message FROM blog JOIN comment ON id = blog_id; id | author | message ----+--------+------------- 2 | bob | who did it? 4 | alice | insane! -- Check Non-RLS JOIN with RLS. SELECT id, author, message FROM comment JOIN blog ON id = blog_id; id | author | message ----+--------+------------- 2 | bob | who did it? 4 | alice | insane! SET SESSION AUTHORIZATION regress_rls_alice; CREATE POLICY comment_1 ON comment USING (blog_id < 4); ALTER TABLE comment ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; -- Check RLS JOIN RLS SELECT id, author, message FROM blog JOIN comment ON id = blog_id; id | author | message ----+--------+------------- 2 | bob | who did it? SELECT id, author, message FROM comment JOIN blog ON id = blog_id; id | author | message ----+--------+------------- 2 | bob | who did it? SET SESSION AUTHORIZATION regress_rls_alice; DROP TABLE blog; DROP TABLE comment; -- -- Default Deny Policy -- RESET SESSION AUTHORIZATION; DROP POLICY p2 ON t1; ALTER TABLE t1 OWNER TO regress_rls_alice; -- Check that default deny does not apply to superuser. RESET SESSION AUTHORIZATION; SELECT * FROM t1; a | b ----+---------------------------------- 1 | c4ca4238a0b923820dcc509a6f75849b 0 | cfcd208495d565ef66e7dff9f98764da 3 | eccbc87e4b5ce2fe28308fd9f2a7baf3 2 | c81e728d9d4c2f636f067f89cc14862c 5 | e4da3b7fbbce2345d7772b0674a318d5 4 | a87ff679a2f3e71d9181a67b7542122c 7 | 8f14e45fceea167a5a36dedd4bea2543 6 | 1679091c5a880faf6fb5e6087eb1b2dc 9 | 45c48cce2e2d7fbdea1afc51c7c6ad26 8 | c9f0f895fb98ab9159f51fd0297e236d 11 | 6512bd43d9caa6e02c990b0a82652dca 10 | d3d9446802a44259755d38e6d163e820 13 | c51ce410c124a10e0db5e4b97fc2af39 12 | c20ad4d76fe97759aa27a0c99bff6710 15 | 9bf31c7ff062936a96d3c8bd1f8f2ff3 14 | aab3238922bcc25a6f606eb525ffdc56 17 | 70efdf2ec9b086079795c442636b55fb 16 | c74d97b01eae257e44aa9d5bade97baf 19 | 1f0e3dad99908345f7439f8ffabdffc4 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 20 | Success EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 -> Seq Scan on _hyper_15_69_chunk t1_2 -> Seq Scan on _hyper_15_70_chunk t1_3 -> Seq Scan on _hyper_15_71_chunk t1_4 -> Seq Scan on _hyper_15_72_chunk t1_5 -> Seq Scan on _hyper_15_73_chunk t1_6 -> Seq Scan on _hyper_15_74_chunk t1_7 -> Seq Scan on _hyper_15_75_chunk t1_8 -> Seq Scan on _hyper_15_76_chunk t1_9 -> Seq Scan on _hyper_15_77_chunk t1_10 -> Seq Scan on _hyper_15_78_chunk t1_11 -> Seq Scan on _hyper_15_79_chunk t1_12 -- Check that default deny does not apply to table owner. SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM t1; a | b ----+---------------------------------- 1 | c4ca4238a0b923820dcc509a6f75849b 0 | cfcd208495d565ef66e7dff9f98764da 3 | eccbc87e4b5ce2fe28308fd9f2a7baf3 2 | c81e728d9d4c2f636f067f89cc14862c 5 | e4da3b7fbbce2345d7772b0674a318d5 4 | a87ff679a2f3e71d9181a67b7542122c 7 | 8f14e45fceea167a5a36dedd4bea2543 6 | 1679091c5a880faf6fb5e6087eb1b2dc 9 | 45c48cce2e2d7fbdea1afc51c7c6ad26 8 | c9f0f895fb98ab9159f51fd0297e236d 11 | 6512bd43d9caa6e02c990b0a82652dca 10 | d3d9446802a44259755d38e6d163e820 13 | c51ce410c124a10e0db5e4b97fc2af39 12 | c20ad4d76fe97759aa27a0c99bff6710 15 | 9bf31c7ff062936a96d3c8bd1f8f2ff3 14 | aab3238922bcc25a6f606eb525ffdc56 17 | 70efdf2ec9b086079795c442636b55fb 16 | c74d97b01eae257e44aa9d5bade97baf 19 | 1f0e3dad99908345f7439f8ffabdffc4 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 20 | Success EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 -> Seq Scan on _hyper_15_69_chunk t1_2 -> Seq Scan on _hyper_15_70_chunk t1_3 -> Seq Scan on _hyper_15_71_chunk t1_4 -> Seq Scan on _hyper_15_72_chunk t1_5 -> Seq Scan on _hyper_15_73_chunk t1_6 -> Seq Scan on _hyper_15_74_chunk t1_7 -> Seq Scan on _hyper_15_75_chunk t1_8 -> Seq Scan on _hyper_15_76_chunk t1_9 -> Seq Scan on _hyper_15_77_chunk t1_10 -> Seq Scan on _hyper_15_78_chunk t1_11 -> Seq Scan on _hyper_15_79_chunk t1_12 -- Check that default deny applies to non-owner/non-superuser when RLS on. SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; SELECT * FROM t1; a | b ---+--- EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; --- QUERY PLAN --- Result One-Time Filter: false SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM t1; a | b ---+--- EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; --- QUERY PLAN --- Result One-Time Filter: false -- -- COPY TO/FROM -- RESET SESSION AUTHORIZATION; DROP TABLE copy_t CASCADE; ERROR: table "copy_t" does not exist CREATE TABLE copy_t (a integer, b text); SELECT public.create_hypertable('copy_t', 'a', chunk_time_interval=>2); create_hypertable ---------------------------------- (19,regress_rls_schema,copy_t,t) CREATE POLICY p1 ON copy_t USING (a % 2 = 0); ALTER TABLE copy_t ENABLE ROW LEVEL SECURITY; GRANT ALL ON copy_t TO regress_rls_bob, regress_rls_exempt_user; INSERT INTO copy_t (SELECT x, md5(x::text) FROM generate_series(0,10) x); -- Check COPY TO as Superuser/owner. RESET SESSION AUTHORIZATION; SET row_security TO OFF; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; 0,cfcd208495d565ef66e7dff9f98764da 1,c4ca4238a0b923820dcc509a6f75849b 2,c81e728d9d4c2f636f067f89cc14862c 3,eccbc87e4b5ce2fe28308fd9f2a7baf3 4,a87ff679a2f3e71d9181a67b7542122c 5,e4da3b7fbbce2345d7772b0674a318d5 6,1679091c5a880faf6fb5e6087eb1b2dc 7,8f14e45fceea167a5a36dedd4bea2543 8,c9f0f895fb98ab9159f51fd0297e236d 9,45c48cce2e2d7fbdea1afc51c7c6ad26 10,d3d9446802a44259755d38e6d163e820 SET row_security TO ON; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; 0,cfcd208495d565ef66e7dff9f98764da 1,c4ca4238a0b923820dcc509a6f75849b 2,c81e728d9d4c2f636f067f89cc14862c 3,eccbc87e4b5ce2fe28308fd9f2a7baf3 4,a87ff679a2f3e71d9181a67b7542122c 5,e4da3b7fbbce2345d7772b0674a318d5 6,1679091c5a880faf6fb5e6087eb1b2dc 7,8f14e45fceea167a5a36dedd4bea2543 8,c9f0f895fb98ab9159f51fd0297e236d 9,45c48cce2e2d7fbdea1afc51c7c6ad26 10,d3d9446802a44259755d38e6d163e820 -- Check COPY TO as user with permissions. SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO OFF; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --fail - would be affected by RLS ERROR: query would be affected by row-level security policy for table "copy_t" SET row_security TO ON; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --ok 0,cfcd208495d565ef66e7dff9f98764da 2,c81e728d9d4c2f636f067f89cc14862c 4,a87ff679a2f3e71d9181a67b7542122c 6,1679091c5a880faf6fb5e6087eb1b2dc 8,c9f0f895fb98ab9159f51fd0297e236d 10,d3d9446802a44259755d38e6d163e820 -- Check COPY TO as user with permissions and BYPASSRLS SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --ok 0,cfcd208495d565ef66e7dff9f98764da 1,c4ca4238a0b923820dcc509a6f75849b 2,c81e728d9d4c2f636f067f89cc14862c 3,eccbc87e4b5ce2fe28308fd9f2a7baf3 4,a87ff679a2f3e71d9181a67b7542122c 5,e4da3b7fbbce2345d7772b0674a318d5 6,1679091c5a880faf6fb5e6087eb1b2dc 7,8f14e45fceea167a5a36dedd4bea2543 8,c9f0f895fb98ab9159f51fd0297e236d 9,45c48cce2e2d7fbdea1afc51c7c6ad26 10,d3d9446802a44259755d38e6d163e820 SET row_security TO ON; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --ok 0,cfcd208495d565ef66e7dff9f98764da 1,c4ca4238a0b923820dcc509a6f75849b 2,c81e728d9d4c2f636f067f89cc14862c 3,eccbc87e4b5ce2fe28308fd9f2a7baf3 4,a87ff679a2f3e71d9181a67b7542122c 5,e4da3b7fbbce2345d7772b0674a318d5 6,1679091c5a880faf6fb5e6087eb1b2dc 7,8f14e45fceea167a5a36dedd4bea2543 8,c9f0f895fb98ab9159f51fd0297e236d 9,45c48cce2e2d7fbdea1afc51c7c6ad26 10,d3d9446802a44259755d38e6d163e820 -- Check COPY TO as user without permissions. SET row_security TO OFF; SET SESSION AUTHORIZATION regress_rls_carol; SET row_security TO OFF; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --fail - would be affected by RLS ERROR: query would be affected by row-level security policy for table "copy_t" SET row_security TO ON; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --fail - permission denied ERROR: permission denied for table copy_t -- Check COPY relation TO; keep it just one row to avoid reordering issues RESET SESSION AUTHORIZATION; SET row_security TO ON; CREATE TABLE copy_rel_to (a integer, b text); SELECT public.create_hypertable('copy_rel_to', 'a', chunk_time_interval=>2); create_hypertable --------------------------------------- (20,regress_rls_schema,copy_rel_to,t) CREATE POLICY p1 ON copy_rel_to USING (a % 2 = 0); ALTER TABLE copy_rel_to ENABLE ROW LEVEL SECURITY; GRANT ALL ON copy_rel_to TO regress_rls_bob, regress_rls_exempt_user; INSERT INTO copy_rel_to VALUES (1, md5('1')); -- Check COPY TO as Superuser/owner. RESET SESSION AUTHORIZATION; SET row_security TO OFF; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; 1,c4ca4238a0b923820dcc509a6f75849b SET row_security TO ON; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; 1,c4ca4238a0b923820dcc509a6f75849b -- Check COPY TO as user with permissions. SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO OFF; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --fail - would be affected by RLS ERROR: query would be affected by row-level security policy for table "copy_rel_to" SET row_security TO ON; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --ok -- Check COPY TO as user with permissions and BYPASSRLS SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --ok 1,c4ca4238a0b923820dcc509a6f75849b SET row_security TO ON; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --ok 1,c4ca4238a0b923820dcc509a6f75849b -- Check COPY TO as user without permissions. SET row_security TO OFF; SET SESSION AUTHORIZATION regress_rls_carol; SET row_security TO OFF; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --fail - permission denied ERROR: query would be affected by row-level security policy for table "copy_rel_to" SET row_security TO ON; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --fail - permission denied ERROR: permission denied for table copy_rel_to -- Check COPY FROM as Superuser/owner. RESET SESSION AUTHORIZATION; SET row_security TO OFF; COPY copy_t FROM STDIN; --ok SET row_security TO ON; COPY copy_t FROM STDIN; --ok -- Check COPY FROM as user with permissions. SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO OFF; COPY copy_t FROM STDIN; --fail - would be affected by RLS. ERROR: query would be affected by row-level security policy for table "copy_t" SET row_security TO ON; COPY copy_t FROM STDIN; --fail - COPY FROM not supported by RLS. ERROR: COPY FROM not supported with row-level security HINT: Use INSERT statements instead. -- Check COPY FROM as user with permissions and BYPASSRLS SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO ON; COPY copy_t FROM STDIN; --ok -- Check COPY FROM as user without permissions. SET SESSION AUTHORIZATION regress_rls_carol; SET row_security TO OFF; COPY copy_t FROM STDIN; --fail - permission denied. ERROR: permission denied for table copy_t SET row_security TO ON; COPY copy_t FROM STDIN; --fail - permission denied. ERROR: permission denied for table copy_t RESET SESSION AUTHORIZATION; DROP TABLE copy_t; DROP TABLE copy_rel_to CASCADE; -- Check WHERE CURRENT OF SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE current_check (currentid int, payload text, rlsuser text); SELECT public.create_hypertable('current_check', 'currentid', chunk_time_interval=>10); create_hypertable ----------------------------------------- (21,regress_rls_schema,current_check,t) GRANT ALL ON current_check TO PUBLIC; INSERT INTO current_check VALUES (1, 'abc', 'regress_rls_bob'), (2, 'bcd', 'regress_rls_bob'), (3, 'cde', 'regress_rls_bob'), (4, 'def', 'regress_rls_bob'); CREATE POLICY p1 ON current_check FOR SELECT USING (currentid % 2 = 0); CREATE POLICY p2 ON current_check FOR DELETE USING (currentid = 4 AND rlsuser = current_user); CREATE POLICY p3 ON current_check FOR UPDATE USING (currentid = 4) WITH CHECK (rlsuser = current_user); ALTER TABLE current_check ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; -- Can SELECT even rows SELECT * FROM current_check; currentid | payload | rlsuser -----------+---------+----------------- 2 | bcd | regress_rls_bob 4 | def | regress_rls_bob -- Cannot UPDATE row 2 UPDATE current_check SET payload = payload || '_new' WHERE currentid = 2 RETURNING *; currentid | payload | rlsuser -----------+---------+--------- BEGIN; -- WHERE CURRENT OF does not work with custom scan nodes -- so we have to disable chunk append here SET timescaledb.enable_chunk_append TO false; DECLARE current_check_cursor SCROLL CURSOR FOR SELECT * FROM current_check; -- Returns rows that can be seen according to SELECT policy, like plain SELECT -- above (even rows) FETCH ABSOLUTE 1 FROM current_check_cursor; currentid | payload | rlsuser -----------+---------+----------------- 2 | bcd | regress_rls_bob -- Still cannot UPDATE row 2 through cursor UPDATE current_check SET payload = payload || '_new' WHERE CURRENT OF current_check_cursor RETURNING *; currentid | payload | rlsuser -----------+---------+--------- -- Can update row 4 through cursor, which is the next visible row FETCH RELATIVE 1 FROM current_check_cursor; currentid | payload | rlsuser -----------+---------+----------------- 4 | def | regress_rls_bob UPDATE current_check SET payload = payload || '_new' WHERE CURRENT OF current_check_cursor RETURNING *; currentid | payload | rlsuser -----------+---------+----------------- 4 | def_new | regress_rls_bob SELECT * FROM current_check; currentid | payload | rlsuser -----------+---------+----------------- 2 | bcd | regress_rls_bob 4 | def_new | regress_rls_bob -- Plan should be a subquery TID scan EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE current_check SET payload = payload WHERE CURRENT OF current_check_cursor; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Update on current_check Update on _hyper_21_104_chunk current_check_1 -> Tid Scan on _hyper_21_104_chunk current_check_1 TID Cond: CURRENT OF current_check_cursor Filter: ((currentid = 4) AND ((currentid % 2) = 0)) -- Similarly can only delete row 4 FETCH ABSOLUTE 1 FROM current_check_cursor; currentid | payload | rlsuser -----------+---------+----------------- 2 | bcd | regress_rls_bob DELETE FROM current_check WHERE CURRENT OF current_check_cursor RETURNING *; currentid | payload | rlsuser -----------+---------+--------- FETCH RELATIVE 1 FROM current_check_cursor; currentid | payload | rlsuser -----------+---------+----------------- 4 | def | regress_rls_bob DELETE FROM current_check WHERE CURRENT OF current_check_cursor RETURNING *; currentid | payload | rlsuser -----------+---------+----------------- 4 | def_new | regress_rls_bob SELECT * FROM current_check; currentid | payload | rlsuser -----------+---------+----------------- 2 | bcd | regress_rls_bob RESET timescaledb.enable_chunk_append; COMMIT; -- -- check pg_stats view filtering -- SET row_security TO ON; SET SESSION AUTHORIZATION regress_rls_alice; ANALYZE current_check; -- Stats visible SELECT row_security_active('current_check'); row_security_active --------------------- f SELECT attname, most_common_vals FROM pg_stats WHERE tablename = 'current_check' ORDER BY 1; attname | most_common_vals -----------+------------------- currentid | payload | rlsuser | {regress_rls_bob} SET SESSION AUTHORIZATION regress_rls_bob; -- Stats not visible SELECT row_security_active('current_check'); row_security_active --------------------- t SELECT attname, most_common_vals FROM pg_stats WHERE tablename = 'current_check' ORDER BY 1; attname | most_common_vals ---------+------------------ -- -- Collation support -- BEGIN; CREATE TABLE coll_t (c) AS VALUES ('bar'::text); CREATE POLICY coll_p ON coll_t USING (c < ('foo'::text COLLATE "C")); ALTER TABLE coll_t ENABLE ROW LEVEL SECURITY; GRANT SELECT ON coll_t TO regress_rls_alice; SELECT (string_to_array(polqual, ':'))[7] AS inputcollid FROM pg_policy WHERE polrelid = 'coll_t'::regclass; inputcollid ------------------ inputcollid 950 SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM coll_t; c ----- bar ROLLBACK; -- -- Shared Object Dependencies -- RESET SESSION AUTHORIZATION; BEGIN; CREATE ROLE regress_rls_eve; CREATE ROLE regress_rls_frank; CREATE TABLE tbl1 (c) AS VALUES ('bar'::text); GRANT SELECT ON TABLE tbl1 TO regress_rls_eve; CREATE POLICY P ON tbl1 TO regress_rls_eve, regress_rls_frank USING (true); SELECT refclassid::regclass, deptype FROM pg_depend WHERE classid = 'pg_policy'::regclass AND refobjid = 'tbl1'::regclass; refclassid | deptype ------------+--------- pg_class | a SELECT refclassid::regclass, deptype FROM pg_shdepend WHERE classid = 'pg_policy'::regclass AND refobjid IN ('regress_rls_eve'::regrole, 'regress_rls_frank'::regrole); refclassid | deptype ------------+--------- pg_authid | r pg_authid | r SAVEPOINT q; DROP ROLE regress_rls_eve; --fails due to dependency on POLICY p ERROR: role "regress_rls_eve" cannot be dropped because some objects depend on it DETAIL: privileges for table tbl1 target of policy p on table tbl1 ROLLBACK TO q; ALTER POLICY p ON tbl1 TO regress_rls_frank USING (true); SAVEPOINT q; DROP ROLE regress_rls_eve; --fails due to dependency on GRANT SELECT ERROR: role "regress_rls_eve" cannot be dropped because some objects depend on it DETAIL: privileges for table tbl1 ROLLBACK TO q; REVOKE ALL ON TABLE tbl1 FROM regress_rls_eve; SAVEPOINT q; DROP ROLE regress_rls_eve; --succeeds ROLLBACK TO q; SAVEPOINT q; DROP ROLE regress_rls_frank; --fails due to dependency on POLICY p ERROR: role "regress_rls_frank" cannot be dropped because some objects depend on it DETAIL: target of policy p on table tbl1 ROLLBACK TO q; DROP POLICY p ON tbl1; SAVEPOINT q; DROP ROLE regress_rls_frank; -- succeeds ROLLBACK TO q; ROLLBACK; -- cleanup -- -- Converting table to view -- BEGIN; CREATE TABLE t (c int); SELECT public.create_hypertable('t', 'c', chunk_time_interval=>2); create_hypertable ----------------------------- (22,regress_rls_schema,t,t) CREATE POLICY p ON t USING (c % 2 = 1); ALTER TABLE t ENABLE ROW LEVEL SECURITY; SAVEPOINT q; CREATE RULE "_RETURN" AS ON SELECT TO t DO INSTEAD SELECT * FROM generate_series(1,5) t0(c); -- fails due to row level security enabled ERROR: hypertables do not support rules ROLLBACK TO q; ALTER TABLE t DISABLE ROW LEVEL SECURITY; SAVEPOINT q; CREATE RULE "_RETURN" AS ON SELECT TO t DO INSTEAD SELECT * FROM generate_series(1,5) t0(c); -- fails due to policy p on t ERROR: hypertables do not support rules ROLLBACK TO q; DROP POLICY p ON t; CREATE RULE "_RETURN" AS ON SELECT TO t DO INSTEAD SELECT * FROM generate_series(1,5) t0(c); -- succeeds ERROR: hypertables do not support rules ROLLBACK; -- -- Policy expression handling -- BEGIN; CREATE TABLE t (c) AS VALUES ('bar'::text); CREATE POLICY p ON t USING (max(c)); -- fails: aggregate functions are not allowed in policy expressions ERROR: aggregate functions are not allowed in policy expressions ROLLBACK; -- -- Non-target relations are only subject to SELECT policies -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE r1 (a int); SELECT public.create_hypertable('r1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (23,regress_rls_schema,r1,t) CREATE TABLE r2 (a int); SELECT public.create_hypertable('r2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (24,regress_rls_schema,r2,t) INSERT INTO r1 VALUES (10), (20); INSERT INTO r2 VALUES (10), (20); GRANT ALL ON r1, r2 TO regress_rls_bob; CREATE POLICY p1 ON r1 USING (true); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; CREATE POLICY p1 ON r2 FOR SELECT USING (true); CREATE POLICY p2 ON r2 FOR INSERT WITH CHECK (false); CREATE POLICY p3 ON r2 FOR UPDATE USING (false); CREATE POLICY p4 ON r2 FOR DELETE USING (false); ALTER TABLE r2 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM r1; a ---- 10 20 SELECT * FROM r2; a ---- 10 20 -- r2 is read-only INSERT INTO r2 VALUES (2); -- Not allowed ERROR: new row violates row-level security policy for table "r2" \pset tuples_only 1 UPDATE r2 SET a = 2 RETURNING *; -- Updates nothing DELETE FROM r2 RETURNING *; -- Deletes nothing \pset tuples_only 0 -- r2 can be used as a non-target relation in DML INSERT INTO r1 SELECT a + 1 FROM r2 RETURNING *; -- OK a ---- 11 21 UPDATE r1 SET a = r2.a + 2 FROM r2 WHERE r1.a = r2.a RETURNING *; -- OK ERROR: new row for relation "_hyper_23_105_chunk" violates check constraint "constraint_105" DELETE FROM r1 USING r2 WHERE r1.a = r2.a + 2 RETURNING *; -- OK a | a ---+--- SELECT * FROM r1; a ---- 10 11 20 21 SELECT * FROM r2; a ---- 10 20 SET SESSION AUTHORIZATION regress_rls_alice; DROP TABLE r1; DROP TABLE r2; -- -- FORCE ROW LEVEL SECURITY applies RLS to owners too -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security = on; CREATE TABLE r1 (a int); SELECT public.create_hypertable('r1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (25,regress_rls_schema,r1,t) INSERT INTO r1 VALUES (10), (20); CREATE POLICY p1 ON r1 USING (false); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- No error, but no rows TABLE r1; a --- -- RLS error INSERT INTO r1 VALUES (1); ERROR: new row violates row-level security policy for table "r1" -- No error (unable to see any rows to update) UPDATE r1 SET a = 1; TABLE r1; a --- -- No error (unable to see any rows to delete) DELETE FROM r1; TABLE r1; a --- SET row_security = off; -- these all fail, would be affected by RLS TABLE r1; ERROR: query would be affected by row-level security policy for table "r1" HINT: To disable the policy for the table's owner, use ALTER TABLE NO FORCE ROW LEVEL SECURITY. UPDATE r1 SET a = 1; ERROR: query would be affected by row-level security policy for table "r1" HINT: To disable the policy for the table's owner, use ALTER TABLE NO FORCE ROW LEVEL SECURITY. DELETE FROM r1; ERROR: query would be affected by row-level security policy for table "r1" HINT: To disable the policy for the table's owner, use ALTER TABLE NO FORCE ROW LEVEL SECURITY. DROP TABLE r1; -- -- FORCE ROW LEVEL SECURITY does not break RI -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security = on; CREATE TABLE r1 (a int PRIMARY KEY); -- r1 is not a hypertable since r1.a is referenced by r2 CREATE TABLE r2 (a int REFERENCES r1); SELECT public.create_hypertable('r2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (26,regress_rls_schema,r2,t) INSERT INTO r1 VALUES (10), (20); INSERT INTO r2 VALUES (10), (20); -- Create policies on r2 which prevent the -- owner from seeing any rows, but RI should -- still see them. CREATE POLICY p1 ON r2 USING (false); ALTER TABLE r2 ENABLE ROW LEVEL SECURITY; ALTER TABLE r2 FORCE ROW LEVEL SECURITY; -- Errors due to rows in r2 DELETE FROM r1; ERROR: update or delete on table "r1" violates foreign key constraint "r2_a_fkey" on table "r2" DETAIL: Key (a)=(10) is still referenced from table "r2". -- Reset r2 to no-RLS DROP POLICY p1 ON r2; ALTER TABLE r2 NO FORCE ROW LEVEL SECURITY; ALTER TABLE r2 DISABLE ROW LEVEL SECURITY; -- clean out r2 for INSERT test below DELETE FROM r2; -- Change r1 to not allow rows to be seen CREATE POLICY p1 ON r1 USING (false); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- No rows seen TABLE r1; a --- -- No error, RI still sees that row exists in r1 INSERT INTO r2 VALUES (10); DROP TABLE r2; DROP TABLE r1; -- Ensure cascaded DELETE works CREATE TABLE r1 (a int PRIMARY KEY); -- r1 is not a hypertable since r1.a is referenced by r2 CREATE TABLE r2 (a int REFERENCES r1 ON DELETE CASCADE); SELECT public.create_hypertable('r2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (27,regress_rls_schema,r2,t) INSERT INTO r1 VALUES (10), (20); INSERT INTO r2 VALUES (10), (20); -- Create policies on r2 which prevent the -- owner from seeing any rows, but RI should -- still see them. CREATE POLICY p1 ON r2 USING (false); ALTER TABLE r2 ENABLE ROW LEVEL SECURITY; ALTER TABLE r2 FORCE ROW LEVEL SECURITY; -- Deletes all records from both DELETE FROM r1; -- Remove FORCE from r2 ALTER TABLE r2 NO FORCE ROW LEVEL SECURITY; -- As owner, we now bypass RLS -- verify no rows in r2 now TABLE r2; a --- DROP TABLE r2; DROP TABLE r1; -- Ensure cascaded UPDATE works CREATE TABLE r1 (a int PRIMARY KEY); -- r1 is not a hypertable since r1.a is referenced by r2 CREATE TABLE r2 (a int REFERENCES r1 ON UPDATE CASCADE); SELECT public.create_hypertable('r2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (28,regress_rls_schema,r2,t) INSERT INTO r1 VALUES (10), (20); INSERT INTO r2 VALUES (10), (20); -- Create policies on r2 which prevent the -- owner from seeing any rows, but RI should -- still see them. CREATE POLICY p1 ON r2 USING (false); ALTER TABLE r2 ENABLE ROW LEVEL SECURITY; ALTER TABLE r2 FORCE ROW LEVEL SECURITY; -- Updates records in both (terse output to not print CONTEXT, which can be different). \set VERBOSITY terse UPDATE r1 SET a = a+5; ERROR: new row for relation "_hyper_28_117_chunk" violates check constraint "constraint_117" \set VERBOSITY default -- Remove FORCE from r2 ALTER TABLE r2 NO FORCE ROW LEVEL SECURITY; -- As owner, we now bypass RLS -- verify records in r2 updated TABLE r2; a ---- 10 20 DROP TABLE r2; DROP TABLE r1; -- -- Test INSERT+RETURNING applies SELECT policies as -- WithCheckOptions (meaning an error is thrown) -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security = on; CREATE TABLE r1 (a int); SELECT public.create_hypertable('r1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (29,regress_rls_schema,r1,t) CREATE POLICY p1 ON r1 FOR SELECT USING (false); CREATE POLICY p2 ON r1 FOR INSERT WITH CHECK (true); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- Works fine INSERT INTO r1 VALUES (10), (20); -- No error, but no rows TABLE r1; a --- SET row_security = off; -- fail, would be affected by RLS TABLE r1; ERROR: query would be affected by row-level security policy for table "r1" HINT: To disable the policy for the table's owner, use ALTER TABLE NO FORCE ROW LEVEL SECURITY. SET row_security = on; -- Error INSERT INTO r1 VALUES (10), (20) RETURNING *; ERROR: new row violates row-level security policy for table "r1" DROP TABLE r1; -- -- Test UPDATE+RETURNING applies SELECT policies as -- WithCheckOptions (meaning an error is thrown) -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security = on; CREATE TABLE r1 (a int PRIMARY KEY); SELECT public.create_hypertable('r1', 'a', chunk_time_interval=>100); create_hypertable ------------------------------ (30,regress_rls_schema,r1,t) CREATE POLICY p1 ON r1 FOR SELECT USING (a < 20); CREATE POLICY p2 ON r1 FOR UPDATE USING (a < 20) WITH CHECK (true); CREATE POLICY p3 ON r1 FOR INSERT WITH CHECK (true); INSERT INTO r1 VALUES (10); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- Works fine UPDATE r1 SET a = 30; -- Show updated rows ALTER TABLE r1 NO FORCE ROW LEVEL SECURITY; TABLE r1; a ---- 30 -- reset value in r1 for test with RETURNING UPDATE r1 SET a = 10; -- Verify row reset TABLE r1; a ---- 10 ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- Error UPDATE r1 SET a = 30 RETURNING *; ERROR: new row violates row-level security policy for table "r1" -- UPDATE path of INSERT ... ON CONFLICT DO UPDATE should also error out INSERT INTO r1 VALUES (10) ON CONFLICT (a) DO UPDATE SET a = 30 RETURNING *; ERROR: new row violates row-level security policy for table "r1" -- Should still error out without RETURNING (use of arbiter always requires -- SELECT permissions) INSERT INTO r1 VALUES (10) ON CONFLICT (a) DO UPDATE SET a = 30; ERROR: new row violates row-level security policy for table "r1" -- ON CONFLICT ON CONSTRAINT INSERT INTO r1 VALUES (10) ON CONFLICT ON CONSTRAINT r1_pkey DO UPDATE SET a = 30; ERROR: new row violates row-level security policy for table "r1" DROP TABLE r1; -- Check dependency handling RESET SESSION AUTHORIZATION; CREATE TABLE dep1 (c1 int); SELECT public.create_hypertable('dep1', 'c1', chunk_time_interval=>2); create_hypertable -------------------------------- (31,regress_rls_schema,dep1,t) CREATE TABLE dep2 (c1 int); SELECT public.create_hypertable('dep2', 'c1', chunk_time_interval=>2); create_hypertable -------------------------------- (32,regress_rls_schema,dep2,t) CREATE POLICY dep_p1 ON dep1 TO regress_rls_bob USING (c1 > (select max(dep2.c1) from dep2)); ALTER POLICY dep_p1 ON dep1 TO regress_rls_bob,regress_rls_carol; -- Should return one SELECT count(*) = 1 FROM pg_depend WHERE objid = (SELECT oid FROM pg_policy WHERE polname = 'dep_p1') AND refobjid = (SELECT oid FROM pg_class WHERE relname = 'dep2'); ?column? ---------- t ALTER POLICY dep_p1 ON dep1 USING (true); -- Should return one SELECT count(*) = 1 FROM pg_shdepend WHERE objid = (SELECT oid FROM pg_policy WHERE polname = 'dep_p1') AND refobjid = (SELECT oid FROM pg_authid WHERE rolname = 'regress_rls_bob'); ?column? ---------- t -- Should return one SELECT count(*) = 1 FROM pg_shdepend WHERE objid = (SELECT oid FROM pg_policy WHERE polname = 'dep_p1') AND refobjid = (SELECT oid FROM pg_authid WHERE rolname = 'regress_rls_carol'); ?column? ---------- t -- Should return zero SELECT count(*) = 0 FROM pg_depend WHERE objid = (SELECT oid FROM pg_policy WHERE polname = 'dep_p1') AND refobjid = (SELECT oid FROM pg_class WHERE relname = 'dep2'); ?column? ---------- t -- DROP OWNED BY testing RESET SESSION AUTHORIZATION; CREATE ROLE regress_rls_dob_role1; CREATE ROLE regress_rls_dob_role2; CREATE TABLE dob_t1 (c1 int); SELECT public.create_hypertable('dob_t1', 'c1', chunk_time_interval=>2); create_hypertable ---------------------------------- (33,regress_rls_schema,dob_t1,t) CREATE TABLE dob_t2 (c1 int) PARTITION BY RANGE (c1); CREATE POLICY p1 ON dob_t1 TO regress_rls_dob_role1 USING (true); DROP OWNED BY regress_rls_dob_role1; DROP POLICY p1 ON dob_t1; -- should fail, already gone ERROR: policy "p1" for table "dob_t1" does not exist CREATE POLICY p1 ON dob_t1 TO regress_rls_dob_role1,regress_rls_dob_role2 USING (true); DROP OWNED BY regress_rls_dob_role1; DROP POLICY p1 ON dob_t1; -- should succeed CREATE POLICY p1 ON dob_t2 TO regress_rls_dob_role1,regress_rls_dob_role2 USING (true); DROP OWNED BY regress_rls_dob_role1; DROP POLICY p1 ON dob_t2; -- should succeed DROP USER regress_rls_dob_role1; DROP USER regress_rls_dob_role2; -- -- Clean up objects -- RESET SESSION AUTHORIZATION; \set VERBOSITY terse \\ -- suppress cascade details DROP SCHEMA regress_rls_schema CASCADE; NOTICE: drop cascades to 116 other objects \set VERBOSITY default DROP USER regress_rls_alice; DROP USER regress_rls_bob; DROP USER regress_rls_carol; DROP USER regress_rls_dave; DROP USER regress_rls_exempt_user; DROP ROLE regress_rls_group1; DROP ROLE regress_rls_group2; -- Arrange to have a few policies left over, for testing -- pg_dump/pg_restore CREATE SCHEMA regress_rls_schema; CREATE TABLE rls_tbl (c1 int); SELECT public.create_hypertable('rls_tbl', 'c1', chunk_time_interval=>2); create_hypertable ----------------------------------- (34,regress_rls_schema,rls_tbl,t) ALTER TABLE rls_tbl ENABLE ROW LEVEL SECURITY; CREATE POLICY p1 ON rls_tbl USING (c1 > 5); CREATE POLICY p2 ON rls_tbl FOR SELECT USING (c1 <= 3); CREATE POLICY p3 ON rls_tbl FOR UPDATE USING (c1 <= 3) WITH CHECK (c1 > 5); CREATE POLICY p4 ON rls_tbl FOR DELETE USING (c1 <= 3); CREATE TABLE rls_tbl_force (c1 int); SELECT public.create_hypertable('rls_tbl_force', 'c1', chunk_time_interval=>2); create_hypertable ----------------------------------------- (35,regress_rls_schema,rls_tbl_force,t) ALTER TABLE rls_tbl_force ENABLE ROW LEVEL SECURITY; ALTER TABLE rls_tbl_force FORCE ROW LEVEL SECURITY; CREATE POLICY p1 ON rls_tbl_force USING (c1 = 5) WITH CHECK (c1 < 5); CREATE POLICY p2 ON rls_tbl_force FOR SELECT USING (c1 = 8); CREATE POLICY p3 ON rls_tbl_force FOR UPDATE USING (c1 = 8) WITH CHECK (c1 >= 5); CREATE POLICY p4 ON rls_tbl_force FOR DELETE USING (c1 = 8); ================================================ FILE: test/expected/rowsecurity-17.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- -- Test of Row-level security feature -- -- Clean up in case a prior regression run failed \c :TEST_DBNAME :ROLE_SUPERUSER \set ON_ERROR_STOP 0 \set VERBOSITY default SET timescaledb.enable_constraint_exclusion TO off; -- Suppress NOTICE messages when users/groups don't exist SET client_min_messages TO 'warning'; DROP USER IF EXISTS regress_rls_alice; DROP USER IF EXISTS regress_rls_bob; DROP USER IF EXISTS regress_rls_carol; DROP USER IF EXISTS regress_rls_dave; DROP USER IF EXISTS regress_rls_exempt_user; DROP ROLE IF EXISTS regress_rls_group1; DROP ROLE IF EXISTS regress_rls_group2; DROP SCHEMA IF EXISTS regress_rls_schema CASCADE; RESET client_min_messages; -- initial setup CREATE USER regress_rls_alice NOLOGIN; CREATE USER regress_rls_bob NOLOGIN; CREATE USER regress_rls_carol NOLOGIN; CREATE USER regress_rls_dave NOLOGIN; CREATE USER regress_rls_exempt_user BYPASSRLS NOLOGIN; CREATE ROLE regress_rls_group1 NOLOGIN; CREATE ROLE regress_rls_group2 NOLOGIN; GRANT regress_rls_group1 TO regress_rls_bob; GRANT regress_rls_group2 TO regress_rls_carol; CREATE SCHEMA regress_rls_schema; GRANT ALL ON SCHEMA regress_rls_schema to public; SET search_path = regress_rls_schema; -- setup of malicious function CREATE OR REPLACE FUNCTION f_leak(text) RETURNS bool COST 0.0000001 LANGUAGE plpgsql AS 'BEGIN RAISE NOTICE ''f_leak => %'', $1; RETURN true; END'; GRANT EXECUTE ON FUNCTION f_leak(text) TO public; -- BASIC Row-Level Security Scenario SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE uaccount ( pguser name primary key, seclv int ); GRANT SELECT ON uaccount TO public; INSERT INTO uaccount VALUES ('regress_rls_alice', 99), ('regress_rls_bob', 1), ('regress_rls_carol', 2), ('regress_rls_dave', 3); CREATE TABLE category ( cid int primary key, cname text ); GRANT ALL ON category TO public; INSERT INTO category VALUES (11, 'novel'), (22, 'science fiction'), (33, 'technology'), (44, 'manga'); CREATE TABLE document ( did int primary key, cid int references category(cid), dlevel int not null, dauthor name, dtitle text ); GRANT ALL ON document TO public; SELECT public.create_hypertable('document', 'did', chunk_time_interval=>2); create_hypertable ----------------------------------- (1,regress_rls_schema,document,t) INSERT INTO document VALUES ( 1, 11, 1, 'regress_rls_bob', 'my first novel'), ( 2, 11, 2, 'regress_rls_bob', 'my second novel'), ( 3, 22, 2, 'regress_rls_bob', 'my science fiction'), ( 4, 44, 1, 'regress_rls_bob', 'my first manga'), ( 5, 44, 2, 'regress_rls_bob', 'my second manga'), ( 6, 22, 1, 'regress_rls_carol', 'great science fiction'), ( 7, 33, 2, 'regress_rls_carol', 'great technology book'), ( 8, 44, 1, 'regress_rls_carol', 'great manga'), ( 9, 22, 1, 'regress_rls_dave', 'awesome science fiction'), (10, 33, 2, 'regress_rls_dave', 'awesome technology book'); ALTER TABLE document ENABLE ROW LEVEL SECURITY; -- user's security level must be higher than or equal to document's CREATE POLICY p1 ON document AS PERMISSIVE USING (dlevel <= (SELECT seclv FROM uaccount WHERE pguser = current_user)); -- try to create a policy of bogus type CREATE POLICY p1 ON document AS UGLY USING (dlevel <= (SELECT seclv FROM uaccount WHERE pguser = current_user)); ERROR: unrecognized row security option "ugly" LINE 1: CREATE POLICY p1 ON document AS UGLY ^ HINT: Only PERMISSIVE or RESTRICTIVE policies are supported currently. -- but Dave isn't allowed to anything at cid 50 or above -- this is to make sure that we sort the policies by name first -- when applying WITH CHECK, a later INSERT by Dave should fail due -- to p1r first CREATE POLICY p2r ON document AS RESTRICTIVE TO regress_rls_dave USING (cid <> 44 AND cid < 50); -- and Dave isn't allowed to see manga documents CREATE POLICY p1r ON document AS RESTRICTIVE TO regress_rls_dave USING (cid <> 44); \dp Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------------------+----------+-------+----------------------------------------------+-------------------+-------------------------------------------- regress_rls_schema | category | table | regress_rls_alice=arwdDxtm/regress_rls_alice+| | | | | =arwdDxtm/regress_rls_alice | | regress_rls_schema | document | table | regress_rls_alice=arwdDxtm/regress_rls_alice+| | p1: + | | | =arwdDxtm/regress_rls_alice | | (u): (dlevel <= ( SELECT uaccount.seclv + | | | | | FROM uaccount + | | | | | WHERE (uaccount.pguser = CURRENT_USER)))+ | | | | | p2r (RESTRICTIVE): + | | | | | (u): ((cid <> 44) AND (cid < 50)) + | | | | | to: regress_rls_dave + | | | | | p1r (RESTRICTIVE): + | | | | | (u): (cid <> 44) + | | | | | to: regress_rls_dave regress_rls_schema | uaccount | table | regress_rls_alice=arwdDxtm/regress_rls_alice+| | | | | =r/regress_rls_alice | | \d document Table "regress_rls_schema.document" Column | Type | Collation | Nullable | Default ---------+---------+-----------+----------+--------- did | integer | | not null | cid | integer | | | dlevel | integer | | not null | dauthor | name | | | dtitle | text | | | Indexes: "document_pkey" PRIMARY KEY, btree (did) Foreign-key constraints: "document_cid_fkey" FOREIGN KEY (cid) REFERENCES category(cid) Policies: POLICY "p1" USING ((dlevel <= ( SELECT uaccount.seclv FROM uaccount WHERE (uaccount.pguser = CURRENT_USER)))) POLICY "p1r" AS RESTRICTIVE TO regress_rls_dave USING ((cid <> 44)) POLICY "p2r" AS RESTRICTIVE TO regress_rls_dave USING (((cid <> 44) AND (cid < 50))) Number of child tables: 6 (Use \d+ to list them.) SELECT * FROM pg_policies WHERE schemaname = 'regress_rls_schema' AND tablename = 'document' ORDER BY policyname; schemaname | tablename | policyname | permissive | roles | cmd | qual | with_check --------------------+-----------+------------+-------------+--------------------+-----+--------------------------------------------+------------ regress_rls_schema | document | p1 | PERMISSIVE | {public} | ALL | (dlevel <= ( SELECT uaccount.seclv +| | | | | | | FROM uaccount +| | | | | | | WHERE (uaccount.pguser = CURRENT_USER))) | regress_rls_schema | document | p1r | RESTRICTIVE | {regress_rls_dave} | ALL | (cid <> 44) | regress_rls_schema | document | p2r | RESTRICTIVE | {regress_rls_dave} | ALL | ((cid <> 44) AND (cid < 50)) | -- viewpoint from regress_rls_bob SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my first manga NOTICE: f_leak => great science fiction NOTICE: f_leak => great manga NOTICE: f_leak => awesome science fiction did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 4 | 44 | 1 | regress_rls_bob | my first manga 6 | 22 | 1 | regress_rls_carol | great science fiction 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my first manga NOTICE: f_leak => great science fiction NOTICE: f_leak => great manga NOTICE: f_leak => awesome science fiction cid | did | dlevel | dauthor | dtitle | cname -----+-----+--------+-------------------+-------------------------+----------------- 11 | 1 | 1 | regress_rls_bob | my first novel | novel 44 | 4 | 1 | regress_rls_bob | my first manga | manga 22 | 6 | 1 | regress_rls_carol | great science fiction | science fiction 44 | 8 | 1 | regress_rls_carol | great manga | manga 22 | 9 | 1 | regress_rls_dave | awesome science fiction | science fiction -- try a sampled version SELECT * FROM document TABLESAMPLE BERNOULLI(50) REPEATABLE(0) WHERE f_leak(dtitle) ORDER BY did; did | cid | dlevel | dauthor | dtitle -----+-----+--------+---------+-------- -- viewpoint from regress_rls_carol SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => my first manga NOTICE: f_leak => my second manga NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great manga NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => my first manga NOTICE: f_leak => my second manga NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great manga NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book cid | did | dlevel | dauthor | dtitle | cname -----+-----+--------+-------------------+-------------------------+----------------- 11 | 1 | 1 | regress_rls_bob | my first novel | novel 11 | 2 | 2 | regress_rls_bob | my second novel | novel 22 | 3 | 2 | regress_rls_bob | my science fiction | science fiction 44 | 4 | 1 | regress_rls_bob | my first manga | manga 44 | 5 | 2 | regress_rls_bob | my second manga | manga 22 | 6 | 1 | regress_rls_carol | great science fiction | science fiction 33 | 7 | 2 | regress_rls_carol | great technology book | technology 44 | 8 | 1 | regress_rls_carol | great manga | manga 22 | 9 | 1 | regress_rls_dave | awesome science fiction | science fiction 33 | 10 | 2 | regress_rls_dave | awesome technology book | technology -- try a sampled version SELECT * FROM document TABLESAMPLE BERNOULLI(50) REPEATABLE(0) WHERE f_leak(dtitle) ORDER BY did; did | cid | dlevel | dauthor | dtitle -----+-----+--------+---------+-------- EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 InitPlan 1 -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on document document_1 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle); --- QUERY PLAN --- Hash Join Hash Cond: (document.cid = category.cid) InitPlan 1 -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 -> Seq Scan on document document_1 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Hash -> Seq Scan on category -- viewpoint from regress_rls_dave SET SESSION AUTHORIZATION regress_rls_dave; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book cid | did | dlevel | dauthor | dtitle | cname -----+-----+--------+-------------------+-------------------------+----------------- 11 | 1 | 1 | regress_rls_bob | my first novel | novel 11 | 2 | 2 | regress_rls_bob | my second novel | novel 22 | 3 | 2 | regress_rls_bob | my science fiction | science fiction 22 | 6 | 1 | regress_rls_carol | great science fiction | science fiction 33 | 7 | 2 | regress_rls_carol | great technology book | technology 22 | 9 | 1 | regress_rls_dave | awesome science fiction | science fiction 33 | 10 | 2 | regress_rls_dave | awesome technology book | technology EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 InitPlan 1 -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on document document_1 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle); --- QUERY PLAN --- Hash Join Hash Cond: (category.cid = document.cid) InitPlan 1 -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on category -> Hash -> Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 -> Seq Scan on document document_1 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -- 44 would technically fail for both p2r and p1r, but we should get an error -- back from p1r for this because it sorts first INSERT INTO document VALUES (100, 44, 1, 'regress_rls_dave', 'testing sorting of policies'); -- fail ERROR: new row violates row-level security policy "p1r" for table "document" -- Just to see a p2r error INSERT INTO document VALUES (100, 55, 1, 'regress_rls_dave', 'testing sorting of policies'); -- fail ERROR: new row violates row-level security policy "p2r" for table "document" -- only owner can change policies ALTER POLICY p1 ON document USING (true); --fail ERROR: must be owner of table document DROP POLICY p1 ON document; --fail ERROR: must be owner of relation document SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY p1 ON document USING (dauthor = current_user); -- viewpoint from regress_rls_bob again SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => my first manga NOTICE: f_leak => my second manga did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+-------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER by did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => my first manga NOTICE: f_leak => my second manga cid | did | dlevel | dauthor | dtitle | cname -----+-----+--------+-----------------+--------------------+----------------- 11 | 1 | 1 | regress_rls_bob | my first novel | novel 11 | 2 | 2 | regress_rls_bob | my second novel | novel 22 | 3 | 2 | regress_rls_bob | my science fiction | science fiction 44 | 4 | 1 | regress_rls_bob | my first manga | manga 44 | 5 | 2 | regress_rls_bob | my second manga | manga -- viewpoint from rls_regres_carol again SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great manga did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+----------------------- 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER by did; NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great manga cid | did | dlevel | dauthor | dtitle | cname -----+-----+--------+-------------------+-----------------------+----------------- 22 | 6 | 1 | regress_rls_carol | great science fiction | science fiction 33 | 7 | 2 | regress_rls_carol | great technology book | technology 44 | 8 | 1 | regress_rls_carol | great manga | manga EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 -> Seq Scan on document document_1 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle); --- QUERY PLAN --- Nested Loop -> Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 -> Seq Scan on document document_1 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Index Scan using category_pkey on category Index Cond: (cid = document.cid) -- interaction of FK/PK constraints SET SESSION AUTHORIZATION regress_rls_alice; CREATE POLICY p2 ON category USING (CASE WHEN current_user = 'regress_rls_bob' THEN cid IN (11, 33) WHEN current_user = 'regress_rls_carol' THEN cid IN (22, 44) ELSE false END); ALTER TABLE category ENABLE ROW LEVEL SECURITY; -- cannot delete PK referenced by invisible FK SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM document d FULL OUTER JOIN category c on d.cid = c.cid ORDER BY d.did, c.cid; did | cid | dlevel | dauthor | dtitle | cid | cname -----+-----+--------+-----------------+--------------------+-----+------------ 1 | 11 | 1 | regress_rls_bob | my first novel | 11 | novel 2 | 11 | 2 | regress_rls_bob | my second novel | 11 | novel 3 | 22 | 2 | regress_rls_bob | my science fiction | | 4 | 44 | 1 | regress_rls_bob | my first manga | | 5 | 44 | 2 | regress_rls_bob | my second manga | | | | | | | 33 | technology \set VERBOSITY sqlstate DELETE FROM category WHERE cid = 33; -- fails with FK violation ERROR: 23503 \set VERBOSITY default -- can insert FK referencing invisible PK SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM document d FULL OUTER JOIN category c on d.cid = c.cid ORDER BY d.did, c.cid; did | cid | dlevel | dauthor | dtitle | cid | cname -----+-----+--------+-------------------+-----------------------+-----+----------------- 6 | 22 | 1 | regress_rls_carol | great science fiction | 22 | science fiction 7 | 33 | 2 | regress_rls_carol | great technology book | | 8 | 44 | 1 | regress_rls_carol | great manga | 44 | manga INSERT INTO document VALUES (11, 33, 1, current_user, 'hoge'); -- UNIQUE or PRIMARY KEY constraint violation DOES reveal presence of row SET SESSION AUTHORIZATION regress_rls_bob; INSERT INTO document VALUES (8, 44, 1, 'regress_rls_bob', 'my third manga'); -- Must fail with unique violation, revealing presence of did we can't see ERROR: duplicate key value violates unique constraint "5_10_document_pkey" DETAIL: Key (did)=(8) already exists. SELECT * FROM document WHERE did = 8; -- and confirm we can't see it did | cid | dlevel | dauthor | dtitle -----+-----+--------+---------+-------- -- RLS policies are checked before constraints INSERT INTO document VALUES (8, 44, 1, 'regress_rls_carol', 'my third manga'); -- Should fail with RLS check violation, not duplicate key violation ERROR: new row violates row-level security policy for table "document" UPDATE document SET did = 8, dauthor = 'regress_rls_carol' WHERE did = 5; -- Should fail with RLS check violation, not duplicate key violation ERROR: new row violates row-level security policy for table "document" -- database superuser does bypass RLS policy when enabled RESET SESSION AUTHORIZATION; SET row_security TO ON; SELECT * FROM document; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book 11 | 33 | 1 | regress_rls_carol | hoge SELECT * FROM category; cid | cname -----+----------------- 11 | novel 22 | science fiction 33 | technology 44 | manga -- database superuser does bypass RLS policy when disabled RESET SESSION AUTHORIZATION; SET row_security TO OFF; SELECT * FROM document; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book 11 | 33 | 1 | regress_rls_carol | hoge SELECT * FROM category; cid | cname -----+----------------- 11 | novel 22 | science fiction 33 | technology 44 | manga -- database non-superuser with bypass privilege can bypass RLS policy when disabled SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; SELECT * FROM document; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book 11 | 33 | 1 | regress_rls_carol | hoge SELECT * FROM category; cid | cname -----+----------------- 11 | novel 22 | science fiction 33 | technology 44 | manga -- RLS policy does not apply to table owner when RLS enabled. SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; SELECT * FROM document; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book 11 | 33 | 1 | regress_rls_carol | hoge SELECT * FROM category; cid | cname -----+----------------- 11 | novel 22 | science fiction 33 | technology 44 | manga -- RLS policy does not apply to table owner when RLS disabled. SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO OFF; SELECT * FROM document; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book 11 | 33 | 1 | regress_rls_carol | hoge SELECT * FROM category; cid | cname -----+----------------- 11 | novel 22 | science fiction 33 | technology 44 | manga -- -- Table inheritance and RLS policy -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; CREATE TABLE t1 (a int, junk1 text, b text); ALTER TABLE t1 DROP COLUMN junk1; -- just a disturbing factor GRANT ALL ON t1 TO public; COPY t1 FROM stdin; CREATE TABLE t2 (c float) INHERITS (t1); GRANT ALL ON t2 TO public; COPY t2 FROM stdin; CREATE TABLE t3 (c text, b text, a int); ALTER TABLE t3 INHERIT t1; GRANT ALL ON t3 TO public; COPY t3(a,b,c) FROM stdin; CREATE POLICY p1 ON t1 FOR ALL TO PUBLIC USING (a % 2 = 0); -- be even number CREATE POLICY p2 ON t2 FOR ALL TO PUBLIC USING (a % 2 = 1); -- be odd number ALTER TABLE t1 ENABLE ROW LEVEL SECURITY; ALTER TABLE t2 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM t1; a | b ---+----- 2 | bbb 4 | dad 2 | bcd 4 | def 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: ((a % 2) = 0) -> Seq Scan on t2 t1_2 Filter: ((a % 2) = 0) -> Seq Scan on t3 t1_3 Filter: ((a % 2) = 0) SELECT * FROM t1 WHERE f_leak(b); NOTICE: f_leak => bbb NOTICE: f_leak => dad NOTICE: f_leak => bcd NOTICE: f_leak => def NOTICE: f_leak => yyy a | b ---+----- 2 | bbb 4 | dad 2 | bcd 4 | def 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -- reference to system column SELECT ctid, * FROM t1; ctid | a | b -------+---+----- (0,2) | 2 | bbb (0,4) | 4 | dad (0,2) | 2 | bcd (0,4) | 4 | def (0,2) | 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT *, t1 FROM t1; --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: ((a % 2) = 0) -> Seq Scan on t2 t1_2 Filter: ((a % 2) = 0) -> Seq Scan on t3 t1_3 Filter: ((a % 2) = 0) -- reference to whole-row reference SELECT *, t1 FROM t1; a | b | t1 ---+-----+--------- 2 | bbb | (2,bbb) 4 | dad | (4,dad) 2 | bcd | (2,bcd) 4 | def | (4,def) 2 | yyy | (2,yyy) EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT *, t1 FROM t1; --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: ((a % 2) = 0) -> Seq Scan on t2 t1_2 Filter: ((a % 2) = 0) -> Seq Scan on t3 t1_3 Filter: ((a % 2) = 0) -- for share/update lock SELECT * FROM t1 FOR SHARE; a | b ---+----- 2 | bbb 4 | dad 2 | bcd 4 | def 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 FOR SHARE; --- QUERY PLAN --- LockRows -> Append -> Seq Scan on t1 t1_1 Filter: ((a % 2) = 0) -> Seq Scan on t2 t1_2 Filter: ((a % 2) = 0) -> Seq Scan on t3 t1_3 Filter: ((a % 2) = 0) SELECT * FROM t1 WHERE f_leak(b) FOR SHARE; NOTICE: f_leak => bbb NOTICE: f_leak => dad NOTICE: f_leak => bcd NOTICE: f_leak => def NOTICE: f_leak => yyy a | b ---+----- 2 | bbb 4 | dad 2 | bcd 4 | def 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b) FOR SHARE; --- QUERY PLAN --- LockRows -> Append -> Seq Scan on t1 t1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -- union all query SELECT a, b, ctid FROM t2 UNION ALL SELECT a, b, ctid FROM t3; a | b | ctid ---+-----+------- 1 | abc | (0,1) 3 | cde | (0,3) 1 | xxx | (0,1) 2 | yyy | (0,2) 3 | zzz | (0,3) EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT a, b, ctid FROM t2 UNION ALL SELECT a, b, ctid FROM t3; --- QUERY PLAN --- Append -> Seq Scan on t2 Filter: ((a % 2) = 1) -> Seq Scan on t3 -- superuser is allowed to bypass RLS checks RESET SESSION AUTHORIZATION; SET row_security TO OFF; SELECT * FROM t1 WHERE f_leak(b); NOTICE: f_leak => aba NOTICE: f_leak => bbb NOTICE: f_leak => ccc NOTICE: f_leak => dad NOTICE: f_leak => abc NOTICE: f_leak => bcd NOTICE: f_leak => cde NOTICE: f_leak => def NOTICE: f_leak => xxx NOTICE: f_leak => yyy NOTICE: f_leak => zzz a | b ---+----- 1 | aba 2 | bbb 3 | ccc 4 | dad 1 | abc 2 | bcd 3 | cde 4 | def 1 | xxx 2 | yyy 3 | zzz EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: f_leak(b) -> Seq Scan on t2 t1_2 Filter: f_leak(b) -> Seq Scan on t3 t1_3 Filter: f_leak(b) -- non-superuser with bypass privilege can bypass RLS policy when disabled SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; SELECT * FROM t1 WHERE f_leak(b); NOTICE: f_leak => aba NOTICE: f_leak => bbb NOTICE: f_leak => ccc NOTICE: f_leak => dad NOTICE: f_leak => abc NOTICE: f_leak => bcd NOTICE: f_leak => cde NOTICE: f_leak => def NOTICE: f_leak => xxx NOTICE: f_leak => yyy NOTICE: f_leak => zzz a | b ---+----- 1 | aba 2 | bbb 3 | ccc 4 | dad 1 | abc 2 | bcd 3 | cde 4 | def 1 | xxx 2 | yyy 3 | zzz EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: f_leak(b) -> Seq Scan on t2 t1_2 Filter: f_leak(b) -> Seq Scan on t3 t1_3 Filter: f_leak(b) -- -- Hyper Tables -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE hyper_document ( did int, cid int, dlevel int not null, dauthor name, dtitle text ); GRANT ALL ON hyper_document TO public; SELECT public.create_hypertable('hyper_document', 'did', chunk_time_interval=>2); create_hypertable ----------------------------------------- (2,regress_rls_schema,hyper_document,t) INSERT INTO hyper_document VALUES ( 1, 11, 1, 'regress_rls_bob', 'my first novel'), ( 2, 11, 2, 'regress_rls_bob', 'my second novel'), ( 3, 99, 2, 'regress_rls_bob', 'my science textbook'), ( 4, 55, 1, 'regress_rls_bob', 'my first satire'), ( 5, 99, 2, 'regress_rls_bob', 'my history book'), ( 6, 11, 1, 'regress_rls_carol', 'great science fiction'), ( 7, 99, 2, 'regress_rls_carol', 'great technology book'), ( 8, 55, 2, 'regress_rls_carol', 'great satire'), ( 9, 11, 1, 'regress_rls_dave', 'awesome science fiction'), (10, 99, 2, 'regress_rls_dave', 'awesome technology book'); ALTER TABLE hyper_document ENABLE ROW LEVEL SECURITY; GRANT ALL ON _timescaledb_internal._hyper_2_9_chunk TO public; -- Create policy on parent -- user's security level must be higher than or equal to document's CREATE POLICY pp1 ON hyper_document AS PERMISSIVE USING (dlevel <= (SELECT seclv FROM uaccount WHERE pguser = current_user)); -- Dave is only allowed to see cid < 55 CREATE POLICY pp1r ON hyper_document AS RESTRICTIVE TO regress_rls_dave USING (cid < 55); \d+ hyper_document Table "regress_rls_schema.hyper_document" Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- did | integer | | not null | | plain | | cid | integer | | | | plain | | dlevel | integer | | not null | | plain | | dauthor | name | | | | plain | | dtitle | text | | | | extended | | Indexes: "hyper_document_did_idx" btree (did DESC) Policies: POLICY "pp1" USING ((dlevel <= ( SELECT uaccount.seclv FROM uaccount WHERE (uaccount.pguser = CURRENT_USER)))) POLICY "pp1r" AS RESTRICTIVE TO regress_rls_dave USING ((cid < 55)) Child tables: _timescaledb_internal._hyper_2_10_chunk, _timescaledb_internal._hyper_2_11_chunk, _timescaledb_internal._hyper_2_12_chunk, _timescaledb_internal._hyper_2_13_chunk, _timescaledb_internal._hyper_2_14_chunk, _timescaledb_internal._hyper_2_9_chunk SELECT * FROM pg_policies WHERE schemaname = 'regress_rls_schema' AND tablename like '%hyper_document%' ORDER BY policyname; schemaname | tablename | policyname | permissive | roles | cmd | qual | with_check --------------------+----------------+------------+-------------+--------------------+-----+--------------------------------------------+------------ regress_rls_schema | hyper_document | pp1 | PERMISSIVE | {public} | ALL | (dlevel <= ( SELECT uaccount.seclv +| | | | | | | FROM uaccount +| | | | | | | WHERE (uaccount.pguser = CURRENT_USER))) | regress_rls_schema | hyper_document | pp1r | RESTRICTIVE | {regress_rls_dave} | ALL | (cid < 55) | -- viewpoint from regress_rls_bob SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my first satire NOTICE: f_leak => great science fiction NOTICE: f_leak => awesome science fiction did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 4 | 55 | 1 | regress_rls_bob | my first satire 6 | 11 | 1 | regress_rls_carol | great science fiction 9 | 11 | 1 | regress_rls_dave | awesome science fiction EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 InitPlan 1 -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on hyper_document hyper_document_1 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -- viewpoint from regress_rls_carol SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science textbook NOTICE: f_leak => my first satire NOTICE: f_leak => my history book NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great satire NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire 9 | 11 | 1 | regress_rls_dave | awesome science fiction 10 | 99 | 2 | regress_rls_dave | awesome technology book EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 InitPlan 1 -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on hyper_document hyper_document_1 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -- viewpoint from regress_rls_dave SET SESSION AUTHORIZATION regress_rls_dave; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => great science fiction NOTICE: f_leak => awesome science fiction did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 6 | 11 | 1 | regress_rls_carol | great science fiction 9 | 11 | 1 | regress_rls_dave | awesome science fiction EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 InitPlan 1 -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on hyper_document hyper_document_1 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -- pp1 ERROR INSERT INTO hyper_document VALUES (1, 11, 5, 'regress_rls_dave', 'testing pp1'); -- fail ERROR: new row violates row-level security policy for table "hyper_document" -- pp1r ERROR INSERT INTO hyper_document VALUES (1, 99, 1, 'regress_rls_dave', 'testing pp1r'); -- fail ERROR: new row violates row-level security policy "pp1r" for table "hyper_document" -- Show that RLS policy does not apply for direct inserts to children -- This should fail with RLS POLICY pp1r violation. INSERT INTO hyper_document VALUES (1, 55, 1, 'regress_rls_dave', 'testing RLS with hypertables'); -- fail ERROR: new row violates row-level security policy "pp1r" for table "hyper_document" -- But this should succeed. INSERT INTO _timescaledb_internal._hyper_2_9_chunk VALUES (1, 55, 1, 'regress_rls_dave', 'testing RLS with hypertables'); -- success -- We still cannot see the row using the parent SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => great science fiction NOTICE: f_leak => awesome science fiction did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 6 | 11 | 1 | regress_rls_carol | great science fiction 9 | 11 | 1 | regress_rls_dave | awesome science fiction -- But we can if we look directly SELECT * FROM _timescaledb_internal._hyper_2_9_chunk WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => my first novel NOTICE: f_leak => testing RLS with hypertables did | cid | dlevel | dauthor | dtitle -----+-----+--------+------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables -- Turn on RLS and create policy on child to show RLS is checked before constraints SET SESSION AUTHORIZATION regress_rls_alice; ALTER TABLE _timescaledb_internal._hyper_2_9_chunk ENABLE ROW LEVEL SECURITY; CREATE POLICY pp3 ON _timescaledb_internal._hyper_2_9_chunk AS RESTRICTIVE USING (cid < 55); -- This should fail with RLS violation now. SET SESSION AUTHORIZATION regress_rls_dave; INSERT INTO _timescaledb_internal._hyper_2_9_chunk VALUES (1, 55, 1, 'regress_rls_dave', 'testing RLS with hypertables - round 2'); -- fail ERROR: new row violates row-level security policy for table "_hyper_2_9_chunk" -- And now we cannot see directly into the partition either, due to RLS SELECT * FROM _timescaledb_internal._hyper_2_9_chunk WHERE f_leak(dtitle) ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+---------+-------- -- The parent looks same as before -- viewpoint from regress_rls_dave SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => great science fiction NOTICE: f_leak => awesome science fiction did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 6 | 11 | 1 | regress_rls_carol | great science fiction 9 | 11 | 1 | regress_rls_dave | awesome science fiction EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 InitPlan 1 -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on hyper_document hyper_document_1 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -- viewpoint from regress_rls_carol SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => my first novel NOTICE: f_leak => testing RLS with hypertables NOTICE: f_leak => my second novel NOTICE: f_leak => my science textbook NOTICE: f_leak => my first satire NOTICE: f_leak => my history book NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great satire NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire 9 | 11 | 1 | regress_rls_dave | awesome science fiction 10 | 99 | 2 | regress_rls_dave | awesome technology book EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 InitPlan 1 -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on hyper_document hyper_document_1 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -- only owner can change policies ALTER POLICY pp1 ON hyper_document USING (true); --fail ERROR: must be owner of table hyper_document DROP POLICY pp1 ON hyper_document; --fail ERROR: must be owner of relation hyper_document SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY pp1 ON hyper_document USING (dauthor = current_user); -- viewpoint from regress_rls_bob again SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science textbook NOTICE: f_leak => my first satire NOTICE: f_leak => my history book did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+--------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book -- viewpoint from rls_regres_carol again SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great satire did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+----------------------- 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 -> Seq Scan on hyper_document hyper_document_1 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -- database superuser does bypass RLS policy when enabled RESET SESSION AUTHORIZATION; SET row_security TO ON; SELECT * FROM hyper_document ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire 9 | 11 | 1 | regress_rls_dave | awesome science fiction 10 | 99 | 2 | regress_rls_dave | awesome technology book SELECT * FROM _timescaledb_internal._hyper_2_9_chunk ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables -- database non-superuser with bypass privilege can bypass RLS policy when disabled SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; SELECT * FROM hyper_document ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire 9 | 11 | 1 | regress_rls_dave | awesome science fiction 10 | 99 | 2 | regress_rls_dave | awesome technology book SELECT * FROM _timescaledb_internal._hyper_2_9_chunk ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables -- RLS policy does not apply to table owner when RLS enabled. SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; SELECT * FROM hyper_document ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire 9 | 11 | 1 | regress_rls_dave | awesome science fiction 10 | 99 | 2 | regress_rls_dave | awesome technology book SELECT * FROM _timescaledb_internal._hyper_2_9_chunk ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables -- When RLS disabled, other users get ERROR. SET SESSION AUTHORIZATION regress_rls_dave; SET row_security TO OFF; SELECT * FROM hyper_document ORDER BY did, cid; ERROR: query would be affected by row-level security policy for table "hyper_document" SELECT * FROM _timescaledb_internal._hyper_2_9_chunk ORDER BY did, cid; ERROR: query would be affected by row-level security policy for table "_hyper_2_9_chunk" -- Check behavior with a policy that uses a SubPlan not an InitPlan. SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; CREATE POLICY pp3 ON hyper_document AS RESTRICTIVE USING ((SELECT dlevel <= seclv FROM uaccount WHERE pguser = current_user)); SET SESSION AUTHORIZATION regress_rls_carol; INSERT INTO hyper_document VALUES (100, 11, 5, 'regress_rls_carol', 'testing pp3'); -- fail ERROR: new row violates row-level security policy "pp3" for table "hyper_document" ----- Dependencies ----- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; CREATE TABLE dependee (x integer, y integer); SELECT public.create_hypertable('dependee', 'x', chunk_time_interval=>2); create_hypertable ----------------------------------- (3,regress_rls_schema,dependee,t) CREATE TABLE dependent (x integer, y integer); SELECT public.create_hypertable('dependent', 'x', chunk_time_interval=>2); create_hypertable ------------------------------------ (4,regress_rls_schema,dependent,t) CREATE POLICY d1 ON dependent FOR ALL TO PUBLIC USING (x = (SELECT d.x FROM dependee d WHERE d.y = y)); DROP TABLE dependee; -- Should fail without CASCADE due to dependency on row security qual? ERROR: cannot drop table dependee because other objects depend on it DETAIL: policy d1 on table dependent depends on table dependee HINT: Use DROP ... CASCADE to drop the dependent objects too. DROP TABLE dependee CASCADE; NOTICE: drop cascades to policy d1 on table dependent EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM dependent; -- After drop, should be unqualified --- QUERY PLAN --- Seq Scan on dependent ----- RECURSION ---- -- -- Simple recursion -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE rec1 (x integer, y integer); SELECT public.create_hypertable('rec1', 'x', chunk_time_interval=>2); create_hypertable ------------------------------- (5,regress_rls_schema,rec1,t) CREATE POLICY r1 ON rec1 USING (x = (SELECT r.x FROM rec1 r WHERE y = r.y)); ALTER TABLE rec1 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rec1; -- fail, direct recursion ERROR: infinite recursion detected in policy for relation "rec1" -- -- Mutual recursion -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE rec2 (a integer, b integer); SELECT public.create_hypertable('rec2', 'x', chunk_time_interval=>2); ERROR: column "x" does not exist ALTER POLICY r1 ON rec1 USING (x = (SELECT a FROM rec2 WHERE b = y)); CREATE POLICY r2 ON rec2 USING (a = (SELECT x FROM rec1 WHERE y = b)); ALTER TABLE rec2 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rec1; -- fail, mutual recursion ERROR: infinite recursion detected in policy for relation "rec1" -- -- Mutual recursion via views -- SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW rec1v AS SELECT * FROM rec1; CREATE VIEW rec2v AS SELECT * FROM rec2; SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY r1 ON rec1 USING (x = (SELECT a FROM rec2v WHERE b = y)); ALTER POLICY r2 ON rec2 USING (a = (SELECT x FROM rec1v WHERE y = b)); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rec1; -- fail, mutual recursion via views ERROR: infinite recursion detected in policy for relation "rec1" -- -- Mutual recursion via .s.b views -- SET SESSION AUTHORIZATION regress_rls_bob; \set VERBOSITY terse \\ -- suppress cascade details DROP VIEW rec1v, rec2v CASCADE; NOTICE: drop cascades to 2 other objects \set VERBOSITY default CREATE VIEW rec1v WITH (security_barrier) AS SELECT * FROM rec1; CREATE VIEW rec2v WITH (security_barrier) AS SELECT * FROM rec2; SET SESSION AUTHORIZATION regress_rls_alice; CREATE POLICY r1 ON rec1 USING (x = (SELECT a FROM rec2v WHERE b = y)); CREATE POLICY r2 ON rec2 USING (a = (SELECT x FROM rec1v WHERE y = b)); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rec1; -- fail, mutual recursion via s.b. views ERROR: infinite recursion detected in policy for relation "rec1" -- -- recursive RLS and VIEWs in policy -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE s1 (a int, b text); SELECT public.create_hypertable('s1', 'a', chunk_time_interval=>2); create_hypertable ----------------------------- (6,regress_rls_schema,s1,t) INSERT INTO s1 (SELECT x, md5(x::text) FROM generate_series(-10,10) x); CREATE TABLE s2 (x int, y text); SELECT public.create_hypertable('s2', 'x', chunk_time_interval=>2); create_hypertable ----------------------------- (7,regress_rls_schema,s2,t) INSERT INTO s2 (SELECT x, md5(x::text) FROM generate_series(-6,6) x); GRANT SELECT ON s1, s2 TO regress_rls_bob; CREATE POLICY p1 ON s1 USING (a in (select x from s2 where y like '%2f%')); CREATE POLICY p2 ON s2 USING (x in (select a from s1 where b like '%22%')); CREATE POLICY p3 ON s1 FOR INSERT WITH CHECK (a = (SELECT a FROM s1)); ALTER TABLE s1 ENABLE ROW LEVEL SECURITY; ALTER TABLE s2 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW v2 AS SELECT * FROM s2 WHERE y like '%af%'; SELECT * FROM s1 WHERE f_leak(b); -- fail (infinite recursion) ERROR: infinite recursion detected in policy for relation "s1" INSERT INTO s1 VALUES (1, 'foo'); -- fail (infinite recursion) ERROR: infinite recursion detected in policy for relation "s1" SET SESSION AUTHORIZATION regress_rls_alice; DROP POLICY p3 on s1; ALTER POLICY p2 ON s2 USING (x % 2 = 0); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM s1 WHERE f_leak(b); -- OK NOTICE: f_leak => c81e728d9d4c2f636f067f89cc14862c NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c a | b ---+---------------------------------- 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM only s1 WHERE f_leak(b); --- QUERY PLAN --- Seq Scan on s1 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) SubPlan 1 -> Append -> Seq Scan on s2 s2_1 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_27_chunk s2_2 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_28_chunk s2_3 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_29_chunk s2_4 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_30_chunk s2_5 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_31_chunk s2_6 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_32_chunk s2_7 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_33_chunk s2_8 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY p1 ON s1 USING (a in (select x from v2)); -- using VIEW in RLS policy SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM s1 WHERE f_leak(b); -- OK NOTICE: f_leak => 0267aaf632e87a63288a08331f22c7c3 NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc a | b ----+---------------------------------- -4 | 0267aaf632e87a63288a08331f22c7c3 6 | 1679091c5a880faf6fb5e6087eb1b2dc EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM s1 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on s1 Chunks excluded during startup: 0 -> Seq Scan on s1 s1_1 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) SubPlan 1 -> Append -> Seq Scan on s2 s2_1 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_27_chunk s2_2 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_28_chunk s2_3 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_29_chunk s2_4 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_30_chunk s2_5 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_31_chunk s2_6 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_32_chunk s2_7 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_33_chunk s2_8 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_6_16_chunk s1_2 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) -> Seq Scan on _hyper_6_17_chunk s1_3 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) -> Seq Scan on _hyper_6_18_chunk s1_4 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) -> Seq Scan on _hyper_6_19_chunk s1_5 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) -> Seq Scan on _hyper_6_20_chunk s1_6 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) -> Seq Scan on _hyper_6_21_chunk s1_7 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) -> Seq Scan on _hyper_6_22_chunk s1_8 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) -> Seq Scan on _hyper_6_23_chunk s1_9 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) -> Seq Scan on _hyper_6_24_chunk s1_10 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) -> Seq Scan on _hyper_6_25_chunk s1_11 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) -> Seq Scan on _hyper_6_26_chunk s1_12 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) SELECT (SELECT x FROM s1 LIMIT 1) xx, * FROM s2 WHERE y like '%28%'; xx | x | y ----+----+---------------------------------- -6 | -6 | 596a3d04481816330f07e4f97510c28f -4 | -4 | 0267aaf632e87a63288a08331f22c7c3 2 | 2 | c81e728d9d4c2f636f067f89cc14862c EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT (SELECT x FROM s1 LIMIT 1) xx, * FROM s2 WHERE y like '%28%'; --- QUERY PLAN --- Result -> Append -> Seq Scan on s2 s2_1 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_27_chunk s2_2 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_28_chunk s2_3 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_29_chunk s2_4 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_30_chunk s2_5 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_31_chunk s2_6 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_32_chunk s2_7 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_33_chunk s2_8 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) SubPlan 2 -> Limit -> Result -> Custom Scan (ChunkAppend) on s1 -> Seq Scan on s1 s1_1 Filter: (ANY (a = (hashed SubPlan 1).col1)) SubPlan 1 -> Append -> Seq Scan on s2 s2_10 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_27_chunk s2_11 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_28_chunk s2_12 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_29_chunk s2_13 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_30_chunk s2_14 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_31_chunk s2_15 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_32_chunk s2_16 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_33_chunk s2_17 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_6_16_chunk s1_2 Filter: (ANY (a = (hashed SubPlan 1).col1)) -> Seq Scan on _hyper_6_17_chunk s1_3 Filter: (ANY (a = (hashed SubPlan 1).col1)) -> Seq Scan on _hyper_6_18_chunk s1_4 Filter: (ANY (a = (hashed SubPlan 1).col1)) -> Seq Scan on _hyper_6_19_chunk s1_5 Filter: (ANY (a = (hashed SubPlan 1).col1)) -> Seq Scan on _hyper_6_20_chunk s1_6 Filter: (ANY (a = (hashed SubPlan 1).col1)) -> Seq Scan on _hyper_6_21_chunk s1_7 Filter: (ANY (a = (hashed SubPlan 1).col1)) -> Seq Scan on _hyper_6_22_chunk s1_8 Filter: (ANY (a = (hashed SubPlan 1).col1)) -> Seq Scan on _hyper_6_23_chunk s1_9 Filter: (ANY (a = (hashed SubPlan 1).col1)) -> Seq Scan on _hyper_6_24_chunk s1_10 Filter: (ANY (a = (hashed SubPlan 1).col1)) -> Seq Scan on _hyper_6_25_chunk s1_11 Filter: (ANY (a = (hashed SubPlan 1).col1)) -> Seq Scan on _hyper_6_26_chunk s1_12 Filter: (ANY (a = (hashed SubPlan 1).col1)) SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY p2 ON s2 USING (x in (select a from s1 where b like '%d2%')); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM s1 WHERE f_leak(b); -- fail (infinite recursion via view) ERROR: infinite recursion detected in policy for relation "s1" -- prepared statement with regress_rls_alice privilege PREPARE p1(int) AS SELECT * FROM t1 WHERE a <= $1; EXECUTE p1(2); a | b ---+----- 2 | bbb 2 | bcd 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE p1(2); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: ((a <= 2) AND ((a % 2) = 0)) -> Seq Scan on t2 t1_2 Filter: ((a <= 2) AND ((a % 2) = 0)) -> Seq Scan on t3 t1_3 Filter: ((a <= 2) AND ((a % 2) = 0)) -- superuser is allowed to bypass RLS checks RESET SESSION AUTHORIZATION; SET row_security TO OFF; SELECT * FROM t1 WHERE f_leak(b); NOTICE: f_leak => aba NOTICE: f_leak => bbb NOTICE: f_leak => ccc NOTICE: f_leak => dad NOTICE: f_leak => abc NOTICE: f_leak => bcd NOTICE: f_leak => cde NOTICE: f_leak => def NOTICE: f_leak => xxx NOTICE: f_leak => yyy NOTICE: f_leak => zzz a | b ---+----- 1 | aba 2 | bbb 3 | ccc 4 | dad 1 | abc 2 | bcd 3 | cde 4 | def 1 | xxx 2 | yyy 3 | zzz EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: f_leak(b) -> Seq Scan on t2 t1_2 Filter: f_leak(b) -> Seq Scan on t3 t1_3 Filter: f_leak(b) -- plan cache should be invalidated EXECUTE p1(2); a | b ---+----- 1 | aba 2 | bbb 1 | abc 2 | bcd 1 | xxx 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE p1(2); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: (a <= 2) -> Seq Scan on t2 t1_2 Filter: (a <= 2) -> Seq Scan on t3 t1_3 Filter: (a <= 2) PREPARE p2(int) AS SELECT * FROM t1 WHERE a = $1; EXECUTE p2(2); a | b ---+----- 2 | bbb 2 | bcd 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE p2(2); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: (a = 2) -> Seq Scan on t2 t1_2 Filter: (a = 2) -> Seq Scan on t3 t1_3 Filter: (a = 2) -- also, case when privilege switch from superuser SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; EXECUTE p2(2); a | b ---+----- 2 | bbb 2 | bcd 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE p2(2); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: ((a = 2) AND ((a % 2) = 0)) -> Seq Scan on t2 t1_2 Filter: ((a = 2) AND ((a % 2) = 0)) -> Seq Scan on t3 t1_3 Filter: ((a = 2) AND ((a % 2) = 0)) -- -- UPDATE / DELETE and Row-level security -- SET SESSION AUTHORIZATION regress_rls_bob; EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t1 SET b = b || b WHERE f_leak(b); --- QUERY PLAN --- Update on t1 Update on t1 t1_1 Update on t2 t1_2 Update on t3 t1_3 -> Result -> Append -> Seq Scan on t1 t1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: (((a % 2) = 0) AND f_leak(b)) UPDATE t1 SET b = b || b WHERE f_leak(b); NOTICE: f_leak => bbb NOTICE: f_leak => dad NOTICE: f_leak => bcd NOTICE: f_leak => def NOTICE: f_leak => yyy EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE only t1 SET b = b || '_updt' WHERE f_leak(b); --- QUERY PLAN --- Update on t1 -> Seq Scan on t1 Filter: (((a % 2) = 0) AND f_leak(b)) UPDATE only t1 SET b = b || '_updt' WHERE f_leak(b); NOTICE: f_leak => bbbbbb NOTICE: f_leak => daddad -- returning clause with system column UPDATE only t1 SET b = b WHERE f_leak(b) RETURNING ctid, *, t1; NOTICE: f_leak => bbbbbb_updt NOTICE: f_leak => daddad_updt ctid | a | b | t1 --------+---+-------------+----------------- (0,9) | 2 | bbbbbb_updt | (2,bbbbbb_updt) (0,10) | 4 | daddad_updt | (4,daddad_updt) UPDATE t1 SET b = b WHERE f_leak(b) RETURNING *; NOTICE: f_leak => bbbbbb_updt NOTICE: f_leak => daddad_updt NOTICE: f_leak => bcdbcd NOTICE: f_leak => defdef NOTICE: f_leak => yyyyyy a | b ---+------------- 2 | bbbbbb_updt 4 | daddad_updt 2 | bcdbcd 4 | defdef 2 | yyyyyy UPDATE t1 SET b = b WHERE f_leak(b) RETURNING ctid, *, t1; NOTICE: f_leak => bbbbbb_updt NOTICE: f_leak => daddad_updt NOTICE: f_leak => bcdbcd NOTICE: f_leak => defdef NOTICE: f_leak => yyyyyy ctid | a | b | t1 --------+---+-------------+----------------- (0,13) | 2 | bbbbbb_updt | (2,bbbbbb_updt) (0,14) | 4 | daddad_updt | (4,daddad_updt) (0,9) | 2 | bcdbcd | (2,bcdbcd) (0,10) | 4 | defdef | (4,defdef) (0,6) | 2 | yyyyyy | (2,yyyyyy) -- updates with from clause EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t2 SET b=t2.b FROM t3 WHERE t2.a = 3 and t3.a = 2 AND f_leak(t2.b) AND f_leak(t3.b); --- QUERY PLAN --- Update on t2 -> Nested Loop -> Seq Scan on t2 Filter: ((a = 3) AND ((a % 2) = 1) AND f_leak(b)) -> Seq Scan on t3 Filter: ((a = 2) AND f_leak(b)) UPDATE t2 SET b=t2.b FROM t3 WHERE t2.a = 3 and t3.a = 2 AND f_leak(t2.b) AND f_leak(t3.b); NOTICE: f_leak => cde NOTICE: f_leak => yyyyyy EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t1 SET b=t1.b FROM t2 WHERE t1.a = 3 and t2.a = 3 AND f_leak(t1.b) AND f_leak(t2.b); --- QUERY PLAN --- Update on t1 Update on t1 t1_1 Update on t2 t1_2 Update on t3 t1_3 -> Nested Loop -> Seq Scan on t2 Filter: ((a = 3) AND ((a % 2) = 1) AND f_leak(b)) -> Append -> Seq Scan on t1 t1_1 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) UPDATE t1 SET b=t1.b FROM t2 WHERE t1.a = 3 and t2.a = 3 AND f_leak(t1.b) AND f_leak(t2.b); NOTICE: f_leak => cde EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t2 SET b=t2.b FROM t1 WHERE t1.a = 3 and t2.a = 3 AND f_leak(t1.b) AND f_leak(t2.b); --- QUERY PLAN --- Update on t2 -> Nested Loop -> Seq Scan on t2 Filter: ((a = 3) AND ((a % 2) = 1) AND f_leak(b)) -> Append -> Seq Scan on t1 t1_1 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) UPDATE t2 SET b=t2.b FROM t1 WHERE t1.a = 3 and t2.a = 3 AND f_leak(t1.b) AND f_leak(t2.b); NOTICE: f_leak => cde -- updates with from clause self join EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t2 t2_1 SET b = t2_2.b FROM t2 t2_2 WHERE t2_1.a = 3 AND t2_2.a = t2_1.a AND t2_2.b = t2_1.b AND f_leak(t2_1.b) AND f_leak(t2_2.b) RETURNING *, t2_1, t2_2; --- QUERY PLAN --- Update on t2 t2_1 -> Nested Loop Join Filter: (t2_1.b = t2_2.b) -> Seq Scan on t2 t2_1 Filter: ((a = 3) AND ((a % 2) = 1) AND f_leak(b)) -> Seq Scan on t2 t2_2 Filter: ((a = 3) AND ((a % 2) = 1) AND f_leak(b)) UPDATE t2 t2_1 SET b = t2_2.b FROM t2 t2_2 WHERE t2_1.a = 3 AND t2_2.a = t2_1.a AND t2_2.b = t2_1.b AND f_leak(t2_1.b) AND f_leak(t2_2.b) RETURNING *, t2_1, t2_2; NOTICE: f_leak => cde NOTICE: f_leak => cde a | b | c | a | b | c | t2_1 | t2_2 ---+-----+-----+---+-----+-----+-------------+------------- 3 | cde | 3.3 | 3 | cde | 3.3 | (3,cde,3.3) | (3,cde,3.3) EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t1 t1_1 SET b = t1_2.b FROM t1 t1_2 WHERE t1_1.a = 4 AND t1_2.a = t1_1.a AND t1_2.b = t1_1.b AND f_leak(t1_1.b) AND f_leak(t1_2.b) RETURNING *, t1_1, t1_2; --- QUERY PLAN --- Update on t1 t1_1 Update on t1 t1_1_1 Update on t2 t1_1_2 Update on t3 t1_1_3 -> Nested Loop Join Filter: (t1_1.b = t1_2.b) -> Append -> Seq Scan on t1 t1_1_1 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_1_2 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_1_3 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) -> Materialize -> Append -> Seq Scan on t1 t1_2_1 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2_2 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_2_3 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) UPDATE t1 t1_1 SET b = t1_2.b FROM t1 t1_2 WHERE t1_1.a = 4 AND t1_2.a = t1_1.a AND t1_2.b = t1_1.b AND f_leak(t1_1.b) AND f_leak(t1_2.b) RETURNING *, t1_1, t1_2; NOTICE: f_leak => daddad_updt NOTICE: f_leak => daddad_updt NOTICE: f_leak => defdef NOTICE: f_leak => defdef a | b | a | b | t1_1 | t1_2 ---+-------------+---+-------------+-----------------+----------------- 4 | daddad_updt | 4 | daddad_updt | (4,daddad_updt) | (4,daddad_updt) 4 | defdef | 4 | defdef | (4,defdef) | (4,defdef) RESET SESSION AUTHORIZATION; SET row_security TO OFF; SELECT * FROM t1 ORDER BY a,b; a | b ---+------------- 1 | aba 1 | abc 1 | xxx 2 | bbbbbb_updt 2 | bcdbcd 2 | yyyyyy 3 | ccc 3 | cde 3 | zzz 4 | daddad_updt 4 | defdef SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; EXPLAIN (BUFFERS OFF, COSTS OFF) DELETE FROM only t1 WHERE f_leak(b); --- QUERY PLAN --- Delete on t1 -> Seq Scan on t1 Filter: (((a % 2) = 0) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) DELETE FROM t1 WHERE f_leak(b); --- QUERY PLAN --- Delete on t1 Delete on t1 t1_1 Delete on t2 t1_2 Delete on t3 t1_3 -> Append -> Seq Scan on t1 t1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: (((a % 2) = 0) AND f_leak(b)) DELETE FROM only t1 WHERE f_leak(b) RETURNING ctid, *, t1; NOTICE: f_leak => bbbbbb_updt NOTICE: f_leak => daddad_updt ctid | a | b | t1 --------+---+-------------+----------------- (0,13) | 2 | bbbbbb_updt | (2,bbbbbb_updt) (0,15) | 4 | daddad_updt | (4,daddad_updt) DELETE FROM t1 WHERE f_leak(b) RETURNING ctid, *, t1; NOTICE: f_leak => bcdbcd NOTICE: f_leak => defdef NOTICE: f_leak => yyyyyy ctid | a | b | t1 --------+---+--------+------------ (0,9) | 2 | bcdbcd | (2,bcdbcd) (0,13) | 4 | defdef | (4,defdef) (0,6) | 2 | yyyyyy | (2,yyyyyy) -- -- S.b. view on top of Row-level security -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE b1 (a int, b text); SELECT public.create_hypertable('b1', 'a', chunk_time_interval=>2); create_hypertable ----------------------------- (8,regress_rls_schema,b1,t) INSERT INTO b1 (SELECT x, md5(x::text) FROM generate_series(-10,10) x); CREATE POLICY p1 ON b1 USING (a % 2 = 0); ALTER TABLE b1 ENABLE ROW LEVEL SECURITY; GRANT ALL ON b1 TO regress_rls_bob; SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW bv1 WITH (security_barrier) AS SELECT * FROM b1 WHERE a > 0 WITH CHECK OPTION; GRANT ALL ON bv1 TO regress_rls_carol; SET SESSION AUTHORIZATION regress_rls_carol; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM bv1 WHERE f_leak(b); --- QUERY PLAN --- Subquery Scan on bv1 Filter: f_leak(bv1.b) -> Append -> Seq Scan on b1 b1_1 Filter: ((a > 0) AND ((a % 2) = 0)) -> Index Scan using _hyper_8_39_chunk_b1_a_idx on _hyper_8_39_chunk b1_2 Index Cond: (a > 0) Filter: ((a % 2) = 0) -> Index Scan using _hyper_8_40_chunk_b1_a_idx on _hyper_8_40_chunk b1_3 Index Cond: (a > 0) Filter: ((a % 2) = 0) -> Index Scan using _hyper_8_41_chunk_b1_a_idx on _hyper_8_41_chunk b1_4 Index Cond: (a > 0) Filter: ((a % 2) = 0) -> Index Scan using _hyper_8_42_chunk_b1_a_idx on _hyper_8_42_chunk b1_5 Index Cond: (a > 0) Filter: ((a % 2) = 0) -> Index Scan using _hyper_8_43_chunk_b1_a_idx on _hyper_8_43_chunk b1_6 Index Cond: (a > 0) Filter: ((a % 2) = 0) -> Index Scan using _hyper_8_44_chunk_b1_a_idx on _hyper_8_44_chunk b1_7 Index Cond: (a > 0) Filter: ((a % 2) = 0) SELECT * FROM bv1 WHERE f_leak(b); NOTICE: f_leak => c81e728d9d4c2f636f067f89cc14862c NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc NOTICE: f_leak => c9f0f895fb98ab9159f51fd0297e236d NOTICE: f_leak => d3d9446802a44259755d38e6d163e820 a | b ----+---------------------------------- 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 INSERT INTO bv1 VALUES (-1, 'xxx'); -- should fail view WCO ERROR: new row violates row-level security policy for table "b1" INSERT INTO bv1 VALUES (11, 'xxx'); -- should fail RLS check ERROR: new row violates row-level security policy for table "b1" INSERT INTO bv1 VALUES (12, 'xxx'); -- ok EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE bv1 SET b = 'yyy' WHERE a = 4 AND f_leak(b); --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Update on b1 Update on _hyper_8_41_chunk b1_1 -> Result -> Custom Scan (ChunkAppend) on b1 Chunks excluded during startup: 0 -> Index Scan using _hyper_8_41_chunk_b1_a_idx on _hyper_8_41_chunk b1_1 Index Cond: ((a > 0) AND (a = 4)) Filter: (((a % 2) = 0) AND f_leak(b)) UPDATE bv1 SET b = 'yyy' WHERE a = 4 AND f_leak(b); NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c EXPLAIN (BUFFERS OFF, COSTS OFF) DELETE FROM bv1 WHERE a = 6 AND f_leak(b); --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Delete on b1 Delete on _hyper_8_42_chunk b1_1 -> Custom Scan (ChunkAppend) on b1 Chunks excluded during startup: 0 -> Index Scan using _hyper_8_42_chunk_b1_a_idx on _hyper_8_42_chunk b1_1 Index Cond: ((a > 0) AND (a = 6)) Filter: (((a % 2) = 0) AND f_leak(b)) DELETE FROM bv1 WHERE a = 6 AND f_leak(b); NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM b1; a | b -----+---------------------------------- -10 | 1b0fd9efa5279c4203b7c70233f86dbf -9 | 252e691406782824eec43d7eadc3d256 -8 | a8d2ec85eaf98407310b72eb73dda247 -7 | 74687a12d3915d3c4d83f1af7b3683d5 -6 | 596a3d04481816330f07e4f97510c28f -5 | 47c1b025fa18ea96c33fbb6718688c0f -4 | 0267aaf632e87a63288a08331f22c7c3 -3 | b3149ecea4628efd23d2f86e5a723472 -2 | 5d7b9adcbe1c629ec722529dd12e5129 -1 | 6bb61e3b7bce0931da574d19d1d82c88 0 | cfcd208495d565ef66e7dff9f98764da 1 | c4ca4238a0b923820dcc509a6f75849b 2 | c81e728d9d4c2f636f067f89cc14862c 3 | eccbc87e4b5ce2fe28308fd9f2a7baf3 5 | e4da3b7fbbce2345d7772b0674a318d5 4 | yyy 7 | 8f14e45fceea167a5a36dedd4bea2543 8 | c9f0f895fb98ab9159f51fd0297e236d 9 | 45c48cce2e2d7fbdea1afc51c7c6ad26 10 | d3d9446802a44259755d38e6d163e820 12 | xxx -- -- INSERT ... ON CONFLICT DO UPDATE and Row-level security -- SET SESSION AUTHORIZATION regress_rls_alice; DROP POLICY p1 ON document; DROP POLICY p1r ON document; CREATE POLICY p1 ON document FOR SELECT USING (true); CREATE POLICY p2 ON document FOR INSERT WITH CHECK (dauthor = current_user); CREATE POLICY p3 ON document FOR UPDATE USING (cid = (SELECT cid from category WHERE cname = 'novel')) WITH CHECK (dauthor = current_user); SET SESSION AUTHORIZATION regress_rls_bob; -- Exists... SELECT * FROM document WHERE did = 2; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+----------------- 2 | 11 | 2 | regress_rls_bob | my second novel -- ...so violates actual WITH CHECK OPTION within UPDATE (not INSERT, since -- alternative UPDATE path happens to be taken): INSERT INTO document VALUES (2, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_carol', 'my first novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, dauthor = EXCLUDED.dauthor; ERROR: new row violates row-level security policy for table "document" -- Violates USING qual for UPDATE policy p3. -- -- UPDATE path is taken, but UPDATE fails purely because *existing* row to be -- updated is not a "novel"/cid 11 (row is not leaked, even though we have -- SELECT privileges sufficient to see the row in this instance): INSERT INTO document VALUES (33, 22, 1, 'regress_rls_bob', 'okay science fiction'); -- preparation for next statement INSERT INTO document VALUES (33, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'Some novel, replaces sci-fi') -- takes UPDATE path ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle; ERROR: new row violates row-level security policy (USING expression) for table "document" -- Fine (we UPDATE, since INSERT WCOs and UPDATE security barrier quals + WCOs -- not violated): INSERT INTO document VALUES (2, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'my first novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+---------------- 2 | 11 | 2 | regress_rls_bob | my first novel -- Fine (we INSERT, so "cid = 33" ("technology") isn't evaluated): INSERT INTO document VALUES (78, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'some technology novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33 RETURNING *; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+----------------------- 78 | 11 | 1 | regress_rls_bob | some technology novel -- Fine (same query, but we UPDATE, so "cid = 33", ("technology") is not the -- case in respect of *existing* tuple): INSERT INTO document VALUES (78, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'some technology novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33 RETURNING *; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+----------------------- 78 | 33 | 1 | regress_rls_bob | some technology novel -- Same query a third time, but now fails due to existing tuple finally not -- passing quals: INSERT INTO document VALUES (78, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'some technology novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33 RETURNING *; ERROR: new row violates row-level security policy (USING expression) for table "document" -- Don't fail just because INSERT doesn't satisfy WITH CHECK option that -- originated as a barrier/USING() qual from the UPDATE. Note that the UPDATE -- path *isn't* taken, and so UPDATE-related policy does not apply: INSERT INTO document VALUES (79, (SELECT cid from category WHERE cname = 'technology'), 1, 'regress_rls_bob', 'technology book, can only insert') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+---------------------------------- 79 | 33 | 1 | regress_rls_bob | technology book, can only insert -- But this time, the same statement fails, because the UPDATE path is taken, -- and updating the row just inserted falls afoul of security barrier qual -- (enforced as WCO) -- what we might have updated target tuple to is -- irrelevant, in fact. INSERT INTO document VALUES (79, (SELECT cid from category WHERE cname = 'technology'), 1, 'regress_rls_bob', 'technology book, can only insert') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; ERROR: new row violates row-level security policy (USING expression) for table "document" -- Test default USING qual enforced as WCO SET SESSION AUTHORIZATION regress_rls_alice; DROP POLICY p1 ON document; DROP POLICY p2 ON document; DROP POLICY p3 ON document; CREATE POLICY p3_with_default ON document FOR UPDATE USING (cid = (SELECT cid from category WHERE cname = 'novel')); SET SESSION AUTHORIZATION regress_rls_bob; -- Just because WCO-style enforcement of USING quals occurs with -- existing/target tuple does not mean that the implementation can be allowed -- to fail to also enforce this qual against the final tuple appended to -- relation (since in the absence of an explicit WCO, this is also interpreted -- as an UPDATE/ALL WCO in general). -- -- UPDATE path is taken here (fails due to existing tuple). Note that this is -- not reported as a "USING expression", because it's an RLS UPDATE check that originated as -- a USING qual for the purposes of RLS in general, as opposed to an explicit -- USING qual that is ordinarily a security barrier. We leave it up to the -- UPDATE to make this fail: INSERT INTO document VALUES (79, (SELECT cid from category WHERE cname = 'technology'), 1, 'regress_rls_bob', 'technology book, can only insert') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; ERROR: new row violates row-level security policy for table "document" -- UPDATE path is taken here. Existing tuple passes, since it's cid -- corresponds to "novel", but default USING qual is enforced against -- post-UPDATE tuple too (as always when updating with a policy that lacks an -- explicit WCO), and so this fails: INSERT INTO document VALUES (2, (SELECT cid from category WHERE cname = 'technology'), 1, 'regress_rls_bob', 'my first novel') ON CONFLICT (did) DO UPDATE SET cid = EXCLUDED.cid, dtitle = EXCLUDED.dtitle RETURNING *; ERROR: new row violates row-level security policy for table "document" SET SESSION AUTHORIZATION regress_rls_alice; DROP POLICY p3_with_default ON document; -- -- Test ALL policies with ON CONFLICT DO UPDATE (much the same as existing UPDATE -- tests) -- CREATE POLICY p3_with_all ON document FOR ALL USING (cid = (SELECT cid from category WHERE cname = 'novel')) WITH CHECK (dauthor = current_user); SET SESSION AUTHORIZATION regress_rls_bob; -- Fails, since ALL WCO is enforced in insert path: INSERT INTO document VALUES (80, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_carol', 'my first novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33; ERROR: new row violates row-level security policy for table "document" -- Fails, since ALL policy USING qual is enforced (existing, target tuple is in -- violation, since it has the "manga" cid): INSERT INTO document VALUES (4, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'my first novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle; ERROR: new row violates row-level security policy (USING expression) for table "document" -- Fails, since ALL WCO are enforced: INSERT INTO document VALUES (1, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'my first novel') ON CONFLICT (did) DO UPDATE SET dauthor = 'regress_rls_carol'; ERROR: new row violates row-level security policy for table "document" -- -- ROLE/GROUP -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE z1 (a int, b text); SELECT public.create_hypertable('z1', 'a', chunk_time_interval=>2); create_hypertable ----------------------------- (9,regress_rls_schema,z1,t) CREATE TABLE z2 (a int, b text); SELECT public.create_hypertable('z2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (10,regress_rls_schema,z2,t) GRANT SELECT ON z1,z2 TO regress_rls_group1, regress_rls_group2, regress_rls_bob, regress_rls_carol; INSERT INTO z1 VALUES (1, 'aba'), (2, 'bbb'), (3, 'ccc'), (4, 'dad'); CREATE POLICY p1 ON z1 TO regress_rls_group1 USING (a % 2 = 0); CREATE POLICY p2 ON z1 TO regress_rls_group2 USING (a % 2 = 1); ALTER TABLE z1 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM z1 WHERE f_leak(b); NOTICE: f_leak => bbb NOTICE: f_leak => dad a | b ---+----- 2 | bbb 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM z1 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) PREPARE plancache_test AS SELECT * FROM z1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) PREPARE plancache_test2 AS WITH q AS MATERIALIZED (SELECT * FROM z1 WHERE f_leak(b)) SELECT * FROM q,z2; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test2; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 PREPARE plancache_test4 AS WITH q AS (SELECT * FROM z1 WHERE f_leak(b)) SELECT * FROM q,z2; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test4; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 PREPARE plancache_test6 AS WITH q AS NOT MATERIALIZED (SELECT * FROM z1 WHERE f_leak(b)) SELECT * FROM q,z2; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test6; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 PREPARE plancache_test3 AS WITH q AS MATERIALIZED (SELECT * FROM z2) SELECT * FROM q,z1 WHERE f_leak(z1.b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test3; --- QUERY PLAN --- Nested Loop CTE q -> Seq Scan on z2 -> CTE Scan on q -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) PREPARE plancache_test5 AS WITH q AS (SELECT * FROM z2) SELECT * FROM q,z1 WHERE f_leak(z1.b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test5; --- QUERY PLAN --- Nested Loop -> Seq Scan on z2 -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) PREPARE plancache_test7 AS WITH q AS NOT MATERIALIZED (SELECT * FROM z2) SELECT * FROM q,z1 WHERE f_leak(z1.b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test7; --- QUERY PLAN --- Nested Loop -> Seq Scan on z2 -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) SET ROLE regress_rls_group1; SELECT * FROM z1 WHERE f_leak(b); NOTICE: f_leak => bbb NOTICE: f_leak => dad a | b ---+----- 2 | bbb 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM z1 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test2; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test4; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test3; --- QUERY PLAN --- Nested Loop CTE q -> Seq Scan on z2 -> CTE Scan on q -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test5; --- QUERY PLAN --- Nested Loop -> Seq Scan on z2 -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM z1 WHERE f_leak(b); NOTICE: f_leak => aba NOTICE: f_leak => ccc a | b ---+----- 1 | aba 3 | ccc EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM z1 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test2; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test4; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test3; --- QUERY PLAN --- Nested Loop CTE q -> Seq Scan on z2 -> CTE Scan on q -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test5; --- QUERY PLAN --- Nested Loop -> Seq Scan on z2 -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) SET ROLE regress_rls_group2; SELECT * FROM z1 WHERE f_leak(b); NOTICE: f_leak => aba NOTICE: f_leak => ccc a | b ---+----- 1 | aba 3 | ccc EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM z1 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test2; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test4; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test3; --- QUERY PLAN --- Nested Loop CTE q -> Seq Scan on z2 -> CTE Scan on q -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test5; --- QUERY PLAN --- Nested Loop -> Seq Scan on z2 -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) -- -- Views should follow policy for view owner. -- -- View and Table owner are the same. SET SESSION AUTHORIZATION regress_rls_alice; CREATE VIEW rls_view AS SELECT * FROM z1 WHERE f_leak(b); GRANT SELECT ON rls_view TO regress_rls_bob; -- Query as role that is not owner of view or table. Should return all records. SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rls_view; NOTICE: f_leak => aba NOTICE: f_leak => bbb NOTICE: f_leak => ccc NOTICE: f_leak => dad a | b ---+----- 1 | aba 2 | bbb 3 | ccc 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: f_leak(b) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: f_leak(b) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: f_leak(b) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: f_leak(b) -- Query as view/table owner. Should return all records. SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM rls_view; NOTICE: f_leak => aba NOTICE: f_leak => bbb NOTICE: f_leak => ccc NOTICE: f_leak => dad a | b ---+----- 1 | aba 2 | bbb 3 | ccc 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: f_leak(b) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: f_leak(b) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: f_leak(b) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: f_leak(b) DROP VIEW rls_view; -- View and Table owners are different. SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW rls_view AS SELECT * FROM z1 WHERE f_leak(b); GRANT SELECT ON rls_view TO regress_rls_alice; -- Query as role that is not owner of view but is owner of table. -- Should return records based on view owner policies. SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM rls_view; NOTICE: f_leak => bbb NOTICE: f_leak => dad a | b ---+----- 2 | bbb 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -- Query as role that is not owner of table but is owner of view. -- Should return records based on view owner policies. SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rls_view; NOTICE: f_leak => bbb NOTICE: f_leak => dad a | b ---+----- 2 | bbb 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -- Query as role that is not the owner of the table or view without permissions. SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM rls_view; --fail - permission denied. ERROR: permission denied for view rls_view EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --fail - permission denied. ERROR: permission denied for view rls_view -- Query as role that is not the owner of the table or view with permissions. SET SESSION AUTHORIZATION regress_rls_bob; GRANT SELECT ON rls_view TO regress_rls_carol; SELECT * FROM rls_view; NOTICE: f_leak => bbb NOTICE: f_leak => dad a | b ---+----- 2 | bbb 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) SET SESSION AUTHORIZATION regress_rls_bob; DROP VIEW rls_view; -- -- Command specific -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE x1 (a int, b text, c text); SELECT public.create_hypertable('x1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (11,regress_rls_schema,x1,t) GRANT ALL ON x1 TO PUBLIC; INSERT INTO x1 VALUES (1, 'abc', 'regress_rls_bob'), (2, 'bcd', 'regress_rls_bob'), (3, 'cde', 'regress_rls_carol'), (4, 'def', 'regress_rls_carol'), (5, 'efg', 'regress_rls_bob'), (6, 'fgh', 'regress_rls_bob'), (7, 'fgh', 'regress_rls_carol'), (8, 'fgh', 'regress_rls_carol'); CREATE POLICY p0 ON x1 FOR ALL USING (c = current_user); CREATE POLICY p1 ON x1 FOR SELECT USING (a % 2 = 0); CREATE POLICY p2 ON x1 FOR INSERT WITH CHECK (a % 2 = 1); CREATE POLICY p3 ON x1 FOR UPDATE USING (a % 2 = 0); CREATE POLICY p4 ON x1 FOR DELETE USING (a < 8); ALTER TABLE x1 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM x1 WHERE f_leak(b) ORDER BY a ASC; NOTICE: f_leak => abc NOTICE: f_leak => bcd NOTICE: f_leak => def NOTICE: f_leak => efg NOTICE: f_leak => fgh NOTICE: f_leak => fgh a | b | c ---+-----+------------------- 1 | abc | regress_rls_bob 2 | bcd | regress_rls_bob 4 | def | regress_rls_carol 5 | efg | regress_rls_bob 6 | fgh | regress_rls_bob 8 | fgh | regress_rls_carol UPDATE x1 SET b = b || '_updt' WHERE f_leak(b) RETURNING *; NOTICE: f_leak => abc NOTICE: f_leak => bcd NOTICE: f_leak => def NOTICE: f_leak => efg NOTICE: f_leak => fgh NOTICE: f_leak => fgh a | b | c ---+----------+------------------- 1 | abc_updt | regress_rls_bob 2 | bcd_updt | regress_rls_bob 4 | def_updt | regress_rls_carol 5 | efg_updt | regress_rls_bob 6 | fgh_updt | regress_rls_bob 8 | fgh_updt | regress_rls_carol SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM x1 WHERE f_leak(b) ORDER BY a ASC; NOTICE: f_leak => cde NOTICE: f_leak => bcd_updt NOTICE: f_leak => def_updt NOTICE: f_leak => fgh NOTICE: f_leak => fgh_updt NOTICE: f_leak => fgh_updt a | b | c ---+----------+------------------- 2 | bcd_updt | regress_rls_bob 3 | cde | regress_rls_carol 4 | def_updt | regress_rls_carol 6 | fgh_updt | regress_rls_bob 7 | fgh | regress_rls_carol 8 | fgh_updt | regress_rls_carol UPDATE x1 SET b = b || '_updt' WHERE f_leak(b) RETURNING *; NOTICE: f_leak => cde NOTICE: f_leak => bcd_updt NOTICE: f_leak => def_updt NOTICE: f_leak => fgh NOTICE: f_leak => fgh_updt NOTICE: f_leak => fgh_updt a | b | c ---+---------------+------------------- 3 | cde_updt | regress_rls_carol 2 | bcd_updt_updt | regress_rls_bob 4 | def_updt_updt | regress_rls_carol 7 | fgh_updt | regress_rls_carol 6 | fgh_updt_updt | regress_rls_bob 8 | fgh_updt_updt | regress_rls_carol DELETE FROM x1 WHERE f_leak(b) RETURNING *; NOTICE: f_leak => cde_updt NOTICE: f_leak => bcd_updt_updt NOTICE: f_leak => def_updt_updt NOTICE: f_leak => fgh_updt NOTICE: f_leak => fgh_updt_updt NOTICE: f_leak => fgh_updt_updt a | b | c ---+---------------+------------------- 3 | cde_updt | regress_rls_carol 2 | bcd_updt_updt | regress_rls_bob 4 | def_updt_updt | regress_rls_carol 7 | fgh_updt | regress_rls_carol 6 | fgh_updt_updt | regress_rls_bob 8 | fgh_updt_updt | regress_rls_carol -- -- Duplicate Policy Names -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE y1 (a int, b text); SELECT public.create_hypertable('y1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (12,regress_rls_schema,y1,t) INSERT INTO y1 VALUES(1,2); CREATE TABLE y2 (a int, b text); SELECT public.create_hypertable('y2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (13,regress_rls_schema,y2,t) GRANT ALL ON y1, y2 TO regress_rls_bob; CREATE POLICY p1 ON y1 FOR ALL USING (a % 2 = 0); CREATE POLICY p2 ON y1 FOR SELECT USING (a > 2); CREATE POLICY p1 ON y1 FOR SELECT USING (a % 2 = 1); --fail ERROR: policy "p1" for table "y1" already exists CREATE POLICY p1 ON y2 FOR ALL USING (a % 2 = 0); --OK ALTER TABLE y1 ENABLE ROW LEVEL SECURITY; ALTER TABLE y2 ENABLE ROW LEVEL SECURITY; -- -- Expression structure with SBV -- -- Create view as table owner. RLS should NOT be applied. SET SESSION AUTHORIZATION regress_rls_alice; CREATE VIEW rls_sbv WITH (security_barrier) AS SELECT * FROM y1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_sbv WHERE (a = 1); --- QUERY PLAN --- Custom Scan (ChunkAppend) on y1 Chunks excluded during startup: 0 -> Seq Scan on y1 y1_1 Filter: (f_leak(b) AND (a = 1)) -> Index Scan using _hyper_12_57_chunk_y1_a_idx on _hyper_12_57_chunk y1_2 Index Cond: (a = 1) Filter: f_leak(b) DROP VIEW rls_sbv; -- Create view as role that does not own table. RLS should be applied. SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW rls_sbv WITH (security_barrier) AS SELECT * FROM y1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_sbv WHERE (a = 1); --- QUERY PLAN --- Custom Scan (ChunkAppend) on y1 Chunks excluded during startup: 0 -> Seq Scan on y1 y1_1 Filter: ((a = 1) AND ((a > 2) OR ((a % 2) = 0)) AND f_leak(b)) -> Index Scan using _hyper_12_57_chunk_y1_a_idx on _hyper_12_57_chunk y1_2 Index Cond: (a = 1) Filter: (((a > 2) OR ((a % 2) = 0)) AND f_leak(b)) DROP VIEW rls_sbv; -- -- Expression structure -- SET SESSION AUTHORIZATION regress_rls_alice; INSERT INTO y2 (SELECT x, md5(x::text) FROM generate_series(0,20) x); CREATE POLICY p2 ON y2 USING (a % 3 = 0); CREATE POLICY p3 ON y2 USING (a % 4 = 0); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM y2 WHERE f_leak(b); NOTICE: f_leak => cfcd208495d565ef66e7dff9f98764da NOTICE: f_leak => c81e728d9d4c2f636f067f89cc14862c NOTICE: f_leak => eccbc87e4b5ce2fe28308fd9f2a7baf3 NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc NOTICE: f_leak => c9f0f895fb98ab9159f51fd0297e236d NOTICE: f_leak => 45c48cce2e2d7fbdea1afc51c7c6ad26 NOTICE: f_leak => d3d9446802a44259755d38e6d163e820 NOTICE: f_leak => c20ad4d76fe97759aa27a0c99bff6710 NOTICE: f_leak => aab3238922bcc25a6f606eb525ffdc56 NOTICE: f_leak => 9bf31c7ff062936a96d3c8bd1f8f2ff3 NOTICE: f_leak => c74d97b01eae257e44aa9d5bade97baf NOTICE: f_leak => 6f4922f45568161a8cdf4ad2299f6d23 NOTICE: f_leak => 98f13708210194c475687be6106a3b84 a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 3 | eccbc87e4b5ce2fe28308fd9f2a7baf3 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 9 | 45c48cce2e2d7fbdea1afc51c7c6ad26 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 15 | 9bf31c7ff062936a96d3c8bd1f8f2ff3 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM y2 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on y2 Chunks excluded during startup: 0 -> Seq Scan on y2 y2_1 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_58_chunk y2_2 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_59_chunk y2_3 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_60_chunk y2_4 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_61_chunk y2_5 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_62_chunk y2_6 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_63_chunk y2_7 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_64_chunk y2_8 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_65_chunk y2_9 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_66_chunk y2_10 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_67_chunk y2_11 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_68_chunk y2_12 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -- -- Qual push-down of leaky functions, when not referring to table -- SELECT * FROM y2 WHERE f_leak('abc'); NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 3 | eccbc87e4b5ce2fe28308fd9f2a7baf3 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 9 | 45c48cce2e2d7fbdea1afc51c7c6ad26 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 15 | 9bf31c7ff062936a96d3c8bd1f8f2ff3 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM y2 WHERE f_leak('abc'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on y2 Chunks excluded during startup: 0 -> Seq Scan on y2 y2_1 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_58_chunk y2_2 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_59_chunk y2_3 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_60_chunk y2_4 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_61_chunk y2_5 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_62_chunk y2_6 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_63_chunk y2_7 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_64_chunk y2_8 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_65_chunk y2_9 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_66_chunk y2_10 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_67_chunk y2_11 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_68_chunk y2_12 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) CREATE TABLE test_qual_pushdown ( abc text ); INSERT INTO test_qual_pushdown VALUES ('abc'),('def'); SELECT * FROM y2 JOIN test_qual_pushdown ON (b = abc) WHERE f_leak(abc); NOTICE: f_leak => abc NOTICE: f_leak => def a | b | abc ---+---+----- EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM y2 JOIN test_qual_pushdown ON (b = abc) WHERE f_leak(abc); --- QUERY PLAN --- Hash Join Hash Cond: (y2.b = test_qual_pushdown.abc) -> Append -> Seq Scan on y2 y2_1 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_58_chunk y2_2 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_59_chunk y2_3 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_60_chunk y2_4 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_61_chunk y2_5 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_62_chunk y2_6 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_63_chunk y2_7 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_64_chunk y2_8 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_65_chunk y2_9 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_66_chunk y2_10 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_67_chunk y2_11 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_68_chunk y2_12 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Hash -> Seq Scan on test_qual_pushdown Filter: f_leak(abc) SELECT * FROM y2 JOIN test_qual_pushdown ON (b = abc) WHERE f_leak(b); NOTICE: f_leak => cfcd208495d565ef66e7dff9f98764da NOTICE: f_leak => c81e728d9d4c2f636f067f89cc14862c NOTICE: f_leak => eccbc87e4b5ce2fe28308fd9f2a7baf3 NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc NOTICE: f_leak => c9f0f895fb98ab9159f51fd0297e236d NOTICE: f_leak => 45c48cce2e2d7fbdea1afc51c7c6ad26 NOTICE: f_leak => d3d9446802a44259755d38e6d163e820 NOTICE: f_leak => c20ad4d76fe97759aa27a0c99bff6710 NOTICE: f_leak => aab3238922bcc25a6f606eb525ffdc56 NOTICE: f_leak => 9bf31c7ff062936a96d3c8bd1f8f2ff3 NOTICE: f_leak => c74d97b01eae257e44aa9d5bade97baf NOTICE: f_leak => 6f4922f45568161a8cdf4ad2299f6d23 NOTICE: f_leak => 98f13708210194c475687be6106a3b84 a | b | abc ---+---+----- EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM y2 JOIN test_qual_pushdown ON (b = abc) WHERE f_leak(b); --- QUERY PLAN --- Hash Join Hash Cond: (test_qual_pushdown.abc = y2.b) -> Seq Scan on test_qual_pushdown -> Hash -> Custom Scan (ChunkAppend) on y2 Chunks excluded during startup: 0 -> Seq Scan on y2 y2_1 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_58_chunk y2_2 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_59_chunk y2_3 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_60_chunk y2_4 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_61_chunk y2_5 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_62_chunk y2_6 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_63_chunk y2_7 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_64_chunk y2_8 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_65_chunk y2_9 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_66_chunk y2_10 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_67_chunk y2_11 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_68_chunk y2_12 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) DROP TABLE test_qual_pushdown; -- -- Plancache invalidate on user change. -- RESET SESSION AUTHORIZATION; \set VERBOSITY terse \\ -- suppress cascade details DROP TABLE t1 CASCADE; NOTICE: drop cascades to 2 other objects \set VERBOSITY default CREATE TABLE t1 (a integer); SELECT public.create_hypertable('t1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (14,regress_rls_schema,t1,t) GRANT SELECT ON t1 TO regress_rls_bob, regress_rls_carol; CREATE POLICY p1 ON t1 TO regress_rls_bob USING ((a % 2) = 0); CREATE POLICY p2 ON t1 TO regress_rls_carol USING ((a % 4) = 0); ALTER TABLE t1 ENABLE ROW LEVEL SECURITY; -- Prepare as regress_rls_bob SET ROLE regress_rls_bob; PREPARE role_inval AS SELECT * FROM t1; -- Check plan EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE role_inval; --- QUERY PLAN --- Seq Scan on t1 Filter: ((a % 2) = 0) -- Change to regress_rls_carol SET ROLE regress_rls_carol; -- Check plan- should be different EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE role_inval; --- QUERY PLAN --- Seq Scan on t1 Filter: ((a % 4) = 0) -- Change back to regress_rls_bob SET ROLE regress_rls_bob; -- Check plan- should be back to original EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE role_inval; --- QUERY PLAN --- Seq Scan on t1 Filter: ((a % 2) = 0) -- -- CTE and RLS -- RESET SESSION AUTHORIZATION; DROP TABLE t1 CASCADE; CREATE TABLE t1 (a integer, b text); SELECT public.create_hypertable('t1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (15,regress_rls_schema,t1,t) CREATE POLICY p1 ON t1 USING (a % 2 = 0); ALTER TABLE t1 ENABLE ROW LEVEL SECURITY; GRANT ALL ON t1 TO regress_rls_bob; INSERT INTO t1 (SELECT x, md5(x::text) FROM generate_series(0,20) x); SET SESSION AUTHORIZATION regress_rls_bob; WITH cte1 AS (SELECT * FROM t1 WHERE f_leak(b)) SELECT * FROM cte1; NOTICE: f_leak => cfcd208495d565ef66e7dff9f98764da NOTICE: f_leak => c81e728d9d4c2f636f067f89cc14862c NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc NOTICE: f_leak => c9f0f895fb98ab9159f51fd0297e236d NOTICE: f_leak => d3d9446802a44259755d38e6d163e820 NOTICE: f_leak => c20ad4d76fe97759aa27a0c99bff6710 NOTICE: f_leak => aab3238922bcc25a6f606eb525ffdc56 NOTICE: f_leak => c74d97b01eae257e44aa9d5bade97baf NOTICE: f_leak => 6f4922f45568161a8cdf4ad2299f6d23 NOTICE: f_leak => 98f13708210194c475687be6106a3b84 a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 EXPLAIN (BUFFERS OFF, COSTS OFF) WITH cte1 AS (SELECT * FROM t1 WHERE f_leak(b)) SELECT * FROM cte1; --- QUERY PLAN --- CTE Scan on cte1 CTE cte1 -> Custom Scan (ChunkAppend) on t1 Chunks excluded during startup: 0 -> Seq Scan on t1 t1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_69_chunk t1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_70_chunk t1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_71_chunk t1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_72_chunk t1_5 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_73_chunk t1_6 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_74_chunk t1_7 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_75_chunk t1_8 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_76_chunk t1_9 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_77_chunk t1_10 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_78_chunk t1_11 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_79_chunk t1_12 Filter: (((a % 2) = 0) AND f_leak(b)) WITH cte1 AS (UPDATE t1 SET a = a + 1 RETURNING *) SELECT * FROM cte1; --fail ERROR: new row violates row-level security policy for table "t1" WITH cte1 AS (UPDATE t1 SET a = a RETURNING *) SELECT * FROM cte1; --ok a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 WITH cte1 AS (INSERT INTO t1 VALUES (21, 'Fail') RETURNING *) SELECT * FROM cte1; --fail ERROR: new row violates row-level security policy for table "t1" WITH cte1 AS (INSERT INTO t1 VALUES (20, 'Success') RETURNING *) SELECT * FROM cte1; --ok a | b ----+--------- 20 | Success -- -- Rename Policy -- RESET SESSION AUTHORIZATION; ALTER POLICY p1 ON t1 RENAME TO p1; --fail ERROR: policy "p1" for table "t1" already exists SELECT polname, relname FROM pg_policy pol JOIN pg_class pc ON (pc.oid = pol.polrelid) WHERE relname = 't1'; polname | relname ---------+--------- p1 | t1 ALTER POLICY p1 ON t1 RENAME TO p2; --ok SELECT polname, relname FROM pg_policy pol JOIN pg_class pc ON (pc.oid = pol.polrelid) WHERE relname = 't1'; polname | relname ---------+--------- p2 | t1 -- -- Check INSERT SELECT -- SET SESSION AUTHORIZATION regress_rls_bob; CREATE TABLE t2 (a integer, b text); SELECT public.create_hypertable('t2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (16,regress_rls_schema,t2,t) INSERT INTO t2 (SELECT * FROM t1); EXPLAIN (BUFFERS OFF, COSTS OFF) INSERT INTO t2 (SELECT * FROM t1); --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on t2 -> Append -> Seq Scan on t1 t1_1 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_69_chunk t1_2 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_70_chunk t1_3 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_71_chunk t1_4 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_72_chunk t1_5 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_73_chunk t1_6 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_74_chunk t1_7 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_75_chunk t1_8 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_76_chunk t1_9 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_77_chunk t1_10 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_78_chunk t1_11 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_79_chunk t1_12 Filter: ((a % 2) = 0) SELECT * FROM t2; a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 20 | Success EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t2; --- QUERY PLAN --- Append -> Seq Scan on t2 t2_1 -> Seq Scan on _hyper_16_80_chunk t2_2 -> Seq Scan on _hyper_16_81_chunk t2_3 -> Seq Scan on _hyper_16_82_chunk t2_4 -> Seq Scan on _hyper_16_83_chunk t2_5 -> Seq Scan on _hyper_16_84_chunk t2_6 -> Seq Scan on _hyper_16_85_chunk t2_7 -> Seq Scan on _hyper_16_86_chunk t2_8 -> Seq Scan on _hyper_16_87_chunk t2_9 -> Seq Scan on _hyper_16_88_chunk t2_10 -> Seq Scan on _hyper_16_89_chunk t2_11 -> Seq Scan on _hyper_16_90_chunk t2_12 CREATE TABLE t3 AS SELECT * FROM t1; SELECT public.create_hypertable('t2', 'a', chunk_time_interval=>2); ERROR: table "t2" is already a hypertable SELECT * FROM t3; a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 20 | Success SELECT * INTO t4 FROM t1; SELECT * FROM t4; a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 20 | Success -- -- RLS with JOIN -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE blog (id integer, author text, post text); SELECT public.create_hypertable('blog', 'id', chunk_time_interval=>2); create_hypertable -------------------------------- (17,regress_rls_schema,blog,t) CREATE TABLE comment (blog_id integer, message text); SELECT public.create_hypertable('comment', 'blog_id', chunk_time_interval=>2); create_hypertable ----------------------------------- (18,regress_rls_schema,comment,t) GRANT ALL ON blog, comment TO regress_rls_bob; CREATE POLICY blog_1 ON blog USING (id % 2 = 0); ALTER TABLE blog ENABLE ROW LEVEL SECURITY; INSERT INTO blog VALUES (1, 'alice', 'blog #1'), (2, 'bob', 'blog #1'), (3, 'alice', 'blog #2'), (4, 'alice', 'blog #3'), (5, 'john', 'blog #1'); INSERT INTO comment VALUES (1, 'cool blog'), (1, 'fun blog'), (3, 'crazy blog'), (5, 'what?'), (4, 'insane!'), (2, 'who did it?'); SET SESSION AUTHORIZATION regress_rls_bob; -- Check RLS JOIN with Non-RLS. SELECT id, author, message FROM blog JOIN comment ON id = blog_id; id | author | message ----+--------+------------- 2 | bob | who did it? 4 | alice | insane! -- Check Non-RLS JOIN with RLS. SELECT id, author, message FROM comment JOIN blog ON id = blog_id; id | author | message ----+--------+------------- 2 | bob | who did it? 4 | alice | insane! SET SESSION AUTHORIZATION regress_rls_alice; CREATE POLICY comment_1 ON comment USING (blog_id < 4); ALTER TABLE comment ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; -- Check RLS JOIN RLS SELECT id, author, message FROM blog JOIN comment ON id = blog_id; id | author | message ----+--------+------------- 2 | bob | who did it? SELECT id, author, message FROM comment JOIN blog ON id = blog_id; id | author | message ----+--------+------------- 2 | bob | who did it? SET SESSION AUTHORIZATION regress_rls_alice; DROP TABLE blog; DROP TABLE comment; -- -- Default Deny Policy -- RESET SESSION AUTHORIZATION; DROP POLICY p2 ON t1; ALTER TABLE t1 OWNER TO regress_rls_alice; -- Check that default deny does not apply to superuser. RESET SESSION AUTHORIZATION; SELECT * FROM t1; a | b ----+---------------------------------- 1 | c4ca4238a0b923820dcc509a6f75849b 0 | cfcd208495d565ef66e7dff9f98764da 3 | eccbc87e4b5ce2fe28308fd9f2a7baf3 2 | c81e728d9d4c2f636f067f89cc14862c 5 | e4da3b7fbbce2345d7772b0674a318d5 4 | a87ff679a2f3e71d9181a67b7542122c 7 | 8f14e45fceea167a5a36dedd4bea2543 6 | 1679091c5a880faf6fb5e6087eb1b2dc 9 | 45c48cce2e2d7fbdea1afc51c7c6ad26 8 | c9f0f895fb98ab9159f51fd0297e236d 11 | 6512bd43d9caa6e02c990b0a82652dca 10 | d3d9446802a44259755d38e6d163e820 13 | c51ce410c124a10e0db5e4b97fc2af39 12 | c20ad4d76fe97759aa27a0c99bff6710 15 | 9bf31c7ff062936a96d3c8bd1f8f2ff3 14 | aab3238922bcc25a6f606eb525ffdc56 17 | 70efdf2ec9b086079795c442636b55fb 16 | c74d97b01eae257e44aa9d5bade97baf 19 | 1f0e3dad99908345f7439f8ffabdffc4 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 20 | Success EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 -> Seq Scan on _hyper_15_69_chunk t1_2 -> Seq Scan on _hyper_15_70_chunk t1_3 -> Seq Scan on _hyper_15_71_chunk t1_4 -> Seq Scan on _hyper_15_72_chunk t1_5 -> Seq Scan on _hyper_15_73_chunk t1_6 -> Seq Scan on _hyper_15_74_chunk t1_7 -> Seq Scan on _hyper_15_75_chunk t1_8 -> Seq Scan on _hyper_15_76_chunk t1_9 -> Seq Scan on _hyper_15_77_chunk t1_10 -> Seq Scan on _hyper_15_78_chunk t1_11 -> Seq Scan on _hyper_15_79_chunk t1_12 -- Check that default deny does not apply to table owner. SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM t1; a | b ----+---------------------------------- 1 | c4ca4238a0b923820dcc509a6f75849b 0 | cfcd208495d565ef66e7dff9f98764da 3 | eccbc87e4b5ce2fe28308fd9f2a7baf3 2 | c81e728d9d4c2f636f067f89cc14862c 5 | e4da3b7fbbce2345d7772b0674a318d5 4 | a87ff679a2f3e71d9181a67b7542122c 7 | 8f14e45fceea167a5a36dedd4bea2543 6 | 1679091c5a880faf6fb5e6087eb1b2dc 9 | 45c48cce2e2d7fbdea1afc51c7c6ad26 8 | c9f0f895fb98ab9159f51fd0297e236d 11 | 6512bd43d9caa6e02c990b0a82652dca 10 | d3d9446802a44259755d38e6d163e820 13 | c51ce410c124a10e0db5e4b97fc2af39 12 | c20ad4d76fe97759aa27a0c99bff6710 15 | 9bf31c7ff062936a96d3c8bd1f8f2ff3 14 | aab3238922bcc25a6f606eb525ffdc56 17 | 70efdf2ec9b086079795c442636b55fb 16 | c74d97b01eae257e44aa9d5bade97baf 19 | 1f0e3dad99908345f7439f8ffabdffc4 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 20 | Success EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 -> Seq Scan on _hyper_15_69_chunk t1_2 -> Seq Scan on _hyper_15_70_chunk t1_3 -> Seq Scan on _hyper_15_71_chunk t1_4 -> Seq Scan on _hyper_15_72_chunk t1_5 -> Seq Scan on _hyper_15_73_chunk t1_6 -> Seq Scan on _hyper_15_74_chunk t1_7 -> Seq Scan on _hyper_15_75_chunk t1_8 -> Seq Scan on _hyper_15_76_chunk t1_9 -> Seq Scan on _hyper_15_77_chunk t1_10 -> Seq Scan on _hyper_15_78_chunk t1_11 -> Seq Scan on _hyper_15_79_chunk t1_12 -- Check that default deny applies to non-owner/non-superuser when RLS on. SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; SELECT * FROM t1; a | b ---+--- EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; --- QUERY PLAN --- Result One-Time Filter: false SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM t1; a | b ---+--- EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; --- QUERY PLAN --- Result One-Time Filter: false -- -- COPY TO/FROM -- RESET SESSION AUTHORIZATION; DROP TABLE copy_t CASCADE; ERROR: table "copy_t" does not exist CREATE TABLE copy_t (a integer, b text); SELECT public.create_hypertable('copy_t', 'a', chunk_time_interval=>2); create_hypertable ---------------------------------- (19,regress_rls_schema,copy_t,t) CREATE POLICY p1 ON copy_t USING (a % 2 = 0); ALTER TABLE copy_t ENABLE ROW LEVEL SECURITY; GRANT ALL ON copy_t TO regress_rls_bob, regress_rls_exempt_user; INSERT INTO copy_t (SELECT x, md5(x::text) FROM generate_series(0,10) x); -- Check COPY TO as Superuser/owner. RESET SESSION AUTHORIZATION; SET row_security TO OFF; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; 0,cfcd208495d565ef66e7dff9f98764da 1,c4ca4238a0b923820dcc509a6f75849b 2,c81e728d9d4c2f636f067f89cc14862c 3,eccbc87e4b5ce2fe28308fd9f2a7baf3 4,a87ff679a2f3e71d9181a67b7542122c 5,e4da3b7fbbce2345d7772b0674a318d5 6,1679091c5a880faf6fb5e6087eb1b2dc 7,8f14e45fceea167a5a36dedd4bea2543 8,c9f0f895fb98ab9159f51fd0297e236d 9,45c48cce2e2d7fbdea1afc51c7c6ad26 10,d3d9446802a44259755d38e6d163e820 SET row_security TO ON; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; 0,cfcd208495d565ef66e7dff9f98764da 1,c4ca4238a0b923820dcc509a6f75849b 2,c81e728d9d4c2f636f067f89cc14862c 3,eccbc87e4b5ce2fe28308fd9f2a7baf3 4,a87ff679a2f3e71d9181a67b7542122c 5,e4da3b7fbbce2345d7772b0674a318d5 6,1679091c5a880faf6fb5e6087eb1b2dc 7,8f14e45fceea167a5a36dedd4bea2543 8,c9f0f895fb98ab9159f51fd0297e236d 9,45c48cce2e2d7fbdea1afc51c7c6ad26 10,d3d9446802a44259755d38e6d163e820 -- Check COPY TO as user with permissions. SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO OFF; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --fail - would be affected by RLS ERROR: query would be affected by row-level security policy for table "copy_t" SET row_security TO ON; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --ok 0,cfcd208495d565ef66e7dff9f98764da 2,c81e728d9d4c2f636f067f89cc14862c 4,a87ff679a2f3e71d9181a67b7542122c 6,1679091c5a880faf6fb5e6087eb1b2dc 8,c9f0f895fb98ab9159f51fd0297e236d 10,d3d9446802a44259755d38e6d163e820 -- Check COPY TO as user with permissions and BYPASSRLS SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --ok 0,cfcd208495d565ef66e7dff9f98764da 1,c4ca4238a0b923820dcc509a6f75849b 2,c81e728d9d4c2f636f067f89cc14862c 3,eccbc87e4b5ce2fe28308fd9f2a7baf3 4,a87ff679a2f3e71d9181a67b7542122c 5,e4da3b7fbbce2345d7772b0674a318d5 6,1679091c5a880faf6fb5e6087eb1b2dc 7,8f14e45fceea167a5a36dedd4bea2543 8,c9f0f895fb98ab9159f51fd0297e236d 9,45c48cce2e2d7fbdea1afc51c7c6ad26 10,d3d9446802a44259755d38e6d163e820 SET row_security TO ON; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --ok 0,cfcd208495d565ef66e7dff9f98764da 1,c4ca4238a0b923820dcc509a6f75849b 2,c81e728d9d4c2f636f067f89cc14862c 3,eccbc87e4b5ce2fe28308fd9f2a7baf3 4,a87ff679a2f3e71d9181a67b7542122c 5,e4da3b7fbbce2345d7772b0674a318d5 6,1679091c5a880faf6fb5e6087eb1b2dc 7,8f14e45fceea167a5a36dedd4bea2543 8,c9f0f895fb98ab9159f51fd0297e236d 9,45c48cce2e2d7fbdea1afc51c7c6ad26 10,d3d9446802a44259755d38e6d163e820 -- Check COPY TO as user without permissions. SET row_security TO OFF; SET SESSION AUTHORIZATION regress_rls_carol; SET row_security TO OFF; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --fail - would be affected by RLS ERROR: query would be affected by row-level security policy for table "copy_t" SET row_security TO ON; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --fail - permission denied ERROR: permission denied for table copy_t -- Check COPY relation TO; keep it just one row to avoid reordering issues RESET SESSION AUTHORIZATION; SET row_security TO ON; CREATE TABLE copy_rel_to (a integer, b text); SELECT public.create_hypertable('copy_rel_to', 'a', chunk_time_interval=>2); create_hypertable --------------------------------------- (20,regress_rls_schema,copy_rel_to,t) CREATE POLICY p1 ON copy_rel_to USING (a % 2 = 0); ALTER TABLE copy_rel_to ENABLE ROW LEVEL SECURITY; GRANT ALL ON copy_rel_to TO regress_rls_bob, regress_rls_exempt_user; INSERT INTO copy_rel_to VALUES (1, md5('1')); -- Check COPY TO as Superuser/owner. RESET SESSION AUTHORIZATION; SET row_security TO OFF; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; 1,c4ca4238a0b923820dcc509a6f75849b SET row_security TO ON; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; 1,c4ca4238a0b923820dcc509a6f75849b -- Check COPY TO as user with permissions. SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO OFF; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --fail - would be affected by RLS ERROR: query would be affected by row-level security policy for table "copy_rel_to" SET row_security TO ON; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --ok -- Check COPY TO as user with permissions and BYPASSRLS SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --ok 1,c4ca4238a0b923820dcc509a6f75849b SET row_security TO ON; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --ok 1,c4ca4238a0b923820dcc509a6f75849b -- Check COPY TO as user without permissions. SET row_security TO OFF; SET SESSION AUTHORIZATION regress_rls_carol; SET row_security TO OFF; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --fail - permission denied ERROR: query would be affected by row-level security policy for table "copy_rel_to" SET row_security TO ON; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --fail - permission denied ERROR: permission denied for table copy_rel_to -- Check COPY FROM as Superuser/owner. RESET SESSION AUTHORIZATION; SET row_security TO OFF; COPY copy_t FROM STDIN; --ok SET row_security TO ON; COPY copy_t FROM STDIN; --ok -- Check COPY FROM as user with permissions. SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO OFF; COPY copy_t FROM STDIN; --fail - would be affected by RLS. ERROR: query would be affected by row-level security policy for table "copy_t" SET row_security TO ON; COPY copy_t FROM STDIN; --fail - COPY FROM not supported by RLS. ERROR: COPY FROM not supported with row-level security HINT: Use INSERT statements instead. -- Check COPY FROM as user with permissions and BYPASSRLS SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO ON; COPY copy_t FROM STDIN; --ok -- Check COPY FROM as user without permissions. SET SESSION AUTHORIZATION regress_rls_carol; SET row_security TO OFF; COPY copy_t FROM STDIN; --fail - permission denied. ERROR: permission denied for table copy_t SET row_security TO ON; COPY copy_t FROM STDIN; --fail - permission denied. ERROR: permission denied for table copy_t RESET SESSION AUTHORIZATION; DROP TABLE copy_t; DROP TABLE copy_rel_to CASCADE; -- Check WHERE CURRENT OF SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE current_check (currentid int, payload text, rlsuser text); SELECT public.create_hypertable('current_check', 'currentid', chunk_time_interval=>10); create_hypertable ----------------------------------------- (21,regress_rls_schema,current_check,t) GRANT ALL ON current_check TO PUBLIC; INSERT INTO current_check VALUES (1, 'abc', 'regress_rls_bob'), (2, 'bcd', 'regress_rls_bob'), (3, 'cde', 'regress_rls_bob'), (4, 'def', 'regress_rls_bob'); CREATE POLICY p1 ON current_check FOR SELECT USING (currentid % 2 = 0); CREATE POLICY p2 ON current_check FOR DELETE USING (currentid = 4 AND rlsuser = current_user); CREATE POLICY p3 ON current_check FOR UPDATE USING (currentid = 4) WITH CHECK (rlsuser = current_user); ALTER TABLE current_check ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; -- Can SELECT even rows SELECT * FROM current_check; currentid | payload | rlsuser -----------+---------+----------------- 2 | bcd | regress_rls_bob 4 | def | regress_rls_bob -- Cannot UPDATE row 2 UPDATE current_check SET payload = payload || '_new' WHERE currentid = 2 RETURNING *; currentid | payload | rlsuser -----------+---------+--------- BEGIN; -- WHERE CURRENT OF does not work with custom scan nodes -- so we have to disable chunk append here SET timescaledb.enable_chunk_append TO false; DECLARE current_check_cursor SCROLL CURSOR FOR SELECT * FROM current_check; -- Returns rows that can be seen according to SELECT policy, like plain SELECT -- above (even rows) FETCH ABSOLUTE 1 FROM current_check_cursor; currentid | payload | rlsuser -----------+---------+----------------- 2 | bcd | regress_rls_bob -- Still cannot UPDATE row 2 through cursor UPDATE current_check SET payload = payload || '_new' WHERE CURRENT OF current_check_cursor RETURNING *; currentid | payload | rlsuser -----------+---------+--------- -- Can update row 4 through cursor, which is the next visible row FETCH RELATIVE 1 FROM current_check_cursor; currentid | payload | rlsuser -----------+---------+----------------- 4 | def | regress_rls_bob UPDATE current_check SET payload = payload || '_new' WHERE CURRENT OF current_check_cursor RETURNING *; currentid | payload | rlsuser -----------+---------+----------------- 4 | def_new | regress_rls_bob SELECT * FROM current_check; currentid | payload | rlsuser -----------+---------+----------------- 2 | bcd | regress_rls_bob 4 | def_new | regress_rls_bob -- Plan should be a subquery TID scan EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE current_check SET payload = payload WHERE CURRENT OF current_check_cursor; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Update on current_check Update on _hyper_21_104_chunk current_check_1 -> Tid Scan on _hyper_21_104_chunk current_check_1 TID Cond: CURRENT OF current_check_cursor Filter: ((currentid = 4) AND ((currentid % 2) = 0)) -- Similarly can only delete row 4 FETCH ABSOLUTE 1 FROM current_check_cursor; currentid | payload | rlsuser -----------+---------+----------------- 2 | bcd | regress_rls_bob DELETE FROM current_check WHERE CURRENT OF current_check_cursor RETURNING *; currentid | payload | rlsuser -----------+---------+--------- FETCH RELATIVE 1 FROM current_check_cursor; currentid | payload | rlsuser -----------+---------+----------------- 4 | def | regress_rls_bob DELETE FROM current_check WHERE CURRENT OF current_check_cursor RETURNING *; currentid | payload | rlsuser -----------+---------+----------------- 4 | def_new | regress_rls_bob SELECT * FROM current_check; currentid | payload | rlsuser -----------+---------+----------------- 2 | bcd | regress_rls_bob RESET timescaledb.enable_chunk_append; COMMIT; -- -- check pg_stats view filtering -- SET row_security TO ON; SET SESSION AUTHORIZATION regress_rls_alice; ANALYZE current_check; -- Stats visible SELECT row_security_active('current_check'); row_security_active --------------------- f SELECT attname, most_common_vals FROM pg_stats WHERE tablename = 'current_check' ORDER BY 1; attname | most_common_vals -----------+------------------- currentid | payload | rlsuser | {regress_rls_bob} SET SESSION AUTHORIZATION regress_rls_bob; -- Stats not visible SELECT row_security_active('current_check'); row_security_active --------------------- t SELECT attname, most_common_vals FROM pg_stats WHERE tablename = 'current_check' ORDER BY 1; attname | most_common_vals ---------+------------------ -- -- Collation support -- BEGIN; CREATE TABLE coll_t (c) AS VALUES ('bar'::text); CREATE POLICY coll_p ON coll_t USING (c < ('foo'::text COLLATE "C")); ALTER TABLE coll_t ENABLE ROW LEVEL SECURITY; GRANT SELECT ON coll_t TO regress_rls_alice; SELECT (string_to_array(polqual, ':'))[7] AS inputcollid FROM pg_policy WHERE polrelid = 'coll_t'::regclass; inputcollid ------------------ inputcollid 950 SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM coll_t; c ----- bar ROLLBACK; -- -- Shared Object Dependencies -- RESET SESSION AUTHORIZATION; BEGIN; CREATE ROLE regress_rls_eve; CREATE ROLE regress_rls_frank; CREATE TABLE tbl1 (c) AS VALUES ('bar'::text); GRANT SELECT ON TABLE tbl1 TO regress_rls_eve; CREATE POLICY P ON tbl1 TO regress_rls_eve, regress_rls_frank USING (true); SELECT refclassid::regclass, deptype FROM pg_depend WHERE classid = 'pg_policy'::regclass AND refobjid = 'tbl1'::regclass; refclassid | deptype ------------+--------- pg_class | a SELECT refclassid::regclass, deptype FROM pg_shdepend WHERE classid = 'pg_policy'::regclass AND refobjid IN ('regress_rls_eve'::regrole, 'regress_rls_frank'::regrole); refclassid | deptype ------------+--------- pg_authid | r pg_authid | r SAVEPOINT q; DROP ROLE regress_rls_eve; --fails due to dependency on POLICY p ERROR: role "regress_rls_eve" cannot be dropped because some objects depend on it DETAIL: privileges for table tbl1 target of policy p on table tbl1 ROLLBACK TO q; ALTER POLICY p ON tbl1 TO regress_rls_frank USING (true); SAVEPOINT q; DROP ROLE regress_rls_eve; --fails due to dependency on GRANT SELECT ERROR: role "regress_rls_eve" cannot be dropped because some objects depend on it DETAIL: privileges for table tbl1 ROLLBACK TO q; REVOKE ALL ON TABLE tbl1 FROM regress_rls_eve; SAVEPOINT q; DROP ROLE regress_rls_eve; --succeeds ROLLBACK TO q; SAVEPOINT q; DROP ROLE regress_rls_frank; --fails due to dependency on POLICY p ERROR: role "regress_rls_frank" cannot be dropped because some objects depend on it DETAIL: target of policy p on table tbl1 ROLLBACK TO q; DROP POLICY p ON tbl1; SAVEPOINT q; DROP ROLE regress_rls_frank; -- succeeds ROLLBACK TO q; ROLLBACK; -- cleanup -- -- Converting table to view -- BEGIN; CREATE TABLE t (c int); SELECT public.create_hypertable('t', 'c', chunk_time_interval=>2); create_hypertable ----------------------------- (22,regress_rls_schema,t,t) CREATE POLICY p ON t USING (c % 2 = 1); ALTER TABLE t ENABLE ROW LEVEL SECURITY; SAVEPOINT q; CREATE RULE "_RETURN" AS ON SELECT TO t DO INSTEAD SELECT * FROM generate_series(1,5) t0(c); -- fails due to row level security enabled ERROR: hypertables do not support rules ROLLBACK TO q; ALTER TABLE t DISABLE ROW LEVEL SECURITY; SAVEPOINT q; CREATE RULE "_RETURN" AS ON SELECT TO t DO INSTEAD SELECT * FROM generate_series(1,5) t0(c); -- fails due to policy p on t ERROR: hypertables do not support rules ROLLBACK TO q; DROP POLICY p ON t; CREATE RULE "_RETURN" AS ON SELECT TO t DO INSTEAD SELECT * FROM generate_series(1,5) t0(c); -- succeeds ERROR: hypertables do not support rules ROLLBACK; -- -- Policy expression handling -- BEGIN; CREATE TABLE t (c) AS VALUES ('bar'::text); CREATE POLICY p ON t USING (max(c)); -- fails: aggregate functions are not allowed in policy expressions ERROR: aggregate functions are not allowed in policy expressions ROLLBACK; -- -- Non-target relations are only subject to SELECT policies -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE r1 (a int); SELECT public.create_hypertable('r1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (23,regress_rls_schema,r1,t) CREATE TABLE r2 (a int); SELECT public.create_hypertable('r2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (24,regress_rls_schema,r2,t) INSERT INTO r1 VALUES (10), (20); INSERT INTO r2 VALUES (10), (20); GRANT ALL ON r1, r2 TO regress_rls_bob; CREATE POLICY p1 ON r1 USING (true); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; CREATE POLICY p1 ON r2 FOR SELECT USING (true); CREATE POLICY p2 ON r2 FOR INSERT WITH CHECK (false); CREATE POLICY p3 ON r2 FOR UPDATE USING (false); CREATE POLICY p4 ON r2 FOR DELETE USING (false); ALTER TABLE r2 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM r1; a ---- 10 20 SELECT * FROM r2; a ---- 10 20 -- r2 is read-only INSERT INTO r2 VALUES (2); -- Not allowed ERROR: new row violates row-level security policy for table "r2" \pset tuples_only 1 UPDATE r2 SET a = 2 RETURNING *; -- Updates nothing DELETE FROM r2 RETURNING *; -- Deletes nothing \pset tuples_only 0 -- r2 can be used as a non-target relation in DML INSERT INTO r1 SELECT a + 1 FROM r2 RETURNING *; -- OK a ---- 11 21 UPDATE r1 SET a = r2.a + 2 FROM r2 WHERE r1.a = r2.a RETURNING *; -- OK ERROR: new row for relation "_hyper_23_105_chunk" violates check constraint "constraint_105" DELETE FROM r1 USING r2 WHERE r1.a = r2.a + 2 RETURNING *; -- OK a | a ---+--- SELECT * FROM r1; a ---- 10 11 20 21 SELECT * FROM r2; a ---- 10 20 SET SESSION AUTHORIZATION regress_rls_alice; DROP TABLE r1; DROP TABLE r2; -- -- FORCE ROW LEVEL SECURITY applies RLS to owners too -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security = on; CREATE TABLE r1 (a int); SELECT public.create_hypertable('r1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (25,regress_rls_schema,r1,t) INSERT INTO r1 VALUES (10), (20); CREATE POLICY p1 ON r1 USING (false); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- No error, but no rows TABLE r1; a --- -- RLS error INSERT INTO r1 VALUES (1); ERROR: new row violates row-level security policy for table "r1" -- No error (unable to see any rows to update) UPDATE r1 SET a = 1; TABLE r1; a --- -- No error (unable to see any rows to delete) DELETE FROM r1; TABLE r1; a --- SET row_security = off; -- these all fail, would be affected by RLS TABLE r1; ERROR: query would be affected by row-level security policy for table "r1" HINT: To disable the policy for the table's owner, use ALTER TABLE NO FORCE ROW LEVEL SECURITY. UPDATE r1 SET a = 1; ERROR: query would be affected by row-level security policy for table "r1" HINT: To disable the policy for the table's owner, use ALTER TABLE NO FORCE ROW LEVEL SECURITY. DELETE FROM r1; ERROR: query would be affected by row-level security policy for table "r1" HINT: To disable the policy for the table's owner, use ALTER TABLE NO FORCE ROW LEVEL SECURITY. DROP TABLE r1; -- -- FORCE ROW LEVEL SECURITY does not break RI -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security = on; CREATE TABLE r1 (a int PRIMARY KEY); -- r1 is not a hypertable since r1.a is referenced by r2 CREATE TABLE r2 (a int REFERENCES r1); SELECT public.create_hypertable('r2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (26,regress_rls_schema,r2,t) INSERT INTO r1 VALUES (10), (20); INSERT INTO r2 VALUES (10), (20); -- Create policies on r2 which prevent the -- owner from seeing any rows, but RI should -- still see them. CREATE POLICY p1 ON r2 USING (false); ALTER TABLE r2 ENABLE ROW LEVEL SECURITY; ALTER TABLE r2 FORCE ROW LEVEL SECURITY; -- Errors due to rows in r2 DELETE FROM r1; ERROR: update or delete on table "r1" violates foreign key constraint "r2_a_fkey" on table "r2" DETAIL: Key (a)=(10) is still referenced from table "r2". -- Reset r2 to no-RLS DROP POLICY p1 ON r2; ALTER TABLE r2 NO FORCE ROW LEVEL SECURITY; ALTER TABLE r2 DISABLE ROW LEVEL SECURITY; -- clean out r2 for INSERT test below DELETE FROM r2; -- Change r1 to not allow rows to be seen CREATE POLICY p1 ON r1 USING (false); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- No rows seen TABLE r1; a --- -- No error, RI still sees that row exists in r1 INSERT INTO r2 VALUES (10); DROP TABLE r2; DROP TABLE r1; -- Ensure cascaded DELETE works CREATE TABLE r1 (a int PRIMARY KEY); -- r1 is not a hypertable since r1.a is referenced by r2 CREATE TABLE r2 (a int REFERENCES r1 ON DELETE CASCADE); SELECT public.create_hypertable('r2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (27,regress_rls_schema,r2,t) INSERT INTO r1 VALUES (10), (20); INSERT INTO r2 VALUES (10), (20); -- Create policies on r2 which prevent the -- owner from seeing any rows, but RI should -- still see them. CREATE POLICY p1 ON r2 USING (false); ALTER TABLE r2 ENABLE ROW LEVEL SECURITY; ALTER TABLE r2 FORCE ROW LEVEL SECURITY; -- Deletes all records from both DELETE FROM r1; -- Remove FORCE from r2 ALTER TABLE r2 NO FORCE ROW LEVEL SECURITY; -- As owner, we now bypass RLS -- verify no rows in r2 now TABLE r2; a --- DROP TABLE r2; DROP TABLE r1; -- Ensure cascaded UPDATE works CREATE TABLE r1 (a int PRIMARY KEY); -- r1 is not a hypertable since r1.a is referenced by r2 CREATE TABLE r2 (a int REFERENCES r1 ON UPDATE CASCADE); SELECT public.create_hypertable('r2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (28,regress_rls_schema,r2,t) INSERT INTO r1 VALUES (10), (20); INSERT INTO r2 VALUES (10), (20); -- Create policies on r2 which prevent the -- owner from seeing any rows, but RI should -- still see them. CREATE POLICY p1 ON r2 USING (false); ALTER TABLE r2 ENABLE ROW LEVEL SECURITY; ALTER TABLE r2 FORCE ROW LEVEL SECURITY; -- Updates records in both (terse output to not print CONTEXT, which can be different). \set VERBOSITY terse UPDATE r1 SET a = a+5; ERROR: new row for relation "_hyper_28_117_chunk" violates check constraint "constraint_117" \set VERBOSITY default -- Remove FORCE from r2 ALTER TABLE r2 NO FORCE ROW LEVEL SECURITY; -- As owner, we now bypass RLS -- verify records in r2 updated TABLE r2; a ---- 10 20 DROP TABLE r2; DROP TABLE r1; -- -- Test INSERT+RETURNING applies SELECT policies as -- WithCheckOptions (meaning an error is thrown) -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security = on; CREATE TABLE r1 (a int); SELECT public.create_hypertable('r1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (29,regress_rls_schema,r1,t) CREATE POLICY p1 ON r1 FOR SELECT USING (false); CREATE POLICY p2 ON r1 FOR INSERT WITH CHECK (true); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- Works fine INSERT INTO r1 VALUES (10), (20); -- No error, but no rows TABLE r1; a --- SET row_security = off; -- fail, would be affected by RLS TABLE r1; ERROR: query would be affected by row-level security policy for table "r1" HINT: To disable the policy for the table's owner, use ALTER TABLE NO FORCE ROW LEVEL SECURITY. SET row_security = on; -- Error INSERT INTO r1 VALUES (10), (20) RETURNING *; ERROR: new row violates row-level security policy for table "r1" DROP TABLE r1; -- -- Test UPDATE+RETURNING applies SELECT policies as -- WithCheckOptions (meaning an error is thrown) -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security = on; CREATE TABLE r1 (a int PRIMARY KEY); SELECT public.create_hypertable('r1', 'a', chunk_time_interval=>100); create_hypertable ------------------------------ (30,regress_rls_schema,r1,t) CREATE POLICY p1 ON r1 FOR SELECT USING (a < 20); CREATE POLICY p2 ON r1 FOR UPDATE USING (a < 20) WITH CHECK (true); CREATE POLICY p3 ON r1 FOR INSERT WITH CHECK (true); INSERT INTO r1 VALUES (10); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- Works fine UPDATE r1 SET a = 30; -- Show updated rows ALTER TABLE r1 NO FORCE ROW LEVEL SECURITY; TABLE r1; a ---- 30 -- reset value in r1 for test with RETURNING UPDATE r1 SET a = 10; -- Verify row reset TABLE r1; a ---- 10 ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- Error UPDATE r1 SET a = 30 RETURNING *; ERROR: new row violates row-level security policy for table "r1" -- UPDATE path of INSERT ... ON CONFLICT DO UPDATE should also error out INSERT INTO r1 VALUES (10) ON CONFLICT (a) DO UPDATE SET a = 30 RETURNING *; ERROR: new row violates row-level security policy for table "r1" -- Should still error out without RETURNING (use of arbiter always requires -- SELECT permissions) INSERT INTO r1 VALUES (10) ON CONFLICT (a) DO UPDATE SET a = 30; ERROR: new row violates row-level security policy for table "r1" -- ON CONFLICT ON CONSTRAINT INSERT INTO r1 VALUES (10) ON CONFLICT ON CONSTRAINT r1_pkey DO UPDATE SET a = 30; ERROR: new row violates row-level security policy for table "r1" DROP TABLE r1; -- Check dependency handling RESET SESSION AUTHORIZATION; CREATE TABLE dep1 (c1 int); SELECT public.create_hypertable('dep1', 'c1', chunk_time_interval=>2); create_hypertable -------------------------------- (31,regress_rls_schema,dep1,t) CREATE TABLE dep2 (c1 int); SELECT public.create_hypertable('dep2', 'c1', chunk_time_interval=>2); create_hypertable -------------------------------- (32,regress_rls_schema,dep2,t) CREATE POLICY dep_p1 ON dep1 TO regress_rls_bob USING (c1 > (select max(dep2.c1) from dep2)); ALTER POLICY dep_p1 ON dep1 TO regress_rls_bob,regress_rls_carol; -- Should return one SELECT count(*) = 1 FROM pg_depend WHERE objid = (SELECT oid FROM pg_policy WHERE polname = 'dep_p1') AND refobjid = (SELECT oid FROM pg_class WHERE relname = 'dep2'); ?column? ---------- t ALTER POLICY dep_p1 ON dep1 USING (true); -- Should return one SELECT count(*) = 1 FROM pg_shdepend WHERE objid = (SELECT oid FROM pg_policy WHERE polname = 'dep_p1') AND refobjid = (SELECT oid FROM pg_authid WHERE rolname = 'regress_rls_bob'); ?column? ---------- t -- Should return one SELECT count(*) = 1 FROM pg_shdepend WHERE objid = (SELECT oid FROM pg_policy WHERE polname = 'dep_p1') AND refobjid = (SELECT oid FROM pg_authid WHERE rolname = 'regress_rls_carol'); ?column? ---------- t -- Should return zero SELECT count(*) = 0 FROM pg_depend WHERE objid = (SELECT oid FROM pg_policy WHERE polname = 'dep_p1') AND refobjid = (SELECT oid FROM pg_class WHERE relname = 'dep2'); ?column? ---------- t -- DROP OWNED BY testing RESET SESSION AUTHORIZATION; CREATE ROLE regress_rls_dob_role1; CREATE ROLE regress_rls_dob_role2; CREATE TABLE dob_t1 (c1 int); SELECT public.create_hypertable('dob_t1', 'c1', chunk_time_interval=>2); create_hypertable ---------------------------------- (33,regress_rls_schema,dob_t1,t) CREATE TABLE dob_t2 (c1 int) PARTITION BY RANGE (c1); CREATE POLICY p1 ON dob_t1 TO regress_rls_dob_role1 USING (true); DROP OWNED BY regress_rls_dob_role1; DROP POLICY p1 ON dob_t1; -- should fail, already gone ERROR: policy "p1" for table "dob_t1" does not exist CREATE POLICY p1 ON dob_t1 TO regress_rls_dob_role1,regress_rls_dob_role2 USING (true); DROP OWNED BY regress_rls_dob_role1; DROP POLICY p1 ON dob_t1; -- should succeed CREATE POLICY p1 ON dob_t2 TO regress_rls_dob_role1,regress_rls_dob_role2 USING (true); DROP OWNED BY regress_rls_dob_role1; DROP POLICY p1 ON dob_t2; -- should succeed DROP USER regress_rls_dob_role1; DROP USER regress_rls_dob_role2; -- -- Clean up objects -- RESET SESSION AUTHORIZATION; \set VERBOSITY terse \\ -- suppress cascade details DROP SCHEMA regress_rls_schema CASCADE; NOTICE: drop cascades to 116 other objects \set VERBOSITY default DROP USER regress_rls_alice; DROP USER regress_rls_bob; DROP USER regress_rls_carol; DROP USER regress_rls_dave; DROP USER regress_rls_exempt_user; DROP ROLE regress_rls_group1; DROP ROLE regress_rls_group2; -- Arrange to have a few policies left over, for testing -- pg_dump/pg_restore CREATE SCHEMA regress_rls_schema; CREATE TABLE rls_tbl (c1 int); SELECT public.create_hypertable('rls_tbl', 'c1', chunk_time_interval=>2); create_hypertable ----------------------------------- (34,regress_rls_schema,rls_tbl,t) ALTER TABLE rls_tbl ENABLE ROW LEVEL SECURITY; CREATE POLICY p1 ON rls_tbl USING (c1 > 5); CREATE POLICY p2 ON rls_tbl FOR SELECT USING (c1 <= 3); CREATE POLICY p3 ON rls_tbl FOR UPDATE USING (c1 <= 3) WITH CHECK (c1 > 5); CREATE POLICY p4 ON rls_tbl FOR DELETE USING (c1 <= 3); CREATE TABLE rls_tbl_force (c1 int); SELECT public.create_hypertable('rls_tbl_force', 'c1', chunk_time_interval=>2); create_hypertable ----------------------------------------- (35,regress_rls_schema,rls_tbl_force,t) ALTER TABLE rls_tbl_force ENABLE ROW LEVEL SECURITY; ALTER TABLE rls_tbl_force FORCE ROW LEVEL SECURITY; CREATE POLICY p1 ON rls_tbl_force USING (c1 = 5) WITH CHECK (c1 < 5); CREATE POLICY p2 ON rls_tbl_force FOR SELECT USING (c1 = 8); CREATE POLICY p3 ON rls_tbl_force FOR UPDATE USING (c1 = 8) WITH CHECK (c1 >= 5); CREATE POLICY p4 ON rls_tbl_force FOR DELETE USING (c1 = 8); ================================================ FILE: test/expected/rowsecurity-18.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- -- Test of Row-level security feature -- -- Clean up in case a prior regression run failed \c :TEST_DBNAME :ROLE_SUPERUSER \set ON_ERROR_STOP 0 \set VERBOSITY default SET timescaledb.enable_constraint_exclusion TO off; -- Suppress NOTICE messages when users/groups don't exist SET client_min_messages TO 'warning'; DROP USER IF EXISTS regress_rls_alice; DROP USER IF EXISTS regress_rls_bob; DROP USER IF EXISTS regress_rls_carol; DROP USER IF EXISTS regress_rls_dave; DROP USER IF EXISTS regress_rls_exempt_user; DROP ROLE IF EXISTS regress_rls_group1; DROP ROLE IF EXISTS regress_rls_group2; DROP SCHEMA IF EXISTS regress_rls_schema CASCADE; RESET client_min_messages; -- initial setup CREATE USER regress_rls_alice NOLOGIN; CREATE USER regress_rls_bob NOLOGIN; CREATE USER regress_rls_carol NOLOGIN; CREATE USER regress_rls_dave NOLOGIN; CREATE USER regress_rls_exempt_user BYPASSRLS NOLOGIN; CREATE ROLE regress_rls_group1 NOLOGIN; CREATE ROLE regress_rls_group2 NOLOGIN; GRANT regress_rls_group1 TO regress_rls_bob; GRANT regress_rls_group2 TO regress_rls_carol; CREATE SCHEMA regress_rls_schema; GRANT ALL ON SCHEMA regress_rls_schema to public; SET search_path = regress_rls_schema; -- setup of malicious function CREATE OR REPLACE FUNCTION f_leak(text) RETURNS bool COST 0.0000001 LANGUAGE plpgsql AS 'BEGIN RAISE NOTICE ''f_leak => %'', $1; RETURN true; END'; GRANT EXECUTE ON FUNCTION f_leak(text) TO public; -- BASIC Row-Level Security Scenario SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE uaccount ( pguser name primary key, seclv int ); GRANT SELECT ON uaccount TO public; INSERT INTO uaccount VALUES ('regress_rls_alice', 99), ('regress_rls_bob', 1), ('regress_rls_carol', 2), ('regress_rls_dave', 3); CREATE TABLE category ( cid int primary key, cname text ); GRANT ALL ON category TO public; INSERT INTO category VALUES (11, 'novel'), (22, 'science fiction'), (33, 'technology'), (44, 'manga'); CREATE TABLE document ( did int primary key, cid int references category(cid), dlevel int not null, dauthor name, dtitle text ); GRANT ALL ON document TO public; SELECT public.create_hypertable('document', 'did', chunk_time_interval=>2); create_hypertable ----------------------------------- (1,regress_rls_schema,document,t) INSERT INTO document VALUES ( 1, 11, 1, 'regress_rls_bob', 'my first novel'), ( 2, 11, 2, 'regress_rls_bob', 'my second novel'), ( 3, 22, 2, 'regress_rls_bob', 'my science fiction'), ( 4, 44, 1, 'regress_rls_bob', 'my first manga'), ( 5, 44, 2, 'regress_rls_bob', 'my second manga'), ( 6, 22, 1, 'regress_rls_carol', 'great science fiction'), ( 7, 33, 2, 'regress_rls_carol', 'great technology book'), ( 8, 44, 1, 'regress_rls_carol', 'great manga'), ( 9, 22, 1, 'regress_rls_dave', 'awesome science fiction'), (10, 33, 2, 'regress_rls_dave', 'awesome technology book'); ALTER TABLE document ENABLE ROW LEVEL SECURITY; -- user's security level must be higher than or equal to document's CREATE POLICY p1 ON document AS PERMISSIVE USING (dlevel <= (SELECT seclv FROM uaccount WHERE pguser = current_user)); -- try to create a policy of bogus type CREATE POLICY p1 ON document AS UGLY USING (dlevel <= (SELECT seclv FROM uaccount WHERE pguser = current_user)); ERROR: unrecognized row security option "ugly" LINE 1: CREATE POLICY p1 ON document AS UGLY ^ HINT: Only PERMISSIVE or RESTRICTIVE policies are supported currently. -- but Dave isn't allowed to anything at cid 50 or above -- this is to make sure that we sort the policies by name first -- when applying WITH CHECK, a later INSERT by Dave should fail due -- to p1r first CREATE POLICY p2r ON document AS RESTRICTIVE TO regress_rls_dave USING (cid <> 44 AND cid < 50); -- and Dave isn't allowed to see manga documents CREATE POLICY p1r ON document AS RESTRICTIVE TO regress_rls_dave USING (cid <> 44); \dp Access privileges Schema | Name | Type | Access privileges | Column privileges | Policies --------------------+----------+-------+----------------------------------------------+-------------------+-------------------------------------------- regress_rls_schema | category | table | regress_rls_alice=arwdDxtm/regress_rls_alice+| | | | | =arwdDxtm/regress_rls_alice | | regress_rls_schema | document | table | regress_rls_alice=arwdDxtm/regress_rls_alice+| | p1: + | | | =arwdDxtm/regress_rls_alice | | (u): (dlevel <= ( SELECT uaccount.seclv + | | | | | FROM uaccount + | | | | | WHERE (uaccount.pguser = CURRENT_USER)))+ | | | | | p2r (RESTRICTIVE): + | | | | | (u): ((cid <> 44) AND (cid < 50)) + | | | | | to: regress_rls_dave + | | | | | p1r (RESTRICTIVE): + | | | | | (u): (cid <> 44) + | | | | | to: regress_rls_dave regress_rls_schema | uaccount | table | regress_rls_alice=arwdDxtm/regress_rls_alice+| | | | | =r/regress_rls_alice | | \d document Table "regress_rls_schema.document" Column | Type | Collation | Nullable | Default ---------+---------+-----------+----------+--------- did | integer | | not null | cid | integer | | | dlevel | integer | | not null | dauthor | name | | | dtitle | text | | | Indexes: "document_pkey" PRIMARY KEY, btree (did) Foreign-key constraints: "document_cid_fkey" FOREIGN KEY (cid) REFERENCES category(cid) Policies: POLICY "p1" USING ((dlevel <= ( SELECT uaccount.seclv FROM uaccount WHERE (uaccount.pguser = CURRENT_USER)))) POLICY "p1r" AS RESTRICTIVE TO regress_rls_dave USING ((cid <> 44)) POLICY "p2r" AS RESTRICTIVE TO regress_rls_dave USING (((cid <> 44) AND (cid < 50))) Number of child tables: 6 (Use \d+ to list them.) SELECT * FROM pg_policies WHERE schemaname = 'regress_rls_schema' AND tablename = 'document' ORDER BY policyname; schemaname | tablename | policyname | permissive | roles | cmd | qual | with_check --------------------+-----------+------------+-------------+--------------------+-----+--------------------------------------------+------------ regress_rls_schema | document | p1 | PERMISSIVE | {public} | ALL | (dlevel <= ( SELECT uaccount.seclv +| | | | | | | FROM uaccount +| | | | | | | WHERE (uaccount.pguser = CURRENT_USER))) | regress_rls_schema | document | p1r | RESTRICTIVE | {regress_rls_dave} | ALL | (cid <> 44) | regress_rls_schema | document | p2r | RESTRICTIVE | {regress_rls_dave} | ALL | ((cid <> 44) AND (cid < 50)) | -- viewpoint from regress_rls_bob SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my first manga NOTICE: f_leak => great science fiction NOTICE: f_leak => great manga NOTICE: f_leak => awesome science fiction did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 4 | 44 | 1 | regress_rls_bob | my first manga 6 | 22 | 1 | regress_rls_carol | great science fiction 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my first manga NOTICE: f_leak => great science fiction NOTICE: f_leak => great manga NOTICE: f_leak => awesome science fiction cid | did | dlevel | dauthor | dtitle | cname -----+-----+--------+-------------------+-------------------------+----------------- 11 | 1 | 1 | regress_rls_bob | my first novel | novel 44 | 4 | 1 | regress_rls_bob | my first manga | manga 22 | 6 | 1 | regress_rls_carol | great science fiction | science fiction 44 | 8 | 1 | regress_rls_carol | great manga | manga 22 | 9 | 1 | regress_rls_dave | awesome science fiction | science fiction -- try a sampled version SELECT * FROM document TABLESAMPLE BERNOULLI(50) REPEATABLE(0) WHERE f_leak(dtitle) ORDER BY did; did | cid | dlevel | dauthor | dtitle -----+-----+--------+---------+-------- -- viewpoint from regress_rls_carol SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => my first manga NOTICE: f_leak => my second manga NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great manga NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => my first manga NOTICE: f_leak => my second manga NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great manga NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book cid | did | dlevel | dauthor | dtitle | cname -----+-----+--------+-------------------+-------------------------+----------------- 11 | 1 | 1 | regress_rls_bob | my first novel | novel 11 | 2 | 2 | regress_rls_bob | my second novel | novel 22 | 3 | 2 | regress_rls_bob | my science fiction | science fiction 44 | 4 | 1 | regress_rls_bob | my first manga | manga 44 | 5 | 2 | regress_rls_bob | my second manga | manga 22 | 6 | 1 | regress_rls_carol | great science fiction | science fiction 33 | 7 | 2 | regress_rls_carol | great technology book | technology 44 | 8 | 1 | regress_rls_carol | great manga | manga 22 | 9 | 1 | regress_rls_dave | awesome science fiction | science fiction 33 | 10 | 2 | regress_rls_dave | awesome technology book | technology -- try a sampled version SELECT * FROM document TABLESAMPLE BERNOULLI(50) REPEATABLE(0) WHERE f_leak(dtitle) ORDER BY did; did | cid | dlevel | dauthor | dtitle -----+-----+--------+---------+-------- EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 InitPlan 1 -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on document document_1 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle); --- QUERY PLAN --- Hash Join Hash Cond: (document.cid = category.cid) InitPlan 1 -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 -> Seq Scan on document document_1 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Hash -> Seq Scan on category -- viewpoint from regress_rls_dave SET SESSION AUTHORIZATION regress_rls_dave; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book cid | did | dlevel | dauthor | dtitle | cname -----+-----+--------+-------------------+-------------------------+----------------- 11 | 1 | 1 | regress_rls_bob | my first novel | novel 11 | 2 | 2 | regress_rls_bob | my second novel | novel 22 | 3 | 2 | regress_rls_bob | my science fiction | science fiction 22 | 6 | 1 | regress_rls_carol | great science fiction | science fiction 33 | 7 | 2 | regress_rls_carol | great technology book | technology 22 | 9 | 1 | regress_rls_dave | awesome science fiction | science fiction 33 | 10 | 2 | regress_rls_dave | awesome technology book | technology EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 InitPlan 1 -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on document document_1 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle); --- QUERY PLAN --- Nested Loop InitPlan 1 -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 -> Seq Scan on document document_1 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((cid <> 44) AND (cid <> 44) AND (cid < 50) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Index Scan using category_pkey on category Index Cond: (cid = document.cid) -- 44 would technically fail for both p2r and p1r, but we should get an error -- back from p1r for this because it sorts first INSERT INTO document VALUES (100, 44, 1, 'regress_rls_dave', 'testing sorting of policies'); -- fail ERROR: new row violates row-level security policy "p1r" for table "document" -- Just to see a p2r error INSERT INTO document VALUES (100, 55, 1, 'regress_rls_dave', 'testing sorting of policies'); -- fail ERROR: new row violates row-level security policy "p2r" for table "document" -- only owner can change policies ALTER POLICY p1 ON document USING (true); --fail ERROR: must be owner of table document DROP POLICY p1 ON document; --fail ERROR: must be owner of relation document SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY p1 ON document USING (dauthor = current_user); -- viewpoint from regress_rls_bob again SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => my first manga NOTICE: f_leak => my second manga did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+-------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER by did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science fiction NOTICE: f_leak => my first manga NOTICE: f_leak => my second manga cid | did | dlevel | dauthor | dtitle | cname -----+-----+--------+-----------------+--------------------+----------------- 11 | 1 | 1 | regress_rls_bob | my first novel | novel 11 | 2 | 2 | regress_rls_bob | my second novel | novel 22 | 3 | 2 | regress_rls_bob | my science fiction | science fiction 44 | 4 | 1 | regress_rls_bob | my first manga | manga 44 | 5 | 2 | regress_rls_bob | my second manga | manga -- viewpoint from rls_regres_carol again SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great manga did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+----------------------- 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER by did; NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great manga cid | did | dlevel | dauthor | dtitle | cname -----+-----+--------+-------------------+-----------------------+----------------- 22 | 6 | 1 | regress_rls_carol | great science fiction | science fiction 33 | 7 | 2 | regress_rls_carol | great technology book | technology 44 | 8 | 1 | regress_rls_carol | great manga | manga EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 -> Seq Scan on document document_1 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle); --- QUERY PLAN --- Nested Loop -> Custom Scan (ChunkAppend) on document Chunks excluded during startup: 0 -> Seq Scan on document document_1 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_1_chunk document_2 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_2_chunk document_3 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_3_chunk document_4 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_4_chunk document_5 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_5_chunk document_6 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_1_6_chunk document_7 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Index Scan using category_pkey on category Index Cond: (cid = document.cid) -- interaction of FK/PK constraints SET SESSION AUTHORIZATION regress_rls_alice; CREATE POLICY p2 ON category USING (CASE WHEN current_user = 'regress_rls_bob' THEN cid IN (11, 33) WHEN current_user = 'regress_rls_carol' THEN cid IN (22, 44) ELSE false END); ALTER TABLE category ENABLE ROW LEVEL SECURITY; -- cannot delete PK referenced by invisible FK SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM document d FULL OUTER JOIN category c on d.cid = c.cid ORDER BY d.did, c.cid; did | cid | dlevel | dauthor | dtitle | cid | cname -----+-----+--------+-----------------+--------------------+-----+------------ 1 | 11 | 1 | regress_rls_bob | my first novel | 11 | novel 2 | 11 | 2 | regress_rls_bob | my second novel | 11 | novel 3 | 22 | 2 | regress_rls_bob | my science fiction | | 4 | 44 | 1 | regress_rls_bob | my first manga | | 5 | 44 | 2 | regress_rls_bob | my second manga | | | | | | | 33 | technology \set VERBOSITY sqlstate DELETE FROM category WHERE cid = 33; -- fails with FK violation ERROR: 23503 \set VERBOSITY default -- can insert FK referencing invisible PK SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM document d FULL OUTER JOIN category c on d.cid = c.cid ORDER BY d.did, c.cid; did | cid | dlevel | dauthor | dtitle | cid | cname -----+-----+--------+-------------------+-----------------------+-----+----------------- 6 | 22 | 1 | regress_rls_carol | great science fiction | 22 | science fiction 7 | 33 | 2 | regress_rls_carol | great technology book | | 8 | 44 | 1 | regress_rls_carol | great manga | 44 | manga INSERT INTO document VALUES (11, 33, 1, current_user, 'hoge'); -- UNIQUE or PRIMARY KEY constraint violation DOES reveal presence of row SET SESSION AUTHORIZATION regress_rls_bob; INSERT INTO document VALUES (8, 44, 1, 'regress_rls_bob', 'my third manga'); -- Must fail with unique violation, revealing presence of did we can't see ERROR: duplicate key value violates unique constraint "5_10_document_pkey" DETAIL: Key (did)=(8) already exists. SELECT * FROM document WHERE did = 8; -- and confirm we can't see it did | cid | dlevel | dauthor | dtitle -----+-----+--------+---------+-------- -- RLS policies are checked before constraints INSERT INTO document VALUES (8, 44, 1, 'regress_rls_carol', 'my third manga'); -- Should fail with RLS check violation, not duplicate key violation ERROR: new row violates row-level security policy for table "document" UPDATE document SET did = 8, dauthor = 'regress_rls_carol' WHERE did = 5; -- Should fail with RLS check violation, not duplicate key violation ERROR: new row violates row-level security policy for table "document" -- database superuser does bypass RLS policy when enabled RESET SESSION AUTHORIZATION; SET row_security TO ON; SELECT * FROM document; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book 11 | 33 | 1 | regress_rls_carol | hoge SELECT * FROM category; cid | cname -----+----------------- 11 | novel 22 | science fiction 33 | technology 44 | manga -- database superuser does bypass RLS policy when disabled RESET SESSION AUTHORIZATION; SET row_security TO OFF; SELECT * FROM document; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book 11 | 33 | 1 | regress_rls_carol | hoge SELECT * FROM category; cid | cname -----+----------------- 11 | novel 22 | science fiction 33 | technology 44 | manga -- database non-superuser with bypass privilege can bypass RLS policy when disabled SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; SELECT * FROM document; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book 11 | 33 | 1 | regress_rls_carol | hoge SELECT * FROM category; cid | cname -----+----------------- 11 | novel 22 | science fiction 33 | technology 44 | manga -- RLS policy does not apply to table owner when RLS enabled. SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; SELECT * FROM document; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book 11 | 33 | 1 | regress_rls_carol | hoge SELECT * FROM category; cid | cname -----+----------------- 11 | novel 22 | science fiction 33 | technology 44 | manga -- RLS policy does not apply to table owner when RLS disabled. SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO OFF; SELECT * FROM document; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 22 | 2 | regress_rls_bob | my science fiction 4 | 44 | 1 | regress_rls_bob | my first manga 5 | 44 | 2 | regress_rls_bob | my second manga 6 | 22 | 1 | regress_rls_carol | great science fiction 7 | 33 | 2 | regress_rls_carol | great technology book 8 | 44 | 1 | regress_rls_carol | great manga 9 | 22 | 1 | regress_rls_dave | awesome science fiction 10 | 33 | 2 | regress_rls_dave | awesome technology book 11 | 33 | 1 | regress_rls_carol | hoge SELECT * FROM category; cid | cname -----+----------------- 11 | novel 22 | science fiction 33 | technology 44 | manga -- -- Table inheritance and RLS policy -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; CREATE TABLE t1 (a int, junk1 text, b text); ALTER TABLE t1 DROP COLUMN junk1; -- just a disturbing factor GRANT ALL ON t1 TO public; COPY t1 FROM stdin; CREATE TABLE t2 (c float) INHERITS (t1); GRANT ALL ON t2 TO public; COPY t2 FROM stdin; CREATE TABLE t3 (c text, b text, a int); ALTER TABLE t3 INHERIT t1; GRANT ALL ON t3 TO public; COPY t3(a,b,c) FROM stdin; CREATE POLICY p1 ON t1 FOR ALL TO PUBLIC USING (a % 2 = 0); -- be even number CREATE POLICY p2 ON t2 FOR ALL TO PUBLIC USING (a % 2 = 1); -- be odd number ALTER TABLE t1 ENABLE ROW LEVEL SECURITY; ALTER TABLE t2 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM t1; a | b ---+----- 2 | bbb 4 | dad 2 | bcd 4 | def 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: ((a % 2) = 0) -> Seq Scan on t2 t1_2 Filter: ((a % 2) = 0) -> Seq Scan on t3 t1_3 Filter: ((a % 2) = 0) SELECT * FROM t1 WHERE f_leak(b); NOTICE: f_leak => bbb NOTICE: f_leak => dad NOTICE: f_leak => bcd NOTICE: f_leak => def NOTICE: f_leak => yyy a | b ---+----- 2 | bbb 4 | dad 2 | bcd 4 | def 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -- reference to system column SELECT ctid, * FROM t1; ctid | a | b -------+---+----- (0,2) | 2 | bbb (0,4) | 4 | dad (0,2) | 2 | bcd (0,4) | 4 | def (0,2) | 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT *, t1 FROM t1; --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: ((a % 2) = 0) -> Seq Scan on t2 t1_2 Filter: ((a % 2) = 0) -> Seq Scan on t3 t1_3 Filter: ((a % 2) = 0) -- reference to whole-row reference SELECT *, t1 FROM t1; a | b | t1 ---+-----+--------- 2 | bbb | (2,bbb) 4 | dad | (4,dad) 2 | bcd | (2,bcd) 4 | def | (4,def) 2 | yyy | (2,yyy) EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT *, t1 FROM t1; --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: ((a % 2) = 0) -> Seq Scan on t2 t1_2 Filter: ((a % 2) = 0) -> Seq Scan on t3 t1_3 Filter: ((a % 2) = 0) -- for share/update lock SELECT * FROM t1 FOR SHARE; a | b ---+----- 2 | bbb 4 | dad 2 | bcd 4 | def 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 FOR SHARE; --- QUERY PLAN --- LockRows -> Append -> Seq Scan on t1 t1_1 Filter: ((a % 2) = 0) -> Seq Scan on t2 t1_2 Filter: ((a % 2) = 0) -> Seq Scan on t3 t1_3 Filter: ((a % 2) = 0) SELECT * FROM t1 WHERE f_leak(b) FOR SHARE; NOTICE: f_leak => bbb NOTICE: f_leak => dad NOTICE: f_leak => bcd NOTICE: f_leak => def NOTICE: f_leak => yyy a | b ---+----- 2 | bbb 4 | dad 2 | bcd 4 | def 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b) FOR SHARE; --- QUERY PLAN --- LockRows -> Append -> Seq Scan on t1 t1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -- union all query SELECT a, b, ctid FROM t2 UNION ALL SELECT a, b, ctid FROM t3; a | b | ctid ---+-----+------- 1 | abc | (0,1) 3 | cde | (0,3) 1 | xxx | (0,1) 2 | yyy | (0,2) 3 | zzz | (0,3) EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT a, b, ctid FROM t2 UNION ALL SELECT a, b, ctid FROM t3; --- QUERY PLAN --- Append -> Seq Scan on t2 Filter: ((a % 2) = 1) -> Seq Scan on t3 -- superuser is allowed to bypass RLS checks RESET SESSION AUTHORIZATION; SET row_security TO OFF; SELECT * FROM t1 WHERE f_leak(b); NOTICE: f_leak => aba NOTICE: f_leak => bbb NOTICE: f_leak => ccc NOTICE: f_leak => dad NOTICE: f_leak => abc NOTICE: f_leak => bcd NOTICE: f_leak => cde NOTICE: f_leak => def NOTICE: f_leak => xxx NOTICE: f_leak => yyy NOTICE: f_leak => zzz a | b ---+----- 1 | aba 2 | bbb 3 | ccc 4 | dad 1 | abc 2 | bcd 3 | cde 4 | def 1 | xxx 2 | yyy 3 | zzz EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: f_leak(b) -> Seq Scan on t2 t1_2 Filter: f_leak(b) -> Seq Scan on t3 t1_3 Filter: f_leak(b) -- non-superuser with bypass privilege can bypass RLS policy when disabled SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; SELECT * FROM t1 WHERE f_leak(b); NOTICE: f_leak => aba NOTICE: f_leak => bbb NOTICE: f_leak => ccc NOTICE: f_leak => dad NOTICE: f_leak => abc NOTICE: f_leak => bcd NOTICE: f_leak => cde NOTICE: f_leak => def NOTICE: f_leak => xxx NOTICE: f_leak => yyy NOTICE: f_leak => zzz a | b ---+----- 1 | aba 2 | bbb 3 | ccc 4 | dad 1 | abc 2 | bcd 3 | cde 4 | def 1 | xxx 2 | yyy 3 | zzz EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: f_leak(b) -> Seq Scan on t2 t1_2 Filter: f_leak(b) -> Seq Scan on t3 t1_3 Filter: f_leak(b) -- -- Hyper Tables -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE hyper_document ( did int, cid int, dlevel int not null, dauthor name, dtitle text ); GRANT ALL ON hyper_document TO public; SELECT public.create_hypertable('hyper_document', 'did', chunk_time_interval=>2); create_hypertable ----------------------------------------- (2,regress_rls_schema,hyper_document,t) INSERT INTO hyper_document VALUES ( 1, 11, 1, 'regress_rls_bob', 'my first novel'), ( 2, 11, 2, 'regress_rls_bob', 'my second novel'), ( 3, 99, 2, 'regress_rls_bob', 'my science textbook'), ( 4, 55, 1, 'regress_rls_bob', 'my first satire'), ( 5, 99, 2, 'regress_rls_bob', 'my history book'), ( 6, 11, 1, 'regress_rls_carol', 'great science fiction'), ( 7, 99, 2, 'regress_rls_carol', 'great technology book'), ( 8, 55, 2, 'regress_rls_carol', 'great satire'), ( 9, 11, 1, 'regress_rls_dave', 'awesome science fiction'), (10, 99, 2, 'regress_rls_dave', 'awesome technology book'); ALTER TABLE hyper_document ENABLE ROW LEVEL SECURITY; GRANT ALL ON _timescaledb_internal._hyper_2_9_chunk TO public; -- Create policy on parent -- user's security level must be higher than or equal to document's CREATE POLICY pp1 ON hyper_document AS PERMISSIVE USING (dlevel <= (SELECT seclv FROM uaccount WHERE pguser = current_user)); -- Dave is only allowed to see cid < 55 CREATE POLICY pp1r ON hyper_document AS RESTRICTIVE TO regress_rls_dave USING (cid < 55); \d+ hyper_document Table "regress_rls_schema.hyper_document" Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- did | integer | | not null | | plain | | cid | integer | | | | plain | | dlevel | integer | | not null | | plain | | dauthor | name | | | | plain | | dtitle | text | | | | extended | | Indexes: "hyper_document_did_idx" btree (did DESC) Policies: POLICY "pp1" USING ((dlevel <= ( SELECT uaccount.seclv FROM uaccount WHERE (uaccount.pguser = CURRENT_USER)))) POLICY "pp1r" AS RESTRICTIVE TO regress_rls_dave USING ((cid < 55)) Not-null constraints: "hyper_document_did_not_null" NOT NULL "did" "hyper_document_dlevel_not_null" NOT NULL "dlevel" Child tables: _timescaledb_internal._hyper_2_10_chunk, _timescaledb_internal._hyper_2_11_chunk, _timescaledb_internal._hyper_2_12_chunk, _timescaledb_internal._hyper_2_13_chunk, _timescaledb_internal._hyper_2_14_chunk, _timescaledb_internal._hyper_2_9_chunk SELECT * FROM pg_policies WHERE schemaname = 'regress_rls_schema' AND tablename like '%hyper_document%' ORDER BY policyname; schemaname | tablename | policyname | permissive | roles | cmd | qual | with_check --------------------+----------------+------------+-------------+--------------------+-----+--------------------------------------------+------------ regress_rls_schema | hyper_document | pp1 | PERMISSIVE | {public} | ALL | (dlevel <= ( SELECT uaccount.seclv +| | | | | | | FROM uaccount +| | | | | | | WHERE (uaccount.pguser = CURRENT_USER))) | regress_rls_schema | hyper_document | pp1r | RESTRICTIVE | {regress_rls_dave} | ALL | (cid < 55) | -- viewpoint from regress_rls_bob SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my first satire NOTICE: f_leak => great science fiction NOTICE: f_leak => awesome science fiction did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 4 | 55 | 1 | regress_rls_bob | my first satire 6 | 11 | 1 | regress_rls_carol | great science fiction 9 | 11 | 1 | regress_rls_dave | awesome science fiction EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 InitPlan 1 -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on hyper_document hyper_document_1 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -- viewpoint from regress_rls_carol SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science textbook NOTICE: f_leak => my first satire NOTICE: f_leak => my history book NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great satire NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire 9 | 11 | 1 | regress_rls_dave | awesome science fiction 10 | 99 | 2 | regress_rls_dave | awesome technology book EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 InitPlan 1 -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on hyper_document hyper_document_1 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -- viewpoint from regress_rls_dave SET SESSION AUTHORIZATION regress_rls_dave; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => great science fiction NOTICE: f_leak => awesome science fiction did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 6 | 11 | 1 | regress_rls_carol | great science fiction 9 | 11 | 1 | regress_rls_dave | awesome science fiction EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 InitPlan 1 -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on hyper_document hyper_document_1 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -- pp1 ERROR INSERT INTO hyper_document VALUES (1, 11, 5, 'regress_rls_dave', 'testing pp1'); -- fail ERROR: new row violates row-level security policy for table "hyper_document" -- pp1r ERROR INSERT INTO hyper_document VALUES (1, 99, 1, 'regress_rls_dave', 'testing pp1r'); -- fail ERROR: new row violates row-level security policy "pp1r" for table "hyper_document" -- Show that RLS policy does not apply for direct inserts to children -- This should fail with RLS POLICY pp1r violation. INSERT INTO hyper_document VALUES (1, 55, 1, 'regress_rls_dave', 'testing RLS with hypertables'); -- fail ERROR: new row violates row-level security policy "pp1r" for table "hyper_document" -- But this should succeed. INSERT INTO _timescaledb_internal._hyper_2_9_chunk VALUES (1, 55, 1, 'regress_rls_dave', 'testing RLS with hypertables'); -- success -- We still cannot see the row using the parent SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => great science fiction NOTICE: f_leak => awesome science fiction did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 6 | 11 | 1 | regress_rls_carol | great science fiction 9 | 11 | 1 | regress_rls_dave | awesome science fiction -- But we can if we look directly SELECT * FROM _timescaledb_internal._hyper_2_9_chunk WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => my first novel NOTICE: f_leak => testing RLS with hypertables did | cid | dlevel | dauthor | dtitle -----+-----+--------+------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables -- Turn on RLS and create policy on child to show RLS is checked before constraints SET SESSION AUTHORIZATION regress_rls_alice; ALTER TABLE _timescaledb_internal._hyper_2_9_chunk ENABLE ROW LEVEL SECURITY; CREATE POLICY pp3 ON _timescaledb_internal._hyper_2_9_chunk AS RESTRICTIVE USING (cid < 55); -- This should fail with RLS violation now. SET SESSION AUTHORIZATION regress_rls_dave; INSERT INTO _timescaledb_internal._hyper_2_9_chunk VALUES (1, 55, 1, 'regress_rls_dave', 'testing RLS with hypertables - round 2'); -- fail ERROR: new row violates row-level security policy for table "_hyper_2_9_chunk" -- And now we cannot see directly into the partition either, due to RLS SELECT * FROM _timescaledb_internal._hyper_2_9_chunk WHERE f_leak(dtitle) ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+---------+-------- -- The parent looks same as before -- viewpoint from regress_rls_dave SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => great science fiction NOTICE: f_leak => awesome science fiction did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 6 | 11 | 1 | regress_rls_carol | great science fiction 9 | 11 | 1 | regress_rls_dave | awesome science fiction EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 InitPlan 1 -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on hyper_document hyper_document_1 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((cid < 55) AND (dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -- viewpoint from regress_rls_carol SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => my first novel NOTICE: f_leak => testing RLS with hypertables NOTICE: f_leak => my second novel NOTICE: f_leak => my science textbook NOTICE: f_leak => my first satire NOTICE: f_leak => my history book NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great satire NOTICE: f_leak => awesome science fiction NOTICE: f_leak => awesome technology book did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire 9 | 11 | 1 | regress_rls_dave | awesome science fiction 10 | 99 | 2 | regress_rls_dave | awesome technology book EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 InitPlan 1 -> Index Scan using uaccount_pkey on uaccount Index Cond: (pguser = CURRENT_USER) -> Seq Scan on hyper_document hyper_document_1 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((dlevel <= (InitPlan 1).col1) AND f_leak(dtitle)) -- only owner can change policies ALTER POLICY pp1 ON hyper_document USING (true); --fail ERROR: must be owner of table hyper_document DROP POLICY pp1 ON hyper_document; --fail ERROR: must be owner of relation hyper_document SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY pp1 ON hyper_document USING (dauthor = current_user); -- viewpoint from regress_rls_bob again SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => my first novel NOTICE: f_leak => my second novel NOTICE: f_leak => my science textbook NOTICE: f_leak => my first satire NOTICE: f_leak => my history book did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+--------------------- 1 | 11 | 1 | regress_rls_bob | my first novel 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book -- viewpoint from rls_regres_carol again SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; NOTICE: f_leak => great science fiction NOTICE: f_leak => great technology book NOTICE: f_leak => great satire did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+----------------------- 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); --- QUERY PLAN --- Custom Scan (ChunkAppend) on hyper_document Chunks excluded during startup: 0 -> Seq Scan on hyper_document hyper_document_1 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_9_chunk hyper_document_2 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_10_chunk hyper_document_3 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_11_chunk hyper_document_4 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_12_chunk hyper_document_5 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_13_chunk hyper_document_6 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -> Seq Scan on _hyper_2_14_chunk hyper_document_7 Filter: ((dauthor = CURRENT_USER) AND f_leak(dtitle)) -- database superuser does bypass RLS policy when enabled RESET SESSION AUTHORIZATION; SET row_security TO ON; SELECT * FROM hyper_document ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire 9 | 11 | 1 | regress_rls_dave | awesome science fiction 10 | 99 | 2 | regress_rls_dave | awesome technology book SELECT * FROM _timescaledb_internal._hyper_2_9_chunk ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables -- database non-superuser with bypass privilege can bypass RLS policy when disabled SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; SELECT * FROM hyper_document ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire 9 | 11 | 1 | regress_rls_dave | awesome science fiction 10 | 99 | 2 | regress_rls_dave | awesome technology book SELECT * FROM _timescaledb_internal._hyper_2_9_chunk ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables -- RLS policy does not apply to table owner when RLS enabled. SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; SELECT * FROM hyper_document ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables 2 | 11 | 2 | regress_rls_bob | my second novel 3 | 99 | 2 | regress_rls_bob | my science textbook 4 | 55 | 1 | regress_rls_bob | my first satire 5 | 99 | 2 | regress_rls_bob | my history book 6 | 11 | 1 | regress_rls_carol | great science fiction 7 | 99 | 2 | regress_rls_carol | great technology book 8 | 55 | 2 | regress_rls_carol | great satire 9 | 11 | 1 | regress_rls_dave | awesome science fiction 10 | 99 | 2 | regress_rls_dave | awesome technology book SELECT * FROM _timescaledb_internal._hyper_2_9_chunk ORDER BY did, cid; did | cid | dlevel | dauthor | dtitle -----+-----+--------+------------------+------------------------------ 1 | 11 | 1 | regress_rls_bob | my first novel 1 | 55 | 1 | regress_rls_dave | testing RLS with hypertables -- When RLS disabled, other users get ERROR. SET SESSION AUTHORIZATION regress_rls_dave; SET row_security TO OFF; SELECT * FROM hyper_document ORDER BY did, cid; ERROR: query would be affected by row-level security policy for table "hyper_document" SELECT * FROM _timescaledb_internal._hyper_2_9_chunk ORDER BY did, cid; ERROR: query would be affected by row-level security policy for table "_hyper_2_9_chunk" -- Check behavior with a policy that uses a SubPlan not an InitPlan. SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; CREATE POLICY pp3 ON hyper_document AS RESTRICTIVE USING ((SELECT dlevel <= seclv FROM uaccount WHERE pguser = current_user)); SET SESSION AUTHORIZATION regress_rls_carol; INSERT INTO hyper_document VALUES (100, 11, 5, 'regress_rls_carol', 'testing pp3'); -- fail ERROR: new row violates row-level security policy "pp3" for table "hyper_document" ----- Dependencies ----- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; CREATE TABLE dependee (x integer, y integer); SELECT public.create_hypertable('dependee', 'x', chunk_time_interval=>2); create_hypertable ----------------------------------- (3,regress_rls_schema,dependee,t) CREATE TABLE dependent (x integer, y integer); SELECT public.create_hypertable('dependent', 'x', chunk_time_interval=>2); create_hypertable ------------------------------------ (4,regress_rls_schema,dependent,t) CREATE POLICY d1 ON dependent FOR ALL TO PUBLIC USING (x = (SELECT d.x FROM dependee d WHERE d.y = y)); DROP TABLE dependee; -- Should fail without CASCADE due to dependency on row security qual? ERROR: cannot drop table dependee because other objects depend on it DETAIL: policy d1 on table dependent depends on table dependee HINT: Use DROP ... CASCADE to drop the dependent objects too. DROP TABLE dependee CASCADE; NOTICE: drop cascades to policy d1 on table dependent EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM dependent; -- After drop, should be unqualified --- QUERY PLAN --- Seq Scan on dependent ----- RECURSION ---- -- -- Simple recursion -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE rec1 (x integer, y integer); SELECT public.create_hypertable('rec1', 'x', chunk_time_interval=>2); create_hypertable ------------------------------- (5,regress_rls_schema,rec1,t) CREATE POLICY r1 ON rec1 USING (x = (SELECT r.x FROM rec1 r WHERE y = r.y)); ALTER TABLE rec1 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rec1; -- fail, direct recursion ERROR: infinite recursion detected in policy for relation "rec1" -- -- Mutual recursion -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE rec2 (a integer, b integer); SELECT public.create_hypertable('rec2', 'x', chunk_time_interval=>2); ERROR: column "x" does not exist ALTER POLICY r1 ON rec1 USING (x = (SELECT a FROM rec2 WHERE b = y)); CREATE POLICY r2 ON rec2 USING (a = (SELECT x FROM rec1 WHERE y = b)); ALTER TABLE rec2 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rec1; -- fail, mutual recursion ERROR: infinite recursion detected in policy for relation "rec1" -- -- Mutual recursion via views -- SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW rec1v AS SELECT * FROM rec1; CREATE VIEW rec2v AS SELECT * FROM rec2; SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY r1 ON rec1 USING (x = (SELECT a FROM rec2v WHERE b = y)); ALTER POLICY r2 ON rec2 USING (a = (SELECT x FROM rec1v WHERE y = b)); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rec1; -- fail, mutual recursion via views ERROR: infinite recursion detected in policy for relation "rec1" -- -- Mutual recursion via .s.b views -- SET SESSION AUTHORIZATION regress_rls_bob; \set VERBOSITY terse \\ -- suppress cascade details DROP VIEW rec1v, rec2v CASCADE; NOTICE: drop cascades to 2 other objects \set VERBOSITY default CREATE VIEW rec1v WITH (security_barrier) AS SELECT * FROM rec1; CREATE VIEW rec2v WITH (security_barrier) AS SELECT * FROM rec2; SET SESSION AUTHORIZATION regress_rls_alice; CREATE POLICY r1 ON rec1 USING (x = (SELECT a FROM rec2v WHERE b = y)); CREATE POLICY r2 ON rec2 USING (a = (SELECT x FROM rec1v WHERE y = b)); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rec1; -- fail, mutual recursion via s.b. views ERROR: infinite recursion detected in policy for relation "rec1" -- -- recursive RLS and VIEWs in policy -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE s1 (a int, b text); SELECT public.create_hypertable('s1', 'a', chunk_time_interval=>2); create_hypertable ----------------------------- (6,regress_rls_schema,s1,t) INSERT INTO s1 (SELECT x, md5(x::text) FROM generate_series(-10,10) x); CREATE TABLE s2 (x int, y text); SELECT public.create_hypertable('s2', 'x', chunk_time_interval=>2); create_hypertable ----------------------------- (7,regress_rls_schema,s2,t) INSERT INTO s2 (SELECT x, md5(x::text) FROM generate_series(-6,6) x); GRANT SELECT ON s1, s2 TO regress_rls_bob; CREATE POLICY p1 ON s1 USING (a in (select x from s2 where y like '%2f%')); CREATE POLICY p2 ON s2 USING (x in (select a from s1 where b like '%22%')); CREATE POLICY p3 ON s1 FOR INSERT WITH CHECK (a = (SELECT a FROM s1)); ALTER TABLE s1 ENABLE ROW LEVEL SECURITY; ALTER TABLE s2 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW v2 AS SELECT * FROM s2 WHERE y like '%af%'; SELECT * FROM s1 WHERE f_leak(b); -- fail (infinite recursion) ERROR: infinite recursion detected in policy for relation "s1" INSERT INTO s1 VALUES (1, 'foo'); -- fail (infinite recursion) ERROR: infinite recursion detected in policy for relation "s1" SET SESSION AUTHORIZATION regress_rls_alice; DROP POLICY p3 on s1; ALTER POLICY p2 ON s2 USING (x % 2 = 0); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM s1 WHERE f_leak(b); -- OK NOTICE: f_leak => c81e728d9d4c2f636f067f89cc14862c NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c a | b ---+---------------------------------- 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM only s1 WHERE f_leak(b); --- QUERY PLAN --- Seq Scan on s1 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) SubPlan 1 -> Append -> Seq Scan on s2 s2_1 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_27_chunk s2_2 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_28_chunk s2_3 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_29_chunk s2_4 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_30_chunk s2_5 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_31_chunk s2_6 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_32_chunk s2_7 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) -> Seq Scan on _hyper_7_33_chunk s2_8 Filter: (((x % 2) = 0) AND (y ~~ '%2f%'::text)) SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY p1 ON s1 USING (a in (select x from v2)); -- using VIEW in RLS policy SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM s1 WHERE f_leak(b); -- OK NOTICE: f_leak => 0267aaf632e87a63288a08331f22c7c3 NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc a | b ----+---------------------------------- -4 | 0267aaf632e87a63288a08331f22c7c3 6 | 1679091c5a880faf6fb5e6087eb1b2dc EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM s1 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on s1 Chunks excluded during startup: 0 -> Seq Scan on s1 s1_1 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) SubPlan 1 -> Append -> Seq Scan on s2 s2_1 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_27_chunk s2_2 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_28_chunk s2_3 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_29_chunk s2_4 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_30_chunk s2_5 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_31_chunk s2_6 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_32_chunk s2_7 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_33_chunk s2_8 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_6_16_chunk s1_2 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) -> Seq Scan on _hyper_6_17_chunk s1_3 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) -> Seq Scan on _hyper_6_18_chunk s1_4 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) -> Seq Scan on _hyper_6_19_chunk s1_5 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) -> Seq Scan on _hyper_6_20_chunk s1_6 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) -> Seq Scan on _hyper_6_21_chunk s1_7 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) -> Seq Scan on _hyper_6_22_chunk s1_8 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) -> Seq Scan on _hyper_6_23_chunk s1_9 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) -> Seq Scan on _hyper_6_24_chunk s1_10 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) -> Seq Scan on _hyper_6_25_chunk s1_11 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) -> Seq Scan on _hyper_6_26_chunk s1_12 Filter: ((ANY (a = (hashed SubPlan 1).col1)) AND f_leak(b)) SELECT (SELECT x FROM s1 LIMIT 1) xx, * FROM s2 WHERE y like '%28%'; xx | x | y ----+----+---------------------------------- -6 | -6 | 596a3d04481816330f07e4f97510c28f -4 | -4 | 0267aaf632e87a63288a08331f22c7c3 2 | 2 | c81e728d9d4c2f636f067f89cc14862c EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT (SELECT x FROM s1 LIMIT 1) xx, * FROM s2 WHERE y like '%28%'; --- QUERY PLAN --- Result -> Append -> Seq Scan on s2 s2_1 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_27_chunk s2_2 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_28_chunk s2_3 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_29_chunk s2_4 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_30_chunk s2_5 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_31_chunk s2_6 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_32_chunk s2_7 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) -> Seq Scan on _hyper_7_33_chunk s2_8 Filter: (((x % 2) = 0) AND (y ~~ '%28%'::text)) SubPlan 2 -> Limit -> Result -> Custom Scan (ChunkAppend) on s1 -> Seq Scan on s1 s1_1 Filter: (ANY (a = (hashed SubPlan 1).col1)) SubPlan 1 -> Append -> Seq Scan on s2 s2_10 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_27_chunk s2_11 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_28_chunk s2_12 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_29_chunk s2_13 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_30_chunk s2_14 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_31_chunk s2_15 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_32_chunk s2_16 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_7_33_chunk s2_17 Filter: (((x % 2) = 0) AND (y ~~ '%af%'::text)) -> Seq Scan on _hyper_6_16_chunk s1_2 Filter: (ANY (a = (hashed SubPlan 1).col1)) -> Seq Scan on _hyper_6_17_chunk s1_3 Filter: (ANY (a = (hashed SubPlan 1).col1)) -> Seq Scan on _hyper_6_18_chunk s1_4 Filter: (ANY (a = (hashed SubPlan 1).col1)) -> Seq Scan on _hyper_6_19_chunk s1_5 Filter: (ANY (a = (hashed SubPlan 1).col1)) -> Seq Scan on _hyper_6_20_chunk s1_6 Filter: (ANY (a = (hashed SubPlan 1).col1)) -> Seq Scan on _hyper_6_21_chunk s1_7 Filter: (ANY (a = (hashed SubPlan 1).col1)) -> Seq Scan on _hyper_6_22_chunk s1_8 Filter: (ANY (a = (hashed SubPlan 1).col1)) -> Seq Scan on _hyper_6_23_chunk s1_9 Filter: (ANY (a = (hashed SubPlan 1).col1)) -> Seq Scan on _hyper_6_24_chunk s1_10 Filter: (ANY (a = (hashed SubPlan 1).col1)) -> Seq Scan on _hyper_6_25_chunk s1_11 Filter: (ANY (a = (hashed SubPlan 1).col1)) -> Seq Scan on _hyper_6_26_chunk s1_12 Filter: (ANY (a = (hashed SubPlan 1).col1)) SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY p2 ON s2 USING (x in (select a from s1 where b like '%d2%')); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM s1 WHERE f_leak(b); -- fail (infinite recursion via view) ERROR: infinite recursion detected in policy for relation "s1" -- prepared statement with regress_rls_alice privilege PREPARE p1(int) AS SELECT * FROM t1 WHERE a <= $1; EXECUTE p1(2); a | b ---+----- 2 | bbb 2 | bcd 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE p1(2); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: ((a <= 2) AND ((a % 2) = 0)) -> Seq Scan on t2 t1_2 Filter: ((a <= 2) AND ((a % 2) = 0)) -> Seq Scan on t3 t1_3 Filter: ((a <= 2) AND ((a % 2) = 0)) -- superuser is allowed to bypass RLS checks RESET SESSION AUTHORIZATION; SET row_security TO OFF; SELECT * FROM t1 WHERE f_leak(b); NOTICE: f_leak => aba NOTICE: f_leak => bbb NOTICE: f_leak => ccc NOTICE: f_leak => dad NOTICE: f_leak => abc NOTICE: f_leak => bcd NOTICE: f_leak => cde NOTICE: f_leak => def NOTICE: f_leak => xxx NOTICE: f_leak => yyy NOTICE: f_leak => zzz a | b ---+----- 1 | aba 2 | bbb 3 | ccc 4 | dad 1 | abc 2 | bcd 3 | cde 4 | def 1 | xxx 2 | yyy 3 | zzz EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: f_leak(b) -> Seq Scan on t2 t1_2 Filter: f_leak(b) -> Seq Scan on t3 t1_3 Filter: f_leak(b) -- plan cache should be invalidated EXECUTE p1(2); a | b ---+----- 1 | aba 2 | bbb 1 | abc 2 | bcd 1 | xxx 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE p1(2); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: (a <= 2) -> Seq Scan on t2 t1_2 Filter: (a <= 2) -> Seq Scan on t3 t1_3 Filter: (a <= 2) PREPARE p2(int) AS SELECT * FROM t1 WHERE a = $1; EXECUTE p2(2); a | b ---+----- 2 | bbb 2 | bcd 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE p2(2); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: (a = 2) -> Seq Scan on t2 t1_2 Filter: (a = 2) -> Seq Scan on t3 t1_3 Filter: (a = 2) -- also, case when privilege switch from superuser SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; EXECUTE p2(2); a | b ---+----- 2 | bbb 2 | bcd 2 | yyy EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE p2(2); --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 Filter: ((a = 2) AND ((a % 2) = 0)) -> Seq Scan on t2 t1_2 Filter: ((a = 2) AND ((a % 2) = 0)) -> Seq Scan on t3 t1_3 Filter: ((a = 2) AND ((a % 2) = 0)) -- -- UPDATE / DELETE and Row-level security -- SET SESSION AUTHORIZATION regress_rls_bob; EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t1 SET b = b || b WHERE f_leak(b); --- QUERY PLAN --- Update on t1 Update on t1 t1_1 Update on t2 t1_2 Update on t3 t1_3 -> Result -> Append -> Seq Scan on t1 t1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: (((a % 2) = 0) AND f_leak(b)) UPDATE t1 SET b = b || b WHERE f_leak(b); NOTICE: f_leak => bbb NOTICE: f_leak => dad NOTICE: f_leak => bcd NOTICE: f_leak => def NOTICE: f_leak => yyy EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE only t1 SET b = b || '_updt' WHERE f_leak(b); --- QUERY PLAN --- Update on t1 -> Seq Scan on t1 Filter: (((a % 2) = 0) AND f_leak(b)) UPDATE only t1 SET b = b || '_updt' WHERE f_leak(b); NOTICE: f_leak => bbbbbb NOTICE: f_leak => daddad -- returning clause with system column UPDATE only t1 SET b = b WHERE f_leak(b) RETURNING ctid, *, t1; NOTICE: f_leak => bbbbbb_updt NOTICE: f_leak => daddad_updt ctid | a | b | t1 --------+---+-------------+----------------- (0,9) | 2 | bbbbbb_updt | (2,bbbbbb_updt) (0,10) | 4 | daddad_updt | (4,daddad_updt) UPDATE t1 SET b = b WHERE f_leak(b) RETURNING *; NOTICE: f_leak => bbbbbb_updt NOTICE: f_leak => daddad_updt NOTICE: f_leak => bcdbcd NOTICE: f_leak => defdef NOTICE: f_leak => yyyyyy a | b ---+------------- 2 | bbbbbb_updt 4 | daddad_updt 2 | bcdbcd 4 | defdef 2 | yyyyyy UPDATE t1 SET b = b WHERE f_leak(b) RETURNING ctid, *, t1; NOTICE: f_leak => bbbbbb_updt NOTICE: f_leak => daddad_updt NOTICE: f_leak => bcdbcd NOTICE: f_leak => defdef NOTICE: f_leak => yyyyyy ctid | a | b | t1 --------+---+-------------+----------------- (0,13) | 2 | bbbbbb_updt | (2,bbbbbb_updt) (0,14) | 4 | daddad_updt | (4,daddad_updt) (0,9) | 2 | bcdbcd | (2,bcdbcd) (0,10) | 4 | defdef | (4,defdef) (0,6) | 2 | yyyyyy | (2,yyyyyy) -- updates with from clause EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t2 SET b=t2.b FROM t3 WHERE t2.a = 3 and t3.a = 2 AND f_leak(t2.b) AND f_leak(t3.b); --- QUERY PLAN --- Update on t2 -> Nested Loop -> Seq Scan on t2 Filter: ((a = 3) AND ((a % 2) = 1) AND f_leak(b)) -> Seq Scan on t3 Filter: ((a = 2) AND f_leak(b)) UPDATE t2 SET b=t2.b FROM t3 WHERE t2.a = 3 and t3.a = 2 AND f_leak(t2.b) AND f_leak(t3.b); NOTICE: f_leak => cde NOTICE: f_leak => yyyyyy EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t1 SET b=t1.b FROM t2 WHERE t1.a = 3 and t2.a = 3 AND f_leak(t1.b) AND f_leak(t2.b); --- QUERY PLAN --- Update on t1 Update on t1 t1_1 Update on t2 t1_2 Update on t3 t1_3 -> Nested Loop -> Seq Scan on t2 Filter: ((a = 3) AND ((a % 2) = 1) AND f_leak(b)) -> Append -> Seq Scan on t1 t1_1 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) UPDATE t1 SET b=t1.b FROM t2 WHERE t1.a = 3 and t2.a = 3 AND f_leak(t1.b) AND f_leak(t2.b); NOTICE: f_leak => cde EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t2 SET b=t2.b FROM t1 WHERE t1.a = 3 and t2.a = 3 AND f_leak(t1.b) AND f_leak(t2.b); --- QUERY PLAN --- Update on t2 -> Nested Loop -> Seq Scan on t2 Filter: ((a = 3) AND ((a % 2) = 1) AND f_leak(b)) -> Append -> Seq Scan on t1 t1_1 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: ((a = 3) AND ((a % 2) = 0) AND f_leak(b)) UPDATE t2 SET b=t2.b FROM t1 WHERE t1.a = 3 and t2.a = 3 AND f_leak(t1.b) AND f_leak(t2.b); NOTICE: f_leak => cde -- updates with from clause self join EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t2 t2_1 SET b = t2_2.b FROM t2 t2_2 WHERE t2_1.a = 3 AND t2_2.a = t2_1.a AND t2_2.b = t2_1.b AND f_leak(t2_1.b) AND f_leak(t2_2.b) RETURNING *, t2_1, t2_2; --- QUERY PLAN --- Update on t2 t2_1 -> Nested Loop Join Filter: (t2_1.b = t2_2.b) -> Seq Scan on t2 t2_1 Filter: ((a = 3) AND ((a % 2) = 1) AND f_leak(b)) -> Seq Scan on t2 t2_2 Filter: ((a = 3) AND ((a % 2) = 1) AND f_leak(b)) UPDATE t2 t2_1 SET b = t2_2.b FROM t2 t2_2 WHERE t2_1.a = 3 AND t2_2.a = t2_1.a AND t2_2.b = t2_1.b AND f_leak(t2_1.b) AND f_leak(t2_2.b) RETURNING *, t2_1, t2_2; NOTICE: f_leak => cde NOTICE: f_leak => cde a | b | c | a | b | c | t2_1 | t2_2 ---+-----+-----+---+-----+-----+-------------+------------- 3 | cde | 3.3 | 3 | cde | 3.3 | (3,cde,3.3) | (3,cde,3.3) EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t1 t1_1 SET b = t1_2.b FROM t1 t1_2 WHERE t1_1.a = 4 AND t1_2.a = t1_1.a AND t1_2.b = t1_1.b AND f_leak(t1_1.b) AND f_leak(t1_2.b) RETURNING *, t1_1, t1_2; --- QUERY PLAN --- Update on t1 t1_1 Update on t1 t1_1_1 Update on t2 t1_1_2 Update on t3 t1_1_3 -> Nested Loop Join Filter: (t1_1.b = t1_2.b) -> Append -> Seq Scan on t1 t1_1_1 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_1_2 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_1_3 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) -> Materialize -> Append -> Seq Scan on t1 t1_2_1 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2_2 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_2_3 Filter: ((a = 4) AND ((a % 2) = 0) AND f_leak(b)) UPDATE t1 t1_1 SET b = t1_2.b FROM t1 t1_2 WHERE t1_1.a = 4 AND t1_2.a = t1_1.a AND t1_2.b = t1_1.b AND f_leak(t1_1.b) AND f_leak(t1_2.b) RETURNING *, t1_1, t1_2; NOTICE: f_leak => daddad_updt NOTICE: f_leak => daddad_updt NOTICE: f_leak => defdef NOTICE: f_leak => defdef a | b | a | b | t1_1 | t1_2 ---+-------------+---+-------------+-----------------+----------------- 4 | daddad_updt | 4 | daddad_updt | (4,daddad_updt) | (4,daddad_updt) 4 | defdef | 4 | defdef | (4,defdef) | (4,defdef) RESET SESSION AUTHORIZATION; SET row_security TO OFF; SELECT * FROM t1 ORDER BY a,b; a | b ---+------------- 1 | aba 1 | abc 1 | xxx 2 | bbbbbb_updt 2 | bcdbcd 2 | yyyyyy 3 | ccc 3 | cde 3 | zzz 4 | daddad_updt 4 | defdef SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; EXPLAIN (BUFFERS OFF, COSTS OFF) DELETE FROM only t1 WHERE f_leak(b); --- QUERY PLAN --- Delete on t1 -> Seq Scan on t1 Filter: (((a % 2) = 0) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) DELETE FROM t1 WHERE f_leak(b); --- QUERY PLAN --- Delete on t1 Delete on t1 t1_1 Delete on t2 t1_2 Delete on t3 t1_3 -> Append -> Seq Scan on t1 t1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t2 t1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on t3 t1_3 Filter: (((a % 2) = 0) AND f_leak(b)) DELETE FROM only t1 WHERE f_leak(b) RETURNING ctid, *, t1; NOTICE: f_leak => bbbbbb_updt NOTICE: f_leak => daddad_updt ctid | a | b | t1 --------+---+-------------+----------------- (0,13) | 2 | bbbbbb_updt | (2,bbbbbb_updt) (0,15) | 4 | daddad_updt | (4,daddad_updt) DELETE FROM t1 WHERE f_leak(b) RETURNING ctid, *, t1; NOTICE: f_leak => bcdbcd NOTICE: f_leak => defdef NOTICE: f_leak => yyyyyy ctid | a | b | t1 --------+---+--------+------------ (0,9) | 2 | bcdbcd | (2,bcdbcd) (0,13) | 4 | defdef | (4,defdef) (0,6) | 2 | yyyyyy | (2,yyyyyy) -- -- S.b. view on top of Row-level security -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE b1 (a int, b text); SELECT public.create_hypertable('b1', 'a', chunk_time_interval=>2); create_hypertable ----------------------------- (8,regress_rls_schema,b1,t) INSERT INTO b1 (SELECT x, md5(x::text) FROM generate_series(-10,10) x); CREATE POLICY p1 ON b1 USING (a % 2 = 0); ALTER TABLE b1 ENABLE ROW LEVEL SECURITY; GRANT ALL ON b1 TO regress_rls_bob; SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW bv1 WITH (security_barrier) AS SELECT * FROM b1 WHERE a > 0 WITH CHECK OPTION; GRANT ALL ON bv1 TO regress_rls_carol; SET SESSION AUTHORIZATION regress_rls_carol; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM bv1 WHERE f_leak(b); --- QUERY PLAN --- Subquery Scan on bv1 Filter: f_leak(bv1.b) -> Append -> Seq Scan on b1 b1_1 Filter: ((a > 0) AND ((a % 2) = 0)) -> Index Scan using _hyper_8_39_chunk_b1_a_idx on _hyper_8_39_chunk b1_2 Index Cond: (a > 0) Filter: ((a % 2) = 0) -> Index Scan using _hyper_8_40_chunk_b1_a_idx on _hyper_8_40_chunk b1_3 Index Cond: (a > 0) Filter: ((a % 2) = 0) -> Index Scan using _hyper_8_41_chunk_b1_a_idx on _hyper_8_41_chunk b1_4 Index Cond: (a > 0) Filter: ((a % 2) = 0) -> Index Scan using _hyper_8_42_chunk_b1_a_idx on _hyper_8_42_chunk b1_5 Index Cond: (a > 0) Filter: ((a % 2) = 0) -> Index Scan using _hyper_8_43_chunk_b1_a_idx on _hyper_8_43_chunk b1_6 Index Cond: (a > 0) Filter: ((a % 2) = 0) -> Index Scan using _hyper_8_44_chunk_b1_a_idx on _hyper_8_44_chunk b1_7 Index Cond: (a > 0) Filter: ((a % 2) = 0) SELECT * FROM bv1 WHERE f_leak(b); NOTICE: f_leak => c81e728d9d4c2f636f067f89cc14862c NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc NOTICE: f_leak => c9f0f895fb98ab9159f51fd0297e236d NOTICE: f_leak => d3d9446802a44259755d38e6d163e820 a | b ----+---------------------------------- 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 INSERT INTO bv1 VALUES (-1, 'xxx'); -- should fail view WCO ERROR: new row violates row-level security policy for table "b1" INSERT INTO bv1 VALUES (11, 'xxx'); -- should fail RLS check ERROR: new row violates row-level security policy for table "b1" INSERT INTO bv1 VALUES (12, 'xxx'); -- ok EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE bv1 SET b = 'yyy' WHERE a = 4 AND f_leak(b); --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Update on b1 Update on _hyper_8_41_chunk b1_1 -> Result -> Custom Scan (ChunkAppend) on b1 Chunks excluded during startup: 0 -> Index Scan using _hyper_8_41_chunk_b1_a_idx on _hyper_8_41_chunk b1_1 Index Cond: ((a > 0) AND (a = 4)) Filter: (((a % 2) = 0) AND f_leak(b)) UPDATE bv1 SET b = 'yyy' WHERE a = 4 AND f_leak(b); NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c EXPLAIN (BUFFERS OFF, COSTS OFF) DELETE FROM bv1 WHERE a = 6 AND f_leak(b); --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Delete on b1 Delete on _hyper_8_42_chunk b1_1 -> Custom Scan (ChunkAppend) on b1 Chunks excluded during startup: 0 -> Index Scan using _hyper_8_42_chunk_b1_a_idx on _hyper_8_42_chunk b1_1 Index Cond: ((a > 0) AND (a = 6)) Filter: (((a % 2) = 0) AND f_leak(b)) DELETE FROM bv1 WHERE a = 6 AND f_leak(b); NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM b1; a | b -----+---------------------------------- -10 | 1b0fd9efa5279c4203b7c70233f86dbf -9 | 252e691406782824eec43d7eadc3d256 -8 | a8d2ec85eaf98407310b72eb73dda247 -7 | 74687a12d3915d3c4d83f1af7b3683d5 -6 | 596a3d04481816330f07e4f97510c28f -5 | 47c1b025fa18ea96c33fbb6718688c0f -4 | 0267aaf632e87a63288a08331f22c7c3 -3 | b3149ecea4628efd23d2f86e5a723472 -2 | 5d7b9adcbe1c629ec722529dd12e5129 -1 | 6bb61e3b7bce0931da574d19d1d82c88 0 | cfcd208495d565ef66e7dff9f98764da 1 | c4ca4238a0b923820dcc509a6f75849b 2 | c81e728d9d4c2f636f067f89cc14862c 3 | eccbc87e4b5ce2fe28308fd9f2a7baf3 5 | e4da3b7fbbce2345d7772b0674a318d5 4 | yyy 7 | 8f14e45fceea167a5a36dedd4bea2543 8 | c9f0f895fb98ab9159f51fd0297e236d 9 | 45c48cce2e2d7fbdea1afc51c7c6ad26 10 | d3d9446802a44259755d38e6d163e820 12 | xxx -- -- INSERT ... ON CONFLICT DO UPDATE and Row-level security -- SET SESSION AUTHORIZATION regress_rls_alice; DROP POLICY p1 ON document; DROP POLICY p1r ON document; CREATE POLICY p1 ON document FOR SELECT USING (true); CREATE POLICY p2 ON document FOR INSERT WITH CHECK (dauthor = current_user); CREATE POLICY p3 ON document FOR UPDATE USING (cid = (SELECT cid from category WHERE cname = 'novel')) WITH CHECK (dauthor = current_user); SET SESSION AUTHORIZATION regress_rls_bob; -- Exists... SELECT * FROM document WHERE did = 2; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+----------------- 2 | 11 | 2 | regress_rls_bob | my second novel -- ...so violates actual WITH CHECK OPTION within UPDATE (not INSERT, since -- alternative UPDATE path happens to be taken): INSERT INTO document VALUES (2, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_carol', 'my first novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, dauthor = EXCLUDED.dauthor; ERROR: new row violates row-level security policy for table "document" -- Violates USING qual for UPDATE policy p3. -- -- UPDATE path is taken, but UPDATE fails purely because *existing* row to be -- updated is not a "novel"/cid 11 (row is not leaked, even though we have -- SELECT privileges sufficient to see the row in this instance): INSERT INTO document VALUES (33, 22, 1, 'regress_rls_bob', 'okay science fiction'); -- preparation for next statement INSERT INTO document VALUES (33, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'Some novel, replaces sci-fi') -- takes UPDATE path ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle; ERROR: new row violates row-level security policy (USING expression) for table "document" -- Fine (we UPDATE, since INSERT WCOs and UPDATE security barrier quals + WCOs -- not violated): INSERT INTO document VALUES (2, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'my first novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+---------------- 2 | 11 | 2 | regress_rls_bob | my first novel -- Fine (we INSERT, so "cid = 33" ("technology") isn't evaluated): INSERT INTO document VALUES (78, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'some technology novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33 RETURNING *; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+----------------------- 78 | 11 | 1 | regress_rls_bob | some technology novel -- Fine (same query, but we UPDATE, so "cid = 33", ("technology") is not the -- case in respect of *existing* tuple): INSERT INTO document VALUES (78, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'some technology novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33 RETURNING *; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+----------------------- 78 | 33 | 1 | regress_rls_bob | some technology novel -- Same query a third time, but now fails due to existing tuple finally not -- passing quals: INSERT INTO document VALUES (78, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'some technology novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33 RETURNING *; ERROR: new row violates row-level security policy (USING expression) for table "document" -- Don't fail just because INSERT doesn't satisfy WITH CHECK option that -- originated as a barrier/USING() qual from the UPDATE. Note that the UPDATE -- path *isn't* taken, and so UPDATE-related policy does not apply: INSERT INTO document VALUES (79, (SELECT cid from category WHERE cname = 'technology'), 1, 'regress_rls_bob', 'technology book, can only insert') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; did | cid | dlevel | dauthor | dtitle -----+-----+--------+-----------------+---------------------------------- 79 | 33 | 1 | regress_rls_bob | technology book, can only insert -- But this time, the same statement fails, because the UPDATE path is taken, -- and updating the row just inserted falls afoul of security barrier qual -- (enforced as WCO) -- what we might have updated target tuple to is -- irrelevant, in fact. INSERT INTO document VALUES (79, (SELECT cid from category WHERE cname = 'technology'), 1, 'regress_rls_bob', 'technology book, can only insert') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; ERROR: new row violates row-level security policy (USING expression) for table "document" -- Test default USING qual enforced as WCO SET SESSION AUTHORIZATION regress_rls_alice; DROP POLICY p1 ON document; DROP POLICY p2 ON document; DROP POLICY p3 ON document; CREATE POLICY p3_with_default ON document FOR UPDATE USING (cid = (SELECT cid from category WHERE cname = 'novel')); SET SESSION AUTHORIZATION regress_rls_bob; -- Just because WCO-style enforcement of USING quals occurs with -- existing/target tuple does not mean that the implementation can be allowed -- to fail to also enforce this qual against the final tuple appended to -- relation (since in the absence of an explicit WCO, this is also interpreted -- as an UPDATE/ALL WCO in general). -- -- UPDATE path is taken here (fails due to existing tuple). Note that this is -- not reported as a "USING expression", because it's an RLS UPDATE check that originated as -- a USING qual for the purposes of RLS in general, as opposed to an explicit -- USING qual that is ordinarily a security barrier. We leave it up to the -- UPDATE to make this fail: INSERT INTO document VALUES (79, (SELECT cid from category WHERE cname = 'technology'), 1, 'regress_rls_bob', 'technology book, can only insert') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; ERROR: new row violates row-level security policy for table "document" -- UPDATE path is taken here. Existing tuple passes, since it's cid -- corresponds to "novel", but default USING qual is enforced against -- post-UPDATE tuple too (as always when updating with a policy that lacks an -- explicit WCO), and so this fails: INSERT INTO document VALUES (2, (SELECT cid from category WHERE cname = 'technology'), 1, 'regress_rls_bob', 'my first novel') ON CONFLICT (did) DO UPDATE SET cid = EXCLUDED.cid, dtitle = EXCLUDED.dtitle RETURNING *; ERROR: new row violates row-level security policy for table "document" SET SESSION AUTHORIZATION regress_rls_alice; DROP POLICY p3_with_default ON document; -- -- Test ALL policies with ON CONFLICT DO UPDATE (much the same as existing UPDATE -- tests) -- CREATE POLICY p3_with_all ON document FOR ALL USING (cid = (SELECT cid from category WHERE cname = 'novel')) WITH CHECK (dauthor = current_user); SET SESSION AUTHORIZATION regress_rls_bob; -- Fails, since ALL WCO is enforced in insert path: INSERT INTO document VALUES (80, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_carol', 'my first novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33; ERROR: new row violates row-level security policy for table "document" -- Fails, since ALL policy USING qual is enforced (existing, target tuple is in -- violation, since it has the "manga" cid): INSERT INTO document VALUES (4, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'my first novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle; ERROR: new row violates row-level security policy (USING expression) for table "document" -- Fails, since ALL WCO are enforced: INSERT INTO document VALUES (1, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'my first novel') ON CONFLICT (did) DO UPDATE SET dauthor = 'regress_rls_carol'; ERROR: new row violates row-level security policy for table "document" -- -- ROLE/GROUP -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE z1 (a int, b text); SELECT public.create_hypertable('z1', 'a', chunk_time_interval=>2); create_hypertable ----------------------------- (9,regress_rls_schema,z1,t) CREATE TABLE z2 (a int, b text); SELECT public.create_hypertable('z2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (10,regress_rls_schema,z2,t) GRANT SELECT ON z1,z2 TO regress_rls_group1, regress_rls_group2, regress_rls_bob, regress_rls_carol; INSERT INTO z1 VALUES (1, 'aba'), (2, 'bbb'), (3, 'ccc'), (4, 'dad'); CREATE POLICY p1 ON z1 TO regress_rls_group1 USING (a % 2 = 0); CREATE POLICY p2 ON z1 TO regress_rls_group2 USING (a % 2 = 1); ALTER TABLE z1 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM z1 WHERE f_leak(b); NOTICE: f_leak => bbb NOTICE: f_leak => dad a | b ---+----- 2 | bbb 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM z1 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) PREPARE plancache_test AS SELECT * FROM z1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) PREPARE plancache_test2 AS WITH q AS MATERIALIZED (SELECT * FROM z1 WHERE f_leak(b)) SELECT * FROM q,z2; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test2; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 PREPARE plancache_test4 AS WITH q AS (SELECT * FROM z1 WHERE f_leak(b)) SELECT * FROM q,z2; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test4; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 PREPARE plancache_test6 AS WITH q AS NOT MATERIALIZED (SELECT * FROM z1 WHERE f_leak(b)) SELECT * FROM q,z2; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test6; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 PREPARE plancache_test3 AS WITH q AS MATERIALIZED (SELECT * FROM z2) SELECT * FROM q,z1 WHERE f_leak(z1.b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test3; --- QUERY PLAN --- Nested Loop CTE q -> Seq Scan on z2 -> CTE Scan on q -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) PREPARE plancache_test5 AS WITH q AS (SELECT * FROM z2) SELECT * FROM q,z1 WHERE f_leak(z1.b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test5; --- QUERY PLAN --- Nested Loop -> Seq Scan on z2 -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) PREPARE plancache_test7 AS WITH q AS NOT MATERIALIZED (SELECT * FROM z2) SELECT * FROM q,z1 WHERE f_leak(z1.b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test7; --- QUERY PLAN --- Nested Loop -> Seq Scan on z2 -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) SET ROLE regress_rls_group1; SELECT * FROM z1 WHERE f_leak(b); NOTICE: f_leak => bbb NOTICE: f_leak => dad a | b ---+----- 2 | bbb 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM z1 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test2; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test4; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test3; --- QUERY PLAN --- Nested Loop CTE q -> Seq Scan on z2 -> CTE Scan on q -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test5; --- QUERY PLAN --- Nested Loop -> Seq Scan on z2 -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM z1 WHERE f_leak(b); NOTICE: f_leak => aba NOTICE: f_leak => ccc a | b ---+----- 1 | aba 3 | ccc EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM z1 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test2; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test4; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test3; --- QUERY PLAN --- Nested Loop CTE q -> Seq Scan on z2 -> CTE Scan on q -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test5; --- QUERY PLAN --- Nested Loop -> Seq Scan on z2 -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) SET ROLE regress_rls_group2; SELECT * FROM z1 WHERE f_leak(b); NOTICE: f_leak => aba NOTICE: f_leak => ccc a | b ---+----- 1 | aba 3 | ccc EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM z1 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test2; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test4; --- QUERY PLAN --- Nested Loop CTE q -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) -> CTE Scan on q -> Materialize -> Seq Scan on z2 EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test3; --- QUERY PLAN --- Nested Loop CTE q -> Seq Scan on z2 -> CTE Scan on q -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test5; --- QUERY PLAN --- Nested Loop -> Seq Scan on z2 -> Materialize -> Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 1) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 1) AND f_leak(b)) -- -- Views should follow policy for view owner. -- -- View and Table owner are the same. SET SESSION AUTHORIZATION regress_rls_alice; CREATE VIEW rls_view AS SELECT * FROM z1 WHERE f_leak(b); GRANT SELECT ON rls_view TO regress_rls_bob; -- Query as role that is not owner of view or table. Should return all records. SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rls_view; NOTICE: f_leak => aba NOTICE: f_leak => bbb NOTICE: f_leak => ccc NOTICE: f_leak => dad a | b ---+----- 1 | aba 2 | bbb 3 | ccc 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: f_leak(b) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: f_leak(b) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: f_leak(b) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: f_leak(b) -- Query as view/table owner. Should return all records. SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM rls_view; NOTICE: f_leak => aba NOTICE: f_leak => bbb NOTICE: f_leak => ccc NOTICE: f_leak => dad a | b ---+----- 1 | aba 2 | bbb 3 | ccc 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: f_leak(b) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: f_leak(b) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: f_leak(b) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: f_leak(b) DROP VIEW rls_view; -- View and Table owners are different. SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW rls_view AS SELECT * FROM z1 WHERE f_leak(b); GRANT SELECT ON rls_view TO regress_rls_alice; -- Query as role that is not owner of view but is owner of table. -- Should return records based on view owner policies. SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM rls_view; NOTICE: f_leak => bbb NOTICE: f_leak => dad a | b ---+----- 2 | bbb 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -- Query as role that is not owner of table but is owner of view. -- Should return records based on view owner policies. SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rls_view; NOTICE: f_leak => bbb NOTICE: f_leak => dad a | b ---+----- 2 | bbb 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -- Query as role that is not the owner of the table or view without permissions. SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM rls_view; --fail - permission denied. ERROR: permission denied for view rls_view EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --fail - permission denied. ERROR: permission denied for view rls_view -- Query as role that is not the owner of the table or view with permissions. SET SESSION AUTHORIZATION regress_rls_bob; GRANT SELECT ON rls_view TO regress_rls_carol; SELECT * FROM rls_view; NOTICE: f_leak => bbb NOTICE: f_leak => dad a | b ---+----- 2 | bbb 4 | dad EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --- QUERY PLAN --- Custom Scan (ChunkAppend) on z1 Chunks excluded during startup: 0 -> Seq Scan on z1 z1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_49_chunk z1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_50_chunk z1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_9_51_chunk z1_4 Filter: (((a % 2) = 0) AND f_leak(b)) SET SESSION AUTHORIZATION regress_rls_bob; DROP VIEW rls_view; -- -- Command specific -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE x1 (a int, b text, c text); SELECT public.create_hypertable('x1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (11,regress_rls_schema,x1,t) GRANT ALL ON x1 TO PUBLIC; INSERT INTO x1 VALUES (1, 'abc', 'regress_rls_bob'), (2, 'bcd', 'regress_rls_bob'), (3, 'cde', 'regress_rls_carol'), (4, 'def', 'regress_rls_carol'), (5, 'efg', 'regress_rls_bob'), (6, 'fgh', 'regress_rls_bob'), (7, 'fgh', 'regress_rls_carol'), (8, 'fgh', 'regress_rls_carol'); CREATE POLICY p0 ON x1 FOR ALL USING (c = current_user); CREATE POLICY p1 ON x1 FOR SELECT USING (a % 2 = 0); CREATE POLICY p2 ON x1 FOR INSERT WITH CHECK (a % 2 = 1); CREATE POLICY p3 ON x1 FOR UPDATE USING (a % 2 = 0); CREATE POLICY p4 ON x1 FOR DELETE USING (a < 8); ALTER TABLE x1 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM x1 WHERE f_leak(b) ORDER BY a ASC; NOTICE: f_leak => abc NOTICE: f_leak => bcd NOTICE: f_leak => def NOTICE: f_leak => efg NOTICE: f_leak => fgh NOTICE: f_leak => fgh a | b | c ---+-----+------------------- 1 | abc | regress_rls_bob 2 | bcd | regress_rls_bob 4 | def | regress_rls_carol 5 | efg | regress_rls_bob 6 | fgh | regress_rls_bob 8 | fgh | regress_rls_carol UPDATE x1 SET b = b || '_updt' WHERE f_leak(b) RETURNING *; NOTICE: f_leak => abc NOTICE: f_leak => bcd NOTICE: f_leak => def NOTICE: f_leak => efg NOTICE: f_leak => fgh NOTICE: f_leak => fgh a | b | c ---+----------+------------------- 1 | abc_updt | regress_rls_bob 2 | bcd_updt | regress_rls_bob 4 | def_updt | regress_rls_carol 5 | efg_updt | regress_rls_bob 6 | fgh_updt | regress_rls_bob 8 | fgh_updt | regress_rls_carol SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM x1 WHERE f_leak(b) ORDER BY a ASC; NOTICE: f_leak => cde NOTICE: f_leak => bcd_updt NOTICE: f_leak => def_updt NOTICE: f_leak => fgh NOTICE: f_leak => fgh_updt NOTICE: f_leak => fgh_updt a | b | c ---+----------+------------------- 2 | bcd_updt | regress_rls_bob 3 | cde | regress_rls_carol 4 | def_updt | regress_rls_carol 6 | fgh_updt | regress_rls_bob 7 | fgh | regress_rls_carol 8 | fgh_updt | regress_rls_carol UPDATE x1 SET b = b || '_updt' WHERE f_leak(b) RETURNING *; NOTICE: f_leak => cde NOTICE: f_leak => bcd_updt NOTICE: f_leak => def_updt NOTICE: f_leak => fgh NOTICE: f_leak => fgh_updt NOTICE: f_leak => fgh_updt a | b | c ---+---------------+------------------- 3 | cde_updt | regress_rls_carol 2 | bcd_updt_updt | regress_rls_bob 4 | def_updt_updt | regress_rls_carol 7 | fgh_updt | regress_rls_carol 6 | fgh_updt_updt | regress_rls_bob 8 | fgh_updt_updt | regress_rls_carol DELETE FROM x1 WHERE f_leak(b) RETURNING *; NOTICE: f_leak => cde_updt NOTICE: f_leak => bcd_updt_updt NOTICE: f_leak => def_updt_updt NOTICE: f_leak => fgh_updt NOTICE: f_leak => fgh_updt_updt NOTICE: f_leak => fgh_updt_updt a | b | c ---+---------------+------------------- 3 | cde_updt | regress_rls_carol 2 | bcd_updt_updt | regress_rls_bob 4 | def_updt_updt | regress_rls_carol 7 | fgh_updt | regress_rls_carol 6 | fgh_updt_updt | regress_rls_bob 8 | fgh_updt_updt | regress_rls_carol -- -- Duplicate Policy Names -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE y1 (a int, b text); SELECT public.create_hypertable('y1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (12,regress_rls_schema,y1,t) INSERT INTO y1 VALUES(1,2); CREATE TABLE y2 (a int, b text); SELECT public.create_hypertable('y2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (13,regress_rls_schema,y2,t) GRANT ALL ON y1, y2 TO regress_rls_bob; CREATE POLICY p1 ON y1 FOR ALL USING (a % 2 = 0); CREATE POLICY p2 ON y1 FOR SELECT USING (a > 2); CREATE POLICY p1 ON y1 FOR SELECT USING (a % 2 = 1); --fail ERROR: policy "p1" for table "y1" already exists CREATE POLICY p1 ON y2 FOR ALL USING (a % 2 = 0); --OK ALTER TABLE y1 ENABLE ROW LEVEL SECURITY; ALTER TABLE y2 ENABLE ROW LEVEL SECURITY; -- -- Expression structure with SBV -- -- Create view as table owner. RLS should NOT be applied. SET SESSION AUTHORIZATION regress_rls_alice; CREATE VIEW rls_sbv WITH (security_barrier) AS SELECT * FROM y1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_sbv WHERE (a = 1); --- QUERY PLAN --- Custom Scan (ChunkAppend) on y1 Chunks excluded during startup: 0 -> Seq Scan on y1 y1_1 Filter: (f_leak(b) AND (a = 1)) -> Index Scan using _hyper_12_57_chunk_y1_a_idx on _hyper_12_57_chunk y1_2 Index Cond: (a = 1) Filter: f_leak(b) DROP VIEW rls_sbv; -- Create view as role that does not own table. RLS should be applied. SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW rls_sbv WITH (security_barrier) AS SELECT * FROM y1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_sbv WHERE (a = 1); --- QUERY PLAN --- Custom Scan (ChunkAppend) on y1 Chunks excluded during startup: 0 -> Seq Scan on y1 y1_1 Filter: ((a = 1) AND ((a > 2) OR ((a % 2) = 0)) AND f_leak(b)) -> Index Scan using _hyper_12_57_chunk_y1_a_idx on _hyper_12_57_chunk y1_2 Index Cond: (a = 1) Filter: (((a > 2) OR ((a % 2) = 0)) AND f_leak(b)) DROP VIEW rls_sbv; -- -- Expression structure -- SET SESSION AUTHORIZATION regress_rls_alice; INSERT INTO y2 (SELECT x, md5(x::text) FROM generate_series(0,20) x); CREATE POLICY p2 ON y2 USING (a % 3 = 0); CREATE POLICY p3 ON y2 USING (a % 4 = 0); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM y2 WHERE f_leak(b); NOTICE: f_leak => cfcd208495d565ef66e7dff9f98764da NOTICE: f_leak => c81e728d9d4c2f636f067f89cc14862c NOTICE: f_leak => eccbc87e4b5ce2fe28308fd9f2a7baf3 NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc NOTICE: f_leak => c9f0f895fb98ab9159f51fd0297e236d NOTICE: f_leak => 45c48cce2e2d7fbdea1afc51c7c6ad26 NOTICE: f_leak => d3d9446802a44259755d38e6d163e820 NOTICE: f_leak => c20ad4d76fe97759aa27a0c99bff6710 NOTICE: f_leak => aab3238922bcc25a6f606eb525ffdc56 NOTICE: f_leak => 9bf31c7ff062936a96d3c8bd1f8f2ff3 NOTICE: f_leak => c74d97b01eae257e44aa9d5bade97baf NOTICE: f_leak => 6f4922f45568161a8cdf4ad2299f6d23 NOTICE: f_leak => 98f13708210194c475687be6106a3b84 a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 3 | eccbc87e4b5ce2fe28308fd9f2a7baf3 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 9 | 45c48cce2e2d7fbdea1afc51c7c6ad26 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 15 | 9bf31c7ff062936a96d3c8bd1f8f2ff3 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM y2 WHERE f_leak(b); --- QUERY PLAN --- Custom Scan (ChunkAppend) on y2 Chunks excluded during startup: 0 -> Seq Scan on y2 y2_1 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_58_chunk y2_2 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_59_chunk y2_3 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_60_chunk y2_4 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_61_chunk y2_5 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_62_chunk y2_6 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_63_chunk y2_7 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_64_chunk y2_8 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_65_chunk y2_9 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_66_chunk y2_10 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_67_chunk y2_11 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_68_chunk y2_12 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -- -- Qual push-down of leaky functions, when not referring to table -- SELECT * FROM y2 WHERE f_leak('abc'); NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc NOTICE: f_leak => abc a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 3 | eccbc87e4b5ce2fe28308fd9f2a7baf3 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 9 | 45c48cce2e2d7fbdea1afc51c7c6ad26 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 15 | 9bf31c7ff062936a96d3c8bd1f8f2ff3 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM y2 WHERE f_leak('abc'); --- QUERY PLAN --- Custom Scan (ChunkAppend) on y2 Chunks excluded during startup: 0 -> Seq Scan on y2 y2_1 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_58_chunk y2_2 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_59_chunk y2_3 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_60_chunk y2_4 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_61_chunk y2_5 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_62_chunk y2_6 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_63_chunk y2_7 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_64_chunk y2_8 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_65_chunk y2_9 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_66_chunk y2_10 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_67_chunk y2_11 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) -> Seq Scan on _hyper_13_68_chunk y2_12 Filter: (f_leak('abc'::text) AND (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0))) CREATE TABLE test_qual_pushdown ( abc text ); INSERT INTO test_qual_pushdown VALUES ('abc'),('def'); SELECT * FROM y2 JOIN test_qual_pushdown ON (b = abc) WHERE f_leak(abc); NOTICE: f_leak => abc NOTICE: f_leak => def a | b | abc ---+---+----- EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM y2 JOIN test_qual_pushdown ON (b = abc) WHERE f_leak(abc); --- QUERY PLAN --- Hash Join Hash Cond: (y2.b = test_qual_pushdown.abc) -> Append -> Seq Scan on y2 y2_1 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_58_chunk y2_2 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_59_chunk y2_3 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_60_chunk y2_4 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_61_chunk y2_5 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_62_chunk y2_6 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_63_chunk y2_7 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_64_chunk y2_8 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_65_chunk y2_9 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_66_chunk y2_10 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_67_chunk y2_11 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Seq Scan on _hyper_13_68_chunk y2_12 Filter: (((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) -> Hash -> Seq Scan on test_qual_pushdown Filter: f_leak(abc) SELECT * FROM y2 JOIN test_qual_pushdown ON (b = abc) WHERE f_leak(b); NOTICE: f_leak => cfcd208495d565ef66e7dff9f98764da NOTICE: f_leak => c81e728d9d4c2f636f067f89cc14862c NOTICE: f_leak => eccbc87e4b5ce2fe28308fd9f2a7baf3 NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc NOTICE: f_leak => c9f0f895fb98ab9159f51fd0297e236d NOTICE: f_leak => 45c48cce2e2d7fbdea1afc51c7c6ad26 NOTICE: f_leak => d3d9446802a44259755d38e6d163e820 NOTICE: f_leak => c20ad4d76fe97759aa27a0c99bff6710 NOTICE: f_leak => aab3238922bcc25a6f606eb525ffdc56 NOTICE: f_leak => 9bf31c7ff062936a96d3c8bd1f8f2ff3 NOTICE: f_leak => c74d97b01eae257e44aa9d5bade97baf NOTICE: f_leak => 6f4922f45568161a8cdf4ad2299f6d23 NOTICE: f_leak => 98f13708210194c475687be6106a3b84 a | b | abc ---+---+----- EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM y2 JOIN test_qual_pushdown ON (b = abc) WHERE f_leak(b); --- QUERY PLAN --- Hash Join Hash Cond: (test_qual_pushdown.abc = y2.b) -> Seq Scan on test_qual_pushdown -> Hash -> Custom Scan (ChunkAppend) on y2 Chunks excluded during startup: 0 -> Seq Scan on y2 y2_1 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_58_chunk y2_2 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_59_chunk y2_3 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_60_chunk y2_4 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_61_chunk y2_5 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_62_chunk y2_6 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_63_chunk y2_7 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_64_chunk y2_8 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_65_chunk y2_9 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_66_chunk y2_10 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_67_chunk y2_11 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) -> Seq Scan on _hyper_13_68_chunk y2_12 Filter: ((((a % 4) = 0) OR ((a % 3) = 0) OR ((a % 2) = 0)) AND f_leak(b)) DROP TABLE test_qual_pushdown; -- -- Plancache invalidate on user change. -- RESET SESSION AUTHORIZATION; \set VERBOSITY terse \\ -- suppress cascade details DROP TABLE t1 CASCADE; NOTICE: drop cascades to 2 other objects \set VERBOSITY default CREATE TABLE t1 (a integer); SELECT public.create_hypertable('t1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (14,regress_rls_schema,t1,t) GRANT SELECT ON t1 TO regress_rls_bob, regress_rls_carol; CREATE POLICY p1 ON t1 TO regress_rls_bob USING ((a % 2) = 0); CREATE POLICY p2 ON t1 TO regress_rls_carol USING ((a % 4) = 0); ALTER TABLE t1 ENABLE ROW LEVEL SECURITY; -- Prepare as regress_rls_bob SET ROLE regress_rls_bob; PREPARE role_inval AS SELECT * FROM t1; -- Check plan EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE role_inval; --- QUERY PLAN --- Seq Scan on t1 Filter: ((a % 2) = 0) -- Change to regress_rls_carol SET ROLE regress_rls_carol; -- Check plan- should be different EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE role_inval; --- QUERY PLAN --- Seq Scan on t1 Filter: ((a % 4) = 0) -- Change back to regress_rls_bob SET ROLE regress_rls_bob; -- Check plan- should be back to original EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE role_inval; --- QUERY PLAN --- Seq Scan on t1 Filter: ((a % 2) = 0) -- -- CTE and RLS -- RESET SESSION AUTHORIZATION; DROP TABLE t1 CASCADE; CREATE TABLE t1 (a integer, b text); SELECT public.create_hypertable('t1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (15,regress_rls_schema,t1,t) CREATE POLICY p1 ON t1 USING (a % 2 = 0); ALTER TABLE t1 ENABLE ROW LEVEL SECURITY; GRANT ALL ON t1 TO regress_rls_bob; INSERT INTO t1 (SELECT x, md5(x::text) FROM generate_series(0,20) x); SET SESSION AUTHORIZATION regress_rls_bob; WITH cte1 AS (SELECT * FROM t1 WHERE f_leak(b)) SELECT * FROM cte1; NOTICE: f_leak => cfcd208495d565ef66e7dff9f98764da NOTICE: f_leak => c81e728d9d4c2f636f067f89cc14862c NOTICE: f_leak => a87ff679a2f3e71d9181a67b7542122c NOTICE: f_leak => 1679091c5a880faf6fb5e6087eb1b2dc NOTICE: f_leak => c9f0f895fb98ab9159f51fd0297e236d NOTICE: f_leak => d3d9446802a44259755d38e6d163e820 NOTICE: f_leak => c20ad4d76fe97759aa27a0c99bff6710 NOTICE: f_leak => aab3238922bcc25a6f606eb525ffdc56 NOTICE: f_leak => c74d97b01eae257e44aa9d5bade97baf NOTICE: f_leak => 6f4922f45568161a8cdf4ad2299f6d23 NOTICE: f_leak => 98f13708210194c475687be6106a3b84 a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 EXPLAIN (BUFFERS OFF, COSTS OFF) WITH cte1 AS (SELECT * FROM t1 WHERE f_leak(b)) SELECT * FROM cte1; --- QUERY PLAN --- CTE Scan on cte1 CTE cte1 -> Custom Scan (ChunkAppend) on t1 Chunks excluded during startup: 0 -> Seq Scan on t1 t1_1 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_69_chunk t1_2 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_70_chunk t1_3 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_71_chunk t1_4 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_72_chunk t1_5 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_73_chunk t1_6 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_74_chunk t1_7 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_75_chunk t1_8 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_76_chunk t1_9 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_77_chunk t1_10 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_78_chunk t1_11 Filter: (((a % 2) = 0) AND f_leak(b)) -> Seq Scan on _hyper_15_79_chunk t1_12 Filter: (((a % 2) = 0) AND f_leak(b)) WITH cte1 AS (UPDATE t1 SET a = a + 1 RETURNING *) SELECT * FROM cte1; --fail ERROR: new row violates row-level security policy for table "t1" WITH cte1 AS (UPDATE t1 SET a = a RETURNING *) SELECT * FROM cte1; --ok a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 WITH cte1 AS (INSERT INTO t1 VALUES (21, 'Fail') RETURNING *) SELECT * FROM cte1; --fail ERROR: new row violates row-level security policy for table "t1" WITH cte1 AS (INSERT INTO t1 VALUES (20, 'Success') RETURNING *) SELECT * FROM cte1; --ok a | b ----+--------- 20 | Success -- -- Rename Policy -- RESET SESSION AUTHORIZATION; ALTER POLICY p1 ON t1 RENAME TO p1; --fail ERROR: policy "p1" for table "t1" already exists SELECT polname, relname FROM pg_policy pol JOIN pg_class pc ON (pc.oid = pol.polrelid) WHERE relname = 't1'; polname | relname ---------+--------- p1 | t1 ALTER POLICY p1 ON t1 RENAME TO p2; --ok SELECT polname, relname FROM pg_policy pol JOIN pg_class pc ON (pc.oid = pol.polrelid) WHERE relname = 't1'; polname | relname ---------+--------- p2 | t1 -- -- Check INSERT SELECT -- SET SESSION AUTHORIZATION regress_rls_bob; CREATE TABLE t2 (a integer, b text); SELECT public.create_hypertable('t2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (16,regress_rls_schema,t2,t) INSERT INTO t2 (SELECT * FROM t1); EXPLAIN (BUFFERS OFF, COSTS OFF) INSERT INTO t2 (SELECT * FROM t1); --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Insert on t2 -> Append -> Seq Scan on t1 t1_1 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_69_chunk t1_2 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_70_chunk t1_3 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_71_chunk t1_4 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_72_chunk t1_5 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_73_chunk t1_6 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_74_chunk t1_7 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_75_chunk t1_8 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_76_chunk t1_9 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_77_chunk t1_10 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_78_chunk t1_11 Filter: ((a % 2) = 0) -> Seq Scan on _hyper_15_79_chunk t1_12 Filter: ((a % 2) = 0) SELECT * FROM t2; a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 20 | Success EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t2; --- QUERY PLAN --- Append -> Seq Scan on t2 t2_1 -> Seq Scan on _hyper_16_80_chunk t2_2 -> Seq Scan on _hyper_16_81_chunk t2_3 -> Seq Scan on _hyper_16_82_chunk t2_4 -> Seq Scan on _hyper_16_83_chunk t2_5 -> Seq Scan on _hyper_16_84_chunk t2_6 -> Seq Scan on _hyper_16_85_chunk t2_7 -> Seq Scan on _hyper_16_86_chunk t2_8 -> Seq Scan on _hyper_16_87_chunk t2_9 -> Seq Scan on _hyper_16_88_chunk t2_10 -> Seq Scan on _hyper_16_89_chunk t2_11 -> Seq Scan on _hyper_16_90_chunk t2_12 CREATE TABLE t3 AS SELECT * FROM t1; SELECT public.create_hypertable('t2', 'a', chunk_time_interval=>2); ERROR: table "t2" is already a hypertable SELECT * FROM t3; a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 20 | Success SELECT * INTO t4 FROM t1; SELECT * FROM t4; a | b ----+---------------------------------- 0 | cfcd208495d565ef66e7dff9f98764da 2 | c81e728d9d4c2f636f067f89cc14862c 4 | a87ff679a2f3e71d9181a67b7542122c 6 | 1679091c5a880faf6fb5e6087eb1b2dc 8 | c9f0f895fb98ab9159f51fd0297e236d 10 | d3d9446802a44259755d38e6d163e820 12 | c20ad4d76fe97759aa27a0c99bff6710 14 | aab3238922bcc25a6f606eb525ffdc56 16 | c74d97b01eae257e44aa9d5bade97baf 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 20 | Success -- -- RLS with JOIN -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE blog (id integer, author text, post text); SELECT public.create_hypertable('blog', 'id', chunk_time_interval=>2); create_hypertable -------------------------------- (17,regress_rls_schema,blog,t) CREATE TABLE comment (blog_id integer, message text); SELECT public.create_hypertable('comment', 'blog_id', chunk_time_interval=>2); create_hypertable ----------------------------------- (18,regress_rls_schema,comment,t) GRANT ALL ON blog, comment TO regress_rls_bob; CREATE POLICY blog_1 ON blog USING (id % 2 = 0); ALTER TABLE blog ENABLE ROW LEVEL SECURITY; INSERT INTO blog VALUES (1, 'alice', 'blog #1'), (2, 'bob', 'blog #1'), (3, 'alice', 'blog #2'), (4, 'alice', 'blog #3'), (5, 'john', 'blog #1'); INSERT INTO comment VALUES (1, 'cool blog'), (1, 'fun blog'), (3, 'crazy blog'), (5, 'what?'), (4, 'insane!'), (2, 'who did it?'); SET SESSION AUTHORIZATION regress_rls_bob; -- Check RLS JOIN with Non-RLS. SELECT id, author, message FROM blog JOIN comment ON id = blog_id; id | author | message ----+--------+------------- 2 | bob | who did it? 4 | alice | insane! -- Check Non-RLS JOIN with RLS. SELECT id, author, message FROM comment JOIN blog ON id = blog_id; id | author | message ----+--------+------------- 2 | bob | who did it? 4 | alice | insane! SET SESSION AUTHORIZATION regress_rls_alice; CREATE POLICY comment_1 ON comment USING (blog_id < 4); ALTER TABLE comment ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; -- Check RLS JOIN RLS SELECT id, author, message FROM blog JOIN comment ON id = blog_id; id | author | message ----+--------+------------- 2 | bob | who did it? SELECT id, author, message FROM comment JOIN blog ON id = blog_id; id | author | message ----+--------+------------- 2 | bob | who did it? SET SESSION AUTHORIZATION regress_rls_alice; DROP TABLE blog; DROP TABLE comment; -- -- Default Deny Policy -- RESET SESSION AUTHORIZATION; DROP POLICY p2 ON t1; ALTER TABLE t1 OWNER TO regress_rls_alice; -- Check that default deny does not apply to superuser. RESET SESSION AUTHORIZATION; SELECT * FROM t1; a | b ----+---------------------------------- 1 | c4ca4238a0b923820dcc509a6f75849b 0 | cfcd208495d565ef66e7dff9f98764da 3 | eccbc87e4b5ce2fe28308fd9f2a7baf3 2 | c81e728d9d4c2f636f067f89cc14862c 5 | e4da3b7fbbce2345d7772b0674a318d5 4 | a87ff679a2f3e71d9181a67b7542122c 7 | 8f14e45fceea167a5a36dedd4bea2543 6 | 1679091c5a880faf6fb5e6087eb1b2dc 9 | 45c48cce2e2d7fbdea1afc51c7c6ad26 8 | c9f0f895fb98ab9159f51fd0297e236d 11 | 6512bd43d9caa6e02c990b0a82652dca 10 | d3d9446802a44259755d38e6d163e820 13 | c51ce410c124a10e0db5e4b97fc2af39 12 | c20ad4d76fe97759aa27a0c99bff6710 15 | 9bf31c7ff062936a96d3c8bd1f8f2ff3 14 | aab3238922bcc25a6f606eb525ffdc56 17 | 70efdf2ec9b086079795c442636b55fb 16 | c74d97b01eae257e44aa9d5bade97baf 19 | 1f0e3dad99908345f7439f8ffabdffc4 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 20 | Success EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 -> Seq Scan on _hyper_15_69_chunk t1_2 -> Seq Scan on _hyper_15_70_chunk t1_3 -> Seq Scan on _hyper_15_71_chunk t1_4 -> Seq Scan on _hyper_15_72_chunk t1_5 -> Seq Scan on _hyper_15_73_chunk t1_6 -> Seq Scan on _hyper_15_74_chunk t1_7 -> Seq Scan on _hyper_15_75_chunk t1_8 -> Seq Scan on _hyper_15_76_chunk t1_9 -> Seq Scan on _hyper_15_77_chunk t1_10 -> Seq Scan on _hyper_15_78_chunk t1_11 -> Seq Scan on _hyper_15_79_chunk t1_12 -- Check that default deny does not apply to table owner. SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM t1; a | b ----+---------------------------------- 1 | c4ca4238a0b923820dcc509a6f75849b 0 | cfcd208495d565ef66e7dff9f98764da 3 | eccbc87e4b5ce2fe28308fd9f2a7baf3 2 | c81e728d9d4c2f636f067f89cc14862c 5 | e4da3b7fbbce2345d7772b0674a318d5 4 | a87ff679a2f3e71d9181a67b7542122c 7 | 8f14e45fceea167a5a36dedd4bea2543 6 | 1679091c5a880faf6fb5e6087eb1b2dc 9 | 45c48cce2e2d7fbdea1afc51c7c6ad26 8 | c9f0f895fb98ab9159f51fd0297e236d 11 | 6512bd43d9caa6e02c990b0a82652dca 10 | d3d9446802a44259755d38e6d163e820 13 | c51ce410c124a10e0db5e4b97fc2af39 12 | c20ad4d76fe97759aa27a0c99bff6710 15 | 9bf31c7ff062936a96d3c8bd1f8f2ff3 14 | aab3238922bcc25a6f606eb525ffdc56 17 | 70efdf2ec9b086079795c442636b55fb 16 | c74d97b01eae257e44aa9d5bade97baf 19 | 1f0e3dad99908345f7439f8ffabdffc4 18 | 6f4922f45568161a8cdf4ad2299f6d23 20 | 98f13708210194c475687be6106a3b84 20 | Success EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; --- QUERY PLAN --- Append -> Seq Scan on t1 t1_1 -> Seq Scan on _hyper_15_69_chunk t1_2 -> Seq Scan on _hyper_15_70_chunk t1_3 -> Seq Scan on _hyper_15_71_chunk t1_4 -> Seq Scan on _hyper_15_72_chunk t1_5 -> Seq Scan on _hyper_15_73_chunk t1_6 -> Seq Scan on _hyper_15_74_chunk t1_7 -> Seq Scan on _hyper_15_75_chunk t1_8 -> Seq Scan on _hyper_15_76_chunk t1_9 -> Seq Scan on _hyper_15_77_chunk t1_10 -> Seq Scan on _hyper_15_78_chunk t1_11 -> Seq Scan on _hyper_15_79_chunk t1_12 -- Check that default deny applies to non-owner/non-superuser when RLS on. SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; SELECT * FROM t1; a | b ---+--- EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; --- QUERY PLAN --- Result One-Time Filter: false SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM t1; a | b ---+--- EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; --- QUERY PLAN --- Result One-Time Filter: false -- -- COPY TO/FROM -- RESET SESSION AUTHORIZATION; DROP TABLE copy_t CASCADE; ERROR: table "copy_t" does not exist CREATE TABLE copy_t (a integer, b text); SELECT public.create_hypertable('copy_t', 'a', chunk_time_interval=>2); create_hypertable ---------------------------------- (19,regress_rls_schema,copy_t,t) CREATE POLICY p1 ON copy_t USING (a % 2 = 0); ALTER TABLE copy_t ENABLE ROW LEVEL SECURITY; GRANT ALL ON copy_t TO regress_rls_bob, regress_rls_exempt_user; INSERT INTO copy_t (SELECT x, md5(x::text) FROM generate_series(0,10) x); -- Check COPY TO as Superuser/owner. RESET SESSION AUTHORIZATION; SET row_security TO OFF; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; 0,cfcd208495d565ef66e7dff9f98764da 1,c4ca4238a0b923820dcc509a6f75849b 2,c81e728d9d4c2f636f067f89cc14862c 3,eccbc87e4b5ce2fe28308fd9f2a7baf3 4,a87ff679a2f3e71d9181a67b7542122c 5,e4da3b7fbbce2345d7772b0674a318d5 6,1679091c5a880faf6fb5e6087eb1b2dc 7,8f14e45fceea167a5a36dedd4bea2543 8,c9f0f895fb98ab9159f51fd0297e236d 9,45c48cce2e2d7fbdea1afc51c7c6ad26 10,d3d9446802a44259755d38e6d163e820 SET row_security TO ON; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; 0,cfcd208495d565ef66e7dff9f98764da 1,c4ca4238a0b923820dcc509a6f75849b 2,c81e728d9d4c2f636f067f89cc14862c 3,eccbc87e4b5ce2fe28308fd9f2a7baf3 4,a87ff679a2f3e71d9181a67b7542122c 5,e4da3b7fbbce2345d7772b0674a318d5 6,1679091c5a880faf6fb5e6087eb1b2dc 7,8f14e45fceea167a5a36dedd4bea2543 8,c9f0f895fb98ab9159f51fd0297e236d 9,45c48cce2e2d7fbdea1afc51c7c6ad26 10,d3d9446802a44259755d38e6d163e820 -- Check COPY TO as user with permissions. SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO OFF; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --fail - would be affected by RLS ERROR: query would be affected by row-level security policy for table "copy_t" SET row_security TO ON; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --ok 0,cfcd208495d565ef66e7dff9f98764da 2,c81e728d9d4c2f636f067f89cc14862c 4,a87ff679a2f3e71d9181a67b7542122c 6,1679091c5a880faf6fb5e6087eb1b2dc 8,c9f0f895fb98ab9159f51fd0297e236d 10,d3d9446802a44259755d38e6d163e820 -- Check COPY TO as user with permissions and BYPASSRLS SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --ok 0,cfcd208495d565ef66e7dff9f98764da 1,c4ca4238a0b923820dcc509a6f75849b 2,c81e728d9d4c2f636f067f89cc14862c 3,eccbc87e4b5ce2fe28308fd9f2a7baf3 4,a87ff679a2f3e71d9181a67b7542122c 5,e4da3b7fbbce2345d7772b0674a318d5 6,1679091c5a880faf6fb5e6087eb1b2dc 7,8f14e45fceea167a5a36dedd4bea2543 8,c9f0f895fb98ab9159f51fd0297e236d 9,45c48cce2e2d7fbdea1afc51c7c6ad26 10,d3d9446802a44259755d38e6d163e820 SET row_security TO ON; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --ok 0,cfcd208495d565ef66e7dff9f98764da 1,c4ca4238a0b923820dcc509a6f75849b 2,c81e728d9d4c2f636f067f89cc14862c 3,eccbc87e4b5ce2fe28308fd9f2a7baf3 4,a87ff679a2f3e71d9181a67b7542122c 5,e4da3b7fbbce2345d7772b0674a318d5 6,1679091c5a880faf6fb5e6087eb1b2dc 7,8f14e45fceea167a5a36dedd4bea2543 8,c9f0f895fb98ab9159f51fd0297e236d 9,45c48cce2e2d7fbdea1afc51c7c6ad26 10,d3d9446802a44259755d38e6d163e820 -- Check COPY TO as user without permissions. SET row_security TO OFF; SET SESSION AUTHORIZATION regress_rls_carol; SET row_security TO OFF; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --fail - would be affected by RLS ERROR: query would be affected by row-level security policy for table "copy_t" SET row_security TO ON; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --fail - permission denied ERROR: permission denied for table copy_t -- Check COPY relation TO; keep it just one row to avoid reordering issues RESET SESSION AUTHORIZATION; SET row_security TO ON; CREATE TABLE copy_rel_to (a integer, b text); SELECT public.create_hypertable('copy_rel_to', 'a', chunk_time_interval=>2); create_hypertable --------------------------------------- (20,regress_rls_schema,copy_rel_to,t) CREATE POLICY p1 ON copy_rel_to USING (a % 2 = 0); ALTER TABLE copy_rel_to ENABLE ROW LEVEL SECURITY; GRANT ALL ON copy_rel_to TO regress_rls_bob, regress_rls_exempt_user; INSERT INTO copy_rel_to VALUES (1, md5('1')); -- Check COPY TO as Superuser/owner. RESET SESSION AUTHORIZATION; SET row_security TO OFF; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; 1,c4ca4238a0b923820dcc509a6f75849b SET row_security TO ON; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; 1,c4ca4238a0b923820dcc509a6f75849b -- Check COPY TO as user with permissions. SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO OFF; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --fail - would be affected by RLS ERROR: query would be affected by row-level security policy for table "copy_rel_to" SET row_security TO ON; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --ok -- Check COPY TO as user with permissions and BYPASSRLS SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --ok 1,c4ca4238a0b923820dcc509a6f75849b SET row_security TO ON; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --ok 1,c4ca4238a0b923820dcc509a6f75849b -- Check COPY TO as user without permissions. SET row_security TO OFF; SET SESSION AUTHORIZATION regress_rls_carol; SET row_security TO OFF; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --fail - permission denied ERROR: query would be affected by row-level security policy for table "copy_rel_to" SET row_security TO ON; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --fail - permission denied ERROR: permission denied for table copy_rel_to -- Check COPY FROM as Superuser/owner. RESET SESSION AUTHORIZATION; SET row_security TO OFF; COPY copy_t FROM STDIN; --ok SET row_security TO ON; COPY copy_t FROM STDIN; --ok -- Check COPY FROM as user with permissions. SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO OFF; COPY copy_t FROM STDIN; --fail - would be affected by RLS. ERROR: query would be affected by row-level security policy for table "copy_t" SET row_security TO ON; COPY copy_t FROM STDIN; --fail - COPY FROM not supported by RLS. ERROR: COPY FROM not supported with row-level security HINT: Use INSERT statements instead. -- Check COPY FROM as user with permissions and BYPASSRLS SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO ON; COPY copy_t FROM STDIN; --ok -- Check COPY FROM as user without permissions. SET SESSION AUTHORIZATION regress_rls_carol; SET row_security TO OFF; COPY copy_t FROM STDIN; --fail - permission denied. ERROR: permission denied for table copy_t SET row_security TO ON; COPY copy_t FROM STDIN; --fail - permission denied. ERROR: permission denied for table copy_t RESET SESSION AUTHORIZATION; DROP TABLE copy_t; DROP TABLE copy_rel_to CASCADE; -- Check WHERE CURRENT OF SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE current_check (currentid int, payload text, rlsuser text); SELECT public.create_hypertable('current_check', 'currentid', chunk_time_interval=>10); create_hypertable ----------------------------------------- (21,regress_rls_schema,current_check,t) GRANT ALL ON current_check TO PUBLIC; INSERT INTO current_check VALUES (1, 'abc', 'regress_rls_bob'), (2, 'bcd', 'regress_rls_bob'), (3, 'cde', 'regress_rls_bob'), (4, 'def', 'regress_rls_bob'); CREATE POLICY p1 ON current_check FOR SELECT USING (currentid % 2 = 0); CREATE POLICY p2 ON current_check FOR DELETE USING (currentid = 4 AND rlsuser = current_user); CREATE POLICY p3 ON current_check FOR UPDATE USING (currentid = 4) WITH CHECK (rlsuser = current_user); ALTER TABLE current_check ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; -- Can SELECT even rows SELECT * FROM current_check; currentid | payload | rlsuser -----------+---------+----------------- 2 | bcd | regress_rls_bob 4 | def | regress_rls_bob -- Cannot UPDATE row 2 UPDATE current_check SET payload = payload || '_new' WHERE currentid = 2 RETURNING *; currentid | payload | rlsuser -----------+---------+--------- BEGIN; -- WHERE CURRENT OF does not work with custom scan nodes -- so we have to disable chunk append here SET timescaledb.enable_chunk_append TO false; DECLARE current_check_cursor SCROLL CURSOR FOR SELECT * FROM current_check; -- Returns rows that can be seen according to SELECT policy, like plain SELECT -- above (even rows) FETCH ABSOLUTE 1 FROM current_check_cursor; currentid | payload | rlsuser -----------+---------+----------------- 2 | bcd | regress_rls_bob -- Still cannot UPDATE row 2 through cursor UPDATE current_check SET payload = payload || '_new' WHERE CURRENT OF current_check_cursor RETURNING *; currentid | payload | rlsuser -----------+---------+--------- -- Can update row 4 through cursor, which is the next visible row FETCH RELATIVE 1 FROM current_check_cursor; currentid | payload | rlsuser -----------+---------+----------------- 4 | def | regress_rls_bob UPDATE current_check SET payload = payload || '_new' WHERE CURRENT OF current_check_cursor RETURNING *; currentid | payload | rlsuser -----------+---------+----------------- 4 | def_new | regress_rls_bob SELECT * FROM current_check; currentid | payload | rlsuser -----------+---------+----------------- 2 | bcd | regress_rls_bob 4 | def_new | regress_rls_bob -- Plan should be a subquery TID scan EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE current_check SET payload = payload WHERE CURRENT OF current_check_cursor; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Update on current_check Update on _hyper_21_104_chunk current_check_1 -> Tid Scan on _hyper_21_104_chunk current_check_1 TID Cond: CURRENT OF current_check_cursor Filter: ((currentid = 4) AND ((currentid % 2) = 0)) -- Similarly can only delete row 4 FETCH ABSOLUTE 1 FROM current_check_cursor; currentid | payload | rlsuser -----------+---------+----------------- 2 | bcd | regress_rls_bob DELETE FROM current_check WHERE CURRENT OF current_check_cursor RETURNING *; currentid | payload | rlsuser -----------+---------+--------- FETCH RELATIVE 1 FROM current_check_cursor; currentid | payload | rlsuser -----------+---------+----------------- 4 | def | regress_rls_bob DELETE FROM current_check WHERE CURRENT OF current_check_cursor RETURNING *; currentid | payload | rlsuser -----------+---------+----------------- 4 | def_new | regress_rls_bob SELECT * FROM current_check; currentid | payload | rlsuser -----------+---------+----------------- 2 | bcd | regress_rls_bob RESET timescaledb.enable_chunk_append; COMMIT; -- -- check pg_stats view filtering -- SET row_security TO ON; SET SESSION AUTHORIZATION regress_rls_alice; ANALYZE current_check; -- Stats visible SELECT row_security_active('current_check'); row_security_active --------------------- f SELECT attname, most_common_vals FROM pg_stats WHERE tablename = 'current_check' ORDER BY 1; attname | most_common_vals -----------+------------------- currentid | payload | rlsuser | {regress_rls_bob} SET SESSION AUTHORIZATION regress_rls_bob; -- Stats not visible SELECT row_security_active('current_check'); row_security_active --------------------- t SELECT attname, most_common_vals FROM pg_stats WHERE tablename = 'current_check' ORDER BY 1; attname | most_common_vals ---------+------------------ -- -- Collation support -- BEGIN; CREATE TABLE coll_t (c) AS VALUES ('bar'::text); CREATE POLICY coll_p ON coll_t USING (c < ('foo'::text COLLATE "C")); ALTER TABLE coll_t ENABLE ROW LEVEL SECURITY; GRANT SELECT ON coll_t TO regress_rls_alice; SELECT (string_to_array(polqual, ':'))[7] AS inputcollid FROM pg_policy WHERE polrelid = 'coll_t'::regclass; inputcollid ------------------ inputcollid 950 SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM coll_t; c ----- bar ROLLBACK; -- -- Shared Object Dependencies -- RESET SESSION AUTHORIZATION; BEGIN; CREATE ROLE regress_rls_eve; CREATE ROLE regress_rls_frank; CREATE TABLE tbl1 (c) AS VALUES ('bar'::text); GRANT SELECT ON TABLE tbl1 TO regress_rls_eve; CREATE POLICY P ON tbl1 TO regress_rls_eve, regress_rls_frank USING (true); SELECT refclassid::regclass, deptype FROM pg_depend WHERE classid = 'pg_policy'::regclass AND refobjid = 'tbl1'::regclass; refclassid | deptype ------------+--------- pg_class | a SELECT refclassid::regclass, deptype FROM pg_shdepend WHERE classid = 'pg_policy'::regclass AND refobjid IN ('regress_rls_eve'::regrole, 'regress_rls_frank'::regrole); refclassid | deptype ------------+--------- pg_authid | r pg_authid | r SAVEPOINT q; DROP ROLE regress_rls_eve; --fails due to dependency on POLICY p ERROR: role "regress_rls_eve" cannot be dropped because some objects depend on it DETAIL: privileges for table tbl1 target of policy p on table tbl1 ROLLBACK TO q; ALTER POLICY p ON tbl1 TO regress_rls_frank USING (true); SAVEPOINT q; DROP ROLE regress_rls_eve; --fails due to dependency on GRANT SELECT ERROR: role "regress_rls_eve" cannot be dropped because some objects depend on it DETAIL: privileges for table tbl1 ROLLBACK TO q; REVOKE ALL ON TABLE tbl1 FROM regress_rls_eve; SAVEPOINT q; DROP ROLE regress_rls_eve; --succeeds ROLLBACK TO q; SAVEPOINT q; DROP ROLE regress_rls_frank; --fails due to dependency on POLICY p ERROR: role "regress_rls_frank" cannot be dropped because some objects depend on it DETAIL: target of policy p on table tbl1 ROLLBACK TO q; DROP POLICY p ON tbl1; SAVEPOINT q; DROP ROLE regress_rls_frank; -- succeeds ROLLBACK TO q; ROLLBACK; -- cleanup -- -- Converting table to view -- BEGIN; CREATE TABLE t (c int); SELECT public.create_hypertable('t', 'c', chunk_time_interval=>2); create_hypertable ----------------------------- (22,regress_rls_schema,t,t) CREATE POLICY p ON t USING (c % 2 = 1); ALTER TABLE t ENABLE ROW LEVEL SECURITY; SAVEPOINT q; CREATE RULE "_RETURN" AS ON SELECT TO t DO INSTEAD SELECT * FROM generate_series(1,5) t0(c); -- fails due to row level security enabled ERROR: hypertables do not support rules ROLLBACK TO q; ALTER TABLE t DISABLE ROW LEVEL SECURITY; SAVEPOINT q; CREATE RULE "_RETURN" AS ON SELECT TO t DO INSTEAD SELECT * FROM generate_series(1,5) t0(c); -- fails due to policy p on t ERROR: hypertables do not support rules ROLLBACK TO q; DROP POLICY p ON t; CREATE RULE "_RETURN" AS ON SELECT TO t DO INSTEAD SELECT * FROM generate_series(1,5) t0(c); -- succeeds ERROR: hypertables do not support rules ROLLBACK; -- -- Policy expression handling -- BEGIN; CREATE TABLE t (c) AS VALUES ('bar'::text); CREATE POLICY p ON t USING (max(c)); -- fails: aggregate functions are not allowed in policy expressions ERROR: aggregate functions are not allowed in policy expressions ROLLBACK; -- -- Non-target relations are only subject to SELECT policies -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE r1 (a int); SELECT public.create_hypertable('r1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (23,regress_rls_schema,r1,t) CREATE TABLE r2 (a int); SELECT public.create_hypertable('r2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (24,regress_rls_schema,r2,t) INSERT INTO r1 VALUES (10), (20); INSERT INTO r2 VALUES (10), (20); GRANT ALL ON r1, r2 TO regress_rls_bob; CREATE POLICY p1 ON r1 USING (true); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; CREATE POLICY p1 ON r2 FOR SELECT USING (true); CREATE POLICY p2 ON r2 FOR INSERT WITH CHECK (false); CREATE POLICY p3 ON r2 FOR UPDATE USING (false); CREATE POLICY p4 ON r2 FOR DELETE USING (false); ALTER TABLE r2 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM r1; a ---- 10 20 SELECT * FROM r2; a ---- 10 20 -- r2 is read-only INSERT INTO r2 VALUES (2); -- Not allowed ERROR: new row violates row-level security policy for table "r2" \pset tuples_only 1 UPDATE r2 SET a = 2 RETURNING *; -- Updates nothing DELETE FROM r2 RETURNING *; -- Deletes nothing \pset tuples_only 0 -- r2 can be used as a non-target relation in DML INSERT INTO r1 SELECT a + 1 FROM r2 RETURNING *; -- OK a ---- 11 21 UPDATE r1 SET a = r2.a + 2 FROM r2 WHERE r1.a = r2.a RETURNING *; -- OK ERROR: new row for relation "_hyper_23_105_chunk" violates check constraint "constraint_105" DELETE FROM r1 USING r2 WHERE r1.a = r2.a + 2 RETURNING *; -- OK a | a ---+--- SELECT * FROM r1; a ---- 10 11 20 21 SELECT * FROM r2; a ---- 10 20 SET SESSION AUTHORIZATION regress_rls_alice; DROP TABLE r1; DROP TABLE r2; -- -- FORCE ROW LEVEL SECURITY applies RLS to owners too -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security = on; CREATE TABLE r1 (a int); SELECT public.create_hypertable('r1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (25,regress_rls_schema,r1,t) INSERT INTO r1 VALUES (10), (20); CREATE POLICY p1 ON r1 USING (false); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- No error, but no rows TABLE r1; a --- -- RLS error INSERT INTO r1 VALUES (1); ERROR: new row violates row-level security policy for table "r1" -- No error (unable to see any rows to update) UPDATE r1 SET a = 1; TABLE r1; a --- -- No error (unable to see any rows to delete) DELETE FROM r1; TABLE r1; a --- SET row_security = off; -- these all fail, would be affected by RLS TABLE r1; ERROR: query would be affected by row-level security policy for table "r1" HINT: To disable the policy for the table's owner, use ALTER TABLE NO FORCE ROW LEVEL SECURITY. UPDATE r1 SET a = 1; ERROR: query would be affected by row-level security policy for table "r1" HINT: To disable the policy for the table's owner, use ALTER TABLE NO FORCE ROW LEVEL SECURITY. DELETE FROM r1; ERROR: query would be affected by row-level security policy for table "r1" HINT: To disable the policy for the table's owner, use ALTER TABLE NO FORCE ROW LEVEL SECURITY. DROP TABLE r1; -- -- FORCE ROW LEVEL SECURITY does not break RI -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security = on; CREATE TABLE r1 (a int PRIMARY KEY); -- r1 is not a hypertable since r1.a is referenced by r2 CREATE TABLE r2 (a int REFERENCES r1); SELECT public.create_hypertable('r2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (26,regress_rls_schema,r2,t) INSERT INTO r1 VALUES (10), (20); INSERT INTO r2 VALUES (10), (20); -- Create policies on r2 which prevent the -- owner from seeing any rows, but RI should -- still see them. CREATE POLICY p1 ON r2 USING (false); ALTER TABLE r2 ENABLE ROW LEVEL SECURITY; ALTER TABLE r2 FORCE ROW LEVEL SECURITY; -- Errors due to rows in r2 DELETE FROM r1; ERROR: update or delete on table "r1" violates foreign key constraint "r2_a_fkey" on table "r2" DETAIL: Key (a)=(10) is still referenced from table "r2". -- Reset r2 to no-RLS DROP POLICY p1 ON r2; ALTER TABLE r2 NO FORCE ROW LEVEL SECURITY; ALTER TABLE r2 DISABLE ROW LEVEL SECURITY; -- clean out r2 for INSERT test below DELETE FROM r2; -- Change r1 to not allow rows to be seen CREATE POLICY p1 ON r1 USING (false); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- No rows seen TABLE r1; a --- -- No error, RI still sees that row exists in r1 INSERT INTO r2 VALUES (10); DROP TABLE r2; DROP TABLE r1; -- Ensure cascaded DELETE works CREATE TABLE r1 (a int PRIMARY KEY); -- r1 is not a hypertable since r1.a is referenced by r2 CREATE TABLE r2 (a int REFERENCES r1 ON DELETE CASCADE); SELECT public.create_hypertable('r2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (27,regress_rls_schema,r2,t) INSERT INTO r1 VALUES (10), (20); INSERT INTO r2 VALUES (10), (20); -- Create policies on r2 which prevent the -- owner from seeing any rows, but RI should -- still see them. CREATE POLICY p1 ON r2 USING (false); ALTER TABLE r2 ENABLE ROW LEVEL SECURITY; ALTER TABLE r2 FORCE ROW LEVEL SECURITY; -- Deletes all records from both DELETE FROM r1; -- Remove FORCE from r2 ALTER TABLE r2 NO FORCE ROW LEVEL SECURITY; -- As owner, we now bypass RLS -- verify no rows in r2 now TABLE r2; a --- DROP TABLE r2; DROP TABLE r1; -- Ensure cascaded UPDATE works CREATE TABLE r1 (a int PRIMARY KEY); -- r1 is not a hypertable since r1.a is referenced by r2 CREATE TABLE r2 (a int REFERENCES r1 ON UPDATE CASCADE); SELECT public.create_hypertable('r2', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (28,regress_rls_schema,r2,t) INSERT INTO r1 VALUES (10), (20); INSERT INTO r2 VALUES (10), (20); -- Create policies on r2 which prevent the -- owner from seeing any rows, but RI should -- still see them. CREATE POLICY p1 ON r2 USING (false); ALTER TABLE r2 ENABLE ROW LEVEL SECURITY; ALTER TABLE r2 FORCE ROW LEVEL SECURITY; -- Updates records in both (terse output to not print CONTEXT, which can be different). \set VERBOSITY terse UPDATE r1 SET a = a+5; ERROR: new row for relation "_hyper_28_117_chunk" violates check constraint "constraint_117" \set VERBOSITY default -- Remove FORCE from r2 ALTER TABLE r2 NO FORCE ROW LEVEL SECURITY; -- As owner, we now bypass RLS -- verify records in r2 updated TABLE r2; a ---- 10 20 DROP TABLE r2; DROP TABLE r1; -- -- Test INSERT+RETURNING applies SELECT policies as -- WithCheckOptions (meaning an error is thrown) -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security = on; CREATE TABLE r1 (a int); SELECT public.create_hypertable('r1', 'a', chunk_time_interval=>2); create_hypertable ------------------------------ (29,regress_rls_schema,r1,t) CREATE POLICY p1 ON r1 FOR SELECT USING (false); CREATE POLICY p2 ON r1 FOR INSERT WITH CHECK (true); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- Works fine INSERT INTO r1 VALUES (10), (20); -- No error, but no rows TABLE r1; a --- SET row_security = off; -- fail, would be affected by RLS TABLE r1; ERROR: query would be affected by row-level security policy for table "r1" HINT: To disable the policy for the table's owner, use ALTER TABLE NO FORCE ROW LEVEL SECURITY. SET row_security = on; -- Error INSERT INTO r1 VALUES (10), (20) RETURNING *; ERROR: new row violates row-level security policy for table "r1" DROP TABLE r1; -- -- Test UPDATE+RETURNING applies SELECT policies as -- WithCheckOptions (meaning an error is thrown) -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security = on; CREATE TABLE r1 (a int PRIMARY KEY); SELECT public.create_hypertable('r1', 'a', chunk_time_interval=>100); create_hypertable ------------------------------ (30,regress_rls_schema,r1,t) CREATE POLICY p1 ON r1 FOR SELECT USING (a < 20); CREATE POLICY p2 ON r1 FOR UPDATE USING (a < 20) WITH CHECK (true); CREATE POLICY p3 ON r1 FOR INSERT WITH CHECK (true); INSERT INTO r1 VALUES (10); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- Works fine UPDATE r1 SET a = 30; -- Show updated rows ALTER TABLE r1 NO FORCE ROW LEVEL SECURITY; TABLE r1; a ---- 30 -- reset value in r1 for test with RETURNING UPDATE r1 SET a = 10; -- Verify row reset TABLE r1; a ---- 10 ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- Error UPDATE r1 SET a = 30 RETURNING *; ERROR: new row violates row-level security policy for table "r1" -- UPDATE path of INSERT ... ON CONFLICT DO UPDATE should also error out INSERT INTO r1 VALUES (10) ON CONFLICT (a) DO UPDATE SET a = 30 RETURNING *; ERROR: new row violates row-level security policy for table "r1" -- Should still error out without RETURNING (use of arbiter always requires -- SELECT permissions) INSERT INTO r1 VALUES (10) ON CONFLICT (a) DO UPDATE SET a = 30; ERROR: new row violates row-level security policy for table "r1" -- ON CONFLICT ON CONSTRAINT INSERT INTO r1 VALUES (10) ON CONFLICT ON CONSTRAINT r1_pkey DO UPDATE SET a = 30; ERROR: new row violates row-level security policy for table "r1" DROP TABLE r1; -- Check dependency handling RESET SESSION AUTHORIZATION; CREATE TABLE dep1 (c1 int); SELECT public.create_hypertable('dep1', 'c1', chunk_time_interval=>2); create_hypertable -------------------------------- (31,regress_rls_schema,dep1,t) CREATE TABLE dep2 (c1 int); SELECT public.create_hypertable('dep2', 'c1', chunk_time_interval=>2); create_hypertable -------------------------------- (32,regress_rls_schema,dep2,t) CREATE POLICY dep_p1 ON dep1 TO regress_rls_bob USING (c1 > (select max(dep2.c1) from dep2)); ALTER POLICY dep_p1 ON dep1 TO regress_rls_bob,regress_rls_carol; -- Should return one SELECT count(*) = 1 FROM pg_depend WHERE objid = (SELECT oid FROM pg_policy WHERE polname = 'dep_p1') AND refobjid = (SELECT oid FROM pg_class WHERE relname = 'dep2'); ?column? ---------- t ALTER POLICY dep_p1 ON dep1 USING (true); -- Should return one SELECT count(*) = 1 FROM pg_shdepend WHERE objid = (SELECT oid FROM pg_policy WHERE polname = 'dep_p1') AND refobjid = (SELECT oid FROM pg_authid WHERE rolname = 'regress_rls_bob'); ?column? ---------- t -- Should return one SELECT count(*) = 1 FROM pg_shdepend WHERE objid = (SELECT oid FROM pg_policy WHERE polname = 'dep_p1') AND refobjid = (SELECT oid FROM pg_authid WHERE rolname = 'regress_rls_carol'); ?column? ---------- t -- Should return zero SELECT count(*) = 0 FROM pg_depend WHERE objid = (SELECT oid FROM pg_policy WHERE polname = 'dep_p1') AND refobjid = (SELECT oid FROM pg_class WHERE relname = 'dep2'); ?column? ---------- t -- DROP OWNED BY testing RESET SESSION AUTHORIZATION; CREATE ROLE regress_rls_dob_role1; CREATE ROLE regress_rls_dob_role2; CREATE TABLE dob_t1 (c1 int); SELECT public.create_hypertable('dob_t1', 'c1', chunk_time_interval=>2); create_hypertable ---------------------------------- (33,regress_rls_schema,dob_t1,t) CREATE TABLE dob_t2 (c1 int) PARTITION BY RANGE (c1); CREATE POLICY p1 ON dob_t1 TO regress_rls_dob_role1 USING (true); DROP OWNED BY regress_rls_dob_role1; DROP POLICY p1 ON dob_t1; -- should fail, already gone ERROR: policy "p1" for table "dob_t1" does not exist CREATE POLICY p1 ON dob_t1 TO regress_rls_dob_role1,regress_rls_dob_role2 USING (true); DROP OWNED BY regress_rls_dob_role1; DROP POLICY p1 ON dob_t1; -- should succeed CREATE POLICY p1 ON dob_t2 TO regress_rls_dob_role1,regress_rls_dob_role2 USING (true); DROP OWNED BY regress_rls_dob_role1; DROP POLICY p1 ON dob_t2; -- should succeed DROP USER regress_rls_dob_role1; DROP USER regress_rls_dob_role2; -- -- Clean up objects -- RESET SESSION AUTHORIZATION; \set VERBOSITY terse \\ -- suppress cascade details DROP SCHEMA regress_rls_schema CASCADE; NOTICE: drop cascades to 116 other objects \set VERBOSITY default DROP USER regress_rls_alice; DROP USER regress_rls_bob; DROP USER regress_rls_carol; DROP USER regress_rls_dave; DROP USER regress_rls_exempt_user; DROP ROLE regress_rls_group1; DROP ROLE regress_rls_group2; -- Arrange to have a few policies left over, for testing -- pg_dump/pg_restore CREATE SCHEMA regress_rls_schema; CREATE TABLE rls_tbl (c1 int); SELECT public.create_hypertable('rls_tbl', 'c1', chunk_time_interval=>2); create_hypertable ----------------------------------- (34,regress_rls_schema,rls_tbl,t) ALTER TABLE rls_tbl ENABLE ROW LEVEL SECURITY; CREATE POLICY p1 ON rls_tbl USING (c1 > 5); CREATE POLICY p2 ON rls_tbl FOR SELECT USING (c1 <= 3); CREATE POLICY p3 ON rls_tbl FOR UPDATE USING (c1 <= 3) WITH CHECK (c1 > 5); CREATE POLICY p4 ON rls_tbl FOR DELETE USING (c1 <= 3); CREATE TABLE rls_tbl_force (c1 int); SELECT public.create_hypertable('rls_tbl_force', 'c1', chunk_time_interval=>2); create_hypertable ----------------------------------------- (35,regress_rls_schema,rls_tbl_force,t) ALTER TABLE rls_tbl_force ENABLE ROW LEVEL SECURITY; ALTER TABLE rls_tbl_force FORCE ROW LEVEL SECURITY; CREATE POLICY p1 ON rls_tbl_force USING (c1 = 5) WITH CHECK (c1 < 5); CREATE POLICY p2 ON rls_tbl_force FOR SELECT USING (c1 = 8); CREATE POLICY p3 ON rls_tbl_force FOR UPDATE USING (c1 = 8) WITH CHECK (c1 >= 5); CREATE POLICY p4 ON rls_tbl_force FOR DELETE USING (c1 = 8); ================================================ FILE: test/expected/size_utils.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \ir include/insert_two_partitions.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."two_Partitions" ( "timeCustom" BIGINT NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."two_Partitions" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_0) WHERE series_0 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_1) WHERE series_1 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_2) WHERE series_2 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_bool) WHERE series_bool IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, device_id); SELECT * FROM create_hypertable('"public"."two_Partitions"'::regclass, 'timeCustom'::name, 'device_id'::name, associated_schema_name=>'_timescaledb_internal'::text, number_partitions => 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+----------------+--------- 1 | public | two_Partitions | t \set QUIET off BEGIN; BEGIN \COPY public."two_Partitions" FROM 'data/ds1_dev1_1.tsv' NULL AS ''; COPY 7 COMMIT; COMMIT INSERT INTO public."two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257987600000000000, 'dev1', 1.5, 1), (1257987600000000000, 'dev1', 1.5, 2), (1257894000000000000, 'dev2', 1.5, 1), (1257894002000000000, 'dev1', 2.5, 3); INSERT 0 4 INSERT INTO "two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257894000000000000, 'dev2', 1.5, 2); INSERT 0 1 \set QUIET on SELECT * FROM hypertable_detailed_size('"public"."two_Partitions"'); table_bytes | index_bytes | toast_bytes | total_bytes | node_name -------------+-------------+-------------+-------------+----------- 32768 | 475136 | 40960 | 548864 | SELECT * FROM hypertable_index_size('"public"."two_Partitions_device_id_timeCustom_idx"'); hypertable_index_size ----------------------- 73728 SELECT * FROM hypertable_index_size('"public"."two_Partitions_timeCustom_device_id_idx"'); hypertable_index_size ----------------------- 73728 SELECT * FROM hypertable_index_size('"public"."two_Partitions_timeCustom_idx"'); hypertable_index_size ----------------------- 73728 SELECT * FROM hypertable_index_size('"public"."two_Partitions_timeCustom_series_0_idx"'); hypertable_index_size ----------------------- 73728 SELECT * FROM hypertable_index_size('"public"."two_Partitions_timeCustom_series_1_idx"'); hypertable_index_size ----------------------- 73728 SELECT * FROM hypertable_index_size('"public"."two_Partitions_timeCustom_series_2_idx"'); hypertable_index_size ----------------------- 49152 SELECT * FROM hypertable_index_size('"public"."two_Partitions_timeCustom_series_bool_idx"'); hypertable_index_size ----------------------- 57344 SELECT * FROM chunks_detailed_size('"public"."two_Partitions"') order by chunk_name; chunk_schema | chunk_name | table_bytes | index_bytes | toast_bytes | total_bytes | node_name -----------------------+------------------+-------------+-------------+-------------+-------------+----------- _timescaledb_internal | _hyper_1_1_chunk | 8192 | 114688 | 8192 | 131072 | _timescaledb_internal | _hyper_1_2_chunk | 8192 | 106496 | 8192 | 122880 | _timescaledb_internal | _hyper_1_3_chunk | 8192 | 98304 | 8192 | 114688 | _timescaledb_internal | _hyper_1_4_chunk | 8192 | 98304 | 8192 | 114688 | CREATE TABLE timestamp_partitioned(time TIMESTAMP, value TEXT); SELECT * FROM create_hypertable('timestamp_partitioned', 'time', 'value', 2); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices hypertable_id | schema_name | table_name | created ---------------+-------------+-----------------------+--------- 2 | public | timestamp_partitioned | t INSERT INTO timestamp_partitioned VALUES('2004-10-19 10:23:54', '10'); INSERT INTO timestamp_partitioned VALUES('2004-12-19 10:23:54', '30'); SELECT * FROM chunks_detailed_size('timestamp_partitioned') order by chunk_name; chunk_schema | chunk_name | table_bytes | index_bytes | toast_bytes | total_bytes | node_name -----------------------+------------------+-------------+-------------+-------------+-------------+----------- _timescaledb_internal | _hyper_2_5_chunk | 8192 | 32768 | 8192 | 49152 | _timescaledb_internal | _hyper_2_6_chunk | 8192 | 32768 | 8192 | 49152 | CREATE TABLE timestamp_partitioned_2(time TIMESTAMP, value CHAR(9)); SELECT * FROM create_hypertable('timestamp_partitioned_2', 'time', 'value', 2); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices hypertable_id | schema_name | table_name | created ---------------+-------------+-------------------------+--------- 3 | public | timestamp_partitioned_2 | t INSERT INTO timestamp_partitioned_2 VALUES('2004-10-19 10:23:54', '10'); INSERT INTO timestamp_partitioned_2 VALUES('2004-12-19 10:23:54', '30'); SELECT * FROM chunks_detailed_size('timestamp_partitioned_2') order by chunk_name; chunk_schema | chunk_name | table_bytes | index_bytes | toast_bytes | total_bytes | node_name -----------------------+------------------+-------------+-------------+-------------+-------------+----------- _timescaledb_internal | _hyper_3_7_chunk | 8192 | 32768 | 0 | 40960 | _timescaledb_internal | _hyper_3_8_chunk | 8192 | 32768 | 0 | 40960 | CREATE TABLE toast_test(time TIMESTAMP, value TEXT); -- Set storage type to EXTERNAL to prevent PostgreSQL from compressing my -- easily compressable string and instead store it with TOAST ALTER TABLE toast_test ALTER COLUMN value SET STORAGE EXTERNAL; SELECT * FROM create_hypertable('toast_test', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 4 | public | toast_test | t INSERT INTO toast_test VALUES('2004-10-19 10:23:54', $$ this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. $$); SELECT * FROM chunks_detailed_size('toast_test'); chunk_schema | chunk_name | table_bytes | index_bytes | toast_bytes | total_bytes | node_name -----------------------+------------------+-------------+-------------+-------------+-------------+----------- _timescaledb_internal | _hyper_4_9_chunk | 8192 | 16384 | 24576 | 49152 | -- -- Tests for approximate_row_count() -- -- Regular table -- CREATE TABLE approx_count(time TIMESTAMP, value int); INSERT INTO approx_count VALUES('2004-01-01 10:00:01', 1); INSERT INTO approx_count VALUES('2004-01-01 10:00:02', 2); INSERT INTO approx_count VALUES('2004-01-01 10:00:03', 3); INSERT INTO approx_count VALUES('2004-01-01 10:00:04', 4); INSERT INTO approx_count VALUES('2004-01-01 10:00:05', 5); INSERT INTO approx_count VALUES('2004-01-01 10:00:06', 6); INSERT INTO approx_count VALUES('2004-01-01 10:00:07', 7); SELECT * FROM approximate_row_count('approx_count'); approximate_row_count ----------------------- 0 ANALYZE approx_count; SELECT count(*) FROM approx_count; count ------- 7 SELECT * FROM approximate_row_count('approx_count'); approximate_row_count ----------------------- 7 DROP TABLE approx_count; -- Regular table with basic inheritance -- CREATE TABLE approx_count(id int); INSERT INTO approx_count VALUES(1); SELECT count(*) FROM approx_count; count ------- 1 SELECT * FROM approximate_row_count('approx_count'); approximate_row_count ----------------------- 0 ANALYZE approx_count; SELECT * FROM approximate_row_count('approx_count'); approximate_row_count ----------------------- 1 CREATE TABLE approx_count_child(id2 int) INHERITS (approx_count); INSERT INTO approx_count_child VALUES(0); SELECT count(*) FROM approx_count; count ------- 2 SELECT * FROM approximate_row_count('approx_count'); approximate_row_count ----------------------- 1 ANALYZE approx_count_child; SELECT * FROM approximate_row_count('approx_count'); approximate_row_count ----------------------- 2 DROP TABLE approx_count CASCADE; NOTICE: drop cascades to table approx_count_child -- Regular table with nested inheritance -- CREATE TABLE approx_count(id int); CREATE TABLE approx_count_a(id2 int) INHERITS (approx_count); CREATE TABLE approx_count_b(id3 int) INHERITS (approx_count_a); CREATE TABLE approx_count_c(id4 int) INHERITS (approx_count_b); INSERT INTO approx_count_a VALUES(0); INSERT INTO approx_count_b VALUES(1); INSERT INTO approx_count_c VALUES(2); INSERT INTO approx_count VALUES(3); SELECT * FROM approximate_row_count('approx_count'); approximate_row_count ----------------------- 0 ANALYZE approx_count_a; ANALYZE approx_count_b; ANALYZE approx_count_c; ANALYZE approx_count; SELECT count(*) FROM approx_count; count ------- 4 SELECT * FROM approximate_row_count('approx_count'); approximate_row_count ----------------------- 4 SELECT count(*) FROM approx_count_a; count ------- 3 SELECT * FROM approximate_row_count('approx_count_a'); approximate_row_count ----------------------- 3 SELECT count(*) FROM approx_count_b; count ------- 2 SELECT * FROM approximate_row_count('approx_count_b'); approximate_row_count ----------------------- 2 SELECT count(*) FROM approx_count_c; count ------- 1 SELECT * FROM approximate_row_count('approx_count_c'); approximate_row_count ----------------------- 1 DROP TABLE approx_count CASCADE; NOTICE: drop cascades to 3 other objects -- table with declarative partitioning -- CREATE TABLE approx_count_dp(time TIMESTAMP, value int) PARTITION BY RANGE(time); CREATE TABLE approx_count_dp0 PARTITION OF approx_count_dp FOR VALUES FROM ('2004-01-01 00:00:00') TO ('2005-01-01 00:00:00'); CREATE TABLE approx_count_dp1 PARTITION OF approx_count_dp FOR VALUES FROM ('2005-01-01 00:00:00') TO ('2006-01-01 00:00:00'); CREATE TABLE approx_count_dp2 PARTITION OF approx_count_dp FOR VALUES FROM ('2006-01-01 00:00:00') TO ('2007-01-01 00:00:00'); INSERT INTO approx_count_dp VALUES('2004-01-01 10:00:00', 1); INSERT INTO approx_count_dp VALUES('2004-01-01 11:00:00', 1); INSERT INTO approx_count_dp VALUES('2004-01-01 12:00:01', 1); INSERT INTO approx_count_dp VALUES('2005-01-01 10:00:00', 1); INSERT INTO approx_count_dp VALUES('2005-01-01 11:00:00', 1); INSERT INTO approx_count_dp VALUES('2005-01-01 12:00:01', 1); INSERT INTO approx_count_dp VALUES('2006-01-01 10:00:00', 1); INSERT INTO approx_count_dp VALUES('2006-01-01 11:00:00', 1); INSERT INTO approx_count_dp VALUES('2006-01-01 12:00:01', 1); SELECT count(*) FROM approx_count_dp; count ------- 9 SELECT count(*) FROM approx_count_dp0; count ------- 3 SELECT count(*) FROM approx_count_dp1; count ------- 3 SELECT count(*) FROM approx_count_dp2; count ------- 3 SELECT * FROM approximate_row_count('approx_count_dp'); approximate_row_count ----------------------- 0 ANALYZE approx_count_dp; SELECT * FROM approximate_row_count('approx_count_dp'); approximate_row_count ----------------------- 9 SELECT * FROM approximate_row_count('approx_count_dp0'); approximate_row_count ----------------------- 3 SELECT * FROM approximate_row_count('approx_count_dp1'); approximate_row_count ----------------------- 3 SELECT * FROM approximate_row_count('approx_count_dp2'); approximate_row_count ----------------------- 3 CREATE TABLE approx_count_dp_nested(time TIMESTAMP, device_id int, value int) PARTITION BY RANGE(time); CREATE TABLE approx_count_dp_nested_0 PARTITION OF approx_count_dp_nested FOR VALUES FROM ('2004-01-01 00:00:00') TO ('2005-01-01 00:00:00') PARTITION BY RANGE (device_id); CREATE TABLE approx_count_dp_nested_0_0 PARTITION OF approx_count_dp_nested_0 FOR VALUES FROM (0) TO (10); CREATE TABLE approx_count_dp_nested_0_1 PARTITION OF approx_count_dp_nested_0 FOR VALUES FROM (10) TO (20); CREATE TABLE approx_count_dp_nested_1 PARTITION OF approx_count_dp_nested FOR VALUES FROM ('2005-01-01 00:00:00') TO ('2006-01-01 00:00:00') PARTITION BY RANGE (device_id); CREATE TABLE approx_count_dp_nested_1_0 PARTITION OF approx_count_dp_nested_1 FOR VALUES FROM (0) TO (10); CREATE TABLE approx_count_dp_nested_1_1 PARTITION OF approx_count_dp_nested_1 FOR VALUES FROM (10) TO (20); INSERT INTO approx_count_dp_nested VALUES('2004-01-01 10:00:00', 1, 1); INSERT INTO approx_count_dp_nested VALUES('2004-01-01 10:00:00', 2, 1); INSERT INTO approx_count_dp_nested VALUES('2004-01-01 10:00:00', 3, 1); INSERT INTO approx_count_dp_nested VALUES('2004-01-01 10:00:00', 11, 1); INSERT INTO approx_count_dp_nested VALUES('2004-01-01 10:00:00', 12, 1); INSERT INTO approx_count_dp_nested VALUES('2004-01-01 10:00:00', 13, 1); INSERT INTO approx_count_dp_nested VALUES('2005-01-01 10:00:00', 1, 1); INSERT INTO approx_count_dp_nested VALUES('2005-01-01 10:00:00', 2, 1); INSERT INTO approx_count_dp_nested VALUES('2005-01-01 10:00:00', 3, 1); INSERT INTO approx_count_dp_nested VALUES('2005-01-01 10:00:00', 11, 1); INSERT INTO approx_count_dp_nested VALUES('2005-01-01 10:00:00', 12, 1); INSERT INTO approx_count_dp_nested VALUES('2005-01-01 10:00:00', 13, 1); SELECT * FROM approximate_row_count('approx_count_dp_nested'); approximate_row_count ----------------------- 0 ANALYZE approx_count_dp_nested; SELECT (SELECT count(*) FROM approx_count_dp_nested) AS dp_nested, (SELECT count(*) FROM approx_count_dp_nested_0) AS dp_nested_0, (SELECT count(*) FROM approx_count_dp_nested_0_0) AS dp_nested_0_0, (SELECT count(*) FROM approx_count_dp_nested_0_1) AS dp_nested_0_1, (SELECT count(*) FROM approx_count_dp_nested_1) AS dp_nested_1, (SELECT count(*) FROM approx_count_dp_nested_1_0) AS dp_nested_1_0, (SELECT count(*) FROM approx_count_dp_nested_1_1) AS dp_nested_1_1 UNION ALL SELECT approximate_row_count('approx_count_dp_nested'), approximate_row_count('approx_count_dp_nested_0'), approximate_row_count('approx_count_dp_nested_0_0'), approximate_row_count('approx_count_dp_nested_0_1'), approximate_row_count('approx_count_dp_nested_1'), approximate_row_count('approx_count_dp_nested_1_0'), approximate_row_count('approx_count_dp_nested_1_1'); dp_nested | dp_nested_0 | dp_nested_0_0 | dp_nested_0_1 | dp_nested_1 | dp_nested_1_0 | dp_nested_1_1 -----------+-------------+---------------+---------------+-------------+---------------+--------------- 12 | 6 | 3 | 3 | 6 | 3 | 3 12 | 6 | 3 | 3 | 6 | 3 | 3 -- Hypertable -- CREATE TABLE approx_count(time TIMESTAMP, value int); SELECT * FROM create_hypertable('approx_count', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices hypertable_id | schema_name | table_name | created ---------------+-------------+--------------+--------- 5 | public | approx_count | t INSERT INTO approx_count VALUES('2004-01-01 10:00:01', 1); INSERT INTO approx_count VALUES('2004-01-01 10:00:02', 2); INSERT INTO approx_count VALUES('2004-01-01 10:00:03', 3); INSERT INTO approx_count VALUES('2004-01-01 10:00:04', 4); INSERT INTO approx_count VALUES('2004-01-01 10:00:05', 5); INSERT INTO approx_count VALUES('2004-01-01 10:00:06', 6); INSERT INTO approx_count VALUES('2004-01-01 10:00:07', 7); INSERT INTO approx_count VALUES('2004-01-01 10:00:08', 8); INSERT INTO approx_count VALUES('2004-01-01 10:00:09', 9); INSERT INTO approx_count VALUES('2004-01-01 10:00:10', 10); SELECT count(*) FROM approx_count; count ------- 10 SELECT * FROM approximate_row_count('approx_count'); approximate_row_count ----------------------- 0 ANALYZE approx_count; SELECT * FROM approximate_row_count('approx_count'); approximate_row_count ----------------------- 10 \set ON_ERROR_STOP 0 SELECT * FROM approximate_row_count('unexisting'); ERROR: relation "unexisting" does not exist at character 37 SELECT * FROM approximate_row_count(); ERROR: function approximate_row_count() does not exist at character 15 SELECT * FROM approximate_row_count(NULL); approximate_row_count ----------------------- \set ON_ERROR_STOP 1 -- Test size functions with invalid or non-existing OID SELECT * FROM hypertable_size(0); hypertable_size ----------------- SELECT * FROM hypertable_detailed_size(0) ORDER BY node_name; table_bytes | index_bytes | toast_bytes | total_bytes | node_name -------------+-------------+-------------+-------------+----------- SELECT * FROM chunks_detailed_size(0) ORDER BY node_name; chunk_schema | chunk_name | table_bytes | index_bytes | toast_bytes | total_bytes | node_name --------------+------------+-------------+-------------+-------------+-------------+----------- SELECT * FROM hypertable_compression_stats(0) ORDER BY node_name; total_chunks | number_compressed_chunks | before_compression_table_bytes | before_compression_index_bytes | before_compression_toast_bytes | before_compression_total_bytes | after_compression_table_bytes | after_compression_index_bytes | after_compression_toast_bytes | after_compression_total_bytes | node_name --------------+--------------------------+--------------------------------+--------------------------------+--------------------------------+--------------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------+----------- SELECT * FROM chunk_compression_stats(0) ORDER BY node_name; chunk_schema | chunk_name | compression_status | before_compression_table_bytes | before_compression_index_bytes | before_compression_toast_bytes | before_compression_total_bytes | after_compression_table_bytes | after_compression_index_bytes | after_compression_toast_bytes | after_compression_total_bytes | node_name --------------+------------+--------------------+--------------------------------+--------------------------------+--------------------------------+--------------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------+----------- SELECT * FROM hypertable_index_size(0); hypertable_index_size ----------------------- SELECT * FROM _timescaledb_functions.relation_size(0); total_size | heap_size | index_size | toast_size ------------+-----------+------------+------------ | | | SELECT * FROM hypertable_size(1); hypertable_size ----------------- SELECT * FROM hypertable_detailed_size(1) ORDER BY node_name; table_bytes | index_bytes | toast_bytes | total_bytes | node_name -------------+-------------+-------------+-------------+----------- SELECT * FROM chunks_detailed_size(1) ORDER BY node_name; chunk_schema | chunk_name | table_bytes | index_bytes | toast_bytes | total_bytes | node_name --------------+------------+-------------+-------------+-------------+-------------+----------- SELECT * FROM hypertable_compression_stats(1) ORDER BY node_name; total_chunks | number_compressed_chunks | before_compression_table_bytes | before_compression_index_bytes | before_compression_toast_bytes | before_compression_total_bytes | after_compression_table_bytes | after_compression_index_bytes | after_compression_toast_bytes | after_compression_total_bytes | node_name --------------+--------------------------+--------------------------------+--------------------------------+--------------------------------+--------------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------+----------- SELECT * FROM chunk_compression_stats(1) ORDER BY node_name; chunk_schema | chunk_name | compression_status | before_compression_table_bytes | before_compression_index_bytes | before_compression_toast_bytes | before_compression_total_bytes | after_compression_table_bytes | after_compression_index_bytes | after_compression_toast_bytes | after_compression_total_bytes | node_name --------------+------------+--------------------+--------------------------------+--------------------------------+--------------------------------+--------------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------+----------- SELECT * FROM hypertable_index_size(1); hypertable_index_size ----------------------- SELECT * FROM _timescaledb_functions.relation_size(1); total_size | heap_size | index_size | toast_size ------------+-----------+------------+------------ 0 | 0 | 0 | 0 -- Test size functions with NULL input SELECT * FROM hypertable_size(NULL); hypertable_size ----------------- SELECT * FROM hypertable_detailed_size(NULL) ORDER BY node_name; table_bytes | index_bytes | toast_bytes | total_bytes | node_name -------------+-------------+-------------+-------------+----------- SELECT * FROM chunks_detailed_size(NULL) ORDER BY node_name; chunk_schema | chunk_name | table_bytes | index_bytes | toast_bytes | total_bytes | node_name --------------+------------+-------------+-------------+-------------+-------------+----------- SELECT * FROM hypertable_compression_stats(NULL) ORDER BY node_name; total_chunks | number_compressed_chunks | before_compression_table_bytes | before_compression_index_bytes | before_compression_toast_bytes | before_compression_total_bytes | after_compression_table_bytes | after_compression_index_bytes | after_compression_toast_bytes | after_compression_total_bytes | node_name --------------+--------------------------+--------------------------------+--------------------------------+--------------------------------+--------------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------+----------- SELECT * FROM chunk_compression_stats(NULL) ORDER BY node_name; chunk_schema | chunk_name | compression_status | before_compression_table_bytes | before_compression_index_bytes | before_compression_toast_bytes | before_compression_total_bytes | after_compression_table_bytes | after_compression_index_bytes | after_compression_toast_bytes | after_compression_total_bytes | node_name --------------+------------+--------------------+--------------------------------+--------------------------------+--------------------------------+--------------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------+----------- SELECT * FROM hypertable_index_size(NULL); hypertable_index_size ----------------------- SELECT * FROM _timescaledb_functions.relation_size(NULL); total_size | heap_size | index_size | toast_size ------------+-----------+------------+------------ | | | -- Test approximate size functions with invalid input SELECT * FROM hypertable_approximate_size(0); hypertable_approximate_size ----------------------------- SELECT * FROM hypertable_approximate_detailed_size(0); table_bytes | index_bytes | toast_bytes | total_bytes -------------+-------------+-------------+------------- | | | SELECT * FROM _timescaledb_functions.relation_approximate_size(0); total_size | heap_size | index_size | toast_size ------------+-----------+------------+------------ | | | SELECT * FROM hypertable_approximate_size(NULL); hypertable_approximate_size ----------------------------- SELECT * FROM hypertable_approximate_detailed_size(NULL); table_bytes | index_bytes | toast_bytes | total_bytes -------------+-------------+-------------+------------- | | | SELECT * FROM _timescaledb_functions.relation_approximate_size(NULL); total_size | heap_size | index_size | toast_size ------------+-----------+------------+------------ -- Test size on view, sequence and composite type CREATE VIEW view1 as SELECT 1; SELECT * FROM _timescaledb_functions.relation_approximate_size('view1'); total_size | heap_size | index_size | toast_size ------------+-----------+------------+------------ 0 | 0 | 0 | 0 SELECT * FROM _timescaledb_functions.relation_size('view1'); total_size | heap_size | index_size | toast_size ------------+-----------+------------+------------ 0 | 0 | 0 | 0 CREATE SEQUENCE test_id_seq INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 9223372036854775807 CACHE 1; SELECT * FROM _timescaledb_functions.relation_approximate_size('test_id_seq'); total_size | heap_size | index_size | toast_size ------------+-----------+------------+------------ 8192 | 8192 | 0 | 0 SELECT * FROM _timescaledb_functions.relation_size('test_id_seq'); total_size | heap_size | index_size | toast_size ------------+-----------+------------+------------ 8192 | 8192 | 0 | 0 CREATE TYPE test_type AS (time timestamp, temp float); SELECT * FROM _timescaledb_functions.relation_approximate_size('test_type'); total_size | heap_size | index_size | toast_size ------------+-----------+------------+------------ 0 | 0 | 0 | 0 SELECT * FROM _timescaledb_functions.relation_size('test_type'); total_size | heap_size | index_size | toast_size ------------+-----------+------------+------------ 0 | 0 | 0 | 0 -- Test size functions on regular table CREATE TABLE hypersize(time timestamptz, device int); CREATE INDEX hypersize_time_idx ON hypersize (time); \set ON_ERROR_STOP 0 \set VERBOSITY default \set SHOW_CONTEXT never SELECT pg_relation_size('hypersize'), pg_table_size('hypersize'), pg_indexes_size('hypersize'), pg_total_relation_size('hypersize'), pg_relation_size('hypersize_time_idx'); pg_relation_size | pg_table_size | pg_indexes_size | pg_total_relation_size | pg_relation_size ------------------+---------------+-----------------+------------------------+------------------ 0 | 0 | 8192 | 8192 | 8192 SELECT * FROM _timescaledb_functions.relation_size('hypersize'); total_size | heap_size | index_size | toast_size ------------+-----------+------------+------------ 8192 | 0 | 8192 | 0 SELECT * FROM hypertable_size('hypersize'); hypertable_size ----------------- SELECT * FROM hypertable_detailed_size('hypersize') ORDER BY node_name; table_bytes | index_bytes | toast_bytes | total_bytes | node_name -------------+-------------+-------------+-------------+----------- SELECT * FROM chunks_detailed_size('hypersize') ORDER BY node_name; chunk_schema | chunk_name | table_bytes | index_bytes | toast_bytes | total_bytes | node_name --------------+------------+-------------+-------------+-------------+-------------+----------- SELECT * FROM hypertable_compression_stats('hypersize') ORDER BY node_name; total_chunks | number_compressed_chunks | before_compression_table_bytes | before_compression_index_bytes | before_compression_toast_bytes | before_compression_total_bytes | after_compression_table_bytes | after_compression_index_bytes | after_compression_toast_bytes | after_compression_total_bytes | node_name --------------+--------------------------+--------------------------------+--------------------------------+--------------------------------+--------------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------+----------- SELECT * FROM chunk_compression_stats('hypersize') ORDER BY node_name; chunk_schema | chunk_name | compression_status | before_compression_table_bytes | before_compression_index_bytes | before_compression_toast_bytes | before_compression_total_bytes | after_compression_table_bytes | after_compression_index_bytes | after_compression_toast_bytes | after_compression_total_bytes | node_name --------------+------------+--------------------+--------------------------------+--------------------------------+--------------------------------+--------------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------+----------- SELECT * FROM hypertable_index_size('hypersize_time_idx'); hypertable_index_size ----------------------- 8192 SELECT * FROM _timescaledb_functions.relation_approximate_size('hypersize'); total_size | heap_size | index_size | toast_size ------------+-----------+------------+------------ 8192 | 0 | 8192 | 0 SELECT * FROM hypertable_approximate_size('hypersize'); ERROR: "hypersize" is not a hypertable or a continuous aggregate HINT: The operation is only possible on a hypertable or continuous aggregate. SELECT * FROM hypertable_approximate_detailed_size('hypersize'); ERROR: "hypersize" is not a hypertable or a continuous aggregate HINT: The operation is only possible on a hypertable or continuous aggregate. \set VERBOSITY terse \set ON_ERROR_STOP 1 -- Test size functions on empty hypertable SELECT * FROM create_hypertable('hypersize', 'time'); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 6 | public | hypersize | t SELECT pg_relation_size('hypersize'), pg_table_size('hypersize'), pg_indexes_size('hypersize'), pg_total_relation_size('hypersize'), pg_relation_size('hypersize_time_idx'); pg_relation_size | pg_table_size | pg_indexes_size | pg_total_relation_size | pg_relation_size ------------------+---------------+-----------------+------------------------+------------------ 0 | 0 | 8192 | 8192 | 8192 SELECT * FROM _timescaledb_functions.relation_size('hypersize'); total_size | heap_size | index_size | toast_size ------------+-----------+------------+------------ 8192 | 0 | 8192 | 0 SELECT * FROM hypertable_size('hypersize'); hypertable_size ----------------- 8192 SELECT * FROM hypertable_detailed_size('hypersize') ORDER BY node_name; table_bytes | index_bytes | toast_bytes | total_bytes | node_name -------------+-------------+-------------+-------------+----------- 0 | 8192 | 0 | 8192 | SELECT * FROM chunks_detailed_size('hypersize') ORDER BY node_name; chunk_schema | chunk_name | table_bytes | index_bytes | toast_bytes | total_bytes | node_name --------------+------------+-------------+-------------+-------------+-------------+----------- SELECT * FROM hypertable_compression_stats('hypersize') ORDER BY node_name; total_chunks | number_compressed_chunks | before_compression_table_bytes | before_compression_index_bytes | before_compression_toast_bytes | before_compression_total_bytes | after_compression_table_bytes | after_compression_index_bytes | after_compression_toast_bytes | after_compression_total_bytes | node_name --------------+--------------------------+--------------------------------+--------------------------------+--------------------------------+--------------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------+----------- SELECT * FROM chunk_compression_stats('hypersize') ORDER BY node_name; chunk_schema | chunk_name | compression_status | before_compression_table_bytes | before_compression_index_bytes | before_compression_toast_bytes | before_compression_total_bytes | after_compression_table_bytes | after_compression_index_bytes | after_compression_toast_bytes | after_compression_total_bytes | node_name --------------+------------+--------------------+--------------------------------+--------------------------------+--------------------------------+--------------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------+----------- SELECT * FROM hypertable_index_size('hypersize_time_idx'); hypertable_index_size ----------------------- 8192 SELECT * FROM _timescaledb_functions.relation_approximate_size('hypersize'); total_size | heap_size | index_size | toast_size ------------+-----------+------------+------------ 8192 | 0 | 8192 | 0 SELECT * FROM hypertable_approximate_size('hypersize'); hypertable_approximate_size ----------------------------- 8192 SELECT * FROM hypertable_approximate_detailed_size('hypersize'); table_bytes | index_bytes | toast_bytes | total_bytes -------------+-------------+-------------+------------- 0 | 8192 | 0 | 8192 -- Test size functions on non-empty hypertable INSERT INTO hypersize VALUES('2021-02-25', 1); SELECT pg_relation_size('hypersize'), pg_table_size('hypersize'), pg_indexes_size('hypersize'), pg_total_relation_size('hypersize'), pg_relation_size('hypersize_time_idx'); pg_relation_size | pg_table_size | pg_indexes_size | pg_total_relation_size | pg_relation_size ------------------+---------------+-----------------+------------------------+------------------ 0 | 0 | 8192 | 8192 | 8192 SELECT pg_relation_size(ch), pg_table_size(ch), pg_indexes_size(ch), pg_total_relation_size(ch) FROM show_chunks('hypersize') ch ORDER BY ch; pg_relation_size | pg_table_size | pg_indexes_size | pg_total_relation_size ------------------+---------------+-----------------+------------------------ 8192 | 8192 | 16384 | 24576 SELECT * FROM show_chunks('hypersize') ch JOIN LATERAL _timescaledb_functions.relation_size(ch) ON true; ch | total_size | heap_size | index_size | toast_size -----------------------------------------+------------+-----------+------------+------------ _timescaledb_internal._hyper_6_11_chunk | 24576 | 8192 | 16384 | 0 SELECT * FROM hypertable_size('hypersize'); hypertable_size ----------------- 32768 SELECT * FROM hypertable_detailed_size('hypersize') ORDER BY node_name; table_bytes | index_bytes | toast_bytes | total_bytes | node_name -------------+-------------+-------------+-------------+----------- 8192 | 24576 | 0 | 32768 | SELECT * FROM chunks_detailed_size('hypersize') ORDER BY node_name; chunk_schema | chunk_name | table_bytes | index_bytes | toast_bytes | total_bytes | node_name -----------------------+-------------------+-------------+-------------+-------------+-------------+----------- _timescaledb_internal | _hyper_6_11_chunk | 8192 | 16384 | 0 | 24576 | SELECT * FROM hypertable_compression_stats('hypersize') ORDER BY node_name; total_chunks | number_compressed_chunks | before_compression_table_bytes | before_compression_index_bytes | before_compression_toast_bytes | before_compression_total_bytes | after_compression_table_bytes | after_compression_index_bytes | after_compression_toast_bytes | after_compression_total_bytes | node_name --------------+--------------------------+--------------------------------+--------------------------------+--------------------------------+--------------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------+----------- SELECT * FROM chunk_compression_stats('hypersize') ORDER BY node_name; chunk_schema | chunk_name | compression_status | before_compression_table_bytes | before_compression_index_bytes | before_compression_toast_bytes | before_compression_total_bytes | after_compression_table_bytes | after_compression_index_bytes | after_compression_toast_bytes | after_compression_total_bytes | node_name --------------+------------+--------------------+--------------------------------+--------------------------------+--------------------------------+--------------------------------+-------------------------------+-------------------------------+-------------------------------+-------------------------------+----------- SELECT * FROM hypertable_index_size('hypersize_time_idx'); hypertable_index_size ----------------------- 24576 SELECT * FROM _timescaledb_functions.relation_approximate_size('hypersize'); total_size | heap_size | index_size | toast_size ------------+-----------+------------+------------ 8192 | 0 | 8192 | 0 SELECT * FROM hypertable_approximate_size('hypersize'); hypertable_approximate_size ----------------------------- 32768 SELECT * FROM hypertable_approximate_detailed_size('hypersize'); table_bytes | index_bytes | toast_bytes | total_bytes -------------+-------------+-------------+------------- 8192 | 24576 | 0 | 32768 -- Test approx size functions with toast entries SELECT * FROM _timescaledb_functions.relation_approximate_size('toast_test'); total_size | heap_size | index_size | toast_size ------------+-----------+------------+------------ 16384 | 0 | 8192 | 8192 SELECT * FROM hypertable_approximate_size('toast_test'); hypertable_approximate_size ----------------------------- 65536 SELECT * FROM hypertable_approximate_detailed_size('toast_test'); table_bytes | index_bytes | toast_bytes | total_bytes -------------+-------------+-------------+------------- 8192 | 24576 | 32768 | 65536 -- Test approx size function against a regular table \set ON_ERROR_STOP 0 CREATE TABLE regular(time TIMESTAMP, value TEXT); SELECT * FROM hypertable_approximate_size('regular'); ERROR: "regular" is not a hypertable or a continuous aggregate \set ON_ERROR_STOP 1 -- Test approx size functions with dropped chunks CREATE TABLE drop_chunks_table(time BIGINT NOT NULL, data INTEGER); SELECT hypertable_id AS drop_chunks_table_id FROM create_hypertable('drop_chunks_table', 'time', chunk_time_interval => 10) \gset INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(0, 29) AS i; SELECT * FROM hypertable_approximate_size('drop_chunks_table'); hypertable_approximate_size ----------------------------- 81920 SELECT drop_chunks('drop_chunks_table', older_than => 19); drop_chunks ----------------------------------------- _timescaledb_internal._hyper_7_12_chunk SELECT * FROM hypertable_approximate_size('drop_chunks_table'); hypertable_approximate_size ----------------------------- 57344 -- github issue #4857 -- below procedure should not crash SET client_min_messages = ERROR; do $$ DECLARE o INT; BEGIN FOR c IN 1..20 LOOP ANALYZE; END LOOP; END; $$; RESET client_min_messages; ================================================ FILE: test/expected/sort_optimization.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set PREFIX 'EXPLAIN (BUFFERS OFF, COSTS OFF) ' CREATE TABLE order_test(time int NOT NULL, device_id int, value float); CREATE INDEX ON order_test(time,device_id); CREATE INDEX ON order_test(device_id,time); SELECT create_hypertable('order_test','time',chunk_time_interval:=1000); create_hypertable ------------------------- (1,public,order_test,t) INSERT INTO order_test SELECT 0,10,0.5; INSERT INTO order_test SELECT 1,9,0.5; INSERT INTO order_test SELECT 2,8,0.5; -- we want to see here that index scans are possible for the chosen expressions -- so we disable seqscan so we dont need to worry about other factors which would -- make PostgreSQL prefer seqscan over index scan SET enable_seqscan TO off; -- test sort optimization with single member order by SELECT time_bucket(10,time),device_id,value FROM order_test ORDER BY 1; time_bucket | device_id | value -------------+-----------+------- 0 | 10 | 0.5 0 | 9 | 0.5 0 | 8 | 0.5 -- should use index scan :PREFIX SELECT time_bucket(10,time),device_id,value FROM order_test ORDER BY 1; --- QUERY PLAN --- Result -> Index Scan Backward using _hyper_1_1_chunk_order_test_time_idx on _hyper_1_1_chunk -- test sort optimization with ordering by multiple columns and time_bucket not last SELECT time_bucket(10,time),device_id,value FROM order_test ORDER BY 1,2; time_bucket | device_id | value -------------+-----------+------- 0 | 8 | 0.5 0 | 9 | 0.5 0 | 10 | 0.5 SET enable_seqscan TO default; -- must not use index scan :PREFIX SELECT time_bucket(10,time),device_id,value FROM order_test ORDER BY 1,2; --- QUERY PLAN --- Sort Sort Key: (time_bucket(10, _hyper_1_1_chunk."time")), _hyper_1_1_chunk.device_id -> Result -> Seq Scan on _hyper_1_1_chunk SET enable_seqscan TO off; -- test sort optimization with ordering by multiple columns and time_bucket as last member SELECT time_bucket(10,time),device_id,value FROM order_test ORDER BY 2,1; time_bucket | device_id | value -------------+-----------+------- 0 | 8 | 0.5 0 | 9 | 0.5 0 | 10 | 0.5 -- should use index scan :PREFIX SELECT time_bucket(10,time),device_id,value FROM order_test ORDER BY 2,1; --- QUERY PLAN --- Result -> Index Scan using _hyper_1_1_chunk_order_test_device_id_time_idx on _hyper_1_1_chunk -- test sort optimization with interval calculation with non-fixed interval -- #7097 CREATE TABLE i7097_1(time timestamptz NOT NULL, quantity float, "isText" boolean); CREATE TABLE i7097_2(time timestamptz NOT NULL, quantity float, "isText" boolean); CREATE INDEX ON i7097_1(time) WHERE "isText" IS NULL; CREATE INDEX ON i7097_2(time) WHERE "isText" IS NULL; SELECT table_name FROM create_hypertable('i7097_1', 'time', create_default_indexes => false); table_name ------------ i7097_1 SELECT table_name FROM create_hypertable('i7097_2', 'time', create_default_indexes => false); table_name ------------ i7097_2 INSERT INTO i7097_1(time, quantity) SELECT time, round((random() * (100-3) + 3)::NUMERIC) AS quantity FROM generate_series('2023-01-01T00:00:00+01:00', '2023-05-01T00:00:00+01:00', interval 'PT10M') AS t(time); INSERT INTO i7097_2(time, quantity) SELECT time, round((random() * (100-3) + 3)::NUMERIC) AS quantity FROM generate_series('2023-01-01T00:00:00+01:00', '2023-05-01T00:00:00+01:00', interval 'PT10M') AS t(time); VACUUM ANALYZE i7097_1, i7097_2; SET TIME ZONE 'Europe/Paris'; WITH "cte1" AS (SELECT time + interval 'P1Y' AS time, avg(quantity) AS quantity FROM i7097_1 WHERE time >= '2024-03-31T00:00:00+01:00'::timestamptz - interval 'P1Y' AND time < '2024-03-31T23:59:59+02:00'::timestamptz + (- interval 'P1Y') AND "isText" IS NULL GROUP BY 1 ORDER BY 1 ASC), "cte2" AS (SELECT time + interval 'P1Y' AS time, avg(quantity) AS quantity FROM i7097_2 WHERE time >= '2024-03-31T00:00:00+01:00'::timestamptz - interval 'P1Y' AND time < '2024-03-31T23:59:59+02:00'::timestamptz + (- interval 'P1Y') AND "isText" IS NULL GROUP BY 1 ORDER BY 1 ASC) SELECT count(*) FROM (SELECT time, cte1.quantity + cte2.quantity FROM cte1 FULL OUTER JOIN cte2 USING (time)) j; count ------- 138 -- github issue 9214 -- test off-by one error in sort optimization CREATE TABLE i9214(time timestamptz NOT NULL, machine_id INT NOT NULL, name TEXT NOT NULL, value FLOAT NOT NULL) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column INSERT INTO i9214 VALUES ('2026-01-30 10:00:00+00', 1, 'tag1', 10.5), ('2026-01-30 10:00:00+00', 1, 'tag2', 20.5), ('2026-01-30 10:01:00+00', 1, 'tag1', 11.0), ('2026-01-30 10:01:00+00', 1, 'tag2', 21.0); WITH rule1 AS ( SELECT date_trunc('minute', time) AS time, machine_id FROM i9214 WHERE machine_id = 1 AND name = 'tag1' AND value > 5 ), row_numbered AS ( SELECT time, machine_id, row_number() OVER (ORDER BY time) AS seqnum FROM rule1 ) SELECT min(time) AS start_time, machine_id, count(*) AS duration_minutes FROM row_numbered GROUP BY machine_id, (time - (seqnum * interval '1 minute')) ORDER BY min(time); start_time | machine_id | duration_minutes ------------------------------+------------+------------------ Fri Jan 30 11:00:00 2026 CET | 1 | 2 WITH rule1 AS ( SELECT date_trunc('minute', time) AS time, machine_id FROM i9214 WHERE machine_id = 1 AND name = 'tag1' AND value > 5 ), rule2 AS ( SELECT date_trunc('minute', time) AS time, machine_id FROM i9214 WHERE machine_id = 1 AND name = 'tag2' AND value > 5 ), joined_rules AS ( SELECT r1.time, r1.machine_id FROM rule1 r1 INNER JOIN rule2 r2 USING(time,machine_id) ), row_numbered AS ( SELECT time, machine_id, row_number() OVER (ORDER BY time) AS seqnum FROM joined_rules ) SELECT min(time) AS start_time, machine_id, count(*) AS duration_minutes FROM row_numbered GROUP BY machine_id, (time - (seqnum * interval '1 minute')) ORDER BY min(time); start_time | machine_id | duration_minutes ------------------------------+------------+------------------ Fri Jan 30 11:00:00 2026 CET | 1 | 2 ================================================ FILE: test/expected/sql_query.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \o /dev/null \ir include/insert_two_partitions.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."two_Partitions" ( "timeCustom" BIGINT NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."two_Partitions" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_0) WHERE series_0 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_1) WHERE series_1 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_2) WHERE series_2 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_bool) WHERE series_bool IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, device_id); SELECT * FROM create_hypertable('"public"."two_Partitions"'::regclass, 'timeCustom'::name, 'device_id'::name, associated_schema_name=>'_timescaledb_internal'::text, number_partitions => 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); \set QUIET off BEGIN; \COPY public."two_Partitions" FROM 'data/ds1_dev1_1.tsv' NULL AS ''; COMMIT; INSERT INTO public."two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257987600000000000, 'dev1', 1.5, 1), (1257987600000000000, 'dev1', 1.5, 2), (1257894000000000000, 'dev2', 1.5, 1), (1257894002000000000, 'dev1', 2.5, 3); INSERT INTO "two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257894000000000000, 'dev2', 1.5, 2); \set QUIET on \o SELECT * FROM PUBLIC."two_Partitions"; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257894000000000000 | dev1 | 1.5 | 1 | 2 | t 1257894000000000000 | dev1 | 1.5 | 2 | | 1257894000000001000 | dev1 | 2.5 | 3 | | 1257894001000000000 | dev1 | 3.5 | 4 | | 1257894002000000000 | dev1 | 5.5 | 6 | | t 1257894002000000000 | dev1 | 5.5 | 7 | | f 1257894002000000000 | dev1 | 2.5 | 3 | | 1257897600000000000 | dev1 | 4.5 | 5 | | f 1257987600000000000 | dev1 | 1.5 | 1 | | 1257987600000000000 | dev1 | 1.5 | 2 | | 1257894000000000000 | dev2 | 1.5 | 1 | | 1257894000000000000 | dev2 | 1.5 | 2 | | EXPLAIN (verbose ON, buffers off, costs off) SELECT * FROM PUBLIC."two_Partitions"; --- QUERY PLAN --- Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."timeCustom", _hyper_1_1_chunk.device_id, _hyper_1_1_chunk.series_0, _hyper_1_1_chunk.series_1, _hyper_1_1_chunk.series_2, _hyper_1_1_chunk.series_bool -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."timeCustom", _hyper_1_2_chunk.device_id, _hyper_1_2_chunk.series_0, _hyper_1_2_chunk.series_1, _hyper_1_2_chunk.series_2, _hyper_1_2_chunk.series_bool -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."timeCustom", _hyper_1_3_chunk.device_id, _hyper_1_3_chunk.series_0, _hyper_1_3_chunk.series_1, _hyper_1_3_chunk.series_2, _hyper_1_3_chunk.series_bool -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."timeCustom", _hyper_1_4_chunk.device_id, _hyper_1_4_chunk.series_0, _hyper_1_4_chunk.series_1, _hyper_1_4_chunk.series_2, _hyper_1_4_chunk.series_bool \echo "The following queries should NOT scan two_Partitions._hyper_1_1_chunk" "The following queries should NOT scan two_Partitions._hyper_1_1_chunk" EXPLAIN (verbose ON, buffers off, costs off) SELECT * FROM PUBLIC."two_Partitions" WHERE device_id = 'dev2'; --- QUERY PLAN --- Index Scan using "_hyper_1_4_chunk_two_Partitions_device_id_timeCustom_idx" on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."timeCustom", _hyper_1_4_chunk.device_id, _hyper_1_4_chunk.series_0, _hyper_1_4_chunk.series_1, _hyper_1_4_chunk.series_2, _hyper_1_4_chunk.series_bool Index Cond: (_hyper_1_4_chunk.device_id = 'dev2'::text) EXPLAIN (verbose ON, buffers off, costs off) SELECT * FROM PUBLIC."two_Partitions" WHERE device_id = 'dev'||'2'; --- QUERY PLAN --- Index Scan using "_hyper_1_4_chunk_two_Partitions_device_id_timeCustom_idx" on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."timeCustom", _hyper_1_4_chunk.device_id, _hyper_1_4_chunk.series_0, _hyper_1_4_chunk.series_1, _hyper_1_4_chunk.series_2, _hyper_1_4_chunk.series_bool Index Cond: (_hyper_1_4_chunk.device_id = 'dev2'::text) EXPLAIN (verbose ON, buffers off, costs off) SELECT * FROM PUBLIC."two_Partitions" WHERE 'dev'||'2' = device_id; --- QUERY PLAN --- Index Scan using "_hyper_1_4_chunk_two_Partitions_device_id_timeCustom_idx" on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."timeCustom", _hyper_1_4_chunk.device_id, _hyper_1_4_chunk.series_0, _hyper_1_4_chunk.series_1, _hyper_1_4_chunk.series_2, _hyper_1_4_chunk.series_bool Index Cond: (_hyper_1_4_chunk.device_id = 'dev2'::text) --test integer partition key CREATE TABLE "int_part"(time timestamp, object_id int, temp float); SELECT create_hypertable('"int_part"', 'time', 'object_id', 2); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ----------------------- (2,public,int_part,t) INSERT INTO "int_part" VALUES('2017-01-20T09:00:01', 1, 22.5); INSERT INTO "int_part" VALUES('2017-01-20T09:00:01', 2, 22.5); --check that there are two chunks SELECT * FROM test.show_subtables('int_part'); Child | Tablespace ----------------------------------------+------------ _timescaledb_internal._hyper_2_5_chunk | _timescaledb_internal._hyper_2_6_chunk | SELECT * FROM "int_part" WHERE object_id = 1; time | object_id | temp --------------------------+-----------+------ Fri Jan 20 09:00:01 2017 | 1 | 22.5 --check that queries with IN/ANY/= work for the "time" column SELECT * FROM "int_part" WHERE time IN (NULL); time | object_id | temp ------+-----------+------ SELECT * FROM "int_part" WHERE time = ANY (NULL); time | object_id | temp ------+-----------+------ SELECT * FROM "int_part" WHERE time = NULL; time | object_id | temp ------+-----------+------ --make sure this touches only one partititon EXPLAIN (verbose ON, buffers off, costs off) SELECT * FROM "int_part" WHERE object_id = 1; --- QUERY PLAN --- Index Scan using _hyper_2_5_chunk_int_part_object_id_time_idx on _timescaledb_internal._hyper_2_5_chunk Output: _hyper_2_5_chunk."time", _hyper_2_5_chunk.object_id, _hyper_2_5_chunk.temp Index Cond: (_hyper_2_5_chunk.object_id = 1) --Need to verify space partitions are currently pruned in this query --EXPLAIN (verbose ON, buffers off, costs off) SELECT * FROM "two_Partitions" WHERE device_id IN ('dev2', 'dev21'); \echo "The following shows non-aggregated queries with time desc using merge append" "The following shows non-aggregated queries with time desc using merge append" EXPLAIN (verbose ON, buffers off, costs off)SELECT * FROM PUBLIC."two_Partitions" ORDER BY "timeCustom" DESC NULLS LAST limit 2; --- QUERY PLAN --- Limit Output: "two_Partitions"."timeCustom", "two_Partitions".device_id, "two_Partitions".series_0, "two_Partitions".series_1, "two_Partitions".series_2, "two_Partitions".series_bool -> Custom Scan (ChunkAppend) on public."two_Partitions" Output: "two_Partitions"."timeCustom", "two_Partitions".device_id, "two_Partitions".series_0, "two_Partitions".series_1, "two_Partitions".series_2, "two_Partitions".series_bool Order: "two_Partitions"."timeCustom" DESC NULLS LAST Startup Exclusion: false Runtime Exclusion: false -> Index Scan using "_hyper_1_3_chunk_two_Partitions_timeCustom_device_id_idx" on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."timeCustom", _hyper_1_3_chunk.device_id, _hyper_1_3_chunk.series_0, _hyper_1_3_chunk.series_1, _hyper_1_3_chunk.series_2, _hyper_1_3_chunk.series_bool -> Index Scan using "_hyper_1_2_chunk_two_Partitions_timeCustom_device_id_idx" on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."timeCustom", _hyper_1_2_chunk.device_id, _hyper_1_2_chunk.series_0, _hyper_1_2_chunk.series_1, _hyper_1_2_chunk.series_2, _hyper_1_2_chunk.series_bool -> Merge Append Sort Key: "two_Partitions"."timeCustom" DESC NULLS LAST -> Index Scan using "_hyper_1_4_chunk_two_Partitions_timeCustom_device_id_idx" on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."timeCustom", _hyper_1_4_chunk.device_id, _hyper_1_4_chunk.series_0, _hyper_1_4_chunk.series_1, _hyper_1_4_chunk.series_2, _hyper_1_4_chunk.series_bool -> Index Scan using "_hyper_1_1_chunk_two_Partitions_timeCustom_device_id_idx" on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."timeCustom", _hyper_1_1_chunk.device_id, _hyper_1_1_chunk.series_0, _hyper_1_1_chunk.series_1, _hyper_1_1_chunk.series_2, _hyper_1_1_chunk.series_bool --shows that more specific indexes are used if the WHERE clauses "match", uses the series_1 index here. EXPLAIN (verbose ON, buffers off, costs off)SELECT * FROM PUBLIC."two_Partitions" WHERE series_1 IS NOT NULL ORDER BY "timeCustom" DESC NULLS LAST limit 2; --- QUERY PLAN --- Limit Output: "two_Partitions"."timeCustom", "two_Partitions".device_id, "two_Partitions".series_0, "two_Partitions".series_1, "two_Partitions".series_2, "two_Partitions".series_bool -> Custom Scan (ChunkAppend) on public."two_Partitions" Output: "two_Partitions"."timeCustom", "two_Partitions".device_id, "two_Partitions".series_0, "two_Partitions".series_1, "two_Partitions".series_2, "two_Partitions".series_bool Order: "two_Partitions"."timeCustom" DESC NULLS LAST Startup Exclusion: false Runtime Exclusion: false -> Index Scan using "_hyper_1_3_chunk_two_Partitions_timeCustom_series_1_idx" on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."timeCustom", _hyper_1_3_chunk.device_id, _hyper_1_3_chunk.series_0, _hyper_1_3_chunk.series_1, _hyper_1_3_chunk.series_2, _hyper_1_3_chunk.series_bool -> Index Scan using "_hyper_1_2_chunk_two_Partitions_timeCustom_series_1_idx" on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."timeCustom", _hyper_1_2_chunk.device_id, _hyper_1_2_chunk.series_0, _hyper_1_2_chunk.series_1, _hyper_1_2_chunk.series_2, _hyper_1_2_chunk.series_bool -> Merge Append Sort Key: "two_Partitions"."timeCustom" DESC NULLS LAST -> Index Scan using "_hyper_1_4_chunk_two_Partitions_timeCustom_series_1_idx" on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."timeCustom", _hyper_1_4_chunk.device_id, _hyper_1_4_chunk.series_0, _hyper_1_4_chunk.series_1, _hyper_1_4_chunk.series_2, _hyper_1_4_chunk.series_bool -> Index Scan using "_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."timeCustom", _hyper_1_1_chunk.device_id, _hyper_1_1_chunk.series_0, _hyper_1_1_chunk.series_1, _hyper_1_1_chunk.series_2, _hyper_1_1_chunk.series_bool --here the "match" is implication series_1 > 1 => series_1 IS NOT NULL EXPLAIN (verbose ON, buffers off, costs off)SELECT * FROM PUBLIC."two_Partitions" WHERE series_1 > 1 ORDER BY "timeCustom" DESC NULLS LAST limit 2; --- QUERY PLAN --- Limit Output: "two_Partitions"."timeCustom", "two_Partitions".device_id, "two_Partitions".series_0, "two_Partitions".series_1, "two_Partitions".series_2, "two_Partitions".series_bool -> Custom Scan (ChunkAppend) on public."two_Partitions" Output: "two_Partitions"."timeCustom", "two_Partitions".device_id, "two_Partitions".series_0, "two_Partitions".series_1, "two_Partitions".series_2, "two_Partitions".series_bool Order: "two_Partitions"."timeCustom" DESC NULLS LAST Startup Exclusion: false Runtime Exclusion: false -> Index Scan using "_hyper_1_3_chunk_two_Partitions_timeCustom_series_1_idx" on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."timeCustom", _hyper_1_3_chunk.device_id, _hyper_1_3_chunk.series_0, _hyper_1_3_chunk.series_1, _hyper_1_3_chunk.series_2, _hyper_1_3_chunk.series_bool Index Cond: (_hyper_1_3_chunk.series_1 > '1'::double precision) -> Index Scan using "_hyper_1_2_chunk_two_Partitions_timeCustom_series_1_idx" on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."timeCustom", _hyper_1_2_chunk.device_id, _hyper_1_2_chunk.series_0, _hyper_1_2_chunk.series_1, _hyper_1_2_chunk.series_2, _hyper_1_2_chunk.series_bool Index Cond: (_hyper_1_2_chunk.series_1 > '1'::double precision) -> Merge Append Sort Key: "two_Partitions"."timeCustom" DESC NULLS LAST -> Index Scan using "_hyper_1_4_chunk_two_Partitions_timeCustom_series_1_idx" on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."timeCustom", _hyper_1_4_chunk.device_id, _hyper_1_4_chunk.series_0, _hyper_1_4_chunk.series_1, _hyper_1_4_chunk.series_2, _hyper_1_4_chunk.series_bool Index Cond: (_hyper_1_4_chunk.series_1 > '1'::double precision) -> Index Scan using "_hyper_1_1_chunk_two_Partitions_timeCustom_series_1_idx" on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."timeCustom", _hyper_1_1_chunk.device_id, _hyper_1_1_chunk.series_0, _hyper_1_1_chunk.series_1, _hyper_1_1_chunk.series_2, _hyper_1_1_chunk.series_bool Index Cond: (_hyper_1_1_chunk.series_1 > '1'::double precision) --note that without time transform things work too EXPLAIN (verbose ON, buffers off, costs off)SELECT "timeCustom" t, min(series_0) FROM PUBLIC."two_Partitions" GROUP BY t ORDER BY t DESC NULLS LAST limit 2; --- QUERY PLAN --- Limit Output: "two_Partitions"."timeCustom", (min("two_Partitions".series_0)) -> GroupAggregate Output: "two_Partitions"."timeCustom", min("two_Partitions".series_0) Group Key: "two_Partitions"."timeCustom" -> Custom Scan (ChunkAppend) on public."two_Partitions" Output: "two_Partitions"."timeCustom", "two_Partitions".series_0 Order: "two_Partitions"."timeCustom" DESC NULLS LAST Startup Exclusion: false Runtime Exclusion: false -> Index Scan using "_hyper_1_3_chunk_two_Partitions_timeCustom_device_id_idx" on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."timeCustom", _hyper_1_3_chunk.series_0 -> Index Scan using "_hyper_1_2_chunk_two_Partitions_timeCustom_device_id_idx" on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."timeCustom", _hyper_1_2_chunk.series_0 -> Merge Append Sort Key: "two_Partitions"."timeCustom" DESC NULLS LAST -> Index Scan using "_hyper_1_4_chunk_two_Partitions_timeCustom_device_id_idx" on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."timeCustom", _hyper_1_4_chunk.series_0 -> Index Scan using "_hyper_1_1_chunk_two_Partitions_timeCustom_device_id_idx" on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."timeCustom", _hyper_1_1_chunk.series_0 --The query should still use the index on timeCustom, even though the GROUP BY/ORDER BY is on the transformed time 't'. --However, current query plans show that it does not. EXPLAIN (verbose ON, buffers off, costs off)SELECT "timeCustom"/10 t, min(series_0) FROM PUBLIC."two_Partitions" GROUP BY t ORDER BY t DESC NULLS LAST limit 2; --- QUERY PLAN --- Limit Output: (("two_Partitions"."timeCustom" / 10)), (min("two_Partitions".series_0)) -> Sort Output: (("two_Partitions"."timeCustom" / 10)), (min("two_Partitions".series_0)) Sort Key: (("two_Partitions"."timeCustom" / 10)) DESC NULLS LAST -> HashAggregate Output: (("two_Partitions"."timeCustom" / 10)), min("two_Partitions".series_0) Group Key: ("two_Partitions"."timeCustom" / 10) -> Result Output: ("two_Partitions"."timeCustom" / 10), "two_Partitions".series_0 -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."timeCustom", _hyper_1_1_chunk.series_0 -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."timeCustom", _hyper_1_2_chunk.series_0 -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."timeCustom", _hyper_1_3_chunk.series_0 -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."timeCustom", _hyper_1_4_chunk.series_0 EXPLAIN (verbose ON, buffers off, costs off)SELECT "timeCustom"%10 t, min(series_0) FROM PUBLIC."two_Partitions" GROUP BY t ORDER BY t DESC NULLS LAST limit 2; --- QUERY PLAN --- Limit Output: (("two_Partitions"."timeCustom" % '10'::bigint)), (min("two_Partitions".series_0)) -> Sort Output: (("two_Partitions"."timeCustom" % '10'::bigint)), (min("two_Partitions".series_0)) Sort Key: (("two_Partitions"."timeCustom" % '10'::bigint)) DESC NULLS LAST -> HashAggregate Output: (("two_Partitions"."timeCustom" % '10'::bigint)), min("two_Partitions".series_0) Group Key: ("two_Partitions"."timeCustom" % '10'::bigint) -> Result Output: ("two_Partitions"."timeCustom" % '10'::bigint), "two_Partitions".series_0 -> Append -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk."timeCustom", _hyper_1_1_chunk.series_0 -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk."timeCustom", _hyper_1_2_chunk.series_0 -> Seq Scan on _timescaledb_internal._hyper_1_3_chunk Output: _hyper_1_3_chunk."timeCustom", _hyper_1_3_chunk.series_0 -> Seq Scan on _timescaledb_internal._hyper_1_4_chunk Output: _hyper_1_4_chunk."timeCustom", _hyper_1_4_chunk.series_0 ================================================ FILE: test/expected/symbol_conflict.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER -- Test for symbol conflicts between the loader module and the -- versioned extension module. -- This test fails on, e.g. Linux, unless compiled with -fvisibility=hidden CREATE OR REPLACE FUNCTION hello_loader() RETURNS TEXT AS 'timescaledb', 'loader_hello' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; SELECT hello_loader(); hello_loader ------------------- hello from loader CREATE OR REPLACE FUNCTION hello_timescaledb() RETURNS TEXT AS :MODULE_PATHNAME, 'timescaledb_hello' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; -- This calls an internal function with a conflicting name in the loader SELECT hello_loader(); hello_loader ------------------- hello from loader -- This calls the identically named internal function in the versioned extension SELECT hello_timescaledb(); hello_timescaledb ------------------------ hello from timescaledb ================================================ FILE: test/expected/tableam.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test support for setting table access method on hypertables \c :TEST_DBNAME :ROLE_SUPERUSER -- create a new access method that reuses the heap handler CREATE ACCESS METHOD testam TYPE TABLE HANDLER heap_tableam_handler; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE TABLE testam (time timestamptz, device int, temp float) USING testam; SELECT create_hypertable('testam', 'time', 'device', 2); create_hypertable --------------------- (1,public,testam,t) -- show that the hypertable is using the 'testam' table access method SELECT amname AS hypertable_amname FROM pg_class cl, pg_am am WHERE cl.oid = 'testam'::regclass AND cl.relam = am.oid; hypertable_amname ------------------- testam -- insert data to create a chunk INSERT INTO testam VALUES('2020-01-22:11:30', 1, 29.3); -- make sure the table access method for a chunk is the same as the -- hypertable root SELECT amname AS chunk_amname FROM pg_class cl, pg_am am, show_chunks('testam') ch WHERE cl.oid = ch AND cl.relam = am.oid; chunk_amname -------------- testam ================================================ FILE: test/expected/tableam_alter.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test support for setting table access method on hypertables using -- ALTER TABLE. It should propagate to the chunks. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE ACCESS METHOD testam TYPE TABLE HANDLER heap_tableam_handler; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE VIEW chunk_info AS SELECT hypertable_name AS hypertable, chunk_name AS chunk, amname FROM timescaledb_information.chunks ch JOIN pg_class cl ON (format('%I.%I', ch.chunk_schema, ch.chunk_name)::regclass = cl.oid) JOIN pg_am am ON (am.oid = cl.relam); CREATE TABLE test_table (time timestamptz not null, device int, temp float); SELECT create_hypertable('test_table', by_range('time')); create_hypertable ------------------- (1,t) INSERT INTO test_table SELECT ts, 10 * random(), 100 * random() FROM generate_series('2001-01-01'::timestamp, '2001-02-01', '1d'::interval) as x(ts); SELECT cl.relname, amname FROM pg_class cl JOIN pg_am am ON cl.relam = am.oid WHERE cl.relname = 'test_table'; relname | amname ------------+-------- test_table | heap SELECT * FROM chunk_info WHERE hypertable = 'test_table'; hypertable | chunk | amname ------------+------------------+-------- test_table | _hyper_1_1_chunk | heap test_table | _hyper_1_2_chunk | heap test_table | _hyper_1_3_chunk | heap test_table | _hyper_1_4_chunk | heap test_table | _hyper_1_5_chunk | heap test_table | _hyper_1_6_chunk | heap -- Test setting the access method together with other options. This -- should not generate an error. ALTER TABLE test_table SET ACCESS METHOD testam, SET (autovacuum_vacuum_threshold = 100); -- Create more chunks. These will use the new access method, but the -- old chunks will use the old access method. INSERT INTO test_table SELECT ts, 10 * random(), 100 * random() FROM generate_series('2001-02-01'::timestamp, '2001-03-01', '1d'::interval) as x(ts); SELECT cl.relname, amname FROM pg_class cl JOIN pg_am am ON cl.relam = am.oid WHERE cl.relname = 'test_table'; relname | amname ------------+-------- test_table | testam SELECT * FROM chunk_info WHERE hypertable = 'test_table'; hypertable | chunk | amname ------------+-------------------+-------- test_table | _hyper_1_1_chunk | heap test_table | _hyper_1_2_chunk | heap test_table | _hyper_1_3_chunk | heap test_table | _hyper_1_4_chunk | heap test_table | _hyper_1_5_chunk | heap test_table | _hyper_1_6_chunk | heap test_table | _hyper_1_7_chunk | testam test_table | _hyper_1_8_chunk | testam test_table | _hyper_1_9_chunk | testam test_table | _hyper_1_10_chunk | testam ================================================ FILE: test/expected/tableam_alter_defaults.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test support for setting table access method on hypertables using -- ALTER TABLE for version 17 and later. This is in addition to the -- tests in tableam_alter.sql. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE ACCESS METHOD testam TYPE TABLE HANDLER heap_tableam_handler; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE VIEW chunk_info AS SELECT hypertable_name AS hypertable, chunk_name AS chunk, amname FROM timescaledb_information.chunks ch JOIN pg_class cl ON (format('%I.%I', ch.chunk_schema, ch.chunk_name)::regclass = cl.oid) JOIN pg_am am ON (am.oid = cl.relam); CREATE TABLE test_table (time timestamptz not null, device int, temp float); SELECT cl.relname, amname FROM pg_class cl JOIN pg_am am ON cl.relam = am.oid WHERE cl.relname = 'test_table'; relname | amname ------------+-------- test_table | heap -- Check that setting default access method of a normal table works. ALTER TABLE test_table SET ACCESS METHOD DEFAULT; -- Check that changing the access method and then changing it back -- works. ALTER TABLE test_table SET ACCESS METHOD testam; ALTER TABLE test_table SET ACCESS METHOD DEFAULT; -- Check that setting default access method of a hypertable works. SELECT create_hypertable('test_table', by_range('time')); create_hypertable ------------------- (1,t) ALTER TABLE test_table SET ACCESS METHOD DEFAULT; SELECT cl.relname, amname FROM pg_class cl JOIN pg_am am ON cl.relam = am.oid WHERE cl.relname = 'test_table'; relname | amname ------------+-------- test_table | heap -- Test setting the access method together with other options. This -- should not generate an error. ALTER TABLE test_table SET ACCESS METHOD testam, SET (autovacuum_vacuum_threshold = 100); -- Add some rows to generate a chunk. This should get the access -- method of the hypertable. INSERT INTO test_table SELECT ts, 10 * random(), 100 * random() FROM generate_series('2001-01-01'::timestamp, '2001-01-14', '1d'::interval) as x(ts); SELECT * FROM chunk_info WHERE hypertable = 'test_table'; hypertable | chunk | amname ------------+------------------+-------- test_table | _hyper_1_1_chunk | testam test_table | _hyper_1_2_chunk | testam test_table | _hyper_1_3_chunk | testam -- Setting it to the default method after we have set it to a test -- access method should work fine also on a hypertable. SELECT cl.relname, amname FROM pg_class cl JOIN pg_am am ON cl.relam = am.oid WHERE cl.relname = 'test_table'; relname | amname ------------+-------- test_table | testam ALTER TABLE test_table SET ACCESS METHOD DEFAULT; SELECT cl.relname, amname FROM pg_class cl JOIN pg_am am ON cl.relam = am.oid WHERE cl.relname = 'test_table'; relname | amname ------------+-------- test_table | heap SELECT chunk FROM show_chunks('test_table') t(chunk) limit 1 \gset ALTER TABLE :chunk SET ACCESS METHOD DEFAULT; SELECT * FROM chunk_info WHERE hypertable = 'test_table'; hypertable | chunk | amname ------------+------------------+-------- test_table | _hyper_1_1_chunk | heap test_table | _hyper_1_2_chunk | testam test_table | _hyper_1_3_chunk | testam ================================================ FILE: test/expected/tablespace.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set ON_ERROR_STOP 0 \c :TEST_DBNAME :ROLE_SUPERUSER CREATE VIEW hypertable_tablespaces AS SELECT cls.relname AS hypertable, (SELECT spcname FROM pg_tablespace WHERE oid = reltablespace) AS tablespace FROM _timescaledb_catalog.hypertable, LATERAL (SELECT * FROM pg_class WHERE oid = format('%I.%I', schema_name, table_name)::regclass) AS cls ORDER BY hypertable, tablespace; GRANT SELECT ON hypertable_tablespaces TO PUBLIC; --Test hypertable with tablespace. Tablespaces are cluster-wide, so we --attach the test name as prefix to allow tests to be executed in --parallel. CREATE TABLESPACE tablespace1 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE1_PATH; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --assigning a tablespace via the main table should work CREATE TABLE tspace_2dim(time timestamp, temp float, device text) TABLESPACE tablespace1; SELECT create_hypertable('tspace_2dim', 'time', 'device', 2); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------- (1,public,tspace_2dim,t) INSERT INTO tspace_2dim VALUES ('2017-01-20T09:00:01', 24.3, 'blue'); -- Tablespace for tspace_2dim should be set SELECT * FROM hypertable_tablespaces WHERE hypertable = 'tspace_2dim'; hypertable | tablespace -------------+------------- tspace_2dim | tablespace1 SELECT show_tablespaces('tspace_2dim'); show_tablespaces ------------------ tablespace1 --verify that the table chunk has the correct tablespace SELECT relname, spcname FROM pg_class c INNER JOIN pg_tablespace t ON (c.reltablespace = t.oid) INNER JOIN _timescaledb_catalog.chunk ch ON (ch.table_name = c.relname); relname | spcname ------------------+------------- _hyper_1_1_chunk | tablespace1 --check some error conditions SELECT attach_tablespace(NULL,NULL); ERROR: invalid tablespace name SELECT attach_tablespace('tablespace2', NULL); ERROR: invalid hypertable SELECT attach_tablespace(NULL, 'tspace_2dim'); ERROR: invalid tablespace name SELECT attach_tablespace('none_existing_tablespace', 'tspace_2dim'); ERROR: tablespace "none_existing_tablespace" does not exist SELECT attach_tablespace('tablespace2', 'none_existing_table'); ERROR: relation "none_existing_table" does not exist at character 41 SELECT detach_tablespace(NULL); ERROR: invalid tablespace name SELECT detach_tablespaces(NULL); ERROR: invalid argument SELECT show_tablespaces(NULL); show_tablespaces ------------------ --attach another tablespace without first creating it --> should generate error SELECT attach_tablespace('tablespace2', 'tspace_2dim'); ERROR: tablespace "tablespace2" does not exist --attach the same tablespace twice to same table should also generate error SELECT attach_tablespace('tablespace1', 'tspace_2dim'); ERROR: tablespace "tablespace1" is already attached to hypertable "tspace_2dim" --no error if if_not_attached is given SELECT attach_tablespace('tablespace1', 'tspace_2dim', if_not_attached => true); NOTICE: tablespace "tablespace1" is already attached to hypertable "tspace_2dim", skipping attach_tablespace ------------------- \c :TEST_DBNAME :ROLE_SUPERUSER --Tablespaces are cluster-wide, so we attach the test name as prefix --to allow tests to be executed in parallel. CREATE TABLESPACE tablespace2 OWNER :ROLE_DEFAULT_PERM_USER_2 LOCATION :TEST_TABLESPACE2_PATH; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 --attach without permissions on the table should fail SELECT attach_tablespace('tablespace2', 'tspace_2dim'); ERROR: must be owner of hypertable "tspace_2dim" \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --attach without permissions on the tablespace should also fail SELECT attach_tablespace('tablespace2', 'tspace_2dim'); ERROR: permission denied for tablespace "tablespace2" by table owner "default_perm_user" \c :TEST_DBNAME :ROLE_SUPERUSER GRANT :ROLE_DEFAULT_PERM_USER_2 TO :ROLE_DEFAULT_PERM_USER; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --should work with permissions on both the table and the tablespace SELECT attach_tablespace('tablespace2', 'tspace_2dim'); attach_tablespace ------------------- SELECT * FROM _timescaledb_catalog.tablespace; id | hypertable_id | tablespace_name ----+---------------+----------------- 1 | 1 | tablespace1 2 | 1 | tablespace2 SELECT * FROM show_tablespaces('tspace_2dim'); show_tablespaces ------------------ tablespace1 tablespace2 --insert into another chunk INSERT INTO tspace_2dim VALUES ('2017-01-20T09:00:01', 24.3, 'brown'); SELECT * FROM test.show_subtables('tspace_2dim'); Child | Tablespace ----------------------------------------+------------- _timescaledb_internal._hyper_1_1_chunk | tablespace1 _timescaledb_internal._hyper_1_2_chunk | tablespace2 --indexes should inherit the tablespace of their chunk SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+--------------------------------------------------------------------+---------------+------+--------+---------+-----------+------------- _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_tspace_2dim_time_idx | {time} | | f | f | f | tablespace1 _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_tspace_2dim_device_time_idx | {device,time} | | f | f | f | tablespace1 _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_tspace_2dim_time_idx | {time} | | f | f | f | tablespace1 _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_tspace_2dim_device_time_idx | {device,time} | | f | f | f | tablespace1 \x SELECT * FROM timescaledb_information.hypertables ORDER BY hypertable_schema, hypertable_name; -[ RECORD 1 ]----------+---------------------------- hypertable_schema | public hypertable_name | tspace_2dim owner | default_perm_user num_dimensions | 2 num_chunks | 2 compression_enabled | f tablespaces | {tablespace1,tablespace2} primary_dimension | time primary_dimension_type | timestamp without time zone SELECT hypertable_schema, hypertable_name, chunk_schema, chunk_name, chunk_tablespace FROM timescaledb_information.chunks ORDER BY chunk_name; -[ RECORD 1 ]-----+---------------------- hypertable_schema | public hypertable_name | tspace_2dim chunk_schema | _timescaledb_internal chunk_name | _hyper_1_1_chunk chunk_tablespace | tablespace1 -[ RECORD 2 ]-----+---------------------- hypertable_schema | public hypertable_name | tspace_2dim chunk_schema | _timescaledb_internal chunk_name | _hyper_1_2_chunk chunk_tablespace | tablespace2 \x -- SET ROLE :ROLE_DEFAULT_PERM_USER_2; CREATE TABLE tspace_1dim(time timestamp, temp float, device text); SELECT create_hypertable('tspace_1dim', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------- (2,public,tspace_1dim,t) --user doesn't have permission on tablespace1 --> error SELECT attach_tablespace('tablespace1', 'tspace_1dim'); ERROR: permission denied for tablespace "tablespace1" by table owner "default_perm_user_2" --grant permission to tablespace1 SET ROLE :ROLE_DEFAULT_PERM_USER; GRANT CREATE ON TABLESPACE tablespace1 TO :ROLE_DEFAULT_PERM_USER_2; SET ROLE :ROLE_DEFAULT_PERM_USER_2; --should work fine now. Test SELECT INTO utility statements to ensure --internal alter table function call works with event triggers. SELECT true INTO attached FROM attach_tablespace('tablespace1', 'tspace_1dim'); SELECT attach_tablespace('tablespace2', 'tspace_1dim'); attach_tablespace ------------------- -- Tablespace for tspace_1dim should be set and attached SELECT * FROM hypertable_tablespaces WHERE hypertable = 'tspace_1dim'; hypertable | tablespace -------------+------------- tspace_1dim | tablespace1 SELECT show_tablespaces('tspace_1dim'); show_tablespaces ------------------ tablespace1 tablespace2 --trying to revoke permissions while attached should fail SET ROLE :ROLE_DEFAULT_PERM_USER; REVOKE CREATE ON TABLESPACE tablespace1 FROM :ROLE_DEFAULT_PERM_USER_2; ERROR: cannot revoke privilege while tablespace "tablespace1" is attached to hypertable "tspace_1dim" REVOKE ALL ON TABLESPACE tablespace1 FROM :ROLE_DEFAULT_PERM_USER_2; ERROR: cannot revoke privilege while tablespace "tablespace1" is attached to hypertable "tspace_1dim" SET ROLE :ROLE_DEFAULT_PERM_USER_2; SELECT * FROM _timescaledb_catalog.tablespace; id | hypertable_id | tablespace_name ----+---------------+----------------- 1 | 1 | tablespace1 2 | 1 | tablespace2 3 | 2 | tablespace1 4 | 2 | tablespace2 INSERT INTO tspace_1dim VALUES ('2017-01-20T09:00:01', 24.3, 'blue'); INSERT INTO tspace_1dim VALUES ('2017-03-20T09:00:01', 24.3, 'brown'); SELECT * FROM test.show_subtablesp('tspace_%'); Parent | Child | Tablespace -------------+----------------------------------------+------------- tspace_2dim | _timescaledb_internal._hyper_1_1_chunk | tablespace1 tspace_2dim | _timescaledb_internal._hyper_1_2_chunk | tablespace2 tspace_1dim | _timescaledb_internal._hyper_2_3_chunk | tablespace1 tspace_1dim | _timescaledb_internal._hyper_2_4_chunk | tablespace2 --indexes should inherit the tablespace of their chunk, unless the --parent index has a tablespace set, in which case the chunks' --corresponding indexes are pinned to the parent index's --tablespace. The parent index can have a tablespace set in two cases: --(1) if explicitly set in CREATE INDEX, or (2) if the main table was --created with a tablespace, because then default indexes will be --created in that tablespace too. SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace ----------------------------------------+--------------------------------------------------------------------+---------------+------+--------+---------+-----------+------------- _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_tspace_2dim_time_idx | {time} | | f | f | f | tablespace1 _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk_tspace_2dim_device_time_idx | {device,time} | | f | f | f | tablespace1 _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_tspace_2dim_time_idx | {time} | | f | f | f | tablespace1 _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_2_chunk_tspace_2dim_device_time_idx | {device,time} | | f | f | f | tablespace1 _timescaledb_internal._hyper_2_3_chunk | _timescaledb_internal._hyper_2_3_chunk_tspace_1dim_time_idx | {time} | | f | f | f | tablespace2 _timescaledb_internal._hyper_2_4_chunk | _timescaledb_internal._hyper_2_4_chunk_tspace_1dim_time_idx | {time} | | f | f | f | tablespace1 --detach tablespace1 from tspace_2dim should fail due to lack of permissions SELECT detach_tablespace('tablespace1', 'tspace_2dim'); ERROR: must be owner of hypertable "tspace_2dim" --detach tablespace1 from all tables. Should only detach from --'tspace_1dim' (1 tablespace) due to lack of permissions SELECT * FROM hypertable_tablespaces; hypertable | tablespace -------------+------------- tspace_1dim | tablespace1 tspace_2dim | tablespace1 SELECT * INTO detached FROM detach_tablespace('tablespace1'); NOTICE: tablespace "tablespace1" remains attached to 1 hypertable(s) due to lack of permissions SELECT * FROM detached; detach_tablespace ------------------- 1 SELECT * FROM _timescaledb_catalog.tablespace; id | hypertable_id | tablespace_name ----+---------------+----------------- 1 | 1 | tablespace1 2 | 1 | tablespace2 4 | 2 | tablespace2 SELECT * FROM show_tablespaces('tspace_1dim'); show_tablespaces ------------------ tablespace2 SELECT * FROM show_tablespaces('tspace_2dim'); show_tablespaces ------------------ tablespace1 tablespace2 SELECT * FROM hypertable_tablespaces; hypertable | tablespace -------------+------------- tspace_1dim | tspace_2dim | tablespace1 --it should now be possible to revoke permissions on tablespace1 SET ROLE :ROLE_DEFAULT_PERM_USER; REVOKE CREATE ON TABLESPACE tablespace1 FROM :ROLE_DEFAULT_PERM_USER_2; SET ROLE :ROLE_DEFAULT_PERM_USER_2; --detach the other tablespace SELECT detach_tablespace('tablespace2', 'tspace_1dim'); detach_tablespace ------------------- 1 SELECT * FROM _timescaledb_catalog.tablespace; id | hypertable_id | tablespace_name ----+---------------+----------------- 1 | 1 | tablespace1 2 | 1 | tablespace2 SELECT * FROM show_tablespaces('tspace_1dim'); show_tablespaces ------------------ SELECT * FROM show_tablespaces('tspace_2dim'); show_tablespaces ------------------ tablespace1 tablespace2 SELECT * FROM hypertable_tablespaces; hypertable | tablespace -------------+------------- tspace_1dim | tspace_2dim | tablespace1 --detaching tablespace2 from a table without permissions should fail SELECT detach_tablespace('tablespace2', 'tspace_2dim'); ERROR: must be owner of hypertable "tspace_2dim" SELECT detach_tablespaces('tspace_2dim'); ERROR: must be owner of hypertable "tspace_2dim" \c :TEST_DBNAME :ROLE_SUPERUSER -- PERM_USER_2 owns tablespace2, and PERM_USER owns the table -- 'tspace_2dim', which has tablespace2 attached. Revoking PERM_USER_2 -- FROM PERM_USER should therefore fail REVOKE :ROLE_DEFAULT_PERM_USER_2 FROM :ROLE_DEFAULT_PERM_USER; ERROR: cannot revoke privilege while tablespace "tablespace2" is attached to hypertable "tspace_2dim" SET ROLE :ROLE_DEFAULT_PERM_USER_2; --set other user should make detach work SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT * INTO detached_all FROM detach_tablespaces('tspace_2dim'); SELECT * FROM detached_all; detach_tablespaces -------------------- 2 SELECT * FROM _timescaledb_catalog.tablespace; id | hypertable_id | tablespace_name ----+---------------+----------------- SELECT * FROM show_tablespaces('tspace_1dim'); show_tablespaces ------------------ SELECT * FROM show_tablespaces('tspace_2dim'); show_tablespaces ------------------ \c :TEST_DBNAME :ROLE_SUPERUSER -- It should now be possible to revoke PERM_USER_2 from PERM_USER -- since tablespace2 is no longer attched to tspace_2dim REVOKE :ROLE_DEFAULT_PERM_USER_2 FROM :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; --detaching twice should fail SELECT detach_tablespace('tablespace2', 'tspace_2dim'); ERROR: tablespace "tablespace2" is not attached to hypertable "tspace_2dim" --adding if_attached should only generate notice SELECT detach_tablespace('tablespace2', 'tspace_2dim', if_attached => true); NOTICE: tablespace "tablespace2" is not attached to hypertable "tspace_2dim", skipping detach_tablespace ------------------- 0 --attach tablespaces again to verify that tablespaces are cleaned up --when tables are dropped \c :TEST_DBNAME :ROLE_SUPERUSER SELECT attach_tablespace('tablespace2', 'tspace_1dim'); attach_tablespace ------------------- SELECT attach_tablespace('tablespace1', 'tspace_2dim'); attach_tablespace ------------------- SELECT * FROM _timescaledb_catalog.tablespace; id | hypertable_id | tablespace_name ----+---------------+----------------- 5 | 2 | tablespace2 6 | 1 | tablespace1 DROP TABLE tspace_1dim; SELECT * FROM _timescaledb_catalog.tablespace; id | hypertable_id | tablespace_name ----+---------------+----------------- 6 | 1 | tablespace1 DROP TABLE tspace_2dim; SELECT * FROM _timescaledb_catalog.tablespace; id | hypertable_id | tablespace_name ----+---------------+----------------- -- Create two tables and attach multiple tablespaces to them. Verify -- that dropping a tablespace from multiple tables work as expected. CREATE TABLE tbl_1(time timestamp, temp float, device text); SELECT create_hypertable('tbl_1', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------- (3,public,tbl_1,t) CREATE TABLE tbl_2(time timestamp, temp float, device text); SELECT create_hypertable('tbl_2', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------- (4,public,tbl_2,t) CREATE TABLE tbl_3(time timestamp, temp float, device text); SELECT create_hypertable('tbl_3', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------- (5,public,tbl_3,t) SELECT * FROM hypertable_tablespaces; hypertable | tablespace ------------+------------ tbl_1 | tbl_2 | tbl_3 | SELECT * FROM show_tablespaces('tbl_1'); show_tablespaces ------------------ SELECT * FROM show_tablespaces('tbl_2'); show_tablespaces ------------------ SELECT * FROM show_tablespaces('tbl_3'); show_tablespaces ------------------ SELECT attach_tablespace('tablespace1', 'tbl_1'); attach_tablespace ------------------- SELECT attach_tablespace('tablespace2', 'tbl_1'); attach_tablespace ------------------- SELECT attach_tablespace('tablespace2', 'tbl_2'); attach_tablespace ------------------- SELECT attach_tablespace('tablespace2', 'tbl_3'); attach_tablespace ------------------- SELECT * FROM hypertable_tablespaces; hypertable | tablespace ------------+------------- tbl_1 | tablespace1 tbl_2 | tablespace2 tbl_3 | tablespace2 SELECT * FROM show_tablespaces('tbl_1'); show_tablespaces ------------------ tablespace1 tablespace2 SELECT * FROM show_tablespaces('tbl_2'); show_tablespaces ------------------ tablespace2 SELECT * FROM show_tablespaces('tbl_3'); show_tablespaces ------------------ tablespace2 SELECT detach_tablespace('tablespace2'); detach_tablespace ------------------- 3 SELECT * FROM hypertable_tablespaces; hypertable | tablespace ------------+------------- tbl_1 | tablespace1 tbl_2 | tbl_3 | SELECT * FROM show_tablespaces('tbl_1'); show_tablespaces ------------------ tablespace1 SELECT * FROM show_tablespaces('tbl_2'); show_tablespaces ------------------ SELECT * FROM show_tablespaces('tbl_3'); show_tablespaces ------------------ DROP TABLE tbl_1; DROP TABLE tbl_2; DROP TABLE tbl_3; -- verify that one cannot DROP a tablespace while it is attached to a -- hypertable CREATE TABLE tbl_1(time timestamp, temp float, device text); SELECT create_hypertable('tbl_1', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------- (6,public,tbl_1,t) SELECT attach_tablespace('tablespace1', 'tbl_1'); attach_tablespace ------------------- SELECT * FROM show_tablespaces('tbl_1'); show_tablespaces ------------------ tablespace1 DROP TABLESPACE tablespace1; ERROR: tablespace "tablespace1" is still attached to 1 hypertables --after detaching we should now be able to drop the tablespace SELECT detach_tablespace('tablespace1', 'tbl_1'); detach_tablespace ------------------- 1 DROP TABLESPACE tablespace1; DROP TABLESPACE tablespace2; ================================================ FILE: test/expected/telemetry.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION _timescaledb_internal.test_status(int) RETURNS JSONB AS :MODULE_PATHNAME, 'ts_test_status' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_status_ssl(int) RETURNS JSONB AS :MODULE_PATHNAME, 'ts_test_status_ssl' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_status_mock(text) RETURNS JSONB AS :MODULE_PATHNAME, 'ts_test_status_mock' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_validate_server_version(response text) RETURNS TEXT AS :MODULE_PATHNAME, 'ts_test_validate_server_version' LANGUAGE C IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_telemetry_main_conn(text, text) RETURNS BOOLEAN AS :MODULE_PATHNAME, 'ts_test_telemetry_main_conn' LANGUAGE C IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_telemetry(host text = NULL, servname text = NULL, port int = NULL) RETURNS JSONB AS :MODULE_PATHNAME, 'ts_test_telemetry' LANGUAGE C IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_privacy() RETURNS BOOLEAN AS :MODULE_PATHNAME, 'ts_test_privacy' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION test_check_version_response(response text) RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_check_version_response' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; INSERT INTO _timescaledb_catalog.metadata VALUES ('foo','bar',TRUE); INSERT INTO _timescaledb_catalog.metadata VALUES ('bar','baz',FALSE); \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT _timescaledb_internal.test_status_ssl(200); test_status_ssl ----------------- {"status": 200} SELECT _timescaledb_internal.test_status_ssl(201); test_status_ssl ----------------- {"status": 201} \set ON_ERROR_STOP 0 SELECT _timescaledb_internal.test_status_ssl(304); ERROR: endpoint sent back unexpected HTTP status: 304 SELECT _timescaledb_internal.test_status_ssl(400); ERROR: endpoint sent back unexpected HTTP status: 400 SELECT _timescaledb_internal.test_status_ssl(401); ERROR: endpoint sent back unexpected HTTP status: 401 SELECT _timescaledb_internal.test_status_ssl(404); ERROR: endpoint sent back unexpected HTTP status: 404 SELECT _timescaledb_internal.test_status_ssl(500); ERROR: endpoint sent back unexpected HTTP status: 500 SELECT _timescaledb_internal.test_status_ssl(503); ERROR: endpoint sent back unexpected HTTP status: 503 \set ON_ERROR_STOP 1 -- This function runs the test 5 times, because each time the internal C function is choosing a random length to send from the server on each socket read. We hit many cases this way. CREATE OR REPLACE FUNCTION mocker(TEXT) RETURNS SETOF TEXT AS $BODY$ DECLARE r TEXT; BEGIN FOR i in 1..5 LOOP SELECT _timescaledb_internal.test_status_mock($1) INTO r; RETURN NEXT r; END LOOP; RETURN; END $BODY$ LANGUAGE 'plpgsql'; select * from mocker( E'HTTP/1.1 200 OK\r\n' 'Content-Type: application/json; charset=utf-8\r\n' 'Date: Thu, 12 Jul 2018 18:33:04 GMT\r\n' 'ETag: W/\"e-upYEWCL+q6R/++2nWHz5b76hBgo\"\r\n' 'Server: nginx\r\n' 'Vary: Accept-Encoding\r\n' 'Content-Length: 14\r\n' 'Connection: Close\r\n\r\n' '{\"status\":200}'); mocker ----------------- {"status": 200} {"status": 200} {"status": 200} {"status": 200} {"status": 200} select * from mocker( E'HTTP/1.1 201 OK\r\n' 'Content-Type: application/json; charset=utf-8\r\n' 'Vary: Accept-Encoding\r\n' 'Content-Length: 14\r\n' 'Connection: Close\r\n\r\n' '{\"status\":201}'); mocker ----------------- {"status": 201} {"status": 201} {"status": 201} {"status": 201} {"status": 201} \set ON_ERROR_STOP 0 \set test_string 'HTTP/1.1 404 Not Found\r\nContent-Length: 14\r\nConnection: Close\r\n\r\n{\"status\":404}'; SELECT _timescaledb_internal.test_status_mock(:'test_string'); ERROR: endpoint sent back unexpected HTTP status: 404 SELECT _timescaledb_internal.test_status_mock(:'test_string'); ERROR: endpoint sent back unexpected HTTP status: 404 SELECT _timescaledb_internal.test_status_mock(:'test_string'); ERROR: endpoint sent back unexpected HTTP status: 404 SELECT _timescaledb_internal.test_status_mock(:'test_string'); ERROR: endpoint sent back unexpected HTTP status: 404 SELECT _timescaledb_internal.test_status_mock(:'test_string'); ERROR: endpoint sent back unexpected HTTP status: 404 \set test_string 'Content-Length: 14\r\nConnection: Close\r\n\r\n{\"status\":404}'; SELECT _timescaledb_internal.test_status_mock(:'test_string'); ERROR: could not parse HTTP response SELECT _timescaledb_internal.test_status_mock(:'test_string'); ERROR: could not parse HTTP response SELECT _timescaledb_internal.test_status_mock(:'test_string'); ERROR: could not parse HTTP response SELECT _timescaledb_internal.test_status_mock(:'test_string'); ERROR: could not parse HTTP response SELECT _timescaledb_internal.test_status_mock(:'test_string'); ERROR: could not parse HTTP response \set test_string 'HTTP/1.1 404 Not Found\r\nContent-Length: 14\r\nConnection: Close\r\n{\"status\":404}'; SELECT _timescaledb_internal.test_status_mock(:'test_string'); ERROR: could not parse HTTP response SELECT _timescaledb_internal.test_status_mock(:'test_string'); ERROR: could not parse HTTP response SELECT _timescaledb_internal.test_status_mock(:'test_string'); ERROR: could not parse HTTP response SELECT _timescaledb_internal.test_status_mock(:'test_string'); ERROR: could not parse HTTP response SELECT _timescaledb_internal.test_status_mock(:'test_string'); ERROR: could not parse HTTP response \set ON_ERROR_STOP 1 -- Test parsing version response SELECT * FROM _timescaledb_internal.test_validate_server_version('{"status": "200", "current_timescaledb_version": "10.1.0"}'); test_validate_server_version ------------------------------ 10.1.0 SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "10.1"}'); test_validate_server_version ------------------------------ 10.1 SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "10"}'); test_validate_server_version ------------------------------ 10 SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "9.2.0"}'); test_validate_server_version ------------------------------ 9.2.0 SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "9.1.2"}'); test_validate_server_version ------------------------------ 9.1.2 SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "1.0.0"}'); test_validate_server_version ------------------------------ 1.0.0 SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "1.0.0-rc1"}'); test_validate_server_version ------------------------------ 1.0.0-rc1 SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "1.0.0-rc2"}'); test_validate_server_version ------------------------------ 1.0.0-rc2 SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "1.0.0-rc1"}'); test_validate_server_version ------------------------------ 1.0.0-rc1 SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "1.0.0-alpha"}'); test_validate_server_version ------------------------------ 1.0.0-alpha SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "123456789"}'); test_validate_server_version ------------------------------ 123456789 SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "!@#$%"}'); test_validate_server_version ------------------------------ SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": ""}'); test_validate_server_version ------------------------------ SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": " 10 "}'); test_validate_server_version ------------------------------ SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "a"}'); test_validate_server_version ------------------------------ a SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "a.b.c"}'); test_validate_server_version ------------------------------ a.b.c SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "10.1.1a"}'); test_validate_server_version ------------------------------ 10.1.1a SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "10.1.1+rc1"}'); test_validate_server_version ------------------------------ SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "10.1.1.1"}'); test_validate_server_version ------------------------------ 10.1.1.1 SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "1.0.0-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"}'); test_validate_server_version ------------------------------ ---------------------------------------------------------------- -- Test well-formed response and valid versions SELECT test_check_version_response('{"current_timescaledb_version": "1.6.1", "is_up_to_date": true}'); NOTICE: the "timescaledb" extension is up-to-date test_check_version_response ----------------------------- SELECT test_check_version_response('{"current_timescaledb_version": "1.6.1", "is_up_to_date": false}'); test_check_version_response ----------------------------- SELECT test_check_version_response('{"current_timescaledb_version": "10.1", "is_up_to_date": false}'); test_check_version_response ----------------------------- SELECT test_check_version_response('{"current_timescaledb_version": "10.1.1-rc1", "is_up_to_date": false}'); test_check_version_response ----------------------------- ---------------------------------------------------------------- -- Test well-formed response but invalid versions SELECT test_check_version_response('{"current_timescaledb_version": "1.0.0-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", "is_up_to_date": false}'); NOTICE: server did not return a valid TimescaleDB version: version string is too long test_check_version_response ----------------------------- SELECT test_check_version_response('{"current_timescaledb_version": "10.1.1+rc1", "is_up_to_date": false}'); NOTICE: server did not return a valid TimescaleDB version: version string has invalid characters test_check_version_response ----------------------------- SELECT test_check_version_response('{"current_timescaledb_version": "@10.1.1", "is_up_to_date": false}'); NOTICE: server did not return a valid TimescaleDB version: version string has invalid characters test_check_version_response ----------------------------- SELECT test_check_version_response('{"current_timescaledb_version": "10.1.1@", "is_up_to_date": false}'); NOTICE: server did not return a valid TimescaleDB version: version string has invalid characters test_check_version_response ----------------------------- ---------------------------------------------------------------- -- Test malformed responses \set ON_ERROR_STOP 0 -- Empty response SELECT test_check_version_response('{}'); ERROR: malformed telemetry response body -- Field "is_up_to_date" missing SELECT test_check_version_response('{"current_timescaledb_version": "1.6.1"}'); ERROR: malformed telemetry response body -- Field "current_timescaledb_version" is missing SELECT test_check_version_response('{"is_up_to_date": false}'); ERROR: malformed telemetry response body \set ON_ERROR_STOP 1 SET timescaledb.last_tune_time = '2024-01-01 00:00:00+00'; SET timescaledb.last_tune_version = '1.2.3'; SET timescaledb.telemetry_level=basic; -- Connect to a bogus host and path to test error handling in telemetry_main() SELECT _timescaledb_internal.test_telemetry_main_conn('noservice.timescale.com', 'path'); NOTICE: telemetry could not connect to "noservice.timescale.com" test_telemetry_main_conn -------------------------- f -- Test telemetry report contents SET timescaledb.telemetry_level=basic; SELECT * FROM jsonb_object_keys(get_telemetry_report()) AS key WHERE key != 'os_name_pretty'; key ------------------------------------ db_uuid license os_name relations os_release os_version data_volume db_metadata replication build_os_name access_methods functions_used install_method installed_time last_tuned_time build_os_version exported_db_uuid instance_metadata stats_by_job_type telemetry_version build_architecture last_tuned_version postgresql_version related_extensions db_telemetry_events timescaledb_version errors_by_sqlerrcode num_reorder_policies num_retention_policies num_compression_policies num_user_defined_actions num_reorder_policies_fixed build_architecture_bit_size num_continuous_aggs_policies num_retention_policies_fixed num_compression_policies_fixed num_user_defined_actions_fixed num_continuous_aggs_policies_fixed CREATE MATERIALIZED VIEW telemetry_report AS SELECT t FROM get_telemetry_report() t; -- check telemetry picks up flagged content from metadata SELECT t -> 'db_metadata' FROM telemetry_report; ?column? ---------------- {"foo": "bar"} -- check timescaledb_telemetry.cloud SELECT t -> 'instance_metadata' FROM telemetry_report; ?column? ----------------- {"cloud": "ci"} -- Check access methods SELECT t->'access_methods' ? 'btree', t->'access_methods' ? 'heap', CAST(t->'access_methods'->'btree'->'pages' AS int) > 0, CAST(t->'access_methods'->'btree'->'instances' AS int) > 0 FROM telemetry_report; ?column? | ?column? | ?column? | ?column? ----------+----------+----------+---------- t | t | t | t WITH t AS ( SELECT t -> 'relations' AS rels FROM telemetry_report ) SELECT rels -> 'hypertables' -> 'num_relations' AS num_hypertables, rels -> 'continuous_aggregates' -> 'num_relations' AS num_caggs FROM t; num_hypertables | num_caggs -----------------+----------- 0 | 0 CREATE TABLE device_readings ( observation_time TIMESTAMPTZ NOT NULL ); SELECT table_name FROM create_hypertable('device_readings', 'observation_time'); table_name ----------------- device_readings REFRESH MATERIALIZED VIEW telemetry_report; WITH t AS ( SELECT t -> 'relations' AS rels FROM telemetry_report ) SELECT rels -> 'hypertables' -> 'num_relations' AS num_hypertables, rels -> 'continuous_aggregates' -> 'num_relations' AS num_caggs FROM t; num_hypertables | num_caggs -----------------+----------- 1 | 0 set datestyle to iso; -- check that installed_time formatting in telemetry report does not depend on local date settings SELECT t -> 'installed_time' AS installed_time FROM telemetry_report \gset set datestyle to sql; SELECT t-> 'installed_time' AS installed_time2 FROM telemetry_report \gset SELECT :'installed_time' = :'installed_time2' AS equal, length(:'installed_time'), length(:'installed_time2'); equal | length | length -------+--------+-------- t | 24 | 24 RESET datestyle; -- test function call telemetry CREATE FUNCTION not_visible_in_telemetry() RETURNS INT AS $$ SELECT 1; $$ LANGUAGE SQL; -- drain old function call telemetry so we have fixed out put; SELECT FROM get_telemetry_report(); -- -- call some arbirary functions SELECT 1 + 1, not_visible_in_telemetry(), 1 + 1, abs(-1), not_visible_in_telemetry() WHERE 1 + 1 = 2; ?column? | not_visible_in_telemetry | ?column? | abs | not_visible_in_telemetry ----------+--------------------------+----------+-----+-------------------------- 2 | 1 | 2 | 1 | 1 -- call some aggregates SELECT min(not_visible_in_telemetry()), sum(not_visible_in_telemetry()); min | sum -----+----- 1 | 1 -- check that we can record from a prepared statement PREPARE record_from_prepared AS SELECT 1 - 1; -- execute 10 times to make sure turning it into generic plan works EXECUTE record_from_prepared; ?column? ---------- 0 EXECUTE record_from_prepared; ?column? ---------- 0 EXECUTE record_from_prepared; ?column? ---------- 0 EXECUTE record_from_prepared; ?column? ---------- 0 EXECUTE record_from_prepared; ?column? ---------- 0 EXECUTE record_from_prepared; ?column? ---------- 0 EXECUTE record_from_prepared; ?column? ---------- 0 EXECUTE record_from_prepared; ?column? ---------- 0 EXECUTE record_from_prepared; ?column? ---------- 0 EXECUTE record_from_prepared; ?column? ---------- 0 DEALLOCATE record_from_prepared; SELECT get_telemetry_report()->'functions_used'; ?column? --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- {"pg_catalog.count()": 1, "pg_catalog.sum(bigint)": 4, "pg_catalog.abs(integer)": 1, "pg_catalog.max(integer)": 2, "pg_catalog.min(integer)": 1, "pg_catalog.sum(integer)": 1, "pg_catalog.int8(numeric)": 4, "pg_catalog.sum(interval)": 2, "pg_catalog.current_database()": 1, "public.get_telemetry_report()": 1, "pg_catalog.text(pg_catalog.name)": 1, "pg_catalog.int4eq(integer,integer)": 3, "pg_catalog.int4mi(integer,integer)": 11, "pg_catalog.int4pl(integer,integer)": 3, "pg_catalog.concat(pg_catalog.\"any\")": 3, "pg_catalog.pg_get_userbyid(pg_catalog.oid)": 1, "pg_catalog.nameeq(pg_catalog.name,pg_catalog.name)": 2, "pg_catalog.texteq(pg_catalog.text,pg_catalog.text)": 1, "pg_catalog.nameregexeq(pg_catalog.name,pg_catalog.text)": 1, "pg_catalog.textregexeq(pg_catalog.text,pg_catalog.text)": 1, "pg_catalog.jsonb_object_agg(pg_catalog.\"any\",pg_catalog.\"any\")": 1, "pg_catalog.jsonb_object_field(pg_catalog.jsonb,pg_catalog.text)": 16, "pg_catalog.jsonb_object_field_text(pg_catalog.jsonb,pg_catalog.text)": 15, "pg_catalog.pg_has_role(pg_catalog.name,pg_catalog.oid,pg_catalog.text)": 1, "pg_catalog.pg_has_role(pg_catalog.name,pg_catalog.name,pg_catalog.text)": 1} -- check the report again to see if resetting works SELECT get_telemetry_report()->'functions_used'; ?column? ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- {"pg_catalog.count()": 1, "pg_catalog.sum(bigint)": 4, "pg_catalog.max(integer)": 2, "pg_catalog.int8(numeric)": 4, "pg_catalog.sum(interval)": 2, "pg_catalog.current_database()": 1, "public.get_telemetry_report()": 1, "pg_catalog.text(pg_catalog.name)": 1, "pg_catalog.int4eq(integer,integer)": 2, "pg_catalog.concat(pg_catalog.\"any\")": 3, "pg_catalog.pg_get_userbyid(pg_catalog.oid)": 1, "pg_catalog.nameeq(pg_catalog.name,pg_catalog.name)": 2, "pg_catalog.texteq(pg_catalog.text,pg_catalog.text)": 1, "pg_catalog.nameregexeq(pg_catalog.name,pg_catalog.text)": 1, "pg_catalog.textregexeq(pg_catalog.text,pg_catalog.text)": 1, "pg_catalog.jsonb_object_agg(pg_catalog.\"any\",pg_catalog.\"any\")": 1, "pg_catalog.jsonb_object_field(pg_catalog.jsonb,pg_catalog.text)": 16, "pg_catalog.jsonb_object_field_text(pg_catalog.jsonb,pg_catalog.text)": 15, "pg_catalog.pg_has_role(pg_catalog.name,pg_catalog.oid,pg_catalog.text)": 1, "pg_catalog.pg_has_role(pg_catalog.name,pg_catalog.name,pg_catalog.text)": 1} \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE _timescaledb_catalog.metadata; SET timescaledb.telemetry_level=off; -- returns false which means telemetry got canceled SELECT * FROM _timescaledb_internal.test_privacy(); test_privacy -------------- f RESET timescaledb.telemetry_level; -- returns false which means telemetry got canceled SELECT * FROM _timescaledb_internal.test_privacy(); test_privacy -------------- f -- To make sure nothing was sent, we check the UUID table to make sure no exported UUID row was created SELECT key from _timescaledb_catalog.metadata; key ----- \set ON_ERROR_STOP 0 -- test that the telemetry gathering code doesn't break nonexistent statements EXECUTE noexistent_statement; ERROR: prepared statement "noexistent_statement" does not exist \c :TEST_DBNAME :ROLE_SUPERUSER -- Insert some data into the telemetry event table INSERT INTO _timescaledb_catalog.telemetry_event(tag, body) VALUES ('ummagumma', '{"title": "Careful with that Axe Eugene!"}'), ('kaboom', '{"title": "Where is that kaboom?"}'); -- Check that it is present in the telemetry report SELECT * FROM jsonb_to_recordset(get_telemetry_report()->'db_telemetry_events') AS x(tag name, body text); tag | body -----------+-------------------------------------------- ummagumma | {"title": "Careful with that Axe Eugene!"} kaboom | {"title": "Where is that kaboom?"} ================================================ FILE: test/expected/test_tss_callbacks.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION test.setup_tss_hook_v0() RETURNS VOID AS :MODULE_PATHNAME, 'ts_setup_tss_hook_v0' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION test.setup_tss_hook_v1() RETURNS VOID AS :MODULE_PATHNAME, 'ts_setup_tss_hook_v1' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION test.teardown_tss_hook_v1() RETURNS VOID AS :MODULE_PATHNAME, 'ts_teardown_tss_hook_v1' LANGUAGE C VOLATILE; SELECT test.setup_tss_hook_v1(); setup_tss_hook_v1 ------------------- CREATE TABLE copy_test ( "time" timestamptz NOT NULL, "value" double precision NOT NULL ); SELECT create_hypertable('copy_test', 'time'); create_hypertable ------------------------ (1,public,copy_test,t) -- We should se a mock message COPY copy_test FROM STDIN DELIMITER ','; NOTICE: test_tss_callbacks (mock): query=COPY copy_test FROM STDIN DELIMITER ',';, len=39, id=0, rows=2 SELECT test.teardown_tss_hook_v1(); teardown_tss_hook_v1 ---------------------- -- Without the hook registered we cannot see the mock message COPY copy_test FROM STDIN DELIMITER ','; -- Test for mismatch version SELECT test.setup_tss_hook_v0(); setup_tss_hook_v0 ------------------- -- Warning because the mismatch versions COPY copy_test FROM STDIN DELIMITER ','; WARNING: version mismatch between timescaledb and ts_stat_statements callbacks WARNING: version mismatch between timescaledb and ts_stat_statements callbacks ================================================ FILE: test/expected/test_utils.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION test.condition() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_utils_condition' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION test.int64_eq() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_utils_int64_eq' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION test.ptr_eq() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_utils_ptr_eq' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION test.double_eq() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_utils_double_eq' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; SET ROLE :ROLE_DEFAULT_PERM_USER; -- We're testing that the test utils work and generate errors on -- failing conditions \set ON_ERROR_STOP 0 SELECT test.condition(); WARNING: TestFailure in ts_test_utils_condition() at line:43 ERROR: TestFailure | (true_value == false_value) SELECT test.int64_eq(); WARNING: TestFailure in ts_test_utils_int64_eq() at line:53 ERROR: TestFailure | (big == small) [32532978 == 3242234] SELECT test.ptr_eq(); WARNING: TestFailure in ts_test_utils_ptr_eq() at line:67 ERROR: TestFailure | (true_ptr == false_ptr) SELECT test.double_eq(); WARNING: TestFailure in ts_test_utils_double_eq() at line:78 ERROR: TestFailure | (big_double == small_double) [923423478.324200 == 324.300000] \set ON_ERROR_STOP 1 -- Test debug points -- \set ECHO all \c :TEST_DBNAME :ROLE_SUPERUSER -- debug point already enabled SELECT debug_waitpoint_enable('test_debug_point'); debug_waitpoint_enable ------------------------ \set ON_ERROR_STOP 0 SELECT debug_waitpoint_enable('test_debug_point'); ERROR: debug point "test_debug_point" already enabled \set ON_ERROR_STOP 1 SELECT debug_waitpoint_release('test_debug_point'); debug_waitpoint_release ------------------------- -- debug point not enabled \set ON_ERROR_STOP 0 SELECT debug_waitpoint_release('test_debug_point'); WARNING: you don't own a lock of type ExclusiveLock ERROR: cannot release debug point "test_debug_point" \set ON_ERROR_STOP 1 -- error injections -- CREATE OR REPLACE FUNCTION test_error_injection(TEXT) RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_error_injection' LANGUAGE C VOLATILE STRICT; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT test_error_injection('test_error'); test_error_injection ---------------------- SELECT debug_waitpoint_enable('test_error'); debug_waitpoint_enable ------------------------ \set ON_ERROR_STOP 0 SELECT test_error_injection('test_error'); ERROR: error injected at debug point 'test_error' \set ON_ERROR_STOP 1 SELECT debug_waitpoint_release('test_error'); debug_waitpoint_release ------------------------- SELECT test_error_injection('test_error'); test_error_injection ---------------------- -- Test Scanner RESET ROLE; CREATE OR REPLACE FUNCTION test.scanner() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_scanner' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; SET ROLE :ROLE_DEFAULT_PERM_USER; -- Create two chunks to scan in the test CREATE TABLE hyper (time timestamptz, temp float); SELECT create_hypertable('hyper', 'time'); create_hypertable -------------------- (1,public,hyper,t) INSERT INTO hyper VALUES ('2021-01-01', 1.0), ('2022-01-01', 2.0); SELECT test.scanner(); NOTICE: 1. Scan: "_timescaledb_internal._hyper_1_1_chunk" NOTICE: 1. Scan: "_timescaledb_internal._hyper_1_2_chunk" NOTICE: 2. Scan with filter: "_timescaledb_internal._hyper_1_1_chunk" NOTICE: 3. ReScan: "_timescaledb_internal._hyper_1_2_chunk" NOTICE: 4. IndexScan: "_timescaledb_internal._hyper_1_2_chunk" scanner --------- -- Test errdata_to_jsonb RESET ROLE; CREATE OR REPLACE FUNCTION test.errdata_to_jsonb() RETURNS JSONB AS :MODULE_PATHNAME, 'ts_test_errdata_to_jsonb' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; SELECT test.errdata_to_jsonb(); errdata_to_jsonb ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- {"hint": "test error hint", "detail": "test error detail", "domain": "test error domain", "lineno": 123, "context": "test error context", "message": "test error message", "filename": "test error filename", "funcname": "test error function", "proc_name": "proc_name", "detail_log": "test error detail log", "sqlerrcode": "22023", "table_name": "test error table", "column_name": "test error column", "proc_schema": "proc_schema", "schema_name": "test error schema", "datatype_name": "test error datatype", "internalquery": "test error internal query", "context_domain": "test error context domain", "constraint_name": "test error constraint"} ================================================ FILE: test/expected/timestamp-15.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Utility function for grouping/slotting time with a given interval. CREATE OR REPLACE FUNCTION date_group( field timestamp, group_interval interval ) RETURNS timestamp LANGUAGE SQL STABLE AS $BODY$ SELECT to_timestamp((EXTRACT(EPOCH from $1)::int / EXTRACT(EPOCH from group_interval)::int) * EXTRACT(EPOCH from group_interval)::int)::timestamp; $BODY$; CREATE TABLE PUBLIC."testNs" ( "timeCustom" TIMESTAMP NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."testNs" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA "testNs" AUTHORIZATION :ROLE_DEFAULT_PERM_USER; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT * FROM create_hypertable('"public"."testNs"', 'timeCustom', 'device_id', 2, associated_schema_name=>'testNs' ); WARNING: column type "timestamp without time zone" used for "timeCustom" does not follow best practices hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | testNs | t \c :TEST_DBNAME INSERT INTO PUBLIC."testNs"("timeCustom", device_id, series_0, series_1) VALUES ('2009-11-12T01:00:00+00:00', 'dev1', 1.5, 1), ('2009-11-12T01:00:00+00:00', 'dev1', 1.5, 2), ('2009-11-10T23:00:02+00:00', 'dev1', 2.5, 3); INSERT INTO PUBLIC."testNs"("timeCustom", device_id, series_0, series_1) VALUES ('2009-11-10T23:00:00+00:00', 'dev2', 1.5, 1), ('2009-11-10T23:00:00+00:00', 'dev2', 1.5, 2); SELECT * FROM PUBLIC."testNs"; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool --------------------------+-----------+----------+----------+----------+------------- Thu Nov 12 01:00:00 2009 | dev1 | 1.5 | 1 | | Thu Nov 12 01:00:00 2009 | dev1 | 1.5 | 2 | | Tue Nov 10 23:00:02 2009 | dev1 | 2.5 | 3 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 1 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 2 | | SET client_min_messages = WARNING; \echo 'The next 2 queries will differ in output between UTC and EST since the mod is on the 100th hour UTC' The next 2 queries will differ in output between UTC and EST since the mod is on the 100th hour UTC SET timezone = 'UTC'; SELECT date_group("timeCustom", '100 days') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC; time | sum --------------------------+----- Sun Sep 13 00:00:00 2009 | 8.5 SET timezone = 'EST'; SELECT date_group("timeCustom", '100 days') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC; time | sum --------------------------+----- Sat Sep 12 19:00:00 2009 | 8.5 \echo 'The rest of the queries will be the same in output between UTC and EST' The rest of the queries will be the same in output between UTC and EST SET timezone = 'UTC'; SELECT date_group("timeCustom", '1 day') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC; time | sum --------------------------+----- Tue Nov 10 00:00:00 2009 | 5.5 Thu Nov 12 00:00:00 2009 | 3 SET timezone = 'EST'; SELECT date_group("timeCustom", '1 day') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC; time | sum --------------------------+----- Mon Nov 09 19:00:00 2009 | 5.5 Wed Nov 11 19:00:00 2009 | 3 SET timezone = 'UTC'; SELECT * FROM PUBLIC."testNs" WHERE "timeCustom" >= TIMESTAMP '2009-11-10T23:00:00' AND "timeCustom" < TIMESTAMP '2009-11-12T01:00:00' ORDER BY "timeCustom" DESC, device_id, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool --------------------------+-----------+----------+----------+----------+------------- Tue Nov 10 23:00:02 2009 | dev1 | 2.5 | 3 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 1 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 2 | | SET timezone = 'EST'; SELECT * FROM PUBLIC."testNs" WHERE "timeCustom" >= TIMESTAMP '2009-11-10T23:00:00' AND "timeCustom" < TIMESTAMP '2009-11-12T01:00:00' ORDER BY "timeCustom" DESC, device_id, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool --------------------------+-----------+----------+----------+----------+------------- Tue Nov 10 23:00:02 2009 | dev1 | 2.5 | 3 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 1 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 2 | | SET timezone = 'UTC'; SELECT date_group("timeCustom", '1 day') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC LIMIT 2; time | sum --------------------------+----- Tue Nov 10 00:00:00 2009 | 5.5 Thu Nov 12 00:00:00 2009 | 3 SET timezone = 'EST'; SELECT date_group("timeCustom", '1 day') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC LIMIT 2; time | sum --------------------------+----- Mon Nov 09 19:00:00 2009 | 5.5 Wed Nov 11 19:00:00 2009 | 3 ------------------------------------ -- Test time conversion functions -- ------------------------------------ \set ON_ERROR_STOP 0 SET timezone = 'UTC'; -- Conversion to timestamp using Postgres built-in function taking -- double. Gives inaccurate result on Postgres <= 9.6.2. Accurate on -- Postgres >= 9.6.3. SELECT to_timestamp(1486480176.236538); to_timestamp ------------------------------------- Tue Feb 07 15:09:36.236538 2017 UTC -- extension-specific version taking microsecond UNIX timestamp SELECT _timescaledb_functions.to_timestamp(1486480176236538); to_timestamp ------------------------------------- Tue Feb 07 15:09:36.236538 2017 UTC -- Should be the inverse of the statement above. SELECT _timescaledb_functions.to_unix_microseconds('2017-02-07 15:09:36.236538+00'); to_unix_microseconds ---------------------- 1486480176236538 -- For timestamps, BIGINT MAX represents +Infinity and BIGINT MIN -- -Infinity. We keep this notion for UNIX epoch time: SELECT _timescaledb_functions.to_unix_microseconds('+infinity'); ERROR: invalid input syntax for type timestamp with time zone: "+infinity" at character 52 SELECT _timescaledb_functions.to_timestamp(9223372036854775807); to_timestamp -------------- infinity SELECT _timescaledb_functions.to_unix_microseconds('-infinity'); to_unix_microseconds ---------------------- -9223372036854775808 SELECT _timescaledb_functions.to_timestamp(-9223372036854775808); to_timestamp -------------- -infinity -- In UNIX microseconds, the largest bigint value below infinity -- (BIGINT MAX) is smaller than internal date upper bound and should -- therefore be OK. Further, converting to the internal postgres epoch -- cannot overflow a 64-bit INTEGER since the postgres epoch is at a -- later date compared to the UNIX epoch, and is therefore represented -- by a smaller number SELECT _timescaledb_functions.to_timestamp(9223372036854775806); to_timestamp --------------------------------------- Sun Jan 10 04:00:54.775806 294247 UTC -- Julian day zero is -210866803200000000 microseconds from UNIX epoch SELECT _timescaledb_functions.to_timestamp(-210866803200000000); to_timestamp --------------------------------- Mon Nov 24 00:00:00 4714 UTC BC \set VERBOSITY default -- Going beyond Julian day zero should give out-of-range error SELECT _timescaledb_functions.to_timestamp(-210866803200000001); ERROR: timestamp out of range -- Lower bound on date (should return the Julian day zero UNIX timestamp above) SELECT _timescaledb_functions.to_unix_microseconds('4714-11-24 00:00:00+00 BC'); to_unix_microseconds ---------------------- -210866803200000000 -- Going beyond lower bound on date should return out-of-range SELECT _timescaledb_functions.to_unix_microseconds('4714-11-23 23:59:59.999999+00 BC'); ERROR: timestamp out of range: "4714-11-23 23:59:59.999999+00 BC" LINE 1: ...ELECT _timescaledb_functions.to_unix_microseconds('4714-11-2... ^ -- The upper bound for Postgres TIMESTAMPTZ SELECT timestamp '294276-12-31 23:59:59.999999+00'; timestamp ----------------------------------- Sun Dec 31 23:59:59.999999 294276 -- Going beyond the upper bound, should fail SELECT timestamp '294276-12-31 23:59:59.999999+00' + interval '1 us'; ERROR: timestamp out of range -- Cannot represent the upper bound timestamp with a UNIX microsecond timestamp -- since the Postgres epoch is at a later date than the UNIX epoch. SELECT _timescaledb_functions.to_unix_microseconds('294276-12-31 23:59:59.999999+00'); ERROR: timestamp out of range -- Subtracting the difference between the two epochs (10957 days) should bring -- us within range. SELECT timestamp '294276-12-31 23:59:59.999999+00' - interval '10957 days'; ?column? ----------------------------------- Fri Jan 01 23:59:59.999999 294247 SELECT _timescaledb_functions.to_unix_microseconds('294247-01-01 23:59:59.999999'); to_unix_microseconds ---------------------- 9223371331199999999 -- Adding one microsecond should take us out-of-range again SELECT timestamp '294247-01-01 23:59:59.999999' + interval '1 us'; ?column? ---------------------------- Sat Jan 02 00:00:00 294247 SELECT _timescaledb_functions.to_unix_microseconds(timestamp '294247-01-01 23:59:59.999999' + interval '1 us'); ERROR: timestamp out of range --no time_bucketing of dates not by integer # of days SELECT time_bucket('1 hour', DATE '2012-01-01'); ERROR: interval must not have sub-day precision SELECT time_bucket('25 hour', DATE '2012-01-01'); ERROR: interval must be a multiple of a day \set ON_ERROR_STOP 1 SELECT time_bucket(INTERVAL '1 day', TIMESTAMP '2011-01-02 01:01:01'); time_bucket -------------------------- Sun Jan 02 00:00:00 2011 SELECT time, time_bucket(INTERVAL '2 day ', time) FROM unnest(ARRAY[ TIMESTAMP '2011-01-01 01:01:01', TIMESTAMP '2011-01-02 01:01:01', TIMESTAMP '2011-01-03 01:01:01', TIMESTAMP '2011-01-04 01:01:01' ]) AS time; time | time_bucket --------------------------+-------------------------- Sat Jan 01 01:01:01 2011 | Sat Jan 01 00:00:00 2011 Sun Jan 02 01:01:01 2011 | Sat Jan 01 00:00:00 2011 Mon Jan 03 01:01:01 2011 | Mon Jan 03 00:00:00 2011 Tue Jan 04 01:01:01 2011 | Mon Jan 03 00:00:00 2011 SELECT int_def, time_bucket(int_def,TIMESTAMP '2011-01-02 01:01:01.111') FROM unnest(ARRAY[ INTERVAL '1 millisecond', INTERVAL '1 second', INTERVAL '1 minute', INTERVAL '1 hour', INTERVAL '1 day', INTERVAL '2 millisecond', INTERVAL '2 second', INTERVAL '2 minute', INTERVAL '2 hour', INTERVAL '2 day' ]) AS int_def; int_def | time_bucket --------------+------------------------------ @ 0.001 secs | Sun Jan 02 01:01:01.111 2011 @ 1 sec | Sun Jan 02 01:01:01 2011 @ 1 min | Sun Jan 02 01:01:00 2011 @ 1 hour | Sun Jan 02 01:00:00 2011 @ 1 day | Sun Jan 02 00:00:00 2011 @ 0.002 secs | Sun Jan 02 01:01:01.11 2011 @ 2 secs | Sun Jan 02 01:01:00 2011 @ 2 mins | Sun Jan 02 01:00:00 2011 @ 2 hours | Sun Jan 02 00:00:00 2011 @ 2 days | Sat Jan 01 00:00:00 2011 \set ON_ERROR_STOP 0 SELECT time_bucket(INTERVAL '1 year 1d',TIMESTAMP '2011-01-02 01:01:01.111'); ERROR: month intervals cannot have day or time component SELECT time_bucket(INTERVAL '1 month 1 minute',TIMESTAMP '2011-01-02 01:01:01.111'); ERROR: month intervals cannot have day or time component \set ON_ERROR_STOP 1 SELECT time, time_bucket(INTERVAL '5 minute', time) FROM unnest(ARRAY[ TIMESTAMP '1970-01-01 00:59:59.999999', TIMESTAMP '1970-01-01 01:01:00', TIMESTAMP '1970-01-01 01:04:59.999999', TIMESTAMP '1970-01-01 01:05:00' ]) AS time; time | time_bucket ---------------------------------+-------------------------- Thu Jan 01 00:59:59.999999 1970 | Thu Jan 01 00:55:00 1970 Thu Jan 01 01:01:00 1970 | Thu Jan 01 01:00:00 1970 Thu Jan 01 01:04:59.999999 1970 | Thu Jan 01 01:00:00 1970 Thu Jan 01 01:05:00 1970 | Thu Jan 01 01:05:00 1970 SELECT time, time_bucket(INTERVAL '5 minute', time) FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:04:59.999999', TIMESTAMP '2011-01-02 01:05:00', TIMESTAMP '2011-01-02 01:09:59.999999', TIMESTAMP '2011-01-02 01:10:00' ]) AS time; time | time_bucket ---------------------------------+-------------------------- Sun Jan 02 01:04:59.999999 2011 | Sun Jan 02 01:00:00 2011 Sun Jan 02 01:05:00 2011 | Sun Jan 02 01:05:00 2011 Sun Jan 02 01:09:59.999999 2011 | Sun Jan 02 01:05:00 2011 Sun Jan 02 01:10:00 2011 | Sun Jan 02 01:10:00 2011 --offset with interval SELECT time, time_bucket(INTERVAL '5 minute', time , INTERVAL '2 minutes') FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:01:59.999999', TIMESTAMP '2011-01-02 01:02:00', TIMESTAMP '2011-01-02 01:06:59.999999', TIMESTAMP '2011-01-02 01:07:00' ]) AS time; time | time_bucket ---------------------------------+-------------------------- Sun Jan 02 01:01:59.999999 2011 | Sun Jan 02 00:57:00 2011 Sun Jan 02 01:02:00 2011 | Sun Jan 02 01:02:00 2011 Sun Jan 02 01:06:59.999999 2011 | Sun Jan 02 01:02:00 2011 Sun Jan 02 01:07:00 2011 | Sun Jan 02 01:07:00 2011 SELECT time, time_bucket(INTERVAL '5 minute', time , - INTERVAL '2 minutes') FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:02:59.999999', TIMESTAMP '2011-01-02 01:03:00', TIMESTAMP '2011-01-02 01:07:59.999999', TIMESTAMP '2011-01-02 01:08:00' ]) AS time; time | time_bucket ---------------------------------+-------------------------- Sun Jan 02 01:02:59.999999 2011 | Sun Jan 02 00:58:00 2011 Sun Jan 02 01:03:00 2011 | Sun Jan 02 01:03:00 2011 Sun Jan 02 01:07:59.999999 2011 | Sun Jan 02 01:03:00 2011 Sun Jan 02 01:08:00 2011 | Sun Jan 02 01:08:00 2011 --offset with infinity -- timestamp SELECT time, time_bucket(INTERVAL '1 week', time, INTERVAL '1 day') FROM unnest(ARRAY[ timestamp '-Infinity', timestamp 'Infinity' ]) AS time; time | time_bucket -----------+------------- -infinity | -infinity infinity | infinity -- timestamptz SELECT time, time_bucket(INTERVAL '1 week', time, INTERVAL '1 day') FROM unnest(ARRAY[ timestamp with time zone '-Infinity', timestamp with time zone 'Infinity' ]) AS time; time | time_bucket -----------+------------- -infinity | -infinity infinity | infinity -- Date SELECT date, time_bucket(INTERVAL '1 week', date, INTERVAL '1 day') FROM unnest(ARRAY[ date '-Infinity', date 'Infinity' ]) AS date; date | time_bucket -----------+------------- -infinity | -infinity infinity | infinity --example to align with an origin SELECT time, time_bucket(INTERVAL '5 minute', time - (TIMESTAMP '2011-01-02 00:02:00' - TIMESTAMP 'epoch')) + (TIMESTAMP '2011-01-02 00:02:00'-TIMESTAMP 'epoch') FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:01:59.999999', TIMESTAMP '2011-01-02 01:02:00', TIMESTAMP '2011-01-02 01:06:59.999999', TIMESTAMP '2011-01-02 01:07:00' ]) AS time; time | ?column? ---------------------------------+-------------------------- Sun Jan 02 01:01:59.999999 2011 | Sun Jan 02 00:57:00 2011 Sun Jan 02 01:02:00 2011 | Sun Jan 02 01:02:00 2011 Sun Jan 02 01:06:59.999999 2011 | Sun Jan 02 01:02:00 2011 Sun Jan 02 01:07:00 2011 | Sun Jan 02 01:07:00 2011 --rounding version SELECT time, time_bucket(INTERVAL '5 minute', time , - INTERVAL '2.5 minutes') + INTERVAL '2 minutes 30 seconds' FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:05:01', TIMESTAMP '2011-01-02 01:07:29', TIMESTAMP '2011-01-02 01:02:30', TIMESTAMP '2011-01-02 01:07:30', TIMESTAMP '2011-01-02 01:02:29' ]) AS time; time | ?column? --------------------------+-------------------------- Sun Jan 02 01:05:01 2011 | Sun Jan 02 01:05:00 2011 Sun Jan 02 01:07:29 2011 | Sun Jan 02 01:05:00 2011 Sun Jan 02 01:02:30 2011 | Sun Jan 02 01:05:00 2011 Sun Jan 02 01:07:30 2011 | Sun Jan 02 01:10:00 2011 Sun Jan 02 01:02:29 2011 | Sun Jan 02 01:00:00 2011 --time_bucket with timezone should mimick date_trunc SET timezone TO 'UTC'; SELECT time, time_bucket(INTERVAL '1 hour', time), date_trunc('hour', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Jan 02 01:01:01 2011 UTC | Sun Jan 02 01:00:00 2011 UTC | Sun Jan 02 01:00:00 2011 UTC Sun Jan 02 00:01:01 2011 UTC | Sun Jan 02 00:00:00 2011 UTC | Sun Jan 02 00:00:00 2011 UTC Sat Jan 01 23:01:01 2011 UTC | Sat Jan 01 23:00:00 2011 UTC | Sat Jan 01 23:00:00 2011 UTC SELECT time, time_bucket(INTERVAL '1 day', time), date_trunc('day', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Jan 02 01:01:01 2011 UTC | Sun Jan 02 00:00:00 2011 UTC | Sun Jan 02 00:00:00 2011 UTC Sun Jan 02 00:01:01 2011 UTC | Sun Jan 02 00:00:00 2011 UTC | Sun Jan 02 00:00:00 2011 UTC Sat Jan 01 23:01:01 2011 UTC | Sat Jan 01 00:00:00 2011 UTC | Sat Jan 01 00:00:00 2011 UTC --what happens with a local tz SET timezone TO 'America/New_York'; SELECT time, time_bucket(INTERVAL '1 hour', time), date_trunc('hour', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Jan 02 01:01:01 2011 EST | Sun Jan 02 01:00:00 2011 EST | Sun Jan 02 01:00:00 2011 EST Sat Jan 01 19:01:01 2011 EST | Sat Jan 01 19:00:00 2011 EST | Sat Jan 01 19:00:00 2011 EST Sat Jan 01 18:01:01 2011 EST | Sat Jan 01 18:00:00 2011 EST | Sat Jan 01 18:00:00 2011 EST --Note the timestamp tz input is aligned with UTC day /not/ local day. different than date_trunc. SELECT time, time_bucket(INTERVAL '1 day', time), date_trunc('day', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-03 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-04 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Jan 02 01:01:01 2011 EST | Sat Jan 01 19:00:00 2011 EST | Sun Jan 02 00:00:00 2011 EST Sun Jan 02 19:01:01 2011 EST | Sun Jan 02 19:00:00 2011 EST | Sun Jan 02 00:00:00 2011 EST Mon Jan 03 18:01:01 2011 EST | Sun Jan 02 19:00:00 2011 EST | Mon Jan 03 00:00:00 2011 EST --can force local bucketing with simple cast. SELECT time, time_bucket(INTERVAL '1 day', time::timestamp), date_trunc('day', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-03 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-04 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+--------------------------+------------------------------ Sun Jan 02 01:01:01 2011 EST | Sun Jan 02 00:00:00 2011 | Sun Jan 02 00:00:00 2011 EST Sun Jan 02 19:01:01 2011 EST | Sun Jan 02 00:00:00 2011 | Sun Jan 02 00:00:00 2011 EST Mon Jan 03 18:01:01 2011 EST | Mon Jan 03 00:00:00 2011 | Mon Jan 03 00:00:00 2011 EST --can also use interval to correct SELECT time, time_bucket(INTERVAL '1 day', time, -INTERVAL '19 hours'), date_trunc('day', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-03 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-04 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Jan 02 01:01:01 2011 EST | Sun Jan 02 00:00:00 2011 EST | Sun Jan 02 00:00:00 2011 EST Sun Jan 02 19:01:01 2011 EST | Sun Jan 02 00:00:00 2011 EST | Sun Jan 02 00:00:00 2011 EST Mon Jan 03 18:01:01 2011 EST | Mon Jan 03 00:00:00 2011 EST | Mon Jan 03 00:00:00 2011 EST --dst: same local hour bucketed as two different hours. SELECT time, time_bucket(INTERVAL '1 hour', time), date_trunc('hour', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2017-11-05 12:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 13:05:00+07' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Nov 05 01:05:00 2017 EDT | Sun Nov 05 01:00:00 2017 EDT | Sun Nov 05 01:00:00 2017 EDT Sun Nov 05 01:05:00 2017 EST | Sun Nov 05 01:00:00 2017 EST | Sun Nov 05 01:00:00 2017 EST --local alignment changes when bucketing by UTC across dst boundary SELECT time, time_bucket(INTERVAL '2 hour', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2017-11-05 10:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 12:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 13:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 15:05:00+07' ]) AS time; time | time_bucket ------------------------------+------------------------------ Sat Nov 04 23:05:00 2017 EDT | Sat Nov 04 22:00:00 2017 EDT Sun Nov 05 01:05:00 2017 EDT | Sun Nov 05 00:00:00 2017 EDT Sun Nov 05 01:05:00 2017 EST | Sun Nov 05 01:00:00 2017 EST Sun Nov 05 03:05:00 2017 EST | Sun Nov 05 03:00:00 2017 EST --local alignment is preserved when bucketing by local time across DST boundary. SELECT time, time_bucket(INTERVAL '2 hour', time::timestamp) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2017-11-05 10:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 12:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 13:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 15:05:00+07' ]) AS time; time | time_bucket ------------------------------+-------------------------- Sat Nov 04 23:05:00 2017 EDT | Sat Nov 04 22:00:00 2017 Sun Nov 05 01:05:00 2017 EDT | Sun Nov 05 00:00:00 2017 Sun Nov 05 01:05:00 2017 EST | Sun Nov 05 00:00:00 2017 Sun Nov 05 03:05:00 2017 EST | Sun Nov 05 02:00:00 2017 -- GitHub issue #7059: time_bucket with timezone + offset across DST boundary -- Asia/Amman: clocks skip from 00:00 to 01:00 on 2021-03-26 -- Input: 01:00+03 = 22:00 UTC → Result: 22:15 UTC = 01:15 local (00:00 + 15min offset) SELECT time_bucket('1 day', '2021-03-26 01:00:00+03'::timestamptz, timezone := 'Asia/Amman', "offset" := '15 minutes'::interval); time_bucket ------------------------------ Wed Mar 24 18:15:00 2021 EDT -- GitHub issue #8851: time_bucket with negative offset during DST fall-back -- Europe/Berlin: clocks repeat 02:00-02:59 on 2025-10-26 -- Input: 02:00+02 = 00:00 UTC → Result: 23:59:45 UTC = 01:59:45 local (02:00 - 15s offset) SELECT time_bucket('30 seconds', '2025-10-26 02:00:00+02'::timestamptz, timezone := 'Europe/Berlin', "offset" := '-15 seconds'::interval); time_bucket ------------------------------ Sat Oct 25 19:59:45 2025 EDT -- Additional DST edge cases for coverage of DST direction × offset sign combinations -- Spring-forward + negative offset -- Input: 01:30+03 = 22:30 UTC → Result: 22:45 UTC = 01:45 local (01:00 + 45min = 02:00 - 15min) SELECT time_bucket('1 hour', '2021-03-26 01:30:00+03'::timestamptz, timezone := 'Asia/Amman', "offset" := '-15 minutes'::interval); time_bucket ------------------------------ Thu Mar 25 17:45:00 2021 EDT -- Fall-back + positive offset -- Input: 02:30+01 = 01:30 UTC → Result: 01:15 UTC = 02:15 local (02:00 + 15min offset) SELECT time_bucket('1 hour', '2025-10-26 02:30:00+01'::timestamptz, timezone := 'Europe/Berlin', "offset" := '15 minutes'::interval); time_bucket ------------------------------ Sat Oct 25 21:15:00 2025 EDT -- Input exactly at DST spring-forward transition -- Input: 22:00 UTC = 00:00 local (the moment clocks jump to 01:00) -- Result: 22:15 UTC = 01:15 local (01:00 + 15min offset) SELECT time_bucket('1 hour', '2021-03-25 22:00:00+00'::timestamptz, timezone := 'Asia/Amman', "offset" := '15 minutes'::interval); time_bucket ------------------------------ Thu Mar 25 17:15:00 2021 EDT -- Input exactly at DST fall-back transition -- Input: 01:00 UTC = 03:00 CEST (the moment clocks go back to 02:00 CET) -- Result: 23:15 UTC = 01:15 local (01:00 + 15min offset, but in CET now) SELECT time_bucket('1 hour', '2025-10-26 01:00:00+00'::timestamptz, timezone := 'Europe/Berlin', "offset" := '15 minutes'::interval); time_bucket ------------------------------ Sat Oct 25 20:15:00 2025 EDT -- Offset larger than bucket size (1h offset with 30min bucket) -- Input: 01:30+03 = 22:30 UTC → Result: 22:30 UTC = 01:30 local (01:00 + 30min = 00:30 + 1h) SELECT time_bucket('30 minutes', '2021-03-26 01:30:00+03'::timestamptz, timezone := 'Asia/Amman', "offset" := '1 hour'::interval); time_bucket ------------------------------ Thu Mar 25 18:30:00 2021 EDT -- GitHub issue #9136: time_bucket with origin during DST fall-back -- When origin is in standard time but timestamp is in daylight time, -- the bucket could incorrectly start AFTER the timestamp. -- America/New_York: clocks go back at 02:00 EDT on 2024-11-03 -- Input: 01:30-04 (EDT) = 05:30 UTC; origin in EST -- Result should have bucket start <= timestamp (bucket in EDT, not EST) SELECT time_bucket('1 hour', '2024-11-03 01:30:00-04'::timestamptz, 'America/New_York', '2000-01-01 00:00:00 America/New_York'::timestamptz) as bucket, '2024-11-03 01:30:00-04'::timestamptz < time_bucket('1 hour', '2024-11-03 01:30:00-04'::timestamptz, 'America/New_York', '2000-01-01 00:00:00 America/New_York'::timestamptz) as ts_before_bucket; bucket | ts_before_bucket ------------------------------+------------------ Sun Nov 03 01:00:00 2024 EDT | f SELECT time, time_bucket(10::smallint, time) AS time_bucket_smallint, time_bucket(10::int, time) AS time_bucket_int, time_bucket(10::bigint, time) AS time_bucket_bigint FROM unnest(ARRAY[ '-11', '-10', '-9', '-1', '0', '1', '99', '100', '109', '110' ]::smallint[]) AS time; time | time_bucket_smallint | time_bucket_int | time_bucket_bigint ------+----------------------+-----------------+-------------------- -11 | -20 | -20 | -20 -10 | -10 | -10 | -10 -9 | -10 | -10 | -10 -1 | -10 | -10 | -10 0 | 0 | 0 | 0 1 | 0 | 0 | 0 99 | 90 | 90 | 90 100 | 100 | 100 | 100 109 | 100 | 100 | 100 110 | 110 | 110 | 110 SELECT time, time_bucket(10::smallint, time, 2::smallint) AS time_bucket_smallint, time_bucket(10::int, time, 2::int) AS time_bucket_int, time_bucket(10::bigint, time, 2::bigint) AS time_bucket_bigint FROM unnest(ARRAY[ '-9', '-8', '-7', '1', '2', '3', '101', '102', '111', '112' ]::smallint[]) AS time; time | time_bucket_smallint | time_bucket_int | time_bucket_bigint ------+----------------------+-----------------+-------------------- -9 | -18 | -18 | -18 -8 | -8 | -8 | -8 -7 | -8 | -8 | -8 1 | -8 | -8 | -8 2 | 2 | 2 | 2 3 | 2 | 2 | 2 101 | 92 | 92 | 92 102 | 102 | 102 | 102 111 | 102 | 102 | 102 112 | 112 | 112 | 112 SELECT time, time_bucket(10::smallint, time, -2::smallint) AS time_bucket_smallint, time_bucket(10::int, time, -2::int) AS time_bucket_int, time_bucket(10::bigint, time, -2::bigint) AS time_bucket_bigint FROM unnest(ARRAY[ '-13', '-12', '-11', '-3', '-2', '-1', '97', '98', '107', '108' ]::smallint[]) AS time; time | time_bucket_smallint | time_bucket_int | time_bucket_bigint ------+----------------------+-----------------+-------------------- -13 | -22 | -22 | -22 -12 | -12 | -12 | -12 -11 | -12 | -12 | -12 -3 | -12 | -12 | -12 -2 | -2 | -2 | -2 -1 | -2 | -2 | -2 97 | 88 | 88 | 88 98 | 98 | 98 | 98 107 | 98 | 98 | 98 108 | 108 | 108 | 108 \set ON_ERROR_STOP 0 SELECT time_bucket(10::smallint, '-32768'::smallint); ERROR: timestamp out of range SELECT time_bucket(10::smallint, '-32761'::smallint); ERROR: timestamp out of range select time_bucket(10::smallint, '-32768'::smallint, 1000::smallint); ERROR: timestamp out of range select time_bucket(10::smallint, '-32768'::smallint, '32767'::smallint); ERROR: timestamp out of range select time_bucket(10::smallint, '32767'::smallint, '-32768'::smallint); ERROR: timestamp out of range \set ON_ERROR_STOP 1 SELECT time, time_bucket(10::smallint, time) FROM unnest(ARRAY[ '-32760', '-32759', '32767' ]::smallint[]) AS time; time | time_bucket --------+------------- -32760 | -32760 -32759 | -32760 32767 | 32760 \set ON_ERROR_STOP 0 SELECT time_bucket(10::int, '-2147483648'::int); ERROR: timestamp out of range SELECT time_bucket(10::int, '-2147483641'::int); ERROR: timestamp out of range SELECT time_bucket(1000::int, '-2147483000'::int, 1::int); ERROR: timestamp out of range SELECT time_bucket(1000::int, '-2147483648'::int, '2147483647'::int); ERROR: timestamp out of range SELECT time_bucket(1000::int, '2147483647'::int, '-2147483648'::int); ERROR: timestamp out of range \set ON_ERROR_STOP 1 SELECT time, time_bucket(10::int, time) FROM unnest(ARRAY[ '-2147483640', '-2147483639', '2147483647' ]::int[]) AS time; time | time_bucket -------------+------------- -2147483640 | -2147483640 -2147483639 | -2147483640 2147483647 | 2147483640 \set ON_ERROR_STOP 0 SELECT time_bucket(10::bigint, '-9223372036854775808'::bigint); ERROR: timestamp out of range SELECT time_bucket(10::bigint, '-9223372036854775801'::bigint); ERROR: timestamp out of range SELECT time_bucket(1000::bigint, '-9223372036854775000'::bigint, 1::bigint); ERROR: timestamp out of range SELECT time_bucket(1000::bigint, '-9223372036854775808'::bigint, '9223372036854775807'::bigint); ERROR: timestamp out of range SELECT time_bucket(1000::bigint, '9223372036854775807'::bigint, '-9223372036854775808'::bigint); ERROR: timestamp out of range \set ON_ERROR_STOP 1 SELECT time, time_bucket(10::bigint, time) FROM unnest(ARRAY[ '-9223372036854775800', '-9223372036854775799', '9223372036854775807' ]::bigint[]) AS time; time | time_bucket ----------------------+---------------------- -9223372036854775800 | -9223372036854775800 -9223372036854775799 | -9223372036854775800 9223372036854775807 | 9223372036854775800 SELECT time, time_bucket(INTERVAL '1 day', time::date) FROM unnest(ARRAY[ date '2017-11-05', date '2017-11-06' ]) AS time; time | time_bucket ------------+------------- 11-05-2017 | 11-05-2017 11-06-2017 | 11-06-2017 SELECT time, time_bucket(INTERVAL '4 day', time::date) FROM unnest(ARRAY[ date '2017-11-04', date '2017-11-05', date '2017-11-08', date '2017-11-09' ]) AS time; time | time_bucket ------------+------------- 11-04-2017 | 11-01-2017 11-05-2017 | 11-05-2017 11-08-2017 | 11-05-2017 11-09-2017 | 11-09-2017 SELECT time, time_bucket(INTERVAL '4 day', time::date, INTERVAL '2 day') FROM unnest(ARRAY[ date '2017-11-06', date '2017-11-07', date '2017-11-10', date '2017-11-11' ]) AS time; time | time_bucket ------------+------------- 11-06-2017 | 11-03-2017 11-07-2017 | 11-07-2017 11-10-2017 | 11-07-2017 11-11-2017 | 11-11-2017 -- 2019-09-24 is a Monday, and we want to ensure that time_bucket returns the week starting with a Monday as date_trunc does, -- Rather than a Saturday which is the date of the PostgreSQL epoch SELECT time, time_bucket(INTERVAL '1 week', time::date) FROM unnest(ARRAY[ date '2018-09-16', date '2018-09-17', date '2018-09-23', date '2018-09-24' ]) AS time; time | time_bucket ------------+------------- 09-16-2018 | 09-10-2018 09-17-2018 | 09-17-2018 09-23-2018 | 09-17-2018 09-24-2018 | 09-24-2018 SELECT time, time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp without time zone '2018-09-16', timestamp without time zone '2018-09-17', timestamp without time zone '2018-09-23', timestamp without time zone '2018-09-24' ]) AS time; time | time_bucket --------------------------+-------------------------- Sun Sep 16 00:00:00 2018 | Mon Sep 10 00:00:00 2018 Mon Sep 17 00:00:00 2018 | Mon Sep 17 00:00:00 2018 Sun Sep 23 00:00:00 2018 | Mon Sep 17 00:00:00 2018 Mon Sep 24 00:00:00 2018 | Mon Sep 24 00:00:00 2018 SELECT time, time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp with time zone '2018-09-16', timestamp with time zone '2018-09-17', timestamp with time zone '2018-09-23', timestamp with time zone '2018-09-24' ]) AS time; time | time_bucket ------------------------------+------------------------------ Sun Sep 16 00:00:00 2018 EDT | Sun Sep 09 20:00:00 2018 EDT Mon Sep 17 00:00:00 2018 EDT | Sun Sep 16 20:00:00 2018 EDT Sun Sep 23 00:00:00 2018 EDT | Sun Sep 16 20:00:00 2018 EDT Mon Sep 24 00:00:00 2018 EDT | Sun Sep 23 20:00:00 2018 EDT SELECT time, time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp with time zone '-Infinity', timestamp with time zone 'Infinity' ]) AS time; time | time_bucket -----------+------------- -infinity | -infinity infinity | infinity SELECT time, time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp without time zone '-Infinity', timestamp without time zone 'Infinity' ]) AS time; time | time_bucket -----------+------------- -infinity | -infinity infinity | infinity SELECT time, time_bucket(INTERVAL '1 week', time), date_trunc('week', time) = time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp without time zone '4714-11-24 01:01:01.0 BC', timestamp without time zone '294276-12-31 23:59:59.9999' ]) AS time; time | time_bucket | ?column? ---------------------------------+-----------------------------+---------- Mon Nov 24 01:01:01 4714 BC | Mon Nov 24 00:00:00 4714 BC | t Sun Dec 31 23:59:59.9999 294276 | Mon Dec 25 00:00:00 294276 | t --1000 years later weeks still align. SELECT time, time_bucket(INTERVAL '1 week', time), date_trunc('week', time) = time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp without time zone '3018-09-14', timestamp without time zone '3018-09-20', timestamp without time zone '3018-09-21', timestamp without time zone '3018-09-22' ]) AS time; time | time_bucket | ?column? --------------------------+--------------------------+---------- Mon Sep 14 00:00:00 3018 | Mon Sep 14 00:00:00 3018 | t Sun Sep 20 00:00:00 3018 | Mon Sep 14 00:00:00 3018 | t Mon Sep 21 00:00:00 3018 | Mon Sep 21 00:00:00 3018 | t Tue Sep 22 00:00:00 3018 | Mon Sep 21 00:00:00 3018 | t --weeks align for timestamptz as well if cast to local time, (but not if done at UTC). SELECT time, date_trunc('week', time) = time_bucket(INTERVAL '1 week', time), date_trunc('week', time) = time_bucket(INTERVAL '1 week', time::timestamp) FROM unnest(ARRAY[ timestamp with time zone '3018-09-14', timestamp with time zone '3018-09-20', timestamp with time zone '3018-09-21', timestamp with time zone '3018-09-22' ]) AS time; time | ?column? | ?column? ------------------------------+----------+---------- Mon Sep 14 00:00:00 3018 EDT | f | t Sun Sep 20 00:00:00 3018 EDT | f | t Mon Sep 21 00:00:00 3018 EDT | f | t Tue Sep 22 00:00:00 3018 EDT | f | t --check functions with origin --note that the default origin is at 0 UTC, using origin parameter it is easy to provide a EDT origin point \x SELECT time, time_bucket(INTERVAL '1 week', time) no_epoch, time_bucket(INTERVAL '1 week', time::timestamp) no_epoch_local, time_bucket(INTERVAL '1 week', time) = time_bucket(INTERVAL '1 week', time, timestamptz '2000-01-03 00:00:00+0') always_true, time_bucket(INTERVAL '1 week', time, timestamptz '2000-01-01 00:00:00+0') pg_epoch, time_bucket(INTERVAL '1 week', time, timestamptz 'epoch') unix_epoch, time_bucket(INTERVAL '1 week', time, timestamptz '3018-09-13') custom_1, time_bucket(INTERVAL '1 week', time, timestamptz '3018-09-14') custom_2 FROM unnest(ARRAY[ timestamp with time zone '2000-01-01 00:00:00+0'- interval '1 second', timestamp with time zone '2000-01-01 00:00:00+0', timestamp with time zone '2000-01-03 00:00:00+0'- interval '1 second', timestamp with time zone '2000-01-03 00:00:00+0', timestamp with time zone '2000-01-01', timestamp with time zone '2000-01-02', timestamp with time zone '2000-01-03', timestamp with time zone '3018-09-12', timestamp with time zone '3018-09-13', timestamp with time zone '3018-09-14', timestamp with time zone '3018-09-15' ]) AS time; -[ RECORD 1 ]--+----------------------------- time | Fri Dec 31 18:59:59 1999 EST no_epoch | Sun Dec 26 19:00:00 1999 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 24 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Dec 25 23:00:00 1999 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 2 ]--+----------------------------- time | Fri Dec 31 19:00:00 1999 EST no_epoch | Sun Dec 26 19:00:00 1999 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Dec 25 23:00:00 1999 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 3 ]--+----------------------------- time | Sun Jan 02 18:59:59 2000 EST no_epoch | Sun Dec 26 19:00:00 1999 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Jan 01 23:00:00 2000 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 4 ]--+----------------------------- time | Sun Jan 02 19:00:00 2000 EST no_epoch | Sun Jan 02 19:00:00 2000 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Jan 01 23:00:00 2000 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 5 ]--+----------------------------- time | Sat Jan 01 00:00:00 2000 EST no_epoch | Sun Dec 26 19:00:00 1999 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Dec 25 23:00:00 1999 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 6 ]--+----------------------------- time | Sun Jan 02 00:00:00 2000 EST no_epoch | Sun Dec 26 19:00:00 1999 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Jan 01 23:00:00 2000 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 7 ]--+----------------------------- time | Mon Jan 03 00:00:00 2000 EST no_epoch | Sun Jan 02 19:00:00 2000 EST no_epoch_local | Mon Jan 03 00:00:00 2000 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Jan 01 23:00:00 2000 EST custom_2 | Sun Jan 02 23:00:00 2000 EST -[ RECORD 8 ]--+----------------------------- time | Sat Sep 12 00:00:00 3018 EDT no_epoch | Sun Sep 06 20:00:00 3018 EDT no_epoch_local | Mon Sep 07 00:00:00 3018 always_true | t pg_epoch | Fri Sep 11 20:00:00 3018 EDT unix_epoch | Wed Sep 09 20:00:00 3018 EDT custom_1 | Sun Sep 06 00:00:00 3018 EDT custom_2 | Mon Sep 07 00:00:00 3018 EDT -[ RECORD 9 ]--+----------------------------- time | Sun Sep 13 00:00:00 3018 EDT no_epoch | Sun Sep 06 20:00:00 3018 EDT no_epoch_local | Mon Sep 07 00:00:00 3018 always_true | t pg_epoch | Fri Sep 11 20:00:00 3018 EDT unix_epoch | Wed Sep 09 20:00:00 3018 EDT custom_1 | Sun Sep 13 00:00:00 3018 EDT custom_2 | Mon Sep 07 00:00:00 3018 EDT -[ RECORD 10 ]-+----------------------------- time | Mon Sep 14 00:00:00 3018 EDT no_epoch | Sun Sep 13 20:00:00 3018 EDT no_epoch_local | Mon Sep 14 00:00:00 3018 always_true | t pg_epoch | Fri Sep 11 20:00:00 3018 EDT unix_epoch | Wed Sep 09 20:00:00 3018 EDT custom_1 | Sun Sep 13 00:00:00 3018 EDT custom_2 | Mon Sep 14 00:00:00 3018 EDT -[ RECORD 11 ]-+----------------------------- time | Tue Sep 15 00:00:00 3018 EDT no_epoch | Sun Sep 13 20:00:00 3018 EDT no_epoch_local | Mon Sep 14 00:00:00 3018 always_true | t pg_epoch | Fri Sep 11 20:00:00 3018 EDT unix_epoch | Wed Sep 09 20:00:00 3018 EDT custom_1 | Sun Sep 13 00:00:00 3018 EDT custom_2 | Mon Sep 14 00:00:00 3018 EDT SELECT time, time_bucket(INTERVAL '1 week', time) no_epoch, time_bucket(INTERVAL '1 week', time) = time_bucket(INTERVAL '1 week', time, timestamp '2000-01-03 00:00:00') always_true, time_bucket(INTERVAL '1 week', time, timestamp '2000-01-01 00:00:00+0') pg_epoch, time_bucket(INTERVAL '1 week', time, timestamp 'epoch') unix_epoch, time_bucket(INTERVAL '1 week', time, timestamp '3018-09-13') custom_1, time_bucket(INTERVAL '1 week', time, timestamp '3018-09-14') custom_2 FROM unnest(ARRAY[ timestamp without time zone '2000-01-01 00:00:00'- interval '1 second', timestamp without time zone '2000-01-01 00:00:00', timestamp without time zone '2000-01-03 00:00:00'- interval '1 second', timestamp without time zone '2000-01-03 00:00:00', timestamp without time zone '2000-01-01', timestamp without time zone '2000-01-02', timestamp without time zone '2000-01-03', timestamp without time zone '3018-09-12', timestamp without time zone '3018-09-13', timestamp without time zone '3018-09-14', timestamp without time zone '3018-09-15' ]) AS time; -[ RECORD 1 ]------------------------- time | Fri Dec 31 23:59:59 1999 no_epoch | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Sat Dec 25 00:00:00 1999 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Dec 26 00:00:00 1999 custom_2 | Mon Dec 27 00:00:00 1999 -[ RECORD 2 ]------------------------- time | Sat Jan 01 00:00:00 2000 no_epoch | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Dec 26 00:00:00 1999 custom_2 | Mon Dec 27 00:00:00 1999 -[ RECORD 3 ]------------------------- time | Sun Jan 02 23:59:59 2000 no_epoch | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Jan 02 00:00:00 2000 custom_2 | Mon Dec 27 00:00:00 1999 -[ RECORD 4 ]------------------------- time | Mon Jan 03 00:00:00 2000 no_epoch | Mon Jan 03 00:00:00 2000 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Jan 02 00:00:00 2000 custom_2 | Mon Jan 03 00:00:00 2000 -[ RECORD 5 ]------------------------- time | Sat Jan 01 00:00:00 2000 no_epoch | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Dec 26 00:00:00 1999 custom_2 | Mon Dec 27 00:00:00 1999 -[ RECORD 6 ]------------------------- time | Sun Jan 02 00:00:00 2000 no_epoch | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Jan 02 00:00:00 2000 custom_2 | Mon Dec 27 00:00:00 1999 -[ RECORD 7 ]------------------------- time | Mon Jan 03 00:00:00 2000 no_epoch | Mon Jan 03 00:00:00 2000 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Jan 02 00:00:00 2000 custom_2 | Mon Jan 03 00:00:00 2000 -[ RECORD 8 ]------------------------- time | Sat Sep 12 00:00:00 3018 no_epoch | Mon Sep 07 00:00:00 3018 always_true | t pg_epoch | Sat Sep 12 00:00:00 3018 unix_epoch | Thu Sep 10 00:00:00 3018 custom_1 | Sun Sep 06 00:00:00 3018 custom_2 | Mon Sep 07 00:00:00 3018 -[ RECORD 9 ]------------------------- time | Sun Sep 13 00:00:00 3018 no_epoch | Mon Sep 07 00:00:00 3018 always_true | t pg_epoch | Sat Sep 12 00:00:00 3018 unix_epoch | Thu Sep 10 00:00:00 3018 custom_1 | Sun Sep 13 00:00:00 3018 custom_2 | Mon Sep 07 00:00:00 3018 -[ RECORD 10 ]------------------------ time | Mon Sep 14 00:00:00 3018 no_epoch | Mon Sep 14 00:00:00 3018 always_true | t pg_epoch | Sat Sep 12 00:00:00 3018 unix_epoch | Thu Sep 10 00:00:00 3018 custom_1 | Sun Sep 13 00:00:00 3018 custom_2 | Mon Sep 14 00:00:00 3018 -[ RECORD 11 ]------------------------ time | Tue Sep 15 00:00:00 3018 no_epoch | Mon Sep 14 00:00:00 3018 always_true | t pg_epoch | Sat Sep 12 00:00:00 3018 unix_epoch | Thu Sep 10 00:00:00 3018 custom_1 | Sun Sep 13 00:00:00 3018 custom_2 | Mon Sep 14 00:00:00 3018 SELECT time, time_bucket(INTERVAL '1 week', time) no_epoch, time_bucket(INTERVAL '1 week', time) = time_bucket(INTERVAL '1 week', time, date '2000-01-03') always_true, time_bucket(INTERVAL '1 week', time, date '2000-01-01') pg_epoch, time_bucket(INTERVAL '1 week', time, (timestamp 'epoch')::date) unix_epoch, time_bucket(INTERVAL '1 week', time, date '3018-09-13') custom_1, time_bucket(INTERVAL '1 week', time, date '3018-09-14') custom_2 FROM unnest(ARRAY[ date '1999-12-31', date '2000-01-01', date '2000-01-02', date '2000-01-03', date '3018-09-12', date '3018-09-13', date '3018-09-14', date '3018-09-15' ]) AS time; -[ RECORD 1 ]----------- time | 12-31-1999 no_epoch | 12-27-1999 always_true | t pg_epoch | 12-25-1999 unix_epoch | 12-30-1999 custom_1 | 12-26-1999 custom_2 | 12-27-1999 -[ RECORD 2 ]----------- time | 01-01-2000 no_epoch | 12-27-1999 always_true | t pg_epoch | 01-01-2000 unix_epoch | 12-30-1999 custom_1 | 12-26-1999 custom_2 | 12-27-1999 -[ RECORD 3 ]----------- time | 01-02-2000 no_epoch | 12-27-1999 always_true | t pg_epoch | 01-01-2000 unix_epoch | 12-30-1999 custom_1 | 01-02-2000 custom_2 | 12-27-1999 -[ RECORD 4 ]----------- time | 01-03-2000 no_epoch | 01-03-2000 always_true | t pg_epoch | 01-01-2000 unix_epoch | 12-30-1999 custom_1 | 01-02-2000 custom_2 | 01-03-2000 -[ RECORD 5 ]----------- time | 09-12-3018 no_epoch | 09-07-3018 always_true | t pg_epoch | 09-12-3018 unix_epoch | 09-10-3018 custom_1 | 09-06-3018 custom_2 | 09-07-3018 -[ RECORD 6 ]----------- time | 09-13-3018 no_epoch | 09-07-3018 always_true | t pg_epoch | 09-12-3018 unix_epoch | 09-10-3018 custom_1 | 09-13-3018 custom_2 | 09-07-3018 -[ RECORD 7 ]----------- time | 09-14-3018 no_epoch | 09-14-3018 always_true | t pg_epoch | 09-12-3018 unix_epoch | 09-10-3018 custom_1 | 09-13-3018 custom_2 | 09-14-3018 -[ RECORD 8 ]----------- time | 09-15-3018 no_epoch | 09-14-3018 always_true | t pg_epoch | 09-12-3018 unix_epoch | 09-10-3018 custom_1 | 09-13-3018 custom_2 | 09-14-3018 \x --really old origin works if date around that time SELECT time, time_bucket(INTERVAL '1 week', time, timestamp without time zone '4710-11-24 01:01:01.0 BC') FROM unnest(ARRAY[ timestamp without time zone '4710-11-24 01:01:01.0 BC', timestamp without time zone '4710-11-25 01:01:01.0 BC', timestamp without time zone '2001-01-01', timestamp without time zone '3001-01-01' ]) AS time; time | time_bucket -----------------------------+----------------------------- Sat Nov 24 01:01:01 4710 BC | Sat Nov 24 01:01:01 4710 BC Sun Nov 25 01:01:01 4710 BC | Sat Nov 24 01:01:01 4710 BC Mon Jan 01 00:00:00 2001 | Sat Dec 30 01:01:01 2000 Thu Jan 01 00:00:00 3001 | Sat Dec 27 01:01:01 3000 SELECT time, time_bucket(INTERVAL '1 week', time, timestamp without time zone '294270-12-30 23:59:59.9999') FROM unnest(ARRAY[ timestamp without time zone '294270-12-29 23:59:59.9999', timestamp without time zone '294270-12-30 23:59:59.9999', timestamp without time zone '294270-12-31 23:59:59.9999', timestamp without time zone '2001-01-01', timestamp without time zone '3001-01-01' ]) AS time; time | time_bucket ---------------------------------+--------------------------------- Thu Dec 29 23:59:59.9999 294270 | Fri Dec 23 23:59:59.9999 294270 Fri Dec 30 23:59:59.9999 294270 | Fri Dec 30 23:59:59.9999 294270 Sat Dec 31 23:59:59.9999 294270 | Fri Dec 30 23:59:59.9999 294270 Mon Jan 01 00:00:00 2001 | Fri Dec 29 23:59:59.9999 2000 Thu Jan 01 00:00:00 3001 | Fri Dec 26 23:59:59.9999 3000 \set ON_ERROR_STOP 0 --really old origin + very new data + long period errors SELECT time, time_bucket(INTERVAL '100000 day', time, timestamp without time zone '4710-11-24 01:01:01.0 BC') FROM unnest(ARRAY[ timestamp without time zone '294270-12-31 23:59:59.9999' ]) AS time; ERROR: timestamp out of range SELECT time, time_bucket(INTERVAL '100000 day', time, timestamp with time zone '4710-11-25 01:01:01.0 BC') FROM unnest(ARRAY[ timestamp with time zone '294270-12-30 23:59:59.9999' ]) AS time; ERROR: timestamp out of range --really high origin + old data + long period errors out SELECT time, time_bucket(INTERVAL '10000000 day', time, timestamp without time zone '294270-12-31 23:59:59.9999') FROM unnest(ARRAY[ timestamp without time zone '4710-11-24 01:01:01.0 BC' ]) AS time; ERROR: timestamp out of range SELECT time, time_bucket(INTERVAL '10000000 day', time, timestamp with time zone '294270-12-31 23:59:59.9999') FROM unnest(ARRAY[ timestamp with time zone '4710-11-24 01:01:01.0 BC' ]) AS time; ERROR: timestamp out of range \set ON_ERROR_STOP 1 ------------------------------------------- --- Test time_bucket with month periods --- ------------------------------------------- SET datestyle TO ISO; SELECT time::date, time_bucket('1 month', time::date) AS "1m", time_bucket('2 month', time::date) AS "2m", time_bucket('3 month', time::date) AS "3m", time_bucket('1 month', time::date, '2000-02-01'::date) AS "1m origin", time_bucket('2 month', time::date, '2000-02-01'::date) AS "2m origin", time_bucket('3 month', time::date, '2000-02-01'::date) AS "3m origin" FROM generate_series('1990-01-03'::date,'1990-06-03'::date,'1month'::interval) time; time | 1m | 2m | 3m | 1m origin | 2m origin | 3m origin ------------+------------+------------+------------+------------+------------+------------ 1990-01-03 | 1990-01-01 | 1990-01-01 | 1990-01-01 | 1990-01-01 | 1989-12-01 | 1989-11-01 1990-02-03 | 1990-02-01 | 1990-01-01 | 1990-01-01 | 1990-02-01 | 1990-02-01 | 1990-02-01 1990-03-03 | 1990-03-01 | 1990-03-01 | 1990-01-01 | 1990-03-01 | 1990-02-01 | 1990-02-01 1990-04-03 | 1990-04-01 | 1990-03-01 | 1990-04-01 | 1990-04-01 | 1990-04-01 | 1990-02-01 1990-05-03 | 1990-05-01 | 1990-05-01 | 1990-04-01 | 1990-05-01 | 1990-04-01 | 1990-05-01 1990-06-03 | 1990-06-01 | 1990-05-01 | 1990-04-01 | 1990-06-01 | 1990-06-01 | 1990-05-01 SELECT time, time_bucket('1 month', time) AS "1m", time_bucket('2 month', time) AS "2m", time_bucket('3 month', time) AS "3m", time_bucket('1 month', time, '2000-02-01'::timestamp) AS "1m origin", time_bucket('2 month', time, '2000-02-01'::timestamp) AS "2m origin", time_bucket('3 month', time, '2000-02-01'::timestamp) AS "3m origin" FROM generate_series('1990-01-03'::timestamp,'1990-06-03'::timestamp,'1month'::interval) time; time | 1m | 2m | 3m | 1m origin | 2m origin | 3m origin ---------------------+---------------------+---------------------+---------------------+---------------------+---------------------+--------------------- 1990-01-03 00:00:00 | 1990-01-01 00:00:00 | 1990-01-01 00:00:00 | 1990-01-01 00:00:00 | 1990-01-01 00:00:00 | 1989-12-01 00:00:00 | 1989-11-01 00:00:00 1990-02-03 00:00:00 | 1990-02-01 00:00:00 | 1990-01-01 00:00:00 | 1990-01-01 00:00:00 | 1990-02-01 00:00:00 | 1990-02-01 00:00:00 | 1990-02-01 00:00:00 1990-03-03 00:00:00 | 1990-03-01 00:00:00 | 1990-03-01 00:00:00 | 1990-01-01 00:00:00 | 1990-03-01 00:00:00 | 1990-02-01 00:00:00 | 1990-02-01 00:00:00 1990-04-03 00:00:00 | 1990-04-01 00:00:00 | 1990-03-01 00:00:00 | 1990-04-01 00:00:00 | 1990-04-01 00:00:00 | 1990-04-01 00:00:00 | 1990-02-01 00:00:00 1990-05-03 00:00:00 | 1990-05-01 00:00:00 | 1990-05-01 00:00:00 | 1990-04-01 00:00:00 | 1990-05-01 00:00:00 | 1990-04-01 00:00:00 | 1990-05-01 00:00:00 1990-06-03 00:00:00 | 1990-06-01 00:00:00 | 1990-05-01 00:00:00 | 1990-04-01 00:00:00 | 1990-06-01 00:00:00 | 1990-06-01 00:00:00 | 1990-05-01 00:00:00 SELECT time, time_bucket('1 month', time) AS "1m", time_bucket('2 month', time) AS "2m", time_bucket('3 month', time) AS "3m", time_bucket('1 month', time, '2000-02-01'::timestamptz) AS "1m origin", time_bucket('2 month', time, '2000-02-01'::timestamptz) AS "2m origin", time_bucket('3 month', time, '2000-02-01'::timestamptz) AS "3m origin" FROM generate_series('1990-01-03'::timestamptz,'1990-06-03'::timestamptz,'1month'::interval) time; time | 1m | 2m | 3m | 1m origin | 2m origin | 3m origin ------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+------------------------ 1990-01-03 00:00:00-05 | 1989-12-31 19:00:00-05 | 1989-12-31 19:00:00-05 | 1989-12-31 19:00:00-05 | 1989-12-31 19:00:00-05 | 1989-11-30 19:00:00-05 | 1989-10-31 19:00:00-05 1990-02-03 00:00:00-05 | 1990-01-31 19:00:00-05 | 1989-12-31 19:00:00-05 | 1989-12-31 19:00:00-05 | 1990-01-31 19:00:00-05 | 1990-01-31 19:00:00-05 | 1990-01-31 19:00:00-05 1990-03-03 00:00:00-05 | 1990-02-28 19:00:00-05 | 1990-02-28 19:00:00-05 | 1989-12-31 19:00:00-05 | 1990-02-28 19:00:00-05 | 1990-01-31 19:00:00-05 | 1990-01-31 19:00:00-05 1990-04-03 00:00:00-04 | 1990-03-31 19:00:00-05 | 1990-02-28 19:00:00-05 | 1990-03-31 19:00:00-05 | 1990-03-31 19:00:00-05 | 1990-03-31 19:00:00-05 | 1990-01-31 19:00:00-05 1990-05-03 00:00:00-04 | 1990-04-30 20:00:00-04 | 1990-04-30 20:00:00-04 | 1990-03-31 19:00:00-05 | 1990-04-30 20:00:00-04 | 1990-03-31 19:00:00-05 | 1990-04-30 20:00:00-04 1990-06-03 00:00:00-04 | 1990-05-31 20:00:00-04 | 1990-04-30 20:00:00-04 | 1990-03-31 19:00:00-05 | 1990-05-31 20:00:00-04 | 1990-05-31 20:00:00-04 | 1990-04-30 20:00:00-04 --------------------------------------- --- Test time_bucket with timezones --- --------------------------------------- -- test NULL args SELECT time_bucket(NULL::interval,now(),'Europe/Berlin'), time_bucket('1day',NULL::timestamptz,'Europe/Berlin'), time_bucket('1day',now(),NULL::text), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin',NULL), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin','2020-04-01',NULL), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin',NULL,NULL), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin',"offset":=NULL::interval), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin',origin:=NULL::timestamptz); time_bucket | time_bucket | time_bucket | time_bucket | time_bucket | time_bucket | time_bucket | time_bucket -------------+-------------+-------------+------------------------+------------------------+------------------------+------------------------+------------------------ | | | 2020-02-02 18:00:00-05 | 2020-02-03 00:00:00-05 | 2020-02-02 18:00:00-05 | 2020-02-02 18:00:00-05 | 2020-02-02 18:00:00-05 SET datestyle TO ISO; SELECT time_bucket('1day', ts) AS "UTC", time_bucket('1day', ts, 'Europe/Berlin') AS "Berlin", time_bucket('1day', ts, 'Europe/London') AS "London", time_bucket('1day', ts, 'America/New_York') AS "New York", time_bucket('1day', ts, 'PST') AS "PST", time_bucket('1day', ts, current_setting('timezone')) AS "current" FROM generate_series('1999-12-31 17:00'::timestamptz,'2000-01-02 3:00'::timestamptz, '1hour'::interval) ts; UTC | Berlin | London | New York | PST | current ------------------------+------------------------+------------------------+------------------------+------------------------+------------------------ 1999-12-30 19:00:00-05 | 1999-12-30 18:00:00-05 | 1999-12-30 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-30 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-30 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-31 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-31 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-31 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 2000-01-01 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-02 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-02 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-02 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-02 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-02 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-02 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-02 00:00:00-05 | 2000-01-02 03:00:00-05 | 2000-01-02 00:00:00-05 SELECT time_bucket('1month', ts) AS "UTC", time_bucket('1month', ts, 'Europe/Berlin') AS "Berlin", time_bucket('1month', ts, 'America/New_York') AS "New York", time_bucket('1month', ts, current_setting('timezone')) AS "current", time_bucket('2month', ts, current_setting('timezone')) AS "2m", time_bucket('2month', ts, current_setting('timezone'), '2000-02-01'::timestamp) AS "2m origin", time_bucket('2month', ts, current_setting('timezone'), "offset":='14 day'::interval) AS "2m offset", time_bucket('2month', ts, current_setting('timezone'), '2000-02-01'::timestamp, '7 day'::interval) AS "2m offset + origin" FROM generate_series('1999-12-01'::timestamptz,'2000-09-01'::timestamptz, '9 day'::interval) ts; UTC | Berlin | New York | current | 2m | 2m origin | 2m offset | 2m offset + origin ------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+------------------------ 1999-11-30 19:00:00-05 | 1999-11-30 18:00:00-05 | 1999-12-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-15 00:00:00-05 | 1999-10-08 00:00:00-04 1999-11-30 19:00:00-05 | 1999-11-30 18:00:00-05 | 1999-12-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-15 00:00:00-05 | 1999-12-08 00:00:00-05 1999-11-30 19:00:00-05 | 1999-11-30 18:00:00-05 | 1999-12-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-15 00:00:00-05 | 1999-12-08 00:00:00-05 1999-11-30 19:00:00-05 | 1999-11-30 18:00:00-05 | 1999-12-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-15 00:00:00-05 | 1999-12-08 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-15 00:00:00-05 | 1999-12-08 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 1999-12-08 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 1999-12-08 00:00:00-05 2000-01-31 19:00:00-05 | 2000-01-31 18:00:00-05 | 2000-02-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 1999-12-08 00:00:00-05 2000-01-31 19:00:00-05 | 2000-01-31 18:00:00-05 | 2000-02-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-01-31 19:00:00-05 | 2000-01-31 18:00:00-05 | 2000-02-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-01-31 19:00:00-05 | 2000-01-31 18:00:00-05 | 2000-02-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-02-29 19:00:00-05 | 2000-02-29 18:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-02-29 19:00:00-05 | 2000-02-29 18:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-02-29 19:00:00-05 | 2000-02-29 18:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-03-31 19:00:00-05 | 2000-03-31 17:00:00-05 | 2000-04-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-03-31 19:00:00-05 | 2000-03-31 17:00:00-05 | 2000-04-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-04-08 00:00:00-04 2000-03-31 19:00:00-05 | 2000-03-31 17:00:00-05 | 2000-04-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-04-08 00:00:00-04 2000-04-30 20:00:00-04 | 2000-04-30 18:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-04-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-04-08 00:00:00-04 2000-04-30 20:00:00-04 | 2000-04-30 18:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-04-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-04-08 00:00:00-04 2000-04-30 20:00:00-04 | 2000-04-30 18:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-04-01 00:00:00-05 | 2000-05-15 00:00:00-04 | 2000-04-08 00:00:00-04 2000-04-30 20:00:00-04 | 2000-04-30 18:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-04-01 00:00:00-05 | 2000-05-15 00:00:00-04 | 2000-04-08 00:00:00-04 2000-05-31 20:00:00-04 | 2000-05-31 18:00:00-04 | 2000-06-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-15 00:00:00-04 | 2000-04-08 00:00:00-04 2000-05-31 20:00:00-04 | 2000-05-31 18:00:00-04 | 2000-06-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-05-31 20:00:00-04 | 2000-05-31 18:00:00-04 | 2000-06-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-06-30 20:00:00-04 | 2000-06-30 18:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-06-30 20:00:00-04 | 2000-06-30 18:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-06-30 20:00:00-04 | 2000-06-30 18:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-07-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-06-30 20:00:00-04 | 2000-06-30 18:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-07-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-07-31 20:00:00-04 | 2000-07-31 18:00:00-04 | 2000-08-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-15 00:00:00-04 | 2000-08-08 00:00:00-04 2000-07-31 20:00:00-04 | 2000-07-31 18:00:00-04 | 2000-08-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-15 00:00:00-04 | 2000-08-08 00:00:00-04 2000-07-31 20:00:00-04 | 2000-07-31 18:00:00-04 | 2000-08-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-15 00:00:00-04 | 2000-08-08 00:00:00-04 RESET datestyle; ------------------------------------- --- Test time input functions -- ------------------------------------- \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION test.interval_to_internal(coltype REGTYPE, value ANYELEMENT = NULL::BIGINT) RETURNS BIGINT AS :MODULE_PATHNAME, 'ts_dimension_interval_to_internal_test' LANGUAGE C VOLATILE; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT test.interval_to_internal('TIMESTAMP'::regtype, INTERVAL '1 day'); interval_to_internal ---------------------- 86400000000 SELECT test.interval_to_internal('TIMESTAMP'::regtype, 86400000000); interval_to_internal ---------------------- 86400000000 ---should give warning SELECT test.interval_to_internal('TIMESTAMP'::regtype, 86400); WARNING: unexpected interval: smaller than one second HINT: The interval is specified in microseconds. interval_to_internal ---------------------- 86400 SELECT test.interval_to_internal('TIMESTAMP'::regtype); interval_to_internal ---------------------- 604800000000 SELECT test.interval_to_internal('BIGINT'::regtype, 2147483649::bigint); interval_to_internal ---------------------- 2147483649 -- Default interval for integer is supported as part of -- hypertable generalization SELECT test.interval_to_internal('INT'::regtype); interval_to_internal ---------------------- 100000 SELECT test.interval_to_internal('SMALLINT'::regtype); interval_to_internal ---------------------- 10000 SELECT test.interval_to_internal('BIGINT'::regtype); interval_to_internal ---------------------- 1000000 SELECT test.interval_to_internal('TIMESTAMPTZ'::regtype); interval_to_internal ---------------------- 604800000000 SELECT test.interval_to_internal('TIMESTAMP'::regtype); interval_to_internal ---------------------- 604800000000 SELECT test.interval_to_internal('DATE'::regtype); interval_to_internal ---------------------- 604800000000 \set VERBOSITY terse \set ON_ERROR_STOP 0 SELECT test.interval_to_internal('INT'::regtype, 2147483649::bigint); ERROR: invalid interval: must be between 1 and 2147483647 SELECT test.interval_to_internal('SMALLINT'::regtype, 32768::bigint); ERROR: invalid interval: must be between 1 and 32767 SELECT test.interval_to_internal('TEXT'::regtype, 32768::bigint); ERROR: invalid type for dimension "testcol" SELECT test.interval_to_internal('INT'::regtype, INTERVAL '1 day'); ERROR: invalid interval type for integer dimension \set ON_ERROR_STOP 1 ================================================ FILE: test/expected/timestamp-16.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Utility function for grouping/slotting time with a given interval. CREATE OR REPLACE FUNCTION date_group( field timestamp, group_interval interval ) RETURNS timestamp LANGUAGE SQL STABLE AS $BODY$ SELECT to_timestamp((EXTRACT(EPOCH from $1)::int / EXTRACT(EPOCH from group_interval)::int) * EXTRACT(EPOCH from group_interval)::int)::timestamp; $BODY$; CREATE TABLE PUBLIC."testNs" ( "timeCustom" TIMESTAMP NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."testNs" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA "testNs" AUTHORIZATION :ROLE_DEFAULT_PERM_USER; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT * FROM create_hypertable('"public"."testNs"', 'timeCustom', 'device_id', 2, associated_schema_name=>'testNs' ); WARNING: column type "timestamp without time zone" used for "timeCustom" does not follow best practices hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | testNs | t \c :TEST_DBNAME INSERT INTO PUBLIC."testNs"("timeCustom", device_id, series_0, series_1) VALUES ('2009-11-12T01:00:00+00:00', 'dev1', 1.5, 1), ('2009-11-12T01:00:00+00:00', 'dev1', 1.5, 2), ('2009-11-10T23:00:02+00:00', 'dev1', 2.5, 3); INSERT INTO PUBLIC."testNs"("timeCustom", device_id, series_0, series_1) VALUES ('2009-11-10T23:00:00+00:00', 'dev2', 1.5, 1), ('2009-11-10T23:00:00+00:00', 'dev2', 1.5, 2); SELECT * FROM PUBLIC."testNs"; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool --------------------------+-----------+----------+----------+----------+------------- Thu Nov 12 01:00:00 2009 | dev1 | 1.5 | 1 | | Thu Nov 12 01:00:00 2009 | dev1 | 1.5 | 2 | | Tue Nov 10 23:00:02 2009 | dev1 | 2.5 | 3 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 1 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 2 | | SET client_min_messages = WARNING; \echo 'The next 2 queries will differ in output between UTC and EST since the mod is on the 100th hour UTC' The next 2 queries will differ in output between UTC and EST since the mod is on the 100th hour UTC SET timezone = 'UTC'; SELECT date_group("timeCustom", '100 days') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC; time | sum --------------------------+----- Sun Sep 13 00:00:00 2009 | 8.5 SET timezone = 'EST'; SELECT date_group("timeCustom", '100 days') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC; time | sum --------------------------+----- Sat Sep 12 19:00:00 2009 | 8.5 \echo 'The rest of the queries will be the same in output between UTC and EST' The rest of the queries will be the same in output between UTC and EST SET timezone = 'UTC'; SELECT date_group("timeCustom", '1 day') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC; time | sum --------------------------+----- Tue Nov 10 00:00:00 2009 | 5.5 Thu Nov 12 00:00:00 2009 | 3 SET timezone = 'EST'; SELECT date_group("timeCustom", '1 day') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC; time | sum --------------------------+----- Mon Nov 09 19:00:00 2009 | 5.5 Wed Nov 11 19:00:00 2009 | 3 SET timezone = 'UTC'; SELECT * FROM PUBLIC."testNs" WHERE "timeCustom" >= TIMESTAMP '2009-11-10T23:00:00' AND "timeCustom" < TIMESTAMP '2009-11-12T01:00:00' ORDER BY "timeCustom" DESC, device_id, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool --------------------------+-----------+----------+----------+----------+------------- Tue Nov 10 23:00:02 2009 | dev1 | 2.5 | 3 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 1 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 2 | | SET timezone = 'EST'; SELECT * FROM PUBLIC."testNs" WHERE "timeCustom" >= TIMESTAMP '2009-11-10T23:00:00' AND "timeCustom" < TIMESTAMP '2009-11-12T01:00:00' ORDER BY "timeCustom" DESC, device_id, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool --------------------------+-----------+----------+----------+----------+------------- Tue Nov 10 23:00:02 2009 | dev1 | 2.5 | 3 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 1 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 2 | | SET timezone = 'UTC'; SELECT date_group("timeCustom", '1 day') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC LIMIT 2; time | sum --------------------------+----- Tue Nov 10 00:00:00 2009 | 5.5 Thu Nov 12 00:00:00 2009 | 3 SET timezone = 'EST'; SELECT date_group("timeCustom", '1 day') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC LIMIT 2; time | sum --------------------------+----- Mon Nov 09 19:00:00 2009 | 5.5 Wed Nov 11 19:00:00 2009 | 3 ------------------------------------ -- Test time conversion functions -- ------------------------------------ \set ON_ERROR_STOP 0 SET timezone = 'UTC'; -- Conversion to timestamp using Postgres built-in function taking -- double. Gives inaccurate result on Postgres <= 9.6.2. Accurate on -- Postgres >= 9.6.3. SELECT to_timestamp(1486480176.236538); to_timestamp ------------------------------------- Tue Feb 07 15:09:36.236538 2017 UTC -- extension-specific version taking microsecond UNIX timestamp SELECT _timescaledb_functions.to_timestamp(1486480176236538); to_timestamp ------------------------------------- Tue Feb 07 15:09:36.236538 2017 UTC -- Should be the inverse of the statement above. SELECT _timescaledb_functions.to_unix_microseconds('2017-02-07 15:09:36.236538+00'); to_unix_microseconds ---------------------- 1486480176236538 -- For timestamps, BIGINT MAX represents +Infinity and BIGINT MIN -- -Infinity. We keep this notion for UNIX epoch time: SELECT _timescaledb_functions.to_unix_microseconds('+infinity'); to_unix_microseconds ---------------------- 9223372036854775807 SELECT _timescaledb_functions.to_timestamp(9223372036854775807); to_timestamp -------------- infinity SELECT _timescaledb_functions.to_unix_microseconds('-infinity'); to_unix_microseconds ---------------------- -9223372036854775808 SELECT _timescaledb_functions.to_timestamp(-9223372036854775808); to_timestamp -------------- -infinity -- In UNIX microseconds, the largest bigint value below infinity -- (BIGINT MAX) is smaller than internal date upper bound and should -- therefore be OK. Further, converting to the internal postgres epoch -- cannot overflow a 64-bit INTEGER since the postgres epoch is at a -- later date compared to the UNIX epoch, and is therefore represented -- by a smaller number SELECT _timescaledb_functions.to_timestamp(9223372036854775806); to_timestamp --------------------------------------- Sun Jan 10 04:00:54.775806 294247 UTC -- Julian day zero is -210866803200000000 microseconds from UNIX epoch SELECT _timescaledb_functions.to_timestamp(-210866803200000000); to_timestamp --------------------------------- Mon Nov 24 00:00:00 4714 UTC BC \set VERBOSITY default -- Going beyond Julian day zero should give out-of-range error SELECT _timescaledb_functions.to_timestamp(-210866803200000001); ERROR: timestamp out of range -- Lower bound on date (should return the Julian day zero UNIX timestamp above) SELECT _timescaledb_functions.to_unix_microseconds('4714-11-24 00:00:00+00 BC'); to_unix_microseconds ---------------------- -210866803200000000 -- Going beyond lower bound on date should return out-of-range SELECT _timescaledb_functions.to_unix_microseconds('4714-11-23 23:59:59.999999+00 BC'); ERROR: timestamp out of range: "4714-11-23 23:59:59.999999+00 BC" LINE 1: ...ELECT _timescaledb_functions.to_unix_microseconds('4714-11-2... ^ -- The upper bound for Postgres TIMESTAMPTZ SELECT timestamp '294276-12-31 23:59:59.999999+00'; timestamp ----------------------------------- Sun Dec 31 23:59:59.999999 294276 -- Going beyond the upper bound, should fail SELECT timestamp '294276-12-31 23:59:59.999999+00' + interval '1 us'; ERROR: timestamp out of range -- Cannot represent the upper bound timestamp with a UNIX microsecond timestamp -- since the Postgres epoch is at a later date than the UNIX epoch. SELECT _timescaledb_functions.to_unix_microseconds('294276-12-31 23:59:59.999999+00'); ERROR: timestamp out of range -- Subtracting the difference between the two epochs (10957 days) should bring -- us within range. SELECT timestamp '294276-12-31 23:59:59.999999+00' - interval '10957 days'; ?column? ----------------------------------- Fri Jan 01 23:59:59.999999 294247 SELECT _timescaledb_functions.to_unix_microseconds('294247-01-01 23:59:59.999999'); to_unix_microseconds ---------------------- 9223371331199999999 -- Adding one microsecond should take us out-of-range again SELECT timestamp '294247-01-01 23:59:59.999999' + interval '1 us'; ?column? ---------------------------- Sat Jan 02 00:00:00 294247 SELECT _timescaledb_functions.to_unix_microseconds(timestamp '294247-01-01 23:59:59.999999' + interval '1 us'); ERROR: timestamp out of range --no time_bucketing of dates not by integer # of days SELECT time_bucket('1 hour', DATE '2012-01-01'); ERROR: interval must not have sub-day precision SELECT time_bucket('25 hour', DATE '2012-01-01'); ERROR: interval must be a multiple of a day \set ON_ERROR_STOP 1 SELECT time_bucket(INTERVAL '1 day', TIMESTAMP '2011-01-02 01:01:01'); time_bucket -------------------------- Sun Jan 02 00:00:00 2011 SELECT time, time_bucket(INTERVAL '2 day ', time) FROM unnest(ARRAY[ TIMESTAMP '2011-01-01 01:01:01', TIMESTAMP '2011-01-02 01:01:01', TIMESTAMP '2011-01-03 01:01:01', TIMESTAMP '2011-01-04 01:01:01' ]) AS time; time | time_bucket --------------------------+-------------------------- Sat Jan 01 01:01:01 2011 | Sat Jan 01 00:00:00 2011 Sun Jan 02 01:01:01 2011 | Sat Jan 01 00:00:00 2011 Mon Jan 03 01:01:01 2011 | Mon Jan 03 00:00:00 2011 Tue Jan 04 01:01:01 2011 | Mon Jan 03 00:00:00 2011 SELECT int_def, time_bucket(int_def,TIMESTAMP '2011-01-02 01:01:01.111') FROM unnest(ARRAY[ INTERVAL '1 millisecond', INTERVAL '1 second', INTERVAL '1 minute', INTERVAL '1 hour', INTERVAL '1 day', INTERVAL '2 millisecond', INTERVAL '2 second', INTERVAL '2 minute', INTERVAL '2 hour', INTERVAL '2 day' ]) AS int_def; int_def | time_bucket --------------+------------------------------ @ 0.001 secs | Sun Jan 02 01:01:01.111 2011 @ 1 sec | Sun Jan 02 01:01:01 2011 @ 1 min | Sun Jan 02 01:01:00 2011 @ 1 hour | Sun Jan 02 01:00:00 2011 @ 1 day | Sun Jan 02 00:00:00 2011 @ 0.002 secs | Sun Jan 02 01:01:01.11 2011 @ 2 secs | Sun Jan 02 01:01:00 2011 @ 2 mins | Sun Jan 02 01:00:00 2011 @ 2 hours | Sun Jan 02 00:00:00 2011 @ 2 days | Sat Jan 01 00:00:00 2011 \set ON_ERROR_STOP 0 SELECT time_bucket(INTERVAL '1 year 1d',TIMESTAMP '2011-01-02 01:01:01.111'); ERROR: month intervals cannot have day or time component SELECT time_bucket(INTERVAL '1 month 1 minute',TIMESTAMP '2011-01-02 01:01:01.111'); ERROR: month intervals cannot have day or time component \set ON_ERROR_STOP 1 SELECT time, time_bucket(INTERVAL '5 minute', time) FROM unnest(ARRAY[ TIMESTAMP '1970-01-01 00:59:59.999999', TIMESTAMP '1970-01-01 01:01:00', TIMESTAMP '1970-01-01 01:04:59.999999', TIMESTAMP '1970-01-01 01:05:00' ]) AS time; time | time_bucket ---------------------------------+-------------------------- Thu Jan 01 00:59:59.999999 1970 | Thu Jan 01 00:55:00 1970 Thu Jan 01 01:01:00 1970 | Thu Jan 01 01:00:00 1970 Thu Jan 01 01:04:59.999999 1970 | Thu Jan 01 01:00:00 1970 Thu Jan 01 01:05:00 1970 | Thu Jan 01 01:05:00 1970 SELECT time, time_bucket(INTERVAL '5 minute', time) FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:04:59.999999', TIMESTAMP '2011-01-02 01:05:00', TIMESTAMP '2011-01-02 01:09:59.999999', TIMESTAMP '2011-01-02 01:10:00' ]) AS time; time | time_bucket ---------------------------------+-------------------------- Sun Jan 02 01:04:59.999999 2011 | Sun Jan 02 01:00:00 2011 Sun Jan 02 01:05:00 2011 | Sun Jan 02 01:05:00 2011 Sun Jan 02 01:09:59.999999 2011 | Sun Jan 02 01:05:00 2011 Sun Jan 02 01:10:00 2011 | Sun Jan 02 01:10:00 2011 --offset with interval SELECT time, time_bucket(INTERVAL '5 minute', time , INTERVAL '2 minutes') FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:01:59.999999', TIMESTAMP '2011-01-02 01:02:00', TIMESTAMP '2011-01-02 01:06:59.999999', TIMESTAMP '2011-01-02 01:07:00' ]) AS time; time | time_bucket ---------------------------------+-------------------------- Sun Jan 02 01:01:59.999999 2011 | Sun Jan 02 00:57:00 2011 Sun Jan 02 01:02:00 2011 | Sun Jan 02 01:02:00 2011 Sun Jan 02 01:06:59.999999 2011 | Sun Jan 02 01:02:00 2011 Sun Jan 02 01:07:00 2011 | Sun Jan 02 01:07:00 2011 SELECT time, time_bucket(INTERVAL '5 minute', time , - INTERVAL '2 minutes') FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:02:59.999999', TIMESTAMP '2011-01-02 01:03:00', TIMESTAMP '2011-01-02 01:07:59.999999', TIMESTAMP '2011-01-02 01:08:00' ]) AS time; time | time_bucket ---------------------------------+-------------------------- Sun Jan 02 01:02:59.999999 2011 | Sun Jan 02 00:58:00 2011 Sun Jan 02 01:03:00 2011 | Sun Jan 02 01:03:00 2011 Sun Jan 02 01:07:59.999999 2011 | Sun Jan 02 01:03:00 2011 Sun Jan 02 01:08:00 2011 | Sun Jan 02 01:08:00 2011 --offset with infinity -- timestamp SELECT time, time_bucket(INTERVAL '1 week', time, INTERVAL '1 day') FROM unnest(ARRAY[ timestamp '-Infinity', timestamp 'Infinity' ]) AS time; time | time_bucket -----------+------------- -infinity | -infinity infinity | infinity -- timestamptz SELECT time, time_bucket(INTERVAL '1 week', time, INTERVAL '1 day') FROM unnest(ARRAY[ timestamp with time zone '-Infinity', timestamp with time zone 'Infinity' ]) AS time; time | time_bucket -----------+------------- -infinity | -infinity infinity | infinity -- Date SELECT date, time_bucket(INTERVAL '1 week', date, INTERVAL '1 day') FROM unnest(ARRAY[ date '-Infinity', date 'Infinity' ]) AS date; date | time_bucket -----------+------------- -infinity | -infinity infinity | infinity --example to align with an origin SELECT time, time_bucket(INTERVAL '5 minute', time - (TIMESTAMP '2011-01-02 00:02:00' - TIMESTAMP 'epoch')) + (TIMESTAMP '2011-01-02 00:02:00'-TIMESTAMP 'epoch') FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:01:59.999999', TIMESTAMP '2011-01-02 01:02:00', TIMESTAMP '2011-01-02 01:06:59.999999', TIMESTAMP '2011-01-02 01:07:00' ]) AS time; time | ?column? ---------------------------------+-------------------------- Sun Jan 02 01:01:59.999999 2011 | Sun Jan 02 00:57:00 2011 Sun Jan 02 01:02:00 2011 | Sun Jan 02 01:02:00 2011 Sun Jan 02 01:06:59.999999 2011 | Sun Jan 02 01:02:00 2011 Sun Jan 02 01:07:00 2011 | Sun Jan 02 01:07:00 2011 --rounding version SELECT time, time_bucket(INTERVAL '5 minute', time , - INTERVAL '2.5 minutes') + INTERVAL '2 minutes 30 seconds' FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:05:01', TIMESTAMP '2011-01-02 01:07:29', TIMESTAMP '2011-01-02 01:02:30', TIMESTAMP '2011-01-02 01:07:30', TIMESTAMP '2011-01-02 01:02:29' ]) AS time; time | ?column? --------------------------+-------------------------- Sun Jan 02 01:05:01 2011 | Sun Jan 02 01:05:00 2011 Sun Jan 02 01:07:29 2011 | Sun Jan 02 01:05:00 2011 Sun Jan 02 01:02:30 2011 | Sun Jan 02 01:05:00 2011 Sun Jan 02 01:07:30 2011 | Sun Jan 02 01:10:00 2011 Sun Jan 02 01:02:29 2011 | Sun Jan 02 01:00:00 2011 --time_bucket with timezone should mimick date_trunc SET timezone TO 'UTC'; SELECT time, time_bucket(INTERVAL '1 hour', time), date_trunc('hour', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Jan 02 01:01:01 2011 UTC | Sun Jan 02 01:00:00 2011 UTC | Sun Jan 02 01:00:00 2011 UTC Sun Jan 02 00:01:01 2011 UTC | Sun Jan 02 00:00:00 2011 UTC | Sun Jan 02 00:00:00 2011 UTC Sat Jan 01 23:01:01 2011 UTC | Sat Jan 01 23:00:00 2011 UTC | Sat Jan 01 23:00:00 2011 UTC SELECT time, time_bucket(INTERVAL '1 day', time), date_trunc('day', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Jan 02 01:01:01 2011 UTC | Sun Jan 02 00:00:00 2011 UTC | Sun Jan 02 00:00:00 2011 UTC Sun Jan 02 00:01:01 2011 UTC | Sun Jan 02 00:00:00 2011 UTC | Sun Jan 02 00:00:00 2011 UTC Sat Jan 01 23:01:01 2011 UTC | Sat Jan 01 00:00:00 2011 UTC | Sat Jan 01 00:00:00 2011 UTC --what happens with a local tz SET timezone TO 'America/New_York'; SELECT time, time_bucket(INTERVAL '1 hour', time), date_trunc('hour', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Jan 02 01:01:01 2011 EST | Sun Jan 02 01:00:00 2011 EST | Sun Jan 02 01:00:00 2011 EST Sat Jan 01 19:01:01 2011 EST | Sat Jan 01 19:00:00 2011 EST | Sat Jan 01 19:00:00 2011 EST Sat Jan 01 18:01:01 2011 EST | Sat Jan 01 18:00:00 2011 EST | Sat Jan 01 18:00:00 2011 EST --Note the timestamp tz input is aligned with UTC day /not/ local day. different than date_trunc. SELECT time, time_bucket(INTERVAL '1 day', time), date_trunc('day', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-03 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-04 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Jan 02 01:01:01 2011 EST | Sat Jan 01 19:00:00 2011 EST | Sun Jan 02 00:00:00 2011 EST Sun Jan 02 19:01:01 2011 EST | Sun Jan 02 19:00:00 2011 EST | Sun Jan 02 00:00:00 2011 EST Mon Jan 03 18:01:01 2011 EST | Sun Jan 02 19:00:00 2011 EST | Mon Jan 03 00:00:00 2011 EST --can force local bucketing with simple cast. SELECT time, time_bucket(INTERVAL '1 day', time::timestamp), date_trunc('day', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-03 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-04 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+--------------------------+------------------------------ Sun Jan 02 01:01:01 2011 EST | Sun Jan 02 00:00:00 2011 | Sun Jan 02 00:00:00 2011 EST Sun Jan 02 19:01:01 2011 EST | Sun Jan 02 00:00:00 2011 | Sun Jan 02 00:00:00 2011 EST Mon Jan 03 18:01:01 2011 EST | Mon Jan 03 00:00:00 2011 | Mon Jan 03 00:00:00 2011 EST --can also use interval to correct SELECT time, time_bucket(INTERVAL '1 day', time, -INTERVAL '19 hours'), date_trunc('day', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-03 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-04 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Jan 02 01:01:01 2011 EST | Sun Jan 02 00:00:00 2011 EST | Sun Jan 02 00:00:00 2011 EST Sun Jan 02 19:01:01 2011 EST | Sun Jan 02 00:00:00 2011 EST | Sun Jan 02 00:00:00 2011 EST Mon Jan 03 18:01:01 2011 EST | Mon Jan 03 00:00:00 2011 EST | Mon Jan 03 00:00:00 2011 EST --dst: same local hour bucketed as two different hours. SELECT time, time_bucket(INTERVAL '1 hour', time), date_trunc('hour', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2017-11-05 12:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 13:05:00+07' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Nov 05 01:05:00 2017 EDT | Sun Nov 05 01:00:00 2017 EDT | Sun Nov 05 01:00:00 2017 EDT Sun Nov 05 01:05:00 2017 EST | Sun Nov 05 01:00:00 2017 EST | Sun Nov 05 01:00:00 2017 EST --local alignment changes when bucketing by UTC across dst boundary SELECT time, time_bucket(INTERVAL '2 hour', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2017-11-05 10:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 12:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 13:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 15:05:00+07' ]) AS time; time | time_bucket ------------------------------+------------------------------ Sat Nov 04 23:05:00 2017 EDT | Sat Nov 04 22:00:00 2017 EDT Sun Nov 05 01:05:00 2017 EDT | Sun Nov 05 00:00:00 2017 EDT Sun Nov 05 01:05:00 2017 EST | Sun Nov 05 01:00:00 2017 EST Sun Nov 05 03:05:00 2017 EST | Sun Nov 05 03:00:00 2017 EST --local alignment is preserved when bucketing by local time across DST boundary. SELECT time, time_bucket(INTERVAL '2 hour', time::timestamp) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2017-11-05 10:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 12:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 13:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 15:05:00+07' ]) AS time; time | time_bucket ------------------------------+-------------------------- Sat Nov 04 23:05:00 2017 EDT | Sat Nov 04 22:00:00 2017 Sun Nov 05 01:05:00 2017 EDT | Sun Nov 05 00:00:00 2017 Sun Nov 05 01:05:00 2017 EST | Sun Nov 05 00:00:00 2017 Sun Nov 05 03:05:00 2017 EST | Sun Nov 05 02:00:00 2017 -- GitHub issue #7059: time_bucket with timezone + offset across DST boundary -- Asia/Amman: clocks skip from 00:00 to 01:00 on 2021-03-26 -- Input: 01:00+03 = 22:00 UTC → Result: 22:15 UTC = 01:15 local (00:00 + 15min offset) SELECT time_bucket('1 day', '2021-03-26 01:00:00+03'::timestamptz, timezone := 'Asia/Amman', "offset" := '15 minutes'::interval); time_bucket ------------------------------ Wed Mar 24 18:15:00 2021 EDT -- GitHub issue #8851: time_bucket with negative offset during DST fall-back -- Europe/Berlin: clocks repeat 02:00-02:59 on 2025-10-26 -- Input: 02:00+02 = 00:00 UTC → Result: 23:59:45 UTC = 01:59:45 local (02:00 - 15s offset) SELECT time_bucket('30 seconds', '2025-10-26 02:00:00+02'::timestamptz, timezone := 'Europe/Berlin', "offset" := '-15 seconds'::interval); time_bucket ------------------------------ Sat Oct 25 19:59:45 2025 EDT -- Additional DST edge cases for coverage of DST direction × offset sign combinations -- Spring-forward + negative offset -- Input: 01:30+03 = 22:30 UTC → Result: 22:45 UTC = 01:45 local (01:00 + 45min = 02:00 - 15min) SELECT time_bucket('1 hour', '2021-03-26 01:30:00+03'::timestamptz, timezone := 'Asia/Amman', "offset" := '-15 minutes'::interval); time_bucket ------------------------------ Thu Mar 25 17:45:00 2021 EDT -- Fall-back + positive offset -- Input: 02:30+01 = 01:30 UTC → Result: 01:15 UTC = 02:15 local (02:00 + 15min offset) SELECT time_bucket('1 hour', '2025-10-26 02:30:00+01'::timestamptz, timezone := 'Europe/Berlin', "offset" := '15 minutes'::interval); time_bucket ------------------------------ Sat Oct 25 21:15:00 2025 EDT -- Input exactly at DST spring-forward transition -- Input: 22:00 UTC = 00:00 local (the moment clocks jump to 01:00) -- Result: 22:15 UTC = 01:15 local (01:00 + 15min offset) SELECT time_bucket('1 hour', '2021-03-25 22:00:00+00'::timestamptz, timezone := 'Asia/Amman', "offset" := '15 minutes'::interval); time_bucket ------------------------------ Thu Mar 25 17:15:00 2021 EDT -- Input exactly at DST fall-back transition -- Input: 01:00 UTC = 03:00 CEST (the moment clocks go back to 02:00 CET) -- Result: 23:15 UTC = 01:15 local (01:00 + 15min offset, but in CET now) SELECT time_bucket('1 hour', '2025-10-26 01:00:00+00'::timestamptz, timezone := 'Europe/Berlin', "offset" := '15 minutes'::interval); time_bucket ------------------------------ Sat Oct 25 20:15:00 2025 EDT -- Offset larger than bucket size (1h offset with 30min bucket) -- Input: 01:30+03 = 22:30 UTC → Result: 22:30 UTC = 01:30 local (01:00 + 30min = 00:30 + 1h) SELECT time_bucket('30 minutes', '2021-03-26 01:30:00+03'::timestamptz, timezone := 'Asia/Amman', "offset" := '1 hour'::interval); time_bucket ------------------------------ Thu Mar 25 18:30:00 2021 EDT -- GitHub issue #9136: time_bucket with origin during DST fall-back -- When origin is in standard time but timestamp is in daylight time, -- the bucket could incorrectly start AFTER the timestamp. -- America/New_York: clocks go back at 02:00 EDT on 2024-11-03 -- Input: 01:30-04 (EDT) = 05:30 UTC; origin in EST -- Result should have bucket start <= timestamp (bucket in EDT, not EST) SELECT time_bucket('1 hour', '2024-11-03 01:30:00-04'::timestamptz, 'America/New_York', '2000-01-01 00:00:00 America/New_York'::timestamptz) as bucket, '2024-11-03 01:30:00-04'::timestamptz < time_bucket('1 hour', '2024-11-03 01:30:00-04'::timestamptz, 'America/New_York', '2000-01-01 00:00:00 America/New_York'::timestamptz) as ts_before_bucket; bucket | ts_before_bucket ------------------------------+------------------ Sun Nov 03 01:00:00 2024 EDT | f SELECT time, time_bucket(10::smallint, time) AS time_bucket_smallint, time_bucket(10::int, time) AS time_bucket_int, time_bucket(10::bigint, time) AS time_bucket_bigint FROM unnest(ARRAY[ '-11', '-10', '-9', '-1', '0', '1', '99', '100', '109', '110' ]::smallint[]) AS time; time | time_bucket_smallint | time_bucket_int | time_bucket_bigint ------+----------------------+-----------------+-------------------- -11 | -20 | -20 | -20 -10 | -10 | -10 | -10 -9 | -10 | -10 | -10 -1 | -10 | -10 | -10 0 | 0 | 0 | 0 1 | 0 | 0 | 0 99 | 90 | 90 | 90 100 | 100 | 100 | 100 109 | 100 | 100 | 100 110 | 110 | 110 | 110 SELECT time, time_bucket(10::smallint, time, 2::smallint) AS time_bucket_smallint, time_bucket(10::int, time, 2::int) AS time_bucket_int, time_bucket(10::bigint, time, 2::bigint) AS time_bucket_bigint FROM unnest(ARRAY[ '-9', '-8', '-7', '1', '2', '3', '101', '102', '111', '112' ]::smallint[]) AS time; time | time_bucket_smallint | time_bucket_int | time_bucket_bigint ------+----------------------+-----------------+-------------------- -9 | -18 | -18 | -18 -8 | -8 | -8 | -8 -7 | -8 | -8 | -8 1 | -8 | -8 | -8 2 | 2 | 2 | 2 3 | 2 | 2 | 2 101 | 92 | 92 | 92 102 | 102 | 102 | 102 111 | 102 | 102 | 102 112 | 112 | 112 | 112 SELECT time, time_bucket(10::smallint, time, -2::smallint) AS time_bucket_smallint, time_bucket(10::int, time, -2::int) AS time_bucket_int, time_bucket(10::bigint, time, -2::bigint) AS time_bucket_bigint FROM unnest(ARRAY[ '-13', '-12', '-11', '-3', '-2', '-1', '97', '98', '107', '108' ]::smallint[]) AS time; time | time_bucket_smallint | time_bucket_int | time_bucket_bigint ------+----------------------+-----------------+-------------------- -13 | -22 | -22 | -22 -12 | -12 | -12 | -12 -11 | -12 | -12 | -12 -3 | -12 | -12 | -12 -2 | -2 | -2 | -2 -1 | -2 | -2 | -2 97 | 88 | 88 | 88 98 | 98 | 98 | 98 107 | 98 | 98 | 98 108 | 108 | 108 | 108 \set ON_ERROR_STOP 0 SELECT time_bucket(10::smallint, '-32768'::smallint); ERROR: timestamp out of range SELECT time_bucket(10::smallint, '-32761'::smallint); ERROR: timestamp out of range select time_bucket(10::smallint, '-32768'::smallint, 1000::smallint); ERROR: timestamp out of range select time_bucket(10::smallint, '-32768'::smallint, '32767'::smallint); ERROR: timestamp out of range select time_bucket(10::smallint, '32767'::smallint, '-32768'::smallint); ERROR: timestamp out of range \set ON_ERROR_STOP 1 SELECT time, time_bucket(10::smallint, time) FROM unnest(ARRAY[ '-32760', '-32759', '32767' ]::smallint[]) AS time; time | time_bucket --------+------------- -32760 | -32760 -32759 | -32760 32767 | 32760 \set ON_ERROR_STOP 0 SELECT time_bucket(10::int, '-2147483648'::int); ERROR: timestamp out of range SELECT time_bucket(10::int, '-2147483641'::int); ERROR: timestamp out of range SELECT time_bucket(1000::int, '-2147483000'::int, 1::int); ERROR: timestamp out of range SELECT time_bucket(1000::int, '-2147483648'::int, '2147483647'::int); ERROR: timestamp out of range SELECT time_bucket(1000::int, '2147483647'::int, '-2147483648'::int); ERROR: timestamp out of range \set ON_ERROR_STOP 1 SELECT time, time_bucket(10::int, time) FROM unnest(ARRAY[ '-2147483640', '-2147483639', '2147483647' ]::int[]) AS time; time | time_bucket -------------+------------- -2147483640 | -2147483640 -2147483639 | -2147483640 2147483647 | 2147483640 \set ON_ERROR_STOP 0 SELECT time_bucket(10::bigint, '-9223372036854775808'::bigint); ERROR: timestamp out of range SELECT time_bucket(10::bigint, '-9223372036854775801'::bigint); ERROR: timestamp out of range SELECT time_bucket(1000::bigint, '-9223372036854775000'::bigint, 1::bigint); ERROR: timestamp out of range SELECT time_bucket(1000::bigint, '-9223372036854775808'::bigint, '9223372036854775807'::bigint); ERROR: timestamp out of range SELECT time_bucket(1000::bigint, '9223372036854775807'::bigint, '-9223372036854775808'::bigint); ERROR: timestamp out of range \set ON_ERROR_STOP 1 SELECT time, time_bucket(10::bigint, time) FROM unnest(ARRAY[ '-9223372036854775800', '-9223372036854775799', '9223372036854775807' ]::bigint[]) AS time; time | time_bucket ----------------------+---------------------- -9223372036854775800 | -9223372036854775800 -9223372036854775799 | -9223372036854775800 9223372036854775807 | 9223372036854775800 SELECT time, time_bucket(INTERVAL '1 day', time::date) FROM unnest(ARRAY[ date '2017-11-05', date '2017-11-06' ]) AS time; time | time_bucket ------------+------------- 11-05-2017 | 11-05-2017 11-06-2017 | 11-06-2017 SELECT time, time_bucket(INTERVAL '4 day', time::date) FROM unnest(ARRAY[ date '2017-11-04', date '2017-11-05', date '2017-11-08', date '2017-11-09' ]) AS time; time | time_bucket ------------+------------- 11-04-2017 | 11-01-2017 11-05-2017 | 11-05-2017 11-08-2017 | 11-05-2017 11-09-2017 | 11-09-2017 SELECT time, time_bucket(INTERVAL '4 day', time::date, INTERVAL '2 day') FROM unnest(ARRAY[ date '2017-11-06', date '2017-11-07', date '2017-11-10', date '2017-11-11' ]) AS time; time | time_bucket ------------+------------- 11-06-2017 | 11-03-2017 11-07-2017 | 11-07-2017 11-10-2017 | 11-07-2017 11-11-2017 | 11-11-2017 -- 2019-09-24 is a Monday, and we want to ensure that time_bucket returns the week starting with a Monday as date_trunc does, -- Rather than a Saturday which is the date of the PostgreSQL epoch SELECT time, time_bucket(INTERVAL '1 week', time::date) FROM unnest(ARRAY[ date '2018-09-16', date '2018-09-17', date '2018-09-23', date '2018-09-24' ]) AS time; time | time_bucket ------------+------------- 09-16-2018 | 09-10-2018 09-17-2018 | 09-17-2018 09-23-2018 | 09-17-2018 09-24-2018 | 09-24-2018 SELECT time, time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp without time zone '2018-09-16', timestamp without time zone '2018-09-17', timestamp without time zone '2018-09-23', timestamp without time zone '2018-09-24' ]) AS time; time | time_bucket --------------------------+-------------------------- Sun Sep 16 00:00:00 2018 | Mon Sep 10 00:00:00 2018 Mon Sep 17 00:00:00 2018 | Mon Sep 17 00:00:00 2018 Sun Sep 23 00:00:00 2018 | Mon Sep 17 00:00:00 2018 Mon Sep 24 00:00:00 2018 | Mon Sep 24 00:00:00 2018 SELECT time, time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp with time zone '2018-09-16', timestamp with time zone '2018-09-17', timestamp with time zone '2018-09-23', timestamp with time zone '2018-09-24' ]) AS time; time | time_bucket ------------------------------+------------------------------ Sun Sep 16 00:00:00 2018 EDT | Sun Sep 09 20:00:00 2018 EDT Mon Sep 17 00:00:00 2018 EDT | Sun Sep 16 20:00:00 2018 EDT Sun Sep 23 00:00:00 2018 EDT | Sun Sep 16 20:00:00 2018 EDT Mon Sep 24 00:00:00 2018 EDT | Sun Sep 23 20:00:00 2018 EDT SELECT time, time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp with time zone '-Infinity', timestamp with time zone 'Infinity' ]) AS time; time | time_bucket -----------+------------- -infinity | -infinity infinity | infinity SELECT time, time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp without time zone '-Infinity', timestamp without time zone 'Infinity' ]) AS time; time | time_bucket -----------+------------- -infinity | -infinity infinity | infinity SELECT time, time_bucket(INTERVAL '1 week', time), date_trunc('week', time) = time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp without time zone '4714-11-24 01:01:01.0 BC', timestamp without time zone '294276-12-31 23:59:59.9999' ]) AS time; time | time_bucket | ?column? ---------------------------------+-----------------------------+---------- Mon Nov 24 01:01:01 4714 BC | Mon Nov 24 00:00:00 4714 BC | t Sun Dec 31 23:59:59.9999 294276 | Mon Dec 25 00:00:00 294276 | t --1000 years later weeks still align. SELECT time, time_bucket(INTERVAL '1 week', time), date_trunc('week', time) = time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp without time zone '3018-09-14', timestamp without time zone '3018-09-20', timestamp without time zone '3018-09-21', timestamp without time zone '3018-09-22' ]) AS time; time | time_bucket | ?column? --------------------------+--------------------------+---------- Mon Sep 14 00:00:00 3018 | Mon Sep 14 00:00:00 3018 | t Sun Sep 20 00:00:00 3018 | Mon Sep 14 00:00:00 3018 | t Mon Sep 21 00:00:00 3018 | Mon Sep 21 00:00:00 3018 | t Tue Sep 22 00:00:00 3018 | Mon Sep 21 00:00:00 3018 | t --weeks align for timestamptz as well if cast to local time, (but not if done at UTC). SELECT time, date_trunc('week', time) = time_bucket(INTERVAL '1 week', time), date_trunc('week', time) = time_bucket(INTERVAL '1 week', time::timestamp) FROM unnest(ARRAY[ timestamp with time zone '3018-09-14', timestamp with time zone '3018-09-20', timestamp with time zone '3018-09-21', timestamp with time zone '3018-09-22' ]) AS time; time | ?column? | ?column? ------------------------------+----------+---------- Mon Sep 14 00:00:00 3018 EDT | f | t Sun Sep 20 00:00:00 3018 EDT | f | t Mon Sep 21 00:00:00 3018 EDT | f | t Tue Sep 22 00:00:00 3018 EDT | f | t --check functions with origin --note that the default origin is at 0 UTC, using origin parameter it is easy to provide a EDT origin point \x SELECT time, time_bucket(INTERVAL '1 week', time) no_epoch, time_bucket(INTERVAL '1 week', time::timestamp) no_epoch_local, time_bucket(INTERVAL '1 week', time) = time_bucket(INTERVAL '1 week', time, timestamptz '2000-01-03 00:00:00+0') always_true, time_bucket(INTERVAL '1 week', time, timestamptz '2000-01-01 00:00:00+0') pg_epoch, time_bucket(INTERVAL '1 week', time, timestamptz 'epoch') unix_epoch, time_bucket(INTERVAL '1 week', time, timestamptz '3018-09-13') custom_1, time_bucket(INTERVAL '1 week', time, timestamptz '3018-09-14') custom_2 FROM unnest(ARRAY[ timestamp with time zone '2000-01-01 00:00:00+0'- interval '1 second', timestamp with time zone '2000-01-01 00:00:00+0', timestamp with time zone '2000-01-03 00:00:00+0'- interval '1 second', timestamp with time zone '2000-01-03 00:00:00+0', timestamp with time zone '2000-01-01', timestamp with time zone '2000-01-02', timestamp with time zone '2000-01-03', timestamp with time zone '3018-09-12', timestamp with time zone '3018-09-13', timestamp with time zone '3018-09-14', timestamp with time zone '3018-09-15' ]) AS time; -[ RECORD 1 ]--+----------------------------- time | Fri Dec 31 18:59:59 1999 EST no_epoch | Sun Dec 26 19:00:00 1999 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 24 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Dec 25 23:00:00 1999 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 2 ]--+----------------------------- time | Fri Dec 31 19:00:00 1999 EST no_epoch | Sun Dec 26 19:00:00 1999 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Dec 25 23:00:00 1999 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 3 ]--+----------------------------- time | Sun Jan 02 18:59:59 2000 EST no_epoch | Sun Dec 26 19:00:00 1999 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Jan 01 23:00:00 2000 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 4 ]--+----------------------------- time | Sun Jan 02 19:00:00 2000 EST no_epoch | Sun Jan 02 19:00:00 2000 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Jan 01 23:00:00 2000 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 5 ]--+----------------------------- time | Sat Jan 01 00:00:00 2000 EST no_epoch | Sun Dec 26 19:00:00 1999 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Dec 25 23:00:00 1999 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 6 ]--+----------------------------- time | Sun Jan 02 00:00:00 2000 EST no_epoch | Sun Dec 26 19:00:00 1999 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Jan 01 23:00:00 2000 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 7 ]--+----------------------------- time | Mon Jan 03 00:00:00 2000 EST no_epoch | Sun Jan 02 19:00:00 2000 EST no_epoch_local | Mon Jan 03 00:00:00 2000 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Jan 01 23:00:00 2000 EST custom_2 | Sun Jan 02 23:00:00 2000 EST -[ RECORD 8 ]--+----------------------------- time | Sat Sep 12 00:00:00 3018 EDT no_epoch | Sun Sep 06 20:00:00 3018 EDT no_epoch_local | Mon Sep 07 00:00:00 3018 always_true | t pg_epoch | Fri Sep 11 20:00:00 3018 EDT unix_epoch | Wed Sep 09 20:00:00 3018 EDT custom_1 | Sun Sep 06 00:00:00 3018 EDT custom_2 | Mon Sep 07 00:00:00 3018 EDT -[ RECORD 9 ]--+----------------------------- time | Sun Sep 13 00:00:00 3018 EDT no_epoch | Sun Sep 06 20:00:00 3018 EDT no_epoch_local | Mon Sep 07 00:00:00 3018 always_true | t pg_epoch | Fri Sep 11 20:00:00 3018 EDT unix_epoch | Wed Sep 09 20:00:00 3018 EDT custom_1 | Sun Sep 13 00:00:00 3018 EDT custom_2 | Mon Sep 07 00:00:00 3018 EDT -[ RECORD 10 ]-+----------------------------- time | Mon Sep 14 00:00:00 3018 EDT no_epoch | Sun Sep 13 20:00:00 3018 EDT no_epoch_local | Mon Sep 14 00:00:00 3018 always_true | t pg_epoch | Fri Sep 11 20:00:00 3018 EDT unix_epoch | Wed Sep 09 20:00:00 3018 EDT custom_1 | Sun Sep 13 00:00:00 3018 EDT custom_2 | Mon Sep 14 00:00:00 3018 EDT -[ RECORD 11 ]-+----------------------------- time | Tue Sep 15 00:00:00 3018 EDT no_epoch | Sun Sep 13 20:00:00 3018 EDT no_epoch_local | Mon Sep 14 00:00:00 3018 always_true | t pg_epoch | Fri Sep 11 20:00:00 3018 EDT unix_epoch | Wed Sep 09 20:00:00 3018 EDT custom_1 | Sun Sep 13 00:00:00 3018 EDT custom_2 | Mon Sep 14 00:00:00 3018 EDT SELECT time, time_bucket(INTERVAL '1 week', time) no_epoch, time_bucket(INTERVAL '1 week', time) = time_bucket(INTERVAL '1 week', time, timestamp '2000-01-03 00:00:00') always_true, time_bucket(INTERVAL '1 week', time, timestamp '2000-01-01 00:00:00+0') pg_epoch, time_bucket(INTERVAL '1 week', time, timestamp 'epoch') unix_epoch, time_bucket(INTERVAL '1 week', time, timestamp '3018-09-13') custom_1, time_bucket(INTERVAL '1 week', time, timestamp '3018-09-14') custom_2 FROM unnest(ARRAY[ timestamp without time zone '2000-01-01 00:00:00'- interval '1 second', timestamp without time zone '2000-01-01 00:00:00', timestamp without time zone '2000-01-03 00:00:00'- interval '1 second', timestamp without time zone '2000-01-03 00:00:00', timestamp without time zone '2000-01-01', timestamp without time zone '2000-01-02', timestamp without time zone '2000-01-03', timestamp without time zone '3018-09-12', timestamp without time zone '3018-09-13', timestamp without time zone '3018-09-14', timestamp without time zone '3018-09-15' ]) AS time; -[ RECORD 1 ]------------------------- time | Fri Dec 31 23:59:59 1999 no_epoch | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Sat Dec 25 00:00:00 1999 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Dec 26 00:00:00 1999 custom_2 | Mon Dec 27 00:00:00 1999 -[ RECORD 2 ]------------------------- time | Sat Jan 01 00:00:00 2000 no_epoch | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Dec 26 00:00:00 1999 custom_2 | Mon Dec 27 00:00:00 1999 -[ RECORD 3 ]------------------------- time | Sun Jan 02 23:59:59 2000 no_epoch | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Jan 02 00:00:00 2000 custom_2 | Mon Dec 27 00:00:00 1999 -[ RECORD 4 ]------------------------- time | Mon Jan 03 00:00:00 2000 no_epoch | Mon Jan 03 00:00:00 2000 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Jan 02 00:00:00 2000 custom_2 | Mon Jan 03 00:00:00 2000 -[ RECORD 5 ]------------------------- time | Sat Jan 01 00:00:00 2000 no_epoch | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Dec 26 00:00:00 1999 custom_2 | Mon Dec 27 00:00:00 1999 -[ RECORD 6 ]------------------------- time | Sun Jan 02 00:00:00 2000 no_epoch | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Jan 02 00:00:00 2000 custom_2 | Mon Dec 27 00:00:00 1999 -[ RECORD 7 ]------------------------- time | Mon Jan 03 00:00:00 2000 no_epoch | Mon Jan 03 00:00:00 2000 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Jan 02 00:00:00 2000 custom_2 | Mon Jan 03 00:00:00 2000 -[ RECORD 8 ]------------------------- time | Sat Sep 12 00:00:00 3018 no_epoch | Mon Sep 07 00:00:00 3018 always_true | t pg_epoch | Sat Sep 12 00:00:00 3018 unix_epoch | Thu Sep 10 00:00:00 3018 custom_1 | Sun Sep 06 00:00:00 3018 custom_2 | Mon Sep 07 00:00:00 3018 -[ RECORD 9 ]------------------------- time | Sun Sep 13 00:00:00 3018 no_epoch | Mon Sep 07 00:00:00 3018 always_true | t pg_epoch | Sat Sep 12 00:00:00 3018 unix_epoch | Thu Sep 10 00:00:00 3018 custom_1 | Sun Sep 13 00:00:00 3018 custom_2 | Mon Sep 07 00:00:00 3018 -[ RECORD 10 ]------------------------ time | Mon Sep 14 00:00:00 3018 no_epoch | Mon Sep 14 00:00:00 3018 always_true | t pg_epoch | Sat Sep 12 00:00:00 3018 unix_epoch | Thu Sep 10 00:00:00 3018 custom_1 | Sun Sep 13 00:00:00 3018 custom_2 | Mon Sep 14 00:00:00 3018 -[ RECORD 11 ]------------------------ time | Tue Sep 15 00:00:00 3018 no_epoch | Mon Sep 14 00:00:00 3018 always_true | t pg_epoch | Sat Sep 12 00:00:00 3018 unix_epoch | Thu Sep 10 00:00:00 3018 custom_1 | Sun Sep 13 00:00:00 3018 custom_2 | Mon Sep 14 00:00:00 3018 SELECT time, time_bucket(INTERVAL '1 week', time) no_epoch, time_bucket(INTERVAL '1 week', time) = time_bucket(INTERVAL '1 week', time, date '2000-01-03') always_true, time_bucket(INTERVAL '1 week', time, date '2000-01-01') pg_epoch, time_bucket(INTERVAL '1 week', time, (timestamp 'epoch')::date) unix_epoch, time_bucket(INTERVAL '1 week', time, date '3018-09-13') custom_1, time_bucket(INTERVAL '1 week', time, date '3018-09-14') custom_2 FROM unnest(ARRAY[ date '1999-12-31', date '2000-01-01', date '2000-01-02', date '2000-01-03', date '3018-09-12', date '3018-09-13', date '3018-09-14', date '3018-09-15' ]) AS time; -[ RECORD 1 ]----------- time | 12-31-1999 no_epoch | 12-27-1999 always_true | t pg_epoch | 12-25-1999 unix_epoch | 12-30-1999 custom_1 | 12-26-1999 custom_2 | 12-27-1999 -[ RECORD 2 ]----------- time | 01-01-2000 no_epoch | 12-27-1999 always_true | t pg_epoch | 01-01-2000 unix_epoch | 12-30-1999 custom_1 | 12-26-1999 custom_2 | 12-27-1999 -[ RECORD 3 ]----------- time | 01-02-2000 no_epoch | 12-27-1999 always_true | t pg_epoch | 01-01-2000 unix_epoch | 12-30-1999 custom_1 | 01-02-2000 custom_2 | 12-27-1999 -[ RECORD 4 ]----------- time | 01-03-2000 no_epoch | 01-03-2000 always_true | t pg_epoch | 01-01-2000 unix_epoch | 12-30-1999 custom_1 | 01-02-2000 custom_2 | 01-03-2000 -[ RECORD 5 ]----------- time | 09-12-3018 no_epoch | 09-07-3018 always_true | t pg_epoch | 09-12-3018 unix_epoch | 09-10-3018 custom_1 | 09-06-3018 custom_2 | 09-07-3018 -[ RECORD 6 ]----------- time | 09-13-3018 no_epoch | 09-07-3018 always_true | t pg_epoch | 09-12-3018 unix_epoch | 09-10-3018 custom_1 | 09-13-3018 custom_2 | 09-07-3018 -[ RECORD 7 ]----------- time | 09-14-3018 no_epoch | 09-14-3018 always_true | t pg_epoch | 09-12-3018 unix_epoch | 09-10-3018 custom_1 | 09-13-3018 custom_2 | 09-14-3018 -[ RECORD 8 ]----------- time | 09-15-3018 no_epoch | 09-14-3018 always_true | t pg_epoch | 09-12-3018 unix_epoch | 09-10-3018 custom_1 | 09-13-3018 custom_2 | 09-14-3018 \x --really old origin works if date around that time SELECT time, time_bucket(INTERVAL '1 week', time, timestamp without time zone '4710-11-24 01:01:01.0 BC') FROM unnest(ARRAY[ timestamp without time zone '4710-11-24 01:01:01.0 BC', timestamp without time zone '4710-11-25 01:01:01.0 BC', timestamp without time zone '2001-01-01', timestamp without time zone '3001-01-01' ]) AS time; time | time_bucket -----------------------------+----------------------------- Sat Nov 24 01:01:01 4710 BC | Sat Nov 24 01:01:01 4710 BC Sun Nov 25 01:01:01 4710 BC | Sat Nov 24 01:01:01 4710 BC Mon Jan 01 00:00:00 2001 | Sat Dec 30 01:01:01 2000 Thu Jan 01 00:00:00 3001 | Sat Dec 27 01:01:01 3000 SELECT time, time_bucket(INTERVAL '1 week', time, timestamp without time zone '294270-12-30 23:59:59.9999') FROM unnest(ARRAY[ timestamp without time zone '294270-12-29 23:59:59.9999', timestamp without time zone '294270-12-30 23:59:59.9999', timestamp without time zone '294270-12-31 23:59:59.9999', timestamp without time zone '2001-01-01', timestamp without time zone '3001-01-01' ]) AS time; time | time_bucket ---------------------------------+--------------------------------- Thu Dec 29 23:59:59.9999 294270 | Fri Dec 23 23:59:59.9999 294270 Fri Dec 30 23:59:59.9999 294270 | Fri Dec 30 23:59:59.9999 294270 Sat Dec 31 23:59:59.9999 294270 | Fri Dec 30 23:59:59.9999 294270 Mon Jan 01 00:00:00 2001 | Fri Dec 29 23:59:59.9999 2000 Thu Jan 01 00:00:00 3001 | Fri Dec 26 23:59:59.9999 3000 \set ON_ERROR_STOP 0 --really old origin + very new data + long period errors SELECT time, time_bucket(INTERVAL '100000 day', time, timestamp without time zone '4710-11-24 01:01:01.0 BC') FROM unnest(ARRAY[ timestamp without time zone '294270-12-31 23:59:59.9999' ]) AS time; ERROR: timestamp out of range SELECT time, time_bucket(INTERVAL '100000 day', time, timestamp with time zone '4710-11-25 01:01:01.0 BC') FROM unnest(ARRAY[ timestamp with time zone '294270-12-30 23:59:59.9999' ]) AS time; ERROR: timestamp out of range --really high origin + old data + long period errors out SELECT time, time_bucket(INTERVAL '10000000 day', time, timestamp without time zone '294270-12-31 23:59:59.9999') FROM unnest(ARRAY[ timestamp without time zone '4710-11-24 01:01:01.0 BC' ]) AS time; ERROR: timestamp out of range SELECT time, time_bucket(INTERVAL '10000000 day', time, timestamp with time zone '294270-12-31 23:59:59.9999') FROM unnest(ARRAY[ timestamp with time zone '4710-11-24 01:01:01.0 BC' ]) AS time; ERROR: timestamp out of range \set ON_ERROR_STOP 1 ------------------------------------------- --- Test time_bucket with month periods --- ------------------------------------------- SET datestyle TO ISO; SELECT time::date, time_bucket('1 month', time::date) AS "1m", time_bucket('2 month', time::date) AS "2m", time_bucket('3 month', time::date) AS "3m", time_bucket('1 month', time::date, '2000-02-01'::date) AS "1m origin", time_bucket('2 month', time::date, '2000-02-01'::date) AS "2m origin", time_bucket('3 month', time::date, '2000-02-01'::date) AS "3m origin" FROM generate_series('1990-01-03'::date,'1990-06-03'::date,'1month'::interval) time; time | 1m | 2m | 3m | 1m origin | 2m origin | 3m origin ------------+------------+------------+------------+------------+------------+------------ 1990-01-03 | 1990-01-01 | 1990-01-01 | 1990-01-01 | 1990-01-01 | 1989-12-01 | 1989-11-01 1990-02-03 | 1990-02-01 | 1990-01-01 | 1990-01-01 | 1990-02-01 | 1990-02-01 | 1990-02-01 1990-03-03 | 1990-03-01 | 1990-03-01 | 1990-01-01 | 1990-03-01 | 1990-02-01 | 1990-02-01 1990-04-03 | 1990-04-01 | 1990-03-01 | 1990-04-01 | 1990-04-01 | 1990-04-01 | 1990-02-01 1990-05-03 | 1990-05-01 | 1990-05-01 | 1990-04-01 | 1990-05-01 | 1990-04-01 | 1990-05-01 1990-06-03 | 1990-06-01 | 1990-05-01 | 1990-04-01 | 1990-06-01 | 1990-06-01 | 1990-05-01 SELECT time, time_bucket('1 month', time) AS "1m", time_bucket('2 month', time) AS "2m", time_bucket('3 month', time) AS "3m", time_bucket('1 month', time, '2000-02-01'::timestamp) AS "1m origin", time_bucket('2 month', time, '2000-02-01'::timestamp) AS "2m origin", time_bucket('3 month', time, '2000-02-01'::timestamp) AS "3m origin" FROM generate_series('1990-01-03'::timestamp,'1990-06-03'::timestamp,'1month'::interval) time; time | 1m | 2m | 3m | 1m origin | 2m origin | 3m origin ---------------------+---------------------+---------------------+---------------------+---------------------+---------------------+--------------------- 1990-01-03 00:00:00 | 1990-01-01 00:00:00 | 1990-01-01 00:00:00 | 1990-01-01 00:00:00 | 1990-01-01 00:00:00 | 1989-12-01 00:00:00 | 1989-11-01 00:00:00 1990-02-03 00:00:00 | 1990-02-01 00:00:00 | 1990-01-01 00:00:00 | 1990-01-01 00:00:00 | 1990-02-01 00:00:00 | 1990-02-01 00:00:00 | 1990-02-01 00:00:00 1990-03-03 00:00:00 | 1990-03-01 00:00:00 | 1990-03-01 00:00:00 | 1990-01-01 00:00:00 | 1990-03-01 00:00:00 | 1990-02-01 00:00:00 | 1990-02-01 00:00:00 1990-04-03 00:00:00 | 1990-04-01 00:00:00 | 1990-03-01 00:00:00 | 1990-04-01 00:00:00 | 1990-04-01 00:00:00 | 1990-04-01 00:00:00 | 1990-02-01 00:00:00 1990-05-03 00:00:00 | 1990-05-01 00:00:00 | 1990-05-01 00:00:00 | 1990-04-01 00:00:00 | 1990-05-01 00:00:00 | 1990-04-01 00:00:00 | 1990-05-01 00:00:00 1990-06-03 00:00:00 | 1990-06-01 00:00:00 | 1990-05-01 00:00:00 | 1990-04-01 00:00:00 | 1990-06-01 00:00:00 | 1990-06-01 00:00:00 | 1990-05-01 00:00:00 SELECT time, time_bucket('1 month', time) AS "1m", time_bucket('2 month', time) AS "2m", time_bucket('3 month', time) AS "3m", time_bucket('1 month', time, '2000-02-01'::timestamptz) AS "1m origin", time_bucket('2 month', time, '2000-02-01'::timestamptz) AS "2m origin", time_bucket('3 month', time, '2000-02-01'::timestamptz) AS "3m origin" FROM generate_series('1990-01-03'::timestamptz,'1990-06-03'::timestamptz,'1month'::interval) time; time | 1m | 2m | 3m | 1m origin | 2m origin | 3m origin ------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+------------------------ 1990-01-03 00:00:00-05 | 1989-12-31 19:00:00-05 | 1989-12-31 19:00:00-05 | 1989-12-31 19:00:00-05 | 1989-12-31 19:00:00-05 | 1989-11-30 19:00:00-05 | 1989-10-31 19:00:00-05 1990-02-03 00:00:00-05 | 1990-01-31 19:00:00-05 | 1989-12-31 19:00:00-05 | 1989-12-31 19:00:00-05 | 1990-01-31 19:00:00-05 | 1990-01-31 19:00:00-05 | 1990-01-31 19:00:00-05 1990-03-03 00:00:00-05 | 1990-02-28 19:00:00-05 | 1990-02-28 19:00:00-05 | 1989-12-31 19:00:00-05 | 1990-02-28 19:00:00-05 | 1990-01-31 19:00:00-05 | 1990-01-31 19:00:00-05 1990-04-03 00:00:00-04 | 1990-03-31 19:00:00-05 | 1990-02-28 19:00:00-05 | 1990-03-31 19:00:00-05 | 1990-03-31 19:00:00-05 | 1990-03-31 19:00:00-05 | 1990-01-31 19:00:00-05 1990-05-03 00:00:00-04 | 1990-04-30 20:00:00-04 | 1990-04-30 20:00:00-04 | 1990-03-31 19:00:00-05 | 1990-04-30 20:00:00-04 | 1990-03-31 19:00:00-05 | 1990-04-30 20:00:00-04 1990-06-03 00:00:00-04 | 1990-05-31 20:00:00-04 | 1990-04-30 20:00:00-04 | 1990-03-31 19:00:00-05 | 1990-05-31 20:00:00-04 | 1990-05-31 20:00:00-04 | 1990-04-30 20:00:00-04 --------------------------------------- --- Test time_bucket with timezones --- --------------------------------------- -- test NULL args SELECT time_bucket(NULL::interval,now(),'Europe/Berlin'), time_bucket('1day',NULL::timestamptz,'Europe/Berlin'), time_bucket('1day',now(),NULL::text), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin',NULL), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin','2020-04-01',NULL), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin',NULL,NULL), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin',"offset":=NULL::interval), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin',origin:=NULL::timestamptz); time_bucket | time_bucket | time_bucket | time_bucket | time_bucket | time_bucket | time_bucket | time_bucket -------------+-------------+-------------+------------------------+------------------------+------------------------+------------------------+------------------------ | | | 2020-02-02 18:00:00-05 | 2020-02-03 00:00:00-05 | 2020-02-02 18:00:00-05 | 2020-02-02 18:00:00-05 | 2020-02-02 18:00:00-05 SET datestyle TO ISO; SELECT time_bucket('1day', ts) AS "UTC", time_bucket('1day', ts, 'Europe/Berlin') AS "Berlin", time_bucket('1day', ts, 'Europe/London') AS "London", time_bucket('1day', ts, 'America/New_York') AS "New York", time_bucket('1day', ts, 'PST') AS "PST", time_bucket('1day', ts, current_setting('timezone')) AS "current" FROM generate_series('1999-12-31 17:00'::timestamptz,'2000-01-02 3:00'::timestamptz, '1hour'::interval) ts; UTC | Berlin | London | New York | PST | current ------------------------+------------------------+------------------------+------------------------+------------------------+------------------------ 1999-12-30 19:00:00-05 | 1999-12-30 18:00:00-05 | 1999-12-30 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-30 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-30 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-31 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-31 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-31 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 2000-01-01 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-02 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-02 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-02 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-02 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-02 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-02 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-02 00:00:00-05 | 2000-01-02 03:00:00-05 | 2000-01-02 00:00:00-05 SELECT time_bucket('1month', ts) AS "UTC", time_bucket('1month', ts, 'Europe/Berlin') AS "Berlin", time_bucket('1month', ts, 'America/New_York') AS "New York", time_bucket('1month', ts, current_setting('timezone')) AS "current", time_bucket('2month', ts, current_setting('timezone')) AS "2m", time_bucket('2month', ts, current_setting('timezone'), '2000-02-01'::timestamp) AS "2m origin", time_bucket('2month', ts, current_setting('timezone'), "offset":='14 day'::interval) AS "2m offset", time_bucket('2month', ts, current_setting('timezone'), '2000-02-01'::timestamp, '7 day'::interval) AS "2m offset + origin" FROM generate_series('1999-12-01'::timestamptz,'2000-09-01'::timestamptz, '9 day'::interval) ts; UTC | Berlin | New York | current | 2m | 2m origin | 2m offset | 2m offset + origin ------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+------------------------ 1999-11-30 19:00:00-05 | 1999-11-30 18:00:00-05 | 1999-12-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-15 00:00:00-05 | 1999-10-08 00:00:00-04 1999-11-30 19:00:00-05 | 1999-11-30 18:00:00-05 | 1999-12-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-15 00:00:00-05 | 1999-12-08 00:00:00-05 1999-11-30 19:00:00-05 | 1999-11-30 18:00:00-05 | 1999-12-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-15 00:00:00-05 | 1999-12-08 00:00:00-05 1999-11-30 19:00:00-05 | 1999-11-30 18:00:00-05 | 1999-12-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-15 00:00:00-05 | 1999-12-08 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-15 00:00:00-05 | 1999-12-08 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 1999-12-08 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 1999-12-08 00:00:00-05 2000-01-31 19:00:00-05 | 2000-01-31 18:00:00-05 | 2000-02-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 1999-12-08 00:00:00-05 2000-01-31 19:00:00-05 | 2000-01-31 18:00:00-05 | 2000-02-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-01-31 19:00:00-05 | 2000-01-31 18:00:00-05 | 2000-02-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-01-31 19:00:00-05 | 2000-01-31 18:00:00-05 | 2000-02-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-02-29 19:00:00-05 | 2000-02-29 18:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-02-29 19:00:00-05 | 2000-02-29 18:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-02-29 19:00:00-05 | 2000-02-29 18:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-03-31 19:00:00-05 | 2000-03-31 17:00:00-05 | 2000-04-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-03-31 19:00:00-05 | 2000-03-31 17:00:00-05 | 2000-04-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-04-08 00:00:00-04 2000-03-31 19:00:00-05 | 2000-03-31 17:00:00-05 | 2000-04-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-04-08 00:00:00-04 2000-04-30 20:00:00-04 | 2000-04-30 18:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-04-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-04-08 00:00:00-04 2000-04-30 20:00:00-04 | 2000-04-30 18:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-04-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-04-08 00:00:00-04 2000-04-30 20:00:00-04 | 2000-04-30 18:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-04-01 00:00:00-05 | 2000-05-15 00:00:00-04 | 2000-04-08 00:00:00-04 2000-04-30 20:00:00-04 | 2000-04-30 18:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-04-01 00:00:00-05 | 2000-05-15 00:00:00-04 | 2000-04-08 00:00:00-04 2000-05-31 20:00:00-04 | 2000-05-31 18:00:00-04 | 2000-06-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-15 00:00:00-04 | 2000-04-08 00:00:00-04 2000-05-31 20:00:00-04 | 2000-05-31 18:00:00-04 | 2000-06-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-05-31 20:00:00-04 | 2000-05-31 18:00:00-04 | 2000-06-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-06-30 20:00:00-04 | 2000-06-30 18:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-06-30 20:00:00-04 | 2000-06-30 18:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-06-30 20:00:00-04 | 2000-06-30 18:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-07-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-06-30 20:00:00-04 | 2000-06-30 18:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-07-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-07-31 20:00:00-04 | 2000-07-31 18:00:00-04 | 2000-08-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-15 00:00:00-04 | 2000-08-08 00:00:00-04 2000-07-31 20:00:00-04 | 2000-07-31 18:00:00-04 | 2000-08-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-15 00:00:00-04 | 2000-08-08 00:00:00-04 2000-07-31 20:00:00-04 | 2000-07-31 18:00:00-04 | 2000-08-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-15 00:00:00-04 | 2000-08-08 00:00:00-04 RESET datestyle; ------------------------------------- --- Test time input functions -- ------------------------------------- \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION test.interval_to_internal(coltype REGTYPE, value ANYELEMENT = NULL::BIGINT) RETURNS BIGINT AS :MODULE_PATHNAME, 'ts_dimension_interval_to_internal_test' LANGUAGE C VOLATILE; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT test.interval_to_internal('TIMESTAMP'::regtype, INTERVAL '1 day'); interval_to_internal ---------------------- 86400000000 SELECT test.interval_to_internal('TIMESTAMP'::regtype, 86400000000); interval_to_internal ---------------------- 86400000000 ---should give warning SELECT test.interval_to_internal('TIMESTAMP'::regtype, 86400); WARNING: unexpected interval: smaller than one second HINT: The interval is specified in microseconds. interval_to_internal ---------------------- 86400 SELECT test.interval_to_internal('TIMESTAMP'::regtype); interval_to_internal ---------------------- 604800000000 SELECT test.interval_to_internal('BIGINT'::regtype, 2147483649::bigint); interval_to_internal ---------------------- 2147483649 -- Default interval for integer is supported as part of -- hypertable generalization SELECT test.interval_to_internal('INT'::regtype); interval_to_internal ---------------------- 100000 SELECT test.interval_to_internal('SMALLINT'::regtype); interval_to_internal ---------------------- 10000 SELECT test.interval_to_internal('BIGINT'::regtype); interval_to_internal ---------------------- 1000000 SELECT test.interval_to_internal('TIMESTAMPTZ'::regtype); interval_to_internal ---------------------- 604800000000 SELECT test.interval_to_internal('TIMESTAMP'::regtype); interval_to_internal ---------------------- 604800000000 SELECT test.interval_to_internal('DATE'::regtype); interval_to_internal ---------------------- 604800000000 \set VERBOSITY terse \set ON_ERROR_STOP 0 SELECT test.interval_to_internal('INT'::regtype, 2147483649::bigint); ERROR: invalid interval: must be between 1 and 2147483647 SELECT test.interval_to_internal('SMALLINT'::regtype, 32768::bigint); ERROR: invalid interval: must be between 1 and 32767 SELECT test.interval_to_internal('TEXT'::regtype, 32768::bigint); ERROR: invalid type for dimension "testcol" SELECT test.interval_to_internal('INT'::regtype, INTERVAL '1 day'); ERROR: invalid interval type for integer dimension \set ON_ERROR_STOP 1 ================================================ FILE: test/expected/timestamp-17.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Utility function for grouping/slotting time with a given interval. CREATE OR REPLACE FUNCTION date_group( field timestamp, group_interval interval ) RETURNS timestamp LANGUAGE SQL STABLE AS $BODY$ SELECT to_timestamp((EXTRACT(EPOCH from $1)::int / EXTRACT(EPOCH from group_interval)::int) * EXTRACT(EPOCH from group_interval)::int)::timestamp; $BODY$; CREATE TABLE PUBLIC."testNs" ( "timeCustom" TIMESTAMP NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."testNs" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA "testNs" AUTHORIZATION :ROLE_DEFAULT_PERM_USER; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT * FROM create_hypertable('"public"."testNs"', 'timeCustom', 'device_id', 2, associated_schema_name=>'testNs' ); WARNING: column type "timestamp without time zone" used for "timeCustom" does not follow best practices hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | testNs | t \c :TEST_DBNAME INSERT INTO PUBLIC."testNs"("timeCustom", device_id, series_0, series_1) VALUES ('2009-11-12T01:00:00+00:00', 'dev1', 1.5, 1), ('2009-11-12T01:00:00+00:00', 'dev1', 1.5, 2), ('2009-11-10T23:00:02+00:00', 'dev1', 2.5, 3); INSERT INTO PUBLIC."testNs"("timeCustom", device_id, series_0, series_1) VALUES ('2009-11-10T23:00:00+00:00', 'dev2', 1.5, 1), ('2009-11-10T23:00:00+00:00', 'dev2', 1.5, 2); SELECT * FROM PUBLIC."testNs"; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool --------------------------+-----------+----------+----------+----------+------------- Thu Nov 12 01:00:00 2009 | dev1 | 1.5 | 1 | | Thu Nov 12 01:00:00 2009 | dev1 | 1.5 | 2 | | Tue Nov 10 23:00:02 2009 | dev1 | 2.5 | 3 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 1 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 2 | | SET client_min_messages = WARNING; \echo 'The next 2 queries will differ in output between UTC and EST since the mod is on the 100th hour UTC' The next 2 queries will differ in output between UTC and EST since the mod is on the 100th hour UTC SET timezone = 'UTC'; SELECT date_group("timeCustom", '100 days') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC; time | sum --------------------------+----- Sun Sep 13 00:00:00 2009 | 8.5 SET timezone = 'EST'; SELECT date_group("timeCustom", '100 days') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC; time | sum --------------------------+----- Sat Sep 12 19:00:00 2009 | 8.5 \echo 'The rest of the queries will be the same in output between UTC and EST' The rest of the queries will be the same in output between UTC and EST SET timezone = 'UTC'; SELECT date_group("timeCustom", '1 day') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC; time | sum --------------------------+----- Tue Nov 10 00:00:00 2009 | 5.5 Thu Nov 12 00:00:00 2009 | 3 SET timezone = 'EST'; SELECT date_group("timeCustom", '1 day') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC; time | sum --------------------------+----- Mon Nov 09 19:00:00 2009 | 5.5 Wed Nov 11 19:00:00 2009 | 3 SET timezone = 'UTC'; SELECT * FROM PUBLIC."testNs" WHERE "timeCustom" >= TIMESTAMP '2009-11-10T23:00:00' AND "timeCustom" < TIMESTAMP '2009-11-12T01:00:00' ORDER BY "timeCustom" DESC, device_id, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool --------------------------+-----------+----------+----------+----------+------------- Tue Nov 10 23:00:02 2009 | dev1 | 2.5 | 3 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 1 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 2 | | SET timezone = 'EST'; SELECT * FROM PUBLIC."testNs" WHERE "timeCustom" >= TIMESTAMP '2009-11-10T23:00:00' AND "timeCustom" < TIMESTAMP '2009-11-12T01:00:00' ORDER BY "timeCustom" DESC, device_id, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool --------------------------+-----------+----------+----------+----------+------------- Tue Nov 10 23:00:02 2009 | dev1 | 2.5 | 3 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 1 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 2 | | SET timezone = 'UTC'; SELECT date_group("timeCustom", '1 day') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC LIMIT 2; time | sum --------------------------+----- Tue Nov 10 00:00:00 2009 | 5.5 Thu Nov 12 00:00:00 2009 | 3 SET timezone = 'EST'; SELECT date_group("timeCustom", '1 day') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC LIMIT 2; time | sum --------------------------+----- Mon Nov 09 19:00:00 2009 | 5.5 Wed Nov 11 19:00:00 2009 | 3 ------------------------------------ -- Test time conversion functions -- ------------------------------------ \set ON_ERROR_STOP 0 SET timezone = 'UTC'; -- Conversion to timestamp using Postgres built-in function taking -- double. Gives inaccurate result on Postgres <= 9.6.2. Accurate on -- Postgres >= 9.6.3. SELECT to_timestamp(1486480176.236538); to_timestamp ------------------------------------- Tue Feb 07 15:09:36.236538 2017 UTC -- extension-specific version taking microsecond UNIX timestamp SELECT _timescaledb_functions.to_timestamp(1486480176236538); to_timestamp ------------------------------------- Tue Feb 07 15:09:36.236538 2017 UTC -- Should be the inverse of the statement above. SELECT _timescaledb_functions.to_unix_microseconds('2017-02-07 15:09:36.236538+00'); to_unix_microseconds ---------------------- 1486480176236538 -- For timestamps, BIGINT MAX represents +Infinity and BIGINT MIN -- -Infinity. We keep this notion for UNIX epoch time: SELECT _timescaledb_functions.to_unix_microseconds('+infinity'); to_unix_microseconds ---------------------- 9223372036854775807 SELECT _timescaledb_functions.to_timestamp(9223372036854775807); to_timestamp -------------- infinity SELECT _timescaledb_functions.to_unix_microseconds('-infinity'); to_unix_microseconds ---------------------- -9223372036854775808 SELECT _timescaledb_functions.to_timestamp(-9223372036854775808); to_timestamp -------------- -infinity -- In UNIX microseconds, the largest bigint value below infinity -- (BIGINT MAX) is smaller than internal date upper bound and should -- therefore be OK. Further, converting to the internal postgres epoch -- cannot overflow a 64-bit INTEGER since the postgres epoch is at a -- later date compared to the UNIX epoch, and is therefore represented -- by a smaller number SELECT _timescaledb_functions.to_timestamp(9223372036854775806); to_timestamp --------------------------------------- Sun Jan 10 04:00:54.775806 294247 UTC -- Julian day zero is -210866803200000000 microseconds from UNIX epoch SELECT _timescaledb_functions.to_timestamp(-210866803200000000); to_timestamp --------------------------------- Mon Nov 24 00:00:00 4714 UTC BC \set VERBOSITY default -- Going beyond Julian day zero should give out-of-range error SELECT _timescaledb_functions.to_timestamp(-210866803200000001); ERROR: timestamp out of range -- Lower bound on date (should return the Julian day zero UNIX timestamp above) SELECT _timescaledb_functions.to_unix_microseconds('4714-11-24 00:00:00+00 BC'); to_unix_microseconds ---------------------- -210866803200000000 -- Going beyond lower bound on date should return out-of-range SELECT _timescaledb_functions.to_unix_microseconds('4714-11-23 23:59:59.999999+00 BC'); ERROR: timestamp out of range: "4714-11-23 23:59:59.999999+00 BC" LINE 1: ...ELECT _timescaledb_functions.to_unix_microseconds('4714-11-2... ^ -- The upper bound for Postgres TIMESTAMPTZ SELECT timestamp '294276-12-31 23:59:59.999999+00'; timestamp ----------------------------------- Sun Dec 31 23:59:59.999999 294276 -- Going beyond the upper bound, should fail SELECT timestamp '294276-12-31 23:59:59.999999+00' + interval '1 us'; ERROR: timestamp out of range -- Cannot represent the upper bound timestamp with a UNIX microsecond timestamp -- since the Postgres epoch is at a later date than the UNIX epoch. SELECT _timescaledb_functions.to_unix_microseconds('294276-12-31 23:59:59.999999+00'); ERROR: timestamp out of range -- Subtracting the difference between the two epochs (10957 days) should bring -- us within range. SELECT timestamp '294276-12-31 23:59:59.999999+00' - interval '10957 days'; ?column? ----------------------------------- Fri Jan 01 23:59:59.999999 294247 SELECT _timescaledb_functions.to_unix_microseconds('294247-01-01 23:59:59.999999'); to_unix_microseconds ---------------------- 9223371331199999999 -- Adding one microsecond should take us out-of-range again SELECT timestamp '294247-01-01 23:59:59.999999' + interval '1 us'; ?column? ---------------------------- Sat Jan 02 00:00:00 294247 SELECT _timescaledb_functions.to_unix_microseconds(timestamp '294247-01-01 23:59:59.999999' + interval '1 us'); ERROR: timestamp out of range --no time_bucketing of dates not by integer # of days SELECT time_bucket('1 hour', DATE '2012-01-01'); ERROR: interval must not have sub-day precision SELECT time_bucket('25 hour', DATE '2012-01-01'); ERROR: interval must be a multiple of a day \set ON_ERROR_STOP 1 SELECT time_bucket(INTERVAL '1 day', TIMESTAMP '2011-01-02 01:01:01'); time_bucket -------------------------- Sun Jan 02 00:00:00 2011 SELECT time, time_bucket(INTERVAL '2 day ', time) FROM unnest(ARRAY[ TIMESTAMP '2011-01-01 01:01:01', TIMESTAMP '2011-01-02 01:01:01', TIMESTAMP '2011-01-03 01:01:01', TIMESTAMP '2011-01-04 01:01:01' ]) AS time; time | time_bucket --------------------------+-------------------------- Sat Jan 01 01:01:01 2011 | Sat Jan 01 00:00:00 2011 Sun Jan 02 01:01:01 2011 | Sat Jan 01 00:00:00 2011 Mon Jan 03 01:01:01 2011 | Mon Jan 03 00:00:00 2011 Tue Jan 04 01:01:01 2011 | Mon Jan 03 00:00:00 2011 SELECT int_def, time_bucket(int_def,TIMESTAMP '2011-01-02 01:01:01.111') FROM unnest(ARRAY[ INTERVAL '1 millisecond', INTERVAL '1 second', INTERVAL '1 minute', INTERVAL '1 hour', INTERVAL '1 day', INTERVAL '2 millisecond', INTERVAL '2 second', INTERVAL '2 minute', INTERVAL '2 hour', INTERVAL '2 day' ]) AS int_def; int_def | time_bucket --------------+------------------------------ @ 0.001 secs | Sun Jan 02 01:01:01.111 2011 @ 1 sec | Sun Jan 02 01:01:01 2011 @ 1 min | Sun Jan 02 01:01:00 2011 @ 1 hour | Sun Jan 02 01:00:00 2011 @ 1 day | Sun Jan 02 00:00:00 2011 @ 0.002 secs | Sun Jan 02 01:01:01.11 2011 @ 2 secs | Sun Jan 02 01:01:00 2011 @ 2 mins | Sun Jan 02 01:00:00 2011 @ 2 hours | Sun Jan 02 00:00:00 2011 @ 2 days | Sat Jan 01 00:00:00 2011 \set ON_ERROR_STOP 0 SELECT time_bucket(INTERVAL '1 year 1d',TIMESTAMP '2011-01-02 01:01:01.111'); ERROR: month intervals cannot have day or time component SELECT time_bucket(INTERVAL '1 month 1 minute',TIMESTAMP '2011-01-02 01:01:01.111'); ERROR: month intervals cannot have day or time component \set ON_ERROR_STOP 1 SELECT time, time_bucket(INTERVAL '5 minute', time) FROM unnest(ARRAY[ TIMESTAMP '1970-01-01 00:59:59.999999', TIMESTAMP '1970-01-01 01:01:00', TIMESTAMP '1970-01-01 01:04:59.999999', TIMESTAMP '1970-01-01 01:05:00' ]) AS time; time | time_bucket ---------------------------------+-------------------------- Thu Jan 01 00:59:59.999999 1970 | Thu Jan 01 00:55:00 1970 Thu Jan 01 01:01:00 1970 | Thu Jan 01 01:00:00 1970 Thu Jan 01 01:04:59.999999 1970 | Thu Jan 01 01:00:00 1970 Thu Jan 01 01:05:00 1970 | Thu Jan 01 01:05:00 1970 SELECT time, time_bucket(INTERVAL '5 minute', time) FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:04:59.999999', TIMESTAMP '2011-01-02 01:05:00', TIMESTAMP '2011-01-02 01:09:59.999999', TIMESTAMP '2011-01-02 01:10:00' ]) AS time; time | time_bucket ---------------------------------+-------------------------- Sun Jan 02 01:04:59.999999 2011 | Sun Jan 02 01:00:00 2011 Sun Jan 02 01:05:00 2011 | Sun Jan 02 01:05:00 2011 Sun Jan 02 01:09:59.999999 2011 | Sun Jan 02 01:05:00 2011 Sun Jan 02 01:10:00 2011 | Sun Jan 02 01:10:00 2011 --offset with interval SELECT time, time_bucket(INTERVAL '5 minute', time , INTERVAL '2 minutes') FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:01:59.999999', TIMESTAMP '2011-01-02 01:02:00', TIMESTAMP '2011-01-02 01:06:59.999999', TIMESTAMP '2011-01-02 01:07:00' ]) AS time; time | time_bucket ---------------------------------+-------------------------- Sun Jan 02 01:01:59.999999 2011 | Sun Jan 02 00:57:00 2011 Sun Jan 02 01:02:00 2011 | Sun Jan 02 01:02:00 2011 Sun Jan 02 01:06:59.999999 2011 | Sun Jan 02 01:02:00 2011 Sun Jan 02 01:07:00 2011 | Sun Jan 02 01:07:00 2011 SELECT time, time_bucket(INTERVAL '5 minute', time , - INTERVAL '2 minutes') FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:02:59.999999', TIMESTAMP '2011-01-02 01:03:00', TIMESTAMP '2011-01-02 01:07:59.999999', TIMESTAMP '2011-01-02 01:08:00' ]) AS time; time | time_bucket ---------------------------------+-------------------------- Sun Jan 02 01:02:59.999999 2011 | Sun Jan 02 00:58:00 2011 Sun Jan 02 01:03:00 2011 | Sun Jan 02 01:03:00 2011 Sun Jan 02 01:07:59.999999 2011 | Sun Jan 02 01:03:00 2011 Sun Jan 02 01:08:00 2011 | Sun Jan 02 01:08:00 2011 --offset with infinity -- timestamp SELECT time, time_bucket(INTERVAL '1 week', time, INTERVAL '1 day') FROM unnest(ARRAY[ timestamp '-Infinity', timestamp 'Infinity' ]) AS time; time | time_bucket -----------+------------- -infinity | -infinity infinity | infinity -- timestamptz SELECT time, time_bucket(INTERVAL '1 week', time, INTERVAL '1 day') FROM unnest(ARRAY[ timestamp with time zone '-Infinity', timestamp with time zone 'Infinity' ]) AS time; time | time_bucket -----------+------------- -infinity | -infinity infinity | infinity -- Date SELECT date, time_bucket(INTERVAL '1 week', date, INTERVAL '1 day') FROM unnest(ARRAY[ date '-Infinity', date 'Infinity' ]) AS date; date | time_bucket -----------+------------- -infinity | -infinity infinity | infinity --example to align with an origin SELECT time, time_bucket(INTERVAL '5 minute', time - (TIMESTAMP '2011-01-02 00:02:00' - TIMESTAMP 'epoch')) + (TIMESTAMP '2011-01-02 00:02:00'-TIMESTAMP 'epoch') FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:01:59.999999', TIMESTAMP '2011-01-02 01:02:00', TIMESTAMP '2011-01-02 01:06:59.999999', TIMESTAMP '2011-01-02 01:07:00' ]) AS time; time | ?column? ---------------------------------+-------------------------- Sun Jan 02 01:01:59.999999 2011 | Sun Jan 02 00:57:00 2011 Sun Jan 02 01:02:00 2011 | Sun Jan 02 01:02:00 2011 Sun Jan 02 01:06:59.999999 2011 | Sun Jan 02 01:02:00 2011 Sun Jan 02 01:07:00 2011 | Sun Jan 02 01:07:00 2011 --rounding version SELECT time, time_bucket(INTERVAL '5 minute', time , - INTERVAL '2.5 minutes') + INTERVAL '2 minutes 30 seconds' FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:05:01', TIMESTAMP '2011-01-02 01:07:29', TIMESTAMP '2011-01-02 01:02:30', TIMESTAMP '2011-01-02 01:07:30', TIMESTAMP '2011-01-02 01:02:29' ]) AS time; time | ?column? --------------------------+-------------------------- Sun Jan 02 01:05:01 2011 | Sun Jan 02 01:05:00 2011 Sun Jan 02 01:07:29 2011 | Sun Jan 02 01:05:00 2011 Sun Jan 02 01:02:30 2011 | Sun Jan 02 01:05:00 2011 Sun Jan 02 01:07:30 2011 | Sun Jan 02 01:10:00 2011 Sun Jan 02 01:02:29 2011 | Sun Jan 02 01:00:00 2011 --time_bucket with timezone should mimick date_trunc SET timezone TO 'UTC'; SELECT time, time_bucket(INTERVAL '1 hour', time), date_trunc('hour', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Jan 02 01:01:01 2011 UTC | Sun Jan 02 01:00:00 2011 UTC | Sun Jan 02 01:00:00 2011 UTC Sun Jan 02 00:01:01 2011 UTC | Sun Jan 02 00:00:00 2011 UTC | Sun Jan 02 00:00:00 2011 UTC Sat Jan 01 23:01:01 2011 UTC | Sat Jan 01 23:00:00 2011 UTC | Sat Jan 01 23:00:00 2011 UTC SELECT time, time_bucket(INTERVAL '1 day', time), date_trunc('day', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Jan 02 01:01:01 2011 UTC | Sun Jan 02 00:00:00 2011 UTC | Sun Jan 02 00:00:00 2011 UTC Sun Jan 02 00:01:01 2011 UTC | Sun Jan 02 00:00:00 2011 UTC | Sun Jan 02 00:00:00 2011 UTC Sat Jan 01 23:01:01 2011 UTC | Sat Jan 01 00:00:00 2011 UTC | Sat Jan 01 00:00:00 2011 UTC --what happens with a local tz SET timezone TO 'America/New_York'; SELECT time, time_bucket(INTERVAL '1 hour', time), date_trunc('hour', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Jan 02 01:01:01 2011 EST | Sun Jan 02 01:00:00 2011 EST | Sun Jan 02 01:00:00 2011 EST Sat Jan 01 19:01:01 2011 EST | Sat Jan 01 19:00:00 2011 EST | Sat Jan 01 19:00:00 2011 EST Sat Jan 01 18:01:01 2011 EST | Sat Jan 01 18:00:00 2011 EST | Sat Jan 01 18:00:00 2011 EST --Note the timestamp tz input is aligned with UTC day /not/ local day. different than date_trunc. SELECT time, time_bucket(INTERVAL '1 day', time), date_trunc('day', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-03 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-04 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Jan 02 01:01:01 2011 EST | Sat Jan 01 19:00:00 2011 EST | Sun Jan 02 00:00:00 2011 EST Sun Jan 02 19:01:01 2011 EST | Sun Jan 02 19:00:00 2011 EST | Sun Jan 02 00:00:00 2011 EST Mon Jan 03 18:01:01 2011 EST | Sun Jan 02 19:00:00 2011 EST | Mon Jan 03 00:00:00 2011 EST --can force local bucketing with simple cast. SELECT time, time_bucket(INTERVAL '1 day', time::timestamp), date_trunc('day', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-03 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-04 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+--------------------------+------------------------------ Sun Jan 02 01:01:01 2011 EST | Sun Jan 02 00:00:00 2011 | Sun Jan 02 00:00:00 2011 EST Sun Jan 02 19:01:01 2011 EST | Sun Jan 02 00:00:00 2011 | Sun Jan 02 00:00:00 2011 EST Mon Jan 03 18:01:01 2011 EST | Mon Jan 03 00:00:00 2011 | Mon Jan 03 00:00:00 2011 EST --can also use interval to correct SELECT time, time_bucket(INTERVAL '1 day', time, -INTERVAL '19 hours'), date_trunc('day', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-03 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-04 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Jan 02 01:01:01 2011 EST | Sun Jan 02 00:00:00 2011 EST | Sun Jan 02 00:00:00 2011 EST Sun Jan 02 19:01:01 2011 EST | Sun Jan 02 00:00:00 2011 EST | Sun Jan 02 00:00:00 2011 EST Mon Jan 03 18:01:01 2011 EST | Mon Jan 03 00:00:00 2011 EST | Mon Jan 03 00:00:00 2011 EST --dst: same local hour bucketed as two different hours. SELECT time, time_bucket(INTERVAL '1 hour', time), date_trunc('hour', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2017-11-05 12:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 13:05:00+07' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Nov 05 01:05:00 2017 EDT | Sun Nov 05 01:00:00 2017 EDT | Sun Nov 05 01:00:00 2017 EDT Sun Nov 05 01:05:00 2017 EST | Sun Nov 05 01:00:00 2017 EST | Sun Nov 05 01:00:00 2017 EST --local alignment changes when bucketing by UTC across dst boundary SELECT time, time_bucket(INTERVAL '2 hour', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2017-11-05 10:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 12:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 13:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 15:05:00+07' ]) AS time; time | time_bucket ------------------------------+------------------------------ Sat Nov 04 23:05:00 2017 EDT | Sat Nov 04 22:00:00 2017 EDT Sun Nov 05 01:05:00 2017 EDT | Sun Nov 05 00:00:00 2017 EDT Sun Nov 05 01:05:00 2017 EST | Sun Nov 05 01:00:00 2017 EST Sun Nov 05 03:05:00 2017 EST | Sun Nov 05 03:00:00 2017 EST --local alignment is preserved when bucketing by local time across DST boundary. SELECT time, time_bucket(INTERVAL '2 hour', time::timestamp) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2017-11-05 10:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 12:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 13:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 15:05:00+07' ]) AS time; time | time_bucket ------------------------------+-------------------------- Sat Nov 04 23:05:00 2017 EDT | Sat Nov 04 22:00:00 2017 Sun Nov 05 01:05:00 2017 EDT | Sun Nov 05 00:00:00 2017 Sun Nov 05 01:05:00 2017 EST | Sun Nov 05 00:00:00 2017 Sun Nov 05 03:05:00 2017 EST | Sun Nov 05 02:00:00 2017 -- GitHub issue #7059: time_bucket with timezone + offset across DST boundary -- Asia/Amman: clocks skip from 00:00 to 01:00 on 2021-03-26 -- Input: 01:00+03 = 22:00 UTC → Result: 22:15 UTC = 01:15 local (00:00 + 15min offset) SELECT time_bucket('1 day', '2021-03-26 01:00:00+03'::timestamptz, timezone := 'Asia/Amman', "offset" := '15 minutes'::interval); time_bucket ------------------------------ Wed Mar 24 18:15:00 2021 EDT -- GitHub issue #8851: time_bucket with negative offset during DST fall-back -- Europe/Berlin: clocks repeat 02:00-02:59 on 2025-10-26 -- Input: 02:00+02 = 00:00 UTC → Result: 23:59:45 UTC = 01:59:45 local (02:00 - 15s offset) SELECT time_bucket('30 seconds', '2025-10-26 02:00:00+02'::timestamptz, timezone := 'Europe/Berlin', "offset" := '-15 seconds'::interval); time_bucket ------------------------------ Sat Oct 25 19:59:45 2025 EDT -- Additional DST edge cases for coverage of DST direction × offset sign combinations -- Spring-forward + negative offset -- Input: 01:30+03 = 22:30 UTC → Result: 22:45 UTC = 01:45 local (01:00 + 45min = 02:00 - 15min) SELECT time_bucket('1 hour', '2021-03-26 01:30:00+03'::timestamptz, timezone := 'Asia/Amman', "offset" := '-15 minutes'::interval); time_bucket ------------------------------ Thu Mar 25 17:45:00 2021 EDT -- Fall-back + positive offset -- Input: 02:30+01 = 01:30 UTC → Result: 01:15 UTC = 02:15 local (02:00 + 15min offset) SELECT time_bucket('1 hour', '2025-10-26 02:30:00+01'::timestamptz, timezone := 'Europe/Berlin', "offset" := '15 minutes'::interval); time_bucket ------------------------------ Sat Oct 25 21:15:00 2025 EDT -- Input exactly at DST spring-forward transition -- Input: 22:00 UTC = 00:00 local (the moment clocks jump to 01:00) -- Result: 22:15 UTC = 01:15 local (01:00 + 15min offset) SELECT time_bucket('1 hour', '2021-03-25 22:00:00+00'::timestamptz, timezone := 'Asia/Amman', "offset" := '15 minutes'::interval); time_bucket ------------------------------ Thu Mar 25 17:15:00 2021 EDT -- Input exactly at DST fall-back transition -- Input: 01:00 UTC = 03:00 CEST (the moment clocks go back to 02:00 CET) -- Result: 23:15 UTC = 01:15 local (01:00 + 15min offset, but in CET now) SELECT time_bucket('1 hour', '2025-10-26 01:00:00+00'::timestamptz, timezone := 'Europe/Berlin', "offset" := '15 minutes'::interval); time_bucket ------------------------------ Sat Oct 25 20:15:00 2025 EDT -- Offset larger than bucket size (1h offset with 30min bucket) -- Input: 01:30+03 = 22:30 UTC → Result: 22:30 UTC = 01:30 local (01:00 + 30min = 00:30 + 1h) SELECT time_bucket('30 minutes', '2021-03-26 01:30:00+03'::timestamptz, timezone := 'Asia/Amman', "offset" := '1 hour'::interval); time_bucket ------------------------------ Thu Mar 25 18:30:00 2021 EDT -- GitHub issue #9136: time_bucket with origin during DST fall-back -- When origin is in standard time but timestamp is in daylight time, -- the bucket could incorrectly start AFTER the timestamp. -- America/New_York: clocks go back at 02:00 EDT on 2024-11-03 -- Input: 01:30-04 (EDT) = 05:30 UTC; origin in EST -- Result should have bucket start <= timestamp (bucket in EDT, not EST) SELECT time_bucket('1 hour', '2024-11-03 01:30:00-04'::timestamptz, 'America/New_York', '2000-01-01 00:00:00 America/New_York'::timestamptz) as bucket, '2024-11-03 01:30:00-04'::timestamptz < time_bucket('1 hour', '2024-11-03 01:30:00-04'::timestamptz, 'America/New_York', '2000-01-01 00:00:00 America/New_York'::timestamptz) as ts_before_bucket; bucket | ts_before_bucket ------------------------------+------------------ Sun Nov 03 01:00:00 2024 EDT | f SELECT time, time_bucket(10::smallint, time) AS time_bucket_smallint, time_bucket(10::int, time) AS time_bucket_int, time_bucket(10::bigint, time) AS time_bucket_bigint FROM unnest(ARRAY[ '-11', '-10', '-9', '-1', '0', '1', '99', '100', '109', '110' ]::smallint[]) AS time; time | time_bucket_smallint | time_bucket_int | time_bucket_bigint ------+----------------------+-----------------+-------------------- -11 | -20 | -20 | -20 -10 | -10 | -10 | -10 -9 | -10 | -10 | -10 -1 | -10 | -10 | -10 0 | 0 | 0 | 0 1 | 0 | 0 | 0 99 | 90 | 90 | 90 100 | 100 | 100 | 100 109 | 100 | 100 | 100 110 | 110 | 110 | 110 SELECT time, time_bucket(10::smallint, time, 2::smallint) AS time_bucket_smallint, time_bucket(10::int, time, 2::int) AS time_bucket_int, time_bucket(10::bigint, time, 2::bigint) AS time_bucket_bigint FROM unnest(ARRAY[ '-9', '-8', '-7', '1', '2', '3', '101', '102', '111', '112' ]::smallint[]) AS time; time | time_bucket_smallint | time_bucket_int | time_bucket_bigint ------+----------------------+-----------------+-------------------- -9 | -18 | -18 | -18 -8 | -8 | -8 | -8 -7 | -8 | -8 | -8 1 | -8 | -8 | -8 2 | 2 | 2 | 2 3 | 2 | 2 | 2 101 | 92 | 92 | 92 102 | 102 | 102 | 102 111 | 102 | 102 | 102 112 | 112 | 112 | 112 SELECT time, time_bucket(10::smallint, time, -2::smallint) AS time_bucket_smallint, time_bucket(10::int, time, -2::int) AS time_bucket_int, time_bucket(10::bigint, time, -2::bigint) AS time_bucket_bigint FROM unnest(ARRAY[ '-13', '-12', '-11', '-3', '-2', '-1', '97', '98', '107', '108' ]::smallint[]) AS time; time | time_bucket_smallint | time_bucket_int | time_bucket_bigint ------+----------------------+-----------------+-------------------- -13 | -22 | -22 | -22 -12 | -12 | -12 | -12 -11 | -12 | -12 | -12 -3 | -12 | -12 | -12 -2 | -2 | -2 | -2 -1 | -2 | -2 | -2 97 | 88 | 88 | 88 98 | 98 | 98 | 98 107 | 98 | 98 | 98 108 | 108 | 108 | 108 \set ON_ERROR_STOP 0 SELECT time_bucket(10::smallint, '-32768'::smallint); ERROR: timestamp out of range SELECT time_bucket(10::smallint, '-32761'::smallint); ERROR: timestamp out of range select time_bucket(10::smallint, '-32768'::smallint, 1000::smallint); ERROR: timestamp out of range select time_bucket(10::smallint, '-32768'::smallint, '32767'::smallint); ERROR: timestamp out of range select time_bucket(10::smallint, '32767'::smallint, '-32768'::smallint); ERROR: timestamp out of range \set ON_ERROR_STOP 1 SELECT time, time_bucket(10::smallint, time) FROM unnest(ARRAY[ '-32760', '-32759', '32767' ]::smallint[]) AS time; time | time_bucket --------+------------- -32760 | -32760 -32759 | -32760 32767 | 32760 \set ON_ERROR_STOP 0 SELECT time_bucket(10::int, '-2147483648'::int); ERROR: timestamp out of range SELECT time_bucket(10::int, '-2147483641'::int); ERROR: timestamp out of range SELECT time_bucket(1000::int, '-2147483000'::int, 1::int); ERROR: timestamp out of range SELECT time_bucket(1000::int, '-2147483648'::int, '2147483647'::int); ERROR: timestamp out of range SELECT time_bucket(1000::int, '2147483647'::int, '-2147483648'::int); ERROR: timestamp out of range \set ON_ERROR_STOP 1 SELECT time, time_bucket(10::int, time) FROM unnest(ARRAY[ '-2147483640', '-2147483639', '2147483647' ]::int[]) AS time; time | time_bucket -------------+------------- -2147483640 | -2147483640 -2147483639 | -2147483640 2147483647 | 2147483640 \set ON_ERROR_STOP 0 SELECT time_bucket(10::bigint, '-9223372036854775808'::bigint); ERROR: timestamp out of range SELECT time_bucket(10::bigint, '-9223372036854775801'::bigint); ERROR: timestamp out of range SELECT time_bucket(1000::bigint, '-9223372036854775000'::bigint, 1::bigint); ERROR: timestamp out of range SELECT time_bucket(1000::bigint, '-9223372036854775808'::bigint, '9223372036854775807'::bigint); ERROR: timestamp out of range SELECT time_bucket(1000::bigint, '9223372036854775807'::bigint, '-9223372036854775808'::bigint); ERROR: timestamp out of range \set ON_ERROR_STOP 1 SELECT time, time_bucket(10::bigint, time) FROM unnest(ARRAY[ '-9223372036854775800', '-9223372036854775799', '9223372036854775807' ]::bigint[]) AS time; time | time_bucket ----------------------+---------------------- -9223372036854775800 | -9223372036854775800 -9223372036854775799 | -9223372036854775800 9223372036854775807 | 9223372036854775800 SELECT time, time_bucket(INTERVAL '1 day', time::date) FROM unnest(ARRAY[ date '2017-11-05', date '2017-11-06' ]) AS time; time | time_bucket ------------+------------- 11-05-2017 | 11-05-2017 11-06-2017 | 11-06-2017 SELECT time, time_bucket(INTERVAL '4 day', time::date) FROM unnest(ARRAY[ date '2017-11-04', date '2017-11-05', date '2017-11-08', date '2017-11-09' ]) AS time; time | time_bucket ------------+------------- 11-04-2017 | 11-01-2017 11-05-2017 | 11-05-2017 11-08-2017 | 11-05-2017 11-09-2017 | 11-09-2017 SELECT time, time_bucket(INTERVAL '4 day', time::date, INTERVAL '2 day') FROM unnest(ARRAY[ date '2017-11-06', date '2017-11-07', date '2017-11-10', date '2017-11-11' ]) AS time; time | time_bucket ------------+------------- 11-06-2017 | 11-03-2017 11-07-2017 | 11-07-2017 11-10-2017 | 11-07-2017 11-11-2017 | 11-11-2017 -- 2019-09-24 is a Monday, and we want to ensure that time_bucket returns the week starting with a Monday as date_trunc does, -- Rather than a Saturday which is the date of the PostgreSQL epoch SELECT time, time_bucket(INTERVAL '1 week', time::date) FROM unnest(ARRAY[ date '2018-09-16', date '2018-09-17', date '2018-09-23', date '2018-09-24' ]) AS time; time | time_bucket ------------+------------- 09-16-2018 | 09-10-2018 09-17-2018 | 09-17-2018 09-23-2018 | 09-17-2018 09-24-2018 | 09-24-2018 SELECT time, time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp without time zone '2018-09-16', timestamp without time zone '2018-09-17', timestamp without time zone '2018-09-23', timestamp without time zone '2018-09-24' ]) AS time; time | time_bucket --------------------------+-------------------------- Sun Sep 16 00:00:00 2018 | Mon Sep 10 00:00:00 2018 Mon Sep 17 00:00:00 2018 | Mon Sep 17 00:00:00 2018 Sun Sep 23 00:00:00 2018 | Mon Sep 17 00:00:00 2018 Mon Sep 24 00:00:00 2018 | Mon Sep 24 00:00:00 2018 SELECT time, time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp with time zone '2018-09-16', timestamp with time zone '2018-09-17', timestamp with time zone '2018-09-23', timestamp with time zone '2018-09-24' ]) AS time; time | time_bucket ------------------------------+------------------------------ Sun Sep 16 00:00:00 2018 EDT | Sun Sep 09 20:00:00 2018 EDT Mon Sep 17 00:00:00 2018 EDT | Sun Sep 16 20:00:00 2018 EDT Sun Sep 23 00:00:00 2018 EDT | Sun Sep 16 20:00:00 2018 EDT Mon Sep 24 00:00:00 2018 EDT | Sun Sep 23 20:00:00 2018 EDT SELECT time, time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp with time zone '-Infinity', timestamp with time zone 'Infinity' ]) AS time; time | time_bucket -----------+------------- -infinity | -infinity infinity | infinity SELECT time, time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp without time zone '-Infinity', timestamp without time zone 'Infinity' ]) AS time; time | time_bucket -----------+------------- -infinity | -infinity infinity | infinity SELECT time, time_bucket(INTERVAL '1 week', time), date_trunc('week', time) = time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp without time zone '4714-11-24 01:01:01.0 BC', timestamp without time zone '294276-12-31 23:59:59.9999' ]) AS time; time | time_bucket | ?column? ---------------------------------+-----------------------------+---------- Mon Nov 24 01:01:01 4714 BC | Mon Nov 24 00:00:00 4714 BC | t Sun Dec 31 23:59:59.9999 294276 | Mon Dec 25 00:00:00 294276 | t --1000 years later weeks still align. SELECT time, time_bucket(INTERVAL '1 week', time), date_trunc('week', time) = time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp without time zone '3018-09-14', timestamp without time zone '3018-09-20', timestamp without time zone '3018-09-21', timestamp without time zone '3018-09-22' ]) AS time; time | time_bucket | ?column? --------------------------+--------------------------+---------- Mon Sep 14 00:00:00 3018 | Mon Sep 14 00:00:00 3018 | t Sun Sep 20 00:00:00 3018 | Mon Sep 14 00:00:00 3018 | t Mon Sep 21 00:00:00 3018 | Mon Sep 21 00:00:00 3018 | t Tue Sep 22 00:00:00 3018 | Mon Sep 21 00:00:00 3018 | t --weeks align for timestamptz as well if cast to local time, (but not if done at UTC). SELECT time, date_trunc('week', time) = time_bucket(INTERVAL '1 week', time), date_trunc('week', time) = time_bucket(INTERVAL '1 week', time::timestamp) FROM unnest(ARRAY[ timestamp with time zone '3018-09-14', timestamp with time zone '3018-09-20', timestamp with time zone '3018-09-21', timestamp with time zone '3018-09-22' ]) AS time; time | ?column? | ?column? ------------------------------+----------+---------- Mon Sep 14 00:00:00 3018 EDT | f | t Sun Sep 20 00:00:00 3018 EDT | f | t Mon Sep 21 00:00:00 3018 EDT | f | t Tue Sep 22 00:00:00 3018 EDT | f | t --check functions with origin --note that the default origin is at 0 UTC, using origin parameter it is easy to provide a EDT origin point \x SELECT time, time_bucket(INTERVAL '1 week', time) no_epoch, time_bucket(INTERVAL '1 week', time::timestamp) no_epoch_local, time_bucket(INTERVAL '1 week', time) = time_bucket(INTERVAL '1 week', time, timestamptz '2000-01-03 00:00:00+0') always_true, time_bucket(INTERVAL '1 week', time, timestamptz '2000-01-01 00:00:00+0') pg_epoch, time_bucket(INTERVAL '1 week', time, timestamptz 'epoch') unix_epoch, time_bucket(INTERVAL '1 week', time, timestamptz '3018-09-13') custom_1, time_bucket(INTERVAL '1 week', time, timestamptz '3018-09-14') custom_2 FROM unnest(ARRAY[ timestamp with time zone '2000-01-01 00:00:00+0'- interval '1 second', timestamp with time zone '2000-01-01 00:00:00+0', timestamp with time zone '2000-01-03 00:00:00+0'- interval '1 second', timestamp with time zone '2000-01-03 00:00:00+0', timestamp with time zone '2000-01-01', timestamp with time zone '2000-01-02', timestamp with time zone '2000-01-03', timestamp with time zone '3018-09-12', timestamp with time zone '3018-09-13', timestamp with time zone '3018-09-14', timestamp with time zone '3018-09-15' ]) AS time; -[ RECORD 1 ]--+----------------------------- time | Fri Dec 31 18:59:59 1999 EST no_epoch | Sun Dec 26 19:00:00 1999 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 24 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Dec 25 23:00:00 1999 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 2 ]--+----------------------------- time | Fri Dec 31 19:00:00 1999 EST no_epoch | Sun Dec 26 19:00:00 1999 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Dec 25 23:00:00 1999 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 3 ]--+----------------------------- time | Sun Jan 02 18:59:59 2000 EST no_epoch | Sun Dec 26 19:00:00 1999 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Jan 01 23:00:00 2000 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 4 ]--+----------------------------- time | Sun Jan 02 19:00:00 2000 EST no_epoch | Sun Jan 02 19:00:00 2000 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Jan 01 23:00:00 2000 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 5 ]--+----------------------------- time | Sat Jan 01 00:00:00 2000 EST no_epoch | Sun Dec 26 19:00:00 1999 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Dec 25 23:00:00 1999 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 6 ]--+----------------------------- time | Sun Jan 02 00:00:00 2000 EST no_epoch | Sun Dec 26 19:00:00 1999 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Jan 01 23:00:00 2000 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 7 ]--+----------------------------- time | Mon Jan 03 00:00:00 2000 EST no_epoch | Sun Jan 02 19:00:00 2000 EST no_epoch_local | Mon Jan 03 00:00:00 2000 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Jan 01 23:00:00 2000 EST custom_2 | Sun Jan 02 23:00:00 2000 EST -[ RECORD 8 ]--+----------------------------- time | Sat Sep 12 00:00:00 3018 EDT no_epoch | Sun Sep 06 20:00:00 3018 EDT no_epoch_local | Mon Sep 07 00:00:00 3018 always_true | t pg_epoch | Fri Sep 11 20:00:00 3018 EDT unix_epoch | Wed Sep 09 20:00:00 3018 EDT custom_1 | Sun Sep 06 00:00:00 3018 EDT custom_2 | Mon Sep 07 00:00:00 3018 EDT -[ RECORD 9 ]--+----------------------------- time | Sun Sep 13 00:00:00 3018 EDT no_epoch | Sun Sep 06 20:00:00 3018 EDT no_epoch_local | Mon Sep 07 00:00:00 3018 always_true | t pg_epoch | Fri Sep 11 20:00:00 3018 EDT unix_epoch | Wed Sep 09 20:00:00 3018 EDT custom_1 | Sun Sep 13 00:00:00 3018 EDT custom_2 | Mon Sep 07 00:00:00 3018 EDT -[ RECORD 10 ]-+----------------------------- time | Mon Sep 14 00:00:00 3018 EDT no_epoch | Sun Sep 13 20:00:00 3018 EDT no_epoch_local | Mon Sep 14 00:00:00 3018 always_true | t pg_epoch | Fri Sep 11 20:00:00 3018 EDT unix_epoch | Wed Sep 09 20:00:00 3018 EDT custom_1 | Sun Sep 13 00:00:00 3018 EDT custom_2 | Mon Sep 14 00:00:00 3018 EDT -[ RECORD 11 ]-+----------------------------- time | Tue Sep 15 00:00:00 3018 EDT no_epoch | Sun Sep 13 20:00:00 3018 EDT no_epoch_local | Mon Sep 14 00:00:00 3018 always_true | t pg_epoch | Fri Sep 11 20:00:00 3018 EDT unix_epoch | Wed Sep 09 20:00:00 3018 EDT custom_1 | Sun Sep 13 00:00:00 3018 EDT custom_2 | Mon Sep 14 00:00:00 3018 EDT SELECT time, time_bucket(INTERVAL '1 week', time) no_epoch, time_bucket(INTERVAL '1 week', time) = time_bucket(INTERVAL '1 week', time, timestamp '2000-01-03 00:00:00') always_true, time_bucket(INTERVAL '1 week', time, timestamp '2000-01-01 00:00:00+0') pg_epoch, time_bucket(INTERVAL '1 week', time, timestamp 'epoch') unix_epoch, time_bucket(INTERVAL '1 week', time, timestamp '3018-09-13') custom_1, time_bucket(INTERVAL '1 week', time, timestamp '3018-09-14') custom_2 FROM unnest(ARRAY[ timestamp without time zone '2000-01-01 00:00:00'- interval '1 second', timestamp without time zone '2000-01-01 00:00:00', timestamp without time zone '2000-01-03 00:00:00'- interval '1 second', timestamp without time zone '2000-01-03 00:00:00', timestamp without time zone '2000-01-01', timestamp without time zone '2000-01-02', timestamp without time zone '2000-01-03', timestamp without time zone '3018-09-12', timestamp without time zone '3018-09-13', timestamp without time zone '3018-09-14', timestamp without time zone '3018-09-15' ]) AS time; -[ RECORD 1 ]------------------------- time | Fri Dec 31 23:59:59 1999 no_epoch | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Sat Dec 25 00:00:00 1999 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Dec 26 00:00:00 1999 custom_2 | Mon Dec 27 00:00:00 1999 -[ RECORD 2 ]------------------------- time | Sat Jan 01 00:00:00 2000 no_epoch | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Dec 26 00:00:00 1999 custom_2 | Mon Dec 27 00:00:00 1999 -[ RECORD 3 ]------------------------- time | Sun Jan 02 23:59:59 2000 no_epoch | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Jan 02 00:00:00 2000 custom_2 | Mon Dec 27 00:00:00 1999 -[ RECORD 4 ]------------------------- time | Mon Jan 03 00:00:00 2000 no_epoch | Mon Jan 03 00:00:00 2000 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Jan 02 00:00:00 2000 custom_2 | Mon Jan 03 00:00:00 2000 -[ RECORD 5 ]------------------------- time | Sat Jan 01 00:00:00 2000 no_epoch | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Dec 26 00:00:00 1999 custom_2 | Mon Dec 27 00:00:00 1999 -[ RECORD 6 ]------------------------- time | Sun Jan 02 00:00:00 2000 no_epoch | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Jan 02 00:00:00 2000 custom_2 | Mon Dec 27 00:00:00 1999 -[ RECORD 7 ]------------------------- time | Mon Jan 03 00:00:00 2000 no_epoch | Mon Jan 03 00:00:00 2000 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Jan 02 00:00:00 2000 custom_2 | Mon Jan 03 00:00:00 2000 -[ RECORD 8 ]------------------------- time | Sat Sep 12 00:00:00 3018 no_epoch | Mon Sep 07 00:00:00 3018 always_true | t pg_epoch | Sat Sep 12 00:00:00 3018 unix_epoch | Thu Sep 10 00:00:00 3018 custom_1 | Sun Sep 06 00:00:00 3018 custom_2 | Mon Sep 07 00:00:00 3018 -[ RECORD 9 ]------------------------- time | Sun Sep 13 00:00:00 3018 no_epoch | Mon Sep 07 00:00:00 3018 always_true | t pg_epoch | Sat Sep 12 00:00:00 3018 unix_epoch | Thu Sep 10 00:00:00 3018 custom_1 | Sun Sep 13 00:00:00 3018 custom_2 | Mon Sep 07 00:00:00 3018 -[ RECORD 10 ]------------------------ time | Mon Sep 14 00:00:00 3018 no_epoch | Mon Sep 14 00:00:00 3018 always_true | t pg_epoch | Sat Sep 12 00:00:00 3018 unix_epoch | Thu Sep 10 00:00:00 3018 custom_1 | Sun Sep 13 00:00:00 3018 custom_2 | Mon Sep 14 00:00:00 3018 -[ RECORD 11 ]------------------------ time | Tue Sep 15 00:00:00 3018 no_epoch | Mon Sep 14 00:00:00 3018 always_true | t pg_epoch | Sat Sep 12 00:00:00 3018 unix_epoch | Thu Sep 10 00:00:00 3018 custom_1 | Sun Sep 13 00:00:00 3018 custom_2 | Mon Sep 14 00:00:00 3018 SELECT time, time_bucket(INTERVAL '1 week', time) no_epoch, time_bucket(INTERVAL '1 week', time) = time_bucket(INTERVAL '1 week', time, date '2000-01-03') always_true, time_bucket(INTERVAL '1 week', time, date '2000-01-01') pg_epoch, time_bucket(INTERVAL '1 week', time, (timestamp 'epoch')::date) unix_epoch, time_bucket(INTERVAL '1 week', time, date '3018-09-13') custom_1, time_bucket(INTERVAL '1 week', time, date '3018-09-14') custom_2 FROM unnest(ARRAY[ date '1999-12-31', date '2000-01-01', date '2000-01-02', date '2000-01-03', date '3018-09-12', date '3018-09-13', date '3018-09-14', date '3018-09-15' ]) AS time; -[ RECORD 1 ]----------- time | 12-31-1999 no_epoch | 12-27-1999 always_true | t pg_epoch | 12-25-1999 unix_epoch | 12-30-1999 custom_1 | 12-26-1999 custom_2 | 12-27-1999 -[ RECORD 2 ]----------- time | 01-01-2000 no_epoch | 12-27-1999 always_true | t pg_epoch | 01-01-2000 unix_epoch | 12-30-1999 custom_1 | 12-26-1999 custom_2 | 12-27-1999 -[ RECORD 3 ]----------- time | 01-02-2000 no_epoch | 12-27-1999 always_true | t pg_epoch | 01-01-2000 unix_epoch | 12-30-1999 custom_1 | 01-02-2000 custom_2 | 12-27-1999 -[ RECORD 4 ]----------- time | 01-03-2000 no_epoch | 01-03-2000 always_true | t pg_epoch | 01-01-2000 unix_epoch | 12-30-1999 custom_1 | 01-02-2000 custom_2 | 01-03-2000 -[ RECORD 5 ]----------- time | 09-12-3018 no_epoch | 09-07-3018 always_true | t pg_epoch | 09-12-3018 unix_epoch | 09-10-3018 custom_1 | 09-06-3018 custom_2 | 09-07-3018 -[ RECORD 6 ]----------- time | 09-13-3018 no_epoch | 09-07-3018 always_true | t pg_epoch | 09-12-3018 unix_epoch | 09-10-3018 custom_1 | 09-13-3018 custom_2 | 09-07-3018 -[ RECORD 7 ]----------- time | 09-14-3018 no_epoch | 09-14-3018 always_true | t pg_epoch | 09-12-3018 unix_epoch | 09-10-3018 custom_1 | 09-13-3018 custom_2 | 09-14-3018 -[ RECORD 8 ]----------- time | 09-15-3018 no_epoch | 09-14-3018 always_true | t pg_epoch | 09-12-3018 unix_epoch | 09-10-3018 custom_1 | 09-13-3018 custom_2 | 09-14-3018 \x --really old origin works if date around that time SELECT time, time_bucket(INTERVAL '1 week', time, timestamp without time zone '4710-11-24 01:01:01.0 BC') FROM unnest(ARRAY[ timestamp without time zone '4710-11-24 01:01:01.0 BC', timestamp without time zone '4710-11-25 01:01:01.0 BC', timestamp without time zone '2001-01-01', timestamp without time zone '3001-01-01' ]) AS time; time | time_bucket -----------------------------+----------------------------- Sat Nov 24 01:01:01 4710 BC | Sat Nov 24 01:01:01 4710 BC Sun Nov 25 01:01:01 4710 BC | Sat Nov 24 01:01:01 4710 BC Mon Jan 01 00:00:00 2001 | Sat Dec 30 01:01:01 2000 Thu Jan 01 00:00:00 3001 | Sat Dec 27 01:01:01 3000 SELECT time, time_bucket(INTERVAL '1 week', time, timestamp without time zone '294270-12-30 23:59:59.9999') FROM unnest(ARRAY[ timestamp without time zone '294270-12-29 23:59:59.9999', timestamp without time zone '294270-12-30 23:59:59.9999', timestamp without time zone '294270-12-31 23:59:59.9999', timestamp without time zone '2001-01-01', timestamp without time zone '3001-01-01' ]) AS time; time | time_bucket ---------------------------------+--------------------------------- Thu Dec 29 23:59:59.9999 294270 | Fri Dec 23 23:59:59.9999 294270 Fri Dec 30 23:59:59.9999 294270 | Fri Dec 30 23:59:59.9999 294270 Sat Dec 31 23:59:59.9999 294270 | Fri Dec 30 23:59:59.9999 294270 Mon Jan 01 00:00:00 2001 | Fri Dec 29 23:59:59.9999 2000 Thu Jan 01 00:00:00 3001 | Fri Dec 26 23:59:59.9999 3000 \set ON_ERROR_STOP 0 --really old origin + very new data + long period errors SELECT time, time_bucket(INTERVAL '100000 day', time, timestamp without time zone '4710-11-24 01:01:01.0 BC') FROM unnest(ARRAY[ timestamp without time zone '294270-12-31 23:59:59.9999' ]) AS time; ERROR: timestamp out of range SELECT time, time_bucket(INTERVAL '100000 day', time, timestamp with time zone '4710-11-25 01:01:01.0 BC') FROM unnest(ARRAY[ timestamp with time zone '294270-12-30 23:59:59.9999' ]) AS time; ERROR: timestamp out of range --really high origin + old data + long period errors out SELECT time, time_bucket(INTERVAL '10000000 day', time, timestamp without time zone '294270-12-31 23:59:59.9999') FROM unnest(ARRAY[ timestamp without time zone '4710-11-24 01:01:01.0 BC' ]) AS time; ERROR: timestamp out of range SELECT time, time_bucket(INTERVAL '10000000 day', time, timestamp with time zone '294270-12-31 23:59:59.9999') FROM unnest(ARRAY[ timestamp with time zone '4710-11-24 01:01:01.0 BC' ]) AS time; ERROR: timestamp out of range \set ON_ERROR_STOP 1 ------------------------------------------- --- Test time_bucket with month periods --- ------------------------------------------- SET datestyle TO ISO; SELECT time::date, time_bucket('1 month', time::date) AS "1m", time_bucket('2 month', time::date) AS "2m", time_bucket('3 month', time::date) AS "3m", time_bucket('1 month', time::date, '2000-02-01'::date) AS "1m origin", time_bucket('2 month', time::date, '2000-02-01'::date) AS "2m origin", time_bucket('3 month', time::date, '2000-02-01'::date) AS "3m origin" FROM generate_series('1990-01-03'::date,'1990-06-03'::date,'1month'::interval) time; time | 1m | 2m | 3m | 1m origin | 2m origin | 3m origin ------------+------------+------------+------------+------------+------------+------------ 1990-01-03 | 1990-01-01 | 1990-01-01 | 1990-01-01 | 1990-01-01 | 1989-12-01 | 1989-11-01 1990-02-03 | 1990-02-01 | 1990-01-01 | 1990-01-01 | 1990-02-01 | 1990-02-01 | 1990-02-01 1990-03-03 | 1990-03-01 | 1990-03-01 | 1990-01-01 | 1990-03-01 | 1990-02-01 | 1990-02-01 1990-04-03 | 1990-04-01 | 1990-03-01 | 1990-04-01 | 1990-04-01 | 1990-04-01 | 1990-02-01 1990-05-03 | 1990-05-01 | 1990-05-01 | 1990-04-01 | 1990-05-01 | 1990-04-01 | 1990-05-01 1990-06-03 | 1990-06-01 | 1990-05-01 | 1990-04-01 | 1990-06-01 | 1990-06-01 | 1990-05-01 SELECT time, time_bucket('1 month', time) AS "1m", time_bucket('2 month', time) AS "2m", time_bucket('3 month', time) AS "3m", time_bucket('1 month', time, '2000-02-01'::timestamp) AS "1m origin", time_bucket('2 month', time, '2000-02-01'::timestamp) AS "2m origin", time_bucket('3 month', time, '2000-02-01'::timestamp) AS "3m origin" FROM generate_series('1990-01-03'::timestamp,'1990-06-03'::timestamp,'1month'::interval) time; time | 1m | 2m | 3m | 1m origin | 2m origin | 3m origin ---------------------+---------------------+---------------------+---------------------+---------------------+---------------------+--------------------- 1990-01-03 00:00:00 | 1990-01-01 00:00:00 | 1990-01-01 00:00:00 | 1990-01-01 00:00:00 | 1990-01-01 00:00:00 | 1989-12-01 00:00:00 | 1989-11-01 00:00:00 1990-02-03 00:00:00 | 1990-02-01 00:00:00 | 1990-01-01 00:00:00 | 1990-01-01 00:00:00 | 1990-02-01 00:00:00 | 1990-02-01 00:00:00 | 1990-02-01 00:00:00 1990-03-03 00:00:00 | 1990-03-01 00:00:00 | 1990-03-01 00:00:00 | 1990-01-01 00:00:00 | 1990-03-01 00:00:00 | 1990-02-01 00:00:00 | 1990-02-01 00:00:00 1990-04-03 00:00:00 | 1990-04-01 00:00:00 | 1990-03-01 00:00:00 | 1990-04-01 00:00:00 | 1990-04-01 00:00:00 | 1990-04-01 00:00:00 | 1990-02-01 00:00:00 1990-05-03 00:00:00 | 1990-05-01 00:00:00 | 1990-05-01 00:00:00 | 1990-04-01 00:00:00 | 1990-05-01 00:00:00 | 1990-04-01 00:00:00 | 1990-05-01 00:00:00 1990-06-03 00:00:00 | 1990-06-01 00:00:00 | 1990-05-01 00:00:00 | 1990-04-01 00:00:00 | 1990-06-01 00:00:00 | 1990-06-01 00:00:00 | 1990-05-01 00:00:00 SELECT time, time_bucket('1 month', time) AS "1m", time_bucket('2 month', time) AS "2m", time_bucket('3 month', time) AS "3m", time_bucket('1 month', time, '2000-02-01'::timestamptz) AS "1m origin", time_bucket('2 month', time, '2000-02-01'::timestamptz) AS "2m origin", time_bucket('3 month', time, '2000-02-01'::timestamptz) AS "3m origin" FROM generate_series('1990-01-03'::timestamptz,'1990-06-03'::timestamptz,'1month'::interval) time; time | 1m | 2m | 3m | 1m origin | 2m origin | 3m origin ------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+------------------------ 1990-01-03 00:00:00-05 | 1989-12-31 19:00:00-05 | 1989-12-31 19:00:00-05 | 1989-12-31 19:00:00-05 | 1989-12-31 19:00:00-05 | 1989-11-30 19:00:00-05 | 1989-10-31 19:00:00-05 1990-02-03 00:00:00-05 | 1990-01-31 19:00:00-05 | 1989-12-31 19:00:00-05 | 1989-12-31 19:00:00-05 | 1990-01-31 19:00:00-05 | 1990-01-31 19:00:00-05 | 1990-01-31 19:00:00-05 1990-03-03 00:00:00-05 | 1990-02-28 19:00:00-05 | 1990-02-28 19:00:00-05 | 1989-12-31 19:00:00-05 | 1990-02-28 19:00:00-05 | 1990-01-31 19:00:00-05 | 1990-01-31 19:00:00-05 1990-04-03 00:00:00-04 | 1990-03-31 19:00:00-05 | 1990-02-28 19:00:00-05 | 1990-03-31 19:00:00-05 | 1990-03-31 19:00:00-05 | 1990-03-31 19:00:00-05 | 1990-01-31 19:00:00-05 1990-05-03 00:00:00-04 | 1990-04-30 20:00:00-04 | 1990-04-30 20:00:00-04 | 1990-03-31 19:00:00-05 | 1990-04-30 20:00:00-04 | 1990-03-31 19:00:00-05 | 1990-04-30 20:00:00-04 1990-06-03 00:00:00-04 | 1990-05-31 20:00:00-04 | 1990-04-30 20:00:00-04 | 1990-03-31 19:00:00-05 | 1990-05-31 20:00:00-04 | 1990-05-31 20:00:00-04 | 1990-04-30 20:00:00-04 --------------------------------------- --- Test time_bucket with timezones --- --------------------------------------- -- test NULL args SELECT time_bucket(NULL::interval,now(),'Europe/Berlin'), time_bucket('1day',NULL::timestamptz,'Europe/Berlin'), time_bucket('1day',now(),NULL::text), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin',NULL), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin','2020-04-01',NULL), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin',NULL,NULL), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin',"offset":=NULL::interval), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin',origin:=NULL::timestamptz); time_bucket | time_bucket | time_bucket | time_bucket | time_bucket | time_bucket | time_bucket | time_bucket -------------+-------------+-------------+------------------------+------------------------+------------------------+------------------------+------------------------ | | | 2020-02-02 18:00:00-05 | 2020-02-03 00:00:00-05 | 2020-02-02 18:00:00-05 | 2020-02-02 18:00:00-05 | 2020-02-02 18:00:00-05 SET datestyle TO ISO; SELECT time_bucket('1day', ts) AS "UTC", time_bucket('1day', ts, 'Europe/Berlin') AS "Berlin", time_bucket('1day', ts, 'Europe/London') AS "London", time_bucket('1day', ts, 'America/New_York') AS "New York", time_bucket('1day', ts, 'PST') AS "PST", time_bucket('1day', ts, current_setting('timezone')) AS "current" FROM generate_series('1999-12-31 17:00'::timestamptz,'2000-01-02 3:00'::timestamptz, '1hour'::interval) ts; UTC | Berlin | London | New York | PST | current ------------------------+------------------------+------------------------+------------------------+------------------------+------------------------ 1999-12-30 19:00:00-05 | 1999-12-30 18:00:00-05 | 1999-12-30 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-30 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-30 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-31 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-31 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-31 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 2000-01-01 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-02 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-02 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-02 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-02 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-02 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-02 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-02 00:00:00-05 | 2000-01-02 03:00:00-05 | 2000-01-02 00:00:00-05 SELECT time_bucket('1month', ts) AS "UTC", time_bucket('1month', ts, 'Europe/Berlin') AS "Berlin", time_bucket('1month', ts, 'America/New_York') AS "New York", time_bucket('1month', ts, current_setting('timezone')) AS "current", time_bucket('2month', ts, current_setting('timezone')) AS "2m", time_bucket('2month', ts, current_setting('timezone'), '2000-02-01'::timestamp) AS "2m origin", time_bucket('2month', ts, current_setting('timezone'), "offset":='14 day'::interval) AS "2m offset", time_bucket('2month', ts, current_setting('timezone'), '2000-02-01'::timestamp, '7 day'::interval) AS "2m offset + origin" FROM generate_series('1999-12-01'::timestamptz,'2000-09-01'::timestamptz, '9 day'::interval) ts; UTC | Berlin | New York | current | 2m | 2m origin | 2m offset | 2m offset + origin ------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+------------------------ 1999-11-30 19:00:00-05 | 1999-11-30 18:00:00-05 | 1999-12-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-15 00:00:00-05 | 1999-10-08 00:00:00-04 1999-11-30 19:00:00-05 | 1999-11-30 18:00:00-05 | 1999-12-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-15 00:00:00-05 | 1999-12-08 00:00:00-05 1999-11-30 19:00:00-05 | 1999-11-30 18:00:00-05 | 1999-12-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-15 00:00:00-05 | 1999-12-08 00:00:00-05 1999-11-30 19:00:00-05 | 1999-11-30 18:00:00-05 | 1999-12-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-15 00:00:00-05 | 1999-12-08 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-15 00:00:00-05 | 1999-12-08 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 1999-12-08 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 1999-12-08 00:00:00-05 2000-01-31 19:00:00-05 | 2000-01-31 18:00:00-05 | 2000-02-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 1999-12-08 00:00:00-05 2000-01-31 19:00:00-05 | 2000-01-31 18:00:00-05 | 2000-02-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-01-31 19:00:00-05 | 2000-01-31 18:00:00-05 | 2000-02-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-01-31 19:00:00-05 | 2000-01-31 18:00:00-05 | 2000-02-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-02-29 19:00:00-05 | 2000-02-29 18:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-02-29 19:00:00-05 | 2000-02-29 18:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-02-29 19:00:00-05 | 2000-02-29 18:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-03-31 19:00:00-05 | 2000-03-31 17:00:00-05 | 2000-04-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-03-31 19:00:00-05 | 2000-03-31 17:00:00-05 | 2000-04-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-04-08 00:00:00-04 2000-03-31 19:00:00-05 | 2000-03-31 17:00:00-05 | 2000-04-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-04-08 00:00:00-04 2000-04-30 20:00:00-04 | 2000-04-30 18:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-04-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-04-08 00:00:00-04 2000-04-30 20:00:00-04 | 2000-04-30 18:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-04-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-04-08 00:00:00-04 2000-04-30 20:00:00-04 | 2000-04-30 18:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-04-01 00:00:00-05 | 2000-05-15 00:00:00-04 | 2000-04-08 00:00:00-04 2000-04-30 20:00:00-04 | 2000-04-30 18:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-04-01 00:00:00-05 | 2000-05-15 00:00:00-04 | 2000-04-08 00:00:00-04 2000-05-31 20:00:00-04 | 2000-05-31 18:00:00-04 | 2000-06-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-15 00:00:00-04 | 2000-04-08 00:00:00-04 2000-05-31 20:00:00-04 | 2000-05-31 18:00:00-04 | 2000-06-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-05-31 20:00:00-04 | 2000-05-31 18:00:00-04 | 2000-06-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-06-30 20:00:00-04 | 2000-06-30 18:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-06-30 20:00:00-04 | 2000-06-30 18:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-06-30 20:00:00-04 | 2000-06-30 18:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-07-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-06-30 20:00:00-04 | 2000-06-30 18:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-07-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-07-31 20:00:00-04 | 2000-07-31 18:00:00-04 | 2000-08-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-15 00:00:00-04 | 2000-08-08 00:00:00-04 2000-07-31 20:00:00-04 | 2000-07-31 18:00:00-04 | 2000-08-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-15 00:00:00-04 | 2000-08-08 00:00:00-04 2000-07-31 20:00:00-04 | 2000-07-31 18:00:00-04 | 2000-08-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-15 00:00:00-04 | 2000-08-08 00:00:00-04 RESET datestyle; ------------------------------------- --- Test time input functions -- ------------------------------------- \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION test.interval_to_internal(coltype REGTYPE, value ANYELEMENT = NULL::BIGINT) RETURNS BIGINT AS :MODULE_PATHNAME, 'ts_dimension_interval_to_internal_test' LANGUAGE C VOLATILE; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT test.interval_to_internal('TIMESTAMP'::regtype, INTERVAL '1 day'); interval_to_internal ---------------------- 86400000000 SELECT test.interval_to_internal('TIMESTAMP'::regtype, 86400000000); interval_to_internal ---------------------- 86400000000 ---should give warning SELECT test.interval_to_internal('TIMESTAMP'::regtype, 86400); WARNING: unexpected interval: smaller than one second HINT: The interval is specified in microseconds. interval_to_internal ---------------------- 86400 SELECT test.interval_to_internal('TIMESTAMP'::regtype); interval_to_internal ---------------------- 604800000000 SELECT test.interval_to_internal('BIGINT'::regtype, 2147483649::bigint); interval_to_internal ---------------------- 2147483649 -- Default interval for integer is supported as part of -- hypertable generalization SELECT test.interval_to_internal('INT'::regtype); interval_to_internal ---------------------- 100000 SELECT test.interval_to_internal('SMALLINT'::regtype); interval_to_internal ---------------------- 10000 SELECT test.interval_to_internal('BIGINT'::regtype); interval_to_internal ---------------------- 1000000 SELECT test.interval_to_internal('TIMESTAMPTZ'::regtype); interval_to_internal ---------------------- 604800000000 SELECT test.interval_to_internal('TIMESTAMP'::regtype); interval_to_internal ---------------------- 604800000000 SELECT test.interval_to_internal('DATE'::regtype); interval_to_internal ---------------------- 604800000000 \set VERBOSITY terse \set ON_ERROR_STOP 0 SELECT test.interval_to_internal('INT'::regtype, 2147483649::bigint); ERROR: invalid interval: must be between 1 and 2147483647 SELECT test.interval_to_internal('SMALLINT'::regtype, 32768::bigint); ERROR: invalid interval: must be between 1 and 32767 SELECT test.interval_to_internal('TEXT'::regtype, 32768::bigint); ERROR: invalid type for dimension "testcol" SELECT test.interval_to_internal('INT'::regtype, INTERVAL '1 day'); ERROR: invalid interval type for integer dimension \set ON_ERROR_STOP 1 ================================================ FILE: test/expected/timestamp-18.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Utility function for grouping/slotting time with a given interval. CREATE OR REPLACE FUNCTION date_group( field timestamp, group_interval interval ) RETURNS timestamp LANGUAGE SQL STABLE AS $BODY$ SELECT to_timestamp((EXTRACT(EPOCH from $1)::int / EXTRACT(EPOCH from group_interval)::int) * EXTRACT(EPOCH from group_interval)::int)::timestamp; $BODY$; CREATE TABLE PUBLIC."testNs" ( "timeCustom" TIMESTAMP NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."testNs" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA "testNs" AUTHORIZATION :ROLE_DEFAULT_PERM_USER; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT * FROM create_hypertable('"public"."testNs"', 'timeCustom', 'device_id', 2, associated_schema_name=>'testNs' ); WARNING: column type "timestamp without time zone" used for "timeCustom" does not follow best practices hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | testNs | t \c :TEST_DBNAME INSERT INTO PUBLIC."testNs"("timeCustom", device_id, series_0, series_1) VALUES ('2009-11-12T01:00:00+00:00', 'dev1', 1.5, 1), ('2009-11-12T01:00:00+00:00', 'dev1', 1.5, 2), ('2009-11-10T23:00:02+00:00', 'dev1', 2.5, 3); INSERT INTO PUBLIC."testNs"("timeCustom", device_id, series_0, series_1) VALUES ('2009-11-10T23:00:00+00:00', 'dev2', 1.5, 1), ('2009-11-10T23:00:00+00:00', 'dev2', 1.5, 2); SELECT * FROM PUBLIC."testNs"; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool --------------------------+-----------+----------+----------+----------+------------- Thu Nov 12 01:00:00 2009 | dev1 | 1.5 | 1 | | Thu Nov 12 01:00:00 2009 | dev1 | 1.5 | 2 | | Tue Nov 10 23:00:02 2009 | dev1 | 2.5 | 3 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 1 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 2 | | SET client_min_messages = WARNING; \echo 'The next 2 queries will differ in output between UTC and EST since the mod is on the 100th hour UTC' The next 2 queries will differ in output between UTC and EST since the mod is on the 100th hour UTC SET timezone = 'UTC'; SELECT date_group("timeCustom", '100 days') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC; time | sum --------------------------+----- Sun Sep 13 00:00:00 2009 | 8.5 SET timezone = 'EST'; SELECT date_group("timeCustom", '100 days') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC; time | sum --------------------------+----- Sat Sep 12 19:00:00 2009 | 8.5 \echo 'The rest of the queries will be the same in output between UTC and EST' The rest of the queries will be the same in output between UTC and EST SET timezone = 'UTC'; SELECT date_group("timeCustom", '1 day') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC; time | sum --------------------------+----- Tue Nov 10 00:00:00 2009 | 5.5 Thu Nov 12 00:00:00 2009 | 3 SET timezone = 'EST'; SELECT date_group("timeCustom", '1 day') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC; time | sum --------------------------+----- Mon Nov 09 19:00:00 2009 | 5.5 Wed Nov 11 19:00:00 2009 | 3 SET timezone = 'UTC'; SELECT * FROM PUBLIC."testNs" WHERE "timeCustom" >= TIMESTAMP '2009-11-10T23:00:00' AND "timeCustom" < TIMESTAMP '2009-11-12T01:00:00' ORDER BY "timeCustom" DESC, device_id, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool --------------------------+-----------+----------+----------+----------+------------- Tue Nov 10 23:00:02 2009 | dev1 | 2.5 | 3 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 1 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 2 | | SET timezone = 'EST'; SELECT * FROM PUBLIC."testNs" WHERE "timeCustom" >= TIMESTAMP '2009-11-10T23:00:00' AND "timeCustom" < TIMESTAMP '2009-11-12T01:00:00' ORDER BY "timeCustom" DESC, device_id, series_1; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool --------------------------+-----------+----------+----------+----------+------------- Tue Nov 10 23:00:02 2009 | dev1 | 2.5 | 3 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 1 | | Tue Nov 10 23:00:00 2009 | dev2 | 1.5 | 2 | | SET timezone = 'UTC'; SELECT date_group("timeCustom", '1 day') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC LIMIT 2; time | sum --------------------------+----- Tue Nov 10 00:00:00 2009 | 5.5 Thu Nov 12 00:00:00 2009 | 3 SET timezone = 'EST'; SELECT date_group("timeCustom", '1 day') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC LIMIT 2; time | sum --------------------------+----- Mon Nov 09 19:00:00 2009 | 5.5 Wed Nov 11 19:00:00 2009 | 3 ------------------------------------ -- Test time conversion functions -- ------------------------------------ \set ON_ERROR_STOP 0 SET timezone = 'UTC'; -- Conversion to timestamp using Postgres built-in function taking -- double. Gives inaccurate result on Postgres <= 9.6.2. Accurate on -- Postgres >= 9.6.3. SELECT to_timestamp(1486480176.236538); to_timestamp ------------------------------------- Tue Feb 07 15:09:36.236538 2017 UTC -- extension-specific version taking microsecond UNIX timestamp SELECT _timescaledb_functions.to_timestamp(1486480176236538); to_timestamp ------------------------------------- Tue Feb 07 15:09:36.236538 2017 UTC -- Should be the inverse of the statement above. SELECT _timescaledb_functions.to_unix_microseconds('2017-02-07 15:09:36.236538+00'); to_unix_microseconds ---------------------- 1486480176236538 -- For timestamps, BIGINT MAX represents +Infinity and BIGINT MIN -- -Infinity. We keep this notion for UNIX epoch time: SELECT _timescaledb_functions.to_unix_microseconds('+infinity'); to_unix_microseconds ---------------------- 9223372036854775807 SELECT _timescaledb_functions.to_timestamp(9223372036854775807); to_timestamp -------------- infinity SELECT _timescaledb_functions.to_unix_microseconds('-infinity'); to_unix_microseconds ---------------------- -9223372036854775808 SELECT _timescaledb_functions.to_timestamp(-9223372036854775808); to_timestamp -------------- -infinity -- In UNIX microseconds, the largest bigint value below infinity -- (BIGINT MAX) is smaller than internal date upper bound and should -- therefore be OK. Further, converting to the internal postgres epoch -- cannot overflow a 64-bit INTEGER since the postgres epoch is at a -- later date compared to the UNIX epoch, and is therefore represented -- by a smaller number SELECT _timescaledb_functions.to_timestamp(9223372036854775806); to_timestamp --------------------------------------- Sun Jan 10 04:00:54.775806 294247 UTC -- Julian day zero is -210866803200000000 microseconds from UNIX epoch SELECT _timescaledb_functions.to_timestamp(-210866803200000000); to_timestamp --------------------------------- Mon Nov 24 00:00:00 4714 UTC BC \set VERBOSITY default -- Going beyond Julian day zero should give out-of-range error SELECT _timescaledb_functions.to_timestamp(-210866803200000001); ERROR: timestamp out of range -- Lower bound on date (should return the Julian day zero UNIX timestamp above) SELECT _timescaledb_functions.to_unix_microseconds('4714-11-24 00:00:00+00 BC'); to_unix_microseconds ---------------------- -210866803200000000 -- Going beyond lower bound on date should return out-of-range SELECT _timescaledb_functions.to_unix_microseconds('4714-11-23 23:59:59.999999+00 BC'); ERROR: timestamp out of range: "4714-11-23 23:59:59.999999+00 BC" LINE 1: ...ELECT _timescaledb_functions.to_unix_microseconds('4714-11-2... ^ -- The upper bound for Postgres TIMESTAMPTZ SELECT timestamp '294276-12-31 23:59:59.999999+00'; timestamp ----------------------------------- Sun Dec 31 23:59:59.999999 294276 -- Going beyond the upper bound, should fail SELECT timestamp '294276-12-31 23:59:59.999999+00' + interval '1 us'; ERROR: timestamp out of range -- Cannot represent the upper bound timestamp with a UNIX microsecond timestamp -- since the Postgres epoch is at a later date than the UNIX epoch. SELECT _timescaledb_functions.to_unix_microseconds('294276-12-31 23:59:59.999999+00'); ERROR: timestamp out of range -- Subtracting the difference between the two epochs (10957 days) should bring -- us within range. SELECT timestamp '294276-12-31 23:59:59.999999+00' - interval '10957 days'; ?column? ----------------------------------- Fri Jan 01 23:59:59.999999 294247 SELECT _timescaledb_functions.to_unix_microseconds('294247-01-01 23:59:59.999999'); to_unix_microseconds ---------------------- 9223371331199999999 -- Adding one microsecond should take us out-of-range again SELECT timestamp '294247-01-01 23:59:59.999999' + interval '1 us'; ?column? ---------------------------- Sat Jan 02 00:00:00 294247 SELECT _timescaledb_functions.to_unix_microseconds(timestamp '294247-01-01 23:59:59.999999' + interval '1 us'); ERROR: timestamp out of range --no time_bucketing of dates not by integer # of days SELECT time_bucket('1 hour', DATE '2012-01-01'); ERROR: interval must not have sub-day precision SELECT time_bucket('25 hour', DATE '2012-01-01'); ERROR: interval must be a multiple of a day \set ON_ERROR_STOP 1 SELECT time_bucket(INTERVAL '1 day', TIMESTAMP '2011-01-02 01:01:01'); time_bucket -------------------------- Sun Jan 02 00:00:00 2011 SELECT time, time_bucket(INTERVAL '2 day ', time) FROM unnest(ARRAY[ TIMESTAMP '2011-01-01 01:01:01', TIMESTAMP '2011-01-02 01:01:01', TIMESTAMP '2011-01-03 01:01:01', TIMESTAMP '2011-01-04 01:01:01' ]) AS time; time | time_bucket --------------------------+-------------------------- Sat Jan 01 01:01:01 2011 | Sat Jan 01 00:00:00 2011 Sun Jan 02 01:01:01 2011 | Sat Jan 01 00:00:00 2011 Mon Jan 03 01:01:01 2011 | Mon Jan 03 00:00:00 2011 Tue Jan 04 01:01:01 2011 | Mon Jan 03 00:00:00 2011 SELECT int_def, time_bucket(int_def,TIMESTAMP '2011-01-02 01:01:01.111') FROM unnest(ARRAY[ INTERVAL '1 millisecond', INTERVAL '1 second', INTERVAL '1 minute', INTERVAL '1 hour', INTERVAL '1 day', INTERVAL '2 millisecond', INTERVAL '2 second', INTERVAL '2 minute', INTERVAL '2 hour', INTERVAL '2 day' ]) AS int_def; int_def | time_bucket --------------+------------------------------ @ 0.001 secs | Sun Jan 02 01:01:01.111 2011 @ 1 sec | Sun Jan 02 01:01:01 2011 @ 1 min | Sun Jan 02 01:01:00 2011 @ 1 hour | Sun Jan 02 01:00:00 2011 @ 1 day | Sun Jan 02 00:00:00 2011 @ 0.002 secs | Sun Jan 02 01:01:01.11 2011 @ 2 secs | Sun Jan 02 01:01:00 2011 @ 2 mins | Sun Jan 02 01:00:00 2011 @ 2 hours | Sun Jan 02 00:00:00 2011 @ 2 days | Sat Jan 01 00:00:00 2011 \set ON_ERROR_STOP 0 SELECT time_bucket(INTERVAL '1 year 1d',TIMESTAMP '2011-01-02 01:01:01.111'); ERROR: month intervals cannot have day or time component SELECT time_bucket(INTERVAL '1 month 1 minute',TIMESTAMP '2011-01-02 01:01:01.111'); ERROR: month intervals cannot have day or time component \set ON_ERROR_STOP 1 SELECT time, time_bucket(INTERVAL '5 minute', time) FROM unnest(ARRAY[ TIMESTAMP '1970-01-01 00:59:59.999999', TIMESTAMP '1970-01-01 01:01:00', TIMESTAMP '1970-01-01 01:04:59.999999', TIMESTAMP '1970-01-01 01:05:00' ]) AS time; time | time_bucket ---------------------------------+-------------------------- Thu Jan 01 00:59:59.999999 1970 | Thu Jan 01 00:55:00 1970 Thu Jan 01 01:01:00 1970 | Thu Jan 01 01:00:00 1970 Thu Jan 01 01:04:59.999999 1970 | Thu Jan 01 01:00:00 1970 Thu Jan 01 01:05:00 1970 | Thu Jan 01 01:05:00 1970 SELECT time, time_bucket(INTERVAL '5 minute', time) FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:04:59.999999', TIMESTAMP '2011-01-02 01:05:00', TIMESTAMP '2011-01-02 01:09:59.999999', TIMESTAMP '2011-01-02 01:10:00' ]) AS time; time | time_bucket ---------------------------------+-------------------------- Sun Jan 02 01:04:59.999999 2011 | Sun Jan 02 01:00:00 2011 Sun Jan 02 01:05:00 2011 | Sun Jan 02 01:05:00 2011 Sun Jan 02 01:09:59.999999 2011 | Sun Jan 02 01:05:00 2011 Sun Jan 02 01:10:00 2011 | Sun Jan 02 01:10:00 2011 --offset with interval SELECT time, time_bucket(INTERVAL '5 minute', time , INTERVAL '2 minutes') FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:01:59.999999', TIMESTAMP '2011-01-02 01:02:00', TIMESTAMP '2011-01-02 01:06:59.999999', TIMESTAMP '2011-01-02 01:07:00' ]) AS time; time | time_bucket ---------------------------------+-------------------------- Sun Jan 02 01:01:59.999999 2011 | Sun Jan 02 00:57:00 2011 Sun Jan 02 01:02:00 2011 | Sun Jan 02 01:02:00 2011 Sun Jan 02 01:06:59.999999 2011 | Sun Jan 02 01:02:00 2011 Sun Jan 02 01:07:00 2011 | Sun Jan 02 01:07:00 2011 SELECT time, time_bucket(INTERVAL '5 minute', time , - INTERVAL '2 minutes') FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:02:59.999999', TIMESTAMP '2011-01-02 01:03:00', TIMESTAMP '2011-01-02 01:07:59.999999', TIMESTAMP '2011-01-02 01:08:00' ]) AS time; time | time_bucket ---------------------------------+-------------------------- Sun Jan 02 01:02:59.999999 2011 | Sun Jan 02 00:58:00 2011 Sun Jan 02 01:03:00 2011 | Sun Jan 02 01:03:00 2011 Sun Jan 02 01:07:59.999999 2011 | Sun Jan 02 01:03:00 2011 Sun Jan 02 01:08:00 2011 | Sun Jan 02 01:08:00 2011 --offset with infinity -- timestamp SELECT time, time_bucket(INTERVAL '1 week', time, INTERVAL '1 day') FROM unnest(ARRAY[ timestamp '-Infinity', timestamp 'Infinity' ]) AS time; time | time_bucket -----------+------------- -infinity | -infinity infinity | infinity -- timestamptz SELECT time, time_bucket(INTERVAL '1 week', time, INTERVAL '1 day') FROM unnest(ARRAY[ timestamp with time zone '-Infinity', timestamp with time zone 'Infinity' ]) AS time; time | time_bucket -----------+------------- -infinity | -infinity infinity | infinity -- Date SELECT date, time_bucket(INTERVAL '1 week', date, INTERVAL '1 day') FROM unnest(ARRAY[ date '-Infinity', date 'Infinity' ]) AS date; date | time_bucket -----------+------------- -infinity | -infinity infinity | infinity --example to align with an origin SELECT time, time_bucket(INTERVAL '5 minute', time - (TIMESTAMP '2011-01-02 00:02:00' - TIMESTAMP 'epoch')) + (TIMESTAMP '2011-01-02 00:02:00'-TIMESTAMP 'epoch') FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:01:59.999999', TIMESTAMP '2011-01-02 01:02:00', TIMESTAMP '2011-01-02 01:06:59.999999', TIMESTAMP '2011-01-02 01:07:00' ]) AS time; time | ?column? ---------------------------------+-------------------------- Sun Jan 02 01:01:59.999999 2011 | Sun Jan 02 00:57:00 2011 Sun Jan 02 01:02:00 2011 | Sun Jan 02 01:02:00 2011 Sun Jan 02 01:06:59.999999 2011 | Sun Jan 02 01:02:00 2011 Sun Jan 02 01:07:00 2011 | Sun Jan 02 01:07:00 2011 --rounding version SELECT time, time_bucket(INTERVAL '5 minute', time , - INTERVAL '2.5 minutes') + INTERVAL '2 minutes 30 seconds' FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:05:01', TIMESTAMP '2011-01-02 01:07:29', TIMESTAMP '2011-01-02 01:02:30', TIMESTAMP '2011-01-02 01:07:30', TIMESTAMP '2011-01-02 01:02:29' ]) AS time; time | ?column? --------------------------+-------------------------- Sun Jan 02 01:05:01 2011 | Sun Jan 02 01:05:00 2011 Sun Jan 02 01:07:29 2011 | Sun Jan 02 01:05:00 2011 Sun Jan 02 01:02:30 2011 | Sun Jan 02 01:05:00 2011 Sun Jan 02 01:07:30 2011 | Sun Jan 02 01:10:00 2011 Sun Jan 02 01:02:29 2011 | Sun Jan 02 01:00:00 2011 --time_bucket with timezone should mimick date_trunc SET timezone TO 'UTC'; SELECT time, time_bucket(INTERVAL '1 hour', time), date_trunc('hour', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Jan 02 01:01:01 2011 UTC | Sun Jan 02 01:00:00 2011 UTC | Sun Jan 02 01:00:00 2011 UTC Sun Jan 02 00:01:01 2011 UTC | Sun Jan 02 00:00:00 2011 UTC | Sun Jan 02 00:00:00 2011 UTC Sat Jan 01 23:01:01 2011 UTC | Sat Jan 01 23:00:00 2011 UTC | Sat Jan 01 23:00:00 2011 UTC SELECT time, time_bucket(INTERVAL '1 day', time), date_trunc('day', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Jan 02 01:01:01 2011 UTC | Sun Jan 02 00:00:00 2011 UTC | Sun Jan 02 00:00:00 2011 UTC Sun Jan 02 00:01:01 2011 UTC | Sun Jan 02 00:00:00 2011 UTC | Sun Jan 02 00:00:00 2011 UTC Sat Jan 01 23:01:01 2011 UTC | Sat Jan 01 00:00:00 2011 UTC | Sat Jan 01 00:00:00 2011 UTC --what happens with a local tz SET timezone TO 'America/New_York'; SELECT time, time_bucket(INTERVAL '1 hour', time), date_trunc('hour', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Jan 02 01:01:01 2011 EST | Sun Jan 02 01:00:00 2011 EST | Sun Jan 02 01:00:00 2011 EST Sat Jan 01 19:01:01 2011 EST | Sat Jan 01 19:00:00 2011 EST | Sat Jan 01 19:00:00 2011 EST Sat Jan 01 18:01:01 2011 EST | Sat Jan 01 18:00:00 2011 EST | Sat Jan 01 18:00:00 2011 EST --Note the timestamp tz input is aligned with UTC day /not/ local day. different than date_trunc. SELECT time, time_bucket(INTERVAL '1 day', time), date_trunc('day', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-03 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-04 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Jan 02 01:01:01 2011 EST | Sat Jan 01 19:00:00 2011 EST | Sun Jan 02 00:00:00 2011 EST Sun Jan 02 19:01:01 2011 EST | Sun Jan 02 19:00:00 2011 EST | Sun Jan 02 00:00:00 2011 EST Mon Jan 03 18:01:01 2011 EST | Sun Jan 02 19:00:00 2011 EST | Mon Jan 03 00:00:00 2011 EST --can force local bucketing with simple cast. SELECT time, time_bucket(INTERVAL '1 day', time::timestamp), date_trunc('day', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-03 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-04 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+--------------------------+------------------------------ Sun Jan 02 01:01:01 2011 EST | Sun Jan 02 00:00:00 2011 | Sun Jan 02 00:00:00 2011 EST Sun Jan 02 19:01:01 2011 EST | Sun Jan 02 00:00:00 2011 | Sun Jan 02 00:00:00 2011 EST Mon Jan 03 18:01:01 2011 EST | Mon Jan 03 00:00:00 2011 | Mon Jan 03 00:00:00 2011 EST --can also use interval to correct SELECT time, time_bucket(INTERVAL '1 day', time, -INTERVAL '19 hours'), date_trunc('day', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-03 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-04 01:01:01+02' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Jan 02 01:01:01 2011 EST | Sun Jan 02 00:00:00 2011 EST | Sun Jan 02 00:00:00 2011 EST Sun Jan 02 19:01:01 2011 EST | Sun Jan 02 00:00:00 2011 EST | Sun Jan 02 00:00:00 2011 EST Mon Jan 03 18:01:01 2011 EST | Mon Jan 03 00:00:00 2011 EST | Mon Jan 03 00:00:00 2011 EST --dst: same local hour bucketed as two different hours. SELECT time, time_bucket(INTERVAL '1 hour', time), date_trunc('hour', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2017-11-05 12:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 13:05:00+07' ]) AS time; time | time_bucket | date_trunc ------------------------------+------------------------------+------------------------------ Sun Nov 05 01:05:00 2017 EDT | Sun Nov 05 01:00:00 2017 EDT | Sun Nov 05 01:00:00 2017 EDT Sun Nov 05 01:05:00 2017 EST | Sun Nov 05 01:00:00 2017 EST | Sun Nov 05 01:00:00 2017 EST --local alignment changes when bucketing by UTC across dst boundary SELECT time, time_bucket(INTERVAL '2 hour', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2017-11-05 10:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 12:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 13:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 15:05:00+07' ]) AS time; time | time_bucket ------------------------------+------------------------------ Sat Nov 04 23:05:00 2017 EDT | Sat Nov 04 22:00:00 2017 EDT Sun Nov 05 01:05:00 2017 EDT | Sun Nov 05 00:00:00 2017 EDT Sun Nov 05 01:05:00 2017 EST | Sun Nov 05 01:00:00 2017 EST Sun Nov 05 03:05:00 2017 EST | Sun Nov 05 03:00:00 2017 EST --local alignment is preserved when bucketing by local time across DST boundary. SELECT time, time_bucket(INTERVAL '2 hour', time::timestamp) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2017-11-05 10:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 12:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 13:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 15:05:00+07' ]) AS time; time | time_bucket ------------------------------+-------------------------- Sat Nov 04 23:05:00 2017 EDT | Sat Nov 04 22:00:00 2017 Sun Nov 05 01:05:00 2017 EDT | Sun Nov 05 00:00:00 2017 Sun Nov 05 01:05:00 2017 EST | Sun Nov 05 00:00:00 2017 Sun Nov 05 03:05:00 2017 EST | Sun Nov 05 02:00:00 2017 -- GitHub issue #7059: time_bucket with timezone + offset across DST boundary -- Asia/Amman: clocks skip from 00:00 to 01:00 on 2021-03-26 -- Input: 01:00+03 = 22:00 UTC → Result: 22:15 UTC = 01:15 local (00:00 + 15min offset) SELECT time_bucket('1 day', '2021-03-26 01:00:00+03'::timestamptz, timezone := 'Asia/Amman', "offset" := '15 minutes'::interval); time_bucket ------------------------------ Wed Mar 24 18:15:00 2021 EDT -- GitHub issue #8851: time_bucket with negative offset during DST fall-back -- Europe/Berlin: clocks repeat 02:00-02:59 on 2025-10-26 -- Input: 02:00+02 = 00:00 UTC → Result: 23:59:45 UTC = 01:59:45 local (02:00 - 15s offset) SELECT time_bucket('30 seconds', '2025-10-26 02:00:00+02'::timestamptz, timezone := 'Europe/Berlin', "offset" := '-15 seconds'::interval); time_bucket ------------------------------ Sat Oct 25 19:59:45 2025 EDT -- Additional DST edge cases for coverage of DST direction × offset sign combinations -- Spring-forward + negative offset -- Input: 01:30+03 = 22:30 UTC → Result: 22:45 UTC = 01:45 local (01:00 + 45min = 02:00 - 15min) SELECT time_bucket('1 hour', '2021-03-26 01:30:00+03'::timestamptz, timezone := 'Asia/Amman', "offset" := '-15 minutes'::interval); time_bucket ------------------------------ Thu Mar 25 17:45:00 2021 EDT -- Fall-back + positive offset -- Input: 02:30+01 = 01:30 UTC → Result: 01:15 UTC = 02:15 local (02:00 + 15min offset) SELECT time_bucket('1 hour', '2025-10-26 02:30:00+01'::timestamptz, timezone := 'Europe/Berlin', "offset" := '15 minutes'::interval); time_bucket ------------------------------ Sat Oct 25 21:15:00 2025 EDT -- Input exactly at DST spring-forward transition -- Input: 22:00 UTC = 00:00 local (the moment clocks jump to 01:00) -- Result: 22:15 UTC = 01:15 local (01:00 + 15min offset) SELECT time_bucket('1 hour', '2021-03-25 22:00:00+00'::timestamptz, timezone := 'Asia/Amman', "offset" := '15 minutes'::interval); time_bucket ------------------------------ Thu Mar 25 17:15:00 2021 EDT -- Input exactly at DST fall-back transition -- Input: 01:00 UTC = 03:00 CEST (the moment clocks go back to 02:00 CET) -- Result: 23:15 UTC = 01:15 local (01:00 + 15min offset, but in CET now) SELECT time_bucket('1 hour', '2025-10-26 01:00:00+00'::timestamptz, timezone := 'Europe/Berlin', "offset" := '15 minutes'::interval); time_bucket ------------------------------ Sat Oct 25 20:15:00 2025 EDT -- Offset larger than bucket size (1h offset with 30min bucket) -- Input: 01:30+03 = 22:30 UTC → Result: 22:30 UTC = 01:30 local (01:00 + 30min = 00:30 + 1h) SELECT time_bucket('30 minutes', '2021-03-26 01:30:00+03'::timestamptz, timezone := 'Asia/Amman', "offset" := '1 hour'::interval); time_bucket ------------------------------ Thu Mar 25 18:30:00 2021 EDT -- GitHub issue #9136: time_bucket with origin during DST fall-back -- When origin is in standard time but timestamp is in daylight time, -- the bucket could incorrectly start AFTER the timestamp. -- America/New_York: clocks go back at 02:00 EDT on 2024-11-03 -- Input: 01:30-04 (EDT) = 05:30 UTC; origin in EST -- Result should have bucket start <= timestamp (bucket in EDT, not EST) SELECT time_bucket('1 hour', '2024-11-03 01:30:00-04'::timestamptz, 'America/New_York', '2000-01-01 00:00:00 America/New_York'::timestamptz) as bucket, '2024-11-03 01:30:00-04'::timestamptz < time_bucket('1 hour', '2024-11-03 01:30:00-04'::timestamptz, 'America/New_York', '2000-01-01 00:00:00 America/New_York'::timestamptz) as ts_before_bucket; bucket | ts_before_bucket ------------------------------+------------------ Sun Nov 03 01:00:00 2024 EDT | f SELECT time, time_bucket(10::smallint, time) AS time_bucket_smallint, time_bucket(10::int, time) AS time_bucket_int, time_bucket(10::bigint, time) AS time_bucket_bigint FROM unnest(ARRAY[ '-11', '-10', '-9', '-1', '0', '1', '99', '100', '109', '110' ]::smallint[]) AS time; time | time_bucket_smallint | time_bucket_int | time_bucket_bigint ------+----------------------+-----------------+-------------------- -11 | -20 | -20 | -20 -10 | -10 | -10 | -10 -9 | -10 | -10 | -10 -1 | -10 | -10 | -10 0 | 0 | 0 | 0 1 | 0 | 0 | 0 99 | 90 | 90 | 90 100 | 100 | 100 | 100 109 | 100 | 100 | 100 110 | 110 | 110 | 110 SELECT time, time_bucket(10::smallint, time, 2::smallint) AS time_bucket_smallint, time_bucket(10::int, time, 2::int) AS time_bucket_int, time_bucket(10::bigint, time, 2::bigint) AS time_bucket_bigint FROM unnest(ARRAY[ '-9', '-8', '-7', '1', '2', '3', '101', '102', '111', '112' ]::smallint[]) AS time; time | time_bucket_smallint | time_bucket_int | time_bucket_bigint ------+----------------------+-----------------+-------------------- -9 | -18 | -18 | -18 -8 | -8 | -8 | -8 -7 | -8 | -8 | -8 1 | -8 | -8 | -8 2 | 2 | 2 | 2 3 | 2 | 2 | 2 101 | 92 | 92 | 92 102 | 102 | 102 | 102 111 | 102 | 102 | 102 112 | 112 | 112 | 112 SELECT time, time_bucket(10::smallint, time, -2::smallint) AS time_bucket_smallint, time_bucket(10::int, time, -2::int) AS time_bucket_int, time_bucket(10::bigint, time, -2::bigint) AS time_bucket_bigint FROM unnest(ARRAY[ '-13', '-12', '-11', '-3', '-2', '-1', '97', '98', '107', '108' ]::smallint[]) AS time; time | time_bucket_smallint | time_bucket_int | time_bucket_bigint ------+----------------------+-----------------+-------------------- -13 | -22 | -22 | -22 -12 | -12 | -12 | -12 -11 | -12 | -12 | -12 -3 | -12 | -12 | -12 -2 | -2 | -2 | -2 -1 | -2 | -2 | -2 97 | 88 | 88 | 88 98 | 98 | 98 | 98 107 | 98 | 98 | 98 108 | 108 | 108 | 108 \set ON_ERROR_STOP 0 SELECT time_bucket(10::smallint, '-32768'::smallint); ERROR: timestamp out of range SELECT time_bucket(10::smallint, '-32761'::smallint); ERROR: timestamp out of range select time_bucket(10::smallint, '-32768'::smallint, 1000::smallint); ERROR: timestamp out of range select time_bucket(10::smallint, '-32768'::smallint, '32767'::smallint); ERROR: timestamp out of range select time_bucket(10::smallint, '32767'::smallint, '-32768'::smallint); ERROR: timestamp out of range \set ON_ERROR_STOP 1 SELECT time, time_bucket(10::smallint, time) FROM unnest(ARRAY[ '-32760', '-32759', '32767' ]::smallint[]) AS time; time | time_bucket --------+------------- -32760 | -32760 -32759 | -32760 32767 | 32760 \set ON_ERROR_STOP 0 SELECT time_bucket(10::int, '-2147483648'::int); ERROR: timestamp out of range SELECT time_bucket(10::int, '-2147483641'::int); ERROR: timestamp out of range SELECT time_bucket(1000::int, '-2147483000'::int, 1::int); ERROR: timestamp out of range SELECT time_bucket(1000::int, '-2147483648'::int, '2147483647'::int); ERROR: timestamp out of range SELECT time_bucket(1000::int, '2147483647'::int, '-2147483648'::int); ERROR: timestamp out of range \set ON_ERROR_STOP 1 SELECT time, time_bucket(10::int, time) FROM unnest(ARRAY[ '-2147483640', '-2147483639', '2147483647' ]::int[]) AS time; time | time_bucket -------------+------------- -2147483640 | -2147483640 -2147483639 | -2147483640 2147483647 | 2147483640 \set ON_ERROR_STOP 0 SELECT time_bucket(10::bigint, '-9223372036854775808'::bigint); ERROR: timestamp out of range SELECT time_bucket(10::bigint, '-9223372036854775801'::bigint); ERROR: timestamp out of range SELECT time_bucket(1000::bigint, '-9223372036854775000'::bigint, 1::bigint); ERROR: timestamp out of range SELECT time_bucket(1000::bigint, '-9223372036854775808'::bigint, '9223372036854775807'::bigint); ERROR: timestamp out of range SELECT time_bucket(1000::bigint, '9223372036854775807'::bigint, '-9223372036854775808'::bigint); ERROR: timestamp out of range \set ON_ERROR_STOP 1 SELECT time, time_bucket(10::bigint, time) FROM unnest(ARRAY[ '-9223372036854775800', '-9223372036854775799', '9223372036854775807' ]::bigint[]) AS time; time | time_bucket ----------------------+---------------------- -9223372036854775800 | -9223372036854775800 -9223372036854775799 | -9223372036854775800 9223372036854775807 | 9223372036854775800 SELECT time, time_bucket(INTERVAL '1 day', time::date) FROM unnest(ARRAY[ date '2017-11-05', date '2017-11-06' ]) AS time; time | time_bucket ------------+------------- 11-05-2017 | 11-05-2017 11-06-2017 | 11-06-2017 SELECT time, time_bucket(INTERVAL '4 day', time::date) FROM unnest(ARRAY[ date '2017-11-04', date '2017-11-05', date '2017-11-08', date '2017-11-09' ]) AS time; time | time_bucket ------------+------------- 11-04-2017 | 11-01-2017 11-05-2017 | 11-05-2017 11-08-2017 | 11-05-2017 11-09-2017 | 11-09-2017 SELECT time, time_bucket(INTERVAL '4 day', time::date, INTERVAL '2 day') FROM unnest(ARRAY[ date '2017-11-06', date '2017-11-07', date '2017-11-10', date '2017-11-11' ]) AS time; time | time_bucket ------------+------------- 11-06-2017 | 11-03-2017 11-07-2017 | 11-07-2017 11-10-2017 | 11-07-2017 11-11-2017 | 11-11-2017 -- 2019-09-24 is a Monday, and we want to ensure that time_bucket returns the week starting with a Monday as date_trunc does, -- Rather than a Saturday which is the date of the PostgreSQL epoch SELECT time, time_bucket(INTERVAL '1 week', time::date) FROM unnest(ARRAY[ date '2018-09-16', date '2018-09-17', date '2018-09-23', date '2018-09-24' ]) AS time; time | time_bucket ------------+------------- 09-16-2018 | 09-10-2018 09-17-2018 | 09-17-2018 09-23-2018 | 09-17-2018 09-24-2018 | 09-24-2018 SELECT time, time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp without time zone '2018-09-16', timestamp without time zone '2018-09-17', timestamp without time zone '2018-09-23', timestamp without time zone '2018-09-24' ]) AS time; time | time_bucket --------------------------+-------------------------- Sun Sep 16 00:00:00 2018 | Mon Sep 10 00:00:00 2018 Mon Sep 17 00:00:00 2018 | Mon Sep 17 00:00:00 2018 Sun Sep 23 00:00:00 2018 | Mon Sep 17 00:00:00 2018 Mon Sep 24 00:00:00 2018 | Mon Sep 24 00:00:00 2018 SELECT time, time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp with time zone '2018-09-16', timestamp with time zone '2018-09-17', timestamp with time zone '2018-09-23', timestamp with time zone '2018-09-24' ]) AS time; time | time_bucket ------------------------------+------------------------------ Sun Sep 16 00:00:00 2018 EDT | Sun Sep 09 20:00:00 2018 EDT Mon Sep 17 00:00:00 2018 EDT | Sun Sep 16 20:00:00 2018 EDT Sun Sep 23 00:00:00 2018 EDT | Sun Sep 16 20:00:00 2018 EDT Mon Sep 24 00:00:00 2018 EDT | Sun Sep 23 20:00:00 2018 EDT SELECT time, time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp with time zone '-Infinity', timestamp with time zone 'Infinity' ]) AS time; time | time_bucket -----------+------------- -infinity | -infinity infinity | infinity SELECT time, time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp without time zone '-Infinity', timestamp without time zone 'Infinity' ]) AS time; time | time_bucket -----------+------------- -infinity | -infinity infinity | infinity SELECT time, time_bucket(INTERVAL '1 week', time), date_trunc('week', time) = time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp without time zone '4714-11-24 01:01:01.0 BC', timestamp without time zone '294276-12-31 23:59:59.9999' ]) AS time; time | time_bucket | ?column? ---------------------------------+-----------------------------+---------- Mon Nov 24 01:01:01 4714 BC | Mon Nov 24 00:00:00 4714 BC | t Sun Dec 31 23:59:59.9999 294276 | Mon Dec 25 00:00:00 294276 | t --1000 years later weeks still align. SELECT time, time_bucket(INTERVAL '1 week', time), date_trunc('week', time) = time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp without time zone '3018-09-14', timestamp without time zone '3018-09-20', timestamp without time zone '3018-09-21', timestamp without time zone '3018-09-22' ]) AS time; time | time_bucket | ?column? --------------------------+--------------------------+---------- Mon Sep 14 00:00:00 3018 | Mon Sep 14 00:00:00 3018 | t Sun Sep 20 00:00:00 3018 | Mon Sep 14 00:00:00 3018 | t Mon Sep 21 00:00:00 3018 | Mon Sep 21 00:00:00 3018 | t Tue Sep 22 00:00:00 3018 | Mon Sep 21 00:00:00 3018 | t --weeks align for timestamptz as well if cast to local time, (but not if done at UTC). SELECT time, date_trunc('week', time) = time_bucket(INTERVAL '1 week', time), date_trunc('week', time) = time_bucket(INTERVAL '1 week', time::timestamp) FROM unnest(ARRAY[ timestamp with time zone '3018-09-14', timestamp with time zone '3018-09-20', timestamp with time zone '3018-09-21', timestamp with time zone '3018-09-22' ]) AS time; time | ?column? | ?column? ------------------------------+----------+---------- Mon Sep 14 00:00:00 3018 EDT | f | t Sun Sep 20 00:00:00 3018 EDT | f | t Mon Sep 21 00:00:00 3018 EDT | f | t Tue Sep 22 00:00:00 3018 EDT | f | t --check functions with origin --note that the default origin is at 0 UTC, using origin parameter it is easy to provide a EDT origin point \x SELECT time, time_bucket(INTERVAL '1 week', time) no_epoch, time_bucket(INTERVAL '1 week', time::timestamp) no_epoch_local, time_bucket(INTERVAL '1 week', time) = time_bucket(INTERVAL '1 week', time, timestamptz '2000-01-03 00:00:00+0') always_true, time_bucket(INTERVAL '1 week', time, timestamptz '2000-01-01 00:00:00+0') pg_epoch, time_bucket(INTERVAL '1 week', time, timestamptz 'epoch') unix_epoch, time_bucket(INTERVAL '1 week', time, timestamptz '3018-09-13') custom_1, time_bucket(INTERVAL '1 week', time, timestamptz '3018-09-14') custom_2 FROM unnest(ARRAY[ timestamp with time zone '2000-01-01 00:00:00+0'- interval '1 second', timestamp with time zone '2000-01-01 00:00:00+0', timestamp with time zone '2000-01-03 00:00:00+0'- interval '1 second', timestamp with time zone '2000-01-03 00:00:00+0', timestamp with time zone '2000-01-01', timestamp with time zone '2000-01-02', timestamp with time zone '2000-01-03', timestamp with time zone '3018-09-12', timestamp with time zone '3018-09-13', timestamp with time zone '3018-09-14', timestamp with time zone '3018-09-15' ]) AS time; -[ RECORD 1 ]--+----------------------------- time | Fri Dec 31 18:59:59 1999 EST no_epoch | Sun Dec 26 19:00:00 1999 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 24 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Dec 25 23:00:00 1999 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 2 ]--+----------------------------- time | Fri Dec 31 19:00:00 1999 EST no_epoch | Sun Dec 26 19:00:00 1999 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Dec 25 23:00:00 1999 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 3 ]--+----------------------------- time | Sun Jan 02 18:59:59 2000 EST no_epoch | Sun Dec 26 19:00:00 1999 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Jan 01 23:00:00 2000 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 4 ]--+----------------------------- time | Sun Jan 02 19:00:00 2000 EST no_epoch | Sun Jan 02 19:00:00 2000 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Jan 01 23:00:00 2000 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 5 ]--+----------------------------- time | Sat Jan 01 00:00:00 2000 EST no_epoch | Sun Dec 26 19:00:00 1999 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Dec 25 23:00:00 1999 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 6 ]--+----------------------------- time | Sun Jan 02 00:00:00 2000 EST no_epoch | Sun Dec 26 19:00:00 1999 EST no_epoch_local | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Jan 01 23:00:00 2000 EST custom_2 | Sun Dec 26 23:00:00 1999 EST -[ RECORD 7 ]--+----------------------------- time | Mon Jan 03 00:00:00 2000 EST no_epoch | Sun Jan 02 19:00:00 2000 EST no_epoch_local | Mon Jan 03 00:00:00 2000 always_true | t pg_epoch | Fri Dec 31 19:00:00 1999 EST unix_epoch | Wed Dec 29 19:00:00 1999 EST custom_1 | Sat Jan 01 23:00:00 2000 EST custom_2 | Sun Jan 02 23:00:00 2000 EST -[ RECORD 8 ]--+----------------------------- time | Sat Sep 12 00:00:00 3018 EDT no_epoch | Sun Sep 06 20:00:00 3018 EDT no_epoch_local | Mon Sep 07 00:00:00 3018 always_true | t pg_epoch | Fri Sep 11 20:00:00 3018 EDT unix_epoch | Wed Sep 09 20:00:00 3018 EDT custom_1 | Sun Sep 06 00:00:00 3018 EDT custom_2 | Mon Sep 07 00:00:00 3018 EDT -[ RECORD 9 ]--+----------------------------- time | Sun Sep 13 00:00:00 3018 EDT no_epoch | Sun Sep 06 20:00:00 3018 EDT no_epoch_local | Mon Sep 07 00:00:00 3018 always_true | t pg_epoch | Fri Sep 11 20:00:00 3018 EDT unix_epoch | Wed Sep 09 20:00:00 3018 EDT custom_1 | Sun Sep 13 00:00:00 3018 EDT custom_2 | Mon Sep 07 00:00:00 3018 EDT -[ RECORD 10 ]-+----------------------------- time | Mon Sep 14 00:00:00 3018 EDT no_epoch | Sun Sep 13 20:00:00 3018 EDT no_epoch_local | Mon Sep 14 00:00:00 3018 always_true | t pg_epoch | Fri Sep 11 20:00:00 3018 EDT unix_epoch | Wed Sep 09 20:00:00 3018 EDT custom_1 | Sun Sep 13 00:00:00 3018 EDT custom_2 | Mon Sep 14 00:00:00 3018 EDT -[ RECORD 11 ]-+----------------------------- time | Tue Sep 15 00:00:00 3018 EDT no_epoch | Sun Sep 13 20:00:00 3018 EDT no_epoch_local | Mon Sep 14 00:00:00 3018 always_true | t pg_epoch | Fri Sep 11 20:00:00 3018 EDT unix_epoch | Wed Sep 09 20:00:00 3018 EDT custom_1 | Sun Sep 13 00:00:00 3018 EDT custom_2 | Mon Sep 14 00:00:00 3018 EDT SELECT time, time_bucket(INTERVAL '1 week', time) no_epoch, time_bucket(INTERVAL '1 week', time) = time_bucket(INTERVAL '1 week', time, timestamp '2000-01-03 00:00:00') always_true, time_bucket(INTERVAL '1 week', time, timestamp '2000-01-01 00:00:00+0') pg_epoch, time_bucket(INTERVAL '1 week', time, timestamp 'epoch') unix_epoch, time_bucket(INTERVAL '1 week', time, timestamp '3018-09-13') custom_1, time_bucket(INTERVAL '1 week', time, timestamp '3018-09-14') custom_2 FROM unnest(ARRAY[ timestamp without time zone '2000-01-01 00:00:00'- interval '1 second', timestamp without time zone '2000-01-01 00:00:00', timestamp without time zone '2000-01-03 00:00:00'- interval '1 second', timestamp without time zone '2000-01-03 00:00:00', timestamp without time zone '2000-01-01', timestamp without time zone '2000-01-02', timestamp without time zone '2000-01-03', timestamp without time zone '3018-09-12', timestamp without time zone '3018-09-13', timestamp without time zone '3018-09-14', timestamp without time zone '3018-09-15' ]) AS time; -[ RECORD 1 ]------------------------- time | Fri Dec 31 23:59:59 1999 no_epoch | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Sat Dec 25 00:00:00 1999 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Dec 26 00:00:00 1999 custom_2 | Mon Dec 27 00:00:00 1999 -[ RECORD 2 ]------------------------- time | Sat Jan 01 00:00:00 2000 no_epoch | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Dec 26 00:00:00 1999 custom_2 | Mon Dec 27 00:00:00 1999 -[ RECORD 3 ]------------------------- time | Sun Jan 02 23:59:59 2000 no_epoch | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Jan 02 00:00:00 2000 custom_2 | Mon Dec 27 00:00:00 1999 -[ RECORD 4 ]------------------------- time | Mon Jan 03 00:00:00 2000 no_epoch | Mon Jan 03 00:00:00 2000 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Jan 02 00:00:00 2000 custom_2 | Mon Jan 03 00:00:00 2000 -[ RECORD 5 ]------------------------- time | Sat Jan 01 00:00:00 2000 no_epoch | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Dec 26 00:00:00 1999 custom_2 | Mon Dec 27 00:00:00 1999 -[ RECORD 6 ]------------------------- time | Sun Jan 02 00:00:00 2000 no_epoch | Mon Dec 27 00:00:00 1999 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Jan 02 00:00:00 2000 custom_2 | Mon Dec 27 00:00:00 1999 -[ RECORD 7 ]------------------------- time | Mon Jan 03 00:00:00 2000 no_epoch | Mon Jan 03 00:00:00 2000 always_true | t pg_epoch | Sat Jan 01 00:00:00 2000 unix_epoch | Thu Dec 30 00:00:00 1999 custom_1 | Sun Jan 02 00:00:00 2000 custom_2 | Mon Jan 03 00:00:00 2000 -[ RECORD 8 ]------------------------- time | Sat Sep 12 00:00:00 3018 no_epoch | Mon Sep 07 00:00:00 3018 always_true | t pg_epoch | Sat Sep 12 00:00:00 3018 unix_epoch | Thu Sep 10 00:00:00 3018 custom_1 | Sun Sep 06 00:00:00 3018 custom_2 | Mon Sep 07 00:00:00 3018 -[ RECORD 9 ]------------------------- time | Sun Sep 13 00:00:00 3018 no_epoch | Mon Sep 07 00:00:00 3018 always_true | t pg_epoch | Sat Sep 12 00:00:00 3018 unix_epoch | Thu Sep 10 00:00:00 3018 custom_1 | Sun Sep 13 00:00:00 3018 custom_2 | Mon Sep 07 00:00:00 3018 -[ RECORD 10 ]------------------------ time | Mon Sep 14 00:00:00 3018 no_epoch | Mon Sep 14 00:00:00 3018 always_true | t pg_epoch | Sat Sep 12 00:00:00 3018 unix_epoch | Thu Sep 10 00:00:00 3018 custom_1 | Sun Sep 13 00:00:00 3018 custom_2 | Mon Sep 14 00:00:00 3018 -[ RECORD 11 ]------------------------ time | Tue Sep 15 00:00:00 3018 no_epoch | Mon Sep 14 00:00:00 3018 always_true | t pg_epoch | Sat Sep 12 00:00:00 3018 unix_epoch | Thu Sep 10 00:00:00 3018 custom_1 | Sun Sep 13 00:00:00 3018 custom_2 | Mon Sep 14 00:00:00 3018 SELECT time, time_bucket(INTERVAL '1 week', time) no_epoch, time_bucket(INTERVAL '1 week', time) = time_bucket(INTERVAL '1 week', time, date '2000-01-03') always_true, time_bucket(INTERVAL '1 week', time, date '2000-01-01') pg_epoch, time_bucket(INTERVAL '1 week', time, (timestamp 'epoch')::date) unix_epoch, time_bucket(INTERVAL '1 week', time, date '3018-09-13') custom_1, time_bucket(INTERVAL '1 week', time, date '3018-09-14') custom_2 FROM unnest(ARRAY[ date '1999-12-31', date '2000-01-01', date '2000-01-02', date '2000-01-03', date '3018-09-12', date '3018-09-13', date '3018-09-14', date '3018-09-15' ]) AS time; -[ RECORD 1 ]----------- time | 12-31-1999 no_epoch | 12-27-1999 always_true | t pg_epoch | 12-25-1999 unix_epoch | 12-30-1999 custom_1 | 12-26-1999 custom_2 | 12-27-1999 -[ RECORD 2 ]----------- time | 01-01-2000 no_epoch | 12-27-1999 always_true | t pg_epoch | 01-01-2000 unix_epoch | 12-30-1999 custom_1 | 12-26-1999 custom_2 | 12-27-1999 -[ RECORD 3 ]----------- time | 01-02-2000 no_epoch | 12-27-1999 always_true | t pg_epoch | 01-01-2000 unix_epoch | 12-30-1999 custom_1 | 01-02-2000 custom_2 | 12-27-1999 -[ RECORD 4 ]----------- time | 01-03-2000 no_epoch | 01-03-2000 always_true | t pg_epoch | 01-01-2000 unix_epoch | 12-30-1999 custom_1 | 01-02-2000 custom_2 | 01-03-2000 -[ RECORD 5 ]----------- time | 09-12-3018 no_epoch | 09-07-3018 always_true | t pg_epoch | 09-12-3018 unix_epoch | 09-10-3018 custom_1 | 09-06-3018 custom_2 | 09-07-3018 -[ RECORD 6 ]----------- time | 09-13-3018 no_epoch | 09-07-3018 always_true | t pg_epoch | 09-12-3018 unix_epoch | 09-10-3018 custom_1 | 09-13-3018 custom_2 | 09-07-3018 -[ RECORD 7 ]----------- time | 09-14-3018 no_epoch | 09-14-3018 always_true | t pg_epoch | 09-12-3018 unix_epoch | 09-10-3018 custom_1 | 09-13-3018 custom_2 | 09-14-3018 -[ RECORD 8 ]----------- time | 09-15-3018 no_epoch | 09-14-3018 always_true | t pg_epoch | 09-12-3018 unix_epoch | 09-10-3018 custom_1 | 09-13-3018 custom_2 | 09-14-3018 \x --really old origin works if date around that time SELECT time, time_bucket(INTERVAL '1 week', time, timestamp without time zone '4710-11-24 01:01:01.0 BC') FROM unnest(ARRAY[ timestamp without time zone '4710-11-24 01:01:01.0 BC', timestamp without time zone '4710-11-25 01:01:01.0 BC', timestamp without time zone '2001-01-01', timestamp without time zone '3001-01-01' ]) AS time; time | time_bucket -----------------------------+----------------------------- Sat Nov 24 01:01:01 4710 BC | Sat Nov 24 01:01:01 4710 BC Sun Nov 25 01:01:01 4710 BC | Sat Nov 24 01:01:01 4710 BC Mon Jan 01 00:00:00 2001 | Sat Dec 30 01:01:01 2000 Thu Jan 01 00:00:00 3001 | Sat Dec 27 01:01:01 3000 SELECT time, time_bucket(INTERVAL '1 week', time, timestamp without time zone '294270-12-30 23:59:59.9999') FROM unnest(ARRAY[ timestamp without time zone '294270-12-29 23:59:59.9999', timestamp without time zone '294270-12-30 23:59:59.9999', timestamp without time zone '294270-12-31 23:59:59.9999', timestamp without time zone '2001-01-01', timestamp without time zone '3001-01-01' ]) AS time; time | time_bucket ---------------------------------+--------------------------------- Thu Dec 29 23:59:59.9999 294270 | Fri Dec 23 23:59:59.9999 294270 Fri Dec 30 23:59:59.9999 294270 | Fri Dec 30 23:59:59.9999 294270 Sat Dec 31 23:59:59.9999 294270 | Fri Dec 30 23:59:59.9999 294270 Mon Jan 01 00:00:00 2001 | Fri Dec 29 23:59:59.9999 2000 Thu Jan 01 00:00:00 3001 | Fri Dec 26 23:59:59.9999 3000 \set ON_ERROR_STOP 0 --really old origin + very new data + long period errors SELECT time, time_bucket(INTERVAL '100000 day', time, timestamp without time zone '4710-11-24 01:01:01.0 BC') FROM unnest(ARRAY[ timestamp without time zone '294270-12-31 23:59:59.9999' ]) AS time; ERROR: timestamp out of range SELECT time, time_bucket(INTERVAL '100000 day', time, timestamp with time zone '4710-11-25 01:01:01.0 BC') FROM unnest(ARRAY[ timestamp with time zone '294270-12-30 23:59:59.9999' ]) AS time; ERROR: timestamp out of range --really high origin + old data + long period errors out SELECT time, time_bucket(INTERVAL '10000000 day', time, timestamp without time zone '294270-12-31 23:59:59.9999') FROM unnest(ARRAY[ timestamp without time zone '4710-11-24 01:01:01.0 BC' ]) AS time; ERROR: timestamp out of range SELECT time, time_bucket(INTERVAL '10000000 day', time, timestamp with time zone '294270-12-31 23:59:59.9999') FROM unnest(ARRAY[ timestamp with time zone '4710-11-24 01:01:01.0 BC' ]) AS time; ERROR: timestamp out of range \set ON_ERROR_STOP 1 ------------------------------------------- --- Test time_bucket with month periods --- ------------------------------------------- SET datestyle TO ISO; SELECT time::date, time_bucket('1 month', time::date) AS "1m", time_bucket('2 month', time::date) AS "2m", time_bucket('3 month', time::date) AS "3m", time_bucket('1 month', time::date, '2000-02-01'::date) AS "1m origin", time_bucket('2 month', time::date, '2000-02-01'::date) AS "2m origin", time_bucket('3 month', time::date, '2000-02-01'::date) AS "3m origin" FROM generate_series('1990-01-03'::date,'1990-06-03'::date,'1month'::interval) time; time | 1m | 2m | 3m | 1m origin | 2m origin | 3m origin ------------+------------+------------+------------+------------+------------+------------ 1990-01-03 | 1990-01-01 | 1990-01-01 | 1990-01-01 | 1990-01-01 | 1989-12-01 | 1989-11-01 1990-02-03 | 1990-02-01 | 1990-01-01 | 1990-01-01 | 1990-02-01 | 1990-02-01 | 1990-02-01 1990-03-03 | 1990-03-01 | 1990-03-01 | 1990-01-01 | 1990-03-01 | 1990-02-01 | 1990-02-01 1990-04-03 | 1990-04-01 | 1990-03-01 | 1990-04-01 | 1990-04-01 | 1990-04-01 | 1990-02-01 1990-05-03 | 1990-05-01 | 1990-05-01 | 1990-04-01 | 1990-05-01 | 1990-04-01 | 1990-05-01 1990-06-03 | 1990-06-01 | 1990-05-01 | 1990-04-01 | 1990-06-01 | 1990-06-01 | 1990-05-01 SELECT time, time_bucket('1 month', time) AS "1m", time_bucket('2 month', time) AS "2m", time_bucket('3 month', time) AS "3m", time_bucket('1 month', time, '2000-02-01'::timestamp) AS "1m origin", time_bucket('2 month', time, '2000-02-01'::timestamp) AS "2m origin", time_bucket('3 month', time, '2000-02-01'::timestamp) AS "3m origin" FROM generate_series('1990-01-03'::timestamp,'1990-06-03'::timestamp,'1month'::interval) time; time | 1m | 2m | 3m | 1m origin | 2m origin | 3m origin ---------------------+---------------------+---------------------+---------------------+---------------------+---------------------+--------------------- 1990-01-03 00:00:00 | 1990-01-01 00:00:00 | 1990-01-01 00:00:00 | 1990-01-01 00:00:00 | 1990-01-01 00:00:00 | 1989-12-01 00:00:00 | 1989-11-01 00:00:00 1990-02-03 00:00:00 | 1990-02-01 00:00:00 | 1990-01-01 00:00:00 | 1990-01-01 00:00:00 | 1990-02-01 00:00:00 | 1990-02-01 00:00:00 | 1990-02-01 00:00:00 1990-03-03 00:00:00 | 1990-03-01 00:00:00 | 1990-03-01 00:00:00 | 1990-01-01 00:00:00 | 1990-03-01 00:00:00 | 1990-02-01 00:00:00 | 1990-02-01 00:00:00 1990-04-03 00:00:00 | 1990-04-01 00:00:00 | 1990-03-01 00:00:00 | 1990-04-01 00:00:00 | 1990-04-01 00:00:00 | 1990-04-01 00:00:00 | 1990-02-01 00:00:00 1990-05-03 00:00:00 | 1990-05-01 00:00:00 | 1990-05-01 00:00:00 | 1990-04-01 00:00:00 | 1990-05-01 00:00:00 | 1990-04-01 00:00:00 | 1990-05-01 00:00:00 1990-06-03 00:00:00 | 1990-06-01 00:00:00 | 1990-05-01 00:00:00 | 1990-04-01 00:00:00 | 1990-06-01 00:00:00 | 1990-06-01 00:00:00 | 1990-05-01 00:00:00 SELECT time, time_bucket('1 month', time) AS "1m", time_bucket('2 month', time) AS "2m", time_bucket('3 month', time) AS "3m", time_bucket('1 month', time, '2000-02-01'::timestamptz) AS "1m origin", time_bucket('2 month', time, '2000-02-01'::timestamptz) AS "2m origin", time_bucket('3 month', time, '2000-02-01'::timestamptz) AS "3m origin" FROM generate_series('1990-01-03'::timestamptz,'1990-06-03'::timestamptz,'1month'::interval) time; time | 1m | 2m | 3m | 1m origin | 2m origin | 3m origin ------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+------------------------ 1990-01-03 00:00:00-05 | 1989-12-31 19:00:00-05 | 1989-12-31 19:00:00-05 | 1989-12-31 19:00:00-05 | 1989-12-31 19:00:00-05 | 1989-11-30 19:00:00-05 | 1989-10-31 19:00:00-05 1990-02-03 00:00:00-05 | 1990-01-31 19:00:00-05 | 1989-12-31 19:00:00-05 | 1989-12-31 19:00:00-05 | 1990-01-31 19:00:00-05 | 1990-01-31 19:00:00-05 | 1990-01-31 19:00:00-05 1990-03-03 00:00:00-05 | 1990-02-28 19:00:00-05 | 1990-02-28 19:00:00-05 | 1989-12-31 19:00:00-05 | 1990-02-28 19:00:00-05 | 1990-01-31 19:00:00-05 | 1990-01-31 19:00:00-05 1990-04-03 00:00:00-04 | 1990-03-31 19:00:00-05 | 1990-02-28 19:00:00-05 | 1990-03-31 19:00:00-05 | 1990-03-31 19:00:00-05 | 1990-03-31 19:00:00-05 | 1990-01-31 19:00:00-05 1990-05-03 00:00:00-04 | 1990-04-30 20:00:00-04 | 1990-04-30 20:00:00-04 | 1990-03-31 19:00:00-05 | 1990-04-30 20:00:00-04 | 1990-03-31 19:00:00-05 | 1990-04-30 20:00:00-04 1990-06-03 00:00:00-04 | 1990-05-31 20:00:00-04 | 1990-04-30 20:00:00-04 | 1990-03-31 19:00:00-05 | 1990-05-31 20:00:00-04 | 1990-05-31 20:00:00-04 | 1990-04-30 20:00:00-04 --------------------------------------- --- Test time_bucket with timezones --- --------------------------------------- -- test NULL args SELECT time_bucket(NULL::interval,now(),'Europe/Berlin'), time_bucket('1day',NULL::timestamptz,'Europe/Berlin'), time_bucket('1day',now(),NULL::text), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin',NULL), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin','2020-04-01',NULL), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin',NULL,NULL), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin',"offset":=NULL::interval), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin',origin:=NULL::timestamptz); time_bucket | time_bucket | time_bucket | time_bucket | time_bucket | time_bucket | time_bucket | time_bucket -------------+-------------+-------------+------------------------+------------------------+------------------------+------------------------+------------------------ | | | 2020-02-02 18:00:00-05 | 2020-02-03 00:00:00-05 | 2020-02-02 18:00:00-05 | 2020-02-02 18:00:00-05 | 2020-02-02 18:00:00-05 SET datestyle TO ISO; SELECT time_bucket('1day', ts) AS "UTC", time_bucket('1day', ts, 'Europe/Berlin') AS "Berlin", time_bucket('1day', ts, 'Europe/London') AS "London", time_bucket('1day', ts, 'America/New_York') AS "New York", time_bucket('1day', ts, 'PST') AS "PST", time_bucket('1day', ts, current_setting('timezone')) AS "current" FROM generate_series('1999-12-31 17:00'::timestamptz,'2000-01-02 3:00'::timestamptz, '1hour'::interval) ts; UTC | Berlin | London | New York | PST | current ------------------------+------------------------+------------------------+------------------------+------------------------+------------------------ 1999-12-30 19:00:00-05 | 1999-12-30 18:00:00-05 | 1999-12-30 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-30 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-30 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 1999-12-31 00:00:00-05 | 1999-12-31 03:00:00-05 | 1999-12-31 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-31 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-31 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-31 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 1999-12-31 19:00:00-05 | 2000-01-01 18:00:00-05 | 1999-12-31 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-01 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-02 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-02 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-02 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-02 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-02 00:00:00-05 | 2000-01-01 03:00:00-05 | 2000-01-02 00:00:00-05 2000-01-01 19:00:00-05 | 2000-01-01 18:00:00-05 | 2000-01-01 19:00:00-05 | 2000-01-02 00:00:00-05 | 2000-01-02 03:00:00-05 | 2000-01-02 00:00:00-05 SELECT time_bucket('1month', ts) AS "UTC", time_bucket('1month', ts, 'Europe/Berlin') AS "Berlin", time_bucket('1month', ts, 'America/New_York') AS "New York", time_bucket('1month', ts, current_setting('timezone')) AS "current", time_bucket('2month', ts, current_setting('timezone')) AS "2m", time_bucket('2month', ts, current_setting('timezone'), '2000-02-01'::timestamp) AS "2m origin", time_bucket('2month', ts, current_setting('timezone'), "offset":='14 day'::interval) AS "2m offset", time_bucket('2month', ts, current_setting('timezone'), '2000-02-01'::timestamp, '7 day'::interval) AS "2m offset + origin" FROM generate_series('1999-12-01'::timestamptz,'2000-09-01'::timestamptz, '9 day'::interval) ts; UTC | Berlin | New York | current | 2m | 2m origin | 2m offset | 2m offset + origin ------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+------------------------+------------------------ 1999-11-30 19:00:00-05 | 1999-11-30 18:00:00-05 | 1999-12-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-15 00:00:00-05 | 1999-10-08 00:00:00-04 1999-11-30 19:00:00-05 | 1999-11-30 18:00:00-05 | 1999-12-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-15 00:00:00-05 | 1999-12-08 00:00:00-05 1999-11-30 19:00:00-05 | 1999-11-30 18:00:00-05 | 1999-12-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-15 00:00:00-05 | 1999-12-08 00:00:00-05 1999-11-30 19:00:00-05 | 1999-11-30 18:00:00-05 | 1999-12-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-15 00:00:00-05 | 1999-12-08 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 1999-11-15 00:00:00-05 | 1999-12-08 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 1999-12-08 00:00:00-05 1999-12-31 19:00:00-05 | 1999-12-31 18:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 1999-12-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 1999-12-08 00:00:00-05 2000-01-31 19:00:00-05 | 2000-01-31 18:00:00-05 | 2000-02-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 1999-12-08 00:00:00-05 2000-01-31 19:00:00-05 | 2000-01-31 18:00:00-05 | 2000-02-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-01-31 19:00:00-05 | 2000-01-31 18:00:00-05 | 2000-02-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-01-31 19:00:00-05 | 2000-01-31 18:00:00-05 | 2000-02-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-02-29 19:00:00-05 | 2000-02-29 18:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-01-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-02-29 19:00:00-05 | 2000-02-29 18:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-02-29 19:00:00-05 | 2000-02-29 18:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-02-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-03-31 19:00:00-05 | 2000-03-31 17:00:00-05 | 2000-04-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-02-08 00:00:00-05 2000-03-31 19:00:00-05 | 2000-03-31 17:00:00-05 | 2000-04-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-04-08 00:00:00-04 2000-03-31 19:00:00-05 | 2000-03-31 17:00:00-05 | 2000-04-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-01 00:00:00-05 | 2000-04-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-04-08 00:00:00-04 2000-04-30 20:00:00-04 | 2000-04-30 18:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-04-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-04-08 00:00:00-04 2000-04-30 20:00:00-04 | 2000-04-30 18:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-04-01 00:00:00-05 | 2000-03-15 00:00:00-05 | 2000-04-08 00:00:00-04 2000-04-30 20:00:00-04 | 2000-04-30 18:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-04-01 00:00:00-05 | 2000-05-15 00:00:00-04 | 2000-04-08 00:00:00-04 2000-04-30 20:00:00-04 | 2000-04-30 18:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-04-01 00:00:00-05 | 2000-05-15 00:00:00-04 | 2000-04-08 00:00:00-04 2000-05-31 20:00:00-04 | 2000-05-31 18:00:00-04 | 2000-06-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-15 00:00:00-04 | 2000-04-08 00:00:00-04 2000-05-31 20:00:00-04 | 2000-05-31 18:00:00-04 | 2000-06-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-05-31 20:00:00-04 | 2000-05-31 18:00:00-04 | 2000-06-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-06-30 20:00:00-04 | 2000-06-30 18:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-06-30 20:00:00-04 | 2000-06-30 18:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-05-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-06-30 20:00:00-04 | 2000-06-30 18:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-07-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-06-30 20:00:00-04 | 2000-06-30 18:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-06-01 00:00:00-04 | 2000-07-15 00:00:00-04 | 2000-06-08 00:00:00-04 2000-07-31 20:00:00-04 | 2000-07-31 18:00:00-04 | 2000-08-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-15 00:00:00-04 | 2000-08-08 00:00:00-04 2000-07-31 20:00:00-04 | 2000-07-31 18:00:00-04 | 2000-08-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-15 00:00:00-04 | 2000-08-08 00:00:00-04 2000-07-31 20:00:00-04 | 2000-07-31 18:00:00-04 | 2000-08-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-01 00:00:00-04 | 2000-08-01 00:00:00-04 | 2000-07-15 00:00:00-04 | 2000-08-08 00:00:00-04 RESET datestyle; ------------------------------------- --- Test time input functions -- ------------------------------------- \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION test.interval_to_internal(coltype REGTYPE, value ANYELEMENT = NULL::BIGINT) RETURNS BIGINT AS :MODULE_PATHNAME, 'ts_dimension_interval_to_internal_test' LANGUAGE C VOLATILE; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT test.interval_to_internal('TIMESTAMP'::regtype, INTERVAL '1 day'); interval_to_internal ---------------------- 86400000000 SELECT test.interval_to_internal('TIMESTAMP'::regtype, 86400000000); interval_to_internal ---------------------- 86400000000 ---should give warning SELECT test.interval_to_internal('TIMESTAMP'::regtype, 86400); WARNING: unexpected interval: smaller than one second HINT: The interval is specified in microseconds. interval_to_internal ---------------------- 86400 SELECT test.interval_to_internal('TIMESTAMP'::regtype); interval_to_internal ---------------------- 604800000000 SELECT test.interval_to_internal('BIGINT'::regtype, 2147483649::bigint); interval_to_internal ---------------------- 2147483649 -- Default interval for integer is supported as part of -- hypertable generalization SELECT test.interval_to_internal('INT'::regtype); interval_to_internal ---------------------- 100000 SELECT test.interval_to_internal('SMALLINT'::regtype); interval_to_internal ---------------------- 10000 SELECT test.interval_to_internal('BIGINT'::regtype); interval_to_internal ---------------------- 1000000 SELECT test.interval_to_internal('TIMESTAMPTZ'::regtype); interval_to_internal ---------------------- 604800000000 SELECT test.interval_to_internal('TIMESTAMP'::regtype); interval_to_internal ---------------------- 604800000000 SELECT test.interval_to_internal('DATE'::regtype); interval_to_internal ---------------------- 604800000000 \set VERBOSITY terse \set ON_ERROR_STOP 0 SELECT test.interval_to_internal('INT'::regtype, 2147483649::bigint); ERROR: invalid interval: must be between 1 and 2147483647 SELECT test.interval_to_internal('SMALLINT'::regtype, 32768::bigint); ERROR: invalid interval: must be between 1 and 32767 SELECT test.interval_to_internal('TEXT'::regtype, 32768::bigint); ERROR: invalid type for dimension "testcol" SELECT test.interval_to_internal('INT'::regtype, INTERVAL '1 day'); ERROR: invalid interval type for integer dimension \set ON_ERROR_STOP 1 ================================================ FILE: test/expected/triggers.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE hyper ( time BIGINT NOT NULL, device_id TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 ); CREATE OR REPLACE FUNCTION test_trigger() RETURNS TRIGGER LANGUAGE PLPGSQL AS $BODY$ DECLARE cnt INTEGER; BEGIN SELECT count(*) INTO cnt FROM hyper; RAISE WARNING 'FIRING trigger when: % level: % op: % cnt: % trigger_name %', tg_when, tg_level, tg_op, cnt, tg_name; IF TG_OP = 'DELETE' THEN RETURN OLD; END IF; RETURN NEW; END $BODY$; -- row triggers: BEFORE CREATE TRIGGER _0_test_trigger_insert BEFORE INSERT ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_update BEFORE UPDATE ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_delete BEFORE delete ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER z_test_trigger_all BEFORE INSERT OR UPDATE OR DELETE ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); -- row triggers: AFTER CREATE TRIGGER _0_test_trigger_insert_after AFTER INSERT ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_insert_after_when_dev1 AFTER INSERT ON hyper FOR EACH ROW WHEN (NEW.device_id = 'dev1') EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_update_after AFTER UPDATE ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_delete_after AFTER delete ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER z_test_trigger_all_after AFTER INSERT OR UPDATE OR DELETE ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); -- statement triggers: BEFORE CREATE TRIGGER _0_test_trigger_insert_s_before BEFORE INSERT ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_update_s_before BEFORE UPDATE ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_delete_s_before BEFORE DELETE ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); -- statement triggers: AFTER CREATE TRIGGER _0_test_trigger_insert_s_after AFTER INSERT ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_update_s_after AFTER UPDATE ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_delete_s_after AFTER DELETE ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); -- CONSTRAINT TRIGGER CREATE CONSTRAINT TRIGGER _0_test_trigger_constraint_insert AFTER INSERT ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE CONSTRAINT TRIGGER _0_test_trigger_constraint_update AFTER UPDATE ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE CONSTRAINT TRIGGER _0_test_trigger_constraint_delete AFTER DELETE ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); SELECT * FROM create_hypertable('hyper', 'time', chunk_time_interval => 10); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | hyper | t --test triggers before create_hypertable INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987600000000000, 'dev1', 1); WARNING: FIRING trigger when: BEFORE level: STATEMENT op: INSERT cnt: 0 trigger_name _0_test_trigger_insert_s_before WARNING: FIRING trigger when: BEFORE level: ROW op: INSERT cnt: 0 trigger_name _0_test_trigger_insert WARNING: FIRING trigger when: BEFORE level: ROW op: INSERT cnt: 0 trigger_name z_test_trigger_all WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 1 trigger_name _0_test_trigger_constraint_insert WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 1 trigger_name _0_test_trigger_insert_after WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 1 trigger_name _0_test_trigger_insert_after_when_dev1 WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 1 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: STATEMENT op: INSERT cnt: 1 trigger_name _0_test_trigger_insert_s_after INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 1), (1257987800000000000, 'dev2', 1); WARNING: FIRING trigger when: BEFORE level: STATEMENT op: INSERT cnt: 1 trigger_name _0_test_trigger_insert_s_before WARNING: FIRING trigger when: BEFORE level: ROW op: INSERT cnt: 1 trigger_name _0_test_trigger_insert WARNING: FIRING trigger when: BEFORE level: ROW op: INSERT cnt: 1 trigger_name z_test_trigger_all WARNING: FIRING trigger when: BEFORE level: ROW op: INSERT cnt: 2 trigger_name _0_test_trigger_insert WARNING: FIRING trigger when: BEFORE level: ROW op: INSERT cnt: 2 trigger_name z_test_trigger_all WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 3 trigger_name _0_test_trigger_constraint_insert WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 3 trigger_name _0_test_trigger_insert_after WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 3 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 3 trigger_name _0_test_trigger_constraint_insert WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 3 trigger_name _0_test_trigger_insert_after WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 3 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: STATEMENT op: INSERT cnt: 3 trigger_name _0_test_trigger_insert_s_after UPDATE hyper SET sensor_1 = 2; WARNING: FIRING trigger when: BEFORE level: STATEMENT op: UPDATE cnt: 3 trigger_name _0_test_trigger_update_s_before WARNING: FIRING trigger when: BEFORE level: ROW op: UPDATE cnt: 3 trigger_name _0_test_trigger_update WARNING: FIRING trigger when: BEFORE level: ROW op: UPDATE cnt: 3 trigger_name z_test_trigger_all WARNING: FIRING trigger when: BEFORE level: ROW op: UPDATE cnt: 3 trigger_name _0_test_trigger_update WARNING: FIRING trigger when: BEFORE level: ROW op: UPDATE cnt: 3 trigger_name z_test_trigger_all WARNING: FIRING trigger when: BEFORE level: ROW op: UPDATE cnt: 3 trigger_name _0_test_trigger_update WARNING: FIRING trigger when: BEFORE level: ROW op: UPDATE cnt: 3 trigger_name z_test_trigger_all WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name _0_test_trigger_constraint_update WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name _0_test_trigger_update_after WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name _0_test_trigger_constraint_update WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name _0_test_trigger_update_after WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name _0_test_trigger_constraint_update WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name _0_test_trigger_update_after WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: STATEMENT op: UPDATE cnt: 3 trigger_name _0_test_trigger_update_s_after DELETE FROM hyper; WARNING: FIRING trigger when: BEFORE level: STATEMENT op: DELETE cnt: 3 trigger_name _0_test_trigger_delete_s_before WARNING: FIRING trigger when: BEFORE level: ROW op: DELETE cnt: 3 trigger_name _0_test_trigger_delete WARNING: FIRING trigger when: BEFORE level: ROW op: DELETE cnt: 3 trigger_name z_test_trigger_all WARNING: FIRING trigger when: BEFORE level: ROW op: DELETE cnt: 2 trigger_name _0_test_trigger_delete WARNING: FIRING trigger when: BEFORE level: ROW op: DELETE cnt: 2 trigger_name z_test_trigger_all WARNING: FIRING trigger when: BEFORE level: ROW op: DELETE cnt: 1 trigger_name _0_test_trigger_delete WARNING: FIRING trigger when: BEFORE level: ROW op: DELETE cnt: 1 trigger_name z_test_trigger_all WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name _0_test_trigger_constraint_delete WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name _0_test_trigger_delete_after WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name _0_test_trigger_constraint_delete WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name _0_test_trigger_delete_after WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name _0_test_trigger_constraint_delete WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name _0_test_trigger_delete_after WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: STATEMENT op: DELETE cnt: 0 trigger_name _0_test_trigger_delete_s_after --test drop trigger DROP TRIGGER _0_test_trigger_insert ON hyper; DROP TRIGGER _0_test_trigger_insert_s_before ON hyper; DROP TRIGGER _0_test_trigger_insert_after ON hyper; DROP TRIGGER _0_test_trigger_insert_s_after ON hyper; INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987600000000000, 'dev1', 1); WARNING: FIRING trigger when: BEFORE level: ROW op: INSERT cnt: 0 trigger_name z_test_trigger_all WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 1 trigger_name _0_test_trigger_constraint_insert WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 1 trigger_name _0_test_trigger_insert_after_when_dev1 WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 1 trigger_name z_test_trigger_all_after INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 1), (1257987800000000000, 'dev2', 1); WARNING: FIRING trigger when: BEFORE level: ROW op: INSERT cnt: 1 trigger_name z_test_trigger_all WARNING: FIRING trigger when: BEFORE level: ROW op: INSERT cnt: 2 trigger_name z_test_trigger_all WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 3 trigger_name _0_test_trigger_constraint_insert WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 3 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 3 trigger_name _0_test_trigger_constraint_insert WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 3 trigger_name z_test_trigger_all_after DROP TRIGGER _0_test_trigger_update ON hyper; DROP TRIGGER _0_test_trigger_update_s_before ON hyper; DROP TRIGGER _0_test_trigger_update_after ON hyper; DROP TRIGGER _0_test_trigger_update_s_after ON hyper; UPDATE hyper SET sensor_1 = 2; WARNING: FIRING trigger when: BEFORE level: ROW op: UPDATE cnt: 3 trigger_name z_test_trigger_all WARNING: FIRING trigger when: BEFORE level: ROW op: UPDATE cnt: 3 trigger_name z_test_trigger_all WARNING: FIRING trigger when: BEFORE level: ROW op: UPDATE cnt: 3 trigger_name z_test_trigger_all WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name _0_test_trigger_constraint_update WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name _0_test_trigger_constraint_update WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name _0_test_trigger_constraint_update WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name z_test_trigger_all_after DROP TRIGGER _0_test_trigger_delete ON hyper; DROP TRIGGER _0_test_trigger_delete_s_before ON hyper; DROP TRIGGER _0_test_trigger_delete_after ON hyper; DROP TRIGGER _0_test_trigger_delete_s_after ON hyper; DELETE FROM hyper; WARNING: FIRING trigger when: BEFORE level: ROW op: DELETE cnt: 3 trigger_name z_test_trigger_all WARNING: FIRING trigger when: BEFORE level: ROW op: DELETE cnt: 2 trigger_name z_test_trigger_all WARNING: FIRING trigger when: BEFORE level: ROW op: DELETE cnt: 1 trigger_name z_test_trigger_all WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name _0_test_trigger_constraint_delete WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name _0_test_trigger_constraint_delete WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name _0_test_trigger_constraint_delete WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name z_test_trigger_all_after DROP TRIGGER z_test_trigger_all ON hyper; DROP TRIGGER z_test_trigger_all_after ON hyper; --test create trigger on hypertable -- row triggers: BEFORE CREATE TRIGGER _0_test_trigger_insert BEFORE INSERT ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_update BEFORE UPDATE ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_delete BEFORE delete ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER z_test_trigger_all BEFORE INSERT OR UPDATE OR DELETE ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); -- row triggers: AFTER CREATE TRIGGER _0_test_trigger_insert_after AFTER INSERT ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_update_after AFTER UPDATE ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_delete_after AFTER delete ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER z_test_trigger_all_after AFTER INSERT OR UPDATE OR DELETE ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); -- statement triggers: BEFORE CREATE TRIGGER _0_test_trigger_insert_s_before BEFORE INSERT ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_update_s_before BEFORE UPDATE ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_delete_s_before BEFORE DELETE ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); -- statement triggers: AFTER CREATE TRIGGER _0_test_trigger_insert_s_after AFTER INSERT ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_update_s_after AFTER UPDATE ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_delete_s_after AFTER DELETE ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987600000000000, 'dev1', 1); WARNING: FIRING trigger when: BEFORE level: STATEMENT op: INSERT cnt: 0 trigger_name _0_test_trigger_insert_s_before WARNING: FIRING trigger when: BEFORE level: ROW op: INSERT cnt: 0 trigger_name _0_test_trigger_insert WARNING: FIRING trigger when: BEFORE level: ROW op: INSERT cnt: 0 trigger_name z_test_trigger_all WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 1 trigger_name _0_test_trigger_constraint_insert WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 1 trigger_name _0_test_trigger_insert_after WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 1 trigger_name _0_test_trigger_insert_after_when_dev1 WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 1 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: STATEMENT op: INSERT cnt: 1 trigger_name _0_test_trigger_insert_s_after INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 1), (1257987800000000000, 'dev2', 1); WARNING: FIRING trigger when: BEFORE level: STATEMENT op: INSERT cnt: 1 trigger_name _0_test_trigger_insert_s_before WARNING: FIRING trigger when: BEFORE level: ROW op: INSERT cnt: 1 trigger_name _0_test_trigger_insert WARNING: FIRING trigger when: BEFORE level: ROW op: INSERT cnt: 1 trigger_name z_test_trigger_all WARNING: FIRING trigger when: BEFORE level: ROW op: INSERT cnt: 2 trigger_name _0_test_trigger_insert WARNING: FIRING trigger when: BEFORE level: ROW op: INSERT cnt: 2 trigger_name z_test_trigger_all WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 3 trigger_name _0_test_trigger_constraint_insert WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 3 trigger_name _0_test_trigger_insert_after WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 3 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 3 trigger_name _0_test_trigger_constraint_insert WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 3 trigger_name _0_test_trigger_insert_after WARNING: FIRING trigger when: AFTER level: ROW op: INSERT cnt: 3 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: STATEMENT op: INSERT cnt: 3 trigger_name _0_test_trigger_insert_s_after UPDATE hyper SET sensor_1 = 2; WARNING: FIRING trigger when: BEFORE level: STATEMENT op: UPDATE cnt: 3 trigger_name _0_test_trigger_update_s_before WARNING: FIRING trigger when: BEFORE level: ROW op: UPDATE cnt: 3 trigger_name _0_test_trigger_update WARNING: FIRING trigger when: BEFORE level: ROW op: UPDATE cnt: 3 trigger_name z_test_trigger_all WARNING: FIRING trigger when: BEFORE level: ROW op: UPDATE cnt: 3 trigger_name _0_test_trigger_update WARNING: FIRING trigger when: BEFORE level: ROW op: UPDATE cnt: 3 trigger_name z_test_trigger_all WARNING: FIRING trigger when: BEFORE level: ROW op: UPDATE cnt: 3 trigger_name _0_test_trigger_update WARNING: FIRING trigger when: BEFORE level: ROW op: UPDATE cnt: 3 trigger_name z_test_trigger_all WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name _0_test_trigger_constraint_update WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name _0_test_trigger_update_after WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name _0_test_trigger_constraint_update WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name _0_test_trigger_update_after WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name _0_test_trigger_constraint_update WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name _0_test_trigger_update_after WARNING: FIRING trigger when: AFTER level: ROW op: UPDATE cnt: 3 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: STATEMENT op: UPDATE cnt: 3 trigger_name _0_test_trigger_update_s_after DELETE FROM hyper; WARNING: FIRING trigger when: BEFORE level: STATEMENT op: DELETE cnt: 3 trigger_name _0_test_trigger_delete_s_before WARNING: FIRING trigger when: BEFORE level: ROW op: DELETE cnt: 3 trigger_name _0_test_trigger_delete WARNING: FIRING trigger when: BEFORE level: ROW op: DELETE cnt: 3 trigger_name z_test_trigger_all WARNING: FIRING trigger when: BEFORE level: ROW op: DELETE cnt: 2 trigger_name _0_test_trigger_delete WARNING: FIRING trigger when: BEFORE level: ROW op: DELETE cnt: 2 trigger_name z_test_trigger_all WARNING: FIRING trigger when: BEFORE level: ROW op: DELETE cnt: 1 trigger_name _0_test_trigger_delete WARNING: FIRING trigger when: BEFORE level: ROW op: DELETE cnt: 1 trigger_name z_test_trigger_all WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name _0_test_trigger_constraint_delete WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name _0_test_trigger_delete_after WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name _0_test_trigger_constraint_delete WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name _0_test_trigger_delete_after WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name _0_test_trigger_constraint_delete WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name _0_test_trigger_delete_after WARNING: FIRING trigger when: AFTER level: ROW op: DELETE cnt: 0 trigger_name z_test_trigger_all_after WARNING: FIRING trigger when: AFTER level: STATEMENT op: DELETE cnt: 0 trigger_name _0_test_trigger_delete_s_after CREATE TABLE vehicles ( vehicle_id INTEGER PRIMARY KEY, vin_number CHAR(17), last_checkup TIMESTAMP ); CREATE TABLE color ( color_id INTEGER PRIMARY KEY, notes text ); CREATE TABLE location ( time TIMESTAMP NOT NULL, vehicle_id INTEGER REFERENCES vehicles (vehicle_id), color_id INTEGER, --no reference since gonna populate a hypertable latitude FLOAT, longitude FLOAT ); CREATE OR REPLACE FUNCTION create_vehicle_trigger_fn() RETURNS TRIGGER LANGUAGE PLPGSQL AS $BODY$ BEGIN INSERT INTO vehicles VALUES(NEW.vehicle_id, NULL, NULL) ON CONFLICT DO NOTHING; RETURN NEW; END $BODY$; CREATE OR REPLACE FUNCTION create_color_trigger_fn() RETURNS TRIGGER LANGUAGE PLPGSQL AS $BODY$ BEGIN --test subtxns within triggers BEGIN INSERT INTO color VALUES(NEW.color_id, 'n/a'); EXCEPTION WHEN unique_violation THEN -- Nothing to do, just continue END; RETURN NEW; END $BODY$; CREATE TRIGGER create_color_trigger BEFORE INSERT OR UPDATE ON location FOR EACH ROW EXECUTE FUNCTION create_color_trigger_fn(); SELECT create_hypertable('location', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ----------------------- (2,public,location,t) --make color also a hypertable SELECT create_hypertable('color', 'color_id', chunk_time_interval=>10); create_hypertable -------------------- (3,public,color,t) -- Test that we can create and use triggers with another user GRANT TRIGGER, INSERT, SELECT, UPDATE ON location TO :ROLE_DEFAULT_PERM_USER_2; GRANT SELECT, INSERT, UPDATE ON color TO :ROLE_DEFAULT_PERM_USER_2; GRANT SELECT, INSERT, UPDATE ON vehicles TO :ROLE_DEFAULT_PERM_USER_2; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2; CREATE TRIGGER create_vehicle_trigger BEFORE INSERT OR UPDATE ON location FOR EACH ROW EXECUTE FUNCTION create_vehicle_trigger_fn(); INSERT INTO location VALUES('2017-01-01 01:02:03', 1, 1, 40.7493226,-73.9771259); INSERT INTO location VALUES('2017-01-01 01:02:04', 1, 20, 24.7493226,-73.9771259); INSERT INTO location VALUES('2017-01-01 01:02:03', 23, 1, 40.7493226,-73.9771269); INSERT INTO location VALUES('2017-01-01 01:02:03', 53, 20, 40.7493226,-73.9771269); UPDATE location SET vehicle_id = 52 WHERE vehicle_id = 53; SELECT * FROM location; time | vehicle_id | color_id | latitude | longitude --------------------------+------------+----------+------------+------------- Sun Jan 01 01:02:03 2017 | 1 | 1 | 40.7493226 | -73.9771259 Sun Jan 01 01:02:04 2017 | 1 | 20 | 24.7493226 | -73.9771259 Sun Jan 01 01:02:03 2017 | 23 | 1 | 40.7493226 | -73.9771269 Sun Jan 01 01:02:03 2017 | 52 | 20 | 40.7493226 | -73.9771269 SELECT * FROM vehicles; vehicle_id | vin_number | last_checkup ------------+------------+-------------- 1 | | 23 | | 53 | | 52 | | SELECT * FROM color; color_id | notes ----------+------- 1 | n/a 20 | n/a -- switch back to default user to run some dropping tests \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER; \set ON_ERROR_STOP 0 -- test that disable trigger is disallowed ALTER TABLE location DISABLE TRIGGER create_vehicle_trigger; ERROR: hypertables do not support enabling or disabling triggers. \set ON_ERROR_STOP 1 -- test that drop trigger works DROP TRIGGER create_color_trigger ON location; DROP TRIGGER create_vehicle_trigger ON location; -- test that drop trigger doesn't cause leftovers that mean that dropping chunks or hypertables no longer works SELECT count(1) FROM pg_depend d WHERE d.classid = 'pg_trigger'::regclass AND NOT EXISTS (SELECT 1 FROM pg_trigger WHERE oid = d.objid); count ------- 0 DROP TABLE location; -- test triggers with transition tables -- test creating hypertable from table with triggers with transition tables CREATE TABLE transition_test(time timestamptz NOT NULL); CREATE TRIGGER t1_stmt AFTER INSERT ON transition_test REFERENCING NEW TABLE AS new_trans FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER t1_row AFTER INSERT ON transition_test REFERENCING NEW TABLE AS new_trans FOR EACH ROW EXECUTE FUNCTION test_trigger(); -- We do not support ROW triggers with transition tables, so we need -- to remove it to be able to create the hypertable. \set ON_ERROR_STOP 0 SELECT create_hypertable('transition_test','time'); ERROR: ROW triggers with transition tables are not supported on hypertables \set ON_ERROR_STOP 1 DROP TRIGGER t1_row ON transition_test; SELECT create_hypertable('transition_test','time'); create_hypertable ------------------------------ (4,public,transition_test,t) -- Insert some rows to create a chunk INSERT INTO transition_test values ('2020-01-10'); WARNING: FIRING trigger when: AFTER level: STATEMENT op: INSERT cnt: 0 trigger_name t1_stmt SELECT chunk FROM show_chunks('transition_test') tbl(chunk) limit 1 \gset -- test creating trigger with transition tables on existing hypertable CREATE TRIGGER t3 AFTER UPDATE ON transition_test REFERENCING NEW TABLE AS new_trans OLD TABLE AS old_trans FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER t4 AFTER DELETE ON transition_test REFERENCING OLD TABLE AS old_trans FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); INSERT INTO transition_test values ('2020-01-11'); WARNING: FIRING trigger when: AFTER level: STATEMENT op: INSERT cnt: 0 trigger_name t1_stmt COPY transition_test FROM STDIN; WARNING: FIRING trigger when: AFTER level: STATEMENT op: INSERT cnt: 0 trigger_name t1_stmt UPDATE transition_test SET time = '2020-01-12' WHERE time = '2020-01-11'; WARNING: FIRING trigger when: AFTER level: STATEMENT op: UPDATE cnt: 0 trigger_name t3 DELETE FROM transition_test WHERE time = '2020-01-12'; WARNING: FIRING trigger when: AFTER level: STATEMENT op: DELETE cnt: 0 trigger_name t4 \set ON_ERROR_STOP 0 CREATE TRIGGER t3 AFTER UPDATE ON :chunk REFERENCING NEW TABLE AS new_trans OLD TABLE AS old_trans FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); ERROR: triggers with transition tables are not supported on hypertable chunks CREATE TRIGGER t4 AFTER DELETE ON :chunk REFERENCING OLD TABLE AS old_trans FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); ERROR: triggers with transition tables are not supported on hypertable chunks CREATE TRIGGER t5 AFTER INSERT ON transition_test REFERENCING NEW TABLE AS new_trans FOR EACH ROW EXECUTE FUNCTION test_trigger(); ERROR: ROW triggers with transition tables are not supported on hypertables CREATE TRIGGER t6 AFTER UPDATE ON transition_test REFERENCING NEW TABLE AS new_trans OLD TABLE AS old_trans FOR EACH ROW EXECUTE FUNCTION test_trigger(); ERROR: ROW triggers with transition tables are not supported on hypertables CREATE TRIGGER t7 AFTER DELETE ON transition_test REFERENCING OLD TABLE AS old_trans FOR EACH ROW EXECUTE FUNCTION test_trigger(); ERROR: ROW triggers with transition tables are not supported on hypertables \set ON_ERROR_STOP 1 ================================================ FILE: test/expected/truncate.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \o /dev/null \ir include/insert_two_partitions.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."two_Partitions" ( "timeCustom" BIGINT NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."two_Partitions" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_0) WHERE series_0 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_1) WHERE series_1 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_2) WHERE series_2 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_bool) WHERE series_bool IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, device_id); SELECT * FROM create_hypertable('"public"."two_Partitions"'::regclass, 'timeCustom'::name, 'device_id'::name, associated_schema_name=>'_timescaledb_internal'::text, number_partitions => 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); \set QUIET off BEGIN; \COPY public."two_Partitions" FROM 'data/ds1_dev1_1.tsv' NULL AS ''; COMMIT; INSERT INTO public."two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257987600000000000, 'dev1', 1.5, 1), (1257987600000000000, 'dev1', 1.5, 2), (1257894000000000000, 'dev2', 1.5, 1), (1257894002000000000, 'dev1', 2.5, 3); INSERT INTO "two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257894000000000000, 'dev2', 1.5, 2); \set QUIET on \o SELECT * FROM _timescaledb_catalog.hypertable; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+----------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 1 | public | two_Partitions | _timescaledb_internal | _hyper_1 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-----------------------+------------------+---------------------+--------+----------- 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | | 0 | f 2 | 1 | _timescaledb_internal | _hyper_1_2_chunk | | 0 | f 3 | 1 | _timescaledb_internal | _hyper_1_3_chunk | | 0 | f 4 | 1 | _timescaledb_internal | _hyper_1_4_chunk | | 0 | f SELECT * FROM test.show_subtables('"two_Partitions"'); Child | Tablespace ----------------------------------------+------------ _timescaledb_internal._hyper_1_1_chunk | _timescaledb_internal._hyper_1_2_chunk | _timescaledb_internal._hyper_1_3_chunk | _timescaledb_internal._hyper_1_4_chunk | SELECT * FROM "two_Partitions"; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257894000000000000 | dev1 | 1.5 | 1 | 2 | t 1257894000000000000 | dev1 | 1.5 | 2 | | 1257894000000001000 | dev1 | 2.5 | 3 | | 1257894001000000000 | dev1 | 3.5 | 4 | | 1257894002000000000 | dev1 | 5.5 | 6 | | t 1257894002000000000 | dev1 | 5.5 | 7 | | f 1257894002000000000 | dev1 | 2.5 | 3 | | 1257897600000000000 | dev1 | 4.5 | 5 | | f 1257987600000000000 | dev1 | 1.5 | 1 | | 1257987600000000000 | dev1 | 1.5 | 2 | | 1257894000000000000 | dev2 | 1.5 | 1 | | 1257894000000000000 | dev2 | 1.5 | 2 | | SET client_min_messages = WARNING; TRUNCATE "two_Partitions"; SELECT * FROM _timescaledb_catalog.hypertable; id | schema_name | table_name | associated_schema_name | associated_table_prefix | num_dimensions | chunk_sizing_func_schema | chunk_sizing_func_name | chunk_target_size | compression_state | compressed_hypertable_id | status ----+-------------+----------------+------------------------+-------------------------+----------------+--------------------------+--------------------------+-------------------+-------------------+--------------------------+-------- 1 | public | two_Partitions | _timescaledb_internal | _hyper_1 | 2 | _timescaledb_functions | calculate_chunk_interval | 0 | 0 | | 0 SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-------------+------------+---------------------+--------+----------- -- should be empty SELECT * FROM test.show_subtables('"two_Partitions"'); Child | Tablespace -------+------------ SELECT * FROM "two_Partitions"; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ------------+-----------+----------+----------+----------+------------- INSERT INTO public."two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257987600000000000, 'dev1', 1.5, 1), (1257987600000000000, 'dev1', 1.5, 2), (1257894000000000000, 'dev2', 1.5, 1), (1257894002000000000, 'dev1', 2.5, 3); SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; id | hypertable_id | schema_name | table_name | compressed_chunk_id | status | osm_chunk ----+---------------+-----------------------+------------------+---------------------+--------+----------- 5 | 1 | _timescaledb_internal | _hyper_1_5_chunk | | 0 | f 6 | 1 | _timescaledb_internal | _hyper_1_6_chunk | | 0 | f 7 | 1 | _timescaledb_internal | _hyper_1_7_chunk | | 0 | f CREATE VIEW dependent_view AS SELECT * FROM _timescaledb_internal._hyper_1_5_chunk; CREATE OR REPLACE FUNCTION test_trigger() RETURNS TRIGGER LANGUAGE PLPGSQL AS $BODY$ DECLARE cnt INTEGER; BEGIN RAISE WARNING 'FIRING trigger when: % level: % op: % cnt: % trigger_name %', tg_when, tg_level, tg_op, cnt, tg_name; IF TG_OP = 'DELETE' THEN RETURN OLD; END IF; RETURN NEW; END $BODY$; -- test truncate on a chunk CREATE TRIGGER _test_truncate_before BEFORE TRUNCATE ON _timescaledb_internal._hyper_1_5_chunk FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _test_truncate_after AFTER TRUNCATE ON _timescaledb_internal._hyper_1_5_chunk FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); \set ON_ERROR_STOP 0 TRUNCATE "two_Partitions"; WARNING: FIRING trigger when: BEFORE level: STATEMENT op: TRUNCATE cnt: <NULL> trigger_name _test_truncate_before WARNING: FIRING trigger when: AFTER level: STATEMENT op: TRUNCATE cnt: <NULL> trigger_name _test_truncate_after ERROR: cannot drop table _timescaledb_internal._hyper_1_5_chunk because other objects depend on it -- cannot TRUNCATE ONLY a hypertable TRUNCATE ONLY "two_Partitions" CASCADE; ERROR: cannot truncate only a hypertable \set ON_ERROR_STOP 1 -- create a regular table to make sure we can truncate it in the same call CREATE TABLE truncate_normal (color int); INSERT INTO truncate_normal VALUES (1); SELECT * FROM truncate_normal; color ------- 1 -- fix for bug #3580 \set ON_ERROR_STOP 0 TRUNCATE nonexistentrelation; ERROR: relation "nonexistentrelation" does not exist \set ON_ERROR_STOP 1 CREATE TABLE truncate_nested (color int); INSERT INTO truncate_nested VALUES (2); SELECT * FROM truncate_normal, truncate_nested; color | color -------+------- 1 | 2 CREATE FUNCTION fn_truncate_nested() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN TRUNCATE truncate_nested; RETURN NEW; END; $$; CREATE TRIGGER tg_truncate_nested BEFORE TRUNCATE ON truncate_normal FOR EACH STATEMENT EXECUTE FUNCTION fn_truncate_nested(); TRUNCATE truncate_normal; SELECT * FROM truncate_normal, truncate_nested; color | color -------+------- INSERT INTO truncate_normal VALUES (3); INSERT INTO truncate_nested VALUES (4); SELECT * FROM truncate_normal, truncate_nested; color | color -------+------- 3 | 4 TRUNCATE truncate_normal; SELECT * FROM truncate_normal, truncate_nested; color | color -------+------- INSERT INTO truncate_normal VALUES (5); INSERT INTO truncate_nested VALUES (6); SELECT * FROM truncate_normal, truncate_nested; color | color -------+------- 5 | 6 SELECT * FROM test.show_subtables('"two_Partitions"'); Child | Tablespace ----------------------------------------+------------ _timescaledb_internal._hyper_1_5_chunk | _timescaledb_internal._hyper_1_6_chunk | _timescaledb_internal._hyper_1_7_chunk | TRUNCATE "two_Partitions", truncate_normal CASCADE; WARNING: FIRING trigger when: BEFORE level: STATEMENT op: TRUNCATE cnt: <NULL> trigger_name _test_truncate_before WARNING: FIRING trigger when: AFTER level: STATEMENT op: TRUNCATE cnt: <NULL> trigger_name _test_truncate_after -- should be empty SELECT * FROM test.show_subtables('"two_Partitions"'); Child | Tablespace -------+------------ SELECT * FROM "two_Partitions"; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ------------+-----------+----------+----------+----------+------------- SELECT * FROM truncate_normal, truncate_nested; color | color -------+------- -- test TRUNCATE can be performed by a user -- with TRUNCATE privilege who is not table owner \c :TEST_DBNAME :ROLE_SUPERUSER CREATE ROLE owner WITH LOGIN; CREATE ROLE truncator WITH LOGIN; CREATE DATABASE test_trunc_ht OWNER owner; \c test_trunc_ht :ROLE_SUPERUSER SET client_min_messages = ERROR; CREATE EXTENSION timescaledb; RESET client_min_messages; \c test_trunc_ht owner CREATE TABLE test_hypertable (time TIMESTAMP WITHOUT TIME ZONE NOT NULL, value DOUBLE PRECISION); SELECT create_hypertable('test_hypertable', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ------------------------------ (1,public,test_hypertable,t) -- fail since we don't have TRUNCATE privileges yet \set ON_ERROR_STOP 0 \c test_trunc_ht truncator TRUNCATE TABLE test_hypertable; ERROR: permission denied for table test_hypertable \set ON_ERROR_STOP 1 \c test_trunc_ht owner GRANT TRUNCATE ON test_hypertable TO truncator; -- now succeed after privilege was granted \c test_trunc_ht truncator; TRUNCATE TABLE test_hypertable; \c :TEST_DBNAME :ROLE_SUPERUSER -- set client_min_messages to ERROR to suppress warnings about orphaned files SET client_min_messages TO ERROR; DROP DATABASE test_trunc_ht WITH (FORCE); DROP ROLE owner; DROP ROLE truncator; ================================================ FILE: test/expected/trusted_extension.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE DATABASE trusted_test; GRANT CREATE ON DATABASE trusted_test TO :ROLE_1; \c trusted_test :ROLE_READ_ONLY \set ON_ERROR_STOP 0 CREATE EXTENSION timescaledb; ERROR: permission denied to create extension "timescaledb" \set ON_ERROR_STOP 1 \c trusted_test :ROLE_1 -- user shouldn't have superuser privilege SELECT rolsuper FROM pg_roles WHERE rolname=user; rolsuper ---------- f SET client_min_messages TO ERROR; CREATE EXTENSION timescaledb; RESET client_min_messages; CREATE TABLE t(time timestamptz); SELECT create_hypertable('t','time'); create_hypertable ------------------- (1,public,t,t) INSERT INTO t VALUES ('2000-01-01'), ('2001-01-01'); SELECT * FROM t ORDER BY 1; time ------------------------------ Sat Jan 01 00:00:00 2000 PST Mon Jan 01 00:00:00 2001 PST SELECT * FROM timescaledb_information.hypertables; hypertable_schema | hypertable_name | owner | num_dimensions | num_chunks | compression_enabled | tablespaces | primary_dimension | primary_dimension_type -------------------+-----------------+-------------+----------------+------------+---------------------+-------------+-------------------+-------------------------- public | t | test_role_1 | 1 | 2 | f | | time | timestamp with time zone SELECT * FROM test.relation WHERE schema = '_timescaledb_internal' AND name LIKE '\_hyper%'; schema | name | type | owner -----------------------+------------------+-------+------------- _timescaledb_internal | _hyper_1_1_chunk | table | test_role_1 _timescaledb_internal | _hyper_1_2_chunk | table | test_role_1 DROP EXTENSION timescaledb CASCADE; NOTICE: drop cascades to 2 other objects \c :TEST_DBNAME :ROLE_SUPERUSER DROP DATABASE trusted_test WITH (FORCE); ================================================ FILE: test/expected/ts_merge-15.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER \set ON_ERROR_STOP 0 \set VERBOSITY default SET client_min_messages TO error; \set TEST_BASE_NAME ts_merge SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') AS "TEST_LOAD_NAME", format('include/%s_load_ht.sql', :'TEST_BASE_NAME') AS "TEST_LOAD_HT_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') AS "TEST_QUERY_NAME", format('%s/results/%s_ht_results.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') AS "TEST_RESULTS_WITH_HYPERTABLE", format('%s/results/%s_results.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') AS "TEST_RESULTS_WITH_NO_HYPERTABLE" \gset SELECT format('\! diff -u --label "Base pg table results" --label "Hypertable results" %s %s', :'TEST_RESULTS_WITH_HYPERTABLE', :'TEST_RESULTS_WITH_NO_HYPERTABLE') AS "DIFF_CMD" \gset \ir :TEST_LOAD_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE USER regress_merge_privs; CREATE USER regress_merge_no_privs; DROP TABLE IF EXISTS target; DROP TABLE IF EXISTS source; CREATE TABLE target (tid integer, balance integer) WITH (autovacuum_enabled=off); CREATE TABLE source (sid integer, delta integer) -- no index WITH (autovacuum_enabled=off); INSERT INTO target VALUES (1, 10); INSERT INTO target VALUES (2, 20); INSERT INTO target VALUES (3, 30); SELECT t.ctid is not null as matched, t.*, s.* FROM source s FULL OUTER JOIN target t ON s.sid = t.tid ORDER BY t.tid, s.sid; matched | tid | balance | sid | delta ---------+-----+---------+-----+------- t | 1 | 10 | | t | 2 | 20 | | t | 3 | 30 | | ALTER TABLE target OWNER TO regress_merge_privs; ALTER TABLE source OWNER TO regress_merge_privs; CREATE TABLE target2 (tid integer, balance integer) WITH (autovacuum_enabled=off); CREATE TABLE source2 (sid integer, delta integer) WITH (autovacuum_enabled=off); ALTER TABLE target2 OWNER TO regress_merge_no_privs; ALTER TABLE source2 OWNER TO regress_merge_no_privs; GRANT INSERT ON target TO regress_merge_no_privs; GRANT CREATE ON SCHEMA public TO regress_merge_privs; SET SESSION AUTHORIZATION regress_merge_privs; CREATE TABLE sq_target (tid integer NOT NULL, balance integer) WITH (autovacuum_enabled=off); CREATE TABLE sq_source (delta integer, sid integer, balance integer DEFAULT 0) WITH (autovacuum_enabled=off); INSERT INTO sq_target(tid, balance) VALUES (1,100), (2,200), (3,300); INSERT INTO sq_source(sid, delta) VALUES (1,10), (2,20), (4,40); -- conditional WHEN clause CREATE TABLE wq_target (tid integer not null, balance integer DEFAULT -1) WITH (autovacuum_enabled=off); CREATE TABLE wq_source (balance integer, sid integer) WITH (autovacuum_enabled=off); INSERT INTO wq_source (sid, balance) VALUES (1, 100); CREATE TABLE cj_target (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE cj_source1 (sid1 integer, scat integer, delta integer) WITH (autovacuum_enabled=off); CREATE TABLE cj_source2 (sid2 integer, sval text) WITH (autovacuum_enabled=off); INSERT INTO cj_source1 VALUES (1, 10, 100); INSERT INTO cj_source1 VALUES (1, 20, 200); INSERT INTO cj_source1 VALUES (2, 20, 300); INSERT INTO cj_source1 VALUES (3, 10, 400); INSERT INTO cj_source2 VALUES (1, 'initial source2'); INSERT INTO cj_source2 VALUES (2, 'initial source2'); INSERT INTO cj_source2 VALUES (3, 'initial source2'); CREATE TABLE fs_target (a int, b int, c text) WITH (autovacuum_enabled=off); -- run tests on normal table \o :TEST_RESULTS_WITH_NO_HYPERTABLE \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- -- Errors -- MERGE INTO target t RANDOMWORD USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:12: ERROR: syntax error at or near "RANDOMWORD" LINE 1: MERGE INTO target t RANDOMWORD ^ -- MATCHED/INSERT error MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:18: ERROR: syntax error at or near "INSERT" LINE 5: INSERT DEFAULT VALUES; ^ -- incorrectly specifying INTO target MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT INTO target DEFAULT VALUES; psql:include/ts_merge_query.sql:24: ERROR: syntax error at or near "INTO" LINE 5: INSERT INTO target DEFAULT VALUES; ^ -- Multiple VALUES clause MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (1,1), (2,2); psql:include/ts_merge_query.sql:30: ERROR: syntax error at or near "," LINE 5: INSERT VALUES (1,1), (2,2); ^ -- SELECT query for INSERT MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT SELECT (1, 1); psql:include/ts_merge_query.sql:36: ERROR: syntax error at or near "SELECT" LINE 5: INSERT SELECT (1, 1); ^ -- NOT MATCHED/UPDATE MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:42: ERROR: syntax error at or near "UPDATE" LINE 5: UPDATE SET balance = 0; ^ -- UPDATE tablename MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE target SET balance = 0; psql:include/ts_merge_query.sql:48: ERROR: syntax error at or near "target" LINE 5: UPDATE target SET balance = 0; ^ -- source and target names the same MERGE INTO target USING target ON tid = tid WHEN MATCHED THEN DO NOTHING; psql:include/ts_merge_query.sql:53: ERROR: name "target" specified more than once DETAIL: The name is used both as MERGE target table and data source. -- used in a CTE WITH foo AS ( MERGE INTO target USING source ON (true) WHEN MATCHED THEN DELETE ) SELECT * FROM foo; psql:include/ts_merge_query.sql:58: ERROR: MERGE not supported in WITH query LINE 1: WITH foo AS ( ^ -- used in COPY COPY ( MERGE INTO target USING source ON (true) WHEN MATCHED THEN DELETE ) TO stdout; psql:include/ts_merge_query.sql:63: ERROR: MERGE not supported in COPY -- unsupported relation types -- view CREATE VIEW tv AS SELECT * FROM target; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:72: ERROR: cannot execute MERGE on relation "tv" DETAIL: This operation is not supported for views. DROP VIEW tv; -- materialized view CREATE MATERIALIZED VIEW mv AS SELECT * FROM target; MERGE INTO mv t USING source s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:81: ERROR: cannot execute MERGE on relation "mv" DETAIL: This operation is not supported for materialized views. DROP MATERIALIZED VIEW mv; -- permissions MERGE INTO target USING source2 ON target.tid = source2.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:90: ERROR: permission denied for table source2 GRANT INSERT ON target TO regress_merge_no_privs; SET SESSION AUTHORIZATION regress_merge_no_privs; MERGE INTO target USING source2 ON target.tid = source2.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:99: ERROR: permission denied for table target GRANT UPDATE ON target2 TO regress_merge_privs; SET SESSION AUTHORIZATION regress_merge_privs; MERGE INTO target2 USING source ON target2.tid = source.sid WHEN MATCHED THEN DELETE; psql:include/ts_merge_query.sql:108: ERROR: permission denied for table target2 MERGE INTO target2 USING source ON target2.tid = source.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:114: ERROR: permission denied for table target2 -- check if the target can be accessed from source relation subquery; we should -- not be able to do so MERGE INTO target t USING (SELECT * FROM source WHERE t.tid > sid) s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:122: ERROR: invalid reference to FROM-clause entry for table "t" LINE 2: USING (SELECT * FROM source WHERE t.tid > sid) s ^ HINT: There is an entry for table "t", but it cannot be referenced from this part of the query. -- -- initial tests -- -- zero rows in source has no effect MERGE INTO target USING source ON target.tid = source.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; ROLLBACK; -- insert some non-matching source rows to work from INSERT INTO source VALUES (4, 40); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN DO NOTHING; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (5, 50); SELECT * FROM target ORDER BY tid; ROLLBACK; -- index plans INSERT INTO target SELECT generate_series(1000,2500), 0; ALTER TABLE target ADD PRIMARY KEY (tid); ANALYZE target; DELETE FROM target WHERE tid > 100; ANALYZE target; -- insert some matching source rows to work from INSERT INTO source VALUES (2, 5); INSERT INTO source VALUES (3, 20); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- equivalent of an UPDATE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; SELECT * FROM target ORDER BY tid; ROLLBACK; -- equivalent of a DELETE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DO NOTHING; SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, NULL); SELECT * FROM target ORDER BY tid; ROLLBACK; -- duplicate source row causes multiple target row update ERROR INSERT INTO source VALUES (2, 5); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:241: ERROR: MERGE command cannot affect row a second time HINT: Ensure that not more than one source row matches any one target row. ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; psql:include/ts_merge_query.sql:249: ERROR: MERGE command cannot affect row a second time HINT: Ensure that not more than one source row matches any one target row. ROLLBACK; -- remove duplicate MATCHED data from source data DELETE FROM source WHERE sid = 2; INSERT INTO source VALUES (2, 5); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- duplicate source row on INSERT should fail because of target_pkey INSERT INTO source VALUES (4, 40); BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, NULL); psql:include/ts_merge_query.sql:265: ERROR: duplicate key value violates unique constraint "target_pkey" DETAIL: Key (tid)=(4) already exists. SELECT * FROM target ORDER BY tid; psql:include/ts_merge_query.sql:266: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; -- remove duplicate NOT MATCHED data from source data DELETE FROM source WHERE sid = 4; INSERT INTO source VALUES (4, 40); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- remove constraints alter table target drop CONSTRAINT target_pkey; alter table target alter column tid drop not null; -- multiple actions BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, 4) WHEN MATCHED THEN UPDATE SET balance = 0; SELECT * FROM target ORDER BY tid; ROLLBACK; -- should be equivalent BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0 WHEN NOT MATCHED THEN INSERT VALUES (4, 4); SELECT * FROM target ORDER BY tid; ROLLBACK; -- column references -- do a simple equivalent of an UPDATE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance + s.delta; SELECT * FROM target ORDER BY tid; ROLLBACK; -- do a simple equivalent of an INSERT SELECT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- and again with duplicate source rows INSERT INTO source VALUES (5, 50); INSERT INTO source VALUES (5, 50); -- do a simple equivalent of an INSERT SELECT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- removing duplicate source rows DELETE FROM source WHERE sid = 5; -- and again with explicitly identified column list BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- and again with a subtle error: referring to non-existent target row for NOT MATCHED MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (t.tid, s.delta); psql:include/ts_merge_query.sql:356: ERROR: invalid reference to FROM-clause entry for table "t" LINE 5: INSERT (tid, balance) VALUES (t.tid, s.delta); ^ HINT: There is an entry for table "t", but it cannot be referenced from this part of the query. -- and again with a constant ON clause BEGIN; MERGE INTO target t USING source AS s ON (SELECT true) WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (t.tid, s.delta); psql:include/ts_merge_query.sql:364: ERROR: invalid reference to FROM-clause entry for table "t" LINE 5: INSERT (tid, balance) VALUES (t.tid, s.delta); ^ HINT: There is an entry for table "t", but it cannot be referenced from this part of the query. SELECT * FROM target ORDER BY tid; psql:include/ts_merge_query.sql:365: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; -- now the classic UPSERT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance + s.delta WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- this time with a FALSE condition MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND FALSE THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; -- this time with an actual condition which returns false MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance <> 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; BEGIN; -- and now with a condition which returns true MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance = 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; ROLLBACK; -- conditions in the NOT MATCHED clause can only refer to source columns BEGIN; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND t.balance = 100 THEN INSERT (tid) VALUES (s.sid); psql:include/ts_merge_query.sql:408: ERROR: invalid reference to FROM-clause entry for table "t" LINE 3: WHEN NOT MATCHED AND t.balance = 100 THEN ^ HINT: There is an entry for table "t", but it cannot be referenced from this part of the query. SELECT * FROM wq_target; psql:include/ts_merge_query.sql:409: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance = 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; -- conditions in MATCHED clause can refer to both source and target SELECT * FROM wq_source; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND s.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check if AND works MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 AND s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 AND s.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check if OR works MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 OR s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 199 OR s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check source-side whole-row references BEGIN; MERGE INTO wq_target t USING wq_source s ON (t.tid = s.sid) WHEN matched and t = s or t.tid = s.sid THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; ROLLBACK; -- check if subqueries work in the conditions? MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance > (SELECT max(balance) FROM target) THEN UPDATE SET balance = t.balance + s.balance; -- check if we can access system columns in the conditions MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.xmin = t.xmax THEN UPDATE SET balance = t.balance + s.balance; psql:include/ts_merge_query.sql:477: ERROR: cannot use system column "xmin" in MERGE WHEN condition LINE 3: WHEN MATCHED AND t.xmin = t.xmax THEN ^ MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.tableoid >= 0 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; DROP TABLE wq_target CASCADE; DROP TABLE wq_source; -- test triggers create or replace function merge_trigfunc () returns trigger language plpgsql as $$ DECLARE line text; BEGIN SELECT INTO line format('%s %s %s trigger%s', TG_WHEN, TG_OP, TG_LEVEL, CASE WHEN TG_OP = 'INSERT' AND TG_LEVEL = 'ROW' THEN format(' row: %s', NEW) WHEN TG_OP = 'UPDATE' AND TG_LEVEL = 'ROW' THEN format(' row: %s -> %s', OLD, NEW) WHEN TG_OP = 'DELETE' AND TG_LEVEL = 'ROW' THEN format(' row: %s', OLD) END); RAISE NOTICE '%', line; IF (TG_WHEN = 'BEFORE' AND TG_LEVEL = 'ROW') THEN IF (TG_OP = 'DELETE') THEN RETURN OLD; ELSE RETURN NEW; END IF; ELSE RETURN NULL; END IF; END; $$; CREATE TRIGGER merge_bsi BEFORE INSERT ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bsu BEFORE UPDATE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bsd BEFORE DELETE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asi AFTER INSERT ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asu AFTER UPDATE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asd AFTER DELETE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bri BEFORE INSERT ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bru BEFORE UPDATE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_brd BEFORE DELETE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_ari AFTER INSERT ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_aru AFTER UPDATE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_ard AFTER DELETE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); -- now the classic UPSERT, with a DELETE BEGIN; UPDATE target SET balance = 0 WHERE tid = 3; --EXPLAIN (ANALYZE ON, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND t.balance > s.delta THEN UPDATE SET balance = t.balance - s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- Test behavior of triggers that turn UPDATE/DELETE into no-ops create or replace function skip_merge_op() returns trigger language plpgsql as $$ BEGIN RETURN NULL; END; $$; SELECT * FROM target full outer join source on (sid = tid); create trigger merge_skip BEFORE INSERT OR UPDATE or DELETE ON target FOR EACH ROW EXECUTE FUNCTION skip_merge_op(); DO $$ DECLARE result integer; BEGIN MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND s.sid = 3 THEN UPDATE SET balance = t.balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT VALUES (sid, delta); IF FOUND THEN RAISE NOTICE 'Found'; ELSE RAISE NOTICE 'Not found'; END IF; GET DIAGNOSTICS result := ROW_COUNT; RAISE NOTICE 'ROW_COUNT = %', result; END; $$; SELECT * FROM target FULL OUTER JOIN source ON (sid = tid); DROP TRIGGER merge_skip ON target; DROP FUNCTION skip_merge_op(); -- test from PL/pgSQL -- make sure MERGE INTO isn't interpreted to mean returning variables like SELECT INTO BEGIN; DO LANGUAGE plpgsql $$ BEGIN MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND t.balance > s.delta THEN UPDATE SET balance = t.balance - s.delta; END; $$; ROLLBACK; --source constants BEGIN; MERGE INTO target t USING (SELECT 9 AS sid, 57 AS delta) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; --source query BEGIN; MERGE INTO target t USING (SELECT sid, delta FROM source WHERE delta > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT sid, delta as newname FROM source WHERE delta > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.newname); SELECT * FROM target ORDER BY tid; ROLLBACK; --self-merge BEGIN; MERGE INTO target t1 USING target t2 ON t1.tid = t2.tid WHEN MATCHED THEN UPDATE SET balance = t1.balance + t2.balance WHEN NOT MATCHED THEN INSERT VALUES (t2.tid, t2.balance); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT tid as sid, balance as delta FROM target WHERE balance > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT sid, max(delta) AS delta FROM source GROUP BY sid HAVING count(*) = 1 ORDER BY sid ASC) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- plpgsql parameters and results BEGIN; CREATE FUNCTION merge_func (p_id integer, p_bal integer) RETURNS INTEGER LANGUAGE plpgsql AS $$ DECLARE result integer; BEGIN MERGE INTO target t USING (SELECT p_id AS sid) AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance - p_bal; IF FOUND THEN GET DIAGNOSTICS result := ROW_COUNT; END IF; RETURN result; END; $$; SELECT merge_func(3, 4); SELECT * FROM target ORDER BY tid; ROLLBACK; -- PREPARE BEGIN; prepare foom as merge into target t using (select 1 as sid) s on (t.tid = s.sid) when matched then update set balance = 1; execute foom; ROLLBACK; BEGIN; PREPARE foom2 (integer, integer) AS MERGE INTO target t USING (SELECT 1) s ON t.tid = $1 WHEN MATCHED THEN UPDATE SET balance = $2; --EXPLAIN (ANALYZE ON, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) execute foom2 (1, 1); ROLLBACK; -- subqueries in source relation BEGIN; MERGE INTO sq_target t USING (SELECT * FROM sq_source) s ON tid = sid WHEN MATCHED AND t.balance > delta THEN UPDATE SET balance = t.balance + delta; SELECT * FROM sq_target ORDER BY tid; ROLLBACK; -- try a view CREATE VIEW v AS SELECT * FROM sq_source WHERE sid < 2; BEGIN; MERGE INTO sq_target USING v ON tid = sid WHEN MATCHED THEN UPDATE SET balance = v.balance + delta; SELECT * FROM sq_target ORDER BY tid; ROLLBACK; -- ambiguous reference to a column BEGIN; MERGE INTO sq_target USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; psql:include/ts_merge_query.sql:732: ERROR: column reference "balance" is ambiguous LINE 5: UPDATE SET balance = balance + delta ^ ROLLBACK; BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; SELECT * FROM sq_target; ROLLBACK; -- CTEs BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); WITH targq AS ( SELECT * FROM v ) MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; ROLLBACK; -- RETURNING BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE RETURNING *; psql:include/ts_merge_query.sql:778: ERROR: syntax error at or near "RETURNING" LINE 10: RETURNING *; ^ ROLLBACK; -- PG17-specific tests for views, returning and merge_action. These throw syntax errors for previous versions of Postgres. -- However, since the error is the same for both hypertables and regular tables, this test should still pass for previous versions. -- RETURNING INSERT INTO source(sid, delta) VALUES(1, 40), (5, 50); BEGIN; MERGE INTO target t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta); SELECT * from target; ROLLBACK; BEGIN; MERGE INTO target t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 1 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta) RETURNING merge_action(), t.*; psql:include/ts_merge_query.sql:811: ERROR: syntax error at or near "RETURNING" LINE 10: RETURNING merge_action(), t.*; ^ ROLLBACK; -- Views CREATE VIEW tv AS SELECT * FROM target; BEGIN; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta); psql:include/ts_merge_query.sql:826: ERROR: cannot execute MERGE on relation "tv" DETAIL: This operation is not supported for views. SELECT * from tv; psql:include/ts_merge_query.sql:827: ERROR: current transaction is aborted, commands ignored until end of transaction block SELECT * from target; -- should also update the underlying table psql:include/ts_merge_query.sql:828: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; BEGIN; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta) RETURNING merge_action(), t.*; psql:include/ts_merge_query.sql:841: ERROR: syntax error at or near "RETURNING" LINE 10: RETURNING merge_action(), t.*; ^ ROLLBACK; DROP VIEW tv; DELETE FROM source where sid in (1, 5); -- EXPLAIN CREATE TABLE ex_mtarget (a int, b int) WITH (autovacuum_enabled=off); CREATE TABLE ex_msource (a int, b int) WITH (autovacuum_enabled=off); INSERT INTO ex_mtarget SELECT i, i*10 FROM generate_series(1,100,2) i; INSERT INTO ex_msource SELECT i, i*10 FROM generate_series(1,100,1) i; CREATE FUNCTION explain_merge(query text) RETURNS SETOF text LANGUAGE plpgsql AS $$ DECLARE ln text; BEGIN FOR ln IN EXECUTE 'explain (analyze, timing off, summary off, buffers off, costs off) ' || query LOOP ln := regexp_replace(ln, '(Memory( Usage)?|Buckets|Batches): \S*', '\1: xxx', 'g'); RETURN NEXT ln; END LOOP; END; $$; -- only updates SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED THEN UPDATE SET b = t.b + 1'); -- only updates to selected tuples SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1'); -- updates + deletes SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1 WHEN MATCHED AND t.a >= 10 AND t.a <= 20 THEN DELETE'); -- only inserts SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN NOT MATCHED AND s.a < 10 THEN INSERT VALUES (a, b)'); -- all three SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1 WHEN MATCHED AND t.a >= 30 AND t.a <= 40 THEN DELETE WHEN NOT MATCHED AND s.a < 20 THEN INSERT VALUES (a, b)'); -- nothing SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a AND t.a < -1000 WHEN MATCHED AND t.a < 10 THEN DO NOTHING'); DROP TABLE ex_msource, ex_mtarget; DROP FUNCTION explain_merge(text); -- Subqueries BEGIN; MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED THEN UPDATE SET balance = (SELECT count(*) FROM sq_target); SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; BEGIN; MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND (SELECT count(*) > 0 FROM sq_target) THEN UPDATE SET balance = 42; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; BEGIN; MERGE INTO sq_target t USING v ON tid = sid AND (SELECT count(*) > 0 FROM sq_target) WHEN MATCHED THEN UPDATE SET balance = 42; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; -- Test RETURNING with subqueries BEGIN; MERGE INTO sq_target t USING v ON tid = sid AND (SELECT count(*) > 0 FROM sq_target) WHEN MATCHED THEN UPDATE SET balance = 42 RETURNING *; psql:include/ts_merge_query.sql:952: ERROR: syntax error at or near "RETURNING" LINE 6: RETURNING *; ^ SELECT * FROM sq_target WHERE tid = 1; psql:include/ts_merge_query.sql:953: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; DROP TABLE sq_target CASCADE; DROP TABLE sq_source CASCADE; CREATE TABLE pa_target (tid integer, balance float, val text) PARTITION BY LIST (tid); CREATE TABLE part1 PARTITION OF pa_target FOR VALUES IN (1,4) WITH (autovacuum_enabled=off); CREATE TABLE part2 PARTITION OF pa_target FOR VALUES IN (2,5,6) WITH (autovacuum_enabled=off); CREATE TABLE part3 PARTITION OF pa_target FOR VALUES IN (3,8,9) WITH (autovacuum_enabled=off); CREATE TABLE part4 PARTITION OF pa_target DEFAULT WITH (autovacuum_enabled=off); CREATE TABLE pa_source (sid integer, delta float); -- insert many rows to the source table INSERT INTO pa_source SELECT id, id * 10 FROM generate_series(1,14) AS id; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT id, id * 100, 'initial' FROM generate_series(1,14,2) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- same with a constant qual BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid AND tid = 1 WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- try updating the partition key column BEGIN; CREATE FUNCTION merge_func() RETURNS integer LANGUAGE plpgsql AS $$ DECLARE result integer; BEGIN MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET tid = tid + 1, balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); IF FOUND THEN GET DIAGNOSTICS result := ROW_COUNT; END IF; RETURN result; END; $$; SELECT merge_func(); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_target CASCADE; -- The target table is partitioned in the same way, but this time by attaching -- partitions which have columns in different order, dropped columns etc. CREATE TABLE pa_target (tid integer, balance float, val text) PARTITION BY LIST (tid); CREATE TABLE part1 (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE part2 (balance float, tid integer, val text) WITH (autovacuum_enabled=off); CREATE TABLE part3 (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE part4 (extraid text, tid integer, balance float, val text) WITH (autovacuum_enabled=off); ALTER TABLE part4 DROP COLUMN extraid; ALTER TABLE pa_target ATTACH PARTITION part1 FOR VALUES IN (1,4); ALTER TABLE pa_target ATTACH PARTITION part2 FOR VALUES IN (2,5,6); ALTER TABLE pa_target ATTACH PARTITION part3 FOR VALUES IN (3,8,9); ALTER TABLE pa_target ATTACH PARTITION part4 DEFAULT; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT id, id * 100, 'initial' FROM generate_series(1,14,2) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- same with a constant qual BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid AND tid IN (1, 5) WHEN MATCHED AND tid % 5 = 0 THEN DELETE WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- try updating the partition key column BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET tid = tid + 1, balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_source; DROP TABLE pa_target CASCADE; -- Sub-partitioning CREATE TABLE pa_target (logts timestamp, tid integer, balance float, val text) PARTITION BY RANGE (logts); CREATE TABLE part_m01 PARTITION OF pa_target FOR VALUES FROM ('2017-01-01') TO ('2017-02-01') PARTITION BY LIST (tid); CREATE TABLE part_m01_odd PARTITION OF part_m01 FOR VALUES IN (1,3,5,7,9) WITH (autovacuum_enabled=off); CREATE TABLE part_m01_even PARTITION OF part_m01 FOR VALUES IN (2,4,6,8) WITH (autovacuum_enabled=off); CREATE TABLE part_m02 PARTITION OF pa_target FOR VALUES FROM ('2017-02-01') TO ('2017-03-01') PARTITION BY LIST (tid); CREATE TABLE part_m02_odd PARTITION OF part_m02 FOR VALUES IN (1,3,5,7,9) WITH (autovacuum_enabled=off); CREATE TABLE part_m02_even PARTITION OF part_m02 FOR VALUES IN (2,4,6,8) WITH (autovacuum_enabled=off); CREATE TABLE pa_source (sid integer, delta float) WITH (autovacuum_enabled=off); -- insert many rows to the source table INSERT INTO pa_source SELECT id, id * 10 FROM generate_series(1,14) AS id; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT '2017-01-31', id, id * 100, 'initial' FROM generate_series(1,9,3) AS id; INSERT INTO pa_target SELECT '2017-02-28', id, id * 100, 'initial' FROM generate_series(2,9,3) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING (SELECT '2017-01-15' AS slogts, * FROM pa_source WHERE sid < 10) s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (slogts::timestamp, sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_source; DROP TABLE pa_target CASCADE; -- some complex joins on the source side -- source relation is an unaliased join MERGE INTO cj_target t USING cj_source1 s1 INNER JOIN cj_source2 s2 ON sid1 = sid2 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid1, delta, sval); -- try accessing columns from either side of the source join MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 AND scat = 20 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid2, delta, sval) WHEN MATCHED THEN DELETE; -- some simple expressions in INSERT targetlist MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid2, delta + scat, sval) WHEN MATCHED THEN UPDATE SET val = val || ' updated by merge'; MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 AND scat = 20 ON t.tid = sid1 WHEN MATCHED THEN UPDATE SET val = val || ' ' || delta::text; SELECT * FROM cj_target ORDER BY tid; ALTER TABLE cj_source1 RENAME COLUMN sid1 TO sid; ALTER TABLE cj_source2 RENAME COLUMN sid2 TO sid; TRUNCATE cj_target; MERGE INTO cj_target t USING cj_source1 s1 INNER JOIN cj_source2 s2 ON s1.sid = s2.sid ON t.tid = s1.sid WHEN NOT MATCHED THEN INSERT VALUES (s2.sid, delta, sval); DROP TABLE cj_source2, cj_source1; DROP TABLE cj_target CASCADE; -- Function scans MERGE INTO fs_target t USING generate_series(1,100,1) AS id ON t.a = id WHEN MATCHED THEN UPDATE SET b = b + id WHEN NOT MATCHED THEN INSERT VALUES (id, -1); MERGE INTO fs_target t USING generate_series(1,100,2) AS id ON t.a = id WHEN MATCHED THEN UPDATE SET b = b + id, c = 'updated '|| id.*::text WHEN NOT MATCHED THEN INSERT VALUES (id, -1, 'inserted ' || id.*::text); SELECT count(*) FROM fs_target; DROP TABLE fs_target CASCADE; -- SERIALIZABLE test -- handled in isolation tests -- Inheritance-based partitioning CREATE TABLE measurement ( city_id int not null, logdate date not null, peaktemp int, unitsales int ) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2006m02 ( CHECK ( logdate >= DATE '2006-02-01' AND logdate < DATE '2006-03-01' ) ) INHERITS (measurement) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2006m03 ( CHECK ( logdate >= DATE '2006-03-01' AND logdate < DATE '2006-04-01' ) ) INHERITS (measurement) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2007m01 ( filler text, peaktemp int, logdate date not null, city_id int not null, unitsales int CHECK ( logdate >= DATE '2007-01-01' AND logdate < DATE '2007-02-01') ) WITH (autovacuum_enabled=off); ALTER TABLE measurement_y2007m01 DROP COLUMN filler; ALTER TABLE measurement_y2007m01 INHERIT measurement; INSERT INTO measurement VALUES (0, '2005-07-21', 5, 15); CREATE OR REPLACE FUNCTION measurement_insert_trigger() RETURNS TRIGGER AS $$ BEGIN IF ( NEW.logdate >= DATE '2006-02-01' AND NEW.logdate < DATE '2006-03-01' ) THEN INSERT INTO measurement_y2006m02 VALUES (NEW.*); ELSIF ( NEW.logdate >= DATE '2006-03-01' AND NEW.logdate < DATE '2006-04-01' ) THEN INSERT INTO measurement_y2006m03 VALUES (NEW.*); ELSIF ( NEW.logdate >= DATE '2007-01-01' AND NEW.logdate < DATE '2007-02-01' ) THEN INSERT INTO measurement_y2007m01 (city_id, logdate, peaktemp, unitsales) VALUES (NEW.*); ELSE RAISE EXCEPTION 'Date out of range. Fix the measurement_insert_trigger() function!'; END IF; RETURN NULL; END; $$ LANGUAGE plpgsql ; CREATE TRIGGER insert_measurement_trigger BEFORE INSERT ON measurement FOR EACH ROW EXECUTE PROCEDURE measurement_insert_trigger(); INSERT INTO measurement VALUES (1, '2006-02-10', 35, 10); INSERT INTO measurement VALUES (1, '2006-02-16', 45, 20); INSERT INTO measurement VALUES (1, '2006-03-17', 25, 10); INSERT INTO measurement VALUES (1, '2006-03-27', 15, 40); INSERT INTO measurement VALUES (1, '2007-01-15', 10, 10); INSERT INTO measurement VALUES (1, '2007-01-17', 10, 10); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate; CREATE TABLE new_measurement (LIKE measurement) WITH (autovacuum_enabled=off); INSERT INTO new_measurement VALUES (0, '2005-07-21', 25, 20); INSERT INTO new_measurement VALUES (1, '2006-03-01', 20, 10); INSERT INTO new_measurement VALUES (1, '2006-02-16', 50, 10); INSERT INTO new_measurement VALUES (2, '2006-02-10', 20, 20); INSERT INTO new_measurement VALUES (1, '2006-03-27', NULL, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-17', NULL, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-15', 5, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-16', 10, 10); BEGIN; MERGE INTO ONLY measurement m USING new_measurement nm ON (m.city_id = nm.city_id and m.logdate=nm.logdate) WHEN MATCHED AND nm.peaktemp IS NULL THEN DELETE WHEN MATCHED THEN UPDATE SET peaktemp = greatest(m.peaktemp, nm.peaktemp), unitsales = m.unitsales + coalesce(nm.unitsales, 0) WHEN NOT MATCHED THEN INSERT (city_id, logdate, peaktemp, unitsales) VALUES (city_id, logdate, peaktemp, unitsales); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate, peaktemp; ROLLBACK; MERGE into measurement m USING new_measurement nm ON (m.city_id = nm.city_id and m.logdate=nm.logdate) WHEN MATCHED AND nm.peaktemp IS NULL THEN DELETE WHEN MATCHED THEN UPDATE SET peaktemp = greatest(m.peaktemp, nm.peaktemp), unitsales = m.unitsales + coalesce(nm.unitsales, 0) WHEN NOT MATCHED THEN INSERT (city_id, logdate, peaktemp, unitsales) VALUES (city_id, logdate, peaktemp, unitsales); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate; BEGIN; MERGE INTO new_measurement nm USING ONLY measurement m ON (nm.city_id = m.city_id and nm.logdate=m.logdate) WHEN MATCHED THEN DELETE; SELECT * FROM new_measurement ORDER BY city_id, logdate; ROLLBACK; MERGE INTO new_measurement nm USING measurement m ON (nm.city_id = m.city_id and nm.logdate=m.logdate) WHEN MATCHED THEN DELETE; SELECT * FROM new_measurement ORDER BY city_id, logdate; DROP TABLE measurement, new_measurement CASCADE; DROP FUNCTION measurement_insert_trigger(); RESET SESSION AUTHORIZATION; DROP TABLE target CASCADE; DROP TABLE target2 CASCADE; DROP TABLE source, source2; DROP FUNCTION merge_trigfunc(); REVOKE CREATE ON SCHEMA public FROM regress_merge_privs; DROP USER regress_merge_privs; DROP USER regress_merge_no_privs; \o \ir :TEST_LOAD_HT_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE USER regress_merge_privs; CREATE USER regress_merge_no_privs; DROP TABLE IF EXISTS target; DROP TABLE IF EXISTS source; CREATE TABLE target (tid integer, balance integer) WITH (autovacuum_enabled=off); SELECT create_hypertable('target', 'tid', chunk_time_interval => 3); create_hypertable --------------------- (1,public,target,t) CREATE TABLE source (sid integer, delta integer) -- no index WITH (autovacuum_enabled=off); INSERT INTO target VALUES (1, 10); INSERT INTO target VALUES (2, 20); INSERT INTO target VALUES (3, 30); SELECT t.ctid is not null as matched, t.*, s.* FROM source s FULL OUTER JOIN target t ON s.sid = t.tid ORDER BY t.tid, s.sid; matched | tid | balance | sid | delta ---------+-----+---------+-----+------- t | 1 | 10 | | t | 2 | 20 | | t | 3 | 30 | | ALTER TABLE target OWNER TO regress_merge_privs; ALTER TABLE source OWNER TO regress_merge_privs; CREATE TABLE target2 (tid integer, balance integer) WITH (autovacuum_enabled=off); SELECT create_hypertable('target2', 'tid', chunk_time_interval => 3); create_hypertable ---------------------- (2,public,target2,t) CREATE TABLE source2 (sid integer, delta integer) WITH (autovacuum_enabled=off); ALTER TABLE target2 OWNER TO regress_merge_no_privs; ALTER TABLE source2 OWNER TO regress_merge_no_privs; GRANT INSERT ON target TO regress_merge_no_privs; GRANT CREATE ON SCHEMA public TO regress_merge_privs; SET SESSION AUTHORIZATION regress_merge_privs; CREATE TABLE sq_target (tid integer NOT NULL, balance integer) WITH (autovacuum_enabled=off); SELECT create_hypertable('sq_target', 'tid', chunk_time_interval => 3); create_hypertable ------------------------ (3,public,sq_target,t) CREATE TABLE sq_source (delta integer, sid integer, balance integer DEFAULT 0) WITH (autovacuum_enabled=off); INSERT INTO sq_target(tid, balance) VALUES (1,100), (2,200), (3,300); INSERT INTO sq_source(sid, delta) VALUES (1,10), (2,20), (4,40); -- conditional WHEN clause CREATE TABLE wq_target (tid integer not null, balance integer DEFAULT -1) WITH (autovacuum_enabled=off); SELECT create_hypertable('wq_target', 'tid', chunk_time_interval => 3); create_hypertable ------------------------ (4,public,wq_target,t) CREATE TABLE wq_source (balance integer, sid integer) WITH (autovacuum_enabled=off); INSERT INTO wq_source (sid, balance) VALUES (1, 100); -- some complex joins on the source side CREATE TABLE cj_target (tid integer, balance float, val text) WITH (autovacuum_enabled=off); SELECT create_hypertable('cj_target', 'tid', chunk_time_interval => 3); create_hypertable ------------------------ (5,public,cj_target,t) CREATE TABLE cj_source1 (sid1 integer, scat integer, delta integer) WITH (autovacuum_enabled=off); CREATE TABLE cj_source2 (sid2 integer, sval text) WITH (autovacuum_enabled=off); INSERT INTO cj_source1 VALUES (1, 10, 100); INSERT INTO cj_source1 VALUES (1, 20, 200); INSERT INTO cj_source1 VALUES (2, 20, 300); INSERT INTO cj_source1 VALUES (3, 10, 400); INSERT INTO cj_source2 VALUES (1, 'initial source2'); INSERT INTO cj_source2 VALUES (2, 'initial source2'); INSERT INTO cj_source2 VALUES (3, 'initial source2'); CREATE TABLE fs_target (a int, b int, c text) WITH (autovacuum_enabled=off); SELECT create_hypertable('fs_target', 'a', chunk_time_interval => 3); create_hypertable ------------------------ (6,public,fs_target,t) -- run tests on hypertable \o :TEST_RESULTS_WITH_HYPERTABLE \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- -- Errors -- MERGE INTO target t RANDOMWORD USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:12: ERROR: syntax error at or near "RANDOMWORD" LINE 1: MERGE INTO target t RANDOMWORD ^ -- MATCHED/INSERT error MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:18: ERROR: syntax error at or near "INSERT" LINE 5: INSERT DEFAULT VALUES; ^ -- incorrectly specifying INTO target MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT INTO target DEFAULT VALUES; psql:include/ts_merge_query.sql:24: ERROR: syntax error at or near "INTO" LINE 5: INSERT INTO target DEFAULT VALUES; ^ -- Multiple VALUES clause MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (1,1), (2,2); psql:include/ts_merge_query.sql:30: ERROR: syntax error at or near "," LINE 5: INSERT VALUES (1,1), (2,2); ^ -- SELECT query for INSERT MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT SELECT (1, 1); psql:include/ts_merge_query.sql:36: ERROR: syntax error at or near "SELECT" LINE 5: INSERT SELECT (1, 1); ^ -- NOT MATCHED/UPDATE MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:42: ERROR: syntax error at or near "UPDATE" LINE 5: UPDATE SET balance = 0; ^ -- UPDATE tablename MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE target SET balance = 0; psql:include/ts_merge_query.sql:48: ERROR: syntax error at or near "target" LINE 5: UPDATE target SET balance = 0; ^ -- source and target names the same MERGE INTO target USING target ON tid = tid WHEN MATCHED THEN DO NOTHING; psql:include/ts_merge_query.sql:53: ERROR: name "target" specified more than once DETAIL: The name is used both as MERGE target table and data source. -- used in a CTE WITH foo AS ( MERGE INTO target USING source ON (true) WHEN MATCHED THEN DELETE ) SELECT * FROM foo; psql:include/ts_merge_query.sql:58: ERROR: MERGE not supported in WITH query LINE 1: WITH foo AS ( ^ -- used in COPY COPY ( MERGE INTO target USING source ON (true) WHEN MATCHED THEN DELETE ) TO stdout; psql:include/ts_merge_query.sql:63: ERROR: MERGE not supported in COPY -- unsupported relation types -- view CREATE VIEW tv AS SELECT * FROM target; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:72: ERROR: cannot execute MERGE on relation "tv" DETAIL: This operation is not supported for views. DROP VIEW tv; -- materialized view CREATE MATERIALIZED VIEW mv AS SELECT * FROM target; MERGE INTO mv t USING source s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:81: ERROR: cannot execute MERGE on relation "mv" DETAIL: This operation is not supported for materialized views. DROP MATERIALIZED VIEW mv; -- permissions MERGE INTO target USING source2 ON target.tid = source2.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:90: ERROR: permission denied for table source2 GRANT INSERT ON target TO regress_merge_no_privs; SET SESSION AUTHORIZATION regress_merge_no_privs; MERGE INTO target USING source2 ON target.tid = source2.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:99: ERROR: permission denied for table target GRANT UPDATE ON target2 TO regress_merge_privs; SET SESSION AUTHORIZATION regress_merge_privs; MERGE INTO target2 USING source ON target2.tid = source.sid WHEN MATCHED THEN DELETE; psql:include/ts_merge_query.sql:108: ERROR: permission denied for table target2 MERGE INTO target2 USING source ON target2.tid = source.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:114: ERROR: permission denied for table target2 -- check if the target can be accessed from source relation subquery; we should -- not be able to do so MERGE INTO target t USING (SELECT * FROM source WHERE t.tid > sid) s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:122: ERROR: invalid reference to FROM-clause entry for table "t" LINE 2: USING (SELECT * FROM source WHERE t.tid > sid) s ^ HINT: There is an entry for table "t", but it cannot be referenced from this part of the query. -- -- initial tests -- -- zero rows in source has no effect MERGE INTO target USING source ON target.tid = source.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; ROLLBACK; -- insert some non-matching source rows to work from INSERT INTO source VALUES (4, 40); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN DO NOTHING; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (5, 50); SELECT * FROM target ORDER BY tid; ROLLBACK; -- index plans INSERT INTO target SELECT generate_series(1000,2500), 0; ALTER TABLE target ADD PRIMARY KEY (tid); ANALYZE target; DELETE FROM target WHERE tid > 100; ANALYZE target; -- insert some matching source rows to work from INSERT INTO source VALUES (2, 5); INSERT INTO source VALUES (3, 20); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- equivalent of an UPDATE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; SELECT * FROM target ORDER BY tid; ROLLBACK; -- equivalent of a DELETE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DO NOTHING; SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, NULL); SELECT * FROM target ORDER BY tid; ROLLBACK; -- duplicate source row causes multiple target row update ERROR INSERT INTO source VALUES (2, 5); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:241: ERROR: MERGE command cannot affect row a second time HINT: Ensure that not more than one source row matches any one target row. ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; psql:include/ts_merge_query.sql:249: ERROR: MERGE command cannot affect row a second time HINT: Ensure that not more than one source row matches any one target row. ROLLBACK; -- remove duplicate MATCHED data from source data DELETE FROM source WHERE sid = 2; INSERT INTO source VALUES (2, 5); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- duplicate source row on INSERT should fail because of target_pkey INSERT INTO source VALUES (4, 40); BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, NULL); psql:include/ts_merge_query.sql:265: ERROR: duplicate key value violates unique constraint "2_2_target_pkey" DETAIL: Key (tid)=(4) already exists. SELECT * FROM target ORDER BY tid; psql:include/ts_merge_query.sql:266: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; -- remove duplicate NOT MATCHED data from source data DELETE FROM source WHERE sid = 4; INSERT INTO source VALUES (4, 40); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- remove constraints alter table target drop CONSTRAINT target_pkey; alter table target alter column tid drop not null; psql:include/ts_merge_query.sql:277: ERROR: cannot drop not-null constraint from a time-partitioned column -- multiple actions BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, 4) WHEN MATCHED THEN UPDATE SET balance = 0; SELECT * FROM target ORDER BY tid; ROLLBACK; -- should be equivalent BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0 WHEN NOT MATCHED THEN INSERT VALUES (4, 4); SELECT * FROM target ORDER BY tid; ROLLBACK; -- column references -- do a simple equivalent of an UPDATE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance + s.delta; SELECT * FROM target ORDER BY tid; ROLLBACK; -- do a simple equivalent of an INSERT SELECT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- and again with duplicate source rows INSERT INTO source VALUES (5, 50); INSERT INTO source VALUES (5, 50); -- do a simple equivalent of an INSERT SELECT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- removing duplicate source rows DELETE FROM source WHERE sid = 5; -- and again with explicitly identified column list BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- and again with a subtle error: referring to non-existent target row for NOT MATCHED MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (t.tid, s.delta); psql:include/ts_merge_query.sql:356: ERROR: invalid reference to FROM-clause entry for table "t" LINE 5: INSERT (tid, balance) VALUES (t.tid, s.delta); ^ HINT: There is an entry for table "t", but it cannot be referenced from this part of the query. -- and again with a constant ON clause BEGIN; MERGE INTO target t USING source AS s ON (SELECT true) WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (t.tid, s.delta); psql:include/ts_merge_query.sql:364: ERROR: invalid reference to FROM-clause entry for table "t" LINE 5: INSERT (tid, balance) VALUES (t.tid, s.delta); ^ HINT: There is an entry for table "t", but it cannot be referenced from this part of the query. SELECT * FROM target ORDER BY tid; psql:include/ts_merge_query.sql:365: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; -- now the classic UPSERT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance + s.delta WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- this time with a FALSE condition MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND FALSE THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; -- this time with an actual condition which returns false MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance <> 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; BEGIN; -- and now with a condition which returns true MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance = 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; ROLLBACK; -- conditions in the NOT MATCHED clause can only refer to source columns BEGIN; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND t.balance = 100 THEN INSERT (tid) VALUES (s.sid); psql:include/ts_merge_query.sql:408: ERROR: invalid reference to FROM-clause entry for table "t" LINE 3: WHEN NOT MATCHED AND t.balance = 100 THEN ^ HINT: There is an entry for table "t", but it cannot be referenced from this part of the query. SELECT * FROM wq_target; psql:include/ts_merge_query.sql:409: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance = 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; -- conditions in MATCHED clause can refer to both source and target SELECT * FROM wq_source; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND s.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check if AND works MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 AND s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 AND s.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check if OR works MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 OR s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 199 OR s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check source-side whole-row references BEGIN; MERGE INTO wq_target t USING wq_source s ON (t.tid = s.sid) WHEN matched and t = s or t.tid = s.sid THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; ROLLBACK; -- check if subqueries work in the conditions? MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance > (SELECT max(balance) FROM target) THEN UPDATE SET balance = t.balance + s.balance; -- check if we can access system columns in the conditions MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.xmin = t.xmax THEN UPDATE SET balance = t.balance + s.balance; psql:include/ts_merge_query.sql:477: ERROR: cannot use system column "xmin" in MERGE WHEN condition LINE 3: WHEN MATCHED AND t.xmin = t.xmax THEN ^ MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.tableoid >= 0 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; DROP TABLE wq_target CASCADE; DROP TABLE wq_source; -- test triggers create or replace function merge_trigfunc () returns trigger language plpgsql as $$ DECLARE line text; BEGIN SELECT INTO line format('%s %s %s trigger%s', TG_WHEN, TG_OP, TG_LEVEL, CASE WHEN TG_OP = 'INSERT' AND TG_LEVEL = 'ROW' THEN format(' row: %s', NEW) WHEN TG_OP = 'UPDATE' AND TG_LEVEL = 'ROW' THEN format(' row: %s -> %s', OLD, NEW) WHEN TG_OP = 'DELETE' AND TG_LEVEL = 'ROW' THEN format(' row: %s', OLD) END); RAISE NOTICE '%', line; IF (TG_WHEN = 'BEFORE' AND TG_LEVEL = 'ROW') THEN IF (TG_OP = 'DELETE') THEN RETURN OLD; ELSE RETURN NEW; END IF; ELSE RETURN NULL; END IF; END; $$; CREATE TRIGGER merge_bsi BEFORE INSERT ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bsu BEFORE UPDATE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bsd BEFORE DELETE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asi AFTER INSERT ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asu AFTER UPDATE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asd AFTER DELETE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bri BEFORE INSERT ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bru BEFORE UPDATE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_brd BEFORE DELETE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_ari AFTER INSERT ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_aru AFTER UPDATE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_ard AFTER DELETE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); -- now the classic UPSERT, with a DELETE BEGIN; UPDATE target SET balance = 0 WHERE tid = 3; --EXPLAIN (ANALYZE ON, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND t.balance > s.delta THEN UPDATE SET balance = t.balance - s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- Test behavior of triggers that turn UPDATE/DELETE into no-ops create or replace function skip_merge_op() returns trigger language plpgsql as $$ BEGIN RETURN NULL; END; $$; SELECT * FROM target full outer join source on (sid = tid); create trigger merge_skip BEFORE INSERT OR UPDATE or DELETE ON target FOR EACH ROW EXECUTE FUNCTION skip_merge_op(); DO $$ DECLARE result integer; BEGIN MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND s.sid = 3 THEN UPDATE SET balance = t.balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT VALUES (sid, delta); IF FOUND THEN RAISE NOTICE 'Found'; ELSE RAISE NOTICE 'Not found'; END IF; GET DIAGNOSTICS result := ROW_COUNT; RAISE NOTICE 'ROW_COUNT = %', result; END; $$; SELECT * FROM target FULL OUTER JOIN source ON (sid = tid); DROP TRIGGER merge_skip ON target; DROP FUNCTION skip_merge_op(); -- test from PL/pgSQL -- make sure MERGE INTO isn't interpreted to mean returning variables like SELECT INTO BEGIN; DO LANGUAGE plpgsql $$ BEGIN MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND t.balance > s.delta THEN UPDATE SET balance = t.balance - s.delta; END; $$; ROLLBACK; --source constants BEGIN; MERGE INTO target t USING (SELECT 9 AS sid, 57 AS delta) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; --source query BEGIN; MERGE INTO target t USING (SELECT sid, delta FROM source WHERE delta > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT sid, delta as newname FROM source WHERE delta > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.newname); SELECT * FROM target ORDER BY tid; ROLLBACK; --self-merge BEGIN; MERGE INTO target t1 USING target t2 ON t1.tid = t2.tid WHEN MATCHED THEN UPDATE SET balance = t1.balance + t2.balance WHEN NOT MATCHED THEN INSERT VALUES (t2.tid, t2.balance); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT tid as sid, balance as delta FROM target WHERE balance > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT sid, max(delta) AS delta FROM source GROUP BY sid HAVING count(*) = 1 ORDER BY sid ASC) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- plpgsql parameters and results BEGIN; CREATE FUNCTION merge_func (p_id integer, p_bal integer) RETURNS INTEGER LANGUAGE plpgsql AS $$ DECLARE result integer; BEGIN MERGE INTO target t USING (SELECT p_id AS sid) AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance - p_bal; IF FOUND THEN GET DIAGNOSTICS result := ROW_COUNT; END IF; RETURN result; END; $$; SELECT merge_func(3, 4); SELECT * FROM target ORDER BY tid; ROLLBACK; -- PREPARE BEGIN; prepare foom as merge into target t using (select 1 as sid) s on (t.tid = s.sid) when matched then update set balance = 1; psql:include/ts_merge_query.sql:685: ERROR: prepared statement "foom" already exists execute foom; psql:include/ts_merge_query.sql:686: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; BEGIN; PREPARE foom2 (integer, integer) AS MERGE INTO target t USING (SELECT 1) s ON t.tid = $1 WHEN MATCHED THEN UPDATE SET balance = $2; psql:include/ts_merge_query.sql:695: ERROR: prepared statement "foom2" already exists --EXPLAIN (ANALYZE ON, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) execute foom2 (1, 1); psql:include/ts_merge_query.sql:697: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; -- subqueries in source relation BEGIN; MERGE INTO sq_target t USING (SELECT * FROM sq_source) s ON tid = sid WHEN MATCHED AND t.balance > delta THEN UPDATE SET balance = t.balance + delta; SELECT * FROM sq_target ORDER BY tid; ROLLBACK; -- try a view CREATE VIEW v AS SELECT * FROM sq_source WHERE sid < 2; BEGIN; MERGE INTO sq_target USING v ON tid = sid WHEN MATCHED THEN UPDATE SET balance = v.balance + delta; SELECT * FROM sq_target ORDER BY tid; ROLLBACK; -- ambiguous reference to a column BEGIN; MERGE INTO sq_target USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; psql:include/ts_merge_query.sql:732: ERROR: column reference "balance" is ambiguous LINE 5: UPDATE SET balance = balance + delta ^ ROLLBACK; BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; SELECT * FROM sq_target; ROLLBACK; -- CTEs BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); WITH targq AS ( SELECT * FROM v ) MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; ROLLBACK; -- RETURNING BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE RETURNING *; psql:include/ts_merge_query.sql:778: ERROR: syntax error at or near "RETURNING" LINE 10: RETURNING *; ^ ROLLBACK; -- PG17-specific tests for views, returning and merge_action. These throw syntax errors for previous versions of Postgres. -- However, since the error is the same for both hypertables and regular tables, this test should still pass for previous versions. -- RETURNING INSERT INTO source(sid, delta) VALUES(1, 40), (5, 50); BEGIN; MERGE INTO target t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta); SELECT * from target; ROLLBACK; BEGIN; MERGE INTO target t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 1 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta) RETURNING merge_action(), t.*; psql:include/ts_merge_query.sql:811: ERROR: syntax error at or near "RETURNING" LINE 10: RETURNING merge_action(), t.*; ^ ROLLBACK; -- Views CREATE VIEW tv AS SELECT * FROM target; BEGIN; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta); psql:include/ts_merge_query.sql:826: ERROR: cannot execute MERGE on relation "tv" DETAIL: This operation is not supported for views. SELECT * from tv; psql:include/ts_merge_query.sql:827: ERROR: current transaction is aborted, commands ignored until end of transaction block SELECT * from target; -- should also update the underlying table psql:include/ts_merge_query.sql:828: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; BEGIN; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta) RETURNING merge_action(), t.*; psql:include/ts_merge_query.sql:841: ERROR: syntax error at or near "RETURNING" LINE 10: RETURNING merge_action(), t.*; ^ ROLLBACK; DROP VIEW tv; DELETE FROM source where sid in (1, 5); -- EXPLAIN CREATE TABLE ex_mtarget (a int, b int) WITH (autovacuum_enabled=off); CREATE TABLE ex_msource (a int, b int) WITH (autovacuum_enabled=off); INSERT INTO ex_mtarget SELECT i, i*10 FROM generate_series(1,100,2) i; INSERT INTO ex_msource SELECT i, i*10 FROM generate_series(1,100,1) i; CREATE FUNCTION explain_merge(query text) RETURNS SETOF text LANGUAGE plpgsql AS $$ DECLARE ln text; BEGIN FOR ln IN EXECUTE 'explain (analyze, timing off, summary off, buffers off, costs off) ' || query LOOP ln := regexp_replace(ln, '(Memory( Usage)?|Buckets|Batches): \S*', '\1: xxx', 'g'); RETURN NEXT ln; END LOOP; END; $$; -- only updates SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED THEN UPDATE SET b = t.b + 1'); -- only updates to selected tuples SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1'); -- updates + deletes SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1 WHEN MATCHED AND t.a >= 10 AND t.a <= 20 THEN DELETE'); -- only inserts SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN NOT MATCHED AND s.a < 10 THEN INSERT VALUES (a, b)'); -- all three SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1 WHEN MATCHED AND t.a >= 30 AND t.a <= 40 THEN DELETE WHEN NOT MATCHED AND s.a < 20 THEN INSERT VALUES (a, b)'); -- nothing SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a AND t.a < -1000 WHEN MATCHED AND t.a < 10 THEN DO NOTHING'); DROP TABLE ex_msource, ex_mtarget; DROP FUNCTION explain_merge(text); -- Subqueries BEGIN; MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED THEN UPDATE SET balance = (SELECT count(*) FROM sq_target); SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; BEGIN; MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND (SELECT count(*) > 0 FROM sq_target) THEN UPDATE SET balance = 42; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; BEGIN; MERGE INTO sq_target t USING v ON tid = sid AND (SELECT count(*) > 0 FROM sq_target) WHEN MATCHED THEN UPDATE SET balance = 42; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; -- Test RETURNING with subqueries BEGIN; MERGE INTO sq_target t USING v ON tid = sid AND (SELECT count(*) > 0 FROM sq_target) WHEN MATCHED THEN UPDATE SET balance = 42 RETURNING *; psql:include/ts_merge_query.sql:952: ERROR: syntax error at or near "RETURNING" LINE 6: RETURNING *; ^ SELECT * FROM sq_target WHERE tid = 1; psql:include/ts_merge_query.sql:953: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; DROP TABLE sq_target CASCADE; DROP TABLE sq_source CASCADE; CREATE TABLE pa_target (tid integer, balance float, val text) PARTITION BY LIST (tid); CREATE TABLE part1 PARTITION OF pa_target FOR VALUES IN (1,4) WITH (autovacuum_enabled=off); CREATE TABLE part2 PARTITION OF pa_target FOR VALUES IN (2,5,6) WITH (autovacuum_enabled=off); CREATE TABLE part3 PARTITION OF pa_target FOR VALUES IN (3,8,9) WITH (autovacuum_enabled=off); CREATE TABLE part4 PARTITION OF pa_target DEFAULT WITH (autovacuum_enabled=off); CREATE TABLE pa_source (sid integer, delta float); -- insert many rows to the source table INSERT INTO pa_source SELECT id, id * 10 FROM generate_series(1,14) AS id; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT id, id * 100, 'initial' FROM generate_series(1,14,2) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- same with a constant qual BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid AND tid = 1 WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- try updating the partition key column BEGIN; CREATE FUNCTION merge_func() RETURNS integer LANGUAGE plpgsql AS $$ DECLARE result integer; BEGIN MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET tid = tid + 1, balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); IF FOUND THEN GET DIAGNOSTICS result := ROW_COUNT; END IF; RETURN result; END; $$; SELECT merge_func(); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_target CASCADE; -- The target table is partitioned in the same way, but this time by attaching -- partitions which have columns in different order, dropped columns etc. CREATE TABLE pa_target (tid integer, balance float, val text) PARTITION BY LIST (tid); CREATE TABLE part1 (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE part2 (balance float, tid integer, val text) WITH (autovacuum_enabled=off); CREATE TABLE part3 (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE part4 (extraid text, tid integer, balance float, val text) WITH (autovacuum_enabled=off); ALTER TABLE part4 DROP COLUMN extraid; ALTER TABLE pa_target ATTACH PARTITION part1 FOR VALUES IN (1,4); ALTER TABLE pa_target ATTACH PARTITION part2 FOR VALUES IN (2,5,6); ALTER TABLE pa_target ATTACH PARTITION part3 FOR VALUES IN (3,8,9); ALTER TABLE pa_target ATTACH PARTITION part4 DEFAULT; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT id, id * 100, 'initial' FROM generate_series(1,14,2) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- same with a constant qual BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid AND tid IN (1, 5) WHEN MATCHED AND tid % 5 = 0 THEN DELETE WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- try updating the partition key column BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET tid = tid + 1, balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_source; DROP TABLE pa_target CASCADE; -- Sub-partitioning CREATE TABLE pa_target (logts timestamp, tid integer, balance float, val text) PARTITION BY RANGE (logts); CREATE TABLE part_m01 PARTITION OF pa_target FOR VALUES FROM ('2017-01-01') TO ('2017-02-01') PARTITION BY LIST (tid); CREATE TABLE part_m01_odd PARTITION OF part_m01 FOR VALUES IN (1,3,5,7,9) WITH (autovacuum_enabled=off); CREATE TABLE part_m01_even PARTITION OF part_m01 FOR VALUES IN (2,4,6,8) WITH (autovacuum_enabled=off); CREATE TABLE part_m02 PARTITION OF pa_target FOR VALUES FROM ('2017-02-01') TO ('2017-03-01') PARTITION BY LIST (tid); CREATE TABLE part_m02_odd PARTITION OF part_m02 FOR VALUES IN (1,3,5,7,9) WITH (autovacuum_enabled=off); CREATE TABLE part_m02_even PARTITION OF part_m02 FOR VALUES IN (2,4,6,8) WITH (autovacuum_enabled=off); CREATE TABLE pa_source (sid integer, delta float) WITH (autovacuum_enabled=off); -- insert many rows to the source table INSERT INTO pa_source SELECT id, id * 10 FROM generate_series(1,14) AS id; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT '2017-01-31', id, id * 100, 'initial' FROM generate_series(1,9,3) AS id; INSERT INTO pa_target SELECT '2017-02-28', id, id * 100, 'initial' FROM generate_series(2,9,3) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING (SELECT '2017-01-15' AS slogts, * FROM pa_source WHERE sid < 10) s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (slogts::timestamp, sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_source; DROP TABLE pa_target CASCADE; -- some complex joins on the source side -- source relation is an unaliased join MERGE INTO cj_target t USING cj_source1 s1 INNER JOIN cj_source2 s2 ON sid1 = sid2 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid1, delta, sval); -- try accessing columns from either side of the source join MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 AND scat = 20 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid2, delta, sval) WHEN MATCHED THEN DELETE; -- some simple expressions in INSERT targetlist MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid2, delta + scat, sval) WHEN MATCHED THEN UPDATE SET val = val || ' updated by merge'; MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 AND scat = 20 ON t.tid = sid1 WHEN MATCHED THEN UPDATE SET val = val || ' ' || delta::text; SELECT * FROM cj_target ORDER BY tid; ALTER TABLE cj_source1 RENAME COLUMN sid1 TO sid; ALTER TABLE cj_source2 RENAME COLUMN sid2 TO sid; TRUNCATE cj_target; MERGE INTO cj_target t USING cj_source1 s1 INNER JOIN cj_source2 s2 ON s1.sid = s2.sid ON t.tid = s1.sid WHEN NOT MATCHED THEN INSERT VALUES (s2.sid, delta, sval); DROP TABLE cj_source2, cj_source1; DROP TABLE cj_target CASCADE; -- Function scans MERGE INTO fs_target t USING generate_series(1,100,1) AS id ON t.a = id WHEN MATCHED THEN UPDATE SET b = b + id WHEN NOT MATCHED THEN INSERT VALUES (id, -1); MERGE INTO fs_target t USING generate_series(1,100,2) AS id ON t.a = id WHEN MATCHED THEN UPDATE SET b = b + id, c = 'updated '|| id.*::text WHEN NOT MATCHED THEN INSERT VALUES (id, -1, 'inserted ' || id.*::text); SELECT count(*) FROM fs_target; DROP TABLE fs_target CASCADE; -- SERIALIZABLE test -- handled in isolation tests -- Inheritance-based partitioning CREATE TABLE measurement ( city_id int not null, logdate date not null, peaktemp int, unitsales int ) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2006m02 ( CHECK ( logdate >= DATE '2006-02-01' AND logdate < DATE '2006-03-01' ) ) INHERITS (measurement) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2006m03 ( CHECK ( logdate >= DATE '2006-03-01' AND logdate < DATE '2006-04-01' ) ) INHERITS (measurement) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2007m01 ( filler text, peaktemp int, logdate date not null, city_id int not null, unitsales int CHECK ( logdate >= DATE '2007-01-01' AND logdate < DATE '2007-02-01') ) WITH (autovacuum_enabled=off); ALTER TABLE measurement_y2007m01 DROP COLUMN filler; ALTER TABLE measurement_y2007m01 INHERIT measurement; INSERT INTO measurement VALUES (0, '2005-07-21', 5, 15); CREATE OR REPLACE FUNCTION measurement_insert_trigger() RETURNS TRIGGER AS $$ BEGIN IF ( NEW.logdate >= DATE '2006-02-01' AND NEW.logdate < DATE '2006-03-01' ) THEN INSERT INTO measurement_y2006m02 VALUES (NEW.*); ELSIF ( NEW.logdate >= DATE '2006-03-01' AND NEW.logdate < DATE '2006-04-01' ) THEN INSERT INTO measurement_y2006m03 VALUES (NEW.*); ELSIF ( NEW.logdate >= DATE '2007-01-01' AND NEW.logdate < DATE '2007-02-01' ) THEN INSERT INTO measurement_y2007m01 (city_id, logdate, peaktemp, unitsales) VALUES (NEW.*); ELSE RAISE EXCEPTION 'Date out of range. Fix the measurement_insert_trigger() function!'; END IF; RETURN NULL; END; $$ LANGUAGE plpgsql ; CREATE TRIGGER insert_measurement_trigger BEFORE INSERT ON measurement FOR EACH ROW EXECUTE PROCEDURE measurement_insert_trigger(); INSERT INTO measurement VALUES (1, '2006-02-10', 35, 10); INSERT INTO measurement VALUES (1, '2006-02-16', 45, 20); INSERT INTO measurement VALUES (1, '2006-03-17', 25, 10); INSERT INTO measurement VALUES (1, '2006-03-27', 15, 40); INSERT INTO measurement VALUES (1, '2007-01-15', 10, 10); INSERT INTO measurement VALUES (1, '2007-01-17', 10, 10); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate; CREATE TABLE new_measurement (LIKE measurement) WITH (autovacuum_enabled=off); INSERT INTO new_measurement VALUES (0, '2005-07-21', 25, 20); INSERT INTO new_measurement VALUES (1, '2006-03-01', 20, 10); INSERT INTO new_measurement VALUES (1, '2006-02-16', 50, 10); INSERT INTO new_measurement VALUES (2, '2006-02-10', 20, 20); INSERT INTO new_measurement VALUES (1, '2006-03-27', NULL, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-17', NULL, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-15', 5, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-16', 10, 10); BEGIN; MERGE INTO ONLY measurement m USING new_measurement nm ON (m.city_id = nm.city_id and m.logdate=nm.logdate) WHEN MATCHED AND nm.peaktemp IS NULL THEN DELETE WHEN MATCHED THEN UPDATE SET peaktemp = greatest(m.peaktemp, nm.peaktemp), unitsales = m.unitsales + coalesce(nm.unitsales, 0) WHEN NOT MATCHED THEN INSERT (city_id, logdate, peaktemp, unitsales) VALUES (city_id, logdate, peaktemp, unitsales); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate, peaktemp; ROLLBACK; MERGE into measurement m USING new_measurement nm ON (m.city_id = nm.city_id and m.logdate=nm.logdate) WHEN MATCHED AND nm.peaktemp IS NULL THEN DELETE WHEN MATCHED THEN UPDATE SET peaktemp = greatest(m.peaktemp, nm.peaktemp), unitsales = m.unitsales + coalesce(nm.unitsales, 0) WHEN NOT MATCHED THEN INSERT (city_id, logdate, peaktemp, unitsales) VALUES (city_id, logdate, peaktemp, unitsales); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate; BEGIN; MERGE INTO new_measurement nm USING ONLY measurement m ON (nm.city_id = m.city_id and nm.logdate=m.logdate) WHEN MATCHED THEN DELETE; SELECT * FROM new_measurement ORDER BY city_id, logdate; ROLLBACK; MERGE INTO new_measurement nm USING measurement m ON (nm.city_id = m.city_id and nm.logdate=m.logdate) WHEN MATCHED THEN DELETE; SELECT * FROM new_measurement ORDER BY city_id, logdate; DROP TABLE measurement, new_measurement CASCADE; DROP FUNCTION measurement_insert_trigger(); RESET SESSION AUTHORIZATION; DROP TABLE target CASCADE; DROP TABLE target2 CASCADE; DROP TABLE source, source2; DROP FUNCTION merge_trigfunc(); REVOKE CREATE ON SCHEMA public FROM regress_merge_privs; DROP USER regress_merge_privs; DROP USER regress_merge_no_privs; \o :DIFF_CMD ================================================ FILE: test/expected/ts_merge-16.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER \set ON_ERROR_STOP 0 \set VERBOSITY default SET client_min_messages TO error; \set TEST_BASE_NAME ts_merge SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') AS "TEST_LOAD_NAME", format('include/%s_load_ht.sql', :'TEST_BASE_NAME') AS "TEST_LOAD_HT_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') AS "TEST_QUERY_NAME", format('%s/results/%s_ht_results.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') AS "TEST_RESULTS_WITH_HYPERTABLE", format('%s/results/%s_results.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') AS "TEST_RESULTS_WITH_NO_HYPERTABLE" \gset SELECT format('\! diff -u --label "Base pg table results" --label "Hypertable results" %s %s', :'TEST_RESULTS_WITH_HYPERTABLE', :'TEST_RESULTS_WITH_NO_HYPERTABLE') AS "DIFF_CMD" \gset \ir :TEST_LOAD_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE USER regress_merge_privs; CREATE USER regress_merge_no_privs; DROP TABLE IF EXISTS target; DROP TABLE IF EXISTS source; CREATE TABLE target (tid integer, balance integer) WITH (autovacuum_enabled=off); CREATE TABLE source (sid integer, delta integer) -- no index WITH (autovacuum_enabled=off); INSERT INTO target VALUES (1, 10); INSERT INTO target VALUES (2, 20); INSERT INTO target VALUES (3, 30); SELECT t.ctid is not null as matched, t.*, s.* FROM source s FULL OUTER JOIN target t ON s.sid = t.tid ORDER BY t.tid, s.sid; matched | tid | balance | sid | delta ---------+-----+---------+-----+------- t | 1 | 10 | | t | 2 | 20 | | t | 3 | 30 | | ALTER TABLE target OWNER TO regress_merge_privs; ALTER TABLE source OWNER TO regress_merge_privs; CREATE TABLE target2 (tid integer, balance integer) WITH (autovacuum_enabled=off); CREATE TABLE source2 (sid integer, delta integer) WITH (autovacuum_enabled=off); ALTER TABLE target2 OWNER TO regress_merge_no_privs; ALTER TABLE source2 OWNER TO regress_merge_no_privs; GRANT INSERT ON target TO regress_merge_no_privs; GRANT CREATE ON SCHEMA public TO regress_merge_privs; SET SESSION AUTHORIZATION regress_merge_privs; CREATE TABLE sq_target (tid integer NOT NULL, balance integer) WITH (autovacuum_enabled=off); CREATE TABLE sq_source (delta integer, sid integer, balance integer DEFAULT 0) WITH (autovacuum_enabled=off); INSERT INTO sq_target(tid, balance) VALUES (1,100), (2,200), (3,300); INSERT INTO sq_source(sid, delta) VALUES (1,10), (2,20), (4,40); -- conditional WHEN clause CREATE TABLE wq_target (tid integer not null, balance integer DEFAULT -1) WITH (autovacuum_enabled=off); CREATE TABLE wq_source (balance integer, sid integer) WITH (autovacuum_enabled=off); INSERT INTO wq_source (sid, balance) VALUES (1, 100); CREATE TABLE cj_target (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE cj_source1 (sid1 integer, scat integer, delta integer) WITH (autovacuum_enabled=off); CREATE TABLE cj_source2 (sid2 integer, sval text) WITH (autovacuum_enabled=off); INSERT INTO cj_source1 VALUES (1, 10, 100); INSERT INTO cj_source1 VALUES (1, 20, 200); INSERT INTO cj_source1 VALUES (2, 20, 300); INSERT INTO cj_source1 VALUES (3, 10, 400); INSERT INTO cj_source2 VALUES (1, 'initial source2'); INSERT INTO cj_source2 VALUES (2, 'initial source2'); INSERT INTO cj_source2 VALUES (3, 'initial source2'); CREATE TABLE fs_target (a int, b int, c text) WITH (autovacuum_enabled=off); -- run tests on normal table \o :TEST_RESULTS_WITH_NO_HYPERTABLE \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- -- Errors -- MERGE INTO target t RANDOMWORD USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:12: ERROR: syntax error at or near "RANDOMWORD" LINE 1: MERGE INTO target t RANDOMWORD ^ -- MATCHED/INSERT error MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:18: ERROR: syntax error at or near "INSERT" LINE 5: INSERT DEFAULT VALUES; ^ -- incorrectly specifying INTO target MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT INTO target DEFAULT VALUES; psql:include/ts_merge_query.sql:24: ERROR: syntax error at or near "INTO" LINE 5: INSERT INTO target DEFAULT VALUES; ^ -- Multiple VALUES clause MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (1,1), (2,2); psql:include/ts_merge_query.sql:30: ERROR: syntax error at or near "," LINE 5: INSERT VALUES (1,1), (2,2); ^ -- SELECT query for INSERT MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT SELECT (1, 1); psql:include/ts_merge_query.sql:36: ERROR: syntax error at or near "SELECT" LINE 5: INSERT SELECT (1, 1); ^ -- NOT MATCHED/UPDATE MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:42: ERROR: syntax error at or near "UPDATE" LINE 5: UPDATE SET balance = 0; ^ -- UPDATE tablename MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE target SET balance = 0; psql:include/ts_merge_query.sql:48: ERROR: syntax error at or near "target" LINE 5: UPDATE target SET balance = 0; ^ -- source and target names the same MERGE INTO target USING target ON tid = tid WHEN MATCHED THEN DO NOTHING; psql:include/ts_merge_query.sql:53: ERROR: name "target" specified more than once DETAIL: The name is used both as MERGE target table and data source. -- used in a CTE WITH foo AS ( MERGE INTO target USING source ON (true) WHEN MATCHED THEN DELETE ) SELECT * FROM foo; psql:include/ts_merge_query.sql:58: ERROR: MERGE not supported in WITH query LINE 1: WITH foo AS ( ^ -- used in COPY COPY ( MERGE INTO target USING source ON (true) WHEN MATCHED THEN DELETE ) TO stdout; psql:include/ts_merge_query.sql:63: ERROR: MERGE not supported in COPY -- unsupported relation types -- view CREATE VIEW tv AS SELECT * FROM target; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:72: ERROR: cannot execute MERGE on relation "tv" DETAIL: This operation is not supported for views. DROP VIEW tv; -- materialized view CREATE MATERIALIZED VIEW mv AS SELECT * FROM target; MERGE INTO mv t USING source s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:81: ERROR: cannot execute MERGE on relation "mv" DETAIL: This operation is not supported for materialized views. DROP MATERIALIZED VIEW mv; -- permissions MERGE INTO target USING source2 ON target.tid = source2.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:90: ERROR: permission denied for table source2 GRANT INSERT ON target TO regress_merge_no_privs; SET SESSION AUTHORIZATION regress_merge_no_privs; MERGE INTO target USING source2 ON target.tid = source2.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:99: ERROR: permission denied for table target GRANT UPDATE ON target2 TO regress_merge_privs; SET SESSION AUTHORIZATION regress_merge_privs; MERGE INTO target2 USING source ON target2.tid = source.sid WHEN MATCHED THEN DELETE; psql:include/ts_merge_query.sql:108: ERROR: permission denied for table target2 MERGE INTO target2 USING source ON target2.tid = source.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:114: ERROR: permission denied for table target2 -- check if the target can be accessed from source relation subquery; we should -- not be able to do so MERGE INTO target t USING (SELECT * FROM source WHERE t.tid > sid) s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:122: ERROR: invalid reference to FROM-clause entry for table "t" LINE 2: USING (SELECT * FROM source WHERE t.tid > sid) s ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. -- -- initial tests -- -- zero rows in source has no effect MERGE INTO target USING source ON target.tid = source.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; ROLLBACK; -- insert some non-matching source rows to work from INSERT INTO source VALUES (4, 40); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN DO NOTHING; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (5, 50); SELECT * FROM target ORDER BY tid; ROLLBACK; -- index plans INSERT INTO target SELECT generate_series(1000,2500), 0; ALTER TABLE target ADD PRIMARY KEY (tid); ANALYZE target; DELETE FROM target WHERE tid > 100; ANALYZE target; -- insert some matching source rows to work from INSERT INTO source VALUES (2, 5); INSERT INTO source VALUES (3, 20); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- equivalent of an UPDATE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; SELECT * FROM target ORDER BY tid; ROLLBACK; -- equivalent of a DELETE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DO NOTHING; SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, NULL); SELECT * FROM target ORDER BY tid; ROLLBACK; -- duplicate source row causes multiple target row update ERROR INSERT INTO source VALUES (2, 5); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:241: ERROR: MERGE command cannot affect row a second time HINT: Ensure that not more than one source row matches any one target row. ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; psql:include/ts_merge_query.sql:249: ERROR: MERGE command cannot affect row a second time HINT: Ensure that not more than one source row matches any one target row. ROLLBACK; -- remove duplicate MATCHED data from source data DELETE FROM source WHERE sid = 2; INSERT INTO source VALUES (2, 5); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- duplicate source row on INSERT should fail because of target_pkey INSERT INTO source VALUES (4, 40); BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, NULL); psql:include/ts_merge_query.sql:265: ERROR: duplicate key value violates unique constraint "target_pkey" DETAIL: Key (tid)=(4) already exists. SELECT * FROM target ORDER BY tid; psql:include/ts_merge_query.sql:266: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; -- remove duplicate NOT MATCHED data from source data DELETE FROM source WHERE sid = 4; INSERT INTO source VALUES (4, 40); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- remove constraints alter table target drop CONSTRAINT target_pkey; alter table target alter column tid drop not null; -- multiple actions BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, 4) WHEN MATCHED THEN UPDATE SET balance = 0; SELECT * FROM target ORDER BY tid; ROLLBACK; -- should be equivalent BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0 WHEN NOT MATCHED THEN INSERT VALUES (4, 4); SELECT * FROM target ORDER BY tid; ROLLBACK; -- column references -- do a simple equivalent of an UPDATE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance + s.delta; SELECT * FROM target ORDER BY tid; ROLLBACK; -- do a simple equivalent of an INSERT SELECT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- and again with duplicate source rows INSERT INTO source VALUES (5, 50); INSERT INTO source VALUES (5, 50); -- do a simple equivalent of an INSERT SELECT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- removing duplicate source rows DELETE FROM source WHERE sid = 5; -- and again with explicitly identified column list BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- and again with a subtle error: referring to non-existent target row for NOT MATCHED MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (t.tid, s.delta); psql:include/ts_merge_query.sql:356: ERROR: invalid reference to FROM-clause entry for table "t" LINE 5: INSERT (tid, balance) VALUES (t.tid, s.delta); ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. -- and again with a constant ON clause BEGIN; MERGE INTO target t USING source AS s ON (SELECT true) WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (t.tid, s.delta); psql:include/ts_merge_query.sql:364: ERROR: invalid reference to FROM-clause entry for table "t" LINE 5: INSERT (tid, balance) VALUES (t.tid, s.delta); ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. SELECT * FROM target ORDER BY tid; psql:include/ts_merge_query.sql:365: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; -- now the classic UPSERT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance + s.delta WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- this time with a FALSE condition MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND FALSE THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; -- this time with an actual condition which returns false MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance <> 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; BEGIN; -- and now with a condition which returns true MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance = 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; ROLLBACK; -- conditions in the NOT MATCHED clause can only refer to source columns BEGIN; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND t.balance = 100 THEN INSERT (tid) VALUES (s.sid); psql:include/ts_merge_query.sql:408: ERROR: invalid reference to FROM-clause entry for table "t" LINE 3: WHEN NOT MATCHED AND t.balance = 100 THEN ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. SELECT * FROM wq_target; psql:include/ts_merge_query.sql:409: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance = 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; -- conditions in MATCHED clause can refer to both source and target SELECT * FROM wq_source; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND s.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check if AND works MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 AND s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 AND s.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check if OR works MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 OR s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 199 OR s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check source-side whole-row references BEGIN; MERGE INTO wq_target t USING wq_source s ON (t.tid = s.sid) WHEN matched and t = s or t.tid = s.sid THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; ROLLBACK; -- check if subqueries work in the conditions? MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance > (SELECT max(balance) FROM target) THEN UPDATE SET balance = t.balance + s.balance; -- check if we can access system columns in the conditions MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.xmin = t.xmax THEN UPDATE SET balance = t.balance + s.balance; psql:include/ts_merge_query.sql:477: ERROR: cannot use system column "xmin" in MERGE WHEN condition LINE 3: WHEN MATCHED AND t.xmin = t.xmax THEN ^ MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.tableoid >= 0 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; DROP TABLE wq_target CASCADE; DROP TABLE wq_source; -- test triggers create or replace function merge_trigfunc () returns trigger language plpgsql as $$ DECLARE line text; BEGIN SELECT INTO line format('%s %s %s trigger%s', TG_WHEN, TG_OP, TG_LEVEL, CASE WHEN TG_OP = 'INSERT' AND TG_LEVEL = 'ROW' THEN format(' row: %s', NEW) WHEN TG_OP = 'UPDATE' AND TG_LEVEL = 'ROW' THEN format(' row: %s -> %s', OLD, NEW) WHEN TG_OP = 'DELETE' AND TG_LEVEL = 'ROW' THEN format(' row: %s', OLD) END); RAISE NOTICE '%', line; IF (TG_WHEN = 'BEFORE' AND TG_LEVEL = 'ROW') THEN IF (TG_OP = 'DELETE') THEN RETURN OLD; ELSE RETURN NEW; END IF; ELSE RETURN NULL; END IF; END; $$; CREATE TRIGGER merge_bsi BEFORE INSERT ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bsu BEFORE UPDATE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bsd BEFORE DELETE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asi AFTER INSERT ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asu AFTER UPDATE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asd AFTER DELETE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bri BEFORE INSERT ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bru BEFORE UPDATE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_brd BEFORE DELETE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_ari AFTER INSERT ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_aru AFTER UPDATE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_ard AFTER DELETE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); -- now the classic UPSERT, with a DELETE BEGIN; UPDATE target SET balance = 0 WHERE tid = 3; --EXPLAIN (ANALYZE ON, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND t.balance > s.delta THEN UPDATE SET balance = t.balance - s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- Test behavior of triggers that turn UPDATE/DELETE into no-ops create or replace function skip_merge_op() returns trigger language plpgsql as $$ BEGIN RETURN NULL; END; $$; SELECT * FROM target full outer join source on (sid = tid); create trigger merge_skip BEFORE INSERT OR UPDATE or DELETE ON target FOR EACH ROW EXECUTE FUNCTION skip_merge_op(); DO $$ DECLARE result integer; BEGIN MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND s.sid = 3 THEN UPDATE SET balance = t.balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT VALUES (sid, delta); IF FOUND THEN RAISE NOTICE 'Found'; ELSE RAISE NOTICE 'Not found'; END IF; GET DIAGNOSTICS result := ROW_COUNT; RAISE NOTICE 'ROW_COUNT = %', result; END; $$; SELECT * FROM target FULL OUTER JOIN source ON (sid = tid); DROP TRIGGER merge_skip ON target; DROP FUNCTION skip_merge_op(); -- test from PL/pgSQL -- make sure MERGE INTO isn't interpreted to mean returning variables like SELECT INTO BEGIN; DO LANGUAGE plpgsql $$ BEGIN MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND t.balance > s.delta THEN UPDATE SET balance = t.balance - s.delta; END; $$; ROLLBACK; --source constants BEGIN; MERGE INTO target t USING (SELECT 9 AS sid, 57 AS delta) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; --source query BEGIN; MERGE INTO target t USING (SELECT sid, delta FROM source WHERE delta > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT sid, delta as newname FROM source WHERE delta > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.newname); SELECT * FROM target ORDER BY tid; ROLLBACK; --self-merge BEGIN; MERGE INTO target t1 USING target t2 ON t1.tid = t2.tid WHEN MATCHED THEN UPDATE SET balance = t1.balance + t2.balance WHEN NOT MATCHED THEN INSERT VALUES (t2.tid, t2.balance); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT tid as sid, balance as delta FROM target WHERE balance > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT sid, max(delta) AS delta FROM source GROUP BY sid HAVING count(*) = 1 ORDER BY sid ASC) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- plpgsql parameters and results BEGIN; CREATE FUNCTION merge_func (p_id integer, p_bal integer) RETURNS INTEGER LANGUAGE plpgsql AS $$ DECLARE result integer; BEGIN MERGE INTO target t USING (SELECT p_id AS sid) AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance - p_bal; IF FOUND THEN GET DIAGNOSTICS result := ROW_COUNT; END IF; RETURN result; END; $$; SELECT merge_func(3, 4); SELECT * FROM target ORDER BY tid; ROLLBACK; -- PREPARE BEGIN; prepare foom as merge into target t using (select 1 as sid) s on (t.tid = s.sid) when matched then update set balance = 1; execute foom; ROLLBACK; BEGIN; PREPARE foom2 (integer, integer) AS MERGE INTO target t USING (SELECT 1) s ON t.tid = $1 WHEN MATCHED THEN UPDATE SET balance = $2; --EXPLAIN (ANALYZE ON, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) execute foom2 (1, 1); ROLLBACK; -- subqueries in source relation BEGIN; MERGE INTO sq_target t USING (SELECT * FROM sq_source) s ON tid = sid WHEN MATCHED AND t.balance > delta THEN UPDATE SET balance = t.balance + delta; SELECT * FROM sq_target ORDER BY tid; ROLLBACK; -- try a view CREATE VIEW v AS SELECT * FROM sq_source WHERE sid < 2; BEGIN; MERGE INTO sq_target USING v ON tid = sid WHEN MATCHED THEN UPDATE SET balance = v.balance + delta; SELECT * FROM sq_target ORDER BY tid; ROLLBACK; -- ambiguous reference to a column BEGIN; MERGE INTO sq_target USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; psql:include/ts_merge_query.sql:732: ERROR: column reference "balance" is ambiguous LINE 5: UPDATE SET balance = balance + delta ^ ROLLBACK; BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; SELECT * FROM sq_target; ROLLBACK; -- CTEs BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); WITH targq AS ( SELECT * FROM v ) MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; ROLLBACK; -- RETURNING BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE RETURNING *; psql:include/ts_merge_query.sql:778: ERROR: syntax error at or near "RETURNING" LINE 10: RETURNING *; ^ ROLLBACK; -- PG17-specific tests for views, returning and merge_action. These throw syntax errors for previous versions of Postgres. -- However, since the error is the same for both hypertables and regular tables, this test should still pass for previous versions. -- RETURNING INSERT INTO source(sid, delta) VALUES(1, 40), (5, 50); BEGIN; MERGE INTO target t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta); SELECT * from target; ROLLBACK; BEGIN; MERGE INTO target t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 1 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta) RETURNING merge_action(), t.*; psql:include/ts_merge_query.sql:811: ERROR: syntax error at or near "RETURNING" LINE 10: RETURNING merge_action(), t.*; ^ ROLLBACK; -- Views CREATE VIEW tv AS SELECT * FROM target; BEGIN; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta); psql:include/ts_merge_query.sql:826: ERROR: cannot execute MERGE on relation "tv" DETAIL: This operation is not supported for views. SELECT * from tv; psql:include/ts_merge_query.sql:827: ERROR: current transaction is aborted, commands ignored until end of transaction block SELECT * from target; -- should also update the underlying table psql:include/ts_merge_query.sql:828: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; BEGIN; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta) RETURNING merge_action(), t.*; psql:include/ts_merge_query.sql:841: ERROR: syntax error at or near "RETURNING" LINE 10: RETURNING merge_action(), t.*; ^ ROLLBACK; DROP VIEW tv; DELETE FROM source where sid in (1, 5); -- EXPLAIN CREATE TABLE ex_mtarget (a int, b int) WITH (autovacuum_enabled=off); CREATE TABLE ex_msource (a int, b int) WITH (autovacuum_enabled=off); INSERT INTO ex_mtarget SELECT i, i*10 FROM generate_series(1,100,2) i; INSERT INTO ex_msource SELECT i, i*10 FROM generate_series(1,100,1) i; CREATE FUNCTION explain_merge(query text) RETURNS SETOF text LANGUAGE plpgsql AS $$ DECLARE ln text; BEGIN FOR ln IN EXECUTE 'explain (analyze, timing off, summary off, buffers off, costs off) ' || query LOOP ln := regexp_replace(ln, '(Memory( Usage)?|Buckets|Batches): \S*', '\1: xxx', 'g'); RETURN NEXT ln; END LOOP; END; $$; -- only updates SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED THEN UPDATE SET b = t.b + 1'); -- only updates to selected tuples SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1'); -- updates + deletes SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1 WHEN MATCHED AND t.a >= 10 AND t.a <= 20 THEN DELETE'); -- only inserts SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN NOT MATCHED AND s.a < 10 THEN INSERT VALUES (a, b)'); -- all three SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1 WHEN MATCHED AND t.a >= 30 AND t.a <= 40 THEN DELETE WHEN NOT MATCHED AND s.a < 20 THEN INSERT VALUES (a, b)'); -- nothing SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a AND t.a < -1000 WHEN MATCHED AND t.a < 10 THEN DO NOTHING'); DROP TABLE ex_msource, ex_mtarget; DROP FUNCTION explain_merge(text); -- Subqueries BEGIN; MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED THEN UPDATE SET balance = (SELECT count(*) FROM sq_target); SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; BEGIN; MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND (SELECT count(*) > 0 FROM sq_target) THEN UPDATE SET balance = 42; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; BEGIN; MERGE INTO sq_target t USING v ON tid = sid AND (SELECT count(*) > 0 FROM sq_target) WHEN MATCHED THEN UPDATE SET balance = 42; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; -- Test RETURNING with subqueries BEGIN; MERGE INTO sq_target t USING v ON tid = sid AND (SELECT count(*) > 0 FROM sq_target) WHEN MATCHED THEN UPDATE SET balance = 42 RETURNING *; psql:include/ts_merge_query.sql:952: ERROR: syntax error at or near "RETURNING" LINE 6: RETURNING *; ^ SELECT * FROM sq_target WHERE tid = 1; psql:include/ts_merge_query.sql:953: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; DROP TABLE sq_target CASCADE; DROP TABLE sq_source CASCADE; CREATE TABLE pa_target (tid integer, balance float, val text) PARTITION BY LIST (tid); CREATE TABLE part1 PARTITION OF pa_target FOR VALUES IN (1,4) WITH (autovacuum_enabled=off); CREATE TABLE part2 PARTITION OF pa_target FOR VALUES IN (2,5,6) WITH (autovacuum_enabled=off); CREATE TABLE part3 PARTITION OF pa_target FOR VALUES IN (3,8,9) WITH (autovacuum_enabled=off); CREATE TABLE part4 PARTITION OF pa_target DEFAULT WITH (autovacuum_enabled=off); CREATE TABLE pa_source (sid integer, delta float); -- insert many rows to the source table INSERT INTO pa_source SELECT id, id * 10 FROM generate_series(1,14) AS id; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT id, id * 100, 'initial' FROM generate_series(1,14,2) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- same with a constant qual BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid AND tid = 1 WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- try updating the partition key column BEGIN; CREATE FUNCTION merge_func() RETURNS integer LANGUAGE plpgsql AS $$ DECLARE result integer; BEGIN MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET tid = tid + 1, balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); IF FOUND THEN GET DIAGNOSTICS result := ROW_COUNT; END IF; RETURN result; END; $$; SELECT merge_func(); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_target CASCADE; -- The target table is partitioned in the same way, but this time by attaching -- partitions which have columns in different order, dropped columns etc. CREATE TABLE pa_target (tid integer, balance float, val text) PARTITION BY LIST (tid); CREATE TABLE part1 (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE part2 (balance float, tid integer, val text) WITH (autovacuum_enabled=off); CREATE TABLE part3 (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE part4 (extraid text, tid integer, balance float, val text) WITH (autovacuum_enabled=off); ALTER TABLE part4 DROP COLUMN extraid; ALTER TABLE pa_target ATTACH PARTITION part1 FOR VALUES IN (1,4); ALTER TABLE pa_target ATTACH PARTITION part2 FOR VALUES IN (2,5,6); ALTER TABLE pa_target ATTACH PARTITION part3 FOR VALUES IN (3,8,9); ALTER TABLE pa_target ATTACH PARTITION part4 DEFAULT; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT id, id * 100, 'initial' FROM generate_series(1,14,2) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- same with a constant qual BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid AND tid IN (1, 5) WHEN MATCHED AND tid % 5 = 0 THEN DELETE WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- try updating the partition key column BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET tid = tid + 1, balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_source; DROP TABLE pa_target CASCADE; -- Sub-partitioning CREATE TABLE pa_target (logts timestamp, tid integer, balance float, val text) PARTITION BY RANGE (logts); CREATE TABLE part_m01 PARTITION OF pa_target FOR VALUES FROM ('2017-01-01') TO ('2017-02-01') PARTITION BY LIST (tid); CREATE TABLE part_m01_odd PARTITION OF part_m01 FOR VALUES IN (1,3,5,7,9) WITH (autovacuum_enabled=off); CREATE TABLE part_m01_even PARTITION OF part_m01 FOR VALUES IN (2,4,6,8) WITH (autovacuum_enabled=off); CREATE TABLE part_m02 PARTITION OF pa_target FOR VALUES FROM ('2017-02-01') TO ('2017-03-01') PARTITION BY LIST (tid); CREATE TABLE part_m02_odd PARTITION OF part_m02 FOR VALUES IN (1,3,5,7,9) WITH (autovacuum_enabled=off); CREATE TABLE part_m02_even PARTITION OF part_m02 FOR VALUES IN (2,4,6,8) WITH (autovacuum_enabled=off); CREATE TABLE pa_source (sid integer, delta float) WITH (autovacuum_enabled=off); -- insert many rows to the source table INSERT INTO pa_source SELECT id, id * 10 FROM generate_series(1,14) AS id; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT '2017-01-31', id, id * 100, 'initial' FROM generate_series(1,9,3) AS id; INSERT INTO pa_target SELECT '2017-02-28', id, id * 100, 'initial' FROM generate_series(2,9,3) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING (SELECT '2017-01-15' AS slogts, * FROM pa_source WHERE sid < 10) s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (slogts::timestamp, sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_source; DROP TABLE pa_target CASCADE; -- some complex joins on the source side -- source relation is an unaliased join MERGE INTO cj_target t USING cj_source1 s1 INNER JOIN cj_source2 s2 ON sid1 = sid2 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid1, delta, sval); -- try accessing columns from either side of the source join MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 AND scat = 20 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid2, delta, sval) WHEN MATCHED THEN DELETE; -- some simple expressions in INSERT targetlist MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid2, delta + scat, sval) WHEN MATCHED THEN UPDATE SET val = val || ' updated by merge'; MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 AND scat = 20 ON t.tid = sid1 WHEN MATCHED THEN UPDATE SET val = val || ' ' || delta::text; SELECT * FROM cj_target ORDER BY tid; ALTER TABLE cj_source1 RENAME COLUMN sid1 TO sid; ALTER TABLE cj_source2 RENAME COLUMN sid2 TO sid; TRUNCATE cj_target; MERGE INTO cj_target t USING cj_source1 s1 INNER JOIN cj_source2 s2 ON s1.sid = s2.sid ON t.tid = s1.sid WHEN NOT MATCHED THEN INSERT VALUES (s2.sid, delta, sval); DROP TABLE cj_source2, cj_source1; DROP TABLE cj_target CASCADE; -- Function scans MERGE INTO fs_target t USING generate_series(1,100,1) AS id ON t.a = id WHEN MATCHED THEN UPDATE SET b = b + id WHEN NOT MATCHED THEN INSERT VALUES (id, -1); MERGE INTO fs_target t USING generate_series(1,100,2) AS id ON t.a = id WHEN MATCHED THEN UPDATE SET b = b + id, c = 'updated '|| id.*::text WHEN NOT MATCHED THEN INSERT VALUES (id, -1, 'inserted ' || id.*::text); SELECT count(*) FROM fs_target; DROP TABLE fs_target CASCADE; -- SERIALIZABLE test -- handled in isolation tests -- Inheritance-based partitioning CREATE TABLE measurement ( city_id int not null, logdate date not null, peaktemp int, unitsales int ) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2006m02 ( CHECK ( logdate >= DATE '2006-02-01' AND logdate < DATE '2006-03-01' ) ) INHERITS (measurement) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2006m03 ( CHECK ( logdate >= DATE '2006-03-01' AND logdate < DATE '2006-04-01' ) ) INHERITS (measurement) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2007m01 ( filler text, peaktemp int, logdate date not null, city_id int not null, unitsales int CHECK ( logdate >= DATE '2007-01-01' AND logdate < DATE '2007-02-01') ) WITH (autovacuum_enabled=off); ALTER TABLE measurement_y2007m01 DROP COLUMN filler; ALTER TABLE measurement_y2007m01 INHERIT measurement; INSERT INTO measurement VALUES (0, '2005-07-21', 5, 15); CREATE OR REPLACE FUNCTION measurement_insert_trigger() RETURNS TRIGGER AS $$ BEGIN IF ( NEW.logdate >= DATE '2006-02-01' AND NEW.logdate < DATE '2006-03-01' ) THEN INSERT INTO measurement_y2006m02 VALUES (NEW.*); ELSIF ( NEW.logdate >= DATE '2006-03-01' AND NEW.logdate < DATE '2006-04-01' ) THEN INSERT INTO measurement_y2006m03 VALUES (NEW.*); ELSIF ( NEW.logdate >= DATE '2007-01-01' AND NEW.logdate < DATE '2007-02-01' ) THEN INSERT INTO measurement_y2007m01 (city_id, logdate, peaktemp, unitsales) VALUES (NEW.*); ELSE RAISE EXCEPTION 'Date out of range. Fix the measurement_insert_trigger() function!'; END IF; RETURN NULL; END; $$ LANGUAGE plpgsql ; CREATE TRIGGER insert_measurement_trigger BEFORE INSERT ON measurement FOR EACH ROW EXECUTE PROCEDURE measurement_insert_trigger(); INSERT INTO measurement VALUES (1, '2006-02-10', 35, 10); INSERT INTO measurement VALUES (1, '2006-02-16', 45, 20); INSERT INTO measurement VALUES (1, '2006-03-17', 25, 10); INSERT INTO measurement VALUES (1, '2006-03-27', 15, 40); INSERT INTO measurement VALUES (1, '2007-01-15', 10, 10); INSERT INTO measurement VALUES (1, '2007-01-17', 10, 10); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate; CREATE TABLE new_measurement (LIKE measurement) WITH (autovacuum_enabled=off); INSERT INTO new_measurement VALUES (0, '2005-07-21', 25, 20); INSERT INTO new_measurement VALUES (1, '2006-03-01', 20, 10); INSERT INTO new_measurement VALUES (1, '2006-02-16', 50, 10); INSERT INTO new_measurement VALUES (2, '2006-02-10', 20, 20); INSERT INTO new_measurement VALUES (1, '2006-03-27', NULL, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-17', NULL, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-15', 5, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-16', 10, 10); BEGIN; MERGE INTO ONLY measurement m USING new_measurement nm ON (m.city_id = nm.city_id and m.logdate=nm.logdate) WHEN MATCHED AND nm.peaktemp IS NULL THEN DELETE WHEN MATCHED THEN UPDATE SET peaktemp = greatest(m.peaktemp, nm.peaktemp), unitsales = m.unitsales + coalesce(nm.unitsales, 0) WHEN NOT MATCHED THEN INSERT (city_id, logdate, peaktemp, unitsales) VALUES (city_id, logdate, peaktemp, unitsales); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate, peaktemp; ROLLBACK; MERGE into measurement m USING new_measurement nm ON (m.city_id = nm.city_id and m.logdate=nm.logdate) WHEN MATCHED AND nm.peaktemp IS NULL THEN DELETE WHEN MATCHED THEN UPDATE SET peaktemp = greatest(m.peaktemp, nm.peaktemp), unitsales = m.unitsales + coalesce(nm.unitsales, 0) WHEN NOT MATCHED THEN INSERT (city_id, logdate, peaktemp, unitsales) VALUES (city_id, logdate, peaktemp, unitsales); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate; BEGIN; MERGE INTO new_measurement nm USING ONLY measurement m ON (nm.city_id = m.city_id and nm.logdate=m.logdate) WHEN MATCHED THEN DELETE; SELECT * FROM new_measurement ORDER BY city_id, logdate; ROLLBACK; MERGE INTO new_measurement nm USING measurement m ON (nm.city_id = m.city_id and nm.logdate=m.logdate) WHEN MATCHED THEN DELETE; SELECT * FROM new_measurement ORDER BY city_id, logdate; DROP TABLE measurement, new_measurement CASCADE; DROP FUNCTION measurement_insert_trigger(); RESET SESSION AUTHORIZATION; DROP TABLE target CASCADE; DROP TABLE target2 CASCADE; DROP TABLE source, source2; DROP FUNCTION merge_trigfunc(); REVOKE CREATE ON SCHEMA public FROM regress_merge_privs; DROP USER regress_merge_privs; DROP USER regress_merge_no_privs; \o \ir :TEST_LOAD_HT_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE USER regress_merge_privs; CREATE USER regress_merge_no_privs; DROP TABLE IF EXISTS target; DROP TABLE IF EXISTS source; CREATE TABLE target (tid integer, balance integer) WITH (autovacuum_enabled=off); SELECT create_hypertable('target', 'tid', chunk_time_interval => 3); create_hypertable --------------------- (1,public,target,t) CREATE TABLE source (sid integer, delta integer) -- no index WITH (autovacuum_enabled=off); INSERT INTO target VALUES (1, 10); INSERT INTO target VALUES (2, 20); INSERT INTO target VALUES (3, 30); SELECT t.ctid is not null as matched, t.*, s.* FROM source s FULL OUTER JOIN target t ON s.sid = t.tid ORDER BY t.tid, s.sid; matched | tid | balance | sid | delta ---------+-----+---------+-----+------- t | 1 | 10 | | t | 2 | 20 | | t | 3 | 30 | | ALTER TABLE target OWNER TO regress_merge_privs; ALTER TABLE source OWNER TO regress_merge_privs; CREATE TABLE target2 (tid integer, balance integer) WITH (autovacuum_enabled=off); SELECT create_hypertable('target2', 'tid', chunk_time_interval => 3); create_hypertable ---------------------- (2,public,target2,t) CREATE TABLE source2 (sid integer, delta integer) WITH (autovacuum_enabled=off); ALTER TABLE target2 OWNER TO regress_merge_no_privs; ALTER TABLE source2 OWNER TO regress_merge_no_privs; GRANT INSERT ON target TO regress_merge_no_privs; GRANT CREATE ON SCHEMA public TO regress_merge_privs; SET SESSION AUTHORIZATION regress_merge_privs; CREATE TABLE sq_target (tid integer NOT NULL, balance integer) WITH (autovacuum_enabled=off); SELECT create_hypertable('sq_target', 'tid', chunk_time_interval => 3); create_hypertable ------------------------ (3,public,sq_target,t) CREATE TABLE sq_source (delta integer, sid integer, balance integer DEFAULT 0) WITH (autovacuum_enabled=off); INSERT INTO sq_target(tid, balance) VALUES (1,100), (2,200), (3,300); INSERT INTO sq_source(sid, delta) VALUES (1,10), (2,20), (4,40); -- conditional WHEN clause CREATE TABLE wq_target (tid integer not null, balance integer DEFAULT -1) WITH (autovacuum_enabled=off); SELECT create_hypertable('wq_target', 'tid', chunk_time_interval => 3); create_hypertable ------------------------ (4,public,wq_target,t) CREATE TABLE wq_source (balance integer, sid integer) WITH (autovacuum_enabled=off); INSERT INTO wq_source (sid, balance) VALUES (1, 100); -- some complex joins on the source side CREATE TABLE cj_target (tid integer, balance float, val text) WITH (autovacuum_enabled=off); SELECT create_hypertable('cj_target', 'tid', chunk_time_interval => 3); create_hypertable ------------------------ (5,public,cj_target,t) CREATE TABLE cj_source1 (sid1 integer, scat integer, delta integer) WITH (autovacuum_enabled=off); CREATE TABLE cj_source2 (sid2 integer, sval text) WITH (autovacuum_enabled=off); INSERT INTO cj_source1 VALUES (1, 10, 100); INSERT INTO cj_source1 VALUES (1, 20, 200); INSERT INTO cj_source1 VALUES (2, 20, 300); INSERT INTO cj_source1 VALUES (3, 10, 400); INSERT INTO cj_source2 VALUES (1, 'initial source2'); INSERT INTO cj_source2 VALUES (2, 'initial source2'); INSERT INTO cj_source2 VALUES (3, 'initial source2'); CREATE TABLE fs_target (a int, b int, c text) WITH (autovacuum_enabled=off); SELECT create_hypertable('fs_target', 'a', chunk_time_interval => 3); create_hypertable ------------------------ (6,public,fs_target,t) -- run tests on hypertable \o :TEST_RESULTS_WITH_HYPERTABLE \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- -- Errors -- MERGE INTO target t RANDOMWORD USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:12: ERROR: syntax error at or near "RANDOMWORD" LINE 1: MERGE INTO target t RANDOMWORD ^ -- MATCHED/INSERT error MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:18: ERROR: syntax error at or near "INSERT" LINE 5: INSERT DEFAULT VALUES; ^ -- incorrectly specifying INTO target MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT INTO target DEFAULT VALUES; psql:include/ts_merge_query.sql:24: ERROR: syntax error at or near "INTO" LINE 5: INSERT INTO target DEFAULT VALUES; ^ -- Multiple VALUES clause MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (1,1), (2,2); psql:include/ts_merge_query.sql:30: ERROR: syntax error at or near "," LINE 5: INSERT VALUES (1,1), (2,2); ^ -- SELECT query for INSERT MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT SELECT (1, 1); psql:include/ts_merge_query.sql:36: ERROR: syntax error at or near "SELECT" LINE 5: INSERT SELECT (1, 1); ^ -- NOT MATCHED/UPDATE MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:42: ERROR: syntax error at or near "UPDATE" LINE 5: UPDATE SET balance = 0; ^ -- UPDATE tablename MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE target SET balance = 0; psql:include/ts_merge_query.sql:48: ERROR: syntax error at or near "target" LINE 5: UPDATE target SET balance = 0; ^ -- source and target names the same MERGE INTO target USING target ON tid = tid WHEN MATCHED THEN DO NOTHING; psql:include/ts_merge_query.sql:53: ERROR: name "target" specified more than once DETAIL: The name is used both as MERGE target table and data source. -- used in a CTE WITH foo AS ( MERGE INTO target USING source ON (true) WHEN MATCHED THEN DELETE ) SELECT * FROM foo; psql:include/ts_merge_query.sql:58: ERROR: MERGE not supported in WITH query LINE 1: WITH foo AS ( ^ -- used in COPY COPY ( MERGE INTO target USING source ON (true) WHEN MATCHED THEN DELETE ) TO stdout; psql:include/ts_merge_query.sql:63: ERROR: MERGE not supported in COPY -- unsupported relation types -- view CREATE VIEW tv AS SELECT * FROM target; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:72: ERROR: cannot execute MERGE on relation "tv" DETAIL: This operation is not supported for views. DROP VIEW tv; -- materialized view CREATE MATERIALIZED VIEW mv AS SELECT * FROM target; MERGE INTO mv t USING source s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:81: ERROR: cannot execute MERGE on relation "mv" DETAIL: This operation is not supported for materialized views. DROP MATERIALIZED VIEW mv; -- permissions MERGE INTO target USING source2 ON target.tid = source2.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:90: ERROR: permission denied for table source2 GRANT INSERT ON target TO regress_merge_no_privs; SET SESSION AUTHORIZATION regress_merge_no_privs; MERGE INTO target USING source2 ON target.tid = source2.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:99: ERROR: permission denied for table target GRANT UPDATE ON target2 TO regress_merge_privs; SET SESSION AUTHORIZATION regress_merge_privs; MERGE INTO target2 USING source ON target2.tid = source.sid WHEN MATCHED THEN DELETE; psql:include/ts_merge_query.sql:108: ERROR: permission denied for table target2 MERGE INTO target2 USING source ON target2.tid = source.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:114: ERROR: permission denied for table target2 -- check if the target can be accessed from source relation subquery; we should -- not be able to do so MERGE INTO target t USING (SELECT * FROM source WHERE t.tid > sid) s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:122: ERROR: invalid reference to FROM-clause entry for table "t" LINE 2: USING (SELECT * FROM source WHERE t.tid > sid) s ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. -- -- initial tests -- -- zero rows in source has no effect MERGE INTO target USING source ON target.tid = source.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; ROLLBACK; -- insert some non-matching source rows to work from INSERT INTO source VALUES (4, 40); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN DO NOTHING; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (5, 50); SELECT * FROM target ORDER BY tid; ROLLBACK; -- index plans INSERT INTO target SELECT generate_series(1000,2500), 0; ALTER TABLE target ADD PRIMARY KEY (tid); ANALYZE target; DELETE FROM target WHERE tid > 100; ANALYZE target; -- insert some matching source rows to work from INSERT INTO source VALUES (2, 5); INSERT INTO source VALUES (3, 20); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- equivalent of an UPDATE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; SELECT * FROM target ORDER BY tid; ROLLBACK; -- equivalent of a DELETE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DO NOTHING; SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, NULL); SELECT * FROM target ORDER BY tid; ROLLBACK; -- duplicate source row causes multiple target row update ERROR INSERT INTO source VALUES (2, 5); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:241: ERROR: MERGE command cannot affect row a second time HINT: Ensure that not more than one source row matches any one target row. ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; psql:include/ts_merge_query.sql:249: ERROR: MERGE command cannot affect row a second time HINT: Ensure that not more than one source row matches any one target row. ROLLBACK; -- remove duplicate MATCHED data from source data DELETE FROM source WHERE sid = 2; INSERT INTO source VALUES (2, 5); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- duplicate source row on INSERT should fail because of target_pkey INSERT INTO source VALUES (4, 40); BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, NULL); psql:include/ts_merge_query.sql:265: ERROR: duplicate key value violates unique constraint "2_2_target_pkey" DETAIL: Key (tid)=(4) already exists. SELECT * FROM target ORDER BY tid; psql:include/ts_merge_query.sql:266: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; -- remove duplicate NOT MATCHED data from source data DELETE FROM source WHERE sid = 4; INSERT INTO source VALUES (4, 40); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- remove constraints alter table target drop CONSTRAINT target_pkey; alter table target alter column tid drop not null; psql:include/ts_merge_query.sql:277: ERROR: cannot drop not-null constraint from a time-partitioned column -- multiple actions BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, 4) WHEN MATCHED THEN UPDATE SET balance = 0; SELECT * FROM target ORDER BY tid; ROLLBACK; -- should be equivalent BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0 WHEN NOT MATCHED THEN INSERT VALUES (4, 4); SELECT * FROM target ORDER BY tid; ROLLBACK; -- column references -- do a simple equivalent of an UPDATE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance + s.delta; SELECT * FROM target ORDER BY tid; ROLLBACK; -- do a simple equivalent of an INSERT SELECT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- and again with duplicate source rows INSERT INTO source VALUES (5, 50); INSERT INTO source VALUES (5, 50); -- do a simple equivalent of an INSERT SELECT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- removing duplicate source rows DELETE FROM source WHERE sid = 5; -- and again with explicitly identified column list BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- and again with a subtle error: referring to non-existent target row for NOT MATCHED MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (t.tid, s.delta); psql:include/ts_merge_query.sql:356: ERROR: invalid reference to FROM-clause entry for table "t" LINE 5: INSERT (tid, balance) VALUES (t.tid, s.delta); ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. -- and again with a constant ON clause BEGIN; MERGE INTO target t USING source AS s ON (SELECT true) WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (t.tid, s.delta); psql:include/ts_merge_query.sql:364: ERROR: invalid reference to FROM-clause entry for table "t" LINE 5: INSERT (tid, balance) VALUES (t.tid, s.delta); ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. SELECT * FROM target ORDER BY tid; psql:include/ts_merge_query.sql:365: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; -- now the classic UPSERT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance + s.delta WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- this time with a FALSE condition MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND FALSE THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; -- this time with an actual condition which returns false MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance <> 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; BEGIN; -- and now with a condition which returns true MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance = 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; ROLLBACK; -- conditions in the NOT MATCHED clause can only refer to source columns BEGIN; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND t.balance = 100 THEN INSERT (tid) VALUES (s.sid); psql:include/ts_merge_query.sql:408: ERROR: invalid reference to FROM-clause entry for table "t" LINE 3: WHEN NOT MATCHED AND t.balance = 100 THEN ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. SELECT * FROM wq_target; psql:include/ts_merge_query.sql:409: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance = 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; -- conditions in MATCHED clause can refer to both source and target SELECT * FROM wq_source; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND s.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check if AND works MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 AND s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 AND s.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check if OR works MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 OR s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 199 OR s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check source-side whole-row references BEGIN; MERGE INTO wq_target t USING wq_source s ON (t.tid = s.sid) WHEN matched and t = s or t.tid = s.sid THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; ROLLBACK; -- check if subqueries work in the conditions? MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance > (SELECT max(balance) FROM target) THEN UPDATE SET balance = t.balance + s.balance; -- check if we can access system columns in the conditions MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.xmin = t.xmax THEN UPDATE SET balance = t.balance + s.balance; psql:include/ts_merge_query.sql:477: ERROR: cannot use system column "xmin" in MERGE WHEN condition LINE 3: WHEN MATCHED AND t.xmin = t.xmax THEN ^ MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.tableoid >= 0 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; DROP TABLE wq_target CASCADE; DROP TABLE wq_source; -- test triggers create or replace function merge_trigfunc () returns trigger language plpgsql as $$ DECLARE line text; BEGIN SELECT INTO line format('%s %s %s trigger%s', TG_WHEN, TG_OP, TG_LEVEL, CASE WHEN TG_OP = 'INSERT' AND TG_LEVEL = 'ROW' THEN format(' row: %s', NEW) WHEN TG_OP = 'UPDATE' AND TG_LEVEL = 'ROW' THEN format(' row: %s -> %s', OLD, NEW) WHEN TG_OP = 'DELETE' AND TG_LEVEL = 'ROW' THEN format(' row: %s', OLD) END); RAISE NOTICE '%', line; IF (TG_WHEN = 'BEFORE' AND TG_LEVEL = 'ROW') THEN IF (TG_OP = 'DELETE') THEN RETURN OLD; ELSE RETURN NEW; END IF; ELSE RETURN NULL; END IF; END; $$; CREATE TRIGGER merge_bsi BEFORE INSERT ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bsu BEFORE UPDATE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bsd BEFORE DELETE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asi AFTER INSERT ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asu AFTER UPDATE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asd AFTER DELETE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bri BEFORE INSERT ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bru BEFORE UPDATE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_brd BEFORE DELETE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_ari AFTER INSERT ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_aru AFTER UPDATE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_ard AFTER DELETE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); -- now the classic UPSERT, with a DELETE BEGIN; UPDATE target SET balance = 0 WHERE tid = 3; --EXPLAIN (ANALYZE ON, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND t.balance > s.delta THEN UPDATE SET balance = t.balance - s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- Test behavior of triggers that turn UPDATE/DELETE into no-ops create or replace function skip_merge_op() returns trigger language plpgsql as $$ BEGIN RETURN NULL; END; $$; SELECT * FROM target full outer join source on (sid = tid); create trigger merge_skip BEFORE INSERT OR UPDATE or DELETE ON target FOR EACH ROW EXECUTE FUNCTION skip_merge_op(); DO $$ DECLARE result integer; BEGIN MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND s.sid = 3 THEN UPDATE SET balance = t.balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT VALUES (sid, delta); IF FOUND THEN RAISE NOTICE 'Found'; ELSE RAISE NOTICE 'Not found'; END IF; GET DIAGNOSTICS result := ROW_COUNT; RAISE NOTICE 'ROW_COUNT = %', result; END; $$; SELECT * FROM target FULL OUTER JOIN source ON (sid = tid); DROP TRIGGER merge_skip ON target; DROP FUNCTION skip_merge_op(); -- test from PL/pgSQL -- make sure MERGE INTO isn't interpreted to mean returning variables like SELECT INTO BEGIN; DO LANGUAGE plpgsql $$ BEGIN MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND t.balance > s.delta THEN UPDATE SET balance = t.balance - s.delta; END; $$; ROLLBACK; --source constants BEGIN; MERGE INTO target t USING (SELECT 9 AS sid, 57 AS delta) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; --source query BEGIN; MERGE INTO target t USING (SELECT sid, delta FROM source WHERE delta > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT sid, delta as newname FROM source WHERE delta > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.newname); SELECT * FROM target ORDER BY tid; ROLLBACK; --self-merge BEGIN; MERGE INTO target t1 USING target t2 ON t1.tid = t2.tid WHEN MATCHED THEN UPDATE SET balance = t1.balance + t2.balance WHEN NOT MATCHED THEN INSERT VALUES (t2.tid, t2.balance); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT tid as sid, balance as delta FROM target WHERE balance > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT sid, max(delta) AS delta FROM source GROUP BY sid HAVING count(*) = 1 ORDER BY sid ASC) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- plpgsql parameters and results BEGIN; CREATE FUNCTION merge_func (p_id integer, p_bal integer) RETURNS INTEGER LANGUAGE plpgsql AS $$ DECLARE result integer; BEGIN MERGE INTO target t USING (SELECT p_id AS sid) AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance - p_bal; IF FOUND THEN GET DIAGNOSTICS result := ROW_COUNT; END IF; RETURN result; END; $$; SELECT merge_func(3, 4); SELECT * FROM target ORDER BY tid; ROLLBACK; -- PREPARE BEGIN; prepare foom as merge into target t using (select 1 as sid) s on (t.tid = s.sid) when matched then update set balance = 1; psql:include/ts_merge_query.sql:685: ERROR: prepared statement "foom" already exists execute foom; psql:include/ts_merge_query.sql:686: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; BEGIN; PREPARE foom2 (integer, integer) AS MERGE INTO target t USING (SELECT 1) s ON t.tid = $1 WHEN MATCHED THEN UPDATE SET balance = $2; psql:include/ts_merge_query.sql:695: ERROR: prepared statement "foom2" already exists --EXPLAIN (ANALYZE ON, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) execute foom2 (1, 1); psql:include/ts_merge_query.sql:697: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; -- subqueries in source relation BEGIN; MERGE INTO sq_target t USING (SELECT * FROM sq_source) s ON tid = sid WHEN MATCHED AND t.balance > delta THEN UPDATE SET balance = t.balance + delta; SELECT * FROM sq_target ORDER BY tid; ROLLBACK; -- try a view CREATE VIEW v AS SELECT * FROM sq_source WHERE sid < 2; BEGIN; MERGE INTO sq_target USING v ON tid = sid WHEN MATCHED THEN UPDATE SET balance = v.balance + delta; SELECT * FROM sq_target ORDER BY tid; ROLLBACK; -- ambiguous reference to a column BEGIN; MERGE INTO sq_target USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; psql:include/ts_merge_query.sql:732: ERROR: column reference "balance" is ambiguous LINE 5: UPDATE SET balance = balance + delta ^ ROLLBACK; BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; SELECT * FROM sq_target; ROLLBACK; -- CTEs BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); WITH targq AS ( SELECT * FROM v ) MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; ROLLBACK; -- RETURNING BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE RETURNING *; psql:include/ts_merge_query.sql:778: ERROR: syntax error at or near "RETURNING" LINE 10: RETURNING *; ^ ROLLBACK; -- PG17-specific tests for views, returning and merge_action. These throw syntax errors for previous versions of Postgres. -- However, since the error is the same for both hypertables and regular tables, this test should still pass for previous versions. -- RETURNING INSERT INTO source(sid, delta) VALUES(1, 40), (5, 50); BEGIN; MERGE INTO target t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta); SELECT * from target; ROLLBACK; BEGIN; MERGE INTO target t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 1 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta) RETURNING merge_action(), t.*; psql:include/ts_merge_query.sql:811: ERROR: syntax error at or near "RETURNING" LINE 10: RETURNING merge_action(), t.*; ^ ROLLBACK; -- Views CREATE VIEW tv AS SELECT * FROM target; BEGIN; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta); psql:include/ts_merge_query.sql:826: ERROR: cannot execute MERGE on relation "tv" DETAIL: This operation is not supported for views. SELECT * from tv; psql:include/ts_merge_query.sql:827: ERROR: current transaction is aborted, commands ignored until end of transaction block SELECT * from target; -- should also update the underlying table psql:include/ts_merge_query.sql:828: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; BEGIN; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta) RETURNING merge_action(), t.*; psql:include/ts_merge_query.sql:841: ERROR: syntax error at or near "RETURNING" LINE 10: RETURNING merge_action(), t.*; ^ ROLLBACK; DROP VIEW tv; DELETE FROM source where sid in (1, 5); -- EXPLAIN CREATE TABLE ex_mtarget (a int, b int) WITH (autovacuum_enabled=off); CREATE TABLE ex_msource (a int, b int) WITH (autovacuum_enabled=off); INSERT INTO ex_mtarget SELECT i, i*10 FROM generate_series(1,100,2) i; INSERT INTO ex_msource SELECT i, i*10 FROM generate_series(1,100,1) i; CREATE FUNCTION explain_merge(query text) RETURNS SETOF text LANGUAGE plpgsql AS $$ DECLARE ln text; BEGIN FOR ln IN EXECUTE 'explain (analyze, timing off, summary off, buffers off, costs off) ' || query LOOP ln := regexp_replace(ln, '(Memory( Usage)?|Buckets|Batches): \S*', '\1: xxx', 'g'); RETURN NEXT ln; END LOOP; END; $$; -- only updates SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED THEN UPDATE SET b = t.b + 1'); -- only updates to selected tuples SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1'); -- updates + deletes SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1 WHEN MATCHED AND t.a >= 10 AND t.a <= 20 THEN DELETE'); -- only inserts SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN NOT MATCHED AND s.a < 10 THEN INSERT VALUES (a, b)'); -- all three SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1 WHEN MATCHED AND t.a >= 30 AND t.a <= 40 THEN DELETE WHEN NOT MATCHED AND s.a < 20 THEN INSERT VALUES (a, b)'); -- nothing SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a AND t.a < -1000 WHEN MATCHED AND t.a < 10 THEN DO NOTHING'); DROP TABLE ex_msource, ex_mtarget; DROP FUNCTION explain_merge(text); -- Subqueries BEGIN; MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED THEN UPDATE SET balance = (SELECT count(*) FROM sq_target); SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; BEGIN; MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND (SELECT count(*) > 0 FROM sq_target) THEN UPDATE SET balance = 42; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; BEGIN; MERGE INTO sq_target t USING v ON tid = sid AND (SELECT count(*) > 0 FROM sq_target) WHEN MATCHED THEN UPDATE SET balance = 42; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; -- Test RETURNING with subqueries BEGIN; MERGE INTO sq_target t USING v ON tid = sid AND (SELECT count(*) > 0 FROM sq_target) WHEN MATCHED THEN UPDATE SET balance = 42 RETURNING *; psql:include/ts_merge_query.sql:952: ERROR: syntax error at or near "RETURNING" LINE 6: RETURNING *; ^ SELECT * FROM sq_target WHERE tid = 1; psql:include/ts_merge_query.sql:953: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; DROP TABLE sq_target CASCADE; DROP TABLE sq_source CASCADE; CREATE TABLE pa_target (tid integer, balance float, val text) PARTITION BY LIST (tid); CREATE TABLE part1 PARTITION OF pa_target FOR VALUES IN (1,4) WITH (autovacuum_enabled=off); CREATE TABLE part2 PARTITION OF pa_target FOR VALUES IN (2,5,6) WITH (autovacuum_enabled=off); CREATE TABLE part3 PARTITION OF pa_target FOR VALUES IN (3,8,9) WITH (autovacuum_enabled=off); CREATE TABLE part4 PARTITION OF pa_target DEFAULT WITH (autovacuum_enabled=off); CREATE TABLE pa_source (sid integer, delta float); -- insert many rows to the source table INSERT INTO pa_source SELECT id, id * 10 FROM generate_series(1,14) AS id; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT id, id * 100, 'initial' FROM generate_series(1,14,2) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- same with a constant qual BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid AND tid = 1 WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- try updating the partition key column BEGIN; CREATE FUNCTION merge_func() RETURNS integer LANGUAGE plpgsql AS $$ DECLARE result integer; BEGIN MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET tid = tid + 1, balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); IF FOUND THEN GET DIAGNOSTICS result := ROW_COUNT; END IF; RETURN result; END; $$; SELECT merge_func(); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_target CASCADE; -- The target table is partitioned in the same way, but this time by attaching -- partitions which have columns in different order, dropped columns etc. CREATE TABLE pa_target (tid integer, balance float, val text) PARTITION BY LIST (tid); CREATE TABLE part1 (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE part2 (balance float, tid integer, val text) WITH (autovacuum_enabled=off); CREATE TABLE part3 (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE part4 (extraid text, tid integer, balance float, val text) WITH (autovacuum_enabled=off); ALTER TABLE part4 DROP COLUMN extraid; ALTER TABLE pa_target ATTACH PARTITION part1 FOR VALUES IN (1,4); ALTER TABLE pa_target ATTACH PARTITION part2 FOR VALUES IN (2,5,6); ALTER TABLE pa_target ATTACH PARTITION part3 FOR VALUES IN (3,8,9); ALTER TABLE pa_target ATTACH PARTITION part4 DEFAULT; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT id, id * 100, 'initial' FROM generate_series(1,14,2) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- same with a constant qual BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid AND tid IN (1, 5) WHEN MATCHED AND tid % 5 = 0 THEN DELETE WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- try updating the partition key column BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET tid = tid + 1, balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_source; DROP TABLE pa_target CASCADE; -- Sub-partitioning CREATE TABLE pa_target (logts timestamp, tid integer, balance float, val text) PARTITION BY RANGE (logts); CREATE TABLE part_m01 PARTITION OF pa_target FOR VALUES FROM ('2017-01-01') TO ('2017-02-01') PARTITION BY LIST (tid); CREATE TABLE part_m01_odd PARTITION OF part_m01 FOR VALUES IN (1,3,5,7,9) WITH (autovacuum_enabled=off); CREATE TABLE part_m01_even PARTITION OF part_m01 FOR VALUES IN (2,4,6,8) WITH (autovacuum_enabled=off); CREATE TABLE part_m02 PARTITION OF pa_target FOR VALUES FROM ('2017-02-01') TO ('2017-03-01') PARTITION BY LIST (tid); CREATE TABLE part_m02_odd PARTITION OF part_m02 FOR VALUES IN (1,3,5,7,9) WITH (autovacuum_enabled=off); CREATE TABLE part_m02_even PARTITION OF part_m02 FOR VALUES IN (2,4,6,8) WITH (autovacuum_enabled=off); CREATE TABLE pa_source (sid integer, delta float) WITH (autovacuum_enabled=off); -- insert many rows to the source table INSERT INTO pa_source SELECT id, id * 10 FROM generate_series(1,14) AS id; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT '2017-01-31', id, id * 100, 'initial' FROM generate_series(1,9,3) AS id; INSERT INTO pa_target SELECT '2017-02-28', id, id * 100, 'initial' FROM generate_series(2,9,3) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING (SELECT '2017-01-15' AS slogts, * FROM pa_source WHERE sid < 10) s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (slogts::timestamp, sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_source; DROP TABLE pa_target CASCADE; -- some complex joins on the source side -- source relation is an unaliased join MERGE INTO cj_target t USING cj_source1 s1 INNER JOIN cj_source2 s2 ON sid1 = sid2 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid1, delta, sval); -- try accessing columns from either side of the source join MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 AND scat = 20 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid2, delta, sval) WHEN MATCHED THEN DELETE; -- some simple expressions in INSERT targetlist MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid2, delta + scat, sval) WHEN MATCHED THEN UPDATE SET val = val || ' updated by merge'; MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 AND scat = 20 ON t.tid = sid1 WHEN MATCHED THEN UPDATE SET val = val || ' ' || delta::text; SELECT * FROM cj_target ORDER BY tid; ALTER TABLE cj_source1 RENAME COLUMN sid1 TO sid; ALTER TABLE cj_source2 RENAME COLUMN sid2 TO sid; TRUNCATE cj_target; MERGE INTO cj_target t USING cj_source1 s1 INNER JOIN cj_source2 s2 ON s1.sid = s2.sid ON t.tid = s1.sid WHEN NOT MATCHED THEN INSERT VALUES (s2.sid, delta, sval); DROP TABLE cj_source2, cj_source1; DROP TABLE cj_target CASCADE; -- Function scans MERGE INTO fs_target t USING generate_series(1,100,1) AS id ON t.a = id WHEN MATCHED THEN UPDATE SET b = b + id WHEN NOT MATCHED THEN INSERT VALUES (id, -1); MERGE INTO fs_target t USING generate_series(1,100,2) AS id ON t.a = id WHEN MATCHED THEN UPDATE SET b = b + id, c = 'updated '|| id.*::text WHEN NOT MATCHED THEN INSERT VALUES (id, -1, 'inserted ' || id.*::text); SELECT count(*) FROM fs_target; DROP TABLE fs_target CASCADE; -- SERIALIZABLE test -- handled in isolation tests -- Inheritance-based partitioning CREATE TABLE measurement ( city_id int not null, logdate date not null, peaktemp int, unitsales int ) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2006m02 ( CHECK ( logdate >= DATE '2006-02-01' AND logdate < DATE '2006-03-01' ) ) INHERITS (measurement) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2006m03 ( CHECK ( logdate >= DATE '2006-03-01' AND logdate < DATE '2006-04-01' ) ) INHERITS (measurement) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2007m01 ( filler text, peaktemp int, logdate date not null, city_id int not null, unitsales int CHECK ( logdate >= DATE '2007-01-01' AND logdate < DATE '2007-02-01') ) WITH (autovacuum_enabled=off); ALTER TABLE measurement_y2007m01 DROP COLUMN filler; ALTER TABLE measurement_y2007m01 INHERIT measurement; INSERT INTO measurement VALUES (0, '2005-07-21', 5, 15); CREATE OR REPLACE FUNCTION measurement_insert_trigger() RETURNS TRIGGER AS $$ BEGIN IF ( NEW.logdate >= DATE '2006-02-01' AND NEW.logdate < DATE '2006-03-01' ) THEN INSERT INTO measurement_y2006m02 VALUES (NEW.*); ELSIF ( NEW.logdate >= DATE '2006-03-01' AND NEW.logdate < DATE '2006-04-01' ) THEN INSERT INTO measurement_y2006m03 VALUES (NEW.*); ELSIF ( NEW.logdate >= DATE '2007-01-01' AND NEW.logdate < DATE '2007-02-01' ) THEN INSERT INTO measurement_y2007m01 (city_id, logdate, peaktemp, unitsales) VALUES (NEW.*); ELSE RAISE EXCEPTION 'Date out of range. Fix the measurement_insert_trigger() function!'; END IF; RETURN NULL; END; $$ LANGUAGE plpgsql ; CREATE TRIGGER insert_measurement_trigger BEFORE INSERT ON measurement FOR EACH ROW EXECUTE PROCEDURE measurement_insert_trigger(); INSERT INTO measurement VALUES (1, '2006-02-10', 35, 10); INSERT INTO measurement VALUES (1, '2006-02-16', 45, 20); INSERT INTO measurement VALUES (1, '2006-03-17', 25, 10); INSERT INTO measurement VALUES (1, '2006-03-27', 15, 40); INSERT INTO measurement VALUES (1, '2007-01-15', 10, 10); INSERT INTO measurement VALUES (1, '2007-01-17', 10, 10); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate; CREATE TABLE new_measurement (LIKE measurement) WITH (autovacuum_enabled=off); INSERT INTO new_measurement VALUES (0, '2005-07-21', 25, 20); INSERT INTO new_measurement VALUES (1, '2006-03-01', 20, 10); INSERT INTO new_measurement VALUES (1, '2006-02-16', 50, 10); INSERT INTO new_measurement VALUES (2, '2006-02-10', 20, 20); INSERT INTO new_measurement VALUES (1, '2006-03-27', NULL, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-17', NULL, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-15', 5, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-16', 10, 10); BEGIN; MERGE INTO ONLY measurement m USING new_measurement nm ON (m.city_id = nm.city_id and m.logdate=nm.logdate) WHEN MATCHED AND nm.peaktemp IS NULL THEN DELETE WHEN MATCHED THEN UPDATE SET peaktemp = greatest(m.peaktemp, nm.peaktemp), unitsales = m.unitsales + coalesce(nm.unitsales, 0) WHEN NOT MATCHED THEN INSERT (city_id, logdate, peaktemp, unitsales) VALUES (city_id, logdate, peaktemp, unitsales); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate, peaktemp; ROLLBACK; MERGE into measurement m USING new_measurement nm ON (m.city_id = nm.city_id and m.logdate=nm.logdate) WHEN MATCHED AND nm.peaktemp IS NULL THEN DELETE WHEN MATCHED THEN UPDATE SET peaktemp = greatest(m.peaktemp, nm.peaktemp), unitsales = m.unitsales + coalesce(nm.unitsales, 0) WHEN NOT MATCHED THEN INSERT (city_id, logdate, peaktemp, unitsales) VALUES (city_id, logdate, peaktemp, unitsales); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate; BEGIN; MERGE INTO new_measurement nm USING ONLY measurement m ON (nm.city_id = m.city_id and nm.logdate=m.logdate) WHEN MATCHED THEN DELETE; SELECT * FROM new_measurement ORDER BY city_id, logdate; ROLLBACK; MERGE INTO new_measurement nm USING measurement m ON (nm.city_id = m.city_id and nm.logdate=m.logdate) WHEN MATCHED THEN DELETE; SELECT * FROM new_measurement ORDER BY city_id, logdate; DROP TABLE measurement, new_measurement CASCADE; DROP FUNCTION measurement_insert_trigger(); RESET SESSION AUTHORIZATION; DROP TABLE target CASCADE; DROP TABLE target2 CASCADE; DROP TABLE source, source2; DROP FUNCTION merge_trigfunc(); REVOKE CREATE ON SCHEMA public FROM regress_merge_privs; DROP USER regress_merge_privs; DROP USER regress_merge_no_privs; \o :DIFF_CMD ================================================ FILE: test/expected/ts_merge-17.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER \set ON_ERROR_STOP 0 \set VERBOSITY default SET client_min_messages TO error; \set TEST_BASE_NAME ts_merge SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') AS "TEST_LOAD_NAME", format('include/%s_load_ht.sql', :'TEST_BASE_NAME') AS "TEST_LOAD_HT_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') AS "TEST_QUERY_NAME", format('%s/results/%s_ht_results.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') AS "TEST_RESULTS_WITH_HYPERTABLE", format('%s/results/%s_results.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') AS "TEST_RESULTS_WITH_NO_HYPERTABLE" \gset SELECT format('\! diff -u --label "Base pg table results" --label "Hypertable results" %s %s', :'TEST_RESULTS_WITH_HYPERTABLE', :'TEST_RESULTS_WITH_NO_HYPERTABLE') AS "DIFF_CMD" \gset \ir :TEST_LOAD_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE USER regress_merge_privs; CREATE USER regress_merge_no_privs; DROP TABLE IF EXISTS target; DROP TABLE IF EXISTS source; CREATE TABLE target (tid integer, balance integer) WITH (autovacuum_enabled=off); CREATE TABLE source (sid integer, delta integer) -- no index WITH (autovacuum_enabled=off); INSERT INTO target VALUES (1, 10); INSERT INTO target VALUES (2, 20); INSERT INTO target VALUES (3, 30); SELECT t.ctid is not null as matched, t.*, s.* FROM source s FULL OUTER JOIN target t ON s.sid = t.tid ORDER BY t.tid, s.sid; matched | tid | balance | sid | delta ---------+-----+---------+-----+------- t | 1 | 10 | | t | 2 | 20 | | t | 3 | 30 | | ALTER TABLE target OWNER TO regress_merge_privs; ALTER TABLE source OWNER TO regress_merge_privs; CREATE TABLE target2 (tid integer, balance integer) WITH (autovacuum_enabled=off); CREATE TABLE source2 (sid integer, delta integer) WITH (autovacuum_enabled=off); ALTER TABLE target2 OWNER TO regress_merge_no_privs; ALTER TABLE source2 OWNER TO regress_merge_no_privs; GRANT INSERT ON target TO regress_merge_no_privs; GRANT CREATE ON SCHEMA public TO regress_merge_privs; SET SESSION AUTHORIZATION regress_merge_privs; CREATE TABLE sq_target (tid integer NOT NULL, balance integer) WITH (autovacuum_enabled=off); CREATE TABLE sq_source (delta integer, sid integer, balance integer DEFAULT 0) WITH (autovacuum_enabled=off); INSERT INTO sq_target(tid, balance) VALUES (1,100), (2,200), (3,300); INSERT INTO sq_source(sid, delta) VALUES (1,10), (2,20), (4,40); -- conditional WHEN clause CREATE TABLE wq_target (tid integer not null, balance integer DEFAULT -1) WITH (autovacuum_enabled=off); CREATE TABLE wq_source (balance integer, sid integer) WITH (autovacuum_enabled=off); INSERT INTO wq_source (sid, balance) VALUES (1, 100); CREATE TABLE cj_target (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE cj_source1 (sid1 integer, scat integer, delta integer) WITH (autovacuum_enabled=off); CREATE TABLE cj_source2 (sid2 integer, sval text) WITH (autovacuum_enabled=off); INSERT INTO cj_source1 VALUES (1, 10, 100); INSERT INTO cj_source1 VALUES (1, 20, 200); INSERT INTO cj_source1 VALUES (2, 20, 300); INSERT INTO cj_source1 VALUES (3, 10, 400); INSERT INTO cj_source2 VALUES (1, 'initial source2'); INSERT INTO cj_source2 VALUES (2, 'initial source2'); INSERT INTO cj_source2 VALUES (3, 'initial source2'); CREATE TABLE fs_target (a int, b int, c text) WITH (autovacuum_enabled=off); -- run tests on normal table \o :TEST_RESULTS_WITH_NO_HYPERTABLE \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- -- Errors -- MERGE INTO target t RANDOMWORD USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:12: ERROR: syntax error at or near "RANDOMWORD" LINE 1: MERGE INTO target t RANDOMWORD ^ -- MATCHED/INSERT error MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:18: ERROR: syntax error at or near "INSERT" LINE 5: INSERT DEFAULT VALUES; ^ -- incorrectly specifying INTO target MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT INTO target DEFAULT VALUES; psql:include/ts_merge_query.sql:24: ERROR: syntax error at or near "INTO" LINE 5: INSERT INTO target DEFAULT VALUES; ^ -- Multiple VALUES clause MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (1,1), (2,2); psql:include/ts_merge_query.sql:30: ERROR: syntax error at or near "," LINE 5: INSERT VALUES (1,1), (2,2); ^ -- SELECT query for INSERT MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT SELECT (1, 1); psql:include/ts_merge_query.sql:36: ERROR: syntax error at or near "SELECT" LINE 5: INSERT SELECT (1, 1); ^ -- NOT MATCHED/UPDATE MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:42: ERROR: syntax error at or near "UPDATE" LINE 5: UPDATE SET balance = 0; ^ -- UPDATE tablename MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE target SET balance = 0; psql:include/ts_merge_query.sql:48: ERROR: syntax error at or near "target" LINE 5: UPDATE target SET balance = 0; ^ -- source and target names the same MERGE INTO target USING target ON tid = tid WHEN MATCHED THEN DO NOTHING; psql:include/ts_merge_query.sql:53: ERROR: name "target" specified more than once DETAIL: The name is used both as MERGE target table and data source. -- used in a CTE WITH foo AS ( MERGE INTO target USING source ON (true) WHEN MATCHED THEN DELETE ) SELECT * FROM foo; psql:include/ts_merge_query.sql:58: ERROR: WITH query "foo" does not have a RETURNING clause LINE 4: ) SELECT * FROM foo; ^ -- used in COPY COPY ( MERGE INTO target USING source ON (true) WHEN MATCHED THEN DELETE ) TO stdout; psql:include/ts_merge_query.sql:63: ERROR: COPY query must have a RETURNING clause -- unsupported relation types -- view CREATE VIEW tv AS SELECT * FROM target; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; DROP VIEW tv; -- materialized view CREATE MATERIALIZED VIEW mv AS SELECT * FROM target; MERGE INTO mv t USING source s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:81: ERROR: cannot execute MERGE on relation "mv" DETAIL: This operation is not supported for materialized views. DROP MATERIALIZED VIEW mv; -- permissions MERGE INTO target USING source2 ON target.tid = source2.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:90: ERROR: permission denied for table source2 GRANT INSERT ON target TO regress_merge_no_privs; SET SESSION AUTHORIZATION regress_merge_no_privs; MERGE INTO target USING source2 ON target.tid = source2.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:99: ERROR: permission denied for table target GRANT UPDATE ON target2 TO regress_merge_privs; SET SESSION AUTHORIZATION regress_merge_privs; MERGE INTO target2 USING source ON target2.tid = source.sid WHEN MATCHED THEN DELETE; psql:include/ts_merge_query.sql:108: ERROR: permission denied for table target2 MERGE INTO target2 USING source ON target2.tid = source.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:114: ERROR: permission denied for table target2 -- check if the target can be accessed from source relation subquery; we should -- not be able to do so MERGE INTO target t USING (SELECT * FROM source WHERE t.tid > sid) s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:122: ERROR: invalid reference to FROM-clause entry for table "t" LINE 2: USING (SELECT * FROM source WHERE t.tid > sid) s ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. -- -- initial tests -- -- zero rows in source has no effect MERGE INTO target USING source ON target.tid = source.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; ROLLBACK; -- insert some non-matching source rows to work from INSERT INTO source VALUES (4, 40); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN DO NOTHING; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (5, 50); SELECT * FROM target ORDER BY tid; ROLLBACK; -- index plans INSERT INTO target SELECT generate_series(1000,2500), 0; ALTER TABLE target ADD PRIMARY KEY (tid); ANALYZE target; DELETE FROM target WHERE tid > 100; ANALYZE target; -- insert some matching source rows to work from INSERT INTO source VALUES (2, 5); INSERT INTO source VALUES (3, 20); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- equivalent of an UPDATE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; SELECT * FROM target ORDER BY tid; ROLLBACK; -- equivalent of a DELETE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DO NOTHING; SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, NULL); SELECT * FROM target ORDER BY tid; ROLLBACK; -- duplicate source row causes multiple target row update ERROR INSERT INTO source VALUES (2, 5); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:241: ERROR: MERGE command cannot affect row a second time HINT: Ensure that not more than one source row matches any one target row. ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; psql:include/ts_merge_query.sql:249: ERROR: MERGE command cannot affect row a second time HINT: Ensure that not more than one source row matches any one target row. ROLLBACK; -- remove duplicate MATCHED data from source data DELETE FROM source WHERE sid = 2; INSERT INTO source VALUES (2, 5); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- duplicate source row on INSERT should fail because of target_pkey INSERT INTO source VALUES (4, 40); BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, NULL); psql:include/ts_merge_query.sql:265: ERROR: duplicate key value violates unique constraint "target_pkey" DETAIL: Key (tid)=(4) already exists. SELECT * FROM target ORDER BY tid; psql:include/ts_merge_query.sql:266: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; -- remove duplicate NOT MATCHED data from source data DELETE FROM source WHERE sid = 4; INSERT INTO source VALUES (4, 40); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- remove constraints alter table target drop CONSTRAINT target_pkey; alter table target alter column tid drop not null; -- multiple actions BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, 4) WHEN MATCHED THEN UPDATE SET balance = 0; SELECT * FROM target ORDER BY tid; ROLLBACK; -- should be equivalent BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0 WHEN NOT MATCHED THEN INSERT VALUES (4, 4); SELECT * FROM target ORDER BY tid; ROLLBACK; -- column references -- do a simple equivalent of an UPDATE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance + s.delta; SELECT * FROM target ORDER BY tid; ROLLBACK; -- do a simple equivalent of an INSERT SELECT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- and again with duplicate source rows INSERT INTO source VALUES (5, 50); INSERT INTO source VALUES (5, 50); -- do a simple equivalent of an INSERT SELECT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- removing duplicate source rows DELETE FROM source WHERE sid = 5; -- and again with explicitly identified column list BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- and again with a subtle error: referring to non-existent target row for NOT MATCHED MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (t.tid, s.delta); psql:include/ts_merge_query.sql:356: ERROR: invalid reference to FROM-clause entry for table "t" LINE 5: INSERT (tid, balance) VALUES (t.tid, s.delta); ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. -- and again with a constant ON clause BEGIN; MERGE INTO target t USING source AS s ON (SELECT true) WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (t.tid, s.delta); psql:include/ts_merge_query.sql:364: ERROR: invalid reference to FROM-clause entry for table "t" LINE 5: INSERT (tid, balance) VALUES (t.tid, s.delta); ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. SELECT * FROM target ORDER BY tid; psql:include/ts_merge_query.sql:365: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; -- now the classic UPSERT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance + s.delta WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- this time with a FALSE condition MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND FALSE THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; -- this time with an actual condition which returns false MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance <> 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; BEGIN; -- and now with a condition which returns true MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance = 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; ROLLBACK; -- conditions in the NOT MATCHED clause can only refer to source columns BEGIN; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND t.balance = 100 THEN INSERT (tid) VALUES (s.sid); psql:include/ts_merge_query.sql:408: ERROR: invalid reference to FROM-clause entry for table "t" LINE 3: WHEN NOT MATCHED AND t.balance = 100 THEN ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. SELECT * FROM wq_target; psql:include/ts_merge_query.sql:409: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance = 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; -- conditions in MATCHED clause can refer to both source and target SELECT * FROM wq_source; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND s.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check if AND works MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 AND s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 AND s.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check if OR works MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 OR s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 199 OR s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check source-side whole-row references BEGIN; MERGE INTO wq_target t USING wq_source s ON (t.tid = s.sid) WHEN matched and t = s or t.tid = s.sid THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; ROLLBACK; -- check if subqueries work in the conditions? MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance > (SELECT max(balance) FROM target) THEN UPDATE SET balance = t.balance + s.balance; -- check if we can access system columns in the conditions MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.xmin = t.xmax THEN UPDATE SET balance = t.balance + s.balance; psql:include/ts_merge_query.sql:477: ERROR: cannot use system column "xmin" in MERGE WHEN condition LINE 3: WHEN MATCHED AND t.xmin = t.xmax THEN ^ MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.tableoid >= 0 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; DROP TABLE wq_target CASCADE; DROP TABLE wq_source; -- test triggers create or replace function merge_trigfunc () returns trigger language plpgsql as $$ DECLARE line text; BEGIN SELECT INTO line format('%s %s %s trigger%s', TG_WHEN, TG_OP, TG_LEVEL, CASE WHEN TG_OP = 'INSERT' AND TG_LEVEL = 'ROW' THEN format(' row: %s', NEW) WHEN TG_OP = 'UPDATE' AND TG_LEVEL = 'ROW' THEN format(' row: %s -> %s', OLD, NEW) WHEN TG_OP = 'DELETE' AND TG_LEVEL = 'ROW' THEN format(' row: %s', OLD) END); RAISE NOTICE '%', line; IF (TG_WHEN = 'BEFORE' AND TG_LEVEL = 'ROW') THEN IF (TG_OP = 'DELETE') THEN RETURN OLD; ELSE RETURN NEW; END IF; ELSE RETURN NULL; END IF; END; $$; CREATE TRIGGER merge_bsi BEFORE INSERT ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bsu BEFORE UPDATE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bsd BEFORE DELETE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asi AFTER INSERT ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asu AFTER UPDATE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asd AFTER DELETE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bri BEFORE INSERT ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bru BEFORE UPDATE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_brd BEFORE DELETE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_ari AFTER INSERT ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_aru AFTER UPDATE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_ard AFTER DELETE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); -- now the classic UPSERT, with a DELETE BEGIN; UPDATE target SET balance = 0 WHERE tid = 3; --EXPLAIN (ANALYZE ON, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND t.balance > s.delta THEN UPDATE SET balance = t.balance - s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- Test behavior of triggers that turn UPDATE/DELETE into no-ops create or replace function skip_merge_op() returns trigger language plpgsql as $$ BEGIN RETURN NULL; END; $$; SELECT * FROM target full outer join source on (sid = tid); create trigger merge_skip BEFORE INSERT OR UPDATE or DELETE ON target FOR EACH ROW EXECUTE FUNCTION skip_merge_op(); DO $$ DECLARE result integer; BEGIN MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND s.sid = 3 THEN UPDATE SET balance = t.balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT VALUES (sid, delta); IF FOUND THEN RAISE NOTICE 'Found'; ELSE RAISE NOTICE 'Not found'; END IF; GET DIAGNOSTICS result := ROW_COUNT; RAISE NOTICE 'ROW_COUNT = %', result; END; $$; SELECT * FROM target FULL OUTER JOIN source ON (sid = tid); DROP TRIGGER merge_skip ON target; DROP FUNCTION skip_merge_op(); -- test from PL/pgSQL -- make sure MERGE INTO isn't interpreted to mean returning variables like SELECT INTO BEGIN; DO LANGUAGE plpgsql $$ BEGIN MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND t.balance > s.delta THEN UPDATE SET balance = t.balance - s.delta; END; $$; ROLLBACK; --source constants BEGIN; MERGE INTO target t USING (SELECT 9 AS sid, 57 AS delta) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; --source query BEGIN; MERGE INTO target t USING (SELECT sid, delta FROM source WHERE delta > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT sid, delta as newname FROM source WHERE delta > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.newname); SELECT * FROM target ORDER BY tid; ROLLBACK; --self-merge BEGIN; MERGE INTO target t1 USING target t2 ON t1.tid = t2.tid WHEN MATCHED THEN UPDATE SET balance = t1.balance + t2.balance WHEN NOT MATCHED THEN INSERT VALUES (t2.tid, t2.balance); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT tid as sid, balance as delta FROM target WHERE balance > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT sid, max(delta) AS delta FROM source GROUP BY sid HAVING count(*) = 1 ORDER BY sid ASC) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- plpgsql parameters and results BEGIN; CREATE FUNCTION merge_func (p_id integer, p_bal integer) RETURNS INTEGER LANGUAGE plpgsql AS $$ DECLARE result integer; BEGIN MERGE INTO target t USING (SELECT p_id AS sid) AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance - p_bal; IF FOUND THEN GET DIAGNOSTICS result := ROW_COUNT; END IF; RETURN result; END; $$; SELECT merge_func(3, 4); SELECT * FROM target ORDER BY tid; ROLLBACK; -- PREPARE BEGIN; prepare foom as merge into target t using (select 1 as sid) s on (t.tid = s.sid) when matched then update set balance = 1; execute foom; ROLLBACK; BEGIN; PREPARE foom2 (integer, integer) AS MERGE INTO target t USING (SELECT 1) s ON t.tid = $1 WHEN MATCHED THEN UPDATE SET balance = $2; --EXPLAIN (ANALYZE ON, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) execute foom2 (1, 1); ROLLBACK; -- subqueries in source relation BEGIN; MERGE INTO sq_target t USING (SELECT * FROM sq_source) s ON tid = sid WHEN MATCHED AND t.balance > delta THEN UPDATE SET balance = t.balance + delta; SELECT * FROM sq_target ORDER BY tid; ROLLBACK; -- try a view CREATE VIEW v AS SELECT * FROM sq_source WHERE sid < 2; BEGIN; MERGE INTO sq_target USING v ON tid = sid WHEN MATCHED THEN UPDATE SET balance = v.balance + delta; SELECT * FROM sq_target ORDER BY tid; ROLLBACK; -- ambiguous reference to a column BEGIN; MERGE INTO sq_target USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; psql:include/ts_merge_query.sql:732: ERROR: column reference "balance" is ambiguous LINE 5: UPDATE SET balance = balance + delta ^ ROLLBACK; BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; SELECT * FROM sq_target; ROLLBACK; -- CTEs BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); WITH targq AS ( SELECT * FROM v ) MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; ROLLBACK; -- RETURNING BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE RETURNING *; ROLLBACK; -- PG17-specific tests for views, returning and merge_action. These throw syntax errors for previous versions of Postgres. -- However, since the error is the same for both hypertables and regular tables, this test should still pass for previous versions. -- RETURNING INSERT INTO source(sid, delta) VALUES(1, 40), (5, 50); BEGIN; MERGE INTO target t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta); SELECT * from target; ROLLBACK; BEGIN; MERGE INTO target t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 1 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta) RETURNING merge_action(), t.*; ROLLBACK; -- Views CREATE VIEW tv AS SELECT * FROM target; BEGIN; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta); SELECT * from tv; SELECT * from target; -- should also update the underlying table ROLLBACK; BEGIN; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta) RETURNING merge_action(), t.*; ROLLBACK; DROP VIEW tv; DELETE FROM source where sid in (1, 5); -- EXPLAIN CREATE TABLE ex_mtarget (a int, b int) WITH (autovacuum_enabled=off); CREATE TABLE ex_msource (a int, b int) WITH (autovacuum_enabled=off); INSERT INTO ex_mtarget SELECT i, i*10 FROM generate_series(1,100,2) i; INSERT INTO ex_msource SELECT i, i*10 FROM generate_series(1,100,1) i; CREATE FUNCTION explain_merge(query text) RETURNS SETOF text LANGUAGE plpgsql AS $$ DECLARE ln text; BEGIN FOR ln IN EXECUTE 'explain (analyze, timing off, summary off, buffers off, costs off) ' || query LOOP ln := regexp_replace(ln, '(Memory( Usage)?|Buckets|Batches): \S*', '\1: xxx', 'g'); RETURN NEXT ln; END LOOP; END; $$; -- only updates SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED THEN UPDATE SET b = t.b + 1'); -- only updates to selected tuples SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1'); -- updates + deletes SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1 WHEN MATCHED AND t.a >= 10 AND t.a <= 20 THEN DELETE'); -- only inserts SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN NOT MATCHED AND s.a < 10 THEN INSERT VALUES (a, b)'); -- all three SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1 WHEN MATCHED AND t.a >= 30 AND t.a <= 40 THEN DELETE WHEN NOT MATCHED AND s.a < 20 THEN INSERT VALUES (a, b)'); -- nothing SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a AND t.a < -1000 WHEN MATCHED AND t.a < 10 THEN DO NOTHING'); DROP TABLE ex_msource, ex_mtarget; DROP FUNCTION explain_merge(text); -- Subqueries BEGIN; MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED THEN UPDATE SET balance = (SELECT count(*) FROM sq_target); SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; BEGIN; MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND (SELECT count(*) > 0 FROM sq_target) THEN UPDATE SET balance = 42; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; BEGIN; MERGE INTO sq_target t USING v ON tid = sid AND (SELECT count(*) > 0 FROM sq_target) WHEN MATCHED THEN UPDATE SET balance = 42; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; -- Test RETURNING with subqueries BEGIN; MERGE INTO sq_target t USING v ON tid = sid AND (SELECT count(*) > 0 FROM sq_target) WHEN MATCHED THEN UPDATE SET balance = 42 RETURNING *; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; DROP TABLE sq_target CASCADE; DROP TABLE sq_source CASCADE; CREATE TABLE pa_target (tid integer, balance float, val text) PARTITION BY LIST (tid); CREATE TABLE part1 PARTITION OF pa_target FOR VALUES IN (1,4) WITH (autovacuum_enabled=off); CREATE TABLE part2 PARTITION OF pa_target FOR VALUES IN (2,5,6) WITH (autovacuum_enabled=off); CREATE TABLE part3 PARTITION OF pa_target FOR VALUES IN (3,8,9) WITH (autovacuum_enabled=off); CREATE TABLE part4 PARTITION OF pa_target DEFAULT WITH (autovacuum_enabled=off); CREATE TABLE pa_source (sid integer, delta float); -- insert many rows to the source table INSERT INTO pa_source SELECT id, id * 10 FROM generate_series(1,14) AS id; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT id, id * 100, 'initial' FROM generate_series(1,14,2) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- same with a constant qual BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid AND tid = 1 WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- try updating the partition key column BEGIN; CREATE FUNCTION merge_func() RETURNS integer LANGUAGE plpgsql AS $$ DECLARE result integer; BEGIN MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET tid = tid + 1, balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); IF FOUND THEN GET DIAGNOSTICS result := ROW_COUNT; END IF; RETURN result; END; $$; SELECT merge_func(); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_target CASCADE; -- The target table is partitioned in the same way, but this time by attaching -- partitions which have columns in different order, dropped columns etc. CREATE TABLE pa_target (tid integer, balance float, val text) PARTITION BY LIST (tid); CREATE TABLE part1 (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE part2 (balance float, tid integer, val text) WITH (autovacuum_enabled=off); CREATE TABLE part3 (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE part4 (extraid text, tid integer, balance float, val text) WITH (autovacuum_enabled=off); ALTER TABLE part4 DROP COLUMN extraid; ALTER TABLE pa_target ATTACH PARTITION part1 FOR VALUES IN (1,4); ALTER TABLE pa_target ATTACH PARTITION part2 FOR VALUES IN (2,5,6); ALTER TABLE pa_target ATTACH PARTITION part3 FOR VALUES IN (3,8,9); ALTER TABLE pa_target ATTACH PARTITION part4 DEFAULT; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT id, id * 100, 'initial' FROM generate_series(1,14,2) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- same with a constant qual BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid AND tid IN (1, 5) WHEN MATCHED AND tid % 5 = 0 THEN DELETE WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- try updating the partition key column BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET tid = tid + 1, balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_source; DROP TABLE pa_target CASCADE; -- Sub-partitioning CREATE TABLE pa_target (logts timestamp, tid integer, balance float, val text) PARTITION BY RANGE (logts); CREATE TABLE part_m01 PARTITION OF pa_target FOR VALUES FROM ('2017-01-01') TO ('2017-02-01') PARTITION BY LIST (tid); CREATE TABLE part_m01_odd PARTITION OF part_m01 FOR VALUES IN (1,3,5,7,9) WITH (autovacuum_enabled=off); CREATE TABLE part_m01_even PARTITION OF part_m01 FOR VALUES IN (2,4,6,8) WITH (autovacuum_enabled=off); CREATE TABLE part_m02 PARTITION OF pa_target FOR VALUES FROM ('2017-02-01') TO ('2017-03-01') PARTITION BY LIST (tid); CREATE TABLE part_m02_odd PARTITION OF part_m02 FOR VALUES IN (1,3,5,7,9) WITH (autovacuum_enabled=off); CREATE TABLE part_m02_even PARTITION OF part_m02 FOR VALUES IN (2,4,6,8) WITH (autovacuum_enabled=off); CREATE TABLE pa_source (sid integer, delta float) WITH (autovacuum_enabled=off); -- insert many rows to the source table INSERT INTO pa_source SELECT id, id * 10 FROM generate_series(1,14) AS id; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT '2017-01-31', id, id * 100, 'initial' FROM generate_series(1,9,3) AS id; INSERT INTO pa_target SELECT '2017-02-28', id, id * 100, 'initial' FROM generate_series(2,9,3) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING (SELECT '2017-01-15' AS slogts, * FROM pa_source WHERE sid < 10) s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (slogts::timestamp, sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_source; DROP TABLE pa_target CASCADE; -- some complex joins on the source side -- source relation is an unaliased join MERGE INTO cj_target t USING cj_source1 s1 INNER JOIN cj_source2 s2 ON sid1 = sid2 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid1, delta, sval); -- try accessing columns from either side of the source join MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 AND scat = 20 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid2, delta, sval) WHEN MATCHED THEN DELETE; -- some simple expressions in INSERT targetlist MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid2, delta + scat, sval) WHEN MATCHED THEN UPDATE SET val = val || ' updated by merge'; MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 AND scat = 20 ON t.tid = sid1 WHEN MATCHED THEN UPDATE SET val = val || ' ' || delta::text; SELECT * FROM cj_target ORDER BY tid; ALTER TABLE cj_source1 RENAME COLUMN sid1 TO sid; ALTER TABLE cj_source2 RENAME COLUMN sid2 TO sid; TRUNCATE cj_target; MERGE INTO cj_target t USING cj_source1 s1 INNER JOIN cj_source2 s2 ON s1.sid = s2.sid ON t.tid = s1.sid WHEN NOT MATCHED THEN INSERT VALUES (s2.sid, delta, sval); DROP TABLE cj_source2, cj_source1; DROP TABLE cj_target CASCADE; -- Function scans MERGE INTO fs_target t USING generate_series(1,100,1) AS id ON t.a = id WHEN MATCHED THEN UPDATE SET b = b + id WHEN NOT MATCHED THEN INSERT VALUES (id, -1); MERGE INTO fs_target t USING generate_series(1,100,2) AS id ON t.a = id WHEN MATCHED THEN UPDATE SET b = b + id, c = 'updated '|| id.*::text WHEN NOT MATCHED THEN INSERT VALUES (id, -1, 'inserted ' || id.*::text); SELECT count(*) FROM fs_target; DROP TABLE fs_target CASCADE; -- SERIALIZABLE test -- handled in isolation tests -- Inheritance-based partitioning CREATE TABLE measurement ( city_id int not null, logdate date not null, peaktemp int, unitsales int ) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2006m02 ( CHECK ( logdate >= DATE '2006-02-01' AND logdate < DATE '2006-03-01' ) ) INHERITS (measurement) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2006m03 ( CHECK ( logdate >= DATE '2006-03-01' AND logdate < DATE '2006-04-01' ) ) INHERITS (measurement) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2007m01 ( filler text, peaktemp int, logdate date not null, city_id int not null, unitsales int CHECK ( logdate >= DATE '2007-01-01' AND logdate < DATE '2007-02-01') ) WITH (autovacuum_enabled=off); ALTER TABLE measurement_y2007m01 DROP COLUMN filler; ALTER TABLE measurement_y2007m01 INHERIT measurement; INSERT INTO measurement VALUES (0, '2005-07-21', 5, 15); CREATE OR REPLACE FUNCTION measurement_insert_trigger() RETURNS TRIGGER AS $$ BEGIN IF ( NEW.logdate >= DATE '2006-02-01' AND NEW.logdate < DATE '2006-03-01' ) THEN INSERT INTO measurement_y2006m02 VALUES (NEW.*); ELSIF ( NEW.logdate >= DATE '2006-03-01' AND NEW.logdate < DATE '2006-04-01' ) THEN INSERT INTO measurement_y2006m03 VALUES (NEW.*); ELSIF ( NEW.logdate >= DATE '2007-01-01' AND NEW.logdate < DATE '2007-02-01' ) THEN INSERT INTO measurement_y2007m01 (city_id, logdate, peaktemp, unitsales) VALUES (NEW.*); ELSE RAISE EXCEPTION 'Date out of range. Fix the measurement_insert_trigger() function!'; END IF; RETURN NULL; END; $$ LANGUAGE plpgsql ; CREATE TRIGGER insert_measurement_trigger BEFORE INSERT ON measurement FOR EACH ROW EXECUTE PROCEDURE measurement_insert_trigger(); INSERT INTO measurement VALUES (1, '2006-02-10', 35, 10); INSERT INTO measurement VALUES (1, '2006-02-16', 45, 20); INSERT INTO measurement VALUES (1, '2006-03-17', 25, 10); INSERT INTO measurement VALUES (1, '2006-03-27', 15, 40); INSERT INTO measurement VALUES (1, '2007-01-15', 10, 10); INSERT INTO measurement VALUES (1, '2007-01-17', 10, 10); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate; CREATE TABLE new_measurement (LIKE measurement) WITH (autovacuum_enabled=off); INSERT INTO new_measurement VALUES (0, '2005-07-21', 25, 20); INSERT INTO new_measurement VALUES (1, '2006-03-01', 20, 10); INSERT INTO new_measurement VALUES (1, '2006-02-16', 50, 10); INSERT INTO new_measurement VALUES (2, '2006-02-10', 20, 20); INSERT INTO new_measurement VALUES (1, '2006-03-27', NULL, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-17', NULL, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-15', 5, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-16', 10, 10); BEGIN; MERGE INTO ONLY measurement m USING new_measurement nm ON (m.city_id = nm.city_id and m.logdate=nm.logdate) WHEN MATCHED AND nm.peaktemp IS NULL THEN DELETE WHEN MATCHED THEN UPDATE SET peaktemp = greatest(m.peaktemp, nm.peaktemp), unitsales = m.unitsales + coalesce(nm.unitsales, 0) WHEN NOT MATCHED THEN INSERT (city_id, logdate, peaktemp, unitsales) VALUES (city_id, logdate, peaktemp, unitsales); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate, peaktemp; ROLLBACK; MERGE into measurement m USING new_measurement nm ON (m.city_id = nm.city_id and m.logdate=nm.logdate) WHEN MATCHED AND nm.peaktemp IS NULL THEN DELETE WHEN MATCHED THEN UPDATE SET peaktemp = greatest(m.peaktemp, nm.peaktemp), unitsales = m.unitsales + coalesce(nm.unitsales, 0) WHEN NOT MATCHED THEN INSERT (city_id, logdate, peaktemp, unitsales) VALUES (city_id, logdate, peaktemp, unitsales); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate; BEGIN; MERGE INTO new_measurement nm USING ONLY measurement m ON (nm.city_id = m.city_id and nm.logdate=m.logdate) WHEN MATCHED THEN DELETE; SELECT * FROM new_measurement ORDER BY city_id, logdate; ROLLBACK; MERGE INTO new_measurement nm USING measurement m ON (nm.city_id = m.city_id and nm.logdate=m.logdate) WHEN MATCHED THEN DELETE; SELECT * FROM new_measurement ORDER BY city_id, logdate; DROP TABLE measurement, new_measurement CASCADE; DROP FUNCTION measurement_insert_trigger(); RESET SESSION AUTHORIZATION; DROP TABLE target CASCADE; DROP TABLE target2 CASCADE; DROP TABLE source, source2; DROP FUNCTION merge_trigfunc(); REVOKE CREATE ON SCHEMA public FROM regress_merge_privs; DROP USER regress_merge_privs; DROP USER regress_merge_no_privs; \o \ir :TEST_LOAD_HT_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE USER regress_merge_privs; CREATE USER regress_merge_no_privs; DROP TABLE IF EXISTS target; DROP TABLE IF EXISTS source; CREATE TABLE target (tid integer, balance integer) WITH (autovacuum_enabled=off); SELECT create_hypertable('target', 'tid', chunk_time_interval => 3); create_hypertable --------------------- (1,public,target,t) CREATE TABLE source (sid integer, delta integer) -- no index WITH (autovacuum_enabled=off); INSERT INTO target VALUES (1, 10); INSERT INTO target VALUES (2, 20); INSERT INTO target VALUES (3, 30); SELECT t.ctid is not null as matched, t.*, s.* FROM source s FULL OUTER JOIN target t ON s.sid = t.tid ORDER BY t.tid, s.sid; matched | tid | balance | sid | delta ---------+-----+---------+-----+------- t | 1 | 10 | | t | 2 | 20 | | t | 3 | 30 | | ALTER TABLE target OWNER TO regress_merge_privs; ALTER TABLE source OWNER TO regress_merge_privs; CREATE TABLE target2 (tid integer, balance integer) WITH (autovacuum_enabled=off); SELECT create_hypertable('target2', 'tid', chunk_time_interval => 3); create_hypertable ---------------------- (2,public,target2,t) CREATE TABLE source2 (sid integer, delta integer) WITH (autovacuum_enabled=off); ALTER TABLE target2 OWNER TO regress_merge_no_privs; ALTER TABLE source2 OWNER TO regress_merge_no_privs; GRANT INSERT ON target TO regress_merge_no_privs; GRANT CREATE ON SCHEMA public TO regress_merge_privs; SET SESSION AUTHORIZATION regress_merge_privs; CREATE TABLE sq_target (tid integer NOT NULL, balance integer) WITH (autovacuum_enabled=off); SELECT create_hypertable('sq_target', 'tid', chunk_time_interval => 3); create_hypertable ------------------------ (3,public,sq_target,t) CREATE TABLE sq_source (delta integer, sid integer, balance integer DEFAULT 0) WITH (autovacuum_enabled=off); INSERT INTO sq_target(tid, balance) VALUES (1,100), (2,200), (3,300); INSERT INTO sq_source(sid, delta) VALUES (1,10), (2,20), (4,40); -- conditional WHEN clause CREATE TABLE wq_target (tid integer not null, balance integer DEFAULT -1) WITH (autovacuum_enabled=off); SELECT create_hypertable('wq_target', 'tid', chunk_time_interval => 3); create_hypertable ------------------------ (4,public,wq_target,t) CREATE TABLE wq_source (balance integer, sid integer) WITH (autovacuum_enabled=off); INSERT INTO wq_source (sid, balance) VALUES (1, 100); -- some complex joins on the source side CREATE TABLE cj_target (tid integer, balance float, val text) WITH (autovacuum_enabled=off); SELECT create_hypertable('cj_target', 'tid', chunk_time_interval => 3); create_hypertable ------------------------ (5,public,cj_target,t) CREATE TABLE cj_source1 (sid1 integer, scat integer, delta integer) WITH (autovacuum_enabled=off); CREATE TABLE cj_source2 (sid2 integer, sval text) WITH (autovacuum_enabled=off); INSERT INTO cj_source1 VALUES (1, 10, 100); INSERT INTO cj_source1 VALUES (1, 20, 200); INSERT INTO cj_source1 VALUES (2, 20, 300); INSERT INTO cj_source1 VALUES (3, 10, 400); INSERT INTO cj_source2 VALUES (1, 'initial source2'); INSERT INTO cj_source2 VALUES (2, 'initial source2'); INSERT INTO cj_source2 VALUES (3, 'initial source2'); CREATE TABLE fs_target (a int, b int, c text) WITH (autovacuum_enabled=off); SELECT create_hypertable('fs_target', 'a', chunk_time_interval => 3); create_hypertable ------------------------ (6,public,fs_target,t) -- run tests on hypertable \o :TEST_RESULTS_WITH_HYPERTABLE \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- -- Errors -- MERGE INTO target t RANDOMWORD USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:12: ERROR: syntax error at or near "RANDOMWORD" LINE 1: MERGE INTO target t RANDOMWORD ^ -- MATCHED/INSERT error MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:18: ERROR: syntax error at or near "INSERT" LINE 5: INSERT DEFAULT VALUES; ^ -- incorrectly specifying INTO target MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT INTO target DEFAULT VALUES; psql:include/ts_merge_query.sql:24: ERROR: syntax error at or near "INTO" LINE 5: INSERT INTO target DEFAULT VALUES; ^ -- Multiple VALUES clause MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (1,1), (2,2); psql:include/ts_merge_query.sql:30: ERROR: syntax error at or near "," LINE 5: INSERT VALUES (1,1), (2,2); ^ -- SELECT query for INSERT MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT SELECT (1, 1); psql:include/ts_merge_query.sql:36: ERROR: syntax error at or near "SELECT" LINE 5: INSERT SELECT (1, 1); ^ -- NOT MATCHED/UPDATE MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:42: ERROR: syntax error at or near "UPDATE" LINE 5: UPDATE SET balance = 0; ^ -- UPDATE tablename MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE target SET balance = 0; psql:include/ts_merge_query.sql:48: ERROR: syntax error at or near "target" LINE 5: UPDATE target SET balance = 0; ^ -- source and target names the same MERGE INTO target USING target ON tid = tid WHEN MATCHED THEN DO NOTHING; psql:include/ts_merge_query.sql:53: ERROR: name "target" specified more than once DETAIL: The name is used both as MERGE target table and data source. -- used in a CTE WITH foo AS ( MERGE INTO target USING source ON (true) WHEN MATCHED THEN DELETE ) SELECT * FROM foo; psql:include/ts_merge_query.sql:58: ERROR: WITH query "foo" does not have a RETURNING clause LINE 4: ) SELECT * FROM foo; ^ -- used in COPY COPY ( MERGE INTO target USING source ON (true) WHEN MATCHED THEN DELETE ) TO stdout; psql:include/ts_merge_query.sql:63: ERROR: COPY query must have a RETURNING clause -- unsupported relation types -- view CREATE VIEW tv AS SELECT * FROM target; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; DROP VIEW tv; -- materialized view CREATE MATERIALIZED VIEW mv AS SELECT * FROM target; MERGE INTO mv t USING source s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:81: ERROR: cannot execute MERGE on relation "mv" DETAIL: This operation is not supported for materialized views. DROP MATERIALIZED VIEW mv; -- permissions MERGE INTO target USING source2 ON target.tid = source2.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:90: ERROR: permission denied for table source2 GRANT INSERT ON target TO regress_merge_no_privs; SET SESSION AUTHORIZATION regress_merge_no_privs; MERGE INTO target USING source2 ON target.tid = source2.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:99: ERROR: permission denied for table target GRANT UPDATE ON target2 TO regress_merge_privs; SET SESSION AUTHORIZATION regress_merge_privs; MERGE INTO target2 USING source ON target2.tid = source.sid WHEN MATCHED THEN DELETE; psql:include/ts_merge_query.sql:108: ERROR: permission denied for table target2 MERGE INTO target2 USING source ON target2.tid = source.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:114: ERROR: permission denied for table target2 -- check if the target can be accessed from source relation subquery; we should -- not be able to do so MERGE INTO target t USING (SELECT * FROM source WHERE t.tid > sid) s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:122: ERROR: invalid reference to FROM-clause entry for table "t" LINE 2: USING (SELECT * FROM source WHERE t.tid > sid) s ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. -- -- initial tests -- -- zero rows in source has no effect MERGE INTO target USING source ON target.tid = source.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; ROLLBACK; -- insert some non-matching source rows to work from INSERT INTO source VALUES (4, 40); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN DO NOTHING; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (5, 50); SELECT * FROM target ORDER BY tid; ROLLBACK; -- index plans INSERT INTO target SELECT generate_series(1000,2500), 0; ALTER TABLE target ADD PRIMARY KEY (tid); ANALYZE target; DELETE FROM target WHERE tid > 100; ANALYZE target; -- insert some matching source rows to work from INSERT INTO source VALUES (2, 5); INSERT INTO source VALUES (3, 20); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- equivalent of an UPDATE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; SELECT * FROM target ORDER BY tid; ROLLBACK; -- equivalent of a DELETE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DO NOTHING; SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, NULL); SELECT * FROM target ORDER BY tid; ROLLBACK; -- duplicate source row causes multiple target row update ERROR INSERT INTO source VALUES (2, 5); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:241: ERROR: MERGE command cannot affect row a second time HINT: Ensure that not more than one source row matches any one target row. ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; psql:include/ts_merge_query.sql:249: ERROR: MERGE command cannot affect row a second time HINT: Ensure that not more than one source row matches any one target row. ROLLBACK; -- remove duplicate MATCHED data from source data DELETE FROM source WHERE sid = 2; INSERT INTO source VALUES (2, 5); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- duplicate source row on INSERT should fail because of target_pkey INSERT INTO source VALUES (4, 40); BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, NULL); psql:include/ts_merge_query.sql:265: ERROR: duplicate key value violates unique constraint "2_2_target_pkey" DETAIL: Key (tid)=(4) already exists. SELECT * FROM target ORDER BY tid; psql:include/ts_merge_query.sql:266: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; -- remove duplicate NOT MATCHED data from source data DELETE FROM source WHERE sid = 4; INSERT INTO source VALUES (4, 40); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- remove constraints alter table target drop CONSTRAINT target_pkey; alter table target alter column tid drop not null; psql:include/ts_merge_query.sql:277: ERROR: cannot drop not-null constraint from a time-partitioned column -- multiple actions BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, 4) WHEN MATCHED THEN UPDATE SET balance = 0; SELECT * FROM target ORDER BY tid; ROLLBACK; -- should be equivalent BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0 WHEN NOT MATCHED THEN INSERT VALUES (4, 4); SELECT * FROM target ORDER BY tid; ROLLBACK; -- column references -- do a simple equivalent of an UPDATE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance + s.delta; SELECT * FROM target ORDER BY tid; ROLLBACK; -- do a simple equivalent of an INSERT SELECT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- and again with duplicate source rows INSERT INTO source VALUES (5, 50); INSERT INTO source VALUES (5, 50); -- do a simple equivalent of an INSERT SELECT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- removing duplicate source rows DELETE FROM source WHERE sid = 5; -- and again with explicitly identified column list BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- and again with a subtle error: referring to non-existent target row for NOT MATCHED MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (t.tid, s.delta); psql:include/ts_merge_query.sql:356: ERROR: invalid reference to FROM-clause entry for table "t" LINE 5: INSERT (tid, balance) VALUES (t.tid, s.delta); ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. -- and again with a constant ON clause BEGIN; MERGE INTO target t USING source AS s ON (SELECT true) WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (t.tid, s.delta); psql:include/ts_merge_query.sql:364: ERROR: invalid reference to FROM-clause entry for table "t" LINE 5: INSERT (tid, balance) VALUES (t.tid, s.delta); ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. SELECT * FROM target ORDER BY tid; psql:include/ts_merge_query.sql:365: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; -- now the classic UPSERT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance + s.delta WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- this time with a FALSE condition MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND FALSE THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; -- this time with an actual condition which returns false MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance <> 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; BEGIN; -- and now with a condition which returns true MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance = 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; ROLLBACK; -- conditions in the NOT MATCHED clause can only refer to source columns BEGIN; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND t.balance = 100 THEN INSERT (tid) VALUES (s.sid); psql:include/ts_merge_query.sql:408: ERROR: invalid reference to FROM-clause entry for table "t" LINE 3: WHEN NOT MATCHED AND t.balance = 100 THEN ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. SELECT * FROM wq_target; psql:include/ts_merge_query.sql:409: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance = 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; -- conditions in MATCHED clause can refer to both source and target SELECT * FROM wq_source; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND s.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check if AND works MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 AND s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 AND s.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check if OR works MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 OR s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 199 OR s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check source-side whole-row references BEGIN; MERGE INTO wq_target t USING wq_source s ON (t.tid = s.sid) WHEN matched and t = s or t.tid = s.sid THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; ROLLBACK; -- check if subqueries work in the conditions? MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance > (SELECT max(balance) FROM target) THEN UPDATE SET balance = t.balance + s.balance; -- check if we can access system columns in the conditions MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.xmin = t.xmax THEN UPDATE SET balance = t.balance + s.balance; psql:include/ts_merge_query.sql:477: ERROR: cannot use system column "xmin" in MERGE WHEN condition LINE 3: WHEN MATCHED AND t.xmin = t.xmax THEN ^ MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.tableoid >= 0 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; DROP TABLE wq_target CASCADE; DROP TABLE wq_source; -- test triggers create or replace function merge_trigfunc () returns trigger language plpgsql as $$ DECLARE line text; BEGIN SELECT INTO line format('%s %s %s trigger%s', TG_WHEN, TG_OP, TG_LEVEL, CASE WHEN TG_OP = 'INSERT' AND TG_LEVEL = 'ROW' THEN format(' row: %s', NEW) WHEN TG_OP = 'UPDATE' AND TG_LEVEL = 'ROW' THEN format(' row: %s -> %s', OLD, NEW) WHEN TG_OP = 'DELETE' AND TG_LEVEL = 'ROW' THEN format(' row: %s', OLD) END); RAISE NOTICE '%', line; IF (TG_WHEN = 'BEFORE' AND TG_LEVEL = 'ROW') THEN IF (TG_OP = 'DELETE') THEN RETURN OLD; ELSE RETURN NEW; END IF; ELSE RETURN NULL; END IF; END; $$; CREATE TRIGGER merge_bsi BEFORE INSERT ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bsu BEFORE UPDATE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bsd BEFORE DELETE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asi AFTER INSERT ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asu AFTER UPDATE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asd AFTER DELETE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bri BEFORE INSERT ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bru BEFORE UPDATE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_brd BEFORE DELETE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_ari AFTER INSERT ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_aru AFTER UPDATE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_ard AFTER DELETE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); -- now the classic UPSERT, with a DELETE BEGIN; UPDATE target SET balance = 0 WHERE tid = 3; --EXPLAIN (ANALYZE ON, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND t.balance > s.delta THEN UPDATE SET balance = t.balance - s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- Test behavior of triggers that turn UPDATE/DELETE into no-ops create or replace function skip_merge_op() returns trigger language plpgsql as $$ BEGIN RETURN NULL; END; $$; SELECT * FROM target full outer join source on (sid = tid); create trigger merge_skip BEFORE INSERT OR UPDATE or DELETE ON target FOR EACH ROW EXECUTE FUNCTION skip_merge_op(); DO $$ DECLARE result integer; BEGIN MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND s.sid = 3 THEN UPDATE SET balance = t.balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT VALUES (sid, delta); IF FOUND THEN RAISE NOTICE 'Found'; ELSE RAISE NOTICE 'Not found'; END IF; GET DIAGNOSTICS result := ROW_COUNT; RAISE NOTICE 'ROW_COUNT = %', result; END; $$; SELECT * FROM target FULL OUTER JOIN source ON (sid = tid); DROP TRIGGER merge_skip ON target; DROP FUNCTION skip_merge_op(); -- test from PL/pgSQL -- make sure MERGE INTO isn't interpreted to mean returning variables like SELECT INTO BEGIN; DO LANGUAGE plpgsql $$ BEGIN MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND t.balance > s.delta THEN UPDATE SET balance = t.balance - s.delta; END; $$; ROLLBACK; --source constants BEGIN; MERGE INTO target t USING (SELECT 9 AS sid, 57 AS delta) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; --source query BEGIN; MERGE INTO target t USING (SELECT sid, delta FROM source WHERE delta > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT sid, delta as newname FROM source WHERE delta > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.newname); SELECT * FROM target ORDER BY tid; ROLLBACK; --self-merge BEGIN; MERGE INTO target t1 USING target t2 ON t1.tid = t2.tid WHEN MATCHED THEN UPDATE SET balance = t1.balance + t2.balance WHEN NOT MATCHED THEN INSERT VALUES (t2.tid, t2.balance); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT tid as sid, balance as delta FROM target WHERE balance > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT sid, max(delta) AS delta FROM source GROUP BY sid HAVING count(*) = 1 ORDER BY sid ASC) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- plpgsql parameters and results BEGIN; CREATE FUNCTION merge_func (p_id integer, p_bal integer) RETURNS INTEGER LANGUAGE plpgsql AS $$ DECLARE result integer; BEGIN MERGE INTO target t USING (SELECT p_id AS sid) AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance - p_bal; IF FOUND THEN GET DIAGNOSTICS result := ROW_COUNT; END IF; RETURN result; END; $$; SELECT merge_func(3, 4); SELECT * FROM target ORDER BY tid; ROLLBACK; -- PREPARE BEGIN; prepare foom as merge into target t using (select 1 as sid) s on (t.tid = s.sid) when matched then update set balance = 1; psql:include/ts_merge_query.sql:685: ERROR: prepared statement "foom" already exists execute foom; psql:include/ts_merge_query.sql:686: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; BEGIN; PREPARE foom2 (integer, integer) AS MERGE INTO target t USING (SELECT 1) s ON t.tid = $1 WHEN MATCHED THEN UPDATE SET balance = $2; psql:include/ts_merge_query.sql:695: ERROR: prepared statement "foom2" already exists --EXPLAIN (ANALYZE ON, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) execute foom2 (1, 1); psql:include/ts_merge_query.sql:697: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; -- subqueries in source relation BEGIN; MERGE INTO sq_target t USING (SELECT * FROM sq_source) s ON tid = sid WHEN MATCHED AND t.balance > delta THEN UPDATE SET balance = t.balance + delta; SELECT * FROM sq_target ORDER BY tid; ROLLBACK; -- try a view CREATE VIEW v AS SELECT * FROM sq_source WHERE sid < 2; BEGIN; MERGE INTO sq_target USING v ON tid = sid WHEN MATCHED THEN UPDATE SET balance = v.balance + delta; SELECT * FROM sq_target ORDER BY tid; ROLLBACK; -- ambiguous reference to a column BEGIN; MERGE INTO sq_target USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; psql:include/ts_merge_query.sql:732: ERROR: column reference "balance" is ambiguous LINE 5: UPDATE SET balance = balance + delta ^ ROLLBACK; BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; SELECT * FROM sq_target; ROLLBACK; -- CTEs BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); WITH targq AS ( SELECT * FROM v ) MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; ROLLBACK; -- RETURNING BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE RETURNING *; ROLLBACK; -- PG17-specific tests for views, returning and merge_action. These throw syntax errors for previous versions of Postgres. -- However, since the error is the same for both hypertables and regular tables, this test should still pass for previous versions. -- RETURNING INSERT INTO source(sid, delta) VALUES(1, 40), (5, 50); BEGIN; MERGE INTO target t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta); SELECT * from target; ROLLBACK; BEGIN; MERGE INTO target t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 1 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta) RETURNING merge_action(), t.*; ROLLBACK; -- Views CREATE VIEW tv AS SELECT * FROM target; BEGIN; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta); SELECT * from tv; SELECT * from target; -- should also update the underlying table ROLLBACK; BEGIN; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta) RETURNING merge_action(), t.*; ROLLBACK; DROP VIEW tv; DELETE FROM source where sid in (1, 5); -- EXPLAIN CREATE TABLE ex_mtarget (a int, b int) WITH (autovacuum_enabled=off); CREATE TABLE ex_msource (a int, b int) WITH (autovacuum_enabled=off); INSERT INTO ex_mtarget SELECT i, i*10 FROM generate_series(1,100,2) i; INSERT INTO ex_msource SELECT i, i*10 FROM generate_series(1,100,1) i; CREATE FUNCTION explain_merge(query text) RETURNS SETOF text LANGUAGE plpgsql AS $$ DECLARE ln text; BEGIN FOR ln IN EXECUTE 'explain (analyze, timing off, summary off, buffers off, costs off) ' || query LOOP ln := regexp_replace(ln, '(Memory( Usage)?|Buckets|Batches): \S*', '\1: xxx', 'g'); RETURN NEXT ln; END LOOP; END; $$; -- only updates SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED THEN UPDATE SET b = t.b + 1'); -- only updates to selected tuples SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1'); -- updates + deletes SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1 WHEN MATCHED AND t.a >= 10 AND t.a <= 20 THEN DELETE'); -- only inserts SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN NOT MATCHED AND s.a < 10 THEN INSERT VALUES (a, b)'); -- all three SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1 WHEN MATCHED AND t.a >= 30 AND t.a <= 40 THEN DELETE WHEN NOT MATCHED AND s.a < 20 THEN INSERT VALUES (a, b)'); -- nothing SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a AND t.a < -1000 WHEN MATCHED AND t.a < 10 THEN DO NOTHING'); DROP TABLE ex_msource, ex_mtarget; DROP FUNCTION explain_merge(text); -- Subqueries BEGIN; MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED THEN UPDATE SET balance = (SELECT count(*) FROM sq_target); SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; BEGIN; MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND (SELECT count(*) > 0 FROM sq_target) THEN UPDATE SET balance = 42; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; BEGIN; MERGE INTO sq_target t USING v ON tid = sid AND (SELECT count(*) > 0 FROM sq_target) WHEN MATCHED THEN UPDATE SET balance = 42; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; -- Test RETURNING with subqueries BEGIN; MERGE INTO sq_target t USING v ON tid = sid AND (SELECT count(*) > 0 FROM sq_target) WHEN MATCHED THEN UPDATE SET balance = 42 RETURNING *; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; DROP TABLE sq_target CASCADE; DROP TABLE sq_source CASCADE; CREATE TABLE pa_target (tid integer, balance float, val text) PARTITION BY LIST (tid); CREATE TABLE part1 PARTITION OF pa_target FOR VALUES IN (1,4) WITH (autovacuum_enabled=off); CREATE TABLE part2 PARTITION OF pa_target FOR VALUES IN (2,5,6) WITH (autovacuum_enabled=off); CREATE TABLE part3 PARTITION OF pa_target FOR VALUES IN (3,8,9) WITH (autovacuum_enabled=off); CREATE TABLE part4 PARTITION OF pa_target DEFAULT WITH (autovacuum_enabled=off); CREATE TABLE pa_source (sid integer, delta float); -- insert many rows to the source table INSERT INTO pa_source SELECT id, id * 10 FROM generate_series(1,14) AS id; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT id, id * 100, 'initial' FROM generate_series(1,14,2) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- same with a constant qual BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid AND tid = 1 WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- try updating the partition key column BEGIN; CREATE FUNCTION merge_func() RETURNS integer LANGUAGE plpgsql AS $$ DECLARE result integer; BEGIN MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET tid = tid + 1, balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); IF FOUND THEN GET DIAGNOSTICS result := ROW_COUNT; END IF; RETURN result; END; $$; SELECT merge_func(); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_target CASCADE; -- The target table is partitioned in the same way, but this time by attaching -- partitions which have columns in different order, dropped columns etc. CREATE TABLE pa_target (tid integer, balance float, val text) PARTITION BY LIST (tid); CREATE TABLE part1 (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE part2 (balance float, tid integer, val text) WITH (autovacuum_enabled=off); CREATE TABLE part3 (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE part4 (extraid text, tid integer, balance float, val text) WITH (autovacuum_enabled=off); ALTER TABLE part4 DROP COLUMN extraid; ALTER TABLE pa_target ATTACH PARTITION part1 FOR VALUES IN (1,4); ALTER TABLE pa_target ATTACH PARTITION part2 FOR VALUES IN (2,5,6); ALTER TABLE pa_target ATTACH PARTITION part3 FOR VALUES IN (3,8,9); ALTER TABLE pa_target ATTACH PARTITION part4 DEFAULT; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT id, id * 100, 'initial' FROM generate_series(1,14,2) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- same with a constant qual BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid AND tid IN (1, 5) WHEN MATCHED AND tid % 5 = 0 THEN DELETE WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- try updating the partition key column BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET tid = tid + 1, balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_source; DROP TABLE pa_target CASCADE; -- Sub-partitioning CREATE TABLE pa_target (logts timestamp, tid integer, balance float, val text) PARTITION BY RANGE (logts); CREATE TABLE part_m01 PARTITION OF pa_target FOR VALUES FROM ('2017-01-01') TO ('2017-02-01') PARTITION BY LIST (tid); CREATE TABLE part_m01_odd PARTITION OF part_m01 FOR VALUES IN (1,3,5,7,9) WITH (autovacuum_enabled=off); CREATE TABLE part_m01_even PARTITION OF part_m01 FOR VALUES IN (2,4,6,8) WITH (autovacuum_enabled=off); CREATE TABLE part_m02 PARTITION OF pa_target FOR VALUES FROM ('2017-02-01') TO ('2017-03-01') PARTITION BY LIST (tid); CREATE TABLE part_m02_odd PARTITION OF part_m02 FOR VALUES IN (1,3,5,7,9) WITH (autovacuum_enabled=off); CREATE TABLE part_m02_even PARTITION OF part_m02 FOR VALUES IN (2,4,6,8) WITH (autovacuum_enabled=off); CREATE TABLE pa_source (sid integer, delta float) WITH (autovacuum_enabled=off); -- insert many rows to the source table INSERT INTO pa_source SELECT id, id * 10 FROM generate_series(1,14) AS id; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT '2017-01-31', id, id * 100, 'initial' FROM generate_series(1,9,3) AS id; INSERT INTO pa_target SELECT '2017-02-28', id, id * 100, 'initial' FROM generate_series(2,9,3) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING (SELECT '2017-01-15' AS slogts, * FROM pa_source WHERE sid < 10) s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (slogts::timestamp, sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_source; DROP TABLE pa_target CASCADE; -- some complex joins on the source side -- source relation is an unaliased join MERGE INTO cj_target t USING cj_source1 s1 INNER JOIN cj_source2 s2 ON sid1 = sid2 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid1, delta, sval); -- try accessing columns from either side of the source join MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 AND scat = 20 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid2, delta, sval) WHEN MATCHED THEN DELETE; -- some simple expressions in INSERT targetlist MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid2, delta + scat, sval) WHEN MATCHED THEN UPDATE SET val = val || ' updated by merge'; MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 AND scat = 20 ON t.tid = sid1 WHEN MATCHED THEN UPDATE SET val = val || ' ' || delta::text; SELECT * FROM cj_target ORDER BY tid; ALTER TABLE cj_source1 RENAME COLUMN sid1 TO sid; ALTER TABLE cj_source2 RENAME COLUMN sid2 TO sid; TRUNCATE cj_target; MERGE INTO cj_target t USING cj_source1 s1 INNER JOIN cj_source2 s2 ON s1.sid = s2.sid ON t.tid = s1.sid WHEN NOT MATCHED THEN INSERT VALUES (s2.sid, delta, sval); DROP TABLE cj_source2, cj_source1; DROP TABLE cj_target CASCADE; -- Function scans MERGE INTO fs_target t USING generate_series(1,100,1) AS id ON t.a = id WHEN MATCHED THEN UPDATE SET b = b + id WHEN NOT MATCHED THEN INSERT VALUES (id, -1); MERGE INTO fs_target t USING generate_series(1,100,2) AS id ON t.a = id WHEN MATCHED THEN UPDATE SET b = b + id, c = 'updated '|| id.*::text WHEN NOT MATCHED THEN INSERT VALUES (id, -1, 'inserted ' || id.*::text); SELECT count(*) FROM fs_target; DROP TABLE fs_target CASCADE; -- SERIALIZABLE test -- handled in isolation tests -- Inheritance-based partitioning CREATE TABLE measurement ( city_id int not null, logdate date not null, peaktemp int, unitsales int ) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2006m02 ( CHECK ( logdate >= DATE '2006-02-01' AND logdate < DATE '2006-03-01' ) ) INHERITS (measurement) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2006m03 ( CHECK ( logdate >= DATE '2006-03-01' AND logdate < DATE '2006-04-01' ) ) INHERITS (measurement) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2007m01 ( filler text, peaktemp int, logdate date not null, city_id int not null, unitsales int CHECK ( logdate >= DATE '2007-01-01' AND logdate < DATE '2007-02-01') ) WITH (autovacuum_enabled=off); ALTER TABLE measurement_y2007m01 DROP COLUMN filler; ALTER TABLE measurement_y2007m01 INHERIT measurement; INSERT INTO measurement VALUES (0, '2005-07-21', 5, 15); CREATE OR REPLACE FUNCTION measurement_insert_trigger() RETURNS TRIGGER AS $$ BEGIN IF ( NEW.logdate >= DATE '2006-02-01' AND NEW.logdate < DATE '2006-03-01' ) THEN INSERT INTO measurement_y2006m02 VALUES (NEW.*); ELSIF ( NEW.logdate >= DATE '2006-03-01' AND NEW.logdate < DATE '2006-04-01' ) THEN INSERT INTO measurement_y2006m03 VALUES (NEW.*); ELSIF ( NEW.logdate >= DATE '2007-01-01' AND NEW.logdate < DATE '2007-02-01' ) THEN INSERT INTO measurement_y2007m01 (city_id, logdate, peaktemp, unitsales) VALUES (NEW.*); ELSE RAISE EXCEPTION 'Date out of range. Fix the measurement_insert_trigger() function!'; END IF; RETURN NULL; END; $$ LANGUAGE plpgsql ; CREATE TRIGGER insert_measurement_trigger BEFORE INSERT ON measurement FOR EACH ROW EXECUTE PROCEDURE measurement_insert_trigger(); INSERT INTO measurement VALUES (1, '2006-02-10', 35, 10); INSERT INTO measurement VALUES (1, '2006-02-16', 45, 20); INSERT INTO measurement VALUES (1, '2006-03-17', 25, 10); INSERT INTO measurement VALUES (1, '2006-03-27', 15, 40); INSERT INTO measurement VALUES (1, '2007-01-15', 10, 10); INSERT INTO measurement VALUES (1, '2007-01-17', 10, 10); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate; CREATE TABLE new_measurement (LIKE measurement) WITH (autovacuum_enabled=off); INSERT INTO new_measurement VALUES (0, '2005-07-21', 25, 20); INSERT INTO new_measurement VALUES (1, '2006-03-01', 20, 10); INSERT INTO new_measurement VALUES (1, '2006-02-16', 50, 10); INSERT INTO new_measurement VALUES (2, '2006-02-10', 20, 20); INSERT INTO new_measurement VALUES (1, '2006-03-27', NULL, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-17', NULL, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-15', 5, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-16', 10, 10); BEGIN; MERGE INTO ONLY measurement m USING new_measurement nm ON (m.city_id = nm.city_id and m.logdate=nm.logdate) WHEN MATCHED AND nm.peaktemp IS NULL THEN DELETE WHEN MATCHED THEN UPDATE SET peaktemp = greatest(m.peaktemp, nm.peaktemp), unitsales = m.unitsales + coalesce(nm.unitsales, 0) WHEN NOT MATCHED THEN INSERT (city_id, logdate, peaktemp, unitsales) VALUES (city_id, logdate, peaktemp, unitsales); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate, peaktemp; ROLLBACK; MERGE into measurement m USING new_measurement nm ON (m.city_id = nm.city_id and m.logdate=nm.logdate) WHEN MATCHED AND nm.peaktemp IS NULL THEN DELETE WHEN MATCHED THEN UPDATE SET peaktemp = greatest(m.peaktemp, nm.peaktemp), unitsales = m.unitsales + coalesce(nm.unitsales, 0) WHEN NOT MATCHED THEN INSERT (city_id, logdate, peaktemp, unitsales) VALUES (city_id, logdate, peaktemp, unitsales); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate; BEGIN; MERGE INTO new_measurement nm USING ONLY measurement m ON (nm.city_id = m.city_id and nm.logdate=m.logdate) WHEN MATCHED THEN DELETE; SELECT * FROM new_measurement ORDER BY city_id, logdate; ROLLBACK; MERGE INTO new_measurement nm USING measurement m ON (nm.city_id = m.city_id and nm.logdate=m.logdate) WHEN MATCHED THEN DELETE; SELECT * FROM new_measurement ORDER BY city_id, logdate; DROP TABLE measurement, new_measurement CASCADE; DROP FUNCTION measurement_insert_trigger(); RESET SESSION AUTHORIZATION; DROP TABLE target CASCADE; DROP TABLE target2 CASCADE; DROP TABLE source, source2; DROP FUNCTION merge_trigfunc(); REVOKE CREATE ON SCHEMA public FROM regress_merge_privs; DROP USER regress_merge_privs; DROP USER regress_merge_no_privs; \o :DIFF_CMD ================================================ FILE: test/expected/ts_merge-18.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER \set ON_ERROR_STOP 0 \set VERBOSITY default SET client_min_messages TO error; \set TEST_BASE_NAME ts_merge SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') AS "TEST_LOAD_NAME", format('include/%s_load_ht.sql', :'TEST_BASE_NAME') AS "TEST_LOAD_HT_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') AS "TEST_QUERY_NAME", format('%s/results/%s_ht_results.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') AS "TEST_RESULTS_WITH_HYPERTABLE", format('%s/results/%s_results.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') AS "TEST_RESULTS_WITH_NO_HYPERTABLE" \gset SELECT format('\! diff -u --label "Base pg table results" --label "Hypertable results" %s %s', :'TEST_RESULTS_WITH_HYPERTABLE', :'TEST_RESULTS_WITH_NO_HYPERTABLE') AS "DIFF_CMD" \gset \ir :TEST_LOAD_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE USER regress_merge_privs; CREATE USER regress_merge_no_privs; DROP TABLE IF EXISTS target; DROP TABLE IF EXISTS source; CREATE TABLE target (tid integer, balance integer) WITH (autovacuum_enabled=off); CREATE TABLE source (sid integer, delta integer) -- no index WITH (autovacuum_enabled=off); INSERT INTO target VALUES (1, 10); INSERT INTO target VALUES (2, 20); INSERT INTO target VALUES (3, 30); SELECT t.ctid is not null as matched, t.*, s.* FROM source s FULL OUTER JOIN target t ON s.sid = t.tid ORDER BY t.tid, s.sid; matched | tid | balance | sid | delta ---------+-----+---------+-----+------- t | 1 | 10 | | t | 2 | 20 | | t | 3 | 30 | | ALTER TABLE target OWNER TO regress_merge_privs; ALTER TABLE source OWNER TO regress_merge_privs; CREATE TABLE target2 (tid integer, balance integer) WITH (autovacuum_enabled=off); CREATE TABLE source2 (sid integer, delta integer) WITH (autovacuum_enabled=off); ALTER TABLE target2 OWNER TO regress_merge_no_privs; ALTER TABLE source2 OWNER TO regress_merge_no_privs; GRANT INSERT ON target TO regress_merge_no_privs; GRANT CREATE ON SCHEMA public TO regress_merge_privs; SET SESSION AUTHORIZATION regress_merge_privs; CREATE TABLE sq_target (tid integer NOT NULL, balance integer) WITH (autovacuum_enabled=off); CREATE TABLE sq_source (delta integer, sid integer, balance integer DEFAULT 0) WITH (autovacuum_enabled=off); INSERT INTO sq_target(tid, balance) VALUES (1,100), (2,200), (3,300); INSERT INTO sq_source(sid, delta) VALUES (1,10), (2,20), (4,40); -- conditional WHEN clause CREATE TABLE wq_target (tid integer not null, balance integer DEFAULT -1) WITH (autovacuum_enabled=off); CREATE TABLE wq_source (balance integer, sid integer) WITH (autovacuum_enabled=off); INSERT INTO wq_source (sid, balance) VALUES (1, 100); CREATE TABLE cj_target (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE cj_source1 (sid1 integer, scat integer, delta integer) WITH (autovacuum_enabled=off); CREATE TABLE cj_source2 (sid2 integer, sval text) WITH (autovacuum_enabled=off); INSERT INTO cj_source1 VALUES (1, 10, 100); INSERT INTO cj_source1 VALUES (1, 20, 200); INSERT INTO cj_source1 VALUES (2, 20, 300); INSERT INTO cj_source1 VALUES (3, 10, 400); INSERT INTO cj_source2 VALUES (1, 'initial source2'); INSERT INTO cj_source2 VALUES (2, 'initial source2'); INSERT INTO cj_source2 VALUES (3, 'initial source2'); CREATE TABLE fs_target (a int, b int, c text) WITH (autovacuum_enabled=off); -- run tests on normal table \o :TEST_RESULTS_WITH_NO_HYPERTABLE \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- -- Errors -- MERGE INTO target t RANDOMWORD USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:12: ERROR: syntax error at or near "RANDOMWORD" LINE 1: MERGE INTO target t RANDOMWORD ^ -- MATCHED/INSERT error MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:18: ERROR: syntax error at or near "INSERT" LINE 5: INSERT DEFAULT VALUES; ^ -- incorrectly specifying INTO target MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT INTO target DEFAULT VALUES; psql:include/ts_merge_query.sql:24: ERROR: syntax error at or near "INTO" LINE 5: INSERT INTO target DEFAULT VALUES; ^ -- Multiple VALUES clause MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (1,1), (2,2); psql:include/ts_merge_query.sql:30: ERROR: syntax error at or near "," LINE 5: INSERT VALUES (1,1), (2,2); ^ -- SELECT query for INSERT MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT SELECT (1, 1); psql:include/ts_merge_query.sql:36: ERROR: syntax error at or near "SELECT" LINE 5: INSERT SELECT (1, 1); ^ -- NOT MATCHED/UPDATE MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:42: ERROR: syntax error at or near "UPDATE" LINE 5: UPDATE SET balance = 0; ^ -- UPDATE tablename MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE target SET balance = 0; psql:include/ts_merge_query.sql:48: ERROR: syntax error at or near "target" LINE 5: UPDATE target SET balance = 0; ^ -- source and target names the same MERGE INTO target USING target ON tid = tid WHEN MATCHED THEN DO NOTHING; psql:include/ts_merge_query.sql:53: ERROR: name "target" specified more than once DETAIL: The name is used both as MERGE target table and data source. -- used in a CTE WITH foo AS ( MERGE INTO target USING source ON (true) WHEN MATCHED THEN DELETE ) SELECT * FROM foo; psql:include/ts_merge_query.sql:58: ERROR: WITH query "foo" does not have a RETURNING clause LINE 4: ) SELECT * FROM foo; ^ -- used in COPY COPY ( MERGE INTO target USING source ON (true) WHEN MATCHED THEN DELETE ) TO stdout; psql:include/ts_merge_query.sql:63: ERROR: COPY query must have a RETURNING clause -- unsupported relation types -- view CREATE VIEW tv AS SELECT * FROM target; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; DROP VIEW tv; -- materialized view CREATE MATERIALIZED VIEW mv AS SELECT * FROM target; MERGE INTO mv t USING source s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:81: ERROR: cannot execute MERGE on relation "mv" DETAIL: This operation is not supported for materialized views. DROP MATERIALIZED VIEW mv; -- permissions MERGE INTO target USING source2 ON target.tid = source2.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:90: ERROR: permission denied for table source2 GRANT INSERT ON target TO regress_merge_no_privs; SET SESSION AUTHORIZATION regress_merge_no_privs; MERGE INTO target USING source2 ON target.tid = source2.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:99: ERROR: permission denied for table target GRANT UPDATE ON target2 TO regress_merge_privs; SET SESSION AUTHORIZATION regress_merge_privs; MERGE INTO target2 USING source ON target2.tid = source.sid WHEN MATCHED THEN DELETE; psql:include/ts_merge_query.sql:108: ERROR: permission denied for table target2 MERGE INTO target2 USING source ON target2.tid = source.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:114: ERROR: permission denied for table target2 -- check if the target can be accessed from source relation subquery; we should -- not be able to do so MERGE INTO target t USING (SELECT * FROM source WHERE t.tid > sid) s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:122: ERROR: invalid reference to FROM-clause entry for table "t" LINE 2: USING (SELECT * FROM source WHERE t.tid > sid) s ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. -- -- initial tests -- -- zero rows in source has no effect MERGE INTO target USING source ON target.tid = source.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; ROLLBACK; -- insert some non-matching source rows to work from INSERT INTO source VALUES (4, 40); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN DO NOTHING; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (5, 50); SELECT * FROM target ORDER BY tid; ROLLBACK; -- index plans INSERT INTO target SELECT generate_series(1000,2500), 0; ALTER TABLE target ADD PRIMARY KEY (tid); ANALYZE target; DELETE FROM target WHERE tid > 100; ANALYZE target; -- insert some matching source rows to work from INSERT INTO source VALUES (2, 5); INSERT INTO source VALUES (3, 20); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- equivalent of an UPDATE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; SELECT * FROM target ORDER BY tid; ROLLBACK; -- equivalent of a DELETE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DO NOTHING; SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, NULL); SELECT * FROM target ORDER BY tid; ROLLBACK; -- duplicate source row causes multiple target row update ERROR INSERT INTO source VALUES (2, 5); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:241: ERROR: MERGE command cannot affect row a second time HINT: Ensure that not more than one source row matches any one target row. ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; psql:include/ts_merge_query.sql:249: ERROR: MERGE command cannot affect row a second time HINT: Ensure that not more than one source row matches any one target row. ROLLBACK; -- remove duplicate MATCHED data from source data DELETE FROM source WHERE sid = 2; INSERT INTO source VALUES (2, 5); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- duplicate source row on INSERT should fail because of target_pkey INSERT INTO source VALUES (4, 40); BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, NULL); psql:include/ts_merge_query.sql:265: ERROR: duplicate key value violates unique constraint "target_pkey" DETAIL: Key (tid)=(4) already exists. SELECT * FROM target ORDER BY tid; psql:include/ts_merge_query.sql:266: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; -- remove duplicate NOT MATCHED data from source data DELETE FROM source WHERE sid = 4; INSERT INTO source VALUES (4, 40); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- remove constraints alter table target drop CONSTRAINT target_pkey; alter table target alter column tid drop not null; -- multiple actions BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, 4) WHEN MATCHED THEN UPDATE SET balance = 0; SELECT * FROM target ORDER BY tid; ROLLBACK; -- should be equivalent BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0 WHEN NOT MATCHED THEN INSERT VALUES (4, 4); SELECT * FROM target ORDER BY tid; ROLLBACK; -- column references -- do a simple equivalent of an UPDATE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance + s.delta; SELECT * FROM target ORDER BY tid; ROLLBACK; -- do a simple equivalent of an INSERT SELECT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- and again with duplicate source rows INSERT INTO source VALUES (5, 50); INSERT INTO source VALUES (5, 50); -- do a simple equivalent of an INSERT SELECT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- removing duplicate source rows DELETE FROM source WHERE sid = 5; -- and again with explicitly identified column list BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- and again with a subtle error: referring to non-existent target row for NOT MATCHED MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (t.tid, s.delta); psql:include/ts_merge_query.sql:356: ERROR: invalid reference to FROM-clause entry for table "t" LINE 5: INSERT (tid, balance) VALUES (t.tid, s.delta); ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. -- and again with a constant ON clause BEGIN; MERGE INTO target t USING source AS s ON (SELECT true) WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (t.tid, s.delta); psql:include/ts_merge_query.sql:364: ERROR: invalid reference to FROM-clause entry for table "t" LINE 5: INSERT (tid, balance) VALUES (t.tid, s.delta); ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. SELECT * FROM target ORDER BY tid; psql:include/ts_merge_query.sql:365: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; -- now the classic UPSERT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance + s.delta WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- this time with a FALSE condition MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND FALSE THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; -- this time with an actual condition which returns false MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance <> 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; BEGIN; -- and now with a condition which returns true MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance = 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; ROLLBACK; -- conditions in the NOT MATCHED clause can only refer to source columns BEGIN; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND t.balance = 100 THEN INSERT (tid) VALUES (s.sid); psql:include/ts_merge_query.sql:408: ERROR: invalid reference to FROM-clause entry for table "t" LINE 3: WHEN NOT MATCHED AND t.balance = 100 THEN ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. SELECT * FROM wq_target; psql:include/ts_merge_query.sql:409: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance = 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; -- conditions in MATCHED clause can refer to both source and target SELECT * FROM wq_source; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND s.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check if AND works MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 AND s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 AND s.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check if OR works MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 OR s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 199 OR s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check source-side whole-row references BEGIN; MERGE INTO wq_target t USING wq_source s ON (t.tid = s.sid) WHEN matched and t = s or t.tid = s.sid THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; ROLLBACK; -- check if subqueries work in the conditions? MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance > (SELECT max(balance) FROM target) THEN UPDATE SET balance = t.balance + s.balance; -- check if we can access system columns in the conditions MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.xmin = t.xmax THEN UPDATE SET balance = t.balance + s.balance; psql:include/ts_merge_query.sql:477: ERROR: cannot use system column "xmin" in MERGE WHEN condition LINE 3: WHEN MATCHED AND t.xmin = t.xmax THEN ^ MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.tableoid >= 0 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; DROP TABLE wq_target CASCADE; DROP TABLE wq_source; -- test triggers create or replace function merge_trigfunc () returns trigger language plpgsql as $$ DECLARE line text; BEGIN SELECT INTO line format('%s %s %s trigger%s', TG_WHEN, TG_OP, TG_LEVEL, CASE WHEN TG_OP = 'INSERT' AND TG_LEVEL = 'ROW' THEN format(' row: %s', NEW) WHEN TG_OP = 'UPDATE' AND TG_LEVEL = 'ROW' THEN format(' row: %s -> %s', OLD, NEW) WHEN TG_OP = 'DELETE' AND TG_LEVEL = 'ROW' THEN format(' row: %s', OLD) END); RAISE NOTICE '%', line; IF (TG_WHEN = 'BEFORE' AND TG_LEVEL = 'ROW') THEN IF (TG_OP = 'DELETE') THEN RETURN OLD; ELSE RETURN NEW; END IF; ELSE RETURN NULL; END IF; END; $$; CREATE TRIGGER merge_bsi BEFORE INSERT ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bsu BEFORE UPDATE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bsd BEFORE DELETE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asi AFTER INSERT ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asu AFTER UPDATE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asd AFTER DELETE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bri BEFORE INSERT ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bru BEFORE UPDATE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_brd BEFORE DELETE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_ari AFTER INSERT ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_aru AFTER UPDATE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_ard AFTER DELETE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); -- now the classic UPSERT, with a DELETE BEGIN; UPDATE target SET balance = 0 WHERE tid = 3; --EXPLAIN (ANALYZE ON, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND t.balance > s.delta THEN UPDATE SET balance = t.balance - s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- Test behavior of triggers that turn UPDATE/DELETE into no-ops create or replace function skip_merge_op() returns trigger language plpgsql as $$ BEGIN RETURN NULL; END; $$; SELECT * FROM target full outer join source on (sid = tid); create trigger merge_skip BEFORE INSERT OR UPDATE or DELETE ON target FOR EACH ROW EXECUTE FUNCTION skip_merge_op(); DO $$ DECLARE result integer; BEGIN MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND s.sid = 3 THEN UPDATE SET balance = t.balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT VALUES (sid, delta); IF FOUND THEN RAISE NOTICE 'Found'; ELSE RAISE NOTICE 'Not found'; END IF; GET DIAGNOSTICS result := ROW_COUNT; RAISE NOTICE 'ROW_COUNT = %', result; END; $$; SELECT * FROM target FULL OUTER JOIN source ON (sid = tid); DROP TRIGGER merge_skip ON target; DROP FUNCTION skip_merge_op(); -- test from PL/pgSQL -- make sure MERGE INTO isn't interpreted to mean returning variables like SELECT INTO BEGIN; DO LANGUAGE plpgsql $$ BEGIN MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND t.balance > s.delta THEN UPDATE SET balance = t.balance - s.delta; END; $$; ROLLBACK; --source constants BEGIN; MERGE INTO target t USING (SELECT 9 AS sid, 57 AS delta) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; --source query BEGIN; MERGE INTO target t USING (SELECT sid, delta FROM source WHERE delta > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT sid, delta as newname FROM source WHERE delta > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.newname); SELECT * FROM target ORDER BY tid; ROLLBACK; --self-merge BEGIN; MERGE INTO target t1 USING target t2 ON t1.tid = t2.tid WHEN MATCHED THEN UPDATE SET balance = t1.balance + t2.balance WHEN NOT MATCHED THEN INSERT VALUES (t2.tid, t2.balance); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT tid as sid, balance as delta FROM target WHERE balance > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT sid, max(delta) AS delta FROM source GROUP BY sid HAVING count(*) = 1 ORDER BY sid ASC) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- plpgsql parameters and results BEGIN; CREATE FUNCTION merge_func (p_id integer, p_bal integer) RETURNS INTEGER LANGUAGE plpgsql AS $$ DECLARE result integer; BEGIN MERGE INTO target t USING (SELECT p_id AS sid) AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance - p_bal; IF FOUND THEN GET DIAGNOSTICS result := ROW_COUNT; END IF; RETURN result; END; $$; SELECT merge_func(3, 4); SELECT * FROM target ORDER BY tid; ROLLBACK; -- PREPARE BEGIN; prepare foom as merge into target t using (select 1 as sid) s on (t.tid = s.sid) when matched then update set balance = 1; execute foom; ROLLBACK; BEGIN; PREPARE foom2 (integer, integer) AS MERGE INTO target t USING (SELECT 1) s ON t.tid = $1 WHEN MATCHED THEN UPDATE SET balance = $2; --EXPLAIN (ANALYZE ON, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) execute foom2 (1, 1); ROLLBACK; -- subqueries in source relation BEGIN; MERGE INTO sq_target t USING (SELECT * FROM sq_source) s ON tid = sid WHEN MATCHED AND t.balance > delta THEN UPDATE SET balance = t.balance + delta; SELECT * FROM sq_target ORDER BY tid; ROLLBACK; -- try a view CREATE VIEW v AS SELECT * FROM sq_source WHERE sid < 2; BEGIN; MERGE INTO sq_target USING v ON tid = sid WHEN MATCHED THEN UPDATE SET balance = v.balance + delta; SELECT * FROM sq_target ORDER BY tid; ROLLBACK; -- ambiguous reference to a column BEGIN; MERGE INTO sq_target USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; psql:include/ts_merge_query.sql:732: ERROR: column reference "balance" is ambiguous LINE 5: UPDATE SET balance = balance + delta ^ ROLLBACK; BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; SELECT * FROM sq_target; ROLLBACK; -- CTEs BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); WITH targq AS ( SELECT * FROM v ) MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; ROLLBACK; -- RETURNING BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE RETURNING *; ROLLBACK; -- PG17-specific tests for views, returning and merge_action. These throw syntax errors for previous versions of Postgres. -- However, since the error is the same for both hypertables and regular tables, this test should still pass for previous versions. -- RETURNING INSERT INTO source(sid, delta) VALUES(1, 40), (5, 50); BEGIN; MERGE INTO target t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta); SELECT * from target; ROLLBACK; BEGIN; MERGE INTO target t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 1 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta) RETURNING merge_action(), t.*; ROLLBACK; -- Views CREATE VIEW tv AS SELECT * FROM target; BEGIN; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta); SELECT * from tv; SELECT * from target; -- should also update the underlying table ROLLBACK; BEGIN; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta) RETURNING merge_action(), t.*; ROLLBACK; DROP VIEW tv; DELETE FROM source where sid in (1, 5); -- EXPLAIN CREATE TABLE ex_mtarget (a int, b int) WITH (autovacuum_enabled=off); CREATE TABLE ex_msource (a int, b int) WITH (autovacuum_enabled=off); INSERT INTO ex_mtarget SELECT i, i*10 FROM generate_series(1,100,2) i; INSERT INTO ex_msource SELECT i, i*10 FROM generate_series(1,100,1) i; CREATE FUNCTION explain_merge(query text) RETURNS SETOF text LANGUAGE plpgsql AS $$ DECLARE ln text; BEGIN FOR ln IN EXECUTE 'explain (analyze, timing off, summary off, buffers off, costs off) ' || query LOOP ln := regexp_replace(ln, '(Memory( Usage)?|Buckets|Batches): \S*', '\1: xxx', 'g'); RETURN NEXT ln; END LOOP; END; $$; -- only updates SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED THEN UPDATE SET b = t.b + 1'); -- only updates to selected tuples SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1'); -- updates + deletes SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1 WHEN MATCHED AND t.a >= 10 AND t.a <= 20 THEN DELETE'); -- only inserts SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN NOT MATCHED AND s.a < 10 THEN INSERT VALUES (a, b)'); -- all three SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1 WHEN MATCHED AND t.a >= 30 AND t.a <= 40 THEN DELETE WHEN NOT MATCHED AND s.a < 20 THEN INSERT VALUES (a, b)'); -- nothing SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a AND t.a < -1000 WHEN MATCHED AND t.a < 10 THEN DO NOTHING'); DROP TABLE ex_msource, ex_mtarget; DROP FUNCTION explain_merge(text); -- Subqueries BEGIN; MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED THEN UPDATE SET balance = (SELECT count(*) FROM sq_target); SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; BEGIN; MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND (SELECT count(*) > 0 FROM sq_target) THEN UPDATE SET balance = 42; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; BEGIN; MERGE INTO sq_target t USING v ON tid = sid AND (SELECT count(*) > 0 FROM sq_target) WHEN MATCHED THEN UPDATE SET balance = 42; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; -- Test RETURNING with subqueries BEGIN; MERGE INTO sq_target t USING v ON tid = sid AND (SELECT count(*) > 0 FROM sq_target) WHEN MATCHED THEN UPDATE SET balance = 42 RETURNING *; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; DROP TABLE sq_target CASCADE; DROP TABLE sq_source CASCADE; CREATE TABLE pa_target (tid integer, balance float, val text) PARTITION BY LIST (tid); CREATE TABLE part1 PARTITION OF pa_target FOR VALUES IN (1,4) WITH (autovacuum_enabled=off); CREATE TABLE part2 PARTITION OF pa_target FOR VALUES IN (2,5,6) WITH (autovacuum_enabled=off); CREATE TABLE part3 PARTITION OF pa_target FOR VALUES IN (3,8,9) WITH (autovacuum_enabled=off); CREATE TABLE part4 PARTITION OF pa_target DEFAULT WITH (autovacuum_enabled=off); CREATE TABLE pa_source (sid integer, delta float); -- insert many rows to the source table INSERT INTO pa_source SELECT id, id * 10 FROM generate_series(1,14) AS id; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT id, id * 100, 'initial' FROM generate_series(1,14,2) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- same with a constant qual BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid AND tid = 1 WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- try updating the partition key column BEGIN; CREATE FUNCTION merge_func() RETURNS integer LANGUAGE plpgsql AS $$ DECLARE result integer; BEGIN MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET tid = tid + 1, balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); IF FOUND THEN GET DIAGNOSTICS result := ROW_COUNT; END IF; RETURN result; END; $$; SELECT merge_func(); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_target CASCADE; -- The target table is partitioned in the same way, but this time by attaching -- partitions which have columns in different order, dropped columns etc. CREATE TABLE pa_target (tid integer, balance float, val text) PARTITION BY LIST (tid); CREATE TABLE part1 (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE part2 (balance float, tid integer, val text) WITH (autovacuum_enabled=off); CREATE TABLE part3 (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE part4 (extraid text, tid integer, balance float, val text) WITH (autovacuum_enabled=off); ALTER TABLE part4 DROP COLUMN extraid; ALTER TABLE pa_target ATTACH PARTITION part1 FOR VALUES IN (1,4); ALTER TABLE pa_target ATTACH PARTITION part2 FOR VALUES IN (2,5,6); ALTER TABLE pa_target ATTACH PARTITION part3 FOR VALUES IN (3,8,9); ALTER TABLE pa_target ATTACH PARTITION part4 DEFAULT; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT id, id * 100, 'initial' FROM generate_series(1,14,2) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- same with a constant qual BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid AND tid IN (1, 5) WHEN MATCHED AND tid % 5 = 0 THEN DELETE WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- try updating the partition key column BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET tid = tid + 1, balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_source; DROP TABLE pa_target CASCADE; -- Sub-partitioning CREATE TABLE pa_target (logts timestamp, tid integer, balance float, val text) PARTITION BY RANGE (logts); CREATE TABLE part_m01 PARTITION OF pa_target FOR VALUES FROM ('2017-01-01') TO ('2017-02-01') PARTITION BY LIST (tid); CREATE TABLE part_m01_odd PARTITION OF part_m01 FOR VALUES IN (1,3,5,7,9) WITH (autovacuum_enabled=off); CREATE TABLE part_m01_even PARTITION OF part_m01 FOR VALUES IN (2,4,6,8) WITH (autovacuum_enabled=off); CREATE TABLE part_m02 PARTITION OF pa_target FOR VALUES FROM ('2017-02-01') TO ('2017-03-01') PARTITION BY LIST (tid); CREATE TABLE part_m02_odd PARTITION OF part_m02 FOR VALUES IN (1,3,5,7,9) WITH (autovacuum_enabled=off); CREATE TABLE part_m02_even PARTITION OF part_m02 FOR VALUES IN (2,4,6,8) WITH (autovacuum_enabled=off); CREATE TABLE pa_source (sid integer, delta float) WITH (autovacuum_enabled=off); -- insert many rows to the source table INSERT INTO pa_source SELECT id, id * 10 FROM generate_series(1,14) AS id; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT '2017-01-31', id, id * 100, 'initial' FROM generate_series(1,9,3) AS id; INSERT INTO pa_target SELECT '2017-02-28', id, id * 100, 'initial' FROM generate_series(2,9,3) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING (SELECT '2017-01-15' AS slogts, * FROM pa_source WHERE sid < 10) s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (slogts::timestamp, sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_source; DROP TABLE pa_target CASCADE; -- some complex joins on the source side -- source relation is an unaliased join MERGE INTO cj_target t USING cj_source1 s1 INNER JOIN cj_source2 s2 ON sid1 = sid2 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid1, delta, sval); -- try accessing columns from either side of the source join MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 AND scat = 20 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid2, delta, sval) WHEN MATCHED THEN DELETE; -- some simple expressions in INSERT targetlist MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid2, delta + scat, sval) WHEN MATCHED THEN UPDATE SET val = val || ' updated by merge'; MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 AND scat = 20 ON t.tid = sid1 WHEN MATCHED THEN UPDATE SET val = val || ' ' || delta::text; SELECT * FROM cj_target ORDER BY tid; ALTER TABLE cj_source1 RENAME COLUMN sid1 TO sid; ALTER TABLE cj_source2 RENAME COLUMN sid2 TO sid; TRUNCATE cj_target; MERGE INTO cj_target t USING cj_source1 s1 INNER JOIN cj_source2 s2 ON s1.sid = s2.sid ON t.tid = s1.sid WHEN NOT MATCHED THEN INSERT VALUES (s2.sid, delta, sval); DROP TABLE cj_source2, cj_source1; DROP TABLE cj_target CASCADE; -- Function scans MERGE INTO fs_target t USING generate_series(1,100,1) AS id ON t.a = id WHEN MATCHED THEN UPDATE SET b = b + id WHEN NOT MATCHED THEN INSERT VALUES (id, -1); MERGE INTO fs_target t USING generate_series(1,100,2) AS id ON t.a = id WHEN MATCHED THEN UPDATE SET b = b + id, c = 'updated '|| id.*::text WHEN NOT MATCHED THEN INSERT VALUES (id, -1, 'inserted ' || id.*::text); SELECT count(*) FROM fs_target; DROP TABLE fs_target CASCADE; -- SERIALIZABLE test -- handled in isolation tests -- Inheritance-based partitioning CREATE TABLE measurement ( city_id int not null, logdate date not null, peaktemp int, unitsales int ) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2006m02 ( CHECK ( logdate >= DATE '2006-02-01' AND logdate < DATE '2006-03-01' ) ) INHERITS (measurement) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2006m03 ( CHECK ( logdate >= DATE '2006-03-01' AND logdate < DATE '2006-04-01' ) ) INHERITS (measurement) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2007m01 ( filler text, peaktemp int, logdate date not null, city_id int not null, unitsales int CHECK ( logdate >= DATE '2007-01-01' AND logdate < DATE '2007-02-01') ) WITH (autovacuum_enabled=off); ALTER TABLE measurement_y2007m01 DROP COLUMN filler; ALTER TABLE measurement_y2007m01 INHERIT measurement; INSERT INTO measurement VALUES (0, '2005-07-21', 5, 15); CREATE OR REPLACE FUNCTION measurement_insert_trigger() RETURNS TRIGGER AS $$ BEGIN IF ( NEW.logdate >= DATE '2006-02-01' AND NEW.logdate < DATE '2006-03-01' ) THEN INSERT INTO measurement_y2006m02 VALUES (NEW.*); ELSIF ( NEW.logdate >= DATE '2006-03-01' AND NEW.logdate < DATE '2006-04-01' ) THEN INSERT INTO measurement_y2006m03 VALUES (NEW.*); ELSIF ( NEW.logdate >= DATE '2007-01-01' AND NEW.logdate < DATE '2007-02-01' ) THEN INSERT INTO measurement_y2007m01 (city_id, logdate, peaktemp, unitsales) VALUES (NEW.*); ELSE RAISE EXCEPTION 'Date out of range. Fix the measurement_insert_trigger() function!'; END IF; RETURN NULL; END; $$ LANGUAGE plpgsql ; CREATE TRIGGER insert_measurement_trigger BEFORE INSERT ON measurement FOR EACH ROW EXECUTE PROCEDURE measurement_insert_trigger(); INSERT INTO measurement VALUES (1, '2006-02-10', 35, 10); INSERT INTO measurement VALUES (1, '2006-02-16', 45, 20); INSERT INTO measurement VALUES (1, '2006-03-17', 25, 10); INSERT INTO measurement VALUES (1, '2006-03-27', 15, 40); INSERT INTO measurement VALUES (1, '2007-01-15', 10, 10); INSERT INTO measurement VALUES (1, '2007-01-17', 10, 10); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate; CREATE TABLE new_measurement (LIKE measurement) WITH (autovacuum_enabled=off); INSERT INTO new_measurement VALUES (0, '2005-07-21', 25, 20); INSERT INTO new_measurement VALUES (1, '2006-03-01', 20, 10); INSERT INTO new_measurement VALUES (1, '2006-02-16', 50, 10); INSERT INTO new_measurement VALUES (2, '2006-02-10', 20, 20); INSERT INTO new_measurement VALUES (1, '2006-03-27', NULL, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-17', NULL, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-15', 5, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-16', 10, 10); BEGIN; MERGE INTO ONLY measurement m USING new_measurement nm ON (m.city_id = nm.city_id and m.logdate=nm.logdate) WHEN MATCHED AND nm.peaktemp IS NULL THEN DELETE WHEN MATCHED THEN UPDATE SET peaktemp = greatest(m.peaktemp, nm.peaktemp), unitsales = m.unitsales + coalesce(nm.unitsales, 0) WHEN NOT MATCHED THEN INSERT (city_id, logdate, peaktemp, unitsales) VALUES (city_id, logdate, peaktemp, unitsales); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate, peaktemp; ROLLBACK; MERGE into measurement m USING new_measurement nm ON (m.city_id = nm.city_id and m.logdate=nm.logdate) WHEN MATCHED AND nm.peaktemp IS NULL THEN DELETE WHEN MATCHED THEN UPDATE SET peaktemp = greatest(m.peaktemp, nm.peaktemp), unitsales = m.unitsales + coalesce(nm.unitsales, 0) WHEN NOT MATCHED THEN INSERT (city_id, logdate, peaktemp, unitsales) VALUES (city_id, logdate, peaktemp, unitsales); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate; BEGIN; MERGE INTO new_measurement nm USING ONLY measurement m ON (nm.city_id = m.city_id and nm.logdate=m.logdate) WHEN MATCHED THEN DELETE; SELECT * FROM new_measurement ORDER BY city_id, logdate; ROLLBACK; MERGE INTO new_measurement nm USING measurement m ON (nm.city_id = m.city_id and nm.logdate=m.logdate) WHEN MATCHED THEN DELETE; SELECT * FROM new_measurement ORDER BY city_id, logdate; DROP TABLE measurement, new_measurement CASCADE; DROP FUNCTION measurement_insert_trigger(); RESET SESSION AUTHORIZATION; DROP TABLE target CASCADE; DROP TABLE target2 CASCADE; DROP TABLE source, source2; DROP FUNCTION merge_trigfunc(); REVOKE CREATE ON SCHEMA public FROM regress_merge_privs; DROP USER regress_merge_privs; DROP USER regress_merge_no_privs; \o \ir :TEST_LOAD_HT_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE USER regress_merge_privs; CREATE USER regress_merge_no_privs; DROP TABLE IF EXISTS target; DROP TABLE IF EXISTS source; CREATE TABLE target (tid integer, balance integer) WITH (autovacuum_enabled=off); SELECT create_hypertable('target', 'tid', chunk_time_interval => 3); create_hypertable --------------------- (1,public,target,t) CREATE TABLE source (sid integer, delta integer) -- no index WITH (autovacuum_enabled=off); INSERT INTO target VALUES (1, 10); INSERT INTO target VALUES (2, 20); INSERT INTO target VALUES (3, 30); SELECT t.ctid is not null as matched, t.*, s.* FROM source s FULL OUTER JOIN target t ON s.sid = t.tid ORDER BY t.tid, s.sid; matched | tid | balance | sid | delta ---------+-----+---------+-----+------- t | 1 | 10 | | t | 2 | 20 | | t | 3 | 30 | | ALTER TABLE target OWNER TO regress_merge_privs; ALTER TABLE source OWNER TO regress_merge_privs; CREATE TABLE target2 (tid integer, balance integer) WITH (autovacuum_enabled=off); SELECT create_hypertable('target2', 'tid', chunk_time_interval => 3); create_hypertable ---------------------- (2,public,target2,t) CREATE TABLE source2 (sid integer, delta integer) WITH (autovacuum_enabled=off); ALTER TABLE target2 OWNER TO regress_merge_no_privs; ALTER TABLE source2 OWNER TO regress_merge_no_privs; GRANT INSERT ON target TO regress_merge_no_privs; GRANT CREATE ON SCHEMA public TO regress_merge_privs; SET SESSION AUTHORIZATION regress_merge_privs; CREATE TABLE sq_target (tid integer NOT NULL, balance integer) WITH (autovacuum_enabled=off); SELECT create_hypertable('sq_target', 'tid', chunk_time_interval => 3); create_hypertable ------------------------ (3,public,sq_target,t) CREATE TABLE sq_source (delta integer, sid integer, balance integer DEFAULT 0) WITH (autovacuum_enabled=off); INSERT INTO sq_target(tid, balance) VALUES (1,100), (2,200), (3,300); INSERT INTO sq_source(sid, delta) VALUES (1,10), (2,20), (4,40); -- conditional WHEN clause CREATE TABLE wq_target (tid integer not null, balance integer DEFAULT -1) WITH (autovacuum_enabled=off); SELECT create_hypertable('wq_target', 'tid', chunk_time_interval => 3); create_hypertable ------------------------ (4,public,wq_target,t) CREATE TABLE wq_source (balance integer, sid integer) WITH (autovacuum_enabled=off); INSERT INTO wq_source (sid, balance) VALUES (1, 100); -- some complex joins on the source side CREATE TABLE cj_target (tid integer, balance float, val text) WITH (autovacuum_enabled=off); SELECT create_hypertable('cj_target', 'tid', chunk_time_interval => 3); create_hypertable ------------------------ (5,public,cj_target,t) CREATE TABLE cj_source1 (sid1 integer, scat integer, delta integer) WITH (autovacuum_enabled=off); CREATE TABLE cj_source2 (sid2 integer, sval text) WITH (autovacuum_enabled=off); INSERT INTO cj_source1 VALUES (1, 10, 100); INSERT INTO cj_source1 VALUES (1, 20, 200); INSERT INTO cj_source1 VALUES (2, 20, 300); INSERT INTO cj_source1 VALUES (3, 10, 400); INSERT INTO cj_source2 VALUES (1, 'initial source2'); INSERT INTO cj_source2 VALUES (2, 'initial source2'); INSERT INTO cj_source2 VALUES (3, 'initial source2'); CREATE TABLE fs_target (a int, b int, c text) WITH (autovacuum_enabled=off); SELECT create_hypertable('fs_target', 'a', chunk_time_interval => 3); create_hypertable ------------------------ (6,public,fs_target,t) -- run tests on hypertable \o :TEST_RESULTS_WITH_HYPERTABLE \ir :TEST_QUERY_NAME -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- -- Errors -- MERGE INTO target t RANDOMWORD USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:12: ERROR: syntax error at or near "RANDOMWORD" LINE 1: MERGE INTO target t RANDOMWORD ^ -- MATCHED/INSERT error MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:18: ERROR: syntax error at or near "INSERT" LINE 5: INSERT DEFAULT VALUES; ^ -- incorrectly specifying INTO target MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT INTO target DEFAULT VALUES; psql:include/ts_merge_query.sql:24: ERROR: syntax error at or near "INTO" LINE 5: INSERT INTO target DEFAULT VALUES; ^ -- Multiple VALUES clause MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (1,1), (2,2); psql:include/ts_merge_query.sql:30: ERROR: syntax error at or near "," LINE 5: INSERT VALUES (1,1), (2,2); ^ -- SELECT query for INSERT MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT SELECT (1, 1); psql:include/ts_merge_query.sql:36: ERROR: syntax error at or near "SELECT" LINE 5: INSERT SELECT (1, 1); ^ -- NOT MATCHED/UPDATE MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:42: ERROR: syntax error at or near "UPDATE" LINE 5: UPDATE SET balance = 0; ^ -- UPDATE tablename MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE target SET balance = 0; psql:include/ts_merge_query.sql:48: ERROR: syntax error at or near "target" LINE 5: UPDATE target SET balance = 0; ^ -- source and target names the same MERGE INTO target USING target ON tid = tid WHEN MATCHED THEN DO NOTHING; psql:include/ts_merge_query.sql:53: ERROR: name "target" specified more than once DETAIL: The name is used both as MERGE target table and data source. -- used in a CTE WITH foo AS ( MERGE INTO target USING source ON (true) WHEN MATCHED THEN DELETE ) SELECT * FROM foo; psql:include/ts_merge_query.sql:58: ERROR: WITH query "foo" does not have a RETURNING clause LINE 4: ) SELECT * FROM foo; ^ -- used in COPY COPY ( MERGE INTO target USING source ON (true) WHEN MATCHED THEN DELETE ) TO stdout; psql:include/ts_merge_query.sql:63: ERROR: COPY query must have a RETURNING clause -- unsupported relation types -- view CREATE VIEW tv AS SELECT * FROM target; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; DROP VIEW tv; -- materialized view CREATE MATERIALIZED VIEW mv AS SELECT * FROM target; MERGE INTO mv t USING source s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:81: ERROR: cannot execute MERGE on relation "mv" DETAIL: This operation is not supported for materialized views. DROP MATERIALIZED VIEW mv; -- permissions MERGE INTO target USING source2 ON target.tid = source2.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:90: ERROR: permission denied for table source2 GRANT INSERT ON target TO regress_merge_no_privs; SET SESSION AUTHORIZATION regress_merge_no_privs; MERGE INTO target USING source2 ON target.tid = source2.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:99: ERROR: permission denied for table target GRANT UPDATE ON target2 TO regress_merge_privs; SET SESSION AUTHORIZATION regress_merge_privs; MERGE INTO target2 USING source ON target2.tid = source.sid WHEN MATCHED THEN DELETE; psql:include/ts_merge_query.sql:108: ERROR: permission denied for table target2 MERGE INTO target2 USING source ON target2.tid = source.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:114: ERROR: permission denied for table target2 -- check if the target can be accessed from source relation subquery; we should -- not be able to do so MERGE INTO target t USING (SELECT * FROM source WHERE t.tid > sid) s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; psql:include/ts_merge_query.sql:122: ERROR: invalid reference to FROM-clause entry for table "t" LINE 2: USING (SELECT * FROM source WHERE t.tid > sid) s ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. -- -- initial tests -- -- zero rows in source has no effect MERGE INTO target USING source ON target.tid = source.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; ROLLBACK; -- insert some non-matching source rows to work from INSERT INTO source VALUES (4, 40); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN DO NOTHING; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (5, 50); SELECT * FROM target ORDER BY tid; ROLLBACK; -- index plans INSERT INTO target SELECT generate_series(1000,2500), 0; ALTER TABLE target ADD PRIMARY KEY (tid); ANALYZE target; DELETE FROM target WHERE tid > 100; ANALYZE target; -- insert some matching source rows to work from INSERT INTO source VALUES (2, 5); INSERT INTO source VALUES (3, 20); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- equivalent of an UPDATE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; SELECT * FROM target ORDER BY tid; ROLLBACK; -- equivalent of a DELETE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DO NOTHING; SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, NULL); SELECT * FROM target ORDER BY tid; ROLLBACK; -- duplicate source row causes multiple target row update ERROR INSERT INTO source VALUES (2, 5); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; psql:include/ts_merge_query.sql:241: ERROR: MERGE command cannot affect row a second time HINT: Ensure that not more than one source row matches any one target row. ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; psql:include/ts_merge_query.sql:249: ERROR: MERGE command cannot affect row a second time HINT: Ensure that not more than one source row matches any one target row. ROLLBACK; -- remove duplicate MATCHED data from source data DELETE FROM source WHERE sid = 2; INSERT INTO source VALUES (2, 5); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- duplicate source row on INSERT should fail because of target_pkey INSERT INTO source VALUES (4, 40); BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, NULL); psql:include/ts_merge_query.sql:265: ERROR: duplicate key value violates unique constraint "2_2_target_pkey" DETAIL: Key (tid)=(4) already exists. SELECT * FROM target ORDER BY tid; psql:include/ts_merge_query.sql:266: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; -- remove duplicate NOT MATCHED data from source data DELETE FROM source WHERE sid = 4; INSERT INTO source VALUES (4, 40); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- remove constraints alter table target drop CONSTRAINT target_pkey; alter table target alter column tid drop not null; psql:include/ts_merge_query.sql:277: ERROR: cannot drop not-null constraint from a time-partitioned column -- multiple actions BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, 4) WHEN MATCHED THEN UPDATE SET balance = 0; SELECT * FROM target ORDER BY tid; ROLLBACK; -- should be equivalent BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0 WHEN NOT MATCHED THEN INSERT VALUES (4, 4); SELECT * FROM target ORDER BY tid; ROLLBACK; -- column references -- do a simple equivalent of an UPDATE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance + s.delta; SELECT * FROM target ORDER BY tid; ROLLBACK; -- do a simple equivalent of an INSERT SELECT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- and again with duplicate source rows INSERT INTO source VALUES (5, 50); INSERT INTO source VALUES (5, 50); -- do a simple equivalent of an INSERT SELECT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- removing duplicate source rows DELETE FROM source WHERE sid = 5; -- and again with explicitly identified column list BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- and again with a subtle error: referring to non-existent target row for NOT MATCHED MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (t.tid, s.delta); psql:include/ts_merge_query.sql:356: ERROR: invalid reference to FROM-clause entry for table "t" LINE 5: INSERT (tid, balance) VALUES (t.tid, s.delta); ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. -- and again with a constant ON clause BEGIN; MERGE INTO target t USING source AS s ON (SELECT true) WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (t.tid, s.delta); psql:include/ts_merge_query.sql:364: ERROR: invalid reference to FROM-clause entry for table "t" LINE 5: INSERT (tid, balance) VALUES (t.tid, s.delta); ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. SELECT * FROM target ORDER BY tid; psql:include/ts_merge_query.sql:365: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; -- now the classic UPSERT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance + s.delta WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- this time with a FALSE condition MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND FALSE THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; -- this time with an actual condition which returns false MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance <> 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; BEGIN; -- and now with a condition which returns true MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance = 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; ROLLBACK; -- conditions in the NOT MATCHED clause can only refer to source columns BEGIN; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND t.balance = 100 THEN INSERT (tid) VALUES (s.sid); psql:include/ts_merge_query.sql:408: ERROR: invalid reference to FROM-clause entry for table "t" LINE 3: WHEN NOT MATCHED AND t.balance = 100 THEN ^ DETAIL: There is an entry for table "t", but it cannot be referenced from this part of the query. SELECT * FROM wq_target; psql:include/ts_merge_query.sql:409: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance = 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; -- conditions in MATCHED clause can refer to both source and target SELECT * FROM wq_source; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND s.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check if AND works MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 AND s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 AND s.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check if OR works MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 OR s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 199 OR s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check source-side whole-row references BEGIN; MERGE INTO wq_target t USING wq_source s ON (t.tid = s.sid) WHEN matched and t = s or t.tid = s.sid THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; ROLLBACK; -- check if subqueries work in the conditions? MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance > (SELECT max(balance) FROM target) THEN UPDATE SET balance = t.balance + s.balance; -- check if we can access system columns in the conditions MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.xmin = t.xmax THEN UPDATE SET balance = t.balance + s.balance; psql:include/ts_merge_query.sql:477: ERROR: cannot use system column "xmin" in MERGE WHEN condition LINE 3: WHEN MATCHED AND t.xmin = t.xmax THEN ^ MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.tableoid >= 0 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; DROP TABLE wq_target CASCADE; DROP TABLE wq_source; -- test triggers create or replace function merge_trigfunc () returns trigger language plpgsql as $$ DECLARE line text; BEGIN SELECT INTO line format('%s %s %s trigger%s', TG_WHEN, TG_OP, TG_LEVEL, CASE WHEN TG_OP = 'INSERT' AND TG_LEVEL = 'ROW' THEN format(' row: %s', NEW) WHEN TG_OP = 'UPDATE' AND TG_LEVEL = 'ROW' THEN format(' row: %s -> %s', OLD, NEW) WHEN TG_OP = 'DELETE' AND TG_LEVEL = 'ROW' THEN format(' row: %s', OLD) END); RAISE NOTICE '%', line; IF (TG_WHEN = 'BEFORE' AND TG_LEVEL = 'ROW') THEN IF (TG_OP = 'DELETE') THEN RETURN OLD; ELSE RETURN NEW; END IF; ELSE RETURN NULL; END IF; END; $$; CREATE TRIGGER merge_bsi BEFORE INSERT ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bsu BEFORE UPDATE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bsd BEFORE DELETE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asi AFTER INSERT ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asu AFTER UPDATE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asd AFTER DELETE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bri BEFORE INSERT ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bru BEFORE UPDATE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_brd BEFORE DELETE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_ari AFTER INSERT ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_aru AFTER UPDATE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_ard AFTER DELETE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); -- now the classic UPSERT, with a DELETE BEGIN; UPDATE target SET balance = 0 WHERE tid = 3; --EXPLAIN (ANALYZE ON, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND t.balance > s.delta THEN UPDATE SET balance = t.balance - s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- Test behavior of triggers that turn UPDATE/DELETE into no-ops create or replace function skip_merge_op() returns trigger language plpgsql as $$ BEGIN RETURN NULL; END; $$; SELECT * FROM target full outer join source on (sid = tid); create trigger merge_skip BEFORE INSERT OR UPDATE or DELETE ON target FOR EACH ROW EXECUTE FUNCTION skip_merge_op(); DO $$ DECLARE result integer; BEGIN MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND s.sid = 3 THEN UPDATE SET balance = t.balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT VALUES (sid, delta); IF FOUND THEN RAISE NOTICE 'Found'; ELSE RAISE NOTICE 'Not found'; END IF; GET DIAGNOSTICS result := ROW_COUNT; RAISE NOTICE 'ROW_COUNT = %', result; END; $$; SELECT * FROM target FULL OUTER JOIN source ON (sid = tid); DROP TRIGGER merge_skip ON target; DROP FUNCTION skip_merge_op(); -- test from PL/pgSQL -- make sure MERGE INTO isn't interpreted to mean returning variables like SELECT INTO BEGIN; DO LANGUAGE plpgsql $$ BEGIN MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND t.balance > s.delta THEN UPDATE SET balance = t.balance - s.delta; END; $$; ROLLBACK; --source constants BEGIN; MERGE INTO target t USING (SELECT 9 AS sid, 57 AS delta) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; --source query BEGIN; MERGE INTO target t USING (SELECT sid, delta FROM source WHERE delta > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT sid, delta as newname FROM source WHERE delta > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.newname); SELECT * FROM target ORDER BY tid; ROLLBACK; --self-merge BEGIN; MERGE INTO target t1 USING target t2 ON t1.tid = t2.tid WHEN MATCHED THEN UPDATE SET balance = t1.balance + t2.balance WHEN NOT MATCHED THEN INSERT VALUES (t2.tid, t2.balance); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT tid as sid, balance as delta FROM target WHERE balance > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT sid, max(delta) AS delta FROM source GROUP BY sid HAVING count(*) = 1 ORDER BY sid ASC) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- plpgsql parameters and results BEGIN; CREATE FUNCTION merge_func (p_id integer, p_bal integer) RETURNS INTEGER LANGUAGE plpgsql AS $$ DECLARE result integer; BEGIN MERGE INTO target t USING (SELECT p_id AS sid) AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance - p_bal; IF FOUND THEN GET DIAGNOSTICS result := ROW_COUNT; END IF; RETURN result; END; $$; SELECT merge_func(3, 4); SELECT * FROM target ORDER BY tid; ROLLBACK; -- PREPARE BEGIN; prepare foom as merge into target t using (select 1 as sid) s on (t.tid = s.sid) when matched then update set balance = 1; psql:include/ts_merge_query.sql:685: ERROR: prepared statement "foom" already exists execute foom; psql:include/ts_merge_query.sql:686: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; BEGIN; PREPARE foom2 (integer, integer) AS MERGE INTO target t USING (SELECT 1) s ON t.tid = $1 WHEN MATCHED THEN UPDATE SET balance = $2; psql:include/ts_merge_query.sql:695: ERROR: prepared statement "foom2" already exists --EXPLAIN (ANALYZE ON, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) execute foom2 (1, 1); psql:include/ts_merge_query.sql:697: ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; -- subqueries in source relation BEGIN; MERGE INTO sq_target t USING (SELECT * FROM sq_source) s ON tid = sid WHEN MATCHED AND t.balance > delta THEN UPDATE SET balance = t.balance + delta; SELECT * FROM sq_target ORDER BY tid; ROLLBACK; -- try a view CREATE VIEW v AS SELECT * FROM sq_source WHERE sid < 2; BEGIN; MERGE INTO sq_target USING v ON tid = sid WHEN MATCHED THEN UPDATE SET balance = v.balance + delta; SELECT * FROM sq_target ORDER BY tid; ROLLBACK; -- ambiguous reference to a column BEGIN; MERGE INTO sq_target USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; psql:include/ts_merge_query.sql:732: ERROR: column reference "balance" is ambiguous LINE 5: UPDATE SET balance = balance + delta ^ ROLLBACK; BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; SELECT * FROM sq_target; ROLLBACK; -- CTEs BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); WITH targq AS ( SELECT * FROM v ) MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; ROLLBACK; -- RETURNING BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE RETURNING *; ROLLBACK; -- PG17-specific tests for views, returning and merge_action. These throw syntax errors for previous versions of Postgres. -- However, since the error is the same for both hypertables and regular tables, this test should still pass for previous versions. -- RETURNING INSERT INTO source(sid, delta) VALUES(1, 40), (5, 50); BEGIN; MERGE INTO target t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta); SELECT * from target; ROLLBACK; BEGIN; MERGE INTO target t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 1 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta) RETURNING merge_action(), t.*; ROLLBACK; -- Views CREATE VIEW tv AS SELECT * FROM target; BEGIN; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta); SELECT * from tv; SELECT * from target; -- should also update the underlying table ROLLBACK; BEGIN; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta) RETURNING merge_action(), t.*; ROLLBACK; DROP VIEW tv; DELETE FROM source where sid in (1, 5); -- EXPLAIN CREATE TABLE ex_mtarget (a int, b int) WITH (autovacuum_enabled=off); CREATE TABLE ex_msource (a int, b int) WITH (autovacuum_enabled=off); INSERT INTO ex_mtarget SELECT i, i*10 FROM generate_series(1,100,2) i; INSERT INTO ex_msource SELECT i, i*10 FROM generate_series(1,100,1) i; CREATE FUNCTION explain_merge(query text) RETURNS SETOF text LANGUAGE plpgsql AS $$ DECLARE ln text; BEGIN FOR ln IN EXECUTE 'explain (analyze, timing off, summary off, buffers off, costs off) ' || query LOOP ln := regexp_replace(ln, '(Memory( Usage)?|Buckets|Batches): \S*', '\1: xxx', 'g'); RETURN NEXT ln; END LOOP; END; $$; -- only updates SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED THEN UPDATE SET b = t.b + 1'); -- only updates to selected tuples SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1'); -- updates + deletes SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1 WHEN MATCHED AND t.a >= 10 AND t.a <= 20 THEN DELETE'); -- only inserts SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN NOT MATCHED AND s.a < 10 THEN INSERT VALUES (a, b)'); -- all three SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1 WHEN MATCHED AND t.a >= 30 AND t.a <= 40 THEN DELETE WHEN NOT MATCHED AND s.a < 20 THEN INSERT VALUES (a, b)'); -- nothing SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a AND t.a < -1000 WHEN MATCHED AND t.a < 10 THEN DO NOTHING'); DROP TABLE ex_msource, ex_mtarget; DROP FUNCTION explain_merge(text); -- Subqueries BEGIN; MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED THEN UPDATE SET balance = (SELECT count(*) FROM sq_target); SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; BEGIN; MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND (SELECT count(*) > 0 FROM sq_target) THEN UPDATE SET balance = 42; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; BEGIN; MERGE INTO sq_target t USING v ON tid = sid AND (SELECT count(*) > 0 FROM sq_target) WHEN MATCHED THEN UPDATE SET balance = 42; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; -- Test RETURNING with subqueries BEGIN; MERGE INTO sq_target t USING v ON tid = sid AND (SELECT count(*) > 0 FROM sq_target) WHEN MATCHED THEN UPDATE SET balance = 42 RETURNING *; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; DROP TABLE sq_target CASCADE; DROP TABLE sq_source CASCADE; CREATE TABLE pa_target (tid integer, balance float, val text) PARTITION BY LIST (tid); CREATE TABLE part1 PARTITION OF pa_target FOR VALUES IN (1,4) WITH (autovacuum_enabled=off); CREATE TABLE part2 PARTITION OF pa_target FOR VALUES IN (2,5,6) WITH (autovacuum_enabled=off); CREATE TABLE part3 PARTITION OF pa_target FOR VALUES IN (3,8,9) WITH (autovacuum_enabled=off); CREATE TABLE part4 PARTITION OF pa_target DEFAULT WITH (autovacuum_enabled=off); CREATE TABLE pa_source (sid integer, delta float); -- insert many rows to the source table INSERT INTO pa_source SELECT id, id * 10 FROM generate_series(1,14) AS id; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT id, id * 100, 'initial' FROM generate_series(1,14,2) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- same with a constant qual BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid AND tid = 1 WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- try updating the partition key column BEGIN; CREATE FUNCTION merge_func() RETURNS integer LANGUAGE plpgsql AS $$ DECLARE result integer; BEGIN MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET tid = tid + 1, balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); IF FOUND THEN GET DIAGNOSTICS result := ROW_COUNT; END IF; RETURN result; END; $$; SELECT merge_func(); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_target CASCADE; -- The target table is partitioned in the same way, but this time by attaching -- partitions which have columns in different order, dropped columns etc. CREATE TABLE pa_target (tid integer, balance float, val text) PARTITION BY LIST (tid); CREATE TABLE part1 (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE part2 (balance float, tid integer, val text) WITH (autovacuum_enabled=off); CREATE TABLE part3 (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE part4 (extraid text, tid integer, balance float, val text) WITH (autovacuum_enabled=off); ALTER TABLE part4 DROP COLUMN extraid; ALTER TABLE pa_target ATTACH PARTITION part1 FOR VALUES IN (1,4); ALTER TABLE pa_target ATTACH PARTITION part2 FOR VALUES IN (2,5,6); ALTER TABLE pa_target ATTACH PARTITION part3 FOR VALUES IN (3,8,9); ALTER TABLE pa_target ATTACH PARTITION part4 DEFAULT; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT id, id * 100, 'initial' FROM generate_series(1,14,2) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- same with a constant qual BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid AND tid IN (1, 5) WHEN MATCHED AND tid % 5 = 0 THEN DELETE WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- try updating the partition key column BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET tid = tid + 1, balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_source; DROP TABLE pa_target CASCADE; -- Sub-partitioning CREATE TABLE pa_target (logts timestamp, tid integer, balance float, val text) PARTITION BY RANGE (logts); CREATE TABLE part_m01 PARTITION OF pa_target FOR VALUES FROM ('2017-01-01') TO ('2017-02-01') PARTITION BY LIST (tid); CREATE TABLE part_m01_odd PARTITION OF part_m01 FOR VALUES IN (1,3,5,7,9) WITH (autovacuum_enabled=off); CREATE TABLE part_m01_even PARTITION OF part_m01 FOR VALUES IN (2,4,6,8) WITH (autovacuum_enabled=off); CREATE TABLE part_m02 PARTITION OF pa_target FOR VALUES FROM ('2017-02-01') TO ('2017-03-01') PARTITION BY LIST (tid); CREATE TABLE part_m02_odd PARTITION OF part_m02 FOR VALUES IN (1,3,5,7,9) WITH (autovacuum_enabled=off); CREATE TABLE part_m02_even PARTITION OF part_m02 FOR VALUES IN (2,4,6,8) WITH (autovacuum_enabled=off); CREATE TABLE pa_source (sid integer, delta float) WITH (autovacuum_enabled=off); -- insert many rows to the source table INSERT INTO pa_source SELECT id, id * 10 FROM generate_series(1,14) AS id; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT '2017-01-31', id, id * 100, 'initial' FROM generate_series(1,9,3) AS id; INSERT INTO pa_target SELECT '2017-02-28', id, id * 100, 'initial' FROM generate_series(2,9,3) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING (SELECT '2017-01-15' AS slogts, * FROM pa_source WHERE sid < 10) s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (slogts::timestamp, sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_source; DROP TABLE pa_target CASCADE; -- some complex joins on the source side -- source relation is an unaliased join MERGE INTO cj_target t USING cj_source1 s1 INNER JOIN cj_source2 s2 ON sid1 = sid2 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid1, delta, sval); -- try accessing columns from either side of the source join MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 AND scat = 20 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid2, delta, sval) WHEN MATCHED THEN DELETE; -- some simple expressions in INSERT targetlist MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid2, delta + scat, sval) WHEN MATCHED THEN UPDATE SET val = val || ' updated by merge'; MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 AND scat = 20 ON t.tid = sid1 WHEN MATCHED THEN UPDATE SET val = val || ' ' || delta::text; SELECT * FROM cj_target ORDER BY tid; ALTER TABLE cj_source1 RENAME COLUMN sid1 TO sid; ALTER TABLE cj_source2 RENAME COLUMN sid2 TO sid; TRUNCATE cj_target; MERGE INTO cj_target t USING cj_source1 s1 INNER JOIN cj_source2 s2 ON s1.sid = s2.sid ON t.tid = s1.sid WHEN NOT MATCHED THEN INSERT VALUES (s2.sid, delta, sval); DROP TABLE cj_source2, cj_source1; DROP TABLE cj_target CASCADE; -- Function scans MERGE INTO fs_target t USING generate_series(1,100,1) AS id ON t.a = id WHEN MATCHED THEN UPDATE SET b = b + id WHEN NOT MATCHED THEN INSERT VALUES (id, -1); MERGE INTO fs_target t USING generate_series(1,100,2) AS id ON t.a = id WHEN MATCHED THEN UPDATE SET b = b + id, c = 'updated '|| id.*::text WHEN NOT MATCHED THEN INSERT VALUES (id, -1, 'inserted ' || id.*::text); SELECT count(*) FROM fs_target; DROP TABLE fs_target CASCADE; -- SERIALIZABLE test -- handled in isolation tests -- Inheritance-based partitioning CREATE TABLE measurement ( city_id int not null, logdate date not null, peaktemp int, unitsales int ) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2006m02 ( CHECK ( logdate >= DATE '2006-02-01' AND logdate < DATE '2006-03-01' ) ) INHERITS (measurement) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2006m03 ( CHECK ( logdate >= DATE '2006-03-01' AND logdate < DATE '2006-04-01' ) ) INHERITS (measurement) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2007m01 ( filler text, peaktemp int, logdate date not null, city_id int not null, unitsales int CHECK ( logdate >= DATE '2007-01-01' AND logdate < DATE '2007-02-01') ) WITH (autovacuum_enabled=off); ALTER TABLE measurement_y2007m01 DROP COLUMN filler; ALTER TABLE measurement_y2007m01 INHERIT measurement; INSERT INTO measurement VALUES (0, '2005-07-21', 5, 15); CREATE OR REPLACE FUNCTION measurement_insert_trigger() RETURNS TRIGGER AS $$ BEGIN IF ( NEW.logdate >= DATE '2006-02-01' AND NEW.logdate < DATE '2006-03-01' ) THEN INSERT INTO measurement_y2006m02 VALUES (NEW.*); ELSIF ( NEW.logdate >= DATE '2006-03-01' AND NEW.logdate < DATE '2006-04-01' ) THEN INSERT INTO measurement_y2006m03 VALUES (NEW.*); ELSIF ( NEW.logdate >= DATE '2007-01-01' AND NEW.logdate < DATE '2007-02-01' ) THEN INSERT INTO measurement_y2007m01 (city_id, logdate, peaktemp, unitsales) VALUES (NEW.*); ELSE RAISE EXCEPTION 'Date out of range. Fix the measurement_insert_trigger() function!'; END IF; RETURN NULL; END; $$ LANGUAGE plpgsql ; CREATE TRIGGER insert_measurement_trigger BEFORE INSERT ON measurement FOR EACH ROW EXECUTE PROCEDURE measurement_insert_trigger(); INSERT INTO measurement VALUES (1, '2006-02-10', 35, 10); INSERT INTO measurement VALUES (1, '2006-02-16', 45, 20); INSERT INTO measurement VALUES (1, '2006-03-17', 25, 10); INSERT INTO measurement VALUES (1, '2006-03-27', 15, 40); INSERT INTO measurement VALUES (1, '2007-01-15', 10, 10); INSERT INTO measurement VALUES (1, '2007-01-17', 10, 10); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate; CREATE TABLE new_measurement (LIKE measurement) WITH (autovacuum_enabled=off); INSERT INTO new_measurement VALUES (0, '2005-07-21', 25, 20); INSERT INTO new_measurement VALUES (1, '2006-03-01', 20, 10); INSERT INTO new_measurement VALUES (1, '2006-02-16', 50, 10); INSERT INTO new_measurement VALUES (2, '2006-02-10', 20, 20); INSERT INTO new_measurement VALUES (1, '2006-03-27', NULL, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-17', NULL, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-15', 5, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-16', 10, 10); BEGIN; MERGE INTO ONLY measurement m USING new_measurement nm ON (m.city_id = nm.city_id and m.logdate=nm.logdate) WHEN MATCHED AND nm.peaktemp IS NULL THEN DELETE WHEN MATCHED THEN UPDATE SET peaktemp = greatest(m.peaktemp, nm.peaktemp), unitsales = m.unitsales + coalesce(nm.unitsales, 0) WHEN NOT MATCHED THEN INSERT (city_id, logdate, peaktemp, unitsales) VALUES (city_id, logdate, peaktemp, unitsales); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate, peaktemp; ROLLBACK; MERGE into measurement m USING new_measurement nm ON (m.city_id = nm.city_id and m.logdate=nm.logdate) WHEN MATCHED AND nm.peaktemp IS NULL THEN DELETE WHEN MATCHED THEN UPDATE SET peaktemp = greatest(m.peaktemp, nm.peaktemp), unitsales = m.unitsales + coalesce(nm.unitsales, 0) WHEN NOT MATCHED THEN INSERT (city_id, logdate, peaktemp, unitsales) VALUES (city_id, logdate, peaktemp, unitsales); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate; BEGIN; MERGE INTO new_measurement nm USING ONLY measurement m ON (nm.city_id = m.city_id and nm.logdate=m.logdate) WHEN MATCHED THEN DELETE; SELECT * FROM new_measurement ORDER BY city_id, logdate; ROLLBACK; MERGE INTO new_measurement nm USING measurement m ON (nm.city_id = m.city_id and nm.logdate=m.logdate) WHEN MATCHED THEN DELETE; SELECT * FROM new_measurement ORDER BY city_id, logdate; DROP TABLE measurement, new_measurement CASCADE; DROP FUNCTION measurement_insert_trigger(); RESET SESSION AUTHORIZATION; DROP TABLE target CASCADE; DROP TABLE target2 CASCADE; DROP TABLE source, source2; DROP FUNCTION merge_trigfunc(); REVOKE CREATE ON SCHEMA public FROM regress_merge_privs; DROP USER regress_merge_privs; DROP USER regress_merge_no_privs; \o :DIFF_CMD ================================================ FILE: test/expected/update.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \o /dev/null \ir include/insert_single.sql -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."one_Partition" ( "timeCustom" BIGINT NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."one_Partition" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; CREATE INDEX ON PUBLIC."one_Partition" ("timeCustom" DESC NULLS LAST, series_0) WHERE series_0 IS NOT NULL; CREATE INDEX ON PUBLIC."one_Partition" ("timeCustom" DESC NULLS LAST, series_1) WHERE series_1 IS NOT NULL; CREATE INDEX ON PUBLIC."one_Partition" ("timeCustom" DESC NULLS LAST, series_2) WHERE series_2 IS NOT NULL; CREATE INDEX ON PUBLIC."one_Partition" ("timeCustom" DESC NULLS LAST, series_bool) WHERE series_bool IS NOT NULL; \c :DBNAME :ROLE_SUPERUSER CREATE SCHEMA "one_Partition" AUTHORIZATION :ROLE_DEFAULT_PERM_USER; \c :DBNAME :ROLE_DEFAULT_PERM_USER; SELECT * FROM create_hypertable('"public"."one_Partition"', 'timeCustom', associated_schema_name=>'one_Partition', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); --output command tags \set QUIET off BEGIN; \COPY "one_Partition" FROM 'data/ds1_dev1_1.tsv' NULL AS ''; COMMIT; INSERT INTO "one_Partition"("timeCustom", device_id, series_0, series_1) VALUES (1257987600000000000, 'dev1', 1.5, 1), (1257987600000000000, 'dev1', 1.5, 2), (1257894000000000000, 'dev2', 1.5, 1), (1257894002000000000, 'dev1', 2.5, 3); INSERT INTO "one_Partition"("timeCustom", device_id, series_0, series_1) VALUES (1257894000000000000, 'dev2', 1.5, 2); \set QUIET on \o -- Make sure UPDATE isn't optimized if it includes Append plans -- Need to turn of nestloop to make append appear the same on PG96 and PG10 set enable_nestloop = 'off'; CREATE OR REPLACE FUNCTION series_val() RETURNS integer LANGUAGE PLPGSQL STABLE AS $BODY$ BEGIN RETURN 5; END; $BODY$; -- ConstraintAwareAppend applied for SELECT EXPLAIN (buffers off, costs off) SELECT FROM "one_Partition" WHERE series_1 IN (SELECT series_1 FROM "one_Partition" WHERE series_1 > series_val()); --- QUERY PLAN --- Hash Join Hash Cond: ("one_Partition".series_1 = "one_Partition_1".series_1) -> Custom Scan (ChunkAppend) on "one_Partition" Chunks excluded during startup: 0 -> Index Only Scan using "_hyper_1_1_chunk_one_Partition_timeCustom_series_1_idx" on _hyper_1_1_chunk Index Cond: (series_1 > (series_val())::double precision) -> Index Only Scan using "_hyper_1_2_chunk_one_Partition_timeCustom_series_1_idx" on _hyper_1_2_chunk Index Cond: (series_1 > (series_val())::double precision) -> Index Only Scan using "_hyper_1_3_chunk_one_Partition_timeCustom_series_1_idx" on _hyper_1_3_chunk Index Cond: (series_1 > (series_val())::double precision) -> Hash -> HashAggregate Group Key: "one_Partition_1".series_1 -> Custom Scan (ChunkAppend) on "one_Partition" "one_Partition_1" Chunks excluded during startup: 0 -> Index Only Scan using "_hyper_1_1_chunk_one_Partition_timeCustom_series_1_idx" on _hyper_1_1_chunk _hyper_1_1_chunk_1 Index Cond: (series_1 > (series_val())::double precision) -> Index Only Scan using "_hyper_1_2_chunk_one_Partition_timeCustom_series_1_idx" on _hyper_1_2_chunk _hyper_1_2_chunk_1 Index Cond: (series_1 > (series_val())::double precision) -> Index Only Scan using "_hyper_1_3_chunk_one_Partition_timeCustom_series_1_idx" on _hyper_1_3_chunk _hyper_1_3_chunk_1 Index Cond: (series_1 > (series_val())::double precision) -- ConstraintAwareAppend NOT applied for UPDATE EXPLAIN (buffers off, costs off) UPDATE "one_Partition" SET series_1 = 8 WHERE series_1 IN (SELECT series_1 FROM "one_Partition" WHERE series_1 > series_val()); --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Update on "one_Partition" Update on _hyper_1_1_chunk "one_Partition_2" Update on _hyper_1_2_chunk "one_Partition_3" Update on _hyper_1_3_chunk "one_Partition_4" -> Hash Join Hash Cond: ("one_Partition".series_1 = "one_Partition_1".series_1) -> Append -> Seq Scan on _hyper_1_1_chunk "one_Partition_2" -> Seq Scan on _hyper_1_2_chunk "one_Partition_3" -> Seq Scan on _hyper_1_3_chunk "one_Partition_4" -> Hash -> HashAggregate Group Key: "one_Partition_1".series_1 -> Append -> Index Scan using "_hyper_1_1_chunk_one_Partition_timeCustom_series_1_idx" on _hyper_1_1_chunk "one_Partition_5" Index Cond: (series_1 > (series_val())::double precision) -> Index Scan using "_hyper_1_2_chunk_one_Partition_timeCustom_series_1_idx" on _hyper_1_2_chunk "one_Partition_6" Index Cond: (series_1 > (series_val())::double precision) -> Index Scan using "_hyper_1_3_chunk_one_Partition_timeCustom_series_1_idx" on _hyper_1_3_chunk "one_Partition_7" Index Cond: (series_1 > (series_val())::double precision) SELECT * FROM "one_Partition" ORDER BY "timeCustom", device_id, series_0, series_1, series_2; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257894000000000000 | dev1 | 1.5 | 1 | 2 | t 1257894000000000000 | dev1 | 1.5 | 2 | | 1257894000000000000 | dev2 | 1.5 | 1 | | 1257894000000000000 | dev2 | 1.5 | 2 | | 1257894000000001000 | dev1 | 2.5 | 3 | | 1257894001000000000 | dev1 | 3.5 | 4 | | 1257894002000000000 | dev1 | 2.5 | 3 | | 1257894002000000000 | dev1 | 5.5 | 6 | | t 1257894002000000000 | dev1 | 5.5 | 7 | | f 1257897600000000000 | dev1 | 4.5 | 5 | | f 1257987600000000000 | dev1 | 1.5 | 1 | | 1257987600000000000 | dev1 | 1.5 | 2 | | UPDATE "one_Partition" SET series_1 = 8 WHERE series_1 IN (SELECT series_1 FROM "one_Partition" WHERE series_1 > series_val()); SELECT * FROM "one_Partition" ORDER BY "timeCustom", device_id, series_0, series_1, series_2; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257894000000000000 | dev1 | 1.5 | 1 | 2 | t 1257894000000000000 | dev1 | 1.5 | 2 | | 1257894000000000000 | dev2 | 1.5 | 1 | | 1257894000000000000 | dev2 | 1.5 | 2 | | 1257894000000001000 | dev1 | 2.5 | 3 | | 1257894001000000000 | dev1 | 3.5 | 4 | | 1257894002000000000 | dev1 | 2.5 | 3 | | 1257894002000000000 | dev1 | 5.5 | 8 | | f 1257894002000000000 | dev1 | 5.5 | 8 | | t 1257897600000000000 | dev1 | 4.5 | 5 | | f 1257987600000000000 | dev1 | 1.5 | 1 | | 1257987600000000000 | dev1 | 1.5 | 2 | | UPDATE "one_Partition" SET series_1 = 47; UPDATE "one_Partition" SET series_bool = true; SELECT * FROM "one_Partition" ORDER BY "timeCustom", device_id, series_0, series_1, series_2; timeCustom | device_id | series_0 | series_1 | series_2 | series_bool ---------------------+-----------+----------+----------+----------+------------- 1257894000000000000 | dev1 | 1.5 | 47 | 2 | t 1257894000000000000 | dev1 | 1.5 | 47 | | t 1257894000000000000 | dev2 | 1.5 | 47 | | t 1257894000000000000 | dev2 | 1.5 | 47 | | t 1257894000000001000 | dev1 | 2.5 | 47 | | t 1257894001000000000 | dev1 | 3.5 | 47 | | t 1257894002000000000 | dev1 | 2.5 | 47 | | t 1257894002000000000 | dev1 | 5.5 | 47 | | t 1257894002000000000 | dev1 | 5.5 | 47 | | t 1257897600000000000 | dev1 | 4.5 | 47 | | t 1257987600000000000 | dev1 | 1.5 | 47 | | t 1257987600000000000 | dev1 | 1.5 | 47 | | t -- test update on chunks directly CREATE TABLE direct_update(time timestamptz) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column INSERT INTO direct_update VALUES ('2020-01-01'); SELECT show_chunks('direct_update') AS "CHUNK" \gset --should have ModifyHyperable node EXPLAIN (costs off, timing off, summary off) UPDATE :CHUNK SET time = time + INTERVAL '1 minute'; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Update on _hyper_2_4_chunk -> Seq Scan on _hyper_2_4_chunk EXPLAIN (costs off, timing off, summary off) UPDATE ONLY :CHUNK SET time = time + INTERVAL '1 minute'; --- QUERY PLAN --- Custom Scan (ModifyHypertable) -> Update on _hyper_2_4_chunk -> Seq Scan on _hyper_2_4_chunk -- correct time range should succeed UPDATE :CHUNK SET time = time + INTERVAL '1 minute' RETURNING *; time ------------------------------ Wed Jan 01 00:01:00 2020 PST UPDATE ONLY :CHUNK SET time = time + INTERVAL '1 minute' RETURNING *; time ------------------------------ Wed Jan 01 00:02:00 2020 PST -- crossing chunk boundary should fail \set ON_ERROR_STOP 0 UPDATE :CHUNK SET time = time + INTERVAL '1 month' RETURNING *; ERROR: new row for relation "_hyper_2_4_chunk" violates check constraint "constraint_4" UPDATE ONLY :CHUNK SET time = time + INTERVAL '1 month' RETURNING *; ERROR: new row for relation "_hyper_2_4_chunk" violates check constraint "constraint_4" \set ON_ERROR_STOP 1 -- github issue #6790 -- test UPDATE with WHERE EXISTS on hypertable CREATE TABLE i6790_update(time timestamptz NOT NULL, device int, value float) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column INSERT INTO i6790_update SELECT t, 1, 0.1 FROM generate_series('2026-01-01'::timestamptz, '2026-01-03'::timestamptz, interval '12 hours') t; -- UPDATE with simple EXISTS - creates gating Result node(s) wrapping ChunkAppend UPDATE i6790_update SET value = 0.2 WHERE EXISTS (SELECT 1); SELECT count(*) FROM i6790_update WHERE value = 0.2; count ------- 5 -- UPDATE with correlated EXISTS UPDATE i6790_update SET value = 0.3 WHERE EXISTS (SELECT 1 FROM i6790_update g WHERE g.device = i6790_update.device); SELECT count(*) FROM i6790_update WHERE value = 0.3; count ------- 5 ================================================ FILE: test/expected/upsert.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE upsert_test(time timestamp PRIMARY KEY, temp float, color text); SELECT create_hypertable('upsert_test', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------- (1,public,upsert_test,t) INSERT INTO upsert_test VALUES ('2017-01-20T09:00:01', 22.5, 'yellow') RETURNING *; time | temp | color --------------------------+------+-------- Fri Jan 20 09:00:01 2017 | 22.5 | yellow INSERT INTO upsert_test VALUES ('2017-01-20T09:00:01', 23.8, 'yellow') ON CONFLICT (time) DO UPDATE SET temp = 23.8 RETURNING *; time | temp | color --------------------------+------+-------- Fri Jan 20 09:00:01 2017 | 23.8 | yellow INSERT INTO upsert_test VALUES ('2017-01-20T09:00:01', 78.4, 'yellow') ON CONFLICT DO NOTHING; SELECT * FROM upsert_test; time | temp | color --------------------------+------+-------- Fri Jan 20 09:00:01 2017 | 23.8 | yellow -- Test 'Tuples Inserted' and 'Conflicting Tuples' values in EXPLAIN ANALYZE EXPLAIN (VERBOSE, ANALYZE, BUFFERS FALSE, COSTS FALSE, TIMING FALSE, SUMMARY FALSE) INSERT INTO upsert_test VALUES ('2017-01-20T09:00:01', 28.5, 'blue'), ('2017-01-20T09:00:01', 21.9, 'red'), ('2017-01-20T10:00:01', 2.4, 'pink') ON CONFLICT DO NOTHING; --- QUERY PLAN --- Custom Scan (ModifyHypertable) (actual rows=0.00 loops=1) -> Insert on public.upsert_test (actual rows=0.00 loops=1) Conflict Resolution: NOTHING Tuples Inserted: 1 Conflicting Tuples: 2 -> Values Scan on "*VALUES*" (actual rows=3.00 loops=1) Output: "*VALUES*".column1, "*VALUES*".column2, "*VALUES*".column3 -- Test ON CONFLICT ON CONSTRAINT INSERT INTO upsert_test VALUES ('2017-01-20T09:00:01', 12.3, 'yellow') ON CONFLICT ON CONSTRAINT upsert_test_pkey DO UPDATE SET temp = 12.3 RETURNING time, temp, color; time | temp | color --------------------------+------+-------- Fri Jan 20 09:00:01 2017 | 12.3 | yellow -- Test that update generates error on conflicts \set ON_ERROR_STOP 0 INSERT INTO upsert_test VALUES ('2017-01-21T09:00:01', 22.5, 'yellow') RETURNING *; time | temp | color --------------------------+------+-------- Sat Jan 21 09:00:01 2017 | 22.5 | yellow UPDATE upsert_test SET time = '2017-01-20T09:00:01'; ERROR: duplicate key value violates unique constraint "1_1_upsert_test_pkey" \set ON_ERROR_STOP 1 -- Test with UNIQUE index on multiple columns instead of PRIMARY KEY constraint CREATE TABLE upsert_test_unique(time timestamp, temp float, color text); SELECT create_hypertable('upsert_test_unique', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable --------------------------------- (2,public,upsert_test_unique,t) CREATE UNIQUE INDEX time_color_idx ON upsert_test_unique (time, color); INSERT INTO upsert_test_unique VALUES ('2017-01-20T09:00:01', 22.5, 'yellow') RETURNING *; time | temp | color --------------------------+------+-------- Fri Jan 20 09:00:01 2017 | 22.5 | yellow INSERT INTO upsert_test_unique VALUES ('2017-01-20T09:00:01', 21.2, 'brown'); SELECT * FROM upsert_test_unique ORDER BY time, color DESC; time | temp | color --------------------------+------+-------- Fri Jan 20 09:00:01 2017 | 22.5 | yellow Fri Jan 20 09:00:01 2017 | 21.2 | brown INSERT INTO upsert_test_unique VALUES ('2017-01-20T09:00:01', 31.8, 'yellow') ON CONFLICT (time, color) DO UPDATE SET temp = 31.8; INSERT INTO upsert_test_unique VALUES ('2017-01-20T09:00:01', 54.3, 'yellow') ON CONFLICT DO NOTHING; SELECT * FROM upsert_test_unique ORDER BY time, color DESC; time | temp | color --------------------------+------+-------- Fri Jan 20 09:00:01 2017 | 31.8 | yellow Fri Jan 20 09:00:01 2017 | 21.2 | brown -- Test with multiple UNIQUE indexes CREATE TABLE upsert_test_multi_unique(time timestamp, temp float, color text); SELECT create_hypertable('upsert_test_multi_unique', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable --------------------------------------- (3,public,upsert_test_multi_unique,t) ALTER TABLE upsert_test_multi_unique ADD CONSTRAINT multi_time_temp UNIQUE (time, temp); CREATE UNIQUE INDEX multi_time_color_idx ON upsert_test_multi_unique (time, color); INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 25.9, 'yellow'); INSERT INTO upsert_test_multi_unique VALUES ('2017-01-21T09:00:01', 25.9, 'yellow'); INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 23.5, 'brown'); INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 25.9, 'purple') ON CONFLICT DO NOTHING; SELECT * FROM upsert_test_multi_unique ORDER BY time, color DESC; time | temp | color --------------------------+------+-------- Fri Jan 20 09:00:01 2017 | 25.9 | yellow Fri Jan 20 09:00:01 2017 | 23.5 | brown Sat Jan 21 09:00:01 2017 | 25.9 | yellow INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 25.9, 'blue') ON CONFLICT (time, temp) DO UPDATE SET color = 'blue'; INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 23.5, 'orange') ON CONFLICT ON CONSTRAINT multi_time_temp DO UPDATE SET color = excluded.color; SELECT * FROM upsert_test_multi_unique ORDER BY time, color DESC; time | temp | color --------------------------+------+-------- Fri Jan 20 09:00:01 2017 | 23.5 | orange Fri Jan 20 09:00:01 2017 | 25.9 | blue Sat Jan 21 09:00:01 2017 | 25.9 | yellow INSERT INTO upsert_test_multi_unique VALUES ('2017-01-21T09:00:01', 45.7, 'yellow') ON CONFLICT (time, color) DO UPDATE SET temp = 45.7; SELECT * FROM upsert_test_multi_unique ORDER BY time, color DESC; time | temp | color --------------------------+------+-------- Fri Jan 20 09:00:01 2017 | 23.5 | orange Fri Jan 20 09:00:01 2017 | 25.9 | blue Sat Jan 21 09:00:01 2017 | 45.7 | yellow \set ON_ERROR_STOP 0 -- Here the constraint in the ON CONFLICT clause is not the one that is -- actually violated by the INSERT, so it should still fail. INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 23.5, 'purple') ON CONFLICT (time, color) DO UPDATE set temp = 23.5; ERROR: duplicate key value violates unique constraint "3_2_multi_time_temp" INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 22.5, 'orange') ON CONFLICT ON CONSTRAINT multi_time_temp DO UPDATE set color = 'orange'; ERROR: duplicate key value violates unique constraint "_hyper_3_3_chunk_multi_time_color_idx" \set ON_ERROR_STOP 1 CREATE TABLE upsert_test_space(time timestamp, device_id_1 char(20), to_drop int, temp float, color text); --drop two columns; create one. ALTER TABLE upsert_test_space DROP to_drop; ALTER TABLE upsert_test_space DROP device_id_1, ADD device_id char(20); ALTER TABLE upsert_test_space ADD CONSTRAINT time_space_constraint UNIQUE (time, device_id); SELECT create_hypertable('upsert_test_space', 'time', 'device_id', 2, partitioning_func=>'_timescaledb_functions.get_partition_for_key'::regproc); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------------- (4,public,upsert_test_space,t) INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev1', 25.9, 'yellow') RETURNING *; time | temp | color | device_id --------------------------+------+--------+---------------------- Fri Jan 20 09:00:01 2017 | 25.9 | yellow | dev1 INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev2', 25.9, 'yellow'); INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev1', 23.5, 'green') ON CONFLICT (time, device_id) DO UPDATE SET color = excluded.color; INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev1', 23.5, 'orange') ON CONFLICT ON CONSTRAINT time_space_constraint DO UPDATE SET color = excluded.color; INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev2', 23.5, 'orange3') ON CONFLICT (time, device_id) DO UPDATE SET color = excluded.color||' (originally '|| upsert_test_space.color ||')' RETURNING *; time | temp | color | device_id --------------------------+------+-----------------------------+---------------------- Fri Jan 20 09:00:01 2017 | 25.9 | orange3 (originally yellow) | dev2 INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev3', 23.5, 'orange3.1') ON CONFLICT (time, device_id) DO UPDATE SET color = excluded.color||' (originally '|| upsert_test_space.color ||')' RETURNING *; time | temp | color | device_id --------------------------+------+-----------+---------------------- Fri Jan 20 09:00:01 2017 | 23.5 | orange3.1 | dev3 INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev2', 23.5, 'orange4') ON CONFLICT (time, device_id) DO NOTHING RETURNING *; time | temp | color | device_id ------+------+-------+----------- INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev4', 23.5, 'orange5') ON CONFLICT (time, device_id) DO NOTHING RETURNING *; time | temp | color | device_id --------------------------+------+---------+---------------------- Fri Jan 20 09:00:01 2017 | 23.5 | orange5 | dev4 INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev5', 23.5, 'orange5') ON CONFLICT (time, device_id) DO NOTHING RETURNING *; time | temp | color | device_id --------------------------+------+---------+---------------------- Fri Jan 20 09:00:01 2017 | 23.5 | orange5 | dev5 INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev5', 23.5, 'orange6') ON CONFLICT ON CONSTRAINT time_space_constraint DO NOTHING RETURNING *; time | temp | color | device_id ------+------+-------+----------- --restore a column with the same name as a previously deleted one; ALTER TABLE upsert_test_space ADD device_id_1 char(20); INSERT INTO upsert_test_space (time, device_id, temp, color, device_id_1) VALUES ('2017-01-20T09:00:01', 'dev4', 23.5, 'orange5.1', 'dev-id-1') ON CONFLICT (time, device_id) DO UPDATE SET color = excluded.color||' (originally '|| upsert_test_space.color ||')' RETURNING *; time | temp | color | device_id | device_id_1 --------------------------+------+--------------------------------+----------------------+------------- Fri Jan 20 09:00:01 2017 | 23.5 | orange5.1 (originally orange5) | dev4 | INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev5', 23.5, 'orange6') ON CONFLICT (time, device_id) DO UPDATE SET color = excluded.color WHERE upsert_test_space.temp < 20 RETURNING *; time | temp | color | device_id | device_id_1 ------+------+-------+-----------+------------- INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev5', 23.5, 'orange7') ON CONFLICT (time, device_id) DO UPDATE SET color = excluded.color WHERE excluded.temp < 20 RETURNING *; time | temp | color | device_id | device_id_1 ------+------+-------+-----------+------------- INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev5', 3.5, 'orange7') ON CONFLICT (time, device_id) DO UPDATE SET color = excluded.color, temp=excluded.temp WHERE excluded.temp < 20 RETURNING *; time | temp | color | device_id | device_id_1 --------------------------+------+---------+----------------------+------------- Fri Jan 20 09:00:01 2017 | 3.5 | orange7 | dev5 | INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev5', 43.5, 'orange8') ON CONFLICT (time, device_id) DO UPDATE SET color = excluded.color WHERE upsert_test_space.temp < 20 RETURNING *; time | temp | color | device_id | device_id_1 --------------------------+------+---------+----------------------+------------- Fri Jan 20 09:00:01 2017 | 3.5 | orange8 | dev5 | INSERT INTO upsert_test_space (time, device_id, temp, color, device_id_1) VALUES ('2017-01-20T09:00:01', 'dev5', 43.5, 'orange8', 'device-id-1-new') ON CONFLICT (time, device_id) DO UPDATE SET device_id_1 = excluded.device_id_1 RETURNING *; time | temp | color | device_id | device_id_1 --------------------------+------+---------+----------------------+---------------------- Fri Jan 20 09:00:01 2017 | 3.5 | orange8 | dev5 | device-id-1-new INSERT INTO upsert_test_space (time, device_id, temp, color, device_id_1) VALUES ('2017-01-20T09:00:01', 'dev5', 43.5, 'orange8', 'device-id-1-new') ON CONFLICT (time, device_id) DO UPDATE SET device_id_1 = 'device-id-1-new-2', color = 'orange9' RETURNING *; time | temp | color | device_id | device_id_1 --------------------------+------+---------+----------------------+---------------------- Fri Jan 20 09:00:01 2017 | 3.5 | orange9 | dev5 | device-id-1-new-2 SELECT * FROM upsert_test_space; time | temp | color | device_id | device_id_1 --------------------------+------+--------------------------------+----------------------+---------------------- Fri Jan 20 09:00:01 2017 | 25.9 | orange | dev1 | Fri Jan 20 09:00:01 2017 | 25.9 | orange3 (originally yellow) | dev2 | Fri Jan 20 09:00:01 2017 | 23.5 | orange3.1 | dev3 | Fri Jan 20 09:00:01 2017 | 23.5 | orange5.1 (originally orange5) | dev4 | Fri Jan 20 09:00:01 2017 | 3.5 | orange9 | dev5 | device-id-1-new-2 ALTER TABLE upsert_test_space DROP device_id_1, ADD device_id_2 char(20); INSERT INTO upsert_test_space (time, device_id, temp, color, device_id_2) VALUES ('2017-01-20T09:00:01', 'dev5', 43.5, 'orange8', 'device-id-2') ON CONFLICT (time, device_id) DO UPDATE SET device_id_2 = 'device-id-2-new', color = 'orange10' RETURNING *; time | temp | color | device_id | device_id_2 --------------------------+------+----------+----------------------+---------------------- Fri Jan 20 09:00:01 2017 | 3.5 | orange10 | dev5 | device-id-2-new --test inserting to to a chunk already in the chunk dispatch cache again. INSERT INTO upsert_test_space as current (time, device_id, temp, color, device_id_2) VALUES ('2017-01-20T09:00:01', 'dev5', 43.5, 'orange8', 'device-id-2'), ('2018-01-20T09:00:01', 'dev5', 43.5, 'orange8', 'device-id-2'), ('2017-01-20T09:00:01', 'dev3', 43.5, 'orange7', 'device-id-2'), ('2018-01-21T09:00:01', 'dev5', 43.5, 'orange9', 'device-id-2') ON CONFLICT (time, device_id) DO UPDATE SET device_id_2 = coalesce(excluded.device_id_2,current.device_id_2), color = coalesce(excluded.color,current.color) RETURNING *; time | temp | color | device_id | device_id_2 --------------------------+------+---------+----------------------+---------------------- Fri Jan 20 09:00:01 2017 | 3.5 | orange8 | dev5 | device-id-2 Sat Jan 20 09:00:01 2018 | 43.5 | orange8 | dev5 | device-id-2 Fri Jan 20 09:00:01 2017 | 23.5 | orange7 | dev3 | device-id-2 Sun Jan 21 09:00:01 2018 | 43.5 | orange9 | dev5 | device-id-2 WITH CTE AS ( INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 25.9, 'purple') ON CONFLICT DO NOTHING RETURNING * ) SELECT 1; ?column? ---------- 1 WITH CTE AS ( INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 25.9, 'purple'), ('2017-01-20T09:00:01', 29.9, 'purple1') ON CONFLICT DO NOTHING RETURNING * ) SELECT * FROM CTE; time | temp | color --------------------------+------+--------- Fri Jan 20 09:00:01 2017 | 29.9 | purple1 WITH CTE AS ( INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 25.9, 'blue') ON CONFLICT (time, temp) DO UPDATE SET color = 'blue' RETURNING * ) SELECT * FROM CTE; time | temp | color --------------------------+------+------- Fri Jan 20 09:00:01 2017 | 25.9 | blue --test error conditions when an index is dropped on a chunk DROP INDEX _timescaledb_internal._hyper_3_3_chunk_multi_time_color_idx; --everything is ok if not used as an arbiter index INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 25.9, 'purple') ON CONFLICT DO NOTHING RETURNING *; time | temp | color ------+------+------- --errors out if used as an arbiter index \set ON_ERROR_STOP 0 INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 25.9, 'purple') ON CONFLICT (time, color) DO NOTHING RETURNING *; ERROR: could not find arbiter index for hypertable index "multi_time_color_idx" on chunk "_hyper_3_3_chunk" \set ON_ERROR_STOP 1 --create table with one chunk that has a tup_conv_map and one that does not --to ensure this, create a chunk before altering the table this chunk will not have a tup_conv_map CREATE TABLE upsert_test_diffchunk(time timestamp, device_id char(20), to_drop int, temp float, color text); SELECT create_hypertable('upsert_test_diffchunk', 'time', chunk_time_interval=> interval '1 month'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ------------------------------------ (5,public,upsert_test_diffchunk,t) CREATE UNIQUE INDEX time_device_idx ON upsert_test_diffchunk (time, device_id); --this is the chunk with no tup_conv_map INSERT INTO upsert_test_diffchunk (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev1', 25.9, 'yellow') RETURNING *; time | device_id | to_drop | temp | color --------------------------+----------------------+---------+------+-------- Fri Jan 20 09:00:01 2017 | dev1 | | 25.9 | yellow INSERT INTO upsert_test_diffchunk (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev2', 25.9, 'yellow') RETURNING *; time | device_id | to_drop | temp | color --------------------------+----------------------+---------+------+-------- Fri Jan 20 09:00:01 2017 | dev2 | | 25.9 | yellow --alter the table ALTER TABLE upsert_test_diffchunk DROP to_drop; ALTER TABLE upsert_test_diffchunk ADD device_id_2 char(20); --new chunk that does have a tup conv map INSERT INTO upsert_test_diffchunk (time, device_id, temp, color) VALUES ('2019-01-20T09:00:01', 'dev1', 23.5, 'orange') ; INSERT INTO upsert_test_diffchunk (time, device_id, temp, color) VALUES ('2019-01-20T09:00:01', 'dev2', 23.5, 'orange') ; select * from upsert_test_diffchunk order by time, device_id; time | device_id | temp | color | device_id_2 --------------------------+----------------------+------+--------+------------- Fri Jan 20 09:00:01 2017 | dev1 | 25.9 | yellow | Fri Jan 20 09:00:01 2017 | dev2 | 25.9 | yellow | Sun Jan 20 09:00:01 2019 | dev1 | 23.5 | orange | Sun Jan 20 09:00:01 2019 | dev2 | 23.5 | orange | --make sure current works INSERT INTO upsert_test_diffchunk as current (time, device_id, temp, color, device_id_2) VALUES ('2019-01-20T09:00:01', 'dev1', 43.5, 'orange2', 'device-id-2'), ('2017-01-20T09:00:01', 'dev1', 43.5, 'yellow2', 'device-id-2'), ('2019-01-20T09:00:01', 'dev2', 43.5, 'orange2', 'device-id-2') ON CONFLICT (time, device_id) DO UPDATE SET device_id_2 = coalesce(excluded.device_id_2,current.device_id_2), temp = coalesce(excluded.temp,current.temp) , color = coalesce(excluded.color,current.color); select * from upsert_test_diffchunk order by time, device_id; time | device_id | temp | color | device_id_2 --------------------------+----------------------+------+---------+---------------------- Fri Jan 20 09:00:01 2017 | dev1 | 43.5 | yellow2 | device-id-2 Fri Jan 20 09:00:01 2017 | dev2 | 25.9 | yellow | Sun Jan 20 09:00:01 2019 | dev1 | 43.5 | orange2 | device-id-2 Sun Jan 20 09:00:01 2019 | dev2 | 43.5 | orange2 | device-id-2 --arbiter index tests CREATE TABLE upsert_test_arbiter(time timestamp, to_drop int); SELECT create_hypertable('upsert_test_arbiter', 'time', chunk_time_interval=> interval '1 month'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ---------------------------------- (6,public,upsert_test_arbiter,t) --this is the chunk with no tup_conv_map INSERT INTO upsert_test_arbiter (time, to_drop) VALUES ('2017-01-20T09:00:01', 1) RETURNING *; time | to_drop --------------------------+--------- Fri Jan 20 09:00:01 2017 | 1 INSERT INTO upsert_test_arbiter (time, to_drop) VALUES ('2017-01-21T09:00:01', 2) RETURNING *; time | to_drop --------------------------+--------- Sat Jan 21 09:00:01 2017 | 2 INSERT INTO upsert_test_arbiter (time, to_drop) VALUES ('2017-03-20T09:00:01', 3) RETURNING *; time | to_drop --------------------------+--------- Mon Mar 20 09:00:01 2017 | 3 --alter the table ALTER TABLE upsert_test_arbiter DROP to_drop; ALTER TABLE upsert_test_arbiter ADD device_id char(20) DEFAULT 'dev1'; CREATE UNIQUE INDEX arbiter_time_device_idx ON upsert_test_arbiter (time, device_id); INSERT INTO upsert_test_arbiter as current (time, device_id) VALUES ('2018-01-21T09:00:01', 'dev1'), ('2017-01-20T09:00:01', 'dev1'), ('2017-01-21T09:00:01', 'dev2'), ('2018-01-21T09:00:01', 'dev2') ON CONFLICT (time, device_id) DO UPDATE SET device_id = coalesce(excluded.device_id,current.device_id) RETURNING *; time | device_id --------------------------+---------------------- Sun Jan 21 09:00:01 2018 | dev1 Fri Jan 20 09:00:01 2017 | dev1 Sat Jan 21 09:00:01 2017 | dev2 Sun Jan 21 09:00:01 2018 | dev2 with cte as ( INSERT INTO upsert_test_arbiter (time, device_id) VALUES ('2017-01-21T09:00:01', 'dev2'), ('2018-01-21T09:00:01', 'dev2') ON CONFLICT (time, device_id) DO UPDATE SET device_id = 'dev3' RETURNING *) select * from cte; time | device_id --------------------------+---------------------- Sat Jan 21 09:00:01 2017 | dev3 Sun Jan 21 09:00:01 2018 | dev3 -- test ON CONFLICT with prepared statements CREATE TABLE prepared_test(time timestamptz PRIMARY KEY, value float CHECK(value > 0)); SELECT create_hypertable('prepared_test','time'); create_hypertable ---------------------------- (7,public,prepared_test,t) CREATE TABLE source_data(time timestamptz PRIMARY KEY, value float); INSERT INTO source_data VALUES('2000-01-01',0.5), ('2001-01-01',0.5); -- at some point PostgreSQL will turn the plan into a generic plan -- so we execute the prepared statement 10 times -- check that an error in the prepared statement does not lead to the plan becoming unusable PREPARE prep_insert_select AS INSERT INTO prepared_test select * from source_data ON CONFLICT (time) DO UPDATE SET value = EXCLUDED.value; EXECUTE prep_insert_select; EXECUTE prep_insert_select; EXECUTE prep_insert_select; EXECUTE prep_insert_select; EXECUTE prep_insert_select; EXECUTE prep_insert_select; EXECUTE prep_insert_select; EXECUTE prep_insert_select; EXECUTE prep_insert_select; EXECUTE prep_insert_select; --this insert will create an invalid tuple in source_data --so that future calls to prep_insert_select will fail INSERT INTO source_data VALUES('2000-01-02',-0.5); \set ON_ERROR_STOP 0 EXECUTE prep_insert_select; ERROR: new row for relation "_hyper_7_11_chunk" violates check constraint "prepared_test_value_check" EXECUTE prep_insert_select; ERROR: new row for relation "_hyper_7_11_chunk" violates check constraint "prepared_test_value_check" \set ON_ERROR_STOP 1 DELETE FROM source_data WHERE value <= 0; EXECUTE prep_insert_select; PREPARE prep_insert AS INSERT INTO prepared_test VALUES('2000-01-01',0.5) ON CONFLICT (time) DO UPDATE SET value = EXCLUDED.value; -- at some point PostgreSQL will turn the plan into a generic plan -- so we execute the prepared statement 10 times EXECUTE prep_insert; EXECUTE prep_insert; EXECUTE prep_insert; EXECUTE prep_insert; EXECUTE prep_insert; EXECUTE prep_insert; EXECUTE prep_insert; EXECUTE prep_insert; EXECUTE prep_insert; EXECUTE prep_insert; SELECT * FROM prepared_test; time | value ------------------------------+------- Sat Jan 01 00:00:00 2000 PST | 0.5 Mon Jan 01 00:00:00 2001 PST | 0.5 DELETE FROM prepared_test; -- test ON CONFLICT with functions CREATE OR REPLACE FUNCTION test_upsert(t timestamptz, v float) RETURNS VOID AS $sql$ BEGIN INSERT INTO prepared_test VALUES(t,v) ON CONFLICT (time) DO UPDATE SET value = EXCLUDED.value; END; $sql$ LANGUAGE PLPGSQL; -- at some point PostgreSQL will turn the plan into a generic plan -- so we execute the function 10 times SELECT counter,test_upsert('2000-01-01',0.5) FROM generate_series(1,10) AS g(counter); counter | test_upsert ---------+------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | SELECT * FROM prepared_test; time | value ------------------------------+------- Sat Jan 01 00:00:00 2000 PST | 0.5 DELETE FROM prepared_test; -- at some point PostgreSQL will turn the plan into a generic plan -- so we execute the function 10 times SELECT counter,test_upsert('2000-01-01',0.5) FROM generate_series(1,10) AS g(counter); counter | test_upsert ---------+------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | SELECT * FROM prepared_test; time | value ------------------------------+------- Sat Jan 01 00:00:00 2000 PST | 0.5 DELETE FROM prepared_test; -- run it again to ensure INSERT path is still working as well SELECT counter,test_upsert('2000-01-01',0.5) FROM generate_series(1,10) AS g(counter); counter | test_upsert ---------+------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | SELECT * FROM prepared_test; time | value ------------------------------+------- Sat Jan 01 00:00:00 2000 PST | 0.5 DELETE FROM prepared_test; -- test ON CONFLICT with functions CREATE OR REPLACE FUNCTION test_upsert2(t timestamptz, v float) RETURNS VOID AS $sql$ BEGIN INSERT INTO prepared_test VALUES(t,v) ON CONFLICT (time) DO UPDATE SET value = prepared_test.value + 1.0; END; $sql$ LANGUAGE PLPGSQL; -- at some point PostgreSQL will turn the plan into a generic plan -- so we execute the function 10 times SELECT counter,test_upsert2('2000-01-01',1.0) FROM generate_series(1,10) AS g(counter); counter | test_upsert2 ---------+-------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | SELECT * FROM prepared_test; time | value ------------------------------+------- Sat Jan 01 00:00:00 2000 PST | 10 ================================================ FILE: test/expected/util.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set ECHO errors item ------------------------------------ db_util_wizard=a/db_util_wizard db_util_wizard=ar/db_util_wizard db_util_wizard=a*/db_util_wizard db_util_wizard=a*r*/db_util_wizard [NULL] [NULL] [NULL] [NULL] =a*r*/db_util_wizard db_util_wizard=a*r*/0 SELECT _timescaledb_functions.align_to_bucket('5 minutes'::interval, rng::tstzrange) FROM data; align_to_bucket ----------------------------------------------------------------- ["Fri Apr 25 02:10:00 2025 PDT","Fri Apr 25 02:15:00 2025 PDT") ["Fri Apr 25 02:10:00 2025 PDT","Fri Apr 25 02:20:00 2025 PDT") ["Fri Apr 25 02:10:00 2025 PDT","Fri Apr 25 02:20:00 2025 PDT") \set ON_ERROR_STOP 0 SELECT _timescaledb_functions.align_to_bucket(null, null); ERROR: could not determine polymorphic type because input has type unknown SELECT _timescaledb_functions.align_to_bucket(null::interval, null::tstzrange); align_to_bucket ----------------- [NULL] SELECT _timescaledb_functions.align_to_bucket( null::interval, '["2025-04-25 11:10:00+02","2025-04-25 11:14:00+02"]'::tstzrange ); align_to_bucket ----------------- [NULL] \set ON_ERROR_STOP 1 SELECT typ, _timescaledb_functions.get_internal_time_min(typ), _timescaledb_functions.get_internal_time_max(typ) FROM (VALUES ('bigint'::regtype), ('int'::regtype), ('smallint'::regtype), ('timestamp'::regtype), ('timestamptz'::regtype), ('date'::regtype), (null::regtype) ) t(typ); typ | get_internal_time_min | get_internal_time_max -----------------------------+-----------------------+----------------------- bigint | -9223372036854775808 | 9223372036854775807 integer | -2147483648 | 2147483647 smallint | -32768 | 32767 timestamp without time zone | -210866803200000000 | 9223371331199999999 timestamp with time zone | -210866803200000000 | 9223371331199999999 date | -210866803200000000 | 9223371331199999999 [NULL] | [NULL] | [NULL] \set ON_ERROR_STOP 0 SELECT _timescaledb_functions.get_internal_time_min(0); ERROR: unsupported time type "-" SELECT _timescaledb_functions.get_internal_time_max(0); ERROR: unsupported time type "-" \set ON_ERROR_STOP 1 WITH tstzranges AS ( SELECT vid, rng::tstzrange, lower(rng::tstzrange) AS lower_ts, upper(rng::tstzrange) AS upper_ts FROM data ), usecranges AS ( SELECT vid, _timescaledb_functions.to_unix_microseconds(lower_ts) AS lower_usec, _timescaledb_functions.to_unix_microseconds(upper_ts) AS upper_usec FROM tstzranges ) SELECT _timescaledb_functions.make_multirange_from_internal_time(rng, lower_usec, upper_usec), _timescaledb_functions.make_range_from_internal_time(rng, lower_ts, upper_ts) FROM tstzranges join usecranges using (vid); make_multirange_from_internal_time | make_range_from_internal_time -------------------------------------------------------------------+----------------------------------------------------------------- {["Fri Apr 25 02:10:00 2025 PDT","Fri Apr 25 02:14:00 2025 PDT")} | ["Fri Apr 25 02:10:00 2025 PDT","Fri Apr 25 02:14:00 2025 PDT") {["Fri Apr 25 02:10:00 2025 PDT","Fri Apr 25 02:17:00 2025 PDT")} | ["Fri Apr 25 02:10:00 2025 PDT","Fri Apr 25 02:17:00 2025 PDT") {["Fri Apr 25 02:10:00 2025 PDT","Fri Apr 25 02:20:00 2025 PDT")} | ["Fri Apr 25 02:10:00 2025 PDT","Fri Apr 25 02:20:00 2025 PDT") WITH tsranges AS ( SELECT vid, rng::tsrange, lower(rng::tsrange) AS lower_ts, upper(rng::tsrange) AS upper_ts FROM data ), usecranges AS ( SELECT vid, _timescaledb_functions.to_unix_microseconds(lower_ts) AS lower_usec, _timescaledb_functions.to_unix_microseconds(upper_ts) AS upper_usec FROM tsranges ) SELECT _timescaledb_functions.make_multirange_from_internal_time(rng, lower_usec, upper_usec), _timescaledb_functions.make_range_from_internal_time(rng, lower_ts, upper_ts) FROM tsranges join usecranges using (vid); make_multirange_from_internal_time | make_range_from_internal_time -----------------------------------------------------------+--------------------------------------------------------- {["Fri Apr 25 18:10:00 2025","Fri Apr 25 18:14:00 2025")} | ["Fri Apr 25 11:10:00 2025","Fri Apr 25 11:14:00 2025") {["Fri Apr 25 18:10:00 2025","Fri Apr 25 18:17:00 2025")} | ["Fri Apr 25 11:10:00 2025","Fri Apr 25 11:17:00 2025") {["Fri Apr 25 18:10:00 2025","Fri Apr 25 18:20:00 2025")} | ["Fri Apr 25 11:10:00 2025","Fri Apr 25 11:20:00 2025") ================================================ FILE: test/expected/uuid.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- -- -- Test "time" partitioning on UUIDv7 -- -- CREATE TABLE uuid_events(id uuid primary key, device int, temp float); \set ON_ERROR_STOP 0 -- Test invalid interval type SELECT create_hypertable('uuid_events', 'id', chunk_time_interval => true); ERROR: invalid interval type for uuid dimension \set ON_ERROR_STOP 1 SELECT create_hypertable('uuid_events', 'id', chunk_time_interval => interval '1 day'); create_hypertable -------------------------- (1,public,uuid_events,t) SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'uuid_events'; time_interval --------------- @ 1 day -- -- Test that inserting boundary values generates the right constraints -- on chunks. -- -- First value with min time: 00000000-0000-7000-8000-000000000000 BEGIN; INSERT INTO uuid_events VALUES ('00000000-0000-7000-8000-000000000000', 1, 1.0); SELECT (test.show_constraints(ch)).* from show_chunks('uuid_events') ch; Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated ----------------------+------+---------+----------------------------------------------+----------------------------------------------------------------------------------------------------------------+------------+----------+----------- 1_1_uuid_events_pkey | p | {id} | _timescaledb_internal."1_1_uuid_events_pkey" | | f | f | t constraint_1 | c | {id} | - | ((id >= '00000000-0000-0000-0000-000000000000'::uuid) AND (id < '00000526-5c00-0000-0000-000000000000'::uuid)) | f | f | t SELECT uuid_timestamp(id), device, temp FROM uuid_events ORDER BY id; uuid_timestamp | device | temp ------------------------------+--------+------ Wed Dec 31 16:00:00 1969 PST | 1 | 1 -- Update v7 UUID to a v4 UUID that doesn't violate the chunk's range -- constraint. Currently we don't prevent this "loophole". UPDATE uuid_events SET id = '00000000-0001-4000-8000-000000000000' WHERE id = '00000000-0000-7000-8000-000000000000'; SELECT uuid_timestamp(id), device, temp FROM uuid_events ORDER BY id; uuid_timestamp | device | temp ----------------+--------+------ | 1 | 1 -- Update v7 UUID to a v4 that violates the chunk constraint: \set ON_ERROR_STOP 0 UPDATE uuid_events SET id = 'ffff0000-0000-4000-8000-000000000000' WHERE id = '00000000-0001-4000-8000-000000000000'; ERROR: new row for relation "_hyper_1_1_chunk" violates check constraint "constraint_1" \set ON_ERROR_STOP 1 ROLLBACK; -- Last value with min time: 00000000-0000-7fff-bfff-ffffffffffff BEGIN; INSERT INTO uuid_events VALUES ('00000000-0000-7fff-bfff-ffffffffffff', 1, 1.0); SELECT (test.show_constraints(ch)).* from show_chunks('uuid_events') ch; Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated ----------------------+------+---------+----------------------------------------------+----------------------------------------------------------------------------------------------------------------+------------+----------+----------- 2_2_uuid_events_pkey | p | {id} | _timescaledb_internal."2_2_uuid_events_pkey" | | f | f | t constraint_2 | c | {id} | - | ((id >= '00000000-0000-0000-0000-000000000000'::uuid) AND (id < '00000526-5c00-0000-0000-000000000000'::uuid)) | f | f | t ROLLBACK; -- First value with max time: ffffffff-ffff-7000-8000-000000000000 BEGIN; INSERT INTO uuid_events VALUES ('ffffffff-ffff-7000-8000-000000000000', 1, 1.0); SELECT (test.show_constraints(ch)).* from show_chunks('uuid_events') ch; Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated ----------------------+------+---------+----------------------------------------------+------------------------------------------------------+------------+----------+----------- 3_3_uuid_events_pkey | p | {id} | _timescaledb_internal."3_3_uuid_events_pkey" | | f | f | t constraint_3 | c | {id} | - | (id >= 'fffffed0-3000-0000-0000-000000000000'::uuid) | f | f | t ROLLBACK; -- (Max time with min value) + 1 BEGIN; INSERT INTO uuid_events VALUES ('ffffffff-ffff-7000-8000-000000000001', 1, 1.0); SELECT (test.show_constraints(ch)).* from show_chunks('uuid_events') ch; Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated ----------------------+------+---------+----------------------------------------------+------------------------------------------------------+------------+----------+----------- 4_4_uuid_events_pkey | p | {id} | _timescaledb_internal."4_4_uuid_events_pkey" | | f | f | t constraint_4 | c | {id} | - | (id >= 'fffffed0-3000-0000-0000-000000000000'::uuid) | f | f | t ROLLBACK; -- Last value with max time: ffffffff-ffff-7fff-bfff-ffffffffffff BEGIN; INSERT INTO uuid_events VALUES ('ffffffff-ffff-7fff-bfff-ffffffffffff', 1, 1.0); SELECT (test.show_constraints(ch)).* from show_chunks('uuid_events') ch; Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated ----------------------+------+---------+----------------------------------------------+------------------------------------------------------+------------+----------+----------- 5_5_uuid_events_pkey | p | {id} | _timescaledb_internal."5_5_uuid_events_pkey" | | f | f | t constraint_5 | c | {id} | - | (id >= 'fffffed0-3000-0000-0000-000000000000'::uuid) | f | f | t ROLLBACK; -- -- It is possible to generate UUIDs like follows, but the random -- generator used doesn't respect setseed() so used constant UUIDs for -- determinism. -- -- (_timescaledb_functions.uuid_v7_from_timestamptz('2025-01-01 01:00 PST'), 1, 1.0), -- (_timescaledb_functions.uuid_v7_from_timestamptz('2025-01-01 02:00 PST'), 2, 2.0), -- (_timescaledb_functions.uuid_v7_from_timestamptz('2025-01-02 01:00 PST'), 3, 3.0), -- (_timescaledb_functions.uuid_v7_from_timestamptz('2025-01-02 02:00 PST'), 4, 4.0), -- (_timescaledb_functions.uuid_v7_from_timestamptz('2025-01-03 03:00 PST'), 5, 5.0), -- (_timescaledb_functions.uuid_v7_from_timestamptz('2025-01-03 10:00 PST'), 6, 6.0); -- INSERT INTO uuid_events VALUES ('0194214e-cd00-7000-a9a7-63f1416dab45', 2, 2.0), ('01942117-de80-7000-8121-f12b2b69dd96', 1, 1.0), ('0194263e-3a80-7000-8f40-82c987b1bc1f', 3, 3.0), ('01942675-2900-7000-8db1-a98694b18785', 4, 4.0), ('01942bd2-7380-7000-9bc4-5f97443907b8', 5, 5.0), ('01942d52-f900-7000-866e-07d6404d53c1', 6, 6.0); SELECT * FROM show_chunks('uuid_events'); show_chunks ---------------------------------------- _timescaledb_internal._hyper_1_6_chunk _timescaledb_internal._hyper_1_7_chunk _timescaledb_internal._hyper_1_8_chunk SELECT (test.show_constraints(ch)).* from show_chunks('uuid_events') ch; Constraint | Type | Columns | Index | Expr | Deferrable | Deferred | Validated ----------------------+------+---------+----------------------------------------------+----------------------------------------------------------------------------------------------------------------+------------+----------+----------- 6_6_uuid_events_pkey | p | {id} | _timescaledb_internal."6_6_uuid_events_pkey" | | f | f | t constraint_6 | c | {id} | - | ((id >= '01941f29-7c00-0000-0000-000000000000'::uuid) AND (id < '0194244f-d800-0000-0000-000000000000'::uuid)) | f | f | t 7_7_uuid_events_pkey | p | {id} | _timescaledb_internal."7_7_uuid_events_pkey" | | f | f | t constraint_7 | c | {id} | - | ((id >= '0194244f-d800-0000-0000-000000000000'::uuid) AND (id < '01942976-3400-0000-0000-000000000000'::uuid)) | f | f | t 8_8_uuid_events_pkey | p | {id} | _timescaledb_internal."8_8_uuid_events_pkey" | | f | f | t constraint_8 | c | {id} | - | ((id >= '01942976-3400-0000-0000-000000000000'::uuid) AND (id < '01942e9c-9000-0000-0000-000000000000'::uuid)) | f | f | t SELECT id, device, temp FROM uuid_events; id | device | temp --------------------------------------+--------+------ 0194214e-cd00-7000-a9a7-63f1416dab45 | 2 | 2 01942117-de80-7000-8121-f12b2b69dd96 | 1 | 1 0194263e-3a80-7000-8f40-82c987b1bc1f | 3 | 3 01942675-2900-7000-8db1-a98694b18785 | 4 | 4 01942bd2-7380-7000-9bc4-5f97443907b8 | 5 | 5 01942d52-f900-7000-866e-07d6404d53c1 | 6 | 6 SELECT uuid_timestamp(id), device, temp FROM uuid_events; uuid_timestamp | device | temp ------------------------------+--------+------ Wed Jan 01 02:00:00 2025 PST | 2 | 2 Wed Jan 01 01:00:00 2025 PST | 1 | 1 Thu Jan 02 01:00:00 2025 PST | 3 | 3 Thu Jan 02 02:00:00 2025 PST | 4 | 4 Fri Jan 03 03:00:00 2025 PST | 5 | 5 Fri Jan 03 10:00:00 2025 PST | 6 | 6 SELECT uuid_timestamp(id), device, temp FROM uuid_events ORDER BY id; uuid_timestamp | device | temp ------------------------------+--------+------ Wed Jan 01 01:00:00 2025 PST | 1 | 1 Wed Jan 01 02:00:00 2025 PST | 2 | 2 Thu Jan 02 01:00:00 2025 PST | 3 | 3 Thu Jan 02 02:00:00 2025 PST | 4 | 4 Fri Jan 03 03:00:00 2025 PST | 5 | 5 Fri Jan 03 10:00:00 2025 PST | 6 | 6 SELECT _timescaledb_functions.to_timestamp(range_start) AS range_start, _timescaledb_functions.to_timestamp(range_end) AS range_end FROM _timescaledb_catalog.dimension_slice ds JOIN _timescaledb_catalog.dimension d ON (ds.dimension_id = d.id) JOIN _timescaledb_catalog.hypertable h ON (d.hypertable_id = h.id) WHERE h.table_name = 'uuid_events'; range_start | range_end ------------------------------+------------------------------ Tue Dec 31 16:00:00 2024 PST | Wed Jan 01 16:00:00 2025 PST Wed Jan 01 16:00:00 2025 PST | Thu Jan 02 16:00:00 2025 PST Thu Jan 02 16:00:00 2025 PST | Fri Jan 03 16:00:00 2025 PST SELECT _timescaledb_functions.to_timestamp(range_start) AS chunk_range_start, _timescaledb_functions.to_timestamp(range_end) AS chunk_range_end FROM _timescaledb_catalog.dimension_slice ds JOIN _timescaledb_catalog.dimension d ON (ds.dimension_id = d.id) JOIN _timescaledb_catalog.hypertable h ON (d.hypertable_id = h.id) WHERE h.table_name = 'uuid_events' LIMIT 1 OFFSET 1 \gset -- Test that chunk exclusion on uuidv7 column works SELECT :'chunk_range_start', to_uuidv7_boundary(:'chunk_range_start'); ?column? | to_uuidv7_boundary ------------------------------+-------------------------------------- Wed Jan 01 16:00:00 2025 PST | 0194244f-d800-7000-8000-000000000000 -- Exclude all but one chunk EXPLAIN (verbose, buffers off, costs off, timing off) SELECT uuid_timestamp(id), device, temp FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_start'); --- QUERY PLAN --- Result Output: uuid_timestamp(_hyper_1_6_chunk.id), _hyper_1_6_chunk.device, _hyper_1_6_chunk.temp -> Seq Scan on _timescaledb_internal._hyper_1_6_chunk Output: _hyper_1_6_chunk.id, _hyper_1_6_chunk.device, _hyper_1_6_chunk.temp SELECT uuid_timestamp(id), device, temp FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_start') ORDER BY id; uuid_timestamp | device | temp ------------------------------+--------+------ Wed Jan 01 01:00:00 2025 PST | 1 | 1 Wed Jan 01 02:00:00 2025 PST | 2 | 2 -- Exclude only one chunk. Add ordering (DESC) EXPLAIN (verbose, buffers off, costs off, timing off) SELECT uuid_timestamp(id), device, temp FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_end') ORDER BY id DESC; --- QUERY PLAN --- Result Output: uuid_timestamp(uuid_events.id), uuid_events.device, uuid_events.temp, uuid_events.id -> Custom Scan (ChunkAppend) on public.uuid_events Output: uuid_events.id, uuid_events.device, uuid_events.temp Order: uuid_events.id DESC Startup Exclusion: false Runtime Exclusion: false -> Index Scan Backward using "7_7_uuid_events_pkey" on _timescaledb_internal._hyper_1_7_chunk Output: _hyper_1_7_chunk.id, _hyper_1_7_chunk.device, _hyper_1_7_chunk.temp -> Index Scan Backward using "6_6_uuid_events_pkey" on _timescaledb_internal._hyper_1_6_chunk Output: _hyper_1_6_chunk.id, _hyper_1_6_chunk.device, _hyper_1_6_chunk.temp SELECT uuid_timestamp(id), device, temp FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_end') ORDER BY id DESC; uuid_timestamp | device | temp ------------------------------+--------+------ Thu Jan 02 02:00:00 2025 PST | 4 | 4 Thu Jan 02 01:00:00 2025 PST | 3 | 3 Wed Jan 01 02:00:00 2025 PST | 2 | 2 Wed Jan 01 01:00:00 2025 PST | 1 | 1 SELECT time_bucket('1 day', id) AS day, avg(temp) FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_end') GROUP BY id ORDER BY id DESC; day | avg ------------------------------+----- Wed Jan 01 16:00:00 2025 PST | 4 Wed Jan 01 16:00:00 2025 PST | 3 Tue Dec 31 16:00:00 2024 PST | 2 Tue Dec 31 16:00:00 2024 PST | 1 SELECT time_bucket('1 day', uuid_timestamp(id)) AS day, avg(temp) FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_end') GROUP BY id ORDER BY id DESC; day | avg ------------------------------+----- Wed Jan 01 16:00:00 2025 PST | 4 Wed Jan 01 16:00:00 2025 PST | 3 Tue Dec 31 16:00:00 2024 PST | 2 Tue Dec 31 16:00:00 2024 PST | 1 -- Bucket with offset SELECT time_bucket('1 day', id, "offset" => '1 week') AS day, avg(temp) FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_end') GROUP BY id ORDER BY id DESC; day | avg ------------------------------+----- Wed Jan 01 16:00:00 2025 PST | 4 Wed Jan 01 16:00:00 2025 PST | 3 Tue Dec 31 16:00:00 2024 PST | 2 Tue Dec 31 16:00:00 2024 PST | 1 -- Bucket with origin SELECT time_bucket('1 day', id, timestamptz '2000-01-01 00:00') AS day, avg(temp) FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_end') GROUP BY id ORDER BY id DESC; day | avg ------------------------------+----- Thu Jan 02 00:00:00 2025 PST | 4 Thu Jan 02 00:00:00 2025 PST | 3 Wed Jan 01 00:00:00 2025 PST | 2 Wed Jan 01 00:00:00 2025 PST | 1 -- Bucket with time zone SELECT time_bucket('1 day', id, 'Europe/Stockholm') at time zone 'Europe/Stockholm' as day, avg(temp) FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_end') GROUP BY id ORDER BY id DESC; day | avg --------------------------+----- Thu Jan 02 00:00:00 2025 | 4 Thu Jan 02 00:00:00 2025 | 3 Wed Jan 01 00:00:00 2025 | 2 Wed Jan 01 00:00:00 2025 | 1 -- Test NULL arguments SELECT time_bucket('1 day', id, 'Europe/Stockholm'::text, NULL::timestamptz) AS day, avg(temp) FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_end') GROUP BY id ORDER BY id DESC; day | avg ------------------------------+----- Wed Jan 01 15:00:00 2025 PST | 4 Wed Jan 01 15:00:00 2025 PST | 3 Tue Dec 31 15:00:00 2024 PST | 2 Tue Dec 31 15:00:00 2024 PST | 1 SELECT time_bucket('1 day', id, 'Europe/Stockholm'::text, '2000-01-01 00:00'::timestamptz, NULL::interval) AS day, avg(temp) FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_end') GROUP BY id ORDER BY id DESC; day | avg ------------------------------+----- Thu Jan 02 00:00:00 2025 PST | 4 Thu Jan 02 00:00:00 2025 PST | 3 Wed Jan 01 00:00:00 2025 PST | 2 Wed Jan 01 00:00:00 2025 PST | 1 -- Test UUID time_bucket in WHERE clause. Note that there is currently no chunk -- exclusion when using time_bucket() in the WHERE clause qual. This requires -- special handling of the time_bucket() transform optimizatios for different -- operators. EXPLAIN (COSTS OFF) SELECT time_bucket('1 day', id) AS day, avg(temp) FROM uuid_events WHERE time_bucket('1 day', id) >= :'chunk_range_end' GROUP BY id ORDER BY id DESC; --- QUERY PLAN --- Sort Sort Key: _hyper_1_8_chunk.id DESC -> HashAggregate Group Key: _hyper_1_8_chunk.id -> Seq Scan on _hyper_1_8_chunk Filter: (time_bucket('@ 1 day'::interval, id) >= 'Thu Jan 02 16:00:00 2025 PST'::timestamp with time zone) SELECT time_bucket('1 day', id) AS day, avg(temp) FROM uuid_events WHERE time_bucket('1 day', id) >= :'chunk_range_end' GROUP BY id ORDER BY id DESC; day | avg ------------------------------+----- Thu Jan 02 16:00:00 2025 PST | 6 Thu Jan 02 16:00:00 2025 PST | 5 EXPLAIN (COSTS OFF) SELECT time_bucket('1 day', id) AS day, avg(temp) FROM uuid_events WHERE time_bucket('1 day', id) > :'chunk_range_start' GROUP BY id ORDER BY id DESC; --- QUERY PLAN --- GroupAggregate Group Key: uuid_events.id -> Custom Scan (ChunkAppend) on uuid_events Order: uuid_events.id DESC -> Index Scan Backward using "8_8_uuid_events_pkey" on _hyper_1_8_chunk Filter: (time_bucket('@ 1 day'::interval, id) > 'Wed Jan 01 16:00:00 2025 PST'::timestamp with time zone) -> Index Scan Backward using "7_7_uuid_events_pkey" on _hyper_1_7_chunk Index Cond: (id > '0194244f-d800-7000-8000-000000000000'::uuid) Filter: (time_bucket('@ 1 day'::interval, id) > 'Wed Jan 01 16:00:00 2025 PST'::timestamp with time zone) SELECT time_bucket('1 day', id) AS day, avg(temp) FROM uuid_events WHERE time_bucket('1 day', id) > :'chunk_range_start' GROUP BY id ORDER BY id DESC; day | avg ------------------------------+----- Thu Jan 02 16:00:00 2025 PST | 6 Thu Jan 02 16:00:00 2025 PST | 5 EXPLAIN (COSTS OFF) SELECT time_bucket('1 day', id) AS day, avg(temp) FROM uuid_events WHERE time_bucket('1 day', id) < :'chunk_range_end' GROUP BY id ORDER BY id DESC; --- QUERY PLAN --- Sort Sort Key: uuid_events.id DESC -> HashAggregate Group Key: uuid_events.id -> Append -> Seq Scan on _hyper_1_7_chunk Filter: (time_bucket('@ 1 day'::interval, id) < 'Thu Jan 02 16:00:00 2025 PST'::timestamp with time zone) -> Seq Scan on _hyper_1_6_chunk Filter: (time_bucket('@ 1 day'::interval, id) < 'Thu Jan 02 16:00:00 2025 PST'::timestamp with time zone) SELECT time_bucket('1 day', id) AS day, avg(temp) FROM uuid_events WHERE time_bucket('1 day', id) < :'chunk_range_end' GROUP BY id ORDER BY id DESC; day | avg ------------------------------+----- Wed Jan 01 16:00:00 2025 PST | 4 Wed Jan 01 16:00:00 2025 PST | 3 Tue Dec 31 16:00:00 2024 PST | 2 Tue Dec 31 16:00:00 2024 PST | 1 EXPLAIN (COSTS OFF) SELECT time_bucket('1 day', id) AS day, avg(temp) FROM uuid_events WHERE time_bucket('1 day', id) <= :'chunk_range_start' GROUP BY id ORDER BY id DESC; --- QUERY PLAN --- Sort Sort Key: uuid_events.id DESC -> HashAggregate Group Key: uuid_events.id -> Append -> Index Scan Backward using "8_8_uuid_events_pkey" on _hyper_1_8_chunk Index Cond: (id <= '01942976-3400-7000-8000-000000000000'::uuid) Filter: (time_bucket('@ 1 day'::interval, id) <= 'Wed Jan 01 16:00:00 2025 PST'::timestamp with time zone) -> Seq Scan on _hyper_1_7_chunk Filter: (time_bucket('@ 1 day'::interval, id) <= 'Wed Jan 01 16:00:00 2025 PST'::timestamp with time zone) -> Seq Scan on _hyper_1_6_chunk Filter: (time_bucket('@ 1 day'::interval, id) <= 'Wed Jan 01 16:00:00 2025 PST'::timestamp with time zone) SELECT time_bucket('1 day', id) AS day, avg(temp) FROM uuid_events WHERE time_bucket('1 day', id) <= :'chunk_range_start' GROUP BY id ORDER BY id DESC; day | avg ------------------------------+----- Wed Jan 01 16:00:00 2025 PST | 4 Wed Jan 01 16:00:00 2025 PST | 3 Tue Dec 31 16:00:00 2024 PST | 2 Tue Dec 31 16:00:00 2024 PST | 1 -- Test time_bucket on non-v7 UUID \set ON_ERROR_STOP 0 SELECT time_bucket('1 day', 'ffff0000-0000-4000-8000-000000000000'::uuid); ERROR: not a version 7 UUID: ffff0000-0000-4000-8000-000000000000 \set ON_ERROR_STOP 1 CREATE VIEW chunk_ranges AS SELECT chunk_name, range_start, range_end FROM timescaledb_information.chunks WHERE hypertable_name = 'uuid_events'; SELECT * FROM chunk_ranges; chunk_name | range_start | range_end ------------------+------------------------------+------------------------------ _hyper_1_6_chunk | Tue Dec 31 16:00:00 2024 PST | Wed Jan 01 16:00:00 2025 PST _hyper_1_7_chunk | Wed Jan 01 16:00:00 2025 PST | Thu Jan 02 16:00:00 2025 PST _hyper_1_8_chunk | Thu Jan 02 16:00:00 2025 PST | Fri Jan 03 16:00:00 2025 PST SELECT show_chunks('uuid_events', older_than => INTERVAL '1 day'); show_chunks ---------------------------------------- _timescaledb_internal._hyper_1_6_chunk _timescaledb_internal._hyper_1_7_chunk _timescaledb_internal._hyper_1_8_chunk SELECT show_chunks('uuid_events', older_than => '2025-01-02'); show_chunks ---------------------------------------- _timescaledb_internal._hyper_1_6_chunk SELECT show_chunks('uuid_events', newer_than => '2025-01-02'); show_chunks ---------------------------------------- _timescaledb_internal._hyper_1_8_chunk SELECT drop_chunks('uuid_events', older_than => '2025-01-02'); drop_chunks ---------------------------------------- _timescaledb_internal._hyper_1_6_chunk SELECT show_chunks('uuid_events'); show_chunks ---------------------------------------- _timescaledb_internal._hyper_1_7_chunk _timescaledb_internal._hyper_1_8_chunk -- Insert non-v7 UUIDs \set ON_ERROR_STOP 0 INSERT INTO uuid_events SELECT 'a8961135-cd89-4c4b-aa05-79df642407dd', 5, 5.0; ERROR: a8961135-cd89-4c4b-aa05-79df642407dd is not a version 7 UUID \set ON_ERROR_STOP 1 -- Insert as v7 UUID and later change to non-v7 to show effect on show_chunks() -- and drop_chunks() INSERT INTO uuid_events SELECT 'a8961135-cd89-7000-aa05-79df642407dd', 5, 5.0; SELECT * FROM chunk_ranges; chunk_name | range_start | range_end ------------------+------------------------------+------------------------------ _hyper_1_7_chunk | Wed Jan 01 16:00:00 2025 PST | Thu Jan 02 16:00:00 2025 PST _hyper_1_8_chunk | Thu Jan 02 16:00:00 2025 PST | Fri Jan 03 16:00:00 2025 PST _hyper_1_9_chunk | Sun Nov 26 16:00:00 7843 PST | Mon Nov 27 16:00:00 7843 PST UPDATE uuid_events SET id = 'a8961135-cd89-4c4b-aa05-79df642407dd' WHERE id = 'a8961135-cd89-7000-aa05-79df642407dd'; SELECT show_chunks('uuid_events', newer_than => '2025-01-02'); show_chunks ---------------------------------------- _timescaledb_internal._hyper_1_8_chunk _timescaledb_internal._hyper_1_9_chunk SELECT drop_chunks('uuid_events', newer_than => '2025-01-02'); drop_chunks ---------------------------------------- _timescaledb_internal._hyper_1_8_chunk _timescaledb_internal._hyper_1_9_chunk SELECT * FROM chunk_ranges; chunk_name | range_start | range_end ------------------+------------------------------+------------------------------ _hyper_1_7_chunk | Wed Jan 01 16:00:00 2025 PST | Thu Jan 02 16:00:00 2025 PST INSERT INTO uuid_events SELECT to_uuidv7(now()), 6, 6.0; SELECT show_chunks('uuid_events', newer_than => INTERVAL '2 months'); show_chunks ----------------------------------------- _timescaledb_internal._hyper_1_10_chunk SELECT drop_chunks('uuid_events', newer_than => INTERVAL '2 months'); drop_chunks ----------------------------------------- _timescaledb_internal._hyper_1_10_chunk SELECT show_chunks('uuid_events', newer_than => INTERVAL '2 months'); show_chunks ------------- DROP TABLE uuid_events; BEGIN; -- Test UUID partition when using CREATE TABLE ... WITH CREATE TABLE IF NOT EXISTS events ( event_id UUID NOT NULL, entity_id VARCHAR(100) NOT NULL, ts TIMESTAMPTZ NOT NULL, event_type VARCHAR(100) NOT NULL, metadata JSONB, PRIMARY KEY (event_id) ) WITH ( tsdb.hypertable, tsdb.partition_column='event_id', tsdb.chunk_interval='2 hours' ); -- Verify that the chunk time interval is two hours SELECT ((interval_length/1000000)/60)/60 AS hours FROM _timescaledb_catalog.dimension d JOIN _timescaledb_catalog.hypertable h ON (h.id = d.hypertable_id) WHERE h.table_name='events'; hours ------- 2 ROLLBACK; -- Test a different interval BEGIN; CREATE TABLE IF NOT EXISTS events ( event_id UUID NOT NULL, entity_id VARCHAR(100) NOT NULL, ts TIMESTAMPTZ NOT NULL, event_type VARCHAR(100) NOT NULL, metadata JSONB, PRIMARY KEY (event_id) ) WITH ( tsdb.hypertable, tsdb.partition_column='event_id', tsdb.chunk_interval='2 months' ); -- Verify that the chunk time interval is two hours SELECT (((interval_length/1000000)/60)/60)/24 AS days FROM _timescaledb_catalog.dimension d JOIN _timescaledb_catalog.hypertable h ON (h.id = d.hypertable_id) WHERE h.table_name='events'; days ------ 60 ROLLBACK; -- Verify same behavior without CREATE TABLE WITH: BEGIN; CREATE TABLE IF NOT EXISTS events ( event_id UUID NOT NULL, entity_id VARCHAR(100) NOT NULL, ts TIMESTAMPTZ NOT NULL, event_type VARCHAR(100) NOT NULL, metadata JSONB, PRIMARY KEY (event_id) ); SELECT create_hypertable('events', 'event_id', chunk_time_interval => interval '2 hours'); WARNING: column type "character varying" used for "entity_id" does not follow best practices WARNING: column type "character varying" used for "event_type" does not follow best practices create_hypertable --------------------- (4,public,events,t) -- Verify that the chunk time interval is two hours SELECT ((interval_length/1000000)/60)/60 AS hours FROM _timescaledb_catalog.dimension d JOIN _timescaledb_catalog.hypertable h ON (h.id = d.hypertable_id) WHERE h.table_name='events'; hours ------- 2 ROLLBACK; BEGIN; CREATE TABLE IF NOT EXISTS events ( event_id UUID NOT NULL, entity_id VARCHAR(100) NOT NULL, ts TIMESTAMPTZ NOT NULL, event_type VARCHAR(100) NOT NULL, metadata JSONB, PRIMARY KEY (event_id) ); SELECT create_hypertable('events', 'event_id', chunk_time_interval => interval '2 months'); WARNING: column type "character varying" used for "entity_id" does not follow best practices WARNING: column type "character varying" used for "event_type" does not follow best practices create_hypertable --------------------- (5,public,events,t) -- Verify that the chunk time interval is two hours SELECT (((interval_length/1000000)/60)/60)/24 AS days FROM _timescaledb_catalog.dimension d JOIN _timescaledb_catalog.hypertable h ON (h.id = d.hypertable_id) WHERE h.table_name='events'; days ------ 60 ROLLBACK; ================================================ FILE: test/expected/vacuum.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE vacuum_test(time timestamp, temp float); -- create hypertable with three chunks SELECT create_hypertable('vacuum_test', 'time', chunk_time_interval => 2628000000000, create_default_indexes => false); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------- (1,public,vacuum_test,t) INSERT INTO vacuum_test VALUES ('2017-01-20T16:00:01', 17.5), ('2017-01-21T16:00:01', 19.1), ('2017-04-20T16:00:01', 89.5), ('2017-04-21T16:00:01', 17.1), ('2017-06-20T16:00:01', 18.5), ('2017-06-21T16:00:01', 11.0); -- no stats SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = '_timescaledb_internal' AND tablename LIKE '_hyper_%_chunk' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); tablename | attname | histogram_bounds | n_distinct -----------+---------+------------------+------------ SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = 'public' AND tablename LIKE 'vacuum_test' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); tablename | attname | histogram_bounds | n_distinct -----------+---------+------------------+------------ VACUUM ANALYZE vacuum_test; -- stats should exist for all three chunks SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = '_timescaledb_internal' AND tablename LIKE '_hyper_%_chunk' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); tablename | attname | histogram_bounds | n_distinct ------------------+---------+---------------------------------------------------------+------------ _hyper_1_1_chunk | temp | {17.5,19.1} | -1 _hyper_1_1_chunk | time | {"Fri Jan 20 16:00:01 2017","Sat Jan 21 16:00:01 2017"} | -1 _hyper_1_2_chunk | temp | {17.1,89.5} | -1 _hyper_1_2_chunk | time | {"Thu Apr 20 16:00:01 2017","Fri Apr 21 16:00:01 2017"} | -1 _hyper_1_3_chunk | temp | {11,18.5} | -1 _hyper_1_3_chunk | time | {"Tue Jun 20 16:00:01 2017","Wed Jun 21 16:00:01 2017"} | -1 -- stats should exist on parent hypertable SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = 'public' AND tablename LIKE 'vacuum_test' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); tablename | attname | histogram_bounds | n_distinct -------------+---------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------ vacuum_test | temp | {11,17.1,17.5,18.5,19.1,89.5} | -1 vacuum_test | time | {"Fri Jan 20 16:00:01 2017","Sat Jan 21 16:00:01 2017","Thu Apr 20 16:00:01 2017","Fri Apr 21 16:00:01 2017","Tue Jun 20 16:00:01 2017","Wed Jun 21 16:00:01 2017"} | -1 DROP TABLE vacuum_test; --test plain analyze (no_vacuum) CREATE TABLE analyze_test(time timestamp, temp float); SELECT create_hypertable('analyze_test', 'time', chunk_time_interval => 2628000000000, create_default_indexes => false); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable --------------------------- (2,public,analyze_test,t) INSERT INTO analyze_test VALUES ('2017-01-20T16:00:01', 17.5), ('2017-01-21T16:00:01', 19.1), ('2017-04-20T16:00:01', 89.5), ('2017-04-21T16:00:01', 17.1), ('2017-06-20T16:00:01', 18.5), ('2017-06-21T16:00:01', 11.0); -- no stats SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = '_timescaledb_internal' AND tablename LIKE '_hyper_%_chunk' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); tablename | attname | histogram_bounds | n_distinct -----------+---------+------------------+------------ SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = 'public' AND tablename LIKE 'analyze_test' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); tablename | attname | histogram_bounds | n_distinct -----------+---------+------------------+------------ ANALYZE analyze_test; -- stats should exist for all three chunks SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = '_timescaledb_internal' AND tablename LIKE '_hyper_%_chunk' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); tablename | attname | histogram_bounds | n_distinct ------------------+---------+---------------------------------------------------------+------------ _hyper_2_4_chunk | temp | {17.5,19.1} | -1 _hyper_2_4_chunk | time | {"Fri Jan 20 16:00:01 2017","Sat Jan 21 16:00:01 2017"} | -1 _hyper_2_5_chunk | temp | {17.1,89.5} | -1 _hyper_2_5_chunk | time | {"Thu Apr 20 16:00:01 2017","Fri Apr 21 16:00:01 2017"} | -1 _hyper_2_6_chunk | temp | {11,18.5} | -1 _hyper_2_6_chunk | time | {"Tue Jun 20 16:00:01 2017","Wed Jun 21 16:00:01 2017"} | -1 -- stats should exist on parent hypertable SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = 'public' AND tablename LIKE 'analyze_test' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); tablename | attname | histogram_bounds | n_distinct --------------+---------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------ analyze_test | temp | {11,17.1,17.5,18.5,19.1,89.5} | -1 analyze_test | time | {"Fri Jan 20 16:00:01 2017","Sat Jan 21 16:00:01 2017","Thu Apr 20 16:00:01 2017","Fri Apr 21 16:00:01 2017","Tue Jun 20 16:00:01 2017","Wed Jun 21 16:00:01 2017"} | -1 DROP TABLE analyze_test; -- Run vacuum on a normal (non-hypertable) table CREATE TABLE vacuum_norm(time timestamp, temp float); INSERT INTO vacuum_norm VALUES ('2017-01-20T09:00:01', 17.5), ('2017-01-21T09:00:01', 19.1), ('2017-04-20T09:00:01', 89.5), ('2017-04-21T09:00:01', 17.1), ('2017-06-20T09:00:01', 18.5), ('2017-06-21T09:00:01', 11.0); VACUUM ANALYZE vacuum_norm; DROP TABLE vacuum_norm; --Similar to normal vacuum tests, but PG11 introduced ability to vacuum multiple tables at once, we make sure that works for hypertables as well. CREATE TABLE vacuum_test(time timestamp, temp float); -- create hypertable with three chunks SELECT create_hypertable('vacuum_test', 'time', chunk_time_interval => 2628000000000, create_default_indexes => false); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------- (3,public,vacuum_test,t) INSERT INTO vacuum_test VALUES ('2017-01-20T16:00:01', 17.5), ('2017-01-21T16:00:01', 19.1), ('2017-04-20T16:00:01', 89.5), ('2017-04-21T16:00:01', 17.1), ('2017-06-20T16:00:01', 18.5), ('2017-06-21T16:00:01', 11.0); CREATE TABLE analyze_test(time timestamp, temp float); SELECT create_hypertable('analyze_test', 'time', chunk_time_interval => 2628000000000, create_default_indexes => false); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable --------------------------- (4,public,analyze_test,t) INSERT INTO analyze_test VALUES ('2017-01-20T16:00:01', 17.5), ('2017-01-21T16:00:01', 19.1), ('2017-04-20T16:00:01', 89.5), ('2017-04-21T16:00:01', 17.1), ('2017-06-20T16:00:01', 18.5), ('2017-06-21T16:00:01', 11.0); CREATE TABLE vacuum_norm(time timestamp, temp float); INSERT INTO vacuum_norm VALUES ('2017-01-20T09:00:01', 17.5), ('2017-01-21T09:00:01', 19.1), ('2017-04-20T09:00:01', 89.5), ('2017-04-21T09:00:01', 17.1), ('2017-06-20T09:00:01', 18.5), ('2017-06-21T09:00:01', 11.0); -- no stats SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = '_timescaledb_internal' AND tablename LIKE '_hyper_%_chunk' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); tablename | attname | histogram_bounds | n_distinct -----------+---------+------------------+------------ SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = 'public' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); tablename | attname | histogram_bounds | n_distinct -----------+---------+------------------+------------ VACUUM ANALYZE vacuum_norm, vacuum_test, analyze_test; -- stats should exist for all 6 chunks SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = '_timescaledb_internal' AND tablename LIKE '_hyper_%_chunk' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); tablename | attname | histogram_bounds | n_distinct -------------------+---------+---------------------------------------------------------+------------ _hyper_3_7_chunk | temp | {17.5,19.1} | -1 _hyper_3_7_chunk | time | {"Fri Jan 20 16:00:01 2017","Sat Jan 21 16:00:01 2017"} | -1 _hyper_3_8_chunk | temp | {17.1,89.5} | -1 _hyper_3_8_chunk | time | {"Thu Apr 20 16:00:01 2017","Fri Apr 21 16:00:01 2017"} | -1 _hyper_3_9_chunk | temp | {11,18.5} | -1 _hyper_3_9_chunk | time | {"Tue Jun 20 16:00:01 2017","Wed Jun 21 16:00:01 2017"} | -1 _hyper_4_10_chunk | temp | {17.5,19.1} | -1 _hyper_4_10_chunk | time | {"Fri Jan 20 16:00:01 2017","Sat Jan 21 16:00:01 2017"} | -1 _hyper_4_11_chunk | temp | {17.1,89.5} | -1 _hyper_4_11_chunk | time | {"Thu Apr 20 16:00:01 2017","Fri Apr 21 16:00:01 2017"} | -1 _hyper_4_12_chunk | temp | {11,18.5} | -1 _hyper_4_12_chunk | time | {"Tue Jun 20 16:00:01 2017","Wed Jun 21 16:00:01 2017"} | -1 -- stats should exist on parent hypertable and normal table SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = 'public' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); tablename | attname | histogram_bounds | n_distinct --------------+---------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------ analyze_test | temp | {11,17.1,17.5,18.5,19.1,89.5} | -1 analyze_test | time | {"Fri Jan 20 16:00:01 2017","Sat Jan 21 16:00:01 2017","Thu Apr 20 16:00:01 2017","Fri Apr 21 16:00:01 2017","Tue Jun 20 16:00:01 2017","Wed Jun 21 16:00:01 2017"} | -1 vacuum_norm | temp | {11,17.1,17.5,18.5,19.1,89.5} | -1 vacuum_norm | time | {"Fri Jan 20 09:00:01 2017","Sat Jan 21 09:00:01 2017","Thu Apr 20 09:00:01 2017","Fri Apr 21 09:00:01 2017","Tue Jun 20 09:00:01 2017","Wed Jun 21 09:00:01 2017"} | -1 vacuum_test | temp | {11,17.1,17.5,18.5,19.1,89.5} | -1 vacuum_test | time | {"Fri Jan 20 16:00:01 2017","Sat Jan 21 16:00:01 2017","Thu Apr 20 16:00:01 2017","Fri Apr 21 16:00:01 2017","Tue Jun 20 16:00:01 2017","Wed Jun 21 16:00:01 2017"} | -1 ================================================ FILE: test/expected/vacuum_parallel.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- PG13 introduced parallel VACUUM functionality. It gets invoked when a table -- has two or more indexes on it. Read up more at -- https://www.postgresql.org/docs/13/sql-vacuum.html#PARALLEL CREATE TABLE vacuum_test(time timestamp NOT NULL, temp1 float, temp2 int); -- create hypertable -- we create chunks in public schema cause otherwise we would need -- elevated privileges to create indexes directly SELECT create_hypertable('vacuum_test', 'time', create_default_indexes => false, associated_schema_name => 'public'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------- (1,public,vacuum_test,t) -- parallel vacuum needs the index size to be larger than min_parallel_index_scan_size to kick in SET min_parallel_index_scan_size TO 0; INSERT INTO vacuum_test SELECT TIMESTAMP 'epoch' + (i * INTERVAL '4h'), i, i+1 FROM generate_series(1, 100) as T(i); -- create indexes on the temp columns -- we create indexes manually because otherwise vacuum verbose output -- would be different between 13.2 and 13.3+ -- 13.2 would try to vacuum the parent table index too while 13.3+ wouldn't CREATE INDEX ON _hyper_1_1_chunk(time); CREATE INDEX ON _hyper_1_1_chunk(temp1); CREATE INDEX ON _hyper_1_1_chunk(temp2); CREATE INDEX ON _hyper_1_2_chunk(time); CREATE INDEX ON _hyper_1_2_chunk(temp1); CREATE INDEX ON _hyper_1_2_chunk(temp2); CREATE INDEX ON _hyper_1_3_chunk(time); CREATE INDEX ON _hyper_1_3_chunk(temp1); CREATE INDEX ON _hyper_1_3_chunk(temp2); -- INSERT only will not trigger vacuum on indexes for PG13.3+ UPDATE vacuum_test SET time = time + '1s'::interval, temp1 = random(), temp2 = random(); -- we should see two parallel workers for each chunk VACUUM (PARALLEL 3) vacuum_test; DROP TABLE vacuum_test; ================================================ FILE: test/expected/version.out ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test that get_os_info returns 3 x text select pg_typeof(sysname) AS sysname_type,pg_typeof(version) AS version_type,pg_typeof(release) AS release_type from _timescaledb_functions.get_os_info(); sysname_type | version_type | release_type --------------+--------------+-------------- text | text | text ================================================ FILE: test/isolation/CMakeLists.txt ================================================ add_subdirectory(specs) ================================================ FILE: test/isolation/expected/concurrent_add_dimension.out ================================================ unused step name: s1_wp_enable unused step name: s1_wp_release Parsed test spec with 3 sessions starting permutation: s3_wp_enable s1_add_dimension s2_add_dimension s3_wp_release s3_query step s3_wp_enable: SELECT debug_waitpoint_enable('add_dimension_ht_lock'); debug_waitpoint_enable ---------------------- step s1_add_dimension: SELECT column_name FROM add_dimension('dim_test', 'device', 2); <waiting ...> step s2_add_dimension: SELECT column_name FROM add_dimension('dim_test', 'device', 1); <waiting ...> step s3_wp_release: SELECT debug_waitpoint_release('add_dimension_ht_lock'); debug_waitpoint_release ----------------------- step s1_add_dimension: <... completed> column_name ----------- device step s2_add_dimension: <... completed> ERROR: column "device" is already a dimension step s3_query: SELECT count(*) FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN _timescaledb_catalog.dimension td ON (h.id = td.hypertable_id) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = td.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.table_name = 'dim_test'; count ----- 2 starting permutation: s3_chunk_wp_enable s1_create_chunk s2_add_dimension2 s3_chunk_wp_release s3_query step s3_chunk_wp_enable: SELECT debug_waitpoint_enable('chunk_create_for_point'); debug_waitpoint_enable ---------------------- step s1_create_chunk: INSERT INTO dim_test VALUES ('2004-10-20 00:00:00+00', 1, 2); <waiting ...> step s2_add_dimension2: SELECT column_name FROM add_dimension('dim_test', 'device2', 1); <waiting ...> step s3_chunk_wp_release: SELECT debug_waitpoint_release('chunk_create_for_point'); debug_waitpoint_release ----------------------- step s1_create_chunk: <... completed> step s2_add_dimension2: <... completed> column_name ----------- device2 step s3_query: SELECT count(*) FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN _timescaledb_catalog.dimension td ON (h.id = td.hypertable_id) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = td.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.table_name = 'dim_test'; count ----- 4 ================================================ FILE: test/isolation/expected/concurrent_query_and_drop_chunks.out ================================================ Parsed test spec with 4 sessions starting permutation: s2_query s1_wp_enable s2_query s1_drop_chunks s1_wp_release s2_show_num_chunks step s2_query: SELECT * FROM measurements ORDER BY 1; time |device|temp ----------------------------+------+---- Fri Jan 03 10:30:00 2020 PST| 1| 1 Sun Jan 03 10:30:00 2021 PST| 2| 2 step s1_wp_enable: SELECT debug_waitpoint_enable('hypertable_expansion_before_lock_chunk'); debug_waitpoint_enable ---------------------- step s2_query: SELECT * FROM measurements ORDER BY 1; <waiting ...> step s1_drop_chunks: SELECT count(*) FROM drop_chunks('measurements', TIMESTAMPTZ '2020-03-01'); count ----- 1 step s1_wp_release: SELECT debug_waitpoint_release('hypertable_expansion_before_lock_chunk'); debug_waitpoint_release ----------------------- step s2_query: <... completed> time |device|temp ----------------------------+------+---- Sun Jan 03 10:30:00 2021 PST| 2| 2 step s2_show_num_chunks: SELECT count(*) FROM show_chunks('measurements') ORDER BY 1; count ----- 1 starting permutation: s3_wp_enable s4_hypertable_size s3_drop_chunks s3_wp_release step s3_wp_enable: SELECT debug_waitpoint_enable('relation_size_before_lock'); debug_waitpoint_enable ---------------------- step s4_hypertable_size: SELECT count(*) FROM hypertable_size('measurements'); <waiting ...> step s3_drop_chunks: SELECT count(*) FROM drop_chunks('measurements', TIMESTAMPTZ '2020-03-01'); count ----- 1 step s3_wp_release: SELECT debug_waitpoint_release('relation_size_before_lock'); debug_waitpoint_release ----------------------- step s4_hypertable_size: <... completed> count ----- 1 ================================================ FILE: test/isolation/expected/deadlock_dropchunks_select.out ================================================ Parsed test spec with 2 sessions starting permutation: s1a s1b s2a s2b step s1a: SELECT count (*) FROM drop_chunks('dt', '2018-12-25 00:00'::timestamptz); count ----- 24 step s1b: COMMIT; step s2a: SELECT typ, loc, mtim FROM DT , SL , ST WHERE SL.lid = DT.lid AND ST.sid = DT.sid AND mtim >= '2018-12-01 03:00:00+00' AND mtim <= '2018-12-01 04:00:00+00' AND typ = 'T1' ; typ|loc|mtim ---+---+---- step s2b: COMMIT; starting permutation: s1a s2a s1b s2b step s1a: SELECT count (*) FROM drop_chunks('dt', '2018-12-25 00:00'::timestamptz); count ----- 24 step s2a: SELECT typ, loc, mtim FROM DT , SL , ST WHERE SL.lid = DT.lid AND ST.sid = DT.sid AND mtim >= '2018-12-01 03:00:00+00' AND mtim <= '2018-12-01 04:00:00+00' AND typ = 'T1' ; <waiting ...> step s1b: COMMIT; step s2a: <... completed> typ|loc|mtim ---+---+---- step s2b: COMMIT; starting permutation: s1a s2a s2b s1b step s1a: SELECT count (*) FROM drop_chunks('dt', '2018-12-25 00:00'::timestamptz); count ----- 24 step s2a: SELECT typ, loc, mtim FROM DT , SL , ST WHERE SL.lid = DT.lid AND ST.sid = DT.sid AND mtim >= '2018-12-01 03:00:00+00' AND mtim <= '2018-12-01 04:00:00+00' AND typ = 'T1' ; <waiting ...> step s2a: <... completed> ERROR: canceling statement due to lock timeout step s2b: COMMIT; step s1b: COMMIT; starting permutation: s2a s1a s1b s2b step s2a: SELECT typ, loc, mtim FROM DT , SL , ST WHERE SL.lid = DT.lid AND ST.sid = DT.sid AND mtim >= '2018-12-01 03:00:00+00' AND mtim <= '2018-12-01 04:00:00+00' AND typ = 'T1' ; typ|loc|mtim ---+---+---- step s1a: SELECT count (*) FROM drop_chunks('dt', '2018-12-25 00:00'::timestamptz); <waiting ...> step s1a: <... completed> ERROR: canceling statement due to lock timeout step s1b: COMMIT; step s2b: COMMIT; starting permutation: s2a s1a s2b s1b step s2a: SELECT typ, loc, mtim FROM DT , SL , ST WHERE SL.lid = DT.lid AND ST.sid = DT.sid AND mtim >= '2018-12-01 03:00:00+00' AND mtim <= '2018-12-01 04:00:00+00' AND typ = 'T1' ; typ|loc|mtim ---+---+---- step s1a: SELECT count (*) FROM drop_chunks('dt', '2018-12-25 00:00'::timestamptz); <waiting ...> step s2b: COMMIT; step s1a: <... completed> count ----- 24 step s1b: COMMIT; starting permutation: s2a s2b s1a s1b step s2a: SELECT typ, loc, mtim FROM DT , SL , ST WHERE SL.lid = DT.lid AND ST.sid = DT.sid AND mtim >= '2018-12-01 03:00:00+00' AND mtim <= '2018-12-01 04:00:00+00' AND typ = 'T1' ; typ|loc|mtim ---+---+---- step s2b: COMMIT; step s1a: SELECT count (*) FROM drop_chunks('dt', '2018-12-25 00:00'::timestamptz); count ----- 24 step s1b: COMMIT; ================================================ FILE: test/isolation/expected/dropchunks_race.out ================================================ Parsed test spec with 5 sessions starting permutation: s3_chunks_found_wait s1_drop_chunks s2_drop_chunks s3_chunks_found_release s3_show_missing_slices s3_show_num_chunks s3_show_data step s3_chunks_found_wait: SELECT debug_waitpoint_enable('drop_chunks_chunks_found'); debug_waitpoint_enable ---------------------- step s1_drop_chunks: SELECT count(*) FROM drop_chunks('dropchunks_race_t1', TIMESTAMPTZ '2020-03-01'); <waiting ...> step s2_drop_chunks: SELECT count(*) FROM drop_chunks('dropchunks_race_t1', TIMESTAMPTZ '2020-03-01'); <waiting ...> step s3_chunks_found_release: SELECT debug_waitpoint_release('drop_chunks_chunks_found'); debug_waitpoint_release ----------------------- step s1_drop_chunks: <... completed> count ----- 1 step s2_drop_chunks: <... completed> count ----- 0 step s3_show_missing_slices: SELECT count(*) FROM _timescaledb_catalog.chunk_constraint WHERE dimension_slice_id NOT IN (SELECT id FROM _timescaledb_catalog.dimension_slice); count ----- 0 step s3_show_num_chunks: SELECT count(*) FROM show_chunks('dropchunks_race_t1') ORDER BY 1; count ----- 0 step s3_show_data: SELECT * FROM dropchunks_race_t1 ORDER BY 1; time|device|temp ----+------+---- starting permutation: s4_chunks_dropped_wait s1_drop_chunks s5_insert_new_chunk s4_chunks_dropped_release s3_show_missing_slices s3_show_num_chunks s3_show_data step s4_chunks_dropped_wait: SELECT debug_waitpoint_enable('drop_chunks_end'); debug_waitpoint_enable ---------------------- step s1_drop_chunks: SELECT count(*) FROM drop_chunks('dropchunks_race_t1', TIMESTAMPTZ '2020-03-01'); <waiting ...> step s5_insert_new_chunk: INSERT INTO dropchunks_race_t1 VALUES ('2020-03-01 10:30', 1, 2.2); <waiting ...> step s4_chunks_dropped_release: SELECT debug_waitpoint_release('drop_chunks_end'); debug_waitpoint_release ----------------------- step s1_drop_chunks: <... completed> count ----- 1 step s5_insert_new_chunk: <... completed> step s3_show_missing_slices: SELECT count(*) FROM _timescaledb_catalog.chunk_constraint WHERE dimension_slice_id NOT IN (SELECT id FROM _timescaledb_catalog.dimension_slice); count ----- 0 step s3_show_num_chunks: SELECT count(*) FROM show_chunks('dropchunks_race_t1') ORDER BY 1; count ----- 1 step s3_show_data: SELECT * FROM dropchunks_race_t1 ORDER BY 1; time |device|temp ----------------------------+------+---- Sun Mar 01 10:30:00 2020 PST| 1| 2.2 starting permutation: s4_chunks_dropped_wait s1_drop_chunks s5_insert_old_chunk s4_chunks_dropped_release s3_show_missing_slices s3_show_num_chunks s3_show_data step s4_chunks_dropped_wait: SELECT debug_waitpoint_enable('drop_chunks_end'); debug_waitpoint_enable ---------------------- step s1_drop_chunks: SELECT count(*) FROM drop_chunks('dropchunks_race_t1', TIMESTAMPTZ '2020-03-01'); <waiting ...> step s5_insert_old_chunk: INSERT INTO dropchunks_race_t1 VALUES ('2020-01-02 10:31', 1, 1.1); <waiting ...> step s4_chunks_dropped_release: SELECT debug_waitpoint_release('drop_chunks_end'); debug_waitpoint_release ----------------------- step s1_drop_chunks: <... completed> count ----- 1 step s5_insert_old_chunk: <... completed> step s3_show_missing_slices: SELECT count(*) FROM _timescaledb_catalog.chunk_constraint WHERE dimension_slice_id NOT IN (SELECT id FROM _timescaledb_catalog.dimension_slice); count ----- 0 step s3_show_num_chunks: SELECT count(*) FROM show_chunks('dropchunks_race_t1') ORDER BY 1; count ----- 1 step s3_show_data: SELECT * FROM dropchunks_race_t1 ORDER BY 1; time |device|temp ----------------------------+------+---- Thu Jan 02 10:31:00 2020 PST| 1| 1.1 ================================================ FILE: test/isolation/expected/insert_dropchunks_race.out ================================================ Parsed test spec with 2 sessions starting permutation: s1a s2a s1b s2b s1c step s1a: INSERT INTO insert_dropchunks_race_t1 VALUES ('2020-01-03 10:30', 3, 33.4); step s2a: SELECT COUNT(*) FROM drop_chunks('insert_dropchunks_race_t1', TIMESTAMPTZ '2020-03-01'); <waiting ...> step s1b: COMMIT; step s2a: <... completed> count ----- 2 step s2b: COMMIT; step s1c: SELECT COUNT(*) FROM _timescaledb_catalog.chunk_constraint LEFT JOIN _timescaledb_catalog.dimension_slice sl ON dimension_slice_id = sl.id WHERE sl.id IS NULL; count ----- 0 ================================================ FILE: test/isolation/expected/isolation_nop.out ================================================ Parsed test spec with 1 sessions starting permutation: s1a table_name --------------- ts_cluster_test step s1a: SELECT pg_sleep(0); pg_sleep -------- ================================================ FILE: test/isolation/expected/multi_transaction_indexing.out ================================================ Parsed test spec with 8 sessions starting permutation: CI I1 Ic Bc P Sc step CI: CREATE INDEX test_index ON ts_index_test(location) WITH (timescaledb.transaction_per_chunk, timescaledb.barrier_table='barrier'); <waiting ...> step I1: INSERT INTO ts_index_test VALUES (31, 6.4, 1); step Ic: COMMIT; step Bc: ROLLBACK; step CI: <... completed> step P: SELECT * FROM hypertable_index_size('test_index'); hypertable_index_size --------------------- 73728 step Sc: COMMIT; starting permutation: I1 CI Bc Ic P Sc step I1: INSERT INTO ts_index_test VALUES (31, 6.4, 1); step CI: CREATE INDEX test_index ON ts_index_test(location) WITH (timescaledb.transaction_per_chunk, timescaledb.barrier_table='barrier'); <waiting ...> step Bc: ROLLBACK; step Ic: COMMIT; step CI: <... completed> step P: SELECT * FROM hypertable_index_size('test_index'); hypertable_index_size --------------------- 73728 step Sc: COMMIT; starting permutation: S1 CI Bc Sc P Ic step S1: SELECT * FROM ts_index_test; time|temp|location ----+----+-------- 1|23.4| 1 11|21.3| 2 21|19.5| 3 step CI: CREATE INDEX test_index ON ts_index_test(location) WITH (timescaledb.transaction_per_chunk, timescaledb.barrier_table='barrier'); <waiting ...> step Bc: ROLLBACK; step CI: <... completed> step Sc: COMMIT; step P: SELECT * FROM hypertable_index_size('test_index'); hypertable_index_size --------------------- 57344 step Ic: COMMIT; starting permutation: F WPE CI DI Bc WPR P Ic Sc step F: SET client_min_messages TO 'error'; step WPE: SELECT debug_waitpoint_enable('process_index_start_indexing_done'); debug_waitpoint_enable ---------------------- step CI: CREATE INDEX test_index ON ts_index_test(location) WITH (timescaledb.transaction_per_chunk, timescaledb.barrier_table='barrier'); <waiting ...> step DI: DROP INDEX test_index; <waiting ...> step Bc: ROLLBACK; step DI: <... completed> step WPR: SELECT debug_waitpoint_release('process_index_start_indexing_done'); debug_waitpoint_release ----------------------- step CI: <... completed> step P: SELECT * FROM hypertable_index_size('test_index'); ERROR: relation "test_index" does not exist step Ic: COMMIT; step Sc: COMMIT; starting permutation: CI RI Bc P Ic Sc step CI: CREATE INDEX test_index ON ts_index_test(location) WITH (timescaledb.transaction_per_chunk, timescaledb.barrier_table='barrier'); <waiting ...> step RI: ALTER TABLE test_index RENAME COLUMN location TO height; <waiting ...> step Bc: ROLLBACK; step CI: <... completed> step RI: <... completed> step P: SELECT * FROM hypertable_index_size('test_index'); hypertable_index_size --------------------- 57344 step Ic: COMMIT; step Sc: COMMIT; ================================================ FILE: test/isolation/expected/read_committed_insert.out ================================================ Parsed test spec with 2 sessions starting permutation: s1a s1c s2a s2b table_name --------------- ts_cluster_test step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090001', 23.4, 1); step s1c: COMMIT; step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090002', 0.72, 1); step s2b: COMMIT; starting permutation: s1a s2a s1c s2b table_name --------------- ts_cluster_test step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090001', 23.4, 1); step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090002', 0.72, 1); <waiting ...> step s1c: COMMIT; step s2a: <... completed> step s2b: COMMIT; starting permutation: s1a s2a s2b s1c table_name --------------- ts_cluster_test step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090001', 23.4, 1); step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090002', 0.72, 1); <waiting ...> step s2a: <... completed> ERROR: canceling statement due to lock timeout step s2b: COMMIT; step s1c: COMMIT; starting permutation: s2a s1a s1c s2b table_name --------------- ts_cluster_test step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090002', 0.72, 1); step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090001', 23.4, 1); <waiting ...> step s1a: <... completed> ERROR: canceling statement due to lock timeout step s1c: COMMIT; step s2b: COMMIT; starting permutation: s2a s1a s2b s1c table_name --------------- ts_cluster_test step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090002', 0.72, 1); step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090001', 23.4, 1); <waiting ...> step s2b: COMMIT; step s1a: <... completed> step s1c: COMMIT; starting permutation: s2a s2b s1a s1c table_name --------------- ts_cluster_test step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090002', 0.72, 1); step s2b: COMMIT; step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090001', 23.4, 1); step s1c: COMMIT; ================================================ FILE: test/isolation/expected/read_uncommitted_insert.out ================================================ Parsed test spec with 2 sessions starting permutation: s1a s1c s2a s2b table_name --------------- ts_cluster_test step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090001', 23.4, 1); step s1c: COMMIT; step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090002', 0.72, 1); step s2b: COMMIT; starting permutation: s1a s2a s1c s2b table_name --------------- ts_cluster_test step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090001', 23.4, 1); step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090002', 0.72, 1); <waiting ...> step s1c: COMMIT; step s2a: <... completed> step s2b: COMMIT; starting permutation: s1a s2a s2b s1c table_name --------------- ts_cluster_test step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090001', 23.4, 1); step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090002', 0.72, 1); <waiting ...> step s2a: <... completed> ERROR: canceling statement due to lock timeout step s2b: COMMIT; step s1c: COMMIT; starting permutation: s2a s1a s1c s2b table_name --------------- ts_cluster_test step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090002', 0.72, 1); step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090001', 23.4, 1); <waiting ...> step s1a: <... completed> ERROR: canceling statement due to lock timeout step s1c: COMMIT; step s2b: COMMIT; starting permutation: s2a s1a s2b s1c table_name --------------- ts_cluster_test step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090002', 0.72, 1); step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090001', 23.4, 1); <waiting ...> step s2b: COMMIT; step s1a: <... completed> step s1c: COMMIT; starting permutation: s2a s2b s1a s1c table_name --------------- ts_cluster_test step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090002', 0.72, 1); step s2b: COMMIT; step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090001', 23.4, 1); step s1c: COMMIT; ================================================ FILE: test/isolation/expected/repeatable_read_insert.out ================================================ Parsed test spec with 2 sessions starting permutation: s1a s1c s2a s2b table_name --------------- ts_cluster_test step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090001', 23.4, 1); step s1c: COMMIT; step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090002', 0.72, 1); step s2b: COMMIT; starting permutation: s1a s2a s1c s2b table_name --------------- ts_cluster_test step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090001', 23.4, 1); step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090002', 0.72, 1); <waiting ...> step s1c: COMMIT; step s2a: <... completed> step s2b: COMMIT; starting permutation: s1a s2a s2b s1c table_name --------------- ts_cluster_test step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090001', 23.4, 1); step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090002', 0.72, 1); <waiting ...> step s2a: <... completed> ERROR: canceling statement due to lock timeout step s2b: COMMIT; step s1c: COMMIT; starting permutation: s2a s1a s1c s2b table_name --------------- ts_cluster_test step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090002', 0.72, 1); step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090001', 23.4, 1); <waiting ...> step s1a: <... completed> ERROR: canceling statement due to lock timeout step s1c: COMMIT; step s2b: COMMIT; starting permutation: s2a s1a s2b s1c table_name --------------- ts_cluster_test step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090002', 0.72, 1); step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090001', 23.4, 1); <waiting ...> step s2b: COMMIT; step s1a: <... completed> step s1c: COMMIT; starting permutation: s2a s2b s1a s1c table_name --------------- ts_cluster_test step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090002', 0.72, 1); step s2b: COMMIT; step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T090001', 23.4, 1); step s1c: COMMIT; ================================================ FILE: test/isolation/expected/serializable_insert.out ================================================ Parsed test spec with 2 sessions starting permutation: s1a s1c s2a s2c schema_name|table_name -----------+--------------- public |ts_cluster_test step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:01', 23.4, 1); step s1c: COMMIT; step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:02', 0.72, 1); step s2c: COMMIT; starting permutation: s1a s2a s1c s2c schema_name|table_name -----------+--------------- public |ts_cluster_test step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:01', 23.4, 1); step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:02', 0.72, 1); <waiting ...> step s1c: COMMIT; step s2a: <... completed> step s2c: COMMIT; starting permutation: s1a s2a s2c s1c schema_name|table_name -----------+--------------- public |ts_cluster_test step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:01', 23.4, 1); step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:02', 0.72, 1); <waiting ...> step s2a: <... completed> ERROR: canceling statement due to lock timeout step s2c: COMMIT; step s1c: COMMIT; starting permutation: s2a s1a s1c s2c schema_name|table_name -----------+--------------- public |ts_cluster_test step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:02', 0.72, 1); step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:01', 23.4, 1); <waiting ...> step s1a: <... completed> ERROR: canceling statement due to lock timeout step s1c: COMMIT; step s2c: COMMIT; starting permutation: s2a s1a s2c s1c schema_name|table_name -----------+--------------- public |ts_cluster_test step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:02', 0.72, 1); step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:01', 23.4, 1); <waiting ...> step s2c: COMMIT; step s1a: <... completed> step s1c: COMMIT; starting permutation: s2a s2c s1a s1c schema_name|table_name -----------+--------------- public |ts_cluster_test step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:02', 0.72, 1); step s2c: COMMIT; step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:01', 23.4, 1); step s1c: COMMIT; ================================================ FILE: test/isolation/expected/serializable_insert_rollback.out ================================================ Parsed test spec with 2 sessions starting permutation: s1a s1c s2a s2c schema_name|table_name -----------+--------------- public |ts_cluster_test step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:01', 23.4, 1); step s1c: ROLLBACK; step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:02', 0.72, 1); step s2c: COMMIT; starting permutation: s1a s2a s1c s2c schema_name|table_name -----------+--------------- public |ts_cluster_test step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:01', 23.4, 1); step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:02', 0.72, 1); <waiting ...> step s1c: ROLLBACK; step s2a: <... completed> step s2c: COMMIT; starting permutation: s1a s2a s2c s1c schema_name|table_name -----------+--------------- public |ts_cluster_test step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:01', 23.4, 1); step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:02', 0.72, 1); <waiting ...> step s2a: <... completed> ERROR: canceling statement due to lock timeout step s2c: COMMIT; step s1c: ROLLBACK; starting permutation: s2a s1a s1c s2c schema_name|table_name -----------+--------------- public |ts_cluster_test step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:02', 0.72, 1); step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:01', 23.4, 1); <waiting ...> step s1a: <... completed> ERROR: canceling statement due to lock timeout step s1c: ROLLBACK; step s2c: COMMIT; starting permutation: s2a s1a s2c s1c schema_name|table_name -----------+--------------- public |ts_cluster_test step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:02', 0.72, 1); step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:01', 23.4, 1); <waiting ...> step s2c: COMMIT; step s1a: <... completed> step s1c: ROLLBACK; starting permutation: s2a s2c s1a s1c schema_name|table_name -----------+--------------- public |ts_cluster_test step s2a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:02', 0.72, 1); step s2c: COMMIT; step s1a: INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:01', 23.4, 1); step s1c: ROLLBACK; ================================================ FILE: test/isolation/specs/CMakeLists.txt ================================================ set(TEST_FILES deadlock_dropchunks_select.spec insert_dropchunks_race.spec isolation_nop.spec read_committed_insert.spec read_uncommitted_insert.spec repeatable_read_insert.spec serializable_insert_rollback.spec serializable_insert.spec) file(REMOVE ${ISOLATION_TEST_SCHEDULE}) set(TEST_TEMPLATES) if(CMAKE_BUILD_TYPE MATCHES Debug) list(APPEND TEST_FILES concurrent_add_dimension.spec concurrent_query_and_drop_chunks.spec dropchunks_race.spec multi_transaction_indexing.spec) endif(CMAKE_BUILD_TYPE MATCHES Debug) foreach(TEST_FILE ${TEST_FILES}) string(REGEX REPLACE "(.+)\.spec" "\\1" TESTS_TO_RUN ${TEST_FILE}) file(APPEND ${ISOLATION_TEST_SCHEDULE} "test: ${TESTS_TO_RUN}\n") endforeach(TEST_FILE) ================================================ FILE: test/isolation/specs/concurrent_add_dimension.spec ================================================ # This file and its contents are licensed under the Apache License 2.0. # Please see the included NOTICE for copyright information and # LICENSE-APACHE for a copy of the license. setup { DROP TABLE IF EXISTS dim_test; CREATE TABLE dim_test(time TIMESTAMPTZ, device int, device2 int); SELECT table_name FROM create_hypertable('dim_test', 'time', chunk_time_interval => INTERVAL '1 day'); INSERT INTO dim_test VALUES ('2004-10-10 00:00:00+00', 1, 1); } teardown { DROP TABLE dim_test; } session "s1" step "s1_wp_enable" { SELECT debug_waitpoint_enable('add_dimension_ht_lock'); } step "s1_wp_release" { SELECT debug_waitpoint_release('add_dimension_ht_lock'); } step "s1_add_dimension" { SELECT column_name FROM add_dimension('dim_test', 'device', 2); } step "s1_create_chunk" { INSERT INTO dim_test VALUES ('2004-10-20 00:00:00+00', 1, 2); } session "s2" step "s2_add_dimension" { SELECT column_name FROM add_dimension('dim_test', 'device', 1); } step "s2_add_dimension2" { SELECT column_name FROM add_dimension('dim_test', 'device2', 1); } session "s3" step "s3_wp_enable" { SELECT debug_waitpoint_enable('add_dimension_ht_lock'); } step "s3_wp_release" { SELECT debug_waitpoint_release('add_dimension_ht_lock'); } step "s3_chunk_wp_enable" { SELECT debug_waitpoint_enable('chunk_create_for_point'); } step "s3_chunk_wp_release" { SELECT debug_waitpoint_release('chunk_create_for_point'); } step "s3_query" { SELECT count(*) FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN _timescaledb_catalog.dimension td ON (h.id = td.hypertable_id) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = td.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.table_name = 'dim_test'; } # Test concurrent add_dimension() call with existing data # permutation "s3_wp_enable" "s1_add_dimension" "s2_add_dimension" "s3_wp_release" "s3_query" # Test concurrent chunk creation during add_dimension() call # permutation "s3_chunk_wp_enable" "s1_create_chunk" "s2_add_dimension2" "s3_chunk_wp_release" "s3_query" ================================================ FILE: test/isolation/specs/concurrent_query_and_drop_chunks.spec ================================================ # This file and its contents are licensed under the Apache License 2.0. # Please see the included NOTICE for copyright information and # LICENSE-APACHE for a copy of the license. setup { DROP TABLE IF EXISTS measurements; CREATE TABLE measurements (time timestamptz, device int, temp float); SELECT create_hypertable('measurements', 'time', 'device', 2); INSERT INTO measurements VALUES ('2020-01-03 10:30', 1, 1.0), ('2021-01-03 10:30', 2, 2.0); } teardown { DROP TABLE measurements; } # # Test concurrent querying and drop chunks. # session "s1" step "s1_wp_enable" { SELECT debug_waitpoint_enable('hypertable_expansion_before_lock_chunk'); } step "s1_wp_release" { SELECT debug_waitpoint_release('hypertable_expansion_before_lock_chunk'); } step "s1_drop_chunks" { SELECT count(*) FROM drop_chunks('measurements', TIMESTAMPTZ '2020-03-01'); } session "s2" step "s2_show_num_chunks" { SELECT count(*) FROM show_chunks('measurements') ORDER BY 1; } step "s2_query" { SELECT * FROM measurements ORDER BY 1; } session "s3" step "s3_wp_enable" { SELECT debug_waitpoint_enable('relation_size_before_lock'); } step "s3_wp_release" { SELECT debug_waitpoint_release('relation_size_before_lock'); } step "s3_drop_chunks" { SELECT count(*) FROM drop_chunks('measurements', TIMESTAMPTZ '2020-03-01'); } session "s4" step "s4_hypertable_size" { SELECT count(*) FROM hypertable_size('measurements'); } # The wait point happens after chunks have been found for table # expansion, but before the chunks are locked. Because one chunk # will dropped before the lock is acqurired, the chunk should # also be ignored. permutation "s2_query" "s1_wp_enable" "s2_query" "s1_drop_chunks" "s1_wp_release" "s2_show_num_chunks" # The wait point happens before the relation_size get the lock # for the relation and one chunk will be dropped in another session # don't leading to race conditions permutation "s3_wp_enable" "s4_hypertable_size" "s3_drop_chunks" "s3_wp_release" ================================================ FILE: test/isolation/specs/deadlock_dropchunks_select.spec ================================================ # This file and its contents are licensed under the Apache License 2.0. # Please see the included NOTICE for copyright information and # LICENSE-APACHE for a copy of the license. ##github issue 865 deadlock between select and drop chunks setup { CREATE TABLE ST ( sid int PRIMARY KEY, typ text) ; CREATE TABLE SL ( lid int PRIMARY KEY, loc text) ; CREATE TABLE DT ( sid int REFERENCES ST(sid), lid int REFERENCES SL(lid), mtim timestamp with time zone ) ; SELECT create_hypertable('DT', 'mtim', chunk_time_interval => interval '1 day'); INSERT INTO SL VALUES (1, 'LA'); INSERT INTO ST VALUES (1, 'T1'); INSERT INTO DT (sid,lid,mtim) SELECT 1,1, generate_series( '2018-12-01 00:00'::timestamp, '2018-12-31 12:00','1 minute') ; } teardown { DROP TABLE DT; DROP table ST; DROP table SL; } session "s1" setup { BEGIN; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SET LOCAL lock_timeout = '5000ms'; SET LOCAL deadlock_timeout = '10ms'; } step "s1a" { SELECT count (*) FROM drop_chunks('dt', '2018-12-25 00:00'::timestamptz); } step "s1b" { COMMIT; } session "s2" setup { BEGIN; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SET LOCAL lock_timeout = '5000ms'; SET LOCAL deadlock_timeout = '10ms'; } step "s2a" { SELECT typ, loc, mtim FROM DT , SL , ST WHERE SL.lid = DT.lid AND ST.sid = DT.sid AND mtim >= '2018-12-01 03:00:00+00' AND mtim <= '2018-12-01 04:00:00+00' AND typ = 'T1' ; } step "s2b" { COMMIT; } ================================================ FILE: test/isolation/specs/dropchunks_race.spec ================================================ # This file and its contents are licensed under the Apache License 2.0. # Please see the included NOTICE for copyright information and # LICENSE-APACHE for a copy of the license. setup { DROP TABLE IF EXISTS dropchunks_race_t1; CREATE TABLE dropchunks_race_t1 (time timestamptz, device int, temp float); SELECT create_hypertable('dropchunks_race_t1', 'time', 'device', 2); INSERT INTO dropchunks_race_t1 VALUES ('2020-01-03 10:30', 1, 32.2); } teardown { DROP TABLE dropchunks_race_t1; } session "s1" step "s1_drop_chunks" { SELECT count(*) FROM drop_chunks('dropchunks_race_t1', TIMESTAMPTZ '2020-03-01'); } session "s2" step "s2_drop_chunks" { SELECT count(*) FROM drop_chunks('dropchunks_race_t1', TIMESTAMPTZ '2020-03-01'); } session "s3" step "s3_chunks_found_wait" { SELECT debug_waitpoint_enable('drop_chunks_chunks_found'); } step "s3_chunks_found_release" { SELECT debug_waitpoint_release('drop_chunks_chunks_found'); } step "s3_show_missing_slices" { SELECT count(*) FROM _timescaledb_catalog.chunk_constraint WHERE dimension_slice_id NOT IN (SELECT id FROM _timescaledb_catalog.dimension_slice); } step "s3_show_num_chunks" { SELECT count(*) FROM show_chunks('dropchunks_race_t1') ORDER BY 1; } step "s3_show_data" { SELECT * FROM dropchunks_race_t1 ORDER BY 1; } session "s4" step "s4_chunks_dropped_wait" { SELECT debug_waitpoint_enable('drop_chunks_end'); } step "s4_chunks_dropped_release" { SELECT debug_waitpoint_release('drop_chunks_end'); } session "s5" step "s5_insert_old_chunk" { INSERT INTO dropchunks_race_t1 VALUES ('2020-01-02 10:31', 1, 1.1); } step "s5_insert_new_chunk" { INSERT INTO dropchunks_race_t1 VALUES ('2020-03-01 10:30', 1, 2.2); } # Test race between two drop_chunks processes. permutation "s3_chunks_found_wait" "s1_drop_chunks" "s2_drop_chunks" "s3_chunks_found_release" "s3_show_missing_slices" "s3_show_num_chunks" "s3_show_data" # Test race between drop_chunks and an insert into a new chunk. The # new chunk will share a slice with the chunk that is about to be # dropped. The shared slice must persist after drop_chunks completes, # or otherwise the new chunk will lack one slice. permutation "s4_chunks_dropped_wait" "s1_drop_chunks" "s5_insert_new_chunk" "s4_chunks_dropped_release" "s3_show_missing_slices" "s3_show_num_chunks" "s3_show_data" # Test race between drop_chunks and an insert into the chunk being # concurrently dropped. The chunk and slices should be recreated. permutation "s4_chunks_dropped_wait" "s1_drop_chunks" "s5_insert_old_chunk" "s4_chunks_dropped_release" "s3_show_missing_slices" "s3_show_num_chunks" "s3_show_data" ================================================ FILE: test/isolation/specs/insert_dropchunks_race.spec ================================================ # This file and its contents are licensed under the Apache License 2.0. # Please see the included NOTICE for copyright information and # LICENSE-APACHE for a copy of the license. # Race condition between insert and drop_chunks # # If an insert need to create a new chunk, it will look for existing # dimension slices to see if any need to be added: if slices already # exist, they do not need to be re-constructed and constraints can be # added that reference these slices. If chunks are dropped, there is a # cleanup of unreferenced dimension slices which can possibly remove # unreferenced dimension slices if transactions creating new chunks do # not lock the dimension slices for read. # # This isolation test check that a concurrent insert and drop_chunks # do not accidentally create a broken state by adding chunk # constraints that reference non-existing dimension slices. setup { DROP TABLE IF EXISTS insert_dropchunks_race_t1; CREATE TABLE insert_dropchunks_race_t1 (time timestamptz, device int, temp float); SELECT create_hypertable('insert_dropchunks_race_t1', 'time', 'device', 2); INSERT INTO insert_dropchunks_race_t1 VALUES ('2020-01-03 10:30', 1, 32.2); } teardown { DROP TABLE insert_dropchunks_race_t1; } session "s1" setup { BEGIN; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; } step "s1a" { INSERT INTO insert_dropchunks_race_t1 VALUES ('2020-01-03 10:30', 3, 33.4); } step "s1b" { COMMIT; } step "s1c" { SELECT COUNT(*) FROM _timescaledb_catalog.chunk_constraint LEFT JOIN _timescaledb_catalog.dimension_slice sl ON dimension_slice_id = sl.id WHERE sl.id IS NULL; } session "s2" setup { BEGIN; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; } step "s2a" { SELECT COUNT(*) FROM drop_chunks('insert_dropchunks_race_t1', TIMESTAMPTZ '2020-03-01'); } step "s2b" { COMMIT; } permutation "s1a" "s2a" "s1b" "s2b" "s1c" ================================================ FILE: test/isolation/specs/isolation_nop.spec ================================================ # This file and its contents are licensed under the Apache License 2.0. # Please see the included NOTICE for copyright information and # LICENSE-APACHE for a copy of the license. setup{ CREATE TABLE ts_cluster_test(time timestamptz, temp float, location int); SELECT table_name from create_hypertable('ts_cluster_test', 'time', chunk_time_interval => interval '1 day'); } teardown { DROP TABLE ts_cluster_test; } session "s1" step "s1a" { SELECT pg_sleep(0); } ================================================ FILE: test/isolation/specs/multi_transaction_indexing.spec ================================================ # This file and its contents are licensed under the Apache License 2.0. # Please see the included NOTICE for copyright information and # LICENSE-APACHE for a copy of the license. setup { CREATE TABLE ts_index_test(time int, temp float, location int); SELECT create_hypertable('ts_index_test', 'time', chunk_time_interval => 10, create_default_indexes => false); INSERT INTO ts_index_test VALUES (1, 23.4, 1), (11, 21.3, 2), (21, 19.5, 3); CREATE TABLE barrier(i INTEGER); } teardown { DROP TABLE ts_index_test; DROP TABLE barrier; } session "Waitpoints" step "WPE" { SELECT debug_waitpoint_enable('process_index_start_indexing_done'); } step "WPR" { SELECT debug_waitpoint_release('process_index_start_indexing_done'); } session "CREATE INDEX" step "F" { SET client_min_messages TO 'error'; } step "CI" { CREATE INDEX test_index ON ts_index_test(location) WITH (timescaledb.transaction_per_chunk, timescaledb.barrier_table='barrier'); } session "RELEASE BARRIER" setup { BEGIN; SET LOCAL lock_timeout = '500ms'; SET LOCAL deadlock_timeout = '10ms'; LOCK TABLE barrier;} step "Bc" { ROLLBACK; } session "SELECT" setup { BEGIN; SET LOCAL lock_timeout = '500ms'; SET LOCAL deadlock_timeout = '10ms'; } step "S1" { SELECT * FROM ts_index_test; } step "Sc" { COMMIT; } session "INSERT CHUNK" setup { BEGIN; SET LOCAL lock_timeout = '500ms'; SET LOCAL deadlock_timeout = '10ms'; } step "I1" { INSERT INTO ts_index_test VALUES (31, 6.4, 1); } step "Ic" { COMMIT; } session "DROP INDEX" step "DI" { DROP INDEX test_index; } session "RENAME COLUMN" step "RI" { ALTER TABLE test_index RENAME COLUMN location TO height; } session "COUNT INDEXES" step "P" { SELECT * FROM hypertable_index_size('test_index'); } # we need to COMMIT every transaction started in setup regardless of whether we use them # inserts work between chunks permutation "CI" "I1" "Ic" "Bc" "P" "Sc" # create blocks on insert permutation "I1" "CI" "Bc" "Ic" "P" "Sc" # create blocks on select permutation "S1" "CI" "Bc" "Sc" "P" "Ic" # drop works (the error message outputs an OID, remove the "F" to see the error) permutation "F" "WPE" "CI" "DI" "Bc" "WPR" "P" "Ic" "Sc" # rename should block permutation "CI" "RI" "Bc" "P" "Ic" "Sc" # Ideally we would declare these functions in setup, and use them to check that the actual index # exist on the relevant chunks in these tests. Unfortunately, in older versions of postgres # (IIRC until 10.4) there was an arbitrary limit that each SQL statement in an isolation test could # be no longer than 1024 characters. Instead, we currently use the number of bytes indexs take # as a proxy. Once we deprecate the old version, or add some other way to get index info we should # switch to this. # # -- functions from testsupport.sql duplicated here becasue we cannot include sql files # CREATE OR REPLACE FUNCTION show_columns(rel regclass) # RETURNS TABLE("Column" name, # "Type" text, # "Nullable" boolean) LANGUAGE SQL STABLE AS # $BODY$ # SELECT a.attname, # format_type(t.oid, t.typtypmod), # a.attnotnull # FROM pg_attribute a, pg_type t # WHERE a.attrelid = rel # AND a.atttypid = t.oid # AND a.attnum >= 0 # ORDER BY a.attnum; # $BODY$; # # CREATE OR REPLACE FUNCTION show_indexesp(pattern text) # RETURNS TABLE("Table" regclass, # "Index" regclass, # "Columns" name[], # "Expr" text, # "Unique" boolean, # "Primary" boolean, # "Exclusion" boolean, # "Tablespace" name) LANGUAGE PLPGSQL STABLE AS # $BODY$ # DECLARE # schema_name name = split_part(pattern, '.', 1); # table_name name = split_part(pattern, '.', 2); # BEGIN # IF schema_name = '' OR table_name = '' THEN # schema_name := current_schema(); # table_name := pattern; # END IF; # # RETURN QUERY # SELECT c.oid::regclass, # i.indexrelid::regclass, # array(SELECT "Column" FROM show_columns(i.indexrelid)), # pg_get_expr(i.indexprs, c.oid, true), # i.indisunique, # i.indisprimary, # i.indisexclusion, # (SELECT t.spcname FROM pg_class cc, pg_tablespace t WHERE cc.oid = i.indexrelid AND t.oid = cc.reltablespace) # FROM pg_class c, pg_index i # WHERE format('%I.%I', c.relnamespace::regnamespace::name, c.relname) LIKE format('%I.%s', schema_name, table_name) # AND c.oid = i.indrelid # ORDER BY c.oid, i.indexrelid; # END # $BODY$; ================================================ FILE: test/isolation/specs/read_committed_insert.spec ================================================ # This file and its contents are licensed under the Apache License 2.0. # Please see the included NOTICE for copyright information and # LICENSE-APACHE for a copy of the license. setup { CREATE TABLE ts_cluster_test(time timestamptz, temp float, location int); SELECT table_name from create_hypertable('ts_cluster_test', 'time', chunk_time_interval => interval '1 day'); } teardown { DROP TABLE ts_cluster_test; } session "s1" setup { BEGIN; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SET LOCAL lock_timeout = '500ms'; SET LOCAL deadlock_timeout = '10ms'; } step "s1a" { INSERT INTO ts_cluster_test VALUES ('2017-01-20T090001', 23.4, 1); } step "s1c" { COMMIT; } session "s2" setup { BEGIN; SET TRANSACTION ISOLATION LEVEL READ COMMITTED; SET LOCAL lock_timeout = '500ms'; SET LOCAL deadlock_timeout = '10ms'; } step "s2a" { INSERT INTO ts_cluster_test VALUES ('2017-01-20T090002', 0.72, 1); } step "s2b" { COMMIT; } ================================================ FILE: test/isolation/specs/read_uncommitted_insert.spec ================================================ # This file and its contents are licensed under the Apache License 2.0. # Please see the included NOTICE for copyright information and # LICENSE-APACHE for a copy of the license. setup { CREATE TABLE ts_cluster_test(time timestamptz, temp float, location int); SELECT table_name from create_hypertable('ts_cluster_test', 'time', chunk_time_interval => interval '1 day'); } teardown { DROP TABLE ts_cluster_test; } session "s1" setup { BEGIN; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; SET LOCAL lock_timeout = '500ms'; SET LOCAL deadlock_timeout = '10ms'; } step "s1a" { INSERT INTO ts_cluster_test VALUES ('2017-01-20T090001', 23.4, 1); } step "s1c" { COMMIT; } session "s2" setup { BEGIN; SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; SET LOCAL lock_timeout = '500ms'; SET LOCAL deadlock_timeout = '10ms'; } step "s2a" { INSERT INTO ts_cluster_test VALUES ('2017-01-20T090002', 0.72, 1); } step "s2b" { COMMIT; } ================================================ FILE: test/isolation/specs/repeatable_read_insert.spec ================================================ # This file and its contents are licensed under the Apache License 2.0. # Please see the included NOTICE for copyright information and # LICENSE-APACHE for a copy of the license. setup { CREATE TABLE ts_cluster_test(time timestamptz, temp float, location int); SELECT table_name from create_hypertable('ts_cluster_test', 'time', chunk_time_interval => interval '1 day'); } teardown { DROP TABLE ts_cluster_test; } session "s1" setup { BEGIN; SET TRANSACTION ISOLATION LEVEL REPEATABLE READ; SET LOCAL lock_timeout = '500ms'; SET LOCAL deadlock_timeout = '10ms'; } step "s1a" { INSERT INTO ts_cluster_test VALUES ('2017-01-20T090001', 23.4, 1); } step "s1c" { COMMIT; } session "s2" setup { BEGIN; SET TRANSACTION ISOLATION LEVEL REPEATABLE READ; SET LOCAL lock_timeout = '500ms'; SET LOCAL deadlock_timeout = '10ms'; } step "s2a" { INSERT INTO ts_cluster_test VALUES ('2017-01-20T090002', 0.72, 1); } step "s2b" { COMMIT; } ================================================ FILE: test/isolation/specs/serializable_insert.spec ================================================ # This file and its contents are licensed under the Apache License 2.0. # Please see the included NOTICE for copyright information and # LICENSE-APACHE for a copy of the license. setup { CREATE TABLE ts_cluster_test(time timestamptz, temp float, location int); SELECT schema_name, table_name FROM create_hypertable('ts_cluster_test', 'time', chunk_time_interval => interval '1 day'); } teardown { DROP TABLE ts_cluster_test; } session "s1" setup { BEGIN; SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; SET LOCAL lock_timeout = '500ms'; SET LOCAL deadlock_timeout = '10ms'; } step "s1a" { INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:01', 23.4, 1); } step "s1c" { COMMIT; } session "s2" setup { BEGIN; SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; SET LOCAL lock_timeout = '500ms'; SET LOCAL deadlock_timeout = '10ms'; } step "s2a" { INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:02', 0.72, 1); } step "s2c" { COMMIT; } ================================================ FILE: test/isolation/specs/serializable_insert_rollback.spec ================================================ # This file and its contents are licensed under the Apache License 2.0. # Please see the included NOTICE for copyright information and # LICENSE-APACHE for a copy of the license. setup { CREATE TABLE ts_cluster_test(time timestamptz, temp float, location int); SELECT schema_name, table_name FROM create_hypertable('ts_cluster_test', 'time', chunk_time_interval => interval '1 day'); } teardown { DROP TABLE ts_cluster_test; } session "s1" setup { BEGIN; SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; SET LOCAL lock_timeout = '500ms'; SET LOCAL deadlock_timeout = '10ms'; } step "s1a" { INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:01', 23.4, 1); } step "s1c" { ROLLBACK; } session "s2" setup { BEGIN; SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; SET LOCAL lock_timeout = '500ms'; SET LOCAL deadlock_timeout = '10ms'; } step "s2a" { INSERT INTO ts_cluster_test VALUES ('2017-01-20T09:00:02', 0.72, 1); } step "s2c" { COMMIT; } ================================================ FILE: test/perl/CMakeLists.txt ================================================ set(PERL_FILES TimescaleNode.pm) # Check if PostgreSQL was compiled with --enable-tap-tests if(TAP_CHECKS AND EXISTS "${PG_PKGLIBDIR}/pgxs/src/test/perl") install(FILES ${PERL_FILES} DESTINATION "${PG_PKGLIBDIR}/pgxs/src/test/perl") endif() ================================================ FILE: test/perl/README.md ================================================ # Perl-based TAP tests `test/perl/` contains shared infrastructure that's used by Perl-based tests across the source tree The tests are invoked via perl's `prove` command. By default every test in the t/ subdirectory is run. Individual test(s) can be run instead by passing something like `PROVE_TESTS="t/001_testname.pl t/002_othertestname.pl"` to make. You should prefer to write tests using `pg_regress`, or isolation tester specs, if possible. Note that all tests and test tools should have perltidy run on them using perltidy, for example: ``` perltidy --profile=$TS_SRC_DIR/.perltidyrc /path/to/taptest ``` ## Writing tests Tests are written using Perl's `Test::More` with some PostgreSQL-specific infrastructure from `src/test/perl` providing node management, support for invoking `psql` to run queries and get results, etc. You should read the documentation for `Test::More` before trying to write tests. The PG specific infrastructure has been extended via the `TimescaleNode` class in this directory to add timescale specific configuration parameters and some often used helper functions. Test scripts in the t/ subdirectory of a suite are executed in alphabetical order. Each test script should begin with: ``` use strict; use warnings; use TimescaleNode; use TestLib; # Replace with the number of tests to execute: use Test::More tests => 1; ``` then it will generally need to set up one or more nodes, run commands against them and evaluate the results. For example: ``` my $node = get_new_ts_node('access_node'); $node->init; $node->start; my $ret = $node->safe_psql('postgres', 'SELECT 1'); is($ret, '1', 'SELECT 1 returns 1'); $node->stop('fast'); ``` `Test::More::like` entails use of the qr// operator. Avoid Perl 5.8.8 bug #39185 by not using the "$" regular expression metacharacter in qr// when also using the "/m" modifier. Instead of "$", use "\n" or "(?=\n|\z)". Read the `Test::More` documentation for more on how to write tests: ``` perldoc Test::More ``` For available PostgreSQL-specific test methods and some example tests read the perldoc for the test modules, e.g.: ``` cd $COMMUNITY_PG_SRCS perldoc src/test/perl/PostgresNode.pm ``` ## Required Perl Tests must run on perl `5.8.0` and newer. `perlbrew` is a good way to obtain such a Perl; see http://perlbrew.pl . Just install and ``` perlbrew --force install 5.8.0 perlbrew use 5.8.0 perlbrew install-cpanm cpanm install IPC::Run ``` then re-run configure to ensure the correct Perl is used when running tests. To verify that the right Perl was found: ``` grep ^PERL= config.log ``` ================================================ FILE: test/perl/TimescaleNode.pm ================================================ # This file and its contents are licensed under the Timescale License. # Please see the included NOTICE for copyright information and # LICENSE-TIMESCALE for a copy of the license. # This class extends PostgresNode with Timescale-specific # routines for setup. package TimescaleNode; # Using linebreaks here to prevent perltidy from performing vertical alignment. # This functionality has changed in recent perltidy versions (e.g., 2021 10 29) # and would restrict the versions of perltidy that can be used to format the # sources. use parent PostgreSQL::Test::Cluster; use PostgreSQL::Test::Utils qw(slurp_file); use strict; use warnings; use Carp 'verbose'; $SIG{__DIE__} = \&Carp::confess; sub create { my ($class, $name, %kwargs) = @_; my $self = $class->new($name); $self->init(%kwargs); $self->start(%kwargs); $self->safe_psql('postgres', 'CREATE EXTENSION timescaledb'); return $self; } # initialize the data directory and add TS specific parameters sub init { my ($self, %kwargs) = @_; $self->SUPER::init(%kwargs); # append into postgresql.conf from Timescale # template config file $self->append_conf('postgresql.conf', slurp_file("$ENV{'CONFDIR'}/postgresql.conf")); $self->append_conf('postgresql.conf', 'datestyle=ISO'); } # helper function to check output from PSQL for a query sub psql_is { my ($self, $db, $query, $expected_stdout, $testname) = @_; my ($psql_rc, $psql_out, $psql_err) = $self->SUPER::psql($db, $query); PostgreSQL::Test::Cluster::ok(!$psql_rc, "$testname: err_code check"); PostgreSQL::Test::Cluster::is($psql_err, '', "$testname: error_msg check"); PostgreSQL::Test::Cluster::is($psql_out, $expected_stdout, "$testname: psql output check"); } # remove leading and trailing whitespace sub strip { my ($str) = @_; $str =~ s/^\s+|\s+$//g; return $str; } sub safe_psql { my ($self, $db, $query) = @_; my $psql_out = $self->SUPER::safe_psql($db, $query); return strip($psql_out); } 1; ================================================ FILE: test/pg_prove.sh ================================================ #!/usr/bin/env bash # Wrapper around perl prove utility to control running of TAP tests # # The following control variable is supported: # # PROVE_TESTS only run TAP tests from this list # e.g make provecheck PROVE_TESTS="t/foo.pl t/bar.pl" # # Note that you can also use regular expressions to run multiple # taps tests matching the pattern: # # e.g make provecheck PROVE_TESTS="t/*chunk*" # PROVE_TESTS=${PROVE_TESTS:-} PROVE=${PROVE:-prove} echo "SKIPS: ${SKIPS}" # If PROVE_TESTS is specified then run those subset of TAP tests even if # TESTS is also specified if [ -z "$PROVE_TESTS" ] && [ -z "${SKIPS}" ] then # Exit early if we are running with TESTS=expr if [ -n "$TESTS" ] then exit 0 fi FINAL_TESTS=$(ls -1 t/*.pl 2>/dev/null) elif [ -z "$PROVE_TESTS" ] && [ -n "${SKIPS}" ] then ALL_TESTS=$(ls -1 t/*.pl 2>/dev/null) FILTERED_TESTS="" # disable path expansion to make SKIPS='*' work set -f # to support wildcards in SKIPS we match the SKIPS # list against the actual list of tests for test_name in ${ALL_TESTS}; do for test_pattern in ${SKIPS}; do # shellcheck disable=SC2053 # We do want to match globs in $test_pattern here. if [[ $test_name == t/${test_pattern}.pl ]]; then continue 2 fi done FILTERED_TESTS="${FILTERED_TESTS}\n${test_name}" done FINAL_TESTS=$(echo -e "${FILTERED_TESTS}" | tr '\n' ' ' | sed -e 's/^ *//') else FINAL_TESTS=$PROVE_TESTS fi if [ -z "$FINAL_TESTS" ] then echo "No TAP tests to run for the current configuration, skipping..." exit 0; fi PG_VERSION_MAJOR=${PG_VERSION_MAJOR} ${PROVE} \ -I "${SRC_DIR}/src/test/perl" \ -I "${CM_SRC_DIR}/test/perl" \ -I "${PG_LIBDIR}/pgxs/src/test/perl" \ -I "${PG_PKGLIBDIR}/pgxs/src/test/perl" \ -I "${PG_LIBDIR}/postgresql/pgxs/src/test/perl" \ $FINAL_TESTS ================================================ FILE: test/pg_regress.sh ================================================ #!/usr/bin/env bash # shellcheck disable=SC2053 # Wrapper around pg_regress and pg_isolation_regress to be able to control the schedule with environment variables # # The following control variables are supported: # # TESTS only run tests from this list # IGNORES failure of tests in this list will not lead to test failure # SKIPS tests from this list are not run # # In TESTS you may use wildcards to match multiple test names # TESTS="compression*" will match all tests whose name starts with compression # TESTS="*compression*" will match all tests that have compression anywhere in the name # Wildcard matching also applies to version specific tests so compression-13 # would also be matched by those patterns. CURRENT_DIR=$(dirname $0) EXE_DIR=${EXE_DIR:-${CURRENT_DIR}} PG_REGRESS=${PG_REGRESS:-pg_regress} PG_REGRESS_DIFF_OPTS=-u TEST_SCHEDULE=${TEST_SCHEDULE:-} TEMP_SCHEDULE=${CURRENT_DIR}/temp_schedule SCHEDULE= TESTS=${TESTS:-} IGNORES=${IGNORES:-} SKIPS=${SKIPS:-} # PG_BINDIR is passed from CMake via environment PSQL="${PSQL:-${PG_BINDIR}/psql} -X" # Prevent any .psqlrc files from being executed during the tests PG_VERSION_MAJOR=$(${PSQL} --version | awk '{split($3,v,"[.a-z]"); print v[1]}') # check if test matches any of the patterns in a list # $1 list of patterns or test names # $2 test name # we use == intentionally and not =~ because the pattern syntax differs between # those two and == allows for simpler patterns. With == the pattern to match # all bgw tests would be "*bgw*" while with =~ it would be ".*bgw.*" matches() { for pattern in $1; do if [[ $2 == $pattern ]]; then return 0 fi done return 1 } if [[ -z ${TEST_SCHEDULE} ]]; then echo "No test schedule supplied please set TEST_SCHEDULE" exit 1; fi # PG16 removed the `ignore` feature from `pg_regress` # so as an workaround if we have any IGNORES entry then # we merge it together with SKIPS and cleanup the IGNORES # https://github.com/postgres/postgres/commit/bd8d453e9b5f8b632a400a9e796fc041aed76d82 if [[ ${PG_VERSION_MAJOR} -ge 16 ]]; then if [[ -n ${IGNORES} ]]; then if [[ -n ${SKIPS} ]]; then SKIPS="${SKIPS} ${IGNORES}" else SKIPS="${IGNORES}" fi IGNORES="" fi fi echo "TESTS ${TESTS}" if [[ ${PG_VERSION_MAJOR} -lt 16 ]]; then echo "IGNORES ${IGNORES}" fi echo "SKIPS ${SKIPS}" if [[ -z ${TESTS} ]] && [[ -z ${SKIPS} ]] && [[ -z ${IGNORES} ]]; then # no filter variables set # nothing to do here and we can use the cmake generated schedule SCHEDULE=${TEST_SCHEDULE} elif [[ -z ${TESTS} && ( -n ${SKIPS} || -n ${IGNORES} ) ]]; then # If we only have IGNORES or SKIPS we can use the cmake created schedule # and just prepend ignore lines for the tests whose result should be # ignored and strip out the skipped tests. This will allow us to retain # the parallel groupings from the supplied schedule. echo > ${TEMP_SCHEDULE} ALL_TESTS=$(grep -a '^test: ' ${TEST_SCHEDULE} | sed -e 's!^test: !!' |tr '\n' ' ') # to support wildcards in IGNORES we match the IGNORES # list against the actual list of tests if [[ -n ${IGNORES} ]]; then for test_pattern in ${IGNORES}; do for test_name in ${ALL_TESTS}; do if [[ -n ${test_name} ]] && [[ $test_name == $test_pattern ]]; then echo "ignore: ${test_name}" >> ${TEMP_SCHEDULE} fi done for test_name in ${SKIPS} do if [[ -n ${test_name} && ${test_name} == ${test_pattern} ]] then echo "The ignored pattern '${test_name}' matches the skipped pattern '${test_pattern}'. This is not allowed." exit 1 fi done done fi cat ${TEST_SCHEDULE} >> ${TEMP_SCHEDULE} # to support wildcards in SKIPS we match the SKIPS # list against the actual list of tests if [[ -n ${SKIPS} ]]; then for test_pattern in ${SKIPS}; do for test_name in ${ALL_TESTS}; do if [[ $test_name == $test_pattern ]]; then sed -e "s!^test:\s*${test_name}\s*\$!!" -i.backup ${TEMP_SCHEDULE} sed -e "s!\b${test_name}\b!!" -i.backup ${TEMP_SCHEDULE} fi done done fi SCHEDULE=${TEMP_SCHEDULE} else # TESTS was specified so we need to create a new schedule based on that ALL_TESTS=$(grep -a '^test: ' ${TEST_SCHEDULE} | sed -e 's!^test: !!' |tr '\n' ' ') if [[ -z "${TESTS}" ]]; then TESTS=${ALL_TESTS} fi # build new test list in current_tests removing entries in SKIPS and # validating against schedule as TESTS might contain tests from # multiple suites and not apply to current run current_tests="" for test_pattern in ${TESTS}; do for test_name in ${ALL_TESTS}; do if ! matches "${SKIPS}" "${test_name}"; then if [[ $test_name == $test_pattern ]]; then current_tests="${current_tests} ${test_name}" elif [[ $test_name =~ ^${test_pattern}-[1-9][0-9]$ ]]; then current_tests="${current_tests} ${test_name}" fi fi done done # if none of the tests survived filtering we can exit early if [[ -z "${current_tests}" ]]; then exit 0 fi current_tests=$(echo "${current_tests}" | tr ' ' '\n' | sort) TESTS=${current_tests} echo > ${TEMP_SCHEDULE} # to support wildcards in IGNORES we match the IGNORES # list against the actual list of tests for test_pattern in ${IGNORES}; do for test_name in ${ALL_TESTS}; do if ! matches "${SKIPS}" "${test_name}"; then if [[ $test_name == $test_pattern ]]; then echo "ignore: ${test_name}" >> ${TEMP_SCHEDULE} fi fi done done for t in ${TESTS}; do echo "test: ${t}" >> ${TEMP_SCHEDULE} done SCHEDULE=${TEMP_SCHEDULE} fi function cleanup() { rm -rf ${EXE_DIR}/sql/dump rm -rf ${TEST_TABLESPACE1_PREFIX} rm -rf ${TEST_TABLESPACE2_PREFIX} rm -rf ${TEST_TABLESPACE3_PREFIX} rm -f ${TEMP_SCHEDULE} cat <<EOF | ${PSQL} -U ${USER} -h ${TEST_PGHOST} -p ${TEST_PGPORT} -d template1 >/dev/null 2>&1 DROP TABLESPACE IF EXISTS tablespace1; DROP TABLESPACE IF EXISTS tablespace2; DROP TABLESPACE IF EXISTS tablespace3; EOF rm -rf ${TEST_OUTPUT_DIR}/.pg_init } trap cleanup EXIT # Generating a prefix directory for all test tablespaces. This should # be used to build a full path for the tablespace. Note that we # terminate the prefix with the directory separator so that we can # easily generate paths independent of the OS. # # This mktemp line will work on both OSX and GNU systems TEST_TABLESPACE1_PREFIX=${TEST_TABLESPACE1_PREFIX:-$(mktemp -d 2>/dev/null || mktemp -d -t 'timescaledb_regress')/} TEST_TABLESPACE2_PREFIX=${TEST_TABLESPACE2_PREFIX:-$(mktemp -d 2>/dev/null || mktemp -d -t 'timescaledb_regress')/} TEST_TABLESPACE3_PREFIX=${TEST_TABLESPACE3_PREFIX:-$(mktemp -d 2>/dev/null || mktemp -d -t 'timescaledb_regress')/} # Creating some defaults for transitioning tests to use the prefix. TEST_TABLESPACE1_PATH=${TEST_TABLESPACE1_PATH:-${TEST_TABLESPACE1_PREFIX}_default} TEST_TABLESPACE2_PATH=${TEST_TABLESPACE2_PATH:-${TEST_TABLESPACE2_PREFIX}_default} TEST_TABLESPACE3_PATH=${TEST_TABLESPACE3_PATH:-${TEST_TABLESPACE3_PREFIX}_default} mkdir -p $TEST_TABLESPACE1_PATH $TEST_TABLESPACE2_PATH $TEST_TABLESPACE3_PATH export TEST_TABLESPACE1_PREFIX TEST_TABLESPACE2_PREFIX TEST_TABLESPACE3_PREFIX export TEST_TABLESPACE1_PATH TEST_TABLESPACE2_PATH TEST_TABLESPACE3_PATH rm -rf ${TEST_OUTPUT_DIR}/.pg_init mkdir -p ${EXE_DIR}/sql/dump export PG_REGRESS_DIFF_OPTS # If so configured, we run the tests with faketime utility to change the current # time. This helps catch the mistakes with using the current time in test # references. We can't do this for isolation tests because this breaks the # waiting mechanism in isolation tester. if [[ "${PG_REGRESS_USE_FAKETIME}" == "1" ]] then PG_REGRESS_FAKETIME="${FAKETIME}" fi PG_REGRESS_OPTS="${PG_REGRESS_OPTS} --schedule=${SCHEDULE}" ${PG_REGRESS_FAKETIME} ${PG_REGRESS} "$@" ${PG_REGRESS_OPTS} ================================================ FILE: test/pgpass.conf.in ================================================ # Only TEST_ROLE_2 should have password in passfile # TEST_ROLE_3 needs to rely on, e.g., user mappings in the DB *:*:*:@TEST_ROLE_2@:@TEST_ROLE_2_PASS@ ================================================ FILE: test/pgtest/CMakeLists.txt ================================================ set(PG_REGRESS_DIR ${PG_SOURCE_DIR}/src/test/regress CACHE PATH "Path to PostgreSQL's regress directory") # input and output directory got removed in PG15 set(PGTEST_DIRS ${PG_REGRESS_DIR}/data ${PG_REGRESS_DIR}/sql ${PG_REGRESS_DIR}/expected) if(EXISTS ${PG_REGRESS_DIR}/input AND EXISTS ${PG_REGRESS_DIR}/output) list(APPEND PGTEST_DIRS ${PG_REGRESS_DIR}/input ${PG_REGRESS_DIR}/output) endif() # Copy the input and output files from PostgreSQL's test suite. The test suite # generates some SQL scripts and output files from template source files and # require directories to be colocated file(COPY ${PGTEST_DIRS} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) file(READ ${PG_REGRESS_DIR}/parallel_schedule PG_TEST_SCHEDULE) # create directory for tablespace test file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/testtablespace) # Tests to ignore set(PG_IGNORE_TESTS amutils database event_trigger jsonb_jsonpath opr_sanity sanity_check type_sanity create_am # Ignoring because it spawns different number of workers in different # versions. select_parallel psql) # Modify the test schedule to ignore some tests foreach(IGNORE_TEST ${PG_IGNORE_TESTS}) # ignored schedules was removed in PG16 # https://github.com/postgres/postgres/commit/bd8d453e9b5f8b632a400a9e796fc041aed76d82 if(${PG_VERSION_MAJOR} LESS "16") string(CONCAT PG_TEST_SCHEDULE "ignore: ${IGNORE_TEST}\n" ${PG_TEST_SCHEDULE}) else() # remove the ignored test from the schedule string(REPLACE "test: ${IGNORE_TEST}\n" "" PG_TEST_SCHEDULE "${PG_TEST_SCHEDULE}") string(REPLACE " ${IGNORE_TEST} " " " PG_TEST_SCHEDULE "${PG_TEST_SCHEDULE}") string(REPLACE " ${IGNORE_TEST}\n" "\n" PG_TEST_SCHEDULE "${PG_TEST_SCHEDULE}") endif() endforeach(IGNORE_TEST) # Write the final test schedule file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/schedule ${PG_TEST_SCHEDULE}) # Need --dlpath set to PostgreSQL's test directory so that the tests can load # libraries there set(PG_REGRESS_OPTS_PGTEST --schedule=${CMAKE_CURRENT_BINARY_DIR}/schedule --load-extension=timescaledb --dlpath=${PG_REGRESS_DIR}) add_custom_target( pginstallcheck COMMAND ${PG_REGRESS} ${PG_REGRESS_OPTS_BASE} ${PG_REGRESS_OPTS_PGTEST} ${PG_REGRESS_OPTS_TEMP_INSTANCE_PGTEST} USES_TERMINAL) add_custom_target( pginstallchecklocal COMMAND ${PG_REGRESS} ${PG_REGRESS_OPTS_BASE} ${PG_REGRESS_OPTS_PGTEST} ${PG_REGRESS_OPTS_LOCAL_INSTANCE} USES_TERMINAL) ================================================ FILE: test/pgtest/README.md ================================================ # PostgreSQL tests for TimescaleDB The CMake configuration within this directory makes it possible to run the standard PostgreSQL test suite with the TimescaleDB extension loaded. This is useful to ensure that TimescaleDBs modifications planner and DDL hooks are compatible with standard PostgreSQL. ## Running The configuration within adds a new CMake target, `pginstallcheck`, that allows running the PostgreSQL test suite using a modified test schedule. The target requires access to the PostgreSQL source code, which can be configured via the `PG_SOURCE_DIR` CMake variable. The source tree needs to be compiled, at least the `src/test/regress` directory. If the path to a PostgreSQL source tree is not auto-detected, this variable can be set manually to point to the right location. ``` # In top-level directory of a TimescaleDB source tree $ mkdir build && cd build $ cmake -DPG_SOURCE_DIR=<path/to/pg/source> .. ``` Once CMake is correctly configured, run: ``` $ make pginstallcheck ``` ================================================ FILE: test/pgtest.conf.in ================================================ # postgresql.conf settings for PostgreSQL test suite shared_preload_libraries=timescaledb @TELEMETRY_DEFAULT_SETTING@ ================================================ FILE: test/postgres-asan-instrumentation-PG18GE.patch ================================================ diff --git a/src/backend/utils/misc/stack_depth.c b/src/backend/utils/misc/stack_depth.c index 8f7cf531fbc..2f7b1cebbe6 100644 --- a/src/backend/utils/misc/stack_depth.c +++ b/src/backend/utils/misc/stack_depth.c @@ -108,6 +108,12 @@ check_stack_depth(void) bool stack_is_too_deep(void) { + /* + * Pointer arithmetics to determine stack depth doesn't work under + * AddressSanitizer. + */ + return false; + char stack_top_loc; ssize_t stack_depth; diff --git a/src/include/utils/memdebug.h b/src/include/utils/memdebug.h index e88b4c6e8e..4ccbbf0146 100644 --- a/src/include/utils/memdebug.h +++ b/src/include/utils/memdebug.h @@ -19,6 +19,31 @@ #ifdef USE_VALGRIND #include <valgrind/memcheck.h> + +#elif __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__) + +#include <sanitizer/asan_interface.h> + +#define VALGRIND_MAKE_MEM_DEFINED(addr, size) \ + ASAN_UNPOISON_MEMORY_REGION(addr, size) + +#define VALGRIND_MAKE_MEM_NOACCESS(addr, size) \ + ASAN_POISON_MEMORY_REGION(addr, size) + +#define VALGRIND_MAKE_MEM_UNDEFINED(addr, size) \ + ASAN_UNPOISON_MEMORY_REGION(addr, size) + +#define VALGRIND_MEMPOOL_ALLOC(context, addr, size) \ + ASAN_UNPOISON_MEMORY_REGION(addr, size) + +#define VALGRIND_MEMPOOL_FREE(context, addr) \ + ASAN_POISON_MEMORY_REGION(addr, 1 /* Length unknown, poison first byte. */) + +#define VALGRIND_CHECK_MEM_IS_DEFINED(addr, size) do {} while (0) +#define VALGRIND_CREATE_MEMPOOL(context, redzones, zeroed) do {} while (0) +#define VALGRIND_DESTROY_MEMPOOL(context) do {} while (0) +#define VALGRIND_MEMPOOL_CHANGE(context, optr, nptr, size) do {} while (0) + #else #define VALGRIND_CHECK_MEM_IS_DEFINED(addr, size) do {} while (0) #define VALGRIND_CREATE_MEMPOOL(context, redzones, zeroed) do {} while (0) ================================================ FILE: test/postgres-asan-instrumentation.patch ================================================ diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 2c50575b37..11b6c688c7 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -3492,6 +4476,12 @@ check_stack_depth(void) bool stack_is_too_deep(void) { + /* + * Pointer arithmetics to determine stack depth doesn't work under + * AddressSanitizer. + */ + return false; + char stack_top_loc; long stack_depth; diff --git a/src/include/utils/memdebug.h b/src/include/utils/memdebug.h index e88b4c6e8e..4ccbbf0146 100644 --- a/src/include/utils/memdebug.h +++ b/src/include/utils/memdebug.h @@ -19,6 +19,31 @@ #ifdef USE_VALGRIND #include <valgrind/memcheck.h> + +#elif __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__) + +#include <sanitizer/asan_interface.h> + +#define VALGRIND_MAKE_MEM_DEFINED(addr, size) \ + ASAN_UNPOISON_MEMORY_REGION(addr, size) + +#define VALGRIND_MAKE_MEM_NOACCESS(addr, size) \ + ASAN_POISON_MEMORY_REGION(addr, size) + +#define VALGRIND_MAKE_MEM_UNDEFINED(addr, size) \ + ASAN_UNPOISON_MEMORY_REGION(addr, size) + +#define VALGRIND_MEMPOOL_ALLOC(context, addr, size) \ + ASAN_UNPOISON_MEMORY_REGION(addr, size) + +#define VALGRIND_MEMPOOL_FREE(context, addr) \ + ASAN_POISON_MEMORY_REGION(addr, 1 /* Length unknown, poison first byte. */) + +#define VALGRIND_CHECK_MEM_IS_DEFINED(addr, size) do {} while (0) +#define VALGRIND_CREATE_MEMPOOL(context, redzones, zeroed) do {} while (0) +#define VALGRIND_DESTROY_MEMPOOL(context) do {} while (0) +#define VALGRIND_MEMPOOL_CHANGE(context, optr, nptr, size) do {} while (0) + #else #define VALGRIND_CHECK_MEM_IS_DEFINED(addr, size) do {} while (0) #define VALGRIND_CREATE_MEMPOOL(context, redzones, zeroed) do {} while (0) ================================================ FILE: test/postgresql.conf.in ================================================ # NOTE: any changes here require changes to tsl/test/postgresql.conf. Its prefix # must be the same as this file. autovacuum=true datestyle='Postgres, MDY' log_destination='jsonlog,stderr' log_directory='@TEST_PG_LOG_DIRECTORY@' log_filename='postmaster.log' log_line_prefix='%m: %u [%p] %d ' logging_collector=true max_worker_processes=24 random_page_cost=1.0 shared_preload_libraries=timescaledb timescaledb.last_tuned='1971-02-03 04:05:06.789012 -0300' timescaledb.last_tuned_version='0.0.1' timescaledb.passfile='@TEST_PASSFILE@' timescaledb_telemetry.cloud='ci' timezone='US/Pacific' # Set extra_float_digits=0 to retain the pre PG12 rounding behaviour # of floating point numbers, which are needed to make our tests work. extra_float_digits=0 @TELEMETRY_DEFAULT_SETTING@ timescaledb.license='apache' timescaledb.enable_compression_ratio_warnings=false # Some tests use logical entries in WAL wal_level = logical ================================================ FILE: test/runner.sh ================================================ #!/usr/bin/env bash set -u set -e CURRENT_DIR=$(dirname $0) EXE_DIR=${EXE_DIR:-${CURRENT_DIR}} PG_REGRESS_PSQL=$1 PSQL=${PSQL:-$PG_REGRESS_PSQL} PSQL="${PSQL} -X" # Prevent any .psqlrc files from being executed during the tests TEST_PGUSER=${TEST_PGUSER:-postgres} TEST_INPUT_DIR=${TEST_INPUT_DIR:-${EXE_DIR}} TEST_OUTPUT_DIR=${TEST_OUTPUT_DIR:-${EXE_DIR}} TEST_SUPPORT_FILE=${CURRENT_DIR}/sql/utils/testsupport.sql TEST_SUPPORT_FILE_INIT=${CURRENT_DIR}/sql/utils/testsupport_init.sql TEST_TIMEOUT=${TEST_TIMEOUT:-120} # PGAPPNAME will be 'pg_regress/test' so we cut off the prefix # to get the name of the test CURRENT_TEST=${PGAPPNAME##pg_regress/} # Since different PG version tests cannot run in parallel in the same instance, # we remove the trailing version suffix to get a good symbol that can be # used as identifier as well. TEST_DBNAME="db_${CURRENT_TEST%%-[0-9][0-9]}" # Read the extension version from version.config read -r VERSION < ${CURRENT_DIR}/../version.config EXT_VERSION=${VERSION##version = } # on macos check if proper timeout is available if [ "$(uname)" == "Darwin" ]; then if ! command -v gtimeout >/dev/null 2>&1 then TIMEOUT_CMD="" else TIMEOUT_CMD="gtimeout -v ${TEST_TIMEOUT}s" fi else TIMEOUT_CMD="timeout -v ${TEST_TIMEOUT}s" fi #docker doesn't set user USER=${USER:-$(whoami)} TEST_SPINWAIT_ITERS=${TEST_SPINWAIT_ITERS:-1000} TEST_ROLE_SUPERUSER=${TEST_ROLE_SUPERUSER:-super_user} TEST_ROLE_DEFAULT_PERM_USER=${TEST_ROLE_DEFAULT_PERM_USER:-default_perm_user} TEST_ROLE_DEFAULT_PERM_USER_2=${TEST_ROLE_DEFAULT_PERM_USER_2:-default_perm_user_2} # Users for clustering. These users have password auth enabled in pg_hba.conf TEST_ROLE_1=${TEST_ROLE_1:-test_role_1} TEST_ROLE_2=${TEST_ROLE_2:-test_role_2} TEST_ROLE_2_PASS=${TEST_ROLE_2_PASS:-pass} TEST_ROLE_3=${TEST_ROLE_3:-test_role_3} TEST_ROLE_3_PASS=${TEST_ROLE_3_PASS:-pass} TEST_ROLE_4=${TEST_ROLE_4:-test_role_4} TEST_ROLE_4_PASS=${TEST_ROLE_4_PASS:-pass} TEST_ROLE_READ_ONLY=${TEST_ROLE_READ_ONLY:-test_role_read_only} shift # Drop test database and make it less verbose in case of dropping a # distributed database. function cleanup { cat <<EOF | ${PSQL} "$@" -U $TEST_ROLE_SUPERUSER -d postgres -v ECHO=none >/dev/null 2>&1 SET client_min_messages=ERROR; DROP DATABASE "${TEST_DBNAME}"; EOF } trap cleanup EXIT # setup clusterwide settings on first run # we use mkdir here because it is an atomic operation unlike existence of a lockfile # where creating and checking are 2 separate operations if mkdir ${TEST_OUTPUT_DIR}/.pg_init 2>/dev/null; then ${PSQL} "$@" -U ${USER} -d template1 -v ECHO=none >/dev/null 2>&1 <<EOF SET client_min_messages=ERROR; DO \$\$ BEGIN IF current_setting('server_version_num')::int >= 150000 THEN GRANT CREATE ON SCHEMA public TO ${TEST_PGUSER}; GRANT CREATE ON SCHEMA public TO ${TEST_ROLE_DEFAULT_PERM_USER}; GRANT CREATE ON SCHEMA public TO ${TEST_ROLE_DEFAULT_PERM_USER_2}; GRANT CREATE ON SCHEMA public TO ${TEST_ROLE_1}; GRANT CREATE ON SCHEMA public TO ${TEST_ROLE_2}; GRANT CREATE ON SCHEMA public TO ${TEST_ROLE_3}; GRANT CREATE ON SCHEMA public TO ${TEST_ROLE_4}; END IF; END \$\$ LANGUAGE PLPGSQL; ALTER USER ${TEST_ROLE_SUPERUSER} WITH SUPERUSER; ALTER USER ${TEST_ROLE_1} WITH CREATEDB CREATEROLE; ALTER USER ${TEST_ROLE_2} WITH CREATEDB PASSWORD '${TEST_ROLE_2_PASS}'; ALTER USER ${TEST_ROLE_3} WITH CREATEDB PASSWORD '${TEST_ROLE_3_PASS}'; ALTER USER ${TEST_ROLE_4} WITH CREATEDB PASSWORD '${TEST_ROLE_4_PASS}'; EOF ${PSQL} "$@" -U $TEST_ROLE_SUPERUSER -d template1 \ -v ECHO=none \ -v MODULE_PATHNAME="'timescaledb-${EXT_VERSION}'" \ -v TSL_MODULE_PATHNAME="'timescaledb-tsl-${EXT_VERSION}'" \ -v TEST_SPINWAIT_ITERS=${TEST_SPINWAIT_ITERS} \ -f ${TEST_SUPPORT_FILE} >/dev/null 2>&1 ${PSQL} "$@" -U ${USER} -d postgres -v ECHO=none -c "ALTER USER ${TEST_ROLE_SUPERUSER} WITH SUPERUSER;" >/dev/null touch ${TEST_OUTPUT_DIR}/.pg_init/done fi # we need to wait for cluster setup to finish cause with parallel schedule # multiple instances will be running and mkdir will only succeed on the first runner while [ ! -f ${TEST_OUTPUT_DIR}/.pg_init/done ]; do sleep 0.2; done cd ${EXE_DIR}/sql # create database and install timescaledb ${PSQL} "$@" -U $TEST_ROLE_SUPERUSER -d postgres -v ECHO=none -c "CREATE DATABASE \"${TEST_DBNAME}\";" ${PSQL} "$@" -U $TEST_ROLE_SUPERUSER -d ${TEST_DBNAME} -v ECHO=none -c "SET client_min_messages=error; CREATE EXTENSION timescaledb;" ${PSQL} "$@" -U $TEST_ROLE_SUPERUSER -d ${TEST_DBNAME} \ -v ECHO=none \ -v MODULE_PATHNAME="'timescaledb-${EXT_VERSION}'" \ -v TSL_MODULE_PATHNAME="'timescaledb-tsl-${EXT_VERSION}'" \ -v TEST_SPINWAIT_ITERS=${TEST_SPINWAIT_ITERS} \ -f ${TEST_SUPPORT_FILE_INIT} >/dev/null 2>&1 export TEST_DBNAME # we strip out any output between <exclude_from_test></exclude_from_test> # and the part about memory usage in EXPLAIN ANALYZE output of Sort nodes # also ignore the Postgres rehashing catalog debug messages from 'src/backend/utils/cache/catcache.c' ${TIMEOUT_CMD} ${PSQL} -U ${TEST_PGUSER} \ -v ON_ERROR_STOP=1 \ -v VERBOSITY=terse \ -v ECHO=all \ -v TEST_DBNAME="${TEST_DBNAME}" \ -v TEST_TABLESPACE1_PREFIX=${TEST_TABLESPACE1_PREFIX} \ -v TEST_TABLESPACE2_PREFIX=${TEST_TABLESPACE2_PREFIX} \ -v TEST_TABLESPACE3_PREFIX=${TEST_TABLESPACE3_PREFIX} \ -v TEST_TABLESPACE1_PATH=\'${TEST_TABLESPACE1_PATH}\' \ -v TEST_TABLESPACE2_PATH=\'${TEST_TABLESPACE2_PATH}\' \ -v TEST_TABLESPACE3_PATH=\'${TEST_TABLESPACE3_PATH}\' \ -v TEST_INPUT_DIR=${TEST_INPUT_DIR} \ -v TEST_OUTPUT_DIR=${TEST_OUTPUT_DIR} \ -v TEST_SPINWAIT_ITERS=${TEST_SPINWAIT_ITERS} \ -v ROLE_SUPERUSER=${TEST_ROLE_SUPERUSER} \ -v ROLE_DEFAULT_PERM_USER=${TEST_ROLE_DEFAULT_PERM_USER} \ -v ROLE_DEFAULT_PERM_USER_2=${TEST_ROLE_DEFAULT_PERM_USER_2} \ -v ROLE_1=${TEST_ROLE_1} \ -v ROLE_2=${TEST_ROLE_2} \ -v ROLE_3=${TEST_ROLE_3} \ -v ROLE_4=${TEST_ROLE_4} \ -v ROLE_READ_ONLY=${TEST_ROLE_READ_ONLY} \ -v ROLE_2_PASS=${TEST_ROLE_2_PASS} \ -v ROLE_3_PASS=${TEST_ROLE_3_PASS} \ -v ROLE_4_PASS=${TEST_ROLE_4_PASS} \ -v MODULE_PATHNAME="'timescaledb-${EXT_VERSION}'" \ -v TSL_MODULE_PATHNAME="'timescaledb-tsl-${EXT_VERSION}'" \ -v TEST_SUPPORT_FILE=${TEST_SUPPORT_FILE} \ -v TEST_SUPPORT_FILE_INIT=${TEST_SUPPORT_FILE_INIT} \ "$@" -d ${TEST_DBNAME} 2>&1 | ${CURRENT_DIR}/runner_cleanup_output.sh ================================================ FILE: test/runner_cleanup_output.sh ================================================ #!/usr/bin/env bash set -u set -e RUNNER=${1:-""} sed -E -e '/<exclude_from_test>/,/<\/exclude_from_test>/d' \ -e 's! Disk: [0-9]+kB!!' \ -e 's! Memory: [0-9]+kB!!' \ -e 's! Memory Usage: [0-9]+kB!!' \ -e 's! Average Peak Memory: [0-9]+kB!!' \ -e 's!ERROR: permission denied for materialized view!ERROR: permission denied for view!' \ -e 's!"*_ts_meta_v2_bl[0-9A-Za-z]+_([_0-9A-Za-z]+)"*!regress-test-bloom_\1!g' \ -e 's/(actual rows=[0-9]+) /\1.00 /' \ -e '/^ ?\([0-9]+ row[s]?\)$/d' \ -e '/ +QUERY PLAN +/{N;s/ +QUERY PLAN +\n-+/--- QUERY PLAN ---/;}' \ -e '/Disabled: true/d' \ -e '/Heap Fetches: [0-9]+/d' \ -e '/Buckets: [0-9]\+/d' \ -e '/Index Searches: [0-9]+/d' \ -e '/Storage: Memory Maximum Storage: [0-9]+kB/d' \ -e '/Window: /d' \ -e '/Batches: [0-9]+/d' \ -e '/found [0-9]+ removable, [0-9]+ nonremovable row versions in [0-9]+ pages/d' | \ grep -av 'DEBUG: rehashing catalog cache id' | \ grep -av 'DEBUG: compacted fsync request queue from' | \ grep -av 'DEBUG: creating and filling new WAL file' | \ grep -av 'DEBUG: done creating and filling new WAL file' | \ grep -av 'DEBUG: flushed relation because a checkpoint occurred concurrently' | \ grep -av 'NOTICE: cancelling the background worker for job' | \ if [ "${RUNNER}" = "shared" ]; then \ sed -e 's!_[0-9]\{1,\}_[0-9]\{1,\}_chunk!_X_X_chunk!g'; \ else \ cat; \ fi | \ if [ "${RUNNER}" = "isolation" ]; then \ sed -e 's!_[0-9]\{1,\}_[0-9]\{1,\}_chunk!_X_X_chunk!g' \ -e 's!hypertable_[0-9]\{1,\}!hypertable_X!g' \ -e 's!constraint_[0-9]\{1,\}!constraint_X!g' \ -e 's!with OID [0-9]\{1,\}!with OID X!g'; \ else \ cat; \ fi ================================================ FILE: test/runner_isolation.sh ================================================ #!/usr/bin/env bash # # Wrapper for the PostgreSQL isolationtest runner. It replaces # the chunks IDs in the output of the tests by _X_X_. So, even # if the IDs change, the tests will not fail. ############################################################## set -e set -u CURRENT_DIR=$(dirname $0) ISOLATIONTEST=$1 shift # Note that removing the chunk numbers is not enough. The chunk numbers also # influence the alignment of the EXPLAIN output, so not only we have to replace # them, we also have to remove the "----"s and the trailing spaces. The aligned # output format in isolation tester is hardcoded, we cannot change it. Moreover, # the chunk numbers influence the names of indexes if they are long enough to be # truncated, so the only way to get a stable explain output is to run such a test # in a separate database. $ISOLATIONTEST "$@" | ${CURRENT_DIR}/runner_cleanup_output.sh "isolation" ================================================ FILE: test/runner_shared.sh ================================================ #!/usr/bin/env bash set -u set -e CURRENT_DIR=$(dirname $0) EXE_DIR=${EXE_DIR:-${CURRENT_DIR}} PG_REGRESS_PSQL=$1 PSQL=${PSQL:-$PG_REGRESS_PSQL} PSQL="${PSQL} -X" # Prevent any .psqlrc files from being executed during the tests TEST_PGUSER=${TEST_PGUSER:-postgres} TEST_INPUT_DIR=${TEST_INPUT_DIR:-${EXE_DIR}} TEST_OUTPUT_DIR=${TEST_OUTPUT_DIR:-${EXE_DIR}} TEST_SUPPORT_FILE=${CURRENT_DIR}/sql/utils/testsupport.sql TEST_SUPPORT_FILE_INIT=${CURRENT_DIR}/sql/utils/testsupport_init.sql TEST_TIMEOUT=${TEST_TIMEOUT:-120} # Read the extension version from version.config read -r VERSION < ${CURRENT_DIR}/../version.config EXT_VERSION=${VERSION##version = } # PGAPPNAME will be 'pg_regress/test' so we cut off the prefix # to get the name of the test TEST_BASE_NAME=${PGAPPNAME##pg_regress/} # if this is a versioned test our name will have version as suffix # so we cut off suffix to get base name if [[ ${TEST_BASE_NAME} == *-1[0-9] ]]; then TEST_BASE_NAME=${TEST_BASE_NAME%???} fi # on macos check if proper timeout is available if [ "$(uname)" == "Darwin" ]; then if ! command -v gtimeout >/dev/null 2>&1 then TIMEOUT_CMD="" else TIMEOUT_CMD="gtimeout -v ${TEST_TIMEOUT}s" fi else TIMEOUT_CMD="timeout -v ${TEST_TIMEOUT}s" fi #docker doesn't set user USER=${USER:-$(whoami)} TEST_ROLE_SUPERUSER=${TEST_ROLE_SUPERUSER:-super_user} TEST_ROLE_DEFAULT_PERM_USER=${TEST_ROLE_DEFAULT_PERM_USER:-default_perm_user} TEST_ROLE_DEFAULT_PERM_USER_2=${TEST_ROLE_DEFAULT_PERM_USER_2:-default_perm_user_2} shift # setup clusterwide settings on first run # we use mkdir here because it is an atomic operation unlike existence of a lockfile # where creating and checking are 2 separate operations if mkdir ${TEST_OUTPUT_DIR}/.pg_init 2>/dev/null; then ${PSQL} "$@" -U ${USER} -d postgres -v ECHO=none -c "ALTER USER ${TEST_ROLE_SUPERUSER} WITH SUPERUSER;" >/dev/null ${PSQL} -U ${USER} \ -v MODULE_PATHNAME="'timescaledb-${EXT_VERSION}'" \ -v ROLE_DEFAULT_PERM_USER=${TEST_ROLE_DEFAULT_PERM_USER} \ -v ROLE_DEFAULT_PERM_USER_2=${TEST_ROLE_DEFAULT_PERM_USER_2} \ -v ROLE_SUPERUSER=${TEST_ROLE_SUPERUSER} \ -v TEST_BASE_NAME=${TEST_BASE_NAME} \ -v TEST_DBNAME="${TEST_DBNAME}" \ -v TEST_INPUT_DIR=${TEST_INPUT_DIR} \ -v TEST_OUTPUT_DIR=${TEST_OUTPUT_DIR} \ -v TEST_SUPPORT_FILE=${TEST_SUPPORT_FILE} \ -v TEST_SUPPORT_FILE_INIT=${TEST_SUPPORT_FILE_INIT} \ -v TSL_MODULE_PATHNAME="'timescaledb-tsl-${EXT_VERSION}'" \ "$@" -d ${TEST_DBNAME} < ${TEST_INPUT_DIR}/shared/sql/include/shared_setup.sql >/dev/null touch ${TEST_OUTPUT_DIR}/.pg_init/done fi # we need to wait for cluster setup to finish cause with parallel schedule # multiple instances will be running and mkdir will only succeed on the first runner while [ ! -f ${TEST_OUTPUT_DIR}/.pg_init/done ]; do sleep 0.2; done cd ${EXE_DIR}/sql # we strip out any output between <exclude_from_test></exclude_from_test> # and the part about memory usage in EXPLAIN ANALYZE output of Sort nodes # also ignore the Postgres rehashing catalog debug messages from 'src/backend/utils/cache/catcache.c' ${TIMEOUT_CMD} ${PSQL} -U ${TEST_PGUSER} \ -v ON_ERROR_STOP=1 \ -v VERBOSITY=terse \ -v ECHO=all \ -v TEST_DBNAME="${TEST_DBNAME}" \ -v TEST_BASE_NAME=${TEST_BASE_NAME} \ -v TEST_INPUT_DIR=${TEST_INPUT_DIR} \ -v TEST_OUTPUT_DIR=${TEST_OUTPUT_DIR} \ -v ROLE_SUPERUSER=${TEST_ROLE_SUPERUSER} \ -v ROLE_DEFAULT_PERM_USER=${TEST_ROLE_DEFAULT_PERM_USER} \ -v ROLE_DEFAULT_PERM_USER_2=${TEST_ROLE_DEFAULT_PERM_USER_2} \ -v MODULE_PATHNAME="'timescaledb-${EXT_VERSION}'" \ -v TSL_MODULE_PATHNAME="'timescaledb-tsl-${EXT_VERSION}'" \ "$@" -d ${TEST_DBNAME} 2>&1 | ${CURRENT_DIR}/runner_cleanup_output.sh "shared" ================================================ FILE: test/sql/.gitignore ================================================ /agg_bookends-*.sql /alternate_users-*.sql /append-*.sql /cluster-*.sql /drop_owned-*.sql /grant_hypertable-*.sql /histogram_test-*.sql /insert-*.sql /insert_many-*.sql /null_exclusion-*.sql /parallel-*.sql /partitioning-*.sql /partitionwise-*.sql /plan_expand_hypertable-*.sql /plan_hashagg-*.sql /plan_hashagg_optimized-*.sql /plan_hypertable_cache-*.sql /plan_ordered_append-*.sql /rowsecurity-*.sql /timestamp-*.sql /ts_merge-*.sql /loader-*.sql ================================================ FILE: test/sql/CMakeLists.txt ================================================ include(GenerateTestSchedule) set(TEST_FILES alter.sql alternate_users.sql baserel_cache.sql catalog_corruption.sql chunk_adaptive.sql chunk_publication.sql chunk_utils.sql chunks.sql cluster.sql constraint.sql copy.sql copy_where.sql create_chunks.sql create_hypertable.sql create_table.sql create_table_with.sql cursor.sql ddl.sql ddl_errors.sql ddl_extra.sql debug_utils.sql delete.sql drop_extension.sql drop_hypertable.sql drop_rename_hypertable.sql drop_schema.sql dump_meta.sql extension_scripts.sql generated_as_identity.sql hash.sql index.sql information_views.sql insert_many.sql insert_returning.sql insert_single.sql lateral.sql merge.sql partition.sql partition_coercion.sql partitioning.sql pg_dump_unprivileged.sql pg_join.sql plain.sql plan_hypertable_inline.sql query.sql relocate_extension.sql reloptions.sql repair.sql size_utils.sql sort_optimization.sql sql_query.sql tableam.sql tableam_alter.sql tablespace.sql triggers.sql truncate.sql trusted_extension.sql update.sql upsert.sql util.sql uuid.sql vacuum.sql vacuum_parallel.sql version.sql license.sql) set(TEST_TEMPLATES agg_bookends.sql.in append.sql.in drop_owned.sql.in grant_hypertable.sql.in histogram_test.sql.in insert.sql.in null_exclusion.sql.in plan_hashagg.sql.in rowsecurity.sql.in parallel.sql.in partitionwise.sql.in plan_expand_hypertable.sql.in plan_ordered_append.sql.in timestamp.sql.in ts_merge.sql.in) # Loader test must distinguish between Apache and TSL builds so we parametrize # this here set(LOADER_TEST_FILE loader-${TEST_LICENSE_SUFFIX}) configure_file(loader.sql.in ${CMAKE_CURRENT_SOURCE_DIR}/${LOADER_TEST_FILE}.sql) # tests that fail or are unreliable when run in parallel bgw tests need to run # first otherwise they are flaky set(SOLO_TESTS alter alternate_users bgw_launcher chunk_utils index pg_dump_unprivileged tablespace telemetry net) list(APPEND SOLO_TESTS ${LOADER_TEST_FILE}) if(CMAKE_BUILD_TYPE MATCHES Debug) list( APPEND TEST_FILES bgw_launcher.sql c_unit_tests.sql copy_memory_usage.sql metadata.sql multi_transaction_index.sql net.sql partitioned_hypertable.sql pg_dump.sql symbol_conflict.sql test_tss_callbacks.sql test_utils.sql ${LOADER_TEST_FILE}.sql) if(USE_TELEMETRY) list(APPEND TEST_FILES telemetry.sql) endif() endif(CMAKE_BUILD_TYPE MATCHES Debug) if((${PG_VERSION_MAJOR} GREATER_EQUAL "17")) list(APPEND TEST_FILES tableam_alter_defaults.sql) endif() if((${PG_VERSION_MAJOR} GREATER_EQUAL "18")) list(APPEND TEST_FILES insert_returning_old_new.sql) endif() # only test custom type if we are in 64-bit architecture if("${CMAKE_SIZEOF_VOID_P}" STREQUAL "8") list(APPEND TEST_FILES custom_type.sql) endif() # Regression tests that vary with PostgreSQL version. Generated test files are # put in the original source directory since all tests must be in the same # directory. These files are updated when the template is edited, but not when # the output file is deleted. If the output is deleted either recreate it # manually, or rerun cmake on the root dir. foreach(TEMPLATE_FILE ${TEST_TEMPLATES}) string(LENGTH ${TEMPLATE_FILE} TEMPLATE_NAME_LEN) math(EXPR TEMPLATE_NAME_LEN ${TEMPLATE_NAME_LEN}-7) string(SUBSTRING ${TEMPLATE_FILE} 0 ${TEMPLATE_NAME_LEN} TEMPLATE) set(TEST_FILE ${TEMPLATE}-${TEST_VERSION_SUFFIX}.sql) configure_file(${TEMPLATE_FILE} ${CMAKE_CURRENT_SOURCE_DIR}/${TEST_FILE} @ONLY) list(APPEND TEST_FILES ${TEST_FILE}) endforeach(TEMPLATE_FILE) if(NOT TEST_GROUP_SIZE) set(PARALLEL_GROUP_SIZE 20) else() set(PARALLEL_GROUP_SIZE ${TEST_GROUP_SIZE}) endif() # Generate a test schedule for each configuration. generate_test_schedule( ${TEST_SCHEDULE} TEST_FILES ${TEST_FILES} SOLO ${SOLO_TESTS} GROUP_SIZE ${PARALLEL_GROUP_SIZE}) add_subdirectory(loader) ================================================ FILE: test/sql/agg_bookends.sql.in ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set TEST_BASE_NAME agg_bookends SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') as "TEST_LOAD_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') as "TEST_QUERY_NAME", format('%s/results/%s_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_OPTIMIZED", format('%s/results/%s_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_UNOPTIMIZED" \gset SELECT format('\! diff -u --label "Unoptimized result" --label "Optimized result" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') as "DIFF_CMD" \gset \set PREFIX 'EXPLAIN (analyze, buffers off, costs off, timing off, summary off)' \ir :TEST_LOAD_NAME \ir :TEST_QUERY_NAME -- we want test results as part of the output too to make sure we produce correct output \set PREFIX '' \ir :TEST_QUERY_NAME -- diff results with optimizations disabled and enabled \o :TEST_RESULTS_UNOPTIMIZED SET timescaledb.enable_optimizations TO false; \ir :TEST_QUERY_NAME \o \o :TEST_RESULTS_OPTIMIZED SET timescaledb.enable_optimizations TO true; \ir :TEST_QUERY_NAME \o :DIFF_CMD -- Test partial aggregation CREATE TABLE partial_aggregation (time timestamptz NOT NULL, quantity numeric, longvalue text); SELECT schema_name, table_name, created FROM create_hypertable('partial_aggregation', 'time'); INSERT INTO partial_aggregation VALUES('2018-01-20T09:00:43', NULL, NULL); INSERT INTO partial_aggregation VALUES('2018-01-20T09:00:44', NULL, NULL); INSERT INTO partial_aggregation VALUES('2019-01-20T09:00:43', 1, 'hello'); INSERT INTO partial_aggregation VALUES('2019-01-20T09:00:44', 2, 'world'); INSERT INTO partial_aggregation VALUES('2020-01-20T09:00:43', 3.1, 'some1'); INSERT INTO partial_aggregation VALUES('2020-01-20T09:00:44', 3.2, 'more1'); INSERT INTO partial_aggregation VALUES('2021-01-20T09:00:43', 3.3, 'some2'); INSERT INTO partial_aggregation VALUES('2021-01-20T09:00:44', 3.4, 'more2'); INSERT INTO partial_aggregation VALUES('2022-01-20T09:00:43', 4, 'word1'); INSERT INTO partial_aggregation VALUES('2022-01-20T09:00:44', 5, 'word2'); INSERT INTO partial_aggregation VALUES('2023-01-20T09:00:43', 6, 'word3'); INSERT INTO partial_aggregation VALUES('2023-01-20T09:00:44', 7, 'word4'); -- Use enable_partitionwise_aggregate to create partial aggregates per chunk SET enable_partitionwise_aggregate = ON; SELECT format('SELECT %3$s, %1$s FROM partial_aggregation WHERE %2$s GROUP BY %3$s ORDER BY 1, 2;', function, condition, grouping) FROM unnest(array[ 'first(time, quantity), last(time, quantity)', 'last(longvalue, quantity)', 'last(quantity, longvalue)', 'last(quantity, time)', 'last(time, longvalue)']) AS function, unnest(array[ 'true', $$time < '2021-01-01'$$, 'quantity is null', 'quantity is not null', 'quantity >= 4']) AS condition, unnest(array[ '777::text' /* dummy grouping column */, 'longvalue', 'quantity', $$time_bucket('1 year', time)$$, $$time_bucket('3 year', time)$$]) AS grouping \gexec SET enable_partitionwise_aggregate = OFF; ================================================ FILE: test/sql/alter.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Set this variable to avoid using a hard-coded path each time query -- results are compared \set QUERY_RESULT_TEST_EQUAL_RELPATH 'include/query_result_test_equal.sql' -- DROP a table's column before making it a hypertable CREATE TABLE alter_before(id serial, time timestamp, temp float, colorid integer, notes text, notes_2 text); ALTER TABLE alter_before DROP COLUMN id; ALTER TABLE alter_before ALTER COLUMN temp SET (n_distinct = 10); ALTER TABLE alter_before ALTER COLUMN colorid SET (n_distinct = 11); ALTER TABLE alter_before ALTER COLUMN colorid RESET (n_distinct); ALTER TABLE alter_before ALTER COLUMN temp SET STATISTICS 100; ALTER TABLE alter_before ALTER COLUMN notes SET STORAGE EXTERNAL; SELECT create_hypertable('alter_before', 'time', chunk_time_interval => 2628000000000); -- Test error hint for invalid timescaledb options on ALTER TABLE \set ON_ERROR_STOP 0 -- Invalid timescaledb option should show hint with valid options \set VERBOSITY default ALTER TABLE alter_before SET (tsdb.invalid_option = true); ALTER TABLE alter_before SET (timescaledb.nonexistent = false); \set ON_ERROR_STOP 1 \set VERBOSITY terse INSERT INTO alter_before VALUES ('2017-03-22T09:18:22', 23.5, 1); SELECT * FROM alter_before; -- Show that deleted column is marked as dropped and that attnums are -- now different for the root table and the chunk -- PG17 made attstattarget NULLABLE and changed the default from -1 to NULL SELECT c.relname, a.attname, a.attnum, a.attoptions, CASE WHEN a.attstattarget = -1 OR (a.attisdropped AND a.attstattarget = 0) THEN NULL ELSE a.attstattarget END attstattarget, a.attstorage FROM pg_attribute a, pg_class c WHERE a.attrelid = c.oid AND (c.relname LIKE '_hyper_1%_chunk' OR c.relname = 'alter_before') AND a.attnum > 0 ORDER BY c.relname, a.attnum; -- DROP a table's column after making it a hypertable and having data CREATE TABLE alter_after(id serial, time timestamp, temp float, colorid integer, notes text, notes_2 text); SELECT create_hypertable('alter_after', 'time', chunk_time_interval => 2628000000000); -- Create first chunk INSERT INTO alter_after (time, temp, colorid) VALUES ('2017-03-22T09:18:22', 23.5, 1); ALTER TABLE alter_after DROP COLUMN id; ALTER TABLE alter_after ALTER COLUMN temp SET (n_distinct = 10); ALTER TABLE alter_after ALTER COLUMN colorid SET (n_distinct = 11); ALTER TABLE alter_after ALTER COLUMN colorid RESET (n_distinct); ALTER TABLE alter_after ALTER COLUMN colorid SET STATISTICS 101; ALTER TABLE alter_after ALTER COLUMN notes_2 SET STORAGE EXTERNAL; -- Creating new chunks after dropping a column should work just fine INSERT INTO alter_after VALUES ('2017-03-22T09:18:23', 21.5, 1), ('2017-05-22T09:18:22', 36.2, 2), ('2017-05-22T09:18:23', 15.2, 2); -- Make sure tuple conversion also works with COPY \COPY alter_after FROM 'data/alter.tsv' NULL AS ''; -- Data should look OK SELECT * FROM alter_after; -- Show that attnums are different for chunks created after DROP -- column SELECT c.relname, a.attname, a.attnum FROM pg_attribute a, pg_class c WHERE a.attrelid = c.oid AND (c.relname LIKE '_hyper_2%_chunk' OR c.relname = 'alter_after') AND a.attnum > 0 ORDER BY c.relname, a.attnum; -- Add an ID column again ALTER TABLE alter_after ADD COLUMN id serial; INSERT INTO alter_after (time, temp, colorid) VALUES ('2017-08-22T09:19:14', 12.5, 3); --test thing that we are allowed to do on chunks ALTER TABLE _timescaledb_internal._hyper_2_3_chunk ALTER COLUMN temp RESET (n_distinct); ALTER TABLE _timescaledb_internal._hyper_2_4_chunk ALTER COLUMN temp SET (n_distinct = 20); ALTER TABLE _timescaledb_internal._hyper_2_4_chunk ALTER COLUMN temp SET STATISTICS 201; ALTER TABLE _timescaledb_internal._hyper_2_4_chunk ALTER COLUMN notes SET STORAGE EXTERNAL; -- PG17 made attstattarget NULLABLE and changed the default from -1 to NULL SELECT c.relname, a.attname, a.attnum, a.attoptions, CASE WHEN a.attstattarget = -1 OR (a.attisdropped AND a.attstattarget = 0) THEN NULL ELSE a.attstattarget END attstattarget, a.attstorage FROM pg_attribute a, pg_class c WHERE a.attrelid = c.oid AND (c.relname LIKE '_hyper_2%_chunk' OR c.relname = 'alter_after') AND a.attnum > 0 ORDER BY c.relname, a.attnum; SELECT * FROM alter_after; -- test setting reloptions ALTER TABLE _timescaledb_internal._hyper_2_3_chunk SET (parallel_workers=2); ALTER TABLE _timescaledb_internal._hyper_2_4_chunk SET (parallel_workers=4); ALTER TABLE _timescaledb_internal._hyper_2_4_chunk RESET (parallel_workers); SELECT relname, reloptions FROM pg_class WHERE relname IN ('_hyper_2_3_chunk','_hyper_2_4_chunk'); -- Need superuser to ALTER chunks in _timescaledb_internal schema \c :TEST_DBNAME :ROLE_SUPERUSER SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk WHERE id = 2; -- Rename chunk ALTER TABLE _timescaledb_internal._hyper_2_2_chunk RENAME TO new_chunk_name; SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk WHERE id = 2; -- Set schema ALTER TABLE _timescaledb_internal.new_chunk_name SET SCHEMA public; SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk WHERE id = 2; -- Test that we cannot rename chunk columns \set ON_ERROR_STOP 0 ALTER TABLE public.new_chunk_name RENAME COLUMN time TO newtime; \set ON_ERROR_STOP 1 -- Test that we can set tablespace of a hypertable \c :TEST_DBNAME :ROLE_SUPERUSER SET client_min_messages = ERROR; DROP TABLESPACE IF EXISTS tablespace1; DROP TABLESPACE IF EXISTS tablespace2; SET client_min_messages = NOTICE; --test hypertable with tables space CREATE TABLESPACE tablespace1 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE1_PATH; CREATE TABLESPACE tablespace2 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE2_PATH; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- Test that we can directly change chunk tablespace ALTER TABLE public.new_chunk_name SET TABLESPACE tablespace1; SELECT tablespace FROM pg_tables WHERE tablename = 'new_chunk_name'; -- drop all tables to make checking the tests below easier DROP TABLE alter_before; DROP TABLE alter_after; -- should return 0 rows SELECT tablename, tablespace FROM pg_tables WHERE tablename = 'hyper_in_space' OR tablename LIKE '\_hyper\__\__\_chunk' ORDER BY tablename; CREATE TABLE hyper_in_space(time bigint, temp float, device int); SELECT create_hypertable('hyper_in_space', 'time', 'device', 4, chunk_time_interval=>1); INSERT INTO hyper_in_space(time, temp, device) VALUES (1, 20, 1); INSERT INTO hyper_in_space(time, temp, device) VALUES (3, 21, 2); INSERT INTO hyper_in_space(time, temp, device) VALUES (5, 23, 1); SELECT tablename FROM pg_tables WHERE tablespace = 'tablespace1' ORDER BY tablename; SET default_tablespace = tablespace1; -- should be inserted in tablespace1 which is now default INSERT INTO hyper_in_space(time, temp, device) VALUES (11, 24, 3); SELECT tablename, tablespace FROM pg_tables WHERE tablename = 'hyper_in_space' OR tablename LIKE '\_hyper\__\__\_chunk' ORDER BY tablename; SET default_tablespace TO DEFAULT; ALTER TABLE hyper_in_space SET TABLESPACE tablespace1; SELECT tablename FROM pg_tables WHERE tablespace = 'tablespace1' ORDER BY tablename; -- should be inserted in an existing chunk in the new tablespace, -- no new chunks INSERT INTO hyper_in_space(time, temp, device) VALUES (5, 27, 1); -- the new chunk should be create in the new tablespace INSERT INTO hyper_in_space(time, temp, device) VALUES (8, 24, 2); SELECT tablename, tablespace FROM pg_tables WHERE tablename = 'hyper_in_space' OR tablename LIKE '\_hyper\__\__\_chunk' ORDER BY tablename; -- should not fail (unlike attach_tablespace) ALTER TABLE hyper_in_space SET TABLESPACE tablespace1; \set ON_ERROR_STOP 0 -- not an empty tablespace DROP TABLESPACE tablespace1; \set ON_ERROR_STOP 1 -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'hyper_in_space\', 22)::NAME' \set QUERY2 'SELECT drop_chunks(\'hyper_in_space\', 22)::NAME' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT tablename, tablespace FROM pg_tables WHERE tablespace = 'tablespace1' ORDER BY tablename; \set ON_ERROR_STOP 0 -- should not be able to drop tablespace if a hypertable depends on it -- even when there are no chunks DROP TABLESPACE tablespace1; \set ON_ERROR_STOP 1 DROP TABLE hyper_in_space; CREATE TABLE hyper_in_space(time bigint, temp float, device int) TABLESPACE tablespace1; SELECT create_hypertable('hyper_in_space', 'time', 'device', 4, chunk_time_interval=>1); INSERT INTO hyper_in_space(time, temp, device) VALUES (1, 20, 1); INSERT INTO hyper_in_space(time, temp, device) VALUES (3, 21, 2); INSERT INTO hyper_in_space(time, temp, device) VALUES (5, 23, 1); SELECT tablename, tablespace FROM pg_tables WHERE tablename = 'hyper_in_space' OR tablename ~ '_hyper_\d+_\d+_chunk' ORDER BY tablename; SELECT attach_tablespace('tablespace2', 'hyper_in_space'); \set ON_ERROR_STOP 0 -- should fail as >1 tablespaces are attached ALTER TABLE hyper_in_space SET TABLESPACE tablespace1; \set ON_ERROR_STOP 1 SELECT detach_tablespace('tablespace2', 'hyper_in_space'); SELECT * FROM _timescaledb_catalog.tablespace; -- make sure when using ALTER TABLE, table spaces are not accumulated -- as in case of attach_tablespace -- should have one result SELECT * FROM _timescaledb_catalog.tablespace; ALTER TABLE hyper_in_space SET TABLESPACE tablespace2; -- should have one result SELECT * FROM _timescaledb_catalog.tablespace; ALTER TABLE hyper_in_space SET TABLESPACE tablespace1; -- should have one result, (same as the first in the block) SELECT * FROM _timescaledb_catalog.tablespace; SELECT tablename, tablespace FROM pg_tables WHERE tablename = 'hyper_in_space' OR tablename ~ '_hyper_\d+_\d+_chunk' ORDER BY tablename; -- attach tb2 <-> ALTER SET tb1 <-> detach tb1 should work SELECT detach_tablespace('tablespace1', 'hyper_in_space'); INSERT INTO hyper_in_space(time, temp, device) VALUES (5, 23, 1); INSERT INTO hyper_in_space(time, temp, device) VALUES (7, 23, 1); -- Since we have detached tablespace1 the new chunk should not be -- placed there. SELECT tablename, tablespace FROM pg_tables WHERE tablename = 'hyper_in_space' OR tablename ~ '_hyper_\d+_\d+_chunk' ORDER BY tablename; SELECT * FROM _timescaledb_catalog.tablespace; -- tablespace functions should handle the default tablespace just as they do others SELECT attach_tablespace('pg_default', 'hyper_in_space'); SELECT attach_tablespace('tablespace2', 'hyper_in_space'); SELECT tablename, tablespace FROM pg_tables WHERE tablename = 'hyper_in_space' OR tablename ~ '_hyper_\d+_\d+_chunk' ORDER BY tablename; SELECT * FROM _timescaledb_catalog.tablespace; INSERT INTO hyper_in_space(time, temp, device) VALUES (12, 22, 1); INSERT INTO hyper_in_space(time, temp, device) VALUES (13, 23, 3); SELECT tablename, tablespace FROM pg_tables WHERE tablename = 'hyper_in_space' OR tablename ~ '_hyper_\d+_\d+_chunk' ORDER BY tablename; SELECT detach_tablespace('pg_default', 'hyper_in_space'); ALTER TABLE hyper_in_space SET TABLESPACE pg_default; SELECT tablename, tablespace FROM pg_tables WHERE tablename = 'hyper_in_space' OR tablename ~ '_hyper_\d+_\d+_chunk' ORDER BY tablename; SELECT detach_tablespace('pg_default', 'hyper_in_space'); DROP TABLE hyper_in_space; -- test altering tablespace on index, issue #903 CREATE TABLE series( time timestamptz not null, device int, value float, CONSTRAINT series_pk PRIMARY KEY (time, device) USING INDEX TABLESPACE tablespace1); SELECT create_hypertable('series', 'time', create_default_indexes => FALSE); INSERT INTO series VALUES ('2019-04-21 10:12', 1, 1.01); CREATE INDEX series_value ON series (value, time) TABLESPACE tablespace2; SELECT schemaname, tablename, indexname, tablespace FROM pg_indexes WHERE indexname LIKE '%series%' ORDER BY indexname; ALTER INDEX series_pk SET TABLESPACE tablespace2; CREATE INDEX ON series (time) TABLESPACE tablespace1; ALTER INDEX series_value SET TABLESPACE pg_default; INSERT INTO series VALUES ('2019-04-29 10:12', 2, 1.31); SELECT schemaname, tablename, indexname, tablespace FROM pg_indexes WHERE indexname LIKE '%series%' ORDER BY indexname; DROP TABLE series; DROP TABLESPACE tablespace1; DROP TABLESPACE tablespace2; -- Make sure we handle ALTER SCHEMA RENAME for hypertable schemas \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA IF NOT EXISTS original_name; CREATE TABLE original_name.my_table ( date timestamp with time zone NOT NULL, quantity double precision ); SELECT create_hypertable('original_name.my_table','date'); INSERT INTO original_name.my_table (date, quantity) VALUES ('2018-07-04T21:00:00+00:00', 8); ALTER SCHEMA original_name RENAME TO new_name; DROP TABLE new_name.my_table; DROP SCHEMA new_name; -- Now make sure schema is renamed for multiple hypertables, but not hypertables not in the schema CREATE SCHEMA IF NOT EXISTS original_name; CREATE TABLE original_name.my_table ( date timestamp with time zone NOT NULL, quantity double precision ); CREATE TABLE original_name.my_table2 ( date timestamp with time zone NOT NULL, quantity double precision ); CREATE TABLE regular_table ( date timestamp with time zone NOT NULL, quantity double precision ); SELECT create_hypertable('original_name.my_table','date'); SELECT create_hypertable('original_name.my_table2','date'); SELECT create_hypertable('regular_table','date'); INSERT INTO original_name.my_table (date, quantity) VALUES ('2018-07-04T21:00:00+00:00', 8); INSERT INTO original_name.my_table2 (date, quantity) VALUES ('2018-07-04T21:00:00+00:00', 8); INSERT INTO regular_table (date, quantity) VALUES ('2018-07-04T21:00:00+00:00', 8); ALTER SCHEMA original_name RENAME TO new_name; DROP TABLE new_name.my_table; DROP TABLE new_name.my_table2; DROP TABLE regular_table; DROP SCHEMA new_name; -- These tables should also drop when we drop the whole schema CREATE SCHEMA IF NOT EXISTS original_name; CREATE TABLE original_name.my_table ( date timestamp with time zone NOT NULL, quantity double precision ); CREATE TABLE original_name.my_table2 ( date timestamp with time zone NOT NULL, quantity double precision ); SELECT create_hypertable('original_name.my_table','date'); SELECT create_hypertable('original_name.my_table2','date'); INSERT INTO original_name.my_table (date, quantity) VALUES ('2018-07-04T21:00:00+00:00', 8); INSERT INTO original_name.my_table2 (date, quantity) VALUES ('2018-07-04T21:00:00+00:00', 8); ALTER SCHEMA original_name RENAME TO new_name; DROP SCHEMA new_name CASCADE; SELECT * FROM test.relation WHERE schema = 'new_name'; -- Make sure we can't rename internal schemas \set ON_ERROR_STOP 0 ALTER SCHEMA _timescaledb_internal RENAME TO my_new_schema_name; ALTER SCHEMA _timescaledb_catalog RENAME TO my_new_schema_name; ALTER SCHEMA _timescaledb_cache RENAME TO my_new_schema_name; \set ON_ERROR_STOP 1 -- Make sure we can rename associated schemas CREATE TABLE my_table ( date timestamp with time zone NOT NULL, quantity double precision ); SELECT create_hypertable('my_table','date', associated_schema_name => 'my_associated_schema'); INSERT INTO my_table (date, quantity) VALUES ('2018-07-04T21:00:00+00:00', 8); ALTER SCHEMA my_associated_schema RENAME TO new_associated_schema; INSERT INTO my_table (date, quantity) VALUES ('2018-08-10T23:00:00+00:00', 20); -- Make sure the schema name is changed in both catalog tables SELECT * from _timescaledb_catalog.hypertable; SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk from _timescaledb_catalog.chunk; DROP TABLE my_table; -- test renaming unique constraints/indexes CREATE TABLE t_hypertable ( id INTEGER NOT NULL, time TIMESTAMPTZ NOT NULL, value FLOAT NOT NULL CHECK (value > 0), UNIQUE(id, time)); SELECT create_hypertable('t_hypertable', 'time'); INSERT INTO t_hypertable AS h VALUES ( 1, '2020-01-01 00:00:00', 3.2) ON CONFLICT (id, time) DO UPDATE SET value = h.value + EXCLUDED.value; INSERT INTO t_hypertable AS h VALUES ( 1, '2021-01-01 00:00:00', 3.2) ON CONFLICT (id, time) DO UPDATE SET value = h.value + EXCLUDED.value; BEGIN; ALTER INDEX t_hypertable_id_time_key RENAME TO t_new_constraint; -- chunk_constraint should have updated constraint names SELECT hypertable_constraint_name, constraint_name from _timescaledb_catalog.chunk_constraint WHERE hypertable_constraint_name = 't_new_constraint' ORDER BY 1,2; INSERT INTO t_hypertable AS h VALUES ( 1, '2020-01-01 00:01:00', 3.2) ON CONFLICT (id, time) DO UPDATE SET value = h.value + EXCLUDED.value; ROLLBACK; BEGIN; ALTER TABLE t_hypertable RENAME CONSTRAINT t_hypertable_id_time_key TO t_new_constraint; -- chunk_constraint should have updated constraint names SELECT hypertable_constraint_name, constraint_name from _timescaledb_catalog.chunk_constraint WHERE hypertable_constraint_name = 't_new_constraint' ORDER BY 1,2; INSERT INTO t_hypertable AS h VALUES ( 1, '2020-01-01 00:01:00', 3.2) ON CONFLICT (id, time) DO UPDATE SET value = h.value + EXCLUDED.value; ROLLBACK; -- predicate reconstruction when attnos are different in hypertable and chunk CREATE TABLE p_hypertable (a integer not null, b integer, c integer); SELECT create_hypertable('p_hypertable', 'a', chunk_time_interval => int '3'); BEGIN; ALTER TABLE p_hypertable DROP COLUMN b, ADD COLUMN d boolean; CREATE INDEX idx_ht ON p_hypertable(a, c) WHERE d = FALSE; END; INSERT INTO p_hypertable(a, c, d) VALUES (1, 1, FALSE); \d _timescaledb_internal._hyper_14_28_chunk DROP TABLE p_hypertable; -- check none of our hooks interact badly with normal alter view handling CREATE VIEW v1 AS SELECT random(); \set ON_ERROR_STOP 0 -- should error with unrecognized parameter ALTER VIEW v1 SET (autovacuum_enabled = false); \set ON_ERROR_STOP 1 -- issue 4474 -- test hypertable with non-default statistics target -- and chunk creation triggered by non-owner CREATE ROLE role_4474; CREATE TABLE i4474(time timestamptz NOT NULL); SELECT table_name FROM public.create_hypertable( 'i4474', 'time'); GRANT SELECT, INSERT on i4474 TO role_4474; -- create chunk as owner INSERT INTO i4474 SELECT '2020-01-01'; -- set statistics ALTER TABLE i4474 ALTER COLUMN time SET statistics 10; -- create chunk as non-owner SET ROLE role_4474; INSERT INTO i4474 SELECT '2021-01-01'; RESET ROLE; DROP TABLE i4474 CASCADE; DROP ROLE role_4474; -- verify that setting replica identity works and chunks inherit the -- root table's setting CREATE TABLE replid(time timestamptz, value int); SELECT create_hypertable('replid', 'time', chunk_time_interval => interval '1 day', create_default_indexes => false); -- replica identity set to default SELECT relreplident FROM pg_class WHERE relname = 'replid'; INSERT INTO replid VALUES ('2023-01-01', 1); -- the new chunk should have the same replica identity setting SELECT relname, relreplident FROM show_chunks('replid') ch INNER JOIN pg_class c ON (ch = c.oid) ORDER BY relname; -- test change to replica identity full ALTER TABLE replid REPLICA IDENTITY FULL; SELECT relname, relreplident FROM pg_class WHERE relname = 'replid' ORDER BY relname; -- the chunk's setting should also change to FULL SELECT relname, relreplident FROM show_chunks('replid') ch INNER JOIN pg_class c ON (ch = c.oid) ORDER BY relname; -- change to replica identity index CREATE UNIQUE INDEX time_key ON replid (time); ALTER TABLE replid REPLICA IDENTITY USING INDEX time_key; SELECT relname, relreplident FROM pg_class WHERE relname = 'replid' ORDER BY relname; SELECT relname, relreplident FROM show_chunks('replid') ch INNER JOIN pg_class c ON (ch = c.oid) ORDER BY relname; SELECT indexrelid::regclass::text AS index_name FROM show_chunks('replid') chid INNER JOIN pg_index i ON (i.indrelid = chid) AND indisreplident=true ORDER BY index_name; INSERT INTO replid VALUES ('2023-01-02', 2); -- the new chunk will also have replica identity "index" SELECT relname, relreplident FROM show_chunks('replid') ch INNER JOIN pg_class c ON (ch = c.oid) ORDER BY relname; SELECT indexrelid::regclass::text AS index_name FROM show_chunks('replid') chid INNER JOIN pg_index i ON (i.indrelid = chid) AND indisreplident=true ORDER BY index_name; -- drop the replica identity index and create a new chunk. The new -- chunk should have replica identity "NOTHING" since this is the -- behavior of replica identity index when the index is dropped. DROP INDEX time_key; INSERT INTO replid VALUES ('2023-01-03', 3); -- no indexes left SELECT relname, relreplident FROM show_chunks('replid') ch INNER JOIN pg_class c ON (ch = c.oid) ORDER BY relname; SELECT indexrelid::regclass::text AS index_name FROM show_chunks('replid') chid INNER JOIN pg_index i ON (i.indrelid = chid) AND indisreplident=true ORDER BY index_name; -- recreate the unique index after drop and insert to create a new chunk. -- This is a regression test for a bug where rd_replidindex was stale -- after relcache invalidation from chunk index creation, leading to -- "could not open relation with OID 0" error. CREATE UNIQUE INDEX time_key ON replid (time); INSERT INTO replid VALUES ('2023-01-04', 4); SELECT relname, relreplident FROM show_chunks('replid') ch INNER JOIN pg_class c ON (ch = c.oid) ORDER BY relname; -- Alter replica identity directly on a chunk is not supported SELECT ch AS chunk_name FROM show_chunks('replid') ch ORDER BY chunk_name LIMIT 1 \gset \set ON_ERROR_STOP 0 ALTER TABLE :chunk_name REPLICA IDENTITY FULL; \set ON_ERROR_STOP 1 SELECT relname, relreplident FROM show_chunks('replid') ch INNER JOIN pg_class c ON (ch = c.oid) ORDER BY relname; -- test implicit constraints gh issue #9132 CREATE TABLE i9132(time timestamptz) WITH (tsdb.hypertable); INSERT INTO i9132 VALUES ('2024-01-01'), ('2024-02-02'); ALTER TABLE i9132 ADD COLUMN id serial, ADD CONSTRAINT implicit_pk PRIMARY KEY (id, time); ================================================ FILE: test/sql/alternate_users.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \ir include/insert_single.sql \c :TEST_DBNAME :ROLE_SUPERUSER -- make sure tablespace1 exists -- since there is no CREATE TABLESPACE IF EXISTS we drop with if exists and recreate SET client_min_messages TO error; DROP TABLESPACE IF EXISTS tablespace1; RESET client_min_messages; CREATE TABLESPACE tablespace1 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE1_PATH; --needed for ddl ops: CREATE SCHEMA IF NOT EXISTS "customSchema" AUTHORIZATION :ROLE_DEFAULT_PERM_USER_2; --needed for ROLE_DEFAULT_PERM_USER_2 to write to the 'one_Partition' schema which --is owned by ROLE_DEFAULT_PERM_USER GRANT CREATE ON SCHEMA "one_Partition" TO :ROLE_DEFAULT_PERM_USER_2; --test creating and using schema as non-superuser \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 SELECT * FROM test.relation WHERE schema='public' ORDER BY schema, name; \set ON_ERROR_STOP 0 SELECT * FROM "one_Partition"; SELECT set_chunk_time_interval('"one_Partition"', 1::bigint); select add_dimension('"one_Partition"', 'device_id', 2); select attach_tablespace('tablespace1', '"one_Partition"'); \set ON_ERROR_STOP 1 CREATE TABLE "1dim"(time timestamp, temp float); SELECT create_hypertable('"1dim"', 'time'); INSERT INTO "1dim" VALUES('2017-01-20T09:00:01', 22.5); INSERT INTO "1dim" VALUES('2017-01-20T09:00:21', 21.2); INSERT INTO "1dim" VALUES('2017-01-20T09:00:47', 25.1); SELECT * FROM "1dim"; \ir include/ddl_ops_1.sql \ir include/ddl_ops_2.sql --test proper denials for all security definer functions: \c :TEST_DBNAME :ROLE_SUPERUSER CREATE TABLE plain_table_su (time timestamp, temp float); CREATE TABLE hypertable_su (time timestamp, temp float); SELECT create_hypertable('hypertable_su', 'time'); CREATE INDEX "ind_1" ON hypertable_su (time); INSERT INTO hypertable_su VALUES('2017-01-20T09:00:01', 22.5); \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 --all of the following should produce errors \set ON_ERROR_STOP 0 SELECT create_hypertable('plain_table_su', 'time'); CREATE INDEX ON plain_table_su (time, temp); CREATE INDEX ON hypertable_su (time, temp); DROP INDEX "ind_1"; ALTER INDEX "ind_1" RENAME TO "ind_2"; \set ON_ERROR_STOP 1 --test that I can't do anything to a non-owned hypertable. \set ON_ERROR_STOP 0 CREATE INDEX ON hypertable_su (time, temp); SELECT * FROM hypertable_su; INSERT INTO hypertable_su VALUES('2017-01-20T09:00:01', 22.5); ALTER TABLE hypertable_su ADD COLUMN val2 integer; \set ON_ERROR_STOP 1 --grant read permissions \c :TEST_DBNAME :ROLE_SUPERUSER GRANT SELECT ON hypertable_su TO :ROLE_DEFAULT_PERM_USER_2; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 SELECT * FROM hypertable_su; \set ON_ERROR_STOP 0 CREATE INDEX ON hypertable_su (time, temp); INSERT INTO hypertable_su VALUES('2017-01-20T09:00:01', 22.5); ALTER TABLE hypertable_su ADD COLUMN val2 integer; \set ON_ERROR_STOP 1 --grant read, insert permissions \c :TEST_DBNAME :ROLE_SUPERUSER GRANT SELECT, INSERT ON hypertable_su TO :ROLE_DEFAULT_PERM_USER_2; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 INSERT INTO hypertable_su VALUES('2017-01-20T09:00:01', 22.5); SELECT * FROM hypertable_su; \set ON_ERROR_STOP 0 CREATE INDEX ON hypertable_su (time, temp); ALTER TABLE hypertable_su ADD COLUMN val2 integer; \set ON_ERROR_STOP 1 --change owner \c :TEST_DBNAME :ROLE_SUPERUSER ALTER TABLE hypertable_su OWNER TO :ROLE_DEFAULT_PERM_USER_2; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 INSERT INTO hypertable_su VALUES('2017-01-20T09:00:01', 22.5); SELECT * FROM hypertable_su; CREATE INDEX ON hypertable_su (time, temp); ALTER TABLE hypertable_su ADD COLUMN val2 integer; ================================================ FILE: test/sql/append.sql.in ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set TEST_BASE_NAME append SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') as "TEST_LOAD_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') as "TEST_QUERY_NAME", format('%s/results/%s_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_OPTIMIZED", format('%s/results/%s_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_UNOPTIMIZED" \gset SELECT format('\! diff -u --label "Unoptimized results" --label "Optimized results" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') as "DIFF_CMD" \gset SET timescaledb.enable_now_constify TO false; -- disable memoize node to avoid flaky results SET enable_memoize TO 'off'; -- disable index only scans to avoid some flaky results SET enable_indexonlyscan TO FALSE; \set PREFIX 'EXPLAIN (analyze, buffers off, costs off, timing off, summary off)' \ir :TEST_LOAD_NAME \ir :TEST_QUERY_NAME --generate the results into two different files \set ECHO errors SET client_min_messages TO error; \set PREFIX '' -- get results with optimizations disabled \o :TEST_RESULTS_UNOPTIMIZED SET timescaledb.enable_optimizations TO false; \ir :TEST_QUERY_NAME \o -- get query results with all optimizations \o :TEST_RESULTS_OPTIMIZED SET timescaledb.enable_optimizations TO true; \ir :TEST_QUERY_NAME \o :DIFF_CMD -- get query results with constraint aware append \o :TEST_RESULTS_OPTIMIZED SET timescaledb.enable_chunk_append TO false; \ir :TEST_QUERY_NAME \o :DIFF_CMD ================================================ FILE: test/sql/baserel_cache.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test that the baserel cache is not clobbered if there's an error -- in a SQL function. CREATE TABLE valid_ids ( id UUID PRIMARY KEY ); CREATE FUNCTION DEFAULT_UUID(TEXT DEFAULT '') RETURNS UUID AS $$ BEGIN RETURN COALESCE($1, '')::UUID; EXCEPTION WHEN invalid_text_representation THEN RETURN '00000000-0000-0000-0000-000000000000'; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE FUNCTION KNOWN_ID(UUID, TEXT) RETURNS UUID AS $$ SELECT COALESCE( (SELECT id FROM valid_ids WHERE id = $1), DEFAULT_UUID() ); $$ LANGUAGE SQL; SELECT KNOWN_ID(NULL, ''), KNOWN_ID(NULL, ''); ================================================ FILE: test/sql/bgw_launcher.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set TEST_DBNAME_2 :TEST_DBNAME _2 \c :TEST_DBNAME :ROLE_SUPERUSER -- start bgw since they are stopped for tests by default SELECT _timescaledb_functions.start_background_workers(); CREATE DATABASE :TEST_DBNAME_2; \c :TEST_DBNAME_2 :ROLE_SUPERUSER \ir include/bgw_launcher_utils.sql -- When we've connected to test db 2, we should be able to see the cluster launcher -- and the scheduler for test db in pg_stat_activity -- but test db 2 shouldn't have a scheduler because ext not created yet SELECT wait_worker_counts(1,1,0,0); -- Now create the extension in test db 2 SET client_min_messages = ERROR; CREATE EXTENSION timescaledb CASCADE; RESET client_min_messages; SELECT wait_worker_counts(1,1,1,0); DROP DATABASE :TEST_DBNAME WITH (FORCE); -- Now the db_scheduler for test db should have disappeared SELECT wait_worker_counts(1,0,1,0); -- Now let's restart the scheduler in test db 2 and make sure our backend_start changed SELECT backend_start as orig_backend_start FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2' \gset -- We'll do this in a txn so that we can see that the worker locks on our txn before continuing BEGIN; SELECT _timescaledb_functions.restart_background_workers(); SELECT wait_worker_counts(1,0,1,0); SELECT (backend_start > :'orig_backend_start'::timestamptz) backend_start_changed, (wait_event = 'virtualxid') wait_event_changed FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2'; COMMIT; SELECT wait_worker_counts(1,0,1,0); SELECT (wait_event IS DISTINCT FROM 'virtualxid') wait_event_changed FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2'; -- Test stop SELECT _timescaledb_functions.stop_background_workers(); SELECT wait_worker_counts(1,0,0,0); -- Make sure it doesn't break if we stop twice in a row SELECT _timescaledb_functions.stop_background_workers(); SELECT wait_worker_counts(1,0,0,0); -- test start SELECT _timescaledb_functions.start_background_workers(); SELECT wait_worker_counts(1,0,1,0); -- make sure start is idempotent SELECT backend_start as orig_backend_start FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2' \gset -- Since we're doing idempotency tests, we're also going to exercise our queue and start 20 times SELECT _timescaledb_functions.start_background_workers() as start_background_workers, * FROM generate_series(1,20); -- Here we're waiting to see if something shows up in pg_stat_activity, -- so we have to condition our loop in the opposite way. We'll only wait -- half a second in total as well so that tests don't take too long. CREATE FUNCTION wait_equals(TIMESTAMPTZ, TEXT) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE r BOOLEAN; BEGIN FOR i in 1..5 LOOP SELECT (backend_start = $1::timestamptz) backend_start_unchanged FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = $2 into r; if(r) THEN PERFORM pg_sleep(0.1); PERFORM pg_stat_clear_snapshot(); ELSE RETURN FALSE; END IF; END LOOP; RETURN TRUE; END $BODY$; select wait_equals(:'orig_backend_start', :'TEST_DBNAME_2'); -- Make sure restart starts a worker even if it is stopped SELECT _timescaledb_functions.stop_background_workers(); SELECT wait_worker_counts(1,0,0,0); SELECT _timescaledb_functions.restart_background_workers(); SELECT wait_worker_counts(1,0,1,0); -- Make sure drop extension statement restarts the worker and on rollback it keeps running -- Now let's restart the scheduler and make sure our backend_start changed SELECT backend_start as orig_backend_start FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2' \gset BEGIN; DROP EXTENSION timescaledb; SELECT wait_worker_counts(1,0,1,0); ROLLBACK; CREATE FUNCTION wait_greater(TIMESTAMPTZ,TEXT) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE r BOOLEAN; BEGIN FOR i in 1..10 LOOP SELECT (backend_start > $1::timestamptz) backend_start_changed FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = $2 into r; if(NOT r) THEN PERFORM pg_sleep(0.1); PERFORM pg_stat_clear_snapshot(); ELSE RETURN TRUE; END IF; END LOOP; RETURN FALSE; END $BODY$; SELECT wait_greater(:'orig_backend_start',:'TEST_DBNAME_2'); -- Make sure canceling the launcher backend causes a restart of schedulers SELECT backend_start as orig_backend_start FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2' \gset SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Launcher'; SELECT wait_worker_counts(1,0,1,0); SELECT wait_greater(:'orig_backend_start', :'TEST_DBNAME_2'); -- Make sure running pre_restore function stops background workers SELECT timescaledb_pre_restore(); SELECT wait_worker_counts(1,0,0,0); -- Make sure a restart with restoring on first starts the background worker BEGIN; SELECT _timescaledb_functions.restart_background_workers(); SELECT wait_worker_counts(1,0,1,0); COMMIT; -- Then the worker dies when it sees that restoring is on after the txn commits SELECT wait_worker_counts(1,0,0,0); --And post_restore starts them BEGIN; SELECT timescaledb_post_restore(); SELECT wait_worker_counts(1,0,1,0); COMMIT; -- And they stay started SELECT wait_worker_counts(1,0,1,0); -- Make sure dropping the extension means that the scheduler is stopped BEGIN; DROP EXTENSION timescaledb; COMMIT; SELECT wait_worker_counts(1,0,0,0); -- Test that background workers are stopped with DROP OWNED ALTER ROLE :ROLE_DEFAULT_PERM_USER WITH SUPERUSER; \c :TEST_DBNAME_2 :ROLE_DEFAULT_PERM_USER SET client_min_messages = ERROR; CREATE EXTENSION timescaledb CASCADE; RESET client_min_messages; -- Make sure there is 1 launcher and 1 bgw in test db 2 SELECT wait_worker_counts(launcher_ct=>1, scheduler1_ct=> 0, scheduler2_ct=>1, template1_ct=>0); -- drop a non-owner of the extension results in no change to worker counts DROP OWNED BY :ROLE_DEFAULT_PERM_USER_2; SELECT wait_worker_counts(launcher_ct=>1, scheduler1_ct=> 0, scheduler2_ct=>1, template1_ct=>0); -- drop of owner of extension results in extension drop and a stop to the bgw DROP OWNED BY :ROLE_DEFAULT_PERM_USER; -- The worker in test db 2 is dead. Note that 0s are respected SELECT wait_worker_counts(launcher_ct=>1, scheduler1_ct=>0, scheduler2_ct=>0, template1_ct=>0); \c :TEST_DBNAME_2 :ROLE_SUPERUSER ALTER ROLE :ROLE_DEFAULT_PERM_USER WITH NOSUPERUSER; -- Connect to the template1 database \c template1 \ir include/bgw_launcher_utils.sql BEGIN; -- Then create extension there in a txn and make sure we see a scheduler start SET client_min_messages = ERROR; CREATE EXTENSION timescaledb CASCADE; RESET client_min_messages; SELECT wait_worker_counts(1,0,0,1); COMMIT; -- End our transaction and it should immediately exit because it's a template database. SELECT wait_worker_counts(1,0,0,0); \c :TEST_DBNAME_2 -- Now try creating a DB from a template with the extension already installed. -- Make sure we see a scheduler start. CREATE DATABASE :TEST_DBNAME; SELECT wait_worker_counts(1,1,0,0); DROP DATABASE :TEST_DBNAME WITH (FORCE); -- Now make sure that there's no race between create database and create extension. -- Although to be honest, this race probably wouldn't manifest in this test. \c template1 DROP EXTENSION timescaledb; \c :TEST_DBNAME_2 CREATE DATABASE :TEST_DBNAME; \c :TEST_DBNAME SET client_min_messages = ERROR; CREATE EXTENSION timescaledb; RESET client_min_messages; \c :TEST_DBNAME_2 SELECT wait_worker_counts(1,1,0,0); -- test rename database CREATE DATABASE db_rename_test; \c db_rename_test :ROLE_SUPERUSER SET client_min_messages=error; CREATE EXTENSION timescaledb; \c :TEST_DBNAME_2 :ROLE_SUPERUSER SELECT wait_for_bgw_scheduler('db_rename_test'); ALTER DATABASE db_rename_test RENAME TO db_rename_test2; DROP DATABASE db_rename_test2 WITH (FORCE); -- test create database with timescaledb database as template SELECT wait_for_bgw_scheduler(:'TEST_DBNAME'); CREATE DATABASE db_from_template WITH TEMPLATE :TEST_DBNAME; SELECT wait_for_bgw_scheduler(:'TEST_DBNAME'); DROP DATABASE db_from_template WITH (FORCE); -- test alter database set tablespace SET client_min_messages TO error; DROP TABLESPACE IF EXISTS tablespace1; RESET client_min_messages; CREATE TABLESPACE tablespace1 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE1_PATH; -- Stop background worker before we change the tablespace of the database (otherwise, the database might be used) SELECT wait_for_bgw_scheduler(:'TEST_DBNAME'); -- Connect to TEST_DBNAME (_timescaledb_functions.stop_background_workers() is not available in TEST_DBNAME_2) \c :TEST_DBNAME :ROLE_SUPERUSER SELECT _timescaledb_functions.stop_background_workers(); SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE backend_type = 'TimescaleDB Background Worker Launcher'; \c :TEST_DBNAME_2 :ROLE_SUPERUSER -- make sure nobody is using it REVOKE CONNECT ON DATABASE :TEST_DBNAME FROM public; CALL kill_database_backends(:'TEST_DBNAME'); -- Change tablespace ALTER DATABASE :TEST_DBNAME SET TABLESPACE tablespace1; -- tear down test and clean up additional database \c :TEST_DBNAME :ROLE_SUPERUSER SELECT _timescaledb_functions.stop_background_workers() \gset REVOKE CONNECT ON DATABASE :TEST_DBNAME_2 FROM public; CALL kill_database_backends(:'TEST_DBNAME_2'); SELECT * FROM pg_stat_activity WHERE datname = :'TEST_DBNAME_2'; DROP DATABASE :TEST_DBNAME_2 WITH (force); -- Clean up the template database, removing our test utilities etc \c template1 \ir include/bgw_launcher_utils_cleanup.sql ================================================ FILE: test/sql/c_unit_tests.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION test.time_to_internal_conversion() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_time_to_internal_conversion' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION test.interval_to_internal_conversion() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_interval_to_internal_conversion' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION test.adts() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_adts' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION test.time_utils() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_time_utils' LANGUAGE C; CREATE OR REPLACE FUNCTION test.bmslist_utils() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_bmslist_utils' LANGUAGE C; CREATE OR REPLACE FUNCTION test.jsonb_utils() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_jsonb_utils' LANGUAGE C; CREATE OR REPLACE FUNCTION test.compression_settings() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_compression_settings' LANGUAGE C; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT test.time_to_internal_conversion(); SELECT test.interval_to_internal_conversion(); SELECT test.adts(); SELECT test.time_utils(); SELECT test.bmslist_utils(); SELECT test.jsonb_utils(); SELECT test.compression_settings(); ================================================ FILE: test/sql/catalog_corruption.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER --- Test handling of missing dimension slices CREATE TABLE dim_test(time TIMESTAMPTZ, device int); SELECT create_hypertable('dim_test', 'time', chunk_time_interval => INTERVAL '1 day'); -- Create two chunks INSERT INTO dim_test values('2000-01-01 00:00:00', 1); INSERT INTO dim_test values('2020-01-01 00:00:00', 1); SELECT id AS dim_slice_id FROM _timescaledb_catalog.dimension_slice ORDER BY id DESC LIMIT 1 \gset -- Delete the dimension slice for the second chunk DELETE FROM _timescaledb_catalog.chunk_constraint WHERE dimension_slice_id = :dim_slice_id; \set ON_ERROR_STOP 0 -- Select data SELECT * FROM dim_test; -- Select data using ordered append SELECT * FROM dim_test ORDER BY time; \set ON_ERROR_STOP 1 DROP TABLE dim_test; ================================================ FILE: test/sql/chunk_adaptive.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- test error handling _timescaledb_functions.calculate_chunk_interval \set ON_ERROR_STOP 0 SELECT _timescaledb_functions.calculate_chunk_interval(0,0,-0); SELECT _timescaledb_functions.calculate_chunk_interval(1,0,-1); \set ON_ERROR_STOP 1 -- Valid chunk sizing function for testing CREATE OR REPLACE FUNCTION calculate_chunk_interval( dimension_id INTEGER, dimension_coord BIGINT, chunk_target_size BIGINT ) RETURNS BIGINT LANGUAGE PLPGSQL AS $BODY$ DECLARE BEGIN RETURN -1; END $BODY$; -- Chunk sizing function with bad signature CREATE OR REPLACE FUNCTION bad_calculate_chunk_interval( dimension_id INTEGER ) RETURNS BIGINT LANGUAGE PLPGSQL AS $BODY$ DECLARE BEGIN RETURN -1; END $BODY$; -- Set a fixed memory cache size to make tests determinstic -- (independent of available machine memory) SELECT * FROM test.set_memory_cache_size('2GB'); -- test NULL handling \set ON_ERROR_STOP 0 SELECT * FROM set_adaptive_chunking(NULL,NULL); \set ON_ERROR_STOP 1 CREATE TABLE test_adaptive(time timestamptz, temp float, location int); \set ON_ERROR_STOP 0 -- Bad signature of sizing func should fail SELECT create_hypertable('test_adaptive', 'time', chunk_target_size => '1MB', chunk_sizing_func => 'bad_calculate_chunk_interval'); \set ON_ERROR_STOP 1 -- Setting sizing func with correct signature should work SELECT create_hypertable('test_adaptive', 'time', chunk_target_size => '1MB', chunk_sizing_func => 'calculate_chunk_interval'); DROP TABLE test_adaptive; CREATE TABLE test_adaptive(time timestamptz, temp float, location int); -- Size but no explicit func should use default func SELECT create_hypertable('test_adaptive', 'time', chunk_target_size => '1MB', create_default_indexes => true); SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; -- Check that adaptive chunking sets a 1 day default chunk time -- interval => 86400000000 microseconds SELECT * FROM _timescaledb_catalog.dimension; -- Change the target size SELECT * FROM set_adaptive_chunking('test_adaptive', '2MB'); SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; \set ON_ERROR_STOP 0 -- Setting NULL func should fail SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB', NULL); \set ON_ERROR_STOP 1 -- Setting NULL size disables adaptive chunking SELECT * FROM set_adaptive_chunking('test_adaptive', NULL); SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB'); -- Setting size to 'off' should also disable SELECT * FROM set_adaptive_chunking('test_adaptive', 'off'); SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; -- Setting 0 size should also disable SELECT * FROM set_adaptive_chunking('test_adaptive', '0MB'); SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB'); -- No warning about small target size if > 10MB SELECT * FROM set_adaptive_chunking('test_adaptive', '11MB'); -- Setting size to 'estimate' should also estimate size SELECT * FROM set_adaptive_chunking('test_adaptive', 'estimate'); SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; -- Use a lower memory setting to test that the calculated chunk_target_size is reduced SELECT * FROM test.set_memory_cache_size('512MB'); SELECT * FROM set_adaptive_chunking('test_adaptive', 'estimate'); SELECT table_name, chunk_sizing_func_schema, chunk_sizing_func_name, chunk_target_size FROM _timescaledb_catalog.hypertable; -- Reset memory settings SELECT * FROM test.set_memory_cache_size('2GB'); -- Set a reasonable test value SELECT * FROM set_adaptive_chunking('test_adaptive', '1MB'); -- Show the interval length before and after adaptation SELECT id, hypertable_id, interval_length FROM _timescaledb_catalog.dimension; -- Generate data to create chunks. We use the hash of the time value -- to get determinstic location IDs so that we always spread these -- values the same way across space partitions INSERT INTO test_adaptive SELECT time, random() * 35, _timescaledb_functions.get_partition_hash(time) FROM generate_series('2017-03-07T18:18:03+00'::timestamptz - interval '175 days', '2017-03-07T18:18:03+00'::timestamptz, '2 minutes') as time; SELECT chunk_name, primary_dimension, range_start, range_end FROM timescaledb_information.chunks WHERE hypertable_name = 'test_adaptive' ORDER BY chunk_name; -- Do same thing without an index on the time column. This affects -- both the calculation of fill-factor of the chunk and its size CREATE TABLE test_adaptive_no_index(time timestamptz, temp float, location int); -- Size but no explicit func should use default func -- No default indexes should warn and use heap scan for min and max SELECT create_hypertable('test_adaptive_no_index', 'time', chunk_target_size => '1MB', create_default_indexes => false); SELECT id, hypertable_id, interval_length FROM _timescaledb_catalog.dimension; INSERT INTO test_adaptive_no_index SELECT time, random() * 35, _timescaledb_functions.get_partition_hash(time) FROM generate_series('2017-03-07T18:18:03+00'::timestamptz - interval '175 days', '2017-03-07T18:18:03+00'::timestamptz, '2 minutes') as time; SELECT chunk_name, primary_dimension, range_start, range_end FROM timescaledb_information.chunks WHERE hypertable_name = 'test_adaptive_no_index' ORDER BY chunk_name; -- Test added to check that the correct index (i.e. time index) is being used -- to find the min and max. Previously a bug selected the first index listed, -- which in this case is location rather than time and therefore could return -- the wrong min and max if items at the start and end of the index did not have -- the correct min and max timestamps. -- -- In this test, we create chunks with a lot of locations with only one reading -- that is at the beginning of the time frame, and then one location in the middle -- of the range that has two readings, one that is the same as the others and one -- that is larger. The algorithm should use these two readings for min & max; however, -- if it's broken (as it was before), it would choose just the reading that is common -- to all the locations. CREATE TABLE test_adaptive_correct_index(time timestamptz, temp float, location int); SELECT create_hypertable('test_adaptive_correct_index', 'time', chunk_target_size => '100MB', chunk_time_interval => 86400000000, create_default_indexes => false); CREATE INDEX ON test_adaptive_correct_index(location); CREATE INDEX ON test_adaptive_correct_index(time DESC); -- First chunk INSERT INTO test_adaptive_correct_index SELECT '2018-01-01T00:00:00+00'::timestamptz, val, val + 1 FROM generate_series(1, 1000) as val; INSERT INTO test_adaptive_correct_index SELECT time, 0.0, '1500' FROM generate_series('2018-01-01T00:00:00+00'::timestamptz, '2018-01-01T20:00:00+00'::timestamptz, '10 hours') as time; INSERT INTO test_adaptive_correct_index SELECT '2018-01-01T00:00:00+00'::timestamptz, val, val + 1 FROM generate_series(2001, 3000) as val; -- Second chunk INSERT INTO test_adaptive_correct_index SELECT '2018-01-02T00:00:00+00'::timestamptz, val, val + 1 FROM generate_series(1, 1000) as val; INSERT INTO test_adaptive_correct_index SELECT time, 0.0, '1500' FROM generate_series('2018-01-02T00:00:00+00'::timestamptz, '2018-01-02T20:00:00+00'::timestamptz, '10 hours') as time; INSERT INTO test_adaptive_correct_index SELECT '2018-01-02T00:00:00+00'::timestamptz, val, val + 1 FROM generate_series(2001, 3000) as val; -- Third chunk INSERT INTO test_adaptive_correct_index SELECT '2018-01-03T00:00:00+00'::timestamptz, val, val + 1 FROM generate_series(1, 1000) as val; INSERT INTO test_adaptive_correct_index SELECT time, 0.0, '1500' FROM generate_series('2018-01-03T00:00:00+00'::timestamptz, '2018-01-03T20:00:00+00'::timestamptz, '10 hours') as time; INSERT INTO test_adaptive_correct_index SELECT '2018-01-03T00:00:00+00'::timestamptz, val, val + 1 FROM generate_series(2001, 3000) as val; -- This should be the start of the fourth chunk INSERT INTO test_adaptive_correct_index SELECT '2018-01-04T00:00:00+00'::timestamptz, val, val + 1 FROM generate_series(1, 1000) as val; INSERT INTO test_adaptive_correct_index SELECT time, 0.0, '1500' FROM generate_series('2018-01-04T00:00:00+00'::timestamptz, '2018-01-04T20:00:00+00'::timestamptz, '10 hours') as time; INSERT INTO test_adaptive_correct_index SELECT '2018-01-04T00:00:00+00'::timestamptz, val, val + 1 FROM generate_series(2001, 3000) as val; -- If working correctly, this goes in the 4th chunk, otherwise its a separate 5th chunk INSERT INTO test_adaptive_correct_index SELECT '2018-01-05T00:00:00+00'::timestamptz, val, val + 1 FROM generate_series(1, 1000) as val; INSERT INTO test_adaptive_correct_index SELECT time, 0.0, '1500' FROM generate_series('2018-01-05T00:00:00+00'::timestamptz, '2018-01-05T20:00:00+00'::timestamptz, '10 hours') as time; INSERT INTO test_adaptive_correct_index SELECT '2018-01-05T00:00:00+00'::timestamptz, val, val + 1 FROM generate_series(2001, 3000) as val; -- This should show 4 chunks, rather than 5 SELECT count(*) FROM timescaledb_information.chunks WHERE hypertable_name = 'test_adaptive_correct_index'; -- The interval_length should no longer be 86400000000 for our hypertable, so 3rd column so be true. -- Note: the exact interval_length is non-deterministic, so we can't use its actual value for tests SELECT id, hypertable_id, interval_length > 86400000000 FROM _timescaledb_catalog.dimension; -- Drop because it's size and estimated chunk_interval is non-deterministic so -- we don't want to make other tests flaky. DROP TABLE test_adaptive_correct_index; -- Test with space partitioning. This might affect the estimation -- since there are more chunks in the same time interval and space -- chunks might be unevenly filled. CREATE TABLE test_adaptive_space(time timestamptz, temp float, location int); SELECT create_hypertable('test_adaptive_space', 'time', 'location', 2, chunk_target_size => '1MB', create_default_indexes => true); SELECT id, hypertable_id, interval_length FROM _timescaledb_catalog.dimension; INSERT INTO test_adaptive_space SELECT time, random() * 35, _timescaledb_functions.get_partition_hash(time) FROM generate_series('2017-03-07T18:18:03+00'::timestamptz - interval '175 days', '2017-03-07T18:18:03+00'::timestamptz, '2 minutes') as time; \x SELECT chunk_name, range_start, range_end FROM timescaledb_information.chunks WHERE hypertable_name = 'test_adaptive_space' ORDER BY chunk_name; SELECT * FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_adaptive_space' ORDER BY dimension_number; \x SELECT * FROM chunks_detailed_size('test_adaptive_space') ORDER BY chunk_name; SELECT id, hypertable_id, interval_length FROM _timescaledb_catalog.dimension; -- A previous version stopped working as soon as hypertable_id stopped being -- equal to dimension_id (i.e., there was a hypertable with more than 1 dimension). -- This test comes after test_adaptive_space, which has 2 dimensions, and makes -- sure that it still works. CREATE TABLE test_adaptive_after_multiple_dims(time timestamptz, temp float, location int); SELECT create_hypertable('test_adaptive_after_multiple_dims', 'time', chunk_target_size => '100MB', create_default_indexes => true); INSERT INTO test_adaptive_after_multiple_dims VALUES('2018-01-01T00:00:00+00'::timestamptz, 0.0, 5); \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 \set ON_ERROR_STOP 0 SELECT * FROM set_adaptive_chunking('test_adaptive', '2MB'); \set ON_ERROR_STOP 1 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- Now make sure renaming schema gets propagated to the func_schema DROP TABLE test_adaptive; \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA IF NOT EXISTS my_chunk_func_schema; CREATE OR REPLACE FUNCTION my_chunk_func_schema.calculate_chunk_interval( dimension_id INTEGER, dimension_coord BIGINT, chunk_target_size BIGINT ) RETURNS BIGINT LANGUAGE PLPGSQL AS $BODY$ DECLARE BEGIN RETURN 2; END $BODY$; CREATE TABLE test_adaptive(time timestamptz, temp float, location int); SELECT create_hypertable('test_adaptive', 'time', chunk_target_size => '1MB', chunk_sizing_func => 'my_chunk_func_schema.calculate_chunk_interval'); ALTER SCHEMA my_chunk_func_schema RENAME TO new_chunk_func_schema; INSERT INTO test_adaptive VALUES (now(), 1.0, 1); ================================================ FILE: test/sql/chunk_publication.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test automatic addition of chunks to publications -- Publications require superuser privileges \c :TEST_DBNAME :ROLE_SUPERUSER SET client_min_messages = WARNING; SET timescaledb.enable_chunk_auto_publication = true; -- Test 1: Basic single publication CREATE TABLE test_hypertable (time timestamptz NOT NULL, device_id int, value float, extra text); SELECT create_hypertable('test_hypertable', 'time', chunk_time_interval => interval '1 day'); -- Insert to create first chunk INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1'); -- Create publication and add hypertable CREATE PUBLICATION test_pub FOR TABLE test_hypertable; -- Verify initial state (1 chunk) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub' ORDER BY schemaname, tablename; -- Insert to create 2 more chunks (total 3 chunks) INSERT INTO test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0, 'data2'); INSERT INTO test_hypertable VALUES ('2024-01-03 00:00:00+00', 3, 3.0, 'data3'); -- Verify state (3 chunks) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub' ORDER BY schemaname, tablename; -- Insert to create 5 more chunks (total 8 chunks) INSERT INTO test_hypertable VALUES ('2024-01-04 00:00:00+00', 4, 4.0, 'data4'); INSERT INTO test_hypertable VALUES ('2024-01-05 00:00:00+00', 5, 5.0, 'data5'); INSERT INTO test_hypertable VALUES ('2024-01-06 00:00:00+00', 6, 6.0, 'data6'); INSERT INTO test_hypertable VALUES ('2024-01-07 00:00:00+00', 7, 7.0, 'data7'); INSERT INTO test_hypertable VALUES ('2024-01-08 00:00:00+00', 8, 8.0, 'data8'); -- Verify final state (8 chunks) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub' ORDER BY schemaname, tablename; -- Verify chunk removal via DROP TABLE SELECT chunk_schema || '.' || chunk_name as "CHUNK_TO_DROP" FROM timescaledb_information.chunks WHERE hypertable_name = 'test_hypertable' ORDER BY chunk_schema, chunk_name LIMIT 1 \gset -- Verify chunk removal via DROP TABLE DROP TABLE :CHUNK_TO_DROP; -- Verify chunk was removed from publication (7 chunks remaining) SELECT chunk_schema, chunk_name FROM timescaledb_information.chunks WHERE hypertable_name = 'test_hypertable' ORDER BY chunk_schema, chunk_name; SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub' ORDER BY schemaname, tablename; -- Verify chunk removal via drop_chunks() SELECT drop_chunks('test_hypertable', older_than => '2024-01-07 00:00:00+00'::timestamptz); -- Verify dropped chunks were removed from publication (2 chunks remaining: 2024-01-07 and 2024-01-08) SELECT chunk_schema, chunk_name FROM timescaledb_information.chunks WHERE hypertable_name = 'test_hypertable' ORDER BY chunk_schema, chunk_name; SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub' ORDER BY schemaname, tablename; -- Verify chunk removal via TRUNCATE TRUNCATE TABLE test_hypertable; -- Verify all chunks were removed from publication (0 chunks remaining) SELECT chunk_schema, chunk_name FROM timescaledb_information.chunks WHERE hypertable_name = 'test_hypertable' ORDER BY chunk_schema, chunk_name; SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub' ORDER BY schemaname, tablename; -- Cleanup DROP PUBLICATION test_pub CASCADE; DROP TABLE test_hypertable CASCADE; -- Test 2: Multiple publications CREATE TABLE test_hypertable (time timestamptz NOT NULL, device_id int, value float, extra text); SELECT create_hypertable('test_hypertable', 'time', chunk_time_interval => interval '1 day'); -- Insert to create first chunk INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1'); -- Create pub1 and add hypertable CREATE PUBLICATION test_pub1 FOR TABLE test_hypertable; -- Verify (1 chunk in pub1) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub1' ORDER BY schemaname, tablename; -- Insert to create 2 more chunks (total 3 chunks) INSERT INTO test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0, 'data2'); INSERT INTO test_hypertable VALUES ('2024-01-03 00:00:00+00', 3, 3.0, 'data3'); -- Verify (3 chunks in pub1) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub1' ORDER BY schemaname, tablename; -- Create pub2 and add hypertable CREATE PUBLICATION test_pub2 FOR TABLE test_hypertable; -- Verify (3 chunks in pub1, 3 chunks in pub2) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub1' ORDER BY schemaname, tablename; SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub2' ORDER BY schemaname, tablename; -- Insert to create 5 more chunks (total 8 chunks) INSERT INTO test_hypertable VALUES ('2024-01-04 00:00:00+00', 4, 4.0, 'data4'); INSERT INTO test_hypertable VALUES ('2024-01-05 00:00:00+00', 5, 5.0, 'data5'); INSERT INTO test_hypertable VALUES ('2024-01-06 00:00:00+00', 6, 6.0, 'data6'); INSERT INTO test_hypertable VALUES ('2024-01-07 00:00:00+00', 7, 7.0, 'data7'); INSERT INTO test_hypertable VALUES ('2024-01-08 00:00:00+00', 8, 8.0, 'data8'); -- Verify (8 chunks in pub1, 8 chunks in pub2) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub1' ORDER BY schemaname, tablename; SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub2' ORDER BY schemaname, tablename; -- Cleanup DROP PUBLICATION test_pub1 CASCADE; DROP PUBLICATION test_pub2 CASCADE; DROP TABLE test_hypertable CASCADE; -- Test 3: Row filtering (WHERE clause with multiple conditions) CREATE TABLE test_hypertable (time timestamptz NOT NULL, device_id int, value float, extra text); SELECT create_hypertable('test_hypertable', 'time', chunk_time_interval => interval '1 day'); -- Insert to create first chunk INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1'); -- Create publication with row filter (multiple conditions) CREATE PUBLICATION test_pub_row_filter FOR TABLE test_hypertable WHERE (device_id > 10 AND value > 1000); -- Verify initial state (1 chunk with row filter) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_row_filter' ORDER BY schemaname, tablename; -- Insert to create 2 more chunks (total 3 chunks) INSERT INTO test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0, 'data2'); INSERT INTO test_hypertable VALUES ('2024-01-03 00:00:00+00', 3, 3.0, 'data3'); -- Verify state (3 chunks with row filters) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_row_filter' ORDER BY schemaname, tablename; -- Insert to create 5 more chunks (total 8 chunks) INSERT INTO test_hypertable VALUES ('2024-01-04 00:00:00+00', 4, 4.0, 'data4'); INSERT INTO test_hypertable VALUES ('2024-01-05 00:00:00+00', 5, 5.0, 'data5'); INSERT INTO test_hypertable VALUES ('2024-01-06 00:00:00+00', 6, 6.0, 'data6'); INSERT INTO test_hypertable VALUES ('2024-01-07 00:00:00+00', 7, 7.0, 'data7'); INSERT INTO test_hypertable VALUES ('2024-01-08 00:00:00+00', 8, 8.0, 'data8'); -- Verify final state (8 chunks with row filters) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_row_filter' ORDER BY schemaname, tablename; -- Cleanup DROP PUBLICATION test_pub_row_filter CASCADE; DROP TABLE test_hypertable CASCADE; -- Test 4: Column filtering CREATE TABLE test_hypertable (time timestamptz NOT NULL, device_id int, value float, extra text); SELECT create_hypertable('test_hypertable', 'time', chunk_time_interval => interval '1 day'); -- Insert to create first chunk INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1'); -- Create publication with column filter CREATE PUBLICATION test_pub_col_filter FOR TABLE test_hypertable (time, device_id); -- Verify initial state (1 chunk with column filter) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_col_filter' ORDER BY schemaname, tablename; -- Insert to create 2 more chunks (total 3 chunks) INSERT INTO test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0, 'data2'); INSERT INTO test_hypertable VALUES ('2024-01-03 00:00:00+00', 3, 3.0, 'data3'); -- Verify state (3 chunks with column filters) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_col_filter' ORDER BY schemaname, tablename; -- Insert to create 5 more chunks (total 8 chunks) INSERT INTO test_hypertable VALUES ('2024-01-04 00:00:00+00', 4, 4.0, 'data4'); INSERT INTO test_hypertable VALUES ('2024-01-05 00:00:00+00', 5, 5.0, 'data5'); INSERT INTO test_hypertable VALUES ('2024-01-06 00:00:00+00', 6, 6.0, 'data6'); INSERT INTO test_hypertable VALUES ('2024-01-07 00:00:00+00', 7, 7.0, 'data7'); INSERT INTO test_hypertable VALUES ('2024-01-08 00:00:00+00', 8, 8.0, 'data8'); -- Verify final state (8 chunks with column filters) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_col_filter' ORDER BY schemaname, tablename; -- Cleanup DROP PUBLICATION test_pub_col_filter CASCADE; DROP TABLE test_hypertable CASCADE; -- Test 5: Combined row + column filtering CREATE TABLE test_hypertable (time timestamptz NOT NULL, device_id int, value float, extra text); SELECT create_hypertable('test_hypertable', 'time', chunk_time_interval => interval '1 day'); -- Insert to create first chunk INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1'); -- Create publication with both row and column filters CREATE PUBLICATION test_pub_combined FOR TABLE test_hypertable (time, device_id) WHERE (device_id > 10); -- Verify initial state (1 chunk with both filters) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_combined' ORDER BY schemaname, tablename; -- Insert to create 2 more chunks (total 3 chunks) INSERT INTO test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0, 'data2'); INSERT INTO test_hypertable VALUES ('2024-01-03 00:00:00+00', 3, 3.0, 'data3'); -- Verify state (3 chunks with both filters) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_combined' ORDER BY schemaname, tablename; -- Insert to create 5 more chunks (total 8 chunks) INSERT INTO test_hypertable VALUES ('2024-01-04 00:00:00+00', 4, 4.0, 'data4'); INSERT INTO test_hypertable VALUES ('2024-01-05 00:00:00+00', 5, 5.0, 'data5'); INSERT INTO test_hypertable VALUES ('2024-01-06 00:00:00+00', 6, 6.0, 'data6'); INSERT INTO test_hypertable VALUES ('2024-01-07 00:00:00+00', 7, 7.0, 'data7'); INSERT INTO test_hypertable VALUES ('2024-01-08 00:00:00+00', 8, 8.0, 'data8'); -- Verify final state (8 chunks with both filters) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_combined' ORDER BY schemaname, tablename; -- Cleanup DROP PUBLICATION test_pub_combined CASCADE; DROP TABLE test_hypertable CASCADE; -- Test 6: FOR ALL TABLES publication CREATE TABLE test_hypertable (time timestamptz NOT NULL, device_id int, value float, extra text); SELECT create_hypertable('test_hypertable', 'time', chunk_time_interval => interval '1 day'); -- Insert to create first chunk INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1'); -- Create FOR ALL TABLES publication CREATE PUBLICATION test_pub_all_tables FOR ALL TABLES; -- Verify initial state (1 chunk) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_all_tables' AND tablename LIKE '%test_hypertable%' OR tablename LIKE '_hyper_%' ORDER BY schemaname, tablename; -- Insert to create 2 more chunks (total 3 chunks) INSERT INTO test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0, 'data2'); INSERT INTO test_hypertable VALUES ('2024-01-03 00:00:00+00', 3, 3.0, 'data3'); -- Verify state (3 chunks) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_all_tables' AND tablename LIKE '%test_hypertable%' OR tablename LIKE '_hyper_%' ORDER BY schemaname, tablename; -- Insert to create 5 more chunks (total 8 chunks) INSERT INTO test_hypertable VALUES ('2024-01-04 00:00:00+00', 4, 4.0, 'data4'); INSERT INTO test_hypertable VALUES ('2024-01-05 00:00:00+00', 5, 5.0, 'data5'); INSERT INTO test_hypertable VALUES ('2024-01-06 00:00:00+00', 6, 6.0, 'data6'); INSERT INTO test_hypertable VALUES ('2024-01-07 00:00:00+00', 7, 7.0, 'data7'); INSERT INTO test_hypertable VALUES ('2024-01-08 00:00:00+00', 8, 8.0, 'data8'); -- Verify final state (8 chunks) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub_all_tables' AND tablename LIKE '%test_hypertable%' OR tablename LIKE '_hyper_%' ORDER BY schemaname, tablename; -- Cleanup DROP PUBLICATION test_pub_all_tables CASCADE; DROP TABLE test_hypertable CASCADE; -- Test 7: Edge case - Hypertable not in any publication CREATE TABLE test_hypertable (time timestamptz NOT NULL, device_id int, value float, extra text); SELECT create_hypertable('test_hypertable', 'time', chunk_time_interval => interval '1 day'); -- Insert to create 8 chunks without any publication INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1'); INSERT INTO test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0, 'data2'); INSERT INTO test_hypertable VALUES ('2024-01-03 00:00:00+00', 3, 3.0, 'data3'); INSERT INTO test_hypertable VALUES ('2024-01-04 00:00:00+00', 4, 4.0, 'data4'); INSERT INTO test_hypertable VALUES ('2024-01-05 00:00:00+00', 5, 5.0, 'data5'); INSERT INTO test_hypertable VALUES ('2024-01-06 00:00:00+00', 6, 6.0, 'data6'); INSERT INTO test_hypertable VALUES ('2024-01-07 00:00:00+00', 7, 7.0, 'data7'); INSERT INTO test_hypertable VALUES ('2024-01-08 00:00:00+00', 8, 8.0, 'data8'); -- Verify chunks were created successfully SELECT COUNT(*) as chunks_created FROM timescaledb_information.chunks WHERE hypertable_name = 'test_hypertable'; -- Cleanup DROP TABLE test_hypertable CASCADE; -- Test 8: Edge case - Publication dropped before chunk creation CREATE TABLE test_hypertable (time timestamptz NOT NULL, device_id int, value float, extra text); SELECT create_hypertable('test_hypertable', 'time', chunk_time_interval => interval '1 day'); -- Insert to create first chunk INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1'); -- Create publication and add hypertable CREATE PUBLICATION test_pub FOR TABLE test_hypertable; -- Verify (1 chunk in publication) SELECT schemaname, tablename, attnames, rowfilter FROM pg_publication_tables WHERE pubname = 'test_pub' ORDER BY schemaname, tablename; -- Drop the publication DROP PUBLICATION test_pub; -- Insert to create 2 more chunks (total 3 chunks) -- Should succeed with WARNING, not error INSERT INTO test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0, 'data2'); INSERT INTO test_hypertable VALUES ('2024-01-03 00:00:00+00', 3, 3.0, 'data3'); -- Verify chunks were created successfully despite missing publication SELECT COUNT(*) as chunks_after_pub_drop FROM timescaledb_information.chunks WHERE hypertable_name = 'test_hypertable'; -- Cleanup DROP TABLE test_hypertable CASCADE; -- Test 9: GUC control of chunk publication CREATE TABLE test_hypertable (time timestamptz NOT NULL, device_id int, value float, extra text); SELECT create_hypertable('test_hypertable', 'time', chunk_time_interval => interval '1 day'); -- Insert to create first chunk INSERT INTO test_hypertable VALUES ('2024-01-01 00:00:00+00', 1, 1.0, 'data1'); -- Create publication CREATE PUBLICATION test_pub_guc FOR TABLE test_hypertable; -- Verify initial state (1 chunk) SELECT schemaname, tablename FROM pg_publication_tables WHERE pubname = 'test_pub_guc' ORDER BY schemaname, tablename; -- Test Part 1: GUC enabled - chunks should be added to publication automatically -- Insert to create a new chunk - should be added to publication automatically INSERT INTO test_hypertable VALUES ('2024-01-02 00:00:00+00', 2, 2.0, 'data2'); -- Verify (2 chunks in publication) SELECT schemaname, tablename FROM pg_publication_tables WHERE pubname = 'test_pub_guc' ORDER BY schemaname, tablename; -- Test Part 2: Disable the GUC and create another chunk SET timescaledb.enable_chunk_auto_publication = false; -- Insert to create a new chunk - should NOT be added to publication INSERT INTO test_hypertable VALUES ('2024-01-03 00:00:00+00', 3, 3.0, 'data3'); -- Verify (still 2 chunks in publication, chunk 3 should not be there) SELECT schemaname, tablename FROM pg_publication_tables WHERE pubname = 'test_pub_guc' ORDER BY schemaname, tablename; -- Verify that chunk 3 exists but is not in the publication SELECT chunk_schema, chunk_name FROM timescaledb_information.chunks WHERE hypertable_name = 'test_hypertable'; -- Test Part 3: Re-enable the GUC and create another chunk SET timescaledb.enable_chunk_auto_publication = true; -- Insert to create a new chunk - should be added to publication again INSERT INTO test_hypertable VALUES ('2024-01-04 00:00:00+00', 4, 4.0, 'data4'); -- Verify (3 chunks in publication: chunk 1, 2, and 4; chunk 3 still missing) SELECT schemaname, tablename FROM pg_publication_tables WHERE pubname = 'test_pub_guc' ORDER BY schemaname, tablename; SELECT chunk_schema, chunk_name FROM timescaledb_information.chunks WHERE hypertable_name = 'test_hypertable'; -- Cleanup DROP PUBLICATION test_pub_guc CASCADE; DROP TABLE test_hypertable CASCADE; RESET client_min_messages; ================================================ FILE: test/sql/chunk_utils.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Set this variable to avoid using a hard-coded path each time query -- results are compared \set QUERY_RESULT_TEST_EQUAL_RELPATH 'include/query_result_test_equal.sql' CREATE OR REPLACE FUNCTION dimension_get_time( hypertable_id INT ) RETURNS _timescaledb_catalog.dimension LANGUAGE SQL STABLE AS $BODY$ SELECT * FROM _timescaledb_catalog.dimension d WHERE d.hypertable_id = dimension_get_time.hypertable_id AND d.interval_length IS NOT NULL $BODY$; CREATE TABLE PUBLIC.drop_chunk_test1(time bigint, temp float8, device_id text); CREATE TABLE PUBLIC.drop_chunk_test2(time bigint, temp float8, device_id text); CREATE TABLE PUBLIC.drop_chunk_test3(time bigint, temp float8, device_id text); CREATE INDEX ON drop_chunk_test1(time DESC); -- show_chunks() without specifying a table is not allowed \set ON_ERROR_STOP 0 SELECT show_chunks(NULL); \set ON_ERROR_STOP 1 SELECT create_hypertable('public.drop_chunk_test1', 'time', chunk_time_interval => 1, create_default_indexes=>false); SELECT create_hypertable('public.drop_chunk_test2', 'time', chunk_time_interval => 1, create_default_indexes=>false); SELECT create_hypertable('public.drop_chunk_test3', 'time', chunk_time_interval => 1, create_default_indexes=>false); -- Add space dimensions to ensure chunks share dimension slices SELECT add_dimension('public.drop_chunk_test1', 'device_id', 2); SELECT add_dimension('public.drop_chunk_test2', 'device_id', 2); SELECT add_dimension('public.drop_chunk_test3', 'device_id', 2); SELECT c.id AS chunk_id, c.hypertable_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN dimension_get_time(h.id) time_dimension ON(true) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = time_dimension.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.schema_name = 'public' AND (h.table_name = 'drop_chunk_test1' OR h.table_name = 'drop_chunk_test2') ORDER BY c.id; SELECT * FROM test.relation WHERE schema = '_timescaledb_internal' AND name LIKE '\_hyper%'; SELECT _timescaledb_functions.get_partition_for_key('dev1'::text); SELECT _timescaledb_functions.get_partition_for_key('dev7'::varchar(5)); INSERT INTO PUBLIC.drop_chunk_test1 VALUES(1, 1.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test1 VALUES(2, 2.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test1 VALUES(3, 3.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test1 VALUES(4, 4.0, 'dev7'); INSERT INTO PUBLIC.drop_chunk_test1 VALUES(5, 5.0, 'dev7'); INSERT INTO PUBLIC.drop_chunk_test1 VALUES(6, 6.0, 'dev7'); INSERT INTO PUBLIC.drop_chunk_test2 VALUES(1, 1.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test2 VALUES(2, 2.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test2 VALUES(3, 3.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test2 VALUES(4, 4.0, 'dev7'); INSERT INTO PUBLIC.drop_chunk_test2 VALUES(5, 5.0, 'dev7'); INSERT INTO PUBLIC.drop_chunk_test2 VALUES(6, 6.0, 'dev7'); INSERT INTO PUBLIC.drop_chunk_test3 VALUES(1, 1.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test3 VALUES(2, 2.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test3 VALUES(3, 3.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test3 VALUES(4, 4.0, 'dev7'); INSERT INTO PUBLIC.drop_chunk_test3 VALUES(5, 5.0, 'dev7'); INSERT INTO PUBLIC.drop_chunk_test3 VALUES(6, 6.0, 'dev7'); SELECT c.id AS chunk_id, c.hypertable_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN dimension_get_time(h.id) time_dimension ON(true) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = time_dimension.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.schema_name = 'public' AND (h.table_name = 'drop_chunk_test1' OR h.table_name = 'drop_chunk_test2') ORDER BY c.id; SELECT * FROM test.relation WHERE schema = '_timescaledb_internal' AND name LIKE '\_hyper%'; -- next two calls of show_chunks should give same set of chunks as above when combined SELECT show_chunks('drop_chunk_test1'); SELECT * FROM show_chunks('drop_chunk_test2'); CREATE VIEW dependent_view AS SELECT * FROM _timescaledb_internal._hyper_1_1_chunk; \set ON_ERROR_STOP 0 SELECT drop_chunks('drop_chunk_test1'); SELECT drop_chunks('drop_chunk_test1', older_than => 2); SELECT drop_chunks('drop_chunk_test1', older_than => NULL::interval); SELECT drop_chunks('drop_chunk_test1', older_than => NULL::int); DROP VIEW dependent_view; -- should error because of wrong relative order of time constraints SELECT show_chunks('drop_chunk_test1', older_than=>3, newer_than=>4); -- Should error because NULL was used for the table name. SELECT drop_chunks(NULL, older_than => 3); -- should error because there is no relation with that OID. SELECT drop_chunks(3533, older_than => 3); \set ON_ERROR_STOP 1 -- show created constraints and dimension slices for each chunk SELECT c.table_name, cc.constraint_name, ds.id AS dimension_slice_id, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (c.id = cc.chunk_id) FULL OUTER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.id = cc.dimension_slice_id) ORDER BY c.id; SELECT * FROM _timescaledb_catalog.dimension_slice ORDER BY id; -- Test that truncating chunks works SELECT count(*) FROM _timescaledb_internal._hyper_2_7_chunk; TRUNCATE TABLE _timescaledb_internal._hyper_2_7_chunk; SELECT count(*) FROM _timescaledb_internal._hyper_2_7_chunk; -- Drop one chunk "manually" and verify that dimension slices and -- constraints are cleaned up. Each chunk has two constraints and two -- dimension slices. Both constraints should be deleted, but only one -- slice should be deleted since the space-dimension slice is shared -- with other chunks in the same hypertable DROP TABLE _timescaledb_internal._hyper_2_7_chunk; -- Two constraints deleted compared to above SELECT c.table_name, cc.constraint_name, ds.id AS dimension_slice_id, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (c.id = cc.chunk_id) FULL OUTER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.id = cc.dimension_slice_id) ORDER BY c.id; -- Only one dimension slice deleted SELECT * FROM _timescaledb_catalog.dimension_slice ORDER BY id; -- We drop all chunks older than timestamp 2 in all hypertable. This -- is added only to avoid making the diff for this commit larger than -- necessary and make reviews easier. SELECT drop_chunks(format('%1$I.%2$I', schema_name, table_name)::regclass, older_than => 2) FROM _timescaledb_catalog.hypertable; SELECT c.table_name, cc.constraint_name, ds.id AS dimension_slice_id, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (c.id = cc.chunk_id) FULL OUTER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.id = cc.dimension_slice_id) ORDER BY c.id; SELECT * FROM _timescaledb_catalog.dimension_slice ORDER BY id; SELECT c.id AS chunk_id, c.hypertable_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN dimension_get_time(h.id) time_dimension ON(true) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = time_dimension.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.schema_name = 'public' AND (h.table_name = 'drop_chunk_test1' OR h.table_name = 'drop_chunk_test2') ORDER BY c.id; -- next two calls of show_chunks should give same set of chunks as above when combined SELECT show_chunks('drop_chunk_test1'); SELECT * FROM show_chunks('drop_chunk_test2'); SELECT * FROM test.relation WHERE schema = '_timescaledb_internal' AND name LIKE '\_hyper%'; -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test1\', older_than => 3)::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test1\', older_than => 3)::NAME' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT c.id AS chunk_id, c.hypertable_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN dimension_get_time(h.id) time_dimension ON(true) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = time_dimension.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.schema_name = 'public' AND (h.table_name = 'drop_chunk_test1' OR h.table_name = 'drop_chunk_test2') ORDER BY c.id; SELECT * FROM test.relation WHERE schema = '_timescaledb_internal' AND name LIKE '\_hyper%'; -- next two calls of show_chunks should give same set of chunks as above when combined SELECT show_chunks('drop_chunk_test1'); SELECT * FROM show_chunks('drop_chunk_test2'); -- 2,147,483,647 is the largest int so this tests that BIGINTs work -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test3\', older_than => 2147483648)::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test3\', older_than => 2147483648)::NAME' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT c.id AS chunk_id, c.hypertable_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN dimension_get_time(h.id) time_dimension ON(true) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = time_dimension.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.schema_name = 'public' AND (h.table_name = 'drop_chunk_test1' OR h.table_name = 'drop_chunk_test2' OR h.table_name = 'drop_chunk_test3') ORDER BY c.id; SELECT * FROM test.relation WHERE schema = '_timescaledb_internal' AND name LIKE '\_hyper%'; \set ON_ERROR_STOP 0 -- should error because no hypertable SELECT drop_chunks('drop_chunk_test4', older_than => 5); SELECT show_chunks('drop_chunk_test4'); SELECT show_chunks('drop_chunk_test4', 5); \set ON_ERROR_STOP 1 DROP TABLE _timescaledb_internal._hyper_1_6_chunk; SELECT c.id AS chunk_id, c.hypertable_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN dimension_get_time(h.id) time_dimension ON(true) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = time_dimension.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.schema_name = 'public' AND (h.table_name = 'drop_chunk_test1' OR h.table_name = 'drop_chunk_test2') ORDER BY c.id; SELECT * FROM test.relation WHERE schema = '_timescaledb_internal' AND name LIKE '\_hyper%'; -- newer_than tests -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test1\', newer_than=>5)::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test1\', newer_than=>5, verbose => true)::NAME' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT c.id AS chunk_id, c.hypertable_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN dimension_get_time(h.id) time_dimension ON(true) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = time_dimension.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.schema_name = 'public' AND (h.table_name = 'drop_chunk_test1') ORDER BY c.id; SELECT show_chunks('drop_chunk_test1'); SELECT * FROM test.relation WHERE schema = '_timescaledb_internal' AND name LIKE '_hyper%'; -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test1\', older_than=>4, newer_than=>3)::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test1\', older_than=>4, newer_than=>3)::NAME' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT c.id AS chunk_id, c.hypertable_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN dimension_get_time(h.id) time_dimension ON(true) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = time_dimension.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.schema_name = 'public' AND (h.table_name = 'drop_chunk_test1') ORDER BY c.id; -- the call of show_chunks should give same set of chunks as above SELECT show_chunks('drop_chunk_test1'); SELECT c.id AS chunk_id, c.hypertable_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN dimension_get_time(h.id) time_dimension ON(true) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = time_dimension.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.schema_name = 'public' ORDER BY c.id; -- We support show/drop chunks using timestamps/interval even with integer partitioning -- the chunk creation time gets used for these. But we need to use "created_before, created_after" -- for these \set ON_ERROR_STOP 0 SELECT show_chunks('drop_chunk_test3', older_than => now()); SELECT show_chunks('drop_chunk_test2', older_than => now()); SELECT show_chunks('drop_chunk_test1', newer_than => INTERVAL '15 minutes'); SELECT show_chunks('drop_chunk_test1', older_than => now(), newer_than => INTERVAL '15 minutes'); SELECT drop_chunks('drop_chunk_test1', older_than => now()); -- mix of older_than/newer_than and created_after/created_before doesn't work SELECT show_chunks('drop_chunk_test1', older_than => now(), created_after => INTERVAL '15 minutes'); SELECT show_chunks('drop_chunk_test1', created_before => now(), newer_than => INTERVAL '15 minutes'); \set ON_ERROR_STOP 1 SELECT show_chunks('drop_chunk_test3', created_before => now() + INTERVAL '1 hour'); SELECT show_chunks('drop_chunk_test2', created_before => now() + INTERVAL '1 hour'); SELECT show_chunks('drop_chunk_test1', created_after => INTERVAL '15 minutes'); SELECT show_chunks('drop_chunk_test1', created_before => now() + INTERVAL '1 hour', created_after => INTERVAL '1 hour'); SELECT drop_chunks('drop_chunk_test1', created_before => now() + INTERVAL '1 hour'); SELECT drop_chunks(format('%1$I.%2$I', schema_name, table_name)::regclass, older_than => 5, newer_than => 4) FROM _timescaledb_catalog.hypertable WHERE schema_name = 'public'; CREATE TABLE PUBLIC.drop_chunk_test_ts(time timestamp, temp float8, device_id text); SELECT create_hypertable('public.drop_chunk_test_ts', 'time', chunk_time_interval => interval '1 minute', create_default_indexes=>false); CREATE TABLE PUBLIC.drop_chunk_test_tstz(time timestamptz, temp float8, device_id text); SELECT create_hypertable('public.drop_chunk_test_tstz', 'time', chunk_time_interval => interval '1 minute', create_default_indexes=>false); SET timezone = '+1'; INSERT INTO PUBLIC.drop_chunk_test_ts VALUES(now()-INTERVAL '5 minutes', 1.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test_ts VALUES(now()+INTERVAL '5 minutes', 1.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test_tstz VALUES(now()-INTERVAL '5 minutes', 1.0, 'dev1'); INSERT INTO PUBLIC.drop_chunk_test_tstz VALUES(now()+INTERVAL '5 minutes', 1.0, 'dev1'); SELECT * FROM test.show_subtables('drop_chunk_test_ts'); SELECT * FROM test.show_subtables('drop_chunk_test_tstz'); BEGIN; SELECT show_chunks('drop_chunk_test_ts'); SELECT show_chunks('drop_chunk_test_ts', now()::timestamp-interval '1 minute'); -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test_ts\', newer_than => interval \'1 minute\')::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_ts\', newer_than => interval \'1 minute\')::NAME' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all \set QUERY1 'SELECT show_chunks(\'drop_chunk_test_ts\', older_than => interval \'6 minute\')::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_ts\', older_than => interval \'6 minute\')::NAME' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT * FROM test.show_subtables('drop_chunk_test_ts'); \set QUERY1 'SELECT show_chunks(\'drop_chunk_test_ts\', older_than => interval \'1 minute\')::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_ts\', interval \'1 minute\')::NAME' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT * FROM test.show_subtables('drop_chunk_test_ts'); SELECT show_chunks('drop_chunk_test_tstz'); SELECT show_chunks('drop_chunk_test_tstz', older_than => now() - interval '1 minute', newer_than => now() - interval '6 minute'); SELECT show_chunks('drop_chunk_test_tstz', newer_than => now() - interval '1 minute'); SELECT show_chunks('drop_chunk_test_tstz', older_than => now() - interval '1 minute'); \set QUERY1 'SELECT show_chunks(older_than => interval \'1 minute\', relation => \'drop_chunk_test_tstz\')::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_tstz\', interval \'1 minute\')::NAME' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT * FROM test.show_subtables('drop_chunk_test_tstz'); ROLLBACK; BEGIN; -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test_ts\', newer_than => interval \'6 minute\')::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_ts\', newer_than => interval \'6 minute\')::NAME' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT * FROM test.show_subtables('drop_chunk_test_ts'); ROLLBACK; BEGIN; -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test_ts\', older_than => interval \'1 minute\')::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_ts\', older_than => interval \'1 minute\')::NAME' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT * FROM test.show_subtables('drop_chunk_test_ts'); \set QUERY1 'SELECT show_chunks(\'drop_chunk_test_tstz\', older_than => interval \'1 minute\')::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_tstz\', older_than => interval \'1 minute\')::NAME' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT * FROM test.show_subtables('drop_chunk_test_tstz'); ROLLBACK; BEGIN; -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test_ts\', older_than => now()::timestamp-interval \'1 minute\')::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_ts\', older_than => now()::timestamp-interval \'1 minute\')::NAME' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT * FROM test.show_subtables('drop_chunk_test_ts'); \set QUERY1 'SELECT show_chunks(\'drop_chunk_test_tstz\', older_than => now()-interval \'1 minute\')::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_tstz\', older_than => now()-interval \'1 minute\')::NAME' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT * FROM test.show_subtables('drop_chunk_test_tstz'); ROLLBACK; SELECT * FROM test.relation WHERE schema = '_timescaledb_internal' AND name LIKE '\_hyper%'; \set ON_ERROR_STOP 0 SELECT drop_chunks(interval '1 minute'); SELECT drop_chunks('drop_chunk_test_ts', (now()-interval '1 minute')); SELECT drop_chunks('drop_chunk_test3', verbose => true); SELECT drop_chunks('drop_chunk_test3', interval '1 minute'); \set ON_ERROR_STOP 1 -- Interval boundary for INTEGER type columns. It uses chunk creation -- time to identify the affected chunks. SELECT drop_chunks('drop_chunk_test3', created_after => interval '1 minute'); SELECT * FROM test.relation WHERE schema = '_timescaledb_internal' AND name LIKE '\_hyper%'; CREATE TABLE PUBLIC.drop_chunk_test_date(time date, temp float8, device_id text); SELECT create_hypertable('public.drop_chunk_test_date', 'time', chunk_time_interval => interval '1 day', create_default_indexes=>false); SET timezone = '+100'; INSERT INTO PUBLIC.drop_chunk_test_date VALUES(now()-INTERVAL '2 day', 1.0, 'dev1'); BEGIN; -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test_date\', older_than => interval \'1 day\')::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_date\', older_than => interval \'1 day\')::NAME' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT * FROM test.show_subtables('drop_chunk_test_date'); ROLLBACK; BEGIN; -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test_date\', older_than => (now()-interval \'1 day\')::date)::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test_date\', older_than => (now()-interval \'1 day\')::date)::NAME' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT * FROM test.show_subtables('drop_chunk_test_date'); ROLLBACK; SET timezone TO '-5'; CREATE TABLE chunk_id_from_relid_test(time bigint, temp float8, device_id int); SELECT hypertable_id FROM create_hypertable('chunk_id_from_relid_test', 'time', chunk_time_interval => 10) \gset INSERT INTO chunk_id_from_relid_test VALUES (0, 1.1, 0), (0, 1.3, 11), (12, 2.0, 0), (12, 0.1, 11); SELECT _timescaledb_functions.chunk_id_from_relid(tableoid) FROM chunk_id_from_relid_test; DROP TABLE chunk_id_from_relid_test; CREATE TABLE chunk_id_from_relid_test(time bigint, temp float8, device_id int); SELECT hypertable_id FROM create_hypertable('chunk_id_from_relid_test', 'time', chunk_time_interval => 10, partitioning_column => 'device_id', number_partitions => 3) \gset INSERT INTO chunk_id_from_relid_test VALUES (0, 1.1, 2), (0, 1.3, 11), (12, 2.0, 2), (12, 0.1, 11); SELECT _timescaledb_functions.chunk_id_from_relid(tableoid) FROM chunk_id_from_relid_test; \set ON_ERROR_STOP 0 SELECT _timescaledb_functions.chunk_id_from_relid('pg_type'::regclass); SELECT _timescaledb_functions.chunk_id_from_relid('chunk_id_from_relid_test'::regclass); -- test drop/show_chunks on custom partition types CREATE FUNCTION extract_time(a jsonb) RETURNS TIMESTAMPTZ LANGUAGE SQL AS $$ SELECT (a->>'time')::TIMESTAMPTZ $$ IMMUTABLE; CREATE TABLE test_weird_type(a jsonb); SELECT create_hypertable('test_weird_type', 'a', time_partitioning_func=>'extract_time'::regproc, chunk_time_interval=>'2 hours'::interval); INSERT INTO test_weird_type VALUES ('{"time":"2019/06/06 1:00+0"}'), ('{"time":"2019/06/06 5:00+0"}'); SELECT * FROM test.show_subtables('test_weird_type'); SELECT show_chunks('test_weird_type', older_than=>'2019/06/06 4:00+0'::TIMESTAMPTZ); SELECT show_chunks('test_weird_type', older_than=>'2019/06/06 10:00+0'::TIMESTAMPTZ); -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'test_weird_type\', older_than => \'2019/06/06 5:00+0\'::TIMESTAMPTZ)::NAME' \set QUERY2 'SELECT drop_chunks(\'test_weird_type\', older_than => \'2019/06/06 5:00+0\'::TIMESTAMPTZ)::NAME' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT * FROM test.show_subtables('test_weird_type'); SELECT show_chunks('test_weird_type', older_than=>'2019/06/06 4:00+0'::TIMESTAMPTZ); SELECT show_chunks('test_weird_type', older_than=>'2019/06/06 10:00+0'::TIMESTAMPTZ); -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'test_weird_type\', older_than => \'2019/06/06 6:00+0\'::TIMESTAMPTZ)::NAME' \set QUERY2 'SELECT drop_chunks(\'test_weird_type\', older_than => \'2019/06/06 6:00+0\'::TIMESTAMPTZ)::NAME' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT * FROM test.show_subtables('test_weird_type'); SELECT show_chunks('test_weird_type', older_than=>'2019/06/06 10:00+0'::TIMESTAMPTZ); DROP TABLE test_weird_type; CREATE FUNCTION extract_int_time(a jsonb) RETURNS BIGINT LANGUAGE SQL AS $$ SELECT (a->>'time')::BIGINT $$ IMMUTABLE; CREATE TABLE test_weird_type_i(a jsonb); SELECT create_hypertable('test_weird_type_i', 'a', time_partitioning_func=>'extract_int_time'::regproc, chunk_time_interval=>5); INSERT INTO test_weird_type_i VALUES ('{"time":"0"}'), ('{"time":"5"}'); SELECT * FROM test.show_subtables('test_weird_type_i'); SELECT show_chunks('test_weird_type_i', older_than=>5); SELECT show_chunks('test_weird_type_i', older_than=>10); -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'test_weird_type_i\', older_than=>5)::NAME' \set QUERY2 'SELECT drop_chunks(\'test_weird_type_i\', older_than => 5)::NAME' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT * FROM test.show_subtables('test_weird_type_i'); SELECT show_chunks('test_weird_type_i', older_than=>5); SELECT show_chunks('test_weird_type_i', older_than=>10); -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'test_weird_type_i\', older_than=>10)::NAME' \set QUERY2 'SELECT drop_chunks(\'test_weird_type_i\', older_than => 10)::NAME' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT * FROM test.show_subtables('test_weird_type_i'); SELECT show_chunks('test_weird_type_i', older_than=>10); DROP TABLE test_weird_type_i CASCADE; \c :TEST_DBNAME :ROLE_SUPERUSER ALTER TABLE drop_chunk_test2 OWNER TO :ROLE_DEFAULT_PERM_USER_2; --drop chunks 3 will have a chunk we a dependent object (a view) --we create the dependent object now INSERT INTO PUBLIC.drop_chunk_test3 VALUES(1, 1.0, 'dev1'); SELECT c.schema_name as chunk_schema, c.table_name as chunk_table FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) WHERE h.schema_name = 'public' AND h.table_name = 'drop_chunk_test3' ORDER BY c.id \gset create view dependent_view as SELECT * FROM :"chunk_schema".:"chunk_table"; create view dependent_view2 as SELECT * FROM :"chunk_schema".:"chunk_table"; ALTER TABLE drop_chunk_test3 OWNER TO :ROLE_DEFAULT_PERM_USER_2; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 \set ON_ERROR_STOP 0 SELECT drop_chunks('drop_chunk_test1', older_than=>4, newer_than=>3); --works with modified owner tables -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT show_chunks(\'drop_chunk_test2\', older_than=>4, newer_than=>3)::NAME' \set QUERY2 'SELECT drop_chunks(\'drop_chunk_test2\', older_than=>4, newer_than=>3)::NAME' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all \set VERBOSITY default --this fails because there are dependent objects SELECT drop_chunks('drop_chunk_test3', older_than=>100); \set VERBOSITY terse \c :TEST_DBNAME :ROLE_SUPERUSER DROP VIEW dependent_view; DROP VIEW dependent_view2; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 \set ON_ERROR_STOP 1 --drop chunks from hypertable with same name in different schema -- order of schema in search_path matters -- \c :TEST_DBNAME :ROLE_SUPERUSER drop table chunk_id_from_relid_test; drop table drop_chunk_test1; drop table drop_chunk_test2; drop table drop_chunk_test3; CREATE SCHEMA try_schema; CREATE SCHEMA test1; CREATE SCHEMA test2; CREATE SCHEMA test3; GRANT CREATE ON SCHEMA try_schema, test1, test2, test3 TO :ROLE_DEFAULT_PERM_USER; GRANT USAGE ON SCHEMA try_schema, test1, test2, test3 TO :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE TABLE try_schema.drop_chunk_test_date(time date, temp float8, device_id text); SELECT create_hypertable('try_schema.drop_chunk_test_date', 'time', chunk_time_interval => interval '1 day', create_default_indexes=>false); INSERT INTO public.drop_chunk_test_date VALUES( '2020-01-10', 100, 'hello'); INSERT INTO try_schema.drop_chunk_test_date VALUES( '2020-01-10', 100, 'hello'); set search_path to try_schema, test1, test2, test3, public; SELECT show_chunks('public.drop_chunk_test_date', older_than=>'1 day'::interval); SELECT show_chunks('try_schema.drop_chunk_test_date', older_than=>'1 day'::interval); SELECT drop_chunks('drop_chunk_test_date', older_than=> '1 day'::interval); -- test drop chunks across two tables within the same schema CREATE TABLE test1.hyper1 (time bigint, temp float); CREATE TABLE test1.hyper2 (time bigint, temp float); SELECT create_hypertable('test1.hyper1', 'time', chunk_time_interval => 10); SELECT create_hypertable('test1.hyper2', 'time', chunk_time_interval => 10); INSERT INTO test1.hyper1 VALUES (10, 0.5); INSERT INTO test1.hyper2 VALUES (10, 0.7); SELECT show_chunks('test1.hyper1'); SELECT show_chunks('test1.hyper2'); -- test drop chunks for given table name across all schemas CREATE TABLE test2.hyperx (time bigint, temp float); CREATE TABLE test3.hyperx (time bigint, temp float); SELECT create_hypertable('test2.hyperx', 'time', chunk_time_interval => 10); SELECT create_hypertable('test3.hyperx', 'time', chunk_time_interval => 10); INSERT INTO test2.hyperx VALUES (10, 0.5); INSERT INTO test3.hyperx VALUES (10, 0.7); SELECT show_chunks('test2.hyperx'); SELECT show_chunks('test3.hyperx'); -- This will only drop from one of the tables since the one that is -- first in the search path will hide the other one. SELECT drop_chunks('hyperx', older_than => 100); SELECT show_chunks('test2.hyperx'); SELECT show_chunks('test3.hyperx'); -- Check CTAS behavior when internal ALTER TABLE gets fired CREATE TABLE PUBLIC.drop_chunk_test4(time bigint, temp float8, device_id text); CREATE TABLE drop_chunks_table_id AS SELECT hypertable_id FROM create_hypertable('public.drop_chunk_test4', 'time', chunk_time_interval => 1); -- TEST for internal api that drops a single chunk -- this drops the table and removes entry from the catalog. -- does not affect any materialized cagg data INSERT INTO test1.hyper1 VALUES (20, 0.5); SELECT chunk_schema as "CHSCHEMA", chunk_name as "CHNAME" FROM timescaledb_information.chunks WHERE hypertable_name = 'hyper1' and hypertable_schema = 'test1' ORDER BY chunk_name ; --drop one of the chunks SELECT chunk_schema || '.' || chunk_name as "CHNAME" FROM timescaledb_information.chunks WHERE hypertable_name = 'hyper1' and hypertable_schema = 'test1' ORDER BY chunk_name LIMIT 1 \gset SELECT _timescaledb_functions.drop_chunk(:'CHNAME'); SELECT chunk_schema as "CHSCHEMA", chunk_name as "CHNAME" FROM timescaledb_information.chunks WHERE hypertable_name = 'hyper1' and hypertable_schema = 'test1' ORDER BY chunk_name ; -- "created_before/after" can be used with time partitioning in drop/show chunks SELECT show_chunks('drop_chunk_test_tstz', created_before => now() - INTERVAL '1 hour'); SELECT drop_chunks('drop_chunk_test_tstz', created_before => now() + INTERVAL '1 hour'); SELECT show_chunks('drop_chunk_test_ts'); -- "created_before/after" accept timestamptz even though partitioning col is just -- timestamp SELECT show_chunks('drop_chunk_test_ts', created_after => now() - INTERVAL '1 hour', created_before => now()); SELECT drop_chunks('drop_chunk_test_ts', created_after => INTERVAL '1 hour', created_before => now()); -- Test views on top of hypertables CREATE TABLE view_test (project_id INT, ts TIMESTAMPTZ NOT NULL); SELECT create_hypertable('view_test', by_range('ts', INTERVAL '1 day')); -- exactly one partition per project_id SELECT * FROM add_dimension('view_test', 'project_id', chunk_time_interval => 1); -- exactly one partition per project; works for *integer* types INSERT INTO view_test (project_id, ts) SELECT g % 25 + 1 AS project_id, i.ts + (g * interval '1 week') / i.total AS ts FROM (SELECT timestamptz '2024-01-01 00:00:00+0', 600) i(ts, total), generate_series(1, i.total) g; -- Create a view on top of this hypertable CREATE VIEW test_view_part_few AS SELECT project_id, ts FROM view_test WHERE project_id = ANY (ARRAY[5, 10, 15]); -- Complicated query on a view involving a range check and a sort SELECT * FROM test_view_part_few WHERE ts BETWEEN '2024-01-04 00:00:00+00'AND '2024-01-05 00:00:00' ORDER BY ts LIMIT 1000; -- Test chunk_status_text function CREATE TABLE chunk_status_test(time timestamptz) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.columnstore=false); INSERT INTO chunk_status_test VALUES ('2025-01-01'),('2025-02-01'),('2025-03-01'); SELECT _timescaledb_functions.chunk_status_text(i) FROM generate_series(0,15) i; SELECT chunk, _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('chunk_status_test') chunk; SELECT _timescaledb_functions.chunk_status_text(NULL::int); SELECT _timescaledb_functions.chunk_status_text(NULL::regclass); \set ON_ERROR_STOP 0 SELECT _timescaledb_functions.chunk_status_text(-1); SELECT _timescaledb_functions.chunk_status_text(16); SELECT _timescaledb_functions.chunk_status_text(1000); SELECT _timescaledb_functions.chunk_status_text(0::regclass); SELECT _timescaledb_functions.chunk_status_text('pg_class'::regclass); \set ON_ERROR_STOP 1 -- Test that function exists and returns an array type SELECT pg_typeof(_timescaledb_functions.chunk_status_text(0)); ================================================ FILE: test/sql/chunks.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \unset ECHO \o /dev/null \ir include/test_utils.sql \o \set ECHO errors \set VERBOSITY default \o /dev/null \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION _timescaledb_internal.dimension_calculate_default_range_open( dimension_value BIGINT, interval_length BIGINT, dimension_type CSTRING, OUT range_start BIGINT, OUT range_end BIGINT) AS :MODULE_PATHNAME, 'ts_dimension_calculate_open_range_default' LANGUAGE C STABLE; CREATE OR REPLACE FUNCTION _timescaledb_internal.dimension_calculate_default_range_closed( dimension_value BIGINT, num_slices SMALLINT, OUT range_start BIGINT, OUT range_end BIGINT) AS :MODULE_PATHNAME, 'ts_dimension_calculate_closed_range_default' LANGUAGE C STABLE; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --open SELECT assert_equal(0::bigint, actual_range_start), assert_equal(10::bigint, actual_range_end) FROM _timescaledb_internal.dimension_calculate_default_range_open(0,10, 'int8') AS res(actual_range_start, actual_range_end); SELECT assert_equal(0::bigint, actual_range_start), assert_equal(10::bigint, actual_range_end) FROM _timescaledb_internal.dimension_calculate_default_range_open(9,10, 'int4') AS res(actual_range_start, actual_range_end); SELECT assert_equal(10::bigint, actual_range_start), assert_equal(20::bigint, actual_range_end) FROM _timescaledb_internal.dimension_calculate_default_range_open(10,10, 'int2') AS res(actual_range_start, actual_range_end); SELECT assert_equal(-10::bigint, actual_range_start), assert_equal(0::bigint, actual_range_end) FROM _timescaledb_internal.dimension_calculate_default_range_open(-1,10, 'int8') AS res(actual_range_start, actual_range_end); SELECT assert_equal(-10::bigint, actual_range_start), assert_equal(0::bigint, actual_range_end) FROM _timescaledb_internal.dimension_calculate_default_range_open(-10,10, 'int4') AS res(actual_range_start, actual_range_end); SELECT assert_equal(-20::bigint, actual_range_start), assert_equal(-10::bigint, actual_range_end) FROM _timescaledb_internal.dimension_calculate_default_range_open(-11,10, 'int2') AS res(actual_range_start, actual_range_end); --test that the ends are cut as needed to prevent overflow/undeflow. --------------- -- BIGINT --------------- SELECT assert_equal(-9223372036854775808, actual_range_start), assert_equal(-9223372036854775800::bigint, actual_range_end) FROM _timescaledb_internal.dimension_calculate_default_range_open(-9223372036854775808,10, 'int8') AS res(actual_range_start, actual_range_end); SELECT assert_equal(-9223372036854775808, actual_range_start), assert_equal(-9223372036854775800::bigint, actual_range_end) FROM _timescaledb_internal.dimension_calculate_default_range_open(-9223372036854775807,10, 'int8') AS res(actual_range_start, actual_range_end); SELECT assert_equal(9223372036854775800::bigint, actual_range_start), assert_equal(9223372036854775807::bigint, actual_range_end) FROM _timescaledb_internal.dimension_calculate_default_range_open(9223372036854775807,10, 'int8') AS res(actual_range_start, actual_range_end); SELECT assert_equal(9223372036854775800::bigint, actual_range_start), assert_equal(9223372036854775807::bigint, actual_range_end) FROM _timescaledb_internal.dimension_calculate_default_range_open(9223372036854775806,10, 'int8') AS res(actual_range_start, actual_range_end); --------------- -- INT --------------- SELECT assert_equal(-9223372036854775808, actual_range_start), assert_equal(-2147483640::bigint, actual_range_end) FROM _timescaledb_internal.dimension_calculate_default_range_open(-2147483648,10, 'int4') AS res(actual_range_start, actual_range_end); SELECT assert_equal(-9223372036854775808, actual_range_start), assert_equal(-2147483640::bigint, actual_range_end) FROM _timescaledb_internal.dimension_calculate_default_range_open(-2147483647,10, 'int4') AS res(actual_range_start, actual_range_end); SELECT assert_equal(2147483640::bigint, actual_range_start), assert_equal(9223372036854775807::bigint, actual_range_end) FROM _timescaledb_internal.dimension_calculate_default_range_open(2147483647,10, 'int4') AS res(actual_range_start, actual_range_end); SELECT assert_equal(2147483640::bigint, actual_range_start), assert_equal(9223372036854775807::bigint, actual_range_end) FROM _timescaledb_internal.dimension_calculate_default_range_open(2147483646,10, 'int4') AS res(actual_range_start, actual_range_end); --------------- -- SMALLINT --------------- SELECT assert_equal(-9223372036854775808, actual_range_start), assert_equal(-32760::bigint, actual_range_end) FROM _timescaledb_internal.dimension_calculate_default_range_open(-32768,10, 'int2') AS res(actual_range_start, actual_range_end); SELECT assert_equal(-9223372036854775808, actual_range_start), assert_equal(-32760::bigint, actual_range_end) FROM _timescaledb_internal.dimension_calculate_default_range_open(-32767,10, 'int2') AS res(actual_range_start, actual_range_end); SELECT assert_equal(32760::bigint, actual_range_start), assert_equal(9223372036854775807::bigint, actual_range_end) FROM _timescaledb_internal.dimension_calculate_default_range_open(32767,10, 'int2') AS res(actual_range_start, actual_range_end); SELECT assert_equal(32760::bigint, actual_range_start), assert_equal(9223372036854775807::bigint, actual_range_end) FROM _timescaledb_internal.dimension_calculate_default_range_open(32766,10, 'int2') AS res(actual_range_start, actual_range_end); --closed SELECT assert_equal((-9223372036854775808)::bigint, actual_range_start), assert_equal(1073741823::bigint, actual_range_end) FROM _timescaledb_internal.dimension_calculate_default_range_closed(0,2::smallint) AS res(actual_range_start, actual_range_end); SELECT assert_equal(1073741823::bigint, actual_range_start), assert_equal(9223372036854775807::bigint, actual_range_end) FROM _timescaledb_internal.dimension_calculate_default_range_closed(1073741824,2::smallint) AS res(actual_range_start, actual_range_end); SELECT assert_equal((-9223372036854775808)::bigint, actual_range_start), assert_equal(9223372036854775807::bigint, actual_range_end) FROM _timescaledb_internal.dimension_calculate_default_range_closed(1073741824,1::smallint) AS res(actual_range_start, actual_range_end); ================================================ FILE: test/sql/cluster.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE cluster_test(time timestamptz, temp float, location int); SELECT create_hypertable('cluster_test', 'time', chunk_time_interval => interval '1 day'); -- Show default indexes SELECT * FROM test.show_indexes('cluster_test'); -- Create two chunks INSERT INTO cluster_test VALUES ('2017-01-20T09:00:01', 23.4, 1), ('2017-01-21T09:00:01', 21.3, 2); -- Run cluster CLUSTER VERBOSE cluster_test USING cluster_test_time_idx; -- Create a third chunk INSERT INTO cluster_test VALUES ('2017-01-22T09:00:01', 19.5, 3); -- Show clustered indexes SELECT indexrelid::regclass, indisclustered FROM pg_index WHERE indisclustered = true ORDER BY 1; -- Reorder just our table CLUSTER VERBOSE cluster_test; -- Show clustered indexes, including new chunk SELECT indexrelid::regclass, indisclustered FROM pg_index WHERE indisclustered = true ORDER BY 1; -- Reorder all tables (although will only be our test table) CLUSTER VERBOSE; -- Change the clustered index CREATE INDEX ON cluster_test (time, location); CLUSTER VERBOSE cluster_test using cluster_test_time_location_idx; -- Show updated clustered indexes SELECT indexrelid::regclass, indisclustered FROM pg_index WHERE indisclustered = true ORDER BY 1; --check the setting of cluster indexes on hypertables and chunks ALTER TABLE cluster_test CLUSTER ON cluster_test_time_idx; SELECT indexrelid::regclass, indisclustered FROM pg_index WHERE indisclustered = true ORDER BY 1,2; CLUSTER VERBOSE cluster_test; ALTER TABLE cluster_test SET WITHOUT CLUSTER; SELECT indexrelid::regclass, indisclustered FROM pg_index WHERE indisclustered = true ORDER BY 1,2; \set ON_ERROR_STOP 0 CLUSTER VERBOSE cluster_test; \set ON_ERROR_STOP 1 ALTER TABLE _timescaledb_internal._hyper_1_1_chunk CLUSTER ON _hyper_1_1_chunk_cluster_test_time_idx; SELECT indexrelid::regclass, indisclustered FROM pg_index WHERE indisclustered = true ORDER BY 1,2; CLUSTER VERBOSE _timescaledb_internal._hyper_1_1_chunk; ALTER TABLE _timescaledb_internal._hyper_1_1_chunk SET WITHOUT CLUSTER; SELECT indexrelid::regclass, indisclustered FROM pg_index WHERE indisclustered = true ORDER BY 1,2; \set ON_ERROR_STOP 0 CLUSTER VERBOSE _timescaledb_internal._hyper_1_1_chunk; \set ON_ERROR_STOP 1 -- test alter column type on hypertable with clustering CREATE TABLE cluster_alter(time timestamp, id text, val int); CREATE INDEX idstuff ON cluster_alter USING btree (id ASC NULLS LAST, time); SELECT table_name FROM create_hypertable('cluster_alter', 'time'); INSERT INTO cluster_alter VALUES('2020-01-01', '123', 1); CLUSTER cluster_alter using idstuff; --attempt the alter table ALTER TABLE cluster_alter ALTER COLUMN id TYPE int USING id::int; CLUSTER cluster_alter; CLUSTER cluster_alter using idstuff; ================================================ FILE: test/sql/constraint.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE hyper ( time BIGINT NOT NULL, device_id TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10) ); SELECT * FROM create_hypertable('hyper', 'time', chunk_time_interval => 10); --check and not-null constraints are inherited through regular inheritance. \set ON_ERROR_STOP 0 INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 9); INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987700000000000, NULL, 11); ALTER TABLE hyper ALTER COLUMN time DROP NOT NULL; ALTER TABLE ONLY hyper ALTER COLUMN sensor_1 SET NOT NULL; ALTER TABLE ONLY hyper ALTER COLUMN device_id DROP NOT NULL; \set ON_ERROR_STOP 1 INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); ALTER TABLE hyper ALTER COLUMN device_id DROP NOT NULL; INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987700000000000, NULL, 11); --make sure validate works \set ON_ERROR_STOP 0 ALTER TABLE hyper ADD CONSTRAINT bad_check_const CHECK (sensor_1 > 100); \set ON_ERROR_STOP 1 ALTER TABLE hyper ADD CONSTRAINT bad_check_const CHECK (sensor_1 > 100) NOT VALID; \set ON_ERROR_STOP 0 ALTER TABLE hyper VALIDATE CONSTRAINT bad_check_const; \set ON_ERROR_STOP 1 ----------------------- UNIQUE CONSTRAINTS ------------------ CREATE TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name ( time BIGINT NOT NULL UNIQUE, device_id TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10) ); SELECT * FROM create_hypertable('hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name', 'time', chunk_time_interval => 10); INSERT INTO hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); INSERT INTO hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name(time, device_id,sensor_1) VALUES (1257987800000000000, 'dev2', 11); \set ON_ERROR_STOP 0 INSERT INTO hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); \set ON_ERROR_STOP 1 -- Show constraints on main tables SELECT * FROM _timescaledb_catalog.chunk_constraint; SELECT * FROM test.show_constraints('hyper'); SELECT * FROM test.show_constraints('hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name'); --should have unique constraint not just unique index SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_2_4_chunk'); ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name DROP CONSTRAINT hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key; -- The constraint should have been removed from the chunk as well SELECT * FROM _timescaledb_catalog.chunk_constraint; SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_2_4_chunk'); --uniqueness not enforced INSERT INTO hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev3', 11); --shouldn't be able to create constraint \set ON_ERROR_STOP 0 ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name ADD CONSTRAINT hyper_unique_time_key UNIQUE (time); \set ON_ERROR_STOP 1 DELETE FROM hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name WHERE device_id = 'dev3'; -- Try multi-alter table statement with a constraint without a name ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name ADD CHECK (time > 0), ADD UNIQUE (time) DEFERRABLE INITIALLY DEFERRED; \set ON_ERROR_STOP 0 BEGIN; --testing deferred checking. The following row has an error, which will not appear until the commit INSERT INTO hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev3', 11); SELECT 1; COMMIT; \set ON_ERROR_STOP 1 SELECT * FROM _timescaledb_catalog.chunk_constraint; SELECT * FROM test.show_constraints('hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name'); SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_2_4_chunk'); ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name DROP CONSTRAINT hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key, DROP CONSTRAINT hyper_unique_with_looooooooooooooooooooooooooooooooo_time_check; SELECT * FROM _timescaledb_catalog.chunk_constraint; SELECT * FROM test.show_constraints('hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name'); SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_2_4_chunk'); CREATE UNIQUE INDEX hyper_unique_with_looooooooooooooooooooooooooooooooo_time_idx ON hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name (time); \set ON_ERROR_STOP 0 -- Try adding constraint using existing index ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name ADD CONSTRAINT hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key UNIQUE USING INDEX hyper_unique_with_looooooooooooooooooooooooooooooooo_time_idx; \set ON_ERROR_STOP 1 DROP INDEX hyper_unique_with_looooooooooooooooooooooooooooooooo_time_idx; --now can create ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name ADD CONSTRAINT hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key UNIQUE (time); SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_2_4_chunk'); --test adding constraint with same name to different table -- should fail \set ON_ERROR_STOP 0 ALTER TABLE hyper ADD CONSTRAINT hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key UNIQUE (time); \set ON_ERROR_STOP 1 --uniquness violation fails \set ON_ERROR_STOP 0 INSERT INTO hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); \set ON_ERROR_STOP 1 --cannot create unique constraint on non-partition column \set ON_ERROR_STOP 0 ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name ADD CONSTRAINT hyper_unique_invalid UNIQUE (device_id); ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name ADD COLUMN new_device_id int UNIQUE; ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name DROP COLUMN device_id, ADD COLUMN new_device_id int UNIQUE; \set ON_ERROR_STOP 1 ----------------------- RENAME CONSTRAINT ------------------ ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name RENAME CONSTRAINT hyper_unique_with_looooooooooooooooooooooooooooooooooo_time_key TO new_name; ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name * RENAME CONSTRAINT new_name TO new_name2; ALTER TABLE IF EXISTS hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name RENAME CONSTRAINT hyper_unique_with_looooooooooooooooooooooooooooo_sensor_1_check TO check_2; SELECT * FROM test.show_constraints('hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name'); SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_2_4_chunk'); SELECT * FROM _timescaledb_catalog.chunk_constraint; \set ON_ERROR_STOP 0 ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name RENAME CONSTRAINT new_name TO new_name2; ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name RENAME CONSTRAINT new_name2 TO check_2; ALTER TABLE ONLY hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name RENAME CONSTRAINT new_name2 TO new_name; ALTER TABLE _timescaledb_internal._hyper_2_4_chunk RENAME CONSTRAINT "4_10_new_name2" TO new_name; \set ON_ERROR_STOP 1 ----------------------- PRIMARY KEY ------------------ CREATE TABLE hyper_pk ( time BIGINT NOT NULL PRIMARY KEY, device_id TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10) ); SELECT * FROM create_hypertable('hyper_pk', 'time', chunk_time_interval => 10); INSERT INTO hyper_pk(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); \set ON_ERROR_STOP 0 INSERT INTO hyper_pk(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); \set ON_ERROR_STOP 1 --should have unique constraint not just unique index SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_3_6_chunk'); ALTER TABLE hyper_pk DROP CONSTRAINT hyper_pk_pkey; SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_3_6_chunk'); --uniqueness not enforced INSERT INTO hyper_pk(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev3', 11); --shouldn't be able to create pk \set ON_ERROR_STOP 0 ALTER TABLE hyper_pk ADD CONSTRAINT hyper_pk_pkey PRIMARY KEY (time); ALTER TABLE hyper_unique_with_looooooooooooooooooooooooooooooooooooong_name ADD COLUMN new_device_id int PRIMARY KEY; \set ON_ERROR_STOP 1 DELETE FROM hyper_pk WHERE device_id = 'dev3'; --cannot create pk constraint on non-partition column \set ON_ERROR_STOP 0 ALTER TABLE hyper_pk ADD CONSTRAINT hyper_pk_invalid PRIMARY KEY (device_id); \set ON_ERROR_STOP 1 --now can create ALTER TABLE hyper_pk ADD CONSTRAINT hyper_pk_pkey PRIMARY KEY (time) DEFERRABLE INITIALLY DEFERRED; SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_3_6_chunk'); --test adding constraint with same name to different table -- should fail \set ON_ERROR_STOP 0 ALTER TABLE hyper ADD CONSTRAINT hyper_pk_pkey UNIQUE (time); \set ON_ERROR_STOP 1 --uniquness violation fails \set ON_ERROR_STOP 0 BEGIN; --error here deferred until commit INSERT INTO hyper_pk(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); SELECT 1; COMMIT; \set ON_ERROR_STOP 1 ----------------------- FOREIGN KEY ------------------ CREATE TABLE devices( device_id TEXT NOT NULL, PRIMARY KEY (device_id) ); CREATE TABLE hyper_fk ( time BIGINT NOT NULL PRIMARY KEY, device_id TEXT NOT NULL REFERENCES devices(device_id), sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10) ); SELECT * FROM create_hypertable('hyper_fk', 'time', chunk_time_interval => 10); --fail fk constraint \set ON_ERROR_STOP 0 INSERT INTO hyper_fk(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); \set ON_ERROR_STOP 1 INSERT INTO devices VALUES ('dev2'); INSERT INTO hyper_fk(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); --delete should fail \set ON_ERROR_STOP 0 DELETE FROM devices; \set ON_ERROR_STOP 1 ALTER TABLE hyper_fk DROP CONSTRAINT hyper_fk_device_id_fkey; --should now be able to add non-fk rows INSERT INTO hyper_fk(time, device_id,sensor_1) VALUES (1257987700000000001, 'dev3', 11); --can't add fk because of dev3 row \set ON_ERROR_STOP 0 ALTER TABLE hyper_fk ADD CONSTRAINT hyper_fk_device_id_fkey FOREIGN KEY (device_id) REFERENCES devices(device_id); \set ON_ERROR_STOP 1 --but can add a NOT VALID one ALTER TABLE hyper_fk ADD CONSTRAINT hyper_fk_device_id_fkey FOREIGN KEY (device_id) REFERENCES devices(device_id) NOT VALID; --which will fail when validated \set ON_ERROR_STOP 0 ALTER TABLE hyper_fk VALIDATE CONSTRAINT hyper_fk_device_id_fkey; \set ON_ERROR_STOP 1 ALTER TABLE hyper_fk DROP CONSTRAINT hyper_fk_device_id_fkey; DELETE FROM hyper_fk WHERE device_id = 'dev3'; ALTER TABLE hyper_fk ADD CONSTRAINT hyper_fk_device_id_fkey FOREIGN KEY (device_id) REFERENCES devices(device_id); \set ON_ERROR_STOP 0 INSERT INTO hyper_fk(time, device_id,sensor_1) VALUES (1257987700000000002, 'dev3', 11); \set ON_ERROR_STOP 1 SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_4_8_chunk'); SELECT * FROM _timescaledb_catalog.chunk_constraint; --test CASCADE drop behavior DROP TABLE devices CASCADE; SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_4_8_chunk'); SELECT * FROM _timescaledb_catalog.chunk_constraint; --the fk went away. INSERT INTO hyper_fk(time, device_id,sensor_1) VALUES (1257987700000000002, 'dev3', 11); CREATE TABLE devices( device_id TEXT NOT NULL, PRIMARY KEY (device_id) ); INSERT INTO devices VALUES ('dev2'), ('dev3'); ALTER TABLE hyper_fk ADD CONSTRAINT hyper_fk_device_id_fkey FOREIGN KEY (device_id) REFERENCES devices(device_id) DEFERRABLE INITIALLY DEFERRED; \set ON_ERROR_STOP 0 BEGIN; --error deferred until commmit INSERT INTO hyper_fk(time, device_id,sensor_1) VALUES (1257987700000000003, 'dev4', 11); SELECT 1; COMMIT; \set ON_ERROR_STOP 1 ALTER TABLE hyper_fk ALTER CONSTRAINT hyper_fk_device_id_fkey NOT DEFERRABLE; \set ON_ERROR_STOP 0 BEGIN; --error detected right away INSERT INTO hyper_fk(time, device_id,sensor_1) VALUES (1257987700000000003, 'dev4', 11); SELECT 1; COMMIT; \set ON_ERROR_STOP 1 --this tests that there are no extra chunk_constraints left on hyper_fk TRUNCATE hyper_fk; ----------------------- FOREIGN KEY INTO A HYPERTABLE ------------------ --FOREIGN KEY references into a hypertable are currently broken. --The referencing table will never find the corresponding row in the hypertable --since it will only search the parent. Thus any insert will result in an ERROR --Block such foreign keys or fix. (Hard to block on create table so punting for now) CREATE TABLE hyper_for_ref ( time BIGINT NOT NULL PRIMARY KEY, device_id TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10) ); SELECT * FROM create_hypertable('hyper_for_ref', 'time', chunk_time_interval => 10); \set ON_ERROR_STOP 0 CREATE TABLE referrer ( time BIGINT NOT NULL REFERENCES hyper_for_ref(time) ); \set ON_ERROR_STOP 1 CREATE TABLE referrer2 ( time BIGINT NOT NULL ); \set ON_ERROR_STOP 0 ALTER TABLE referrer2 ADD CONSTRAINT hyper_fk_device_id_fkey FOREIGN KEY (time) REFERENCES hyper_for_ref(time); \set ON_ERROR_STOP 1 -- github issue 8082: FK referencing hypertable with composite unique index -- fails on first insert because chunk indexes are created after FK propagation CREATE TABLE messages_ref ( time_received TIMESTAMPTZ NOT NULL, message_id BIGSERIAL, message_type SMALLINT NOT NULL ); SELECT create_hypertable('messages_ref', by_range('time_received')); CREATE UNIQUE INDEX ON messages_ref(time_received, message_id); -- Create FK referencing the hypertable BEFORE any data exists CREATE TABLE contents_ref ( content_id BIGSERIAL, time_received TIMESTAMPTZ NOT NULL, message_id BIGINT NOT NULL, content CHAR(10), FOREIGN KEY (time_received, message_id) REFERENCES messages_ref(time_received, message_id) ON DELETE CASCADE ); -- This insert creates a new chunk. Previously it would fail with -- "index for constraint not found on chunk" because FK propagation -- happened before chunk indexes were created. INSERT INTO messages_ref (time_received, message_type) VALUES ('2025-05-05 14:56:58.000 UTC', 2); INSERT INTO contents_ref (message_id, time_received, content) VALUES (CURRVAL('messages_ref_message_id_seq'), '2025-05-05 14:56:58.000 UTC', 'HEJ'); -- Insert into a second chunk INSERT INTO messages_ref (time_received, message_type) VALUES ('2025-06-05 14:57:58.000 UTC', 3); INSERT INTO contents_ref (message_id, time_received, content) VALUES (CURRVAL('messages_ref_message_id_seq'), '2025-06-05 14:57:58.000 UTC', 'HEJ2'); -- Verify data SELECT message_type FROM messages_ref ORDER BY time_received; SELECT content FROM contents_ref ORDER BY time_received; -- Verify FK enforcement \set ON_ERROR_STOP 0 INSERT INTO contents_ref (message_id, time_received, content) VALUES (9999, '2025-05-05 14:56:58.000 UTC', 'FAIL'); \set ON_ERROR_STOP 1 -- Verify cascade delete DELETE FROM messages_ref WHERE message_type = 2; SELECT content FROM contents_ref ORDER BY time_received; DROP TABLE contents_ref; DROP TABLE messages_ref; ----------------------- EXCLUSION CONSTRAINT ------------------ CREATE TABLE hyper_ex ( time BIGINT, device_id TEXT NOT NULL REFERENCES devices(device_id), sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10), canceled boolean DEFAULT false, EXCLUDE USING btree ( time WITH =, device_id WITH = ) WHERE (not canceled) ); SELECT * FROM create_hypertable('hyper_ex', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); INSERT INTO hyper_ex(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); \set ON_ERROR_STOP 0 INSERT INTO hyper_ex(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 12); \set ON_ERROR_STOP 1 ALTER TABLE hyper_ex DROP CONSTRAINT hyper_ex_time_device_id_excl; --can now add INSERT INTO hyper_ex(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 12); --cannot add because of conflicts \set ON_ERROR_STOP 0 ALTER TABLE hyper_ex ADD CONSTRAINT hyper_ex_time_device_id_excl EXCLUDE USING btree ( time WITH =, device_id WITH = ) WHERE (not canceled) ; \set ON_ERROR_STOP 1 DELETE FROM hyper_ex WHERE sensor_1 = 12; ALTER TABLE hyper_ex ADD CONSTRAINT hyper_ex_time_device_id_excl EXCLUDE USING btree ( time WITH =, device_id WITH = ) WHERE (not canceled) DEFERRABLE INITIALLY DEFERRED ; \set ON_ERROR_STOP 0 BEGIN; --error deferred til commit INSERT INTO hyper_ex(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 12); SELECT 1; COMMIT; \set ON_ERROR_STOP 1 --cannot add exclusion constraint without partition key. CREATE TABLE hyper_ex_invalid ( time BIGINT, device_id TEXT NOT NULL REFERENCES devices(device_id), sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10), canceled boolean DEFAULT false, EXCLUDE USING btree ( device_id WITH = ) WHERE (not canceled) ); \set ON_ERROR_STOP 0 SELECT * FROM create_hypertable('hyper_ex_invalid', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); \set ON_ERROR_STOP 1 --- NO INHERIT constraints (not allowed) ---- CREATE TABLE hyper_noinherit ( time BIGINT, sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 0) NO INHERIT ); SELECT * FROM test.show_constraints('hyper_noinherit'); \set ON_ERROR_STOP 0 SELECT * FROM create_hypertable('hyper_noinherit', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); \set ON_ERROR_STOP 1 CREATE TABLE hyper_noinherit_alter ( time BIGINT, sensor_1 NUMERIC NULL DEFAULT 1 ); SELECT * FROM create_hypertable('hyper_noinherit_alter', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); \set ON_ERROR_STOP 0 ALTER TABLE hyper_noinherit_alter ADD CONSTRAINT check_noinherit CHECK (sensor_1 > 0) NO INHERIT; -- CREATE TABLE WITH DEFERRED CONSTRAINTS -- CREATE TABLE hyper_unique_deferred ( time BIGINT UNIQUE DEFERRABLE INITIALLY DEFERRED, device_id TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10) ); SELECT * FROM create_hypertable('hyper_unique_deferred', 'time', chunk_time_interval => 10); INSERT INTO hyper_unique_deferred(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); \set ON_ERROR_STOP 0 BEGIN; --error here deferred until commit INSERT INTO hyper_unique_deferred(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); SELECT 1; COMMIT; \set ON_ERROR_STOP 1 --test deferred on create table CREATE TABLE hyper_pk_deferred ( time BIGINT NOT NULL PRIMARY KEY DEFERRABLE INITIALLY DEFERRED, device_id TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10) ); SELECT * FROM create_hypertable('hyper_pk_deferred', 'time', chunk_time_interval => 10); INSERT INTO hyper_pk_deferred(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); \set ON_ERROR_STOP 0 BEGIN; --error here deferred until commit INSERT INTO hyper_pk_deferred(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); SELECT 1; COMMIT; \set ON_ERROR_STOP 1 --test that deferred works on create table too CREATE TABLE hyper_fk_deferred ( time BIGINT NOT NULL PRIMARY KEY, device_id TEXT NOT NULL REFERENCES devices(device_id) DEFERRABLE INITIALLY DEFERRED, sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10) ); SELECT * FROM create_hypertable('hyper_fk_deferred', 'time', chunk_time_interval => 10); \set ON_ERROR_STOP 0 BEGIN; --error deferred until commmit INSERT INTO hyper_fk_deferred(time, device_id,sensor_1) VALUES (1257987700000000003, 'dev4', 11); SELECT 1; COMMIT; \set ON_ERROR_STOP 1 CREATE TABLE hyper_ex_deferred ( time BIGINT, device_id TEXT NOT NULL REFERENCES devices(device_id), sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10), canceled boolean DEFAULT false, EXCLUDE USING btree ( time WITH =, device_id WITH = ) WHERE (not canceled) DEFERRABLE INITIALLY DEFERRED ); SELECT * FROM create_hypertable('hyper_ex_deferred', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); INSERT INTO hyper_ex_deferred(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 12); \set ON_ERROR_STOP 0 BEGIN; --error deferred til commit INSERT INTO hyper_ex_deferred(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 12); SELECT 1; COMMIT; \set ON_ERROR_STOP 1 -- Make sure renaming schemas won't break dropping constraints \c :TEST_DBNAME :ROLE_SUPERUSER CREATE TABLE hyper_unique ( time BIGINT NOT NULL UNIQUE, device_id TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 CHECK (sensor_1 > 10) ); SELECT * FROM create_hypertable('hyper_unique', 'time', chunk_time_interval => 10, associated_schema_name => 'my_associated_schema'); INSERT INTO hyper_unique(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 11); ALTER SCHEMA my_associated_schema RENAME TO new_associated_schema; ALTER TABLE hyper_unique DROP CONSTRAINT hyper_unique_time_key; -- test for constraint validation crash, see #1183 CREATE TABLE test_validate(time timestamp NOT NULL, a TEXT, b TEXT); SELECT * FROM create_hypertable('test_validate', 'time'); INSERT INTO test_validate values(now(), 'a', 'b'); ALTER TABLE test_validate ADD COLUMN c TEXT, ADD CONSTRAINT c_not_null CHECK (c IS NOT NULL) NOT VALID; UPDATE test_validate SET c = ''; ALTER TABLE test_validate VALIDATE CONSTRAINT c_not_null; DROP TABLE test_validate; -- test for hypertables constraints both using index tablespaces and not See #2604 SET client_min_messages = ERROR; DROP TABLESPACE IF EXISTS tablespace1; SET client_min_messages = NOTICE; CREATE TABLESPACE tablespace1 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE1_PATH; CREATE TABLE fk_tbl ( id int, CONSTRAINT pkfk PRIMARY KEY (id) USING INDEX TABLESPACE tablespace1); CREATE TABLE tbl ( fk_id int, id int, time timestamp, CONSTRAINT pk PRIMARY KEY (time, id) USING INDEX TABLESPACE tablespace1 DEFERRABLE INITIALLY DEFERRED); SELECT create_hypertable('tbl', 'time'); ALTER TABLE tbl ADD CONSTRAINT fk_con FOREIGN KEY (fk_id) REFERENCES fk_tbl(id) ON UPDATE SET NULL ON DELETE SET NULL; INSERT INTO fk_tbl VALUES(1); INSERT INTO tbl VALUES ( 1, 1, now() ); DROP TABLE tbl; DROP TABLE fk_tbl; DROP TABLESPACE IF EXISTS tablespace1; ================================================ FILE: test/sql/copy.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \o /dev/null \ir include/insert_two_partitions.sql \o --old chunks COPY "two_Partitions"("timeCustom", device_id, series_0, series_1) FROM STDIN DELIMITER ','; 1257894000000000000,dev3,1.5,2 \. \copy "two_Partitions"("timeCustom", device_id, series_0, series_1) FROM STDIN DELIMITER ','; 1257894000000000000,dev3,1.5,2 \. --new chunks COPY "two_Partitions"("timeCustom", device_id, series_0, series_1) FROM STDIN DELIMITER ','; 2257894000000000000,dev3,1.5,2 \. \copy "two_Partitions"("timeCustom", device_id, series_0, series_1) FROM STDIN DELIMITER ','; 2257894000000000000,dev3,1.5,2 \. COPY (SELECT * FROM "two_Partitions" ORDER BY "timeCustom", device_id, series_0, series_1) TO STDOUT; ---test hypertable with FK CREATE TABLE "meta" ("id" serial PRIMARY KEY); CREATE TABLE "hyper" ( "meta_id" integer NOT NULL REFERENCES meta(id), "time" bigint NOT NULL, "value" double precision NOT NULL ); SELECT create_hypertable('hyper', 'time', chunk_time_interval => 100); INSERT INTO "meta" ("id") values (1); \copy hyper (time, meta_id, value) FROM STDIN DELIMITER ','; 1,1,1 \. COPY hyper (time, meta_id, value) FROM STDIN DELIMITER ','; 2,1,1 \. \set ON_ERROR_STOP 0 \copy hyper (time, meta_id, value) FROM STDIN DELIMITER ','; 1,2,1 \. COPY hyper (time, meta_id, value) FROM STDIN DELIMITER ','; 2,2,1 \. \set ON_ERROR_STOP 1 COPY (SELECT * FROM hyper ORDER BY time, meta_id) TO STDOUT; --test that copy works with a low setting for max_open_chunks_per_insert set timescaledb.max_open_chunks_per_insert = 1; CREATE TABLE "hyper2" ( "time" bigint NOT NULL, "value" double precision NOT NULL ); SELECT create_hypertable('hyper2', 'time', chunk_time_interval => 10); \copy hyper2 from data/copy_data.csv with csv header ; -- test copy with blocking trigger CREATE FUNCTION gt_10() RETURNS trigger AS $func$ BEGIN IF NEW."time" < 11 THEN RETURN NULL; END IF; RETURN NEW; END $func$ LANGUAGE plpgsql; CREATE TABLE "trigger_test" ( "time" bigint NOT NULL, "value" double precision NOT NULL ); SELECT create_hypertable('trigger_test', 'time', chunk_time_interval => 10); CREATE TRIGGER check_time BEFORE INSERT ON trigger_test FOR EACH ROW EXECUTE FUNCTION gt_10(); \copy trigger_test from data/copy_data.csv with csv header ; SELECT * FROM trigger_test ORDER BY time; -- Test that if we copy from stdin to a hypertable and violate a null -- constraint, it does not crash and generate an appropriate error -- message. CREATE TABLE test(a INT NOT NULL, b TIMESTAMPTZ); SELECT create_hypertable('test', 'b'); \set ON_ERROR_STOP 0 COPY TEST (a,b) FROM STDIN (delimiter ',', null 'N'); N,'2020-01-01' \. \c :TEST_DBNAME :ROLE_SUPERUSER SET client_min_messages TO NOTICE; -- Do a basic test of COPY with a wrong PROGRAM COPY hyper FROM PROGRAM 'error'; \set ON_ERROR_STOP 1 ---------------------------------------------------------------- -- Testing COPY TO. ---------------------------------------------------------------- -- COPY TO using a hypertable will not copy any tuples, but should -- show a notice. COPY hyper TO STDOUT DELIMITER ','; -- COPY TO using a query should display all the tuples and not show a -- notice. COPY (SELECT * FROM hyper) TO STDOUT DELIMITER ','; ---------------------------------------------------------------- -- Testing multi-buffer optimization. ---------------------------------------------------------------- CREATE TABLE "hyper_copy" ( "time" bigint NOT NULL, "value" double precision NOT NULL ); SELECT create_hypertable('hyper_copy', 'time', chunk_time_interval => 2); -- First copy call with default client_min_messages, to get rid of the -- building index "_hyper_XXX_chunk_hyper_copy_time_idx" on table "_hyper_XXX_chunk" serially -- messages \copy hyper_copy FROM data/copy_data.csv WITH csv header; SET client_min_messages TO DEBUG1; \copy hyper_copy FROM data/copy_data.csv WITH csv header; SELECT count(*) FROM hyper_copy; -- Limit number of open chunks SET timescaledb.max_open_chunks_per_insert = 1; \copy hyper_copy FROM data/copy_data.csv WITH csv header; SELECT count(*) FROM hyper_copy; -- Before trigger disable the multi-buffer optimization CREATE OR REPLACE FUNCTION empty_test_trigger() RETURNS TRIGGER LANGUAGE PLPGSQL AS $BODY$ BEGIN IF TG_OP = 'DELETE' THEN RETURN OLD; END IF; RETURN NEW; END $BODY$; -- Before trigger (CIM_SINGLE should be used) CREATE TRIGGER hyper_copy_trigger_insert_before BEFORE INSERT ON hyper_copy FOR EACH ROW EXECUTE FUNCTION empty_test_trigger(); \copy hyper_copy FROM data/copy_data.csv WITH csv header; SELECT count(*) FROM hyper_copy; -- Suppress 'DEBUG: EventTriggerInvoke XXXX' messages RESET client_min_messages; DROP TRIGGER hyper_copy_trigger_insert_before ON hyper_copy; SET client_min_messages TO DEBUG1; -- After trigger (CIM_MULTI_CONDITIONAL should be used) CREATE TRIGGER hyper_copy_trigger_insert_after AFTER INSERT ON hyper_copy FOR EACH ROW EXECUTE FUNCTION empty_test_trigger(); \copy hyper_copy FROM data/copy_data.csv WITH csv header; SELECT count(*) FROM hyper_copy; -- Insert data into the chunks in random order COPY hyper_copy FROM STDIN DELIMITER ',' NULL AS 'null'; 5,1 7,1 1,1 0,5 15,3 0,7 17,2 20,1 5,6 19,1 18,2 17,1 16,1 15,1 14,1 13,1 12,1 11,1 10,1 11,1 12,2 13,2 14,2 15,2 16,2 17,2 18,2 19,2 20,2 \. SELECT count(*) FROM hyper_copy; RESET client_min_messages; RESET timescaledb.max_open_chunks_per_insert; ---------------------------------------------------------------- -- Testing multi-buffer optimization -- (no index on destination hypertable). ---------------------------------------------------------------- CREATE TABLE "hyper_copy_noindex" ( "time" bigint NOT NULL, "value" double precision NOT NULL ); SELECT create_hypertable('hyper_copy_noindex', 'time', chunk_time_interval => 10, create_default_indexes => false); -- No trigger \copy hyper_copy_noindex FROM data/copy_data.csv WITH csv header; SET client_min_messages TO DEBUG1; \copy hyper_copy_noindex FROM data/copy_data.csv WITH csv header; RESET client_min_messages; SELECT count(*) FROM hyper_copy_noindex; -- Before trigger (CIM_SINGLE should be used) CREATE TRIGGER hyper_copy_trigger_insert_before BEFORE INSERT ON hyper_copy_noindex FOR EACH ROW EXECUTE FUNCTION empty_test_trigger(); \copy hyper_copy_noindex FROM data/copy_data.csv WITH csv header; SET client_min_messages TO DEBUG1; \copy hyper_copy_noindex FROM data/copy_data.csv WITH csv header; RESET client_min_messages; SELECT count(*) FROM hyper_copy_noindex; -- After trigger (CIM_MULTI_CONDITIONAL should be used) DROP TRIGGER hyper_copy_trigger_insert_before ON hyper_copy_noindex; CREATE TRIGGER hyper_copy_trigger_insert_after AFTER INSERT ON hyper_copy_noindex FOR EACH ROW EXECUTE FUNCTION empty_test_trigger(); \copy hyper_copy_noindex FROM data/copy_data.csv WITH csv header; SET client_min_messages TO DEBUG1; \copy hyper_copy_noindex FROM data/copy_data.csv WITH csv header; RESET client_min_messages; SELECT count(*) FROM hyper_copy_noindex; ---------------------------------------------------------------- -- Testing multi-buffer optimization -- (more chunks than MAX_PARTITION_BUFFERS). ---------------------------------------------------------------- CREATE TABLE "hyper_copy_large" ( "time" timestamp NOT NULL, "value" double precision NOT NULL ); -- Genate data that will create more than 32 (MAX_PARTITION_BUFFERS) -- chunks on the 10 second chunk_time_interval partitioned hypertable. INSERT INTO hyper_copy_large SELECT time, random() AS value FROM generate_series('2022-01-01', '2022-01-31', INTERVAL '1 hour') AS g1(time) ORDER BY time; SELECT COUNT(*) FROM hyper_copy_large; -- Migrate data to chunks by using copy SELECT create_hypertable('hyper_copy_large', 'time', chunk_time_interval => INTERVAL '1 hour', migrate_data => 'true'); SELECT COUNT(*) FROM hyper_copy_large; ---------------------------------------------------------------- -- Testing multi-buffer optimization -- (triggers on chunks). ---------------------------------------------------------------- CREATE TABLE "table_with_chunk_trigger" ( "time" bigint NOT NULL, "value" double precision NOT NULL ); -- This trigger counts the already inserted tuples in -- the table table_with_chunk_trigger. CREATE OR REPLACE FUNCTION count_test_chunk_trigger() RETURNS TRIGGER LANGUAGE PLPGSQL AS $BODY$ DECLARE cnt INTEGER; BEGIN SELECT count(*) FROM table_with_chunk_trigger INTO cnt; RAISE WARNING 'Trigger counted % tuples in table table_with_chunk_trigger', cnt; IF TG_OP = 'DELETE' THEN RETURN OLD; END IF; RETURN NEW; END $BODY$; -- Create hypertable and chunks SELECT create_hypertable('table_with_chunk_trigger', 'time', chunk_time_interval => 1); -- Insert data to create all missing chunks \copy table_with_chunk_trigger from data/copy_data.csv with csv header; SELECT count(*) FROM table_with_chunk_trigger; -- Chunk 1: 1-2, Chunk 2: 2-3, Chunk 3: 3-4, Chunk 4: 4-5 SELECT chunk_schema, chunk_name FROM timescaledb_information.chunks WHERE hypertable_name = 'table_with_chunk_trigger' AND range_end_integer=5 \gset -- Create before trigger on the 4th chunk CREATE TRIGGER table_with_chunk_trigger_before_trigger BEFORE INSERT ON :chunk_schema.:chunk_name FOR EACH ROW EXECUTE FUNCTION count_test_chunk_trigger(); -- Insert data -- 25 tuples are already imported. The trigger is executed before tuples -- are copied into the 4th chunk. So, the trigger should report 25+3 = 28 -- This test requires that the multi-insert buffers of the other chunks -- are flushed before the trigger is executed. SET client_min_messages TO DEBUG1; \copy table_with_chunk_trigger FROM data/copy_data.csv WITH csv header; RESET client_min_messages; SELECT count(*) FROM table_with_chunk_trigger; DROP TRIGGER table_with_chunk_trigger_before_trigger ON :chunk_schema.:chunk_name; -- Create after trigger CREATE TRIGGER table_with_chunk_trigger_after_trigger AFTER INSERT ON :chunk_schema.:chunk_name FOR EACH ROW EXECUTE FUNCTION count_test_chunk_trigger(); -- Insert data -- 50 tuples are already imported. The trigger is executed after all -- tuples are imported. So, the trigger should report 50+25 = 75 SET client_min_messages TO DEBUG1; \copy table_with_chunk_trigger FROM data/copy_data.csv WITH csv header; RESET client_min_messages; SELECT count(*) FROM table_with_chunk_trigger; -- Hypertable with after row trigger and no index DROP TABLE table_with_chunk_trigger; CREATE TABLE "table_with_chunk_trigger" ( "time" bigint NOT NULL, "value" double precision NOT NULL ); -- Create hypertable and chunks SELECT create_hypertable('table_with_chunk_trigger', 'time', chunk_time_interval => 1, create_default_indexes => false); -- Insert data to create all missing chunks \copy table_with_chunk_trigger from data/copy_data.csv with csv header; SELECT count(*) FROM table_with_chunk_trigger; -- Chunk 1: 1-2, Chunk 2: 2-3, Chunk 3: 3-4, Chunk 4: 4-5 SELECT chunk_schema, chunk_name FROM timescaledb_information.chunks WHERE hypertable_name = 'table_with_chunk_trigger' AND range_end_integer=5 \gset -- Create after trigger CREATE TRIGGER table_with_chunk_trigger_after_trigger AFTER INSERT ON :chunk_schema.:chunk_name FOR EACH ROW EXECUTE FUNCTION count_test_chunk_trigger(); \copy table_with_chunk_trigger from data/copy_data.csv with csv header; SELECT count(*) FROM table_with_chunk_trigger; ---------------------------------------------------------------- -- Testing multi-buffer optimization -- (Hypertable without before insert trigger) ---------------------------------------------------------------- CREATE TABLE "table_without_bf_trigger" ( "time" bigint NOT NULL, "value" double precision NOT NULL ); SELECT create_hypertable('table_without_bf_trigger', 'time', chunk_time_interval => 1); \copy table_without_bf_trigger from data/copy_data.csv with csv header; SET client_min_messages TO DEBUG1; \copy table_without_bf_trigger from data/copy_data.csv with csv header; RESET client_min_messages; SELECT count(*) FROM table_without_bf_trigger; -- After trigger (CIM_MULTI_CONDITIONAL should be used) CREATE TRIGGER table_with_chunk_trigger_after_trigger AFTER INSERT ON table_without_bf_trigger FOR EACH ROW EXECUTE FUNCTION empty_test_trigger(); SET client_min_messages TO DEBUG1; \copy table_without_bf_trigger from data/copy_data.csv with csv header; RESET client_min_messages; SELECT count(*) FROM table_without_bf_trigger; ---------------------------------------------------------------- -- Testing multi-buffer optimization -- (Chunks with different layouts) ---------------------------------------------------------------- -- Time is not the first attribute of the hypertable CREATE TABLE "table_with_layout_change" ( "value1" real NOT NULL DEFAULT 1, "value2" smallint DEFAULT NULL, "value3" bigint DEFAULT NULL, "time" bigint NOT NULL, "value4" double precision NOT NULL DEFAULT 4, "value5" double precision NOT NULL DEFAULT 5 ); SELECT create_hypertable('table_with_layout_change', 'time', chunk_time_interval => 1); -- Chunk 1 (time = 1) COPY table_with_layout_change FROM STDIN DELIMITER ',' NULL AS 'null'; 100,200,300,1,400,500 \. SELECT * FROM table_with_layout_change; -- Drop the first attribute ALTER TABLE table_with_layout_change DROP COLUMN value1; SELECT * FROM table_with_layout_change; -- COPY into existing chunk (time = 1) COPY table_with_layout_change FROM STDIN DELIMITER ',' NULL AS 'null'; 201,301,1,401,501 \. -- Create new chunk (time = 2) COPY table_with_layout_change FROM STDIN DELIMITER ',' NULL AS 'null'; 202,302,2,402,502 \. SELECT * FROM table_with_layout_change ORDER BY time, value2, value3, value4, value5; -- Create new chunk (time = 2), insert in different order COPY table_with_layout_change (time, value5, value4, value3, value2) FROM STDIN DELIMITER ',' NULL AS 'null'; 2,503,403,303,203 \. COPY table_with_layout_change (value5, value4, value3, value2, time) FROM STDIN DELIMITER ',' NULL AS 'null'; 504,404,304,204,2 \. COPY table_with_layout_change (value5, value4, value3, time, value2) FROM STDIN DELIMITER ',' NULL AS 'null'; 505,405,305,2,205 \. SELECT * FROM table_with_layout_change ORDER BY time, value2, value3, value4, value5; -- Drop the last attribute and add a new one ALTER TABLE table_with_layout_change DROP COLUMN value5; ALTER TABLE table_with_layout_change ADD COLUMN value6 double precision NOT NULL default 600; SELECT * FROM table_with_layout_change ORDER BY time, value2, value3, value4, value6; -- COPY in first chunk (time = 1) COPY table_with_layout_change (time, value2, value3, value4, value6) FROM STDIN DELIMITER ',' NULL AS 'null'; 1,206,306,406,606 \. -- COPY in second chunk (time = 2) COPY table_with_layout_change (time, value2, value3, value4, value6) FROM STDIN DELIMITER ',' NULL AS 'null'; 2,207,307,407,607 \. -- COPY in new chunk (time = 3) COPY table_with_layout_change (time, value2, value3, value4, value6) FROM STDIN DELIMITER ',' NULL AS 'null'; 3,208,308,408,608 \. -- COPY in all chunks, different attribute order COPY table_with_layout_change (value3, value4, time, value6, value2) FROM STDIN DELIMITER ',' NULL AS 'null'; 309,409,3,609,209 310,410,2,610,210 311,411,1,611,211 \. SELECT * FROM table_with_layout_change ORDER BY time, value2, value3, value4, value6; -- Drop first column ALTER TABLE table_with_layout_change DROP COLUMN value2; SELECT * FROM table_with_layout_change ORDER BY time, value3, value4, value6; -- COPY in all exiting chunks and create a new one (time 4) COPY table_with_layout_change (value3, value4, time, value6) FROM STDIN DELIMITER ',' NULL AS 'null'; 312,412,3,612 313,413,2,613 314,414,4,614 315,415,1,615 \. SELECT * FROM table_with_layout_change ORDER BY time, value3, value4, value6; -- Drop the last two columns ALTER TABLE table_with_layout_change DROP COLUMN value4; ALTER TABLE table_with_layout_change DROP COLUMN value6; -- COPY in all exiting chunks and create a new one (time 5) COPY table_with_layout_change (value3, time) FROM STDIN DELIMITER ',' NULL AS 'null'; 316,2 317,1 318,3 319,5 320,4 \. SELECT * FROM table_with_layout_change ORDER BY time, value3; -- Drop the last of the initial attributes and add a new one ALTER TABLE table_with_layout_change DROP COLUMN value3; ALTER TABLE table_with_layout_change ADD COLUMN value7 double precision NOT NULL default 700; -- COPY in all exiting chunks and create a new one (time 6) COPY table_with_layout_change (value7, time) FROM STDIN DELIMITER ',' NULL AS 'null'; 721,2 722,1 723,3 724,5 725,6 726,4 \. SELECT * FROM table_with_layout_change ORDER BY time, value7; -- verify check constraints work CREATE TABLE test_check(a INT, b TIMESTAMPTZ); ALTER TABLE test_check ADD CONSTRAINT c1 CHECK (a > 7); SELECT table_name FROM create_hypertable('test_check', 'b'); COPY test_check(a,b) FROM STDIN (delimiter ',', null 'N'); 8,'2020-01-01' \. \set ON_ERROR_STOP 0 COPY test_check(a,b) FROM STDIN (delimiter ',', null 'N'); 3,'2020-01-01' \. \set ON_ERROR_STOP 1 ================================================ FILE: test/sql/copy_memory_usage.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test that transaction memory usage with COPY doesn't grow. -- We need memory usage in PortalContext after the completion of the query, so -- we'll have to log it from a trigger that runs after the query is completed. \c :TEST_DBNAME :ROLE_SUPERUSER; create table uk_price_paid(price integer, "date" date, postcode1 text, postcode2 text, type smallint, is_new bool, duration smallint, addr1 text, addr2 text, street text, locality text, town text, district text, country text, category smallint); -- Aim to about 100 partitions, the data is from 1995 to 2022. select create_hypertable('uk_price_paid', 'date', chunk_time_interval => interval '90 day'); -- This is where we log the memory usage. create table portal_memory_log(id serial, bytes bigint); -- Returns the amount of memory currently allocated in a given -- memory context. Only works for PortalContext, and doesn't work for PG 12. create or replace function ts_debug_allocated_bytes(text) returns bigint as :MODULE_PATHNAME, 'ts_debug_allocated_bytes' language c strict volatile; -- Log current memory usage into the log table. create function log_memory() returns trigger as $$ begin insert into portal_memory_log values (default, ts_debug_allocated_bytes('PortalContext')); return new; end; $$ language plpgsql; -- Prepare version dependent TopTransactionContext total memory usage query. -- Using prepared statements to avoid contaminating memory usage numbers. -- PG18 removed parent column so we have to use path to get TopTransactionContext child entries. -- https://github.com/postgres/postgres/commit/f0d11275 create or replace function prepare_transaction_total_memory_usage_stmt() returns void language plpgsql as $$ begin if current_setting('server_version_num')::int < 180000 then prepare total_stmt as select sum(total_bytes) from pg_backend_memory_contexts where parent = 'TopTransactionContext'; else prepare total_stmt as select sum(m.total_bytes) from pg_backend_memory_contexts m inner join pg_backend_memory_contexts p on (m.path[m.level-1] = p.path[p.level]) where p.name = 'TopTransactionContext'; end if; end; $$; -- Add a trigger that runs after completion of each INSERT/COPY and logs the -- current memory usage. create trigger check_update after insert on uk_price_paid for each statement execute function log_memory(); -- Memory leaks often happen on cache invalidation, so make sure they are -- invalidated often and independently (at co-prime periods). set timescaledb.max_open_chunks_per_insert = 2; set timescaledb.max_cached_chunks_per_hypertable = 3; -- Try increasingly larger data sets by concatenating the same file multiple -- times. \copy uk_price_paid from program 'bash -c "cat <(zcat < data/prices-10k-random-1.tsv.gz)"'; \copy uk_price_paid from program 'bash -c "cat <(zcat < data/prices-10k-random-1.tsv.gz) <(zcat < data/prices-10k-random-1.tsv.gz)"'; \copy uk_price_paid from program 'bash -c "cat <(zcat < data/prices-10k-random-1.tsv.gz) <(zcat < data/prices-10k-random-1.tsv.gz) <(zcat < data/prices-10k-random-1.tsv.gz)"'; \copy uk_price_paid from program 'bash -c "cat <(zcat < data/prices-10k-random-1.tsv.gz) <(zcat < data/prices-10k-random-1.tsv.gz) <(zcat < data/prices-10k-random-1.tsv.gz) <(zcat < data/prices-10k-random-1.tsv.gz)"'; \copy uk_price_paid from program 'bash -c "cat <(zcat < data/prices-10k-random-1.tsv.gz) <(zcat < data/prices-10k-random-1.tsv.gz) <(zcat < data/prices-10k-random-1.tsv.gz) <(zcat < data/prices-10k-random-1.tsv.gz) <(zcat < data/prices-10k-random-1.tsv.gz)"'; select count(*) from portal_memory_log; -- Check that the memory doesn't increase with file size by using linear regression. select * from portal_memory_log where ( select regr_slope(bytes, id - 1) / regr_intercept(bytes, id - 1)::float > 0.05 from portal_memory_log ); -- Test plpgsql leaks CREATE TABLE test_ht(tm timestamptz, val float8); SELECT * FROM create_hypertable('test_ht', 'tm'); -- Use a plpgsql function to insert into the hypertable CREATE OR REPLACE FUNCTION to_double(_in text, INOUT _out double precision) LANGUAGE plpgsql IMMUTABLE parallel safe AS $$ BEGIN SELECT CAST(_in AS double precision) INTO _out; EXCEPTION WHEN others THEN --do nothing: _out already carries default END; $$; -- TopTransactionContext usage needs to remain the same after every insert -- There was a leak earlier in the child CurTransactionContext SELECT prepare_transaction_total_memory_usage_stmt(); BEGIN; INSERT INTO test_ht VALUES ('1980-01-01 00:00:00-00', to_double('23.11', 0)); EXECUTE total_stmt; INSERT INTO test_ht VALUES ('1980-02-01 00:00:00-00', to_double('24.11', 0)); EXECUTE total_stmt; COMMIT; ================================================ FILE: test/sql/copy_where.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. ------- TEST 1: Restrictive copy from file CREATE TABLE "copy_golden" ( "time" bigint NOT NULL, "value" double precision NOT NULL ); \COPY copy_golden (time, value) FROM data/copy_data.csv WITH CSV HEADER SELECT * FROM copy_golden ORDER BY TIME; CREATE TABLE "copy_control" ( "time" bigint NOT NULL, "value" double precision NOT NULL ); \COPY copy_control (time, value) FROM data/copy_data.csv WITH CSV HEADER WHERE time > 10; SELECT * FROM copy_control ORDER BY TIME; CREATE TABLE "copy_test" ( "time" bigint NOT NULL, "value" double precision NOT NULL ); SELECT create_hypertable('copy_test', 'time', chunk_time_interval => 10); \COPY copy_test (time, value) FROM data/copy_data.csv WITH CSV HEADER WHERE time > 10; SELECT * FROM copy_test ORDER BY TIME; -- Verify attempting to use subqueries fails the same as non-hypertables \set ON_ERROR_STOP 0 \COPY copy_control (time, value) FROM data/copy_data.csv WITH CSV HEADER WHERE time IN (SELECT time FROM copy_golden); \COPY copy_test (time, value) FROM data/copy_data.csv WITH CSV HEADER WHERE time IN (SELECT time FROM copy_golden); \set ON_ERROR_STOP 1 DROP TABLE copy_golden; DROP TABLE copy_control; DROP TABLE copy_test; ================================================ FILE: test/sql/create_chunks.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- -- This test will create chunks in two dimenisions, time (x) and -- space (y), where the time dimension is aligned. The figure below -- shows the expected result. The chunk number in the figure -- indicates the creation order. -- -- + -- + -- + +-----+ +-----+ -- + | 2 | | 3 | -- + | +---+-+ | -- + +-----+ 5 |6+-----+ -- + | 1 +---+-+-----+ +---------+ -- + | | |4| 7 | | 8 | -- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+- -- 0 5 10 15 20 -- -- Partitioning: -- -- Chunk # | time | space -- 1 | 3 | 2 -- 4 | 1 | 3 -- 5 | 5 | 3 -- CREATE TABLE chunk_test(time integer, temp float8, tag integer, color integer); SELECT create_hypertable('chunk_test', 'time', 'tag', 2, chunk_time_interval => 3); INSERT INTO chunk_test VALUES (4, 24.3, 1, 1); SELECT * FROM _timescaledb_catalog.dimension_slice; INSERT INTO chunk_test VALUES (4, 24.3, 2, 1); INSERT INTO chunk_test VALUES (10, 24.3, 2, 1); SELECT c.table_name AS chunk_name, d.id AS dimension_id, ds.id AS slice_id, range_start, range_end FROM _timescaledb_catalog.chunk c LEFT JOIN _timescaledb_catalog.chunk_constraint cc ON (c.id = cc.chunk_id) LEFT JOIN _timescaledb_catalog.dimension_slice ds ON (ds.id = cc.dimension_slice_id) LEFT JOIN _timescaledb_catalog.dimension d ON (d.id = ds.dimension_id) LEFT JOIN _timescaledb_catalog.hypertable h ON (d.hypertable_id = h.id) WHERE h.schema_name = 'public' AND h.table_name = 'chunk_test' ORDER BY c.id, d.id; \c :TEST_DBNAME :ROLE_SUPERUSER SELECT set_number_partitions('chunk_test', 3); \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT set_chunk_time_interval('chunk_test', 1::bigint); INSERT INTO chunk_test VALUES (8, 24.3, 11233, 1); SELECT set_chunk_time_interval('chunk_test', 5::bigint); SELECT * FROM _timescaledb_catalog.dimension; INSERT INTO chunk_test VALUES (7, 24.3, 79669, 1); INSERT INTO chunk_test VALUES (8, 24.3, 79669, 1); INSERT INTO chunk_test VALUES (10, 24.3, 11233, 1); INSERT INTO chunk_test VALUES (16, 24.3, 11233, 1); SELECT c.table_name AS chunk_name, d.id AS dimension_id, ds.id AS slice_id, range_start, range_end FROM _timescaledb_catalog.chunk c LEFT JOIN _timescaledb_catalog.chunk_constraint cc ON (c.id = cc.chunk_id) LEFT JOIN _timescaledb_catalog.dimension_slice ds ON (ds.id = cc.dimension_slice_id) LEFT JOIN _timescaledb_catalog.dimension d ON (d.id = ds.dimension_id) LEFT JOIN _timescaledb_catalog.hypertable h ON (d.hypertable_id = h.id) WHERE h.schema_name = 'public' AND h.table_name = 'chunk_test' ORDER BY c.id, d.id; --test the edges of an open partition -- INT_64_MAX and INT_64_MIN. CREATE TABLE chunk_test_ends(time bigint, temp float8, tag integer, color integer); SELECT create_hypertable('chunk_test_ends', 'time', chunk_time_interval => 5); INSERT INTO chunk_test_ends VALUES ((-9223372036854775808)::bigint, 23.1, 11233, 1); INSERT INTO chunk_test_ends VALUES (9223372036854775807::bigint, 24.1, 11233, 1); --try to hit cache INSERT INTO chunk_test_ends VALUES (9223372036854775807::bigint, 24.2, 11233, 1); INSERT INTO chunk_test_ends VALUES (9223372036854775807::bigint, 24.3, 11233, 1), (9223372036854775807::bigint, 24.4, 11233, 1); INSERT INTO chunk_test_ends VALUES ((-9223372036854775808)::bigint, 23.2, 11233, 1); INSERT INTO chunk_test_ends VALUES ((-9223372036854775808)::bigint, 23.3, 11233, 1), ((-9223372036854775808)::bigint, 23.4, 11233, 1); SELECT * FROM chunk_test_ends ORDER BY time asc, tag, temp; --further tests of set_chunk_time_interval CREATE TABLE chunk_test2(time TIMESTAMPTZ, temp float8, tag integer, color integer); SELECT create_hypertable('chunk_test2', 'time', 'tag', 2, chunk_time_interval => 3); SELECT interval_length FROM _timescaledb_catalog.dimension d LEFT JOIN _timescaledb_catalog.hypertable h ON (d.hypertable_id = h.id) WHERE h.schema_name = 'public' AND h.table_name = 'chunk_test2' ORDER BY d.id; -- should work since time column is non-INT SELECT set_chunk_time_interval('chunk_test2', INTERVAL '1 minute'); SELECT interval_length FROM _timescaledb_catalog.dimension d LEFT JOIN _timescaledb_catalog.hypertable h ON (d.hypertable_id = h.id) WHERE h.schema_name = 'public' AND h.table_name = 'chunk_test2' ORDER BY d.id; -- should still work for non-INT time columns SELECT set_chunk_time_interval('chunk_test2', 1000000); SELECT interval_length FROM _timescaledb_catalog.dimension d LEFT JOIN _timescaledb_catalog.hypertable h ON (d.hypertable_id = h.id) WHERE h.schema_name = 'public' AND h.table_name = 'chunk_test2' ORDER BY d.id; \set ON_ERROR_STOP 0 select set_chunk_time_interval(NULL,NULL::interval); -- should fail since time column is an int SELECT set_chunk_time_interval('chunk_test', INTERVAL '1 minute'); -- should fail since its not a valid way to represent time SELECT set_chunk_time_interval('chunk_test', 'foo'::TEXT); SELECT set_chunk_time_interval('chunk_test', NULL::BIGINT); SELECT set_chunk_time_interval('chunk_test2', NULL::BIGINT); SELECT set_chunk_time_interval('chunk_test2', NULL::INTERVAL); \set ON_ERROR_STOP 1 -- Issue https://github.com/timescale/timescaledb/issues/7406 CREATE TABLE test_ht (time TIMESTAMPTZ, v1 INTEGER); SELECT create_hypertable('test_ht', by_range('time', INTERVAL '1 hour')); CREATE TABLE test_tb (time TIMESTAMPTZ, v1 INTEGER); CREATE OR REPLACE FUNCTION test_tb_trg_insert() RETURNS TRIGGER AS $$ BEGIN INSERT INTO test_ht VALUES (NEW.time, NEW.v1); RETURN NEW; END; $$ LANGUAGE plpgsql; CREATE TRIGGER test_tb_trg_insert AFTER INSERT ON test_tb FOR EACH ROW EXECUTE FUNCTION test_tb_trg_insert(); -- Creating new chunk inside a trigger called by -- a DDL statement should not fail. CREATE TABLE test_output AS WITH inserted AS ( INSERT INTO test_tb VALUES (NOW(), 1), (NOW(), 2) RETURNING * ) SELECT * FROM inserted; -- Check the DEFAULT REPLICA IDENTITY of the chunks SELECT relname, relreplident FROM show_chunks('test_ht') ch JOIN pg_class c ON (ch = c.oid) ORDER BY relname; -- Clean up TRUNCATE test_ht, test_tb; DROP TABLE test_output; -- Change the DEFAULT REPLICA IDENTITY of the chunks ALTER TABLE test_ht REPLICA IDENTITY FULL; -- Internally we force new chunks have the same REPLICA IDENTITY -- as the parent table. CREATE TABLE test_output AS WITH inserted AS ( INSERT INTO test_tb VALUES (NOW(), 1), (NOW(), 2) RETURNING * ) SELECT * FROM inserted; -- Check current new REPLICA IDENTITY FULL in the chunks SELECT relname, relreplident FROM show_chunks('test_ht') ch JOIN pg_class c ON (ch = c.oid) ORDER BY relname; -- All tables should have the same number of rows SELECT count(*) FROM test_tb; SELECT count(*) FROM test_ht; SELECT count(*) FROM test_output; -- test ALTER TABLE SET (tsdb.chunk_interval) on a hypertable CREATE TABLE t_with(time timestamptz not null, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 't_with'; ALTER TABLE t_with SET (tsdb.chunk_interval = '1 hour'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 't_with'; ================================================ FILE: test/sql/create_hypertable.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER create schema test_schema AUTHORIZATION :ROLE_DEFAULT_PERM_USER; create schema chunk_schema AUTHORIZATION :ROLE_DEFAULT_PERM_USER_2; SET ROLE :ROLE_DEFAULT_PERM_USER; create table test_schema.test_table(time BIGINT, temp float8, device_id text, device_type text, location text, id int, id2 int); \set ON_ERROR_STOP 0 -- get_create_command should fail since hypertable isn't made yet SELECT * FROM _timescaledb_functions.get_create_command('test_table'); \set ON_ERROR_STOP 1 SELECT * FROM test.relation WHERE schema = 'test_schema'; \d _timescaledb_catalog.chunk create table test_schema.test_table_no_not_null(time BIGINT, device_id text); \set ON_ERROR_STOP 0 -- Permission denied with unprivileged role SET ROLE :ROLE_DEFAULT_PERM_USER_2; select * from create_hypertable('test_schema.test_table_no_not_null', 'time', 'device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); -- CREATE on schema is not enough SET ROLE :ROLE_DEFAULT_PERM_USER; GRANT ALL ON SCHEMA test_schema TO :ROLE_DEFAULT_PERM_USER_2; SET ROLE :ROLE_DEFAULT_PERM_USER_2; select * from create_hypertable('test_schema.test_table_no_not_null', 'time', 'device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); \set ON_ERROR_STOP 1 -- Should work with when granted table owner role RESET ROLE; GRANT :ROLE_DEFAULT_PERM_USER TO :ROLE_DEFAULT_PERM_USER_2; SET ROLE :ROLE_DEFAULT_PERM_USER_2; select * from create_hypertable('test_schema.test_table_no_not_null', 'time', 'device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); \set ON_ERROR_STOP 0 insert into test_schema.test_table_no_not_null (device_id) VALUES('foo'); \set ON_ERROR_STOP 1 insert into test_schema.test_table_no_not_null (time, device_id) VALUES(1, 'foo'); RESET ROLE; SET ROLE :ROLE_DEFAULT_PERM_USER; \set ON_ERROR_STOP 0 -- No permissions on associated schema should fail select * from create_hypertable('test_schema.test_table', 'time', 'device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month'), associated_schema_name => 'chunk_schema'); \set ON_ERROR_STOP 1 -- Granting permissions on chunk_schema should make things work RESET ROLE; GRANT CREATE ON SCHEMA chunk_schema TO :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; select * from create_hypertable('test_schema.test_table', 'time', 'device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month'), associated_schema_name => 'chunk_schema'); -- Check that the insert block trigger exists SELECT * FROM test.show_triggers('test_schema.test_table'); SELECT * FROM _timescaledb_functions.get_create_command('test_table'); --test adding one more closed dimension select add_dimension('test_schema.test_table', 'location', 4); select * from _timescaledb_catalog.hypertable where table_name = 'test_table'; select * from _timescaledb_catalog.dimension; --test that we can change the number of partitions and that 1 is allowed SELECT set_number_partitions('test_schema.test_table', 1, 'location'); select * from _timescaledb_catalog.dimension WHERE column_name = 'location'; SELECT set_number_partitions('test_schema.test_table', 2, 'location'); select * from _timescaledb_catalog.dimension WHERE column_name = 'location'; \set ON_ERROR_STOP 0 --must give an explicit dimension when there are multiple space dimensions SELECT set_number_partitions('test_schema.test_table', 3); --too few SELECT set_number_partitions('test_schema.test_table', 0, 'location'); -- Too many SELECT set_number_partitions('test_schema.test_table', 32768, 'location'); -- get_create_command only works on tables w/ 1 or 2 dimensions SELECT * FROM _timescaledb_functions.get_create_command('test_table'); \set ON_ERROR_STOP 1 --test adding one more open dimension select add_dimension('test_schema.test_table', 'id', chunk_time_interval => 1000); select * from _timescaledb_catalog.hypertable where table_name = 'test_table'; select * from _timescaledb_catalog.dimension; -- Test add_dimension: can use interval types for TIMESTAMPTZ columns CREATE TABLE dim_test_time(time TIMESTAMPTZ, time2 TIMESTAMPTZ, time3 BIGINT, temp float8, device int, location int); SELECT create_hypertable('dim_test_time', 'time'); SELECT add_dimension('dim_test_time', 'time2', chunk_time_interval => INTERVAL '1 day'); -- Test add_dimension: only integral should work on BIGINT columns \set ON_ERROR_STOP 0 SELECT add_dimension('dim_test_time', 'time3', chunk_time_interval => INTERVAL '1 day'); -- string is not a valid type SELECT add_dimension('dim_test_time', 'time3', chunk_time_interval => 'foo'::TEXT); \set ON_ERROR_STOP 1 SELECT add_dimension('dim_test_time', 'time3', chunk_time_interval => 500); -- Test add_dimension: integrals should work on TIMESTAMPTZ columns CREATE TABLE dim_test_time2(time TIMESTAMPTZ, time2 TIMESTAMPTZ, temp float8, device int, location int); SELECT create_hypertable('dim_test_time2', 'time'); SELECT add_dimension('dim_test_time2', 'time2', chunk_time_interval => 500); --adding a dimension twice should not fail with 'if_not_exists' SELECT add_dimension('dim_test_time2', 'time2', chunk_time_interval => 500, if_not_exists => true); \set ON_ERROR_STOP 0 --adding on a non-hypertable CREATE TABLE not_hypertable(time TIMESTAMPTZ, temp float8, device int, location int); SELECT add_dimension('not_hypertable', 'time', chunk_time_interval => 500); --adding a non-exist column SELECT add_dimension('test_schema.test_table', 'nope', 2); --adding the same dimension twice should fail select add_dimension('test_schema.test_table', 'location', 2); --adding dimension with both number_partitions and chunk_time_interval should fail select add_dimension('test_schema.test_table', 'id2', number_partitions => 2, chunk_time_interval => 1000); \set ON_ERROR_STOP 1 -- test adding a new dimension on a non-empty table CREATE TABLE dim_test(time TIMESTAMPTZ, device int); SELECT create_hypertable('dim_test', 'time', chunk_time_interval => INTERVAL '1 day'); CREATE VIEW dim_test_slices AS SELECT c.id AS chunk_id, c.hypertable_id, ds.dimension_id, cc.dimension_slice_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN _timescaledb_catalog.dimension td ON (h.id = td.hypertable_id) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = td.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.table_name = 'dim_test' ORDER BY c.id, ds.dimension_id; INSERT INTO dim_test VALUES ('2004-10-10 00:00:00+00', 1); INSERT INTO dim_test VALUES ('2004-10-20 00:00:00+00', 2); SELECT * FROM dim_test_slices; SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_5_2_chunk'); SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_5_3_chunk'); -- add dimension to the existing chunks by adding -inf/inf dimension slices SELECT add_dimension('dim_test', 'device', 2); SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_5_2_chunk'); SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_5_3_chunk'); SELECT * FROM dim_test_slices; -- newer chunks have proper dimension slices range INSERT INTO dim_test VALUES ('2004-10-30 00:00:00+00', 3); SELECT * FROM dim_test_slices; SELECT * FROM dim_test ORDER BY time; DROP VIEW dim_test_slices; DROP TABLE dim_test; -- test add_dimension() with existing data on table with space partitioning CREATE TABLE dim_test(time TIMESTAMPTZ, device int, data int); SELECT create_hypertable('dim_test', 'time', 'device', 2, chunk_time_interval => INTERVAL '1 day'); CREATE VIEW dim_test_slices AS SELECT c.id AS chunk_id, c.hypertable_id, ds.dimension_id, cc.dimension_slice_id, c.schema_name AS chunk_schema, c.table_name AS chunk_table, ds.range_start, ds.range_end FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable h ON (c.hypertable_id = h.id) INNER JOIN _timescaledb_catalog.dimension td ON (h.id = td.hypertable_id) INNER JOIN _timescaledb_catalog.dimension_slice ds ON (ds.dimension_id = td.id) INNER JOIN _timescaledb_catalog.chunk_constraint cc ON (cc.dimension_slice_id = ds.id AND cc.chunk_id = c.id) WHERE h.table_name = 'dim_test' ORDER BY c.id, ds.dimension_id; INSERT INTO dim_test VALUES ('2004-10-10 00:00:00+00', 1, 3); INSERT INTO dim_test VALUES ('2004-10-20 00:00:00+00', 2, 2); SELECT * FROM dim_test_slices; -- new dimension slice will cover full range on existing chunks SELECT add_dimension('dim_test', 'data', 1); SELECT * FROM dim_test_slices; INSERT INTO dim_test VALUES ('2004-10-30 00:00:00+00', 3, 1); SELECT * FROM dim_test_slices; SELECT * FROM dim_test ORDER BY time; DROP VIEW dim_test_slices; DROP TABLE dim_test; -- should not fail on non-empty table with 'if_not_exists' in case the dimension exists select add_dimension('test_schema.test_table', 'location', 2, if_not_exists => true); --test partitioning in only time dimension create table test_schema.test_1dim(time timestamp, temp float); select create_hypertable('test_schema.test_1dim', 'time'); SELECT * FROM _timescaledb_functions.get_create_command('test_1dim'); SELECT * FROM test.relation WHERE schema = 'test_schema'; select create_hypertable('test_schema.test_1dim', 'time', if_not_exists => true); -- Should error when creating again without if_not_exists set to true \set ON_ERROR_STOP 0 select create_hypertable('test_schema.test_1dim', 'time'); \set ON_ERROR_STOP 1 -- if_not_exist should also work with data in the hypertable insert into test_schema.test_1dim VALUES ('2004-10-19 10:23:54+02', 1.0); select create_hypertable('test_schema.test_1dim', 'time', if_not_exists => true); -- Should error when creating again without if_not_exists set to true \set ON_ERROR_STOP 0 select create_hypertable('test_schema.test_1dim', 'time'); \set ON_ERROR_STOP 1 -- Test partitioning functions CREATE OR REPLACE FUNCTION invalid_partfunc(source integer) RETURNS INTEGER LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RETURN NULL; END $BODY$; CREATE OR REPLACE FUNCTION time_partfunc(source text) RETURNS TIMESTAMPTZ LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RETURN timezone('UTC', to_timestamp(source)); END $BODY$; CREATE TABLE test_schema.test_invalid_func(time timestamptz, temp float8, device text); \set ON_ERROR_STOP 0 -- should fail due to invalid signature SELECT create_hypertable('test_schema.test_invalid_func', 'time', 'device', 2, partitioning_func => 'invalid_partfunc'); SELECT create_hypertable('test_schema.test_invalid_func', 'time'); -- should also fail due to invalid signature SELECT add_dimension('test_schema.test_invalid_func', 'device', 2, partitioning_func => 'invalid_partfunc'); \set ON_ERROR_STOP 1 -- Test open-dimension function CREATE TABLE test_schema.open_dim_part_func(time text, temp float8, device text, event_time text); \set ON_ERROR_STOP 0 -- should fail due to invalid signature SELECT create_hypertable('test_schema.open_dim_part_func', 'time', time_partitioning_func => 'invalid_partfunc'); \set ON_ERROR_STOP 1 SELECT create_hypertable('test_schema.open_dim_part_func', 'time', time_partitioning_func => 'time_partfunc'); \set ON_ERROR_STOP 0 -- should fail due to invalid signature SELECT add_dimension('test_schema.open_dim_part_func', 'event_time', chunk_time_interval => interval '1 day', partitioning_func => 'invalid_partfunc'); \set ON_ERROR_STOP 1 SELECT add_dimension('test_schema.open_dim_part_func', 'event_time', chunk_time_interval => interval '1 day', partitioning_func => 'time_partfunc'); --test data migration create table test_schema.test_migrate(time timestamp, temp float); insert into test_schema.test_migrate VALUES ('2004-10-19 10:23:54+02', 1.0), ('2004-12-19 10:23:54+02', 2.0); select * from only test_schema.test_migrate; \set ON_ERROR_STOP 0 --should fail without migrate_data => true select create_hypertable('test_schema.test_migrate', 'time'); \set ON_ERROR_STOP 1 select create_hypertable('test_schema.test_migrate', 'time', migrate_data => true); --there should be two new chunks select * from _timescaledb_catalog.hypertable where table_name = 'test_migrate'; select id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk from _timescaledb_catalog.chunk; select * from test_schema.test_migrate; --main table should now be empty select * from only test_schema.test_migrate; select * from only _timescaledb_internal._hyper_10_9_chunk; select * from only _timescaledb_internal._hyper_10_10_chunk; create table test_schema.test_migrate_empty(time timestamp, temp float); select create_hypertable('test_schema.test_migrate_empty', 'time', migrate_data => true); CREATE TYPE test_type AS (time timestamp, temp float); CREATE TABLE test_table_of_type OF test_type; SELECT create_hypertable('test_table_of_type', 'time'); INSERT INTO test_table_of_type VALUES ('2004-10-19 10:23:54+02', 1.0), ('2004-12-19 10:23:54+02', 2.0); \set ON_ERROR_STOP 0 DROP TYPE test_type; \set ON_ERROR_STOP 1 DROP TYPE test_type CASCADE; CREATE TABLE test_table_of_type (time timestamp, temp float); SELECT create_hypertable('test_table_of_type', 'time'); INSERT INTO test_table_of_type VALUES ('2004-10-19 10:23:54+02', 1.0), ('2004-12-19 10:23:54+02', 2.0); CREATE TYPE test_type AS (time timestamp, temp float); ALTER TABLE test_table_of_type OF test_type; \set ON_ERROR_STOP 0 DROP TYPE test_type; \set ON_ERROR_STOP 1 BEGIN; DROP TYPE test_type CASCADE; ROLLBACK; ALTER TABLE test_table_of_type NOT OF; DROP TYPE test_type; -- Reset GRANTS \c :TEST_DBNAME :ROLE_SUPERUSER REVOKE :ROLE_DEFAULT_PERM_USER FROM :ROLE_DEFAULT_PERM_USER_2; -- Test custom partitioning functions CREATE OR REPLACE FUNCTION partfunc_not_immutable(source anyelement) RETURNS INTEGER LANGUAGE PLPGSQL AS $BODY$ BEGIN RETURN _timescaledb_functions.get_partition_hash(source); END $BODY$; CREATE OR REPLACE FUNCTION partfunc_bad_return_type(source anyelement) RETURNS BIGINT LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RETURN _timescaledb_functions.get_partition_hash(source); END $BODY$; CREATE OR REPLACE FUNCTION partfunc_bad_arg_type(source text) RETURNS INTEGER LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RETURN _timescaledb_functions.get_partition_hash(source); END $BODY$; CREATE OR REPLACE FUNCTION partfunc_bad_multi_arg(source anyelement, extra_arg integer) RETURNS INTEGER LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RETURN _timescaledb_functions.get_partition_hash(source); END $BODY$; CREATE OR REPLACE FUNCTION partfunc_valid(source anyelement) RETURNS INTEGER LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RETURN _timescaledb_functions.get_partition_hash(source); END $BODY$; create table test_schema.test_partfunc(time timestamptz, temp float, device int); -- Test that create_hypertable fails due to invalid partitioning function \set ON_ERROR_STOP 0 select create_hypertable('test_schema.test_partfunc', 'time', 'device', 2, partitioning_func => 'partfunc_not_immutable'); select create_hypertable('test_schema.test_partfunc', 'time', 'device', 2, partitioning_func => 'partfunc_bad_return_type'); select create_hypertable('test_schema.test_partfunc', 'time', 'device', 2, partitioning_func => 'partfunc_bad_arg_type'); select create_hypertable('test_schema.test_partfunc', 'time', 'device', 2, partitioning_func => 'partfunc_bad_multi_arg'); \set ON_ERROR_STOP 1 -- Test that add_dimension fails due to invalid partitioning function select create_hypertable('test_schema.test_partfunc', 'time'); \set ON_ERROR_STOP 0 select add_dimension('test_schema.test_partfunc', 'device', 2, partitioning_func => 'partfunc_not_immutable'); select add_dimension('test_schema.test_partfunc', 'device', 2, partitioning_func => 'partfunc_bad_return_type'); select add_dimension('test_schema.test_partfunc', 'device', 2, partitioning_func => 'partfunc_bad_arg_type'); select add_dimension('test_schema.test_partfunc', 'device', 2, partitioning_func => 'partfunc_bad_multi_arg'); \set ON_ERROR_STOP 1 -- A valid function should work: select add_dimension('test_schema.test_partfunc', 'device', 2, partitioning_func => 'partfunc_valid'); -- check get_create_command produces valid command CREATE TABLE test_schema.test_sql_cmd(time TIMESTAMPTZ, temp FLOAT8, device_id TEXT, device_type TEXT, location TEXT, id INT, id2 INT); SELECT create_hypertable('test_schema.test_sql_cmd','time'); SELECT * FROM _timescaledb_functions.get_create_command('test_sql_cmd'); SELECT _timescaledb_functions.get_create_command('test_sql_cmd') AS create_cmd; \gset DROP TABLE test_schema.test_sql_cmd CASCADE; CREATE TABLE test_schema.test_sql_cmd(time TIMESTAMPTZ, temp FLOAT8, device_id TEXT, device_type TEXT, location TEXT, id INT, id2 INT); SELECT test.execute_sql(:'create_cmd'); \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE test_table_int(time bigint, junk int); SELECT hypertable_id AS "TEST_TABLE_INT_HYPERTABLE_ID" FROM create_hypertable('test_table_int', 'time', chunk_time_interval => 1) \gset \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA IF NOT EXISTS my_schema; create or replace function my_schema.dummy_now2() returns BIGINT LANGUAGE SQL IMMUTABLE as 'SELECT 1::BIGINT'; grant execute on ALL FUNCTIONS IN SCHEMA my_schema to public; create or replace function dummy_now3() returns BIGINT LANGUAGE SQL IMMUTABLE as 'SELECT 1::BIGINT'; grant execute on ALL FUNCTIONS IN SCHEMA my_schema to public; REVOKE execute ON function dummy_now3() FROM PUBLIC; CREATE SCHEMA IF NOT EXISTS my_user_schema; GRANT ALL ON SCHEMA my_user_schema to PUBLIC; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER create or replace function dummy_now() returns BIGINT LANGUAGE SQL IMMUTABLE as 'SELECT 1::BIGINT'; create or replace function my_user_schema.dummy_now4() returns BIGINT LANGUAGE SQL IMMUTABLE as 'SELECT 1::BIGINT'; select set_integer_now_func('test_table_int', 'dummy_now'); select * from _timescaledb_catalog.dimension WHERE hypertable_id = :TEST_TABLE_INT_HYPERTABLE_ID; -- show chunks works with "created_before" and errors out with time used in "older_than" SELECT SHOW_CHUNKS('test_table_int', older_than => 10); SELECT SHOW_CHUNKS('test_table_int', created_before => now()); \set ON_ERROR_STOP 0 SELECT SHOW_CHUNKS('test_table_int', older_than => now()); select set_integer_now_func('test_table_int', 'dummy_now'); select set_integer_now_func('test_table_int', 'my_schema.dummy_now2', replace_if_exists => TRUE); select set_integer_now_func('test_table_int', 'dummy_now3', replace_if_exists => TRUE); -- test invalid oid as the integer_now_func select set_integer_now_func('test_table_int', 1, replace_if_exists => TRUE); \set ON_ERROR_STOP select set_integer_now_func('test_table_int', 'my_user_schema.dummy_now4', replace_if_exists => TRUE); \c :TEST_DBNAME :ROLE_SUPERUSER ALTER SCHEMA my_user_schema RENAME TO my_new_schema; select * from _timescaledb_catalog.dimension WHERE hypertable_id = :TEST_TABLE_INT_HYPERTABLE_ID; -- github issue #4650 CREATE TABLE sample_table ( cpu double precision null, time TIMESTAMP WITH TIME ZONE NOT NULL, sensor_id INTEGER NOT NULL, name varchar(100) default 'this is a default string value', UNIQUE(sensor_id, time) ); ALTER TABLE sample_table DROP COLUMN name; -- below creation should not report any warnings. SELECT * FROM create_hypertable('sample_table', 'time'); -- cleanup DROP TABLE sample_table CASCADE; -- github issue 4684 -- test PARTITION BY HASH CREATE TABLE regular( id INT NOT NULL, dev INT NOT NULL, value INT, CONSTRAINT cstr_regular_pky PRIMARY KEY (id) ) PARTITION BY HASH (id); DO $$ BEGIN FOR i IN 1..2 LOOP EXECUTE format(' CREATE TABLE %I PARTITION OF regular FOR VALUES WITH (MODULUS 2, REMAINDER %s)', 'regular_' || i, i - 1 ); END LOOP; END; $$; INSERT INTO regular SELECT generate_series(1,1000), 44,55; CREATE TABLE timescale ( ts TIMESTAMP WITH TIME ZONE NOT NULL, id INT NOT NULL, dev INT NOT NULL, FOREIGN KEY (id) REFERENCES regular(id) ON DELETE CASCADE ); SELECT create_hypertable( relation => 'timescale', time_column_name => 'ts' ); -- creates chunk1 INSERT INTO timescale SELECT now(), generate_series(1,200), 43; -- creates chunk2 INSERT INTO timescale SELECT now() + interval '20' day, generate_series(1,200), 43; -- creates chunk3 INSERT INTO timescale SELECT now() + interval '40' day, generate_series(1,200), 43; -- show chunks SELECT SHOW_CHUNKS('timescale'); \set ON_ERROR_STOP 0 -- record goes into chunk1 violating FK constraint as value 1001 is not present in regular table INSERT INTO timescale SELECT now(), 1001, 43; -- record goes into chunk2 violating FK constraint as value 1002 is not present in regular table INSERT INTO timescale SELECT now() + interval '20' day, 1002, 43; -- record goes into chunk3 violating FK constraint as value 1003 is not present in regular table INSERT INTO timescale SELECT now() + interval '40' day, 1003, 43; \set ON_ERROR_STOP 1 -- cleanup DROP TABLE regular cascade; DROP TABLE timescale cascade; -- test PARTITION BY RANGE CREATE TABLE regular( id INT NOT NULL, dev INT NOT NULL, value INT, CONSTRAINT cstr_regular_pky PRIMARY KEY (id) ) PARTITION BY RANGE (id); CREATE TABLE regular_1_500 PARTITION OF regular FOR VALUES FROM (1) TO (500); CREATE TABLE regular_500_1000 PARTITION OF regular FOR VALUES FROM (500) TO (801); INSERT INTO regular SELECT generate_series(1,800), 44,55; CREATE TABLE timescale ( ts TIMESTAMP WITH TIME ZONE NOT NULL, id INT NOT NULL, dev INT NOT NULL, FOREIGN KEY (id) REFERENCES regular(id) ON DELETE CASCADE ); SELECT create_hypertable( relation => 'timescale', time_column_name => 'ts' ); -- creates chunk1 INSERT INTO timescale SELECT now(), generate_series(1,200), 43; -- creates chunk2 INSERT INTO timescale SELECT now() + interval '20' day, generate_series(200,400), 43; -- creates chunk3 INSERT INTO timescale SELECT now() + interval '40' day, generate_series(400,600), 43; -- show chunks SELECT SHOW_CHUNKS('timescale'); \set ON_ERROR_STOP 0 -- FK constraint violation as value 801 is not present in regular table INSERT INTO timescale SELECT now(), 801, 43; -- FK constraint violation as value 902 is not present in regular table INSERT INTO timescale SELECT now() + interval '20' day, 902, 43; -- FK constraint violation as value 1003 is not present in regular table INSERT INTO timescale SELECT now() + interval '40' day, 1003, 43; \set ON_ERROR_STOP 1 -- cleanup DROP TABLE regular cascade; DROP TABLE timescale cascade; -- test PARTITION BY LIST CREATE TABLE regular( id INT NOT NULL, dev INT NOT NULL, value INT, CONSTRAINT cstr_regular_pky PRIMARY KEY (id) ) PARTITION BY LIST (id); CREATE TABLE regular_1_2_3_4 PARTITION OF regular FOR VALUES IN (1,2,3,4); CREATE TABLE regular_5_6_7_8 PARTITION OF regular FOR VALUES IN (5,6,7,8); INSERT INTO regular SELECT generate_series(1,8), 44,55; CREATE TABLE timescale ( ts TIMESTAMP WITH TIME ZONE NOT NULL, id INT NOT NULL, dev INT NOT NULL, FOREIGN KEY (id) REFERENCES regular(id) ON DELETE CASCADE ); SELECT create_hypertable( relation => 'timescale', time_column_name => 'ts' ); insert into timescale values (now(), 1,2); insert into timescale values (now(), 2,2); insert into timescale values (now(), 3,2); insert into timescale values (now(), 4,2); insert into timescale values (now(), 5,2); insert into timescale values (now(), 6,2); insert into timescale values (now(), 7,2); insert into timescale values (now(), 8,2); \set ON_ERROR_STOP 0 -- FK constraint violation as value 9 is not present in regular table insert into timescale values (now(), 9,2); -- FK constraint violation as value 10 is not present in regular table insert into timescale values (now(), 10,2); -- FK constraint violation as value 111 is not present in regular table insert into timescale values (now(), 111,2); \set ON_ERROR_STOP 1 -- cleanup DROP TABLE regular cascade; DROP TABLE timescale cascade; -- github issue 4872 -- If subplan of ChunkAppend is TidRangeScan, then SELECT on -- hypertable fails with error "invalid child of chunk append: Node (26)" create table tidrangescan_test ( time timestamp with time zone, some_column bigint ); select create_hypertable('tidrangescan_test', 'time'); insert into tidrangescan_test (time, some_column) values ('2023-02-12 00:00:00+02:40', 1); insert into tidrangescan_test (time, some_column) values ('2023-02-12 00:00:10+02:40', 2); insert into tidrangescan_test (time, some_column) values ('2023-02-12 00:00:20+02:40', 3); -- Below query will generate plan as -- Custom Scan (ChunkAppend) -- -> Tid Range Scan -- However when traversing ChunkAppend node, Tid Range Scan node is not -- recognised as a valid child node of ChunkAppend which causes error -- "invalid child of chunk append: Node (26)" when below query is executed select * from tidrangescan_test where time > '2023-02-12 00:00:00+02:40'::timestamp with time zone - interval '5 years' and ctid < '(1,1)'::tid ORDER BY time; drop table tidrangescan_test; \set VERBOSITY default set client_min_messages = WARNING; -- test creating a hypertable from table referenced by a foreign key fails with -- error "cannot have FOREIGN KEY constraints to hypertable". create table test_schema.fk_parent(time timestamptz, id int, unique(time, id)); create table test_schema.fk_child( time timestamptz, id int, foreign key (time, id) references test_schema.fk_parent(time, id) ); select create_hypertable ('test_schema.fk_child', 'time'); \set ON_ERROR_STOP 0 select create_hypertable ('test_schema.fk_parent', 'time'); \set ON_ERROR_STOP 1 -- create default indexes on chunks when migrating data CREATE TABLE test(time TIMESTAMPTZ, val BIGINT); CREATE INDEX test_val_idx ON test(val); INSERT INTO test VALUES('2024-01-01 00:00:00-03', 500); SELECT FROM create_hypertable('test', 'time', migrate_data=>TRUE); -- should return ALL indexes for hypertable and chunk SELECT * FROM test.show_indexes('test') ORDER BY 1; SELECT * FROM show_chunks('test') ch, LATERAL test.show_indexes(ch) ORDER BY 1, 2; DROP TABLE test; -- don't create default indexes on chunks when migrating data CREATE TABLE test(time TIMESTAMPTZ, val BIGINT); CREATE INDEX test_val_idx ON test(val); INSERT INTO test VALUES('2024-01-01 00:00:00-03', 500); SELECT FROM create_hypertable('test', 'time', create_default_indexes => FALSE, migrate_data=>TRUE); -- should NOT return default indexes for hypertable and chunk -- only user indexes should be returned SELECT * FROM test.show_indexes('test') ORDER BY 1; SELECT * FROM show_chunks('test') ch, LATERAL test.show_indexes(ch) ORDER BY 1, 2; DROP TABLE test; -- test creating a hypertable with a primary key where the partitioning column is not part of the primary key CREATE TABLE test_schema.partition_not_pk (id INT NOT NULL, device_id INT NOT NULL, time TIMESTAMPTZ NOT NULL, a TEXT NOT NULL, PRIMARY KEY (id)); \set ON_ERROR_STOP 0 select create_hypertable ('test_schema.partition_not_pk', 'time'); \set ON_ERROR_STOP 1 DROP TABLE test_schema.partition_not_pk; -- test creating a hypertable with a composite key where the partitioning column is not part of the composite key CREATE TABLE test_schema.partition_not_pk (id INT NOT NULL, device_id INT NOT NULL, time TIMESTAMPTZ NOT NULL, a TEXT NOT NULL, PRIMARY KEY (id, device_id)); \set ON_ERROR_STOP 0 select create_hypertable ('test_schema.partition_not_pk', 'time'); \set ON_ERROR_STOP 1 DROP TABLE test_schema.partition_not_pk; -- test hypertable is not created for a table that is a part of a publication explicitly SET client_min_messages = ERROR; CREATE TABLE test (timestamp TIMESTAMPTZ NOT NULL); CREATE PUBLICATION publication_test; ALTER PUBLICATION publication_test ADD TABLE test; \set ON_ERROR_STOP 0 SELECT create_hypertable('test', 'timestamp'); \set ON_ERROR_STOP 1 INSERT INTO test (timestamp) values (now()); ALTER PUBLICATION publication_test DROP TABLE test; DROP PUBLICATION publication_test; DROP TABLE test; CREATE TABLE test (timestamp TIMESTAMPTZ NOT NULL); CREATE PUBLICATION publication_test1; CREATE PUBLICATION publication_test2; ALTER PUBLICATION publication_test1 ADD TABLE test; ALTER PUBLICATION publication_test2 ADD TABLE test; \set ON_ERROR_STOP 0 SELECT create_hypertable('test', 'timestamp'); \set ON_ERROR_STOP 1 INSERT INTO test (timestamp) values (now()); ALTER PUBLICATION publication_test1 DROP TABLE test; ALTER PUBLICATION publication_test2 DROP TABLE test; DROP PUBLICATION publication_test1; DROP PUBLICATION publication_test2; DROP TABLE test; -- test hypertable is not created for a table that is a part of a publication implicitly CREATE PUBLICATION publication_test FOR ALL tables; CREATE TABLE test (timestamp TIMESTAMPTZ NOT NULL); \set ON_ERROR_STOP 0 SELECT create_hypertable('test', 'timestamp'); \set ON_ERROR_STOP 1 DROP PUBLICATION publication_test; DROP TABLE test; CREATE TABLE test (timestamp TIMESTAMPTZ NOT NULL); CREATE PUBLICATION publication_test FOR ALL tables; \set ON_ERROR_STOP 0 SELECT create_hypertable('test', 'timestamp'); \set ON_ERROR_STOP 1 DROP PUBLICATION publication_test; DROP TABLE test; RESET client_min_messages; -- Test default_chunk_time_interval GUC -- Tests that the GUC correctly sets the default chunk interval for hypertables -- with different time column types (timestamp, timestamptz, date) and integer types. -- Show initial state (should be NULL meaning legacy defaults) SHOW timescaledb.default_chunk_time_interval; -- No default_chunk_time_interval set (NULL) - uses legacy defaults CREATE TABLE test_no_guc_timestamptz(time TIMESTAMPTZ NOT NULL, val INT); SELECT create_hypertable('test_no_guc_timestamptz', 'time'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_no_guc_timestamptz'; DROP TABLE test_no_guc_timestamptz; CREATE TABLE test_no_guc_timestamp(time TIMESTAMP NOT NULL, val INT); SELECT create_hypertable('test_no_guc_timestamp', 'time'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_no_guc_timestamp'; DROP TABLE test_no_guc_timestamp; CREATE TABLE test_no_guc_date(time DATE NOT NULL, val INT); SELECT create_hypertable('test_no_guc_date', 'time'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_no_guc_date'; DROP TABLE test_no_guc_date; CREATE TABLE test_no_guc_uuid(time UUID NOT NULL, val INT); SELECT create_hypertable('test_no_guc_uuid', 'time'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_no_guc_uuid'; DROP TABLE test_no_guc_uuid; -- Set default_chunk_time_interval to '1 week' and create hypertables SET timescaledb.default_chunk_time_interval = '1 week'; SHOW timescaledb.default_chunk_time_interval; CREATE TABLE test_guc_timestamptz(time TIMESTAMPTZ NOT NULL, val INT); SELECT create_hypertable('test_guc_timestamptz', 'time'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_guc_timestamptz'; DROP TABLE test_guc_timestamptz; CREATE TABLE test_guc_timestamp(time TIMESTAMP NOT NULL, val INT); SELECT create_hypertable('test_guc_timestamp', 'time'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_guc_timestamp'; DROP TABLE test_guc_timestamp; CREATE TABLE test_guc_date(time DATE NOT NULL, val INT); SELECT create_hypertable('test_guc_date', 'time'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_guc_date'; DROP TABLE test_guc_date; CREATE TABLE test_guc_uuid(time UUID NOT NULL, val INT); SELECT create_hypertable('test_guc_uuid', 'time'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_guc_uuid'; DROP TABLE test_guc_uuid; -- Set default_chunk_time_interval to '1 day' SET timescaledb.default_chunk_time_interval = '1 day'; SHOW timescaledb.default_chunk_time_interval; CREATE TABLE test_guc_1day_timestamptz(time TIMESTAMPTZ NOT NULL, val INT); SELECT create_hypertable('test_guc_1day_timestamptz', 'time'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_guc_1day_timestamptz'; DROP TABLE test_guc_1day_timestamptz; -- Set default_chunk_time_interval to '1 month' SET timescaledb.default_chunk_time_interval = '1 month'; SHOW timescaledb.default_chunk_time_interval; CREATE TABLE test_guc_1month_timestamptz(time TIMESTAMPTZ NOT NULL, val INT); SELECT create_hypertable('test_guc_1month_timestamptz', 'time'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_guc_1month_timestamptz'; DROP TABLE test_guc_1month_timestamptz; -- Integer partition types have their own defaults and do not use the GUC SET timescaledb.default_chunk_time_interval = '1 week'; CREATE TABLE test_guc_bigint(time BIGINT NOT NULL, val INT); SELECT create_hypertable('test_guc_bigint', 'time'); SELECT integer_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_guc_bigint'; DROP TABLE test_guc_bigint; CREATE TABLE test_guc_int(time INT NOT NULL, val INT); SELECT create_hypertable('test_guc_int', 'time'); SELECT integer_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_guc_int'; DROP TABLE test_guc_int; CREATE TABLE test_guc_smallint(time SMALLINT NOT NULL, val INT); SELECT create_hypertable('test_guc_smallint', 'time'); SELECT integer_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_guc_smallint'; DROP TABLE test_guc_smallint; -- Explicit chunk_time_interval should override the GUC SET timescaledb.default_chunk_time_interval = '1 week'; CREATE TABLE test_override_timestamptz(time TIMESTAMPTZ NOT NULL, val INT); SELECT create_hypertable('test_override_timestamptz', 'time', chunk_time_interval => INTERVAL '2 days'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_override_timestamptz'; DROP TABLE test_override_timestamptz; -- Reset GUC to NULL (legacy behavior) RESET timescaledb.default_chunk_time_interval; SHOW timescaledb.default_chunk_time_interval; CREATE TABLE test_reset_timestamptz(time TIMESTAMPTZ NOT NULL, val INT); SELECT create_hypertable('test_reset_timestamptz', 'time'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_reset_timestamptz'; DROP TABLE test_reset_timestamptz; -- Invalid interval should fail \set ON_ERROR_STOP 0 SET timescaledb.default_chunk_time_interval = 'not_an_interval'; SET timescaledb.default_chunk_time_interval = '123abc'; \set ON_ERROR_STOP 1 -- add_dimension does not use the GUC, keeps legacy behavior requiring explicit interval RESET timescaledb.default_chunk_time_interval; SET timescaledb.default_chunk_time_interval = '3 days'; CREATE TABLE test_add_dim(time TIMESTAMPTZ NOT NULL, time2 TIMESTAMPTZ NOT NULL, val INT); SELECT create_hypertable('test_add_dim', 'time'); -- First dimension uses GUC SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_add_dim'; -- add_dimension without explicit interval should fail \set ON_ERROR_STOP 0 SELECT add_dimension('test_add_dim', 'time2'); \set ON_ERROR_STOP 1 -- add_dimension with explicit interval works SELECT add_dimension('test_add_dim', 'time2', chunk_time_interval => INTERVAL '1 day'); SELECT column_name, time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_add_dim' ORDER BY dimension_number; DROP TABLE test_add_dim; -- Session-level GUC changes RESET timescaledb.default_chunk_time_interval; SET timescaledb.default_chunk_time_interval = '5 days'; CREATE TABLE test_session_1(time TIMESTAMPTZ NOT NULL, val INT); SELECT create_hypertable('test_session_1', 'time'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_session_1'; -- Change GUC mid-session SET timescaledb.default_chunk_time_interval = '10 days'; CREATE TABLE test_session_2(time TIMESTAMPTZ NOT NULL, val INT); SELECT create_hypertable('test_session_2', 'time'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_session_2'; DROP TABLE test_session_1; DROP TABLE test_session_2; -- Transaction-level GUC with SET LOCAL RESET timescaledb.default_chunk_time_interval; SET timescaledb.default_chunk_time_interval = '1 week'; BEGIN; SET LOCAL timescaledb.default_chunk_time_interval = '2 weeks'; CREATE TABLE test_local_guc(time TIMESTAMPTZ NOT NULL, val INT); SELECT create_hypertable('test_local_guc', 'time'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_local_guc'; COMMIT; -- After commit, GUC should be back to session level (1 week) SHOW timescaledb.default_chunk_time_interval; DROP TABLE test_local_guc; -- UUID partition type with various intervals RESET timescaledb.default_chunk_time_interval; SET timescaledb.default_chunk_time_interval = '1 day'; CREATE TABLE test_uuid_1day(time UUID NOT NULL, val INT); SELECT create_hypertable('test_uuid_1day', 'time'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_uuid_1day'; DROP TABLE test_uuid_1day; SET timescaledb.default_chunk_time_interval = '1 hour'; CREATE TABLE test_uuid_1hour(time UUID NOT NULL, val INT); SELECT create_hypertable('test_uuid_1hour', 'time'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_uuid_1hour'; DROP TABLE test_uuid_1hour; -- UUID with explicit override should work SET timescaledb.default_chunk_time_interval = '1 week'; CREATE TABLE test_uuid_override(time UUID NOT NULL, val INT); SELECT create_hypertable('test_uuid_override', 'time', chunk_time_interval => INTERVAL '12 hours'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_uuid_override'; DROP TABLE test_uuid_override; -- UUID with no GUC set (legacy default) RESET timescaledb.default_chunk_time_interval; CREATE TABLE test_uuid_legacy(time UUID NOT NULL, val INT); SELECT create_hypertable('test_uuid_legacy', 'time'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'test_uuid_legacy'; DROP TABLE test_uuid_legacy; -- Cleanup RESET timescaledb.default_chunk_time_interval; -- Check that we produce the proper error code for nonexistent tables. select create_hypertable(77777777, 'nonexistent'); ================================================ FILE: test/sql/create_table.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test that we can verify constraints on regular tables CREATE TABLE test_hyper_pk(time TIMESTAMPTZ PRIMARY KEY, temp FLOAT, device INT); CREATE TABLE test_pk(device INT PRIMARY KEY); CREATE TABLE test_like(LIKE test_pk); SELECT create_hypertable('test_hyper_pk', 'time'); \set ON_ERROR_STOP 0 -- Foreign key constraints that reference hypertables are currently unsupported CREATE TABLE test_fk(time TIMESTAMPTZ REFERENCES test_hyper_pk(time)); \set ON_ERROR_STOP 1 CREATE TABLE test_delete(time timestamp with time zone PRIMARY KEY, temp float); SELECT create_hypertable('test_delete', 'time'); INSERT INTO test_delete VALUES('2017-01-20T09:00:01', 22.5); INSERT INTO test_delete VALUES('2017-01-20T09:00:21', 21.2); INSERT INTO test_delete VALUES('2017-01-20T09:00:47', 25.1); INSERT INTO test_delete VALUES('2020-01-20T09:00:47', 25.1); INSERT INTO test_delete VALUES('2021-01-20T09:00:47', 25.1); SELECT * FROM test_delete WHERE temp = 25.1 ORDER BY time; CREATE OR replace FUNCTION test_delete_row_count() RETURNS void AS $$ DECLARE v_cnt numeric; BEGIN v_cnt := 0; DELETE FROM test_delete WHERE temp = 25.1; GET DIAGNOSTICS v_cnt = ROW_COUNT; IF v_cnt != 3 THEN RAISE EXCEPTION 'unexpected result'; END IF; END; $$ LANGUAGE plpgsql; SELECT test_delete_row_count(); ================================================ FILE: test/sql/create_table_with.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- our user needs permission to create schema for the schema tests \c :TEST_DBNAME :ROLE_SUPERUSER GRANT CREATE ON DATABASE :TEST_DBNAME TO :ROLE_DEFAULT_PERM_USER; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- create table with non-tsdb option should not be affected CREATE TABLE t1(time timestamptz, device text, value float) WITH (autovacuum_enabled); DROP TABLE t1; -- test error cases \set ON_ERROR_STOP 0 \set VERBOSITY default CREATE TABLE t2(time float, device text, value float) WITH (tsdb.hypertable); CREATE TABLE t2(time float, device text, value float) WITH (timescaledb.hypertable); CREATE TABLE t2(time timestamptz, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column=NULL); CREATE TABLE t2(time timestamptz, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column=''); CREATE TABLE t2(time timestamptz, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='foo'); CREATE TABLE t2(time timestamptz, device text, value float) WITH (tsdb.partition_column='time'); CREATE TABLE t2(time timestamptz, device text, value float) WITH (timescaledb.partition_column='time'); CREATE TABLE t2(time timestamptz , device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval='foo'); CREATE TABLE t2(time int2 NOT NULL, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval='3 months'); CREATE TABLE t2(time timestamptz, device text, value float) WITH (tsdb.create_default_indexes='time'); CREATE TABLE t2(time timestamptz, device text, value float) WITH (tsdb.create_default_indexes=2); CREATE TABLE t2(time timestamptz, device text, value float) WITH (tsdb.create_default_indexes=-1); CREATE TABLE t2(time timestamptz NOT NULL, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.columnstore=true); CREATE TABLE t2(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore,tsdb.hypertable,tsdb.partition_column='time'); -- Test error hint for invalid timescaledb options during CREATE TABLE CREATE TABLE t2(time timestamptz, device text, value float) WITH (tsdb.invalid_option = true); CREATE TABLE t2(time timestamptz, device text, value float) WITH (timescaledb.nonexistent_param = false); \set ON_ERROR_STOP 1 \set VERBOSITY terse BEGIN; CREATE TABLE t3(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time'); CREATE TABLE t4(time timestamp, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,timescaledb.partition_column='time'); CREATE TABLE t5(time date, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time',autovacuum_enabled); CREATE TABLE t6(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,timescaledb.hypertable,tsdb.partition_column='time'); CREATE TABLE t7(time timestamptz, device text, value float) WITH (timescaledb.hypertable,tsdb.partition_column='time'); SELECT hypertable_name FROM timescaledb_information.hypertables ORDER BY 1; ROLLBACK; -- IF NOT EXISTS BEGIN; CREATE TABLE IF NOT EXISTS t7(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time'); CREATE TABLE IF NOT EXISTS t7(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time'); CREATE TABLE IF NOT EXISTS t7(time timestamptz NOT NULL, device text, value float); SELECT hypertable_name FROM timescaledb_information.hypertables ORDER BY 1; ROLLBACK; -- table won't be converted to hypertable unless it is in the initial CREATE TABLE BEGIN; CREATE TABLE IF NOT EXISTS t8(time timestamptz NOT NULL, device text, value float); CREATE TABLE IF NOT EXISTS t8(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time'); CREATE TABLE IF NOT EXISTS t8(time timestamptz NOT NULL, device text, value float); SELECT hypertable_name FROM timescaledb_information.hypertables ORDER BY 1; ROLLBACK; -- chunk_interval BEGIN; CREATE TABLE IF NOT EXISTS t9(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval='8weeks'); SELECT hypertable_name, column_name, column_type, time_interval FROM timescaledb_information.dimensions; ROLLBACK; BEGIN; CREATE TABLE IF NOT EXISTS t9(time timestamp NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval='23 days'); SELECT hypertable_name, column_name, column_type, time_interval FROM timescaledb_information.dimensions; ROLLBACK; BEGIN; CREATE TABLE IF NOT EXISTS t9(time date NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval='3 months'); SELECT hypertable_name, column_name, column_type, time_interval FROM timescaledb_information.dimensions; ROLLBACK; BEGIN; CREATE TABLE IF NOT EXISTS t9(time int2 NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval=12); SELECT hypertable_name, column_name, column_type, integer_interval FROM timescaledb_information.dimensions; ROLLBACK; BEGIN; CREATE TABLE IF NOT EXISTS t9(time int4 NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval=3453); SELECT hypertable_name, column_name, column_type, integer_interval FROM timescaledb_information.dimensions; ROLLBACK; BEGIN; CREATE TABLE IF NOT EXISTS t9(time int8 NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval=32768); SELECT hypertable_name, column_name, column_type, integer_interval FROM timescaledb_information.dimensions; ROLLBACK; -- create_default_indexes BEGIN; CREATE TABLE t10(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time'); SELECT indexrelid::regclass from pg_index where indrelid='t10'::regclass ORDER BY indexrelid::regclass::text; ROLLBACK; BEGIN; CREATE TABLE t10(time timestamptz NOT NULL PRIMARY KEY, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time'); SELECT indexrelid::regclass from pg_index where indrelid='t10'::regclass ORDER BY indexrelid::regclass::text; ROLLBACK; BEGIN; CREATE TABLE t10(time timestamptz NOT NULL UNIQUE, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time'); SELECT indexrelid::regclass from pg_index where indrelid='t10'::regclass ORDER BY indexrelid::regclass::text; ROLLBACK; BEGIN; CREATE TABLE t10(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time',tsdb.create_default_indexes=true); SELECT indexrelid::regclass from pg_index where indrelid='t10'::regclass ORDER BY indexrelid::regclass::text; ROLLBACK; BEGIN; CREATE TABLE t10(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time',tsdb.create_default_indexes=false); SELECT indexrelid::regclass from pg_index where indrelid='t10'::regclass ORDER BY indexrelid::regclass::text; ROLLBACK; -- associated_schema BEGIN; CREATE TABLE t11(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time'); SELECT associated_schema_name FROM _timescaledb_catalog.hypertable WHERE table_name = 't11'; ROLLBACK; BEGIN; CREATE TABLE t11(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time', tsdb.associated_schema='abc'); SELECT associated_schema_name FROM _timescaledb_catalog.hypertable WHERE table_name = 't11'; INSERT INTO t11 SELECT '2025-01-01', 'd1', 0.1; SELECT relname from pg_class where relnamespace = 'abc'::regnamespace ORDER BY 1; ROLLBACK; BEGIN; CREATE SCHEMA abc2; CREATE TABLE t11(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time', tsdb.associated_schema='abc2'); SELECT associated_schema_name FROM _timescaledb_catalog.hypertable WHERE table_name = 't11'; INSERT INTO t11 SELECT '2025-01-01', 'd1', 0.1; SELECT relname from pg_class where relnamespace = 'abc2'::regnamespace ORDER BY 1; ROLLBACK; -- associated_table_prefix BEGIN; CREATE TABLE t12(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time'); SELECT associated_table_prefix FROM _timescaledb_catalog.hypertable WHERE table_name = 't12'; ROLLBACK; BEGIN; CREATE TABLE t12(time timestamptz NOT NULL, device text, value float) WITH (tsdb.columnstore=false,tsdb.hypertable,tsdb.partition_column='time', tsdb.associated_schema='abc', tsdb.associated_table_prefix='tbl_prefix'); SELECT associated_table_prefix FROM _timescaledb_catalog.hypertable WHERE table_name = 't12'; INSERT INTO t12 SELECT '2025-01-01', 'd1', 0.1; SELECT relname from pg_class where relnamespace = 'abc'::regnamespace ORDER BY 1; ROLLBACK; -- default partition column BEGIN; CREATE TABLE t13(time timestamptz, device text, value float) WITH (tsdb.hypertable); CREATE TABLE t14("TiMe" timestamptz, device text, value float) WITH (tsdb.hypertable); SELECT hypertable_name, column_name FROM timescaledb_information.dimensions WHERE hypertable_name IN ('t13','t14') ORDER BY 1; ROLLBACK; -- Test default_chunk_time_interval GUC interaction with CREATE TABLE WITH -- Tests that the GUC correctly sets the default chunk interval when using -- CREATE TABLE WITH syntax instead of create_hypertable(). -- GUC set to '1 week' should be used by CREATE TABLE WITH BEGIN; SET timescaledb.default_chunk_time_interval = '1 week'; CREATE TABLE t_guc_week(time timestamptz NOT NULL, device text, value float) WITH (tsdb.hypertable, tsdb.partition_column='time'); SELECT hypertable_name, time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 't_guc_week'; ROLLBACK; -- GUC set to '1 day' with different time types BEGIN; SET timescaledb.default_chunk_time_interval = '1 day'; CREATE TABLE t_guc_timestamptz(time timestamptz NOT NULL, device text, value float) WITH (tsdb.hypertable, tsdb.partition_column='time'); CREATE TABLE t_guc_timestamp(time timestamp NOT NULL, device text, value float) WITH (tsdb.hypertable, tsdb.partition_column='time'); CREATE TABLE t_guc_date(time date NOT NULL, device text, value float) WITH (tsdb.hypertable, tsdb.partition_column='time'); SELECT hypertable_name, time_interval FROM timescaledb_information.dimensions WHERE hypertable_name LIKE 't_guc_%' ORDER BY hypertable_name; ROLLBACK; -- Explicit tsdb.chunk_interval should override the GUC BEGIN; SET timescaledb.default_chunk_time_interval = '1 week'; CREATE TABLE t_guc_override(time timestamptz NOT NULL, device text, value float) WITH (tsdb.hypertable, tsdb.partition_column='time', tsdb.chunk_interval='2 days'); SELECT hypertable_name, time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 't_guc_override'; ROLLBACK; -- Integer partition types have their own default and do not use the GUC BEGIN; SET timescaledb.default_chunk_time_interval = '1 week'; CREATE TABLE t_guc_int(time int8 NOT NULL, device text, value float) WITH (tsdb.hypertable, tsdb.partition_column='time'); SELECT hypertable_name, integer_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 't_guc_int'; ROLLBACK; -- No GUC set (NULL) should use legacy defaults BEGIN; RESET timescaledb.default_chunk_time_interval; CREATE TABLE t_no_guc(time timestamptz NOT NULL, device text, value float) WITH (tsdb.hypertable, tsdb.partition_column='time'); SELECT hypertable_name, time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 't_no_guc'; ROLLBACK; -- GUC with UUID partition type BEGIN; SET timescaledb.default_chunk_time_interval = '12 hours'; CREATE TABLE t_guc_uuid(time uuid NOT NULL, device text, value float) WITH (tsdb.hypertable, tsdb.partition_column='time'); SELECT hypertable_name, time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 't_guc_uuid'; ROLLBACK; -- Cleanup RESET timescaledb.default_chunk_time_interval; ================================================ FILE: test/sql/cursor.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE cursor_test(time timestamptz, device_id int, temp float); SELECT create_hypertable('cursor_test','time'); INSERT INTO cursor_test SELECT '2000-01-01',1,0.5; INSERT INTO cursor_test SELECT '2001-01-01',1,0.5; INSERT INTO cursor_test SELECT '2002-01-01',1,0.5; \set ON_ERROR_STOP 0 BEGIN; DECLARE c1 SCROLL CURSOR FOR SELECT * FROM cursor_test; FETCH NEXT FROM c1; -- this will produce an error on PG < 14 because PostgreSQL checks -- for the existence of a scan node with the relation id for every relation -- used in the update plan in the plan of the cursor. UPDATE cursor_test SET temp = 0.7 WHERE CURRENT OF c1; COMMIT; -- test cursor with no chunks left after runtime exclusion BEGIN; DECLARE c1 SCROLL CURSOR FOR SELECT * FROM cursor_test WHERE time > now(); UPDATE cursor_test SET temp = 0.7 WHERE CURRENT OF c1; COMMIT; -- test cursor with no chunks left after planning exclusion BEGIN; DECLARE c1 SCROLL CURSOR FOR SELECT * FROM cursor_test WHERE time > '2010-01-01'; UPDATE cursor_test SET temp = 0.7 WHERE CURRENT OF c1; COMMIT; \set ON_ERROR_STOP 1 SET timescaledb.enable_constraint_exclusion TO off; BEGIN; DECLARE c1 SCROLL CURSOR FOR SELECT * FROM cursor_test; FETCH NEXT FROM c1; UPDATE cursor_test SET temp = 0.7 WHERE CURRENT OF c1; COMMIT; ================================================ FILE: test/sql/custom_type.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER SET client_min_messages TO WARNING; CREATE OR REPLACE FUNCTION customtype_in(cstring) RETURNS customtype AS 'timestamptz_in' LANGUAGE internal IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION customtype_out(customtype) RETURNS cstring AS 'timestamptz_out' LANGUAGE internal IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION customtype_recv(internal) RETURNS customtype AS 'timestamptz_recv' LANGUAGE internal IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION customtype_send(customtype) RETURNS bytea AS 'timestamptz_send' LANGUAGE internal IMMUTABLE STRICT; SET client_min_messages TO DEFAULT; CREATE TYPE customtype ( INPUT = customtype_in, OUTPUT = customtype_out, RECEIVE = customtype_recv, SEND = customtype_send, LIKE = TIMESTAMPTZ ); CREATE CAST (customtype AS bigint) WITHOUT FUNCTION AS ASSIGNMENT; CREATE CAST (bigint AS customtype) WITHOUT FUNCTION AS IMPLICIT; CREATE CAST (customtype AS timestamptz) WITHOUT FUNCTION AS ASSIGNMENT; CREATE CAST (timestamptz AS customtype) WITHOUT FUNCTION AS ASSIGNMENT; CREATE OR REPLACE FUNCTION customtype_lt(customtype, customtype) RETURNS bool AS 'timestamp_lt' LANGUAGE internal IMMUTABLE STRICT; CREATE OPERATOR < ( LEFTARG = customtype, RIGHTARG = customtype, PROCEDURE = customtype_lt, COMMUTATOR = >, NEGATOR = >=, RESTRICT = scalarltsel, JOIN = scalarltjoinsel ); CREATE OR REPLACE FUNCTION customtype_ge(customtype, customtype) RETURNS bool AS 'timestamp_ge' LANGUAGE internal IMMUTABLE STRICT; CREATE OPERATOR >= ( LEFTARG = customtype, RIGHTARG = customtype, PROCEDURE = customtype_ge, COMMUTATOR = <=, NEGATOR = <, RESTRICT = scalargtsel, JOIN = scalargtjoinsel ); \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE customtype_test(time_custom customtype, val int); \set ON_ERROR_STOP 0 -- Using interval type for chunk time interval should fail with custom time type SELECT create_hypertable('customtype_test', 'time_custom', chunk_time_interval => INTERVAL '1 day', create_default_indexes=>false); \set ON_ERROR_STOP 1 SELECT create_hypertable('customtype_test', 'time_custom', chunk_time_interval => 10e6::bigint, create_default_indexes=>false); INSERT INTO customtype_test VALUES ('2001-01-01 01:02:03'::customtype, 10); INSERT INTO customtype_test VALUES ('2001-01-01 01:02:03'::customtype, 10); INSERT INTO customtype_test VALUES ('2001-01-01 01:02:03'::customtype, 10); EXPLAIN (buffers off, costs off) SELECT * FROM customtype_test; INSERT INTO customtype_test VALUES ('2001-01-01 01:02:23'::customtype, 11); EXPLAIN (buffers off, costs off) SELECT * FROM customtype_test; SELECT * FROM customtype_test; ================================================ FILE: test/sql/data/alter.tsv ================================================ 2017-08-22T09:19:22 21.4 3 nr1 n2r1 2017-08-23T09:20:17 31.5 2 nr2 n2r2 ================================================ FILE: test/sql/data/copy_data.csv ================================================ generate_series,random 1,0.951734602451324 2,0.717823888640851 3,0.543408489786088 4,0.641131274402142 5,0.12689296528697 6,0.0126486560329795 7,0.213605496101081 8,0.132784110959619 9,0.381155731156468 10,0.284836102742702 11,0.795640022493899 12,0.631451691035181 13,0.0958626130595803 14,0.929304684977978 15,0.524866581428796 16,0.919249163009226 17,0.878917074296623 18,0.68551931809634 19,0.594833800103515 20,0.819584367796779 21,0.474171321373433 22,0.938535195309669 23,0.333933369256556 24,0.274582070298493 25,0.602348630782217 ================================================ FILE: test/sql/data/ds1_dev1_1.tsv ================================================ 1257894000000000000 dev1 1.5 1 2 true 1257894000000000000 dev1 1.5 2 1257894000000001000 dev1 2.5 3 1257894001000000000 dev1 3.5 4 1257897600000000000 dev1 4.5 5 false 1257894002000000000 dev1 5.5 6 true 1257894002000000000 dev1 5.5 7 false ================================================ FILE: test/sql/data/onek.data ================================================ 147 0 1 3 7 7 7 47 147 147 147 14 15 RFAAAA AAAAAA AAAAxx 931 1 1 3 1 11 1 31 131 431 931 2 3 VJAAAA BAAAAA HHHHxx 714 2 0 2 4 14 4 14 114 214 714 8 9 MBAAAA CAAAAA OOOOxx 711 3 1 3 1 11 1 11 111 211 711 2 3 JBAAAA DAAAAA VVVVxx 883 4 1 3 3 3 3 83 83 383 883 6 7 ZHAAAA EAAAAA AAAAxx 439 5 1 3 9 19 9 39 39 439 439 18 19 XQAAAA FAAAAA HHHHxx 670 6 0 2 0 10 0 70 70 170 670 0 1 UZAAAA GAAAAA OOOOxx 543 7 1 3 3 3 3 43 143 43 543 6 7 XUAAAA HAAAAA VVVVxx 425 8 1 1 5 5 5 25 25 425 425 10 11 JQAAAA IAAAAA AAAAxx 800 9 0 0 0 0 0 0 0 300 800 0 1 UEAAAA JAAAAA HHHHxx 489 10 1 1 9 9 9 89 89 489 489 18 19 VSAAAA KAAAAA OOOOxx 494 11 0 2 4 14 4 94 94 494 494 8 9 ATAAAA LAAAAA VVVVxx 880 12 0 0 0 0 0 80 80 380 880 0 1 WHAAAA MAAAAA AAAAxx 611 13 1 3 1 11 1 11 11 111 611 2 3 NXAAAA NAAAAA HHHHxx 226 14 0 2 6 6 6 26 26 226 226 12 13 SIAAAA OAAAAA OOOOxx 774 15 0 2 4 14 4 74 174 274 774 8 9 UDAAAA PAAAAA VVVVxx 298 16 0 2 8 18 8 98 98 298 298 16 17 MLAAAA QAAAAA AAAAxx 682 17 0 2 2 2 2 82 82 182 682 4 5 GAAAAA RAAAAA HHHHxx 864 18 0 0 4 4 4 64 64 364 864 8 9 GHAAAA SAAAAA OOOOxx 183 19 1 3 3 3 3 83 183 183 183 6 7 BHAAAA TAAAAA VVVVxx 885 20 1 1 5 5 5 85 85 385 885 10 11 BIAAAA UAAAAA AAAAxx 997 21 1 1 7 17 7 97 197 497 997 14 15 JMAAAA VAAAAA HHHHxx 966 22 0 2 6 6 6 66 166 466 966 12 13 ELAAAA WAAAAA OOOOxx 389 23 1 1 9 9 9 89 189 389 389 18 19 ZOAAAA XAAAAA VVVVxx 846 24 0 2 6 6 6 46 46 346 846 12 13 OGAAAA YAAAAA AAAAxx 206 25 0 2 6 6 6 6 6 206 206 12 13 YHAAAA ZAAAAA HHHHxx 239 26 1 3 9 19 9 39 39 239 239 18 19 FJAAAA ABAAAA OOOOxx 365 27 1 1 5 5 5 65 165 365 365 10 11 BOAAAA BBAAAA VVVVxx 204 28 0 0 4 4 4 4 4 204 204 8 9 WHAAAA CBAAAA AAAAxx 690 29 0 2 0 10 0 90 90 190 690 0 1 OAAAAA DBAAAA HHHHxx 69 30 1 1 9 9 9 69 69 69 69 18 19 RCAAAA EBAAAA OOOOxx 358 31 0 2 8 18 8 58 158 358 358 16 17 UNAAAA FBAAAA VVVVxx 269 32 1 1 9 9 9 69 69 269 269 18 19 JKAAAA GBAAAA AAAAxx 663 33 1 3 3 3 3 63 63 163 663 6 7 NZAAAA HBAAAA HHHHxx 608 34 0 0 8 8 8 8 8 108 608 16 17 KXAAAA IBAAAA OOOOxx 398 35 0 2 8 18 8 98 198 398 398 16 17 IPAAAA JBAAAA VVVVxx 330 36 0 2 0 10 0 30 130 330 330 0 1 SMAAAA KBAAAA AAAAxx 529 37 1 1 9 9 9 29 129 29 529 18 19 JUAAAA LBAAAA HHHHxx 555 38 1 3 5 15 5 55 155 55 555 10 11 JVAAAA MBAAAA OOOOxx 746 39 0 2 6 6 6 46 146 246 746 12 13 SCAAAA NBAAAA VVVVxx 558 40 0 2 8 18 8 58 158 58 558 16 17 MVAAAA OBAAAA AAAAxx 574 41 0 2 4 14 4 74 174 74 574 8 9 CWAAAA PBAAAA HHHHxx 343 42 1 3 3 3 3 43 143 343 343 6 7 FNAAAA QBAAAA OOOOxx 120 43 0 0 0 0 0 20 120 120 120 0 1 QEAAAA RBAAAA VVVVxx 461 44 1 1 1 1 1 61 61 461 461 2 3 TRAAAA SBAAAA AAAAxx 754 45 0 2 4 14 4 54 154 254 754 8 9 ADAAAA TBAAAA HHHHxx 772 46 0 0 2 12 2 72 172 272 772 4 5 SDAAAA UBAAAA OOOOxx 749 47 1 1 9 9 9 49 149 249 749 18 19 VCAAAA VBAAAA VVVVxx 386 48 0 2 6 6 6 86 186 386 386 12 13 WOAAAA WBAAAA AAAAxx 9 49 1 1 9 9 9 9 9 9 9 18 19 JAAAAA XBAAAA HHHHxx 771 50 1 3 1 11 1 71 171 271 771 2 3 RDAAAA YBAAAA OOOOxx 470 51 0 2 0 10 0 70 70 470 470 0 1 CSAAAA ZBAAAA VVVVxx 238 52 0 2 8 18 8 38 38 238 238 16 17 EJAAAA ACAAAA AAAAxx 86 53 0 2 6 6 6 86 86 86 86 12 13 IDAAAA BCAAAA HHHHxx 56 54 0 0 6 16 6 56 56 56 56 12 13 ECAAAA CCAAAA OOOOxx 767 55 1 3 7 7 7 67 167 267 767 14 15 NDAAAA DCAAAA VVVVxx 363 56 1 3 3 3 3 63 163 363 363 6 7 ZNAAAA ECAAAA AAAAxx 655 57 1 3 5 15 5 55 55 155 655 10 11 FZAAAA FCAAAA HHHHxx 394 58 0 2 4 14 4 94 194 394 394 8 9 EPAAAA GCAAAA OOOOxx 223 59 1 3 3 3 3 23 23 223 223 6 7 PIAAAA HCAAAA VVVVxx 946 60 0 2 6 6 6 46 146 446 946 12 13 KKAAAA ICAAAA AAAAxx 863 61 1 3 3 3 3 63 63 363 863 6 7 FHAAAA JCAAAA HHHHxx 913 62 1 1 3 13 3 13 113 413 913 6 7 DJAAAA KCAAAA OOOOxx 737 63 1 1 7 17 7 37 137 237 737 14 15 JCAAAA LCAAAA VVVVxx 65 64 1 1 5 5 5 65 65 65 65 10 11 NCAAAA MCAAAA AAAAxx 251 65 1 3 1 11 1 51 51 251 251 2 3 RJAAAA NCAAAA HHHHxx 686 66 0 2 6 6 6 86 86 186 686 12 13 KAAAAA OCAAAA OOOOxx 971 67 1 3 1 11 1 71 171 471 971 2 3 JLAAAA PCAAAA VVVVxx 775 68 1 3 5 15 5 75 175 275 775 10 11 VDAAAA QCAAAA AAAAxx 577 69 1 1 7 17 7 77 177 77 577 14 15 FWAAAA RCAAAA HHHHxx 830 70 0 2 0 10 0 30 30 330 830 0 1 YFAAAA SCAAAA OOOOxx 787 71 1 3 7 7 7 87 187 287 787 14 15 HEAAAA TCAAAA VVVVxx 898 72 0 2 8 18 8 98 98 398 898 16 17 OIAAAA UCAAAA AAAAxx 588 73 0 0 8 8 8 88 188 88 588 16 17 QWAAAA VCAAAA HHHHxx 872 74 0 0 2 12 2 72 72 372 872 4 5 OHAAAA WCAAAA OOOOxx 397 75 1 1 7 17 7 97 197 397 397 14 15 HPAAAA XCAAAA VVVVxx 51 76 1 3 1 11 1 51 51 51 51 2 3 ZBAAAA YCAAAA AAAAxx 381 77 1 1 1 1 1 81 181 381 381 2 3 ROAAAA ZCAAAA HHHHxx 632 78 0 0 2 12 2 32 32 132 632 4 5 IYAAAA ADAAAA OOOOxx 31 79 1 3 1 11 1 31 31 31 31 2 3 FBAAAA BDAAAA VVVVxx 855 80 1 3 5 15 5 55 55 355 855 10 11 XGAAAA CDAAAA AAAAxx 699 81 1 3 9 19 9 99 99 199 699 18 19 XAAAAA DDAAAA HHHHxx 562 82 0 2 2 2 2 62 162 62 562 4 5 QVAAAA EDAAAA OOOOxx 681 83 1 1 1 1 1 81 81 181 681 2 3 FAAAAA FDAAAA VVVVxx 585 84 1 1 5 5 5 85 185 85 585 10 11 NWAAAA GDAAAA AAAAxx 35 85 1 3 5 15 5 35 35 35 35 10 11 JBAAAA HDAAAA HHHHxx 962 86 0 2 2 2 2 62 162 462 962 4 5 ALAAAA IDAAAA OOOOxx 282 87 0 2 2 2 2 82 82 282 282 4 5 WKAAAA JDAAAA VVVVxx 254 88 0 2 4 14 4 54 54 254 254 8 9 UJAAAA KDAAAA AAAAxx 514 89 0 2 4 14 4 14 114 14 514 8 9 UTAAAA LDAAAA HHHHxx 406 90 0 2 6 6 6 6 6 406 406 12 13 QPAAAA MDAAAA OOOOxx 544 91 0 0 4 4 4 44 144 44 544 8 9 YUAAAA NDAAAA VVVVxx 704 92 0 0 4 4 4 4 104 204 704 8 9 CBAAAA ODAAAA AAAAxx 948 93 0 0 8 8 8 48 148 448 948 16 17 MKAAAA PDAAAA HHHHxx 412 94 0 0 2 12 2 12 12 412 412 4 5 WPAAAA QDAAAA OOOOxx 200 95 0 0 0 0 0 0 0 200 200 0 1 SHAAAA RDAAAA VVVVxx 583 96 1 3 3 3 3 83 183 83 583 6 7 LWAAAA SDAAAA AAAAxx 486 97 0 2 6 6 6 86 86 486 486 12 13 SSAAAA TDAAAA HHHHxx 666 98 0 2 6 6 6 66 66 166 666 12 13 QZAAAA UDAAAA OOOOxx 436 99 0 0 6 16 6 36 36 436 436 12 13 UQAAAA VDAAAA VVVVxx 842 100 0 2 2 2 2 42 42 342 842 4 5 KGAAAA WDAAAA AAAAxx 99 101 1 3 9 19 9 99 99 99 99 18 19 VDAAAA XDAAAA HHHHxx 656 102 0 0 6 16 6 56 56 156 656 12 13 GZAAAA YDAAAA OOOOxx 673 103 1 1 3 13 3 73 73 173 673 6 7 XZAAAA ZDAAAA VVVVxx 371 104 1 3 1 11 1 71 171 371 371 2 3 HOAAAA AEAAAA AAAAxx 869 105 1 1 9 9 9 69 69 369 869 18 19 LHAAAA BEAAAA HHHHxx 569 106 1 1 9 9 9 69 169 69 569 18 19 XVAAAA CEAAAA OOOOxx 616 107 0 0 6 16 6 16 16 116 616 12 13 SXAAAA DEAAAA VVVVxx 612 108 0 0 2 12 2 12 12 112 612 4 5 OXAAAA EEAAAA AAAAxx 505 109 1 1 5 5 5 5 105 5 505 10 11 LTAAAA FEAAAA HHHHxx 922 110 0 2 2 2 2 22 122 422 922 4 5 MJAAAA GEAAAA OOOOxx 221 111 1 1 1 1 1 21 21 221 221 2 3 NIAAAA HEAAAA VVVVxx 388 112 0 0 8 8 8 88 188 388 388 16 17 YOAAAA IEAAAA AAAAxx 567 113 1 3 7 7 7 67 167 67 567 14 15 VVAAAA JEAAAA HHHHxx 58 114 0 2 8 18 8 58 58 58 58 16 17 GCAAAA KEAAAA OOOOxx 316 115 0 0 6 16 6 16 116 316 316 12 13 EMAAAA LEAAAA VVVVxx 659 116 1 3 9 19 9 59 59 159 659 18 19 JZAAAA MEAAAA AAAAxx 501 117 1 1 1 1 1 1 101 1 501 2 3 HTAAAA NEAAAA HHHHxx 815 118 1 3 5 15 5 15 15 315 815 10 11 JFAAAA OEAAAA OOOOxx 638 119 0 2 8 18 8 38 38 138 638 16 17 OYAAAA PEAAAA VVVVxx 696 120 0 0 6 16 6 96 96 196 696 12 13 UAAAAA QEAAAA AAAAxx 734 121 0 2 4 14 4 34 134 234 734 8 9 GCAAAA REAAAA HHHHxx 237 122 1 1 7 17 7 37 37 237 237 14 15 DJAAAA SEAAAA OOOOxx 816 123 0 0 6 16 6 16 16 316 816 12 13 KFAAAA TEAAAA VVVVxx 917 124 1 1 7 17 7 17 117 417 917 14 15 HJAAAA UEAAAA AAAAxx 844 125 0 0 4 4 4 44 44 344 844 8 9 MGAAAA VEAAAA HHHHxx 657 126 1 1 7 17 7 57 57 157 657 14 15 HZAAAA WEAAAA OOOOxx 952 127 0 0 2 12 2 52 152 452 952 4 5 QKAAAA XEAAAA VVVVxx 519 128 1 3 9 19 9 19 119 19 519 18 19 ZTAAAA YEAAAA AAAAxx 792 129 0 0 2 12 2 92 192 292 792 4 5 MEAAAA ZEAAAA HHHHxx 275 130 1 3 5 15 5 75 75 275 275 10 11 PKAAAA AFAAAA OOOOxx 319 131 1 3 9 19 9 19 119 319 319 18 19 HMAAAA BFAAAA VVVVxx 487 132 1 3 7 7 7 87 87 487 487 14 15 TSAAAA CFAAAA AAAAxx 945 133 1 1 5 5 5 45 145 445 945 10 11 JKAAAA DFAAAA HHHHxx 584 134 0 0 4 4 4 84 184 84 584 8 9 MWAAAA EFAAAA OOOOxx 765 135 1 1 5 5 5 65 165 265 765 10 11 LDAAAA FFAAAA VVVVxx 814 136 0 2 4 14 4 14 14 314 814 8 9 IFAAAA GFAAAA AAAAxx 359 137 1 3 9 19 9 59 159 359 359 18 19 VNAAAA HFAAAA HHHHxx 548 138 0 0 8 8 8 48 148 48 548 16 17 CVAAAA IFAAAA OOOOxx 811 139 1 3 1 11 1 11 11 311 811 2 3 FFAAAA JFAAAA VVVVxx 531 140 1 3 1 11 1 31 131 31 531 2 3 LUAAAA KFAAAA AAAAxx 104 141 0 0 4 4 4 4 104 104 104 8 9 AEAAAA LFAAAA HHHHxx 33 142 1 1 3 13 3 33 33 33 33 6 7 HBAAAA MFAAAA OOOOxx 404 143 0 0 4 4 4 4 4 404 404 8 9 OPAAAA NFAAAA VVVVxx 995 144 1 3 5 15 5 95 195 495 995 10 11 HMAAAA OFAAAA AAAAxx 408 145 0 0 8 8 8 8 8 408 408 16 17 SPAAAA PFAAAA HHHHxx 93 146 1 1 3 13 3 93 93 93 93 6 7 PDAAAA QFAAAA OOOOxx 794 147 0 2 4 14 4 94 194 294 794 8 9 OEAAAA RFAAAA VVVVxx 833 148 1 1 3 13 3 33 33 333 833 6 7 BGAAAA SFAAAA AAAAxx 615 149 1 3 5 15 5 15 15 115 615 10 11 RXAAAA TFAAAA HHHHxx 333 150 1 1 3 13 3 33 133 333 333 6 7 VMAAAA UFAAAA OOOOxx 357 151 1 1 7 17 7 57 157 357 357 14 15 TNAAAA VFAAAA VVVVxx 999 152 1 3 9 19 9 99 199 499 999 18 19 LMAAAA WFAAAA AAAAxx 515 153 1 3 5 15 5 15 115 15 515 10 11 VTAAAA XFAAAA HHHHxx 685 154 1 1 5 5 5 85 85 185 685 10 11 JAAAAA YFAAAA OOOOxx 692 155 0 0 2 12 2 92 92 192 692 4 5 QAAAAA ZFAAAA VVVVxx 627 156 1 3 7 7 7 27 27 127 627 14 15 DYAAAA AGAAAA AAAAxx 654 157 0 2 4 14 4 54 54 154 654 8 9 EZAAAA BGAAAA HHHHxx 115 158 1 3 5 15 5 15 115 115 115 10 11 LEAAAA CGAAAA OOOOxx 75 159 1 3 5 15 5 75 75 75 75 10 11 XCAAAA DGAAAA VVVVxx 14 160 0 2 4 14 4 14 14 14 14 8 9 OAAAAA EGAAAA AAAAxx 148 161 0 0 8 8 8 48 148 148 148 16 17 SFAAAA FGAAAA HHHHxx 201 162 1 1 1 1 1 1 1 201 201 2 3 THAAAA GGAAAA OOOOxx 862 163 0 2 2 2 2 62 62 362 862 4 5 EHAAAA HGAAAA VVVVxx 634 164 0 2 4 14 4 34 34 134 634 8 9 KYAAAA IGAAAA AAAAxx 589 165 1 1 9 9 9 89 189 89 589 18 19 RWAAAA JGAAAA HHHHxx 142 166 0 2 2 2 2 42 142 142 142 4 5 MFAAAA KGAAAA OOOOxx 545 167 1 1 5 5 5 45 145 45 545 10 11 ZUAAAA LGAAAA VVVVxx 983 168 1 3 3 3 3 83 183 483 983 6 7 VLAAAA MGAAAA AAAAxx 87 169 1 3 7 7 7 87 87 87 87 14 15 JDAAAA NGAAAA HHHHxx 335 170 1 3 5 15 5 35 135 335 335 10 11 XMAAAA OGAAAA OOOOxx 915 171 1 3 5 15 5 15 115 415 915 10 11 FJAAAA PGAAAA VVVVxx 286 172 0 2 6 6 6 86 86 286 286 12 13 ALAAAA QGAAAA AAAAxx 361 173 1 1 1 1 1 61 161 361 361 2 3 XNAAAA RGAAAA HHHHxx 97 174 1 1 7 17 7 97 97 97 97 14 15 TDAAAA SGAAAA OOOOxx 98 175 0 2 8 18 8 98 98 98 98 16 17 UDAAAA TGAAAA VVVVxx 377 176 1 1 7 17 7 77 177 377 377 14 15 NOAAAA UGAAAA AAAAxx 525 177 1 1 5 5 5 25 125 25 525 10 11 FUAAAA VGAAAA HHHHxx 448 178 0 0 8 8 8 48 48 448 448 16 17 GRAAAA WGAAAA OOOOxx 154 179 0 2 4 14 4 54 154 154 154 8 9 YFAAAA XGAAAA VVVVxx 866 180 0 2 6 6 6 66 66 366 866 12 13 IHAAAA YGAAAA AAAAxx 741 181 1 1 1 1 1 41 141 241 741 2 3 NCAAAA ZGAAAA HHHHxx 172 182 0 0 2 12 2 72 172 172 172 4 5 QGAAAA AHAAAA OOOOxx 843 183 1 3 3 3 3 43 43 343 843 6 7 LGAAAA BHAAAA VVVVxx 378 184 0 2 8 18 8 78 178 378 378 16 17 OOAAAA CHAAAA AAAAxx 804 185 0 0 4 4 4 4 4 304 804 8 9 YEAAAA DHAAAA HHHHxx 596 186 0 0 6 16 6 96 196 96 596 12 13 YWAAAA EHAAAA OOOOxx 77 187 1 1 7 17 7 77 77 77 77 14 15 ZCAAAA FHAAAA VVVVxx 572 188 0 0 2 12 2 72 172 72 572 4 5 AWAAAA GHAAAA AAAAxx 444 189 0 0 4 4 4 44 44 444 444 8 9 CRAAAA HHAAAA HHHHxx 47 190 1 3 7 7 7 47 47 47 47 14 15 VBAAAA IHAAAA OOOOxx 274 191 0 2 4 14 4 74 74 274 274 8 9 OKAAAA JHAAAA VVVVxx 40 192 0 0 0 0 0 40 40 40 40 0 1 OBAAAA KHAAAA AAAAxx 339 193 1 3 9 19 9 39 139 339 339 18 19 BNAAAA LHAAAA HHHHxx 13 194 1 1 3 13 3 13 13 13 13 6 7 NAAAAA MHAAAA OOOOxx 878 195 0 2 8 18 8 78 78 378 878 16 17 UHAAAA NHAAAA VVVVxx 53 196 1 1 3 13 3 53 53 53 53 6 7 BCAAAA OHAAAA AAAAxx 939 197 1 3 9 19 9 39 139 439 939 18 19 DKAAAA PHAAAA HHHHxx 928 198 0 0 8 8 8 28 128 428 928 16 17 SJAAAA QHAAAA OOOOxx 886 199 0 2 6 6 6 86 86 386 886 12 13 CIAAAA RHAAAA VVVVxx 267 200 1 3 7 7 7 67 67 267 267 14 15 HKAAAA SHAAAA AAAAxx 105 201 1 1 5 5 5 5 105 105 105 10 11 BEAAAA THAAAA HHHHxx 312 202 0 0 2 12 2 12 112 312 312 4 5 AMAAAA UHAAAA OOOOxx 552 203 0 0 2 12 2 52 152 52 552 4 5 GVAAAA VHAAAA VVVVxx 918 204 0 2 8 18 8 18 118 418 918 16 17 IJAAAA WHAAAA AAAAxx 114 205 0 2 4 14 4 14 114 114 114 8 9 KEAAAA XHAAAA HHHHxx 805 206 1 1 5 5 5 5 5 305 805 10 11 ZEAAAA YHAAAA OOOOxx 875 207 1 3 5 15 5 75 75 375 875 10 11 RHAAAA ZHAAAA VVVVxx 225 208 1 1 5 5 5 25 25 225 225 10 11 RIAAAA AIAAAA AAAAxx 495 209 1 3 5 15 5 95 95 495 495 10 11 BTAAAA BIAAAA HHHHxx 150 210 0 2 0 10 0 50 150 150 150 0 1 UFAAAA CIAAAA OOOOxx 759 211 1 3 9 19 9 59 159 259 759 18 19 FDAAAA DIAAAA VVVVxx 149 212 1 1 9 9 9 49 149 149 149 18 19 TFAAAA EIAAAA AAAAxx 480 213 0 0 0 0 0 80 80 480 480 0 1 MSAAAA FIAAAA HHHHxx 1 214 1 1 1 1 1 1 1 1 1 2 3 BAAAAA GIAAAA OOOOxx 557 215 1 1 7 17 7 57 157 57 557 14 15 LVAAAA HIAAAA VVVVxx 295 216 1 3 5 15 5 95 95 295 295 10 11 JLAAAA IIAAAA AAAAxx 854 217 0 2 4 14 4 54 54 354 854 8 9 WGAAAA JIAAAA HHHHxx 420 218 0 0 0 0 0 20 20 420 420 0 1 EQAAAA KIAAAA OOOOxx 414 219 0 2 4 14 4 14 14 414 414 8 9 YPAAAA LIAAAA VVVVxx 758 220 0 2 8 18 8 58 158 258 758 16 17 EDAAAA MIAAAA AAAAxx 879 221 1 3 9 19 9 79 79 379 879 18 19 VHAAAA NIAAAA HHHHxx 332 222 0 0 2 12 2 32 132 332 332 4 5 UMAAAA OIAAAA OOOOxx 78 223 0 2 8 18 8 78 78 78 78 16 17 ADAAAA PIAAAA VVVVxx 851 224 1 3 1 11 1 51 51 351 851 2 3 TGAAAA QIAAAA AAAAxx 592 225 0 0 2 12 2 92 192 92 592 4 5 UWAAAA RIAAAA HHHHxx 979 226 1 3 9 19 9 79 179 479 979 18 19 RLAAAA SIAAAA OOOOxx 989 227 1 1 9 9 9 89 189 489 989 18 19 BMAAAA TIAAAA VVVVxx 752 228 0 0 2 12 2 52 152 252 752 4 5 YCAAAA UIAAAA AAAAxx 214 229 0 2 4 14 4 14 14 214 214 8 9 GIAAAA VIAAAA HHHHxx 453 230 1 1 3 13 3 53 53 453 453 6 7 LRAAAA WIAAAA OOOOxx 540 231 0 0 0 0 0 40 140 40 540 0 1 UUAAAA XIAAAA VVVVxx 597 232 1 1 7 17 7 97 197 97 597 14 15 ZWAAAA YIAAAA AAAAxx 356 233 0 0 6 16 6 56 156 356 356 12 13 SNAAAA ZIAAAA HHHHxx 720 234 0 0 0 0 0 20 120 220 720 0 1 SBAAAA AJAAAA OOOOxx 367 235 1 3 7 7 7 67 167 367 367 14 15 DOAAAA BJAAAA VVVVxx 762 236 0 2 2 2 2 62 162 262 762 4 5 IDAAAA CJAAAA AAAAxx 986 237 0 2 6 6 6 86 186 486 986 12 13 YLAAAA DJAAAA HHHHxx 924 238 0 0 4 4 4 24 124 424 924 8 9 OJAAAA EJAAAA OOOOxx 779 239 1 3 9 19 9 79 179 279 779 18 19 ZDAAAA FJAAAA VVVVxx 684 240 0 0 4 4 4 84 84 184 684 8 9 IAAAAA GJAAAA AAAAxx 413 241 1 1 3 13 3 13 13 413 413 6 7 XPAAAA HJAAAA HHHHxx 479 242 1 3 9 19 9 79 79 479 479 18 19 LSAAAA IJAAAA OOOOxx 731 243 1 3 1 11 1 31 131 231 731 2 3 DCAAAA JJAAAA VVVVxx 409 244 1 1 9 9 9 9 9 409 409 18 19 TPAAAA KJAAAA AAAAxx 372 245 0 0 2 12 2 72 172 372 372 4 5 IOAAAA LJAAAA HHHHxx 139 246 1 3 9 19 9 39 139 139 139 18 19 JFAAAA MJAAAA OOOOxx 717 247 1 1 7 17 7 17 117 217 717 14 15 PBAAAA NJAAAA VVVVxx 539 248 1 3 9 19 9 39 139 39 539 18 19 TUAAAA OJAAAA AAAAxx 318 249 0 2 8 18 8 18 118 318 318 16 17 GMAAAA PJAAAA HHHHxx 208 250 0 0 8 8 8 8 8 208 208 16 17 AIAAAA QJAAAA OOOOxx 797 251 1 1 7 17 7 97 197 297 797 14 15 REAAAA RJAAAA VVVVxx 661 252 1 1 1 1 1 61 61 161 661 2 3 LZAAAA SJAAAA AAAAxx 50 253 0 2 0 10 0 50 50 50 50 0 1 YBAAAA TJAAAA HHHHxx 102 254 0 2 2 2 2 2 102 102 102 4 5 YDAAAA UJAAAA OOOOxx 484 255 0 0 4 4 4 84 84 484 484 8 9 QSAAAA VJAAAA VVVVxx 108 256 0 0 8 8 8 8 108 108 108 16 17 EEAAAA WJAAAA AAAAxx 140 257 0 0 0 0 0 40 140 140 140 0 1 KFAAAA XJAAAA HHHHxx 996 258 0 0 6 16 6 96 196 496 996 12 13 IMAAAA YJAAAA OOOOxx 687 259 1 3 7 7 7 87 87 187 687 14 15 LAAAAA ZJAAAA VVVVxx 241 260 1 1 1 1 1 41 41 241 241 2 3 HJAAAA AKAAAA AAAAxx 923 261 1 3 3 3 3 23 123 423 923 6 7 NJAAAA BKAAAA HHHHxx 500 262 0 0 0 0 0 0 100 0 500 0 1 GTAAAA CKAAAA OOOOxx 536 263 0 0 6 16 6 36 136 36 536 12 13 QUAAAA DKAAAA VVVVxx 490 264 0 2 0 10 0 90 90 490 490 0 1 WSAAAA EKAAAA AAAAxx 773 265 1 1 3 13 3 73 173 273 773 6 7 TDAAAA FKAAAA HHHHxx 19 266 1 3 9 19 9 19 19 19 19 18 19 TAAAAA GKAAAA OOOOxx 534 267 0 2 4 14 4 34 134 34 534 8 9 OUAAAA HKAAAA VVVVxx 941 268 1 1 1 1 1 41 141 441 941 2 3 FKAAAA IKAAAA AAAAxx 477 269 1 1 7 17 7 77 77 477 477 14 15 JSAAAA JKAAAA HHHHxx 173 270 1 1 3 13 3 73 173 173 173 6 7 RGAAAA KKAAAA OOOOxx 113 271 1 1 3 13 3 13 113 113 113 6 7 JEAAAA LKAAAA VVVVxx 526 272 0 2 6 6 6 26 126 26 526 12 13 GUAAAA MKAAAA AAAAxx 727 273 1 3 7 7 7 27 127 227 727 14 15 ZBAAAA NKAAAA HHHHxx 302 274 0 2 2 2 2 2 102 302 302 4 5 QLAAAA OKAAAA OOOOxx 789 275 1 1 9 9 9 89 189 289 789 18 19 JEAAAA PKAAAA VVVVxx 447 276 1 3 7 7 7 47 47 447 447 14 15 FRAAAA QKAAAA AAAAxx 884 277 0 0 4 4 4 84 84 384 884 8 9 AIAAAA RKAAAA HHHHxx 718 278 0 2 8 18 8 18 118 218 718 16 17 QBAAAA SKAAAA OOOOxx 818 279 0 2 8 18 8 18 18 318 818 16 17 MFAAAA TKAAAA VVVVxx 466 280 0 2 6 6 6 66 66 466 466 12 13 YRAAAA UKAAAA AAAAxx 131 281 1 3 1 11 1 31 131 131 131 2 3 BFAAAA VKAAAA HHHHxx 503 282 1 3 3 3 3 3 103 3 503 6 7 JTAAAA WKAAAA OOOOxx 364 283 0 0 4 4 4 64 164 364 364 8 9 AOAAAA XKAAAA VVVVxx 934 284 0 2 4 14 4 34 134 434 934 8 9 YJAAAA YKAAAA AAAAxx 542 285 0 2 2 2 2 42 142 42 542 4 5 WUAAAA ZKAAAA HHHHxx 146 286 0 2 6 6 6 46 146 146 146 12 13 QFAAAA ALAAAA OOOOxx 652 287 0 0 2 12 2 52 52 152 652 4 5 CZAAAA BLAAAA VVVVxx 566 288 0 2 6 6 6 66 166 66 566 12 13 UVAAAA CLAAAA AAAAxx 788 289 0 0 8 8 8 88 188 288 788 16 17 IEAAAA DLAAAA HHHHxx 168 290 0 0 8 8 8 68 168 168 168 16 17 MGAAAA ELAAAA OOOOxx 736 291 0 0 6 16 6 36 136 236 736 12 13 ICAAAA FLAAAA VVVVxx 795 292 1 3 5 15 5 95 195 295 795 10 11 PEAAAA GLAAAA AAAAxx 103 293 1 3 3 3 3 3 103 103 103 6 7 ZDAAAA HLAAAA HHHHxx 763 294 1 3 3 3 3 63 163 263 763 6 7 JDAAAA ILAAAA OOOOxx 256 295 0 0 6 16 6 56 56 256 256 12 13 WJAAAA JLAAAA VVVVxx 63 296 1 3 3 3 3 63 63 63 63 6 7 LCAAAA KLAAAA AAAAxx 702 297 0 2 2 2 2 2 102 202 702 4 5 ABAAAA LLAAAA HHHHxx 390 298 0 2 0 10 0 90 190 390 390 0 1 APAAAA MLAAAA OOOOxx 116 299 0 0 6 16 6 16 116 116 116 12 13 MEAAAA NLAAAA VVVVxx 354 300 0 2 4 14 4 54 154 354 354 8 9 QNAAAA OLAAAA AAAAxx 162 301 0 2 2 2 2 62 162 162 162 4 5 GGAAAA PLAAAA HHHHxx 71 302 1 3 1 11 1 71 71 71 71 2 3 TCAAAA QLAAAA OOOOxx 916 303 0 0 6 16 6 16 116 416 916 12 13 GJAAAA RLAAAA VVVVxx 565 304 1 1 5 5 5 65 165 65 565 10 11 TVAAAA SLAAAA AAAAxx 509 305 1 1 9 9 9 9 109 9 509 18 19 PTAAAA TLAAAA HHHHxx 20 306 0 0 0 0 0 20 20 20 20 0 1 UAAAAA ULAAAA OOOOxx 813 307 1 1 3 13 3 13 13 313 813 6 7 HFAAAA VLAAAA VVVVxx 80 308 0 0 0 0 0 80 80 80 80 0 1 CDAAAA WLAAAA AAAAxx 400 309 0 0 0 0 0 0 0 400 400 0 1 KPAAAA XLAAAA HHHHxx 888 310 0 0 8 8 8 88 88 388 888 16 17 EIAAAA YLAAAA OOOOxx 825 311 1 1 5 5 5 25 25 325 825 10 11 TFAAAA ZLAAAA VVVVxx 401 312 1 1 1 1 1 1 1 401 401 2 3 LPAAAA AMAAAA AAAAxx 158 313 0 2 8 18 8 58 158 158 158 16 17 CGAAAA BMAAAA HHHHxx 973 314 1 1 3 13 3 73 173 473 973 6 7 LLAAAA CMAAAA OOOOxx 324 315 0 0 4 4 4 24 124 324 324 8 9 MMAAAA DMAAAA VVVVxx 873 316 1 1 3 13 3 73 73 373 873 6 7 PHAAAA EMAAAA AAAAxx 676 317 0 0 6 16 6 76 76 176 676 12 13 AAAAAA FMAAAA HHHHxx 199 318 1 3 9 19 9 99 199 199 199 18 19 RHAAAA GMAAAA OOOOxx 304 319 0 0 4 4 4 4 104 304 304 8 9 SLAAAA HMAAAA VVVVxx 338 320 0 2 8 18 8 38 138 338 338 16 17 ANAAAA IMAAAA AAAAxx 743 321 1 3 3 3 3 43 143 243 743 6 7 PCAAAA JMAAAA HHHHxx 730 322 0 2 0 10 0 30 130 230 730 0 1 CCAAAA KMAAAA OOOOxx 130 323 0 2 0 10 0 30 130 130 130 0 1 AFAAAA LMAAAA VVVVxx 224 324 0 0 4 4 4 24 24 224 224 8 9 QIAAAA MMAAAA AAAAxx 216 325 0 0 6 16 6 16 16 216 216 12 13 IIAAAA NMAAAA HHHHxx 2 326 0 2 2 2 2 2 2 2 2 4 5 CAAAAA OMAAAA OOOOxx 836 327 0 0 6 16 6 36 36 336 836 12 13 EGAAAA PMAAAA VVVVxx 443 328 1 3 3 3 3 43 43 443 443 6 7 BRAAAA QMAAAA AAAAxx 777 329 1 1 7 17 7 77 177 277 777 14 15 XDAAAA RMAAAA HHHHxx 126 330 0 2 6 6 6 26 126 126 126 12 13 WEAAAA SMAAAA OOOOxx 117 331 1 1 7 17 7 17 117 117 117 14 15 NEAAAA TMAAAA VVVVxx 633 332 1 1 3 13 3 33 33 133 633 6 7 JYAAAA UMAAAA AAAAxx 310 333 0 2 0 10 0 10 110 310 310 0 1 YLAAAA VMAAAA HHHHxx 622 334 0 2 2 2 2 22 22 122 622 4 5 YXAAAA WMAAAA OOOOxx 268 335 0 0 8 8 8 68 68 268 268 16 17 IKAAAA XMAAAA VVVVxx 384 336 0 0 4 4 4 84 184 384 384 8 9 UOAAAA YMAAAA AAAAxx 460 337 0 0 0 0 0 60 60 460 460 0 1 SRAAAA ZMAAAA HHHHxx 475 338 1 3 5 15 5 75 75 475 475 10 11 HSAAAA ANAAAA OOOOxx 624 339 0 0 4 4 4 24 24 124 624 8 9 AYAAAA BNAAAA VVVVxx 826 340 0 2 6 6 6 26 26 326 826 12 13 UFAAAA CNAAAA AAAAxx 680 341 0 0 0 0 0 80 80 180 680 0 1 EAAAAA DNAAAA HHHHxx 306 342 0 2 6 6 6 6 106 306 306 12 13 ULAAAA ENAAAA OOOOxx 896 343 0 0 6 16 6 96 96 396 896 12 13 MIAAAA FNAAAA VVVVxx 30 344 0 2 0 10 0 30 30 30 30 0 1 EBAAAA GNAAAA AAAAxx 576 345 0 0 6 16 6 76 176 76 576 12 13 EWAAAA HNAAAA HHHHxx 551 346 1 3 1 11 1 51 151 51 551 2 3 FVAAAA INAAAA OOOOxx 639 347 1 3 9 19 9 39 39 139 639 18 19 PYAAAA JNAAAA VVVVxx 975 348 1 3 5 15 5 75 175 475 975 10 11 NLAAAA KNAAAA AAAAxx 882 349 0 2 2 2 2 82 82 382 882 4 5 YHAAAA LNAAAA HHHHxx 160 350 0 0 0 0 0 60 160 160 160 0 1 EGAAAA MNAAAA OOOOxx 522 351 0 2 2 2 2 22 122 22 522 4 5 CUAAAA NNAAAA VVVVxx 620 352 0 0 0 0 0 20 20 120 620 0 1 WXAAAA ONAAAA AAAAxx 719 353 1 3 9 19 9 19 119 219 719 18 19 RBAAAA PNAAAA HHHHxx 88 354 0 0 8 8 8 88 88 88 88 16 17 KDAAAA QNAAAA OOOOxx 614 355 0 2 4 14 4 14 14 114 614 8 9 QXAAAA RNAAAA VVVVxx 54 356 0 2 4 14 4 54 54 54 54 8 9 CCAAAA SNAAAA AAAAxx 209 357 1 1 9 9 9 9 9 209 209 18 19 BIAAAA TNAAAA HHHHxx 67 358 1 3 7 7 7 67 67 67 67 14 15 PCAAAA UNAAAA OOOOxx 809 359 1 1 9 9 9 9 9 309 809 18 19 DFAAAA VNAAAA VVVVxx 982 360 0 2 2 2 2 82 182 482 982 4 5 ULAAAA WNAAAA AAAAxx 817 361 1 1 7 17 7 17 17 317 817 14 15 LFAAAA XNAAAA HHHHxx 187 362 1 3 7 7 7 87 187 187 187 14 15 FHAAAA YNAAAA OOOOxx 992 363 0 0 2 12 2 92 192 492 992 4 5 EMAAAA ZNAAAA VVVVxx 580 364 0 0 0 0 0 80 180 80 580 0 1 IWAAAA AOAAAA AAAAxx 658 365 0 2 8 18 8 58 58 158 658 16 17 IZAAAA BOAAAA HHHHxx 222 366 0 2 2 2 2 22 22 222 222 4 5 OIAAAA COAAAA OOOOxx 667 367 1 3 7 7 7 67 67 167 667 14 15 RZAAAA DOAAAA VVVVxx 715 368 1 3 5 15 5 15 115 215 715 10 11 NBAAAA EOAAAA AAAAxx 990 369 0 2 0 10 0 90 190 490 990 0 1 CMAAAA FOAAAA HHHHxx 22 370 0 2 2 2 2 22 22 22 22 4 5 WAAAAA GOAAAA OOOOxx 362 371 0 2 2 2 2 62 162 362 362 4 5 YNAAAA HOAAAA VVVVxx 376 372 0 0 6 16 6 76 176 376 376 12 13 MOAAAA IOAAAA AAAAxx 246 373 0 2 6 6 6 46 46 246 246 12 13 MJAAAA JOAAAA HHHHxx 300 374 0 0 0 0 0 0 100 300 300 0 1 OLAAAA KOAAAA OOOOxx 231 375 1 3 1 11 1 31 31 231 231 2 3 XIAAAA LOAAAA VVVVxx 151 376 1 3 1 11 1 51 151 151 151 2 3 VFAAAA MOAAAA AAAAxx 29 377 1 1 9 9 9 29 29 29 29 18 19 DBAAAA NOAAAA HHHHxx 297 378 1 1 7 17 7 97 97 297 297 14 15 LLAAAA OOAAAA OOOOxx 403 379 1 3 3 3 3 3 3 403 403 6 7 NPAAAA POAAAA VVVVxx 716 380 0 0 6 16 6 16 116 216 716 12 13 OBAAAA QOAAAA AAAAxx 260 381 0 0 0 0 0 60 60 260 260 0 1 AKAAAA ROAAAA HHHHxx 170 382 0 2 0 10 0 70 170 170 170 0 1 OGAAAA SOAAAA OOOOxx 285 383 1 1 5 5 5 85 85 285 285 10 11 ZKAAAA TOAAAA VVVVxx 82 384 0 2 2 2 2 82 82 82 82 4 5 EDAAAA UOAAAA AAAAxx 958 385 0 2 8 18 8 58 158 458 958 16 17 WKAAAA VOAAAA HHHHxx 175 386 1 3 5 15 5 75 175 175 175 10 11 TGAAAA WOAAAA OOOOxx 671 387 1 3 1 11 1 71 71 171 671 2 3 VZAAAA XOAAAA VVVVxx 822 388 0 2 2 2 2 22 22 322 822 4 5 QFAAAA YOAAAA AAAAxx 573 389 1 1 3 13 3 73 173 73 573 6 7 BWAAAA ZOAAAA HHHHxx 723 390 1 3 3 3 3 23 123 223 723 6 7 VBAAAA APAAAA OOOOxx 195 391 1 3 5 15 5 95 195 195 195 10 11 NHAAAA BPAAAA VVVVxx 197 392 1 1 7 17 7 97 197 197 197 14 15 PHAAAA CPAAAA AAAAxx 755 393 1 3 5 15 5 55 155 255 755 10 11 BDAAAA DPAAAA HHHHxx 42 394 0 2 2 2 2 42 42 42 42 4 5 QBAAAA EPAAAA OOOOxx 897 395 1 1 7 17 7 97 97 397 897 14 15 NIAAAA FPAAAA VVVVxx 309 396 1 1 9 9 9 9 109 309 309 18 19 XLAAAA GPAAAA AAAAxx 724 397 0 0 4 4 4 24 124 224 724 8 9 WBAAAA HPAAAA HHHHxx 474 398 0 2 4 14 4 74 74 474 474 8 9 GSAAAA IPAAAA OOOOxx 345 399 1 1 5 5 5 45 145 345 345 10 11 HNAAAA JPAAAA VVVVxx 678 400 0 2 8 18 8 78 78 178 678 16 17 CAAAAA KPAAAA AAAAxx 757 401 1 1 7 17 7 57 157 257 757 14 15 DDAAAA LPAAAA HHHHxx 600 402 0 0 0 0 0 0 0 100 600 0 1 CXAAAA MPAAAA OOOOxx 184 403 0 0 4 4 4 84 184 184 184 8 9 CHAAAA NPAAAA VVVVxx 155 404 1 3 5 15 5 55 155 155 155 10 11 ZFAAAA OPAAAA AAAAxx 136 405 0 0 6 16 6 36 136 136 136 12 13 GFAAAA PPAAAA HHHHxx 889 406 1 1 9 9 9 89 89 389 889 18 19 FIAAAA QPAAAA OOOOxx 95 407 1 3 5 15 5 95 95 95 95 10 11 RDAAAA RPAAAA VVVVxx 549 408 1 1 9 9 9 49 149 49 549 18 19 DVAAAA SPAAAA AAAAxx 81 409 1 1 1 1 1 81 81 81 81 2 3 DDAAAA TPAAAA HHHHxx 679 410 1 3 9 19 9 79 79 179 679 18 19 DAAAAA UPAAAA OOOOxx 27 411 1 3 7 7 7 27 27 27 27 14 15 BBAAAA VPAAAA VVVVxx 748 412 0 0 8 8 8 48 148 248 748 16 17 UCAAAA WPAAAA AAAAxx 107 413 1 3 7 7 7 7 107 107 107 14 15 DEAAAA XPAAAA HHHHxx 870 414 0 2 0 10 0 70 70 370 870 0 1 MHAAAA YPAAAA OOOOxx 848 415 0 0 8 8 8 48 48 348 848 16 17 QGAAAA ZPAAAA VVVVxx 764 416 0 0 4 4 4 64 164 264 764 8 9 KDAAAA AQAAAA AAAAxx 535 417 1 3 5 15 5 35 135 35 535 10 11 PUAAAA BQAAAA HHHHxx 211 418 1 3 1 11 1 11 11 211 211 2 3 DIAAAA CQAAAA OOOOxx 625 419 1 1 5 5 5 25 25 125 625 10 11 BYAAAA DQAAAA VVVVxx 96 420 0 0 6 16 6 96 96 96 96 12 13 SDAAAA EQAAAA AAAAxx 828 421 0 0 8 8 8 28 28 328 828 16 17 WFAAAA FQAAAA HHHHxx 229 422 1 1 9 9 9 29 29 229 229 18 19 VIAAAA GQAAAA OOOOxx 602 423 0 2 2 2 2 2 2 102 602 4 5 EXAAAA HQAAAA VVVVxx 742 424 0 2 2 2 2 42 142 242 742 4 5 OCAAAA IQAAAA AAAAxx 451 425 1 3 1 11 1 51 51 451 451 2 3 JRAAAA JQAAAA HHHHxx 991 426 1 3 1 11 1 91 191 491 991 2 3 DMAAAA KQAAAA OOOOxx 301 427 1 1 1 1 1 1 101 301 301 2 3 PLAAAA LQAAAA VVVVxx 510 428 0 2 0 10 0 10 110 10 510 0 1 QTAAAA MQAAAA AAAAxx 299 429 1 3 9 19 9 99 99 299 299 18 19 NLAAAA NQAAAA HHHHxx 961 430 1 1 1 1 1 61 161 461 961 2 3 ZKAAAA OQAAAA OOOOxx 3 431 1 3 3 3 3 3 3 3 3 6 7 DAAAAA PQAAAA VVVVxx 106 432 0 2 6 6 6 6 106 106 106 12 13 CEAAAA QQAAAA AAAAxx 591 433 1 3 1 11 1 91 191 91 591 2 3 TWAAAA RQAAAA HHHHxx 700 434 0 0 0 0 0 0 100 200 700 0 1 YAAAAA SQAAAA OOOOxx 841 435 1 1 1 1 1 41 41 341 841 2 3 JGAAAA TQAAAA VVVVxx 829 436 1 1 9 9 9 29 29 329 829 18 19 XFAAAA UQAAAA AAAAxx 508 437 0 0 8 8 8 8 108 8 508 16 17 OTAAAA VQAAAA HHHHxx 750 438 0 2 0 10 0 50 150 250 750 0 1 WCAAAA WQAAAA OOOOxx 665 439 1 1 5 5 5 65 65 165 665 10 11 PZAAAA XQAAAA VVVVxx 157 440 1 1 7 17 7 57 157 157 157 14 15 BGAAAA YQAAAA AAAAxx 694 441 0 2 4 14 4 94 94 194 694 8 9 SAAAAA ZQAAAA HHHHxx 176 442 0 0 6 16 6 76 176 176 176 12 13 UGAAAA ARAAAA OOOOxx 950 443 0 2 0 10 0 50 150 450 950 0 1 OKAAAA BRAAAA VVVVxx 970 444 0 2 0 10 0 70 170 470 970 0 1 ILAAAA CRAAAA AAAAxx 496 445 0 0 6 16 6 96 96 496 496 12 13 CTAAAA DRAAAA HHHHxx 429 446 1 1 9 9 9 29 29 429 429 18 19 NQAAAA ERAAAA OOOOxx 907 447 1 3 7 7 7 7 107 407 907 14 15 XIAAAA FRAAAA VVVVxx 72 448 0 0 2 12 2 72 72 72 72 4 5 UCAAAA GRAAAA AAAAxx 186 449 0 2 6 6 6 86 186 186 186 12 13 EHAAAA HRAAAA HHHHxx 713 450 1 1 3 13 3 13 113 213 713 6 7 LBAAAA IRAAAA OOOOxx 432 451 0 0 2 12 2 32 32 432 432 4 5 QQAAAA JRAAAA VVVVxx 735 452 1 3 5 15 5 35 135 235 735 10 11 HCAAAA KRAAAA AAAAxx 516 453 0 0 6 16 6 16 116 16 516 12 13 WTAAAA LRAAAA HHHHxx 964 454 0 0 4 4 4 64 164 464 964 8 9 CLAAAA MRAAAA OOOOxx 840 455 0 0 0 0 0 40 40 340 840 0 1 IGAAAA NRAAAA VVVVxx 550 456 0 2 0 10 0 50 150 50 550 0 1 EVAAAA ORAAAA AAAAxx 360 457 0 0 0 0 0 60 160 360 360 0 1 WNAAAA PRAAAA HHHHxx 827 458 1 3 7 7 7 27 27 327 827 14 15 VFAAAA QRAAAA OOOOxx 959 459 1 3 9 19 9 59 159 459 959 18 19 XKAAAA RRAAAA VVVVxx 454 460 0 2 4 14 4 54 54 454 454 8 9 MRAAAA SRAAAA AAAAxx 819 461 1 3 9 19 9 19 19 319 819 18 19 NFAAAA TRAAAA HHHHxx 745 462 1 1 5 5 5 45 145 245 745 10 11 RCAAAA URAAAA OOOOxx 279 463 1 3 9 19 9 79 79 279 279 18 19 TKAAAA VRAAAA VVVVxx 426 464 0 2 6 6 6 26 26 426 426 12 13 KQAAAA WRAAAA AAAAxx 70 465 0 2 0 10 0 70 70 70 70 0 1 SCAAAA XRAAAA HHHHxx 637 466 1 1 7 17 7 37 37 137 637 14 15 NYAAAA YRAAAA OOOOxx 417 467 1 1 7 17 7 17 17 417 417 14 15 BQAAAA ZRAAAA VVVVxx 586 468 0 2 6 6 6 86 186 86 586 12 13 OWAAAA ASAAAA AAAAxx 314 469 0 2 4 14 4 14 114 314 314 8 9 CMAAAA BSAAAA HHHHxx 101 470 1 1 1 1 1 1 101 101 101 2 3 XDAAAA CSAAAA OOOOxx 205 471 1 1 5 5 5 5 5 205 205 10 11 XHAAAA DSAAAA VVVVxx 969 472 1 1 9 9 9 69 169 469 969 18 19 HLAAAA ESAAAA AAAAxx 217 473 1 1 7 17 7 17 17 217 217 14 15 JIAAAA FSAAAA HHHHxx 281 474 1 1 1 1 1 81 81 281 281 2 3 VKAAAA GSAAAA OOOOxx 984 475 0 0 4 4 4 84 184 484 984 8 9 WLAAAA HSAAAA VVVVxx 366 476 0 2 6 6 6 66 166 366 366 12 13 COAAAA ISAAAA AAAAxx 483 477 1 3 3 3 3 83 83 483 483 6 7 PSAAAA JSAAAA HHHHxx 838 478 0 2 8 18 8 38 38 338 838 16 17 GGAAAA KSAAAA OOOOxx 64 479 0 0 4 4 4 64 64 64 64 8 9 MCAAAA LSAAAA VVVVxx 981 480 1 1 1 1 1 81 181 481 981 2 3 TLAAAA MSAAAA AAAAxx 538 481 0 2 8 18 8 38 138 38 538 16 17 SUAAAA NSAAAA HHHHxx 39 482 1 3 9 19 9 39 39 39 39 18 19 NBAAAA OSAAAA OOOOxx 60 483 0 0 0 0 0 60 60 60 60 0 1 ICAAAA PSAAAA VVVVxx 874 484 0 2 4 14 4 74 74 374 874 8 9 QHAAAA QSAAAA AAAAxx 955 485 1 3 5 15 5 55 155 455 955 10 11 TKAAAA RSAAAA HHHHxx 347 486 1 3 7 7 7 47 147 347 347 14 15 JNAAAA SSAAAA OOOOxx 227 487 1 3 7 7 7 27 27 227 227 14 15 TIAAAA TSAAAA VVVVxx 44 488 0 0 4 4 4 44 44 44 44 8 9 SBAAAA USAAAA AAAAxx 446 489 0 2 6 6 6 46 46 446 446 12 13 ERAAAA VSAAAA HHHHxx 605 490 1 1 5 5 5 5 5 105 605 10 11 HXAAAA WSAAAA OOOOxx 570 491 0 2 0 10 0 70 170 70 570 0 1 YVAAAA XSAAAA VVVVxx 895 492 1 3 5 15 5 95 95 395 895 10 11 LIAAAA YSAAAA AAAAxx 760 493 0 0 0 0 0 60 160 260 760 0 1 GDAAAA ZSAAAA HHHHxx 428 494 0 0 8 8 8 28 28 428 428 16 17 MQAAAA ATAAAA OOOOxx 628 495 0 0 8 8 8 28 28 128 628 16 17 EYAAAA BTAAAA VVVVxx 933 496 1 1 3 13 3 33 133 433 933 6 7 XJAAAA CTAAAA AAAAxx 263 497 1 3 3 3 3 63 63 263 263 6 7 DKAAAA DTAAAA HHHHxx 729 498 1 1 9 9 9 29 129 229 729 18 19 BCAAAA ETAAAA OOOOxx 860 499 0 0 0 0 0 60 60 360 860 0 1 CHAAAA FTAAAA VVVVxx 76 500 0 0 6 16 6 76 76 76 76 12 13 YCAAAA GTAAAA AAAAxx 293 501 1 1 3 13 3 93 93 293 293 6 7 HLAAAA HTAAAA HHHHxx 296 502 0 0 6 16 6 96 96 296 296 12 13 KLAAAA ITAAAA OOOOxx 124 503 0 0 4 4 4 24 124 124 124 8 9 UEAAAA JTAAAA VVVVxx 568 504 0 0 8 8 8 68 168 68 568 16 17 WVAAAA KTAAAA AAAAxx 337 505 1 1 7 17 7 37 137 337 337 14 15 ZMAAAA LTAAAA HHHHxx 464 506 0 0 4 4 4 64 64 464 464 8 9 WRAAAA MTAAAA OOOOxx 582 507 0 2 2 2 2 82 182 82 582 4 5 KWAAAA NTAAAA VVVVxx 207 508 1 3 7 7 7 7 7 207 207 14 15 ZHAAAA OTAAAA AAAAxx 518 509 0 2 8 18 8 18 118 18 518 16 17 YTAAAA PTAAAA HHHHxx 513 510 1 1 3 13 3 13 113 13 513 6 7 TTAAAA QTAAAA OOOOxx 127 511 1 3 7 7 7 27 127 127 127 14 15 XEAAAA RTAAAA VVVVxx 396 512 0 0 6 16 6 96 196 396 396 12 13 GPAAAA STAAAA AAAAxx 781 513 1 1 1 1 1 81 181 281 781 2 3 BEAAAA TTAAAA HHHHxx 233 514 1 1 3 13 3 33 33 233 233 6 7 ZIAAAA UTAAAA OOOOxx 709 515 1 1 9 9 9 9 109 209 709 18 19 HBAAAA VTAAAA VVVVxx 325 516 1 1 5 5 5 25 125 325 325 10 11 NMAAAA WTAAAA AAAAxx 143 517 1 3 3 3 3 43 143 143 143 6 7 NFAAAA XTAAAA HHHHxx 824 518 0 0 4 4 4 24 24 324 824 8 9 SFAAAA YTAAAA OOOOxx 122 519 0 2 2 2 2 22 122 122 122 4 5 SEAAAA ZTAAAA VVVVxx 10 520 0 2 0 10 0 10 10 10 10 0 1 KAAAAA AUAAAA AAAAxx 41 521 1 1 1 1 1 41 41 41 41 2 3 PBAAAA BUAAAA HHHHxx 618 522 0 2 8 18 8 18 18 118 618 16 17 UXAAAA CUAAAA OOOOxx 161 523 1 1 1 1 1 61 161 161 161 2 3 FGAAAA DUAAAA VVVVxx 801 524 1 1 1 1 1 1 1 301 801 2 3 VEAAAA EUAAAA AAAAxx 768 525 0 0 8 8 8 68 168 268 768 16 17 ODAAAA FUAAAA HHHHxx 642 526 0 2 2 2 2 42 42 142 642 4 5 SYAAAA GUAAAA OOOOxx 803 527 1 3 3 3 3 3 3 303 803 6 7 XEAAAA HUAAAA VVVVxx 317 528 1 1 7 17 7 17 117 317 317 14 15 FMAAAA IUAAAA AAAAxx 938 529 0 2 8 18 8 38 138 438 938 16 17 CKAAAA JUAAAA HHHHxx 649 530 1 1 9 9 9 49 49 149 649 18 19 ZYAAAA KUAAAA OOOOxx 738 531 0 2 8 18 8 38 138 238 738 16 17 KCAAAA LUAAAA VVVVxx 344 532 0 0 4 4 4 44 144 344 344 8 9 GNAAAA MUAAAA AAAAxx 399 533 1 3 9 19 9 99 199 399 399 18 19 JPAAAA NUAAAA HHHHxx 609 534 1 1 9 9 9 9 9 109 609 18 19 LXAAAA OUAAAA OOOOxx 677 535 1 1 7 17 7 77 77 177 677 14 15 BAAAAA PUAAAA VVVVxx 478 536 0 2 8 18 8 78 78 478 478 16 17 KSAAAA QUAAAA AAAAxx 452 537 0 0 2 12 2 52 52 452 452 4 5 KRAAAA RUAAAA HHHHxx 261 538 1 1 1 1 1 61 61 261 261 2 3 BKAAAA SUAAAA OOOOxx 449 539 1 1 9 9 9 49 49 449 449 18 19 HRAAAA TUAAAA VVVVxx 433 540 1 1 3 13 3 33 33 433 433 6 7 RQAAAA UUAAAA AAAAxx 5 541 1 1 5 5 5 5 5 5 5 10 11 FAAAAA VUAAAA HHHHxx 664 542 0 0 4 4 4 64 64 164 664 8 9 OZAAAA WUAAAA OOOOxx 887 543 1 3 7 7 7 87 87 387 887 14 15 DIAAAA XUAAAA VVVVxx 546 544 0 2 6 6 6 46 146 46 546 12 13 AVAAAA YUAAAA AAAAxx 253 545 1 1 3 13 3 53 53 253 253 6 7 TJAAAA ZUAAAA HHHHxx 235 546 1 3 5 15 5 35 35 235 235 10 11 BJAAAA AVAAAA OOOOxx 258 547 0 2 8 18 8 58 58 258 258 16 17 YJAAAA BVAAAA VVVVxx 621 548 1 1 1 1 1 21 21 121 621 2 3 XXAAAA CVAAAA AAAAxx 998 549 0 2 8 18 8 98 198 498 998 16 17 KMAAAA DVAAAA HHHHxx 236 550 0 0 6 16 6 36 36 236 236 12 13 CJAAAA EVAAAA OOOOxx 537 551 1 1 7 17 7 37 137 37 537 14 15 RUAAAA FVAAAA VVVVxx 769 552 1 1 9 9 9 69 169 269 769 18 19 PDAAAA GVAAAA AAAAxx 921 553 1 1 1 1 1 21 121 421 921 2 3 LJAAAA HVAAAA HHHHxx 951 554 1 3 1 11 1 51 151 451 951 2 3 PKAAAA IVAAAA OOOOxx 240 555 0 0 0 0 0 40 40 240 240 0 1 GJAAAA JVAAAA VVVVxx 644 556 0 0 4 4 4 44 44 144 644 8 9 UYAAAA KVAAAA AAAAxx 352 557 0 0 2 12 2 52 152 352 352 4 5 ONAAAA LVAAAA HHHHxx 613 558 1 1 3 13 3 13 13 113 613 6 7 PXAAAA MVAAAA OOOOxx 784 559 0 0 4 4 4 84 184 284 784 8 9 EEAAAA NVAAAA VVVVxx 61 560 1 1 1 1 1 61 61 61 61 2 3 JCAAAA OVAAAA AAAAxx 144 561 0 0 4 4 4 44 144 144 144 8 9 OFAAAA PVAAAA HHHHxx 94 562 0 2 4 14 4 94 94 94 94 8 9 QDAAAA QVAAAA OOOOxx 270 563 0 2 0 10 0 70 70 270 270 0 1 KKAAAA RVAAAA VVVVxx 942 564 0 2 2 2 2 42 142 442 942 4 5 GKAAAA SVAAAA AAAAxx 756 565 0 0 6 16 6 56 156 256 756 12 13 CDAAAA TVAAAA HHHHxx 321 566 1 1 1 1 1 21 121 321 321 2 3 JMAAAA UVAAAA OOOOxx 36 567 0 0 6 16 6 36 36 36 36 12 13 KBAAAA VVAAAA VVVVxx 232 568 0 0 2 12 2 32 32 232 232 4 5 YIAAAA WVAAAA AAAAxx 430 569 0 2 0 10 0 30 30 430 430 0 1 OQAAAA XVAAAA HHHHxx 177 570 1 1 7 17 7 77 177 177 177 14 15 VGAAAA YVAAAA OOOOxx 220 571 0 0 0 0 0 20 20 220 220 0 1 MIAAAA ZVAAAA VVVVxx 109 572 1 1 9 9 9 9 109 109 109 18 19 FEAAAA AWAAAA AAAAxx 419 573 1 3 9 19 9 19 19 419 419 18 19 DQAAAA BWAAAA HHHHxx 135 574 1 3 5 15 5 35 135 135 135 10 11 FFAAAA CWAAAA OOOOxx 610 575 0 2 0 10 0 10 10 110 610 0 1 MXAAAA DWAAAA VVVVxx 956 576 0 0 6 16 6 56 156 456 956 12 13 UKAAAA EWAAAA AAAAxx 626 577 0 2 6 6 6 26 26 126 626 12 13 CYAAAA FWAAAA HHHHxx 375 578 1 3 5 15 5 75 175 375 375 10 11 LOAAAA GWAAAA OOOOxx 976 579 0 0 6 16 6 76 176 476 976 12 13 OLAAAA HWAAAA VVVVxx 152 580 0 0 2 12 2 52 152 152 152 4 5 WFAAAA IWAAAA AAAAxx 308 581 0 0 8 8 8 8 108 308 308 16 17 WLAAAA JWAAAA HHHHxx 445 582 1 1 5 5 5 45 45 445 445 10 11 DRAAAA KWAAAA OOOOxx 326 583 0 2 6 6 6 26 126 326 326 12 13 OMAAAA LWAAAA VVVVxx 422 584 0 2 2 2 2 22 22 422 422 4 5 GQAAAA MWAAAA AAAAxx 972 585 0 0 2 12 2 72 172 472 972 4 5 KLAAAA NWAAAA HHHHxx 45 586 1 1 5 5 5 45 45 45 45 10 11 TBAAAA OWAAAA OOOOxx 725 587 1 1 5 5 5 25 125 225 725 10 11 XBAAAA PWAAAA VVVVxx 753 588 1 1 3 13 3 53 153 253 753 6 7 ZCAAAA QWAAAA AAAAxx 493 589 1 1 3 13 3 93 93 493 493 6 7 ZSAAAA RWAAAA HHHHxx 601 590 1 1 1 1 1 1 1 101 601 2 3 DXAAAA SWAAAA OOOOxx 463 591 1 3 3 3 3 63 63 463 463 6 7 VRAAAA TWAAAA VVVVxx 303 592 1 3 3 3 3 3 103 303 303 6 7 RLAAAA UWAAAA AAAAxx 59 593 1 3 9 19 9 59 59 59 59 18 19 HCAAAA VWAAAA HHHHxx 595 594 1 3 5 15 5 95 195 95 595 10 11 XWAAAA WWAAAA OOOOxx 807 595 1 3 7 7 7 7 7 307 807 14 15 BFAAAA XWAAAA VVVVxx 424 596 0 0 4 4 4 24 24 424 424 8 9 IQAAAA YWAAAA AAAAxx 521 597 1 1 1 1 1 21 121 21 521 2 3 BUAAAA ZWAAAA HHHHxx 341 598 1 1 1 1 1 41 141 341 341 2 3 DNAAAA AXAAAA OOOOxx 571 599 1 3 1 11 1 71 171 71 571 2 3 ZVAAAA BXAAAA VVVVxx 165 600 1 1 5 5 5 65 165 165 165 10 11 JGAAAA CXAAAA AAAAxx 908 601 0 0 8 8 8 8 108 408 908 16 17 YIAAAA DXAAAA HHHHxx 351 602 1 3 1 11 1 51 151 351 351 2 3 NNAAAA EXAAAA OOOOxx 334 603 0 2 4 14 4 34 134 334 334 8 9 WMAAAA FXAAAA VVVVxx 636 604 0 0 6 16 6 36 36 136 636 12 13 MYAAAA GXAAAA AAAAxx 138 605 0 2 8 18 8 38 138 138 138 16 17 IFAAAA HXAAAA HHHHxx 438 606 0 2 8 18 8 38 38 438 438 16 17 WQAAAA IXAAAA OOOOxx 391 607 1 3 1 11 1 91 191 391 391 2 3 BPAAAA JXAAAA VVVVxx 395 608 1 3 5 15 5 95 195 395 395 10 11 FPAAAA KXAAAA AAAAxx 502 609 0 2 2 2 2 2 102 2 502 4 5 ITAAAA LXAAAA HHHHxx 85 610 1 1 5 5 5 85 85 85 85 10 11 HDAAAA MXAAAA OOOOxx 786 611 0 2 6 6 6 86 186 286 786 12 13 GEAAAA NXAAAA VVVVxx 619 612 1 3 9 19 9 19 19 119 619 18 19 VXAAAA OXAAAA AAAAxx 440 613 0 0 0 0 0 40 40 440 440 0 1 YQAAAA PXAAAA HHHHxx 949 614 1 1 9 9 9 49 149 449 949 18 19 NKAAAA QXAAAA OOOOxx 691 615 1 3 1 11 1 91 91 191 691 2 3 PAAAAA RXAAAA VVVVxx 348 616 0 0 8 8 8 48 148 348 348 16 17 KNAAAA SXAAAA AAAAxx 506 617 0 2 6 6 6 6 106 6 506 12 13 MTAAAA TXAAAA HHHHxx 192 618 0 0 2 12 2 92 192 192 192 4 5 KHAAAA UXAAAA OOOOxx 369 619 1 1 9 9 9 69 169 369 369 18 19 FOAAAA VXAAAA VVVVxx 311 620 1 3 1 11 1 11 111 311 311 2 3 ZLAAAA WXAAAA AAAAxx 273 621 1 1 3 13 3 73 73 273 273 6 7 NKAAAA XXAAAA HHHHxx 770 622 0 2 0 10 0 70 170 270 770 0 1 QDAAAA YXAAAA OOOOxx 191 623 1 3 1 11 1 91 191 191 191 2 3 JHAAAA ZXAAAA VVVVxx 90 624 0 2 0 10 0 90 90 90 90 0 1 MDAAAA AYAAAA AAAAxx 163 625 1 3 3 3 3 63 163 163 163 6 7 HGAAAA BYAAAA HHHHxx 350 626 0 2 0 10 0 50 150 350 350 0 1 MNAAAA CYAAAA OOOOxx 55 627 1 3 5 15 5 55 55 55 55 10 11 DCAAAA DYAAAA VVVVxx 488 628 0 0 8 8 8 88 88 488 488 16 17 USAAAA EYAAAA AAAAxx 215 629 1 3 5 15 5 15 15 215 215 10 11 HIAAAA FYAAAA HHHHxx 732 630 0 0 2 12 2 32 132 232 732 4 5 ECAAAA GYAAAA OOOOxx 688 631 0 0 8 8 8 88 88 188 688 16 17 MAAAAA HYAAAA VVVVxx 520 632 0 0 0 0 0 20 120 20 520 0 1 AUAAAA IYAAAA AAAAxx 62 633 0 2 2 2 2 62 62 62 62 4 5 KCAAAA JYAAAA HHHHxx 423 634 1 3 3 3 3 23 23 423 423 6 7 HQAAAA KYAAAA OOOOxx 242 635 0 2 2 2 2 42 42 242 242 4 5 IJAAAA LYAAAA VVVVxx 193 636 1 1 3 13 3 93 193 193 193 6 7 LHAAAA MYAAAA AAAAxx 648 637 0 0 8 8 8 48 48 148 648 16 17 YYAAAA NYAAAA HHHHxx 459 638 1 3 9 19 9 59 59 459 459 18 19 RRAAAA OYAAAA OOOOxx 196 639 0 0 6 16 6 96 196 196 196 12 13 OHAAAA PYAAAA VVVVxx 476 640 0 0 6 16 6 76 76 476 476 12 13 ISAAAA QYAAAA AAAAxx 903 641 1 3 3 3 3 3 103 403 903 6 7 TIAAAA RYAAAA HHHHxx 974 642 0 2 4 14 4 74 174 474 974 8 9 MLAAAA SYAAAA OOOOxx 603 643 1 3 3 3 3 3 3 103 603 6 7 FXAAAA TYAAAA VVVVxx 12 644 0 0 2 12 2 12 12 12 12 4 5 MAAAAA UYAAAA AAAAxx 599 645 1 3 9 19 9 99 199 99 599 18 19 BXAAAA VYAAAA HHHHxx 914 646 0 2 4 14 4 14 114 414 914 8 9 EJAAAA WYAAAA OOOOxx 7 647 1 3 7 7 7 7 7 7 7 14 15 HAAAAA XYAAAA VVVVxx 213 648 1 1 3 13 3 13 13 213 213 6 7 FIAAAA YYAAAA AAAAxx 174 649 0 2 4 14 4 74 174 174 174 8 9 SGAAAA ZYAAAA HHHHxx 392 650 0 0 2 12 2 92 192 392 392 4 5 CPAAAA AZAAAA OOOOxx 674 651 0 2 4 14 4 74 74 174 674 8 9 YZAAAA BZAAAA VVVVxx 650 652 0 2 0 10 0 50 50 150 650 0 1 AZAAAA CZAAAA AAAAxx 8 653 0 0 8 8 8 8 8 8 8 16 17 IAAAAA DZAAAA HHHHxx 492 654 0 0 2 12 2 92 92 492 492 4 5 YSAAAA EZAAAA OOOOxx 322 655 0 2 2 2 2 22 122 322 322 4 5 KMAAAA FZAAAA VVVVxx 315 656 1 3 5 15 5 15 115 315 315 10 11 DMAAAA GZAAAA AAAAxx 380 657 0 0 0 0 0 80 180 380 380 0 1 QOAAAA HZAAAA HHHHxx 353 658 1 1 3 13 3 53 153 353 353 6 7 PNAAAA IZAAAA OOOOxx 892 659 0 0 2 12 2 92 92 392 892 4 5 IIAAAA JZAAAA VVVVxx 932 660 0 0 2 12 2 32 132 432 932 4 5 WJAAAA KZAAAA AAAAxx 993 661 1 1 3 13 3 93 193 493 993 6 7 FMAAAA LZAAAA HHHHxx 859 662 1 3 9 19 9 59 59 359 859 18 19 BHAAAA MZAAAA OOOOxx 806 663 0 2 6 6 6 6 6 306 806 12 13 AFAAAA NZAAAA VVVVxx 145 664 1 1 5 5 5 45 145 145 145 10 11 PFAAAA OZAAAA AAAAxx 373 665 1 1 3 13 3 73 173 373 373 6 7 JOAAAA PZAAAA HHHHxx 418 666 0 2 8 18 8 18 18 418 418 16 17 CQAAAA QZAAAA OOOOxx 865 667 1 1 5 5 5 65 65 365 865 10 11 HHAAAA RZAAAA VVVVxx 462 668 0 2 2 2 2 62 62 462 462 4 5 URAAAA SZAAAA AAAAxx 24 669 0 0 4 4 4 24 24 24 24 8 9 YAAAAA TZAAAA HHHHxx 920 670 0 0 0 0 0 20 120 420 920 0 1 KJAAAA UZAAAA OOOOxx 672 671 0 0 2 12 2 72 72 172 672 4 5 WZAAAA VZAAAA VVVVxx 92 672 0 0 2 12 2 92 92 92 92 4 5 ODAAAA WZAAAA AAAAxx 721 673 1 1 1 1 1 21 121 221 721 2 3 TBAAAA XZAAAA HHHHxx 646 674 0 2 6 6 6 46 46 146 646 12 13 WYAAAA YZAAAA OOOOxx 910 675 0 2 0 10 0 10 110 410 910 0 1 AJAAAA ZZAAAA VVVVxx 909 676 1 1 9 9 9 9 109 409 909 18 19 ZIAAAA AABAAA AAAAxx 630 677 0 2 0 10 0 30 30 130 630 0 1 GYAAAA BABAAA HHHHxx 482 678 0 2 2 2 2 82 82 482 482 4 5 OSAAAA CABAAA OOOOxx 559 679 1 3 9 19 9 59 159 59 559 18 19 NVAAAA DABAAA VVVVxx 853 680 1 1 3 13 3 53 53 353 853 6 7 VGAAAA EABAAA AAAAxx 141 681 1 1 1 1 1 41 141 141 141 2 3 LFAAAA FABAAA HHHHxx 266 682 0 2 6 6 6 66 66 266 266 12 13 GKAAAA GABAAA OOOOxx 835 683 1 3 5 15 5 35 35 335 835 10 11 DGAAAA HABAAA VVVVxx 164 684 0 0 4 4 4 64 164 164 164 8 9 IGAAAA IABAAA AAAAxx 629 685 1 1 9 9 9 29 29 129 629 18 19 FYAAAA JABAAA HHHHxx 203 686 1 3 3 3 3 3 3 203 203 6 7 VHAAAA KABAAA OOOOxx 411 687 1 3 1 11 1 11 11 411 411 2 3 VPAAAA LABAAA VVVVxx 930 688 0 2 0 10 0 30 130 430 930 0 1 UJAAAA MABAAA AAAAxx 435 689 1 3 5 15 5 35 35 435 435 10 11 TQAAAA NABAAA HHHHxx 563 690 1 3 3 3 3 63 163 63 563 6 7 RVAAAA OABAAA OOOOxx 960 691 0 0 0 0 0 60 160 460 960 0 1 YKAAAA PABAAA VVVVxx 733 692 1 1 3 13 3 33 133 233 733 6 7 FCAAAA QABAAA AAAAxx 967 693 1 3 7 7 7 67 167 467 967 14 15 FLAAAA RABAAA HHHHxx 668 694 0 0 8 8 8 68 68 168 668 16 17 SZAAAA SABAAA OOOOxx 994 695 0 2 4 14 4 94 194 494 994 8 9 GMAAAA TABAAA VVVVxx 129 696 1 1 9 9 9 29 129 129 129 18 19 ZEAAAA UABAAA AAAAxx 954 697 0 2 4 14 4 54 154 454 954 8 9 SKAAAA VABAAA HHHHxx 68 698 0 0 8 8 8 68 68 68 68 16 17 QCAAAA WABAAA OOOOxx 79 699 1 3 9 19 9 79 79 79 79 18 19 BDAAAA XABAAA VVVVxx 121 700 1 1 1 1 1 21 121 121 121 2 3 REAAAA YABAAA AAAAxx 740 701 0 0 0 0 0 40 140 240 740 0 1 MCAAAA ZABAAA HHHHxx 902 702 0 2 2 2 2 2 102 402 902 4 5 SIAAAA ABBAAA OOOOxx 695 703 1 3 5 15 5 95 95 195 695 10 11 TAAAAA BBBAAA VVVVxx 455 704 1 3 5 15 5 55 55 455 455 10 11 NRAAAA CBBAAA AAAAxx 89 705 1 1 9 9 9 89 89 89 89 18 19 LDAAAA DBBAAA HHHHxx 893 706 1 1 3 13 3 93 93 393 893 6 7 JIAAAA EBBAAA OOOOxx 202 707 0 2 2 2 2 2 2 202 202 4 5 UHAAAA FBBAAA VVVVxx 132 708 0 0 2 12 2 32 132 132 132 4 5 CFAAAA GBBAAA AAAAxx 782 709 0 2 2 2 2 82 182 282 782 4 5 CEAAAA HBBAAA HHHHxx 512 710 0 0 2 12 2 12 112 12 512 4 5 STAAAA IBBAAA OOOOxx 857 711 1 1 7 17 7 57 57 357 857 14 15 ZGAAAA JBBAAA VVVVxx 248 712 0 0 8 8 8 48 48 248 248 16 17 OJAAAA KBBAAA AAAAxx 858 713 0 2 8 18 8 58 58 358 858 16 17 AHAAAA LBBAAA HHHHxx 527 714 1 3 7 7 7 27 127 27 527 14 15 HUAAAA MBBAAA OOOOxx 450 715 0 2 0 10 0 50 50 450 450 0 1 IRAAAA NBBAAA VVVVxx 712 716 0 0 2 12 2 12 112 212 712 4 5 KBAAAA OBBAAA AAAAxx 153 717 1 1 3 13 3 53 153 153 153 6 7 XFAAAA PBBAAA HHHHxx 587 718 1 3 7 7 7 87 187 87 587 14 15 PWAAAA QBBAAA OOOOxx 593 719 1 1 3 13 3 93 193 93 593 6 7 VWAAAA RBBAAA VVVVxx 249 720 1 1 9 9 9 49 49 249 249 18 19 PJAAAA SBBAAA AAAAxx 128 721 0 0 8 8 8 28 128 128 128 16 17 YEAAAA TBBAAA HHHHxx 675 722 1 3 5 15 5 75 75 175 675 10 11 ZZAAAA UBBAAA OOOOxx 929 723 1 1 9 9 9 29 129 429 929 18 19 TJAAAA VBBAAA VVVVxx 156 724 0 0 6 16 6 56 156 156 156 12 13 AGAAAA WBBAAA AAAAxx 415 725 1 3 5 15 5 15 15 415 415 10 11 ZPAAAA XBBAAA HHHHxx 28 726 0 0 8 8 8 28 28 28 28 16 17 CBAAAA YBBAAA OOOOxx 18 727 0 2 8 18 8 18 18 18 18 16 17 SAAAAA ZBBAAA VVVVxx 255 728 1 3 5 15 5 55 55 255 255 10 11 VJAAAA ACBAAA AAAAxx 793 729 1 1 3 13 3 93 193 293 793 6 7 NEAAAA BCBAAA HHHHxx 554 730 0 2 4 14 4 54 154 54 554 8 9 IVAAAA CCBAAA OOOOxx 467 731 1 3 7 7 7 67 67 467 467 14 15 ZRAAAA DCBAAA VVVVxx 410 732 0 2 0 10 0 10 10 410 410 0 1 UPAAAA ECBAAA AAAAxx 651 733 1 3 1 11 1 51 51 151 651 2 3 BZAAAA FCBAAA HHHHxx 287 734 1 3 7 7 7 87 87 287 287 14 15 BLAAAA GCBAAA OOOOxx 640 735 0 0 0 0 0 40 40 140 640 0 1 QYAAAA HCBAAA VVVVxx 245 736 1 1 5 5 5 45 45 245 245 10 11 LJAAAA ICBAAA AAAAxx 21 737 1 1 1 1 1 21 21 21 21 2 3 VAAAAA JCBAAA HHHHxx 83 738 1 3 3 3 3 83 83 83 83 6 7 FDAAAA KCBAAA OOOOxx 228 739 0 0 8 8 8 28 28 228 228 16 17 UIAAAA LCBAAA VVVVxx 323 740 1 3 3 3 3 23 123 323 323 6 7 LMAAAA MCBAAA AAAAxx 594 741 0 2 4 14 4 94 194 94 594 8 9 WWAAAA NCBAAA HHHHxx 528 742 0 0 8 8 8 28 128 28 528 16 17 IUAAAA OCBAAA OOOOxx 276 743 0 0 6 16 6 76 76 276 276 12 13 QKAAAA PCBAAA VVVVxx 598 744 0 2 8 18 8 98 198 98 598 16 17 AXAAAA QCBAAA AAAAxx 635 745 1 3 5 15 5 35 35 135 635 10 11 LYAAAA RCBAAA HHHHxx 868 746 0 0 8 8 8 68 68 368 868 16 17 KHAAAA SCBAAA OOOOxx 290 747 0 2 0 10 0 90 90 290 290 0 1 ELAAAA TCBAAA VVVVxx 468 748 0 0 8 8 8 68 68 468 468 16 17 ASAAAA UCBAAA AAAAxx 689 749 1 1 9 9 9 89 89 189 689 18 19 NAAAAA VCBAAA HHHHxx 799 750 1 3 9 19 9 99 199 299 799 18 19 TEAAAA WCBAAA OOOOxx 210 751 0 2 0 10 0 10 10 210 210 0 1 CIAAAA XCBAAA VVVVxx 346 752 0 2 6 6 6 46 146 346 346 12 13 INAAAA YCBAAA AAAAxx 957 753 1 1 7 17 7 57 157 457 957 14 15 VKAAAA ZCBAAA HHHHxx 905 754 1 1 5 5 5 5 105 405 905 10 11 VIAAAA ADBAAA OOOOxx 523 755 1 3 3 3 3 23 123 23 523 6 7 DUAAAA BDBAAA VVVVxx 899 756 1 3 9 19 9 99 99 399 899 18 19 PIAAAA CDBAAA AAAAxx 867 757 1 3 7 7 7 67 67 367 867 14 15 JHAAAA DDBAAA HHHHxx 11 758 1 3 1 11 1 11 11 11 11 2 3 LAAAAA EDBAAA OOOOxx 320 759 0 0 0 0 0 20 120 320 320 0 1 IMAAAA FDBAAA VVVVxx 766 760 0 2 6 6 6 66 166 266 766 12 13 MDAAAA GDBAAA AAAAxx 84 761 0 0 4 4 4 84 84 84 84 8 9 GDAAAA HDBAAA HHHHxx 507 762 1 3 7 7 7 7 107 7 507 14 15 NTAAAA IDBAAA OOOOxx 471 763 1 3 1 11 1 71 71 471 471 2 3 DSAAAA JDBAAA VVVVxx 517 764 1 1 7 17 7 17 117 17 517 14 15 XTAAAA KDBAAA AAAAxx 234 765 0 2 4 14 4 34 34 234 234 8 9 AJAAAA LDBAAA HHHHxx 988 766 0 0 8 8 8 88 188 488 988 16 17 AMAAAA MDBAAA OOOOxx 473 767 1 1 3 13 3 73 73 473 473 6 7 FSAAAA NDBAAA VVVVxx 66 768 0 2 6 6 6 66 66 66 66 12 13 OCAAAA ODBAAA AAAAxx 530 769 0 2 0 10 0 30 130 30 530 0 1 KUAAAA PDBAAA HHHHxx 834 770 0 2 4 14 4 34 34 334 834 8 9 CGAAAA QDBAAA OOOOxx 894 771 0 2 4 14 4 94 94 394 894 8 9 KIAAAA RDBAAA VVVVxx 481 772 1 1 1 1 1 81 81 481 481 2 3 NSAAAA SDBAAA AAAAxx 280 773 0 0 0 0 0 80 80 280 280 0 1 UKAAAA TDBAAA HHHHxx 705 774 1 1 5 5 5 5 105 205 705 10 11 DBAAAA UDBAAA OOOOxx 218 775 0 2 8 18 8 18 18 218 218 16 17 KIAAAA VDBAAA VVVVxx 560 776 0 0 0 0 0 60 160 60 560 0 1 OVAAAA WDBAAA AAAAxx 123 777 1 3 3 3 3 23 123 123 123 6 7 TEAAAA XDBAAA HHHHxx 289 778 1 1 9 9 9 89 89 289 289 18 19 DLAAAA YDBAAA OOOOxx 189 779 1 1 9 9 9 89 189 189 189 18 19 HHAAAA ZDBAAA VVVVxx 541 780 1 1 1 1 1 41 141 41 541 2 3 VUAAAA AEBAAA AAAAxx 876 781 0 0 6 16 6 76 76 376 876 12 13 SHAAAA BEBAAA HHHHxx 504 782 0 0 4 4 4 4 104 4 504 8 9 KTAAAA CEBAAA OOOOxx 643 783 1 3 3 3 3 43 43 143 643 6 7 TYAAAA DEBAAA VVVVxx 73 784 1 1 3 13 3 73 73 73 73 6 7 VCAAAA EEBAAA AAAAxx 465 785 1 1 5 5 5 65 65 465 465 10 11 XRAAAA FEBAAA HHHHxx 861 786 1 1 1 1 1 61 61 361 861 2 3 DHAAAA GEBAAA OOOOxx 355 787 1 3 5 15 5 55 155 355 355 10 11 RNAAAA HEBAAA VVVVxx 441 788 1 1 1 1 1 41 41 441 441 2 3 ZQAAAA IEBAAA AAAAxx 219 789 1 3 9 19 9 19 19 219 219 18 19 LIAAAA JEBAAA HHHHxx 839 790 1 3 9 19 9 39 39 339 839 18 19 HGAAAA KEBAAA OOOOxx 271 791 1 3 1 11 1 71 71 271 271 2 3 LKAAAA LEBAAA VVVVxx 212 792 0 0 2 12 2 12 12 212 212 4 5 EIAAAA MEBAAA AAAAxx 904 793 0 0 4 4 4 4 104 404 904 8 9 UIAAAA NEBAAA HHHHxx 244 794 0 0 4 4 4 44 44 244 244 8 9 KJAAAA OEBAAA OOOOxx 751 795 1 3 1 11 1 51 151 251 751 2 3 XCAAAA PEBAAA VVVVxx 944 796 0 0 4 4 4 44 144 444 944 8 9 IKAAAA QEBAAA AAAAxx 305 797 1 1 5 5 5 5 105 305 305 10 11 TLAAAA REBAAA HHHHxx 617 798 1 1 7 17 7 17 17 117 617 14 15 TXAAAA SEBAAA OOOOxx 891 799 1 3 1 11 1 91 91 391 891 2 3 HIAAAA TEBAAA VVVVxx 653 800 1 1 3 13 3 53 53 153 653 6 7 DZAAAA UEBAAA AAAAxx 845 801 1 1 5 5 5 45 45 345 845 10 11 NGAAAA VEBAAA HHHHxx 936 802 0 0 6 16 6 36 136 436 936 12 13 AKAAAA WEBAAA OOOOxx 91 803 1 3 1 11 1 91 91 91 91 2 3 NDAAAA XEBAAA VVVVxx 442 804 0 2 2 2 2 42 42 442 442 4 5 ARAAAA YEBAAA AAAAxx 498 805 0 2 8 18 8 98 98 498 498 16 17 ETAAAA ZEBAAA HHHHxx 987 806 1 3 7 7 7 87 187 487 987 14 15 ZLAAAA AFBAAA OOOOxx 194 807 0 2 4 14 4 94 194 194 194 8 9 MHAAAA BFBAAA VVVVxx 927 808 1 3 7 7 7 27 127 427 927 14 15 RJAAAA CFBAAA AAAAxx 607 809 1 3 7 7 7 7 7 107 607 14 15 JXAAAA DFBAAA HHHHxx 119 810 1 3 9 19 9 19 119 119 119 18 19 PEAAAA EFBAAA OOOOxx 182 811 0 2 2 2 2 82 182 182 182 4 5 AHAAAA FFBAAA VVVVxx 606 812 0 2 6 6 6 6 6 106 606 12 13 IXAAAA GFBAAA AAAAxx 849 813 1 1 9 9 9 49 49 349 849 18 19 RGAAAA HFBAAA HHHHxx 34 814 0 2 4 14 4 34 34 34 34 8 9 IBAAAA IFBAAA OOOOxx 683 815 1 3 3 3 3 83 83 183 683 6 7 HAAAAA JFBAAA VVVVxx 134 816 0 2 4 14 4 34 134 134 134 8 9 EFAAAA KFBAAA AAAAxx 331 817 1 3 1 11 1 31 131 331 331 2 3 TMAAAA LFBAAA HHHHxx 808 818 0 0 8 8 8 8 8 308 808 16 17 CFAAAA MFBAAA OOOOxx 703 819 1 3 3 3 3 3 103 203 703 6 7 BBAAAA NFBAAA VVVVxx 669 820 1 1 9 9 9 69 69 169 669 18 19 TZAAAA OFBAAA AAAAxx 264 821 0 0 4 4 4 64 64 264 264 8 9 EKAAAA PFBAAA HHHHxx 277 822 1 1 7 17 7 77 77 277 277 14 15 RKAAAA QFBAAA OOOOxx 877 823 1 1 7 17 7 77 77 377 877 14 15 THAAAA RFBAAA VVVVxx 783 824 1 3 3 3 3 83 183 283 783 6 7 DEAAAA SFBAAA AAAAxx 791 825 1 3 1 11 1 91 191 291 791 2 3 LEAAAA TFBAAA HHHHxx 171 826 1 3 1 11 1 71 171 171 171 2 3 PGAAAA UFBAAA OOOOxx 564 827 0 0 4 4 4 64 164 64 564 8 9 SVAAAA VFBAAA VVVVxx 230 828 0 2 0 10 0 30 30 230 230 0 1 WIAAAA WFBAAA AAAAxx 881 829 1 1 1 1 1 81 81 381 881 2 3 XHAAAA XFBAAA HHHHxx 890 830 0 2 0 10 0 90 90 390 890 0 1 GIAAAA YFBAAA OOOOxx 374 831 0 2 4 14 4 74 174 374 374 8 9 KOAAAA ZFBAAA VVVVxx 697 832 1 1 7 17 7 97 97 197 697 14 15 VAAAAA AGBAAA AAAAxx 4 833 0 0 4 4 4 4 4 4 4 8 9 EAAAAA BGBAAA HHHHxx 385 834 1 1 5 5 5 85 185 385 385 10 11 VOAAAA CGBAAA OOOOxx 739 835 1 3 9 19 9 39 139 239 739 18 19 LCAAAA DGBAAA VVVVxx 623 836 1 3 3 3 3 23 23 123 623 6 7 ZXAAAA EGBAAA AAAAxx 547 837 1 3 7 7 7 47 147 47 547 14 15 BVAAAA FGBAAA HHHHxx 532 838 0 0 2 12 2 32 132 32 532 4 5 MUAAAA GGBAAA OOOOxx 383 839 1 3 3 3 3 83 183 383 383 6 7 TOAAAA HGBAAA VVVVxx 181 840 1 1 1 1 1 81 181 181 181 2 3 ZGAAAA IGBAAA AAAAxx 327 841 1 3 7 7 7 27 127 327 327 14 15 PMAAAA JGBAAA HHHHxx 701 842 1 1 1 1 1 1 101 201 701 2 3 ZAAAAA KGBAAA OOOOxx 111 843 1 3 1 11 1 11 111 111 111 2 3 HEAAAA LGBAAA VVVVxx 977 844 1 1 7 17 7 77 177 477 977 14 15 PLAAAA MGBAAA AAAAxx 431 845 1 3 1 11 1 31 31 431 431 2 3 PQAAAA NGBAAA HHHHxx 456 846 0 0 6 16 6 56 56 456 456 12 13 ORAAAA OGBAAA OOOOxx 368 847 0 0 8 8 8 68 168 368 368 16 17 EOAAAA PGBAAA VVVVxx 32 848 0 0 2 12 2 32 32 32 32 4 5 GBAAAA QGBAAA AAAAxx 125 849 1 1 5 5 5 25 125 125 125 10 11 VEAAAA RGBAAA HHHHxx 847 850 1 3 7 7 7 47 47 347 847 14 15 PGAAAA SGBAAA OOOOxx 485 851 1 1 5 5 5 85 85 485 485 10 11 RSAAAA TGBAAA VVVVxx 387 852 1 3 7 7 7 87 187 387 387 14 15 XOAAAA UGBAAA AAAAxx 288 853 0 0 8 8 8 88 88 288 288 16 17 CLAAAA VGBAAA HHHHxx 919 854 1 3 9 19 9 19 119 419 919 18 19 JJAAAA WGBAAA OOOOxx 393 855 1 1 3 13 3 93 193 393 393 6 7 DPAAAA XGBAAA VVVVxx 953 856 1 1 3 13 3 53 153 453 953 6 7 RKAAAA YGBAAA AAAAxx 798 857 0 2 8 18 8 98 198 298 798 16 17 SEAAAA ZGBAAA HHHHxx 940 858 0 0 0 0 0 40 140 440 940 0 1 EKAAAA AHBAAA OOOOxx 198 859 0 2 8 18 8 98 198 198 198 16 17 QHAAAA BHBAAA VVVVxx 25 860 1 1 5 5 5 25 25 25 25 10 11 ZAAAAA CHBAAA AAAAxx 190 861 0 2 0 10 0 90 190 190 190 0 1 IHAAAA DHBAAA HHHHxx 820 862 0 0 0 0 0 20 20 320 820 0 1 OFAAAA EHBAAA OOOOxx 15 863 1 3 5 15 5 15 15 15 15 10 11 PAAAAA FHBAAA VVVVxx 427 864 1 3 7 7 7 27 27 427 427 14 15 LQAAAA GHBAAA AAAAxx 349 865 1 1 9 9 9 49 149 349 349 18 19 LNAAAA HHBAAA HHHHxx 785 866 1 1 5 5 5 85 185 285 785 10 11 FEAAAA IHBAAA OOOOxx 340 867 0 0 0 0 0 40 140 340 340 0 1 CNAAAA JHBAAA VVVVxx 292 868 0 0 2 12 2 92 92 292 292 4 5 GLAAAA KHBAAA AAAAxx 17 869 1 1 7 17 7 17 17 17 17 14 15 RAAAAA LHBAAA HHHHxx 985 870 1 1 5 5 5 85 185 485 985 10 11 XLAAAA MHBAAA OOOOxx 645 871 1 1 5 5 5 45 45 145 645 10 11 VYAAAA NHBAAA VVVVxx 631 872 1 3 1 11 1 31 31 131 631 2 3 HYAAAA OHBAAA AAAAxx 761 873 1 1 1 1 1 61 161 261 761 2 3 HDAAAA PHBAAA HHHHxx 707 874 1 3 7 7 7 7 107 207 707 14 15 FBAAAA QHBAAA OOOOxx 776 875 0 0 6 16 6 76 176 276 776 12 13 WDAAAA RHBAAA VVVVxx 856 876 0 0 6 16 6 56 56 356 856 12 13 YGAAAA SHBAAA AAAAxx 978 877 0 2 8 18 8 78 178 478 978 16 17 QLAAAA THBAAA HHHHxx 710 878 0 2 0 10 0 10 110 210 710 0 1 IBAAAA UHBAAA OOOOxx 604 879 0 0 4 4 4 4 4 104 604 8 9 GXAAAA VHBAAA VVVVxx 291 880 1 3 1 11 1 91 91 291 291 2 3 FLAAAA WHBAAA AAAAxx 747 881 1 3 7 7 7 47 147 247 747 14 15 TCAAAA XHBAAA HHHHxx 837 882 1 1 7 17 7 37 37 337 837 14 15 FGAAAA YHBAAA OOOOxx 722 883 0 2 2 2 2 22 122 222 722 4 5 UBAAAA ZHBAAA VVVVxx 925 884 1 1 5 5 5 25 125 425 925 10 11 PJAAAA AIBAAA AAAAxx 49 885 1 1 9 9 9 49 49 49 49 18 19 XBAAAA BIBAAA HHHHxx 832 886 0 0 2 12 2 32 32 332 832 4 5 AGAAAA CIBAAA OOOOxx 336 887 0 0 6 16 6 36 136 336 336 12 13 YMAAAA DIBAAA VVVVxx 185 888 1 1 5 5 5 85 185 185 185 10 11 DHAAAA EIBAAA AAAAxx 434 889 0 2 4 14 4 34 34 434 434 8 9 SQAAAA FIBAAA HHHHxx 284 890 0 0 4 4 4 84 84 284 284 8 9 YKAAAA GIBAAA OOOOxx 812 891 0 0 2 12 2 12 12 312 812 4 5 GFAAAA HIBAAA VVVVxx 810 892 0 2 0 10 0 10 10 310 810 0 1 EFAAAA IIBAAA AAAAxx 252 893 0 0 2 12 2 52 52 252 252 4 5 SJAAAA JIBAAA HHHHxx 965 894 1 1 5 5 5 65 165 465 965 10 11 DLAAAA KIBAAA OOOOxx 110 895 0 2 0 10 0 10 110 110 110 0 1 GEAAAA LIBAAA VVVVxx 698 896 0 2 8 18 8 98 98 198 698 16 17 WAAAAA MIBAAA AAAAxx 283 897 1 3 3 3 3 83 83 283 283 6 7 XKAAAA NIBAAA HHHHxx 533 898 1 1 3 13 3 33 133 33 533 6 7 NUAAAA OIBAAA OOOOxx 662 899 0 2 2 2 2 62 62 162 662 4 5 MZAAAA PIBAAA VVVVxx 329 900 1 1 9 9 9 29 129 329 329 18 19 RMAAAA QIBAAA AAAAxx 250 901 0 2 0 10 0 50 50 250 250 0 1 QJAAAA RIBAAA HHHHxx 407 902 1 3 7 7 7 7 7 407 407 14 15 RPAAAA SIBAAA OOOOxx 823 903 1 3 3 3 3 23 23 323 823 6 7 RFAAAA TIBAAA VVVVxx 852 904 0 0 2 12 2 52 52 352 852 4 5 UGAAAA UIBAAA AAAAxx 871 905 1 3 1 11 1 71 71 371 871 2 3 NHAAAA VIBAAA HHHHxx 118 906 0 2 8 18 8 18 118 118 118 16 17 OEAAAA WIBAAA OOOOxx 912 907 0 0 2 12 2 12 112 412 912 4 5 CJAAAA XIBAAA VVVVxx 458 908 0 2 8 18 8 58 58 458 458 16 17 QRAAAA YIBAAA AAAAxx 926 909 0 2 6 6 6 26 126 426 926 12 13 QJAAAA ZIBAAA HHHHxx 328 910 0 0 8 8 8 28 128 328 328 16 17 QMAAAA AJBAAA OOOOxx 980 911 0 0 0 0 0 80 180 480 980 0 1 SLAAAA BJBAAA VVVVxx 259 912 1 3 9 19 9 59 59 259 259 18 19 ZJAAAA CJBAAA AAAAxx 900 913 0 0 0 0 0 0 100 400 900 0 1 QIAAAA DJBAAA HHHHxx 137 914 1 1 7 17 7 37 137 137 137 14 15 HFAAAA EJBAAA OOOOxx 159 915 1 3 9 19 9 59 159 159 159 18 19 DGAAAA FJBAAA VVVVxx 243 916 1 3 3 3 3 43 43 243 243 6 7 JJAAAA GJBAAA AAAAxx 472 917 0 0 2 12 2 72 72 472 472 4 5 ESAAAA HJBAAA HHHHxx 796 918 0 0 6 16 6 96 196 296 796 12 13 QEAAAA IJBAAA OOOOxx 382 919 0 2 2 2 2 82 182 382 382 4 5 SOAAAA JJBAAA VVVVxx 911 920 1 3 1 11 1 11 111 411 911 2 3 BJAAAA KJBAAA AAAAxx 179 921 1 3 9 19 9 79 179 179 179 18 19 XGAAAA LJBAAA HHHHxx 778 922 0 2 8 18 8 78 178 278 778 16 17 YDAAAA MJBAAA OOOOxx 405 923 1 1 5 5 5 5 5 405 405 10 11 PPAAAA NJBAAA VVVVxx 265 924 1 1 5 5 5 65 65 265 265 10 11 FKAAAA OJBAAA AAAAxx 556 925 0 0 6 16 6 56 156 56 556 12 13 KVAAAA PJBAAA HHHHxx 16 926 0 0 6 16 6 16 16 16 16 12 13 QAAAAA QJBAAA OOOOxx 706 927 0 2 6 6 6 6 106 206 706 12 13 EBAAAA RJBAAA VVVVxx 497 928 1 1 7 17 7 97 97 497 497 14 15 DTAAAA SJBAAA AAAAxx 708 929 0 0 8 8 8 8 108 208 708 16 17 GBAAAA TJBAAA HHHHxx 46 930 0 2 6 6 6 46 46 46 46 12 13 UBAAAA UJBAAA OOOOxx 901 931 1 1 1 1 1 1 101 401 901 2 3 RIAAAA VJBAAA VVVVxx 416 932 0 0 6 16 6 16 16 416 416 12 13 AQAAAA WJBAAA AAAAxx 307 933 1 3 7 7 7 7 107 307 307 14 15 VLAAAA XJBAAA HHHHxx 166 934 0 2 6 6 6 66 166 166 166 12 13 KGAAAA YJBAAA OOOOxx 178 935 0 2 8 18 8 78 178 178 178 16 17 WGAAAA ZJBAAA VVVVxx 499 936 1 3 9 19 9 99 99 499 499 18 19 FTAAAA AKBAAA AAAAxx 257 937 1 1 7 17 7 57 57 257 257 14 15 XJAAAA BKBAAA HHHHxx 342 938 0 2 2 2 2 42 142 342 342 4 5 ENAAAA CKBAAA OOOOxx 850 939 0 2 0 10 0 50 50 350 850 0 1 SGAAAA DKBAAA VVVVxx 313 940 1 1 3 13 3 13 113 313 313 6 7 BMAAAA EKBAAA AAAAxx 831 941 1 3 1 11 1 31 31 331 831 2 3 ZFAAAA FKBAAA HHHHxx 57 942 1 1 7 17 7 57 57 57 57 14 15 FCAAAA GKBAAA OOOOxx 37 943 1 1 7 17 7 37 37 37 37 14 15 LBAAAA HKBAAA VVVVxx 511 944 1 3 1 11 1 11 111 11 511 2 3 RTAAAA IKBAAA AAAAxx 578 945 0 2 8 18 8 78 178 78 578 16 17 GWAAAA JKBAAA HHHHxx 100 946 0 0 0 0 0 0 100 100 100 0 1 WDAAAA KKBAAA OOOOxx 935 947 1 3 5 15 5 35 135 435 935 10 11 ZJAAAA LKBAAA VVVVxx 821 948 1 1 1 1 1 21 21 321 821 2 3 PFAAAA MKBAAA AAAAxx 294 949 0 2 4 14 4 94 94 294 294 8 9 ILAAAA NKBAAA HHHHxx 575 950 1 3 5 15 5 75 175 75 575 10 11 DWAAAA OKBAAA OOOOxx 272 951 0 0 2 12 2 72 72 272 272 4 5 MKAAAA PKBAAA VVVVxx 491 952 1 3 1 11 1 91 91 491 491 2 3 XSAAAA QKBAAA AAAAxx 43 953 1 3 3 3 3 43 43 43 43 6 7 RBAAAA RKBAAA HHHHxx 167 954 1 3 7 7 7 67 167 167 167 14 15 LGAAAA SKBAAA OOOOxx 457 955 1 1 7 17 7 57 57 457 457 14 15 PRAAAA TKBAAA VVVVxx 647 956 1 3 7 7 7 47 47 147 647 14 15 XYAAAA UKBAAA AAAAxx 180 957 0 0 0 0 0 80 180 180 180 0 1 YGAAAA VKBAAA HHHHxx 48 958 0 0 8 8 8 48 48 48 48 16 17 WBAAAA WKBAAA OOOOxx 553 959 1 1 3 13 3 53 153 53 553 6 7 HVAAAA XKBAAA VVVVxx 188 960 0 0 8 8 8 88 188 188 188 16 17 GHAAAA YKBAAA AAAAxx 262 961 0 2 2 2 2 62 62 262 262 4 5 CKAAAA ZKBAAA HHHHxx 728 962 0 0 8 8 8 28 128 228 728 16 17 ACAAAA ALBAAA OOOOxx 581 963 1 1 1 1 1 81 181 81 581 2 3 JWAAAA BLBAAA VVVVxx 937 964 1 1 7 17 7 37 137 437 937 14 15 BKAAAA CLBAAA AAAAxx 370 965 0 2 0 10 0 70 170 370 370 0 1 GOAAAA DLBAAA HHHHxx 590 966 0 2 0 10 0 90 190 90 590 0 1 SWAAAA ELBAAA OOOOxx 421 967 1 1 1 1 1 21 21 421 421 2 3 FQAAAA FLBAAA VVVVxx 693 968 1 1 3 13 3 93 93 193 693 6 7 RAAAAA GLBAAA AAAAxx 906 969 0 2 6 6 6 6 106 406 906 12 13 WIAAAA HLBAAA HHHHxx 802 970 0 2 2 2 2 2 2 302 802 4 5 WEAAAA ILBAAA OOOOxx 38 971 0 2 8 18 8 38 38 38 38 16 17 MBAAAA JLBAAA VVVVxx 790 972 0 2 0 10 0 90 190 290 790 0 1 KEAAAA KLBAAA AAAAxx 726 973 0 2 6 6 6 26 126 226 726 12 13 YBAAAA LLBAAA HHHHxx 23 974 1 3 3 3 3 23 23 23 23 6 7 XAAAAA MLBAAA OOOOxx 641 975 1 1 1 1 1 41 41 141 641 2 3 RYAAAA NLBAAA VVVVxx 524 976 0 0 4 4 4 24 124 24 524 8 9 EUAAAA OLBAAA AAAAxx 169 977 1 1 9 9 9 69 169 169 169 18 19 NGAAAA PLBAAA HHHHxx 6 978 0 2 6 6 6 6 6 6 6 12 13 GAAAAA QLBAAA OOOOxx 943 979 1 3 3 3 3 43 143 443 943 6 7 HKAAAA RLBAAA VVVVxx 26 980 0 2 6 6 6 26 26 26 26 12 13 ABAAAA SLBAAA AAAAxx 469 981 1 1 9 9 9 69 69 469 469 18 19 BSAAAA TLBAAA HHHHxx 968 982 0 0 8 8 8 68 168 468 968 16 17 GLAAAA ULBAAA OOOOxx 947 983 1 3 7 7 7 47 147 447 947 14 15 LKAAAA VLBAAA VVVVxx 133 984 1 1 3 13 3 33 133 133 133 6 7 DFAAAA WLBAAA AAAAxx 52 985 0 0 2 12 2 52 52 52 52 4 5 ACAAAA XLBAAA HHHHxx 660 986 0 0 0 0 0 60 60 160 660 0 1 KZAAAA YLBAAA OOOOxx 780 987 0 0 0 0 0 80 180 280 780 0 1 AEAAAA ZLBAAA VVVVxx 963 988 1 3 3 3 3 63 163 463 963 6 7 BLAAAA AMBAAA AAAAxx 561 989 1 1 1 1 1 61 161 61 561 2 3 PVAAAA BMBAAA HHHHxx 402 990 0 2 2 2 2 2 2 402 402 4 5 MPAAAA CMBAAA OOOOxx 437 991 1 1 7 17 7 37 37 437 437 14 15 VQAAAA DMBAAA VVVVxx 112 992 0 0 2 12 2 12 112 112 112 4 5 IEAAAA EMBAAA AAAAxx 247 993 1 3 7 7 7 47 47 247 247 14 15 NJAAAA FMBAAA HHHHxx 579 994 1 3 9 19 9 79 179 79 579 18 19 HWAAAA GMBAAA OOOOxx 379 995 1 3 9 19 9 79 179 379 379 18 19 POAAAA HMBAAA VVVVxx 74 996 0 2 4 14 4 74 74 74 74 8 9 WCAAAA IMBAAA AAAAxx 744 997 0 0 4 4 4 44 144 244 744 8 9 QCAAAA JMBAAA HHHHxx 0 998 0 0 0 0 0 0 0 0 0 0 1 AAAAAA KMBAAA OOOOxx 278 999 0 2 8 18 8 78 78 278 278 16 17 SKAAAA LMBAAA VVVVxx ================================================ FILE: test/sql/data/tenk.data ================================================ 8800 0 0 0 0 0 0 800 800 3800 8800 0 1 MAAAAA AAAAAA AAAAxx 1891 1 1 3 1 11 91 891 1891 1891 1891 182 183 TUAAAA BAAAAA HHHHxx 3420 2 0 0 0 0 20 420 1420 3420 3420 40 41 OBAAAA CAAAAA OOOOxx 9850 3 0 2 0 10 50 850 1850 4850 9850 100 101 WOAAAA DAAAAA VVVVxx 7164 4 0 0 4 4 64 164 1164 2164 7164 128 129 OPAAAA EAAAAA AAAAxx 8009 5 1 1 9 9 9 9 9 3009 8009 18 19 BWAAAA FAAAAA HHHHxx 5057 6 1 1 7 17 57 57 1057 57 5057 114 115 NMAAAA GAAAAA OOOOxx 6701 7 1 1 1 1 1 701 701 1701 6701 2 3 TXAAAA HAAAAA VVVVxx 4321 8 1 1 1 1 21 321 321 4321 4321 42 43 FKAAAA IAAAAA AAAAxx 3043 9 1 3 3 3 43 43 1043 3043 3043 86 87 BNAAAA JAAAAA HHHHxx 1314 10 0 2 4 14 14 314 1314 1314 1314 28 29 OYAAAA KAAAAA OOOOxx 1504 11 0 0 4 4 4 504 1504 1504 1504 8 9 WFAAAA LAAAAA VVVVxx 5222 12 0 2 2 2 22 222 1222 222 5222 44 45 WSAAAA MAAAAA AAAAxx 6243 13 1 3 3 3 43 243 243 1243 6243 86 87 DGAAAA NAAAAA HHHHxx 5471 14 1 3 1 11 71 471 1471 471 5471 142 143 LCAAAA OAAAAA OOOOxx 5006 15 0 2 6 6 6 6 1006 6 5006 12 13 OKAAAA PAAAAA VVVVxx 5387 16 1 3 7 7 87 387 1387 387 5387 174 175 FZAAAA QAAAAA AAAAxx 5785 17 1 1 5 5 85 785 1785 785 5785 170 171 NOAAAA RAAAAA HHHHxx 6621 18 1 1 1 1 21 621 621 1621 6621 42 43 RUAAAA SAAAAA OOOOxx 6969 19 1 1 9 9 69 969 969 1969 6969 138 139 BIAAAA TAAAAA VVVVxx 9460 20 0 0 0 0 60 460 1460 4460 9460 120 121 WZAAAA UAAAAA AAAAxx 59 21 1 3 9 19 59 59 59 59 59 118 119 HCAAAA VAAAAA HHHHxx 8020 22 0 0 0 0 20 20 20 3020 8020 40 41 MWAAAA WAAAAA OOOOxx 7695 23 1 3 5 15 95 695 1695 2695 7695 190 191 ZJAAAA XAAAAA VVVVxx 3442 24 0 2 2 2 42 442 1442 3442 3442 84 85 KCAAAA YAAAAA AAAAxx 5119 25 1 3 9 19 19 119 1119 119 5119 38 39 XOAAAA ZAAAAA HHHHxx 646 26 0 2 6 6 46 646 646 646 646 92 93 WYAAAA ABAAAA OOOOxx 9605 27 1 1 5 5 5 605 1605 4605 9605 10 11 LFAAAA BBAAAA VVVVxx 263 28 1 3 3 3 63 263 263 263 263 126 127 DKAAAA CBAAAA AAAAxx 3269 29 1 1 9 9 69 269 1269 3269 3269 138 139 TVAAAA DBAAAA HHHHxx 1839 30 1 3 9 19 39 839 1839 1839 1839 78 79 TSAAAA EBAAAA OOOOxx 9144 31 0 0 4 4 44 144 1144 4144 9144 88 89 SNAAAA FBAAAA VVVVxx 2513 32 1 1 3 13 13 513 513 2513 2513 26 27 RSAAAA GBAAAA AAAAxx 8850 33 0 2 0 10 50 850 850 3850 8850 100 101 KCAAAA HBAAAA HHHHxx 236 34 0 0 6 16 36 236 236 236 236 72 73 CJAAAA IBAAAA OOOOxx 3162 35 0 2 2 2 62 162 1162 3162 3162 124 125 QRAAAA JBAAAA VVVVxx 4380 36 0 0 0 0 80 380 380 4380 4380 160 161 MMAAAA KBAAAA AAAAxx 8095 37 1 3 5 15 95 95 95 3095 8095 190 191 JZAAAA LBAAAA HHHHxx 209 38 1 1 9 9 9 209 209 209 209 18 19 BIAAAA MBAAAA OOOOxx 3055 39 1 3 5 15 55 55 1055 3055 3055 110 111 NNAAAA NBAAAA VVVVxx 6921 40 1 1 1 1 21 921 921 1921 6921 42 43 FGAAAA OBAAAA AAAAxx 7046 41 0 2 6 6 46 46 1046 2046 7046 92 93 ALAAAA PBAAAA HHHHxx 7912 42 0 0 2 12 12 912 1912 2912 7912 24 25 ISAAAA QBAAAA OOOOxx 7267 43 1 3 7 7 67 267 1267 2267 7267 134 135 NTAAAA RBAAAA VVVVxx 3599 44 1 3 9 19 99 599 1599 3599 3599 198 199 LIAAAA SBAAAA AAAAxx 923 45 1 3 3 3 23 923 923 923 923 46 47 NJAAAA TBAAAA HHHHxx 1437 46 1 1 7 17 37 437 1437 1437 1437 74 75 HDAAAA UBAAAA OOOOxx 6439 47 1 3 9 19 39 439 439 1439 6439 78 79 RNAAAA VBAAAA VVVVxx 6989 48 1 1 9 9 89 989 989 1989 6989 178 179 VIAAAA WBAAAA AAAAxx 8798 49 0 2 8 18 98 798 798 3798 8798 196 197 KAAAAA XBAAAA HHHHxx 5960 50 0 0 0 0 60 960 1960 960 5960 120 121 GVAAAA YBAAAA OOOOxx 5832 51 0 0 2 12 32 832 1832 832 5832 64 65 IQAAAA ZBAAAA VVVVxx 6066 52 0 2 6 6 66 66 66 1066 6066 132 133 IZAAAA ACAAAA AAAAxx 322 53 0 2 2 2 22 322 322 322 322 44 45 KMAAAA BCAAAA HHHHxx 8321 54 1 1 1 1 21 321 321 3321 8321 42 43 BIAAAA CCAAAA OOOOxx 734 55 0 2 4 14 34 734 734 734 734 68 69 GCAAAA DCAAAA VVVVxx 688 56 0 0 8 8 88 688 688 688 688 176 177 MAAAAA ECAAAA AAAAxx 4212 57 0 0 2 12 12 212 212 4212 4212 24 25 AGAAAA FCAAAA HHHHxx 9653 58 1 1 3 13 53 653 1653 4653 9653 106 107 HHAAAA GCAAAA OOOOxx 2677 59 1 1 7 17 77 677 677 2677 2677 154 155 ZYAAAA HCAAAA VVVVxx 5423 60 1 3 3 3 23 423 1423 423 5423 46 47 PAAAAA ICAAAA AAAAxx 2592 61 0 0 2 12 92 592 592 2592 2592 184 185 SVAAAA JCAAAA HHHHxx 3233 62 1 1 3 13 33 233 1233 3233 3233 66 67 JUAAAA KCAAAA OOOOxx 5032 63 0 0 2 12 32 32 1032 32 5032 64 65 OLAAAA LCAAAA VVVVxx 2525 64 1 1 5 5 25 525 525 2525 2525 50 51 DTAAAA MCAAAA AAAAxx 4450 65 0 2 0 10 50 450 450 4450 4450 100 101 EPAAAA NCAAAA HHHHxx 5778 66 0 2 8 18 78 778 1778 778 5778 156 157 GOAAAA OCAAAA OOOOxx 5852 67 0 0 2 12 52 852 1852 852 5852 104 105 CRAAAA PCAAAA VVVVxx 5404 68 0 0 4 4 4 404 1404 404 5404 8 9 WZAAAA QCAAAA AAAAxx 6223 69 1 3 3 3 23 223 223 1223 6223 46 47 JFAAAA RCAAAA HHHHxx 6133 70 1 1 3 13 33 133 133 1133 6133 66 67 XBAAAA SCAAAA OOOOxx 9112 71 0 0 2 12 12 112 1112 4112 9112 24 25 MMAAAA TCAAAA VVVVxx 7575 72 1 3 5 15 75 575 1575 2575 7575 150 151 JFAAAA UCAAAA AAAAxx 7414 73 0 2 4 14 14 414 1414 2414 7414 28 29 EZAAAA VCAAAA HHHHxx 9741 74 1 1 1 1 41 741 1741 4741 9741 82 83 RKAAAA WCAAAA OOOOxx 3767 75 1 3 7 7 67 767 1767 3767 3767 134 135 XOAAAA XCAAAA VVVVxx 9372 76 0 0 2 12 72 372 1372 4372 9372 144 145 MWAAAA YCAAAA AAAAxx 8976 77 0 0 6 16 76 976 976 3976 8976 152 153 GHAAAA ZCAAAA HHHHxx 4071 78 1 3 1 11 71 71 71 4071 4071 142 143 PAAAAA ADAAAA OOOOxx 1311 79 1 3 1 11 11 311 1311 1311 1311 22 23 LYAAAA BDAAAA VVVVxx 2604 80 0 0 4 4 4 604 604 2604 2604 8 9 EWAAAA CDAAAA AAAAxx 8840 81 0 0 0 0 40 840 840 3840 8840 80 81 ACAAAA DDAAAA HHHHxx 567 82 1 3 7 7 67 567 567 567 567 134 135 VVAAAA EDAAAA OOOOxx 5215 83 1 3 5 15 15 215 1215 215 5215 30 31 PSAAAA FDAAAA VVVVxx 5474 84 0 2 4 14 74 474 1474 474 5474 148 149 OCAAAA GDAAAA AAAAxx 3906 85 0 2 6 6 6 906 1906 3906 3906 12 13 GUAAAA HDAAAA HHHHxx 1769 86 1 1 9 9 69 769 1769 1769 1769 138 139 BQAAAA IDAAAA OOOOxx 1454 87 0 2 4 14 54 454 1454 1454 1454 108 109 YDAAAA JDAAAA VVVVxx 6877 88 1 1 7 17 77 877 877 1877 6877 154 155 NEAAAA KDAAAA AAAAxx 6501 89 1 1 1 1 1 501 501 1501 6501 2 3 BQAAAA LDAAAA HHHHxx 934 90 0 2 4 14 34 934 934 934 934 68 69 YJAAAA MDAAAA OOOOxx 4075 91 1 3 5 15 75 75 75 4075 4075 150 151 TAAAAA NDAAAA VVVVxx 3180 92 0 0 0 0 80 180 1180 3180 3180 160 161 ISAAAA ODAAAA AAAAxx 7787 93 1 3 7 7 87 787 1787 2787 7787 174 175 NNAAAA PDAAAA HHHHxx 6401 94 1 1 1 1 1 401 401 1401 6401 2 3 FMAAAA QDAAAA OOOOxx 4244 95 0 0 4 4 44 244 244 4244 4244 88 89 GHAAAA RDAAAA VVVVxx 4591 96 1 3 1 11 91 591 591 4591 4591 182 183 PUAAAA SDAAAA AAAAxx 4113 97 1 1 3 13 13 113 113 4113 4113 26 27 FCAAAA TDAAAA HHHHxx 5925 98 1 1 5 5 25 925 1925 925 5925 50 51 XTAAAA UDAAAA OOOOxx 1987 99 1 3 7 7 87 987 1987 1987 1987 174 175 LYAAAA VDAAAA VVVVxx 8248 100 0 0 8 8 48 248 248 3248 8248 96 97 GFAAAA WDAAAA AAAAxx 4151 101 1 3 1 11 51 151 151 4151 4151 102 103 RDAAAA XDAAAA HHHHxx 8670 102 0 2 0 10 70 670 670 3670 8670 140 141 MVAAAA YDAAAA OOOOxx 6194 103 0 2 4 14 94 194 194 1194 6194 188 189 GEAAAA ZDAAAA VVVVxx 88 104 0 0 8 8 88 88 88 88 88 176 177 KDAAAA AEAAAA AAAAxx 4058 105 0 2 8 18 58 58 58 4058 4058 116 117 CAAAAA BEAAAA HHHHxx 2742 106 0 2 2 2 42 742 742 2742 2742 84 85 MBAAAA CEAAAA OOOOxx 8275 107 1 3 5 15 75 275 275 3275 8275 150 151 HGAAAA DEAAAA VVVVxx 4258 108 0 2 8 18 58 258 258 4258 4258 116 117 UHAAAA EEAAAA AAAAxx 6129 109 1 1 9 9 29 129 129 1129 6129 58 59 TBAAAA FEAAAA HHHHxx 7243 110 1 3 3 3 43 243 1243 2243 7243 86 87 PSAAAA GEAAAA OOOOxx 2392 111 0 0 2 12 92 392 392 2392 2392 184 185 AOAAAA HEAAAA VVVVxx 9853 112 1 1 3 13 53 853 1853 4853 9853 106 107 ZOAAAA IEAAAA AAAAxx 6064 113 0 0 4 4 64 64 64 1064 6064 128 129 GZAAAA JEAAAA HHHHxx 4391 114 1 3 1 11 91 391 391 4391 4391 182 183 XMAAAA KEAAAA OOOOxx 726 115 0 2 6 6 26 726 726 726 726 52 53 YBAAAA LEAAAA VVVVxx 6957 116 1 1 7 17 57 957 957 1957 6957 114 115 PHAAAA MEAAAA AAAAxx 3853 117 1 1 3 13 53 853 1853 3853 3853 106 107 FSAAAA NEAAAA HHHHxx 4524 118 0 0 4 4 24 524 524 4524 4524 48 49 ASAAAA OEAAAA OOOOxx 5330 119 0 2 0 10 30 330 1330 330 5330 60 61 AXAAAA PEAAAA VVVVxx 6671 120 1 3 1 11 71 671 671 1671 6671 142 143 PWAAAA QEAAAA AAAAxx 5314 121 0 2 4 14 14 314 1314 314 5314 28 29 KWAAAA REAAAA HHHHxx 9202 122 0 2 2 2 2 202 1202 4202 9202 4 5 YPAAAA SEAAAA OOOOxx 4596 123 0 0 6 16 96 596 596 4596 4596 192 193 UUAAAA TEAAAA VVVVxx 8951 124 1 3 1 11 51 951 951 3951 8951 102 103 HGAAAA UEAAAA AAAAxx 9902 125 0 2 2 2 2 902 1902 4902 9902 4 5 WQAAAA VEAAAA HHHHxx 1440 126 0 0 0 0 40 440 1440 1440 1440 80 81 KDAAAA WEAAAA OOOOxx 5339 127 1 3 9 19 39 339 1339 339 5339 78 79 JXAAAA XEAAAA VVVVxx 3371 128 1 3 1 11 71 371 1371 3371 3371 142 143 RZAAAA YEAAAA AAAAxx 4467 129 1 3 7 7 67 467 467 4467 4467 134 135 VPAAAA ZEAAAA HHHHxx 6216 130 0 0 6 16 16 216 216 1216 6216 32 33 CFAAAA AFAAAA OOOOxx 5364 131 0 0 4 4 64 364 1364 364 5364 128 129 IYAAAA BFAAAA VVVVxx 7547 132 1 3 7 7 47 547 1547 2547 7547 94 95 HEAAAA CFAAAA AAAAxx 4338 133 0 2 8 18 38 338 338 4338 4338 76 77 WKAAAA DFAAAA HHHHxx 3481 134 1 1 1 1 81 481 1481 3481 3481 162 163 XDAAAA EFAAAA OOOOxx 826 135 0 2 6 6 26 826 826 826 826 52 53 UFAAAA FFAAAA VVVVxx 3647 136 1 3 7 7 47 647 1647 3647 3647 94 95 HKAAAA GFAAAA AAAAxx 3337 137 1 1 7 17 37 337 1337 3337 3337 74 75 JYAAAA HFAAAA HHHHxx 3591 138 1 3 1 11 91 591 1591 3591 3591 182 183 DIAAAA IFAAAA OOOOxx 7192 139 0 0 2 12 92 192 1192 2192 7192 184 185 QQAAAA JFAAAA VVVVxx 1078 140 0 2 8 18 78 78 1078 1078 1078 156 157 MPAAAA KFAAAA AAAAxx 1310 141 0 2 0 10 10 310 1310 1310 1310 20 21 KYAAAA LFAAAA HHHHxx 9642 142 0 2 2 2 42 642 1642 4642 9642 84 85 WGAAAA MFAAAA OOOOxx 39 143 1 3 9 19 39 39 39 39 39 78 79 NBAAAA NFAAAA VVVVxx 8682 144 0 2 2 2 82 682 682 3682 8682 164 165 YVAAAA OFAAAA AAAAxx 1794 145 0 2 4 14 94 794 1794 1794 1794 188 189 ARAAAA PFAAAA HHHHxx 5630 146 0 2 0 10 30 630 1630 630 5630 60 61 OIAAAA QFAAAA OOOOxx 6748 147 0 0 8 8 48 748 748 1748 6748 96 97 OZAAAA RFAAAA VVVVxx 3766 148 0 2 6 6 66 766 1766 3766 3766 132 133 WOAAAA SFAAAA AAAAxx 6403 149 1 3 3 3 3 403 403 1403 6403 6 7 HMAAAA TFAAAA HHHHxx 175 150 1 3 5 15 75 175 175 175 175 150 151 TGAAAA UFAAAA OOOOxx 2179 151 1 3 9 19 79 179 179 2179 2179 158 159 VFAAAA VFAAAA VVVVxx 7897 152 1 1 7 17 97 897 1897 2897 7897 194 195 TRAAAA WFAAAA AAAAxx 2760 153 0 0 0 0 60 760 760 2760 2760 120 121 ECAAAA XFAAAA HHHHxx 1675 154 1 3 5 15 75 675 1675 1675 1675 150 151 LMAAAA YFAAAA OOOOxx 2564 155 0 0 4 4 64 564 564 2564 2564 128 129 QUAAAA ZFAAAA VVVVxx 157 156 1 1 7 17 57 157 157 157 157 114 115 BGAAAA AGAAAA AAAAxx 8779 157 1 3 9 19 79 779 779 3779 8779 158 159 RZAAAA BGAAAA HHHHxx 9591 158 1 3 1 11 91 591 1591 4591 9591 182 183 XEAAAA CGAAAA OOOOxx 8732 159 0 0 2 12 32 732 732 3732 8732 64 65 WXAAAA DGAAAA VVVVxx 139 160 1 3 9 19 39 139 139 139 139 78 79 JFAAAA EGAAAA AAAAxx 5372 161 0 0 2 12 72 372 1372 372 5372 144 145 QYAAAA FGAAAA HHHHxx 1278 162 0 2 8 18 78 278 1278 1278 1278 156 157 EXAAAA GGAAAA OOOOxx 4697 163 1 1 7 17 97 697 697 4697 4697 194 195 RYAAAA HGAAAA VVVVxx 8610 164 0 2 0 10 10 610 610 3610 8610 20 21 ETAAAA IGAAAA AAAAxx 8180 165 0 0 0 0 80 180 180 3180 8180 160 161 QCAAAA JGAAAA HHHHxx 2399 166 1 3 9 19 99 399 399 2399 2399 198 199 HOAAAA KGAAAA OOOOxx 615 167 1 3 5 15 15 615 615 615 615 30 31 RXAAAA LGAAAA VVVVxx 7629 168 1 1 9 9 29 629 1629 2629 7629 58 59 LHAAAA MGAAAA AAAAxx 7628 169 0 0 8 8 28 628 1628 2628 7628 56 57 KHAAAA NGAAAA HHHHxx 4659 170 1 3 9 19 59 659 659 4659 4659 118 119 FXAAAA OGAAAA OOOOxx 5865 171 1 1 5 5 65 865 1865 865 5865 130 131 PRAAAA PGAAAA VVVVxx 3973 172 1 1 3 13 73 973 1973 3973 3973 146 147 VWAAAA QGAAAA AAAAxx 552 173 0 0 2 12 52 552 552 552 552 104 105 GVAAAA RGAAAA HHHHxx 708 174 0 0 8 8 8 708 708 708 708 16 17 GBAAAA SGAAAA OOOOxx 3550 175 0 2 0 10 50 550 1550 3550 3550 100 101 OGAAAA TGAAAA VVVVxx 5547 176 1 3 7 7 47 547 1547 547 5547 94 95 JFAAAA UGAAAA AAAAxx 489 177 1 1 9 9 89 489 489 489 489 178 179 VSAAAA VGAAAA HHHHxx 3794 178 0 2 4 14 94 794 1794 3794 3794 188 189 YPAAAA WGAAAA OOOOxx 9479 179 1 3 9 19 79 479 1479 4479 9479 158 159 PAAAAA XGAAAA VVVVxx 6435 180 1 3 5 15 35 435 435 1435 6435 70 71 NNAAAA YGAAAA AAAAxx 5120 181 0 0 0 0 20 120 1120 120 5120 40 41 YOAAAA ZGAAAA HHHHxx 3615 182 1 3 5 15 15 615 1615 3615 3615 30 31 BJAAAA AHAAAA OOOOxx 8399 183 1 3 9 19 99 399 399 3399 8399 198 199 BLAAAA BHAAAA VVVVxx 2155 184 1 3 5 15 55 155 155 2155 2155 110 111 XEAAAA CHAAAA AAAAxx 6690 185 0 2 0 10 90 690 690 1690 6690 180 181 IXAAAA DHAAAA HHHHxx 1683 186 1 3 3 3 83 683 1683 1683 1683 166 167 TMAAAA EHAAAA OOOOxx 6302 187 0 2 2 2 2 302 302 1302 6302 4 5 KIAAAA FHAAAA VVVVxx 516 188 0 0 6 16 16 516 516 516 516 32 33 WTAAAA GHAAAA AAAAxx 3901 189 1 1 1 1 1 901 1901 3901 3901 2 3 BUAAAA HHAAAA HHHHxx 6938 190 0 2 8 18 38 938 938 1938 6938 76 77 WGAAAA IHAAAA OOOOxx 7484 191 0 0 4 4 84 484 1484 2484 7484 168 169 WBAAAA JHAAAA VVVVxx 7424 192 0 0 4 4 24 424 1424 2424 7424 48 49 OZAAAA KHAAAA AAAAxx 9410 193 0 2 0 10 10 410 1410 4410 9410 20 21 YXAAAA LHAAAA HHHHxx 1714 194 0 2 4 14 14 714 1714 1714 1714 28 29 YNAAAA MHAAAA OOOOxx 8278 195 0 2 8 18 78 278 278 3278 8278 156 157 KGAAAA NHAAAA VVVVxx 3158 196 0 2 8 18 58 158 1158 3158 3158 116 117 MRAAAA OHAAAA AAAAxx 2511 197 1 3 1 11 11 511 511 2511 2511 22 23 PSAAAA PHAAAA HHHHxx 2912 198 0 0 2 12 12 912 912 2912 2912 24 25 AIAAAA QHAAAA OOOOxx 2648 199 0 0 8 8 48 648 648 2648 2648 96 97 WXAAAA RHAAAA VVVVxx 9385 200 1 1 5 5 85 385 1385 4385 9385 170 171 ZWAAAA SHAAAA AAAAxx 7545 201 1 1 5 5 45 545 1545 2545 7545 90 91 FEAAAA THAAAA HHHHxx 8407 202 1 3 7 7 7 407 407 3407 8407 14 15 JLAAAA UHAAAA OOOOxx 5893 203 1 1 3 13 93 893 1893 893 5893 186 187 RSAAAA VHAAAA VVVVxx 7049 204 1 1 9 9 49 49 1049 2049 7049 98 99 DLAAAA WHAAAA AAAAxx 6812 205 0 0 2 12 12 812 812 1812 6812 24 25 ACAAAA XHAAAA HHHHxx 3649 206 1 1 9 9 49 649 1649 3649 3649 98 99 JKAAAA YHAAAA OOOOxx 9275 207 1 3 5 15 75 275 1275 4275 9275 150 151 TSAAAA ZHAAAA VVVVxx 1179 208 1 3 9 19 79 179 1179 1179 1179 158 159 JTAAAA AIAAAA AAAAxx 969 209 1 1 9 9 69 969 969 969 969 138 139 HLAAAA BIAAAA HHHHxx 7920 210 0 0 0 0 20 920 1920 2920 7920 40 41 QSAAAA CIAAAA OOOOxx 998 211 0 2 8 18 98 998 998 998 998 196 197 KMAAAA DIAAAA VVVVxx 3958 212 0 2 8 18 58 958 1958 3958 3958 116 117 GWAAAA EIAAAA AAAAxx 6052 213 0 0 2 12 52 52 52 1052 6052 104 105 UYAAAA FIAAAA HHHHxx 8791 214 1 3 1 11 91 791 791 3791 8791 182 183 DAAAAA GIAAAA OOOOxx 5191 215 1 3 1 11 91 191 1191 191 5191 182 183 RRAAAA HIAAAA VVVVxx 4267 216 1 3 7 7 67 267 267 4267 4267 134 135 DIAAAA IIAAAA AAAAxx 2829 217 1 1 9 9 29 829 829 2829 2829 58 59 VEAAAA JIAAAA HHHHxx 6396 218 0 0 6 16 96 396 396 1396 6396 192 193 AMAAAA KIAAAA OOOOxx 9413 219 1 1 3 13 13 413 1413 4413 9413 26 27 BYAAAA LIAAAA VVVVxx 614 220 0 2 4 14 14 614 614 614 614 28 29 QXAAAA MIAAAA AAAAxx 4660 221 0 0 0 0 60 660 660 4660 4660 120 121 GXAAAA NIAAAA HHHHxx 8834 222 0 2 4 14 34 834 834 3834 8834 68 69 UBAAAA OIAAAA OOOOxx 2767 223 1 3 7 7 67 767 767 2767 2767 134 135 LCAAAA PIAAAA VVVVxx 2444 224 0 0 4 4 44 444 444 2444 2444 88 89 AQAAAA QIAAAA AAAAxx 4129 225 1 1 9 9 29 129 129 4129 4129 58 59 VCAAAA RIAAAA HHHHxx 3394 226 0 2 4 14 94 394 1394 3394 3394 188 189 OAAAAA SIAAAA OOOOxx 2705 227 1 1 5 5 5 705 705 2705 2705 10 11 BAAAAA TIAAAA VVVVxx 8499 228 1 3 9 19 99 499 499 3499 8499 198 199 XOAAAA UIAAAA AAAAxx 8852 229 0 0 2 12 52 852 852 3852 8852 104 105 MCAAAA VIAAAA HHHHxx 6174 230 0 2 4 14 74 174 174 1174 6174 148 149 MDAAAA WIAAAA OOOOxx 750 231 0 2 0 10 50 750 750 750 750 100 101 WCAAAA XIAAAA VVVVxx 8164 232 0 0 4 4 64 164 164 3164 8164 128 129 ACAAAA YIAAAA AAAAxx 4930 233 0 2 0 10 30 930 930 4930 4930 60 61 QHAAAA ZIAAAA HHHHxx 9904 234 0 0 4 4 4 904 1904 4904 9904 8 9 YQAAAA AJAAAA OOOOxx 7378 235 0 2 8 18 78 378 1378 2378 7378 156 157 UXAAAA BJAAAA VVVVxx 2927 236 1 3 7 7 27 927 927 2927 2927 54 55 PIAAAA CJAAAA AAAAxx 7155 237 1 3 5 15 55 155 1155 2155 7155 110 111 FPAAAA DJAAAA HHHHxx 1302 238 0 2 2 2 2 302 1302 1302 1302 4 5 CYAAAA EJAAAA OOOOxx 5904 239 0 0 4 4 4 904 1904 904 5904 8 9 CTAAAA FJAAAA VVVVxx 9687 240 1 3 7 7 87 687 1687 4687 9687 174 175 PIAAAA GJAAAA AAAAxx 3553 241 1 1 3 13 53 553 1553 3553 3553 106 107 RGAAAA HJAAAA HHHHxx 4447 242 1 3 7 7 47 447 447 4447 4447 94 95 BPAAAA IJAAAA OOOOxx 6878 243 0 2 8 18 78 878 878 1878 6878 156 157 OEAAAA JJAAAA VVVVxx 9470 244 0 2 0 10 70 470 1470 4470 9470 140 141 GAAAAA KJAAAA AAAAxx 9735 245 1 3 5 15 35 735 1735 4735 9735 70 71 LKAAAA LJAAAA HHHHxx 5967 246 1 3 7 7 67 967 1967 967 5967 134 135 NVAAAA MJAAAA OOOOxx 6601 247 1 1 1 1 1 601 601 1601 6601 2 3 XTAAAA NJAAAA VVVVxx 7631 248 1 3 1 11 31 631 1631 2631 7631 62 63 NHAAAA OJAAAA AAAAxx 3559 249 1 3 9 19 59 559 1559 3559 3559 118 119 XGAAAA PJAAAA HHHHxx 2247 250 1 3 7 7 47 247 247 2247 2247 94 95 LIAAAA QJAAAA OOOOxx 9649 251 1 1 9 9 49 649 1649 4649 9649 98 99 DHAAAA RJAAAA VVVVxx 808 252 0 0 8 8 8 808 808 808 808 16 17 CFAAAA SJAAAA AAAAxx 240 253 0 0 0 0 40 240 240 240 240 80 81 GJAAAA TJAAAA HHHHxx 5031 254 1 3 1 11 31 31 1031 31 5031 62 63 NLAAAA UJAAAA OOOOxx 9563 255 1 3 3 3 63 563 1563 4563 9563 126 127 VDAAAA VJAAAA VVVVxx 5656 256 0 0 6 16 56 656 1656 656 5656 112 113 OJAAAA WJAAAA AAAAxx 3886 257 0 2 6 6 86 886 1886 3886 3886 172 173 MTAAAA XJAAAA HHHHxx 2431 258 1 3 1 11 31 431 431 2431 2431 62 63 NPAAAA YJAAAA OOOOxx 5560 259 0 0 0 0 60 560 1560 560 5560 120 121 WFAAAA ZJAAAA VVVVxx 9065 260 1 1 5 5 65 65 1065 4065 9065 130 131 RKAAAA AKAAAA AAAAxx 8130 261 0 2 0 10 30 130 130 3130 8130 60 61 SAAAAA BKAAAA HHHHxx 4054 262 0 2 4 14 54 54 54 4054 4054 108 109 YZAAAA CKAAAA OOOOxx 873 263 1 1 3 13 73 873 873 873 873 146 147 PHAAAA DKAAAA VVVVxx 3092 264 0 0 2 12 92 92 1092 3092 3092 184 185 YOAAAA EKAAAA AAAAxx 6697 265 1 1 7 17 97 697 697 1697 6697 194 195 PXAAAA FKAAAA HHHHxx 2452 266 0 0 2 12 52 452 452 2452 2452 104 105 IQAAAA GKAAAA OOOOxx 7867 267 1 3 7 7 67 867 1867 2867 7867 134 135 PQAAAA HKAAAA VVVVxx 3753 268 1 1 3 13 53 753 1753 3753 3753 106 107 JOAAAA IKAAAA AAAAxx 7834 269 0 2 4 14 34 834 1834 2834 7834 68 69 IPAAAA JKAAAA HHHHxx 5846 270 0 2 6 6 46 846 1846 846 5846 92 93 WQAAAA KKAAAA OOOOxx 7604 271 0 0 4 4 4 604 1604 2604 7604 8 9 MGAAAA LKAAAA VVVVxx 3452 272 0 0 2 12 52 452 1452 3452 3452 104 105 UCAAAA MKAAAA AAAAxx 4788 273 0 0 8 8 88 788 788 4788 4788 176 177 ECAAAA NKAAAA HHHHxx 8600 274 0 0 0 0 0 600 600 3600 8600 0 1 USAAAA OKAAAA OOOOxx 8511 275 1 3 1 11 11 511 511 3511 8511 22 23 JPAAAA PKAAAA VVVVxx 4452 276 0 0 2 12 52 452 452 4452 4452 104 105 GPAAAA QKAAAA AAAAxx 1709 277 1 1 9 9 9 709 1709 1709 1709 18 19 TNAAAA RKAAAA HHHHxx 3440 278 0 0 0 0 40 440 1440 3440 3440 80 81 ICAAAA SKAAAA OOOOxx 9188 279 0 0 8 8 88 188 1188 4188 9188 176 177 KPAAAA TKAAAA VVVVxx 3058 280 0 2 8 18 58 58 1058 3058 3058 116 117 QNAAAA UKAAAA AAAAxx 5821 281 1 1 1 1 21 821 1821 821 5821 42 43 XPAAAA VKAAAA HHHHxx 3428 282 0 0 8 8 28 428 1428 3428 3428 56 57 WBAAAA WKAAAA OOOOxx 3581 283 1 1 1 1 81 581 1581 3581 3581 162 163 THAAAA XKAAAA VVVVxx 7523 284 1 3 3 3 23 523 1523 2523 7523 46 47 JDAAAA YKAAAA AAAAxx 3131 285 1 3 1 11 31 131 1131 3131 3131 62 63 LQAAAA ZKAAAA HHHHxx 2404 286 0 0 4 4 4 404 404 2404 2404 8 9 MOAAAA ALAAAA OOOOxx 5453 287 1 1 3 13 53 453 1453 453 5453 106 107 TBAAAA BLAAAA VVVVxx 1599 288 1 3 9 19 99 599 1599 1599 1599 198 199 NJAAAA CLAAAA AAAAxx 7081 289 1 1 1 1 81 81 1081 2081 7081 162 163 JMAAAA DLAAAA HHHHxx 1750 290 0 2 0 10 50 750 1750 1750 1750 100 101 IPAAAA ELAAAA OOOOxx 5085 291 1 1 5 5 85 85 1085 85 5085 170 171 PNAAAA FLAAAA VVVVxx 9777 292 1 1 7 17 77 777 1777 4777 9777 154 155 BMAAAA GLAAAA AAAAxx 574 293 0 2 4 14 74 574 574 574 574 148 149 CWAAAA HLAAAA HHHHxx 5984 294 0 0 4 4 84 984 1984 984 5984 168 169 EWAAAA ILAAAA OOOOxx 7039 295 1 3 9 19 39 39 1039 2039 7039 78 79 TKAAAA JLAAAA VVVVxx 7143 296 1 3 3 3 43 143 1143 2143 7143 86 87 TOAAAA KLAAAA AAAAxx 5702 297 0 2 2 2 2 702 1702 702 5702 4 5 ILAAAA LLAAAA HHHHxx 362 298 0 2 2 2 62 362 362 362 362 124 125 YNAAAA MLAAAA OOOOxx 6997 299 1 1 7 17 97 997 997 1997 6997 194 195 DJAAAA NLAAAA VVVVxx 2529 300 1 1 9 9 29 529 529 2529 2529 58 59 HTAAAA OLAAAA AAAAxx 6319 301 1 3 9 19 19 319 319 1319 6319 38 39 BJAAAA PLAAAA HHHHxx 954 302 0 2 4 14 54 954 954 954 954 108 109 SKAAAA QLAAAA OOOOxx 3413 303 1 1 3 13 13 413 1413 3413 3413 26 27 HBAAAA RLAAAA VVVVxx 9081 304 1 1 1 1 81 81 1081 4081 9081 162 163 HLAAAA SLAAAA AAAAxx 5599 305 1 3 9 19 99 599 1599 599 5599 198 199 JHAAAA TLAAAA HHHHxx 4772 306 0 0 2 12 72 772 772 4772 4772 144 145 OBAAAA ULAAAA OOOOxx 1124 307 0 0 4 4 24 124 1124 1124 1124 48 49 GRAAAA VLAAAA VVVVxx 7793 308 1 1 3 13 93 793 1793 2793 7793 186 187 TNAAAA WLAAAA AAAAxx 4201 309 1 1 1 1 1 201 201 4201 4201 2 3 PFAAAA XLAAAA HHHHxx 7015 310 1 3 5 15 15 15 1015 2015 7015 30 31 VJAAAA YLAAAA OOOOxx 5936 311 0 0 6 16 36 936 1936 936 5936 72 73 IUAAAA ZLAAAA VVVVxx 4625 312 1 1 5 5 25 625 625 4625 4625 50 51 XVAAAA AMAAAA AAAAxx 4989 313 1 1 9 9 89 989 989 4989 4989 178 179 XJAAAA BMAAAA HHHHxx 4949 314 1 1 9 9 49 949 949 4949 4949 98 99 JIAAAA CMAAAA OOOOxx 6273 315 1 1 3 13 73 273 273 1273 6273 146 147 HHAAAA DMAAAA VVVVxx 4478 316 0 2 8 18 78 478 478 4478 4478 156 157 GQAAAA EMAAAA AAAAxx 8854 317 0 2 4 14 54 854 854 3854 8854 108 109 OCAAAA FMAAAA HHHHxx 2105 318 1 1 5 5 5 105 105 2105 2105 10 11 ZCAAAA GMAAAA OOOOxx 8345 319 1 1 5 5 45 345 345 3345 8345 90 91 ZIAAAA HMAAAA VVVVxx 1941 320 1 1 1 1 41 941 1941 1941 1941 82 83 RWAAAA IMAAAA AAAAxx 1765 321 1 1 5 5 65 765 1765 1765 1765 130 131 XPAAAA JMAAAA HHHHxx 9592 322 0 0 2 12 92 592 1592 4592 9592 184 185 YEAAAA KMAAAA OOOOxx 1694 323 0 2 4 14 94 694 1694 1694 1694 188 189 ENAAAA LMAAAA VVVVxx 8940 324 0 0 0 0 40 940 940 3940 8940 80 81 WFAAAA MMAAAA AAAAxx 7264 325 0 0 4 4 64 264 1264 2264 7264 128 129 KTAAAA NMAAAA HHHHxx 4699 326 1 3 9 19 99 699 699 4699 4699 198 199 TYAAAA OMAAAA OOOOxx 4541 327 1 1 1 1 41 541 541 4541 4541 82 83 RSAAAA PMAAAA VVVVxx 5768 328 0 0 8 8 68 768 1768 768 5768 136 137 WNAAAA QMAAAA AAAAxx 6183 329 1 3 3 3 83 183 183 1183 6183 166 167 VDAAAA RMAAAA HHHHxx 7457 330 1 1 7 17 57 457 1457 2457 7457 114 115 VAAAAA SMAAAA OOOOxx 7317 331 1 1 7 17 17 317 1317 2317 7317 34 35 LVAAAA TMAAAA VVVVxx 1944 332 0 0 4 4 44 944 1944 1944 1944 88 89 UWAAAA UMAAAA AAAAxx 665 333 1 1 5 5 65 665 665 665 665 130 131 PZAAAA VMAAAA HHHHxx 5974 334 0 2 4 14 74 974 1974 974 5974 148 149 UVAAAA WMAAAA OOOOxx 7370 335 0 2 0 10 70 370 1370 2370 7370 140 141 MXAAAA XMAAAA VVVVxx 9196 336 0 0 6 16 96 196 1196 4196 9196 192 193 SPAAAA YMAAAA AAAAxx 6796 337 0 0 6 16 96 796 796 1796 6796 192 193 KBAAAA ZMAAAA HHHHxx 6180 338 0 0 0 0 80 180 180 1180 6180 160 161 SDAAAA ANAAAA OOOOxx 8557 339 1 1 7 17 57 557 557 3557 8557 114 115 DRAAAA BNAAAA VVVVxx 928 340 0 0 8 8 28 928 928 928 928 56 57 SJAAAA CNAAAA AAAAxx 6275 341 1 3 5 15 75 275 275 1275 6275 150 151 JHAAAA DNAAAA HHHHxx 409 342 1 1 9 9 9 409 409 409 409 18 19 TPAAAA ENAAAA OOOOxx 6442 343 0 2 2 2 42 442 442 1442 6442 84 85 UNAAAA FNAAAA VVVVxx 5889 344 1 1 9 9 89 889 1889 889 5889 178 179 NSAAAA GNAAAA AAAAxx 5180 345 0 0 0 0 80 180 1180 180 5180 160 161 GRAAAA HNAAAA HHHHxx 1629 346 1 1 9 9 29 629 1629 1629 1629 58 59 RKAAAA INAAAA OOOOxx 6088 347 0 0 8 8 88 88 88 1088 6088 176 177 EAAAAA JNAAAA VVVVxx 5598 348 0 2 8 18 98 598 1598 598 5598 196 197 IHAAAA KNAAAA AAAAxx 1803 349 1 3 3 3 3 803 1803 1803 1803 6 7 JRAAAA LNAAAA HHHHxx 2330 350 0 2 0 10 30 330 330 2330 2330 60 61 QLAAAA MNAAAA OOOOxx 5901 351 1 1 1 1 1 901 1901 901 5901 2 3 ZSAAAA NNAAAA VVVVxx 780 352 0 0 0 0 80 780 780 780 780 160 161 AEAAAA ONAAAA AAAAxx 7171 353 1 3 1 11 71 171 1171 2171 7171 142 143 VPAAAA PNAAAA HHHHxx 8778 354 0 2 8 18 78 778 778 3778 8778 156 157 QZAAAA QNAAAA OOOOxx 6622 355 0 2 2 2 22 622 622 1622 6622 44 45 SUAAAA RNAAAA VVVVxx 9938 356 0 2 8 18 38 938 1938 4938 9938 76 77 GSAAAA SNAAAA AAAAxx 8254 357 0 2 4 14 54 254 254 3254 8254 108 109 MFAAAA TNAAAA HHHHxx 1951 358 1 3 1 11 51 951 1951 1951 1951 102 103 BXAAAA UNAAAA OOOOxx 1434 359 0 2 4 14 34 434 1434 1434 1434 68 69 EDAAAA VNAAAA VVVVxx 7539 360 1 3 9 19 39 539 1539 2539 7539 78 79 ZDAAAA WNAAAA AAAAxx 600 361 0 0 0 0 0 600 600 600 600 0 1 CXAAAA XNAAAA HHHHxx 3122 362 0 2 2 2 22 122 1122 3122 3122 44 45 CQAAAA YNAAAA OOOOxx 5704 363 0 0 4 4 4 704 1704 704 5704 8 9 KLAAAA ZNAAAA VVVVxx 6300 364 0 0 0 0 0 300 300 1300 6300 0 1 IIAAAA AOAAAA AAAAxx 4585 365 1 1 5 5 85 585 585 4585 4585 170 171 JUAAAA BOAAAA HHHHxx 6313 366 1 1 3 13 13 313 313 1313 6313 26 27 VIAAAA COAAAA OOOOxx 3154 367 0 2 4 14 54 154 1154 3154 3154 108 109 IRAAAA DOAAAA VVVVxx 642 368 0 2 2 2 42 642 642 642 642 84 85 SYAAAA EOAAAA AAAAxx 7736 369 0 0 6 16 36 736 1736 2736 7736 72 73 OLAAAA FOAAAA HHHHxx 5087 370 1 3 7 7 87 87 1087 87 5087 174 175 RNAAAA GOAAAA OOOOxx 5708 371 0 0 8 8 8 708 1708 708 5708 16 17 OLAAAA HOAAAA VVVVxx 8169 372 1 1 9 9 69 169 169 3169 8169 138 139 FCAAAA IOAAAA AAAAxx 9768 373 0 0 8 8 68 768 1768 4768 9768 136 137 SLAAAA JOAAAA HHHHxx 3874 374 0 2 4 14 74 874 1874 3874 3874 148 149 ATAAAA KOAAAA OOOOxx 6831 375 1 3 1 11 31 831 831 1831 6831 62 63 TCAAAA LOAAAA VVVVxx 18 376 0 2 8 18 18 18 18 18 18 36 37 SAAAAA MOAAAA AAAAxx 6375 377 1 3 5 15 75 375 375 1375 6375 150 151 FLAAAA NOAAAA HHHHxx 7106 378 0 2 6 6 6 106 1106 2106 7106 12 13 INAAAA OOAAAA OOOOxx 5926 379 0 2 6 6 26 926 1926 926 5926 52 53 YTAAAA POAAAA VVVVxx 4956 380 0 0 6 16 56 956 956 4956 4956 112 113 QIAAAA QOAAAA AAAAxx 7042 381 0 2 2 2 42 42 1042 2042 7042 84 85 WKAAAA ROAAAA HHHHxx 6043 382 1 3 3 3 43 43 43 1043 6043 86 87 LYAAAA SOAAAA OOOOxx 2084 383 0 0 4 4 84 84 84 2084 2084 168 169 ECAAAA TOAAAA VVVVxx 6038 384 0 2 8 18 38 38 38 1038 6038 76 77 GYAAAA UOAAAA AAAAxx 7253 385 1 1 3 13 53 253 1253 2253 7253 106 107 ZSAAAA VOAAAA HHHHxx 2061 386 1 1 1 1 61 61 61 2061 2061 122 123 HBAAAA WOAAAA OOOOxx 7800 387 0 0 0 0 0 800 1800 2800 7800 0 1 AOAAAA XOAAAA VVVVxx 4970 388 0 2 0 10 70 970 970 4970 4970 140 141 EJAAAA YOAAAA AAAAxx 8580 389 0 0 0 0 80 580 580 3580 8580 160 161 ASAAAA ZOAAAA HHHHxx 9173 390 1 1 3 13 73 173 1173 4173 9173 146 147 VOAAAA APAAAA OOOOxx 8558 391 0 2 8 18 58 558 558 3558 8558 116 117 ERAAAA BPAAAA VVVVxx 3897 392 1 1 7 17 97 897 1897 3897 3897 194 195 XTAAAA CPAAAA AAAAxx 5069 393 1 1 9 9 69 69 1069 69 5069 138 139 ZMAAAA DPAAAA HHHHxx 2301 394 1 1 1 1 1 301 301 2301 2301 2 3 NKAAAA EPAAAA OOOOxx 9863 395 1 3 3 3 63 863 1863 4863 9863 126 127 JPAAAA FPAAAA VVVVxx 5733 396 1 1 3 13 33 733 1733 733 5733 66 67 NMAAAA GPAAAA AAAAxx 2338 397 0 2 8 18 38 338 338 2338 2338 76 77 YLAAAA HPAAAA HHHHxx 9639 398 1 3 9 19 39 639 1639 4639 9639 78 79 TGAAAA IPAAAA OOOOxx 1139 399 1 3 9 19 39 139 1139 1139 1139 78 79 VRAAAA JPAAAA VVVVxx 2293 400 1 1 3 13 93 293 293 2293 2293 186 187 FKAAAA KPAAAA AAAAxx 6125 401 1 1 5 5 25 125 125 1125 6125 50 51 PBAAAA LPAAAA HHHHxx 5374 402 0 2 4 14 74 374 1374 374 5374 148 149 SYAAAA MPAAAA OOOOxx 7216 403 0 0 6 16 16 216 1216 2216 7216 32 33 ORAAAA NPAAAA VVVVxx 2285 404 1 1 5 5 85 285 285 2285 2285 170 171 XJAAAA OPAAAA AAAAxx 2387 405 1 3 7 7 87 387 387 2387 2387 174 175 VNAAAA PPAAAA HHHHxx 5015 406 1 3 5 15 15 15 1015 15 5015 30 31 XKAAAA QPAAAA OOOOxx 2087 407 1 3 7 7 87 87 87 2087 2087 174 175 HCAAAA RPAAAA VVVVxx 4938 408 0 2 8 18 38 938 938 4938 4938 76 77 YHAAAA SPAAAA AAAAxx 3635 409 1 3 5 15 35 635 1635 3635 3635 70 71 VJAAAA TPAAAA HHHHxx 7737 410 1 1 7 17 37 737 1737 2737 7737 74 75 PLAAAA UPAAAA OOOOxx 8056 411 0 0 6 16 56 56 56 3056 8056 112 113 WXAAAA VPAAAA VVVVxx 4502 412 0 2 2 2 2 502 502 4502 4502 4 5 ERAAAA WPAAAA AAAAxx 54 413 0 2 4 14 54 54 54 54 54 108 109 CCAAAA XPAAAA HHHHxx 3182 414 0 2 2 2 82 182 1182 3182 3182 164 165 KSAAAA YPAAAA OOOOxx 3718 415 0 2 8 18 18 718 1718 3718 3718 36 37 ANAAAA ZPAAAA VVVVxx 3989 416 1 1 9 9 89 989 1989 3989 3989 178 179 LXAAAA AQAAAA AAAAxx 8028 417 0 0 8 8 28 28 28 3028 8028 56 57 UWAAAA BQAAAA HHHHxx 1426 418 0 2 6 6 26 426 1426 1426 1426 52 53 WCAAAA CQAAAA OOOOxx 3801 419 1 1 1 1 1 801 1801 3801 3801 2 3 FQAAAA DQAAAA VVVVxx 241 420 1 1 1 1 41 241 241 241 241 82 83 HJAAAA EQAAAA AAAAxx 8000 421 0 0 0 0 0 0 0 3000 8000 0 1 SVAAAA FQAAAA HHHHxx 8357 422 1 1 7 17 57 357 357 3357 8357 114 115 LJAAAA GQAAAA OOOOxx 7548 423 0 0 8 8 48 548 1548 2548 7548 96 97 IEAAAA HQAAAA VVVVxx 7307 424 1 3 7 7 7 307 1307 2307 7307 14 15 BVAAAA IQAAAA AAAAxx 2275 425 1 3 5 15 75 275 275 2275 2275 150 151 NJAAAA JQAAAA HHHHxx 2718 426 0 2 8 18 18 718 718 2718 2718 36 37 OAAAAA KQAAAA OOOOxx 7068 427 0 0 8 8 68 68 1068 2068 7068 136 137 WLAAAA LQAAAA VVVVxx 3181 428 1 1 1 1 81 181 1181 3181 3181 162 163 JSAAAA MQAAAA AAAAxx 749 429 1 1 9 9 49 749 749 749 749 98 99 VCAAAA NQAAAA HHHHxx 5195 430 1 3 5 15 95 195 1195 195 5195 190 191 VRAAAA OQAAAA OOOOxx 6136 431 0 0 6 16 36 136 136 1136 6136 72 73 ACAAAA PQAAAA VVVVxx 8012 432 0 0 2 12 12 12 12 3012 8012 24 25 EWAAAA QQAAAA AAAAxx 3957 433 1 1 7 17 57 957 1957 3957 3957 114 115 FWAAAA RQAAAA HHHHxx 3083 434 1 3 3 3 83 83 1083 3083 3083 166 167 POAAAA SQAAAA OOOOxx 9997 435 1 1 7 17 97 997 1997 4997 9997 194 195 NUAAAA TQAAAA VVVVxx 3299 436 1 3 9 19 99 299 1299 3299 3299 198 199 XWAAAA UQAAAA AAAAxx 846 437 0 2 6 6 46 846 846 846 846 92 93 OGAAAA VQAAAA HHHHxx 2985 438 1 1 5 5 85 985 985 2985 2985 170 171 VKAAAA WQAAAA OOOOxx 9238 439 0 2 8 18 38 238 1238 4238 9238 76 77 IRAAAA XQAAAA VVVVxx 1403 440 1 3 3 3 3 403 1403 1403 1403 6 7 ZBAAAA YQAAAA AAAAxx 5563 441 1 3 3 3 63 563 1563 563 5563 126 127 ZFAAAA ZQAAAA HHHHxx 7965 442 1 1 5 5 65 965 1965 2965 7965 130 131 JUAAAA ARAAAA OOOOxx 4512 443 0 0 2 12 12 512 512 4512 4512 24 25 ORAAAA BRAAAA VVVVxx 9730 444 0 2 0 10 30 730 1730 4730 9730 60 61 GKAAAA CRAAAA AAAAxx 1129 445 1 1 9 9 29 129 1129 1129 1129 58 59 LRAAAA DRAAAA HHHHxx 2624 446 0 0 4 4 24 624 624 2624 2624 48 49 YWAAAA ERAAAA OOOOxx 8178 447 0 2 8 18 78 178 178 3178 8178 156 157 OCAAAA FRAAAA VVVVxx 6468 448 0 0 8 8 68 468 468 1468 6468 136 137 UOAAAA GRAAAA AAAAxx 3027 449 1 3 7 7 27 27 1027 3027 3027 54 55 LMAAAA HRAAAA HHHHxx 3845 450 1 1 5 5 45 845 1845 3845 3845 90 91 XRAAAA IRAAAA OOOOxx 786 451 0 2 6 6 86 786 786 786 786 172 173 GEAAAA JRAAAA VVVVxx 4971 452 1 3 1 11 71 971 971 4971 4971 142 143 FJAAAA KRAAAA AAAAxx 1542 453 0 2 2 2 42 542 1542 1542 1542 84 85 IHAAAA LRAAAA HHHHxx 7967 454 1 3 7 7 67 967 1967 2967 7967 134 135 LUAAAA MRAAAA OOOOxx 443 455 1 3 3 3 43 443 443 443 443 86 87 BRAAAA NRAAAA VVVVxx 7318 456 0 2 8 18 18 318 1318 2318 7318 36 37 MVAAAA ORAAAA AAAAxx 4913 457 1 1 3 13 13 913 913 4913 4913 26 27 ZGAAAA PRAAAA HHHHxx 9466 458 0 2 6 6 66 466 1466 4466 9466 132 133 CAAAAA QRAAAA OOOOxx 7866 459 0 2 6 6 66 866 1866 2866 7866 132 133 OQAAAA RRAAAA VVVVxx 784 460 0 0 4 4 84 784 784 784 784 168 169 EEAAAA SRAAAA AAAAxx 9040 461 0 0 0 0 40 40 1040 4040 9040 80 81 SJAAAA TRAAAA HHHHxx 3954 462 0 2 4 14 54 954 1954 3954 3954 108 109 CWAAAA URAAAA OOOOxx 4183 463 1 3 3 3 83 183 183 4183 4183 166 167 XEAAAA VRAAAA VVVVxx 3608 464 0 0 8 8 8 608 1608 3608 3608 16 17 UIAAAA WRAAAA AAAAxx 7630 465 0 2 0 10 30 630 1630 2630 7630 60 61 MHAAAA XRAAAA HHHHxx 590 466 0 2 0 10 90 590 590 590 590 180 181 SWAAAA YRAAAA OOOOxx 3453 467 1 1 3 13 53 453 1453 3453 3453 106 107 VCAAAA ZRAAAA VVVVxx 7757 468 1 1 7 17 57 757 1757 2757 7757 114 115 JMAAAA ASAAAA AAAAxx 7394 469 0 2 4 14 94 394 1394 2394 7394 188 189 KYAAAA BSAAAA HHHHxx 396 470 0 0 6 16 96 396 396 396 396 192 193 GPAAAA CSAAAA OOOOxx 7873 471 1 1 3 13 73 873 1873 2873 7873 146 147 VQAAAA DSAAAA VVVVxx 1553 472 1 1 3 13 53 553 1553 1553 1553 106 107 THAAAA ESAAAA AAAAxx 598 473 0 2 8 18 98 598 598 598 598 196 197 AXAAAA FSAAAA HHHHxx 7191 474 1 3 1 11 91 191 1191 2191 7191 182 183 PQAAAA GSAAAA OOOOxx 8116 475 0 0 6 16 16 116 116 3116 8116 32 33 EAAAAA HSAAAA VVVVxx 2516 476 0 0 6 16 16 516 516 2516 2516 32 33 USAAAA ISAAAA AAAAxx 7750 477 0 2 0 10 50 750 1750 2750 7750 100 101 CMAAAA JSAAAA HHHHxx 6625 478 1 1 5 5 25 625 625 1625 6625 50 51 VUAAAA KSAAAA OOOOxx 8838 479 0 2 8 18 38 838 838 3838 8838 76 77 YBAAAA LSAAAA VVVVxx 4636 480 0 0 6 16 36 636 636 4636 4636 72 73 IWAAAA MSAAAA AAAAxx 7627 481 1 3 7 7 27 627 1627 2627 7627 54 55 JHAAAA NSAAAA HHHHxx 1690 482 0 2 0 10 90 690 1690 1690 1690 180 181 ANAAAA OSAAAA OOOOxx 7071 483 1 3 1 11 71 71 1071 2071 7071 142 143 ZLAAAA PSAAAA VVVVxx 2081 484 1 1 1 1 81 81 81 2081 2081 162 163 BCAAAA QSAAAA AAAAxx 7138 485 0 2 8 18 38 138 1138 2138 7138 76 77 OOAAAA RSAAAA HHHHxx 864 486 0 0 4 4 64 864 864 864 864 128 129 GHAAAA SSAAAA OOOOxx 6392 487 0 0 2 12 92 392 392 1392 6392 184 185 WLAAAA TSAAAA VVVVxx 7544 488 0 0 4 4 44 544 1544 2544 7544 88 89 EEAAAA USAAAA AAAAxx 5438 489 0 2 8 18 38 438 1438 438 5438 76 77 EBAAAA VSAAAA HHHHxx 7099 490 1 3 9 19 99 99 1099 2099 7099 198 199 BNAAAA WSAAAA OOOOxx 5157 491 1 1 7 17 57 157 1157 157 5157 114 115 JQAAAA XSAAAA VVVVxx 3391 492 1 3 1 11 91 391 1391 3391 3391 182 183 LAAAAA YSAAAA AAAAxx 3805 493 1 1 5 5 5 805 1805 3805 3805 10 11 JQAAAA ZSAAAA HHHHxx 2110 494 0 2 0 10 10 110 110 2110 2110 20 21 EDAAAA ATAAAA OOOOxx 3176 495 0 0 6 16 76 176 1176 3176 3176 152 153 ESAAAA BTAAAA VVVVxx 5918 496 0 2 8 18 18 918 1918 918 5918 36 37 QTAAAA CTAAAA AAAAxx 1218 497 0 2 8 18 18 218 1218 1218 1218 36 37 WUAAAA DTAAAA HHHHxx 6683 498 1 3 3 3 83 683 683 1683 6683 166 167 BXAAAA ETAAAA OOOOxx 914 499 0 2 4 14 14 914 914 914 914 28 29 EJAAAA FTAAAA VVVVxx 4737 500 1 1 7 17 37 737 737 4737 4737 74 75 FAAAAA GTAAAA AAAAxx 7286 501 0 2 6 6 86 286 1286 2286 7286 172 173 GUAAAA HTAAAA HHHHxx 9975 502 1 3 5 15 75 975 1975 4975 9975 150 151 RTAAAA ITAAAA OOOOxx 8030 503 0 2 0 10 30 30 30 3030 8030 60 61 WWAAAA JTAAAA VVVVxx 7364 504 0 0 4 4 64 364 1364 2364 7364 128 129 GXAAAA KTAAAA AAAAxx 1389 505 1 1 9 9 89 389 1389 1389 1389 178 179 LBAAAA LTAAAA HHHHxx 4025 506 1 1 5 5 25 25 25 4025 4025 50 51 VYAAAA MTAAAA OOOOxx 4835 507 1 3 5 15 35 835 835 4835 4835 70 71 ZDAAAA NTAAAA VVVVxx 8045 508 1 1 5 5 45 45 45 3045 8045 90 91 LXAAAA OTAAAA AAAAxx 1864 509 0 0 4 4 64 864 1864 1864 1864 128 129 STAAAA PTAAAA HHHHxx 3313 510 1 1 3 13 13 313 1313 3313 3313 26 27 LXAAAA QTAAAA OOOOxx 2384 511 0 0 4 4 84 384 384 2384 2384 168 169 SNAAAA RTAAAA VVVVxx 6115 512 1 3 5 15 15 115 115 1115 6115 30 31 FBAAAA STAAAA AAAAxx 5705 513 1 1 5 5 5 705 1705 705 5705 10 11 LLAAAA TTAAAA HHHHxx 9269 514 1 1 9 9 69 269 1269 4269 9269 138 139 NSAAAA UTAAAA OOOOxx 3379 515 1 3 9 19 79 379 1379 3379 3379 158 159 ZZAAAA VTAAAA VVVVxx 8205 516 1 1 5 5 5 205 205 3205 8205 10 11 PDAAAA WTAAAA AAAAxx 6575 517 1 3 5 15 75 575 575 1575 6575 150 151 XSAAAA XTAAAA HHHHxx 486 518 0 2 6 6 86 486 486 486 486 172 173 SSAAAA YTAAAA OOOOxx 4894 519 0 2 4 14 94 894 894 4894 4894 188 189 GGAAAA ZTAAAA VVVVxx 3090 520 0 2 0 10 90 90 1090 3090 3090 180 181 WOAAAA AUAAAA AAAAxx 759 521 1 3 9 19 59 759 759 759 759 118 119 FDAAAA BUAAAA HHHHxx 4864 522 0 0 4 4 64 864 864 4864 4864 128 129 CFAAAA CUAAAA OOOOxx 4083 523 1 3 3 3 83 83 83 4083 4083 166 167 BBAAAA DUAAAA VVVVxx 6918 524 0 2 8 18 18 918 918 1918 6918 36 37 CGAAAA EUAAAA AAAAxx 8146 525 0 2 6 6 46 146 146 3146 8146 92 93 IBAAAA FUAAAA HHHHxx 1523 526 1 3 3 3 23 523 1523 1523 1523 46 47 PGAAAA GUAAAA OOOOxx 1591 527 1 3 1 11 91 591 1591 1591 1591 182 183 FJAAAA HUAAAA VVVVxx 3343 528 1 3 3 3 43 343 1343 3343 3343 86 87 PYAAAA IUAAAA AAAAxx 1391 529 1 3 1 11 91 391 1391 1391 1391 182 183 NBAAAA JUAAAA HHHHxx 9963 530 1 3 3 3 63 963 1963 4963 9963 126 127 FTAAAA KUAAAA OOOOxx 2423 531 1 3 3 3 23 423 423 2423 2423 46 47 FPAAAA LUAAAA VVVVxx 1822 532 0 2 2 2 22 822 1822 1822 1822 44 45 CSAAAA MUAAAA AAAAxx 8706 533 0 2 6 6 6 706 706 3706 8706 12 13 WWAAAA NUAAAA HHHHxx 3001 534 1 1 1 1 1 1 1001 3001 3001 2 3 LLAAAA OUAAAA OOOOxx 6707 535 1 3 7 7 7 707 707 1707 6707 14 15 ZXAAAA PUAAAA VVVVxx 2121 536 1 1 1 1 21 121 121 2121 2121 42 43 PDAAAA QUAAAA AAAAxx 5814 537 0 2 4 14 14 814 1814 814 5814 28 29 QPAAAA RUAAAA HHHHxx 2659 538 1 3 9 19 59 659 659 2659 2659 118 119 HYAAAA SUAAAA OOOOxx 2016 539 0 0 6 16 16 16 16 2016 2016 32 33 OZAAAA TUAAAA VVVVxx 4286 540 0 2 6 6 86 286 286 4286 4286 172 173 WIAAAA UUAAAA AAAAxx 9205 541 1 1 5 5 5 205 1205 4205 9205 10 11 BQAAAA VUAAAA HHHHxx 3496 542 0 0 6 16 96 496 1496 3496 3496 192 193 MEAAAA WUAAAA OOOOxx 5333 543 1 1 3 13 33 333 1333 333 5333 66 67 DXAAAA XUAAAA VVVVxx 5571 544 1 3 1 11 71 571 1571 571 5571 142 143 HGAAAA YUAAAA AAAAxx 1696 545 0 0 6 16 96 696 1696 1696 1696 192 193 GNAAAA ZUAAAA HHHHxx 4871 546 1 3 1 11 71 871 871 4871 4871 142 143 JFAAAA AVAAAA OOOOxx 4852 547 0 0 2 12 52 852 852 4852 4852 104 105 QEAAAA BVAAAA VVVVxx 8483 548 1 3 3 3 83 483 483 3483 8483 166 167 HOAAAA CVAAAA AAAAxx 1376 549 0 0 6 16 76 376 1376 1376 1376 152 153 YAAAAA DVAAAA HHHHxx 5456 550 0 0 6 16 56 456 1456 456 5456 112 113 WBAAAA EVAAAA OOOOxx 499 551 1 3 9 19 99 499 499 499 499 198 199 FTAAAA FVAAAA VVVVxx 3463 552 1 3 3 3 63 463 1463 3463 3463 126 127 FDAAAA GVAAAA AAAAxx 7426 553 0 2 6 6 26 426 1426 2426 7426 52 53 QZAAAA HVAAAA HHHHxx 5341 554 1 1 1 1 41 341 1341 341 5341 82 83 LXAAAA IVAAAA OOOOxx 9309 555 1 1 9 9 9 309 1309 4309 9309 18 19 BUAAAA JVAAAA VVVVxx 2055 556 1 3 5 15 55 55 55 2055 2055 110 111 BBAAAA KVAAAA AAAAxx 2199 557 1 3 9 19 99 199 199 2199 2199 198 199 PGAAAA LVAAAA HHHHxx 7235 558 1 3 5 15 35 235 1235 2235 7235 70 71 HSAAAA MVAAAA OOOOxx 8661 559 1 1 1 1 61 661 661 3661 8661 122 123 DVAAAA NVAAAA VVVVxx 9494 560 0 2 4 14 94 494 1494 4494 9494 188 189 EBAAAA OVAAAA AAAAxx 935 561 1 3 5 15 35 935 935 935 935 70 71 ZJAAAA PVAAAA HHHHxx 7044 562 0 0 4 4 44 44 1044 2044 7044 88 89 YKAAAA QVAAAA OOOOxx 1974 563 0 2 4 14 74 974 1974 1974 1974 148 149 YXAAAA RVAAAA VVVVxx 9679 564 1 3 9 19 79 679 1679 4679 9679 158 159 HIAAAA SVAAAA AAAAxx 9822 565 0 2 2 2 22 822 1822 4822 9822 44 45 UNAAAA TVAAAA HHHHxx 4088 566 0 0 8 8 88 88 88 4088 4088 176 177 GBAAAA UVAAAA OOOOxx 1749 567 1 1 9 9 49 749 1749 1749 1749 98 99 HPAAAA VVAAAA VVVVxx 2116 568 0 0 6 16 16 116 116 2116 2116 32 33 KDAAAA WVAAAA AAAAxx 976 569 0 0 6 16 76 976 976 976 976 152 153 OLAAAA XVAAAA HHHHxx 8689 570 1 1 9 9 89 689 689 3689 8689 178 179 FWAAAA YVAAAA OOOOxx 2563 571 1 3 3 3 63 563 563 2563 2563 126 127 PUAAAA ZVAAAA VVVVxx 7195 572 1 3 5 15 95 195 1195 2195 7195 190 191 TQAAAA AWAAAA AAAAxx 9985 573 1 1 5 5 85 985 1985 4985 9985 170 171 BUAAAA BWAAAA HHHHxx 7699 574 1 3 9 19 99 699 1699 2699 7699 198 199 DKAAAA CWAAAA OOOOxx 5311 575 1 3 1 11 11 311 1311 311 5311 22 23 HWAAAA DWAAAA VVVVxx 295 576 1 3 5 15 95 295 295 295 295 190 191 JLAAAA EWAAAA AAAAxx 8214 577 0 2 4 14 14 214 214 3214 8214 28 29 YDAAAA FWAAAA HHHHxx 3275 578 1 3 5 15 75 275 1275 3275 3275 150 151 ZVAAAA GWAAAA OOOOxx 9646 579 0 2 6 6 46 646 1646 4646 9646 92 93 AHAAAA HWAAAA VVVVxx 1908 580 0 0 8 8 8 908 1908 1908 1908 16 17 KVAAAA IWAAAA AAAAxx 3858 581 0 2 8 18 58 858 1858 3858 3858 116 117 KSAAAA JWAAAA HHHHxx 9362 582 0 2 2 2 62 362 1362 4362 9362 124 125 CWAAAA KWAAAA OOOOxx 9307 583 1 3 7 7 7 307 1307 4307 9307 14 15 ZTAAAA LWAAAA VVVVxx 6124 584 0 0 4 4 24 124 124 1124 6124 48 49 OBAAAA MWAAAA AAAAxx 2405 585 1 1 5 5 5 405 405 2405 2405 10 11 NOAAAA NWAAAA HHHHxx 8422 586 0 2 2 2 22 422 422 3422 8422 44 45 YLAAAA OWAAAA OOOOxx 393 587 1 1 3 13 93 393 393 393 393 186 187 DPAAAA PWAAAA VVVVxx 8973 588 1 1 3 13 73 973 973 3973 8973 146 147 DHAAAA QWAAAA AAAAxx 5171 589 1 3 1 11 71 171 1171 171 5171 142 143 XQAAAA RWAAAA HHHHxx 4929 590 1 1 9 9 29 929 929 4929 4929 58 59 PHAAAA SWAAAA OOOOxx 6935 591 1 3 5 15 35 935 935 1935 6935 70 71 TGAAAA TWAAAA VVVVxx 8584 592 0 0 4 4 84 584 584 3584 8584 168 169 ESAAAA UWAAAA AAAAxx 1035 593 1 3 5 15 35 35 1035 1035 1035 70 71 VNAAAA VWAAAA HHHHxx 3734 594 0 2 4 14 34 734 1734 3734 3734 68 69 QNAAAA WWAAAA OOOOxx 1458 595 0 2 8 18 58 458 1458 1458 1458 116 117 CEAAAA XWAAAA VVVVxx 8746 596 0 2 6 6 46 746 746 3746 8746 92 93 KYAAAA YWAAAA AAAAxx 1677 597 1 1 7 17 77 677 1677 1677 1677 154 155 NMAAAA ZWAAAA HHHHxx 8502 598 0 2 2 2 2 502 502 3502 8502 4 5 APAAAA AXAAAA OOOOxx 7752 599 0 0 2 12 52 752 1752 2752 7752 104 105 EMAAAA BXAAAA VVVVxx 2556 600 0 0 6 16 56 556 556 2556 2556 112 113 IUAAAA CXAAAA AAAAxx 6426 601 0 2 6 6 26 426 426 1426 6426 52 53 ENAAAA DXAAAA HHHHxx 8420 602 0 0 0 0 20 420 420 3420 8420 40 41 WLAAAA EXAAAA OOOOxx 4462 603 0 2 2 2 62 462 462 4462 4462 124 125 QPAAAA FXAAAA VVVVxx 1378 604 0 2 8 18 78 378 1378 1378 1378 156 157 ABAAAA GXAAAA AAAAxx 1387 605 1 3 7 7 87 387 1387 1387 1387 174 175 JBAAAA HXAAAA HHHHxx 8094 606 0 2 4 14 94 94 94 3094 8094 188 189 IZAAAA IXAAAA OOOOxx 7247 607 1 3 7 7 47 247 1247 2247 7247 94 95 TSAAAA JXAAAA VVVVxx 4261 608 1 1 1 1 61 261 261 4261 4261 122 123 XHAAAA KXAAAA AAAAxx 5029 609 1 1 9 9 29 29 1029 29 5029 58 59 LLAAAA LXAAAA HHHHxx 3625 610 1 1 5 5 25 625 1625 3625 3625 50 51 LJAAAA MXAAAA OOOOxx 8068 611 0 0 8 8 68 68 68 3068 8068 136 137 IYAAAA NXAAAA VVVVxx 102 612 0 2 2 2 2 102 102 102 102 4 5 YDAAAA OXAAAA AAAAxx 5596 613 0 0 6 16 96 596 1596 596 5596 192 193 GHAAAA PXAAAA HHHHxx 5872 614 0 0 2 12 72 872 1872 872 5872 144 145 WRAAAA QXAAAA OOOOxx 4742 615 0 2 2 2 42 742 742 4742 4742 84 85 KAAAAA RXAAAA VVVVxx 2117 616 1 1 7 17 17 117 117 2117 2117 34 35 LDAAAA SXAAAA AAAAxx 3945 617 1 1 5 5 45 945 1945 3945 3945 90 91 TVAAAA TXAAAA HHHHxx 7483 618 1 3 3 3 83 483 1483 2483 7483 166 167 VBAAAA UXAAAA OOOOxx 4455 619 1 3 5 15 55 455 455 4455 4455 110 111 JPAAAA VXAAAA VVVVxx 609 620 1 1 9 9 9 609 609 609 609 18 19 LXAAAA WXAAAA AAAAxx 9829 621 1 1 9 9 29 829 1829 4829 9829 58 59 BOAAAA XXAAAA HHHHxx 4857 622 1 1 7 17 57 857 857 4857 4857 114 115 VEAAAA YXAAAA OOOOxx 3314 623 0 2 4 14 14 314 1314 3314 3314 28 29 MXAAAA ZXAAAA VVVVxx 5353 624 1 1 3 13 53 353 1353 353 5353 106 107 XXAAAA AYAAAA AAAAxx 4909 625 1 1 9 9 9 909 909 4909 4909 18 19 VGAAAA BYAAAA HHHHxx 7597 626 1 1 7 17 97 597 1597 2597 7597 194 195 FGAAAA CYAAAA OOOOxx 2683 627 1 3 3 3 83 683 683 2683 2683 166 167 FZAAAA DYAAAA VVVVxx 3223 628 1 3 3 3 23 223 1223 3223 3223 46 47 ZTAAAA EYAAAA AAAAxx 5363 629 1 3 3 3 63 363 1363 363 5363 126 127 HYAAAA FYAAAA HHHHxx 4578 630 0 2 8 18 78 578 578 4578 4578 156 157 CUAAAA GYAAAA OOOOxx 5544 631 0 0 4 4 44 544 1544 544 5544 88 89 GFAAAA HYAAAA VVVVxx 1589 632 1 1 9 9 89 589 1589 1589 1589 178 179 DJAAAA IYAAAA AAAAxx 7412 633 0 0 2 12 12 412 1412 2412 7412 24 25 CZAAAA JYAAAA HHHHxx 3803 634 1 3 3 3 3 803 1803 3803 3803 6 7 HQAAAA KYAAAA OOOOxx 6179 635 1 3 9 19 79 179 179 1179 6179 158 159 RDAAAA LYAAAA VVVVxx 5588 636 0 0 8 8 88 588 1588 588 5588 176 177 YGAAAA MYAAAA AAAAxx 2134 637 0 2 4 14 34 134 134 2134 2134 68 69 CEAAAA NYAAAA HHHHxx 4383 638 1 3 3 3 83 383 383 4383 4383 166 167 PMAAAA OYAAAA OOOOxx 6995 639 1 3 5 15 95 995 995 1995 6995 190 191 BJAAAA PYAAAA VVVVxx 6598 640 0 2 8 18 98 598 598 1598 6598 196 197 UTAAAA QYAAAA AAAAxx 8731 641 1 3 1 11 31 731 731 3731 8731 62 63 VXAAAA RYAAAA HHHHxx 7177 642 1 1 7 17 77 177 1177 2177 7177 154 155 BQAAAA SYAAAA OOOOxx 6578 643 0 2 8 18 78 578 578 1578 6578 156 157 ATAAAA TYAAAA VVVVxx 9393 644 1 1 3 13 93 393 1393 4393 9393 186 187 HXAAAA UYAAAA AAAAxx 1276 645 0 0 6 16 76 276 1276 1276 1276 152 153 CXAAAA VYAAAA HHHHxx 8766 646 0 2 6 6 66 766 766 3766 8766 132 133 EZAAAA WYAAAA OOOOxx 1015 647 1 3 5 15 15 15 1015 1015 1015 30 31 BNAAAA XYAAAA VVVVxx 4396 648 0 0 6 16 96 396 396 4396 4396 192 193 CNAAAA YYAAAA AAAAxx 5564 649 0 0 4 4 64 564 1564 564 5564 128 129 AGAAAA ZYAAAA HHHHxx 927 650 1 3 7 7 27 927 927 927 927 54 55 RJAAAA AZAAAA OOOOxx 3306 651 0 2 6 6 6 306 1306 3306 3306 12 13 EXAAAA BZAAAA VVVVxx 1615 652 1 3 5 15 15 615 1615 1615 1615 30 31 DKAAAA CZAAAA AAAAxx 4550 653 0 2 0 10 50 550 550 4550 4550 100 101 ATAAAA DZAAAA HHHHxx 2468 654 0 0 8 8 68 468 468 2468 2468 136 137 YQAAAA EZAAAA OOOOxx 5336 655 0 0 6 16 36 336 1336 336 5336 72 73 GXAAAA FZAAAA VVVVxx 4471 656 1 3 1 11 71 471 471 4471 4471 142 143 ZPAAAA GZAAAA AAAAxx 8085 657 1 1 5 5 85 85 85 3085 8085 170 171 ZYAAAA HZAAAA HHHHxx 540 658 0 0 0 0 40 540 540 540 540 80 81 UUAAAA IZAAAA OOOOxx 5108 659 0 0 8 8 8 108 1108 108 5108 16 17 MOAAAA JZAAAA VVVVxx 8015 660 1 3 5 15 15 15 15 3015 8015 30 31 HWAAAA KZAAAA AAAAxx 2857 661 1 1 7 17 57 857 857 2857 2857 114 115 XFAAAA LZAAAA HHHHxx 9472 662 0 0 2 12 72 472 1472 4472 9472 144 145 IAAAAA MZAAAA OOOOxx 5666 663 0 2 6 6 66 666 1666 666 5666 132 133 YJAAAA NZAAAA VVVVxx 3555 664 1 3 5 15 55 555 1555 3555 3555 110 111 TGAAAA OZAAAA AAAAxx 378 665 0 2 8 18 78 378 378 378 378 156 157 OOAAAA PZAAAA HHHHxx 4466 666 0 2 6 6 66 466 466 4466 4466 132 133 UPAAAA QZAAAA OOOOxx 3247 667 1 3 7 7 47 247 1247 3247 3247 94 95 XUAAAA RZAAAA VVVVxx 6570 668 0 2 0 10 70 570 570 1570 6570 140 141 SSAAAA SZAAAA AAAAxx 5655 669 1 3 5 15 55 655 1655 655 5655 110 111 NJAAAA TZAAAA HHHHxx 917 670 1 1 7 17 17 917 917 917 917 34 35 HJAAAA UZAAAA OOOOxx 3637 671 1 1 7 17 37 637 1637 3637 3637 74 75 XJAAAA VZAAAA VVVVxx 3668 672 0 0 8 8 68 668 1668 3668 3668 136 137 CLAAAA WZAAAA AAAAxx 5644 673 0 0 4 4 44 644 1644 644 5644 88 89 CJAAAA XZAAAA HHHHxx 8286 674 0 2 6 6 86 286 286 3286 8286 172 173 SGAAAA YZAAAA OOOOxx 6896 675 0 0 6 16 96 896 896 1896 6896 192 193 GFAAAA ZZAAAA VVVVxx 2870 676 0 2 0 10 70 870 870 2870 2870 140 141 KGAAAA AABAAA AAAAxx 8041 677 1 1 1 1 41 41 41 3041 8041 82 83 HXAAAA BABAAA HHHHxx 8137 678 1 1 7 17 37 137 137 3137 8137 74 75 ZAAAAA CABAAA OOOOxx 4823 679 1 3 3 3 23 823 823 4823 4823 46 47 NDAAAA DABAAA VVVVxx 2438 680 0 2 8 18 38 438 438 2438 2438 76 77 UPAAAA EABAAA AAAAxx 6329 681 1 1 9 9 29 329 329 1329 6329 58 59 LJAAAA FABAAA HHHHxx 623 682 1 3 3 3 23 623 623 623 623 46 47 ZXAAAA GABAAA OOOOxx 1360 683 0 0 0 0 60 360 1360 1360 1360 120 121 IAAAAA HABAAA VVVVxx 7987 684 1 3 7 7 87 987 1987 2987 7987 174 175 FVAAAA IABAAA AAAAxx 9788 685 0 0 8 8 88 788 1788 4788 9788 176 177 MMAAAA JABAAA HHHHxx 3212 686 0 0 2 12 12 212 1212 3212 3212 24 25 OTAAAA KABAAA OOOOxx 2725 687 1 1 5 5 25 725 725 2725 2725 50 51 VAAAAA LABAAA VVVVxx 7837 688 1 1 7 17 37 837 1837 2837 7837 74 75 LPAAAA MABAAA AAAAxx 4746 689 0 2 6 6 46 746 746 4746 4746 92 93 OAAAAA NABAAA HHHHxx 3986 690 0 2 6 6 86 986 1986 3986 3986 172 173 IXAAAA OABAAA OOOOxx 9128 691 0 0 8 8 28 128 1128 4128 9128 56 57 CNAAAA PABAAA VVVVxx 5044 692 0 0 4 4 44 44 1044 44 5044 88 89 AMAAAA QABAAA AAAAxx 8132 693 0 0 2 12 32 132 132 3132 8132 64 65 UAAAAA RABAAA HHHHxx 9992 694 0 0 2 12 92 992 1992 4992 9992 184 185 IUAAAA SABAAA OOOOxx 8468 695 0 0 8 8 68 468 468 3468 8468 136 137 SNAAAA TABAAA VVVVxx 6876 696 0 0 6 16 76 876 876 1876 6876 152 153 MEAAAA UABAAA AAAAxx 3532 697 0 0 2 12 32 532 1532 3532 3532 64 65 WFAAAA VABAAA HHHHxx 2140 698 0 0 0 0 40 140 140 2140 2140 80 81 IEAAAA WABAAA OOOOxx 2183 699 1 3 3 3 83 183 183 2183 2183 166 167 ZFAAAA XABAAA VVVVxx 9766 700 0 2 6 6 66 766 1766 4766 9766 132 133 QLAAAA YABAAA AAAAxx 7943 701 1 3 3 3 43 943 1943 2943 7943 86 87 NTAAAA ZABAAA HHHHxx 9243 702 1 3 3 3 43 243 1243 4243 9243 86 87 NRAAAA ABBAAA OOOOxx 6241 703 1 1 1 1 41 241 241 1241 6241 82 83 BGAAAA BBBAAA VVVVxx 9540 704 0 0 0 0 40 540 1540 4540 9540 80 81 YCAAAA CBBAAA AAAAxx 7418 705 0 2 8 18 18 418 1418 2418 7418 36 37 IZAAAA DBBAAA HHHHxx 1603 706 1 3 3 3 3 603 1603 1603 1603 6 7 RJAAAA EBBAAA OOOOxx 8950 707 0 2 0 10 50 950 950 3950 8950 100 101 GGAAAA FBBAAA VVVVxx 6933 708 1 1 3 13 33 933 933 1933 6933 66 67 RGAAAA GBBAAA AAAAxx 2646 709 0 2 6 6 46 646 646 2646 2646 92 93 UXAAAA HBBAAA HHHHxx 3447 710 1 3 7 7 47 447 1447 3447 3447 94 95 PCAAAA IBBAAA OOOOxx 9957 711 1 1 7 17 57 957 1957 4957 9957 114 115 ZSAAAA JBBAAA VVVVxx 4623 712 1 3 3 3 23 623 623 4623 4623 46 47 VVAAAA KBBAAA AAAAxx 9058 713 0 2 8 18 58 58 1058 4058 9058 116 117 KKAAAA LBBAAA HHHHxx 7361 714 1 1 1 1 61 361 1361 2361 7361 122 123 DXAAAA MBBAAA OOOOxx 2489 715 1 1 9 9 89 489 489 2489 2489 178 179 TRAAAA NBBAAA VVVVxx 7643 716 1 3 3 3 43 643 1643 2643 7643 86 87 ZHAAAA OBBAAA AAAAxx 9166 717 0 2 6 6 66 166 1166 4166 9166 132 133 OOAAAA PBBAAA HHHHxx 7789 718 1 1 9 9 89 789 1789 2789 7789 178 179 PNAAAA QBBAAA OOOOxx 2332 719 0 0 2 12 32 332 332 2332 2332 64 65 SLAAAA RBBAAA VVVVxx 1832 720 0 0 2 12 32 832 1832 1832 1832 64 65 MSAAAA SBBAAA AAAAxx 8375 721 1 3 5 15 75 375 375 3375 8375 150 151 DKAAAA TBBAAA HHHHxx 948 722 0 0 8 8 48 948 948 948 948 96 97 MKAAAA UBBAAA OOOOxx 5613 723 1 1 3 13 13 613 1613 613 5613 26 27 XHAAAA VBBAAA VVVVxx 6310 724 0 2 0 10 10 310 310 1310 6310 20 21 SIAAAA WBBAAA AAAAxx 4254 725 0 2 4 14 54 254 254 4254 4254 108 109 QHAAAA XBBAAA HHHHxx 4260 726 0 0 0 0 60 260 260 4260 4260 120 121 WHAAAA YBBAAA OOOOxx 2060 727 0 0 0 0 60 60 60 2060 2060 120 121 GBAAAA ZBBAAA VVVVxx 4831 728 1 3 1 11 31 831 831 4831 4831 62 63 VDAAAA ACBAAA AAAAxx 6176 729 0 0 6 16 76 176 176 1176 6176 152 153 ODAAAA BCBAAA HHHHxx 6688 730 0 0 8 8 88 688 688 1688 6688 176 177 GXAAAA CCBAAA OOOOxx 5752 731 0 0 2 12 52 752 1752 752 5752 104 105 GNAAAA DCBAAA VVVVxx 8714 732 0 2 4 14 14 714 714 3714 8714 28 29 EXAAAA ECBAAA AAAAxx 6739 733 1 3 9 19 39 739 739 1739 6739 78 79 FZAAAA FCBAAA HHHHxx 7066 734 0 2 6 6 66 66 1066 2066 7066 132 133 ULAAAA GCBAAA OOOOxx 7250 735 0 2 0 10 50 250 1250 2250 7250 100 101 WSAAAA HCBAAA VVVVxx 3161 736 1 1 1 1 61 161 1161 3161 3161 122 123 PRAAAA ICBAAA AAAAxx 1411 737 1 3 1 11 11 411 1411 1411 1411 22 23 HCAAAA JCBAAA HHHHxx 9301 738 1 1 1 1 1 301 1301 4301 9301 2 3 TTAAAA KCBAAA OOOOxx 8324 739 0 0 4 4 24 324 324 3324 8324 48 49 EIAAAA LCBAAA VVVVxx 9641 740 1 1 1 1 41 641 1641 4641 9641 82 83 VGAAAA MCBAAA AAAAxx 7077 741 1 1 7 17 77 77 1077 2077 7077 154 155 FMAAAA NCBAAA HHHHxx 9888 742 0 0 8 8 88 888 1888 4888 9888 176 177 IQAAAA OCBAAA OOOOxx 9909 743 1 1 9 9 9 909 1909 4909 9909 18 19 DRAAAA PCBAAA VVVVxx 2209 744 1 1 9 9 9 209 209 2209 2209 18 19 ZGAAAA QCBAAA AAAAxx 6904 745 0 0 4 4 4 904 904 1904 6904 8 9 OFAAAA RCBAAA HHHHxx 6608 746 0 0 8 8 8 608 608 1608 6608 16 17 EUAAAA SCBAAA OOOOxx 8400 747 0 0 0 0 0 400 400 3400 8400 0 1 CLAAAA TCBAAA VVVVxx 5124 748 0 0 4 4 24 124 1124 124 5124 48 49 CPAAAA UCBAAA AAAAxx 5484 749 0 0 4 4 84 484 1484 484 5484 168 169 YCAAAA VCBAAA HHHHxx 3575 750 1 3 5 15 75 575 1575 3575 3575 150 151 NHAAAA WCBAAA OOOOxx 9723 751 1 3 3 3 23 723 1723 4723 9723 46 47 ZJAAAA XCBAAA VVVVxx 360 752 0 0 0 0 60 360 360 360 360 120 121 WNAAAA YCBAAA AAAAxx 1059 753 1 3 9 19 59 59 1059 1059 1059 118 119 TOAAAA ZCBAAA HHHHxx 4941 754 1 1 1 1 41 941 941 4941 4941 82 83 BIAAAA ADBAAA OOOOxx 2535 755 1 3 5 15 35 535 535 2535 2535 70 71 NTAAAA BDBAAA VVVVxx 4119 756 1 3 9 19 19 119 119 4119 4119 38 39 LCAAAA CDBAAA AAAAxx 3725 757 1 1 5 5 25 725 1725 3725 3725 50 51 HNAAAA DDBAAA HHHHxx 4758 758 0 2 8 18 58 758 758 4758 4758 116 117 ABAAAA EDBAAA OOOOxx 9593 759 1 1 3 13 93 593 1593 4593 9593 186 187 ZEAAAA FDBAAA VVVVxx 4663 760 1 3 3 3 63 663 663 4663 4663 126 127 JXAAAA GDBAAA AAAAxx 7734 761 0 2 4 14 34 734 1734 2734 7734 68 69 MLAAAA HDBAAA HHHHxx 9156 762 0 0 6 16 56 156 1156 4156 9156 112 113 EOAAAA IDBAAA OOOOxx 8120 763 0 0 0 0 20 120 120 3120 8120 40 41 IAAAAA JDBAAA VVVVxx 4385 764 1 1 5 5 85 385 385 4385 4385 170 171 RMAAAA KDBAAA AAAAxx 2926 765 0 2 6 6 26 926 926 2926 2926 52 53 OIAAAA LDBAAA HHHHxx 4186 766 0 2 6 6 86 186 186 4186 4186 172 173 AFAAAA MDBAAA OOOOxx 2508 767 0 0 8 8 8 508 508 2508 2508 16 17 MSAAAA NDBAAA VVVVxx 4012 768 0 0 2 12 12 12 12 4012 4012 24 25 IYAAAA ODBAAA AAAAxx 6266 769 0 2 6 6 66 266 266 1266 6266 132 133 AHAAAA PDBAAA HHHHxx 3709 770 1 1 9 9 9 709 1709 3709 3709 18 19 RMAAAA QDBAAA OOOOxx 7289 771 1 1 9 9 89 289 1289 2289 7289 178 179 JUAAAA RDBAAA VVVVxx 8875 772 1 3 5 15 75 875 875 3875 8875 150 151 JDAAAA SDBAAA AAAAxx 4412 773 0 0 2 12 12 412 412 4412 4412 24 25 SNAAAA TDBAAA HHHHxx 3033 774 1 1 3 13 33 33 1033 3033 3033 66 67 RMAAAA UDBAAA OOOOxx 1645 775 1 1 5 5 45 645 1645 1645 1645 90 91 HLAAAA VDBAAA VVVVxx 3557 776 1 1 7 17 57 557 1557 3557 3557 114 115 VGAAAA WDBAAA AAAAxx 6316 777 0 0 6 16 16 316 316 1316 6316 32 33 YIAAAA XDBAAA HHHHxx 2054 778 0 2 4 14 54 54 54 2054 2054 108 109 ABAAAA YDBAAA OOOOxx 7031 779 1 3 1 11 31 31 1031 2031 7031 62 63 LKAAAA ZDBAAA VVVVxx 3405 780 1 1 5 5 5 405 1405 3405 3405 10 11 ZAAAAA AEBAAA AAAAxx 5343 781 1 3 3 3 43 343 1343 343 5343 86 87 NXAAAA BEBAAA HHHHxx 5240 782 0 0 0 0 40 240 1240 240 5240 80 81 OTAAAA CEBAAA OOOOxx 9650 783 0 2 0 10 50 650 1650 4650 9650 100 101 EHAAAA DEBAAA VVVVxx 3777 784 1 1 7 17 77 777 1777 3777 3777 154 155 HPAAAA EEBAAA AAAAxx 9041 785 1 1 1 1 41 41 1041 4041 9041 82 83 TJAAAA FEBAAA HHHHxx 6923 786 1 3 3 3 23 923 923 1923 6923 46 47 HGAAAA GEBAAA OOOOxx 2977 787 1 1 7 17 77 977 977 2977 2977 154 155 NKAAAA HEBAAA VVVVxx 5500 788 0 0 0 0 0 500 1500 500 5500 0 1 ODAAAA IEBAAA AAAAxx 1044 789 0 0 4 4 44 44 1044 1044 1044 88 89 EOAAAA JEBAAA HHHHxx 434 790 0 2 4 14 34 434 434 434 434 68 69 SQAAAA KEBAAA OOOOxx 611 791 1 3 1 11 11 611 611 611 611 22 23 NXAAAA LEBAAA VVVVxx 5760 792 0 0 0 0 60 760 1760 760 5760 120 121 ONAAAA MEBAAA AAAAxx 2445 793 1 1 5 5 45 445 445 2445 2445 90 91 BQAAAA NEBAAA HHHHxx 7098 794 0 2 8 18 98 98 1098 2098 7098 196 197 ANAAAA OEBAAA OOOOxx 2188 795 0 0 8 8 88 188 188 2188 2188 176 177 EGAAAA PEBAAA VVVVxx 4597 796 1 1 7 17 97 597 597 4597 4597 194 195 VUAAAA QEBAAA AAAAxx 1913 797 1 1 3 13 13 913 1913 1913 1913 26 27 PVAAAA REBAAA HHHHxx 8696 798 0 0 6 16 96 696 696 3696 8696 192 193 MWAAAA SEBAAA OOOOxx 3332 799 0 0 2 12 32 332 1332 3332 3332 64 65 EYAAAA TEBAAA VVVVxx 8760 800 0 0 0 0 60 760 760 3760 8760 120 121 YYAAAA UEBAAA AAAAxx 3215 801 1 3 5 15 15 215 1215 3215 3215 30 31 RTAAAA VEBAAA HHHHxx 1625 802 1 1 5 5 25 625 1625 1625 1625 50 51 NKAAAA WEBAAA OOOOxx 4219 803 1 3 9 19 19 219 219 4219 4219 38 39 HGAAAA XEBAAA VVVVxx 415 804 1 3 5 15 15 415 415 415 415 30 31 ZPAAAA YEBAAA AAAAxx 4242 805 0 2 2 2 42 242 242 4242 4242 84 85 EHAAAA ZEBAAA HHHHxx 8660 806 0 0 0 0 60 660 660 3660 8660 120 121 CVAAAA AFBAAA OOOOxx 6525 807 1 1 5 5 25 525 525 1525 6525 50 51 ZQAAAA BFBAAA VVVVxx 2141 808 1 1 1 1 41 141 141 2141 2141 82 83 JEAAAA CFBAAA AAAAxx 5152 809 0 0 2 12 52 152 1152 152 5152 104 105 EQAAAA DFBAAA HHHHxx 8560 810 0 0 0 0 60 560 560 3560 8560 120 121 GRAAAA EFBAAA OOOOxx 9835 811 1 3 5 15 35 835 1835 4835 9835 70 71 HOAAAA FFBAAA VVVVxx 2657 812 1 1 7 17 57 657 657 2657 2657 114 115 FYAAAA GFBAAA AAAAxx 6085 813 1 1 5 5 85 85 85 1085 6085 170 171 BAAAAA HFBAAA HHHHxx 6698 814 0 2 8 18 98 698 698 1698 6698 196 197 QXAAAA IFBAAA OOOOxx 5421 815 1 1 1 1 21 421 1421 421 5421 42 43 NAAAAA JFBAAA VVVVxx 6661 816 1 1 1 1 61 661 661 1661 6661 122 123 FWAAAA KFBAAA AAAAxx 5645 817 1 1 5 5 45 645 1645 645 5645 90 91 DJAAAA LFBAAA HHHHxx 1248 818 0 0 8 8 48 248 1248 1248 1248 96 97 AWAAAA MFBAAA OOOOxx 5690 819 0 2 0 10 90 690 1690 690 5690 180 181 WKAAAA NFBAAA VVVVxx 4762 820 0 2 2 2 62 762 762 4762 4762 124 125 EBAAAA OFBAAA AAAAxx 1455 821 1 3 5 15 55 455 1455 1455 1455 110 111 ZDAAAA PFBAAA HHHHxx 9846 822 0 2 6 6 46 846 1846 4846 9846 92 93 SOAAAA QFBAAA OOOOxx 5295 823 1 3 5 15 95 295 1295 295 5295 190 191 RVAAAA RFBAAA VVVVxx 2826 824 0 2 6 6 26 826 826 2826 2826 52 53 SEAAAA SFBAAA AAAAxx 7496 825 0 0 6 16 96 496 1496 2496 7496 192 193 ICAAAA TFBAAA HHHHxx 3024 826 0 0 4 4 24 24 1024 3024 3024 48 49 IMAAAA UFBAAA OOOOxx 4945 827 1 1 5 5 45 945 945 4945 4945 90 91 FIAAAA VFBAAA VVVVxx 4404 828 0 0 4 4 4 404 404 4404 4404 8 9 KNAAAA WFBAAA AAAAxx 9302 829 0 2 2 2 2 302 1302 4302 9302 4 5 UTAAAA XFBAAA HHHHxx 1286 830 0 2 6 6 86 286 1286 1286 1286 172 173 MXAAAA YFBAAA OOOOxx 8435 831 1 3 5 15 35 435 435 3435 8435 70 71 LMAAAA ZFBAAA VVVVxx 8969 832 1 1 9 9 69 969 969 3969 8969 138 139 ZGAAAA AGBAAA AAAAxx 3302 833 0 2 2 2 2 302 1302 3302 3302 4 5 AXAAAA BGBAAA HHHHxx 9753 834 1 1 3 13 53 753 1753 4753 9753 106 107 DLAAAA CGBAAA OOOOxx 9374 835 0 2 4 14 74 374 1374 4374 9374 148 149 OWAAAA DGBAAA VVVVxx 4907 836 1 3 7 7 7 907 907 4907 4907 14 15 TGAAAA EGBAAA AAAAxx 1659 837 1 3 9 19 59 659 1659 1659 1659 118 119 VLAAAA FGBAAA HHHHxx 5095 838 1 3 5 15 95 95 1095 95 5095 190 191 ZNAAAA GGBAAA OOOOxx 9446 839 0 2 6 6 46 446 1446 4446 9446 92 93 IZAAAA HGBAAA VVVVxx 8528 840 0 0 8 8 28 528 528 3528 8528 56 57 AQAAAA IGBAAA AAAAxx 4890 841 0 2 0 10 90 890 890 4890 4890 180 181 CGAAAA JGBAAA HHHHxx 1221 842 1 1 1 1 21 221 1221 1221 1221 42 43 ZUAAAA KGBAAA OOOOxx 5583 843 1 3 3 3 83 583 1583 583 5583 166 167 TGAAAA LGBAAA VVVVxx 7303 844 1 3 3 3 3 303 1303 2303 7303 6 7 XUAAAA MGBAAA AAAAxx 406 845 0 2 6 6 6 406 406 406 406 12 13 QPAAAA NGBAAA HHHHxx 7542 846 0 2 2 2 42 542 1542 2542 7542 84 85 CEAAAA OGBAAA OOOOxx 9507 847 1 3 7 7 7 507 1507 4507 9507 14 15 RBAAAA PGBAAA VVVVxx 9511 848 1 3 1 11 11 511 1511 4511 9511 22 23 VBAAAA QGBAAA AAAAxx 1373 849 1 1 3 13 73 373 1373 1373 1373 146 147 VAAAAA RGBAAA HHHHxx 6556 850 0 0 6 16 56 556 556 1556 6556 112 113 ESAAAA SGBAAA OOOOxx 4117 851 1 1 7 17 17 117 117 4117 4117 34 35 JCAAAA TGBAAA VVVVxx 7794 852 0 2 4 14 94 794 1794 2794 7794 188 189 UNAAAA UGBAAA AAAAxx 7170 853 0 2 0 10 70 170 1170 2170 7170 140 141 UPAAAA VGBAAA HHHHxx 5809 854 1 1 9 9 9 809 1809 809 5809 18 19 LPAAAA WGBAAA OOOOxx 7828 855 0 0 8 8 28 828 1828 2828 7828 56 57 CPAAAA XGBAAA VVVVxx 8046 856 0 2 6 6 46 46 46 3046 8046 92 93 MXAAAA YGBAAA AAAAxx 4833 857 1 1 3 13 33 833 833 4833 4833 66 67 XDAAAA ZGBAAA HHHHxx 2107 858 1 3 7 7 7 107 107 2107 2107 14 15 BDAAAA AHBAAA OOOOxx 4276 859 0 0 6 16 76 276 276 4276 4276 152 153 MIAAAA BHBAAA VVVVxx 9536 860 0 0 6 16 36 536 1536 4536 9536 72 73 UCAAAA CHBAAA AAAAxx 5549 861 1 1 9 9 49 549 1549 549 5549 98 99 LFAAAA DHBAAA HHHHxx 6427 862 1 3 7 7 27 427 427 1427 6427 54 55 FNAAAA EHBAAA OOOOxx 1382 863 0 2 2 2 82 382 1382 1382 1382 164 165 EBAAAA FHBAAA VVVVxx 3256 864 0 0 6 16 56 256 1256 3256 3256 112 113 GVAAAA GHBAAA AAAAxx 3270 865 0 2 0 10 70 270 1270 3270 3270 140 141 UVAAAA HHBAAA HHHHxx 4808 866 0 0 8 8 8 808 808 4808 4808 16 17 YCAAAA IHBAAA OOOOxx 7938 867 0 2 8 18 38 938 1938 2938 7938 76 77 ITAAAA JHBAAA VVVVxx 4405 868 1 1 5 5 5 405 405 4405 4405 10 11 LNAAAA KHBAAA AAAAxx 2264 869 0 0 4 4 64 264 264 2264 2264 128 129 CJAAAA LHBAAA HHHHxx 80 870 0 0 0 0 80 80 80 80 80 160 161 CDAAAA MHBAAA OOOOxx 320 871 0 0 0 0 20 320 320 320 320 40 41 IMAAAA NHBAAA VVVVxx 2383 872 1 3 3 3 83 383 383 2383 2383 166 167 RNAAAA OHBAAA AAAAxx 3146 873 0 2 6 6 46 146 1146 3146 3146 92 93 ARAAAA PHBAAA HHHHxx 6911 874 1 3 1 11 11 911 911 1911 6911 22 23 VFAAAA QHBAAA OOOOxx 7377 875 1 1 7 17 77 377 1377 2377 7377 154 155 TXAAAA RHBAAA VVVVxx 9965 876 1 1 5 5 65 965 1965 4965 9965 130 131 HTAAAA SHBAAA AAAAxx 8361 877 1 1 1 1 61 361 361 3361 8361 122 123 PJAAAA THBAAA HHHHxx 9417 878 1 1 7 17 17 417 1417 4417 9417 34 35 FYAAAA UHBAAA OOOOxx 2483 879 1 3 3 3 83 483 483 2483 2483 166 167 NRAAAA VHBAAA VVVVxx 9843 880 1 3 3 3 43 843 1843 4843 9843 86 87 POAAAA WHBAAA AAAAxx 6395 881 1 3 5 15 95 395 395 1395 6395 190 191 ZLAAAA XHBAAA HHHHxx 6444 882 0 0 4 4 44 444 444 1444 6444 88 89 WNAAAA YHBAAA OOOOxx 1820 883 0 0 0 0 20 820 1820 1820 1820 40 41 ASAAAA ZHBAAA VVVVxx 2768 884 0 0 8 8 68 768 768 2768 2768 136 137 MCAAAA AIBAAA AAAAxx 5413 885 1 1 3 13 13 413 1413 413 5413 26 27 FAAAAA BIBAAA HHHHxx 2923 886 1 3 3 3 23 923 923 2923 2923 46 47 LIAAAA CIBAAA OOOOxx 5286 887 0 2 6 6 86 286 1286 286 5286 172 173 IVAAAA DIBAAA VVVVxx 6126 888 0 2 6 6 26 126 126 1126 6126 52 53 QBAAAA EIBAAA AAAAxx 8343 889 1 3 3 3 43 343 343 3343 8343 86 87 XIAAAA FIBAAA HHHHxx 6010 890 0 2 0 10 10 10 10 1010 6010 20 21 EXAAAA GIBAAA OOOOxx 4177 891 1 1 7 17 77 177 177 4177 4177 154 155 REAAAA HIBAAA VVVVxx 5808 892 0 0 8 8 8 808 1808 808 5808 16 17 KPAAAA IIBAAA AAAAxx 4859 893 1 3 9 19 59 859 859 4859 4859 118 119 XEAAAA JIBAAA HHHHxx 9252 894 0 0 2 12 52 252 1252 4252 9252 104 105 WRAAAA KIBAAA OOOOxx 2941 895 1 1 1 1 41 941 941 2941 2941 82 83 DJAAAA LIBAAA VVVVxx 8693 896 1 1 3 13 93 693 693 3693 8693 186 187 JWAAAA MIBAAA AAAAxx 4432 897 0 0 2 12 32 432 432 4432 4432 64 65 MOAAAA NIBAAA HHHHxx 2371 898 1 3 1 11 71 371 371 2371 2371 142 143 FNAAAA OIBAAA OOOOxx 7546 899 0 2 6 6 46 546 1546 2546 7546 92 93 GEAAAA PIBAAA VVVVxx 1369 900 1 1 9 9 69 369 1369 1369 1369 138 139 RAAAAA QIBAAA AAAAxx 4687 901 1 3 7 7 87 687 687 4687 4687 174 175 HYAAAA RIBAAA HHHHxx 8941 902 1 1 1 1 41 941 941 3941 8941 82 83 XFAAAA SIBAAA OOOOxx 226 903 0 2 6 6 26 226 226 226 226 52 53 SIAAAA TIBAAA VVVVxx 3493 904 1 1 3 13 93 493 1493 3493 3493 186 187 JEAAAA UIBAAA AAAAxx 6433 905 1 1 3 13 33 433 433 1433 6433 66 67 LNAAAA VIBAAA HHHHxx 9189 906 1 1 9 9 89 189 1189 4189 9189 178 179 LPAAAA WIBAAA OOOOxx 6027 907 1 3 7 7 27 27 27 1027 6027 54 55 VXAAAA XIBAAA VVVVxx 4615 908 1 3 5 15 15 615 615 4615 4615 30 31 NVAAAA YIBAAA AAAAxx 5320 909 0 0 0 0 20 320 1320 320 5320 40 41 QWAAAA ZIBAAA HHHHxx 7002 910 0 2 2 2 2 2 1002 2002 7002 4 5 IJAAAA AJBAAA OOOOxx 7367 911 1 3 7 7 67 367 1367 2367 7367 134 135 JXAAAA BJBAAA VVVVxx 289 912 1 1 9 9 89 289 289 289 289 178 179 DLAAAA CJBAAA AAAAxx 407 913 1 3 7 7 7 407 407 407 407 14 15 RPAAAA DJBAAA HHHHxx 504 914 0 0 4 4 4 504 504 504 504 8 9 KTAAAA EJBAAA OOOOxx 8301 915 1 1 1 1 1 301 301 3301 8301 2 3 HHAAAA FJBAAA VVVVxx 1396 916 0 0 6 16 96 396 1396 1396 1396 192 193 SBAAAA GJBAAA AAAAxx 4794 917 0 2 4 14 94 794 794 4794 4794 188 189 KCAAAA HJBAAA HHHHxx 6400 918 0 0 0 0 0 400 400 1400 6400 0 1 EMAAAA IJBAAA OOOOxx 1275 919 1 3 5 15 75 275 1275 1275 1275 150 151 BXAAAA JJBAAA VVVVxx 5797 920 1 1 7 17 97 797 1797 797 5797 194 195 ZOAAAA KJBAAA AAAAxx 2221 921 1 1 1 1 21 221 221 2221 2221 42 43 LHAAAA LJBAAA HHHHxx 2504 922 0 0 4 4 4 504 504 2504 2504 8 9 ISAAAA MJBAAA OOOOxx 2143 923 1 3 3 3 43 143 143 2143 2143 86 87 LEAAAA NJBAAA VVVVxx 1083 924 1 3 3 3 83 83 1083 1083 1083 166 167 RPAAAA OJBAAA AAAAxx 6148 925 0 0 8 8 48 148 148 1148 6148 96 97 MCAAAA PJBAAA HHHHxx 3612 926 0 0 2 12 12 612 1612 3612 3612 24 25 YIAAAA QJBAAA OOOOxx 9499 927 1 3 9 19 99 499 1499 4499 9499 198 199 JBAAAA RJBAAA VVVVxx 5773 928 1 1 3 13 73 773 1773 773 5773 146 147 BOAAAA SJBAAA AAAAxx 1014 929 0 2 4 14 14 14 1014 1014 1014 28 29 ANAAAA TJBAAA HHHHxx 1427 930 1 3 7 7 27 427 1427 1427 1427 54 55 XCAAAA UJBAAA OOOOxx 6770 931 0 2 0 10 70 770 770 1770 6770 140 141 KAAAAA VJBAAA VVVVxx 9042 932 0 2 2 2 42 42 1042 4042 9042 84 85 UJAAAA WJBAAA AAAAxx 9892 933 0 0 2 12 92 892 1892 4892 9892 184 185 MQAAAA XJBAAA HHHHxx 1771 934 1 3 1 11 71 771 1771 1771 1771 142 143 DQAAAA YJBAAA OOOOxx 7392 935 0 0 2 12 92 392 1392 2392 7392 184 185 IYAAAA ZJBAAA VVVVxx 4465 936 1 1 5 5 65 465 465 4465 4465 130 131 TPAAAA AKBAAA AAAAxx 278 937 0 2 8 18 78 278 278 278 278 156 157 SKAAAA BKBAAA HHHHxx 7776 938 0 0 6 16 76 776 1776 2776 7776 152 153 CNAAAA CKBAAA OOOOxx 3763 939 1 3 3 3 63 763 1763 3763 3763 126 127 TOAAAA DKBAAA VVVVxx 7503 940 1 3 3 3 3 503 1503 2503 7503 6 7 PCAAAA EKBAAA AAAAxx 3793 941 1 1 3 13 93 793 1793 3793 3793 186 187 XPAAAA FKBAAA HHHHxx 6510 942 0 2 0 10 10 510 510 1510 6510 20 21 KQAAAA GKBAAA OOOOxx 7641 943 1 1 1 1 41 641 1641 2641 7641 82 83 XHAAAA HKBAAA VVVVxx 3228 944 0 0 8 8 28 228 1228 3228 3228 56 57 EUAAAA IKBAAA AAAAxx 194 945 0 2 4 14 94 194 194 194 194 188 189 MHAAAA JKBAAA HHHHxx 8555 946 1 3 5 15 55 555 555 3555 8555 110 111 BRAAAA KKBAAA OOOOxx 4997 947 1 1 7 17 97 997 997 4997 4997 194 195 FKAAAA LKBAAA VVVVxx 8687 948 1 3 7 7 87 687 687 3687 8687 174 175 DWAAAA MKBAAA AAAAxx 6632 949 0 0 2 12 32 632 632 1632 6632 64 65 CVAAAA NKBAAA HHHHxx 9607 950 1 3 7 7 7 607 1607 4607 9607 14 15 NFAAAA OKBAAA OOOOxx 6201 951 1 1 1 1 1 201 201 1201 6201 2 3 NEAAAA PKBAAA VVVVxx 857 952 1 1 7 17 57 857 857 857 857 114 115 ZGAAAA QKBAAA AAAAxx 5623 953 1 3 3 3 23 623 1623 623 5623 46 47 HIAAAA RKBAAA HHHHxx 5979 954 1 3 9 19 79 979 1979 979 5979 158 159 ZVAAAA SKBAAA OOOOxx 2201 955 1 1 1 1 1 201 201 2201 2201 2 3 RGAAAA TKBAAA VVVVxx 3166 956 0 2 6 6 66 166 1166 3166 3166 132 133 URAAAA UKBAAA AAAAxx 6249 957 1 1 9 9 49 249 249 1249 6249 98 99 JGAAAA VKBAAA HHHHxx 3271 958 1 3 1 11 71 271 1271 3271 3271 142 143 VVAAAA WKBAAA OOOOxx 7777 959 1 1 7 17 77 777 1777 2777 7777 154 155 DNAAAA XKBAAA VVVVxx 6732 960 0 0 2 12 32 732 732 1732 6732 64 65 YYAAAA YKBAAA AAAAxx 6297 961 1 1 7 17 97 297 297 1297 6297 194 195 FIAAAA ZKBAAA HHHHxx 5685 962 1 1 5 5 85 685 1685 685 5685 170 171 RKAAAA ALBAAA OOOOxx 9931 963 1 3 1 11 31 931 1931 4931 9931 62 63 ZRAAAA BLBAAA VVVVxx 7485 964 1 1 5 5 85 485 1485 2485 7485 170 171 XBAAAA CLBAAA AAAAxx 386 965 0 2 6 6 86 386 386 386 386 172 173 WOAAAA DLBAAA HHHHxx 8204 966 0 0 4 4 4 204 204 3204 8204 8 9 ODAAAA ELBAAA OOOOxx 3606 967 0 2 6 6 6 606 1606 3606 3606 12 13 SIAAAA FLBAAA VVVVxx 1692 968 0 0 2 12 92 692 1692 1692 1692 184 185 CNAAAA GLBAAA AAAAxx 3002 969 0 2 2 2 2 2 1002 3002 3002 4 5 MLAAAA HLBAAA HHHHxx 9676 970 0 0 6 16 76 676 1676 4676 9676 152 153 EIAAAA ILBAAA OOOOxx 915 971 1 3 5 15 15 915 915 915 915 30 31 FJAAAA JLBAAA VVVVxx 7706 972 0 2 6 6 6 706 1706 2706 7706 12 13 KKAAAA KLBAAA AAAAxx 6080 973 0 0 0 0 80 80 80 1080 6080 160 161 WZAAAA LLBAAA HHHHxx 1860 974 0 0 0 0 60 860 1860 1860 1860 120 121 OTAAAA MLBAAA OOOOxx 1444 975 0 0 4 4 44 444 1444 1444 1444 88 89 ODAAAA NLBAAA VVVVxx 7208 976 0 0 8 8 8 208 1208 2208 7208 16 17 GRAAAA OLBAAA AAAAxx 8554 977 0 2 4 14 54 554 554 3554 8554 108 109 ARAAAA PLBAAA HHHHxx 2028 978 0 0 8 8 28 28 28 2028 2028 56 57 AAAAAA QLBAAA OOOOxx 9893 979 1 1 3 13 93 893 1893 4893 9893 186 187 NQAAAA RLBAAA VVVVxx 4740 980 0 0 0 0 40 740 740 4740 4740 80 81 IAAAAA SLBAAA AAAAxx 6186 981 0 2 6 6 86 186 186 1186 6186 172 173 YDAAAA TLBAAA HHHHxx 6357 982 1 1 7 17 57 357 357 1357 6357 114 115 NKAAAA ULBAAA OOOOxx 3699 983 1 3 9 19 99 699 1699 3699 3699 198 199 HMAAAA VLBAAA VVVVxx 7620 984 0 0 0 0 20 620 1620 2620 7620 40 41 CHAAAA WLBAAA AAAAxx 921 985 1 1 1 1 21 921 921 921 921 42 43 LJAAAA XLBAAA HHHHxx 5506 986 0 2 6 6 6 506 1506 506 5506 12 13 UDAAAA YLBAAA OOOOxx 8851 987 1 3 1 11 51 851 851 3851 8851 102 103 LCAAAA ZLBAAA VVVVxx 3205 988 1 1 5 5 5 205 1205 3205 3205 10 11 HTAAAA AMBAAA AAAAxx 1956 989 0 0 6 16 56 956 1956 1956 1956 112 113 GXAAAA BMBAAA HHHHxx 6272 990 0 0 2 12 72 272 272 1272 6272 144 145 GHAAAA CMBAAA OOOOxx 1509 991 1 1 9 9 9 509 1509 1509 1509 18 19 BGAAAA DMBAAA VVVVxx 53 992 1 1 3 13 53 53 53 53 53 106 107 BCAAAA EMBAAA AAAAxx 213 993 1 1 3 13 13 213 213 213 213 26 27 FIAAAA FMBAAA HHHHxx 4924 994 0 0 4 4 24 924 924 4924 4924 48 49 KHAAAA GMBAAA OOOOxx 2097 995 1 1 7 17 97 97 97 2097 2097 194 195 RCAAAA HMBAAA VVVVxx 4607 996 1 3 7 7 7 607 607 4607 4607 14 15 FVAAAA IMBAAA AAAAxx 1582 997 0 2 2 2 82 582 1582 1582 1582 164 165 WIAAAA JMBAAA HHHHxx 6643 998 1 3 3 3 43 643 643 1643 6643 86 87 NVAAAA KMBAAA OOOOxx 2238 999 0 2 8 18 38 238 238 2238 2238 76 77 CIAAAA LMBAAA VVVVxx 2942 1000 0 2 2 2 42 942 942 2942 2942 84 85 EJAAAA MMBAAA AAAAxx 1655 1001 1 3 5 15 55 655 1655 1655 1655 110 111 RLAAAA NMBAAA HHHHxx 3226 1002 0 2 6 6 26 226 1226 3226 3226 52 53 CUAAAA OMBAAA OOOOxx 4263 1003 1 3 3 3 63 263 263 4263 4263 126 127 ZHAAAA PMBAAA VVVVxx 960 1004 0 0 0 0 60 960 960 960 960 120 121 YKAAAA QMBAAA AAAAxx 1213 1005 1 1 3 13 13 213 1213 1213 1213 26 27 RUAAAA RMBAAA HHHHxx 1845 1006 1 1 5 5 45 845 1845 1845 1845 90 91 ZSAAAA SMBAAA OOOOxx 6944 1007 0 0 4 4 44 944 944 1944 6944 88 89 CHAAAA TMBAAA VVVVxx 5284 1008 0 0 4 4 84 284 1284 284 5284 168 169 GVAAAA UMBAAA AAAAxx 188 1009 0 0 8 8 88 188 188 188 188 176 177 GHAAAA VMBAAA HHHHxx 748 1010 0 0 8 8 48 748 748 748 748 96 97 UCAAAA WMBAAA OOOOxx 2226 1011 0 2 6 6 26 226 226 2226 2226 52 53 QHAAAA XMBAAA VVVVxx 7342 1012 0 2 2 2 42 342 1342 2342 7342 84 85 KWAAAA YMBAAA AAAAxx 6120 1013 0 0 0 0 20 120 120 1120 6120 40 41 KBAAAA ZMBAAA HHHHxx 536 1014 0 0 6 16 36 536 536 536 536 72 73 QUAAAA ANBAAA OOOOxx 3239 1015 1 3 9 19 39 239 1239 3239 3239 78 79 PUAAAA BNBAAA VVVVxx 2832 1016 0 0 2 12 32 832 832 2832 2832 64 65 YEAAAA CNBAAA AAAAxx 5296 1017 0 0 6 16 96 296 1296 296 5296 192 193 SVAAAA DNBAAA HHHHxx 5795 1018 1 3 5 15 95 795 1795 795 5795 190 191 XOAAAA ENBAAA OOOOxx 6290 1019 0 2 0 10 90 290 290 1290 6290 180 181 YHAAAA FNBAAA VVVVxx 4916 1020 0 0 6 16 16 916 916 4916 4916 32 33 CHAAAA GNBAAA AAAAxx 8366 1021 0 2 6 6 66 366 366 3366 8366 132 133 UJAAAA HNBAAA HHHHxx 4248 1022 0 0 8 8 48 248 248 4248 4248 96 97 KHAAAA INBAAA OOOOxx 6460 1023 0 0 0 0 60 460 460 1460 6460 120 121 MOAAAA JNBAAA VVVVxx 9296 1024 0 0 6 16 96 296 1296 4296 9296 192 193 OTAAAA KNBAAA AAAAxx 3486 1025 0 2 6 6 86 486 1486 3486 3486 172 173 CEAAAA LNBAAA HHHHxx 5664 1026 0 0 4 4 64 664 1664 664 5664 128 129 WJAAAA MNBAAA OOOOxx 7624 1027 0 0 4 4 24 624 1624 2624 7624 48 49 GHAAAA NNBAAA VVVVxx 2790 1028 0 2 0 10 90 790 790 2790 2790 180 181 IDAAAA ONBAAA AAAAxx 682 1029 0 2 2 2 82 682 682 682 682 164 165 GAAAAA PNBAAA HHHHxx 6412 1030 0 0 2 12 12 412 412 1412 6412 24 25 QMAAAA QNBAAA OOOOxx 6882 1031 0 2 2 2 82 882 882 1882 6882 164 165 SEAAAA RNBAAA VVVVxx 1332 1032 0 0 2 12 32 332 1332 1332 1332 64 65 GZAAAA SNBAAA AAAAxx 4911 1033 1 3 1 11 11 911 911 4911 4911 22 23 XGAAAA TNBAAA HHHHxx 3528 1034 0 0 8 8 28 528 1528 3528 3528 56 57 SFAAAA UNBAAA OOOOxx 271 1035 1 3 1 11 71 271 271 271 271 142 143 LKAAAA VNBAAA VVVVxx 7007 1036 1 3 7 7 7 7 1007 2007 7007 14 15 NJAAAA WNBAAA AAAAxx 2198 1037 0 2 8 18 98 198 198 2198 2198 196 197 OGAAAA XNBAAA HHHHxx 4266 1038 0 2 6 6 66 266 266 4266 4266 132 133 CIAAAA YNBAAA OOOOxx 9867 1039 1 3 7 7 67 867 1867 4867 9867 134 135 NPAAAA ZNBAAA VVVVxx 7602 1040 0 2 2 2 2 602 1602 2602 7602 4 5 KGAAAA AOBAAA AAAAxx 7521 1041 1 1 1 1 21 521 1521 2521 7521 42 43 HDAAAA BOBAAA HHHHxx 7200 1042 0 0 0 0 0 200 1200 2200 7200 0 1 YQAAAA COBAAA OOOOxx 4816 1043 0 0 6 16 16 816 816 4816 4816 32 33 GDAAAA DOBAAA VVVVxx 1669 1044 1 1 9 9 69 669 1669 1669 1669 138 139 FMAAAA EOBAAA AAAAxx 4764 1045 0 0 4 4 64 764 764 4764 4764 128 129 GBAAAA FOBAAA HHHHxx 7393 1046 1 1 3 13 93 393 1393 2393 7393 186 187 JYAAAA GOBAAA OOOOxx 7434 1047 0 2 4 14 34 434 1434 2434 7434 68 69 YZAAAA HOBAAA VVVVxx 9079 1048 1 3 9 19 79 79 1079 4079 9079 158 159 FLAAAA IOBAAA AAAAxx 9668 1049 0 0 8 8 68 668 1668 4668 9668 136 137 WHAAAA JOBAAA HHHHxx 7184 1050 0 0 4 4 84 184 1184 2184 7184 168 169 IQAAAA KOBAAA OOOOxx 7347 1051 1 3 7 7 47 347 1347 2347 7347 94 95 PWAAAA LOBAAA VVVVxx 951 1052 1 3 1 11 51 951 951 951 951 102 103 PKAAAA MOBAAA AAAAxx 4513 1053 1 1 3 13 13 513 513 4513 4513 26 27 PRAAAA NOBAAA HHHHxx 2692 1054 0 0 2 12 92 692 692 2692 2692 184 185 OZAAAA OOBAAA OOOOxx 9930 1055 0 2 0 10 30 930 1930 4930 9930 60 61 YRAAAA POBAAA VVVVxx 4516 1056 0 0 6 16 16 516 516 4516 4516 32 33 SRAAAA QOBAAA AAAAxx 1592 1057 0 0 2 12 92 592 1592 1592 1592 184 185 GJAAAA ROBAAA HHHHxx 6312 1058 0 0 2 12 12 312 312 1312 6312 24 25 UIAAAA SOBAAA OOOOxx 185 1059 1 1 5 5 85 185 185 185 185 170 171 DHAAAA TOBAAA VVVVxx 1848 1060 0 0 8 8 48 848 1848 1848 1848 96 97 CTAAAA UOBAAA AAAAxx 5844 1061 0 0 4 4 44 844 1844 844 5844 88 89 UQAAAA VOBAAA HHHHxx 1666 1062 0 2 6 6 66 666 1666 1666 1666 132 133 CMAAAA WOBAAA OOOOxx 5864 1063 0 0 4 4 64 864 1864 864 5864 128 129 ORAAAA XOBAAA VVVVxx 1004 1064 0 0 4 4 4 4 1004 1004 1004 8 9 QMAAAA YOBAAA AAAAxx 1758 1065 0 2 8 18 58 758 1758 1758 1758 116 117 QPAAAA ZOBAAA HHHHxx 8823 1066 1 3 3 3 23 823 823 3823 8823 46 47 JBAAAA APBAAA OOOOxx 129 1067 1 1 9 9 29 129 129 129 129 58 59 ZEAAAA BPBAAA VVVVxx 5703 1068 1 3 3 3 3 703 1703 703 5703 6 7 JLAAAA CPBAAA AAAAxx 3331 1069 1 3 1 11 31 331 1331 3331 3331 62 63 DYAAAA DPBAAA HHHHxx 5791 1070 1 3 1 11 91 791 1791 791 5791 182 183 TOAAAA EPBAAA OOOOxx 4421 1071 1 1 1 1 21 421 421 4421 4421 42 43 BOAAAA FPBAAA VVVVxx 9740 1072 0 0 0 0 40 740 1740 4740 9740 80 81 QKAAAA GPBAAA AAAAxx 798 1073 0 2 8 18 98 798 798 798 798 196 197 SEAAAA HPBAAA HHHHxx 571 1074 1 3 1 11 71 571 571 571 571 142 143 ZVAAAA IPBAAA OOOOxx 7084 1075 0 0 4 4 84 84 1084 2084 7084 168 169 MMAAAA JPBAAA VVVVxx 650 1076 0 2 0 10 50 650 650 650 650 100 101 AZAAAA KPBAAA AAAAxx 1467 1077 1 3 7 7 67 467 1467 1467 1467 134 135 LEAAAA LPBAAA HHHHxx 5446 1078 0 2 6 6 46 446 1446 446 5446 92 93 MBAAAA MPBAAA OOOOxx 830 1079 0 2 0 10 30 830 830 830 830 60 61 YFAAAA NPBAAA VVVVxx 5516 1080 0 0 6 16 16 516 1516 516 5516 32 33 EEAAAA OPBAAA AAAAxx 8520 1081 0 0 0 0 20 520 520 3520 8520 40 41 SPAAAA PPBAAA HHHHxx 1152 1082 0 0 2 12 52 152 1152 1152 1152 104 105 ISAAAA QPBAAA OOOOxx 862 1083 0 2 2 2 62 862 862 862 862 124 125 EHAAAA RPBAAA VVVVxx 454 1084 0 2 4 14 54 454 454 454 454 108 109 MRAAAA SPBAAA AAAAxx 9956 1085 0 0 6 16 56 956 1956 4956 9956 112 113 YSAAAA TPBAAA HHHHxx 1654 1086 0 2 4 14 54 654 1654 1654 1654 108 109 QLAAAA UPBAAA OOOOxx 257 1087 1 1 7 17 57 257 257 257 257 114 115 XJAAAA VPBAAA VVVVxx 5469 1088 1 1 9 9 69 469 1469 469 5469 138 139 JCAAAA WPBAAA AAAAxx 9075 1089 1 3 5 15 75 75 1075 4075 9075 150 151 BLAAAA XPBAAA HHHHxx 7799 1090 1 3 9 19 99 799 1799 2799 7799 198 199 ZNAAAA YPBAAA OOOOxx 2001 1091 1 1 1 1 1 1 1 2001 2001 2 3 ZYAAAA ZPBAAA VVVVxx 9786 1092 0 2 6 6 86 786 1786 4786 9786 172 173 KMAAAA AQBAAA AAAAxx 7281 1093 1 1 1 1 81 281 1281 2281 7281 162 163 BUAAAA BQBAAA HHHHxx 5137 1094 1 1 7 17 37 137 1137 137 5137 74 75 PPAAAA CQBAAA OOOOxx 4053 1095 1 1 3 13 53 53 53 4053 4053 106 107 XZAAAA DQBAAA VVVVxx 7911 1096 1 3 1 11 11 911 1911 2911 7911 22 23 HSAAAA EQBAAA AAAAxx 4298 1097 0 2 8 18 98 298 298 4298 4298 196 197 IJAAAA FQBAAA HHHHxx 4805 1098 1 1 5 5 5 805 805 4805 4805 10 11 VCAAAA GQBAAA OOOOxx 9038 1099 0 2 8 18 38 38 1038 4038 9038 76 77 QJAAAA HQBAAA VVVVxx 8023 1100 1 3 3 3 23 23 23 3023 8023 46 47 PWAAAA IQBAAA AAAAxx 6595 1101 1 3 5 15 95 595 595 1595 6595 190 191 RTAAAA JQBAAA HHHHxx 9831 1102 1 3 1 11 31 831 1831 4831 9831 62 63 DOAAAA KQBAAA OOOOxx 788 1103 0 0 8 8 88 788 788 788 788 176 177 IEAAAA LQBAAA VVVVxx 902 1104 0 2 2 2 2 902 902 902 902 4 5 SIAAAA MQBAAA AAAAxx 9137 1105 1 1 7 17 37 137 1137 4137 9137 74 75 LNAAAA NQBAAA HHHHxx 1744 1106 0 0 4 4 44 744 1744 1744 1744 88 89 CPAAAA OQBAAA OOOOxx 7285 1107 1 1 5 5 85 285 1285 2285 7285 170 171 FUAAAA PQBAAA VVVVxx 7006 1108 0 2 6 6 6 6 1006 2006 7006 12 13 MJAAAA QQBAAA AAAAxx 9236 1109 0 0 6 16 36 236 1236 4236 9236 72 73 GRAAAA RQBAAA HHHHxx 5472 1110 0 0 2 12 72 472 1472 472 5472 144 145 MCAAAA SQBAAA OOOOxx 7975 1111 1 3 5 15 75 975 1975 2975 7975 150 151 TUAAAA TQBAAA VVVVxx 4181 1112 1 1 1 1 81 181 181 4181 4181 162 163 VEAAAA UQBAAA AAAAxx 7677 1113 1 1 7 17 77 677 1677 2677 7677 154 155 HJAAAA VQBAAA HHHHxx 35 1114 1 3 5 15 35 35 35 35 35 70 71 JBAAAA WQBAAA OOOOxx 6813 1115 1 1 3 13 13 813 813 1813 6813 26 27 BCAAAA XQBAAA VVVVxx 6618 1116 0 2 8 18 18 618 618 1618 6618 36 37 OUAAAA YQBAAA AAAAxx 8069 1117 1 1 9 9 69 69 69 3069 8069 138 139 JYAAAA ZQBAAA HHHHxx 3071 1118 1 3 1 11 71 71 1071 3071 3071 142 143 DOAAAA ARBAAA OOOOxx 4390 1119 0 2 0 10 90 390 390 4390 4390 180 181 WMAAAA BRBAAA VVVVxx 7764 1120 0 0 4 4 64 764 1764 2764 7764 128 129 QMAAAA CRBAAA AAAAxx 8163 1121 1 3 3 3 63 163 163 3163 8163 126 127 ZBAAAA DRBAAA HHHHxx 1961 1122 1 1 1 1 61 961 1961 1961 1961 122 123 LXAAAA ERBAAA OOOOxx 1103 1123 1 3 3 3 3 103 1103 1103 1103 6 7 LQAAAA FRBAAA VVVVxx 5486 1124 0 2 6 6 86 486 1486 486 5486 172 173 ADAAAA GRBAAA AAAAxx 9513 1125 1 1 3 13 13 513 1513 4513 9513 26 27 XBAAAA HRBAAA HHHHxx 7311 1126 1 3 1 11 11 311 1311 2311 7311 22 23 FVAAAA IRBAAA OOOOxx 4144 1127 0 0 4 4 44 144 144 4144 4144 88 89 KDAAAA JRBAAA VVVVxx 7901 1128 1 1 1 1 1 901 1901 2901 7901 2 3 XRAAAA KRBAAA AAAAxx 4629 1129 1 1 9 9 29 629 629 4629 4629 58 59 BWAAAA LRBAAA HHHHxx 6858 1130 0 2 8 18 58 858 858 1858 6858 116 117 UDAAAA MRBAAA OOOOxx 125 1131 1 1 5 5 25 125 125 125 125 50 51 VEAAAA NRBAAA VVVVxx 3834 1132 0 2 4 14 34 834 1834 3834 3834 68 69 MRAAAA ORBAAA AAAAxx 8155 1133 1 3 5 15 55 155 155 3155 8155 110 111 RBAAAA PRBAAA HHHHxx 8230 1134 0 2 0 10 30 230 230 3230 8230 60 61 OEAAAA QRBAAA OOOOxx 744 1135 0 0 4 4 44 744 744 744 744 88 89 QCAAAA RRBAAA VVVVxx 357 1136 1 1 7 17 57 357 357 357 357 114 115 TNAAAA SRBAAA AAAAxx 2159 1137 1 3 9 19 59 159 159 2159 2159 118 119 BFAAAA TRBAAA HHHHxx 8559 1138 1 3 9 19 59 559 559 3559 8559 118 119 FRAAAA URBAAA OOOOxx 6866 1139 0 2 6 6 66 866 866 1866 6866 132 133 CEAAAA VRBAAA VVVVxx 3863 1140 1 3 3 3 63 863 1863 3863 3863 126 127 PSAAAA WRBAAA AAAAxx 4193 1141 1 1 3 13 93 193 193 4193 4193 186 187 HFAAAA XRBAAA HHHHxx 3277 1142 1 1 7 17 77 277 1277 3277 3277 154 155 BWAAAA YRBAAA OOOOxx 5577 1143 1 1 7 17 77 577 1577 577 5577 154 155 NGAAAA ZRBAAA VVVVxx 9503 1144 1 3 3 3 3 503 1503 4503 9503 6 7 NBAAAA ASBAAA AAAAxx 7642 1145 0 2 2 2 42 642 1642 2642 7642 84 85 YHAAAA BSBAAA HHHHxx 6197 1146 1 1 7 17 97 197 197 1197 6197 194 195 JEAAAA CSBAAA OOOOxx 8995 1147 1 3 5 15 95 995 995 3995 8995 190 191 ZHAAAA DSBAAA VVVVxx 440 1148 0 0 0 0 40 440 440 440 440 80 81 YQAAAA ESBAAA AAAAxx 8418 1149 0 2 8 18 18 418 418 3418 8418 36 37 ULAAAA FSBAAA HHHHxx 8531 1150 1 3 1 11 31 531 531 3531 8531 62 63 DQAAAA GSBAAA OOOOxx 3790 1151 0 2 0 10 90 790 1790 3790 3790 180 181 UPAAAA HSBAAA VVVVxx 7610 1152 0 2 0 10 10 610 1610 2610 7610 20 21 SGAAAA ISBAAA AAAAxx 1252 1153 0 0 2 12 52 252 1252 1252 1252 104 105 EWAAAA JSBAAA HHHHxx 7559 1154 1 3 9 19 59 559 1559 2559 7559 118 119 TEAAAA KSBAAA OOOOxx 9945 1155 1 1 5 5 45 945 1945 4945 9945 90 91 NSAAAA LSBAAA VVVVxx 9023 1156 1 3 3 3 23 23 1023 4023 9023 46 47 BJAAAA MSBAAA AAAAxx 3516 1157 0 0 6 16 16 516 1516 3516 3516 32 33 GFAAAA NSBAAA HHHHxx 4671 1158 1 3 1 11 71 671 671 4671 4671 142 143 RXAAAA OSBAAA OOOOxx 1465 1159 1 1 5 5 65 465 1465 1465 1465 130 131 JEAAAA PSBAAA VVVVxx 9515 1160 1 3 5 15 15 515 1515 4515 9515 30 31 ZBAAAA QSBAAA AAAAxx 3242 1161 0 2 2 2 42 242 1242 3242 3242 84 85 SUAAAA RSBAAA HHHHxx 1732 1162 0 0 2 12 32 732 1732 1732 1732 64 65 QOAAAA SSBAAA OOOOxx 1678 1163 0 2 8 18 78 678 1678 1678 1678 156 157 OMAAAA TSBAAA VVVVxx 1464 1164 0 0 4 4 64 464 1464 1464 1464 128 129 IEAAAA USBAAA AAAAxx 6546 1165 0 2 6 6 46 546 546 1546 6546 92 93 URAAAA VSBAAA HHHHxx 4448 1166 0 0 8 8 48 448 448 4448 4448 96 97 CPAAAA WSBAAA OOOOxx 9847 1167 1 3 7 7 47 847 1847 4847 9847 94 95 TOAAAA XSBAAA VVVVxx 8264 1168 0 0 4 4 64 264 264 3264 8264 128 129 WFAAAA YSBAAA AAAAxx 1620 1169 0 0 0 0 20 620 1620 1620 1620 40 41 IKAAAA ZSBAAA HHHHxx 9388 1170 0 0 8 8 88 388 1388 4388 9388 176 177 CXAAAA ATBAAA OOOOxx 6445 1171 1 1 5 5 45 445 445 1445 6445 90 91 XNAAAA BTBAAA VVVVxx 4789 1172 1 1 9 9 89 789 789 4789 4789 178 179 FCAAAA CTBAAA AAAAxx 1562 1173 0 2 2 2 62 562 1562 1562 1562 124 125 CIAAAA DTBAAA HHHHxx 7305 1174 1 1 5 5 5 305 1305 2305 7305 10 11 ZUAAAA ETBAAA OOOOxx 6344 1175 0 0 4 4 44 344 344 1344 6344 88 89 AKAAAA FTBAAA VVVVxx 5130 1176 0 2 0 10 30 130 1130 130 5130 60 61 IPAAAA GTBAAA AAAAxx 3284 1177 0 0 4 4 84 284 1284 3284 3284 168 169 IWAAAA HTBAAA HHHHxx 6346 1178 0 2 6 6 46 346 346 1346 6346 92 93 CKAAAA ITBAAA OOOOxx 1061 1179 1 1 1 1 61 61 1061 1061 1061 122 123 VOAAAA JTBAAA VVVVxx 872 1180 0 0 2 12 72 872 872 872 872 144 145 OHAAAA KTBAAA AAAAxx 123 1181 1 3 3 3 23 123 123 123 123 46 47 TEAAAA LTBAAA HHHHxx 7903 1182 1 3 3 3 3 903 1903 2903 7903 6 7 ZRAAAA MTBAAA OOOOxx 560 1183 0 0 0 0 60 560 560 560 560 120 121 OVAAAA NTBAAA VVVVxx 4446 1184 0 2 6 6 46 446 446 4446 4446 92 93 APAAAA OTBAAA AAAAxx 3909 1185 1 1 9 9 9 909 1909 3909 3909 18 19 JUAAAA PTBAAA HHHHxx 669 1186 1 1 9 9 69 669 669 669 669 138 139 TZAAAA QTBAAA OOOOxx 7843 1187 1 3 3 3 43 843 1843 2843 7843 86 87 RPAAAA RTBAAA VVVVxx 2546 1188 0 2 6 6 46 546 546 2546 2546 92 93 YTAAAA STBAAA AAAAxx 6757 1189 1 1 7 17 57 757 757 1757 6757 114 115 XZAAAA TTBAAA HHHHxx 466 1190 0 2 6 6 66 466 466 466 466 132 133 YRAAAA UTBAAA OOOOxx 5556 1191 0 0 6 16 56 556 1556 556 5556 112 113 SFAAAA VTBAAA VVVVxx 7196 1192 0 0 6 16 96 196 1196 2196 7196 192 193 UQAAAA WTBAAA AAAAxx 2947 1193 1 3 7 7 47 947 947 2947 2947 94 95 JJAAAA XTBAAA HHHHxx 6493 1194 1 1 3 13 93 493 493 1493 6493 186 187 TPAAAA YTBAAA OOOOxx 7203 1195 1 3 3 3 3 203 1203 2203 7203 6 7 BRAAAA ZTBAAA VVVVxx 3716 1196 0 0 6 16 16 716 1716 3716 3716 32 33 YMAAAA AUBAAA AAAAxx 8058 1197 0 2 8 18 58 58 58 3058 8058 116 117 YXAAAA BUBAAA HHHHxx 433 1198 1 1 3 13 33 433 433 433 433 66 67 RQAAAA CUBAAA OOOOxx 7649 1199 1 1 9 9 49 649 1649 2649 7649 98 99 FIAAAA DUBAAA VVVVxx 6966 1200 0 2 6 6 66 966 966 1966 6966 132 133 YHAAAA EUBAAA AAAAxx 553 1201 1 1 3 13 53 553 553 553 553 106 107 HVAAAA FUBAAA HHHHxx 3677 1202 1 1 7 17 77 677 1677 3677 3677 154 155 LLAAAA GUBAAA OOOOxx 2344 1203 0 0 4 4 44 344 344 2344 2344 88 89 EMAAAA HUBAAA VVVVxx 7439 1204 1 3 9 19 39 439 1439 2439 7439 78 79 DAAAAA IUBAAA AAAAxx 3910 1205 0 2 0 10 10 910 1910 3910 3910 20 21 KUAAAA JUBAAA HHHHxx 3638 1206 0 2 8 18 38 638 1638 3638 3638 76 77 YJAAAA KUBAAA OOOOxx 6637 1207 1 1 7 17 37 637 637 1637 6637 74 75 HVAAAA LUBAAA VVVVxx 4438 1208 0 2 8 18 38 438 438 4438 4438 76 77 SOAAAA MUBAAA AAAAxx 171 1209 1 3 1 11 71 171 171 171 171 142 143 PGAAAA NUBAAA HHHHxx 310 1210 0 2 0 10 10 310 310 310 310 20 21 YLAAAA OUBAAA OOOOxx 2714 1211 0 2 4 14 14 714 714 2714 2714 28 29 KAAAAA PUBAAA VVVVxx 5199 1212 1 3 9 19 99 199 1199 199 5199 198 199 ZRAAAA QUBAAA AAAAxx 8005 1213 1 1 5 5 5 5 5 3005 8005 10 11 XVAAAA RUBAAA HHHHxx 3188 1214 0 0 8 8 88 188 1188 3188 3188 176 177 QSAAAA SUBAAA OOOOxx 1518 1215 0 2 8 18 18 518 1518 1518 1518 36 37 KGAAAA TUBAAA VVVVxx 6760 1216 0 0 0 0 60 760 760 1760 6760 120 121 AAAAAA UUBAAA AAAAxx 9373 1217 1 1 3 13 73 373 1373 4373 9373 146 147 NWAAAA VUBAAA HHHHxx 1938 1218 0 2 8 18 38 938 1938 1938 1938 76 77 OWAAAA WUBAAA OOOOxx 2865 1219 1 1 5 5 65 865 865 2865 2865 130 131 FGAAAA XUBAAA VVVVxx 3203 1220 1 3 3 3 3 203 1203 3203 3203 6 7 FTAAAA YUBAAA AAAAxx 6025 1221 1 1 5 5 25 25 25 1025 6025 50 51 TXAAAA ZUBAAA HHHHxx 8684 1222 0 0 4 4 84 684 684 3684 8684 168 169 AWAAAA AVBAAA OOOOxx 7732 1223 0 0 2 12 32 732 1732 2732 7732 64 65 KLAAAA BVBAAA VVVVxx 3218 1224 0 2 8 18 18 218 1218 3218 3218 36 37 UTAAAA CVBAAA AAAAxx 525 1225 1 1 5 5 25 525 525 525 525 50 51 FUAAAA DVBAAA HHHHxx 601 1226 1 1 1 1 1 601 601 601 601 2 3 DXAAAA EVBAAA OOOOxx 6091 1227 1 3 1 11 91 91 91 1091 6091 182 183 HAAAAA FVBAAA VVVVxx 4498 1228 0 2 8 18 98 498 498 4498 4498 196 197 ARAAAA GVBAAA AAAAxx 8192 1229 0 0 2 12 92 192 192 3192 8192 184 185 CDAAAA HVBAAA HHHHxx 8006 1230 0 2 6 6 6 6 6 3006 8006 12 13 YVAAAA IVBAAA OOOOxx 6157 1231 1 1 7 17 57 157 157 1157 6157 114 115 VCAAAA JVBAAA VVVVxx 312 1232 0 0 2 12 12 312 312 312 312 24 25 AMAAAA KVBAAA AAAAxx 8652 1233 0 0 2 12 52 652 652 3652 8652 104 105 UUAAAA LVBAAA HHHHxx 2787 1234 1 3 7 7 87 787 787 2787 2787 174 175 FDAAAA MVBAAA OOOOxx 1782 1235 0 2 2 2 82 782 1782 1782 1782 164 165 OQAAAA NVBAAA VVVVxx 23 1236 1 3 3 3 23 23 23 23 23 46 47 XAAAAA OVBAAA AAAAxx 1206 1237 0 2 6 6 6 206 1206 1206 1206 12 13 KUAAAA PVBAAA HHHHxx 1076 1238 0 0 6 16 76 76 1076 1076 1076 152 153 KPAAAA QVBAAA OOOOxx 5379 1239 1 3 9 19 79 379 1379 379 5379 158 159 XYAAAA RVBAAA VVVVxx 2047 1240 1 3 7 7 47 47 47 2047 2047 94 95 TAAAAA SVBAAA AAAAxx 6262 1241 0 2 2 2 62 262 262 1262 6262 124 125 WGAAAA TVBAAA HHHHxx 1840 1242 0 0 0 0 40 840 1840 1840 1840 80 81 USAAAA UVBAAA OOOOxx 2106 1243 0 2 6 6 6 106 106 2106 2106 12 13 ADAAAA VVBAAA VVVVxx 1307 1244 1 3 7 7 7 307 1307 1307 1307 14 15 HYAAAA WVBAAA AAAAxx 735 1245 1 3 5 15 35 735 735 735 735 70 71 HCAAAA XVBAAA HHHHxx 3657 1246 1 1 7 17 57 657 1657 3657 3657 114 115 RKAAAA YVBAAA OOOOxx 3006 1247 0 2 6 6 6 6 1006 3006 3006 12 13 QLAAAA ZVBAAA VVVVxx 1538 1248 0 2 8 18 38 538 1538 1538 1538 76 77 EHAAAA AWBAAA AAAAxx 6098 1249 0 2 8 18 98 98 98 1098 6098 196 197 OAAAAA BWBAAA HHHHxx 5267 1250 1 3 7 7 67 267 1267 267 5267 134 135 PUAAAA CWBAAA OOOOxx 9757 1251 1 1 7 17 57 757 1757 4757 9757 114 115 HLAAAA DWBAAA VVVVxx 1236 1252 0 0 6 16 36 236 1236 1236 1236 72 73 OVAAAA EWBAAA AAAAxx 83 1253 1 3 3 3 83 83 83 83 83 166 167 FDAAAA FWBAAA HHHHxx 9227 1254 1 3 7 7 27 227 1227 4227 9227 54 55 XQAAAA GWBAAA OOOOxx 8772 1255 0 0 2 12 72 772 772 3772 8772 144 145 KZAAAA HWBAAA VVVVxx 8822 1256 0 2 2 2 22 822 822 3822 8822 44 45 IBAAAA IWBAAA AAAAxx 7167 1257 1 3 7 7 67 167 1167 2167 7167 134 135 RPAAAA JWBAAA HHHHxx 6909 1258 1 1 9 9 9 909 909 1909 6909 18 19 TFAAAA KWBAAA OOOOxx 1439 1259 1 3 9 19 39 439 1439 1439 1439 78 79 JDAAAA LWBAAA VVVVxx 2370 1260 0 2 0 10 70 370 370 2370 2370 140 141 ENAAAA MWBAAA AAAAxx 4577 1261 1 1 7 17 77 577 577 4577 4577 154 155 BUAAAA NWBAAA HHHHxx 2575 1262 1 3 5 15 75 575 575 2575 2575 150 151 BVAAAA OWBAAA OOOOxx 2795 1263 1 3 5 15 95 795 795 2795 2795 190 191 NDAAAA PWBAAA VVVVxx 5520 1264 0 0 0 0 20 520 1520 520 5520 40 41 IEAAAA QWBAAA AAAAxx 382 1265 0 2 2 2 82 382 382 382 382 164 165 SOAAAA RWBAAA HHHHxx 6335 1266 1 3 5 15 35 335 335 1335 6335 70 71 RJAAAA SWBAAA OOOOxx 8430 1267 0 2 0 10 30 430 430 3430 8430 60 61 GMAAAA TWBAAA VVVVxx 4131 1268 1 3 1 11 31 131 131 4131 4131 62 63 XCAAAA UWBAAA AAAAxx 9332 1269 0 0 2 12 32 332 1332 4332 9332 64 65 YUAAAA VWBAAA HHHHxx 293 1270 1 1 3 13 93 293 293 293 293 186 187 HLAAAA WWBAAA OOOOxx 2276 1271 0 0 6 16 76 276 276 2276 2276 152 153 OJAAAA XWBAAA VVVVxx 5687 1272 1 3 7 7 87 687 1687 687 5687 174 175 TKAAAA YWBAAA AAAAxx 5862 1273 0 2 2 2 62 862 1862 862 5862 124 125 MRAAAA ZWBAAA HHHHxx 5073 1274 1 1 3 13 73 73 1073 73 5073 146 147 DNAAAA AXBAAA OOOOxx 4170 1275 0 2 0 10 70 170 170 4170 4170 140 141 KEAAAA BXBAAA VVVVxx 5039 1276 1 3 9 19 39 39 1039 39 5039 78 79 VLAAAA CXBAAA AAAAxx 3294 1277 0 2 4 14 94 294 1294 3294 3294 188 189 SWAAAA DXBAAA HHHHxx 6015 1278 1 3 5 15 15 15 15 1015 6015 30 31 JXAAAA EXBAAA OOOOxx 9015 1279 1 3 5 15 15 15 1015 4015 9015 30 31 TIAAAA FXBAAA VVVVxx 9785 1280 1 1 5 5 85 785 1785 4785 9785 170 171 JMAAAA GXBAAA AAAAxx 4312 1281 0 0 2 12 12 312 312 4312 4312 24 25 WJAAAA HXBAAA HHHHxx 6343 1282 1 3 3 3 43 343 343 1343 6343 86 87 ZJAAAA IXBAAA OOOOxx 2161 1283 1 1 1 1 61 161 161 2161 2161 122 123 DFAAAA JXBAAA VVVVxx 4490 1284 0 2 0 10 90 490 490 4490 4490 180 181 SQAAAA KXBAAA AAAAxx 4454 1285 0 2 4 14 54 454 454 4454 4454 108 109 IPAAAA LXBAAA HHHHxx 7647 1286 1 3 7 7 47 647 1647 2647 7647 94 95 DIAAAA MXBAAA OOOOxx 1028 1287 0 0 8 8 28 28 1028 1028 1028 56 57 ONAAAA NXBAAA VVVVxx 2965 1288 1 1 5 5 65 965 965 2965 2965 130 131 BKAAAA OXBAAA AAAAxx 9900 1289 0 0 0 0 0 900 1900 4900 9900 0 1 UQAAAA PXBAAA HHHHxx 5509 1290 1 1 9 9 9 509 1509 509 5509 18 19 XDAAAA QXBAAA OOOOxx 7751 1291 1 3 1 11 51 751 1751 2751 7751 102 103 DMAAAA RXBAAA VVVVxx 9594 1292 0 2 4 14 94 594 1594 4594 9594 188 189 AFAAAA SXBAAA AAAAxx 7632 1293 0 0 2 12 32 632 1632 2632 7632 64 65 OHAAAA TXBAAA HHHHxx 6528 1294 0 0 8 8 28 528 528 1528 6528 56 57 CRAAAA UXBAAA OOOOxx 1041 1295 1 1 1 1 41 41 1041 1041 1041 82 83 BOAAAA VXBAAA VVVVxx 1534 1296 0 2 4 14 34 534 1534 1534 1534 68 69 AHAAAA WXBAAA AAAAxx 4229 1297 1 1 9 9 29 229 229 4229 4229 58 59 RGAAAA XXBAAA HHHHxx 84 1298 0 0 4 4 84 84 84 84 84 168 169 GDAAAA YXBAAA OOOOxx 2189 1299 1 1 9 9 89 189 189 2189 2189 178 179 FGAAAA ZXBAAA VVVVxx 7566 1300 0 2 6 6 66 566 1566 2566 7566 132 133 AFAAAA AYBAAA AAAAxx 707 1301 1 3 7 7 7 707 707 707 707 14 15 FBAAAA BYBAAA HHHHxx 581 1302 1 1 1 1 81 581 581 581 581 162 163 JWAAAA CYBAAA OOOOxx 6753 1303 1 1 3 13 53 753 753 1753 6753 106 107 TZAAAA DYBAAA VVVVxx 8604 1304 0 0 4 4 4 604 604 3604 8604 8 9 YSAAAA EYBAAA AAAAxx 373 1305 1 1 3 13 73 373 373 373 373 146 147 JOAAAA FYBAAA HHHHxx 9635 1306 1 3 5 15 35 635 1635 4635 9635 70 71 PGAAAA GYBAAA OOOOxx 9277 1307 1 1 7 17 77 277 1277 4277 9277 154 155 VSAAAA HYBAAA VVVVxx 7117 1308 1 1 7 17 17 117 1117 2117 7117 34 35 TNAAAA IYBAAA AAAAxx 8564 1309 0 0 4 4 64 564 564 3564 8564 128 129 KRAAAA JYBAAA HHHHxx 1697 1310 1 1 7 17 97 697 1697 1697 1697 194 195 HNAAAA KYBAAA OOOOxx 7840 1311 0 0 0 0 40 840 1840 2840 7840 80 81 OPAAAA LYBAAA VVVVxx 3646 1312 0 2 6 6 46 646 1646 3646 3646 92 93 GKAAAA MYBAAA AAAAxx 368 1313 0 0 8 8 68 368 368 368 368 136 137 EOAAAA NYBAAA HHHHxx 4797 1314 1 1 7 17 97 797 797 4797 4797 194 195 NCAAAA OYBAAA OOOOxx 5300 1315 0 0 0 0 0 300 1300 300 5300 0 1 WVAAAA PYBAAA VVVVxx 7664 1316 0 0 4 4 64 664 1664 2664 7664 128 129 UIAAAA QYBAAA AAAAxx 1466 1317 0 2 6 6 66 466 1466 1466 1466 132 133 KEAAAA RYBAAA HHHHxx 2477 1318 1 1 7 17 77 477 477 2477 2477 154 155 HRAAAA SYBAAA OOOOxx 2036 1319 0 0 6 16 36 36 36 2036 2036 72 73 IAAAAA TYBAAA VVVVxx 3624 1320 0 0 4 4 24 624 1624 3624 3624 48 49 KJAAAA UYBAAA AAAAxx 5099 1321 1 3 9 19 99 99 1099 99 5099 198 199 DOAAAA VYBAAA HHHHxx 1308 1322 0 0 8 8 8 308 1308 1308 1308 16 17 IYAAAA WYBAAA OOOOxx 3704 1323 0 0 4 4 4 704 1704 3704 3704 8 9 MMAAAA XYBAAA VVVVxx 2451 1324 1 3 1 11 51 451 451 2451 2451 102 103 HQAAAA YYBAAA AAAAxx 4898 1325 0 2 8 18 98 898 898 4898 4898 196 197 KGAAAA ZYBAAA HHHHxx 4959 1326 1 3 9 19 59 959 959 4959 4959 118 119 TIAAAA AZBAAA OOOOxx 5942 1327 0 2 2 2 42 942 1942 942 5942 84 85 OUAAAA BZBAAA VVVVxx 2425 1328 1 1 5 5 25 425 425 2425 2425 50 51 HPAAAA CZBAAA AAAAxx 7760 1329 0 0 0 0 60 760 1760 2760 7760 120 121 MMAAAA DZBAAA HHHHxx 6294 1330 0 2 4 14 94 294 294 1294 6294 188 189 CIAAAA EZBAAA OOOOxx 6785 1331 1 1 5 5 85 785 785 1785 6785 170 171 ZAAAAA FZBAAA VVVVxx 3542 1332 0 2 2 2 42 542 1542 3542 3542 84 85 GGAAAA GZBAAA AAAAxx 1809 1333 1 1 9 9 9 809 1809 1809 1809 18 19 PRAAAA HZBAAA HHHHxx 130 1334 0 2 0 10 30 130 130 130 130 60 61 AFAAAA IZBAAA OOOOxx 8672 1335 0 0 2 12 72 672 672 3672 8672 144 145 OVAAAA JZBAAA VVVVxx 2125 1336 1 1 5 5 25 125 125 2125 2125 50 51 TDAAAA KZBAAA AAAAxx 7683 1337 1 3 3 3 83 683 1683 2683 7683 166 167 NJAAAA LZBAAA HHHHxx 7842 1338 0 2 2 2 42 842 1842 2842 7842 84 85 QPAAAA MZBAAA OOOOxx 9584 1339 0 0 4 4 84 584 1584 4584 9584 168 169 QEAAAA NZBAAA VVVVxx 7963 1340 1 3 3 3 63 963 1963 2963 7963 126 127 HUAAAA OZBAAA AAAAxx 8581 1341 1 1 1 1 81 581 581 3581 8581 162 163 BSAAAA PZBAAA HHHHxx 2135 1342 1 3 5 15 35 135 135 2135 2135 70 71 DEAAAA QZBAAA OOOOxx 7352 1343 0 0 2 12 52 352 1352 2352 7352 104 105 UWAAAA RZBAAA VVVVxx 5789 1344 1 1 9 9 89 789 1789 789 5789 178 179 ROAAAA SZBAAA AAAAxx 8490 1345 0 2 0 10 90 490 490 3490 8490 180 181 OOAAAA TZBAAA HHHHxx 2145 1346 1 1 5 5 45 145 145 2145 2145 90 91 NEAAAA UZBAAA OOOOxx 7021 1347 1 1 1 1 21 21 1021 2021 7021 42 43 BKAAAA VZBAAA VVVVxx 3736 1348 0 0 6 16 36 736 1736 3736 3736 72 73 SNAAAA WZBAAA AAAAxx 7396 1349 0 0 6 16 96 396 1396 2396 7396 192 193 MYAAAA XZBAAA HHHHxx 6334 1350 0 2 4 14 34 334 334 1334 6334 68 69 QJAAAA YZBAAA OOOOxx 5461 1351 1 1 1 1 61 461 1461 461 5461 122 123 BCAAAA ZZBAAA VVVVxx 5337 1352 1 1 7 17 37 337 1337 337 5337 74 75 HXAAAA AACAAA AAAAxx 7440 1353 0 0 0 0 40 440 1440 2440 7440 80 81 EAAAAA BACAAA HHHHxx 6879 1354 1 3 9 19 79 879 879 1879 6879 158 159 PEAAAA CACAAA OOOOxx 2432 1355 0 0 2 12 32 432 432 2432 2432 64 65 OPAAAA DACAAA VVVVxx 8529 1356 1 1 9 9 29 529 529 3529 8529 58 59 BQAAAA EACAAA AAAAxx 7859 1357 1 3 9 19 59 859 1859 2859 7859 118 119 HQAAAA FACAAA HHHHxx 15 1358 1 3 5 15 15 15 15 15 15 30 31 PAAAAA GACAAA OOOOxx 7475 1359 1 3 5 15 75 475 1475 2475 7475 150 151 NBAAAA HACAAA VVVVxx 717 1360 1 1 7 17 17 717 717 717 717 34 35 PBAAAA IACAAA AAAAxx 250 1361 0 2 0 10 50 250 250 250 250 100 101 QJAAAA JACAAA HHHHxx 4700 1362 0 0 0 0 0 700 700 4700 4700 0 1 UYAAAA KACAAA OOOOxx 7510 1363 0 2 0 10 10 510 1510 2510 7510 20 21 WCAAAA LACAAA VVVVxx 4562 1364 0 2 2 2 62 562 562 4562 4562 124 125 MTAAAA MACAAA AAAAxx 8075 1365 1 3 5 15 75 75 75 3075 8075 150 151 PYAAAA NACAAA HHHHxx 871 1366 1 3 1 11 71 871 871 871 871 142 143 NHAAAA OACAAA OOOOxx 7161 1367 1 1 1 1 61 161 1161 2161 7161 122 123 LPAAAA PACAAA VVVVxx 9109 1368 1 1 9 9 9 109 1109 4109 9109 18 19 JMAAAA QACAAA AAAAxx 8675 1369 1 3 5 15 75 675 675 3675 8675 150 151 RVAAAA RACAAA HHHHxx 1025 1370 1 1 5 5 25 25 1025 1025 1025 50 51 LNAAAA SACAAA OOOOxx 4065 1371 1 1 5 5 65 65 65 4065 4065 130 131 JAAAAA TACAAA VVVVxx 3511 1372 1 3 1 11 11 511 1511 3511 3511 22 23 BFAAAA UACAAA AAAAxx 9840 1373 0 0 0 0 40 840 1840 4840 9840 80 81 MOAAAA VACAAA HHHHxx 7495 1374 1 3 5 15 95 495 1495 2495 7495 190 191 HCAAAA WACAAA OOOOxx 55 1375 1 3 5 15 55 55 55 55 55 110 111 DCAAAA XACAAA VVVVxx 6151 1376 1 3 1 11 51 151 151 1151 6151 102 103 PCAAAA YACAAA AAAAxx 2512 1377 0 0 2 12 12 512 512 2512 2512 24 25 QSAAAA ZACAAA HHHHxx 5881 1378 1 1 1 1 81 881 1881 881 5881 162 163 FSAAAA ABCAAA OOOOxx 1442 1379 0 2 2 2 42 442 1442 1442 1442 84 85 MDAAAA BBCAAA VVVVxx 1270 1380 0 2 0 10 70 270 1270 1270 1270 140 141 WWAAAA CBCAAA AAAAxx 959 1381 1 3 9 19 59 959 959 959 959 118 119 XKAAAA DBCAAA HHHHxx 8251 1382 1 3 1 11 51 251 251 3251 8251 102 103 JFAAAA EBCAAA OOOOxx 3051 1383 1 3 1 11 51 51 1051 3051 3051 102 103 JNAAAA FBCAAA VVVVxx 5052 1384 0 0 2 12 52 52 1052 52 5052 104 105 IMAAAA GBCAAA AAAAxx 1863 1385 1 3 3 3 63 863 1863 1863 1863 126 127 RTAAAA HBCAAA HHHHxx 344 1386 0 0 4 4 44 344 344 344 344 88 89 GNAAAA IBCAAA OOOOxx 3590 1387 0 2 0 10 90 590 1590 3590 3590 180 181 CIAAAA JBCAAA VVVVxx 4223 1388 1 3 3 3 23 223 223 4223 4223 46 47 LGAAAA KBCAAA AAAAxx 2284 1389 0 0 4 4 84 284 284 2284 2284 168 169 WJAAAA LBCAAA HHHHxx 9425 1390 1 1 5 5 25 425 1425 4425 9425 50 51 NYAAAA MBCAAA OOOOxx 6221 1391 1 1 1 1 21 221 221 1221 6221 42 43 HFAAAA NBCAAA VVVVxx 195 1392 1 3 5 15 95 195 195 195 195 190 191 NHAAAA OBCAAA AAAAxx 1517 1393 1 1 7 17 17 517 1517 1517 1517 34 35 JGAAAA PBCAAA HHHHxx 3791 1394 1 3 1 11 91 791 1791 3791 3791 182 183 VPAAAA QBCAAA OOOOxx 572 1395 0 0 2 12 72 572 572 572 572 144 145 AWAAAA RBCAAA VVVVxx 46 1396 0 2 6 6 46 46 46 46 46 92 93 UBAAAA SBCAAA AAAAxx 9451 1397 1 3 1 11 51 451 1451 4451 9451 102 103 NZAAAA TBCAAA HHHHxx 3359 1398 1 3 9 19 59 359 1359 3359 3359 118 119 FZAAAA UBCAAA OOOOxx 8867 1399 1 3 7 7 67 867 867 3867 8867 134 135 BDAAAA VBCAAA VVVVxx 674 1400 0 2 4 14 74 674 674 674 674 148 149 YZAAAA WBCAAA AAAAxx 2674 1401 0 2 4 14 74 674 674 2674 2674 148 149 WYAAAA XBCAAA HHHHxx 6523 1402 1 3 3 3 23 523 523 1523 6523 46 47 XQAAAA YBCAAA OOOOxx 6210 1403 0 2 0 10 10 210 210 1210 6210 20 21 WEAAAA ZBCAAA VVVVxx 7564 1404 0 0 4 4 64 564 1564 2564 7564 128 129 YEAAAA ACCAAA AAAAxx 4776 1405 0 0 6 16 76 776 776 4776 4776 152 153 SBAAAA BCCAAA HHHHxx 2993 1406 1 1 3 13 93 993 993 2993 2993 186 187 DLAAAA CCCAAA OOOOxx 2969 1407 1 1 9 9 69 969 969 2969 2969 138 139 FKAAAA DCCAAA VVVVxx 1762 1408 0 2 2 2 62 762 1762 1762 1762 124 125 UPAAAA ECCAAA AAAAxx 685 1409 1 1 5 5 85 685 685 685 685 170 171 JAAAAA FCCAAA HHHHxx 5312 1410 0 0 2 12 12 312 1312 312 5312 24 25 IWAAAA GCCAAA OOOOxx 3264 1411 0 0 4 4 64 264 1264 3264 3264 128 129 OVAAAA HCCAAA VVVVxx 7008 1412 0 0 8 8 8 8 1008 2008 7008 16 17 OJAAAA ICCAAA AAAAxx 5167 1413 1 3 7 7 67 167 1167 167 5167 134 135 TQAAAA JCCAAA HHHHxx 3060 1414 0 0 0 0 60 60 1060 3060 3060 120 121 SNAAAA KCCAAA OOOOxx 1752 1415 0 0 2 12 52 752 1752 1752 1752 104 105 KPAAAA LCCAAA VVVVxx 1016 1416 0 0 6 16 16 16 1016 1016 1016 32 33 CNAAAA MCCAAA AAAAxx 7365 1417 1 1 5 5 65 365 1365 2365 7365 130 131 HXAAAA NCCAAA HHHHxx 4358 1418 0 2 8 18 58 358 358 4358 4358 116 117 QLAAAA OCCAAA OOOOxx 2819 1419 1 3 9 19 19 819 819 2819 2819 38 39 LEAAAA PCCAAA VVVVxx 6727 1420 1 3 7 7 27 727 727 1727 6727 54 55 TYAAAA QCCAAA AAAAxx 1459 1421 1 3 9 19 59 459 1459 1459 1459 118 119 DEAAAA RCCAAA HHHHxx 1708 1422 0 0 8 8 8 708 1708 1708 1708 16 17 SNAAAA SCCAAA OOOOxx 471 1423 1 3 1 11 71 471 471 471 471 142 143 DSAAAA TCCAAA VVVVxx 387 1424 1 3 7 7 87 387 387 387 387 174 175 XOAAAA UCCAAA AAAAxx 1166 1425 0 2 6 6 66 166 1166 1166 1166 132 133 WSAAAA VCCAAA HHHHxx 2400 1426 0 0 0 0 0 400 400 2400 2400 0 1 IOAAAA WCCAAA OOOOxx 3584 1427 0 0 4 4 84 584 1584 3584 3584 168 169 WHAAAA XCCAAA VVVVxx 6423 1428 1 3 3 3 23 423 423 1423 6423 46 47 BNAAAA YCCAAA AAAAxx 9520 1429 0 0 0 0 20 520 1520 4520 9520 40 41 ECAAAA ZCCAAA HHHHxx 8080 1430 0 0 0 0 80 80 80 3080 8080 160 161 UYAAAA ADCAAA OOOOxx 5709 1431 1 1 9 9 9 709 1709 709 5709 18 19 PLAAAA BDCAAA VVVVxx 1131 1432 1 3 1 11 31 131 1131 1131 1131 62 63 NRAAAA CDCAAA AAAAxx 8562 1433 0 2 2 2 62 562 562 3562 8562 124 125 IRAAAA DDCAAA HHHHxx 5766 1434 0 2 6 6 66 766 1766 766 5766 132 133 UNAAAA EDCAAA OOOOxx 245 1435 1 1 5 5 45 245 245 245 245 90 91 LJAAAA FDCAAA VVVVxx 9869 1436 1 1 9 9 69 869 1869 4869 9869 138 139 PPAAAA GDCAAA AAAAxx 3533 1437 1 1 3 13 33 533 1533 3533 3533 66 67 XFAAAA HDCAAA HHHHxx 5109 1438 1 1 9 9 9 109 1109 109 5109 18 19 NOAAAA IDCAAA OOOOxx 977 1439 1 1 7 17 77 977 977 977 977 154 155 PLAAAA JDCAAA VVVVxx 1651 1440 1 3 1 11 51 651 1651 1651 1651 102 103 NLAAAA KDCAAA AAAAxx 1357 1441 1 1 7 17 57 357 1357 1357 1357 114 115 FAAAAA LDCAAA HHHHxx 9087 1442 1 3 7 7 87 87 1087 4087 9087 174 175 NLAAAA MDCAAA OOOOxx 3399 1443 1 3 9 19 99 399 1399 3399 3399 198 199 TAAAAA NDCAAA VVVVxx 7543 1444 1 3 3 3 43 543 1543 2543 7543 86 87 DEAAAA ODCAAA AAAAxx 2469 1445 1 1 9 9 69 469 469 2469 2469 138 139 ZQAAAA PDCAAA HHHHxx 8305 1446 1 1 5 5 5 305 305 3305 8305 10 11 LHAAAA QDCAAA OOOOxx 3265 1447 1 1 5 5 65 265 1265 3265 3265 130 131 PVAAAA RDCAAA VVVVxx 9977 1448 1 1 7 17 77 977 1977 4977 9977 154 155 TTAAAA SDCAAA AAAAxx 3961 1449 1 1 1 1 61 961 1961 3961 3961 122 123 JWAAAA TDCAAA HHHHxx 4952 1450 0 0 2 12 52 952 952 4952 4952 104 105 MIAAAA UDCAAA OOOOxx 5173 1451 1 1 3 13 73 173 1173 173 5173 146 147 ZQAAAA VDCAAA VVVVxx 860 1452 0 0 0 0 60 860 860 860 860 120 121 CHAAAA WDCAAA AAAAxx 4523 1453 1 3 3 3 23 523 523 4523 4523 46 47 ZRAAAA XDCAAA HHHHxx 2361 1454 1 1 1 1 61 361 361 2361 2361 122 123 VMAAAA YDCAAA OOOOxx 7877 1455 1 1 7 17 77 877 1877 2877 7877 154 155 ZQAAAA ZDCAAA VVVVxx 3422 1456 0 2 2 2 22 422 1422 3422 3422 44 45 QBAAAA AECAAA AAAAxx 5781 1457 1 1 1 1 81 781 1781 781 5781 162 163 JOAAAA BECAAA HHHHxx 4752 1458 0 0 2 12 52 752 752 4752 4752 104 105 UAAAAA CECAAA OOOOxx 1786 1459 0 2 6 6 86 786 1786 1786 1786 172 173 SQAAAA DECAAA VVVVxx 1892 1460 0 0 2 12 92 892 1892 1892 1892 184 185 UUAAAA EECAAA AAAAxx 6389 1461 1 1 9 9 89 389 389 1389 6389 178 179 TLAAAA FECAAA HHHHxx 8644 1462 0 0 4 4 44 644 644 3644 8644 88 89 MUAAAA GECAAA OOOOxx 9056 1463 0 0 6 16 56 56 1056 4056 9056 112 113 IKAAAA HECAAA VVVVxx 1423 1464 1 3 3 3 23 423 1423 1423 1423 46 47 TCAAAA IECAAA AAAAxx 4901 1465 1 1 1 1 1 901 901 4901 4901 2 3 NGAAAA JECAAA HHHHxx 3859 1466 1 3 9 19 59 859 1859 3859 3859 118 119 LSAAAA KECAAA OOOOxx 2324 1467 0 0 4 4 24 324 324 2324 2324 48 49 KLAAAA LECAAA VVVVxx 8101 1468 1 1 1 1 1 101 101 3101 8101 2 3 PZAAAA MECAAA AAAAxx 8016 1469 0 0 6 16 16 16 16 3016 8016 32 33 IWAAAA NECAAA HHHHxx 5826 1470 0 2 6 6 26 826 1826 826 5826 52 53 CQAAAA OECAAA OOOOxx 8266 1471 0 2 6 6 66 266 266 3266 8266 132 133 YFAAAA PECAAA VVVVxx 7558 1472 0 2 8 18 58 558 1558 2558 7558 116 117 SEAAAA QECAAA AAAAxx 6976 1473 0 0 6 16 76 976 976 1976 6976 152 153 IIAAAA RECAAA HHHHxx 222 1474 0 2 2 2 22 222 222 222 222 44 45 OIAAAA SECAAA OOOOxx 1624 1475 0 0 4 4 24 624 1624 1624 1624 48 49 MKAAAA TECAAA VVVVxx 1250 1476 0 2 0 10 50 250 1250 1250 1250 100 101 CWAAAA UECAAA AAAAxx 1621 1477 1 1 1 1 21 621 1621 1621 1621 42 43 JKAAAA VECAAA HHHHxx 2350 1478 0 2 0 10 50 350 350 2350 2350 100 101 KMAAAA WECAAA OOOOxx 5239 1479 1 3 9 19 39 239 1239 239 5239 78 79 NTAAAA XECAAA VVVVxx 6681 1480 1 1 1 1 81 681 681 1681 6681 162 163 ZWAAAA YECAAA AAAAxx 4983 1481 1 3 3 3 83 983 983 4983 4983 166 167 RJAAAA ZECAAA HHHHxx 7149 1482 1 1 9 9 49 149 1149 2149 7149 98 99 ZOAAAA AFCAAA OOOOxx 3502 1483 0 2 2 2 2 502 1502 3502 3502 4 5 SEAAAA BFCAAA VVVVxx 3133 1484 1 1 3 13 33 133 1133 3133 3133 66 67 NQAAAA CFCAAA AAAAxx 8342 1485 0 2 2 2 42 342 342 3342 8342 84 85 WIAAAA DFCAAA HHHHxx 3041 1486 1 1 1 1 41 41 1041 3041 3041 82 83 ZMAAAA EFCAAA OOOOxx 5383 1487 1 3 3 3 83 383 1383 383 5383 166 167 BZAAAA FFCAAA VVVVxx 3916 1488 0 0 6 16 16 916 1916 3916 3916 32 33 QUAAAA GFCAAA AAAAxx 1438 1489 0 2 8 18 38 438 1438 1438 1438 76 77 IDAAAA HFCAAA HHHHxx 9408 1490 0 0 8 8 8 408 1408 4408 9408 16 17 WXAAAA IFCAAA OOOOxx 5783 1491 1 3 3 3 83 783 1783 783 5783 166 167 LOAAAA JFCAAA VVVVxx 683 1492 1 3 3 3 83 683 683 683 683 166 167 HAAAAA KFCAAA AAAAxx 9381 1493 1 1 1 1 81 381 1381 4381 9381 162 163 VWAAAA LFCAAA HHHHxx 5676 1494 0 0 6 16 76 676 1676 676 5676 152 153 IKAAAA MFCAAA OOOOxx 3224 1495 0 0 4 4 24 224 1224 3224 3224 48 49 AUAAAA NFCAAA VVVVxx 8332 1496 0 0 2 12 32 332 332 3332 8332 64 65 MIAAAA OFCAAA AAAAxx 3372 1497 0 0 2 12 72 372 1372 3372 3372 144 145 SZAAAA PFCAAA HHHHxx 7436 1498 0 0 6 16 36 436 1436 2436 7436 72 73 AAAAAA QFCAAA OOOOxx 5010 1499 0 2 0 10 10 10 1010 10 5010 20 21 SKAAAA RFCAAA VVVVxx 7256 1500 0 0 6 16 56 256 1256 2256 7256 112 113 CTAAAA SFCAAA AAAAxx 961 1501 1 1 1 1 61 961 961 961 961 122 123 ZKAAAA TFCAAA HHHHxx 4182 1502 0 2 2 2 82 182 182 4182 4182 164 165 WEAAAA UFCAAA OOOOxx 639 1503 1 3 9 19 39 639 639 639 639 78 79 PYAAAA VFCAAA VVVVxx 8836 1504 0 0 6 16 36 836 836 3836 8836 72 73 WBAAAA WFCAAA AAAAxx 8705 1505 1 1 5 5 5 705 705 3705 8705 10 11 VWAAAA XFCAAA HHHHxx 32 1506 0 0 2 12 32 32 32 32 32 64 65 GBAAAA YFCAAA OOOOxx 7913 1507 1 1 3 13 13 913 1913 2913 7913 26 27 JSAAAA ZFCAAA VVVVxx 229 1508 1 1 9 9 29 229 229 229 229 58 59 VIAAAA AGCAAA AAAAxx 2393 1509 1 1 3 13 93 393 393 2393 2393 186 187 BOAAAA BGCAAA HHHHxx 2815 1510 1 3 5 15 15 815 815 2815 2815 30 31 HEAAAA CGCAAA OOOOxx 4858 1511 0 2 8 18 58 858 858 4858 4858 116 117 WEAAAA DGCAAA VVVVxx 6283 1512 1 3 3 3 83 283 283 1283 6283 166 167 RHAAAA EGCAAA AAAAxx 4147 1513 1 3 7 7 47 147 147 4147 4147 94 95 NDAAAA FGCAAA HHHHxx 6801 1514 1 1 1 1 1 801 801 1801 6801 2 3 PBAAAA GGCAAA OOOOxx 1011 1515 1 3 1 11 11 11 1011 1011 1011 22 23 XMAAAA HGCAAA VVVVxx 2527 1516 1 3 7 7 27 527 527 2527 2527 54 55 FTAAAA IGCAAA AAAAxx 381 1517 1 1 1 1 81 381 381 381 381 162 163 ROAAAA JGCAAA HHHHxx 3366 1518 0 2 6 6 66 366 1366 3366 3366 132 133 MZAAAA KGCAAA OOOOxx 9636 1519 0 0 6 16 36 636 1636 4636 9636 72 73 QGAAAA LGCAAA VVVVxx 2239 1520 1 3 9 19 39 239 239 2239 2239 78 79 DIAAAA MGCAAA AAAAxx 5911 1521 1 3 1 11 11 911 1911 911 5911 22 23 JTAAAA NGCAAA HHHHxx 449 1522 1 1 9 9 49 449 449 449 449 98 99 HRAAAA OGCAAA OOOOxx 5118 1523 0 2 8 18 18 118 1118 118 5118 36 37 WOAAAA PGCAAA VVVVxx 7684 1524 0 0 4 4 84 684 1684 2684 7684 168 169 OJAAAA QGCAAA AAAAxx 804 1525 0 0 4 4 4 804 804 804 804 8 9 YEAAAA RGCAAA HHHHxx 8378 1526 0 2 8 18 78 378 378 3378 8378 156 157 GKAAAA SGCAAA OOOOxx 9855 1527 1 3 5 15 55 855 1855 4855 9855 110 111 BPAAAA TGCAAA VVVVxx 1995 1528 1 3 5 15 95 995 1995 1995 1995 190 191 TYAAAA UGCAAA AAAAxx 1979 1529 1 3 9 19 79 979 1979 1979 1979 158 159 DYAAAA VGCAAA HHHHxx 4510 1530 0 2 0 10 10 510 510 4510 4510 20 21 MRAAAA WGCAAA OOOOxx 3792 1531 0 0 2 12 92 792 1792 3792 3792 184 185 WPAAAA XGCAAA VVVVxx 3541 1532 1 1 1 1 41 541 1541 3541 3541 82 83 FGAAAA YGCAAA AAAAxx 8847 1533 1 3 7 7 47 847 847 3847 8847 94 95 HCAAAA ZGCAAA HHHHxx 1336 1534 0 0 6 16 36 336 1336 1336 1336 72 73 KZAAAA AHCAAA OOOOxx 6780 1535 0 0 0 0 80 780 780 1780 6780 160 161 UAAAAA BHCAAA VVVVxx 8711 1536 1 3 1 11 11 711 711 3711 8711 22 23 BXAAAA CHCAAA AAAAxx 7839 1537 1 3 9 19 39 839 1839 2839 7839 78 79 NPAAAA DHCAAA HHHHxx 677 1538 1 1 7 17 77 677 677 677 677 154 155 BAAAAA EHCAAA OOOOxx 1574 1539 0 2 4 14 74 574 1574 1574 1574 148 149 OIAAAA FHCAAA VVVVxx 2905 1540 1 1 5 5 5 905 905 2905 2905 10 11 THAAAA GHCAAA AAAAxx 1879 1541 1 3 9 19 79 879 1879 1879 1879 158 159 HUAAAA HHCAAA HHHHxx 7820 1542 0 0 0 0 20 820 1820 2820 7820 40 41 UOAAAA IHCAAA OOOOxx 4308 1543 0 0 8 8 8 308 308 4308 4308 16 17 SJAAAA JHCAAA VVVVxx 4474 1544 0 2 4 14 74 474 474 4474 4474 148 149 CQAAAA KHCAAA AAAAxx 6985 1545 1 1 5 5 85 985 985 1985 6985 170 171 RIAAAA LHCAAA HHHHxx 6929 1546 1 1 9 9 29 929 929 1929 6929 58 59 NGAAAA MHCAAA OOOOxx 777 1547 1 1 7 17 77 777 777 777 777 154 155 XDAAAA NHCAAA VVVVxx 8271 1548 1 3 1 11 71 271 271 3271 8271 142 143 DGAAAA OHCAAA AAAAxx 2389 1549 1 1 9 9 89 389 389 2389 2389 178 179 XNAAAA PHCAAA HHHHxx 946 1550 0 2 6 6 46 946 946 946 946 92 93 KKAAAA QHCAAA OOOOxx 9682 1551 0 2 2 2 82 682 1682 4682 9682 164 165 KIAAAA RHCAAA VVVVxx 8722 1552 0 2 2 2 22 722 722 3722 8722 44 45 MXAAAA SHCAAA AAAAxx 470 1553 0 2 0 10 70 470 470 470 470 140 141 CSAAAA THCAAA HHHHxx 7425 1554 1 1 5 5 25 425 1425 2425 7425 50 51 PZAAAA UHCAAA OOOOxx 2372 1555 0 0 2 12 72 372 372 2372 2372 144 145 GNAAAA VHCAAA VVVVxx 508 1556 0 0 8 8 8 508 508 508 508 16 17 OTAAAA WHCAAA AAAAxx 163 1557 1 3 3 3 63 163 163 163 163 126 127 HGAAAA XHCAAA HHHHxx 6579 1558 1 3 9 19 79 579 579 1579 6579 158 159 BTAAAA YHCAAA OOOOxx 2355 1559 1 3 5 15 55 355 355 2355 2355 110 111 PMAAAA ZHCAAA VVVVxx 70 1560 0 2 0 10 70 70 70 70 70 140 141 SCAAAA AICAAA AAAAxx 651 1561 1 3 1 11 51 651 651 651 651 102 103 BZAAAA BICAAA HHHHxx 4436 1562 0 0 6 16 36 436 436 4436 4436 72 73 QOAAAA CICAAA OOOOxx 4240 1563 0 0 0 0 40 240 240 4240 4240 80 81 CHAAAA DICAAA VVVVxx 2722 1564 0 2 2 2 22 722 722 2722 2722 44 45 SAAAAA EICAAA AAAAxx 8937 1565 1 1 7 17 37 937 937 3937 8937 74 75 TFAAAA FICAAA HHHHxx 8364 1566 0 0 4 4 64 364 364 3364 8364 128 129 SJAAAA GICAAA OOOOxx 8317 1567 1 1 7 17 17 317 317 3317 8317 34 35 XHAAAA HICAAA VVVVxx 8872 1568 0 0 2 12 72 872 872 3872 8872 144 145 GDAAAA IICAAA AAAAxx 5512 1569 0 0 2 12 12 512 1512 512 5512 24 25 AEAAAA JICAAA HHHHxx 6651 1570 1 3 1 11 51 651 651 1651 6651 102 103 VVAAAA KICAAA OOOOxx 5976 1571 0 0 6 16 76 976 1976 976 5976 152 153 WVAAAA LICAAA VVVVxx 3301 1572 1 1 1 1 1 301 1301 3301 3301 2 3 ZWAAAA MICAAA AAAAxx 6784 1573 0 0 4 4 84 784 784 1784 6784 168 169 YAAAAA NICAAA HHHHxx 573 1574 1 1 3 13 73 573 573 573 573 146 147 BWAAAA OICAAA OOOOxx 3015 1575 1 3 5 15 15 15 1015 3015 3015 30 31 ZLAAAA PICAAA VVVVxx 8245 1576 1 1 5 5 45 245 245 3245 8245 90 91 DFAAAA QICAAA AAAAxx 5251 1577 1 3 1 11 51 251 1251 251 5251 102 103 ZTAAAA RICAAA HHHHxx 2281 1578 1 1 1 1 81 281 281 2281 2281 162 163 TJAAAA SICAAA OOOOxx 518 1579 0 2 8 18 18 518 518 518 518 36 37 YTAAAA TICAAA VVVVxx 9839 1580 1 3 9 19 39 839 1839 4839 9839 78 79 LOAAAA UICAAA AAAAxx 4526 1581 0 2 6 6 26 526 526 4526 4526 52 53 CSAAAA VICAAA HHHHxx 1261 1582 1 1 1 1 61 261 1261 1261 1261 122 123 NWAAAA WICAAA OOOOxx 4259 1583 1 3 9 19 59 259 259 4259 4259 118 119 VHAAAA XICAAA VVVVxx 9098 1584 0 2 8 18 98 98 1098 4098 9098 196 197 YLAAAA YICAAA AAAAxx 6037 1585 1 1 7 17 37 37 37 1037 6037 74 75 FYAAAA ZICAAA HHHHxx 4284 1586 0 0 4 4 84 284 284 4284 4284 168 169 UIAAAA AJCAAA OOOOxx 3267 1587 1 3 7 7 67 267 1267 3267 3267 134 135 RVAAAA BJCAAA VVVVxx 5908 1588 0 0 8 8 8 908 1908 908 5908 16 17 GTAAAA CJCAAA AAAAxx 1549 1589 1 1 9 9 49 549 1549 1549 1549 98 99 PHAAAA DJCAAA HHHHxx 8736 1590 0 0 6 16 36 736 736 3736 8736 72 73 AYAAAA EJCAAA OOOOxx 2008 1591 0 0 8 8 8 8 8 2008 2008 16 17 GZAAAA FJCAAA VVVVxx 548 1592 0 0 8 8 48 548 548 548 548 96 97 CVAAAA GJCAAA AAAAxx 8846 1593 0 2 6 6 46 846 846 3846 8846 92 93 GCAAAA HJCAAA HHHHxx 8374 1594 0 2 4 14 74 374 374 3374 8374 148 149 CKAAAA IJCAAA OOOOxx 7986 1595 0 2 6 6 86 986 1986 2986 7986 172 173 EVAAAA JJCAAA VVVVxx 6819 1596 1 3 9 19 19 819 819 1819 6819 38 39 HCAAAA KJCAAA AAAAxx 4418 1597 0 2 8 18 18 418 418 4418 4418 36 37 YNAAAA LJCAAA HHHHxx 833 1598 1 1 3 13 33 833 833 833 833 66 67 BGAAAA MJCAAA OOOOxx 4416 1599 0 0 6 16 16 416 416 4416 4416 32 33 WNAAAA NJCAAA VVVVxx 4902 1600 0 2 2 2 2 902 902 4902 4902 4 5 OGAAAA OJCAAA AAAAxx 6828 1601 0 0 8 8 28 828 828 1828 6828 56 57 QCAAAA PJCAAA HHHHxx 1118 1602 0 2 8 18 18 118 1118 1118 1118 36 37 ARAAAA QJCAAA OOOOxx 9993 1603 1 1 3 13 93 993 1993 4993 9993 186 187 JUAAAA RJCAAA VVVVxx 1430 1604 0 2 0 10 30 430 1430 1430 1430 60 61 ADAAAA SJCAAA AAAAxx 5670 1605 0 2 0 10 70 670 1670 670 5670 140 141 CKAAAA TJCAAA HHHHxx 5424 1606 0 0 4 4 24 424 1424 424 5424 48 49 QAAAAA UJCAAA OOOOxx 5561 1607 1 1 1 1 61 561 1561 561 5561 122 123 XFAAAA VJCAAA VVVVxx 2027 1608 1 3 7 7 27 27 27 2027 2027 54 55 ZZAAAA WJCAAA AAAAxx 6924 1609 0 0 4 4 24 924 924 1924 6924 48 49 IGAAAA XJCAAA HHHHxx 5946 1610 0 2 6 6 46 946 1946 946 5946 92 93 SUAAAA YJCAAA OOOOxx 4294 1611 0 2 4 14 94 294 294 4294 4294 188 189 EJAAAA ZJCAAA VVVVxx 2936 1612 0 0 6 16 36 936 936 2936 2936 72 73 YIAAAA AKCAAA AAAAxx 3855 1613 1 3 5 15 55 855 1855 3855 3855 110 111 HSAAAA BKCAAA HHHHxx 455 1614 1 3 5 15 55 455 455 455 455 110 111 NRAAAA CKCAAA OOOOxx 2918 1615 0 2 8 18 18 918 918 2918 2918 36 37 GIAAAA DKCAAA VVVVxx 448 1616 0 0 8 8 48 448 448 448 448 96 97 GRAAAA EKCAAA AAAAxx 2149 1617 1 1 9 9 49 149 149 2149 2149 98 99 REAAAA FKCAAA HHHHxx 8890 1618 0 2 0 10 90 890 890 3890 8890 180 181 YDAAAA GKCAAA OOOOxx 8919 1619 1 3 9 19 19 919 919 3919 8919 38 39 BFAAAA HKCAAA VVVVxx 4957 1620 1 1 7 17 57 957 957 4957 4957 114 115 RIAAAA IKCAAA AAAAxx 4 1621 0 0 4 4 4 4 4 4 4 8 9 EAAAAA JKCAAA HHHHxx 4837 1622 1 1 7 17 37 837 837 4837 4837 74 75 BEAAAA KKCAAA OOOOxx 3976 1623 0 0 6 16 76 976 1976 3976 3976 152 153 YWAAAA LKCAAA VVVVxx 9459 1624 1 3 9 19 59 459 1459 4459 9459 118 119 VZAAAA MKCAAA AAAAxx 7097 1625 1 1 7 17 97 97 1097 2097 7097 194 195 ZMAAAA NKCAAA HHHHxx 9226 1626 0 2 6 6 26 226 1226 4226 9226 52 53 WQAAAA OKCAAA OOOOxx 5803 1627 1 3 3 3 3 803 1803 803 5803 6 7 FPAAAA PKCAAA VVVVxx 21 1628 1 1 1 1 21 21 21 21 21 42 43 VAAAAA QKCAAA AAAAxx 5275 1629 1 3 5 15 75 275 1275 275 5275 150 151 XUAAAA RKCAAA HHHHxx 3488 1630 0 0 8 8 88 488 1488 3488 3488 176 177 EEAAAA SKCAAA OOOOxx 1595 1631 1 3 5 15 95 595 1595 1595 1595 190 191 JJAAAA TKCAAA VVVVxx 5212 1632 0 0 2 12 12 212 1212 212 5212 24 25 MSAAAA UKCAAA AAAAxx 6574 1633 0 2 4 14 74 574 574 1574 6574 148 149 WSAAAA VKCAAA HHHHxx 7524 1634 0 0 4 4 24 524 1524 2524 7524 48 49 KDAAAA WKCAAA OOOOxx 6100 1635 0 0 0 0 0 100 100 1100 6100 0 1 QAAAAA XKCAAA VVVVxx 1198 1636 0 2 8 18 98 198 1198 1198 1198 196 197 CUAAAA YKCAAA AAAAxx 7345 1637 1 1 5 5 45 345 1345 2345 7345 90 91 NWAAAA ZKCAAA HHHHxx 5020 1638 0 0 0 0 20 20 1020 20 5020 40 41 CLAAAA ALCAAA OOOOxx 6925 1639 1 1 5 5 25 925 925 1925 6925 50 51 JGAAAA BLCAAA VVVVxx 8915 1640 1 3 5 15 15 915 915 3915 8915 30 31 XEAAAA CLCAAA AAAAxx 3088 1641 0 0 8 8 88 88 1088 3088 3088 176 177 UOAAAA DLCAAA HHHHxx 4828 1642 0 0 8 8 28 828 828 4828 4828 56 57 SDAAAA ELCAAA OOOOxx 7276 1643 0 0 6 16 76 276 1276 2276 7276 152 153 WTAAAA FLCAAA VVVVxx 299 1644 1 3 9 19 99 299 299 299 299 198 199 NLAAAA GLCAAA AAAAxx 76 1645 0 0 6 16 76 76 76 76 76 152 153 YCAAAA HLCAAA HHHHxx 8458 1646 0 2 8 18 58 458 458 3458 8458 116 117 INAAAA ILCAAA OOOOxx 7207 1647 1 3 7 7 7 207 1207 2207 7207 14 15 FRAAAA JLCAAA VVVVxx 5585 1648 1 1 5 5 85 585 1585 585 5585 170 171 VGAAAA KLCAAA AAAAxx 3234 1649 0 2 4 14 34 234 1234 3234 3234 68 69 KUAAAA LLCAAA HHHHxx 8001 1650 1 1 1 1 1 1 1 3001 8001 2 3 TVAAAA MLCAAA OOOOxx 1319 1651 1 3 9 19 19 319 1319 1319 1319 38 39 TYAAAA NLCAAA VVVVxx 6342 1652 0 2 2 2 42 342 342 1342 6342 84 85 YJAAAA OLCAAA AAAAxx 9199 1653 1 3 9 19 99 199 1199 4199 9199 198 199 VPAAAA PLCAAA HHHHxx 5696 1654 0 0 6 16 96 696 1696 696 5696 192 193 CLAAAA QLCAAA OOOOxx 2562 1655 0 2 2 2 62 562 562 2562 2562 124 125 OUAAAA RLCAAA VVVVxx 4226 1656 0 2 6 6 26 226 226 4226 4226 52 53 OGAAAA SLCAAA AAAAxx 1184 1657 0 0 4 4 84 184 1184 1184 1184 168 169 OTAAAA TLCAAA HHHHxx 5807 1658 1 3 7 7 7 807 1807 807 5807 14 15 JPAAAA ULCAAA OOOOxx 1890 1659 0 2 0 10 90 890 1890 1890 1890 180 181 SUAAAA VLCAAA VVVVxx 451 1660 1 3 1 11 51 451 451 451 451 102 103 JRAAAA WLCAAA AAAAxx 1049 1661 1 1 9 9 49 49 1049 1049 1049 98 99 JOAAAA XLCAAA HHHHxx 5272 1662 0 0 2 12 72 272 1272 272 5272 144 145 UUAAAA YLCAAA OOOOxx 4588 1663 0 0 8 8 88 588 588 4588 4588 176 177 MUAAAA ZLCAAA VVVVxx 5213 1664 1 1 3 13 13 213 1213 213 5213 26 27 NSAAAA AMCAAA AAAAxx 9543 1665 1 3 3 3 43 543 1543 4543 9543 86 87 BDAAAA BMCAAA HHHHxx 6318 1666 0 2 8 18 18 318 318 1318 6318 36 37 AJAAAA CMCAAA OOOOxx 7992 1667 0 0 2 12 92 992 1992 2992 7992 184 185 KVAAAA DMCAAA VVVVxx 4619 1668 1 3 9 19 19 619 619 4619 4619 38 39 RVAAAA EMCAAA AAAAxx 7189 1669 1 1 9 9 89 189 1189 2189 7189 178 179 NQAAAA FMCAAA HHHHxx 2178 1670 0 2 8 18 78 178 178 2178 2178 156 157 UFAAAA GMCAAA OOOOxx 4928 1671 0 0 8 8 28 928 928 4928 4928 56 57 OHAAAA HMCAAA VVVVxx 3966 1672 0 2 6 6 66 966 1966 3966 3966 132 133 OWAAAA IMCAAA AAAAxx 9790 1673 0 2 0 10 90 790 1790 4790 9790 180 181 OMAAAA JMCAAA HHHHxx 9150 1674 0 2 0 10 50 150 1150 4150 9150 100 101 YNAAAA KMCAAA OOOOxx 313 1675 1 1 3 13 13 313 313 313 313 26 27 BMAAAA LMCAAA VVVVxx 1614 1676 0 2 4 14 14 614 1614 1614 1614 28 29 CKAAAA MMCAAA AAAAxx 1581 1677 1 1 1 1 81 581 1581 1581 1581 162 163 VIAAAA NMCAAA HHHHxx 3674 1678 0 2 4 14 74 674 1674 3674 3674 148 149 ILAAAA OMCAAA OOOOxx 3444 1679 0 0 4 4 44 444 1444 3444 3444 88 89 MCAAAA PMCAAA VVVVxx 1050 1680 0 2 0 10 50 50 1050 1050 1050 100 101 KOAAAA QMCAAA AAAAxx 8241 1681 1 1 1 1 41 241 241 3241 8241 82 83 ZEAAAA RMCAAA HHHHxx 3382 1682 0 2 2 2 82 382 1382 3382 3382 164 165 CAAAAA SMCAAA OOOOxx 7105 1683 1 1 5 5 5 105 1105 2105 7105 10 11 HNAAAA TMCAAA VVVVxx 2957 1684 1 1 7 17 57 957 957 2957 2957 114 115 TJAAAA UMCAAA AAAAxx 6162 1685 0 2 2 2 62 162 162 1162 6162 124 125 ADAAAA VMCAAA HHHHxx 5150 1686 0 2 0 10 50 150 1150 150 5150 100 101 CQAAAA WMCAAA OOOOxx 2622 1687 0 2 2 2 22 622 622 2622 2622 44 45 WWAAAA XMCAAA VVVVxx 2240 1688 0 0 0 0 40 240 240 2240 2240 80 81 EIAAAA YMCAAA AAAAxx 8880 1689 0 0 0 0 80 880 880 3880 8880 160 161 ODAAAA ZMCAAA HHHHxx 9250 1690 0 2 0 10 50 250 1250 4250 9250 100 101 URAAAA ANCAAA OOOOxx 7010 1691 0 2 0 10 10 10 1010 2010 7010 20 21 QJAAAA BNCAAA VVVVxx 1098 1692 0 2 8 18 98 98 1098 1098 1098 196 197 GQAAAA CNCAAA AAAAxx 648 1693 0 0 8 8 48 648 648 648 648 96 97 YYAAAA DNCAAA HHHHxx 5536 1694 0 0 6 16 36 536 1536 536 5536 72 73 YEAAAA ENCAAA OOOOxx 7858 1695 0 2 8 18 58 858 1858 2858 7858 116 117 GQAAAA FNCAAA VVVVxx 7053 1696 1 1 3 13 53 53 1053 2053 7053 106 107 HLAAAA GNCAAA AAAAxx 8681 1697 1 1 1 1 81 681 681 3681 8681 162 163 XVAAAA HNCAAA HHHHxx 8832 1698 0 0 2 12 32 832 832 3832 8832 64 65 SBAAAA INCAAA OOOOxx 6836 1699 0 0 6 16 36 836 836 1836 6836 72 73 YCAAAA JNCAAA VVVVxx 4856 1700 0 0 6 16 56 856 856 4856 4856 112 113 UEAAAA KNCAAA AAAAxx 345 1701 1 1 5 5 45 345 345 345 345 90 91 HNAAAA LNCAAA HHHHxx 6559 1702 1 3 9 19 59 559 559 1559 6559 118 119 HSAAAA MNCAAA OOOOxx 3017 1703 1 1 7 17 17 17 1017 3017 3017 34 35 BMAAAA NNCAAA VVVVxx 4176 1704 0 0 6 16 76 176 176 4176 4176 152 153 QEAAAA ONCAAA AAAAxx 2839 1705 1 3 9 19 39 839 839 2839 2839 78 79 FFAAAA PNCAAA HHHHxx 6065 1706 1 1 5 5 65 65 65 1065 6065 130 131 HZAAAA QNCAAA OOOOxx 7360 1707 0 0 0 0 60 360 1360 2360 7360 120 121 CXAAAA RNCAAA VVVVxx 9527 1708 1 3 7 7 27 527 1527 4527 9527 54 55 LCAAAA SNCAAA AAAAxx 8849 1709 1 1 9 9 49 849 849 3849 8849 98 99 JCAAAA TNCAAA HHHHxx 7274 1710 0 2 4 14 74 274 1274 2274 7274 148 149 UTAAAA UNCAAA OOOOxx 4368 1711 0 0 8 8 68 368 368 4368 4368 136 137 AMAAAA VNCAAA VVVVxx 2488 1712 0 0 8 8 88 488 488 2488 2488 176 177 SRAAAA WNCAAA AAAAxx 4674 1713 0 2 4 14 74 674 674 4674 4674 148 149 UXAAAA XNCAAA HHHHxx 365 1714 1 1 5 5 65 365 365 365 365 130 131 BOAAAA YNCAAA OOOOxx 5897 1715 1 1 7 17 97 897 1897 897 5897 194 195 VSAAAA ZNCAAA VVVVxx 8918 1716 0 2 8 18 18 918 918 3918 8918 36 37 AFAAAA AOCAAA AAAAxx 1988 1717 0 0 8 8 88 988 1988 1988 1988 176 177 MYAAAA BOCAAA HHHHxx 1210 1718 0 2 0 10 10 210 1210 1210 1210 20 21 OUAAAA COCAAA OOOOxx 2945 1719 1 1 5 5 45 945 945 2945 2945 90 91 HJAAAA DOCAAA VVVVxx 555 1720 1 3 5 15 55 555 555 555 555 110 111 JVAAAA EOCAAA AAAAxx 9615 1721 1 3 5 15 15 615 1615 4615 9615 30 31 VFAAAA FOCAAA HHHHxx 9939 1722 1 3 9 19 39 939 1939 4939 9939 78 79 HSAAAA GOCAAA OOOOxx 1216 1723 0 0 6 16 16 216 1216 1216 1216 32 33 UUAAAA HOCAAA VVVVxx 745 1724 1 1 5 5 45 745 745 745 745 90 91 RCAAAA IOCAAA AAAAxx 3326 1725 0 2 6 6 26 326 1326 3326 3326 52 53 YXAAAA JOCAAA HHHHxx 953 1726 1 1 3 13 53 953 953 953 953 106 107 RKAAAA KOCAAA OOOOxx 444 1727 0 0 4 4 44 444 444 444 444 88 89 CRAAAA LOCAAA VVVVxx 280 1728 0 0 0 0 80 280 280 280 280 160 161 UKAAAA MOCAAA AAAAxx 3707 1729 1 3 7 7 7 707 1707 3707 3707 14 15 PMAAAA NOCAAA HHHHxx 1351 1730 1 3 1 11 51 351 1351 1351 1351 102 103 ZZAAAA OOCAAA OOOOxx 1280 1731 0 0 0 0 80 280 1280 1280 1280 160 161 GXAAAA POCAAA VVVVxx 628 1732 0 0 8 8 28 628 628 628 628 56 57 EYAAAA QOCAAA AAAAxx 6198 1733 0 2 8 18 98 198 198 1198 6198 196 197 KEAAAA ROCAAA HHHHxx 1957 1734 1 1 7 17 57 957 1957 1957 1957 114 115 HXAAAA SOCAAA OOOOxx 9241 1735 1 1 1 1 41 241 1241 4241 9241 82 83 LRAAAA TOCAAA VVVVxx 303 1736 1 3 3 3 3 303 303 303 303 6 7 RLAAAA UOCAAA AAAAxx 1945 1737 1 1 5 5 45 945 1945 1945 1945 90 91 VWAAAA VOCAAA HHHHxx 3634 1738 0 2 4 14 34 634 1634 3634 3634 68 69 UJAAAA WOCAAA OOOOxx 4768 1739 0 0 8 8 68 768 768 4768 4768 136 137 KBAAAA XOCAAA VVVVxx 9262 1740 0 2 2 2 62 262 1262 4262 9262 124 125 GSAAAA YOCAAA AAAAxx 2610 1741 0 2 0 10 10 610 610 2610 2610 20 21 KWAAAA ZOCAAA HHHHxx 6640 1742 0 0 0 0 40 640 640 1640 6640 80 81 KVAAAA APCAAA OOOOxx 3338 1743 0 2 8 18 38 338 1338 3338 3338 76 77 KYAAAA BPCAAA VVVVxx 6560 1744 0 0 0 0 60 560 560 1560 6560 120 121 ISAAAA CPCAAA AAAAxx 5986 1745 0 2 6 6 86 986 1986 986 5986 172 173 GWAAAA DPCAAA HHHHxx 2970 1746 0 2 0 10 70 970 970 2970 2970 140 141 GKAAAA EPCAAA OOOOxx 4731 1747 1 3 1 11 31 731 731 4731 4731 62 63 ZZAAAA FPCAAA VVVVxx 9486 1748 0 2 6 6 86 486 1486 4486 9486 172 173 WAAAAA GPCAAA AAAAxx 7204 1749 0 0 4 4 4 204 1204 2204 7204 8 9 CRAAAA HPCAAA HHHHxx 6685 1750 1 1 5 5 85 685 685 1685 6685 170 171 DXAAAA IPCAAA OOOOxx 6852 1751 0 0 2 12 52 852 852 1852 6852 104 105 ODAAAA JPCAAA VVVVxx 2325 1752 1 1 5 5 25 325 325 2325 2325 50 51 LLAAAA KPCAAA AAAAxx 1063 1753 1 3 3 3 63 63 1063 1063 1063 126 127 XOAAAA LPCAAA HHHHxx 6810 1754 0 2 0 10 10 810 810 1810 6810 20 21 YBAAAA MPCAAA OOOOxx 7718 1755 0 2 8 18 18 718 1718 2718 7718 36 37 WKAAAA NPCAAA VVVVxx 1680 1756 0 0 0 0 80 680 1680 1680 1680 160 161 QMAAAA OPCAAA AAAAxx 7402 1757 0 2 2 2 2 402 1402 2402 7402 4 5 SYAAAA PPCAAA HHHHxx 4134 1758 0 2 4 14 34 134 134 4134 4134 68 69 ADAAAA QPCAAA OOOOxx 8232 1759 0 0 2 12 32 232 232 3232 8232 64 65 QEAAAA RPCAAA VVVVxx 6682 1760 0 2 2 2 82 682 682 1682 6682 164 165 AXAAAA SPCAAA AAAAxx 7952 1761 0 0 2 12 52 952 1952 2952 7952 104 105 WTAAAA TPCAAA HHHHxx 5943 1762 1 3 3 3 43 943 1943 943 5943 86 87 PUAAAA UPCAAA OOOOxx 5394 1763 0 2 4 14 94 394 1394 394 5394 188 189 MZAAAA VPCAAA VVVVxx 6554 1764 0 2 4 14 54 554 554 1554 6554 108 109 CSAAAA WPCAAA AAAAxx 8186 1765 0 2 6 6 86 186 186 3186 8186 172 173 WCAAAA XPCAAA HHHHxx 199 1766 1 3 9 19 99 199 199 199 199 198 199 RHAAAA YPCAAA OOOOxx 3386 1767 0 2 6 6 86 386 1386 3386 3386 172 173 GAAAAA ZPCAAA VVVVxx 8974 1768 0 2 4 14 74 974 974 3974 8974 148 149 EHAAAA AQCAAA AAAAxx 8140 1769 0 0 0 0 40 140 140 3140 8140 80 81 CBAAAA BQCAAA HHHHxx 3723 1770 1 3 3 3 23 723 1723 3723 3723 46 47 FNAAAA CQCAAA OOOOxx 8827 1771 1 3 7 7 27 827 827 3827 8827 54 55 NBAAAA DQCAAA VVVVxx 1998 1772 0 2 8 18 98 998 1998 1998 1998 196 197 WYAAAA EQCAAA AAAAxx 879 1773 1 3 9 19 79 879 879 879 879 158 159 VHAAAA FQCAAA HHHHxx 892 1774 0 0 2 12 92 892 892 892 892 184 185 IIAAAA GQCAAA OOOOxx 9468 1775 0 0 8 8 68 468 1468 4468 9468 136 137 EAAAAA HQCAAA VVVVxx 3797 1776 1 1 7 17 97 797 1797 3797 3797 194 195 BQAAAA IQCAAA AAAAxx 8379 1777 1 3 9 19 79 379 379 3379 8379 158 159 HKAAAA JQCAAA HHHHxx 2817 1778 1 1 7 17 17 817 817 2817 2817 34 35 JEAAAA KQCAAA OOOOxx 789 1779 1 1 9 9 89 789 789 789 789 178 179 JEAAAA LQCAAA VVVVxx 3871 1780 1 3 1 11 71 871 1871 3871 3871 142 143 XSAAAA MQCAAA AAAAxx 7931 1781 1 3 1 11 31 931 1931 2931 7931 62 63 BTAAAA NQCAAA HHHHxx 3636 1782 0 0 6 16 36 636 1636 3636 3636 72 73 WJAAAA OQCAAA OOOOxx 699 1783 1 3 9 19 99 699 699 699 699 198 199 XAAAAA PQCAAA VVVVxx 6850 1784 0 2 0 10 50 850 850 1850 6850 100 101 MDAAAA QQCAAA AAAAxx 6394 1785 0 2 4 14 94 394 394 1394 6394 188 189 YLAAAA RQCAAA HHHHxx 3475 1786 1 3 5 15 75 475 1475 3475 3475 150 151 RDAAAA SQCAAA OOOOxx 3026 1787 0 2 6 6 26 26 1026 3026 3026 52 53 KMAAAA TQCAAA VVVVxx 876 1788 0 0 6 16 76 876 876 876 876 152 153 SHAAAA UQCAAA AAAAxx 1992 1789 0 0 2 12 92 992 1992 1992 1992 184 185 QYAAAA VQCAAA HHHHxx 3079 1790 1 3 9 19 79 79 1079 3079 3079 158 159 LOAAAA WQCAAA OOOOxx 8128 1791 0 0 8 8 28 128 128 3128 8128 56 57 QAAAAA XQCAAA VVVVxx 8123 1792 1 3 3 3 23 123 123 3123 8123 46 47 LAAAAA YQCAAA AAAAxx 3285 1793 1 1 5 5 85 285 1285 3285 3285 170 171 JWAAAA ZQCAAA HHHHxx 9315 1794 1 3 5 15 15 315 1315 4315 9315 30 31 HUAAAA ARCAAA OOOOxx 9862 1795 0 2 2 2 62 862 1862 4862 9862 124 125 IPAAAA BRCAAA VVVVxx 2764 1796 0 0 4 4 64 764 764 2764 2764 128 129 ICAAAA CRCAAA AAAAxx 3544 1797 0 0 4 4 44 544 1544 3544 3544 88 89 IGAAAA DRCAAA HHHHxx 7747 1798 1 3 7 7 47 747 1747 2747 7747 94 95 ZLAAAA ERCAAA OOOOxx 7725 1799 1 1 5 5 25 725 1725 2725 7725 50 51 DLAAAA FRCAAA VVVVxx 2449 1800 1 1 9 9 49 449 449 2449 2449 98 99 FQAAAA GRCAAA AAAAxx 8967 1801 1 3 7 7 67 967 967 3967 8967 134 135 XGAAAA HRCAAA HHHHxx 7371 1802 1 3 1 11 71 371 1371 2371 7371 142 143 NXAAAA IRCAAA OOOOxx 2158 1803 0 2 8 18 58 158 158 2158 2158 116 117 AFAAAA JRCAAA VVVVxx 5590 1804 0 2 0 10 90 590 1590 590 5590 180 181 AHAAAA KRCAAA AAAAxx 8072 1805 0 0 2 12 72 72 72 3072 8072 144 145 MYAAAA LRCAAA HHHHxx 1971 1806 1 3 1 11 71 971 1971 1971 1971 142 143 VXAAAA MRCAAA OOOOxx 772 1807 0 0 2 12 72 772 772 772 772 144 145 SDAAAA NRCAAA VVVVxx 3433 1808 1 1 3 13 33 433 1433 3433 3433 66 67 BCAAAA ORCAAA AAAAxx 8419 1809 1 3 9 19 19 419 419 3419 8419 38 39 VLAAAA PRCAAA HHHHxx 1493 1810 1 1 3 13 93 493 1493 1493 1493 186 187 LFAAAA QRCAAA OOOOxx 2584 1811 0 0 4 4 84 584 584 2584 2584 168 169 KVAAAA RRCAAA VVVVxx 9502 1812 0 2 2 2 2 502 1502 4502 9502 4 5 MBAAAA SRCAAA AAAAxx 4673 1813 1 1 3 13 73 673 673 4673 4673 146 147 TXAAAA TRCAAA HHHHxx 7403 1814 1 3 3 3 3 403 1403 2403 7403 6 7 TYAAAA URCAAA OOOOxx 7103 1815 1 3 3 3 3 103 1103 2103 7103 6 7 FNAAAA VRCAAA VVVVxx 7026 1816 0 2 6 6 26 26 1026 2026 7026 52 53 GKAAAA WRCAAA AAAAxx 8574 1817 0 2 4 14 74 574 574 3574 8574 148 149 URAAAA XRCAAA HHHHxx 1366 1818 0 2 6 6 66 366 1366 1366 1366 132 133 OAAAAA YRCAAA OOOOxx 5787 1819 1 3 7 7 87 787 1787 787 5787 174 175 POAAAA ZRCAAA VVVVxx 2552 1820 0 0 2 12 52 552 552 2552 2552 104 105 EUAAAA ASCAAA AAAAxx 4557 1821 1 1 7 17 57 557 557 4557 4557 114 115 HTAAAA BSCAAA HHHHxx 3237 1822 1 1 7 17 37 237 1237 3237 3237 74 75 NUAAAA CSCAAA OOOOxx 6901 1823 1 1 1 1 1 901 901 1901 6901 2 3 LFAAAA DSCAAA VVVVxx 7708 1824 0 0 8 8 8 708 1708 2708 7708 16 17 MKAAAA ESCAAA AAAAxx 2011 1825 1 3 1 11 11 11 11 2011 2011 22 23 JZAAAA FSCAAA HHHHxx 9455 1826 1 3 5 15 55 455 1455 4455 9455 110 111 RZAAAA GSCAAA OOOOxx 5228 1827 0 0 8 8 28 228 1228 228 5228 56 57 CTAAAA HSCAAA VVVVxx 4043 1828 1 3 3 3 43 43 43 4043 4043 86 87 NZAAAA ISCAAA AAAAxx 8242 1829 0 2 2 2 42 242 242 3242 8242 84 85 AFAAAA JSCAAA HHHHxx 6351 1830 1 3 1 11 51 351 351 1351 6351 102 103 HKAAAA KSCAAA OOOOxx 5899 1831 1 3 9 19 99 899 1899 899 5899 198 199 XSAAAA LSCAAA VVVVxx 4849 1832 1 1 9 9 49 849 849 4849 4849 98 99 NEAAAA MSCAAA AAAAxx 9583 1833 1 3 3 3 83 583 1583 4583 9583 166 167 PEAAAA NSCAAA HHHHxx 4994 1834 0 2 4 14 94 994 994 4994 4994 188 189 CKAAAA OSCAAA OOOOxx 9787 1835 1 3 7 7 87 787 1787 4787 9787 174 175 LMAAAA PSCAAA VVVVxx 243 1836 1 3 3 3 43 243 243 243 243 86 87 JJAAAA QSCAAA AAAAxx 3931 1837 1 3 1 11 31 931 1931 3931 3931 62 63 FVAAAA RSCAAA HHHHxx 5945 1838 1 1 5 5 45 945 1945 945 5945 90 91 RUAAAA SSCAAA OOOOxx 1325 1839 1 1 5 5 25 325 1325 1325 1325 50 51 ZYAAAA TSCAAA VVVVxx 4142 1840 0 2 2 2 42 142 142 4142 4142 84 85 IDAAAA USCAAA AAAAxx 1963 1841 1 3 3 3 63 963 1963 1963 1963 126 127 NXAAAA VSCAAA HHHHxx 7041 1842 1 1 1 1 41 41 1041 2041 7041 82 83 VKAAAA WSCAAA OOOOxx 3074 1843 0 2 4 14 74 74 1074 3074 3074 148 149 GOAAAA XSCAAA VVVVxx 3290 1844 0 2 0 10 90 290 1290 3290 3290 180 181 OWAAAA YSCAAA AAAAxx 4146 1845 0 2 6 6 46 146 146 4146 4146 92 93 MDAAAA ZSCAAA HHHHxx 3832 1846 0 0 2 12 32 832 1832 3832 3832 64 65 KRAAAA ATCAAA OOOOxx 2217 1847 1 1 7 17 17 217 217 2217 2217 34 35 HHAAAA BTCAAA VVVVxx 635 1848 1 3 5 15 35 635 635 635 635 70 71 LYAAAA CTCAAA AAAAxx 6967 1849 1 3 7 7 67 967 967 1967 6967 134 135 ZHAAAA DTCAAA HHHHxx 3522 1850 0 2 2 2 22 522 1522 3522 3522 44 45 MFAAAA ETCAAA OOOOxx 2471 1851 1 3 1 11 71 471 471 2471 2471 142 143 BRAAAA FTCAAA VVVVxx 4236 1852 0 0 6 16 36 236 236 4236 4236 72 73 YGAAAA GTCAAA AAAAxx 853 1853 1 1 3 13 53 853 853 853 853 106 107 VGAAAA HTCAAA HHHHxx 3754 1854 0 2 4 14 54 754 1754 3754 3754 108 109 KOAAAA ITCAAA OOOOxx 796 1855 0 0 6 16 96 796 796 796 796 192 193 QEAAAA JTCAAA VVVVxx 4640 1856 0 0 0 0 40 640 640 4640 4640 80 81 MWAAAA KTCAAA AAAAxx 9496 1857 0 0 6 16 96 496 1496 4496 9496 192 193 GBAAAA LTCAAA HHHHxx 6873 1858 1 1 3 13 73 873 873 1873 6873 146 147 JEAAAA MTCAAA OOOOxx 4632 1859 0 0 2 12 32 632 632 4632 4632 64 65 EWAAAA NTCAAA VVVVxx 5758 1860 0 2 8 18 58 758 1758 758 5758 116 117 MNAAAA OTCAAA AAAAxx 6514 1861 0 2 4 14 14 514 514 1514 6514 28 29 OQAAAA PTCAAA HHHHxx 9510 1862 0 2 0 10 10 510 1510 4510 9510 20 21 UBAAAA QTCAAA OOOOxx 8411 1863 1 3 1 11 11 411 411 3411 8411 22 23 NLAAAA RTCAAA VVVVxx 7762 1864 0 2 2 2 62 762 1762 2762 7762 124 125 OMAAAA STCAAA AAAAxx 2225 1865 1 1 5 5 25 225 225 2225 2225 50 51 PHAAAA TTCAAA HHHHxx 4373 1866 1 1 3 13 73 373 373 4373 4373 146 147 FMAAAA UTCAAA OOOOxx 7326 1867 0 2 6 6 26 326 1326 2326 7326 52 53 UVAAAA VTCAAA VVVVxx 8651 1868 1 3 1 11 51 651 651 3651 8651 102 103 TUAAAA WTCAAA AAAAxx 9825 1869 1 1 5 5 25 825 1825 4825 9825 50 51 XNAAAA XTCAAA HHHHxx 2988 1870 0 0 8 8 88 988 988 2988 2988 176 177 YKAAAA YTCAAA OOOOxx 8138 1871 0 2 8 18 38 138 138 3138 8138 76 77 ABAAAA ZTCAAA VVVVxx 7792 1872 0 0 2 12 92 792 1792 2792 7792 184 185 SNAAAA AUCAAA AAAAxx 1232 1873 0 0 2 12 32 232 1232 1232 1232 64 65 KVAAAA BUCAAA HHHHxx 8221 1874 1 1 1 1 21 221 221 3221 8221 42 43 FEAAAA CUCAAA OOOOxx 4044 1875 0 0 4 4 44 44 44 4044 4044 88 89 OZAAAA DUCAAA VVVVxx 1204 1876 0 0 4 4 4 204 1204 1204 1204 8 9 IUAAAA EUCAAA AAAAxx 5145 1877 1 1 5 5 45 145 1145 145 5145 90 91 XPAAAA FUCAAA HHHHxx 7791 1878 1 3 1 11 91 791 1791 2791 7791 182 183 RNAAAA GUCAAA OOOOxx 8270 1879 0 2 0 10 70 270 270 3270 8270 140 141 CGAAAA HUCAAA VVVVxx 9427 1880 1 3 7 7 27 427 1427 4427 9427 54 55 PYAAAA IUCAAA AAAAxx 2152 1881 0 0 2 12 52 152 152 2152 2152 104 105 UEAAAA JUCAAA HHHHxx 7790 1882 0 2 0 10 90 790 1790 2790 7790 180 181 QNAAAA KUCAAA OOOOxx 5301 1883 1 1 1 1 1 301 1301 301 5301 2 3 XVAAAA LUCAAA VVVVxx 626 1884 0 2 6 6 26 626 626 626 626 52 53 CYAAAA MUCAAA AAAAxx 260 1885 0 0 0 0 60 260 260 260 260 120 121 AKAAAA NUCAAA HHHHxx 4369 1886 1 1 9 9 69 369 369 4369 4369 138 139 BMAAAA OUCAAA OOOOxx 5457 1887 1 1 7 17 57 457 1457 457 5457 114 115 XBAAAA PUCAAA VVVVxx 3468 1888 0 0 8 8 68 468 1468 3468 3468 136 137 KDAAAA QUCAAA AAAAxx 2257 1889 1 1 7 17 57 257 257 2257 2257 114 115 VIAAAA RUCAAA HHHHxx 9318 1890 0 2 8 18 18 318 1318 4318 9318 36 37 KUAAAA SUCAAA OOOOxx 8762 1891 0 2 2 2 62 762 762 3762 8762 124 125 AZAAAA TUCAAA VVVVxx 9153 1892 1 1 3 13 53 153 1153 4153 9153 106 107 BOAAAA UUCAAA AAAAxx 9220 1893 0 0 0 0 20 220 1220 4220 9220 40 41 QQAAAA VUCAAA HHHHxx 8003 1894 1 3 3 3 3 3 3 3003 8003 6 7 VVAAAA WUCAAA OOOOxx 7257 1895 1 1 7 17 57 257 1257 2257 7257 114 115 DTAAAA XUCAAA VVVVxx 3930 1896 0 2 0 10 30 930 1930 3930 3930 60 61 EVAAAA YUCAAA AAAAxx 2976 1897 0 0 6 16 76 976 976 2976 2976 152 153 MKAAAA ZUCAAA HHHHxx 2531 1898 1 3 1 11 31 531 531 2531 2531 62 63 JTAAAA AVCAAA OOOOxx 2250 1899 0 2 0 10 50 250 250 2250 2250 100 101 OIAAAA BVCAAA VVVVxx 8549 1900 1 1 9 9 49 549 549 3549 8549 98 99 VQAAAA CVCAAA AAAAxx 7197 1901 1 1 7 17 97 197 1197 2197 7197 194 195 VQAAAA DVCAAA HHHHxx 5916 1902 0 0 6 16 16 916 1916 916 5916 32 33 OTAAAA EVCAAA OOOOxx 5287 1903 1 3 7 7 87 287 1287 287 5287 174 175 JVAAAA FVCAAA VVVVxx 9095 1904 1 3 5 15 95 95 1095 4095 9095 190 191 VLAAAA GVCAAA AAAAxx 7137 1905 1 1 7 17 37 137 1137 2137 7137 74 75 NOAAAA HVCAAA HHHHxx 7902 1906 0 2 2 2 2 902 1902 2902 7902 4 5 YRAAAA IVCAAA OOOOxx 7598 1907 0 2 8 18 98 598 1598 2598 7598 196 197 GGAAAA JVCAAA VVVVxx 5652 1908 0 0 2 12 52 652 1652 652 5652 104 105 KJAAAA KVCAAA AAAAxx 2017 1909 1 1 7 17 17 17 17 2017 2017 34 35 PZAAAA LVCAAA HHHHxx 7255 1910 1 3 5 15 55 255 1255 2255 7255 110 111 BTAAAA MVCAAA OOOOxx 7999 1911 1 3 9 19 99 999 1999 2999 7999 198 199 RVAAAA NVCAAA VVVVxx 5388 1912 0 0 8 8 88 388 1388 388 5388 176 177 GZAAAA OVCAAA AAAAxx 8754 1913 0 2 4 14 54 754 754 3754 8754 108 109 SYAAAA PVCAAA HHHHxx 5415 1914 1 3 5 15 15 415 1415 415 5415 30 31 HAAAAA QVCAAA OOOOxx 8861 1915 1 1 1 1 61 861 861 3861 8861 122 123 VCAAAA RVCAAA VVVVxx 2874 1916 0 2 4 14 74 874 874 2874 2874 148 149 OGAAAA SVCAAA AAAAxx 9910 1917 0 2 0 10 10 910 1910 4910 9910 20 21 ERAAAA TVCAAA HHHHxx 5178 1918 0 2 8 18 78 178 1178 178 5178 156 157 ERAAAA UVCAAA OOOOxx 5698 1919 0 2 8 18 98 698 1698 698 5698 196 197 ELAAAA VVCAAA VVVVxx 8500 1920 0 0 0 0 0 500 500 3500 8500 0 1 YOAAAA WVCAAA AAAAxx 1814 1921 0 2 4 14 14 814 1814 1814 1814 28 29 URAAAA XVCAAA HHHHxx 4968 1922 0 0 8 8 68 968 968 4968 4968 136 137 CJAAAA YVCAAA OOOOxx 2642 1923 0 2 2 2 42 642 642 2642 2642 84 85 QXAAAA ZVCAAA VVVVxx 1578 1924 0 2 8 18 78 578 1578 1578 1578 156 157 SIAAAA AWCAAA AAAAxx 4774 1925 0 2 4 14 74 774 774 4774 4774 148 149 QBAAAA BWCAAA HHHHxx 7062 1926 0 2 2 2 62 62 1062 2062 7062 124 125 QLAAAA CWCAAA OOOOxx 5381 1927 1 1 1 1 81 381 1381 381 5381 162 163 ZYAAAA DWCAAA VVVVxx 7985 1928 1 1 5 5 85 985 1985 2985 7985 170 171 DVAAAA EWCAAA AAAAxx 3850 1929 0 2 0 10 50 850 1850 3850 3850 100 101 CSAAAA FWCAAA HHHHxx 5624 1930 0 0 4 4 24 624 1624 624 5624 48 49 IIAAAA GWCAAA OOOOxx 8948 1931 0 0 8 8 48 948 948 3948 8948 96 97 EGAAAA HWCAAA VVVVxx 995 1932 1 3 5 15 95 995 995 995 995 190 191 HMAAAA IWCAAA AAAAxx 5058 1933 0 2 8 18 58 58 1058 58 5058 116 117 OMAAAA JWCAAA HHHHxx 9670 1934 0 2 0 10 70 670 1670 4670 9670 140 141 YHAAAA KWCAAA OOOOxx 3115 1935 1 3 5 15 15 115 1115 3115 3115 30 31 VPAAAA LWCAAA VVVVxx 4935 1936 1 3 5 15 35 935 935 4935 4935 70 71 VHAAAA MWCAAA AAAAxx 4735 1937 1 3 5 15 35 735 735 4735 4735 70 71 DAAAAA NWCAAA HHHHxx 1348 1938 0 0 8 8 48 348 1348 1348 1348 96 97 WZAAAA OWCAAA OOOOxx 2380 1939 0 0 0 0 80 380 380 2380 2380 160 161 ONAAAA PWCAAA VVVVxx 4246 1940 0 2 6 6 46 246 246 4246 4246 92 93 IHAAAA QWCAAA AAAAxx 522 1941 0 2 2 2 22 522 522 522 522 44 45 CUAAAA RWCAAA HHHHxx 1701 1942 1 1 1 1 1 701 1701 1701 1701 2 3 LNAAAA SWCAAA OOOOxx 9709 1943 1 1 9 9 9 709 1709 4709 9709 18 19 LJAAAA TWCAAA VVVVxx 8829 1944 1 1 9 9 29 829 829 3829 8829 58 59 PBAAAA UWCAAA AAAAxx 7936 1945 0 0 6 16 36 936 1936 2936 7936 72 73 GTAAAA VWCAAA HHHHxx 8474 1946 0 2 4 14 74 474 474 3474 8474 148 149 YNAAAA WWCAAA OOOOxx 4676 1947 0 0 6 16 76 676 676 4676 4676 152 153 WXAAAA XWCAAA VVVVxx 6303 1948 1 3 3 3 3 303 303 1303 6303 6 7 LIAAAA YWCAAA AAAAxx 3485 1949 1 1 5 5 85 485 1485 3485 3485 170 171 BEAAAA ZWCAAA HHHHxx 2695 1950 1 3 5 15 95 695 695 2695 2695 190 191 RZAAAA AXCAAA OOOOxx 8830 1951 0 2 0 10 30 830 830 3830 8830 60 61 QBAAAA BXCAAA VVVVxx 898 1952 0 2 8 18 98 898 898 898 898 196 197 OIAAAA CXCAAA AAAAxx 7268 1953 0 0 8 8 68 268 1268 2268 7268 136 137 OTAAAA DXCAAA HHHHxx 6568 1954 0 0 8 8 68 568 568 1568 6568 136 137 QSAAAA EXCAAA OOOOxx 9724 1955 0 0 4 4 24 724 1724 4724 9724 48 49 AKAAAA FXCAAA VVVVxx 3329 1956 1 1 9 9 29 329 1329 3329 3329 58 59 BYAAAA GXCAAA AAAAxx 9860 1957 0 0 0 0 60 860 1860 4860 9860 120 121 GPAAAA HXCAAA HHHHxx 6833 1958 1 1 3 13 33 833 833 1833 6833 66 67 VCAAAA IXCAAA OOOOxx 5956 1959 0 0 6 16 56 956 1956 956 5956 112 113 CVAAAA JXCAAA VVVVxx 3963 1960 1 3 3 3 63 963 1963 3963 3963 126 127 LWAAAA KXCAAA AAAAxx 883 1961 1 3 3 3 83 883 883 883 883 166 167 ZHAAAA LXCAAA HHHHxx 2761 1962 1 1 1 1 61 761 761 2761 2761 122 123 FCAAAA MXCAAA OOOOxx 4644 1963 0 0 4 4 44 644 644 4644 4644 88 89 QWAAAA NXCAAA VVVVxx 1358 1964 0 2 8 18 58 358 1358 1358 1358 116 117 GAAAAA OXCAAA AAAAxx 2049 1965 1 1 9 9 49 49 49 2049 2049 98 99 VAAAAA PXCAAA HHHHxx 2193 1966 1 1 3 13 93 193 193 2193 2193 186 187 JGAAAA QXCAAA OOOOxx 9435 1967 1 3 5 15 35 435 1435 4435 9435 70 71 XYAAAA RXCAAA VVVVxx 5890 1968 0 2 0 10 90 890 1890 890 5890 180 181 OSAAAA SXCAAA AAAAxx 8149 1969 1 1 9 9 49 149 149 3149 8149 98 99 LBAAAA TXCAAA HHHHxx 423 1970 1 3 3 3 23 423 423 423 423 46 47 HQAAAA UXCAAA OOOOxx 7980 1971 0 0 0 0 80 980 1980 2980 7980 160 161 YUAAAA VXCAAA VVVVxx 9019 1972 1 3 9 19 19 19 1019 4019 9019 38 39 XIAAAA WXCAAA AAAAxx 1647 1973 1 3 7 7 47 647 1647 1647 1647 94 95 JLAAAA XXCAAA HHHHxx 9495 1974 1 3 5 15 95 495 1495 4495 9495 190 191 FBAAAA YXCAAA OOOOxx 3904 1975 0 0 4 4 4 904 1904 3904 3904 8 9 EUAAAA ZXCAAA VVVVxx 5838 1976 0 2 8 18 38 838 1838 838 5838 76 77 OQAAAA AYCAAA AAAAxx 3866 1977 0 2 6 6 66 866 1866 3866 3866 132 133 SSAAAA BYCAAA HHHHxx 3093 1978 1 1 3 13 93 93 1093 3093 3093 186 187 ZOAAAA CYCAAA OOOOxx 9666 1979 0 2 6 6 66 666 1666 4666 9666 132 133 UHAAAA DYCAAA VVVVxx 1246 1980 0 2 6 6 46 246 1246 1246 1246 92 93 YVAAAA EYCAAA AAAAxx 9759 1981 1 3 9 19 59 759 1759 4759 9759 118 119 JLAAAA FYCAAA HHHHxx 7174 1982 0 2 4 14 74 174 1174 2174 7174 148 149 YPAAAA GYCAAA OOOOxx 7678 1983 0 2 8 18 78 678 1678 2678 7678 156 157 IJAAAA HYCAAA VVVVxx 3004 1984 0 0 4 4 4 4 1004 3004 3004 8 9 OLAAAA IYCAAA AAAAxx 5607 1985 1 3 7 7 7 607 1607 607 5607 14 15 RHAAAA JYCAAA HHHHxx 8510 1986 0 2 0 10 10 510 510 3510 8510 20 21 IPAAAA KYCAAA OOOOxx 1483 1987 1 3 3 3 83 483 1483 1483 1483 166 167 BFAAAA LYCAAA VVVVxx 2915 1988 1 3 5 15 15 915 915 2915 2915 30 31 DIAAAA MYCAAA AAAAxx 1548 1989 0 0 8 8 48 548 1548 1548 1548 96 97 OHAAAA NYCAAA HHHHxx 5767 1990 1 3 7 7 67 767 1767 767 5767 134 135 VNAAAA OYCAAA OOOOxx 3214 1991 0 2 4 14 14 214 1214 3214 3214 28 29 QTAAAA PYCAAA VVVVxx 8663 1992 1 3 3 3 63 663 663 3663 8663 126 127 FVAAAA QYCAAA AAAAxx 5425 1993 1 1 5 5 25 425 1425 425 5425 50 51 RAAAAA RYCAAA HHHHxx 8530 1994 0 2 0 10 30 530 530 3530 8530 60 61 CQAAAA SYCAAA OOOOxx 821 1995 1 1 1 1 21 821 821 821 821 42 43 PFAAAA TYCAAA VVVVxx 8816 1996 0 0 6 16 16 816 816 3816 8816 32 33 CBAAAA UYCAAA AAAAxx 9367 1997 1 3 7 7 67 367 1367 4367 9367 134 135 HWAAAA VYCAAA HHHHxx 4138 1998 0 2 8 18 38 138 138 4138 4138 76 77 EDAAAA WYCAAA OOOOxx 94 1999 0 2 4 14 94 94 94 94 94 188 189 QDAAAA XYCAAA VVVVxx 1858 2000 0 2 8 18 58 858 1858 1858 1858 116 117 MTAAAA YYCAAA AAAAxx 5513 2001 1 1 3 13 13 513 1513 513 5513 26 27 BEAAAA ZYCAAA HHHHxx 9620 2002 0 0 0 0 20 620 1620 4620 9620 40 41 AGAAAA AZCAAA OOOOxx 4770 2003 0 2 0 10 70 770 770 4770 4770 140 141 MBAAAA BZCAAA VVVVxx 5193 2004 1 1 3 13 93 193 1193 193 5193 186 187 TRAAAA CZCAAA AAAAxx 198 2005 0 2 8 18 98 198 198 198 198 196 197 QHAAAA DZCAAA HHHHxx 417 2006 1 1 7 17 17 417 417 417 417 34 35 BQAAAA EZCAAA OOOOxx 173 2007 1 1 3 13 73 173 173 173 173 146 147 RGAAAA FZCAAA VVVVxx 6248 2008 0 0 8 8 48 248 248 1248 6248 96 97 IGAAAA GZCAAA AAAAxx 302 2009 0 2 2 2 2 302 302 302 302 4 5 QLAAAA HZCAAA HHHHxx 8983 2010 1 3 3 3 83 983 983 3983 8983 166 167 NHAAAA IZCAAA OOOOxx 4840 2011 0 0 0 0 40 840 840 4840 4840 80 81 EEAAAA JZCAAA VVVVxx 2876 2012 0 0 6 16 76 876 876 2876 2876 152 153 QGAAAA KZCAAA AAAAxx 5841 2013 1 1 1 1 41 841 1841 841 5841 82 83 RQAAAA LZCAAA HHHHxx 2766 2014 0 2 6 6 66 766 766 2766 2766 132 133 KCAAAA MZCAAA OOOOxx 9482 2015 0 2 2 2 82 482 1482 4482 9482 164 165 SAAAAA NZCAAA VVVVxx 5335 2016 1 3 5 15 35 335 1335 335 5335 70 71 FXAAAA OZCAAA AAAAxx 1502 2017 0 2 2 2 2 502 1502 1502 1502 4 5 UFAAAA PZCAAA HHHHxx 9291 2018 1 3 1 11 91 291 1291 4291 9291 182 183 JTAAAA QZCAAA OOOOxx 8655 2019 1 3 5 15 55 655 655 3655 8655 110 111 XUAAAA RZCAAA VVVVxx 1687 2020 1 3 7 7 87 687 1687 1687 1687 174 175 XMAAAA SZCAAA AAAAxx 8171 2021 1 3 1 11 71 171 171 3171 8171 142 143 HCAAAA TZCAAA HHHHxx 5699 2022 1 3 9 19 99 699 1699 699 5699 198 199 FLAAAA UZCAAA OOOOxx 1462 2023 0 2 2 2 62 462 1462 1462 1462 124 125 GEAAAA VZCAAA VVVVxx 608 2024 0 0 8 8 8 608 608 608 608 16 17 KXAAAA WZCAAA AAAAxx 6860 2025 0 0 0 0 60 860 860 1860 6860 120 121 WDAAAA XZCAAA HHHHxx 6063 2026 1 3 3 3 63 63 63 1063 6063 126 127 FZAAAA YZCAAA OOOOxx 1422 2027 0 2 2 2 22 422 1422 1422 1422 44 45 SCAAAA ZZCAAA VVVVxx 1932 2028 0 0 2 12 32 932 1932 1932 1932 64 65 IWAAAA AADAAA AAAAxx 5065 2029 1 1 5 5 65 65 1065 65 5065 130 131 VMAAAA BADAAA HHHHxx 432 2030 0 0 2 12 32 432 432 432 432 64 65 QQAAAA CADAAA OOOOxx 4680 2031 0 0 0 0 80 680 680 4680 4680 160 161 AYAAAA DADAAA VVVVxx 8172 2032 0 0 2 12 72 172 172 3172 8172 144 145 ICAAAA EADAAA AAAAxx 8668 2033 0 0 8 8 68 668 668 3668 8668 136 137 KVAAAA FADAAA HHHHxx 256 2034 0 0 6 16 56 256 256 256 256 112 113 WJAAAA GADAAA OOOOxx 2500 2035 0 0 0 0 0 500 500 2500 2500 0 1 ESAAAA HADAAA VVVVxx 274 2036 0 2 4 14 74 274 274 274 274 148 149 OKAAAA IADAAA AAAAxx 5907 2037 1 3 7 7 7 907 1907 907 5907 14 15 FTAAAA JADAAA HHHHxx 8587 2038 1 3 7 7 87 587 587 3587 8587 174 175 HSAAAA KADAAA OOOOxx 9942 2039 0 2 2 2 42 942 1942 4942 9942 84 85 KSAAAA LADAAA VVVVxx 116 2040 0 0 6 16 16 116 116 116 116 32 33 MEAAAA MADAAA AAAAxx 7134 2041 0 2 4 14 34 134 1134 2134 7134 68 69 KOAAAA NADAAA HHHHxx 9002 2042 0 2 2 2 2 2 1002 4002 9002 4 5 GIAAAA OADAAA OOOOxx 1209 2043 1 1 9 9 9 209 1209 1209 1209 18 19 NUAAAA PADAAA VVVVxx 9983 2044 1 3 3 3 83 983 1983 4983 9983 166 167 ZTAAAA QADAAA AAAAxx 1761 2045 1 1 1 1 61 761 1761 1761 1761 122 123 TPAAAA RADAAA HHHHxx 7723 2046 1 3 3 3 23 723 1723 2723 7723 46 47 BLAAAA SADAAA OOOOxx 6518 2047 0 2 8 18 18 518 518 1518 6518 36 37 SQAAAA TADAAA VVVVxx 1372 2048 0 0 2 12 72 372 1372 1372 1372 144 145 UAAAAA UADAAA AAAAxx 3587 2049 1 3 7 7 87 587 1587 3587 3587 174 175 ZHAAAA VADAAA HHHHxx 5323 2050 1 3 3 3 23 323 1323 323 5323 46 47 TWAAAA WADAAA OOOOxx 5902 2051 0 2 2 2 2 902 1902 902 5902 4 5 ATAAAA XADAAA VVVVxx 3749 2052 1 1 9 9 49 749 1749 3749 3749 98 99 FOAAAA YADAAA AAAAxx 5965 2053 1 1 5 5 65 965 1965 965 5965 130 131 LVAAAA ZADAAA HHHHxx 663 2054 1 3 3 3 63 663 663 663 663 126 127 NZAAAA ABDAAA OOOOxx 36 2055 0 0 6 16 36 36 36 36 36 72 73 KBAAAA BBDAAA VVVVxx 9782 2056 0 2 2 2 82 782 1782 4782 9782 164 165 GMAAAA CBDAAA AAAAxx 5412 2057 0 0 2 12 12 412 1412 412 5412 24 25 EAAAAA DBDAAA HHHHxx 9961 2058 1 1 1 1 61 961 1961 4961 9961 122 123 DTAAAA EBDAAA OOOOxx 6492 2059 0 0 2 12 92 492 492 1492 6492 184 185 SPAAAA FBDAAA VVVVxx 4234 2060 0 2 4 14 34 234 234 4234 4234 68 69 WGAAAA GBDAAA AAAAxx 4922 2061 0 2 2 2 22 922 922 4922 4922 44 45 IHAAAA HBDAAA HHHHxx 6166 2062 0 2 6 6 66 166 166 1166 6166 132 133 EDAAAA IBDAAA OOOOxx 7019 2063 1 3 9 19 19 19 1019 2019 7019 38 39 ZJAAAA JBDAAA VVVVxx 7805 2064 1 1 5 5 5 805 1805 2805 7805 10 11 FOAAAA KBDAAA AAAAxx 9808 2065 0 0 8 8 8 808 1808 4808 9808 16 17 GNAAAA LBDAAA HHHHxx 2550 2066 0 2 0 10 50 550 550 2550 2550 100 101 CUAAAA MBDAAA OOOOxx 8626 2067 0 2 6 6 26 626 626 3626 8626 52 53 UTAAAA NBDAAA VVVVxx 5649 2068 1 1 9 9 49 649 1649 649 5649 98 99 HJAAAA OBDAAA AAAAxx 3117 2069 1 1 7 17 17 117 1117 3117 3117 34 35 XPAAAA PBDAAA HHHHxx 866 2070 0 2 6 6 66 866 866 866 866 132 133 IHAAAA QBDAAA OOOOxx 2323 2071 1 3 3 3 23 323 323 2323 2323 46 47 JLAAAA RBDAAA VVVVxx 5132 2072 0 0 2 12 32 132 1132 132 5132 64 65 KPAAAA SBDAAA AAAAxx 9222 2073 0 2 2 2 22 222 1222 4222 9222 44 45 SQAAAA TBDAAA HHHHxx 3934 2074 0 2 4 14 34 934 1934 3934 3934 68 69 IVAAAA UBDAAA OOOOxx 4845 2075 1 1 5 5 45 845 845 4845 4845 90 91 JEAAAA VBDAAA VVVVxx 7714 2076 0 2 4 14 14 714 1714 2714 7714 28 29 SKAAAA WBDAAA AAAAxx 9818 2077 0 2 8 18 18 818 1818 4818 9818 36 37 QNAAAA XBDAAA HHHHxx 2219 2078 1 3 9 19 19 219 219 2219 2219 38 39 JHAAAA YBDAAA OOOOxx 6573 2079 1 1 3 13 73 573 573 1573 6573 146 147 VSAAAA ZBDAAA VVVVxx 4555 2080 1 3 5 15 55 555 555 4555 4555 110 111 FTAAAA ACDAAA AAAAxx 7306 2081 0 2 6 6 6 306 1306 2306 7306 12 13 AVAAAA BCDAAA HHHHxx 9313 2082 1 1 3 13 13 313 1313 4313 9313 26 27 FUAAAA CCDAAA OOOOxx 3924 2083 0 0 4 4 24 924 1924 3924 3924 48 49 YUAAAA DCDAAA VVVVxx 5176 2084 0 0 6 16 76 176 1176 176 5176 152 153 CRAAAA ECDAAA AAAAxx 9767 2085 1 3 7 7 67 767 1767 4767 9767 134 135 RLAAAA FCDAAA HHHHxx 905 2086 1 1 5 5 5 905 905 905 905 10 11 VIAAAA GCDAAA OOOOxx 8037 2087 1 1 7 17 37 37 37 3037 8037 74 75 DXAAAA HCDAAA VVVVxx 8133 2088 1 1 3 13 33 133 133 3133 8133 66 67 VAAAAA ICDAAA AAAAxx 2954 2089 0 2 4 14 54 954 954 2954 2954 108 109 QJAAAA JCDAAA HHHHxx 7262 2090 0 2 2 2 62 262 1262 2262 7262 124 125 ITAAAA KCDAAA OOOOxx 8768 2091 0 0 8 8 68 768 768 3768 8768 136 137 GZAAAA LCDAAA VVVVxx 6953 2092 1 1 3 13 53 953 953 1953 6953 106 107 LHAAAA MCDAAA AAAAxx 1984 2093 0 0 4 4 84 984 1984 1984 1984 168 169 IYAAAA NCDAAA HHHHxx 9348 2094 0 0 8 8 48 348 1348 4348 9348 96 97 OVAAAA OCDAAA OOOOxx 7769 2095 1 1 9 9 69 769 1769 2769 7769 138 139 VMAAAA PCDAAA VVVVxx 2994 2096 0 2 4 14 94 994 994 2994 2994 188 189 ELAAAA QCDAAA AAAAxx 5938 2097 0 2 8 18 38 938 1938 938 5938 76 77 KUAAAA RCDAAA HHHHxx 556 2098 0 0 6 16 56 556 556 556 556 112 113 KVAAAA SCDAAA OOOOxx 2577 2099 1 1 7 17 77 577 577 2577 2577 154 155 DVAAAA TCDAAA VVVVxx 8733 2100 1 1 3 13 33 733 733 3733 8733 66 67 XXAAAA UCDAAA AAAAxx 3108 2101 0 0 8 8 8 108 1108 3108 3108 16 17 OPAAAA VCDAAA HHHHxx 4166 2102 0 2 6 6 66 166 166 4166 4166 132 133 GEAAAA WCDAAA OOOOxx 3170 2103 0 2 0 10 70 170 1170 3170 3170 140 141 YRAAAA XCDAAA VVVVxx 8118 2104 0 2 8 18 18 118 118 3118 8118 36 37 GAAAAA YCDAAA AAAAxx 8454 2105 0 2 4 14 54 454 454 3454 8454 108 109 ENAAAA ZCDAAA HHHHxx 5338 2106 0 2 8 18 38 338 1338 338 5338 76 77 IXAAAA ADDAAA OOOOxx 402 2107 0 2 2 2 2 402 402 402 402 4 5 MPAAAA BDDAAA VVVVxx 5673 2108 1 1 3 13 73 673 1673 673 5673 146 147 FKAAAA CDDAAA AAAAxx 4324 2109 0 0 4 4 24 324 324 4324 4324 48 49 IKAAAA DDDAAA HHHHxx 1943 2110 1 3 3 3 43 943 1943 1943 1943 86 87 TWAAAA EDDAAA OOOOxx 7703 2111 1 3 3 3 3 703 1703 2703 7703 6 7 HKAAAA FDDAAA VVVVxx 7180 2112 0 0 0 0 80 180 1180 2180 7180 160 161 EQAAAA GDDAAA AAAAxx 5478 2113 0 2 8 18 78 478 1478 478 5478 156 157 SCAAAA HDDAAA HHHHxx 5775 2114 1 3 5 15 75 775 1775 775 5775 150 151 DOAAAA IDDAAA OOOOxx 6952 2115 0 0 2 12 52 952 952 1952 6952 104 105 KHAAAA JDDAAA VVVVxx 9022 2116 0 2 2 2 22 22 1022 4022 9022 44 45 AJAAAA KDDAAA AAAAxx 547 2117 1 3 7 7 47 547 547 547 547 94 95 BVAAAA LDDAAA HHHHxx 5877 2118 1 1 7 17 77 877 1877 877 5877 154 155 BSAAAA MDDAAA OOOOxx 9580 2119 0 0 0 0 80 580 1580 4580 9580 160 161 MEAAAA NDDAAA VVVVxx 6094 2120 0 2 4 14 94 94 94 1094 6094 188 189 KAAAAA ODDAAA AAAAxx 3398 2121 0 2 8 18 98 398 1398 3398 3398 196 197 SAAAAA PDDAAA HHHHxx 4574 2122 0 2 4 14 74 574 574 4574 4574 148 149 YTAAAA QDDAAA OOOOxx 3675 2123 1 3 5 15 75 675 1675 3675 3675 150 151 JLAAAA RDDAAA VVVVxx 6413 2124 1 1 3 13 13 413 413 1413 6413 26 27 RMAAAA SDDAAA AAAAxx 9851 2125 1 3 1 11 51 851 1851 4851 9851 102 103 XOAAAA TDDAAA HHHHxx 126 2126 0 2 6 6 26 126 126 126 126 52 53 WEAAAA UDDAAA OOOOxx 6803 2127 1 3 3 3 3 803 803 1803 6803 6 7 RBAAAA VDDAAA VVVVxx 6949 2128 1 1 9 9 49 949 949 1949 6949 98 99 HHAAAA WDDAAA AAAAxx 115 2129 1 3 5 15 15 115 115 115 115 30 31 LEAAAA XDDAAA HHHHxx 4165 2130 1 1 5 5 65 165 165 4165 4165 130 131 FEAAAA YDDAAA OOOOxx 201 2131 1 1 1 1 1 201 201 201 201 2 3 THAAAA ZDDAAA VVVVxx 9324 2132 0 0 4 4 24 324 1324 4324 9324 48 49 QUAAAA AEDAAA AAAAxx 6562 2133 0 2 2 2 62 562 562 1562 6562 124 125 KSAAAA BEDAAA HHHHxx 1917 2134 1 1 7 17 17 917 1917 1917 1917 34 35 TVAAAA CEDAAA OOOOxx 558 2135 0 2 8 18 58 558 558 558 558 116 117 MVAAAA DEDAAA VVVVxx 8515 2136 1 3 5 15 15 515 515 3515 8515 30 31 NPAAAA EEDAAA AAAAxx 6321 2137 1 1 1 1 21 321 321 1321 6321 42 43 DJAAAA FEDAAA HHHHxx 6892 2138 0 0 2 12 92 892 892 1892 6892 184 185 CFAAAA GEDAAA OOOOxx 1001 2139 1 1 1 1 1 1 1001 1001 1001 2 3 NMAAAA HEDAAA VVVVxx 2858 2140 0 2 8 18 58 858 858 2858 2858 116 117 YFAAAA IEDAAA AAAAxx 2434 2141 0 2 4 14 34 434 434 2434 2434 68 69 QPAAAA JEDAAA HHHHxx 4460 2142 0 0 0 0 60 460 460 4460 4460 120 121 OPAAAA KEDAAA OOOOxx 5447 2143 1 3 7 7 47 447 1447 447 5447 94 95 NBAAAA LEDAAA VVVVxx 3799 2144 1 3 9 19 99 799 1799 3799 3799 198 199 DQAAAA MEDAAA AAAAxx 4310 2145 0 2 0 10 10 310 310 4310 4310 20 21 UJAAAA NEDAAA HHHHxx 405 2146 1 1 5 5 5 405 405 405 405 10 11 PPAAAA OEDAAA OOOOxx 4573 2147 1 1 3 13 73 573 573 4573 4573 146 147 XTAAAA PEDAAA VVVVxx 706 2148 0 2 6 6 6 706 706 706 706 12 13 EBAAAA QEDAAA AAAAxx 7619 2149 1 3 9 19 19 619 1619 2619 7619 38 39 BHAAAA REDAAA HHHHxx 7959 2150 1 3 9 19 59 959 1959 2959 7959 118 119 DUAAAA SEDAAA OOOOxx 6712 2151 0 0 2 12 12 712 712 1712 6712 24 25 EYAAAA TEDAAA VVVVxx 6959 2152 1 3 9 19 59 959 959 1959 6959 118 119 RHAAAA UEDAAA AAAAxx 9791 2153 1 3 1 11 91 791 1791 4791 9791 182 183 PMAAAA VEDAAA HHHHxx 2112 2154 0 0 2 12 12 112 112 2112 2112 24 25 GDAAAA WEDAAA OOOOxx 9114 2155 0 2 4 14 14 114 1114 4114 9114 28 29 OMAAAA XEDAAA VVVVxx 3506 2156 0 2 6 6 6 506 1506 3506 3506 12 13 WEAAAA YEDAAA AAAAxx 5002 2157 0 2 2 2 2 2 1002 2 5002 4 5 KKAAAA ZEDAAA HHHHxx 3518 2158 0 2 8 18 18 518 1518 3518 3518 36 37 IFAAAA AFDAAA OOOOxx 602 2159 0 2 2 2 2 602 602 602 602 4 5 EXAAAA BFDAAA VVVVxx 9060 2160 0 0 0 0 60 60 1060 4060 9060 120 121 MKAAAA CFDAAA AAAAxx 3292 2161 0 0 2 12 92 292 1292 3292 3292 184 185 QWAAAA DFDAAA HHHHxx 77 2162 1 1 7 17 77 77 77 77 77 154 155 ZCAAAA EFDAAA OOOOxx 1420 2163 0 0 0 0 20 420 1420 1420 1420 40 41 QCAAAA FFDAAA VVVVxx 6001 2164 1 1 1 1 1 1 1 1001 6001 2 3 VWAAAA GFDAAA AAAAxx 7477 2165 1 1 7 17 77 477 1477 2477 7477 154 155 PBAAAA HFDAAA HHHHxx 6655 2166 1 3 5 15 55 655 655 1655 6655 110 111 ZVAAAA IFDAAA OOOOxx 7845 2167 1 1 5 5 45 845 1845 2845 7845 90 91 TPAAAA JFDAAA VVVVxx 8484 2168 0 0 4 4 84 484 484 3484 8484 168 169 IOAAAA KFDAAA AAAAxx 4345 2169 1 1 5 5 45 345 345 4345 4345 90 91 DLAAAA LFDAAA HHHHxx 4250 2170 0 2 0 10 50 250 250 4250 4250 100 101 MHAAAA MFDAAA OOOOxx 2391 2171 1 3 1 11 91 391 391 2391 2391 182 183 ZNAAAA NFDAAA VVVVxx 6884 2172 0 0 4 4 84 884 884 1884 6884 168 169 UEAAAA OFDAAA AAAAxx 7270 2173 0 2 0 10 70 270 1270 2270 7270 140 141 QTAAAA PFDAAA HHHHxx 2499 2174 1 3 9 19 99 499 499 2499 2499 198 199 DSAAAA QFDAAA OOOOxx 7312 2175 0 0 2 12 12 312 1312 2312 7312 24 25 GVAAAA RFDAAA VVVVxx 7113 2176 1 1 3 13 13 113 1113 2113 7113 26 27 PNAAAA SFDAAA AAAAxx 6695 2177 1 3 5 15 95 695 695 1695 6695 190 191 NXAAAA TFDAAA HHHHxx 6521 2178 1 1 1 1 21 521 521 1521 6521 42 43 VQAAAA UFDAAA OOOOxx 272 2179 0 0 2 12 72 272 272 272 272 144 145 MKAAAA VFDAAA VVVVxx 9976 2180 0 0 6 16 76 976 1976 4976 9976 152 153 STAAAA WFDAAA AAAAxx 992 2181 0 0 2 12 92 992 992 992 992 184 185 EMAAAA XFDAAA HHHHxx 6158 2182 0 2 8 18 58 158 158 1158 6158 116 117 WCAAAA YFDAAA OOOOxx 3281 2183 1 1 1 1 81 281 1281 3281 3281 162 163 FWAAAA ZFDAAA VVVVxx 7446 2184 0 2 6 6 46 446 1446 2446 7446 92 93 KAAAAA AGDAAA AAAAxx 4679 2185 1 3 9 19 79 679 679 4679 4679 158 159 ZXAAAA BGDAAA HHHHxx 5203 2186 1 3 3 3 3 203 1203 203 5203 6 7 DSAAAA CGDAAA OOOOxx 9874 2187 0 2 4 14 74 874 1874 4874 9874 148 149 UPAAAA DGDAAA VVVVxx 8371 2188 1 3 1 11 71 371 371 3371 8371 142 143 ZJAAAA EGDAAA AAAAxx 9086 2189 0 2 6 6 86 86 1086 4086 9086 172 173 MLAAAA FGDAAA HHHHxx 430 2190 0 2 0 10 30 430 430 430 430 60 61 OQAAAA GGDAAA OOOOxx 8749 2191 1 1 9 9 49 749 749 3749 8749 98 99 NYAAAA HGDAAA VVVVxx 577 2192 1 1 7 17 77 577 577 577 577 154 155 FWAAAA IGDAAA AAAAxx 4884 2193 0 0 4 4 84 884 884 4884 4884 168 169 WFAAAA JGDAAA HHHHxx 3421 2194 1 1 1 1 21 421 1421 3421 3421 42 43 PBAAAA KGDAAA OOOOxx 2812 2195 0 0 2 12 12 812 812 2812 2812 24 25 EEAAAA LGDAAA VVVVxx 5958 2196 0 2 8 18 58 958 1958 958 5958 116 117 EVAAAA MGDAAA AAAAxx 9901 2197 1 1 1 1 1 901 1901 4901 9901 2 3 VQAAAA NGDAAA HHHHxx 8478 2198 0 2 8 18 78 478 478 3478 8478 156 157 COAAAA OGDAAA OOOOxx 6545 2199 1 1 5 5 45 545 545 1545 6545 90 91 TRAAAA PGDAAA VVVVxx 1479 2200 1 3 9 19 79 479 1479 1479 1479 158 159 XEAAAA QGDAAA AAAAxx 1046 2201 0 2 6 6 46 46 1046 1046 1046 92 93 GOAAAA RGDAAA HHHHxx 6372 2202 0 0 2 12 72 372 372 1372 6372 144 145 CLAAAA SGDAAA OOOOxx 8206 2203 0 2 6 6 6 206 206 3206 8206 12 13 QDAAAA TGDAAA VVVVxx 9544 2204 0 0 4 4 44 544 1544 4544 9544 88 89 CDAAAA UGDAAA AAAAxx 9287 2205 1 3 7 7 87 287 1287 4287 9287 174 175 FTAAAA VGDAAA HHHHxx 6786 2206 0 2 6 6 86 786 786 1786 6786 172 173 ABAAAA WGDAAA OOOOxx 6511 2207 1 3 1 11 11 511 511 1511 6511 22 23 LQAAAA XGDAAA VVVVxx 603 2208 1 3 3 3 3 603 603 603 603 6 7 FXAAAA YGDAAA AAAAxx 2022 2209 0 2 2 2 22 22 22 2022 2022 44 45 UZAAAA ZGDAAA HHHHxx 2086 2210 0 2 6 6 86 86 86 2086 2086 172 173 GCAAAA AHDAAA OOOOxx 1969 2211 1 1 9 9 69 969 1969 1969 1969 138 139 TXAAAA BHDAAA VVVVxx 4841 2212 1 1 1 1 41 841 841 4841 4841 82 83 FEAAAA CHDAAA AAAAxx 5845 2213 1 1 5 5 45 845 1845 845 5845 90 91 VQAAAA DHDAAA HHHHxx 4635 2214 1 3 5 15 35 635 635 4635 4635 70 71 HWAAAA EHDAAA OOOOxx 4658 2215 0 2 8 18 58 658 658 4658 4658 116 117 EXAAAA FHDAAA VVVVxx 2896 2216 0 0 6 16 96 896 896 2896 2896 192 193 KHAAAA GHDAAA AAAAxx 5179 2217 1 3 9 19 79 179 1179 179 5179 158 159 FRAAAA HHDAAA HHHHxx 8667 2218 1 3 7 7 67 667 667 3667 8667 134 135 JVAAAA IHDAAA OOOOxx 7294 2219 0 2 4 14 94 294 1294 2294 7294 188 189 OUAAAA JHDAAA VVVVxx 3706 2220 0 2 6 6 6 706 1706 3706 3706 12 13 OMAAAA KHDAAA AAAAxx 8389 2221 1 1 9 9 89 389 389 3389 8389 178 179 RKAAAA LHDAAA HHHHxx 2486 2222 0 2 6 6 86 486 486 2486 2486 172 173 QRAAAA MHDAAA OOOOxx 8743 2223 1 3 3 3 43 743 743 3743 8743 86 87 HYAAAA NHDAAA VVVVxx 2777 2224 1 1 7 17 77 777 777 2777 2777 154 155 VCAAAA OHDAAA AAAAxx 2113 2225 1 1 3 13 13 113 113 2113 2113 26 27 HDAAAA PHDAAA HHHHxx 2076 2226 0 0 6 16 76 76 76 2076 2076 152 153 WBAAAA QHDAAA OOOOxx 2300 2227 0 0 0 0 0 300 300 2300 2300 0 1 MKAAAA RHDAAA VVVVxx 6894 2228 0 2 4 14 94 894 894 1894 6894 188 189 EFAAAA SHDAAA AAAAxx 6939 2229 1 3 9 19 39 939 939 1939 6939 78 79 XGAAAA THDAAA HHHHxx 446 2230 0 2 6 6 46 446 446 446 446 92 93 ERAAAA UHDAAA OOOOxx 6218 2231 0 2 8 18 18 218 218 1218 6218 36 37 EFAAAA VHDAAA VVVVxx 1295 2232 1 3 5 15 95 295 1295 1295 1295 190 191 VXAAAA WHDAAA AAAAxx 5135 2233 1 3 5 15 35 135 1135 135 5135 70 71 NPAAAA XHDAAA HHHHxx 8122 2234 0 2 2 2 22 122 122 3122 8122 44 45 KAAAAA YHDAAA OOOOxx 316 2235 0 0 6 16 16 316 316 316 316 32 33 EMAAAA ZHDAAA VVVVxx 514 2236 0 2 4 14 14 514 514 514 514 28 29 UTAAAA AIDAAA AAAAxx 7970 2237 0 2 0 10 70 970 1970 2970 7970 140 141 OUAAAA BIDAAA HHHHxx 9350 2238 0 2 0 10 50 350 1350 4350 9350 100 101 QVAAAA CIDAAA OOOOxx 3700 2239 0 0 0 0 0 700 1700 3700 3700 0 1 IMAAAA DIDAAA VVVVxx 582 2240 0 2 2 2 82 582 582 582 582 164 165 KWAAAA EIDAAA AAAAxx 9722 2241 0 2 2 2 22 722 1722 4722 9722 44 45 YJAAAA FIDAAA HHHHxx 7398 2242 0 2 8 18 98 398 1398 2398 7398 196 197 OYAAAA GIDAAA OOOOxx 2265 2243 1 1 5 5 65 265 265 2265 2265 130 131 DJAAAA HIDAAA VVVVxx 3049 2244 1 1 9 9 49 49 1049 3049 3049 98 99 HNAAAA IIDAAA AAAAxx 9121 2245 1 1 1 1 21 121 1121 4121 9121 42 43 VMAAAA JIDAAA HHHHxx 4275 2246 1 3 5 15 75 275 275 4275 4275 150 151 LIAAAA KIDAAA OOOOxx 6567 2247 1 3 7 7 67 567 567 1567 6567 134 135 PSAAAA LIDAAA VVVVxx 6755 2248 1 3 5 15 55 755 755 1755 6755 110 111 VZAAAA MIDAAA AAAAxx 4535 2249 1 3 5 15 35 535 535 4535 4535 70 71 LSAAAA NIDAAA HHHHxx 7968 2250 0 0 8 8 68 968 1968 2968 7968 136 137 MUAAAA OIDAAA OOOOxx 3412 2251 0 0 2 12 12 412 1412 3412 3412 24 25 GBAAAA PIDAAA VVVVxx 6112 2252 0 0 2 12 12 112 112 1112 6112 24 25 CBAAAA QIDAAA AAAAxx 6805 2253 1 1 5 5 5 805 805 1805 6805 10 11 TBAAAA RIDAAA HHHHxx 2880 2254 0 0 0 0 80 880 880 2880 2880 160 161 UGAAAA SIDAAA OOOOxx 7710 2255 0 2 0 10 10 710 1710 2710 7710 20 21 OKAAAA TIDAAA VVVVxx 7949 2256 1 1 9 9 49 949 1949 2949 7949 98 99 TTAAAA UIDAAA AAAAxx 7043 2257 1 3 3 3 43 43 1043 2043 7043 86 87 XKAAAA VIDAAA HHHHxx 9012 2258 0 0 2 12 12 12 1012 4012 9012 24 25 QIAAAA WIDAAA OOOOxx 878 2259 0 2 8 18 78 878 878 878 878 156 157 UHAAAA XIDAAA VVVVxx 7930 2260 0 2 0 10 30 930 1930 2930 7930 60 61 ATAAAA YIDAAA AAAAxx 667 2261 1 3 7 7 67 667 667 667 667 134 135 RZAAAA ZIDAAA HHHHxx 1905 2262 1 1 5 5 5 905 1905 1905 1905 10 11 HVAAAA AJDAAA OOOOxx 4958 2263 0 2 8 18 58 958 958 4958 4958 116 117 SIAAAA BJDAAA VVVVxx 2973 2264 1 1 3 13 73 973 973 2973 2973 146 147 JKAAAA CJDAAA AAAAxx 3631 2265 1 3 1 11 31 631 1631 3631 3631 62 63 RJAAAA DJDAAA HHHHxx 5868 2266 0 0 8 8 68 868 1868 868 5868 136 137 SRAAAA EJDAAA OOOOxx 2873 2267 1 1 3 13 73 873 873 2873 2873 146 147 NGAAAA FJDAAA VVVVxx 6941 2268 1 1 1 1 41 941 941 1941 6941 82 83 ZGAAAA GJDAAA AAAAxx 6384 2269 0 0 4 4 84 384 384 1384 6384 168 169 OLAAAA HJDAAA HHHHxx 3806 2270 0 2 6 6 6 806 1806 3806 3806 12 13 KQAAAA IJDAAA OOOOxx 5079 2271 1 3 9 19 79 79 1079 79 5079 158 159 JNAAAA JJDAAA VVVVxx 1970 2272 0 2 0 10 70 970 1970 1970 1970 140 141 UXAAAA KJDAAA AAAAxx 7810 2273 0 2 0 10 10 810 1810 2810 7810 20 21 KOAAAA LJDAAA HHHHxx 4639 2274 1 3 9 19 39 639 639 4639 4639 78 79 LWAAAA MJDAAA OOOOxx 6527 2275 1 3 7 7 27 527 527 1527 6527 54 55 BRAAAA NJDAAA VVVVxx 8079 2276 1 3 9 19 79 79 79 3079 8079 158 159 TYAAAA OJDAAA AAAAxx 2740 2277 0 0 0 0 40 740 740 2740 2740 80 81 KBAAAA PJDAAA HHHHxx 2337 2278 1 1 7 17 37 337 337 2337 2337 74 75 XLAAAA QJDAAA OOOOxx 6670 2279 0 2 0 10 70 670 670 1670 6670 140 141 OWAAAA RJDAAA VVVVxx 2345 2280 1 1 5 5 45 345 345 2345 2345 90 91 FMAAAA SJDAAA AAAAxx 401 2281 1 1 1 1 1 401 401 401 401 2 3 LPAAAA TJDAAA HHHHxx 2704 2282 0 0 4 4 4 704 704 2704 2704 8 9 AAAAAA UJDAAA OOOOxx 5530 2283 0 2 0 10 30 530 1530 530 5530 60 61 SEAAAA VJDAAA VVVVxx 51 2284 1 3 1 11 51 51 51 51 51 102 103 ZBAAAA WJDAAA AAAAxx 4282 2285 0 2 2 2 82 282 282 4282 4282 164 165 SIAAAA XJDAAA HHHHxx 7336 2286 0 0 6 16 36 336 1336 2336 7336 72 73 EWAAAA YJDAAA OOOOxx 8320 2287 0 0 0 0 20 320 320 3320 8320 40 41 AIAAAA ZJDAAA VVVVxx 7772 2288 0 0 2 12 72 772 1772 2772 7772 144 145 YMAAAA AKDAAA AAAAxx 1894 2289 0 2 4 14 94 894 1894 1894 1894 188 189 WUAAAA BKDAAA HHHHxx 2320 2290 0 0 0 0 20 320 320 2320 2320 40 41 GLAAAA CKDAAA OOOOxx 6232 2291 0 0 2 12 32 232 232 1232 6232 64 65 SFAAAA DKDAAA VVVVxx 2833 2292 1 1 3 13 33 833 833 2833 2833 66 67 ZEAAAA EKDAAA AAAAxx 8265 2293 1 1 5 5 65 265 265 3265 8265 130 131 XFAAAA FKDAAA HHHHxx 4589 2294 1 1 9 9 89 589 589 4589 4589 178 179 NUAAAA GKDAAA OOOOxx 8182 2295 0 2 2 2 82 182 182 3182 8182 164 165 SCAAAA HKDAAA VVVVxx 8337 2296 1 1 7 17 37 337 337 3337 8337 74 75 RIAAAA IKDAAA AAAAxx 8210 2297 0 2 0 10 10 210 210 3210 8210 20 21 UDAAAA JKDAAA HHHHxx 1406 2298 0 2 6 6 6 406 1406 1406 1406 12 13 CCAAAA KKDAAA OOOOxx 4463 2299 1 3 3 3 63 463 463 4463 4463 126 127 RPAAAA LKDAAA VVVVxx 4347 2300 1 3 7 7 47 347 347 4347 4347 94 95 FLAAAA MKDAAA AAAAxx 181 2301 1 1 1 1 81 181 181 181 181 162 163 ZGAAAA NKDAAA HHHHxx 9986 2302 0 2 6 6 86 986 1986 4986 9986 172 173 CUAAAA OKDAAA OOOOxx 661 2303 1 1 1 1 61 661 661 661 661 122 123 LZAAAA PKDAAA VVVVxx 4105 2304 1 1 5 5 5 105 105 4105 4105 10 11 XBAAAA QKDAAA AAAAxx 2187 2305 1 3 7 7 87 187 187 2187 2187 174 175 DGAAAA RKDAAA HHHHxx 1628 2306 0 0 8 8 28 628 1628 1628 1628 56 57 QKAAAA SKDAAA OOOOxx 3119 2307 1 3 9 19 19 119 1119 3119 3119 38 39 ZPAAAA TKDAAA VVVVxx 6804 2308 0 0 4 4 4 804 804 1804 6804 8 9 SBAAAA UKDAAA AAAAxx 9918 2309 0 2 8 18 18 918 1918 4918 9918 36 37 MRAAAA VKDAAA HHHHxx 8916 2310 0 0 6 16 16 916 916 3916 8916 32 33 YEAAAA WKDAAA OOOOxx 6057 2311 1 1 7 17 57 57 57 1057 6057 114 115 ZYAAAA XKDAAA VVVVxx 3622 2312 0 2 2 2 22 622 1622 3622 3622 44 45 IJAAAA YKDAAA AAAAxx 9168 2313 0 0 8 8 68 168 1168 4168 9168 136 137 QOAAAA ZKDAAA HHHHxx 3720 2314 0 0 0 0 20 720 1720 3720 3720 40 41 CNAAAA ALDAAA OOOOxx 9927 2315 1 3 7 7 27 927 1927 4927 9927 54 55 VRAAAA BLDAAA VVVVxx 5616 2316 0 0 6 16 16 616 1616 616 5616 32 33 AIAAAA CLDAAA AAAAxx 5210 2317 0 2 0 10 10 210 1210 210 5210 20 21 KSAAAA DLDAAA HHHHxx 636 2318 0 0 6 16 36 636 636 636 636 72 73 MYAAAA ELDAAA OOOOxx 9936 2319 0 0 6 16 36 936 1936 4936 9936 72 73 ESAAAA FLDAAA VVVVxx 2316 2320 0 0 6 16 16 316 316 2316 2316 32 33 CLAAAA GLDAAA AAAAxx 4363 2321 1 3 3 3 63 363 363 4363 4363 126 127 VLAAAA HLDAAA HHHHxx 7657 2322 1 1 7 17 57 657 1657 2657 7657 114 115 NIAAAA ILDAAA OOOOxx 697 2323 1 1 7 17 97 697 697 697 697 194 195 VAAAAA JLDAAA VVVVxx 912 2324 0 0 2 12 12 912 912 912 912 24 25 CJAAAA KLDAAA AAAAxx 8806 2325 0 2 6 6 6 806 806 3806 8806 12 13 SAAAAA LLDAAA HHHHxx 9698 2326 0 2 8 18 98 698 1698 4698 9698 196 197 AJAAAA MLDAAA OOOOxx 6191 2327 1 3 1 11 91 191 191 1191 6191 182 183 DEAAAA NLDAAA VVVVxx 1188 2328 0 0 8 8 88 188 1188 1188 1188 176 177 STAAAA OLDAAA AAAAxx 7676 2329 0 0 6 16 76 676 1676 2676 7676 152 153 GJAAAA PLDAAA HHHHxx 7073 2330 1 1 3 13 73 73 1073 2073 7073 146 147 BMAAAA QLDAAA OOOOxx 8019 2331 1 3 9 19 19 19 19 3019 8019 38 39 LWAAAA RLDAAA VVVVxx 4726 2332 0 2 6 6 26 726 726 4726 4726 52 53 UZAAAA SLDAAA AAAAxx 4648 2333 0 0 8 8 48 648 648 4648 4648 96 97 UWAAAA TLDAAA HHHHxx 3227 2334 1 3 7 7 27 227 1227 3227 3227 54 55 DUAAAA ULDAAA OOOOxx 7232 2335 0 0 2 12 32 232 1232 2232 7232 64 65 ESAAAA VLDAAA VVVVxx 9761 2336 1 1 1 1 61 761 1761 4761 9761 122 123 LLAAAA WLDAAA AAAAxx 3105 2337 1 1 5 5 5 105 1105 3105 3105 10 11 LPAAAA XLDAAA HHHHxx 5266 2338 0 2 6 6 66 266 1266 266 5266 132 133 OUAAAA YLDAAA OOOOxx 6788 2339 0 0 8 8 88 788 788 1788 6788 176 177 CBAAAA ZLDAAA VVVVxx 2442 2340 0 2 2 2 42 442 442 2442 2442 84 85 YPAAAA AMDAAA AAAAxx 8198 2341 0 2 8 18 98 198 198 3198 8198 196 197 IDAAAA BMDAAA HHHHxx 5806 2342 0 2 6 6 6 806 1806 806 5806 12 13 IPAAAA CMDAAA OOOOxx 8928 2343 0 0 8 8 28 928 928 3928 8928 56 57 KFAAAA DMDAAA VVVVxx 1657 2344 1 1 7 17 57 657 1657 1657 1657 114 115 TLAAAA EMDAAA AAAAxx 9164 2345 0 0 4 4 64 164 1164 4164 9164 128 129 MOAAAA FMDAAA HHHHxx 1851 2346 1 3 1 11 51 851 1851 1851 1851 102 103 FTAAAA GMDAAA OOOOxx 4744 2347 0 0 4 4 44 744 744 4744 4744 88 89 MAAAAA HMDAAA VVVVxx 8055 2348 1 3 5 15 55 55 55 3055 8055 110 111 VXAAAA IMDAAA AAAAxx 1533 2349 1 1 3 13 33 533 1533 1533 1533 66 67 ZGAAAA JMDAAA HHHHxx 1260 2350 0 0 0 0 60 260 1260 1260 1260 120 121 MWAAAA KMDAAA OOOOxx 1290 2351 0 2 0 10 90 290 1290 1290 1290 180 181 QXAAAA LMDAAA VVVVxx 297 2352 1 1 7 17 97 297 297 297 297 194 195 LLAAAA MMDAAA AAAAxx 4145 2353 1 1 5 5 45 145 145 4145 4145 90 91 LDAAAA NMDAAA HHHHxx 863 2354 1 3 3 3 63 863 863 863 863 126 127 FHAAAA OMDAAA OOOOxx 3423 2355 1 3 3 3 23 423 1423 3423 3423 46 47 RBAAAA PMDAAA VVVVxx 8750 2356 0 2 0 10 50 750 750 3750 8750 100 101 OYAAAA QMDAAA AAAAxx 3546 2357 0 2 6 6 46 546 1546 3546 3546 92 93 KGAAAA RMDAAA HHHHxx 3678 2358 0 2 8 18 78 678 1678 3678 3678 156 157 MLAAAA SMDAAA OOOOxx 5313 2359 1 1 3 13 13 313 1313 313 5313 26 27 JWAAAA TMDAAA VVVVxx 6233 2360 1 1 3 13 33 233 233 1233 6233 66 67 TFAAAA UMDAAA AAAAxx 5802 2361 0 2 2 2 2 802 1802 802 5802 4 5 EPAAAA VMDAAA HHHHxx 7059 2362 1 3 9 19 59 59 1059 2059 7059 118 119 NLAAAA WMDAAA OOOOxx 6481 2363 1 1 1 1 81 481 481 1481 6481 162 163 HPAAAA XMDAAA VVVVxx 1596 2364 0 0 6 16 96 596 1596 1596 1596 192 193 KJAAAA YMDAAA AAAAxx 8181 2365 1 1 1 1 81 181 181 3181 8181 162 163 RCAAAA ZMDAAA HHHHxx 5368 2366 0 0 8 8 68 368 1368 368 5368 136 137 MYAAAA ANDAAA OOOOxx 9416 2367 0 0 6 16 16 416 1416 4416 9416 32 33 EYAAAA BNDAAA VVVVxx 9521 2368 1 1 1 1 21 521 1521 4521 9521 42 43 FCAAAA CNDAAA AAAAxx 1042 2369 0 2 2 2 42 42 1042 1042 1042 84 85 COAAAA DNDAAA HHHHxx 4503 2370 1 3 3 3 3 503 503 4503 4503 6 7 FRAAAA ENDAAA OOOOxx 3023 2371 1 3 3 3 23 23 1023 3023 3023 46 47 HMAAAA FNDAAA VVVVxx 1976 2372 0 0 6 16 76 976 1976 1976 1976 152 153 AYAAAA GNDAAA AAAAxx 5610 2373 0 2 0 10 10 610 1610 610 5610 20 21 UHAAAA HNDAAA HHHHxx 7410 2374 0 2 0 10 10 410 1410 2410 7410 20 21 AZAAAA INDAAA OOOOxx 7872 2375 0 0 2 12 72 872 1872 2872 7872 144 145 UQAAAA JNDAAA VVVVxx 8591 2376 1 3 1 11 91 591 591 3591 8591 182 183 LSAAAA KNDAAA AAAAxx 1804 2377 0 0 4 4 4 804 1804 1804 1804 8 9 KRAAAA LNDAAA HHHHxx 5299 2378 1 3 9 19 99 299 1299 299 5299 198 199 VVAAAA MNDAAA OOOOxx 4695 2379 1 3 5 15 95 695 695 4695 4695 190 191 PYAAAA NNDAAA VVVVxx 2672 2380 0 0 2 12 72 672 672 2672 2672 144 145 UYAAAA ONDAAA AAAAxx 585 2381 1 1 5 5 85 585 585 585 585 170 171 NWAAAA PNDAAA HHHHxx 8622 2382 0 2 2 2 22 622 622 3622 8622 44 45 QTAAAA QNDAAA OOOOxx 3780 2383 0 0 0 0 80 780 1780 3780 3780 160 161 KPAAAA RNDAAA VVVVxx 7941 2384 1 1 1 1 41 941 1941 2941 7941 82 83 LTAAAA SNDAAA AAAAxx 3305 2385 1 1 5 5 5 305 1305 3305 3305 10 11 DXAAAA TNDAAA HHHHxx 8653 2386 1 1 3 13 53 653 653 3653 8653 106 107 VUAAAA UNDAAA OOOOxx 5756 2387 0 0 6 16 56 756 1756 756 5756 112 113 KNAAAA VNDAAA VVVVxx 576 2388 0 0 6 16 76 576 576 576 576 152 153 EWAAAA WNDAAA AAAAxx 1915 2389 1 3 5 15 15 915 1915 1915 1915 30 31 RVAAAA XNDAAA HHHHxx 4627 2390 1 3 7 7 27 627 627 4627 4627 54 55 ZVAAAA YNDAAA OOOOxx 920 2391 0 0 0 0 20 920 920 920 920 40 41 KJAAAA ZNDAAA VVVVxx 2537 2392 1 1 7 17 37 537 537 2537 2537 74 75 PTAAAA AODAAA AAAAxx 50 2393 0 2 0 10 50 50 50 50 50 100 101 YBAAAA BODAAA HHHHxx 1313 2394 1 1 3 13 13 313 1313 1313 1313 26 27 NYAAAA CODAAA OOOOxx 8542 2395 0 2 2 2 42 542 542 3542 8542 84 85 OQAAAA DODAAA VVVVxx 6428 2396 0 0 8 8 28 428 428 1428 6428 56 57 GNAAAA EODAAA AAAAxx 4351 2397 1 3 1 11 51 351 351 4351 4351 102 103 JLAAAA FODAAA HHHHxx 2050 2398 0 2 0 10 50 50 50 2050 2050 100 101 WAAAAA GODAAA OOOOxx 5162 2399 0 2 2 2 62 162 1162 162 5162 124 125 OQAAAA HODAAA VVVVxx 8229 2400 1 1 9 9 29 229 229 3229 8229 58 59 NEAAAA IODAAA AAAAxx 7782 2401 0 2 2 2 82 782 1782 2782 7782 164 165 INAAAA JODAAA HHHHxx 1563 2402 1 3 3 3 63 563 1563 1563 1563 126 127 DIAAAA KODAAA OOOOxx 267 2403 1 3 7 7 67 267 267 267 267 134 135 HKAAAA LODAAA VVVVxx 5138 2404 0 2 8 18 38 138 1138 138 5138 76 77 QPAAAA MODAAA AAAAxx 7022 2405 0 2 2 2 22 22 1022 2022 7022 44 45 CKAAAA NODAAA HHHHxx 6705 2406 1 1 5 5 5 705 705 1705 6705 10 11 XXAAAA OODAAA OOOOxx 6190 2407 0 2 0 10 90 190 190 1190 6190 180 181 CEAAAA PODAAA VVVVxx 8226 2408 0 2 6 6 26 226 226 3226 8226 52 53 KEAAAA QODAAA AAAAxx 8882 2409 0 2 2 2 82 882 882 3882 8882 164 165 QDAAAA RODAAA HHHHxx 5181 2410 1 1 1 1 81 181 1181 181 5181 162 163 HRAAAA SODAAA OOOOxx 4598 2411 0 2 8 18 98 598 598 4598 4598 196 197 WUAAAA TODAAA VVVVxx 4882 2412 0 2 2 2 82 882 882 4882 4882 164 165 UFAAAA UODAAA AAAAxx 7490 2413 0 2 0 10 90 490 1490 2490 7490 180 181 CCAAAA VODAAA HHHHxx 5224 2414 0 0 4 4 24 224 1224 224 5224 48 49 YSAAAA WODAAA OOOOxx 2174 2415 0 2 4 14 74 174 174 2174 2174 148 149 QFAAAA XODAAA VVVVxx 3059 2416 1 3 9 19 59 59 1059 3059 3059 118 119 RNAAAA YODAAA AAAAxx 8790 2417 0 2 0 10 90 790 790 3790 8790 180 181 CAAAAA ZODAAA HHHHxx 2222 2418 0 2 2 2 22 222 222 2222 2222 44 45 MHAAAA APDAAA OOOOxx 5473 2419 1 1 3 13 73 473 1473 473 5473 146 147 NCAAAA BPDAAA VVVVxx 937 2420 1 1 7 17 37 937 937 937 937 74 75 BKAAAA CPDAAA AAAAxx 2975 2421 1 3 5 15 75 975 975 2975 2975 150 151 LKAAAA DPDAAA HHHHxx 9569 2422 1 1 9 9 69 569 1569 4569 9569 138 139 BEAAAA EPDAAA OOOOxx 3456 2423 0 0 6 16 56 456 1456 3456 3456 112 113 YCAAAA FPDAAA VVVVxx 6657 2424 1 1 7 17 57 657 657 1657 6657 114 115 BWAAAA GPDAAA AAAAxx 3776 2425 0 0 6 16 76 776 1776 3776 3776 152 153 GPAAAA HPDAAA HHHHxx 6072 2426 0 0 2 12 72 72 72 1072 6072 144 145 OZAAAA IPDAAA OOOOxx 8129 2427 1 1 9 9 29 129 129 3129 8129 58 59 RAAAAA JPDAAA VVVVxx 1085 2428 1 1 5 5 85 85 1085 1085 1085 170 171 TPAAAA KPDAAA AAAAxx 2079 2429 1 3 9 19 79 79 79 2079 2079 158 159 ZBAAAA LPDAAA HHHHxx 1200 2430 0 0 0 0 0 200 1200 1200 1200 0 1 EUAAAA MPDAAA OOOOxx 3276 2431 0 0 6 16 76 276 1276 3276 3276 152 153 AWAAAA NPDAAA VVVVxx 2608 2432 0 0 8 8 8 608 608 2608 2608 16 17 IWAAAA OPDAAA AAAAxx 702 2433 0 2 2 2 2 702 702 702 702 4 5 ABAAAA PPDAAA HHHHxx 5750 2434 0 2 0 10 50 750 1750 750 5750 100 101 ENAAAA QPDAAA OOOOxx 2776 2435 0 0 6 16 76 776 776 2776 2776 152 153 UCAAAA RPDAAA VVVVxx 9151 2436 1 3 1 11 51 151 1151 4151 9151 102 103 ZNAAAA SPDAAA AAAAxx 3282 2437 0 2 2 2 82 282 1282 3282 3282 164 165 GWAAAA TPDAAA HHHHxx 408 2438 0 0 8 8 8 408 408 408 408 16 17 SPAAAA UPDAAA OOOOxx 3473 2439 1 1 3 13 73 473 1473 3473 3473 146 147 PDAAAA VPDAAA VVVVxx 7095 2440 1 3 5 15 95 95 1095 2095 7095 190 191 XMAAAA WPDAAA AAAAxx 3288 2441 0 0 8 8 88 288 1288 3288 3288 176 177 MWAAAA XPDAAA HHHHxx 8215 2442 1 3 5 15 15 215 215 3215 8215 30 31 ZDAAAA YPDAAA OOOOxx 6244 2443 0 0 4 4 44 244 244 1244 6244 88 89 EGAAAA ZPDAAA VVVVxx 8440 2444 0 0 0 0 40 440 440 3440 8440 80 81 QMAAAA AQDAAA AAAAxx 3800 2445 0 0 0 0 0 800 1800 3800 3800 0 1 EQAAAA BQDAAA HHHHxx 7279 2446 1 3 9 19 79 279 1279 2279 7279 158 159 ZTAAAA CQDAAA OOOOxx 9206 2447 0 2 6 6 6 206 1206 4206 9206 12 13 CQAAAA DQDAAA VVVVxx 6465 2448 1 1 5 5 65 465 465 1465 6465 130 131 ROAAAA EQDAAA AAAAxx 4127 2449 1 3 7 7 27 127 127 4127 4127 54 55 TCAAAA FQDAAA HHHHxx 7463 2450 1 3 3 3 63 463 1463 2463 7463 126 127 BBAAAA GQDAAA OOOOxx 5117 2451 1 1 7 17 17 117 1117 117 5117 34 35 VOAAAA HQDAAA VVVVxx 4715 2452 1 3 5 15 15 715 715 4715 4715 30 31 JZAAAA IQDAAA AAAAxx 2010 2453 0 2 0 10 10 10 10 2010 2010 20 21 IZAAAA JQDAAA HHHHxx 6486 2454 0 2 6 6 86 486 486 1486 6486 172 173 MPAAAA KQDAAA OOOOxx 6434 2455 0 2 4 14 34 434 434 1434 6434 68 69 MNAAAA LQDAAA VVVVxx 2151 2456 1 3 1 11 51 151 151 2151 2151 102 103 TEAAAA MQDAAA AAAAxx 4821 2457 1 1 1 1 21 821 821 4821 4821 42 43 LDAAAA NQDAAA HHHHxx 6507 2458 1 3 7 7 7 507 507 1507 6507 14 15 HQAAAA OQDAAA OOOOxx 8741 2459 1 1 1 1 41 741 741 3741 8741 82 83 FYAAAA PQDAAA VVVVxx 6846 2460 0 2 6 6 46 846 846 1846 6846 92 93 IDAAAA QQDAAA AAAAxx 4525 2461 1 1 5 5 25 525 525 4525 4525 50 51 BSAAAA RQDAAA HHHHxx 8299 2462 1 3 9 19 99 299 299 3299 8299 198 199 FHAAAA SQDAAA OOOOxx 5465 2463 1 1 5 5 65 465 1465 465 5465 130 131 FCAAAA TQDAAA VVVVxx 7206 2464 0 2 6 6 6 206 1206 2206 7206 12 13 ERAAAA UQDAAA AAAAxx 2616 2465 0 0 6 16 16 616 616 2616 2616 32 33 QWAAAA VQDAAA HHHHxx 4440 2466 0 0 0 0 40 440 440 4440 4440 80 81 UOAAAA WQDAAA OOOOxx 6109 2467 1 1 9 9 9 109 109 1109 6109 18 19 ZAAAAA XQDAAA VVVVxx 7905 2468 1 1 5 5 5 905 1905 2905 7905 10 11 BSAAAA YQDAAA AAAAxx 6498 2469 0 2 8 18 98 498 498 1498 6498 196 197 YPAAAA ZQDAAA HHHHxx 2034 2470 0 2 4 14 34 34 34 2034 2034 68 69 GAAAAA ARDAAA OOOOxx 7693 2471 1 1 3 13 93 693 1693 2693 7693 186 187 XJAAAA BRDAAA VVVVxx 7511 2472 1 3 1 11 11 511 1511 2511 7511 22 23 XCAAAA CRDAAA AAAAxx 7531 2473 1 3 1 11 31 531 1531 2531 7531 62 63 RDAAAA DRDAAA HHHHxx 6869 2474 1 1 9 9 69 869 869 1869 6869 138 139 FEAAAA ERDAAA OOOOxx 2763 2475 1 3 3 3 63 763 763 2763 2763 126 127 HCAAAA FRDAAA VVVVxx 575 2476 1 3 5 15 75 575 575 575 575 150 151 DWAAAA GRDAAA AAAAxx 8953 2477 1 1 3 13 53 953 953 3953 8953 106 107 JGAAAA HRDAAA HHHHxx 5833 2478 1 1 3 13 33 833 1833 833 5833 66 67 JQAAAA IRDAAA OOOOxx 9035 2479 1 3 5 15 35 35 1035 4035 9035 70 71 NJAAAA JRDAAA VVVVxx 9123 2480 1 3 3 3 23 123 1123 4123 9123 46 47 XMAAAA KRDAAA AAAAxx 206 2481 0 2 6 6 6 206 206 206 206 12 13 YHAAAA LRDAAA HHHHxx 4155 2482 1 3 5 15 55 155 155 4155 4155 110 111 VDAAAA MRDAAA OOOOxx 532 2483 0 0 2 12 32 532 532 532 532 64 65 MUAAAA NRDAAA VVVVxx 1370 2484 0 2 0 10 70 370 1370 1370 1370 140 141 SAAAAA ORDAAA AAAAxx 7656 2485 0 0 6 16 56 656 1656 2656 7656 112 113 MIAAAA PRDAAA HHHHxx 7735 2486 1 3 5 15 35 735 1735 2735 7735 70 71 NLAAAA QRDAAA OOOOxx 2118 2487 0 2 8 18 18 118 118 2118 2118 36 37 MDAAAA RRDAAA VVVVxx 6914 2488 0 2 4 14 14 914 914 1914 6914 28 29 YFAAAA SRDAAA AAAAxx 6277 2489 1 1 7 17 77 277 277 1277 6277 154 155 LHAAAA TRDAAA HHHHxx 6347 2490 1 3 7 7 47 347 347 1347 6347 94 95 DKAAAA URDAAA OOOOxx 4030 2491 0 2 0 10 30 30 30 4030 4030 60 61 AZAAAA VRDAAA VVVVxx 9673 2492 1 1 3 13 73 673 1673 4673 9673 146 147 BIAAAA WRDAAA AAAAxx 2015 2493 1 3 5 15 15 15 15 2015 2015 30 31 NZAAAA XRDAAA HHHHxx 1317 2494 1 1 7 17 17 317 1317 1317 1317 34 35 RYAAAA YRDAAA OOOOxx 404 2495 0 0 4 4 4 404 404 404 404 8 9 OPAAAA ZRDAAA VVVVxx 1604 2496 0 0 4 4 4 604 1604 1604 1604 8 9 SJAAAA ASDAAA AAAAxx 1912 2497 0 0 2 12 12 912 1912 1912 1912 24 25 OVAAAA BSDAAA HHHHxx 5727 2498 1 3 7 7 27 727 1727 727 5727 54 55 HMAAAA CSDAAA OOOOxx 4538 2499 0 2 8 18 38 538 538 4538 4538 76 77 OSAAAA DSDAAA VVVVxx 6868 2500 0 0 8 8 68 868 868 1868 6868 136 137 EEAAAA ESDAAA AAAAxx 9801 2501 1 1 1 1 1 801 1801 4801 9801 2 3 ZMAAAA FSDAAA HHHHxx 1781 2502 1 1 1 1 81 781 1781 1781 1781 162 163 NQAAAA GSDAAA OOOOxx 7061 2503 1 1 1 1 61 61 1061 2061 7061 122 123 PLAAAA HSDAAA VVVVxx 2412 2504 0 0 2 12 12 412 412 2412 2412 24 25 UOAAAA ISDAAA AAAAxx 9191 2505 1 3 1 11 91 191 1191 4191 9191 182 183 NPAAAA JSDAAA HHHHxx 1958 2506 0 2 8 18 58 958 1958 1958 1958 116 117 IXAAAA KSDAAA OOOOxx 2203 2507 1 3 3 3 3 203 203 2203 2203 6 7 TGAAAA LSDAAA VVVVxx 9104 2508 0 0 4 4 4 104 1104 4104 9104 8 9 EMAAAA MSDAAA AAAAxx 3837 2509 1 1 7 17 37 837 1837 3837 3837 74 75 PRAAAA NSDAAA HHHHxx 7055 2510 1 3 5 15 55 55 1055 2055 7055 110 111 JLAAAA OSDAAA OOOOxx 4612 2511 0 0 2 12 12 612 612 4612 4612 24 25 KVAAAA PSDAAA VVVVxx 6420 2512 0 0 0 0 20 420 420 1420 6420 40 41 YMAAAA QSDAAA AAAAxx 613 2513 1 1 3 13 13 613 613 613 613 26 27 PXAAAA RSDAAA HHHHxx 1691 2514 1 3 1 11 91 691 1691 1691 1691 182 183 BNAAAA SSDAAA OOOOxx 33 2515 1 1 3 13 33 33 33 33 33 66 67 HBAAAA TSDAAA VVVVxx 875 2516 1 3 5 15 75 875 875 875 875 150 151 RHAAAA USDAAA AAAAxx 9030 2517 0 2 0 10 30 30 1030 4030 9030 60 61 IJAAAA VSDAAA HHHHxx 4285 2518 1 1 5 5 85 285 285 4285 4285 170 171 VIAAAA WSDAAA OOOOxx 6236 2519 0 0 6 16 36 236 236 1236 6236 72 73 WFAAAA XSDAAA VVVVxx 4702 2520 0 2 2 2 2 702 702 4702 4702 4 5 WYAAAA YSDAAA AAAAxx 3441 2521 1 1 1 1 41 441 1441 3441 3441 82 83 JCAAAA ZSDAAA HHHHxx 2150 2522 0 2 0 10 50 150 150 2150 2150 100 101 SEAAAA ATDAAA OOOOxx 1852 2523 0 0 2 12 52 852 1852 1852 1852 104 105 GTAAAA BTDAAA VVVVxx 7713 2524 1 1 3 13 13 713 1713 2713 7713 26 27 RKAAAA CTDAAA AAAAxx 6849 2525 1 1 9 9 49 849 849 1849 6849 98 99 LDAAAA DTDAAA HHHHxx 3425 2526 1 1 5 5 25 425 1425 3425 3425 50 51 TBAAAA ETDAAA OOOOxx 4681 2527 1 1 1 1 81 681 681 4681 4681 162 163 BYAAAA FTDAAA VVVVxx 1134 2528 0 2 4 14 34 134 1134 1134 1134 68 69 QRAAAA GTDAAA AAAAxx 7462 2529 0 2 2 2 62 462 1462 2462 7462 124 125 ABAAAA HTDAAA HHHHxx 2148 2530 0 0 8 8 48 148 148 2148 2148 96 97 QEAAAA ITDAAA OOOOxx 5921 2531 1 1 1 1 21 921 1921 921 5921 42 43 TTAAAA JTDAAA VVVVxx 118 2532 0 2 8 18 18 118 118 118 118 36 37 OEAAAA KTDAAA AAAAxx 3065 2533 1 1 5 5 65 65 1065 3065 3065 130 131 XNAAAA LTDAAA HHHHxx 6590 2534 0 2 0 10 90 590 590 1590 6590 180 181 MTAAAA MTDAAA OOOOxx 4993 2535 1 1 3 13 93 993 993 4993 4993 186 187 BKAAAA NTDAAA VVVVxx 6818 2536 0 2 8 18 18 818 818 1818 6818 36 37 GCAAAA OTDAAA AAAAxx 1449 2537 1 1 9 9 49 449 1449 1449 1449 98 99 TDAAAA PTDAAA HHHHxx 2039 2538 1 3 9 19 39 39 39 2039 2039 78 79 LAAAAA QTDAAA OOOOxx 2524 2539 0 0 4 4 24 524 524 2524 2524 48 49 CTAAAA RTDAAA VVVVxx 1481 2540 1 1 1 1 81 481 1481 1481 1481 162 163 ZEAAAA STDAAA AAAAxx 6984 2541 0 0 4 4 84 984 984 1984 6984 168 169 QIAAAA TTDAAA HHHHxx 3960 2542 0 0 0 0 60 960 1960 3960 3960 120 121 IWAAAA UTDAAA OOOOxx 1983 2543 1 3 3 3 83 983 1983 1983 1983 166 167 HYAAAA VTDAAA VVVVxx 6379 2544 1 3 9 19 79 379 379 1379 6379 158 159 JLAAAA WTDAAA AAAAxx 8975 2545 1 3 5 15 75 975 975 3975 8975 150 151 FHAAAA XTDAAA HHHHxx 1102 2546 0 2 2 2 2 102 1102 1102 1102 4 5 KQAAAA YTDAAA OOOOxx 2517 2547 1 1 7 17 17 517 517 2517 2517 34 35 VSAAAA ZTDAAA VVVVxx 712 2548 0 0 2 12 12 712 712 712 712 24 25 KBAAAA AUDAAA AAAAxx 5419 2549 1 3 9 19 19 419 1419 419 5419 38 39 LAAAAA BUDAAA HHHHxx 723 2550 1 3 3 3 23 723 723 723 723 46 47 VBAAAA CUDAAA OOOOxx 8057 2551 1 1 7 17 57 57 57 3057 8057 114 115 XXAAAA DUDAAA VVVVxx 7471 2552 1 3 1 11 71 471 1471 2471 7471 142 143 JBAAAA EUDAAA AAAAxx 8855 2553 1 3 5 15 55 855 855 3855 8855 110 111 PCAAAA FUDAAA HHHHxx 5074 2554 0 2 4 14 74 74 1074 74 5074 148 149 ENAAAA GUDAAA OOOOxx 7139 2555 1 3 9 19 39 139 1139 2139 7139 78 79 POAAAA HUDAAA VVVVxx 3833 2556 1 1 3 13 33 833 1833 3833 3833 66 67 LRAAAA IUDAAA AAAAxx 5186 2557 0 2 6 6 86 186 1186 186 5186 172 173 MRAAAA JUDAAA HHHHxx 9436 2558 0 0 6 16 36 436 1436 4436 9436 72 73 YYAAAA KUDAAA OOOOxx 8859 2559 1 3 9 19 59 859 859 3859 8859 118 119 TCAAAA LUDAAA VVVVxx 6943 2560 1 3 3 3 43 943 943 1943 6943 86 87 BHAAAA MUDAAA AAAAxx 2315 2561 1 3 5 15 15 315 315 2315 2315 30 31 BLAAAA NUDAAA HHHHxx 1394 2562 0 2 4 14 94 394 1394 1394 1394 188 189 QBAAAA OUDAAA OOOOxx 8863 2563 1 3 3 3 63 863 863 3863 8863 126 127 XCAAAA PUDAAA VVVVxx 8812 2564 0 0 2 12 12 812 812 3812 8812 24 25 YAAAAA QUDAAA AAAAxx 7498 2565 0 2 8 18 98 498 1498 2498 7498 196 197 KCAAAA RUDAAA HHHHxx 8962 2566 0 2 2 2 62 962 962 3962 8962 124 125 SGAAAA SUDAAA OOOOxx 2533 2567 1 1 3 13 33 533 533 2533 2533 66 67 LTAAAA TUDAAA VVVVxx 8188 2568 0 0 8 8 88 188 188 3188 8188 176 177 YCAAAA UUDAAA AAAAxx 6137 2569 1 1 7 17 37 137 137 1137 6137 74 75 BCAAAA VUDAAA HHHHxx 974 2570 0 2 4 14 74 974 974 974 974 148 149 MLAAAA WUDAAA OOOOxx 2751 2571 1 3 1 11 51 751 751 2751 2751 102 103 VBAAAA XUDAAA VVVVxx 4975 2572 1 3 5 15 75 975 975 4975 4975 150 151 JJAAAA YUDAAA AAAAxx 3411 2573 1 3 1 11 11 411 1411 3411 3411 22 23 FBAAAA ZUDAAA HHHHxx 3143 2574 1 3 3 3 43 143 1143 3143 3143 86 87 XQAAAA AVDAAA OOOOxx 8011 2575 1 3 1 11 11 11 11 3011 8011 22 23 DWAAAA BVDAAA VVVVxx 988 2576 0 0 8 8 88 988 988 988 988 176 177 AMAAAA CVDAAA AAAAxx 4289 2577 1 1 9 9 89 289 289 4289 4289 178 179 ZIAAAA DVDAAA HHHHxx 8105 2578 1 1 5 5 5 105 105 3105 8105 10 11 TZAAAA EVDAAA OOOOxx 9885 2579 1 1 5 5 85 885 1885 4885 9885 170 171 FQAAAA FVDAAA VVVVxx 1002 2580 0 2 2 2 2 2 1002 1002 1002 4 5 OMAAAA GVDAAA AAAAxx 5827 2581 1 3 7 7 27 827 1827 827 5827 54 55 DQAAAA HVDAAA HHHHxx 1228 2582 0 0 8 8 28 228 1228 1228 1228 56 57 GVAAAA IVDAAA OOOOxx 6352 2583 0 0 2 12 52 352 352 1352 6352 104 105 IKAAAA JVDAAA VVVVxx 8868 2584 0 0 8 8 68 868 868 3868 8868 136 137 CDAAAA KVDAAA AAAAxx 3643 2585 1 3 3 3 43 643 1643 3643 3643 86 87 DKAAAA LVDAAA HHHHxx 1468 2586 0 0 8 8 68 468 1468 1468 1468 136 137 MEAAAA MVDAAA OOOOxx 8415 2587 1 3 5 15 15 415 415 3415 8415 30 31 RLAAAA NVDAAA VVVVxx 9631 2588 1 3 1 11 31 631 1631 4631 9631 62 63 LGAAAA OVDAAA AAAAxx 7408 2589 0 0 8 8 8 408 1408 2408 7408 16 17 YYAAAA PVDAAA HHHHxx 1934 2590 0 2 4 14 34 934 1934 1934 1934 68 69 KWAAAA QVDAAA OOOOxx 996 2591 0 0 6 16 96 996 996 996 996 192 193 IMAAAA RVDAAA VVVVxx 8027 2592 1 3 7 7 27 27 27 3027 8027 54 55 TWAAAA SVDAAA AAAAxx 8464 2593 0 0 4 4 64 464 464 3464 8464 128 129 ONAAAA TVDAAA HHHHxx 5007 2594 1 3 7 7 7 7 1007 7 5007 14 15 PKAAAA UVDAAA OOOOxx 8356 2595 0 0 6 16 56 356 356 3356 8356 112 113 KJAAAA VVDAAA VVVVxx 4579 2596 1 3 9 19 79 579 579 4579 4579 158 159 DUAAAA WVDAAA AAAAxx 8513 2597 1 1 3 13 13 513 513 3513 8513 26 27 LPAAAA XVDAAA HHHHxx 383 2598 1 3 3 3 83 383 383 383 383 166 167 TOAAAA YVDAAA OOOOxx 9304 2599 0 0 4 4 4 304 1304 4304 9304 8 9 WTAAAA ZVDAAA VVVVxx 7224 2600 0 0 4 4 24 224 1224 2224 7224 48 49 WRAAAA AWDAAA AAAAxx 6023 2601 1 3 3 3 23 23 23 1023 6023 46 47 RXAAAA BWDAAA HHHHxx 2746 2602 0 2 6 6 46 746 746 2746 2746 92 93 QBAAAA CWDAAA OOOOxx 137 2603 1 1 7 17 37 137 137 137 137 74 75 HFAAAA DWDAAA VVVVxx 9441 2604 1 1 1 1 41 441 1441 4441 9441 82 83 DZAAAA EWDAAA AAAAxx 3690 2605 0 2 0 10 90 690 1690 3690 3690 180 181 YLAAAA FWDAAA HHHHxx 913 2606 1 1 3 13 13 913 913 913 913 26 27 DJAAAA GWDAAA OOOOxx 1768 2607 0 0 8 8 68 768 1768 1768 1768 136 137 AQAAAA HWDAAA VVVVxx 8492 2608 0 0 2 12 92 492 492 3492 8492 184 185 QOAAAA IWDAAA AAAAxx 8083 2609 1 3 3 3 83 83 83 3083 8083 166 167 XYAAAA JWDAAA HHHHxx 4609 2610 1 1 9 9 9 609 609 4609 4609 18 19 HVAAAA KWDAAA OOOOxx 7520 2611 0 0 0 0 20 520 1520 2520 7520 40 41 GDAAAA LWDAAA VVVVxx 4231 2612 1 3 1 11 31 231 231 4231 4231 62 63 TGAAAA MWDAAA AAAAxx 6022 2613 0 2 2 2 22 22 22 1022 6022 44 45 QXAAAA NWDAAA HHHHxx 9784 2614 0 0 4 4 84 784 1784 4784 9784 168 169 IMAAAA OWDAAA OOOOxx 1343 2615 1 3 3 3 43 343 1343 1343 1343 86 87 RZAAAA PWDAAA VVVVxx 7549 2616 1 1 9 9 49 549 1549 2549 7549 98 99 JEAAAA QWDAAA AAAAxx 269 2617 1 1 9 9 69 269 269 269 269 138 139 JKAAAA RWDAAA HHHHxx 1069 2618 1 1 9 9 69 69 1069 1069 1069 138 139 DPAAAA SWDAAA OOOOxx 4610 2619 0 2 0 10 10 610 610 4610 4610 20 21 IVAAAA TWDAAA VVVVxx 482 2620 0 2 2 2 82 482 482 482 482 164 165 OSAAAA UWDAAA AAAAxx 3025 2621 1 1 5 5 25 25 1025 3025 3025 50 51 JMAAAA VWDAAA HHHHxx 7914 2622 0 2 4 14 14 914 1914 2914 7914 28 29 KSAAAA WWDAAA OOOOxx 3198 2623 0 2 8 18 98 198 1198 3198 3198 196 197 ATAAAA XWDAAA VVVVxx 1187 2624 1 3 7 7 87 187 1187 1187 1187 174 175 RTAAAA YWDAAA AAAAxx 4707 2625 1 3 7 7 7 707 707 4707 4707 14 15 BZAAAA ZWDAAA HHHHxx 8279 2626 1 3 9 19 79 279 279 3279 8279 158 159 LGAAAA AXDAAA OOOOxx 6127 2627 1 3 7 7 27 127 127 1127 6127 54 55 RBAAAA BXDAAA VVVVxx 1305 2628 1 1 5 5 5 305 1305 1305 1305 10 11 FYAAAA CXDAAA AAAAxx 4804 2629 0 0 4 4 4 804 804 4804 4804 8 9 UCAAAA DXDAAA HHHHxx 6069 2630 1 1 9 9 69 69 69 1069 6069 138 139 LZAAAA EXDAAA OOOOxx 9229 2631 1 1 9 9 29 229 1229 4229 9229 58 59 ZQAAAA FXDAAA VVVVxx 4703 2632 1 3 3 3 3 703 703 4703 4703 6 7 XYAAAA GXDAAA AAAAxx 6410 2633 0 2 0 10 10 410 410 1410 6410 20 21 OMAAAA HXDAAA HHHHxx 944 2634 0 0 4 4 44 944 944 944 944 88 89 IKAAAA IXDAAA OOOOxx 3744 2635 0 0 4 4 44 744 1744 3744 3744 88 89 AOAAAA JXDAAA VVVVxx 1127 2636 1 3 7 7 27 127 1127 1127 1127 54 55 JRAAAA KXDAAA AAAAxx 6693 2637 1 1 3 13 93 693 693 1693 6693 186 187 LXAAAA LXDAAA HHHHxx 583 2638 1 3 3 3 83 583 583 583 583 166 167 LWAAAA MXDAAA OOOOxx 2684 2639 0 0 4 4 84 684 684 2684 2684 168 169 GZAAAA NXDAAA VVVVxx 6192 2640 0 0 2 12 92 192 192 1192 6192 184 185 EEAAAA OXDAAA AAAAxx 4157 2641 1 1 7 17 57 157 157 4157 4157 114 115 XDAAAA PXDAAA HHHHxx 6470 2642 0 2 0 10 70 470 470 1470 6470 140 141 WOAAAA QXDAAA OOOOxx 8965 2643 1 1 5 5 65 965 965 3965 8965 130 131 VGAAAA RXDAAA VVVVxx 1433 2644 1 1 3 13 33 433 1433 1433 1433 66 67 DDAAAA SXDAAA AAAAxx 4570 2645 0 2 0 10 70 570 570 4570 4570 140 141 UTAAAA TXDAAA HHHHxx 1806 2646 0 2 6 6 6 806 1806 1806 1806 12 13 MRAAAA UXDAAA OOOOxx 1230 2647 0 2 0 10 30 230 1230 1230 1230 60 61 IVAAAA VXDAAA VVVVxx 2283 2648 1 3 3 3 83 283 283 2283 2283 166 167 VJAAAA WXDAAA AAAAxx 6456 2649 0 0 6 16 56 456 456 1456 6456 112 113 IOAAAA XXDAAA HHHHxx 7427 2650 1 3 7 7 27 427 1427 2427 7427 54 55 RZAAAA YXDAAA OOOOxx 8310 2651 0 2 0 10 10 310 310 3310 8310 20 21 QHAAAA ZXDAAA VVVVxx 8103 2652 1 3 3 3 3 103 103 3103 8103 6 7 RZAAAA AYDAAA AAAAxx 3947 2653 1 3 7 7 47 947 1947 3947 3947 94 95 VVAAAA BYDAAA HHHHxx 3414 2654 0 2 4 14 14 414 1414 3414 3414 28 29 IBAAAA CYDAAA OOOOxx 2043 2655 1 3 3 3 43 43 43 2043 2043 86 87 PAAAAA DYDAAA VVVVxx 4393 2656 1 1 3 13 93 393 393 4393 4393 186 187 ZMAAAA EYDAAA AAAAxx 6664 2657 0 0 4 4 64 664 664 1664 6664 128 129 IWAAAA FYDAAA HHHHxx 4545 2658 1 1 5 5 45 545 545 4545 4545 90 91 VSAAAA GYDAAA OOOOxx 7637 2659 1 1 7 17 37 637 1637 2637 7637 74 75 THAAAA HYDAAA VVVVxx 1359 2660 1 3 9 19 59 359 1359 1359 1359 118 119 HAAAAA IYDAAA AAAAxx 5018 2661 0 2 8 18 18 18 1018 18 5018 36 37 ALAAAA JYDAAA HHHHxx 987 2662 1 3 7 7 87 987 987 987 987 174 175 ZLAAAA KYDAAA OOOOxx 1320 2663 0 0 0 0 20 320 1320 1320 1320 40 41 UYAAAA LYDAAA VVVVxx 9311 2664 1 3 1 11 11 311 1311 4311 9311 22 23 DUAAAA MYDAAA AAAAxx 7993 2665 1 1 3 13 93 993 1993 2993 7993 186 187 LVAAAA NYDAAA HHHHxx 7588 2666 0 0 8 8 88 588 1588 2588 7588 176 177 WFAAAA OYDAAA OOOOxx 5983 2667 1 3 3 3 83 983 1983 983 5983 166 167 DWAAAA PYDAAA VVVVxx 4070 2668 0 2 0 10 70 70 70 4070 4070 140 141 OAAAAA QYDAAA AAAAxx 8349 2669 1 1 9 9 49 349 349 3349 8349 98 99 DJAAAA RYDAAA HHHHxx 3810 2670 0 2 0 10 10 810 1810 3810 3810 20 21 OQAAAA SYDAAA OOOOxx 6948 2671 0 0 8 8 48 948 948 1948 6948 96 97 GHAAAA TYDAAA VVVVxx 7153 2672 1 1 3 13 53 153 1153 2153 7153 106 107 DPAAAA UYDAAA AAAAxx 5371 2673 1 3 1 11 71 371 1371 371 5371 142 143 PYAAAA VYDAAA HHHHxx 8316 2674 0 0 6 16 16 316 316 3316 8316 32 33 WHAAAA WYDAAA OOOOxx 5903 2675 1 3 3 3 3 903 1903 903 5903 6 7 BTAAAA XYDAAA VVVVxx 6718 2676 0 2 8 18 18 718 718 1718 6718 36 37 KYAAAA YYDAAA AAAAxx 4759 2677 1 3 9 19 59 759 759 4759 4759 118 119 BBAAAA ZYDAAA HHHHxx 2555 2678 1 3 5 15 55 555 555 2555 2555 110 111 HUAAAA AZDAAA OOOOxx 3457 2679 1 1 7 17 57 457 1457 3457 3457 114 115 ZCAAAA BZDAAA VVVVxx 9626 2680 0 2 6 6 26 626 1626 4626 9626 52 53 GGAAAA CZDAAA AAAAxx 2570 2681 0 2 0 10 70 570 570 2570 2570 140 141 WUAAAA DZDAAA HHHHxx 7964 2682 0 0 4 4 64 964 1964 2964 7964 128 129 IUAAAA EZDAAA OOOOxx 1543 2683 1 3 3 3 43 543 1543 1543 1543 86 87 JHAAAA FZDAAA VVVVxx 929 2684 1 1 9 9 29 929 929 929 929 58 59 TJAAAA GZDAAA AAAAxx 9244 2685 0 0 4 4 44 244 1244 4244 9244 88 89 ORAAAA HZDAAA HHHHxx 9210 2686 0 2 0 10 10 210 1210 4210 9210 20 21 GQAAAA IZDAAA OOOOxx 8334 2687 0 2 4 14 34 334 334 3334 8334 68 69 OIAAAA JZDAAA VVVVxx 9310 2688 0 2 0 10 10 310 1310 4310 9310 20 21 CUAAAA KZDAAA AAAAxx 5024 2689 0 0 4 4 24 24 1024 24 5024 48 49 GLAAAA LZDAAA HHHHxx 8794 2690 0 2 4 14 94 794 794 3794 8794 188 189 GAAAAA MZDAAA OOOOxx 4091 2691 1 3 1 11 91 91 91 4091 4091 182 183 JBAAAA NZDAAA VVVVxx 649 2692 1 1 9 9 49 649 649 649 649 98 99 ZYAAAA OZDAAA AAAAxx 8505 2693 1 1 5 5 5 505 505 3505 8505 10 11 DPAAAA PZDAAA HHHHxx 6652 2694 0 0 2 12 52 652 652 1652 6652 104 105 WVAAAA QZDAAA OOOOxx 8945 2695 1 1 5 5 45 945 945 3945 8945 90 91 BGAAAA RZDAAA VVVVxx 2095 2696 1 3 5 15 95 95 95 2095 2095 190 191 PCAAAA SZDAAA AAAAxx 8676 2697 0 0 6 16 76 676 676 3676 8676 152 153 SVAAAA TZDAAA HHHHxx 3994 2698 0 2 4 14 94 994 1994 3994 3994 188 189 QXAAAA UZDAAA OOOOxx 2859 2699 1 3 9 19 59 859 859 2859 2859 118 119 ZFAAAA VZDAAA VVVVxx 5403 2700 1 3 3 3 3 403 1403 403 5403 6 7 VZAAAA WZDAAA AAAAxx 3254 2701 0 2 4 14 54 254 1254 3254 3254 108 109 EVAAAA XZDAAA HHHHxx 7339 2702 1 3 9 19 39 339 1339 2339 7339 78 79 HWAAAA YZDAAA OOOOxx 7220 2703 0 0 0 0 20 220 1220 2220 7220 40 41 SRAAAA ZZDAAA VVVVxx 4154 2704 0 2 4 14 54 154 154 4154 4154 108 109 UDAAAA AAEAAA AAAAxx 7570 2705 0 2 0 10 70 570 1570 2570 7570 140 141 EFAAAA BAEAAA HHHHxx 2576 2706 0 0 6 16 76 576 576 2576 2576 152 153 CVAAAA CAEAAA OOOOxx 5764 2707 0 0 4 4 64 764 1764 764 5764 128 129 SNAAAA DAEAAA VVVVxx 4314 2708 0 2 4 14 14 314 314 4314 4314 28 29 YJAAAA EAEAAA AAAAxx 2274 2709 0 2 4 14 74 274 274 2274 2274 148 149 MJAAAA FAEAAA HHHHxx 9756 2710 0 0 6 16 56 756 1756 4756 9756 112 113 GLAAAA GAEAAA OOOOxx 8274 2711 0 2 4 14 74 274 274 3274 8274 148 149 GGAAAA HAEAAA VVVVxx 1289 2712 1 1 9 9 89 289 1289 1289 1289 178 179 PXAAAA IAEAAA AAAAxx 7335 2713 1 3 5 15 35 335 1335 2335 7335 70 71 DWAAAA JAEAAA HHHHxx 5351 2714 1 3 1 11 51 351 1351 351 5351 102 103 VXAAAA KAEAAA OOOOxx 8978 2715 0 2 8 18 78 978 978 3978 8978 156 157 IHAAAA LAEAAA VVVVxx 2 2716 0 2 2 2 2 2 2 2 2 4 5 CAAAAA MAEAAA AAAAxx 8906 2717 0 2 6 6 6 906 906 3906 8906 12 13 OEAAAA NAEAAA HHHHxx 6388 2718 0 0 8 8 88 388 388 1388 6388 176 177 SLAAAA OAEAAA OOOOxx 5675 2719 1 3 5 15 75 675 1675 675 5675 150 151 HKAAAA PAEAAA VVVVxx 255 2720 1 3 5 15 55 255 255 255 255 110 111 VJAAAA QAEAAA AAAAxx 9538 2721 0 2 8 18 38 538 1538 4538 9538 76 77 WCAAAA RAEAAA HHHHxx 1480 2722 0 0 0 0 80 480 1480 1480 1480 160 161 YEAAAA SAEAAA OOOOxx 4015 2723 1 3 5 15 15 15 15 4015 4015 30 31 LYAAAA TAEAAA VVVVxx 5166 2724 0 2 6 6 66 166 1166 166 5166 132 133 SQAAAA UAEAAA AAAAxx 91 2725 1 3 1 11 91 91 91 91 91 182 183 NDAAAA VAEAAA HHHHxx 2958 2726 0 2 8 18 58 958 958 2958 2958 116 117 UJAAAA WAEAAA OOOOxx 9131 2727 1 3 1 11 31 131 1131 4131 9131 62 63 FNAAAA XAEAAA VVVVxx 3944 2728 0 0 4 4 44 944 1944 3944 3944 88 89 SVAAAA YAEAAA AAAAxx 4514 2729 0 2 4 14 14 514 514 4514 4514 28 29 QRAAAA ZAEAAA HHHHxx 5661 2730 1 1 1 1 61 661 1661 661 5661 122 123 TJAAAA ABEAAA OOOOxx 8724 2731 0 0 4 4 24 724 724 3724 8724 48 49 OXAAAA BBEAAA VVVVxx 6408 2732 0 0 8 8 8 408 408 1408 6408 16 17 MMAAAA CBEAAA AAAAxx 5013 2733 1 1 3 13 13 13 1013 13 5013 26 27 VKAAAA DBEAAA HHHHxx 6156 2734 0 0 6 16 56 156 156 1156 6156 112 113 UCAAAA EBEAAA OOOOxx 7350 2735 0 2 0 10 50 350 1350 2350 7350 100 101 SWAAAA FBEAAA VVVVxx 9858 2736 0 2 8 18 58 858 1858 4858 9858 116 117 EPAAAA GBEAAA AAAAxx 895 2737 1 3 5 15 95 895 895 895 895 190 191 LIAAAA HBEAAA HHHHxx 8368 2738 0 0 8 8 68 368 368 3368 8368 136 137 WJAAAA IBEAAA OOOOxx 179 2739 1 3 9 19 79 179 179 179 179 158 159 XGAAAA JBEAAA VVVVxx 4048 2740 0 0 8 8 48 48 48 4048 4048 96 97 SZAAAA KBEAAA AAAAxx 3073 2741 1 1 3 13 73 73 1073 3073 3073 146 147 FOAAAA LBEAAA HHHHxx 321 2742 1 1 1 1 21 321 321 321 321 42 43 JMAAAA MBEAAA OOOOxx 5352 2743 0 0 2 12 52 352 1352 352 5352 104 105 WXAAAA NBEAAA VVVVxx 1940 2744 0 0 0 0 40 940 1940 1940 1940 80 81 QWAAAA OBEAAA AAAAxx 8803 2745 1 3 3 3 3 803 803 3803 8803 6 7 PAAAAA PBEAAA HHHHxx 791 2746 1 3 1 11 91 791 791 791 791 182 183 LEAAAA QBEAAA OOOOxx 9809 2747 1 1 9 9 9 809 1809 4809 9809 18 19 HNAAAA RBEAAA VVVVxx 5519 2748 1 3 9 19 19 519 1519 519 5519 38 39 HEAAAA SBEAAA AAAAxx 7420 2749 0 0 0 0 20 420 1420 2420 7420 40 41 KZAAAA TBEAAA HHHHxx 7541 2750 1 1 1 1 41 541 1541 2541 7541 82 83 BEAAAA UBEAAA OOOOxx 6538 2751 0 2 8 18 38 538 538 1538 6538 76 77 MRAAAA VBEAAA VVVVxx 710 2752 0 2 0 10 10 710 710 710 710 20 21 IBAAAA WBEAAA AAAAxx 9488 2753 0 0 8 8 88 488 1488 4488 9488 176 177 YAAAAA XBEAAA HHHHxx 3135 2754 1 3 5 15 35 135 1135 3135 3135 70 71 PQAAAA YBEAAA OOOOxx 4273 2755 1 1 3 13 73 273 273 4273 4273 146 147 JIAAAA ZBEAAA VVVVxx 629 2756 1 1 9 9 29 629 629 629 629 58 59 FYAAAA ACEAAA AAAAxx 9167 2757 1 3 7 7 67 167 1167 4167 9167 134 135 POAAAA BCEAAA HHHHxx 751 2758 1 3 1 11 51 751 751 751 751 102 103 XCAAAA CCEAAA OOOOxx 1126 2759 0 2 6 6 26 126 1126 1126 1126 52 53 IRAAAA DCEAAA VVVVxx 3724 2760 0 0 4 4 24 724 1724 3724 3724 48 49 GNAAAA ECEAAA AAAAxx 1789 2761 1 1 9 9 89 789 1789 1789 1789 178 179 VQAAAA FCEAAA HHHHxx 792 2762 0 0 2 12 92 792 792 792 792 184 185 MEAAAA GCEAAA OOOOxx 2771 2763 1 3 1 11 71 771 771 2771 2771 142 143 PCAAAA HCEAAA VVVVxx 4313 2764 1 1 3 13 13 313 313 4313 4313 26 27 XJAAAA ICEAAA AAAAxx 9312 2765 0 0 2 12 12 312 1312 4312 9312 24 25 EUAAAA JCEAAA HHHHxx 955 2766 1 3 5 15 55 955 955 955 955 110 111 TKAAAA KCEAAA OOOOxx 6382 2767 0 2 2 2 82 382 382 1382 6382 164 165 MLAAAA LCEAAA VVVVxx 7875 2768 1 3 5 15 75 875 1875 2875 7875 150 151 XQAAAA MCEAAA AAAAxx 7491 2769 1 3 1 11 91 491 1491 2491 7491 182 183 DCAAAA NCEAAA HHHHxx 8193 2770 1 1 3 13 93 193 193 3193 8193 186 187 DDAAAA OCEAAA OOOOxx 968 2771 0 0 8 8 68 968 968 968 968 136 137 GLAAAA PCEAAA VVVVxx 4951 2772 1 3 1 11 51 951 951 4951 4951 102 103 LIAAAA QCEAAA AAAAxx 2204 2773 0 0 4 4 4 204 204 2204 2204 8 9 UGAAAA RCEAAA HHHHxx 2066 2774 0 2 6 6 66 66 66 2066 2066 132 133 MBAAAA SCEAAA OOOOxx 2631 2775 1 3 1 11 31 631 631 2631 2631 62 63 FXAAAA TCEAAA VVVVxx 8947 2776 1 3 7 7 47 947 947 3947 8947 94 95 DGAAAA UCEAAA AAAAxx 8033 2777 1 1 3 13 33 33 33 3033 8033 66 67 ZWAAAA VCEAAA HHHHxx 6264 2778 0 0 4 4 64 264 264 1264 6264 128 129 YGAAAA WCEAAA OOOOxx 7778 2779 0 2 8 18 78 778 1778 2778 7778 156 157 ENAAAA XCEAAA VVVVxx 9701 2780 1 1 1 1 1 701 1701 4701 9701 2 3 DJAAAA YCEAAA AAAAxx 5091 2781 1 3 1 11 91 91 1091 91 5091 182 183 VNAAAA ZCEAAA HHHHxx 7577 2782 1 1 7 17 77 577 1577 2577 7577 154 155 LFAAAA ADEAAA OOOOxx 3345 2783 1 1 5 5 45 345 1345 3345 3345 90 91 RYAAAA BDEAAA VVVVxx 7329 2784 1 1 9 9 29 329 1329 2329 7329 58 59 XVAAAA CDEAAA AAAAxx 7551 2785 1 3 1 11 51 551 1551 2551 7551 102 103 LEAAAA DDEAAA HHHHxx 6207 2786 1 3 7 7 7 207 207 1207 6207 14 15 TEAAAA EDEAAA OOOOxx 8664 2787 0 0 4 4 64 664 664 3664 8664 128 129 GVAAAA FDEAAA VVVVxx 8394 2788 0 2 4 14 94 394 394 3394 8394 188 189 WKAAAA GDEAAA AAAAxx 7324 2789 0 0 4 4 24 324 1324 2324 7324 48 49 SVAAAA HDEAAA HHHHxx 2713 2790 1 1 3 13 13 713 713 2713 2713 26 27 JAAAAA IDEAAA OOOOxx 2230 2791 0 2 0 10 30 230 230 2230 2230 60 61 UHAAAA JDEAAA VVVVxx 9211 2792 1 3 1 11 11 211 1211 4211 9211 22 23 HQAAAA KDEAAA AAAAxx 1296 2793 0 0 6 16 96 296 1296 1296 1296 192 193 WXAAAA LDEAAA HHHHxx 8104 2794 0 0 4 4 4 104 104 3104 8104 8 9 SZAAAA MDEAAA OOOOxx 6916 2795 0 0 6 16 16 916 916 1916 6916 32 33 AGAAAA NDEAAA VVVVxx 2208 2796 0 0 8 8 8 208 208 2208 2208 16 17 YGAAAA ODEAAA AAAAxx 3935 2797 1 3 5 15 35 935 1935 3935 3935 70 71 JVAAAA PDEAAA HHHHxx 7814 2798 0 2 4 14 14 814 1814 2814 7814 28 29 OOAAAA QDEAAA OOOOxx 6508 2799 0 0 8 8 8 508 508 1508 6508 16 17 IQAAAA RDEAAA VVVVxx 1703 2800 1 3 3 3 3 703 1703 1703 1703 6 7 NNAAAA SDEAAA AAAAxx 5640 2801 0 0 0 0 40 640 1640 640 5640 80 81 YIAAAA TDEAAA HHHHxx 6417 2802 1 1 7 17 17 417 417 1417 6417 34 35 VMAAAA UDEAAA OOOOxx 1713 2803 1 1 3 13 13 713 1713 1713 1713 26 27 XNAAAA VDEAAA VVVVxx 5309 2804 1 1 9 9 9 309 1309 309 5309 18 19 FWAAAA WDEAAA AAAAxx 4364 2805 0 0 4 4 64 364 364 4364 4364 128 129 WLAAAA XDEAAA HHHHxx 619 2806 1 3 9 19 19 619 619 619 619 38 39 VXAAAA YDEAAA OOOOxx 9498 2807 0 2 8 18 98 498 1498 4498 9498 196 197 IBAAAA ZDEAAA VVVVxx 2804 2808 0 0 4 4 4 804 804 2804 2804 8 9 WDAAAA AEEAAA AAAAxx 2220 2809 0 0 0 0 20 220 220 2220 2220 40 41 KHAAAA BEEAAA HHHHxx 9542 2810 0 2 2 2 42 542 1542 4542 9542 84 85 ADAAAA CEEAAA OOOOxx 3349 2811 1 1 9 9 49 349 1349 3349 3349 98 99 VYAAAA DEEAAA VVVVxx 9198 2812 0 2 8 18 98 198 1198 4198 9198 196 197 UPAAAA EEEAAA AAAAxx 2727 2813 1 3 7 7 27 727 727 2727 2727 54 55 XAAAAA FEEAAA HHHHxx 3768 2814 0 0 8 8 68 768 1768 3768 3768 136 137 YOAAAA GEEAAA OOOOxx 2334 2815 0 2 4 14 34 334 334 2334 2334 68 69 ULAAAA HEEAAA VVVVxx 7770 2816 0 2 0 10 70 770 1770 2770 7770 140 141 WMAAAA IEEAAA AAAAxx 5963 2817 1 3 3 3 63 963 1963 963 5963 126 127 JVAAAA JEEAAA HHHHxx 4732 2818 0 0 2 12 32 732 732 4732 4732 64 65 AAAAAA KEEAAA OOOOxx 2448 2819 0 0 8 8 48 448 448 2448 2448 96 97 EQAAAA LEEAAA VVVVxx 5998 2820 0 2 8 18 98 998 1998 998 5998 196 197 SWAAAA MEEAAA AAAAxx 8577 2821 1 1 7 17 77 577 577 3577 8577 154 155 XRAAAA NEEAAA HHHHxx 266 2822 0 2 6 6 66 266 266 266 266 132 133 GKAAAA OEEAAA OOOOxx 2169 2823 1 1 9 9 69 169 169 2169 2169 138 139 LFAAAA PEEAAA VVVVxx 8228 2824 0 0 8 8 28 228 228 3228 8228 56 57 MEAAAA QEEAAA AAAAxx 4813 2825 1 1 3 13 13 813 813 4813 4813 26 27 DDAAAA REEAAA HHHHxx 2769 2826 1 1 9 9 69 769 769 2769 2769 138 139 NCAAAA SEEAAA OOOOxx 8382 2827 0 2 2 2 82 382 382 3382 8382 164 165 KKAAAA TEEAAA VVVVxx 1717 2828 1 1 7 17 17 717 1717 1717 1717 34 35 BOAAAA UEEAAA AAAAxx 7178 2829 0 2 8 18 78 178 1178 2178 7178 156 157 CQAAAA VEEAAA HHHHxx 9547 2830 1 3 7 7 47 547 1547 4547 9547 94 95 FDAAAA WEEAAA OOOOxx 8187 2831 1 3 7 7 87 187 187 3187 8187 174 175 XCAAAA XEEAAA VVVVxx 3168 2832 0 0 8 8 68 168 1168 3168 3168 136 137 WRAAAA YEEAAA AAAAxx 2180 2833 0 0 0 0 80 180 180 2180 2180 160 161 WFAAAA ZEEAAA HHHHxx 859 2834 1 3 9 19 59 859 859 859 859 118 119 BHAAAA AFEAAA OOOOxx 1554 2835 0 2 4 14 54 554 1554 1554 1554 108 109 UHAAAA BFEAAA VVVVxx 3567 2836 1 3 7 7 67 567 1567 3567 3567 134 135 FHAAAA CFEAAA AAAAxx 5985 2837 1 1 5 5 85 985 1985 985 5985 170 171 FWAAAA DFEAAA HHHHxx 1 2838 1 1 1 1 1 1 1 1 1 2 3 BAAAAA EFEAAA OOOOxx 5937 2839 1 1 7 17 37 937 1937 937 5937 74 75 JUAAAA FFEAAA VVVVxx 7594 2840 0 2 4 14 94 594 1594 2594 7594 188 189 CGAAAA GFEAAA AAAAxx 3783 2841 1 3 3 3 83 783 1783 3783 3783 166 167 NPAAAA HFEAAA HHHHxx 6841 2842 1 1 1 1 41 841 841 1841 6841 82 83 DDAAAA IFEAAA OOOOxx 9694 2843 0 2 4 14 94 694 1694 4694 9694 188 189 WIAAAA JFEAAA VVVVxx 4322 2844 0 2 2 2 22 322 322 4322 4322 44 45 GKAAAA KFEAAA AAAAxx 6012 2845 0 0 2 12 12 12 12 1012 6012 24 25 GXAAAA LFEAAA HHHHxx 108 2846 0 0 8 8 8 108 108 108 108 16 17 EEAAAA MFEAAA OOOOxx 3396 2847 0 0 6 16 96 396 1396 3396 3396 192 193 QAAAAA NFEAAA VVVVxx 8643 2848 1 3 3 3 43 643 643 3643 8643 86 87 LUAAAA OFEAAA AAAAxx 6087 2849 1 3 7 7 87 87 87 1087 6087 174 175 DAAAAA PFEAAA HHHHxx 2629 2850 1 1 9 9 29 629 629 2629 2629 58 59 DXAAAA QFEAAA OOOOxx 3009 2851 1 1 9 9 9 9 1009 3009 3009 18 19 TLAAAA RFEAAA VVVVxx 438 2852 0 2 8 18 38 438 438 438 438 76 77 WQAAAA SFEAAA AAAAxx 2480 2853 0 0 0 0 80 480 480 2480 2480 160 161 KRAAAA TFEAAA HHHHxx 936 2854 0 0 6 16 36 936 936 936 936 72 73 AKAAAA UFEAAA OOOOxx 6 2855 0 2 6 6 6 6 6 6 6 12 13 GAAAAA VFEAAA VVVVxx 768 2856 0 0 8 8 68 768 768 768 768 136 137 ODAAAA WFEAAA AAAAxx 1564 2857 0 0 4 4 64 564 1564 1564 1564 128 129 EIAAAA XFEAAA HHHHxx 3236 2858 0 0 6 16 36 236 1236 3236 3236 72 73 MUAAAA YFEAAA OOOOxx 3932 2859 0 0 2 12 32 932 1932 3932 3932 64 65 GVAAAA ZFEAAA VVVVxx 8914 2860 0 2 4 14 14 914 914 3914 8914 28 29 WEAAAA AGEAAA AAAAxx 119 2861 1 3 9 19 19 119 119 119 119 38 39 PEAAAA BGEAAA HHHHxx 6034 2862 0 2 4 14 34 34 34 1034 6034 68 69 CYAAAA CGEAAA OOOOxx 5384 2863 0 0 4 4 84 384 1384 384 5384 168 169 CZAAAA DGEAAA VVVVxx 6885 2864 1 1 5 5 85 885 885 1885 6885 170 171 VEAAAA EGEAAA AAAAxx 232 2865 0 0 2 12 32 232 232 232 232 64 65 YIAAAA FGEAAA HHHHxx 1293 2866 1 1 3 13 93 293 1293 1293 1293 186 187 TXAAAA GGEAAA OOOOxx 9204 2867 0 0 4 4 4 204 1204 4204 9204 8 9 AQAAAA HGEAAA VVVVxx 527 2868 1 3 7 7 27 527 527 527 527 54 55 HUAAAA IGEAAA AAAAxx 6539 2869 1 3 9 19 39 539 539 1539 6539 78 79 NRAAAA JGEAAA HHHHxx 3679 2870 1 3 9 19 79 679 1679 3679 3679 158 159 NLAAAA KGEAAA OOOOxx 8282 2871 0 2 2 2 82 282 282 3282 8282 164 165 OGAAAA LGEAAA VVVVxx 5027 2872 1 3 7 7 27 27 1027 27 5027 54 55 JLAAAA MGEAAA AAAAxx 7694 2873 0 2 4 14 94 694 1694 2694 7694 188 189 YJAAAA NGEAAA HHHHxx 473 2874 1 1 3 13 73 473 473 473 473 146 147 FSAAAA OGEAAA OOOOxx 6325 2875 1 1 5 5 25 325 325 1325 6325 50 51 HJAAAA PGEAAA VVVVxx 8761 2876 1 1 1 1 61 761 761 3761 8761 122 123 ZYAAAA QGEAAA AAAAxx 6184 2877 0 0 4 4 84 184 184 1184 6184 168 169 WDAAAA RGEAAA HHHHxx 419 2878 1 3 9 19 19 419 419 419 419 38 39 DQAAAA SGEAAA OOOOxx 6111 2879 1 3 1 11 11 111 111 1111 6111 22 23 BBAAAA TGEAAA VVVVxx 3836 2880 0 0 6 16 36 836 1836 3836 3836 72 73 ORAAAA UGEAAA AAAAxx 4086 2881 0 2 6 6 86 86 86 4086 4086 172 173 EBAAAA VGEAAA HHHHxx 5818 2882 0 2 8 18 18 818 1818 818 5818 36 37 UPAAAA WGEAAA OOOOxx 4528 2883 0 0 8 8 28 528 528 4528 4528 56 57 ESAAAA XGEAAA VVVVxx 7199 2884 1 3 9 19 99 199 1199 2199 7199 198 199 XQAAAA YGEAAA AAAAxx 1847 2885 1 3 7 7 47 847 1847 1847 1847 94 95 BTAAAA ZGEAAA HHHHxx 2875 2886 1 3 5 15 75 875 875 2875 2875 150 151 PGAAAA AHEAAA OOOOxx 2872 2887 0 0 2 12 72 872 872 2872 2872 144 145 MGAAAA BHEAAA VVVVxx 3972 2888 0 0 2 12 72 972 1972 3972 3972 144 145 UWAAAA CHEAAA AAAAxx 7590 2889 0 2 0 10 90 590 1590 2590 7590 180 181 YFAAAA DHEAAA HHHHxx 1914 2890 0 2 4 14 14 914 1914 1914 1914 28 29 QVAAAA EHEAAA OOOOxx 1658 2891 0 2 8 18 58 658 1658 1658 1658 116 117 ULAAAA FHEAAA VVVVxx 2126 2892 0 2 6 6 26 126 126 2126 2126 52 53 UDAAAA GHEAAA AAAAxx 645 2893 1 1 5 5 45 645 645 645 645 90 91 VYAAAA HHEAAA HHHHxx 6636 2894 0 0 6 16 36 636 636 1636 6636 72 73 GVAAAA IHEAAA OOOOxx 1469 2895 1 1 9 9 69 469 1469 1469 1469 138 139 NEAAAA JHEAAA VVVVxx 1377 2896 1 1 7 17 77 377 1377 1377 1377 154 155 ZAAAAA KHEAAA AAAAxx 8425 2897 1 1 5 5 25 425 425 3425 8425 50 51 BMAAAA LHEAAA HHHHxx 9300 2898 0 0 0 0 0 300 1300 4300 9300 0 1 STAAAA MHEAAA OOOOxx 5355 2899 1 3 5 15 55 355 1355 355 5355 110 111 ZXAAAA NHEAAA VVVVxx 840 2900 0 0 0 0 40 840 840 840 840 80 81 IGAAAA OHEAAA AAAAxx 5185 2901 1 1 5 5 85 185 1185 185 5185 170 171 LRAAAA PHEAAA HHHHxx 6467 2902 1 3 7 7 67 467 467 1467 6467 134 135 TOAAAA QHEAAA OOOOxx 58 2903 0 2 8 18 58 58 58 58 58 116 117 GCAAAA RHEAAA VVVVxx 5051 2904 1 3 1 11 51 51 1051 51 5051 102 103 HMAAAA SHEAAA AAAAxx 8901 2905 1 1 1 1 1 901 901 3901 8901 2 3 JEAAAA THEAAA HHHHxx 1550 2906 0 2 0 10 50 550 1550 1550 1550 100 101 QHAAAA UHEAAA OOOOxx 1698 2907 0 2 8 18 98 698 1698 1698 1698 196 197 INAAAA VHEAAA VVVVxx 802 2908 0 2 2 2 2 802 802 802 802 4 5 WEAAAA WHEAAA AAAAxx 2440 2909 0 0 0 0 40 440 440 2440 2440 80 81 WPAAAA XHEAAA HHHHxx 2260 2910 0 0 0 0 60 260 260 2260 2260 120 121 YIAAAA YHEAAA OOOOxx 8218 2911 0 2 8 18 18 218 218 3218 8218 36 37 CEAAAA ZHEAAA VVVVxx 5144 2912 0 0 4 4 44 144 1144 144 5144 88 89 WPAAAA AIEAAA AAAAxx 4822 2913 0 2 2 2 22 822 822 4822 4822 44 45 MDAAAA BIEAAA HHHHxx 9476 2914 0 0 6 16 76 476 1476 4476 9476 152 153 MAAAAA CIEAAA OOOOxx 7535 2915 1 3 5 15 35 535 1535 2535 7535 70 71 VDAAAA DIEAAA VVVVxx 8738 2916 0 2 8 18 38 738 738 3738 8738 76 77 CYAAAA EIEAAA AAAAxx 7946 2917 0 2 6 6 46 946 1946 2946 7946 92 93 QTAAAA FIEAAA HHHHxx 8143 2918 1 3 3 3 43 143 143 3143 8143 86 87 FBAAAA GIEAAA OOOOxx 2623 2919 1 3 3 3 23 623 623 2623 2623 46 47 XWAAAA HIEAAA VVVVxx 5209 2920 1 1 9 9 9 209 1209 209 5209 18 19 JSAAAA IIEAAA AAAAxx 7674 2921 0 2 4 14 74 674 1674 2674 7674 148 149 EJAAAA JIEAAA HHHHxx 1135 2922 1 3 5 15 35 135 1135 1135 1135 70 71 RRAAAA KIEAAA OOOOxx 424 2923 0 0 4 4 24 424 424 424 424 48 49 IQAAAA LIEAAA VVVVxx 942 2924 0 2 2 2 42 942 942 942 942 84 85 GKAAAA MIEAAA AAAAxx 7813 2925 1 1 3 13 13 813 1813 2813 7813 26 27 NOAAAA NIEAAA HHHHxx 3539 2926 1 3 9 19 39 539 1539 3539 3539 78 79 DGAAAA OIEAAA OOOOxx 2909 2927 1 1 9 9 9 909 909 2909 2909 18 19 XHAAAA PIEAAA VVVVxx 3748 2928 0 0 8 8 48 748 1748 3748 3748 96 97 EOAAAA QIEAAA AAAAxx 2996 2929 0 0 6 16 96 996 996 2996 2996 192 193 GLAAAA RIEAAA HHHHxx 1869 2930 1 1 9 9 69 869 1869 1869 1869 138 139 XTAAAA SIEAAA OOOOxx 8151 2931 1 3 1 11 51 151 151 3151 8151 102 103 NBAAAA TIEAAA VVVVxx 6361 2932 1 1 1 1 61 361 361 1361 6361 122 123 RKAAAA UIEAAA AAAAxx 5568 2933 0 0 8 8 68 568 1568 568 5568 136 137 EGAAAA VIEAAA HHHHxx 2796 2934 0 0 6 16 96 796 796 2796 2796 192 193 ODAAAA WIEAAA OOOOxx 8489 2935 1 1 9 9 89 489 489 3489 8489 178 179 NOAAAA XIEAAA VVVVxx 9183 2936 1 3 3 3 83 183 1183 4183 9183 166 167 FPAAAA YIEAAA AAAAxx 8227 2937 1 3 7 7 27 227 227 3227 8227 54 55 LEAAAA ZIEAAA HHHHxx 1844 2938 0 0 4 4 44 844 1844 1844 1844 88 89 YSAAAA AJEAAA OOOOxx 3975 2939 1 3 5 15 75 975 1975 3975 3975 150 151 XWAAAA BJEAAA VVVVxx 6490 2940 0 2 0 10 90 490 490 1490 6490 180 181 QPAAAA CJEAAA AAAAxx 8303 2941 1 3 3 3 3 303 303 3303 8303 6 7 JHAAAA DJEAAA HHHHxx 7334 2942 0 2 4 14 34 334 1334 2334 7334 68 69 CWAAAA EJEAAA OOOOxx 2382 2943 0 2 2 2 82 382 382 2382 2382 164 165 QNAAAA FJEAAA VVVVxx 177 2944 1 1 7 17 77 177 177 177 177 154 155 VGAAAA GJEAAA AAAAxx 8117 2945 1 1 7 17 17 117 117 3117 8117 34 35 FAAAAA HJEAAA HHHHxx 5485 2946 1 1 5 5 85 485 1485 485 5485 170 171 ZCAAAA IJEAAA OOOOxx 6544 2947 0 0 4 4 44 544 544 1544 6544 88 89 SRAAAA JJEAAA VVVVxx 8517 2948 1 1 7 17 17 517 517 3517 8517 34 35 PPAAAA KJEAAA AAAAxx 2252 2949 0 0 2 12 52 252 252 2252 2252 104 105 QIAAAA LJEAAA HHHHxx 4480 2950 0 0 0 0 80 480 480 4480 4480 160 161 IQAAAA MJEAAA OOOOxx 4785 2951 1 1 5 5 85 785 785 4785 4785 170 171 BCAAAA NJEAAA VVVVxx 9700 2952 0 0 0 0 0 700 1700 4700 9700 0 1 CJAAAA OJEAAA AAAAxx 2122 2953 0 2 2 2 22 122 122 2122 2122 44 45 QDAAAA PJEAAA HHHHxx 8783 2954 1 3 3 3 83 783 783 3783 8783 166 167 VZAAAA QJEAAA OOOOxx 1453 2955 1 1 3 13 53 453 1453 1453 1453 106 107 XDAAAA RJEAAA VVVVxx 3908 2956 0 0 8 8 8 908 1908 3908 3908 16 17 IUAAAA SJEAAA AAAAxx 7707 2957 1 3 7 7 7 707 1707 2707 7707 14 15 LKAAAA TJEAAA HHHHxx 9049 2958 1 1 9 9 49 49 1049 4049 9049 98 99 BKAAAA UJEAAA OOOOxx 654 2959 0 2 4 14 54 654 654 654 654 108 109 EZAAAA VJEAAA VVVVxx 3336 2960 0 0 6 16 36 336 1336 3336 3336 72 73 IYAAAA WJEAAA AAAAxx 622 2961 0 2 2 2 22 622 622 622 622 44 45 YXAAAA XJEAAA HHHHxx 8398 2962 0 2 8 18 98 398 398 3398 8398 196 197 ALAAAA YJEAAA OOOOxx 9193 2963 1 1 3 13 93 193 1193 4193 9193 186 187 PPAAAA ZJEAAA VVVVxx 7896 2964 0 0 6 16 96 896 1896 2896 7896 192 193 SRAAAA AKEAAA AAAAxx 9798 2965 0 2 8 18 98 798 1798 4798 9798 196 197 WMAAAA BKEAAA HHHHxx 2881 2966 1 1 1 1 81 881 881 2881 2881 162 163 VGAAAA CKEAAA OOOOxx 672 2967 0 0 2 12 72 672 672 672 672 144 145 WZAAAA DKEAAA VVVVxx 6743 2968 1 3 3 3 43 743 743 1743 6743 86 87 JZAAAA EKEAAA AAAAxx 8935 2969 1 3 5 15 35 935 935 3935 8935 70 71 RFAAAA FKEAAA HHHHxx 2426 2970 0 2 6 6 26 426 426 2426 2426 52 53 IPAAAA GKEAAA OOOOxx 722 2971 0 2 2 2 22 722 722 722 722 44 45 UBAAAA HKEAAA VVVVxx 5088 2972 0 0 8 8 88 88 1088 88 5088 176 177 SNAAAA IKEAAA AAAAxx 8677 2973 1 1 7 17 77 677 677 3677 8677 154 155 TVAAAA JKEAAA HHHHxx 6963 2974 1 3 3 3 63 963 963 1963 6963 126 127 VHAAAA KKEAAA OOOOxx 1653 2975 1 1 3 13 53 653 1653 1653 1653 106 107 PLAAAA LKEAAA VVVVxx 7295 2976 1 3 5 15 95 295 1295 2295 7295 190 191 PUAAAA MKEAAA AAAAxx 6675 2977 1 3 5 15 75 675 675 1675 6675 150 151 TWAAAA NKEAAA HHHHxx 7183 2978 1 3 3 3 83 183 1183 2183 7183 166 167 HQAAAA OKEAAA OOOOxx 4378 2979 0 2 8 18 78 378 378 4378 4378 156 157 KMAAAA PKEAAA VVVVxx 2157 2980 1 1 7 17 57 157 157 2157 2157 114 115 ZEAAAA QKEAAA AAAAxx 2621 2981 1 1 1 1 21 621 621 2621 2621 42 43 VWAAAA RKEAAA HHHHxx 9278 2982 0 2 8 18 78 278 1278 4278 9278 156 157 WSAAAA SKEAAA OOOOxx 79 2983 1 3 9 19 79 79 79 79 79 158 159 BDAAAA TKEAAA VVVVxx 7358 2984 0 2 8 18 58 358 1358 2358 7358 116 117 AXAAAA UKEAAA AAAAxx 3589 2985 1 1 9 9 89 589 1589 3589 3589 178 179 BIAAAA VKEAAA HHHHxx 1254 2986 0 2 4 14 54 254 1254 1254 1254 108 109 GWAAAA WKEAAA OOOOxx 3490 2987 0 2 0 10 90 490 1490 3490 3490 180 181 GEAAAA XKEAAA VVVVxx 7533 2988 1 1 3 13 33 533 1533 2533 7533 66 67 TDAAAA YKEAAA AAAAxx 2800 2989 0 0 0 0 0 800 800 2800 2800 0 1 SDAAAA ZKEAAA HHHHxx 351 2990 1 3 1 11 51 351 351 351 351 102 103 NNAAAA ALEAAA OOOOxx 4359 2991 1 3 9 19 59 359 359 4359 4359 118 119 RLAAAA BLEAAA VVVVxx 5788 2992 0 0 8 8 88 788 1788 788 5788 176 177 QOAAAA CLEAAA AAAAxx 5521 2993 1 1 1 1 21 521 1521 521 5521 42 43 JEAAAA DLEAAA HHHHxx 3351 2994 1 3 1 11 51 351 1351 3351 3351 102 103 XYAAAA ELEAAA OOOOxx 5129 2995 1 1 9 9 29 129 1129 129 5129 58 59 HPAAAA FLEAAA VVVVxx 315 2996 1 3 5 15 15 315 315 315 315 30 31 DMAAAA GLEAAA AAAAxx 7552 2997 0 0 2 12 52 552 1552 2552 7552 104 105 MEAAAA HLEAAA HHHHxx 9176 2998 0 0 6 16 76 176 1176 4176 9176 152 153 YOAAAA ILEAAA OOOOxx 7458 2999 0 2 8 18 58 458 1458 2458 7458 116 117 WAAAAA JLEAAA VVVVxx 279 3000 1 3 9 19 79 279 279 279 279 158 159 TKAAAA KLEAAA AAAAxx 738 3001 0 2 8 18 38 738 738 738 738 76 77 KCAAAA LLEAAA HHHHxx 2557 3002 1 1 7 17 57 557 557 2557 2557 114 115 JUAAAA MLEAAA OOOOxx 9395 3003 1 3 5 15 95 395 1395 4395 9395 190 191 JXAAAA NLEAAA VVVVxx 7214 3004 0 2 4 14 14 214 1214 2214 7214 28 29 MRAAAA OLEAAA AAAAxx 6354 3005 0 2 4 14 54 354 354 1354 6354 108 109 KKAAAA PLEAAA HHHHxx 4799 3006 1 3 9 19 99 799 799 4799 4799 198 199 PCAAAA QLEAAA OOOOxx 1231 3007 1 3 1 11 31 231 1231 1231 1231 62 63 JVAAAA RLEAAA VVVVxx 5252 3008 0 0 2 12 52 252 1252 252 5252 104 105 AUAAAA SLEAAA AAAAxx 5250 3009 0 2 0 10 50 250 1250 250 5250 100 101 YTAAAA TLEAAA HHHHxx 9319 3010 1 3 9 19 19 319 1319 4319 9319 38 39 LUAAAA ULEAAA OOOOxx 1724 3011 0 0 4 4 24 724 1724 1724 1724 48 49 IOAAAA VLEAAA VVVVxx 7947 3012 1 3 7 7 47 947 1947 2947 7947 94 95 RTAAAA WLEAAA AAAAxx 1105 3013 1 1 5 5 5 105 1105 1105 1105 10 11 NQAAAA XLEAAA HHHHxx 1417 3014 1 1 7 17 17 417 1417 1417 1417 34 35 NCAAAA YLEAAA OOOOxx 7101 3015 1 1 1 1 1 101 1101 2101 7101 2 3 DNAAAA ZLEAAA VVVVxx 1088 3016 0 0 8 8 88 88 1088 1088 1088 176 177 WPAAAA AMEAAA AAAAxx 979 3017 1 3 9 19 79 979 979 979 979 158 159 RLAAAA BMEAAA HHHHxx 7589 3018 1 1 9 9 89 589 1589 2589 7589 178 179 XFAAAA CMEAAA OOOOxx 8952 3019 0 0 2 12 52 952 952 3952 8952 104 105 IGAAAA DMEAAA VVVVxx 2864 3020 0 0 4 4 64 864 864 2864 2864 128 129 EGAAAA EMEAAA AAAAxx 234 3021 0 2 4 14 34 234 234 234 234 68 69 AJAAAA FMEAAA HHHHxx 7231 3022 1 3 1 11 31 231 1231 2231 7231 62 63 DSAAAA GMEAAA OOOOxx 6792 3023 0 0 2 12 92 792 792 1792 6792 184 185 GBAAAA HMEAAA VVVVxx 4311 3024 1 3 1 11 11 311 311 4311 4311 22 23 VJAAAA IMEAAA AAAAxx 3374 3025 0 2 4 14 74 374 1374 3374 3374 148 149 UZAAAA JMEAAA HHHHxx 3367 3026 1 3 7 7 67 367 1367 3367 3367 134 135 NZAAAA KMEAAA OOOOxx 2598 3027 0 2 8 18 98 598 598 2598 2598 196 197 YVAAAA LMEAAA VVVVxx 1033 3028 1 1 3 13 33 33 1033 1033 1033 66 67 TNAAAA MMEAAA AAAAxx 7803 3029 1 3 3 3 3 803 1803 2803 7803 6 7 DOAAAA NMEAAA HHHHxx 3870 3030 0 2 0 10 70 870 1870 3870 3870 140 141 WSAAAA OMEAAA OOOOxx 4962 3031 0 2 2 2 62 962 962 4962 4962 124 125 WIAAAA PMEAAA VVVVxx 4842 3032 0 2 2 2 42 842 842 4842 4842 84 85 GEAAAA QMEAAA AAAAxx 8814 3033 0 2 4 14 14 814 814 3814 8814 28 29 ABAAAA RMEAAA HHHHxx 3429 3034 1 1 9 9 29 429 1429 3429 3429 58 59 XBAAAA SMEAAA OOOOxx 6550 3035 0 2 0 10 50 550 550 1550 6550 100 101 YRAAAA TMEAAA VVVVxx 6317 3036 1 1 7 17 17 317 317 1317 6317 34 35 ZIAAAA UMEAAA AAAAxx 5023 3037 1 3 3 3 23 23 1023 23 5023 46 47 FLAAAA VMEAAA HHHHxx 5825 3038 1 1 5 5 25 825 1825 825 5825 50 51 BQAAAA WMEAAA OOOOxx 5297 3039 1 1 7 17 97 297 1297 297 5297 194 195 TVAAAA XMEAAA VVVVxx 8764 3040 0 0 4 4 64 764 764 3764 8764 128 129 CZAAAA YMEAAA AAAAxx 5084 3041 0 0 4 4 84 84 1084 84 5084 168 169 ONAAAA ZMEAAA HHHHxx 6808 3042 0 0 8 8 8 808 808 1808 6808 16 17 WBAAAA ANEAAA OOOOxx 1780 3043 0 0 0 0 80 780 1780 1780 1780 160 161 MQAAAA BNEAAA VVVVxx 4092 3044 0 0 2 12 92 92 92 4092 4092 184 185 KBAAAA CNEAAA AAAAxx 3618 3045 0 2 8 18 18 618 1618 3618 3618 36 37 EJAAAA DNEAAA HHHHxx 7299 3046 1 3 9 19 99 299 1299 2299 7299 198 199 TUAAAA ENEAAA OOOOxx 8544 3047 0 0 4 4 44 544 544 3544 8544 88 89 QQAAAA FNEAAA VVVVxx 2359 3048 1 3 9 19 59 359 359 2359 2359 118 119 TMAAAA GNEAAA AAAAxx 1939 3049 1 3 9 19 39 939 1939 1939 1939 78 79 PWAAAA HNEAAA HHHHxx 5834 3050 0 2 4 14 34 834 1834 834 5834 68 69 KQAAAA INEAAA OOOOxx 1997 3051 1 1 7 17 97 997 1997 1997 1997 194 195 VYAAAA JNEAAA VVVVxx 7917 3052 1 1 7 17 17 917 1917 2917 7917 34 35 NSAAAA KNEAAA AAAAxx 2098 3053 0 2 8 18 98 98 98 2098 2098 196 197 SCAAAA LNEAAA HHHHxx 7576 3054 0 0 6 16 76 576 1576 2576 7576 152 153 KFAAAA MNEAAA OOOOxx 376 3055 0 0 6 16 76 376 376 376 376 152 153 MOAAAA NNEAAA VVVVxx 8535 3056 1 3 5 15 35 535 535 3535 8535 70 71 HQAAAA ONEAAA AAAAxx 5659 3057 1 3 9 19 59 659 1659 659 5659 118 119 RJAAAA PNEAAA HHHHxx 2786 3058 0 2 6 6 86 786 786 2786 2786 172 173 EDAAAA QNEAAA OOOOxx 8820 3059 0 0 0 0 20 820 820 3820 8820 40 41 GBAAAA RNEAAA VVVVxx 1229 3060 1 1 9 9 29 229 1229 1229 1229 58 59 HVAAAA SNEAAA AAAAxx 9321 3061 1 1 1 1 21 321 1321 4321 9321 42 43 NUAAAA TNEAAA HHHHxx 7662 3062 0 2 2 2 62 662 1662 2662 7662 124 125 SIAAAA UNEAAA OOOOxx 5535 3063 1 3 5 15 35 535 1535 535 5535 70 71 XEAAAA VNEAAA VVVVxx 4889 3064 1 1 9 9 89 889 889 4889 4889 178 179 BGAAAA WNEAAA AAAAxx 8259 3065 1 3 9 19 59 259 259 3259 8259 118 119 RFAAAA XNEAAA HHHHxx 6789 3066 1 1 9 9 89 789 789 1789 6789 178 179 DBAAAA YNEAAA OOOOxx 5411 3067 1 3 1 11 11 411 1411 411 5411 22 23 DAAAAA ZNEAAA VVVVxx 6992 3068 0 0 2 12 92 992 992 1992 6992 184 185 YIAAAA AOEAAA AAAAxx 7698 3069 0 2 8 18 98 698 1698 2698 7698 196 197 CKAAAA BOEAAA HHHHxx 2342 3070 0 2 2 2 42 342 342 2342 2342 84 85 CMAAAA COEAAA OOOOxx 1501 3071 1 1 1 1 1 501 1501 1501 1501 2 3 TFAAAA DOEAAA VVVVxx 6322 3072 0 2 2 2 22 322 322 1322 6322 44 45 EJAAAA EOEAAA AAAAxx 9861 3073 1 1 1 1 61 861 1861 4861 9861 122 123 HPAAAA FOEAAA HHHHxx 9802 3074 0 2 2 2 2 802 1802 4802 9802 4 5 ANAAAA GOEAAA OOOOxx 4750 3075 0 2 0 10 50 750 750 4750 4750 100 101 SAAAAA HOEAAA VVVVxx 5855 3076 1 3 5 15 55 855 1855 855 5855 110 111 FRAAAA IOEAAA AAAAxx 4304 3077 0 0 4 4 4 304 304 4304 4304 8 9 OJAAAA JOEAAA HHHHxx 2605 3078 1 1 5 5 5 605 605 2605 2605 10 11 FWAAAA KOEAAA OOOOxx 1802 3079 0 2 2 2 2 802 1802 1802 1802 4 5 IRAAAA LOEAAA VVVVxx 9368 3080 0 0 8 8 68 368 1368 4368 9368 136 137 IWAAAA MOEAAA AAAAxx 7107 3081 1 3 7 7 7 107 1107 2107 7107 14 15 JNAAAA NOEAAA HHHHxx 8895 3082 1 3 5 15 95 895 895 3895 8895 190 191 DEAAAA OOEAAA OOOOxx 3750 3083 0 2 0 10 50 750 1750 3750 3750 100 101 GOAAAA POEAAA VVVVxx 8934 3084 0 2 4 14 34 934 934 3934 8934 68 69 QFAAAA QOEAAA AAAAxx 9464 3085 0 0 4 4 64 464 1464 4464 9464 128 129 AAAAAA ROEAAA HHHHxx 1928 3086 0 0 8 8 28 928 1928 1928 1928 56 57 EWAAAA SOEAAA OOOOxx 3196 3087 0 0 6 16 96 196 1196 3196 3196 192 193 YSAAAA TOEAAA VVVVxx 5256 3088 0 0 6 16 56 256 1256 256 5256 112 113 EUAAAA UOEAAA AAAAxx 7119 3089 1 3 9 19 19 119 1119 2119 7119 38 39 VNAAAA VOEAAA HHHHxx 4495 3090 1 3 5 15 95 495 495 4495 4495 190 191 XQAAAA WOEAAA OOOOxx 9292 3091 0 0 2 12 92 292 1292 4292 9292 184 185 KTAAAA XOEAAA VVVVxx 1617 3092 1 1 7 17 17 617 1617 1617 1617 34 35 FKAAAA YOEAAA AAAAxx 481 3093 1 1 1 1 81 481 481 481 481 162 163 NSAAAA ZOEAAA HHHHxx 56 3094 0 0 6 16 56 56 56 56 56 112 113 ECAAAA APEAAA OOOOxx 9120 3095 0 0 0 0 20 120 1120 4120 9120 40 41 UMAAAA BPEAAA VVVVxx 1306 3096 0 2 6 6 6 306 1306 1306 1306 12 13 GYAAAA CPEAAA AAAAxx 7773 3097 1 1 3 13 73 773 1773 2773 7773 146 147 ZMAAAA DPEAAA HHHHxx 4863 3098 1 3 3 3 63 863 863 4863 4863 126 127 BFAAAA EPEAAA OOOOxx 1114 3099 0 2 4 14 14 114 1114 1114 1114 28 29 WQAAAA FPEAAA VVVVxx 8124 3100 0 0 4 4 24 124 124 3124 8124 48 49 MAAAAA GPEAAA AAAAxx 6254 3101 0 2 4 14 54 254 254 1254 6254 108 109 OGAAAA HPEAAA HHHHxx 8109 3102 1 1 9 9 9 109 109 3109 8109 18 19 XZAAAA IPEAAA OOOOxx 1747 3103 1 3 7 7 47 747 1747 1747 1747 94 95 FPAAAA JPEAAA VVVVxx 6185 3104 1 1 5 5 85 185 185 1185 6185 170 171 XDAAAA KPEAAA AAAAxx 3388 3105 0 0 8 8 88 388 1388 3388 3388 176 177 IAAAAA LPEAAA HHHHxx 4905 3106 1 1 5 5 5 905 905 4905 4905 10 11 RGAAAA MPEAAA OOOOxx 5728 3107 0 0 8 8 28 728 1728 728 5728 56 57 IMAAAA NPEAAA VVVVxx 7507 3108 1 3 7 7 7 507 1507 2507 7507 14 15 TCAAAA OPEAAA AAAAxx 5662 3109 0 2 2 2 62 662 1662 662 5662 124 125 UJAAAA PPEAAA HHHHxx 1686 3110 0 2 6 6 86 686 1686 1686 1686 172 173 WMAAAA QPEAAA OOOOxx 5202 3111 0 2 2 2 2 202 1202 202 5202 4 5 CSAAAA RPEAAA VVVVxx 6905 3112 1 1 5 5 5 905 905 1905 6905 10 11 PFAAAA SPEAAA AAAAxx 9577 3113 1 1 7 17 77 577 1577 4577 9577 154 155 JEAAAA TPEAAA HHHHxx 7194 3114 0 2 4 14 94 194 1194 2194 7194 188 189 SQAAAA UPEAAA OOOOxx 7016 3115 0 0 6 16 16 16 1016 2016 7016 32 33 WJAAAA VPEAAA VVVVxx 8905 3116 1 1 5 5 5 905 905 3905 8905 10 11 NEAAAA WPEAAA AAAAxx 3419 3117 1 3 9 19 19 419 1419 3419 3419 38 39 NBAAAA XPEAAA HHHHxx 6881 3118 1 1 1 1 81 881 881 1881 6881 162 163 REAAAA YPEAAA OOOOxx 8370 3119 0 2 0 10 70 370 370 3370 8370 140 141 YJAAAA ZPEAAA VVVVxx 6117 3120 1 1 7 17 17 117 117 1117 6117 34 35 HBAAAA AQEAAA AAAAxx 1636 3121 0 0 6 16 36 636 1636 1636 1636 72 73 YKAAAA BQEAAA HHHHxx 6857 3122 1 1 7 17 57 857 857 1857 6857 114 115 TDAAAA CQEAAA OOOOxx 7163 3123 1 3 3 3 63 163 1163 2163 7163 126 127 NPAAAA DQEAAA VVVVxx 5040 3124 0 0 0 0 40 40 1040 40 5040 80 81 WLAAAA EQEAAA AAAAxx 6263 3125 1 3 3 3 63 263 263 1263 6263 126 127 XGAAAA FQEAAA HHHHxx 4809 3126 1 1 9 9 9 809 809 4809 4809 18 19 ZCAAAA GQEAAA OOOOxx 900 3127 0 0 0 0 0 900 900 900 900 0 1 QIAAAA HQEAAA VVVVxx 3199 3128 1 3 9 19 99 199 1199 3199 3199 198 199 BTAAAA IQEAAA AAAAxx 4156 3129 0 0 6 16 56 156 156 4156 4156 112 113 WDAAAA JQEAAA HHHHxx 3501 3130 1 1 1 1 1 501 1501 3501 3501 2 3 REAAAA KQEAAA OOOOxx 164 3131 0 0 4 4 64 164 164 164 164 128 129 IGAAAA LQEAAA VVVVxx 9548 3132 0 0 8 8 48 548 1548 4548 9548 96 97 GDAAAA MQEAAA AAAAxx 1149 3133 1 1 9 9 49 149 1149 1149 1149 98 99 FSAAAA NQEAAA HHHHxx 1962 3134 0 2 2 2 62 962 1962 1962 1962 124 125 MXAAAA OQEAAA OOOOxx 4072 3135 0 0 2 12 72 72 72 4072 4072 144 145 QAAAAA PQEAAA VVVVxx 4280 3136 0 0 0 0 80 280 280 4280 4280 160 161 QIAAAA QQEAAA AAAAxx 1398 3137 0 2 8 18 98 398 1398 1398 1398 196 197 UBAAAA RQEAAA HHHHxx 725 3138 1 1 5 5 25 725 725 725 725 50 51 XBAAAA SQEAAA OOOOxx 3988 3139 0 0 8 8 88 988 1988 3988 3988 176 177 KXAAAA TQEAAA VVVVxx 5059 3140 1 3 9 19 59 59 1059 59 5059 118 119 PMAAAA UQEAAA AAAAxx 2632 3141 0 0 2 12 32 632 632 2632 2632 64 65 GXAAAA VQEAAA HHHHxx 1909 3142 1 1 9 9 9 909 1909 1909 1909 18 19 LVAAAA WQEAAA OOOOxx 6827 3143 1 3 7 7 27 827 827 1827 6827 54 55 PCAAAA XQEAAA VVVVxx 8156 3144 0 0 6 16 56 156 156 3156 8156 112 113 SBAAAA YQEAAA AAAAxx 1192 3145 0 0 2 12 92 192 1192 1192 1192 184 185 WTAAAA ZQEAAA HHHHxx 9545 3146 1 1 5 5 45 545 1545 4545 9545 90 91 DDAAAA AREAAA OOOOxx 2249 3147 1 1 9 9 49 249 249 2249 2249 98 99 NIAAAA BREAAA VVVVxx 5580 3148 0 0 0 0 80 580 1580 580 5580 160 161 QGAAAA CREAAA AAAAxx 8403 3149 1 3 3 3 3 403 403 3403 8403 6 7 FLAAAA DREAAA HHHHxx 4024 3150 0 0 4 4 24 24 24 4024 4024 48 49 UYAAAA EREAAA OOOOxx 1866 3151 0 2 6 6 66 866 1866 1866 1866 132 133 UTAAAA FREAAA VVVVxx 9251 3152 1 3 1 11 51 251 1251 4251 9251 102 103 VRAAAA GREAAA AAAAxx 9979 3153 1 3 9 19 79 979 1979 4979 9979 158 159 VTAAAA HREAAA HHHHxx 9899 3154 1 3 9 19 99 899 1899 4899 9899 198 199 TQAAAA IREAAA OOOOxx 2540 3155 0 0 0 0 40 540 540 2540 2540 80 81 STAAAA JREAAA VVVVxx 8957 3156 1 1 7 17 57 957 957 3957 8957 114 115 NGAAAA KREAAA AAAAxx 7702 3157 0 2 2 2 2 702 1702 2702 7702 4 5 GKAAAA LREAAA HHHHxx 4211 3158 1 3 1 11 11 211 211 4211 4211 22 23 ZFAAAA MREAAA OOOOxx 6684 3159 0 0 4 4 84 684 684 1684 6684 168 169 CXAAAA NREAAA VVVVxx 3883 3160 1 3 3 3 83 883 1883 3883 3883 166 167 JTAAAA OREAAA AAAAxx 3531 3161 1 3 1 11 31 531 1531 3531 3531 62 63 VFAAAA PREAAA HHHHxx 9178 3162 0 2 8 18 78 178 1178 4178 9178 156 157 APAAAA QREAAA OOOOxx 3389 3163 1 1 9 9 89 389 1389 3389 3389 178 179 JAAAAA RREAAA VVVVxx 7874 3164 0 2 4 14 74 874 1874 2874 7874 148 149 WQAAAA SREAAA AAAAxx 4522 3165 0 2 2 2 22 522 522 4522 4522 44 45 YRAAAA TREAAA HHHHxx 9399 3166 1 3 9 19 99 399 1399 4399 9399 198 199 NXAAAA UREAAA OOOOxx 9083 3167 1 3 3 3 83 83 1083 4083 9083 166 167 JLAAAA VREAAA VVVVxx 1530 3168 0 2 0 10 30 530 1530 1530 1530 60 61 WGAAAA WREAAA AAAAxx 2360 3169 0 0 0 0 60 360 360 2360 2360 120 121 UMAAAA XREAAA HHHHxx 4908 3170 0 0 8 8 8 908 908 4908 4908 16 17 UGAAAA YREAAA OOOOxx 4628 3171 0 0 8 8 28 628 628 4628 4628 56 57 AWAAAA ZREAAA VVVVxx 3889 3172 1 1 9 9 89 889 1889 3889 3889 178 179 PTAAAA ASEAAA AAAAxx 1331 3173 1 3 1 11 31 331 1331 1331 1331 62 63 FZAAAA BSEAAA HHHHxx 1942 3174 0 2 2 2 42 942 1942 1942 1942 84 85 SWAAAA CSEAAA OOOOxx 4734 3175 0 2 4 14 34 734 734 4734 4734 68 69 CAAAAA DSEAAA VVVVxx 8386 3176 0 2 6 6 86 386 386 3386 8386 172 173 OKAAAA ESEAAA AAAAxx 3586 3177 0 2 6 6 86 586 1586 3586 3586 172 173 YHAAAA FSEAAA HHHHxx 2354 3178 0 2 4 14 54 354 354 2354 2354 108 109 OMAAAA GSEAAA OOOOxx 7108 3179 0 0 8 8 8 108 1108 2108 7108 16 17 KNAAAA HSEAAA VVVVxx 1857 3180 1 1 7 17 57 857 1857 1857 1857 114 115 LTAAAA ISEAAA AAAAxx 2544 3181 0 0 4 4 44 544 544 2544 2544 88 89 WTAAAA JSEAAA HHHHxx 819 3182 1 3 9 19 19 819 819 819 819 38 39 NFAAAA KSEAAA OOOOxx 2878 3183 0 2 8 18 78 878 878 2878 2878 156 157 SGAAAA LSEAAA VVVVxx 1772 3184 0 0 2 12 72 772 1772 1772 1772 144 145 EQAAAA MSEAAA AAAAxx 354 3185 0 2 4 14 54 354 354 354 354 108 109 QNAAAA NSEAAA HHHHxx 3259 3186 1 3 9 19 59 259 1259 3259 3259 118 119 JVAAAA OSEAAA OOOOxx 2170 3187 0 2 0 10 70 170 170 2170 2170 140 141 MFAAAA PSEAAA VVVVxx 1190 3188 0 2 0 10 90 190 1190 1190 1190 180 181 UTAAAA QSEAAA AAAAxx 3607 3189 1 3 7 7 7 607 1607 3607 3607 14 15 TIAAAA RSEAAA HHHHxx 4661 3190 1 1 1 1 61 661 661 4661 4661 122 123 HXAAAA SSEAAA OOOOxx 1796 3191 0 0 6 16 96 796 1796 1796 1796 192 193 CRAAAA TSEAAA VVVVxx 1561 3192 1 1 1 1 61 561 1561 1561 1561 122 123 BIAAAA USEAAA AAAAxx 4336 3193 0 0 6 16 36 336 336 4336 4336 72 73 UKAAAA VSEAAA HHHHxx 7550 3194 0 2 0 10 50 550 1550 2550 7550 100 101 KEAAAA WSEAAA OOOOxx 3238 3195 0 2 8 18 38 238 1238 3238 3238 76 77 OUAAAA XSEAAA VVVVxx 9870 3196 0 2 0 10 70 870 1870 4870 9870 140 141 QPAAAA YSEAAA AAAAxx 6502 3197 0 2 2 2 2 502 502 1502 6502 4 5 CQAAAA ZSEAAA HHHHxx 3903 3198 1 3 3 3 3 903 1903 3903 3903 6 7 DUAAAA ATEAAA OOOOxx 2869 3199 1 1 9 9 69 869 869 2869 2869 138 139 JGAAAA BTEAAA VVVVxx 5072 3200 0 0 2 12 72 72 1072 72 5072 144 145 CNAAAA CTEAAA AAAAxx 1201 3201 1 1 1 1 1 201 1201 1201 1201 2 3 FUAAAA DTEAAA HHHHxx 6245 3202 1 1 5 5 45 245 245 1245 6245 90 91 FGAAAA ETEAAA OOOOxx 1402 3203 0 2 2 2 2 402 1402 1402 1402 4 5 YBAAAA FTEAAA VVVVxx 2594 3204 0 2 4 14 94 594 594 2594 2594 188 189 UVAAAA GTEAAA AAAAxx 9171 3205 1 3 1 11 71 171 1171 4171 9171 142 143 TOAAAA HTEAAA HHHHxx 2620 3206 0 0 0 0 20 620 620 2620 2620 40 41 UWAAAA ITEAAA OOOOxx 6309 3207 1 1 9 9 9 309 309 1309 6309 18 19 RIAAAA JTEAAA VVVVxx 1285 3208 1 1 5 5 85 285 1285 1285 1285 170 171 LXAAAA KTEAAA AAAAxx 5466 3209 0 2 6 6 66 466 1466 466 5466 132 133 GCAAAA LTEAAA HHHHxx 168 3210 0 0 8 8 68 168 168 168 168 136 137 MGAAAA MTEAAA OOOOxx 1410 3211 0 2 0 10 10 410 1410 1410 1410 20 21 GCAAAA NTEAAA VVVVxx 6332 3212 0 0 2 12 32 332 332 1332 6332 64 65 OJAAAA OTEAAA AAAAxx 9530 3213 0 2 0 10 30 530 1530 4530 9530 60 61 OCAAAA PTEAAA HHHHxx 7749 3214 1 1 9 9 49 749 1749 2749 7749 98 99 BMAAAA QTEAAA OOOOxx 3656 3215 0 0 6 16 56 656 1656 3656 3656 112 113 QKAAAA RTEAAA VVVVxx 37 3216 1 1 7 17 37 37 37 37 37 74 75 LBAAAA STEAAA AAAAxx 2744 3217 0 0 4 4 44 744 744 2744 2744 88 89 OBAAAA TTEAAA HHHHxx 4206 3218 0 2 6 6 6 206 206 4206 4206 12 13 UFAAAA UTEAAA OOOOxx 1846 3219 0 2 6 6 46 846 1846 1846 1846 92 93 ATAAAA VTEAAA VVVVxx 9913 3220 1 1 3 13 13 913 1913 4913 9913 26 27 HRAAAA WTEAAA AAAAxx 4078 3221 0 2 8 18 78 78 78 4078 4078 156 157 WAAAAA XTEAAA HHHHxx 2080 3222 0 0 0 0 80 80 80 2080 2080 160 161 ACAAAA YTEAAA OOOOxx 4169 3223 1 1 9 9 69 169 169 4169 4169 138 139 JEAAAA ZTEAAA VVVVxx 2070 3224 0 2 0 10 70 70 70 2070 2070 140 141 QBAAAA AUEAAA AAAAxx 4500 3225 0 0 0 0 0 500 500 4500 4500 0 1 CRAAAA BUEAAA HHHHxx 4123 3226 1 3 3 3 23 123 123 4123 4123 46 47 PCAAAA CUEAAA OOOOxx 5594 3227 0 2 4 14 94 594 1594 594 5594 188 189 EHAAAA DUEAAA VVVVxx 9941 3228 1 1 1 1 41 941 1941 4941 9941 82 83 JSAAAA EUEAAA AAAAxx 7154 3229 0 2 4 14 54 154 1154 2154 7154 108 109 EPAAAA FUEAAA HHHHxx 8340 3230 0 0 0 0 40 340 340 3340 8340 80 81 UIAAAA GUEAAA OOOOxx 7110 3231 0 2 0 10 10 110 1110 2110 7110 20 21 MNAAAA HUEAAA VVVVxx 7795 3232 1 3 5 15 95 795 1795 2795 7795 190 191 VNAAAA IUEAAA AAAAxx 132 3233 0 0 2 12 32 132 132 132 132 64 65 CFAAAA JUEAAA HHHHxx 4603 3234 1 3 3 3 3 603 603 4603 4603 6 7 BVAAAA KUEAAA OOOOxx 9720 3235 0 0 0 0 20 720 1720 4720 9720 40 41 WJAAAA LUEAAA VVVVxx 1460 3236 0 0 0 0 60 460 1460 1460 1460 120 121 EEAAAA MUEAAA AAAAxx 4677 3237 1 1 7 17 77 677 677 4677 4677 154 155 XXAAAA NUEAAA HHHHxx 9272 3238 0 0 2 12 72 272 1272 4272 9272 144 145 QSAAAA OUEAAA OOOOxx 2279 3239 1 3 9 19 79 279 279 2279 2279 158 159 RJAAAA PUEAAA VVVVxx 4587 3240 1 3 7 7 87 587 587 4587 4587 174 175 LUAAAA QUEAAA AAAAxx 2244 3241 0 0 4 4 44 244 244 2244 2244 88 89 IIAAAA RUEAAA HHHHxx 742 3242 0 2 2 2 42 742 742 742 742 84 85 OCAAAA SUEAAA OOOOxx 4426 3243 0 2 6 6 26 426 426 4426 4426 52 53 GOAAAA TUEAAA VVVVxx 4571 3244 1 3 1 11 71 571 571 4571 4571 142 143 VTAAAA UUEAAA AAAAxx 4775 3245 1 3 5 15 75 775 775 4775 4775 150 151 RBAAAA VUEAAA HHHHxx 24 3246 0 0 4 4 24 24 24 24 24 48 49 YAAAAA WUEAAA OOOOxx 4175 3247 1 3 5 15 75 175 175 4175 4175 150 151 PEAAAA XUEAAA VVVVxx 9877 3248 1 1 7 17 77 877 1877 4877 9877 154 155 XPAAAA YUEAAA AAAAxx 7271 3249 1 3 1 11 71 271 1271 2271 7271 142 143 RTAAAA ZUEAAA HHHHxx 5468 3250 0 0 8 8 68 468 1468 468 5468 136 137 ICAAAA AVEAAA OOOOxx 6106 3251 0 2 6 6 6 106 106 1106 6106 12 13 WAAAAA BVEAAA VVVVxx 9005 3252 1 1 5 5 5 5 1005 4005 9005 10 11 JIAAAA CVEAAA AAAAxx 109 3253 1 1 9 9 9 109 109 109 109 18 19 FEAAAA DVEAAA HHHHxx 6365 3254 1 1 5 5 65 365 365 1365 6365 130 131 VKAAAA EVEAAA OOOOxx 7437 3255 1 1 7 17 37 437 1437 2437 7437 74 75 BAAAAA FVEAAA VVVVxx 7979 3256 1 3 9 19 79 979 1979 2979 7979 158 159 XUAAAA GVEAAA AAAAxx 6050 3257 0 2 0 10 50 50 50 1050 6050 100 101 SYAAAA HVEAAA HHHHxx 2853 3258 1 1 3 13 53 853 853 2853 2853 106 107 TFAAAA IVEAAA OOOOxx 7603 3259 1 3 3 3 3 603 1603 2603 7603 6 7 LGAAAA JVEAAA VVVVxx 483 3260 1 3 3 3 83 483 483 483 483 166 167 PSAAAA KVEAAA AAAAxx 5994 3261 0 2 4 14 94 994 1994 994 5994 188 189 OWAAAA LVEAAA HHHHxx 6708 3262 0 0 8 8 8 708 708 1708 6708 16 17 AYAAAA MVEAAA OOOOxx 5090 3263 0 2 0 10 90 90 1090 90 5090 180 181 UNAAAA NVEAAA VVVVxx 4608 3264 0 0 8 8 8 608 608 4608 4608 16 17 GVAAAA OVEAAA AAAAxx 4551 3265 1 3 1 11 51 551 551 4551 4551 102 103 BTAAAA PVEAAA HHHHxx 5437 3266 1 1 7 17 37 437 1437 437 5437 74 75 DBAAAA QVEAAA OOOOxx 4130 3267 0 2 0 10 30 130 130 4130 4130 60 61 WCAAAA RVEAAA VVVVxx 6363 3268 1 3 3 3 63 363 363 1363 6363 126 127 TKAAAA SVEAAA AAAAxx 1499 3269 1 3 9 19 99 499 1499 1499 1499 198 199 RFAAAA TVEAAA HHHHxx 384 3270 0 0 4 4 84 384 384 384 384 168 169 UOAAAA UVEAAA OOOOxx 2266 3271 0 2 6 6 66 266 266 2266 2266 132 133 EJAAAA VVEAAA VVVVxx 6018 3272 0 2 8 18 18 18 18 1018 6018 36 37 MXAAAA WVEAAA AAAAxx 7915 3273 1 3 5 15 15 915 1915 2915 7915 30 31 LSAAAA XVEAAA HHHHxx 6167 3274 1 3 7 7 67 167 167 1167 6167 134 135 FDAAAA YVEAAA OOOOxx 9988 3275 0 0 8 8 88 988 1988 4988 9988 176 177 EUAAAA ZVEAAA VVVVxx 6599 3276 1 3 9 19 99 599 599 1599 6599 198 199 VTAAAA AWEAAA AAAAxx 1693 3277 1 1 3 13 93 693 1693 1693 1693 186 187 DNAAAA BWEAAA HHHHxx 5971 3278 1 3 1 11 71 971 1971 971 5971 142 143 RVAAAA CWEAAA OOOOxx 8470 3279 0 2 0 10 70 470 470 3470 8470 140 141 UNAAAA DWEAAA VVVVxx 2807 3280 1 3 7 7 7 807 807 2807 2807 14 15 ZDAAAA EWEAAA AAAAxx 1120 3281 0 0 0 0 20 120 1120 1120 1120 40 41 CRAAAA FWEAAA HHHHxx 5924 3282 0 0 4 4 24 924 1924 924 5924 48 49 WTAAAA GWEAAA OOOOxx 9025 3283 1 1 5 5 25 25 1025 4025 9025 50 51 DJAAAA HWEAAA VVVVxx 9454 3284 0 2 4 14 54 454 1454 4454 9454 108 109 QZAAAA IWEAAA AAAAxx 2259 3285 1 3 9 19 59 259 259 2259 2259 118 119 XIAAAA JWEAAA HHHHxx 5249 3286 1 1 9 9 49 249 1249 249 5249 98 99 XTAAAA KWEAAA OOOOxx 6350 3287 0 2 0 10 50 350 350 1350 6350 100 101 GKAAAA LWEAAA VVVVxx 2930 3288 0 2 0 10 30 930 930 2930 2930 60 61 SIAAAA MWEAAA AAAAxx 6055 3289 1 3 5 15 55 55 55 1055 6055 110 111 XYAAAA NWEAAA HHHHxx 7691 3290 1 3 1 11 91 691 1691 2691 7691 182 183 VJAAAA OWEAAA OOOOxx 1573 3291 1 1 3 13 73 573 1573 1573 1573 146 147 NIAAAA PWEAAA VVVVxx 9943 3292 1 3 3 3 43 943 1943 4943 9943 86 87 LSAAAA QWEAAA AAAAxx 3085 3293 1 1 5 5 85 85 1085 3085 3085 170 171 ROAAAA RWEAAA HHHHxx 5928 3294 0 0 8 8 28 928 1928 928 5928 56 57 AUAAAA SWEAAA OOOOxx 887 3295 1 3 7 7 87 887 887 887 887 174 175 DIAAAA TWEAAA VVVVxx 4630 3296 0 2 0 10 30 630 630 4630 4630 60 61 CWAAAA UWEAAA AAAAxx 9827 3297 1 3 7 7 27 827 1827 4827 9827 54 55 ZNAAAA VWEAAA HHHHxx 8926 3298 0 2 6 6 26 926 926 3926 8926 52 53 IFAAAA WWEAAA OOOOxx 5726 3299 0 2 6 6 26 726 1726 726 5726 52 53 GMAAAA XWEAAA VVVVxx 1569 3300 1 1 9 9 69 569 1569 1569 1569 138 139 JIAAAA YWEAAA AAAAxx 8074 3301 0 2 4 14 74 74 74 3074 8074 148 149 OYAAAA ZWEAAA HHHHxx 7909 3302 1 1 9 9 9 909 1909 2909 7909 18 19 FSAAAA AXEAAA OOOOxx 8367 3303 1 3 7 7 67 367 367 3367 8367 134 135 VJAAAA BXEAAA VVVVxx 7217 3304 1 1 7 17 17 217 1217 2217 7217 34 35 PRAAAA CXEAAA AAAAxx 5254 3305 0 2 4 14 54 254 1254 254 5254 108 109 CUAAAA DXEAAA HHHHxx 1181 3306 1 1 1 1 81 181 1181 1181 1181 162 163 LTAAAA EXEAAA OOOOxx 6907 3307 1 3 7 7 7 907 907 1907 6907 14 15 RFAAAA FXEAAA VVVVxx 5508 3308 0 0 8 8 8 508 1508 508 5508 16 17 WDAAAA GXEAAA AAAAxx 4782 3309 0 2 2 2 82 782 782 4782 4782 164 165 YBAAAA HXEAAA HHHHxx 793 3310 1 1 3 13 93 793 793 793 793 186 187 NEAAAA IXEAAA OOOOxx 5740 3311 0 0 0 0 40 740 1740 740 5740 80 81 UMAAAA JXEAAA VVVVxx 3107 3312 1 3 7 7 7 107 1107 3107 3107 14 15 NPAAAA KXEAAA AAAAxx 1197 3313 1 1 7 17 97 197 1197 1197 1197 194 195 BUAAAA LXEAAA HHHHxx 4376 3314 0 0 6 16 76 376 376 4376 4376 152 153 IMAAAA MXEAAA OOOOxx 6226 3315 0 2 6 6 26 226 226 1226 6226 52 53 MFAAAA NXEAAA VVVVxx 5033 3316 1 1 3 13 33 33 1033 33 5033 66 67 PLAAAA OXEAAA AAAAxx 5494 3317 0 2 4 14 94 494 1494 494 5494 188 189 IDAAAA PXEAAA HHHHxx 3244 3318 0 0 4 4 44 244 1244 3244 3244 88 89 UUAAAA QXEAAA OOOOxx 7670 3319 0 2 0 10 70 670 1670 2670 7670 140 141 AJAAAA RXEAAA VVVVxx 9273 3320 1 1 3 13 73 273 1273 4273 9273 146 147 RSAAAA SXEAAA AAAAxx 5248 3321 0 0 8 8 48 248 1248 248 5248 96 97 WTAAAA TXEAAA HHHHxx 3381 3322 1 1 1 1 81 381 1381 3381 3381 162 163 BAAAAA UXEAAA OOOOxx 4136 3323 0 0 6 16 36 136 136 4136 4136 72 73 CDAAAA VXEAAA VVVVxx 4163 3324 1 3 3 3 63 163 163 4163 4163 126 127 DEAAAA WXEAAA AAAAxx 4270 3325 0 2 0 10 70 270 270 4270 4270 140 141 GIAAAA XXEAAA HHHHxx 1729 3326 1 1 9 9 29 729 1729 1729 1729 58 59 NOAAAA YXEAAA OOOOxx 2778 3327 0 2 8 18 78 778 778 2778 2778 156 157 WCAAAA ZXEAAA VVVVxx 5082 3328 0 2 2 2 82 82 1082 82 5082 164 165 MNAAAA AYEAAA AAAAxx 870 3329 0 2 0 10 70 870 870 870 870 140 141 MHAAAA BYEAAA HHHHxx 4192 3330 0 0 2 12 92 192 192 4192 4192 184 185 GFAAAA CYEAAA OOOOxx 308 3331 0 0 8 8 8 308 308 308 308 16 17 WLAAAA DYEAAA VVVVxx 6783 3332 1 3 3 3 83 783 783 1783 6783 166 167 XAAAAA EYEAAA AAAAxx 7611 3333 1 3 1 11 11 611 1611 2611 7611 22 23 TGAAAA FYEAAA HHHHxx 4221 3334 1 1 1 1 21 221 221 4221 4221 42 43 JGAAAA GYEAAA OOOOxx 6353 3335 1 1 3 13 53 353 353 1353 6353 106 107 JKAAAA HYEAAA VVVVxx 1830 3336 0 2 0 10 30 830 1830 1830 1830 60 61 KSAAAA IYEAAA AAAAxx 2437 3337 1 1 7 17 37 437 437 2437 2437 74 75 TPAAAA JYEAAA HHHHxx 3360 3338 0 0 0 0 60 360 1360 3360 3360 120 121 GZAAAA KYEAAA OOOOxx 1829 3339 1 1 9 9 29 829 1829 1829 1829 58 59 JSAAAA LYEAAA VVVVxx 9475 3340 1 3 5 15 75 475 1475 4475 9475 150 151 LAAAAA MYEAAA AAAAxx 4566 3341 0 2 6 6 66 566 566 4566 4566 132 133 QTAAAA NYEAAA HHHHxx 9944 3342 0 0 4 4 44 944 1944 4944 9944 88 89 MSAAAA OYEAAA OOOOxx 6054 3343 0 2 4 14 54 54 54 1054 6054 108 109 WYAAAA PYEAAA VVVVxx 4722 3344 0 2 2 2 22 722 722 4722 4722 44 45 QZAAAA QYEAAA AAAAxx 2779 3345 1 3 9 19 79 779 779 2779 2779 158 159 XCAAAA RYEAAA HHHHxx 8051 3346 1 3 1 11 51 51 51 3051 8051 102 103 RXAAAA SYEAAA OOOOxx 9671 3347 1 3 1 11 71 671 1671 4671 9671 142 143 ZHAAAA TYEAAA VVVVxx 6084 3348 0 0 4 4 84 84 84 1084 6084 168 169 AAAAAA UYEAAA AAAAxx 3729 3349 1 1 9 9 29 729 1729 3729 3729 58 59 LNAAAA VYEAAA HHHHxx 6627 3350 1 3 7 7 27 627 627 1627 6627 54 55 XUAAAA WYEAAA OOOOxx 4769 3351 1 1 9 9 69 769 769 4769 4769 138 139 LBAAAA XYEAAA VVVVxx 2224 3352 0 0 4 4 24 224 224 2224 2224 48 49 OHAAAA YYEAAA AAAAxx 1404 3353 0 0 4 4 4 404 1404 1404 1404 8 9 ACAAAA ZYEAAA HHHHxx 8532 3354 0 0 2 12 32 532 532 3532 8532 64 65 EQAAAA AZEAAA OOOOxx 6759 3355 1 3 9 19 59 759 759 1759 6759 118 119 ZZAAAA BZEAAA VVVVxx 6404 3356 0 0 4 4 4 404 404 1404 6404 8 9 IMAAAA CZEAAA AAAAxx 3144 3357 0 0 4 4 44 144 1144 3144 3144 88 89 YQAAAA DZEAAA HHHHxx 973 3358 1 1 3 13 73 973 973 973 973 146 147 LLAAAA EZEAAA OOOOxx 9789 3359 1 1 9 9 89 789 1789 4789 9789 178 179 NMAAAA FZEAAA VVVVxx 6181 3360 1 1 1 1 81 181 181 1181 6181 162 163 TDAAAA GZEAAA AAAAxx 1519 3361 1 3 9 19 19 519 1519 1519 1519 38 39 LGAAAA HZEAAA HHHHxx 9729 3362 1 1 9 9 29 729 1729 4729 9729 58 59 FKAAAA IZEAAA OOOOxx 8167 3363 1 3 7 7 67 167 167 3167 8167 134 135 DCAAAA JZEAAA VVVVxx 3830 3364 0 2 0 10 30 830 1830 3830 3830 60 61 IRAAAA KZEAAA AAAAxx 6286 3365 0 2 6 6 86 286 286 1286 6286 172 173 UHAAAA LZEAAA HHHHxx 3047 3366 1 3 7 7 47 47 1047 3047 3047 94 95 FNAAAA MZEAAA OOOOxx 3183 3367 1 3 3 3 83 183 1183 3183 3183 166 167 LSAAAA NZEAAA VVVVxx 6687 3368 1 3 7 7 87 687 687 1687 6687 174 175 FXAAAA OZEAAA AAAAxx 2783 3369 1 3 3 3 83 783 783 2783 2783 166 167 BDAAAA PZEAAA HHHHxx 9920 3370 0 0 0 0 20 920 1920 4920 9920 40 41 ORAAAA QZEAAA OOOOxx 4847 3371 1 3 7 7 47 847 847 4847 4847 94 95 LEAAAA RZEAAA VVVVxx 3645 3372 1 1 5 5 45 645 1645 3645 3645 90 91 FKAAAA SZEAAA AAAAxx 7406 3373 0 2 6 6 6 406 1406 2406 7406 12 13 WYAAAA TZEAAA HHHHxx 6003 3374 1 3 3 3 3 3 3 1003 6003 6 7 XWAAAA UZEAAA OOOOxx 3408 3375 0 0 8 8 8 408 1408 3408 3408 16 17 CBAAAA VZEAAA VVVVxx 4243 3376 1 3 3 3 43 243 243 4243 4243 86 87 FHAAAA WZEAAA AAAAxx 1622 3377 0 2 2 2 22 622 1622 1622 1622 44 45 KKAAAA XZEAAA HHHHxx 5319 3378 1 3 9 19 19 319 1319 319 5319 38 39 PWAAAA YZEAAA OOOOxx 4033 3379 1 1 3 13 33 33 33 4033 4033 66 67 DZAAAA ZZEAAA VVVVxx 8573 3380 1 1 3 13 73 573 573 3573 8573 146 147 TRAAAA AAFAAA AAAAxx 8404 3381 0 0 4 4 4 404 404 3404 8404 8 9 GLAAAA BAFAAA HHHHxx 6993 3382 1 1 3 13 93 993 993 1993 6993 186 187 ZIAAAA CAFAAA OOOOxx 660 3383 0 0 0 0 60 660 660 660 660 120 121 KZAAAA DAFAAA VVVVxx 1136 3384 0 0 6 16 36 136 1136 1136 1136 72 73 SRAAAA EAFAAA AAAAxx 3393 3385 1 1 3 13 93 393 1393 3393 3393 186 187 NAAAAA FAFAAA HHHHxx 9743 3386 1 3 3 3 43 743 1743 4743 9743 86 87 TKAAAA GAFAAA OOOOxx 9705 3387 1 1 5 5 5 705 1705 4705 9705 10 11 HJAAAA HAFAAA VVVVxx 6960 3388 0 0 0 0 60 960 960 1960 6960 120 121 SHAAAA IAFAAA AAAAxx 2753 3389 1 1 3 13 53 753 753 2753 2753 106 107 XBAAAA JAFAAA HHHHxx 906 3390 0 2 6 6 6 906 906 906 906 12 13 WIAAAA KAFAAA OOOOxx 999 3391 1 3 9 19 99 999 999 999 999 198 199 LMAAAA LAFAAA VVVVxx 6927 3392 1 3 7 7 27 927 927 1927 6927 54 55 LGAAAA MAFAAA AAAAxx 4846 3393 0 2 6 6 46 846 846 4846 4846 92 93 KEAAAA NAFAAA HHHHxx 676 3394 0 0 6 16 76 676 676 676 676 152 153 AAAAAA OAFAAA OOOOxx 8612 3395 0 0 2 12 12 612 612 3612 8612 24 25 GTAAAA PAFAAA VVVVxx 4111 3396 1 3 1 11 11 111 111 4111 4111 22 23 DCAAAA QAFAAA AAAAxx 9994 3397 0 2 4 14 94 994 1994 4994 9994 188 189 KUAAAA RAFAAA HHHHxx 4399 3398 1 3 9 19 99 399 399 4399 4399 198 199 FNAAAA SAFAAA OOOOxx 4464 3399 0 0 4 4 64 464 464 4464 4464 128 129 SPAAAA TAFAAA VVVVxx 7316 3400 0 0 6 16 16 316 1316 2316 7316 32 33 KVAAAA UAFAAA AAAAxx 8982 3401 0 2 2 2 82 982 982 3982 8982 164 165 MHAAAA VAFAAA HHHHxx 1871 3402 1 3 1 11 71 871 1871 1871 1871 142 143 ZTAAAA WAFAAA OOOOxx 4082 3403 0 2 2 2 82 82 82 4082 4082 164 165 ABAAAA XAFAAA VVVVxx 3949 3404 1 1 9 9 49 949 1949 3949 3949 98 99 XVAAAA YAFAAA AAAAxx 9352 3405 0 0 2 12 52 352 1352 4352 9352 104 105 SVAAAA ZAFAAA HHHHxx 9638 3406 0 2 8 18 38 638 1638 4638 9638 76 77 SGAAAA ABFAAA OOOOxx 8177 3407 1 1 7 17 77 177 177 3177 8177 154 155 NCAAAA BBFAAA VVVVxx 3499 3408 1 3 9 19 99 499 1499 3499 3499 198 199 PEAAAA CBFAAA AAAAxx 4233 3409 1 1 3 13 33 233 233 4233 4233 66 67 VGAAAA DBFAAA HHHHxx 1953 3410 1 1 3 13 53 953 1953 1953 1953 106 107 DXAAAA EBFAAA OOOOxx 7372 3411 0 0 2 12 72 372 1372 2372 7372 144 145 OXAAAA FBFAAA VVVVxx 5127 3412 1 3 7 7 27 127 1127 127 5127 54 55 FPAAAA GBFAAA AAAAxx 4384 3413 0 0 4 4 84 384 384 4384 4384 168 169 QMAAAA HBFAAA HHHHxx 9964 3414 0 0 4 4 64 964 1964 4964 9964 128 129 GTAAAA IBFAAA OOOOxx 5392 3415 0 0 2 12 92 392 1392 392 5392 184 185 KZAAAA JBFAAA VVVVxx 616 3416 0 0 6 16 16 616 616 616 616 32 33 SXAAAA KBFAAA AAAAxx 591 3417 1 3 1 11 91 591 591 591 591 182 183 TWAAAA LBFAAA HHHHxx 6422 3418 0 2 2 2 22 422 422 1422 6422 44 45 ANAAAA MBFAAA OOOOxx 6551 3419 1 3 1 11 51 551 551 1551 6551 102 103 ZRAAAA NBFAAA VVVVxx 9286 3420 0 2 6 6 86 286 1286 4286 9286 172 173 ETAAAA OBFAAA AAAAxx 3817 3421 1 1 7 17 17 817 1817 3817 3817 34 35 VQAAAA PBFAAA HHHHxx 7717 3422 1 1 7 17 17 717 1717 2717 7717 34 35 VKAAAA QBFAAA OOOOxx 8718 3423 0 2 8 18 18 718 718 3718 8718 36 37 IXAAAA RBFAAA VVVVxx 8608 3424 0 0 8 8 8 608 608 3608 8608 16 17 CTAAAA SBFAAA AAAAxx 2242 3425 0 2 2 2 42 242 242 2242 2242 84 85 GIAAAA TBFAAA HHHHxx 4811 3426 1 3 1 11 11 811 811 4811 4811 22 23 BDAAAA UBFAAA OOOOxx 6838 3427 0 2 8 18 38 838 838 1838 6838 76 77 ADAAAA VBFAAA VVVVxx 787 3428 1 3 7 7 87 787 787 787 787 174 175 HEAAAA WBFAAA AAAAxx 7940 3429 0 0 0 0 40 940 1940 2940 7940 80 81 KTAAAA XBFAAA HHHHxx 336 3430 0 0 6 16 36 336 336 336 336 72 73 YMAAAA YBFAAA OOOOxx 9859 3431 1 3 9 19 59 859 1859 4859 9859 118 119 FPAAAA ZBFAAA VVVVxx 3864 3432 0 0 4 4 64 864 1864 3864 3864 128 129 QSAAAA ACFAAA AAAAxx 7162 3433 0 2 2 2 62 162 1162 2162 7162 124 125 MPAAAA BCFAAA HHHHxx 2071 3434 1 3 1 11 71 71 71 2071 2071 142 143 RBAAAA CCFAAA OOOOxx 7469 3435 1 1 9 9 69 469 1469 2469 7469 138 139 HBAAAA DCFAAA VVVVxx 2917 3436 1 1 7 17 17 917 917 2917 2917 34 35 FIAAAA ECFAAA AAAAxx 7486 3437 0 2 6 6 86 486 1486 2486 7486 172 173 YBAAAA FCFAAA HHHHxx 3355 3438 1 3 5 15 55 355 1355 3355 3355 110 111 BZAAAA GCFAAA OOOOxx 6998 3439 0 2 8 18 98 998 998 1998 6998 196 197 EJAAAA HCFAAA VVVVxx 5498 3440 0 2 8 18 98 498 1498 498 5498 196 197 MDAAAA ICFAAA AAAAxx 5113 3441 1 1 3 13 13 113 1113 113 5113 26 27 ROAAAA JCFAAA HHHHxx 2846 3442 0 2 6 6 46 846 846 2846 2846 92 93 MFAAAA KCFAAA OOOOxx 6834 3443 0 2 4 14 34 834 834 1834 6834 68 69 WCAAAA LCFAAA VVVVxx 8925 3444 1 1 5 5 25 925 925 3925 8925 50 51 HFAAAA MCFAAA AAAAxx 2757 3445 1 1 7 17 57 757 757 2757 2757 114 115 BCAAAA NCFAAA HHHHxx 2775 3446 1 3 5 15 75 775 775 2775 2775 150 151 TCAAAA OCFAAA OOOOxx 6182 3447 0 2 2 2 82 182 182 1182 6182 164 165 UDAAAA PCFAAA VVVVxx 4488 3448 0 0 8 8 88 488 488 4488 4488 176 177 QQAAAA QCFAAA AAAAxx 8523 3449 1 3 3 3 23 523 523 3523 8523 46 47 VPAAAA RCFAAA HHHHxx 52 3450 0 0 2 12 52 52 52 52 52 104 105 ACAAAA SCFAAA OOOOxx 7251 3451 1 3 1 11 51 251 1251 2251 7251 102 103 XSAAAA TCFAAA VVVVxx 6130 3452 0 2 0 10 30 130 130 1130 6130 60 61 UBAAAA UCFAAA AAAAxx 205 3453 1 1 5 5 5 205 205 205 205 10 11 XHAAAA VCFAAA HHHHxx 1186 3454 0 2 6 6 86 186 1186 1186 1186 172 173 QTAAAA WCFAAA OOOOxx 1738 3455 0 2 8 18 38 738 1738 1738 1738 76 77 WOAAAA XCFAAA VVVVxx 9485 3456 1 1 5 5 85 485 1485 4485 9485 170 171 VAAAAA YCFAAA AAAAxx 4235 3457 1 3 5 15 35 235 235 4235 4235 70 71 XGAAAA ZCFAAA HHHHxx 7891 3458 1 3 1 11 91 891 1891 2891 7891 182 183 NRAAAA ADFAAA OOOOxx 4960 3459 0 0 0 0 60 960 960 4960 4960 120 121 UIAAAA BDFAAA VVVVxx 8911 3460 1 3 1 11 11 911 911 3911 8911 22 23 TEAAAA CDFAAA AAAAxx 1219 3461 1 3 9 19 19 219 1219 1219 1219 38 39 XUAAAA DDFAAA HHHHxx 9652 3462 0 0 2 12 52 652 1652 4652 9652 104 105 GHAAAA EDFAAA OOOOxx 9715 3463 1 3 5 15 15 715 1715 4715 9715 30 31 RJAAAA FDFAAA VVVVxx 6629 3464 1 1 9 9 29 629 629 1629 6629 58 59 ZUAAAA GDFAAA AAAAxx 700 3465 0 0 0 0 0 700 700 700 700 0 1 YAAAAA HDFAAA HHHHxx 9819 3466 1 3 9 19 19 819 1819 4819 9819 38 39 RNAAAA IDFAAA OOOOxx 5188 3467 0 0 8 8 88 188 1188 188 5188 176 177 ORAAAA JDFAAA VVVVxx 5367 3468 1 3 7 7 67 367 1367 367 5367 134 135 LYAAAA KDFAAA AAAAxx 6447 3469 1 3 7 7 47 447 447 1447 6447 94 95 ZNAAAA LDFAAA HHHHxx 720 3470 0 0 0 0 20 720 720 720 720 40 41 SBAAAA MDFAAA OOOOxx 9157 3471 1 1 7 17 57 157 1157 4157 9157 114 115 FOAAAA NDFAAA VVVVxx 1082 3472 0 2 2 2 82 82 1082 1082 1082 164 165 QPAAAA ODFAAA AAAAxx 3179 3473 1 3 9 19 79 179 1179 3179 3179 158 159 HSAAAA PDFAAA HHHHxx 4818 3474 0 2 8 18 18 818 818 4818 4818 36 37 IDAAAA QDFAAA OOOOxx 7607 3475 1 3 7 7 7 607 1607 2607 7607 14 15 PGAAAA RDFAAA VVVVxx 2352 3476 0 0 2 12 52 352 352 2352 2352 104 105 MMAAAA SDFAAA AAAAxx 1170 3477 0 2 0 10 70 170 1170 1170 1170 140 141 ATAAAA TDFAAA HHHHxx 4269 3478 1 1 9 9 69 269 269 4269 4269 138 139 FIAAAA UDFAAA OOOOxx 8767 3479 1 3 7 7 67 767 767 3767 8767 134 135 FZAAAA VDFAAA VVVVxx 3984 3480 0 0 4 4 84 984 1984 3984 3984 168 169 GXAAAA WDFAAA AAAAxx 3190 3481 0 2 0 10 90 190 1190 3190 3190 180 181 SSAAAA XDFAAA HHHHxx 7456 3482 0 0 6 16 56 456 1456 2456 7456 112 113 UAAAAA YDFAAA OOOOxx 4348 3483 0 0 8 8 48 348 348 4348 4348 96 97 GLAAAA ZDFAAA VVVVxx 3150 3484 0 2 0 10 50 150 1150 3150 3150 100 101 ERAAAA AEFAAA AAAAxx 8780 3485 0 0 0 0 80 780 780 3780 8780 160 161 SZAAAA BEFAAA HHHHxx 2553 3486 1 1 3 13 53 553 553 2553 2553 106 107 FUAAAA CEFAAA OOOOxx 7526 3487 0 2 6 6 26 526 1526 2526 7526 52 53 MDAAAA DEFAAA VVVVxx 2031 3488 1 3 1 11 31 31 31 2031 2031 62 63 DAAAAA EEFAAA AAAAxx 8793 3489 1 1 3 13 93 793 793 3793 8793 186 187 FAAAAA FEFAAA HHHHxx 1122 3490 0 2 2 2 22 122 1122 1122 1122 44 45 ERAAAA GEFAAA OOOOxx 1855 3491 1 3 5 15 55 855 1855 1855 1855 110 111 JTAAAA HEFAAA VVVVxx 6613 3492 1 1 3 13 13 613 613 1613 6613 26 27 JUAAAA IEFAAA AAAAxx 3231 3493 1 3 1 11 31 231 1231 3231 3231 62 63 HUAAAA JEFAAA HHHHxx 9101 3494 1 1 1 1 1 101 1101 4101 9101 2 3 BMAAAA KEFAAA OOOOxx 4937 3495 1 1 7 17 37 937 937 4937 4937 74 75 XHAAAA LEFAAA VVVVxx 666 3496 0 2 6 6 66 666 666 666 666 132 133 QZAAAA MEFAAA AAAAxx 8943 3497 1 3 3 3 43 943 943 3943 8943 86 87 ZFAAAA NEFAAA HHHHxx 6164 3498 0 0 4 4 64 164 164 1164 6164 128 129 CDAAAA OEFAAA OOOOxx 1081 3499 1 1 1 1 81 81 1081 1081 1081 162 163 PPAAAA PEFAAA VVVVxx 210 3500 0 2 0 10 10 210 210 210 210 20 21 CIAAAA QEFAAA AAAAxx 6024 3501 0 0 4 4 24 24 24 1024 6024 48 49 SXAAAA REFAAA HHHHxx 5715 3502 1 3 5 15 15 715 1715 715 5715 30 31 VLAAAA SEFAAA OOOOxx 8938 3503 0 2 8 18 38 938 938 3938 8938 76 77 UFAAAA TEFAAA VVVVxx 1326 3504 0 2 6 6 26 326 1326 1326 1326 52 53 AZAAAA UEFAAA AAAAxx 7111 3505 1 3 1 11 11 111 1111 2111 7111 22 23 NNAAAA VEFAAA HHHHxx 757 3506 1 1 7 17 57 757 757 757 757 114 115 DDAAAA WEFAAA OOOOxx 8933 3507 1 1 3 13 33 933 933 3933 8933 66 67 PFAAAA XEFAAA VVVVxx 6495 3508 1 3 5 15 95 495 495 1495 6495 190 191 VPAAAA YEFAAA AAAAxx 3134 3509 0 2 4 14 34 134 1134 3134 3134 68 69 OQAAAA ZEFAAA HHHHxx 1304 3510 0 0 4 4 4 304 1304 1304 1304 8 9 EYAAAA AFFAAA OOOOxx 1835 3511 1 3 5 15 35 835 1835 1835 1835 70 71 PSAAAA BFFAAA VVVVxx 7275 3512 1 3 5 15 75 275 1275 2275 7275 150 151 VTAAAA CFFAAA AAAAxx 7337 3513 1 1 7 17 37 337 1337 2337 7337 74 75 FWAAAA DFFAAA HHHHxx 1282 3514 0 2 2 2 82 282 1282 1282 1282 164 165 IXAAAA EFFAAA OOOOxx 6566 3515 0 2 6 6 66 566 566 1566 6566 132 133 OSAAAA FFFAAA VVVVxx 3786 3516 0 2 6 6 86 786 1786 3786 3786 172 173 QPAAAA GFFAAA AAAAxx 5741 3517 1 1 1 1 41 741 1741 741 5741 82 83 VMAAAA HFFAAA HHHHxx 6076 3518 0 0 6 16 76 76 76 1076 6076 152 153 SZAAAA IFFAAA OOOOxx 9998 3519 0 2 8 18 98 998 1998 4998 9998 196 197 OUAAAA JFFAAA VVVVxx 6268 3520 0 0 8 8 68 268 268 1268 6268 136 137 CHAAAA KFFAAA AAAAxx 9647 3521 1 3 7 7 47 647 1647 4647 9647 94 95 BHAAAA LFFAAA HHHHxx 4877 3522 1 1 7 17 77 877 877 4877 4877 154 155 PFAAAA MFFAAA OOOOxx 2652 3523 0 0 2 12 52 652 652 2652 2652 104 105 AYAAAA NFFAAA VVVVxx 1247 3524 1 3 7 7 47 247 1247 1247 1247 94 95 ZVAAAA OFFAAA AAAAxx 2721 3525 1 1 1 1 21 721 721 2721 2721 42 43 RAAAAA PFFAAA HHHHxx 5968 3526 0 0 8 8 68 968 1968 968 5968 136 137 OVAAAA QFFAAA OOOOxx 9570 3527 0 2 0 10 70 570 1570 4570 9570 140 141 CEAAAA RFFAAA VVVVxx 6425 3528 1 1 5 5 25 425 425 1425 6425 50 51 DNAAAA SFFAAA AAAAxx 5451 3529 1 3 1 11 51 451 1451 451 5451 102 103 RBAAAA TFFAAA HHHHxx 5668 3530 0 0 8 8 68 668 1668 668 5668 136 137 AKAAAA UFFAAA OOOOxx 9493 3531 1 1 3 13 93 493 1493 4493 9493 186 187 DBAAAA VFFAAA VVVVxx 7973 3532 1 1 3 13 73 973 1973 2973 7973 146 147 RUAAAA WFFAAA AAAAxx 8250 3533 0 2 0 10 50 250 250 3250 8250 100 101 IFAAAA XFFAAA HHHHxx 82 3534 0 2 2 2 82 82 82 82 82 164 165 EDAAAA YFFAAA OOOOxx 6258 3535 0 2 8 18 58 258 258 1258 6258 116 117 SGAAAA ZFFAAA VVVVxx 9978 3536 0 2 8 18 78 978 1978 4978 9978 156 157 UTAAAA AGFAAA AAAAxx 6930 3537 0 2 0 10 30 930 930 1930 6930 60 61 OGAAAA BGFAAA HHHHxx 3746 3538 0 2 6 6 46 746 1746 3746 3746 92 93 COAAAA CGFAAA OOOOxx 7065 3539 1 1 5 5 65 65 1065 2065 7065 130 131 TLAAAA DGFAAA VVVVxx 4281 3540 1 1 1 1 81 281 281 4281 4281 162 163 RIAAAA EGFAAA AAAAxx 4367 3541 1 3 7 7 67 367 367 4367 4367 134 135 ZLAAAA FGFAAA HHHHxx 9526 3542 0 2 6 6 26 526 1526 4526 9526 52 53 KCAAAA GGFAAA OOOOxx 5880 3543 0 0 0 0 80 880 1880 880 5880 160 161 ESAAAA HGFAAA VVVVxx 8480 3544 0 0 0 0 80 480 480 3480 8480 160 161 EOAAAA IGFAAA AAAAxx 2476 3545 0 0 6 16 76 476 476 2476 2476 152 153 GRAAAA JGFAAA HHHHxx 9074 3546 0 2 4 14 74 74 1074 4074 9074 148 149 ALAAAA KGFAAA OOOOxx 4830 3547 0 2 0 10 30 830 830 4830 4830 60 61 UDAAAA LGFAAA VVVVxx 3207 3548 1 3 7 7 7 207 1207 3207 3207 14 15 JTAAAA MGFAAA AAAAxx 7894 3549 0 2 4 14 94 894 1894 2894 7894 188 189 QRAAAA NGFAAA HHHHxx 3860 3550 0 0 0 0 60 860 1860 3860 3860 120 121 MSAAAA OGFAAA OOOOxx 5293 3551 1 1 3 13 93 293 1293 293 5293 186 187 PVAAAA PGFAAA VVVVxx 6895 3552 1 3 5 15 95 895 895 1895 6895 190 191 FFAAAA QGFAAA AAAAxx 9908 3553 0 0 8 8 8 908 1908 4908 9908 16 17 CRAAAA RGFAAA HHHHxx 9247 3554 1 3 7 7 47 247 1247 4247 9247 94 95 RRAAAA SGFAAA OOOOxx 8110 3555 0 2 0 10 10 110 110 3110 8110 20 21 YZAAAA TGFAAA VVVVxx 4716 3556 0 0 6 16 16 716 716 4716 4716 32 33 KZAAAA UGFAAA AAAAxx 4979 3557 1 3 9 19 79 979 979 4979 4979 158 159 NJAAAA VGFAAA HHHHxx 5280 3558 0 0 0 0 80 280 1280 280 5280 160 161 CVAAAA WGFAAA OOOOxx 8326 3559 0 2 6 6 26 326 326 3326 8326 52 53 GIAAAA XGFAAA VVVVxx 5572 3560 0 0 2 12 72 572 1572 572 5572 144 145 IGAAAA YGFAAA AAAAxx 4665 3561 1 1 5 5 65 665 665 4665 4665 130 131 LXAAAA ZGFAAA HHHHxx 3665 3562 1 1 5 5 65 665 1665 3665 3665 130 131 ZKAAAA AHFAAA OOOOxx 6744 3563 0 0 4 4 44 744 744 1744 6744 88 89 KZAAAA BHFAAA VVVVxx 1897 3564 1 1 7 17 97 897 1897 1897 1897 194 195 ZUAAAA CHFAAA AAAAxx 1220 3565 0 0 0 0 20 220 1220 1220 1220 40 41 YUAAAA DHFAAA HHHHxx 2614 3566 0 2 4 14 14 614 614 2614 2614 28 29 OWAAAA EHFAAA OOOOxx 8509 3567 1 1 9 9 9 509 509 3509 8509 18 19 HPAAAA FHFAAA VVVVxx 8521 3568 1 1 1 1 21 521 521 3521 8521 42 43 TPAAAA GHFAAA AAAAxx 4121 3569 1 1 1 1 21 121 121 4121 4121 42 43 NCAAAA HHFAAA HHHHxx 9663 3570 1 3 3 3 63 663 1663 4663 9663 126 127 RHAAAA IHFAAA OOOOxx 2346 3571 0 2 6 6 46 346 346 2346 2346 92 93 GMAAAA JHFAAA VVVVxx 3370 3572 0 2 0 10 70 370 1370 3370 3370 140 141 QZAAAA KHFAAA AAAAxx 1498 3573 0 2 8 18 98 498 1498 1498 1498 196 197 QFAAAA LHFAAA HHHHxx 7422 3574 0 2 2 2 22 422 1422 2422 7422 44 45 MZAAAA MHFAAA OOOOxx 3472 3575 0 0 2 12 72 472 1472 3472 3472 144 145 ODAAAA NHFAAA VVVVxx 4126 3576 0 2 6 6 26 126 126 4126 4126 52 53 SCAAAA OHFAAA AAAAxx 4494 3577 0 2 4 14 94 494 494 4494 4494 188 189 WQAAAA PHFAAA HHHHxx 6323 3578 1 3 3 3 23 323 323 1323 6323 46 47 FJAAAA QHFAAA OOOOxx 2823 3579 1 3 3 3 23 823 823 2823 2823 46 47 PEAAAA RHFAAA VVVVxx 8596 3580 0 0 6 16 96 596 596 3596 8596 192 193 QSAAAA SHFAAA AAAAxx 6642 3581 0 2 2 2 42 642 642 1642 6642 84 85 MVAAAA THFAAA HHHHxx 9276 3582 0 0 6 16 76 276 1276 4276 9276 152 153 USAAAA UHFAAA OOOOxx 4148 3583 0 0 8 8 48 148 148 4148 4148 96 97 ODAAAA VHFAAA VVVVxx 9770 3584 0 2 0 10 70 770 1770 4770 9770 140 141 ULAAAA WHFAAA AAAAxx 9812 3585 0 0 2 12 12 812 1812 4812 9812 24 25 KNAAAA XHFAAA HHHHxx 4419 3586 1 3 9 19 19 419 419 4419 4419 38 39 ZNAAAA YHFAAA OOOOxx 3802 3587 0 2 2 2 2 802 1802 3802 3802 4 5 GQAAAA ZHFAAA VVVVxx 3210 3588 0 2 0 10 10 210 1210 3210 3210 20 21 MTAAAA AIFAAA AAAAxx 6794 3589 0 2 4 14 94 794 794 1794 6794 188 189 IBAAAA BIFAAA HHHHxx 242 3590 0 2 2 2 42 242 242 242 242 84 85 IJAAAA CIFAAA OOOOxx 962 3591 0 2 2 2 62 962 962 962 962 124 125 ALAAAA DIFAAA VVVVxx 7151 3592 1 3 1 11 51 151 1151 2151 7151 102 103 BPAAAA EIFAAA AAAAxx 9440 3593 0 0 0 0 40 440 1440 4440 9440 80 81 CZAAAA FIFAAA HHHHxx 721 3594 1 1 1 1 21 721 721 721 721 42 43 TBAAAA GIFAAA OOOOxx 2119 3595 1 3 9 19 19 119 119 2119 2119 38 39 NDAAAA HIFAAA VVVVxx 9883 3596 1 3 3 3 83 883 1883 4883 9883 166 167 DQAAAA IIFAAA AAAAxx 5071 3597 1 3 1 11 71 71 1071 71 5071 142 143 BNAAAA JIFAAA HHHHxx 8239 3598 1 3 9 19 39 239 239 3239 8239 78 79 XEAAAA KIFAAA OOOOxx 7451 3599 1 3 1 11 51 451 1451 2451 7451 102 103 PAAAAA LIFAAA VVVVxx 9517 3600 1 1 7 17 17 517 1517 4517 9517 34 35 BCAAAA MIFAAA AAAAxx 9180 3601 0 0 0 0 80 180 1180 4180 9180 160 161 CPAAAA NIFAAA HHHHxx 9327 3602 1 3 7 7 27 327 1327 4327 9327 54 55 TUAAAA OIFAAA OOOOxx 5462 3603 0 2 2 2 62 462 1462 462 5462 124 125 CCAAAA PIFAAA VVVVxx 8306 3604 0 2 6 6 6 306 306 3306 8306 12 13 MHAAAA QIFAAA AAAAxx 6234 3605 0 2 4 14 34 234 234 1234 6234 68 69 UFAAAA RIFAAA HHHHxx 8771 3606 1 3 1 11 71 771 771 3771 8771 142 143 JZAAAA SIFAAA OOOOxx 5853 3607 1 1 3 13 53 853 1853 853 5853 106 107 DRAAAA TIFAAA VVVVxx 8373 3608 1 1 3 13 73 373 373 3373 8373 146 147 BKAAAA UIFAAA AAAAxx 5017 3609 1 1 7 17 17 17 1017 17 5017 34 35 ZKAAAA VIFAAA HHHHxx 8025 3610 1 1 5 5 25 25 25 3025 8025 50 51 RWAAAA WIFAAA OOOOxx 2526 3611 0 2 6 6 26 526 526 2526 2526 52 53 ETAAAA XIFAAA VVVVxx 7419 3612 1 3 9 19 19 419 1419 2419 7419 38 39 JZAAAA YIFAAA AAAAxx 4572 3613 0 0 2 12 72 572 572 4572 4572 144 145 WTAAAA ZIFAAA HHHHxx 7744 3614 0 0 4 4 44 744 1744 2744 7744 88 89 WLAAAA AJFAAA OOOOxx 8825 3615 1 1 5 5 25 825 825 3825 8825 50 51 LBAAAA BJFAAA VVVVxx 6067 3616 1 3 7 7 67 67 67 1067 6067 134 135 JZAAAA CJFAAA AAAAxx 3291 3617 1 3 1 11 91 291 1291 3291 3291 182 183 PWAAAA DJFAAA HHHHxx 7115 3618 1 3 5 15 15 115 1115 2115 7115 30 31 RNAAAA EJFAAA OOOOxx 2626 3619 0 2 6 6 26 626 626 2626 2626 52 53 AXAAAA FJFAAA VVVVxx 4109 3620 1 1 9 9 9 109 109 4109 4109 18 19 BCAAAA GJFAAA AAAAxx 4056 3621 0 0 6 16 56 56 56 4056 4056 112 113 AAAAAA HJFAAA HHHHxx 6811 3622 1 3 1 11 11 811 811 1811 6811 22 23 ZBAAAA IJFAAA OOOOxx 680 3623 0 0 0 0 80 680 680 680 680 160 161 EAAAAA JJFAAA VVVVxx 474 3624 0 2 4 14 74 474 474 474 474 148 149 GSAAAA KJFAAA AAAAxx 9294 3625 0 2 4 14 94 294 1294 4294 9294 188 189 MTAAAA LJFAAA HHHHxx 7555 3626 1 3 5 15 55 555 1555 2555 7555 110 111 PEAAAA MJFAAA OOOOxx 8076 3627 0 0 6 16 76 76 76 3076 8076 152 153 QYAAAA NJFAAA VVVVxx 3840 3628 0 0 0 0 40 840 1840 3840 3840 80 81 SRAAAA OJFAAA AAAAxx 5955 3629 1 3 5 15 55 955 1955 955 5955 110 111 BVAAAA PJFAAA HHHHxx 994 3630 0 2 4 14 94 994 994 994 994 188 189 GMAAAA QJFAAA OOOOxx 2089 3631 1 1 9 9 89 89 89 2089 2089 178 179 JCAAAA RJFAAA VVVVxx 869 3632 1 1 9 9 69 869 869 869 869 138 139 LHAAAA SJFAAA AAAAxx 1223 3633 1 3 3 3 23 223 1223 1223 1223 46 47 BVAAAA TJFAAA HHHHxx 1514 3634 0 2 4 14 14 514 1514 1514 1514 28 29 GGAAAA UJFAAA OOOOxx 4891 3635 1 3 1 11 91 891 891 4891 4891 182 183 DGAAAA VJFAAA VVVVxx 4190 3636 0 2 0 10 90 190 190 4190 4190 180 181 EFAAAA WJFAAA AAAAxx 4377 3637 1 1 7 17 77 377 377 4377 4377 154 155 JMAAAA XJFAAA HHHHxx 9195 3638 1 3 5 15 95 195 1195 4195 9195 190 191 RPAAAA YJFAAA OOOOxx 3827 3639 1 3 7 7 27 827 1827 3827 3827 54 55 FRAAAA ZJFAAA VVVVxx 7386 3640 0 2 6 6 86 386 1386 2386 7386 172 173 CYAAAA AKFAAA AAAAxx 6665 3641 1 1 5 5 65 665 665 1665 6665 130 131 JWAAAA BKFAAA HHHHxx 7514 3642 0 2 4 14 14 514 1514 2514 7514 28 29 ADAAAA CKFAAA OOOOxx 6431 3643 1 3 1 11 31 431 431 1431 6431 62 63 JNAAAA DKFAAA VVVVxx 3251 3644 1 3 1 11 51 251 1251 3251 3251 102 103 BVAAAA EKFAAA AAAAxx 8439 3645 1 3 9 19 39 439 439 3439 8439 78 79 PMAAAA FKFAAA HHHHxx 831 3646 1 3 1 11 31 831 831 831 831 62 63 ZFAAAA GKFAAA OOOOxx 8485 3647 1 1 5 5 85 485 485 3485 8485 170 171 JOAAAA HKFAAA VVVVxx 7314 3648 0 2 4 14 14 314 1314 2314 7314 28 29 IVAAAA IKFAAA AAAAxx 3044 3649 0 0 4 4 44 44 1044 3044 3044 88 89 CNAAAA JKFAAA HHHHxx 4283 3650 1 3 3 3 83 283 283 4283 4283 166 167 TIAAAA KKFAAA OOOOxx 298 3651 0 2 8 18 98 298 298 298 298 196 197 MLAAAA LKFAAA VVVVxx 7114 3652 0 2 4 14 14 114 1114 2114 7114 28 29 QNAAAA MKFAAA AAAAxx 9664 3653 0 0 4 4 64 664 1664 4664 9664 128 129 SHAAAA NKFAAA HHHHxx 5315 3654 1 3 5 15 15 315 1315 315 5315 30 31 LWAAAA OKFAAA OOOOxx 2164 3655 0 0 4 4 64 164 164 2164 2164 128 129 GFAAAA PKFAAA VVVVxx 3390 3656 0 2 0 10 90 390 1390 3390 3390 180 181 KAAAAA QKFAAA AAAAxx 836 3657 0 0 6 16 36 836 836 836 836 72 73 EGAAAA RKFAAA HHHHxx 3316 3658 0 0 6 16 16 316 1316 3316 3316 32 33 OXAAAA SKFAAA OOOOxx 1284 3659 0 0 4 4 84 284 1284 1284 1284 168 169 KXAAAA TKFAAA VVVVxx 2497 3660 1 1 7 17 97 497 497 2497 2497 194 195 BSAAAA UKFAAA AAAAxx 1374 3661 0 2 4 14 74 374 1374 1374 1374 148 149 WAAAAA VKFAAA HHHHxx 9525 3662 1 1 5 5 25 525 1525 4525 9525 50 51 JCAAAA WKFAAA OOOOxx 2911 3663 1 3 1 11 11 911 911 2911 2911 22 23 ZHAAAA XKFAAA VVVVxx 9686 3664 0 2 6 6 86 686 1686 4686 9686 172 173 OIAAAA YKFAAA AAAAxx 584 3665 0 0 4 4 84 584 584 584 584 168 169 MWAAAA ZKFAAA HHHHxx 5653 3666 1 1 3 13 53 653 1653 653 5653 106 107 LJAAAA ALFAAA OOOOxx 4986 3667 0 2 6 6 86 986 986 4986 4986 172 173 UJAAAA BLFAAA VVVVxx 6049 3668 1 1 9 9 49 49 49 1049 6049 98 99 RYAAAA CLFAAA AAAAxx 9891 3669 1 3 1 11 91 891 1891 4891 9891 182 183 LQAAAA DLFAAA HHHHxx 8809 3670 1 1 9 9 9 809 809 3809 8809 18 19 VAAAAA ELFAAA OOOOxx 8598 3671 0 2 8 18 98 598 598 3598 8598 196 197 SSAAAA FLFAAA VVVVxx 2573 3672 1 1 3 13 73 573 573 2573 2573 146 147 ZUAAAA GLFAAA AAAAxx 6864 3673 0 0 4 4 64 864 864 1864 6864 128 129 AEAAAA HLFAAA HHHHxx 7932 3674 0 0 2 12 32 932 1932 2932 7932 64 65 CTAAAA ILFAAA OOOOxx 6605 3675 1 1 5 5 5 605 605 1605 6605 10 11 BUAAAA JLFAAA VVVVxx 9500 3676 0 0 0 0 0 500 1500 4500 9500 0 1 KBAAAA KLFAAA AAAAxx 8742 3677 0 2 2 2 42 742 742 3742 8742 84 85 GYAAAA LLFAAA HHHHxx 9815 3678 1 3 5 15 15 815 1815 4815 9815 30 31 NNAAAA MLFAAA OOOOxx 3319 3679 1 3 9 19 19 319 1319 3319 3319 38 39 RXAAAA NLFAAA VVVVxx 184 3680 0 0 4 4 84 184 184 184 184 168 169 CHAAAA OLFAAA AAAAxx 8886 3681 0 2 6 6 86 886 886 3886 8886 172 173 UDAAAA PLFAAA HHHHxx 7050 3682 0 2 0 10 50 50 1050 2050 7050 100 101 ELAAAA QLFAAA OOOOxx 9781 3683 1 1 1 1 81 781 1781 4781 9781 162 163 FMAAAA RLFAAA VVVVxx 2443 3684 1 3 3 3 43 443 443 2443 2443 86 87 ZPAAAA SLFAAA AAAAxx 1160 3685 0 0 0 0 60 160 1160 1160 1160 120 121 QSAAAA TLFAAA HHHHxx 4600 3686 0 0 0 0 0 600 600 4600 4600 0 1 YUAAAA ULFAAA OOOOxx 813 3687 1 1 3 13 13 813 813 813 813 26 27 HFAAAA VLFAAA VVVVxx 5078 3688 0 2 8 18 78 78 1078 78 5078 156 157 INAAAA WLFAAA AAAAxx 9008 3689 0 0 8 8 8 8 1008 4008 9008 16 17 MIAAAA XLFAAA HHHHxx 9016 3690 0 0 6 16 16 16 1016 4016 9016 32 33 UIAAAA YLFAAA OOOOxx 2747 3691 1 3 7 7 47 747 747 2747 2747 94 95 RBAAAA ZLFAAA VVVVxx 3106 3692 0 2 6 6 6 106 1106 3106 3106 12 13 MPAAAA AMFAAA AAAAxx 8235 3693 1 3 5 15 35 235 235 3235 8235 70 71 TEAAAA BMFAAA HHHHxx 5582 3694 0 2 2 2 82 582 1582 582 5582 164 165 SGAAAA CMFAAA OOOOxx 4334 3695 0 2 4 14 34 334 334 4334 4334 68 69 SKAAAA DMFAAA VVVVxx 1612 3696 0 0 2 12 12 612 1612 1612 1612 24 25 AKAAAA EMFAAA AAAAxx 5650 3697 0 2 0 10 50 650 1650 650 5650 100 101 IJAAAA FMFAAA HHHHxx 6086 3698 0 2 6 6 86 86 86 1086 6086 172 173 CAAAAA GMFAAA OOOOxx 9667 3699 1 3 7 7 67 667 1667 4667 9667 134 135 VHAAAA HMFAAA VVVVxx 4215 3700 1 3 5 15 15 215 215 4215 4215 30 31 DGAAAA IMFAAA AAAAxx 8553 3701 1 1 3 13 53 553 553 3553 8553 106 107 ZQAAAA JMFAAA HHHHxx 9066 3702 0 2 6 6 66 66 1066 4066 9066 132 133 SKAAAA KMFAAA OOOOxx 1092 3703 0 0 2 12 92 92 1092 1092 1092 184 185 AQAAAA LMFAAA VVVVxx 2848 3704 0 0 8 8 48 848 848 2848 2848 96 97 OFAAAA MMFAAA AAAAxx 2765 3705 1 1 5 5 65 765 765 2765 2765 130 131 JCAAAA NMFAAA HHHHxx 6513 3706 1 1 3 13 13 513 513 1513 6513 26 27 NQAAAA OMFAAA OOOOxx 6541 3707 1 1 1 1 41 541 541 1541 6541 82 83 PRAAAA PMFAAA VVVVxx 9617 3708 1 1 7 17 17 617 1617 4617 9617 34 35 XFAAAA QMFAAA AAAAxx 5870 3709 0 2 0 10 70 870 1870 870 5870 140 141 URAAAA RMFAAA HHHHxx 8811 3710 1 3 1 11 11 811 811 3811 8811 22 23 XAAAAA SMFAAA OOOOxx 4529 3711 1 1 9 9 29 529 529 4529 4529 58 59 FSAAAA TMFAAA VVVVxx 161 3712 1 1 1 1 61 161 161 161 161 122 123 FGAAAA UMFAAA AAAAxx 641 3713 1 1 1 1 41 641 641 641 641 82 83 RYAAAA VMFAAA HHHHxx 4767 3714 1 3 7 7 67 767 767 4767 4767 134 135 JBAAAA WMFAAA OOOOxx 6293 3715 1 1 3 13 93 293 293 1293 6293 186 187 BIAAAA XMFAAA VVVVxx 3816 3716 0 0 6 16 16 816 1816 3816 3816 32 33 UQAAAA YMFAAA AAAAxx 4748 3717 0 0 8 8 48 748 748 4748 4748 96 97 QAAAAA ZMFAAA HHHHxx 9924 3718 0 0 4 4 24 924 1924 4924 9924 48 49 SRAAAA ANFAAA OOOOxx 6716 3719 0 0 6 16 16 716 716 1716 6716 32 33 IYAAAA BNFAAA VVVVxx 8828 3720 0 0 8 8 28 828 828 3828 8828 56 57 OBAAAA CNFAAA AAAAxx 4967 3721 1 3 7 7 67 967 967 4967 4967 134 135 BJAAAA DNFAAA HHHHxx 9680 3722 0 0 0 0 80 680 1680 4680 9680 160 161 IIAAAA ENFAAA OOOOxx 2784 3723 0 0 4 4 84 784 784 2784 2784 168 169 CDAAAA FNFAAA VVVVxx 2882 3724 0 2 2 2 82 882 882 2882 2882 164 165 WGAAAA GNFAAA AAAAxx 3641 3725 1 1 1 1 41 641 1641 3641 3641 82 83 BKAAAA HNFAAA HHHHxx 5537 3726 1 1 7 17 37 537 1537 537 5537 74 75 ZEAAAA INFAAA OOOOxx 820 3727 0 0 0 0 20 820 820 820 820 40 41 OFAAAA JNFAAA VVVVxx 5847 3728 1 3 7 7 47 847 1847 847 5847 94 95 XQAAAA KNFAAA AAAAxx 566 3729 0 2 6 6 66 566 566 566 566 132 133 UVAAAA LNFAAA HHHHxx 2246 3730 0 2 6 6 46 246 246 2246 2246 92 93 KIAAAA MNFAAA OOOOxx 6680 3731 0 0 0 0 80 680 680 1680 6680 160 161 YWAAAA NNFAAA VVVVxx 2014 3732 0 2 4 14 14 14 14 2014 2014 28 29 MZAAAA ONFAAA AAAAxx 8355 3733 1 3 5 15 55 355 355 3355 8355 110 111 JJAAAA PNFAAA HHHHxx 1610 3734 0 2 0 10 10 610 1610 1610 1610 20 21 YJAAAA QNFAAA OOOOxx 9719 3735 1 3 9 19 19 719 1719 4719 9719 38 39 VJAAAA RNFAAA VVVVxx 8498 3736 0 2 8 18 98 498 498 3498 8498 196 197 WOAAAA SNFAAA AAAAxx 5883 3737 1 3 3 3 83 883 1883 883 5883 166 167 HSAAAA TNFAAA HHHHxx 7380 3738 0 0 0 0 80 380 1380 2380 7380 160 161 WXAAAA UNFAAA OOOOxx 8865 3739 1 1 5 5 65 865 865 3865 8865 130 131 ZCAAAA VNFAAA VVVVxx 4743 3740 1 3 3 3 43 743 743 4743 4743 86 87 LAAAAA WNFAAA AAAAxx 5086 3741 0 2 6 6 86 86 1086 86 5086 172 173 QNAAAA XNFAAA HHHHxx 2739 3742 1 3 9 19 39 739 739 2739 2739 78 79 JBAAAA YNFAAA OOOOxx 9375 3743 1 3 5 15 75 375 1375 4375 9375 150 151 PWAAAA ZNFAAA VVVVxx 7876 3744 0 0 6 16 76 876 1876 2876 7876 152 153 YQAAAA AOFAAA AAAAxx 453 3745 1 1 3 13 53 453 453 453 453 106 107 LRAAAA BOFAAA HHHHxx 6987 3746 1 3 7 7 87 987 987 1987 6987 174 175 TIAAAA COFAAA OOOOxx 2860 3747 0 0 0 0 60 860 860 2860 2860 120 121 AGAAAA DOFAAA VVVVxx 8372 3748 0 0 2 12 72 372 372 3372 8372 144 145 AKAAAA EOFAAA AAAAxx 2048 3749 0 0 8 8 48 48 48 2048 2048 96 97 UAAAAA FOFAAA HHHHxx 9231 3750 1 3 1 11 31 231 1231 4231 9231 62 63 BRAAAA GOFAAA OOOOxx 634 3751 0 2 4 14 34 634 634 634 634 68 69 KYAAAA HOFAAA VVVVxx 3998 3752 0 2 8 18 98 998 1998 3998 3998 196 197 UXAAAA IOFAAA AAAAxx 4728 3753 0 0 8 8 28 728 728 4728 4728 56 57 WZAAAA JOFAAA HHHHxx 579 3754 1 3 9 19 79 579 579 579 579 158 159 HWAAAA KOFAAA OOOOxx 815 3755 1 3 5 15 15 815 815 815 815 30 31 JFAAAA LOFAAA VVVVxx 1009 3756 1 1 9 9 9 9 1009 1009 1009 18 19 VMAAAA MOFAAA AAAAxx 6596 3757 0 0 6 16 96 596 596 1596 6596 192 193 STAAAA NOFAAA HHHHxx 2793 3758 1 1 3 13 93 793 793 2793 2793 186 187 LDAAAA OOFAAA OOOOxx 9589 3759 1 1 9 9 89 589 1589 4589 9589 178 179 VEAAAA POFAAA VVVVxx 2794 3760 0 2 4 14 94 794 794 2794 2794 188 189 MDAAAA QOFAAA AAAAxx 2551 3761 1 3 1 11 51 551 551 2551 2551 102 103 DUAAAA ROFAAA HHHHxx 1588 3762 0 0 8 8 88 588 1588 1588 1588 176 177 CJAAAA SOFAAA OOOOxx 4443 3763 1 3 3 3 43 443 443 4443 4443 86 87 XOAAAA TOFAAA VVVVxx 5009 3764 1 1 9 9 9 9 1009 9 5009 18 19 RKAAAA UOFAAA AAAAxx 4287 3765 1 3 7 7 87 287 287 4287 4287 174 175 XIAAAA VOFAAA HHHHxx 2167 3766 1 3 7 7 67 167 167 2167 2167 134 135 JFAAAA WOFAAA OOOOxx 2290 3767 0 2 0 10 90 290 290 2290 2290 180 181 CKAAAA XOFAAA VVVVxx 7225 3768 1 1 5 5 25 225 1225 2225 7225 50 51 XRAAAA YOFAAA AAAAxx 8992 3769 0 0 2 12 92 992 992 3992 8992 184 185 WHAAAA ZOFAAA HHHHxx 1540 3770 0 0 0 0 40 540 1540 1540 1540 80 81 GHAAAA APFAAA OOOOxx 2029 3771 1 1 9 9 29 29 29 2029 2029 58 59 BAAAAA BPFAAA VVVVxx 2855 3772 1 3 5 15 55 855 855 2855 2855 110 111 VFAAAA CPFAAA AAAAxx 3534 3773 0 2 4 14 34 534 1534 3534 3534 68 69 YFAAAA DPFAAA HHHHxx 8078 3774 0 2 8 18 78 78 78 3078 8078 156 157 SYAAAA EPFAAA OOOOxx 9778 3775 0 2 8 18 78 778 1778 4778 9778 156 157 CMAAAA FPFAAA VVVVxx 3543 3776 1 3 3 3 43 543 1543 3543 3543 86 87 HGAAAA GPFAAA AAAAxx 4778 3777 0 2 8 18 78 778 778 4778 4778 156 157 UBAAAA HPFAAA HHHHxx 8931 3778 1 3 1 11 31 931 931 3931 8931 62 63 NFAAAA IPFAAA OOOOxx 557 3779 1 1 7 17 57 557 557 557 557 114 115 LVAAAA JPFAAA VVVVxx 5546 3780 0 2 6 6 46 546 1546 546 5546 92 93 IFAAAA KPFAAA AAAAxx 7527 3781 1 3 7 7 27 527 1527 2527 7527 54 55 NDAAAA LPFAAA HHHHxx 5000 3782 0 0 0 0 0 0 1000 0 5000 0 1 IKAAAA MPFAAA OOOOxx 7587 3783 1 3 7 7 87 587 1587 2587 7587 174 175 VFAAAA NPFAAA VVVVxx 3014 3784 0 2 4 14 14 14 1014 3014 3014 28 29 YLAAAA OPFAAA AAAAxx 5276 3785 0 0 6 16 76 276 1276 276 5276 152 153 YUAAAA PPFAAA HHHHxx 6457 3786 1 1 7 17 57 457 457 1457 6457 114 115 JOAAAA QPFAAA OOOOxx 389 3787 1 1 9 9 89 389 389 389 389 178 179 ZOAAAA RPFAAA VVVVxx 7104 3788 0 0 4 4 4 104 1104 2104 7104 8 9 GNAAAA SPFAAA AAAAxx 9995 3789 1 3 5 15 95 995 1995 4995 9995 190 191 LUAAAA TPFAAA HHHHxx 7368 3790 0 0 8 8 68 368 1368 2368 7368 136 137 KXAAAA UPFAAA OOOOxx 3258 3791 0 2 8 18 58 258 1258 3258 3258 116 117 IVAAAA VPFAAA VVVVxx 9208 3792 0 0 8 8 8 208 1208 4208 9208 16 17 EQAAAA WPFAAA AAAAxx 2396 3793 0 0 6 16 96 396 396 2396 2396 192 193 EOAAAA XPFAAA HHHHxx 1715 3794 1 3 5 15 15 715 1715 1715 1715 30 31 ZNAAAA YPFAAA OOOOxx 1240 3795 0 0 0 0 40 240 1240 1240 1240 80 81 SVAAAA ZPFAAA VVVVxx 1952 3796 0 0 2 12 52 952 1952 1952 1952 104 105 CXAAAA AQFAAA AAAAxx 4403 3797 1 3 3 3 3 403 403 4403 4403 6 7 JNAAAA BQFAAA HHHHxx 6333 3798 1 1 3 13 33 333 333 1333 6333 66 67 PJAAAA CQFAAA OOOOxx 2492 3799 0 0 2 12 92 492 492 2492 2492 184 185 WRAAAA DQFAAA VVVVxx 6543 3800 1 3 3 3 43 543 543 1543 6543 86 87 RRAAAA EQFAAA AAAAxx 5548 3801 0 0 8 8 48 548 1548 548 5548 96 97 KFAAAA FQFAAA HHHHxx 3458 3802 0 2 8 18 58 458 1458 3458 3458 116 117 ADAAAA GQFAAA OOOOxx 2588 3803 0 0 8 8 88 588 588 2588 2588 176 177 OVAAAA HQFAAA VVVVxx 1364 3804 0 0 4 4 64 364 1364 1364 1364 128 129 MAAAAA IQFAAA AAAAxx 9856 3805 0 0 6 16 56 856 1856 4856 9856 112 113 CPAAAA JQFAAA HHHHxx 4964 3806 0 0 4 4 64 964 964 4964 4964 128 129 YIAAAA KQFAAA OOOOxx 773 3807 1 1 3 13 73 773 773 773 773 146 147 TDAAAA LQFAAA VVVVxx 6402 3808 0 2 2 2 2 402 402 1402 6402 4 5 GMAAAA MQFAAA AAAAxx 7213 3809 1 1 3 13 13 213 1213 2213 7213 26 27 LRAAAA NQFAAA HHHHxx 3385 3810 1 1 5 5 85 385 1385 3385 3385 170 171 FAAAAA OQFAAA OOOOxx 6005 3811 1 1 5 5 5 5 5 1005 6005 10 11 ZWAAAA PQFAAA VVVVxx 9346 3812 0 2 6 6 46 346 1346 4346 9346 92 93 MVAAAA QQFAAA AAAAxx 1831 3813 1 3 1 11 31 831 1831 1831 1831 62 63 LSAAAA RQFAAA HHHHxx 5406 3814 0 2 6 6 6 406 1406 406 5406 12 13 YZAAAA SQFAAA OOOOxx 2154 3815 0 2 4 14 54 154 154 2154 2154 108 109 WEAAAA TQFAAA VVVVxx 3721 3816 1 1 1 1 21 721 1721 3721 3721 42 43 DNAAAA UQFAAA AAAAxx 2889 3817 1 1 9 9 89 889 889 2889 2889 178 179 DHAAAA VQFAAA HHHHxx 4410 3818 0 2 0 10 10 410 410 4410 4410 20 21 QNAAAA WQFAAA OOOOxx 7102 3819 0 2 2 2 2 102 1102 2102 7102 4 5 ENAAAA XQFAAA VVVVxx 4057 3820 1 1 7 17 57 57 57 4057 4057 114 115 BAAAAA YQFAAA AAAAxx 9780 3821 0 0 0 0 80 780 1780 4780 9780 160 161 EMAAAA ZQFAAA HHHHxx 9481 3822 1 1 1 1 81 481 1481 4481 9481 162 163 RAAAAA ARFAAA OOOOxx 2366 3823 0 2 6 6 66 366 366 2366 2366 132 133 ANAAAA BRFAAA VVVVxx 2708 3824 0 0 8 8 8 708 708 2708 2708 16 17 EAAAAA CRFAAA AAAAxx 7399 3825 1 3 9 19 99 399 1399 2399 7399 198 199 PYAAAA DRFAAA HHHHxx 5234 3826 0 2 4 14 34 234 1234 234 5234 68 69 ITAAAA ERFAAA OOOOxx 1843 3827 1 3 3 3 43 843 1843 1843 1843 86 87 XSAAAA FRFAAA VVVVxx 1006 3828 0 2 6 6 6 6 1006 1006 1006 12 13 SMAAAA GRFAAA AAAAxx 7696 3829 0 0 6 16 96 696 1696 2696 7696 192 193 AKAAAA HRFAAA HHHHxx 6411 3830 1 3 1 11 11 411 411 1411 6411 22 23 PMAAAA IRFAAA OOOOxx 3913 3831 1 1 3 13 13 913 1913 3913 3913 26 27 NUAAAA JRFAAA VVVVxx 2538 3832 0 2 8 18 38 538 538 2538 2538 76 77 QTAAAA KRFAAA AAAAxx 3019 3833 1 3 9 19 19 19 1019 3019 3019 38 39 DMAAAA LRFAAA HHHHxx 107 3834 1 3 7 7 7 107 107 107 107 14 15 DEAAAA MRFAAA OOOOxx 427 3835 1 3 7 7 27 427 427 427 427 54 55 LQAAAA NRFAAA VVVVxx 9849 3836 1 1 9 9 49 849 1849 4849 9849 98 99 VOAAAA ORFAAA AAAAxx 4195 3837 1 3 5 15 95 195 195 4195 4195 190 191 JFAAAA PRFAAA HHHHxx 9215 3838 1 3 5 15 15 215 1215 4215 9215 30 31 LQAAAA QRFAAA OOOOxx 3165 3839 1 1 5 5 65 165 1165 3165 3165 130 131 TRAAAA RRFAAA VVVVxx 3280 3840 0 0 0 0 80 280 1280 3280 3280 160 161 EWAAAA SRFAAA AAAAxx 4477 3841 1 1 7 17 77 477 477 4477 4477 154 155 FQAAAA TRFAAA HHHHxx 5885 3842 1 1 5 5 85 885 1885 885 5885 170 171 JSAAAA URFAAA OOOOxx 3311 3843 1 3 1 11 11 311 1311 3311 3311 22 23 JXAAAA VRFAAA VVVVxx 6453 3844 1 1 3 13 53 453 453 1453 6453 106 107 FOAAAA WRFAAA AAAAxx 8527 3845 1 3 7 7 27 527 527 3527 8527 54 55 ZPAAAA XRFAAA HHHHxx 1921 3846 1 1 1 1 21 921 1921 1921 1921 42 43 XVAAAA YRFAAA OOOOxx 2427 3847 1 3 7 7 27 427 427 2427 2427 54 55 JPAAAA ZRFAAA VVVVxx 3691 3848 1 3 1 11 91 691 1691 3691 3691 182 183 ZLAAAA ASFAAA AAAAxx 3882 3849 0 2 2 2 82 882 1882 3882 3882 164 165 ITAAAA BSFAAA HHHHxx 562 3850 0 2 2 2 62 562 562 562 562 124 125 QVAAAA CSFAAA OOOOxx 377 3851 1 1 7 17 77 377 377 377 377 154 155 NOAAAA DSFAAA VVVVxx 1497 3852 1 1 7 17 97 497 1497 1497 1497 194 195 PFAAAA ESFAAA AAAAxx 4453 3853 1 1 3 13 53 453 453 4453 4453 106 107 HPAAAA FSFAAA HHHHxx 4678 3854 0 2 8 18 78 678 678 4678 4678 156 157 YXAAAA GSFAAA OOOOxx 2234 3855 0 2 4 14 34 234 234 2234 2234 68 69 YHAAAA HSFAAA VVVVxx 1073 3856 1 1 3 13 73 73 1073 1073 1073 146 147 HPAAAA ISFAAA AAAAxx 6479 3857 1 3 9 19 79 479 479 1479 6479 158 159 FPAAAA JSFAAA HHHHxx 5665 3858 1 1 5 5 65 665 1665 665 5665 130 131 XJAAAA KSFAAA OOOOxx 586 3859 0 2 6 6 86 586 586 586 586 172 173 OWAAAA LSFAAA VVVVxx 1584 3860 0 0 4 4 84 584 1584 1584 1584 168 169 YIAAAA MSFAAA AAAAxx 2574 3861 0 2 4 14 74 574 574 2574 2574 148 149 AVAAAA NSFAAA HHHHxx 9833 3862 1 1 3 13 33 833 1833 4833 9833 66 67 FOAAAA OSFAAA OOOOxx 6726 3863 0 2 6 6 26 726 726 1726 6726 52 53 SYAAAA PSFAAA VVVVxx 8497 3864 1 1 7 17 97 497 497 3497 8497 194 195 VOAAAA QSFAAA AAAAxx 2914 3865 0 2 4 14 14 914 914 2914 2914 28 29 CIAAAA RSFAAA HHHHxx 8586 3866 0 2 6 6 86 586 586 3586 8586 172 173 GSAAAA SSFAAA OOOOxx 6973 3867 1 1 3 13 73 973 973 1973 6973 146 147 FIAAAA TSFAAA VVVVxx 1322 3868 0 2 2 2 22 322 1322 1322 1322 44 45 WYAAAA USFAAA AAAAxx 5242 3869 0 2 2 2 42 242 1242 242 5242 84 85 QTAAAA VSFAAA HHHHxx 5581 3870 1 1 1 1 81 581 1581 581 5581 162 163 RGAAAA WSFAAA OOOOxx 1365 3871 1 1 5 5 65 365 1365 1365 1365 130 131 NAAAAA XSFAAA VVVVxx 2818 3872 0 2 8 18 18 818 818 2818 2818 36 37 KEAAAA YSFAAA AAAAxx 3758 3873 0 2 8 18 58 758 1758 3758 3758 116 117 OOAAAA ZSFAAA HHHHxx 2665 3874 1 1 5 5 65 665 665 2665 2665 130 131 NYAAAA ATFAAA OOOOxx 9823 3875 1 3 3 3 23 823 1823 4823 9823 46 47 VNAAAA BTFAAA VVVVxx 7057 3876 1 1 7 17 57 57 1057 2057 7057 114 115 LLAAAA CTFAAA AAAAxx 543 3877 1 3 3 3 43 543 543 543 543 86 87 XUAAAA DTFAAA HHHHxx 4008 3878 0 0 8 8 8 8 8 4008 4008 16 17 EYAAAA ETFAAA OOOOxx 4397 3879 1 1 7 17 97 397 397 4397 4397 194 195 DNAAAA FTFAAA VVVVxx 8533 3880 1 1 3 13 33 533 533 3533 8533 66 67 FQAAAA GTFAAA AAAAxx 9728 3881 0 0 8 8 28 728 1728 4728 9728 56 57 EKAAAA HTFAAA HHHHxx 5198 3882 0 2 8 18 98 198 1198 198 5198 196 197 YRAAAA ITFAAA OOOOxx 5036 3883 0 0 6 16 36 36 1036 36 5036 72 73 SLAAAA JTFAAA VVVVxx 4394 3884 0 2 4 14 94 394 394 4394 4394 188 189 ANAAAA KTFAAA AAAAxx 9633 3885 1 1 3 13 33 633 1633 4633 9633 66 67 NGAAAA LTFAAA HHHHxx 3339 3886 1 3 9 19 39 339 1339 3339 3339 78 79 LYAAAA MTFAAA OOOOxx 9529 3887 1 1 9 9 29 529 1529 4529 9529 58 59 NCAAAA NTFAAA VVVVxx 4780 3888 0 0 0 0 80 780 780 4780 4780 160 161 WBAAAA OTFAAA AAAAxx 4862 3889 0 2 2 2 62 862 862 4862 4862 124 125 AFAAAA PTFAAA HHHHxx 8152 3890 0 0 2 12 52 152 152 3152 8152 104 105 OBAAAA QTFAAA OOOOxx 9330 3891 0 2 0 10 30 330 1330 4330 9330 60 61 WUAAAA RTFAAA VVVVxx 4362 3892 0 2 2 2 62 362 362 4362 4362 124 125 ULAAAA STFAAA AAAAxx 4688 3893 0 0 8 8 88 688 688 4688 4688 176 177 IYAAAA TTFAAA HHHHxx 1903 3894 1 3 3 3 3 903 1903 1903 1903 6 7 FVAAAA UTFAAA OOOOxx 9027 3895 1 3 7 7 27 27 1027 4027 9027 54 55 FJAAAA VTFAAA VVVVxx 5385 3896 1 1 5 5 85 385 1385 385 5385 170 171 DZAAAA WTFAAA AAAAxx 9854 3897 0 2 4 14 54 854 1854 4854 9854 108 109 APAAAA XTFAAA HHHHxx 9033 3898 1 1 3 13 33 33 1033 4033 9033 66 67 LJAAAA YTFAAA OOOOxx 3185 3899 1 1 5 5 85 185 1185 3185 3185 170 171 NSAAAA ZTFAAA VVVVxx 2618 3900 0 2 8 18 18 618 618 2618 2618 36 37 SWAAAA AUFAAA AAAAxx 371 3901 1 3 1 11 71 371 371 371 371 142 143 HOAAAA BUFAAA HHHHxx 3697 3902 1 1 7 17 97 697 1697 3697 3697 194 195 FMAAAA CUFAAA OOOOxx 1682 3903 0 2 2 2 82 682 1682 1682 1682 164 165 SMAAAA DUFAAA VVVVxx 3333 3904 1 1 3 13 33 333 1333 3333 3333 66 67 FYAAAA EUFAAA AAAAxx 1722 3905 0 2 2 2 22 722 1722 1722 1722 44 45 GOAAAA FUFAAA HHHHxx 2009 3906 1 1 9 9 9 9 9 2009 2009 18 19 HZAAAA GUFAAA OOOOxx 3517 3907 1 1 7 17 17 517 1517 3517 3517 34 35 HFAAAA HUFAAA VVVVxx 7640 3908 0 0 0 0 40 640 1640 2640 7640 80 81 WHAAAA IUFAAA AAAAxx 259 3909 1 3 9 19 59 259 259 259 259 118 119 ZJAAAA JUFAAA HHHHxx 1400 3910 0 0 0 0 0 400 1400 1400 1400 0 1 WBAAAA KUFAAA OOOOxx 6663 3911 1 3 3 3 63 663 663 1663 6663 126 127 HWAAAA LUFAAA VVVVxx 1576 3912 0 0 6 16 76 576 1576 1576 1576 152 153 QIAAAA MUFAAA AAAAxx 8843 3913 1 3 3 3 43 843 843 3843 8843 86 87 DCAAAA NUFAAA HHHHxx 9474 3914 0 2 4 14 74 474 1474 4474 9474 148 149 KAAAAA OUFAAA OOOOxx 1597 3915 1 1 7 17 97 597 1597 1597 1597 194 195 LJAAAA PUFAAA VVVVxx 1143 3916 1 3 3 3 43 143 1143 1143 1143 86 87 ZRAAAA QUFAAA AAAAxx 4162 3917 0 2 2 2 62 162 162 4162 4162 124 125 CEAAAA RUFAAA HHHHxx 1301 3918 1 1 1 1 1 301 1301 1301 1301 2 3 BYAAAA SUFAAA OOOOxx 2935 3919 1 3 5 15 35 935 935 2935 2935 70 71 XIAAAA TUFAAA VVVVxx 886 3920 0 2 6 6 86 886 886 886 886 172 173 CIAAAA UUFAAA AAAAxx 1661 3921 1 1 1 1 61 661 1661 1661 1661 122 123 XLAAAA VUFAAA HHHHxx 1026 3922 0 2 6 6 26 26 1026 1026 1026 52 53 MNAAAA WUFAAA OOOOxx 7034 3923 0 2 4 14 34 34 1034 2034 7034 68 69 OKAAAA XUFAAA VVVVxx 2305 3924 1 1 5 5 5 305 305 2305 2305 10 11 RKAAAA YUFAAA AAAAxx 1725 3925 1 1 5 5 25 725 1725 1725 1725 50 51 JOAAAA ZUFAAA HHHHxx 909 3926 1 1 9 9 9 909 909 909 909 18 19 ZIAAAA AVFAAA OOOOxx 9906 3927 0 2 6 6 6 906 1906 4906 9906 12 13 ARAAAA BVFAAA VVVVxx 3309 3928 1 1 9 9 9 309 1309 3309 3309 18 19 HXAAAA CVFAAA AAAAxx 515 3929 1 3 5 15 15 515 515 515 515 30 31 VTAAAA DVFAAA HHHHxx 932 3930 0 0 2 12 32 932 932 932 932 64 65 WJAAAA EVFAAA OOOOxx 8144 3931 0 0 4 4 44 144 144 3144 8144 88 89 GBAAAA FVFAAA VVVVxx 5592 3932 0 0 2 12 92 592 1592 592 5592 184 185 CHAAAA GVFAAA AAAAxx 4003 3933 1 3 3 3 3 3 3 4003 4003 6 7 ZXAAAA HVFAAA HHHHxx 9566 3934 0 2 6 6 66 566 1566 4566 9566 132 133 YDAAAA IVFAAA OOOOxx 4556 3935 0 0 6 16 56 556 556 4556 4556 112 113 GTAAAA JVFAAA VVVVxx 268 3936 0 0 8 8 68 268 268 268 268 136 137 IKAAAA KVFAAA AAAAxx 8107 3937 1 3 7 7 7 107 107 3107 8107 14 15 VZAAAA LVFAAA HHHHxx 5816 3938 0 0 6 16 16 816 1816 816 5816 32 33 SPAAAA MVFAAA OOOOxx 8597 3939 1 1 7 17 97 597 597 3597 8597 194 195 RSAAAA NVFAAA VVVVxx 9611 3940 1 3 1 11 11 611 1611 4611 9611 22 23 RFAAAA OVFAAA AAAAxx 8070 3941 0 2 0 10 70 70 70 3070 8070 140 141 KYAAAA PVFAAA HHHHxx 6040 3942 0 0 0 0 40 40 40 1040 6040 80 81 IYAAAA QVFAAA OOOOxx 3184 3943 0 0 4 4 84 184 1184 3184 3184 168 169 MSAAAA RVFAAA VVVVxx 9656 3944 0 0 6 16 56 656 1656 4656 9656 112 113 KHAAAA SVFAAA AAAAxx 1577 3945 1 1 7 17 77 577 1577 1577 1577 154 155 RIAAAA TVFAAA HHHHxx 1805 3946 1 1 5 5 5 805 1805 1805 1805 10 11 LRAAAA UVFAAA OOOOxx 8268 3947 0 0 8 8 68 268 268 3268 8268 136 137 AGAAAA VVFAAA VVVVxx 3489 3948 1 1 9 9 89 489 1489 3489 3489 178 179 FEAAAA WVFAAA AAAAxx 4564 3949 0 0 4 4 64 564 564 4564 4564 128 129 OTAAAA XVFAAA HHHHxx 4006 3950 0 2 6 6 6 6 6 4006 4006 12 13 CYAAAA YVFAAA OOOOxx 8466 3951 0 2 6 6 66 466 466 3466 8466 132 133 QNAAAA ZVFAAA VVVVxx 938 3952 0 2 8 18 38 938 938 938 938 76 77 CKAAAA AWFAAA AAAAxx 5944 3953 0 0 4 4 44 944 1944 944 5944 88 89 QUAAAA BWFAAA HHHHxx 8363 3954 1 3 3 3 63 363 363 3363 8363 126 127 RJAAAA CWFAAA OOOOxx 5348 3955 0 0 8 8 48 348 1348 348 5348 96 97 SXAAAA DWFAAA VVVVxx 71 3956 1 3 1 11 71 71 71 71 71 142 143 TCAAAA EWFAAA AAAAxx 3620 3957 0 0 0 0 20 620 1620 3620 3620 40 41 GJAAAA FWFAAA HHHHxx 3230 3958 0 2 0 10 30 230 1230 3230 3230 60 61 GUAAAA GWFAAA OOOOxx 6132 3959 0 0 2 12 32 132 132 1132 6132 64 65 WBAAAA HWFAAA VVVVxx 6143 3960 1 3 3 3 43 143 143 1143 6143 86 87 HCAAAA IWFAAA AAAAxx 8781 3961 1 1 1 1 81 781 781 3781 8781 162 163 TZAAAA JWFAAA HHHHxx 5522 3962 0 2 2 2 22 522 1522 522 5522 44 45 KEAAAA KWFAAA OOOOxx 6320 3963 0 0 0 0 20 320 320 1320 6320 40 41 CJAAAA LWFAAA VVVVxx 3923 3964 1 3 3 3 23 923 1923 3923 3923 46 47 XUAAAA MWFAAA AAAAxx 2207 3965 1 3 7 7 7 207 207 2207 2207 14 15 XGAAAA NWFAAA HHHHxx 966 3966 0 2 6 6 66 966 966 966 966 132 133 ELAAAA OWFAAA OOOOxx 9020 3967 0 0 0 0 20 20 1020 4020 9020 40 41 YIAAAA PWFAAA VVVVxx 4616 3968 0 0 6 16 16 616 616 4616 4616 32 33 OVAAAA QWFAAA AAAAxx 8289 3969 1 1 9 9 89 289 289 3289 8289 178 179 VGAAAA RWFAAA HHHHxx 5796 3970 0 0 6 16 96 796 1796 796 5796 192 193 YOAAAA SWFAAA OOOOxx 9259 3971 1 3 9 19 59 259 1259 4259 9259 118 119 DSAAAA TWFAAA VVVVxx 3710 3972 0 2 0 10 10 710 1710 3710 3710 20 21 SMAAAA UWFAAA AAAAxx 251 3973 1 3 1 11 51 251 251 251 251 102 103 RJAAAA VWFAAA HHHHxx 7669 3974 1 1 9 9 69 669 1669 2669 7669 138 139 ZIAAAA WWFAAA OOOOxx 6304 3975 0 0 4 4 4 304 304 1304 6304 8 9 MIAAAA XWFAAA VVVVxx 6454 3976 0 2 4 14 54 454 454 1454 6454 108 109 GOAAAA YWFAAA AAAAxx 1489 3977 1 1 9 9 89 489 1489 1489 1489 178 179 HFAAAA ZWFAAA HHHHxx 715 3978 1 3 5 15 15 715 715 715 715 30 31 NBAAAA AXFAAA OOOOxx 4319 3979 1 3 9 19 19 319 319 4319 4319 38 39 DKAAAA BXFAAA VVVVxx 7112 3980 0 0 2 12 12 112 1112 2112 7112 24 25 ONAAAA CXFAAA AAAAxx 3726 3981 0 2 6 6 26 726 1726 3726 3726 52 53 INAAAA DXFAAA HHHHxx 7727 3982 1 3 7 7 27 727 1727 2727 7727 54 55 FLAAAA EXFAAA OOOOxx 8387 3983 1 3 7 7 87 387 387 3387 8387 174 175 PKAAAA FXFAAA VVVVxx 6555 3984 1 3 5 15 55 555 555 1555 6555 110 111 DSAAAA GXFAAA AAAAxx 1148 3985 0 0 8 8 48 148 1148 1148 1148 96 97 ESAAAA HXFAAA HHHHxx 9000 3986 0 0 0 0 0 0 1000 4000 9000 0 1 EIAAAA IXFAAA OOOOxx 5278 3987 0 2 8 18 78 278 1278 278 5278 156 157 AVAAAA JXFAAA VVVVxx 2388 3988 0 0 8 8 88 388 388 2388 2388 176 177 WNAAAA KXFAAA AAAAxx 7984 3989 0 0 4 4 84 984 1984 2984 7984 168 169 CVAAAA LXFAAA HHHHxx 881 3990 1 1 1 1 81 881 881 881 881 162 163 XHAAAA MXFAAA OOOOxx 6830 3991 0 2 0 10 30 830 830 1830 6830 60 61 SCAAAA NXFAAA VVVVxx 7056 3992 0 0 6 16 56 56 1056 2056 7056 112 113 KLAAAA OXFAAA AAAAxx 7581 3993 1 1 1 1 81 581 1581 2581 7581 162 163 PFAAAA PXFAAA HHHHxx 5214 3994 0 2 4 14 14 214 1214 214 5214 28 29 OSAAAA QXFAAA OOOOxx 2505 3995 1 1 5 5 5 505 505 2505 2505 10 11 JSAAAA RXFAAA VVVVxx 5112 3996 0 0 2 12 12 112 1112 112 5112 24 25 QOAAAA SXFAAA AAAAxx 9884 3997 0 0 4 4 84 884 1884 4884 9884 168 169 EQAAAA TXFAAA HHHHxx 8040 3998 0 0 0 0 40 40 40 3040 8040 80 81 GXAAAA UXFAAA OOOOxx 7033 3999 1 1 3 13 33 33 1033 2033 7033 66 67 NKAAAA VXFAAA VVVVxx 9343 4000 1 3 3 3 43 343 1343 4343 9343 86 87 JVAAAA WXFAAA AAAAxx 2931 4001 1 3 1 11 31 931 931 2931 2931 62 63 TIAAAA XXFAAA HHHHxx 9024 4002 0 0 4 4 24 24 1024 4024 9024 48 49 CJAAAA YXFAAA OOOOxx 6485 4003 1 1 5 5 85 485 485 1485 6485 170 171 LPAAAA ZXFAAA VVVVxx 3465 4004 1 1 5 5 65 465 1465 3465 3465 130 131 HDAAAA AYFAAA AAAAxx 3357 4005 1 1 7 17 57 357 1357 3357 3357 114 115 DZAAAA BYFAAA HHHHxx 2929 4006 1 1 9 9 29 929 929 2929 2929 58 59 RIAAAA CYFAAA OOOOxx 3086 4007 0 2 6 6 86 86 1086 3086 3086 172 173 SOAAAA DYFAAA VVVVxx 8897 4008 1 1 7 17 97 897 897 3897 8897 194 195 FEAAAA EYFAAA AAAAxx 9688 4009 0 0 8 8 88 688 1688 4688 9688 176 177 QIAAAA FYFAAA HHHHxx 6522 4010 0 2 2 2 22 522 522 1522 6522 44 45 WQAAAA GYFAAA OOOOxx 3241 4011 1 1 1 1 41 241 1241 3241 3241 82 83 RUAAAA HYFAAA VVVVxx 8770 4012 0 2 0 10 70 770 770 3770 8770 140 141 IZAAAA IYFAAA AAAAxx 2884 4013 0 0 4 4 84 884 884 2884 2884 168 169 YGAAAA JYFAAA HHHHxx 9579 4014 1 3 9 19 79 579 1579 4579 9579 158 159 LEAAAA KYFAAA OOOOxx 3125 4015 1 1 5 5 25 125 1125 3125 3125 50 51 FQAAAA LYFAAA VVVVxx 4604 4016 0 0 4 4 4 604 604 4604 4604 8 9 CVAAAA MYFAAA AAAAxx 2682 4017 0 2 2 2 82 682 682 2682 2682 164 165 EZAAAA NYFAAA HHHHxx 254 4018 0 2 4 14 54 254 254 254 254 108 109 UJAAAA OYFAAA OOOOxx 6569 4019 1 1 9 9 69 569 569 1569 6569 138 139 RSAAAA PYFAAA VVVVxx 2686 4020 0 2 6 6 86 686 686 2686 2686 172 173 IZAAAA QYFAAA AAAAxx 2123 4021 1 3 3 3 23 123 123 2123 2123 46 47 RDAAAA RYFAAA HHHHxx 1745 4022 1 1 5 5 45 745 1745 1745 1745 90 91 DPAAAA SYFAAA OOOOxx 247 4023 1 3 7 7 47 247 247 247 247 94 95 NJAAAA TYFAAA VVVVxx 5800 4024 0 0 0 0 0 800 1800 800 5800 0 1 CPAAAA UYFAAA AAAAxx 1121 4025 1 1 1 1 21 121 1121 1121 1121 42 43 DRAAAA VYFAAA HHHHxx 8893 4026 1 1 3 13 93 893 893 3893 8893 186 187 BEAAAA WYFAAA OOOOxx 7819 4027 1 3 9 19 19 819 1819 2819 7819 38 39 TOAAAA XYFAAA VVVVxx 1339 4028 1 3 9 19 39 339 1339 1339 1339 78 79 NZAAAA YYFAAA AAAAxx 5680 4029 0 0 0 0 80 680 1680 680 5680 160 161 MKAAAA ZYFAAA HHHHxx 5093 4030 1 1 3 13 93 93 1093 93 5093 186 187 XNAAAA AZFAAA OOOOxx 3508 4031 0 0 8 8 8 508 1508 3508 3508 16 17 YEAAAA BZFAAA VVVVxx 933 4032 1 1 3 13 33 933 933 933 933 66 67 XJAAAA CZFAAA AAAAxx 1106 4033 0 2 6 6 6 106 1106 1106 1106 12 13 OQAAAA DZFAAA HHHHxx 4386 4034 0 2 6 6 86 386 386 4386 4386 172 173 SMAAAA EZFAAA OOOOxx 5895 4035 1 3 5 15 95 895 1895 895 5895 190 191 TSAAAA FZFAAA VVVVxx 2980 4036 0 0 0 0 80 980 980 2980 2980 160 161 QKAAAA GZFAAA AAAAxx 4400 4037 0 0 0 0 0 400 400 4400 4400 0 1 GNAAAA HZFAAA HHHHxx 7433 4038 1 1 3 13 33 433 1433 2433 7433 66 67 XZAAAA IZFAAA OOOOxx 6110 4039 0 2 0 10 10 110 110 1110 6110 20 21 ABAAAA JZFAAA VVVVxx 867 4040 1 3 7 7 67 867 867 867 867 134 135 JHAAAA KZFAAA AAAAxx 5292 4041 0 0 2 12 92 292 1292 292 5292 184 185 OVAAAA LZFAAA HHHHxx 3926 4042 0 2 6 6 26 926 1926 3926 3926 52 53 AVAAAA MZFAAA OOOOxx 1107 4043 1 3 7 7 7 107 1107 1107 1107 14 15 PQAAAA NZFAAA VVVVxx 7355 4044 1 3 5 15 55 355 1355 2355 7355 110 111 XWAAAA OZFAAA AAAAxx 4689 4045 1 1 9 9 89 689 689 4689 4689 178 179 JYAAAA PZFAAA HHHHxx 4872 4046 0 0 2 12 72 872 872 4872 4872 144 145 KFAAAA QZFAAA OOOOxx 7821 4047 1 1 1 1 21 821 1821 2821 7821 42 43 VOAAAA RZFAAA VVVVxx 7277 4048 1 1 7 17 77 277 1277 2277 7277 154 155 XTAAAA SZFAAA AAAAxx 3268 4049 0 0 8 8 68 268 1268 3268 3268 136 137 SVAAAA TZFAAA HHHHxx 8877 4050 1 1 7 17 77 877 877 3877 8877 154 155 LDAAAA UZFAAA OOOOxx 343 4051 1 3 3 3 43 343 343 343 343 86 87 FNAAAA VZFAAA VVVVxx 621 4052 1 1 1 1 21 621 621 621 621 42 43 XXAAAA WZFAAA AAAAxx 5429 4053 1 1 9 9 29 429 1429 429 5429 58 59 VAAAAA XZFAAA HHHHxx 392 4054 0 0 2 12 92 392 392 392 392 184 185 CPAAAA YZFAAA OOOOxx 6004 4055 0 0 4 4 4 4 4 1004 6004 8 9 YWAAAA ZZFAAA VVVVxx 6377 4056 1 1 7 17 77 377 377 1377 6377 154 155 HLAAAA AAGAAA AAAAxx 3037 4057 1 1 7 17 37 37 1037 3037 3037 74 75 VMAAAA BAGAAA HHHHxx 3514 4058 0 2 4 14 14 514 1514 3514 3514 28 29 EFAAAA CAGAAA OOOOxx 8740 4059 0 0 0 0 40 740 740 3740 8740 80 81 EYAAAA DAGAAA VVVVxx 3877 4060 1 1 7 17 77 877 1877 3877 3877 154 155 DTAAAA EAGAAA AAAAxx 5731 4061 1 3 1 11 31 731 1731 731 5731 62 63 LMAAAA FAGAAA HHHHxx 6407 4062 1 3 7 7 7 407 407 1407 6407 14 15 LMAAAA GAGAAA OOOOxx 2044 4063 0 0 4 4 44 44 44 2044 2044 88 89 QAAAAA HAGAAA VVVVxx 7362 4064 0 2 2 2 62 362 1362 2362 7362 124 125 EXAAAA IAGAAA AAAAxx 5458 4065 0 2 8 18 58 458 1458 458 5458 116 117 YBAAAA JAGAAA HHHHxx 6437 4066 1 1 7 17 37 437 437 1437 6437 74 75 PNAAAA KAGAAA OOOOxx 1051 4067 1 3 1 11 51 51 1051 1051 1051 102 103 LOAAAA LAGAAA VVVVxx 1203 4068 1 3 3 3 3 203 1203 1203 1203 6 7 HUAAAA MAGAAA AAAAxx 2176 4069 0 0 6 16 76 176 176 2176 2176 152 153 SFAAAA NAGAAA HHHHxx 8997 4070 1 1 7 17 97 997 997 3997 8997 194 195 BIAAAA OAGAAA OOOOxx 6378 4071 0 2 8 18 78 378 378 1378 6378 156 157 ILAAAA PAGAAA VVVVxx 6006 4072 0 2 6 6 6 6 6 1006 6006 12 13 AXAAAA QAGAAA AAAAxx 2308 4073 0 0 8 8 8 308 308 2308 2308 16 17 UKAAAA RAGAAA HHHHxx 625 4074 1 1 5 5 25 625 625 625 625 50 51 BYAAAA SAGAAA OOOOxx 7298 4075 0 2 8 18 98 298 1298 2298 7298 196 197 SUAAAA TAGAAA VVVVxx 5575 4076 1 3 5 15 75 575 1575 575 5575 150 151 LGAAAA UAGAAA AAAAxx 3565 4077 1 1 5 5 65 565 1565 3565 3565 130 131 DHAAAA VAGAAA HHHHxx 47 4078 1 3 7 7 47 47 47 47 47 94 95 VBAAAA WAGAAA OOOOxx 2413 4079 1 1 3 13 13 413 413 2413 2413 26 27 VOAAAA XAGAAA VVVVxx 2153 4080 1 1 3 13 53 153 153 2153 2153 106 107 VEAAAA YAGAAA AAAAxx 752 4081 0 0 2 12 52 752 752 752 752 104 105 YCAAAA ZAGAAA HHHHxx 4095 4082 1 3 5 15 95 95 95 4095 4095 190 191 NBAAAA ABGAAA OOOOxx 2518 4083 0 2 8 18 18 518 518 2518 2518 36 37 WSAAAA BBGAAA VVVVxx 3681 4084 1 1 1 1 81 681 1681 3681 3681 162 163 PLAAAA CBGAAA AAAAxx 4213 4085 1 1 3 13 13 213 213 4213 4213 26 27 BGAAAA DBGAAA HHHHxx 2615 4086 1 3 5 15 15 615 615 2615 2615 30 31 PWAAAA EBGAAA OOOOxx 1471 4087 1 3 1 11 71 471 1471 1471 1471 142 143 PEAAAA FBGAAA VVVVxx 7315 4088 1 3 5 15 15 315 1315 2315 7315 30 31 JVAAAA GBGAAA AAAAxx 6013 4089 1 1 3 13 13 13 13 1013 6013 26 27 HXAAAA HBGAAA HHHHxx 3077 4090 1 1 7 17 77 77 1077 3077 3077 154 155 JOAAAA IBGAAA OOOOxx 2190 4091 0 2 0 10 90 190 190 2190 2190 180 181 GGAAAA JBGAAA VVVVxx 528 4092 0 0 8 8 28 528 528 528 528 56 57 IUAAAA KBGAAA AAAAxx 9508 4093 0 0 8 8 8 508 1508 4508 9508 16 17 SBAAAA LBGAAA HHHHxx 2473 4094 1 1 3 13 73 473 473 2473 2473 146 147 DRAAAA MBGAAA OOOOxx 167 4095 1 3 7 7 67 167 167 167 167 134 135 LGAAAA NBGAAA VVVVxx 8448 4096 0 0 8 8 48 448 448 3448 8448 96 97 YMAAAA OBGAAA AAAAxx 7538 4097 0 2 8 18 38 538 1538 2538 7538 76 77 YDAAAA PBGAAA HHHHxx 7638 4098 0 2 8 18 38 638 1638 2638 7638 76 77 UHAAAA QBGAAA OOOOxx 4328 4099 0 0 8 8 28 328 328 4328 4328 56 57 MKAAAA RBGAAA VVVVxx 3812 4100 0 0 2 12 12 812 1812 3812 3812 24 25 QQAAAA SBGAAA AAAAxx 2879 4101 1 3 9 19 79 879 879 2879 2879 158 159 TGAAAA TBGAAA HHHHxx 4741 4102 1 1 1 1 41 741 741 4741 4741 82 83 JAAAAA UBGAAA OOOOxx 9155 4103 1 3 5 15 55 155 1155 4155 9155 110 111 DOAAAA VBGAAA VVVVxx 5151 4104 1 3 1 11 51 151 1151 151 5151 102 103 DQAAAA WBGAAA AAAAxx 5591 4105 1 3 1 11 91 591 1591 591 5591 182 183 BHAAAA XBGAAA HHHHxx 1034 4106 0 2 4 14 34 34 1034 1034 1034 68 69 UNAAAA YBGAAA OOOOxx 765 4107 1 1 5 5 65 765 765 765 765 130 131 LDAAAA ZBGAAA VVVVxx 2664 4108 0 0 4 4 64 664 664 2664 2664 128 129 MYAAAA ACGAAA AAAAxx 6854 4109 0 2 4 14 54 854 854 1854 6854 108 109 QDAAAA BCGAAA HHHHxx 8263 4110 1 3 3 3 63 263 263 3263 8263 126 127 VFAAAA CCGAAA OOOOxx 8658 4111 0 2 8 18 58 658 658 3658 8658 116 117 AVAAAA DCGAAA VVVVxx 587 4112 1 3 7 7 87 587 587 587 587 174 175 PWAAAA ECGAAA AAAAxx 4553 4113 1 1 3 13 53 553 553 4553 4553 106 107 DTAAAA FCGAAA HHHHxx 1368 4114 0 0 8 8 68 368 1368 1368 1368 136 137 QAAAAA GCGAAA OOOOxx 1718 4115 0 2 8 18 18 718 1718 1718 1718 36 37 COAAAA HCGAAA VVVVxx 140 4116 0 0 0 0 40 140 140 140 140 80 81 KFAAAA ICGAAA AAAAxx 8341 4117 1 1 1 1 41 341 341 3341 8341 82 83 VIAAAA JCGAAA HHHHxx 72 4118 0 0 2 12 72 72 72 72 72 144 145 UCAAAA KCGAAA OOOOxx 6589 4119 1 1 9 9 89 589 589 1589 6589 178 179 LTAAAA LCGAAA VVVVxx 2024 4120 0 0 4 4 24 24 24 2024 2024 48 49 WZAAAA MCGAAA AAAAxx 8024 4121 0 0 4 4 24 24 24 3024 8024 48 49 QWAAAA NCGAAA HHHHxx 9564 4122 0 0 4 4 64 564 1564 4564 9564 128 129 WDAAAA OCGAAA OOOOxx 8625 4123 1 1 5 5 25 625 625 3625 8625 50 51 TTAAAA PCGAAA VVVVxx 2680 4124 0 0 0 0 80 680 680 2680 2680 160 161 CZAAAA QCGAAA AAAAxx 4323 4125 1 3 3 3 23 323 323 4323 4323 46 47 HKAAAA RCGAAA HHHHxx 8981 4126 1 1 1 1 81 981 981 3981 8981 162 163 LHAAAA SCGAAA OOOOxx 8909 4127 1 1 9 9 9 909 909 3909 8909 18 19 REAAAA TCGAAA VVVVxx 5288 4128 0 0 8 8 88 288 1288 288 5288 176 177 KVAAAA UCGAAA AAAAxx 2057 4129 1 1 7 17 57 57 57 2057 2057 114 115 DBAAAA VCGAAA HHHHxx 5931 4130 1 3 1 11 31 931 1931 931 5931 62 63 DUAAAA WCGAAA OOOOxx 9794 4131 0 2 4 14 94 794 1794 4794 9794 188 189 SMAAAA XCGAAA VVVVxx 1012 4132 0 0 2 12 12 12 1012 1012 1012 24 25 YMAAAA YCGAAA AAAAxx 5496 4133 0 0 6 16 96 496 1496 496 5496 192 193 KDAAAA ZCGAAA HHHHxx 9182 4134 0 2 2 2 82 182 1182 4182 9182 164 165 EPAAAA ADGAAA OOOOxx 5258 4135 0 2 8 18 58 258 1258 258 5258 116 117 GUAAAA BDGAAA VVVVxx 3050 4136 0 2 0 10 50 50 1050 3050 3050 100 101 INAAAA CDGAAA AAAAxx 2083 4137 1 3 3 3 83 83 83 2083 2083 166 167 DCAAAA DDGAAA HHHHxx 3069 4138 1 1 9 9 69 69 1069 3069 3069 138 139 BOAAAA EDGAAA OOOOxx 8459 4139 1 3 9 19 59 459 459 3459 8459 118 119 JNAAAA FDGAAA VVVVxx 169 4140 1 1 9 9 69 169 169 169 169 138 139 NGAAAA GDGAAA AAAAxx 4379 4141 1 3 9 19 79 379 379 4379 4379 158 159 LMAAAA HDGAAA HHHHxx 5126 4142 0 2 6 6 26 126 1126 126 5126 52 53 EPAAAA IDGAAA OOOOxx 1415 4143 1 3 5 15 15 415 1415 1415 1415 30 31 LCAAAA JDGAAA VVVVxx 1163 4144 1 3 3 3 63 163 1163 1163 1163 126 127 TSAAAA KDGAAA AAAAxx 3500 4145 0 0 0 0 0 500 1500 3500 3500 0 1 QEAAAA LDGAAA HHHHxx 7202 4146 0 2 2 2 2 202 1202 2202 7202 4 5 ARAAAA MDGAAA OOOOxx 747 4147 1 3 7 7 47 747 747 747 747 94 95 TCAAAA NDGAAA VVVVxx 9264 4148 0 0 4 4 64 264 1264 4264 9264 128 129 ISAAAA ODGAAA AAAAxx 8548 4149 0 0 8 8 48 548 548 3548 8548 96 97 UQAAAA PDGAAA HHHHxx 4228 4150 0 0 8 8 28 228 228 4228 4228 56 57 QGAAAA QDGAAA OOOOxx 7122 4151 0 2 2 2 22 122 1122 2122 7122 44 45 YNAAAA RDGAAA VVVVxx 3395 4152 1 3 5 15 95 395 1395 3395 3395 190 191 PAAAAA SDGAAA AAAAxx 5674 4153 0 2 4 14 74 674 1674 674 5674 148 149 GKAAAA TDGAAA HHHHxx 7293 4154 1 1 3 13 93 293 1293 2293 7293 186 187 NUAAAA UDGAAA OOOOxx 737 4155 1 1 7 17 37 737 737 737 737 74 75 JCAAAA VDGAAA VVVVxx 9595 4156 1 3 5 15 95 595 1595 4595 9595 190 191 BFAAAA WDGAAA AAAAxx 594 4157 0 2 4 14 94 594 594 594 594 188 189 WWAAAA XDGAAA HHHHxx 5322 4158 0 2 2 2 22 322 1322 322 5322 44 45 SWAAAA YDGAAA OOOOxx 2933 4159 1 1 3 13 33 933 933 2933 2933 66 67 VIAAAA ZDGAAA VVVVxx 4955 4160 1 3 5 15 55 955 955 4955 4955 110 111 PIAAAA AEGAAA AAAAxx 4073 4161 1 1 3 13 73 73 73 4073 4073 146 147 RAAAAA BEGAAA HHHHxx 7249 4162 1 1 9 9 49 249 1249 2249 7249 98 99 VSAAAA CEGAAA OOOOxx 192 4163 0 0 2 12 92 192 192 192 192 184 185 KHAAAA DEGAAA VVVVxx 2617 4164 1 1 7 17 17 617 617 2617 2617 34 35 RWAAAA EEGAAA AAAAxx 7409 4165 1 1 9 9 9 409 1409 2409 7409 18 19 ZYAAAA FEGAAA HHHHxx 4903 4166 1 3 3 3 3 903 903 4903 4903 6 7 PGAAAA GEGAAA OOOOxx 9797 4167 1 1 7 17 97 797 1797 4797 9797 194 195 VMAAAA HEGAAA VVVVxx 9919 4168 1 3 9 19 19 919 1919 4919 9919 38 39 NRAAAA IEGAAA AAAAxx 1878 4169 0 2 8 18 78 878 1878 1878 1878 156 157 GUAAAA JEGAAA HHHHxx 4851 4170 1 3 1 11 51 851 851 4851 4851 102 103 PEAAAA KEGAAA OOOOxx 5514 4171 0 2 4 14 14 514 1514 514 5514 28 29 CEAAAA LEGAAA VVVVxx 2582 4172 0 2 2 2 82 582 582 2582 2582 164 165 IVAAAA MEGAAA AAAAxx 3564 4173 0 0 4 4 64 564 1564 3564 3564 128 129 CHAAAA NEGAAA HHHHxx 7085 4174 1 1 5 5 85 85 1085 2085 7085 170 171 NMAAAA OEGAAA OOOOxx 3619 4175 1 3 9 19 19 619 1619 3619 3619 38 39 FJAAAA PEGAAA VVVVxx 261 4176 1 1 1 1 61 261 261 261 261 122 123 BKAAAA QEGAAA AAAAxx 7338 4177 0 2 8 18 38 338 1338 2338 7338 76 77 GWAAAA REGAAA HHHHxx 4251 4178 1 3 1 11 51 251 251 4251 4251 102 103 NHAAAA SEGAAA OOOOxx 5360 4179 0 0 0 0 60 360 1360 360 5360 120 121 EYAAAA TEGAAA VVVVxx 5678 4180 0 2 8 18 78 678 1678 678 5678 156 157 KKAAAA UEGAAA AAAAxx 9162 4181 0 2 2 2 62 162 1162 4162 9162 124 125 KOAAAA VEGAAA HHHHxx 5920 4182 0 0 0 0 20 920 1920 920 5920 40 41 STAAAA WEGAAA OOOOxx 7156 4183 0 0 6 16 56 156 1156 2156 7156 112 113 GPAAAA XEGAAA VVVVxx 4271 4184 1 3 1 11 71 271 271 4271 4271 142 143 HIAAAA YEGAAA AAAAxx 4698 4185 0 2 8 18 98 698 698 4698 4698 196 197 SYAAAA ZEGAAA HHHHxx 1572 4186 0 0 2 12 72 572 1572 1572 1572 144 145 MIAAAA AFGAAA OOOOxx 6974 4187 0 2 4 14 74 974 974 1974 6974 148 149 GIAAAA BFGAAA VVVVxx 4291 4188 1 3 1 11 91 291 291 4291 4291 182 183 BJAAAA CFGAAA AAAAxx 4036 4189 0 0 6 16 36 36 36 4036 4036 72 73 GZAAAA DFGAAA HHHHxx 7473 4190 1 1 3 13 73 473 1473 2473 7473 146 147 LBAAAA EFGAAA OOOOxx 4786 4191 0 2 6 6 86 786 786 4786 4786 172 173 CCAAAA FFGAAA VVVVxx 2662 4192 0 2 2 2 62 662 662 2662 2662 124 125 KYAAAA GFGAAA AAAAxx 916 4193 0 0 6 16 16 916 916 916 916 32 33 GJAAAA HFGAAA HHHHxx 668 4194 0 0 8 8 68 668 668 668 668 136 137 SZAAAA IFGAAA OOOOxx 4874 4195 0 2 4 14 74 874 874 4874 4874 148 149 MFAAAA JFGAAA VVVVxx 3752 4196 0 0 2 12 52 752 1752 3752 3752 104 105 IOAAAA KFGAAA AAAAxx 4865 4197 1 1 5 5 65 865 865 4865 4865 130 131 DFAAAA LFGAAA HHHHxx 7052 4198 0 0 2 12 52 52 1052 2052 7052 104 105 GLAAAA MFGAAA OOOOxx 5712 4199 0 0 2 12 12 712 1712 712 5712 24 25 SLAAAA NFGAAA VVVVxx 31 4200 1 3 1 11 31 31 31 31 31 62 63 FBAAAA OFGAAA AAAAxx 4944 4201 0 0 4 4 44 944 944 4944 4944 88 89 EIAAAA PFGAAA HHHHxx 1435 4202 1 3 5 15 35 435 1435 1435 1435 70 71 FDAAAA QFGAAA OOOOxx 501 4203 1 1 1 1 1 501 501 501 501 2 3 HTAAAA RFGAAA VVVVxx 9401 4204 1 1 1 1 1 401 1401 4401 9401 2 3 PXAAAA SFGAAA AAAAxx 5014 4205 0 2 4 14 14 14 1014 14 5014 28 29 WKAAAA TFGAAA HHHHxx 9125 4206 1 1 5 5 25 125 1125 4125 9125 50 51 ZMAAAA UFGAAA OOOOxx 6144 4207 0 0 4 4 44 144 144 1144 6144 88 89 ICAAAA VFGAAA VVVVxx 1743 4208 1 3 3 3 43 743 1743 1743 1743 86 87 BPAAAA WFGAAA AAAAxx 4316 4209 0 0 6 16 16 316 316 4316 4316 32 33 AKAAAA XFGAAA HHHHxx 8212 4210 0 0 2 12 12 212 212 3212 8212 24 25 WDAAAA YFGAAA OOOOxx 7344 4211 0 0 4 4 44 344 1344 2344 7344 88 89 MWAAAA ZFGAAA VVVVxx 2051 4212 1 3 1 11 51 51 51 2051 2051 102 103 XAAAAA AGGAAA AAAAxx 8131 4213 1 3 1 11 31 131 131 3131 8131 62 63 TAAAAA BGGAAA HHHHxx 7023 4214 1 3 3 3 23 23 1023 2023 7023 46 47 DKAAAA CGGAAA OOOOxx 9674 4215 0 2 4 14 74 674 1674 4674 9674 148 149 CIAAAA DGGAAA VVVVxx 4984 4216 0 0 4 4 84 984 984 4984 4984 168 169 SJAAAA EGGAAA AAAAxx 111 4217 1 3 1 11 11 111 111 111 111 22 23 HEAAAA FGGAAA HHHHxx 2296 4218 0 0 6 16 96 296 296 2296 2296 192 193 IKAAAA GGGAAA OOOOxx 5025 4219 1 1 5 5 25 25 1025 25 5025 50 51 HLAAAA HGGAAA VVVVxx 1756 4220 0 0 6 16 56 756 1756 1756 1756 112 113 OPAAAA IGGAAA AAAAxx 2885 4221 1 1 5 5 85 885 885 2885 2885 170 171 ZGAAAA JGGAAA HHHHxx 2541 4222 1 1 1 1 41 541 541 2541 2541 82 83 TTAAAA KGGAAA OOOOxx 1919 4223 1 3 9 19 19 919 1919 1919 1919 38 39 VVAAAA LGGAAA VVVVxx 6496 4224 0 0 6 16 96 496 496 1496 6496 192 193 WPAAAA MGGAAA AAAAxx 6103 4225 1 3 3 3 3 103 103 1103 6103 6 7 TAAAAA NGGAAA HHHHxx 98 4226 0 2 8 18 98 98 98 98 98 196 197 UDAAAA OGGAAA OOOOxx 3727 4227 1 3 7 7 27 727 1727 3727 3727 54 55 JNAAAA PGGAAA VVVVxx 689 4228 1 1 9 9 89 689 689 689 689 178 179 NAAAAA QGGAAA AAAAxx 7181 4229 1 1 1 1 81 181 1181 2181 7181 162 163 FQAAAA RGGAAA HHHHxx 8447 4230 1 3 7 7 47 447 447 3447 8447 94 95 XMAAAA SGGAAA OOOOxx 4569 4231 1 1 9 9 69 569 569 4569 4569 138 139 TTAAAA TGGAAA VVVVxx 8844 4232 0 0 4 4 44 844 844 3844 8844 88 89 ECAAAA UGGAAA AAAAxx 2436 4233 0 0 6 16 36 436 436 2436 2436 72 73 SPAAAA VGGAAA HHHHxx 391 4234 1 3 1 11 91 391 391 391 391 182 183 BPAAAA WGGAAA OOOOxx 3035 4235 1 3 5 15 35 35 1035 3035 3035 70 71 TMAAAA XGGAAA VVVVxx 7583 4236 1 3 3 3 83 583 1583 2583 7583 166 167 RFAAAA YGGAAA AAAAxx 1145 4237 1 1 5 5 45 145 1145 1145 1145 90 91 BSAAAA ZGGAAA HHHHxx 93 4238 1 1 3 13 93 93 93 93 93 186 187 PDAAAA AHGAAA OOOOxx 8896 4239 0 0 6 16 96 896 896 3896 8896 192 193 EEAAAA BHGAAA VVVVxx 6719 4240 1 3 9 19 19 719 719 1719 6719 38 39 LYAAAA CHGAAA AAAAxx 7728 4241 0 0 8 8 28 728 1728 2728 7728 56 57 GLAAAA DHGAAA HHHHxx 1349 4242 1 1 9 9 49 349 1349 1349 1349 98 99 XZAAAA EHGAAA OOOOxx 5349 4243 1 1 9 9 49 349 1349 349 5349 98 99 TXAAAA FHGAAA VVVVxx 3040 4244 0 0 0 0 40 40 1040 3040 3040 80 81 YMAAAA GHGAAA AAAAxx 2414 4245 0 2 4 14 14 414 414 2414 2414 28 29 WOAAAA HHGAAA HHHHxx 5122 4246 0 2 2 2 22 122 1122 122 5122 44 45 APAAAA IHGAAA OOOOxx 9553 4247 1 1 3 13 53 553 1553 4553 9553 106 107 LDAAAA JHGAAA VVVVxx 5987 4248 1 3 7 7 87 987 1987 987 5987 174 175 HWAAAA KHGAAA AAAAxx 5939 4249 1 3 9 19 39 939 1939 939 5939 78 79 LUAAAA LHGAAA HHHHxx 3525 4250 1 1 5 5 25 525 1525 3525 3525 50 51 PFAAAA MHGAAA OOOOxx 1371 4251 1 3 1 11 71 371 1371 1371 1371 142 143 TAAAAA NHGAAA VVVVxx 618 4252 0 2 8 18 18 618 618 618 618 36 37 UXAAAA OHGAAA AAAAxx 6529 4253 1 1 9 9 29 529 529 1529 6529 58 59 DRAAAA PHGAAA HHHHxx 4010 4254 0 2 0 10 10 10 10 4010 4010 20 21 GYAAAA QHGAAA OOOOxx 328 4255 0 0 8 8 28 328 328 328 328 56 57 QMAAAA RHGAAA VVVVxx 6121 4256 1 1 1 1 21 121 121 1121 6121 42 43 LBAAAA SHGAAA AAAAxx 3505 4257 1 1 5 5 5 505 1505 3505 3505 10 11 VEAAAA THGAAA HHHHxx 2033 4258 1 1 3 13 33 33 33 2033 2033 66 67 FAAAAA UHGAAA OOOOxx 4724 4259 0 0 4 4 24 724 724 4724 4724 48 49 SZAAAA VHGAAA VVVVxx 8717 4260 1 1 7 17 17 717 717 3717 8717 34 35 HXAAAA WHGAAA AAAAxx 5639 4261 1 3 9 19 39 639 1639 639 5639 78 79 XIAAAA XHGAAA HHHHxx 3448 4262 0 0 8 8 48 448 1448 3448 3448 96 97 QCAAAA YHGAAA OOOOxx 2919 4263 1 3 9 19 19 919 919 2919 2919 38 39 HIAAAA ZHGAAA VVVVxx 3417 4264 1 1 7 17 17 417 1417 3417 3417 34 35 LBAAAA AIGAAA AAAAxx 943 4265 1 3 3 3 43 943 943 943 943 86 87 HKAAAA BIGAAA HHHHxx 775 4266 1 3 5 15 75 775 775 775 775 150 151 VDAAAA CIGAAA OOOOxx 2333 4267 1 1 3 13 33 333 333 2333 2333 66 67 TLAAAA DIGAAA VVVVxx 4801 4268 1 1 1 1 1 801 801 4801 4801 2 3 RCAAAA EIGAAA AAAAxx 7169 4269 1 1 9 9 69 169 1169 2169 7169 138 139 TPAAAA FIGAAA HHHHxx 2840 4270 0 0 0 0 40 840 840 2840 2840 80 81 GFAAAA GIGAAA OOOOxx 9034 4271 0 2 4 14 34 34 1034 4034 9034 68 69 MJAAAA HIGAAA VVVVxx 6154 4272 0 2 4 14 54 154 154 1154 6154 108 109 SCAAAA IIGAAA AAAAxx 1412 4273 0 0 2 12 12 412 1412 1412 1412 24 25 ICAAAA JIGAAA HHHHxx 2263 4274 1 3 3 3 63 263 263 2263 2263 126 127 BJAAAA KIGAAA OOOOxx 7118 4275 0 2 8 18 18 118 1118 2118 7118 36 37 UNAAAA LIGAAA VVVVxx 1526 4276 0 2 6 6 26 526 1526 1526 1526 52 53 SGAAAA MIGAAA AAAAxx 491 4277 1 3 1 11 91 491 491 491 491 182 183 XSAAAA NIGAAA HHHHxx 9732 4278 0 0 2 12 32 732 1732 4732 9732 64 65 IKAAAA OIGAAA OOOOxx 7067 4279 1 3 7 7 67 67 1067 2067 7067 134 135 VLAAAA PIGAAA VVVVxx 212 4280 0 0 2 12 12 212 212 212 212 24 25 EIAAAA QIGAAA AAAAxx 1955 4281 1 3 5 15 55 955 1955 1955 1955 110 111 FXAAAA RIGAAA HHHHxx 3303 4282 1 3 3 3 3 303 1303 3303 3303 6 7 BXAAAA SIGAAA OOOOxx 2715 4283 1 3 5 15 15 715 715 2715 2715 30 31 LAAAAA TIGAAA VVVVxx 8168 4284 0 0 8 8 68 168 168 3168 8168 136 137 ECAAAA UIGAAA AAAAxx 6799 4285 1 3 9 19 99 799 799 1799 6799 198 199 NBAAAA VIGAAA HHHHxx 5080 4286 0 0 0 0 80 80 1080 80 5080 160 161 KNAAAA WIGAAA OOOOxx 4939 4287 1 3 9 19 39 939 939 4939 4939 78 79 ZHAAAA XIGAAA VVVVxx 6604 4288 0 0 4 4 4 604 604 1604 6604 8 9 AUAAAA YIGAAA AAAAxx 6531 4289 1 3 1 11 31 531 531 1531 6531 62 63 FRAAAA ZIGAAA HHHHxx 9948 4290 0 0 8 8 48 948 1948 4948 9948 96 97 QSAAAA AJGAAA OOOOxx 7923 4291 1 3 3 3 23 923 1923 2923 7923 46 47 TSAAAA BJGAAA VVVVxx 9905 4292 1 1 5 5 5 905 1905 4905 9905 10 11 ZQAAAA CJGAAA AAAAxx 340 4293 0 0 0 0 40 340 340 340 340 80 81 CNAAAA DJGAAA HHHHxx 1721 4294 1 1 1 1 21 721 1721 1721 1721 42 43 FOAAAA EJGAAA OOOOxx 9047 4295 1 3 7 7 47 47 1047 4047 9047 94 95 ZJAAAA FJGAAA VVVVxx 4723 4296 1 3 3 3 23 723 723 4723 4723 46 47 RZAAAA GJGAAA AAAAxx 5748 4297 0 0 8 8 48 748 1748 748 5748 96 97 CNAAAA HJGAAA HHHHxx 6845 4298 1 1 5 5 45 845 845 1845 6845 90 91 HDAAAA IJGAAA OOOOxx 1556 4299 0 0 6 16 56 556 1556 1556 1556 112 113 WHAAAA JJGAAA VVVVxx 9505 4300 1 1 5 5 5 505 1505 4505 9505 10 11 PBAAAA KJGAAA AAAAxx 3573 4301 1 1 3 13 73 573 1573 3573 3573 146 147 LHAAAA LJGAAA HHHHxx 3785 4302 1 1 5 5 85 785 1785 3785 3785 170 171 PPAAAA MJGAAA OOOOxx 2772 4303 0 0 2 12 72 772 772 2772 2772 144 145 QCAAAA NJGAAA VVVVxx 7282 4304 0 2 2 2 82 282 1282 2282 7282 164 165 CUAAAA OJGAAA AAAAxx 8106 4305 0 2 6 6 6 106 106 3106 8106 12 13 UZAAAA PJGAAA HHHHxx 2847 4306 1 3 7 7 47 847 847 2847 2847 94 95 NFAAAA QJGAAA OOOOxx 9803 4307 1 3 3 3 3 803 1803 4803 9803 6 7 BNAAAA RJGAAA VVVVxx 7719 4308 1 3 9 19 19 719 1719 2719 7719 38 39 XKAAAA SJGAAA AAAAxx 4649 4309 1 1 9 9 49 649 649 4649 4649 98 99 VWAAAA TJGAAA HHHHxx 6196 4310 0 0 6 16 96 196 196 1196 6196 192 193 IEAAAA UJGAAA OOOOxx 6026 4311 0 2 6 6 26 26 26 1026 6026 52 53 UXAAAA VJGAAA VVVVxx 1646 4312 0 2 6 6 46 646 1646 1646 1646 92 93 ILAAAA WJGAAA AAAAxx 6526 4313 0 2 6 6 26 526 526 1526 6526 52 53 ARAAAA XJGAAA HHHHxx 5110 4314 0 2 0 10 10 110 1110 110 5110 20 21 OOAAAA YJGAAA OOOOxx 3946 4315 0 2 6 6 46 946 1946 3946 3946 92 93 UVAAAA ZJGAAA VVVVxx 445 4316 1 1 5 5 45 445 445 445 445 90 91 DRAAAA AKGAAA AAAAxx 3249 4317 1 1 9 9 49 249 1249 3249 3249 98 99 ZUAAAA BKGAAA HHHHxx 2501 4318 1 1 1 1 1 501 501 2501 2501 2 3 FSAAAA CKGAAA OOOOxx 3243 4319 1 3 3 3 43 243 1243 3243 3243 86 87 TUAAAA DKGAAA VVVVxx 4701 4320 1 1 1 1 1 701 701 4701 4701 2 3 VYAAAA EKGAAA AAAAxx 472 4321 0 0 2 12 72 472 472 472 472 144 145 ESAAAA FKGAAA HHHHxx 3356 4322 0 0 6 16 56 356 1356 3356 3356 112 113 CZAAAA GKGAAA OOOOxx 9967 4323 1 3 7 7 67 967 1967 4967 9967 134 135 JTAAAA HKGAAA VVVVxx 4292 4324 0 0 2 12 92 292 292 4292 4292 184 185 CJAAAA IKGAAA AAAAxx 7005 4325 1 1 5 5 5 5 1005 2005 7005 10 11 LJAAAA JKGAAA HHHHxx 6267 4326 1 3 7 7 67 267 267 1267 6267 134 135 BHAAAA KKGAAA OOOOxx 6678 4327 0 2 8 18 78 678 678 1678 6678 156 157 WWAAAA LKGAAA VVVVxx 6083 4328 1 3 3 3 83 83 83 1083 6083 166 167 ZZAAAA MKGAAA AAAAxx 760 4329 0 0 0 0 60 760 760 760 760 120 121 GDAAAA NKGAAA HHHHxx 7833 4330 1 1 3 13 33 833 1833 2833 7833 66 67 HPAAAA OKGAAA OOOOxx 2877 4331 1 1 7 17 77 877 877 2877 2877 154 155 RGAAAA PKGAAA VVVVxx 8810 4332 0 2 0 10 10 810 810 3810 8810 20 21 WAAAAA QKGAAA AAAAxx 1560 4333 0 0 0 0 60 560 1560 1560 1560 120 121 AIAAAA RKGAAA HHHHxx 1367 4334 1 3 7 7 67 367 1367 1367 1367 134 135 PAAAAA SKGAAA OOOOxx 8756 4335 0 0 6 16 56 756 756 3756 8756 112 113 UYAAAA TKGAAA VVVVxx 1346 4336 0 2 6 6 46 346 1346 1346 1346 92 93 UZAAAA UKGAAA AAAAxx 6449 4337 1 1 9 9 49 449 449 1449 6449 98 99 BOAAAA VKGAAA HHHHxx 6658 4338 0 2 8 18 58 658 658 1658 6658 116 117 CWAAAA WKGAAA OOOOxx 6745 4339 1 1 5 5 45 745 745 1745 6745 90 91 LZAAAA XKGAAA VVVVxx 4866 4340 0 2 6 6 66 866 866 4866 4866 132 133 EFAAAA YKGAAA AAAAxx 14 4341 0 2 4 14 14 14 14 14 14 28 29 OAAAAA ZKGAAA HHHHxx 4506 4342 0 2 6 6 6 506 506 4506 4506 12 13 IRAAAA ALGAAA OOOOxx 1923 4343 1 3 3 3 23 923 1923 1923 1923 46 47 ZVAAAA BLGAAA VVVVxx 8365 4344 1 1 5 5 65 365 365 3365 8365 130 131 TJAAAA CLGAAA AAAAxx 1279 4345 1 3 9 19 79 279 1279 1279 1279 158 159 FXAAAA DLGAAA HHHHxx 7666 4346 0 2 6 6 66 666 1666 2666 7666 132 133 WIAAAA ELGAAA OOOOxx 7404 4347 0 0 4 4 4 404 1404 2404 7404 8 9 UYAAAA FLGAAA VVVVxx 65 4348 1 1 5 5 65 65 65 65 65 130 131 NCAAAA GLGAAA AAAAxx 5820 4349 0 0 0 0 20 820 1820 820 5820 40 41 WPAAAA HLGAAA HHHHxx 459 4350 1 3 9 19 59 459 459 459 459 118 119 RRAAAA ILGAAA OOOOxx 4787 4351 1 3 7 7 87 787 787 4787 4787 174 175 DCAAAA JLGAAA VVVVxx 5631 4352 1 3 1 11 31 631 1631 631 5631 62 63 PIAAAA KLGAAA AAAAxx 9717 4353 1 1 7 17 17 717 1717 4717 9717 34 35 TJAAAA LLGAAA HHHHxx 2560 4354 0 0 0 0 60 560 560 2560 2560 120 121 MUAAAA MLGAAA OOOOxx 8295 4355 1 3 5 15 95 295 295 3295 8295 190 191 BHAAAA NLGAAA VVVVxx 3596 4356 0 0 6 16 96 596 1596 3596 3596 192 193 IIAAAA OLGAAA AAAAxx 2023 4357 1 3 3 3 23 23 23 2023 2023 46 47 VZAAAA PLGAAA HHHHxx 5055 4358 1 3 5 15 55 55 1055 55 5055 110 111 LMAAAA QLGAAA OOOOxx 763 4359 1 3 3 3 63 763 763 763 763 126 127 JDAAAA RLGAAA VVVVxx 6733 4360 1 1 3 13 33 733 733 1733 6733 66 67 ZYAAAA SLGAAA AAAAxx 9266 4361 0 2 6 6 66 266 1266 4266 9266 132 133 KSAAAA TLGAAA HHHHxx 4479 4362 1 3 9 19 79 479 479 4479 4479 158 159 HQAAAA ULGAAA OOOOxx 1816 4363 0 0 6 16 16 816 1816 1816 1816 32 33 WRAAAA VLGAAA VVVVxx 899 4364 1 3 9 19 99 899 899 899 899 198 199 PIAAAA WLGAAA AAAAxx 230 4365 0 2 0 10 30 230 230 230 230 60 61 WIAAAA XLGAAA HHHHxx 5362 4366 0 2 2 2 62 362 1362 362 5362 124 125 GYAAAA YLGAAA OOOOxx 1609 4367 1 1 9 9 9 609 1609 1609 1609 18 19 XJAAAA ZLGAAA VVVVxx 6750 4368 0 2 0 10 50 750 750 1750 6750 100 101 QZAAAA AMGAAA AAAAxx 9704 4369 0 0 4 4 4 704 1704 4704 9704 8 9 GJAAAA BMGAAA HHHHxx 3991 4370 1 3 1 11 91 991 1991 3991 3991 182 183 NXAAAA CMGAAA OOOOxx 3959 4371 1 3 9 19 59 959 1959 3959 3959 118 119 HWAAAA DMGAAA VVVVxx 9021 4372 1 1 1 1 21 21 1021 4021 9021 42 43 ZIAAAA EMGAAA AAAAxx 7585 4373 1 1 5 5 85 585 1585 2585 7585 170 171 TFAAAA FMGAAA HHHHxx 7083 4374 1 3 3 3 83 83 1083 2083 7083 166 167 LMAAAA GMGAAA OOOOxx 7688 4375 0 0 8 8 88 688 1688 2688 7688 176 177 SJAAAA HMGAAA VVVVxx 2673 4376 1 1 3 13 73 673 673 2673 2673 146 147 VYAAAA IMGAAA AAAAxx 3554 4377 0 2 4 14 54 554 1554 3554 3554 108 109 SGAAAA JMGAAA HHHHxx 7416 4378 0 0 6 16 16 416 1416 2416 7416 32 33 GZAAAA KMGAAA OOOOxx 5672 4379 0 0 2 12 72 672 1672 672 5672 144 145 EKAAAA LMGAAA VVVVxx 1355 4380 1 3 5 15 55 355 1355 1355 1355 110 111 DAAAAA MMGAAA AAAAxx 3149 4381 1 1 9 9 49 149 1149 3149 3149 98 99 DRAAAA NMGAAA HHHHxx 5811 4382 1 3 1 11 11 811 1811 811 5811 22 23 NPAAAA OMGAAA OOOOxx 3759 4383 1 3 9 19 59 759 1759 3759 3759 118 119 POAAAA PMGAAA VVVVxx 5634 4384 0 2 4 14 34 634 1634 634 5634 68 69 SIAAAA QMGAAA AAAAxx 8617 4385 1 1 7 17 17 617 617 3617 8617 34 35 LTAAAA RMGAAA HHHHxx 8949 4386 1 1 9 9 49 949 949 3949 8949 98 99 FGAAAA SMGAAA OOOOxx 3964 4387 0 0 4 4 64 964 1964 3964 3964 128 129 MWAAAA TMGAAA VVVVxx 3852 4388 0 0 2 12 52 852 1852 3852 3852 104 105 ESAAAA UMGAAA AAAAxx 1555 4389 1 3 5 15 55 555 1555 1555 1555 110 111 VHAAAA VMGAAA HHHHxx 6536 4390 0 0 6 16 36 536 536 1536 6536 72 73 KRAAAA WMGAAA OOOOxx 4779 4391 1 3 9 19 79 779 779 4779 4779 158 159 VBAAAA XMGAAA VVVVxx 1893 4392 1 1 3 13 93 893 1893 1893 1893 186 187 VUAAAA YMGAAA AAAAxx 9358 4393 0 2 8 18 58 358 1358 4358 9358 116 117 YVAAAA ZMGAAA HHHHxx 7438 4394 0 2 8 18 38 438 1438 2438 7438 76 77 CAAAAA ANGAAA OOOOxx 941 4395 1 1 1 1 41 941 941 941 941 82 83 FKAAAA BNGAAA VVVVxx 4844 4396 0 0 4 4 44 844 844 4844 4844 88 89 IEAAAA CNGAAA AAAAxx 4745 4397 1 1 5 5 45 745 745 4745 4745 90 91 NAAAAA DNGAAA HHHHxx 1017 4398 1 1 7 17 17 17 1017 1017 1017 34 35 DNAAAA ENGAAA OOOOxx 327 4399 1 3 7 7 27 327 327 327 327 54 55 PMAAAA FNGAAA VVVVxx 3152 4400 0 0 2 12 52 152 1152 3152 3152 104 105 GRAAAA GNGAAA AAAAxx 4711 4401 1 3 1 11 11 711 711 4711 4711 22 23 FZAAAA HNGAAA HHHHxx 141 4402 1 1 1 1 41 141 141 141 141 82 83 LFAAAA INGAAA OOOOxx 1303 4403 1 3 3 3 3 303 1303 1303 1303 6 7 DYAAAA JNGAAA VVVVxx 8873 4404 1 1 3 13 73 873 873 3873 8873 146 147 HDAAAA KNGAAA AAAAxx 8481 4405 1 1 1 1 81 481 481 3481 8481 162 163 FOAAAA LNGAAA HHHHxx 5445 4406 1 1 5 5 45 445 1445 445 5445 90 91 LBAAAA MNGAAA OOOOxx 7868 4407 0 0 8 8 68 868 1868 2868 7868 136 137 QQAAAA NNGAAA VVVVxx 6722 4408 0 2 2 2 22 722 722 1722 6722 44 45 OYAAAA ONGAAA AAAAxx 6628 4409 0 0 8 8 28 628 628 1628 6628 56 57 YUAAAA PNGAAA HHHHxx 7738 4410 0 2 8 18 38 738 1738 2738 7738 76 77 QLAAAA QNGAAA OOOOxx 1018 4411 0 2 8 18 18 18 1018 1018 1018 36 37 ENAAAA RNGAAA VVVVxx 3296 4412 0 0 6 16 96 296 1296 3296 3296 192 193 UWAAAA SNGAAA AAAAxx 1946 4413 0 2 6 6 46 946 1946 1946 1946 92 93 WWAAAA TNGAAA HHHHxx 6603 4414 1 3 3 3 3 603 603 1603 6603 6 7 ZTAAAA UNGAAA OOOOxx 3562 4415 0 2 2 2 62 562 1562 3562 3562 124 125 AHAAAA VNGAAA VVVVxx 1147 4416 1 3 7 7 47 147 1147 1147 1147 94 95 DSAAAA WNGAAA AAAAxx 6031 4417 1 3 1 11 31 31 31 1031 6031 62 63 ZXAAAA XNGAAA HHHHxx 6484 4418 0 0 4 4 84 484 484 1484 6484 168 169 KPAAAA YNGAAA OOOOxx 496 4419 0 0 6 16 96 496 496 496 496 192 193 CTAAAA ZNGAAA VVVVxx 4563 4420 1 3 3 3 63 563 563 4563 4563 126 127 NTAAAA AOGAAA AAAAxx 1037 4421 1 1 7 17 37 37 1037 1037 1037 74 75 XNAAAA BOGAAA HHHHxx 9672 4422 0 0 2 12 72 672 1672 4672 9672 144 145 AIAAAA COGAAA OOOOxx 9053 4423 1 1 3 13 53 53 1053 4053 9053 106 107 FKAAAA DOGAAA VVVVxx 2523 4424 1 3 3 3 23 523 523 2523 2523 46 47 BTAAAA EOGAAA AAAAxx 8519 4425 1 3 9 19 19 519 519 3519 8519 38 39 RPAAAA FOGAAA HHHHxx 8190 4426 0 2 0 10 90 190 190 3190 8190 180 181 ADAAAA GOGAAA OOOOxx 2068 4427 0 0 8 8 68 68 68 2068 2068 136 137 OBAAAA HOGAAA VVVVxx 8569 4428 1 1 9 9 69 569 569 3569 8569 138 139 PRAAAA IOGAAA AAAAxx 6535 4429 1 3 5 15 35 535 535 1535 6535 70 71 JRAAAA JOGAAA HHHHxx 1810 4430 0 2 0 10 10 810 1810 1810 1810 20 21 QRAAAA KOGAAA OOOOxx 3099 4431 1 3 9 19 99 99 1099 3099 3099 198 199 FPAAAA LOGAAA VVVVxx 7466 4432 0 2 6 6 66 466 1466 2466 7466 132 133 EBAAAA MOGAAA AAAAxx 4017 4433 1 1 7 17 17 17 17 4017 4017 34 35 NYAAAA NOGAAA HHHHxx 1097 4434 1 1 7 17 97 97 1097 1097 1097 194 195 FQAAAA OOGAAA OOOOxx 7686 4435 0 2 6 6 86 686 1686 2686 7686 172 173 QJAAAA POGAAA VVVVxx 6742 4436 0 2 2 2 42 742 742 1742 6742 84 85 IZAAAA QOGAAA AAAAxx 5966 4437 0 2 6 6 66 966 1966 966 5966 132 133 MVAAAA ROGAAA HHHHxx 3632 4438 0 0 2 12 32 632 1632 3632 3632 64 65 SJAAAA SOGAAA OOOOxx 8837 4439 1 1 7 17 37 837 837 3837 8837 74 75 XBAAAA TOGAAA VVVVxx 1667 4440 1 3 7 7 67 667 1667 1667 1667 134 135 DMAAAA UOGAAA AAAAxx 8833 4441 1 1 3 13 33 833 833 3833 8833 66 67 TBAAAA VOGAAA HHHHxx 9805 4442 1 1 5 5 5 805 1805 4805 9805 10 11 DNAAAA WOGAAA OOOOxx 3650 4443 0 2 0 10 50 650 1650 3650 3650 100 101 KKAAAA XOGAAA VVVVxx 2237 4444 1 1 7 17 37 237 237 2237 2237 74 75 BIAAAA YOGAAA AAAAxx 9980 4445 0 0 0 0 80 980 1980 4980 9980 160 161 WTAAAA ZOGAAA HHHHxx 2861 4446 1 1 1 1 61 861 861 2861 2861 122 123 BGAAAA APGAAA OOOOxx 1334 4447 0 2 4 14 34 334 1334 1334 1334 68 69 IZAAAA BPGAAA VVVVxx 842 4448 0 2 2 2 42 842 842 842 842 84 85 KGAAAA CPGAAA AAAAxx 1116 4449 0 0 6 16 16 116 1116 1116 1116 32 33 YQAAAA DPGAAA HHHHxx 4055 4450 1 3 5 15 55 55 55 4055 4055 110 111 ZZAAAA EPGAAA OOOOxx 3842 4451 0 2 2 2 42 842 1842 3842 3842 84 85 URAAAA FPGAAA VVVVxx 1886 4452 0 2 6 6 86 886 1886 1886 1886 172 173 OUAAAA GPGAAA AAAAxx 8589 4453 1 1 9 9 89 589 589 3589 8589 178 179 JSAAAA HPGAAA HHHHxx 5873 4454 1 1 3 13 73 873 1873 873 5873 146 147 XRAAAA IPGAAA OOOOxx 7711 4455 1 3 1 11 11 711 1711 2711 7711 22 23 PKAAAA JPGAAA VVVVxx 911 4456 1 3 1 11 11 911 911 911 911 22 23 BJAAAA KPGAAA AAAAxx 5837 4457 1 1 7 17 37 837 1837 837 5837 74 75 NQAAAA LPGAAA HHHHxx 897 4458 1 1 7 17 97 897 897 897 897 194 195 NIAAAA MPGAAA OOOOxx 4299 4459 1 3 9 19 99 299 299 4299 4299 198 199 JJAAAA NPGAAA VVVVxx 7774 4460 0 2 4 14 74 774 1774 2774 7774 148 149 ANAAAA OPGAAA AAAAxx 7832 4461 0 0 2 12 32 832 1832 2832 7832 64 65 GPAAAA PPGAAA HHHHxx 9915 4462 1 3 5 15 15 915 1915 4915 9915 30 31 JRAAAA QPGAAA OOOOxx 9 4463 1 1 9 9 9 9 9 9 9 18 19 JAAAAA RPGAAA VVVVxx 9675 4464 1 3 5 15 75 675 1675 4675 9675 150 151 DIAAAA SPGAAA AAAAxx 7953 4465 1 1 3 13 53 953 1953 2953 7953 106 107 XTAAAA TPGAAA HHHHxx 8912 4466 0 0 2 12 12 912 912 3912 8912 24 25 UEAAAA UPGAAA OOOOxx 4188 4467 0 0 8 8 88 188 188 4188 4188 176 177 CFAAAA VPGAAA VVVVxx 8446 4468 0 2 6 6 46 446 446 3446 8446 92 93 WMAAAA WPGAAA AAAAxx 1600 4469 0 0 0 0 0 600 1600 1600 1600 0 1 OJAAAA XPGAAA HHHHxx 43 4470 1 3 3 3 43 43 43 43 43 86 87 RBAAAA YPGAAA OOOOxx 544 4471 0 0 4 4 44 544 544 544 544 88 89 YUAAAA ZPGAAA VVVVxx 6977 4472 1 1 7 17 77 977 977 1977 6977 154 155 JIAAAA AQGAAA AAAAxx 3191 4473 1 3 1 11 91 191 1191 3191 3191 182 183 TSAAAA BQGAAA HHHHxx 418 4474 0 2 8 18 18 418 418 418 418 36 37 CQAAAA CQGAAA OOOOxx 3142 4475 0 2 2 2 42 142 1142 3142 3142 84 85 WQAAAA DQGAAA VVVVxx 5042 4476 0 2 2 2 42 42 1042 42 5042 84 85 YLAAAA EQGAAA AAAAxx 2194 4477 0 2 4 14 94 194 194 2194 2194 188 189 KGAAAA FQGAAA HHHHxx 2397 4478 1 1 7 17 97 397 397 2397 2397 194 195 FOAAAA GQGAAA OOOOxx 4684 4479 0 0 4 4 84 684 684 4684 4684 168 169 EYAAAA HQGAAA VVVVxx 34 4480 0 2 4 14 34 34 34 34 34 68 69 IBAAAA IQGAAA AAAAxx 3844 4481 0 0 4 4 44 844 1844 3844 3844 88 89 WRAAAA JQGAAA HHHHxx 7824 4482 0 0 4 4 24 824 1824 2824 7824 48 49 YOAAAA KQGAAA OOOOxx 6177 4483 1 1 7 17 77 177 177 1177 6177 154 155 PDAAAA LQGAAA VVVVxx 9657 4484 1 1 7 17 57 657 1657 4657 9657 114 115 LHAAAA MQGAAA AAAAxx 4546 4485 0 2 6 6 46 546 546 4546 4546 92 93 WSAAAA NQGAAA HHHHxx 599 4486 1 3 9 19 99 599 599 599 599 198 199 BXAAAA OQGAAA OOOOxx 153 4487 1 1 3 13 53 153 153 153 153 106 107 XFAAAA PQGAAA VVVVxx 6910 4488 0 2 0 10 10 910 910 1910 6910 20 21 UFAAAA QQGAAA AAAAxx 4408 4489 0 0 8 8 8 408 408 4408 4408 16 17 ONAAAA RQGAAA HHHHxx 1164 4490 0 0 4 4 64 164 1164 1164 1164 128 129 USAAAA SQGAAA OOOOxx 6469 4491 1 1 9 9 69 469 469 1469 6469 138 139 VOAAAA TQGAAA VVVVxx 5996 4492 0 0 6 16 96 996 1996 996 5996 192 193 QWAAAA UQGAAA AAAAxx 2639 4493 1 3 9 19 39 639 639 2639 2639 78 79 NXAAAA VQGAAA HHHHxx 2678 4494 0 2 8 18 78 678 678 2678 2678 156 157 AZAAAA WQGAAA OOOOxx 8392 4495 0 0 2 12 92 392 392 3392 8392 184 185 UKAAAA XQGAAA VVVVxx 1386 4496 0 2 6 6 86 386 1386 1386 1386 172 173 IBAAAA YQGAAA AAAAxx 5125 4497 1 1 5 5 25 125 1125 125 5125 50 51 DPAAAA ZQGAAA HHHHxx 8453 4498 1 1 3 13 53 453 453 3453 8453 106 107 DNAAAA ARGAAA OOOOxx 2369 4499 1 1 9 9 69 369 369 2369 2369 138 139 DNAAAA BRGAAA VVVVxx 1608 4500 0 0 8 8 8 608 1608 1608 1608 16 17 WJAAAA CRGAAA AAAAxx 3781 4501 1 1 1 1 81 781 1781 3781 3781 162 163 LPAAAA DRGAAA HHHHxx 903 4502 1 3 3 3 3 903 903 903 903 6 7 TIAAAA ERGAAA OOOOxx 2099 4503 1 3 9 19 99 99 99 2099 2099 198 199 TCAAAA FRGAAA VVVVxx 538 4504 0 2 8 18 38 538 538 538 538 76 77 SUAAAA GRGAAA AAAAxx 9177 4505 1 1 7 17 77 177 1177 4177 9177 154 155 ZOAAAA HRGAAA HHHHxx 420 4506 0 0 0 0 20 420 420 420 420 40 41 EQAAAA IRGAAA OOOOxx 9080 4507 0 0 0 0 80 80 1080 4080 9080 160 161 GLAAAA JRGAAA VVVVxx 2630 4508 0 2 0 10 30 630 630 2630 2630 60 61 EXAAAA KRGAAA AAAAxx 5978 4509 0 2 8 18 78 978 1978 978 5978 156 157 YVAAAA LRGAAA HHHHxx 9239 4510 1 3 9 19 39 239 1239 4239 9239 78 79 JRAAAA MRGAAA OOOOxx 4372 4511 0 0 2 12 72 372 372 4372 4372 144 145 EMAAAA NRGAAA VVVVxx 4357 4512 1 1 7 17 57 357 357 4357 4357 114 115 PLAAAA ORGAAA AAAAxx 9857 4513 1 1 7 17 57 857 1857 4857 9857 114 115 DPAAAA PRGAAA HHHHxx 7933 4514 1 1 3 13 33 933 1933 2933 7933 66 67 DTAAAA QRGAAA OOOOxx 9574 4515 0 2 4 14 74 574 1574 4574 9574 148 149 GEAAAA RRGAAA VVVVxx 8294 4516 0 2 4 14 94 294 294 3294 8294 188 189 AHAAAA SRGAAA AAAAxx 627 4517 1 3 7 7 27 627 627 627 627 54 55 DYAAAA TRGAAA HHHHxx 3229 4518 1 1 9 9 29 229 1229 3229 3229 58 59 FUAAAA URGAAA OOOOxx 3163 4519 1 3 3 3 63 163 1163 3163 3163 126 127 RRAAAA VRGAAA VVVVxx 7349 4520 1 1 9 9 49 349 1349 2349 7349 98 99 RWAAAA WRGAAA AAAAxx 6889 4521 1 1 9 9 89 889 889 1889 6889 178 179 ZEAAAA XRGAAA HHHHxx 2101 4522 1 1 1 1 1 101 101 2101 2101 2 3 VCAAAA YRGAAA OOOOxx 6476 4523 0 0 6 16 76 476 476 1476 6476 152 153 CPAAAA ZRGAAA VVVVxx 6765 4524 1 1 5 5 65 765 765 1765 6765 130 131 FAAAAA ASGAAA AAAAxx 4204 4525 0 0 4 4 4 204 204 4204 4204 8 9 SFAAAA BSGAAA HHHHxx 5915 4526 1 3 5 15 15 915 1915 915 5915 30 31 NTAAAA CSGAAA OOOOxx 2318 4527 0 2 8 18 18 318 318 2318 2318 36 37 ELAAAA DSGAAA VVVVxx 294 4528 0 2 4 14 94 294 294 294 294 188 189 ILAAAA ESGAAA AAAAxx 5245 4529 1 1 5 5 45 245 1245 245 5245 90 91 TTAAAA FSGAAA HHHHxx 4481 4530 1 1 1 1 81 481 481 4481 4481 162 163 JQAAAA GSGAAA OOOOxx 7754 4531 0 2 4 14 54 754 1754 2754 7754 108 109 GMAAAA HSGAAA VVVVxx 8494 4532 0 2 4 14 94 494 494 3494 8494 188 189 SOAAAA ISGAAA AAAAxx 4014 4533 0 2 4 14 14 14 14 4014 4014 28 29 KYAAAA JSGAAA HHHHxx 2197 4534 1 1 7 17 97 197 197 2197 2197 194 195 NGAAAA KSGAAA OOOOxx 1297 4535 1 1 7 17 97 297 1297 1297 1297 194 195 XXAAAA LSGAAA VVVVxx 1066 4536 0 2 6 6 66 66 1066 1066 1066 132 133 APAAAA MSGAAA AAAAxx 5710 4537 0 2 0 10 10 710 1710 710 5710 20 21 QLAAAA NSGAAA HHHHxx 4100 4538 0 0 0 0 0 100 100 4100 4100 0 1 SBAAAA OSGAAA OOOOxx 7356 4539 0 0 6 16 56 356 1356 2356 7356 112 113 YWAAAA PSGAAA VVVVxx 7658 4540 0 2 8 18 58 658 1658 2658 7658 116 117 OIAAAA QSGAAA AAAAxx 3666 4541 0 2 6 6 66 666 1666 3666 3666 132 133 ALAAAA RSGAAA HHHHxx 9713 4542 1 1 3 13 13 713 1713 4713 9713 26 27 PJAAAA SSGAAA OOOOxx 691 4543 1 3 1 11 91 691 691 691 691 182 183 PAAAAA TSGAAA VVVVxx 3112 4544 0 0 2 12 12 112 1112 3112 3112 24 25 SPAAAA USGAAA AAAAxx 6035 4545 1 3 5 15 35 35 35 1035 6035 70 71 DYAAAA VSGAAA HHHHxx 8353 4546 1 1 3 13 53 353 353 3353 8353 106 107 HJAAAA WSGAAA OOOOxx 5679 4547 1 3 9 19 79 679 1679 679 5679 158 159 LKAAAA XSGAAA VVVVxx 2124 4548 0 0 4 4 24 124 124 2124 2124 48 49 SDAAAA YSGAAA AAAAxx 4714 4549 0 2 4 14 14 714 714 4714 4714 28 29 IZAAAA ZSGAAA HHHHxx 9048 4550 0 0 8 8 48 48 1048 4048 9048 96 97 AKAAAA ATGAAA OOOOxx 7692 4551 0 0 2 12 92 692 1692 2692 7692 184 185 WJAAAA BTGAAA VVVVxx 4542 4552 0 2 2 2 42 542 542 4542 4542 84 85 SSAAAA CTGAAA AAAAxx 8737 4553 1 1 7 17 37 737 737 3737 8737 74 75 BYAAAA DTGAAA HHHHxx 4977 4554 1 1 7 17 77 977 977 4977 4977 154 155 LJAAAA ETGAAA OOOOxx 9349 4555 1 1 9 9 49 349 1349 4349 9349 98 99 PVAAAA FTGAAA VVVVxx 731 4556 1 3 1 11 31 731 731 731 731 62 63 DCAAAA GTGAAA AAAAxx 1788 4557 0 0 8 8 88 788 1788 1788 1788 176 177 UQAAAA HTGAAA HHHHxx 7830 4558 0 2 0 10 30 830 1830 2830 7830 60 61 EPAAAA ITGAAA OOOOxx 3977 4559 1 1 7 17 77 977 1977 3977 3977 154 155 ZWAAAA JTGAAA VVVVxx 2421 4560 1 1 1 1 21 421 421 2421 2421 42 43 DPAAAA KTGAAA AAAAxx 5891 4561 1 3 1 11 91 891 1891 891 5891 182 183 PSAAAA LTGAAA HHHHxx 1111 4562 1 3 1 11 11 111 1111 1111 1111 22 23 TQAAAA MTGAAA OOOOxx 9224 4563 0 0 4 4 24 224 1224 4224 9224 48 49 UQAAAA NTGAAA VVVVxx 9872 4564 0 0 2 12 72 872 1872 4872 9872 144 145 SPAAAA OTGAAA AAAAxx 2433 4565 1 1 3 13 33 433 433 2433 2433 66 67 PPAAAA PTGAAA HHHHxx 1491 4566 1 3 1 11 91 491 1491 1491 1491 182 183 JFAAAA QTGAAA OOOOxx 6653 4567 1 1 3 13 53 653 653 1653 6653 106 107 XVAAAA RTGAAA VVVVxx 1907 4568 1 3 7 7 7 907 1907 1907 1907 14 15 JVAAAA STGAAA AAAAxx 889 4569 1 1 9 9 89 889 889 889 889 178 179 FIAAAA TTGAAA HHHHxx 561 4570 1 1 1 1 61 561 561 561 561 122 123 PVAAAA UTGAAA OOOOxx 7415 4571 1 3 5 15 15 415 1415 2415 7415 30 31 FZAAAA VTGAAA VVVVxx 2703 4572 1 3 3 3 3 703 703 2703 2703 6 7 ZZAAAA WTGAAA AAAAxx 2561 4573 1 1 1 1 61 561 561 2561 2561 122 123 NUAAAA XTGAAA HHHHxx 1257 4574 1 1 7 17 57 257 1257 1257 1257 114 115 JWAAAA YTGAAA OOOOxx 2390 4575 0 2 0 10 90 390 390 2390 2390 180 181 YNAAAA ZTGAAA VVVVxx 3915 4576 1 3 5 15 15 915 1915 3915 3915 30 31 PUAAAA AUGAAA AAAAxx 8476 4577 0 0 6 16 76 476 476 3476 8476 152 153 AOAAAA BUGAAA HHHHxx 607 4578 1 3 7 7 7 607 607 607 607 14 15 JXAAAA CUGAAA OOOOxx 3891 4579 1 3 1 11 91 891 1891 3891 3891 182 183 RTAAAA DUGAAA VVVVxx 7269 4580 1 1 9 9 69 269 1269 2269 7269 138 139 PTAAAA EUGAAA AAAAxx 9537 4581 1 1 7 17 37 537 1537 4537 9537 74 75 VCAAAA FUGAAA HHHHxx 8518 4582 0 2 8 18 18 518 518 3518 8518 36 37 QPAAAA GUGAAA OOOOxx 5221 4583 1 1 1 1 21 221 1221 221 5221 42 43 VSAAAA HUGAAA VVVVxx 3274 4584 0 2 4 14 74 274 1274 3274 3274 148 149 YVAAAA IUGAAA AAAAxx 6677 4585 1 1 7 17 77 677 677 1677 6677 154 155 VWAAAA JUGAAA HHHHxx 3114 4586 0 2 4 14 14 114 1114 3114 3114 28 29 UPAAAA KUGAAA OOOOxx 1966 4587 0 2 6 6 66 966 1966 1966 1966 132 133 QXAAAA LUGAAA VVVVxx 5941 4588 1 1 1 1 41 941 1941 941 5941 82 83 NUAAAA MUGAAA AAAAxx 9463 4589 1 3 3 3 63 463 1463 4463 9463 126 127 ZZAAAA NUGAAA HHHHxx 8966 4590 0 2 6 6 66 966 966 3966 8966 132 133 WGAAAA OUGAAA OOOOxx 4402 4591 0 2 2 2 2 402 402 4402 4402 4 5 INAAAA PUGAAA VVVVxx 3364 4592 0 0 4 4 64 364 1364 3364 3364 128 129 KZAAAA QUGAAA AAAAxx 3698 4593 0 2 8 18 98 698 1698 3698 3698 196 197 GMAAAA RUGAAA HHHHxx 4651 4594 1 3 1 11 51 651 651 4651 4651 102 103 XWAAAA SUGAAA OOOOxx 2127 4595 1 3 7 7 27 127 127 2127 2127 54 55 VDAAAA TUGAAA VVVVxx 3614 4596 0 2 4 14 14 614 1614 3614 3614 28 29 AJAAAA UUGAAA AAAAxx 5430 4597 0 2 0 10 30 430 1430 430 5430 60 61 WAAAAA VUGAAA HHHHxx 3361 4598 1 1 1 1 61 361 1361 3361 3361 122 123 HZAAAA WUGAAA OOOOxx 4798 4599 0 2 8 18 98 798 798 4798 4798 196 197 OCAAAA XUGAAA VVVVxx 8269 4600 1 1 9 9 69 269 269 3269 8269 138 139 BGAAAA YUGAAA AAAAxx 6458 4601 0 2 8 18 58 458 458 1458 6458 116 117 KOAAAA ZUGAAA HHHHxx 3358 4602 0 2 8 18 58 358 1358 3358 3358 116 117 EZAAAA AVGAAA OOOOxx 5898 4603 0 2 8 18 98 898 1898 898 5898 196 197 WSAAAA BVGAAA VVVVxx 1880 4604 0 0 0 0 80 880 1880 1880 1880 160 161 IUAAAA CVGAAA AAAAxx 782 4605 0 2 2 2 82 782 782 782 782 164 165 CEAAAA DVGAAA HHHHxx 3102 4606 0 2 2 2 2 102 1102 3102 3102 4 5 IPAAAA EVGAAA OOOOxx 6366 4607 0 2 6 6 66 366 366 1366 6366 132 133 WKAAAA FVGAAA VVVVxx 399 4608 1 3 9 19 99 399 399 399 399 198 199 JPAAAA GVGAAA AAAAxx 6773 4609 1 1 3 13 73 773 773 1773 6773 146 147 NAAAAA HVGAAA HHHHxx 7942 4610 0 2 2 2 42 942 1942 2942 7942 84 85 MTAAAA IVGAAA OOOOxx 6274 4611 0 2 4 14 74 274 274 1274 6274 148 149 IHAAAA JVGAAA VVVVxx 7447 4612 1 3 7 7 47 447 1447 2447 7447 94 95 LAAAAA KVGAAA AAAAxx 7648 4613 0 0 8 8 48 648 1648 2648 7648 96 97 EIAAAA LVGAAA HHHHxx 3997 4614 1 1 7 17 97 997 1997 3997 3997 194 195 TXAAAA MVGAAA OOOOxx 1759 4615 1 3 9 19 59 759 1759 1759 1759 118 119 RPAAAA NVGAAA VVVVxx 1785 4616 1 1 5 5 85 785 1785 1785 1785 170 171 RQAAAA OVGAAA AAAAxx 8930 4617 0 2 0 10 30 930 930 3930 8930 60 61 MFAAAA PVGAAA HHHHxx 7595 4618 1 3 5 15 95 595 1595 2595 7595 190 191 DGAAAA QVGAAA OOOOxx 6752 4619 0 0 2 12 52 752 752 1752 6752 104 105 SZAAAA RVGAAA VVVVxx 5635 4620 1 3 5 15 35 635 1635 635 5635 70 71 TIAAAA SVGAAA AAAAxx 1579 4621 1 3 9 19 79 579 1579 1579 1579 158 159 TIAAAA TVGAAA HHHHxx 7743 4622 1 3 3 3 43 743 1743 2743 7743 86 87 VLAAAA UVGAAA OOOOxx 5856 4623 0 0 6 16 56 856 1856 856 5856 112 113 GRAAAA VVGAAA VVVVxx 7273 4624 1 1 3 13 73 273 1273 2273 7273 146 147 TTAAAA WVGAAA AAAAxx 1399 4625 1 3 9 19 99 399 1399 1399 1399 198 199 VBAAAA XVGAAA HHHHxx 3694 4626 0 2 4 14 94 694 1694 3694 3694 188 189 CMAAAA YVGAAA OOOOxx 2782 4627 0 2 2 2 82 782 782 2782 2782 164 165 ADAAAA ZVGAAA VVVVxx 6951 4628 1 3 1 11 51 951 951 1951 6951 102 103 JHAAAA AWGAAA AAAAxx 6053 4629 1 1 3 13 53 53 53 1053 6053 106 107 VYAAAA BWGAAA HHHHxx 1753 4630 1 1 3 13 53 753 1753 1753 1753 106 107 LPAAAA CWGAAA OOOOxx 3985 4631 1 1 5 5 85 985 1985 3985 3985 170 171 HXAAAA DWGAAA VVVVxx 6159 4632 1 3 9 19 59 159 159 1159 6159 118 119 XCAAAA EWGAAA AAAAxx 6250 4633 0 2 0 10 50 250 250 1250 6250 100 101 KGAAAA FWGAAA HHHHxx 6240 4634 0 0 0 0 40 240 240 1240 6240 80 81 AGAAAA GWGAAA OOOOxx 6571 4635 1 3 1 11 71 571 571 1571 6571 142 143 TSAAAA HWGAAA VVVVxx 8624 4636 0 0 4 4 24 624 624 3624 8624 48 49 STAAAA IWGAAA AAAAxx 9718 4637 0 2 8 18 18 718 1718 4718 9718 36 37 UJAAAA JWGAAA HHHHxx 5529 4638 1 1 9 9 29 529 1529 529 5529 58 59 REAAAA KWGAAA OOOOxx 7089 4639 1 1 9 9 89 89 1089 2089 7089 178 179 RMAAAA LWGAAA VVVVxx 5488 4640 0 0 8 8 88 488 1488 488 5488 176 177 CDAAAA MWGAAA AAAAxx 5444 4641 0 0 4 4 44 444 1444 444 5444 88 89 KBAAAA NWGAAA HHHHxx 4899 4642 1 3 9 19 99 899 899 4899 4899 198 199 LGAAAA OWGAAA OOOOxx 7928 4643 0 0 8 8 28 928 1928 2928 7928 56 57 YSAAAA PWGAAA VVVVxx 4736 4644 0 0 6 16 36 736 736 4736 4736 72 73 EAAAAA QWGAAA AAAAxx 4317 4645 1 1 7 17 17 317 317 4317 4317 34 35 BKAAAA RWGAAA HHHHxx 1174 4646 0 2 4 14 74 174 1174 1174 1174 148 149 ETAAAA SWGAAA OOOOxx 6138 4647 0 2 8 18 38 138 138 1138 6138 76 77 CCAAAA TWGAAA VVVVxx 3943 4648 1 3 3 3 43 943 1943 3943 3943 86 87 RVAAAA UWGAAA AAAAxx 1545 4649 1 1 5 5 45 545 1545 1545 1545 90 91 LHAAAA VWGAAA HHHHxx 6867 4650 1 3 7 7 67 867 867 1867 6867 134 135 DEAAAA WWGAAA OOOOxx 6832 4651 0 0 2 12 32 832 832 1832 6832 64 65 UCAAAA XWGAAA VVVVxx 2987 4652 1 3 7 7 87 987 987 2987 2987 174 175 XKAAAA YWGAAA AAAAxx 5169 4653 1 1 9 9 69 169 1169 169 5169 138 139 VQAAAA ZWGAAA HHHHxx 8998 4654 0 2 8 18 98 998 998 3998 8998 196 197 CIAAAA AXGAAA OOOOxx 9347 4655 1 3 7 7 47 347 1347 4347 9347 94 95 NVAAAA BXGAAA VVVVxx 4800 4656 0 0 0 0 0 800 800 4800 4800 0 1 QCAAAA CXGAAA AAAAxx 4200 4657 0 0 0 0 0 200 200 4200 4200 0 1 OFAAAA DXGAAA HHHHxx 4046 4658 0 2 6 6 46 46 46 4046 4046 92 93 QZAAAA EXGAAA OOOOxx 7142 4659 0 2 2 2 42 142 1142 2142 7142 84 85 SOAAAA FXGAAA VVVVxx 2733 4660 1 1 3 13 33 733 733 2733 2733 66 67 DBAAAA GXGAAA AAAAxx 1568 4661 0 0 8 8 68 568 1568 1568 1568 136 137 IIAAAA HXGAAA HHHHxx 5105 4662 1 1 5 5 5 105 1105 105 5105 10 11 JOAAAA IXGAAA OOOOxx 9115 4663 1 3 5 15 15 115 1115 4115 9115 30 31 PMAAAA JXGAAA VVVVxx 6475 4664 1 3 5 15 75 475 475 1475 6475 150 151 BPAAAA KXGAAA AAAAxx 3796 4665 0 0 6 16 96 796 1796 3796 3796 192 193 AQAAAA LXGAAA HHHHxx 5410 4666 0 2 0 10 10 410 1410 410 5410 20 21 CAAAAA MXGAAA OOOOxx 4023 4667 1 3 3 3 23 23 23 4023 4023 46 47 TYAAAA NXGAAA VVVVxx 8904 4668 0 0 4 4 4 904 904 3904 8904 8 9 MEAAAA OXGAAA AAAAxx 450 4669 0 2 0 10 50 450 450 450 450 100 101 IRAAAA PXGAAA HHHHxx 8087 4670 1 3 7 7 87 87 87 3087 8087 174 175 BZAAAA QXGAAA OOOOxx 6478 4671 0 2 8 18 78 478 478 1478 6478 156 157 EPAAAA RXGAAA VVVVxx 2696 4672 0 0 6 16 96 696 696 2696 2696 192 193 SZAAAA SXGAAA AAAAxx 1792 4673 0 0 2 12 92 792 1792 1792 1792 184 185 YQAAAA TXGAAA HHHHxx 9699 4674 1 3 9 19 99 699 1699 4699 9699 198 199 BJAAAA UXGAAA OOOOxx 9160 4675 0 0 0 0 60 160 1160 4160 9160 120 121 IOAAAA VXGAAA VVVVxx 9989 4676 1 1 9 9 89 989 1989 4989 9989 178 179 FUAAAA WXGAAA AAAAxx 9568 4677 0 0 8 8 68 568 1568 4568 9568 136 137 AEAAAA XXGAAA HHHHxx 487 4678 1 3 7 7 87 487 487 487 487 174 175 TSAAAA YXGAAA OOOOxx 7863 4679 1 3 3 3 63 863 1863 2863 7863 126 127 LQAAAA ZXGAAA VVVVxx 1884 4680 0 0 4 4 84 884 1884 1884 1884 168 169 MUAAAA AYGAAA AAAAxx 2651 4681 1 3 1 11 51 651 651 2651 2651 102 103 ZXAAAA BYGAAA HHHHxx 8285 4682 1 1 5 5 85 285 285 3285 8285 170 171 RGAAAA CYGAAA OOOOxx 3927 4683 1 3 7 7 27 927 1927 3927 3927 54 55 BVAAAA DYGAAA VVVVxx 4076 4684 0 0 6 16 76 76 76 4076 4076 152 153 UAAAAA EYGAAA AAAAxx 6149 4685 1 1 9 9 49 149 149 1149 6149 98 99 NCAAAA FYGAAA HHHHxx 6581 4686 1 1 1 1 81 581 581 1581 6581 162 163 DTAAAA GYGAAA OOOOxx 8293 4687 1 1 3 13 93 293 293 3293 8293 186 187 ZGAAAA HYGAAA VVVVxx 7665 4688 1 1 5 5 65 665 1665 2665 7665 130 131 VIAAAA IYGAAA AAAAxx 4435 4689 1 3 5 15 35 435 435 4435 4435 70 71 POAAAA JYGAAA HHHHxx 1271 4690 1 3 1 11 71 271 1271 1271 1271 142 143 XWAAAA KYGAAA OOOOxx 3928 4691 0 0 8 8 28 928 1928 3928 3928 56 57 CVAAAA LYGAAA VVVVxx 7045 4692 1 1 5 5 45 45 1045 2045 7045 90 91 ZKAAAA MYGAAA AAAAxx 4943 4693 1 3 3 3 43 943 943 4943 4943 86 87 DIAAAA NYGAAA HHHHxx 8473 4694 1 1 3 13 73 473 473 3473 8473 146 147 XNAAAA OYGAAA OOOOxx 1707 4695 1 3 7 7 7 707 1707 1707 1707 14 15 RNAAAA PYGAAA VVVVxx 7509 4696 1 1 9 9 9 509 1509 2509 7509 18 19 VCAAAA QYGAAA AAAAxx 1593 4697 1 1 3 13 93 593 1593 1593 1593 186 187 HJAAAA RYGAAA HHHHxx 9281 4698 1 1 1 1 81 281 1281 4281 9281 162 163 ZSAAAA SYGAAA OOOOxx 8986 4699 0 2 6 6 86 986 986 3986 8986 172 173 QHAAAA TYGAAA VVVVxx 3740 4700 0 0 0 0 40 740 1740 3740 3740 80 81 WNAAAA UYGAAA AAAAxx 9265 4701 1 1 5 5 65 265 1265 4265 9265 130 131 JSAAAA VYGAAA HHHHxx 1510 4702 0 2 0 10 10 510 1510 1510 1510 20 21 CGAAAA WYGAAA OOOOxx 3022 4703 0 2 2 2 22 22 1022 3022 3022 44 45 GMAAAA XYGAAA VVVVxx 9014 4704 0 2 4 14 14 14 1014 4014 9014 28 29 SIAAAA YYGAAA AAAAxx 6816 4705 0 0 6 16 16 816 816 1816 6816 32 33 ECAAAA ZYGAAA HHHHxx 5518 4706 0 2 8 18 18 518 1518 518 5518 36 37 GEAAAA AZGAAA OOOOxx 4451 4707 1 3 1 11 51 451 451 4451 4451 102 103 FPAAAA BZGAAA VVVVxx 8747 4708 1 3 7 7 47 747 747 3747 8747 94 95 LYAAAA CZGAAA AAAAxx 4646 4709 0 2 6 6 46 646 646 4646 4646 92 93 SWAAAA DZGAAA HHHHxx 7296 4710 0 0 6 16 96 296 1296 2296 7296 192 193 QUAAAA EZGAAA OOOOxx 9644 4711 0 0 4 4 44 644 1644 4644 9644 88 89 YGAAAA FZGAAA VVVVxx 5977 4712 1 1 7 17 77 977 1977 977 5977 154 155 XVAAAA GZGAAA AAAAxx 6270 4713 0 2 0 10 70 270 270 1270 6270 140 141 EHAAAA HZGAAA HHHHxx 5578 4714 0 2 8 18 78 578 1578 578 5578 156 157 OGAAAA IZGAAA OOOOxx 2465 4715 1 1 5 5 65 465 465 2465 2465 130 131 VQAAAA JZGAAA VVVVxx 6436 4716 0 0 6 16 36 436 436 1436 6436 72 73 ONAAAA KZGAAA AAAAxx 8089 4717 1 1 9 9 89 89 89 3089 8089 178 179 DZAAAA LZGAAA HHHHxx 2409 4718 1 1 9 9 9 409 409 2409 2409 18 19 ROAAAA MZGAAA OOOOxx 284 4719 0 0 4 4 84 284 284 284 284 168 169 YKAAAA NZGAAA VVVVxx 5576 4720 0 0 6 16 76 576 1576 576 5576 152 153 MGAAAA OZGAAA AAAAxx 6534 4721 0 2 4 14 34 534 534 1534 6534 68 69 IRAAAA PZGAAA HHHHxx 8848 4722 0 0 8 8 48 848 848 3848 8848 96 97 ICAAAA QZGAAA OOOOxx 4305 4723 1 1 5 5 5 305 305 4305 4305 10 11 PJAAAA RZGAAA VVVVxx 5574 4724 0 2 4 14 74 574 1574 574 5574 148 149 KGAAAA SZGAAA AAAAxx 596 4725 0 0 6 16 96 596 596 596 596 192 193 YWAAAA TZGAAA HHHHxx 1253 4726 1 1 3 13 53 253 1253 1253 1253 106 107 FWAAAA UZGAAA OOOOxx 521 4727 1 1 1 1 21 521 521 521 521 42 43 BUAAAA VZGAAA VVVVxx 8739 4728 1 3 9 19 39 739 739 3739 8739 78 79 DYAAAA WZGAAA AAAAxx 908 4729 0 0 8 8 8 908 908 908 908 16 17 YIAAAA XZGAAA HHHHxx 6937 4730 1 1 7 17 37 937 937 1937 6937 74 75 VGAAAA YZGAAA OOOOxx 4515 4731 1 3 5 15 15 515 515 4515 4515 30 31 RRAAAA ZZGAAA VVVVxx 8630 4732 0 2 0 10 30 630 630 3630 8630 60 61 YTAAAA AAHAAA AAAAxx 7518 4733 0 2 8 18 18 518 1518 2518 7518 36 37 EDAAAA BAHAAA HHHHxx 8300 4734 0 0 0 0 0 300 300 3300 8300 0 1 GHAAAA CAHAAA OOOOxx 8434 4735 0 2 4 14 34 434 434 3434 8434 68 69 KMAAAA DAHAAA VVVVxx 6000 4736 0 0 0 0 0 0 0 1000 6000 0 1 UWAAAA EAHAAA AAAAxx 4508 4737 0 0 8 8 8 508 508 4508 4508 16 17 KRAAAA FAHAAA HHHHxx 7861 4738 1 1 1 1 61 861 1861 2861 7861 122 123 JQAAAA GAHAAA OOOOxx 5953 4739 1 1 3 13 53 953 1953 953 5953 106 107 ZUAAAA HAHAAA VVVVxx 5063 4740 1 3 3 3 63 63 1063 63 5063 126 127 TMAAAA IAHAAA AAAAxx 4501 4741 1 1 1 1 1 501 501 4501 4501 2 3 DRAAAA JAHAAA HHHHxx 7092 4742 0 0 2 12 92 92 1092 2092 7092 184 185 UMAAAA KAHAAA OOOOxx 4388 4743 0 0 8 8 88 388 388 4388 4388 176 177 UMAAAA LAHAAA VVVVxx 1826 4744 0 2 6 6 26 826 1826 1826 1826 52 53 GSAAAA MAHAAA AAAAxx 568 4745 0 0 8 8 68 568 568 568 568 136 137 WVAAAA NAHAAA HHHHxx 8184 4746 0 0 4 4 84 184 184 3184 8184 168 169 UCAAAA OAHAAA OOOOxx 4268 4747 0 0 8 8 68 268 268 4268 4268 136 137 EIAAAA PAHAAA VVVVxx 5798 4748 0 2 8 18 98 798 1798 798 5798 196 197 APAAAA QAHAAA AAAAxx 5190 4749 0 2 0 10 90 190 1190 190 5190 180 181 QRAAAA RAHAAA HHHHxx 1298 4750 0 2 8 18 98 298 1298 1298 1298 196 197 YXAAAA SAHAAA OOOOxx 4035 4751 1 3 5 15 35 35 35 4035 4035 70 71 FZAAAA TAHAAA VVVVxx 4504 4752 0 0 4 4 4 504 504 4504 4504 8 9 GRAAAA UAHAAA AAAAxx 5992 4753 0 0 2 12 92 992 1992 992 5992 184 185 MWAAAA VAHAAA HHHHxx 770 4754 0 2 0 10 70 770 770 770 770 140 141 QDAAAA WAHAAA OOOOxx 7502 4755 0 2 2 2 2 502 1502 2502 7502 4 5 OCAAAA XAHAAA VVVVxx 824 4756 0 0 4 4 24 824 824 824 824 48 49 SFAAAA YAHAAA AAAAxx 7716 4757 0 0 6 16 16 716 1716 2716 7716 32 33 UKAAAA ZAHAAA HHHHxx 5749 4758 1 1 9 9 49 749 1749 749 5749 98 99 DNAAAA ABHAAA OOOOxx 9814 4759 0 2 4 14 14 814 1814 4814 9814 28 29 MNAAAA BBHAAA VVVVxx 350 4760 0 2 0 10 50 350 350 350 350 100 101 MNAAAA CBHAAA AAAAxx 1390 4761 0 2 0 10 90 390 1390 1390 1390 180 181 MBAAAA DBHAAA HHHHxx 6994 4762 0 2 4 14 94 994 994 1994 6994 188 189 AJAAAA EBHAAA OOOOxx 3629 4763 1 1 9 9 29 629 1629 3629 3629 58 59 PJAAAA FBHAAA VVVVxx 9937 4764 1 1 7 17 37 937 1937 4937 9937 74 75 FSAAAA GBHAAA AAAAxx 5285 4765 1 1 5 5 85 285 1285 285 5285 170 171 HVAAAA HBHAAA HHHHxx 3157 4766 1 1 7 17 57 157 1157 3157 3157 114 115 LRAAAA IBHAAA OOOOxx 9549 4767 1 1 9 9 49 549 1549 4549 9549 98 99 HDAAAA JBHAAA VVVVxx 4118 4768 0 2 8 18 18 118 118 4118 4118 36 37 KCAAAA KBHAAA AAAAxx 756 4769 0 0 6 16 56 756 756 756 756 112 113 CDAAAA LBHAAA HHHHxx 5964 4770 0 0 4 4 64 964 1964 964 5964 128 129 KVAAAA MBHAAA OOOOxx 7701 4771 1 1 1 1 1 701 1701 2701 7701 2 3 FKAAAA NBHAAA VVVVxx 1242 4772 0 2 2 2 42 242 1242 1242 1242 84 85 UVAAAA OBHAAA AAAAxx 7890 4773 0 2 0 10 90 890 1890 2890 7890 180 181 MRAAAA PBHAAA HHHHxx 1991 4774 1 3 1 11 91 991 1991 1991 1991 182 183 PYAAAA QBHAAA OOOOxx 110 4775 0 2 0 10 10 110 110 110 110 20 21 GEAAAA RBHAAA VVVVxx 9334 4776 0 2 4 14 34 334 1334 4334 9334 68 69 AVAAAA SBHAAA AAAAxx 6231 4777 1 3 1 11 31 231 231 1231 6231 62 63 RFAAAA TBHAAA HHHHxx 9871 4778 1 3 1 11 71 871 1871 4871 9871 142 143 RPAAAA UBHAAA OOOOxx 9471 4779 1 3 1 11 71 471 1471 4471 9471 142 143 HAAAAA VBHAAA VVVVxx 2697 4780 1 1 7 17 97 697 697 2697 2697 194 195 TZAAAA WBHAAA AAAAxx 4761 4781 1 1 1 1 61 761 761 4761 4761 122 123 DBAAAA XBHAAA HHHHxx 8493 4782 1 1 3 13 93 493 493 3493 8493 186 187 ROAAAA YBHAAA OOOOxx 1045 4783 1 1 5 5 45 45 1045 1045 1045 90 91 FOAAAA ZBHAAA VVVVxx 3403 4784 1 3 3 3 3 403 1403 3403 3403 6 7 XAAAAA ACHAAA AAAAxx 9412 4785 0 0 2 12 12 412 1412 4412 9412 24 25 AYAAAA BCHAAA HHHHxx 7652 4786 0 0 2 12 52 652 1652 2652 7652 104 105 IIAAAA CCHAAA OOOOxx 5866 4787 0 2 6 6 66 866 1866 866 5866 132 133 QRAAAA DCHAAA VVVVxx 6942 4788 0 2 2 2 42 942 942 1942 6942 84 85 AHAAAA ECHAAA AAAAxx 9353 4789 1 1 3 13 53 353 1353 4353 9353 106 107 TVAAAA FCHAAA HHHHxx 2600 4790 0 0 0 0 0 600 600 2600 2600 0 1 AWAAAA GCHAAA OOOOxx 6971 4791 1 3 1 11 71 971 971 1971 6971 142 143 DIAAAA HCHAAA VVVVxx 5391 4792 1 3 1 11 91 391 1391 391 5391 182 183 JZAAAA ICHAAA AAAAxx 7654 4793 0 2 4 14 54 654 1654 2654 7654 108 109 KIAAAA JCHAAA HHHHxx 1797 4794 1 1 7 17 97 797 1797 1797 1797 194 195 DRAAAA KCHAAA OOOOxx 4530 4795 0 2 0 10 30 530 530 4530 4530 60 61 GSAAAA LCHAAA VVVVxx 3130 4796 0 2 0 10 30 130 1130 3130 3130 60 61 KQAAAA MCHAAA AAAAxx 9442 4797 0 2 2 2 42 442 1442 4442 9442 84 85 EZAAAA NCHAAA HHHHxx 6659 4798 1 3 9 19 59 659 659 1659 6659 118 119 DWAAAA OCHAAA OOOOxx 9714 4799 0 2 4 14 14 714 1714 4714 9714 28 29 QJAAAA PCHAAA VVVVxx 3660 4800 0 0 0 0 60 660 1660 3660 3660 120 121 UKAAAA QCHAAA AAAAxx 1906 4801 0 2 6 6 6 906 1906 1906 1906 12 13 IVAAAA RCHAAA HHHHxx 7927 4802 1 3 7 7 27 927 1927 2927 7927 54 55 XSAAAA SCHAAA OOOOxx 1767 4803 1 3 7 7 67 767 1767 1767 1767 134 135 ZPAAAA TCHAAA VVVVxx 5523 4804 1 3 3 3 23 523 1523 523 5523 46 47 LEAAAA UCHAAA AAAAxx 9289 4805 1 1 9 9 89 289 1289 4289 9289 178 179 HTAAAA VCHAAA HHHHxx 2717 4806 1 1 7 17 17 717 717 2717 2717 34 35 NAAAAA WCHAAA OOOOxx 4099 4807 1 3 9 19 99 99 99 4099 4099 198 199 RBAAAA XCHAAA VVVVxx 4387 4808 1 3 7 7 87 387 387 4387 4387 174 175 TMAAAA YCHAAA AAAAxx 8864 4809 0 0 4 4 64 864 864 3864 8864 128 129 YCAAAA ZCHAAA HHHHxx 1774 4810 0 2 4 14 74 774 1774 1774 1774 148 149 GQAAAA ADHAAA OOOOxx 6292 4811 0 0 2 12 92 292 292 1292 6292 184 185 AIAAAA BDHAAA VVVVxx 847 4812 1 3 7 7 47 847 847 847 847 94 95 PGAAAA CDHAAA AAAAxx 5954 4813 0 2 4 14 54 954 1954 954 5954 108 109 AVAAAA DDHAAA HHHHxx 8032 4814 0 0 2 12 32 32 32 3032 8032 64 65 YWAAAA EDHAAA OOOOxx 3295 4815 1 3 5 15 95 295 1295 3295 3295 190 191 TWAAAA FDHAAA VVVVxx 8984 4816 0 0 4 4 84 984 984 3984 8984 168 169 OHAAAA GDHAAA AAAAxx 7809 4817 1 1 9 9 9 809 1809 2809 7809 18 19 JOAAAA HDHAAA HHHHxx 1670 4818 0 2 0 10 70 670 1670 1670 1670 140 141 GMAAAA IDHAAA OOOOxx 7733 4819 1 1 3 13 33 733 1733 2733 7733 66 67 LLAAAA JDHAAA VVVVxx 6187 4820 1 3 7 7 87 187 187 1187 6187 174 175 ZDAAAA KDHAAA AAAAxx 9326 4821 0 2 6 6 26 326 1326 4326 9326 52 53 SUAAAA LDHAAA HHHHxx 2493 4822 1 1 3 13 93 493 493 2493 2493 186 187 XRAAAA MDHAAA OOOOxx 9512 4823 0 0 2 12 12 512 1512 4512 9512 24 25 WBAAAA NDHAAA VVVVxx 4342 4824 0 2 2 2 42 342 342 4342 4342 84 85 ALAAAA ODHAAA AAAAxx 5350 4825 0 2 0 10 50 350 1350 350 5350 100 101 UXAAAA PDHAAA HHHHxx 6009 4826 1 1 9 9 9 9 9 1009 6009 18 19 DXAAAA QDHAAA OOOOxx 1208 4827 0 0 8 8 8 208 1208 1208 1208 16 17 MUAAAA RDHAAA VVVVxx 7014 4828 0 2 4 14 14 14 1014 2014 7014 28 29 UJAAAA SDHAAA AAAAxx 2967 4829 1 3 7 7 67 967 967 2967 2967 134 135 DKAAAA TDHAAA HHHHxx 5831 4830 1 3 1 11 31 831 1831 831 5831 62 63 HQAAAA UDHAAA OOOOxx 3097 4831 1 1 7 17 97 97 1097 3097 3097 194 195 DPAAAA VDHAAA VVVVxx 1528 4832 0 0 8 8 28 528 1528 1528 1528 56 57 UGAAAA WDHAAA AAAAxx 6429 4833 1 1 9 9 29 429 429 1429 6429 58 59 HNAAAA XDHAAA HHHHxx 7320 4834 0 0 0 0 20 320 1320 2320 7320 40 41 OVAAAA YDHAAA OOOOxx 844 4835 0 0 4 4 44 844 844 844 844 88 89 MGAAAA ZDHAAA VVVVxx 7054 4836 0 2 4 14 54 54 1054 2054 7054 108 109 ILAAAA AEHAAA AAAAxx 1643 4837 1 3 3 3 43 643 1643 1643 1643 86 87 FLAAAA BEHAAA HHHHxx 7626 4838 0 2 6 6 26 626 1626 2626 7626 52 53 IHAAAA CEHAAA OOOOxx 8728 4839 0 0 8 8 28 728 728 3728 8728 56 57 SXAAAA DEHAAA VVVVxx 8277 4840 1 1 7 17 77 277 277 3277 8277 154 155 JGAAAA EEHAAA AAAAxx 189 4841 1 1 9 9 89 189 189 189 189 178 179 HHAAAA FEHAAA HHHHxx 3717 4842 1 1 7 17 17 717 1717 3717 3717 34 35 ZMAAAA GEHAAA OOOOxx 1020 4843 0 0 0 0 20 20 1020 1020 1020 40 41 GNAAAA HEHAAA VVVVxx 9234 4844 0 2 4 14 34 234 1234 4234 9234 68 69 ERAAAA IEHAAA AAAAxx 9541 4845 1 1 1 1 41 541 1541 4541 9541 82 83 ZCAAAA JEHAAA HHHHxx 380 4846 0 0 0 0 80 380 380 380 380 160 161 QOAAAA KEHAAA OOOOxx 397 4847 1 1 7 17 97 397 397 397 397 194 195 HPAAAA LEHAAA VVVVxx 835 4848 1 3 5 15 35 835 835 835 835 70 71 DGAAAA MEHAAA AAAAxx 347 4849 1 3 7 7 47 347 347 347 347 94 95 JNAAAA NEHAAA HHHHxx 2490 4850 0 2 0 10 90 490 490 2490 2490 180 181 URAAAA OEHAAA OOOOxx 605 4851 1 1 5 5 5 605 605 605 605 10 11 HXAAAA PEHAAA VVVVxx 7960 4852 0 0 0 0 60 960 1960 2960 7960 120 121 EUAAAA QEHAAA AAAAxx 9681 4853 1 1 1 1 81 681 1681 4681 9681 162 163 JIAAAA REHAAA HHHHxx 5753 4854 1 1 3 13 53 753 1753 753 5753 106 107 HNAAAA SEHAAA OOOOxx 1676 4855 0 0 6 16 76 676 1676 1676 1676 152 153 MMAAAA TEHAAA VVVVxx 5533 4856 1 1 3 13 33 533 1533 533 5533 66 67 VEAAAA UEHAAA AAAAxx 8958 4857 0 2 8 18 58 958 958 3958 8958 116 117 OGAAAA VEHAAA HHHHxx 664 4858 0 0 4 4 64 664 664 664 664 128 129 OZAAAA WEHAAA OOOOxx 3005 4859 1 1 5 5 5 5 1005 3005 3005 10 11 PLAAAA XEHAAA VVVVxx 8576 4860 0 0 6 16 76 576 576 3576 8576 152 153 WRAAAA YEHAAA AAAAxx 7304 4861 0 0 4 4 4 304 1304 2304 7304 8 9 YUAAAA ZEHAAA HHHHxx 3375 4862 1 3 5 15 75 375 1375 3375 3375 150 151 VZAAAA AFHAAA OOOOxx 6336 4863 0 0 6 16 36 336 336 1336 6336 72 73 SJAAAA BFHAAA VVVVxx 1392 4864 0 0 2 12 92 392 1392 1392 1392 184 185 OBAAAA CFHAAA AAAAxx 2925 4865 1 1 5 5 25 925 925 2925 2925 50 51 NIAAAA DFHAAA HHHHxx 1217 4866 1 1 7 17 17 217 1217 1217 1217 34 35 VUAAAA EFHAAA OOOOxx 3714 4867 0 2 4 14 14 714 1714 3714 3714 28 29 WMAAAA FFHAAA VVVVxx 2120 4868 0 0 0 0 20 120 120 2120 2120 40 41 ODAAAA GFHAAA AAAAxx 2845 4869 1 1 5 5 45 845 845 2845 2845 90 91 LFAAAA HFHAAA HHHHxx 3865 4870 1 1 5 5 65 865 1865 3865 3865 130 131 RSAAAA IFHAAA OOOOxx 124 4871 0 0 4 4 24 124 124 124 124 48 49 UEAAAA JFHAAA VVVVxx 865 4872 1 1 5 5 65 865 865 865 865 130 131 HHAAAA KFHAAA AAAAxx 9361 4873 1 1 1 1 61 361 1361 4361 9361 122 123 BWAAAA LFHAAA HHHHxx 6338 4874 0 2 8 18 38 338 338 1338 6338 76 77 UJAAAA MFHAAA OOOOxx 7330 4875 0 2 0 10 30 330 1330 2330 7330 60 61 YVAAAA NFHAAA VVVVxx 513 4876 1 1 3 13 13 513 513 513 513 26 27 TTAAAA OFHAAA AAAAxx 5001 4877 1 1 1 1 1 1 1001 1 5001 2 3 JKAAAA PFHAAA HHHHxx 549 4878 1 1 9 9 49 549 549 549 549 98 99 DVAAAA QFHAAA OOOOxx 1808 4879 0 0 8 8 8 808 1808 1808 1808 16 17 ORAAAA RFHAAA VVVVxx 7168 4880 0 0 8 8 68 168 1168 2168 7168 136 137 SPAAAA SFHAAA AAAAxx 9878 4881 0 2 8 18 78 878 1878 4878 9878 156 157 YPAAAA TFHAAA HHHHxx 233 4882 1 1 3 13 33 233 233 233 233 66 67 ZIAAAA UFHAAA OOOOxx 4262 4883 0 2 2 2 62 262 262 4262 4262 124 125 YHAAAA VFHAAA VVVVxx 7998 4884 0 2 8 18 98 998 1998 2998 7998 196 197 QVAAAA WFHAAA AAAAxx 2419 4885 1 3 9 19 19 419 419 2419 2419 38 39 BPAAAA XFHAAA HHHHxx 9960 4886 0 0 0 0 60 960 1960 4960 9960 120 121 CTAAAA YFHAAA OOOOxx 3523 4887 1 3 3 3 23 523 1523 3523 3523 46 47 NFAAAA ZFHAAA VVVVxx 5440 4888 0 0 0 0 40 440 1440 440 5440 80 81 GBAAAA AGHAAA AAAAxx 3030 4889 0 2 0 10 30 30 1030 3030 3030 60 61 OMAAAA BGHAAA HHHHxx 2745 4890 1 1 5 5 45 745 745 2745 2745 90 91 PBAAAA CGHAAA OOOOxx 7175 4891 1 3 5 15 75 175 1175 2175 7175 150 151 ZPAAAA DGHAAA VVVVxx 640 4892 0 0 0 0 40 640 640 640 640 80 81 QYAAAA EGHAAA AAAAxx 1798 4893 0 2 8 18 98 798 1798 1798 1798 196 197 ERAAAA FGHAAA HHHHxx 7499 4894 1 3 9 19 99 499 1499 2499 7499 198 199 LCAAAA GGHAAA OOOOxx 1924 4895 0 0 4 4 24 924 1924 1924 1924 48 49 AWAAAA HGHAAA VVVVxx 1327 4896 1 3 7 7 27 327 1327 1327 1327 54 55 BZAAAA IGHAAA AAAAxx 73 4897 1 1 3 13 73 73 73 73 73 146 147 VCAAAA JGHAAA HHHHxx 9558 4898 0 2 8 18 58 558 1558 4558 9558 116 117 QDAAAA KGHAAA OOOOxx 818 4899 0 2 8 18 18 818 818 818 818 36 37 MFAAAA LGHAAA VVVVxx 9916 4900 0 0 6 16 16 916 1916 4916 9916 32 33 KRAAAA MGHAAA AAAAxx 2978 4901 0 2 8 18 78 978 978 2978 2978 156 157 OKAAAA NGHAAA HHHHxx 8469 4902 1 1 9 9 69 469 469 3469 8469 138 139 TNAAAA OGHAAA OOOOxx 9845 4903 1 1 5 5 45 845 1845 4845 9845 90 91 ROAAAA PGHAAA VVVVxx 2326 4904 0 2 6 6 26 326 326 2326 2326 52 53 MLAAAA QGHAAA AAAAxx 4032 4905 0 0 2 12 32 32 32 4032 4032 64 65 CZAAAA RGHAAA HHHHxx 5604 4906 0 0 4 4 4 604 1604 604 5604 8 9 OHAAAA SGHAAA OOOOxx 9610 4907 0 2 0 10 10 610 1610 4610 9610 20 21 QFAAAA TGHAAA VVVVxx 5101 4908 1 1 1 1 1 101 1101 101 5101 2 3 FOAAAA UGHAAA AAAAxx 7246 4909 0 2 6 6 46 246 1246 2246 7246 92 93 SSAAAA VGHAAA HHHHxx 1292 4910 0 0 2 12 92 292 1292 1292 1292 184 185 SXAAAA WGHAAA OOOOxx 6235 4911 1 3 5 15 35 235 235 1235 6235 70 71 VFAAAA XGHAAA VVVVxx 1733 4912 1 1 3 13 33 733 1733 1733 1733 66 67 ROAAAA YGHAAA AAAAxx 4647 4913 1 3 7 7 47 647 647 4647 4647 94 95 TWAAAA ZGHAAA HHHHxx 258 4914 0 2 8 18 58 258 258 258 258 116 117 YJAAAA AHHAAA OOOOxx 8438 4915 0 2 8 18 38 438 438 3438 8438 76 77 OMAAAA BHHAAA VVVVxx 7869 4916 1 1 9 9 69 869 1869 2869 7869 138 139 RQAAAA CHHAAA AAAAxx 9691 4917 1 3 1 11 91 691 1691 4691 9691 182 183 TIAAAA DHHAAA HHHHxx 5422 4918 0 2 2 2 22 422 1422 422 5422 44 45 OAAAAA EHHAAA OOOOxx 9630 4919 0 2 0 10 30 630 1630 4630 9630 60 61 KGAAAA FHHAAA VVVVxx 4439 4920 1 3 9 19 39 439 439 4439 4439 78 79 TOAAAA GHHAAA AAAAxx 3140 4921 0 0 0 0 40 140 1140 3140 3140 80 81 UQAAAA HHHAAA HHHHxx 9111 4922 1 3 1 11 11 111 1111 4111 9111 22 23 LMAAAA IHHAAA OOOOxx 4606 4923 0 2 6 6 6 606 606 4606 4606 12 13 EVAAAA JHHAAA VVVVxx 8620 4924 0 0 0 0 20 620 620 3620 8620 40 41 OTAAAA KHHAAA AAAAxx 7849 4925 1 1 9 9 49 849 1849 2849 7849 98 99 XPAAAA LHHAAA HHHHxx 346 4926 0 2 6 6 46 346 346 346 346 92 93 INAAAA MHHAAA OOOOxx 9528 4927 0 0 8 8 28 528 1528 4528 9528 56 57 MCAAAA NHHAAA VVVVxx 1811 4928 1 3 1 11 11 811 1811 1811 1811 22 23 RRAAAA OHHAAA AAAAxx 6068 4929 0 0 8 8 68 68 68 1068 6068 136 137 KZAAAA PHHAAA HHHHxx 6260 4930 0 0 0 0 60 260 260 1260 6260 120 121 UGAAAA QHHAAA OOOOxx 5909 4931 1 1 9 9 9 909 1909 909 5909 18 19 HTAAAA RHHAAA VVVVxx 4518 4932 0 2 8 18 18 518 518 4518 4518 36 37 URAAAA SHHAAA AAAAxx 7530 4933 0 2 0 10 30 530 1530 2530 7530 60 61 QDAAAA THHAAA HHHHxx 3900 4934 0 0 0 0 0 900 1900 3900 3900 0 1 AUAAAA UHHAAA OOOOxx 3969 4935 1 1 9 9 69 969 1969 3969 3969 138 139 RWAAAA VHHAAA VVVVxx 8690 4936 0 2 0 10 90 690 690 3690 8690 180 181 GWAAAA WHHAAA AAAAxx 5532 4937 0 0 2 12 32 532 1532 532 5532 64 65 UEAAAA XHHAAA HHHHxx 5989 4938 1 1 9 9 89 989 1989 989 5989 178 179 JWAAAA YHHAAA OOOOxx 1870 4939 0 2 0 10 70 870 1870 1870 1870 140 141 YTAAAA ZHHAAA VVVVxx 1113 4940 1 1 3 13 13 113 1113 1113 1113 26 27 VQAAAA AIHAAA AAAAxx 5155 4941 1 3 5 15 55 155 1155 155 5155 110 111 HQAAAA BIHAAA HHHHxx 7460 4942 0 0 0 0 60 460 1460 2460 7460 120 121 YAAAAA CIHAAA OOOOxx 6217 4943 1 1 7 17 17 217 217 1217 6217 34 35 DFAAAA DIHAAA VVVVxx 8333 4944 1 1 3 13 33 333 333 3333 8333 66 67 NIAAAA EIHAAA AAAAxx 6341 4945 1 1 1 1 41 341 341 1341 6341 82 83 XJAAAA FIHAAA HHHHxx 6230 4946 0 2 0 10 30 230 230 1230 6230 60 61 QFAAAA GIHAAA OOOOxx 6902 4947 0 2 2 2 2 902 902 1902 6902 4 5 MFAAAA HIHAAA VVVVxx 670 4948 0 2 0 10 70 670 670 670 670 140 141 UZAAAA IIHAAA AAAAxx 805 4949 1 1 5 5 5 805 805 805 805 10 11 ZEAAAA JIHAAA HHHHxx 1340 4950 0 0 0 0 40 340 1340 1340 1340 80 81 OZAAAA KIHAAA OOOOxx 8649 4951 1 1 9 9 49 649 649 3649 8649 98 99 RUAAAA LIHAAA VVVVxx 3887 4952 1 3 7 7 87 887 1887 3887 3887 174 175 NTAAAA MIHAAA AAAAxx 5400 4953 0 0 0 0 0 400 1400 400 5400 0 1 SZAAAA NIHAAA HHHHxx 4354 4954 0 2 4 14 54 354 354 4354 4354 108 109 MLAAAA OIHAAA OOOOxx 950 4955 0 2 0 10 50 950 950 950 950 100 101 OKAAAA PIHAAA VVVVxx 1544 4956 0 0 4 4 44 544 1544 1544 1544 88 89 KHAAAA QIHAAA AAAAxx 3898 4957 0 2 8 18 98 898 1898 3898 3898 196 197 YTAAAA RIHAAA HHHHxx 8038 4958 0 2 8 18 38 38 38 3038 8038 76 77 EXAAAA SIHAAA OOOOxx 1095 4959 1 3 5 15 95 95 1095 1095 1095 190 191 DQAAAA TIHAAA VVVVxx 1748 4960 0 0 8 8 48 748 1748 1748 1748 96 97 GPAAAA UIHAAA AAAAxx 9154 4961 0 2 4 14 54 154 1154 4154 9154 108 109 COAAAA VIHAAA HHHHxx 2182 4962 0 2 2 2 82 182 182 2182 2182 164 165 YFAAAA WIHAAA OOOOxx 6797 4963 1 1 7 17 97 797 797 1797 6797 194 195 LBAAAA XIHAAA VVVVxx 9149 4964 1 1 9 9 49 149 1149 4149 9149 98 99 XNAAAA YIHAAA AAAAxx 7351 4965 1 3 1 11 51 351 1351 2351 7351 102 103 TWAAAA ZIHAAA HHHHxx 2820 4966 0 0 0 0 20 820 820 2820 2820 40 41 MEAAAA AJHAAA OOOOxx 9696 4967 0 0 6 16 96 696 1696 4696 9696 192 193 YIAAAA BJHAAA VVVVxx 253 4968 1 1 3 13 53 253 253 253 253 106 107 TJAAAA CJHAAA AAAAxx 3600 4969 0 0 0 0 0 600 1600 3600 3600 0 1 MIAAAA DJHAAA HHHHxx 3892 4970 0 0 2 12 92 892 1892 3892 3892 184 185 STAAAA EJHAAA OOOOxx 231 4971 1 3 1 11 31 231 231 231 231 62 63 XIAAAA FJHAAA VVVVxx 8331 4972 1 3 1 11 31 331 331 3331 8331 62 63 LIAAAA GJHAAA AAAAxx 403 4973 1 3 3 3 3 403 403 403 403 6 7 NPAAAA HJHAAA HHHHxx 8642 4974 0 2 2 2 42 642 642 3642 8642 84 85 KUAAAA IJHAAA OOOOxx 3118 4975 0 2 8 18 18 118 1118 3118 3118 36 37 YPAAAA JJHAAA VVVVxx 3835 4976 1 3 5 15 35 835 1835 3835 3835 70 71 NRAAAA KJHAAA AAAAxx 1117 4977 1 1 7 17 17 117 1117 1117 1117 34 35 ZQAAAA LJHAAA HHHHxx 7024 4978 0 0 4 4 24 24 1024 2024 7024 48 49 EKAAAA MJHAAA OOOOxx 2636 4979 0 0 6 16 36 636 636 2636 2636 72 73 KXAAAA NJHAAA VVVVxx 3778 4980 0 2 8 18 78 778 1778 3778 3778 156 157 IPAAAA OJHAAA AAAAxx 2003 4981 1 3 3 3 3 3 3 2003 2003 6 7 BZAAAA PJHAAA HHHHxx 5717 4982 1 1 7 17 17 717 1717 717 5717 34 35 XLAAAA QJHAAA OOOOxx 4869 4983 1 1 9 9 69 869 869 4869 4869 138 139 HFAAAA RJHAAA VVVVxx 8921 4984 1 1 1 1 21 921 921 3921 8921 42 43 DFAAAA SJHAAA AAAAxx 888 4985 0 0 8 8 88 888 888 888 888 176 177 EIAAAA TJHAAA HHHHxx 7599 4986 1 3 9 19 99 599 1599 2599 7599 198 199 HGAAAA UJHAAA OOOOxx 8621 4987 1 1 1 1 21 621 621 3621 8621 42 43 PTAAAA VJHAAA VVVVxx 811 4988 1 3 1 11 11 811 811 811 811 22 23 FFAAAA WJHAAA AAAAxx 9147 4989 1 3 7 7 47 147 1147 4147 9147 94 95 VNAAAA XJHAAA HHHHxx 1413 4990 1 1 3 13 13 413 1413 1413 1413 26 27 JCAAAA YJHAAA OOOOxx 5232 4991 0 0 2 12 32 232 1232 232 5232 64 65 GTAAAA ZJHAAA VVVVxx 5912 4992 0 0 2 12 12 912 1912 912 5912 24 25 KTAAAA AKHAAA AAAAxx 3418 4993 0 2 8 18 18 418 1418 3418 3418 36 37 MBAAAA BKHAAA HHHHxx 3912 4994 0 0 2 12 12 912 1912 3912 3912 24 25 MUAAAA CKHAAA OOOOxx 9576 4995 0 0 6 16 76 576 1576 4576 9576 152 153 IEAAAA DKHAAA VVVVxx 4225 4996 1 1 5 5 25 225 225 4225 4225 50 51 NGAAAA EKHAAA AAAAxx 8222 4997 0 2 2 2 22 222 222 3222 8222 44 45 GEAAAA FKHAAA HHHHxx 7013 4998 1 1 3 13 13 13 1013 2013 7013 26 27 TJAAAA GKHAAA OOOOxx 7037 4999 1 1 7 17 37 37 1037 2037 7037 74 75 RKAAAA HKHAAA VVVVxx 1205 5000 1 1 5 5 5 205 1205 1205 1205 10 11 JUAAAA IKHAAA AAAAxx 8114 5001 0 2 4 14 14 114 114 3114 8114 28 29 CAAAAA JKHAAA HHHHxx 6585 5002 1 1 5 5 85 585 585 1585 6585 170 171 HTAAAA KKHAAA OOOOxx 155 5003 1 3 5 15 55 155 155 155 155 110 111 ZFAAAA LKHAAA VVVVxx 2841 5004 1 1 1 1 41 841 841 2841 2841 82 83 HFAAAA MKHAAA AAAAxx 1996 5005 0 0 6 16 96 996 1996 1996 1996 192 193 UYAAAA NKHAAA HHHHxx 4948 5006 0 0 8 8 48 948 948 4948 4948 96 97 IIAAAA OKHAAA OOOOxx 3304 5007 0 0 4 4 4 304 1304 3304 3304 8 9 CXAAAA PKHAAA VVVVxx 5684 5008 0 0 4 4 84 684 1684 684 5684 168 169 QKAAAA QKHAAA AAAAxx 6962 5009 0 2 2 2 62 962 962 1962 6962 124 125 UHAAAA RKHAAA HHHHxx 8691 5010 1 3 1 11 91 691 691 3691 8691 182 183 HWAAAA SKHAAA OOOOxx 8501 5011 1 1 1 1 1 501 501 3501 8501 2 3 ZOAAAA TKHAAA VVVVxx 4783 5012 1 3 3 3 83 783 783 4783 4783 166 167 ZBAAAA UKHAAA AAAAxx 3762 5013 0 2 2 2 62 762 1762 3762 3762 124 125 SOAAAA VKHAAA HHHHxx 4534 5014 0 2 4 14 34 534 534 4534 4534 68 69 KSAAAA WKHAAA OOOOxx 4999 5015 1 3 9 19 99 999 999 4999 4999 198 199 HKAAAA XKHAAA VVVVxx 4618 5016 0 2 8 18 18 618 618 4618 4618 36 37 QVAAAA YKHAAA AAAAxx 4220 5017 0 0 0 0 20 220 220 4220 4220 40 41 IGAAAA ZKHAAA HHHHxx 3384 5018 0 0 4 4 84 384 1384 3384 3384 168 169 EAAAAA ALHAAA OOOOxx 3036 5019 0 0 6 16 36 36 1036 3036 3036 72 73 UMAAAA BLHAAA VVVVxx 545 5020 1 1 5 5 45 545 545 545 545 90 91 ZUAAAA CLHAAA AAAAxx 9946 5021 0 2 6 6 46 946 1946 4946 9946 92 93 OSAAAA DLHAAA HHHHxx 1985 5022 1 1 5 5 85 985 1985 1985 1985 170 171 JYAAAA ELHAAA OOOOxx 2310 5023 0 2 0 10 10 310 310 2310 2310 20 21 WKAAAA FLHAAA VVVVxx 6563 5024 1 3 3 3 63 563 563 1563 6563 126 127 LSAAAA GLHAAA AAAAxx 4886 5025 0 2 6 6 86 886 886 4886 4886 172 173 YFAAAA HLHAAA HHHHxx 9359 5026 1 3 9 19 59 359 1359 4359 9359 118 119 ZVAAAA ILHAAA OOOOxx 400 5027 0 0 0 0 0 400 400 400 400 0 1 KPAAAA JLHAAA VVVVxx 9742 5028 0 2 2 2 42 742 1742 4742 9742 84 85 SKAAAA KLHAAA AAAAxx 6736 5029 0 0 6 16 36 736 736 1736 6736 72 73 CZAAAA LLHAAA HHHHxx 8166 5030 0 2 6 6 66 166 166 3166 8166 132 133 CCAAAA MLHAAA OOOOxx 861 5031 1 1 1 1 61 861 861 861 861 122 123 DHAAAA NLHAAA VVVVxx 7492 5032 0 0 2 12 92 492 1492 2492 7492 184 185 ECAAAA OLHAAA AAAAxx 1155 5033 1 3 5 15 55 155 1155 1155 1155 110 111 LSAAAA PLHAAA HHHHxx 9769 5034 1 1 9 9 69 769 1769 4769 9769 138 139 TLAAAA QLHAAA OOOOxx 6843 5035 1 3 3 3 43 843 843 1843 6843 86 87 FDAAAA RLHAAA VVVVxx 5625 5036 1 1 5 5 25 625 1625 625 5625 50 51 JIAAAA SLHAAA AAAAxx 1910 5037 0 2 0 10 10 910 1910 1910 1910 20 21 MVAAAA TLHAAA HHHHxx 9796 5038 0 0 6 16 96 796 1796 4796 9796 192 193 UMAAAA ULHAAA OOOOxx 6950 5039 0 2 0 10 50 950 950 1950 6950 100 101 IHAAAA VLHAAA VVVVxx 3084 5040 0 0 4 4 84 84 1084 3084 3084 168 169 QOAAAA WLHAAA AAAAxx 2959 5041 1 3 9 19 59 959 959 2959 2959 118 119 VJAAAA XLHAAA HHHHxx 2093 5042 1 1 3 13 93 93 93 2093 2093 186 187 NCAAAA YLHAAA OOOOxx 2738 5043 0 2 8 18 38 738 738 2738 2738 76 77 IBAAAA ZLHAAA VVVVxx 6406 5044 0 2 6 6 6 406 406 1406 6406 12 13 KMAAAA AMHAAA AAAAxx 9082 5045 0 2 2 2 82 82 1082 4082 9082 164 165 ILAAAA BMHAAA HHHHxx 8568 5046 0 0 8 8 68 568 568 3568 8568 136 137 ORAAAA CMHAAA OOOOxx 3566 5047 0 2 6 6 66 566 1566 3566 3566 132 133 EHAAAA DMHAAA VVVVxx 3016 5048 0 0 6 16 16 16 1016 3016 3016 32 33 AMAAAA EMHAAA AAAAxx 1207 5049 1 3 7 7 7 207 1207 1207 1207 14 15 LUAAAA FMHAAA HHHHxx 4045 5050 1 1 5 5 45 45 45 4045 4045 90 91 PZAAAA GMHAAA OOOOxx 4173 5051 1 1 3 13 73 173 173 4173 4173 146 147 NEAAAA HMHAAA VVVVxx 3939 5052 1 3 9 19 39 939 1939 3939 3939 78 79 NVAAAA IMHAAA AAAAxx 9683 5053 1 3 3 3 83 683 1683 4683 9683 166 167 LIAAAA JMHAAA HHHHxx 1684 5054 0 0 4 4 84 684 1684 1684 1684 168 169 UMAAAA KMHAAA OOOOxx 9271 5055 1 3 1 11 71 271 1271 4271 9271 142 143 PSAAAA LMHAAA VVVVxx 9317 5056 1 1 7 17 17 317 1317 4317 9317 34 35 JUAAAA MMHAAA AAAAxx 5793 5057 1 1 3 13 93 793 1793 793 5793 186 187 VOAAAA NMHAAA HHHHxx 352 5058 0 0 2 12 52 352 352 352 352 104 105 ONAAAA OMHAAA OOOOxx 7328 5059 0 0 8 8 28 328 1328 2328 7328 56 57 WVAAAA PMHAAA VVVVxx 4582 5060 0 2 2 2 82 582 582 4582 4582 164 165 GUAAAA QMHAAA AAAAxx 7413 5061 1 1 3 13 13 413 1413 2413 7413 26 27 DZAAAA RMHAAA HHHHxx 6772 5062 0 0 2 12 72 772 772 1772 6772 144 145 MAAAAA SMHAAA OOOOxx 4973 5063 1 1 3 13 73 973 973 4973 4973 146 147 HJAAAA TMHAAA VVVVxx 7480 5064 0 0 0 0 80 480 1480 2480 7480 160 161 SBAAAA UMHAAA AAAAxx 5555 5065 1 3 5 15 55 555 1555 555 5555 110 111 RFAAAA VMHAAA HHHHxx 4227 5066 1 3 7 7 27 227 227 4227 4227 54 55 PGAAAA WMHAAA OOOOxx 4153 5067 1 1 3 13 53 153 153 4153 4153 106 107 TDAAAA XMHAAA VVVVxx 4601 5068 1 1 1 1 1 601 601 4601 4601 2 3 ZUAAAA YMHAAA AAAAxx 3782 5069 0 2 2 2 82 782 1782 3782 3782 164 165 MPAAAA ZMHAAA HHHHxx 3872 5070 0 0 2 12 72 872 1872 3872 3872 144 145 YSAAAA ANHAAA OOOOxx 893 5071 1 1 3 13 93 893 893 893 893 186 187 JIAAAA BNHAAA VVVVxx 2430 5072 0 2 0 10 30 430 430 2430 2430 60 61 MPAAAA CNHAAA AAAAxx 2591 5073 1 3 1 11 91 591 591 2591 2591 182 183 RVAAAA DNHAAA HHHHxx 264 5074 0 0 4 4 64 264 264 264 264 128 129 EKAAAA ENHAAA OOOOxx 6238 5075 0 2 8 18 38 238 238 1238 6238 76 77 YFAAAA FNHAAA VVVVxx 633 5076 1 1 3 13 33 633 633 633 633 66 67 JYAAAA GNHAAA AAAAxx 1029 5077 1 1 9 9 29 29 1029 1029 1029 58 59 PNAAAA HNHAAA HHHHxx 5934 5078 0 2 4 14 34 934 1934 934 5934 68 69 GUAAAA INHAAA OOOOxx 8694 5079 0 2 4 14 94 694 694 3694 8694 188 189 KWAAAA JNHAAA VVVVxx 7401 5080 1 1 1 1 1 401 1401 2401 7401 2 3 RYAAAA KNHAAA AAAAxx 1165 5081 1 1 5 5 65 165 1165 1165 1165 130 131 VSAAAA LNHAAA HHHHxx 9438 5082 0 2 8 18 38 438 1438 4438 9438 76 77 AZAAAA MNHAAA OOOOxx 4790 5083 0 2 0 10 90 790 790 4790 4790 180 181 GCAAAA NNHAAA VVVVxx 4531 5084 1 3 1 11 31 531 531 4531 4531 62 63 HSAAAA ONHAAA AAAAxx 6099 5085 1 3 9 19 99 99 99 1099 6099 198 199 PAAAAA PNHAAA HHHHxx 8236 5086 0 0 6 16 36 236 236 3236 8236 72 73 UEAAAA QNHAAA OOOOxx 8551 5087 1 3 1 11 51 551 551 3551 8551 102 103 XQAAAA RNHAAA VVVVxx 3128 5088 0 0 8 8 28 128 1128 3128 3128 56 57 IQAAAA SNHAAA AAAAxx 3504 5089 0 0 4 4 4 504 1504 3504 3504 8 9 UEAAAA TNHAAA HHHHxx 9071 5090 1 3 1 11 71 71 1071 4071 9071 142 143 XKAAAA UNHAAA OOOOxx 5930 5091 0 2 0 10 30 930 1930 930 5930 60 61 CUAAAA VNHAAA VVVVxx 6825 5092 1 1 5 5 25 825 825 1825 6825 50 51 NCAAAA WNHAAA AAAAxx 2218 5093 0 2 8 18 18 218 218 2218 2218 36 37 IHAAAA XNHAAA HHHHxx 3604 5094 0 0 4 4 4 604 1604 3604 3604 8 9 QIAAAA YNHAAA OOOOxx 5761 5095 1 1 1 1 61 761 1761 761 5761 122 123 PNAAAA ZNHAAA VVVVxx 5414 5096 0 2 4 14 14 414 1414 414 5414 28 29 GAAAAA AOHAAA AAAAxx 5892 5097 0 0 2 12 92 892 1892 892 5892 184 185 QSAAAA BOHAAA HHHHxx 4080 5098 0 0 0 0 80 80 80 4080 4080 160 161 YAAAAA COHAAA OOOOxx 8018 5099 0 2 8 18 18 18 18 3018 8018 36 37 KWAAAA DOHAAA VVVVxx 1757 5100 1 1 7 17 57 757 1757 1757 1757 114 115 PPAAAA EOHAAA AAAAxx 5854 5101 0 2 4 14 54 854 1854 854 5854 108 109 ERAAAA FOHAAA HHHHxx 1335 5102 1 3 5 15 35 335 1335 1335 1335 70 71 JZAAAA GOHAAA OOOOxx 3811 5103 1 3 1 11 11 811 1811 3811 3811 22 23 PQAAAA HOHAAA VVVVxx 9917 5104 1 1 7 17 17 917 1917 4917 9917 34 35 LRAAAA IOHAAA AAAAxx 5947 5105 1 3 7 7 47 947 1947 947 5947 94 95 TUAAAA JOHAAA HHHHxx 7263 5106 1 3 3 3 63 263 1263 2263 7263 126 127 JTAAAA KOHAAA OOOOxx 1730 5107 0 2 0 10 30 730 1730 1730 1730 60 61 OOAAAA LOHAAA VVVVxx 5747 5108 1 3 7 7 47 747 1747 747 5747 94 95 BNAAAA MOHAAA AAAAxx 3876 5109 0 0 6 16 76 876 1876 3876 3876 152 153 CTAAAA NOHAAA HHHHxx 2762 5110 0 2 2 2 62 762 762 2762 2762 124 125 GCAAAA OOHAAA OOOOxx 7613 5111 1 1 3 13 13 613 1613 2613 7613 26 27 VGAAAA POHAAA VVVVxx 152 5112 0 0 2 12 52 152 152 152 152 104 105 WFAAAA QOHAAA AAAAxx 3941 5113 1 1 1 1 41 941 1941 3941 3941 82 83 PVAAAA ROHAAA HHHHxx 5614 5114 0 2 4 14 14 614 1614 614 5614 28 29 YHAAAA SOHAAA OOOOxx 9279 5115 1 3 9 19 79 279 1279 4279 9279 158 159 XSAAAA TOHAAA VVVVxx 3048 5116 0 0 8 8 48 48 1048 3048 3048 96 97 GNAAAA UOHAAA AAAAxx 6152 5117 0 0 2 12 52 152 152 1152 6152 104 105 QCAAAA VOHAAA HHHHxx 5481 5118 1 1 1 1 81 481 1481 481 5481 162 163 VCAAAA WOHAAA OOOOxx 4675 5119 1 3 5 15 75 675 675 4675 4675 150 151 VXAAAA XOHAAA VVVVxx 3334 5120 0 2 4 14 34 334 1334 3334 3334 68 69 GYAAAA YOHAAA AAAAxx 4691 5121 1 3 1 11 91 691 691 4691 4691 182 183 LYAAAA ZOHAAA HHHHxx 803 5122 1 3 3 3 3 803 803 803 803 6 7 XEAAAA APHAAA OOOOxx 5409 5123 1 1 9 9 9 409 1409 409 5409 18 19 BAAAAA BPHAAA VVVVxx 1054 5124 0 2 4 14 54 54 1054 1054 1054 108 109 OOAAAA CPHAAA AAAAxx 103 5125 1 3 3 3 3 103 103 103 103 6 7 ZDAAAA DPHAAA HHHHxx 8565 5126 1 1 5 5 65 565 565 3565 8565 130 131 LRAAAA EPHAAA OOOOxx 4666 5127 0 2 6 6 66 666 666 4666 4666 132 133 MXAAAA FPHAAA VVVVxx 6634 5128 0 2 4 14 34 634 634 1634 6634 68 69 EVAAAA GPHAAA AAAAxx 5538 5129 0 2 8 18 38 538 1538 538 5538 76 77 AFAAAA HPHAAA HHHHxx 3789 5130 1 1 9 9 89 789 1789 3789 3789 178 179 TPAAAA IPHAAA OOOOxx 4641 5131 1 1 1 1 41 641 641 4641 4641 82 83 NWAAAA JPHAAA VVVVxx 2458 5132 0 2 8 18 58 458 458 2458 2458 116 117 OQAAAA KPHAAA AAAAxx 5667 5133 1 3 7 7 67 667 1667 667 5667 134 135 ZJAAAA LPHAAA HHHHxx 6524 5134 0 0 4 4 24 524 524 1524 6524 48 49 YQAAAA MPHAAA OOOOxx 9179 5135 1 3 9 19 79 179 1179 4179 9179 158 159 BPAAAA NPHAAA VVVVxx 6358 5136 0 2 8 18 58 358 358 1358 6358 116 117 OKAAAA OPHAAA AAAAxx 6668 5137 0 0 8 8 68 668 668 1668 6668 136 137 MWAAAA PPHAAA HHHHxx 6414 5138 0 2 4 14 14 414 414 1414 6414 28 29 SMAAAA QPHAAA OOOOxx 2813 5139 1 1 3 13 13 813 813 2813 2813 26 27 FEAAAA RPHAAA VVVVxx 8927 5140 1 3 7 7 27 927 927 3927 8927 54 55 JFAAAA SPHAAA AAAAxx 8695 5141 1 3 5 15 95 695 695 3695 8695 190 191 LWAAAA TPHAAA HHHHxx 363 5142 1 3 3 3 63 363 363 363 363 126 127 ZNAAAA UPHAAA OOOOxx 9966 5143 0 2 6 6 66 966 1966 4966 9966 132 133 ITAAAA VPHAAA VVVVxx 1323 5144 1 3 3 3 23 323 1323 1323 1323 46 47 XYAAAA WPHAAA AAAAxx 8211 5145 1 3 1 11 11 211 211 3211 8211 22 23 VDAAAA XPHAAA HHHHxx 4375 5146 1 3 5 15 75 375 375 4375 4375 150 151 HMAAAA YPHAAA OOOOxx 3257 5147 1 1 7 17 57 257 1257 3257 3257 114 115 HVAAAA ZPHAAA VVVVxx 6239 5148 1 3 9 19 39 239 239 1239 6239 78 79 ZFAAAA AQHAAA AAAAxx 3602 5149 0 2 2 2 2 602 1602 3602 3602 4 5 OIAAAA BQHAAA HHHHxx 9830 5150 0 2 0 10 30 830 1830 4830 9830 60 61 COAAAA CQHAAA OOOOxx 7826 5151 0 2 6 6 26 826 1826 2826 7826 52 53 APAAAA DQHAAA VVVVxx 2108 5152 0 0 8 8 8 108 108 2108 2108 16 17 CDAAAA EQHAAA AAAAxx 7245 5153 1 1 5 5 45 245 1245 2245 7245 90 91 RSAAAA FQHAAA HHHHxx 8330 5154 0 2 0 10 30 330 330 3330 8330 60 61 KIAAAA GQHAAA OOOOxx 7441 5155 1 1 1 1 41 441 1441 2441 7441 82 83 FAAAAA HQHAAA VVVVxx 9848 5156 0 0 8 8 48 848 1848 4848 9848 96 97 UOAAAA IQHAAA AAAAxx 1226 5157 0 2 6 6 26 226 1226 1226 1226 52 53 EVAAAA JQHAAA HHHHxx 414 5158 0 2 4 14 14 414 414 414 414 28 29 YPAAAA KQHAAA OOOOxx 1273 5159 1 1 3 13 73 273 1273 1273 1273 146 147 ZWAAAA LQHAAA VVVVxx 9866 5160 0 2 6 6 66 866 1866 4866 9866 132 133 MPAAAA MQHAAA AAAAxx 4633 5161 1 1 3 13 33 633 633 4633 4633 66 67 FWAAAA NQHAAA HHHHxx 8727 5162 1 3 7 7 27 727 727 3727 8727 54 55 RXAAAA OQHAAA OOOOxx 5308 5163 0 0 8 8 8 308 1308 308 5308 16 17 EWAAAA PQHAAA VVVVxx 1395 5164 1 3 5 15 95 395 1395 1395 1395 190 191 RBAAAA QQHAAA AAAAxx 1825 5165 1 1 5 5 25 825 1825 1825 1825 50 51 FSAAAA RQHAAA HHHHxx 7606 5166 0 2 6 6 6 606 1606 2606 7606 12 13 OGAAAA SQHAAA OOOOxx 9390 5167 0 2 0 10 90 390 1390 4390 9390 180 181 EXAAAA TQHAAA VVVVxx 2376 5168 0 0 6 16 76 376 376 2376 2376 152 153 KNAAAA UQHAAA AAAAxx 2377 5169 1 1 7 17 77 377 377 2377 2377 154 155 LNAAAA VQHAAA HHHHxx 5346 5170 0 2 6 6 46 346 1346 346 5346 92 93 QXAAAA WQHAAA OOOOxx 4140 5171 0 0 0 0 40 140 140 4140 4140 80 81 GDAAAA XQHAAA VVVVxx 6032 5172 0 0 2 12 32 32 32 1032 6032 64 65 AYAAAA YQHAAA AAAAxx 9453 5173 1 1 3 13 53 453 1453 4453 9453 106 107 PZAAAA ZQHAAA HHHHxx 9297 5174 1 1 7 17 97 297 1297 4297 9297 194 195 PTAAAA ARHAAA OOOOxx 6455 5175 1 3 5 15 55 455 455 1455 6455 110 111 HOAAAA BRHAAA VVVVxx 4458 5176 0 2 8 18 58 458 458 4458 4458 116 117 MPAAAA CRHAAA AAAAxx 9516 5177 0 0 6 16 16 516 1516 4516 9516 32 33 ACAAAA DRHAAA HHHHxx 6211 5178 1 3 1 11 11 211 211 1211 6211 22 23 XEAAAA ERHAAA OOOOxx 526 5179 0 2 6 6 26 526 526 526 526 52 53 GUAAAA FRHAAA VVVVxx 3570 5180 0 2 0 10 70 570 1570 3570 3570 140 141 IHAAAA GRHAAA AAAAxx 4885 5181 1 1 5 5 85 885 885 4885 4885 170 171 XFAAAA HRHAAA HHHHxx 6390 5182 0 2 0 10 90 390 390 1390 6390 180 181 ULAAAA IRHAAA OOOOxx 1606 5183 0 2 6 6 6 606 1606 1606 1606 12 13 UJAAAA JRHAAA VVVVxx 7850 5184 0 2 0 10 50 850 1850 2850 7850 100 101 YPAAAA KRHAAA AAAAxx 3315 5185 1 3 5 15 15 315 1315 3315 3315 30 31 NXAAAA LRHAAA HHHHxx 8322 5186 0 2 2 2 22 322 322 3322 8322 44 45 CIAAAA MRHAAA OOOOxx 3703 5187 1 3 3 3 3 703 1703 3703 3703 6 7 LMAAAA NRHAAA VVVVxx 9489 5188 1 1 9 9 89 489 1489 4489 9489 178 179 ZAAAAA ORHAAA AAAAxx 6104 5189 0 0 4 4 4 104 104 1104 6104 8 9 UAAAAA PRHAAA HHHHxx 3067 5190 1 3 7 7 67 67 1067 3067 3067 134 135 ZNAAAA QRHAAA OOOOxx 2521 5191 1 1 1 1 21 521 521 2521 2521 42 43 ZSAAAA RRHAAA VVVVxx 2581 5192 1 1 1 1 81 581 581 2581 2581 162 163 HVAAAA SRHAAA AAAAxx 595 5193 1 3 5 15 95 595 595 595 595 190 191 XWAAAA TRHAAA HHHHxx 8291 5194 1 3 1 11 91 291 291 3291 8291 182 183 XGAAAA URHAAA OOOOxx 1727 5195 1 3 7 7 27 727 1727 1727 1727 54 55 LOAAAA VRHAAA VVVVxx 6847 5196 1 3 7 7 47 847 847 1847 6847 94 95 JDAAAA WRHAAA AAAAxx 7494 5197 0 2 4 14 94 494 1494 2494 7494 188 189 GCAAAA XRHAAA HHHHxx 7093 5198 1 1 3 13 93 93 1093 2093 7093 186 187 VMAAAA YRHAAA OOOOxx 7357 5199 1 1 7 17 57 357 1357 2357 7357 114 115 ZWAAAA ZRHAAA VVVVxx 620 5200 0 0 0 0 20 620 620 620 620 40 41 WXAAAA ASHAAA AAAAxx 2460 5201 0 0 0 0 60 460 460 2460 2460 120 121 QQAAAA BSHAAA HHHHxx 1598 5202 0 2 8 18 98 598 1598 1598 1598 196 197 MJAAAA CSHAAA OOOOxx 4112 5203 0 0 2 12 12 112 112 4112 4112 24 25 ECAAAA DSHAAA VVVVxx 2956 5204 0 0 6 16 56 956 956 2956 2956 112 113 SJAAAA ESHAAA AAAAxx 3193 5205 1 1 3 13 93 193 1193 3193 3193 186 187 VSAAAA FSHAAA HHHHxx 6356 5206 0 0 6 16 56 356 356 1356 6356 112 113 MKAAAA GSHAAA OOOOxx 730 5207 0 2 0 10 30 730 730 730 730 60 61 CCAAAA HSHAAA VVVVxx 8826 5208 0 2 6 6 26 826 826 3826 8826 52 53 MBAAAA ISHAAA AAAAxx 9036 5209 0 0 6 16 36 36 1036 4036 9036 72 73 OJAAAA JSHAAA HHHHxx 2085 5210 1 1 5 5 85 85 85 2085 2085 170 171 FCAAAA KSHAAA OOOOxx 9007 5211 1 3 7 7 7 7 1007 4007 9007 14 15 LIAAAA LSHAAA VVVVxx 6047 5212 1 3 7 7 47 47 47 1047 6047 94 95 PYAAAA MSHAAA AAAAxx 3953 5213 1 1 3 13 53 953 1953 3953 3953 106 107 BWAAAA NSHAAA HHHHxx 1214 5214 0 2 4 14 14 214 1214 1214 1214 28 29 SUAAAA OSHAAA OOOOxx 4814 5215 0 2 4 14 14 814 814 4814 4814 28 29 EDAAAA PSHAAA VVVVxx 5738 5216 0 2 8 18 38 738 1738 738 5738 76 77 SMAAAA QSHAAA AAAAxx 7176 5217 0 0 6 16 76 176 1176 2176 7176 152 153 AQAAAA RSHAAA HHHHxx 3609 5218 1 1 9 9 9 609 1609 3609 3609 18 19 VIAAAA SSHAAA OOOOxx 592 5219 0 0 2 12 92 592 592 592 592 184 185 UWAAAA TSHAAA VVVVxx 9391 5220 1 3 1 11 91 391 1391 4391 9391 182 183 FXAAAA USHAAA AAAAxx 5345 5221 1 1 5 5 45 345 1345 345 5345 90 91 PXAAAA VSHAAA HHHHxx 1171 5222 1 3 1 11 71 171 1171 1171 1171 142 143 BTAAAA WSHAAA OOOOxx 7238 5223 0 2 8 18 38 238 1238 2238 7238 76 77 KSAAAA XSHAAA VVVVxx 7561 5224 1 1 1 1 61 561 1561 2561 7561 122 123 VEAAAA YSHAAA AAAAxx 5876 5225 0 0 6 16 76 876 1876 876 5876 152 153 ASAAAA ZSHAAA HHHHxx 6611 5226 1 3 1 11 11 611 611 1611 6611 22 23 HUAAAA ATHAAA OOOOxx 7300 5227 0 0 0 0 0 300 1300 2300 7300 0 1 UUAAAA BTHAAA VVVVxx 1506 5228 0 2 6 6 6 506 1506 1506 1506 12 13 YFAAAA CTHAAA AAAAxx 1153 5229 1 1 3 13 53 153 1153 1153 1153 106 107 JSAAAA DTHAAA HHHHxx 3831 5230 1 3 1 11 31 831 1831 3831 3831 62 63 JRAAAA ETHAAA OOOOxx 9255 5231 1 3 5 15 55 255 1255 4255 9255 110 111 ZRAAAA FTHAAA VVVVxx 1841 5232 1 1 1 1 41 841 1841 1841 1841 82 83 VSAAAA GTHAAA AAAAxx 5075 5233 1 3 5 15 75 75 1075 75 5075 150 151 FNAAAA HTHAAA HHHHxx 101 5234 1 1 1 1 1 101 101 101 101 2 3 XDAAAA ITHAAA OOOOxx 2627 5235 1 3 7 7 27 627 627 2627 2627 54 55 BXAAAA JTHAAA VVVVxx 7078 5236 0 2 8 18 78 78 1078 2078 7078 156 157 GMAAAA KTHAAA AAAAxx 2850 5237 0 2 0 10 50 850 850 2850 2850 100 101 QFAAAA LTHAAA HHHHxx 8703 5238 1 3 3 3 3 703 703 3703 8703 6 7 TWAAAA MTHAAA OOOOxx 4101 5239 1 1 1 1 1 101 101 4101 4101 2 3 TBAAAA NTHAAA VVVVxx 318 5240 0 2 8 18 18 318 318 318 318 36 37 GMAAAA OTHAAA AAAAxx 6452 5241 0 0 2 12 52 452 452 1452 6452 104 105 EOAAAA PTHAAA HHHHxx 5558 5242 0 2 8 18 58 558 1558 558 5558 116 117 UFAAAA QTHAAA OOOOxx 3127 5243 1 3 7 7 27 127 1127 3127 3127 54 55 HQAAAA RTHAAA VVVVxx 535 5244 1 3 5 15 35 535 535 535 535 70 71 PUAAAA STHAAA AAAAxx 270 5245 0 2 0 10 70 270 270 270 270 140 141 KKAAAA TTHAAA HHHHxx 4038 5246 0 2 8 18 38 38 38 4038 4038 76 77 IZAAAA UTHAAA OOOOxx 3404 5247 0 0 4 4 4 404 1404 3404 3404 8 9 YAAAAA VTHAAA VVVVxx 2374 5248 0 2 4 14 74 374 374 2374 2374 148 149 INAAAA WTHAAA AAAAxx 6446 5249 0 2 6 6 46 446 446 1446 6446 92 93 YNAAAA XTHAAA HHHHxx 7758 5250 0 2 8 18 58 758 1758 2758 7758 116 117 KMAAAA YTHAAA OOOOxx 356 5251 0 0 6 16 56 356 356 356 356 112 113 SNAAAA ZTHAAA VVVVxx 9197 5252 1 1 7 17 97 197 1197 4197 9197 194 195 TPAAAA AUHAAA AAAAxx 9765 5253 1 1 5 5 65 765 1765 4765 9765 130 131 PLAAAA BUHAAA HHHHxx 4974 5254 0 2 4 14 74 974 974 4974 4974 148 149 IJAAAA CUHAAA OOOOxx 442 5255 0 2 2 2 42 442 442 442 442 84 85 ARAAAA DUHAAA VVVVxx 4349 5256 1 1 9 9 49 349 349 4349 4349 98 99 HLAAAA EUHAAA AAAAxx 6119 5257 1 3 9 19 19 119 119 1119 6119 38 39 JBAAAA FUHAAA HHHHxx 7574 5258 0 2 4 14 74 574 1574 2574 7574 148 149 IFAAAA GUHAAA OOOOxx 4445 5259 1 1 5 5 45 445 445 4445 4445 90 91 ZOAAAA HUHAAA VVVVxx 940 5260 0 0 0 0 40 940 940 940 940 80 81 EKAAAA IUHAAA AAAAxx 1875 5261 1 3 5 15 75 875 1875 1875 1875 150 151 DUAAAA JUHAAA HHHHxx 5951 5262 1 3 1 11 51 951 1951 951 5951 102 103 XUAAAA KUHAAA OOOOxx 9132 5263 0 0 2 12 32 132 1132 4132 9132 64 65 GNAAAA LUHAAA VVVVxx 6913 5264 1 1 3 13 13 913 913 1913 6913 26 27 XFAAAA MUHAAA AAAAxx 3308 5265 0 0 8 8 8 308 1308 3308 3308 16 17 GXAAAA NUHAAA HHHHxx 7553 5266 1 1 3 13 53 553 1553 2553 7553 106 107 NEAAAA OUHAAA OOOOxx 2138 5267 0 2 8 18 38 138 138 2138 2138 76 77 GEAAAA PUHAAA VVVVxx 6252 5268 0 0 2 12 52 252 252 1252 6252 104 105 MGAAAA QUHAAA AAAAxx 2171 5269 1 3 1 11 71 171 171 2171 2171 142 143 NFAAAA RUHAAA HHHHxx 4159 5270 1 3 9 19 59 159 159 4159 4159 118 119 ZDAAAA SUHAAA OOOOxx 2401 5271 1 1 1 1 1 401 401 2401 2401 2 3 JOAAAA TUHAAA VVVVxx 6553 5272 1 1 3 13 53 553 553 1553 6553 106 107 BSAAAA UUHAAA AAAAxx 5217 5273 1 1 7 17 17 217 1217 217 5217 34 35 RSAAAA VUHAAA HHHHxx 1405 5274 1 1 5 5 5 405 1405 1405 1405 10 11 BCAAAA WUHAAA OOOOxx 1494 5275 0 2 4 14 94 494 1494 1494 1494 188 189 MFAAAA XUHAAA VVVVxx 5553 5276 1 1 3 13 53 553 1553 553 5553 106 107 PFAAAA YUHAAA AAAAxx 8296 5277 0 0 6 16 96 296 296 3296 8296 192 193 CHAAAA ZUHAAA HHHHxx 6565 5278 1 1 5 5 65 565 565 1565 6565 130 131 NSAAAA AVHAAA OOOOxx 817 5279 1 1 7 17 17 817 817 817 817 34 35 LFAAAA BVHAAA VVVVxx 6947 5280 1 3 7 7 47 947 947 1947 6947 94 95 FHAAAA CVHAAA AAAAxx 4184 5281 0 0 4 4 84 184 184 4184 4184 168 169 YEAAAA DVHAAA HHHHxx 6577 5282 1 1 7 17 77 577 577 1577 6577 154 155 ZSAAAA EVHAAA OOOOxx 6424 5283 0 0 4 4 24 424 424 1424 6424 48 49 CNAAAA FVHAAA VVVVxx 2482 5284 0 2 2 2 82 482 482 2482 2482 164 165 MRAAAA GVHAAA AAAAxx 6874 5285 0 2 4 14 74 874 874 1874 6874 148 149 KEAAAA HVHAAA HHHHxx 7601 5286 1 1 1 1 1 601 1601 2601 7601 2 3 JGAAAA IVHAAA OOOOxx 4552 5287 0 0 2 12 52 552 552 4552 4552 104 105 CTAAAA JVHAAA VVVVxx 8406 5288 0 2 6 6 6 406 406 3406 8406 12 13 ILAAAA KVHAAA AAAAxx 2924 5289 0 0 4 4 24 924 924 2924 2924 48 49 MIAAAA LVHAAA HHHHxx 8255 5290 1 3 5 15 55 255 255 3255 8255 110 111 NFAAAA MVHAAA OOOOxx 4920 5291 0 0 0 0 20 920 920 4920 4920 40 41 GHAAAA NVHAAA VVVVxx 228 5292 0 0 8 8 28 228 228 228 228 56 57 UIAAAA OVHAAA AAAAxx 9431 5293 1 3 1 11 31 431 1431 4431 9431 62 63 TYAAAA PVHAAA HHHHxx 4021 5294 1 1 1 1 21 21 21 4021 4021 42 43 RYAAAA QVHAAA OOOOxx 2966 5295 0 2 6 6 66 966 966 2966 2966 132 133 CKAAAA RVHAAA VVVVxx 2862 5296 0 2 2 2 62 862 862 2862 2862 124 125 CGAAAA SVHAAA AAAAxx 4303 5297 1 3 3 3 3 303 303 4303 4303 6 7 NJAAAA TVHAAA HHHHxx 9643 5298 1 3 3 3 43 643 1643 4643 9643 86 87 XGAAAA UVHAAA OOOOxx 3008 5299 0 0 8 8 8 8 1008 3008 3008 16 17 SLAAAA VVHAAA VVVVxx 7476 5300 0 0 6 16 76 476 1476 2476 7476 152 153 OBAAAA WVHAAA AAAAxx 3686 5301 0 2 6 6 86 686 1686 3686 3686 172 173 ULAAAA XVHAAA HHHHxx 9051 5302 1 3 1 11 51 51 1051 4051 9051 102 103 DKAAAA YVHAAA OOOOxx 6592 5303 0 0 2 12 92 592 592 1592 6592 184 185 OTAAAA ZVHAAA VVVVxx 924 5304 0 0 4 4 24 924 924 924 924 48 49 OJAAAA AWHAAA AAAAxx 4406 5305 0 2 6 6 6 406 406 4406 4406 12 13 MNAAAA BWHAAA HHHHxx 5233 5306 1 1 3 13 33 233 1233 233 5233 66 67 HTAAAA CWHAAA OOOOxx 8881 5307 1 1 1 1 81 881 881 3881 8881 162 163 PDAAAA DWHAAA VVVVxx 2212 5308 0 0 2 12 12 212 212 2212 2212 24 25 CHAAAA EWHAAA AAAAxx 5804 5309 0 0 4 4 4 804 1804 804 5804 8 9 GPAAAA FWHAAA HHHHxx 2990 5310 0 2 0 10 90 990 990 2990 2990 180 181 ALAAAA GWHAAA OOOOxx 4069 5311 1 1 9 9 69 69 69 4069 4069 138 139 NAAAAA HWHAAA VVVVxx 5380 5312 0 0 0 0 80 380 1380 380 5380 160 161 YYAAAA IWHAAA AAAAxx 5016 5313 0 0 6 16 16 16 1016 16 5016 32 33 YKAAAA JWHAAA HHHHxx 5056 5314 0 0 6 16 56 56 1056 56 5056 112 113 MMAAAA KWHAAA OOOOxx 3732 5315 0 0 2 12 32 732 1732 3732 3732 64 65 ONAAAA LWHAAA VVVVxx 5527 5316 1 3 7 7 27 527 1527 527 5527 54 55 PEAAAA MWHAAA AAAAxx 1151 5317 1 3 1 11 51 151 1151 1151 1151 102 103 HSAAAA NWHAAA HHHHxx 7900 5318 0 0 0 0 0 900 1900 2900 7900 0 1 WRAAAA OWHAAA OOOOxx 1660 5319 0 0 0 0 60 660 1660 1660 1660 120 121 WLAAAA PWHAAA VVVVxx 8064 5320 0 0 4 4 64 64 64 3064 8064 128 129 EYAAAA QWHAAA AAAAxx 8240 5321 0 0 0 0 40 240 240 3240 8240 80 81 YEAAAA RWHAAA HHHHxx 413 5322 1 1 3 13 13 413 413 413 413 26 27 XPAAAA SWHAAA OOOOxx 8311 5323 1 3 1 11 11 311 311 3311 8311 22 23 RHAAAA TWHAAA VVVVxx 1065 5324 1 1 5 5 65 65 1065 1065 1065 130 131 ZOAAAA UWHAAA AAAAxx 2741 5325 1 1 1 1 41 741 741 2741 2741 82 83 LBAAAA VWHAAA HHHHxx 5306 5326 0 2 6 6 6 306 1306 306 5306 12 13 CWAAAA WWHAAA OOOOxx 5464 5327 0 0 4 4 64 464 1464 464 5464 128 129 ECAAAA XWHAAA VVVVxx 4237 5328 1 1 7 17 37 237 237 4237 4237 74 75 ZGAAAA YWHAAA AAAAxx 3822 5329 0 2 2 2 22 822 1822 3822 3822 44 45 ARAAAA ZWHAAA HHHHxx 2548 5330 0 0 8 8 48 548 548 2548 2548 96 97 AUAAAA AXHAAA OOOOxx 2688 5331 0 0 8 8 88 688 688 2688 2688 176 177 KZAAAA BXHAAA VVVVxx 8061 5332 1 1 1 1 61 61 61 3061 8061 122 123 BYAAAA CXHAAA AAAAxx 9340 5333 0 0 0 0 40 340 1340 4340 9340 80 81 GVAAAA DXHAAA HHHHxx 4031 5334 1 3 1 11 31 31 31 4031 4031 62 63 BZAAAA EXHAAA OOOOxx 2635 5335 1 3 5 15 35 635 635 2635 2635 70 71 JXAAAA FXHAAA VVVVxx 809 5336 1 1 9 9 9 809 809 809 809 18 19 DFAAAA GXHAAA AAAAxx 3209 5337 1 1 9 9 9 209 1209 3209 3209 18 19 LTAAAA HXHAAA HHHHxx 3825 5338 1 1 5 5 25 825 1825 3825 3825 50 51 DRAAAA IXHAAA OOOOxx 1448 5339 0 0 8 8 48 448 1448 1448 1448 96 97 SDAAAA JXHAAA VVVVxx 9077 5340 1 1 7 17 77 77 1077 4077 9077 154 155 DLAAAA KXHAAA AAAAxx 3730 5341 0 2 0 10 30 730 1730 3730 3730 60 61 MNAAAA LXHAAA HHHHxx 9596 5342 0 0 6 16 96 596 1596 4596 9596 192 193 CFAAAA MXHAAA OOOOxx 3563 5343 1 3 3 3 63 563 1563 3563 3563 126 127 BHAAAA NXHAAA VVVVxx 4116 5344 0 0 6 16 16 116 116 4116 4116 32 33 ICAAAA OXHAAA AAAAxx 4825 5345 1 1 5 5 25 825 825 4825 4825 50 51 PDAAAA PXHAAA HHHHxx 8376 5346 0 0 6 16 76 376 376 3376 8376 152 153 EKAAAA QXHAAA OOOOxx 3917 5347 1 1 7 17 17 917 1917 3917 3917 34 35 RUAAAA RXHAAA VVVVxx 4407 5348 1 3 7 7 7 407 407 4407 4407 14 15 NNAAAA SXHAAA AAAAxx 8202 5349 0 2 2 2 2 202 202 3202 8202 4 5 MDAAAA TXHAAA HHHHxx 7675 5350 1 3 5 15 75 675 1675 2675 7675 150 151 FJAAAA UXHAAA OOOOxx 4104 5351 0 0 4 4 4 104 104 4104 4104 8 9 WBAAAA VXHAAA VVVVxx 9225 5352 1 1 5 5 25 225 1225 4225 9225 50 51 VQAAAA WXHAAA AAAAxx 2834 5353 0 2 4 14 34 834 834 2834 2834 68 69 AFAAAA XXHAAA HHHHxx 1227 5354 1 3 7 7 27 227 1227 1227 1227 54 55 FVAAAA YXHAAA OOOOxx 3383 5355 1 3 3 3 83 383 1383 3383 3383 166 167 DAAAAA ZXHAAA VVVVxx 67 5356 1 3 7 7 67 67 67 67 67 134 135 PCAAAA AYHAAA AAAAxx 1751 5357 1 3 1 11 51 751 1751 1751 1751 102 103 JPAAAA BYHAAA HHHHxx 8054 5358 0 2 4 14 54 54 54 3054 8054 108 109 UXAAAA CYHAAA OOOOxx 8571 5359 1 3 1 11 71 571 571 3571 8571 142 143 RRAAAA DYHAAA VVVVxx 2466 5360 0 2 6 6 66 466 466 2466 2466 132 133 WQAAAA EYHAAA AAAAxx 9405 5361 1 1 5 5 5 405 1405 4405 9405 10 11 TXAAAA FYHAAA HHHHxx 6883 5362 1 3 3 3 83 883 883 1883 6883 166 167 TEAAAA GYHAAA OOOOxx 4301 5363 1 1 1 1 1 301 301 4301 4301 2 3 LJAAAA HYHAAA VVVVxx 3705 5364 1 1 5 5 5 705 1705 3705 3705 10 11 NMAAAA IYHAAA AAAAxx 5420 5365 0 0 0 0 20 420 1420 420 5420 40 41 MAAAAA JYHAAA HHHHxx 3692 5366 0 0 2 12 92 692 1692 3692 3692 184 185 AMAAAA KYHAAA OOOOxx 6851 5367 1 3 1 11 51 851 851 1851 6851 102 103 NDAAAA LYHAAA VVVVxx 9363 5368 1 3 3 3 63 363 1363 4363 9363 126 127 DWAAAA MYHAAA AAAAxx 2269 5369 1 1 9 9 69 269 269 2269 2269 138 139 HJAAAA NYHAAA HHHHxx 4918 5370 0 2 8 18 18 918 918 4918 4918 36 37 EHAAAA OYHAAA OOOOxx 4297 5371 1 1 7 17 97 297 297 4297 4297 194 195 HJAAAA PYHAAA VVVVxx 1836 5372 0 0 6 16 36 836 1836 1836 1836 72 73 QSAAAA QYHAAA AAAAxx 237 5373 1 1 7 17 37 237 237 237 237 74 75 DJAAAA RYHAAA HHHHxx 6131 5374 1 3 1 11 31 131 131 1131 6131 62 63 VBAAAA SYHAAA OOOOxx 3174 5375 0 2 4 14 74 174 1174 3174 3174 148 149 CSAAAA TYHAAA VVVVxx 9987 5376 1 3 7 7 87 987 1987 4987 9987 174 175 DUAAAA UYHAAA AAAAxx 3630 5377 0 2 0 10 30 630 1630 3630 3630 60 61 QJAAAA VYHAAA HHHHxx 2899 5378 1 3 9 19 99 899 899 2899 2899 198 199 NHAAAA WYHAAA OOOOxx 4079 5379 1 3 9 19 79 79 79 4079 4079 158 159 XAAAAA XYHAAA VVVVxx 5049 5380 1 1 9 9 49 49 1049 49 5049 98 99 FMAAAA YYHAAA AAAAxx 2963 5381 1 3 3 3 63 963 963 2963 2963 126 127 ZJAAAA ZYHAAA HHHHxx 3962 5382 0 2 2 2 62 962 1962 3962 3962 124 125 KWAAAA AZHAAA OOOOxx 7921 5383 1 1 1 1 21 921 1921 2921 7921 42 43 RSAAAA BZHAAA VVVVxx 3967 5384 1 3 7 7 67 967 1967 3967 3967 134 135 PWAAAA CZHAAA AAAAxx 2752 5385 0 0 2 12 52 752 752 2752 2752 104 105 WBAAAA DZHAAA HHHHxx 7944 5386 0 0 4 4 44 944 1944 2944 7944 88 89 OTAAAA EZHAAA OOOOxx 2205 5387 1 1 5 5 5 205 205 2205 2205 10 11 VGAAAA FZHAAA VVVVxx 5035 5388 1 3 5 15 35 35 1035 35 5035 70 71 RLAAAA GZHAAA AAAAxx 1425 5389 1 1 5 5 25 425 1425 1425 1425 50 51 VCAAAA HZHAAA HHHHxx 832 5390 0 0 2 12 32 832 832 832 832 64 65 AGAAAA IZHAAA OOOOxx 1447 5391 1 3 7 7 47 447 1447 1447 1447 94 95 RDAAAA JZHAAA VVVVxx 6108 5392 0 0 8 8 8 108 108 1108 6108 16 17 YAAAAA KZHAAA AAAAxx 4936 5393 0 0 6 16 36 936 936 4936 4936 72 73 WHAAAA LZHAAA HHHHxx 7704 5394 0 0 4 4 4 704 1704 2704 7704 8 9 IKAAAA MZHAAA OOOOxx 142 5395 0 2 2 2 42 142 142 142 142 84 85 MFAAAA NZHAAA VVVVxx 4272 5396 0 0 2 12 72 272 272 4272 4272 144 145 IIAAAA OZHAAA AAAAxx 7667 5397 1 3 7 7 67 667 1667 2667 7667 134 135 XIAAAA PZHAAA HHHHxx 366 5398 0 2 6 6 66 366 366 366 366 132 133 COAAAA QZHAAA OOOOxx 8866 5399 0 2 6 6 66 866 866 3866 8866 132 133 ADAAAA RZHAAA VVVVxx 7712 5400 0 0 2 12 12 712 1712 2712 7712 24 25 QKAAAA SZHAAA AAAAxx 3880 5401 0 0 0 0 80 880 1880 3880 3880 160 161 GTAAAA TZHAAA HHHHxx 4631 5402 1 3 1 11 31 631 631 4631 4631 62 63 DWAAAA UZHAAA OOOOxx 2789 5403 1 1 9 9 89 789 789 2789 2789 178 179 HDAAAA VZHAAA VVVVxx 7720 5404 0 0 0 0 20 720 1720 2720 7720 40 41 YKAAAA WZHAAA AAAAxx 7618 5405 0 2 8 18 18 618 1618 2618 7618 36 37 AHAAAA XZHAAA HHHHxx 4990 5406 0 2 0 10 90 990 990 4990 4990 180 181 YJAAAA YZHAAA OOOOxx 7918 5407 0 2 8 18 18 918 1918 2918 7918 36 37 OSAAAA ZZHAAA VVVVxx 5067 5408 1 3 7 7 67 67 1067 67 5067 134 135 XMAAAA AAIAAA AAAAxx 6370 5409 0 2 0 10 70 370 370 1370 6370 140 141 ALAAAA BAIAAA HHHHxx 2268 5410 0 0 8 8 68 268 268 2268 2268 136 137 GJAAAA CAIAAA OOOOxx 1949 5411 1 1 9 9 49 949 1949 1949 1949 98 99 ZWAAAA DAIAAA VVVVxx 5503 5412 1 3 3 3 3 503 1503 503 5503 6 7 RDAAAA EAIAAA AAAAxx 9951 5413 1 3 1 11 51 951 1951 4951 9951 102 103 TSAAAA FAIAAA HHHHxx 6823 5414 1 3 3 3 23 823 823 1823 6823 46 47 LCAAAA GAIAAA OOOOxx 6287 5415 1 3 7 7 87 287 287 1287 6287 174 175 VHAAAA HAIAAA VVVVxx 6016 5416 0 0 6 16 16 16 16 1016 6016 32 33 KXAAAA IAIAAA AAAAxx 1977 5417 1 1 7 17 77 977 1977 1977 1977 154 155 BYAAAA JAIAAA HHHHxx 8579 5418 1 3 9 19 79 579 579 3579 8579 158 159 ZRAAAA KAIAAA OOOOxx 6204 5419 0 0 4 4 4 204 204 1204 6204 8 9 QEAAAA LAIAAA VVVVxx 9764 5420 0 0 4 4 64 764 1764 4764 9764 128 129 OLAAAA MAIAAA AAAAxx 2005 5421 1 1 5 5 5 5 5 2005 2005 10 11 DZAAAA NAIAAA HHHHxx 1648 5422 0 0 8 8 48 648 1648 1648 1648 96 97 KLAAAA OAIAAA OOOOxx 2457 5423 1 1 7 17 57 457 457 2457 2457 114 115 NQAAAA PAIAAA VVVVxx 2698 5424 0 2 8 18 98 698 698 2698 2698 196 197 UZAAAA QAIAAA AAAAxx 7730 5425 0 2 0 10 30 730 1730 2730 7730 60 61 ILAAAA RAIAAA HHHHxx 7287 5426 1 3 7 7 87 287 1287 2287 7287 174 175 HUAAAA SAIAAA OOOOxx 2937 5427 1 1 7 17 37 937 937 2937 2937 74 75 ZIAAAA TAIAAA VVVVxx 6824 5428 0 0 4 4 24 824 824 1824 6824 48 49 MCAAAA UAIAAA AAAAxx 9256 5429 0 0 6 16 56 256 1256 4256 9256 112 113 ASAAAA VAIAAA HHHHxx 4810 5430 0 2 0 10 10 810 810 4810 4810 20 21 ADAAAA WAIAAA OOOOxx 3869 5431 1 1 9 9 69 869 1869 3869 3869 138 139 VSAAAA XAIAAA VVVVxx 1993 5432 1 1 3 13 93 993 1993 1993 1993 186 187 RYAAAA YAIAAA AAAAxx 6048 5433 0 0 8 8 48 48 48 1048 6048 96 97 QYAAAA ZAIAAA HHHHxx 6922 5434 0 2 2 2 22 922 922 1922 6922 44 45 GGAAAA ABIAAA OOOOxx 8 5435 0 0 8 8 8 8 8 8 8 16 17 IAAAAA BBIAAA VVVVxx 6706 5436 0 2 6 6 6 706 706 1706 6706 12 13 YXAAAA CBIAAA AAAAxx 9159 5437 1 3 9 19 59 159 1159 4159 9159 118 119 HOAAAA DBIAAA HHHHxx 7020 5438 0 0 0 0 20 20 1020 2020 7020 40 41 AKAAAA EBIAAA OOOOxx 767 5439 1 3 7 7 67 767 767 767 767 134 135 NDAAAA FBIAAA VVVVxx 8602 5440 0 2 2 2 2 602 602 3602 8602 4 5 WSAAAA GBIAAA AAAAxx 4442 5441 0 2 2 2 42 442 442 4442 4442 84 85 WOAAAA HBIAAA HHHHxx 2040 5442 0 0 0 0 40 40 40 2040 2040 80 81 MAAAAA IBIAAA OOOOxx 5493 5443 1 1 3 13 93 493 1493 493 5493 186 187 HDAAAA JBIAAA VVVVxx 275 5444 1 3 5 15 75 275 275 275 275 150 151 PKAAAA KBIAAA AAAAxx 8876 5445 0 0 6 16 76 876 876 3876 8876 152 153 KDAAAA LBIAAA HHHHxx 7381 5446 1 1 1 1 81 381 1381 2381 7381 162 163 XXAAAA MBIAAA OOOOxx 1827 5447 1 3 7 7 27 827 1827 1827 1827 54 55 HSAAAA NBIAAA VVVVxx 3537 5448 1 1 7 17 37 537 1537 3537 3537 74 75 BGAAAA OBIAAA AAAAxx 6978 5449 0 2 8 18 78 978 978 1978 6978 156 157 KIAAAA PBIAAA HHHHxx 6160 5450 0 0 0 0 60 160 160 1160 6160 120 121 YCAAAA QBIAAA OOOOxx 9219 5451 1 3 9 19 19 219 1219 4219 9219 38 39 PQAAAA RBIAAA VVVVxx 5034 5452 0 2 4 14 34 34 1034 34 5034 68 69 QLAAAA SBIAAA AAAAxx 8463 5453 1 3 3 3 63 463 463 3463 8463 126 127 NNAAAA TBIAAA HHHHxx 2038 5454 0 2 8 18 38 38 38 2038 2038 76 77 KAAAAA UBIAAA OOOOxx 9562 5455 0 2 2 2 62 562 1562 4562 9562 124 125 UDAAAA VBIAAA VVVVxx 2687 5456 1 3 7 7 87 687 687 2687 2687 174 175 JZAAAA WBIAAA AAAAxx 5092 5457 0 0 2 12 92 92 1092 92 5092 184 185 WNAAAA XBIAAA HHHHxx 539 5458 1 3 9 19 39 539 539 539 539 78 79 TUAAAA YBIAAA OOOOxx 2139 5459 1 3 9 19 39 139 139 2139 2139 78 79 HEAAAA ZBIAAA VVVVxx 9221 5460 1 1 1 1 21 221 1221 4221 9221 42 43 RQAAAA ACIAAA AAAAxx 965 5461 1 1 5 5 65 965 965 965 965 130 131 DLAAAA BCIAAA HHHHxx 6051 5462 1 3 1 11 51 51 51 1051 6051 102 103 TYAAAA CCIAAA OOOOxx 5822 5463 0 2 2 2 22 822 1822 822 5822 44 45 YPAAAA DCIAAA VVVVxx 6397 5464 1 1 7 17 97 397 397 1397 6397 194 195 BMAAAA ECIAAA AAAAxx 2375 5465 1 3 5 15 75 375 375 2375 2375 150 151 JNAAAA FCIAAA HHHHxx 9415 5466 1 3 5 15 15 415 1415 4415 9415 30 31 DYAAAA GCIAAA OOOOxx 6552 5467 0 0 2 12 52 552 552 1552 6552 104 105 ASAAAA HCIAAA VVVVxx 2248 5468 0 0 8 8 48 248 248 2248 2248 96 97 MIAAAA ICIAAA AAAAxx 2611 5469 1 3 1 11 11 611 611 2611 2611 22 23 LWAAAA JCIAAA HHHHxx 9609 5470 1 1 9 9 9 609 1609 4609 9609 18 19 PFAAAA KCIAAA OOOOxx 2132 5471 0 0 2 12 32 132 132 2132 2132 64 65 AEAAAA LCIAAA VVVVxx 8452 5472 0 0 2 12 52 452 452 3452 8452 104 105 CNAAAA MCIAAA AAAAxx 9407 5473 1 3 7 7 7 407 1407 4407 9407 14 15 VXAAAA NCIAAA HHHHxx 2814 5474 0 2 4 14 14 814 814 2814 2814 28 29 GEAAAA OCIAAA OOOOxx 1889 5475 1 1 9 9 89 889 1889 1889 1889 178 179 RUAAAA PCIAAA VVVVxx 7489 5476 1 1 9 9 89 489 1489 2489 7489 178 179 BCAAAA QCIAAA AAAAxx 2255 5477 1 3 5 15 55 255 255 2255 2255 110 111 TIAAAA RCIAAA HHHHxx 3380 5478 0 0 0 0 80 380 1380 3380 3380 160 161 AAAAAA SCIAAA OOOOxx 1167 5479 1 3 7 7 67 167 1167 1167 1167 134 135 XSAAAA TCIAAA VVVVxx 5369 5480 1 1 9 9 69 369 1369 369 5369 138 139 NYAAAA UCIAAA AAAAxx 2378 5481 0 2 8 18 78 378 378 2378 2378 156 157 MNAAAA VCIAAA HHHHxx 8315 5482 1 3 5 15 15 315 315 3315 8315 30 31 VHAAAA WCIAAA OOOOxx 2934 5483 0 2 4 14 34 934 934 2934 2934 68 69 WIAAAA XCIAAA VVVVxx 7924 5484 0 0 4 4 24 924 1924 2924 7924 48 49 USAAAA YCIAAA AAAAxx 2867 5485 1 3 7 7 67 867 867 2867 2867 134 135 HGAAAA ZCIAAA HHHHxx 9141 5486 1 1 1 1 41 141 1141 4141 9141 82 83 PNAAAA ADIAAA OOOOxx 3613 5487 1 1 3 13 13 613 1613 3613 3613 26 27 ZIAAAA BDIAAA VVVVxx 2461 5488 1 1 1 1 61 461 461 2461 2461 122 123 RQAAAA CDIAAA AAAAxx 4567 5489 1 3 7 7 67 567 567 4567 4567 134 135 RTAAAA DDIAAA HHHHxx 2906 5490 0 2 6 6 6 906 906 2906 2906 12 13 UHAAAA EDIAAA OOOOxx 4848 5491 0 0 8 8 48 848 848 4848 4848 96 97 MEAAAA FDIAAA VVVVxx 6614 5492 0 2 4 14 14 614 614 1614 6614 28 29 KUAAAA GDIAAA AAAAxx 6200 5493 0 0 0 0 0 200 200 1200 6200 0 1 MEAAAA HDIAAA HHHHxx 7895 5494 1 3 5 15 95 895 1895 2895 7895 190 191 RRAAAA IDIAAA OOOOxx 6829 5495 1 1 9 9 29 829 829 1829 6829 58 59 RCAAAA JDIAAA VVVVxx 4087 5496 1 3 7 7 87 87 87 4087 4087 174 175 FBAAAA KDIAAA AAAAxx 8787 5497 1 3 7 7 87 787 787 3787 8787 174 175 ZZAAAA LDIAAA HHHHxx 3322 5498 0 2 2 2 22 322 1322 3322 3322 44 45 UXAAAA MDIAAA OOOOxx 9091 5499 1 3 1 11 91 91 1091 4091 9091 182 183 RLAAAA NDIAAA VVVVxx 5268 5500 0 0 8 8 68 268 1268 268 5268 136 137 QUAAAA ODIAAA AAAAxx 2719 5501 1 3 9 19 19 719 719 2719 2719 38 39 PAAAAA PDIAAA HHHHxx 30 5502 0 2 0 10 30 30 30 30 30 60 61 EBAAAA QDIAAA OOOOxx 1975 5503 1 3 5 15 75 975 1975 1975 1975 150 151 ZXAAAA RDIAAA VVVVxx 2641 5504 1 1 1 1 41 641 641 2641 2641 82 83 PXAAAA SDIAAA AAAAxx 8616 5505 0 0 6 16 16 616 616 3616 8616 32 33 KTAAAA TDIAAA HHHHxx 5980 5506 0 0 0 0 80 980 1980 980 5980 160 161 AWAAAA UDIAAA OOOOxx 5170 5507 0 2 0 10 70 170 1170 170 5170 140 141 WQAAAA VDIAAA VVVVxx 1960 5508 0 0 0 0 60 960 1960 1960 1960 120 121 KXAAAA WDIAAA AAAAxx 8141 5509 1 1 1 1 41 141 141 3141 8141 82 83 DBAAAA XDIAAA HHHHxx 6692 5510 0 0 2 12 92 692 692 1692 6692 184 185 KXAAAA YDIAAA OOOOxx 7621 5511 1 1 1 1 21 621 1621 2621 7621 42 43 DHAAAA ZDIAAA VVVVxx 3890 5512 0 2 0 10 90 890 1890 3890 3890 180 181 QTAAAA AEIAAA AAAAxx 4300 5513 0 0 0 0 0 300 300 4300 4300 0 1 KJAAAA BEIAAA HHHHxx 736 5514 0 0 6 16 36 736 736 736 736 72 73 ICAAAA CEIAAA OOOOxx 6626 5515 0 2 6 6 26 626 626 1626 6626 52 53 WUAAAA DEIAAA VVVVxx 1800 5516 0 0 0 0 0 800 1800 1800 1800 0 1 GRAAAA EEIAAA AAAAxx 3430 5517 0 2 0 10 30 430 1430 3430 3430 60 61 YBAAAA FEIAAA HHHHxx 9519 5518 1 3 9 19 19 519 1519 4519 9519 38 39 DCAAAA GEIAAA OOOOxx 5111 5519 1 3 1 11 11 111 1111 111 5111 22 23 POAAAA HEIAAA VVVVxx 6915 5520 1 3 5 15 15 915 915 1915 6915 30 31 ZFAAAA IEIAAA AAAAxx 9246 5521 0 2 6 6 46 246 1246 4246 9246 92 93 QRAAAA JEIAAA HHHHxx 5141 5522 1 1 1 1 41 141 1141 141 5141 82 83 TPAAAA KEIAAA OOOOxx 5922 5523 0 2 2 2 22 922 1922 922 5922 44 45 UTAAAA LEIAAA VVVVxx 3087 5524 1 3 7 7 87 87 1087 3087 3087 174 175 TOAAAA MEIAAA AAAAxx 1859 5525 1 3 9 19 59 859 1859 1859 1859 118 119 NTAAAA NEIAAA HHHHxx 8482 5526 0 2 2 2 82 482 482 3482 8482 164 165 GOAAAA OEIAAA OOOOxx 8414 5527 0 2 4 14 14 414 414 3414 8414 28 29 QLAAAA PEIAAA VVVVxx 6662 5528 0 2 2 2 62 662 662 1662 6662 124 125 GWAAAA QEIAAA AAAAxx 8614 5529 0 2 4 14 14 614 614 3614 8614 28 29 ITAAAA REIAAA HHHHxx 42 5530 0 2 2 2 42 42 42 42 42 84 85 QBAAAA SEIAAA OOOOxx 7582 5531 0 2 2 2 82 582 1582 2582 7582 164 165 QFAAAA TEIAAA VVVVxx 8183 5532 1 3 3 3 83 183 183 3183 8183 166 167 TCAAAA UEIAAA AAAAxx 1299 5533 1 3 9 19 99 299 1299 1299 1299 198 199 ZXAAAA VEIAAA HHHHxx 7004 5534 0 0 4 4 4 4 1004 2004 7004 8 9 KJAAAA WEIAAA OOOOxx 3298 5535 0 2 8 18 98 298 1298 3298 3298 196 197 WWAAAA XEIAAA VVVVxx 7884 5536 0 0 4 4 84 884 1884 2884 7884 168 169 GRAAAA YEIAAA AAAAxx 4191 5537 1 3 1 11 91 191 191 4191 4191 182 183 FFAAAA ZEIAAA HHHHxx 7346 5538 0 2 6 6 46 346 1346 2346 7346 92 93 OWAAAA AFIAAA OOOOxx 7989 5539 1 1 9 9 89 989 1989 2989 7989 178 179 HVAAAA BFIAAA VVVVxx 5719 5540 1 3 9 19 19 719 1719 719 5719 38 39 ZLAAAA CFIAAA AAAAxx 800 5541 0 0 0 0 0 800 800 800 800 0 1 UEAAAA DFIAAA HHHHxx 6509 5542 1 1 9 9 9 509 509 1509 6509 18 19 JQAAAA EFIAAA OOOOxx 4672 5543 0 0 2 12 72 672 672 4672 4672 144 145 SXAAAA FFIAAA VVVVxx 4434 5544 0 2 4 14 34 434 434 4434 4434 68 69 OOAAAA GFIAAA AAAAxx 8309 5545 1 1 9 9 9 309 309 3309 8309 18 19 PHAAAA HFIAAA HHHHxx 5134 5546 0 2 4 14 34 134 1134 134 5134 68 69 MPAAAA IFIAAA OOOOxx 5153 5547 1 1 3 13 53 153 1153 153 5153 106 107 FQAAAA JFIAAA VVVVxx 1522 5548 0 2 2 2 22 522 1522 1522 1522 44 45 OGAAAA KFIAAA AAAAxx 8629 5549 1 1 9 9 29 629 629 3629 8629 58 59 XTAAAA LFIAAA HHHHxx 4549 5550 1 1 9 9 49 549 549 4549 4549 98 99 ZSAAAA MFIAAA OOOOxx 9506 5551 0 2 6 6 6 506 1506 4506 9506 12 13 QBAAAA NFIAAA VVVVxx 6542 5552 0 2 2 2 42 542 542 1542 6542 84 85 QRAAAA OFIAAA AAAAxx 2579 5553 1 3 9 19 79 579 579 2579 2579 158 159 FVAAAA PFIAAA HHHHxx 4664 5554 0 0 4 4 64 664 664 4664 4664 128 129 KXAAAA QFIAAA OOOOxx 696 5555 0 0 6 16 96 696 696 696 696 192 193 UAAAAA RFIAAA VVVVxx 7950 5556 0 2 0 10 50 950 1950 2950 7950 100 101 UTAAAA SFIAAA AAAAxx 5 5557 1 1 5 5 5 5 5 5 5 10 11 FAAAAA TFIAAA HHHHxx 7806 5558 0 2 6 6 6 806 1806 2806 7806 12 13 GOAAAA UFIAAA OOOOxx 2770 5559 0 2 0 10 70 770 770 2770 2770 140 141 OCAAAA VFIAAA VVVVxx 1344 5560 0 0 4 4 44 344 1344 1344 1344 88 89 SZAAAA WFIAAA AAAAxx 511 5561 1 3 1 11 11 511 511 511 511 22 23 RTAAAA XFIAAA HHHHxx 9070 5562 0 2 0 10 70 70 1070 4070 9070 140 141 WKAAAA YFIAAA OOOOxx 2961 5563 1 1 1 1 61 961 961 2961 2961 122 123 XJAAAA ZFIAAA VVVVxx 8031 5564 1 3 1 11 31 31 31 3031 8031 62 63 XWAAAA AGIAAA AAAAxx 326 5565 0 2 6 6 26 326 326 326 326 52 53 OMAAAA BGIAAA HHHHxx 183 5566 1 3 3 3 83 183 183 183 183 166 167 BHAAAA CGIAAA OOOOxx 5917 5567 1 1 7 17 17 917 1917 917 5917 34 35 PTAAAA DGIAAA VVVVxx 8256 5568 0 0 6 16 56 256 256 3256 8256 112 113 OFAAAA EGIAAA AAAAxx 7889 5569 1 1 9 9 89 889 1889 2889 7889 178 179 LRAAAA FGIAAA HHHHxx 9029 5570 1 1 9 9 29 29 1029 4029 9029 58 59 HJAAAA GGIAAA OOOOxx 1316 5571 0 0 6 16 16 316 1316 1316 1316 32 33 QYAAAA HGIAAA VVVVxx 7442 5572 0 2 2 2 42 442 1442 2442 7442 84 85 GAAAAA IGIAAA AAAAxx 2810 5573 0 2 0 10 10 810 810 2810 2810 20 21 CEAAAA JGIAAA HHHHxx 20 5574 0 0 0 0 20 20 20 20 20 40 41 UAAAAA KGIAAA OOOOxx 2306 5575 0 2 6 6 6 306 306 2306 2306 12 13 SKAAAA LGIAAA VVVVxx 4694 5576 0 2 4 14 94 694 694 4694 4694 188 189 OYAAAA MGIAAA AAAAxx 9710 5577 0 2 0 10 10 710 1710 4710 9710 20 21 MJAAAA NGIAAA HHHHxx 1791 5578 1 3 1 11 91 791 1791 1791 1791 182 183 XQAAAA OGIAAA OOOOxx 6730 5579 0 2 0 10 30 730 730 1730 6730 60 61 WYAAAA PGIAAA VVVVxx 359 5580 1 3 9 19 59 359 359 359 359 118 119 VNAAAA QGIAAA AAAAxx 8097 5581 1 1 7 17 97 97 97 3097 8097 194 195 LZAAAA RGIAAA HHHHxx 6147 5582 1 3 7 7 47 147 147 1147 6147 94 95 LCAAAA SGIAAA OOOOxx 643 5583 1 3 3 3 43 643 643 643 643 86 87 TYAAAA TGIAAA VVVVxx 698 5584 0 2 8 18 98 698 698 698 698 196 197 WAAAAA UGIAAA AAAAxx 3881 5585 1 1 1 1 81 881 1881 3881 3881 162 163 HTAAAA VGIAAA HHHHxx 7600 5586 0 0 0 0 0 600 1600 2600 7600 0 1 IGAAAA WGIAAA OOOOxx 1583 5587 1 3 3 3 83 583 1583 1583 1583 166 167 XIAAAA XGIAAA VVVVxx 9612 5588 0 0 2 12 12 612 1612 4612 9612 24 25 SFAAAA YGIAAA AAAAxx 1032 5589 0 0 2 12 32 32 1032 1032 1032 64 65 SNAAAA ZGIAAA HHHHxx 4834 5590 0 2 4 14 34 834 834 4834 4834 68 69 YDAAAA AHIAAA OOOOxx 5076 5591 0 0 6 16 76 76 1076 76 5076 152 153 GNAAAA BHIAAA VVVVxx 3070 5592 0 2 0 10 70 70 1070 3070 3070 140 141 COAAAA CHIAAA AAAAxx 1421 5593 1 1 1 1 21 421 1421 1421 1421 42 43 RCAAAA DHIAAA HHHHxx 8970 5594 0 2 0 10 70 970 970 3970 8970 140 141 AHAAAA EHIAAA OOOOxx 6271 5595 1 3 1 11 71 271 271 1271 6271 142 143 FHAAAA FHIAAA VVVVxx 8547 5596 1 3 7 7 47 547 547 3547 8547 94 95 TQAAAA GHIAAA AAAAxx 1259 5597 1 3 9 19 59 259 1259 1259 1259 118 119 LWAAAA HHIAAA HHHHxx 8328 5598 0 0 8 8 28 328 328 3328 8328 56 57 IIAAAA IHIAAA OOOOxx 1503 5599 1 3 3 3 3 503 1503 1503 1503 6 7 VFAAAA JHIAAA VVVVxx 2253 5600 1 1 3 13 53 253 253 2253 2253 106 107 RIAAAA KHIAAA AAAAxx 7449 5601 1 1 9 9 49 449 1449 2449 7449 98 99 NAAAAA LHIAAA HHHHxx 3579 5602 1 3 9 19 79 579 1579 3579 3579 158 159 RHAAAA MHIAAA OOOOxx 1585 5603 1 1 5 5 85 585 1585 1585 1585 170 171 ZIAAAA NHIAAA VVVVxx 5543 5604 1 3 3 3 43 543 1543 543 5543 86 87 FFAAAA OHIAAA AAAAxx 8627 5605 1 3 7 7 27 627 627 3627 8627 54 55 VTAAAA PHIAAA HHHHxx 8618 5606 0 2 8 18 18 618 618 3618 8618 36 37 MTAAAA QHIAAA OOOOxx 1911 5607 1 3 1 11 11 911 1911 1911 1911 22 23 NVAAAA RHIAAA VVVVxx 2758 5608 0 2 8 18 58 758 758 2758 2758 116 117 CCAAAA SHIAAA AAAAxx 5744 5609 0 0 4 4 44 744 1744 744 5744 88 89 YMAAAA THIAAA HHHHxx 4976 5610 0 0 6 16 76 976 976 4976 4976 152 153 KJAAAA UHIAAA OOOOxx 6380 5611 0 0 0 0 80 380 380 1380 6380 160 161 KLAAAA VHIAAA VVVVxx 1937 5612 1 1 7 17 37 937 1937 1937 1937 74 75 NWAAAA WHIAAA AAAAxx 9903 5613 1 3 3 3 3 903 1903 4903 9903 6 7 XQAAAA XHIAAA HHHHxx 4409 5614 1 1 9 9 9 409 409 4409 4409 18 19 PNAAAA YHIAAA OOOOxx 4133 5615 1 1 3 13 33 133 133 4133 4133 66 67 ZCAAAA ZHIAAA VVVVxx 5263 5616 1 3 3 3 63 263 1263 263 5263 126 127 LUAAAA AIIAAA AAAAxx 7888 5617 0 0 8 8 88 888 1888 2888 7888 176 177 KRAAAA BIIAAA HHHHxx 6060 5618 0 0 0 0 60 60 60 1060 6060 120 121 CZAAAA CIIAAA OOOOxx 2522 5619 0 2 2 2 22 522 522 2522 2522 44 45 ATAAAA DIIAAA VVVVxx 5550 5620 0 2 0 10 50 550 1550 550 5550 100 101 MFAAAA EIIAAA AAAAxx 9396 5621 0 0 6 16 96 396 1396 4396 9396 192 193 KXAAAA FIIAAA HHHHxx 176 5622 0 0 6 16 76 176 176 176 176 152 153 UGAAAA GIIAAA OOOOxx 5148 5623 0 0 8 8 48 148 1148 148 5148 96 97 AQAAAA HIIAAA VVVVxx 6691 5624 1 3 1 11 91 691 691 1691 6691 182 183 JXAAAA IIIAAA AAAAxx 4652 5625 0 0 2 12 52 652 652 4652 4652 104 105 YWAAAA JIIAAA HHHHxx 5096 5626 0 0 6 16 96 96 1096 96 5096 192 193 AOAAAA KIIAAA OOOOxx 2408 5627 0 0 8 8 8 408 408 2408 2408 16 17 QOAAAA LIIAAA VVVVxx 7322 5628 0 2 2 2 22 322 1322 2322 7322 44 45 QVAAAA MIIAAA AAAAxx 6782 5629 0 2 2 2 82 782 782 1782 6782 164 165 WAAAAA NIIAAA HHHHxx 4642 5630 0 2 2 2 42 642 642 4642 4642 84 85 OWAAAA OIIAAA OOOOxx 5427 5631 1 3 7 7 27 427 1427 427 5427 54 55 TAAAAA PIIAAA VVVVxx 4461 5632 1 1 1 1 61 461 461 4461 4461 122 123 PPAAAA QIIAAA AAAAxx 8416 5633 0 0 6 16 16 416 416 3416 8416 32 33 SLAAAA RIIAAA HHHHxx 2593 5634 1 1 3 13 93 593 593 2593 2593 186 187 TVAAAA SIIAAA OOOOxx 6202 5635 0 2 2 2 2 202 202 1202 6202 4 5 OEAAAA TIIAAA VVVVxx 3826 5636 0 2 6 6 26 826 1826 3826 3826 52 53 ERAAAA UIIAAA AAAAxx 4417 5637 1 1 7 17 17 417 417 4417 4417 34 35 XNAAAA VIIAAA HHHHxx 7871 5638 1 3 1 11 71 871 1871 2871 7871 142 143 TQAAAA WIIAAA OOOOxx 5622 5639 0 2 2 2 22 622 1622 622 5622 44 45 GIAAAA XIIAAA VVVVxx 3010 5640 0 2 0 10 10 10 1010 3010 3010 20 21 ULAAAA YIIAAA AAAAxx 3407 5641 1 3 7 7 7 407 1407 3407 3407 14 15 BBAAAA ZIIAAA HHHHxx 1274 5642 0 2 4 14 74 274 1274 1274 1274 148 149 AXAAAA AJIAAA OOOOxx 2828 5643 0 0 8 8 28 828 828 2828 2828 56 57 UEAAAA BJIAAA VVVVxx 3427 5644 1 3 7 7 27 427 1427 3427 3427 54 55 VBAAAA CJIAAA AAAAxx 612 5645 0 0 2 12 12 612 612 612 612 24 25 OXAAAA DJIAAA HHHHxx 8729 5646 1 1 9 9 29 729 729 3729 8729 58 59 TXAAAA EJIAAA OOOOxx 1239 5647 1 3 9 19 39 239 1239 1239 1239 78 79 RVAAAA FJIAAA VVVVxx 8990 5648 0 2 0 10 90 990 990 3990 8990 180 181 UHAAAA GJIAAA AAAAxx 5609 5649 1 1 9 9 9 609 1609 609 5609 18 19 THAAAA HJIAAA HHHHxx 4441 5650 1 1 1 1 41 441 441 4441 4441 82 83 VOAAAA IJIAAA OOOOxx 9078 5651 0 2 8 18 78 78 1078 4078 9078 156 157 ELAAAA JJIAAA VVVVxx 6699 5652 1 3 9 19 99 699 699 1699 6699 198 199 RXAAAA KJIAAA AAAAxx 8390 5653 0 2 0 10 90 390 390 3390 8390 180 181 SKAAAA LJIAAA HHHHxx 5455 5654 1 3 5 15 55 455 1455 455 5455 110 111 VBAAAA MJIAAA OOOOxx 7537 5655 1 1 7 17 37 537 1537 2537 7537 74 75 XDAAAA NJIAAA VVVVxx 4669 5656 1 1 9 9 69 669 669 4669 4669 138 139 PXAAAA OJIAAA AAAAxx 5534 5657 0 2 4 14 34 534 1534 534 5534 68 69 WEAAAA PJIAAA HHHHxx 1920 5658 0 0 0 0 20 920 1920 1920 1920 40 41 WVAAAA QJIAAA OOOOxx 9465 5659 1 1 5 5 65 465 1465 4465 9465 130 131 BAAAAA RJIAAA VVVVxx 4897 5660 1 1 7 17 97 897 897 4897 4897 194 195 JGAAAA SJIAAA AAAAxx 1990 5661 0 2 0 10 90 990 1990 1990 1990 180 181 OYAAAA TJIAAA HHHHxx 7148 5662 0 0 8 8 48 148 1148 2148 7148 96 97 YOAAAA UJIAAA OOOOxx 533 5663 1 1 3 13 33 533 533 533 533 66 67 NUAAAA VJIAAA VVVVxx 4339 5664 1 3 9 19 39 339 339 4339 4339 78 79 XKAAAA WJIAAA AAAAxx 6450 5665 0 2 0 10 50 450 450 1450 6450 100 101 COAAAA XJIAAA HHHHxx 9627 5666 1 3 7 7 27 627 1627 4627 9627 54 55 HGAAAA YJIAAA OOOOxx 5539 5667 1 3 9 19 39 539 1539 539 5539 78 79 BFAAAA ZJIAAA VVVVxx 6758 5668 0 2 8 18 58 758 758 1758 6758 116 117 YZAAAA AKIAAA AAAAxx 3435 5669 1 3 5 15 35 435 1435 3435 3435 70 71 DCAAAA BKIAAA HHHHxx 4350 5670 0 2 0 10 50 350 350 4350 4350 100 101 ILAAAA CKIAAA OOOOxx 9088 5671 0 0 8 8 88 88 1088 4088 9088 176 177 OLAAAA DKIAAA VVVVxx 6368 5672 0 0 8 8 68 368 368 1368 6368 136 137 YKAAAA EKIAAA AAAAxx 6337 5673 1 1 7 17 37 337 337 1337 6337 74 75 TJAAAA FKIAAA HHHHxx 4361 5674 1 1 1 1 61 361 361 4361 4361 122 123 TLAAAA GKIAAA OOOOxx 1719 5675 1 3 9 19 19 719 1719 1719 1719 38 39 DOAAAA HKIAAA VVVVxx 3109 5676 1 1 9 9 9 109 1109 3109 3109 18 19 PPAAAA IKIAAA AAAAxx 7135 5677 1 3 5 15 35 135 1135 2135 7135 70 71 LOAAAA JKIAAA HHHHxx 1964 5678 0 0 4 4 64 964 1964 1964 1964 128 129 OXAAAA KKIAAA OOOOxx 3 5679 1 3 3 3 3 3 3 3 3 6 7 DAAAAA LKIAAA VVVVxx 1868 5680 0 0 8 8 68 868 1868 1868 1868 136 137 WTAAAA MKIAAA AAAAxx 5182 5681 0 2 2 2 82 182 1182 182 5182 164 165 IRAAAA NKIAAA HHHHxx 7567 5682 1 3 7 7 67 567 1567 2567 7567 134 135 BFAAAA OKIAAA OOOOxx 3676 5683 0 0 6 16 76 676 1676 3676 3676 152 153 KLAAAA PKIAAA VVVVxx 9382 5684 0 2 2 2 82 382 1382 4382 9382 164 165 WWAAAA QKIAAA AAAAxx 8645 5685 1 1 5 5 45 645 645 3645 8645 90 91 NUAAAA RKIAAA HHHHxx 2018 5686 0 2 8 18 18 18 18 2018 2018 36 37 QZAAAA SKIAAA OOOOxx 217 5687 1 1 7 17 17 217 217 217 217 34 35 JIAAAA TKIAAA VVVVxx 6793 5688 1 1 3 13 93 793 793 1793 6793 186 187 HBAAAA UKIAAA AAAAxx 7280 5689 0 0 0 0 80 280 1280 2280 7280 160 161 AUAAAA VKIAAA HHHHxx 2168 5690 0 0 8 8 68 168 168 2168 2168 136 137 KFAAAA WKIAAA OOOOxx 5259 5691 1 3 9 19 59 259 1259 259 5259 118 119 HUAAAA XKIAAA VVVVxx 6019 5692 1 3 9 19 19 19 19 1019 6019 38 39 NXAAAA YKIAAA AAAAxx 877 5693 1 1 7 17 77 877 877 877 877 154 155 THAAAA ZKIAAA HHHHxx 4961 5694 1 1 1 1 61 961 961 4961 4961 122 123 VIAAAA ALIAAA OOOOxx 1873 5695 1 1 3 13 73 873 1873 1873 1873 146 147 BUAAAA BLIAAA VVVVxx 13 5696 1 1 3 13 13 13 13 13 13 26 27 NAAAAA CLIAAA AAAAxx 1537 5697 1 1 7 17 37 537 1537 1537 1537 74 75 DHAAAA DLIAAA HHHHxx 3129 5698 1 1 9 9 29 129 1129 3129 3129 58 59 JQAAAA ELIAAA OOOOxx 6473 5699 1 1 3 13 73 473 473 1473 6473 146 147 ZOAAAA FLIAAA VVVVxx 7865 5700 1 1 5 5 65 865 1865 2865 7865 130 131 NQAAAA GLIAAA AAAAxx 7822 5701 0 2 2 2 22 822 1822 2822 7822 44 45 WOAAAA HLIAAA HHHHxx 239 5702 1 3 9 19 39 239 239 239 239 78 79 FJAAAA ILIAAA OOOOxx 2062 5703 0 2 2 2 62 62 62 2062 2062 124 125 IBAAAA JLIAAA VVVVxx 762 5704 0 2 2 2 62 762 762 762 762 124 125 IDAAAA KLIAAA AAAAxx 3764 5705 0 0 4 4 64 764 1764 3764 3764 128 129 UOAAAA LLIAAA HHHHxx 465 5706 1 1 5 5 65 465 465 465 465 130 131 XRAAAA MLIAAA OOOOxx 2587 5707 1 3 7 7 87 587 587 2587 2587 174 175 NVAAAA NLIAAA VVVVxx 8402 5708 0 2 2 2 2 402 402 3402 8402 4 5 ELAAAA OLIAAA AAAAxx 1055 5709 1 3 5 15 55 55 1055 1055 1055 110 111 POAAAA PLIAAA HHHHxx 3072 5710 0 0 2 12 72 72 1072 3072 3072 144 145 EOAAAA QLIAAA OOOOxx 7359 5711 1 3 9 19 59 359 1359 2359 7359 118 119 BXAAAA RLIAAA VVVVxx 6558 5712 0 2 8 18 58 558 558 1558 6558 116 117 GSAAAA SLIAAA AAAAxx 48 5713 0 0 8 8 48 48 48 48 48 96 97 WBAAAA TLIAAA HHHHxx 5382 5714 0 2 2 2 82 382 1382 382 5382 164 165 AZAAAA ULIAAA OOOOxx 947 5715 1 3 7 7 47 947 947 947 947 94 95 LKAAAA VLIAAA VVVVxx 2644 5716 0 0 4 4 44 644 644 2644 2644 88 89 SXAAAA WLIAAA AAAAxx 7516 5717 0 0 6 16 16 516 1516 2516 7516 32 33 CDAAAA XLIAAA HHHHxx 2362 5718 0 2 2 2 62 362 362 2362 2362 124 125 WMAAAA YLIAAA OOOOxx 839 5719 1 3 9 19 39 839 839 839 839 78 79 HGAAAA ZLIAAA VVVVxx 2216 5720 0 0 6 16 16 216 216 2216 2216 32 33 GHAAAA AMIAAA AAAAxx 7673 5721 1 1 3 13 73 673 1673 2673 7673 146 147 DJAAAA BMIAAA HHHHxx 8173 5722 1 1 3 13 73 173 173 3173 8173 146 147 JCAAAA CMIAAA OOOOxx 1630 5723 0 2 0 10 30 630 1630 1630 1630 60 61 SKAAAA DMIAAA VVVVxx 9057 5724 1 1 7 17 57 57 1057 4057 9057 114 115 JKAAAA EMIAAA AAAAxx 4392 5725 0 0 2 12 92 392 392 4392 4392 184 185 YMAAAA FMIAAA HHHHxx 3695 5726 1 3 5 15 95 695 1695 3695 3695 190 191 DMAAAA GMIAAA OOOOxx 5751 5727 1 3 1 11 51 751 1751 751 5751 102 103 FNAAAA HMIAAA VVVVxx 5745 5728 1 1 5 5 45 745 1745 745 5745 90 91 ZMAAAA IMIAAA AAAAxx 7945 5729 1 1 5 5 45 945 1945 2945 7945 90 91 PTAAAA JMIAAA HHHHxx 5174 5730 0 2 4 14 74 174 1174 174 5174 148 149 ARAAAA KMIAAA OOOOxx 3829 5731 1 1 9 9 29 829 1829 3829 3829 58 59 HRAAAA LMIAAA VVVVxx 3317 5732 1 1 7 17 17 317 1317 3317 3317 34 35 PXAAAA MMIAAA AAAAxx 4253 5733 1 1 3 13 53 253 253 4253 4253 106 107 PHAAAA NMIAAA HHHHxx 1291 5734 1 3 1 11 91 291 1291 1291 1291 182 183 RXAAAA OMIAAA OOOOxx 3266 5735 0 2 6 6 66 266 1266 3266 3266 132 133 QVAAAA PMIAAA VVVVxx 2939 5736 1 3 9 19 39 939 939 2939 2939 78 79 BJAAAA QMIAAA AAAAxx 2755 5737 1 3 5 15 55 755 755 2755 2755 110 111 ZBAAAA RMIAAA HHHHxx 6844 5738 0 0 4 4 44 844 844 1844 6844 88 89 GDAAAA SMIAAA OOOOxx 8594 5739 0 2 4 14 94 594 594 3594 8594 188 189 OSAAAA TMIAAA VVVVxx 704 5740 0 0 4 4 4 704 704 704 704 8 9 CBAAAA UMIAAA AAAAxx 1681 5741 1 1 1 1 81 681 1681 1681 1681 162 163 RMAAAA VMIAAA HHHHxx 364 5742 0 0 4 4 64 364 364 364 364 128 129 AOAAAA WMIAAA OOOOxx 2928 5743 0 0 8 8 28 928 928 2928 2928 56 57 QIAAAA XMIAAA VVVVxx 117 5744 1 1 7 17 17 117 117 117 117 34 35 NEAAAA YMIAAA AAAAxx 96 5745 0 0 6 16 96 96 96 96 96 192 193 SDAAAA ZMIAAA HHHHxx 7796 5746 0 0 6 16 96 796 1796 2796 7796 192 193 WNAAAA ANIAAA OOOOxx 3101 5747 1 1 1 1 1 101 1101 3101 3101 2 3 HPAAAA BNIAAA VVVVxx 3397 5748 1 1 7 17 97 397 1397 3397 3397 194 195 RAAAAA CNIAAA AAAAxx 1605 5749 1 1 5 5 5 605 1605 1605 1605 10 11 TJAAAA DNIAAA HHHHxx 4881 5750 1 1 1 1 81 881 881 4881 4881 162 163 TFAAAA ENIAAA OOOOxx 4521 5751 1 1 1 1 21 521 521 4521 4521 42 43 XRAAAA FNIAAA VVVVxx 6430 5752 0 2 0 10 30 430 430 1430 6430 60 61 INAAAA GNIAAA AAAAxx 282 5753 0 2 2 2 82 282 282 282 282 164 165 WKAAAA HNIAAA HHHHxx 9645 5754 1 1 5 5 45 645 1645 4645 9645 90 91 ZGAAAA INIAAA OOOOxx 8946 5755 0 2 6 6 46 946 946 3946 8946 92 93 CGAAAA JNIAAA VVVVxx 5064 5756 0 0 4 4 64 64 1064 64 5064 128 129 UMAAAA KNIAAA AAAAxx 7470 5757 0 2 0 10 70 470 1470 2470 7470 140 141 IBAAAA LNIAAA HHHHxx 5886 5758 0 2 6 6 86 886 1886 886 5886 172 173 KSAAAA MNIAAA OOOOxx 6280 5759 0 0 0 0 80 280 280 1280 6280 160 161 OHAAAA NNIAAA VVVVxx 5247 5760 1 3 7 7 47 247 1247 247 5247 94 95 VTAAAA ONIAAA AAAAxx 412 5761 0 0 2 12 12 412 412 412 412 24 25 WPAAAA PNIAAA HHHHxx 5342 5762 0 2 2 2 42 342 1342 342 5342 84 85 MXAAAA QNIAAA OOOOxx 2271 5763 1 3 1 11 71 271 271 2271 2271 142 143 JJAAAA RNIAAA VVVVxx 849 5764 1 1 9 9 49 849 849 849 849 98 99 RGAAAA SNIAAA AAAAxx 1885 5765 1 1 5 5 85 885 1885 1885 1885 170 171 NUAAAA TNIAAA HHHHxx 5620 5766 0 0 0 0 20 620 1620 620 5620 40 41 EIAAAA UNIAAA OOOOxx 7079 5767 1 3 9 19 79 79 1079 2079 7079 158 159 HMAAAA VNIAAA VVVVxx 5819 5768 1 3 9 19 19 819 1819 819 5819 38 39 VPAAAA WNIAAA AAAAxx 7497 5769 1 1 7 17 97 497 1497 2497 7497 194 195 JCAAAA XNIAAA HHHHxx 5993 5770 1 1 3 13 93 993 1993 993 5993 186 187 NWAAAA YNIAAA OOOOxx 3739 5771 1 3 9 19 39 739 1739 3739 3739 78 79 VNAAAA ZNIAAA VVVVxx 6296 5772 0 0 6 16 96 296 296 1296 6296 192 193 EIAAAA AOIAAA AAAAxx 2716 5773 0 0 6 16 16 716 716 2716 2716 32 33 MAAAAA BOIAAA HHHHxx 1130 5774 0 2 0 10 30 130 1130 1130 1130 60 61 MRAAAA COIAAA OOOOxx 5593 5775 1 1 3 13 93 593 1593 593 5593 186 187 DHAAAA DOIAAA VVVVxx 6972 5776 0 0 2 12 72 972 972 1972 6972 144 145 EIAAAA EOIAAA AAAAxx 8360 5777 0 0 0 0 60 360 360 3360 8360 120 121 OJAAAA FOIAAA HHHHxx 6448 5778 0 0 8 8 48 448 448 1448 6448 96 97 AOAAAA GOIAAA OOOOxx 3689 5779 1 1 9 9 89 689 1689 3689 3689 178 179 XLAAAA HOIAAA VVVVxx 7951 5780 1 3 1 11 51 951 1951 2951 7951 102 103 VTAAAA IOIAAA AAAAxx 2974 5781 0 2 4 14 74 974 974 2974 2974 148 149 KKAAAA JOIAAA HHHHxx 6600 5782 0 0 0 0 0 600 600 1600 6600 0 1 WTAAAA KOIAAA OOOOxx 4662 5783 0 2 2 2 62 662 662 4662 4662 124 125 IXAAAA LOIAAA VVVVxx 4765 5784 1 1 5 5 65 765 765 4765 4765 130 131 HBAAAA MOIAAA AAAAxx 355 5785 1 3 5 15 55 355 355 355 355 110 111 RNAAAA NOIAAA HHHHxx 6228 5786 0 0 8 8 28 228 228 1228 6228 56 57 OFAAAA OOIAAA OOOOxx 964 5787 0 0 4 4 64 964 964 964 964 128 129 CLAAAA POIAAA VVVVxx 3082 5788 0 2 2 2 82 82 1082 3082 3082 164 165 OOAAAA QOIAAA AAAAxx 7028 5789 0 0 8 8 28 28 1028 2028 7028 56 57 IKAAAA ROIAAA HHHHxx 4505 5790 1 1 5 5 5 505 505 4505 4505 10 11 HRAAAA SOIAAA OOOOxx 8961 5791 1 1 1 1 61 961 961 3961 8961 122 123 RGAAAA TOIAAA VVVVxx 9571 5792 1 3 1 11 71 571 1571 4571 9571 142 143 DEAAAA UOIAAA AAAAxx 9394 5793 0 2 4 14 94 394 1394 4394 9394 188 189 IXAAAA VOIAAA HHHHxx 4245 5794 1 1 5 5 45 245 245 4245 4245 90 91 HHAAAA WOIAAA OOOOxx 7560 5795 0 0 0 0 60 560 1560 2560 7560 120 121 UEAAAA XOIAAA VVVVxx 2907 5796 1 3 7 7 7 907 907 2907 2907 14 15 VHAAAA YOIAAA AAAAxx 7817 5797 1 1 7 17 17 817 1817 2817 7817 34 35 ROAAAA ZOIAAA HHHHxx 5408 5798 0 0 8 8 8 408 1408 408 5408 16 17 AAAAAA APIAAA OOOOxx 8092 5799 0 0 2 12 92 92 92 3092 8092 184 185 GZAAAA BPIAAA VVVVxx 1309 5800 1 1 9 9 9 309 1309 1309 1309 18 19 JYAAAA CPIAAA AAAAxx 6673 5801 1 1 3 13 73 673 673 1673 6673 146 147 RWAAAA DPIAAA HHHHxx 1245 5802 1 1 5 5 45 245 1245 1245 1245 90 91 XVAAAA EPIAAA OOOOxx 6790 5803 0 2 0 10 90 790 790 1790 6790 180 181 EBAAAA FPIAAA VVVVxx 8380 5804 0 0 0 0 80 380 380 3380 8380 160 161 IKAAAA GPIAAA AAAAxx 5786 5805 0 2 6 6 86 786 1786 786 5786 172 173 OOAAAA HPIAAA HHHHxx 9590 5806 0 2 0 10 90 590 1590 4590 9590 180 181 WEAAAA IPIAAA OOOOxx 5763 5807 1 3 3 3 63 763 1763 763 5763 126 127 RNAAAA JPIAAA VVVVxx 1345 5808 1 1 5 5 45 345 1345 1345 1345 90 91 TZAAAA KPIAAA AAAAxx 3480 5809 0 0 0 0 80 480 1480 3480 3480 160 161 WDAAAA LPIAAA HHHHxx 7864 5810 0 0 4 4 64 864 1864 2864 7864 128 129 MQAAAA MPIAAA OOOOxx 4853 5811 1 1 3 13 53 853 853 4853 4853 106 107 REAAAA NPIAAA VVVVxx 1445 5812 1 1 5 5 45 445 1445 1445 1445 90 91 PDAAAA OPIAAA AAAAxx 170 5813 0 2 0 10 70 170 170 170 170 140 141 OGAAAA PPIAAA HHHHxx 7348 5814 0 0 8 8 48 348 1348 2348 7348 96 97 QWAAAA QPIAAA OOOOxx 3920 5815 0 0 0 0 20 920 1920 3920 3920 40 41 UUAAAA RPIAAA VVVVxx 3307 5816 1 3 7 7 7 307 1307 3307 3307 14 15 FXAAAA SPIAAA AAAAxx 4584 5817 0 0 4 4 84 584 584 4584 4584 168 169 IUAAAA TPIAAA HHHHxx 3344 5818 0 0 4 4 44 344 1344 3344 3344 88 89 QYAAAA UPIAAA OOOOxx 4360 5819 0 0 0 0 60 360 360 4360 4360 120 121 SLAAAA VPIAAA VVVVxx 8757 5820 1 1 7 17 57 757 757 3757 8757 114 115 VYAAAA WPIAAA AAAAxx 4315 5821 1 3 5 15 15 315 315 4315 4315 30 31 ZJAAAA XPIAAA HHHHxx 5243 5822 1 3 3 3 43 243 1243 243 5243 86 87 RTAAAA YPIAAA OOOOxx 8550 5823 0 2 0 10 50 550 550 3550 8550 100 101 WQAAAA ZPIAAA VVVVxx 159 5824 1 3 9 19 59 159 159 159 159 118 119 DGAAAA AQIAAA AAAAxx 4710 5825 0 2 0 10 10 710 710 4710 4710 20 21 EZAAAA BQIAAA HHHHxx 7179 5826 1 3 9 19 79 179 1179 2179 7179 158 159 DQAAAA CQIAAA OOOOxx 2509 5827 1 1 9 9 9 509 509 2509 2509 18 19 NSAAAA DQIAAA VVVVxx 6981 5828 1 1 1 1 81 981 981 1981 6981 162 163 NIAAAA EQIAAA AAAAxx 5060 5829 0 0 0 0 60 60 1060 60 5060 120 121 QMAAAA FQIAAA HHHHxx 5601 5830 1 1 1 1 1 601 1601 601 5601 2 3 LHAAAA GQIAAA OOOOxx 703 5831 1 3 3 3 3 703 703 703 703 6 7 BBAAAA HQIAAA VVVVxx 8719 5832 1 3 9 19 19 719 719 3719 8719 38 39 JXAAAA IQIAAA AAAAxx 1570 5833 0 2 0 10 70 570 1570 1570 1570 140 141 KIAAAA JQIAAA HHHHxx 1036 5834 0 0 6 16 36 36 1036 1036 1036 72 73 WNAAAA KQIAAA OOOOxx 6703 5835 1 3 3 3 3 703 703 1703 6703 6 7 VXAAAA LQIAAA VVVVxx 252 5836 0 0 2 12 52 252 252 252 252 104 105 SJAAAA MQIAAA AAAAxx 631 5837 1 3 1 11 31 631 631 631 631 62 63 HYAAAA NQIAAA HHHHxx 5098 5838 0 2 8 18 98 98 1098 98 5098 196 197 COAAAA OQIAAA OOOOxx 8346 5839 0 2 6 6 46 346 346 3346 8346 92 93 AJAAAA PQIAAA VVVVxx 4910 5840 0 2 0 10 10 910 910 4910 4910 20 21 WGAAAA QQIAAA AAAAxx 559 5841 1 3 9 19 59 559 559 559 559 118 119 NVAAAA RQIAAA HHHHxx 1477 5842 1 1 7 17 77 477 1477 1477 1477 154 155 VEAAAA SQIAAA OOOOxx 5115 5843 1 3 5 15 15 115 1115 115 5115 30 31 TOAAAA TQIAAA VVVVxx 8784 5844 0 0 4 4 84 784 784 3784 8784 168 169 WZAAAA UQIAAA AAAAxx 4422 5845 0 2 2 2 22 422 422 4422 4422 44 45 COAAAA VQIAAA HHHHxx 2702 5846 0 2 2 2 2 702 702 2702 2702 4 5 YZAAAA WQIAAA OOOOxx 9599 5847 1 3 9 19 99 599 1599 4599 9599 198 199 FFAAAA XQIAAA VVVVxx 2463 5848 1 3 3 3 63 463 463 2463 2463 126 127 TQAAAA YQIAAA AAAAxx 498 5849 0 2 8 18 98 498 498 498 498 196 197 ETAAAA ZQIAAA HHHHxx 494 5850 0 2 4 14 94 494 494 494 494 188 189 ATAAAA ARIAAA OOOOxx 8632 5851 0 0 2 12 32 632 632 3632 8632 64 65 AUAAAA BRIAAA VVVVxx 3449 5852 1 1 9 9 49 449 1449 3449 3449 98 99 RCAAAA CRIAAA AAAAxx 5888 5853 0 0 8 8 88 888 1888 888 5888 176 177 MSAAAA DRIAAA HHHHxx 2211 5854 1 3 1 11 11 211 211 2211 2211 22 23 BHAAAA ERIAAA OOOOxx 2835 5855 1 3 5 15 35 835 835 2835 2835 70 71 BFAAAA FRIAAA VVVVxx 4196 5856 0 0 6 16 96 196 196 4196 4196 192 193 KFAAAA GRIAAA AAAAxx 2177 5857 1 1 7 17 77 177 177 2177 2177 154 155 TFAAAA HRIAAA HHHHxx 1959 5858 1 3 9 19 59 959 1959 1959 1959 118 119 JXAAAA IRIAAA OOOOxx 5172 5859 0 0 2 12 72 172 1172 172 5172 144 145 YQAAAA JRIAAA VVVVxx 7898 5860 0 2 8 18 98 898 1898 2898 7898 196 197 URAAAA KRIAAA AAAAxx 5729 5861 1 1 9 9 29 729 1729 729 5729 58 59 JMAAAA LRIAAA HHHHxx 469 5862 1 1 9 9 69 469 469 469 469 138 139 BSAAAA MRIAAA OOOOxx 4456 5863 0 0 6 16 56 456 456 4456 4456 112 113 KPAAAA NRIAAA VVVVxx 3578 5864 0 2 8 18 78 578 1578 3578 3578 156 157 QHAAAA ORIAAA AAAAxx 8623 5865 1 3 3 3 23 623 623 3623 8623 46 47 RTAAAA PRIAAA HHHHxx 6749 5866 1 1 9 9 49 749 749 1749 6749 98 99 PZAAAA QRIAAA OOOOxx 6735 5867 1 3 5 15 35 735 735 1735 6735 70 71 BZAAAA RRIAAA VVVVxx 5197 5868 1 1 7 17 97 197 1197 197 5197 194 195 XRAAAA SRIAAA AAAAxx 2067 5869 1 3 7 7 67 67 67 2067 2067 134 135 NBAAAA TRIAAA HHHHxx 5600 5870 0 0 0 0 0 600 1600 600 5600 0 1 KHAAAA URIAAA OOOOxx 7741 5871 1 1 1 1 41 741 1741 2741 7741 82 83 TLAAAA VRIAAA VVVVxx 9925 5872 1 1 5 5 25 925 1925 4925 9925 50 51 TRAAAA WRIAAA AAAAxx 9685 5873 1 1 5 5 85 685 1685 4685 9685 170 171 NIAAAA XRIAAA HHHHxx 7622 5874 0 2 2 2 22 622 1622 2622 7622 44 45 EHAAAA YRIAAA OOOOxx 6859 5875 1 3 9 19 59 859 859 1859 6859 118 119 VDAAAA ZRIAAA VVVVxx 3094 5876 0 2 4 14 94 94 1094 3094 3094 188 189 APAAAA ASIAAA AAAAxx 2628 5877 0 0 8 8 28 628 628 2628 2628 56 57 CXAAAA BSIAAA HHHHxx 40 5878 0 0 0 0 40 40 40 40 40 80 81 OBAAAA CSIAAA OOOOxx 1644 5879 0 0 4 4 44 644 1644 1644 1644 88 89 GLAAAA DSIAAA VVVVxx 588 5880 0 0 8 8 88 588 588 588 588 176 177 QWAAAA ESIAAA AAAAxx 7522 5881 0 2 2 2 22 522 1522 2522 7522 44 45 IDAAAA FSIAAA HHHHxx 162 5882 0 2 2 2 62 162 162 162 162 124 125 GGAAAA GSIAAA OOOOxx 3610 5883 0 2 0 10 10 610 1610 3610 3610 20 21 WIAAAA HSIAAA VVVVxx 3561 5884 1 1 1 1 61 561 1561 3561 3561 122 123 ZGAAAA ISIAAA AAAAxx 8185 5885 1 1 5 5 85 185 185 3185 8185 170 171 VCAAAA JSIAAA HHHHxx 7237 5886 1 1 7 17 37 237 1237 2237 7237 74 75 JSAAAA KSIAAA OOOOxx 4592 5887 0 0 2 12 92 592 592 4592 4592 184 185 QUAAAA LSIAAA VVVVxx 7082 5888 0 2 2 2 82 82 1082 2082 7082 164 165 KMAAAA MSIAAA AAAAxx 4719 5889 1 3 9 19 19 719 719 4719 4719 38 39 NZAAAA NSIAAA HHHHxx 3879 5890 1 3 9 19 79 879 1879 3879 3879 158 159 FTAAAA OSIAAA OOOOxx 1662 5891 0 2 2 2 62 662 1662 1662 1662 124 125 YLAAAA PSIAAA VVVVxx 3995 5892 1 3 5 15 95 995 1995 3995 3995 190 191 RXAAAA QSIAAA AAAAxx 5828 5893 0 0 8 8 28 828 1828 828 5828 56 57 EQAAAA RSIAAA HHHHxx 4197 5894 1 1 7 17 97 197 197 4197 4197 194 195 LFAAAA SSIAAA OOOOxx 5146 5895 0 2 6 6 46 146 1146 146 5146 92 93 YPAAAA TSIAAA VVVVxx 753 5896 1 1 3 13 53 753 753 753 753 106 107 ZCAAAA USIAAA AAAAxx 7064 5897 0 0 4 4 64 64 1064 2064 7064 128 129 SLAAAA VSIAAA HHHHxx 1312 5898 0 0 2 12 12 312 1312 1312 1312 24 25 MYAAAA WSIAAA OOOOxx 5573 5899 1 1 3 13 73 573 1573 573 5573 146 147 JGAAAA XSIAAA VVVVxx 7634 5900 0 2 4 14 34 634 1634 2634 7634 68 69 QHAAAA YSIAAA AAAAxx 2459 5901 1 3 9 19 59 459 459 2459 2459 118 119 PQAAAA ZSIAAA HHHHxx 8636 5902 0 0 6 16 36 636 636 3636 8636 72 73 EUAAAA ATIAAA OOOOxx 5318 5903 0 2 8 18 18 318 1318 318 5318 36 37 OWAAAA BTIAAA VVVVxx 1064 5904 0 0 4 4 64 64 1064 1064 1064 128 129 YOAAAA CTIAAA AAAAxx 9779 5905 1 3 9 19 79 779 1779 4779 9779 158 159 DMAAAA DTIAAA HHHHxx 6512 5906 0 0 2 12 12 512 512 1512 6512 24 25 MQAAAA ETIAAA OOOOxx 3572 5907 0 0 2 12 72 572 1572 3572 3572 144 145 KHAAAA FTIAAA VVVVxx 816 5908 0 0 6 16 16 816 816 816 816 32 33 KFAAAA GTIAAA AAAAxx 3978 5909 0 2 8 18 78 978 1978 3978 3978 156 157 AXAAAA HTIAAA HHHHxx 5390 5910 0 2 0 10 90 390 1390 390 5390 180 181 IZAAAA ITIAAA OOOOxx 4685 5911 1 1 5 5 85 685 685 4685 4685 170 171 FYAAAA JTIAAA VVVVxx 3003 5912 1 3 3 3 3 3 1003 3003 3003 6 7 NLAAAA KTIAAA AAAAxx 2638 5913 0 2 8 18 38 638 638 2638 2638 76 77 MXAAAA LTIAAA HHHHxx 9716 5914 0 0 6 16 16 716 1716 4716 9716 32 33 SJAAAA MTIAAA OOOOxx 9598 5915 0 2 8 18 98 598 1598 4598 9598 196 197 EFAAAA NTIAAA VVVVxx 9501 5916 1 1 1 1 1 501 1501 4501 9501 2 3 LBAAAA OTIAAA AAAAxx 1704 5917 0 0 4 4 4 704 1704 1704 1704 8 9 ONAAAA PTIAAA HHHHxx 8609 5918 1 1 9 9 9 609 609 3609 8609 18 19 DTAAAA QTIAAA OOOOxx 5211 5919 1 3 1 11 11 211 1211 211 5211 22 23 LSAAAA RTIAAA VVVVxx 3605 5920 1 1 5 5 5 605 1605 3605 3605 10 11 RIAAAA STIAAA AAAAxx 8730 5921 0 2 0 10 30 730 730 3730 8730 60 61 UXAAAA TTIAAA HHHHxx 4208 5922 0 0 8 8 8 208 208 4208 4208 16 17 WFAAAA UTIAAA OOOOxx 7784 5923 0 0 4 4 84 784 1784 2784 7784 168 169 KNAAAA VTIAAA VVVVxx 7501 5924 1 1 1 1 1 501 1501 2501 7501 2 3 NCAAAA WTIAAA AAAAxx 7862 5925 0 2 2 2 62 862 1862 2862 7862 124 125 KQAAAA XTIAAA HHHHxx 8922 5926 0 2 2 2 22 922 922 3922 8922 44 45 EFAAAA YTIAAA OOOOxx 3857 5927 1 1 7 17 57 857 1857 3857 3857 114 115 JSAAAA ZTIAAA VVVVxx 6393 5928 1 1 3 13 93 393 393 1393 6393 186 187 XLAAAA AUIAAA AAAAxx 506 5929 0 2 6 6 6 506 506 506 506 12 13 MTAAAA BUIAAA HHHHxx 4232 5930 0 0 2 12 32 232 232 4232 4232 64 65 UGAAAA CUIAAA OOOOxx 8991 5931 1 3 1 11 91 991 991 3991 8991 182 183 VHAAAA DUIAAA VVVVxx 8578 5932 0 2 8 18 78 578 578 3578 8578 156 157 YRAAAA EUIAAA AAAAxx 3235 5933 1 3 5 15 35 235 1235 3235 3235 70 71 LUAAAA FUIAAA HHHHxx 963 5934 1 3 3 3 63 963 963 963 963 126 127 BLAAAA GUIAAA OOOOxx 113 5935 1 1 3 13 13 113 113 113 113 26 27 JEAAAA HUIAAA VVVVxx 8234 5936 0 2 4 14 34 234 234 3234 8234 68 69 SEAAAA IUIAAA AAAAxx 2613 5937 1 1 3 13 13 613 613 2613 2613 26 27 NWAAAA JUIAAA HHHHxx 5540 5938 0 0 0 0 40 540 1540 540 5540 80 81 CFAAAA KUIAAA OOOOxx 9727 5939 1 3 7 7 27 727 1727 4727 9727 54 55 DKAAAA LUIAAA VVVVxx 2229 5940 1 1 9 9 29 229 229 2229 2229 58 59 THAAAA MUIAAA AAAAxx 6242 5941 0 2 2 2 42 242 242 1242 6242 84 85 CGAAAA NUIAAA HHHHxx 2502 5942 0 2 2 2 2 502 502 2502 2502 4 5 GSAAAA OUIAAA OOOOxx 6212 5943 0 0 2 12 12 212 212 1212 6212 24 25 YEAAAA PUIAAA VVVVxx 3495 5944 1 3 5 15 95 495 1495 3495 3495 190 191 LEAAAA QUIAAA AAAAxx 2364 5945 0 0 4 4 64 364 364 2364 2364 128 129 YMAAAA RUIAAA HHHHxx 6777 5946 1 1 7 17 77 777 777 1777 6777 154 155 RAAAAA SUIAAA OOOOxx 9811 5947 1 3 1 11 11 811 1811 4811 9811 22 23 JNAAAA TUIAAA VVVVxx 1450 5948 0 2 0 10 50 450 1450 1450 1450 100 101 UDAAAA UUIAAA AAAAxx 5008 5949 0 0 8 8 8 8 1008 8 5008 16 17 QKAAAA VUIAAA HHHHxx 1318 5950 0 2 8 18 18 318 1318 1318 1318 36 37 SYAAAA WUIAAA OOOOxx 3373 5951 1 1 3 13 73 373 1373 3373 3373 146 147 TZAAAA XUIAAA VVVVxx 398 5952 0 2 8 18 98 398 398 398 398 196 197 IPAAAA YUIAAA AAAAxx 3804 5953 0 0 4 4 4 804 1804 3804 3804 8 9 IQAAAA ZUIAAA HHHHxx 9148 5954 0 0 8 8 48 148 1148 4148 9148 96 97 WNAAAA AVIAAA OOOOxx 4382 5955 0 2 2 2 82 382 382 4382 4382 164 165 OMAAAA BVIAAA VVVVxx 4026 5956 0 2 6 6 26 26 26 4026 4026 52 53 WYAAAA CVIAAA AAAAxx 7804 5957 0 0 4 4 4 804 1804 2804 7804 8 9 EOAAAA DVIAAA HHHHxx 6839 5958 1 3 9 19 39 839 839 1839 6839 78 79 BDAAAA EVIAAA OOOOxx 3756 5959 0 0 6 16 56 756 1756 3756 3756 112 113 MOAAAA FVIAAA VVVVxx 6734 5960 0 2 4 14 34 734 734 1734 6734 68 69 AZAAAA GVIAAA AAAAxx 2228 5961 0 0 8 8 28 228 228 2228 2228 56 57 SHAAAA HVIAAA HHHHxx 3273 5962 1 1 3 13 73 273 1273 3273 3273 146 147 XVAAAA IVIAAA OOOOxx 3708 5963 0 0 8 8 8 708 1708 3708 3708 16 17 QMAAAA JVIAAA VVVVxx 4320 5964 0 0 0 0 20 320 320 4320 4320 40 41 EKAAAA KVIAAA AAAAxx 74 5965 0 2 4 14 74 74 74 74 74 148 149 WCAAAA LVIAAA HHHHxx 2520 5966 0 0 0 0 20 520 520 2520 2520 40 41 YSAAAA MVIAAA OOOOxx 9619 5967 1 3 9 19 19 619 1619 4619 9619 38 39 ZFAAAA NVIAAA VVVVxx 1801 5968 1 1 1 1 1 801 1801 1801 1801 2 3 HRAAAA OVIAAA AAAAxx 6399 5969 1 3 9 19 99 399 399 1399 6399 198 199 DMAAAA PVIAAA HHHHxx 8313 5970 1 1 3 13 13 313 313 3313 8313 26 27 THAAAA QVIAAA OOOOxx 7003 5971 1 3 3 3 3 3 1003 2003 7003 6 7 JJAAAA RVIAAA VVVVxx 329 5972 1 1 9 9 29 329 329 329 329 58 59 RMAAAA SVIAAA AAAAxx 9090 5973 0 2 0 10 90 90 1090 4090 9090 180 181 QLAAAA TVIAAA HHHHxx 2299 5974 1 3 9 19 99 299 299 2299 2299 198 199 LKAAAA UVIAAA OOOOxx 3925 5975 1 1 5 5 25 925 1925 3925 3925 50 51 ZUAAAA VVIAAA VVVVxx 8145 5976 1 1 5 5 45 145 145 3145 8145 90 91 HBAAAA WVIAAA AAAAxx 8561 5977 1 1 1 1 61 561 561 3561 8561 122 123 HRAAAA XVIAAA HHHHxx 2797 5978 1 1 7 17 97 797 797 2797 2797 194 195 PDAAAA YVIAAA OOOOxx 1451 5979 1 3 1 11 51 451 1451 1451 1451 102 103 VDAAAA ZVIAAA VVVVxx 7977 5980 1 1 7 17 77 977 1977 2977 7977 154 155 VUAAAA AWIAAA AAAAxx 112 5981 0 0 2 12 12 112 112 112 112 24 25 IEAAAA BWIAAA HHHHxx 5265 5982 1 1 5 5 65 265 1265 265 5265 130 131 NUAAAA CWIAAA OOOOxx 3819 5983 1 3 9 19 19 819 1819 3819 3819 38 39 XQAAAA DWIAAA VVVVxx 3648 5984 0 0 8 8 48 648 1648 3648 3648 96 97 IKAAAA EWIAAA AAAAxx 6306 5985 0 2 6 6 6 306 306 1306 6306 12 13 OIAAAA FWIAAA HHHHxx 2385 5986 1 1 5 5 85 385 385 2385 2385 170 171 TNAAAA GWIAAA OOOOxx 9084 5987 0 0 4 4 84 84 1084 4084 9084 168 169 KLAAAA HWIAAA VVVVxx 4499 5988 1 3 9 19 99 499 499 4499 4499 198 199 BRAAAA IWIAAA AAAAxx 1154 5989 0 2 4 14 54 154 1154 1154 1154 108 109 KSAAAA JWIAAA HHHHxx 6800 5990 0 0 0 0 0 800 800 1800 6800 0 1 OBAAAA KWIAAA OOOOxx 8049 5991 1 1 9 9 49 49 49 3049 8049 98 99 PXAAAA LWIAAA VVVVxx 3733 5992 1 1 3 13 33 733 1733 3733 3733 66 67 PNAAAA MWIAAA AAAAxx 8496 5993 0 0 6 16 96 496 496 3496 8496 192 193 UOAAAA NWIAAA HHHHxx 9952 5994 0 0 2 12 52 952 1952 4952 9952 104 105 USAAAA OWIAAA OOOOxx 9792 5995 0 0 2 12 92 792 1792 4792 9792 184 185 QMAAAA PWIAAA VVVVxx 5081 5996 1 1 1 1 81 81 1081 81 5081 162 163 LNAAAA QWIAAA AAAAxx 7908 5997 0 0 8 8 8 908 1908 2908 7908 16 17 ESAAAA RWIAAA HHHHxx 5398 5998 0 2 8 18 98 398 1398 398 5398 196 197 QZAAAA SWIAAA OOOOxx 8423 5999 1 3 3 3 23 423 423 3423 8423 46 47 ZLAAAA TWIAAA VVVVxx 3362 6000 0 2 2 2 62 362 1362 3362 3362 124 125 IZAAAA UWIAAA AAAAxx 7767 6001 1 3 7 7 67 767 1767 2767 7767 134 135 TMAAAA VWIAAA HHHHxx 7063 6002 1 3 3 3 63 63 1063 2063 7063 126 127 RLAAAA WWIAAA OOOOxx 8350 6003 0 2 0 10 50 350 350 3350 8350 100 101 EJAAAA XWIAAA VVVVxx 6779 6004 1 3 9 19 79 779 779 1779 6779 158 159 TAAAAA YWIAAA AAAAxx 5742 6005 0 2 2 2 42 742 1742 742 5742 84 85 WMAAAA ZWIAAA HHHHxx 9045 6006 1 1 5 5 45 45 1045 4045 9045 90 91 XJAAAA AXIAAA OOOOxx 8792 6007 0 0 2 12 92 792 792 3792 8792 184 185 EAAAAA BXIAAA VVVVxx 8160 6008 0 0 0 0 60 160 160 3160 8160 120 121 WBAAAA CXIAAA AAAAxx 3061 6009 1 1 1 1 61 61 1061 3061 3061 122 123 TNAAAA DXIAAA HHHHxx 4721 6010 1 1 1 1 21 721 721 4721 4721 42 43 PZAAAA EXIAAA OOOOxx 9817 6011 1 1 7 17 17 817 1817 4817 9817 34 35 PNAAAA FXIAAA VVVVxx 9257 6012 1 1 7 17 57 257 1257 4257 9257 114 115 BSAAAA GXIAAA AAAAxx 7779 6013 1 3 9 19 79 779 1779 2779 7779 158 159 FNAAAA HXIAAA HHHHxx 2663 6014 1 3 3 3 63 663 663 2663 2663 126 127 LYAAAA IXIAAA OOOOxx 3885 6015 1 1 5 5 85 885 1885 3885 3885 170 171 LTAAAA JXIAAA VVVVxx 9469 6016 1 1 9 9 69 469 1469 4469 9469 138 139 FAAAAA KXIAAA AAAAxx 6766 6017 0 2 6 6 66 766 766 1766 6766 132 133 GAAAAA LXIAAA HHHHxx 7173 6018 1 1 3 13 73 173 1173 2173 7173 146 147 XPAAAA MXIAAA OOOOxx 4709 6019 1 1 9 9 9 709 709 4709 4709 18 19 DZAAAA NXIAAA VVVVxx 4210 6020 0 2 0 10 10 210 210 4210 4210 20 21 YFAAAA OXIAAA AAAAxx 3715 6021 1 3 5 15 15 715 1715 3715 3715 30 31 XMAAAA PXIAAA HHHHxx 5089 6022 1 1 9 9 89 89 1089 89 5089 178 179 TNAAAA QXIAAA OOOOxx 1639 6023 1 3 9 19 39 639 1639 1639 1639 78 79 BLAAAA RXIAAA VVVVxx 5757 6024 1 1 7 17 57 757 1757 757 5757 114 115 LNAAAA SXIAAA AAAAxx 3545 6025 1 1 5 5 45 545 1545 3545 3545 90 91 JGAAAA TXIAAA HHHHxx 709 6026 1 1 9 9 9 709 709 709 709 18 19 HBAAAA UXIAAA OOOOxx 6519 6027 1 3 9 19 19 519 519 1519 6519 38 39 TQAAAA VXIAAA VVVVxx 4341 6028 1 1 1 1 41 341 341 4341 4341 82 83 ZKAAAA WXIAAA AAAAxx 2381 6029 1 1 1 1 81 381 381 2381 2381 162 163 PNAAAA XXIAAA HHHHxx 7215 6030 1 3 5 15 15 215 1215 2215 7215 30 31 NRAAAA YXIAAA OOOOxx 9323 6031 1 3 3 3 23 323 1323 4323 9323 46 47 PUAAAA ZXIAAA VVVVxx 3593 6032 1 1 3 13 93 593 1593 3593 3593 186 187 FIAAAA AYIAAA AAAAxx 3123 6033 1 3 3 3 23 123 1123 3123 3123 46 47 DQAAAA BYIAAA HHHHxx 8673 6034 1 1 3 13 73 673 673 3673 8673 146 147 PVAAAA CYIAAA OOOOxx 5094 6035 0 2 4 14 94 94 1094 94 5094 188 189 YNAAAA DYIAAA VVVVxx 6477 6036 1 1 7 17 77 477 477 1477 6477 154 155 DPAAAA EYIAAA AAAAxx 9734 6037 0 2 4 14 34 734 1734 4734 9734 68 69 KKAAAA FYIAAA HHHHxx 2998 6038 0 2 8 18 98 998 998 2998 2998 196 197 ILAAAA GYIAAA OOOOxx 7807 6039 1 3 7 7 7 807 1807 2807 7807 14 15 HOAAAA HYIAAA VVVVxx 5739 6040 1 3 9 19 39 739 1739 739 5739 78 79 TMAAAA IYIAAA AAAAxx 138 6041 0 2 8 18 38 138 138 138 138 76 77 IFAAAA JYIAAA HHHHxx 2403 6042 1 3 3 3 3 403 403 2403 2403 6 7 LOAAAA KYIAAA OOOOxx 2484 6043 0 0 4 4 84 484 484 2484 2484 168 169 ORAAAA LYIAAA VVVVxx 2805 6044 1 1 5 5 5 805 805 2805 2805 10 11 XDAAAA MYIAAA AAAAxx 5189 6045 1 1 9 9 89 189 1189 189 5189 178 179 PRAAAA NYIAAA HHHHxx 8336 6046 0 0 6 16 36 336 336 3336 8336 72 73 QIAAAA OYIAAA OOOOxx 5241 6047 1 1 1 1 41 241 1241 241 5241 82 83 PTAAAA PYIAAA VVVVxx 2612 6048 0 0 2 12 12 612 612 2612 2612 24 25 MWAAAA QYIAAA AAAAxx 2571 6049 1 3 1 11 71 571 571 2571 2571 142 143 XUAAAA RYIAAA HHHHxx 926 6050 0 2 6 6 26 926 926 926 926 52 53 QJAAAA SYIAAA OOOOxx 337 6051 1 1 7 17 37 337 337 337 337 74 75 ZMAAAA TYIAAA VVVVxx 2821 6052 1 1 1 1 21 821 821 2821 2821 42 43 NEAAAA UYIAAA AAAAxx 2658 6053 0 2 8 18 58 658 658 2658 2658 116 117 GYAAAA VYIAAA HHHHxx 9054 6054 0 2 4 14 54 54 1054 4054 9054 108 109 GKAAAA WYIAAA OOOOxx 5492 6055 0 0 2 12 92 492 1492 492 5492 184 185 GDAAAA XYIAAA VVVVxx 7313 6056 1 1 3 13 13 313 1313 2313 7313 26 27 HVAAAA YYIAAA AAAAxx 75 6057 1 3 5 15 75 75 75 75 75 150 151 XCAAAA ZYIAAA HHHHxx 5489 6058 1 1 9 9 89 489 1489 489 5489 178 179 DDAAAA AZIAAA OOOOxx 8413 6059 1 1 3 13 13 413 413 3413 8413 26 27 PLAAAA BZIAAA VVVVxx 3693 6060 1 1 3 13 93 693 1693 3693 3693 186 187 BMAAAA CZIAAA AAAAxx 9820 6061 0 0 0 0 20 820 1820 4820 9820 40 41 SNAAAA DZIAAA HHHHxx 8157 6062 1 1 7 17 57 157 157 3157 8157 114 115 TBAAAA EZIAAA OOOOxx 4161 6063 1 1 1 1 61 161 161 4161 4161 122 123 BEAAAA FZIAAA VVVVxx 8339 6064 1 3 9 19 39 339 339 3339 8339 78 79 TIAAAA GZIAAA AAAAxx 4141 6065 1 1 1 1 41 141 141 4141 4141 82 83 HDAAAA HZIAAA HHHHxx 9001 6066 1 1 1 1 1 1 1001 4001 9001 2 3 FIAAAA IZIAAA OOOOxx 8247 6067 1 3 7 7 47 247 247 3247 8247 94 95 FFAAAA JZIAAA VVVVxx 1182 6068 0 2 2 2 82 182 1182 1182 1182 164 165 MTAAAA KZIAAA AAAAxx 9876 6069 0 0 6 16 76 876 1876 4876 9876 152 153 WPAAAA LZIAAA HHHHxx 4302 6070 0 2 2 2 2 302 302 4302 4302 4 5 MJAAAA MZIAAA OOOOxx 6674 6071 0 2 4 14 74 674 674 1674 6674 148 149 SWAAAA NZIAAA VVVVxx 4214 6072 0 2 4 14 14 214 214 4214 4214 28 29 CGAAAA OZIAAA AAAAxx 5584 6073 0 0 4 4 84 584 1584 584 5584 168 169 UGAAAA PZIAAA HHHHxx 265 6074 1 1 5 5 65 265 265 265 265 130 131 FKAAAA QZIAAA OOOOxx 9207 6075 1 3 7 7 7 207 1207 4207 9207 14 15 DQAAAA RZIAAA VVVVxx 9434 6076 0 2 4 14 34 434 1434 4434 9434 68 69 WYAAAA SZIAAA AAAAxx 2921 6077 1 1 1 1 21 921 921 2921 2921 42 43 JIAAAA TZIAAA HHHHxx 9355 6078 1 3 5 15 55 355 1355 4355 9355 110 111 VVAAAA UZIAAA OOOOxx 8538 6079 0 2 8 18 38 538 538 3538 8538 76 77 KQAAAA VZIAAA VVVVxx 4559 6080 1 3 9 19 59 559 559 4559 4559 118 119 JTAAAA WZIAAA AAAAxx 9175 6081 1 3 5 15 75 175 1175 4175 9175 150 151 XOAAAA XZIAAA HHHHxx 4489 6082 1 1 9 9 89 489 489 4489 4489 178 179 RQAAAA YZIAAA OOOOxx 1485 6083 1 1 5 5 85 485 1485 1485 1485 170 171 DFAAAA ZZIAAA VVVVxx 8853 6084 1 1 3 13 53 853 853 3853 8853 106 107 NCAAAA AAJAAA AAAAxx 9143 6085 1 3 3 3 43 143 1143 4143 9143 86 87 RNAAAA BAJAAA HHHHxx 9551 6086 1 3 1 11 51 551 1551 4551 9551 102 103 JDAAAA CAJAAA OOOOxx 49 6087 1 1 9 9 49 49 49 49 49 98 99 XBAAAA DAJAAA VVVVxx 8351 6088 1 3 1 11 51 351 351 3351 8351 102 103 FJAAAA EAJAAA AAAAxx 9748 6089 0 0 8 8 48 748 1748 4748 9748 96 97 YKAAAA FAJAAA HHHHxx 4536 6090 0 0 6 16 36 536 536 4536 4536 72 73 MSAAAA GAJAAA OOOOxx 930 6091 0 2 0 10 30 930 930 930 930 60 61 UJAAAA HAJAAA VVVVxx 2206 6092 0 2 6 6 6 206 206 2206 2206 12 13 WGAAAA IAJAAA AAAAxx 8004 6093 0 0 4 4 4 4 4 3004 8004 8 9 WVAAAA JAJAAA HHHHxx 219 6094 1 3 9 19 19 219 219 219 219 38 39 LIAAAA KAJAAA OOOOxx 2724 6095 0 0 4 4 24 724 724 2724 2724 48 49 UAAAAA LAJAAA VVVVxx 4868 6096 0 0 8 8 68 868 868 4868 4868 136 137 GFAAAA MAJAAA AAAAxx 5952 6097 0 0 2 12 52 952 1952 952 5952 104 105 YUAAAA NAJAAA HHHHxx 2094 6098 0 2 4 14 94 94 94 2094 2094 188 189 OCAAAA OAJAAA OOOOxx 5707 6099 1 3 7 7 7 707 1707 707 5707 14 15 NLAAAA PAJAAA VVVVxx 5200 6100 0 0 0 0 0 200 1200 200 5200 0 1 ASAAAA QAJAAA AAAAxx 967 6101 1 3 7 7 67 967 967 967 967 134 135 FLAAAA RAJAAA HHHHxx 1982 6102 0 2 2 2 82 982 1982 1982 1982 164 165 GYAAAA SAJAAA OOOOxx 3410 6103 0 2 0 10 10 410 1410 3410 3410 20 21 EBAAAA TAJAAA VVVVxx 174 6104 0 2 4 14 74 174 174 174 174 148 149 SGAAAA UAJAAA AAAAxx 9217 6105 1 1 7 17 17 217 1217 4217 9217 34 35 NQAAAA VAJAAA HHHHxx 9103 6106 1 3 3 3 3 103 1103 4103 9103 6 7 DMAAAA WAJAAA OOOOxx 868 6107 0 0 8 8 68 868 868 868 868 136 137 KHAAAA XAJAAA VVVVxx 8261 6108 1 1 1 1 61 261 261 3261 8261 122 123 TFAAAA YAJAAA AAAAxx 2720 6109 0 0 0 0 20 720 720 2720 2720 40 41 QAAAAA ZAJAAA HHHHxx 2999 6110 1 3 9 19 99 999 999 2999 2999 198 199 JLAAAA ABJAAA OOOOxx 769 6111 1 1 9 9 69 769 769 769 769 138 139 PDAAAA BBJAAA VVVVxx 4533 6112 1 1 3 13 33 533 533 4533 4533 66 67 JSAAAA CBJAAA AAAAxx 2030 6113 0 2 0 10 30 30 30 2030 2030 60 61 CAAAAA DBJAAA HHHHxx 5824 6114 0 0 4 4 24 824 1824 824 5824 48 49 AQAAAA EBJAAA OOOOxx 2328 6115 0 0 8 8 28 328 328 2328 2328 56 57 OLAAAA FBJAAA VVVVxx 9970 6116 0 2 0 10 70 970 1970 4970 9970 140 141 MTAAAA GBJAAA AAAAxx 3192 6117 0 0 2 12 92 192 1192 3192 3192 184 185 USAAAA HBJAAA HHHHxx 3387 6118 1 3 7 7 87 387 1387 3387 3387 174 175 HAAAAA IBJAAA OOOOxx 1936 6119 0 0 6 16 36 936 1936 1936 1936 72 73 MWAAAA JBJAAA VVVVxx 6934 6120 0 2 4 14 34 934 934 1934 6934 68 69 SGAAAA KBJAAA AAAAxx 5615 6121 1 3 5 15 15 615 1615 615 5615 30 31 ZHAAAA LBJAAA HHHHxx 2241 6122 1 1 1 1 41 241 241 2241 2241 82 83 FIAAAA MBJAAA OOOOxx 1842 6123 0 2 2 2 42 842 1842 1842 1842 84 85 WSAAAA NBJAAA VVVVxx 8044 6124 0 0 4 4 44 44 44 3044 8044 88 89 KXAAAA OBJAAA AAAAxx 8902 6125 0 2 2 2 2 902 902 3902 8902 4 5 KEAAAA PBJAAA HHHHxx 4519 6126 1 3 9 19 19 519 519 4519 4519 38 39 VRAAAA QBJAAA OOOOxx 492 6127 0 0 2 12 92 492 492 492 492 184 185 YSAAAA RBJAAA VVVVxx 2694 6128 0 2 4 14 94 694 694 2694 2694 188 189 QZAAAA SBJAAA AAAAxx 5861 6129 1 1 1 1 61 861 1861 861 5861 122 123 LRAAAA TBJAAA HHHHxx 2104 6130 0 0 4 4 4 104 104 2104 2104 8 9 YCAAAA UBJAAA OOOOxx 5376 6131 0 0 6 16 76 376 1376 376 5376 152 153 UYAAAA VBJAAA VVVVxx 3147 6132 1 3 7 7 47 147 1147 3147 3147 94 95 BRAAAA WBJAAA AAAAxx 9880 6133 0 0 0 0 80 880 1880 4880 9880 160 161 AQAAAA XBJAAA HHHHxx 6171 6134 1 3 1 11 71 171 171 1171 6171 142 143 JDAAAA YBJAAA OOOOxx 1850 6135 0 2 0 10 50 850 1850 1850 1850 100 101 ETAAAA ZBJAAA VVVVxx 1775 6136 1 3 5 15 75 775 1775 1775 1775 150 151 HQAAAA ACJAAA AAAAxx 9261 6137 1 1 1 1 61 261 1261 4261 9261 122 123 FSAAAA BCJAAA HHHHxx 9648 6138 0 0 8 8 48 648 1648 4648 9648 96 97 CHAAAA CCJAAA OOOOxx 7846 6139 0 2 6 6 46 846 1846 2846 7846 92 93 UPAAAA DCJAAA VVVVxx 1446 6140 0 2 6 6 46 446 1446 1446 1446 92 93 QDAAAA ECJAAA AAAAxx 3139 6141 1 3 9 19 39 139 1139 3139 3139 78 79 TQAAAA FCJAAA HHHHxx 6142 6142 0 2 2 2 42 142 142 1142 6142 84 85 GCAAAA GCJAAA OOOOxx 5812 6143 0 0 2 12 12 812 1812 812 5812 24 25 OPAAAA HCJAAA VVVVxx 6728 6144 0 0 8 8 28 728 728 1728 6728 56 57 UYAAAA ICJAAA AAAAxx 4428 6145 0 0 8 8 28 428 428 4428 4428 56 57 IOAAAA JCJAAA HHHHxx 502 6146 0 2 2 2 2 502 502 502 502 4 5 ITAAAA KCJAAA OOOOxx 2363 6147 1 3 3 3 63 363 363 2363 2363 126 127 XMAAAA LCJAAA VVVVxx 3808 6148 0 0 8 8 8 808 1808 3808 3808 16 17 MQAAAA MCJAAA AAAAxx 1010 6149 0 2 0 10 10 10 1010 1010 1010 20 21 WMAAAA NCJAAA HHHHxx 9565 6150 1 1 5 5 65 565 1565 4565 9565 130 131 XDAAAA OCJAAA OOOOxx 1587 6151 1 3 7 7 87 587 1587 1587 1587 174 175 BJAAAA PCJAAA VVVVxx 1474 6152 0 2 4 14 74 474 1474 1474 1474 148 149 SEAAAA QCJAAA AAAAxx 6215 6153 1 3 5 15 15 215 215 1215 6215 30 31 BFAAAA RCJAAA HHHHxx 2395 6154 1 3 5 15 95 395 395 2395 2395 190 191 DOAAAA SCJAAA OOOOxx 8753 6155 1 1 3 13 53 753 753 3753 8753 106 107 RYAAAA TCJAAA VVVVxx 2446 6156 0 2 6 6 46 446 446 2446 2446 92 93 CQAAAA UCJAAA AAAAxx 60 6157 0 0 0 0 60 60 60 60 60 120 121 ICAAAA VCJAAA HHHHxx 982 6158 0 2 2 2 82 982 982 982 982 164 165 ULAAAA WCJAAA OOOOxx 6489 6159 1 1 9 9 89 489 489 1489 6489 178 179 PPAAAA XCJAAA VVVVxx 5334 6160 0 2 4 14 34 334 1334 334 5334 68 69 EXAAAA YCJAAA AAAAxx 8540 6161 0 0 0 0 40 540 540 3540 8540 80 81 MQAAAA ZCJAAA HHHHxx 490 6162 0 2 0 10 90 490 490 490 490 180 181 WSAAAA ADJAAA OOOOxx 6763 6163 1 3 3 3 63 763 763 1763 6763 126 127 DAAAAA BDJAAA VVVVxx 8273 6164 1 1 3 13 73 273 273 3273 8273 146 147 FGAAAA CDJAAA AAAAxx 8327 6165 1 3 7 7 27 327 327 3327 8327 54 55 HIAAAA DDJAAA HHHHxx 8541 6166 1 1 1 1 41 541 541 3541 8541 82 83 NQAAAA EDJAAA OOOOxx 3459 6167 1 3 9 19 59 459 1459 3459 3459 118 119 BDAAAA FDJAAA VVVVxx 5557 6168 1 1 7 17 57 557 1557 557 5557 114 115 TFAAAA GDJAAA AAAAxx 158 6169 0 2 8 18 58 158 158 158 158 116 117 CGAAAA HDJAAA HHHHxx 1741 6170 1 1 1 1 41 741 1741 1741 1741 82 83 ZOAAAA IDJAAA OOOOxx 8385 6171 1 1 5 5 85 385 385 3385 8385 170 171 NKAAAA JDJAAA VVVVxx 617 6172 1 1 7 17 17 617 617 617 617 34 35 TXAAAA KDJAAA AAAAxx 3560 6173 0 0 0 0 60 560 1560 3560 3560 120 121 YGAAAA LDJAAA HHHHxx 5216 6174 0 0 6 16 16 216 1216 216 5216 32 33 QSAAAA MDJAAA OOOOxx 8443 6175 1 3 3 3 43 443 443 3443 8443 86 87 TMAAAA NDJAAA VVVVxx 2700 6176 0 0 0 0 0 700 700 2700 2700 0 1 WZAAAA ODJAAA AAAAxx 3661 6177 1 1 1 1 61 661 1661 3661 3661 122 123 VKAAAA PDJAAA HHHHxx 4875 6178 1 3 5 15 75 875 875 4875 4875 150 151 NFAAAA QDJAAA OOOOxx 6721 6179 1 1 1 1 21 721 721 1721 6721 42 43 NYAAAA RDJAAA VVVVxx 3659 6180 1 3 9 19 59 659 1659 3659 3659 118 119 TKAAAA SDJAAA AAAAxx 8944 6181 0 0 4 4 44 944 944 3944 8944 88 89 AGAAAA TDJAAA HHHHxx 9133 6182 1 1 3 13 33 133 1133 4133 9133 66 67 HNAAAA UDJAAA OOOOxx 9882 6183 0 2 2 2 82 882 1882 4882 9882 164 165 CQAAAA VDJAAA VVVVxx 2102 6184 0 2 2 2 2 102 102 2102 2102 4 5 WCAAAA WDJAAA AAAAxx 9445 6185 1 1 5 5 45 445 1445 4445 9445 90 91 HZAAAA XDJAAA HHHHxx 5559 6186 1 3 9 19 59 559 1559 559 5559 118 119 VFAAAA YDJAAA OOOOxx 6096 6187 0 0 6 16 96 96 96 1096 6096 192 193 MAAAAA ZDJAAA VVVVxx 9336 6188 0 0 6 16 36 336 1336 4336 9336 72 73 CVAAAA AEJAAA AAAAxx 2162 6189 0 2 2 2 62 162 162 2162 2162 124 125 EFAAAA BEJAAA HHHHxx 7459 6190 1 3 9 19 59 459 1459 2459 7459 118 119 XAAAAA CEJAAA OOOOxx 3248 6191 0 0 8 8 48 248 1248 3248 3248 96 97 YUAAAA DEJAAA VVVVxx 9539 6192 1 3 9 19 39 539 1539 4539 9539 78 79 XCAAAA EEJAAA AAAAxx 4449 6193 1 1 9 9 49 449 449 4449 4449 98 99 DPAAAA FEJAAA HHHHxx 2809 6194 1 1 9 9 9 809 809 2809 2809 18 19 BEAAAA GEJAAA OOOOxx 7058 6195 0 2 8 18 58 58 1058 2058 7058 116 117 MLAAAA HEJAAA VVVVxx 3512 6196 0 0 2 12 12 512 1512 3512 3512 24 25 CFAAAA IEJAAA AAAAxx 2802 6197 0 2 2 2 2 802 802 2802 2802 4 5 UDAAAA JEJAAA HHHHxx 6289 6198 1 1 9 9 89 289 289 1289 6289 178 179 XHAAAA KEJAAA OOOOxx 1947 6199 1 3 7 7 47 947 1947 1947 1947 94 95 XWAAAA LEJAAA VVVVxx 9572 6200 0 0 2 12 72 572 1572 4572 9572 144 145 EEAAAA MEJAAA AAAAxx 2356 6201 0 0 6 16 56 356 356 2356 2356 112 113 QMAAAA NEJAAA HHHHxx 3039 6202 1 3 9 19 39 39 1039 3039 3039 78 79 XMAAAA OEJAAA OOOOxx 9452 6203 0 0 2 12 52 452 1452 4452 9452 104 105 OZAAAA PEJAAA VVVVxx 6328 6204 0 0 8 8 28 328 328 1328 6328 56 57 KJAAAA QEJAAA AAAAxx 7661 6205 1 1 1 1 61 661 1661 2661 7661 122 123 RIAAAA REJAAA HHHHxx 2566 6206 0 2 6 6 66 566 566 2566 2566 132 133 SUAAAA SEJAAA OOOOxx 6095 6207 1 3 5 15 95 95 95 1095 6095 190 191 LAAAAA TEJAAA VVVVxx 6367 6208 1 3 7 7 67 367 367 1367 6367 134 135 XKAAAA UEJAAA AAAAxx 3368 6209 0 0 8 8 68 368 1368 3368 3368 136 137 OZAAAA VEJAAA HHHHxx 5567 6210 1 3 7 7 67 567 1567 567 5567 134 135 DGAAAA WEJAAA OOOOxx 9834 6211 0 2 4 14 34 834 1834 4834 9834 68 69 GOAAAA XEJAAA VVVVxx 9695 6212 1 3 5 15 95 695 1695 4695 9695 190 191 XIAAAA YEJAAA AAAAxx 7291 6213 1 3 1 11 91 291 1291 2291 7291 182 183 LUAAAA ZEJAAA HHHHxx 4806 6214 0 2 6 6 6 806 806 4806 4806 12 13 WCAAAA AFJAAA OOOOxx 2000 6215 0 0 0 0 0 0 0 2000 2000 0 1 YYAAAA BFJAAA VVVVxx 6817 6216 1 1 7 17 17 817 817 1817 6817 34 35 FCAAAA CFJAAA AAAAxx 8487 6217 1 3 7 7 87 487 487 3487 8487 174 175 LOAAAA DFJAAA HHHHxx 3245 6218 1 1 5 5 45 245 1245 3245 3245 90 91 VUAAAA EFJAAA OOOOxx 632 6219 0 0 2 12 32 632 632 632 632 64 65 IYAAAA FFJAAA VVVVxx 8067 6220 1 3 7 7 67 67 67 3067 8067 134 135 HYAAAA GFJAAA AAAAxx 7140 6221 0 0 0 0 40 140 1140 2140 7140 80 81 QOAAAA HFJAAA HHHHxx 6802 6222 0 2 2 2 2 802 802 1802 6802 4 5 QBAAAA IFJAAA OOOOxx 3980 6223 0 0 0 0 80 980 1980 3980 3980 160 161 CXAAAA JFJAAA VVVVxx 1321 6224 1 1 1 1 21 321 1321 1321 1321 42 43 VYAAAA KFJAAA AAAAxx 2273 6225 1 1 3 13 73 273 273 2273 2273 146 147 LJAAAA LFJAAA HHHHxx 6787 6226 1 3 7 7 87 787 787 1787 6787 174 175 BBAAAA MFJAAA OOOOxx 9480 6227 0 0 0 0 80 480 1480 4480 9480 160 161 QAAAAA NFJAAA VVVVxx 9404 6228 0 0 4 4 4 404 1404 4404 9404 8 9 SXAAAA OFJAAA AAAAxx 3914 6229 0 2 4 14 14 914 1914 3914 3914 28 29 OUAAAA PFJAAA HHHHxx 5507 6230 1 3 7 7 7 507 1507 507 5507 14 15 VDAAAA QFJAAA OOOOxx 1813 6231 1 1 3 13 13 813 1813 1813 1813 26 27 TRAAAA RFJAAA VVVVxx 1999 6232 1 3 9 19 99 999 1999 1999 1999 198 199 XYAAAA SFJAAA AAAAxx 3848 6233 0 0 8 8 48 848 1848 3848 3848 96 97 ASAAAA TFJAAA HHHHxx 9693 6234 1 1 3 13 93 693 1693 4693 9693 186 187 VIAAAA UFJAAA OOOOxx 1353 6235 1 1 3 13 53 353 1353 1353 1353 106 107 BAAAAA VFJAAA VVVVxx 7218 6236 0 2 8 18 18 218 1218 2218 7218 36 37 QRAAAA WFJAAA AAAAxx 8223 6237 1 3 3 3 23 223 223 3223 8223 46 47 HEAAAA XFJAAA HHHHxx 9982 6238 0 2 2 2 82 982 1982 4982 9982 164 165 YTAAAA YFJAAA OOOOxx 8799 6239 1 3 9 19 99 799 799 3799 8799 198 199 LAAAAA ZFJAAA VVVVxx 8929 6240 1 1 9 9 29 929 929 3929 8929 58 59 LFAAAA AGJAAA AAAAxx 4626 6241 0 2 6 6 26 626 626 4626 4626 52 53 YVAAAA BGJAAA HHHHxx 7958 6242 0 2 8 18 58 958 1958 2958 7958 116 117 CUAAAA CGJAAA OOOOxx 3743 6243 1 3 3 3 43 743 1743 3743 3743 86 87 ZNAAAA DGJAAA VVVVxx 8165 6244 1 1 5 5 65 165 165 3165 8165 130 131 BCAAAA EGJAAA AAAAxx 7899 6245 1 3 9 19 99 899 1899 2899 7899 198 199 VRAAAA FGJAAA HHHHxx 8698 6246 0 2 8 18 98 698 698 3698 8698 196 197 OWAAAA GGJAAA OOOOxx 9270 6247 0 2 0 10 70 270 1270 4270 9270 140 141 OSAAAA HGJAAA VVVVxx 6348 6248 0 0 8 8 48 348 348 1348 6348 96 97 EKAAAA IGJAAA AAAAxx 6999 6249 1 3 9 19 99 999 999 1999 6999 198 199 FJAAAA JGJAAA HHHHxx 8467 6250 1 3 7 7 67 467 467 3467 8467 134 135 RNAAAA KGJAAA OOOOxx 3907 6251 1 3 7 7 7 907 1907 3907 3907 14 15 HUAAAA LGJAAA VVVVxx 4738 6252 0 2 8 18 38 738 738 4738 4738 76 77 GAAAAA MGJAAA AAAAxx 248 6253 0 0 8 8 48 248 248 248 248 96 97 OJAAAA NGJAAA HHHHxx 8769 6254 1 1 9 9 69 769 769 3769 8769 138 139 HZAAAA OGJAAA OOOOxx 9922 6255 0 2 2 2 22 922 1922 4922 9922 44 45 QRAAAA PGJAAA VVVVxx 778 6256 0 2 8 18 78 778 778 778 778 156 157 YDAAAA QGJAAA AAAAxx 1233 6257 1 1 3 13 33 233 1233 1233 1233 66 67 LVAAAA RGJAAA HHHHxx 1183 6258 1 3 3 3 83 183 1183 1183 1183 166 167 NTAAAA SGJAAA OOOOxx 2838 6259 0 2 8 18 38 838 838 2838 2838 76 77 EFAAAA TGJAAA VVVVxx 3096 6260 0 0 6 16 96 96 1096 3096 3096 192 193 CPAAAA UGJAAA AAAAxx 8566 6261 0 2 6 6 66 566 566 3566 8566 132 133 MRAAAA VGJAAA HHHHxx 7635 6262 1 3 5 15 35 635 1635 2635 7635 70 71 RHAAAA WGJAAA OOOOxx 5428 6263 0 0 8 8 28 428 1428 428 5428 56 57 UAAAAA XGJAAA VVVVxx 7430 6264 0 2 0 10 30 430 1430 2430 7430 60 61 UZAAAA YGJAAA AAAAxx 7210 6265 0 2 0 10 10 210 1210 2210 7210 20 21 IRAAAA ZGJAAA HHHHxx 4485 6266 1 1 5 5 85 485 485 4485 4485 170 171 NQAAAA AHJAAA OOOOxx 9623 6267 1 3 3 3 23 623 1623 4623 9623 46 47 DGAAAA BHJAAA VVVVxx 3670 6268 0 2 0 10 70 670 1670 3670 3670 140 141 ELAAAA CHJAAA AAAAxx 1575 6269 1 3 5 15 75 575 1575 1575 1575 150 151 PIAAAA DHJAAA HHHHxx 5874 6270 0 2 4 14 74 874 1874 874 5874 148 149 YRAAAA EHJAAA OOOOxx 673 6271 1 1 3 13 73 673 673 673 673 146 147 XZAAAA FHJAAA VVVVxx 9712 6272 0 0 2 12 12 712 1712 4712 9712 24 25 OJAAAA GHJAAA AAAAxx 7729 6273 1 1 9 9 29 729 1729 2729 7729 58 59 HLAAAA HHJAAA HHHHxx 4318 6274 0 2 8 18 18 318 318 4318 4318 36 37 CKAAAA IHJAAA OOOOxx 4143 6275 1 3 3 3 43 143 143 4143 4143 86 87 JDAAAA JHJAAA VVVVxx 4932 6276 0 0 2 12 32 932 932 4932 4932 64 65 SHAAAA KHJAAA AAAAxx 5835 6277 1 3 5 15 35 835 1835 835 5835 70 71 LQAAAA LHJAAA HHHHxx 4966 6278 0 2 6 6 66 966 966 4966 4966 132 133 AJAAAA MHJAAA OOOOxx 6711 6279 1 3 1 11 11 711 711 1711 6711 22 23 DYAAAA NHJAAA VVVVxx 3990 6280 0 2 0 10 90 990 1990 3990 3990 180 181 MXAAAA OHJAAA AAAAxx 990 6281 0 2 0 10 90 990 990 990 990 180 181 CMAAAA PHJAAA HHHHxx 220 6282 0 0 0 0 20 220 220 220 220 40 41 MIAAAA QHJAAA OOOOxx 5693 6283 1 1 3 13 93 693 1693 693 5693 186 187 ZKAAAA RHJAAA VVVVxx 3662 6284 0 2 2 2 62 662 1662 3662 3662 124 125 WKAAAA SHJAAA AAAAxx 7844 6285 0 0 4 4 44 844 1844 2844 7844 88 89 SPAAAA THJAAA HHHHxx 5515 6286 1 3 5 15 15 515 1515 515 5515 30 31 DEAAAA UHJAAA OOOOxx 5551 6287 1 3 1 11 51 551 1551 551 5551 102 103 NFAAAA VHJAAA VVVVxx 2358 6288 0 2 8 18 58 358 358 2358 2358 116 117 SMAAAA WHJAAA AAAAxx 8977 6289 1 1 7 17 77 977 977 3977 8977 154 155 HHAAAA XHJAAA HHHHxx 7040 6290 0 0 0 0 40 40 1040 2040 7040 80 81 UKAAAA YHJAAA OOOOxx 105 6291 1 1 5 5 5 105 105 105 105 10 11 BEAAAA ZHJAAA VVVVxx 4496 6292 0 0 6 16 96 496 496 4496 4496 192 193 YQAAAA AIJAAA AAAAxx 2254 6293 0 2 4 14 54 254 254 2254 2254 108 109 SIAAAA BIJAAA HHHHxx 411 6294 1 3 1 11 11 411 411 411 411 22 23 VPAAAA CIJAAA OOOOxx 2373 6295 1 1 3 13 73 373 373 2373 2373 146 147 HNAAAA DIJAAA VVVVxx 3477 6296 1 1 7 17 77 477 1477 3477 3477 154 155 TDAAAA EIJAAA AAAAxx 8964 6297 0 0 4 4 64 964 964 3964 8964 128 129 UGAAAA FIJAAA HHHHxx 8471 6298 1 3 1 11 71 471 471 3471 8471 142 143 VNAAAA GIJAAA OOOOxx 5776 6299 0 0 6 16 76 776 1776 776 5776 152 153 EOAAAA HIJAAA VVVVxx 9921 6300 1 1 1 1 21 921 1921 4921 9921 42 43 PRAAAA IIJAAA AAAAxx 7816 6301 0 0 6 16 16 816 1816 2816 7816 32 33 QOAAAA JIJAAA HHHHxx 2439 6302 1 3 9 19 39 439 439 2439 2439 78 79 VPAAAA KIJAAA OOOOxx 9298 6303 0 2 8 18 98 298 1298 4298 9298 196 197 QTAAAA LIJAAA VVVVxx 9424 6304 0 0 4 4 24 424 1424 4424 9424 48 49 MYAAAA MIJAAA AAAAxx 3252 6305 0 0 2 12 52 252 1252 3252 3252 104 105 CVAAAA NIJAAA HHHHxx 1401 6306 1 1 1 1 1 401 1401 1401 1401 2 3 XBAAAA OIJAAA OOOOxx 9632 6307 0 0 2 12 32 632 1632 4632 9632 64 65 MGAAAA PIJAAA VVVVxx 370 6308 0 2 0 10 70 370 370 370 370 140 141 GOAAAA QIJAAA AAAAxx 728 6309 0 0 8 8 28 728 728 728 728 56 57 ACAAAA RIJAAA HHHHxx 2888 6310 0 0 8 8 88 888 888 2888 2888 176 177 CHAAAA SIJAAA OOOOxx 1441 6311 1 1 1 1 41 441 1441 1441 1441 82 83 LDAAAA TIJAAA VVVVxx 8308 6312 0 0 8 8 8 308 308 3308 8308 16 17 OHAAAA UIJAAA AAAAxx 2165 6313 1 1 5 5 65 165 165 2165 2165 130 131 HFAAAA VIJAAA HHHHxx 6359 6314 1 3 9 19 59 359 359 1359 6359 118 119 PKAAAA WIJAAA OOOOxx 9637 6315 1 1 7 17 37 637 1637 4637 9637 74 75 RGAAAA XIJAAA VVVVxx 5208 6316 0 0 8 8 8 208 1208 208 5208 16 17 ISAAAA YIJAAA AAAAxx 4705 6317 1 1 5 5 5 705 705 4705 4705 10 11 ZYAAAA ZIJAAA HHHHxx 2341 6318 1 1 1 1 41 341 341 2341 2341 82 83 BMAAAA AJJAAA OOOOxx 8539 6319 1 3 9 19 39 539 539 3539 8539 78 79 LQAAAA BJJAAA VVVVxx 7528 6320 0 0 8 8 28 528 1528 2528 7528 56 57 ODAAAA CJJAAA AAAAxx 7969 6321 1 1 9 9 69 969 1969 2969 7969 138 139 NUAAAA DJJAAA HHHHxx 6381 6322 1 1 1 1 81 381 381 1381 6381 162 163 LLAAAA EJJAAA OOOOxx 4906 6323 0 2 6 6 6 906 906 4906 4906 12 13 SGAAAA FJJAAA VVVVxx 8697 6324 1 1 7 17 97 697 697 3697 8697 194 195 NWAAAA GJJAAA AAAAxx 6301 6325 1 1 1 1 1 301 301 1301 6301 2 3 JIAAAA HJJAAA HHHHxx 7554 6326 0 2 4 14 54 554 1554 2554 7554 108 109 OEAAAA IJJAAA OOOOxx 5107 6327 1 3 7 7 7 107 1107 107 5107 14 15 LOAAAA JJJAAA VVVVxx 5046 6328 0 2 6 6 46 46 1046 46 5046 92 93 CMAAAA KJJAAA AAAAxx 4063 6329 1 3 3 3 63 63 63 4063 4063 126 127 HAAAAA LJJAAA HHHHxx 7580 6330 0 0 0 0 80 580 1580 2580 7580 160 161 OFAAAA MJJAAA OOOOxx 2245 6331 1 1 5 5 45 245 245 2245 2245 90 91 JIAAAA NJJAAA VVVVxx 3711 6332 1 3 1 11 11 711 1711 3711 3711 22 23 TMAAAA OJJAAA AAAAxx 3220 6333 0 0 0 0 20 220 1220 3220 3220 40 41 WTAAAA PJJAAA HHHHxx 6463 6334 1 3 3 3 63 463 463 1463 6463 126 127 POAAAA QJJAAA OOOOxx 8196 6335 0 0 6 16 96 196 196 3196 8196 192 193 GDAAAA RJJAAA VVVVxx 9875 6336 1 3 5 15 75 875 1875 4875 9875 150 151 VPAAAA SJJAAA AAAAxx 1333 6337 1 1 3 13 33 333 1333 1333 1333 66 67 HZAAAA TJJAAA HHHHxx 7880 6338 0 0 0 0 80 880 1880 2880 7880 160 161 CRAAAA UJJAAA OOOOxx 2322 6339 0 2 2 2 22 322 322 2322 2322 44 45 ILAAAA VJJAAA VVVVxx 2163 6340 1 3 3 3 63 163 163 2163 2163 126 127 FFAAAA WJJAAA AAAAxx 421 6341 1 1 1 1 21 421 421 421 421 42 43 FQAAAA XJJAAA HHHHxx 2042 6342 0 2 2 2 42 42 42 2042 2042 84 85 OAAAAA YJJAAA OOOOxx 1424 6343 0 0 4 4 24 424 1424 1424 1424 48 49 UCAAAA ZJJAAA VVVVxx 7870 6344 0 2 0 10 70 870 1870 2870 7870 140 141 SQAAAA AKJAAA AAAAxx 2653 6345 1 1 3 13 53 653 653 2653 2653 106 107 BYAAAA BKJAAA HHHHxx 4216 6346 0 0 6 16 16 216 216 4216 4216 32 33 EGAAAA CKJAAA OOOOxx 1515 6347 1 3 5 15 15 515 1515 1515 1515 30 31 HGAAAA DKJAAA VVVVxx 7860 6348 0 0 0 0 60 860 1860 2860 7860 120 121 IQAAAA EKJAAA AAAAxx 2984 6349 0 0 4 4 84 984 984 2984 2984 168 169 UKAAAA FKJAAA HHHHxx 6269 6350 1 1 9 9 69 269 269 1269 6269 138 139 DHAAAA GKJAAA OOOOxx 2609 6351 1 1 9 9 9 609 609 2609 2609 18 19 JWAAAA HKJAAA VVVVxx 3671 6352 1 3 1 11 71 671 1671 3671 3671 142 143 FLAAAA IKJAAA AAAAxx 4544 6353 0 0 4 4 44 544 544 4544 4544 88 89 USAAAA JKJAAA HHHHxx 4668 6354 0 0 8 8 68 668 668 4668 4668 136 137 OXAAAA KKJAAA OOOOxx 2565 6355 1 1 5 5 65 565 565 2565 2565 130 131 RUAAAA LKJAAA VVVVxx 3126 6356 0 2 6 6 26 126 1126 3126 3126 52 53 GQAAAA MKJAAA AAAAxx 7573 6357 1 1 3 13 73 573 1573 2573 7573 146 147 HFAAAA NKJAAA HHHHxx 1476 6358 0 0 6 16 76 476 1476 1476 1476 152 153 UEAAAA OKJAAA OOOOxx 2146 6359 0 2 6 6 46 146 146 2146 2146 92 93 OEAAAA PKJAAA VVVVxx 9990 6360 0 2 0 10 90 990 1990 4990 9990 180 181 GUAAAA QKJAAA AAAAxx 2530 6361 0 2 0 10 30 530 530 2530 2530 60 61 ITAAAA RKJAAA HHHHxx 9288 6362 0 0 8 8 88 288 1288 4288 9288 176 177 GTAAAA SKJAAA OOOOxx 9755 6363 1 3 5 15 55 755 1755 4755 9755 110 111 FLAAAA TKJAAA VVVVxx 5305 6364 1 1 5 5 5 305 1305 305 5305 10 11 BWAAAA UKJAAA AAAAxx 2495 6365 1 3 5 15 95 495 495 2495 2495 190 191 ZRAAAA VKJAAA HHHHxx 5443 6366 1 3 3 3 43 443 1443 443 5443 86 87 JBAAAA WKJAAA OOOOxx 1930 6367 0 2 0 10 30 930 1930 1930 1930 60 61 GWAAAA XKJAAA VVVVxx 9134 6368 0 2 4 14 34 134 1134 4134 9134 68 69 INAAAA YKJAAA AAAAxx 2844 6369 0 0 4 4 44 844 844 2844 2844 88 89 KFAAAA ZKJAAA HHHHxx 896 6370 0 0 6 16 96 896 896 896 896 192 193 MIAAAA ALJAAA OOOOxx 1330 6371 0 2 0 10 30 330 1330 1330 1330 60 61 EZAAAA BLJAAA VVVVxx 8980 6372 0 0 0 0 80 980 980 3980 8980 160 161 KHAAAA CLJAAA AAAAxx 5940 6373 0 0 0 0 40 940 1940 940 5940 80 81 MUAAAA DLJAAA HHHHxx 6494 6374 0 2 4 14 94 494 494 1494 6494 188 189 UPAAAA ELJAAA OOOOxx 165 6375 1 1 5 5 65 165 165 165 165 130 131 JGAAAA FLJAAA VVVVxx 2510 6376 0 2 0 10 10 510 510 2510 2510 20 21 OSAAAA GLJAAA AAAAxx 9950 6377 0 2 0 10 50 950 1950 4950 9950 100 101 SSAAAA HLJAAA HHHHxx 3854 6378 0 2 4 14 54 854 1854 3854 3854 108 109 GSAAAA ILJAAA OOOOxx 7493 6379 1 1 3 13 93 493 1493 2493 7493 186 187 FCAAAA JLJAAA VVVVxx 4124 6380 0 0 4 4 24 124 124 4124 4124 48 49 QCAAAA KLJAAA AAAAxx 8563 6381 1 3 3 3 63 563 563 3563 8563 126 127 JRAAAA LLJAAA HHHHxx 8735 6382 1 3 5 15 35 735 735 3735 8735 70 71 ZXAAAA MLJAAA OOOOxx 9046 6383 0 2 6 6 46 46 1046 4046 9046 92 93 YJAAAA NLJAAA VVVVxx 1754 6384 0 2 4 14 54 754 1754 1754 1754 108 109 MPAAAA OLJAAA AAAAxx 6954 6385 0 2 4 14 54 954 954 1954 6954 108 109 MHAAAA PLJAAA HHHHxx 4953 6386 1 1 3 13 53 953 953 4953 4953 106 107 NIAAAA QLJAAA OOOOxx 8142 6387 0 2 2 2 42 142 142 3142 8142 84 85 EBAAAA RLJAAA VVVVxx 9661 6388 1 1 1 1 61 661 1661 4661 9661 122 123 PHAAAA SLJAAA AAAAxx 6415 6389 1 3 5 15 15 415 415 1415 6415 30 31 TMAAAA TLJAAA HHHHxx 5782 6390 0 2 2 2 82 782 1782 782 5782 164 165 KOAAAA ULJAAA OOOOxx 7721 6391 1 1 1 1 21 721 1721 2721 7721 42 43 ZKAAAA VLJAAA VVVVxx 580 6392 0 0 0 0 80 580 580 580 580 160 161 IWAAAA WLJAAA AAAAxx 3784 6393 0 0 4 4 84 784 1784 3784 3784 168 169 OPAAAA XLJAAA HHHHxx 9810 6394 0 2 0 10 10 810 1810 4810 9810 20 21 INAAAA YLJAAA OOOOxx 8488 6395 0 0 8 8 88 488 488 3488 8488 176 177 MOAAAA ZLJAAA VVVVxx 6214 6396 0 2 4 14 14 214 214 1214 6214 28 29 AFAAAA AMJAAA AAAAxx 9433 6397 1 1 3 13 33 433 1433 4433 9433 66 67 VYAAAA BMJAAA HHHHxx 9959 6398 1 3 9 19 59 959 1959 4959 9959 118 119 BTAAAA CMJAAA OOOOxx 554 6399 0 2 4 14 54 554 554 554 554 108 109 IVAAAA DMJAAA VVVVxx 6646 6400 0 2 6 6 46 646 646 1646 6646 92 93 QVAAAA EMJAAA AAAAxx 1138 6401 0 2 8 18 38 138 1138 1138 1138 76 77 URAAAA FMJAAA HHHHxx 9331 6402 1 3 1 11 31 331 1331 4331 9331 62 63 XUAAAA GMJAAA OOOOxx 7331 6403 1 3 1 11 31 331 1331 2331 7331 62 63 ZVAAAA HMJAAA VVVVxx 3482 6404 0 2 2 2 82 482 1482 3482 3482 164 165 YDAAAA IMJAAA AAAAxx 3795 6405 1 3 5 15 95 795 1795 3795 3795 190 191 ZPAAAA JMJAAA HHHHxx 2441 6406 1 1 1 1 41 441 441 2441 2441 82 83 XPAAAA KMJAAA OOOOxx 5229 6407 1 1 9 9 29 229 1229 229 5229 58 59 DTAAAA LMJAAA VVVVxx 7012 6408 0 0 2 12 12 12 1012 2012 7012 24 25 SJAAAA MMJAAA AAAAxx 7036 6409 0 0 6 16 36 36 1036 2036 7036 72 73 QKAAAA NMJAAA HHHHxx 8243 6410 1 3 3 3 43 243 243 3243 8243 86 87 BFAAAA OMJAAA OOOOxx 9320 6411 0 0 0 0 20 320 1320 4320 9320 40 41 MUAAAA PMJAAA VVVVxx 4693 6412 1 1 3 13 93 693 693 4693 4693 186 187 NYAAAA QMJAAA AAAAxx 6741 6413 1 1 1 1 41 741 741 1741 6741 82 83 HZAAAA RMJAAA HHHHxx 2997 6414 1 1 7 17 97 997 997 2997 2997 194 195 HLAAAA SMJAAA OOOOxx 4838 6415 0 2 8 18 38 838 838 4838 4838 76 77 CEAAAA TMJAAA VVVVxx 6945 6416 1 1 5 5 45 945 945 1945 6945 90 91 DHAAAA UMJAAA AAAAxx 8253 6417 1 1 3 13 53 253 253 3253 8253 106 107 LFAAAA VMJAAA HHHHxx 8989 6418 1 1 9 9 89 989 989 3989 8989 178 179 THAAAA WMJAAA OOOOxx 2640 6419 0 0 0 0 40 640 640 2640 2640 80 81 OXAAAA XMJAAA VVVVxx 5647 6420 1 3 7 7 47 647 1647 647 5647 94 95 FJAAAA YMJAAA AAAAxx 7186 6421 0 2 6 6 86 186 1186 2186 7186 172 173 KQAAAA ZMJAAA HHHHxx 3278 6422 0 2 8 18 78 278 1278 3278 3278 156 157 CWAAAA ANJAAA OOOOxx 8546 6423 0 2 6 6 46 546 546 3546 8546 92 93 SQAAAA BNJAAA VVVVxx 8297 6424 1 1 7 17 97 297 297 3297 8297 194 195 DHAAAA CNJAAA AAAAxx 9534 6425 0 2 4 14 34 534 1534 4534 9534 68 69 SCAAAA DNJAAA HHHHxx 9618 6426 0 2 8 18 18 618 1618 4618 9618 36 37 YFAAAA ENJAAA OOOOxx 8839 6427 1 3 9 19 39 839 839 3839 8839 78 79 ZBAAAA FNJAAA VVVVxx 7605 6428 1 1 5 5 5 605 1605 2605 7605 10 11 NGAAAA GNJAAA AAAAxx 6421 6429 1 1 1 1 21 421 421 1421 6421 42 43 ZMAAAA HNJAAA HHHHxx 3582 6430 0 2 2 2 82 582 1582 3582 3582 164 165 UHAAAA INJAAA OOOOxx 485 6431 1 1 5 5 85 485 485 485 485 170 171 RSAAAA JNJAAA VVVVxx 1925 6432 1 1 5 5 25 925 1925 1925 1925 50 51 BWAAAA KNJAAA AAAAxx 4296 6433 0 0 6 16 96 296 296 4296 4296 192 193 GJAAAA LNJAAA HHHHxx 8874 6434 0 2 4 14 74 874 874 3874 8874 148 149 IDAAAA MNJAAA OOOOxx 1443 6435 1 3 3 3 43 443 1443 1443 1443 86 87 NDAAAA NNJAAA VVVVxx 4239 6436 1 3 9 19 39 239 239 4239 4239 78 79 BHAAAA ONJAAA AAAAxx 9760 6437 0 0 0 0 60 760 1760 4760 9760 120 121 KLAAAA PNJAAA HHHHxx 136 6438 0 0 6 16 36 136 136 136 136 72 73 GFAAAA QNJAAA OOOOxx 6472 6439 0 0 2 12 72 472 472 1472 6472 144 145 YOAAAA RNJAAA VVVVxx 4896 6440 0 0 6 16 96 896 896 4896 4896 192 193 IGAAAA SNJAAA AAAAxx 9028 6441 0 0 8 8 28 28 1028 4028 9028 56 57 GJAAAA TNJAAA HHHHxx 8354 6442 0 2 4 14 54 354 354 3354 8354 108 109 IJAAAA UNJAAA OOOOxx 8648 6443 0 0 8 8 48 648 648 3648 8648 96 97 QUAAAA VNJAAA VVVVxx 918 6444 0 2 8 18 18 918 918 918 918 36 37 IJAAAA WNJAAA AAAAxx 6606 6445 0 2 6 6 6 606 606 1606 6606 12 13 CUAAAA XNJAAA HHHHxx 2462 6446 0 2 2 2 62 462 462 2462 2462 124 125 SQAAAA YNJAAA OOOOxx 7536 6447 0 0 6 16 36 536 1536 2536 7536 72 73 WDAAAA ZNJAAA VVVVxx 1700 6448 0 0 0 0 0 700 1700 1700 1700 0 1 KNAAAA AOJAAA AAAAxx 6740 6449 0 0 0 0 40 740 740 1740 6740 80 81 GZAAAA BOJAAA HHHHxx 28 6450 0 0 8 8 28 28 28 28 28 56 57 CBAAAA COJAAA OOOOxx 6044 6451 0 0 4 4 44 44 44 1044 6044 88 89 MYAAAA DOJAAA VVVVxx 5053 6452 1 1 3 13 53 53 1053 53 5053 106 107 JMAAAA EOJAAA AAAAxx 4832 6453 0 0 2 12 32 832 832 4832 4832 64 65 WDAAAA FOJAAA HHHHxx 9145 6454 1 1 5 5 45 145 1145 4145 9145 90 91 TNAAAA GOJAAA OOOOxx 5482 6455 0 2 2 2 82 482 1482 482 5482 164 165 WCAAAA HOJAAA VVVVxx 7644 6456 0 0 4 4 44 644 1644 2644 7644 88 89 AIAAAA IOJAAA AAAAxx 2128 6457 0 0 8 8 28 128 128 2128 2128 56 57 WDAAAA JOJAAA HHHHxx 6583 6458 1 3 3 3 83 583 583 1583 6583 166 167 FTAAAA KOJAAA OOOOxx 4224 6459 0 0 4 4 24 224 224 4224 4224 48 49 MGAAAA LOJAAA VVVVxx 5253 6460 1 1 3 13 53 253 1253 253 5253 106 107 BUAAAA MOJAAA AAAAxx 8219 6461 1 3 9 19 19 219 219 3219 8219 38 39 DEAAAA NOJAAA HHHHxx 8113 6462 1 1 3 13 13 113 113 3113 8113 26 27 BAAAAA OOJAAA OOOOxx 3616 6463 0 0 6 16 16 616 1616 3616 3616 32 33 CJAAAA POJAAA VVVVxx 1361 6464 1 1 1 1 61 361 1361 1361 1361 122 123 JAAAAA QOJAAA AAAAxx 949 6465 1 1 9 9 49 949 949 949 949 98 99 NKAAAA ROJAAA HHHHxx 8582 6466 0 2 2 2 82 582 582 3582 8582 164 165 CSAAAA SOJAAA OOOOxx 5104 6467 0 0 4 4 4 104 1104 104 5104 8 9 IOAAAA TOJAAA VVVVxx 6146 6468 0 2 6 6 46 146 146 1146 6146 92 93 KCAAAA UOJAAA AAAAxx 7681 6469 1 1 1 1 81 681 1681 2681 7681 162 163 LJAAAA VOJAAA HHHHxx 1904 6470 0 0 4 4 4 904 1904 1904 1904 8 9 GVAAAA WOJAAA OOOOxx 1989 6471 1 1 9 9 89 989 1989 1989 1989 178 179 NYAAAA XOJAAA VVVVxx 4179 6472 1 3 9 19 79 179 179 4179 4179 158 159 TEAAAA YOJAAA AAAAxx 1739 6473 1 3 9 19 39 739 1739 1739 1739 78 79 XOAAAA ZOJAAA HHHHxx 2447 6474 1 3 7 7 47 447 447 2447 2447 94 95 DQAAAA APJAAA OOOOxx 3029 6475 1 1 9 9 29 29 1029 3029 3029 58 59 NMAAAA BPJAAA VVVVxx 9783 6476 1 3 3 3 83 783 1783 4783 9783 166 167 HMAAAA CPJAAA AAAAxx 8381 6477 1 1 1 1 81 381 381 3381 8381 162 163 JKAAAA DPJAAA HHHHxx 8755 6478 1 3 5 15 55 755 755 3755 8755 110 111 TYAAAA EPJAAA OOOOxx 8384 6479 0 0 4 4 84 384 384 3384 8384 168 169 MKAAAA FPJAAA VVVVxx 7655 6480 1 3 5 15 55 655 1655 2655 7655 110 111 LIAAAA GPJAAA AAAAxx 4766 6481 0 2 6 6 66 766 766 4766 4766 132 133 IBAAAA HPJAAA HHHHxx 3324 6482 0 0 4 4 24 324 1324 3324 3324 48 49 WXAAAA IPJAAA OOOOxx 5022 6483 0 2 2 2 22 22 1022 22 5022 44 45 ELAAAA JPJAAA VVVVxx 2856 6484 0 0 6 16 56 856 856 2856 2856 112 113 WFAAAA KPJAAA AAAAxx 6503 6485 1 3 3 3 3 503 503 1503 6503 6 7 DQAAAA LPJAAA HHHHxx 6872 6486 0 0 2 12 72 872 872 1872 6872 144 145 IEAAAA MPJAAA OOOOxx 1663 6487 1 3 3 3 63 663 1663 1663 1663 126 127 ZLAAAA NPJAAA VVVVxx 6964 6488 0 0 4 4 64 964 964 1964 6964 128 129 WHAAAA OPJAAA AAAAxx 4622 6489 0 2 2 2 22 622 622 4622 4622 44 45 UVAAAA PPJAAA HHHHxx 6089 6490 1 1 9 9 89 89 89 1089 6089 178 179 FAAAAA QPJAAA OOOOxx 8567 6491 1 3 7 7 67 567 567 3567 8567 134 135 NRAAAA RPJAAA VVVVxx 597 6492 1 1 7 17 97 597 597 597 597 194 195 ZWAAAA SPJAAA AAAAxx 4222 6493 0 2 2 2 22 222 222 4222 4222 44 45 KGAAAA TPJAAA HHHHxx 9322 6494 0 2 2 2 22 322 1322 4322 9322 44 45 OUAAAA UPJAAA OOOOxx 624 6495 0 0 4 4 24 624 624 624 624 48 49 AYAAAA VPJAAA VVVVxx 4329 6496 1 1 9 9 29 329 329 4329 4329 58 59 NKAAAA WPJAAA AAAAxx 6781 6497 1 1 1 1 81 781 781 1781 6781 162 163 VAAAAA XPJAAA HHHHxx 1673 6498 1 1 3 13 73 673 1673 1673 1673 146 147 JMAAAA YPJAAA OOOOxx 6633 6499 1 1 3 13 33 633 633 1633 6633 66 67 DVAAAA ZPJAAA VVVVxx 2569 6500 1 1 9 9 69 569 569 2569 2569 138 139 VUAAAA AQJAAA AAAAxx 4995 6501 1 3 5 15 95 995 995 4995 4995 190 191 DKAAAA BQJAAA HHHHxx 2749 6502 1 1 9 9 49 749 749 2749 2749 98 99 TBAAAA CQJAAA OOOOxx 9044 6503 0 0 4 4 44 44 1044 4044 9044 88 89 WJAAAA DQJAAA VVVVxx 5823 6504 1 3 3 3 23 823 1823 823 5823 46 47 ZPAAAA EQJAAA AAAAxx 9366 6505 0 2 6 6 66 366 1366 4366 9366 132 133 GWAAAA FQJAAA HHHHxx 1169 6506 1 1 9 9 69 169 1169 1169 1169 138 139 ZSAAAA GQJAAA OOOOxx 1300 6507 0 0 0 0 0 300 1300 1300 1300 0 1 AYAAAA HQJAAA VVVVxx 9973 6508 1 1 3 13 73 973 1973 4973 9973 146 147 PTAAAA IQJAAA AAAAxx 2092 6509 0 0 2 12 92 92 92 2092 2092 184 185 MCAAAA JQJAAA HHHHxx 9776 6510 0 0 6 16 76 776 1776 4776 9776 152 153 AMAAAA KQJAAA OOOOxx 7612 6511 0 0 2 12 12 612 1612 2612 7612 24 25 UGAAAA LQJAAA VVVVxx 7190 6512 0 2 0 10 90 190 1190 2190 7190 180 181 OQAAAA MQJAAA AAAAxx 5147 6513 1 3 7 7 47 147 1147 147 5147 94 95 ZPAAAA NQJAAA HHHHxx 3722 6514 0 2 2 2 22 722 1722 3722 3722 44 45 ENAAAA OQJAAA OOOOxx 5858 6515 0 2 8 18 58 858 1858 858 5858 116 117 IRAAAA PQJAAA VVVVxx 3204 6516 0 0 4 4 4 204 1204 3204 3204 8 9 GTAAAA QQJAAA AAAAxx 8994 6517 0 2 4 14 94 994 994 3994 8994 188 189 YHAAAA RQJAAA HHHHxx 7478 6518 0 2 8 18 78 478 1478 2478 7478 156 157 QBAAAA SQJAAA OOOOxx 9624 6519 0 0 4 4 24 624 1624 4624 9624 48 49 EGAAAA TQJAAA VVVVxx 6639 6520 1 3 9 19 39 639 639 1639 6639 78 79 JVAAAA UQJAAA AAAAxx 369 6521 1 1 9 9 69 369 369 369 369 138 139 FOAAAA VQJAAA HHHHxx 7766 6522 0 2 6 6 66 766 1766 2766 7766 132 133 SMAAAA WQJAAA OOOOxx 4094 6523 0 2 4 14 94 94 94 4094 4094 188 189 MBAAAA XQJAAA VVVVxx 9556 6524 0 0 6 16 56 556 1556 4556 9556 112 113 ODAAAA YQJAAA AAAAxx 4887 6525 1 3 7 7 87 887 887 4887 4887 174 175 ZFAAAA ZQJAAA HHHHxx 2321 6526 1 1 1 1 21 321 321 2321 2321 42 43 HLAAAA ARJAAA OOOOxx 9201 6527 1 1 1 1 1 201 1201 4201 9201 2 3 XPAAAA BRJAAA VVVVxx 1627 6528 1 3 7 7 27 627 1627 1627 1627 54 55 PKAAAA CRJAAA AAAAxx 150 6529 0 2 0 10 50 150 150 150 150 100 101 UFAAAA DRJAAA HHHHxx 8010 6530 0 2 0 10 10 10 10 3010 8010 20 21 CWAAAA ERJAAA OOOOxx 8026 6531 0 2 6 6 26 26 26 3026 8026 52 53 SWAAAA FRJAAA VVVVxx 5495 6532 1 3 5 15 95 495 1495 495 5495 190 191 JDAAAA GRJAAA AAAAxx 6213 6533 1 1 3 13 13 213 213 1213 6213 26 27 ZEAAAA HRJAAA HHHHxx 6464 6534 0 0 4 4 64 464 464 1464 6464 128 129 QOAAAA IRJAAA OOOOxx 1158 6535 0 2 8 18 58 158 1158 1158 1158 116 117 OSAAAA JRJAAA VVVVxx 8669 6536 1 1 9 9 69 669 669 3669 8669 138 139 LVAAAA KRJAAA AAAAxx 3225 6537 1 1 5 5 25 225 1225 3225 3225 50 51 BUAAAA LRJAAA HHHHxx 1294 6538 0 2 4 14 94 294 1294 1294 1294 188 189 UXAAAA MRJAAA OOOOxx 2166 6539 0 2 6 6 66 166 166 2166 2166 132 133 IFAAAA NRJAAA VVVVxx 9328 6540 0 0 8 8 28 328 1328 4328 9328 56 57 UUAAAA ORJAAA AAAAxx 8431 6541 1 3 1 11 31 431 431 3431 8431 62 63 HMAAAA PRJAAA HHHHxx 7100 6542 0 0 0 0 0 100 1100 2100 7100 0 1 CNAAAA QRJAAA OOOOxx 8126 6543 0 2 6 6 26 126 126 3126 8126 52 53 OAAAAA RRJAAA VVVVxx 2185 6544 1 1 5 5 85 185 185 2185 2185 170 171 BGAAAA SRJAAA AAAAxx 5697 6545 1 1 7 17 97 697 1697 697 5697 194 195 DLAAAA TRJAAA HHHHxx 5531 6546 1 3 1 11 31 531 1531 531 5531 62 63 TEAAAA URJAAA OOOOxx 3020 6547 0 0 0 0 20 20 1020 3020 3020 40 41 EMAAAA VRJAAA VVVVxx 3076 6548 0 0 6 16 76 76 1076 3076 3076 152 153 IOAAAA WRJAAA AAAAxx 9228 6549 0 0 8 8 28 228 1228 4228 9228 56 57 YQAAAA XRJAAA HHHHxx 1734 6550 0 2 4 14 34 734 1734 1734 1734 68 69 SOAAAA YRJAAA OOOOxx 7616 6551 0 0 6 16 16 616 1616 2616 7616 32 33 YGAAAA ZRJAAA VVVVxx 9059 6552 1 3 9 19 59 59 1059 4059 9059 118 119 LKAAAA ASJAAA AAAAxx 323 6553 1 3 3 3 23 323 323 323 323 46 47 LMAAAA BSJAAA HHHHxx 1283 6554 1 3 3 3 83 283 1283 1283 1283 166 167 JXAAAA CSJAAA OOOOxx 9535 6555 1 3 5 15 35 535 1535 4535 9535 70 71 TCAAAA DSJAAA VVVVxx 2580 6556 0 0 0 0 80 580 580 2580 2580 160 161 GVAAAA ESJAAA AAAAxx 7633 6557 1 1 3 13 33 633 1633 2633 7633 66 67 PHAAAA FSJAAA HHHHxx 9497 6558 1 1 7 17 97 497 1497 4497 9497 194 195 HBAAAA GSJAAA OOOOxx 9842 6559 0 2 2 2 42 842 1842 4842 9842 84 85 OOAAAA HSJAAA VVVVxx 3426 6560 0 2 6 6 26 426 1426 3426 3426 52 53 UBAAAA ISJAAA AAAAxx 7650 6561 0 2 0 10 50 650 1650 2650 7650 100 101 GIAAAA JSJAAA HHHHxx 9935 6562 1 3 5 15 35 935 1935 4935 9935 70 71 DSAAAA KSJAAA OOOOxx 9354 6563 0 2 4 14 54 354 1354 4354 9354 108 109 UVAAAA LSJAAA VVVVxx 5569 6564 1 1 9 9 69 569 1569 569 5569 138 139 FGAAAA MSJAAA AAAAxx 5765 6565 1 1 5 5 65 765 1765 765 5765 130 131 TNAAAA NSJAAA HHHHxx 7283 6566 1 3 3 3 83 283 1283 2283 7283 166 167 DUAAAA OSJAAA OOOOxx 1068 6567 0 0 8 8 68 68 1068 1068 1068 136 137 CPAAAA PSJAAA VVVVxx 1641 6568 1 1 1 1 41 641 1641 1641 1641 82 83 DLAAAA QSJAAA AAAAxx 1688 6569 0 0 8 8 88 688 1688 1688 1688 176 177 YMAAAA RSJAAA HHHHxx 1133 6570 1 1 3 13 33 133 1133 1133 1133 66 67 PRAAAA SSJAAA OOOOxx 4493 6571 1 1 3 13 93 493 493 4493 4493 186 187 VQAAAA TSJAAA VVVVxx 3354 6572 0 2 4 14 54 354 1354 3354 3354 108 109 AZAAAA USJAAA AAAAxx 4029 6573 1 1 9 9 29 29 29 4029 4029 58 59 ZYAAAA VSJAAA HHHHxx 6704 6574 0 0 4 4 4 704 704 1704 6704 8 9 WXAAAA WSJAAA OOOOxx 3221 6575 1 1 1 1 21 221 1221 3221 3221 42 43 XTAAAA XSJAAA VVVVxx 9432 6576 0 0 2 12 32 432 1432 4432 9432 64 65 UYAAAA YSJAAA AAAAxx 6990 6577 0 2 0 10 90 990 990 1990 6990 180 181 WIAAAA ZSJAAA HHHHxx 1760 6578 0 0 0 0 60 760 1760 1760 1760 120 121 SPAAAA ATJAAA OOOOxx 4754 6579 0 2 4 14 54 754 754 4754 4754 108 109 WAAAAA BTJAAA VVVVxx 7724 6580 0 0 4 4 24 724 1724 2724 7724 48 49 CLAAAA CTJAAA AAAAxx 9487 6581 1 3 7 7 87 487 1487 4487 9487 174 175 XAAAAA DTJAAA HHHHxx 166 6582 0 2 6 6 66 166 166 166 166 132 133 KGAAAA ETJAAA OOOOxx 5479 6583 1 3 9 19 79 479 1479 479 5479 158 159 TCAAAA FTJAAA VVVVxx 8744 6584 0 0 4 4 44 744 744 3744 8744 88 89 IYAAAA GTJAAA AAAAxx 5746 6585 0 2 6 6 46 746 1746 746 5746 92 93 ANAAAA HTJAAA HHHHxx 907 6586 1 3 7 7 7 907 907 907 907 14 15 XIAAAA ITJAAA OOOOxx 3968 6587 0 0 8 8 68 968 1968 3968 3968 136 137 QWAAAA JTJAAA VVVVxx 5721 6588 1 1 1 1 21 721 1721 721 5721 42 43 BMAAAA KTJAAA AAAAxx 6738 6589 0 2 8 18 38 738 738 1738 6738 76 77 EZAAAA LTJAAA HHHHxx 4097 6590 1 1 7 17 97 97 97 4097 4097 194 195 PBAAAA MTJAAA OOOOxx 8456 6591 0 0 6 16 56 456 456 3456 8456 112 113 GNAAAA NTJAAA VVVVxx 1269 6592 1 1 9 9 69 269 1269 1269 1269 138 139 VWAAAA OTJAAA AAAAxx 7997 6593 1 1 7 17 97 997 1997 2997 7997 194 195 PVAAAA PTJAAA HHHHxx 9457 6594 1 1 7 17 57 457 1457 4457 9457 114 115 TZAAAA QTJAAA OOOOxx 1159 6595 1 3 9 19 59 159 1159 1159 1159 118 119 PSAAAA RTJAAA VVVVxx 1631 6596 1 3 1 11 31 631 1631 1631 1631 62 63 TKAAAA STJAAA AAAAxx 2019 6597 1 3 9 19 19 19 19 2019 2019 38 39 RZAAAA TTJAAA HHHHxx 3186 6598 0 2 6 6 86 186 1186 3186 3186 172 173 OSAAAA UTJAAA OOOOxx 5587 6599 1 3 7 7 87 587 1587 587 5587 174 175 XGAAAA VTJAAA VVVVxx 9172 6600 0 0 2 12 72 172 1172 4172 9172 144 145 UOAAAA WTJAAA AAAAxx 5589 6601 1 1 9 9 89 589 1589 589 5589 178 179 ZGAAAA XTJAAA HHHHxx 5103 6602 1 3 3 3 3 103 1103 103 5103 6 7 HOAAAA YTJAAA OOOOxx 3177 6603 1 1 7 17 77 177 1177 3177 3177 154 155 FSAAAA ZTJAAA VVVVxx 8887 6604 1 3 7 7 87 887 887 3887 8887 174 175 VDAAAA AUJAAA AAAAxx 12 6605 0 0 2 12 12 12 12 12 12 24 25 MAAAAA BUJAAA HHHHxx 8575 6606 1 3 5 15 75 575 575 3575 8575 150 151 VRAAAA CUJAAA OOOOxx 4335 6607 1 3 5 15 35 335 335 4335 4335 70 71 TKAAAA DUJAAA VVVVxx 4581 6608 1 1 1 1 81 581 581 4581 4581 162 163 FUAAAA EUJAAA AAAAxx 4444 6609 0 0 4 4 44 444 444 4444 4444 88 89 YOAAAA FUJAAA HHHHxx 7978 6610 0 2 8 18 78 978 1978 2978 7978 156 157 WUAAAA GUJAAA OOOOxx 3081 6611 1 1 1 1 81 81 1081 3081 3081 162 163 NOAAAA HUJAAA VVVVxx 4059 6612 1 3 9 19 59 59 59 4059 4059 118 119 DAAAAA IUJAAA AAAAxx 5711 6613 1 3 1 11 11 711 1711 711 5711 22 23 RLAAAA JUJAAA HHHHxx 7069 6614 1 1 9 9 69 69 1069 2069 7069 138 139 XLAAAA KUJAAA OOOOxx 6150 6615 0 2 0 10 50 150 150 1150 6150 100 101 OCAAAA LUJAAA VVVVxx 9550 6616 0 2 0 10 50 550 1550 4550 9550 100 101 IDAAAA MUJAAA AAAAxx 7087 6617 1 3 7 7 87 87 1087 2087 7087 174 175 PMAAAA NUJAAA HHHHxx 9557 6618 1 1 7 17 57 557 1557 4557 9557 114 115 PDAAAA OUJAAA OOOOxx 7856 6619 0 0 6 16 56 856 1856 2856 7856 112 113 EQAAAA PUJAAA VVVVxx 1115 6620 1 3 5 15 15 115 1115 1115 1115 30 31 XQAAAA QUJAAA AAAAxx 1086 6621 0 2 6 6 86 86 1086 1086 1086 172 173 UPAAAA RUJAAA HHHHxx 5048 6622 0 0 8 8 48 48 1048 48 5048 96 97 EMAAAA SUJAAA OOOOxx 5168 6623 0 0 8 8 68 168 1168 168 5168 136 137 UQAAAA TUJAAA VVVVxx 6029 6624 1 1 9 9 29 29 29 1029 6029 58 59 XXAAAA UUJAAA AAAAxx 546 6625 0 2 6 6 46 546 546 546 546 92 93 AVAAAA VUJAAA HHHHxx 2908 6626 0 0 8 8 8 908 908 2908 2908 16 17 WHAAAA WUJAAA OOOOxx 779 6627 1 3 9 19 79 779 779 779 779 158 159 ZDAAAA XUJAAA VVVVxx 4202 6628 0 2 2 2 2 202 202 4202 4202 4 5 QFAAAA YUJAAA AAAAxx 9984 6629 0 0 4 4 84 984 1984 4984 9984 168 169 AUAAAA ZUJAAA HHHHxx 4730 6630 0 2 0 10 30 730 730 4730 4730 60 61 YZAAAA AVJAAA OOOOxx 6517 6631 1 1 7 17 17 517 517 1517 6517 34 35 RQAAAA BVJAAA VVVVxx 8410 6632 0 2 0 10 10 410 410 3410 8410 20 21 MLAAAA CVJAAA AAAAxx 4793 6633 1 1 3 13 93 793 793 4793 4793 186 187 JCAAAA DVJAAA HHHHxx 3431 6634 1 3 1 11 31 431 1431 3431 3431 62 63 ZBAAAA EVJAAA OOOOxx 2481 6635 1 1 1 1 81 481 481 2481 2481 162 163 LRAAAA FVJAAA VVVVxx 3905 6636 1 1 5 5 5 905 1905 3905 3905 10 11 FUAAAA GVJAAA AAAAxx 8807 6637 1 3 7 7 7 807 807 3807 8807 14 15 TAAAAA HVJAAA HHHHxx 2660 6638 0 0 0 0 60 660 660 2660 2660 120 121 IYAAAA IVJAAA OOOOxx 4985 6639 1 1 5 5 85 985 985 4985 4985 170 171 TJAAAA JVJAAA VVVVxx 3080 6640 0 0 0 0 80 80 1080 3080 3080 160 161 MOAAAA KVJAAA AAAAxx 1090 6641 0 2 0 10 90 90 1090 1090 1090 180 181 YPAAAA LVJAAA HHHHxx 6917 6642 1 1 7 17 17 917 917 1917 6917 34 35 BGAAAA MVJAAA OOOOxx 5177 6643 1 1 7 17 77 177 1177 177 5177 154 155 DRAAAA NVJAAA VVVVxx 2729 6644 1 1 9 9 29 729 729 2729 2729 58 59 ZAAAAA OVJAAA AAAAxx 9706 6645 0 2 6 6 6 706 1706 4706 9706 12 13 IJAAAA PVJAAA HHHHxx 9929 6646 1 1 9 9 29 929 1929 4929 9929 58 59 XRAAAA QVJAAA OOOOxx 1547 6647 1 3 7 7 47 547 1547 1547 1547 94 95 NHAAAA RVJAAA VVVVxx 2798 6648 0 2 8 18 98 798 798 2798 2798 196 197 QDAAAA SVJAAA AAAAxx 4420 6649 0 0 0 0 20 420 420 4420 4420 40 41 AOAAAA TVJAAA HHHHxx 6771 6650 1 3 1 11 71 771 771 1771 6771 142 143 LAAAAA UVJAAA OOOOxx 2004 6651 0 0 4 4 4 4 4 2004 2004 8 9 CZAAAA VVJAAA VVVVxx 8686 6652 0 2 6 6 86 686 686 3686 8686 172 173 CWAAAA WVJAAA AAAAxx 3663 6653 1 3 3 3 63 663 1663 3663 3663 126 127 XKAAAA XVJAAA HHHHxx 806 6654 0 2 6 6 6 806 806 806 806 12 13 AFAAAA YVJAAA OOOOxx 4309 6655 1 1 9 9 9 309 309 4309 4309 18 19 TJAAAA ZVJAAA VVVVxx 7443 6656 1 3 3 3 43 443 1443 2443 7443 86 87 HAAAAA AWJAAA AAAAxx 5779 6657 1 3 9 19 79 779 1779 779 5779 158 159 HOAAAA BWJAAA HHHHxx 8821 6658 1 1 1 1 21 821 821 3821 8821 42 43 HBAAAA CWJAAA OOOOxx 4198 6659 0 2 8 18 98 198 198 4198 4198 196 197 MFAAAA DWJAAA VVVVxx 8115 6660 1 3 5 15 15 115 115 3115 8115 30 31 DAAAAA EWJAAA AAAAxx 9554 6661 0 2 4 14 54 554 1554 4554 9554 108 109 MDAAAA FWJAAA HHHHxx 8956 6662 0 0 6 16 56 956 956 3956 8956 112 113 MGAAAA GWJAAA OOOOxx 4733 6663 1 1 3 13 33 733 733 4733 4733 66 67 BAAAAA HWJAAA VVVVxx 5417 6664 1 1 7 17 17 417 1417 417 5417 34 35 JAAAAA IWJAAA AAAAxx 4792 6665 0 0 2 12 92 792 792 4792 4792 184 185 ICAAAA JWJAAA HHHHxx 462 6666 0 2 2 2 62 462 462 462 462 124 125 URAAAA KWJAAA OOOOxx 3687 6667 1 3 7 7 87 687 1687 3687 3687 174 175 VLAAAA LWJAAA VVVVxx 2013 6668 1 1 3 13 13 13 13 2013 2013 26 27 LZAAAA MWJAAA AAAAxx 5386 6669 0 2 6 6 86 386 1386 386 5386 172 173 EZAAAA NWJAAA HHHHxx 2816 6670 0 0 6 16 16 816 816 2816 2816 32 33 IEAAAA OWJAAA OOOOxx 7827 6671 1 3 7 7 27 827 1827 2827 7827 54 55 BPAAAA PWJAAA VVVVxx 5077 6672 1 1 7 17 77 77 1077 77 5077 154 155 HNAAAA QWJAAA AAAAxx 6039 6673 1 3 9 19 39 39 39 1039 6039 78 79 HYAAAA RWJAAA HHHHxx 215 6674 1 3 5 15 15 215 215 215 215 30 31 HIAAAA SWJAAA OOOOxx 855 6675 1 3 5 15 55 855 855 855 855 110 111 XGAAAA TWJAAA VVVVxx 9692 6676 0 0 2 12 92 692 1692 4692 9692 184 185 UIAAAA UWJAAA AAAAxx 8391 6677 1 3 1 11 91 391 391 3391 8391 182 183 TKAAAA VWJAAA HHHHxx 8424 6678 0 0 4 4 24 424 424 3424 8424 48 49 AMAAAA WWJAAA OOOOxx 6331 6679 1 3 1 11 31 331 331 1331 6331 62 63 NJAAAA XWJAAA VVVVxx 6561 6680 1 1 1 1 61 561 561 1561 6561 122 123 JSAAAA YWJAAA AAAAxx 8955 6681 1 3 5 15 55 955 955 3955 8955 110 111 LGAAAA ZWJAAA HHHHxx 1764 6682 0 0 4 4 64 764 1764 1764 1764 128 129 WPAAAA AXJAAA OOOOxx 6623 6683 1 3 3 3 23 623 623 1623 6623 46 47 TUAAAA BXJAAA VVVVxx 2900 6684 0 0 0 0 0 900 900 2900 2900 0 1 OHAAAA CXJAAA AAAAxx 7048 6685 0 0 8 8 48 48 1048 2048 7048 96 97 CLAAAA DXJAAA HHHHxx 3843 6686 1 3 3 3 43 843 1843 3843 3843 86 87 VRAAAA EXJAAA OOOOxx 4855 6687 1 3 5 15 55 855 855 4855 4855 110 111 TEAAAA FXJAAA VVVVxx 7383 6688 1 3 3 3 83 383 1383 2383 7383 166 167 ZXAAAA GXJAAA AAAAxx 7765 6689 1 1 5 5 65 765 1765 2765 7765 130 131 RMAAAA HXJAAA HHHHxx 1125 6690 1 1 5 5 25 125 1125 1125 1125 50 51 HRAAAA IXJAAA OOOOxx 755 6691 1 3 5 15 55 755 755 755 755 110 111 BDAAAA JXJAAA VVVVxx 2995 6692 1 3 5 15 95 995 995 2995 2995 190 191 FLAAAA KXJAAA AAAAxx 8907 6693 1 3 7 7 7 907 907 3907 8907 14 15 PEAAAA LXJAAA HHHHxx 9357 6694 1 1 7 17 57 357 1357 4357 9357 114 115 XVAAAA MXJAAA OOOOxx 4469 6695 1 1 9 9 69 469 469 4469 4469 138 139 XPAAAA NXJAAA VVVVxx 2147 6696 1 3 7 7 47 147 147 2147 2147 94 95 PEAAAA OXJAAA AAAAxx 2952 6697 0 0 2 12 52 952 952 2952 2952 104 105 OJAAAA PXJAAA HHHHxx 1324 6698 0 0 4 4 24 324 1324 1324 1324 48 49 YYAAAA QXJAAA OOOOxx 1173 6699 1 1 3 13 73 173 1173 1173 1173 146 147 DTAAAA RXJAAA VVVVxx 3169 6700 1 1 9 9 69 169 1169 3169 3169 138 139 XRAAAA SXJAAA AAAAxx 5149 6701 1 1 9 9 49 149 1149 149 5149 98 99 BQAAAA TXJAAA HHHHxx 9660 6702 0 0 0 0 60 660 1660 4660 9660 120 121 OHAAAA UXJAAA OOOOxx 3446 6703 0 2 6 6 46 446 1446 3446 3446 92 93 OCAAAA VXJAAA VVVVxx 6988 6704 0 0 8 8 88 988 988 1988 6988 176 177 UIAAAA WXJAAA AAAAxx 5829 6705 1 1 9 9 29 829 1829 829 5829 58 59 FQAAAA XXJAAA HHHHxx 7166 6706 0 2 6 6 66 166 1166 2166 7166 132 133 QPAAAA YXJAAA OOOOxx 3940 6707 0 0 0 0 40 940 1940 3940 3940 80 81 OVAAAA ZXJAAA VVVVxx 2645 6708 1 1 5 5 45 645 645 2645 2645 90 91 TXAAAA AYJAAA AAAAxx 478 6709 0 2 8 18 78 478 478 478 478 156 157 KSAAAA BYJAAA HHHHxx 1156 6710 0 0 6 16 56 156 1156 1156 1156 112 113 MSAAAA CYJAAA OOOOxx 2731 6711 1 3 1 11 31 731 731 2731 2731 62 63 BBAAAA DYJAAA VVVVxx 5637 6712 1 1 7 17 37 637 1637 637 5637 74 75 VIAAAA EYJAAA AAAAxx 7517 6713 1 1 7 17 17 517 1517 2517 7517 34 35 DDAAAA FYJAAA HHHHxx 5331 6714 1 3 1 11 31 331 1331 331 5331 62 63 BXAAAA GYJAAA OOOOxx 9640 6715 0 0 0 0 40 640 1640 4640 9640 80 81 UGAAAA HYJAAA VVVVxx 4108 6716 0 0 8 8 8 108 108 4108 4108 16 17 ACAAAA IYJAAA AAAAxx 1087 6717 1 3 7 7 87 87 1087 1087 1087 174 175 VPAAAA JYJAAA HHHHxx 8017 6718 1 1 7 17 17 17 17 3017 8017 34 35 JWAAAA KYJAAA OOOOxx 8795 6719 1 3 5 15 95 795 795 3795 8795 190 191 HAAAAA LYJAAA VVVVxx 7060 6720 0 0 0 0 60 60 1060 2060 7060 120 121 OLAAAA MYJAAA AAAAxx 9450 6721 0 2 0 10 50 450 1450 4450 9450 100 101 MZAAAA NYJAAA HHHHxx 390 6722 0 2 0 10 90 390 390 390 390 180 181 APAAAA OYJAAA OOOOxx 66 6723 0 2 6 6 66 66 66 66 66 132 133 OCAAAA PYJAAA VVVVxx 8789 6724 1 1 9 9 89 789 789 3789 8789 178 179 BAAAAA QYJAAA AAAAxx 9260 6725 0 0 0 0 60 260 1260 4260 9260 120 121 ESAAAA RYJAAA HHHHxx 6679 6726 1 3 9 19 79 679 679 1679 6679 158 159 XWAAAA SYJAAA OOOOxx 9052 6727 0 0 2 12 52 52 1052 4052 9052 104 105 EKAAAA TYJAAA VVVVxx 9561 6728 1 1 1 1 61 561 1561 4561 9561 122 123 TDAAAA UYJAAA AAAAxx 9725 6729 1 1 5 5 25 725 1725 4725 9725 50 51 BKAAAA VYJAAA HHHHxx 6298 6730 0 2 8 18 98 298 298 1298 6298 196 197 GIAAAA WYJAAA OOOOxx 8654 6731 0 2 4 14 54 654 654 3654 8654 108 109 WUAAAA XYJAAA VVVVxx 8725 6732 1 1 5 5 25 725 725 3725 8725 50 51 PXAAAA YYJAAA AAAAxx 9377 6733 1 1 7 17 77 377 1377 4377 9377 154 155 RWAAAA ZYJAAA HHHHxx 3807 6734 1 3 7 7 7 807 1807 3807 3807 14 15 LQAAAA AZJAAA OOOOxx 8048 6735 0 0 8 8 48 48 48 3048 8048 96 97 OXAAAA BZJAAA VVVVxx 764 6736 0 0 4 4 64 764 764 764 764 128 129 KDAAAA CZJAAA AAAAxx 9702 6737 0 2 2 2 2 702 1702 4702 9702 4 5 EJAAAA DZJAAA HHHHxx 8060 6738 0 0 0 0 60 60 60 3060 8060 120 121 AYAAAA EZJAAA OOOOxx 6371 6739 1 3 1 11 71 371 371 1371 6371 142 143 BLAAAA FZJAAA VVVVxx 5237 6740 1 1 7 17 37 237 1237 237 5237 74 75 LTAAAA GZJAAA AAAAxx 743 6741 1 3 3 3 43 743 743 743 743 86 87 PCAAAA HZJAAA HHHHxx 7395 6742 1 3 5 15 95 395 1395 2395 7395 190 191 LYAAAA IZJAAA OOOOxx 3365 6743 1 1 5 5 65 365 1365 3365 3365 130 131 LZAAAA JZJAAA VVVVxx 6667 6744 1 3 7 7 67 667 667 1667 6667 134 135 LWAAAA KZJAAA AAAAxx 3445 6745 1 1 5 5 45 445 1445 3445 3445 90 91 NCAAAA LZJAAA HHHHxx 4019 6746 1 3 9 19 19 19 19 4019 4019 38 39 PYAAAA MZJAAA OOOOxx 7035 6747 1 3 5 15 35 35 1035 2035 7035 70 71 PKAAAA NZJAAA VVVVxx 5274 6748 0 2 4 14 74 274 1274 274 5274 148 149 WUAAAA OZJAAA AAAAxx 519 6749 1 3 9 19 19 519 519 519 519 38 39 ZTAAAA PZJAAA HHHHxx 2801 6750 1 1 1 1 1 801 801 2801 2801 2 3 TDAAAA QZJAAA OOOOxx 3320 6751 0 0 0 0 20 320 1320 3320 3320 40 41 SXAAAA RZJAAA VVVVxx 3153 6752 1 1 3 13 53 153 1153 3153 3153 106 107 HRAAAA SZJAAA AAAAxx 7680 6753 0 0 0 0 80 680 1680 2680 7680 160 161 KJAAAA TZJAAA HHHHxx 8942 6754 0 2 2 2 42 942 942 3942 8942 84 85 YFAAAA UZJAAA OOOOxx 3195 6755 1 3 5 15 95 195 1195 3195 3195 190 191 XSAAAA VZJAAA VVVVxx 2287 6756 1 3 7 7 87 287 287 2287 2287 174 175 ZJAAAA WZJAAA AAAAxx 8325 6757 1 1 5 5 25 325 325 3325 8325 50 51 FIAAAA XZJAAA HHHHxx 2603 6758 1 3 3 3 3 603 603 2603 2603 6 7 DWAAAA YZJAAA OOOOxx 5871 6759 1 3 1 11 71 871 1871 871 5871 142 143 VRAAAA ZZJAAA VVVVxx 1773 6760 1 1 3 13 73 773 1773 1773 1773 146 147 FQAAAA AAKAAA AAAAxx 3323 6761 1 3 3 3 23 323 1323 3323 3323 46 47 VXAAAA BAKAAA HHHHxx 2053 6762 1 1 3 13 53 53 53 2053 2053 106 107 ZAAAAA CAKAAA OOOOxx 4062 6763 0 2 2 2 62 62 62 4062 4062 124 125 GAAAAA DAKAAA VVVVxx 4611 6764 1 3 1 11 11 611 611 4611 4611 22 23 JVAAAA EAKAAA AAAAxx 3451 6765 1 3 1 11 51 451 1451 3451 3451 102 103 TCAAAA FAKAAA HHHHxx 1819 6766 1 3 9 19 19 819 1819 1819 1819 38 39 ZRAAAA GAKAAA OOOOxx 9806 6767 0 2 6 6 6 806 1806 4806 9806 12 13 ENAAAA HAKAAA VVVVxx 6619 6768 1 3 9 19 19 619 619 1619 6619 38 39 PUAAAA IAKAAA AAAAxx 1031 6769 1 3 1 11 31 31 1031 1031 1031 62 63 RNAAAA JAKAAA HHHHxx 1865 6770 1 1 5 5 65 865 1865 1865 1865 130 131 TTAAAA KAKAAA OOOOxx 6282 6771 0 2 2 2 82 282 282 1282 6282 164 165 QHAAAA LAKAAA VVVVxx 1178 6772 0 2 8 18 78 178 1178 1178 1178 156 157 ITAAAA MAKAAA AAAAxx 8007 6773 1 3 7 7 7 7 7 3007 8007 14 15 ZVAAAA NAKAAA HHHHxx 9126 6774 0 2 6 6 26 126 1126 4126 9126 52 53 ANAAAA OAKAAA OOOOxx 9113 6775 1 1 3 13 13 113 1113 4113 9113 26 27 NMAAAA PAKAAA VVVVxx 537 6776 1 1 7 17 37 537 537 537 537 74 75 RUAAAA QAKAAA AAAAxx 6208 6777 0 0 8 8 8 208 208 1208 6208 16 17 UEAAAA RAKAAA HHHHxx 1626 6778 0 2 6 6 26 626 1626 1626 1626 52 53 OKAAAA SAKAAA OOOOxx 7188 6779 0 0 8 8 88 188 1188 2188 7188 176 177 MQAAAA TAKAAA VVVVxx 9216 6780 0 0 6 16 16 216 1216 4216 9216 32 33 MQAAAA UAKAAA AAAAxx 6134 6781 0 2 4 14 34 134 134 1134 6134 68 69 YBAAAA VAKAAA HHHHxx 2074 6782 0 2 4 14 74 74 74 2074 2074 148 149 UBAAAA WAKAAA OOOOxx 6369 6783 1 1 9 9 69 369 369 1369 6369 138 139 ZKAAAA XAKAAA VVVVxx 9306 6784 0 2 6 6 6 306 1306 4306 9306 12 13 YTAAAA YAKAAA AAAAxx 3155 6785 1 3 5 15 55 155 1155 3155 3155 110 111 JRAAAA ZAKAAA HHHHxx 3611 6786 1 3 1 11 11 611 1611 3611 3611 22 23 XIAAAA ABKAAA OOOOxx 6530 6787 0 2 0 10 30 530 530 1530 6530 60 61 ERAAAA BBKAAA VVVVxx 6979 6788 1 3 9 19 79 979 979 1979 6979 158 159 LIAAAA CBKAAA AAAAxx 9129 6789 1 1 9 9 29 129 1129 4129 9129 58 59 DNAAAA DBKAAA HHHHxx 8013 6790 1 1 3 13 13 13 13 3013 8013 26 27 FWAAAA EBKAAA OOOOxx 6926 6791 0 2 6 6 26 926 926 1926 6926 52 53 KGAAAA FBKAAA VVVVxx 1877 6792 1 1 7 17 77 877 1877 1877 1877 154 155 FUAAAA GBKAAA AAAAxx 1882 6793 0 2 2 2 82 882 1882 1882 1882 164 165 KUAAAA HBKAAA HHHHxx 6720 6794 0 0 0 0 20 720 720 1720 6720 40 41 MYAAAA IBKAAA OOOOxx 690 6795 0 2 0 10 90 690 690 690 690 180 181 OAAAAA JBKAAA VVVVxx 143 6796 1 3 3 3 43 143 143 143 143 86 87 NFAAAA KBKAAA AAAAxx 7241 6797 1 1 1 1 41 241 1241 2241 7241 82 83 NSAAAA LBKAAA HHHHxx 6461 6798 1 1 1 1 61 461 461 1461 6461 122 123 NOAAAA MBKAAA OOOOxx 2258 6799 0 2 8 18 58 258 258 2258 2258 116 117 WIAAAA NBKAAA VVVVxx 2280 6800 0 0 0 0 80 280 280 2280 2280 160 161 SJAAAA OBKAAA AAAAxx 7556 6801 0 0 6 16 56 556 1556 2556 7556 112 113 QEAAAA PBKAAA HHHHxx 1038 6802 0 2 8 18 38 38 1038 1038 1038 76 77 YNAAAA QBKAAA OOOOxx 2634 6803 0 2 4 14 34 634 634 2634 2634 68 69 IXAAAA RBKAAA VVVVxx 7847 6804 1 3 7 7 47 847 1847 2847 7847 94 95 VPAAAA SBKAAA AAAAxx 4415 6805 1 3 5 15 15 415 415 4415 4415 30 31 VNAAAA TBKAAA HHHHxx 1933 6806 1 1 3 13 33 933 1933 1933 1933 66 67 JWAAAA UBKAAA OOOOxx 8034 6807 0 2 4 14 34 34 34 3034 8034 68 69 AXAAAA VBKAAA VVVVxx 9233 6808 1 1 3 13 33 233 1233 4233 9233 66 67 DRAAAA WBKAAA AAAAxx 6572 6809 0 0 2 12 72 572 572 1572 6572 144 145 USAAAA XBKAAA HHHHxx 1586 6810 0 2 6 6 86 586 1586 1586 1586 172 173 AJAAAA YBKAAA OOOOxx 8512 6811 0 0 2 12 12 512 512 3512 8512 24 25 KPAAAA ZBKAAA VVVVxx 7421 6812 1 1 1 1 21 421 1421 2421 7421 42 43 LZAAAA ACKAAA AAAAxx 503 6813 1 3 3 3 3 503 503 503 503 6 7 JTAAAA BCKAAA HHHHxx 5332 6814 0 0 2 12 32 332 1332 332 5332 64 65 CXAAAA CCKAAA OOOOxx 2602 6815 0 2 2 2 2 602 602 2602 2602 4 5 CWAAAA DCKAAA VVVVxx 2902 6816 0 2 2 2 2 902 902 2902 2902 4 5 QHAAAA ECKAAA AAAAxx 2979 6817 1 3 9 19 79 979 979 2979 2979 158 159 PKAAAA FCKAAA HHHHxx 1431 6818 1 3 1 11 31 431 1431 1431 1431 62 63 BDAAAA GCKAAA OOOOxx 8639 6819 1 3 9 19 39 639 639 3639 8639 78 79 HUAAAA HCKAAA VVVVxx 4218 6820 0 2 8 18 18 218 218 4218 4218 36 37 GGAAAA ICKAAA AAAAxx 7453 6821 1 1 3 13 53 453 1453 2453 7453 106 107 RAAAAA JCKAAA HHHHxx 5448 6822 0 0 8 8 48 448 1448 448 5448 96 97 OBAAAA KCKAAA OOOOxx 6768 6823 0 0 8 8 68 768 768 1768 6768 136 137 IAAAAA LCKAAA VVVVxx 3104 6824 0 0 4 4 4 104 1104 3104 3104 8 9 KPAAAA MCKAAA AAAAxx 2297 6825 1 1 7 17 97 297 297 2297 2297 194 195 JKAAAA NCKAAA HHHHxx 7994 6826 0 2 4 14 94 994 1994 2994 7994 188 189 MVAAAA OCKAAA OOOOxx 550 6827 0 2 0 10 50 550 550 550 550 100 101 EVAAAA PCKAAA VVVVxx 4777 6828 1 1 7 17 77 777 777 4777 4777 154 155 TBAAAA QCKAAA AAAAxx 5962 6829 0 2 2 2 62 962 1962 962 5962 124 125 IVAAAA RCKAAA HHHHxx 1763 6830 1 3 3 3 63 763 1763 1763 1763 126 127 VPAAAA SCKAAA OOOOxx 3654 6831 0 2 4 14 54 654 1654 3654 3654 108 109 OKAAAA TCKAAA VVVVxx 4106 6832 0 2 6 6 6 106 106 4106 4106 12 13 YBAAAA UCKAAA AAAAxx 5156 6833 0 0 6 16 56 156 1156 156 5156 112 113 IQAAAA VCKAAA HHHHxx 422 6834 0 2 2 2 22 422 422 422 422 44 45 GQAAAA WCKAAA OOOOxx 5011 6835 1 3 1 11 11 11 1011 11 5011 22 23 TKAAAA XCKAAA VVVVxx 218 6836 0 2 8 18 18 218 218 218 218 36 37 KIAAAA YCKAAA AAAAxx 9762 6837 0 2 2 2 62 762 1762 4762 9762 124 125 MLAAAA ZCKAAA HHHHxx 6074 6838 0 2 4 14 74 74 74 1074 6074 148 149 QZAAAA ADKAAA OOOOxx 4060 6839 0 0 0 0 60 60 60 4060 4060 120 121 EAAAAA BDKAAA VVVVxx 8680 6840 0 0 0 0 80 680 680 3680 8680 160 161 WVAAAA CDKAAA AAAAxx 5863 6841 1 3 3 3 63 863 1863 863 5863 126 127 NRAAAA DDKAAA HHHHxx 8042 6842 0 2 2 2 42 42 42 3042 8042 84 85 IXAAAA EDKAAA OOOOxx 2964 6843 0 0 4 4 64 964 964 2964 2964 128 129 AKAAAA FDKAAA VVVVxx 6931 6844 1 3 1 11 31 931 931 1931 6931 62 63 PGAAAA GDKAAA AAAAxx 6715 6845 1 3 5 15 15 715 715 1715 6715 30 31 HYAAAA HDKAAA HHHHxx 5859 6846 1 3 9 19 59 859 1859 859 5859 118 119 JRAAAA IDKAAA OOOOxx 6173 6847 1 1 3 13 73 173 173 1173 6173 146 147 LDAAAA JDKAAA VVVVxx 7788 6848 0 0 8 8 88 788 1788 2788 7788 176 177 ONAAAA KDKAAA AAAAxx 9370 6849 0 2 0 10 70 370 1370 4370 9370 140 141 KWAAAA LDKAAA HHHHxx 3038 6850 0 2 8 18 38 38 1038 3038 3038 76 77 WMAAAA MDKAAA OOOOxx 6483 6851 1 3 3 3 83 483 483 1483 6483 166 167 JPAAAA NDKAAA VVVVxx 7534 6852 0 2 4 14 34 534 1534 2534 7534 68 69 UDAAAA ODKAAA AAAAxx 5769 6853 1 1 9 9 69 769 1769 769 5769 138 139 XNAAAA PDKAAA HHHHxx 9152 6854 0 0 2 12 52 152 1152 4152 9152 104 105 AOAAAA QDKAAA OOOOxx 6251 6855 1 3 1 11 51 251 251 1251 6251 102 103 LGAAAA RDKAAA VVVVxx 9209 6856 1 1 9 9 9 209 1209 4209 9209 18 19 FQAAAA SDKAAA AAAAxx 5365 6857 1 1 5 5 65 365 1365 365 5365 130 131 JYAAAA TDKAAA HHHHxx 509 6858 1 1 9 9 9 509 509 509 509 18 19 PTAAAA UDKAAA OOOOxx 3132 6859 0 0 2 12 32 132 1132 3132 3132 64 65 MQAAAA VDKAAA VVVVxx 5373 6860 1 1 3 13 73 373 1373 373 5373 146 147 RYAAAA WDKAAA AAAAxx 4247 6861 1 3 7 7 47 247 247 4247 4247 94 95 JHAAAA XDKAAA HHHHxx 3491 6862 1 3 1 11 91 491 1491 3491 3491 182 183 HEAAAA YDKAAA OOOOxx 495 6863 1 3 5 15 95 495 495 495 495 190 191 BTAAAA ZDKAAA VVVVxx 1594 6864 0 2 4 14 94 594 1594 1594 1594 188 189 IJAAAA AEKAAA AAAAxx 2243 6865 1 3 3 3 43 243 243 2243 2243 86 87 HIAAAA BEKAAA HHHHxx 7780 6866 0 0 0 0 80 780 1780 2780 7780 160 161 GNAAAA CEKAAA OOOOxx 5632 6867 0 0 2 12 32 632 1632 632 5632 64 65 QIAAAA DEKAAA VVVVxx 2679 6868 1 3 9 19 79 679 679 2679 2679 158 159 BZAAAA EEKAAA AAAAxx 1354 6869 0 2 4 14 54 354 1354 1354 1354 108 109 CAAAAA FEKAAA HHHHxx 180 6870 0 0 0 0 80 180 180 180 180 160 161 YGAAAA GEKAAA OOOOxx 7017 6871 1 1 7 17 17 17 1017 2017 7017 34 35 XJAAAA HEKAAA VVVVxx 1867 6872 1 3 7 7 67 867 1867 1867 1867 134 135 VTAAAA IEKAAA AAAAxx 2213 6873 1 1 3 13 13 213 213 2213 2213 26 27 DHAAAA JEKAAA HHHHxx 8773 6874 1 1 3 13 73 773 773 3773 8773 146 147 LZAAAA KEKAAA OOOOxx 1784 6875 0 0 4 4 84 784 1784 1784 1784 168 169 QQAAAA LEKAAA VVVVxx 5961 6876 1 1 1 1 61 961 1961 961 5961 122 123 HVAAAA MEKAAA AAAAxx 8801 6877 1 1 1 1 1 801 801 3801 8801 2 3 NAAAAA NEKAAA HHHHxx 4860 6878 0 0 0 0 60 860 860 4860 4860 120 121 YEAAAA OEKAAA OOOOxx 2214 6879 0 2 4 14 14 214 214 2214 2214 28 29 EHAAAA PEKAAA VVVVxx 1735 6880 1 3 5 15 35 735 1735 1735 1735 70 71 TOAAAA QEKAAA AAAAxx 578 6881 0 2 8 18 78 578 578 578 578 156 157 GWAAAA REKAAA HHHHxx 7853 6882 1 1 3 13 53 853 1853 2853 7853 106 107 BQAAAA SEKAAA OOOOxx 2215 6883 1 3 5 15 15 215 215 2215 2215 30 31 FHAAAA TEKAAA VVVVxx 4704 6884 0 0 4 4 4 704 704 4704 4704 8 9 YYAAAA UEKAAA AAAAxx 9379 6885 1 3 9 19 79 379 1379 4379 9379 158 159 TWAAAA VEKAAA HHHHxx 9745 6886 1 1 5 5 45 745 1745 4745 9745 90 91 VKAAAA WEKAAA OOOOxx 5636 6887 0 0 6 16 36 636 1636 636 5636 72 73 UIAAAA XEKAAA VVVVxx 4548 6888 0 0 8 8 48 548 548 4548 4548 96 97 YSAAAA YEKAAA AAAAxx 6537 6889 1 1 7 17 37 537 537 1537 6537 74 75 LRAAAA ZEKAAA HHHHxx 7748 6890 0 0 8 8 48 748 1748 2748 7748 96 97 AMAAAA AFKAAA OOOOxx 687 6891 1 3 7 7 87 687 687 687 687 174 175 LAAAAA BFKAAA VVVVxx 1243 6892 1 3 3 3 43 243 1243 1243 1243 86 87 VVAAAA CFKAAA AAAAxx 852 6893 0 0 2 12 52 852 852 852 852 104 105 UGAAAA DFKAAA HHHHxx 785 6894 1 1 5 5 85 785 785 785 785 170 171 FEAAAA EFKAAA OOOOxx 2002 6895 0 2 2 2 2 2 2 2002 2002 4 5 AZAAAA FFKAAA VVVVxx 2748 6896 0 0 8 8 48 748 748 2748 2748 96 97 SBAAAA GFKAAA AAAAxx 6075 6897 1 3 5 15 75 75 75 1075 6075 150 151 RZAAAA HFKAAA HHHHxx 7029 6898 1 1 9 9 29 29 1029 2029 7029 58 59 JKAAAA IFKAAA OOOOxx 7474 6899 0 2 4 14 74 474 1474 2474 7474 148 149 MBAAAA JFKAAA VVVVxx 7755 6900 1 3 5 15 55 755 1755 2755 7755 110 111 HMAAAA KFKAAA AAAAxx 1456 6901 0 0 6 16 56 456 1456 1456 1456 112 113 AEAAAA LFKAAA HHHHxx 2808 6902 0 0 8 8 8 808 808 2808 2808 16 17 AEAAAA MFKAAA OOOOxx 4089 6903 1 1 9 9 89 89 89 4089 4089 178 179 HBAAAA NFKAAA VVVVxx 4718 6904 0 2 8 18 18 718 718 4718 4718 36 37 MZAAAA OFKAAA AAAAxx 910 6905 0 2 0 10 10 910 910 910 910 20 21 AJAAAA PFKAAA HHHHxx 2868 6906 0 0 8 8 68 868 868 2868 2868 136 137 IGAAAA QFKAAA OOOOxx 2103 6907 1 3 3 3 3 103 103 2103 2103 6 7 XCAAAA RFKAAA VVVVxx 2407 6908 1 3 7 7 7 407 407 2407 2407 14 15 POAAAA SFKAAA AAAAxx 4353 6909 1 1 3 13 53 353 353 4353 4353 106 107 LLAAAA TFKAAA HHHHxx 7988 6910 0 0 8 8 88 988 1988 2988 7988 176 177 GVAAAA UFKAAA OOOOxx 2750 6911 0 2 0 10 50 750 750 2750 2750 100 101 UBAAAA VFKAAA VVVVxx 2006 6912 0 2 6 6 6 6 6 2006 2006 12 13 EZAAAA WFKAAA AAAAxx 4617 6913 1 1 7 17 17 617 617 4617 4617 34 35 PVAAAA XFKAAA HHHHxx 1251 6914 1 3 1 11 51 251 1251 1251 1251 102 103 DWAAAA YFKAAA OOOOxx 4590 6915 0 2 0 10 90 590 590 4590 4590 180 181 OUAAAA ZFKAAA VVVVxx 1144 6916 0 0 4 4 44 144 1144 1144 1144 88 89 ASAAAA AGKAAA AAAAxx 7131 6917 1 3 1 11 31 131 1131 2131 7131 62 63 HOAAAA BGKAAA HHHHxx 95 6918 1 3 5 15 95 95 95 95 95 190 191 RDAAAA CGKAAA OOOOxx 4827 6919 1 3 7 7 27 827 827 4827 4827 54 55 RDAAAA DGKAAA VVVVxx 4307 6920 1 3 7 7 7 307 307 4307 4307 14 15 RJAAAA EGKAAA AAAAxx 1505 6921 1 1 5 5 5 505 1505 1505 1505 10 11 XFAAAA FGKAAA HHHHxx 8191 6922 1 3 1 11 91 191 191 3191 8191 182 183 BDAAAA GGKAAA OOOOxx 5037 6923 1 1 7 17 37 37 1037 37 5037 74 75 TLAAAA HGKAAA VVVVxx 7363 6924 1 3 3 3 63 363 1363 2363 7363 126 127 FXAAAA IGKAAA AAAAxx 8427 6925 1 3 7 7 27 427 427 3427 8427 54 55 DMAAAA JGKAAA HHHHxx 5231 6926 1 3 1 11 31 231 1231 231 5231 62 63 FTAAAA KGKAAA OOOOxx 2943 6927 1 3 3 3 43 943 943 2943 2943 86 87 FJAAAA LGKAAA VVVVxx 4624 6928 0 0 4 4 24 624 624 4624 4624 48 49 WVAAAA MGKAAA AAAAxx 2020 6929 0 0 0 0 20 20 20 2020 2020 40 41 SZAAAA NGKAAA HHHHxx 6155 6930 1 3 5 15 55 155 155 1155 6155 110 111 TCAAAA OGKAAA OOOOxx 4381 6931 1 1 1 1 81 381 381 4381 4381 162 163 NMAAAA PGKAAA VVVVxx 1057 6932 1 1 7 17 57 57 1057 1057 1057 114 115 ROAAAA QGKAAA AAAAxx 9010 6933 0 2 0 10 10 10 1010 4010 9010 20 21 OIAAAA RGKAAA HHHHxx 4947 6934 1 3 7 7 47 947 947 4947 4947 94 95 HIAAAA SGKAAA OOOOxx 335 6935 1 3 5 15 35 335 335 335 335 70 71 XMAAAA TGKAAA VVVVxx 6890 6936 0 2 0 10 90 890 890 1890 6890 180 181 AFAAAA UGKAAA AAAAxx 5070 6937 0 2 0 10 70 70 1070 70 5070 140 141 ANAAAA VGKAAA HHHHxx 5270 6938 0 2 0 10 70 270 1270 270 5270 140 141 SUAAAA WGKAAA OOOOxx 8657 6939 1 1 7 17 57 657 657 3657 8657 114 115 ZUAAAA XGKAAA VVVVxx 7625 6940 1 1 5 5 25 625 1625 2625 7625 50 51 HHAAAA YGKAAA AAAAxx 5759 6941 1 3 9 19 59 759 1759 759 5759 118 119 NNAAAA ZGKAAA HHHHxx 9483 6942 1 3 3 3 83 483 1483 4483 9483 166 167 TAAAAA AHKAAA OOOOxx 8304 6943 0 0 4 4 4 304 304 3304 8304 8 9 KHAAAA BHKAAA VVVVxx 296 6944 0 0 6 16 96 296 296 296 296 192 193 KLAAAA CHKAAA AAAAxx 1176 6945 0 0 6 16 76 176 1176 1176 1176 152 153 GTAAAA DHKAAA HHHHxx 2069 6946 1 1 9 9 69 69 69 2069 2069 138 139 PBAAAA EHKAAA OOOOxx 1531 6947 1 3 1 11 31 531 1531 1531 1531 62 63 XGAAAA FHKAAA VVVVxx 5329 6948 1 1 9 9 29 329 1329 329 5329 58 59 ZWAAAA GHKAAA AAAAxx 3702 6949 0 2 2 2 2 702 1702 3702 3702 4 5 KMAAAA HHKAAA HHHHxx 6520 6950 0 0 0 0 20 520 520 1520 6520 40 41 UQAAAA IHKAAA OOOOxx 7310 6951 0 2 0 10 10 310 1310 2310 7310 20 21 EVAAAA JHKAAA VVVVxx 1175 6952 1 3 5 15 75 175 1175 1175 1175 150 151 FTAAAA KHKAAA AAAAxx 9107 6953 1 3 7 7 7 107 1107 4107 9107 14 15 HMAAAA LHKAAA HHHHxx 2737 6954 1 1 7 17 37 737 737 2737 2737 74 75 HBAAAA MHKAAA OOOOxx 3437 6955 1 1 7 17 37 437 1437 3437 3437 74 75 FCAAAA NHKAAA VVVVxx 281 6956 1 1 1 1 81 281 281 281 281 162 163 VKAAAA OHKAAA AAAAxx 6676 6957 0 0 6 16 76 676 676 1676 6676 152 153 UWAAAA PHKAAA HHHHxx 145 6958 1 1 5 5 45 145 145 145 145 90 91 PFAAAA QHKAAA OOOOxx 3172 6959 0 0 2 12 72 172 1172 3172 3172 144 145 ASAAAA RHKAAA VVVVxx 4049 6960 1 1 9 9 49 49 49 4049 4049 98 99 TZAAAA SHKAAA AAAAxx 6042 6961 0 2 2 2 42 42 42 1042 6042 84 85 KYAAAA THKAAA HHHHxx 9122 6962 0 2 2 2 22 122 1122 4122 9122 44 45 WMAAAA UHKAAA OOOOxx 7244 6963 0 0 4 4 44 244 1244 2244 7244 88 89 QSAAAA VHKAAA VVVVxx 5361 6964 1 1 1 1 61 361 1361 361 5361 122 123 FYAAAA WHKAAA AAAAxx 8647 6965 1 3 7 7 47 647 647 3647 8647 94 95 PUAAAA XHKAAA HHHHxx 7956 6966 0 0 6 16 56 956 1956 2956 7956 112 113 AUAAAA YHKAAA OOOOxx 7812 6967 0 0 2 12 12 812 1812 2812 7812 24 25 MOAAAA ZHKAAA VVVVxx 570 6968 0 2 0 10 70 570 570 570 570 140 141 YVAAAA AIKAAA AAAAxx 4115 6969 1 3 5 15 15 115 115 4115 4115 30 31 HCAAAA BIKAAA HHHHxx 1856 6970 0 0 6 16 56 856 1856 1856 1856 112 113 KTAAAA CIKAAA OOOOxx 9582 6971 0 2 2 2 82 582 1582 4582 9582 164 165 OEAAAA DIKAAA VVVVxx 2025 6972 1 1 5 5 25 25 25 2025 2025 50 51 XZAAAA EIKAAA AAAAxx 986 6973 0 2 6 6 86 986 986 986 986 172 173 YLAAAA FIKAAA HHHHxx 8358 6974 0 2 8 18 58 358 358 3358 8358 116 117 MJAAAA GIKAAA OOOOxx 510 6975 0 2 0 10 10 510 510 510 510 20 21 QTAAAA HIKAAA VVVVxx 6101 6976 1 1 1 1 1 101 101 1101 6101 2 3 RAAAAA IIKAAA AAAAxx 4167 6977 1 3 7 7 67 167 167 4167 4167 134 135 HEAAAA JIKAAA HHHHxx 6139 6978 1 3 9 19 39 139 139 1139 6139 78 79 DCAAAA KIKAAA OOOOxx 6912 6979 0 0 2 12 12 912 912 1912 6912 24 25 WFAAAA LIKAAA VVVVxx 339 6980 1 3 9 19 39 339 339 339 339 78 79 BNAAAA MIKAAA AAAAxx 8759 6981 1 3 9 19 59 759 759 3759 8759 118 119 XYAAAA NIKAAA HHHHxx 246 6982 0 2 6 6 46 246 246 246 246 92 93 MJAAAA OIKAAA OOOOxx 2831 6983 1 3 1 11 31 831 831 2831 2831 62 63 XEAAAA PIKAAA VVVVxx 2327 6984 1 3 7 7 27 327 327 2327 2327 54 55 NLAAAA QIKAAA AAAAxx 7001 6985 1 1 1 1 1 1 1001 2001 7001 2 3 HJAAAA RIKAAA HHHHxx 4398 6986 0 2 8 18 98 398 398 4398 4398 196 197 ENAAAA SIKAAA OOOOxx 1495 6987 1 3 5 15 95 495 1495 1495 1495 190 191 NFAAAA TIKAAA VVVVxx 8522 6988 0 2 2 2 22 522 522 3522 8522 44 45 UPAAAA UIKAAA AAAAxx 7090 6989 0 2 0 10 90 90 1090 2090 7090 180 181 SMAAAA VIKAAA HHHHxx 8457 6990 1 1 7 17 57 457 457 3457 8457 114 115 HNAAAA WIKAAA OOOOxx 4238 6991 0 2 8 18 38 238 238 4238 4238 76 77 AHAAAA XIKAAA VVVVxx 6791 6992 1 3 1 11 91 791 791 1791 6791 182 183 FBAAAA YIKAAA AAAAxx 1342 6993 0 2 2 2 42 342 1342 1342 1342 84 85 QZAAAA ZIKAAA HHHHxx 4580 6994 0 0 0 0 80 580 580 4580 4580 160 161 EUAAAA AJKAAA OOOOxx 1475 6995 1 3 5 15 75 475 1475 1475 1475 150 151 TEAAAA BJKAAA VVVVxx 9184 6996 0 0 4 4 84 184 1184 4184 9184 168 169 GPAAAA CJKAAA AAAAxx 1189 6997 1 1 9 9 89 189 1189 1189 1189 178 179 TTAAAA DJKAAA HHHHxx 638 6998 0 2 8 18 38 638 638 638 638 76 77 OYAAAA EJKAAA OOOOxx 5867 6999 1 3 7 7 67 867 1867 867 5867 134 135 RRAAAA FJKAAA VVVVxx 9911 7000 1 3 1 11 11 911 1911 4911 9911 22 23 FRAAAA GJKAAA AAAAxx 8147 7001 1 3 7 7 47 147 147 3147 8147 94 95 JBAAAA HJKAAA HHHHxx 4492 7002 0 0 2 12 92 492 492 4492 4492 184 185 UQAAAA IJKAAA OOOOxx 385 7003 1 1 5 5 85 385 385 385 385 170 171 VOAAAA JJKAAA VVVVxx 5235 7004 1 3 5 15 35 235 1235 235 5235 70 71 JTAAAA KJKAAA AAAAxx 4812 7005 0 0 2 12 12 812 812 4812 4812 24 25 CDAAAA LJKAAA HHHHxx 9807 7006 1 3 7 7 7 807 1807 4807 9807 14 15 FNAAAA MJKAAA OOOOxx 9588 7007 0 0 8 8 88 588 1588 4588 9588 176 177 UEAAAA NJKAAA VVVVxx 9832 7008 0 0 2 12 32 832 1832 4832 9832 64 65 EOAAAA OJKAAA AAAAxx 3757 7009 1 1 7 17 57 757 1757 3757 3757 114 115 NOAAAA PJKAAA HHHHxx 9703 7010 1 3 3 3 3 703 1703 4703 9703 6 7 FJAAAA QJKAAA OOOOxx 1022 7011 0 2 2 2 22 22 1022 1022 1022 44 45 INAAAA RJKAAA VVVVxx 5165 7012 1 1 5 5 65 165 1165 165 5165 130 131 RQAAAA SJKAAA AAAAxx 7129 7013 1 1 9 9 29 129 1129 2129 7129 58 59 FOAAAA TJKAAA HHHHxx 4164 7014 0 0 4 4 64 164 164 4164 4164 128 129 EEAAAA UJKAAA OOOOxx 7239 7015 1 3 9 19 39 239 1239 2239 7239 78 79 LSAAAA VJKAAA VVVVxx 523 7016 1 3 3 3 23 523 523 523 523 46 47 DUAAAA WJKAAA AAAAxx 4670 7017 0 2 0 10 70 670 670 4670 4670 140 141 QXAAAA XJKAAA HHHHxx 8503 7018 1 3 3 3 3 503 503 3503 8503 6 7 BPAAAA YJKAAA OOOOxx 714 7019 0 2 4 14 14 714 714 714 714 28 29 MBAAAA ZJKAAA VVVVxx 1350 7020 0 2 0 10 50 350 1350 1350 1350 100 101 YZAAAA AKKAAA AAAAxx 8318 7021 0 2 8 18 18 318 318 3318 8318 36 37 YHAAAA BKKAAA HHHHxx 1834 7022 0 2 4 14 34 834 1834 1834 1834 68 69 OSAAAA CKKAAA OOOOxx 4306 7023 0 2 6 6 6 306 306 4306 4306 12 13 QJAAAA DKKAAA VVVVxx 8543 7024 1 3 3 3 43 543 543 3543 8543 86 87 PQAAAA EKKAAA AAAAxx 9397 7025 1 1 7 17 97 397 1397 4397 9397 194 195 LXAAAA FKKAAA HHHHxx 3145 7026 1 1 5 5 45 145 1145 3145 3145 90 91 ZQAAAA GKKAAA OOOOxx 3942 7027 0 2 2 2 42 942 1942 3942 3942 84 85 QVAAAA HKKAAA VVVVxx 8583 7028 1 3 3 3 83 583 583 3583 8583 166 167 DSAAAA IKKAAA AAAAxx 8073 7029 1 1 3 13 73 73 73 3073 8073 146 147 NYAAAA JKKAAA HHHHxx 4940 7030 0 0 0 0 40 940 940 4940 4940 80 81 AIAAAA KKKAAA OOOOxx 9573 7031 1 1 3 13 73 573 1573 4573 9573 146 147 FEAAAA LKKAAA VVVVxx 5325 7032 1 1 5 5 25 325 1325 325 5325 50 51 VWAAAA MKKAAA AAAAxx 1833 7033 1 1 3 13 33 833 1833 1833 1833 66 67 NSAAAA NKKAAA HHHHxx 1337 7034 1 1 7 17 37 337 1337 1337 1337 74 75 LZAAAA OKKAAA OOOOxx 9749 7035 1 1 9 9 49 749 1749 4749 9749 98 99 ZKAAAA PKKAAA VVVVxx 7505 7036 1 1 5 5 5 505 1505 2505 7505 10 11 RCAAAA QKKAAA AAAAxx 9731 7037 1 3 1 11 31 731 1731 4731 9731 62 63 HKAAAA RKKAAA HHHHxx 4098 7038 0 2 8 18 98 98 98 4098 4098 196 197 QBAAAA SKKAAA OOOOxx 1418 7039 0 2 8 18 18 418 1418 1418 1418 36 37 OCAAAA TKKAAA VVVVxx 63 7040 1 3 3 3 63 63 63 63 63 126 127 LCAAAA UKKAAA AAAAxx 9889 7041 1 1 9 9 89 889 1889 4889 9889 178 179 JQAAAA VKKAAA HHHHxx 2871 7042 1 3 1 11 71 871 871 2871 2871 142 143 LGAAAA WKKAAA OOOOxx 1003 7043 1 3 3 3 3 3 1003 1003 1003 6 7 PMAAAA XKKAAA VVVVxx 8796 7044 0 0 6 16 96 796 796 3796 8796 192 193 IAAAAA YKKAAA AAAAxx 22 7045 0 2 2 2 22 22 22 22 22 44 45 WAAAAA ZKKAAA HHHHxx 8244 7046 0 0 4 4 44 244 244 3244 8244 88 89 CFAAAA ALKAAA OOOOxx 2282 7047 0 2 2 2 82 282 282 2282 2282 164 165 UJAAAA BLKAAA VVVVxx 3487 7048 1 3 7 7 87 487 1487 3487 3487 174 175 DEAAAA CLKAAA AAAAxx 8633 7049 1 1 3 13 33 633 633 3633 8633 66 67 BUAAAA DLKAAA HHHHxx 6418 7050 0 2 8 18 18 418 418 1418 6418 36 37 WMAAAA ELKAAA OOOOxx 4682 7051 0 2 2 2 82 682 682 4682 4682 164 165 CYAAAA FLKAAA VVVVxx 4103 7052 1 3 3 3 3 103 103 4103 4103 6 7 VBAAAA GLKAAA AAAAxx 6256 7053 0 0 6 16 56 256 256 1256 6256 112 113 QGAAAA HLKAAA HHHHxx 4040 7054 0 0 0 0 40 40 40 4040 4040 80 81 KZAAAA ILKAAA OOOOxx 9342 7055 0 2 2 2 42 342 1342 4342 9342 84 85 IVAAAA JLKAAA VVVVxx 9969 7056 1 1 9 9 69 969 1969 4969 9969 138 139 LTAAAA KLKAAA AAAAxx 223 7057 1 3 3 3 23 223 223 223 223 46 47 PIAAAA LLKAAA HHHHxx 4593 7058 1 1 3 13 93 593 593 4593 4593 186 187 RUAAAA MLKAAA OOOOxx 44 7059 0 0 4 4 44 44 44 44 44 88 89 SBAAAA NLKAAA VVVVxx 3513 7060 1 1 3 13 13 513 1513 3513 3513 26 27 DFAAAA OLKAAA AAAAxx 5771 7061 1 3 1 11 71 771 1771 771 5771 142 143 ZNAAAA PLKAAA HHHHxx 5083 7062 1 3 3 3 83 83 1083 83 5083 166 167 NNAAAA QLKAAA OOOOxx 3839 7063 1 3 9 19 39 839 1839 3839 3839 78 79 RRAAAA RLKAAA VVVVxx 2986 7064 0 2 6 6 86 986 986 2986 2986 172 173 WKAAAA SLKAAA AAAAxx 2200 7065 0 0 0 0 0 200 200 2200 2200 0 1 QGAAAA TLKAAA HHHHxx 197 7066 1 1 7 17 97 197 197 197 197 194 195 PHAAAA ULKAAA OOOOxx 7455 7067 1 3 5 15 55 455 1455 2455 7455 110 111 TAAAAA VLKAAA VVVVxx 1379 7068 1 3 9 19 79 379 1379 1379 1379 158 159 BBAAAA WLKAAA AAAAxx 4356 7069 0 0 6 16 56 356 356 4356 4356 112 113 OLAAAA XLKAAA HHHHxx 6888 7070 0 0 8 8 88 888 888 1888 6888 176 177 YEAAAA YLKAAA OOOOxx 9139 7071 1 3 9 19 39 139 1139 4139 9139 78 79 NNAAAA ZLKAAA VVVVxx 7682 7072 0 2 2 2 82 682 1682 2682 7682 164 165 MJAAAA AMKAAA AAAAxx 4873 7073 1 1 3 13 73 873 873 4873 4873 146 147 LFAAAA BMKAAA HHHHxx 783 7074 1 3 3 3 83 783 783 783 783 166 167 DEAAAA CMKAAA OOOOxx 6071 7075 1 3 1 11 71 71 71 1071 6071 142 143 NZAAAA DMKAAA VVVVxx 5160 7076 0 0 0 0 60 160 1160 160 5160 120 121 MQAAAA EMKAAA AAAAxx 2291 7077 1 3 1 11 91 291 291 2291 2291 182 183 DKAAAA FMKAAA HHHHxx 187 7078 1 3 7 7 87 187 187 187 187 174 175 FHAAAA GMKAAA OOOOxx 7786 7079 0 2 6 6 86 786 1786 2786 7786 172 173 MNAAAA HMKAAA VVVVxx 3432 7080 0 0 2 12 32 432 1432 3432 3432 64 65 ACAAAA IMKAAA AAAAxx 5450 7081 0 2 0 10 50 450 1450 450 5450 100 101 QBAAAA JMKAAA HHHHxx 2699 7082 1 3 9 19 99 699 699 2699 2699 198 199 VZAAAA KMKAAA OOOOxx 692 7083 0 0 2 12 92 692 692 692 692 184 185 QAAAAA LMKAAA VVVVxx 6081 7084 1 1 1 1 81 81 81 1081 6081 162 163 XZAAAA MMKAAA AAAAxx 4829 7085 1 1 9 9 29 829 829 4829 4829 58 59 TDAAAA NMKAAA HHHHxx 238 7086 0 2 8 18 38 238 238 238 238 76 77 EJAAAA OMKAAA OOOOxx 9100 7087 0 0 0 0 0 100 1100 4100 9100 0 1 AMAAAA PMKAAA VVVVxx 1968 7088 0 0 8 8 68 968 1968 1968 1968 136 137 SXAAAA QMKAAA AAAAxx 1872 7089 0 0 2 12 72 872 1872 1872 1872 144 145 AUAAAA RMKAAA HHHHxx 7051 7090 1 3 1 11 51 51 1051 2051 7051 102 103 FLAAAA SMKAAA OOOOxx 2743 7091 1 3 3 3 43 743 743 2743 2743 86 87 NBAAAA TMKAAA VVVVxx 1237 7092 1 1 7 17 37 237 1237 1237 1237 74 75 PVAAAA UMKAAA AAAAxx 3052 7093 0 0 2 12 52 52 1052 3052 3052 104 105 KNAAAA VMKAAA HHHHxx 8021 7094 1 1 1 1 21 21 21 3021 8021 42 43 NWAAAA WMKAAA OOOOxx 657 7095 1 1 7 17 57 657 657 657 657 114 115 HZAAAA XMKAAA VVVVxx 2236 7096 0 0 6 16 36 236 236 2236 2236 72 73 AIAAAA YMKAAA AAAAxx 7011 7097 1 3 1 11 11 11 1011 2011 7011 22 23 RJAAAA ZMKAAA HHHHxx 4067 7098 1 3 7 7 67 67 67 4067 4067 134 135 LAAAAA ANKAAA OOOOxx 9449 7099 1 1 9 9 49 449 1449 4449 9449 98 99 LZAAAA BNKAAA VVVVxx 7428 7100 0 0 8 8 28 428 1428 2428 7428 56 57 SZAAAA CNKAAA AAAAxx 1272 7101 0 0 2 12 72 272 1272 1272 1272 144 145 YWAAAA DNKAAA HHHHxx 6897 7102 1 1 7 17 97 897 897 1897 6897 194 195 HFAAAA ENKAAA OOOOxx 5839 7103 1 3 9 19 39 839 1839 839 5839 78 79 PQAAAA FNKAAA VVVVxx 6835 7104 1 3 5 15 35 835 835 1835 6835 70 71 XCAAAA GNKAAA AAAAxx 1887 7105 1 3 7 7 87 887 1887 1887 1887 174 175 PUAAAA HNKAAA HHHHxx 1551 7106 1 3 1 11 51 551 1551 1551 1551 102 103 RHAAAA INKAAA OOOOxx 4667 7107 1 3 7 7 67 667 667 4667 4667 134 135 NXAAAA JNKAAA VVVVxx 9603 7108 1 3 3 3 3 603 1603 4603 9603 6 7 JFAAAA KNKAAA AAAAxx 4332 7109 0 0 2 12 32 332 332 4332 4332 64 65 QKAAAA LNKAAA HHHHxx 5681 7110 1 1 1 1 81 681 1681 681 5681 162 163 NKAAAA MNKAAA OOOOxx 8062 7111 0 2 2 2 62 62 62 3062 8062 124 125 CYAAAA NNKAAA VVVVxx 2302 7112 0 2 2 2 2 302 302 2302 2302 4 5 OKAAAA ONKAAA AAAAxx 2825 7113 1 1 5 5 25 825 825 2825 2825 50 51 REAAAA PNKAAA HHHHxx 4527 7114 1 3 7 7 27 527 527 4527 4527 54 55 DSAAAA QNKAAA OOOOxx 4230 7115 0 2 0 10 30 230 230 4230 4230 60 61 SGAAAA RNKAAA VVVVxx 3053 7116 1 1 3 13 53 53 1053 3053 3053 106 107 LNAAAA SNKAAA AAAAxx 983 7117 1 3 3 3 83 983 983 983 983 166 167 VLAAAA TNKAAA HHHHxx 9458 7118 0 2 8 18 58 458 1458 4458 9458 116 117 UZAAAA UNKAAA OOOOxx 4128 7119 0 0 8 8 28 128 128 4128 4128 56 57 UCAAAA VNKAAA VVVVxx 425 7120 1 1 5 5 25 425 425 425 425 50 51 JQAAAA WNKAAA AAAAxx 3911 7121 1 3 1 11 11 911 1911 3911 3911 22 23 LUAAAA XNKAAA HHHHxx 6607 7122 1 3 7 7 7 607 607 1607 6607 14 15 DUAAAA YNKAAA OOOOxx 5431 7123 1 3 1 11 31 431 1431 431 5431 62 63 XAAAAA ZNKAAA VVVVxx 6330 7124 0 2 0 10 30 330 330 1330 6330 60 61 MJAAAA AOKAAA AAAAxx 3592 7125 0 0 2 12 92 592 1592 3592 3592 184 185 EIAAAA BOKAAA HHHHxx 154 7126 0 2 4 14 54 154 154 154 154 108 109 YFAAAA COKAAA OOOOxx 9879 7127 1 3 9 19 79 879 1879 4879 9879 158 159 ZPAAAA DOKAAA VVVVxx 3202 7128 0 2 2 2 2 202 1202 3202 3202 4 5 ETAAAA EOKAAA AAAAxx 3056 7129 0 0 6 16 56 56 1056 3056 3056 112 113 ONAAAA FOKAAA HHHHxx 9890 7130 0 2 0 10 90 890 1890 4890 9890 180 181 KQAAAA GOKAAA OOOOxx 5840 7131 0 0 0 0 40 840 1840 840 5840 80 81 QQAAAA HOKAAA VVVVxx 9804 7132 0 0 4 4 4 804 1804 4804 9804 8 9 CNAAAA IOKAAA AAAAxx 681 7133 1 1 1 1 81 681 681 681 681 162 163 FAAAAA JOKAAA HHHHxx 3443 7134 1 3 3 3 43 443 1443 3443 3443 86 87 LCAAAA KOKAAA OOOOxx 8088 7135 0 0 8 8 88 88 88 3088 8088 176 177 CZAAAA LOKAAA VVVVxx 9447 7136 1 3 7 7 47 447 1447 4447 9447 94 95 JZAAAA MOKAAA AAAAxx 1490 7137 0 2 0 10 90 490 1490 1490 1490 180 181 IFAAAA NOKAAA HHHHxx 3684 7138 0 0 4 4 84 684 1684 3684 3684 168 169 SLAAAA OOKAAA OOOOxx 3113 7139 1 1 3 13 13 113 1113 3113 3113 26 27 TPAAAA POKAAA VVVVxx 9004 7140 0 0 4 4 4 4 1004 4004 9004 8 9 IIAAAA QOKAAA AAAAxx 7147 7141 1 3 7 7 47 147 1147 2147 7147 94 95 XOAAAA ROKAAA HHHHxx 7571 7142 1 3 1 11 71 571 1571 2571 7571 142 143 FFAAAA SOKAAA OOOOxx 5545 7143 1 1 5 5 45 545 1545 545 5545 90 91 HFAAAA TOKAAA VVVVxx 4558 7144 0 2 8 18 58 558 558 4558 4558 116 117 ITAAAA UOKAAA AAAAxx 6206 7145 0 2 6 6 6 206 206 1206 6206 12 13 SEAAAA VOKAAA HHHHxx 5695 7146 1 3 5 15 95 695 1695 695 5695 190 191 BLAAAA WOKAAA OOOOxx 9600 7147 0 0 0 0 0 600 1600 4600 9600 0 1 GFAAAA XOKAAA VVVVxx 5432 7148 0 0 2 12 32 432 1432 432 5432 64 65 YAAAAA YOKAAA AAAAxx 9299 7149 1 3 9 19 99 299 1299 4299 9299 198 199 RTAAAA ZOKAAA HHHHxx 2386 7150 0 2 6 6 86 386 386 2386 2386 172 173 UNAAAA APKAAA OOOOxx 2046 7151 0 2 6 6 46 46 46 2046 2046 92 93 SAAAAA BPKAAA VVVVxx 3293 7152 1 1 3 13 93 293 1293 3293 3293 186 187 RWAAAA CPKAAA AAAAxx 3046 7153 0 2 6 6 46 46 1046 3046 3046 92 93 ENAAAA DPKAAA HHHHxx 214 7154 0 2 4 14 14 214 214 214 214 28 29 GIAAAA EPKAAA OOOOxx 7893 7155 1 1 3 13 93 893 1893 2893 7893 186 187 PRAAAA FPKAAA VVVVxx 891 7156 1 3 1 11 91 891 891 891 891 182 183 HIAAAA GPKAAA AAAAxx 6499 7157 1 3 9 19 99 499 499 1499 6499 198 199 ZPAAAA HPKAAA HHHHxx 5003 7158 1 3 3 3 3 3 1003 3 5003 6 7 LKAAAA IPKAAA OOOOxx 6487 7159 1 3 7 7 87 487 487 1487 6487 174 175 NPAAAA JPKAAA VVVVxx 9403 7160 1 3 3 3 3 403 1403 4403 9403 6 7 RXAAAA KPKAAA AAAAxx 945 7161 1 1 5 5 45 945 945 945 945 90 91 JKAAAA LPKAAA HHHHxx 6713 7162 1 1 3 13 13 713 713 1713 6713 26 27 FYAAAA MPKAAA OOOOxx 9928 7163 0 0 8 8 28 928 1928 4928 9928 56 57 WRAAAA NPKAAA VVVVxx 8585 7164 1 1 5 5 85 585 585 3585 8585 170 171 FSAAAA OPKAAA AAAAxx 4004 7165 0 0 4 4 4 4 4 4004 4004 8 9 AYAAAA PPKAAA HHHHxx 2528 7166 0 0 8 8 28 528 528 2528 2528 56 57 GTAAAA QPKAAA OOOOxx 3350 7167 0 2 0 10 50 350 1350 3350 3350 100 101 WYAAAA RPKAAA VVVVxx 2160 7168 0 0 0 0 60 160 160 2160 2160 120 121 CFAAAA SPKAAA AAAAxx 1521 7169 1 1 1 1 21 521 1521 1521 1521 42 43 NGAAAA TPKAAA HHHHxx 5660 7170 0 0 0 0 60 660 1660 660 5660 120 121 SJAAAA UPKAAA OOOOxx 5755 7171 1 3 5 15 55 755 1755 755 5755 110 111 JNAAAA VPKAAA VVVVxx 7614 7172 0 2 4 14 14 614 1614 2614 7614 28 29 WGAAAA WPKAAA AAAAxx 3121 7173 1 1 1 1 21 121 1121 3121 3121 42 43 BQAAAA XPKAAA HHHHxx 2735 7174 1 3 5 15 35 735 735 2735 2735 70 71 FBAAAA YPKAAA OOOOxx 7506 7175 0 2 6 6 6 506 1506 2506 7506 12 13 SCAAAA ZPKAAA VVVVxx 2693 7176 1 1 3 13 93 693 693 2693 2693 186 187 PZAAAA AQKAAA AAAAxx 2892 7177 0 0 2 12 92 892 892 2892 2892 184 185 GHAAAA BQKAAA HHHHxx 3310 7178 0 2 0 10 10 310 1310 3310 3310 20 21 IXAAAA CQKAAA OOOOxx 3484 7179 0 0 4 4 84 484 1484 3484 3484 168 169 AEAAAA DQKAAA VVVVxx 9733 7180 1 1 3 13 33 733 1733 4733 9733 66 67 JKAAAA EQKAAA AAAAxx 29 7181 1 1 9 9 29 29 29 29 29 58 59 DBAAAA FQKAAA HHHHxx 9013 7182 1 1 3 13 13 13 1013 4013 9013 26 27 RIAAAA GQKAAA OOOOxx 3847 7183 1 3 7 7 47 847 1847 3847 3847 94 95 ZRAAAA HQKAAA VVVVxx 6724 7184 0 0 4 4 24 724 724 1724 6724 48 49 QYAAAA IQKAAA AAAAxx 2559 7185 1 3 9 19 59 559 559 2559 2559 118 119 LUAAAA JQKAAA HHHHxx 5326 7186 0 2 6 6 26 326 1326 326 5326 52 53 WWAAAA KQKAAA OOOOxx 4802 7187 0 2 2 2 2 802 802 4802 4802 4 5 SCAAAA LQKAAA VVVVxx 131 7188 1 3 1 11 31 131 131 131 131 62 63 BFAAAA MQKAAA AAAAxx 1634 7189 0 2 4 14 34 634 1634 1634 1634 68 69 WKAAAA NQKAAA HHHHxx 919 7190 1 3 9 19 19 919 919 919 919 38 39 JJAAAA OQKAAA OOOOxx 9575 7191 1 3 5 15 75 575 1575 4575 9575 150 151 HEAAAA PQKAAA VVVVxx 1256 7192 0 0 6 16 56 256 1256 1256 1256 112 113 IWAAAA QQKAAA AAAAxx 9428 7193 0 0 8 8 28 428 1428 4428 9428 56 57 QYAAAA RQKAAA HHHHxx 5121 7194 1 1 1 1 21 121 1121 121 5121 42 43 ZOAAAA SQKAAA OOOOxx 6584 7195 0 0 4 4 84 584 584 1584 6584 168 169 GTAAAA TQKAAA VVVVxx 7193 7196 1 1 3 13 93 193 1193 2193 7193 186 187 RQAAAA UQKAAA AAAAxx 4047 7197 1 3 7 7 47 47 47 4047 4047 94 95 RZAAAA VQKAAA HHHHxx 104 7198 0 0 4 4 4 104 104 104 104 8 9 AEAAAA WQKAAA OOOOxx 1527 7199 1 3 7 7 27 527 1527 1527 1527 54 55 TGAAAA XQKAAA VVVVxx 3460 7200 0 0 0 0 60 460 1460 3460 3460 120 121 CDAAAA YQKAAA AAAAxx 8526 7201 0 2 6 6 26 526 526 3526 8526 52 53 YPAAAA ZQKAAA HHHHxx 8959 7202 1 3 9 19 59 959 959 3959 8959 118 119 PGAAAA ARKAAA OOOOxx 3633 7203 1 1 3 13 33 633 1633 3633 3633 66 67 TJAAAA BRKAAA VVVVxx 1799 7204 1 3 9 19 99 799 1799 1799 1799 198 199 FRAAAA CRKAAA AAAAxx 461 7205 1 1 1 1 61 461 461 461 461 122 123 TRAAAA DRKAAA HHHHxx 718 7206 0 2 8 18 18 718 718 718 718 36 37 QBAAAA ERKAAA OOOOxx 3219 7207 1 3 9 19 19 219 1219 3219 3219 38 39 VTAAAA FRKAAA VVVVxx 3494 7208 0 2 4 14 94 494 1494 3494 3494 188 189 KEAAAA GRKAAA AAAAxx 9402 7209 0 2 2 2 2 402 1402 4402 9402 4 5 QXAAAA HRKAAA HHHHxx 7983 7210 1 3 3 3 83 983 1983 2983 7983 166 167 BVAAAA IRKAAA OOOOxx 7919 7211 1 3 9 19 19 919 1919 2919 7919 38 39 PSAAAA JRKAAA VVVVxx 8036 7212 0 0 6 16 36 36 36 3036 8036 72 73 CXAAAA KRKAAA AAAAxx 5164 7213 0 0 4 4 64 164 1164 164 5164 128 129 QQAAAA LRKAAA HHHHxx 4160 7214 0 0 0 0 60 160 160 4160 4160 120 121 AEAAAA MRKAAA OOOOxx 5370 7215 0 2 0 10 70 370 1370 370 5370 140 141 OYAAAA NRKAAA VVVVxx 5347 7216 1 3 7 7 47 347 1347 347 5347 94 95 RXAAAA ORKAAA AAAAxx 7109 7217 1 1 9 9 9 109 1109 2109 7109 18 19 LNAAAA PRKAAA HHHHxx 4826 7218 0 2 6 6 26 826 826 4826 4826 52 53 QDAAAA QRKAAA OOOOxx 1338 7219 0 2 8 18 38 338 1338 1338 1338 76 77 MZAAAA RRKAAA VVVVxx 2711 7220 1 3 1 11 11 711 711 2711 2711 22 23 HAAAAA SRKAAA AAAAxx 6299 7221 1 3 9 19 99 299 299 1299 6299 198 199 HIAAAA TRKAAA HHHHxx 1616 7222 0 0 6 16 16 616 1616 1616 1616 32 33 EKAAAA URKAAA OOOOxx 7519 7223 1 3 9 19 19 519 1519 2519 7519 38 39 FDAAAA VRKAAA VVVVxx 1262 7224 0 2 2 2 62 262 1262 1262 1262 124 125 OWAAAA WRKAAA AAAAxx 7228 7225 0 0 8 8 28 228 1228 2228 7228 56 57 ASAAAA XRKAAA HHHHxx 7892 7226 0 0 2 12 92 892 1892 2892 7892 184 185 ORAAAA YRKAAA OOOOxx 7929 7227 1 1 9 9 29 929 1929 2929 7929 58 59 ZSAAAA ZRKAAA VVVVxx 7705 7228 1 1 5 5 5 705 1705 2705 7705 10 11 JKAAAA ASKAAA AAAAxx 3111 7229 1 3 1 11 11 111 1111 3111 3111 22 23 RPAAAA BSKAAA HHHHxx 3066 7230 0 2 6 6 66 66 1066 3066 3066 132 133 YNAAAA CSKAAA OOOOxx 9559 7231 1 3 9 19 59 559 1559 4559 9559 118 119 RDAAAA DSKAAA VVVVxx 3787 7232 1 3 7 7 87 787 1787 3787 3787 174 175 RPAAAA ESKAAA AAAAxx 8710 7233 0 2 0 10 10 710 710 3710 8710 20 21 AXAAAA FSKAAA HHHHxx 4870 7234 0 2 0 10 70 870 870 4870 4870 140 141 IFAAAA GSKAAA OOOOxx 1883 7235 1 3 3 3 83 883 1883 1883 1883 166 167 LUAAAA HSKAAA VVVVxx 9689 7236 1 1 9 9 89 689 1689 4689 9689 178 179 RIAAAA ISKAAA AAAAxx 9491 7237 1 3 1 11 91 491 1491 4491 9491 182 183 BBAAAA JSKAAA HHHHxx 2035 7238 1 3 5 15 35 35 35 2035 2035 70 71 HAAAAA KSKAAA OOOOxx 655 7239 1 3 5 15 55 655 655 655 655 110 111 FZAAAA LSKAAA VVVVxx 6305 7240 1 1 5 5 5 305 305 1305 6305 10 11 NIAAAA MSKAAA AAAAxx 9423 7241 1 3 3 3 23 423 1423 4423 9423 46 47 LYAAAA NSKAAA HHHHxx 283 7242 1 3 3 3 83 283 283 283 283 166 167 XKAAAA OSKAAA OOOOxx 2607 7243 1 3 7 7 7 607 607 2607 2607 14 15 HWAAAA PSKAAA VVVVxx 7740 7244 0 0 0 0 40 740 1740 2740 7740 80 81 SLAAAA QSKAAA AAAAxx 6956 7245 0 0 6 16 56 956 956 1956 6956 112 113 OHAAAA RSKAAA HHHHxx 884 7246 0 0 4 4 84 884 884 884 884 168 169 AIAAAA SSKAAA OOOOxx 5730 7247 0 2 0 10 30 730 1730 730 5730 60 61 KMAAAA TSKAAA VVVVxx 3438 7248 0 2 8 18 38 438 1438 3438 3438 76 77 GCAAAA USKAAA AAAAxx 3250 7249 0 2 0 10 50 250 1250 3250 3250 100 101 AVAAAA VSKAAA HHHHxx 5470 7250 0 2 0 10 70 470 1470 470 5470 140 141 KCAAAA WSKAAA OOOOxx 2037 7251 1 1 7 17 37 37 37 2037 2037 74 75 JAAAAA XSKAAA VVVVxx 6593 7252 1 1 3 13 93 593 593 1593 6593 186 187 PTAAAA YSKAAA AAAAxx 3893 7253 1 1 3 13 93 893 1893 3893 3893 186 187 TTAAAA ZSKAAA HHHHxx 3200 7254 0 0 0 0 0 200 1200 3200 3200 0 1 CTAAAA ATKAAA OOOOxx 7125 7255 1 1 5 5 25 125 1125 2125 7125 50 51 BOAAAA BTKAAA VVVVxx 2295 7256 1 3 5 15 95 295 295 2295 2295 190 191 HKAAAA CTKAAA AAAAxx 2056 7257 0 0 6 16 56 56 56 2056 2056 112 113 CBAAAA DTKAAA HHHHxx 2962 7258 0 2 2 2 62 962 962 2962 2962 124 125 YJAAAA ETKAAA OOOOxx 993 7259 1 1 3 13 93 993 993 993 993 186 187 FMAAAA FTKAAA VVVVxx 9127 7260 1 3 7 7 27 127 1127 4127 9127 54 55 BNAAAA GTKAAA AAAAxx 2075 7261 1 3 5 15 75 75 75 2075 2075 150 151 VBAAAA HTKAAA HHHHxx 9338 7262 0 2 8 18 38 338 1338 4338 9338 76 77 EVAAAA ITKAAA OOOOxx 8100 7263 0 0 0 0 0 100 100 3100 8100 0 1 OZAAAA JTKAAA VVVVxx 5047 7264 1 3 7 7 47 47 1047 47 5047 94 95 DMAAAA KTKAAA AAAAxx 7032 7265 0 0 2 12 32 32 1032 2032 7032 64 65 MKAAAA LTKAAA HHHHxx 6374 7266 0 2 4 14 74 374 374 1374 6374 148 149 ELAAAA MTKAAA OOOOxx 4137 7267 1 1 7 17 37 137 137 4137 4137 74 75 DDAAAA NTKAAA VVVVxx 7132 7268 0 0 2 12 32 132 1132 2132 7132 64 65 IOAAAA OTKAAA AAAAxx 3064 7269 0 0 4 4 64 64 1064 3064 3064 128 129 WNAAAA PTKAAA HHHHxx 3621 7270 1 1 1 1 21 621 1621 3621 3621 42 43 HJAAAA QTKAAA OOOOxx 6199 7271 1 3 9 19 99 199 199 1199 6199 198 199 LEAAAA RTKAAA VVVVxx 4926 7272 0 2 6 6 26 926 926 4926 4926 52 53 MHAAAA STKAAA AAAAxx 8035 7273 1 3 5 15 35 35 35 3035 8035 70 71 BXAAAA TTKAAA HHHHxx 2195 7274 1 3 5 15 95 195 195 2195 2195 190 191 LGAAAA UTKAAA OOOOxx 5366 7275 0 2 6 6 66 366 1366 366 5366 132 133 KYAAAA VTKAAA VVVVxx 3478 7276 0 2 8 18 78 478 1478 3478 3478 156 157 UDAAAA WTKAAA AAAAxx 1926 7277 0 2 6 6 26 926 1926 1926 1926 52 53 CWAAAA XTKAAA HHHHxx 7265 7278 1 1 5 5 65 265 1265 2265 7265 130 131 LTAAAA YTKAAA OOOOxx 7668 7279 0 0 8 8 68 668 1668 2668 7668 136 137 YIAAAA ZTKAAA VVVVxx 3335 7280 1 3 5 15 35 335 1335 3335 3335 70 71 HYAAAA AUKAAA AAAAxx 7660 7281 0 0 0 0 60 660 1660 2660 7660 120 121 QIAAAA BUKAAA HHHHxx 9604 7282 0 0 4 4 4 604 1604 4604 9604 8 9 KFAAAA CUKAAA OOOOxx 7301 7283 1 1 1 1 1 301 1301 2301 7301 2 3 VUAAAA DUKAAA VVVVxx 4475 7284 1 3 5 15 75 475 475 4475 4475 150 151 DQAAAA EUKAAA AAAAxx 9954 7285 0 2 4 14 54 954 1954 4954 9954 108 109 WSAAAA FUKAAA HHHHxx 5723 7286 1 3 3 3 23 723 1723 723 5723 46 47 DMAAAA GUKAAA OOOOxx 2669 7287 1 1 9 9 69 669 669 2669 2669 138 139 RYAAAA HUKAAA VVVVxx 1685 7288 1 1 5 5 85 685 1685 1685 1685 170 171 VMAAAA IUKAAA AAAAxx 2233 7289 1 1 3 13 33 233 233 2233 2233 66 67 XHAAAA JUKAAA HHHHxx 8111 7290 1 3 1 11 11 111 111 3111 8111 22 23 ZZAAAA KUKAAA OOOOxx 7685 7291 1 1 5 5 85 685 1685 2685 7685 170 171 PJAAAA LUKAAA VVVVxx 3773 7292 1 1 3 13 73 773 1773 3773 3773 146 147 DPAAAA MUKAAA AAAAxx 7172 7293 0 0 2 12 72 172 1172 2172 7172 144 145 WPAAAA NUKAAA HHHHxx 1740 7294 0 0 0 0 40 740 1740 1740 1740 80 81 YOAAAA OUKAAA OOOOxx 5416 7295 0 0 6 16 16 416 1416 416 5416 32 33 IAAAAA PUKAAA VVVVxx 1823 7296 1 3 3 3 23 823 1823 1823 1823 46 47 DSAAAA QUKAAA AAAAxx 1668 7297 0 0 8 8 68 668 1668 1668 1668 136 137 EMAAAA RUKAAA HHHHxx 1795 7298 1 3 5 15 95 795 1795 1795 1795 190 191 BRAAAA SUKAAA OOOOxx 8599 7299 1 3 9 19 99 599 599 3599 8599 198 199 TSAAAA TUKAAA VVVVxx 5542 7300 0 2 2 2 42 542 1542 542 5542 84 85 EFAAAA UUKAAA AAAAxx 5658 7301 0 2 8 18 58 658 1658 658 5658 116 117 QJAAAA VUKAAA HHHHxx 9824 7302 0 0 4 4 24 824 1824 4824 9824 48 49 WNAAAA WUKAAA OOOOxx 19 7303 1 3 9 19 19 19 19 19 19 38 39 TAAAAA XUKAAA VVVVxx 9344 7304 0 0 4 4 44 344 1344 4344 9344 88 89 KVAAAA YUKAAA AAAAxx 5900 7305 0 0 0 0 0 900 1900 900 5900 0 1 YSAAAA ZUKAAA HHHHxx 7818 7306 0 2 8 18 18 818 1818 2818 7818 36 37 SOAAAA AVKAAA OOOOxx 8377 7307 1 1 7 17 77 377 377 3377 8377 154 155 FKAAAA BVKAAA VVVVxx 6886 7308 0 2 6 6 86 886 886 1886 6886 172 173 WEAAAA CVKAAA AAAAxx 3201 7309 1 1 1 1 1 201 1201 3201 3201 2 3 DTAAAA DVKAAA HHHHxx 87 7310 1 3 7 7 87 87 87 87 87 174 175 JDAAAA EVKAAA OOOOxx 1089 7311 1 1 9 9 89 89 1089 1089 1089 178 179 XPAAAA FVKAAA VVVVxx 3948 7312 0 0 8 8 48 948 1948 3948 3948 96 97 WVAAAA GVKAAA AAAAxx 6383 7313 1 3 3 3 83 383 383 1383 6383 166 167 NLAAAA HVKAAA HHHHxx 837 7314 1 1 7 17 37 837 837 837 837 74 75 FGAAAA IVKAAA OOOOxx 6285 7315 1 1 5 5 85 285 285 1285 6285 170 171 THAAAA JVKAAA VVVVxx 78 7316 0 2 8 18 78 78 78 78 78 156 157 ADAAAA KVKAAA AAAAxx 4389 7317 1 1 9 9 89 389 389 4389 4389 178 179 VMAAAA LVKAAA HHHHxx 4795 7318 1 3 5 15 95 795 795 4795 4795 190 191 LCAAAA MVKAAA OOOOxx 9369 7319 1 1 9 9 69 369 1369 4369 9369 138 139 JWAAAA NVKAAA VVVVxx 69 7320 1 1 9 9 69 69 69 69 69 138 139 RCAAAA OVKAAA AAAAxx 7689 7321 1 1 9 9 89 689 1689 2689 7689 178 179 TJAAAA PVKAAA HHHHxx 5642 7322 0 2 2 2 42 642 1642 642 5642 84 85 AJAAAA QVKAAA OOOOxx 2348 7323 0 0 8 8 48 348 348 2348 2348 96 97 IMAAAA RVKAAA VVVVxx 9308 7324 0 0 8 8 8 308 1308 4308 9308 16 17 AUAAAA SVKAAA AAAAxx 9093 7325 1 1 3 13 93 93 1093 4093 9093 186 187 TLAAAA TVKAAA HHHHxx 1199 7326 1 3 9 19 99 199 1199 1199 1199 198 199 DUAAAA UVKAAA OOOOxx 307 7327 1 3 7 7 7 307 307 307 307 14 15 VLAAAA VVKAAA VVVVxx 3814 7328 0 2 4 14 14 814 1814 3814 3814 28 29 SQAAAA WVKAAA AAAAxx 8817 7329 1 1 7 17 17 817 817 3817 8817 34 35 DBAAAA XVKAAA HHHHxx 2329 7330 1 1 9 9 29 329 329 2329 2329 58 59 PLAAAA YVKAAA OOOOxx 2932 7331 0 0 2 12 32 932 932 2932 2932 64 65 UIAAAA ZVKAAA VVVVxx 1986 7332 0 2 6 6 86 986 1986 1986 1986 172 173 KYAAAA AWKAAA AAAAxx 5279 7333 1 3 9 19 79 279 1279 279 5279 158 159 BVAAAA BWKAAA HHHHxx 5357 7334 1 1 7 17 57 357 1357 357 5357 114 115 BYAAAA CWKAAA OOOOxx 6778 7335 0 2 8 18 78 778 778 1778 6778 156 157 SAAAAA DWKAAA VVVVxx 2773 7336 1 1 3 13 73 773 773 2773 2773 146 147 RCAAAA EWKAAA AAAAxx 244 7337 0 0 4 4 44 244 244 244 244 88 89 KJAAAA FWKAAA HHHHxx 6900 7338 0 0 0 0 0 900 900 1900 6900 0 1 KFAAAA GWKAAA OOOOxx 4739 7339 1 3 9 19 39 739 739 4739 4739 78 79 HAAAAA HWKAAA VVVVxx 3217 7340 1 1 7 17 17 217 1217 3217 3217 34 35 TTAAAA IWKAAA AAAAxx 7563 7341 1 3 3 3 63 563 1563 2563 7563 126 127 XEAAAA JWKAAA HHHHxx 1807 7342 1 3 7 7 7 807 1807 1807 1807 14 15 NRAAAA KWKAAA OOOOxx 4199 7343 1 3 9 19 99 199 199 4199 4199 198 199 NFAAAA LWKAAA VVVVxx 1077 7344 1 1 7 17 77 77 1077 1077 1077 154 155 LPAAAA MWKAAA AAAAxx 8348 7345 0 0 8 8 48 348 348 3348 8348 96 97 CJAAAA NWKAAA HHHHxx 841 7346 1 1 1 1 41 841 841 841 841 82 83 JGAAAA OWKAAA OOOOxx 8154 7347 0 2 4 14 54 154 154 3154 8154 108 109 QBAAAA PWKAAA VVVVxx 5261 7348 1 1 1 1 61 261 1261 261 5261 122 123 JUAAAA QWKAAA AAAAxx 1950 7349 0 2 0 10 50 950 1950 1950 1950 100 101 AXAAAA RWKAAA HHHHxx 8472 7350 0 0 2 12 72 472 472 3472 8472 144 145 WNAAAA SWKAAA OOOOxx 8745 7351 1 1 5 5 45 745 745 3745 8745 90 91 JYAAAA TWKAAA VVVVxx 8715 7352 1 3 5 15 15 715 715 3715 8715 30 31 FXAAAA UWKAAA AAAAxx 9708 7353 0 0 8 8 8 708 1708 4708 9708 16 17 KJAAAA VWKAAA HHHHxx 5860 7354 0 0 0 0 60 860 1860 860 5860 120 121 KRAAAA WWKAAA OOOOxx 9142 7355 0 2 2 2 42 142 1142 4142 9142 84 85 QNAAAA XWKAAA VVVVxx 6582 7356 0 2 2 2 82 582 582 1582 6582 164 165 ETAAAA YWKAAA AAAAxx 1255 7357 1 3 5 15 55 255 1255 1255 1255 110 111 HWAAAA ZWKAAA HHHHxx 6459 7358 1 3 9 19 59 459 459 1459 6459 118 119 LOAAAA AXKAAA OOOOxx 6327 7359 1 3 7 7 27 327 327 1327 6327 54 55 JJAAAA BXKAAA VVVVxx 4692 7360 0 0 2 12 92 692 692 4692 4692 184 185 MYAAAA CXKAAA AAAAxx 3772 7361 0 0 2 12 72 772 1772 3772 3772 144 145 CPAAAA DXKAAA HHHHxx 4203 7362 1 3 3 3 3 203 203 4203 4203 6 7 RFAAAA EXKAAA OOOOxx 2946 7363 0 2 6 6 46 946 946 2946 2946 92 93 IJAAAA FXKAAA VVVVxx 3524 7364 0 0 4 4 24 524 1524 3524 3524 48 49 OFAAAA GXKAAA AAAAxx 8409 7365 1 1 9 9 9 409 409 3409 8409 18 19 LLAAAA HXKAAA HHHHxx 1824 7366 0 0 4 4 24 824 1824 1824 1824 48 49 ESAAAA IXKAAA OOOOxx 4637 7367 1 1 7 17 37 637 637 4637 4637 74 75 JWAAAA JXKAAA VVVVxx 589 7368 1 1 9 9 89 589 589 589 589 178 179 RWAAAA KXKAAA AAAAxx 484 7369 0 0 4 4 84 484 484 484 484 168 169 QSAAAA LXKAAA HHHHxx 8963 7370 1 3 3 3 63 963 963 3963 8963 126 127 TGAAAA MXKAAA OOOOxx 5502 7371 0 2 2 2 2 502 1502 502 5502 4 5 QDAAAA NXKAAA VVVVxx 6982 7372 0 2 2 2 82 982 982 1982 6982 164 165 OIAAAA OXKAAA AAAAxx 8029 7373 1 1 9 9 29 29 29 3029 8029 58 59 VWAAAA PXKAAA HHHHxx 4395 7374 1 3 5 15 95 395 395 4395 4395 190 191 BNAAAA QXKAAA OOOOxx 2595 7375 1 3 5 15 95 595 595 2595 2595 190 191 VVAAAA RXKAAA VVVVxx 2133 7376 1 1 3 13 33 133 133 2133 2133 66 67 BEAAAA SXKAAA AAAAxx 1414 7377 0 2 4 14 14 414 1414 1414 1414 28 29 KCAAAA TXKAAA HHHHxx 8201 7378 1 1 1 1 1 201 201 3201 8201 2 3 LDAAAA UXKAAA OOOOxx 4706 7379 0 2 6 6 6 706 706 4706 4706 12 13 AZAAAA VXKAAA VVVVxx 5310 7380 0 2 0 10 10 310 1310 310 5310 20 21 GWAAAA WXKAAA AAAAxx 7333 7381 1 1 3 13 33 333 1333 2333 7333 66 67 BWAAAA XXKAAA HHHHxx 9420 7382 0 0 0 0 20 420 1420 4420 9420 40 41 IYAAAA YXKAAA OOOOxx 1383 7383 1 3 3 3 83 383 1383 1383 1383 166 167 FBAAAA ZXKAAA VVVVxx 6225 7384 1 1 5 5 25 225 225 1225 6225 50 51 LFAAAA AYKAAA AAAAxx 2064 7385 0 0 4 4 64 64 64 2064 2064 128 129 KBAAAA BYKAAA HHHHxx 6700 7386 0 0 0 0 0 700 700 1700 6700 0 1 SXAAAA CYKAAA OOOOxx 1352 7387 0 0 2 12 52 352 1352 1352 1352 104 105 AAAAAA DYKAAA VVVVxx 4249 7388 1 1 9 9 49 249 249 4249 4249 98 99 LHAAAA EYKAAA AAAAxx 9429 7389 1 1 9 9 29 429 1429 4429 9429 58 59 RYAAAA FYKAAA HHHHxx 8090 7390 0 2 0 10 90 90 90 3090 8090 180 181 EZAAAA GYKAAA OOOOxx 5378 7391 0 2 8 18 78 378 1378 378 5378 156 157 WYAAAA HYKAAA VVVVxx 9085 7392 1 1 5 5 85 85 1085 4085 9085 170 171 LLAAAA IYKAAA AAAAxx 7468 7393 0 0 8 8 68 468 1468 2468 7468 136 137 GBAAAA JYKAAA HHHHxx 9955 7394 1 3 5 15 55 955 1955 4955 9955 110 111 XSAAAA KYKAAA OOOOxx 8692 7395 0 0 2 12 92 692 692 3692 8692 184 185 IWAAAA LYKAAA VVVVxx 1463 7396 1 3 3 3 63 463 1463 1463 1463 126 127 HEAAAA MYKAAA AAAAxx 3577 7397 1 1 7 17 77 577 1577 3577 3577 154 155 PHAAAA NYKAAA HHHHxx 5654 7398 0 2 4 14 54 654 1654 654 5654 108 109 MJAAAA OYKAAA OOOOxx 7955 7399 1 3 5 15 55 955 1955 2955 7955 110 111 ZTAAAA PYKAAA VVVVxx 4843 7400 1 3 3 3 43 843 843 4843 4843 86 87 HEAAAA QYKAAA AAAAxx 1776 7401 0 0 6 16 76 776 1776 1776 1776 152 153 IQAAAA RYKAAA HHHHxx 2223 7402 1 3 3 3 23 223 223 2223 2223 46 47 NHAAAA SYKAAA OOOOxx 8442 7403 0 2 2 2 42 442 442 3442 8442 84 85 SMAAAA TYKAAA VVVVxx 9738 7404 0 2 8 18 38 738 1738 4738 9738 76 77 OKAAAA UYKAAA AAAAxx 4867 7405 1 3 7 7 67 867 867 4867 4867 134 135 FFAAAA VYKAAA HHHHxx 2983 7406 1 3 3 3 83 983 983 2983 2983 166 167 TKAAAA WYKAAA OOOOxx 3300 7407 0 0 0 0 0 300 1300 3300 3300 0 1 YWAAAA XYKAAA VVVVxx 3815 7408 1 3 5 15 15 815 1815 3815 3815 30 31 TQAAAA YYKAAA AAAAxx 1779 7409 1 3 9 19 79 779 1779 1779 1779 158 159 LQAAAA ZYKAAA HHHHxx 1123 7410 1 3 3 3 23 123 1123 1123 1123 46 47 FRAAAA AZKAAA OOOOxx 4824 7411 0 0 4 4 24 824 824 4824 4824 48 49 ODAAAA BZKAAA VVVVxx 5407 7412 1 3 7 7 7 407 1407 407 5407 14 15 ZZAAAA CZKAAA AAAAxx 5123 7413 1 3 3 3 23 123 1123 123 5123 46 47 BPAAAA DZKAAA HHHHxx 2515 7414 1 3 5 15 15 515 515 2515 2515 30 31 TSAAAA EZKAAA OOOOxx 4781 7415 1 1 1 1 81 781 781 4781 4781 162 163 XBAAAA FZKAAA VVVVxx 7831 7416 1 3 1 11 31 831 1831 2831 7831 62 63 FPAAAA GZKAAA AAAAxx 6946 7417 0 2 6 6 46 946 946 1946 6946 92 93 EHAAAA HZKAAA HHHHxx 1215 7418 1 3 5 15 15 215 1215 1215 1215 30 31 TUAAAA IZKAAA OOOOxx 7783 7419 1 3 3 3 83 783 1783 2783 7783 166 167 JNAAAA JZKAAA VVVVxx 4532 7420 0 0 2 12 32 532 532 4532 4532 64 65 ISAAAA KZKAAA AAAAxx 9068 7421 0 0 8 8 68 68 1068 4068 9068 136 137 UKAAAA LZKAAA HHHHxx 7030 7422 0 2 0 10 30 30 1030 2030 7030 60 61 KKAAAA MZKAAA OOOOxx 436 7423 0 0 6 16 36 436 436 436 436 72 73 UQAAAA NZKAAA VVVVxx 6549 7424 1 1 9 9 49 549 549 1549 6549 98 99 XRAAAA OZKAAA AAAAxx 3348 7425 0 0 8 8 48 348 1348 3348 3348 96 97 UYAAAA PZKAAA HHHHxx 6229 7426 1 1 9 9 29 229 229 1229 6229 58 59 PFAAAA QZKAAA OOOOxx 3933 7427 1 1 3 13 33 933 1933 3933 3933 66 67 HVAAAA RZKAAA VVVVxx 1876 7428 0 0 6 16 76 876 1876 1876 1876 152 153 EUAAAA SZKAAA AAAAxx 8920 7429 0 0 0 0 20 920 920 3920 8920 40 41 CFAAAA TZKAAA HHHHxx 7926 7430 0 2 6 6 26 926 1926 2926 7926 52 53 WSAAAA UZKAAA OOOOxx 8805 7431 1 1 5 5 5 805 805 3805 8805 10 11 RAAAAA VZKAAA VVVVxx 6729 7432 1 1 9 9 29 729 729 1729 6729 58 59 VYAAAA WZKAAA AAAAxx 7397 7433 1 1 7 17 97 397 1397 2397 7397 194 195 NYAAAA XZKAAA HHHHxx 9303 7434 1 3 3 3 3 303 1303 4303 9303 6 7 VTAAAA YZKAAA OOOOxx 4255 7435 1 3 5 15 55 255 255 4255 4255 110 111 RHAAAA ZZKAAA VVVVxx 7229 7436 1 1 9 9 29 229 1229 2229 7229 58 59 BSAAAA AALAAA AAAAxx 854 7437 0 2 4 14 54 854 854 854 854 108 109 WGAAAA BALAAA HHHHxx 6723 7438 1 3 3 3 23 723 723 1723 6723 46 47 PYAAAA CALAAA OOOOxx 9597 7439 1 1 7 17 97 597 1597 4597 9597 194 195 DFAAAA DALAAA VVVVxx 6532 7440 0 0 2 12 32 532 532 1532 6532 64 65 GRAAAA EALAAA AAAAxx 2910 7441 0 2 0 10 10 910 910 2910 2910 20 21 YHAAAA FALAAA HHHHxx 6717 7442 1 1 7 17 17 717 717 1717 6717 34 35 JYAAAA GALAAA OOOOxx 1790 7443 0 2 0 10 90 790 1790 1790 1790 180 181 WQAAAA HALAAA VVVVxx 3761 7444 1 1 1 1 61 761 1761 3761 3761 122 123 ROAAAA IALAAA AAAAxx 1565 7445 1 1 5 5 65 565 1565 1565 1565 130 131 FIAAAA JALAAA HHHHxx 6205 7446 1 1 5 5 5 205 205 1205 6205 10 11 REAAAA KALAAA OOOOxx 2726 7447 0 2 6 6 26 726 726 2726 2726 52 53 WAAAAA LALAAA VVVVxx 799 7448 1 3 9 19 99 799 799 799 799 198 199 TEAAAA MALAAA AAAAxx 3540 7449 0 0 0 0 40 540 1540 3540 3540 80 81 EGAAAA NALAAA HHHHxx 5878 7450 0 2 8 18 78 878 1878 878 5878 156 157 CSAAAA OALAAA OOOOxx 2542 7451 0 2 2 2 42 542 542 2542 2542 84 85 UTAAAA PALAAA VVVVxx 4888 7452 0 0 8 8 88 888 888 4888 4888 176 177 AGAAAA QALAAA AAAAxx 5290 7453 0 2 0 10 90 290 1290 290 5290 180 181 MVAAAA RALAAA HHHHxx 7995 7454 1 3 5 15 95 995 1995 2995 7995 190 191 NVAAAA SALAAA OOOOxx 3519 7455 1 3 9 19 19 519 1519 3519 3519 38 39 JFAAAA TALAAA VVVVxx 3571 7456 1 3 1 11 71 571 1571 3571 3571 142 143 JHAAAA UALAAA AAAAxx 7854 7457 0 2 4 14 54 854 1854 2854 7854 108 109 CQAAAA VALAAA HHHHxx 5184 7458 0 0 4 4 84 184 1184 184 5184 168 169 KRAAAA WALAAA OOOOxx 3498 7459 0 2 8 18 98 498 1498 3498 3498 196 197 OEAAAA XALAAA VVVVxx 1264 7460 0 0 4 4 64 264 1264 1264 1264 128 129 QWAAAA YALAAA AAAAxx 3159 7461 1 3 9 19 59 159 1159 3159 3159 118 119 NRAAAA ZALAAA HHHHxx 5480 7462 0 0 0 0 80 480 1480 480 5480 160 161 UCAAAA ABLAAA OOOOxx 1706 7463 0 2 6 6 6 706 1706 1706 1706 12 13 QNAAAA BBLAAA VVVVxx 4540 7464 0 0 0 0 40 540 540 4540 4540 80 81 QSAAAA CBLAAA AAAAxx 2799 7465 1 3 9 19 99 799 799 2799 2799 198 199 RDAAAA DBLAAA HHHHxx 7389 7466 1 1 9 9 89 389 1389 2389 7389 178 179 FYAAAA EBLAAA OOOOxx 5565 7467 1 1 5 5 65 565 1565 565 5565 130 131 BGAAAA FBLAAA VVVVxx 3896 7468 0 0 6 16 96 896 1896 3896 3896 192 193 WTAAAA GBLAAA AAAAxx 2100 7469 0 0 0 0 0 100 100 2100 2100 0 1 UCAAAA HBLAAA HHHHxx 3507 7470 1 3 7 7 7 507 1507 3507 3507 14 15 XEAAAA IBLAAA OOOOxx 7971 7471 1 3 1 11 71 971 1971 2971 7971 142 143 PUAAAA JBLAAA VVVVxx 2312 7472 0 0 2 12 12 312 312 2312 2312 24 25 YKAAAA KBLAAA AAAAxx 2494 7473 0 2 4 14 94 494 494 2494 2494 188 189 YRAAAA LBLAAA HHHHxx 2474 7474 0 2 4 14 74 474 474 2474 2474 148 149 ERAAAA MBLAAA OOOOxx 3136 7475 0 0 6 16 36 136 1136 3136 3136 72 73 QQAAAA NBLAAA VVVVxx 7242 7476 0 2 2 2 42 242 1242 2242 7242 84 85 OSAAAA OBLAAA AAAAxx 9430 7477 0 2 0 10 30 430 1430 4430 9430 60 61 SYAAAA PBLAAA HHHHxx 1052 7478 0 0 2 12 52 52 1052 1052 1052 104 105 MOAAAA QBLAAA OOOOxx 4172 7479 0 0 2 12 72 172 172 4172 4172 144 145 MEAAAA RBLAAA VVVVxx 970 7480 0 2 0 10 70 970 970 970 970 140 141 ILAAAA SBLAAA AAAAxx 882 7481 0 2 2 2 82 882 882 882 882 164 165 YHAAAA TBLAAA HHHHxx 9799 7482 1 3 9 19 99 799 1799 4799 9799 198 199 XMAAAA UBLAAA OOOOxx 5850 7483 0 2 0 10 50 850 1850 850 5850 100 101 ARAAAA VBLAAA VVVVxx 9473 7484 1 1 3 13 73 473 1473 4473 9473 146 147 JAAAAA WBLAAA AAAAxx 8635 7485 1 3 5 15 35 635 635 3635 8635 70 71 DUAAAA XBLAAA HHHHxx 2349 7486 1 1 9 9 49 349 349 2349 2349 98 99 JMAAAA YBLAAA OOOOxx 2270 7487 0 2 0 10 70 270 270 2270 2270 140 141 IJAAAA ZBLAAA VVVVxx 7887 7488 1 3 7 7 87 887 1887 2887 7887 174 175 JRAAAA ACLAAA AAAAxx 3091 7489 1 3 1 11 91 91 1091 3091 3091 182 183 XOAAAA BCLAAA HHHHxx 3728 7490 0 0 8 8 28 728 1728 3728 3728 56 57 KNAAAA CCLAAA OOOOxx 3658 7491 0 2 8 18 58 658 1658 3658 3658 116 117 SKAAAA DCLAAA VVVVxx 5975 7492 1 3 5 15 75 975 1975 975 5975 150 151 VVAAAA ECLAAA AAAAxx 332 7493 0 0 2 12 32 332 332 332 332 64 65 UMAAAA FCLAAA HHHHxx 7990 7494 0 2 0 10 90 990 1990 2990 7990 180 181 IVAAAA GCLAAA OOOOxx 8688 7495 0 0 8 8 88 688 688 3688 8688 176 177 EWAAAA HCLAAA VVVVxx 9601 7496 1 1 1 1 1 601 1601 4601 9601 2 3 HFAAAA ICLAAA AAAAxx 8401 7497 1 1 1 1 1 401 401 3401 8401 2 3 DLAAAA JCLAAA HHHHxx 8093 7498 1 1 3 13 93 93 93 3093 8093 186 187 HZAAAA KCLAAA OOOOxx 4278 7499 0 2 8 18 78 278 278 4278 4278 156 157 OIAAAA LCLAAA VVVVxx 5467 7500 1 3 7 7 67 467 1467 467 5467 134 135 HCAAAA MCLAAA AAAAxx 3137 7501 1 1 7 17 37 137 1137 3137 3137 74 75 RQAAAA NCLAAA HHHHxx 204 7502 0 0 4 4 4 204 204 204 204 8 9 WHAAAA OCLAAA OOOOxx 8224 7503 0 0 4 4 24 224 224 3224 8224 48 49 IEAAAA PCLAAA VVVVxx 2944 7504 0 0 4 4 44 944 944 2944 2944 88 89 GJAAAA QCLAAA AAAAxx 7593 7505 1 1 3 13 93 593 1593 2593 7593 186 187 BGAAAA RCLAAA HHHHxx 814 7506 0 2 4 14 14 814 814 814 814 28 29 IFAAAA SCLAAA OOOOxx 8047 7507 1 3 7 7 47 47 47 3047 8047 94 95 NXAAAA TCLAAA VVVVxx 7802 7508 0 2 2 2 2 802 1802 2802 7802 4 5 COAAAA UCLAAA AAAAxx 901 7509 1 1 1 1 1 901 901 901 901 2 3 RIAAAA VCLAAA HHHHxx 6168 7510 0 0 8 8 68 168 168 1168 6168 136 137 GDAAAA WCLAAA OOOOxx 2950 7511 0 2 0 10 50 950 950 2950 2950 100 101 MJAAAA XCLAAA VVVVxx 5393 7512 1 1 3 13 93 393 1393 393 5393 186 187 LZAAAA YCLAAA AAAAxx 3585 7513 1 1 5 5 85 585 1585 3585 3585 170 171 XHAAAA ZCLAAA HHHHxx 9392 7514 0 0 2 12 92 392 1392 4392 9392 184 185 GXAAAA ADLAAA OOOOxx 8314 7515 0 2 4 14 14 314 314 3314 8314 28 29 UHAAAA BDLAAA VVVVxx 9972 7516 0 0 2 12 72 972 1972 4972 9972 144 145 OTAAAA CDLAAA AAAAxx 9130 7517 0 2 0 10 30 130 1130 4130 9130 60 61 ENAAAA DDLAAA HHHHxx 975 7518 1 3 5 15 75 975 975 975 975 150 151 NLAAAA EDLAAA OOOOxx 5720 7519 0 0 0 0 20 720 1720 720 5720 40 41 AMAAAA FDLAAA VVVVxx 3769 7520 1 1 9 9 69 769 1769 3769 3769 138 139 ZOAAAA GDLAAA AAAAxx 5303 7521 1 3 3 3 3 303 1303 303 5303 6 7 ZVAAAA HDLAAA HHHHxx 6564 7522 0 0 4 4 64 564 564 1564 6564 128 129 MSAAAA IDLAAA OOOOxx 7855 7523 1 3 5 15 55 855 1855 2855 7855 110 111 DQAAAA JDLAAA VVVVxx 8153 7524 1 1 3 13 53 153 153 3153 8153 106 107 PBAAAA KDLAAA AAAAxx 2292 7525 0 0 2 12 92 292 292 2292 2292 184 185 EKAAAA LDLAAA HHHHxx 3156 7526 0 0 6 16 56 156 1156 3156 3156 112 113 KRAAAA MDLAAA OOOOxx 6580 7527 0 0 0 0 80 580 580 1580 6580 160 161 CTAAAA NDLAAA VVVVxx 5324 7528 0 0 4 4 24 324 1324 324 5324 48 49 UWAAAA ODLAAA AAAAxx 8871 7529 1 3 1 11 71 871 871 3871 8871 142 143 FDAAAA PDLAAA HHHHxx 2543 7530 1 3 3 3 43 543 543 2543 2543 86 87 VTAAAA QDLAAA OOOOxx 7857 7531 1 1 7 17 57 857 1857 2857 7857 114 115 FQAAAA RDLAAA VVVVxx 4084 7532 0 0 4 4 84 84 84 4084 4084 168 169 CBAAAA SDLAAA AAAAxx 9887 7533 1 3 7 7 87 887 1887 4887 9887 174 175 HQAAAA TDLAAA HHHHxx 6940 7534 0 0 0 0 40 940 940 1940 6940 80 81 YGAAAA UDLAAA OOOOxx 3415 7535 1 3 5 15 15 415 1415 3415 3415 30 31 JBAAAA VDLAAA VVVVxx 5012 7536 0 0 2 12 12 12 1012 12 5012 24 25 UKAAAA WDLAAA AAAAxx 3187 7537 1 3 7 7 87 187 1187 3187 3187 174 175 PSAAAA XDLAAA HHHHxx 8556 7538 0 0 6 16 56 556 556 3556 8556 112 113 CRAAAA YDLAAA OOOOxx 7966 7539 0 2 6 6 66 966 1966 2966 7966 132 133 KUAAAA ZDLAAA VVVVxx 7481 7540 1 1 1 1 81 481 1481 2481 7481 162 163 TBAAAA AELAAA AAAAxx 8524 7541 0 0 4 4 24 524 524 3524 8524 48 49 WPAAAA BELAAA HHHHxx 3021 7542 1 1 1 1 21 21 1021 3021 3021 42 43 FMAAAA CELAAA OOOOxx 6045 7543 1 1 5 5 45 45 45 1045 6045 90 91 NYAAAA DELAAA VVVVxx 8022 7544 0 2 2 2 22 22 22 3022 8022 44 45 OWAAAA EELAAA AAAAxx 3626 7545 0 2 6 6 26 626 1626 3626 3626 52 53 MJAAAA FELAAA HHHHxx 1030 7546 0 2 0 10 30 30 1030 1030 1030 60 61 QNAAAA GELAAA OOOOxx 8903 7547 1 3 3 3 3 903 903 3903 8903 6 7 LEAAAA HELAAA VVVVxx 7488 7548 0 0 8 8 88 488 1488 2488 7488 176 177 ACAAAA IELAAA AAAAxx 9293 7549 1 1 3 13 93 293 1293 4293 9293 186 187 LTAAAA JELAAA HHHHxx 4586 7550 0 2 6 6 86 586 586 4586 4586 172 173 KUAAAA KELAAA OOOOxx 9282 7551 0 2 2 2 82 282 1282 4282 9282 164 165 ATAAAA LELAAA VVVVxx 1948 7552 0 0 8 8 48 948 1948 1948 1948 96 97 YWAAAA MELAAA AAAAxx 2534 7553 0 2 4 14 34 534 534 2534 2534 68 69 MTAAAA NELAAA HHHHxx 1150 7554 0 2 0 10 50 150 1150 1150 1150 100 101 GSAAAA OELAAA OOOOxx 4931 7555 1 3 1 11 31 931 931 4931 4931 62 63 RHAAAA PELAAA VVVVxx 2866 7556 0 2 6 6 66 866 866 2866 2866 132 133 GGAAAA QELAAA AAAAxx 6172 7557 0 0 2 12 72 172 172 1172 6172 144 145 KDAAAA RELAAA HHHHxx 4819 7558 1 3 9 19 19 819 819 4819 4819 38 39 JDAAAA SELAAA OOOOxx 569 7559 1 1 9 9 69 569 569 569 569 138 139 XVAAAA TELAAA VVVVxx 1146 7560 0 2 6 6 46 146 1146 1146 1146 92 93 CSAAAA UELAAA AAAAxx 3062 7561 0 2 2 2 62 62 1062 3062 3062 124 125 UNAAAA VELAAA HHHHxx 7690 7562 0 2 0 10 90 690 1690 2690 7690 180 181 UJAAAA WELAAA OOOOxx 8611 7563 1 3 1 11 11 611 611 3611 8611 22 23 FTAAAA XELAAA VVVVxx 1142 7564 0 2 2 2 42 142 1142 1142 1142 84 85 YRAAAA YELAAA AAAAxx 1193 7565 1 1 3 13 93 193 1193 1193 1193 186 187 XTAAAA ZELAAA HHHHxx 2507 7566 1 3 7 7 7 507 507 2507 2507 14 15 LSAAAA AFLAAA OOOOxx 1043 7567 1 3 3 3 43 43 1043 1043 1043 86 87 DOAAAA BFLAAA VVVVxx 7472 7568 0 0 2 12 72 472 1472 2472 7472 144 145 KBAAAA CFLAAA AAAAxx 1817 7569 1 1 7 17 17 817 1817 1817 1817 34 35 XRAAAA DFLAAA HHHHxx 3868 7570 0 0 8 8 68 868 1868 3868 3868 136 137 USAAAA EFLAAA OOOOxx 9031 7571 1 3 1 11 31 31 1031 4031 9031 62 63 JJAAAA FFLAAA VVVVxx 7254 7572 0 2 4 14 54 254 1254 2254 7254 108 109 ATAAAA GFLAAA AAAAxx 5030 7573 0 2 0 10 30 30 1030 30 5030 60 61 MLAAAA HFLAAA HHHHxx 6594 7574 0 2 4 14 94 594 594 1594 6594 188 189 QTAAAA IFLAAA OOOOxx 6862 7575 0 2 2 2 62 862 862 1862 6862 124 125 YDAAAA JFLAAA VVVVxx 1994 7576 0 2 4 14 94 994 1994 1994 1994 188 189 SYAAAA KFLAAA AAAAxx 9017 7577 1 1 7 17 17 17 1017 4017 9017 34 35 VIAAAA LFLAAA HHHHxx 5716 7578 0 0 6 16 16 716 1716 716 5716 32 33 WLAAAA MFLAAA OOOOxx 1900 7579 0 0 0 0 0 900 1900 1900 1900 0 1 CVAAAA NFLAAA VVVVxx 120 7580 0 0 0 0 20 120 120 120 120 40 41 QEAAAA OFLAAA AAAAxx 9003 7581 1 3 3 3 3 3 1003 4003 9003 6 7 HIAAAA PFLAAA HHHHxx 4178 7582 0 2 8 18 78 178 178 4178 4178 156 157 SEAAAA QFLAAA OOOOxx 8777 7583 1 1 7 17 77 777 777 3777 8777 154 155 PZAAAA RFLAAA VVVVxx 3653 7584 1 1 3 13 53 653 1653 3653 3653 106 107 NKAAAA SFLAAA AAAAxx 1137 7585 1 1 7 17 37 137 1137 1137 1137 74 75 TRAAAA TFLAAA HHHHxx 6362 7586 0 2 2 2 62 362 362 1362 6362 124 125 SKAAAA UFLAAA OOOOxx 8537 7587 1 1 7 17 37 537 537 3537 8537 74 75 JQAAAA VFLAAA VVVVxx 1590 7588 0 2 0 10 90 590 1590 1590 1590 180 181 EJAAAA WFLAAA AAAAxx 374 7589 0 2 4 14 74 374 374 374 374 148 149 KOAAAA XFLAAA HHHHxx 2597 7590 1 1 7 17 97 597 597 2597 2597 194 195 XVAAAA YFLAAA OOOOxx 8071 7591 1 3 1 11 71 71 71 3071 8071 142 143 LYAAAA ZFLAAA VVVVxx 9009 7592 1 1 9 9 9 9 1009 4009 9009 18 19 NIAAAA AGLAAA AAAAxx 1978 7593 0 2 8 18 78 978 1978 1978 1978 156 157 CYAAAA BGLAAA HHHHxx 1541 7594 1 1 1 1 41 541 1541 1541 1541 82 83 HHAAAA CGLAAA OOOOxx 4998 7595 0 2 8 18 98 998 998 4998 4998 196 197 GKAAAA DGLAAA VVVVxx 1649 7596 1 1 9 9 49 649 1649 1649 1649 98 99 LLAAAA EGLAAA AAAAxx 5426 7597 0 2 6 6 26 426 1426 426 5426 52 53 SAAAAA FGLAAA HHHHxx 1492 7598 0 0 2 12 92 492 1492 1492 1492 184 185 KFAAAA GGLAAA OOOOxx 9622 7599 0 2 2 2 22 622 1622 4622 9622 44 45 CGAAAA HGLAAA VVVVxx 701 7600 1 1 1 1 1 701 701 701 701 2 3 ZAAAAA IGLAAA AAAAxx 2781 7601 1 1 1 1 81 781 781 2781 2781 162 163 ZCAAAA JGLAAA HHHHxx 3982 7602 0 2 2 2 82 982 1982 3982 3982 164 165 EXAAAA KGLAAA OOOOxx 7259 7603 1 3 9 19 59 259 1259 2259 7259 118 119 FTAAAA LGLAAA VVVVxx 9868 7604 0 0 8 8 68 868 1868 4868 9868 136 137 OPAAAA MGLAAA AAAAxx 564 7605 0 0 4 4 64 564 564 564 564 128 129 SVAAAA NGLAAA HHHHxx 6315 7606 1 3 5 15 15 315 315 1315 6315 30 31 XIAAAA OGLAAA OOOOxx 9092 7607 0 0 2 12 92 92 1092 4092 9092 184 185 SLAAAA PGLAAA VVVVxx 8237 7608 1 1 7 17 37 237 237 3237 8237 74 75 VEAAAA QGLAAA AAAAxx 1513 7609 1 1 3 13 13 513 1513 1513 1513 26 27 FGAAAA RGLAAA HHHHxx 1922 7610 0 2 2 2 22 922 1922 1922 1922 44 45 YVAAAA SGLAAA OOOOxx 5396 7611 0 0 6 16 96 396 1396 396 5396 192 193 OZAAAA TGLAAA VVVVxx 2485 7612 1 1 5 5 85 485 485 2485 2485 170 171 PRAAAA UGLAAA AAAAxx 5774 7613 0 2 4 14 74 774 1774 774 5774 148 149 COAAAA VGLAAA HHHHxx 3983 7614 1 3 3 3 83 983 1983 3983 3983 166 167 FXAAAA WGLAAA OOOOxx 221 7615 1 1 1 1 21 221 221 221 221 42 43 NIAAAA XGLAAA VVVVxx 8662 7616 0 2 2 2 62 662 662 3662 8662 124 125 EVAAAA YGLAAA AAAAxx 2456 7617 0 0 6 16 56 456 456 2456 2456 112 113 MQAAAA ZGLAAA HHHHxx 9736 7618 0 0 6 16 36 736 1736 4736 9736 72 73 MKAAAA AHLAAA OOOOxx 8936 7619 0 0 6 16 36 936 936 3936 8936 72 73 SFAAAA BHLAAA VVVVxx 5395 7620 1 3 5 15 95 395 1395 395 5395 190 191 NZAAAA CHLAAA AAAAxx 9523 7621 1 3 3 3 23 523 1523 4523 9523 46 47 HCAAAA DHLAAA HHHHxx 6980 7622 0 0 0 0 80 980 980 1980 6980 160 161 MIAAAA EHLAAA OOOOxx 2091 7623 1 3 1 11 91 91 91 2091 2091 182 183 LCAAAA FHLAAA VVVVxx 6807 7624 1 3 7 7 7 807 807 1807 6807 14 15 VBAAAA GHLAAA AAAAxx 8818 7625 0 2 8 18 18 818 818 3818 8818 36 37 EBAAAA HHLAAA HHHHxx 5298 7626 0 2 8 18 98 298 1298 298 5298 196 197 UVAAAA IHLAAA OOOOxx 1726 7627 0 2 6 6 26 726 1726 1726 1726 52 53 KOAAAA JHLAAA VVVVxx 3878 7628 0 2 8 18 78 878 1878 3878 3878 156 157 ETAAAA KHLAAA AAAAxx 8700 7629 0 0 0 0 0 700 700 3700 8700 0 1 QWAAAA LHLAAA HHHHxx 5201 7630 1 1 1 1 1 201 1201 201 5201 2 3 BSAAAA MHLAAA OOOOxx 3936 7631 0 0 6 16 36 936 1936 3936 3936 72 73 KVAAAA NHLAAA VVVVxx 776 7632 0 0 6 16 76 776 776 776 776 152 153 WDAAAA OHLAAA AAAAxx 5302 7633 0 2 2 2 2 302 1302 302 5302 4 5 YVAAAA PHLAAA HHHHxx 3595 7634 1 3 5 15 95 595 1595 3595 3595 190 191 HIAAAA QHLAAA OOOOxx 9061 7635 1 1 1 1 61 61 1061 4061 9061 122 123 NKAAAA RHLAAA VVVVxx 6261 7636 1 1 1 1 61 261 261 1261 6261 122 123 VGAAAA SHLAAA AAAAxx 8878 7637 0 2 8 18 78 878 878 3878 8878 156 157 MDAAAA THLAAA HHHHxx 3312 7638 0 0 2 12 12 312 1312 3312 3312 24 25 KXAAAA UHLAAA OOOOxx 9422 7639 0 2 2 2 22 422 1422 4422 9422 44 45 KYAAAA VHLAAA VVVVxx 7321 7640 1 1 1 1 21 321 1321 2321 7321 42 43 PVAAAA WHLAAA AAAAxx 3813 7641 1 1 3 13 13 813 1813 3813 3813 26 27 RQAAAA XHLAAA HHHHxx 5848 7642 0 0 8 8 48 848 1848 848 5848 96 97 YQAAAA YHLAAA OOOOxx 3535 7643 1 3 5 15 35 535 1535 3535 3535 70 71 ZFAAAA ZHLAAA VVVVxx 1040 7644 0 0 0 0 40 40 1040 1040 1040 80 81 AOAAAA AILAAA AAAAxx 8572 7645 0 0 2 12 72 572 572 3572 8572 144 145 SRAAAA BILAAA HHHHxx 5435 7646 1 3 5 15 35 435 1435 435 5435 70 71 BBAAAA CILAAA OOOOxx 8199 7647 1 3 9 19 99 199 199 3199 8199 198 199 JDAAAA DILAAA VVVVxx 8775 7648 1 3 5 15 75 775 775 3775 8775 150 151 NZAAAA EILAAA AAAAxx 7722 7649 0 2 2 2 22 722 1722 2722 7722 44 45 ALAAAA FILAAA HHHHxx 3549 7650 1 1 9 9 49 549 1549 3549 3549 98 99 NGAAAA GILAAA OOOOxx 2578 7651 0 2 8 18 78 578 578 2578 2578 156 157 EVAAAA HILAAA VVVVxx 1695 7652 1 3 5 15 95 695 1695 1695 1695 190 191 FNAAAA IILAAA AAAAxx 1902 7653 0 2 2 2 2 902 1902 1902 1902 4 5 EVAAAA JILAAA HHHHxx 6058 7654 0 2 8 18 58 58 58 1058 6058 116 117 AZAAAA KILAAA OOOOxx 6591 7655 1 3 1 11 91 591 591 1591 6591 182 183 NTAAAA LILAAA VVVVxx 7962 7656 0 2 2 2 62 962 1962 2962 7962 124 125 GUAAAA MILAAA AAAAxx 5612 7657 0 0 2 12 12 612 1612 612 5612 24 25 WHAAAA NILAAA HHHHxx 3341 7658 1 1 1 1 41 341 1341 3341 3341 82 83 NYAAAA OILAAA OOOOxx 5460 7659 0 0 0 0 60 460 1460 460 5460 120 121 ACAAAA PILAAA VVVVxx 2368 7660 0 0 8 8 68 368 368 2368 2368 136 137 CNAAAA QILAAA AAAAxx 8646 7661 0 2 6 6 46 646 646 3646 8646 92 93 OUAAAA RILAAA HHHHxx 4987 7662 1 3 7 7 87 987 987 4987 4987 174 175 VJAAAA SILAAA OOOOxx 9018 7663 0 2 8 18 18 18 1018 4018 9018 36 37 WIAAAA TILAAA VVVVxx 8685 7664 1 1 5 5 85 685 685 3685 8685 170 171 BWAAAA UILAAA AAAAxx 694 7665 0 2 4 14 94 694 694 694 694 188 189 SAAAAA VILAAA HHHHxx 2012 7666 0 0 2 12 12 12 12 2012 2012 24 25 KZAAAA WILAAA OOOOxx 2417 7667 1 1 7 17 17 417 417 2417 2417 34 35 ZOAAAA XILAAA VVVVxx 4022 7668 0 2 2 2 22 22 22 4022 4022 44 45 SYAAAA YILAAA AAAAxx 5935 7669 1 3 5 15 35 935 1935 935 5935 70 71 HUAAAA ZILAAA HHHHxx 1656 7670 0 0 6 16 56 656 1656 1656 1656 112 113 SLAAAA AJLAAA OOOOxx 6195 7671 1 3 5 15 95 195 195 1195 6195 190 191 HEAAAA BJLAAA VVVVxx 3057 7672 1 1 7 17 57 57 1057 3057 3057 114 115 PNAAAA CJLAAA AAAAxx 2852 7673 0 0 2 12 52 852 852 2852 2852 104 105 SFAAAA DJLAAA HHHHxx 4634 7674 0 2 4 14 34 634 634 4634 4634 68 69 GWAAAA EJLAAA OOOOxx 1689 7675 1 1 9 9 89 689 1689 1689 1689 178 179 ZMAAAA FJLAAA VVVVxx 4102 7676 0 2 2 2 2 102 102 4102 4102 4 5 UBAAAA GJLAAA AAAAxx 3287 7677 1 3 7 7 87 287 1287 3287 3287 174 175 LWAAAA HJLAAA HHHHxx 5246 7678 0 2 6 6 46 246 1246 246 5246 92 93 UTAAAA IJLAAA OOOOxx 7450 7679 0 2 0 10 50 450 1450 2450 7450 100 101 OAAAAA JJLAAA VVVVxx 6548 7680 0 0 8 8 48 548 548 1548 6548 96 97 WRAAAA KJLAAA AAAAxx 379 7681 1 3 9 19 79 379 379 379 379 158 159 POAAAA LJLAAA HHHHxx 7435 7682 1 3 5 15 35 435 1435 2435 7435 70 71 ZZAAAA MJLAAA OOOOxx 2041 7683 1 1 1 1 41 41 41 2041 2041 82 83 NAAAAA NJLAAA VVVVxx 8462 7684 0 2 2 2 62 462 462 3462 8462 124 125 MNAAAA OJLAAA AAAAxx 9076 7685 0 0 6 16 76 76 1076 4076 9076 152 153 CLAAAA PJLAAA HHHHxx 761 7686 1 1 1 1 61 761 761 761 761 122 123 HDAAAA QJLAAA OOOOxx 795 7687 1 3 5 15 95 795 795 795 795 190 191 PEAAAA RJLAAA VVVVxx 1671 7688 1 3 1 11 71 671 1671 1671 1671 142 143 HMAAAA SJLAAA AAAAxx 695 7689 1 3 5 15 95 695 695 695 695 190 191 TAAAAA TJLAAA HHHHxx 4981 7690 1 1 1 1 81 981 981 4981 4981 162 163 PJAAAA UJLAAA OOOOxx 1211 7691 1 3 1 11 11 211 1211 1211 1211 22 23 PUAAAA VJLAAA VVVVxx 5914 7692 0 2 4 14 14 914 1914 914 5914 28 29 MTAAAA WJLAAA AAAAxx 9356 7693 0 0 6 16 56 356 1356 4356 9356 112 113 WVAAAA XJLAAA HHHHxx 1500 7694 0 0 0 0 0 500 1500 1500 1500 0 1 SFAAAA YJLAAA OOOOxx 3353 7695 1 1 3 13 53 353 1353 3353 3353 106 107 ZYAAAA ZJLAAA VVVVxx 1060 7696 0 0 0 0 60 60 1060 1060 1060 120 121 UOAAAA AKLAAA AAAAxx 7910 7697 0 2 0 10 10 910 1910 2910 7910 20 21 GSAAAA BKLAAA HHHHxx 1329 7698 1 1 9 9 29 329 1329 1329 1329 58 59 DZAAAA CKLAAA OOOOxx 6011 7699 1 3 1 11 11 11 11 1011 6011 22 23 FXAAAA DKLAAA VVVVxx 7146 7700 0 2 6 6 46 146 1146 2146 7146 92 93 WOAAAA EKLAAA AAAAxx 4602 7701 0 2 2 2 2 602 602 4602 4602 4 5 AVAAAA FKLAAA HHHHxx 6751 7702 1 3 1 11 51 751 751 1751 6751 102 103 RZAAAA GKLAAA OOOOxx 2666 7703 0 2 6 6 66 666 666 2666 2666 132 133 OYAAAA HKLAAA VVVVxx 2785 7704 1 1 5 5 85 785 785 2785 2785 170 171 DDAAAA IKLAAA AAAAxx 5851 7705 1 3 1 11 51 851 1851 851 5851 102 103 BRAAAA JKLAAA HHHHxx 2435 7706 1 3 5 15 35 435 435 2435 2435 70 71 RPAAAA KKLAAA OOOOxx 7429 7707 1 1 9 9 29 429 1429 2429 7429 58 59 TZAAAA LKLAAA VVVVxx 4241 7708 1 1 1 1 41 241 241 4241 4241 82 83 DHAAAA MKLAAA AAAAxx 5691 7709 1 3 1 11 91 691 1691 691 5691 182 183 XKAAAA NKLAAA HHHHxx 7731 7710 1 3 1 11 31 731 1731 2731 7731 62 63 JLAAAA OKLAAA OOOOxx 249 7711 1 1 9 9 49 249 249 249 249 98 99 PJAAAA PKLAAA VVVVxx 1731 7712 1 3 1 11 31 731 1731 1731 1731 62 63 POAAAA QKLAAA AAAAxx 8716 7713 0 0 6 16 16 716 716 3716 8716 32 33 GXAAAA RKLAAA HHHHxx 2670 7714 0 2 0 10 70 670 670 2670 2670 140 141 SYAAAA SKLAAA OOOOxx 4654 7715 0 2 4 14 54 654 654 4654 4654 108 109 AXAAAA TKLAAA VVVVxx 1027 7716 1 3 7 7 27 27 1027 1027 1027 54 55 NNAAAA UKLAAA AAAAxx 1099 7717 1 3 9 19 99 99 1099 1099 1099 198 199 HQAAAA VKLAAA HHHHxx 3617 7718 1 1 7 17 17 617 1617 3617 3617 34 35 DJAAAA WKLAAA OOOOxx 4330 7719 0 2 0 10 30 330 330 4330 4330 60 61 OKAAAA XKLAAA VVVVxx 9750 7720 0 2 0 10 50 750 1750 4750 9750 100 101 ALAAAA YKLAAA AAAAxx 467 7721 1 3 7 7 67 467 467 467 467 134 135 ZRAAAA ZKLAAA HHHHxx 8525 7722 1 1 5 5 25 525 525 3525 8525 50 51 XPAAAA ALLAAA OOOOxx 5990 7723 0 2 0 10 90 990 1990 990 5990 180 181 KWAAAA BLLAAA VVVVxx 4839 7724 1 3 9 19 39 839 839 4839 4839 78 79 DEAAAA CLLAAA AAAAxx 9914 7725 0 2 4 14 14 914 1914 4914 9914 28 29 IRAAAA DLLAAA HHHHxx 7047 7726 1 3 7 7 47 47 1047 2047 7047 94 95 BLAAAA ELLAAA OOOOxx 874 7727 0 2 4 14 74 874 874 874 874 148 149 QHAAAA FLLAAA VVVVxx 6061 7728 1 1 1 1 61 61 61 1061 6061 122 123 DZAAAA GLLAAA AAAAxx 5491 7729 1 3 1 11 91 491 1491 491 5491 182 183 FDAAAA HLLAAA HHHHxx 4344 7730 0 0 4 4 44 344 344 4344 4344 88 89 CLAAAA ILLAAA OOOOxx 1281 7731 1 1 1 1 81 281 1281 1281 1281 162 163 HXAAAA JLLAAA VVVVxx 3597 7732 1 1 7 17 97 597 1597 3597 3597 194 195 JIAAAA KLLAAA AAAAxx 4992 7733 0 0 2 12 92 992 992 4992 4992 184 185 AKAAAA LLLAAA HHHHxx 3849 7734 1 1 9 9 49 849 1849 3849 3849 98 99 BSAAAA MLLAAA OOOOxx 2655 7735 1 3 5 15 55 655 655 2655 2655 110 111 DYAAAA NLLAAA VVVVxx 147 7736 1 3 7 7 47 147 147 147 147 94 95 RFAAAA OLLAAA AAAAxx 9110 7737 0 2 0 10 10 110 1110 4110 9110 20 21 KMAAAA PLLAAA HHHHxx 1637 7738 1 1 7 17 37 637 1637 1637 1637 74 75 ZKAAAA QLLAAA OOOOxx 9826 7739 0 2 6 6 26 826 1826 4826 9826 52 53 YNAAAA RLLAAA VVVVxx 5957 7740 1 1 7 17 57 957 1957 957 5957 114 115 DVAAAA SLLAAA AAAAxx 6932 7741 0 0 2 12 32 932 932 1932 6932 64 65 QGAAAA TLLAAA HHHHxx 9684 7742 0 0 4 4 84 684 1684 4684 9684 168 169 MIAAAA ULLAAA OOOOxx 4653 7743 1 1 3 13 53 653 653 4653 4653 106 107 ZWAAAA VLLAAA VVVVxx 8065 7744 1 1 5 5 65 65 65 3065 8065 130 131 FYAAAA WLLAAA AAAAxx 1202 7745 0 2 2 2 2 202 1202 1202 1202 4 5 GUAAAA XLLAAA HHHHxx 9214 7746 0 2 4 14 14 214 1214 4214 9214 28 29 KQAAAA YLLAAA OOOOxx 196 7747 0 0 6 16 96 196 196 196 196 192 193 OHAAAA ZLLAAA VVVVxx 4486 7748 0 2 6 6 86 486 486 4486 4486 172 173 OQAAAA AMLAAA AAAAxx 2585 7749 1 1 5 5 85 585 585 2585 2585 170 171 LVAAAA BMLAAA HHHHxx 2464 7750 0 0 4 4 64 464 464 2464 2464 128 129 UQAAAA CMLAAA OOOOxx 3467 7751 1 3 7 7 67 467 1467 3467 3467 134 135 JDAAAA DMLAAA VVVVxx 9295 7752 1 3 5 15 95 295 1295 4295 9295 190 191 NTAAAA EMLAAA AAAAxx 517 7753 1 1 7 17 17 517 517 517 517 34 35 XTAAAA FMLAAA HHHHxx 6870 7754 0 2 0 10 70 870 870 1870 6870 140 141 GEAAAA GMLAAA OOOOxx 5732 7755 0 0 2 12 32 732 1732 732 5732 64 65 MMAAAA HMLAAA VVVVxx 9376 7756 0 0 6 16 76 376 1376 4376 9376 152 153 QWAAAA IMLAAA AAAAxx 838 7757 0 2 8 18 38 838 838 838 838 76 77 GGAAAA JMLAAA HHHHxx 9254 7758 0 2 4 14 54 254 1254 4254 9254 108 109 YRAAAA KMLAAA OOOOxx 8879 7759 1 3 9 19 79 879 879 3879 8879 158 159 NDAAAA LMLAAA VVVVxx 6281 7760 1 1 1 1 81 281 281 1281 6281 162 163 PHAAAA MMLAAA AAAAxx 8216 7761 0 0 6 16 16 216 216 3216 8216 32 33 AEAAAA NMLAAA HHHHxx 9213 7762 1 1 3 13 13 213 1213 4213 9213 26 27 JQAAAA OMLAAA OOOOxx 7234 7763 0 2 4 14 34 234 1234 2234 7234 68 69 GSAAAA PMLAAA VVVVxx 5692 7764 0 0 2 12 92 692 1692 692 5692 184 185 YKAAAA QMLAAA AAAAxx 693 7765 1 1 3 13 93 693 693 693 693 186 187 RAAAAA RMLAAA HHHHxx 9050 7766 0 2 0 10 50 50 1050 4050 9050 100 101 CKAAAA SMLAAA OOOOxx 3623 7767 1 3 3 3 23 623 1623 3623 3623 46 47 JJAAAA TMLAAA VVVVxx 2130 7768 0 2 0 10 30 130 130 2130 2130 60 61 YDAAAA UMLAAA AAAAxx 2514 7769 0 2 4 14 14 514 514 2514 2514 28 29 SSAAAA VMLAAA HHHHxx 1812 7770 0 0 2 12 12 812 1812 1812 1812 24 25 SRAAAA WMLAAA OOOOxx 9037 7771 1 1 7 17 37 37 1037 4037 9037 74 75 PJAAAA XMLAAA VVVVxx 5054 7772 0 2 4 14 54 54 1054 54 5054 108 109 KMAAAA YMLAAA AAAAxx 7801 7773 1 1 1 1 1 801 1801 2801 7801 2 3 BOAAAA ZMLAAA HHHHxx 7939 7774 1 3 9 19 39 939 1939 2939 7939 78 79 JTAAAA ANLAAA OOOOxx 7374 7775 0 2 4 14 74 374 1374 2374 7374 148 149 QXAAAA BNLAAA VVVVxx 1058 7776 0 2 8 18 58 58 1058 1058 1058 116 117 SOAAAA CNLAAA AAAAxx 1972 7777 0 0 2 12 72 972 1972 1972 1972 144 145 WXAAAA DNLAAA HHHHxx 3741 7778 1 1 1 1 41 741 1741 3741 3741 82 83 XNAAAA ENLAAA OOOOxx 2227 7779 1 3 7 7 27 227 227 2227 2227 54 55 RHAAAA FNLAAA VVVVxx 304 7780 0 0 4 4 4 304 304 304 304 8 9 SLAAAA GNLAAA AAAAxx 4914 7781 0 2 4 14 14 914 914 4914 4914 28 29 AHAAAA HNLAAA HHHHxx 2428 7782 0 0 8 8 28 428 428 2428 2428 56 57 KPAAAA INLAAA OOOOxx 6660 7783 0 0 0 0 60 660 660 1660 6660 120 121 EWAAAA JNLAAA VVVVxx 2676 7784 0 0 6 16 76 676 676 2676 2676 152 153 YYAAAA KNLAAA AAAAxx 2454 7785 0 2 4 14 54 454 454 2454 2454 108 109 KQAAAA LNLAAA HHHHxx 3798 7786 0 2 8 18 98 798 1798 3798 3798 196 197 CQAAAA MNLAAA OOOOxx 1341 7787 1 1 1 1 41 341 1341 1341 1341 82 83 PZAAAA NNLAAA VVVVxx 1611 7788 1 3 1 11 11 611 1611 1611 1611 22 23 ZJAAAA ONLAAA AAAAxx 2681 7789 1 1 1 1 81 681 681 2681 2681 162 163 DZAAAA PNLAAA HHHHxx 7292 7790 0 0 2 12 92 292 1292 2292 7292 184 185 MUAAAA QNLAAA OOOOxx 7775 7791 1 3 5 15 75 775 1775 2775 7775 150 151 BNAAAA RNLAAA VVVVxx 794 7792 0 2 4 14 94 794 794 794 794 188 189 OEAAAA SNLAAA AAAAxx 8709 7793 1 1 9 9 9 709 709 3709 8709 18 19 ZWAAAA TNLAAA HHHHxx 1901 7794 1 1 1 1 1 901 1901 1901 1901 2 3 DVAAAA UNLAAA OOOOxx 3089 7795 1 1 9 9 89 89 1089 3089 3089 178 179 VOAAAA VNLAAA VVVVxx 7797 7796 1 1 7 17 97 797 1797 2797 7797 194 195 XNAAAA WNLAAA AAAAxx 6070 7797 0 2 0 10 70 70 70 1070 6070 140 141 MZAAAA XNLAAA HHHHxx 2191 7798 1 3 1 11 91 191 191 2191 2191 182 183 HGAAAA YNLAAA OOOOxx 3497 7799 1 1 7 17 97 497 1497 3497 3497 194 195 NEAAAA ZNLAAA VVVVxx 8302 7800 0 2 2 2 2 302 302 3302 8302 4 5 IHAAAA AOLAAA AAAAxx 4365 7801 1 1 5 5 65 365 365 4365 4365 130 131 XLAAAA BOLAAA HHHHxx 3588 7802 0 0 8 8 88 588 1588 3588 3588 176 177 AIAAAA COLAAA OOOOxx 8292 7803 0 0 2 12 92 292 292 3292 8292 184 185 YGAAAA DOLAAA VVVVxx 4696 7804 0 0 6 16 96 696 696 4696 4696 192 193 QYAAAA EOLAAA AAAAxx 5641 7805 1 1 1 1 41 641 1641 641 5641 82 83 ZIAAAA FOLAAA HHHHxx 9386 7806 0 2 6 6 86 386 1386 4386 9386 172 173 AXAAAA GOLAAA OOOOxx 507 7807 1 3 7 7 7 507 507 507 507 14 15 NTAAAA HOLAAA VVVVxx 7201 7808 1 1 1 1 1 201 1201 2201 7201 2 3 ZQAAAA IOLAAA AAAAxx 7785 7809 1 1 5 5 85 785 1785 2785 7785 170 171 LNAAAA JOLAAA HHHHxx 463 7810 1 3 3 3 63 463 463 463 463 126 127 VRAAAA KOLAAA OOOOxx 6656 7811 0 0 6 16 56 656 656 1656 6656 112 113 AWAAAA LOLAAA VVVVxx 807 7812 1 3 7 7 7 807 807 807 807 14 15 BFAAAA MOLAAA AAAAxx 7278 7813 0 2 8 18 78 278 1278 2278 7278 156 157 YTAAAA NOLAAA HHHHxx 6237 7814 1 1 7 17 37 237 237 1237 6237 74 75 XFAAAA OOLAAA OOOOxx 7671 7815 1 3 1 11 71 671 1671 2671 7671 142 143 BJAAAA POLAAA VVVVxx 2235 7816 1 3 5 15 35 235 235 2235 2235 70 71 ZHAAAA QOLAAA AAAAxx 4042 7817 0 2 2 2 42 42 42 4042 4042 84 85 MZAAAA ROLAAA HHHHxx 5273 7818 1 1 3 13 73 273 1273 273 5273 146 147 VUAAAA SOLAAA OOOOxx 7557 7819 1 1 7 17 57 557 1557 2557 7557 114 115 REAAAA TOLAAA VVVVxx 4007 7820 1 3 7 7 7 7 7 4007 4007 14 15 DYAAAA UOLAAA AAAAxx 1428 7821 0 0 8 8 28 428 1428 1428 1428 56 57 YCAAAA VOLAAA HHHHxx 9739 7822 1 3 9 19 39 739 1739 4739 9739 78 79 PKAAAA WOLAAA OOOOxx 7836 7823 0 0 6 16 36 836 1836 2836 7836 72 73 KPAAAA XOLAAA VVVVxx 1777 7824 1 1 7 17 77 777 1777 1777 1777 154 155 JQAAAA YOLAAA AAAAxx 5192 7825 0 0 2 12 92 192 1192 192 5192 184 185 SRAAAA ZOLAAA HHHHxx 7236 7826 0 0 6 16 36 236 1236 2236 7236 72 73 ISAAAA APLAAA OOOOxx 1623 7827 1 3 3 3 23 623 1623 1623 1623 46 47 LKAAAA BPLAAA VVVVxx 8288 7828 0 0 8 8 88 288 288 3288 8288 176 177 UGAAAA CPLAAA AAAAxx 2827 7829 1 3 7 7 27 827 827 2827 2827 54 55 TEAAAA DPLAAA HHHHxx 458 7830 0 2 8 18 58 458 458 458 458 116 117 QRAAAA EPLAAA OOOOxx 1818 7831 0 2 8 18 18 818 1818 1818 1818 36 37 YRAAAA FPLAAA VVVVxx 6837 7832 1 1 7 17 37 837 837 1837 6837 74 75 ZCAAAA GPLAAA AAAAxx 7825 7833 1 1 5 5 25 825 1825 2825 7825 50 51 ZOAAAA HPLAAA HHHHxx 9146 7834 0 2 6 6 46 146 1146 4146 9146 92 93 UNAAAA IPLAAA OOOOxx 8451 7835 1 3 1 11 51 451 451 3451 8451 102 103 BNAAAA JPLAAA VVVVxx 6438 7836 0 2 8 18 38 438 438 1438 6438 76 77 QNAAAA KPLAAA AAAAxx 4020 7837 0 0 0 0 20 20 20 4020 4020 40 41 QYAAAA LPLAAA HHHHxx 4068 7838 0 0 8 8 68 68 68 4068 4068 136 137 MAAAAA MPLAAA OOOOxx 2411 7839 1 3 1 11 11 411 411 2411 2411 22 23 TOAAAA NPLAAA VVVVxx 6222 7840 0 2 2 2 22 222 222 1222 6222 44 45 IFAAAA OPLAAA AAAAxx 3164 7841 0 0 4 4 64 164 1164 3164 3164 128 129 SRAAAA PPLAAA HHHHxx 311 7842 1 3 1 11 11 311 311 311 311 22 23 ZLAAAA QPLAAA OOOOxx 5683 7843 1 3 3 3 83 683 1683 683 5683 166 167 PKAAAA RPLAAA VVVVxx 3993 7844 1 1 3 13 93 993 1993 3993 3993 186 187 PXAAAA SPLAAA AAAAxx 9897 7845 1 1 7 17 97 897 1897 4897 9897 194 195 RQAAAA TPLAAA HHHHxx 6609 7846 1 1 9 9 9 609 609 1609 6609 18 19 FUAAAA UPLAAA OOOOxx 1362 7847 0 2 2 2 62 362 1362 1362 1362 124 125 KAAAAA VPLAAA VVVVxx 3918 7848 0 2 8 18 18 918 1918 3918 3918 36 37 SUAAAA WPLAAA AAAAxx 7376 7849 0 0 6 16 76 376 1376 2376 7376 152 153 SXAAAA XPLAAA HHHHxx 6996 7850 0 0 6 16 96 996 996 1996 6996 192 193 CJAAAA YPLAAA OOOOxx 9567 7851 1 3 7 7 67 567 1567 4567 9567 134 135 ZDAAAA ZPLAAA VVVVxx 7525 7852 1 1 5 5 25 525 1525 2525 7525 50 51 LDAAAA AQLAAA AAAAxx 9069 7853 1 1 9 9 69 69 1069 4069 9069 138 139 VKAAAA BQLAAA HHHHxx 9999 7854 1 3 9 19 99 999 1999 4999 9999 198 199 PUAAAA CQLAAA OOOOxx 9237 7855 1 1 7 17 37 237 1237 4237 9237 74 75 HRAAAA DQLAAA VVVVxx 8441 7856 1 1 1 1 41 441 441 3441 8441 82 83 RMAAAA EQLAAA AAAAxx 6769 7857 1 1 9 9 69 769 769 1769 6769 138 139 JAAAAA FQLAAA HHHHxx 6073 7858 1 1 3 13 73 73 73 1073 6073 146 147 PZAAAA GQLAAA OOOOxx 1091 7859 1 3 1 11 91 91 1091 1091 1091 182 183 ZPAAAA HQLAAA VVVVxx 9886 7860 0 2 6 6 86 886 1886 4886 9886 172 173 GQAAAA IQLAAA AAAAxx 3971 7861 1 3 1 11 71 971 1971 3971 3971 142 143 TWAAAA JQLAAA HHHHxx 4621 7862 1 1 1 1 21 621 621 4621 4621 42 43 TVAAAA KQLAAA OOOOxx 3120 7863 0 0 0 0 20 120 1120 3120 3120 40 41 AQAAAA LQLAAA VVVVxx 9773 7864 1 1 3 13 73 773 1773 4773 9773 146 147 XLAAAA MQLAAA AAAAxx 8712 7865 0 0 2 12 12 712 712 3712 8712 24 25 CXAAAA NQLAAA HHHHxx 801 7866 1 1 1 1 1 801 801 801 801 2 3 VEAAAA OQLAAA OOOOxx 9478 7867 0 2 8 18 78 478 1478 4478 9478 156 157 OAAAAA PQLAAA VVVVxx 3466 7868 0 2 6 6 66 466 1466 3466 3466 132 133 IDAAAA QQLAAA AAAAxx 6326 7869 0 2 6 6 26 326 326 1326 6326 52 53 IJAAAA RQLAAA HHHHxx 1723 7870 1 3 3 3 23 723 1723 1723 1723 46 47 HOAAAA SQLAAA OOOOxx 4978 7871 0 2 8 18 78 978 978 4978 4978 156 157 MJAAAA TQLAAA VVVVxx 2311 7872 1 3 1 11 11 311 311 2311 2311 22 23 XKAAAA UQLAAA AAAAxx 9532 7873 0 0 2 12 32 532 1532 4532 9532 64 65 QCAAAA VQLAAA HHHHxx 3680 7874 0 0 0 0 80 680 1680 3680 3680 160 161 OLAAAA WQLAAA OOOOxx 1244 7875 0 0 4 4 44 244 1244 1244 1244 88 89 WVAAAA XQLAAA VVVVxx 3821 7876 1 1 1 1 21 821 1821 3821 3821 42 43 ZQAAAA YQLAAA AAAAxx 9586 7877 0 2 6 6 86 586 1586 4586 9586 172 173 SEAAAA ZQLAAA HHHHxx 3894 7878 0 2 4 14 94 894 1894 3894 3894 188 189 UTAAAA ARLAAA OOOOxx 6169 7879 1 1 9 9 69 169 169 1169 6169 138 139 HDAAAA BRLAAA VVVVxx 5919 7880 1 3 9 19 19 919 1919 919 5919 38 39 RTAAAA CRLAAA AAAAxx 4187 7881 1 3 7 7 87 187 187 4187 4187 174 175 BFAAAA DRLAAA HHHHxx 5477 7882 1 1 7 17 77 477 1477 477 5477 154 155 RCAAAA ERLAAA OOOOxx 2806 7883 0 2 6 6 6 806 806 2806 2806 12 13 YDAAAA FRLAAA VVVVxx 8158 7884 0 2 8 18 58 158 158 3158 8158 116 117 UBAAAA GRLAAA AAAAxx 7130 7885 0 2 0 10 30 130 1130 2130 7130 60 61 GOAAAA HRLAAA HHHHxx 7133 7886 1 1 3 13 33 133 1133 2133 7133 66 67 JOAAAA IRLAAA OOOOxx 6033 7887 1 1 3 13 33 33 33 1033 6033 66 67 BYAAAA JRLAAA VVVVxx 2415 7888 1 3 5 15 15 415 415 2415 2415 30 31 XOAAAA KRLAAA AAAAxx 8091 7889 1 3 1 11 91 91 91 3091 8091 182 183 FZAAAA LRLAAA HHHHxx 8347 7890 1 3 7 7 47 347 347 3347 8347 94 95 BJAAAA MRLAAA OOOOxx 7879 7891 1 3 9 19 79 879 1879 2879 7879 158 159 BRAAAA NRLAAA VVVVxx 9360 7892 0 0 0 0 60 360 1360 4360 9360 120 121 AWAAAA ORLAAA AAAAxx 3369 7893 1 1 9 9 69 369 1369 3369 3369 138 139 PZAAAA PRLAAA HHHHxx 8536 7894 0 0 6 16 36 536 536 3536 8536 72 73 IQAAAA QRLAAA OOOOxx 8628 7895 0 0 8 8 28 628 628 3628 8628 56 57 WTAAAA RRLAAA VVVVxx 1580 7896 0 0 0 0 80 580 1580 1580 1580 160 161 UIAAAA SRLAAA AAAAxx 705 7897 1 1 5 5 5 705 705 705 705 10 11 DBAAAA TRLAAA HHHHxx 4650 7898 0 2 0 10 50 650 650 4650 4650 100 101 WWAAAA URLAAA OOOOxx 9165 7899 1 1 5 5 65 165 1165 4165 9165 130 131 NOAAAA VRLAAA VVVVxx 4820 7900 0 0 0 0 20 820 820 4820 4820 40 41 KDAAAA WRLAAA AAAAxx 3538 7901 0 2 8 18 38 538 1538 3538 3538 76 77 CGAAAA XRLAAA HHHHxx 9947 7902 1 3 7 7 47 947 1947 4947 9947 94 95 PSAAAA YRLAAA OOOOxx 4954 7903 0 2 4 14 54 954 954 4954 4954 108 109 OIAAAA ZRLAAA VVVVxx 1104 7904 0 0 4 4 4 104 1104 1104 1104 8 9 MQAAAA ASLAAA AAAAxx 8455 7905 1 3 5 15 55 455 455 3455 8455 110 111 FNAAAA BSLAAA HHHHxx 8307 7906 1 3 7 7 7 307 307 3307 8307 14 15 NHAAAA CSLAAA OOOOxx 9203 7907 1 3 3 3 3 203 1203 4203 9203 6 7 ZPAAAA DSLAAA VVVVxx 7565 7908 1 1 5 5 65 565 1565 2565 7565 130 131 ZEAAAA ESLAAA AAAAxx 7745 7909 1 1 5 5 45 745 1745 2745 7745 90 91 XLAAAA FSLAAA HHHHxx 1787 7910 1 3 7 7 87 787 1787 1787 1787 174 175 TQAAAA GSLAAA OOOOxx 4861 7911 1 1 1 1 61 861 861 4861 4861 122 123 ZEAAAA HSLAAA VVVVxx 5183 7912 1 3 3 3 83 183 1183 183 5183 166 167 JRAAAA ISLAAA AAAAxx 529 7913 1 1 9 9 29 529 529 529 529 58 59 JUAAAA JSLAAA HHHHxx 2470 7914 0 2 0 10 70 470 470 2470 2470 140 141 ARAAAA KSLAAA OOOOxx 1267 7915 1 3 7 7 67 267 1267 1267 1267 134 135 TWAAAA LSLAAA VVVVxx 2059 7916 1 3 9 19 59 59 59 2059 2059 118 119 FBAAAA MSLAAA AAAAxx 1862 7917 0 2 2 2 62 862 1862 1862 1862 124 125 QTAAAA NSLAAA HHHHxx 7382 7918 0 2 2 2 82 382 1382 2382 7382 164 165 YXAAAA OSLAAA OOOOxx 4796 7919 0 0 6 16 96 796 796 4796 4796 192 193 MCAAAA PSLAAA VVVVxx 2331 7920 1 3 1 11 31 331 331 2331 2331 62 63 RLAAAA QSLAAA AAAAxx 8870 7921 0 2 0 10 70 870 870 3870 8870 140 141 EDAAAA RSLAAA HHHHxx 9581 7922 1 1 1 1 81 581 1581 4581 9581 162 163 NEAAAA SSLAAA OOOOxx 9063 7923 1 3 3 3 63 63 1063 4063 9063 126 127 PKAAAA TSLAAA VVVVxx 2192 7924 0 0 2 12 92 192 192 2192 2192 184 185 IGAAAA USLAAA AAAAxx 6466 7925 0 2 6 6 66 466 466 1466 6466 132 133 SOAAAA VSLAAA HHHHxx 7096 7926 0 0 6 16 96 96 1096 2096 7096 192 193 YMAAAA WSLAAA OOOOxx 6257 7927 1 1 7 17 57 257 257 1257 6257 114 115 RGAAAA XSLAAA VVVVxx 7009 7928 1 1 9 9 9 9 1009 2009 7009 18 19 PJAAAA YSLAAA AAAAxx 8136 7929 0 0 6 16 36 136 136 3136 8136 72 73 YAAAAA ZSLAAA HHHHxx 1854 7930 0 2 4 14 54 854 1854 1854 1854 108 109 ITAAAA ATLAAA OOOOxx 3644 7931 0 0 4 4 44 644 1644 3644 3644 88 89 EKAAAA BTLAAA VVVVxx 4437 7932 1 1 7 17 37 437 437 4437 4437 74 75 ROAAAA CTLAAA AAAAxx 7209 7933 1 1 9 9 9 209 1209 2209 7209 18 19 HRAAAA DTLAAA HHHHxx 1516 7934 0 0 6 16 16 516 1516 1516 1516 32 33 IGAAAA ETLAAA OOOOxx 822 7935 0 2 2 2 22 822 822 822 822 44 45 QFAAAA FTLAAA VVVVxx 1778 7936 0 2 8 18 78 778 1778 1778 1778 156 157 KQAAAA GTLAAA AAAAxx 8161 7937 1 1 1 1 61 161 161 3161 8161 122 123 XBAAAA HTLAAA HHHHxx 6030 7938 0 2 0 10 30 30 30 1030 6030 60 61 YXAAAA ITLAAA OOOOxx 3515 7939 1 3 5 15 15 515 1515 3515 3515 30 31 FFAAAA JTLAAA VVVVxx 1702 7940 0 2 2 2 2 702 1702 1702 1702 4 5 MNAAAA KTLAAA AAAAxx 2671 7941 1 3 1 11 71 671 671 2671 2671 142 143 TYAAAA LTLAAA HHHHxx 7623 7942 1 3 3 3 23 623 1623 2623 7623 46 47 FHAAAA MTLAAA OOOOxx 9828 7943 0 0 8 8 28 828 1828 4828 9828 56 57 AOAAAA NTLAAA VVVVxx 1888 7944 0 0 8 8 88 888 1888 1888 1888 176 177 QUAAAA OTLAAA AAAAxx 4520 7945 0 0 0 0 20 520 520 4520 4520 40 41 WRAAAA PTLAAA HHHHxx 3461 7946 1 1 1 1 61 461 1461 3461 3461 122 123 DDAAAA QTLAAA OOOOxx 1488 7947 0 0 8 8 88 488 1488 1488 1488 176 177 GFAAAA RTLAAA VVVVxx 7753 7948 1 1 3 13 53 753 1753 2753 7753 106 107 FMAAAA STLAAA AAAAxx 5525 7949 1 1 5 5 25 525 1525 525 5525 50 51 NEAAAA TTLAAA HHHHxx 5220 7950 0 0 0 0 20 220 1220 220 5220 40 41 USAAAA UTLAAA OOOOxx 305 7951 1 1 5 5 5 305 305 305 305 10 11 TLAAAA VTLAAA VVVVxx 7883 7952 1 3 3 3 83 883 1883 2883 7883 166 167 FRAAAA WTLAAA AAAAxx 1222 7953 0 2 2 2 22 222 1222 1222 1222 44 45 AVAAAA XTLAAA HHHHxx 8552 7954 0 0 2 12 52 552 552 3552 8552 104 105 YQAAAA YTLAAA OOOOxx 6097 7955 1 1 7 17 97 97 97 1097 6097 194 195 NAAAAA ZTLAAA VVVVxx 2298 7956 0 2 8 18 98 298 298 2298 2298 196 197 KKAAAA AULAAA AAAAxx 956 7957 0 0 6 16 56 956 956 956 956 112 113 UKAAAA BULAAA HHHHxx 9351 7958 1 3 1 11 51 351 1351 4351 9351 102 103 RVAAAA CULAAA OOOOxx 6669 7959 1 1 9 9 69 669 669 1669 6669 138 139 NWAAAA DULAAA VVVVxx 9383 7960 1 3 3 3 83 383 1383 4383 9383 166 167 XWAAAA EULAAA AAAAxx 1607 7961 1 3 7 7 7 607 1607 1607 1607 14 15 VJAAAA FULAAA HHHHxx 812 7962 0 0 2 12 12 812 812 812 812 24 25 GFAAAA GULAAA OOOOxx 2109 7963 1 1 9 9 9 109 109 2109 2109 18 19 DDAAAA HULAAA VVVVxx 207 7964 1 3 7 7 7 207 207 207 207 14 15 ZHAAAA IULAAA AAAAxx 7124 7965 0 0 4 4 24 124 1124 2124 7124 48 49 AOAAAA JULAAA HHHHxx 9333 7966 1 1 3 13 33 333 1333 4333 9333 66 67 ZUAAAA KULAAA OOOOxx 3262 7967 0 2 2 2 62 262 1262 3262 3262 124 125 MVAAAA LULAAA VVVVxx 1070 7968 0 2 0 10 70 70 1070 1070 1070 140 141 EPAAAA MULAAA AAAAxx 7579 7969 1 3 9 19 79 579 1579 2579 7579 158 159 NFAAAA NULAAA HHHHxx 9283 7970 1 3 3 3 83 283 1283 4283 9283 166 167 BTAAAA OULAAA OOOOxx 4917 7971 1 1 7 17 17 917 917 4917 4917 34 35 DHAAAA PULAAA VVVVxx 1328 7972 0 0 8 8 28 328 1328 1328 1328 56 57 CZAAAA QULAAA AAAAxx 3042 7973 0 2 2 2 42 42 1042 3042 3042 84 85 ANAAAA RULAAA HHHHxx 8352 7974 0 0 2 12 52 352 352 3352 8352 104 105 GJAAAA SULAAA OOOOxx 2710 7975 0 2 0 10 10 710 710 2710 2710 20 21 GAAAAA TULAAA VVVVxx 3330 7976 0 2 0 10 30 330 1330 3330 3330 60 61 CYAAAA UULAAA AAAAxx 2822 7977 0 2 2 2 22 822 822 2822 2822 44 45 OEAAAA VULAAA HHHHxx 5627 7978 1 3 7 7 27 627 1627 627 5627 54 55 LIAAAA WULAAA OOOOxx 7848 7979 0 0 8 8 48 848 1848 2848 7848 96 97 WPAAAA XULAAA VVVVxx 7384 7980 0 0 4 4 84 384 1384 2384 7384 168 169 AYAAAA YULAAA AAAAxx 727 7981 1 3 7 7 27 727 727 727 727 54 55 ZBAAAA ZULAAA HHHHxx 9926 7982 0 2 6 6 26 926 1926 4926 9926 52 53 URAAAA AVLAAA OOOOxx 2647 7983 1 3 7 7 47 647 647 2647 2647 94 95 VXAAAA BVLAAA VVVVxx 6416 7984 0 0 6 16 16 416 416 1416 6416 32 33 UMAAAA CVLAAA AAAAxx 8751 7985 1 3 1 11 51 751 751 3751 8751 102 103 PYAAAA DVLAAA HHHHxx 6515 7986 1 3 5 15 15 515 515 1515 6515 30 31 PQAAAA EVLAAA OOOOxx 2472 7987 0 0 2 12 72 472 472 2472 2472 144 145 CRAAAA FVLAAA VVVVxx 7205 7988 1 1 5 5 5 205 1205 2205 7205 10 11 DRAAAA GVLAAA AAAAxx 9654 7989 0 2 4 14 54 654 1654 4654 9654 108 109 IHAAAA HVLAAA HHHHxx 5646 7990 0 2 6 6 46 646 1646 646 5646 92 93 EJAAAA IVLAAA OOOOxx 4217 7991 1 1 7 17 17 217 217 4217 4217 34 35 FGAAAA JVLAAA VVVVxx 4484 7992 0 0 4 4 84 484 484 4484 4484 168 169 MQAAAA KVLAAA AAAAxx 6654 7993 0 2 4 14 54 654 654 1654 6654 108 109 YVAAAA LVLAAA HHHHxx 4876 7994 0 0 6 16 76 876 876 4876 4876 152 153 OFAAAA MVLAAA OOOOxx 9690 7995 0 2 0 10 90 690 1690 4690 9690 180 181 SIAAAA NVLAAA VVVVxx 2453 7996 1 1 3 13 53 453 453 2453 2453 106 107 JQAAAA OVLAAA AAAAxx 829 7997 1 1 9 9 29 829 829 829 829 58 59 XFAAAA PVLAAA HHHHxx 2547 7998 1 3 7 7 47 547 547 2547 2547 94 95 ZTAAAA QVLAAA OOOOxx 9726 7999 0 2 6 6 26 726 1726 4726 9726 52 53 CKAAAA RVLAAA VVVVxx 9267 8000 1 3 7 7 67 267 1267 4267 9267 134 135 LSAAAA SVLAAA AAAAxx 7448 8001 0 0 8 8 48 448 1448 2448 7448 96 97 MAAAAA TVLAAA HHHHxx 610 8002 0 2 0 10 10 610 610 610 610 20 21 MXAAAA UVLAAA OOOOxx 2791 8003 1 3 1 11 91 791 791 2791 2791 182 183 JDAAAA VVLAAA VVVVxx 3651 8004 1 3 1 11 51 651 1651 3651 3651 102 103 LKAAAA WVLAAA AAAAxx 5206 8005 0 2 6 6 6 206 1206 206 5206 12 13 GSAAAA XVLAAA HHHHxx 8774 8006 0 2 4 14 74 774 774 3774 8774 148 149 MZAAAA YVLAAA OOOOxx 4753 8007 1 1 3 13 53 753 753 4753 4753 106 107 VAAAAA ZVLAAA VVVVxx 4755 8008 1 3 5 15 55 755 755 4755 4755 110 111 XAAAAA AWLAAA AAAAxx 686 8009 0 2 6 6 86 686 686 686 686 172 173 KAAAAA BWLAAA HHHHxx 8281 8010 1 1 1 1 81 281 281 3281 8281 162 163 NGAAAA CWLAAA OOOOxx 2058 8011 0 2 8 18 58 58 58 2058 2058 116 117 EBAAAA DWLAAA VVVVxx 8900 8012 0 0 0 0 0 900 900 3900 8900 0 1 IEAAAA EWLAAA AAAAxx 8588 8013 0 0 8 8 88 588 588 3588 8588 176 177 ISAAAA FWLAAA HHHHxx 2904 8014 0 0 4 4 4 904 904 2904 2904 8 9 SHAAAA GWLAAA OOOOxx 8917 8015 1 1 7 17 17 917 917 3917 8917 34 35 ZEAAAA HWLAAA VVVVxx 9026 8016 0 2 6 6 26 26 1026 4026 9026 52 53 EJAAAA IWLAAA AAAAxx 2416 8017 0 0 6 16 16 416 416 2416 2416 32 33 YOAAAA JWLAAA HHHHxx 1053 8018 1 1 3 13 53 53 1053 1053 1053 106 107 NOAAAA KWLAAA OOOOxx 7141 8019 1 1 1 1 41 141 1141 2141 7141 82 83 ROAAAA LWLAAA VVVVxx 9771 8020 1 3 1 11 71 771 1771 4771 9771 142 143 VLAAAA MWLAAA AAAAxx 2774 8021 0 2 4 14 74 774 774 2774 2774 148 149 SCAAAA NWLAAA HHHHxx 3213 8022 1 1 3 13 13 213 1213 3213 3213 26 27 PTAAAA OWLAAA OOOOxx 5694 8023 0 2 4 14 94 694 1694 694 5694 188 189 ALAAAA PWLAAA VVVVxx 6631 8024 1 3 1 11 31 631 631 1631 6631 62 63 BVAAAA QWLAAA AAAAxx 6638 8025 0 2 8 18 38 638 638 1638 6638 76 77 IVAAAA RWLAAA HHHHxx 7407 8026 1 3 7 7 7 407 1407 2407 7407 14 15 XYAAAA SWLAAA OOOOxx 8972 8027 0 0 2 12 72 972 972 3972 8972 144 145 CHAAAA TWLAAA VVVVxx 2202 8028 0 2 2 2 2 202 202 2202 2202 4 5 SGAAAA UWLAAA AAAAxx 6135 8029 1 3 5 15 35 135 135 1135 6135 70 71 ZBAAAA VWLAAA HHHHxx 5043 8030 1 3 3 3 43 43 1043 43 5043 86 87 ZLAAAA WWLAAA OOOOxx 5163 8031 1 3 3 3 63 163 1163 163 5163 126 127 PQAAAA XWLAAA VVVVxx 1191 8032 1 3 1 11 91 191 1191 1191 1191 182 183 VTAAAA YWLAAA AAAAxx 6576 8033 0 0 6 16 76 576 576 1576 6576 152 153 YSAAAA ZWLAAA HHHHxx 3455 8034 1 3 5 15 55 455 1455 3455 3455 110 111 XCAAAA AXLAAA OOOOxx 3688 8035 0 0 8 8 88 688 1688 3688 3688 176 177 WLAAAA BXLAAA VVVVxx 4982 8036 0 2 2 2 82 982 982 4982 4982 164 165 QJAAAA CXLAAA AAAAxx 4180 8037 0 0 0 0 80 180 180 4180 4180 160 161 UEAAAA DXLAAA HHHHxx 4708 8038 0 0 8 8 8 708 708 4708 4708 16 17 CZAAAA EXLAAA OOOOxx 1241 8039 1 1 1 1 41 241 1241 1241 1241 82 83 TVAAAA FXLAAA VVVVxx 4921 8040 1 1 1 1 21 921 921 4921 4921 42 43 HHAAAA GXLAAA AAAAxx 3197 8041 1 1 7 17 97 197 1197 3197 3197 194 195 ZSAAAA HXLAAA HHHHxx 8225 8042 1 1 5 5 25 225 225 3225 8225 50 51 JEAAAA IXLAAA OOOOxx 5913 8043 1 1 3 13 13 913 1913 913 5913 26 27 LTAAAA JXLAAA VVVVxx 6387 8044 1 3 7 7 87 387 387 1387 6387 174 175 RLAAAA KXLAAA AAAAxx 2706 8045 0 2 6 6 6 706 706 2706 2706 12 13 CAAAAA LXLAAA HHHHxx 1461 8046 1 1 1 1 61 461 1461 1461 1461 122 123 FEAAAA MXLAAA OOOOxx 7646 8047 0 2 6 6 46 646 1646 2646 7646 92 93 CIAAAA NXLAAA VVVVxx 8066 8048 0 2 6 6 66 66 66 3066 8066 132 133 GYAAAA OXLAAA AAAAxx 4171 8049 1 3 1 11 71 171 171 4171 4171 142 143 LEAAAA PXLAAA HHHHxx 8008 8050 0 0 8 8 8 8 8 3008 8008 16 17 AWAAAA QXLAAA OOOOxx 2088 8051 0 0 8 8 88 88 88 2088 2088 176 177 ICAAAA RXLAAA VVVVxx 7907 8052 1 3 7 7 7 907 1907 2907 7907 14 15 DSAAAA SXLAAA AAAAxx 2429 8053 1 1 9 9 29 429 429 2429 2429 58 59 LPAAAA TXLAAA HHHHxx 9629 8054 1 1 9 9 29 629 1629 4629 9629 58 59 JGAAAA UXLAAA OOOOxx 1470 8055 0 2 0 10 70 470 1470 1470 1470 140 141 OEAAAA VXLAAA VVVVxx 4346 8056 0 2 6 6 46 346 346 4346 4346 92 93 ELAAAA WXLAAA AAAAxx 7219 8057 1 3 9 19 19 219 1219 2219 7219 38 39 RRAAAA XXLAAA HHHHxx 1185 8058 1 1 5 5 85 185 1185 1185 1185 170 171 PTAAAA YXLAAA OOOOxx 8776 8059 0 0 6 16 76 776 776 3776 8776 152 153 OZAAAA ZXLAAA VVVVxx 684 8060 0 0 4 4 84 684 684 684 684 168 169 IAAAAA AYLAAA AAAAxx 2343 8061 1 3 3 3 43 343 343 2343 2343 86 87 DMAAAA BYLAAA HHHHxx 4470 8062 0 2 0 10 70 470 470 4470 4470 140 141 YPAAAA CYLAAA OOOOxx 5116 8063 0 0 6 16 16 116 1116 116 5116 32 33 UOAAAA DYLAAA VVVVxx 1746 8064 0 2 6 6 46 746 1746 1746 1746 92 93 EPAAAA EYLAAA AAAAxx 3216 8065 0 0 6 16 16 216 1216 3216 3216 32 33 STAAAA FYLAAA HHHHxx 4594 8066 0 2 4 14 94 594 594 4594 4594 188 189 SUAAAA GYLAAA OOOOxx 3013 8067 1 1 3 13 13 13 1013 3013 3013 26 27 XLAAAA HYLAAA VVVVxx 2307 8068 1 3 7 7 7 307 307 2307 2307 14 15 TKAAAA IYLAAA AAAAxx 7663 8069 1 3 3 3 63 663 1663 2663 7663 126 127 TIAAAA JYLAAA HHHHxx 8504 8070 0 0 4 4 4 504 504 3504 8504 8 9 CPAAAA KYLAAA OOOOxx 3683 8071 1 3 3 3 83 683 1683 3683 3683 166 167 RLAAAA LYLAAA VVVVxx 144 8072 0 0 4 4 44 144 144 144 144 88 89 OFAAAA MYLAAA AAAAxx 203 8073 1 3 3 3 3 203 203 203 203 6 7 VHAAAA NYLAAA HHHHxx 5255 8074 1 3 5 15 55 255 1255 255 5255 110 111 DUAAAA OYLAAA OOOOxx 4150 8075 0 2 0 10 50 150 150 4150 4150 100 101 QDAAAA PYLAAA VVVVxx 5701 8076 1 1 1 1 1 701 1701 701 5701 2 3 HLAAAA QYLAAA AAAAxx 7400 8077 0 0 0 0 0 400 1400 2400 7400 0 1 QYAAAA RYLAAA HHHHxx 8203 8078 1 3 3 3 3 203 203 3203 8203 6 7 NDAAAA SYLAAA OOOOxx 637 8079 1 1 7 17 37 637 637 637 637 74 75 NYAAAA TYLAAA VVVVxx 2898 8080 0 2 8 18 98 898 898 2898 2898 196 197 MHAAAA UYLAAA AAAAxx 1110 8081 0 2 0 10 10 110 1110 1110 1110 20 21 SQAAAA VYLAAA HHHHxx 6255 8082 1 3 5 15 55 255 255 1255 6255 110 111 PGAAAA WYLAAA OOOOxx 1071 8083 1 3 1 11 71 71 1071 1071 1071 142 143 FPAAAA XYLAAA VVVVxx 541 8084 1 1 1 1 41 541 541 541 541 82 83 VUAAAA YYLAAA AAAAxx 8077 8085 1 1 7 17 77 77 77 3077 8077 154 155 RYAAAA ZYLAAA HHHHxx 6809 8086 1 1 9 9 9 809 809 1809 6809 18 19 XBAAAA AZLAAA OOOOxx 4749 8087 1 1 9 9 49 749 749 4749 4749 98 99 RAAAAA BZLAAA VVVVxx 2886 8088 0 2 6 6 86 886 886 2886 2886 172 173 AHAAAA CZLAAA AAAAxx 5510 8089 0 2 0 10 10 510 1510 510 5510 20 21 YDAAAA DZLAAA HHHHxx 713 8090 1 1 3 13 13 713 713 713 713 26 27 LBAAAA EZLAAA OOOOxx 8388 8091 0 0 8 8 88 388 388 3388 8388 176 177 QKAAAA FZLAAA VVVVxx 9524 8092 0 0 4 4 24 524 1524 4524 9524 48 49 ICAAAA GZLAAA AAAAxx 9949 8093 1 1 9 9 49 949 1949 4949 9949 98 99 RSAAAA HZLAAA HHHHxx 885 8094 1 1 5 5 85 885 885 885 885 170 171 BIAAAA IZLAAA OOOOxx 8699 8095 1 3 9 19 99 699 699 3699 8699 198 199 PWAAAA JZLAAA VVVVxx 2232 8096 0 0 2 12 32 232 232 2232 2232 64 65 WHAAAA KZLAAA AAAAxx 5142 8097 0 2 2 2 42 142 1142 142 5142 84 85 UPAAAA LZLAAA HHHHxx 8891 8098 1 3 1 11 91 891 891 3891 8891 182 183 ZDAAAA MZLAAA OOOOxx 1881 8099 1 1 1 1 81 881 1881 1881 1881 162 163 JUAAAA NZLAAA VVVVxx 3751 8100 1 3 1 11 51 751 1751 3751 3751 102 103 HOAAAA OZLAAA AAAAxx 1896 8101 0 0 6 16 96 896 1896 1896 1896 192 193 YUAAAA PZLAAA HHHHxx 8258 8102 0 2 8 18 58 258 258 3258 8258 116 117 QFAAAA QZLAAA OOOOxx 3820 8103 0 0 0 0 20 820 1820 3820 3820 40 41 YQAAAA RZLAAA VVVVxx 6617 8104 1 1 7 17 17 617 617 1617 6617 34 35 NUAAAA SZLAAA AAAAxx 5100 8105 0 0 0 0 0 100 1100 100 5100 0 1 EOAAAA TZLAAA HHHHxx 4277 8106 1 1 7 17 77 277 277 4277 4277 154 155 NIAAAA UZLAAA OOOOxx 2498 8107 0 2 8 18 98 498 498 2498 2498 196 197 CSAAAA VZLAAA VVVVxx 4343 8108 1 3 3 3 43 343 343 4343 4343 86 87 BLAAAA WZLAAA AAAAxx 8319 8109 1 3 9 19 19 319 319 3319 8319 38 39 ZHAAAA XZLAAA HHHHxx 4803 8110 1 3 3 3 3 803 803 4803 4803 6 7 TCAAAA YZLAAA OOOOxx 3100 8111 0 0 0 0 0 100 1100 3100 3100 0 1 GPAAAA ZZLAAA VVVVxx 428 8112 0 0 8 8 28 428 428 428 428 56 57 MQAAAA AAMAAA AAAAxx 2811 8113 1 3 1 11 11 811 811 2811 2811 22 23 DEAAAA BAMAAA HHHHxx 2989 8114 1 1 9 9 89 989 989 2989 2989 178 179 ZKAAAA CAMAAA OOOOxx 1100 8115 0 0 0 0 0 100 1100 1100 1100 0 1 IQAAAA DAMAAA VVVVxx 6586 8116 0 2 6 6 86 586 586 1586 6586 172 173 ITAAAA EAMAAA AAAAxx 3124 8117 0 0 4 4 24 124 1124 3124 3124 48 49 EQAAAA FAMAAA HHHHxx 1635 8118 1 3 5 15 35 635 1635 1635 1635 70 71 XKAAAA GAMAAA OOOOxx 3888 8119 0 0 8 8 88 888 1888 3888 3888 176 177 OTAAAA HAMAAA VVVVxx 8369 8120 1 1 9 9 69 369 369 3369 8369 138 139 XJAAAA IAMAAA AAAAxx 3148 8121 0 0 8 8 48 148 1148 3148 3148 96 97 CRAAAA JAMAAA HHHHxx 2842 8122 0 2 2 2 42 842 842 2842 2842 84 85 IFAAAA KAMAAA OOOOxx 4965 8123 1 1 5 5 65 965 965 4965 4965 130 131 ZIAAAA LAMAAA VVVVxx 3742 8124 0 2 2 2 42 742 1742 3742 3742 84 85 YNAAAA MAMAAA AAAAxx 5196 8125 0 0 6 16 96 196 1196 196 5196 192 193 WRAAAA NAMAAA HHHHxx 9105 8126 1 1 5 5 5 105 1105 4105 9105 10 11 FMAAAA OAMAAA OOOOxx 6806 8127 0 2 6 6 6 806 806 1806 6806 12 13 UBAAAA PAMAAA VVVVxx 5849 8128 1 1 9 9 49 849 1849 849 5849 98 99 ZQAAAA QAMAAA AAAAxx 6504 8129 0 0 4 4 4 504 504 1504 6504 8 9 EQAAAA RAMAAA HHHHxx 9841 8130 1 1 1 1 41 841 1841 4841 9841 82 83 NOAAAA SAMAAA OOOOxx 457 8131 1 1 7 17 57 457 457 457 457 114 115 PRAAAA TAMAAA VVVVxx 8856 8132 0 0 6 16 56 856 856 3856 8856 112 113 QCAAAA UAMAAA AAAAxx 8043 8133 1 3 3 3 43 43 43 3043 8043 86 87 JXAAAA VAMAAA HHHHxx 5933 8134 1 1 3 13 33 933 1933 933 5933 66 67 FUAAAA WAMAAA OOOOxx 5725 8135 1 1 5 5 25 725 1725 725 5725 50 51 FMAAAA XAMAAA VVVVxx 8607 8136 1 3 7 7 7 607 607 3607 8607 14 15 BTAAAA YAMAAA AAAAxx 9280 8137 0 0 0 0 80 280 1280 4280 9280 160 161 YSAAAA ZAMAAA HHHHxx 6017 8138 1 1 7 17 17 17 17 1017 6017 34 35 LXAAAA ABMAAA OOOOxx 4946 8139 0 2 6 6 46 946 946 4946 4946 92 93 GIAAAA BBMAAA VVVVxx 7373 8140 1 1 3 13 73 373 1373 2373 7373 146 147 PXAAAA CBMAAA AAAAxx 8096 8141 0 0 6 16 96 96 96 3096 8096 192 193 KZAAAA DBMAAA HHHHxx 3178 8142 0 2 8 18 78 178 1178 3178 3178 156 157 GSAAAA EBMAAA OOOOxx 1849 8143 1 1 9 9 49 849 1849 1849 1849 98 99 DTAAAA FBMAAA VVVVxx 8813 8144 1 1 3 13 13 813 813 3813 8813 26 27 ZAAAAA GBMAAA AAAAxx 460 8145 0 0 0 0 60 460 460 460 460 120 121 SRAAAA HBMAAA HHHHxx 7756 8146 0 0 6 16 56 756 1756 2756 7756 112 113 IMAAAA IBMAAA OOOOxx 4425 8147 1 1 5 5 25 425 425 4425 4425 50 51 FOAAAA JBMAAA VVVVxx 1602 8148 0 2 2 2 2 602 1602 1602 1602 4 5 QJAAAA KBMAAA AAAAxx 5981 8149 1 1 1 1 81 981 1981 981 5981 162 163 BWAAAA LBMAAA HHHHxx 8139 8150 1 3 9 19 39 139 139 3139 8139 78 79 BBAAAA MBMAAA OOOOxx 754 8151 0 2 4 14 54 754 754 754 754 108 109 ADAAAA NBMAAA VVVVxx 26 8152 0 2 6 6 26 26 26 26 26 52 53 ABAAAA OBMAAA AAAAxx 106 8153 0 2 6 6 6 106 106 106 106 12 13 CEAAAA PBMAAA HHHHxx 7465 8154 1 1 5 5 65 465 1465 2465 7465 130 131 DBAAAA QBMAAA OOOOxx 1048 8155 0 0 8 8 48 48 1048 1048 1048 96 97 IOAAAA RBMAAA VVVVxx 2303 8156 1 3 3 3 3 303 303 2303 2303 6 7 PKAAAA SBMAAA AAAAxx 5794 8157 0 2 4 14 94 794 1794 794 5794 188 189 WOAAAA TBMAAA HHHHxx 3321 8158 1 1 1 1 21 321 1321 3321 3321 42 43 TXAAAA UBMAAA OOOOxx 6122 8159 0 2 2 2 22 122 122 1122 6122 44 45 MBAAAA VBMAAA VVVVxx 6474 8160 0 2 4 14 74 474 474 1474 6474 148 149 APAAAA WBMAAA AAAAxx 827 8161 1 3 7 7 27 827 827 827 827 54 55 VFAAAA XBMAAA HHHHxx 6616 8162 0 0 6 16 16 616 616 1616 6616 32 33 MUAAAA YBMAAA OOOOxx 2131 8163 1 3 1 11 31 131 131 2131 2131 62 63 ZDAAAA ZBMAAA VVVVxx 5483 8164 1 3 3 3 83 483 1483 483 5483 166 167 XCAAAA ACMAAA AAAAxx 606 8165 0 2 6 6 6 606 606 606 606 12 13 IXAAAA BCMAAA HHHHxx 922 8166 0 2 2 2 22 922 922 922 922 44 45 MJAAAA CCMAAA OOOOxx 8475 8167 1 3 5 15 75 475 475 3475 8475 150 151 ZNAAAA DCMAAA VVVVxx 7645 8168 1 1 5 5 45 645 1645 2645 7645 90 91 BIAAAA ECMAAA AAAAxx 5097 8169 1 1 7 17 97 97 1097 97 5097 194 195 BOAAAA FCMAAA HHHHxx 5377 8170 1 1 7 17 77 377 1377 377 5377 154 155 VYAAAA GCMAAA OOOOxx 6116 8171 0 0 6 16 16 116 116 1116 6116 32 33 GBAAAA HCMAAA VVVVxx 8674 8172 0 2 4 14 74 674 674 3674 8674 148 149 QVAAAA ICMAAA AAAAxx 8063 8173 1 3 3 3 63 63 63 3063 8063 126 127 DYAAAA JCMAAA HHHHxx 5271 8174 1 3 1 11 71 271 1271 271 5271 142 143 TUAAAA KCMAAA OOOOxx 1619 8175 1 3 9 19 19 619 1619 1619 1619 38 39 HKAAAA LCMAAA VVVVxx 6419 8176 1 3 9 19 19 419 419 1419 6419 38 39 XMAAAA MCMAAA AAAAxx 7651 8177 1 3 1 11 51 651 1651 2651 7651 102 103 HIAAAA NCMAAA HHHHxx 2897 8178 1 1 7 17 97 897 897 2897 2897 194 195 LHAAAA OCMAAA OOOOxx 8148 8179 0 0 8 8 48 148 148 3148 8148 96 97 KBAAAA PCMAAA VVVVxx 7461 8180 1 1 1 1 61 461 1461 2461 7461 122 123 ZAAAAA QCMAAA AAAAxx 9186 8181 0 2 6 6 86 186 1186 4186 9186 172 173 IPAAAA RCMAAA HHHHxx 7127 8182 1 3 7 7 27 127 1127 2127 7127 54 55 DOAAAA SCMAAA OOOOxx 8233 8183 1 1 3 13 33 233 233 3233 8233 66 67 REAAAA TCMAAA VVVVxx 9651 8184 1 3 1 11 51 651 1651 4651 9651 102 103 FHAAAA UCMAAA AAAAxx 6746 8185 0 2 6 6 46 746 746 1746 6746 92 93 MZAAAA VCMAAA HHHHxx 7835 8186 1 3 5 15 35 835 1835 2835 7835 70 71 JPAAAA WCMAAA OOOOxx 8815 8187 1 3 5 15 15 815 815 3815 8815 30 31 BBAAAA XCMAAA VVVVxx 6398 8188 0 2 8 18 98 398 398 1398 6398 196 197 CMAAAA YCMAAA AAAAxx 5344 8189 0 0 4 4 44 344 1344 344 5344 88 89 OXAAAA ZCMAAA HHHHxx 8209 8190 1 1 9 9 9 209 209 3209 8209 18 19 TDAAAA ADMAAA OOOOxx 8444 8191 0 0 4 4 44 444 444 3444 8444 88 89 UMAAAA BDMAAA VVVVxx 5669 8192 1 1 9 9 69 669 1669 669 5669 138 139 BKAAAA CDMAAA AAAAxx 2455 8193 1 3 5 15 55 455 455 2455 2455 110 111 LQAAAA DDMAAA HHHHxx 6767 8194 1 3 7 7 67 767 767 1767 6767 134 135 HAAAAA EDMAAA OOOOxx 135 8195 1 3 5 15 35 135 135 135 135 70 71 FFAAAA FDMAAA VVVVxx 3503 8196 1 3 3 3 3 503 1503 3503 3503 6 7 TEAAAA GDMAAA AAAAxx 6102 8197 0 2 2 2 2 102 102 1102 6102 4 5 SAAAAA HDMAAA HHHHxx 7136 8198 0 0 6 16 36 136 1136 2136 7136 72 73 MOAAAA IDMAAA OOOOxx 4933 8199 1 1 3 13 33 933 933 4933 4933 66 67 THAAAA JDMAAA VVVVxx 8804 8200 0 0 4 4 4 804 804 3804 8804 8 9 QAAAAA KDMAAA AAAAxx 3760 8201 0 0 0 0 60 760 1760 3760 3760 120 121 QOAAAA LDMAAA HHHHxx 8603 8202 1 3 3 3 3 603 603 3603 8603 6 7 XSAAAA MDMAAA OOOOxx 7411 8203 1 3 1 11 11 411 1411 2411 7411 22 23 BZAAAA NDMAAA VVVVxx 834 8204 0 2 4 14 34 834 834 834 834 68 69 CGAAAA ODMAAA AAAAxx 7385 8205 1 1 5 5 85 385 1385 2385 7385 170 171 BYAAAA PDMAAA HHHHxx 3696 8206 0 0 6 16 96 696 1696 3696 3696 192 193 EMAAAA QDMAAA OOOOxx 8720 8207 0 0 0 0 20 720 720 3720 8720 40 41 KXAAAA RDMAAA VVVVxx 4539 8208 1 3 9 19 39 539 539 4539 4539 78 79 PSAAAA SDMAAA AAAAxx 9837 8209 1 1 7 17 37 837 1837 4837 9837 74 75 JOAAAA TDMAAA HHHHxx 8595 8210 1 3 5 15 95 595 595 3595 8595 190 191 PSAAAA UDMAAA OOOOxx 3673 8211 1 1 3 13 73 673 1673 3673 3673 146 147 HLAAAA VDMAAA VVVVxx 475 8212 1 3 5 15 75 475 475 475 475 150 151 HSAAAA WDMAAA AAAAxx 2256 8213 0 0 6 16 56 256 256 2256 2256 112 113 UIAAAA XDMAAA HHHHxx 6349 8214 1 1 9 9 49 349 349 1349 6349 98 99 FKAAAA YDMAAA OOOOxx 9968 8215 0 0 8 8 68 968 1968 4968 9968 136 137 KTAAAA ZDMAAA VVVVxx 7261 8216 1 1 1 1 61 261 1261 2261 7261 122 123 HTAAAA AEMAAA AAAAxx 5799 8217 1 3 9 19 99 799 1799 799 5799 198 199 BPAAAA BEMAAA HHHHxx 8159 8218 1 3 9 19 59 159 159 3159 8159 118 119 VBAAAA CEMAAA OOOOxx 92 8219 0 0 2 12 92 92 92 92 92 184 185 ODAAAA DEMAAA VVVVxx 5927 8220 1 3 7 7 27 927 1927 927 5927 54 55 ZTAAAA EEMAAA AAAAxx 7925 8221 1 1 5 5 25 925 1925 2925 7925 50 51 VSAAAA FEMAAA HHHHxx 5836 8222 0 0 6 16 36 836 1836 836 5836 72 73 MQAAAA GEMAAA OOOOxx 7935 8223 1 3 5 15 35 935 1935 2935 7935 70 71 FTAAAA HEMAAA VVVVxx 5505 8224 1 1 5 5 5 505 1505 505 5505 10 11 TDAAAA IEMAAA AAAAxx 5882 8225 0 2 2 2 82 882 1882 882 5882 164 165 GSAAAA JEMAAA HHHHxx 4411 8226 1 3 1 11 11 411 411 4411 4411 22 23 RNAAAA KEMAAA OOOOxx 64 8227 0 0 4 4 64 64 64 64 64 128 129 MCAAAA LEMAAA VVVVxx 2851 8228 1 3 1 11 51 851 851 2851 2851 102 103 RFAAAA MEMAAA AAAAxx 1665 8229 1 1 5 5 65 665 1665 1665 1665 130 131 BMAAAA NEMAAA HHHHxx 2895 8230 1 3 5 15 95 895 895 2895 2895 190 191 JHAAAA OEMAAA OOOOxx 2210 8231 0 2 0 10 10 210 210 2210 2210 20 21 AHAAAA PEMAAA VVVVxx 9873 8232 1 1 3 13 73 873 1873 4873 9873 146 147 TPAAAA QEMAAA AAAAxx 5402 8233 0 2 2 2 2 402 1402 402 5402 4 5 UZAAAA REMAAA HHHHxx 285 8234 1 1 5 5 85 285 285 285 285 170 171 ZKAAAA SEMAAA OOOOxx 8545 8235 1 1 5 5 45 545 545 3545 8545 90 91 RQAAAA TEMAAA VVVVxx 5328 8236 0 0 8 8 28 328 1328 328 5328 56 57 YWAAAA UEMAAA AAAAxx 733 8237 1 1 3 13 33 733 733 733 733 66 67 FCAAAA VEMAAA HHHHxx 7726 8238 0 2 6 6 26 726 1726 2726 7726 52 53 ELAAAA WEMAAA OOOOxx 5418 8239 0 2 8 18 18 418 1418 418 5418 36 37 KAAAAA XEMAAA VVVVxx 7761 8240 1 1 1 1 61 761 1761 2761 7761 122 123 NMAAAA YEMAAA AAAAxx 9263 8241 1 3 3 3 63 263 1263 4263 9263 126 127 HSAAAA ZEMAAA HHHHxx 5579 8242 1 3 9 19 79 579 1579 579 5579 158 159 PGAAAA AFMAAA OOOOxx 5434 8243 0 2 4 14 34 434 1434 434 5434 68 69 ABAAAA BFMAAA VVVVxx 5230 8244 0 2 0 10 30 230 1230 230 5230 60 61 ETAAAA CFMAAA AAAAxx 9981 8245 1 1 1 1 81 981 1981 4981 9981 162 163 XTAAAA DFMAAA HHHHxx 5830 8246 0 2 0 10 30 830 1830 830 5830 60 61 GQAAAA EFMAAA OOOOxx 128 8247 0 0 8 8 28 128 128 128 128 56 57 YEAAAA FFMAAA VVVVxx 2734 8248 0 2 4 14 34 734 734 2734 2734 68 69 EBAAAA GFMAAA AAAAxx 4537 8249 1 1 7 17 37 537 537 4537 4537 74 75 NSAAAA HFMAAA HHHHxx 3899 8250 1 3 9 19 99 899 1899 3899 3899 198 199 ZTAAAA IFMAAA OOOOxx 1000 8251 0 0 0 0 0 0 1000 1000 1000 0 1 MMAAAA JFMAAA VVVVxx 9896 8252 0 0 6 16 96 896 1896 4896 9896 192 193 QQAAAA KFMAAA AAAAxx 3640 8253 0 0 0 0 40 640 1640 3640 3640 80 81 AKAAAA LFMAAA HHHHxx 2568 8254 0 0 8 8 68 568 568 2568 2568 136 137 UUAAAA MFMAAA OOOOxx 2026 8255 0 2 6 6 26 26 26 2026 2026 52 53 YZAAAA NFMAAA VVVVxx 3955 8256 1 3 5 15 55 955 1955 3955 3955 110 111 DWAAAA OFMAAA AAAAxx 7152 8257 0 0 2 12 52 152 1152 2152 7152 104 105 CPAAAA PFMAAA HHHHxx 2402 8258 0 2 2 2 2 402 402 2402 2402 4 5 KOAAAA QFMAAA OOOOxx 9522 8259 0 2 2 2 22 522 1522 4522 9522 44 45 GCAAAA RFMAAA VVVVxx 4011 8260 1 3 1 11 11 11 11 4011 4011 22 23 HYAAAA SFMAAA AAAAxx 3297 8261 1 1 7 17 97 297 1297 3297 3297 194 195 VWAAAA TFMAAA HHHHxx 4915 8262 1 3 5 15 15 915 915 4915 4915 30 31 BHAAAA UFMAAA OOOOxx 5397 8263 1 1 7 17 97 397 1397 397 5397 194 195 PZAAAA VFMAAA VVVVxx 5454 8264 0 2 4 14 54 454 1454 454 5454 108 109 UBAAAA WFMAAA AAAAxx 4568 8265 0 0 8 8 68 568 568 4568 4568 136 137 STAAAA XFMAAA HHHHxx 5875 8266 1 3 5 15 75 875 1875 875 5875 150 151 ZRAAAA YFMAAA OOOOxx 3642 8267 0 2 2 2 42 642 1642 3642 3642 84 85 CKAAAA ZFMAAA VVVVxx 8506 8268 0 2 6 6 6 506 506 3506 8506 12 13 EPAAAA AGMAAA AAAAxx 9621 8269 1 1 1 1 21 621 1621 4621 9621 42 43 BGAAAA BGMAAA HHHHxx 7739 8270 1 3 9 19 39 739 1739 2739 7739 78 79 RLAAAA CGMAAA OOOOxx 3987 8271 1 3 7 7 87 987 1987 3987 3987 174 175 JXAAAA DGMAAA VVVVxx 2090 8272 0 2 0 10 90 90 90 2090 2090 180 181 KCAAAA EGMAAA AAAAxx 3838 8273 0 2 8 18 38 838 1838 3838 3838 76 77 QRAAAA FGMAAA HHHHxx 17 8274 1 1 7 17 17 17 17 17 17 34 35 RAAAAA GGMAAA OOOOxx 3406 8275 0 2 6 6 6 406 1406 3406 3406 12 13 ABAAAA HGMAAA VVVVxx 8312 8276 0 0 2 12 12 312 312 3312 8312 24 25 SHAAAA IGMAAA AAAAxx 4034 8277 0 2 4 14 34 34 34 4034 4034 68 69 EZAAAA JGMAAA HHHHxx 1535 8278 1 3 5 15 35 535 1535 1535 1535 70 71 BHAAAA KGMAAA OOOOxx 7198 8279 0 2 8 18 98 198 1198 2198 7198 196 197 WQAAAA LGMAAA VVVVxx 8885 8280 1 1 5 5 85 885 885 3885 8885 170 171 TDAAAA MGMAAA AAAAxx 4081 8281 1 1 1 1 81 81 81 4081 4081 162 163 ZAAAAA NGMAAA HHHHxx 980 8282 0 0 0 0 80 980 980 980 980 160 161 SLAAAA OGMAAA OOOOxx 551 8283 1 3 1 11 51 551 551 551 551 102 103 FVAAAA PGMAAA VVVVxx 7746 8284 0 2 6 6 46 746 1746 2746 7746 92 93 YLAAAA QGMAAA AAAAxx 4756 8285 0 0 6 16 56 756 756 4756 4756 112 113 YAAAAA RGMAAA HHHHxx 3655 8286 1 3 5 15 55 655 1655 3655 3655 110 111 PKAAAA SGMAAA OOOOxx 7075 8287 1 3 5 15 75 75 1075 2075 7075 150 151 DMAAAA TGMAAA VVVVxx 3950 8288 0 2 0 10 50 950 1950 3950 3950 100 101 YVAAAA UGMAAA AAAAxx 2314 8289 0 2 4 14 14 314 314 2314 2314 28 29 ALAAAA VGMAAA HHHHxx 8432 8290 0 0 2 12 32 432 432 3432 8432 64 65 IMAAAA WGMAAA OOOOxx 62 8291 0 2 2 2 62 62 62 62 62 124 125 KCAAAA XGMAAA VVVVxx 6920 8292 0 0 0 0 20 920 920 1920 6920 40 41 EGAAAA YGMAAA AAAAxx 4077 8293 1 1 7 17 77 77 77 4077 4077 154 155 VAAAAA ZGMAAA HHHHxx 9118 8294 0 2 8 18 18 118 1118 4118 9118 36 37 SMAAAA AHMAAA OOOOxx 5375 8295 1 3 5 15 75 375 1375 375 5375 150 151 TYAAAA BHMAAA VVVVxx 178 8296 0 2 8 18 78 178 178 178 178 156 157 WGAAAA CHMAAA AAAAxx 1079 8297 1 3 9 19 79 79 1079 1079 1079 158 159 NPAAAA DHMAAA HHHHxx 4279 8298 1 3 9 19 79 279 279 4279 4279 158 159 PIAAAA EHMAAA OOOOxx 8436 8299 0 0 6 16 36 436 436 3436 8436 72 73 MMAAAA FHMAAA VVVVxx 1931 8300 1 3 1 11 31 931 1931 1931 1931 62 63 HWAAAA GHMAAA AAAAxx 2096 8301 0 0 6 16 96 96 96 2096 2096 192 193 QCAAAA HHMAAA HHHHxx 1638 8302 0 2 8 18 38 638 1638 1638 1638 76 77 ALAAAA IHMAAA OOOOxx 2788 8303 0 0 8 8 88 788 788 2788 2788 176 177 GDAAAA JHMAAA VVVVxx 4751 8304 1 3 1 11 51 751 751 4751 4751 102 103 TAAAAA KHMAAA AAAAxx 8824 8305 0 0 4 4 24 824 824 3824 8824 48 49 KBAAAA LHMAAA HHHHxx 3098 8306 0 2 8 18 98 98 1098 3098 3098 196 197 EPAAAA MHMAAA OOOOxx 4497 8307 1 1 7 17 97 497 497 4497 4497 194 195 ZQAAAA NHMAAA VVVVxx 5223 8308 1 3 3 3 23 223 1223 223 5223 46 47 XSAAAA OHMAAA AAAAxx 9212 8309 0 0 2 12 12 212 1212 4212 9212 24 25 IQAAAA PHMAAA HHHHxx 4265 8310 1 1 5 5 65 265 265 4265 4265 130 131 BIAAAA QHMAAA OOOOxx 6898 8311 0 2 8 18 98 898 898 1898 6898 196 197 IFAAAA RHMAAA VVVVxx 8808 8312 0 0 8 8 8 808 808 3808 8808 16 17 UAAAAA SHMAAA AAAAxx 5629 8313 1 1 9 9 29 629 1629 629 5629 58 59 NIAAAA THMAAA HHHHxx 3779 8314 1 3 9 19 79 779 1779 3779 3779 158 159 JPAAAA UHMAAA OOOOxx 4972 8315 0 0 2 12 72 972 972 4972 4972 144 145 GJAAAA VHMAAA VVVVxx 4511 8316 1 3 1 11 11 511 511 4511 4511 22 23 NRAAAA WHMAAA AAAAxx 6761 8317 1 1 1 1 61 761 761 1761 6761 122 123 BAAAAA XHMAAA HHHHxx 2335 8318 1 3 5 15 35 335 335 2335 2335 70 71 VLAAAA YHMAAA OOOOxx 732 8319 0 0 2 12 32 732 732 732 732 64 65 ECAAAA ZHMAAA VVVVxx 4757 8320 1 1 7 17 57 757 757 4757 4757 114 115 ZAAAAA AIMAAA AAAAxx 6624 8321 0 0 4 4 24 624 624 1624 6624 48 49 UUAAAA BIMAAA HHHHxx 5869 8322 1 1 9 9 69 869 1869 869 5869 138 139 TRAAAA CIMAAA OOOOxx 5842 8323 0 2 2 2 42 842 1842 842 5842 84 85 SQAAAA DIMAAA VVVVxx 5735 8324 1 3 5 15 35 735 1735 735 5735 70 71 PMAAAA EIMAAA AAAAxx 8276 8325 0 0 6 16 76 276 276 3276 8276 152 153 IGAAAA FIMAAA HHHHxx 7227 8326 1 3 7 7 27 227 1227 2227 7227 54 55 ZRAAAA GIMAAA OOOOxx 4923 8327 1 3 3 3 23 923 923 4923 4923 46 47 JHAAAA HIMAAA VVVVxx 9135 8328 1 3 5 15 35 135 1135 4135 9135 70 71 JNAAAA IIMAAA AAAAxx 5813 8329 1 1 3 13 13 813 1813 813 5813 26 27 PPAAAA JIMAAA HHHHxx 9697 8330 1 1 7 17 97 697 1697 4697 9697 194 195 ZIAAAA KIMAAA OOOOxx 3222 8331 0 2 2 2 22 222 1222 3222 3222 44 45 YTAAAA LIMAAA VVVVxx 2394 8332 0 2 4 14 94 394 394 2394 2394 188 189 COAAAA MIMAAA AAAAxx 5784 8333 0 0 4 4 84 784 1784 784 5784 168 169 MOAAAA NIMAAA HHHHxx 3652 8334 0 0 2 12 52 652 1652 3652 3652 104 105 MKAAAA OIMAAA OOOOxx 8175 8335 1 3 5 15 75 175 175 3175 8175 150 151 LCAAAA PIMAAA VVVVxx 7568 8336 0 0 8 8 68 568 1568 2568 7568 136 137 CFAAAA QIMAAA AAAAxx 6645 8337 1 1 5 5 45 645 645 1645 6645 90 91 PVAAAA RIMAAA HHHHxx 8176 8338 0 0 6 16 76 176 176 3176 8176 152 153 MCAAAA SIMAAA OOOOxx 530 8339 0 2 0 10 30 530 530 530 530 60 61 KUAAAA TIMAAA VVVVxx 5439 8340 1 3 9 19 39 439 1439 439 5439 78 79 FBAAAA UIMAAA AAAAxx 61 8341 1 1 1 1 61 61 61 61 61 122 123 JCAAAA VIMAAA HHHHxx 3951 8342 1 3 1 11 51 951 1951 3951 3951 102 103 ZVAAAA WIMAAA OOOOxx 5283 8343 1 3 3 3 83 283 1283 283 5283 166 167 FVAAAA XIMAAA VVVVxx 7226 8344 0 2 6 6 26 226 1226 2226 7226 52 53 YRAAAA YIMAAA AAAAxx 1954 8345 0 2 4 14 54 954 1954 1954 1954 108 109 EXAAAA ZIMAAA HHHHxx 334 8346 0 2 4 14 34 334 334 334 334 68 69 WMAAAA AJMAAA OOOOxx 3921 8347 1 1 1 1 21 921 1921 3921 3921 42 43 VUAAAA BJMAAA VVVVxx 6276 8348 0 0 6 16 76 276 276 1276 6276 152 153 KHAAAA CJMAAA AAAAxx 3378 8349 0 2 8 18 78 378 1378 3378 3378 156 157 YZAAAA DJMAAA HHHHxx 5236 8350 0 0 6 16 36 236 1236 236 5236 72 73 KTAAAA EJMAAA OOOOxx 7781 8351 1 1 1 1 81 781 1781 2781 7781 162 163 HNAAAA FJMAAA VVVVxx 8601 8352 1 1 1 1 1 601 601 3601 8601 2 3 VSAAAA GJMAAA AAAAxx 1473 8353 1 1 3 13 73 473 1473 1473 1473 146 147 REAAAA HJMAAA HHHHxx 3246 8354 0 2 6 6 46 246 1246 3246 3246 92 93 WUAAAA IJMAAA OOOOxx 3601 8355 1 1 1 1 1 601 1601 3601 3601 2 3 NIAAAA JJMAAA VVVVxx 6861 8356 1 1 1 1 61 861 861 1861 6861 122 123 XDAAAA KJMAAA AAAAxx 9032 8357 0 0 2 12 32 32 1032 4032 9032 64 65 KJAAAA LJMAAA HHHHxx 216 8358 0 0 6 16 16 216 216 216 216 32 33 IIAAAA MJMAAA OOOOxx 3824 8359 0 0 4 4 24 824 1824 3824 3824 48 49 CRAAAA NJMAAA VVVVxx 8486 8360 0 2 6 6 86 486 486 3486 8486 172 173 KOAAAA OJMAAA AAAAxx 276 8361 0 0 6 16 76 276 276 276 276 152 153 QKAAAA PJMAAA HHHHxx 1838 8362 0 2 8 18 38 838 1838 1838 1838 76 77 SSAAAA QJMAAA OOOOxx 6175 8363 1 3 5 15 75 175 175 1175 6175 150 151 NDAAAA RJMAAA VVVVxx 3719 8364 1 3 9 19 19 719 1719 3719 3719 38 39 BNAAAA SJMAAA AAAAxx 6958 8365 0 2 8 18 58 958 958 1958 6958 116 117 QHAAAA TJMAAA HHHHxx 6822 8366 0 2 2 2 22 822 822 1822 6822 44 45 KCAAAA UJMAAA OOOOxx 3318 8367 0 2 8 18 18 318 1318 3318 3318 36 37 QXAAAA VJMAAA VVVVxx 7222 8368 0 2 2 2 22 222 1222 2222 7222 44 45 URAAAA WJMAAA AAAAxx 85 8369 1 1 5 5 85 85 85 85 85 170 171 HDAAAA XJMAAA HHHHxx 5158 8370 0 2 8 18 58 158 1158 158 5158 116 117 KQAAAA YJMAAA OOOOxx 6360 8371 0 0 0 0 60 360 360 1360 6360 120 121 QKAAAA ZJMAAA VVVVxx 2599 8372 1 3 9 19 99 599 599 2599 2599 198 199 ZVAAAA AKMAAA AAAAxx 4002 8373 0 2 2 2 2 2 2 4002 4002 4 5 YXAAAA BKMAAA HHHHxx 6597 8374 1 1 7 17 97 597 597 1597 6597 194 195 TTAAAA CKMAAA OOOOxx 5762 8375 0 2 2 2 62 762 1762 762 5762 124 125 QNAAAA DKMAAA VVVVxx 8383 8376 1 3 3 3 83 383 383 3383 8383 166 167 LKAAAA EKMAAA AAAAxx 4686 8377 0 2 6 6 86 686 686 4686 4686 172 173 GYAAAA FKMAAA HHHHxx 5972 8378 0 0 2 12 72 972 1972 972 5972 144 145 SVAAAA GKMAAA OOOOxx 1432 8379 0 0 2 12 32 432 1432 1432 1432 64 65 CDAAAA HKMAAA VVVVxx 1601 8380 1 1 1 1 1 601 1601 1601 1601 2 3 PJAAAA IKMAAA AAAAxx 3012 8381 0 0 2 12 12 12 1012 3012 3012 24 25 WLAAAA JKMAAA HHHHxx 9345 8382 1 1 5 5 45 345 1345 4345 9345 90 91 LVAAAA KKMAAA OOOOxx 8869 8383 1 1 9 9 69 869 869 3869 8869 138 139 DDAAAA LKMAAA VVVVxx 6612 8384 0 0 2 12 12 612 612 1612 6612 24 25 IUAAAA MKMAAA AAAAxx 262 8385 0 2 2 2 62 262 262 262 262 124 125 CKAAAA NKMAAA HHHHxx 300 8386 0 0 0 0 0 300 300 300 300 0 1 OLAAAA OKMAAA OOOOxx 3045 8387 1 1 5 5 45 45 1045 3045 3045 90 91 DNAAAA PKMAAA VVVVxx 7252 8388 0 0 2 12 52 252 1252 2252 7252 104 105 YSAAAA QKMAAA AAAAxx 9099 8389 1 3 9 19 99 99 1099 4099 9099 198 199 ZLAAAA RKMAAA HHHHxx 9006 8390 0 2 6 6 6 6 1006 4006 9006 12 13 KIAAAA SKMAAA OOOOxx 3078 8391 0 2 8 18 78 78 1078 3078 3078 156 157 KOAAAA TKMAAA VVVVxx 5159 8392 1 3 9 19 59 159 1159 159 5159 118 119 LQAAAA UKMAAA AAAAxx 9329 8393 1 1 9 9 29 329 1329 4329 9329 58 59 VUAAAA VKMAAA HHHHxx 1393 8394 1 1 3 13 93 393 1393 1393 1393 186 187 PBAAAA WKMAAA OOOOxx 5894 8395 0 2 4 14 94 894 1894 894 5894 188 189 SSAAAA XKMAAA VVVVxx 11 8396 1 3 1 11 11 11 11 11 11 22 23 LAAAAA YKMAAA AAAAxx 5606 8397 0 2 6 6 6 606 1606 606 5606 12 13 QHAAAA ZKMAAA HHHHxx 5541 8398 1 1 1 1 41 541 1541 541 5541 82 83 DFAAAA ALMAAA OOOOxx 2689 8399 1 1 9 9 89 689 689 2689 2689 178 179 LZAAAA BLMAAA VVVVxx 1023 8400 1 3 3 3 23 23 1023 1023 1023 46 47 JNAAAA CLMAAA AAAAxx 8134 8401 0 2 4 14 34 134 134 3134 8134 68 69 WAAAAA DLMAAA HHHHxx 5923 8402 1 3 3 3 23 923 1923 923 5923 46 47 VTAAAA ELMAAA OOOOxx 6056 8403 0 0 6 16 56 56 56 1056 6056 112 113 YYAAAA FLMAAA VVVVxx 653 8404 1 1 3 13 53 653 653 653 653 106 107 DZAAAA GLMAAA AAAAxx 367 8405 1 3 7 7 67 367 367 367 367 134 135 DOAAAA HLMAAA HHHHxx 1828 8406 0 0 8 8 28 828 1828 1828 1828 56 57 ISAAAA ILMAAA OOOOxx 6506 8407 0 2 6 6 6 506 506 1506 6506 12 13 GQAAAA JLMAAA VVVVxx 5772 8408 0 0 2 12 72 772 1772 772 5772 144 145 AOAAAA KLMAAA AAAAxx 8052 8409 0 0 2 12 52 52 52 3052 8052 104 105 SXAAAA LLMAAA HHHHxx 2633 8410 1 1 3 13 33 633 633 2633 2633 66 67 HXAAAA MLMAAA OOOOxx 4878 8411 0 2 8 18 78 878 878 4878 4878 156 157 QFAAAA NLMAAA VVVVxx 5621 8412 1 1 1 1 21 621 1621 621 5621 42 43 FIAAAA OLMAAA AAAAxx 41 8413 1 1 1 1 41 41 41 41 41 82 83 PBAAAA PLMAAA HHHHxx 4613 8414 1 1 3 13 13 613 613 4613 4613 26 27 LVAAAA QLMAAA OOOOxx 9389 8415 1 1 9 9 89 389 1389 4389 9389 178 179 DXAAAA RLMAAA VVVVxx 9414 8416 0 2 4 14 14 414 1414 4414 9414 28 29 CYAAAA SLMAAA AAAAxx 3583 8417 1 3 3 3 83 583 1583 3583 3583 166 167 VHAAAA TLMAAA HHHHxx 3454 8418 0 2 4 14 54 454 1454 3454 3454 108 109 WCAAAA ULMAAA OOOOxx 719 8419 1 3 9 19 19 719 719 719 719 38 39 RBAAAA VLMAAA VVVVxx 6188 8420 0 0 8 8 88 188 188 1188 6188 176 177 AEAAAA WLMAAA AAAAxx 2288 8421 0 0 8 8 88 288 288 2288 2288 176 177 AKAAAA XLMAAA HHHHxx 1287 8422 1 3 7 7 87 287 1287 1287 1287 174 175 NXAAAA YLMAAA OOOOxx 1397 8423 1 1 7 17 97 397 1397 1397 1397 194 195 TBAAAA ZLMAAA VVVVxx 7763 8424 1 3 3 3 63 763 1763 2763 7763 126 127 PMAAAA AMMAAA AAAAxx 5194 8425 0 2 4 14 94 194 1194 194 5194 188 189 URAAAA BMMAAA HHHHxx 3167 8426 1 3 7 7 67 167 1167 3167 3167 134 135 VRAAAA CMMAAA OOOOxx 9218 8427 0 2 8 18 18 218 1218 4218 9218 36 37 OQAAAA DMMAAA VVVVxx 2065 8428 1 1 5 5 65 65 65 2065 2065 130 131 LBAAAA EMMAAA AAAAxx 9669 8429 1 1 9 9 69 669 1669 4669 9669 138 139 XHAAAA FMMAAA HHHHxx 146 8430 0 2 6 6 46 146 146 146 146 92 93 QFAAAA GMMAAA OOOOxx 6141 8431 1 1 1 1 41 141 141 1141 6141 82 83 FCAAAA HMMAAA VVVVxx 2843 8432 1 3 3 3 43 843 843 2843 2843 86 87 JFAAAA IMMAAA AAAAxx 7934 8433 0 2 4 14 34 934 1934 2934 7934 68 69 ETAAAA JMMAAA HHHHxx 2536 8434 0 0 6 16 36 536 536 2536 2536 72 73 OTAAAA KMMAAA OOOOxx 7088 8435 0 0 8 8 88 88 1088 2088 7088 176 177 QMAAAA LMMAAA VVVVxx 2519 8436 1 3 9 19 19 519 519 2519 2519 38 39 XSAAAA MMMAAA AAAAxx 6650 8437 0 2 0 10 50 650 650 1650 6650 100 101 UVAAAA NMMAAA HHHHxx 3007 8438 1 3 7 7 7 7 1007 3007 3007 14 15 RLAAAA OMMAAA OOOOxx 4507 8439 1 3 7 7 7 507 507 4507 4507 14 15 JRAAAA PMMAAA VVVVxx 4892 8440 0 0 2 12 92 892 892 4892 4892 184 185 EGAAAA QMMAAA AAAAxx 7159 8441 1 3 9 19 59 159 1159 2159 7159 118 119 JPAAAA RMMAAA HHHHxx 3171 8442 1 3 1 11 71 171 1171 3171 3171 142 143 ZRAAAA SMMAAA OOOOxx 1080 8443 0 0 0 0 80 80 1080 1080 1080 160 161 OPAAAA TMMAAA VVVVxx 7248 8444 0 0 8 8 48 248 1248 2248 7248 96 97 USAAAA UMMAAA AAAAxx 7230 8445 0 2 0 10 30 230 1230 2230 7230 60 61 CSAAAA VMMAAA HHHHxx 3823 8446 1 3 3 3 23 823 1823 3823 3823 46 47 BRAAAA WMMAAA OOOOxx 5517 8447 1 1 7 17 17 517 1517 517 5517 34 35 FEAAAA XMMAAA VVVVxx 1482 8448 0 2 2 2 82 482 1482 1482 1482 164 165 AFAAAA YMMAAA AAAAxx 9953 8449 1 1 3 13 53 953 1953 4953 9953 106 107 VSAAAA ZMMAAA HHHHxx 2754 8450 0 2 4 14 54 754 754 2754 2754 108 109 YBAAAA ANMAAA OOOOxx 3875 8451 1 3 5 15 75 875 1875 3875 3875 150 151 BTAAAA BNMAAA VVVVxx 9800 8452 0 0 0 0 0 800 1800 4800 9800 0 1 YMAAAA CNMAAA AAAAxx 8819 8453 1 3 9 19 19 819 819 3819 8819 38 39 FBAAAA DNMAAA HHHHxx 8267 8454 1 3 7 7 67 267 267 3267 8267 134 135 ZFAAAA ENMAAA OOOOxx 520 8455 0 0 0 0 20 520 520 520 520 40 41 AUAAAA FNMAAA VVVVxx 5770 8456 0 2 0 10 70 770 1770 770 5770 140 141 YNAAAA GNMAAA AAAAxx 2114 8457 0 2 4 14 14 114 114 2114 2114 28 29 IDAAAA HNMAAA HHHHxx 5045 8458 1 1 5 5 45 45 1045 45 5045 90 91 BMAAAA INMAAA OOOOxx 1094 8459 0 2 4 14 94 94 1094 1094 1094 188 189 CQAAAA JNMAAA VVVVxx 8786 8460 0 2 6 6 86 786 786 3786 8786 172 173 YZAAAA KNMAAA AAAAxx 353 8461 1 1 3 13 53 353 353 353 353 106 107 PNAAAA LNMAAA HHHHxx 290 8462 0 2 0 10 90 290 290 290 290 180 181 ELAAAA MNMAAA OOOOxx 3376 8463 0 0 6 16 76 376 1376 3376 3376 152 153 WZAAAA NNMAAA VVVVxx 9305 8464 1 1 5 5 5 305 1305 4305 9305 10 11 XTAAAA ONMAAA AAAAxx 186 8465 0 2 6 6 86 186 186 186 186 172 173 EHAAAA PNMAAA HHHHxx 4817 8466 1 1 7 17 17 817 817 4817 4817 34 35 HDAAAA QNMAAA OOOOxx 4638 8467 0 2 8 18 38 638 638 4638 4638 76 77 KWAAAA RNMAAA VVVVxx 3558 8468 0 2 8 18 58 558 1558 3558 3558 116 117 WGAAAA SNMAAA AAAAxx 9285 8469 1 1 5 5 85 285 1285 4285 9285 170 171 DTAAAA TNMAAA HHHHxx 848 8470 0 0 8 8 48 848 848 848 848 96 97 QGAAAA UNMAAA OOOOxx 8923 8471 1 3 3 3 23 923 923 3923 8923 46 47 FFAAAA VNMAAA VVVVxx 6826 8472 0 2 6 6 26 826 826 1826 6826 52 53 OCAAAA WNMAAA AAAAxx 5187 8473 1 3 7 7 87 187 1187 187 5187 174 175 NRAAAA XNMAAA HHHHxx 2398 8474 0 2 8 18 98 398 398 2398 2398 196 197 GOAAAA YNMAAA OOOOxx 7653 8475 1 1 3 13 53 653 1653 2653 7653 106 107 JIAAAA ZNMAAA VVVVxx 8835 8476 1 3 5 15 35 835 835 3835 8835 70 71 VBAAAA AOMAAA AAAAxx 5736 8477 0 0 6 16 36 736 1736 736 5736 72 73 QMAAAA BOMAAA HHHHxx 1238 8478 0 2 8 18 38 238 1238 1238 1238 76 77 QVAAAA COMAAA OOOOxx 6021 8479 1 1 1 1 21 21 21 1021 6021 42 43 PXAAAA DOMAAA VVVVxx 6815 8480 1 3 5 15 15 815 815 1815 6815 30 31 DCAAAA EOMAAA AAAAxx 2549 8481 1 1 9 9 49 549 549 2549 2549 98 99 BUAAAA FOMAAA HHHHxx 5657 8482 1 1 7 17 57 657 1657 657 5657 114 115 PJAAAA GOMAAA OOOOxx 6855 8483 1 3 5 15 55 855 855 1855 6855 110 111 RDAAAA HOMAAA VVVVxx 1225 8484 1 1 5 5 25 225 1225 1225 1225 50 51 DVAAAA IOMAAA AAAAxx 7452 8485 0 0 2 12 52 452 1452 2452 7452 104 105 QAAAAA JOMAAA HHHHxx 2479 8486 1 3 9 19 79 479 479 2479 2479 158 159 JRAAAA KOMAAA OOOOxx 7974 8487 0 2 4 14 74 974 1974 2974 7974 148 149 SUAAAA LOMAAA VVVVxx 1212 8488 0 0 2 12 12 212 1212 1212 1212 24 25 QUAAAA MOMAAA AAAAxx 8883 8489 1 3 3 3 83 883 883 3883 8883 166 167 RDAAAA NOMAAA HHHHxx 8150 8490 0 2 0 10 50 150 150 3150 8150 100 101 MBAAAA OOMAAA OOOOxx 3392 8491 0 0 2 12 92 392 1392 3392 3392 184 185 MAAAAA POMAAA VVVVxx 6774 8492 0 2 4 14 74 774 774 1774 6774 148 149 OAAAAA QOMAAA AAAAxx 904 8493 0 0 4 4 4 904 904 904 904 8 9 UIAAAA ROMAAA HHHHxx 5068 8494 0 0 8 8 68 68 1068 68 5068 136 137 YMAAAA SOMAAA OOOOxx 9339 8495 1 3 9 19 39 339 1339 4339 9339 78 79 FVAAAA TOMAAA VVVVxx 1062 8496 0 2 2 2 62 62 1062 1062 1062 124 125 WOAAAA UOMAAA AAAAxx 3841 8497 1 1 1 1 41 841 1841 3841 3841 82 83 TRAAAA VOMAAA HHHHxx 8924 8498 0 0 4 4 24 924 924 3924 8924 48 49 GFAAAA WOMAAA OOOOxx 9795 8499 1 3 5 15 95 795 1795 4795 9795 190 191 TMAAAA XOMAAA VVVVxx 3981 8500 1 1 1 1 81 981 1981 3981 3981 162 163 DXAAAA YOMAAA AAAAxx 4290 8501 0 2 0 10 90 290 290 4290 4290 180 181 AJAAAA ZOMAAA HHHHxx 1067 8502 1 3 7 7 67 67 1067 1067 1067 134 135 BPAAAA APMAAA OOOOxx 8679 8503 1 3 9 19 79 679 679 3679 8679 158 159 VVAAAA BPMAAA VVVVxx 2894 8504 0 2 4 14 94 894 894 2894 2894 188 189 IHAAAA CPMAAA AAAAxx 9248 8505 0 0 8 8 48 248 1248 4248 9248 96 97 SRAAAA DPMAAA HHHHxx 1072 8506 0 0 2 12 72 72 1072 1072 1072 144 145 GPAAAA EPMAAA OOOOxx 3510 8507 0 2 0 10 10 510 1510 3510 3510 20 21 AFAAAA FPMAAA VVVVxx 6871 8508 1 3 1 11 71 871 871 1871 6871 142 143 HEAAAA GPMAAA AAAAxx 8701 8509 1 1 1 1 1 701 701 3701 8701 2 3 RWAAAA HPMAAA HHHHxx 8170 8510 0 2 0 10 70 170 170 3170 8170 140 141 GCAAAA IPMAAA OOOOxx 2730 8511 0 2 0 10 30 730 730 2730 2730 60 61 ABAAAA JPMAAA VVVVxx 2668 8512 0 0 8 8 68 668 668 2668 2668 136 137 QYAAAA KPMAAA AAAAxx 8723 8513 1 3 3 3 23 723 723 3723 8723 46 47 NXAAAA LPMAAA HHHHxx 3439 8514 1 3 9 19 39 439 1439 3439 3439 78 79 HCAAAA MPMAAA OOOOxx 6219 8515 1 3 9 19 19 219 219 1219 6219 38 39 FFAAAA NPMAAA VVVVxx 4264 8516 0 0 4 4 64 264 264 4264 4264 128 129 AIAAAA OPMAAA AAAAxx 3929 8517 1 1 9 9 29 929 1929 3929 3929 58 59 DVAAAA PPMAAA HHHHxx 7 8518 1 3 7 7 7 7 7 7 7 14 15 HAAAAA QPMAAA OOOOxx 3737 8519 1 1 7 17 37 737 1737 3737 3737 74 75 TNAAAA RPMAAA VVVVxx 358 8520 0 2 8 18 58 358 358 358 358 116 117 UNAAAA SPMAAA AAAAxx 5128 8521 0 0 8 8 28 128 1128 128 5128 56 57 GPAAAA TPMAAA HHHHxx 7353 8522 1 1 3 13 53 353 1353 2353 7353 106 107 VWAAAA UPMAAA OOOOxx 8758 8523 0 2 8 18 58 758 758 3758 8758 116 117 WYAAAA VPMAAA VVVVxx 7284 8524 0 0 4 4 84 284 1284 2284 7284 168 169 EUAAAA WPMAAA AAAAxx 4037 8525 1 1 7 17 37 37 37 4037 4037 74 75 HZAAAA XPMAAA HHHHxx 435 8526 1 3 5 15 35 435 435 435 435 70 71 TQAAAA YPMAAA OOOOxx 3580 8527 0 0 0 0 80 580 1580 3580 3580 160 161 SHAAAA ZPMAAA VVVVxx 4554 8528 0 2 4 14 54 554 554 4554 4554 108 109 ETAAAA AQMAAA AAAAxx 4337 8529 1 1 7 17 37 337 337 4337 4337 74 75 VKAAAA BQMAAA HHHHxx 512 8530 0 0 2 12 12 512 512 512 512 24 25 STAAAA CQMAAA OOOOxx 2032 8531 0 0 2 12 32 32 32 2032 2032 64 65 EAAAAA DQMAAA VVVVxx 1755 8532 1 3 5 15 55 755 1755 1755 1755 110 111 NPAAAA EQMAAA AAAAxx 9923 8533 1 3 3 3 23 923 1923 4923 9923 46 47 RRAAAA FQMAAA HHHHxx 3747 8534 1 3 7 7 47 747 1747 3747 3747 94 95 DOAAAA GQMAAA OOOOxx 27 8535 1 3 7 7 27 27 27 27 27 54 55 BBAAAA HQMAAA VVVVxx 3075 8536 1 3 5 15 75 75 1075 3075 3075 150 151 HOAAAA IQMAAA AAAAxx 6259 8537 1 3 9 19 59 259 259 1259 6259 118 119 TGAAAA JQMAAA HHHHxx 2940 8538 0 0 0 0 40 940 940 2940 2940 80 81 CJAAAA KQMAAA OOOOxx 5724 8539 0 0 4 4 24 724 1724 724 5724 48 49 EMAAAA LQMAAA VVVVxx 5638 8540 0 2 8 18 38 638 1638 638 5638 76 77 WIAAAA MQMAAA AAAAxx 479 8541 1 3 9 19 79 479 479 479 479 158 159 LSAAAA NQMAAA HHHHxx 4125 8542 1 1 5 5 25 125 125 4125 4125 50 51 RCAAAA OQMAAA OOOOxx 1525 8543 1 1 5 5 25 525 1525 1525 1525 50 51 RGAAAA PQMAAA VVVVxx 7529 8544 1 1 9 9 29 529 1529 2529 7529 58 59 PDAAAA QQMAAA AAAAxx 931 8545 1 3 1 11 31 931 931 931 931 62 63 VJAAAA RQMAAA HHHHxx 5175 8546 1 3 5 15 75 175 1175 175 5175 150 151 BRAAAA SQMAAA OOOOxx 6798 8547 0 2 8 18 98 798 798 1798 6798 196 197 MBAAAA TQMAAA VVVVxx 2111 8548 1 3 1 11 11 111 111 2111 2111 22 23 FDAAAA UQMAAA AAAAxx 6145 8549 1 1 5 5 45 145 145 1145 6145 90 91 JCAAAA VQMAAA HHHHxx 4712 8550 0 0 2 12 12 712 712 4712 4712 24 25 GZAAAA WQMAAA OOOOxx 3110 8551 0 2 0 10 10 110 1110 3110 3110 20 21 QPAAAA XQMAAA VVVVxx 97 8552 1 1 7 17 97 97 97 97 97 194 195 TDAAAA YQMAAA AAAAxx 758 8553 0 2 8 18 58 758 758 758 758 116 117 EDAAAA ZQMAAA HHHHxx 1895 8554 1 3 5 15 95 895 1895 1895 1895 190 191 XUAAAA ARMAAA OOOOxx 5289 8555 1 1 9 9 89 289 1289 289 5289 178 179 LVAAAA BRMAAA VVVVxx 5026 8556 0 2 6 6 26 26 1026 26 5026 52 53 ILAAAA CRMAAA AAAAxx 4725 8557 1 1 5 5 25 725 725 4725 4725 50 51 TZAAAA DRMAAA HHHHxx 1679 8558 1 3 9 19 79 679 1679 1679 1679 158 159 PMAAAA ERMAAA OOOOxx 4433 8559 1 1 3 13 33 433 433 4433 4433 66 67 NOAAAA FRMAAA VVVVxx 5340 8560 0 0 0 0 40 340 1340 340 5340 80 81 KXAAAA GRMAAA AAAAxx 6340 8561 0 0 0 0 40 340 340 1340 6340 80 81 WJAAAA HRMAAA HHHHxx 3261 8562 1 1 1 1 61 261 1261 3261 3261 122 123 LVAAAA IRMAAA OOOOxx 8108 8563 0 0 8 8 8 108 108 3108 8108 16 17 WZAAAA JRMAAA VVVVxx 8785 8564 1 1 5 5 85 785 785 3785 8785 170 171 XZAAAA KRMAAA AAAAxx 7391 8565 1 3 1 11 91 391 1391 2391 7391 182 183 HYAAAA LRMAAA HHHHxx 1496 8566 0 0 6 16 96 496 1496 1496 1496 192 193 OFAAAA MRMAAA OOOOxx 1484 8567 0 0 4 4 84 484 1484 1484 1484 168 169 CFAAAA NRMAAA VVVVxx 5884 8568 0 0 4 4 84 884 1884 884 5884 168 169 ISAAAA ORMAAA AAAAxx 342 8569 0 2 2 2 42 342 342 342 342 84 85 ENAAAA PRMAAA HHHHxx 7659 8570 1 3 9 19 59 659 1659 2659 7659 118 119 PIAAAA QRMAAA OOOOxx 6635 8571 1 3 5 15 35 635 635 1635 6635 70 71 FVAAAA RRMAAA VVVVxx 8507 8572 1 3 7 7 7 507 507 3507 8507 14 15 FPAAAA SRMAAA AAAAxx 2583 8573 1 3 3 3 83 583 583 2583 2583 166 167 JVAAAA TRMAAA HHHHxx 6533 8574 1 1 3 13 33 533 533 1533 6533 66 67 HRAAAA URMAAA OOOOxx 5879 8575 1 3 9 19 79 879 1879 879 5879 158 159 DSAAAA VRMAAA VVVVxx 5511 8576 1 3 1 11 11 511 1511 511 5511 22 23 ZDAAAA WRMAAA AAAAxx 3682 8577 0 2 2 2 82 682 1682 3682 3682 164 165 QLAAAA XRMAAA HHHHxx 7182 8578 0 2 2 2 82 182 1182 2182 7182 164 165 GQAAAA YRMAAA OOOOxx 1409 8579 1 1 9 9 9 409 1409 1409 1409 18 19 FCAAAA ZRMAAA VVVVxx 3363 8580 1 3 3 3 63 363 1363 3363 3363 126 127 JZAAAA ASMAAA AAAAxx 729 8581 1 1 9 9 29 729 729 729 729 58 59 BCAAAA BSMAAA HHHHxx 5857 8582 1 1 7 17 57 857 1857 857 5857 114 115 HRAAAA CSMAAA OOOOxx 235 8583 1 3 5 15 35 235 235 235 235 70 71 BJAAAA DSMAAA VVVVxx 193 8584 1 1 3 13 93 193 193 193 193 186 187 LHAAAA ESMAAA AAAAxx 5586 8585 0 2 6 6 86 586 1586 586 5586 172 173 WGAAAA FSMAAA HHHHxx 6203 8586 1 3 3 3 3 203 203 1203 6203 6 7 PEAAAA GSMAAA OOOOxx 6795 8587 1 3 5 15 95 795 795 1795 6795 190 191 JBAAAA HSMAAA VVVVxx 3211 8588 1 3 1 11 11 211 1211 3211 3211 22 23 NTAAAA ISMAAA AAAAxx 9763 8589 1 3 3 3 63 763 1763 4763 9763 126 127 NLAAAA JSMAAA HHHHxx 9043 8590 1 3 3 3 43 43 1043 4043 9043 86 87 VJAAAA KSMAAA OOOOxx 2854 8591 0 2 4 14 54 854 854 2854 2854 108 109 UFAAAA LSMAAA VVVVxx 565 8592 1 1 5 5 65 565 565 565 565 130 131 TVAAAA MSMAAA AAAAxx 9284 8593 0 0 4 4 84 284 1284 4284 9284 168 169 CTAAAA NSMAAA HHHHxx 7886 8594 0 2 6 6 86 886 1886 2886 7886 172 173 IRAAAA OSMAAA OOOOxx 122 8595 0 2 2 2 22 122 122 122 122 44 45 SEAAAA PSMAAA VVVVxx 4934 8596 0 2 4 14 34 934 934 4934 4934 68 69 UHAAAA QSMAAA AAAAxx 1766 8597 0 2 6 6 66 766 1766 1766 1766 132 133 YPAAAA RSMAAA HHHHxx 2554 8598 0 2 4 14 54 554 554 2554 2554 108 109 GUAAAA SSMAAA OOOOxx 488 8599 0 0 8 8 88 488 488 488 488 176 177 USAAAA TSMAAA VVVVxx 825 8600 1 1 5 5 25 825 825 825 825 50 51 TFAAAA USMAAA AAAAxx 678 8601 0 2 8 18 78 678 678 678 678 156 157 CAAAAA VSMAAA HHHHxx 4543 8602 1 3 3 3 43 543 543 4543 4543 86 87 TSAAAA WSMAAA OOOOxx 1699 8603 1 3 9 19 99 699 1699 1699 1699 198 199 JNAAAA XSMAAA VVVVxx 3771 8604 1 3 1 11 71 771 1771 3771 3771 142 143 BPAAAA YSMAAA AAAAxx 1234 8605 0 2 4 14 34 234 1234 1234 1234 68 69 MVAAAA ZSMAAA HHHHxx 4152 8606 0 0 2 12 52 152 152 4152 4152 104 105 SDAAAA ATMAAA OOOOxx 1632 8607 0 0 2 12 32 632 1632 1632 1632 64 65 UKAAAA BTMAAA VVVVxx 4988 8608 0 0 8 8 88 988 988 4988 4988 176 177 WJAAAA CTMAAA AAAAxx 1980 8609 0 0 0 0 80 980 1980 1980 1980 160 161 EYAAAA DTMAAA HHHHxx 7479 8610 1 3 9 19 79 479 1479 2479 7479 158 159 RBAAAA ETMAAA OOOOxx 2586 8611 0 2 6 6 86 586 586 2586 2586 172 173 MVAAAA FTMAAA VVVVxx 5433 8612 1 1 3 13 33 433 1433 433 5433 66 67 ZAAAAA GTMAAA AAAAxx 2261 8613 1 1 1 1 61 261 261 2261 2261 122 123 ZIAAAA HTMAAA HHHHxx 1180 8614 0 0 0 0 80 180 1180 1180 1180 160 161 KTAAAA ITMAAA OOOOxx 3938 8615 0 2 8 18 38 938 1938 3938 3938 76 77 MVAAAA JTMAAA VVVVxx 6714 8616 0 2 4 14 14 714 714 1714 6714 28 29 GYAAAA KTMAAA AAAAxx 2890 8617 0 2 0 10 90 890 890 2890 2890 180 181 EHAAAA LTMAAA HHHHxx 7379 8618 1 3 9 19 79 379 1379 2379 7379 158 159 VXAAAA MTMAAA OOOOxx 5896 8619 0 0 6 16 96 896 1896 896 5896 192 193 USAAAA NTMAAA VVVVxx 5949 8620 1 1 9 9 49 949 1949 949 5949 98 99 VUAAAA OTMAAA AAAAxx 3194 8621 0 2 4 14 94 194 1194 3194 3194 188 189 WSAAAA PTMAAA HHHHxx 9325 8622 1 1 5 5 25 325 1325 4325 9325 50 51 RUAAAA QTMAAA OOOOxx 9531 8623 1 3 1 11 31 531 1531 4531 9531 62 63 PCAAAA RTMAAA VVVVxx 711 8624 1 3 1 11 11 711 711 711 711 22 23 JBAAAA STMAAA AAAAxx 2450 8625 0 2 0 10 50 450 450 2450 2450 100 101 GQAAAA TTMAAA HHHHxx 1929 8626 1 1 9 9 29 929 1929 1929 1929 58 59 FWAAAA UTMAAA OOOOxx 6165 8627 1 1 5 5 65 165 165 1165 6165 130 131 DDAAAA VTMAAA VVVVxx 4050 8628 0 2 0 10 50 50 50 4050 4050 100 101 UZAAAA WTMAAA AAAAxx 9011 8629 1 3 1 11 11 11 1011 4011 9011 22 23 PIAAAA XTMAAA HHHHxx 7916 8630 0 0 6 16 16 916 1916 2916 7916 32 33 MSAAAA YTMAAA OOOOxx 9136 8631 0 0 6 16 36 136 1136 4136 9136 72 73 KNAAAA ZTMAAA VVVVxx 8782 8632 0 2 2 2 82 782 782 3782 8782 164 165 UZAAAA AUMAAA AAAAxx 8491 8633 1 3 1 11 91 491 491 3491 8491 182 183 POAAAA BUMAAA HHHHxx 5114 8634 0 2 4 14 14 114 1114 114 5114 28 29 SOAAAA CUMAAA OOOOxx 5815 8635 1 3 5 15 15 815 1815 815 5815 30 31 RPAAAA DUMAAA VVVVxx 5628 8636 0 0 8 8 28 628 1628 628 5628 56 57 MIAAAA EUMAAA AAAAxx 810 8637 0 2 0 10 10 810 810 810 810 20 21 EFAAAA FUMAAA HHHHxx 6178 8638 0 2 8 18 78 178 178 1178 6178 156 157 QDAAAA GUMAAA OOOOxx 2619 8639 1 3 9 19 19 619 619 2619 2619 38 39 TWAAAA HUMAAA VVVVxx 3340 8640 0 0 0 0 40 340 1340 3340 3340 80 81 MYAAAA IUMAAA AAAAxx 2491 8641 1 3 1 11 91 491 491 2491 2491 182 183 VRAAAA JUMAAA HHHHxx 3574 8642 0 2 4 14 74 574 1574 3574 3574 148 149 MHAAAA KUMAAA OOOOxx 6754 8643 0 2 4 14 54 754 754 1754 6754 108 109 UZAAAA LUMAAA VVVVxx 1566 8644 0 2 6 6 66 566 1566 1566 1566 132 133 GIAAAA MUMAAA AAAAxx 9174 8645 0 2 4 14 74 174 1174 4174 9174 148 149 WOAAAA NUMAAA HHHHxx 1520 8646 0 0 0 0 20 520 1520 1520 1520 40 41 MGAAAA OUMAAA OOOOxx 2691 8647 1 3 1 11 91 691 691 2691 2691 182 183 NZAAAA PUMAAA VVVVxx 6961 8648 1 1 1 1 61 961 961 1961 6961 122 123 THAAAA QUMAAA AAAAxx 5722 8649 0 2 2 2 22 722 1722 722 5722 44 45 CMAAAA RUMAAA HHHHxx 9707 8650 1 3 7 7 7 707 1707 4707 9707 14 15 JJAAAA SUMAAA OOOOxx 2891 8651 1 3 1 11 91 891 891 2891 2891 182 183 FHAAAA TUMAAA VVVVxx 341 8652 1 1 1 1 41 341 341 341 341 82 83 DNAAAA UUMAAA AAAAxx 4690 8653 0 2 0 10 90 690 690 4690 4690 180 181 KYAAAA VUMAAA HHHHxx 7841 8654 1 1 1 1 41 841 1841 2841 7841 82 83 PPAAAA WUMAAA OOOOxx 6615 8655 1 3 5 15 15 615 615 1615 6615 30 31 LUAAAA XUMAAA VVVVxx 9169 8656 1 1 9 9 69 169 1169 4169 9169 138 139 ROAAAA YUMAAA AAAAxx 6689 8657 1 1 9 9 89 689 689 1689 6689 178 179 HXAAAA ZUMAAA HHHHxx 8721 8658 1 1 1 1 21 721 721 3721 8721 42 43 LXAAAA AVMAAA OOOOxx 7508 8659 0 0 8 8 8 508 1508 2508 7508 16 17 UCAAAA BVMAAA VVVVxx 8631 8660 1 3 1 11 31 631 631 3631 8631 62 63 ZTAAAA CVMAAA AAAAxx 480 8661 0 0 0 0 80 480 480 480 480 160 161 MSAAAA DVMAAA HHHHxx 7094 8662 0 2 4 14 94 94 1094 2094 7094 188 189 WMAAAA EVMAAA OOOOxx 319 8663 1 3 9 19 19 319 319 319 319 38 39 HMAAAA FVMAAA VVVVxx 9421 8664 1 1 1 1 21 421 1421 4421 9421 42 43 JYAAAA GVMAAA AAAAxx 4352 8665 0 0 2 12 52 352 352 4352 4352 104 105 KLAAAA HVMAAA HHHHxx 5019 8666 1 3 9 19 19 19 1019 19 5019 38 39 BLAAAA IVMAAA OOOOxx 3956 8667 0 0 6 16 56 956 1956 3956 3956 112 113 EWAAAA JVMAAA VVVVxx 114 8668 0 2 4 14 14 114 114 114 114 28 29 KEAAAA KVMAAA AAAAxx 1196 8669 0 0 6 16 96 196 1196 1196 1196 192 193 AUAAAA LVMAAA HHHHxx 1407 8670 1 3 7 7 7 407 1407 1407 1407 14 15 DCAAAA MVMAAA OOOOxx 7432 8671 0 0 2 12 32 432 1432 2432 7432 64 65 WZAAAA NVMAAA VVVVxx 3141 8672 1 1 1 1 41 141 1141 3141 3141 82 83 VQAAAA OVMAAA AAAAxx 2073 8673 1 1 3 13 73 73 73 2073 2073 146 147 TBAAAA PVMAAA HHHHxx 3400 8674 0 0 0 0 0 400 1400 3400 3400 0 1 UAAAAA QVMAAA OOOOxx 505 8675 1 1 5 5 5 505 505 505 505 10 11 LTAAAA RVMAAA VVVVxx 1263 8676 1 3 3 3 63 263 1263 1263 1263 126 127 PWAAAA SVMAAA AAAAxx 190 8677 0 2 0 10 90 190 190 190 190 180 181 IHAAAA TVMAAA HHHHxx 6686 8678 0 2 6 6 86 686 686 1686 6686 172 173 EXAAAA UVMAAA OOOOxx 9821 8679 1 1 1 1 21 821 1821 4821 9821 42 43 TNAAAA VVMAAA VVVVxx 1119 8680 1 3 9 19 19 119 1119 1119 1119 38 39 BRAAAA WVMAAA AAAAxx 2955 8681 1 3 5 15 55 955 955 2955 2955 110 111 RJAAAA XVMAAA HHHHxx 224 8682 0 0 4 4 24 224 224 224 224 48 49 QIAAAA YVMAAA OOOOxx 7562 8683 0 2 2 2 62 562 1562 2562 7562 124 125 WEAAAA ZVMAAA VVVVxx 8845 8684 1 1 5 5 45 845 845 3845 8845 90 91 FCAAAA AWMAAA AAAAxx 5405 8685 1 1 5 5 5 405 1405 405 5405 10 11 XZAAAA BWMAAA HHHHxx 9192 8686 0 0 2 12 92 192 1192 4192 9192 184 185 OPAAAA CWMAAA OOOOxx 4927 8687 1 3 7 7 27 927 927 4927 4927 54 55 NHAAAA DWMAAA VVVVxx 997 8688 1 1 7 17 97 997 997 997 997 194 195 JMAAAA EWMAAA AAAAxx 989 8689 1 1 9 9 89 989 989 989 989 178 179 BMAAAA FWMAAA HHHHxx 7258 8690 0 2 8 18 58 258 1258 2258 7258 116 117 ETAAAA GWMAAA OOOOxx 6899 8691 1 3 9 19 99 899 899 1899 6899 198 199 JFAAAA HWMAAA VVVVxx 1770 8692 0 2 0 10 70 770 1770 1770 1770 140 141 CQAAAA IWMAAA AAAAxx 4423 8693 1 3 3 3 23 423 423 4423 4423 46 47 DOAAAA JWMAAA HHHHxx 5671 8694 1 3 1 11 71 671 1671 671 5671 142 143 DKAAAA KWMAAA OOOOxx 8393 8695 1 1 3 13 93 393 393 3393 8393 186 187 VKAAAA LWMAAA VVVVxx 4355 8696 1 3 5 15 55 355 355 4355 4355 110 111 NLAAAA MWMAAA AAAAxx 3919 8697 1 3 9 19 19 919 1919 3919 3919 38 39 TUAAAA NWMAAA HHHHxx 338 8698 0 2 8 18 38 338 338 338 338 76 77 ANAAAA OWMAAA OOOOxx 5790 8699 0 2 0 10 90 790 1790 790 5790 180 181 SOAAAA PWMAAA VVVVxx 1452 8700 0 0 2 12 52 452 1452 1452 1452 104 105 WDAAAA QWMAAA AAAAxx 939 8701 1 3 9 19 39 939 939 939 939 78 79 DKAAAA RWMAAA HHHHxx 8913 8702 1 1 3 13 13 913 913 3913 8913 26 27 VEAAAA SWMAAA OOOOxx 7157 8703 1 1 7 17 57 157 1157 2157 7157 114 115 HPAAAA TWMAAA VVVVxx 7240 8704 0 0 0 0 40 240 1240 2240 7240 80 81 MSAAAA UWMAAA AAAAxx 3492 8705 0 0 2 12 92 492 1492 3492 3492 184 185 IEAAAA VWMAAA HHHHxx 3464 8706 0 0 4 4 64 464 1464 3464 3464 128 129 GDAAAA WWMAAA OOOOxx 388 8707 0 0 8 8 88 388 388 388 388 176 177 YOAAAA XWMAAA VVVVxx 4135 8708 1 3 5 15 35 135 135 4135 4135 70 71 BDAAAA YWMAAA AAAAxx 1194 8709 0 2 4 14 94 194 1194 1194 1194 188 189 YTAAAA ZWMAAA HHHHxx 5476 8710 0 0 6 16 76 476 1476 476 5476 152 153 QCAAAA AXMAAA OOOOxx 9844 8711 0 0 4 4 44 844 1844 4844 9844 88 89 QOAAAA BXMAAA VVVVxx 9364 8712 0 0 4 4 64 364 1364 4364 9364 128 129 EWAAAA CXMAAA AAAAxx 5238 8713 0 2 8 18 38 238 1238 238 5238 76 77 MTAAAA DXMAAA HHHHxx 3712 8714 0 0 2 12 12 712 1712 3712 3712 24 25 UMAAAA EXMAAA OOOOxx 6189 8715 1 1 9 9 89 189 189 1189 6189 178 179 BEAAAA FXMAAA VVVVxx 5257 8716 1 1 7 17 57 257 1257 257 5257 114 115 FUAAAA GXMAAA AAAAxx 81 8717 1 1 1 1 81 81 81 81 81 162 163 DDAAAA HXMAAA HHHHxx 3289 8718 1 1 9 9 89 289 1289 3289 3289 178 179 NWAAAA IXMAAA OOOOxx 1177 8719 1 1 7 17 77 177 1177 1177 1177 154 155 HTAAAA JXMAAA VVVVxx 5038 8720 0 2 8 18 38 38 1038 38 5038 76 77 ULAAAA KXMAAA AAAAxx 325 8721 1 1 5 5 25 325 325 325 325 50 51 NMAAAA LXMAAA HHHHxx 7221 8722 1 1 1 1 21 221 1221 2221 7221 42 43 TRAAAA MXMAAA OOOOxx 7123 8723 1 3 3 3 23 123 1123 2123 7123 46 47 ZNAAAA NXMAAA VVVVxx 6364 8724 0 0 4 4 64 364 364 1364 6364 128 129 UKAAAA OXMAAA AAAAxx 4468 8725 0 0 8 8 68 468 468 4468 4468 136 137 WPAAAA PXMAAA HHHHxx 9185 8726 1 1 5 5 85 185 1185 4185 9185 170 171 HPAAAA QXMAAA OOOOxx 4158 8727 0 2 8 18 58 158 158 4158 4158 116 117 YDAAAA RXMAAA VVVVxx 9439 8728 1 3 9 19 39 439 1439 4439 9439 78 79 BZAAAA SXMAAA AAAAxx 7759 8729 1 3 9 19 59 759 1759 2759 7759 118 119 LMAAAA TXMAAA HHHHxx 3325 8730 1 1 5 5 25 325 1325 3325 3325 50 51 XXAAAA UXMAAA OOOOxx 7991 8731 1 3 1 11 91 991 1991 2991 7991 182 183 JVAAAA VXMAAA VVVVxx 1650 8732 0 2 0 10 50 650 1650 1650 1650 100 101 MLAAAA WXMAAA AAAAxx 8395 8733 1 3 5 15 95 395 395 3395 8395 190 191 XKAAAA XXMAAA HHHHxx 286 8734 0 2 6 6 86 286 286 286 286 172 173 ALAAAA YXMAAA OOOOxx 1507 8735 1 3 7 7 7 507 1507 1507 1507 14 15 ZFAAAA ZXMAAA VVVVxx 4122 8736 0 2 2 2 22 122 122 4122 4122 44 45 OCAAAA AYMAAA AAAAxx 2625 8737 1 1 5 5 25 625 625 2625 2625 50 51 ZWAAAA BYMAAA HHHHxx 1140 8738 0 0 0 0 40 140 1140 1140 1140 80 81 WRAAAA CYMAAA OOOOxx 5262 8739 0 2 2 2 62 262 1262 262 5262 124 125 KUAAAA DYMAAA VVVVxx 4919 8740 1 3 9 19 19 919 919 4919 4919 38 39 FHAAAA EYMAAA AAAAxx 7266 8741 0 2 6 6 66 266 1266 2266 7266 132 133 MTAAAA FYMAAA HHHHxx 630 8742 0 2 0 10 30 630 630 630 630 60 61 GYAAAA GYMAAA OOOOxx 2129 8743 1 1 9 9 29 129 129 2129 2129 58 59 XDAAAA HYMAAA VVVVxx 9552 8744 0 0 2 12 52 552 1552 4552 9552 104 105 KDAAAA IYMAAA AAAAxx 3018 8745 0 2 8 18 18 18 1018 3018 3018 36 37 CMAAAA JYMAAA HHHHxx 7145 8746 1 1 5 5 45 145 1145 2145 7145 90 91 VOAAAA KYMAAA OOOOxx 1633 8747 1 1 3 13 33 633 1633 1633 1633 66 67 VKAAAA LYMAAA VVVVxx 7957 8748 1 1 7 17 57 957 1957 2957 7957 114 115 BUAAAA MYMAAA AAAAxx 774 8749 0 2 4 14 74 774 774 774 774 148 149 UDAAAA NYMAAA HHHHxx 9371 8750 1 3 1 11 71 371 1371 4371 9371 142 143 LWAAAA OYMAAA OOOOxx 6007 8751 1 3 7 7 7 7 7 1007 6007 14 15 BXAAAA PYMAAA VVVVxx 5277 8752 1 1 7 17 77 277 1277 277 5277 154 155 ZUAAAA QYMAAA AAAAxx 9426 8753 0 2 6 6 26 426 1426 4426 9426 52 53 OYAAAA RYMAAA HHHHxx 9190 8754 0 2 0 10 90 190 1190 4190 9190 180 181 MPAAAA SYMAAA OOOOxx 8996 8755 0 0 6 16 96 996 996 3996 8996 192 193 AIAAAA TYMAAA VVVVxx 3409 8756 1 1 9 9 9 409 1409 3409 3409 18 19 DBAAAA UYMAAA AAAAxx 7212 8757 0 0 2 12 12 212 1212 2212 7212 24 25 KRAAAA VYMAAA HHHHxx 416 8758 0 0 6 16 16 416 416 416 416 32 33 AQAAAA WYMAAA OOOOxx 7211 8759 1 3 1 11 11 211 1211 2211 7211 22 23 JRAAAA XYMAAA VVVVxx 7454 8760 0 2 4 14 54 454 1454 2454 7454 108 109 SAAAAA YYMAAA AAAAxx 8417 8761 1 1 7 17 17 417 417 3417 8417 34 35 TLAAAA ZYMAAA HHHHxx 5562 8762 0 2 2 2 62 562 1562 562 5562 124 125 YFAAAA AZMAAA OOOOxx 4996 8763 0 0 6 16 96 996 996 4996 4996 192 193 EKAAAA BZMAAA VVVVxx 5718 8764 0 2 8 18 18 718 1718 718 5718 36 37 YLAAAA CZMAAA AAAAxx 7838 8765 0 2 8 18 38 838 1838 2838 7838 76 77 MPAAAA DZMAAA HHHHxx 7715 8766 1 3 5 15 15 715 1715 2715 7715 30 31 TKAAAA EZMAAA OOOOxx 2780 8767 0 0 0 0 80 780 780 2780 2780 160 161 YCAAAA FZMAAA VVVVxx 1013 8768 1 1 3 13 13 13 1013 1013 1013 26 27 ZMAAAA GZMAAA AAAAxx 8465 8769 1 1 5 5 65 465 465 3465 8465 130 131 PNAAAA HZMAAA HHHHxx 7976 8770 0 0 6 16 76 976 1976 2976 7976 152 153 UUAAAA IZMAAA OOOOxx 7150 8771 0 2 0 10 50 150 1150 2150 7150 100 101 APAAAA JZMAAA VVVVxx 6471 8772 1 3 1 11 71 471 471 1471 6471 142 143 XOAAAA KZMAAA AAAAxx 1927 8773 1 3 7 7 27 927 1927 1927 1927 54 55 DWAAAA LZMAAA HHHHxx 227 8774 1 3 7 7 27 227 227 227 227 54 55 TIAAAA MZMAAA OOOOxx 6462 8775 0 2 2 2 62 462 462 1462 6462 124 125 OOAAAA NZMAAA VVVVxx 5227 8776 1 3 7 7 27 227 1227 227 5227 54 55 BTAAAA OZMAAA AAAAxx 1074 8777 0 2 4 14 74 74 1074 1074 1074 148 149 IPAAAA PZMAAA HHHHxx 9448 8778 0 0 8 8 48 448 1448 4448 9448 96 97 KZAAAA QZMAAA OOOOxx 4459 8779 1 3 9 19 59 459 459 4459 4459 118 119 NPAAAA RZMAAA VVVVxx 2478 8780 0 2 8 18 78 478 478 2478 2478 156 157 IRAAAA SZMAAA AAAAxx 5005 8781 1 1 5 5 5 5 1005 5 5005 10 11 NKAAAA TZMAAA HHHHxx 2418 8782 0 2 8 18 18 418 418 2418 2418 36 37 APAAAA UZMAAA OOOOxx 6991 8783 1 3 1 11 91 991 991 1991 6991 182 183 XIAAAA VZMAAA VVVVxx 4729 8784 1 1 9 9 29 729 729 4729 4729 58 59 XZAAAA WZMAAA AAAAxx 3548 8785 0 0 8 8 48 548 1548 3548 3548 96 97 MGAAAA XZMAAA HHHHxx 9616 8786 0 0 6 16 16 616 1616 4616 9616 32 33 WFAAAA YZMAAA OOOOxx 2901 8787 1 1 1 1 1 901 901 2901 2901 2 3 PHAAAA ZZMAAA VVVVxx 10 8788 0 2 0 10 10 10 10 10 10 20 21 KAAAAA AANAAA AAAAxx 2637 8789 1 1 7 17 37 637 637 2637 2637 74 75 LXAAAA BANAAA HHHHxx 6747 8790 1 3 7 7 47 747 747 1747 6747 94 95 NZAAAA CANAAA OOOOxx 797 8791 1 1 7 17 97 797 797 797 797 194 195 REAAAA DANAAA VVVVxx 7609 8792 1 1 9 9 9 609 1609 2609 7609 18 19 RGAAAA EANAAA AAAAxx 8290 8793 0 2 0 10 90 290 290 3290 8290 180 181 WGAAAA FANAAA HHHHxx 8765 8794 1 1 5 5 65 765 765 3765 8765 130 131 DZAAAA GANAAA OOOOxx 8053 8795 1 1 3 13 53 53 53 3053 8053 106 107 TXAAAA HANAAA VVVVxx 5602 8796 0 2 2 2 2 602 1602 602 5602 4 5 MHAAAA IANAAA AAAAxx 3672 8797 0 0 2 12 72 672 1672 3672 3672 144 145 GLAAAA JANAAA HHHHxx 7513 8798 1 1 3 13 13 513 1513 2513 7513 26 27 ZCAAAA KANAAA OOOOxx 3462 8799 0 2 2 2 62 462 1462 3462 3462 124 125 EDAAAA LANAAA VVVVxx 4457 8800 1 1 7 17 57 457 457 4457 4457 114 115 LPAAAA MANAAA AAAAxx 6547 8801 1 3 7 7 47 547 547 1547 6547 94 95 VRAAAA NANAAA HHHHxx 7417 8802 1 1 7 17 17 417 1417 2417 7417 34 35 HZAAAA OANAAA OOOOxx 8641 8803 1 1 1 1 41 641 641 3641 8641 82 83 JUAAAA PANAAA VVVVxx 149 8804 1 1 9 9 49 149 149 149 149 98 99 TFAAAA QANAAA AAAAxx 5041 8805 1 1 1 1 41 41 1041 41 5041 82 83 XLAAAA RANAAA HHHHxx 9232 8806 0 0 2 12 32 232 1232 4232 9232 64 65 CRAAAA SANAAA OOOOxx 3603 8807 1 3 3 3 3 603 1603 3603 3603 6 7 PIAAAA TANAAA VVVVxx 2792 8808 0 0 2 12 92 792 792 2792 2792 184 185 KDAAAA UANAAA AAAAxx 6620 8809 0 0 0 0 20 620 620 1620 6620 40 41 QUAAAA VANAAA HHHHxx 4000 8810 0 0 0 0 0 0 0 4000 4000 0 1 WXAAAA WANAAA OOOOxx 659 8811 1 3 9 19 59 659 659 659 659 118 119 JZAAAA XANAAA VVVVxx 8174 8812 0 2 4 14 74 174 174 3174 8174 148 149 KCAAAA YANAAA AAAAxx 4599 8813 1 3 9 19 99 599 599 4599 4599 198 199 XUAAAA ZANAAA HHHHxx 7851 8814 1 3 1 11 51 851 1851 2851 7851 102 103 ZPAAAA ABNAAA OOOOxx 6284 8815 0 0 4 4 84 284 284 1284 6284 168 169 SHAAAA BBNAAA VVVVxx 7116 8816 0 0 6 16 16 116 1116 2116 7116 32 33 SNAAAA CBNAAA AAAAxx 5595 8817 1 3 5 15 95 595 1595 595 5595 190 191 FHAAAA DBNAAA HHHHxx 2903 8818 1 3 3 3 3 903 903 2903 2903 6 7 RHAAAA EBNAAA OOOOxx 5948 8819 0 0 8 8 48 948 1948 948 5948 96 97 UUAAAA FBNAAA VVVVxx 225 8820 1 1 5 5 25 225 225 225 225 50 51 RIAAAA GBNAAA AAAAxx 524 8821 0 0 4 4 24 524 524 524 524 48 49 EUAAAA HBNAAA HHHHxx 7639 8822 1 3 9 19 39 639 1639 2639 7639 78 79 VHAAAA IBNAAA OOOOxx 7297 8823 1 1 7 17 97 297 1297 2297 7297 194 195 RUAAAA JBNAAA VVVVxx 2606 8824 0 2 6 6 6 606 606 2606 2606 12 13 GWAAAA KBNAAA AAAAxx 4771 8825 1 3 1 11 71 771 771 4771 4771 142 143 NBAAAA LBNAAA HHHHxx 8162 8826 0 2 2 2 62 162 162 3162 8162 124 125 YBAAAA MBNAAA OOOOxx 8999 8827 1 3 9 19 99 999 999 3999 8999 198 199 DIAAAA NBNAAA VVVVxx 2309 8828 1 1 9 9 9 309 309 2309 2309 18 19 VKAAAA OBNAAA AAAAxx 3594 8829 0 2 4 14 94 594 1594 3594 3594 188 189 GIAAAA PBNAAA HHHHxx 6092 8830 0 0 2 12 92 92 92 1092 6092 184 185 IAAAAA QBNAAA OOOOxx 7467 8831 1 3 7 7 67 467 1467 2467 7467 134 135 FBAAAA RBNAAA VVVVxx 6986 8832 0 2 6 6 86 986 986 1986 6986 172 173 SIAAAA SBNAAA AAAAxx 9898 8833 0 2 8 18 98 898 1898 4898 9898 196 197 SQAAAA TBNAAA HHHHxx 9578 8834 0 2 8 18 78 578 1578 4578 9578 156 157 KEAAAA UBNAAA OOOOxx 156 8835 0 0 6 16 56 156 156 156 156 112 113 AGAAAA VBNAAA VVVVxx 5810 8836 0 2 0 10 10 810 1810 810 5810 20 21 MPAAAA WBNAAA AAAAxx 790 8837 0 2 0 10 90 790 790 790 790 180 181 KEAAAA XBNAAA HHHHxx 6840 8838 0 0 0 0 40 840 840 1840 6840 80 81 CDAAAA YBNAAA OOOOxx 6725 8839 1 1 5 5 25 725 725 1725 6725 50 51 RYAAAA ZBNAAA VVVVxx 5528 8840 0 0 8 8 28 528 1528 528 5528 56 57 QEAAAA ACNAAA AAAAxx 4120 8841 0 0 0 0 20 120 120 4120 4120 40 41 MCAAAA BCNAAA HHHHxx 6694 8842 0 2 4 14 94 694 694 1694 6694 188 189 MXAAAA CCNAAA OOOOxx 3552 8843 0 0 2 12 52 552 1552 3552 3552 104 105 QGAAAA DCNAAA VVVVxx 1478 8844 0 2 8 18 78 478 1478 1478 1478 156 157 WEAAAA ECNAAA AAAAxx 8084 8845 0 0 4 4 84 84 84 3084 8084 168 169 YYAAAA FCNAAA HHHHxx 7578 8846 0 2 8 18 78 578 1578 2578 7578 156 157 MFAAAA GCNAAA OOOOxx 6314 8847 0 2 4 14 14 314 314 1314 6314 28 29 WIAAAA HCNAAA VVVVxx 6123 8848 1 3 3 3 23 123 123 1123 6123 46 47 NBAAAA ICNAAA AAAAxx 9443 8849 1 3 3 3 43 443 1443 4443 9443 86 87 FZAAAA JCNAAA HHHHxx 9628 8850 0 0 8 8 28 628 1628 4628 9628 56 57 IGAAAA KCNAAA OOOOxx 8508 8851 0 0 8 8 8 508 508 3508 8508 16 17 GPAAAA LCNAAA VVVVxx 5552 8852 0 0 2 12 52 552 1552 552 5552 104 105 OFAAAA MCNAAA AAAAxx 5327 8853 1 3 7 7 27 327 1327 327 5327 54 55 XWAAAA NCNAAA HHHHxx 7771 8854 1 3 1 11 71 771 1771 2771 7771 142 143 XMAAAA OCNAAA OOOOxx 8932 8855 0 0 2 12 32 932 932 3932 8932 64 65 OFAAAA PCNAAA VVVVxx 3526 8856 0 2 6 6 26 526 1526 3526 3526 52 53 QFAAAA QCNAAA AAAAxx 4340 8857 0 0 0 0 40 340 340 4340 4340 80 81 YKAAAA RCNAAA HHHHxx 9419 8858 1 3 9 19 19 419 1419 4419 9419 38 39 HYAAAA SCNAAA OOOOxx 8421 8859 1 1 1 1 21 421 421 3421 8421 42 43 XLAAAA TCNAAA VVVVxx 7431 8860 1 3 1 11 31 431 1431 2431 7431 62 63 VZAAAA UCNAAA AAAAxx 172 8861 0 0 2 12 72 172 172 172 172 144 145 QGAAAA VCNAAA HHHHxx 3279 8862 1 3 9 19 79 279 1279 3279 3279 158 159 DWAAAA WCNAAA OOOOxx 1508 8863 0 0 8 8 8 508 1508 1508 1508 16 17 AGAAAA XCNAAA VVVVxx 7091 8864 1 3 1 11 91 91 1091 2091 7091 182 183 TMAAAA YCNAAA AAAAxx 1419 8865 1 3 9 19 19 419 1419 1419 1419 38 39 PCAAAA ZCNAAA HHHHxx 3032 8866 0 0 2 12 32 32 1032 3032 3032 64 65 QMAAAA ADNAAA OOOOxx 8683 8867 1 3 3 3 83 683 683 3683 8683 166 167 ZVAAAA BDNAAA VVVVxx 4763 8868 1 3 3 3 63 763 763 4763 4763 126 127 FBAAAA CDNAAA AAAAxx 4424 8869 0 0 4 4 24 424 424 4424 4424 48 49 EOAAAA DDNAAA HHHHxx 8640 8870 0 0 0 0 40 640 640 3640 8640 80 81 IUAAAA EDNAAA OOOOxx 7187 8871 1 3 7 7 87 187 1187 2187 7187 174 175 LQAAAA FDNAAA VVVVxx 6247 8872 1 3 7 7 47 247 247 1247 6247 94 95 HGAAAA GDNAAA AAAAxx 7340 8873 0 0 0 0 40 340 1340 2340 7340 80 81 IWAAAA HDNAAA HHHHxx 182 8874 0 2 2 2 82 182 182 182 182 164 165 AHAAAA IDNAAA OOOOxx 2948 8875 0 0 8 8 48 948 948 2948 2948 96 97 KJAAAA JDNAAA VVVVxx 9462 8876 0 2 2 2 62 462 1462 4462 9462 124 125 YZAAAA KDNAAA AAAAxx 5997 8877 1 1 7 17 97 997 1997 997 5997 194 195 RWAAAA LDNAAA HHHHxx 5608 8878 0 0 8 8 8 608 1608 608 5608 16 17 SHAAAA MDNAAA OOOOxx 1472 8879 0 0 2 12 72 472 1472 1472 1472 144 145 QEAAAA NDNAAA VVVVxx 277 8880 1 1 7 17 77 277 277 277 277 154 155 RKAAAA ODNAAA AAAAxx 4807 8881 1 3 7 7 7 807 807 4807 4807 14 15 XCAAAA PDNAAA HHHHxx 4969 8882 1 1 9 9 69 969 969 4969 4969 138 139 DJAAAA QDNAAA OOOOxx 5611 8883 1 3 1 11 11 611 1611 611 5611 22 23 VHAAAA RDNAAA VVVVxx 372 8884 0 0 2 12 72 372 372 372 372 144 145 IOAAAA SDNAAA AAAAxx 6666 8885 0 2 6 6 66 666 666 1666 6666 132 133 KWAAAA TDNAAA HHHHxx 476 8886 0 0 6 16 76 476 476 476 476 152 153 ISAAAA UDNAAA OOOOxx 5225 8887 1 1 5 5 25 225 1225 225 5225 50 51 ZSAAAA VDNAAA VVVVxx 5143 8888 1 3 3 3 43 143 1143 143 5143 86 87 VPAAAA WDNAAA AAAAxx 1853 8889 1 1 3 13 53 853 1853 1853 1853 106 107 HTAAAA XDNAAA HHHHxx 675 8890 1 3 5 15 75 675 675 675 675 150 151 ZZAAAA YDNAAA OOOOxx 5643 8891 1 3 3 3 43 643 1643 643 5643 86 87 BJAAAA ZDNAAA VVVVxx 5317 8892 1 1 7 17 17 317 1317 317 5317 34 35 NWAAAA AENAAA AAAAxx 8102 8893 0 2 2 2 2 102 102 3102 8102 4 5 QZAAAA BENAAA HHHHxx 978 8894 0 2 8 18 78 978 978 978 978 156 157 QLAAAA CENAAA OOOOxx 4620 8895 0 0 0 0 20 620 620 4620 4620 40 41 SVAAAA DENAAA VVVVxx 151 8896 1 3 1 11 51 151 151 151 151 102 103 VFAAAA EENAAA AAAAxx 972 8897 0 0 2 12 72 972 972 972 972 144 145 KLAAAA FENAAA HHHHxx 6820 8898 0 0 0 0 20 820 820 1820 6820 40 41 ICAAAA GENAAA OOOOxx 7387 8899 1 3 7 7 87 387 1387 2387 7387 174 175 DYAAAA HENAAA VVVVxx 9634 8900 0 2 4 14 34 634 1634 4634 9634 68 69 OGAAAA IENAAA AAAAxx 6308 8901 0 0 8 8 8 308 308 1308 6308 16 17 QIAAAA JENAAA HHHHxx 8323 8902 1 3 3 3 23 323 323 3323 8323 46 47 DIAAAA KENAAA OOOOxx 6672 8903 0 0 2 12 72 672 672 1672 6672 144 145 QWAAAA LENAAA VVVVxx 8283 8904 1 3 3 3 83 283 283 3283 8283 166 167 PGAAAA MENAAA AAAAxx 7996 8905 0 0 6 16 96 996 1996 2996 7996 192 193 OVAAAA NENAAA HHHHxx 6488 8906 0 0 8 8 88 488 488 1488 6488 176 177 OPAAAA OENAAA OOOOxx 2365 8907 1 1 5 5 65 365 365 2365 2365 130 131 ZMAAAA PENAAA VVVVxx 9746 8908 0 2 6 6 46 746 1746 4746 9746 92 93 WKAAAA QENAAA AAAAxx 8605 8909 1 1 5 5 5 605 605 3605 8605 10 11 ZSAAAA RENAAA HHHHxx 3342 8910 0 2 2 2 42 342 1342 3342 3342 84 85 OYAAAA SENAAA OOOOxx 8429 8911 1 1 9 9 29 429 429 3429 8429 58 59 FMAAAA TENAAA VVVVxx 1162 8912 0 2 2 2 62 162 1162 1162 1162 124 125 SSAAAA UENAAA AAAAxx 531 8913 1 3 1 11 31 531 531 531 531 62 63 LUAAAA VENAAA HHHHxx 8408 8914 0 0 8 8 8 408 408 3408 8408 16 17 KLAAAA WENAAA OOOOxx 8862 8915 0 2 2 2 62 862 862 3862 8862 124 125 WCAAAA XENAAA VVVVxx 5843 8916 1 3 3 3 43 843 1843 843 5843 86 87 TQAAAA YENAAA AAAAxx 8704 8917 0 0 4 4 4 704 704 3704 8704 8 9 UWAAAA ZENAAA HHHHxx 7070 8918 0 2 0 10 70 70 1070 2070 7070 140 141 YLAAAA AFNAAA OOOOxx 9119 8919 1 3 9 19 19 119 1119 4119 9119 38 39 TMAAAA BFNAAA VVVVxx 8344 8920 0 0 4 4 44 344 344 3344 8344 88 89 YIAAAA CFNAAA AAAAxx 8979 8921 1 3 9 19 79 979 979 3979 8979 158 159 JHAAAA DFNAAA HHHHxx 2971 8922 1 3 1 11 71 971 971 2971 2971 142 143 HKAAAA EFNAAA OOOOxx 7700 8923 0 0 0 0 0 700 1700 2700 7700 0 1 EKAAAA FFNAAA VVVVxx 8280 8924 0 0 0 0 80 280 280 3280 8280 160 161 MGAAAA GFNAAA AAAAxx 9096 8925 0 0 6 16 96 96 1096 4096 9096 192 193 WLAAAA HFNAAA HHHHxx 99 8926 1 3 9 19 99 99 99 99 99 198 199 VDAAAA IFNAAA OOOOxx 6696 8927 0 0 6 16 96 696 696 1696 6696 192 193 OXAAAA JFNAAA VVVVxx 9490 8928 0 2 0 10 90 490 1490 4490 9490 180 181 ABAAAA KFNAAA AAAAxx 9073 8929 1 1 3 13 73 73 1073 4073 9073 146 147 ZKAAAA LFNAAA HHHHxx 1861 8930 1 1 1 1 61 861 1861 1861 1861 122 123 PTAAAA MFNAAA OOOOxx 4413 8931 1 1 3 13 13 413 413 4413 4413 26 27 TNAAAA NFNAAA VVVVxx 6002 8932 0 2 2 2 2 2 2 1002 6002 4 5 WWAAAA OFNAAA AAAAxx 439 8933 1 3 9 19 39 439 439 439 439 78 79 XQAAAA PFNAAA HHHHxx 5449 8934 1 1 9 9 49 449 1449 449 5449 98 99 PBAAAA QFNAAA OOOOxx 9737 8935 1 1 7 17 37 737 1737 4737 9737 74 75 NKAAAA RFNAAA VVVVxx 1898 8936 0 2 8 18 98 898 1898 1898 1898 196 197 AVAAAA SFNAAA AAAAxx 4189 8937 1 1 9 9 89 189 189 4189 4189 178 179 DFAAAA TFNAAA HHHHxx 1408 8938 0 0 8 8 8 408 1408 1408 1408 16 17 ECAAAA UFNAAA OOOOxx 394 8939 0 2 4 14 94 394 394 394 394 188 189 EPAAAA VFNAAA VVVVxx 1935 8940 1 3 5 15 35 935 1935 1935 1935 70 71 LWAAAA WFNAAA AAAAxx 3965 8941 1 1 5 5 65 965 1965 3965 3965 130 131 NWAAAA XFNAAA HHHHxx 6821 8942 1 1 1 1 21 821 821 1821 6821 42 43 JCAAAA YFNAAA OOOOxx 349 8943 1 1 9 9 49 349 349 349 349 98 99 LNAAAA ZFNAAA VVVVxx 8428 8944 0 0 8 8 28 428 428 3428 8428 56 57 EMAAAA AGNAAA AAAAxx 8200 8945 0 0 0 0 0 200 200 3200 8200 0 1 KDAAAA BGNAAA HHHHxx 1737 8946 1 1 7 17 37 737 1737 1737 1737 74 75 VOAAAA CGNAAA OOOOxx 6516 8947 0 0 6 16 16 516 516 1516 6516 32 33 QQAAAA DGNAAA VVVVxx 5441 8948 1 1 1 1 41 441 1441 441 5441 82 83 HBAAAA EGNAAA AAAAxx 5999 8949 1 3 9 19 99 999 1999 999 5999 198 199 TWAAAA FGNAAA HHHHxx 1539 8950 1 3 9 19 39 539 1539 1539 1539 78 79 FHAAAA GGNAAA OOOOxx 9067 8951 1 3 7 7 67 67 1067 4067 9067 134 135 TKAAAA HGNAAA VVVVxx 4061 8952 1 1 1 1 61 61 61 4061 4061 122 123 FAAAAA IGNAAA AAAAxx 1642 8953 0 2 2 2 42 642 1642 1642 1642 84 85 ELAAAA JGNAAA HHHHxx 4657 8954 1 1 7 17 57 657 657 4657 4657 114 115 DXAAAA KGNAAA OOOOxx 9934 8955 0 2 4 14 34 934 1934 4934 9934 68 69 CSAAAA LGNAAA VVVVxx 6385 8956 1 1 5 5 85 385 385 1385 6385 170 171 PLAAAA MGNAAA AAAAxx 6775 8957 1 3 5 15 75 775 775 1775 6775 150 151 PAAAAA NGNAAA HHHHxx 3873 8958 1 1 3 13 73 873 1873 3873 3873 146 147 ZSAAAA OGNAAA OOOOxx 3862 8959 0 2 2 2 62 862 1862 3862 3862 124 125 OSAAAA PGNAAA VVVVxx 1224 8960 0 0 4 4 24 224 1224 1224 1224 48 49 CVAAAA QGNAAA AAAAxx 4483 8961 1 3 3 3 83 483 483 4483 4483 166 167 LQAAAA RGNAAA HHHHxx 3685 8962 1 1 5 5 85 685 1685 3685 3685 170 171 TLAAAA SGNAAA OOOOxx 6082 8963 0 2 2 2 82 82 82 1082 6082 164 165 YZAAAA TGNAAA VVVVxx 7798 8964 0 2 8 18 98 798 1798 2798 7798 196 197 YNAAAA UGNAAA AAAAxx 9039 8965 1 3 9 19 39 39 1039 4039 9039 78 79 RJAAAA VGNAAA HHHHxx 985 8966 1 1 5 5 85 985 985 985 985 170 171 XLAAAA WGNAAA OOOOxx 5389 8967 1 1 9 9 89 389 1389 389 5389 178 179 HZAAAA XGNAAA VVVVxx 1716 8968 0 0 6 16 16 716 1716 1716 1716 32 33 AOAAAA YGNAAA AAAAxx 4209 8969 1 1 9 9 9 209 209 4209 4209 18 19 XFAAAA ZGNAAA HHHHxx 746 8970 0 2 6 6 46 746 746 746 746 92 93 SCAAAA AHNAAA OOOOxx 6295 8971 1 3 5 15 95 295 295 1295 6295 190 191 DIAAAA BHNAAA VVVVxx 9754 8972 0 2 4 14 54 754 1754 4754 9754 108 109 ELAAAA CHNAAA AAAAxx 2336 8973 0 0 6 16 36 336 336 2336 2336 72 73 WLAAAA DHNAAA HHHHxx 3701 8974 1 1 1 1 1 701 1701 3701 3701 2 3 JMAAAA EHNAAA OOOOxx 3551 8975 1 3 1 11 51 551 1551 3551 3551 102 103 PGAAAA FHNAAA VVVVxx 8516 8976 0 0 6 16 16 516 516 3516 8516 32 33 OPAAAA GHNAAA AAAAxx 9290 8977 0 2 0 10 90 290 1290 4290 9290 180 181 ITAAAA HHNAAA HHHHxx 5686 8978 0 2 6 6 86 686 1686 686 5686 172 173 SKAAAA IHNAAA OOOOxx 2893 8979 1 1 3 13 93 893 893 2893 2893 186 187 HHAAAA JHNAAA VVVVxx 6279 8980 1 3 9 19 79 279 279 1279 6279 158 159 NHAAAA KHNAAA AAAAxx 2278 8981 0 2 8 18 78 278 278 2278 2278 156 157 QJAAAA LHNAAA HHHHxx 1618 8982 0 2 8 18 18 618 1618 1618 1618 36 37 GKAAAA MHNAAA OOOOxx 3450 8983 0 2 0 10 50 450 1450 3450 3450 100 101 SCAAAA NHNAAA VVVVxx 8857 8984 1 1 7 17 57 857 857 3857 8857 114 115 RCAAAA OHNAAA AAAAxx 1005 8985 1 1 5 5 5 5 1005 1005 1005 10 11 RMAAAA PHNAAA HHHHxx 4727 8986 1 3 7 7 27 727 727 4727 4727 54 55 VZAAAA QHNAAA OOOOxx 7617 8987 1 1 7 17 17 617 1617 2617 7617 34 35 ZGAAAA RHNAAA VVVVxx 2021 8988 1 1 1 1 21 21 21 2021 2021 42 43 TZAAAA SHNAAA AAAAxx 9124 8989 0 0 4 4 24 124 1124 4124 9124 48 49 YMAAAA THNAAA HHHHxx 3175 8990 1 3 5 15 75 175 1175 3175 3175 150 151 DSAAAA UHNAAA OOOOxx 2949 8991 1 1 9 9 49 949 949 2949 2949 98 99 LJAAAA VHNAAA VVVVxx 2424 8992 0 0 4 4 24 424 424 2424 2424 48 49 GPAAAA WHNAAA AAAAxx 4791 8993 1 3 1 11 91 791 791 4791 4791 182 183 HCAAAA XHNAAA HHHHxx 7500 8994 0 0 0 0 0 500 1500 2500 7500 0 1 MCAAAA YHNAAA OOOOxx 4893 8995 1 1 3 13 93 893 893 4893 4893 186 187 FGAAAA ZHNAAA VVVVxx 121 8996 1 1 1 1 21 121 121 121 121 42 43 REAAAA AINAAA AAAAxx 1965 8997 1 1 5 5 65 965 1965 1965 1965 130 131 PXAAAA BINAAA HHHHxx 2972 8998 0 0 2 12 72 972 972 2972 2972 144 145 IKAAAA CINAAA OOOOxx 662 8999 0 2 2 2 62 662 662 662 662 124 125 MZAAAA DINAAA VVVVxx 7074 9000 0 2 4 14 74 74 1074 2074 7074 148 149 CMAAAA EINAAA AAAAxx 981 9001 1 1 1 1 81 981 981 981 981 162 163 TLAAAA FINAAA HHHHxx 3520 9002 0 0 0 0 20 520 1520 3520 3520 40 41 KFAAAA GINAAA OOOOxx 6540 9003 0 0 0 0 40 540 540 1540 6540 80 81 ORAAAA HINAAA VVVVxx 6648 9004 0 0 8 8 48 648 648 1648 6648 96 97 SVAAAA IINAAA AAAAxx 7076 9005 0 0 6 16 76 76 1076 2076 7076 152 153 EMAAAA JINAAA HHHHxx 6919 9006 1 3 9 19 19 919 919 1919 6919 38 39 DGAAAA KINAAA OOOOxx 1108 9007 0 0 8 8 8 108 1108 1108 1108 16 17 QQAAAA LINAAA VVVVxx 317 9008 1 1 7 17 17 317 317 317 317 34 35 FMAAAA MINAAA AAAAxx 3483 9009 1 3 3 3 83 483 1483 3483 3483 166 167 ZDAAAA NINAAA HHHHxx 6764 9010 0 0 4 4 64 764 764 1764 6764 128 129 EAAAAA OINAAA OOOOxx 1235 9011 1 3 5 15 35 235 1235 1235 1235 70 71 NVAAAA PINAAA VVVVxx 7121 9012 1 1 1 1 21 121 1121 2121 7121 42 43 XNAAAA QINAAA AAAAxx 426 9013 0 2 6 6 26 426 426 426 426 52 53 KQAAAA RINAAA HHHHxx 6880 9014 0 0 0 0 80 880 880 1880 6880 160 161 QEAAAA SINAAA OOOOxx 5401 9015 1 1 1 1 1 401 1401 401 5401 2 3 TZAAAA TINAAA VVVVxx 7323 9016 1 3 3 3 23 323 1323 2323 7323 46 47 RVAAAA UINAAA AAAAxx 9751 9017 1 3 1 11 51 751 1751 4751 9751 102 103 BLAAAA VINAAA HHHHxx 3436 9018 0 0 6 16 36 436 1436 3436 3436 72 73 ECAAAA WINAAA OOOOxx 7319 9019 1 3 9 19 19 319 1319 2319 7319 38 39 NVAAAA XINAAA VVVVxx 7882 9020 0 2 2 2 82 882 1882 2882 7882 164 165 ERAAAA YINAAA AAAAxx 8260 9021 0 0 0 0 60 260 260 3260 8260 120 121 SFAAAA ZINAAA HHHHxx 9758 9022 0 2 8 18 58 758 1758 4758 9758 116 117 ILAAAA AJNAAA OOOOxx 4205 9023 1 1 5 5 5 205 205 4205 4205 10 11 TFAAAA BJNAAA VVVVxx 8884 9024 0 0 4 4 84 884 884 3884 8884 168 169 SDAAAA CJNAAA AAAAxx 1112 9025 0 0 2 12 12 112 1112 1112 1112 24 25 UQAAAA DJNAAA HHHHxx 2186 9026 0 2 6 6 86 186 186 2186 2186 172 173 CGAAAA EJNAAA OOOOxx 8666 9027 0 2 6 6 66 666 666 3666 8666 132 133 IVAAAA FJNAAA VVVVxx 4325 9028 1 1 5 5 25 325 325 4325 4325 50 51 JKAAAA GJNAAA AAAAxx 4912 9029 0 0 2 12 12 912 912 4912 4912 24 25 YGAAAA HJNAAA HHHHxx 6497 9030 1 1 7 17 97 497 497 1497 6497 194 195 XPAAAA IJNAAA OOOOxx 9072 9031 0 0 2 12 72 72 1072 4072 9072 144 145 YKAAAA JJNAAA VVVVxx 8899 9032 1 3 9 19 99 899 899 3899 8899 198 199 HEAAAA KJNAAA AAAAxx 5619 9033 1 3 9 19 19 619 1619 619 5619 38 39 DIAAAA LJNAAA HHHHxx 4110 9034 0 2 0 10 10 110 110 4110 4110 20 21 CCAAAA MJNAAA OOOOxx 7025 9035 1 1 5 5 25 25 1025 2025 7025 50 51 FKAAAA NJNAAA VVVVxx 5605 9036 1 1 5 5 5 605 1605 605 5605 10 11 PHAAAA OJNAAA AAAAxx 2572 9037 0 0 2 12 72 572 572 2572 2572 144 145 YUAAAA PJNAAA HHHHxx 3895 9038 1 3 5 15 95 895 1895 3895 3895 190 191 VTAAAA QJNAAA OOOOxx 9138 9039 0 2 8 18 38 138 1138 4138 9138 76 77 MNAAAA RJNAAA VVVVxx 4713 9040 1 1 3 13 13 713 713 4713 4713 26 27 HZAAAA SJNAAA AAAAxx 6079 9041 1 3 9 19 79 79 79 1079 6079 158 159 VZAAAA TJNAAA HHHHxx 8898 9042 0 2 8 18 98 898 898 3898 8898 196 197 GEAAAA UJNAAA OOOOxx 2650 9043 0 2 0 10 50 650 650 2650 2650 100 101 YXAAAA VJNAAA VVVVxx 5316 9044 0 0 6 16 16 316 1316 316 5316 32 33 MWAAAA WJNAAA AAAAxx 5133 9045 1 1 3 13 33 133 1133 133 5133 66 67 LPAAAA XJNAAA HHHHxx 2184 9046 0 0 4 4 84 184 184 2184 2184 168 169 AGAAAA YJNAAA OOOOxx 2728 9047 0 0 8 8 28 728 728 2728 2728 56 57 YAAAAA ZJNAAA VVVVxx 6737 9048 1 1 7 17 37 737 737 1737 6737 74 75 DZAAAA AKNAAA AAAAxx 1128 9049 0 0 8 8 28 128 1128 1128 1128 56 57 KRAAAA BKNAAA HHHHxx 9662 9050 0 2 2 2 62 662 1662 4662 9662 124 125 QHAAAA CKNAAA OOOOxx 9384 9051 0 0 4 4 84 384 1384 4384 9384 168 169 YWAAAA DKNAAA VVVVxx 4576 9052 0 0 6 16 76 576 576 4576 4576 152 153 AUAAAA EKNAAA AAAAxx 9613 9053 1 1 3 13 13 613 1613 4613 9613 26 27 TFAAAA FKNAAA HHHHxx 4001 9054 1 1 1 1 1 1 1 4001 4001 2 3 XXAAAA GKNAAA OOOOxx 3628 9055 0 0 8 8 28 628 1628 3628 3628 56 57 OJAAAA HKNAAA VVVVxx 6968 9056 0 0 8 8 68 968 968 1968 6968 136 137 AIAAAA IKNAAA AAAAxx 6491 9057 1 3 1 11 91 491 491 1491 6491 182 183 RPAAAA JKNAAA HHHHxx 1265 9058 1 1 5 5 65 265 1265 1265 1265 130 131 RWAAAA KKNAAA OOOOxx 6128 9059 0 0 8 8 28 128 128 1128 6128 56 57 SBAAAA LKNAAA VVVVxx 4274 9060 0 2 4 14 74 274 274 4274 4274 148 149 KIAAAA MKNAAA AAAAxx 3598 9061 0 2 8 18 98 598 1598 3598 3598 196 197 KIAAAA NKNAAA HHHHxx 7961 9062 1 1 1 1 61 961 1961 2961 7961 122 123 FUAAAA OKNAAA OOOOxx 2643 9063 1 3 3 3 43 643 643 2643 2643 86 87 RXAAAA PKNAAA VVVVxx 4547 9064 1 3 7 7 47 547 547 4547 4547 94 95 XSAAAA QKNAAA AAAAxx 3568 9065 0 0 8 8 68 568 1568 3568 3568 136 137 GHAAAA RKNAAA HHHHxx 8954 9066 0 2 4 14 54 954 954 3954 8954 108 109 KGAAAA SKNAAA OOOOxx 8802 9067 0 2 2 2 2 802 802 3802 8802 4 5 OAAAAA TKNAAA VVVVxx 7829 9068 1 1 9 9 29 829 1829 2829 7829 58 59 DPAAAA UKNAAA AAAAxx 1008 9069 0 0 8 8 8 8 1008 1008 1008 16 17 UMAAAA VKNAAA HHHHxx 3627 9070 1 3 7 7 27 627 1627 3627 3627 54 55 NJAAAA WKNAAA OOOOxx 3999 9071 1 3 9 19 99 999 1999 3999 3999 198 199 VXAAAA XKNAAA VVVVxx 7697 9072 1 1 7 17 97 697 1697 2697 7697 194 195 BKAAAA YKNAAA AAAAxx 9380 9073 0 0 0 0 80 380 1380 4380 9380 160 161 UWAAAA ZKNAAA HHHHxx 2707 9074 1 3 7 7 7 707 707 2707 2707 14 15 DAAAAA ALNAAA OOOOxx 4430 9075 0 2 0 10 30 430 430 4430 4430 60 61 KOAAAA BLNAAA VVVVxx 6440 9076 0 0 0 0 40 440 440 1440 6440 80 81 SNAAAA CLNAAA AAAAxx 9958 9077 0 2 8 18 58 958 1958 4958 9958 116 117 ATAAAA DLNAAA HHHHxx 7592 9078 0 0 2 12 92 592 1592 2592 7592 184 185 AGAAAA ELNAAA OOOOxx 7852 9079 0 0 2 12 52 852 1852 2852 7852 104 105 AQAAAA FLNAAA VVVVxx 9253 9080 1 1 3 13 53 253 1253 4253 9253 106 107 XRAAAA GLNAAA AAAAxx 5910 9081 0 2 0 10 10 910 1910 910 5910 20 21 ITAAAA HLNAAA HHHHxx 7487 9082 1 3 7 7 87 487 1487 2487 7487 174 175 ZBAAAA ILNAAA OOOOxx 6324 9083 0 0 4 4 24 324 324 1324 6324 48 49 GJAAAA JLNAAA VVVVxx 5792 9084 0 0 2 12 92 792 1792 792 5792 184 185 UOAAAA KLNAAA AAAAxx 7390 9085 0 2 0 10 90 390 1390 2390 7390 180 181 GYAAAA LLNAAA HHHHxx 8534 9086 0 2 4 14 34 534 534 3534 8534 68 69 GQAAAA MLNAAA OOOOxx 2690 9087 0 2 0 10 90 690 690 2690 2690 180 181 MZAAAA NLNAAA VVVVxx 3992 9088 0 0 2 12 92 992 1992 3992 3992 184 185 OXAAAA OLNAAA AAAAxx 6928 9089 0 0 8 8 28 928 928 1928 6928 56 57 MGAAAA PLNAAA HHHHxx 7815 9090 1 3 5 15 15 815 1815 2815 7815 30 31 POAAAA QLNAAA OOOOxx 9477 9091 1 1 7 17 77 477 1477 4477 9477 154 155 NAAAAA RLNAAA VVVVxx 497 9092 1 1 7 17 97 497 497 497 497 194 195 DTAAAA SLNAAA AAAAxx 7532 9093 0 0 2 12 32 532 1532 2532 7532 64 65 SDAAAA TLNAAA HHHHxx 9838 9094 0 2 8 18 38 838 1838 4838 9838 76 77 KOAAAA ULNAAA OOOOxx 1557 9095 1 1 7 17 57 557 1557 1557 1557 114 115 XHAAAA VLNAAA VVVVxx 2467 9096 1 3 7 7 67 467 467 2467 2467 134 135 XQAAAA WLNAAA AAAAxx 2367 9097 1 3 7 7 67 367 367 2367 2367 134 135 BNAAAA XLNAAA HHHHxx 5677 9098 1 1 7 17 77 677 1677 677 5677 154 155 JKAAAA YLNAAA OOOOxx 6193 9099 1 1 3 13 93 193 193 1193 6193 186 187 FEAAAA ZLNAAA VVVVxx 7126 9100 0 2 6 6 26 126 1126 2126 7126 52 53 COAAAA AMNAAA AAAAxx 5264 9101 0 0 4 4 64 264 1264 264 5264 128 129 MUAAAA BMNAAA HHHHxx 850 9102 0 2 0 10 50 850 850 850 850 100 101 SGAAAA CMNAAA OOOOxx 4854 9103 0 2 4 14 54 854 854 4854 4854 108 109 SEAAAA DMNAAA VVVVxx 4414 9104 0 2 4 14 14 414 414 4414 4414 28 29 UNAAAA EMNAAA AAAAxx 8971 9105 1 3 1 11 71 971 971 3971 8971 142 143 BHAAAA FMNAAA HHHHxx 9240 9106 0 0 0 0 40 240 1240 4240 9240 80 81 KRAAAA GMNAAA OOOOxx 7341 9107 1 1 1 1 41 341 1341 2341 7341 82 83 JWAAAA HMNAAA VVVVxx 3151 9108 1 3 1 11 51 151 1151 3151 3151 102 103 FRAAAA IMNAAA AAAAxx 1742 9109 0 2 2 2 42 742 1742 1742 1742 84 85 APAAAA JMNAAA HHHHxx 1347 9110 1 3 7 7 47 347 1347 1347 1347 94 95 VZAAAA KMNAAA OOOOxx 9418 9111 0 2 8 18 18 418 1418 4418 9418 36 37 GYAAAA LMNAAA VVVVxx 5452 9112 0 0 2 12 52 452 1452 452 5452 104 105 SBAAAA MMNAAA AAAAxx 8637 9113 1 1 7 17 37 637 637 3637 8637 74 75 FUAAAA NMNAAA HHHHxx 8287 9114 1 3 7 7 87 287 287 3287 8287 174 175 TGAAAA OMNAAA OOOOxx 9865 9115 1 1 5 5 65 865 1865 4865 9865 130 131 LPAAAA PMNAAA VVVVxx 1664 9116 0 0 4 4 64 664 1664 1664 1664 128 129 AMAAAA QMNAAA AAAAxx 9933 9117 1 1 3 13 33 933 1933 4933 9933 66 67 BSAAAA RMNAAA HHHHxx 3416 9118 0 0 6 16 16 416 1416 3416 3416 32 33 KBAAAA SMNAAA OOOOxx 7981 9119 1 1 1 1 81 981 1981 2981 7981 162 163 ZUAAAA TMNAAA VVVVxx 1981 9120 1 1 1 1 81 981 1981 1981 1981 162 163 FYAAAA UMNAAA AAAAxx 441 9121 1 1 1 1 41 441 441 441 441 82 83 ZQAAAA VMNAAA HHHHxx 1380 9122 0 0 0 0 80 380 1380 1380 1380 160 161 CBAAAA WMNAAA OOOOxx 7325 9123 1 1 5 5 25 325 1325 2325 7325 50 51 TVAAAA XMNAAA VVVVxx 5682 9124 0 2 2 2 82 682 1682 682 5682 164 165 OKAAAA YMNAAA AAAAxx 1024 9125 0 0 4 4 24 24 1024 1024 1024 48 49 KNAAAA ZMNAAA HHHHxx 1096 9126 0 0 6 16 96 96 1096 1096 1096 192 193 EQAAAA ANNAAA OOOOxx 4717 9127 1 1 7 17 17 717 717 4717 4717 34 35 LZAAAA BNNAAA VVVVxx 7948 9128 0 0 8 8 48 948 1948 2948 7948 96 97 STAAAA CNNAAA AAAAxx 4074 9129 0 2 4 14 74 74 74 4074 4074 148 149 SAAAAA DNNAAA HHHHxx 211 9130 1 3 1 11 11 211 211 211 211 22 23 DIAAAA ENNAAA OOOOxx 8993 9131 1 1 3 13 93 993 993 3993 8993 186 187 XHAAAA FNNAAA VVVVxx 4509 9132 1 1 9 9 9 509 509 4509 4509 18 19 LRAAAA GNNAAA AAAAxx 823 9133 1 3 3 3 23 823 823 823 823 46 47 RFAAAA HNNAAA HHHHxx 4747 9134 1 3 7 7 47 747 747 4747 4747 94 95 PAAAAA INNAAA OOOOxx 6955 9135 1 3 5 15 55 955 955 1955 6955 110 111 NHAAAA JNNAAA VVVVxx 7922 9136 0 2 2 2 22 922 1922 2922 7922 44 45 SSAAAA KNNAAA AAAAxx 6936 9137 0 0 6 16 36 936 936 1936 6936 72 73 UGAAAA LNNAAA HHHHxx 1546 9138 0 2 6 6 46 546 1546 1546 1546 92 93 MHAAAA MNNAAA OOOOxx 9836 9139 0 0 6 16 36 836 1836 4836 9836 72 73 IOAAAA NNNAAA VVVVxx 5626 9140 0 2 6 6 26 626 1626 626 5626 52 53 KIAAAA ONNAAA AAAAxx 4879 9141 1 3 9 19 79 879 879 4879 4879 158 159 RFAAAA PNNAAA HHHHxx 8590 9142 0 2 0 10 90 590 590 3590 8590 180 181 KSAAAA QNNAAA OOOOxx 8842 9143 0 2 2 2 42 842 842 3842 8842 84 85 CCAAAA RNNAAA VVVVxx 6505 9144 1 1 5 5 5 505 505 1505 6505 10 11 FQAAAA SNNAAA AAAAxx 2803 9145 1 3 3 3 3 803 803 2803 2803 6 7 VDAAAA TNNAAA HHHHxx 9258 9146 0 2 8 18 58 258 1258 4258 9258 116 117 CSAAAA UNNAAA OOOOxx 741 9147 1 1 1 1 41 741 741 741 741 82 83 NCAAAA VNNAAA VVVVxx 1457 9148 1 1 7 17 57 457 1457 1457 1457 114 115 BEAAAA WNNAAA AAAAxx 5777 9149 1 1 7 17 77 777 1777 777 5777 154 155 FOAAAA XNNAAA HHHHxx 2883 9150 1 3 3 3 83 883 883 2883 2883 166 167 XGAAAA YNNAAA OOOOxx 6610 9151 0 2 0 10 10 610 610 1610 6610 20 21 GUAAAA ZNNAAA VVVVxx 4331 9152 1 3 1 11 31 331 331 4331 4331 62 63 PKAAAA AONAAA AAAAxx 2712 9153 0 0 2 12 12 712 712 2712 2712 24 25 IAAAAA BONAAA HHHHxx 9268 9154 0 0 8 8 68 268 1268 4268 9268 136 137 MSAAAA CONAAA OOOOxx 410 9155 0 2 0 10 10 410 410 410 410 20 21 UPAAAA DONAAA VVVVxx 9411 9156 1 3 1 11 11 411 1411 4411 9411 22 23 ZXAAAA EONAAA AAAAxx 4683 9157 1 3 3 3 83 683 683 4683 4683 166 167 DYAAAA FONAAA HHHHxx 7072 9158 0 0 2 12 72 72 1072 2072 7072 144 145 AMAAAA GONAAA OOOOxx 5050 9159 0 2 0 10 50 50 1050 50 5050 100 101 GMAAAA HONAAA VVVVxx 5932 9160 0 0 2 12 32 932 1932 932 5932 64 65 EUAAAA IONAAA AAAAxx 2756 9161 0 0 6 16 56 756 756 2756 2756 112 113 ACAAAA JONAAA HHHHxx 9813 9162 1 1 3 13 13 813 1813 4813 9813 26 27 LNAAAA KONAAA OOOOxx 7388 9163 0 0 8 8 88 388 1388 2388 7388 176 177 EYAAAA LONAAA VVVVxx 2596 9164 0 0 6 16 96 596 596 2596 2596 192 193 WVAAAA MONAAA AAAAxx 5102 9165 0 2 2 2 2 102 1102 102 5102 4 5 GOAAAA NONAAA HHHHxx 208 9166 0 0 8 8 8 208 208 208 208 16 17 AIAAAA OONAAA OOOOxx 86 9167 0 2 6 6 86 86 86 86 86 172 173 IDAAAA PONAAA VVVVxx 8127 9168 1 3 7 7 27 127 127 3127 8127 54 55 PAAAAA QONAAA AAAAxx 5154 9169 0 2 4 14 54 154 1154 154 5154 108 109 GQAAAA RONAAA HHHHxx 4491 9170 1 3 1 11 91 491 491 4491 4491 182 183 TQAAAA SONAAA OOOOxx 7423 9171 1 3 3 3 23 423 1423 2423 7423 46 47 NZAAAA TONAAA VVVVxx 6441 9172 1 1 1 1 41 441 441 1441 6441 82 83 TNAAAA UONAAA AAAAxx 2920 9173 0 0 0 0 20 920 920 2920 2920 40 41 IIAAAA VONAAA HHHHxx 6386 9174 0 2 6 6 86 386 386 1386 6386 172 173 QLAAAA WONAAA OOOOxx 9744 9175 0 0 4 4 44 744 1744 4744 9744 88 89 UKAAAA XONAAA VVVVxx 2667 9176 1 3 7 7 67 667 667 2667 2667 134 135 PYAAAA YONAAA AAAAxx 5754 9177 0 2 4 14 54 754 1754 754 5754 108 109 INAAAA ZONAAA HHHHxx 4645 9178 1 1 5 5 45 645 645 4645 4645 90 91 RWAAAA APNAAA OOOOxx 4327 9179 1 3 7 7 27 327 327 4327 4327 54 55 LKAAAA BPNAAA VVVVxx 843 9180 1 3 3 3 43 843 843 843 843 86 87 LGAAAA CPNAAA AAAAxx 4085 9181 1 1 5 5 85 85 85 4085 4085 170 171 DBAAAA DPNAAA HHHHxx 2849 9182 1 1 9 9 49 849 849 2849 2849 98 99 PFAAAA EPNAAA OOOOxx 5734 9183 0 2 4 14 34 734 1734 734 5734 68 69 OMAAAA FPNAAA VVVVxx 5307 9184 1 3 7 7 7 307 1307 307 5307 14 15 DWAAAA GPNAAA AAAAxx 8433 9185 1 1 3 13 33 433 433 3433 8433 66 67 JMAAAA HPNAAA HHHHxx 3031 9186 1 3 1 11 31 31 1031 3031 3031 62 63 PMAAAA IPNAAA OOOOxx 5714 9187 0 2 4 14 14 714 1714 714 5714 28 29 ULAAAA JPNAAA VVVVxx 5969 9188 1 1 9 9 69 969 1969 969 5969 138 139 PVAAAA KPNAAA AAAAxx 2532 9189 0 0 2 12 32 532 532 2532 2532 64 65 KTAAAA LPNAAA HHHHxx 5219 9190 1 3 9 19 19 219 1219 219 5219 38 39 TSAAAA MPNAAA OOOOxx 7343 9191 1 3 3 3 43 343 1343 2343 7343 86 87 LWAAAA NPNAAA VVVVxx 9089 9192 1 1 9 9 89 89 1089 4089 9089 178 179 PLAAAA OPNAAA AAAAxx 9337 9193 1 1 7 17 37 337 1337 4337 9337 74 75 DVAAAA PPNAAA HHHHxx 5131 9194 1 3 1 11 31 131 1131 131 5131 62 63 JPAAAA QPNAAA OOOOxx 6253 9195 1 1 3 13 53 253 253 1253 6253 106 107 NGAAAA RPNAAA VVVVxx 5140 9196 0 0 0 0 40 140 1140 140 5140 80 81 SPAAAA SPNAAA AAAAxx 2953 9197 1 1 3 13 53 953 953 2953 2953 106 107 PJAAAA TPNAAA HHHHxx 4293 9198 1 1 3 13 93 293 293 4293 4293 186 187 DJAAAA UPNAAA OOOOxx 9974 9199 0 2 4 14 74 974 1974 4974 9974 148 149 QTAAAA VPNAAA VVVVxx 5061 9200 1 1 1 1 61 61 1061 61 5061 122 123 RMAAAA WPNAAA AAAAxx 8570 9201 0 2 0 10 70 570 570 3570 8570 140 141 QRAAAA XPNAAA HHHHxx 9504 9202 0 0 4 4 4 504 1504 4504 9504 8 9 OBAAAA YPNAAA OOOOxx 604 9203 0 0 4 4 4 604 604 604 604 8 9 GXAAAA ZPNAAA VVVVxx 4991 9204 1 3 1 11 91 991 991 4991 4991 182 183 ZJAAAA AQNAAA AAAAxx 880 9205 0 0 0 0 80 880 880 880 880 160 161 WHAAAA BQNAAA HHHHxx 3861 9206 1 1 1 1 61 861 1861 3861 3861 122 123 NSAAAA CQNAAA OOOOxx 8262 9207 0 2 2 2 62 262 262 3262 8262 124 125 UFAAAA DQNAAA VVVVxx 5689 9208 1 1 9 9 89 689 1689 689 5689 178 179 VKAAAA EQNAAA AAAAxx 1793 9209 1 1 3 13 93 793 1793 1793 1793 186 187 ZQAAAA FQNAAA HHHHxx 2661 9210 1 1 1 1 61 661 661 2661 2661 122 123 JYAAAA GQNAAA OOOOxx 7954 9211 0 2 4 14 54 954 1954 2954 7954 108 109 YTAAAA HQNAAA VVVVxx 1874 9212 0 2 4 14 74 874 1874 1874 1874 148 149 CUAAAA IQNAAA AAAAxx 2982 9213 0 2 2 2 82 982 982 2982 2982 164 165 SKAAAA JQNAAA HHHHxx 331 9214 1 3 1 11 31 331 331 331 331 62 63 TMAAAA KQNAAA OOOOxx 5021 9215 1 1 1 1 21 21 1021 21 5021 42 43 DLAAAA LQNAAA VVVVxx 9894 9216 0 2 4 14 94 894 1894 4894 9894 188 189 OQAAAA MQNAAA AAAAxx 7709 9217 1 1 9 9 9 709 1709 2709 7709 18 19 NKAAAA NQNAAA HHHHxx 4980 9218 0 0 0 0 80 980 980 4980 4980 160 161 OJAAAA OQNAAA OOOOxx 8249 9219 1 1 9 9 49 249 249 3249 8249 98 99 HFAAAA PQNAAA VVVVxx 7120 9220 0 0 0 0 20 120 1120 2120 7120 40 41 WNAAAA QQNAAA AAAAxx 7464 9221 0 0 4 4 64 464 1464 2464 7464 128 129 CBAAAA RQNAAA HHHHxx 8086 9222 0 2 6 6 86 86 86 3086 8086 172 173 AZAAAA SQNAAA OOOOxx 3509 9223 1 1 9 9 9 509 1509 3509 3509 18 19 ZEAAAA TQNAAA VVVVxx 3902 9224 0 2 2 2 2 902 1902 3902 3902 4 5 CUAAAA UQNAAA AAAAxx 9907 9225 1 3 7 7 7 907 1907 4907 9907 14 15 BRAAAA VQNAAA HHHHxx 6278 9226 0 2 8 18 78 278 278 1278 6278 156 157 MHAAAA WQNAAA OOOOxx 9316 9227 0 0 6 16 16 316 1316 4316 9316 32 33 IUAAAA XQNAAA VVVVxx 2824 9228 0 0 4 4 24 824 824 2824 2824 48 49 QEAAAA YQNAAA AAAAxx 1558 9229 0 2 8 18 58 558 1558 1558 1558 116 117 YHAAAA ZQNAAA HHHHxx 5436 9230 0 0 6 16 36 436 1436 436 5436 72 73 CBAAAA ARNAAA OOOOxx 1161 9231 1 1 1 1 61 161 1161 1161 1161 122 123 RSAAAA BRNAAA VVVVxx 7569 9232 1 1 9 9 69 569 1569 2569 7569 138 139 DFAAAA CRNAAA AAAAxx 9614 9233 0 2 4 14 14 614 1614 4614 9614 28 29 UFAAAA DRNAAA HHHHxx 6970 9234 0 2 0 10 70 970 970 1970 6970 140 141 CIAAAA ERNAAA OOOOxx 2422 9235 0 2 2 2 22 422 422 2422 2422 44 45 EPAAAA FRNAAA VVVVxx 8860 9236 0 0 0 0 60 860 860 3860 8860 120 121 UCAAAA GRNAAA AAAAxx 9912 9237 0 0 2 12 12 912 1912 4912 9912 24 25 GRAAAA HRNAAA HHHHxx 1109 9238 1 1 9 9 9 109 1109 1109 1109 18 19 RQAAAA IRNAAA OOOOxx 3286 9239 0 2 6 6 86 286 1286 3286 3286 172 173 KWAAAA JRNAAA VVVVxx 2277 9240 1 1 7 17 77 277 277 2277 2277 154 155 PJAAAA KRNAAA AAAAxx 8656 9241 0 0 6 16 56 656 656 3656 8656 112 113 YUAAAA LRNAAA HHHHxx 4656 9242 0 0 6 16 56 656 656 4656 4656 112 113 CXAAAA MRNAAA OOOOxx 6965 9243 1 1 5 5 65 965 965 1965 6965 130 131 XHAAAA NRNAAA VVVVxx 7591 9244 1 3 1 11 91 591 1591 2591 7591 182 183 ZFAAAA ORNAAA AAAAxx 4883 9245 1 3 3 3 83 883 883 4883 4883 166 167 VFAAAA PRNAAA HHHHxx 452 9246 0 0 2 12 52 452 452 452 452 104 105 KRAAAA QRNAAA OOOOxx 4018 9247 0 2 8 18 18 18 18 4018 4018 36 37 OYAAAA RRNAAA VVVVxx 4066 9248 0 2 6 6 66 66 66 4066 4066 132 133 KAAAAA SRNAAA AAAAxx 6480 9249 0 0 0 0 80 480 480 1480 6480 160 161 GPAAAA TRNAAA HHHHxx 8634 9250 0 2 4 14 34 634 634 3634 8634 68 69 CUAAAA URNAAA OOOOxx 9387 9251 1 3 7 7 87 387 1387 4387 9387 174 175 BXAAAA VRNAAA VVVVxx 3476 9252 0 0 6 16 76 476 1476 3476 3476 152 153 SDAAAA WRNAAA AAAAxx 5995 9253 1 3 5 15 95 995 1995 995 5995 190 191 PWAAAA XRNAAA HHHHxx 9677 9254 1 1 7 17 77 677 1677 4677 9677 154 155 FIAAAA YRNAAA OOOOxx 3884 9255 0 0 4 4 84 884 1884 3884 3884 168 169 KTAAAA ZRNAAA VVVVxx 6500 9256 0 0 0 0 0 500 500 1500 6500 0 1 AQAAAA ASNAAA AAAAxx 7972 9257 0 0 2 12 72 972 1972 2972 7972 144 145 QUAAAA BSNAAA HHHHxx 5281 9258 1 1 1 1 81 281 1281 281 5281 162 163 DVAAAA CSNAAA OOOOxx 1288 9259 0 0 8 8 88 288 1288 1288 1288 176 177 OXAAAA DSNAAA VVVVxx 4366 9260 0 2 6 6 66 366 366 4366 4366 132 133 YLAAAA ESNAAA AAAAxx 6557 9261 1 1 7 17 57 557 557 1557 6557 114 115 FSAAAA FSNAAA HHHHxx 7086 9262 0 2 6 6 86 86 1086 2086 7086 172 173 OMAAAA GSNAAA OOOOxx 6588 9263 0 0 8 8 88 588 588 1588 6588 176 177 KTAAAA HSNAAA VVVVxx 9062 9264 0 2 2 2 62 62 1062 4062 9062 124 125 OKAAAA ISNAAA AAAAxx 9230 9265 0 2 0 10 30 230 1230 4230 9230 60 61 ARAAAA JSNAAA HHHHxx 7672 9266 0 0 2 12 72 672 1672 2672 7672 144 145 CJAAAA KSNAAA OOOOxx 5204 9267 0 0 4 4 4 204 1204 204 5204 8 9 ESAAAA LSNAAA VVVVxx 2836 9268 0 0 6 16 36 836 836 2836 2836 72 73 CFAAAA MSNAAA AAAAxx 7165 9269 1 1 5 5 65 165 1165 2165 7165 130 131 PPAAAA NSNAAA HHHHxx 971 9270 1 3 1 11 71 971 971 971 971 142 143 JLAAAA OSNAAA OOOOxx 3851 9271 1 3 1 11 51 851 1851 3851 3851 102 103 DSAAAA PSNAAA VVVVxx 8593 9272 1 1 3 13 93 593 593 3593 8593 186 187 NSAAAA QSNAAA AAAAxx 7742 9273 0 2 2 2 42 742 1742 2742 7742 84 85 ULAAAA RSNAAA HHHHxx 2887 9274 1 3 7 7 87 887 887 2887 2887 174 175 BHAAAA SSNAAA OOOOxx 8479 9275 1 3 9 19 79 479 479 3479 8479 158 159 DOAAAA TSNAAA VVVVxx 9514 9276 0 2 4 14 14 514 1514 4514 9514 28 29 YBAAAA USNAAA AAAAxx 273 9277 1 1 3 13 73 273 273 273 273 146 147 NKAAAA VSNAAA HHHHxx 2938 9278 0 2 8 18 38 938 938 2938 2938 76 77 AJAAAA WSNAAA OOOOxx 9793 9279 1 1 3 13 93 793 1793 4793 9793 186 187 RMAAAA XSNAAA VVVVxx 8050 9280 0 2 0 10 50 50 50 3050 8050 100 101 QXAAAA YSNAAA AAAAxx 6702 9281 0 2 2 2 2 702 702 1702 6702 4 5 UXAAAA ZSNAAA HHHHxx 7290 9282 0 2 0 10 90 290 1290 2290 7290 180 181 KUAAAA ATNAAA OOOOxx 1837 9283 1 1 7 17 37 837 1837 1837 1837 74 75 RSAAAA BTNAAA VVVVxx 3206 9284 0 2 6 6 6 206 1206 3206 3206 12 13 ITAAAA CTNAAA AAAAxx 4925 9285 1 1 5 5 25 925 925 4925 4925 50 51 LHAAAA DTNAAA HHHHxx 5066 9286 0 2 6 6 66 66 1066 66 5066 132 133 WMAAAA ETNAAA OOOOxx 3401 9287 1 1 1 1 1 401 1401 3401 3401 2 3 VAAAAA FTNAAA VVVVxx 3474 9288 0 2 4 14 74 474 1474 3474 3474 148 149 QDAAAA GTNAAA AAAAxx 57 9289 1 1 7 17 57 57 57 57 57 114 115 FCAAAA HTNAAA HHHHxx 2082 9290 0 2 2 2 82 82 82 2082 2082 164 165 CCAAAA ITNAAA OOOOxx 100 9291 0 0 0 0 0 100 100 100 100 0 1 WDAAAA JTNAAA VVVVxx 9665 9292 1 1 5 5 65 665 1665 4665 9665 130 131 THAAAA KTNAAA AAAAxx 8284 9293 0 0 4 4 84 284 284 3284 8284 168 169 QGAAAA LTNAAA HHHHxx 958 9294 0 2 8 18 58 958 958 958 958 116 117 WKAAAA MTNAAA OOOOxx 5282 9295 0 2 2 2 82 282 1282 282 5282 164 165 EVAAAA NTNAAA VVVVxx 4257 9296 1 1 7 17 57 257 257 4257 4257 114 115 THAAAA OTNAAA AAAAxx 3160 9297 0 0 0 0 60 160 1160 3160 3160 120 121 ORAAAA PTNAAA HHHHxx 8449 9298 1 1 9 9 49 449 449 3449 8449 98 99 ZMAAAA QTNAAA OOOOxx 500 9299 0 0 0 0 0 500 500 500 500 0 1 GTAAAA RTNAAA VVVVxx 6432 9300 0 0 2 12 32 432 432 1432 6432 64 65 KNAAAA STNAAA AAAAxx 6220 9301 0 0 0 0 20 220 220 1220 6220 40 41 GFAAAA TTNAAA HHHHxx 7233 9302 1 1 3 13 33 233 1233 2233 7233 66 67 FSAAAA UTNAAA OOOOxx 2723 9303 1 3 3 3 23 723 723 2723 2723 46 47 TAAAAA VTNAAA VVVVxx 1899 9304 1 3 9 19 99 899 1899 1899 1899 198 199 BVAAAA WTNAAA AAAAxx 7158 9305 0 2 8 18 58 158 1158 2158 7158 116 117 IPAAAA XTNAAA HHHHxx 202 9306 0 2 2 2 2 202 202 202 202 4 5 UHAAAA YTNAAA OOOOxx 2286 9307 0 2 6 6 86 286 286 2286 2286 172 173 YJAAAA ZTNAAA VVVVxx 5356 9308 0 0 6 16 56 356 1356 356 5356 112 113 AYAAAA AUNAAA AAAAxx 3809 9309 1 1 9 9 9 809 1809 3809 3809 18 19 NQAAAA BUNAAA HHHHxx 3979 9310 1 3 9 19 79 979 1979 3979 3979 158 159 BXAAAA CUNAAA OOOOxx 8359 9311 1 3 9 19 59 359 359 3359 8359 118 119 NJAAAA DUNAAA VVVVxx 3479 9312 1 3 9 19 79 479 1479 3479 3479 158 159 VDAAAA EUNAAA AAAAxx 4895 9313 1 3 5 15 95 895 895 4895 4895 190 191 HGAAAA FUNAAA HHHHxx 6059 9314 1 3 9 19 59 59 59 1059 6059 118 119 BZAAAA GUNAAA OOOOxx 9560 9315 0 0 0 0 60 560 1560 4560 9560 120 121 SDAAAA HUNAAA VVVVxx 6756 9316 0 0 6 16 56 756 756 1756 6756 112 113 WZAAAA IUNAAA AAAAxx 7504 9317 0 0 4 4 4 504 1504 2504 7504 8 9 QCAAAA JUNAAA HHHHxx 6762 9318 0 2 2 2 62 762 762 1762 6762 124 125 CAAAAA KUNAAA OOOOxx 5304 9319 0 0 4 4 4 304 1304 304 5304 8 9 AWAAAA LUNAAA VVVVxx 9533 9320 1 1 3 13 33 533 1533 4533 9533 66 67 RCAAAA MUNAAA AAAAxx 6649 9321 1 1 9 9 49 649 649 1649 6649 98 99 TVAAAA NUNAAA HHHHxx 38 9322 0 2 8 18 38 38 38 38 38 76 77 MBAAAA OUNAAA OOOOxx 5713 9323 1 1 3 13 13 713 1713 713 5713 26 27 TLAAAA PUNAAA VVVVxx 3000 9324 0 0 0 0 0 0 1000 3000 3000 0 1 KLAAAA QUNAAA AAAAxx 3738 9325 0 2 8 18 38 738 1738 3738 3738 76 77 UNAAAA RUNAAA HHHHxx 3327 9326 1 3 7 7 27 327 1327 3327 3327 54 55 ZXAAAA SUNAAA OOOOxx 3922 9327 0 2 2 2 22 922 1922 3922 3922 44 45 WUAAAA TUNAAA VVVVxx 9245 9328 1 1 5 5 45 245 1245 4245 9245 90 91 PRAAAA UUNAAA AAAAxx 2172 9329 0 0 2 12 72 172 172 2172 2172 144 145 OFAAAA VUNAAA HHHHxx 7128 9330 0 0 8 8 28 128 1128 2128 7128 56 57 EOAAAA WUNAAA OOOOxx 1195 9331 1 3 5 15 95 195 1195 1195 1195 190 191 ZTAAAA XUNAAA VVVVxx 8445 9332 1 1 5 5 45 445 445 3445 8445 90 91 VMAAAA YUNAAA AAAAxx 8638 9333 0 2 8 18 38 638 638 3638 8638 76 77 GUAAAA ZUNAAA HHHHxx 1249 9334 1 1 9 9 49 249 1249 1249 1249 98 99 BWAAAA AVNAAA OOOOxx 8659 9335 1 3 9 19 59 659 659 3659 8659 118 119 BVAAAA BVNAAA VVVVxx 3556 9336 0 0 6 16 56 556 1556 3556 3556 112 113 UGAAAA CVNAAA AAAAxx 3347 9337 1 3 7 7 47 347 1347 3347 3347 94 95 TYAAAA DVNAAA HHHHxx 3260 9338 0 0 0 0 60 260 1260 3260 3260 120 121 KVAAAA EVNAAA OOOOxx 5139 9339 1 3 9 19 39 139 1139 139 5139 78 79 RPAAAA FVNAAA VVVVxx 9991 9340 1 3 1 11 91 991 1991 4991 9991 182 183 HUAAAA GVNAAA AAAAxx 5499 9341 1 3 9 19 99 499 1499 499 5499 198 199 NDAAAA HVNAAA HHHHxx 8082 9342 0 2 2 2 82 82 82 3082 8082 164 165 WYAAAA IVNAAA OOOOxx 1640 9343 0 0 0 0 40 640 1640 1640 1640 80 81 CLAAAA JVNAAA VVVVxx 8726 9344 0 2 6 6 26 726 726 3726 8726 52 53 QXAAAA KVNAAA AAAAxx 2339 9345 1 3 9 19 39 339 339 2339 2339 78 79 ZLAAAA LVNAAA HHHHxx 2601 9346 1 1 1 1 1 601 601 2601 2601 2 3 BWAAAA MVNAAA OOOOxx 9940 9347 0 0 0 0 40 940 1940 4940 9940 80 81 ISAAAA NVNAAA VVVVxx 4185 9348 1 1 5 5 85 185 185 4185 4185 170 171 ZEAAAA OVNAAA AAAAxx 9546 9349 0 2 6 6 46 546 1546 4546 9546 92 93 EDAAAA PVNAAA HHHHxx 5218 9350 0 2 8 18 18 218 1218 218 5218 36 37 SSAAAA QVNAAA OOOOxx 4374 9351 0 2 4 14 74 374 374 4374 4374 148 149 GMAAAA RVNAAA VVVVxx 288 9352 0 0 8 8 88 288 288 288 288 176 177 CLAAAA SVNAAA AAAAxx 7445 9353 1 1 5 5 45 445 1445 2445 7445 90 91 JAAAAA TVNAAA HHHHxx 1710 9354 0 2 0 10 10 710 1710 1710 1710 20 21 UNAAAA UVNAAA OOOOxx 6409 9355 1 1 9 9 9 409 409 1409 6409 18 19 NMAAAA VVNAAA VVVVxx 7982 9356 0 2 2 2 82 982 1982 2982 7982 164 165 AVAAAA WVNAAA AAAAxx 4950 9357 0 2 0 10 50 950 950 4950 4950 100 101 KIAAAA XVNAAA HHHHxx 9242 9358 0 2 2 2 42 242 1242 4242 9242 84 85 MRAAAA YVNAAA OOOOxx 3272 9359 0 0 2 12 72 272 1272 3272 3272 144 145 WVAAAA ZVNAAA VVVVxx 739 9360 1 3 9 19 39 739 739 739 739 78 79 LCAAAA AWNAAA AAAAxx 5526 9361 0 2 6 6 26 526 1526 526 5526 52 53 OEAAAA BWNAAA HHHHxx 8189 9362 1 1 9 9 89 189 189 3189 8189 178 179 ZCAAAA CWNAAA OOOOxx 9106 9363 0 2 6 6 6 106 1106 4106 9106 12 13 GMAAAA DWNAAA VVVVxx 9775 9364 1 3 5 15 75 775 1775 4775 9775 150 151 ZLAAAA EWNAAA AAAAxx 4643 9365 1 3 3 3 43 643 643 4643 4643 86 87 PWAAAA FWNAAA HHHHxx 8396 9366 0 0 6 16 96 396 396 3396 8396 192 193 YKAAAA GWNAAA OOOOxx 3255 9367 1 3 5 15 55 255 1255 3255 3255 110 111 FVAAAA HWNAAA VVVVxx 301 9368 1 1 1 1 1 301 301 301 301 2 3 PLAAAA IWNAAA AAAAxx 6014 9369 0 2 4 14 14 14 14 1014 6014 28 29 IXAAAA JWNAAA HHHHxx 6046 9370 0 2 6 6 46 46 46 1046 6046 92 93 OYAAAA KWNAAA OOOOxx 984 9371 0 0 4 4 84 984 984 984 984 168 169 WLAAAA LWNAAA VVVVxx 2420 9372 0 0 0 0 20 420 420 2420 2420 40 41 CPAAAA MWNAAA AAAAxx 2922 9373 0 2 2 2 22 922 922 2922 2922 44 45 KIAAAA NWNAAA HHHHxx 2317 9374 1 1 7 17 17 317 317 2317 2317 34 35 DLAAAA OWNAAA OOOOxx 7332 9375 0 0 2 12 32 332 1332 2332 7332 64 65 AWAAAA PWNAAA VVVVxx 6451 9376 1 3 1 11 51 451 451 1451 6451 102 103 DOAAAA QWNAAA AAAAxx 2589 9377 1 1 9 9 89 589 589 2589 2589 178 179 PVAAAA RWNAAA HHHHxx 4333 9378 1 1 3 13 33 333 333 4333 4333 66 67 RKAAAA SWNAAA OOOOxx 8650 9379 0 2 0 10 50 650 650 3650 8650 100 101 SUAAAA TWNAAA VVVVxx 6856 9380 0 0 6 16 56 856 856 1856 6856 112 113 SDAAAA UWNAAA AAAAxx 4194 9381 0 2 4 14 94 194 194 4194 4194 188 189 IFAAAA VWNAAA HHHHxx 6246 9382 0 2 6 6 46 246 246 1246 6246 92 93 GGAAAA WWNAAA OOOOxx 4371 9383 1 3 1 11 71 371 371 4371 4371 142 143 DMAAAA XWNAAA VVVVxx 1388 9384 0 0 8 8 88 388 1388 1388 1388 176 177 KBAAAA YWNAAA AAAAxx 1056 9385 0 0 6 16 56 56 1056 1056 1056 112 113 QOAAAA ZWNAAA HHHHxx 6041 9386 1 1 1 1 41 41 41 1041 6041 82 83 JYAAAA AXNAAA OOOOxx 6153 9387 1 1 3 13 53 153 153 1153 6153 106 107 RCAAAA BXNAAA VVVVxx 8450 9388 0 2 0 10 50 450 450 3450 8450 100 101 ANAAAA CXNAAA AAAAxx 3469 9389 1 1 9 9 69 469 1469 3469 3469 138 139 LDAAAA DXNAAA HHHHxx 5226 9390 0 2 6 6 26 226 1226 226 5226 52 53 ATAAAA EXNAAA OOOOxx 8112 9391 0 0 2 12 12 112 112 3112 8112 24 25 AAAAAA FXNAAA VVVVxx 647 9392 1 3 7 7 47 647 647 647 647 94 95 XYAAAA GXNAAA AAAAxx 2567 9393 1 3 7 7 67 567 567 2567 2567 134 135 TUAAAA HXNAAA HHHHxx 9064 9394 0 0 4 4 64 64 1064 4064 9064 128 129 QKAAAA IXNAAA OOOOxx 5161 9395 1 1 1 1 61 161 1161 161 5161 122 123 NQAAAA JXNAAA VVVVxx 5260 9396 0 0 0 0 60 260 1260 260 5260 120 121 IUAAAA KXNAAA AAAAxx 8988 9397 0 0 8 8 88 988 988 3988 8988 176 177 SHAAAA LXNAAA HHHHxx 9678 9398 0 2 8 18 78 678 1678 4678 9678 156 157 GIAAAA MXNAAA OOOOxx 6853 9399 1 1 3 13 53 853 853 1853 6853 106 107 PDAAAA NXNAAA VVVVxx 5294 9400 0 2 4 14 94 294 1294 294 5294 188 189 QVAAAA OXNAAA AAAAxx 9864 9401 0 0 4 4 64 864 1864 4864 9864 128 129 KPAAAA PXNAAA HHHHxx 8702 9402 0 2 2 2 2 702 702 3702 8702 4 5 SWAAAA QXNAAA OOOOxx 1132 9403 0 0 2 12 32 132 1132 1132 1132 64 65 ORAAAA RXNAAA VVVVxx 1524 9404 0 0 4 4 24 524 1524 1524 1524 48 49 QGAAAA SXNAAA AAAAxx 4560 9405 0 0 0 0 60 560 560 4560 4560 120 121 KTAAAA TXNAAA HHHHxx 2137 9406 1 1 7 17 37 137 137 2137 2137 74 75 FEAAAA UXNAAA OOOOxx 3283 9407 1 3 3 3 83 283 1283 3283 3283 166 167 HWAAAA VXNAAA VVVVxx 3377 9408 1 1 7 17 77 377 1377 3377 3377 154 155 XZAAAA WXNAAA AAAAxx 2267 9409 1 3 7 7 67 267 267 2267 2267 134 135 FJAAAA XXNAAA HHHHxx 8987 9410 1 3 7 7 87 987 987 3987 8987 174 175 RHAAAA YXNAAA OOOOxx 6709 9411 1 1 9 9 9 709 709 1709 6709 18 19 BYAAAA ZXNAAA VVVVxx 8059 9412 1 3 9 19 59 59 59 3059 8059 118 119 ZXAAAA AYNAAA AAAAxx 3402 9413 0 2 2 2 2 402 1402 3402 3402 4 5 WAAAAA BYNAAA HHHHxx 6443 9414 1 3 3 3 43 443 443 1443 6443 86 87 VNAAAA CYNAAA OOOOxx 8858 9415 0 2 8 18 58 858 858 3858 8858 116 117 SCAAAA DYNAAA VVVVxx 3974 9416 0 2 4 14 74 974 1974 3974 3974 148 149 WWAAAA EYNAAA AAAAxx 3521 9417 1 1 1 1 21 521 1521 3521 3521 42 43 LFAAAA FYNAAA HHHHxx 9509 9418 1 1 9 9 9 509 1509 4509 9509 18 19 TBAAAA GYNAAA OOOOxx 5442 9419 0 2 2 2 42 442 1442 442 5442 84 85 IBAAAA HYNAAA VVVVxx 8968 9420 0 0 8 8 68 968 968 3968 8968 136 137 YGAAAA IYNAAA AAAAxx 333 9421 1 1 3 13 33 333 333 333 333 66 67 VMAAAA JYNAAA HHHHxx 952 9422 0 0 2 12 52 952 952 952 952 104 105 QKAAAA KYNAAA OOOOxx 7482 9423 0 2 2 2 82 482 1482 2482 7482 164 165 UBAAAA LYNAAA VVVVxx 1486 9424 0 2 6 6 86 486 1486 1486 1486 172 173 EFAAAA MYNAAA AAAAxx 1815 9425 1 3 5 15 15 815 1815 1815 1815 30 31 VRAAAA NYNAAA HHHHxx 7937 9426 1 1 7 17 37 937 1937 2937 7937 74 75 HTAAAA OYNAAA OOOOxx 1436 9427 0 0 6 16 36 436 1436 1436 1436 72 73 GDAAAA PYNAAA VVVVxx 3470 9428 0 2 0 10 70 470 1470 3470 3470 140 141 MDAAAA QYNAAA AAAAxx 8195 9429 1 3 5 15 95 195 195 3195 8195 190 191 FDAAAA RYNAAA HHHHxx 6906 9430 0 2 6 6 6 906 906 1906 6906 12 13 QFAAAA SYNAAA OOOOxx 2539 9431 1 3 9 19 39 539 539 2539 2539 78 79 RTAAAA TYNAAA VVVVxx 5988 9432 0 0 8 8 88 988 1988 988 5988 176 177 IWAAAA UYNAAA AAAAxx 8908 9433 0 0 8 8 8 908 908 3908 8908 16 17 QEAAAA VYNAAA HHHHxx 2319 9434 1 3 9 19 19 319 319 2319 2319 38 39 FLAAAA WYNAAA OOOOxx 3263 9435 1 3 3 3 63 263 1263 3263 3263 126 127 NVAAAA XYNAAA VVVVxx 4039 9436 1 3 9 19 39 39 39 4039 4039 78 79 JZAAAA YYNAAA AAAAxx 6373 9437 1 1 3 13 73 373 373 1373 6373 146 147 DLAAAA ZYNAAA HHHHxx 1168 9438 0 0 8 8 68 168 1168 1168 1168 136 137 YSAAAA AZNAAA OOOOxx 8338 9439 0 2 8 18 38 338 338 3338 8338 76 77 SIAAAA BZNAAA VVVVxx 1172 9440 0 0 2 12 72 172 1172 1172 1172 144 145 CTAAAA CZNAAA AAAAxx 200 9441 0 0 0 0 0 200 200 200 200 0 1 SHAAAA DZNAAA HHHHxx 6355 9442 1 3 5 15 55 355 355 1355 6355 110 111 LKAAAA EZNAAA OOOOxx 7768 9443 0 0 8 8 68 768 1768 2768 7768 136 137 UMAAAA FZNAAA VVVVxx 25 9444 1 1 5 5 25 25 25 25 25 50 51 ZAAAAA GZNAAA AAAAxx 7144 9445 0 0 4 4 44 144 1144 2144 7144 88 89 UOAAAA HZNAAA HHHHxx 8671 9446 1 3 1 11 71 671 671 3671 8671 142 143 NVAAAA IZNAAA OOOOxx 9163 9447 1 3 3 3 63 163 1163 4163 9163 126 127 LOAAAA JZNAAA VVVVxx 8889 9448 1 1 9 9 89 889 889 3889 8889 178 179 XDAAAA KZNAAA AAAAxx 5950 9449 0 2 0 10 50 950 1950 950 5950 100 101 WUAAAA LZNAAA HHHHxx 6163 9450 1 3 3 3 63 163 163 1163 6163 126 127 BDAAAA MZNAAA OOOOxx 8119 9451 1 3 9 19 19 119 119 3119 8119 38 39 HAAAAA NZNAAA VVVVxx 1416 9452 0 0 6 16 16 416 1416 1416 1416 32 33 MCAAAA OZNAAA AAAAxx 4132 9453 0 0 2 12 32 132 132 4132 4132 64 65 YCAAAA PZNAAA HHHHxx 2294 9454 0 2 4 14 94 294 294 2294 2294 188 189 GKAAAA QZNAAA OOOOxx 9094 9455 0 2 4 14 94 94 1094 4094 9094 188 189 ULAAAA RZNAAA VVVVxx 4168 9456 0 0 8 8 68 168 168 4168 4168 136 137 IEAAAA SZNAAA AAAAxx 9108 9457 0 0 8 8 8 108 1108 4108 9108 16 17 IMAAAA TZNAAA HHHHxx 5706 9458 0 2 6 6 6 706 1706 706 5706 12 13 MLAAAA UZNAAA OOOOxx 2231 9459 1 3 1 11 31 231 231 2231 2231 62 63 VHAAAA VZNAAA VVVVxx 2173 9460 1 1 3 13 73 173 173 2173 2173 146 147 PFAAAA WZNAAA AAAAxx 90 9461 0 2 0 10 90 90 90 90 90 180 181 MDAAAA XZNAAA HHHHxx 9996 9462 0 0 6 16 96 996 1996 4996 9996 192 193 MUAAAA YZNAAA OOOOxx 330 9463 0 2 0 10 30 330 330 330 330 60 61 SMAAAA ZZNAAA VVVVxx 2052 9464 0 0 2 12 52 52 52 2052 2052 104 105 YAAAAA AAOAAA AAAAxx 1093 9465 1 1 3 13 93 93 1093 1093 1093 186 187 BQAAAA BAOAAA HHHHxx 5817 9466 1 1 7 17 17 817 1817 817 5817 34 35 TPAAAA CAOAAA OOOOxx 1559 9467 1 3 9 19 59 559 1559 1559 1559 118 119 ZHAAAA DAOAAA VVVVxx 8405 9468 1 1 5 5 5 405 405 3405 8405 10 11 HLAAAA EAOAAA AAAAxx 9962 9469 0 2 2 2 62 962 1962 4962 9962 124 125 ETAAAA FAOAAA HHHHxx 9461 9470 1 1 1 1 61 461 1461 4461 9461 122 123 XZAAAA GAOAAA OOOOxx 3028 9471 0 0 8 8 28 28 1028 3028 3028 56 57 MMAAAA HAOAAA VVVVxx 6814 9472 0 2 4 14 14 814 814 1814 6814 28 29 CCAAAA IAOAAA AAAAxx 9587 9473 1 3 7 7 87 587 1587 4587 9587 174 175 TEAAAA JAOAAA HHHHxx 6863 9474 1 3 3 3 63 863 863 1863 6863 126 127 ZDAAAA KAOAAA OOOOxx 4963 9475 1 3 3 3 63 963 963 4963 4963 126 127 XIAAAA LAOAAA VVVVxx 7811 9476 1 3 1 11 11 811 1811 2811 7811 22 23 LOAAAA MAOAAA AAAAxx 7608 9477 0 0 8 8 8 608 1608 2608 7608 16 17 QGAAAA NAOAAA HHHHxx 5321 9478 1 1 1 1 21 321 1321 321 5321 42 43 RWAAAA OAOAAA OOOOxx 9971 9479 1 3 1 11 71 971 1971 4971 9971 142 143 NTAAAA PAOAAA VVVVxx 6161 9480 1 1 1 1 61 161 161 1161 6161 122 123 ZCAAAA QAOAAA AAAAxx 2181 9481 1 1 1 1 81 181 181 2181 2181 162 163 XFAAAA RAOAAA HHHHxx 3828 9482 0 0 8 8 28 828 1828 3828 3828 56 57 GRAAAA SAOAAA OOOOxx 348 9483 0 0 8 8 48 348 348 348 348 96 97 KNAAAA TAOAAA VVVVxx 5459 9484 1 3 9 19 59 459 1459 459 5459 118 119 ZBAAAA UAOAAA AAAAxx 9406 9485 0 2 6 6 6 406 1406 4406 9406 12 13 UXAAAA VAOAAA HHHHxx 9852 9486 0 0 2 12 52 852 1852 4852 9852 104 105 YOAAAA WAOAAA OOOOxx 3095 9487 1 3 5 15 95 95 1095 3095 3095 190 191 BPAAAA XAOAAA VVVVxx 5597 9488 1 1 7 17 97 597 1597 597 5597 194 195 HHAAAA YAOAAA AAAAxx 8841 9489 1 1 1 1 41 841 841 3841 8841 82 83 BCAAAA ZAOAAA HHHHxx 3536 9490 0 0 6 16 36 536 1536 3536 3536 72 73 AGAAAA ABOAAA OOOOxx 4009 9491 1 1 9 9 9 9 9 4009 4009 18 19 FYAAAA BBOAAA VVVVxx 7366 9492 0 2 6 6 66 366 1366 2366 7366 132 133 IXAAAA CBOAAA AAAAxx 7327 9493 1 3 7 7 27 327 1327 2327 7327 54 55 VVAAAA DBOAAA HHHHxx 1613 9494 1 1 3 13 13 613 1613 1613 1613 26 27 BKAAAA EBOAAA OOOOxx 8619 9495 1 3 9 19 19 619 619 3619 8619 38 39 NTAAAA FBOAAA VVVVxx 4880 9496 0 0 0 0 80 880 880 4880 4880 160 161 SFAAAA GBOAAA AAAAxx 1552 9497 0 0 2 12 52 552 1552 1552 1552 104 105 SHAAAA HBOAAA HHHHxx 7636 9498 0 0 6 16 36 636 1636 2636 7636 72 73 SHAAAA IBOAAA OOOOxx 8397 9499 1 1 7 17 97 397 397 3397 8397 194 195 ZKAAAA JBOAAA VVVVxx 6224 9500 0 0 4 4 24 224 224 1224 6224 48 49 KFAAAA KBOAAA AAAAxx 9102 9501 0 2 2 2 2 102 1102 4102 9102 4 5 CMAAAA LBOAAA HHHHxx 7906 9502 0 2 6 6 6 906 1906 2906 7906 12 13 CSAAAA MBOAAA OOOOxx 9467 9503 1 3 7 7 67 467 1467 4467 9467 134 135 DAAAAA NBOAAA VVVVxx 828 9504 0 0 8 8 28 828 828 828 828 56 57 WFAAAA OBOAAA AAAAxx 9585 9505 1 1 5 5 85 585 1585 4585 9585 170 171 REAAAA PBOAAA HHHHxx 925 9506 1 1 5 5 25 925 925 925 925 50 51 PJAAAA QBOAAA OOOOxx 7375 9507 1 3 5 15 75 375 1375 2375 7375 150 151 RXAAAA RBOAAA VVVVxx 4027 9508 1 3 7 7 27 27 27 4027 4027 54 55 XYAAAA SBOAAA AAAAxx 766 9509 0 2 6 6 66 766 766 766 766 132 133 MDAAAA TBOAAA HHHHxx 5633 9510 1 1 3 13 33 633 1633 633 5633 66 67 RIAAAA UBOAAA OOOOxx 5648 9511 0 0 8 8 48 648 1648 648 5648 96 97 GJAAAA VBOAAA VVVVxx 148 9512 0 0 8 8 48 148 148 148 148 96 97 SFAAAA WBOAAA AAAAxx 2072 9513 0 0 2 12 72 72 72 2072 2072 144 145 SBAAAA XBOAAA HHHHxx 431 9514 1 3 1 11 31 431 431 431 431 62 63 PQAAAA YBOAAA OOOOxx 1711 9515 1 3 1 11 11 711 1711 1711 1711 22 23 VNAAAA ZBOAAA VVVVxx 9378 9516 0 2 8 18 78 378 1378 4378 9378 156 157 SWAAAA ACOAAA AAAAxx 6776 9517 0 0 6 16 76 776 776 1776 6776 152 153 QAAAAA BCOAAA HHHHxx 6842 9518 0 2 2 2 42 842 842 1842 6842 84 85 EDAAAA CCOAAA OOOOxx 2656 9519 0 0 6 16 56 656 656 2656 2656 112 113 EYAAAA DCOAAA VVVVxx 3116 9520 0 0 6 16 16 116 1116 3116 3116 32 33 WPAAAA ECOAAA AAAAxx 7904 9521 0 0 4 4 4 904 1904 2904 7904 8 9 ASAAAA FCOAAA HHHHxx 3529 9522 1 1 9 9 29 529 1529 3529 3529 58 59 TFAAAA GCOAAA OOOOxx 3240 9523 0 0 0 0 40 240 1240 3240 3240 80 81 QUAAAA HCOAAA VVVVxx 5801 9524 1 1 1 1 1 801 1801 801 5801 2 3 DPAAAA ICOAAA AAAAxx 4090 9525 0 2 0 10 90 90 90 4090 4090 180 181 IBAAAA JCOAAA HHHHxx 7687 9526 1 3 7 7 87 687 1687 2687 7687 174 175 RJAAAA KCOAAA OOOOxx 9711 9527 1 3 1 11 11 711 1711 4711 9711 22 23 NJAAAA LCOAAA VVVVxx 4760 9528 0 0 0 0 60 760 760 4760 4760 120 121 CBAAAA MCOAAA AAAAxx 5524 9529 0 0 4 4 24 524 1524 524 5524 48 49 MEAAAA NCOAAA HHHHxx 2251 9530 1 3 1 11 51 251 251 2251 2251 102 103 PIAAAA OCOAAA OOOOxx 1511 9531 1 3 1 11 11 511 1511 1511 1511 22 23 DGAAAA PCOAAA VVVVxx 5991 9532 1 3 1 11 91 991 1991 991 5991 182 183 LWAAAA QCOAAA AAAAxx 7808 9533 0 0 8 8 8 808 1808 2808 7808 16 17 IOAAAA RCOAAA HHHHxx 8708 9534 0 0 8 8 8 708 708 3708 8708 16 17 YWAAAA SCOAAA OOOOxx 8939 9535 1 3 9 19 39 939 939 3939 8939 78 79 VFAAAA TCOAAA VVVVxx 4295 9536 1 3 5 15 95 295 295 4295 4295 190 191 FJAAAA UCOAAA AAAAxx 5905 9537 1 1 5 5 5 905 1905 905 5905 10 11 DTAAAA VCOAAA HHHHxx 2649 9538 1 1 9 9 49 649 649 2649 2649 98 99 XXAAAA WCOAAA OOOOxx 2347 9539 1 3 7 7 47 347 347 2347 2347 94 95 HMAAAA XCOAAA VVVVxx 6339 9540 1 3 9 19 39 339 339 1339 6339 78 79 VJAAAA YCOAAA AAAAxx 292 9541 0 0 2 12 92 292 292 292 292 184 185 GLAAAA ZCOAAA HHHHxx 9314 9542 0 2 4 14 14 314 1314 4314 9314 28 29 GUAAAA ADOAAA OOOOxx 6893 9543 1 1 3 13 93 893 893 1893 6893 186 187 DFAAAA BDOAAA VVVVxx 3970 9544 0 2 0 10 70 970 1970 3970 3970 140 141 SWAAAA CDOAAA AAAAxx 1652 9545 0 0 2 12 52 652 1652 1652 1652 104 105 OLAAAA DDOAAA HHHHxx 4326 9546 0 2 6 6 26 326 326 4326 4326 52 53 KKAAAA EDOAAA OOOOxx 7881 9547 1 1 1 1 81 881 1881 2881 7881 162 163 DRAAAA FDOAAA VVVVxx 5291 9548 1 3 1 11 91 291 1291 291 5291 182 183 NVAAAA GDOAAA AAAAxx 957 9549 1 1 7 17 57 957 957 957 957 114 115 VKAAAA HDOAAA HHHHxx 2313 9550 1 1 3 13 13 313 313 2313 2313 26 27 ZKAAAA IDOAAA OOOOxx 5463 9551 1 3 3 3 63 463 1463 463 5463 126 127 DCAAAA JDOAAA VVVVxx 1268 9552 0 0 8 8 68 268 1268 1268 1268 136 137 UWAAAA KDOAAA AAAAxx 5028 9553 0 0 8 8 28 28 1028 28 5028 56 57 KLAAAA LDOAAA HHHHxx 656 9554 0 0 6 16 56 656 656 656 656 112 113 GZAAAA MDOAAA OOOOxx 9274 9555 0 2 4 14 74 274 1274 4274 9274 148 149 SSAAAA NDOAAA VVVVxx 8217 9556 1 1 7 17 17 217 217 3217 8217 34 35 BEAAAA ODOAAA AAAAxx 2175 9557 1 3 5 15 75 175 175 2175 2175 150 151 RFAAAA PDOAAA HHHHxx 6028 9558 0 0 8 8 28 28 28 1028 6028 56 57 WXAAAA QDOAAA OOOOxx 7584 9559 0 0 4 4 84 584 1584 2584 7584 168 169 SFAAAA RDOAAA VVVVxx 4114 9560 0 2 4 14 14 114 114 4114 4114 28 29 GCAAAA SDOAAA AAAAxx 8894 9561 0 2 4 14 94 894 894 3894 8894 188 189 CEAAAA TDOAAA HHHHxx 781 9562 1 1 1 1 81 781 781 781 781 162 163 BEAAAA UDOAAA OOOOxx 133 9563 1 1 3 13 33 133 133 133 133 66 67 DFAAAA VDOAAA VVVVxx 7572 9564 0 0 2 12 72 572 1572 2572 7572 144 145 GFAAAA WDOAAA AAAAxx 8514 9565 0 2 4 14 14 514 514 3514 8514 28 29 MPAAAA XDOAAA HHHHxx 3352 9566 0 0 2 12 52 352 1352 3352 3352 104 105 YYAAAA YDOAAA OOOOxx 8098 9567 0 2 8 18 98 98 98 3098 8098 196 197 MZAAAA ZDOAAA VVVVxx 9116 9568 0 0 6 16 16 116 1116 4116 9116 32 33 QMAAAA AEOAAA AAAAxx 9444 9569 0 0 4 4 44 444 1444 4444 9444 88 89 GZAAAA BEOAAA HHHHxx 2590 9570 0 2 0 10 90 590 590 2590 2590 180 181 QVAAAA CEOAAA OOOOxx 7302 9571 0 2 2 2 2 302 1302 2302 7302 4 5 WUAAAA DEOAAA VVVVxx 7444 9572 0 0 4 4 44 444 1444 2444 7444 88 89 IAAAAA EEOAAA AAAAxx 8748 9573 0 0 8 8 48 748 748 3748 8748 96 97 MYAAAA FEOAAA HHHHxx 7615 9574 1 3 5 15 15 615 1615 2615 7615 30 31 XGAAAA GEOAAA OOOOxx 6090 9575 0 2 0 10 90 90 90 1090 6090 180 181 GAAAAA HEOAAA VVVVxx 1529 9576 1 1 9 9 29 529 1529 1529 1529 58 59 VGAAAA IEOAAA AAAAxx 9398 9577 0 2 8 18 98 398 1398 4398 9398 196 197 MXAAAA JEOAAA HHHHxx 6114 9578 0 2 4 14 14 114 114 1114 6114 28 29 EBAAAA KEOAAA OOOOxx 2736 9579 0 0 6 16 36 736 736 2736 2736 72 73 GBAAAA LEOAAA VVVVxx 468 9580 0 0 8 8 68 468 468 468 468 136 137 ASAAAA MEOAAA AAAAxx 1487 9581 1 3 7 7 87 487 1487 1487 1487 174 175 FFAAAA NEOAAA HHHHxx 4784 9582 0 0 4 4 84 784 784 4784 4784 168 169 ACAAAA OEOAAA OOOOxx 6731 9583 1 3 1 11 31 731 731 1731 6731 62 63 XYAAAA PEOAAA VVVVxx 3328 9584 0 0 8 8 28 328 1328 3328 3328 56 57 AYAAAA QEOAAA AAAAxx 6891 9585 1 3 1 11 91 891 891 1891 6891 182 183 BFAAAA REOAAA HHHHxx 8039 9586 1 3 9 19 39 39 39 3039 8039 78 79 FXAAAA SEOAAA OOOOxx 4064 9587 0 0 4 4 64 64 64 4064 4064 128 129 IAAAAA TEOAAA VVVVxx 542 9588 0 2 2 2 42 542 542 542 542 84 85 WUAAAA UEOAAA AAAAxx 1039 9589 1 3 9 19 39 39 1039 1039 1039 78 79 ZNAAAA VEOAAA HHHHxx 5603 9590 1 3 3 3 3 603 1603 603 5603 6 7 NHAAAA WEOAAA OOOOxx 6641 9591 1 1 1 1 41 641 641 1641 6641 82 83 LVAAAA XEOAAA VVVVxx 6307 9592 1 3 7 7 7 307 307 1307 6307 14 15 PIAAAA YEOAAA AAAAxx 5354 9593 0 2 4 14 54 354 1354 354 5354 108 109 YXAAAA ZEOAAA HHHHxx 7878 9594 0 2 8 18 78 878 1878 2878 7878 156 157 ARAAAA AFOAAA OOOOxx 6391 9595 1 3 1 11 91 391 391 1391 6391 182 183 VLAAAA BFOAAA VVVVxx 4575 9596 1 3 5 15 75 575 575 4575 4575 150 151 ZTAAAA CFOAAA AAAAxx 6644 9597 0 0 4 4 44 644 644 1644 6644 88 89 OVAAAA DFOAAA HHHHxx 5207 9598 1 3 7 7 7 207 1207 207 5207 14 15 HSAAAA EFOAAA OOOOxx 1736 9599 0 0 6 16 36 736 1736 1736 1736 72 73 UOAAAA FFOAAA VVVVxx 3547 9600 1 3 7 7 47 547 1547 3547 3547 94 95 LGAAAA GFOAAA AAAAxx 6647 9601 1 3 7 7 47 647 647 1647 6647 94 95 RVAAAA HFOAAA HHHHxx 4107 9602 1 3 7 7 7 107 107 4107 4107 14 15 ZBAAAA IFOAAA OOOOxx 8125 9603 1 1 5 5 25 125 125 3125 8125 50 51 NAAAAA JFOAAA VVVVxx 9223 9604 1 3 3 3 23 223 1223 4223 9223 46 47 TQAAAA KFOAAA AAAAxx 6903 9605 1 3 3 3 3 903 903 1903 6903 6 7 NFAAAA LFOAAA HHHHxx 3639 9606 1 3 9 19 39 639 1639 3639 3639 78 79 ZJAAAA MFOAAA OOOOxx 9606 9607 0 2 6 6 6 606 1606 4606 9606 12 13 MFAAAA NFOAAA VVVVxx 3232 9608 0 0 2 12 32 232 1232 3232 3232 64 65 IUAAAA OFOAAA AAAAxx 2063 9609 1 3 3 3 63 63 63 2063 2063 126 127 JBAAAA PFOAAA HHHHxx 3731 9610 1 3 1 11 31 731 1731 3731 3731 62 63 NNAAAA QFOAAA OOOOxx 2558 9611 0 2 8 18 58 558 558 2558 2558 116 117 KUAAAA RFOAAA VVVVxx 2357 9612 1 1 7 17 57 357 357 2357 2357 114 115 RMAAAA SFOAAA AAAAxx 6008 9613 0 0 8 8 8 8 8 1008 6008 16 17 CXAAAA TFOAAA HHHHxx 8246 9614 0 2 6 6 46 246 246 3246 8246 92 93 EFAAAA UFOAAA OOOOxx 8220 9615 0 0 0 0 20 220 220 3220 8220 40 41 EEAAAA VFOAAA VVVVxx 1075 9616 1 3 5 15 75 75 1075 1075 1075 150 151 JPAAAA WFOAAA AAAAxx 2410 9617 0 2 0 10 10 410 410 2410 2410 20 21 SOAAAA XFOAAA HHHHxx 3253 9618 1 1 3 13 53 253 1253 3253 3253 106 107 DVAAAA YFOAAA OOOOxx 4370 9619 0 2 0 10 70 370 370 4370 4370 140 141 CMAAAA ZFOAAA VVVVxx 8426 9620 0 2 6 6 26 426 426 3426 8426 52 53 CMAAAA AGOAAA AAAAxx 2262 9621 0 2 2 2 62 262 262 2262 2262 124 125 AJAAAA BGOAAA HHHHxx 4149 9622 1 1 9 9 49 149 149 4149 4149 98 99 PDAAAA CGOAAA OOOOxx 2732 9623 0 0 2 12 32 732 732 2732 2732 64 65 CBAAAA DGOAAA VVVVxx 8606 9624 0 2 6 6 6 606 606 3606 8606 12 13 ATAAAA EGOAAA AAAAxx 6311 9625 1 3 1 11 11 311 311 1311 6311 22 23 TIAAAA FGOAAA HHHHxx 7223 9626 1 3 3 3 23 223 1223 2223 7223 46 47 VRAAAA GGOAAA OOOOxx 3054 9627 0 2 4 14 54 54 1054 3054 3054 108 109 MNAAAA HGOAAA VVVVxx 3952 9628 0 0 2 12 52 952 1952 3952 3952 104 105 AWAAAA IGOAAA AAAAxx 8252 9629 0 0 2 12 52 252 252 3252 8252 104 105 KFAAAA JGOAAA HHHHxx 6020 9630 0 0 0 0 20 20 20 1020 6020 40 41 OXAAAA KGOAAA OOOOxx 3846 9631 0 2 6 6 46 846 1846 3846 3846 92 93 YRAAAA LGOAAA VVVVxx 3755 9632 1 3 5 15 55 755 1755 3755 3755 110 111 LOAAAA MGOAAA AAAAxx 3765 9633 1 1 5 5 65 765 1765 3765 3765 130 131 VOAAAA NGOAAA HHHHxx 3434 9634 0 2 4 14 34 434 1434 3434 3434 68 69 CCAAAA OGOAAA OOOOxx 1381 9635 1 1 1 1 81 381 1381 1381 1381 162 163 DBAAAA PGOAAA VVVVxx 287 9636 1 3 7 7 87 287 287 287 287 174 175 BLAAAA QGOAAA AAAAxx 4476 9637 0 0 6 16 76 476 476 4476 4476 152 153 EQAAAA RGOAAA HHHHxx 2916 9638 0 0 6 16 16 916 916 2916 2916 32 33 EIAAAA SGOAAA OOOOxx 4517 9639 1 1 7 17 17 517 517 4517 4517 34 35 TRAAAA TGOAAA VVVVxx 4561 9640 1 1 1 1 61 561 561 4561 4561 122 123 LTAAAA UGOAAA AAAAxx 5106 9641 0 2 6 6 6 106 1106 106 5106 12 13 KOAAAA VGOAAA HHHHxx 2077 9642 1 1 7 17 77 77 77 2077 2077 154 155 XBAAAA WGOAAA OOOOxx 5269 9643 1 1 9 9 69 269 1269 269 5269 138 139 RUAAAA XGOAAA VVVVxx 5688 9644 0 0 8 8 88 688 1688 688 5688 176 177 UKAAAA YGOAAA AAAAxx 8831 9645 1 3 1 11 31 831 831 3831 8831 62 63 RBAAAA ZGOAAA HHHHxx 3867 9646 1 3 7 7 67 867 1867 3867 3867 134 135 TSAAAA AHOAAA OOOOxx 6062 9647 0 2 2 2 62 62 62 1062 6062 124 125 EZAAAA BHOAAA VVVVxx 8460 9648 0 0 0 0 60 460 460 3460 8460 120 121 KNAAAA CHOAAA AAAAxx 3138 9649 0 2 8 18 38 138 1138 3138 3138 76 77 SQAAAA DHOAAA HHHHxx 3173 9650 1 1 3 13 73 173 1173 3173 3173 146 147 BSAAAA EHOAAA OOOOxx 7018 9651 0 2 8 18 18 18 1018 2018 7018 36 37 YJAAAA FHOAAA VVVVxx 4836 9652 0 0 6 16 36 836 836 4836 4836 72 73 AEAAAA GHOAAA AAAAxx 1007 9653 1 3 7 7 7 7 1007 1007 1007 14 15 TMAAAA HHOAAA HHHHxx 658 9654 0 2 8 18 58 658 658 658 658 116 117 IZAAAA IHOAAA OOOOxx 5205 9655 1 1 5 5 5 205 1205 205 5205 10 11 FSAAAA JHOAAA VVVVxx 5805 9656 1 1 5 5 5 805 1805 805 5805 10 11 HPAAAA KHOAAA AAAAxx 5959 9657 1 3 9 19 59 959 1959 959 5959 118 119 FVAAAA LHOAAA HHHHxx 2863 9658 1 3 3 3 63 863 863 2863 2863 126 127 DGAAAA MHOAAA OOOOxx 7272 9659 0 0 2 12 72 272 1272 2272 7272 144 145 STAAAA NHOAAA VVVVxx 8437 9660 1 1 7 17 37 437 437 3437 8437 74 75 NMAAAA OHOAAA AAAAxx 4900 9661 0 0 0 0 0 900 900 4900 4900 0 1 MGAAAA PHOAAA HHHHxx 890 9662 0 2 0 10 90 890 890 890 890 180 181 GIAAAA QHOAAA OOOOxx 3530 9663 0 2 0 10 30 530 1530 3530 3530 60 61 UFAAAA RHOAAA VVVVxx 6209 9664 1 1 9 9 9 209 209 1209 6209 18 19 VEAAAA SHOAAA AAAAxx 4595 9665 1 3 5 15 95 595 595 4595 4595 190 191 TUAAAA THOAAA HHHHxx 5982 9666 0 2 2 2 82 982 1982 982 5982 164 165 CWAAAA UHOAAA OOOOxx 1101 9667 1 1 1 1 1 101 1101 1101 1101 2 3 JQAAAA VHOAAA VVVVxx 9555 9668 1 3 5 15 55 555 1555 4555 9555 110 111 NDAAAA WHOAAA AAAAxx 1918 9669 0 2 8 18 18 918 1918 1918 1918 36 37 UVAAAA XHOAAA HHHHxx 3527 9670 1 3 7 7 27 527 1527 3527 3527 54 55 RFAAAA YHOAAA OOOOxx 7309 9671 1 1 9 9 9 309 1309 2309 7309 18 19 DVAAAA ZHOAAA VVVVxx 8213 9672 1 1 3 13 13 213 213 3213 8213 26 27 XDAAAA AIOAAA AAAAxx 306 9673 0 2 6 6 6 306 306 306 306 12 13 ULAAAA BIOAAA HHHHxx 845 9674 1 1 5 5 45 845 845 845 845 90 91 NGAAAA CIOAAA OOOOxx 16 9675 0 0 6 16 16 16 16 16 16 32 33 QAAAAA DIOAAA VVVVxx 437 9676 1 1 7 17 37 437 437 437 437 74 75 VQAAAA EIOAAA AAAAxx 9518 9677 0 2 8 18 18 518 1518 4518 9518 36 37 CCAAAA FIOAAA HHHHxx 2142 9678 0 2 2 2 42 142 142 2142 2142 84 85 KEAAAA GIOAAA OOOOxx 8121 9679 1 1 1 1 21 121 121 3121 8121 42 43 JAAAAA HIOAAA VVVVxx 7354 9680 0 2 4 14 54 354 1354 2354 7354 108 109 WWAAAA IIOAAA AAAAxx 1720 9681 0 0 0 0 20 720 1720 1720 1720 40 41 EOAAAA JIOAAA HHHHxx 6078 9682 0 2 8 18 78 78 78 1078 6078 156 157 UZAAAA KIOAAA OOOOxx 5929 9683 1 1 9 9 29 929 1929 929 5929 58 59 BUAAAA LIOAAA VVVVxx 3856 9684 0 0 6 16 56 856 1856 3856 3856 112 113 ISAAAA MIOAAA AAAAxx 3424 9685 0 0 4 4 24 424 1424 3424 3424 48 49 SBAAAA NIOAAA HHHHxx 1712 9686 0 0 2 12 12 712 1712 1712 1712 24 25 WNAAAA OIOAAA OOOOxx 2340 9687 0 0 0 0 40 340 340 2340 2340 80 81 AMAAAA PIOAAA VVVVxx 5570 9688 0 2 0 10 70 570 1570 570 5570 140 141 GGAAAA QIOAAA AAAAxx 8734 9689 0 2 4 14 34 734 734 3734 8734 68 69 YXAAAA RIOAAA HHHHxx 6077 9690 1 1 7 17 77 77 77 1077 6077 154 155 TZAAAA SIOAAA OOOOxx 2960 9691 0 0 0 0 60 960 960 2960 2960 120 121 WJAAAA TIOAAA VVVVxx 5062 9692 0 2 2 2 62 62 1062 62 5062 124 125 SMAAAA UIOAAA AAAAxx 1532 9693 0 0 2 12 32 532 1532 1532 1532 64 65 YGAAAA VIOAAA HHHHxx 8298 9694 0 2 8 18 98 298 298 3298 8298 196 197 EHAAAA WIOAAA OOOOxx 2496 9695 0 0 6 16 96 496 496 2496 2496 192 193 ASAAAA XIOAAA VVVVxx 8412 9696 0 0 2 12 12 412 412 3412 8412 24 25 OLAAAA YIOAAA AAAAxx 724 9697 0 0 4 4 24 724 724 724 724 48 49 WBAAAA ZIOAAA HHHHxx 1019 9698 1 3 9 19 19 19 1019 1019 1019 38 39 FNAAAA AJOAAA OOOOxx 6265 9699 1 1 5 5 65 265 265 1265 6265 130 131 ZGAAAA BJOAAA VVVVxx 740 9700 0 0 0 0 40 740 740 740 740 80 81 MCAAAA CJOAAA AAAAxx 8495 9701 1 3 5 15 95 495 495 3495 8495 190 191 TOAAAA DJOAAA HHHHxx 6983 9702 1 3 3 3 83 983 983 1983 6983 166 167 PIAAAA EJOAAA OOOOxx 991 9703 1 3 1 11 91 991 991 991 991 182 183 DMAAAA FJOAAA VVVVxx 3189 9704 1 1 9 9 89 189 1189 3189 3189 178 179 RSAAAA GJOAAA AAAAxx 4487 9705 1 3 7 7 87 487 487 4487 4487 174 175 PQAAAA HJOAAA HHHHxx 5554 9706 0 2 4 14 54 554 1554 554 5554 108 109 QFAAAA IJOAAA OOOOxx 1258 9707 0 2 8 18 58 258 1258 1258 1258 116 117 KWAAAA JJOAAA VVVVxx 5359 9708 1 3 9 19 59 359 1359 359 5359 118 119 DYAAAA KJOAAA AAAAxx 2709 9709 1 1 9 9 9 709 709 2709 2709 18 19 FAAAAA LJOAAA HHHHxx 361 9710 1 1 1 1 61 361 361 361 361 122 123 XNAAAA MJOAAA OOOOxx 4028 9711 0 0 8 8 28 28 28 4028 4028 56 57 YYAAAA NJOAAA VVVVxx 3735 9712 1 3 5 15 35 735 1735 3735 3735 70 71 RNAAAA OJOAAA AAAAxx 4427 9713 1 3 7 7 27 427 427 4427 4427 54 55 HOAAAA PJOAAA HHHHxx 7540 9714 0 0 0 0 40 540 1540 2540 7540 80 81 AEAAAA QJOAAA OOOOxx 3569 9715 1 1 9 9 69 569 1569 3569 3569 138 139 HHAAAA RJOAAA VVVVxx 1916 9716 0 0 6 16 16 916 1916 1916 1916 32 33 SVAAAA SJOAAA AAAAxx 7596 9717 0 0 6 16 96 596 1596 2596 7596 192 193 EGAAAA TJOAAA HHHHxx 9721 9718 1 1 1 1 21 721 1721 4721 9721 42 43 XJAAAA UJOAAA OOOOxx 4429 9719 1 1 9 9 29 429 429 4429 4429 58 59 JOAAAA VJOAAA VVVVxx 3471 9720 1 3 1 11 71 471 1471 3471 3471 142 143 NDAAAA WJOAAA AAAAxx 1157 9721 1 1 7 17 57 157 1157 1157 1157 114 115 NSAAAA XJOAAA HHHHxx 5700 9722 0 0 0 0 0 700 1700 700 5700 0 1 GLAAAA YJOAAA OOOOxx 4431 9723 1 3 1 11 31 431 431 4431 4431 62 63 LOAAAA ZJOAAA VVVVxx 9409 9724 1 1 9 9 9 409 1409 4409 9409 18 19 XXAAAA AKOAAA AAAAxx 8752 9725 0 0 2 12 52 752 752 3752 8752 104 105 QYAAAA BKOAAA HHHHxx 9484 9726 0 0 4 4 84 484 1484 4484 9484 168 169 UAAAAA CKOAAA OOOOxx 1266 9727 0 2 6 6 66 266 1266 1266 1266 132 133 SWAAAA DKOAAA VVVVxx 9097 9728 1 1 7 17 97 97 1097 4097 9097 194 195 XLAAAA EKOAAA AAAAxx 3068 9729 0 0 8 8 68 68 1068 3068 3068 136 137 AOAAAA FKOAAA HHHHxx 5490 9730 0 2 0 10 90 490 1490 490 5490 180 181 EDAAAA GKOAAA OOOOxx 1375 9731 1 3 5 15 75 375 1375 1375 1375 150 151 XAAAAA HKOAAA VVVVxx 2487 9732 1 3 7 7 87 487 487 2487 2487 174 175 RRAAAA IKOAAA AAAAxx 1705 9733 1 1 5 5 5 705 1705 1705 1705 10 11 PNAAAA JKOAAA HHHHxx 1571 9734 1 3 1 11 71 571 1571 1571 1571 142 143 LIAAAA KKOAAA OOOOxx 4005 9735 1 1 5 5 5 5 5 4005 4005 10 11 BYAAAA LKOAAA VVVVxx 5497 9736 1 1 7 17 97 497 1497 497 5497 194 195 LDAAAA MKOAAA AAAAxx 2144 9737 0 0 4 4 44 144 144 2144 2144 88 89 MEAAAA NKOAAA HHHHxx 4052 9738 0 0 2 12 52 52 52 4052 4052 104 105 WZAAAA OKOAAA OOOOxx 4942 9739 0 2 2 2 42 942 942 4942 4942 84 85 CIAAAA PKOAAA VVVVxx 5504 9740 0 0 4 4 4 504 1504 504 5504 8 9 SDAAAA QKOAAA AAAAxx 2913 9741 1 1 3 13 13 913 913 2913 2913 26 27 BIAAAA RKOAAA HHHHxx 5617 9742 1 1 7 17 17 617 1617 617 5617 34 35 BIAAAA SKOAAA OOOOxx 8179 9743 1 3 9 19 79 179 179 3179 8179 158 159 PCAAAA TKOAAA VVVVxx 9437 9744 1 1 7 17 37 437 1437 4437 9437 74 75 ZYAAAA UKOAAA AAAAxx 1821 9745 1 1 1 1 21 821 1821 1821 1821 42 43 BSAAAA VKOAAA HHHHxx 5737 9746 1 1 7 17 37 737 1737 737 5737 74 75 RMAAAA WKOAAA OOOOxx 4207 9747 1 3 7 7 7 207 207 4207 4207 14 15 VFAAAA XKOAAA VVVVxx 4815 9748 1 3 5 15 15 815 815 4815 4815 30 31 FDAAAA YKOAAA AAAAxx 8707 9749 1 3 7 7 7 707 707 3707 8707 14 15 XWAAAA ZKOAAA HHHHxx 5970 9750 0 2 0 10 70 970 1970 970 5970 140 141 QVAAAA ALOAAA OOOOxx 5501 9751 1 1 1 1 1 501 1501 501 5501 2 3 PDAAAA BLOAAA VVVVxx 4013 9752 1 1 3 13 13 13 13 4013 4013 26 27 JYAAAA CLOAAA AAAAxx 9235 9753 1 3 5 15 35 235 1235 4235 9235 70 71 FRAAAA DLOAAA HHHHxx 2503 9754 1 3 3 3 3 503 503 2503 2503 6 7 HSAAAA ELOAAA OOOOxx 9181 9755 1 1 1 1 81 181 1181 4181 9181 162 163 DPAAAA FLOAAA VVVVxx 2289 9756 1 1 9 9 89 289 289 2289 2289 178 179 BKAAAA GLOAAA AAAAxx 4256 9757 0 0 6 16 56 256 256 4256 4256 112 113 SHAAAA HLOAAA HHHHxx 191 9758 1 3 1 11 91 191 191 191 191 182 183 JHAAAA ILOAAA OOOOxx 9655 9759 1 3 5 15 55 655 1655 4655 9655 110 111 JHAAAA JLOAAA VVVVxx 8615 9760 1 3 5 15 15 615 615 3615 8615 30 31 JTAAAA KLOAAA AAAAxx 3011 9761 1 3 1 11 11 11 1011 3011 3011 22 23 VLAAAA LLOAAA HHHHxx 6376 9762 0 0 6 16 76 376 376 1376 6376 152 153 GLAAAA MLOAAA OOOOxx 68 9763 0 0 8 8 68 68 68 68 68 136 137 QCAAAA NLOAAA VVVVxx 4720 9764 0 0 0 0 20 720 720 4720 4720 40 41 OZAAAA OLOAAA AAAAxx 6848 9765 0 0 8 8 48 848 848 1848 6848 96 97 KDAAAA PLOAAA HHHHxx 456 9766 0 0 6 16 56 456 456 456 456 112 113 ORAAAA QLOAAA OOOOxx 5887 9767 1 3 7 7 87 887 1887 887 5887 174 175 LSAAAA RLOAAA VVVVxx 9249 9768 1 1 9 9 49 249 1249 4249 9249 98 99 TRAAAA SLOAAA AAAAxx 4041 9769 1 1 1 1 41 41 41 4041 4041 82 83 LZAAAA TLOAAA HHHHxx 2304 9770 0 0 4 4 4 304 304 2304 2304 8 9 QKAAAA ULOAAA OOOOxx 8763 9771 1 3 3 3 63 763 763 3763 8763 126 127 BZAAAA VLOAAA VVVVxx 2115 9772 1 3 5 15 15 115 115 2115 2115 30 31 JDAAAA WLOAAA AAAAxx 8014 9773 0 2 4 14 14 14 14 3014 8014 28 29 GWAAAA XLOAAA HHHHxx 9895 9774 1 3 5 15 95 895 1895 4895 9895 190 191 PQAAAA YLOAAA OOOOxx 671 9775 1 3 1 11 71 671 671 671 671 142 143 VZAAAA ZLOAAA VVVVxx 3774 9776 0 2 4 14 74 774 1774 3774 3774 148 149 EPAAAA AMOAAA AAAAxx 134 9777 0 2 4 14 34 134 134 134 134 68 69 EFAAAA BMOAAA HHHHxx 534 9778 0 2 4 14 34 534 534 534 534 68 69 OUAAAA CMOAAA OOOOxx 7308 9779 0 0 8 8 8 308 1308 2308 7308 16 17 CVAAAA DMOAAA VVVVxx 5244 9780 0 0 4 4 44 244 1244 244 5244 88 89 STAAAA EMOAAA AAAAxx 1512 9781 0 0 2 12 12 512 1512 1512 1512 24 25 EGAAAA FMOAAA HHHHxx 8960 9782 0 0 0 0 60 960 960 3960 8960 120 121 QGAAAA GMOAAA OOOOxx 6602 9783 0 2 2 2 2 602 602 1602 6602 4 5 YTAAAA HMOAAA VVVVxx 593 9784 1 1 3 13 93 593 593 593 593 186 187 VWAAAA IMOAAA AAAAxx 2353 9785 1 1 3 13 53 353 353 2353 2353 106 107 NMAAAA JMOAAA HHHHxx 4139 9786 1 3 9 19 39 139 139 4139 4139 78 79 FDAAAA KMOAAA OOOOxx 3063 9787 1 3 3 3 63 63 1063 3063 3063 126 127 VNAAAA LMOAAA VVVVxx 652 9788 0 0 2 12 52 652 652 652 652 104 105 CZAAAA MMOAAA AAAAxx 7405 9789 1 1 5 5 5 405 1405 2405 7405 10 11 VYAAAA NMOAAA HHHHxx 3034 9790 0 2 4 14 34 34 1034 3034 3034 68 69 SMAAAA OMOAAA OOOOxx 4614 9791 0 2 4 14 14 614 614 4614 4614 28 29 MVAAAA PMOAAA VVVVxx 2351 9792 1 3 1 11 51 351 351 2351 2351 102 103 LMAAAA QMOAAA AAAAxx 8208 9793 0 0 8 8 8 208 208 3208 8208 16 17 SDAAAA RMOAAA HHHHxx 5475 9794 1 3 5 15 75 475 1475 475 5475 150 151 PCAAAA SMOAAA OOOOxx 6875 9795 1 3 5 15 75 875 875 1875 6875 150 151 LEAAAA TMOAAA VVVVxx 563 9796 1 3 3 3 63 563 563 563 563 126 127 RVAAAA UMOAAA AAAAxx 3346 9797 0 2 6 6 46 346 1346 3346 3346 92 93 SYAAAA VMOAAA HHHHxx 291 9798 1 3 1 11 91 291 291 291 291 182 183 FLAAAA WMOAAA OOOOxx 6345 9799 1 1 5 5 45 345 345 1345 6345 90 91 BKAAAA XMOAAA VVVVxx 8099 9800 1 3 9 19 99 99 99 3099 8099 198 199 NZAAAA YMOAAA AAAAxx 2078 9801 0 2 8 18 78 78 78 2078 2078 156 157 YBAAAA ZMOAAA HHHHxx 8238 9802 0 2 8 18 38 238 238 3238 8238 76 77 WEAAAA ANOAAA OOOOxx 4482 9803 0 2 2 2 82 482 482 4482 4482 164 165 KQAAAA BNOAAA VVVVxx 716 9804 0 0 6 16 16 716 716 716 716 32 33 OBAAAA CNOAAA AAAAxx 7288 9805 0 0 8 8 88 288 1288 2288 7288 176 177 IUAAAA DNOAAA HHHHxx 5906 9806 0 2 6 6 6 906 1906 906 5906 12 13 ETAAAA ENOAAA OOOOxx 5618 9807 0 2 8 18 18 618 1618 618 5618 36 37 CIAAAA FNOAAA VVVVxx 1141 9808 1 1 1 1 41 141 1141 1141 1141 82 83 XRAAAA GNOAAA AAAAxx 8231 9809 1 3 1 11 31 231 231 3231 8231 62 63 PEAAAA HNOAAA HHHHxx 3713 9810 1 1 3 13 13 713 1713 3713 3713 26 27 VMAAAA INOAAA OOOOxx 9158 9811 0 2 8 18 58 158 1158 4158 9158 116 117 GOAAAA JNOAAA VVVVxx 4051 9812 1 3 1 11 51 51 51 4051 4051 102 103 VZAAAA KNOAAA AAAAxx 1973 9813 1 1 3 13 73 973 1973 1973 1973 146 147 XXAAAA LNOAAA HHHHxx 6710 9814 0 2 0 10 10 710 710 1710 6710 20 21 CYAAAA MNOAAA OOOOxx 1021 9815 1 1 1 1 21 21 1021 1021 1021 42 43 HNAAAA NNOAAA VVVVxx 2196 9816 0 0 6 16 96 196 196 2196 2196 192 193 MGAAAA ONOAAA AAAAxx 8335 9817 1 3 5 15 35 335 335 3335 8335 70 71 PIAAAA PNOAAA HHHHxx 2272 9818 0 0 2 12 72 272 272 2272 2272 144 145 KJAAAA QNOAAA OOOOxx 3818 9819 0 2 8 18 18 818 1818 3818 3818 36 37 WQAAAA RNOAAA VVVVxx 679 9820 1 3 9 19 79 679 679 679 679 158 159 DAAAAA SNOAAA AAAAxx 7512 9821 0 0 2 12 12 512 1512 2512 7512 24 25 YCAAAA TNOAAA HHHHxx 493 9822 1 1 3 13 93 493 493 493 493 186 187 ZSAAAA UNOAAA OOOOxx 5663 9823 1 3 3 3 63 663 1663 663 5663 126 127 VJAAAA VNOAAA VVVVxx 4655 9824 1 3 5 15 55 655 655 4655 4655 110 111 BXAAAA WNOAAA AAAAxx 3996 9825 0 0 6 16 96 996 1996 3996 3996 192 193 SXAAAA XNOAAA HHHHxx 8797 9826 1 1 7 17 97 797 797 3797 8797 194 195 JAAAAA YNOAAA OOOOxx 2991 9827 1 3 1 11 91 991 991 2991 2991 182 183 BLAAAA ZNOAAA VVVVxx 7038 9828 0 2 8 18 38 38 1038 2038 7038 76 77 SKAAAA AOOAAA AAAAxx 4174 9829 0 2 4 14 74 174 174 4174 4174 148 149 OEAAAA BOOAAA HHHHxx 6908 9830 0 0 8 8 8 908 908 1908 6908 16 17 SFAAAA COOAAA OOOOxx 8477 9831 1 1 7 17 77 477 477 3477 8477 154 155 BOAAAA DOOAAA VVVVxx 3576 9832 0 0 6 16 76 576 1576 3576 3576 152 153 OHAAAA EOOAAA AAAAxx 2685 9833 1 1 5 5 85 685 685 2685 2685 170 171 HZAAAA FOOAAA HHHHxx 9161 9834 1 1 1 1 61 161 1161 4161 9161 122 123 JOAAAA GOOAAA OOOOxx 2951 9835 1 3 1 11 51 951 951 2951 2951 102 103 NJAAAA HOOAAA VVVVxx 8362 9836 0 2 2 2 62 362 362 3362 8362 124 125 QJAAAA IOOAAA AAAAxx 2379 9837 1 3 9 19 79 379 379 2379 2379 158 159 NNAAAA JOOAAA HHHHxx 1277 9838 1 1 7 17 77 277 1277 1277 1277 154 155 DXAAAA KOOAAA OOOOxx 1728 9839 0 0 8 8 28 728 1728 1728 1728 56 57 MOAAAA LOOAAA VVVVxx 9816 9840 0 0 6 16 16 816 1816 4816 9816 32 33 ONAAAA MOOAAA AAAAxx 6288 9841 0 0 8 8 88 288 288 1288 6288 176 177 WHAAAA NOOAAA HHHHxx 8985 9842 1 1 5 5 85 985 985 3985 8985 170 171 PHAAAA OOOAAA OOOOxx 771 9843 1 3 1 11 71 771 771 771 771 142 143 RDAAAA POOAAA VVVVxx 464 9844 0 0 4 4 64 464 464 464 464 128 129 WRAAAA QOOAAA AAAAxx 9625 9845 1 1 5 5 25 625 1625 4625 9625 50 51 FGAAAA ROOAAA HHHHxx 9608 9846 0 0 8 8 8 608 1608 4608 9608 16 17 OFAAAA SOOAAA OOOOxx 9170 9847 0 2 0 10 70 170 1170 4170 9170 140 141 SOAAAA TOOAAA VVVVxx 9658 9848 0 2 8 18 58 658 1658 4658 9658 116 117 MHAAAA UOOAAA AAAAxx 7515 9849 1 3 5 15 15 515 1515 2515 7515 30 31 BDAAAA VOOAAA HHHHxx 9400 9850 0 0 0 0 0 400 1400 4400 9400 0 1 OXAAAA WOOAAA OOOOxx 2045 9851 1 1 5 5 45 45 45 2045 2045 90 91 RAAAAA XOOAAA VVVVxx 324 9852 0 0 4 4 24 324 324 324 324 48 49 MMAAAA YOOAAA AAAAxx 4252 9853 0 0 2 12 52 252 252 4252 4252 104 105 OHAAAA ZOOAAA HHHHxx 8329 9854 1 1 9 9 29 329 329 3329 8329 58 59 JIAAAA APOAAA OOOOxx 4472 9855 0 0 2 12 72 472 472 4472 4472 144 145 AQAAAA BPOAAA VVVVxx 1047 9856 1 3 7 7 47 47 1047 1047 1047 94 95 HOAAAA CPOAAA AAAAxx 9341 9857 1 1 1 1 41 341 1341 4341 9341 82 83 HVAAAA DPOAAA HHHHxx 7000 9858 0 0 0 0 0 0 1000 2000 7000 0 1 GJAAAA EPOAAA OOOOxx 1429 9859 1 1 9 9 29 429 1429 1429 1429 58 59 ZCAAAA FPOAAA VVVVxx 2701 9860 1 1 1 1 1 701 701 2701 2701 2 3 XZAAAA GPOAAA AAAAxx 6630 9861 0 2 0 10 30 630 630 1630 6630 60 61 AVAAAA HPOAAA HHHHxx 3669 9862 1 1 9 9 69 669 1669 3669 3669 138 139 DLAAAA IPOAAA OOOOxx 8613 9863 1 1 3 13 13 613 613 3613 8613 26 27 HTAAAA JPOAAA VVVVxx 7080 9864 0 0 0 0 80 80 1080 2080 7080 160 161 IMAAAA KPOAAA AAAAxx 8788 9865 0 0 8 8 88 788 788 3788 8788 176 177 AAAAAA LPOAAA HHHHxx 6291 9866 1 3 1 11 91 291 291 1291 6291 182 183 ZHAAAA MPOAAA OOOOxx 7885 9867 1 1 5 5 85 885 1885 2885 7885 170 171 HRAAAA NPOAAA VVVVxx 7160 9868 0 0 0 0 60 160 1160 2160 7160 120 121 KPAAAA OPOAAA AAAAxx 6140 9869 0 0 0 0 40 140 140 1140 6140 80 81 ECAAAA PPOAAA HHHHxx 9881 9870 1 1 1 1 81 881 1881 4881 9881 162 163 BQAAAA QPOAAA OOOOxx 9140 9871 0 0 0 0 40 140 1140 4140 9140 80 81 ONAAAA RPOAAA VVVVxx 644 9872 0 0 4 4 44 644 644 644 644 88 89 UYAAAA SPOAAA AAAAxx 3667 9873 1 3 7 7 67 667 1667 3667 3667 134 135 BLAAAA TPOAAA HHHHxx 2675 9874 1 3 5 15 75 675 675 2675 2675 150 151 XYAAAA UPOAAA OOOOxx 9492 9875 0 0 2 12 92 492 1492 4492 9492 184 185 CBAAAA VPOAAA VVVVxx 5004 9876 0 0 4 4 4 4 1004 4 5004 8 9 MKAAAA WPOAAA AAAAxx 9456 9877 0 0 6 16 56 456 1456 4456 9456 112 113 SZAAAA XPOAAA HHHHxx 8197 9878 1 1 7 17 97 197 197 3197 8197 194 195 HDAAAA YPOAAA OOOOxx 2837 9879 1 1 7 17 37 837 837 2837 2837 74 75 DFAAAA ZPOAAA VVVVxx 127 9880 1 3 7 7 27 127 127 127 127 54 55 XEAAAA AQOAAA AAAAxx 9772 9881 0 0 2 12 72 772 1772 4772 9772 144 145 WLAAAA BQOAAA HHHHxx 5743 9882 1 3 3 3 43 743 1743 743 5743 86 87 XMAAAA CQOAAA OOOOxx 2007 9883 1 3 7 7 7 7 7 2007 2007 14 15 FZAAAA DQOAAA VVVVxx 7586 9884 0 2 6 6 86 586 1586 2586 7586 172 173 UFAAAA EQOAAA AAAAxx 45 9885 1 1 5 5 45 45 45 45 45 90 91 TBAAAA FQOAAA HHHHxx 6482 9886 0 2 2 2 82 482 482 1482 6482 164 165 IPAAAA GQOAAA OOOOxx 4565 9887 1 1 5 5 65 565 565 4565 4565 130 131 PTAAAA HQOAAA VVVVxx 6975 9888 1 3 5 15 75 975 975 1975 6975 150 151 HIAAAA IQOAAA AAAAxx 7260 9889 0 0 0 0 60 260 1260 2260 7260 120 121 GTAAAA JQOAAA HHHHxx 2830 9890 0 2 0 10 30 830 830 2830 2830 60 61 WEAAAA KQOAAA OOOOxx 9365 9891 1 1 5 5 65 365 1365 4365 9365 130 131 FWAAAA LQOAAA VVVVxx 8207 9892 1 3 7 7 7 207 207 3207 8207 14 15 RDAAAA MQOAAA AAAAxx 2506 9893 0 2 6 6 6 506 506 2506 2506 12 13 KSAAAA NQOAAA HHHHxx 8081 9894 1 1 1 1 81 81 81 3081 8081 162 163 VYAAAA OQOAAA OOOOxx 8678 9895 0 2 8 18 78 678 678 3678 8678 156 157 UVAAAA PQOAAA VVVVxx 9932 9896 0 0 2 12 32 932 1932 4932 9932 64 65 ASAAAA QQOAAA AAAAxx 447 9897 1 3 7 7 47 447 447 447 447 94 95 FRAAAA RQOAAA HHHHxx 9187 9898 1 3 7 7 87 187 1187 4187 9187 174 175 JPAAAA SQOAAA OOOOxx 89 9899 1 1 9 9 89 89 89 89 89 178 179 LDAAAA TQOAAA VVVVxx 7027 9900 1 3 7 7 27 27 1027 2027 7027 54 55 HKAAAA UQOAAA AAAAxx 1536 9901 0 0 6 16 36 536 1536 1536 1536 72 73 CHAAAA VQOAAA HHHHxx 160 9902 0 0 0 0 60 160 160 160 160 120 121 EGAAAA WQOAAA OOOOxx 7679 9903 1 3 9 19 79 679 1679 2679 7679 158 159 JJAAAA XQOAAA VVVVxx 5973 9904 1 1 3 13 73 973 1973 973 5973 146 147 TVAAAA YQOAAA AAAAxx 4401 9905 1 1 1 1 1 401 401 4401 4401 2 3 HNAAAA ZQOAAA HHHHxx 395 9906 1 3 5 15 95 395 395 395 395 190 191 FPAAAA AROAAA OOOOxx 4904 9907 0 0 4 4 4 904 904 4904 4904 8 9 QGAAAA BROAAA VVVVxx 2759 9908 1 3 9 19 59 759 759 2759 2759 118 119 DCAAAA CROAAA AAAAxx 8713 9909 1 1 3 13 13 713 713 3713 8713 26 27 DXAAAA DROAAA HHHHxx 3770 9910 0 2 0 10 70 770 1770 3770 3770 140 141 APAAAA EROAAA OOOOxx 8272 9911 0 0 2 12 72 272 272 3272 8272 144 145 EGAAAA FROAAA VVVVxx 5358 9912 0 2 8 18 58 358 1358 358 5358 116 117 CYAAAA GROAAA AAAAxx 9747 9913 1 3 7 7 47 747 1747 4747 9747 94 95 XKAAAA HROAAA HHHHxx 1567 9914 1 3 7 7 67 567 1567 1567 1567 134 135 HIAAAA IROAAA OOOOxx 2136 9915 0 0 6 16 36 136 136 2136 2136 72 73 EEAAAA JROAAA VVVVxx 314 9916 0 2 4 14 14 314 314 314 314 28 29 CMAAAA KROAAA AAAAxx 4583 9917 1 3 3 3 83 583 583 4583 4583 166 167 HUAAAA LROAAA HHHHxx 375 9918 1 3 5 15 75 375 375 375 375 150 151 LOAAAA MROAAA OOOOxx 5566 9919 0 2 6 6 66 566 1566 566 5566 132 133 CGAAAA NROAAA VVVVxx 6865 9920 1 1 5 5 65 865 865 1865 6865 130 131 BEAAAA OROAAA AAAAxx 894 9921 0 2 4 14 94 894 894 894 894 188 189 KIAAAA PROAAA HHHHxx 5399 9922 1 3 9 19 99 399 1399 399 5399 198 199 RZAAAA QROAAA OOOOxx 1385 9923 1 1 5 5 85 385 1385 1385 1385 170 171 HBAAAA RROAAA VVVVxx 2156 9924 0 0 6 16 56 156 156 2156 2156 112 113 YEAAAA SROAAA AAAAxx 9659 9925 1 3 9 19 59 659 1659 4659 9659 118 119 NHAAAA TROAAA HHHHxx 477 9926 1 1 7 17 77 477 477 477 477 154 155 JSAAAA UROAAA OOOOxx 8194 9927 0 2 4 14 94 194 194 3194 8194 188 189 EDAAAA VROAAA VVVVxx 3937 9928 1 1 7 17 37 937 1937 3937 3937 74 75 LVAAAA WROAAA AAAAxx 3745 9929 1 1 5 5 45 745 1745 3745 3745 90 91 BOAAAA XROAAA HHHHxx 4096 9930 0 0 6 16 96 96 96 4096 4096 192 193 OBAAAA YROAAA OOOOxx 5487 9931 1 3 7 7 87 487 1487 487 5487 174 175 BDAAAA ZROAAA VVVVxx 2475 9932 1 3 5 15 75 475 475 2475 2475 150 151 FRAAAA ASOAAA AAAAxx 6105 9933 1 1 5 5 5 105 105 1105 6105 10 11 VAAAAA BSOAAA HHHHxx 6036 9934 0 0 6 16 36 36 36 1036 6036 72 73 EYAAAA CSOAAA OOOOxx 1315 9935 1 3 5 15 15 315 1315 1315 1315 30 31 PYAAAA DSOAAA VVVVxx 4473 9936 1 1 3 13 73 473 473 4473 4473 146 147 BQAAAA ESOAAA AAAAxx 4016 9937 0 0 6 16 16 16 16 4016 4016 32 33 MYAAAA FSOAAA HHHHxx 8135 9938 1 3 5 15 35 135 135 3135 8135 70 71 XAAAAA GSOAAA OOOOxx 8892 9939 0 0 2 12 92 892 892 3892 8892 184 185 AEAAAA HSOAAA VVVVxx 4850 9940 0 2 0 10 50 850 850 4850 4850 100 101 OEAAAA ISOAAA AAAAxx 2545 9941 1 1 5 5 45 545 545 2545 2545 90 91 XTAAAA JSOAAA HHHHxx 3788 9942 0 0 8 8 88 788 1788 3788 3788 176 177 SPAAAA KSOAAA OOOOxx 1672 9943 0 0 2 12 72 672 1672 1672 1672 144 145 IMAAAA LSOAAA VVVVxx 3664 9944 0 0 4 4 64 664 1664 3664 3664 128 129 YKAAAA MSOAAA AAAAxx 3775 9945 1 3 5 15 75 775 1775 3775 3775 150 151 FPAAAA NSOAAA HHHHxx 3103 9946 1 3 3 3 3 103 1103 3103 3103 6 7 JPAAAA OSOAAA OOOOxx 9335 9947 1 3 5 15 35 335 1335 4335 9335 70 71 BVAAAA PSOAAA VVVVxx 9200 9948 0 0 0 0 0 200 1200 4200 9200 0 1 WPAAAA QSOAAA AAAAxx 8665 9949 1 1 5 5 65 665 665 3665 8665 130 131 HVAAAA RSOAAA HHHHxx 1356 9950 0 0 6 16 56 356 1356 1356 1356 112 113 EAAAAA SSOAAA OOOOxx 6118 9951 0 2 8 18 18 118 118 1118 6118 36 37 IBAAAA TSOAAA VVVVxx 4605 9952 1 1 5 5 5 605 605 4605 4605 10 11 DVAAAA USOAAA AAAAxx 5651 9953 1 3 1 11 51 651 1651 651 5651 102 103 JJAAAA VSOAAA HHHHxx 9055 9954 1 3 5 15 55 55 1055 4055 9055 110 111 HKAAAA WSOAAA OOOOxx 8461 9955 1 1 1 1 61 461 461 3461 8461 122 123 LNAAAA XSOAAA VVVVxx 6107 9956 1 3 7 7 7 107 107 1107 6107 14 15 XAAAAA YSOAAA AAAAxx 1967 9957 1 3 7 7 67 967 1967 1967 1967 134 135 RXAAAA ZSOAAA HHHHxx 8910 9958 0 2 0 10 10 910 910 3910 8910 20 21 SEAAAA ATOAAA OOOOxx 8257 9959 1 1 7 17 57 257 257 3257 8257 114 115 PFAAAA BTOAAA VVVVxx 851 9960 1 3 1 11 51 851 851 851 851 102 103 TGAAAA CTOAAA AAAAxx 7823 9961 1 3 3 3 23 823 1823 2823 7823 46 47 XOAAAA DTOAAA HHHHxx 3208 9962 0 0 8 8 8 208 1208 3208 3208 16 17 KTAAAA ETOAAA OOOOxx 856 9963 0 0 6 16 56 856 856 856 856 112 113 YGAAAA FTOAAA VVVVxx 2654 9964 0 2 4 14 54 654 654 2654 2654 108 109 CYAAAA GTOAAA AAAAxx 7185 9965 1 1 5 5 85 185 1185 2185 7185 170 171 JQAAAA HTOAAA HHHHxx 309 9966 1 1 9 9 9 309 309 309 309 18 19 XLAAAA ITOAAA OOOOxx 9752 9967 0 0 2 12 52 752 1752 4752 9752 104 105 CLAAAA JTOAAA VVVVxx 6405 9968 1 1 5 5 5 405 405 1405 6405 10 11 JMAAAA KTOAAA AAAAxx 6113 9969 1 1 3 13 13 113 113 1113 6113 26 27 DBAAAA LTOAAA HHHHxx 9774 9970 0 2 4 14 74 774 1774 4774 9774 148 149 YLAAAA MTOAAA OOOOxx 1674 9971 0 2 4 14 74 674 1674 1674 1674 148 149 KMAAAA NTOAAA VVVVxx 9602 9972 0 2 2 2 2 602 1602 4602 9602 4 5 IFAAAA OTOAAA AAAAxx 1363 9973 1 3 3 3 63 363 1363 1363 1363 126 127 LAAAAA PTOAAA HHHHxx 6887 9974 1 3 7 7 87 887 887 1887 6887 174 175 XEAAAA QTOAAA OOOOxx 6170 9975 0 2 0 10 70 170 170 1170 6170 140 141 IDAAAA RTOAAA VVVVxx 8888 9976 0 0 8 8 88 888 888 3888 8888 176 177 WDAAAA STOAAA AAAAxx 2981 9977 1 1 1 1 81 981 981 2981 2981 162 163 RKAAAA TTOAAA HHHHxx 7369 9978 1 1 9 9 69 369 1369 2369 7369 138 139 LXAAAA UTOAAA OOOOxx 6227 9979 1 3 7 7 27 227 227 1227 6227 54 55 NFAAAA VTOAAA VVVVxx 8002 9980 0 2 2 2 2 2 2 3002 8002 4 5 UVAAAA WTOAAA AAAAxx 4288 9981 0 0 8 8 88 288 288 4288 4288 176 177 YIAAAA XTOAAA HHHHxx 5136 9982 0 0 6 16 36 136 1136 136 5136 72 73 OPAAAA YTOAAA OOOOxx 1084 9983 0 0 4 4 84 84 1084 1084 1084 168 169 SPAAAA ZTOAAA VVVVxx 9117 9984 1 1 7 17 17 117 1117 4117 9117 34 35 RMAAAA AUOAAA AAAAxx 2406 9985 0 2 6 6 6 406 406 2406 2406 12 13 OOAAAA BUOAAA HHHHxx 1384 9986 0 0 4 4 84 384 1384 1384 1384 168 169 GBAAAA CUOAAA OOOOxx 9194 9987 0 2 4 14 94 194 1194 4194 9194 188 189 QPAAAA DUOAAA VVVVxx 858 9988 0 2 8 18 58 858 858 858 858 116 117 AHAAAA EUOAAA AAAAxx 8592 9989 0 0 2 12 92 592 592 3592 8592 184 185 MSAAAA FUOAAA HHHHxx 4773 9990 1 1 3 13 73 773 773 4773 4773 146 147 PBAAAA GUOAAA OOOOxx 4093 9991 1 1 3 13 93 93 93 4093 4093 186 187 LBAAAA HUOAAA VVVVxx 6587 9992 1 3 7 7 87 587 587 1587 6587 174 175 JTAAAA IUOAAA AAAAxx 6093 9993 1 1 3 13 93 93 93 1093 6093 186 187 JAAAAA JUOAAA HHHHxx 429 9994 1 1 9 9 29 429 429 429 429 58 59 NQAAAA KUOAAA OOOOxx 5780 9995 0 0 0 0 80 780 1780 780 5780 160 161 IOAAAA LUOAAA VVVVxx 1783 9996 1 3 3 3 83 783 1783 1783 1783 166 167 PQAAAA MUOAAA AAAAxx 2992 9997 0 0 2 12 92 992 992 2992 2992 184 185 CLAAAA NUOAAA HHHHxx 0 9998 0 0 0 0 0 0 0 0 0 0 1 AAAAAA OUOAAA OOOOxx 2968 9999 0 0 8 8 68 968 968 2968 2968 136 137 EKAAAA PUOAAA VVVVxx ================================================ FILE: test/sql/ddl.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA IF NOT EXISTS "customSchema" AUTHORIZATION :ROLE_DEFAULT_PERM_USER; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER \ir include/ddl_ops_1.sql SELECT * FROM PUBLIC."Hypertable_1"; SELECT * FROM ONLY PUBLIC."Hypertable_1"; EXPLAIN (buffers off, costs off) SELECT * FROM ONLY PUBLIC."Hypertable_1"; SELECT * FROM test.show_columns('PUBLIC."Hypertable_1"'); SELECT * FROM test.show_columns('_timescaledb_internal._hyper_1_1_chunk'); \ir include/ddl_ops_2.sql SELECT * FROM test.show_columns('PUBLIC."Hypertable_1"'); SELECT * FROM test.show_columns('_timescaledb_internal._hyper_1_1_chunk'); SELECT * FROM PUBLIC."Hypertable_1"; -- alter column tests CREATE TABLE alter_test(time timestamptz, temp float, color varchar(10)); -- create hypertable with two chunks SELECT create_hypertable('alter_test', 'time', 'color', 2, chunk_time_interval => 2628000000000); INSERT INTO alter_test VALUES ('2017-01-20T09:00:01', 17.5, 'blue'), ('2017-01-21T09:00:01', 19.1, 'yellow'), ('2017-04-20T09:00:01', 89.5, 'green'), ('2017-04-21T09:00:01', 17.1, 'black'); SELECT * FROM test.show_columns('alter_test'); SELECT * FROM test.show_columnsp('_timescaledb_internal._hyper_9_%chunk'); -- show the column name and type of the partitioning dimension in the -- metadata table SELECT * FROM _timescaledb_catalog.dimension WHERE hypertable_id = 9; EXPLAIN (buffers off, costs off) SELECT * FROM alter_test WHERE time > '2017-05-20T10:00:01'; -- rename column and change its type ALTER TABLE alter_test RENAME COLUMN time TO time_us; --converting timestamptz->timestamp should happen under UTC SET timezone = 'UTC'; ALTER TABLE alter_test ALTER COLUMN time_us TYPE timestamp; RESET timezone; ALTER TABLE alter_test RENAME COLUMN color TO colorname; \set ON_ERROR_STOP 0 -- Changing types on hash-partitioned columns is not safe for some -- types and is therefore blocked. ALTER TABLE alter_test ALTER COLUMN colorname TYPE text; \set ON_ERROR_STOP 1 SELECT * FROM test.show_columns('alter_test'); SELECT * FROM test.show_columnsp('_timescaledb_internal._hyper_9_%chunk'); -- show that the metadata has been updated SELECT * FROM _timescaledb_catalog.dimension WHERE hypertable_id = 9; -- constraint exclusion should still work with updated column EXPLAIN (buffers off, costs off) SELECT * FROM alter_test WHERE time_us > '2017-05-20T10:00:01'; \set ON_ERROR_STOP 0 -- verify that we cannot change the column type to something incompatible ALTER TABLE alter_test ALTER COLUMN colorname TYPE varchar(3); -- conversion that messes up partitioning fails ALTER TABLE alter_test ALTER COLUMN time_us TYPE timestamptz USING time_us::timestamptz+INTERVAL '1 year'; -- dropping column that messes up partiitoning fails ALTER TABLE alter_test DROP COLUMN colorname; --ONLY blocked ALTER TABLE ONLY alter_test RENAME COLUMN colorname TO colorname2; ALTER TABLE ONLY alter_test ALTER COLUMN colorname TYPE varchar(10); \set ON_ERROR_STOP 1 CREATE TABLE alter_test_bigint(time bigint, temp float); SELECT create_hypertable('alter_test_bigint', 'time', chunk_time_interval => 2628000000000); \set ON_ERROR_STOP 0 -- Changing type of time dimension to a non-supported type -- shall not be allowed ALTER TABLE alter_test_bigint ALTER COLUMN time TYPE TEXT; -- dropping open time dimension shall not be allowed. ALTER TABLE alter_test_bigint DROP COLUMN time; \set ON_ERROR_STOP 1 -- test expression index creation where physical layout of chunks differs from hypertable CREATE TABLE i2504(time timestamp NOT NULL, a int, b int, c int, d int); select create_hypertable('i2504', 'time'); INSERT INTO i2504 VALUES (now(), 1, 2, 3, 4); ALTER TABLE i2504 DROP COLUMN b; INSERT INTO i2504(time, a, c, d) VALUES (now() - interval '1 year', 1, 2, 3), (now() - interval '2 years', 1, 2, 3); CREATE INDEX idx2 ON i2504(a,d) WHERE c IS NOT NULL; DROP INDEX idx2; CREATE INDEX idx2 ON i2504(a,d) WITH (timescaledb.transaction_per_chunk) WHERE c IS NOT NULL; -- Make sure custom composite types are supported as dimensions CREATE TYPE TUPLE as (val1 int4, val2 int4); CREATE TABLE part_custom_dim (time TIMESTAMPTZ, combo TUPLE, device TEXT); \set ON_ERROR_STOP 0 -- should fail on PG < 14 because no partitioning function supplied and the given custom type -- has no default hash function -- on PG14 custom types are hashable SELECT create_hypertable('part_custom_dim', 'time', 'combo', 4); \set ON_ERROR_STOP 1 -- immutable functions with sub-transaction (issue #4489) CREATE FUNCTION i4489(value TEXT DEFAULT '') RETURNS INTEGER AS $$ BEGIN RETURN value::INTEGER; EXCEPTION WHEN invalid_text_representation THEN RETURN 0; END; $$ LANGUAGE PLPGSQL IMMUTABLE; -- should return 1 (one) in both cases SELECT i4489('1'), i4489('1'); -- should return 0 (zero) in all cases handled by the exception SELECT i4489(), i4489(); SELECT i4489('a'), i4489('a'); -- test ALTER TABLE ONLY for hypertables CREATE TABLE at_test(time timestamptz) WITH (tsdb.hypertable); -- adding column only on the parent table should be blocked \set ON_ERROR_STOP 0 ALTER TABLE ONLY at_test ADD COLUMN value INT; \set ON_ERROR_STOP 1 ALTER TABLE ONLY at_test SET (autovacuum_enabled = false); ALTER TABLE ONLY at_test RESET (autovacuum_enabled); -- test again after creating some chunks INSERT INTO at_test VALUES ('2025-01-01'); INSERT INTO at_test VALUES ('2025-02-01'); ALTER TABLE ONLY at_test SET (autovacuum_enabled = false); ALTER TABLE ONLY at_test RESET (autovacuum_enabled); -- test DDL inside function CREATE OR REPLACE FUNCTION ddl_function() RETURNS VOID LANGUAGE PLPGSQL AS $$ BEGIN DROP TABLE IF EXISTS func_table; CREATE TABLE func_table(time timestamptz) WITH (tsdb.hypertable); END $$; SELECT ddl_function(); SELECT hypertable_name from timescaledb_information.hypertables WHERE hypertable_name='func_table'; SELECT ddl_function(); SELECT hypertable_name from timescaledb_information.hypertables WHERE hypertable_name='func_table'; ================================================ FILE: test/sql/ddl_errors.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."Hypertable_1" ( time BIGINT NOT NULL, "Device_id" TEXT NOT NULL, temp_c int NOT NULL DEFAULT -1 ); CREATE INDEX ON PUBLIC."Hypertable_1" (time, "Device_id"); -- Default integer interval is supported as part of -- hypertable generalization, verify additional secnarios \set ON_ERROR_STOP 0 SELECT * FROM create_hypertable(NULL, NULL); SELECT * FROM create_hypertable('"public"."Hypertable_1"', NULL); -- space dimensions require explicit number of partitions SELECT * FROM create_hypertable('"public"."Hypertable_1"', 'time', 'Device_id', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); SELECT * FROM create_hypertable('"public"."Hypertable_1_mispelled"', 'time', 'Device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); SELECT * FROM create_hypertable('"public"."Hypertable_1"', 'time_mispelled', 'Device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); SELECT * FROM create_hypertable('"public"."Hypertable_1"', 'Device_id', 'Device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); SELECT * FROM create_hypertable('"public"."Hypertable_1"', 'time', 'Device_id_mispelled', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); INSERT INTO PUBLIC."Hypertable_1" VALUES(1,'dev_1', 3); SELECT * FROM create_hypertable('"public"."Hypertable_1"', 'time', 'Device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); DELETE FROM PUBLIC."Hypertable_1" ; \set ON_ERROR_STOP 1 SELECT * FROM create_hypertable('"public"."Hypertable_1"', 'time', 'Device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); \set ON_ERROR_STOP 0 SELECT * FROM create_hypertable('"public"."Hypertable_1"', 'time', 'Device_id', 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); \set ON_ERROR_STOP 1 INSERT INTO "Hypertable_1" VALUES (0, 1, 0); \set ON_ERROR_STOP 0 ALTER TABLE _timescaledb_internal._hyper_1_1_chunk ALTER COLUMN temp_c DROP NOT NULL; \set ON_ERROR_STOP 1 CREATE TABLE PUBLIC."Parent" ( time BIGINT NOT NULL, "Device_id" TEXT NOT NULL, temp_c int NOT NULL DEFAULT -1 ); \set ON_ERROR_STOP 0 ALTER TABLE "Hypertable_1" INHERIT "Parent"; ALTER TABLE _timescaledb_internal._hyper_1_1_chunk INHERIT "Parent"; ALTER TABLE _timescaledb_internal._hyper_1_1_chunk NO INHERIT "Parent"; \set ON_ERROR_STOP 1 CREATE TABLE PUBLIC."Child" () INHERITS ("Parent"); \set ON_ERROR_STOP 0 SELECT * FROM create_hypertable('"public"."Parent"', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); SELECT * FROM create_hypertable('"public"."Child"', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); \set ON_ERROR_STOP 1 \set ON_ERROR_STOP 0 CREATE TEMPORARY TABLE temp_table (time timestamptz) WITH (tsdb.hypertable); \set ON_ERROR_STOP 1 CREATE TEMP TABLE "Hypertable_temp" ( time BIGINT NOT NULL, "Device_id" TEXT NOT NULL, temp_c int NOT NULL DEFAULT -1 ); \set ON_ERROR_STOP 0 SELECT * FROM create_hypertable('"Hypertable_temp"', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); ALTER TABLE "Hypertable_1" SET UNLOGGED; \set ON_ERROR_STOP 1 ALTER TABLE "Hypertable_1" SET LOGGED; CREATE TABLE PUBLIC."Hypertable_1_rule" ( time BIGINT NOT NULL, "Device_id" TEXT NOT NULL, temp_c int NOT NULL DEFAULT -1 ); CREATE RULE notify_me AS ON UPDATE TO "Hypertable_1_rule" DO ALSO NOTIFY "Hypertable_1_rule"; \set ON_ERROR_STOP 0 SELECT * FROM create_hypertable('"public"."Hypertable_1_rule"', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); \set ON_ERROR_STOP 1 ALTER TABLE "Hypertable_1_rule" DISABLE RULE notify_me; \set ON_ERROR_STOP 0 SELECT * FROM create_hypertable('"public"."Hypertable_1_rule"', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); \set ON_ERROR_STOP 1 DROP RULE notify_me ON "Hypertable_1_rule"; SELECT * FROM create_hypertable('"public"."Hypertable_1_rule"', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); \set ON_ERROR_STOP 0 CREATE RULE notify_me AS ON UPDATE TO "Hypertable_1_rule" DO ALSO NOTIFY "Hypertable_1_rule"; \set ON_ERROR_STOP 1 \set ON_ERROR_STOP 0 SELECT add_dimension(NULL,NULL); \set ON_ERROR_STOP 1 \set ON_ERROR_STOP 0 SELECT attach_tablespace(NULL,NULL); \set ON_ERROR_STOP 1 \set ON_ERROR_STOP 0 select set_number_partitions(NULL,NULL); \set ON_ERROR_STOP 1 ================================================ FILE: test/sql/ddl_extra.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE OR REPLACE FUNCTION show_columns_ext(rel regclass) RETURNS TABLE("Column" name, "Type" text, "NotNull" boolean, "Compression" text) LANGUAGE SQL STABLE AS $BODY$ SELECT a.attname, format_type(t.oid, t.typtypmod), a.attnotnull, (CASE WHEN a.attcompression = 'l' THEN 'lz4' WHEN a.attcompression = 'p' THEN 'pglz' ELSE '' END) FROM pg_attribute a, pg_type t WHERE a.attrelid = rel AND a.atttypid = t.oid AND a.attnum >= 0 ORDER BY a.attnum; $BODY$; CREATE TABLE conditions ( time TIMESTAMP NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL ); SELECT create_hypertable('conditions', 'time', chunk_time_interval := '1 day'::interval); INSERT INTO conditions SELECT generate_series('2021-10-10 00:00'::timestamp, '2021-10-11 00:00'::timestamp, '1 day'), 'POR', 55, 75; CREATE VIEW t AS SELECT 'conditions'::regclass AS r UNION ALL SELECT * FROM show_chunks('conditions'); SELECT * FROM t, LATERAL show_columns_ext(r) WHERE "Column" = 'location' ORDER BY 1, 2; ALTER TABLE conditions ALTER COLUMN location SET COMPRESSION pglz; SELECT * FROM t, LATERAL show_columns_ext(r) WHERE "Column" = 'location' ORDER BY 1, 2; INSERT INTO conditions VALUES ('2021-10-12 00:00'::timestamp, 'BRA', 66, 77); SELECT * FROM t, LATERAL show_columns_ext(r) WHERE "Column" = 'location' ORDER BY 1, 2; ALTER TABLE conditions ALTER COLUMN location SET COMPRESSION default; SELECT * FROM t, LATERAL show_columns_ext(r) WHERE "Column" = 'location' ORDER BY 1, 2; \set ON_ERROR_STOP 0 -- failing test because compression is not allowed in "non-TOASTable" datatypes ALTER TABLE conditions ALTER COLUMN temperature SET COMPRESSION pglz; SELECT * FROM t, LATERAL show_columns_ext(r) WHERE "Column" = 'temperature' ORDER BY 1, 2; ================================================ FILE: test/sql/debug_utils.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT _timescaledb_functions.extension_state(); RESET ROLE; DO $$ DECLARE module text; BEGIN SELECT probin INTO module FROM pg_proc WHERE proname = 'extension_state' AND pronamespace = '_timescaledb_functions'::regnamespace; EXECUTE format('CREATE FUNCTION extension_state() RETURNS TEXT AS ''%s'', ''ts_extension_get_state'' LANGUAGE C', module); END $$; DROP EXTENSION timescaledb; SELECT * FROM extension_state(); \c CREATE EXTENSION timescaledb; SELECT * FROM extension_state(); ================================================ FILE: test/sql/delete.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \o /dev/null \ir include/insert_two_partitions.sql \o SELECT * FROM "two_Partitions" ORDER BY "timeCustom", device_id, series_0, series_1; DELETE FROM "two_Partitions" WHERE series_0 = 1.5; DELETE FROM "two_Partitions" WHERE series_0 = 100; SELECT * FROM "two_Partitions" ORDER BY "timeCustom", device_id, series_0, series_1; -- Make sure DELETE isn't optimized if it includes Append plans -- Need to turn of nestloop to make append appear the same on PG96 and PG10 set enable_nestloop = 'off'; CREATE OR REPLACE FUNCTION series_val() RETURNS integer LANGUAGE PLPGSQL STABLE AS $BODY$ BEGIN RETURN 5; END; $BODY$; -- ConstraintAwareAppend applied for SELECT EXPLAIN (buffers off, costs off) SELECT FROM "two_Partitions" WHERE series_1 IN (SELECT series_1 FROM "two_Partitions" WHERE series_1 > series_val()); -- ConstraintAwareAppend NOT applied for DELETE EXPLAIN (buffers off, costs off) DELETE FROM "two_Partitions" WHERE series_1 IN (SELECT series_1 FROM "two_Partitions" WHERE series_1 > series_val()); SELECT * FROM "two_Partitions" ORDER BY "timeCustom", device_id, series_0, series_1; BEGIN; DELETE FROM "two_Partitions" WHERE series_1 IN (SELECT series_1 FROM "two_Partitions" WHERE series_1 > series_val()); SELECT * FROM "two_Partitions" ORDER BY "timeCustom", device_id, series_0, series_1; ROLLBACK; BEGIN; DELETE FROM "two_Partitions" WHERE series_1 IN (SELECT series_1 FROM "two_Partitions" WHERE series_1 > series_val()) RETURNING "timeCustom"; SELECT * FROM "two_Partitions" ORDER BY "timeCustom", device_id, series_0, series_1; ROLLBACK; -- test update on chunks directly CREATE TABLE direct_delete(time timestamptz) WITH (tsdb.hypertable); INSERT INTO direct_delete VALUES ('2020-01-01'); SELECT show_chunks('direct_delete') AS "CHUNK" \gset --should have ModifyHyperable node EXPLAIN (costs off, timing off, summary off) DELETE FROM :CHUNK; EXPLAIN (costs off, timing off, summary off) DELETE FROM ONLY :CHUNK; -- DELETE should succeed BEGIN; DELETE FROM :CHUNK RETURNING *; ROLLBACK; BEGIN; DELETE FROM ONLY :CHUNK RETURNING *; ROLLBACK; -- Test that EXPLAIN VERBOSE on prepared statements does not corrupt cached plans. SET plan_cache_mode = 'force_generic_plan'; CREATE TABLE explain_verbose_ht( time timestamptz NOT NULL, device int, value float) WITH (tsdb.hypertable); INSERT INTO explain_verbose_ht SELECT t, 1, 0.1 FROM generate_series('2026-01-01'::timestamptz, '2026-01-08'::timestamptz, interval '6 hours') t; -- Verify the DELETE plan uses ChunkAppend EXPLAIN (costs off) DELETE FROM explain_verbose_ht WHERE time > '2025-01-01'::text::timestamptz; PREPARE delete_ht AS DELETE FROM explain_verbose_ht WHERE time > '2025-01-01'::text::timestamptz AND device = 2; EXECUTE delete_ht; EXPLAIN (verbose, costs off) EXECUTE delete_ht; EXECUTE delete_ht; DEALLOCATE delete_ht; -- repeat test with explain analyze PREPARE delete_ht AS DELETE FROM explain_verbose_ht WHERE time > '2025-01-01'::text::timestamptz AND device = 2; EXECUTE delete_ht; EXPLAIN (verbose, analyze, buffers off, costs off, timing off, summary off) EXECUTE delete_ht; EXECUTE delete_ht; DEALLOCATE delete_ht; RESET plan_cache_mode; -- github issue #6790 -- test DELETE with WHERE EXISTS on hypertable CREATE TABLE i6790(time timestamptz NOT NULL, device int, value float) WITH (tsdb.hypertable); INSERT INTO i6790 SELECT t, 1, 0.1 FROM generate_series('2026-01-01'::timestamptz, '2026-01-03'::timestamptz, interval '12 hours') t; -- DELETE with simple EXISTS - creates gating Result node wrapping ChunkAppend DELETE FROM i6790 WHERE EXISTS (SELECT 1); -- all rows should be gone SELECT count(*) FROM i6790; -- repopulate for next test INSERT INTO i6790 SELECT t, 1, 0.1 FROM generate_series('2026-01-01'::timestamptz, '2026-01-03'::timestamptz, interval '12 hours') t; -- DELETE with correlated EXISTS DELETE FROM i6790 WHERE EXISTS (SELECT 1 FROM i6790 g WHERE g.device = i6790.device); SELECT count(*) FROM i6790; ================================================ FILE: test/sql/drop_extension.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE drop_test(time timestamp, temp float8, device text); SELECT create_hypertable('drop_test', 'time', 'device', 2); SELECT * FROM _timescaledb_catalog.hypertable; INSERT INTO drop_test VALUES('Mon Mar 20 09:17:00.936242 2017', 23.4, 'dev1'); SELECT * FROM drop_test; \c :TEST_DBNAME :ROLE_SUPERUSER DROP EXTENSION timescaledb CASCADE; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- Querying the original table should not return any rows since all of -- them actually existed in chunks that are now gone SELECT * FROM drop_test; \c :TEST_DBNAME :ROLE_SUPERUSER -- Recreate the extension SET client_min_messages=error; CREATE EXTENSION timescaledb; RESET client_min_messages; -- Test that calling twice generates proper error \set ON_ERROR_STOP 0 CREATE EXTENSION timescaledb; \set ON_ERROR_STOP 1 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- CREATE twice with IF NOT EXISTS should be OK CREATE EXTENSION IF NOT EXISTS timescaledb; -- Make the table a hypertable again SELECT create_hypertable('drop_test', 'time', 'device', 2); SELECT * FROM _timescaledb_catalog.hypertable; INSERT INTO drop_test VALUES('Mon Mar 20 09:18:19.100462 2017', 22.1, 'dev1'); SELECT * FROM drop_test; --test drops thru cascades of other objects \c :TEST_DBNAME :ROLE_SUPERUSER -- Stop background workers to prevent them from interfering with the drop public schema SELECT _timescaledb_functions.stop_background_workers(); SET client_min_messages TO ERROR; REVOKE CONNECT ON DATABASE :TEST_DBNAME FROM public; SELECT count(pg_terminate_backend(pg_stat_activity.pid)) AS TERMINATED FROM pg_stat_activity WHERE pg_stat_activity.datname = :'TEST_DBNAME' AND pg_stat_activity.pid <> pg_backend_pid() \gset RESET client_min_messages; -- drop the public schema and all its objects DROP SCHEMA public CASCADE; \dn -- Recreate the public schema and extension in the same session. -- This should work without requiring a reconnect (issue #5884). CREATE SCHEMA public; SET client_min_messages=error; CREATE EXTENSION timescaledb SCHEMA public; RESET client_min_messages; SELECT extname FROM pg_extension WHERE extname = 'timescaledb'; -- Verify the extension is functional after re-creation CREATE TABLE drop_test2(time timestamptz, temp float8); SELECT create_hypertable('drop_test2', 'time'); INSERT INTO drop_test2 VALUES('2024-01-01', 23.4); SELECT * FROM drop_test2; DROP TABLE drop_test2; -- Test that dropping and recreating extension directly also works in the same session DROP EXTENSION timescaledb CASCADE; SET client_min_messages=error; CREATE EXTENSION timescaledb; RESET client_min_messages; SELECT extname FROM pg_extension WHERE extname = 'timescaledb'; ================================================ FILE: test/sql/drop_hypertable.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SELECT * from _timescaledb_catalog.hypertable; SELECT * from _timescaledb_catalog.dimension; CREATE TABLE should_drop (time timestamp, temp float8); SELECT create_hypertable('should_drop', 'time'); CREATE TABLE hyper_with_dependencies (time timestamp, temp float8); SELECT create_hypertable('hyper_with_dependencies', 'time'); CREATE VIEW dependent_view AS SELECT * FROM hyper_with_dependencies; INSERT INTO hyper_with_dependencies VALUES (now(), 1.0); \set ON_ERROR_STOP 0 DROP TABLE hyper_with_dependencies; \set ON_ERROR_STOP 1 DROP TABLE hyper_with_dependencies CASCADE; -- check that the view is dropped SELECT oid FROM pg_class WHERE relname = 'dependent_view'; CREATE TABLE chunk_with_dependencies (time timestamp, temp float8); SELECT create_hypertable('chunk_with_dependencies', 'time'); INSERT INTO chunk_with_dependencies VALUES (now(), 1.0); CREATE VIEW dependent_view_chunk AS SELECT * FROM _timescaledb_internal._hyper_3_2_chunk; \set ON_ERROR_STOP 0 DROP TABLE chunk_with_dependencies; \set ON_ERROR_STOP 1 DROP TABLE chunk_with_dependencies CASCADE; -- check that the view is dropped SELECT oid FROM pg_class WHERE relname = 'dependent_view_chunk'; -- Calling create hypertable again will increment hypertable ID -- although no new hypertable is created. Make sure we can handle this. SELECT create_hypertable('should_drop', 'time', if_not_exists => true); SELECT * from _timescaledb_catalog.hypertable; SELECT * from _timescaledb_catalog.dimension; DROP TABLE should_drop; CREATE TABLE should_drop (time timestamp, temp float8); SELECT create_hypertable('should_drop', 'time'); INSERT INTO should_drop VALUES (now(), 1.0); SELECT * from _timescaledb_catalog.hypertable; SELECT * from _timescaledb_catalog.dimension; -- test dropping multiple objects at once CREATE TABLE t1 (time timestamptz) WITH (tsdb.hypertable); INSERT INTO t1 VALUES ('2025-01-01'); CREATE TABLE t2 (time timestamptz) WITH (tsdb.hypertable); INSERT INTO t2 VALUES ('2025-01-01'); CREATE TABLE t3 (time timestamptz); DROP TABLE t1, t2, t3; ================================================ FILE: test/sql/drop_owned.sql.in ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA hypertable_schema; GRANT ALL ON SCHEMA hypertable_schema TO :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE TABLE hypertable_schema.default_perm_user (time timestamptz, temp float, location int); SELECT create_hypertable('hypertable_schema.default_perm_user', 'time', 'location', 2); INSERT INTO hypertable_schema.default_perm_user VALUES ('2001-01-01 01:01:01', 23.3, 1); RESET ROLE; CREATE TABLE hypertable_schema.superuser (time timestamptz, temp float, location int); SELECT create_hypertable('hypertable_schema.superuser', 'time', 'location', 2); INSERT INTO hypertable_schema.superuser VALUES ('2001-01-01 01:01:01', 23.3, 1); SELECT * FROM _timescaledb_catalog.hypertable ORDER BY id; SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; DROP OWNED BY :ROLE_DEFAULT_PERM_USER; SELECT * FROM _timescaledb_catalog.hypertable ORDER BY id; SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; DROP TABLE hypertable_schema.superuser; --everything should be cleaned up SELECT * FROM _timescaledb_catalog.hypertable GROUP BY id; SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; SELECT * FROM _timescaledb_catalog.dimension; SELECT * FROM _timescaledb_catalog.dimension_slice; SELECT * FROM _timescaledb_catalog.chunk_constraint; -- test drop owned in database without extension installed \c :TEST_DBNAME :ROLE_SUPERUSER CREATE database test_drop_owned; \c test_drop_owned DROP OWNED BY :ROLE_SUPERUSER; \c :TEST_DBNAME :ROLE_SUPERUSER DROP DATABASE test_drop_owned WITH (FORCE); -- Test that dependencies on roles are added to chunks when creating -- new chunks. If that is not done, DROP OWNED BY will not revoke the -- privilege on the chunk. CREATE TABLE sensor_data(time timestamptz not null, cpu double precision null); SELECT * FROM create_hypertable('sensor_data','time'); INSERT INTO sensor_data SELECT time, random() AS cpu FROM generate_series('2020-01-01'::timestamptz, '2020-01-24'::timestamptz, INTERVAL '10 minute') AS g1(time); \dp sensor_data \dp _timescaledb_internal._hyper_3* GRANT SELECT ON sensor_data TO :ROLE_DEFAULT_PERM_USER; \dp sensor_data \dp _timescaledb_internal._hyper_3* -- Insert more chunks after adding the user to the hypertable. These -- will now get the privileges of the hypertable. INSERT INTO sensor_data SELECT time, random() AS cpu FROM generate_series('2020-01-20'::timestamptz, '2020-02-05'::timestamptz, INTERVAL '10 minute') AS g1(time); \dp _timescaledb_internal._hyper_3* -- This should revoke the privileges on both the hypertable and the chunks. DROP OWNED BY :ROLE_DEFAULT_PERM_USER; \dp sensor_data \dp _timescaledb_internal._hyper_3* ================================================ FILE: test/sql/drop_rename_hypertable.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \o /dev/null \ir include/insert_two_partitions.sql \o SELECT * FROM test.show_columnsp('_timescaledb_internal.%_hyper%'); -- Test that renaming hypertable works SELECT * FROM test.show_columns('_timescaledb_internal._hyper_1_1_chunk'); ALTER TABLE "two_Partitions" RENAME TO "newname"; SELECT * FROM "newname"; SELECT * FROM _timescaledb_catalog.hypertable; \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA "newschema" AUTHORIZATION :ROLE_DEFAULT_PERM_USER; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER ALTER TABLE "newname" SET SCHEMA "newschema"; SELECT * FROM "newschema"."newname"; SELECT * FROM _timescaledb_catalog.hypertable; DROP TABLE "newschema"."newname"; SELECT * FROM _timescaledb_catalog.hypertable; SELECT schema, name FROM test.relation WHERE schema IN ('public', '_timescaledb_catalog', '_timescaledb_internal'); -- Test that renaming ordinary table works CREATE TABLE renametable (foo int); ALTER TABLE "renametable" RENAME TO "newname_none_ht"; SELECT * FROM "newname_none_ht"; ================================================ FILE: test/sql/drop_schema.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA chunk_schema1; CREATE SCHEMA chunk_schema2; CREATE SCHEMA hypertable_schema; CREATE SCHEMA extra_schema; GRANT ALL ON SCHEMA hypertable_schema TO :ROLE_DEFAULT_PERM_USER; GRANT ALL ON SCHEMA chunk_schema1 TO :ROLE_DEFAULT_PERM_USER; GRANT ALL ON SCHEMA chunk_schema2 TO :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE TABLE hypertable_schema.test1 (time timestamptz, temp float, location int); CREATE TABLE hypertable_schema.test2 (time timestamptz, temp float, location int); --create two identical tables with their own chunk schemas SELECT create_hypertable('hypertable_schema.test1', 'time', 'location', 2, associated_schema_name => 'chunk_schema1'); SELECT create_hypertable('hypertable_schema.test2', 'time', 'location', 2, associated_schema_name => 'chunk_schema2'); INSERT INTO hypertable_schema.test1 VALUES ('2001-01-01 01:01:01', 23.3, 1); INSERT INTO hypertable_schema.test2 VALUES ('2001-01-01 01:01:01', 23.3, 1); SELECT * FROM _timescaledb_catalog.hypertable ORDER BY id; SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; RESET ROLE; --drop the associated schema. We drop the extra schema to show we can --handle multi-schema drops DROP SCHEMA chunk_schema1, extra_schema CASCADE; SET ROLE :ROLE_DEFAULT_PERM_USER; --show that the metadata for the table using the dropped schema is --changed. The other table is not affected. SELECT * FROM _timescaledb_catalog.hypertable ORDER BY id; SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; --new chunk should be created in the internal associated schema INSERT INTO hypertable_schema.test1 VALUES ('2001-01-01 01:01:01', 23.3, 1); SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; RESET ROLE; --dropping the internal schema should not work \set ON_ERROR_STOP 0 DROP SCHEMA _timescaledb_internal CASCADE; \set ON_ERROR_STOP 1 --dropping the hypertable schema should delete everything DROP SCHEMA hypertable_schema CASCADE; SET ROLE :ROLE_DEFAULT_PERM_USER; --everything should be cleaned up SELECT * FROM _timescaledb_catalog.hypertable GROUP BY id; SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; SELECT * FROM _timescaledb_catalog.dimension; SELECT * FROM _timescaledb_catalog.dimension_slice; SELECT * FROM _timescaledb_catalog.chunk_constraint; ================================================ FILE: test/sql/dump_meta.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \ir include/insert_two_partitions.sql \ir ../../scripts/dump_meta_data.sql ================================================ FILE: test/sql/extension_scripts.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER DROP EXTENSION timescaledb; -- test that installation script errors when any of our internal schemas already exists \set ON_ERROR_STOP 0 CREATE SCHEMA _timescaledb_catalog; CREATE EXTENSION timescaledb; DROP SCHEMA _timescaledb_catalog; \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA _timescaledb_internal; CREATE EXTENSION timescaledb; DROP SCHEMA _timescaledb_internal; \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA _timescaledb_cache; CREATE EXTENSION timescaledb; DROP SCHEMA _timescaledb_cache; \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA timescaledb_experimental; CREATE EXTENSION timescaledb; DROP SCHEMA timescaledb_experimental; \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA timescaledb_information; CREATE EXTENSION timescaledb; DROP SCHEMA timescaledb_information; -- test that installation script errors when any of the function in public schema already exists -- we don't test every public function but just a few common ones \c :TEST_DBNAME :ROLE_SUPERUSER CREATE FUNCTION time_bucket(int,int) RETURNS int LANGUAGE SQL AS $$ SELECT 1::int; $$; CREATE EXTENSION timescaledb; DROP FUNCTION time_bucket; \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION show_chunks(relation regclass, older_than "any" DEFAULT NULL, newer_than "any" DEFAULT NULL, created_before "any" DEFAULT NULL, created_after "any" DEFAULT NULL) RETURNS SETOF regclass language internal as 'pg_partition_ancestors'; CREATE EXTENSION timescaledb; DROP FUNCTION show_chunks; -- Create a user that is not all-lowercase CREATE USER "FooBar" WITH SUPERUSER; \c :TEST_DBNAME "FooBar" SET client_min_messages TO error; CREATE EXTENSION timescaledb; DROP EXTENSION timescaledb; RESET client_min_messages; \c :TEST_DBNAME :ROLE_SUPERUSER DROP USER "FooBar"; SET client_min_messages TO ERROR; CREATE EXTENSION timescaledb; ================================================ FILE: test/sql/generated_as_identity.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE table test_gen ( id int generated by default AS IDENTITY primary key, payload text ); SELECT create_hypertable('test_gen', 'id', chunk_time_interval=>10); insert into test_gen (payload) select generate_series(1,15) returning *; select * from test_gen; \set ON_ERROR_STOP 0 insert into test_gen values('1', 'a'); \set ON_ERROR_STOP 1 ALTER TABLE test_gen ALTER COLUMN id DROP IDENTITY; \set ON_ERROR_STOP 0 insert into test_gen (payload) select generate_series(15,20) returning *; \set ON_ERROR_STOP 1 ALTER TABLE test_gen ALTER COLUMN id ADD GENERATED BY DEFAULT AS IDENTITY; \set ON_ERROR_STOP 0 insert into test_gen (payload) select generate_series(15,20) returning *; \set ON_ERROR_STOP 1 ALTER TABLE test_gen ALTER COLUMN id SET GENERATED BY DEFAULT RESTART 100; insert into test_gen (payload) select generate_series(15,20) returning *; select * from test_gen; SELECT * FROM test.show_subtables('test_gen'); ================================================ FILE: test/sql/grant_hypertable.sql.in ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE TABLE conditions( time TIMESTAMPTZ NOT NULL, device INTEGER, temperature FLOAT ); -- Create a hypertable and show that it does not have any privileges SELECT * FROM create_hypertable('conditions', 'time', chunk_time_interval => '5 days'::interval); INSERT INTO conditions SELECT time, (random()*30)::int, random()*80 - 40 FROM generate_series('2018-12-01 00:00'::timestamp, '2018-12-10 00:00'::timestamp, '1h') AS time; \z conditions \z _timescaledb_internal.*chunk -- Add privileges and show that they propagate to the chunks GRANT SELECT, INSERT ON conditions TO PUBLIC; \z conditions \z _timescaledb_internal.*chunk -- Create some more chunks and show that they also get the privileges. INSERT INTO conditions SELECT time, (random()*30)::int, random()*80 - 40 FROM generate_series('2018-12-10 00:00'::timestamp, '2018-12-20 00:00'::timestamp, '1h') AS time; \z conditions \z _timescaledb_internal.*chunk -- Revoke one of the privileges and show that it propagate to the -- chunks. REVOKE INSERT ON conditions FROM PUBLIC; \z conditions \z _timescaledb_internal.*chunk -- Add some more chunks and show that it inherits the grants from the -- hypertable. INSERT INTO conditions SELECT time, (random()*30)::int, random()*80 - 40 FROM generate_series('2018-12-20 00:00'::timestamp, '2018-12-30 00:00'::timestamp, '1h') AS time; \z conditions \z _timescaledb_internal.*chunk -- Change grants of one chunk explicitly and check that it is possible \z _timescaledb_internal._hyper_1_1_chunk GRANT UPDATE ON _timescaledb_internal._hyper_1_1_chunk TO PUBLIC; \z _timescaledb_internal._hyper_1_1_chunk REVOKE SELECT ON _timescaledb_internal._hyper_1_1_chunk FROM PUBLIC; \z _timescaledb_internal._hyper_1_1_chunk -- Check that revoking a permission first on the chunk and then on the -- hypertable that was added through the hypertable (INSERT and -- SELECT, in this case) still do not copy permissions from the -- hypertable (so there should not be a select permission to public on -- the chunk but there should be one on the hypertable). GRANT INSERT ON conditions TO PUBLIC; \z conditions \z _timescaledb_internal._hyper_1_2_chunk REVOKE SELECT ON _timescaledb_internal._hyper_1_2_chunk FROM PUBLIC; REVOKE INSERT ON conditions FROM PUBLIC; \z conditions \z _timescaledb_internal._hyper_1_2_chunk -- Check that granting permissions through hypertable does not remove -- separate grants on chunk. GRANT UPDATE ON _timescaledb_internal._hyper_1_3_chunk TO PUBLIC; \z conditions \z _timescaledb_internal._hyper_1_3_chunk GRANT INSERT ON conditions TO PUBLIC; REVOKE INSERT ON conditions FROM PUBLIC; \z conditions \z _timescaledb_internal._hyper_1_3_chunk -- Check that GRANT ALL IN SCHEMA adds privileges to the parent -- and also goes to chunks in another schema GRANT ALL ON ALL TABLES IN SCHEMA public TO :ROLE_DEFAULT_PERM_USER_2; \z conditions \z _timescaledb_internal.*chunk -- Check that REVOKE ALL IN SCHEMA removes privileges of the parent -- and also goes to chunks in another schema REVOKE ALL ON ALL TABLES IN SCHEMA public FROM :ROLE_DEFAULT_PERM_USER_2; \z conditions \z _timescaledb_internal.*chunk -- Create chunks in the same schema as the hypertable and check that -- they also get the same privileges as the hypertable CREATE TABLE measurements( time TIMESTAMPTZ NOT NULL, device INTEGER, temperature FLOAT ); -- Create a hypertable with chunks in the same schema SELECT * FROM create_hypertable('public.measurements', 'time', chunk_time_interval => '5 days'::interval, associated_schema_name => 'public'); INSERT INTO measurements SELECT time, (random()*30)::int, random()*80 - 40 FROM generate_series('2018-12-01 00:00'::timestamp, '2018-12-10 00:00'::timestamp, '1h') AS time; -- GRANT ALL and check privileges GRANT ALL ON ALL TABLES IN SCHEMA public TO :ROLE_DEFAULT_PERM_USER_2; \z measurements \z conditions \z public.*chunk -- REVOKE ALL and check privileges REVOKE ALL ON ALL TABLES IN SCHEMA public FROM :ROLE_DEFAULT_PERM_USER_2; \z measurements \z conditions \z public.*chunk -- GRANT/REVOKE in an empty schema (Issue #4581) CREATE SCHEMA test_grant; GRANT ALL ON ALL TABLES IN SCHEMA test_grant TO :ROLE_DEFAULT_PERM_USER_2; REVOKE ALL ON ALL TABLES IN SCHEMA test_grant FROM :ROLE_DEFAULT_PERM_USER_2; ================================================ FILE: test/sql/hash.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test hashing Const values. We should expect the same hash value for -- all integer types when values are compatible SELECT _timescaledb_functions.get_partition_hash(1::int); SELECT _timescaledb_functions.get_partition_hash(1::bigint); SELECT _timescaledb_functions.get_partition_hash(1::smallint); SELECT _timescaledb_functions.get_partition_hash(true); -- Floating point types should also hash the same for compatible values SELECT _timescaledb_functions.get_partition_hash(1.0::real); SELECT _timescaledb_functions.get_partition_hash(1.0::double precision); -- Float aliases SELECT _timescaledb_functions.get_partition_hash(1.0::float); SELECT _timescaledb_functions.get_partition_hash(1.0::float4); SELECT _timescaledb_functions.get_partition_hash(1.0::float8); SELECT _timescaledb_functions.get_partition_hash(1.0::numeric); -- 'name' and '"char"' are internal PostgreSQL types, which are not -- intended for use by the general user. They are included here only -- for completeness -- https://www.postgresql.org/docs/10/static/datatype-character.html#datatype-character-special-table SELECT _timescaledb_functions.get_partition_hash('c'::name); SELECT _timescaledb_functions.get_partition_hash('c'::"char"); -- String and character hashes should also have the same output for -- compatible values SELECT _timescaledb_functions.get_partition_hash('c'::char); SELECT _timescaledb_functions.get_partition_hash('c'::varchar(2)); SELECT _timescaledb_functions.get_partition_hash('c'::text); -- 'c' is 0x63 in ASCII SELECT _timescaledb_functions.get_partition_hash(E'\\x63'::bytea); -- Time and date types SELECT _timescaledb_functions.get_partition_hash(interval '1 day'); SELECT _timescaledb_functions.get_partition_hash('2017-03-22T09:18:23'::timestamp); SELECT _timescaledb_functions.get_partition_hash('2017-03-22T09:18:23'::timestamptz); SELECT _timescaledb_functions.get_partition_hash('2017-03-22'::date); SELECT _timescaledb_functions.get_partition_hash('10:00:00'::time); SELECT _timescaledb_functions.get_partition_hash('10:00:00-1'::timetz); -- Other types SELECT _timescaledb_functions.get_partition_hash(ARRAY[1,2,3]); SELECT _timescaledb_functions.get_partition_hash('08002b:010203'::macaddr); SELECT _timescaledb_functions.get_partition_hash('192.168.100.128/25'::cidr); SELECT _timescaledb_functions.get_partition_hash('192.168.100.128'::inet); SELECT _timescaledb_functions.get_partition_hash('2001:4f8:3:ba:2e0:81ff:fe22:d1f1'::inet); SELECT _timescaledb_functions.get_partition_hash('2001:4f8:3:ba:2e0:81ff:fe22:d1f1/128'::cidr); SELECT _timescaledb_functions.get_partition_hash('{ "foo": "bar" }'::jsonb); SELECT _timescaledb_functions.get_partition_hash('4b6a5eec-b344-11e7-abc4-cec278b6b50a'::uuid); SELECT _timescaledb_functions.get_partition_hash(1::regclass); SELECT _timescaledb_functions.get_partition_hash(int4range(10, 20)); SELECT _timescaledb_functions.get_partition_hash(int8range(10, 20)); SELECT _timescaledb_functions.get_partition_hash(numrange(10, 20)); SELECT _timescaledb_functions.get_partition_hash(tsrange('2017-03-22T09:18:23', '2017-03-23T09:18:23')); SELECT _timescaledb_functions.get_partition_hash(tstzrange('2017-03-22T09:18:23+01', '2017-03-23T09:18:23+00')); -- Test hashing Var values CREATE TABLE hash_test(id int, value text); INSERT INTO hash_test VALUES (1, 'test'); -- Test Vars SELECT _timescaledb_functions.get_partition_hash(id) FROM hash_test; SELECT _timescaledb_functions.get_partition_hash(value) FROM hash_test; -- Test coerced value SELECT _timescaledb_functions.get_partition_hash(id::text) FROM hash_test; -- Test legacy function that converts values to text first SELECT _timescaledb_functions.get_partition_for_key('4b6a5eec-b344-11e7-abc4-cec278b6b50a'::text); SELECT _timescaledb_functions.get_partition_for_key('4b6a5eec-b344-11e7-abc4-cec278b6b50a'::varchar); SELECT _timescaledb_functions.get_partition_for_key(187); SELECT _timescaledb_functions.get_partition_for_key(187::bigint); SELECT _timescaledb_functions.get_partition_for_key(187::numeric); SELECT _timescaledb_functions.get_partition_for_key(187::double precision); SELECT _timescaledb_functions.get_partition_for_key(int4range(10, 20)); SELECT _timescaledb_functions.get_partition_hash('08002b:010203'::macaddr); -- Test inside IMMUTABLE function (Issue #4575) CREATE FUNCTION my_get_partition_hash(INTEGER) RETURNS INTEGER AS 'SELECT _timescaledb_functions.get_partition_hash($1);' LANGUAGE SQL IMMUTABLE; CREATE FUNCTION my_get_partition_for_key(INTEGER) RETURNS INTEGER AS 'SELECT _timescaledb_functions.get_partition_for_key($1);' LANGUAGE SQL IMMUTABLE; SELECT my_get_partition_hash(1); SELECT my_get_partition_for_key(1); ================================================ FILE: test/sql/histogram_test.sql.in ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- table 1 CREATE TABLE "hitest1"(key real, val varchar(40)); -- insertions INSERT INTO "hitest1" VALUES(0, 'hi'); INSERT INTO "hitest1" VALUES(1, 'sup'); INSERT INTO "hitest1" VALUES(2, 'hello'); INSERT INTO "hitest1" VALUES(3, 'yo'); INSERT INTO "hitest1" VALUES(4, 'howdy'); INSERT INTO "hitest1" VALUES(5, 'hola'); INSERT INTO "hitest1" VALUES(6, 'ya'); INSERT INTO "hitest1" VALUES(1, 'sup'); INSERT INTO "hitest1" VALUES(2, 'hello'); INSERT INTO "hitest1" VALUES(1, 'sup'); -- table 2 CREATE TABLE "hitest2"(name varchar(30), score integer, qualify boolean); -- insertions INSERT INTO "hitest2" VALUES('Tom', 6, TRUE); INSERT INTO "hitest2" VALUES('Mary', 4, FALSE); INSERT INTO "hitest2" VALUES('Jaq', 3, FALSE); INSERT INTO "hitest2" VALUES('Jane', 10, TRUE); -- standard 2 bucket SELECT histogram(key, 0, 9, 2) FROM hitest1; -- standard multi-bucket SELECT histogram(key, 0, 9, 5) FROM hitest1; -- standard 3 bucket SELECT val, histogram(key, 0, 7, 3) FROM hitest1 GROUP BY val ORDER BY val; -- standard element beneath lb SELECT histogram(key, 1, 7, 3) FROM hitest1; -- standard element above ub SELECT histogram(key, 0, 3, 3) FROM hitest1; -- standard element beneath and above lb and ub, respectively SELECT histogram(key, 1, 3, 2) FROM hitest1; -- standard 1 bucket SELECT histogram(key, 1, 3, 1) FROM hitest1; -- standard 2 bucket SELECT qualify, histogram(score, 0, 10, 2) FROM hitest2 GROUP BY qualify ORDER BY qualify; -- standard multi-bucket SELECT qualify, histogram(score, 0, 10, 5) FROM hitest2 GROUP BY qualify ORDER BY qualify; -- check number of buckets is constant \set ON_ERROR_STOP 0 select histogram(i,10,90,case when i=1 then 1 else 1000000 end) FROM generate_series(1,100) i; \set ON_ERROR_STOP 1 CREATE TABLE weather ( time TIMESTAMPTZ NOT NULL, city TEXT, temperature FLOAT, PRIMARY KEY(time, city) ); -- There is a bug in width_bucket() causing a NaN as a result, so we -- check that it is not causing a crash in histogram(). SELECT * FROM create_hypertable('weather', 'time', 'city', 3); INSERT INTO weather VALUES ('2023-02-10 09:16:51.133584+00','city1',10.4), ('2023-02-10 11:16:51.611618+00','city1',10.3), ('2023-02-10 06:58:59.999999+00','city1',10.3), ('2023-02-10 01:58:59.999999+00','city1',10.3), ('2023-02-09 01:58:59.999999+00','city1',10.3), ('2023-02-10 08:58:59.999999+00','city1',10.3), ('2023-03-23 06:12:02.73765+00 ','city1', 9.7), ('2023-03-23 06:12:06.990998+00','city1',11.7); -- This will currently generate an error on PG15 and prior versions \set ON_ERROR_STOP 0 SELECT histogram(temperature, -1.79769e+308, 1.79769e+308,10) FROM weather GROUP BY city; \set ON_ERROR_STOP 1 ================================================ FILE: test/sql/include/agg_bookends_load.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE btest(time timestamp NOT NULL, time_alt timestamp, gp INTEGER, temp float, strid TEXT DEFAULT 'testing'); SELECT schema_name, table_name, created FROM create_hypertable('btest', 'time'); INSERT INTO btest VALUES('2017-01-20T09:00:01', '2017-01-20T10:00:00', 1, 22.5); INSERT INTO btest VALUES('2017-01-20T09:00:21', '2017-01-20T09:00:59', 1, 21.2); INSERT INTO btest VALUES('2017-01-20T09:00:47', '2017-01-20T09:00:58', 1, 25.1); INSERT INTO btest VALUES('2017-01-20T09:00:02', '2017-01-20T09:00:57', 2, 35.5); INSERT INTO btest VALUES('2017-01-20T09:00:21', '2017-01-20T09:00:56', 2, 30.2); --TOASTED; INSERT INTO btest VALUES('2017-01-20T09:00:43', '2017-01-20T09:01:55', 2, 20.1, repeat('xyz', 1000000) ); CREATE TABLE btest_numeric (time timestamp NOT NULL, quantity numeric); SELECT schema_name, table_name, created FROM create_hypertable('btest_numeric', 'time'); ================================================ FILE: test/sql/include/agg_bookends_query.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be only output of results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations')) v(setting); :PREFIX SELECT time, gp, temp FROM btest ORDER BY time; :PREFIX SELECT last(temp, time) FROM btest; :PREFIX SELECT first(temp, time) FROM btest; :PREFIX SELECT last(temp, time_alt) FROM btest; :PREFIX SELECT first(temp, time_alt) FROM btest; :PREFIX SELECT gp, last(temp, time) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, first(temp, time) FROM btest GROUP BY gp ORDER BY gp; --check whole row :PREFIX SELECT gp, first(btest, time) FROM btest GROUP BY gp ORDER BY gp; --check toasted col :PREFIX SELECT gp, left(last(strid, time), 10) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, last(temp, strid) FROM btest GROUP BY gp ORDER BY gp; :PREFIX SELECT gp, last(strid, temp) FROM btest GROUP BY gp ORDER BY gp; BEGIN; --check null value as last element INSERT INTO btest VALUES('2018-01-20T09:00:43', '2017-01-20T09:00:55', 2, NULL); :PREFIX SELECT last(temp, time) FROM btest; --check non-null element "overrides" NULL because it comes after. INSERT INTO btest VALUES('2019-01-20T09:00:43', '2018-01-20T09:00:55', 2, 30.5); :PREFIX SELECT last(temp, time) FROM btest; --check null cmp element is skipped INSERT INTO btest VALUES('2018-01-20T09:00:43', NULL, 2, 32.3); :PREFIX SELECT last(temp, time_alt) FROM btest; -- fist returns NULL value :PREFIX SELECT first(temp, time_alt) FROM btest; -- test first return non NULL value INSERT INTO btest VALUES('2016-01-20T09:00:00', '2016-01-20T09:00:00', 2, 36.5); :PREFIX SELECT first(temp, time_alt) FROM btest; --check non null cmp element insert after null cmp INSERT INTO btest VALUES('2020-01-20T09:00:43', '2020-01-20T09:00:43', 2, 35.3); :PREFIX SELECT last(temp, time_alt) FROM btest; :PREFIX SELECT first(temp, time_alt) FROM btest; --cmp nulls should be ignored and not present in groups :PREFIX SELECT gp, last(temp, time_alt) FROM btest GROUP BY gp ORDER BY gp; --Previously, some bugs were found with NULLS and numeric types, so test that INSERT INTO btest_numeric VALUES ('2019-01-20T09:00:43', NULL); :PREFIX SELECT last(quantity, time) FROM btest_numeric; --check non-null element "overrides" NULL because it comes after. INSERT INTO btest_numeric VALUES('2020-01-20T09:00:43', 30.5); :PREFIX SELECT last(quantity, time) FROM btest_numeric; -- do index scan for last :PREFIX SELECT last(temp, time) FROM btest; -- do index scan for first :PREFIX SELECT first(temp, time) FROM btest; -- can't do index scan when ordering on non-index column :PREFIX SELECT first(temp, time_alt) FROM btest; -- do index scan for subquery :PREFIX SELECT * FROM (SELECT last(temp, time) FROM btest) last; -- can't do index scan when using group by :PREFIX SELECT last(temp, time) FROM btest GROUP BY gp ORDER BY gp; -- do index scan when agg function is used in CTE subquery :PREFIX WITH last_temp AS (SELECT last(temp, time) FROM btest) SELECT * from last_temp; -- do index scan when using both FIRST and LAST aggregate functions :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; -- verify results when using both FIRST and LAST :PREFIX SELECT first(temp, time), last(temp, time) FROM btest; -- do index scan when using WHERE :PREFIX SELECT last(temp, time) FROM btest WHERE time <= '2017-01-20T09:00:02'; -- can't do index scan for MAX and LAST combined (MinMax optimization fails when having different aggregate functions) :PREFIX SELECT max(time), last(temp, time) FROM btest; -- can't do index scan when using FIRST/LAST in ORDER BY :PREFIX SELECT last(temp, time) FROM btest ORDER BY last(temp, time); -- do index scan :PREFIX SELECT last(temp, time) FROM btest WHERE temp < 30; -- SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- do index scan :PREFIX SELECT first(temp, time) FROM btest WHERE time >= '2017-01-20 09:00:47'; -- can't do index scan when using WINDOW function :PREFIX SELECT gp, last(temp, time) OVER (PARTITION BY gp) AS last FROM btest; -- test constants :PREFIX SELECT first(100, 100) FROM btest; -- create an index so we can test optimization CREATE INDEX btest_time_alt_idx ON btest(time_alt); :PREFIX SELECT last(temp, time_alt) FROM btest; --test nested FIRST/LAST - should optimize :PREFIX SELECT abs(last(temp, time)) FROM btest; -- test nested FIRST/LAST in ORDER BY - no optimization possible :PREFIX SELECT abs(last(temp, time)) FROM btest ORDER BY abs(last(temp,time)); ROLLBACK; -- Test with NULL numeric values BEGIN; TRUNCATE btest_numeric; -- Empty table :PREFIX SELECT first(btest_numeric, time) FROM btest_numeric; :PREFIX SELECT last(btest_numeric, time) FROM btest_numeric; -- Only NULL values INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; -- NULL values followed by non-NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; TRUNCATE btest_numeric; -- non-NULL values followed by NULL values INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 1); INSERT INTO btest_numeric VALUES('2019-01-20T09:00:43', 2); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); INSERT INTO btest_numeric VALUES('2018-01-20T09:00:43', NULL); :PREFIX SELECT first(quantity, time) FROM btest_numeric; :PREFIX SELECT last(quantity, time) FROM btest_numeric; :PREFIX SELECT first(time, quantity) FROM btest_numeric; :PREFIX SELECT last(time, quantity) FROM btest_numeric; ROLLBACK; ================================================ FILE: test/sql/include/append_load.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- create a now() function for repeatable testing that always returns -- the same timestamp. It needs to be marked STABLE CREATE OR REPLACE FUNCTION now_s() RETURNS timestamptz LANGUAGE PLPGSQL STABLE PARALLEL SAFE AS $BODY$ BEGIN RAISE NOTICE 'Stable function now_s() called!'; RETURN '2017-08-22T10:00:00'::timestamptz; END; $BODY$; CREATE OR REPLACE FUNCTION now_i() RETURNS timestamptz LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RAISE NOTICE 'Immutable function now_i() called!'; RETURN '2017-08-22T10:00:00'::timestamptz; END; $BODY$; CREATE OR REPLACE FUNCTION now_v() RETURNS timestamptz LANGUAGE PLPGSQL VOLATILE AS $BODY$ BEGIN RAISE NOTICE 'Volatile function now_v() called!'; RETURN '2017-08-22T10:00:00'::timestamptz; END; $BODY$; CREATE OR REPLACE PROCEDURE force_parallel(on_or_off bool) LANGUAGE PLPGSQL AS $$ BEGIN IF current_setting('server_version_num')::int < 160000 THEN IF on_or_off THEN set force_parallel_mode = 'on'; ELSE set force_parallel_mode = 'off'; END IF; ELSE IF on_or_off THEN set debug_parallel_query = 'on'; ELSE set debug_parallel_query = 'off'; END IF; END IF; END; $$; CREATE TABLE append_test(time timestamptz, temp float, colorid integer, attr jsonb); SELECT create_hypertable('append_test', 'time', chunk_time_interval => 2628000000000); -- create three chunks INSERT INTO append_test VALUES ('2017-03-22T09:18:22', 23.5, 1, '{"a": 1, "b": 2}'), ('2017-03-22T09:18:23', 21.5, 1, '{"a": 1, "b": 2}'), ('2017-05-22T09:18:22', 36.2, 2, '{"c": 3, "b": 2}'), ('2017-05-22T09:18:23', 15.2, 2, '{"c": 3}'), ('2017-08-22T09:18:22', 34.1, 3, '{"c": 4}'); VACUUM (ANALYZE) append_test; -- Create another hypertable to join with CREATE TABLE join_test(time timestamptz, temp float, colorid integer); SELECT create_hypertable('join_test', 'time', chunk_time_interval => 2628000000000); INSERT INTO join_test VALUES ('2017-01-22T09:18:22', 15.2, 1), ('2017-02-22T09:18:22', 24.5, 2), ('2017-08-22T09:18:22', 23.1, 3); VACUUM (ANALYZE) join_test; -- Create another table to join with which is not a hypertable. CREATE TABLE join_test_plain(time timestamptz, temp float, colorid integer, attr jsonb); INSERT INTO join_test_plain VALUES ('2017-01-22T09:18:22', 15.2, 1, '{"a": 1}'), ('2017-02-22T09:18:22', 24.5, 2, '{"b": 2}'), ('2017-08-22T09:18:22', 23.1, 3, '{"c": 3}'); VACUUM (ANALYZE) join_test_plain; -- create hypertable with DATE time dimension CREATE TABLE metrics_date(time DATE NOT NULL); SELECT create_hypertable('metrics_date','time'); INSERT INTO metrics_date SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval); VACUUM (ANALYZE) metrics_date; -- create hypertable with TIMESTAMP time dimension CREATE TABLE metrics_timestamp(time TIMESTAMP NOT NULL); SELECT create_hypertable('metrics_timestamp','time'); INSERT INTO metrics_timestamp SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval); VACUUM (ANALYZE) metrics_timestamp; -- create hypertable with TIMESTAMPTZ time dimension CREATE TABLE metrics_timestamptz(time TIMESTAMPTZ NOT NULL, device_id INT NOT NULL); CREATE INDEX ON metrics_timestamptz(device_id,time); SELECT create_hypertable('metrics_timestamptz','time'); INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval), 1; INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval), 2; INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::date, '2000-02-01'::date, '5m'::interval), 3; VACUUM (ANALYZE) metrics_timestamptz; -- create space partitioned hypertable CREATE TABLE metrics_space(time timestamptz NOT NULL, device_id int NOT NULL, v1 float, v2 float, v3 text); SELECT create_hypertable('metrics_space','time','device_id',3); INSERT INTO metrics_space SELECT time, device_id, device_id + 0.25, device_id + 0.75, device_id FROM generate_series('2000-01-01'::timestamptz, '2000-01-14'::timestamptz, '5m'::interval) g1(time), generate_series(1,10,1) g2(device_id) ORDER BY time, device_id; VACUUM (ANALYZE) metrics_space; -- test ChunkAppend projection #2661 CREATE TABLE i2661 ( machine_id int4 NOT NULL, "name" varchar(255) NOT NULL, "timestamp" timestamptz NOT NULL, "first" float4 NULL ); SELECT create_hypertable('i2661', 'timestamp'); INSERT INTO i2661 SELECT 1, 'speed', generate_series('2019-12-31 00:00:00', '2020-01-10 00:00:00', '2m'::interval), 0; VACUUM (ANALYZE) i2661; ================================================ FILE: test/sql/include/append_query.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- canary for results diff -- this should be the only output of the results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations'),('timescaledb.enable_chunk_append')) v(setting); -- query should exclude all chunks with optimization on :PREFIX SELECT * FROM append_test WHERE time > now_s() + '1 month' ORDER BY time DESC; --query should exclude all chunks and be a MergeAppend :PREFIX SELECT * FROM append_test WHERE time > now_s() + '1 month' ORDER BY time DESC limit 1; -- when optimized, the plan should be a constraint-aware append and -- cover only one chunk. It should be a backward index scan due to -- descending index on time. Should also skip the main table, since it -- cannot hold tuples :PREFIX SELECT * FROM append_test WHERE time > now_s() - interval '2 months'; -- adding ORDER BY and LIMIT should turn the plan into an optimized -- ordered append plan :PREFIX SELECT * FROM append_test WHERE time > now_s() - interval '2 months' ORDER BY time LIMIT 3; -- no optimized plan for queries with restrictions that can be -- constified at planning time. Regular planning-time constraint -- exclusion should occur. :PREFIX SELECT * FROM append_test WHERE time > now_i() - interval '2 months' ORDER BY time; -- currently, we cannot distinguish between stable and volatile -- functions as far as applying our modified plan. However, volatile -- function should not be pre-evaluated to constants, so no chunk -- exclusion should occur. :PREFIX SELECT * FROM append_test WHERE time > now_v() - interval '2 months' ORDER BY time; -- prepared statement output should be the same regardless of -- optimizations PREPARE query_opt AS SELECT * FROM append_test WHERE time > now_s() - interval '2 months' ORDER BY time; :PREFIX EXECUTE query_opt; DEALLOCATE query_opt; -- aggregates should produce same output :PREFIX SELECT date_trunc('year', time) t, avg(temp) FROM append_test WHERE time > now_s() - interval '4 months' GROUP BY t ORDER BY t DESC; -- querying outside the time range should return nothing. This tests -- that ConstraintAwareAppend can handle the case when an Append node -- is turned into a Result node due to no children :PREFIX SELECT date_trunc('year', time) t, avg(temp) FROM append_test WHERE time < '2016-03-22' AND date_part('dow', time) between 1 and 5 GROUP BY t ORDER BY t DESC; -- a parameterized query can safely constify params, so won't be -- optimized by constraint-aware append since regular constraint -- exclusion works just fine PREPARE query_param AS SELECT * FROM append_test WHERE time > $1 ORDER BY time; :PREFIX EXECUTE query_param(now_s() - interval '2 months'); DEALLOCATE query_param; --test with cte :PREFIX WITH data AS ( SELECT time_bucket(INTERVAL '30 day', TIME) AS btime, AVG(temp) AS VALUE FROM append_test WHERE TIME > now_s() - INTERVAL '400 day' AND colorid > 0 GROUP BY btime ), period AS ( SELECT time_bucket(INTERVAL '30 day', TIME) AS btime FROM GENERATE_SERIES('2017-03-22T01:01:01', '2017-08-23T01:01:01', INTERVAL '30 day') TIME ) SELECT period.btime, VALUE FROM period LEFT JOIN DATA USING (btime) ORDER BY period.btime; WITH data AS ( SELECT time_bucket(INTERVAL '30 day', TIME) AS btime, AVG(temp) AS VALUE FROM append_test WHERE TIME > now_s() - INTERVAL '400 day' AND colorid > 0 GROUP BY btime ), period AS ( SELECT time_bucket(INTERVAL '30 day', TIME) AS btime FROM GENERATE_SERIES('2017-03-22T01:01:01', '2017-08-23T01:01:01', INTERVAL '30 day') TIME ) SELECT period.btime, VALUE FROM period LEFT JOIN DATA USING (btime) ORDER BY period.btime; -- force nested loop join with no materialization. This tests that the -- inner ConstraintAwareScan supports resetting its scan for every -- iteration of the outer relation loop set enable_hashjoin = 'off'; set enable_mergejoin = 'off'; set enable_material = 'off'; :PREFIX SELECT * FROM append_test a INNER JOIN join_test j ON (a.colorid = j.colorid) WHERE a.time > now_s() - interval '3 hours' AND j.time > now_s() - interval '3 hours'; reset enable_hashjoin; reset enable_mergejoin; reset enable_material; -- test constraint_exclusion with date time dimension and DATE/TIMESTAMP/TIMESTAMPTZ constraints -- the queries should all have 3 chunks :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::date ORDER BY time; :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::timestamp ORDER BY time; :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::timestamptz ORDER BY time; -- test Const OP Var -- the queries should all have 3 chunks :PREFIX SELECT * FROM metrics_date WHERE '2000-01-15'::date < time ORDER BY time; :PREFIX SELECT * FROM metrics_date WHERE '2000-01-15'::timestamp < time ORDER BY time; :PREFIX SELECT * FROM metrics_date WHERE '2000-01-15'::timestamptz < time ORDER BY time; -- test 2 constraints -- the queries should all have 2 chunks :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::date AND time < '2000-01-21'::date ORDER BY time; :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::timestamp AND time < '2000-01-21'::timestamp ORDER BY time; :PREFIX SELECT * FROM metrics_date WHERE time > '2000-01-15'::timestamptz AND time < '2000-01-21'::timestamptz ORDER BY time; -- test constraint_exclusion with timestamp time dimension and DATE/TIMESTAMP/TIMESTAMPTZ constraints -- the queries should all have 3 chunks :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::date ORDER BY time; :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::timestamp ORDER BY time; :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::timestamptz ORDER BY time; -- test Const OP Var -- the queries should all have 3 chunks :PREFIX SELECT * FROM metrics_timestamp WHERE '2000-01-15'::date < time ORDER BY time; :PREFIX SELECT * FROM metrics_timestamp WHERE '2000-01-15'::timestamp < time ORDER BY time; :PREFIX SELECT * FROM metrics_timestamp WHERE '2000-01-15'::timestamptz < time ORDER BY time; -- test 2 constraints -- the queries should all have 2 chunks :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::date AND time < '2000-01-21'::date ORDER BY time; :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::timestamp AND time < '2000-01-21'::timestamp ORDER BY time; :PREFIX SELECT * FROM metrics_timestamp WHERE time > '2000-01-15'::timestamptz AND time < '2000-01-21'::timestamptz ORDER BY time; -- test constraint_exclusion with timestamptz time dimension and DATE/TIMESTAMP/TIMESTAMPTZ constraints -- the queries should all have 3 chunks :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::date ORDER BY time; :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::timestamp ORDER BY time; :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::timestamptz ORDER BY time; -- test Const OP Var -- the queries should all have 3 chunks :PREFIX SELECT time FROM metrics_timestamptz WHERE '2000-01-15'::date < time ORDER BY time; :PREFIX SELECT time FROM metrics_timestamptz WHERE '2000-01-15'::timestamp < time ORDER BY time; :PREFIX SELECT time FROM metrics_timestamptz WHERE '2000-01-15'::timestamptz < time ORDER BY time; -- test 2 constraints -- the queries should all have 2 chunks :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::date AND time < '2000-01-21'::date ORDER BY time; :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::timestamp AND time < '2000-01-21'::timestamp ORDER BY time; :PREFIX SELECT time FROM metrics_timestamptz WHERE time > '2000-01-15'::timestamptz AND time < '2000-01-21'::timestamptz ORDER BY time; -- test constraint_exclusion with space partitioning and DATE/TIMESTAMP/TIMESTAMPTZ constraints -- exclusion for constraints with non-matching datatypes not working for space partitioning atm :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::date ORDER BY time; :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamp ORDER BY time; :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz ORDER BY time; -- test Const OP Var -- exclusion for constraints with non-matching datatypes not working for space partitioning atm :PREFIX SELECT time FROM metrics_space WHERE '2000-01-10'::date < time ORDER BY time; :PREFIX SELECT time FROM metrics_space WHERE '2000-01-10'::timestamp < time ORDER BY time; :PREFIX SELECT time FROM metrics_space WHERE '2000-01-10'::timestamptz < time ORDER BY time; -- test 2 constraints -- exclusion for constraints with non-matching datatypes not working for space partitioning atm :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::date AND time < '2000-01-15'::date ORDER BY time; :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamp AND time < '2000-01-15'::timestamp ORDER BY time; :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND time < '2000-01-15'::timestamptz ORDER BY time; -- test filtering on space partition :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND device_id = 1 ORDER BY time; :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND device_id IN (1,2) ORDER BY time; :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND device_id IN (VALUES(1)) ORDER BY time; :PREFIX SELECT time FROM metrics_space WHERE time > '2000-01-10'::timestamptz AND v3 IN (VALUES('1')) ORDER BY time; :PREFIX SELECT * FROM metrics_space WHERE time = (VALUES ('2019-12-24' at time zone 'UTC')) AND v3 NOT IN (VALUES ('1')); -- test CURRENT_DATE -- should be 0 chunks :PREFIX SELECT time FROM metrics_date WHERE time > CURRENT_DATE ORDER BY time; :PREFIX SELECT time FROM metrics_timestamp WHERE time > CURRENT_DATE ORDER BY time; :PREFIX SELECT time FROM metrics_timestamptz WHERE time > CURRENT_DATE ORDER BY time; :PREFIX SELECT time FROM metrics_space WHERE time > CURRENT_DATE ORDER BY time; -- test CURRENT_TIMESTAMP -- should be 0 chunks :PREFIX SELECT time FROM metrics_date WHERE time > CURRENT_TIMESTAMP ORDER BY time; :PREFIX SELECT time FROM metrics_timestamp WHERE time > CURRENT_TIMESTAMP ORDER BY time; :PREFIX SELECT time FROM metrics_timestamptz WHERE time > CURRENT_TIMESTAMP ORDER BY time; :PREFIX SELECT time FROM metrics_space WHERE time > CURRENT_TIMESTAMP ORDER BY time; -- test now() -- should be 0 chunks :PREFIX SELECT time FROM metrics_date WHERE time > now() ORDER BY time; :PREFIX SELECT time FROM metrics_timestamp WHERE time > now() ORDER BY time; :PREFIX SELECT time FROM metrics_timestamptz WHERE time > now() ORDER BY time; :PREFIX SELECT time FROM metrics_space WHERE time > now() ORDER BY time; -- query with tablesample and planner exclusion :PREFIX SELECT * FROM metrics_date TABLESAMPLE BERNOULLI(5) REPEATABLE(0) WHERE time > '2000-01-15' ORDER BY time DESC; -- query with tablesample and startup exclusion :PREFIX SELECT * FROM metrics_date TABLESAMPLE BERNOULLI(5) REPEATABLE(0) WHERE time > '2000-01-15'::text::date ORDER BY time DESC; -- query with tablesample, space partitioning and planner exclusion :PREFIX SELECT * FROM metrics_space TABLESAMPLE BERNOULLI(5) REPEATABLE(0) WHERE time > '2000-01-10'::timestamptz ORDER BY time DESC, device_id; -- test runtime exclusion -- test runtime exclusion with LATERAL and 2 hypertables :PREFIX SELECT m1.time, m2.time FROM metrics_timestamptz m1 LEFT JOIN LATERAL(SELECT time FROM metrics_timestamptz m2 WHERE m1.time = m2.time LIMIT 1) m2 ON true ORDER BY m1.time; -- test runtime exclusion and startup exclusions :PREFIX SELECT m1.time, m2.time FROM metrics_timestamptz m1 LEFT JOIN LATERAL(SELECT time FROM metrics_timestamptz m2 WHERE m1.time = m2.time AND m2.time < '2000-01-10'::text::timestamptz LIMIT 1) m2 ON true ORDER BY m1.time; -- test runtime exclusion does not activate for constraints on non-partitioning columns -- should not use runtime exclusion :PREFIX SELECT * FROM append_test a LEFT JOIN LATERAL(SELECT * FROM join_test j WHERE a.colorid = j.colorid ORDER BY time DESC LIMIT 1) j ON true ORDER BY a.time LIMIT 1; -- test runtime exclusion with LATERAL and generate_series :PREFIX SELECT g.time FROM generate_series('2000-01-01'::timestamptz, '2000-02-01'::timestamptz, '1d'::interval) g(time) LEFT JOIN LATERAL(SELECT time FROM metrics_timestamptz m WHERE m.time=g.time LIMIT 1) m ON true; :PREFIX SELECT * FROM generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval) AS g(time) INNER JOIN LATERAL (SELECT time FROM metrics_timestamptz m WHERE time=g.time) m ON true; :PREFIX SELECT * FROM generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval) AS g(time) INNER JOIN LATERAL (SELECT time FROM metrics_timestamptz m WHERE time=g.time ORDER BY time) m ON true; :PREFIX SELECT * FROM generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval) AS g(time) INNER JOIN LATERAL (SELECT time FROM metrics_timestamptz m WHERE time>g.time + '1 day' ORDER BY time LIMIT 1) m ON true; -- test runtime exclusion with subquery :PREFIX SELECT m1.time FROM metrics_timestamptz m1 WHERE m1.time=(SELECT max(time) FROM metrics_timestamptz); -- test runtime exclusion with correlated subquery :PREFIX SELECT m1.time, (SELECT m2.time FROM metrics_timestamptz m2 WHERE m2.time < m1.time ORDER BY m2.time DESC LIMIT 1) FROM metrics_timestamptz m1 WHERE m1.time < '2000-01-10' ORDER BY m1.time; -- test EXISTS :PREFIX SELECT m1.time FROM metrics_timestamptz m1 WHERE EXISTS(SELECT 1 FROM metrics_timestamptz m2 WHERE m1.time < m2.time) ORDER BY m1.time DESC limit 1000; -- test constraint exclusion for subqueries with append -- should include 2 chunks :PREFIX SELECT time FROM (SELECT time FROM metrics_timestamptz WHERE time < '2000-01-10'::text::timestamptz ORDER BY time) m; -- test constraint exclusion for subqueries with mergeappend -- should include 2 chunks :PREFIX SELECT device_id, time FROM (SELECT device_id, time FROM metrics_timestamptz WHERE time < '2000-01-10'::text::timestamptz ORDER BY device_id, time) m; -- test LIMIT pushdown -- no aggregates/window functions/SRF should pushdown limit :PREFIX SELECT FROM metrics_timestamptz ORDER BY time LIMIT 1; -- aggregates should prevent pushdown :PREFIX SELECT count(*) FROM metrics_timestamptz LIMIT 1; :PREFIX SELECT count(*) FROM metrics_space LIMIT 1; -- HAVING should prevent pushdown :PREFIX SELECT 1 FROM metrics_timestamptz HAVING count(*) > 1 LIMIT 1; :PREFIX SELECT 1 FROM metrics_space HAVING count(*) > 1 LIMIT 1; -- DISTINCT should prevent pushdown SET enable_hashagg TO false; :PREFIX SELECT DISTINCT device_id FROM metrics_timestamptz ORDER BY device_id LIMIT 3; :PREFIX SELECT DISTINCT device_id FROM metrics_space ORDER BY device_id LIMIT 3; RESET enable_hashagg; -- JOINs should prevent pushdown -- when LIMIT gets pushed to a Sort node it will switch to top-N heapsort -- if more tuples then LIMIT are requested this will trigger an error -- to trigger this we need a Sort node that is below ChunkAppend CREATE TABLE join_limit (time timestamptz, device_id int); SELECT table_name FROM create_hypertable('join_limit','time',create_default_indexes:=false); CREATE INDEX ON join_limit(time,device_id); INSERT INTO join_limit SELECT time, device_id FROM generate_series('2000-01-01'::timestamptz,'2000-01-21','30m') g1(time), generate_series(1,10,1) g2(device_id) ORDER BY time, device_id; VACUUM (ANALYZE) join_limit; -- get 2nd chunk oid SELECT tableoid AS "CHUNK_OID" FROM join_limit WHERE time > '2000-01-07' ORDER BY time LIMIT 1 \gset --get index name for 2nd chunk SELECT indexrelid::regclass AS "INDEX_NAME" FROM pg_index WHERE indrelid = :CHUNK_OID \gset DROP INDEX :INDEX_NAME; :PREFIX SELECT * FROM metrics_timestamptz m1 INNER JOIN join_limit m2 ON m1.time = m2.time AND m1.device_id=m2.device_id WHERE m1.time > '2000-01-07' ORDER BY m1.time, m1.device_id LIMIT 3; DROP TABLE join_limit; -- test ChunkAppend projection #2661 :PREFIX SELECT ts.timestamp, ht.timestamp FROM ( SELECT generate_series( to_timestamp(FLOOR(EXTRACT (EPOCH FROM '2020-01-01T00:01:00Z'::timestamp) / 300) * 300) AT TIME ZONE 'UTC', '2020-01-01T01:00:00Z', '5 minutes'::interval ) AS timestamp ) ts LEFT JOIN i2661 ht ON (FLOOR(EXTRACT (EPOCH FROM ht."timestamp") / 300) * 300 = EXTRACT (EPOCH FROM ts.timestamp)) AND ht.timestamp > '2019-12-30T00:00:00Z'::timestamp ORDER BY ts.timestamp, ht.timestamp; -- #3030 test chunkappend keeps pathkeys when subpath is append -- on PG11 this will not use ChunkAppend but MergeAppend SET enable_seqscan TO FALSE; CREATE TABLE i3030(time timestamptz NOT NULL, a int, b int); SELECT table_name FROM create_hypertable('i3030', 'time', create_default_indexes=>false); CREATE INDEX ON i3030(a,time); INSERT INTO i3030 (time,a) SELECT time, a FROM generate_series('2000-01-01'::timestamptz,'2000-01-01 3:00:00'::timestamptz,'1min'::interval) time, generate_series(1,30) a; VACUUM (ANALYZE) i3030; :PREFIX SELECT * FROM i3030 where time BETWEEN '2000-01-01'::text::timestamptz AND '2000-01-03'::text::timestamptz ORDER BY a,time LIMIT 1; DROP TABLE i3030; RESET enable_seqscan; --parent runtime exclusion tests: --optimization works with ANY (array) :PREFIX SELECT * FROM append_test a WHERE a.attr @> ANY((SELECT coalesce(array_agg(attr), array[]::jsonb[]) FROM join_test_plain WHERE temp > 100)::jsonb[]); --optimization does not work for ANY subquery (does not force an initplan) :PREFIX SELECT * FROM append_test a WHERE a.attr @> ANY((SELECT attr FROM join_test_plain WHERE temp > 100)); --works on any strict operator without ANY :PREFIX SELECT * FROM append_test a WHERE a.attr @> (SELECT attr FROM join_test_plain WHERE temp > 100 limit 1); --optimization works with function calls CREATE OR REPLACE FUNCTION select_tag(_min_temp int) RETURNS jsonb[] LANGUAGE sql STABLE PARALLEL SAFE AS $function$ SELECT coalesce(array_agg(attr), array[]::jsonb[]) FROM join_test_plain WHERE temp > _min_temp $function$; :PREFIX SELECT * FROM append_test a WHERE a.attr @> ANY((SELECT select_tag(100))::jsonb[]); --optimization does not work when result is null :PREFIX SELECT * FROM append_test a WHERE a.attr @> ANY((SELECT array_agg(attr) FROM join_test_plain WHERE temp > 100)::jsonb[]); -- Test that ConstraintAwareAppend properly locks relations in -- parallel query mode set timescaledb.enable_chunk_append=false; call force_parallel(true); :PREFIX select time, avg(temp), colorid from append_test where time > now_s() - interval '3 months 20 days' group by time, colorid; reset timescaledb.enable_chunk_append; reset max_parallel_workers_per_gather; call force_parallel(false); ================================================ FILE: test/sql/include/bgw_launcher_utils.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Note on testing: need a couple wrappers that pg_sleep in a loop to wait for changes -- to appear in pg_stat_activity. -- Further Note: PG 9.6 changed what appeared in pg_stat_activity, so the launcher doesn't actually show up. -- we can still test its interactions with its children, but can't test some of the things specific to the launcher. -- So we've added some bits about the version number as needed. CREATE VIEW worker_counts as SELECT count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Launcher') as launcher, count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME') as single_scheduler, count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = :'TEST_DBNAME_2') as single_2_scheduler, count(*) filter (WHERE backend_type = 'TimescaleDB Background Worker Scheduler' AND datname = 'template1') as template1_scheduler FROM pg_stat_activity; CREATE FUNCTION wait_worker_counts(launcher_ct INTEGER, scheduler1_ct INTEGER, scheduler2_ct INTEGER, template1_ct INTEGER) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE r INTEGER; BEGIN FOR i in 1..10 LOOP SELECT COUNT(*) from worker_counts where launcher = launcher_ct AND single_scheduler = scheduler1_ct AND single_2_scheduler = scheduler2_ct and template1_scheduler = template1_ct into r; if(r < 1) THEN PERFORM pg_sleep(0.1); PERFORM pg_stat_clear_snapshot(); ELSE --We have the correct counts! RETURN TRUE; END IF; END LOOP; RETURN FALSE; END $BODY$; CREATE FUNCTION wait_for_bgw_scheduler(_datname NAME, _count INT DEFAULT 1, _ticks INT DEFAULT 10) RETURNS TEXT LANGUAGE PLPGSQL AS $BODY$ DECLARE r INTEGER; BEGIN FOR i in 1.._ticks LOOP SELECT count(*) FROM pg_stat_activity WHERE application_name = 'TimescaleDB Background Worker Scheduler' AND datname = _datname INTO r; IF(r <> _count) THEN PERFORM pg_sleep(0.1); PERFORM pg_stat_clear_snapshot(); ELSE RETURN 'BGW Scheduler found.'; END IF; END LOOP; RETURN 'BGW Scheduler NOT found.'; END $BODY$; CREATE PROCEDURE kill_database_backends(_datname NAME) LANGUAGE PLPGSQL AS $BODY$ DECLARE r INTEGER; BEGIN FOR i in 1..100 LOOP SELECT count(pg_terminate_backend(pg_stat_activity.pid)) FROM pg_stat_activity WHERE datname = _datname AND pg_stat_activity.pid <> pg_backend_pid() INTO r; IF(r = 0) THEN RETURN; END IF; PERFORM pg_sleep(0.1); PERFORM pg_stat_clear_snapshot(); END LOOP; RAISE 'Failed to terminate backends'; END $BODY$; ================================================ FILE: test/sql/include/bgw_launcher_utils_cleanup.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. DROP FUNCTION wait_worker_counts(integer, integer, integer, integer); DROP VIEW worker_counts; DROP FUNCTION wait_for_bgw_scheduler(name,int,int); DROP PROCEDURE kill_database_backends(name); ================================================ FILE: test/sql/include/ddl_ops_1.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."Hypertable_1" ( time BIGINT NOT NULL, "Device_id" TEXT NOT NULL, temp_c int NOT NULL DEFAULT -1, humidity numeric NULL DEFAULT 0, sensor_1 NUMERIC NULL DEFAULT 1, sensor_2 NUMERIC NOT NULL DEFAULT 1, sensor_3 NUMERIC NOT NULL DEFAULT 1, sensor_4 NUMERIC NOT NULL DEFAULT 1 ); CREATE INDEX ON PUBLIC."Hypertable_1" (time, "Device_id"); CREATE TABLE "customSchema"."Hypertable_1" ( time BIGINT NOT NULL, "Device_id" TEXT NOT NULL, temp_c int NOT NULL DEFAULT -1, humidity numeric NULL DEFAULT 0, sensor_1 NUMERIC NULL DEFAULT 1, sensor_2 NUMERIC NOT NULL DEFAULT 1, sensor_3 NUMERIC NOT NULL DEFAULT 1, sensor_4 NUMERIC NOT NULL DEFAULT 1 ); CREATE INDEX ON "customSchema"."Hypertable_1" (time, "Device_id"); SELECT * FROM create_hypertable('"public"."Hypertable_1"', 'time', 'Device_id', 1, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); SELECT * FROM create_hypertable('"customSchema"."Hypertable_1"', 'time', NULL, 1, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); SELECT * FROM _timescaledb_catalog.hypertable; CREATE INDEX ON PUBLIC."Hypertable_1" (time, "temp_c"); CREATE INDEX "ind_humidity" ON PUBLIC."Hypertable_1" (time, "humidity"); CREATE INDEX "ind_sensor_1" ON PUBLIC."Hypertable_1" (time, "sensor_1"); INSERT INTO PUBLIC."Hypertable_1"(time, "Device_id", temp_c, humidity, sensor_1, sensor_2, sensor_3, sensor_4) VALUES(1257894000000000000, 'dev1', 30, 70, 1, 2, 3, 100); CREATE UNIQUE INDEX "Unique1" ON PUBLIC."Hypertable_1" (time, "Device_id"); CREATE UNIQUE INDEX "Unique1" ON "customSchema"."Hypertable_1" (time); INSERT INTO "customSchema"."Hypertable_1"(time, "Device_id", temp_c, humidity, sensor_1, sensor_2, sensor_3, sensor_4) VALUES(1257894000000000000, 'dev1', 30, 70, 1, 2, 3, 100); INSERT INTO "customSchema"."Hypertable_1"(time, "Device_id", temp_c, humidity, sensor_1, sensor_2, sensor_3, sensor_4) VALUES(1257894000000000001, 'dev1', 30, 70, 1, 2, 3, 100); SELECT * FROM test.show_indexesp('%.%'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper_%'); --expect error cases \set ON_ERROR_STOP 0 INSERT INTO "customSchema"."Hypertable_1"(time, "Device_id", temp_c, humidity, sensor_1, sensor_2, sensor_3, sensor_4) VALUES(1257894000000000000, 'dev1', 31, 71, 72, 4, 1, 102); CREATE UNIQUE INDEX "Unique2" ON PUBLIC."Hypertable_1" ("Device_id"); CREATE UNIQUE INDEX "Unique2" ON PUBLIC."Hypertable_1" (time); CREATE UNIQUE INDEX "Unique2" ON PUBLIC."Hypertable_1" (sensor_1); UPDATE ONLY PUBLIC."Hypertable_1" SET time = 0 WHERE TRUE; DELETE FROM ONLY PUBLIC."Hypertable_1" WHERE "Device_id" = 'dev1'; \set ON_ERROR_STOP 1 CREATE TABLE my_ht (time BIGINT, val integer); SELECT * FROM create_hypertable('my_ht', 'time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); ALTER TABLE my_ht ADD COLUMN val2 integer; SELECT * FROM test.show_columns('my_ht'); -- Should error when adding again \set ON_ERROR_STOP 0 ALTER TABLE my_ht ADD COLUMN val2 integer; \set ON_ERROR_STOP 1 -- Should create ALTER TABLE my_ht ADD COLUMN IF NOT EXISTS val3 integer; SELECT * FROM test.show_columns('my_ht'); -- Should skip and not error ALTER TABLE my_ht ADD COLUMN IF NOT EXISTS val3 integer; SELECT * FROM test.show_columns('my_ht'); -- Should drop ALTER TABLE my_ht DROP COLUMN IF EXISTS val3; SELECT * FROM test.show_columns('my_ht'); -- Should skip and not error ALTER TABLE my_ht DROP COLUMN IF EXISTS val3; SELECT * FROM test.show_columns('my_ht'); --Test default index creation on create_hypertable(). --Make sure that we do not duplicate indexes that already exists -- --No existing indexes: both time and space-time indexes created BEGIN; CREATE TABLE PUBLIC."Hypertable_1_with_default_index_enabled" ( "Time" BIGINT NOT NULL, "Device_id" TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 ); SELECT * FROM create_hypertable('"public"."Hypertable_1_with_default_index_enabled"', 'Time', 'Device_id', 1, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); SELECT * FROM test.show_indexes('"Hypertable_1_with_default_index_enabled"'); ROLLBACK; --Space index exists: only time index created BEGIN; CREATE TABLE PUBLIC."Hypertable_1_with_default_index_enabled" ( "Time" BIGINT NOT NULL, "Device_id" TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 ); CREATE INDEX ON PUBLIC."Hypertable_1_with_default_index_enabled" ("Device_id", "Time" DESC); SELECT * FROM create_hypertable('"public"."Hypertable_1_with_default_index_enabled"', 'Time', 'Device_id', 1, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); SELECT * FROM test.show_indexes('"Hypertable_1_with_default_index_enabled"'); ROLLBACK; --Time index exists, only partition index created BEGIN; CREATE TABLE PUBLIC."Hypertable_1_with_default_index_enabled" ( "Time" BIGINT NOT NULL, "Device_id" TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 ); CREATE INDEX ON PUBLIC."Hypertable_1_with_default_index_enabled" ("Time" DESC); SELECT * FROM create_hypertable('"public"."Hypertable_1_with_default_index_enabled"', 'Time', 'Device_id', 1, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); SELECT * FROM test.show_indexes('"Hypertable_1_with_default_index_enabled"'); ROLLBACK; --No space partitioning, only time index created BEGIN; CREATE TABLE PUBLIC."Hypertable_1_with_default_index_enabled" ( "Time" BIGINT NOT NULL, "Device_id" TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 ); SELECT * FROM create_hypertable('"public"."Hypertable_1_with_default_index_enabled"', 'Time', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); SELECT * FROM test.show_indexes('"Hypertable_1_with_default_index_enabled"'); ROLLBACK; --Disable index creation: no default indexes created BEGIN; CREATE TABLE PUBLIC."Hypertable_1_with_default_index_enabled" ( "Time" BIGINT NOT NULL, "Device_id" TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 ); SELECT * FROM create_hypertable('"public"."Hypertable_1_with_default_index_enabled"', 'Time', 'Device_id', 1, create_default_indexes=>FALSE, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); SELECT * FROM test.show_indexes('"Hypertable_1_with_default_index_enabled"'); ROLLBACK; ================================================ FILE: test/sql/include/ddl_ops_2.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. ALTER TABLE PUBLIC."Hypertable_1" ADD COLUMN temp_f INTEGER NOT NULL DEFAULT 31; ALTER TABLE PUBLIC."Hypertable_1" DROP COLUMN temp_c; ALTER TABLE PUBLIC."Hypertable_1" DROP COLUMN sensor_4; ALTER TABLE PUBLIC."Hypertable_1" ALTER COLUMN humidity SET DEFAULT 100; ALTER TABLE PUBLIC."Hypertable_1" ALTER COLUMN sensor_1 DROP DEFAULT; ALTER TABLE PUBLIC."Hypertable_1" ALTER COLUMN sensor_2 SET DEFAULT NULL; ALTER TABLE PUBLIC."Hypertable_1" ALTER COLUMN sensor_1 SET NOT NULL; ALTER TABLE PUBLIC."Hypertable_1" ALTER COLUMN sensor_2 DROP NOT NULL; ALTER TABLE PUBLIC."Hypertable_1" RENAME COLUMN sensor_2 TO sensor_2_renamed; ALTER TABLE PUBLIC."Hypertable_1" RENAME COLUMN sensor_3 TO sensor_3_renamed; DROP INDEX "ind_sensor_1"; CREATE OR REPLACE FUNCTION empty_trigger_func() RETURNS TRIGGER LANGUAGE PLPGSQL AS $BODY$ BEGIN END $BODY$; CREATE TRIGGER test_trigger BEFORE UPDATE OR DELETE ON PUBLIC."Hypertable_1" FOR EACH STATEMENT EXECUTE FUNCTION empty_trigger_func(); ALTER TABLE PUBLIC."Hypertable_1" ALTER COLUMN sensor_2_renamed SET DATA TYPE int; ALTER INDEX "ind_humidity" RENAME TO "ind_humdity2"; -- Change should be reflected here SELECT * FROM test.show_indexesp('%.%'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%'); --create column with same name as previously renamed one ALTER TABLE PUBLIC."Hypertable_1" ADD COLUMN sensor_3 BIGINT NOT NULL DEFAULT 131; --create column with same name as previously dropped one ALTER TABLE PUBLIC."Hypertable_1" ADD COLUMN sensor_4 BIGINT NOT NULL DEFAULT 131; ================================================ FILE: test/sql/include/insert_single.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."one_Partition" ( "timeCustom" BIGINT NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."one_Partition" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; CREATE INDEX ON PUBLIC."one_Partition" ("timeCustom" DESC NULLS LAST, series_0) WHERE series_0 IS NOT NULL; CREATE INDEX ON PUBLIC."one_Partition" ("timeCustom" DESC NULLS LAST, series_1) WHERE series_1 IS NOT NULL; CREATE INDEX ON PUBLIC."one_Partition" ("timeCustom" DESC NULLS LAST, series_2) WHERE series_2 IS NOT NULL; CREATE INDEX ON PUBLIC."one_Partition" ("timeCustom" DESC NULLS LAST, series_bool) WHERE series_bool IS NOT NULL; \c :DBNAME :ROLE_SUPERUSER CREATE SCHEMA "one_Partition" AUTHORIZATION :ROLE_DEFAULT_PERM_USER; \c :DBNAME :ROLE_DEFAULT_PERM_USER; SELECT * FROM create_hypertable('"public"."one_Partition"', 'timeCustom', associated_schema_name=>'one_Partition', chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); --output command tags \set QUIET off BEGIN; \COPY "one_Partition" FROM 'data/ds1_dev1_1.tsv' NULL AS ''; COMMIT; INSERT INTO "one_Partition"("timeCustom", device_id, series_0, series_1) VALUES (1257987600000000000, 'dev1', 1.5, 1), (1257987600000000000, 'dev1', 1.5, 2), (1257894000000000000, 'dev2', 1.5, 1), (1257894002000000000, 'dev1', 2.5, 3); INSERT INTO "one_Partition"("timeCustom", device_id, series_0, series_1) VALUES (1257894000000000000, 'dev2', 1.5, 2); \set QUIET on ================================================ FILE: test/sql/include/insert_two_partitions.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."two_Partitions" ( "timeCustom" BIGINT NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."two_Partitions" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_0) WHERE series_0 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_1) WHERE series_1 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_2) WHERE series_2 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_bool) WHERE series_bool IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, device_id); SELECT * FROM create_hypertable('"public"."two_Partitions"'::regclass, 'timeCustom'::name, 'device_id'::name, associated_schema_name=>'_timescaledb_internal'::text, number_partitions => 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); \set QUIET off BEGIN; \COPY public."two_Partitions" FROM 'data/ds1_dev1_1.tsv' NULL AS ''; COMMIT; INSERT INTO public."two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257987600000000000, 'dev1', 1.5, 1), (1257987600000000000, 'dev1', 1.5, 2), (1257894000000000000, 'dev2', 1.5, 1), (1257894002000000000, 'dev1', 2.5, 3); INSERT INTO "two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257894000000000000, 'dev2', 1.5, 2); \set QUIET on ================================================ FILE: test/sql/include/join_load.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- these table definitions have been adjusted from -- table defintions of the postgres test suite CREATE TABLE INT2_TBL(f1 int2, ts timestamptz NOT NULL DEFAULT '2000-01-01'); SELECT table_name FROM create_hypertable('int2_tbl','ts'); INSERT INTO INT2_TBL(f1) VALUES ('0 '); INSERT INTO INT2_TBL(f1) VALUES (' 1234 '); INSERT INTO INT2_TBL(f1) VALUES (' -1234'); -- largest and smallest values INSERT INTO INT2_TBL(f1) VALUES ('32767'); INSERT INTO INT2_TBL(f1) VALUES ('-32767'); CREATE TABLE INT4_TBL(f1 int4, ts timestamptz NOT NULL DEFAULT '2000-01-01'); SELECT table_name FROM create_hypertable('int4_tbl','ts'); INSERT INTO INT4_TBL(f1) VALUES (' 0 '); INSERT INTO INT4_TBL(f1) VALUES ('123456 '); INSERT INTO INT4_TBL(f1) VALUES (' -123456'); -- largest and smallest values INSERT INTO INT4_TBL(f1) VALUES ('2147483647'); INSERT INTO INT4_TBL(f1) VALUES ('-2147483647'); CREATE TABLE INT8_TBL(q1 int8, q2 int8, ts timestamptz NOT NULL DEFAULT '2000-01-01'); SELECT table_name FROM create_hypertable('int8_tbl','ts'); INSERT INTO INT8_TBL VALUES(' 123 ',' 456'); INSERT INTO INT8_TBL VALUES('123 ','4567890123456789'); INSERT INTO INT8_TBL VALUES('4567890123456789','123'); INSERT INTO INT8_TBL VALUES(+4567890123456789,'4567890123456789'); INSERT INTO INT8_TBL VALUES('+4567890123456789','-4567890123456789'); CREATE TABLE FLOAT8_TBL(f1 float8, ts timestamptz NOT NULL DEFAULT '2000-01-01'); SELECT table_name FROM create_hypertable('float8_tbl','ts'); INSERT INTO FLOAT8_TBL(f1) VALUES (' 0.0 '); INSERT INTO FLOAT8_TBL(f1) VALUES ('1004.30 '); INSERT INTO FLOAT8_TBL(f1) VALUES (' -34.84'); INSERT INTO FLOAT8_TBL(f1) VALUES ('1.2345678901234e+200'); INSERT INTO FLOAT8_TBL(f1) VALUES ('1.2345678901234e-200'); CREATE TABLE TEXT_TBL (f1 text, ts timestamptz NOT NULL DEFAULT '2000-01-01'); SELECT table_name FROM create_hypertable('text_tbl','ts'); INSERT INTO TEXT_TBL VALUES ('doh!'); INSERT INTO TEXT_TBL VALUES ('hi de ho neighbor'); CREATE TABLE a (aa TEXT); CREATE TABLE b (bb TEXT) INHERITS (a); CREATE TABLE c (cc TEXT) INHERITS (a); CREATE TABLE d (dd TEXT) INHERITS (b,c,a); INSERT INTO a(aa) VALUES('aaa'); INSERT INTO a(aa) VALUES('aaaa'); INSERT INTO a(aa) VALUES('aaaaa'); INSERT INTO a(aa) VALUES('aaaaaa'); INSERT INTO a(aa) VALUES('aaaaaaa'); INSERT INTO a(aa) VALUES('aaaaaaaa'); INSERT INTO b(aa) VALUES('bbb'); INSERT INTO b(aa) VALUES('bbbb'); INSERT INTO b(aa) VALUES('bbbbb'); INSERT INTO b(aa) VALUES('bbbbbb'); INSERT INTO b(aa) VALUES('bbbbbbb'); INSERT INTO b(aa) VALUES('bbbbbbbb'); INSERT INTO c(aa) VALUES('ccc'); INSERT INTO c(aa) VALUES('cccc'); INSERT INTO c(aa) VALUES('ccccc'); INSERT INTO c(aa) VALUES('cccccc'); INSERT INTO c(aa) VALUES('ccccccc'); INSERT INTO c(aa) VALUES('cccccccc'); INSERT INTO d(aa) VALUES('ddd'); INSERT INTO d(aa) VALUES('dddd'); INSERT INTO d(aa) VALUES('ddddd'); INSERT INTO d(aa) VALUES('dddddd'); INSERT INTO d(aa) VALUES('ddddddd'); INSERT INTO d(aa) VALUES('dddddddd'); CREATE TABLE onek ( unique1 int4, unique2 int4, two int4, four int4, ten int4, twenty int4, hundred int4, thousand int4, twothousand int4, fivethous int4, tenthous int4, odd int4, even int4, stringu1 name, stringu2 name, string4 name ); SELECT table_name FROM create_hypertable('onek','unique2',chunk_time_interval:=1000); \copy onek FROM 'data/onek.data' CREATE TABLE tenk1 ( unique1 int4, unique2 int4, two int4, four int4, ten int4, twenty int4, hundred int4, thousand int4, twothousand int4, fivethous int4, tenthous int4, odd int4, even int4, stringu1 name, stringu2 name, string4 name ); SELECT table_name FROM create_hypertable('tenk1','unique2',chunk_time_interval:=1000); \copy tenk1 FROM 'data/tenk.data' CREATE TABLE tenk2 ( unique1 int4, unique2 int4, two int4, four int4, ten int4, twenty int4, hundred int4, thousand int4, twothousand int4, fivethous int4, tenthous int4, odd int4, even int4, stringu1 name, stringu2 name, string4 name ); SELECT table_name FROM create_hypertable('tenk2','unique2',chunk_time_interval:=1000); INSERT INTO tenk2 SELECT * FROM tenk1; ================================================ FILE: test/sql/include/join_query.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- these queries are based on the postgres JOIN tests -- and have been adjusted to run on hypertables -- canary for results diff -- this should be the only output of the results diff SELECT setting, current_setting(setting) AS value from (VALUES ('timescaledb.enable_optimizations')) v(setting); -- -- JOIN -- Test JOIN clauses -- CREATE TABLE J1_TBL ( i integer, j integer, t text, ts_j1 timestamptz NOT NULL DEFAULT '2000-01-01' ); CREATE TABLE J2_TBL ( i integer, k integer, ts_j2 timestamptz NOT NULL DEFAULT '2000-01-02' ); SELECT (SELECT table_name FROM create_hypertable(tbl||'_tbl', 'ts_' || tbl)) FROM (VALUES ('j1'),('j2')) v(tbl); INSERT INTO J1_TBL VALUES (1, 4, 'one'); INSERT INTO J1_TBL VALUES (2, 3, 'two'); INSERT INTO J1_TBL VALUES (3, 2, 'three'); INSERT INTO J1_TBL VALUES (4, 1, 'four'); INSERT INTO J1_TBL VALUES (5, 0, 'five'); INSERT INTO J1_TBL VALUES (6, 6, 'six'); INSERT INTO J1_TBL VALUES (7, 7, 'seven'); INSERT INTO J1_TBL VALUES (8, 8, 'eight'); INSERT INTO J1_TBL VALUES (0, NULL, 'zero'); INSERT INTO J1_TBL VALUES (NULL, NULL, 'null'); INSERT INTO J1_TBL VALUES (NULL, 0, 'zero'); INSERT INTO J2_TBL VALUES (1, -1); INSERT INTO J2_TBL VALUES (2, 2); INSERT INTO J2_TBL VALUES (3, -3); INSERT INTO J2_TBL VALUES (2, 4); INSERT INTO J2_TBL VALUES (5, -5); INSERT INTO J2_TBL VALUES (5, -5); INSERT INTO J2_TBL VALUES (0, NULL); INSERT INTO J2_TBL VALUES (NULL, NULL); INSERT INTO J2_TBL VALUES (NULL, 0); -- -- CORRELATION NAMES -- Make sure that table/column aliases are supported -- before diving into more complex join syntax. -- SELECT '' AS "xxx", * FROM J1_TBL AS tx; SELECT '' AS "xxx", * FROM J1_TBL tx; SELECT '' AS "xxx", * FROM J1_TBL AS t1 (a, b, c); SELECT '' AS "xxx", * FROM J1_TBL t1 (a, b, c); SELECT '' AS "xxx", * FROM J1_TBL t1 (a, b, c), J2_TBL t2 (d, e); SELECT '' AS "xxx", t1.a, t2.e FROM J1_TBL t1 (a, b, c), J2_TBL t2 (d, e) WHERE t1.a = t2.d; -- -- CROSS JOIN -- Qualifications are not allowed on cross joins, -- which degenerate into a standard unqualified inner join. -- SELECT '' AS "xxx", * FROM J1_TBL CROSS JOIN J2_TBL; -- ambiguous column -- SELECT '' AS "xxx", i, k, t -- FROM J1_TBL CROSS JOIN J2_TBL; -- resolve previous ambiguity by specifying the table name SELECT '' AS "xxx", t1.i, k, t FROM J1_TBL t1 CROSS JOIN J2_TBL t2; SELECT '' AS "xxx", ii, tt, kk FROM (J1_TBL CROSS JOIN J2_TBL) AS tx (ii, jj, tt, ii2, kk); SELECT '' AS "xxx", tx.ii, tx.jj, tx.kk FROM (J1_TBL t1 (a, b, c) CROSS JOIN J2_TBL t2 (d, e)) AS tx (ii, jj, tt, ii2, kk); SELECT '' AS "xxx", * FROM J1_TBL CROSS JOIN J2_TBL a CROSS JOIN J2_TBL b; -- -- -- Inner joins (equi-joins) -- -- -- -- Inner joins (equi-joins) with USING clause -- The USING syntax changes the shape of the resulting table -- by including a column in the USING clause only once in the result. -- -- Inner equi-join on specified column SELECT '' AS "xxx", * FROM J1_TBL INNER JOIN J2_TBL USING (i); -- Same as above, slightly different syntax SELECT '' AS "xxx", * FROM J1_TBL JOIN J2_TBL USING (i); SELECT '' AS "xxx", * FROM J1_TBL t1 (a, b, c) JOIN J2_TBL t2 (a, d) USING (a) ORDER BY a, d; SELECT '' AS "xxx", * FROM J1_TBL t1 (a, b, c) JOIN J2_TBL t2 (a, b) USING (b) ORDER BY b, t1.a; -- -- NATURAL JOIN -- Inner equi-join on all columns with the same name -- SELECT '' AS "xxx", * FROM J1_TBL NATURAL JOIN J2_TBL; SELECT '' AS "xxx", * FROM J1_TBL t1 (a, b, c) NATURAL JOIN J2_TBL t2 (a, d); SELECT '' AS "xxx", * FROM J1_TBL t1 (a, b, c) NATURAL JOIN J2_TBL t2 (d, a); -- mismatch number of columns -- currently, Postgres will fill in with underlying names SELECT '' AS "xxx", * FROM J1_TBL t1 (a, b) NATURAL JOIN J2_TBL t2 (a); -- -- Inner joins (equi-joins) -- SELECT '' AS "xxx", * FROM J1_TBL JOIN J2_TBL ON (J1_TBL.i = J2_TBL.i); SELECT '' AS "xxx", * FROM J1_TBL JOIN J2_TBL ON (J1_TBL.i = J2_TBL.k); -- -- Non-equi-joins -- SELECT '' AS "xxx", * FROM J1_TBL JOIN J2_TBL ON (J1_TBL.i <= J2_TBL.k); -- -- Outer joins -- Note that OUTER is a noise word -- SELECT '' AS "xxx", * FROM J1_TBL LEFT OUTER JOIN J2_TBL USING (i) ORDER BY i, k, t; SELECT '' AS "xxx", * FROM J1_TBL LEFT JOIN J2_TBL USING (i) ORDER BY i, k, t; SELECT '' AS "xxx", * FROM J1_TBL RIGHT OUTER JOIN J2_TBL USING (i); SELECT '' AS "xxx", * FROM J1_TBL RIGHT JOIN J2_TBL USING (i); SELECT '' AS "xxx", * FROM J1_TBL FULL OUTER JOIN J2_TBL USING (i) ORDER BY i, k, t; SELECT '' AS "xxx", * FROM J1_TBL FULL JOIN J2_TBL USING (i) ORDER BY i, k, t; SELECT '' AS "xxx", * FROM J1_TBL LEFT JOIN J2_TBL USING (i) WHERE (k = 1); SELECT '' AS "xxx", * FROM J1_TBL LEFT JOIN J2_TBL USING (i) WHERE (i = 1); -- -- semijoin selectivity for <> -- :PREFIX select * from int4_tbl i4, tenk1 a where exists(select * from tenk1 b where a.twothousand = b.twothousand and a.fivethous <> b.fivethous) and i4.f1 = a.tenthous; -- -- More complicated constructs -- -- -- Multiway full join -- CREATE TABLE t1 (name TEXT, n INTEGER, ts_t1 timestamptz NOT NULL DEFAULT '2000-01-01'); CREATE TABLE t2 (name TEXT, n INTEGER, ts_t2 timestamptz NOT NULL DEFAULT '2000-01-02'); CREATE TABLE t3 (name TEXT, n INTEGER, ts_t3 timestamptz NOT NULL DEFAULT '2000-01-03'); SELECT (SELECT table_name FROM create_hypertable(tbl, 'ts_' || tbl)) FROM (VALUES ('t1'),('t2'),('t3')) v(tbl); INSERT INTO t1 VALUES ( 'bb', 11 ); INSERT INTO t2 VALUES ( 'bb', 12 ); INSERT INTO t2 VALUES ( 'cc', 22 ); INSERT INTO t2 VALUES ( 'ee', 42 ); INSERT INTO t3 VALUES ( 'bb', 13 ); INSERT INTO t3 VALUES ( 'cc', 23 ); INSERT INTO t3 VALUES ( 'dd', 33 ); SELECT * FROM t1 FULL JOIN t2 USING (name) FULL JOIN t3 USING (name); -- -- Test interactions of join syntax and subqueries -- -- Basic cases (we expect planner to pull up the subquery here) SELECT * FROM (SELECT * FROM t2) as s2 INNER JOIN (SELECT * FROM t3) s3 USING (name); SELECT * FROM (SELECT * FROM t2) as s2 LEFT JOIN (SELECT * FROM t3) s3 USING (name); SELECT * FROM (SELECT * FROM t2) as s2 FULL JOIN (SELECT * FROM t3) s3 USING (name); -- Cases with non-nullable expressions in subquery results; -- make sure these go to null as expected SELECT * FROM (SELECT name, n as s2_n, 2 as s2_2 FROM t2) as s2 NATURAL INNER JOIN (SELECT name, n as s3_n, 3 as s3_2 FROM t3) s3; SELECT * FROM (SELECT name, n as s2_n, 2 as s2_2 FROM t2) as s2 NATURAL LEFT JOIN (SELECT name, n as s3_n, 3 as s3_2 FROM t3) s3; SELECT * FROM (SELECT name, n as s2_n, 2 as s2_2 FROM t2) as s2 NATURAL FULL JOIN (SELECT name, n as s3_n, 3 as s3_2 FROM t3) s3; SELECT * FROM (SELECT name, n as s1_n, 1 as s1_1 FROM t1) as s1 NATURAL INNER JOIN (SELECT name, n as s2_n, 2 as s2_2 FROM t2) as s2 NATURAL INNER JOIN (SELECT name, n as s3_n, 3 as s3_2 FROM t3) s3; SELECT * FROM (SELECT name, n as s1_n, 1 as s1_1 FROM t1) as s1 NATURAL FULL JOIN (SELECT name, n as s2_n, 2 as s2_2 FROM t2) as s2 NATURAL FULL JOIN (SELECT name, n as s3_n, 3 as s3_2 FROM t3) s3; SELECT * FROM (SELECT name, n as s1_n FROM t1) as s1 NATURAL FULL JOIN (SELECT * FROM (SELECT name, n as s2_n FROM t2) as s2 NATURAL FULL JOIN (SELECT name, n as s3_n FROM t3) as s3 ) ss2; SELECT * FROM (SELECT name, n as s1_n FROM t1) as s1 NATURAL FULL JOIN (SELECT * FROM (SELECT name, n as s2_n, 2 as s2_2 FROM t2) as s2 NATURAL FULL JOIN (SELECT name, n as s3_n FROM t3) as s3 ) ss2; -- Test for propagation of nullability constraints into sub-joins create table x (x1 int, x2 int, ts_x timestamptz NOT NULL DEFAULT '2000-01-01'); SELECT table_name FROM create_hypertable('x','ts_x'); insert into x values (1,11); insert into x values (2,22); insert into x values (3,null); insert into x values (4,44); insert into x values (5,null); create table y (y1 int, y2 int, ts_y timestamptz NOT NULL DEFAULT '2000-01-02'); SELECT table_name FROM create_hypertable('y','ts_y'); insert into y values (1,111); insert into y values (2,222); insert into y values (3,333); insert into y values (4,null); select * from x; select * from y; select * from x left join y on (x1 = y1 and x2 is not null); select * from x left join y on (x1 = y1 and y2 is not null); select * from (x left join y on (x1 = y1)) left join x xx(xx1,xx2) on (x1 = xx1); select * from (x left join y on (x1 = y1)) left join x xx(xx1,xx2) on (x1 = xx1 and x2 is not null); select * from (x left join y on (x1 = y1)) left join x xx(xx1,xx2) on (x1 = xx1 and y2 is not null); select * from (x left join y on (x1 = y1)) left join x xx(xx1,xx2) on (x1 = xx1 and xx2 is not null); -- these should NOT give the same answers as above select * from (x left join y on (x1 = y1)) left join x xx(xx1,xx2) on (x1 = xx1) where (x2 is not null); select * from (x left join y on (x1 = y1)) left join x xx(xx1,xx2) on (x1 = xx1) where (y2 is not null); select * from (x left join y on (x1 = y1)) left join x xx(xx1,xx2) on (x1 = xx1) where (xx2 is not null); -- -- regression test: check for bug with propagation of implied equality -- to outside an IN -- select count(*) from tenk1 a where unique1 in (select unique1 from tenk1 b join tenk1 c using (unique1) where b.unique2 = 42); -- -- regression test: check for failure to generate a plan with multiple -- degenerate IN clauses -- select count(*) from tenk1 x where x.unique1 in (select a.f1 from int4_tbl a,float8_tbl b where a.f1=b.f1) and x.unique1 = 0 and x.unique1 in (select aa.f1 from int4_tbl aa,float8_tbl bb where aa.f1=bb.f1); -- try that with GEQO too begin; set geqo = on; set geqo_threshold = 2; select count(*) from tenk1 x where x.unique1 in (select a.f1 from int4_tbl a,float8_tbl b where a.f1=b.f1) and x.unique1 = 0 and x.unique1 in (select aa.f1 from int4_tbl aa,float8_tbl bb where aa.f1=bb.f1); rollback; -- -- regression test: be sure we cope with proven-dummy append rels -- :PREFIX select aa, bb, unique1, unique1 from tenk1 right join b on aa::int = unique1 where bb < bb and bb is null; -- -- regression test: check handling of empty-FROM subquery underneath outer join -- :PREFIX select * from int8_tbl i1 left join (int8_tbl i2 join (select 123 as x) ss on i2.q1 = x) on i1.q2 = i2.q2 order by 1, 2; -- -- regression test: check a case where join_clause_is_movable_into() gives -- an imprecise result, causing an assertion failure -- select count(*) from (select t3.tenthous as x1, coalesce(t1.stringu1, t2.stringu1) as x2 from tenk1 t1 left join tenk1 t2 on t1.unique1 = t2.unique1 join tenk1 t3 on t1.unique2 = t3.unique2) ss, tenk1 t4, tenk1 t5 where t4.thousand = t5.unique1 and ss.x1 = t4.tenthous and ss.x2 = t5.stringu1; -- -- regression test: check a case where we formerly missed including an EC -- enforcement clause because it was expected to be handled at scan level -- :PREFIX select a.f1, b.f1, t.thousand, t.tenthous from tenk1 t, (select sum(f1)+1 as f1 from int4_tbl i4a) a, (select sum(f1) as f1 from int4_tbl i4b) b where b.f1 = t.thousand and a.f1 = b.f1 and (a.f1+b.f1+999) = t.tenthous; -- -- check a case where we formerly got confused by conflicting sort orders -- in redundant merge join path keys -- :PREFIX select * from j1_tbl full join (select * from j2_tbl order by j2_tbl.i desc, j2_tbl.k asc) j2_tbl on j1_tbl.i = j2_tbl.i and j1_tbl.i = j2_tbl.k; -- -- a different check for handling of redundant sort keys in merge joins -- :PREFIX select count(*) from (select * from tenk1 x order by x.thousand, x.twothousand, x.fivethous) x left join (select * from tenk1 y order by y.unique2) y on x.thousand = y.unique2 and x.twothousand = y.hundred and x.fivethous = y.unique2; -- -- Clean up -- DROP TABLE t1; DROP TABLE t2; DROP TABLE t3; DROP TABLE J1_TBL; DROP TABLE J2_TBL; DROP TABLE x; DROP TABLE y; -- Both DELETE and UPDATE allow the specification of additional tables -- to "join" against to determine which rows should be modified. CREATE TABLE t1 (a int, b int, ts_t1 timestamptz NOT NULL DEFAULT '2000-01-01'); CREATE TABLE t2 (a int, b int, ts_t2 timestamptz NOT NULL DEFAULT '2000-01-02'); CREATE TABLE t3 (x int, y int, ts_t3 timestamptz NOT NULL DEFAULT '2000-01-03'); SELECT (SELECT table_name FROM create_hypertable(tbl, 'ts_' || tbl)) FROM (VALUES ('t1'),('t2'),('t3')) v(tbl); INSERT INTO t1 VALUES (5, 10); INSERT INTO t1 VALUES (15, 20); INSERT INTO t1 VALUES (100, 100); INSERT INTO t1 VALUES (200, 1000); INSERT INTO t2 VALUES (200, 2000); INSERT INTO t3 VALUES (5, 20); INSERT INTO t3 VALUES (6, 7); INSERT INTO t3 VALUES (7, 8); INSERT INTO t3 VALUES (500, 100); DELETE FROM t3 USING t1 table1 WHERE t3.x = table1.a; SELECT * FROM t3; DELETE FROM t3 USING t1 JOIN t2 USING (a) WHERE t3.x > t1.a; SELECT * FROM t3; DELETE FROM t3 USING t3 t3_other WHERE t3.x = t3_other.x AND t3.y = t3_other.y; SELECT * FROM t3; -- Test matching of column name with wrong alias -- select t1.x from t1 join t3 on (t1.a = t3.x); DROP TABLE t1; DROP TABLE t2; DROP TABLE t3; -- -- regression test for 8.1 merge right join bug -- CREATE TABLE tt1 ( tt1_id int4, joincol int4, ts_tt1 timestamptz NOT NULL DEFAULT '2000-01-01' ); SELECT table_name FROM create_hypertable('tt1','ts_tt1'); INSERT INTO tt1 VALUES (1, 11); INSERT INTO tt1 VALUES (2, NULL); CREATE TABLE tt2 ( tt2_id int4, joincol int4, ts_tt2 timestamptz NOT NULL DEFAULT '2000-01-02' ); SELECT table_name FROM create_hypertable('tt2','ts_tt2'); INSERT INTO tt2 VALUES (21, 11); INSERT INTO tt2 VALUES (22, 11); set enable_hashjoin to off; set enable_nestloop to off; -- these should give the same results select tt1.*, tt2.* from tt1 left join tt2 on tt1.joincol = tt2.joincol; select tt1.*, tt2.* from tt2 right join tt1 on tt1.joincol = tt2.joincol; reset enable_hashjoin; reset enable_nestloop; DROP TABLE tt1; DROP TABLE tt2; -- -- regression test for bug #13908 (hash join with skew tuples & nbatch increase) -- set work_mem to '64kB'; set enable_mergejoin to off; :PREFIX select count(*) from tenk1 a, tenk1 b where a.hundred = b.thousand and (b.fivethous % 10) < 10; reset work_mem; reset enable_mergejoin; -- -- regression test for 8.2 bug with improper re-ordering of left joins -- create table tt3(f1 int, f2 text); SELECT table_name FROM create_hypertable('tt3','f1',chunk_time_interval:=2000); insert into tt3 select x, repeat('xyzzy', 100) from generate_series(1,10000) x; create index tt3i on tt3(f1); analyze tt3; create table tt4(f1 int); SELECT table_name FROM create_hypertable('tt4','f1',chunk_time_interval:=2000); insert into tt4 values (0),(1),(9999); analyze tt4; SELECT a.f1 FROM tt4 a LEFT JOIN ( SELECT b.f1 FROM tt3 b LEFT JOIN tt3 c ON (b.f1 = c.f1) WHERE c.f1 IS NULL ) AS d ON (a.f1 = d.f1) WHERE d.f1 IS NULL; DROP TABLE tt3; DROP TABLE tt4; -- -- regression test for proper handling of outer joins within antijoins -- create table tt4x(c1 int, c2 int, c3 int); SELECT table_name FROM create_hypertable('tt4x','c1',chunk_time_interval:=2000); :PREFIX select * from tt4x t1 where not exists ( select 1 from tt4x t2 left join tt4x t3 on t2.c3 = t3.c1 left join ( select t5.c1 as c1 from tt4x t4 left join tt4x t5 on t4.c2 = t5.c1 ) a1 on t3.c2 = a1.c1 where t1.c1 = t2.c2 ); DROP TABLE tt4x; -- -- regression test for problems of the sort depicted in bug #3494 -- create table tt5(f1 int, f2 int); create table tt6(f1 int, f2 int); SELECT table_name FROM create_hypertable('tt5','f1',chunk_time_interval:=2000); SELECT table_name FROM create_hypertable('tt6','f1',chunk_time_interval:=2000); insert into tt5 values(1, 10); insert into tt5 values(1, 11); insert into tt6 values(1, 9); insert into tt6 values(1, 2); insert into tt6 values(2, 9); select * from tt5,tt6 where tt5.f1 = tt6.f1 and tt5.f1 = tt5.f2 - tt6.f2; DROP TABLE tt5; DROP TABLE tt6; -- -- regression test for problems of the sort depicted in bug #3588 -- create table xx (pkxx int); create table yy (pkyy int, pkxx int); select table_name FROM create_hypertable('xx','pkxx',chunk_time_interval:=2000); select table_name FROM create_hypertable('yy','pkyy',chunk_time_interval:=2000); insert into xx values (1); insert into xx values (2); insert into xx values (3); insert into yy values (101, 1); insert into yy values (201, 2); insert into yy values (301, NULL); select yy.pkyy as yy_pkyy, yy.pkxx as yy_pkxx, yya.pkyy as yya_pkyy, xxa.pkxx as xxa_pkxx, xxb.pkxx as xxb_pkxx from yy left join (SELECT * FROM yy where pkyy = 101) as yya ON yy.pkyy = yya.pkyy left join xx xxa on yya.pkxx = xxa.pkxx left join xx xxb on coalesce (xxa.pkxx, 1) = xxb.pkxx; DROP TABLE xx; DROP TABLE yy; -- -- regression test for improper pushing of constants across outer-join clauses -- (as seen in early 8.2.x releases) -- create table zt1 (f1 int primary key); create table zt2 (f2 int primary key); create table zt3 (f3 int primary key); select table_name FROM create_hypertable('zt1','f1',chunk_time_interval:=2000); select table_name FROM create_hypertable('zt2','f2',chunk_time_interval:=2000); select table_name FROM create_hypertable('zt3','f3',chunk_time_interval:=2000); insert into zt1 values(53); insert into zt2 values(53); select * from zt2 left join zt3 on (f2 = f3) left join zt1 on (f3 = f1) where f2 = 53; create temp view zv1 as select *,'dummy'::text AS junk from zt1; select * from zt2 left join zt3 on (f2 = f3) left join zv1 on (f3 = f1) where f2 = 53; drop table zt1 cascade; drop table zt2; drop table zt3; -- -- regression test for improper extraction of OR indexqual conditions -- (as seen in early 8.3.x releases) -- select a.unique2, a.ten, b.tenthous, b.unique2, b.hundred from tenk1 a left join tenk1 b on a.unique2 = b.tenthous where a.unique1 = 42 and ((b.unique2 is null and a.ten = 2) or b.hundred = 3); -- -- test proper positioning of one-time quals in EXISTS (8.4devel bug) -- prepare foo(bool) as select count(*) from tenk1 a left join tenk1 b on (a.unique2 = b.unique1 and exists (select 1 from tenk1 c where c.thousand = b.unique2 and $1)); execute foo(true); execute foo(false); deallocate foo; -- -- test for sane behavior with noncanonical merge clauses, per bug #4926 -- begin; set enable_mergejoin = 1; set enable_hashjoin = 0; set enable_nestloop = 0; create temp table a (i integer); create temp table b (x integer, y integer); select * from a left join b on i = x and i = y and x = i; rollback; -- -- test handling of merge clauses using record_ops -- begin; create type mycomptype as (id int, v bigint); create temp table tidv (idv mycomptype); create index on tidv (idv); :PREFIX select a.idv, b.idv from tidv a, tidv b where a.idv = b.idv; set enable_mergejoin = 0; :PREFIX select a.idv, b.idv from tidv a, tidv b where a.idv = b.idv; rollback; -- -- test NULL behavior of whole-row Vars, per bug #5025 -- select t1.q2, count(t2.*) from int8_tbl t1 left join int8_tbl t2 on (t1.q2 = t2.q1) group by t1.q2 order by 1; select t1.q2, count(t2.*) from int8_tbl t1 left join (select * from int8_tbl) t2 on (t1.q2 = t2.q1) group by t1.q2 order by 1; select t1.q2, count(t2.*) from int8_tbl t1 left join (select * from int8_tbl offset 0) t2 on (t1.q2 = t2.q1) group by t1.q2 order by 1; select t1.q2, count(t2.*) from int8_tbl t1 left join (select q1, case when q2=1 then 1 else q2 end as q2 from int8_tbl) t2 on (t1.q2 = t2.q1) group by t1.q2 order by 1; -- -- test incorrect failure to NULL pulled-up subexpressions -- begin; create temp table a ( code char not null, constraint a_pk primary key (code) ); create temp table b ( a char not null, num integer not null, constraint b_pk primary key (a, num) ); create temp table c ( name char not null, a char, constraint c_pk primary key (name) ); insert into a (code) values ('p'); insert into a (code) values ('q'); insert into b (a, num) values ('p', 1); insert into b (a, num) values ('p', 2); insert into c (name, a) values ('A', 'p'); insert into c (name, a) values ('B', 'q'); insert into c (name, a) values ('C', null); select c.name, ss.code, ss.b_cnt, ss.const from c left join (select a.code, coalesce(b_grp.cnt, 0) as b_cnt, -1 as const from a left join (select count(1) as cnt, b.a from b group by b.a) as b_grp on a.code = b_grp.a ) as ss on (c.a = ss.code) order by c.name; rollback; -- -- test incorrect handling of placeholders that only appear in targetlists, -- per bug #6154 -- SELECT * FROM ( SELECT 1 as key1 ) sub1 LEFT JOIN ( SELECT sub3.key3, sub4.value2, COALESCE(sub4.value2, 66) as value3 FROM ( SELECT 1 as key3 ) sub3 LEFT JOIN ( SELECT sub5.key5, COALESCE(sub6.value1, 1) as value2 FROM ( SELECT 1 as key5 ) sub5 LEFT JOIN ( SELECT 2 as key6, 42 as value1 ) sub6 ON sub5.key5 = sub6.key6 ) sub4 ON sub4.key5 = sub3.key3 ) sub2 ON sub1.key1 = sub2.key3; -- test the path using join aliases, too SELECT * FROM ( SELECT 1 as key1 ) sub1 LEFT JOIN ( SELECT sub3.key3, value2, COALESCE(value2, 66) as value3 FROM ( SELECT 1 as key3 ) sub3 LEFT JOIN ( SELECT sub5.key5, COALESCE(sub6.value1, 1) as value2 FROM ( SELECT 1 as key5 ) sub5 LEFT JOIN ( SELECT 2 as key6, 42 as value1 ) sub6 ON sub5.key5 = sub6.key6 ) sub4 ON sub4.key5 = sub3.key3 ) sub2 ON sub1.key1 = sub2.key3; -- -- test case where a PlaceHolderVar is used as a nestloop parameter -- :PREFIX SELECT qq, unique1 FROM ( SELECT COALESCE(q1, 0) AS qq FROM int8_tbl a ) AS ss1 FULL OUTER JOIN ( SELECT COALESCE(q2, -1) AS qq FROM int8_tbl b ) AS ss2 USING (qq) INNER JOIN tenk1 c ON qq = unique2; -- -- nested nestloops can require nested PlaceHolderVars -- create table nt1 ( id int primary key, a1 boolean, a2 boolean ); create table nt2 ( id int primary key, nt1_id int, b1 boolean, b2 boolean -- foreign key (nt1_id) references nt1(id) ); create table nt3 ( id int primary key, nt2_id int, c1 boolean -- foreign key (nt2_id) references nt2(id) ); select (select table_name from create_hypertable(tbl,'id',chunk_time_interval:=1000)) from (VALUES ('nt1'),('nt2'),('nt3')) v(tbl); insert into nt1 values (1,true,true); insert into nt1 values (2,true,false); insert into nt1 values (3,false,false); insert into nt2 values (1,1,true,true); insert into nt2 values (2,2,true,false); insert into nt2 values (3,3,false,false); insert into nt3 values (1,1,true); insert into nt3 values (2,2,false); insert into nt3 values (3,3,true); :PREFIX select nt3.id from nt3 as nt3 left join (select nt2.*, (nt2.b1 and ss1.a3) AS b3 from nt2 as nt2 left join (select nt1.*, (nt1.id is not null) as a3 from nt1) as ss1 on ss1.id = nt2.nt1_id ) as ss2 on ss2.id = nt3.nt2_id where nt3.id = 1 and ss2.b3; drop table nt1; drop table nt2; drop table nt3; -- -- test case where a PlaceHolderVar is propagated into a subquery -- :PREFIX select * from int8_tbl t1 left join (select q1 as x, 42 as y from int8_tbl t2) ss on t1.q2 = ss.x where 1 = (select 1 from int8_tbl t3 where ss.y is not null limit 1) order by 1,2; -- -- test the corner cases FULL JOIN ON TRUE and FULL JOIN ON FALSE -- select * from int4_tbl a full join int4_tbl b on true; select * from int4_tbl a full join int4_tbl b on false; -- -- test for ability to use a cartesian join when necessary -- :PREFIX select * from tenk1 join int4_tbl on f1 = twothousand, int4(sin(1)) q1, int4(sin(0)) q2 where q1 = thousand or q2 = thousand; :PREFIX select * from tenk1 join int4_tbl on f1 = twothousand, int4(sin(1)) q1, int4(sin(0)) q2 where thousand = (q1 + q2); -- -- test ability to generate a suitable plan for a star-schema query -- :PREFIX select * from tenk1, int8_tbl a, int8_tbl b where thousand = a.q1 and tenthous = b.q1 and a.q2 = 1 and b.q2 = 2; -- -- test a corner case in which we shouldn't apply the star-schema optimization -- :PREFIX select t1.unique2, t1.stringu1, t2.unique1, t2.stringu2 from tenk1 t1 inner join int4_tbl i1 left join (select v1.x2, v2.y1, 11 AS d1 from (values(1,0)) v1(x1,x2) left join (values(3,1)) v2(y1,y2) on v1.x1 = v2.y2) subq1 on (i1.f1 = subq1.x2) on (t1.unique2 = subq1.d1) left join tenk1 t2 on (subq1.y1 = t2.unique1) where t1.unique2 < 42 and t1.stringu1 > t2.stringu2; -- variant that isn't quite a star-schema case :PREFIX select ss1.d1 from tenk1 as t1 inner join tenk1 as t2 on t1.tenthous = t2.ten inner join int8_tbl as i8 left join int4_tbl as i4 inner join (select 64::information_schema.cardinal_number as d1 from tenk1 t3, lateral (select abs(t3.unique1) + random()) ss0(x) where t3.fivethous < 0) as ss1 on i4.f1 = ss1.d1 on i8.q1 = i4.f1 on t1.tenthous = ss1.d1 where t1.unique1 < i4.f1; -- -- test extraction of restriction OR clauses from join OR clause -- (we used to only do this for indexable clauses) -- :PREFIX select * from tenk1 a join tenk1 b on (a.unique1 = 1 and b.unique1 = 2) or (a.unique2 = 3 and b.hundred = 4) ORDER BY a.unique2,b.unique2; :PREFIX select * from tenk1 a join tenk1 b on (a.unique1 = 1 and b.unique1 = 2) or (a.unique2 = 3 and b.ten = 4) ORDER BY a.unique2,b.unique2; :PREFIX select * from tenk1 a join tenk1 b on (a.unique1 = 1 and b.unique1 = 2) or ((a.unique2 = 3 or a.unique2 = 7) and b.hundred = 4) ORDER BY a.unique2,b.unique2; -- -- test placement of movable quals in a parameterized join tree -- :PREFIX select * from tenk1 t1 left join (tenk1 t2 join tenk1 t3 on t2.thousand = t3.unique2) on t1.hundred = t2.hundred and t1.ten = t3.ten where t1.unique1 = 1 ORDER BY t1,t2,t3; :PREFIX select * from tenk1 t1 left join (tenk1 t2 join tenk1 t3 on t2.thousand = t3.unique2) on t1.hundred = t2.hundred and t1.ten + t2.ten = t3.ten where t1.unique1 = 1; :PREFIX select count(*) from tenk1 a join tenk1 b on a.unique1 = b.unique2 left join tenk1 c on a.unique2 = b.unique1 and c.thousand = a.thousand join int4_tbl on b.thousand = f1; :PREFIX select b.unique1 from tenk1 a join tenk1 b on a.unique1 = b.unique2 left join tenk1 c on b.unique1 = 42 and c.thousand = a.thousand join int4_tbl i1 on b.thousand = f1 right join int4_tbl i2 on i2.f1 = b.tenthous order by 1; :PREFIX select * from ( select unique1, q1, coalesce(unique1, -1) + q1 as fault from int8_tbl left join tenk1 on (q2 = unique2) ) ss where fault = 122 order by fault; :PREFIX select * from (values (1, array[10,20]), (2, array[20,30])) as v1(v1x,v1ys) left join (values (1, 10), (2, 20)) as v2(v2x,v2y) on v2x = v1x left join unnest(v1ys) as u1(u1y) on u1y = v2y; -- -- test handling of potential equivalence clauses above outer joins -- :PREFIX select q1, unique2, thousand, hundred from int8_tbl a left join tenk1 b on q1 = unique2 where coalesce(thousand,123) = q1 and q1 = coalesce(hundred,123); :PREFIX select f1, unique2, case when unique2 is null then f1 else 0 end from int4_tbl a left join tenk1 b on f1 = unique2 where (case when unique2 is null then f1 else 0 end) = 0; -- -- another case with equivalence clauses above outer joins (bug #8591) -- :PREFIX select a.unique1, b.unique1, c.unique1, coalesce(b.twothousand, a.twothousand) from tenk1 a left join tenk1 b on b.thousand = a.unique1 left join tenk1 c on c.unique2 = coalesce(b.twothousand, a.twothousand) where a.unique2 < 10 and coalesce(b.twothousand, a.twothousand) = 44; -- -- check handling of join aliases when flattening multiple levels of subquery -- :PREFIX select foo1.join_key as foo1_id, foo3.join_key AS foo3_id, bug_field from (values (0),(1)) foo1(join_key) left join (select join_key, bug_field from (select ss1.join_key, ss1.bug_field from (select f1 as join_key, 666 as bug_field from int4_tbl i1) ss1 ) foo2 left join (select unique2 as join_key from tenk1 i2) ss2 using (join_key) ) foo3 using (join_key); -- -- test successful handling of nested outer joins with degenerate join quals -- :PREFIX select t1.* from text_tbl t1 left join (select *, '***'::text as d1 from int8_tbl i8b1) b1 left join int8_tbl i8 left join (select *, null::int as d2 from int8_tbl i8b2) b2 on (i8.q1 = b2.q1) on (b2.d2 = b1.q2) on (t1.f1 = b1.d1) left join int4_tbl i4 on (i8.q2 = i4.f1); :PREFIX select t1.* from text_tbl t1 left join (select *, '***'::text as d1 from int8_tbl i8b1) b1 left join int8_tbl i8 left join (select *, null::int as d2 from int8_tbl i8b2, int4_tbl i4b2) b2 on (i8.q1 = b2.q1) on (b2.d2 = b1.q2) on (t1.f1 = b1.d1) left join int4_tbl i4 on (i8.q2 = i4.f1); :PREFIX select t1.* from text_tbl t1 left join (select *, '***'::text as d1 from int8_tbl i8b1) b1 left join int8_tbl i8 left join (select *, null::int as d2 from int8_tbl i8b2, int4_tbl i4b2 where q1 = f1) b2 on (i8.q1 = b2.q1) on (b2.d2 = b1.q2) on (t1.f1 = b1.d1) left join int4_tbl i4 on (i8.q2 = i4.f1); :PREFIX select * from text_tbl t1 inner join int8_tbl i8 on i8.q2 = 456 right join text_tbl t2 on t1.f1 = 'doh!' left join int4_tbl i4 on i8.q1 = i4.f1; -- -- test for appropriate join order in the presence of lateral references -- :PREFIX select * from text_tbl t1 left join int8_tbl i8 on i8.q2 = 123, lateral (select i8.q1, t2.f1 from text_tbl t2 limit 1) as ss where t1.f1 = ss.f1; :PREFIX select * from text_tbl t1 left join int8_tbl i8 on i8.q2 = 123, lateral (select i8.q1, t2.f1 from text_tbl t2 limit 1) as ss1, lateral (select ss1.* from text_tbl t3 limit 1) as ss2 where t1.f1 = ss2.f1; :PREFIX select 1 from text_tbl as tt1 inner join text_tbl as tt2 on (tt1.f1 = 'foo') left join text_tbl as tt3 on (tt3.f1 = 'foo') left join text_tbl as tt4 on (tt3.f1 = tt4.f1), lateral (select tt4.f1 as c0 from text_tbl as tt5 limit 1) as ss1 where tt1.f1 = ss1.c0; -- -- check a case in which a PlaceHolderVar forces join order -- :PREFIX select ss2.* from int4_tbl i41 left join int8_tbl i8 join (select i42.f1 as c1, i43.f1 as c2, 42 as c3 from int4_tbl i42, int4_tbl i43) ss1 on i8.q1 = ss1.c2 on i41.f1 = ss1.c1, lateral (select i41.*, i8.*, ss1.* from text_tbl limit 1) ss2 where ss1.c2 = 0; -- -- test successful handling of full join underneath left join (bug #14105) -- :PREFIX select * from (select 1 as id) as xx left join (tenk1 as a1 full join (select 1 as id) as yy on (a1.unique1 = yy.id)) on (xx.id = coalesce(yy.id)); -- -- test ability to push constants through outer join clauses -- :PREFIX select * from int4_tbl a left join tenk1 b on f1 = unique2 where f1 = 0; :PREFIX select * from tenk1 a full join tenk1 b using(unique2) where unique2 = 42; -- -- test that quals attached to an outer join have correct semantics, -- specifically that they don't re-use expressions computed below the join; -- we force a mergejoin so that coalesce(b.q1, 1) appears as a join input -- set enable_hashjoin to off; set enable_nestloop to off; :PREFIX select a.q2, b.q1 from int8_tbl a left join int8_tbl b on a.q2 = coalesce(b.q1, 1) where coalesce(b.q1, 1) > 0; reset enable_hashjoin; reset enable_nestloop; -- -- test join removal -- begin; CREATE TEMP TABLE a (id int PRIMARY KEY, b_id int); CREATE TEMP TABLE b (id int PRIMARY KEY, c_id int); CREATE TEMP TABLE c (id int PRIMARY KEY); CREATE TEMP TABLE d (a int, b int); INSERT INTO a VALUES (0, 0), (1, NULL); INSERT INTO b VALUES (0, 0), (1, NULL); INSERT INTO c VALUES (0), (1); INSERT INTO d VALUES (1,3), (2,2), (3,1); -- all three cases should be optimizable into a simple seqscan :PREFIX SELECT a.* FROM a LEFT JOIN b ON a.b_id = b.id; :PREFIX SELECT b.* FROM b LEFT JOIN c ON b.c_id = c.id; :PREFIX SELECT a.* FROM a LEFT JOIN (b left join c on b.c_id = c.id) ON (a.b_id = b.id); -- check optimization of outer join within another special join :PREFIX select id from a where id in ( select b.id from b left join c on b.id = c.id ); -- check that join removal works for a left join when joining a subquery -- that is guaranteed to be unique by its GROUP BY clause :PREFIX select d.* from d left join (select * from b group by b.id, b.c_id) s on d.a = s.id and d.b = s.c_id; -- similarly, but keying off a DISTINCT clause :PREFIX select d.* from d left join (select distinct * from b) s on d.a = s.id and d.b = s.c_id; -- join removal is not possible when the GROUP BY contains a column that is -- not in the join condition. (Note: as of 9.6, we notice that b.id is a -- primary key and so drop b.c_id from the GROUP BY of the resulting plan; -- but this happens too late for join removal in the outer plan level.) :PREFIX select d.* from d left join (select * from b group by b.id, b.c_id) s on d.a = s.id; -- similarly, but keying off a DISTINCT clause :PREFIX select d.* from d left join (select distinct * from b) s on d.a = s.id; -- check join removal works when uniqueness of the join condition is enforced -- by a UNION :PREFIX select d.* from d left join (select id from a union select id from b) s on d.a = s.id; -- check join removal with a cross-type comparison operator :PREFIX select i8.* from int8_tbl i8 left join (select f1 from int4_tbl group by f1) i4 on i8.q1 = i4.f1; -- check join removal with lateral references :PREFIX select 1 from (select a.id FROM a left join b on a.b_id = b.id) q, lateral generate_series(1, q.id) gs(i) where q.id = gs.i; rollback; create table parent (k int primary key, pd int); create table child (k int unique, cd int); select table_name from create_hypertable('parent','k',chunk_time_interval:=1000); select table_name from create_hypertable('child','k',chunk_time_interval:=1000); insert into parent values (1, 10), (2, 20), (3, 30); insert into child values (1, 100), (4, 400); -- this case is optimizable select p.* from parent p left join child c on (p.k = c.k); :PREFIX select p.* from parent p left join child c on (p.k = c.k); -- this case is not select p.*, linked from parent p left join (select c.*, true as linked from child c) as ss on (p.k = ss.k); :PREFIX select p.*, linked from parent p left join (select c.*, true as linked from child c) as ss on (p.k = ss.k); -- check for a 9.0rc1 bug: join removal breaks pseudoconstant qual handling select p.* from parent p left join child c on (p.k = c.k) where p.k = 1 and p.k = 2; :PREFIX select p.* from parent p left join child c on (p.k = c.k) where p.k = 1 and p.k = 2; select p.* from (parent p left join child c on (p.k = c.k)) join parent x on p.k = x.k where p.k = 1 and p.k = 2; :PREFIX select p.* from (parent p left join child c on (p.k = c.k)) join parent x on p.k = x.k where p.k = 1 and p.k = 2; drop table parent; drop table child; -- bug 5255: this is not optimizable by join removal begin; CREATE TEMP TABLE a (id int PRIMARY KEY); CREATE TEMP TABLE b (id int PRIMARY KEY, a_id int); INSERT INTO a VALUES (0), (1); INSERT INTO b VALUES (0, 0), (1, NULL); SELECT * FROM b LEFT JOIN a ON (b.a_id = a.id) WHERE (a.id IS NULL OR a.id > 0); SELECT b.* FROM b LEFT JOIN a ON (b.a_id = a.id) WHERE (a.id IS NULL OR a.id > 0); rollback; -- another join removal bug: this is not optimizable, either begin; create temp table innertab (id int8 primary key, dat1 int8); insert into innertab values(123, 42); SELECT * FROM (SELECT 1 AS x) ss1 LEFT JOIN (SELECT q1, q2, COALESCE(dat1, q1) AS y FROM int8_tbl LEFT JOIN innertab ON q2 = id) ss2 ON true; rollback; -- another join removal bug: we must clean up correctly when removing a PHV begin; create temp table uniquetbl (f1 text unique); :PREFIX select t1.* from uniquetbl as t1 left join (select *, '***'::text as d1 from uniquetbl) t2 on t1.f1 = t2.f1 left join uniquetbl t3 on t2.d1 = t3.f1; :PREFIX select t0.* from text_tbl t0 left join (select case t1.ten when 0 then 'doh!'::text else null::text end as case1, t1.stringu2 from tenk1 t1 join int4_tbl i4 ON i4.f1 = t1.unique2 left join uniquetbl u1 ON u1.f1 = t1.string4) ss on t0.f1 = ss.case1 where ss.stringu2 !~* ss.case1; select t0.* from text_tbl t0 left join (select case t1.ten when 0 then 'doh!'::text else null::text end as case1, t1.stringu2 from tenk1 t1 join int4_tbl i4 ON i4.f1 = t1.unique2 left join uniquetbl u1 ON u1.f1 = t1.string4) ss on t0.f1 = ss.case1 where ss.stringu2 !~* ss.case1; rollback; -- bug #8444: we've historically allowed duplicate aliases within aliased JOINs -- select * from -- int8_tbl x join (int4_tbl x cross join int4_tbl y) j on q1 = f1; -- error -- select * from -- int8_tbl x join (int4_tbl x cross join int4_tbl y) j on q1 = y.f1; -- error select * from int8_tbl x join (int4_tbl x cross join int4_tbl y(ff)) j on q1 = f1; -- ok -- -- Test hints given on incorrect column references are useful -- -- select t1.uunique1 from -- tenk1 t1 join tenk2 t2 on t1.two = t2.two; -- error, prefer "t1" suggestion -- select t2.uunique1 from -- tenk1 t1 join tenk2 t2 on t1.two = t2.two; -- error, prefer "t2" suggestion -- select uunique1 from -- tenk1 t1 join tenk2 t2 on t1.two = t2.two; -- error, suggest both at once -- -- Take care to reference the correct RTE -- -- select atts.relid::regclass, s.* from pg_stats s join -- pg_attribute a on s.attname = a.attname and s.tablename = -- a.attrelid::regclass::text join (select unnest(indkey) attnum, -- indexrelid from pg_index i) atts on atts.attnum = a.attnum where -- schemaname != 'pg_catalog'; -- -- Test LATERAL -- select unique2, x.* from tenk1 a, lateral (select * from int4_tbl b where f1 = a.unique1) x; :PREFIX select unique2, x.* from tenk1 a, lateral (select * from int4_tbl b where f1 = a.unique1) x; select unique2, x.* from int4_tbl x, lateral (select unique2 from tenk1 where f1 = unique1) ss; :PREFIX select unique2, x.* from int4_tbl x, lateral (select unique2 from tenk1 where f1 = unique1) ss; :PREFIX select unique2, x.* from int4_tbl x cross join lateral (select unique2 from tenk1 where f1 = unique1) ss; select unique2, x.* from int4_tbl x left join lateral (select unique1, unique2 from tenk1 where f1 = unique1) ss on true; :PREFIX select unique2, x.* from int4_tbl x left join lateral (select unique1, unique2 from tenk1 where f1 = unique1) ss on true; -- check scoping of lateral versus parent references -- the first of these should return int8_tbl.q2, the second int8_tbl.q1 select *, (select r from (select q1 as q2) x, (select q2 as r) y) from int8_tbl; select *, (select r from (select q1 as q2) x, lateral (select q2 as r) y) from int8_tbl; -- lateral with function in FROM select count(*) from tenk1 a, lateral generate_series(1,two) g; :PREFIX select count(*) from tenk1 a, lateral generate_series(1,two) g; :PREFIX select count(*) from tenk1 a cross join lateral generate_series(1,two) g; -- don't need the explicit LATERAL keyword for functions :PREFIX select count(*) from tenk1 a, generate_series(1,two) g; -- lateral with UNION ALL subselect :PREFIX select * from generate_series(100,200) g, lateral (select * from int8_tbl a where g = q1 union all select * from int8_tbl b where g = q2) ss; select * from generate_series(100,200) g, lateral (select * from int8_tbl a where g = q1 union all select * from int8_tbl b where g = q2) ss; -- lateral with VALUES :PREFIX select count(*) from tenk1 a, tenk1 b join lateral (values(a.unique1)) ss(x) on b.unique2 = ss.x; select count(*) from tenk1 a, tenk1 b join lateral (values(a.unique1)) ss(x) on b.unique2 = ss.x; -- lateral with VALUES, no flattening possible :PREFIX select count(*) from tenk1 a, tenk1 b join lateral (values(a.unique1),(-1)) ss(x) on b.unique2 = ss.x; select count(*) from tenk1 a, tenk1 b join lateral (values(a.unique1),(-1)) ss(x) on b.unique2 = ss.x; -- lateral injecting a strange outer join condition :PREFIX select * from int8_tbl a, int8_tbl x left join lateral (select a.q1 from int4_tbl y) ss(z) on x.q2 = ss.z order by a.q1, a.q2, x.q1, x.q2, ss.z; select * from int8_tbl a, int8_tbl x left join lateral (select a.q1 from int4_tbl y) ss(z) on x.q2 = ss.z order by a.q1, a.q2, x.q1, x.q2, ss.z; -- lateral reference to a join alias variable select * from (select f1/2 as x from int4_tbl) ss1 join int4_tbl i4 on x = f1, lateral (select x) ss2(y); select * from (select f1 as x from int4_tbl) ss1 join int4_tbl i4 on x = f1, lateral (values(x)) ss2(y); select * from ((select f1/2 as x from int4_tbl) ss1 join int4_tbl i4 on x = f1) j, lateral (select x) ss2(y); -- lateral references requiring pullup select * from (values(1)) x(lb), lateral generate_series(lb,4) x4; select * from (select f1/1000000000 from int4_tbl) x(lb), lateral generate_series(lb,4) x4; select * from (values(1)) x(lb), lateral (values(lb)) y(lbcopy); select * from (values(1)) x(lb), lateral (select lb from int4_tbl) y(lbcopy); select * from int8_tbl x left join (select q1,coalesce(q2,0) q2 from int8_tbl) y on x.q2 = y.q1, lateral (values(x.q1,y.q1,y.q2)) v(xq1,yq1,yq2); select * from int8_tbl x left join (select q1,coalesce(q2,0) q2 from int8_tbl) y on x.q2 = y.q1, lateral (select x.q1,y.q1,y.q2) v(xq1,yq1,yq2); select x.* from int8_tbl x left join (select q1,coalesce(q2,0) q2 from int8_tbl) y on x.q2 = y.q1, lateral (select x.q1,y.q1,y.q2) v(xq1,yq1,yq2); select v.* from (int8_tbl x left join (select q1,coalesce(q2,0) q2 from int8_tbl) y on x.q2 = y.q1) left join int4_tbl z on z.f1 = x.q2, lateral (select x.q1,y.q1 union all select x.q2,y.q2) v(vx,vy); select v.* from (int8_tbl x left join (select q1,(select coalesce(q2,0)) q2 from int8_tbl) y on x.q2 = y.q1) left join int4_tbl z on z.f1 = x.q2, lateral (select x.q1,y.q1 union all select x.q2,y.q2) v(vx,vy); create temp table dual(); insert into dual default values; analyze dual; select v.* from (int8_tbl x left join (select q1,(select coalesce(q2,0)) q2 from int8_tbl) y on x.q2 = y.q1) left join int4_tbl z on z.f1 = x.q2, lateral (select x.q1,y.q1 from dual union all select x.q2,y.q2 from dual) v(vx,vy); drop table dual; :PREFIX select * from int8_tbl a left join lateral (select *, a.q2 as x from int8_tbl b) ss on a.q2 = ss.q1; select * from int8_tbl a left join lateral (select *, a.q2 as x from int8_tbl b) ss on a.q2 = ss.q1; :PREFIX select * from int8_tbl a left join lateral (select *, coalesce(a.q2, 42) as x from int8_tbl b) ss on a.q2 = ss.q1; select * from int8_tbl a left join lateral (select *, coalesce(a.q2, 42) as x from int8_tbl b) ss on a.q2 = ss.q1; -- lateral can result in join conditions appearing below their -- real semantic level :PREFIX select * from int4_tbl i left join lateral (select * from int2_tbl j where i.f1 = j.f1) k on true; select * from int4_tbl i left join lateral (select * from int2_tbl j where i.f1 = j.f1) k on true; :PREFIX select * from int4_tbl i left join lateral (select coalesce(i) from int2_tbl j where i.f1 = j.f1) k on true; select * from int4_tbl i left join lateral (select coalesce(i) from int2_tbl j where i.f1 = j.f1) k on true; :PREFIX select * from int4_tbl a, lateral ( select * from int4_tbl b left join int8_tbl c on (b.f1 = q1 and a.f1 = q2) ) ss; select * from int4_tbl a, lateral ( select * from int4_tbl b left join int8_tbl c on (b.f1 = q1 and a.f1 = q2) ) ss; -- lateral reference in a PlaceHolderVar evaluated at join level :PREFIX select * from int8_tbl a left join lateral (select b.q1 as bq1, c.q1 as cq1, least(a.q1,b.q1,c.q1) from int8_tbl b cross join int8_tbl c) ss on a.q2 = ss.bq1; select * from int8_tbl a left join lateral (select b.q1 as bq1, c.q1 as cq1, least(a.q1,b.q1,c.q1) from int8_tbl b cross join int8_tbl c) ss on a.q2 = ss.bq1; -- case requiring nested PlaceHolderVars :PREFIX select * from int8_tbl c left join ( int8_tbl a left join (select q1, coalesce(q2,42) as x from int8_tbl b) ss1 on a.q2 = ss1.q1 cross join lateral (select q1, coalesce(ss1.x,q2) as y from int8_tbl d) ss2 ) on c.q2 = ss2.q1, lateral (select ss2.y offset 0) ss3; -- case that breaks the old ph_may_need optimization :PREFIX select c.*,a.*,ss1.q1,ss2.q1,ss3.* from int8_tbl c left join ( int8_tbl a left join (select q1, coalesce(q2,f1) as x from int8_tbl b, int4_tbl b2 where q1 < f1) ss1 on a.q2 = ss1.q1 cross join lateral (select q1, coalesce(ss1.x,q2) as y from int8_tbl d) ss2 ) on c.q2 = ss2.q1, lateral (select * from int4_tbl i where ss2.y > f1) ss3; -- check processing of postponed quals (bug #9041) :PREFIX select * from (select 1 as x offset 0) x cross join (select 2 as y offset 0) y left join lateral ( select * from (select 3 as z offset 0) z where z.z = x.x ) zz on zz.z = y.y; -- check dummy rels with lateral references (bug #15694) :PREFIX select * from int8_tbl i8 left join lateral (select *, i8.q2 from int4_tbl where false) ss on true; :PREFIX select * from int8_tbl i8 left join lateral (select *, i8.q2 from int4_tbl i1, int4_tbl i2 where false) ss on true; -- check handling of nested appendrels inside LATERAL select * from ((select 2 as v) union all (select 3 as v)) as q1 cross join lateral ((select * from ((select 4 as v) union all (select 5 as v)) as q3) union all (select q1.v) ) as q2; -- check we don't try to do a unique-ified semijoin with LATERAL :PREFIX select * from (values (0,9998), (1,1000)) v(id,x), lateral (select f1 from int4_tbl where f1 = any (select unique1 from tenk1 where unique2 = v.x offset 0)) ss; select * from (values (0,9998), (1,1000)) v(id,x), lateral (select f1 from int4_tbl where f1 = any (select unique1 from tenk1 where unique2 = v.x offset 0)) ss; -- check proper extParam/allParam handling (this isn't exactly a LATERAL issue, -- but we can make the test case much more compact with LATERAL) :PREFIX select * from (values (0), (1)) v(id), lateral (select * from int8_tbl t1, lateral (select * from (select * from int8_tbl t2 where q1 = any (select q2 from int8_tbl t3 where q2 = (select greatest(t1.q1,t2.q2)) and (select v.id=0)) offset 0) ss2) ss where t1.q1 = ss.q2) ss0; select * from (values (0), (1)) v(id), lateral (select * from int8_tbl t1, lateral (select * from (select * from int8_tbl t2 where q1 = any (select q2 from int8_tbl t3 where q2 = (select greatest(t1.q1,t2.q2)) and (select v.id=0)) offset 0) ss2) ss where t1.q1 = ss.q2) ss0; -- test some error cases where LATERAL should have been used but wasn't -- select f1,g from int4_tbl a, (select f1 as g) ss; -- select f1,g from int4_tbl a, (select a.f1 as g) ss; -- select f1,g from int4_tbl a cross join (select f1 as g) ss; -- select f1,g from int4_tbl a cross join (select a.f1 as g) ss; -- SQL:2008 says the left table is in scope but illegal to access here -- select f1,g from int4_tbl a right join lateral generate_series(0, a.f1) g on true; -- select f1,g from int4_tbl a full join lateral generate_series(0, a.f1) g on true; -- check we complain about ambiguous table references -- select * from -- int8_tbl x cross join (int4_tbl x cross join lateral (select x.f1) ss); -- LATERAL can be used to put an aggregate into the FROM clause of its query -- select 1 from tenk1 a, lateral (select max(a.unique1) from int4_tbl b) ss; -- check behavior of LATERAL in UPDATE/DELETE -- create temp table xx1 as select f1 as x1, -f1 as x2 from int4_tbl; -- error, can't do this: -- update xx1 set x2 = f1 from (select * from int4_tbl where f1 = x1) ss; -- update xx1 set x2 = f1 from (select * from int4_tbl where f1 = xx1.x1) ss; -- can't do it even with LATERAL: -- update xx1 set x2 = f1 from lateral (select * from int4_tbl where f1 = x1) ss; -- we might in future allow something like this, but for now it's an error: -- update xx1 set x2 = f1 from xx1, lateral (select * from int4_tbl where f1 = x1) ss; -- also errors: -- delete from xx1 using (select * from int4_tbl where f1 = x1) ss; -- delete from xx1 using (select * from int4_tbl where f1 = xx1.x1) ss; -- delete from xx1 using lateral (select * from int4_tbl where f1 = x1) ss; -- -- test LATERAL reference propagation down a multi-level inheritance hierarchy -- produced for a multi-level partitioned table hierarchy. -- create table join_pt1 (a int, b int, c varchar) partition by range(a); create table join_pt1p1 partition of join_pt1 for values from (0) to (100) partition by range(b); create table join_pt1p2 partition of join_pt1 for values from (100) to (200); create table join_pt1p1p1 partition of join_pt1p1 for values from (0) to (100); insert into join_pt1 values (1, 1, 'x'), (101, 101, 'y'); create table join_ut1 (a int, b int, c varchar); insert into join_ut1 values (101, 101, 'y'), (2, 2, 'z'); :PREFIX select t1.b, ss.phv from join_ut1 t1 left join lateral (select t2.a as t2a, t3.a t3a, least(t1.a, t2.a, t3.a) phv from join_pt1 t2 join join_ut1 t3 on t2.a = t3.b) ss on t1.a = ss.t2a order by t1.a; select t1.b, ss.phv from join_ut1 t1 left join lateral (select t2.a as t2a, t3.a t3a, least(t1.a, t2.a, t3.a) phv from join_pt1 t2 join join_ut1 t3 on t2.a = t3.b) ss on t1.a = ss.t2a order by t1.a; drop table join_pt1; drop table join_ut1; -- -- test that foreign key join estimation performs sanely for outer joins -- begin; create table fkest (a int, b int, c int unique, primary key(a,b)); create table fkest1 (a int, b int, primary key(a,b)); insert into fkest select x/10, x%10, x from generate_series(1,1000) x; insert into fkest1 select x/10, x%10 from generate_series(1,1000) x; alter table fkest1 add constraint fkest1_a_b_fkey foreign key (a,b) references fkest; analyze fkest; analyze fkest1; :PREFIX select * from fkest f left join fkest1 f1 on f.a = f1.a and f.b = f1.b left join fkest1 f2 on f.a = f2.a and f.b = f2.b left join fkest1 f3 on f.a = f3.a and f.b = f3.b where f.c = 1; rollback; -- -- test planner's ability to mark joins as unique -- create table j1 (id int primary key); create table j2 (id int primary key); create table j3 (id int); SELECT (SELECT table_name FROM create_hypertable(tbl,'id',chunk_time_interval:=1000)) FROM (VALUES ('j1'),('j2'),('j3')) v(tbl); insert into j1 values(1),(2),(3); insert into j2 values(1),(2),(3); insert into j3 values(1),(1); analyze j1; analyze j2; analyze j3; -- ensure join is properly marked as unique :PREFIX select * from j1 inner join j2 on j1.id = j2.id; -- ensure join is not unique when not an equi-join :PREFIX select * from j1 inner join j2 on j1.id > j2.id; -- ensure non-unique rel is not chosen as inner :PREFIX select * from j1 inner join j3 on j1.id = j3.id; -- ensure left join is marked as unique :PREFIX select * from j1 left join j2 on j1.id = j2.id; -- ensure right join is marked as unique :PREFIX select * from j1 right join j2 on j1.id = j2.id; -- ensure full join is marked as unique :PREFIX select * from j1 full join j2 on j1.id = j2.id; -- a clauseless (cross) join can't be unique :PREFIX select * from j1 cross join j2; -- ensure a natural join is marked as unique :PREFIX select * from j1 natural join j2; -- ensure a distinct clause allows the inner to become unique :PREFIX select * from j1 inner join (select distinct id from j3) j3 on j1.id = j3.id; -- ensure group by clause allows the inner to become unique :PREFIX select * from j1 inner join (select id from j3 group by id) j3 on j1.id = j3.id; drop table j1; drop table j2; drop table j3; -- test more complex permutations of unique joins create table j1 (id1 int, id2 int, primary key(id1,id2)); create table j2 (id1 int, id2 int, primary key(id1,id2)); create table j3 (id1 int, id2 int, primary key(id1,id2)); SELECT (SELECT table_name FROM create_hypertable(tbl,'id1',chunk_time_interval:=1000,create_default_indexes:=false)) FROM (VALUES ('j1'),('j2'),('j3')) v(tbl); insert into j1 values(1,1),(1,2); insert into j2 values(1,1); insert into j3 values(1,1); analyze j1; analyze j2; analyze j3; -- ensure there's no unique join when not all columns which are part of the -- unique index are seen in the join clause :PREFIX select * from j1 inner join j2 on j1.id1 = j2.id1; -- ensure proper unique detection with multiple join quals :PREFIX select * from j1 inner join j2 on j1.id1 = j2.id1 and j1.id2 = j2.id2; -- ensure we don't detect the join to be unique when quals are not part of the -- join condition :PREFIX select * from j1 inner join j2 on j1.id1 = j2.id1 where j1.id2 = 1; -- as above, but for left joins. :PREFIX select * from j1 left join j2 on j1.id1 = j2.id1 where j1.id2 = 1; -- validate logic in merge joins which skips mark and restore. -- it should only do this if all quals which were used to detect the unique -- are present as join quals, and not plain quals. set enable_nestloop to 0; set enable_hashjoin to 0; set enable_sort to 0; -- create an index that will be preferred over the PK to perform the join create index j1_id1_idx on j1 (id1) where id1 % 1000 = 1; :PREFIX select * from j1 j1 inner join j1 j2 on j1.id1 = j2.id1 and j1.id2 = j2.id2 where j1.id1 % 1000 = 1 and j2.id1 % 1000 = 1; select * from j1 j1 inner join j1 j2 on j1.id1 = j2.id1 and j1.id2 = j2.id2 where j1.id1 % 1000 = 1 and j2.id1 % 1000 = 1; reset enable_nestloop; reset enable_hashjoin; reset enable_sort; drop table j1; drop table j2; drop table j3; -- check that semijoin inner is not seen as unique for a portion of the outerrel :PREFIX select t1.unique1, t2.hundred from onek t1, tenk1 t2 where exists (select 1 from tenk1 t3 where t3.thousand = t1.unique1 and t3.tenthous = t2.hundred) and t1.unique1 < 1; -- ... unless it actually is unique create table j3 as select unique1, tenthous from onek LIMIT 0; select table_name FROM create_hypertable('j3','unique1',chunk_time_interval:=10000); insert into j3 SELECT unique1, tenthous from onek; vacuum analyze j3; create unique index on j3(unique1, tenthous); :PREFIX select t1.unique1, t2.hundred from onek t1, tenk1 t2 where exists (select 1 from j3 where j3.unique1 = t1.unique1 and j3.tenthous = t2.hundred) and t1.unique1 < 1; drop table j3; -- -- exercises for the hash join code -- begin; set local min_parallel_table_scan_size = 0; set local parallel_setup_cost = 0; set local enable_hashjoin = on; -- Extract bucket and batch counts from an explain analyze plan. In -- general we can't make assertions about how many batches (or -- buckets) will be required because it can vary, but we can in some -- special cases and we can check for growth. create or replace function find_hash(node json) returns json language plpgsql as $$ declare x json; child json; begin if node->>'Node Type' = 'Hash' then return node; else for child in select json_array_elements(node->'Plans') loop x := find_hash(child); if x is not null then return x; end if; end loop; return null; end if; end; $$; create or replace function hash_join_batches(query text) returns table (original int, final int) language plpgsql as $$ declare whole_plan json; hash_node json; begin for whole_plan in execute 'explain (analyze, format ''json'') ' || query loop hash_node := find_hash(json_extract_path(whole_plan, '0', 'Plan')); original := hash_node->>'Original Hash Batches'; final := hash_node->>'Hash Batches'; return next; end loop; end; $$; -- Make a simple relation with well distributed keys and correctly -- estimated size. create table simple(id int, t text); select table_name from create_hypertable('simple','id',chunk_time_interval:=5000); insert into simple select generate_series(1, 20000) AS id, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; alter table simple set (parallel_workers = 2); analyze simple; -- Make a relation whose size we will under-estimate. We want stats -- to say 1000 rows, but actually there are 20,000 rows. create table bigger_than_it_looks(id int, t text); select table_name from create_hypertable('bigger_than_it_looks','id',chunk_time_interval:=5000); insert into bigger_than_it_looks select generate_series(1, 20000) as id, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; alter table bigger_than_it_looks set (autovacuum_enabled = 'false'); alter table bigger_than_it_looks set (parallel_workers = 2); analyze bigger_than_it_looks; update pg_class set reltuples = 1000 where relname = 'bigger_than_it_looks'; -- Make a relation whose size we underestimate and that also has a -- kind of skew that breaks our batching scheme. We want stats to say -- 2 rows, but actually there are 20,000 rows with the same key. create table extremely_skewed (id int, t text); select table_name from create_hypertable('extremely_skewed','id',chunk_time_interval:=5000); alter table extremely_skewed set (autovacuum_enabled = 'false'); alter table extremely_skewed set (parallel_workers = 2); analyze extremely_skewed; insert into extremely_skewed select 42 as id, 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' from generate_series(1, 20000); update pg_class set reltuples = 2, relpages = pg_relation_size('extremely_skewed') / 8192 where relname = 'extremely_skewed'; -- Make a relation with a couple of enormous tuples. create table wide(id int, t text); select table_name from create_hypertable('wide','id',chunk_time_interval:=5000); insert into wide select generate_series(1, 10) as id, rpad('', 320000, 'x') as t; alter table wide set (parallel_workers = 2); -- update statistics analyze wide; -- The "optimal" case: the hash table fits in memory; we plan for 1 -- batch, we stick to that number, and peak memory usage stays within -- our work_mem budget -- non-parallel savepoint settings; set local max_parallel_workers_per_gather = 0; set local work_mem = '4MB'; :PREFIX select count(*) from simple r join simple s using (id); select count(*) from simple r join simple s using (id); select original > 1 as initially_multibatch, final > original as increased_batches from hash_join_batches( $$ select count(*) from simple r join simple s using (id); $$); rollback to settings; -- parallel with parallel-oblivious hash join savepoint settings; set local max_parallel_workers_per_gather = 2; set local work_mem = '4MB'; set local enable_parallel_hash = off; :PREFIX select count(*) from simple r join simple s using (id); select count(*) from simple r join simple s using (id); select original > 1 as initially_multibatch, final > original as increased_batches from hash_join_batches( $$ select count(*) from simple r join simple s using (id); $$); rollback to settings; -- parallel with parallel-aware hash join savepoint settings; set local max_parallel_workers_per_gather = 2; set local work_mem = '4MB'; set local enable_parallel_hash = on; :PREFIX select count(*) from simple r join simple s using (id); select count(*) from simple r join simple s using (id); select original > 1 as initially_multibatch, final > original as increased_batches from hash_join_batches( $$ select count(*) from simple r join simple s using (id); $$); rollback to settings; -- The "good" case: batches required, but we plan the right number; we -- plan for some number of batches, and we stick to that number, and -- peak memory usage says within our work_mem budget -- non-parallel savepoint settings; set local max_parallel_workers_per_gather = 0; set local work_mem = '128kB'; :PREFIX select count(*) from simple r join simple s using (id); select count(*) from simple r join simple s using (id); select original > 1 as initially_multibatch, final > original as increased_batches from hash_join_batches( $$ select count(*) from simple r join simple s using (id); $$); rollback to settings; -- parallel with parallel-oblivious hash join savepoint settings; set local max_parallel_workers_per_gather = 2; set local work_mem = '128kB'; set local enable_parallel_hash = off; :PREFIX select count(*) from simple r join simple s using (id); select count(*) from simple r join simple s using (id); select original > 1 as initially_multibatch, final > original as increased_batches from hash_join_batches( $$ select count(*) from simple r join simple s using (id); $$); rollback to settings; -- parallel with parallel-aware hash join savepoint settings; set local max_parallel_workers_per_gather = 2; set local work_mem = '192kB'; set local enable_parallel_hash = on; :PREFIX select count(*) from simple r join simple s using (id); select count(*) from simple r join simple s using (id); select original > 1 as initially_multibatch, final > original as increased_batches from hash_join_batches( $$ select count(*) from simple r join simple s using (id); $$); rollback to settings; -- The "bad" case: during execution we need to increase number of -- batches; in this case we plan for 1 batch, and increase at least a -- couple of times, and peak memory usage stays within our work_mem -- budget -- non-parallel savepoint settings; set local max_parallel_workers_per_gather = 0; set local work_mem = '128kB'; :PREFIX select count(*) FROM simple r JOIN bigger_than_it_looks s USING (id); select count(*) FROM simple r JOIN bigger_than_it_looks s USING (id); select original > 1 as initially_multibatch, final > original as increased_batches from hash_join_batches( $$ select count(*) FROM simple r JOIN bigger_than_it_looks s USING (id); $$); rollback to settings; -- parallel with parallel-oblivious hash join savepoint settings; set local max_parallel_workers_per_gather = 2; set local work_mem = '128kB'; set local enable_parallel_hash = off; :PREFIX select count(*) from simple r join bigger_than_it_looks s using (id); select count(*) from simple r join bigger_than_it_looks s using (id); select original > 1 as initially_multibatch, final > original as increased_batches from hash_join_batches( $$ select count(*) from simple r join bigger_than_it_looks s using (id); $$); rollback to settings; -- parallel with parallel-aware hash join savepoint settings; set local max_parallel_workers_per_gather = 1; set local work_mem = '192kB'; set local enable_parallel_hash = on; :PREFIX select count(*) from simple r join bigger_than_it_looks s using (id); select count(*) from simple r join bigger_than_it_looks s using (id); select original > 1 as initially_multibatch, final > original as increased_batches from hash_join_batches( $$ select count(*) from simple r join bigger_than_it_looks s using (id); $$); rollback to settings; -- The "ugly" case: increasing the number of batches during execution -- doesn't help, so stop trying to fit in work_mem and hope for the -- best; in this case we plan for 1 batch, increases just once and -- then stop increasing because that didn't help at all, so we blow -- right through the work_mem budget and hope for the best... -- non-parallel savepoint settings; set local max_parallel_workers_per_gather = 0; set local work_mem = '128kB'; set local enable_mergejoin to false; :PREFIX select count(*) from simple r join extremely_skewed s using (id); select count(*) from simple r join extremely_skewed s using (id); select * from hash_join_batches( $$ select count(*) from simple r join extremely_skewed s using (id); $$); rollback to settings; -- parallel with parallel-oblivious hash join savepoint settings; set local max_parallel_workers_per_gather = 2; set local work_mem = '128kB'; set local enable_parallel_hash = off; :PREFIX select count(*) from simple r join extremely_skewed s using (id); select count(*) from simple r join extremely_skewed s using (id); select * from hash_join_batches( $$ select count(*) from simple r join extremely_skewed s using (id); $$); rollback to settings; -- parallel with parallel-aware hash join savepoint settings; set local max_parallel_workers_per_gather = 1; set local work_mem = '128kB'; set local enable_parallel_hash = on; :PREFIX select count(*) from simple r join extremely_skewed s using (id); select count(*) from simple r join extremely_skewed s using (id); select * from hash_join_batches( $$ select count(*) from simple r join extremely_skewed s using (id); $$); rollback to settings; -- A couple of other hash join tests unrelated to work_mem management. -- Check that EXPLAIN ANALYZE has data even if the leader doesn't participate savepoint settings; set local max_parallel_workers_per_gather = 2; set local work_mem = '4MB'; set local parallel_leader_participation = off; select * from hash_join_batches( $$ select count(*) from simple r join simple s using (id); $$); rollback to settings; -- Exercise rescans. We'll turn off parallel_leader_participation so -- that we can check that instrumentation comes back correctly. create table join_foo as select generate_series(1, 3) as id, 'xxxxx'::text as t; alter table join_foo set (parallel_workers = 0); create table join_bar as select generate_series(1, 10000) as id, 'xxxxx'::text as t; alter table join_bar set (parallel_workers = 2); -- update statistics analyze join_foo; analyze join_bar; -- multi-batch with rescan, parallel-oblivious savepoint settings; set enable_parallel_hash = off; set parallel_leader_participation = off; set min_parallel_table_scan_size = 0; set parallel_setup_cost = 0; set parallel_tuple_cost = 0; set max_parallel_workers_per_gather = 2; set enable_material = off; set enable_mergejoin = off; set work_mem = '64kB'; :PREFIX select count(*) from join_foo left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1; select count(*) from join_foo left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1; select final > 1 as multibatch from hash_join_batches( $$ select count(*) from join_foo left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1; $$); rollback to settings; -- single-batch with rescan, parallel-oblivious savepoint settings; set enable_parallel_hash = off; set parallel_leader_participation = off; set min_parallel_table_scan_size = 0; set parallel_setup_cost = 0; set parallel_tuple_cost = 0; set max_parallel_workers_per_gather = 2; set enable_material = off; set enable_mergejoin = off; set work_mem = '4MB'; :PREFIX select count(*) from join_foo left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1; select final > 1 as multibatch from hash_join_batches( $$ select count(*) from join_foo left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1; $$); rollback to settings; -- multi-batch with rescan, parallel-aware savepoint settings; set enable_parallel_hash = on; set parallel_leader_participation = off; set min_parallel_table_scan_size = 0; set parallel_setup_cost = 0; set parallel_tuple_cost = 0; set max_parallel_workers_per_gather = 2; set enable_material = off; set enable_mergejoin = off; set work_mem = '64kB'; :PREFIX select count(*) from join_foo left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1; select count(*) from join_foo left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1; select final > 1 as multibatch from hash_join_batches( $$ select count(*) from join_foo left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1; $$); rollback to settings; -- single-batch with rescan, parallel-aware savepoint settings; set enable_parallel_hash = on; set parallel_leader_participation = off; set min_parallel_table_scan_size = 0; set parallel_setup_cost = 0; set parallel_tuple_cost = 0; set max_parallel_workers_per_gather = 2; set enable_material = off; set enable_mergejoin = off; set work_mem = '4MB'; :PREFIX select count(*) from join_foo left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1; select count(*) from join_foo left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1; select final > 1 as multibatch from hash_join_batches( $$ select count(*) from join_foo left join (select b1.id, b1.t from join_bar b1 join join_bar b2 using (id)) ss on join_foo.id < ss.id + 1 and join_foo.id > ss.id - 1; $$); rollback to settings; -- A full outer join where every record is matched. -- non-parallel savepoint settings; set local max_parallel_workers_per_gather = 0; :PREFIX select count(*) from simple r full outer join simple s using (id); rollback to settings; -- parallelism not possible with parallel-oblivious outer hash join savepoint settings; set local max_parallel_workers_per_gather = 2; :PREFIX select count(*) from simple r full outer join simple s using (id); select count(*) from simple r full outer join simple s using (id); rollback to settings; -- An full outer join where every record is not matched. -- non-parallel savepoint settings; set local max_parallel_workers_per_gather = 0; :PREFIX select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); rollback to settings; -- parallelism not possible with parallel-oblivious outer hash join savepoint settings; set local max_parallel_workers_per_gather = 2; :PREFIX select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); select count(*) from simple r full outer join simple s on (r.id = 0 - s.id); rollback to settings; -- exercise special code paths for huge tuples (note use of non-strict -- expression and left join required to get the detoasted tuple into -- the hash table) -- parallel with parallel-aware hash join (hits ExecParallelHashLoadTuple and -- sts_puttuple oversized tuple cases because it's multi-batch) savepoint settings; set max_parallel_workers_per_gather = 2; set enable_parallel_hash = on; set work_mem = '128kB'; :PREFIX select length(max(s.t)) from wide left join (select id, coalesce(t, '') || '' as t from wide) s using (id); select length(max(s.t)) from wide left join (select id, coalesce(t, '') || '' as t from wide) s using (id); select final > 1 as multibatch from hash_join_batches( $$ select length(max(s.t)) from wide left join (select id, coalesce(t, '') || '' as t from wide) s using (id); $$); rollback to settings; rollback; ================================================ FILE: test/sql/include/plan_expand_hypertable_load.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --single time dimension CREATE TABLE hyper ("time_broken" bigint NOT NULL, "value" integer); ALTER TABLE hyper DROP COLUMN time_broken, ADD COLUMN time BIGINT; SELECT create_hypertable('hyper', 'time', chunk_time_interval => 10); INSERT INTO hyper SELECT g, g FROM generate_series(0,1000) g; --insert a point with INT_MAX_64 INSERT INTO hyper (time, value) SELECT 9223372036854775807::bigint, 0; --time and space CREATE TABLE hyper_w_space ("time_broken" bigint NOT NULL, "device_id" text, "value" integer); ALTER TABLE hyper_w_space DROP COLUMN time_broken, ADD COLUMN time BIGINT; SELECT create_hypertable('hyper_w_space', 'time', 'device_id', 4, chunk_time_interval => 10); INSERT INTO hyper_w_space (time, device_id, value) SELECT g, 'dev' || g, g FROM generate_series(0,30) g; CREATE VIEW hyper_w_space_view AS (SELECT * FROM hyper_w_space); --with timestamp and space CREATE TABLE tag (id serial PRIMARY KEY, name text); CREATE TABLE hyper_ts ("time_broken" timestamptz NOT NULL, "device_id" text, tag_id INT REFERENCES tag(id), "value" integer); ALTER TABLE hyper_ts DROP COLUMN time_broken, ADD COLUMN time TIMESTAMPTZ; SELECT create_hypertable('hyper_ts', 'time', 'device_id', 2, chunk_time_interval => '10 seconds'::interval); INSERT INTO tag(name) SELECT 'tag'||g FROM generate_series(0,10) g; INSERT INTO hyper_ts (time, device_id, tag_id, value) SELECT to_timestamp(g), 'dev' || g, (random() /10)+1, g FROM generate_series(0,30) g; --one in the future INSERT INTO hyper_ts (time, device_id, tag_id, value) VALUES ('2100-01-01 02:03:04 PST', 'dev101', 1, 0); --time partitioning function CREATE OR REPLACE FUNCTION unix_to_timestamp(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT AS $BODY$ SELECT to_timestamp(unixtime); $BODY$; CREATE TABLE hyper_timefunc ("time" float8 NOT NULL, "device_id" text, "value" integer); SELECT create_hypertable('hyper_timefunc', 'time', 'device_id', 4, chunk_time_interval => 10, time_partitioning_func => 'unix_to_timestamp'); INSERT INTO hyper_timefunc (time, device_id, value) SELECT g, 'dev' || g, g FROM generate_series(0,30) g; CREATE TABLE metrics_timestamp(time timestamp); SELECT create_hypertable('metrics_timestamp','time'); INSERT INTO metrics_timestamp SELECT generate_series('2000-01-01'::timestamp,'2000-02-01'::timestamp,'1d'::interval); CREATE TABLE metrics_timestamptz(time timestamptz, device_id int); SELECT create_hypertable('metrics_timestamptz','time'); INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval), 1; INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval), 2; INSERT INTO metrics_timestamptz SELECT generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval), 3; --create a second table to test joins with CREATE TABLE metrics_timestamptz_2 (LIKE metrics_timestamptz); SELECT create_hypertable('metrics_timestamptz_2','time'); INSERT INTO metrics_timestamptz_2 SELECT * FROM metrics_timestamptz; INSERT INTO metrics_timestamptz_2 VALUES ('2000-12-01'::timestamptz, 3); CREATE TABLE metrics_date(time date); SELECT create_hypertable('metrics_date','time'); INSERT INTO metrics_date SELECT generate_series('2000-01-01'::date,'2000-02-01'::date,'1d'::interval); ANALYZE hyper; ANALYZE hyper_w_space; ANALYZE tag; ANALYZE hyper_ts; ANALYZE hyper_timefunc; -- create normal table for JOIN tests CREATE TABLE regular_timestamptz(time timestamptz); INSERT INTO regular_timestamptz SELECT generate_series('2000-01-01'::timestamptz,'2000-02-01'::timestamptz,'1d'::interval); ================================================ FILE: test/sql/include/plan_expand_hypertable_query.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --we want to see how our logic excludes chunks --and not how much work constraint_exclusion does SET constraint_exclusion = 'off'; \qecho test upper bounds :PREFIX SELECT * FROM hyper WHERE time < 10 ORDER BY value; :PREFIX SELECT * FROM hyper WHERE time < 11 ORDER BY value; :PREFIX SELECT * FROM hyper WHERE time = 10 ORDER BY value; :PREFIX SELECT * FROM hyper WHERE 10 >= time ORDER BY value; \qecho test lower bounds :PREFIX SELECT * FROM hyper WHERE time >= 10 and time < 20 ORDER BY value; :PREFIX SELECT * FROM hyper WHERE 10 < time and 20 >= time ORDER BY value; :PREFIX SELECT * FROM hyper WHERE time >= 9 and time < 20 ORDER BY value; :PREFIX SELECT * FROM hyper WHERE time > 9 and time < 20 ORDER BY value; \qecho test empty result :PREFIX SELECT * FROM hyper WHERE time < 0; \qecho test expression evaluation :PREFIX SELECT * FROM hyper WHERE time < (5*2)::smallint; \qecho test logic at INT64_MAX :PREFIX SELECT * FROM hyper WHERE time = 9223372036854775807::bigint ORDER BY value; :PREFIX SELECT * FROM hyper WHERE time = 9223372036854775806::bigint ORDER BY value; :PREFIX SELECT * FROM hyper WHERE time >= 9223372036854775807::bigint ORDER BY value; :PREFIX SELECT * FROM hyper WHERE time > 9223372036854775807::bigint ORDER BY value; :PREFIX SELECT * FROM hyper WHERE time > 9223372036854775806::bigint ORDER BY value; \qecho cte :PREFIX WITH cte AS( SELECT * FROM hyper WHERE time < 10 ) SELECT * FROM cte ORDER BY value; \qecho subquery :PREFIX SELECT 0 = ANY (SELECT value FROM hyper WHERE time < 10); \qecho no space constraint :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 ORDER BY value; \qecho valid space constraint :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 and device_id = 'dev5' ORDER BY value; :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 and 'dev5' = device_id ORDER BY value; :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 and 'dev'||(2+3) = device_id ORDER BY value; \qecho only space constraint :PREFIX SELECT * FROM hyper_w_space WHERE 'dev5' = device_id ORDER BY value; \qecho unhandled space constraint :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 and device_id > 'dev5' ORDER BY value; \qecho use of OR - does not filter chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND (device_id = 'dev5' or device_id = 'dev6') ORDER BY value; \qecho cte :PREFIX WITH cte AS( SELECT * FROM hyper_w_space WHERE time < 10 and device_id = 'dev5' ) SELECT * FROM cte ORDER BY value; \qecho subquery :PREFIX SELECT 0 = ANY (SELECT value FROM hyper_w_space WHERE time < 10 and device_id = 'dev5'); \qecho view :PREFIX SELECT * FROM hyper_w_space_view WHERE time < 10 and device_id = 'dev5' ORDER BY value; \qecho IN statement - simple :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id IN ('dev5') ORDER BY value; \qecho IN statement - two chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id IN ('dev5','dev6') ORDER BY value; \qecho IN statement - one chunk :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id IN ('dev4','dev5') ORDER BY value; \qecho NOT IN - does not filter chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id NOT IN ('dev5','dev6') ORDER BY value; \qecho IN statement with subquery - does not filter chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id IN (SELECT 'dev5'::text) ORDER BY value; \qecho ANY :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id = ANY(ARRAY['dev5','dev6']) ORDER BY value; \qecho ANY with intersection :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id = ANY(ARRAY['dev5','dev6']) AND device_id = ANY(ARRAY['dev6','dev7']) ORDER BY value; \qecho ANY without intersection shouldnt scan any chunks :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND device_id = ANY(ARRAY['dev5','dev6']) AND device_id = ANY(ARRAY['dev8','dev9']) ORDER BY value; \qecho ANY/IN/ALL only works for equals operator :PREFIX SELECT * FROM hyper_w_space WHERE device_id < ANY(ARRAY['dev5','dev6']) ORDER BY value; \qecho ALL with equals and different values shouldnt scan any chunks :PREFIX SELECT * FROM hyper_w_space WHERE device_id = ALL(ARRAY['dev5','dev6']) ORDER BY value; \qecho Multi AND :PREFIX SELECT * FROM hyper_w_space WHERE time < 10 AND time < 100 ORDER BY value; \qecho Time dimension doesnt filter chunks when using non-equality IN/ANY with multiple arguments :PREFIX SELECT * FROM hyper_w_space WHERE time < ANY(ARRAY[1,2]) ORDER BY value; \qecho Time dimension chunk exclusion with IN/ANY equality uses bounding range :PREFIX SELECT * FROM hyper WHERE time IN (5, 15) ORDER BY value; :PREFIX SELECT * FROM hyper WHERE time = ANY(ARRAY[5, 15, 25]) ORDER BY value; :PREFIX SELECT * FROM hyper WHERE time = ANY(ARRAY[25, 15, 5]) ORDER BY value; :PREFIX SELECT * FROM hyper_w_space WHERE time IN (5, 15) ORDER BY value; :PREFIX SELECT * FROM metrics_timestamp WHERE time IN ('2000-01-05'::timestamp, '2000-01-15'::timestamp) ORDER BY time; :PREFIX SELECT * FROM metrics_timestamptz WHERE time IN ('2000-01-05'::timestamptz, '2000-01-15'::timestamptz) ORDER BY time; :PREFIX SELECT * FROM metrics_date WHERE time IN ('2000-01-05'::date, '2000-01-15'::date) ORDER BY time; \qecho cross-type IN/ANY: timestamp to timestamptz column does not use bounding range (stable cast) :PREFIX SELECT * FROM metrics_timestamptz WHERE time IN ('2000-01-05'::timestamp, '2000-01-15'::timestamp) ORDER BY time; \qecho Time dimension chunk filtering works for ANY with single argument :PREFIX SELECT * FROM hyper_w_space WHERE time < ANY(ARRAY[1]) ORDER BY value; \qecho Time dimension chunk filtering works for ALL with single argument :PREFIX SELECT * FROM hyper_w_space WHERE time < ALL(ARRAY[1]) ORDER BY value; \qecho Time dimension chunk filtering works for ALL with multiple arguments :PREFIX SELECT * FROM hyper_w_space WHERE time < ALL(ARRAY[1,10,20,30]) ORDER BY value; \qecho AND intersection using IN and EQUALS :PREFIX SELECT * FROM hyper_w_space WHERE device_id IN ('dev1','dev2') AND device_id = 'dev1' ORDER BY value; \qecho AND with no intersection using IN and EQUALS :PREFIX SELECT * FROM hyper_w_space WHERE device_id IN ('dev1','dev2') AND device_id = 'dev3' ORDER BY value; \qecho timestamps \qecho these should work since they are immutable functions :PREFIX SELECT * FROM hyper_ts WHERE time < 'Wed Dec 31 16:00:10 1969 PST'::timestamptz ORDER BY value; :PREFIX SELECT * FROM hyper_ts WHERE time < to_timestamp(10) ORDER BY value; :PREFIX SELECT * FROM hyper_ts WHERE time < 'Wed Dec 31 16:00:10 1969'::timestamp AT TIME ZONE 'PST' ORDER BY value; :PREFIX SELECT * FROM hyper_ts WHERE time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; \qecho these should not work since uses stable functions; :PREFIX SELECT * FROM hyper_ts WHERE time < 'Wed Dec 31 16:00:10 1969'::timestamp ORDER BY value; :PREFIX SELECT * FROM hyper_ts WHERE time < ('Wed Dec 31 16:00:10 1969'::timestamp::timestamptz) ORDER BY value; :PREFIX SELECT * FROM hyper_ts WHERE NOW() < time ORDER BY value; \qecho joins :PREFIX SELECT * FROM hyper_ts WHERE tag_id IN (SELECT id FROM tag WHERE tag.id=1) and time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; :PREFIX SELECT * FROM hyper_ts WHERE tag_id IN (SELECT id FROM tag WHERE tag.id=1) or (time < to_timestamp(10) and device_id = 'dev1') ORDER BY value; :PREFIX SELECT * FROM hyper_ts WHERE tag_id IN (SELECT id FROM tag WHERE tag.name='tag1') and time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; :PREFIX SELECT * FROM hyper_ts JOIN tag on (hyper_ts.tag_id = tag.id ) WHERE time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; :PREFIX SELECT * FROM hyper_ts JOIN tag on (hyper_ts.tag_id = tag.id ) WHERE tag.name = 'tag1' and time < to_timestamp(10) and device_id = 'dev1' ORDER BY value; \qecho test constraint exclusion for constraints in ON clause of JOINs \qecho should exclude chunks on m1 and propagate qual to m2 because of INNER JOIN :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m1.time < '2000-01-10' ORDER BY m1.time; \qecho should exclude chunks on m2 and propagate qual to m1 because of INNER JOIN :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time < '2000-01-10' ORDER BY m1.time; \qecho must not exclude on m1 :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m1.time < '2000-01-10' ORDER BY m1.time; \qecho should exclude chunks on m2 :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time < '2000-01-10' ORDER BY m1.time; \qecho should exclude chunks on m1 :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m1.time < '2000-01-10' ORDER BY m1.time; \qecho must not exclude chunks on m2 :PREFIX SELECT m1.time,m2.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time < '2000-01-10' ORDER BY m1.time, m2.time; \qecho time_bucket exclusion :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) < 10::bigint ORDER BY time; :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) < 11::bigint ORDER BY time; :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) <= 10::bigint ORDER BY time; :PREFIX SELECT * FROM hyper WHERE 10::bigint > time_bucket(10, time) ORDER BY time; :PREFIX SELECT * FROM hyper WHERE 11::bigint > time_bucket(10, time) ORDER BY time; :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time, 5) < 10::bigint ORDER BY time; :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time, 5) < 11::bigint ORDER BY time; :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time, 5) <= 10::bigint ORDER BY time; :PREFIX SELECT * FROM hyper WHERE 10::bigint > time_bucket(10, time, 5) ORDER BY time; :PREFIX SELECT * FROM hyper WHERE 11::bigint > time_bucket(10, time, 5) ORDER BY time; \qecho timestamp time_bucket exclusion SELECT count(DISTINCT tableoid) FROM metrics_timestamp; :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time) < '2000-01-05' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time) <= '2000-01-05' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time) > '2000-01-25' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time) >= '2000-01-15' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'3d'::interval) < '2000-01-05' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'3d'::interval) <= '2000-01-05' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'3d'::interval) > '2000-01-25' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'3d'::interval) >= '2000-01-25' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'2000-01-10'::timestamp) < '2000-01-05' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'2000-01-10'::timestamp) <= '2000-01-05' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'2000-01-10'::timestamp) > '2000-01-25' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('7d',time,'2000-01-10'::timestamp) >= '2000-01-25' ORDER BY time; \qecho timestamptz time_bucket exclusion SELECT count(DISTINCT tableoid) FROM metrics_timestamptz; :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time) < '2000-01-05' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time) <= '2000-01-05' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time) > '2000-01-25' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time) >= '2000-01-25' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'3d'::interval) < '2000-01-05' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'3d'::interval) <= '2000-01-05' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'3d'::interval) > '2000-01-25' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'3d'::interval) >= '2000-01-25' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'2000-01-10'::timestamptz) < '2000-01-05' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'2000-01-10'::timestamptz) <= '2000-01-05' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'2000-01-10'::timestamptz) > '2000-01-25' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'2000-01-10'::timestamptz) >= '2000-01-25' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'Europe/Berlin') < '2000-01-05' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'Europe/Berlin') <= '2000-01-05' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'Europe/Berlin') > '2000-01-25' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamptz WHERE time_bucket('7d',time,'Europe/Berlin') >= '2000-01-25' ORDER BY time; \qecho test overflow behaviour of time_bucket exclusion :PREFIX SELECT * FROM hyper WHERE time > 950 AND time_bucket(10, time) < '9223372036854775807'::bigint ORDER BY time; \qecho test timestamp upper boundary \qecho there should be no transformation if we are out of the supported (TimescaleDB-specific) range :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('1d',time) < '294276-01-01'::timestamp ORDER BY time; \qecho transformation would be out of range :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('1000d',time) < '294276-01-01'::timestamp ORDER BY time; \qecho test timestamptz upper boundary \qecho there should be no transformation if we are out of the supported (TimescaleDB-specific) range :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1d',time) < '294276-01-01'::timestamptz ORDER BY time; \qecho transformation would be out of range :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1000d',time) < '294276-01-01'::timestamptz ORDER BY time; \qecho time_bucket exclusion with run-time constants -- These queries have a stable time_bucket expression, because the text::interval conversion is stable. PREPARE P1(text , text) AS SELECT * FROM metrics_timestamptz WHERE time_bucket($1::interval, time) > $2::timestamptz ORDER BY time; PREPARE P2(text , text) AS SELECT * FROM metrics_timestamp WHERE time_bucket($1::interval, time) > $2::timestamptz ORDER BY time; PREPARE P3(text , text) AS SELECT * FROM metrics_timestamptz WHERE time_bucket($1::interval, time) > $2::timestamp ORDER BY time; PREPARE P4(text , text) AS SELECT * FROM metrics_timestamp WHERE time_bucket($1::interval, time) > $2::timestamp ORDER BY time; -- These queries have an immutable time_bucket expression, because the parameter is passed as interval, and no conversion is involved. PREPARE P5(interval, text) AS SELECT * FROM metrics_timestamptz WHERE time_bucket($1::interval, time) > $2::timestamptz ORDER BY time; PREPARE P6(interval, text) AS SELECT * FROM metrics_timestamp WHERE time_bucket($1::interval, time) > $2::timestamptz ORDER BY time; PREPARE P7(interval, text) AS SELECT * FROM metrics_timestamptz WHERE time_bucket($1::interval, time) > $2::timestamp ORDER BY time; PREPARE P8(interval, text) AS SELECT * FROM metrics_timestamp WHERE time_bucket($1::interval, time) > $2::timestamp ORDER BY time; SET plan_cache_mode TO 'force_custom_plan'; :PREFIX EXECUTE P1('60 mins', '2024-01-01 UTC'); :PREFIX EXECUTE P2('60 mins', '2024-01-01 UTC'); :PREFIX EXECUTE P3('60 mins', '2024-01-01 UTC'); :PREFIX EXECUTE P4('60 mins', '2024-01-01 UTC'); :PREFIX EXECUTE P5('60 mins', '2024-01-01 UTC'); :PREFIX EXECUTE P6('60 mins', '2024-01-01 UTC'); :PREFIX EXECUTE P7('60 mins', '2024-01-01 UTC'); :PREFIX EXECUTE P8('60 mins', '2024-01-01 UTC'); :PREFIX EXECUTE P1('60 mins', '2000-01-01 UTC'); :PREFIX EXECUTE P2('60 mins', '2000-01-01 UTC'); :PREFIX EXECUTE P3('60 mins', '2000-01-01 UTC'); :PREFIX EXECUTE P4('60 mins', '2000-01-01 UTC'); :PREFIX EXECUTE P5('60 mins', '2000-01-01 UTC'); :PREFIX EXECUTE P6('60 mins', '2000-01-01 UTC'); :PREFIX EXECUTE P7('60 mins', '2000-01-01 UTC'); :PREFIX EXECUTE P8('60 mins', '2000-01-01 UTC'); SET plan_cache_mode TO 'force_generic_plan'; :PREFIX EXECUTE P1('60 mins', '2024-01-01 UTC'); :PREFIX EXECUTE P2('60 mins', '2024-01-01 UTC'); :PREFIX EXECUTE P3('60 mins', '2024-01-01 UTC'); :PREFIX EXECUTE P4('60 mins', '2024-01-01 UTC'); :PREFIX EXECUTE P5('60 mins', '2024-01-01 UTC'); :PREFIX EXECUTE P6('60 mins', '2024-01-01 UTC'); :PREFIX EXECUTE P7('60 mins', '2024-01-01 UTC'); :PREFIX EXECUTE P8('60 mins', '2024-01-01 UTC'); :PREFIX EXECUTE P1('60 mins', '2000-01-01 UTC'); :PREFIX EXECUTE P2('60 mins', '2000-01-01 UTC'); :PREFIX EXECUTE P3('60 mins', '2000-01-01 UTC'); :PREFIX EXECUTE P4('60 mins', '2000-01-01 UTC'); :PREFIX EXECUTE P5('60 mins', '2000-01-01 UTC'); :PREFIX EXECUTE P6('60 mins', '2000-01-01 UTC'); :PREFIX EXECUTE P7('60 mins', '2000-01-01 UTC'); :PREFIX EXECUTE P8('60 mins', '2000-01-01 UTC'); RESET plan_cache_mode; DEALLOCATE P1; DEALLOCATE P2; DEALLOCATE P3; DEALLOCATE P4; DEALLOCATE P5; DEALLOCATE P6; DEALLOCATE P7; DEALLOCATE P8; :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) > 10 AND time_bucket(10, time) < 100 ORDER BY time; :PREFIX SELECT * FROM hyper WHERE time_bucket(10, time) > 10 AND time_bucket(10, time) < 20 ORDER BY time; :PREFIX SELECT * FROM hyper WHERE time_bucket(1, time) > 11 AND time_bucket(1, time) < 19 ORDER BY time; :PREFIX SELECT * FROM hyper WHERE 10 < time_bucket(10, time) AND 20 > time_bucket(10,time) ORDER BY time; \qecho time_bucket exclusion with date :PREFIX SELECT * FROM metrics_date WHERE time_bucket('1d',time) < '2000-01-03' ORDER BY time; :PREFIX SELECT * FROM metrics_date WHERE time_bucket('1d',time) >= '2000-01-03' AND time_bucket('1d',time) <= '2000-01-10' ORDER BY time; \qecho time_bucket exclusion with timestamp :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('1d',time) < '2000-01-03' ORDER BY time; :PREFIX SELECT * FROM metrics_timestamp WHERE time_bucket('1d',time) >= '2000-01-03' AND time_bucket('1d',time) <= '2000-01-10' ORDER BY time; \qecho time_bucket exclusion with timestamptz :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('6h',time) < '2000-01-03' ORDER BY time; :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('6h',time) >= '2000-01-03' AND time_bucket('6h',time) <= '2000-01-10' ORDER BY time; \qecho time_bucket exclusion with timestamptz and day interval :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1d',time) < '2000-01-03' ORDER BY time; :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1d',time) >= '2000-01-03' AND time_bucket('1d',time) <= '2000-01-10' ORDER BY time; :PREFIX SELECT time FROM metrics_timestamptz WHERE time_bucket('1d',time) >= '2000-01-03' AND time_bucket('7d',time) <= '2000-01-10' ORDER BY time; \qecho no transformation :PREFIX SELECT * FROM hyper WHERE time_bucket(10 + floor(random())::int, time) > 10 AND time_bucket(10 + floor(random())::int, time) < 100 AND time < 150 ORDER BY time; \qecho exclude chunks based on time column with partitioning function. This \qecho transparently applies the time partitioning function on the time \qecho value to be able to exclude chunks (similar to a closed dimension). :PREFIX SELECT * FROM hyper_timefunc WHERE time < 4 ORDER BY value; \qecho excluding based on time expression is currently unoptimized :PREFIX SELECT * FROM hyper_timefunc WHERE unix_to_timestamp(time) < 'Wed Dec 31 16:00:04 1969 PST' ORDER BY value; \qecho test qual propagation for joins RESET constraint_exclusion; \qecho nothing to propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1, metrics_timestamptz_2 m2 WHERE m1.time = m2.time ORDER BY m1.time; :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time ORDER BY m1.time; :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time ORDER BY m1.time; :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time ORDER BY m1.time; \qecho OR constraints should not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' OR m1.time > '2001-01-01' ORDER BY m1.time; \qecho test single constraint \qecho constraint should be on both scans \qecho these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column :PREFIX SELECT m1.time FROM metrics_timestamptz m1, metrics_timestamptz_2 m2 WHERE m1.time = m2.time AND m1.time < '2000-01-10' ORDER BY m1.time; :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' ORDER BY m1.time; :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' ORDER BY m1.time; :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' ORDER BY m1.time; \qecho test 2 constraints on single relation \qecho these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column :PREFIX SELECT m1.time FROM metrics_timestamptz m1, metrics_timestamptz_2 m2 WHERE m1.time = m2.time AND m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; \qecho test 2 constraints with 1 constraint on each relation \qecho these will propagate even for LEFT/RIGHT JOIN because the constraints are not in the ON clause and therefore imply a NOT NULL condition on the JOIN column :PREFIX SELECT m1.time FROM metrics_timestamptz m1, metrics_timestamptz_2 m2 WHERE m1.time = m2.time AND m1.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; \qecho test constraints in ON clause of INNER JOIN :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; \qecho test constraints in ON clause of LEFT JOIN \qecho must not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 LEFT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; \qecho test constraints in ON clause of RIGHT JOIN \qecho must not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 RIGHT JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time AND m2.time > '2000-01-01' AND m2.time < '2000-01-10' ORDER BY m1.time; \qecho test equality condition not in ON clause :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON true WHERE m2.time = m1.time AND m2.time < '2000-01-10' ORDER BY m1.time; \qecho test constraints not joined on \qecho device_id constraint must not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON true WHERE m2.time = m1.time AND m2.time < '2000-01-10' AND m1.device_id = 1 ORDER BY m1.time; \qecho test multiple join conditions \qecho device_id constraint should propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON true WHERE m2.time = m1.time AND m1.device_id = m2.device_id AND m2.time < '2000-01-10' AND m1.device_id = 1 ORDER BY m1.time; \qecho test join with 3 tables :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time INNER JOIN metrics_timestamptz m3 ON m2.time=m3.time WHERE m1.time > '2000-01-01' AND m1.time < '2000-01-10' ORDER BY m1.time; \qecho test non-Const constraints :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10'::text::timestamptz ORDER BY m1.time; \qecho test now() :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < now() ORDER BY m1.time; \qecho test volatile function \qecho should not propagate :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m1.time < clock_timestamp() ORDER BY m1.time; :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN metrics_timestamptz_2 m2 ON m1.time = m2.time WHERE m2.time < clock_timestamp() ORDER BY m1.time; \qecho test JOINs with normal table \qecho will not propagate because constraints are only added to hypertables :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN regular_timestamptz m2 ON m1.time = m2.time WHERE m1.time < '2000-01-10' ORDER BY m1.time; \qecho test JOINs with normal table :PREFIX SELECT m1.time FROM metrics_timestamptz m1 INNER JOIN regular_timestamptz m2 ON m1.time = m2.time WHERE m2.time < '2000-01-10' ORDER BY m1.time; \qecho test quals are not pushed into OUTER JOIN CREATE TABLE outer_join_1 (id int, name text,time timestamptz NOT NULL DEFAULT '2000-01-01'); CREATE TABLE outer_join_2 (id int, name text,time timestamptz NOT NULL DEFAULT '2000-01-01'); SELECT (SELECT table_name FROM create_hypertable(tbl, 'time')) FROM (VALUES ('outer_join_1'),('outer_join_2')) v(tbl); INSERT INTO outer_join_1 VALUES(1,'a'), (2,'b'); INSERT INTO outer_join_2 VALUES(1,'a'); :PREFIX SELECT one.id, two.name FROM outer_join_1 one LEFT OUTER JOIN outer_join_2 two ON one.id=two.id WHERE one.id=2; :PREFIX SELECT one.id, two.name FROM outer_join_2 two RIGHT OUTER JOIN outer_join_1 one ON one.id=two.id WHERE one.id=2; DROP TABLE outer_join_1; DROP TABLE outer_join_2; -- test UNION between regular table and hypertable SELECT time FROM regular_timestamptz UNION SELECT time FROM metrics_timestamptz ORDER BY 1; -- test UNION ALL between regular table and hypertable SELECT time FROM regular_timestamptz UNION ALL SELECT time FROM metrics_timestamptz ORDER BY 1; -- test nested join qual propagation :PREFIX SELECT * FROM ( SELECT o1_m1.time FROM metrics_timestamptz o1_m1 INNER JOIN metrics_timestamptz_2 o1_m2 ON true WHERE o1_m2.time = o1_m1.time AND o1_m1.device_id = o1_m2.device_id AND o1_m2.time < '2000-01-10' AND o1_m1.device_id = 1 ) o1 FULL OUTER JOIN ( SELECT o2_m1.time FROM metrics_timestamptz o2_m1 FULL OUTER JOIN metrics_timestamptz_2 o2_m2 ON true WHERE o2_m2.time = o2_m1.time AND o2_m1.device_id = o2_m2.device_id AND o2_m2.time > '2000-01-20' AND o2_m1.device_id = 2 ) o2 ON o1.time = o2.time ORDER BY 1,2; :PREFIX SELECT * FROM ( SELECT o1_m1.time FROM metrics_timestamptz o1_m1 INNER JOIN metrics_timestamptz_2 o1_m2 ON o1_m2.time = o1_m1.time AND o1_m1.device_id = o1_m2.device_id WHERE o1_m2.time < '2000-01-10' AND o1_m1.device_id = 1 ) o1 FULL OUTER JOIN ( SELECT o2_m1.time FROM metrics_timestamptz o2_m1 FULL OUTER JOIN metrics_timestamptz_2 o2_m2 ON o2_m2.time = o2_m1.time AND o2_m1.device_id = o2_m2.device_id WHERE o2_m2.time > '2000-01-20' AND o2_m1.device_id = 2 ) o2 ON o1.time = o2.time ORDER BY 1,2; ================================================ FILE: test/sql/include/plan_hashagg_load.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE metric (id SERIAL PRIMARY KEY, value INT); CREATE TABLE hyper(time TIMESTAMP NOT NULL, time_int BIGINT, time_broken DATE, metricid int, value double precision); CREATE TABLE regular(time TIMESTAMP NOT NULL, time_int BIGINT, time_date DATE, metricid int, value double precision); SELECT create_hypertable('hyper', 'time', chunk_time_interval => interval '20 day', create_default_indexes=>FALSE); ALTER TABLE hyper DROP COLUMN time_broken, ADD COLUMN time_date DATE; INSERT INTO metric(value) SELECT random()*100 FROM generate_series(0,10); INSERT INTO hyper SELECT t, EXTRACT(EPOCH FROM t), (EXTRACT(EPOCH FROM t)::int % 10)+1, 1.0, t::date FROM generate_series('2001-01-01', '2001-01-10', INTERVAL '1 second') t; INSERT INTO regular(time, time_int, time_date, metricid, value) SELECT t, EXTRACT(EPOCH FROM t), t::date, (EXTRACT(EPOCH FROM t)::int % 10) + 1, 1.0 FROM generate_series('2001-01-01', '2001-01-02', INTERVAL '1 second') t; --test some queries before analyze; EXPLAIN (buffers off, costs off) SELECT time_bucket('1 minute', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; EXPLAIN (buffers off, costs off) SELECT date_trunc('minute', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; -- Test partitioning function on an open (time) dimension CREATE OR REPLACE FUNCTION unix_to_timestamp(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE SQL IMMUTABLE AS $BODY$ SELECT to_timestamp(unixtime); $BODY$; CREATE TABLE hyper_timefunc(time float8 NOT NULL, metricid int, VALUE double precision, time_date DATE); SELECT create_hypertable('hyper_timefunc', 'time', chunk_time_interval => interval '20 day', create_default_indexes=>FALSE, time_partitioning_func => 'unix_to_timestamp'); INSERT INTO hyper_timefunc SELECT time_int, metricid, VALUE, time_date FROM hyper; ANALYZE metric; ANALYZE hyper; ANALYZE regular; ANALYZE hyper_timefunc; ================================================ FILE: test/sql/include/plan_hashagg_query.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. :PREFIX SELECT time_bucket('1 minute', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; :PREFIX SELECT time_bucket('1 hour', time) AS MetricMinuteTs, metricid, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs, metricid ORDER BY MetricMinuteTs DESC, metricid; --should be too many groups will not hashaggregate :PREFIX SELECT time_bucket('1 second', time) AS MetricMinuteTs, metricid, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs, metricid ORDER BY MetricMinuteTs DESC, metricid; :PREFIX SELECT time_bucket('1 minute', time, INTERVAL '30 seconds') AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; :PREFIX SELECT time_bucket(60, time_int) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; :PREFIX SELECT time_bucket(60, time_int, 10) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; :PREFIX SELECT time_bucket('1 day', time_date) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; :PREFIX SELECT date_trunc('minute', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; \set ON_ERROR_STOP 0 --can't optimize invalid time unit :PREFIX SELECT date_trunc('invalid', time) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; \set ON_ERROR_STOP 1 :PREFIX SELECT date_trunc('day', time_date) AS MetricMinuteTs, AVG(value) as avg FROM hyper WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; --joins --with hypertable, optimize :PREFIX SELECT time_bucket(3600, time_int, 10) AS MetricMinuteTs, metric.value, AVG(hyper.value) as avg FROM hyper JOIN metric ON (hyper.metricid = metric.id) WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs, metric.id ORDER BY MetricMinuteTs DESC, metric.id; --no hypertable involved, no optimization :PREFIX SELECT time_bucket(3600, time_int, 10) AS MetricMinuteTs, metric.value, AVG(regular.value) as avg FROM regular JOIN metric ON (regular.metricid = metric.id) WHERE time >= '2001-01-04T00:00:00' AND time <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs, metric.id ORDER BY MetricMinuteTs DESC, metric.id; -- Try with time partitioning function. Currently not optimized for hash aggregates :PREFIX SELECT time_bucket('1 minute', unix_to_timestamp(time)) AS MetricMinuteTs, AVG(value) as avg FROM hyper_timefunc WHERE unix_to_timestamp(time) >= '2001-01-04T00:00:00' AND unix_to_timestamp(time) <= '2001-01-05T01:00:00' GROUP BY MetricMinuteTs ORDER BY MetricMinuteTs DESC; ================================================ FILE: test/sql/include/plan_ordered_append_load.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- create a now() function for repeatable testing that always returns -- the same timestamp. It needs to be marked STABLE CREATE OR REPLACE FUNCTION now_s() RETURNS timestamptz LANGUAGE PLPGSQL STABLE AS $BODY$ BEGIN RETURN '2000-01-08T0:00:00+0'::timestamptz; END; $BODY$; CREATE TABLE devices(device_id INT PRIMARY KEY, name TEXT); INSERT INTO devices VALUES (1,'Device 1'), (2,'Device 2'), (3,'Device 3'); -- create a second table where we create chunks in reverse order CREATE TABLE ordered_append_reverse(time timestamptz NOT NULL, device_id INT, value float); SELECT create_hypertable('ordered_append_reverse','time'); INSERT INTO ordered_append_reverse SELECT generate_series('2000-01-18'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 0.5; -- table where dimension column is last column CREATE TABLE IF NOT EXISTS dimension_last( id INT8 NOT NULL, device_id INT NOT NULL, name TEXT NOT NULL, time timestamptz NOT NULL ); SELECT create_hypertable('dimension_last', 'time', chunk_time_interval => interval '1day', if_not_exists => True); -- table with only dimension column CREATE TABLE IF NOT EXISTS dimension_only( time timestamptz NOT NULL ); SELECT create_hypertable('dimension_only', 'time', chunk_time_interval => interval '1day', if_not_exists => True); INSERT INTO dimension_last SELECT 1,1,'Device 1',generate_series('2000-01-01 0:00:00+0'::timestamptz,'2000-01-04 23:59:00+0'::timestamptz,'1m'::interval); INSERT INTO dimension_only VALUES ('2000-01-01'), ('2000-01-03'), ('2000-01-05'), ('2000-01-07'); ANALYZE devices; ANALYZE ordered_append_reverse; ANALYZE dimension_last; ANALYZE dimension_only; -- create hypertable with indexes not on all chunks CREATE TABLE ht_missing_indexes(time timestamptz NOT NULL, device_id int, value float); SELECT create_hypertable('ht_missing_indexes','time'); INSERT INTO ht_missing_indexes SELECT generate_series('2000-01-01'::timestamptz,'2000-01-18'::timestamptz,'1m'::interval), 1, 0.5; INSERT INTO ht_missing_indexes SELECT generate_series('2000-01-01'::timestamptz,'2000-01-18'::timestamptz,'1m'::interval), 2, 1.5; INSERT INTO ht_missing_indexes SELECT generate_series('2000-01-01'::timestamptz,'2000-01-18'::timestamptz,'1m'::interval), 3, 2.5; -- drop index from 2nd chunk of ht_missing_indexes SELECT format('%I.%I',i.schemaname,i.indexname) AS "INDEX_NAME" FROM _timescaledb_catalog.chunk c INNER JOIN _timescaledb_catalog.hypertable ht ON c.hypertable_id = ht.id INNER JOIN pg_indexes i ON i.schemaname = c.schema_name AND i.tablename=c.table_name WHERE ht.table_name = 'ht_missing_indexes' ORDER BY c.id LIMIT 1 OFFSET 1 \gset DROP INDEX :INDEX_NAME; ANALYZE ht_missing_indexes; -- create hypertable with with dropped columns CREATE TABLE ht_dropped_columns(c1 int, c2 int, c3 int, c4 int, c5 int, time timestamptz NOT NULL, device_id int, value float); SELECT create_hypertable('ht_dropped_columns','time'); ALTER TABLE ht_dropped_columns DROP COLUMN c1; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-01'::timestamptz,'2000-01-02'::timestamptz,'1m'::interval), 1, 0.5; ALTER TABLE ht_dropped_columns DROP COLUMN c2; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-08'::timestamptz,'2000-01-09'::timestamptz,'1m'::interval), 1, 0.5; ALTER TABLE ht_dropped_columns DROP COLUMN c3; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-15'::timestamptz,'2000-01-16'::timestamptz,'1m'::interval), 1, 0.5; ALTER TABLE ht_dropped_columns DROP COLUMN c4; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-22'::timestamptz,'2000-01-23'::timestamptz,'1m'::interval), 1, 0.5; ALTER TABLE ht_dropped_columns DROP COLUMN c5; INSERT INTO ht_dropped_columns(time,device_id,value) SELECT generate_series('2000-01-29'::timestamptz,'2000-01-30'::timestamptz,'1m'::interval), 1, 0.5; ANALYZE ht_dropped_columns; CREATE TABLE space2(time timestamptz NOT NULL, device_id int NOT NULL, tag_id int NOT NULL, value float); SELECT create_hypertable('space2','time','device_id',number_partitions:=3); SELECT add_dimension('space2','tag_id',number_partitions:=3); INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 1, 1.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 1, 2.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 3, 1, 3.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 2, 1.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 2, 2.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 3, 2, 3.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 3, 1.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 3, 2.5; INSERT INTO space2 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 3, 3, 3.5; ANALYZE space2; CREATE TABLE space3(time timestamptz NOT NULL, x int NOT NULL, y int NOT NULL, z int NOT NULL, value float); SELECT create_hypertable('space3','time','x',number_partitions:=2); SELECT add_dimension('space3','y',number_partitions:=2); SELECT add_dimension('space3','z',number_partitions:=2); INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 1, 1, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 1, 2, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 2, 1, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 1, 2, 2, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 1, 1, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 1, 2, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 2, 1, 1.5; INSERT INTO space3 SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 2, 2, 2, 1.5; ANALYZE space3; CREATE TABLE sortopt_test(time timestamptz NOT NULL, device TEXT); SELECT create_hypertable('sortopt_test','time',create_default_indexes:=false); -- since alpine does not support locales we cant test collations in our ci -- CREATE COLLATION IF NOT EXISTS en_US(LOCALE='en_US.utf8'); -- CREATE INDEX time_device_utf8 ON sortopt_test(time, device COLLATE "en_US"); CREATE INDEX time_device_nullsfirst ON sortopt_test(time, device NULLS FIRST); CREATE INDEX time_device_nullslast ON sortopt_test(time, device DESC NULLS LAST); INSERT INTO sortopt_test SELECT generate_series('2000-01-10'::timestamptz,'2000-01-01'::timestamptz,'-1m'::interval), 'Device 1'; ANALYZE sortopt_test; ================================================ FILE: test/sql/include/plan_ordered_append_query.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- print chunks ordered by time to ensure ordering we want SELECT ht.table_name AS hypertable, c.table_name AS chunk, ds.range_start FROM _timescaledb_catalog.chunk c INNER JOIN LATERAL(SELECT * FROM _timescaledb_catalog.chunk_constraint cc WHERE c.id = cc.chunk_id ORDER BY cc.dimension_slice_id LIMIT 1) cc ON true INNER JOIN _timescaledb_catalog.dimension_slice ds ON ds.id=cc.dimension_slice_id INNER JOIN _timescaledb_catalog.dimension d ON ds.dimension_id = d.id INNER JOIN _timescaledb_catalog.hypertable ht ON d.hypertable_id = ht.id ORDER BY ht.table_name, range_start, chunk; -- test ASC for reverse ordered chunks :PREFIX SELECT time, device_id, value FROM ordered_append_reverse ORDER BY time ASC LIMIT 1; -- test DESC for reverse ordered chunks :PREFIX SELECT time, device_id, value FROM ordered_append_reverse ORDER BY time DESC LIMIT 1; -- test query with ORDER BY time_bucket, device_id -- must not use ordered append :PREFIX SELECT time_bucket('1d',time), device_id, name FROM dimension_last ORDER BY time_bucket('1d',time), device_id LIMIT 1; -- test query with ORDER BY date_trunc, device_id -- must not use ordered append :PREFIX SELECT date_trunc('day',time), device_id, name FROM dimension_last ORDER BY 1,2 LIMIT 1; -- test with table with only dimension column :PREFIX SELECT * FROM dimension_only ORDER BY time DESC LIMIT 1; -- test LEFT JOIN against hypertable :PREFIX_NO_ANALYZE SELECT * FROM dimension_last LEFT JOIN dimension_only USING (time) ORDER BY dimension_last.time DESC LIMIT 2; -- test INNER JOIN against non-hypertable :PREFIX_NO_ANALYZE SELECT * FROM dimension_last INNER JOIN dimension_only USING (time) ORDER BY dimension_last.time DESC LIMIT 2; -- test join against non-hypertable :PREFIX SELECT * FROM dimension_last INNER JOIN devices USING(device_id) ORDER BY dimension_last.time DESC LIMIT 2; -- test hypertable with index missing on one chunk :PREFIX SELECT time, device_id, value FROM ht_missing_indexes ORDER BY time ASC LIMIT 1; -- test hypertable with index missing on one chunk -- and no data :PREFIX SELECT time, device_id, value FROM ht_missing_indexes WHERE device_id = 2 ORDER BY time DESC LIMIT 1; -- test hypertable with index missing on one chunk -- and no data :PREFIX SELECT time, device_id, value FROM ht_missing_indexes WHERE time > '2000-01-07' ORDER BY time LIMIT 10; -- test hypertable with dropped columns :PREFIX SELECT time, device_id, value FROM ht_dropped_columns ORDER BY time ASC LIMIT 1; -- test hypertable with dropped columns :PREFIX SELECT time, device_id, value FROM ht_dropped_columns WHERE device_id = 1 ORDER BY time DESC; -- test hypertable with 2 space dimensions :PREFIX SELECT time, device_id, value FROM space2 ORDER BY time DESC; -- test hypertable with 3 space dimensions :PREFIX SELECT time FROM space3 ORDER BY time DESC; -- test COLLATION -- cant be tested in our ci because alpine doesnt support locales -- :PREFIX SELECT * FROM sortopt_test ORDER BY time, device COLLATE "en_US.utf8"; -- test NULLS FIRST :PREFIX SELECT * FROM sortopt_test ORDER BY time, device NULLS FIRST; -- test NULLS LAST :PREFIX SELECT * FROM sortopt_test ORDER BY time, device DESC NULLS LAST; ================================================ FILE: test/sql/include/query_load.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC.hyper_1 ( time TIMESTAMP NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL ); CREATE INDEX "time_plain" ON PUBLIC.hyper_1 (time DESC, series_0); SELECT * FROM create_hypertable('"public"."hyper_1"'::regclass, 'time'::name, number_partitions => 1, create_default_indexes=>false); INSERT INTO hyper_1 SELECT to_timestamp(ser), ser, ser+10000, sqrt(ser::numeric) FROM generate_series(0,10000) ser; INSERT INTO hyper_1 SELECT to_timestamp(ser), ser, ser+10000, sqrt(ser::numeric) FROM generate_series(10001,20000) ser; CREATE TABLE PUBLIC.hyper_1_tz ( time TIMESTAMPTZ NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL ); CREATE INDEX "time_plain_tz" ON PUBLIC.hyper_1_tz (time DESC, series_0); SELECT * FROM create_hypertable('"public"."hyper_1_tz"'::regclass, 'time'::name, number_partitions => 1, create_default_indexes=>false); INSERT INTO hyper_1_tz SELECT to_timestamp(ser), ser, ser+10000, sqrt(ser::numeric) FROM generate_series(0,10000) ser; INSERT INTO hyper_1_tz SELECT to_timestamp(ser), ser, ser+10000, sqrt(ser::numeric) FROM generate_series(10001,20000) ser; CREATE TABLE PUBLIC.hyper_1_int ( time int NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL ); CREATE INDEX "time_plain_int" ON PUBLIC.hyper_1_int (time DESC, series_0); SELECT * FROM create_hypertable('"public"."hyper_1_int"'::regclass, 'time'::name, number_partitions => 1, chunk_time_interval=>10000, create_default_indexes=>FALSE); INSERT INTO hyper_1_int SELECT ser, ser, ser+10000, sqrt(ser::numeric) FROM generate_series(0,10000) ser; INSERT INTO hyper_1_int SELECT ser, ser, ser+10000, sqrt(ser::numeric) FROM generate_series(10001,20000) ser; CREATE TABLE PUBLIC.hyper_1_date ( time date NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL ); CREATE INDEX "time_plain_date" ON PUBLIC.hyper_1_date (time DESC, series_0); SELECT * FROM create_hypertable('"public"."hyper_1_date"'::regclass, 'time'::name, number_partitions => 1, chunk_time_interval=>86400000000, create_default_indexes=>FALSE); INSERT INTO hyper_1_date SELECT to_timestamp(ser)::date, ser, ser+10000, sqrt(ser::numeric) FROM generate_series(0,10000) ser; INSERT INTO hyper_1_date SELECT to_timestamp(ser)::date, ser, ser+10000, sqrt(ser::numeric) FROM generate_series(10001,20000) ser; --below needed to create enough unique dates to trigger an index scan INSERT INTO hyper_1_date SELECT to_timestamp(ser*100)::date, ser, ser+10000, sqrt(ser::numeric) FROM generate_series(10001,20000) ser; CREATE TABLE PUBLIC.plain_table ( time TIMESTAMPTZ NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL ); CREATE INDEX "time_plain_plain_table" ON PUBLIC.plain_table (time DESC, series_0); INSERT INTO plain_table SELECT to_timestamp(ser), ser, ser+10000, sqrt(ser::numeric) FROM generate_series(0,10000) ser; INSERT INTO plain_table SELECT to_timestamp(ser), ser, ser+10000, sqrt(ser::numeric) FROM generate_series(10001,20000) ser; -- Table with a time partitioning function CREATE TABLE PUBLIC.hyper_timefunc ( time float8 NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL ); CREATE OR REPLACE FUNCTION unix_to_timestamp(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE SQL IMMUTABLE AS $BODY$ SELECT to_timestamp(unixtime); $BODY$; CREATE INDEX "time_plain_timefunc" ON PUBLIC.hyper_timefunc (to_timestamp(time) DESC, series_0); SELECT * FROM create_hypertable('"public"."hyper_timefunc"'::regclass, 'time'::name, number_partitions => 1, create_default_indexes=>false, time_partitioning_func => 'unix_to_timestamp'); INSERT INTO hyper_timefunc SELECT ser, ser, ser+10000, sqrt(ser::numeric) FROM generate_series(0,10000) ser; INSERT INTO hyper_timefunc SELECT ser, ser, ser+10000, sqrt(ser::numeric) FROM generate_series(10001,20000) ser; ANALYZE plain_table; ANALYZE hyper_timefunc; ANALYZE hyper_1; ANALYZE hyper_1_tz; ANALYZE hyper_1_int; ANALYZE hyper_1_date; ================================================ FILE: test/sql/include/query_query.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SHOW timescaledb.enable_optimizations; --non-aggregates use MergeAppend in both optimized and non-optimized :PREFIX SELECT * FROM hyper_1 ORDER BY "time" DESC limit 2; :PREFIX SELECT * FROM hyper_timefunc ORDER BY unix_to_timestamp("time") DESC limit 2; --Aggregates use MergeAppend only in optimized :PREFIX SELECT date_trunc('minute', time) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1 GROUP BY t ORDER BY t DESC limit 2; :PREFIX SELECT date_trunc('minute', time) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1_date GROUP BY t ORDER BY t DESC limit 2; --the minute and second results should be diff :PREFIX SELECT date_trunc('minute', time) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1 GROUP BY t ORDER BY t DESC limit 2; :PREFIX SELECT date_trunc('second', time) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1 GROUP BY t ORDER BY t DESC limit 2; --test that when index on time used by constraint, still works correctly :PREFIX SELECT date_trunc('minute', time) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1 WHERE time < to_timestamp(900) GROUP BY t ORDER BY t DESC LIMIT 2; --test on table with time partitioning function. Currently not --optimized to use index for ordering since the index is an expression --on time (e.g., timefunc(time)), and we currently don't handle that --case. :PREFIX SELECT date_trunc('minute', to_timestamp(time)) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_timefunc WHERE to_timestamp(time) < to_timestamp(900) GROUP BY t ORDER BY t DESC LIMIT 2; BEGIN; --test that still works with an expression index on data_trunc. DROP INDEX "time_plain"; CREATE INDEX "time_trunc" ON PUBLIC.hyper_1 (date_trunc('minute', time)); ANALYZE hyper_1; :PREFIX SELECT date_trunc('minute', time) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1 GROUP BY t ORDER BY t DESC limit 2; --test that works with both indexes CREATE INDEX "time_plain" ON PUBLIC.hyper_1 (time DESC, series_0); ANALYZE hyper_1; :PREFIX SELECT date_trunc('minute', time) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1 GROUP BY t ORDER BY t DESC limit 2; :PREFIX SELECT time_bucket('1 minute', time) t, avg(series_0), min(series_1), trunc(avg(series_2)::numeric, 5) FROM hyper_1 GROUP BY t ORDER BY t DESC limit 2; :PREFIX SELECT time_bucket('1 minute', time, INTERVAL '30 seconds') t, avg(series_0), min(series_1), trunc(avg(series_2)::numeric,5) FROM hyper_1 GROUP BY t ORDER BY t DESC limit 2; :PREFIX SELECT time_bucket('1 minute', time - INTERVAL '30 seconds') t, avg(series_0), min(series_1), trunc(avg(series_2)::numeric,5) FROM hyper_1 GROUP BY t ORDER BY t DESC limit 2; :PREFIX SELECT time_bucket('1 minute', time - INTERVAL '30 seconds') + INTERVAL '30 seconds' t, avg(series_0), min(series_1), trunc(avg(series_2)::numeric,5) FROM hyper_1 GROUP BY t ORDER BY t DESC limit 2; :PREFIX SELECT time_bucket('1 minute', time) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1_tz GROUP BY t ORDER BY t DESC limit 2; :PREFIX SELECT time_bucket('1 minute', time::timestamp) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1_tz GROUP BY t ORDER BY t DESC limit 2; :PREFIX SELECT time_bucket(10, time) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1_int GROUP BY t ORDER BY t DESC limit 2; :PREFIX SELECT time_bucket(10, time, 2) t, avg(series_0), min(series_1), avg(series_2) FROM hyper_1_int GROUP BY t ORDER BY t DESC limit 2; ROLLBACK; -- sort order optimization should not be applied to non-hypertables :PREFIX SELECT date_trunc('minute', time) t, avg(series_0), min(series_1), avg(series_2) FROM plain_table WHERE time < to_timestamp(900) GROUP BY t ORDER BY t DESC LIMIT 2; ================================================ FILE: test/sql/include/query_result_test_equal.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --expects QUERY1 and QUERY2 to be set, expects data can be compared set enable_hashjoin = off; set enable_mergejoin = on; with query1 AS ( SELECT row_number() OVER(ORDER BY q.*) row_number, * FROM (:QUERY1) as q ), query2 AS ( SELECT row_number() OVER (ORDER BY v.*) row_number, * FROM (:QUERY2) as v ) SELECT count(*) FILTER (WHERE query1.row_number IS DISTINCT FROM query2.row_number OR query1.show_chunks IS DISTINCT FROM query2.drop_chunks) AS "Different Rows", coalesce(max(query1.row_number), 0) AS "Total Rows from Query 1", coalesce(max(query2.row_number), 0) AS "Total Rows from Query 2" FROM query1 FULL OUTER JOIN query2 ON (query1.row_number = query2.row_number); reset enable_hashjoin; reset enable_mergejoin; ================================================ FILE: test/sql/include/test_utils.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE OR REPLACE FUNCTION assert_true( val boolean ) RETURNS VOID LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN IF val IS NOT TRUE THEN RAISE 'Assert failed'; END IF; END $BODY$; CREATE OR REPLACE FUNCTION assert_equal( val1 anyelement, val2 anyelement ) RETURNS VOID LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN IF (val1 = val2) IS NOT TRUE THEN RAISE 'Assert failed: % = %',val1,val2; END IF; END $BODY$; ================================================ FILE: test/sql/include/ts_merge_load.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE USER regress_merge_privs; CREATE USER regress_merge_no_privs; DROP TABLE IF EXISTS target; DROP TABLE IF EXISTS source; CREATE TABLE target (tid integer, balance integer) WITH (autovacuum_enabled=off); CREATE TABLE source (sid integer, delta integer) -- no index WITH (autovacuum_enabled=off); INSERT INTO target VALUES (1, 10); INSERT INTO target VALUES (2, 20); INSERT INTO target VALUES (3, 30); SELECT t.ctid is not null as matched, t.*, s.* FROM source s FULL OUTER JOIN target t ON s.sid = t.tid ORDER BY t.tid, s.sid; ALTER TABLE target OWNER TO regress_merge_privs; ALTER TABLE source OWNER TO regress_merge_privs; CREATE TABLE target2 (tid integer, balance integer) WITH (autovacuum_enabled=off); CREATE TABLE source2 (sid integer, delta integer) WITH (autovacuum_enabled=off); ALTER TABLE target2 OWNER TO regress_merge_no_privs; ALTER TABLE source2 OWNER TO regress_merge_no_privs; GRANT INSERT ON target TO regress_merge_no_privs; GRANT CREATE ON SCHEMA public TO regress_merge_privs; SET SESSION AUTHORIZATION regress_merge_privs; CREATE TABLE sq_target (tid integer NOT NULL, balance integer) WITH (autovacuum_enabled=off); CREATE TABLE sq_source (delta integer, sid integer, balance integer DEFAULT 0) WITH (autovacuum_enabled=off); INSERT INTO sq_target(tid, balance) VALUES (1,100), (2,200), (3,300); INSERT INTO sq_source(sid, delta) VALUES (1,10), (2,20), (4,40); -- conditional WHEN clause CREATE TABLE wq_target (tid integer not null, balance integer DEFAULT -1) WITH (autovacuum_enabled=off); CREATE TABLE wq_source (balance integer, sid integer) WITH (autovacuum_enabled=off); INSERT INTO wq_source (sid, balance) VALUES (1, 100); CREATE TABLE cj_target (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE cj_source1 (sid1 integer, scat integer, delta integer) WITH (autovacuum_enabled=off); CREATE TABLE cj_source2 (sid2 integer, sval text) WITH (autovacuum_enabled=off); INSERT INTO cj_source1 VALUES (1, 10, 100); INSERT INTO cj_source1 VALUES (1, 20, 200); INSERT INTO cj_source1 VALUES (2, 20, 300); INSERT INTO cj_source1 VALUES (3, 10, 400); INSERT INTO cj_source2 VALUES (1, 'initial source2'); INSERT INTO cj_source2 VALUES (2, 'initial source2'); INSERT INTO cj_source2 VALUES (3, 'initial source2'); CREATE TABLE fs_target (a int, b int, c text) WITH (autovacuum_enabled=off); ================================================ FILE: test/sql/include/ts_merge_load_ht.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE USER regress_merge_privs; CREATE USER regress_merge_no_privs; DROP TABLE IF EXISTS target; DROP TABLE IF EXISTS source; CREATE TABLE target (tid integer, balance integer) WITH (autovacuum_enabled=off); SELECT create_hypertable('target', 'tid', chunk_time_interval => 3); CREATE TABLE source (sid integer, delta integer) -- no index WITH (autovacuum_enabled=off); INSERT INTO target VALUES (1, 10); INSERT INTO target VALUES (2, 20); INSERT INTO target VALUES (3, 30); SELECT t.ctid is not null as matched, t.*, s.* FROM source s FULL OUTER JOIN target t ON s.sid = t.tid ORDER BY t.tid, s.sid; ALTER TABLE target OWNER TO regress_merge_privs; ALTER TABLE source OWNER TO regress_merge_privs; CREATE TABLE target2 (tid integer, balance integer) WITH (autovacuum_enabled=off); SELECT create_hypertable('target2', 'tid', chunk_time_interval => 3); CREATE TABLE source2 (sid integer, delta integer) WITH (autovacuum_enabled=off); ALTER TABLE target2 OWNER TO regress_merge_no_privs; ALTER TABLE source2 OWNER TO regress_merge_no_privs; GRANT INSERT ON target TO regress_merge_no_privs; GRANT CREATE ON SCHEMA public TO regress_merge_privs; SET SESSION AUTHORIZATION regress_merge_privs; CREATE TABLE sq_target (tid integer NOT NULL, balance integer) WITH (autovacuum_enabled=off); SELECT create_hypertable('sq_target', 'tid', chunk_time_interval => 3); CREATE TABLE sq_source (delta integer, sid integer, balance integer DEFAULT 0) WITH (autovacuum_enabled=off); INSERT INTO sq_target(tid, balance) VALUES (1,100), (2,200), (3,300); INSERT INTO sq_source(sid, delta) VALUES (1,10), (2,20), (4,40); -- conditional WHEN clause CREATE TABLE wq_target (tid integer not null, balance integer DEFAULT -1) WITH (autovacuum_enabled=off); SELECT create_hypertable('wq_target', 'tid', chunk_time_interval => 3); CREATE TABLE wq_source (balance integer, sid integer) WITH (autovacuum_enabled=off); INSERT INTO wq_source (sid, balance) VALUES (1, 100); -- some complex joins on the source side CREATE TABLE cj_target (tid integer, balance float, val text) WITH (autovacuum_enabled=off); SELECT create_hypertable('cj_target', 'tid', chunk_time_interval => 3); CREATE TABLE cj_source1 (sid1 integer, scat integer, delta integer) WITH (autovacuum_enabled=off); CREATE TABLE cj_source2 (sid2 integer, sval text) WITH (autovacuum_enabled=off); INSERT INTO cj_source1 VALUES (1, 10, 100); INSERT INTO cj_source1 VALUES (1, 20, 200); INSERT INTO cj_source1 VALUES (2, 20, 300); INSERT INTO cj_source1 VALUES (3, 10, 400); INSERT INTO cj_source2 VALUES (1, 'initial source2'); INSERT INTO cj_source2 VALUES (2, 'initial source2'); INSERT INTO cj_source2 VALUES (3, 'initial source2'); CREATE TABLE fs_target (a int, b int, c text) WITH (autovacuum_enabled=off); SELECT create_hypertable('fs_target', 'a', chunk_time_interval => 3); ================================================ FILE: test/sql/include/ts_merge_query.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- -- Errors -- MERGE INTO target t RANDOMWORD USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; -- MATCHED/INSERT error MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN INSERT DEFAULT VALUES; -- incorrectly specifying INTO target MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT INTO target DEFAULT VALUES; -- Multiple VALUES clause MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (1,1), (2,2); -- SELECT query for INSERT MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT SELECT (1, 1); -- NOT MATCHED/UPDATE MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN UPDATE SET balance = 0; -- UPDATE tablename MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE target SET balance = 0; -- source and target names the same MERGE INTO target USING target ON tid = tid WHEN MATCHED THEN DO NOTHING; -- used in a CTE WITH foo AS ( MERGE INTO target USING source ON (true) WHEN MATCHED THEN DELETE ) SELECT * FROM foo; -- used in COPY COPY ( MERGE INTO target USING source ON (true) WHEN MATCHED THEN DELETE ) TO stdout; -- unsupported relation types -- view CREATE VIEW tv AS SELECT * FROM target; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; DROP VIEW tv; -- materialized view CREATE MATERIALIZED VIEW mv AS SELECT * FROM target; MERGE INTO mv t USING source s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; DROP MATERIALIZED VIEW mv; -- permissions MERGE INTO target USING source2 ON target.tid = source2.sid WHEN MATCHED THEN UPDATE SET balance = 0; GRANT INSERT ON target TO regress_merge_no_privs; SET SESSION AUTHORIZATION regress_merge_no_privs; MERGE INTO target USING source2 ON target.tid = source2.sid WHEN MATCHED THEN UPDATE SET balance = 0; GRANT UPDATE ON target2 TO regress_merge_privs; SET SESSION AUTHORIZATION regress_merge_privs; MERGE INTO target2 USING source ON target2.tid = source.sid WHEN MATCHED THEN DELETE; MERGE INTO target2 USING source ON target2.tid = source.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; -- check if the target can be accessed from source relation subquery; we should -- not be able to do so MERGE INTO target t USING (SELECT * FROM source WHERE t.tid > sid) s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; -- -- initial tests -- -- zero rows in source has no effect MERGE INTO target USING source ON target.tid = source.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT DEFAULT VALUES; ROLLBACK; -- insert some non-matching source rows to work from INSERT INTO source VALUES (4, 40); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN DO NOTHING; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (5, 50); SELECT * FROM target ORDER BY tid; ROLLBACK; -- index plans INSERT INTO target SELECT generate_series(1000,2500), 0; ALTER TABLE target ADD PRIMARY KEY (tid); ANALYZE target; DELETE FROM target WHERE tid > 100; ANALYZE target; -- insert some matching source rows to work from INSERT INTO source VALUES (2, 5); INSERT INTO source VALUES (3, 20); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- equivalent of an UPDATE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; SELECT * FROM target ORDER BY tid; ROLLBACK; -- equivalent of a DELETE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DO NOTHING; SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, NULL); SELECT * FROM target ORDER BY tid; ROLLBACK; -- duplicate source row causes multiple target row update ERROR INSERT INTO source VALUES (2, 5); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0; ROLLBACK; BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN DELETE; ROLLBACK; -- remove duplicate MATCHED data from source data DELETE FROM source WHERE sid = 2; INSERT INTO source VALUES (2, 5); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- duplicate source row on INSERT should fail because of target_pkey INSERT INTO source VALUES (4, 40); BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, NULL); SELECT * FROM target ORDER BY tid; ROLLBACK; -- remove duplicate NOT MATCHED data from source data DELETE FROM source WHERE sid = 4; INSERT INTO source VALUES (4, 40); SELECT * FROM source ORDER BY sid; SELECT * FROM target ORDER BY tid; -- remove constraints alter table target drop CONSTRAINT target_pkey; alter table target alter column tid drop not null; -- multiple actions BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (4, 4) WHEN MATCHED THEN UPDATE SET balance = 0; SELECT * FROM target ORDER BY tid; ROLLBACK; -- should be equivalent BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = 0 WHEN NOT MATCHED THEN INSERT VALUES (4, 4); SELECT * FROM target ORDER BY tid; ROLLBACK; -- column references -- do a simple equivalent of an UPDATE join BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance + s.delta; SELECT * FROM target ORDER BY tid; ROLLBACK; -- do a simple equivalent of an INSERT SELECT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- and again with duplicate source rows INSERT INTO source VALUES (5, 50); INSERT INTO source VALUES (5, 50); -- do a simple equivalent of an INSERT SELECT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- removing duplicate source rows DELETE FROM source WHERE sid = 5; -- and again with explicitly identified column list BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- and again with a subtle error: referring to non-existent target row for NOT MATCHED MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (t.tid, s.delta); -- and again with a constant ON clause BEGIN; MERGE INTO target t USING source AS s ON (SELECT true) WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (t.tid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- now the classic UPSERT BEGIN; MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance + s.delta WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- this time with a FALSE condition MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND FALSE THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; -- this time with an actual condition which returns false MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance <> 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; BEGIN; -- and now with a condition which returns true MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance = 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; ROLLBACK; -- conditions in the NOT MATCHED clause can only refer to source columns BEGIN; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND t.balance = 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; ROLLBACK; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN NOT MATCHED AND s.balance = 100 THEN INSERT (tid) VALUES (s.sid); SELECT * FROM wq_target; -- conditions in MATCHED clause can refer to both source and target SELECT * FROM wq_source; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND s.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check if AND works MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 AND s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 AND s.balance = 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check if OR works MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 99 OR s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance = 199 OR s.balance > 100 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; -- check source-side whole-row references BEGIN; MERGE INTO wq_target t USING wq_source s ON (t.tid = s.sid) WHEN matched and t = s or t.tid = s.sid THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; ROLLBACK; -- check if subqueries work in the conditions? MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.balance > (SELECT max(balance) FROM target) THEN UPDATE SET balance = t.balance + s.balance; -- check if we can access system columns in the conditions MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.xmin = t.xmax THEN UPDATE SET balance = t.balance + s.balance; MERGE INTO wq_target t USING wq_source s ON t.tid = s.sid WHEN MATCHED AND t.tableoid >= 0 THEN UPDATE SET balance = t.balance + s.balance; SELECT * FROM wq_target; DROP TABLE wq_target CASCADE; DROP TABLE wq_source; -- test triggers create or replace function merge_trigfunc () returns trigger language plpgsql as $$ DECLARE line text; BEGIN SELECT INTO line format('%s %s %s trigger%s', TG_WHEN, TG_OP, TG_LEVEL, CASE WHEN TG_OP = 'INSERT' AND TG_LEVEL = 'ROW' THEN format(' row: %s', NEW) WHEN TG_OP = 'UPDATE' AND TG_LEVEL = 'ROW' THEN format(' row: %s -> %s', OLD, NEW) WHEN TG_OP = 'DELETE' AND TG_LEVEL = 'ROW' THEN format(' row: %s', OLD) END); RAISE NOTICE '%', line; IF (TG_WHEN = 'BEFORE' AND TG_LEVEL = 'ROW') THEN IF (TG_OP = 'DELETE') THEN RETURN OLD; ELSE RETURN NEW; END IF; ELSE RETURN NULL; END IF; END; $$; CREATE TRIGGER merge_bsi BEFORE INSERT ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bsu BEFORE UPDATE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bsd BEFORE DELETE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asi AFTER INSERT ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asu AFTER UPDATE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_asd AFTER DELETE ON target FOR EACH STATEMENT EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bri BEFORE INSERT ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_bru BEFORE UPDATE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_brd BEFORE DELETE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_ari AFTER INSERT ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_aru AFTER UPDATE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); CREATE TRIGGER merge_ard AFTER DELETE ON target FOR EACH ROW EXECUTE PROCEDURE merge_trigfunc (); -- now the classic UPSERT, with a DELETE BEGIN; UPDATE target SET balance = 0 WHERE tid = 3; --EXPLAIN (ANALYZE ON, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND t.balance > s.delta THEN UPDATE SET balance = t.balance - s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- Test behavior of triggers that turn UPDATE/DELETE into no-ops create or replace function skip_merge_op() returns trigger language plpgsql as $$ BEGIN RETURN NULL; END; $$; SELECT * FROM target full outer join source on (sid = tid); create trigger merge_skip BEFORE INSERT OR UPDATE or DELETE ON target FOR EACH ROW EXECUTE FUNCTION skip_merge_op(); DO $$ DECLARE result integer; BEGIN MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND s.sid = 3 THEN UPDATE SET balance = t.balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT VALUES (sid, delta); IF FOUND THEN RAISE NOTICE 'Found'; ELSE RAISE NOTICE 'Not found'; END IF; GET DIAGNOSTICS result := ROW_COUNT; RAISE NOTICE 'ROW_COUNT = %', result; END; $$; SELECT * FROM target FULL OUTER JOIN source ON (sid = tid); DROP TRIGGER merge_skip ON target; DROP FUNCTION skip_merge_op(); -- test from PL/pgSQL -- make sure MERGE INTO isn't interpreted to mean returning variables like SELECT INTO BEGIN; DO LANGUAGE plpgsql $$ BEGIN MERGE INTO target t USING source AS s ON t.tid = s.sid WHEN MATCHED AND t.balance > s.delta THEN UPDATE SET balance = t.balance - s.delta; END; $$; ROLLBACK; --source constants BEGIN; MERGE INTO target t USING (SELECT 9 AS sid, 57 AS delta) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; --source query BEGIN; MERGE INTO target t USING (SELECT sid, delta FROM source WHERE delta > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT sid, delta as newname FROM source WHERE delta > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.newname); SELECT * FROM target ORDER BY tid; ROLLBACK; --self-merge BEGIN; MERGE INTO target t1 USING target t2 ON t1.tid = t2.tid WHEN MATCHED THEN UPDATE SET balance = t1.balance + t2.balance WHEN NOT MATCHED THEN INSERT VALUES (t2.tid, t2.balance); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT tid as sid, balance as delta FROM target WHERE balance > 0) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; BEGIN; MERGE INTO target t USING (SELECT sid, max(delta) AS delta FROM source GROUP BY sid HAVING count(*) = 1 ORDER BY sid ASC) AS s ON t.tid = s.sid WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (s.sid, s.delta); SELECT * FROM target ORDER BY tid; ROLLBACK; -- plpgsql parameters and results BEGIN; CREATE FUNCTION merge_func (p_id integer, p_bal integer) RETURNS INTEGER LANGUAGE plpgsql AS $$ DECLARE result integer; BEGIN MERGE INTO target t USING (SELECT p_id AS sid) AS s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = t.balance - p_bal; IF FOUND THEN GET DIAGNOSTICS result := ROW_COUNT; END IF; RETURN result; END; $$; SELECT merge_func(3, 4); SELECT * FROM target ORDER BY tid; ROLLBACK; -- PREPARE BEGIN; prepare foom as merge into target t using (select 1 as sid) s on (t.tid = s.sid) when matched then update set balance = 1; execute foom; ROLLBACK; BEGIN; PREPARE foom2 (integer, integer) AS MERGE INTO target t USING (SELECT 1) s ON t.tid = $1 WHEN MATCHED THEN UPDATE SET balance = $2; --EXPLAIN (ANALYZE ON, BUFFERS OFF, COSTS OFF, SUMMARY OFF, TIMING OFF) execute foom2 (1, 1); ROLLBACK; -- subqueries in source relation BEGIN; MERGE INTO sq_target t USING (SELECT * FROM sq_source) s ON tid = sid WHEN MATCHED AND t.balance > delta THEN UPDATE SET balance = t.balance + delta; SELECT * FROM sq_target ORDER BY tid; ROLLBACK; -- try a view CREATE VIEW v AS SELECT * FROM sq_source WHERE sid < 2; BEGIN; MERGE INTO sq_target USING v ON tid = sid WHEN MATCHED THEN UPDATE SET balance = v.balance + delta; SELECT * FROM sq_target ORDER BY tid; ROLLBACK; -- ambiguous reference to a column BEGIN; MERGE INTO sq_target USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; ROLLBACK; BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; SELECT * FROM sq_target; ROLLBACK; -- CTEs BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); WITH targq AS ( SELECT * FROM v ) MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE; ROLLBACK; -- RETURNING BEGIN; INSERT INTO sq_source (sid, balance, delta) VALUES (-1, -1, -10); MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND tid > 2 THEN UPDATE SET balance = t.balance + delta WHEN NOT MATCHED THEN INSERT (balance, tid) VALUES (balance + delta, sid) WHEN MATCHED AND tid < 2 THEN DELETE RETURNING *; ROLLBACK; -- PG17-specific tests for views, returning and merge_action. These throw syntax errors for previous versions of Postgres. -- However, since the error is the same for both hypertables and regular tables, this test should still pass for previous versions. -- RETURNING INSERT INTO source(sid, delta) VALUES(1, 40), (5, 50); BEGIN; MERGE INTO target t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta); SELECT * from target; ROLLBACK; BEGIN; MERGE INTO target t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 1 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta) RETURNING merge_action(), t.*; ROLLBACK; -- Views CREATE VIEW tv AS SELECT * FROM target; BEGIN; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta); SELECT * from tv; SELECT * from target; -- should also update the underlying table ROLLBACK; BEGIN; MERGE INTO tv t USING source s ON t.tid = s.sid WHEN MATCHED AND tid > 2 THEN UPDATE set balance = balance + s.delta WHEN MATCHED THEN DELETE WHEN NOT MATCHED THEN INSERT (tid, balance) VALUES (sid, delta) RETURNING merge_action(), t.*; ROLLBACK; DROP VIEW tv; DELETE FROM source where sid in (1, 5); -- EXPLAIN CREATE TABLE ex_mtarget (a int, b int) WITH (autovacuum_enabled=off); CREATE TABLE ex_msource (a int, b int) WITH (autovacuum_enabled=off); INSERT INTO ex_mtarget SELECT i, i*10 FROM generate_series(1,100,2) i; INSERT INTO ex_msource SELECT i, i*10 FROM generate_series(1,100,1) i; CREATE FUNCTION explain_merge(query text) RETURNS SETOF text LANGUAGE plpgsql AS $$ DECLARE ln text; BEGIN FOR ln IN EXECUTE 'explain (analyze, timing off, summary off, buffers off, costs off) ' || query LOOP ln := regexp_replace(ln, '(Memory( Usage)?|Buckets|Batches): \S*', '\1: xxx', 'g'); RETURN NEXT ln; END LOOP; END; $$; -- only updates SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED THEN UPDATE SET b = t.b + 1'); -- only updates to selected tuples SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1'); -- updates + deletes SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1 WHEN MATCHED AND t.a >= 10 AND t.a <= 20 THEN DELETE'); -- only inserts SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN NOT MATCHED AND s.a < 10 THEN INSERT VALUES (a, b)'); -- all three SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a WHEN MATCHED AND t.a < 10 THEN UPDATE SET b = t.b + 1 WHEN MATCHED AND t.a >= 30 AND t.a <= 40 THEN DELETE WHEN NOT MATCHED AND s.a < 20 THEN INSERT VALUES (a, b)'); -- nothing SELECT explain_merge(' MERGE INTO ex_mtarget t USING ex_msource s ON t.a = s.a AND t.a < -1000 WHEN MATCHED AND t.a < 10 THEN DO NOTHING'); DROP TABLE ex_msource, ex_mtarget; DROP FUNCTION explain_merge(text); -- Subqueries BEGIN; MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED THEN UPDATE SET balance = (SELECT count(*) FROM sq_target); SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; BEGIN; MERGE INTO sq_target t USING v ON tid = sid WHEN MATCHED AND (SELECT count(*) > 0 FROM sq_target) THEN UPDATE SET balance = 42; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; BEGIN; MERGE INTO sq_target t USING v ON tid = sid AND (SELECT count(*) > 0 FROM sq_target) WHEN MATCHED THEN UPDATE SET balance = 42; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; -- Test RETURNING with subqueries BEGIN; MERGE INTO sq_target t USING v ON tid = sid AND (SELECT count(*) > 0 FROM sq_target) WHEN MATCHED THEN UPDATE SET balance = 42 RETURNING *; SELECT * FROM sq_target WHERE tid = 1; ROLLBACK; DROP TABLE sq_target CASCADE; DROP TABLE sq_source CASCADE; CREATE TABLE pa_target (tid integer, balance float, val text) PARTITION BY LIST (tid); CREATE TABLE part1 PARTITION OF pa_target FOR VALUES IN (1,4) WITH (autovacuum_enabled=off); CREATE TABLE part2 PARTITION OF pa_target FOR VALUES IN (2,5,6) WITH (autovacuum_enabled=off); CREATE TABLE part3 PARTITION OF pa_target FOR VALUES IN (3,8,9) WITH (autovacuum_enabled=off); CREATE TABLE part4 PARTITION OF pa_target DEFAULT WITH (autovacuum_enabled=off); CREATE TABLE pa_source (sid integer, delta float); -- insert many rows to the source table INSERT INTO pa_source SELECT id, id * 10 FROM generate_series(1,14) AS id; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT id, id * 100, 'initial' FROM generate_series(1,14,2) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- same with a constant qual BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid AND tid = 1 WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- try updating the partition key column BEGIN; CREATE FUNCTION merge_func() RETURNS integer LANGUAGE plpgsql AS $$ DECLARE result integer; BEGIN MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET tid = tid + 1, balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); IF FOUND THEN GET DIAGNOSTICS result := ROW_COUNT; END IF; RETURN result; END; $$; SELECT merge_func(); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_target CASCADE; -- The target table is partitioned in the same way, but this time by attaching -- partitions which have columns in different order, dropped columns etc. CREATE TABLE pa_target (tid integer, balance float, val text) PARTITION BY LIST (tid); CREATE TABLE part1 (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE part2 (balance float, tid integer, val text) WITH (autovacuum_enabled=off); CREATE TABLE part3 (tid integer, balance float, val text) WITH (autovacuum_enabled=off); CREATE TABLE part4 (extraid text, tid integer, balance float, val text) WITH (autovacuum_enabled=off); ALTER TABLE part4 DROP COLUMN extraid; ALTER TABLE pa_target ATTACH PARTITION part1 FOR VALUES IN (1,4); ALTER TABLE pa_target ATTACH PARTITION part2 FOR VALUES IN (2,5,6); ALTER TABLE pa_target ATTACH PARTITION part3 FOR VALUES IN (3,8,9); ALTER TABLE pa_target ATTACH PARTITION part4 DEFAULT; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT id, id * 100, 'initial' FROM generate_series(1,14,2) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- same with a constant qual BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid AND tid IN (1, 5) WHEN MATCHED AND tid % 5 = 0 THEN DELETE WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; -- try updating the partition key column BEGIN; MERGE INTO pa_target t USING pa_source s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET tid = tid + 1, balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_source; DROP TABLE pa_target CASCADE; -- Sub-partitioning CREATE TABLE pa_target (logts timestamp, tid integer, balance float, val text) PARTITION BY RANGE (logts); CREATE TABLE part_m01 PARTITION OF pa_target FOR VALUES FROM ('2017-01-01') TO ('2017-02-01') PARTITION BY LIST (tid); CREATE TABLE part_m01_odd PARTITION OF part_m01 FOR VALUES IN (1,3,5,7,9) WITH (autovacuum_enabled=off); CREATE TABLE part_m01_even PARTITION OF part_m01 FOR VALUES IN (2,4,6,8) WITH (autovacuum_enabled=off); CREATE TABLE part_m02 PARTITION OF pa_target FOR VALUES FROM ('2017-02-01') TO ('2017-03-01') PARTITION BY LIST (tid); CREATE TABLE part_m02_odd PARTITION OF part_m02 FOR VALUES IN (1,3,5,7,9) WITH (autovacuum_enabled=off); CREATE TABLE part_m02_even PARTITION OF part_m02 FOR VALUES IN (2,4,6,8) WITH (autovacuum_enabled=off); CREATE TABLE pa_source (sid integer, delta float) WITH (autovacuum_enabled=off); -- insert many rows to the source table INSERT INTO pa_source SELECT id, id * 10 FROM generate_series(1,14) AS id; -- insert a few rows in the target table (odd numbered tid) INSERT INTO pa_target SELECT '2017-01-31', id, id * 100, 'initial' FROM generate_series(1,9,3) AS id; INSERT INTO pa_target SELECT '2017-02-28', id, id * 100, 'initial' FROM generate_series(2,9,3) AS id; -- try simple MERGE BEGIN; MERGE INTO pa_target t USING (SELECT '2017-01-15' AS slogts, * FROM pa_source WHERE sid < 10) s ON t.tid = s.sid WHEN MATCHED THEN UPDATE SET balance = balance + delta, val = val || ' updated by merge' WHEN NOT MATCHED THEN INSERT VALUES (slogts::timestamp, sid, delta, 'inserted by merge'); SELECT * FROM pa_target ORDER BY tid; ROLLBACK; DROP TABLE pa_source; DROP TABLE pa_target CASCADE; -- some complex joins on the source side -- source relation is an unaliased join MERGE INTO cj_target t USING cj_source1 s1 INNER JOIN cj_source2 s2 ON sid1 = sid2 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid1, delta, sval); -- try accessing columns from either side of the source join MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 AND scat = 20 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid2, delta, sval) WHEN MATCHED THEN DELETE; -- some simple expressions in INSERT targetlist MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 ON t.tid = sid1 WHEN NOT MATCHED THEN INSERT VALUES (sid2, delta + scat, sval) WHEN MATCHED THEN UPDATE SET val = val || ' updated by merge'; MERGE INTO cj_target t USING cj_source2 s2 INNER JOIN cj_source1 s1 ON sid1 = sid2 AND scat = 20 ON t.tid = sid1 WHEN MATCHED THEN UPDATE SET val = val || ' ' || delta::text; SELECT * FROM cj_target ORDER BY tid; ALTER TABLE cj_source1 RENAME COLUMN sid1 TO sid; ALTER TABLE cj_source2 RENAME COLUMN sid2 TO sid; TRUNCATE cj_target; MERGE INTO cj_target t USING cj_source1 s1 INNER JOIN cj_source2 s2 ON s1.sid = s2.sid ON t.tid = s1.sid WHEN NOT MATCHED THEN INSERT VALUES (s2.sid, delta, sval); DROP TABLE cj_source2, cj_source1; DROP TABLE cj_target CASCADE; -- Function scans MERGE INTO fs_target t USING generate_series(1,100,1) AS id ON t.a = id WHEN MATCHED THEN UPDATE SET b = b + id WHEN NOT MATCHED THEN INSERT VALUES (id, -1); MERGE INTO fs_target t USING generate_series(1,100,2) AS id ON t.a = id WHEN MATCHED THEN UPDATE SET b = b + id, c = 'updated '|| id.*::text WHEN NOT MATCHED THEN INSERT VALUES (id, -1, 'inserted ' || id.*::text); SELECT count(*) FROM fs_target; DROP TABLE fs_target CASCADE; -- SERIALIZABLE test -- handled in isolation tests -- Inheritance-based partitioning CREATE TABLE measurement ( city_id int not null, logdate date not null, peaktemp int, unitsales int ) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2006m02 ( CHECK ( logdate >= DATE '2006-02-01' AND logdate < DATE '2006-03-01' ) ) INHERITS (measurement) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2006m03 ( CHECK ( logdate >= DATE '2006-03-01' AND logdate < DATE '2006-04-01' ) ) INHERITS (measurement) WITH (autovacuum_enabled=off); CREATE TABLE measurement_y2007m01 ( filler text, peaktemp int, logdate date not null, city_id int not null, unitsales int CHECK ( logdate >= DATE '2007-01-01' AND logdate < DATE '2007-02-01') ) WITH (autovacuum_enabled=off); ALTER TABLE measurement_y2007m01 DROP COLUMN filler; ALTER TABLE measurement_y2007m01 INHERIT measurement; INSERT INTO measurement VALUES (0, '2005-07-21', 5, 15); CREATE OR REPLACE FUNCTION measurement_insert_trigger() RETURNS TRIGGER AS $$ BEGIN IF ( NEW.logdate >= DATE '2006-02-01' AND NEW.logdate < DATE '2006-03-01' ) THEN INSERT INTO measurement_y2006m02 VALUES (NEW.*); ELSIF ( NEW.logdate >= DATE '2006-03-01' AND NEW.logdate < DATE '2006-04-01' ) THEN INSERT INTO measurement_y2006m03 VALUES (NEW.*); ELSIF ( NEW.logdate >= DATE '2007-01-01' AND NEW.logdate < DATE '2007-02-01' ) THEN INSERT INTO measurement_y2007m01 (city_id, logdate, peaktemp, unitsales) VALUES (NEW.*); ELSE RAISE EXCEPTION 'Date out of range. Fix the measurement_insert_trigger() function!'; END IF; RETURN NULL; END; $$ LANGUAGE plpgsql ; CREATE TRIGGER insert_measurement_trigger BEFORE INSERT ON measurement FOR EACH ROW EXECUTE PROCEDURE measurement_insert_trigger(); INSERT INTO measurement VALUES (1, '2006-02-10', 35, 10); INSERT INTO measurement VALUES (1, '2006-02-16', 45, 20); INSERT INTO measurement VALUES (1, '2006-03-17', 25, 10); INSERT INTO measurement VALUES (1, '2006-03-27', 15, 40); INSERT INTO measurement VALUES (1, '2007-01-15', 10, 10); INSERT INTO measurement VALUES (1, '2007-01-17', 10, 10); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate; CREATE TABLE new_measurement (LIKE measurement) WITH (autovacuum_enabled=off); INSERT INTO new_measurement VALUES (0, '2005-07-21', 25, 20); INSERT INTO new_measurement VALUES (1, '2006-03-01', 20, 10); INSERT INTO new_measurement VALUES (1, '2006-02-16', 50, 10); INSERT INTO new_measurement VALUES (2, '2006-02-10', 20, 20); INSERT INTO new_measurement VALUES (1, '2006-03-27', NULL, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-17', NULL, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-15', 5, NULL); INSERT INTO new_measurement VALUES (1, '2007-01-16', 10, 10); BEGIN; MERGE INTO ONLY measurement m USING new_measurement nm ON (m.city_id = nm.city_id and m.logdate=nm.logdate) WHEN MATCHED AND nm.peaktemp IS NULL THEN DELETE WHEN MATCHED THEN UPDATE SET peaktemp = greatest(m.peaktemp, nm.peaktemp), unitsales = m.unitsales + coalesce(nm.unitsales, 0) WHEN NOT MATCHED THEN INSERT (city_id, logdate, peaktemp, unitsales) VALUES (city_id, logdate, peaktemp, unitsales); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate, peaktemp; ROLLBACK; MERGE into measurement m USING new_measurement nm ON (m.city_id = nm.city_id and m.logdate=nm.logdate) WHEN MATCHED AND nm.peaktemp IS NULL THEN DELETE WHEN MATCHED THEN UPDATE SET peaktemp = greatest(m.peaktemp, nm.peaktemp), unitsales = m.unitsales + coalesce(nm.unitsales, 0) WHEN NOT MATCHED THEN INSERT (city_id, logdate, peaktemp, unitsales) VALUES (city_id, logdate, peaktemp, unitsales); SELECT tableoid::regclass, * FROM measurement ORDER BY city_id, logdate; BEGIN; MERGE INTO new_measurement nm USING ONLY measurement m ON (nm.city_id = m.city_id and nm.logdate=m.logdate) WHEN MATCHED THEN DELETE; SELECT * FROM new_measurement ORDER BY city_id, logdate; ROLLBACK; MERGE INTO new_measurement nm USING measurement m ON (nm.city_id = m.city_id and nm.logdate=m.logdate) WHEN MATCHED THEN DELETE; SELECT * FROM new_measurement ORDER BY city_id, logdate; DROP TABLE measurement, new_measurement CASCADE; DROP FUNCTION measurement_insert_trigger(); RESET SESSION AUTHORIZATION; DROP TABLE target CASCADE; DROP TABLE target2 CASCADE; DROP TABLE source, source2; DROP FUNCTION merge_trigfunc(); REVOKE CREATE ON SCHEMA public FROM regress_merge_privs; DROP USER regress_merge_privs; DROP USER regress_merge_no_privs; ================================================ FILE: test/sql/index.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE index_test(time timestamptz, temp float); SELECT create_hypertable('index_test', 'time'); -- Default indexes created SELECT * FROM test.show_indexes('index_test'); DROP TABLE index_test; CREATE TABLE index_test(time timestamptz, device integer, temp float); -- Create index before create_hypertable() CREATE UNIQUE INDEX index_test_time_idx ON index_test (time); \set ON_ERROR_STOP 0 -- Creating a hypertable from a table with an index that doesn't cover -- all partitioning columns should fail SELECT create_hypertable('index_test', 'time', 'device', 2); \set ON_ERROR_STOP 1 -- Partitioning on only time should work SELECT create_hypertable('index_test', 'time'); INSERT INTO index_test VALUES ('2017-01-20T09:00:01', 1, 17.5); -- Check that index is also created on chunk SELECT * FROM test.show_indexes('index_test'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); -- Create another chunk INSERT INTO index_test VALUES ('2017-05-20T09:00:01', 3, 17.5); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); -- Delete the index on only one chunk DROP INDEX _timescaledb_internal._hyper_3_1_chunk_index_test_time_idx; SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); -- Recreate table with new partitioning DROP TABLE index_test; CREATE TABLE index_test(id serial, time timestamptz, device integer, temp float); SELECT * FROM test.show_columns('index_test'); -- Test that we can handle difference in attnos across hypertable and -- chunks by dropping the ID column ALTER TABLE index_test DROP COLUMN id; SELECT * FROM test.show_columns('index_test'); -- No pre-existing UNIQUE index, so partitioning on two columns should work SELECT create_hypertable('index_test', 'time', 'device', 2); INSERT INTO index_test VALUES ('2017-01-20T09:00:01', 1, 17.5); \set ON_ERROR_STOP 0 -- Create unique index without all partitioning columns should fail CREATE UNIQUE INDEX index_test_time_device_idx ON index_test (time); \set ON_ERROR_STOP 1 CREATE UNIQUE INDEX index_test_time_device_idx ON index_test (time, device); -- Regular index need not cover all partitioning columns CREATE INDEX ON index_test (time, temp); -- Create another chunk INSERT INTO index_test VALUES ('2017-04-20T09:00:01', 1, 17.5); -- New index should have been recursed to chunks SELECT * FROM test.show_indexes('index_test'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); ALTER INDEX index_test_time_idx RENAME TO index_test_time_idx2; -- Metadata and index should have changed name SELECT * FROM test.show_indexes('index_test'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); DROP INDEX index_test_time_idx2; DROP INDEX index_test_time_device_idx; -- Index should have been dropped SELECT * FROM test.show_indexes('index_test'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); -- Create index with long name to see how this is handled on chunks CREATE INDEX a_hypertable_index_with_a_very_very_long_name_that_truncates ON index_test (time DESC, temp); CREATE INDEX a_hypertable_index_with_a_very_very_long_name_that_truncates_2 ON index_test (time DESC, temp DESC); SELECT * FROM test.show_indexes('index_test'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); DROP INDEX a_hypertable_index_with_a_very_very_long_name_that_truncates; DROP INDEX a_hypertable_index_with_a_very_very_long_name_that_truncates_2; \set ON_ERROR_STOP 0 -- Create index CONCURRENTLY CREATE UNIQUE INDEX CONCURRENTLY index_test_time_device_idx ON index_test (time, device); \set ON_ERROR_STOP 1 -- Test tablespaces. Chunk indexes should end up in same tablespace as -- main index. \c :TEST_DBNAME :ROLE_SUPERUSER SET client_min_messages = ERROR; DROP TABLESPACE IF EXISTS tablespace1; DROP TABLESPACE IF EXISTS tablespace2; SET client_min_messages = NOTICE; CREATE TABLESPACE tablespace1 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE1_PATH; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE INDEX index_test_time_idx ON index_test (time) TABLESPACE tablespace1; SELECT * FROM test.show_indexes('index_test'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); \c :TEST_DBNAME :ROLE_SUPERUSER CREATE TABLESPACE tablespace2 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE2_PATH; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER ALTER INDEX index_test_time_idx SET TABLESPACE tablespace2; SELECT * FROM test.show_indexes('index_test'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); -- Add constraint index ALTER TABLE index_test ADD UNIQUE (time, device); SELECT * FROM test.show_indexes('index_test'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); -- Constraints are added to chunk_constraint table. SELECT * FROM _timescaledb_catalog.chunk_constraint; DROP TABLE index_test; -- Create table in a tablespace CREATE TABLE index_test(time timestamptz, temp float, device int) TABLESPACE tablespace1; -- Default indexes should be in the table's tablespace SELECT create_hypertable('index_test', 'time'); -- Explicitly defining an index tablespace should work and propagate -- to chunks CREATE INDEX ON index_test (time, device) TABLESPACE tablespace2; -- New indexes without explicit tablespaces should use the default -- tablespace CREATE INDEX ON index_test (device); -- Create chunk INSERT INTO index_test VALUES ('2017-01-20T09:00:01', 17.5); -- Check that the tablespaces of chunk indexes match the tablespace of -- the main index SELECT * FROM test.show_indexes('index_test'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); -- Creating a new index should propagate to existing chunks, including -- the given tablespace CREATE INDEX ON index_test (time, temp) TABLESPACE tablespace2; SELECT * FROM test.show_indexes('index_test'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); -- Cleanup DROP TABLE index_test CASCADE; \c :TEST_DBNAME :ROLE_SUPERUSER DROP TABLESPACE tablespace1; DROP TABLESPACE tablespace2; -- Test expression indexes CREATE TABLE index_expr_test(id serial, time timestamptz, temp float, meta jsonb); -- Screw up the attribute numbers ALTER TABLE index_expr_test DROP COLUMN id; CREATE INDEX ON index_expr_test ((meta ->> 'field')) ; INSERT INTO index_expr_test VALUES ('2017-01-20T09:00:01', 17.5, '{"field": "value1"}'); INSERT INTO index_expr_test VALUES ('2017-01-20T09:00:01', 17.5, '{"field": "value2"}'); EXPLAIN (verbose, buffers off, costs off) SELECT * FROM index_expr_test WHERE meta ->> 'field' = 'value1'; SELECT * FROM index_expr_test WHERE meta ->> 'field' = 'value1'; -- Test INDEX DROP error for multiple objects CREATE TABLE index_test(time timestamptz, temp float); CREATE UNIQUE INDEX index_test_idx ON index_test (time, temp); SELECT create_hypertable('index_test', 'time'); CREATE TABLE index_test_2(time timestamptz, temp float); CREATE UNIQUE INDEX index_test_2_idx ON index_test_2 (time, temp); \set ON_ERROR_STOP 0 DROP INDEX index_test_idx, index_test_2_idx; \set ON_ERROR_STOP 1 -- test expression index with dropped columns CREATE TABLE idx_expr_test(filler int, time timestamptz, meta text); SELECT table_name FROM create_hypertable('idx_expr_test', 'time'); ALTER TABLE idx_expr_test DROP COLUMN filler; CREATE INDEX tag_idx ON idx_expr_test(('foo'||meta)); INSERT INTO idx_expr_test(time, meta) VALUES ('2000-01-01', 'bar'); DROP TABLE idx_expr_test CASCADE; -- test multicolumn expression index with dropped columns CREATE TABLE idx_expr_test(filler int, time timestamptz, t1 text, t2 text, t3 text); SELECT table_name FROM create_hypertable('idx_expr_test', 'time'); ALTER TABLE idx_expr_test DROP COLUMN filler; CREATE INDEX tag_idx ON idx_expr_test((t1||t2||t3)); INSERT INTO idx_expr_test(time, t1, t2, t3) VALUES ('2000-01-01', 'foo', 'bar', 'baz'); DROP TABLE idx_expr_test CASCADE; -- test index with predicate and dropped columns CREATE TABLE idx_predicate_test(filler int, time timestamptz); SELECT table_name FROM create_hypertable('idx_predicate_test', 'time'); ALTER TABLE idx_predicate_test DROP COLUMN filler; ALTER TABLE idx_predicate_test ADD COLUMN b1 bool; CREATE INDEX idx_predicate_test_b1 ON idx_predicate_test(b1) WHERE b1=true; INSERT INTO idx_predicate_test VALUES ('2000-01-01',true); DROP TABLE idx_predicate_test; -- test index with table references CREATE TABLE idx_tableref_test(time timestamptz); SELECT table_name FROM create_hypertable('idx_tableref_test', 'time'); -- we use security definer to prevent function inlining CREATE OR REPLACE FUNCTION tableref_func(t idx_tableref_test) RETURNS timestamptz LANGUAGE SQL IMMUTABLE SECURITY DEFINER AS $f$ SELECT t.time; $f$; -- try creating index with no existing chunks CREATE INDEX tableref_idx ON idx_tableref_test(tableref_func(idx_tableref_test)); -- insert data to trigger chunk creation INSERT INTO idx_tableref_test SELECT '2000-01-01'; DROP INDEX tableref_idx; -- try creating index on hypertable with existing chunks CREATE INDEX tableref_idx ON idx_tableref_test(tableref_func(idx_tableref_test)); -- test index creation with if not exists CREATE TABLE idx_exists(time timestamptz NOT NULL); SELECT table_name FROM create_hypertable('idx_exists', 'time'); -- should be skipped since this index was already created by create_hypertable CREATE INDEX IF NOT EXISTS idx_exists_time_idx ON idx_exists(time DESC); -- should create index CREATE INDEX IF NOT EXISTS idx_exists_time_asc_idx ON idx_exists(time ASC); -- should be skipped since it was created in previous command CREATE INDEX IF NOT EXISTS idx_exists_time_asc_idx ON idx_exists(time ASC); DROP INDEX idx_exists_time_asc_idx; INSERT INTO idx_exists VALUES ('2000-01-01'),('2001-01-01'); -- should create index CREATE INDEX IF NOT EXISTS idx_exists_time_asc_idx ON idx_exists(time ASC); -- should be skipped since it was created in previous command CREATE INDEX IF NOT EXISTS idx_exists_time_asc_idx ON idx_exists(time ASC); -- test reindex CREATE TABLE reindex_test(time timestamp, temp float, PRIMARY KEY(time, temp)); CREATE UNIQUE INDEX reindex_test_time_unique_idx ON reindex_test(time); -- create hypertable with three chunks SELECT create_hypertable('reindex_test', 'time', chunk_time_interval => 2628000000000); INSERT INTO reindex_test VALUES ('2017-01-20T09:00:01', 17.5), ('2017-01-21T09:00:01', 19.1), ('2017-04-20T09:00:01', 89.5), ('2017-04-21T09:00:01', 17.1), ('2017-06-20T09:00:01', 18.5), ('2017-06-21T09:00:01', 11.0); SELECT * FROM test.show_columns('reindex_test'); SELECT * FROM test.show_subtables('reindex_test'); -- show reindexing REINDEX (VERBOSE) TABLE reindex_test; \set ON_ERROR_STOP 0 -- REINDEX TABLE CONCURRENTLY on hypertables is not supported REINDEX TABLE CONCURRENTLY reindex_test; -- this one currently doesn't recurse to chunks and instead gives an -- error REINDEX (VERBOSE) INDEX reindex_test_time_unique_idx; \set ON_ERROR_STOP 1 -- show reindexing on a normal table CREATE TABLE reindex_norm(time timestamp, temp float); CREATE UNIQUE INDEX reindex_norm_time_unique_idx ON reindex_norm(time); INSERT INTO reindex_norm VALUES ('2017-01-20T09:00:01', 17.5), ('2017-01-21T09:00:01', 19.1), ('2017-04-20T09:00:01', 89.5), ('2017-04-21T09:00:01', 17.1), ('2017-06-20T09:00:01', 18.5), ('2017-06-21T09:00:01', 11.0); REINDEX (VERBOSE) TABLE reindex_norm; REINDEX (VERBOSE) INDEX reindex_norm_time_unique_idx; SELECT * FROM test.show_constraintsp('_timescaledb_internal._hyper_12%'); SELECT * FROM reindex_norm; CREATE TABLE ht_dropped(time timestamptz, d0 int, d1 int, c0 int, c1 int, c2 int); SELECT create_hypertable('ht_dropped','time'); INSERT INTO ht_dropped(time,c0,c1,c2) SELECT '2000-01-01',1,2,3; ALTER TABLE ht_dropped DROP COLUMN d0; INSERT INTO ht_dropped(time,c0,c1,c2) SELECT '2001-01-01',1,2,3; ALTER TABLE ht_dropped DROP COLUMN d1; INSERT INTO ht_dropped(time,c0,c1,c2) SELECT '2002-01-01',1,2,3; CREATE INDEX ON ht_dropped(c0,c1,c2) WHERE c1 IS NOT NULL; CREATE INDEX ON ht_dropped(c0,c1,c2) WITH(timescaledb.transaction_per_chunk) WHERE c2 IS NOT NULL; SELECT oid::TEXT AS "Chunk", i.* FROM (SELECT tableoid::REGCLASS FROM ht_dropped GROUP BY tableoid) ch (oid) LEFT JOIN LATERAL ( SELECT * FROM test.show_indexes (ch.oid)) i ON TRUE ORDER BY 1, 2; -- #3056 check chunk index column name mapping CREATE TABLE i3056(c int, order_number int NOT NULL, date_created timestamptz NOT NULL); CREATE INDEX ON i3056(order_number) INCLUDE(order_number); CREATE INDEX ON i3056(date_created, (order_number % 5)) INCLUDE(order_number); SELECT table_name FROM create_hypertable('i3056', 'date_created'); ALTER TABLE i3056 DROP COLUMN c; INSERT INTO i3056(order_number,date_created) VALUES (1, '2000-01-01'); -- #5908 test CREATE INDEX ON ONLY main table CREATE TABLE test(time timestamptz, temp float); SELECT create_hypertable('test', 'time'); INSERT INTO test (time,temp) VALUES (generate_series(TIMESTAMP '2019-08-01', TIMESTAMP '2019-08-10', INTERVAL '10 minutes'), ROUND(RANDOM()*10)::float); SELECT * FROM show_chunks('test'); SELECT * FROM test.show_indexes('_timescaledb_internal._hyper_15_21_chunk'); -- create index per chunk CREATE INDEX _hyper_15_21_chunk_test_temp_idx ON _timescaledb_internal._hyper_15_21_chunk(temp); SELECT * FROM test.show_indexes('_timescaledb_internal._hyper_15_21_chunk'); SELECT * FROM test.show_indexes('test'); -- create index only on main table CREATE INDEX test_temp_idx ON ONLY test (time); SELECT * FROM test.show_indexes('test'); SELECT * FROM test.show_indexes('_timescaledb_internal._hyper_15_21_chunk'); ================================================ FILE: test/sql/information_views.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SELECT * FROM timescaledb_information.hypertables; -- create simple hypertable with 1 chunk CREATE TABLE ht1(time TIMESTAMPTZ NOT NULL); SELECT create_hypertable('ht1','time'); INSERT INTO ht1 SELECT '2000-01-01'::TIMESTAMPTZ; -- create simple hypertable with 1 chunk and toasted data CREATE TABLE ht2(time TIMESTAMPTZ NOT NULL, data TEXT); SELECT create_hypertable('ht2','time'); INSERT INTO ht2 SELECT '2000-01-01'::TIMESTAMPTZ, repeat('8k',4096); SELECT * FROM timescaledb_information.hypertables ORDER BY hypertable_schema, hypertable_name; \c :TEST_DBNAME :ROLE_SUPERUSER -- create schema open and hypertable with 3 chunks CREATE SCHEMA open; GRANT USAGE ON SCHEMA open TO :ROLE_DEFAULT_PERM_USER; CREATE TABLE open.open_ht(time TIMESTAMPTZ NOT NULL); SELECT create_hypertable('open.open_ht','time'); INSERT INTO open.open_ht SELECT '2000-01-01'::TIMESTAMPTZ; INSERT INTO open.open_ht SELECT '2001-01-01'::TIMESTAMPTZ; INSERT INTO open.open_ht SELECT '2002-01-01'::TIMESTAMPTZ; -- create schema closed and hypertable CREATE SCHEMA closed; CREATE TABLE closed.closed_ht(time TIMESTAMPTZ NOT NULL); SELECT create_hypertable('closed.closed_ht','time'); INSERT INTO closed.closed_ht SELECT '2000-01-01'::TIMESTAMPTZ; SELECT * FROM timescaledb_information.hypertables ORDER BY hypertable_schema, hypertable_name; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER \set ON_ERROR_STOP 0 \x SELECT * FROM timescaledb_information.hypertables ORDER BY hypertable_schema, hypertable_name; -- filter by schema SELECT * FROM timescaledb_information.hypertables WHERE hypertable_schema = 'closed' ORDER BY hypertable_schema, hypertable_name; -- filter by table name SELECT * FROM timescaledb_information.hypertables WHERE hypertable_name = 'ht1' ORDER BY hypertable_schema, hypertable_name; -- filter by owner SELECT * FROM timescaledb_information.hypertables WHERE owner = 'super_user' ORDER BY hypertable_schema, hypertable_name; \x ---Add integer table -- CREATE TABLE test_table_int(time bigint, junk int); SELECT create_hypertable('test_table_int', 'time', chunk_time_interval => 10); CREATE OR REPLACE function table_int_now() returns BIGINT LANGUAGE SQL IMMUTABLE as 'SELECT 1::BIGINT'; SELECT set_integer_now_func('test_table_int', 'table_int_now'); INSERT into test_table_int SELECT generate_series( 1, 20), 100; \d timescaledb_information.chunks SELECT hypertable_schema, hypertable_name, chunk_schema, chunk_name, primary_dimension, primary_dimension_type, range_start, range_end, range_start_integer, range_end_integer, is_compressed, chunk_tablespace, data_nodes FROM timescaledb_information.chunks WHERE hypertable_name = 'ht1' ORDER BY chunk_name; SELECT hypertable_schema, hypertable_name, chunk_schema, chunk_name, primary_dimension, primary_dimension_type, range_start, range_end, range_start_integer, range_end_integer, is_compressed, chunk_tablespace, data_nodes FROM timescaledb_information.chunks WHERE hypertable_name = 'test_table_int' ORDER BY chunk_name; \x SELECT * FROM timescaledb_information.dimensions ORDER BY hypertable_name, dimension_number; \x ================================================ FILE: test/sql/insert.sql.in ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SET enable_seqscan TO off; \ir include/insert_two_partitions.sql SELECT * FROM test.show_columnsp('_timescaledb_internal.%_hyper%'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%'); SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; SELECT * FROM "two_Partitions" ORDER BY "timeCustom", device_id, series_0, series_1; SELECT * FROM ONLY "two_Partitions"; CREATE TABLE error_test(time timestamp, temp float8, device text NOT NULL); SELECT create_hypertable('error_test', 'time', 'device', 2); \set QUIET off INSERT INTO error_test VALUES ('Mon Mar 20 09:18:20.1 2017', 21.3, 'dev1'); \set ON_ERROR_STOP 0 -- generate insert error INSERT INTO error_test VALUES ('Mon Mar 20 09:18:22.3 2017', 21.1, NULL); \set ON_ERROR_STOP 1 INSERT INTO error_test VALUES ('Mon Mar 20 09:18:25.7 2017', 22.4, 'dev2'); \set QUIET on SELECT * FROM error_test; --test character(9) partition keys since there were issues with padding causing partitioning errors CREATE TABLE tick_character ( symbol character(9) NOT NULL, mid REAL NOT NULL, spread REAL NOT NULL, time TIMESTAMPTZ NOT NULL ); SELECT create_hypertable ('tick_character', 'time', 'symbol', 2); INSERT INTO tick_character ( symbol, mid, spread, time ) VALUES ( 'GBPJPY', 142.639000, 5.80, 'Mon Mar 20 09:18:22.3 2017') RETURNING time, symbol, mid; SELECT * FROM tick_character; CREATE TABLE date_col_test(time date, temp float8, device text NOT NULL); SELECT create_hypertable('date_col_test', 'time', 'device', 1000, chunk_time_interval => INTERVAL '1 Day'); INSERT INTO date_col_test VALUES ('2001-02-01', 98, 'dev1'), ('2001-03-02', 98, 'dev1'); SELECT * FROM date_col_test WHERE time > '2001-01-01'; -- Out-of-order insertion regression test. -- this used to trip an assert in subspace_store.c checking that -- max_open_chunks_per_insert was obeyed set timescaledb.max_open_chunks_per_insert=1; CREATE TABLE chunk_assert_fail(i bigint, j bigint); SELECT create_hypertable('chunk_assert_fail', 'i', 'j', 1000, chunk_time_interval=>1); insert into chunk_assert_fail values (1, 1), (1, 2), (2,1); select * from chunk_assert_fail; CREATE TABLE one_space_test(time timestamp, temp float8, device text NOT NULL); SELECT create_hypertable('one_space_test', 'time', 'device', 1); INSERT INTO one_space_test VALUES ('2001-01-01 01:01:01', 1.0, 'device'), ('2002-01-01 01:02:01', 1.0, 'device'); SELECT * FROM one_space_test; --CTE & EXPLAIN ANALYZE TESTS WITH insert_cte as ( INSERT INTO one_space_test VALUES ('2001-01-01 01:02:01', 1.0, 'device') RETURNING *) SELECT * FROM insert_cte; EXPLAIN (analyze, buffers off, costs off, timing off) --can't turn summary off in 9.6 so instead grep it away at end. WITH insert_cte as ( INSERT INTO one_space_test VALUES ('2001-01-01 01:03:01', 1.0, 'device') ) SELECT 1 \g | grep -v "Planning" | grep -v "Execution" -- INSERTs can exclude chunks based on constraints EXPLAIN (buffers off, costs off) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail; EXPLAIN (buffers off, costs off) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail WHERE i < 1; EXPLAIN (buffers off, costs off) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail WHERE i = 1; EXPLAIN (buffers off, costs off) INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail WHERE i > 1; INSERT INTO chunk_assert_fail SELECT i, j FROM chunk_assert_fail WHERE i > 1; EXPLAIN (buffers off, costs off) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time < 'infinity' LIMIT 1; EXPLAIN (buffers off, costs off) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time >= 'infinity' LIMIT 1; EXPLAIN (buffers off, costs off) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time <= '-infinity' LIMIT 1; EXPLAIN (buffers off, costs off) INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time > '-infinity' LIMIT 1; INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time < 'infinity' LIMIT 1; INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time >= 'infinity' LIMIT 1; INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time <= '-infinity' LIMIT 1; INSERT INTO one_space_test SELECT * FROM one_space_test WHERE time > '-infinity' LIMIT 1; CREATE TABLE timestamp_inf(time TIMESTAMP); SELECT create_hypertable('timestamp_inf', 'time'); INSERT INTO timestamp_inf VALUES ('2018/01/02'), ('2019/01/02'); EXPLAIN (buffers off, costs off) INSERT INTO timestamp_inf SELECT * FROM timestamp_inf WHERE time < 'infinity' LIMIT 1; EXPLAIN (buffers off, costs off) INSERT INTO timestamp_inf SELECT * FROM timestamp_inf WHERE time >= 'infinity' LIMIT 1; EXPLAIN (buffers off, costs off) INSERT INTO timestamp_inf SELECT * FROM timestamp_inf WHERE time <= '-infinity' LIMIT 1; EXPLAIN (buffers off, costs off) INSERT INTO timestamp_inf SELECT * FROM timestamp_inf WHERE time > '-infinity' LIMIT 1; CREATE TABLE date_inf(time DATE); SELECT create_hypertable('date_inf', 'time'); INSERT INTO date_inf VALUES ('2018/01/02'), ('2019/01/02'); EXPLAIN (buffers off, costs off) INSERT INTO date_inf SELECT * FROM date_inf WHERE time < 'infinity' LIMIT 1; EXPLAIN (buffers off, costs off) INSERT INTO date_inf SELECT * FROM date_inf WHERE time >= 'infinity' LIMIT 1; EXPLAIN (buffers off, costs off) INSERT INTO date_inf SELECT * FROM date_inf WHERE time <= '-infinity' LIMIT 1; EXPLAIN (buffers off, costs off) INSERT INTO date_inf SELECT * FROM date_inf WHERE time > '-infinity' LIMIT 1; -- test INSERT with cached plans / plpgsql functions -- https://github.com/timescale/timescaledb/issues/1809 CREATE TABLE status_table(a int, b int, last_ts timestamptz, UNIQUE(a,b)); CREATE TABLE metrics(time timestamptz NOT NULL, value float); CREATE TABLE metrics2(time timestamptz NOT NULL, value float); SELECT (create_hypertable(t,'time')).table_name FROM (VALUES ('metrics'),('metrics2')) v(t); INSERT INTO metrics VALUES ('2000-01-01',random()), ('2000-02-01',random()), ('2000-03-01',random()); CREATE OR REPLACE FUNCTION insert_test() RETURNS VOID LANGUAGE plpgsql AS $$ DECLARE r RECORD; BEGIN FOR r IN SELECT * FROM metrics LOOP WITH foo AS ( INSERT INTO metrics2 SELECT * FROM metrics RETURNING * ) INSERT INTO status_table (a,b, last_ts) VALUES (1,1, now()) ON CONFLICT (a,b) DO UPDATE SET last_ts=(SELECT max(time) FROM metrics); END LOOP; END; $$; SELECT insert_test(), insert_test(), insert_test(); -- test Postgres crashes on INSERT ... SELECT ... WHERE NOT EXISTS with empty table -- https://github.com/timescale/timescaledb/issues/1883 CREATE TABLE readings ( toe TIMESTAMPTZ NOT NULL, sensor_id INT NOT NULL, value INT NOT NULL ); SELECT create_hypertable( 'readings', 'toe', chunk_time_interval => interval '1 day', if_not_exists => TRUE, migrate_data => TRUE ); EXPLAIN (buffers off, costs off) INSERT INTO readings SELECT '2020-05-09 10:34:35.296288+00', 1, 0 WHERE NOT EXISTS ( SELECT 1 FROM readings WHERE sensor_id = 1 AND toe = '2020-05-09 10:34:35.296288+00' ); INSERT INTO readings SELECT '2020-05-09 10:34:35.296288+00', 1, 0 WHERE NOT EXISTS ( SELECT 1 FROM readings WHERE sensor_id = 1 AND toe = '2020-05-09 10:34:35.296288+00' ); DROP TABLE readings; CREATE TABLE sample_table ( sequence INTEGER NOT NULL, time TIMESTAMP WITHOUT TIME ZONE NOT NULL, value NUMERIC NOT NULL, UNIQUE (sequence, time) ); SELECT * FROM create_hypertable('sample_table', 'time', chunk_time_interval => INTERVAL '1 day'); INSERT INTO sample_table (sequence,time,value) VALUES (7, generate_series(TIMESTAMP '2019-08-01', TIMESTAMP '2019-08-10', INTERVAL '10 minutes'), ROUND(RANDOM()*10)::int); \set ON_ERROR_STOP 0 INSERT INTO sample_table (sequence,time,value) VALUES (7, generate_series(TIMESTAMP '2019-07-21', TIMESTAMP '2019-08-01', INTERVAL '10 minutes'), ROUND(RANDOM()*10)::int); \set ON_ERROR_STOP 1 INSERT INTO sample_table (sequence,time,value) VALUES (7,generate_series(TIMESTAMP '2019-01-01', TIMESTAMP '2019-07-01', '10 minutes'), ROUND(RANDOM()*10)::int); DROP TABLE sample_table; -- test on conflict clause on columns with default value -- issue #3037 CREATE TABLE i3037(time timestamptz PRIMARY KEY); SELECT create_hypertable('i3037','time'); ALTER TABLE i3037 ADD COLUMN value float DEFAULT 0; INSERT INTO i3037 VALUES ('2000-01-01'); INSERT INTO i3037 VALUES ('2000-01-01') ON CONFLICT(time) DO UPDATE SET value = EXCLUDED.value; -- test inserting into chunks directly CREATE TABLE direct_insert(time timestamptz, meta text) WITH (tsdb.hypertable); INSERT INTO direct_insert VALUES ('2020-01-01'); SELECT show_chunks('direct_insert') AS "CHUNK" \gset --should have ModifyHyperable node EXPLAIN (costs off, timing off, summary off) INSERT INTO :CHUNK VALUES ('2020-01-01'); -- correct time range should succeed INSERT INTO :CHUNK VALUES ('2020-01-01') RETURNING *; -- incorrect time range should fail \set ON_ERROR_STOP 0 INSERT INTO :CHUNK VALUES ('2020-01-01'), ('2021-01-01') RETURNING *; INSERT INTO :CHUNK VALUES ('2025-01-01') RETURNING *; \set ON_ERROR_STOP 1 -- test that triggers on chunks work CREATE FUNCTION test_trigger() RETURNS TRIGGER AS $$ BEGIN RAISE NOTICE 'trigger called'; NEW.meta = 'triggered'; RETURN NEW; END; $$ LANGUAGE plpgsql; CREATE TRIGGER test_trigger BEFORE INSERT ON direct_insert FOR EACH ROW EXECUTE PROCEDURE test_trigger(); INSERT INTO :CHUNK VALUES ('2020-01-01') RETURNING *; -- test upsert DELETE FROM direct_insert; ALTER TABLE direct_insert ADD CONSTRAINT direct_insert_pkey PRIMARY KEY (time); INSERT INTO :CHUNK VALUES ('2020-01-01') RETURNING *; -- DO NOTHING should succeed INSERT INTO :CHUNK VALUES ('2020-01-01') ON CONFLICT DO NOTHING RETURNING *; INSERT INTO :CHUNK VALUES ('2020-01-01') ON CONFLICT (time) DO UPDATE SET meta = 'update' RETURNING *; \set ON_ERROR_STOP 0 -- conflict should fail INSERT INTO :CHUNK VALUES ('2020-01-01') RETURNING *; INSERT INTO :CHUNK VALUES ('2020-01-01') ON CONFLICT (time) DO UPDATE SET time = '2000-01-01' RETURNING *; INSERT INTO :CHUNK VALUES ('2020-01-01') ON CONFLICT (time) DO UPDATE SET time = EXCLUDED.time + '1 year' RETURNING *; \set ON_ERROR_STOP 1 ================================================ FILE: test/sql/insert_many.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE many_partitions_test(time timestamp, temp float8, device text NOT NULL); SELECT create_hypertable('many_partitions_test', 'time', 'device', 1000); --NOTE: how much slower the first two queries are -- they are creating chunks INSERT INTO many_partitions_test SELECT to_timestamp(ser), ser, ser::text FROM generate_series(1,100) ser; INSERT INTO many_partitions_test SELECT to_timestamp(ser), ser, ser::text FROM generate_series(101,200) ser; INSERT INTO many_partitions_test SELECT to_timestamp(ser), ser, (ser-201)::text FROM generate_series(201,300) ser; SELECT * FROM many_partitions_test ORDER BY time DESC LIMIT 2; SELECT count(*) FROM many_partitions_test; CREATE TABLE many_partitions_test_1m (time timestamp, temp float8, device text NOT NULL); SELECT create_hypertable('many_partitions_test_1m', 'time', 'device', 1000); EXPLAIN (verbose on, buffers off, costs off) INSERT INTO many_partitions_test_1m(time, temp, device) SELECT time_bucket('1 minute', time) AS period, avg(temp), device FROM many_partitions_test GROUP BY period, device; INSERT INTO many_partitions_test_1m(time, temp, device) SELECT time_bucket('1 minute', time) AS period, avg(temp), device FROM many_partitions_test GROUP BY period, device; SELECT * FROM many_partitions_test_1m ORDER BY time, device LIMIT 10; ================================================ FILE: test/sql/insert_returning.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE standard_table ( standard_id integer PRIMARY KEY, name text not null ); CREATE TABLE hypertable ( time timestamptz not null, name text not null, standard_id integer not null ); select * from create_hypertable('hypertable', 'time'); INSERT INTO standard_table (standard_id, name) VALUES (1, 'standard_1'); INSERT INTO hypertable (time, name, standard_id) VALUES ('2021-01-01 01:01:01+00', 'hypertable_1', 1); INSERT INTO hypertable (time, name, standard_id) VALUES ('2022-02-02 02:02:02+00', 'hypertable_2', 1) RETURNING *, EXISTS ( SELECT * FROM standard_table WHERE standard_table.standard_id = hypertable.standard_id ); INSERT INTO hypertable (time, name, standard_id) VALUES ('2023-03-03 03:03:03+00', 'hypertable_3', 1) RETURNING *, EXISTS ( SELECT * FROM standard_table WHERE standard_table.standard_id = hypertable.standard_id ); INSERT INTO hypertable (time, name, standard_id) VALUES ('2024-04-04 04:04:04+00', 'hypertable_4', 2) RETURNING *, EXISTS ( SELECT * FROM standard_table WHERE standard_table.standard_id = hypertable.standard_id ); ================================================ FILE: test/sql/insert_returning_old_new.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test OLD/NEW references in RETURNING clause (PG18+ feature) CREATE TABLE ht_returning( time timestamptz NOT NULL, value int NOT NULL, UNIQUE(time) ) WITH (tsdb.hypertable); -- Insert initial rows INSERT INTO ht_returning(time, value) VALUES ('2024-01-01', 10); INSERT INTO ht_returning(time, value) VALUES ('2024-01-02', 20); -- Test 1: INSERT ON CONFLICT DO UPDATE with RETURNING OLD.col, NEW.col -- This should show the old value (10) and the new value (100) INSERT INTO ht_returning(time, value) VALUES ('2024-01-01', 100) ON CONFLICT (time) DO UPDATE SET value = EXCLUDED.value RETURNING OLD.value AS old_val, NEW.value AS new_val; -- Test 2: INSERT ON CONFLICT DO UPDATE with RETURNING arithmetic on OLD/NEW INSERT INTO ht_returning(time, value) VALUES ('2024-01-02', 50) ON CONFLICT (time) DO UPDATE SET value = EXCLUDED.value RETURNING OLD.value AS old_val, NEW.value AS new_val, NEW.value - OLD.value AS diff; -- Test 3: Plain INSERT with RETURNING NEW.col (OLD should be NULL for fresh inserts) INSERT INTO ht_returning(time, value) VALUES ('2024-01-03', 30) RETURNING OLD.value AS old_val, NEW.value AS new_val; -- Test 4: MERGE with both UPDATE and INSERT paths, returning OLD/NEW values and merge_action() CREATE TABLE t1(time timestamptz NOT NULL, value int NOT NULL); INSERT INTO t1(time, value) VALUES ('2024-01-01', 5); -- Will trigger UPDATE (existing row) INSERT INTO t1(time, value) VALUES ('2024-01-05', 10); -- Will trigger INSERT (new row) MERGE INTO ht_returning AS t USING t1 AS s ON t.time = s.time WHEN MATCHED THEN UPDATE SET value = t.value + s.value WHEN NOT MATCHED THEN INSERT (time, value) VALUES (s.time, s.value) RETURNING NEW.time, NEW.value, t.value, OLD.value, s.value, merge_action(); -- Verify final state SELECT * FROM ht_returning ORDER BY time; -- Cleanup DROP TABLE ht_returning; ================================================ FILE: test/sql/insert_single.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \ir include/insert_single.sql SELECT * FROM test.show_columnsp('"one_Partition".%'); SELECT * FROM "one_Partition" ORDER BY "timeCustom", device_id, series_0, series_1, series_2; --test that we can insert data into a 1-dimensional table (only time partitioning) CREATE TABLE "1dim"(time timestamp PRIMARY KEY, temp float); SELECT create_hypertable('"1dim"', 'time'); INSERT INTO "1dim" VALUES('2017-01-20T09:00:01', 22.5) RETURNING *; INSERT INTO "1dim" VALUES('2017-01-20T09:00:21', 21.2); INSERT INTO "1dim" VALUES('2017-01-20T09:00:47', 25.1); SELECT * FROM "1dim"; CREATE TABLE regular_table (time timestamp, temp float); INSERT INTO regular_table SELECT * FROM "1dim"; SELECT * FROM regular_table; TRUNCATE TABLE regular_table; INSERT INTO regular_table VALUES('2017-01-20T09:00:59', 29.2); INSERT INTO "1dim" SELECT * FROM regular_table; SELECT * FROM "1dim"; SELECT "1dim" FROM "1dim"; --test that we can insert pre-1970 dates CREATE TABLE "1dim_pre1970"(time timestamp PRIMARY KEY, temp float); SELECT create_hypertable('"1dim_pre1970"', 'time', chunk_time_interval=> INTERVAL '1 Month'); INSERT INTO "1dim_pre1970" VALUES('1969-12-01T19:00:00', 21.2); INSERT INTO "1dim_pre1970" VALUES('1969-12-20T09:00:00', 25.1); INSERT INTO "1dim_pre1970" VALUES('1970-01-20T09:00:00', 26.6); INSERT INTO "1dim_pre1970" VALUES('1969-02-20T09:00:00', 29.9); --should show warning BEGIN; CREATE TABLE "1dim_usec_interval"(time timestamp PRIMARY KEY, temp float); SELECT create_hypertable('"1dim_usec_interval"', 'time', chunk_time_interval=> 10); INSERT INTO "1dim_usec_interval" VALUES('1969-12-01T19:00:00', 21.2); ROLLBACK; CREATE TABLE "1dim_usec_interval"(time timestamp PRIMARY KEY, temp float); SELECT create_hypertable('"1dim_usec_interval"', 'time', chunk_time_interval=> 1000000); INSERT INTO "1dim_usec_interval" VALUES('1969-12-01T19:00:00', 21.2); CREATE TABLE "1dim_neg"(time INTEGER, temp float); SELECT create_hypertable('"1dim_neg"', 'time', chunk_time_interval=>10); INSERT INTO "1dim_neg" VALUES (-20, 21.2); INSERT INTO "1dim_neg" VALUES (-19, 21.2); INSERT INTO "1dim_neg" VALUES (-1, 21.2); INSERT INTO "1dim_neg" VALUES (0, 21.2); INSERT INTO "1dim_neg" VALUES (1, 21.2); INSERT INTO "1dim_neg" VALUES (19, 21.2); INSERT INTO "1dim_neg" VALUES (20, 21.2); SELECT * FROM "1dim_pre1970"; SELECT * FROM "1dim_neg"; SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; SELECT * FROM _timescaledb_catalog.dimension_slice; -- Create a three-dimensional table CREATE TABLE "3dim" (time timestamp, temp float, device text, location text); SELECT create_hypertable('"3dim"', 'time', 'device', 2); SELECT add_dimension('"3dim"', 'location', 2); INSERT INTO "3dim" VALUES('2017-01-20T09:00:01', 22.5, 'blue', 'nyc'); INSERT INTO "3dim" VALUES('2017-01-20T09:00:21', 21.2, 'brown', 'sthlm'); INSERT INTO "3dim" VALUES('2017-01-20T09:00:47', 25.1, 'yellow', 'la'); --show the constraints on the three-dimensional chunk SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_7_16_chunk'); --queries should work in three dimensions SELECT * FROM "3dim"; -- test that explain works EXPLAIN (BUFFERS FALSE, COSTS FALSE) INSERT INTO "3dim" VALUES('2017-01-21T09:00:01', 32.9, 'green', 'nyc'), ('2017-01-21T09:00:47', 27.3, 'purple', 'la') RETURNING *; EXPLAIN (BUFFERS FALSE, COSTS FALSE) WITH "3dim_insert" AS ( INSERT INTO "3dim" VALUES('2017-01-21T09:01:44', 19.3, 'black', 'la') RETURNING time, temp ), regular_insert AS ( INSERT INTO regular_table VALUES('2017-01-21T10:00:51', 14.3) RETURNING time, temp ) INSERT INTO "1dim" (SELECT time, temp FROM "3dim_insert" UNION SELECT time, temp FROM regular_insert); -- test prepared statement INSERT PREPARE "1dim_plan" (timestamp, float) AS INSERT INTO "1dim" VALUES($1, $2) ON CONFLICT (time) DO NOTHING; EXECUTE "1dim_plan" ('2017-04-17 23:35', 31.4); EXECUTE "1dim_plan" ('2017-04-17 23:35', 32.6); -- test prepared statement with generic plan (forced when no parameters) PREPARE "1dim_plan_generic" AS INSERT INTO "1dim" VALUES('2017-05-18 17:24', 18.3); EXECUTE "1dim_plan_generic"; SELECT * FROM "1dim" ORDER BY time; SELECT * FROM "3dim" ORDER BY (time, device); -- Test large intervals as default interval for integer is -- supported as part of hypertable generalization \set ON_ERROR_STOP 0 CREATE TABLE "inttime_err"(time INTEGER PRIMARY KEY, temp float); SELECT create_hypertable('"inttime_err"', 'time', chunk_time_interval=>2147483648); \set ON_ERROR_STOP 1 SELECT create_hypertable('"inttime_err"', 'time', chunk_time_interval=>2147483647); -- Test large intervals as default interval is supported -- for integer types as part of hypertable generalization. \set ON_ERROR_STOP 0 CREATE TABLE "smallinttime_err"(time SMALLINT PRIMARY KEY, temp float); SELECT create_hypertable('"smallinttime_err"', 'time', chunk_time_interval=>32768); \set ON_ERROR_STOP 1 SELECT create_hypertable('"smallinttime_err"', 'time', chunk_time_interval=>32767); --make sure date inserts work even when the timezone changes the CREATE TABLE hyper_date(time date, temp float); SELECT create_hypertable('"hyper_date"', 'time'); SET timezone=+1; INSERT INTO "hyper_date" VALUES('2011-01-26', 22.5); RESET timezone; --make sure timestamp inserts work even when the timezone changes the SET timezone = 'UTC'; CREATE TABLE "test_tz"(time timestamp PRIMARY KEY, temp float); SELECT create_hypertable('"test_tz"', 'time', chunk_time_interval=> INTERVAL '1 day'); INSERT INTO "test_tz" VALUES('2017-09-22 10:00:00', 21.2); INSERT INTO "test_tz" VALUES('2017-09-21 19:00:00', 21.2); SET timezone = 'US/central'; INSERT INTO "test_tz" VALUES('2017-09-21 19:01:00', 21.2); SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_10_20_chunk'); SELECT * FROM test_tz; -- test various memory settings -- SET timescaledb.max_open_chunks_per_insert = 10; SET timescaledb.max_cached_chunks_per_hypertable = 10; CREATE TABLE "nondefault_mem_settings"(time timestamp PRIMARY KEY, temp float); SELECT create_hypertable('"nondefault_mem_settings"', 'time', chunk_time_interval=> INTERVAL '1 Month'); INSERT INTO "nondefault_mem_settings" VALUES('2000-12-01T19:00:00', 21.2); INSERT INTO "nondefault_mem_settings" VALUES('2001-12-20T09:00:00', 25.1); --lowest possible SET timescaledb.max_open_chunks_per_insert = 1; SET timescaledb.max_cached_chunks_per_hypertable = 1; INSERT INTO "nondefault_mem_settings" VALUES ('2001-01-20T09:00:00', 26.6), ('2002-02-20T09:00:00', 27.9), ('2003-02-20T09:00:00', 28.9); INSERT INTO "nondefault_mem_settings" VALUES ('2001-03-20T09:00:00', 30.6), ('2002-03-20T09:00:00', 31.9), ('2003-03-20T09:00:00', 32.9); --warning about mismatched cache sizes SET timescaledb.max_open_chunks_per_insert = 100; SET timescaledb.max_cached_chunks_per_hypertable = 10; INSERT INTO "nondefault_mem_settings" VALUES ('2001-05-20T09:00:00', 36.6), ('2002-05-20T09:00:00', 37.9), ('2003-05-20T09:00:00', 38.9); --unlimited SET timescaledb.max_open_chunks_per_insert = 0; SET timescaledb.max_cached_chunks_per_hypertable = 0; INSERT INTO "nondefault_mem_settings" VALUES ('2001-04-20T09:00:00', 33.6), ('2002-04-20T09:00:00', 34.9), ('2003-04-20T09:00:00', 35.9); SELECT * FROM "nondefault_mem_settings"; --test rollback BEGIN; \set QUIET off CREATE TABLE "data_records" ("time" bigint NOT NULL, "value" integer CHECK (VALUE >= 0)); SELECT create_hypertable('data_records', 'time', chunk_time_interval => 2592000000); INSERT INTO "data_records" ("time", "value") VALUES (0, 1); SAVEPOINT savepoint_1; INSERT INTO "data_records" ("time", "value") VALUES (1, 0); ROLLBACK TO SAVEPOINT savepoint_1; INSERT INTO "data_records" ("time", "value") VALUES (2, 1); SAVEPOINT savepoint_2; \set ON_ERROR_STOP 0 INSERT INTO "data_records" ("time", "value") VALUES (3, -1); \set ON_ERROR_STOP 1 ROLLBACK TO SAVEPOINT savepoint_2; INSERT INTO "data_records" ("time", "value") VALUES (4, 1); SAVEPOINT savepoint_3; INSERT INTO "data_records" ("time", "value") VALUES (5, 0); ROLLBACK TO SAVEPOINT savepoint_3; SELECT * FROM data_records; \set QUIET on ROLLBACK; -- Test INSERT into hypertable with a generated column whose type is a -- domain with a NOT NULL constraint CREATE DOMAIN nn_int AS int CHECK (VALUE IS NOT NULL); CREATE TABLE generated_col_ht( time timestamptz NOT NULL, val int NOT NULL, doubled nn_int GENERATED ALWAYS AS (val * 2) STORED ); SELECT create_hypertable('generated_col_ht', 'time'); INSERT INTO generated_col_ht(time, val) VALUES ('2024-01-01', 5); INSERT INTO generated_col_ht(time, val) VALUES ('2024-01-02', 10) RETURNING *; SELECT * FROM generated_col_ht ORDER BY time; DROP TABLE generated_col_ht; DROP DOMAIN nn_int; ================================================ FILE: test/sql/lateral.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE regular_table(name text, junk text); CREATE TABLE ht(time timestamptz NOT NULL, location text); SELECT create_hypertable('ht', 'time'); INSERT INTO ht(time) select timestamp 'epoch' + (i * interval '1 second') from generate_series(1, 100) as T(i); INSERT INTO regular_table values('name', 'junk'); SELECT * FROM regular_table ik LEFT JOIN LATERAL (select max(time::timestamptz) from ht s where ik.name='name' and s.time < now()) s on true; select * from regular_table ik LEFT JOIN LATERAL (select max(time::timestamptz) from ht s where ik.name='name' and s.time > now()) s on true; DROP TABLE regular_table; DROP TABLE ht; CREATE TABLE orders(id int, user_id int, time TIMESTAMPTZ NOT NULL); SELECT create_hypertable('orders', 'time'); INSERT INTO orders values(1,1,timestamp 'epoch' + '1 second'); INSERT INTO orders values(2,1,timestamp 'epoch' + '2 second'); INSERT INTO orders values(3,1,timestamp 'epoch' + '3 second'); INSERT INTO orders values(4,2,timestamp 'epoch' + '4 second'); INSERT INTO orders values(5,1,timestamp 'epoch' + '5 second'); INSERT INTO orders values(6,3,timestamp 'epoch' + '6 second'); INSERT INTO orders values(7,1,timestamp 'epoch' + '7 second'); INSERT INTO orders values(8,4,timestamp 'epoch' + '8 second'); INSERT INTO orders values(9,2,timestamp 'epoch' + '9 second'); -- Need a LATERAL query with a reference to the upper-level table and -- with a restriction on time -- Upper-level table constraint should be a constant in order to trigger -- creation of a one-time filter in the planner SELECT user_id, first_order_time, max_time FROM (SELECT user_id, min(time) AS first_order_time FROM orders GROUP BY user_id) o1 LEFT JOIN LATERAL (SELECT max(time) AS max_time FROM orders WHERE o1.user_id = '2' AND time > now()) o2 ON true ORDER BY user_id, first_order_time, max_time; SELECT user_id, first_order_time, max_time FROM (SELECT user_id, min(time) AS first_order_time FROM orders GROUP BY user_id) o1 LEFT JOIN LATERAL (SELECT max(time) AS max_time FROM orders WHERE o1.user_id = '2' AND time < now()) o2 ON true ORDER BY user_id, first_order_time, max_time; -- Nested LATERALs SELECT user_id, first_order_time, time1, min_time FROM (SELECT user_id, min(time) AS first_order_time FROM orders GROUP BY user_id) o1 LEFT JOIN LATERAL (SELECT user_id as o2user_id, time AS time1 FROM orders WHERE o1.user_id = '2' AND time < now()) o2 ON true LEFT JOIN LATERAL (SELECT min(time) as min_time FROM orders WHERE o2.o2user_id = '1' AND time < now()) o3 ON true ORDER BY user_id, first_order_time, time1, min_time; -- Cleanup DROP TABLE orders; ---- OUTER JOIN tests --- --github issue 2500 CREATE TABLE t1_timescale (a int, b int); CREATE TABLE t2 (a int, b int); SELECT create_hypertable('t1_timescale', 'a', chunk_time_interval=>1000); INSERT into t2 values (3, 3), (15 , 15); INSERT into t1_timescale select generate_series(5, 25, 1), 77; UPDATE t1_timescale SET b = 15 WHERE a = 15; SELECT * FROM t1_timescale FULL OUTER JOIN t2 on t1_timescale.b=t2.b and t2.b between 10 and 20 ORDER BY 1, 2, 3, 4; SELECT * FROM t1_timescale LEFT OUTER JOIN t2 on t1_timescale.b=t2.b and t2.b between 10 and 20 WHERE t1_timescale.a=5 ORDER BY 1, 2, 3, 4; SELECT * FROM t1_timescale RIGHT JOIN t2 on t1_timescale.b=t2.b and t2.b between 10 and 20 ORDER BY 1, 2, 3, 4; SELECT * FROM t1_timescale RIGHT JOIN t2 on t1_timescale.b=t2.b and t2.b between 10 and 20 WHERE t1_timescale.a=5 ORDER BY 1, 2, 3, 4; SELECT * FROM t1_timescale LEFT OUTER JOIN t2 on t1_timescale.a=t2.a and t2.b between 10 and 20 WHERE t1_timescale.a IN ( 10, 15, 20, 25) ORDER BY 1, 2, 3, 4; SELECT * FROM t1_timescale RIGHT OUTER JOIN t2 on t1_timescale.a=t2.a and t2.b between 10 and 20 ORDER BY 1, 2, 3, 4; ================================================ FILE: test/sql/license.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER \set ECHO queries \set VERBOSITY default SHOW timescaledb.license; SELECT _timescaledb_functions.tsl_loaded(); -- User shouldn't be able to change the license in the session \set ON_ERROR_STOP 0 SET timescaledb.license='apache'; SET timescaledb.license='timescale'; SET timescaledb.license='something_else'; \set ON_ERROR_STOP 1 -- make sure apache license blocks tsl features \set ON_ERROR_STOP 0 SELECT locf(1); SELECT interpolate(1); SELECT time_bucket_gapfill(1,1,1,1); CREATE OR REPLACE FUNCTION custom_func(jobid int, args jsonb) RETURNS VOID AS $$ DECLARE BEGIN END; $$ LANGUAGE plpgsql; SELECT add_job('custom_func','1h', config:='{"type":"function"}'::jsonb); DROP FUNCTION custom_func; CREATE TABLE metrics(time timestamptz NOT NULL, value float); SELECT create_hypertable('metrics', 'time'); ALTER TABLE metrics SET (timescaledb.compress); INSERT INTO metrics VALUES ('2022-01-01 00:00:00', 1), ('2022-01-01 01:00:00', 2), ('2022-01-01 02:00:00', 3); CREATE MATERIALIZED VIEW metrics_hourly WITH (timescaledb.continuous) AS SELECT time_bucket(INTERVAL '1 hour', time) AS bucket, AVG(value), MAX(value), MIN(value) FROM metrics GROUP BY bucket WITH NO DATA; CREATE MATERIALIZED VIEW metrics_hourly AS SELECT time_bucket(INTERVAL '1 hour', time) AS bucket, AVG(value), MAX(value), MIN(value) FROM metrics GROUP BY bucket; CALL refresh_continuous_aggregate('metrics_hourly', NULL, NULL); DROP MATERIALIZED VIEW metrics_hourly; DROP TABLE metrics; \set ON_ERROR_STOP 1 ================================================ FILE: test/sql/loader/CMakeLists.txt ================================================ if(CMAKE_BUILD_TYPE MATCHES Debug) install( FILES timescaledb--mock-1.sql timescaledb--mock-2.sql timescaledb--mock-3.sql timescaledb--mock-4.sql timescaledb--mock-5.sql timescaledb--mock-6.sql timescaledb--mock-broken.sql timescaledb--mock-1--mock-2.sql timescaledb--mock-2--mock-3.sql timescaledb--mock-3--mock-4.sql timescaledb--mock-5--mock-6.sql timescaledb--mock-broken--mock-5.sql timescaledb--0.0.0.sql timescaledb_osm.control timescaledb_osm--mock-1.sql DESTINATION "${PG_SHAREDIR}/extension") endif(CMAKE_BUILD_TYPE MATCHES Debug) ================================================ FILE: test/sql/loader/timescaledb--0.0.0.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. ================================================ FILE: test/sql/loader/timescaledb--mock-1--mock-2.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --test that the extension is deactivated during upgrade SELECT 1; SELECT 1; SELECT 1; SELECT 1; CREATE OR REPLACE FUNCTION mock_function() RETURNS VOID AS '$libdir/timescaledb-mock-2', 'ts_mock_function' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; ================================================ FILE: test/sql/loader/timescaledb--mock-1.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE SCHEMA IF NOT EXISTS _timescaledb_cache; CREATE TABLE IF NOT EXISTS _timescaledb_cache.cache_inval_extension(); CREATE OR REPLACE FUNCTION mock_function() RETURNS VOID AS '$libdir/timescaledb-mock-1', 'ts_mock_function' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; ================================================ FILE: test/sql/loader/timescaledb--mock-2--mock-3.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. broken sql; CREATE OR REPLACE FUNCTION mock_function() RETURNS VOID AS '$libdir/timescaledb-mock-3', 'ts_mock_function' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; ================================================ FILE: test/sql/loader/timescaledb--mock-2.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --create function before proxy table CREATE OR REPLACE FUNCTION mock_function() RETURNS VOID AS '$libdir/timescaledb-mock-2', 'ts_mock_function' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE SCHEMA IF NOT EXISTS _timescaledb_cache; CREATE TABLE IF NOT EXISTS _timescaledb_cache.cache_inval_extension(); ================================================ FILE: test/sql/loader/timescaledb--mock-3--mock-4.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --test that the extension is deactivated during upgrade SELECT 1; SELECT 1; SELECT 1; SELECT 1; CREATE OR REPLACE FUNCTION mock_function() RETURNS VOID AS '$libdir/timescaledb-mock-4', 'ts_mock_function' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; ================================================ FILE: test/sql/loader/timescaledb--mock-3.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE SCHEMA IF NOT EXISTS _timescaledb_cache; CREATE TABLE IF NOT EXISTS _timescaledb_cache.cache_inval_extension(); CREATE OR REPLACE FUNCTION mock_function() RETURNS VOID AS '$libdir/timescaledb-mock-3', 'ts_mock_function' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; ================================================ FILE: test/sql/loader/timescaledb--mock-4.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE SCHEMA IF NOT EXISTS _timescaledb_cache; CREATE TABLE IF NOT EXISTS _timescaledb_cache.cache_inval_extension(); CREATE OR REPLACE FUNCTION mock_function() RETURNS VOID AS '$libdir/timescaledb-mock-4', 'ts_mock_function' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; ================================================ FILE: test/sql/loader/timescaledb--mock-5--mock-6.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --test that the extension is deactivated during upgrade SELECT 1; SELECT 1; SELECT 1; SELECT 1; --intentionally forget to updat func --CREATE OR REPLACE FUNCTION mock_function() RETURNS VOID -- AS '$libdir/timescaledb-mock-6', 'ts_mock_function' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; ================================================ FILE: test/sql/loader/timescaledb--mock-5.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE SCHEMA IF NOT EXISTS _timescaledb_cache; CREATE TABLE IF NOT EXISTS _timescaledb_cache.cache_inval_extension(); CREATE OR REPLACE FUNCTION mock_function() RETURNS VOID AS '$libdir/timescaledb-mock-5', 'ts_mock_function' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; ================================================ FILE: test/sql/loader/timescaledb--mock-6.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE SCHEMA IF NOT EXISTS _timescaledb_cache; CREATE TABLE IF NOT EXISTS _timescaledb_cache.cache_inval_extension(); CREATE OR REPLACE FUNCTION mock_function() RETURNS VOID AS '$libdir/timescaledb-mock-6', 'ts_mock_function' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; ================================================ FILE: test/sql/loader/timescaledb--mock-broken--mock-5.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --test that the extension is deactivated during upgrade SELECT 1; SELECT 1; SELECT 1; SELECT 1; CREATE OR REPLACE FUNCTION mock_function() RETURNS VOID AS '$libdir/timescaledb-mock-5', 'ts_mock_function' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; ================================================ FILE: test/sql/loader/timescaledb--mock-broken.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE SCHEMA IF NOT EXISTS _timescaledb_cache; CREATE TABLE IF NOT EXISTS _timescaledb_cache.cache_inval_extension(); CREATE OR REPLACE FUNCTION mock_function() RETURNS VOID AS '$libdir/timescaledb-mock-broken', 'ts_mock_function' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; ================================================ FILE: test/sql/loader/timescaledb_osm--mock-1.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE SCHEMA _osm_catalog; CREATE TABLE _osm_catalog.metadata(); CREATE OR REPLACE FUNCTION mock_osm() RETURNS VOID AS '$libdir/timescaledb_osm-mock-1', 'ts_mock_osm' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; ================================================ FILE: test/sql/loader/timescaledb_osm.control ================================================ comment = 'Manages object storage on S3' default_version = '0' module_pathname = '$libdir/timescaledb_osm-0' relocatable = false superuser = false requires = timescaledb ================================================ FILE: test/sql/loader.sql.in ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set TEST_DBNAME_2 :TEST_DBNAME _2 \c :TEST_DBNAME :ROLE_SUPERUSER CREATE DATABASE :"TEST_DBNAME_2"; DROP EXTENSION timescaledb; --no extension SELECT * FROM test.extension; SELECT 1; \c :TEST_DBNAME :ROLE_SUPERUSER CREATE EXTENSION timescaledb VERSION 'mock-1'; SELECT 1; SELECT * FROM test.extension; CREATE EXTENSION IF NOT EXISTS timescaledb VERSION 'mock-1'; CREATE EXTENSION IF NOT EXISTS timescaledb VERSION 'mock-2'; DROP EXTENSION timescaledb; \set ON_ERROR_STOP 0 --test that we cannot accidentally load another library version CREATE EXTENSION IF NOT EXISTS timescaledb VERSION 'mock-2'; \set ON_ERROR_STOP 1 \c :TEST_DBNAME :ROLE_SUPERUSER --no extension SELECT * FROM test.extension; SELECT 1; CREATE EXTENSION timescaledb VERSION 'mock-1'; --same backend as create extension; SELECT 1; SELECT * FROM test.extension; --start new backend; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT 1; SELECT 1; --test fn call after load SELECT mock_function(); SELECT * FROM test.extension; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --test fn call as first command SELECT mock_function(); --use guc to prevent loading \c :TEST_DBNAME :ROLE_SUPERUSER SET timescaledb.disable_load = 'on'; SELECT 1; SELECT 1; SET timescaledb.disable_load = 'off'; SELECT 1; \set ON_ERROR_STOP 0 SET timescaledb.disable_load = 'not bool'; \set ON_ERROR_STOP 1 \c :TEST_DBNAME :ROLE_SUPERUSER RESET ALL; SELECT 1; \c :TEST_DBNAME :ROLE_SUPERUSER SET timescaledb.disable_load TO DEFAULT; SELECT 1; \c :TEST_DBNAME :ROLE_SUPERUSER RESET timescaledb.disable_load; SELECT 1; \c :TEST_DBNAME :ROLE_SUPERUSER SET timescaledb.other = 'on'; SELECT 1; \set ON_ERROR_STOP 0 --cannot update extension after .so of previous version already loaded ALTER EXTENSION timescaledb UPDATE TO 'mock-2'; \set ON_ERROR_STOP 1 \c :TEST_DBNAME_2 :ROLE_SUPERUSER SELECT * FROM test.extension; CREATE EXTENSION timescaledb VERSION 'mock-1'; SELECT * FROM test.extension; --start a new backend to update \c :TEST_DBNAME_2 :ROLE_SUPERUSER ALTER EXTENSION timescaledb UPDATE TO 'mock-2'; SELECT 1; SELECT * FROM test.extension; --drop extension DROP EXTENSION timescaledb; SELECT 1; SELECT * FROM test.extension; \c :TEST_DBNAME_2 :ROLE_SUPERUSER CREATE EXTENSION timescaledb VERSION 'mock-2'; SELECT 1; SELECT * FROM test.extension; -- test db 1 still has old version \c :TEST_DBNAME :ROLE_SUPERUSER SELECT 1; SELECT * FROM test.extension; --try a broken upgrade \c :TEST_DBNAME_2 :ROLE_SUPERUSER SELECT * FROM test.extension; \set ON_ERROR_STOP 0 ALTER EXTENSION timescaledb UPDATE TO 'mock-3'; \set ON_ERROR_STOP 1 --should still be on mock-2 SELECT 1; SELECT * FROM test.extension; --drop extension DROP EXTENSION timescaledb; SELECT 1; SELECT * FROM test.extension; --create extension anew, only upgrade was broken \c :TEST_DBNAME_2 :ROLE_SUPERUSER CREATE EXTENSION timescaledb VERSION 'mock-3'; SELECT 1; SELECT * FROM test.extension; DROP EXTENSION timescaledb; SELECT 1; --mismatched version errors \c :TEST_DBNAME_2 :ROLE_SUPERUSER --mock-4 has mismatched versions, so the .so load should be fatal SELECT format($$\! utils/test_fatal_command.sh %1$s "CREATE EXTENSION timescaledb VERSION 'mock-4'"$$, :'TEST_DBNAME_2') as command_to_run \gset :command_to_run --mock-4 not installed. SELECT * FROM test.extension; \c :TEST_DBNAME_2 :ROLE_SUPERUSER --broken version and drop CREATE EXTENSION timescaledb VERSION 'mock-broken'; \set ON_ERROR_STOP 0 --intentional broken version SELECT * FROM test.extension; SELECT 1; SELECT 1; --cannot drop extension; already loaded broken version DROP EXTENSION timescaledb; \set ON_ERROR_STOP 1 \c :TEST_DBNAME_2 :ROLE_SUPERUSER --can drop extension now. Since drop first command. DROP EXTENSION timescaledb; SELECT * FROM test.extension; --broken version and update to fixed \c :TEST_DBNAME_2 :ROLE_SUPERUSER CREATE EXTENSION timescaledb VERSION 'mock-broken'; \set ON_ERROR_STOP 0 --intentional broken version SELECT 1; --cannot update extension; already loaded bad version ALTER EXTENSION timescaledb UPDATE TO 'mock-5'; \set ON_ERROR_STOP 1 \c :TEST_DBNAME_2 :ROLE_SUPERUSER --can update extension now. ALTER EXTENSION timescaledb UPDATE TO 'mock-5'; SELECT 1; SELECT mock_function(); \c :TEST_DBNAME_2 :ROLE_SUPERUSER ALTER EXTENSION timescaledb UPDATE TO 'mock-6'; --The mock-5->mock_6 upgrade is intentionally broken. --The mock_function was never changed to point to mock-6 in the update script. --Thus mock_function is defined incorrectly to point to the mock-5.so --This will now be a FATAL error. SELECT format($$\! utils/test_fatal_command.sh %1$s "SELECT mock_function()"$$, :'TEST_DBNAME_2') as command_to_run \gset :command_to_run SELECT * FROM test.extension; --TEST: create extension when old .so already loaded \c :TEST_DBNAME :ROLE_SUPERUSER SELECT * FROM test.extension; DROP EXTENSION timescaledb; SELECT * FROM test.extension; \set ON_ERROR_STOP 0 CREATE EXTENSION timescaledb VERSION 'mock-2'; \set ON_ERROR_STOP 1 SELECT * FROM test.extension; --can create in a new session. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE EXTENSION timescaledb VERSION 'mock-2'; SELECT * FROM test.extension; --make sure parallel workers started after a 'DISCARD ALL' work CREATE TABLE test (i int, j double precision); INSERT INTO test SELECT x, x+0.1 FROM generate_series(1,100) AS x; DISCARD ALL; SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false); SET max_parallel_workers_per_gather = 1; SELECT count(*) FROM test; CREATE EXTENSION timescaledb_osm VERSION 'mock-1'; -- Test that OSM process utility hook works: it should see this DROP TABLE. DROP TABLE test; -- clean up additional database \c :TEST_DBNAME :ROLE_SUPERUSER DROP DATABASE :"TEST_DBNAME_2" WITH (FORCE); ================================================ FILE: test/sql/merge.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER -- Create target table with location and temperature CREATE TABLE target ( time TIMESTAMPTZ NOT NULL, location SMALLINT NOT NULL, temperature DOUBLE PRECISION NULL, to_be_dropped text ); SELECT create_hypertable( 'target', 'time', chunk_time_interval => INTERVAL '5 seconds'); INSERT INTO target SELECT time, location, 14 as temperature FROM generate_series( '2021-01-01 00:00:00', '2021-01-01 00:00:09', INTERVAL '5 seconds' ) as time, generate_series(1,4) as location; -- This makes sure we have one column with attisdropped and one column -- with atthasmissing set to true. These two cases can cause problems -- with chunk dispatch execution when merging using a when-clause with -- inserts. Unfortunately they are hard to trigger, so this is not a -- definitive test. ALTER TABLE target DROP COLUMN to_be_dropped; ALTER TABLE target ADD COLUMN val text default 'string -'; -- Create source table with location and temperature CREATE TABLE source ( time TIMESTAMPTZ NOT NULL, location SMALLINT NOT NULL, temperature DOUBLE PRECISION NULL ); SELECT create_hypertable( 'source', 'time', chunk_time_interval => INTERVAL '5 seconds'); -- Generate data that overlaps with target table INSERT INTO source SELECT time, location, 80 as temperature FROM generate_series( '2021-01-01 00:00:05', '2021-01-01 00:00:14', INTERVAL '5 seconds' ) as time, generate_series(1,4) as location; -- Print table/rows/num of chunks select * from target order by time, location asc; select * from source order by time, location asc; -- CREATE normal PostgreSQL tables CREATE TABLE target_pg AS SELECT * FROM target; CREATE TABLE source_pg AS SELECT * FROM source; -- Merge UPDATE matched rows for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN UPDATE SET temperature = (t.temperature + s.temperature)/2, val = val || ' UPDATED BY MERGE'; -- Merge UPDATE matched rows for hypertables MERGE INTO target t USING source s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN UPDATE SET temperature = (t.temperature + s.temperature)/2, val = val || ' UPDATED BY MERGE'; -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; -- Merge DELETE matched rows for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN DELETE; -- Merge DELETE matched rows for hypertables MERGE INTO target t USING source s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN DELETE; -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; -- clean up tables DELETE FROM target_pg; DELETE FROM target; DELETE FROM source_pg; DELETE FROM source; INSERT INTO target SELECT time, location, 14 as temperature FROM generate_series( '2021-01-01 00:00:00', '2021-01-01 00:00:09', INTERVAL '5 seconds' ) as time, generate_series(1,4) as location; INSERT INTO source SELECT time, location, 80 as temperature FROM generate_series( '2021-01-01 00:00:05', '2021-01-01 00:00:14', INTERVAL '5 seconds' ) as time, generate_series(1,4) as location; INSERT INTO target_pg SELECT * FROM target; INSERT INTO source_pg SELECT * FROM source; -- Merge UPDATE matched rows and INSERT new row for unmatched rows for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN UPDATE SET temperature = (t.temperature + s.temperature)/2, val = val || ' UPDATED BY MERGE' WHEN NOT MATCHED THEN INSERT (time, location, temperature, val) VALUES (s.time, s.location, s.temperature, 'string - INSERTED BY MERGE'); -- Merge UPDATE matched rows and INSERT new row for unmatched rows for hypertables MERGE INTO target t USING source s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN UPDATE SET temperature = (t.temperature + s.temperature)/2, val = val || ' UPDATED BY MERGE' WHEN NOT MATCHED THEN INSERT (time, location, temperature, val) VALUES (s.time, s.location, s.temperature, 'string - INSERTED BY MERGE'); -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; -- Merge INSERT with constant literals for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.location = 1234 WHEN NOT MATCHED THEN INSERT VALUES ('2021-11-01 00:00:05'::timestamp with time zone, 5, 210, 'string - INSERTED BY MERGE'); -- Merge INSERT with constant literals for hypertables MERGE INTO target t USING source s ON t.location = 1234 WHEN NOT MATCHED THEN INSERT VALUES ('2021-11-01 00:00:05'::timestamp with time zone, 5, 210, 'string - INSERTED BY MERGE'); -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; -- Merge with INSERT/DELETE/UPDATE on PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.location = s.location WHEN MATCHED AND t.location = 560076 THEN UPDATE SET temperature = (t.temperature + s.temperature) * 2, val = val || ' UPDATED BY MERGE' WHEN MATCHED AND t.location = 560083 THEN DELETE WHEN NOT MATCHED THEN INSERT (time, location, temperature, val) VALUES (s.time, s.location, s.temperature, 'string - INSERTED BY MERGE'); -- Merge with INSERT/DELETE/UPDATE on hypertables MERGE INTO target t USING source s ON t.time = s.time AND t.location = s.location WHEN MATCHED AND t.location = 560076 THEN UPDATE SET temperature = (t.temperature + s.temperature) * 2, val = val || ' UPDATED BY MERGE' WHEN MATCHED AND t.location = 560083 THEN DELETE WHEN NOT MATCHED THEN INSERT (time, location, temperature, val) VALUES (s.time, s.location, s.temperature, 'string - INSERTED BY MERGE'); -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; -- Merge with Subqueries on PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.location > (SELECT count(*) FROM source_pg) WHEN MATCHED AND t.temperature = 23 THEN UPDATE SET temperature = (SELECT count(*) FROM target_pg) * 2, val = val || ' UPDATED BY MERGE' WHEN MATCHED AND t.temperature = 47 THEN DELETE WHEN NOT MATCHED THEN INSERT (time, location, temperature, val) VALUES (s.time, s.location, s.temperature, 'SUBQUERY string - INSERTED BY MERGE'); -- Merge with Subqueries on hypertables MERGE INTO target t USING source s ON t.time = s.time AND t.location > (SELECT count(*) FROM source) WHEN MATCHED AND t.temperature = 23 THEN UPDATE SET temperature = (SELECT count(*) FROM target) * 2, val = val || ' UPDATED BY MERGE' WHEN MATCHED AND t.temperature = 47 THEN DELETE WHEN NOT MATCHED THEN INSERT (time, location, temperature, val) VALUES (s.time, s.location, s.temperature, 'SUBQUERY string - INSERTED BY MERGE'); -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; -- clean up tables DELETE FROM target_pg; DELETE FROM target; DELETE FROM source_pg; DELETE FROM source; -- TEST with target as hypertable and source as normal PG table INSERT INTO target SELECT time, location, 14 as temperature FROM generate_series( '2021-01-01 00:00:00', '2021-01-01 00:00:09', INTERVAL '5 seconds' ) as time, generate_series(1,4) as location; INSERT INTO source SELECT time, location, 80 as temperature FROM generate_series( '2021-01-01 00:00:05', '2021-01-01 00:00:14', INTERVAL '5 seconds' ) as time, generate_series(1,4) as location; INSERT INTO target_pg SELECT * FROM target; INSERT INTO source_pg SELECT * FROM source; -- Merge UPDATE matched rows for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN UPDATE SET temperature = (t.temperature + s.temperature)/2, val = val || ' UPDATED BY MERGE'; -- Merge UPDATE with target as hypertables and source as normal PG tables MERGE INTO target t USING source_pg s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN UPDATE SET temperature = (t.temperature + s.temperature)/2, val = val || ' UPDATED BY MERGE'; -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; -- Merge DELETE matched rows for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN DELETE; -- Merge DELETE with target as hypertables and source as normal PG tables MERGE INTO target t USING source_pg s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN DELETE; -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; -- Merge INSERT with constant literals for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.location = 1234 WHEN NOT MATCHED THEN INSERT VALUES ('2021-11-01 00:00:05'::timestamp with time zone, 5, 210, 'string - INSERTED BY MERGE'); -- Merge INSERT with constant literals for target as hypertables and source as normal PG tables MERGE INTO target t USING source s ON t.location = 1234 WHEN NOT MATCHED THEN INSERT VALUES ('2021-11-01 00:00:05'::timestamp with time zone, 5, 210, 'string - INSERTED BY MERGE'); -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; -- Merge with INSERT/DELETE/UPDATE on PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.location = s.location WHEN MATCHED AND t.temperature = 23 THEN UPDATE SET temperature = (t.temperature + s.temperature) * 2, val = val || ' UPDATED BY MERGE' WHEN MATCHED AND t.temperature = 47 THEN DELETE WHEN NOT MATCHED THEN INSERT (time, location, temperature, val) VALUES (s.time, s.location, s.temperature, 'string - INSERTED BY MERGE'); -- Merge with INSERT/DELETE/UPDATE on target as hypertables and source as normal PG tables MERGE INTO target t USING source s ON t.time = s.time AND t.location = s.location WHEN MATCHED AND t.temperature = 23 THEN UPDATE SET temperature = (t.temperature + s.temperature) * 2, val = val || ' UPDATED BY MERGE' WHEN MATCHED AND t.temperature = 47 THEN DELETE WHEN NOT MATCHED THEN INSERT (time, location, temperature, val) VALUES (s.time, s.location, s.temperature, 'string - INSERTED BY MERGE'); -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; DROP TABLE target_pg CASCADE; DROP TABLE target CASCADE; DROP TABLE source_pg CASCADE; DROP TABLE source CASCADE; -- test MERGE with source being a PARTITION table CREATE TABLE source_pg( id INT NOT NULL, dev INT NOT NULL, value INT, CONSTRAINT cstr_source_pky PRIMARY KEY (id) ) PARTITION BY LIST (id); CREATE TABLE source_1_2_3_4 PARTITION OF source_pg FOR VALUES IN (1,2,3,4); CREATE TABLE source_5_6_7_8 PARTITION OF source_pg FOR VALUES IN (5,6,7,8); INSERT INTO source_pg SELECT generate_series(1,8), 44,55; CREATE TABLE target ( ts TIMESTAMP WITH TIME ZONE NOT NULL, id INT NOT NULL, dev INT NOT NULL, FOREIGN KEY (id) REFERENCES source_pg(id) ON DELETE CASCADE ); SELECT create_hypertable( relation => 'target', time_column_name => 'ts' ); insert into target values ('2023-01-12 00:00:05'::timestamp with time zone, 1,2); insert into target values ('2023-01-12 00:00:10'::timestamp with time zone, 2,2); insert into target values ('2023-01-12 00:00:15'::timestamp with time zone, 3,2); insert into target values ('2023-01-12 00:00:20'::timestamp with time zone, 4,2); insert into target values ('2023-01-14 00:00:25'::timestamp with time zone, 5,2); insert into target values ('2023-01-14 00:00:30'::timestamp with time zone, 6,2); insert into target values ('2023-01-14 00:00:35'::timestamp with time zone, 7,2); insert into target values ('2023-01-14 00:00:40'::timestamp with time zone, 8,2); CREATE TABLE target_pg AS SELECT * FROM target; -- Merge UPDATE matched rows for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.id = s.id WHEN MATCHED THEN UPDATE SET dev = (t.dev + s.dev)/2; -- Merge UPDATE matched rows for hypertables MERGE INTO target t USING source_pg s ON t.id = s.id WHEN MATCHED THEN UPDATE SET dev = (t.dev + s.dev)/2; -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; -- Merge DELETE matched rows for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.id = s.id WHEN MATCHED THEN DELETE; -- Merge DELETE matched rows for hypertables MERGE INTO target t USING source_pg s ON t.id = s.id WHEN MATCHED THEN DELETE; -- ensure TARGET PG table and hypertable are same SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; -- clean up tables DROP TABLE target_pg CASCADE; DROP TABLE target CASCADE; DROP TABLE source_pg CASCADE; -- test MERGE with hypertables with time and space partitions CREATE TABLE target ( filler_1 int, filler_2 int, filler_3 int, time timestamptz NOT NULL, device_id int, device_id_peer int, v0 int, v1 float, v2 float, v3 float ); SELECT create_hypertable ('target', 'time', 'device_id', 5); SELECT add_dimension('target', 'device_id_peer', 5); SELECT add_dimension('target', 'v2', 5); INSERT INTO target (time, device_id, device_id_peer, v0, v1, v2, v3) SELECT time, device_id, 0, device_id + 1, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-01 0:00:00+0'::timestamptz, '2000-01-05 23:55:00+0', '20m') gtime (time), generate_series(1, 2, 1) gdevice (device_id); CREATE TABLE source ( filler_1 int, filler_2 int, filler_3 int, time timestamptz NOT NULL, device_id int ); SELECT create_hypertable ('source', 'time', 'device_id', 3); INSERT INTO source (time, device_id, filler_2, filler_3, filler_1) SELECT time, device_id, device_id + 134, device_id + 209, device_id + 0.50127 FROM generate_series('2000-01-01 0:00:00+0'::timestamptz, '2000-01-05 23:55:00+0', '20m') gtime (time), generate_series(1, 5, 1) gdevice (device_id); -- create PG tables to compare PG target and hypertable target tables CREATE table target_pg as SELECT * FROM target; CREATE table source_pg as SELECT * FROM source; -- Merge UDPATE matched rows for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.device_id = s.device_id WHEN MATCHED THEN UPDATE SET filler_2 = s.filler_1 + 100; -- Merge UDPATE matched rows for space partitioned hypertables MERGE INTO target t USING source s ON t.time = s.time AND t.device_id = s.device_id WHEN MATCHED THEN UPDATE SET filler_2 = s.filler_1 + 100; SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; -- Merge DELETE matched rows for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.device_id = s.device_id WHEN MATCHED THEN DELETE; -- Merge DELETE matched rows for space partitioned hypertables MERGE INTO target t USING source s ON t.time = s.time AND t.device_id = s.device_id WHEN MATCHED THEN DELETE; SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; -- Merge INSERT matched rows for normal PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.device_id = s.device_id WHEN NOT MATCHED THEN INSERT (filler_1, filler_2, filler_3, time, device_id, device_id_peer, v0, v1, v2, v3) VALUES (s.filler_1, s.filler_2, s.filler_3, s.time, s.device_id, s.device_id + 10, 1,2,3,4); -- Merge INSERT matched rows for space partitioned hypertables MERGE INTO target t USING source s ON t.time = s.time AND t.device_id = s.device_id WHEN NOT MATCHED THEN INSERT (filler_1, filler_2, filler_3, time, device_id, device_id_peer, v0, v1, v2, v3) VALUES (s.filler_1, s.filler_2, s.filler_3, s.time, s.device_id, s.device_id + 10, 1,2,3,4); SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; -- Merge with INSERT/DELETE/UPDATE on PG tables MERGE INTO target_pg t USING source_pg s ON t.time = s.time AND t.device_id = s.device_id WHEN MATCHED AND t.device_id_peer = 2 THEN UPDATE SET filler_2 = s.filler_1 + s.filler_2 + s.filler_3 + 100 WHEN MATCHED AND t.device_id_peer = 7 THEN DELETE WHEN NOT MATCHED THEN INSERT (filler_1, filler_2, filler_3, time, device_id, device_id_peer, v0, v1, v2, v3) VALUES (s.filler_1, s.filler_2, s.filler_3, s.time, s.device_id, s.device_id + 10, 1,2,3,4); -- Merge with INSERT/DELETE/UPDATE on space partitioned hypertables MERGE INTO target t USING source s ON t.time = s.time AND t.device_id = s.device_id WHEN MATCHED AND t.device_id_peer = 2 THEN UPDATE SET filler_2 = s.filler_1 + s.filler_2 + s.filler_3 + 100 WHEN MATCHED AND t.device_id_peer = 7 THEN DELETE WHEN NOT MATCHED THEN INSERT (filler_1, filler_2, filler_3, time, device_id, device_id_peer, v0, v1, v2, v3) VALUES (s.filler_1, s.filler_2, s.filler_3, s.time, s.device_id, s.device_id + 10, 1,2,3,4); SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; -- clean up tables DROP TABLE target_pg CASCADE; DROP TABLE target CASCADE; DROP TABLE source_pg CASCADE; DROP TABLE source CASCADE; -- TEST with parition column place after similar data type column CREATE TABLE target ( filler_1 int, filler_2 int, filler_3 int, time timestamptz NOT NULL, device_id int, device_id_peer int, v0 int, v1 float, v2 float, v3 float, partition_column TIMESTAMPTZ NOT NULL ); SELECT create_hypertable ('target', 'partition_column'); INSERT INTO target (time, device_id, device_id_peer, v0, v1, v2, v3, partition_column) SELECT time, device_id, 0, device_id + 1, device_id + 2, device_id + 0.5, NULL, time + interval '10m' FROM generate_series('2000-01-01 0:00:00+0'::timestamptz, '2000-01-05 23:55:00+0', '20m') gtime (time), generate_series(1, 2, 1) gdevice (device_id); CREATE TABLE source ( filler_1 int, filler_2 int, filler_3 int, time timestamptz NOT NULL, device_id int ); SELECT create_hypertable ('source', 'time', 'device_id', 3); INSERT INTO source (time, device_id, filler_2, filler_3, filler_1) SELECT time, device_id, device_id + 134, device_id + 209, device_id + 0.50127 FROM generate_series('2000-01-01 0:00:00+0'::timestamptz, '2000-01-05 23:55:00+0', '20m') gtime (time), generate_series(1, 5, 1) gdevice (device_id); -- create PG tables to compare PG target and hypertable target tables CREATE table target_pg as SELECT * FROM target; MERGE INTO target_pg t USING source s ON t.time = s.time AND t.device_id = s.device_id WHEN NOT MATCHED THEN INSERT (time, device_id, device_id_peer, v0, v1, v2, v3, partition_column) VALUES ('2010-01-06 05:30:00+05:30', 23, 2, 11, 22, 33, 44, '2023-01-06 05:33:00+05:30'); MERGE INTO target t USING source s ON t.time = s.time AND t.device_id = s.device_id WHEN NOT MATCHED THEN INSERT (time, device_id, device_id_peer, v0, v1, v2, v3, partition_column) VALUES ('2010-01-06 05:30:00+05:30', 23, 2, 11, 22, 33, 44, '2023-01-06 05:33:00+05:30'); SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; MERGE INTO target_pg t USING source s ON t.time = s.time AND t.device_id = s.device_id WHEN MATCHED THEN DELETE; MERGE INTO target t USING source s ON t.time = s.time AND t.device_id = s.device_id WHEN MATCHED THEN DELETE; SELECT CASE WHEN EXISTS (TABLE target EXCEPT TABLE target_pg) OR EXISTS (TABLE target_pg EXCEPT TABLE target) THEN 'different' ELSE 'same' END AS result; MERGE INTO target_pg t USING source s ON t.time = s.time AND t.device_id = s.device_id WHEN NOT MATCHED THEN INSERT (filler_1, filler_2, filler_3, time, device_id, device_id_peer, v0, v1, v2, v3) VALUES (s.filler_1, s.filler_2, s.filler_3, s.time, s.device_id, s.device_id + 10, 1,2,3,4); -- time dimension column is NULL, this will report an null constraint violation error \set ON_ERROR_STOP 0 MERGE INTO target t USING source s ON t.time = s.time AND t.device_id = s.device_id WHEN NOT MATCHED THEN INSERT (filler_1, filler_2, filler_3, time, device_id, device_id_peer, v0, v1, v2, v3) VALUES (s.filler_1, s.filler_2, s.filler_3, s.time, s.device_id, s.device_id + 10, 1,2,3,4); \set ON_ERROR_STOP 1 DROP TABLE target CASCADE; DROP TABLE target_pg CASCADE; DROP TABLE source CASCADE; -- TEST with target table have CHECK constraints CREATE TABLE target ( time TIMESTAMPTZ NOT NULL, location SMALLINT NOT NULL, temperature DOUBLE PRECISION NULL CHECK (temperature > 10), val text default 'string -' ); SELECT create_hypertable( 'target', 'time', chunk_time_interval => INTERVAL '5 seconds'); INSERT INTO target SELECT time, location, 14 as temperature FROM generate_series( '2021-01-01 00:00:00', '2021-01-01 00:00:09', INTERVAL '5 seconds' ) as time, generate_series(1,4) as location; -- Create source table with location and temperature CREATE TABLE source ( time TIMESTAMPTZ NOT NULL, location SMALLINT NOT NULL, temperature DOUBLE PRECISION NULL ); -- Generate data that overlaps with target table INSERT INTO source SELECT time, location, 80 as temperature FROM generate_series( '2021-01-01 00:00:05', '2021-01-01 00:00:14', INTERVAL '5 seconds' ) as time, generate_series(1,4) as location; -- CREATE normal PostgreSQL tables CREATE TABLE target_pg AS SELECT * FROM target; -- Merge UPDATE/DELETE with DO NOTHING on pg tables MERGE INTO target_pg t USING source s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN DO NOTHING WHEN NOT MATCHED THEN DO NOTHING; -- Merge UPDATE/DELETE with DO NOTHING on hypertable MERGE INTO target t USING source s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN DO NOTHING WHEN NOT MATCHED THEN DO NOTHING; -- Error cases for Merge \set ON_ERROR_STOP 0 -- Merge UPDATE should fail with check constraint violation MERGE INTO target_pg t USING source s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN UPDATE SET temperature = 8, val = val || ' UPDATED BY MERGE'; -- Merge UPDATE should fail with check constraint violation MERGE INTO target t USING source s ON t.time = s.time AND t.location = s.location WHEN MATCHED THEN UPDATE SET temperature = 8, val = val || ' UPDATED BY MERGE'; -- Merge error with unreachable WHEN clause on pg tables MERGE INTO target_pg t USING source s ON t.time = s.time AND t.location != s.location WHEN MATCHED THEN UPDATE SET temperature = 8, val = val || ' UPDATED BY MERGE' WHEN MATCHED AND t.time < now() THEN DELETE WHEN NOT MATCHED THEN DO NOTHING; -- Merge error with unreachable WHEN clause on hypertable MERGE INTO target t USING source s ON t.time = s.time AND t.location != s.location WHEN MATCHED THEN UPDATE SET temperature = 8, val = val || ' UPDATED BY MERGE' WHEN MATCHED AND t.time < now() THEN DELETE WHEN NOT MATCHED THEN DO NOTHING; -- Merge error with unknown action in MERGE WHEN MATCHED clause on pg tables MERGE INTO target_pg t USING source s ON t.time = s.time AND t.location != s.location WHEN MATCHED THEN SELECT 1; -- Merge error with unknown action in MERGE WHEN MATCHED clause on hypertable MERGE INTO target t USING source s ON t.time = s.time AND t.location != s.location WHEN MATCHED THEN SELECT 1; -- Merge error cannot affect row a second time on pg tables MERGE INTO target_pg t USING source s ON t.location = s.location WHEN MATCHED THEN UPDATE SET temperature = 28, val = val || ' UPDATED BY MERGE'; -- Merge error cannot affect row a second time on hypertable MERGE INTO target t USING source s ON t.location = s.location WHEN MATCHED THEN UPDATE SET temperature = 28, val = val || ' UPDATED BY MERGE'; \set ON_ERROR_STOP 1 DROP TABLE target CASCADE; DROP TABLE target_pg CASCADE; DROP TABLE source CASCADE; -- TEST for PERMISSIONS CREATE USER priv_user; CREATE USER non_priv_user; CREATE TABLE target ( value DOUBLE PRECISION NOT NULL, time TIMESTAMPTZ NOT NULL ); SELECT table_name FROM create_hypertable( 'target'::regclass, 'time'::name, chunk_time_interval=>interval '8 hours', create_default_indexes=> false); SELECT '2022-10-10 14:33:44.1234+05:30' as start_date \gset INSERT INTO target (value, time) SELECT 1,t from generate_series(:'start_date'::timestamptz, :'start_date'::timestamptz + interval '1 day', '5m') t cross join generate_series(1,3) s; CREATE TABLE source ( time TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP NOT NULL, value DOUBLE PRECISION NOT NULL ); SELECT table_name FROM create_hypertable( 'source'::regclass, 'time'::name, chunk_time_interval=>interval '6 hours', create_default_indexes=> false); ALTER TABLE target OWNER TO priv_user; ALTER TABLE source OWNER TO priv_user; GRANT SELECT ON source TO non_priv_user; SET SESSION AUTHORIZATION non_priv_user; \set ON_ERROR_STOP 0 -- non_priv_user does not have UPDATE privilege on target table MERGE INTO target USING source ON target.time = source.time WHEN MATCHED THEN UPDATE SET value = 0; -- non_priv_user does not have DELETE privilege on target table MERGE INTO target USING source ON target.time = source.time WHEN MATCHED THEN DELETE; -- non_priv_user does not have INSERT privilege on target table MERGE INTO target USING source ON target.time = source.time WHEN NOT MATCHED THEN INSERT VALUES (10, '2023-01-15 00:00:10'::timestamp with time zone); \set ON_ERROR_STOP 1 RESET SESSION AUTHORIZATION; DROP TABLE target; DROP TABLE source; DROP USER priv_user; DROP USER non_priv_user; ================================================ FILE: test/sql/metadata.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION _timescaledb_internal.test_uuid() RETURNS UUID AS :MODULE_PATHNAME, 'ts_test_uuid' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_exported_uuid() RETURNS UUID AS :MODULE_PATHNAME, 'ts_test_exported_uuid' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_install_timestamp() RETURNS TIMESTAMPTZ AS :MODULE_PATHNAME, 'ts_test_install_timestamp' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; INSERT INTO _timescaledb_catalog.metadata (key, value, include_in_telemetry) SELECT 'metadata_test', 'FOO', TRUE; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- uuid and install_timestamp should already be in the table before we generate SELECT COUNT(*) from _timescaledb_catalog.metadata; SELECT _timescaledb_internal.test_uuid() as uuid_1 \gset SELECT _timescaledb_internal.test_exported_uuid() as uuid_ex_1 \gset SELECT _timescaledb_internal.test_install_timestamp() as timestamp_1 \gset -- Check that there is exactly 1 UUID row SELECT COUNT(*) from _timescaledb_catalog.metadata where key='uuid'; -- Check that exported_uuid and timestamp are also generated SELECT COUNT(*) from _timescaledb_catalog.metadata where key='exported_uuid'; SELECT COUNT(*) from _timescaledb_catalog.metadata where key='install_timestamp'; -- Make sure that the UUID is idempotent SELECT _timescaledb_internal.test_uuid() = :'uuid_1' as uuids_equal; SELECT _timescaledb_internal.test_uuid() = :'uuid_1' as uuids_equal; -- Also make sure install_time and exported_uuid are idempotent SELECT _timescaledb_internal.test_exported_uuid() = :'uuid_ex_1' as exported_uuids_equal; SELECT _timescaledb_internal.test_exported_uuid() = :'uuid_ex_1' as exported_uuids_equal; SELECT _timescaledb_internal.test_install_timestamp() = :'timestamp_1' as timestamps_equal; SELECT _timescaledb_internal.test_install_timestamp() = :'timestamp_1' as timestamps_equal; -- Now make sure that only the exported_uuid is exported on pg_dump \c postgres :ROLE_SUPERUSER \setenv PGOPTIONS '--client-min-messages=warning' \! utils/pg_dump_aux_dump.sh dump/instmeta.sql ALTER DATABASE :TEST_DBNAME SET timescaledb.restoring='on'; -- Redirect to /dev/null to suppress NOTICE \! utils/pg_dump_aux_restore.sh dump/instmeta.sql ALTER DATABASE :TEST_DBNAME SET timescaledb.restoring='off'; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- Should have all 3 row, because pg_dump includes the insertion of uuid and timestamp. SELECT COUNT(*) FROM _timescaledb_catalog.metadata; -- Verify that this is the old exported_uuid SELECT _timescaledb_internal.test_exported_uuid() = :'uuid_ex_1' as exported_uuids_equal; -- Verify that the uuid is new SELECT _timescaledb_internal.test_uuid() = :'uuid_1' as exported_uuids_diff; -- Verify that the install_timestamp got restored SELECT _timescaledb_internal.test_install_timestamp() = :'timestamp_1' as timestamps_equal; SELECT * FROM _timescaledb_catalog.metadata WHERE key = 'metadata_test'; -- check metadata version matches expected value SELECT x.extversion = m.value AS "version match" FROM pg_extension x JOIN _timescaledb_catalog.metadata m ON m.key='timescaledb_version' WHERE x.extname='timescaledb'; -- test version check in post_restore \c :TEST_DBNAME :ROLE_SUPERUSER UPDATE _timescaledb_catalog.metadata SET value = '1.2.3' WHERE key = 'timescaledb_version'; \set ON_ERROR_STOP 0 -- set verbosity to sqlstate to suppress version dependant error message \set VERBOSITY sqlstate SELECT timescaledb_post_restore(); \set ON_ERROR_STOP 1 UPDATE _timescaledb_catalog.metadata m SET value = x.extversion FROM pg_extension x WHERE m.key = 'timescaledb_version' AND x.extname='timescaledb'; SELECT timescaledb_post_restore(); ================================================ FILE: test/sql/multi_transaction_index.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE index_test(id serial, time timestamptz, device integer, temp float); SELECT * FROM test.show_columns('index_test'); -- Test that we can handle difference in attnos across hypertable and -- chunks by dropping the ID column ALTER TABLE index_test DROP COLUMN id; SELECT * FROM test.show_columns('index_test'); -- No pre-existing UNIQUE index, so partitioning on two columns should work SELECT create_hypertable('index_test', 'time', 'device', 2); INSERT INTO index_test VALUES ('2017-01-20T09:00:01', 1, 17.5); \set ON_ERROR_STOP 0 -- cannot create a UNIQUE index with transaction_per_chunk CREATE UNIQUE INDEX index_test_time_device_idx ON index_test (time) WITH (timescaledb.transaction_per_chunk); CREATE UNIQUE INDEX index_test_time_device_idx ON index_test (time, device) WITH(timescaledb.transaction_per_chunk); \set ON_ERROR_STOP 1 CREATE INDEX index_test_time_device_idx ON index_test (time, device) WITH (timescaledb.transaction_per_chunk); -- Regular index need not cover all partitioning columns CREATE INDEX ON index_test (time, temp) WITH (timescaledb.transaction_per_chunk); -- Create another chunk INSERT INTO index_test VALUES ('2017-04-20T09:00:01', 1, 17.5); -- New index should have been recursed to chunks SELECT * FROM test.show_indexes('index_test'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk') ORDER BY 1,2; ALTER INDEX index_test_time_idx RENAME TO index_test_time_idx2; -- Metadata and index should have changed name SELECT * FROM test.show_indexes('index_test'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk') ORDER BY 1,2; DROP INDEX index_test_time_idx2; DROP INDEX index_test_time_device_idx; -- Index should have been dropped SELECT * FROM test.show_indexes('index_test'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); -- Create index with long name to see how this is handled on chunks CREATE INDEX a_hypertable_index_with_a_very_very_long_name_that_truncates ON index_test (time ASC, temp DESC) WITH (timescaledb.transaction_per_chunk); CREATE INDEX a_hypertable_index_with_a_very_very_long_name_that_truncates_2 ON index_test (time DESC, temp ASC) WITH (timescaledb.transaction_per_chunk); SELECT * FROM test.show_indexes('index_test'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); DROP INDEX a_hypertable_index_with_a_very_very_long_name_that_truncates; DROP INDEX a_hypertable_index_with_a_very_very_long_name_that_truncates_2; SELECT * FROM test.show_indexes('index_test'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); SELECT * FROM test.show_indexes('index_test'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); -- Add constraint index ALTER TABLE index_test ADD UNIQUE (time, device); SELECT * FROM test.show_indexes('index_test'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); -- Constraints are added to chunk_constraint table. SELECT * FROM _timescaledb_catalog.chunk_constraint; DROP TABLE index_test; -- Test that indexes are planned correctly CREATE TABLE index_expr_test(id serial, time timestamptz, temp float, meta int); select create_hypertable('index_expr_test', 'time'); -- Screw up the attribute numbers ALTER TABLE index_expr_test DROP COLUMN id; CREATE INDEX ON index_expr_test (meta) WITH (timescaledb.transaction_per_chunk); INSERT INTO index_expr_test VALUES ('2017-01-20T09:00:01', 17.5, 1); INSERT INTO index_expr_test VALUES ('2017-01-20T09:00:01', 17.5, 2); SET enable_seqscan TO false; SET enable_bitmapscan TO false; EXPLAIN (verbose, buffers off, costs off) SELECT * FROM index_expr_test WHERE meta = 1; SELECT * FROM index_expr_test WHERE meta = 1; SET enable_seqscan TO default; SET enable_bitmapscan TO default; \set ON_ERROR_STOP 0 -- cannot create a transaction_per_chunk index within a transaction block BEGIN; CREATE INDEX ON index_expr_test (temp) WITH (timescaledb.transaction_per_chunk); ROLLBACK; \set ON_ERROR_STOP 1 DROP TABLE index_expr_test CASCADE; CREATE TABLE partial_index_test(time INTEGER); SELECT create_hypertable('partial_index_test', 'time', chunk_time_interval => 1, create_default_indexes => false); -- create 3 chunks INSERT INTO partial_index_test(time) SELECT generate_series(0, 2); select * from partial_index_test order by 1; -- create indexes on only 1 of the chunks CREATE INDEX ON partial_index_test (time) WITH (timescaledb.transaction_per_chunk, timescaledb.max_chunks='1'); SELECT * FROM test.show_indexes('partial_index_test'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); -- regerssion test for bug fixed by PR #1008. -- this caused an assertion failure when a MergeAppend node contained unsorted children SET enable_bitmapscan TO false; EXPLAIN (verbose, buffers off, costs off) SELECT * FROM partial_index_test WHERE time < 2 ORDER BY time LIMIT 2; SELECT * FROM partial_index_test WHERE time < 2 ORDER BY time LIMIT 2; -- we can drop the partially created index DROP INDEX partial_index_test_time_idx; SELECT * FROM test.show_indexes('partial_index_test'); SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); EXPLAIN (verbose, buffers off, costs off) SELECT * FROM partial_index_test WHERE time < 2 ORDER BY time LIMIT 2; SELECT * FROM partial_index_test WHERE time < 2 ORDER BY time LIMIT 2; SET enable_seqscan TO true; SET enable_bitmapscan TO true; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 \set ON_ERROR_STOP 0 CREATE INDEX ON partial_index_test (time) WITH (timescaledb.transaction_per_chunk, timescaledb.max_chunks='1'); \set ON_ERROR_STOP 1 ================================================ FILE: test/sql/net.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION _timescaledb_internal.test_http_parsing(int) RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_http_parsing' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_http_parsing_full() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_http_parsing_full' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_http_request_build() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_http_request_build' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_conn() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_conn' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT _timescaledb_internal.test_http_parsing(10000); SELECT _timescaledb_internal.test_http_parsing_full(); SELECT _timescaledb_internal.test_http_request_build(); SELECT _timescaledb_internal.test_conn(); ================================================ FILE: test/sql/null_exclusion.sql.in ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. create table metrics(ts timestamp, id int, value float); select create_hypertable('metrics', 'ts'); insert into metrics values ('2022-02-02 02:02:02', 2, 2.), ('2023-03-03 03:03:03', 3, 3.); analyze metrics; -- non-const condition explain (analyze, buffers off, costs off, summary off, timing off) select * from metrics where ts >= (select max(ts) from metrics); -- two non-const conditions explain (analyze, buffers off, costs off, summary off, timing off) select * from metrics where ts >= (select max(ts) from metrics) and id = 1; -- condition that becomes const null after evaluating the param explain (analyze, buffers off, costs off, summary off, timing off) select * from metrics where ts >= (select max(ts) from metrics where id = -1); -- const null condition and some other condition explain (analyze, buffers off, costs off, summary off, timing off) select * from metrics where ts >= (select max(ts) from metrics where id = -1) and id = 1; ================================================ FILE: test/sql/parallel.sql.in ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --parallel queries require big-ish tables so collect them all here --so that we need to generate queries only once. -- output with analyze is not stable because it depends on worker assignment \set PREFIX 'EXPLAIN (buffers off, costs off)' \set CHUNK1 _timescaledb_internal._hyper_1_1_chunk \set CHUNK2 _timescaledb_internal._hyper_1_2_chunk CREATE TABLE test (i int, j double precision, ts timestamp); SELECT create_hypertable('test','i',chunk_time_interval:=500000); INSERT INTO test SELECT x, x+0.1, _timescaledb_functions.to_timestamp(x*1000) FROM generate_series(0,1000000-1,10) AS x; ANALYZE test; ALTER TABLE :CHUNK1 SET (parallel_workers=2); ALTER TABLE :CHUNK2 SET (parallel_workers=2); SET work_mem TO '50MB'; SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false); SET max_parallel_workers_per_gather = 4; SET parallel_setup_cost TO 0; EXPLAIN (buffers off, costs off) SELECT first(i, j) FROM "test"; SELECT first(i, j) FROM "test"; EXPLAIN (buffers off, costs off) SELECT last(i, j) FROM "test"; SELECT last(i, j) FROM "test"; EXPLAIN (buffers off, costs off) SELECT time_bucket('1 second', ts) sec, last(i, j) FROM "test" GROUP BY sec ORDER BY sec LIMIT 5; -- test single copy parallel plan with parallel chunk append :PREFIX SELECT time_bucket('1 second', ts) sec, last(i, j) FROM "test" WHERE length(version()) > 0 GROUP BY sec ORDER BY sec LIMIT 5; SELECT time_bucket('1 second', ts) sec, last(i, j) FROM "test" GROUP BY sec ORDER BY sec LIMIT 5; --test variants of histogram EXPLAIN (buffers off, costs off) SELECT histogram(i, 1, 1000000, 2) FROM "test"; SELECT histogram(i, 1, 1000000, 2) FROM "test"; EXPLAIN (buffers off, costs off) SELECT histogram(i, 1,1000001,10) FROM "test"; SELECT histogram(i, 1, 1000001, 10) FROM "test"; EXPLAIN (buffers off, costs off) SELECT histogram(i, 0,100000,5) FROM "test"; SELECT histogram(i, 0, 100000, 5) FROM "test"; EXPLAIN (buffers off, costs off) SELECT histogram(i, 10,100000,5) FROM "test"; SELECT histogram(i, 10, 100000, 5) FROM "test"; EXPLAIN (buffers off, costs off) SELECT histogram(NULL, 10,100000,5) FROM "test" WHERE i = coalesce(-1,j); SELECT histogram(NULL, 10,100000,5) FROM "test" WHERE i = coalesce(-1,j); -- test parallel ChunkAppend :PREFIX SELECT i FROM "test" WHERE length(version()) > 0; :PREFIX SELECT count(*) FROM "test" WHERE i > 1 AND length(version()) > 0; SELECT count(*) FROM "test" WHERE i > 1 AND length(version()) > 0; -- test parallel ChunkAppend with only work done in the parallel workers SET parallel_leader_participation = off; SELECT count(*) FROM "test" WHERE i > 1 AND length(version()) > 0; RESET parallel_leader_participation; -- Test parallel chunk append is used (index scan is disabled to trigger a parallel chunk append) SET parallel_tuple_cost = 0; SET enable_indexscan = OFF; :PREFIX SELECT * FROM (SELECT * FROM "test" WHERE length(version()) > 0 ORDER BY I LIMIT 10) AS t1 LEFT JOIN (SELECT * FROM "test" WHERE i < 500000 ORDER BY I LIMIT 10) AS t2 ON (t1.i = t2.i) ORDER BY t1.i, t2.i; SELECT * FROM (SELECT * FROM "test" WHERE length(version()) > 0 ORDER BY I LIMIT 10) AS t1 LEFT JOIN (SELECT * FROM "test" WHERE i < 500000 ORDER BY I LIMIT 10) AS t2 ON (t1.i = t2.i) ORDER BY t1.i, t2.i; SET enable_indexscan = ON; -- Test normal chunk append can be used in a parallel worker :PREFIX SELECT * FROM (SELECT * FROM "test" WHERE i >= 999000 ORDER BY i) AS t1 JOIN (SELECT * FROM "test" WHERE i >= 400000 ORDER BY i) AS t2 ON (TRUE) ORDER BY t1.i, t2.i LIMIT 10; SELECT * FROM (SELECT * FROM "test" WHERE i >= 999000 ORDER BY i) AS t1 JOIN (SELECT * FROM "test" WHERE i >= 400000 ORDER BY i) AS t2 ON (TRUE) ORDER BY t1.i, t2.i LIMIT 10; -- Test parallel ChunkAppend reinit SET enable_material = off; SET min_parallel_table_scan_size = 0; SET min_parallel_index_scan_size = 0; SET enable_hashjoin = 'off'; SET enable_nestloop = 'off'; CREATE TABLE sensor_data( time timestamptz NOT NULL, sensor_id integer NOT NULL); SELECT FROM create_hypertable(relation=>'sensor_data', time_column_name=> 'time'); -- Sensors 1 and 2 INSERT INTO sensor_data SELECT time, sensor_id FROM generate_series('2000-01-01 00:00:30', '2022-01-01 00:00:30', INTERVAL '3 months') AS g1(time), generate_series(1, 2, 1) AS g2(sensor_id) ORDER BY time; -- Sensor 100 INSERT INTO sensor_data SELECT time, 100 as sensor_id FROM generate_series('2000-01-01 00:00:30', '2022-01-01 00:00:30', INTERVAL '1 year') AS g1(time) ORDER BY time; :PREFIX SELECT * FROM sensor_data AS s1 JOIN sensor_data AS s2 ON (TRUE) WHERE s1.time > '2020-01-01 00:00:30'::text::timestamptz AND s2.time > '2020-01-01 00:00:30' AND s2.time < '2021-01-01 00:00:30' AND s1.sensor_id > 50 ORDER BY s2.time, s1.time, s1.sensor_id, s2.sensor_id; -- Check query result SELECT * FROM sensor_data AS s1 JOIN sensor_data AS s2 ON (TRUE) WHERE s1.time > '2020-01-01 00:00:30'::text::timestamptz AND s2.time > '2020-01-01 00:00:30' AND s2.time < '2021-01-01 00:00:30' AND s1.sensor_id > 50 ORDER BY s2.time, s1.time, s1.sensor_id, s2.sensor_id; -- Ensure the same result is produced if only the parallel workers have to produce them (i.e., the pstate is reinitialized properly) SET parallel_leader_participation = off; SELECT * FROM sensor_data AS s1 JOIN sensor_data AS s2 ON (TRUE) WHERE s1.time > '2020-01-01 00:00:30'::text::timestamptz AND s2.time > '2020-01-01 00:00:30' AND s2.time < '2021-01-01 00:00:30' AND s1.sensor_id > 50 ORDER BY s2.time, s1.time, s1.sensor_id, s2.sensor_id; RESET parallel_leader_participation; -- Ensure the same query result is produced by a sequencial query SET max_parallel_workers_per_gather TO 0; SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'off', false); SELECT * FROM sensor_data AS s1 JOIN sensor_data AS s2 ON (TRUE) WHERE s1.time > '2020-01-01 00:00:30'::text::timestamptz AND s2.time > '2020-01-01 00:00:30' AND s2.time < '2021-01-01 00:00:30' AND s1.sensor_id > 50 ORDER BY s2.time, s1.time, s1.sensor_id, s2.sensor_id; RESET enable_material; RESET min_parallel_table_scan_size; RESET min_parallel_index_scan_size; RESET enable_hashjoin; RESET enable_nestloop; RESET parallel_tuple_cost; SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false); -- test worker assignment -- first chunk should have 1 worker and second chunk should have 2 SET max_parallel_workers_per_gather TO 2; :PREFIX SELECT count(*) FROM "test" WHERE i >= 400000 AND length(version()) > 0; SELECT count(*) FROM "test" WHERE i >= 400000 AND length(version()) > 0; -- test worker assignment -- first chunk should have 2 worker and second chunk should have 1 :PREFIX SELECT count(*) FROM "test" WHERE i < 600000 AND length(version()) > 0; SELECT count(*) FROM "test" WHERE i < 600000 AND length(version()) > 0; -- test ChunkAppend with # workers < # childs SET max_parallel_workers_per_gather TO 1; :PREFIX SELECT count(*) FROM "test" WHERE length(version()) > 0; SELECT count(*) FROM "test" WHERE length(version()) > 0; -- test ChunkAppend with # workers > # childs SET max_parallel_workers_per_gather TO 2; :PREFIX SELECT count(*) FROM "test" WHERE i >= 500000 AND length(version()) > 0; SELECT count(*) FROM "test" WHERE i >= 500000 AND length(version()) > 0; RESET max_parallel_workers_per_gather; -- test partial and non-partial plans -- these will not be parallel on PG < 11 ALTER TABLE :CHUNK1 SET (parallel_workers=0); ALTER TABLE :CHUNK2 SET (parallel_workers=2); :PREFIX SELECT count(*) FROM "test" WHERE i > 400000 AND length(version()) > 0; ALTER TABLE :CHUNK1 SET (parallel_workers=2); ALTER TABLE :CHUNK2 SET (parallel_workers=0); :PREFIX SELECT count(*) FROM "test" WHERE i < 600000 AND length(version()) > 0; ALTER TABLE :CHUNK1 RESET (parallel_workers); ALTER TABLE :CHUNK2 RESET (parallel_workers); -- now() is not marked parallel safe in PostgreSQL < 12 so using now() -- in a query will prevent parallelism but CURRENT_TIMESTAMP and -- transaction_timestamp() are marked parallel safe :PREFIX SELECT i FROM "test" WHERE ts < CURRENT_TIMESTAMP; :PREFIX SELECT i FROM "test" WHERE ts < transaction_timestamp(); -- this won't be parallel query because now() is parallel restricted in PG < 12 :PREFIX SELECT i FROM "test" WHERE ts < now(); ================================================ FILE: test/sql/partition.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE part_legacy(time timestamptz, temp float, device int); SELECT create_hypertable('part_legacy', 'time', 'device', 2, partitioning_func => '_timescaledb_functions.get_partition_for_key'); -- Show legacy partitioning function is used SELECT * FROM _timescaledb_catalog.dimension; INSERT INTO part_legacy VALUES ('2017-03-22T09:18:23', 23.4, 1); INSERT INTO part_legacy VALUES ('2017-03-22T09:18:23', 23.4, 76); VACUUM part_legacy; -- Show two chunks and CHECK constraint with cast SELECT * FROM test.show_constraintsp('_timescaledb_internal._hyper_1_%_chunk'); -- Make sure constraint exclusion works on device column BEGIN; -- For plan stability between versions SET LOCAL enable_bitmapscan = false; SET LOCAL enable_indexscan = false; EXPLAIN (verbose, buffers off, costs off) SELECT * FROM part_legacy WHERE device = 1; COMMIT; CREATE TABLE part_new(time timestamptz, temp float, device int); SELECT create_hypertable('part_new', 'time', 'device', 2); SELECT * FROM _timescaledb_catalog.dimension; INSERT INTO part_new VALUES ('2017-03-22T09:18:23', 23.4, 1); INSERT INTO part_new VALUES ('2017-03-22T09:18:23', 23.4, 2); VACUUM part_new; -- Show two chunks and CHECK constraint without cast SELECT * FROM test.show_constraintsp('_timescaledb_internal._hyper_2_%_chunk'); -- Make sure constraint exclusion works on device column BEGIN; -- For plan stability between versions SET LOCAL enable_bitmapscan = false; SET LOCAL enable_indexscan = false; EXPLAIN (verbose, buffers off, costs off) SELECT * FROM part_new WHERE device = 1; COMMIT; CREATE TABLE part_new_convert1(time timestamptz, temp float8, device int); SELECT create_hypertable('part_new_convert1', 'time', 'temp', 2); INSERT INTO part_new_convert1 VALUES ('2017-03-22T09:18:23', 1.0, 2); \set ON_ERROR_STOP 0 -- Changing the type of a hash-partitioned column should not be supported ALTER TABLE part_new_convert1 ALTER COLUMN temp TYPE numeric; \set ON_ERROR_STOP 1 -- Should be able to change if not hash partitioned though ALTER TABLE part_new_convert1 ALTER COLUMN time TYPE timestamp; SELECT * FROM test.show_columnsp('_timescaledb_internal._hyper_3_%_chunk'); CREATE TABLE part_add_dim(time timestamptz, temp float8, device int, location int); SELECT create_hypertable('part_add_dim', 'time', 'temp', 2); \set ON_ERROR_STOP 0 SELECT add_dimension('part_add_dim', 'location', 2, partitioning_func => 'bad_func'); \set ON_ERROR_STOP 1 SELECT add_dimension('part_add_dim', 'location', 2, partitioning_func => '_timescaledb_functions.get_partition_for_key'); SELECT * FROM _timescaledb_catalog.dimension; -- Test that we support custom SQL-based partitioning functions and -- that our native partitioning function handles function expressions -- as argument CREATE OR REPLACE FUNCTION custom_partfunc(source anyelement) RETURNS INTEGER LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ DECLARE retval INTEGER; BEGIN retval = _timescaledb_functions.get_partition_hash(substring(source::text FROM '[A-za-z0-9 ]+')); RAISE NOTICE 'hash value for % is %', source, retval; RETURN retval; END $BODY$; CREATE TABLE part_custom_func(time timestamptz, temp float8, device text); SELECT create_hypertable('part_custom_func', 'time', 'device', 2, partitioning_func => 'custom_partfunc'); SELECT _timescaledb_functions.get_partition_hash(substring('dev1' FROM '[A-za-z0-9 ]+')); SELECT _timescaledb_functions.get_partition_hash('dev1'::text); SELECT _timescaledb_functions.get_partition_hash('dev7'::text); INSERT INTO part_custom_func VALUES ('2017-03-22T09:18:23', 23.4, 'dev1'), ('2017-03-22T09:18:23', 23.4, 'dev7'); SELECT * FROM test.show_subtables('part_custom_func'); -- This first test is slightly trivial, but segfaulted in old versions CREATE TYPE simpl AS (val1 int4); CREATE OR REPLACE FUNCTION simpl_type_hash(ANYELEMENT) RETURNS int4 AS $$ SELECT $1.val1; $$ LANGUAGE SQL IMMUTABLE; CREATE TABLE simpl_partition ("timestamp" TIMESTAMPTZ, object simpl); SELECT create_hypertable( 'simpl_partition', 'timestamp', 'object', 1000, chunk_time_interval => interval '1 day', partitioning_func=>'simpl_type_hash'); INSERT INTO simpl_partition VALUES ('2017-03-22T09:18:23', ROW(1)::simpl); SELECT * from simpl_partition; -- Also test that the fix works when we have more chunks than allowed at once SET timescaledb.max_open_chunks_per_insert=1; INSERT INTO simpl_partition VALUES ('2017-03-22T10:18:23', ROW(0)::simpl), ('2017-03-22T10:18:23', ROW(1)::simpl), ('2017-03-22T10:18:23', ROW(2)::simpl), ('2017-03-22T10:18:23', ROW(3)::simpl), ('2017-03-22T10:18:23', ROW(4)::simpl), ('2017-03-22T10:18:23', ROW(5)::simpl); SET timescaledb.max_open_chunks_per_insert=default; SELECT * from simpl_partition; -- Test that index creation is handled correctly. CREATE TABLE hyper_with_index(time timestamptz, temp float, device int); CREATE UNIQUE INDEX temp_index ON hyper_with_index(temp); \set ON_ERROR_STOP 0 SELECT create_hypertable('hyper_with_index', 'time'); SELECT create_hypertable('hyper_with_index', 'time', 'device', 2); SELECT create_hypertable('hyper_with_index', 'time', 'temp', 2); \set ON_ERROR_STOP 1 DROP INDEX temp_index; CREATE UNIQUE INDEX time_index ON hyper_with_index(time); \set ON_ERROR_STOP 0 -- should error because device not in index SELECT create_hypertable('hyper_with_index', 'time', 'device', 4); \set ON_ERROR_STOP 1 SELECT create_hypertable('hyper_with_index', 'time'); -- make sure user created index is used. -- not using \d or \d+ because output syntax differs -- between postgres 9 and postgres 10. SELECT indexname FROM pg_indexes WHERE tablename = 'hyper_with_index'; \set ON_ERROR_STOP 0 SELECT add_dimension('hyper_with_index', 'device', 4); \set ON_ERROR_STOP 1 DROP INDEX time_index; CREATE UNIQUE INDEX time_space_index ON hyper_with_index(time, device); SELECT add_dimension('hyper_with_index', 'device', 4); CREATE TABLE hyper_with_primary(time TIMESTAMPTZ PRIMARY KEY, temp float, device int); \set ON_ERROR_STOP 0 SELECT create_hypertable('hyper_with_primary', 'time', 'device', 4); \set ON_ERROR_STOP 1 SELECT create_hypertable('hyper_with_primary', 'time'); \set ON_ERROR_STOP 0 SELECT add_dimension('hyper_with_primary', 'device', 4); \set ON_ERROR_STOP 1 -- NON-unique indexes can still be created CREATE INDEX temp_index ON hyper_with_index(temp); -- Make sure custom composite types are supported as dimensions CREATE TYPE TUPLE as (val1 int4, val2 int4); CREATE FUNCTION tuple_hash(value ANYELEMENT) RETURNS INT4 LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RAISE NOTICE 'custom hash value is: %', value.val1+value.val2; RETURN value.val1+value.val2; END $BODY$; CREATE TABLE part_custom_dim (time TIMESTAMPTZ, combo TUPLE, device TEXT); SELECT create_hypertable('part_custom_dim', 'time', 'combo', 4, partitioning_func=>'tuple_hash'); INSERT INTO part_custom_dim(time, combo) VALUES (now(), (1,2)); DROP TABLE part_custom_dim; -- Now make sure that renaming partitioning_func_schema will get updated properly \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA IF NOT EXISTS my_partitioning_schema; CREATE FUNCTION my_partitioning_schema.tuple_hash(value ANYELEMENT) RETURNS INT4 LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RAISE NOTICE 'custom hash value is: %', value.val1+value.val2; RETURN value.val1+value.val2; END $BODY$; CREATE TABLE part_custom_dim (time TIMESTAMPTZ, combo TUPLE, device TEXT); SELECT create_hypertable('part_custom_dim', 'time', 'combo', 4, partitioning_func=>'my_partitioning_schema.tuple_hash'); INSERT INTO part_custom_dim(time, combo) VALUES (now(), (1,2)); ALTER SCHEMA my_partitioning_schema RENAME TO new_partitioning_schema; -- Inserts should work even after we rename the schema INSERT INTO part_custom_dim(time, combo) VALUES (now(), (3,4)); -- Test partitioning function on an open (time) dimension CREATE OR REPLACE FUNCTION time_partfunc(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ DECLARE retval TIMESTAMPTZ; BEGIN retval := to_timestamp(unixtime); RAISE NOTICE 'time value for % is %', unixtime, timezone('UTC', retval); RETURN retval; END $BODY$; CREATE OR REPLACE FUNCTION time_partfunc_bad_parameters(unixtime float8, extra text) RETURNS TIMESTAMPTZ LANGUAGE SQL IMMUTABLE AS $BODY$ SELECT to_timestamp(unixtime); $BODY$; CREATE OR REPLACE FUNCTION time_partfunc_bad_return_type(unixtime float8) RETURNS FLOAT8 LANGUAGE SQL IMMUTABLE AS $BODY$ SELECT unixtime; $BODY$; CREATE TABLE part_time_func(time float8, temp float8, device text); \set ON_ERROR_STOP 0 -- Should fail due to invalid time column SELECT create_hypertable('part_time_func', 'time'); -- Should fail due to bad signature of time partitioning function SELECT create_hypertable('part_time_func', 'time', time_partitioning_func => 'time_partfunc_bad_parameters'); SELECT create_hypertable('part_time_func', 'time', time_partitioning_func => 'time_partfunc_bad_return_type'); \set ON_ERROR_STOP 1 -- Should work with time partitioning function that returns a valid time type SELECT create_hypertable('part_time_func', 'time', time_partitioning_func => 'time_partfunc'); INSERT INTO part_time_func VALUES (1530214157.134, 23.4, 'dev1'), (1533214157.8734, 22.3, 'dev7'); SELECT time, temp, device FROM part_time_func; SELECT time_partfunc(time) at time zone 'UTC', temp, device FROM part_time_func; SELECT * FROM test.show_subtables('part_time_func'); SELECT (test.show_constraints("Child")).* FROM test.show_subtables('part_time_func'); SELECT (test.show_indexes("Child")).* FROM test.show_subtables('part_time_func'); -- Check that constraint exclusion works with time partitioning -- function (scan only one chunk) -- No exclusion EXPLAIN (verbose, buffers off, costs off) SELECT * FROM part_time_func; -- Exclude using the function on time EXPLAIN (verbose, buffers off, costs off) SELECT * FROM part_time_func WHERE time_partfunc(time) < '2018-07-01'; -- Exclude using the same date but as a UNIX timestamp. Won't do an -- index scan since the index is on the time function expression EXPLAIN (verbose, buffers off, costs off) SELECT * FROM part_time_func WHERE time < 1530403200.0; -- Check that inserts will fail if we use a time partitioning function -- that returns NULL CREATE OR REPLACE FUNCTION time_partfunc_null_ret(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ BEGIN RETURN NULL; END $BODY$; CREATE TABLE part_time_func_null_ret(time float8, temp float8, device text); SELECT create_hypertable('part_time_func_null_ret', 'time', time_partitioning_func => 'time_partfunc_null_ret'); \set ON_ERROR_STOP 0 INSERT INTO part_time_func_null_ret VALUES (1530214157.134, 23.4, 'dev1'), (1533214157.8734, 22.3, 'dev7'); \set ON_ERROR_STOP 1 ================================================ FILE: test/sql/partition_coercion.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test partition chunk exclusion with cross-type comparisons -- wrong result: text column + name literal CREATE TABLE hash_text(time timestamptz NOT NULL, device text); SELECT create_hypertable('hash_text', 'time'); SELECT add_dimension('hash_text', 'device', number_partitions => 3); INSERT INTO hash_text VALUES ('2000-01-01', 'abc'); SELECT count(*) FROM hash_text WHERE device = 'abc'::name; DROP TABLE hash_text; -- x86_64 wrong result: int4 time column + large int8 literal + range query -- 4294967196::int8 truncates to -100 as signed int4 -- Chunk exclusion uses time < -100, excluding all positive-time chunks CREATE FUNCTION time_part_int4(val int4) RETURNS int4 AS $$ SELECT val $$ LANGUAGE SQL IMMUTABLE; CREATE TABLE time_int4(time int4 NOT NULL, v int); SELECT create_hypertable('time_int4', 'time', chunk_time_interval => 100, time_partitioning_func => 'time_part_int4'); INSERT INTO time_int4 VALUES (100, 1), (200, 2); -- Both rows satisfy time < 4294967196, but bug truncates to time < -100 SELECT count(*) FROM time_int4 WHERE time < 4294967196::int8; DROP TABLE time_int4; DROP FUNCTION time_part_int4; -- i386 crash: int8 time column + int4 literal + custom partitioning -- On i386: SEGFAULT (DatumGetInt64 dereferences byval int4 as pointer) -- On x86_64: works by coincidence (both int4 and int8 are byval) CREATE FUNCTION time_part_int8(val int8) RETURNS int8 AS $$ SELECT val $$ LANGUAGE SQL IMMUTABLE; CREATE TABLE time_int8(time int8 NOT NULL, v int); SELECT create_hypertable('time_int8', 'time', chunk_time_interval => 10, time_partitioning_func => 'time_part_int8'); INSERT INTO time_int8 VALUES (1, 1), (11, 2), (21, 3); SELECT count(*) FROM time_int8 WHERE time = 1::int4; DROP TABLE time_int8; DROP FUNCTION time_part_int8; -- Exact type match: text column + text literal (no coercion needed) CREATE TABLE hash_text_exact(time timestamptz NOT NULL, device text); SELECT create_hypertable('hash_text_exact', 'time'); SELECT add_dimension('hash_text_exact', 'device', number_partitions => 3); INSERT INTO hash_text_exact VALUES ('2000-01-01', 'abc'); SELECT count(*) FROM hash_text_exact WHERE device = 'abc'::text; DROP TABLE hash_text_exact; -- Binary compatible types: text column + varchar literal -- PostgreSQL coerces varchar to text at parse time CREATE TABLE hash_text_varchar(time timestamptz NOT NULL, device text); SELECT create_hypertable('hash_text_varchar', 'time'); SELECT add_dimension('hash_text_varchar', 'device', number_partitions => 3); INSERT INTO hash_text_varchar VALUES ('2000-01-01', 'abc'); SELECT count(*) FROM hash_text_varchar WHERE device = 'abc'::varchar; DROP TABLE hash_text_varchar; -- Array coercion: text column + name[] array (ScalarArrayOpExpr) -- Test both ANY (OR) and ALL (AND) semantics CREATE TABLE hash_text_array(time timestamptz NOT NULL, device text); SELECT create_hypertable('hash_text_array', 'time'); SELECT add_dimension('hash_text_array', 'device', number_partitions => 3); INSERT INTO hash_text_array VALUES ('2000-01-01', 'abc'), ('2000-01-01', 'def'), ('2000-01-01', 'ghi'); -- OR: match any element SELECT count(*) FROM hash_text_array WHERE device = ANY(ARRAY['abc', 'def']::name[]); -- AND: match all elements (logically empty for different values, but exercises code path) SELECT count(*) FROM hash_text_array WHERE device = ALL(ARRAY['abc', 'def']::name[]); -- AND: single element (equivalent to =) SELECT count(*) FROM hash_text_array WHERE device = ALL(ARRAY['abc']::name[]); DROP TABLE hash_text_array; -- Time dimension with SAOP + type coercion (int4 column, int8 array) -- Note: open (time/range) dimensions can't use SAOP with multiple OR values -- for chunk exclusion, but AND with single effective bound works. -- These tests verify correct results and that AND cases use chunk exclusion. CREATE FUNCTION time_part_int4_saop(val int4) RETURNS int4 AS $$ SELECT val $$ LANGUAGE SQL IMMUTABLE; CREATE TABLE time_int4_saop(time int4 NOT NULL, v int); SELECT create_hypertable('time_int4_saop', 'time', chunk_time_interval => 100, time_partitioning_func => 'time_part_int4_saop'); INSERT INTO time_int4_saop VALUES (50, 1), (150, 2), (250, 3); -- AND: time < ALL(array) means time < min(array) = 100 -- Single effective bound, chunk exclusion should work SELECT count(*) FROM time_int4_saop WHERE time < ALL(ARRAY[100, 200]::int8[]); -- AND: time > ALL(array) means time > max(array) = 200 SELECT count(*) FROM time_int4_saop WHERE time > ALL(ARRAY[100, 200]::int8[]); -- OR cases: chunk exclusion not used (multiple OR values rejected), -- but results must still be correct SELECT count(*) FROM time_int4_saop WHERE time < ANY(ARRAY[100, 200]::int8[]); SELECT count(*) FROM time_int4_saop WHERE time > ANY(ARRAY[100, 200]::int8[]); DROP TABLE time_int4_saop; DROP FUNCTION time_part_int4_saop; -- Prepared statement with varchar parameter, text column -- Custom plan: coercion at plan time, chunk exclusion works -- Generic plan: no chunk exclusion (param unknown), but correct result CREATE TABLE hash_prep(time timestamptz NOT NULL, device text); SELECT create_hypertable('hash_prep', 'time'); SELECT add_dimension('hash_prep', 'device', number_partitions => 3); INSERT INTO hash_prep VALUES ('2000-01-01', 'abc'), ('2000-01-01', 'def'); PREPARE hash_q(varchar) AS SELECT count(*) FROM hash_prep WHERE device = $1; SET plan_cache_mode = force_custom_plan; EXECUTE hash_q('abc'); EXECUTE hash_q('def'); SET plan_cache_mode = force_generic_plan; EXECUTE hash_q('abc'); EXECUTE hash_q('def'); RESET plan_cache_mode; DEALLOCATE hash_q; DROP TABLE hash_prep; -- Multiple ANDed restrictions on closed dimension -- Use IN + = to exercise intersection: IN creates list, = intersects with it CREATE TABLE hash_multi(time timestamptz NOT NULL, device text); SELECT create_hypertable('hash_multi', 'time'); SELECT add_dimension('hash_multi', 'device', number_partitions => 3); INSERT INTO hash_multi VALUES ('2000-01-01', 'abc'), ('2000-01-01', 'def'); -- IN creates partition list [abc,def], then = intersects with [abc] => [abc] SELECT count(*) FROM hash_multi WHERE device IN ('abc', 'def') AND device = 'abc'; DROP TABLE hash_multi; -- NULL array elements: exercises branch when iterating arrays with NULLs -- The NULL elements should be skipped, non-NULL elements should work CREATE TABLE hash_null_array(time timestamptz NOT NULL, device text); SELECT create_hypertable('hash_null_array', 'time'); SELECT add_dimension('hash_null_array', 'device', number_partitions => 3); INSERT INTO hash_null_array VALUES ('2000-01-01', 'abc'), ('2000-01-01', 'def'); -- Array with NULL elements (type coercion path) SELECT count(*) FROM hash_null_array WHERE device = ANY(ARRAY['abc', NULL, 'def']::name[]); -- Array with NULL elements (no type coercion path) SELECT count(*) FROM hash_null_array WHERE device = ANY(ARRAY['abc', NULL, 'def']::text[]); DROP TABLE hash_null_array; ================================================ FILE: test/sql/partitioned_hypertable.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test declarative partitioning for hypertables -- Enable declarative partitioning for all subsequent tests SET timescaledb.enable_partitioned_hypertables = true; -- Basic hypertable creation with TIMESTAMPTZ CREATE TABLE metrics( time TIMESTAMP WITH TIME ZONE, device TEXT, value FLOAT ) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); -- Create with TIMESTAMP CREATE TABLE metrics_ts( time TIMESTAMP NOT NULL, device TEXT, value FLOAT ) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); -- Create with DATE CREATE TABLE metrics_date( time DATE NOT NULL, device TEXT, value FLOAT ) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); -- Create with int CREATE TABLE metrics_int( time INT NOT NULL, device TEXT, value FLOAT ) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); -- Create with custom chunk_time_interval CREATE TABLE metrics_custom_interval( time TIMESTAMPTZ NOT NULL, device TEXT, value FLOAT ) WITH (timescaledb.hypertable, timescaledb.partition_column='time', timescaledb.chunk_interval='30 days'); -- Verify hypertables are actually created and partitioned SELECT hypertable_name FROM timescaledb_information.hypertables WHERE hypertable_name IN ('metrics', 'metrics_ts', 'metrics_date', 'metrics_int', 'metrics_custom_interval') ORDER BY hypertable_name; SELECT DISTINCT(relkind) = 'p' FROM pg_class WHERE relname IN ('metrics', 'metrics_ts', 'metrics_date', 'metrics_int', 'metrics_custom_interval'); \set ON_ERROR_STOP 0 -- Try to create with invalid partition column type CREATE TABLE invalid(time TEXT, value FLOAT) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); CREATE TABLE invalid(time TEXT, value FLOAT) WITH (timescaledb.hypertable); \set ON_ERROR_STOP 1 DROP TABLE IF EXISTS metrics_ts; DROP TABLE IF EXISTS metrics_date; DROP TABLE IF EXISTS metrics_int; DROP TABLE IF EXISTS metrics_custom_interval; DROP TABLE IF EXISTS invalid; -- Test PARTITION BY syntax CREATE TABLE metrics_partition_by( time TIMESTAMPTZ NOT NULL, device TEXT, value FLOAT ) PARTITION BY RANGE (time) WITH (timescaledb.hypertable); \set ON_ERROR_STOP 0 CREATE TABLE part_col_specified(time TIMESTAMPTZ, device TEXT) PARTITION BY RANGE (time) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); CREATE TABLE multiple_part_key(time TIMESTAMPTZ, time2 TIMESTAMP, device TEXT) PARTITION BY RANGE (time, time2) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); CREATE TABLE bad_strategy(time TIMESTAMPTZ, device TEXT) PARTITION BY LIST (time) WITH (timescaledb.hypertable); \set ON_ERROR_STOP 1 DROP TABLE IF EXISTS metrics_partition_by; -- Insert Operations and Chunk Creation INSERT INTO metrics VALUES ('2025-01-15 00:00:00+00', 'device1', 11.0), ('2025-02-15 00:00:00+00', 'device2', 12.0), ('2025-03-15 00:00:00+00', 'device3', 13.0); SELECT count(*) FROM show_chunks('metrics'); SELECT count(*) FROM metrics; SELECT * FROM metrics ORDER BY time; -- Verify chunk was created and attached as partition SELECT count(*) FROM show_chunks('metrics'); SELECT child.relname AS chunk, parent.relname AS hypertable FROM pg_inherits JOIN pg_class child ON inhrelid = child.oid JOIN pg_class parent ON inhparent = parent.oid WHERE parent.relname = 'metrics' LIMIT 1; -- Insert with CHECK constraint ALTER TABLE metrics ADD CONSTRAINT valcheck CHECK (value >= 0); -- Try inserting into existing and new chunk to violate CHECK constraint \set ON_ERROR_STOP 0 INSERT INTO metrics VALUES ('2025-03-15 00:00:00+00', 'device1', -10.0); INSERT INTO metrics VALUES ('2025-04-15 00:00:00+00', 'device1', -10.0); \set ON_ERROR_STOP 1 -- SELECT with WHERE on time (partition pruning) EXPLAIN (COSTS OFF) SELECT * FROM metrics WHERE time >= '2025-01-01' AND time < '2025-02-01'; -- FOREIGN KEY from hypertable to regular table CREATE TABLE ref_table(id INT PRIMARY KEY, name TEXT); INSERT INTO ref_table VALUES (1, 'ref1'), (2, 'ref2'); CREATE TABLE fk_table( time TIMESTAMPTZ NOT NULL, ref_id INT REFERENCES ref_table(id), value FLOAT ) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); INSERT INTO fk_table VALUES ('2025-11-01', 1, 10.0); \set ON_ERROR_STOP 0 INSERT INTO fk_table VALUES ('2025-11-01', 999, 20.0); \set ON_ERROR_STOP 1 DROP TABLE ref_table CASCADE; DROP TABLE fk_table CASCADE; -- FOREIGN KEY from hypertable to hypertable CREATE TABLE ref_ht( time TIMESTAMPTZ NOT NULL , id INT, CONSTRAINT ref_ht_pkey PRIMARY KEY (time, id) ) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); INSERT INTO ref_ht VALUES ('2025-06-15 00:00:00+00', 1), ('2025-07-15 00:00:00+00', 2); CREATE TABLE fk_ht( time TIMESTAMPTZ NOT NULL, ref_time TIMESTAMPTZ, ref_id INT, value FLOAT, FOREIGN KEY (ref_time, ref_id) REFERENCES ref_ht(time, id) ) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); INSERT INTO fk_ht VALUES ('2025-08-15 00:00:00+00', '2025-06-15 00:00:00+00', 1, 31.0); \set ON_ERROR_STOP 0 INSERT INTO fk_ht VALUES ('2025-08-15 00:00:00+00', '2025-01-01 00:00:00+00', 999, 32.0); \set ON_ERROR_STOP 1 DROP TABLE ref_ht CASCADE; DROP TABLE fk_ht CASCADE; -- Test if foreign keys to hypertables not using declarative partitioning are still disallowed SET timescaledb.enable_partitioned_hypertables = false; CREATE TABLE ref_ht( time TIMESTAMPTZ NOT NULL , id INT, CONSTRAINT ref_ht_pkey PRIMARY KEY (time, id) ) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); INSERT INTO ref_ht VALUES ('2025-06-15 00:00:00+00', 1), ('2025-07-15 00:00:00+00', 2); SET timescaledb.enable_partitioned_hypertables = true; \set ON_ERROR_STOP 0 CREATE TABLE fk_ht( time TIMESTAMPTZ NOT NULL, ref_time TIMESTAMPTZ, ref_id INT, value FLOAT, FOREIGN KEY (ref_time, ref_id) REFERENCES ref_ht(time, id) ) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); \set ON_ERROR_STOP 1 DROP TABLE ref_ht CASCADE; DROP TABLE IF EXISTS fk_ht CASCADE; -- Test partition wise joins CREATE TABLE metrics_pwj( time TIMESTAMPTZ NOT NULL, device TEXT, value FLOAT ) WITH (timescaledb.hypertable, timescaledb.partition_column='time'); INSERT INTO metrics_pwj VALUES ('2025-01-15 00:00:00+00', 'device1', 11.0), ('2025-02-15 00:00:00+00', 'device2', 12.0), ('2025-03-15 00:00:00+00', 'device3', 13.0); SET enable_partitionwise_join = true; EXPLAIN (COSTS OFF) SELECT m1.device, m2.device FROM metrics AS m1 JOIN metrics_pwj AS m2 ON m1.time = m2.time; SET enable_partitionwise_join = false; -- Transaction - ROLLBACK BEGIN; INSERT INTO metrics VALUES ('2024-12-20', 'rollback_test', 42.0); SELECT count(*)=1 FROM metrics WHERE device = 'rollback_test'; ROLLBACK; SELECT count(*)=0 FROM metrics WHERE device = 'rollback_test'; -- Reset GUC SET timescaledb.enable_partitioned_hypertables = false; -- Cleanup DROP TABLE IF EXISTS metrics CASCADE; DROP TABLE IF EXISTS metrics_pwj CASCADE; ================================================ FILE: test/sql/partitioning.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Should expect an error when creating a hypertable from a partition \set ON_ERROR_STOP 0 CREATE TABLE partitioned_ht_create(time timestamptz, temp float, device int) PARTITION BY RANGE (time); SELECT create_hypertable('partitioned_ht_create', 'time'); \set ON_ERROR_STOP 1 -- Should expect an error when attaching a hypertable to a partition \set ON_ERROR_STOP 0 CREATE TABLE partitioned_attachment_vanilla(time timestamptz, temp float, device int) PARTITION BY RANGE (time); CREATE TABLE attachment_hypertable(time timestamptz, temp float, device int); SELECT create_hypertable('attachment_hypertable', 'time'); ALTER TABLE partitioned_attachment_vanilla ATTACH PARTITION attachment_hypertable FOR VALUES FROM ('2016-07-01') TO ('2016-08-01'); \set ON_ERROR_STOP 1 -- Should not expect an error when attaching a normal table to a partition CREATE TABLE partitioned_vanilla(time timestamptz, temp float, device int) PARTITION BY RANGE (time); CREATE TABLE attachment_vanilla(time timestamptz, temp float, device int); ALTER TABLE partitioned_vanilla ATTACH PARTITION attachment_vanilla FOR VALUES FROM ('2016-07-01') TO ('2016-08-01'); ================================================ FILE: test/sql/partitionwise.sql.in ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set PREFIX 'EXPLAIN (VERBOSE, BUFFERS OFF, COSTS OFF)' -- Create a two dimensional hypertable CREATE TABLE hyper (time timestamptz, device int, temp float); SELECT * FROM create_hypertable('hyper', 'time', 'device', 2); -- Create a similar PostgreSQL partitioned table CREATE TABLE pg2dim (time timestamptz, device int, temp float) PARTITION BY HASH (device); CREATE TABLE pg2dim_h1 PARTITION OF pg2dim FOR VALUES WITH (MODULUS 2, REMAINDER 0) PARTITION BY RANGE(time); CREATE TABLE pg2dim_h2 PARTITION OF pg2dim FOR VALUES WITH (MODULUS 2, REMAINDER 1) PARTITION BY RANGE(time); CREATE TABLE pg2dim_h1_t1 PARTITION OF pg2dim_h1 FOR VALUES FROM ('2018-01-01 00:00') TO ('2018-09-01 00:00'); CREATE TABLE pg2dim_h1_t2 PARTITION OF pg2dim_h1 FOR VALUES FROM ('2018-09-01 00:00') TO ('2018-12-01 00:00'); CREATE TABLE pg2dim_h2_t1 PARTITION OF pg2dim_h2 FOR VALUES FROM ('2018-01-01 00:00') TO ('2018-09-01 00:00'); CREATE TABLE pg2dim_h2_t2 PARTITION OF pg2dim_h2 FOR VALUES FROM ('2018-09-01 00:00') TO ('2018-12-01 00:00'); -- Create a 1-dimensional partitioned table for comparison CREATE TABLE pg1dim (time timestamptz, device int, temp float) PARTITION BY HASH (device); CREATE TABLE pg1dim_h1 PARTITION OF pg1dim FOR VALUES WITH (MODULUS 2, REMAINDER 0); CREATE TABLE pg1dim_h2 PARTITION OF pg1dim FOR VALUES WITH (MODULUS 2, REMAINDER 1); INSERT INTO hyper VALUES ('2018-02-19 13:01', 1, 2.3), ('2018-02-19 13:02', 3, 3.1), ('2018-10-19 13:01', 1, 7.6), ('2018-10-19 13:02', 3, 9.0); INSERT INTO pg2dim VALUES ('2018-02-19 13:01', 1, 2.3), ('2018-02-19 13:02', 3, 3.1), ('2018-10-19 13:01', 1, 7.6), ('2018-10-19 13:02', 3, 9.0); INSERT INTO pg1dim VALUES ('2018-02-19 13:01', 1, 2.3), ('2018-02-19 13:02', 3, 3.1), ('2018-10-19 13:01', 1, 7.6), ('2018-10-19 13:02', 3, 9.0); SELECT * FROM test.show_subtables('hyper'); SELECT * FROM pg2dim_h1_t1; SELECT * FROM pg2dim_h1_t2; SELECT * FROM pg2dim_h2_t1; SELECT * FROM pg2dim_h2_t2; -- Compare partitionwise aggreate enabled/disabled. First run queries -- on PG partitioned tables for reference. -- All partition keys covered by GROUP BY SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT device, avg(temp) FROM pg1dim GROUP BY 1 ORDER BY 1; SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT device, avg(temp) FROM pg1dim GROUP BY 1 ORDER BY 1; -- All partition keys not covered by GROUP BY (partial partitionwise) SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT device, avg(temp) FROM pg2dim GROUP BY 1 ORDER BY 1; SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT device, avg(temp) FROM pg2dim GROUP BY 1 ORDER BY 1; -- All partition keys covered by GROUP BY (full partitionwise) SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT time, device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT time, device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; -- All partition keys not covered by GROUP BY because of date_trunc -- expression on time (partial partitionwise) SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM pg2dim GROUP BY 1, 2 ORDER BY 1, 2; -- Now run on hypertable -- All partition keys not covered by GROUP BY (partial partitionwise) SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT device, avg(temp) FROM hyper GROUP BY 1 ORDER BY 1; SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT device, avg(temp) FROM hyper GROUP BY 1 ORDER BY 1; -- All partition keys covered (full partitionwise) SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT time, device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT time, device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; -- Partial aggregation since date_trunc(time) is not a partition key SET enable_partitionwise_aggregate = 'off'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; -- Partial aggregation pushdown is currently not supported for this query by -- the TSDB pushdown code since a projection is used in the path. SET enable_partitionwise_aggregate = 'on'; :PREFIX SELECT date_trunc('month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; -- Also test time_bucket SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT time_bucket('1 month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT time_bucket('1 month', time), device, avg(temp) FROM hyper GROUP BY 1, 2 ORDER BY 1, 2; -- Test partitionwise joins, mostly to see that we do not break -- anything CREATE TABLE hyper_meta (time timestamptz, device int, info text); SELECT * FROM create_hypertable('hyper_meta', 'time', 'device', 2); INSERT INTO hyper_meta VALUES ('2018-02-19 13:01', 1, 'device_1'), ('2018-02-19 13:02', 3, 'device_3'); SET enable_partitionwise_join = 'off'; :PREFIX SELECT h.time, h.device, h.temp, hm.info FROM hyper h, hyper_meta hm WHERE h.device = hm.device; :PREFIX SELECT pg2.time, pg2.device, pg2.temp, pg1.temp FROM pg2dim pg2, pg1dim pg1 WHERE pg2.device = pg1.device; SET enable_partitionwise_join = 'on'; :PREFIX SELECT h.time, h.device, h.temp, hm.info FROM hyper h, hyper_meta hm WHERE h.device = hm.device; :PREFIX SELECT pg2.time, pg2.device, pg2.temp, pg1.temp FROM pg2dim pg2, pg1dim pg1 WHERE pg2.device = pg1.device; -- Test hypertable with time partitioning function CREATE OR REPLACE FUNCTION time_func(unixtime float8) RETURNS TIMESTAMPTZ LANGUAGE PLPGSQL IMMUTABLE AS $BODY$ DECLARE retval TIMESTAMPTZ; BEGIN retval := to_timestamp(unixtime); RETURN retval; END $BODY$; CREATE TABLE hyper_timepart (time float8, device int, temp float); SELECT * FROM create_hypertable('hyper_timepart', 'time', 'device', 2, time_partitioning_func => 'time_func'); -- Planner won't pick push-down aggs on table with time function -- unless a certain amount of data SELECT setseed(1); INSERT INTO hyper_timepart SELECT x, ceil(random() * 8), random() * 20 FROM generate_series(0,5000-1) AS x; -- All partition keys covered (full partitionwise) SET timescaledb.enable_chunkwise_aggregation = 'off'; :PREFIX SELECT time, device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; :PREFIX SELECT time_func(time), device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; -- Grouping on original time column should be pushed-down SET timescaledb.enable_chunkwise_aggregation = 'on'; :PREFIX SELECT time, device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; -- Applying the time partitioning function should also allow push-down -- on open dimensions :PREFIX SELECT time_func(time), device, avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; -- Partial aggregation pushdown is currently not supported for this query by -- the TSDB pushdown code since a projection is used in the path. :PREFIX SELECT time_func(time), _timescaledb_functions.get_partition_hash(device), avg(temp) FROM hyper_timepart GROUP BY 1, 2 ORDER BY 1, 2 LIMIT 10; -- Test removal of redundant group key optimization in PG16 -- All lower versions include the redundant key on device column :PREFIX SELECT device, avg(temp) FROM hyper_timepart WHERE device = 1 GROUP BY 1 LIMIT 10; ================================================ FILE: test/sql/pg_dump.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set TEST_DBNAME_EXTRA :TEST_DBNAME _extra \o /dev/null \ir include/insert_two_partitions.sql \o \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION bgw_wait(database TEXT, timeout INT, raise_error BOOLEAN DEFAULT TRUE) RETURNS VOID AS :MODULE_PATHNAME, 'ts_bgw_wait' LANGUAGE C VOLATILE; CREATE SCHEMA test_schema AUTHORIZATION :ROLE_DEFAULT_PERM_USER; \c :TEST_DBNAME ALTER TABLE PUBLIC."two_Partitions" SET SCHEMA "test_schema"; -- Test that we can restore constraints ALTER TABLE "test_schema"."two_Partitions" ADD CONSTRAINT timeCustom_device_id_series_2_key UNIQUE ("timeCustom", device_id, series_2); -- Test that we can restore triggers CREATE OR REPLACE FUNCTION test_trigger() RETURNS TRIGGER LANGUAGE PLPGSQL AS $BODY$ BEGIN RETURN NEW; END $BODY$; -- Test that a custom chunk sizing function is restored CREATE OR REPLACE FUNCTION custom_calculate_chunk_interval( dimension_id INTEGER, dimension_coord BIGINT, chunk_target_size BIGINT ) RETURNS BIGINT LANGUAGE PLPGSQL AS $BODY$ DECLARE BEGIN RETURN -1; END $BODY$; SELECT * FROM set_adaptive_chunking('"test_schema"."two_Partitions"', '1 MB', 'custom_calculate_chunk_interval'); -- Chunk sizing func set SELECT * FROM _timescaledb_catalog.hypertable; SELECT proname, pronamespace, pronargs FROM pg_proc WHERE proname = 'custom_calculate_chunk_interval'; CREATE TRIGGER restore_trigger BEFORE INSERT ON "test_schema"."two_Partitions" FOR EACH ROW EXECUTE FUNCTION test_trigger(); -- Save the number of dependent objects so we can make sure we have the same number later SELECT count(*) as num_dependent_objects FROM pg_depend WHERE refclassid = 'pg_extension'::regclass AND refobjid = (SELECT oid FROM pg_extension WHERE extname = 'timescaledb') \gset SELECT * FROM test.show_columns('"test_schema"."two_Partitions"'); SELECT * FROM test.show_columns('_timescaledb_internal._hyper_1_1_chunk'); SELECT * FROM test.show_indexes('"test_schema"."two_Partitions"'); SELECT * FROM test.show_indexes('_timescaledb_internal._hyper_1_1_chunk'); SELECT * FROM test.show_constraints('"test_schema"."two_Partitions"'); SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_1_1_chunk'); SELECT * FROM test.show_triggers('"test_schema"."two_Partitions"'); SELECT * FROM test.show_triggers('_timescaledb_internal._hyper_1_1_chunk'); SELECT * FROM "test_schema"."two_Partitions" ORDER BY "timeCustom", device_id, series_0, series_1; SELECT * FROM _timescaledb_internal._hyper_1_1_chunk ORDER BY "timeCustom", device_id, series_0, series_1; SELECT * FROM _timescaledb_internal._hyper_1_2_chunk ORDER BY "timeCustom", device_id, series_0, series_1; -- Show all constraints SELECT * FROM _timescaledb_catalog.chunk_constraint; INSERT INTO _timescaledb_catalog.metadata VALUES ('exported_uuid', 'original_uuid', true); INSERT INTO _timescaledb_catalog.metadata VALUES ('metadata_test', 'FOO', false); \c postgres :ROLE_SUPERUSER -- We shell out to a script in order to grab the correct hostname from the -- environmental variables that originally called this psql command. Sadly -- vars passed to psql do not work in \! commands so we can't do it that way. \! utils/pg_dump_aux_dump.sh dump/pg_dump.sql \c :TEST_DBNAME SET client_min_messages = ERROR; CREATE EXTENSION timescaledb CASCADE; --create a exported uuid before restoring (mocks telemetry running before restore) INSERT INTO _timescaledb_catalog.metadata VALUES ('exported_uuid', 'new_db_uuid', true); -- disable background jobs UPDATE _timescaledb_catalog.bgw_job SET scheduled = false; RESET client_min_messages; SELECT timescaledb_pre_restore(); SHOW timescaledb.restoring; -- reconnect and check GUC value in new session \c SHOW timescaledb.restoring; \! utils/pg_dump_aux_restore.sh dump/pg_dump.sql -- Now run our post-restore function. SELECT timescaledb_post_restore(); SHOW timescaledb.restoring; -- timescaledb_post_restore restarts background worker so we have to stop them -- to make sure they dont interfere with this database being used as template below SELECT _timescaledb_functions.stop_background_workers(); --should be same as count above SELECT count(*) = :num_dependent_objects as dependent_objects_match FROM pg_depend WHERE refclassid = 'pg_extension'::regclass AND refobjid = (SELECT oid FROM pg_extension WHERE extname = 'timescaledb'); --we should have the original uuid from the backed up db set as the exported_uuid SELECT value = 'original_uuid' FROM _timescaledb_catalog.metadata WHERE key='exported_uuid'; SELECT count(*) = 1 FROM _timescaledb_catalog.metadata WHERE key LIKE 'exported%'; --we should have the original value of metadata_test SELECT * FROM _timescaledb_catalog.metadata WHERE key='metadata_test'; --main table and chunk schemas should be the same SELECT * FROM test.show_columns('"test_schema"."two_Partitions"'); SELECT * FROM test.show_columns('_timescaledb_internal._hyper_1_1_chunk'); SELECT * FROM test.show_indexes('"test_schema"."two_Partitions"'); SELECT * FROM test.show_indexes('_timescaledb_internal._hyper_1_1_chunk'); SELECT * FROM test.show_constraints('"test_schema"."two_Partitions"'); SELECT * FROM test.show_constraints('_timescaledb_internal._hyper_1_1_chunk'); SELECT * FROM test.show_triggers('"test_schema"."two_Partitions"'); SELECT * FROM test.show_triggers('_timescaledb_internal._hyper_1_1_chunk'); --data should be the same SELECT * FROM "test_schema"."two_Partitions" ORDER BY "timeCustom", device_id, series_0, series_1; SELECT * FROM _timescaledb_internal._hyper_1_1_chunk ORDER BY "timeCustom", device_id, series_0, series_1; SELECT * FROM _timescaledb_internal._hyper_1_2_chunk ORDER BY "timeCustom", device_id, series_0, series_1; SELECT * FROM _timescaledb_catalog.chunk_constraint; --Chunk sizing function should have been restored SELECT * FROM _timescaledb_catalog.hypertable; SELECT proname, pronamespace, pronargs FROM pg_proc WHERE proname = 'custom_calculate_chunk_interval'; --check simple ddl still works ALTER TABLE "test_schema"."two_Partitions" ADD COLUMN series_3 integer; INSERT INTO "test_schema"."two_Partitions"("timeCustom", device_id, series_0, series_1, series_3) VALUES (1357894000000000000, 'dev5', 1.5, 2, 4); SELECT * FROM ONLY "test_schema"."two_Partitions"; --query for the extension tables/sequences that will not be dumped by pg_dump (should --be empty except for views and explicitly excluded tables) SELECT objid::regclass FROM pg_catalog.pg_depend WHERE refclassid = 'pg_catalog.pg_extension'::pg_catalog.regclass AND refobjid = (select oid from pg_extension where extname='timescaledb') AND deptype = 'e' AND classid='pg_catalog.pg_class'::pg_catalog.regclass AND objid NOT IN (select unnest(extconfig) from pg_extension where extname='timescaledb') ORDER BY objid::regclass::text COLLATE "C"; -- Make sure we can't run our restoring functions as a normal perm user as that would disable functionality for the whole db \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- Hides error messages in cases where error messages differ between Postgres versions create or replace function get_sqlstate(in_text TEXT) RETURNS TEXT AS $$ BEGIN BEGIN EXECUTE in_text; EXCEPTION WHEN others THEN GET STACKED DIAGNOSTICS in_text = RETURNED_SQLSTATE; END; RETURN in_text; END; $$ LANGUAGE PLPGSQL; SELECT get_sqlstate('SELECT timescaledb_pre_restore()'); SELECT get_sqlstate('SELECT timescaledb_post_restore()'); drop function get_sqlstate(TEXT); -- Check that the extension can be copied from an existing database -- without explicitly installing it. Stop background workers since we -- cannot have any backends connected to the database when cloning it. \c :TEST_DBNAME :ROLE_SUPERUSER SELECT timescaledb_pre_restore(); SELECT bgw_wait(:'TEST_DBNAME', 60, FALSE); -- Force other sessions connected to the TEST_DBNAME to be finished \c postgres :ROLE_SUPERUSER REVOKE CONNECT ON DATABASE :TEST_DBNAME FROM public; ALTER DATABASE :TEST_DBNAME allow_connections = off; SET client_min_messages TO ERROR; SELECT COUNT(pg_catalog.pg_terminate_backend(pid))>=0 FROM pg_stat_activity WHERE datname = ':TEST_DBNAME'; RESET client_min_messages; CREATE DATABASE :TEST_DBNAME_EXTRA WITH TEMPLATE :TEST_DBNAME; ALTER DATABASE :TEST_DBNAME allow_connections = on; GRANT CONNECT ON DATABASE :TEST_DBNAME TO public; -- Connect to the database and do some basic stuff to check that the -- extension works. \c :TEST_DBNAME_EXTRA :ROLE_DEFAULT_PERM_USER CREATE TABLE test_tz(time timestamptz not null, temp float8, device text); SELECT create_hypertable('test_tz', 'time', 'device', 2); SELECT id, schema_name, table_name FROM _timescaledb_catalog.hypertable; INSERT INTO test_tz VALUES('Mon Mar 20 09:17:00.936242 2017', 23.4, 'dev1'); INSERT INTO test_tz VALUES('Mon Mar 20 09:27:00.936242 2017', 22, 'dev2'); INSERT INTO test_tz VALUES('Mon Mar 20 09:28:00.936242 2017', 21.2, 'dev1'); INSERT INTO test_tz VALUES('Mon Mar 20 09:37:00.936242 2017', 30, 'dev3'); SELECT * FROM test_tz ORDER BY time; \c :TEST_DBNAME :ROLE_SUPERUSER -- make sure nobody is using it SET client_min_messages TO ERROR; REVOKE CONNECT ON DATABASE :TEST_DBNAME_EXTRA FROM public; SELECT count(pg_terminate_backend(pg_stat_activity.pid)) AS TERMINATED FROM pg_stat_activity WHERE pg_stat_activity.datname = :'TEST_DBNAME_EXTRA' AND pg_stat_activity.pid <> pg_backend_pid() \gset RESET client_min_messages; DROP DATABASE :TEST_DBNAME_EXTRA WITH (FORCE); ================================================ FILE: test/sql/pg_dump_unprivileged.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c template1 :ROLE_SUPERUSER SET client_min_messages TO ERROR; CREATE EXTENSION IF NOT EXISTS timescaledb; RESET client_min_messages; CREATE USER dump_unprivileged CREATEDB; \c template1 dump_unprivileged CREATE database dump_unprivileged; \! utils/pg_dump_unprivileged.sh \c dump_unprivileged :ROLE_SUPERUSER DROP EXTENSION timescaledb; GRANT ALL ON DATABASE dump_unprivileged TO dump_unprivileged; \c dump_unprivileged dump_unprivileged -- Create the timescale extension and table as underprivileged user CREATE EXTENSION timescaledb; CREATE TABLE t1 (a int); -- pg_dump currently fails when dumped \! utils/pg_dump_unprivileged.sh \c template1 :ROLE_SUPERUSER DROP EXTENSION timescaledb; DROP DATABASE dump_unprivileged WITH (FORCE); DROP USER dump_unprivileged; ================================================ FILE: test/sql/pg_join.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- this test suite is based on the postgres join tests -- -- the tests have been adjusted to work with hypertables -- statements that would generate an error have been commented out -- because errors don't play nicely with psql output redirection -- plan output has been disabled, because we are not interested in -- the actual plans produced but in the correctness of the results -- we need superuser because some of the tests modify statistics \c :TEST_DBNAME :ROLE_SUPERUSER \set TEST_BASE_NAME join SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') as "TEST_LOAD_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') as "TEST_QUERY_NAME", format('%s/results/%s_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_OPTIMIZED", format('%s/results/%s_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_UNOPTIMIZED" \gset SELECT format('\! diff -u --label "Unoptimized results" --label "Optimized results" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') as "DIFF_CMD" \gset set client_min_messages to warning; \ir :TEST_LOAD_NAME \set PREFIX '' \set ECHO errors -- get results with optimizations disabled \o :TEST_RESULTS_UNOPTIMIZED SET timescaledb.enable_optimizations TO false; \ir :TEST_QUERY_NAME \o -- get query results with all optimizations \o :TEST_RESULTS_OPTIMIZED SET timescaledb.enable_optimizations TO true; \ir :TEST_QUERY_NAME \o :DIFF_CMD ================================================ FILE: test/sql/plain.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Tests for plain PostgreSQL commands to ensure that they work while -- the TimescaleDB extension is loaded. This is a mix of statements -- added mostly as regression checks when bugs are discovered and -- fixed. CREATE TABLE regular_table(time timestamp, temp float8, tag text, color integer); -- Renaming indexes should work CREATE INDEX time_color_idx ON regular_table(time, color); ALTER INDEX time_color_idx RENAME TO time_color_idx2; ALTER TABLE regular_table ALTER COLUMN color TYPE bigint; SELECT * FROM test.show_columns('regular_table'); SELECT * FROM test.show_indexes('regular_table'); -- Renaming types should work CREATE TYPE rainbow AS ENUM ('red', 'orange', 'yellow', 'green', 'blue', 'purple'); ALTER TYPE rainbow RENAME TO colors; \dT+ REINDEX TABLE regular_table; \c :TEST_DBNAME :ROLE_SUPERUSER REINDEX SCHEMA public; -- Not only simple statements should work CREATE TABLE a (aa TEXT); CREATE TABLE z (b TEXT, PRIMARY KEY(aa, b)) inherits (a); ================================================ FILE: test/sql/plan_expand_hypertable.sql.in ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set PREFIX 'EXPLAIN (buffers off, costs off) ' \ir include/plan_expand_hypertable_load.sql \ir include/plan_expand_hypertable_query.sql \set ECHO errors \set TEST_BASE_NAME plan_expand_hypertable SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') as "TEST_LOAD_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') as "TEST_QUERY_NAME", format('%s/results/%s_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_OPTIMIZED", format('%s/results/%s_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_UNOPTIMIZED" \gset SELECT format('\! diff -u --label "Unoptimized result" --label "Optimized result" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') as "DIFF_CMD" \gset -- run queries with optimization on and off and diff results \set PREFIX '' \o :TEST_RESULTS_OPTIMIZED SET timescaledb.enable_optimizations TO true; \ir :TEST_QUERY_NAME \o \o :TEST_RESULTS_UNOPTIMIZED SET timescaledb.enable_optimizations TO false; \ir :TEST_QUERY_NAME \o :DIFF_CMD \set ECHO queries \set PREFIX 'EXPLAIN (buffers off, costs off)' RESET timescaledb.enable_optimizations; -- test enable_qual_propagation GUC CREATE TABLE t(time timestamptz NOT NULL); SELECT table_name FROM create_hypertable('t','time'); INSERT INTO t VALUES ('2000-01-01'), ('2010-01-01'), ('2020-01-01'); -- time constraint should be in both scans :PREFIX SELECT * FROM t t1 INNER JOIN t t2 ON t1.time = t2.time WHERE t1.time < timestamptz '2010-01-01'; SET timescaledb.enable_qual_propagation TO false; -- time constraint should only be in t1 scan :PREFIX SELECT * FROM t t1 INNER JOIN t t2 ON t1.time = t2.time WHERE t1.time < timestamptz '2010-01-01'; RESET timescaledb.enable_qual_propagation; -- test hypertable classification when hypertable is not in cache -- https://github.com/timescale/timescaledb/issues/1832 CREATE TABLE test (a int, time timestamptz NOT NULL); SELECT table_name FROM create_hypertable('public.test', 'time'); INSERT INTO test SELECT i, '2020-04-01'::date-10-i from generate_series(1,20) i; CREATE OR REPLACE FUNCTION test_f(_ts timestamptz) RETURNS SETOF test LANGUAGE SQL STABLE PARALLEL SAFE AS $f$ SELECT DISTINCT ON (a) * FROM test WHERE time >= _ts ORDER BY a, time DESC $f$; :PREFIX SELECT * FROM test_f(now()); -- create new session \c -- plan output should be identical to previous session :PREFIX SELECT * FROM test_f(now()); -- test hypertable expansion in nested function calls CREATE TABLE t1 (a int, b int NOT NULL); SELECT create_hypertable('t1', 'b', chunk_time_interval=>10); CREATE TABLE t2 (a int, b int NOT NULL); SELECT create_hypertable('t2', 'b', chunk_time_interval=>10); CREATE OR REPLACE FUNCTION f_t1(_a int, _b int) RETURNS SETOF t1 LANGUAGE SQL STABLE PARALLEL SAFE AS $function$ SELECT DISTINCT ON (a) * FROM t1 WHERE a = _a and b = _b ORDER BY a, b DESC $function$ ; CREATE OR REPLACE FUNCTION f_t2(_a int, _b int) RETURNS SETOF t2 LANGUAGE sql STABLE PARALLEL SAFE AS $function$ SELECT DISTINCT ON (j.a) j.* FROM f_t1(_a, _b) sc, t2 j WHERE j.b = _b AND j.a = _a ORDER BY j.a, j.b DESC $function$ ; CREATE OR REPLACE FUNCTION f_t1_2(_b int) RETURNS SETOF t1 LANGUAGE SQL STABLE PARALLEL SAFE AS $function$ SELECT DISTINCT ON (j.a) jt.* FROM t1 j, f_t1(j.a, _b) jt $function$; :PREFIX SELECT * FROM f_t1_2(10); :PREFIX SELECT * FROM f_t1_2(10) sc, f_t2(sc.a, 10); \set PREFIX 'EXPLAIN (buffers off, costs off, timing off, summary off, analyze)' -- test qual filtering on hypertable with integer time column CREATE TABLE metrics_int1(time int, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval=1); INSERT INTO metrics_int1 SELECT i, i::text, i FROM generate_series(3,7) i; SELECT tableoid::regclass, time FROM metrics_int1 ORDER BY time; -- no contraint on any chunk :PREFIX SELECT * FROM metrics_int1 WHERE time >= 2; :PREFIX SELECT * FROM metrics_int1 WHERE time >= 1 AND time >= 2; -- no constraint on any chunk but first chunk excluded :PREFIX SELECT * FROM metrics_int1 WHERE time >= 4; -- test all four combinations of open/closed interval :PREFIX SELECT * FROM metrics_int1 WHERE time > 4 AND time < 6; :PREFIX SELECT * FROM metrics_int1 WHERE time > 4 AND time <= 6; :PREFIX SELECT * FROM metrics_int1 WHERE time >= 4 AND time < 6; :PREFIX SELECT * FROM metrics_int1 WHERE time >= 4 AND time <= 6; -- test between :PREFIX SELECT * FROM metrics_int1 WHERE time BETWEEN 4 AND 5; -- test equality :PREFIX SELECT * FROM metrics_int1 WHERE time = 5; -- test qual filtering on hypertable with integer time column SET TIMEZONE='UTC'; CREATE TABLE metrics_tstz(time timestamptz, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval='1day'); INSERT INTO metrics_tstz SELECT '2000-01-01'::timestamptz + format('%s day',i)::interval, i::text, i FROM generate_series(2,6) i; SELECT tableoid::regclass, time FROM metrics_tstz ORDER BY time; -- no contraint on any chunk :PREFIX SELECT * FROM metrics_tstz WHERE time >= '2000-01-02'; :PREFIX SELECT * FROM metrics_tstz WHERE time >= '2000-01-01' AND time >= '2000-01-02'; -- no constraint on any chunk but first chunk excluded :PREFIX SELECT * FROM metrics_tstz WHERE time >= '2000-01-04'; -- test all four combinations of open/closed interval :PREFIX SELECT * FROM metrics_tstz WHERE time > '2000-01-04' AND time < '2000-01-06'; :PREFIX SELECT * FROM metrics_tstz WHERE time > '2000-01-04' AND time <= '2000-01-06'; :PREFIX SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time < '2000-01-06'; :PREFIX SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time <= '2000-01-06'; -- test between (between is inclusive on both ends) :PREFIX SELECT * FROM metrics_tstz WHERE time BETWEEN '2000-01-04' AND '2000-01-05'; -- test equality :PREFIX SELECT * FROM metrics_tstz WHERE time = '2000-01-05'; -- constraints on non-dimension columns remain unaffected :PREFIX SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time <= '2000-01-06' AND device = '5'; :PREFIX SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time <= '2000-01-06' AND device IS NOT NULL; -- non-constant constraints on time dimension column remain unaffected :PREFIX SELECT * FROM metrics_tstz WHERE time >= '2000-01-04' AND time <= '2000-01-06' AND time < now(); -- test hypertable with space partitioning CREATE TABLE metrics_space(time timestamptz, device text, value float) WITH (tsdb.hypertable,tsdb.partition_column='time',tsdb.chunk_interval='1day'); SELECT add_dimension('metrics_space', 'device', 4); INSERT INTO metrics_space SELECT '2000-01-01'::timestamptz + format('%s day',i)::interval, i::text, i FROM generate_series(2,6) i; -- no contraints removed due to space partitioning :PREFIX SELECT * FROM metrics_space WHERE time >= '2000-01-02'; \qecho '--TEST END--' ================================================ FILE: test/sql/plan_hashagg.sql.in ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SET max_parallel_workers_per_gather TO 0; \set PREFIX 'EXPLAIN (buffers off, costs off) ' \ir include/plan_hashagg_load.sql \ir include/plan_hashagg_query.sql \set ECHO none \set TEST_BASE_NAME plan_hashagg SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') as "TEST_LOAD_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') as "TEST_QUERY_NAME", format('%s/results/%s_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_OPTIMIZED", format('%s/results/%s_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_UNOPTIMIZED" \gset SELECT format('\! diff -u --label "Unoptimized result" --label "Optimized result" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') as "DIFF_CMD" \gset SET client_min_messages TO error; --generate the results into two different files \set PREFIX '' \o :TEST_RESULTS_OPTIMIZED SET timescaledb.enable_optimizations TO true; \ir :TEST_QUERY_NAME \o \o :TEST_RESULTS_UNOPTIMIZED SET timescaledb.enable_optimizations TO false; \ir :TEST_QUERY_NAME \o :DIFF_CMD ================================================ FILE: test/sql/plan_hypertable_inline.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- test hypertable classification when query is in an inlineable function \set PREFIX 'EXPLAIN (buffers off, costs off)' CREATE TABLE test (a int, b bigint NOT NULL); SELECT create_hypertable('public.test', 'b', chunk_time_interval=>10); INSERT INTO test SELECT i, i FROM generate_series(1, 20) i; CREATE OR REPLACE FUNCTION test_f(_ts bigint) RETURNS SETOF test LANGUAGE SQL STABLE as $f$ SELECT DISTINCT ON (a) * FROM test WHERE b >= _ts AND b <= _ts + 2 $f$; -- plans must be the same in both cases -- specifically, the first plan should not contain the parent hypertable -- as that is a sign the pruning was not done successfully :PREFIX SELECT * FROM test_f(5); :PREFIX SELECT DISTINCT ON (a) * FROM test WHERE b >= 5 AND b <= 5 + 2; -- test with FOR UPDATE CREATE OR REPLACE FUNCTION test_f(_ts bigint) RETURNS SETOF test LANGUAGE SQL STABLE as $f$ SELECT * FROM test WHERE b >= _ts AND b <= _ts + 2 FOR UPDATE $f$; -- pruning should not be done by TimescaleDb in this case -- specifically, the parent hypertable must exist in the output plan :PREFIX SELECT * FROM test_f(5); :PREFIX SELECT * FROM test WHERE b >= 5 AND b <= 5 + 2 FOR UPDATE; -- test with CTE -- these cases are just to make sure we're everything is alright with -- the way we identify hypertables to prune chunks - we abuse ctename -- for this purpose. So double-check if we're not breaking plans -- with CTEs here. CREATE OR REPLACE FUNCTION test_f(_ts bigint) RETURNS SETOF test LANGUAGE SQL STABLE as $f$ WITH ct AS MATERIALIZED ( SELECT DISTINCT ON (a) * FROM test WHERE b >= _ts AND b <= _ts + 2 ) SELECT * FROM ct $f$; :PREFIX SELECT * FROM test_f(5); :PREFIX WITH ct AS MATERIALIZED ( SELECT DISTINCT ON (a) * FROM test WHERE b >= 5 AND b <= 5 + 2 ) SELECT * FROM ct; -- CTE within CTE :PREFIX WITH ct AS MATERIALIZED ( SELECT * FROM test_f(5) ) SELECT * FROM ct; -- CTE within NO MATERIALIZED CTE :PREFIX WITH ct AS NOT MATERIALIZED ( SELECT * FROM test_f(5) ) SELECT * FROM ct; ================================================ FILE: test/sql/plan_ordered_append.sql.in ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- we run these with analyze to confirm that nodes that are not -- needed to fulfill the limit are not executed -- unfortunately this doesn't work on PostgreSQL 9.6 which lacks -- the ability to turn off analyze timing summary so we run -- them without ANALYZE on PostgreSQL 9.6, but since LATERAL plans -- are different across versions we need version specific output -- here anyway. \set TEST_BASE_NAME plan_ordered_append SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') as "TEST_LOAD_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') as "TEST_QUERY_NAME", format('%s/results/%s_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_OPTIMIZED", format('%s/results/%s_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_UNOPTIMIZED" \gset SELECT format('\! diff -u --label "Unoptimized result" --label "Optimized result" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') as "DIFF_CMD" \gset \set PREFIX 'EXPLAIN (analyze, buffers off, costs off, timing off, summary off)' \set PREFIX_NO_ANALYZE 'EXPLAIN (buffers off, costs off)' \ir :TEST_LOAD_NAME \ir :TEST_QUERY_NAME --generate the results into two different files \set ECHO errors --make output contain query results \set PREFIX '' \set PREFIX_NO_ANALYZE '' \o :TEST_RESULTS_OPTIMIZED SET timescaledb.ordered_append = 'on'; \ir :TEST_QUERY_NAME \o \o :TEST_RESULTS_UNOPTIMIZED SET timescaledb.ordered_append = 'off'; \ir :TEST_QUERY_NAME \o :DIFF_CMD ================================================ FILE: test/sql/query.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set TEST_BASE_NAME query SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') as "TEST_LOAD_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') as "TEST_QUERY_NAME", format('%s/results/%s_results_optimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_OPTIMIZED", format('%s/results/%s_results_unoptimized.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_UNOPTIMIZED" \gset SELECT format('\! diff -u --label "Unoptimized result" --label "Optimized result" %s %s', :'TEST_RESULTS_UNOPTIMIZED', :'TEST_RESULTS_OPTIMIZED') as "DIFF_CMD" \gset \set PREFIX 'EXPLAIN (buffers OFF, costs OFF)' \ir :TEST_LOAD_NAME \ir :TEST_QUERY_NAME --generate the results into two different files \set ECHO errors SET client_min_messages TO error; --make output contain query results \set PREFIX '' \o :TEST_RESULTS_OPTIMIZED SET timescaledb.enable_optimizations TO true; \ir :TEST_QUERY_NAME \o \o :TEST_RESULTS_UNOPTIMIZED SET timescaledb.enable_optimizations TO false; \ir :TEST_QUERY_NAME \o :DIFF_CMD SELECT 'Done' ================================================ FILE: test/sql/relocate_extension.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Set this variable to avoid using a hard-coded path each time query -- results are compared \set QUERY_RESULT_TEST_EQUAL_RELPATH 'include/query_result_test_equal.sql' \c postgres :ROLE_SUPERUSER DROP DATABASE :TEST_DBNAME WITH (FORCE); CREATE DATABASE :TEST_DBNAME; \c :TEST_DBNAME CREATE SCHEMA "testSchema0"; SET client_min_messages=error; CREATE EXTENSION IF NOT EXISTS timescaledb SCHEMA "testSchema0"; RESET client_min_messages; CREATE TABLE test_ts(time timestamp, temp float8, device text); CREATE TABLE test_tz(time timestamptz, temp float8, device text); CREATE TABLE test_dt(time date, temp float8, device text); SELECT "testSchema0".create_hypertable('test_ts', 'time', 'device', 2); SELECT "testSchema0".create_hypertable('test_tz', 'time', 'device', 2); SELECT "testSchema0".create_hypertable('test_dt', 'time', 'device', 2); SELECT * FROM _timescaledb_catalog.hypertable; INSERT INTO test_ts VALUES('Mon Mar 20 09:17:00.936242 2017', 23.4, 'dev1'); INSERT INTO test_ts VALUES('Mon Mar 20 09:27:00.936242 2017', 22, 'dev2'); INSERT INTO test_ts VALUES('Mon Mar 20 09:28:00.936242 2017', 21.2, 'dev1'); INSERT INTO test_ts VALUES('Mon Mar 20 09:37:00.936242 2017', 30, 'dev3'); SELECT * FROM test_ts ORDER BY time; INSERT INTO test_tz VALUES('Mon Mar 20 09:17:00.936242 2017', 23.4, 'dev1'); INSERT INTO test_tz VALUES('Mon Mar 20 09:27:00.936242 2017', 22, 'dev2'); INSERT INTO test_tz VALUES('Mon Mar 20 09:28:00.936242 2017', 21.2, 'dev1'); INSERT INTO test_tz VALUES('Mon Mar 20 09:37:00.936242 2017', 30, 'dev3'); SELECT * FROM test_tz ORDER BY time; INSERT INTO test_dt VALUES('Mon Mar 20 09:17:00.936242 2017', 23.4, 'dev1'); INSERT INTO test_dt VALUES('Mon Mar 21 09:27:00.936242 2017', 22, 'dev2'); INSERT INTO test_dt VALUES('Mon Mar 22 09:28:00.936242 2017', 21.2, 'dev1'); INSERT INTO test_dt VALUES('Mon Mar 23 09:37:00.936242 2017', 30, 'dev3'); SELECT * FROM test_dt ORDER BY time; -- testing time_bucket START SELECT AVG(temp) AS avg_tmp, "testSchema0".time_bucket('5 minutes', time, INTERVAL '1 minutes') AS ten_min FROM test_ts GROUP BY ten_min ORDER BY avg_tmp; SELECT AVG(temp) AS avg_tmp, "testSchema0".time_bucket('5 minutes', time, INTERVAL '1 minutes') AS ten_min FROM test_tz GROUP BY ten_min ORDER BY avg_tmp; SELECT AVG(temp) AS avg_tmp, "testSchema0".time_bucket('1 day', time, INTERVAL '-0.5 day') AS ten_min FROM test_dt GROUP BY ten_min ORDER BY avg_tmp; -- testing time_bucket END -- testing drop_chunks START -- show_chunks and drop_chunks output should be the same \set QUERY1 'SELECT "testSchema0".show_chunks(older_than => \'2017-03-01\'::timestamp, relation => \'test_ts\')::REGCLASS::TEXT' \set QUERY2 'SELECT "testSchema0".drop_chunks(\'test_ts\', \'2017-03-01\'::timestamp)::TEXT' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT * FROM test_ts ORDER BY time; \set QUERY1 'SELECT "testSchema0".show_chunks(older_than => interval \'1 minutes\', relation => \'test_tz\')::REGCLASS::TEXT' \set QUERY2 'SELECT "testSchema0".drop_chunks(\'test_tz\', interval \'1 minutes\')::TEXT' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT * FROM test_tz ORDER BY time; \set QUERY1 'SELECT "testSchema0".show_chunks(older_than => interval \'1 minutes\', relation => \'test_dt\')::REGCLASS::TEXT' \set QUERY2 'SELECT "testSchema0".drop_chunks(\'test_dt\', interval \'1 minutes\')::TEXT' \set ECHO errors \ir :QUERY_RESULT_TEST_EQUAL_RELPATH \set ECHO all SELECT * FROM test_dt ORDER BY time; -- testing drop_chunks END -- testing hypertable_detailed_size START SELECT * FROM "testSchema0".hypertable_detailed_size('test_ts'); -- testing hypertable_detailed_size END SELECT * FROM "testSchema0".hypertable_index_size('test_ts_time_idx'); SELECT * FROM "testSchema0".hypertable_index_size('test_ts_device_time_idx'); CREATE SCHEMA "testSchema"; \set ON_ERROR_STOP 0 ALTER EXTENSION timescaledb SET SCHEMA "testSchema"; \set ON_ERROR_STOP 1 ================================================ FILE: test/sql/reloptions.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE reloptions_test(time integer, temp float8, color integer) WITH (fillfactor=75, autovacuum_vacuum_threshold=100); SELECT create_hypertable('reloptions_test', 'time', chunk_time_interval => 3); INSERT INTO reloptions_test VALUES (4, 24.3, 1), (9, 13.3, 2); -- Show that reloptions are inherited by chunks SELECT relname, reloptions FROM pg_class WHERE relname ~ '^_hyper.*' AND relkind = 'r'; -- Alter reloptions. We support multiple options for the ALTER TABLE ALTER TABLE reloptions_test SET (fillfactor=80, parallel_workers=8); ALTER TABLE reloptions_test SET (fillfactor=80), SET (parallel_workers=8); SELECT relname, reloptions FROM pg_class WHERE relname ~ '^_hyper.*' AND relkind = 'r'; ALTER TABLE reloptions_test RESET (fillfactor); SELECT relname, reloptions FROM pg_class WHERE relname ~ '^_hyper.*' AND relkind = 'r'; -- Test reloptions on a regular table CREATE TABLE reloptions_test2(time integer, temp float8, color integer); ALTER TABLE reloptions_test2 SET (fillfactor=80, parallel_workers=8); ALTER TABLE reloptions_test2 SET (fillfactor=80), SET (parallel_workers=8); DROP TABLE reloptions_test2; ================================================ FILE: test/sql/repair.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- We are testing different repair functions here to make sure that -- they work as expected. \c :TEST_DBNAME :ROLE_SUPERUSER \set TMP_USER :TEST_DBNAME _wizard CREATE USER :TMP_USER; CREATE USER "Random L User"; CREATE TABLE test_table_1(time timestamptz not null, temp float); SELECT create_hypertable('test_table_1', by_range('time', '1 day'::interval)); INSERT INTO test_table_1(time,temp) SELECT time, 100 * random() FROM generate_series( '2000-01-01'::timestamptz, '2000-01-05'::timestamptz, '1min'::interval ) time; CREATE TABLE test_table_2(time timestamptz not null, temp float); SELECT create_hypertable('test_table_2', by_range('time', '1 day'::interval)); INSERT INTO test_table_2(time,temp) SELECT time, 100 * random() FROM generate_series( '2000-01-01'::timestamptz, '2000-01-05'::timestamptz, '1min'::interval ) time; GRANT ALL ON test_table_1 TO :TMP_USER; GRANT ALL ON test_table_2 TO :TMP_USER; GRANT SELECT, INSERT ON test_table_1 TO "Random L User"; GRANT INSERT ON test_table_2 TO "Random L User"; -- Break the relacl of the table by deleting users DELETE FROM pg_authid WHERE rolname IN (:'TMP_USER', 'Random L User'); CREATE TABLE saved (LIKE pg_class); INSERT INTO saved SELECT * FROM pg_class; CALL _timescaledb_functions.repair_relation_acls(); -- The only thing we should see here are the relations we broke and -- the privileges we added for that user. No other relations should be -- touched. WITH lhs AS (SELECT oid, aclexplode(relacl) FROM pg_class), rhs AS (SELECT oid, aclexplode(relacl) FROM saved) SELECT rhs.oid::regclass FROM lhs FULL OUTER JOIN rhs ON row(lhs) = row(rhs) WHERE lhs.oid IS NULL AND rhs.oid IS NOT NULL GROUP BY rhs.oid; DROP TABLE saved; DROP TABLE test_table_1; DROP TABLE test_table_2; ================================================ FILE: test/sql/rowsecurity.sql.in ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- -- Test of Row-level security feature -- -- Clean up in case a prior regression run failed \c :TEST_DBNAME :ROLE_SUPERUSER \set ON_ERROR_STOP 0 \set VERBOSITY default SET timescaledb.enable_constraint_exclusion TO off; -- Suppress NOTICE messages when users/groups don't exist SET client_min_messages TO 'warning'; DROP USER IF EXISTS regress_rls_alice; DROP USER IF EXISTS regress_rls_bob; DROP USER IF EXISTS regress_rls_carol; DROP USER IF EXISTS regress_rls_dave; DROP USER IF EXISTS regress_rls_exempt_user; DROP ROLE IF EXISTS regress_rls_group1; DROP ROLE IF EXISTS regress_rls_group2; DROP SCHEMA IF EXISTS regress_rls_schema CASCADE; RESET client_min_messages; -- initial setup CREATE USER regress_rls_alice NOLOGIN; CREATE USER regress_rls_bob NOLOGIN; CREATE USER regress_rls_carol NOLOGIN; CREATE USER regress_rls_dave NOLOGIN; CREATE USER regress_rls_exempt_user BYPASSRLS NOLOGIN; CREATE ROLE regress_rls_group1 NOLOGIN; CREATE ROLE regress_rls_group2 NOLOGIN; GRANT regress_rls_group1 TO regress_rls_bob; GRANT regress_rls_group2 TO regress_rls_carol; CREATE SCHEMA regress_rls_schema; GRANT ALL ON SCHEMA regress_rls_schema to public; SET search_path = regress_rls_schema; -- setup of malicious function CREATE OR REPLACE FUNCTION f_leak(text) RETURNS bool COST 0.0000001 LANGUAGE plpgsql AS 'BEGIN RAISE NOTICE ''f_leak => %'', $1; RETURN true; END'; GRANT EXECUTE ON FUNCTION f_leak(text) TO public; -- BASIC Row-Level Security Scenario SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE uaccount ( pguser name primary key, seclv int ); GRANT SELECT ON uaccount TO public; INSERT INTO uaccount VALUES ('regress_rls_alice', 99), ('regress_rls_bob', 1), ('regress_rls_carol', 2), ('regress_rls_dave', 3); CREATE TABLE category ( cid int primary key, cname text ); GRANT ALL ON category TO public; INSERT INTO category VALUES (11, 'novel'), (22, 'science fiction'), (33, 'technology'), (44, 'manga'); CREATE TABLE document ( did int primary key, cid int references category(cid), dlevel int not null, dauthor name, dtitle text ); GRANT ALL ON document TO public; SELECT public.create_hypertable('document', 'did', chunk_time_interval=>2); INSERT INTO document VALUES ( 1, 11, 1, 'regress_rls_bob', 'my first novel'), ( 2, 11, 2, 'regress_rls_bob', 'my second novel'), ( 3, 22, 2, 'regress_rls_bob', 'my science fiction'), ( 4, 44, 1, 'regress_rls_bob', 'my first manga'), ( 5, 44, 2, 'regress_rls_bob', 'my second manga'), ( 6, 22, 1, 'regress_rls_carol', 'great science fiction'), ( 7, 33, 2, 'regress_rls_carol', 'great technology book'), ( 8, 44, 1, 'regress_rls_carol', 'great manga'), ( 9, 22, 1, 'regress_rls_dave', 'awesome science fiction'), (10, 33, 2, 'regress_rls_dave', 'awesome technology book'); ALTER TABLE document ENABLE ROW LEVEL SECURITY; -- user's security level must be higher than or equal to document's CREATE POLICY p1 ON document AS PERMISSIVE USING (dlevel <= (SELECT seclv FROM uaccount WHERE pguser = current_user)); -- try to create a policy of bogus type CREATE POLICY p1 ON document AS UGLY USING (dlevel <= (SELECT seclv FROM uaccount WHERE pguser = current_user)); -- but Dave isn't allowed to anything at cid 50 or above -- this is to make sure that we sort the policies by name first -- when applying WITH CHECK, a later INSERT by Dave should fail due -- to p1r first CREATE POLICY p2r ON document AS RESTRICTIVE TO regress_rls_dave USING (cid <> 44 AND cid < 50); -- and Dave isn't allowed to see manga documents CREATE POLICY p1r ON document AS RESTRICTIVE TO regress_rls_dave USING (cid <> 44); \dp \d document SELECT * FROM pg_policies WHERE schemaname = 'regress_rls_schema' AND tablename = 'document' ORDER BY policyname; -- viewpoint from regress_rls_bob SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER BY did; -- try a sampled version SELECT * FROM document TABLESAMPLE BERNOULLI(50) REPEATABLE(0) WHERE f_leak(dtitle) ORDER BY did; -- viewpoint from regress_rls_carol SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER BY did; -- try a sampled version SELECT * FROM document TABLESAMPLE BERNOULLI(50) REPEATABLE(0) WHERE f_leak(dtitle) ORDER BY did; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document WHERE f_leak(dtitle); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle); -- viewpoint from regress_rls_dave SET SESSION AUTHORIZATION regress_rls_dave; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER BY did; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document WHERE f_leak(dtitle); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle); -- 44 would technically fail for both p2r and p1r, but we should get an error -- back from p1r for this because it sorts first INSERT INTO document VALUES (100, 44, 1, 'regress_rls_dave', 'testing sorting of policies'); -- fail -- Just to see a p2r error INSERT INTO document VALUES (100, 55, 1, 'regress_rls_dave', 'testing sorting of policies'); -- fail -- only owner can change policies ALTER POLICY p1 ON document USING (true); --fail DROP POLICY p1 ON document; --fail SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY p1 ON document USING (dauthor = current_user); -- viewpoint from regress_rls_bob again SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER by did; -- viewpoint from rls_regres_carol again SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM document WHERE f_leak(dtitle) ORDER BY did; SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle) ORDER by did; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document WHERE f_leak(dtitle); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM document NATURAL JOIN category WHERE f_leak(dtitle); -- interaction of FK/PK constraints SET SESSION AUTHORIZATION regress_rls_alice; CREATE POLICY p2 ON category USING (CASE WHEN current_user = 'regress_rls_bob' THEN cid IN (11, 33) WHEN current_user = 'regress_rls_carol' THEN cid IN (22, 44) ELSE false END); ALTER TABLE category ENABLE ROW LEVEL SECURITY; -- cannot delete PK referenced by invisible FK SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM document d FULL OUTER JOIN category c on d.cid = c.cid ORDER BY d.did, c.cid; \set VERBOSITY sqlstate DELETE FROM category WHERE cid = 33; -- fails with FK violation \set VERBOSITY default -- can insert FK referencing invisible PK SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM document d FULL OUTER JOIN category c on d.cid = c.cid ORDER BY d.did, c.cid; INSERT INTO document VALUES (11, 33, 1, current_user, 'hoge'); -- UNIQUE or PRIMARY KEY constraint violation DOES reveal presence of row SET SESSION AUTHORIZATION regress_rls_bob; INSERT INTO document VALUES (8, 44, 1, 'regress_rls_bob', 'my third manga'); -- Must fail with unique violation, revealing presence of did we can't see SELECT * FROM document WHERE did = 8; -- and confirm we can't see it -- RLS policies are checked before constraints INSERT INTO document VALUES (8, 44, 1, 'regress_rls_carol', 'my third manga'); -- Should fail with RLS check violation, not duplicate key violation UPDATE document SET did = 8, dauthor = 'regress_rls_carol' WHERE did = 5; -- Should fail with RLS check violation, not duplicate key violation -- database superuser does bypass RLS policy when enabled RESET SESSION AUTHORIZATION; SET row_security TO ON; SELECT * FROM document; SELECT * FROM category; -- database superuser does bypass RLS policy when disabled RESET SESSION AUTHORIZATION; SET row_security TO OFF; SELECT * FROM document; SELECT * FROM category; -- database non-superuser with bypass privilege can bypass RLS policy when disabled SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; SELECT * FROM document; SELECT * FROM category; -- RLS policy does not apply to table owner when RLS enabled. SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; SELECT * FROM document; SELECT * FROM category; -- RLS policy does not apply to table owner when RLS disabled. SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO OFF; SELECT * FROM document; SELECT * FROM category; -- -- Table inheritance and RLS policy -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; CREATE TABLE t1 (a int, junk1 text, b text); ALTER TABLE t1 DROP COLUMN junk1; -- just a disturbing factor GRANT ALL ON t1 TO public; COPY t1 FROM stdin; 1 aba 2 bbb 3 ccc 4 dad \. CREATE TABLE t2 (c float) INHERITS (t1); GRANT ALL ON t2 TO public; COPY t2 FROM stdin; 1 abc 1.1 2 bcd 2.2 3 cde 3.3 4 def 4.4 \. CREATE TABLE t3 (c text, b text, a int); ALTER TABLE t3 INHERIT t1; GRANT ALL ON t3 TO public; COPY t3(a,b,c) FROM stdin; 1 xxx X 2 yyy Y 3 zzz Z \. CREATE POLICY p1 ON t1 FOR ALL TO PUBLIC USING (a % 2 = 0); -- be even number CREATE POLICY p2 ON t2 FOR ALL TO PUBLIC USING (a % 2 = 1); -- be odd number ALTER TABLE t1 ENABLE ROW LEVEL SECURITY; ALTER TABLE t2 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM t1; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; SELECT * FROM t1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b); -- reference to system column SELECT ctid, * FROM t1; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT *, t1 FROM t1; -- reference to whole-row reference SELECT *, t1 FROM t1; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT *, t1 FROM t1; -- for share/update lock SELECT * FROM t1 FOR SHARE; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 FOR SHARE; SELECT * FROM t1 WHERE f_leak(b) FOR SHARE; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b) FOR SHARE; -- union all query SELECT a, b, ctid FROM t2 UNION ALL SELECT a, b, ctid FROM t3; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT a, b, ctid FROM t2 UNION ALL SELECT a, b, ctid FROM t3; -- superuser is allowed to bypass RLS checks RESET SESSION AUTHORIZATION; SET row_security TO OFF; SELECT * FROM t1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b); -- non-superuser with bypass privilege can bypass RLS policy when disabled SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; SELECT * FROM t1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b); -- -- Hyper Tables -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE hyper_document ( did int, cid int, dlevel int not null, dauthor name, dtitle text ); GRANT ALL ON hyper_document TO public; SELECT public.create_hypertable('hyper_document', 'did', chunk_time_interval=>2); INSERT INTO hyper_document VALUES ( 1, 11, 1, 'regress_rls_bob', 'my first novel'), ( 2, 11, 2, 'regress_rls_bob', 'my second novel'), ( 3, 99, 2, 'regress_rls_bob', 'my science textbook'), ( 4, 55, 1, 'regress_rls_bob', 'my first satire'), ( 5, 99, 2, 'regress_rls_bob', 'my history book'), ( 6, 11, 1, 'regress_rls_carol', 'great science fiction'), ( 7, 99, 2, 'regress_rls_carol', 'great technology book'), ( 8, 55, 2, 'regress_rls_carol', 'great satire'), ( 9, 11, 1, 'regress_rls_dave', 'awesome science fiction'), (10, 99, 2, 'regress_rls_dave', 'awesome technology book'); ALTER TABLE hyper_document ENABLE ROW LEVEL SECURITY; GRANT ALL ON _timescaledb_internal._hyper_2_9_chunk TO public; -- Create policy on parent -- user's security level must be higher than or equal to document's CREATE POLICY pp1 ON hyper_document AS PERMISSIVE USING (dlevel <= (SELECT seclv FROM uaccount WHERE pguser = current_user)); -- Dave is only allowed to see cid < 55 CREATE POLICY pp1r ON hyper_document AS RESTRICTIVE TO regress_rls_dave USING (cid < 55); \d+ hyper_document SELECT * FROM pg_policies WHERE schemaname = 'regress_rls_schema' AND tablename like '%hyper_document%' ORDER BY policyname; -- viewpoint from regress_rls_bob SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); -- viewpoint from regress_rls_carol SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); -- viewpoint from regress_rls_dave SET SESSION AUTHORIZATION regress_rls_dave; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); -- pp1 ERROR INSERT INTO hyper_document VALUES (1, 11, 5, 'regress_rls_dave', 'testing pp1'); -- fail -- pp1r ERROR INSERT INTO hyper_document VALUES (1, 99, 1, 'regress_rls_dave', 'testing pp1r'); -- fail -- Show that RLS policy does not apply for direct inserts to children -- This should fail with RLS POLICY pp1r violation. INSERT INTO hyper_document VALUES (1, 55, 1, 'regress_rls_dave', 'testing RLS with hypertables'); -- fail -- But this should succeed. INSERT INTO _timescaledb_internal._hyper_2_9_chunk VALUES (1, 55, 1, 'regress_rls_dave', 'testing RLS with hypertables'); -- success -- We still cannot see the row using the parent SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; -- But we can if we look directly SELECT * FROM _timescaledb_internal._hyper_2_9_chunk WHERE f_leak(dtitle) ORDER BY did, cid; -- Turn on RLS and create policy on child to show RLS is checked before constraints SET SESSION AUTHORIZATION regress_rls_alice; ALTER TABLE _timescaledb_internal._hyper_2_9_chunk ENABLE ROW LEVEL SECURITY; CREATE POLICY pp3 ON _timescaledb_internal._hyper_2_9_chunk AS RESTRICTIVE USING (cid < 55); -- This should fail with RLS violation now. SET SESSION AUTHORIZATION regress_rls_dave; INSERT INTO _timescaledb_internal._hyper_2_9_chunk VALUES (1, 55, 1, 'regress_rls_dave', 'testing RLS with hypertables - round 2'); -- fail -- And now we cannot see directly into the partition either, due to RLS SELECT * FROM _timescaledb_internal._hyper_2_9_chunk WHERE f_leak(dtitle) ORDER BY did, cid; -- The parent looks same as before -- viewpoint from regress_rls_dave SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); -- viewpoint from regress_rls_carol SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); -- only owner can change policies ALTER POLICY pp1 ON hyper_document USING (true); --fail DROP POLICY pp1 ON hyper_document; --fail SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY pp1 ON hyper_document USING (dauthor = current_user); -- viewpoint from regress_rls_bob again SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; -- viewpoint from rls_regres_carol again SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM hyper_document WHERE f_leak(dtitle) ORDER BY did, cid; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM hyper_document WHERE f_leak(dtitle); -- database superuser does bypass RLS policy when enabled RESET SESSION AUTHORIZATION; SET row_security TO ON; SELECT * FROM hyper_document ORDER BY did, cid; SELECT * FROM _timescaledb_internal._hyper_2_9_chunk ORDER BY did, cid; -- database non-superuser with bypass privilege can bypass RLS policy when disabled SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; SELECT * FROM hyper_document ORDER BY did, cid; SELECT * FROM _timescaledb_internal._hyper_2_9_chunk ORDER BY did, cid; -- RLS policy does not apply to table owner when RLS enabled. SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; SELECT * FROM hyper_document ORDER BY did, cid; SELECT * FROM _timescaledb_internal._hyper_2_9_chunk ORDER BY did, cid; -- When RLS disabled, other users get ERROR. SET SESSION AUTHORIZATION regress_rls_dave; SET row_security TO OFF; SELECT * FROM hyper_document ORDER BY did, cid; SELECT * FROM _timescaledb_internal._hyper_2_9_chunk ORDER BY did, cid; -- Check behavior with a policy that uses a SubPlan not an InitPlan. SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; CREATE POLICY pp3 ON hyper_document AS RESTRICTIVE USING ((SELECT dlevel <= seclv FROM uaccount WHERE pguser = current_user)); SET SESSION AUTHORIZATION regress_rls_carol; INSERT INTO hyper_document VALUES (100, 11, 5, 'regress_rls_carol', 'testing pp3'); -- fail ----- Dependencies ----- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security TO ON; CREATE TABLE dependee (x integer, y integer); SELECT public.create_hypertable('dependee', 'x', chunk_time_interval=>2); CREATE TABLE dependent (x integer, y integer); SELECT public.create_hypertable('dependent', 'x', chunk_time_interval=>2); CREATE POLICY d1 ON dependent FOR ALL TO PUBLIC USING (x = (SELECT d.x FROM dependee d WHERE d.y = y)); DROP TABLE dependee; -- Should fail without CASCADE due to dependency on row security qual? DROP TABLE dependee CASCADE; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM dependent; -- After drop, should be unqualified ----- RECURSION ---- -- -- Simple recursion -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE rec1 (x integer, y integer); SELECT public.create_hypertable('rec1', 'x', chunk_time_interval=>2); CREATE POLICY r1 ON rec1 USING (x = (SELECT r.x FROM rec1 r WHERE y = r.y)); ALTER TABLE rec1 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rec1; -- fail, direct recursion -- -- Mutual recursion -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE rec2 (a integer, b integer); SELECT public.create_hypertable('rec2', 'x', chunk_time_interval=>2); ALTER POLICY r1 ON rec1 USING (x = (SELECT a FROM rec2 WHERE b = y)); CREATE POLICY r2 ON rec2 USING (a = (SELECT x FROM rec1 WHERE y = b)); ALTER TABLE rec2 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rec1; -- fail, mutual recursion -- -- Mutual recursion via views -- SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW rec1v AS SELECT * FROM rec1; CREATE VIEW rec2v AS SELECT * FROM rec2; SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY r1 ON rec1 USING (x = (SELECT a FROM rec2v WHERE b = y)); ALTER POLICY r2 ON rec2 USING (a = (SELECT x FROM rec1v WHERE y = b)); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rec1; -- fail, mutual recursion via views -- -- Mutual recursion via .s.b views -- SET SESSION AUTHORIZATION regress_rls_bob; \set VERBOSITY terse \\ -- suppress cascade details DROP VIEW rec1v, rec2v CASCADE; \set VERBOSITY default CREATE VIEW rec1v WITH (security_barrier) AS SELECT * FROM rec1; CREATE VIEW rec2v WITH (security_barrier) AS SELECT * FROM rec2; SET SESSION AUTHORIZATION regress_rls_alice; CREATE POLICY r1 ON rec1 USING (x = (SELECT a FROM rec2v WHERE b = y)); CREATE POLICY r2 ON rec2 USING (a = (SELECT x FROM rec1v WHERE y = b)); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rec1; -- fail, mutual recursion via s.b. views -- -- recursive RLS and VIEWs in policy -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE s1 (a int, b text); SELECT public.create_hypertable('s1', 'a', chunk_time_interval=>2); INSERT INTO s1 (SELECT x, md5(x::text) FROM generate_series(-10,10) x); CREATE TABLE s2 (x int, y text); SELECT public.create_hypertable('s2', 'x', chunk_time_interval=>2); INSERT INTO s2 (SELECT x, md5(x::text) FROM generate_series(-6,6) x); GRANT SELECT ON s1, s2 TO regress_rls_bob; CREATE POLICY p1 ON s1 USING (a in (select x from s2 where y like '%2f%')); CREATE POLICY p2 ON s2 USING (x in (select a from s1 where b like '%22%')); CREATE POLICY p3 ON s1 FOR INSERT WITH CHECK (a = (SELECT a FROM s1)); ALTER TABLE s1 ENABLE ROW LEVEL SECURITY; ALTER TABLE s2 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW v2 AS SELECT * FROM s2 WHERE y like '%af%'; SELECT * FROM s1 WHERE f_leak(b); -- fail (infinite recursion) INSERT INTO s1 VALUES (1, 'foo'); -- fail (infinite recursion) SET SESSION AUTHORIZATION regress_rls_alice; DROP POLICY p3 on s1; ALTER POLICY p2 ON s2 USING (x % 2 = 0); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM s1 WHERE f_leak(b); -- OK EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM only s1 WHERE f_leak(b); SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY p1 ON s1 USING (a in (select x from v2)); -- using VIEW in RLS policy SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM s1 WHERE f_leak(b); -- OK EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM s1 WHERE f_leak(b); SELECT (SELECT x FROM s1 LIMIT 1) xx, * FROM s2 WHERE y like '%28%'; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT (SELECT x FROM s1 LIMIT 1) xx, * FROM s2 WHERE y like '%28%'; SET SESSION AUTHORIZATION regress_rls_alice; ALTER POLICY p2 ON s2 USING (x in (select a from s1 where b like '%d2%')); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM s1 WHERE f_leak(b); -- fail (infinite recursion via view) -- prepared statement with regress_rls_alice privilege PREPARE p1(int) AS SELECT * FROM t1 WHERE a <= $1; EXECUTE p1(2); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE p1(2); -- superuser is allowed to bypass RLS checks RESET SESSION AUTHORIZATION; SET row_security TO OFF; SELECT * FROM t1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1 WHERE f_leak(b); -- plan cache should be invalidated EXECUTE p1(2); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE p1(2); PREPARE p2(int) AS SELECT * FROM t1 WHERE a = $1; EXECUTE p2(2); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE p2(2); -- also, case when privilege switch from superuser SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; EXECUTE p2(2); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE p2(2); -- -- UPDATE / DELETE and Row-level security -- SET SESSION AUTHORIZATION regress_rls_bob; EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t1 SET b = b || b WHERE f_leak(b); UPDATE t1 SET b = b || b WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE only t1 SET b = b || '_updt' WHERE f_leak(b); UPDATE only t1 SET b = b || '_updt' WHERE f_leak(b); -- returning clause with system column UPDATE only t1 SET b = b WHERE f_leak(b) RETURNING ctid, *, t1; UPDATE t1 SET b = b WHERE f_leak(b) RETURNING *; UPDATE t1 SET b = b WHERE f_leak(b) RETURNING ctid, *, t1; -- updates with from clause EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t2 SET b=t2.b FROM t3 WHERE t2.a = 3 and t3.a = 2 AND f_leak(t2.b) AND f_leak(t3.b); UPDATE t2 SET b=t2.b FROM t3 WHERE t2.a = 3 and t3.a = 2 AND f_leak(t2.b) AND f_leak(t3.b); EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t1 SET b=t1.b FROM t2 WHERE t1.a = 3 and t2.a = 3 AND f_leak(t1.b) AND f_leak(t2.b); UPDATE t1 SET b=t1.b FROM t2 WHERE t1.a = 3 and t2.a = 3 AND f_leak(t1.b) AND f_leak(t2.b); EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t2 SET b=t2.b FROM t1 WHERE t1.a = 3 and t2.a = 3 AND f_leak(t1.b) AND f_leak(t2.b); UPDATE t2 SET b=t2.b FROM t1 WHERE t1.a = 3 and t2.a = 3 AND f_leak(t1.b) AND f_leak(t2.b); -- updates with from clause self join EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t2 t2_1 SET b = t2_2.b FROM t2 t2_2 WHERE t2_1.a = 3 AND t2_2.a = t2_1.a AND t2_2.b = t2_1.b AND f_leak(t2_1.b) AND f_leak(t2_2.b) RETURNING *, t2_1, t2_2; UPDATE t2 t2_1 SET b = t2_2.b FROM t2 t2_2 WHERE t2_1.a = 3 AND t2_2.a = t2_1.a AND t2_2.b = t2_1.b AND f_leak(t2_1.b) AND f_leak(t2_2.b) RETURNING *, t2_1, t2_2; EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE t1 t1_1 SET b = t1_2.b FROM t1 t1_2 WHERE t1_1.a = 4 AND t1_2.a = t1_1.a AND t1_2.b = t1_1.b AND f_leak(t1_1.b) AND f_leak(t1_2.b) RETURNING *, t1_1, t1_2; UPDATE t1 t1_1 SET b = t1_2.b FROM t1 t1_2 WHERE t1_1.a = 4 AND t1_2.a = t1_1.a AND t1_2.b = t1_1.b AND f_leak(t1_1.b) AND f_leak(t1_2.b) RETURNING *, t1_1, t1_2; RESET SESSION AUTHORIZATION; SET row_security TO OFF; SELECT * FROM t1 ORDER BY a,b; SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; EXPLAIN (BUFFERS OFF, COSTS OFF) DELETE FROM only t1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) DELETE FROM t1 WHERE f_leak(b); DELETE FROM only t1 WHERE f_leak(b) RETURNING ctid, *, t1; DELETE FROM t1 WHERE f_leak(b) RETURNING ctid, *, t1; -- -- S.b. view on top of Row-level security -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE b1 (a int, b text); SELECT public.create_hypertable('b1', 'a', chunk_time_interval=>2); INSERT INTO b1 (SELECT x, md5(x::text) FROM generate_series(-10,10) x); CREATE POLICY p1 ON b1 USING (a % 2 = 0); ALTER TABLE b1 ENABLE ROW LEVEL SECURITY; GRANT ALL ON b1 TO regress_rls_bob; SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW bv1 WITH (security_barrier) AS SELECT * FROM b1 WHERE a > 0 WITH CHECK OPTION; GRANT ALL ON bv1 TO regress_rls_carol; SET SESSION AUTHORIZATION regress_rls_carol; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM bv1 WHERE f_leak(b); SELECT * FROM bv1 WHERE f_leak(b); INSERT INTO bv1 VALUES (-1, 'xxx'); -- should fail view WCO INSERT INTO bv1 VALUES (11, 'xxx'); -- should fail RLS check INSERT INTO bv1 VALUES (12, 'xxx'); -- ok EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE bv1 SET b = 'yyy' WHERE a = 4 AND f_leak(b); UPDATE bv1 SET b = 'yyy' WHERE a = 4 AND f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) DELETE FROM bv1 WHERE a = 6 AND f_leak(b); DELETE FROM bv1 WHERE a = 6 AND f_leak(b); SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM b1; -- -- INSERT ... ON CONFLICT DO UPDATE and Row-level security -- SET SESSION AUTHORIZATION regress_rls_alice; DROP POLICY p1 ON document; DROP POLICY p1r ON document; CREATE POLICY p1 ON document FOR SELECT USING (true); CREATE POLICY p2 ON document FOR INSERT WITH CHECK (dauthor = current_user); CREATE POLICY p3 ON document FOR UPDATE USING (cid = (SELECT cid from category WHERE cname = 'novel')) WITH CHECK (dauthor = current_user); SET SESSION AUTHORIZATION regress_rls_bob; -- Exists... SELECT * FROM document WHERE did = 2; -- ...so violates actual WITH CHECK OPTION within UPDATE (not INSERT, since -- alternative UPDATE path happens to be taken): INSERT INTO document VALUES (2, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_carol', 'my first novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, dauthor = EXCLUDED.dauthor; -- Violates USING qual for UPDATE policy p3. -- -- UPDATE path is taken, but UPDATE fails purely because *existing* row to be -- updated is not a "novel"/cid 11 (row is not leaked, even though we have -- SELECT privileges sufficient to see the row in this instance): INSERT INTO document VALUES (33, 22, 1, 'regress_rls_bob', 'okay science fiction'); -- preparation for next statement INSERT INTO document VALUES (33, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'Some novel, replaces sci-fi') -- takes UPDATE path ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle; -- Fine (we UPDATE, since INSERT WCOs and UPDATE security barrier quals + WCOs -- not violated): INSERT INTO document VALUES (2, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'my first novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; -- Fine (we INSERT, so "cid = 33" ("technology") isn't evaluated): INSERT INTO document VALUES (78, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'some technology novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33 RETURNING *; -- Fine (same query, but we UPDATE, so "cid = 33", ("technology") is not the -- case in respect of *existing* tuple): INSERT INTO document VALUES (78, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'some technology novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33 RETURNING *; -- Same query a third time, but now fails due to existing tuple finally not -- passing quals: INSERT INTO document VALUES (78, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'some technology novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33 RETURNING *; -- Don't fail just because INSERT doesn't satisfy WITH CHECK option that -- originated as a barrier/USING() qual from the UPDATE. Note that the UPDATE -- path *isn't* taken, and so UPDATE-related policy does not apply: INSERT INTO document VALUES (79, (SELECT cid from category WHERE cname = 'technology'), 1, 'regress_rls_bob', 'technology book, can only insert') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; -- But this time, the same statement fails, because the UPDATE path is taken, -- and updating the row just inserted falls afoul of security barrier qual -- (enforced as WCO) -- what we might have updated target tuple to is -- irrelevant, in fact. INSERT INTO document VALUES (79, (SELECT cid from category WHERE cname = 'technology'), 1, 'regress_rls_bob', 'technology book, can only insert') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; -- Test default USING qual enforced as WCO SET SESSION AUTHORIZATION regress_rls_alice; DROP POLICY p1 ON document; DROP POLICY p2 ON document; DROP POLICY p3 ON document; CREATE POLICY p3_with_default ON document FOR UPDATE USING (cid = (SELECT cid from category WHERE cname = 'novel')); SET SESSION AUTHORIZATION regress_rls_bob; -- Just because WCO-style enforcement of USING quals occurs with -- existing/target tuple does not mean that the implementation can be allowed -- to fail to also enforce this qual against the final tuple appended to -- relation (since in the absence of an explicit WCO, this is also interpreted -- as an UPDATE/ALL WCO in general). -- -- UPDATE path is taken here (fails due to existing tuple). Note that this is -- not reported as a "USING expression", because it's an RLS UPDATE check that originated as -- a USING qual for the purposes of RLS in general, as opposed to an explicit -- USING qual that is ordinarily a security barrier. We leave it up to the -- UPDATE to make this fail: INSERT INTO document VALUES (79, (SELECT cid from category WHERE cname = 'technology'), 1, 'regress_rls_bob', 'technology book, can only insert') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle RETURNING *; -- UPDATE path is taken here. Existing tuple passes, since it's cid -- corresponds to "novel", but default USING qual is enforced against -- post-UPDATE tuple too (as always when updating with a policy that lacks an -- explicit WCO), and so this fails: INSERT INTO document VALUES (2, (SELECT cid from category WHERE cname = 'technology'), 1, 'regress_rls_bob', 'my first novel') ON CONFLICT (did) DO UPDATE SET cid = EXCLUDED.cid, dtitle = EXCLUDED.dtitle RETURNING *; SET SESSION AUTHORIZATION regress_rls_alice; DROP POLICY p3_with_default ON document; -- -- Test ALL policies with ON CONFLICT DO UPDATE (much the same as existing UPDATE -- tests) -- CREATE POLICY p3_with_all ON document FOR ALL USING (cid = (SELECT cid from category WHERE cname = 'novel')) WITH CHECK (dauthor = current_user); SET SESSION AUTHORIZATION regress_rls_bob; -- Fails, since ALL WCO is enforced in insert path: INSERT INTO document VALUES (80, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_carol', 'my first novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle, cid = 33; -- Fails, since ALL policy USING qual is enforced (existing, target tuple is in -- violation, since it has the "manga" cid): INSERT INTO document VALUES (4, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'my first novel') ON CONFLICT (did) DO UPDATE SET dtitle = EXCLUDED.dtitle; -- Fails, since ALL WCO are enforced: INSERT INTO document VALUES (1, (SELECT cid from category WHERE cname = 'novel'), 1, 'regress_rls_bob', 'my first novel') ON CONFLICT (did) DO UPDATE SET dauthor = 'regress_rls_carol'; -- -- ROLE/GROUP -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE z1 (a int, b text); SELECT public.create_hypertable('z1', 'a', chunk_time_interval=>2); CREATE TABLE z2 (a int, b text); SELECT public.create_hypertable('z2', 'a', chunk_time_interval=>2); GRANT SELECT ON z1,z2 TO regress_rls_group1, regress_rls_group2, regress_rls_bob, regress_rls_carol; INSERT INTO z1 VALUES (1, 'aba'), (2, 'bbb'), (3, 'ccc'), (4, 'dad'); CREATE POLICY p1 ON z1 TO regress_rls_group1 USING (a % 2 = 0); CREATE POLICY p2 ON z1 TO regress_rls_group2 USING (a % 2 = 1); ALTER TABLE z1 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM z1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM z1 WHERE f_leak(b); PREPARE plancache_test AS SELECT * FROM z1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test; PREPARE plancache_test2 AS WITH q AS MATERIALIZED (SELECT * FROM z1 WHERE f_leak(b)) SELECT * FROM q,z2; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test2; PREPARE plancache_test4 AS WITH q AS (SELECT * FROM z1 WHERE f_leak(b)) SELECT * FROM q,z2; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test4; PREPARE plancache_test6 AS WITH q AS NOT MATERIALIZED (SELECT * FROM z1 WHERE f_leak(b)) SELECT * FROM q,z2; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test6; PREPARE plancache_test3 AS WITH q AS MATERIALIZED (SELECT * FROM z2) SELECT * FROM q,z1 WHERE f_leak(z1.b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test3; PREPARE plancache_test5 AS WITH q AS (SELECT * FROM z2) SELECT * FROM q,z1 WHERE f_leak(z1.b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test5; PREPARE plancache_test7 AS WITH q AS NOT MATERIALIZED (SELECT * FROM z2) SELECT * FROM q,z1 WHERE f_leak(z1.b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test7; SET ROLE regress_rls_group1; SELECT * FROM z1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM z1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test2; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test4; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test3; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test5; SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM z1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM z1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test2; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test4; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test3; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test5; SET ROLE regress_rls_group2; SELECT * FROM z1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM z1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test2; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test4; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test3; EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE plancache_test5; -- -- Views should follow policy for view owner. -- -- View and Table owner are the same. SET SESSION AUTHORIZATION regress_rls_alice; CREATE VIEW rls_view AS SELECT * FROM z1 WHERE f_leak(b); GRANT SELECT ON rls_view TO regress_rls_bob; -- Query as role that is not owner of view or table. Should return all records. SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rls_view; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; -- Query as view/table owner. Should return all records. SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM rls_view; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; DROP VIEW rls_view; -- View and Table owners are different. SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW rls_view AS SELECT * FROM z1 WHERE f_leak(b); GRANT SELECT ON rls_view TO regress_rls_alice; -- Query as role that is not owner of view but is owner of table. -- Should return records based on view owner policies. SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM rls_view; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; -- Query as role that is not owner of table but is owner of view. -- Should return records based on view owner policies. SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM rls_view; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; -- Query as role that is not the owner of the table or view without permissions. SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM rls_view; --fail - permission denied. EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; --fail - permission denied. -- Query as role that is not the owner of the table or view with permissions. SET SESSION AUTHORIZATION regress_rls_bob; GRANT SELECT ON rls_view TO regress_rls_carol; SELECT * FROM rls_view; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_view; SET SESSION AUTHORIZATION regress_rls_bob; DROP VIEW rls_view; -- -- Command specific -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE x1 (a int, b text, c text); SELECT public.create_hypertable('x1', 'a', chunk_time_interval=>2); GRANT ALL ON x1 TO PUBLIC; INSERT INTO x1 VALUES (1, 'abc', 'regress_rls_bob'), (2, 'bcd', 'regress_rls_bob'), (3, 'cde', 'regress_rls_carol'), (4, 'def', 'regress_rls_carol'), (5, 'efg', 'regress_rls_bob'), (6, 'fgh', 'regress_rls_bob'), (7, 'fgh', 'regress_rls_carol'), (8, 'fgh', 'regress_rls_carol'); CREATE POLICY p0 ON x1 FOR ALL USING (c = current_user); CREATE POLICY p1 ON x1 FOR SELECT USING (a % 2 = 0); CREATE POLICY p2 ON x1 FOR INSERT WITH CHECK (a % 2 = 1); CREATE POLICY p3 ON x1 FOR UPDATE USING (a % 2 = 0); CREATE POLICY p4 ON x1 FOR DELETE USING (a < 8); ALTER TABLE x1 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM x1 WHERE f_leak(b) ORDER BY a ASC; UPDATE x1 SET b = b || '_updt' WHERE f_leak(b) RETURNING *; SET SESSION AUTHORIZATION regress_rls_carol; SELECT * FROM x1 WHERE f_leak(b) ORDER BY a ASC; UPDATE x1 SET b = b || '_updt' WHERE f_leak(b) RETURNING *; DELETE FROM x1 WHERE f_leak(b) RETURNING *; -- -- Duplicate Policy Names -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE y1 (a int, b text); SELECT public.create_hypertable('y1', 'a', chunk_time_interval=>2); INSERT INTO y1 VALUES(1,2); CREATE TABLE y2 (a int, b text); SELECT public.create_hypertable('y2', 'a', chunk_time_interval=>2); GRANT ALL ON y1, y2 TO regress_rls_bob; CREATE POLICY p1 ON y1 FOR ALL USING (a % 2 = 0); CREATE POLICY p2 ON y1 FOR SELECT USING (a > 2); CREATE POLICY p1 ON y1 FOR SELECT USING (a % 2 = 1); --fail CREATE POLICY p1 ON y2 FOR ALL USING (a % 2 = 0); --OK ALTER TABLE y1 ENABLE ROW LEVEL SECURITY; ALTER TABLE y2 ENABLE ROW LEVEL SECURITY; -- -- Expression structure with SBV -- -- Create view as table owner. RLS should NOT be applied. SET SESSION AUTHORIZATION regress_rls_alice; CREATE VIEW rls_sbv WITH (security_barrier) AS SELECT * FROM y1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_sbv WHERE (a = 1); DROP VIEW rls_sbv; -- Create view as role that does not own table. RLS should be applied. SET SESSION AUTHORIZATION regress_rls_bob; CREATE VIEW rls_sbv WITH (security_barrier) AS SELECT * FROM y1 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM rls_sbv WHERE (a = 1); DROP VIEW rls_sbv; -- -- Expression structure -- SET SESSION AUTHORIZATION regress_rls_alice; INSERT INTO y2 (SELECT x, md5(x::text) FROM generate_series(0,20) x); CREATE POLICY p2 ON y2 USING (a % 3 = 0); CREATE POLICY p3 ON y2 USING (a % 4 = 0); SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM y2 WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM y2 WHERE f_leak(b); -- -- Qual push-down of leaky functions, when not referring to table -- SELECT * FROM y2 WHERE f_leak('abc'); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM y2 WHERE f_leak('abc'); CREATE TABLE test_qual_pushdown ( abc text ); INSERT INTO test_qual_pushdown VALUES ('abc'),('def'); SELECT * FROM y2 JOIN test_qual_pushdown ON (b = abc) WHERE f_leak(abc); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM y2 JOIN test_qual_pushdown ON (b = abc) WHERE f_leak(abc); SELECT * FROM y2 JOIN test_qual_pushdown ON (b = abc) WHERE f_leak(b); EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM y2 JOIN test_qual_pushdown ON (b = abc) WHERE f_leak(b); DROP TABLE test_qual_pushdown; -- -- Plancache invalidate on user change. -- RESET SESSION AUTHORIZATION; \set VERBOSITY terse \\ -- suppress cascade details DROP TABLE t1 CASCADE; \set VERBOSITY default CREATE TABLE t1 (a integer); SELECT public.create_hypertable('t1', 'a', chunk_time_interval=>2); GRANT SELECT ON t1 TO regress_rls_bob, regress_rls_carol; CREATE POLICY p1 ON t1 TO regress_rls_bob USING ((a % 2) = 0); CREATE POLICY p2 ON t1 TO regress_rls_carol USING ((a % 4) = 0); ALTER TABLE t1 ENABLE ROW LEVEL SECURITY; -- Prepare as regress_rls_bob SET ROLE regress_rls_bob; PREPARE role_inval AS SELECT * FROM t1; -- Check plan EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE role_inval; -- Change to regress_rls_carol SET ROLE regress_rls_carol; -- Check plan- should be different EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE role_inval; -- Change back to regress_rls_bob SET ROLE regress_rls_bob; -- Check plan- should be back to original EXPLAIN (BUFFERS OFF, COSTS OFF) EXECUTE role_inval; -- -- CTE and RLS -- RESET SESSION AUTHORIZATION; DROP TABLE t1 CASCADE; CREATE TABLE t1 (a integer, b text); SELECT public.create_hypertable('t1', 'a', chunk_time_interval=>2); CREATE POLICY p1 ON t1 USING (a % 2 = 0); ALTER TABLE t1 ENABLE ROW LEVEL SECURITY; GRANT ALL ON t1 TO regress_rls_bob; INSERT INTO t1 (SELECT x, md5(x::text) FROM generate_series(0,20) x); SET SESSION AUTHORIZATION regress_rls_bob; WITH cte1 AS (SELECT * FROM t1 WHERE f_leak(b)) SELECT * FROM cte1; EXPLAIN (BUFFERS OFF, COSTS OFF) WITH cte1 AS (SELECT * FROM t1 WHERE f_leak(b)) SELECT * FROM cte1; WITH cte1 AS (UPDATE t1 SET a = a + 1 RETURNING *) SELECT * FROM cte1; --fail WITH cte1 AS (UPDATE t1 SET a = a RETURNING *) SELECT * FROM cte1; --ok WITH cte1 AS (INSERT INTO t1 VALUES (21, 'Fail') RETURNING *) SELECT * FROM cte1; --fail WITH cte1 AS (INSERT INTO t1 VALUES (20, 'Success') RETURNING *) SELECT * FROM cte1; --ok -- -- Rename Policy -- RESET SESSION AUTHORIZATION; ALTER POLICY p1 ON t1 RENAME TO p1; --fail SELECT polname, relname FROM pg_policy pol JOIN pg_class pc ON (pc.oid = pol.polrelid) WHERE relname = 't1'; ALTER POLICY p1 ON t1 RENAME TO p2; --ok SELECT polname, relname FROM pg_policy pol JOIN pg_class pc ON (pc.oid = pol.polrelid) WHERE relname = 't1'; -- -- Check INSERT SELECT -- SET SESSION AUTHORIZATION regress_rls_bob; CREATE TABLE t2 (a integer, b text); SELECT public.create_hypertable('t2', 'a', chunk_time_interval=>2); INSERT INTO t2 (SELECT * FROM t1); EXPLAIN (BUFFERS OFF, COSTS OFF) INSERT INTO t2 (SELECT * FROM t1); SELECT * FROM t2; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t2; CREATE TABLE t3 AS SELECT * FROM t1; SELECT public.create_hypertable('t2', 'a', chunk_time_interval=>2); SELECT * FROM t3; SELECT * INTO t4 FROM t1; SELECT * FROM t4; -- -- RLS with JOIN -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE blog (id integer, author text, post text); SELECT public.create_hypertable('blog', 'id', chunk_time_interval=>2); CREATE TABLE comment (blog_id integer, message text); SELECT public.create_hypertable('comment', 'blog_id', chunk_time_interval=>2); GRANT ALL ON blog, comment TO regress_rls_bob; CREATE POLICY blog_1 ON blog USING (id % 2 = 0); ALTER TABLE blog ENABLE ROW LEVEL SECURITY; INSERT INTO blog VALUES (1, 'alice', 'blog #1'), (2, 'bob', 'blog #1'), (3, 'alice', 'blog #2'), (4, 'alice', 'blog #3'), (5, 'john', 'blog #1'); INSERT INTO comment VALUES (1, 'cool blog'), (1, 'fun blog'), (3, 'crazy blog'), (5, 'what?'), (4, 'insane!'), (2, 'who did it?'); SET SESSION AUTHORIZATION regress_rls_bob; -- Check RLS JOIN with Non-RLS. SELECT id, author, message FROM blog JOIN comment ON id = blog_id; -- Check Non-RLS JOIN with RLS. SELECT id, author, message FROM comment JOIN blog ON id = blog_id; SET SESSION AUTHORIZATION regress_rls_alice; CREATE POLICY comment_1 ON comment USING (blog_id < 4); ALTER TABLE comment ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; -- Check RLS JOIN RLS SELECT id, author, message FROM blog JOIN comment ON id = blog_id; SELECT id, author, message FROM comment JOIN blog ON id = blog_id; SET SESSION AUTHORIZATION regress_rls_alice; DROP TABLE blog; DROP TABLE comment; -- -- Default Deny Policy -- RESET SESSION AUTHORIZATION; DROP POLICY p2 ON t1; ALTER TABLE t1 OWNER TO regress_rls_alice; -- Check that default deny does not apply to superuser. RESET SESSION AUTHORIZATION; SELECT * FROM t1; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; -- Check that default deny does not apply to table owner. SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM t1; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; -- Check that default deny applies to non-owner/non-superuser when RLS on. SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO ON; SELECT * FROM t1; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM t1; EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM t1; -- -- COPY TO/FROM -- RESET SESSION AUTHORIZATION; DROP TABLE copy_t CASCADE; CREATE TABLE copy_t (a integer, b text); SELECT public.create_hypertable('copy_t', 'a', chunk_time_interval=>2); CREATE POLICY p1 ON copy_t USING (a % 2 = 0); ALTER TABLE copy_t ENABLE ROW LEVEL SECURITY; GRANT ALL ON copy_t TO regress_rls_bob, regress_rls_exempt_user; INSERT INTO copy_t (SELECT x, md5(x::text) FROM generate_series(0,10) x); -- Check COPY TO as Superuser/owner. RESET SESSION AUTHORIZATION; SET row_security TO OFF; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; SET row_security TO ON; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; -- Check COPY TO as user with permissions. SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO OFF; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --fail - would be affected by RLS SET row_security TO ON; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --ok -- Check COPY TO as user with permissions and BYPASSRLS SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --ok SET row_security TO ON; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --ok -- Check COPY TO as user without permissions. SET row_security TO OFF; SET SESSION AUTHORIZATION regress_rls_carol; SET row_security TO OFF; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --fail - would be affected by RLS SET row_security TO ON; COPY (SELECT * FROM copy_t ORDER BY a ASC) TO STDOUT WITH DELIMITER ','; --fail - permission denied -- Check COPY relation TO; keep it just one row to avoid reordering issues RESET SESSION AUTHORIZATION; SET row_security TO ON; CREATE TABLE copy_rel_to (a integer, b text); SELECT public.create_hypertable('copy_rel_to', 'a', chunk_time_interval=>2); CREATE POLICY p1 ON copy_rel_to USING (a % 2 = 0); ALTER TABLE copy_rel_to ENABLE ROW LEVEL SECURITY; GRANT ALL ON copy_rel_to TO regress_rls_bob, regress_rls_exempt_user; INSERT INTO copy_rel_to VALUES (1, md5('1')); -- Check COPY TO as Superuser/owner. RESET SESSION AUTHORIZATION; SET row_security TO OFF; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; SET row_security TO ON; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; -- Check COPY TO as user with permissions. SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO OFF; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --fail - would be affected by RLS SET row_security TO ON; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --ok -- Check COPY TO as user with permissions and BYPASSRLS SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO OFF; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --ok SET row_security TO ON; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --ok -- Check COPY TO as user without permissions. SET row_security TO OFF; SET SESSION AUTHORIZATION regress_rls_carol; SET row_security TO OFF; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --fail - permission denied SET row_security TO ON; COPY (SELECT * FROM copy_rel_to) TO STDOUT WITH DELIMITER ','; --fail - permission denied -- Check COPY FROM as Superuser/owner. RESET SESSION AUTHORIZATION; SET row_security TO OFF; COPY copy_t FROM STDIN; --ok 1 abc 2 bcd 3 cde 4 def \. SET row_security TO ON; COPY copy_t FROM STDIN; --ok 1 abc 2 bcd 3 cde 4 def \. -- Check COPY FROM as user with permissions. SET SESSION AUTHORIZATION regress_rls_bob; SET row_security TO OFF; COPY copy_t FROM STDIN; --fail - would be affected by RLS. SET row_security TO ON; COPY copy_t FROM STDIN; --fail - COPY FROM not supported by RLS. -- Check COPY FROM as user with permissions and BYPASSRLS SET SESSION AUTHORIZATION regress_rls_exempt_user; SET row_security TO ON; COPY copy_t FROM STDIN; --ok 1 abc 2 bcd 3 cde 4 def \. -- Check COPY FROM as user without permissions. SET SESSION AUTHORIZATION regress_rls_carol; SET row_security TO OFF; COPY copy_t FROM STDIN; --fail - permission denied. SET row_security TO ON; COPY copy_t FROM STDIN; --fail - permission denied. RESET SESSION AUTHORIZATION; DROP TABLE copy_t; DROP TABLE copy_rel_to CASCADE; -- Check WHERE CURRENT OF SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE current_check (currentid int, payload text, rlsuser text); SELECT public.create_hypertable('current_check', 'currentid', chunk_time_interval=>10); GRANT ALL ON current_check TO PUBLIC; INSERT INTO current_check VALUES (1, 'abc', 'regress_rls_bob'), (2, 'bcd', 'regress_rls_bob'), (3, 'cde', 'regress_rls_bob'), (4, 'def', 'regress_rls_bob'); CREATE POLICY p1 ON current_check FOR SELECT USING (currentid % 2 = 0); CREATE POLICY p2 ON current_check FOR DELETE USING (currentid = 4 AND rlsuser = current_user); CREATE POLICY p3 ON current_check FOR UPDATE USING (currentid = 4) WITH CHECK (rlsuser = current_user); ALTER TABLE current_check ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; -- Can SELECT even rows SELECT * FROM current_check; -- Cannot UPDATE row 2 UPDATE current_check SET payload = payload || '_new' WHERE currentid = 2 RETURNING *; BEGIN; -- WHERE CURRENT OF does not work with custom scan nodes -- so we have to disable chunk append here SET timescaledb.enable_chunk_append TO false; DECLARE current_check_cursor SCROLL CURSOR FOR SELECT * FROM current_check; -- Returns rows that can be seen according to SELECT policy, like plain SELECT -- above (even rows) FETCH ABSOLUTE 1 FROM current_check_cursor; -- Still cannot UPDATE row 2 through cursor UPDATE current_check SET payload = payload || '_new' WHERE CURRENT OF current_check_cursor RETURNING *; -- Can update row 4 through cursor, which is the next visible row FETCH RELATIVE 1 FROM current_check_cursor; UPDATE current_check SET payload = payload || '_new' WHERE CURRENT OF current_check_cursor RETURNING *; SELECT * FROM current_check; -- Plan should be a subquery TID scan EXPLAIN (BUFFERS OFF, COSTS OFF) UPDATE current_check SET payload = payload WHERE CURRENT OF current_check_cursor; -- Similarly can only delete row 4 FETCH ABSOLUTE 1 FROM current_check_cursor; DELETE FROM current_check WHERE CURRENT OF current_check_cursor RETURNING *; FETCH RELATIVE 1 FROM current_check_cursor; DELETE FROM current_check WHERE CURRENT OF current_check_cursor RETURNING *; SELECT * FROM current_check; RESET timescaledb.enable_chunk_append; COMMIT; -- -- check pg_stats view filtering -- SET row_security TO ON; SET SESSION AUTHORIZATION regress_rls_alice; ANALYZE current_check; -- Stats visible SELECT row_security_active('current_check'); SELECT attname, most_common_vals FROM pg_stats WHERE tablename = 'current_check' ORDER BY 1; SET SESSION AUTHORIZATION regress_rls_bob; -- Stats not visible SELECT row_security_active('current_check'); SELECT attname, most_common_vals FROM pg_stats WHERE tablename = 'current_check' ORDER BY 1; -- -- Collation support -- BEGIN; CREATE TABLE coll_t (c) AS VALUES ('bar'::text); CREATE POLICY coll_p ON coll_t USING (c < ('foo'::text COLLATE "C")); ALTER TABLE coll_t ENABLE ROW LEVEL SECURITY; GRANT SELECT ON coll_t TO regress_rls_alice; SELECT (string_to_array(polqual, ':'))[7] AS inputcollid FROM pg_policy WHERE polrelid = 'coll_t'::regclass; SET SESSION AUTHORIZATION regress_rls_alice; SELECT * FROM coll_t; ROLLBACK; -- -- Shared Object Dependencies -- RESET SESSION AUTHORIZATION; BEGIN; CREATE ROLE regress_rls_eve; CREATE ROLE regress_rls_frank; CREATE TABLE tbl1 (c) AS VALUES ('bar'::text); GRANT SELECT ON TABLE tbl1 TO regress_rls_eve; CREATE POLICY P ON tbl1 TO regress_rls_eve, regress_rls_frank USING (true); SELECT refclassid::regclass, deptype FROM pg_depend WHERE classid = 'pg_policy'::regclass AND refobjid = 'tbl1'::regclass; SELECT refclassid::regclass, deptype FROM pg_shdepend WHERE classid = 'pg_policy'::regclass AND refobjid IN ('regress_rls_eve'::regrole, 'regress_rls_frank'::regrole); SAVEPOINT q; DROP ROLE regress_rls_eve; --fails due to dependency on POLICY p ROLLBACK TO q; ALTER POLICY p ON tbl1 TO regress_rls_frank USING (true); SAVEPOINT q; DROP ROLE regress_rls_eve; --fails due to dependency on GRANT SELECT ROLLBACK TO q; REVOKE ALL ON TABLE tbl1 FROM regress_rls_eve; SAVEPOINT q; DROP ROLE regress_rls_eve; --succeeds ROLLBACK TO q; SAVEPOINT q; DROP ROLE regress_rls_frank; --fails due to dependency on POLICY p ROLLBACK TO q; DROP POLICY p ON tbl1; SAVEPOINT q; DROP ROLE regress_rls_frank; -- succeeds ROLLBACK TO q; ROLLBACK; -- cleanup -- -- Converting table to view -- BEGIN; CREATE TABLE t (c int); SELECT public.create_hypertable('t', 'c', chunk_time_interval=>2); CREATE POLICY p ON t USING (c % 2 = 1); ALTER TABLE t ENABLE ROW LEVEL SECURITY; SAVEPOINT q; CREATE RULE "_RETURN" AS ON SELECT TO t DO INSTEAD SELECT * FROM generate_series(1,5) t0(c); -- fails due to row level security enabled ROLLBACK TO q; ALTER TABLE t DISABLE ROW LEVEL SECURITY; SAVEPOINT q; CREATE RULE "_RETURN" AS ON SELECT TO t DO INSTEAD SELECT * FROM generate_series(1,5) t0(c); -- fails due to policy p on t ROLLBACK TO q; DROP POLICY p ON t; CREATE RULE "_RETURN" AS ON SELECT TO t DO INSTEAD SELECT * FROM generate_series(1,5) t0(c); -- succeeds ROLLBACK; -- -- Policy expression handling -- BEGIN; CREATE TABLE t (c) AS VALUES ('bar'::text); CREATE POLICY p ON t USING (max(c)); -- fails: aggregate functions are not allowed in policy expressions ROLLBACK; -- -- Non-target relations are only subject to SELECT policies -- SET SESSION AUTHORIZATION regress_rls_alice; CREATE TABLE r1 (a int); SELECT public.create_hypertable('r1', 'a', chunk_time_interval=>2); CREATE TABLE r2 (a int); SELECT public.create_hypertable('r2', 'a', chunk_time_interval=>2); INSERT INTO r1 VALUES (10), (20); INSERT INTO r2 VALUES (10), (20); GRANT ALL ON r1, r2 TO regress_rls_bob; CREATE POLICY p1 ON r1 USING (true); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; CREATE POLICY p1 ON r2 FOR SELECT USING (true); CREATE POLICY p2 ON r2 FOR INSERT WITH CHECK (false); CREATE POLICY p3 ON r2 FOR UPDATE USING (false); CREATE POLICY p4 ON r2 FOR DELETE USING (false); ALTER TABLE r2 ENABLE ROW LEVEL SECURITY; SET SESSION AUTHORIZATION regress_rls_bob; SELECT * FROM r1; SELECT * FROM r2; -- r2 is read-only INSERT INTO r2 VALUES (2); -- Not allowed \pset tuples_only 1 UPDATE r2 SET a = 2 RETURNING *; -- Updates nothing DELETE FROM r2 RETURNING *; -- Deletes nothing \pset tuples_only 0 -- r2 can be used as a non-target relation in DML INSERT INTO r1 SELECT a + 1 FROM r2 RETURNING *; -- OK UPDATE r1 SET a = r2.a + 2 FROM r2 WHERE r1.a = r2.a RETURNING *; -- OK DELETE FROM r1 USING r2 WHERE r1.a = r2.a + 2 RETURNING *; -- OK SELECT * FROM r1; SELECT * FROM r2; SET SESSION AUTHORIZATION regress_rls_alice; DROP TABLE r1; DROP TABLE r2; -- -- FORCE ROW LEVEL SECURITY applies RLS to owners too -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security = on; CREATE TABLE r1 (a int); SELECT public.create_hypertable('r1', 'a', chunk_time_interval=>2); INSERT INTO r1 VALUES (10), (20); CREATE POLICY p1 ON r1 USING (false); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- No error, but no rows TABLE r1; -- RLS error INSERT INTO r1 VALUES (1); -- No error (unable to see any rows to update) UPDATE r1 SET a = 1; TABLE r1; -- No error (unable to see any rows to delete) DELETE FROM r1; TABLE r1; SET row_security = off; -- these all fail, would be affected by RLS TABLE r1; UPDATE r1 SET a = 1; DELETE FROM r1; DROP TABLE r1; -- -- FORCE ROW LEVEL SECURITY does not break RI -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security = on; CREATE TABLE r1 (a int PRIMARY KEY); -- r1 is not a hypertable since r1.a is referenced by r2 CREATE TABLE r2 (a int REFERENCES r1); SELECT public.create_hypertable('r2', 'a', chunk_time_interval=>2); INSERT INTO r1 VALUES (10), (20); INSERT INTO r2 VALUES (10), (20); -- Create policies on r2 which prevent the -- owner from seeing any rows, but RI should -- still see them. CREATE POLICY p1 ON r2 USING (false); ALTER TABLE r2 ENABLE ROW LEVEL SECURITY; ALTER TABLE r2 FORCE ROW LEVEL SECURITY; -- Errors due to rows in r2 DELETE FROM r1; -- Reset r2 to no-RLS DROP POLICY p1 ON r2; ALTER TABLE r2 NO FORCE ROW LEVEL SECURITY; ALTER TABLE r2 DISABLE ROW LEVEL SECURITY; -- clean out r2 for INSERT test below DELETE FROM r2; -- Change r1 to not allow rows to be seen CREATE POLICY p1 ON r1 USING (false); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- No rows seen TABLE r1; -- No error, RI still sees that row exists in r1 INSERT INTO r2 VALUES (10); DROP TABLE r2; DROP TABLE r1; -- Ensure cascaded DELETE works CREATE TABLE r1 (a int PRIMARY KEY); -- r1 is not a hypertable since r1.a is referenced by r2 CREATE TABLE r2 (a int REFERENCES r1 ON DELETE CASCADE); SELECT public.create_hypertable('r2', 'a', chunk_time_interval=>2); INSERT INTO r1 VALUES (10), (20); INSERT INTO r2 VALUES (10), (20); -- Create policies on r2 which prevent the -- owner from seeing any rows, but RI should -- still see them. CREATE POLICY p1 ON r2 USING (false); ALTER TABLE r2 ENABLE ROW LEVEL SECURITY; ALTER TABLE r2 FORCE ROW LEVEL SECURITY; -- Deletes all records from both DELETE FROM r1; -- Remove FORCE from r2 ALTER TABLE r2 NO FORCE ROW LEVEL SECURITY; -- As owner, we now bypass RLS -- verify no rows in r2 now TABLE r2; DROP TABLE r2; DROP TABLE r1; -- Ensure cascaded UPDATE works CREATE TABLE r1 (a int PRIMARY KEY); -- r1 is not a hypertable since r1.a is referenced by r2 CREATE TABLE r2 (a int REFERENCES r1 ON UPDATE CASCADE); SELECT public.create_hypertable('r2', 'a', chunk_time_interval=>2); INSERT INTO r1 VALUES (10), (20); INSERT INTO r2 VALUES (10), (20); -- Create policies on r2 which prevent the -- owner from seeing any rows, but RI should -- still see them. CREATE POLICY p1 ON r2 USING (false); ALTER TABLE r2 ENABLE ROW LEVEL SECURITY; ALTER TABLE r2 FORCE ROW LEVEL SECURITY; -- Updates records in both (terse output to not print CONTEXT, which can be different). \set VERBOSITY terse UPDATE r1 SET a = a+5; \set VERBOSITY default -- Remove FORCE from r2 ALTER TABLE r2 NO FORCE ROW LEVEL SECURITY; -- As owner, we now bypass RLS -- verify records in r2 updated TABLE r2; DROP TABLE r2; DROP TABLE r1; -- -- Test INSERT+RETURNING applies SELECT policies as -- WithCheckOptions (meaning an error is thrown) -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security = on; CREATE TABLE r1 (a int); SELECT public.create_hypertable('r1', 'a', chunk_time_interval=>2); CREATE POLICY p1 ON r1 FOR SELECT USING (false); CREATE POLICY p2 ON r1 FOR INSERT WITH CHECK (true); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- Works fine INSERT INTO r1 VALUES (10), (20); -- No error, but no rows TABLE r1; SET row_security = off; -- fail, would be affected by RLS TABLE r1; SET row_security = on; -- Error INSERT INTO r1 VALUES (10), (20) RETURNING *; DROP TABLE r1; -- -- Test UPDATE+RETURNING applies SELECT policies as -- WithCheckOptions (meaning an error is thrown) -- SET SESSION AUTHORIZATION regress_rls_alice; SET row_security = on; CREATE TABLE r1 (a int PRIMARY KEY); SELECT public.create_hypertable('r1', 'a', chunk_time_interval=>100); CREATE POLICY p1 ON r1 FOR SELECT USING (a < 20); CREATE POLICY p2 ON r1 FOR UPDATE USING (a < 20) WITH CHECK (true); CREATE POLICY p3 ON r1 FOR INSERT WITH CHECK (true); INSERT INTO r1 VALUES (10); ALTER TABLE r1 ENABLE ROW LEVEL SECURITY; ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- Works fine UPDATE r1 SET a = 30; -- Show updated rows ALTER TABLE r1 NO FORCE ROW LEVEL SECURITY; TABLE r1; -- reset value in r1 for test with RETURNING UPDATE r1 SET a = 10; -- Verify row reset TABLE r1; ALTER TABLE r1 FORCE ROW LEVEL SECURITY; -- Error UPDATE r1 SET a = 30 RETURNING *; -- UPDATE path of INSERT ... ON CONFLICT DO UPDATE should also error out INSERT INTO r1 VALUES (10) ON CONFLICT (a) DO UPDATE SET a = 30 RETURNING *; -- Should still error out without RETURNING (use of arbiter always requires -- SELECT permissions) INSERT INTO r1 VALUES (10) ON CONFLICT (a) DO UPDATE SET a = 30; -- ON CONFLICT ON CONSTRAINT INSERT INTO r1 VALUES (10) ON CONFLICT ON CONSTRAINT r1_pkey DO UPDATE SET a = 30; DROP TABLE r1; -- Check dependency handling RESET SESSION AUTHORIZATION; CREATE TABLE dep1 (c1 int); SELECT public.create_hypertable('dep1', 'c1', chunk_time_interval=>2); CREATE TABLE dep2 (c1 int); SELECT public.create_hypertable('dep2', 'c1', chunk_time_interval=>2); CREATE POLICY dep_p1 ON dep1 TO regress_rls_bob USING (c1 > (select max(dep2.c1) from dep2)); ALTER POLICY dep_p1 ON dep1 TO regress_rls_bob,regress_rls_carol; -- Should return one SELECT count(*) = 1 FROM pg_depend WHERE objid = (SELECT oid FROM pg_policy WHERE polname = 'dep_p1') AND refobjid = (SELECT oid FROM pg_class WHERE relname = 'dep2'); ALTER POLICY dep_p1 ON dep1 USING (true); -- Should return one SELECT count(*) = 1 FROM pg_shdepend WHERE objid = (SELECT oid FROM pg_policy WHERE polname = 'dep_p1') AND refobjid = (SELECT oid FROM pg_authid WHERE rolname = 'regress_rls_bob'); -- Should return one SELECT count(*) = 1 FROM pg_shdepend WHERE objid = (SELECT oid FROM pg_policy WHERE polname = 'dep_p1') AND refobjid = (SELECT oid FROM pg_authid WHERE rolname = 'regress_rls_carol'); -- Should return zero SELECT count(*) = 0 FROM pg_depend WHERE objid = (SELECT oid FROM pg_policy WHERE polname = 'dep_p1') AND refobjid = (SELECT oid FROM pg_class WHERE relname = 'dep2'); -- DROP OWNED BY testing RESET SESSION AUTHORIZATION; CREATE ROLE regress_rls_dob_role1; CREATE ROLE regress_rls_dob_role2; CREATE TABLE dob_t1 (c1 int); SELECT public.create_hypertable('dob_t1', 'c1', chunk_time_interval=>2); CREATE TABLE dob_t2 (c1 int) PARTITION BY RANGE (c1); CREATE POLICY p1 ON dob_t1 TO regress_rls_dob_role1 USING (true); DROP OWNED BY regress_rls_dob_role1; DROP POLICY p1 ON dob_t1; -- should fail, already gone CREATE POLICY p1 ON dob_t1 TO regress_rls_dob_role1,regress_rls_dob_role2 USING (true); DROP OWNED BY regress_rls_dob_role1; DROP POLICY p1 ON dob_t1; -- should succeed CREATE POLICY p1 ON dob_t2 TO regress_rls_dob_role1,regress_rls_dob_role2 USING (true); DROP OWNED BY regress_rls_dob_role1; DROP POLICY p1 ON dob_t2; -- should succeed DROP USER regress_rls_dob_role1; DROP USER regress_rls_dob_role2; -- -- Clean up objects -- RESET SESSION AUTHORIZATION; \set VERBOSITY terse \\ -- suppress cascade details DROP SCHEMA regress_rls_schema CASCADE; \set VERBOSITY default DROP USER regress_rls_alice; DROP USER regress_rls_bob; DROP USER regress_rls_carol; DROP USER regress_rls_dave; DROP USER regress_rls_exempt_user; DROP ROLE regress_rls_group1; DROP ROLE regress_rls_group2; -- Arrange to have a few policies left over, for testing -- pg_dump/pg_restore CREATE SCHEMA regress_rls_schema; CREATE TABLE rls_tbl (c1 int); SELECT public.create_hypertable('rls_tbl', 'c1', chunk_time_interval=>2); ALTER TABLE rls_tbl ENABLE ROW LEVEL SECURITY; CREATE POLICY p1 ON rls_tbl USING (c1 > 5); CREATE POLICY p2 ON rls_tbl FOR SELECT USING (c1 <= 3); CREATE POLICY p3 ON rls_tbl FOR UPDATE USING (c1 <= 3) WITH CHECK (c1 > 5); CREATE POLICY p4 ON rls_tbl FOR DELETE USING (c1 <= 3); CREATE TABLE rls_tbl_force (c1 int); SELECT public.create_hypertable('rls_tbl_force', 'c1', chunk_time_interval=>2); ALTER TABLE rls_tbl_force ENABLE ROW LEVEL SECURITY; ALTER TABLE rls_tbl_force FORCE ROW LEVEL SECURITY; CREATE POLICY p1 ON rls_tbl_force USING (c1 = 5) WITH CHECK (c1 < 5); CREATE POLICY p2 ON rls_tbl_force FOR SELECT USING (c1 = 8); CREATE POLICY p3 ON rls_tbl_force FOR UPDATE USING (c1 = 8) WITH CHECK (c1 >= 5); CREATE POLICY p4 ON rls_tbl_force FOR DELETE USING (c1 = 8); ================================================ FILE: test/sql/size_utils.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \ir include/insert_two_partitions.sql SELECT * FROM hypertable_detailed_size('"public"."two_Partitions"'); SELECT * FROM hypertable_index_size('"public"."two_Partitions_device_id_timeCustom_idx"'); SELECT * FROM hypertable_index_size('"public"."two_Partitions_timeCustom_device_id_idx"'); SELECT * FROM hypertable_index_size('"public"."two_Partitions_timeCustom_idx"'); SELECT * FROM hypertable_index_size('"public"."two_Partitions_timeCustom_series_0_idx"'); SELECT * FROM hypertable_index_size('"public"."two_Partitions_timeCustom_series_1_idx"'); SELECT * FROM hypertable_index_size('"public"."two_Partitions_timeCustom_series_2_idx"'); SELECT * FROM hypertable_index_size('"public"."two_Partitions_timeCustom_series_bool_idx"'); SELECT * FROM chunks_detailed_size('"public"."two_Partitions"') order by chunk_name; CREATE TABLE timestamp_partitioned(time TIMESTAMP, value TEXT); SELECT * FROM create_hypertable('timestamp_partitioned', 'time', 'value', 2); INSERT INTO timestamp_partitioned VALUES('2004-10-19 10:23:54', '10'); INSERT INTO timestamp_partitioned VALUES('2004-12-19 10:23:54', '30'); SELECT * FROM chunks_detailed_size('timestamp_partitioned') order by chunk_name; CREATE TABLE timestamp_partitioned_2(time TIMESTAMP, value CHAR(9)); SELECT * FROM create_hypertable('timestamp_partitioned_2', 'time', 'value', 2); INSERT INTO timestamp_partitioned_2 VALUES('2004-10-19 10:23:54', '10'); INSERT INTO timestamp_partitioned_2 VALUES('2004-12-19 10:23:54', '30'); SELECT * FROM chunks_detailed_size('timestamp_partitioned_2') order by chunk_name; CREATE TABLE toast_test(time TIMESTAMP, value TEXT); -- Set storage type to EXTERNAL to prevent PostgreSQL from compressing my -- easily compressable string and instead store it with TOAST ALTER TABLE toast_test ALTER COLUMN value SET STORAGE EXTERNAL; SELECT * FROM create_hypertable('toast_test', 'time'); INSERT INTO toast_test VALUES('2004-10-19 10:23:54', $$ this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. this must be over 2k. $$); SELECT * FROM chunks_detailed_size('toast_test'); -- -- Tests for approximate_row_count() -- -- Regular table -- CREATE TABLE approx_count(time TIMESTAMP, value int); INSERT INTO approx_count VALUES('2004-01-01 10:00:01', 1); INSERT INTO approx_count VALUES('2004-01-01 10:00:02', 2); INSERT INTO approx_count VALUES('2004-01-01 10:00:03', 3); INSERT INTO approx_count VALUES('2004-01-01 10:00:04', 4); INSERT INTO approx_count VALUES('2004-01-01 10:00:05', 5); INSERT INTO approx_count VALUES('2004-01-01 10:00:06', 6); INSERT INTO approx_count VALUES('2004-01-01 10:00:07', 7); SELECT * FROM approximate_row_count('approx_count'); ANALYZE approx_count; SELECT count(*) FROM approx_count; SELECT * FROM approximate_row_count('approx_count'); DROP TABLE approx_count; -- Regular table with basic inheritance -- CREATE TABLE approx_count(id int); INSERT INTO approx_count VALUES(1); SELECT count(*) FROM approx_count; SELECT * FROM approximate_row_count('approx_count'); ANALYZE approx_count; SELECT * FROM approximate_row_count('approx_count'); CREATE TABLE approx_count_child(id2 int) INHERITS (approx_count); INSERT INTO approx_count_child VALUES(0); SELECT count(*) FROM approx_count; SELECT * FROM approximate_row_count('approx_count'); ANALYZE approx_count_child; SELECT * FROM approximate_row_count('approx_count'); DROP TABLE approx_count CASCADE; -- Regular table with nested inheritance -- CREATE TABLE approx_count(id int); CREATE TABLE approx_count_a(id2 int) INHERITS (approx_count); CREATE TABLE approx_count_b(id3 int) INHERITS (approx_count_a); CREATE TABLE approx_count_c(id4 int) INHERITS (approx_count_b); INSERT INTO approx_count_a VALUES(0); INSERT INTO approx_count_b VALUES(1); INSERT INTO approx_count_c VALUES(2); INSERT INTO approx_count VALUES(3); SELECT * FROM approximate_row_count('approx_count'); ANALYZE approx_count_a; ANALYZE approx_count_b; ANALYZE approx_count_c; ANALYZE approx_count; SELECT count(*) FROM approx_count; SELECT * FROM approximate_row_count('approx_count'); SELECT count(*) FROM approx_count_a; SELECT * FROM approximate_row_count('approx_count_a'); SELECT count(*) FROM approx_count_b; SELECT * FROM approximate_row_count('approx_count_b'); SELECT count(*) FROM approx_count_c; SELECT * FROM approximate_row_count('approx_count_c'); DROP TABLE approx_count CASCADE; -- table with declarative partitioning -- CREATE TABLE approx_count_dp(time TIMESTAMP, value int) PARTITION BY RANGE(time); CREATE TABLE approx_count_dp0 PARTITION OF approx_count_dp FOR VALUES FROM ('2004-01-01 00:00:00') TO ('2005-01-01 00:00:00'); CREATE TABLE approx_count_dp1 PARTITION OF approx_count_dp FOR VALUES FROM ('2005-01-01 00:00:00') TO ('2006-01-01 00:00:00'); CREATE TABLE approx_count_dp2 PARTITION OF approx_count_dp FOR VALUES FROM ('2006-01-01 00:00:00') TO ('2007-01-01 00:00:00'); INSERT INTO approx_count_dp VALUES('2004-01-01 10:00:00', 1); INSERT INTO approx_count_dp VALUES('2004-01-01 11:00:00', 1); INSERT INTO approx_count_dp VALUES('2004-01-01 12:00:01', 1); INSERT INTO approx_count_dp VALUES('2005-01-01 10:00:00', 1); INSERT INTO approx_count_dp VALUES('2005-01-01 11:00:00', 1); INSERT INTO approx_count_dp VALUES('2005-01-01 12:00:01', 1); INSERT INTO approx_count_dp VALUES('2006-01-01 10:00:00', 1); INSERT INTO approx_count_dp VALUES('2006-01-01 11:00:00', 1); INSERT INTO approx_count_dp VALUES('2006-01-01 12:00:01', 1); SELECT count(*) FROM approx_count_dp; SELECT count(*) FROM approx_count_dp0; SELECT count(*) FROM approx_count_dp1; SELECT count(*) FROM approx_count_dp2; SELECT * FROM approximate_row_count('approx_count_dp'); ANALYZE approx_count_dp; SELECT * FROM approximate_row_count('approx_count_dp'); SELECT * FROM approximate_row_count('approx_count_dp0'); SELECT * FROM approximate_row_count('approx_count_dp1'); SELECT * FROM approximate_row_count('approx_count_dp2'); CREATE TABLE approx_count_dp_nested(time TIMESTAMP, device_id int, value int) PARTITION BY RANGE(time); CREATE TABLE approx_count_dp_nested_0 PARTITION OF approx_count_dp_nested FOR VALUES FROM ('2004-01-01 00:00:00') TO ('2005-01-01 00:00:00') PARTITION BY RANGE (device_id); CREATE TABLE approx_count_dp_nested_0_0 PARTITION OF approx_count_dp_nested_0 FOR VALUES FROM (0) TO (10); CREATE TABLE approx_count_dp_nested_0_1 PARTITION OF approx_count_dp_nested_0 FOR VALUES FROM (10) TO (20); CREATE TABLE approx_count_dp_nested_1 PARTITION OF approx_count_dp_nested FOR VALUES FROM ('2005-01-01 00:00:00') TO ('2006-01-01 00:00:00') PARTITION BY RANGE (device_id); CREATE TABLE approx_count_dp_nested_1_0 PARTITION OF approx_count_dp_nested_1 FOR VALUES FROM (0) TO (10); CREATE TABLE approx_count_dp_nested_1_1 PARTITION OF approx_count_dp_nested_1 FOR VALUES FROM (10) TO (20); INSERT INTO approx_count_dp_nested VALUES('2004-01-01 10:00:00', 1, 1); INSERT INTO approx_count_dp_nested VALUES('2004-01-01 10:00:00', 2, 1); INSERT INTO approx_count_dp_nested VALUES('2004-01-01 10:00:00', 3, 1); INSERT INTO approx_count_dp_nested VALUES('2004-01-01 10:00:00', 11, 1); INSERT INTO approx_count_dp_nested VALUES('2004-01-01 10:00:00', 12, 1); INSERT INTO approx_count_dp_nested VALUES('2004-01-01 10:00:00', 13, 1); INSERT INTO approx_count_dp_nested VALUES('2005-01-01 10:00:00', 1, 1); INSERT INTO approx_count_dp_nested VALUES('2005-01-01 10:00:00', 2, 1); INSERT INTO approx_count_dp_nested VALUES('2005-01-01 10:00:00', 3, 1); INSERT INTO approx_count_dp_nested VALUES('2005-01-01 10:00:00', 11, 1); INSERT INTO approx_count_dp_nested VALUES('2005-01-01 10:00:00', 12, 1); INSERT INTO approx_count_dp_nested VALUES('2005-01-01 10:00:00', 13, 1); SELECT * FROM approximate_row_count('approx_count_dp_nested'); ANALYZE approx_count_dp_nested; SELECT (SELECT count(*) FROM approx_count_dp_nested) AS dp_nested, (SELECT count(*) FROM approx_count_dp_nested_0) AS dp_nested_0, (SELECT count(*) FROM approx_count_dp_nested_0_0) AS dp_nested_0_0, (SELECT count(*) FROM approx_count_dp_nested_0_1) AS dp_nested_0_1, (SELECT count(*) FROM approx_count_dp_nested_1) AS dp_nested_1, (SELECT count(*) FROM approx_count_dp_nested_1_0) AS dp_nested_1_0, (SELECT count(*) FROM approx_count_dp_nested_1_1) AS dp_nested_1_1 UNION ALL SELECT approximate_row_count('approx_count_dp_nested'), approximate_row_count('approx_count_dp_nested_0'), approximate_row_count('approx_count_dp_nested_0_0'), approximate_row_count('approx_count_dp_nested_0_1'), approximate_row_count('approx_count_dp_nested_1'), approximate_row_count('approx_count_dp_nested_1_0'), approximate_row_count('approx_count_dp_nested_1_1'); -- Hypertable -- CREATE TABLE approx_count(time TIMESTAMP, value int); SELECT * FROM create_hypertable('approx_count', 'time'); INSERT INTO approx_count VALUES('2004-01-01 10:00:01', 1); INSERT INTO approx_count VALUES('2004-01-01 10:00:02', 2); INSERT INTO approx_count VALUES('2004-01-01 10:00:03', 3); INSERT INTO approx_count VALUES('2004-01-01 10:00:04', 4); INSERT INTO approx_count VALUES('2004-01-01 10:00:05', 5); INSERT INTO approx_count VALUES('2004-01-01 10:00:06', 6); INSERT INTO approx_count VALUES('2004-01-01 10:00:07', 7); INSERT INTO approx_count VALUES('2004-01-01 10:00:08', 8); INSERT INTO approx_count VALUES('2004-01-01 10:00:09', 9); INSERT INTO approx_count VALUES('2004-01-01 10:00:10', 10); SELECT count(*) FROM approx_count; SELECT * FROM approximate_row_count('approx_count'); ANALYZE approx_count; SELECT * FROM approximate_row_count('approx_count'); \set ON_ERROR_STOP 0 SELECT * FROM approximate_row_count('unexisting'); SELECT * FROM approximate_row_count(); SELECT * FROM approximate_row_count(NULL); \set ON_ERROR_STOP 1 -- Test size functions with invalid or non-existing OID SELECT * FROM hypertable_size(0); SELECT * FROM hypertable_detailed_size(0) ORDER BY node_name; SELECT * FROM chunks_detailed_size(0) ORDER BY node_name; SELECT * FROM hypertable_compression_stats(0) ORDER BY node_name; SELECT * FROM chunk_compression_stats(0) ORDER BY node_name; SELECT * FROM hypertable_index_size(0); SELECT * FROM _timescaledb_functions.relation_size(0); SELECT * FROM hypertable_size(1); SELECT * FROM hypertable_detailed_size(1) ORDER BY node_name; SELECT * FROM chunks_detailed_size(1) ORDER BY node_name; SELECT * FROM hypertable_compression_stats(1) ORDER BY node_name; SELECT * FROM chunk_compression_stats(1) ORDER BY node_name; SELECT * FROM hypertable_index_size(1); SELECT * FROM _timescaledb_functions.relation_size(1); -- Test size functions with NULL input SELECT * FROM hypertable_size(NULL); SELECT * FROM hypertable_detailed_size(NULL) ORDER BY node_name; SELECT * FROM chunks_detailed_size(NULL) ORDER BY node_name; SELECT * FROM hypertable_compression_stats(NULL) ORDER BY node_name; SELECT * FROM chunk_compression_stats(NULL) ORDER BY node_name; SELECT * FROM hypertable_index_size(NULL); SELECT * FROM _timescaledb_functions.relation_size(NULL); -- Test approximate size functions with invalid input SELECT * FROM hypertable_approximate_size(0); SELECT * FROM hypertable_approximate_detailed_size(0); SELECT * FROM _timescaledb_functions.relation_approximate_size(0); SELECT * FROM hypertable_approximate_size(NULL); SELECT * FROM hypertable_approximate_detailed_size(NULL); SELECT * FROM _timescaledb_functions.relation_approximate_size(NULL); -- Test size on view, sequence and composite type CREATE VIEW view1 as SELECT 1; SELECT * FROM _timescaledb_functions.relation_approximate_size('view1'); SELECT * FROM _timescaledb_functions.relation_size('view1'); CREATE SEQUENCE test_id_seq INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 9223372036854775807 CACHE 1; SELECT * FROM _timescaledb_functions.relation_approximate_size('test_id_seq'); SELECT * FROM _timescaledb_functions.relation_size('test_id_seq'); CREATE TYPE test_type AS (time timestamp, temp float); SELECT * FROM _timescaledb_functions.relation_approximate_size('test_type'); SELECT * FROM _timescaledb_functions.relation_size('test_type'); -- Test size functions on regular table CREATE TABLE hypersize(time timestamptz, device int); CREATE INDEX hypersize_time_idx ON hypersize (time); \set ON_ERROR_STOP 0 \set VERBOSITY default \set SHOW_CONTEXT never SELECT pg_relation_size('hypersize'), pg_table_size('hypersize'), pg_indexes_size('hypersize'), pg_total_relation_size('hypersize'), pg_relation_size('hypersize_time_idx'); SELECT * FROM _timescaledb_functions.relation_size('hypersize'); SELECT * FROM hypertable_size('hypersize'); SELECT * FROM hypertable_detailed_size('hypersize') ORDER BY node_name; SELECT * FROM chunks_detailed_size('hypersize') ORDER BY node_name; SELECT * FROM hypertable_compression_stats('hypersize') ORDER BY node_name; SELECT * FROM chunk_compression_stats('hypersize') ORDER BY node_name; SELECT * FROM hypertable_index_size('hypersize_time_idx'); SELECT * FROM _timescaledb_functions.relation_approximate_size('hypersize'); SELECT * FROM hypertable_approximate_size('hypersize'); SELECT * FROM hypertable_approximate_detailed_size('hypersize'); \set VERBOSITY terse \set ON_ERROR_STOP 1 -- Test size functions on empty hypertable SELECT * FROM create_hypertable('hypersize', 'time'); SELECT pg_relation_size('hypersize'), pg_table_size('hypersize'), pg_indexes_size('hypersize'), pg_total_relation_size('hypersize'), pg_relation_size('hypersize_time_idx'); SELECT * FROM _timescaledb_functions.relation_size('hypersize'); SELECT * FROM hypertable_size('hypersize'); SELECT * FROM hypertable_detailed_size('hypersize') ORDER BY node_name; SELECT * FROM chunks_detailed_size('hypersize') ORDER BY node_name; SELECT * FROM hypertable_compression_stats('hypersize') ORDER BY node_name; SELECT * FROM chunk_compression_stats('hypersize') ORDER BY node_name; SELECT * FROM hypertable_index_size('hypersize_time_idx'); SELECT * FROM _timescaledb_functions.relation_approximate_size('hypersize'); SELECT * FROM hypertable_approximate_size('hypersize'); SELECT * FROM hypertable_approximate_detailed_size('hypersize'); -- Test size functions on non-empty hypertable INSERT INTO hypersize VALUES('2021-02-25', 1); SELECT pg_relation_size('hypersize'), pg_table_size('hypersize'), pg_indexes_size('hypersize'), pg_total_relation_size('hypersize'), pg_relation_size('hypersize_time_idx'); SELECT pg_relation_size(ch), pg_table_size(ch), pg_indexes_size(ch), pg_total_relation_size(ch) FROM show_chunks('hypersize') ch ORDER BY ch; SELECT * FROM show_chunks('hypersize') ch JOIN LATERAL _timescaledb_functions.relation_size(ch) ON true; SELECT * FROM hypertable_size('hypersize'); SELECT * FROM hypertable_detailed_size('hypersize') ORDER BY node_name; SELECT * FROM chunks_detailed_size('hypersize') ORDER BY node_name; SELECT * FROM hypertable_compression_stats('hypersize') ORDER BY node_name; SELECT * FROM chunk_compression_stats('hypersize') ORDER BY node_name; SELECT * FROM hypertable_index_size('hypersize_time_idx'); SELECT * FROM _timescaledb_functions.relation_approximate_size('hypersize'); SELECT * FROM hypertable_approximate_size('hypersize'); SELECT * FROM hypertable_approximate_detailed_size('hypersize'); -- Test approx size functions with toast entries SELECT * FROM _timescaledb_functions.relation_approximate_size('toast_test'); SELECT * FROM hypertable_approximate_size('toast_test'); SELECT * FROM hypertable_approximate_detailed_size('toast_test'); -- Test approx size function against a regular table \set ON_ERROR_STOP 0 CREATE TABLE regular(time TIMESTAMP, value TEXT); SELECT * FROM hypertable_approximate_size('regular'); \set ON_ERROR_STOP 1 -- Test approx size functions with dropped chunks CREATE TABLE drop_chunks_table(time BIGINT NOT NULL, data INTEGER); SELECT hypertable_id AS drop_chunks_table_id FROM create_hypertable('drop_chunks_table', 'time', chunk_time_interval => 10) \gset INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(0, 29) AS i; SELECT * FROM hypertable_approximate_size('drop_chunks_table'); SELECT drop_chunks('drop_chunks_table', older_than => 19); SELECT * FROM hypertable_approximate_size('drop_chunks_table'); -- github issue #4857 -- below procedure should not crash SET client_min_messages = ERROR; do $$ DECLARE o INT; BEGIN FOR c IN 1..20 LOOP ANALYZE; END LOOP; END; $$; RESET client_min_messages; ================================================ FILE: test/sql/sort_optimization.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set PREFIX 'EXPLAIN (BUFFERS OFF, COSTS OFF) ' CREATE TABLE order_test(time int NOT NULL, device_id int, value float); CREATE INDEX ON order_test(time,device_id); CREATE INDEX ON order_test(device_id,time); SELECT create_hypertable('order_test','time',chunk_time_interval:=1000); INSERT INTO order_test SELECT 0,10,0.5; INSERT INTO order_test SELECT 1,9,0.5; INSERT INTO order_test SELECT 2,8,0.5; -- we want to see here that index scans are possible for the chosen expressions -- so we disable seqscan so we dont need to worry about other factors which would -- make PostgreSQL prefer seqscan over index scan SET enable_seqscan TO off; -- test sort optimization with single member order by SELECT time_bucket(10,time),device_id,value FROM order_test ORDER BY 1; -- should use index scan :PREFIX SELECT time_bucket(10,time),device_id,value FROM order_test ORDER BY 1; -- test sort optimization with ordering by multiple columns and time_bucket not last SELECT time_bucket(10,time),device_id,value FROM order_test ORDER BY 1,2; SET enable_seqscan TO default; -- must not use index scan :PREFIX SELECT time_bucket(10,time),device_id,value FROM order_test ORDER BY 1,2; SET enable_seqscan TO off; -- test sort optimization with ordering by multiple columns and time_bucket as last member SELECT time_bucket(10,time),device_id,value FROM order_test ORDER BY 2,1; -- should use index scan :PREFIX SELECT time_bucket(10,time),device_id,value FROM order_test ORDER BY 2,1; -- test sort optimization with interval calculation with non-fixed interval -- #7097 CREATE TABLE i7097_1(time timestamptz NOT NULL, quantity float, "isText" boolean); CREATE TABLE i7097_2(time timestamptz NOT NULL, quantity float, "isText" boolean); CREATE INDEX ON i7097_1(time) WHERE "isText" IS NULL; CREATE INDEX ON i7097_2(time) WHERE "isText" IS NULL; SELECT table_name FROM create_hypertable('i7097_1', 'time', create_default_indexes => false); SELECT table_name FROM create_hypertable('i7097_2', 'time', create_default_indexes => false); INSERT INTO i7097_1(time, quantity) SELECT time, round((random() * (100-3) + 3)::NUMERIC) AS quantity FROM generate_series('2023-01-01T00:00:00+01:00', '2023-05-01T00:00:00+01:00', interval 'PT10M') AS t(time); INSERT INTO i7097_2(time, quantity) SELECT time, round((random() * (100-3) + 3)::NUMERIC) AS quantity FROM generate_series('2023-01-01T00:00:00+01:00', '2023-05-01T00:00:00+01:00', interval 'PT10M') AS t(time); VACUUM ANALYZE i7097_1, i7097_2; SET TIME ZONE 'Europe/Paris'; WITH "cte1" AS (SELECT time + interval 'P1Y' AS time, avg(quantity) AS quantity FROM i7097_1 WHERE time >= '2024-03-31T00:00:00+01:00'::timestamptz - interval 'P1Y' AND time < '2024-03-31T23:59:59+02:00'::timestamptz + (- interval 'P1Y') AND "isText" IS NULL GROUP BY 1 ORDER BY 1 ASC), "cte2" AS (SELECT time + interval 'P1Y' AS time, avg(quantity) AS quantity FROM i7097_2 WHERE time >= '2024-03-31T00:00:00+01:00'::timestamptz - interval 'P1Y' AND time < '2024-03-31T23:59:59+02:00'::timestamptz + (- interval 'P1Y') AND "isText" IS NULL GROUP BY 1 ORDER BY 1 ASC) SELECT count(*) FROM (SELECT time, cte1.quantity + cte2.quantity FROM cte1 FULL OUTER JOIN cte2 USING (time)) j; -- github issue 9214 -- test off-by one error in sort optimization CREATE TABLE i9214(time timestamptz NOT NULL, machine_id INT NOT NULL, name TEXT NOT NULL, value FLOAT NOT NULL) WITH (tsdb.hypertable); INSERT INTO i9214 VALUES ('2026-01-30 10:00:00+00', 1, 'tag1', 10.5), ('2026-01-30 10:00:00+00', 1, 'tag2', 20.5), ('2026-01-30 10:01:00+00', 1, 'tag1', 11.0), ('2026-01-30 10:01:00+00', 1, 'tag2', 21.0); WITH rule1 AS ( SELECT date_trunc('minute', time) AS time, machine_id FROM i9214 WHERE machine_id = 1 AND name = 'tag1' AND value > 5 ), row_numbered AS ( SELECT time, machine_id, row_number() OVER (ORDER BY time) AS seqnum FROM rule1 ) SELECT min(time) AS start_time, machine_id, count(*) AS duration_minutes FROM row_numbered GROUP BY machine_id, (time - (seqnum * interval '1 minute')) ORDER BY min(time); WITH rule1 AS ( SELECT date_trunc('minute', time) AS time, machine_id FROM i9214 WHERE machine_id = 1 AND name = 'tag1' AND value > 5 ), rule2 AS ( SELECT date_trunc('minute', time) AS time, machine_id FROM i9214 WHERE machine_id = 1 AND name = 'tag2' AND value > 5 ), joined_rules AS ( SELECT r1.time, r1.machine_id FROM rule1 r1 INNER JOIN rule2 r2 USING(time,machine_id) ), row_numbered AS ( SELECT time, machine_id, row_number() OVER (ORDER BY time) AS seqnum FROM joined_rules ) SELECT min(time) AS start_time, machine_id, count(*) AS duration_minutes FROM row_numbered GROUP BY machine_id, (time - (seqnum * interval '1 minute')) ORDER BY min(time); ================================================ FILE: test/sql/sql_query.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \o /dev/null \ir include/insert_two_partitions.sql \o SELECT * FROM PUBLIC."two_Partitions"; EXPLAIN (verbose ON, buffers off, costs off) SELECT * FROM PUBLIC."two_Partitions"; \echo "The following queries should NOT scan two_Partitions._hyper_1_1_chunk" EXPLAIN (verbose ON, buffers off, costs off) SELECT * FROM PUBLIC."two_Partitions" WHERE device_id = 'dev2'; EXPLAIN (verbose ON, buffers off, costs off) SELECT * FROM PUBLIC."two_Partitions" WHERE device_id = 'dev'||'2'; EXPLAIN (verbose ON, buffers off, costs off) SELECT * FROM PUBLIC."two_Partitions" WHERE 'dev'||'2' = device_id; --test integer partition key CREATE TABLE "int_part"(time timestamp, object_id int, temp float); SELECT create_hypertable('"int_part"', 'time', 'object_id', 2); INSERT INTO "int_part" VALUES('2017-01-20T09:00:01', 1, 22.5); INSERT INTO "int_part" VALUES('2017-01-20T09:00:01', 2, 22.5); --check that there are two chunks SELECT * FROM test.show_subtables('int_part'); SELECT * FROM "int_part" WHERE object_id = 1; --check that queries with IN/ANY/= work for the "time" column SELECT * FROM "int_part" WHERE time IN (NULL); SELECT * FROM "int_part" WHERE time = ANY (NULL); SELECT * FROM "int_part" WHERE time = NULL; --make sure this touches only one partititon EXPLAIN (verbose ON, buffers off, costs off) SELECT * FROM "int_part" WHERE object_id = 1; --Need to verify space partitions are currently pruned in this query --EXPLAIN (verbose ON, buffers off, costs off) SELECT * FROM "two_Partitions" WHERE device_id IN ('dev2', 'dev21'); \echo "The following shows non-aggregated queries with time desc using merge append" EXPLAIN (verbose ON, buffers off, costs off)SELECT * FROM PUBLIC."two_Partitions" ORDER BY "timeCustom" DESC NULLS LAST limit 2; --shows that more specific indexes are used if the WHERE clauses "match", uses the series_1 index here. EXPLAIN (verbose ON, buffers off, costs off)SELECT * FROM PUBLIC."two_Partitions" WHERE series_1 IS NOT NULL ORDER BY "timeCustom" DESC NULLS LAST limit 2; --here the "match" is implication series_1 > 1 => series_1 IS NOT NULL EXPLAIN (verbose ON, buffers off, costs off)SELECT * FROM PUBLIC."two_Partitions" WHERE series_1 > 1 ORDER BY "timeCustom" DESC NULLS LAST limit 2; --note that without time transform things work too EXPLAIN (verbose ON, buffers off, costs off)SELECT "timeCustom" t, min(series_0) FROM PUBLIC."two_Partitions" GROUP BY t ORDER BY t DESC NULLS LAST limit 2; --The query should still use the index on timeCustom, even though the GROUP BY/ORDER BY is on the transformed time 't'. --However, current query plans show that it does not. EXPLAIN (verbose ON, buffers off, costs off)SELECT "timeCustom"/10 t, min(series_0) FROM PUBLIC."two_Partitions" GROUP BY t ORDER BY t DESC NULLS LAST limit 2; EXPLAIN (verbose ON, buffers off, costs off)SELECT "timeCustom"%10 t, min(series_0) FROM PUBLIC."two_Partitions" GROUP BY t ORDER BY t DESC NULLS LAST limit 2; ================================================ FILE: test/sql/symbol_conflict.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER -- Test for symbol conflicts between the loader module and the -- versioned extension module. -- This test fails on, e.g. Linux, unless compiled with -fvisibility=hidden CREATE OR REPLACE FUNCTION hello_loader() RETURNS TEXT AS 'timescaledb', 'loader_hello' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; SELECT hello_loader(); CREATE OR REPLACE FUNCTION hello_timescaledb() RETURNS TEXT AS :MODULE_PATHNAME, 'timescaledb_hello' LANGUAGE C IMMUTABLE PARALLEL SAFE STRICT; -- This calls an internal function with a conflicting name in the loader SELECT hello_loader(); -- This calls the identically named internal function in the versioned extension SELECT hello_timescaledb(); ================================================ FILE: test/sql/tableam.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test support for setting table access method on hypertables \c :TEST_DBNAME :ROLE_SUPERUSER -- create a new access method that reuses the heap handler CREATE ACCESS METHOD testam TYPE TABLE HANDLER heap_tableam_handler; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE TABLE testam (time timestamptz, device int, temp float) USING testam; SELECT create_hypertable('testam', 'time', 'device', 2); -- show that the hypertable is using the 'testam' table access method SELECT amname AS hypertable_amname FROM pg_class cl, pg_am am WHERE cl.oid = 'testam'::regclass AND cl.relam = am.oid; -- insert data to create a chunk INSERT INTO testam VALUES('2020-01-22:11:30', 1, 29.3); -- make sure the table access method for a chunk is the same as the -- hypertable root SELECT amname AS chunk_amname FROM pg_class cl, pg_am am, show_chunks('testam') ch WHERE cl.oid = ch AND cl.relam = am.oid; ================================================ FILE: test/sql/tableam_alter.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test support for setting table access method on hypertables using -- ALTER TABLE. It should propagate to the chunks. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE ACCESS METHOD testam TYPE TABLE HANDLER heap_tableam_handler; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE VIEW chunk_info AS SELECT hypertable_name AS hypertable, chunk_name AS chunk, amname FROM timescaledb_information.chunks ch JOIN pg_class cl ON (format('%I.%I', ch.chunk_schema, ch.chunk_name)::regclass = cl.oid) JOIN pg_am am ON (am.oid = cl.relam); CREATE TABLE test_table (time timestamptz not null, device int, temp float); SELECT create_hypertable('test_table', by_range('time')); INSERT INTO test_table SELECT ts, 10 * random(), 100 * random() FROM generate_series('2001-01-01'::timestamp, '2001-02-01', '1d'::interval) as x(ts); SELECT cl.relname, amname FROM pg_class cl JOIN pg_am am ON cl.relam = am.oid WHERE cl.relname = 'test_table'; SELECT * FROM chunk_info WHERE hypertable = 'test_table'; -- Test setting the access method together with other options. This -- should not generate an error. ALTER TABLE test_table SET ACCESS METHOD testam, SET (autovacuum_vacuum_threshold = 100); -- Create more chunks. These will use the new access method, but the -- old chunks will use the old access method. INSERT INTO test_table SELECT ts, 10 * random(), 100 * random() FROM generate_series('2001-02-01'::timestamp, '2001-03-01', '1d'::interval) as x(ts); SELECT cl.relname, amname FROM pg_class cl JOIN pg_am am ON cl.relam = am.oid WHERE cl.relname = 'test_table'; SELECT * FROM chunk_info WHERE hypertable = 'test_table'; ================================================ FILE: test/sql/tableam_alter_defaults.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test support for setting table access method on hypertables using -- ALTER TABLE for version 17 and later. This is in addition to the -- tests in tableam_alter.sql. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE ACCESS METHOD testam TYPE TABLE HANDLER heap_tableam_handler; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE VIEW chunk_info AS SELECT hypertable_name AS hypertable, chunk_name AS chunk, amname FROM timescaledb_information.chunks ch JOIN pg_class cl ON (format('%I.%I', ch.chunk_schema, ch.chunk_name)::regclass = cl.oid) JOIN pg_am am ON (am.oid = cl.relam); CREATE TABLE test_table (time timestamptz not null, device int, temp float); SELECT cl.relname, amname FROM pg_class cl JOIN pg_am am ON cl.relam = am.oid WHERE cl.relname = 'test_table'; -- Check that setting default access method of a normal table works. ALTER TABLE test_table SET ACCESS METHOD DEFAULT; -- Check that changing the access method and then changing it back -- works. ALTER TABLE test_table SET ACCESS METHOD testam; ALTER TABLE test_table SET ACCESS METHOD DEFAULT; -- Check that setting default access method of a hypertable works. SELECT create_hypertable('test_table', by_range('time')); ALTER TABLE test_table SET ACCESS METHOD DEFAULT; SELECT cl.relname, amname FROM pg_class cl JOIN pg_am am ON cl.relam = am.oid WHERE cl.relname = 'test_table'; -- Test setting the access method together with other options. This -- should not generate an error. ALTER TABLE test_table SET ACCESS METHOD testam, SET (autovacuum_vacuum_threshold = 100); -- Add some rows to generate a chunk. This should get the access -- method of the hypertable. INSERT INTO test_table SELECT ts, 10 * random(), 100 * random() FROM generate_series('2001-01-01'::timestamp, '2001-01-14', '1d'::interval) as x(ts); SELECT * FROM chunk_info WHERE hypertable = 'test_table'; -- Setting it to the default method after we have set it to a test -- access method should work fine also on a hypertable. SELECT cl.relname, amname FROM pg_class cl JOIN pg_am am ON cl.relam = am.oid WHERE cl.relname = 'test_table'; ALTER TABLE test_table SET ACCESS METHOD DEFAULT; SELECT cl.relname, amname FROM pg_class cl JOIN pg_am am ON cl.relam = am.oid WHERE cl.relname = 'test_table'; SELECT chunk FROM show_chunks('test_table') t(chunk) limit 1 \gset ALTER TABLE :chunk SET ACCESS METHOD DEFAULT; SELECT * FROM chunk_info WHERE hypertable = 'test_table'; ================================================ FILE: test/sql/tablespace.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set ON_ERROR_STOP 0 \c :TEST_DBNAME :ROLE_SUPERUSER CREATE VIEW hypertable_tablespaces AS SELECT cls.relname AS hypertable, (SELECT spcname FROM pg_tablespace WHERE oid = reltablespace) AS tablespace FROM _timescaledb_catalog.hypertable, LATERAL (SELECT * FROM pg_class WHERE oid = format('%I.%I', schema_name, table_name)::regclass) AS cls ORDER BY hypertable, tablespace; GRANT SELECT ON hypertable_tablespaces TO PUBLIC; --Test hypertable with tablespace. Tablespaces are cluster-wide, so we --attach the test name as prefix to allow tests to be executed in --parallel. CREATE TABLESPACE tablespace1 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE1_PATH; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --assigning a tablespace via the main table should work CREATE TABLE tspace_2dim(time timestamp, temp float, device text) TABLESPACE tablespace1; SELECT create_hypertable('tspace_2dim', 'time', 'device', 2); INSERT INTO tspace_2dim VALUES ('2017-01-20T09:00:01', 24.3, 'blue'); -- Tablespace for tspace_2dim should be set SELECT * FROM hypertable_tablespaces WHERE hypertable = 'tspace_2dim'; SELECT show_tablespaces('tspace_2dim'); --verify that the table chunk has the correct tablespace SELECT relname, spcname FROM pg_class c INNER JOIN pg_tablespace t ON (c.reltablespace = t.oid) INNER JOIN _timescaledb_catalog.chunk ch ON (ch.table_name = c.relname); --check some error conditions SELECT attach_tablespace(NULL,NULL); SELECT attach_tablespace('tablespace2', NULL); SELECT attach_tablespace(NULL, 'tspace_2dim'); SELECT attach_tablespace('none_existing_tablespace', 'tspace_2dim'); SELECT attach_tablespace('tablespace2', 'none_existing_table'); SELECT detach_tablespace(NULL); SELECT detach_tablespaces(NULL); SELECT show_tablespaces(NULL); --attach another tablespace without first creating it --> should generate error SELECT attach_tablespace('tablespace2', 'tspace_2dim'); --attach the same tablespace twice to same table should also generate error SELECT attach_tablespace('tablespace1', 'tspace_2dim'); --no error if if_not_attached is given SELECT attach_tablespace('tablespace1', 'tspace_2dim', if_not_attached => true); \c :TEST_DBNAME :ROLE_SUPERUSER --Tablespaces are cluster-wide, so we attach the test name as prefix --to allow tests to be executed in parallel. CREATE TABLESPACE tablespace2 OWNER :ROLE_DEFAULT_PERM_USER_2 LOCATION :TEST_TABLESPACE2_PATH; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 --attach without permissions on the table should fail SELECT attach_tablespace('tablespace2', 'tspace_2dim'); \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --attach without permissions on the tablespace should also fail SELECT attach_tablespace('tablespace2', 'tspace_2dim'); \c :TEST_DBNAME :ROLE_SUPERUSER GRANT :ROLE_DEFAULT_PERM_USER_2 TO :ROLE_DEFAULT_PERM_USER; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --should work with permissions on both the table and the tablespace SELECT attach_tablespace('tablespace2', 'tspace_2dim'); SELECT * FROM _timescaledb_catalog.tablespace; SELECT * FROM show_tablespaces('tspace_2dim'); --insert into another chunk INSERT INTO tspace_2dim VALUES ('2017-01-20T09:00:01', 24.3, 'brown'); SELECT * FROM test.show_subtables('tspace_2dim'); --indexes should inherit the tablespace of their chunk SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); \x SELECT * FROM timescaledb_information.hypertables ORDER BY hypertable_schema, hypertable_name; SELECT hypertable_schema, hypertable_name, chunk_schema, chunk_name, chunk_tablespace FROM timescaledb_information.chunks ORDER BY chunk_name; \x -- SET ROLE :ROLE_DEFAULT_PERM_USER_2; CREATE TABLE tspace_1dim(time timestamp, temp float, device text); SELECT create_hypertable('tspace_1dim', 'time'); --user doesn't have permission on tablespace1 --> error SELECT attach_tablespace('tablespace1', 'tspace_1dim'); --grant permission to tablespace1 SET ROLE :ROLE_DEFAULT_PERM_USER; GRANT CREATE ON TABLESPACE tablespace1 TO :ROLE_DEFAULT_PERM_USER_2; SET ROLE :ROLE_DEFAULT_PERM_USER_2; --should work fine now. Test SELECT INTO utility statements to ensure --internal alter table function call works with event triggers. SELECT true INTO attached FROM attach_tablespace('tablespace1', 'tspace_1dim'); SELECT attach_tablespace('tablespace2', 'tspace_1dim'); -- Tablespace for tspace_1dim should be set and attached SELECT * FROM hypertable_tablespaces WHERE hypertable = 'tspace_1dim'; SELECT show_tablespaces('tspace_1dim'); --trying to revoke permissions while attached should fail SET ROLE :ROLE_DEFAULT_PERM_USER; REVOKE CREATE ON TABLESPACE tablespace1 FROM :ROLE_DEFAULT_PERM_USER_2; REVOKE ALL ON TABLESPACE tablespace1 FROM :ROLE_DEFAULT_PERM_USER_2; SET ROLE :ROLE_DEFAULT_PERM_USER_2; SELECT * FROM _timescaledb_catalog.tablespace; INSERT INTO tspace_1dim VALUES ('2017-01-20T09:00:01', 24.3, 'blue'); INSERT INTO tspace_1dim VALUES ('2017-03-20T09:00:01', 24.3, 'brown'); SELECT * FROM test.show_subtablesp('tspace_%'); --indexes should inherit the tablespace of their chunk, unless the --parent index has a tablespace set, in which case the chunks' --corresponding indexes are pinned to the parent index's --tablespace. The parent index can have a tablespace set in two cases: --(1) if explicitly set in CREATE INDEX, or (2) if the main table was --created with a tablespace, because then default indexes will be --created in that tablespace too. SELECT * FROM test.show_indexesp('_timescaledb_internal._hyper%_chunk'); --detach tablespace1 from tspace_2dim should fail due to lack of permissions SELECT detach_tablespace('tablespace1', 'tspace_2dim'); --detach tablespace1 from all tables. Should only detach from --'tspace_1dim' (1 tablespace) due to lack of permissions SELECT * FROM hypertable_tablespaces; SELECT * INTO detached FROM detach_tablespace('tablespace1'); SELECT * FROM detached; SELECT * FROM _timescaledb_catalog.tablespace; SELECT * FROM show_tablespaces('tspace_1dim'); SELECT * FROM show_tablespaces('tspace_2dim'); SELECT * FROM hypertable_tablespaces; --it should now be possible to revoke permissions on tablespace1 SET ROLE :ROLE_DEFAULT_PERM_USER; REVOKE CREATE ON TABLESPACE tablespace1 FROM :ROLE_DEFAULT_PERM_USER_2; SET ROLE :ROLE_DEFAULT_PERM_USER_2; --detach the other tablespace SELECT detach_tablespace('tablespace2', 'tspace_1dim'); SELECT * FROM _timescaledb_catalog.tablespace; SELECT * FROM show_tablespaces('tspace_1dim'); SELECT * FROM show_tablespaces('tspace_2dim'); SELECT * FROM hypertable_tablespaces; --detaching tablespace2 from a table without permissions should fail SELECT detach_tablespace('tablespace2', 'tspace_2dim'); SELECT detach_tablespaces('tspace_2dim'); \c :TEST_DBNAME :ROLE_SUPERUSER -- PERM_USER_2 owns tablespace2, and PERM_USER owns the table -- 'tspace_2dim', which has tablespace2 attached. Revoking PERM_USER_2 -- FROM PERM_USER should therefore fail REVOKE :ROLE_DEFAULT_PERM_USER_2 FROM :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER_2; --set other user should make detach work SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT * INTO detached_all FROM detach_tablespaces('tspace_2dim'); SELECT * FROM detached_all; SELECT * FROM _timescaledb_catalog.tablespace; SELECT * FROM show_tablespaces('tspace_1dim'); SELECT * FROM show_tablespaces('tspace_2dim'); \c :TEST_DBNAME :ROLE_SUPERUSER -- It should now be possible to revoke PERM_USER_2 from PERM_USER -- since tablespace2 is no longer attched to tspace_2dim REVOKE :ROLE_DEFAULT_PERM_USER_2 FROM :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; --detaching twice should fail SELECT detach_tablespace('tablespace2', 'tspace_2dim'); --adding if_attached should only generate notice SELECT detach_tablespace('tablespace2', 'tspace_2dim', if_attached => true); --attach tablespaces again to verify that tablespaces are cleaned up --when tables are dropped \c :TEST_DBNAME :ROLE_SUPERUSER SELECT attach_tablespace('tablespace2', 'tspace_1dim'); SELECT attach_tablespace('tablespace1', 'tspace_2dim'); SELECT * FROM _timescaledb_catalog.tablespace; DROP TABLE tspace_1dim; SELECT * FROM _timescaledb_catalog.tablespace; DROP TABLE tspace_2dim; SELECT * FROM _timescaledb_catalog.tablespace; -- Create two tables and attach multiple tablespaces to them. Verify -- that dropping a tablespace from multiple tables work as expected. CREATE TABLE tbl_1(time timestamp, temp float, device text); SELECT create_hypertable('tbl_1', 'time'); CREATE TABLE tbl_2(time timestamp, temp float, device text); SELECT create_hypertable('tbl_2', 'time'); CREATE TABLE tbl_3(time timestamp, temp float, device text); SELECT create_hypertable('tbl_3', 'time'); SELECT * FROM hypertable_tablespaces; SELECT * FROM show_tablespaces('tbl_1'); SELECT * FROM show_tablespaces('tbl_2'); SELECT * FROM show_tablespaces('tbl_3'); SELECT attach_tablespace('tablespace1', 'tbl_1'); SELECT attach_tablespace('tablespace2', 'tbl_1'); SELECT attach_tablespace('tablespace2', 'tbl_2'); SELECT attach_tablespace('tablespace2', 'tbl_3'); SELECT * FROM hypertable_tablespaces; SELECT * FROM show_tablespaces('tbl_1'); SELECT * FROM show_tablespaces('tbl_2'); SELECT * FROM show_tablespaces('tbl_3'); SELECT detach_tablespace('tablespace2'); SELECT * FROM hypertable_tablespaces; SELECT * FROM show_tablespaces('tbl_1'); SELECT * FROM show_tablespaces('tbl_2'); SELECT * FROM show_tablespaces('tbl_3'); DROP TABLE tbl_1; DROP TABLE tbl_2; DROP TABLE tbl_3; -- verify that one cannot DROP a tablespace while it is attached to a -- hypertable CREATE TABLE tbl_1(time timestamp, temp float, device text); SELECT create_hypertable('tbl_1', 'time'); SELECT attach_tablespace('tablespace1', 'tbl_1'); SELECT * FROM show_tablespaces('tbl_1'); DROP TABLESPACE tablespace1; --after detaching we should now be able to drop the tablespace SELECT detach_tablespace('tablespace1', 'tbl_1'); DROP TABLESPACE tablespace1; DROP TABLESPACE tablespace2; ================================================ FILE: test/sql/telemetry.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION _timescaledb_internal.test_status(int) RETURNS JSONB AS :MODULE_PATHNAME, 'ts_test_status' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_status_ssl(int) RETURNS JSONB AS :MODULE_PATHNAME, 'ts_test_status_ssl' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_status_mock(text) RETURNS JSONB AS :MODULE_PATHNAME, 'ts_test_status_mock' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_validate_server_version(response text) RETURNS TEXT AS :MODULE_PATHNAME, 'ts_test_validate_server_version' LANGUAGE C IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_telemetry_main_conn(text, text) RETURNS BOOLEAN AS :MODULE_PATHNAME, 'ts_test_telemetry_main_conn' LANGUAGE C IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_telemetry(host text = NULL, servname text = NULL, port int = NULL) RETURNS JSONB AS :MODULE_PATHNAME, 'ts_test_telemetry' LANGUAGE C IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION _timescaledb_internal.test_privacy() RETURNS BOOLEAN AS :MODULE_PATHNAME, 'ts_test_privacy' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION test_check_version_response(response text) RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_check_version_response' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; INSERT INTO _timescaledb_catalog.metadata VALUES ('foo','bar',TRUE); INSERT INTO _timescaledb_catalog.metadata VALUES ('bar','baz',FALSE); \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT _timescaledb_internal.test_status_ssl(200); SELECT _timescaledb_internal.test_status_ssl(201); \set ON_ERROR_STOP 0 SELECT _timescaledb_internal.test_status_ssl(304); SELECT _timescaledb_internal.test_status_ssl(400); SELECT _timescaledb_internal.test_status_ssl(401); SELECT _timescaledb_internal.test_status_ssl(404); SELECT _timescaledb_internal.test_status_ssl(500); SELECT _timescaledb_internal.test_status_ssl(503); \set ON_ERROR_STOP 1 -- This function runs the test 5 times, because each time the internal C function is choosing a random length to send from the server on each socket read. We hit many cases this way. CREATE OR REPLACE FUNCTION mocker(TEXT) RETURNS SETOF TEXT AS $BODY$ DECLARE r TEXT; BEGIN FOR i in 1..5 LOOP SELECT _timescaledb_internal.test_status_mock($1) INTO r; RETURN NEXT r; END LOOP; RETURN; END $BODY$ LANGUAGE 'plpgsql'; select * from mocker( E'HTTP/1.1 200 OK\r\n' 'Content-Type: application/json; charset=utf-8\r\n' 'Date: Thu, 12 Jul 2018 18:33:04 GMT\r\n' 'ETag: W/\"e-upYEWCL+q6R/++2nWHz5b76hBgo\"\r\n' 'Server: nginx\r\n' 'Vary: Accept-Encoding\r\n' 'Content-Length: 14\r\n' 'Connection: Close\r\n\r\n' '{\"status\":200}'); select * from mocker( E'HTTP/1.1 201 OK\r\n' 'Content-Type: application/json; charset=utf-8\r\n' 'Vary: Accept-Encoding\r\n' 'Content-Length: 14\r\n' 'Connection: Close\r\n\r\n' '{\"status\":201}'); \set ON_ERROR_STOP 0 \set test_string 'HTTP/1.1 404 Not Found\r\nContent-Length: 14\r\nConnection: Close\r\n\r\n{\"status\":404}'; SELECT _timescaledb_internal.test_status_mock(:'test_string'); SELECT _timescaledb_internal.test_status_mock(:'test_string'); SELECT _timescaledb_internal.test_status_mock(:'test_string'); SELECT _timescaledb_internal.test_status_mock(:'test_string'); SELECT _timescaledb_internal.test_status_mock(:'test_string'); \set test_string 'Content-Length: 14\r\nConnection: Close\r\n\r\n{\"status\":404}'; SELECT _timescaledb_internal.test_status_mock(:'test_string'); SELECT _timescaledb_internal.test_status_mock(:'test_string'); SELECT _timescaledb_internal.test_status_mock(:'test_string'); SELECT _timescaledb_internal.test_status_mock(:'test_string'); SELECT _timescaledb_internal.test_status_mock(:'test_string'); \set test_string 'HTTP/1.1 404 Not Found\r\nContent-Length: 14\r\nConnection: Close\r\n{\"status\":404}'; SELECT _timescaledb_internal.test_status_mock(:'test_string'); SELECT _timescaledb_internal.test_status_mock(:'test_string'); SELECT _timescaledb_internal.test_status_mock(:'test_string'); SELECT _timescaledb_internal.test_status_mock(:'test_string'); SELECT _timescaledb_internal.test_status_mock(:'test_string'); \set ON_ERROR_STOP 1 -- Test parsing version response SELECT * FROM _timescaledb_internal.test_validate_server_version('{"status": "200", "current_timescaledb_version": "10.1.0"}'); SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "10.1"}'); SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "10"}'); SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "9.2.0"}'); SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "9.1.2"}'); SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "1.0.0"}'); SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "1.0.0-rc1"}'); SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "1.0.0-rc2"}'); SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "1.0.0-rc1"}'); SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "1.0.0-alpha"}'); SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "123456789"}'); SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "!@#$%"}'); SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": ""}'); SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": " 10 "}'); SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "a"}'); SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "a.b.c"}'); SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "10.1.1a"}'); SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "10.1.1+rc1"}'); SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "10.1.1.1"}'); SELECT * FROM _timescaledb_internal.test_validate_server_version('{"current_timescaledb_version": "1.0.0-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz"}'); ---------------------------------------------------------------- -- Test well-formed response and valid versions SELECT test_check_version_response('{"current_timescaledb_version": "1.6.1", "is_up_to_date": true}'); SELECT test_check_version_response('{"current_timescaledb_version": "1.6.1", "is_up_to_date": false}'); SELECT test_check_version_response('{"current_timescaledb_version": "10.1", "is_up_to_date": false}'); SELECT test_check_version_response('{"current_timescaledb_version": "10.1.1-rc1", "is_up_to_date": false}'); ---------------------------------------------------------------- -- Test well-formed response but invalid versions SELECT test_check_version_response('{"current_timescaledb_version": "1.0.0-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz", "is_up_to_date": false}'); SELECT test_check_version_response('{"current_timescaledb_version": "10.1.1+rc1", "is_up_to_date": false}'); SELECT test_check_version_response('{"current_timescaledb_version": "@10.1.1", "is_up_to_date": false}'); SELECT test_check_version_response('{"current_timescaledb_version": "10.1.1@", "is_up_to_date": false}'); ---------------------------------------------------------------- -- Test malformed responses \set ON_ERROR_STOP 0 -- Empty response SELECT test_check_version_response('{}'); -- Field "is_up_to_date" missing SELECT test_check_version_response('{"current_timescaledb_version": "1.6.1"}'); -- Field "current_timescaledb_version" is missing SELECT test_check_version_response('{"is_up_to_date": false}'); \set ON_ERROR_STOP 1 SET timescaledb.last_tune_time = '2024-01-01 00:00:00+00'; SET timescaledb.last_tune_version = '1.2.3'; SET timescaledb.telemetry_level=basic; -- Connect to a bogus host and path to test error handling in telemetry_main() SELECT _timescaledb_internal.test_telemetry_main_conn('noservice.timescale.com', 'path'); -- Test telemetry report contents SET timescaledb.telemetry_level=basic; SELECT * FROM jsonb_object_keys(get_telemetry_report()) AS key WHERE key != 'os_name_pretty'; CREATE MATERIALIZED VIEW telemetry_report AS SELECT t FROM get_telemetry_report() t; -- check telemetry picks up flagged content from metadata SELECT t -> 'db_metadata' FROM telemetry_report; -- check timescaledb_telemetry.cloud SELECT t -> 'instance_metadata' FROM telemetry_report; -- Check access methods SELECT t->'access_methods' ? 'btree', t->'access_methods' ? 'heap', CAST(t->'access_methods'->'btree'->'pages' AS int) > 0, CAST(t->'access_methods'->'btree'->'instances' AS int) > 0 FROM telemetry_report; WITH t AS ( SELECT t -> 'relations' AS rels FROM telemetry_report ) SELECT rels -> 'hypertables' -> 'num_relations' AS num_hypertables, rels -> 'continuous_aggregates' -> 'num_relations' AS num_caggs FROM t; CREATE TABLE device_readings ( observation_time TIMESTAMPTZ NOT NULL ); SELECT table_name FROM create_hypertable('device_readings', 'observation_time'); REFRESH MATERIALIZED VIEW telemetry_report; WITH t AS ( SELECT t -> 'relations' AS rels FROM telemetry_report ) SELECT rels -> 'hypertables' -> 'num_relations' AS num_hypertables, rels -> 'continuous_aggregates' -> 'num_relations' AS num_caggs FROM t; set datestyle to iso; -- check that installed_time formatting in telemetry report does not depend on local date settings SELECT t -> 'installed_time' AS installed_time FROM telemetry_report \gset set datestyle to sql; SELECT t-> 'installed_time' AS installed_time2 FROM telemetry_report \gset SELECT :'installed_time' = :'installed_time2' AS equal, length(:'installed_time'), length(:'installed_time2'); RESET datestyle; -- test function call telemetry CREATE FUNCTION not_visible_in_telemetry() RETURNS INT AS $$ SELECT 1; $$ LANGUAGE SQL; -- drain old function call telemetry so we have fixed out put; SELECT FROM get_telemetry_report(); -- call some arbirary functions SELECT 1 + 1, not_visible_in_telemetry(), 1 + 1, abs(-1), not_visible_in_telemetry() WHERE 1 + 1 = 2; -- call some aggregates SELECT min(not_visible_in_telemetry()), sum(not_visible_in_telemetry()); -- check that we can record from a prepared statement PREPARE record_from_prepared AS SELECT 1 - 1; -- execute 10 times to make sure turning it into generic plan works EXECUTE record_from_prepared; EXECUTE record_from_prepared; EXECUTE record_from_prepared; EXECUTE record_from_prepared; EXECUTE record_from_prepared; EXECUTE record_from_prepared; EXECUTE record_from_prepared; EXECUTE record_from_prepared; EXECUTE record_from_prepared; EXECUTE record_from_prepared; DEALLOCATE record_from_prepared; SELECT get_telemetry_report()->'functions_used'; -- check the report again to see if resetting works SELECT get_telemetry_report()->'functions_used'; \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE _timescaledb_catalog.metadata; SET timescaledb.telemetry_level=off; -- returns false which means telemetry got canceled SELECT * FROM _timescaledb_internal.test_privacy(); RESET timescaledb.telemetry_level; -- returns false which means telemetry got canceled SELECT * FROM _timescaledb_internal.test_privacy(); -- To make sure nothing was sent, we check the UUID table to make sure no exported UUID row was created SELECT key from _timescaledb_catalog.metadata; \set ON_ERROR_STOP 0 -- test that the telemetry gathering code doesn't break nonexistent statements EXECUTE noexistent_statement; \c :TEST_DBNAME :ROLE_SUPERUSER -- Insert some data into the telemetry event table INSERT INTO _timescaledb_catalog.telemetry_event(tag, body) VALUES ('ummagumma', '{"title": "Careful with that Axe Eugene!"}'), ('kaboom', '{"title": "Where is that kaboom?"}'); -- Check that it is present in the telemetry report SELECT * FROM jsonb_to_recordset(get_telemetry_report()->'db_telemetry_events') AS x(tag name, body text); ================================================ FILE: test/sql/test_tss_callbacks.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION test.setup_tss_hook_v0() RETURNS VOID AS :MODULE_PATHNAME, 'ts_setup_tss_hook_v0' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION test.setup_tss_hook_v1() RETURNS VOID AS :MODULE_PATHNAME, 'ts_setup_tss_hook_v1' LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION test.teardown_tss_hook_v1() RETURNS VOID AS :MODULE_PATHNAME, 'ts_teardown_tss_hook_v1' LANGUAGE C VOLATILE; SELECT test.setup_tss_hook_v1(); CREATE TABLE copy_test ( "time" timestamptz NOT NULL, "value" double precision NOT NULL ); SELECT create_hypertable('copy_test', 'time'); -- We should se a mock message COPY copy_test FROM STDIN DELIMITER ','; 2020-01-01 01:10:00+01,1 2021-01-01 01:10:00+01,1 \. SELECT test.teardown_tss_hook_v1(); -- Without the hook registered we cannot see the mock message COPY copy_test FROM STDIN DELIMITER ','; 2020-01-01 01:10:00+01,1 2021-01-01 01:10:00+01,1 \. -- Test for mismatch version SELECT test.setup_tss_hook_v0(); -- Warning because the mismatch versions COPY copy_test FROM STDIN DELIMITER ','; 2020-01-01 01:10:00+01,1 2021-01-01 01:10:00+01,1 \. ================================================ FILE: test/sql/test_utils.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION test.condition() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_utils_condition' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION test.int64_eq() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_utils_int64_eq' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION test.ptr_eq() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_utils_ptr_eq' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION test.double_eq() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_utils_double_eq' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; SET ROLE :ROLE_DEFAULT_PERM_USER; -- We're testing that the test utils work and generate errors on -- failing conditions \set ON_ERROR_STOP 0 SELECT test.condition(); SELECT test.int64_eq(); SELECT test.ptr_eq(); SELECT test.double_eq(); \set ON_ERROR_STOP 1 -- Test debug points -- \set ECHO all \c :TEST_DBNAME :ROLE_SUPERUSER -- debug point already enabled SELECT debug_waitpoint_enable('test_debug_point'); \set ON_ERROR_STOP 0 SELECT debug_waitpoint_enable('test_debug_point'); \set ON_ERROR_STOP 1 SELECT debug_waitpoint_release('test_debug_point'); -- debug point not enabled \set ON_ERROR_STOP 0 SELECT debug_waitpoint_release('test_debug_point'); \set ON_ERROR_STOP 1 -- error injections -- CREATE OR REPLACE FUNCTION test_error_injection(TEXT) RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_error_injection' LANGUAGE C VOLATILE STRICT; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT test_error_injection('test_error'); SELECT debug_waitpoint_enable('test_error'); \set ON_ERROR_STOP 0 SELECT test_error_injection('test_error'); \set ON_ERROR_STOP 1 SELECT debug_waitpoint_release('test_error'); SELECT test_error_injection('test_error'); -- Test Scanner RESET ROLE; CREATE OR REPLACE FUNCTION test.scanner() RETURNS VOID AS :MODULE_PATHNAME, 'ts_test_scanner' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; SET ROLE :ROLE_DEFAULT_PERM_USER; -- Create two chunks to scan in the test CREATE TABLE hyper (time timestamptz, temp float); SELECT create_hypertable('hyper', 'time'); INSERT INTO hyper VALUES ('2021-01-01', 1.0), ('2022-01-01', 2.0); SELECT test.scanner(); -- Test errdata_to_jsonb RESET ROLE; CREATE OR REPLACE FUNCTION test.errdata_to_jsonb() RETURNS JSONB AS :MODULE_PATHNAME, 'ts_test_errdata_to_jsonb' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; SELECT test.errdata_to_jsonb(); ================================================ FILE: test/sql/timestamp.sql.in ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Utility function for grouping/slotting time with a given interval. CREATE OR REPLACE FUNCTION date_group( field timestamp, group_interval interval ) RETURNS timestamp LANGUAGE SQL STABLE AS $BODY$ SELECT to_timestamp((EXTRACT(EPOCH from $1)::int / EXTRACT(EPOCH from group_interval)::int) * EXTRACT(EPOCH from group_interval)::int)::timestamp; $BODY$; CREATE TABLE PUBLIC."testNs" ( "timeCustom" TIMESTAMP NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON PUBLIC."testNs" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA "testNs" AUTHORIZATION :ROLE_DEFAULT_PERM_USER; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT * FROM create_hypertable('"public"."testNs"', 'timeCustom', 'device_id', 2, associated_schema_name=>'testNs' ); \c :TEST_DBNAME INSERT INTO PUBLIC."testNs"("timeCustom", device_id, series_0, series_1) VALUES ('2009-11-12T01:00:00+00:00', 'dev1', 1.5, 1), ('2009-11-12T01:00:00+00:00', 'dev1', 1.5, 2), ('2009-11-10T23:00:02+00:00', 'dev1', 2.5, 3); INSERT INTO PUBLIC."testNs"("timeCustom", device_id, series_0, series_1) VALUES ('2009-11-10T23:00:00+00:00', 'dev2', 1.5, 1), ('2009-11-10T23:00:00+00:00', 'dev2', 1.5, 2); SELECT * FROM PUBLIC."testNs"; SET client_min_messages = WARNING; \echo 'The next 2 queries will differ in output between UTC and EST since the mod is on the 100th hour UTC' SET timezone = 'UTC'; SELECT date_group("timeCustom", '100 days') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC; SET timezone = 'EST'; SELECT date_group("timeCustom", '100 days') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC; \echo 'The rest of the queries will be the same in output between UTC and EST' SET timezone = 'UTC'; SELECT date_group("timeCustom", '1 day') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC; SET timezone = 'EST'; SELECT date_group("timeCustom", '1 day') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC; SET timezone = 'UTC'; SELECT * FROM PUBLIC."testNs" WHERE "timeCustom" >= TIMESTAMP '2009-11-10T23:00:00' AND "timeCustom" < TIMESTAMP '2009-11-12T01:00:00' ORDER BY "timeCustom" DESC, device_id, series_1; SET timezone = 'EST'; SELECT * FROM PUBLIC."testNs" WHERE "timeCustom" >= TIMESTAMP '2009-11-10T23:00:00' AND "timeCustom" < TIMESTAMP '2009-11-12T01:00:00' ORDER BY "timeCustom" DESC, device_id, series_1; SET timezone = 'UTC'; SELECT date_group("timeCustom", '1 day') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC LIMIT 2; SET timezone = 'EST'; SELECT date_group("timeCustom", '1 day') AS time, sum(series_0) FROM PUBLIC."testNs" GROUP BY time ORDER BY time ASC LIMIT 2; ------------------------------------ -- Test time conversion functions -- ------------------------------------ \set ON_ERROR_STOP 0 SET timezone = 'UTC'; -- Conversion to timestamp using Postgres built-in function taking -- double. Gives inaccurate result on Postgres <= 9.6.2. Accurate on -- Postgres >= 9.6.3. SELECT to_timestamp(1486480176.236538); -- extension-specific version taking microsecond UNIX timestamp SELECT _timescaledb_functions.to_timestamp(1486480176236538); -- Should be the inverse of the statement above. SELECT _timescaledb_functions.to_unix_microseconds('2017-02-07 15:09:36.236538+00'); -- For timestamps, BIGINT MAX represents +Infinity and BIGINT MIN -- -Infinity. We keep this notion for UNIX epoch time: SELECT _timescaledb_functions.to_unix_microseconds('+infinity'); SELECT _timescaledb_functions.to_timestamp(9223372036854775807); SELECT _timescaledb_functions.to_unix_microseconds('-infinity'); SELECT _timescaledb_functions.to_timestamp(-9223372036854775808); -- In UNIX microseconds, the largest bigint value below infinity -- (BIGINT MAX) is smaller than internal date upper bound and should -- therefore be OK. Further, converting to the internal postgres epoch -- cannot overflow a 64-bit INTEGER since the postgres epoch is at a -- later date compared to the UNIX epoch, and is therefore represented -- by a smaller number SELECT _timescaledb_functions.to_timestamp(9223372036854775806); -- Julian day zero is -210866803200000000 microseconds from UNIX epoch SELECT _timescaledb_functions.to_timestamp(-210866803200000000); \set VERBOSITY default -- Going beyond Julian day zero should give out-of-range error SELECT _timescaledb_functions.to_timestamp(-210866803200000001); -- Lower bound on date (should return the Julian day zero UNIX timestamp above) SELECT _timescaledb_functions.to_unix_microseconds('4714-11-24 00:00:00+00 BC'); -- Going beyond lower bound on date should return out-of-range SELECT _timescaledb_functions.to_unix_microseconds('4714-11-23 23:59:59.999999+00 BC'); -- The upper bound for Postgres TIMESTAMPTZ SELECT timestamp '294276-12-31 23:59:59.999999+00'; -- Going beyond the upper bound, should fail SELECT timestamp '294276-12-31 23:59:59.999999+00' + interval '1 us'; -- Cannot represent the upper bound timestamp with a UNIX microsecond timestamp -- since the Postgres epoch is at a later date than the UNIX epoch. SELECT _timescaledb_functions.to_unix_microseconds('294276-12-31 23:59:59.999999+00'); -- Subtracting the difference between the two epochs (10957 days) should bring -- us within range. SELECT timestamp '294276-12-31 23:59:59.999999+00' - interval '10957 days'; SELECT _timescaledb_functions.to_unix_microseconds('294247-01-01 23:59:59.999999'); -- Adding one microsecond should take us out-of-range again SELECT timestamp '294247-01-01 23:59:59.999999' + interval '1 us'; SELECT _timescaledb_functions.to_unix_microseconds(timestamp '294247-01-01 23:59:59.999999' + interval '1 us'); --no time_bucketing of dates not by integer # of days SELECT time_bucket('1 hour', DATE '2012-01-01'); SELECT time_bucket('25 hour', DATE '2012-01-01'); \set ON_ERROR_STOP 1 SELECT time_bucket(INTERVAL '1 day', TIMESTAMP '2011-01-02 01:01:01'); SELECT time, time_bucket(INTERVAL '2 day ', time) FROM unnest(ARRAY[ TIMESTAMP '2011-01-01 01:01:01', TIMESTAMP '2011-01-02 01:01:01', TIMESTAMP '2011-01-03 01:01:01', TIMESTAMP '2011-01-04 01:01:01' ]) AS time; SELECT int_def, time_bucket(int_def,TIMESTAMP '2011-01-02 01:01:01.111') FROM unnest(ARRAY[ INTERVAL '1 millisecond', INTERVAL '1 second', INTERVAL '1 minute', INTERVAL '1 hour', INTERVAL '1 day', INTERVAL '2 millisecond', INTERVAL '2 second', INTERVAL '2 minute', INTERVAL '2 hour', INTERVAL '2 day' ]) AS int_def; \set ON_ERROR_STOP 0 SELECT time_bucket(INTERVAL '1 year 1d',TIMESTAMP '2011-01-02 01:01:01.111'); SELECT time_bucket(INTERVAL '1 month 1 minute',TIMESTAMP '2011-01-02 01:01:01.111'); \set ON_ERROR_STOP 1 SELECT time, time_bucket(INTERVAL '5 minute', time) FROM unnest(ARRAY[ TIMESTAMP '1970-01-01 00:59:59.999999', TIMESTAMP '1970-01-01 01:01:00', TIMESTAMP '1970-01-01 01:04:59.999999', TIMESTAMP '1970-01-01 01:05:00' ]) AS time; SELECT time, time_bucket(INTERVAL '5 minute', time) FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:04:59.999999', TIMESTAMP '2011-01-02 01:05:00', TIMESTAMP '2011-01-02 01:09:59.999999', TIMESTAMP '2011-01-02 01:10:00' ]) AS time; --offset with interval SELECT time, time_bucket(INTERVAL '5 minute', time , INTERVAL '2 minutes') FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:01:59.999999', TIMESTAMP '2011-01-02 01:02:00', TIMESTAMP '2011-01-02 01:06:59.999999', TIMESTAMP '2011-01-02 01:07:00' ]) AS time; SELECT time, time_bucket(INTERVAL '5 minute', time , - INTERVAL '2 minutes') FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:02:59.999999', TIMESTAMP '2011-01-02 01:03:00', TIMESTAMP '2011-01-02 01:07:59.999999', TIMESTAMP '2011-01-02 01:08:00' ]) AS time; --offset with infinity -- timestamp SELECT time, time_bucket(INTERVAL '1 week', time, INTERVAL '1 day') FROM unnest(ARRAY[ timestamp '-Infinity', timestamp 'Infinity' ]) AS time; -- timestamptz SELECT time, time_bucket(INTERVAL '1 week', time, INTERVAL '1 day') FROM unnest(ARRAY[ timestamp with time zone '-Infinity', timestamp with time zone 'Infinity' ]) AS time; -- Date SELECT date, time_bucket(INTERVAL '1 week', date, INTERVAL '1 day') FROM unnest(ARRAY[ date '-Infinity', date 'Infinity' ]) AS date; --example to align with an origin SELECT time, time_bucket(INTERVAL '5 minute', time - (TIMESTAMP '2011-01-02 00:02:00' - TIMESTAMP 'epoch')) + (TIMESTAMP '2011-01-02 00:02:00'-TIMESTAMP 'epoch') FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:01:59.999999', TIMESTAMP '2011-01-02 01:02:00', TIMESTAMP '2011-01-02 01:06:59.999999', TIMESTAMP '2011-01-02 01:07:00' ]) AS time; --rounding version SELECT time, time_bucket(INTERVAL '5 minute', time , - INTERVAL '2.5 minutes') + INTERVAL '2 minutes 30 seconds' FROM unnest(ARRAY[ TIMESTAMP '2011-01-02 01:05:01', TIMESTAMP '2011-01-02 01:07:29', TIMESTAMP '2011-01-02 01:02:30', TIMESTAMP '2011-01-02 01:07:30', TIMESTAMP '2011-01-02 01:02:29' ]) AS time; --time_bucket with timezone should mimick date_trunc SET timezone TO 'UTC'; SELECT time, time_bucket(INTERVAL '1 hour', time), date_trunc('hour', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+02' ]) AS time; SELECT time, time_bucket(INTERVAL '1 day', time), date_trunc('day', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+02' ]) AS time; --what happens with a local tz SET timezone TO 'America/New_York'; SELECT time, time_bucket(INTERVAL '1 hour', time), date_trunc('hour', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01+02' ]) AS time; --Note the timestamp tz input is aligned with UTC day /not/ local day. different than date_trunc. SELECT time, time_bucket(INTERVAL '1 day', time), date_trunc('day', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-03 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-04 01:01:01+02' ]) AS time; --can force local bucketing with simple cast. SELECT time, time_bucket(INTERVAL '1 day', time::timestamp), date_trunc('day', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-03 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-04 01:01:01+02' ]) AS time; --can also use interval to correct SELECT time, time_bucket(INTERVAL '1 day', time, -INTERVAL '19 hours'), date_trunc('day', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2011-01-02 01:01:01', TIMESTAMP WITH TIME ZONE '2011-01-03 01:01:01+01', TIMESTAMP WITH TIME ZONE '2011-01-04 01:01:01+02' ]) AS time; --dst: same local hour bucketed as two different hours. SELECT time, time_bucket(INTERVAL '1 hour', time), date_trunc('hour', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2017-11-05 12:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 13:05:00+07' ]) AS time; --local alignment changes when bucketing by UTC across dst boundary SELECT time, time_bucket(INTERVAL '2 hour', time) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2017-11-05 10:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 12:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 13:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 15:05:00+07' ]) AS time; --local alignment is preserved when bucketing by local time across DST boundary. SELECT time, time_bucket(INTERVAL '2 hour', time::timestamp) FROM unnest(ARRAY[ TIMESTAMP WITH TIME ZONE '2017-11-05 10:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 12:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 13:05:00+07', TIMESTAMP WITH TIME ZONE '2017-11-05 15:05:00+07' ]) AS time; -- GitHub issue #7059: time_bucket with timezone + offset across DST boundary -- Asia/Amman: clocks skip from 00:00 to 01:00 on 2021-03-26 -- Input: 01:00+03 = 22:00 UTC → Result: 22:15 UTC = 01:15 local (00:00 + 15min offset) SELECT time_bucket('1 day', '2021-03-26 01:00:00+03'::timestamptz, timezone := 'Asia/Amman', "offset" := '15 minutes'::interval); -- GitHub issue #8851: time_bucket with negative offset during DST fall-back -- Europe/Berlin: clocks repeat 02:00-02:59 on 2025-10-26 -- Input: 02:00+02 = 00:00 UTC → Result: 23:59:45 UTC = 01:59:45 local (02:00 - 15s offset) SELECT time_bucket('30 seconds', '2025-10-26 02:00:00+02'::timestamptz, timezone := 'Europe/Berlin', "offset" := '-15 seconds'::interval); -- Additional DST edge cases for coverage of DST direction × offset sign combinations -- Spring-forward + negative offset -- Input: 01:30+03 = 22:30 UTC → Result: 22:45 UTC = 01:45 local (01:00 + 45min = 02:00 - 15min) SELECT time_bucket('1 hour', '2021-03-26 01:30:00+03'::timestamptz, timezone := 'Asia/Amman', "offset" := '-15 minutes'::interval); -- Fall-back + positive offset -- Input: 02:30+01 = 01:30 UTC → Result: 01:15 UTC = 02:15 local (02:00 + 15min offset) SELECT time_bucket('1 hour', '2025-10-26 02:30:00+01'::timestamptz, timezone := 'Europe/Berlin', "offset" := '15 minutes'::interval); -- Input exactly at DST spring-forward transition -- Input: 22:00 UTC = 00:00 local (the moment clocks jump to 01:00) -- Result: 22:15 UTC = 01:15 local (01:00 + 15min offset) SELECT time_bucket('1 hour', '2021-03-25 22:00:00+00'::timestamptz, timezone := 'Asia/Amman', "offset" := '15 minutes'::interval); -- Input exactly at DST fall-back transition -- Input: 01:00 UTC = 03:00 CEST (the moment clocks go back to 02:00 CET) -- Result: 23:15 UTC = 01:15 local (01:00 + 15min offset, but in CET now) SELECT time_bucket('1 hour', '2025-10-26 01:00:00+00'::timestamptz, timezone := 'Europe/Berlin', "offset" := '15 minutes'::interval); -- Offset larger than bucket size (1h offset with 30min bucket) -- Input: 01:30+03 = 22:30 UTC → Result: 22:30 UTC = 01:30 local (01:00 + 30min = 00:30 + 1h) SELECT time_bucket('30 minutes', '2021-03-26 01:30:00+03'::timestamptz, timezone := 'Asia/Amman', "offset" := '1 hour'::interval); -- GitHub issue #9136: time_bucket with origin during DST fall-back -- When origin is in standard time but timestamp is in daylight time, -- the bucket could incorrectly start AFTER the timestamp. -- America/New_York: clocks go back at 02:00 EDT on 2024-11-03 -- Input: 01:30-04 (EDT) = 05:30 UTC; origin in EST -- Result should have bucket start <= timestamp (bucket in EDT, not EST) SELECT time_bucket('1 hour', '2024-11-03 01:30:00-04'::timestamptz, 'America/New_York', '2000-01-01 00:00:00 America/New_York'::timestamptz) as bucket, '2024-11-03 01:30:00-04'::timestamptz < time_bucket('1 hour', '2024-11-03 01:30:00-04'::timestamptz, 'America/New_York', '2000-01-01 00:00:00 America/New_York'::timestamptz) as ts_before_bucket; SELECT time, time_bucket(10::smallint, time) AS time_bucket_smallint, time_bucket(10::int, time) AS time_bucket_int, time_bucket(10::bigint, time) AS time_bucket_bigint FROM unnest(ARRAY[ '-11', '-10', '-9', '-1', '0', '1', '99', '100', '109', '110' ]::smallint[]) AS time; SELECT time, time_bucket(10::smallint, time, 2::smallint) AS time_bucket_smallint, time_bucket(10::int, time, 2::int) AS time_bucket_int, time_bucket(10::bigint, time, 2::bigint) AS time_bucket_bigint FROM unnest(ARRAY[ '-9', '-8', '-7', '1', '2', '3', '101', '102', '111', '112' ]::smallint[]) AS time; SELECT time, time_bucket(10::smallint, time, -2::smallint) AS time_bucket_smallint, time_bucket(10::int, time, -2::int) AS time_bucket_int, time_bucket(10::bigint, time, -2::bigint) AS time_bucket_bigint FROM unnest(ARRAY[ '-13', '-12', '-11', '-3', '-2', '-1', '97', '98', '107', '108' ]::smallint[]) AS time; \set ON_ERROR_STOP 0 SELECT time_bucket(10::smallint, '-32768'::smallint); SELECT time_bucket(10::smallint, '-32761'::smallint); select time_bucket(10::smallint, '-32768'::smallint, 1000::smallint); select time_bucket(10::smallint, '-32768'::smallint, '32767'::smallint); select time_bucket(10::smallint, '32767'::smallint, '-32768'::smallint); \set ON_ERROR_STOP 1 SELECT time, time_bucket(10::smallint, time) FROM unnest(ARRAY[ '-32760', '-32759', '32767' ]::smallint[]) AS time; \set ON_ERROR_STOP 0 SELECT time_bucket(10::int, '-2147483648'::int); SELECT time_bucket(10::int, '-2147483641'::int); SELECT time_bucket(1000::int, '-2147483000'::int, 1::int); SELECT time_bucket(1000::int, '-2147483648'::int, '2147483647'::int); SELECT time_bucket(1000::int, '2147483647'::int, '-2147483648'::int); \set ON_ERROR_STOP 1 SELECT time, time_bucket(10::int, time) FROM unnest(ARRAY[ '-2147483640', '-2147483639', '2147483647' ]::int[]) AS time; \set ON_ERROR_STOP 0 SELECT time_bucket(10::bigint, '-9223372036854775808'::bigint); SELECT time_bucket(10::bigint, '-9223372036854775801'::bigint); SELECT time_bucket(1000::bigint, '-9223372036854775000'::bigint, 1::bigint); SELECT time_bucket(1000::bigint, '-9223372036854775808'::bigint, '9223372036854775807'::bigint); SELECT time_bucket(1000::bigint, '9223372036854775807'::bigint, '-9223372036854775808'::bigint); \set ON_ERROR_STOP 1 SELECT time, time_bucket(10::bigint, time) FROM unnest(ARRAY[ '-9223372036854775800', '-9223372036854775799', '9223372036854775807' ]::bigint[]) AS time; SELECT time, time_bucket(INTERVAL '1 day', time::date) FROM unnest(ARRAY[ date '2017-11-05', date '2017-11-06' ]) AS time; SELECT time, time_bucket(INTERVAL '4 day', time::date) FROM unnest(ARRAY[ date '2017-11-04', date '2017-11-05', date '2017-11-08', date '2017-11-09' ]) AS time; SELECT time, time_bucket(INTERVAL '4 day', time::date, INTERVAL '2 day') FROM unnest(ARRAY[ date '2017-11-06', date '2017-11-07', date '2017-11-10', date '2017-11-11' ]) AS time; -- 2019-09-24 is a Monday, and we want to ensure that time_bucket returns the week starting with a Monday as date_trunc does, -- Rather than a Saturday which is the date of the PostgreSQL epoch SELECT time, time_bucket(INTERVAL '1 week', time::date) FROM unnest(ARRAY[ date '2018-09-16', date '2018-09-17', date '2018-09-23', date '2018-09-24' ]) AS time; SELECT time, time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp without time zone '2018-09-16', timestamp without time zone '2018-09-17', timestamp without time zone '2018-09-23', timestamp without time zone '2018-09-24' ]) AS time; SELECT time, time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp with time zone '2018-09-16', timestamp with time zone '2018-09-17', timestamp with time zone '2018-09-23', timestamp with time zone '2018-09-24' ]) AS time; SELECT time, time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp with time zone '-Infinity', timestamp with time zone 'Infinity' ]) AS time; SELECT time, time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp without time zone '-Infinity', timestamp without time zone 'Infinity' ]) AS time; SELECT time, time_bucket(INTERVAL '1 week', time), date_trunc('week', time) = time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp without time zone '4714-11-24 01:01:01.0 BC', timestamp without time zone '294276-12-31 23:59:59.9999' ]) AS time; --1000 years later weeks still align. SELECT time, time_bucket(INTERVAL '1 week', time), date_trunc('week', time) = time_bucket(INTERVAL '1 week', time) FROM unnest(ARRAY[ timestamp without time zone '3018-09-14', timestamp without time zone '3018-09-20', timestamp without time zone '3018-09-21', timestamp without time zone '3018-09-22' ]) AS time; --weeks align for timestamptz as well if cast to local time, (but not if done at UTC). SELECT time, date_trunc('week', time) = time_bucket(INTERVAL '1 week', time), date_trunc('week', time) = time_bucket(INTERVAL '1 week', time::timestamp) FROM unnest(ARRAY[ timestamp with time zone '3018-09-14', timestamp with time zone '3018-09-20', timestamp with time zone '3018-09-21', timestamp with time zone '3018-09-22' ]) AS time; --check functions with origin --note that the default origin is at 0 UTC, using origin parameter it is easy to provide a EDT origin point \x SELECT time, time_bucket(INTERVAL '1 week', time) no_epoch, time_bucket(INTERVAL '1 week', time::timestamp) no_epoch_local, time_bucket(INTERVAL '1 week', time) = time_bucket(INTERVAL '1 week', time, timestamptz '2000-01-03 00:00:00+0') always_true, time_bucket(INTERVAL '1 week', time, timestamptz '2000-01-01 00:00:00+0') pg_epoch, time_bucket(INTERVAL '1 week', time, timestamptz 'epoch') unix_epoch, time_bucket(INTERVAL '1 week', time, timestamptz '3018-09-13') custom_1, time_bucket(INTERVAL '1 week', time, timestamptz '3018-09-14') custom_2 FROM unnest(ARRAY[ timestamp with time zone '2000-01-01 00:00:00+0'- interval '1 second', timestamp with time zone '2000-01-01 00:00:00+0', timestamp with time zone '2000-01-03 00:00:00+0'- interval '1 second', timestamp with time zone '2000-01-03 00:00:00+0', timestamp with time zone '2000-01-01', timestamp with time zone '2000-01-02', timestamp with time zone '2000-01-03', timestamp with time zone '3018-09-12', timestamp with time zone '3018-09-13', timestamp with time zone '3018-09-14', timestamp with time zone '3018-09-15' ]) AS time; SELECT time, time_bucket(INTERVAL '1 week', time) no_epoch, time_bucket(INTERVAL '1 week', time) = time_bucket(INTERVAL '1 week', time, timestamp '2000-01-03 00:00:00') always_true, time_bucket(INTERVAL '1 week', time, timestamp '2000-01-01 00:00:00+0') pg_epoch, time_bucket(INTERVAL '1 week', time, timestamp 'epoch') unix_epoch, time_bucket(INTERVAL '1 week', time, timestamp '3018-09-13') custom_1, time_bucket(INTERVAL '1 week', time, timestamp '3018-09-14') custom_2 FROM unnest(ARRAY[ timestamp without time zone '2000-01-01 00:00:00'- interval '1 second', timestamp without time zone '2000-01-01 00:00:00', timestamp without time zone '2000-01-03 00:00:00'- interval '1 second', timestamp without time zone '2000-01-03 00:00:00', timestamp without time zone '2000-01-01', timestamp without time zone '2000-01-02', timestamp without time zone '2000-01-03', timestamp without time zone '3018-09-12', timestamp without time zone '3018-09-13', timestamp without time zone '3018-09-14', timestamp without time zone '3018-09-15' ]) AS time; SELECT time, time_bucket(INTERVAL '1 week', time) no_epoch, time_bucket(INTERVAL '1 week', time) = time_bucket(INTERVAL '1 week', time, date '2000-01-03') always_true, time_bucket(INTERVAL '1 week', time, date '2000-01-01') pg_epoch, time_bucket(INTERVAL '1 week', time, (timestamp 'epoch')::date) unix_epoch, time_bucket(INTERVAL '1 week', time, date '3018-09-13') custom_1, time_bucket(INTERVAL '1 week', time, date '3018-09-14') custom_2 FROM unnest(ARRAY[ date '1999-12-31', date '2000-01-01', date '2000-01-02', date '2000-01-03', date '3018-09-12', date '3018-09-13', date '3018-09-14', date '3018-09-15' ]) AS time; \x --really old origin works if date around that time SELECT time, time_bucket(INTERVAL '1 week', time, timestamp without time zone '4710-11-24 01:01:01.0 BC') FROM unnest(ARRAY[ timestamp without time zone '4710-11-24 01:01:01.0 BC', timestamp without time zone '4710-11-25 01:01:01.0 BC', timestamp without time zone '2001-01-01', timestamp without time zone '3001-01-01' ]) AS time; SELECT time, time_bucket(INTERVAL '1 week', time, timestamp without time zone '294270-12-30 23:59:59.9999') FROM unnest(ARRAY[ timestamp without time zone '294270-12-29 23:59:59.9999', timestamp without time zone '294270-12-30 23:59:59.9999', timestamp without time zone '294270-12-31 23:59:59.9999', timestamp without time zone '2001-01-01', timestamp without time zone '3001-01-01' ]) AS time; \set ON_ERROR_STOP 0 --really old origin + very new data + long period errors SELECT time, time_bucket(INTERVAL '100000 day', time, timestamp without time zone '4710-11-24 01:01:01.0 BC') FROM unnest(ARRAY[ timestamp without time zone '294270-12-31 23:59:59.9999' ]) AS time; SELECT time, time_bucket(INTERVAL '100000 day', time, timestamp with time zone '4710-11-25 01:01:01.0 BC') FROM unnest(ARRAY[ timestamp with time zone '294270-12-30 23:59:59.9999' ]) AS time; --really high origin + old data + long period errors out SELECT time, time_bucket(INTERVAL '10000000 day', time, timestamp without time zone '294270-12-31 23:59:59.9999') FROM unnest(ARRAY[ timestamp without time zone '4710-11-24 01:01:01.0 BC' ]) AS time; SELECT time, time_bucket(INTERVAL '10000000 day', time, timestamp with time zone '294270-12-31 23:59:59.9999') FROM unnest(ARRAY[ timestamp with time zone '4710-11-24 01:01:01.0 BC' ]) AS time; \set ON_ERROR_STOP 1 ------------------------------------------- --- Test time_bucket with month periods --- ------------------------------------------- SET datestyle TO ISO; SELECT time::date, time_bucket('1 month', time::date) AS "1m", time_bucket('2 month', time::date) AS "2m", time_bucket('3 month', time::date) AS "3m", time_bucket('1 month', time::date, '2000-02-01'::date) AS "1m origin", time_bucket('2 month', time::date, '2000-02-01'::date) AS "2m origin", time_bucket('3 month', time::date, '2000-02-01'::date) AS "3m origin" FROM generate_series('1990-01-03'::date,'1990-06-03'::date,'1month'::interval) time; SELECT time, time_bucket('1 month', time) AS "1m", time_bucket('2 month', time) AS "2m", time_bucket('3 month', time) AS "3m", time_bucket('1 month', time, '2000-02-01'::timestamp) AS "1m origin", time_bucket('2 month', time, '2000-02-01'::timestamp) AS "2m origin", time_bucket('3 month', time, '2000-02-01'::timestamp) AS "3m origin" FROM generate_series('1990-01-03'::timestamp,'1990-06-03'::timestamp,'1month'::interval) time; SELECT time, time_bucket('1 month', time) AS "1m", time_bucket('2 month', time) AS "2m", time_bucket('3 month', time) AS "3m", time_bucket('1 month', time, '2000-02-01'::timestamptz) AS "1m origin", time_bucket('2 month', time, '2000-02-01'::timestamptz) AS "2m origin", time_bucket('3 month', time, '2000-02-01'::timestamptz) AS "3m origin" FROM generate_series('1990-01-03'::timestamptz,'1990-06-03'::timestamptz,'1month'::interval) time; --------------------------------------- --- Test time_bucket with timezones --- --------------------------------------- -- test NULL args SELECT time_bucket(NULL::interval,now(),'Europe/Berlin'), time_bucket('1day',NULL::timestamptz,'Europe/Berlin'), time_bucket('1day',now(),NULL::text), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin',NULL), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin','2020-04-01',NULL), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin',NULL,NULL), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin',"offset":=NULL::interval), time_bucket('1day',timestamptz '2020-02-03','Europe/Berlin',origin:=NULL::timestamptz); SET datestyle TO ISO; SELECT time_bucket('1day', ts) AS "UTC", time_bucket('1day', ts, 'Europe/Berlin') AS "Berlin", time_bucket('1day', ts, 'Europe/London') AS "London", time_bucket('1day', ts, 'America/New_York') AS "New York", time_bucket('1day', ts, 'PST') AS "PST", time_bucket('1day', ts, current_setting('timezone')) AS "current" FROM generate_series('1999-12-31 17:00'::timestamptz,'2000-01-02 3:00'::timestamptz, '1hour'::interval) ts; SELECT time_bucket('1month', ts) AS "UTC", time_bucket('1month', ts, 'Europe/Berlin') AS "Berlin", time_bucket('1month', ts, 'America/New_York') AS "New York", time_bucket('1month', ts, current_setting('timezone')) AS "current", time_bucket('2month', ts, current_setting('timezone')) AS "2m", time_bucket('2month', ts, current_setting('timezone'), '2000-02-01'::timestamp) AS "2m origin", time_bucket('2month', ts, current_setting('timezone'), "offset":='14 day'::interval) AS "2m offset", time_bucket('2month', ts, current_setting('timezone'), '2000-02-01'::timestamp, '7 day'::interval) AS "2m offset + origin" FROM generate_series('1999-12-01'::timestamptz,'2000-09-01'::timestamptz, '9 day'::interval) ts; RESET datestyle; ------------------------------------- --- Test time input functions -- ------------------------------------- \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION test.interval_to_internal(coltype REGTYPE, value ANYELEMENT = NULL::BIGINT) RETURNS BIGINT AS :MODULE_PATHNAME, 'ts_dimension_interval_to_internal_test' LANGUAGE C VOLATILE; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT test.interval_to_internal('TIMESTAMP'::regtype, INTERVAL '1 day'); SELECT test.interval_to_internal('TIMESTAMP'::regtype, 86400000000); ---should give warning SELECT test.interval_to_internal('TIMESTAMP'::regtype, 86400); SELECT test.interval_to_internal('TIMESTAMP'::regtype); SELECT test.interval_to_internal('BIGINT'::regtype, 2147483649::bigint); -- Default interval for integer is supported as part of -- hypertable generalization SELECT test.interval_to_internal('INT'::regtype); SELECT test.interval_to_internal('SMALLINT'::regtype); SELECT test.interval_to_internal('BIGINT'::regtype); SELECT test.interval_to_internal('TIMESTAMPTZ'::regtype); SELECT test.interval_to_internal('TIMESTAMP'::regtype); SELECT test.interval_to_internal('DATE'::regtype); \set VERBOSITY terse \set ON_ERROR_STOP 0 SELECT test.interval_to_internal('INT'::regtype, 2147483649::bigint); SELECT test.interval_to_internal('SMALLINT'::regtype, 32768::bigint); SELECT test.interval_to_internal('TEXT'::regtype, 32768::bigint); SELECT test.interval_to_internal('INT'::regtype, INTERVAL '1 day'); \set ON_ERROR_STOP 1 ================================================ FILE: test/sql/triggers.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE hyper ( time BIGINT NOT NULL, device_id TEXT NOT NULL, sensor_1 NUMERIC NULL DEFAULT 1 ); CREATE OR REPLACE FUNCTION test_trigger() RETURNS TRIGGER LANGUAGE PLPGSQL AS $BODY$ DECLARE cnt INTEGER; BEGIN SELECT count(*) INTO cnt FROM hyper; RAISE WARNING 'FIRING trigger when: % level: % op: % cnt: % trigger_name %', tg_when, tg_level, tg_op, cnt, tg_name; IF TG_OP = 'DELETE' THEN RETURN OLD; END IF; RETURN NEW; END $BODY$; -- row triggers: BEFORE CREATE TRIGGER _0_test_trigger_insert BEFORE INSERT ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_update BEFORE UPDATE ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_delete BEFORE delete ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER z_test_trigger_all BEFORE INSERT OR UPDATE OR DELETE ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); -- row triggers: AFTER CREATE TRIGGER _0_test_trigger_insert_after AFTER INSERT ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_insert_after_when_dev1 AFTER INSERT ON hyper FOR EACH ROW WHEN (NEW.device_id = 'dev1') EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_update_after AFTER UPDATE ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_delete_after AFTER delete ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER z_test_trigger_all_after AFTER INSERT OR UPDATE OR DELETE ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); -- statement triggers: BEFORE CREATE TRIGGER _0_test_trigger_insert_s_before BEFORE INSERT ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_update_s_before BEFORE UPDATE ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_delete_s_before BEFORE DELETE ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); -- statement triggers: AFTER CREATE TRIGGER _0_test_trigger_insert_s_after AFTER INSERT ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_update_s_after AFTER UPDATE ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_delete_s_after AFTER DELETE ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); -- CONSTRAINT TRIGGER CREATE CONSTRAINT TRIGGER _0_test_trigger_constraint_insert AFTER INSERT ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE CONSTRAINT TRIGGER _0_test_trigger_constraint_update AFTER UPDATE ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE CONSTRAINT TRIGGER _0_test_trigger_constraint_delete AFTER DELETE ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); SELECT * FROM create_hypertable('hyper', 'time', chunk_time_interval => 10); --test triggers before create_hypertable INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987600000000000, 'dev1', 1); INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 1), (1257987800000000000, 'dev2', 1); UPDATE hyper SET sensor_1 = 2; DELETE FROM hyper; --test drop trigger DROP TRIGGER _0_test_trigger_insert ON hyper; DROP TRIGGER _0_test_trigger_insert_s_before ON hyper; DROP TRIGGER _0_test_trigger_insert_after ON hyper; DROP TRIGGER _0_test_trigger_insert_s_after ON hyper; INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987600000000000, 'dev1', 1); INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 1), (1257987800000000000, 'dev2', 1); DROP TRIGGER _0_test_trigger_update ON hyper; DROP TRIGGER _0_test_trigger_update_s_before ON hyper; DROP TRIGGER _0_test_trigger_update_after ON hyper; DROP TRIGGER _0_test_trigger_update_s_after ON hyper; UPDATE hyper SET sensor_1 = 2; DROP TRIGGER _0_test_trigger_delete ON hyper; DROP TRIGGER _0_test_trigger_delete_s_before ON hyper; DROP TRIGGER _0_test_trigger_delete_after ON hyper; DROP TRIGGER _0_test_trigger_delete_s_after ON hyper; DELETE FROM hyper; DROP TRIGGER z_test_trigger_all ON hyper; DROP TRIGGER z_test_trigger_all_after ON hyper; --test create trigger on hypertable -- row triggers: BEFORE CREATE TRIGGER _0_test_trigger_insert BEFORE INSERT ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_update BEFORE UPDATE ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_delete BEFORE delete ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER z_test_trigger_all BEFORE INSERT OR UPDATE OR DELETE ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); -- row triggers: AFTER CREATE TRIGGER _0_test_trigger_insert_after AFTER INSERT ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_update_after AFTER UPDATE ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_delete_after AFTER delete ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER z_test_trigger_all_after AFTER INSERT OR UPDATE OR DELETE ON hyper FOR EACH ROW EXECUTE FUNCTION test_trigger(); -- statement triggers: BEFORE CREATE TRIGGER _0_test_trigger_insert_s_before BEFORE INSERT ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_update_s_before BEFORE UPDATE ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_delete_s_before BEFORE DELETE ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); -- statement triggers: AFTER CREATE TRIGGER _0_test_trigger_insert_s_after AFTER INSERT ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_update_s_after AFTER UPDATE ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _0_test_trigger_delete_s_after AFTER DELETE ON hyper FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987600000000000, 'dev1', 1); INSERT INTO hyper(time, device_id,sensor_1) VALUES (1257987700000000000, 'dev2', 1), (1257987800000000000, 'dev2', 1); UPDATE hyper SET sensor_1 = 2; DELETE FROM hyper; CREATE TABLE vehicles ( vehicle_id INTEGER PRIMARY KEY, vin_number CHAR(17), last_checkup TIMESTAMP ); CREATE TABLE color ( color_id INTEGER PRIMARY KEY, notes text ); CREATE TABLE location ( time TIMESTAMP NOT NULL, vehicle_id INTEGER REFERENCES vehicles (vehicle_id), color_id INTEGER, --no reference since gonna populate a hypertable latitude FLOAT, longitude FLOAT ); CREATE OR REPLACE FUNCTION create_vehicle_trigger_fn() RETURNS TRIGGER LANGUAGE PLPGSQL AS $BODY$ BEGIN INSERT INTO vehicles VALUES(NEW.vehicle_id, NULL, NULL) ON CONFLICT DO NOTHING; RETURN NEW; END $BODY$; CREATE OR REPLACE FUNCTION create_color_trigger_fn() RETURNS TRIGGER LANGUAGE PLPGSQL AS $BODY$ BEGIN --test subtxns within triggers BEGIN INSERT INTO color VALUES(NEW.color_id, 'n/a'); EXCEPTION WHEN unique_violation THEN -- Nothing to do, just continue END; RETURN NEW; END $BODY$; CREATE TRIGGER create_color_trigger BEFORE INSERT OR UPDATE ON location FOR EACH ROW EXECUTE FUNCTION create_color_trigger_fn(); SELECT create_hypertable('location', 'time'); --make color also a hypertable SELECT create_hypertable('color', 'color_id', chunk_time_interval=>10); -- Test that we can create and use triggers with another user GRANT TRIGGER, INSERT, SELECT, UPDATE ON location TO :ROLE_DEFAULT_PERM_USER_2; GRANT SELECT, INSERT, UPDATE ON color TO :ROLE_DEFAULT_PERM_USER_2; GRANT SELECT, INSERT, UPDATE ON vehicles TO :ROLE_DEFAULT_PERM_USER_2; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2; CREATE TRIGGER create_vehicle_trigger BEFORE INSERT OR UPDATE ON location FOR EACH ROW EXECUTE FUNCTION create_vehicle_trigger_fn(); INSERT INTO location VALUES('2017-01-01 01:02:03', 1, 1, 40.7493226,-73.9771259); INSERT INTO location VALUES('2017-01-01 01:02:04', 1, 20, 24.7493226,-73.9771259); INSERT INTO location VALUES('2017-01-01 01:02:03', 23, 1, 40.7493226,-73.9771269); INSERT INTO location VALUES('2017-01-01 01:02:03', 53, 20, 40.7493226,-73.9771269); UPDATE location SET vehicle_id = 52 WHERE vehicle_id = 53; SELECT * FROM location; SELECT * FROM vehicles; SELECT * FROM color; -- switch back to default user to run some dropping tests \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER; \set ON_ERROR_STOP 0 -- test that disable trigger is disallowed ALTER TABLE location DISABLE TRIGGER create_vehicle_trigger; \set ON_ERROR_STOP 1 -- test that drop trigger works DROP TRIGGER create_color_trigger ON location; DROP TRIGGER create_vehicle_trigger ON location; -- test that drop trigger doesn't cause leftovers that mean that dropping chunks or hypertables no longer works SELECT count(1) FROM pg_depend d WHERE d.classid = 'pg_trigger'::regclass AND NOT EXISTS (SELECT 1 FROM pg_trigger WHERE oid = d.objid); DROP TABLE location; -- test triggers with transition tables -- test creating hypertable from table with triggers with transition tables CREATE TABLE transition_test(time timestamptz NOT NULL); CREATE TRIGGER t1_stmt AFTER INSERT ON transition_test REFERENCING NEW TABLE AS new_trans FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER t1_row AFTER INSERT ON transition_test REFERENCING NEW TABLE AS new_trans FOR EACH ROW EXECUTE FUNCTION test_trigger(); -- We do not support ROW triggers with transition tables, so we need -- to remove it to be able to create the hypertable. \set ON_ERROR_STOP 0 SELECT create_hypertable('transition_test','time'); \set ON_ERROR_STOP 1 DROP TRIGGER t1_row ON transition_test; SELECT create_hypertable('transition_test','time'); -- Insert some rows to create a chunk INSERT INTO transition_test values ('2020-01-10'); SELECT chunk FROM show_chunks('transition_test') tbl(chunk) limit 1 \gset -- test creating trigger with transition tables on existing hypertable CREATE TRIGGER t3 AFTER UPDATE ON transition_test REFERENCING NEW TABLE AS new_trans OLD TABLE AS old_trans FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER t4 AFTER DELETE ON transition_test REFERENCING OLD TABLE AS old_trans FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); INSERT INTO transition_test values ('2020-01-11'); COPY transition_test FROM STDIN; 2020-01-09 \. UPDATE transition_test SET time = '2020-01-12' WHERE time = '2020-01-11'; DELETE FROM transition_test WHERE time = '2020-01-12'; \set ON_ERROR_STOP 0 CREATE TRIGGER t3 AFTER UPDATE ON :chunk REFERENCING NEW TABLE AS new_trans OLD TABLE AS old_trans FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER t4 AFTER DELETE ON :chunk REFERENCING OLD TABLE AS old_trans FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER t5 AFTER INSERT ON transition_test REFERENCING NEW TABLE AS new_trans FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER t6 AFTER UPDATE ON transition_test REFERENCING NEW TABLE AS new_trans OLD TABLE AS old_trans FOR EACH ROW EXECUTE FUNCTION test_trigger(); CREATE TRIGGER t7 AFTER DELETE ON transition_test REFERENCING OLD TABLE AS old_trans FOR EACH ROW EXECUTE FUNCTION test_trigger(); \set ON_ERROR_STOP 1 ================================================ FILE: test/sql/truncate.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \o /dev/null \ir include/insert_two_partitions.sql \o SELECT * FROM _timescaledb_catalog.hypertable; SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; SELECT * FROM test.show_subtables('"two_Partitions"'); SELECT * FROM "two_Partitions"; SET client_min_messages = WARNING; TRUNCATE "two_Partitions"; SELECT * FROM _timescaledb_catalog.hypertable; SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; -- should be empty SELECT * FROM test.show_subtables('"two_Partitions"'); SELECT * FROM "two_Partitions"; INSERT INTO public."two_Partitions"("timeCustom", device_id, series_0, series_1) VALUES (1257987600000000000, 'dev1', 1.5, 1), (1257987600000000000, 'dev1', 1.5, 2), (1257894000000000000, 'dev2', 1.5, 1), (1257894002000000000, 'dev1', 2.5, 3); SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, status, osm_chunk FROM _timescaledb_catalog.chunk; CREATE VIEW dependent_view AS SELECT * FROM _timescaledb_internal._hyper_1_5_chunk; CREATE OR REPLACE FUNCTION test_trigger() RETURNS TRIGGER LANGUAGE PLPGSQL AS $BODY$ DECLARE cnt INTEGER; BEGIN RAISE WARNING 'FIRING trigger when: % level: % op: % cnt: % trigger_name %', tg_when, tg_level, tg_op, cnt, tg_name; IF TG_OP = 'DELETE' THEN RETURN OLD; END IF; RETURN NEW; END $BODY$; -- test truncate on a chunk CREATE TRIGGER _test_truncate_before BEFORE TRUNCATE ON _timescaledb_internal._hyper_1_5_chunk FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); CREATE TRIGGER _test_truncate_after AFTER TRUNCATE ON _timescaledb_internal._hyper_1_5_chunk FOR EACH STATEMENT EXECUTE FUNCTION test_trigger(); \set ON_ERROR_STOP 0 TRUNCATE "two_Partitions"; -- cannot TRUNCATE ONLY a hypertable TRUNCATE ONLY "two_Partitions" CASCADE; \set ON_ERROR_STOP 1 -- create a regular table to make sure we can truncate it in the same call CREATE TABLE truncate_normal (color int); INSERT INTO truncate_normal VALUES (1); SELECT * FROM truncate_normal; -- fix for bug #3580 \set ON_ERROR_STOP 0 TRUNCATE nonexistentrelation; \set ON_ERROR_STOP 1 CREATE TABLE truncate_nested (color int); INSERT INTO truncate_nested VALUES (2); SELECT * FROM truncate_normal, truncate_nested; CREATE FUNCTION fn_truncate_nested() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN TRUNCATE truncate_nested; RETURN NEW; END; $$; CREATE TRIGGER tg_truncate_nested BEFORE TRUNCATE ON truncate_normal FOR EACH STATEMENT EXECUTE FUNCTION fn_truncate_nested(); TRUNCATE truncate_normal; SELECT * FROM truncate_normal, truncate_nested; INSERT INTO truncate_normal VALUES (3); INSERT INTO truncate_nested VALUES (4); SELECT * FROM truncate_normal, truncate_nested; TRUNCATE truncate_normal; SELECT * FROM truncate_normal, truncate_nested; INSERT INTO truncate_normal VALUES (5); INSERT INTO truncate_nested VALUES (6); SELECT * FROM truncate_normal, truncate_nested; SELECT * FROM test.show_subtables('"two_Partitions"'); TRUNCATE "two_Partitions", truncate_normal CASCADE; -- should be empty SELECT * FROM test.show_subtables('"two_Partitions"'); SELECT * FROM "two_Partitions"; SELECT * FROM truncate_normal, truncate_nested; -- test TRUNCATE can be performed by a user -- with TRUNCATE privilege who is not table owner \c :TEST_DBNAME :ROLE_SUPERUSER CREATE ROLE owner WITH LOGIN; CREATE ROLE truncator WITH LOGIN; CREATE DATABASE test_trunc_ht OWNER owner; \c test_trunc_ht :ROLE_SUPERUSER SET client_min_messages = ERROR; CREATE EXTENSION timescaledb; RESET client_min_messages; \c test_trunc_ht owner CREATE TABLE test_hypertable (time TIMESTAMP WITHOUT TIME ZONE NOT NULL, value DOUBLE PRECISION); SELECT create_hypertable('test_hypertable', 'time'); -- fail since we don't have TRUNCATE privileges yet \set ON_ERROR_STOP 0 \c test_trunc_ht truncator TRUNCATE TABLE test_hypertable; \set ON_ERROR_STOP 1 \c test_trunc_ht owner GRANT TRUNCATE ON test_hypertable TO truncator; -- now succeed after privilege was granted \c test_trunc_ht truncator; TRUNCATE TABLE test_hypertable; \c :TEST_DBNAME :ROLE_SUPERUSER -- set client_min_messages to ERROR to suppress warnings about orphaned files SET client_min_messages TO ERROR; DROP DATABASE test_trunc_ht WITH (FORCE); DROP ROLE owner; DROP ROLE truncator; ================================================ FILE: test/sql/trusted_extension.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE DATABASE trusted_test; GRANT CREATE ON DATABASE trusted_test TO :ROLE_1; \c trusted_test :ROLE_READ_ONLY \set ON_ERROR_STOP 0 CREATE EXTENSION timescaledb; \set ON_ERROR_STOP 1 \c trusted_test :ROLE_1 -- user shouldn't have superuser privilege SELECT rolsuper FROM pg_roles WHERE rolname=user; SET client_min_messages TO ERROR; CREATE EXTENSION timescaledb; RESET client_min_messages; CREATE TABLE t(time timestamptz); SELECT create_hypertable('t','time'); INSERT INTO t VALUES ('2000-01-01'), ('2001-01-01'); SELECT * FROM t ORDER BY 1; SELECT * FROM timescaledb_information.hypertables; SELECT * FROM test.relation WHERE schema = '_timescaledb_internal' AND name LIKE '\_hyper%'; DROP EXTENSION timescaledb CASCADE; \c :TEST_DBNAME :ROLE_SUPERUSER DROP DATABASE trusted_test WITH (FORCE); ================================================ FILE: test/sql/ts_merge.sql.in ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER \set ON_ERROR_STOP 0 \set VERBOSITY default SET client_min_messages TO error; \set TEST_BASE_NAME ts_merge SELECT format('include/%s_load.sql', :'TEST_BASE_NAME') AS "TEST_LOAD_NAME", format('include/%s_load_ht.sql', :'TEST_BASE_NAME') AS "TEST_LOAD_HT_NAME", format('include/%s_query.sql', :'TEST_BASE_NAME') AS "TEST_QUERY_NAME", format('%s/results/%s_ht_results.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') AS "TEST_RESULTS_WITH_HYPERTABLE", format('%s/results/%s_results.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') AS "TEST_RESULTS_WITH_NO_HYPERTABLE" \gset SELECT format('\! diff -u --label "Base pg table results" --label "Hypertable results" %s %s', :'TEST_RESULTS_WITH_HYPERTABLE', :'TEST_RESULTS_WITH_NO_HYPERTABLE') AS "DIFF_CMD" \gset \ir :TEST_LOAD_NAME -- run tests on normal table \o :TEST_RESULTS_WITH_NO_HYPERTABLE \ir :TEST_QUERY_NAME \o \ir :TEST_LOAD_HT_NAME -- run tests on hypertable \o :TEST_RESULTS_WITH_HYPERTABLE \ir :TEST_QUERY_NAME \o :DIFF_CMD ================================================ FILE: test/sql/update.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \o /dev/null \ir include/insert_single.sql \o -- Make sure UPDATE isn't optimized if it includes Append plans -- Need to turn of nestloop to make append appear the same on PG96 and PG10 set enable_nestloop = 'off'; CREATE OR REPLACE FUNCTION series_val() RETURNS integer LANGUAGE PLPGSQL STABLE AS $BODY$ BEGIN RETURN 5; END; $BODY$; -- ConstraintAwareAppend applied for SELECT EXPLAIN (buffers off, costs off) SELECT FROM "one_Partition" WHERE series_1 IN (SELECT series_1 FROM "one_Partition" WHERE series_1 > series_val()); -- ConstraintAwareAppend NOT applied for UPDATE EXPLAIN (buffers off, costs off) UPDATE "one_Partition" SET series_1 = 8 WHERE series_1 IN (SELECT series_1 FROM "one_Partition" WHERE series_1 > series_val()); SELECT * FROM "one_Partition" ORDER BY "timeCustom", device_id, series_0, series_1, series_2; UPDATE "one_Partition" SET series_1 = 8 WHERE series_1 IN (SELECT series_1 FROM "one_Partition" WHERE series_1 > series_val()); SELECT * FROM "one_Partition" ORDER BY "timeCustom", device_id, series_0, series_1, series_2; UPDATE "one_Partition" SET series_1 = 47; UPDATE "one_Partition" SET series_bool = true; SELECT * FROM "one_Partition" ORDER BY "timeCustom", device_id, series_0, series_1, series_2; -- test update on chunks directly CREATE TABLE direct_update(time timestamptz) WITH (tsdb.hypertable); INSERT INTO direct_update VALUES ('2020-01-01'); SELECT show_chunks('direct_update') AS "CHUNK" \gset --should have ModifyHyperable node EXPLAIN (costs off, timing off, summary off) UPDATE :CHUNK SET time = time + INTERVAL '1 minute'; EXPLAIN (costs off, timing off, summary off) UPDATE ONLY :CHUNK SET time = time + INTERVAL '1 minute'; -- correct time range should succeed UPDATE :CHUNK SET time = time + INTERVAL '1 minute' RETURNING *; UPDATE ONLY :CHUNK SET time = time + INTERVAL '1 minute' RETURNING *; -- crossing chunk boundary should fail \set ON_ERROR_STOP 0 UPDATE :CHUNK SET time = time + INTERVAL '1 month' RETURNING *; UPDATE ONLY :CHUNK SET time = time + INTERVAL '1 month' RETURNING *; \set ON_ERROR_STOP 1 -- github issue #6790 -- test UPDATE with WHERE EXISTS on hypertable CREATE TABLE i6790_update(time timestamptz NOT NULL, device int, value float) WITH (tsdb.hypertable); INSERT INTO i6790_update SELECT t, 1, 0.1 FROM generate_series('2026-01-01'::timestamptz, '2026-01-03'::timestamptz, interval '12 hours') t; -- UPDATE with simple EXISTS - creates gating Result node(s) wrapping ChunkAppend UPDATE i6790_update SET value = 0.2 WHERE EXISTS (SELECT 1); SELECT count(*) FROM i6790_update WHERE value = 0.2; -- UPDATE with correlated EXISTS UPDATE i6790_update SET value = 0.3 WHERE EXISTS (SELECT 1 FROM i6790_update g WHERE g.device = i6790_update.device); SELECT count(*) FROM i6790_update WHERE value = 0.3; ================================================ FILE: test/sql/updates/catalog_missing_columns.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. --PG11 added an optimization where columns that were added by --an ALTER TABLE that had a DEFAULT value did not cause a table re-write. --Instead, those columns are filled with the default value on read. --But, this mechanism does not apply to catalog tables and does --not work with our catalog scanning code. --Thus make sure all catalog tables do not have this enabled (a.atthasmissing == false) CREATE OR REPLACE FUNCTION timescaledb_catalog_has_no_missing_columns() RETURNS VOID LANGUAGE PLPGSQL STABLE AS $BODY$ DECLARE cnt INTEGER; rel_names TEXT; BEGIN SELECT count(*), string_agg(c.relname,' ;') FROM pg_namespace n INNER JOIN pg_class c ON (c.relnamespace = n.oid) INNER JOIN pg_attribute a ON attrelid=c.oid WHERE a.attnum >= 0 AND (n.nspname='_timescaledb_catalog' OR n.nspname = '_timescaledb_config' OR n.nspname='_timescaledb_internal') AND a.atthasmissing INTO STRICT cnt, rel_names; IF cnt != 0 THEN RAISE EXCEPTION 'Some catalog tables were altered without a table re-write: %', rel_names; END IF; END; $BODY$; SELECT timescaledb_catalog_has_no_missing_columns(); ================================================ FILE: test/sql/updates/cleanup.bigint.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. DROP TABLE PUBLIC."two_Partitions" CASCADE; ================================================ FILE: test/sql/updates/cleanup.chunk_skipping.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. DROP TABLE IF EXISTS chunkskip; ================================================ FILE: test/sql/updates/cleanup.compression.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. DROP TABLE compress; DROP TYPE custom_type_for_compression; ================================================ FILE: test/sql/updates/cleanup.constraints.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. ALTER TABLE "two_Partitions" DROP CONSTRAINT IF EXISTS two_Partitions_device_id_2_fkey; DROP TABLE IF EXISTS devices; ================================================ FILE: test/sql/updates/cleanup.continuous_aggs.v2.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Collect information about different features so that we can pick -- the right usage. Some of these are changed in the same version, but -- we keep them separate anyway so that we can do additional checking -- if necessary. SELECT extversion < '2.0.0' AS has_refresh_mat_view, extversion < '2.0.0' AS has_drop_chunks_old_interface, extversion < '2.0.0' AS has_ignore_invalidations_older_than, extversion < '2.0.0' AS has_max_interval_per_job, extversion >= '2.0.0' AS has_create_mat_view, extversion >= '2.0.0' AS has_continuous_aggs_policy FROM pg_extension WHERE extname = 'timescaledb' \gset \if :has_continuous_aggs_policy SELECT remove_continuous_aggregate_policy('mat_drop'); \endif \if :has_create_mat_view DROP MATERIALIZED VIEW mat_drop; \else DROP VIEW mat_drop; \endif DROP TABLE drop_test; \if :has_create_mat_view DROP MATERIALIZED VIEW mat_conflict; \else DROP VIEW mat_conflict; \endif DROP TABLE conflict_test; \if :has_create_mat_view DROP MATERIALIZED VIEW mat_inttime; DROP MATERIALIZED VIEW mat_inttime2; \else DROP VIEW mat_inttime; DROP VIEW mat_inttime2; \endif DROP FUNCTION integer_now_test; DROP TABLE IF EXISTS int_time_test; \if :has_create_mat_view DROP MATERIALIZED VIEW mat_inval; \else DROP VIEW mat_inval; \endif DROP TABLE inval_test; \if :has_create_mat_view DROP MATERIALIZED VIEW mat_ignoreinval; \else DROP VIEW mat_ignoreinval; \endif \if :has_create_mat_view DROP MATERIALIZED VIEW cagg.realtime_mat; \else DROP VIEW cagg.realtime_mat; \endif DROP SCHEMA cagg; \if :has_create_mat_view DROP MATERIALIZED VIEW mat_before; \else DROP VIEW mat_before; \endif \if :has_create_mat_view DROP MATERIALIZED VIEW rename_cols; \else DROP VIEW rename_cols; \endif DROP TABLE conditions_before; DROP TYPE custom_type; ================================================ FILE: test/sql/updates/cleanup.policies.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SELECT remove_reorder_policy('policy_test_timestamptz'); SELECT remove_retention_policy('policy_test_timestamptz'); SELECT remove_compression_policy('policy_test_timestamptz'); ================================================ FILE: test/sql/updates/cleanup.sparse_index.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. DROP TABLE IF EXISTS bloom; ================================================ FILE: test/sql/updates/cleanup.timestamp.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. DROP TABLE IF EXISTS PUBLIC.hyper_timestamp; ================================================ FILE: test/sql/updates/cleanup.v10.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \ir cleanup.v9.sql \ir cleanup.sparse_index.sql ================================================ FILE: test/sql/updates/cleanup.v7.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \ir cleanup.bigint.sql \ir cleanup.constraints.sql \ir cleanup.timestamp.sql \ir cleanup.continuous_aggs.v2.sql \ir cleanup.compression.sql \ir cleanup.policies.sql ================================================ FILE: test/sql/updates/cleanup.v8.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \ir cleanup.v7.sql ================================================ FILE: test/sql/updates/cleanup.v9.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \ir cleanup.v8.sql \ir cleanup.chunk_skipping.sql ================================================ FILE: test/sql/updates/post.catalog.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SELECT NOT (extversion >= '2.19.0' AND extversion <= '2.20.3') AS has_fixed_compression_algorithms FROM pg_extension WHERE extname = 'timescaledb' \gset \if :PG_UPGRADE_TEST \else \d+ _timescaledb_catalog.hypertable \d+ _timescaledb_catalog.chunk \d+ _timescaledb_catalog.dimension \d+ _timescaledb_catalog.dimension_slice \d+ _timescaledb_catalog.chunk_constraint \d+ _timescaledb_catalog.tablespace \endif -- since we forgot to add bool and null compression with 2.19.0 to the preinstall -- script fresh installations of 2.19+ won't have these compression algorithms \if :has_fixed_compression_algorithms SELECT * from _timescaledb_catalog.compression_algorithm algo ORDER BY algo; \endif SELECT nspname AS Schema, relname AS Name, -- PG17 introduced MAINTAIN acl (m) so removed it to keep output backward compatible replace(unnest(relacl)::text, 'm', '') as ACL FROM pg_class JOIN pg_namespace ns ON relnamespace = ns.oid WHERE nspname IN ('_timescaledb_catalog', '_timescaledb_config') ORDER BY Schema, Name, ACL; SELECT nspname AS schema, relname AS name, -- PG17 introduced MAINTAIN acl (m) so removed it to keep output backward compatible replace(unnest(initprivs)::text, 'm', '') AS initpriv FROM pg_class cl JOIN pg_namespace ns ON ns.oid = relnamespace LEFT JOIN pg_init_privs ON objoid = cl.oid AND objsubid = 0 WHERE classoid = 'pg_class'::regclass AND nspname IN ('_timescaledb_catalog', '_timescaledb_config') ORDER BY schema, name, initpriv; -- indexes in _timescaledb_catalog schema SELECT n.nspname as "Schema", c.relname as "Name", CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 't' THEN 'TOAST table' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'partitioned table' WHEN 'I' THEN 'partitioned index' END as "Type", pg_catalog.pg_get_userbyid(c.relowner) as "Owner", c2.relname as "Table" FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace LEFT JOIN pg_catalog.pg_am am ON am.oid = c.relam LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid LEFT JOIN pg_catalog.pg_class c2 ON i.indrelid = c2.oid WHERE c.relkind IN ('i','I','') AND n.nspname = '_timescaledb_catalog' AND pg_catalog.pg_table_is_visible(c.oid) ORDER BY 1,2; -- sequences in _timescaledb_catalog schema SELECT n.nspname as "Schema", c.relname as "Name", CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 't' THEN 'TOAST table' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'partitioned table' WHEN 'I' THEN 'partitioned index' END as "Type", pg_catalog.pg_get_userbyid(c.relowner) as "Owner", CASE c.relpersistence WHEN 'p' THEN 'permanent' WHEN 't' THEN 'temporary' WHEN 'u' THEN 'unlogged' END as "Persistence", pg_catalog.pg_size_pretty(pg_catalog.pg_table_size(c.oid)) as "Size", pg_catalog.obj_description(c.oid, 'pg_class') as "Description" FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind IN ('S','') AND n.nspname = '_timescaledb_catalog' AND pg_catalog.pg_table_is_visible(c.oid) ORDER BY 1,2; -- Functions in schemas: -- * _timescaledb_internal -- * _timescaledb_functions -- * public SELECT n.nspname as "Schema", p.proname as "Name", pg_catalog.pg_get_function_result(p.oid) as "Result data type", pg_catalog.pg_get_function_arguments(p.oid) as "Argument data types", CASE p.prokind WHEN 'a' THEN 'agg' WHEN 'w' THEN 'window' WHEN 'p' THEN 'proc' ELSE 'func' END as "Type", CASE WHEN p.provolatile = 'i' THEN 'immutable' WHEN p.provolatile = 's' THEN 'stable' WHEN p.provolatile = 'v' THEN 'volatile' END as "Volatility", CASE WHEN p.proparallel = 'r' THEN 'restricted' WHEN p.proparallel = 's' THEN 'safe' WHEN p.proparallel = 'u' THEN 'unsafe' END as "Parallel", pg_catalog.pg_get_userbyid(p.proowner) as "Owner", CASE WHEN prosecdef THEN 'definer' ELSE 'invoker' END AS "Security", CASE WHEN pg_catalog.array_length(p.proacl, 1) = 0 THEN '(none)' ELSE pg_catalog.array_to_string(p.proacl, E'\n') END AS "Access privileges", l.lanname as "Language", p.prosrc as "Source code", CASE WHEN l.lanname IN ('internal', 'c') THEN p.prosrc END as "Internal name", pg_catalog.obj_description(p.oid, 'pg_proc') as "Description" FROM pg_catalog.pg_proc p LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace LEFT JOIN pg_catalog.pg_language l ON l.oid = p.prolang WHERE n.nspname OPERATOR(pg_catalog.~) '^(_timescaledb_internal|_timescaledb_functions|public)$' COLLATE pg_catalog.default ORDER BY 1, 2, 4; \dy \a \d public.* \a -- Keep the output backward compatible \if :PG_UPGRADE_TEST SELECT oid AS extoid FROM pg_catalog.pg_extension WHERE extname = 'timescaledb' \gset WITH ext AS ( SELECT pg_catalog.pg_describe_object(classid, objid, 0) AS objdesc FROM pg_catalog.pg_depend WHERE refclassid = 'pg_catalog.pg_extension'::pg_catalog.regclass AND refobjid = :'extoid' AND deptype = 'e' ORDER BY 1 ) SELECT objdesc AS "Object description" FROM ext WHERE objdesc !~ '^type' OR objdesc ~ '^type _timescaledb_internal.(compressed_data|dimension_info)$' ORDER BY 1; WITH ext AS ( SELECT pg_catalog.pg_describe_object(classid, objid, 0) AS objdesc FROM pg_catalog.pg_depend WHERE refclassid = 'pg_catalog.pg_extension'::pg_catalog.regclass AND refobjid = :'extoid' AND deptype = 'e' ORDER BY 1 ) SELECT count(*) FROM ext WHERE objdesc !~ '^type' OR objdesc ~ '^type _timescaledb_internal.(compressed_data|dimension_info)$'; \else \dx+ timescaledb SELECT count(*) FROM pg_depend WHERE refclassid = 'pg_extension'::regclass AND refobjid = (SELECT oid FROM pg_extension WHERE extname = 'timescaledb'); \endif -- The list of tables configured to be dumped. SELECT unnest(extconfig)::regclass::text, unnest(extcondition) FROM pg_extension WHERE extname = 'timescaledb' ORDER BY 1; -- Show chunks that include owner in the output SELECT c.id, c.hypertable_id, c.schema_name, c.table_name, cl.relowner::regrole FROM _timescaledb_catalog.chunk c INNER JOIN pg_class cl ON (cl.oid=format('%I.%I', schema_name, table_name)::regclass) ORDER BY c.id, c.hypertable_id; SELECT chunk_constraint.* FROM _timescaledb_catalog.chunk_constraint JOIN _timescaledb_catalog.chunk ON chunk.id = chunk_constraint.chunk_id ORDER BY chunk_constraint.chunk_id, chunk_constraint.dimension_slice_id, chunk_constraint.constraint_name; -- Show attnum of all regclass objects belonging to our extension -- if those are not the same between fresh install/update our update scripts are broken SELECT att.attrelid::regclass, att.attnum, att.attname FROM pg_depend dep INNER JOIN pg_extension ext ON (dep.refobjid=ext.oid AND ext.extname = 'timescaledb') INNER JOIN pg_attribute att ON (att.attrelid=dep.objid AND att.attnum > 0) WHERE classid='pg_class'::regclass ORDER BY attrelid::regclass::text,att.attnum; -- Show constraints SELECT conrelid::regclass::text, conname, pg_get_constraintdef(oid) FROM pg_constraint WHERE conrelid::regclass::text ~ '^_timescaledb_' \if :PG_UPGRADE_TEST AND pg_get_constraintdef(oid) NOT LIKE 'NOT NULL %' \endif ORDER BY 1, 2, 3; SELECT * FROM _timescaledb_catalog.compression_settings ORDER BY relid::regclass; ================================================ FILE: test/sql/updates/post.chunk_skipping.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Show chunk column stats after updating the extension. We need to -- exclude the rows with chunk_id = 0 because we cannot keep those on -- downgrade due to FK constraint. Showing them would mean a diff in -- the output. We can still test that 0 chunk_ids are converted to -- NULL values during upgrades, however. SELECT * FROM _timescaledb_catalog.chunk_column_stats WHERE chunk_id IS NULL OR chunk_id > 0 ORDER BY id; ================================================ FILE: test/sql/updates/post.compression.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SELECT * FROM compress ORDER BY time DESC, small_cardinality, large_cardinality, some_double, some_int, some_custom, some_bool; -- This recompression is necessary only for downgrades from 2.17 to 2.16.1 -- due to downgrade migration requiring to add sequence number metadata -- column and causing compressed chunks to be unordered. -- Recompressing the chunks fully fixes the difference. SELECT count(decompress_chunk(ch, true)) FROM show_chunks('compress') ch; SELECT count(compress_chunk(ch, true)) FROM show_chunks('compress') ch; -- Running this query again to confirm data is consistent even after above recompression SELECT * FROM compress ORDER BY time DESC, small_cardinality, large_cardinality, some_double, some_int, some_custom, some_bool; INSERT INTO compress(time, small_cardinality, large_cardinality, some_double, some_int, some_custom, some_bool) SELECT g, 'QW', g::text, 2, 0, (100,4)::custom_type_for_compression, false FROM generate_series('2019-11-01 00:00'::timestamp, '2019-12-15 00:00'::timestamp, '1 day') g; SELECT count(compress_chunk(ch, true)) FROM show_chunks('compress') ch; SELECT * FROM compress ORDER BY time DESC, small_cardinality, large_cardinality, some_double, some_int, some_custom, some_bool; \x on WITH hypertables AS ( SELECT ht.id hypertable_id, ht.schema_name, ht.table_name, ht.compressed_hypertable_id FROM pg_class cl JOIN pg_namespace ns ON ns.oid = relnamespace JOIN _timescaledb_catalog.hypertable ht ON relname = ht.table_name AND nspname = ht.schema_name ), table_summary AS ( SELECT format('%I.%I', ht1.schema_name, ht1.table_name) AS hypertable_name, format('%I.%I', ht2.schema_name, ht2.table_name) AS compressed_hypertable_name, format('%I.%I', ch2.schema_name, ch2.table_name) AS compressed_chunk_name FROM hypertables ht1 JOIN hypertables ht2 ON ht1.compressed_hypertable_id = ht2.hypertable_id JOIN _timescaledb_catalog.chunk ch2 ON ch2.hypertable_id = ht2.hypertable_id ) SELECT hypertable_name, (SELECT relacl FROM pg_class WHERE oid = hypertable_name::regclass) AS hypertable_acl, compressed_hypertable_name, (SELECT relacl FROM pg_class WHERE oid = compressed_hypertable_name::regclass) AS compressed_hypertable_acl, compressed_chunk_name, (SELECT relacl FROM pg_class WHERE oid = compressed_chunk_name::regclass) AS compressed_chunk_acl FROM table_summary ORDER BY hypertable_name, compressed_hypertable_name, compressed_chunk_name; \x off ================================================ FILE: test/sql/updates/post.continuous_aggs.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SELECT extversion < '2.0.0' AS has_refresh_mat_view FROM pg_extension WHERE extname = 'timescaledb' \gset \if :has_refresh_mat_view REFRESH MATERIALIZED VIEW mat_before; \else CALL refresh_continuous_aggregate('mat_before',NULL,NULL); \endif \x on SELECT * FROM mat_before ORDER BY bucket, location; \x off --cause invalidations in the time range that is already --materialized. However, shift time by one second so that each --(timestamp, location) pair is unique. Otherwise last(temperature, --timec) won't be deterministic. INSERT INTO conditions_before SELECT generate_series('2018-12-01 00:01'::timestamp, '2018-12-31 00:01'::timestamp, '1 day'), 'POR', 165, 75, 40, 70, NULL, (1,2)::custom_type, 2, true; --cause invalidations way in the past INSERT INTO conditions_before SELECT generate_series('2017-12-01 00:01'::timestamp, '2017-12-31 00:01'::timestamp, '1 day'), 'POR', 1065, 75, 40, 70, NULL, (1,2)::custom_type, 2, true; \x on SELECT * FROM mat_before ORDER BY bucket, location; \x off CALL refresh_continuous_aggregate('mat_before',NULL,NULL); --the max of the temp for the POR should now be 165 \x on SELECT * FROM mat_before ORDER BY bucket, location; \x off -- Output the ACLs for each internal cagg object SELECT cl.oid::regclass::text AS reloid, unnest(relacl)::text AS relacl FROM _timescaledb_catalog.continuous_agg ca JOIN _timescaledb_catalog.hypertable h ON (ca.mat_hypertable_id = h.id) JOIN pg_class cl ON (cl.oid IN (format('%I.%I', h.schema_name, h.table_name)::regclass, format('%I.%I', direct_view_schema, direct_view_name)::regclass, format('%I.%I', partial_view_schema, partial_view_name)::regclass)) ORDER BY reloid, relacl; -- Output ACLs for chunks on materialized hypertables SELECT inhparent::regclass::text AS parent, cl.oid::regclass::text AS chunk, unnest(relacl)::text AS acl FROM _timescaledb_catalog.continuous_agg ca JOIN _timescaledb_catalog.hypertable h ON (ca.mat_hypertable_id = h.id) JOIN pg_inherits inh ON (inh.inhparent = format('%I.%I', h.schema_name, h.table_name)::regclass) JOIN pg_class cl ON (cl.oid = inh.inhrelid) ORDER BY parent, chunk, acl; -- Verify privileges on internal cagg objects. The privileges on the -- materialized hypertable, partial view, and direct view should match -- the user-facing user view. DO $$ DECLARE user_view_rel regclass; user_view_acl aclitem[]; rel regclass; acl aclitem[]; acl_matches boolean; BEGIN FOR user_view_rel, user_view_acl IN SELECT cl.oid, cl.relacl FROM pg_class cl JOIN _timescaledb_catalog.continuous_agg ca ON (format('%I.%I', ca.user_view_schema, ca.user_view_name)::regclass = cl.oid) LOOP FOR rel, acl, acl_matches IN SELECT cl.oid, cl.relacl, COALESCE(cl.relacl, ARRAY[]::aclitem[]) @> COALESCE(user_view_acl, ARRAY[]::aclitem[]) FROM _timescaledb_catalog.continuous_agg ca JOIN _timescaledb_catalog.hypertable h ON (ca.mat_hypertable_id = h.id) JOIN pg_class cl ON (cl.oid IN (format('%I.%I', h.schema_name, h.table_name)::regclass, format('%I.%I', direct_view_schema, direct_view_name)::regclass, format('%I.%I', partial_view_schema, partial_view_name)::regclass)) WHERE format('%I.%I', ca.user_view_schema, ca.user_view_name)::regclass = user_view_rel LOOP IF NOT acl_matches THEN RAISE EXCEPTION 'privileges mismatch for continuous aggregate "%"', user_view_rel USING DETAIL = format('Privileges for internal object "%s" are [%s], expected [%s].', rel, acl, user_view_acl); END IF; END LOOP; END LOOP; END $$ LANGUAGE PLPGSQL; ================================================ FILE: test/sql/updates/post.continuous_aggs.v2.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \ir post.continuous_aggs.sql \d cagg.* \x on SELECT * FROM cagg.realtime_mat ORDER BY bucket, location; \x off CALL refresh_continuous_aggregate('cagg.realtime_mat',NULL,NULL); \x on SELECT * FROM cagg.realtime_mat ORDER BY bucket, location; \x off SELECT view_name, materialized_only, materialization_hypertable_name FROM timescaledb_information.continuous_aggregates ORDER BY view_name::text; SELECT schedule_interval FROM timescaledb_information.jobs ORDER BY job_id; SELECT maxtemp FROM mat_ignoreinval ORDER BY 1; SELECT materialization_id FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE lowest_modified_value = -9223372036854775808 ORDER BY 1; SELECT count(*) FROM mat_inval; CALL refresh_continuous_aggregate('mat_inval',NULL,NULL); SELECT pg_sleep(0.1); -- ensure refresh completes SELECT count(*) FROM mat_inval; ================================================ FILE: test/sql/updates/post.continuous_aggs.v3.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \ir post.continuous_aggs.v2.sql -- Ensure CAgg is refreshed SELECT extversion < '2.0.0' AS has_refresh_mat_view FROM pg_extension WHERE extname = 'timescaledb' \gset \if :has_refresh_mat_view REFRESH MATERIALIZED VIEW rename_cols; \else CALL refresh_continuous_aggregate('rename_cols',NULL,NULL); \endif SELECT "time", count(*) from rename_cols GROUP BY 1 ORDER BY 1; --verify compression can be enabled ALTER MATERIALIZED VIEW rename_cols SET ( timescaledb.compress='true'); SELECT "time", count(*) from rename_cols GROUP BY 1 ORDER BY 1; ================================================ FILE: test/sql/updates/post.functions.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- check all functions point to the correct library SELECT oid::REGPROCEDURE, probin FROM pg_proc WHERE probin LIKE '%timescale%' ORDER BY oid::REGPROCEDURE::TEXT; ================================================ FILE: test/sql/updates/post.insert.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- INSERT data to create a new chunk after update or restore. INSERT INTO devices(id,floor) VALUES ('dev5', 5); INSERT INTO "two_Partitions"("timeCustom", device_id, device_id_2, series_0, series_1, series_2) VALUES (1258894000000000000, 'dev5', 'dev1', 2.2, 1, 2); ================================================ FILE: test/sql/updates/post.integrity_test.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- We do not dump the size of the tables here since that might differ -- between an updated node and a restored node. For examples, stats -- tables can have different sizes, and this is not relevant for an -- update test. \dt _timescaledb_internal.* CREATE OR REPLACE FUNCTION timescaledb_integrity_test() RETURNS VOID LANGUAGE PLPGSQL STABLE AS $BODY$ DECLARE dimension_slice RECORD; constraint_row RECORD; index_row RECORD; chunk_count INTEGER; chunk_constraint_count INTEGER; BEGIN -- Check integrity of chunk_constraints FOR constraint_row IN SELECT c.conname, h.id AS hypertable_id FROM _timescaledb_catalog.hypertable h INNER JOIN pg_constraint c ON (c.conrelid = format('%I.%I', h.schema_name, h.table_name)::regclass) WHERE c.contype NOT IN ('c','n') LOOP SELECT count(*) FROM _timescaledb_catalog.chunk c WHERE c.hypertable_id = constraint_row.hypertable_id INTO STRICT chunk_count; SELECT count(cc.*) FROM _timescaledb_catalog.chunk_constraint cc, _timescaledb_catalog.chunk c WHERE hypertable_constraint_name = constraint_row.conname AND c.id = cc.chunk_id AND c.hypertable_id = constraint_row.hypertable_id INTO STRICT chunk_constraint_count; IF chunk_constraint_count != chunk_count THEN RAISE EXCEPTION 'Missing chunk constraints for %. Expected %, but found %', constraint_row.conname, chunk_count, chunk_constraint_count; END IF; END LOOP; FOR dimension_slice IN SELECT chunk_id, dimension_slice_id FROM _timescaledb_catalog.chunk_constraint WHERE dimension_slice_id NOT IN (SELECT id FROM _timescaledb_catalog.dimension_slice) LOOP RAISE EXCEPTION 'Missing dimension slice with id % for chunk %.', dimension_slice.dimension_slice_id, dimension_slice.chunk_id; END LOOP; END; $BODY$; SELECT timescaledb_integrity_test(); -- Verify that the default jobs are the same in bgw_job SELECT relnamespace::regnamespace "JOB_SCHEMA" FROM pg_class WHERE relname='bgw_job' and relkind = 'r' \gset SELECT id, application_name FROM :JOB_SCHEMA.bgw_job ORDER BY id; ================================================ FILE: test/sql/updates/post.pg_upgrade.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \pset format aligned \pset tuples_only off \set PG_UPGRADE_TEST true \ir post.catalog.sql \unset PG_UPGRADE_TEST \ir post.policies.sql \ir post.functions.sql SELECT * FROM cagg_join.measurement_daily ORDER BY 1, 2, 3, 4, 5, 6; ================================================ FILE: test/sql/updates/post.policies.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- SELECT * FROM _timescaledb_config.bgw_job WHERE id <> 1 ORDER BY id; SELECT relnamespace::regnamespace "JOB_SCHEMA" FROM pg_class WHERE relname='bgw_job' and relkind = 'r' \gset SELECT id, application_name, schedule_interval, max_runtime, max_retries, retry_period, proc_schema, proc_name, owner, scheduled, hypertable_id, config FROM :JOB_SCHEMA.bgw_job WHERE id <> 1 ORDER BY id; ================================================ FILE: test/sql/updates/post.repair.hierarchical_cagg.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SELECT count(*) FROM agg_test_monthly; ================================================ FILE: test/sql/updates/post.repair.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \z repair_test_int \z repair_test_extra ================================================ FILE: test/sql/updates/post.sequences.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- check sequences did not get reset SELECT seqrelid::regclass, nextval(seqrelid), seqstart, seqincrement, seqmax, seqmin FROM pg_sequence ORDER BY seqrelid::regclass::text; ================================================ FILE: test/sql/updates/post.sparse_index.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Show chunk column stats after updating the extension. We need to -- exclude the rows with chunk_id = 0 because we cannot keep those on -- downgrade due to FK constraint. Showing them would mean a diff in -- the output. We can still test that 0 chunk_ids are converted to -- NULL values during upgrades, however. SELECT * FROM _timescaledb_catalog.compression_settings ORDER BY relid::regclass::text; SELECT schema_name || '.' || table_name AS chunk FROM _timescaledb_catalog.chunk WHERE id = ( SELECT compressed_chunk_id FROM _timescaledb_catalog.chunk WHERE hypertable_id = ( SELECT id FROM _timescaledb_catalog.hypertable WHERE table_name = 'bloom' ) LIMIT 1 ) \gset -- Note: we can't use \d+ here because it prevents changing the TOAST storage flag -- if we do, like in the UUID compression changes from external to extended, then -- the upgrade tests fail \a \d :chunk \a ================================================ FILE: test/sql/updates/post.v10.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \ir post.v9.sql \ir post.sparse_index.sql ================================================ FILE: test/sql/updates/post.v7.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set PG_UPGRADE_TEST false \ir post.catalog.sql \unset PG_UPGRADE_TEST \ir post.insert.sql \ir post.integrity_test.sql \ir catalog_missing_columns.sql \ir post.compression.sql \ir post.continuous_aggs.v2.sql \ir post.policies.sql \if :WITH_SUPERUSER \ir post.sequences.sql \endif \ir post.functions.sql ================================================ FILE: test/sql/updates/post.v8.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set PG_UPGRADE_TEST false \ir post.catalog.sql \unset PG_UPGRADE_TEST \ir post.insert.sql \ir post.integrity_test.sql \ir catalog_missing_columns.sql \ir post.compression.sql \ir post.continuous_aggs.v3.sql \ir post.policies.sql \ir post.sequences.sql \ir post.functions.sql ================================================ FILE: test/sql/updates/post.v9.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \ir post.v8.sql \ir post.chunk_skipping.sql ================================================ FILE: test/sql/updates/pre.cleanup.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Clean up objects that are created by the setup files. Ideally, we -- should clean them up in each post.*.sql file after generating the -- output, but now we do it here. SET client_min_messages TO WARNING; DROP MATERIALIZED VIEW IF EXISTS mat_inval CASCADE; DROP MATERIALIZED VIEW IF EXISTS mat_drop CASCADE; DROP MATERIALIZED VIEW IF EXISTS mat_before CASCADE; DROP MATERIALIZED VIEW IF EXISTS mat_conflict CASCADE; DROP MATERIALIZED VIEW IF EXISTS mat_inttime CASCADE; DROP MATERIALIZED VIEW IF EXISTS mat_inttime2 CASCADE; DROP MATERIALIZED VIEW IF EXISTS mat_ignoreinval CASCADE; DROP MATERIALIZED VIEW IF EXISTS cagg.realtime_mat CASCADE; DROP TABLE IF EXISTS public.hyper_timestamp; DROP TABLE IF EXISTS public."two_Partitions"; DROP TABLE IF EXISTS conditions_before; DROP TABLE IF EXISTS inval_test; DROP TABLE IF EXISTS int_time_test; DROP TABLE IF EXISTS conflict_test; DROP TABLE IF EXISTS drop_test; DROP TABLE IF EXISTS repair_test_timestamptz; DROP TABLE IF EXISTS repair_test_int; DROP TABLE IF EXISTS repair_test_extra; DROP TABLE IF EXISTS repair_test_timestamp; DROP TABLE IF EXISTS repair_test_date; DROP TABLE IF EXISTS compress; DROP TABLE IF EXISTS skip; DROP TABLE IF EXISTS devices; DROP TABLE IF EXISTS disthyper; DROP TABLE IF EXISTS policy_test_timestamptz; DROP TYPE IF EXISTS custom_type; DROP TYPE IF EXISTS custom_type_for_compression; DROP PROCEDURE IF EXISTS _timescaledb_testing.restart_dimension_slice_id; DROP PROCEDURE IF EXISTS _timescaledb_testing.stop_workers; DROP FUNCTION IF EXISTS timescaledb_integrity_test; DROP FUNCTION IF EXISTS timescaledb_catalog_has_no_missing_columns; DROP FUNCTION IF EXISTS integer_now_test; DROP SCHEMA IF EXISTS cagg; DROP SCHEMA IF EXISTS _timescaledb_testing; RESET client_min_messages; ================================================ FILE: test/sql/updates/pre.smoke.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- These functions are used when running smoke tests. For smoke tests -- we assume that we do not have SUPER privileges. CREATE SCHEMA IF NOT EXISTS _timescaledb_testing; CREATE PROCEDURE _timescaledb_testing.restart_dimension_slice_id() LANGUAGE SQL AS ''; CREATE PROCEDURE _timescaledb_testing.stop_workers() LANGUAGE SQL AS ''; ================================================ FILE: test/sql/updates/pre.testing.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- These functions are used when running normal update tests. CREATE SCHEMA IF NOT EXISTS _timescaledb_testing; CREATE OR REPLACE PROCEDURE _timescaledb_testing.restart_dimension_slice_id() LANGUAGE SQL AS $$ ALTER SEQUENCE _timescaledb_catalog.dimension_slice_id_seq RESTART WITH 100; $$; CREATE OR REPLACE PROCEDURE _timescaledb_testing.stop_workers() LANGUAGE PLPGSQL AS $$ BEGIN IF EXISTS (SELECT FROM pg_proc WHERE proname='stop_background_workers' AND pronamespace='_timescaledb_internal'::regnamespace) THEN PERFORM _timescaledb_internal.stop_background_workers(); ELSE PERFORM _timescaledb_functions.stop_background_workers(); END IF; END $$; ================================================ FILE: test/sql/updates/setup.bigint.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE PUBLIC."two_Partitions" ( "timeCustom" BIGINT NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL, UNIQUE("timeCustom", device_id, series_2) ); CREATE INDEX ON PUBLIC."two_Partitions" (device_id, "timeCustom" DESC NULLS LAST) WHERE device_id IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_0) WHERE series_0 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_1) WHERE series_1 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_2) WHERE series_2 IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, series_bool) WHERE series_bool IS NOT NULL; CREATE INDEX ON PUBLIC."two_Partitions" ("timeCustom" DESC NULLS LAST, device_id); DO $$ BEGIN IF (EXISTS (SELECT FROM pg_proc WHERE proname = 'interval_to_usec' AND pronamespace='_timescaledb_internal'::regnamespace)) THEN PERFORM create_hypertable('"public"."two_Partitions"'::regclass, 'timeCustom'::name, 'device_id'::name, associated_schema_name=>'_timescaledb_internal'::text, number_partitions => 2, chunk_time_interval=>_timescaledb_internal.interval_to_usec('1 month')); ELSE PERFORM create_hypertable('"public"."two_Partitions"'::regclass, 'timeCustom'::name, 'device_id'::name, associated_schema_name=>'_timescaledb_internal'::text, number_partitions => 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); END IF; END; $$; ================================================ FILE: test/sql/updates/setup.catalog.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Catalog tables are occasionally rewritten as part of updates, so -- this is to test that privileges are maintained over updates of the -- extension. We could verify that other properties (e.g., comments) -- are maintained here as well, but this is not something we use right -- now. -- -- We do not alter the privileges on _timescaledb_internal since this -- affects both internal objects and two tables that are metadata -- placed in the _timescaledb_internal schema. GRANT SELECT ON ALL TABLES IN SCHEMA _timescaledb_catalog TO tsdbadmin; GRANT SELECT ON ALL SEQUENCES IN SCHEMA _timescaledb_catalog TO tsdbadmin; ALTER DEFAULT PRIVILEGES IN SCHEMA _timescaledb_catalog GRANT SELECT ON TABLES TO tsdbadmin; ALTER DEFAULT PRIVILEGES IN SCHEMA _timescaledb_catalog GRANT SELECT ON SEQUENCES TO tsdbadmin; DO $$ BEGIN IF EXISTS(SELECT FROM pg_namespace WHERE nspname = '_timescaledb_config') THEN GRANT SELECT ON ALL TABLES IN SCHEMA _timescaledb_config TO tsdbadmin; GRANT SELECT ON ALL SEQUENCES IN SCHEMA _timescaledb_config TO tsdbadmin; ALTER DEFAULT PRIVILEGES IN SCHEMA _timescaledb_config GRANT SELECT ON TABLES TO tsdbadmin; ALTER DEFAULT PRIVILEGES IN SCHEMA _timescaledb_config GRANT SELECT ON SEQUENCES TO tsdbadmin; END IF; END $$; ================================================ FILE: test/sql/updates/setup.check.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \echo **** Missing dimension slices **** SELECT hypertable_id, ( SELECT format('%I.%I', schema_name, table_name)::regclass FROM _timescaledb_catalog.hypertable ht WHERE ht.id = ch.hypertable_id ) AS hypertable, chunk_id, dimension_slice_id, constraint_name, attname AS column_name, pg_get_expr(conbin, conrelid) AS constraint_expr FROM _timescaledb_catalog.chunk_constraint cc JOIN _timescaledb_catalog.chunk ch ON cc.chunk_id = ch.id JOIN pg_constraint ON conname = constraint_name JOIN pg_namespace ns ON connamespace = ns.oid AND ns.nspname = ch.schema_name JOIN pg_attribute ON attnum = conkey[1] AND attrelid = conrelid WHERE dimension_slice_id NOT IN (SELECT id FROM _timescaledb_catalog.dimension_slice); ================================================ FILE: test/sql/updates/setup.chunk_skipping.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE chunkskip ( time TIMESTAMPTZ NOT NULL, updated_at TIMESTAMPTZ NOT NULL, location INT, temp FLOAT ); SELECT setseed(0.1); SELECT table_name FROM create_hypertable( 'chunkskip', 'time'); INSERT INTO chunkskip SELECT g, g+interval '1 hour', ceil(random()*20), random()*30 FROM generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day') g; ALTER TABLE chunkskip SET (timescaledb.compress, timescaledb.compress_segmentby='location', timescaledb.compress_orderby='"time" desc'); SELECT count(compress_chunk(ch, true)) FROM show_chunks('chunkskip') ch; DO $$ DECLARE version text[] := (SELECT regexp_split_to_array(extversion,'(\.|-)') FROM pg_extension WHERE extname = 'timescaledb'); BEGIN -- Enable chunk skipping. Doing this after compression to have a -- "special" chunk_id entry of 0. We check that this is changed to -- NULL during upgrade. IF version[1]::int >= 2 AND version[2]::int >= 17 AND (version[2]::int > 17 OR version[3]::int > 0) THEN SET timescaledb.enable_chunk_skipping = true; END IF; END $$; SELECT enable_chunk_skipping('chunkskip', 'updated_at'); ================================================ FILE: test/sql/updates/setup.compression.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TYPE custom_type_for_compression AS (high int, low int); CREATE TABLE compress ( time TIMESTAMPTZ NOT NULL, small_cardinality TEXT NULL, large_cardinality TEXT NULL, dropped TEXT NULL, some_double DOUBLE PRECISION NULL, some_int integer NULL, some_custom custom_type_for_compression NULL, some_bool boolean NULL ); SELECT table_name FROM create_hypertable( 'compress', 'time'); INSERT INTO compress SELECT g, 'POR', g::text, 'lint', 75.0, 40, (1,2)::custom_type_for_compression, true FROM generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day') g; INSERT INTO compress SELECT g, 'POR', NULL, 'lint', NULL, NULL, NULL, NULL FROM generate_series('2018-11-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day') g; INSERT INTO compress SELECT g, 'POR', g::text, 'lint', 94.0, 45, (3,4)::custom_type_for_compression, true FROM generate_series('2018-11-01 00:00'::timestamp, '2018-12-15 00:00'::timestamp, '1 day') g; DO $$ DECLARE ts_minor int := (SELECT (string_to_array(extversion,'.'))[2]::int FROM pg_extension WHERE extname = 'timescaledb'); BEGIN -- support for dropping columns on compressed hypertables was added in 2.10.0 -- so we drop before compressing if the version is before 2.10.0 IF ts_minor < 10 THEN ALTER TABLE compress DROP COLUMN dropped; END IF; ALTER TABLE compress SET (timescaledb.compress, timescaledb.compress_segmentby='small_cardinality', timescaledb.compress_orderby='"time" desc'); PERFORM count(compress_chunk(ch, true)) FROM show_chunks('compress') ch; IF ts_minor >= 10 THEN ALTER TABLE compress DROP COLUMN dropped; END IF; END $$; \if :WITH_ROLES GRANT SELECT ON compress TO tsdbadmin; \endif ================================================ FILE: test/sql/updates/setup.constraints.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Secondary devices table to test foreign keys in "two_Partitions" CREATE TABLE devices ( id TEXT PRIMARY KEY, floor INTEGER ); INSERT INTO devices(id,floor) VALUES ('dev1', 1), ('dev2', 2), ('dev3', 3); -- Setup "two_Partitions" to use foreign key constraints ALTER TABLE "two_Partitions" ADD COLUMN device_id_2 TEXT NOT NULL; ALTER TABLE "two_Partitions" ADD CONSTRAINT two_Partitions_device_id_2_fkey FOREIGN KEY (device_id_2) REFERENCES devices(id); ================================================ FILE: test/sql/updates/setup.continuous_aggs.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Collect information about different features so that we can pick -- the right usage. Some of these are changed in the same version, but -- we keep them separate anyway so that we can do additional checking -- if necessary. -- disable background workers to prevent deadlocks between background processes -- on timescaledb 1.7.x CALL _timescaledb_testing.stop_workers(); -- disable chunkwise aggregation and hash aggregation, because it might lead to -- different order of chunk creation in the cagg table, based on the underlying -- aggregation plan. SET timescaledb.enable_chunkwise_aggregation TO OFF; SET enable_hashagg TO OFF; CREATE TYPE custom_type AS (high int, low int); CREATE TABLE conditions_before ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null, highlow custom_type null, bit_int smallint, good_life boolean ); SELECT table_name FROM create_hypertable( 'conditions_before', 'timec'); INSERT INTO conditions_before SELECT generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'POR', 55, 75, 40, 70, NULL, (1,2)::custom_type, 2, true; INSERT INTO conditions_before SELECT generate_series('2018-11-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'NYC', 35, 45, 50, 40, NULL, (3,4)::custom_type, 4, false; INSERT INTO conditions_before SELECT generate_series('2018-11-01 00:00'::timestamp, '2018-12-15 00:00'::timestamp, '1 day'), 'LA', 73, 55, NULL, 28, NULL, NULL, 8, true; -- rename_cols cagg view is also used for another test: if we can enable -- compression on a cagg after an upgrade -- This view has 3 cols which is fewer than the number of cols on the table -- we had a bug related to that and need to verify if compression can be -- enabled on such a view CREATE MATERIALIZED VIEW rename_cols WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1 week', timec) AS bucket, location, round(avg(humidity)) AS humidity FROM conditions_before GROUP BY bucket, location WITH NO DATA; CREATE MATERIALIZED VIEW IF NOT EXISTS mat_before WITH ( timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1week', timec) as bucket, location, round(min(allnull)) as min_allnull, round(max(temperature)) as max_temp, round(sum(temperature)+sum(humidity)) as agg_sum_expr, round(avg(humidity)) AS avg_humidity, round(stddev(humidity)) as stddev, bit_and(bit_int), bit_or(bit_int), bool_and(good_life), every(temperature > 0), bool_or(good_life), count(*) as count_rows, count(temperature) as count_temp, count(allnull) as count_zero, round(corr(temperature, humidity)) as corr, round(covar_pop(temperature, humidity)) as covar_pop, round(covar_samp(temperature, humidity)) as covar_samp, round(regr_avgx(temperature, humidity)) as regr_avgx, round(regr_avgy(temperature, humidity)) as regr_avgy, round(regr_count(temperature, humidity)) as regr_count, round(regr_intercept(temperature, humidity)) as regr_intercept, round(regr_r2(temperature, humidity)) as regr_r2, round(regr_slope(temperature, humidity)) as regr_slope, round(regr_sxx(temperature, humidity)) as regr_sxx, round(regr_sxy(temperature, humidity)) as regr_sxy, round(regr_syy(temperature, humidity)) as regr_syy, round(stddev(temperature)) as stddev_temp, round(stddev_pop(temperature)) as stddev_pop, round(stddev_samp(temperature)) as stddev_samp, round(variance(temperature)) as variance, round(var_pop(temperature)) as var_pop, round(var_samp(temperature)) as var_samp, last(temperature, timec) as last_temp, last(highlow, timec) as last_hl, first(highlow, timec) as first_hl, histogram(temperature, 0, 100, 5) FROM conditions_before GROUP BY bucket, location HAVING min(location) >= 'NYC' and avg(temperature) > 2 WITH NO DATA; ALTER MATERIALIZED VIEW rename_cols RENAME COLUMN bucket TO "time"; \if :WITH_SUPERUSER GRANT SELECT ON mat_before TO cagg_user WITH GRANT OPTION; \endif CALL refresh_continuous_aggregate('rename_cols',NULL,NULL); CALL refresh_continuous_aggregate('mat_before',NULL,NULL); -- we create separate schema for realtime agg since we dump all view definitions in public schema -- but realtime agg view definition is not stable across versions CREATE SCHEMA cagg; CREATE MATERIALIZED VIEW IF NOT EXISTS cagg.realtime_mat WITH ( timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec) as bucket, location, round(min(allnull)) as min_allnull, round(max(temperature)) as max_temp, round(sum(temperature)+sum(humidity)) as agg_sum_expr, round(avg(humidity)) AS avg_humidity, round(stddev(humidity)) as stddev_humidity, bit_and(bit_int), bit_or(bit_int), bool_and(good_life), every(temperature > 0), bool_or(good_life), count(*) as count_rows, count(temperature) as count_temp, count(allnull) as count_zero, round(corr(temperature, humidity)) as corr, round(covar_pop(temperature, humidity)) as covar_pop, round(covar_samp(temperature, humidity)) as covar_samp, round(regr_avgx(temperature, humidity)) as regr_avgx, round(regr_avgy(temperature, humidity)) as regr_avgy, round(regr_count(temperature, humidity)) as regr_count, round(regr_intercept(temperature, humidity)) as regr_intercept, round(regr_r2(temperature, humidity)) as regr_r2, round(regr_slope(temperature, humidity)) as regr_slope, round(regr_sxx(temperature, humidity)) as regr_sxx, round(regr_sxy(temperature, humidity)) as regr_sxy, round(regr_syy(temperature, humidity)) as regr_syy, round(stddev(temperature)) as stddev_temp, round(stddev_pop(temperature)) as stddev_pop, round(stddev_samp(temperature)) as stddev_samp, round(variance(temperature)) as variance, round(var_pop(temperature)) as var_pop, round(var_samp(temperature)) as var_samp, last(temperature, timec) as last_temp, last(highlow, timec) as last_hl, first(highlow, timec) as first_hl, histogram(temperature, 0, 100, 5) FROM conditions_before GROUP BY bucket, location HAVING min(location) >= 'NYC' and avg(temperature) > 2 WITH NO DATA; \if :WITH_SUPERUSER GRANT SELECT ON cagg.realtime_mat TO cagg_user; \endif CALL refresh_continuous_aggregate('cagg.realtime_mat',NULL,NULL); -- test ignore_invalidation_older_than migration -- CREATE MATERIALIZED VIEW IF NOT EXISTS mat_ignoreinval WITH ( timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1 week', timec) as bucket, max(temperature) as maxtemp FROM conditions_before GROUP BY bucket WITH NO DATA; SELECT add_continuous_aggregate_policy('mat_ignoreinval', '30 days'::interval, '-30 days'::interval, '336 h'); CALL refresh_continuous_aggregate('mat_ignoreinval',NULL,NULL); -- test new data beyond the invalidation threshold is properly handled -- CREATE TABLE inval_test (time TIMESTAMPTZ NOT NULL, location TEXT, temperature DOUBLE PRECISION); SELECT create_hypertable('inval_test', 'time', chunk_time_interval => INTERVAL '1 week'); INSERT INTO inval_test SELECT generate_series('2018-12-01 00:00'::timestamp, '2018-12-20 00:00'::timestamp, '1 day'), 'POR', generate_series(40.5, 50.0, 0.5); INSERT INTO inval_test SELECT generate_series('2018-12-01 00:00'::timestamp, '2018-12-20 00:00'::timestamp, '1 day'), 'NYC', generate_series(31.0, 50.0, 1.0); CREATE MATERIALIZED VIEW mat_inval WITH ( timescaledb.continuous, timescaledb.materialized_only=true ) AS SELECT time_bucket('10 minute', time) as bucket, location, min(temperature) as min_temp, max(temperature) as max_temp, round(avg(temperature)) as avg_temp FROM inval_test GROUP BY bucket, location WITH NO DATA; SELECT add_continuous_aggregate_policy('mat_inval', NULL, '-20 days'::interval, '12 hours'); CALL refresh_continuous_aggregate('mat_inval',NULL,NULL); INSERT INTO inval_test SELECT generate_series('2118-12-01 00:00'::timestamp, '2118-12-20 00:00'::timestamp, '1 day'), 'POR', generate_series(135.25, 140.0, 0.25); INSERT INTO inval_test SELECT generate_series('2118-12-01 00:00'::timestamp, '2118-12-20 00:00'::timestamp, '1 day'), 'NYC', generate_series(131.0, 150.0, 1.0); -- Add an integer base table to ensure we handle it correctly CREATE TABLE int_time_test(timeval integer not null, col1 integer, col2 integer); select create_hypertable('int_time_test', 'timeval', chunk_time_interval=> 2); CREATE OR REPLACE FUNCTION integer_now_test() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timeval), 0) FROM public.int_time_test $$; SELECT set_integer_now_func('int_time_test', 'integer_now_test'); INSERT INTO int_time_test VALUES (10, - 4, 1), (11, - 3, 5), (12, - 3, 7), (13, - 3, 9), (14,-4, 11), (15, -4, 22), (16, -4, 23); CREATE MATERIALIZED VIEW mat_inttime WITH ( timescaledb.continuous, timescaledb.materialized_only=true ) AS SELECT time_bucket( 2, timeval), COUNT(col1) FROM int_time_test GROUP BY 1 WITH NO DATA; CREATE MATERIALIZED VIEW mat_inttime2 WITH ( timescaledb.continuous, timescaledb.materialized_only=true ) AS SELECT time_bucket( 2, timeval), COUNT(col1) FROM int_time_test GROUP BY 1 WITH NO DATA; SELECT add_continuous_aggregate_policy('mat_inttime', 6, 2, '12 hours'); SELECT add_continuous_aggregate_policy('mat_inttime2', NULL, 2, '12 hours'); CALL refresh_continuous_aggregate('mat_inttime',NULL,NULL); CALL refresh_continuous_aggregate('mat_inttime2',NULL,NULL); -- Test that retention policies that conflict with continuous aggs are disabled -- CREATE TABLE conflict_test (time TIMESTAMPTZ NOT NULL, location TEXT, temperature DOUBLE PRECISION); SELECT create_hypertable('conflict_test', 'time', chunk_time_interval => INTERVAL '1 week'); CREATE MATERIALIZED VIEW mat_conflict WITH ( timescaledb.continuous, timescaledb.materialized_only=true ) AS SELECT time_bucket('10 minute', time) as bucket, location, min(temperature) as min_temp, max(temperature) as max_temp, round(avg(temperature)) as avg_temp FROM conflict_test GROUP BY bucket, location WITH NO DATA; SELECT add_continuous_aggregate_policy('mat_conflict', '28 days', '1 day', '12 hours'); SELECT add_retention_policy('conflict_test', '14 days'::interval) AS retention_jobid \gset SELECT alter_job(:retention_jobid, scheduled=>false); \if :WITH_SUPERUSER GRANT SELECT, TRIGGER, UPDATE ON mat_conflict TO cagg_user WITH GRANT OPTION; \endif -- Test that calling drop chunks on the hypertable does not break the -- update process when chunks are marked as dropped rather than -- removed. This happens when a continuous aggregate is defined on the -- hypertable, so we create a hypertable and a continuous aggregate -- here and then drop chunks from the hypertable and make sure that -- the update from 1.7 to 2.0 works as expected. CREATE TABLE drop_test ( time timestamptz not null, location INT, temperature double PRECISION ); SELECT create_hypertable ('drop_test', 'time', chunk_time_interval => interval '1 week'); INSERT INTO drop_test SELECT time, (random() * 3 + 1)::int, random() * 100.0 FROM generate_series(now() - interval '28 days', now(), '1 hour') AS time; CREATE MATERIALIZED VIEW mat_drop WITH ( timescaledb.materialized_only = TRUE, timescaledb.continuous ) AS SELECT time_bucket ('10 minute',time) AS bucket, LOCATION, min(temperature) AS min_temp, max(temperature) AS max_temp, round(avg(temperature)) AS avg_temp FROM drop_test GROUP BY bucket, LOCATION; SELECT add_continuous_aggregate_policy('mat_drop', '7 days', '-30 days'::interval, '20 min'); CALL refresh_continuous_aggregate('mat_drop',NULL,NULL); SELECT drop_chunks('drop_test', NOW() - INTERVAL '7 days'); RESET timescaledb.enable_chunkwise_aggregation; RESET enable_hashagg; ================================================ FILE: test/sql/updates/setup.databases.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE DATABASE single; \c single CREATE EXTENSION IF NOT EXISTS timescaledb; ================================================ FILE: test/sql/updates/setup.drop_meta.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- DROP some chunks to test metadata cleanup \if :WITH_CHUNK DROP TABLE _timescaledb_internal._hyper_1_2_chunk; DROP TABLE _timescaledb_internal._hyper_1_3_chunk; \endif ================================================ FILE: test/sql/updates/setup.fix_sparse_index_migration.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Fix wrong migration by removing all sparse index configurations -- which only contain auto sparse indexing definitions on hypertable DO $$ DECLARE rec RECORD; num_config INT; BEGIN IF NOT EXISTS ( SELECT column_name FROM information_schema.columns WHERE table_schema = '_timescaledb_catalog' AND table_name = 'compression_settings' AND column_name = 'index') THEN RETURN; END IF; FOR rec IN SELECT relid, compress_relid, index FROM _timescaledb_catalog.compression_settings WHERE compress_relid IS NULL LOOP num_config:=0; SELECT count(*) INTO num_config FROM jsonb_array_elements(rec.index) AS idx WHERE idx::jsonb @> '{"storage":"config"}'::jsonb; IF num_config = 0 THEN UPDATE _timescaledb_catalog.compression_settings SET index = NULL WHERE relid = rec.relid; END IF; END LOOP; END $$; ================================================ FILE: test/sql/updates/setup.insert_bigint.v1.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. INSERT INTO public."two_Partitions"("timeCustom", device_id, series_0, series_1, series_2) VALUES (1257987600000000000, 'dev1', 1.5, 1, 1), (1257987600000000000, 'dev1', 1.5, 2, 2), (1257894000000000000, 'dev2', 1.5, 1, 3), (1257894002000000000, 'dev1', 2.5, 3, 4); INSERT INTO "two_Partitions"("timeCustom", device_id, series_0, series_1, series_2) VALUES (1257894000000000000, 'dev2', 1.5, 2, 6); ================================================ FILE: test/sql/updates/setup.insert_bigint.v2.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. INSERT INTO public."two_Partitions"("timeCustom", device_id, device_id_2, series_0, series_1, series_2) VALUES (1257987600000000000, 'dev1', 'dev2', 1.5, 2, 2), (1257894000000000000, 'dev2', 'dev2', 1.5, 1, 3), (1257987600000000000, 'dev3', 'dev2', 1.5, 1, 1), (1257894002000000000, 'dev1', 'dev2', 2.5, 3, 4); INSERT INTO "two_Partitions"("timeCustom", device_id, device_id_2, series_0, series_1, series_2) VALUES (1257894100000000000, 'dev2', 'dev2', 1.5, 2, 6); ================================================ FILE: test/sql/updates/setup.insert_timestamp.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. INSERT INTO hyper_timestamp VALUES ('2017-01-20T09:00:01', 'dev1', 1), ('2017-01-20T08:00:01', 'dev2', 2), ('2016-01-20T09:00:01', 'dev1', 3); ================================================ FILE: test/sql/updates/setup.pg_upgrade.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE SCHEMA cagg_join; CREATE TABLE cagg_join.sensor( id SERIAL PRIMARY KEY, name TEXT, enabled BOOLEAN ); CREATE TABLE cagg_join.measurement( sensor_id INTEGER REFERENCES cagg_join.sensor(id), observed TIMESTAMPTZ, value FLOAT ); SELECT create_hypertable('cagg_join.measurement', 'observed'); CREATE MATERIALIZED VIEW cagg_join.measurement_daily WITH (timescaledb.continuous) AS -- Column s.name is functionally dependent on s.id (primary key) SELECT s.id, s.name, time_bucket(interval '1 day', observed) as bucket, avg(value), min(value), max(value) FROM cagg_join.sensor s JOIN cagg_join.measurement m on (s.id = m.sensor_id) GROUP BY s.id, bucket; ================================================ FILE: test/sql/updates/setup.policies.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE policy_test_timestamptz(time timestamptz not null, device_id int, value float); SELECT table_name FROM create_hypertable('policy_test_timestamptz','time'); ALTER TABLE policy_test_timestamptz SET (timescaledb.compress, timescaledb.compress_orderby = '"time" desc'); INSERT INTO policy_test_timestamptz(time, device_id, value) VALUES ('3020-01-01 00:00:00', 1, 1.0); SELECT compress_chunk(show_chunks('policy_test_timestamptz')); DO LANGUAGE PLPGSQL $$ DECLARE ts_major INT; ts_minor INT; BEGIN WITH timescale_version AS ( SELECT string_to_array(extversion,'.') AS v FROM pg_extension WHERE extname = 'timescaledb' ) SELECT v[1], v[2] INTO ts_major, ts_minor FROM timescale_version; PERFORM add_reorder_policy('policy_test_timestamptz','policy_test_timestamptz_time_idx'); PERFORM add_retention_policy('policy_test_timestamptz','60d'::interval); -- some policy API functions got renamed for 2.0 so we need to make -- sure to use the right name for the version. The schedule_interval -- parameter of add_compression_policy was introduced in 2.8.0 IF ts_major = 2 AND ts_minor < 8 THEN PERFORM add_compression_policy('policy_test_timestamptz','10d'::interval); ELSE PERFORM add_compression_policy('policy_test_timestamptz','10d'::interval, schedule_interval => '3 days 12:00:00'::interval); END IF; END $$; \if :WITH_ROLES -- For PostgreSQL 15 and later GRANT ALL ON SCHEMA PUBLIC TO "dotted.name"; GRANT ALL ON SCHEMA PUBLIC TO "Kim Possible"; SET ROLE "dotted.name"; CREATE TABLE policy_test_user_1(time timestamptz not null, device_id int, value float); SELECT table_name FROM create_hypertable('policy_test_user_1','time'); SELECT add_retention_policy('policy_test_user_1', '14 days'::interval); SET ROLE "Kim Possible"; CREATE TABLE policy_test_user_2(time timestamptz not null, device_id int, value float); SELECT table_name FROM create_hypertable('policy_test_user_2','time'); SELECT add_retention_policy('policy_test_user_2', '14 days'::interval); RESET ROLE; \endif ================================================ FILE: test/sql/updates/setup.post-downgrade.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- When running a downgrade tests, the extension is first updated to -- the later version and then downgraded to the previous version. This -- means that in some cases, changes done by the update is not -- reversed by the downgrade. If these changes are harmless, we can -- apply changes to the clean rerun to incorporate these changes -- directly and prevent a diff between the clean-rerun version and the -- upgrade-downgrade version of the database. SELECT split_part(extversion, '.', 1)::int * 100000 + split_part(extversion, '.', 2)::int * 100 AS extversion_num FROM pg_extension WHERE extname = 'timescaledb' \gset SELECT :extversion_num >= 200000 AS has_create_mat_view \gset -- Rebuild the user views based on the renamed views \if :has_create_mat_view ALTER MATERIALIZED VIEW rename_cols SET (timescaledb.materialized_only = FALSE); \else ALTER VIEW rename_cols SET (timescaledb.materialized_only = FALSE); \endif ================================================ FILE: test/sql/updates/setup.repair.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test file to check that the repair script works. It will create a -- bunch of tables and "break" them by removing dimension slices from -- the dimension slice table. The repair script should then repair all -- of them and there should be no dimension slices missing. CREATE USER wizard; CREATE USER "Random L User"; CREATE TABLE repair_test_int(time integer not null, temp float8, tag integer, color integer); CREATE TABLE repair_test_timestamptz(time timestamptz not null, temp float8, tag integer, color integer); CREATE TABLE repair_test_extra(time timestamptz not null, temp float8, tag integer, color integer); CREATE TABLE repair_test_timestamp(time timestamp not null, temp float8, tag integer, color integer); CREATE TABLE repair_test_date(time date not null, temp float8, tag integer, color integer); -- We only break the dimension slice table if there is repair that is -- going to be done, but we create the tables regardless so that we -- can compare the databases. SELECT create_hypertable('repair_test_int', 'time', 'tag', 2, chunk_time_interval => '3'::bigint); SELECT create_hypertable('repair_test_timestamptz', 'time', 'tag', 2, chunk_time_interval => '1 day'::interval); SELECT create_hypertable('repair_test_extra', 'time', 'tag', 2, chunk_time_interval => '1 day'::interval); SELECT create_hypertable('repair_test_timestamp', 'time', 'tag', 2, chunk_time_interval => '1 day'::interval); SELECT create_hypertable('repair_test_date', 'time', 'tag', 2, chunk_time_interval => '1 day'::interval); -- These rows will create four constraints for each table. INSERT INTO repair_test_int VALUES (4, 24.3, 1, 1), (4, 24.3, 2, 1), (10, 24.3, 2, 1); INSERT INTO repair_test_timestamptz VALUES ('2020-01-01 10:11:12', 24.3, 1, 1), ('2020-01-01 10:11:13', 24.3, 2, 1), ('2020-01-02 10:11:14', 24.3, 2, 1); INSERT INTO repair_test_extra VALUES ('2020-01-01 10:11:12', 24.3, 1, 1), ('2020-01-01 10:11:13', 24.3, 2, 1), ('2020-01-02 10:11:14', 24.3, 2, 1); INSERT INTO repair_test_timestamp VALUES ('2020-01-01 10:11:12', 24.3, 1, 1), ('2020-01-01 10:11:13', 24.3, 2, 1), ('2020-01-02 10:11:14', 24.3, 2, 1); INSERT INTO repair_test_date VALUES ('2020-01-01 10:11:12', 24.3, 1, 1), ('2020-01-01 10:11:13', 24.3, 2, 1), ('2020-01-02 10:11:14', 24.3, 2, 1); -- We always drop the constraint and restore it in the -- post.repair.sql. -- -- This way if there are constraint violations remaining that wasn't -- repaired properly, we will notice them when restoring the -- constraint. ALTER TABLE _timescaledb_catalog.chunk_constraint DROP CONSTRAINT chunk_constraint_dimension_slice_id_fkey; -- Grant privileges to some tables above. All should be repaired. GRANT ALL ON repair_test_int TO wizard; GRANT ALL ON repair_test_extra TO wizard; GRANT SELECT, INSERT ON repair_test_int TO "Random L User"; GRANT INSERT ON repair_test_extra TO "Random L User"; -- Break the relacl of the table by deleting users directly from -- pg_authid table. DELETE FROM pg_authid WHERE rolname IN ('wizard', 'Random L User'); ================================================ FILE: test/sql/updates/setup.roles.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE ROLE cagg_user; CREATE USER tsdbadmin; -- These are used to test job creation and updating job owners. CREATE USER "dotted.name"; --non-identifier character in name CREATE USER "Kim Possible"; --case-sensitive names ================================================ FILE: test/sql/updates/setup.sparse_index.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test a table with auto sparse indexes CREATE TABLE bloom ( x INT, v1 TEXT, u UUID, ts TIMESTAMP ); SELECT create_hypertable('bloom', 'x'); INSERT INTO bloom SELECT x, md5(x::text), CASE WHEN x = 7134 THEN '90ec9e8e-4501-4232-9d03-6d7cf6132815' ELSE '6c1d0998-05f3-452c-abd3-45afe72bbcab'::uuid END, '2021-01-01'::timestamp + (INTERVAL '1 hour') * x FROM generate_series(1, 10000) x; CREATE INDEX ON bloom USING brin(v1 text_bloom_ops); CREATE INDEX ON bloom USING brin(u uuid_bloom_ops); CREATE INDEX ON bloom USING brin(ts timestamp_minmax_ops); ALTER TABLE bloom SET ( timescaledb.compress, timescaledb.compress_segmentby = 'v1', timescaledb.compress_orderby = 'x' ); SELECT COUNT(compress_chunk(x)) FROM show_chunks('bloom') x; VACUUM FULL ANALYZE bloom; SELECT * FROM _timescaledb_catalog.compression_settings; SELECT schema_name || '.' || table_name AS chunk FROM _timescaledb_catalog.chunk WHERE id = ( SELECT compressed_chunk_id FROM _timescaledb_catalog.chunk WHERE hypertable_id = ( SELECT id FROM _timescaledb_catalog.hypertable WHERE table_name = 'bloom' ) LIMIT 1 ) \gset \d+ :chunk ================================================ FILE: test/sql/updates/setup.timestamp.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test a hypertable using timestamps CREATE TABLE PUBLIC.hyper_timestamp ( time timestamp NOT NULL, device_id TEXT NOT NULL, value int NOT NULL ); DO $$ BEGIN IF (EXISTS (SELECT FROM pg_proc WHERE proname = 'interval_to_usec' AND pronamespace='_timescaledb_internal'::regnamespace)) THEN PERFORM create_hypertable('hyper_timestamp'::regclass, 'time'::name, 'device_id'::name, number_partitions => 2, chunk_time_interval=> _timescaledb_internal.interval_to_usec('1 minute')); ELSE PERFORM create_hypertable('hyper_timestamp'::regclass, 'time'::name, 'device_id'::name, number_partitions => 2, chunk_time_interval=> _timescaledb_functions.interval_to_usec('1 minute')); END IF; END; $$; --some old versions use more slice_ids than newer ones. Make this uniform CALL _timescaledb_testing.restart_dimension_slice_id(); ================================================ FILE: test/sql/updates/setup.v10.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \ir setup.v9.sql \ir setup.sparse_index.sql \ir setup.fix_sparse_index_migration.sql ================================================ FILE: test/sql/updates/setup.v7.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \ir setup.catalog.sql \ir setup.bigint.sql \ir setup.constraints.sql \ir setup.insert_bigint.v2.sql \ir setup.timestamp.sql ALTER TABLE PUBLIC.hyper_timestamp ADD CONSTRAINT exclude_const EXCLUDE USING btree ( "time" WITH =, device_id WITH = ) WHERE (value > 0); \ir setup.insert_timestamp.sql \ir setup.drop_meta.sql \ir setup.continuous_aggs.sql \ir setup.compression.sql \ir setup.policies.sql ================================================ FILE: test/sql/updates/setup.v8.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \ir setup.v7.sql ================================================ FILE: test/sql/updates/setup.v9.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \ir setup.v8.sql \ir setup.chunk_skipping.sql ================================================ FILE: test/sql/upsert.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE upsert_test(time timestamp PRIMARY KEY, temp float, color text); SELECT create_hypertable('upsert_test', 'time'); INSERT INTO upsert_test VALUES ('2017-01-20T09:00:01', 22.5, 'yellow') RETURNING *; INSERT INTO upsert_test VALUES ('2017-01-20T09:00:01', 23.8, 'yellow') ON CONFLICT (time) DO UPDATE SET temp = 23.8 RETURNING *; INSERT INTO upsert_test VALUES ('2017-01-20T09:00:01', 78.4, 'yellow') ON CONFLICT DO NOTHING; SELECT * FROM upsert_test; -- Test 'Tuples Inserted' and 'Conflicting Tuples' values in EXPLAIN ANALYZE EXPLAIN (VERBOSE, ANALYZE, BUFFERS FALSE, COSTS FALSE, TIMING FALSE, SUMMARY FALSE) INSERT INTO upsert_test VALUES ('2017-01-20T09:00:01', 28.5, 'blue'), ('2017-01-20T09:00:01', 21.9, 'red'), ('2017-01-20T10:00:01', 2.4, 'pink') ON CONFLICT DO NOTHING; -- Test ON CONFLICT ON CONSTRAINT INSERT INTO upsert_test VALUES ('2017-01-20T09:00:01', 12.3, 'yellow') ON CONFLICT ON CONSTRAINT upsert_test_pkey DO UPDATE SET temp = 12.3 RETURNING time, temp, color; -- Test that update generates error on conflicts \set ON_ERROR_STOP 0 INSERT INTO upsert_test VALUES ('2017-01-21T09:00:01', 22.5, 'yellow') RETURNING *; UPDATE upsert_test SET time = '2017-01-20T09:00:01'; \set ON_ERROR_STOP 1 -- Test with UNIQUE index on multiple columns instead of PRIMARY KEY constraint CREATE TABLE upsert_test_unique(time timestamp, temp float, color text); SELECT create_hypertable('upsert_test_unique', 'time'); CREATE UNIQUE INDEX time_color_idx ON upsert_test_unique (time, color); INSERT INTO upsert_test_unique VALUES ('2017-01-20T09:00:01', 22.5, 'yellow') RETURNING *; INSERT INTO upsert_test_unique VALUES ('2017-01-20T09:00:01', 21.2, 'brown'); SELECT * FROM upsert_test_unique ORDER BY time, color DESC; INSERT INTO upsert_test_unique VALUES ('2017-01-20T09:00:01', 31.8, 'yellow') ON CONFLICT (time, color) DO UPDATE SET temp = 31.8; INSERT INTO upsert_test_unique VALUES ('2017-01-20T09:00:01', 54.3, 'yellow') ON CONFLICT DO NOTHING; SELECT * FROM upsert_test_unique ORDER BY time, color DESC; -- Test with multiple UNIQUE indexes CREATE TABLE upsert_test_multi_unique(time timestamp, temp float, color text); SELECT create_hypertable('upsert_test_multi_unique', 'time'); ALTER TABLE upsert_test_multi_unique ADD CONSTRAINT multi_time_temp UNIQUE (time, temp); CREATE UNIQUE INDEX multi_time_color_idx ON upsert_test_multi_unique (time, color); INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 25.9, 'yellow'); INSERT INTO upsert_test_multi_unique VALUES ('2017-01-21T09:00:01', 25.9, 'yellow'); INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 23.5, 'brown'); INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 25.9, 'purple') ON CONFLICT DO NOTHING; SELECT * FROM upsert_test_multi_unique ORDER BY time, color DESC; INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 25.9, 'blue') ON CONFLICT (time, temp) DO UPDATE SET color = 'blue'; INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 23.5, 'orange') ON CONFLICT ON CONSTRAINT multi_time_temp DO UPDATE SET color = excluded.color; SELECT * FROM upsert_test_multi_unique ORDER BY time, color DESC; INSERT INTO upsert_test_multi_unique VALUES ('2017-01-21T09:00:01', 45.7, 'yellow') ON CONFLICT (time, color) DO UPDATE SET temp = 45.7; SELECT * FROM upsert_test_multi_unique ORDER BY time, color DESC; \set ON_ERROR_STOP 0 -- Here the constraint in the ON CONFLICT clause is not the one that is -- actually violated by the INSERT, so it should still fail. INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 23.5, 'purple') ON CONFLICT (time, color) DO UPDATE set temp = 23.5; INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 22.5, 'orange') ON CONFLICT ON CONSTRAINT multi_time_temp DO UPDATE set color = 'orange'; \set ON_ERROR_STOP 1 CREATE TABLE upsert_test_space(time timestamp, device_id_1 char(20), to_drop int, temp float, color text); --drop two columns; create one. ALTER TABLE upsert_test_space DROP to_drop; ALTER TABLE upsert_test_space DROP device_id_1, ADD device_id char(20); ALTER TABLE upsert_test_space ADD CONSTRAINT time_space_constraint UNIQUE (time, device_id); SELECT create_hypertable('upsert_test_space', 'time', 'device_id', 2, partitioning_func=>'_timescaledb_functions.get_partition_for_key'::regproc); INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev1', 25.9, 'yellow') RETURNING *; INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev2', 25.9, 'yellow'); INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev1', 23.5, 'green') ON CONFLICT (time, device_id) DO UPDATE SET color = excluded.color; INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev1', 23.5, 'orange') ON CONFLICT ON CONSTRAINT time_space_constraint DO UPDATE SET color = excluded.color; INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev2', 23.5, 'orange3') ON CONFLICT (time, device_id) DO UPDATE SET color = excluded.color||' (originally '|| upsert_test_space.color ||')' RETURNING *; INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev3', 23.5, 'orange3.1') ON CONFLICT (time, device_id) DO UPDATE SET color = excluded.color||' (originally '|| upsert_test_space.color ||')' RETURNING *; INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev2', 23.5, 'orange4') ON CONFLICT (time, device_id) DO NOTHING RETURNING *; INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev4', 23.5, 'orange5') ON CONFLICT (time, device_id) DO NOTHING RETURNING *; INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev5', 23.5, 'orange5') ON CONFLICT (time, device_id) DO NOTHING RETURNING *; INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev5', 23.5, 'orange6') ON CONFLICT ON CONSTRAINT time_space_constraint DO NOTHING RETURNING *; --restore a column with the same name as a previously deleted one; ALTER TABLE upsert_test_space ADD device_id_1 char(20); INSERT INTO upsert_test_space (time, device_id, temp, color, device_id_1) VALUES ('2017-01-20T09:00:01', 'dev4', 23.5, 'orange5.1', 'dev-id-1') ON CONFLICT (time, device_id) DO UPDATE SET color = excluded.color||' (originally '|| upsert_test_space.color ||')' RETURNING *; INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev5', 23.5, 'orange6') ON CONFLICT (time, device_id) DO UPDATE SET color = excluded.color WHERE upsert_test_space.temp < 20 RETURNING *; INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev5', 23.5, 'orange7') ON CONFLICT (time, device_id) DO UPDATE SET color = excluded.color WHERE excluded.temp < 20 RETURNING *; INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev5', 3.5, 'orange7') ON CONFLICT (time, device_id) DO UPDATE SET color = excluded.color, temp=excluded.temp WHERE excluded.temp < 20 RETURNING *; INSERT INTO upsert_test_space (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev5', 43.5, 'orange8') ON CONFLICT (time, device_id) DO UPDATE SET color = excluded.color WHERE upsert_test_space.temp < 20 RETURNING *; INSERT INTO upsert_test_space (time, device_id, temp, color, device_id_1) VALUES ('2017-01-20T09:00:01', 'dev5', 43.5, 'orange8', 'device-id-1-new') ON CONFLICT (time, device_id) DO UPDATE SET device_id_1 = excluded.device_id_1 RETURNING *; INSERT INTO upsert_test_space (time, device_id, temp, color, device_id_1) VALUES ('2017-01-20T09:00:01', 'dev5', 43.5, 'orange8', 'device-id-1-new') ON CONFLICT (time, device_id) DO UPDATE SET device_id_1 = 'device-id-1-new-2', color = 'orange9' RETURNING *; SELECT * FROM upsert_test_space; ALTER TABLE upsert_test_space DROP device_id_1, ADD device_id_2 char(20); INSERT INTO upsert_test_space (time, device_id, temp, color, device_id_2) VALUES ('2017-01-20T09:00:01', 'dev5', 43.5, 'orange8', 'device-id-2') ON CONFLICT (time, device_id) DO UPDATE SET device_id_2 = 'device-id-2-new', color = 'orange10' RETURNING *; --test inserting to to a chunk already in the chunk dispatch cache again. INSERT INTO upsert_test_space as current (time, device_id, temp, color, device_id_2) VALUES ('2017-01-20T09:00:01', 'dev5', 43.5, 'orange8', 'device-id-2'), ('2018-01-20T09:00:01', 'dev5', 43.5, 'orange8', 'device-id-2'), ('2017-01-20T09:00:01', 'dev3', 43.5, 'orange7', 'device-id-2'), ('2018-01-21T09:00:01', 'dev5', 43.5, 'orange9', 'device-id-2') ON CONFLICT (time, device_id) DO UPDATE SET device_id_2 = coalesce(excluded.device_id_2,current.device_id_2), color = coalesce(excluded.color,current.color) RETURNING *; WITH CTE AS ( INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 25.9, 'purple') ON CONFLICT DO NOTHING RETURNING * ) SELECT 1; WITH CTE AS ( INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 25.9, 'purple'), ('2017-01-20T09:00:01', 29.9, 'purple1') ON CONFLICT DO NOTHING RETURNING * ) SELECT * FROM CTE; WITH CTE AS ( INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 25.9, 'blue') ON CONFLICT (time, temp) DO UPDATE SET color = 'blue' RETURNING * ) SELECT * FROM CTE; --test error conditions when an index is dropped on a chunk DROP INDEX _timescaledb_internal._hyper_3_3_chunk_multi_time_color_idx; --everything is ok if not used as an arbiter index INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 25.9, 'purple') ON CONFLICT DO NOTHING RETURNING *; --errors out if used as an arbiter index \set ON_ERROR_STOP 0 INSERT INTO upsert_test_multi_unique VALUES ('2017-01-20T09:00:01', 25.9, 'purple') ON CONFLICT (time, color) DO NOTHING RETURNING *; \set ON_ERROR_STOP 1 --create table with one chunk that has a tup_conv_map and one that does not --to ensure this, create a chunk before altering the table this chunk will not have a tup_conv_map CREATE TABLE upsert_test_diffchunk(time timestamp, device_id char(20), to_drop int, temp float, color text); SELECT create_hypertable('upsert_test_diffchunk', 'time', chunk_time_interval=> interval '1 month'); CREATE UNIQUE INDEX time_device_idx ON upsert_test_diffchunk (time, device_id); --this is the chunk with no tup_conv_map INSERT INTO upsert_test_diffchunk (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev1', 25.9, 'yellow') RETURNING *; INSERT INTO upsert_test_diffchunk (time, device_id, temp, color) VALUES ('2017-01-20T09:00:01', 'dev2', 25.9, 'yellow') RETURNING *; --alter the table ALTER TABLE upsert_test_diffchunk DROP to_drop; ALTER TABLE upsert_test_diffchunk ADD device_id_2 char(20); --new chunk that does have a tup conv map INSERT INTO upsert_test_diffchunk (time, device_id, temp, color) VALUES ('2019-01-20T09:00:01', 'dev1', 23.5, 'orange') ; INSERT INTO upsert_test_diffchunk (time, device_id, temp, color) VALUES ('2019-01-20T09:00:01', 'dev2', 23.5, 'orange') ; select * from upsert_test_diffchunk order by time, device_id; --make sure current works INSERT INTO upsert_test_diffchunk as current (time, device_id, temp, color, device_id_2) VALUES ('2019-01-20T09:00:01', 'dev1', 43.5, 'orange2', 'device-id-2'), ('2017-01-20T09:00:01', 'dev1', 43.5, 'yellow2', 'device-id-2'), ('2019-01-20T09:00:01', 'dev2', 43.5, 'orange2', 'device-id-2') ON CONFLICT (time, device_id) DO UPDATE SET device_id_2 = coalesce(excluded.device_id_2,current.device_id_2), temp = coalesce(excluded.temp,current.temp) , color = coalesce(excluded.color,current.color); select * from upsert_test_diffchunk order by time, device_id; --arbiter index tests CREATE TABLE upsert_test_arbiter(time timestamp, to_drop int); SELECT create_hypertable('upsert_test_arbiter', 'time', chunk_time_interval=> interval '1 month'); --this is the chunk with no tup_conv_map INSERT INTO upsert_test_arbiter (time, to_drop) VALUES ('2017-01-20T09:00:01', 1) RETURNING *; INSERT INTO upsert_test_arbiter (time, to_drop) VALUES ('2017-01-21T09:00:01', 2) RETURNING *; INSERT INTO upsert_test_arbiter (time, to_drop) VALUES ('2017-03-20T09:00:01', 3) RETURNING *; --alter the table ALTER TABLE upsert_test_arbiter DROP to_drop; ALTER TABLE upsert_test_arbiter ADD device_id char(20) DEFAULT 'dev1'; CREATE UNIQUE INDEX arbiter_time_device_idx ON upsert_test_arbiter (time, device_id); INSERT INTO upsert_test_arbiter as current (time, device_id) VALUES ('2018-01-21T09:00:01', 'dev1'), ('2017-01-20T09:00:01', 'dev1'), ('2017-01-21T09:00:01', 'dev2'), ('2018-01-21T09:00:01', 'dev2') ON CONFLICT (time, device_id) DO UPDATE SET device_id = coalesce(excluded.device_id,current.device_id) RETURNING *; with cte as ( INSERT INTO upsert_test_arbiter (time, device_id) VALUES ('2017-01-21T09:00:01', 'dev2'), ('2018-01-21T09:00:01', 'dev2') ON CONFLICT (time, device_id) DO UPDATE SET device_id = 'dev3' RETURNING *) select * from cte; -- test ON CONFLICT with prepared statements CREATE TABLE prepared_test(time timestamptz PRIMARY KEY, value float CHECK(value > 0)); SELECT create_hypertable('prepared_test','time'); CREATE TABLE source_data(time timestamptz PRIMARY KEY, value float); INSERT INTO source_data VALUES('2000-01-01',0.5), ('2001-01-01',0.5); -- at some point PostgreSQL will turn the plan into a generic plan -- so we execute the prepared statement 10 times -- check that an error in the prepared statement does not lead to the plan becoming unusable PREPARE prep_insert_select AS INSERT INTO prepared_test select * from source_data ON CONFLICT (time) DO UPDATE SET value = EXCLUDED.value; EXECUTE prep_insert_select; EXECUTE prep_insert_select; EXECUTE prep_insert_select; EXECUTE prep_insert_select; EXECUTE prep_insert_select; EXECUTE prep_insert_select; EXECUTE prep_insert_select; EXECUTE prep_insert_select; EXECUTE prep_insert_select; EXECUTE prep_insert_select; --this insert will create an invalid tuple in source_data --so that future calls to prep_insert_select will fail INSERT INTO source_data VALUES('2000-01-02',-0.5); \set ON_ERROR_STOP 0 EXECUTE prep_insert_select; EXECUTE prep_insert_select; \set ON_ERROR_STOP 1 DELETE FROM source_data WHERE value <= 0; EXECUTE prep_insert_select; PREPARE prep_insert AS INSERT INTO prepared_test VALUES('2000-01-01',0.5) ON CONFLICT (time) DO UPDATE SET value = EXCLUDED.value; -- at some point PostgreSQL will turn the plan into a generic plan -- so we execute the prepared statement 10 times EXECUTE prep_insert; EXECUTE prep_insert; EXECUTE prep_insert; EXECUTE prep_insert; EXECUTE prep_insert; EXECUTE prep_insert; EXECUTE prep_insert; EXECUTE prep_insert; EXECUTE prep_insert; EXECUTE prep_insert; SELECT * FROM prepared_test; DELETE FROM prepared_test; -- test ON CONFLICT with functions CREATE OR REPLACE FUNCTION test_upsert(t timestamptz, v float) RETURNS VOID AS $sql$ BEGIN INSERT INTO prepared_test VALUES(t,v) ON CONFLICT (time) DO UPDATE SET value = EXCLUDED.value; END; $sql$ LANGUAGE PLPGSQL; -- at some point PostgreSQL will turn the plan into a generic plan -- so we execute the function 10 times SELECT counter,test_upsert('2000-01-01',0.5) FROM generate_series(1,10) AS g(counter); SELECT * FROM prepared_test; DELETE FROM prepared_test; -- at some point PostgreSQL will turn the plan into a generic plan -- so we execute the function 10 times SELECT counter,test_upsert('2000-01-01',0.5) FROM generate_series(1,10) AS g(counter); SELECT * FROM prepared_test; DELETE FROM prepared_test; -- run it again to ensure INSERT path is still working as well SELECT counter,test_upsert('2000-01-01',0.5) FROM generate_series(1,10) AS g(counter); SELECT * FROM prepared_test; DELETE FROM prepared_test; -- test ON CONFLICT with functions CREATE OR REPLACE FUNCTION test_upsert2(t timestamptz, v float) RETURNS VOID AS $sql$ BEGIN INSERT INTO prepared_test VALUES(t,v) ON CONFLICT (time) DO UPDATE SET value = prepared_test.value + 1.0; END; $sql$ LANGUAGE PLPGSQL; -- at some point PostgreSQL will turn the plan into a generic plan -- so we execute the function 10 times SELECT counter,test_upsert2('2000-01-01',1.0) FROM generate_series(1,10) AS g(counter); SELECT * FROM prepared_test; ================================================ FILE: test/sql/util.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. \set ECHO errors \set VERBOSITY default \c :TEST_DBNAME :ROLE_SUPERUSER \set TMP_USER :TEST_DBNAME _wizard DO $$ BEGIN ASSERT( _timescaledb_functions.get_partition_for_key(''::text) = 669664877 ); ASSERT( _timescaledb_functions.get_partition_for_key('dev1'::text) = 1129986420 ); ASSERT( _timescaledb_functions.get_partition_for_key('longlonglonglongpartitionkey'::text) = 1169179734); END$$; \pset null '[NULL]' CREATE USER :TMP_USER; SELECT * FROM ( VALUES (_timescaledb_functions.makeaclitem(:'TMP_USER', :'TMP_USER', 'insert', false)), (_timescaledb_functions.makeaclitem(:'TMP_USER', :'TMP_USER', 'insert,select', false)), (_timescaledb_functions.makeaclitem(:'TMP_USER', :'TMP_USER', 'insert', true)), (_timescaledb_functions.makeaclitem(:'TMP_USER', :'TMP_USER', 'insert,select', true)), (_timescaledb_functions.makeaclitem(NULL, :'TMP_USER', 'insert,select', true)), (_timescaledb_functions.makeaclitem(:'TMP_USER', NULL, 'insert,select', true)), (_timescaledb_functions.makeaclitem(:'TMP_USER', :'TMP_USER', NULL, true)), (_timescaledb_functions.makeaclitem(:'TMP_USER', :'TMP_USER', 'insert,select', NULL)), (_timescaledb_functions.makeaclitem(0, :'TMP_USER', 'insert,select', true)), (_timescaledb_functions.makeaclitem(:'TMP_USER', 0, 'insert,select', true)) ) AS t(item); DROP USER :TMP_USER; CREATE TABLE data (vid serial, rng text); INSERT INTO data(rng) VALUES ('["2025-04-25 11:10:00+02","2025-04-25 11:14:00+02"]'), ('["2025-04-25 11:10:00+02","2025-04-25 11:17:00+02"]'), ('["2025-04-25 11:10:00+02","2025-04-25 11:20:00+02")'); \set ECHO all SELECT _timescaledb_functions.align_to_bucket('5 minutes'::interval, rng::tstzrange) FROM data; \set ON_ERROR_STOP 0 SELECT _timescaledb_functions.align_to_bucket(null, null); SELECT _timescaledb_functions.align_to_bucket(null::interval, null::tstzrange); SELECT _timescaledb_functions.align_to_bucket( null::interval, '["2025-04-25 11:10:00+02","2025-04-25 11:14:00+02"]'::tstzrange ); \set ON_ERROR_STOP 1 SELECT typ, _timescaledb_functions.get_internal_time_min(typ), _timescaledb_functions.get_internal_time_max(typ) FROM (VALUES ('bigint'::regtype), ('int'::regtype), ('smallint'::regtype), ('timestamp'::regtype), ('timestamptz'::regtype), ('date'::regtype), (null::regtype) ) t(typ); \set ON_ERROR_STOP 0 SELECT _timescaledb_functions.get_internal_time_min(0); SELECT _timescaledb_functions.get_internal_time_max(0); \set ON_ERROR_STOP 1 WITH tstzranges AS ( SELECT vid, rng::tstzrange, lower(rng::tstzrange) AS lower_ts, upper(rng::tstzrange) AS upper_ts FROM data ), usecranges AS ( SELECT vid, _timescaledb_functions.to_unix_microseconds(lower_ts) AS lower_usec, _timescaledb_functions.to_unix_microseconds(upper_ts) AS upper_usec FROM tstzranges ) SELECT _timescaledb_functions.make_multirange_from_internal_time(rng, lower_usec, upper_usec), _timescaledb_functions.make_range_from_internal_time(rng, lower_ts, upper_ts) FROM tstzranges join usecranges using (vid); WITH tsranges AS ( SELECT vid, rng::tsrange, lower(rng::tsrange) AS lower_ts, upper(rng::tsrange) AS upper_ts FROM data ), usecranges AS ( SELECT vid, _timescaledb_functions.to_unix_microseconds(lower_ts) AS lower_usec, _timescaledb_functions.to_unix_microseconds(upper_ts) AS upper_usec FROM tsranges ) SELECT _timescaledb_functions.make_multirange_from_internal_time(rng, lower_usec, upper_usec), _timescaledb_functions.make_range_from_internal_time(rng, lower_ts, upper_ts) FROM tsranges join usecranges using (vid); ================================================ FILE: test/sql/utils/pg_dump_aux_dump.sh ================================================ DUMPFILE=${DUMPFILE:-$1} EXTRA_PGOPTIONS=${EXTRA_PGOPTIONS:-$2} # Override PGOPTIONS to remove verbose output PGOPTIONS="--client-min-messages=warning $EXTRA_PGOPTIONS" export PGOPTIONS ${PG_BINDIR}/pg_dump -h ${PGHOST} -U ${TEST_ROLE_SUPERUSER} -Fc ${TEST_DBNAME} > /dev/null 2>&1 -f ${DUMPFILE} ${PG_BINDIR}/dropdb -f -h ${PGHOST} -U ${TEST_ROLE_SUPERUSER} ${TEST_DBNAME} ${PG_BINDIR}/createdb -h ${PGHOST} -U ${TEST_ROLE_SUPERUSER} ${TEST_DBNAME} ================================================ FILE: test/sql/utils/pg_dump_aux_plain_dump.sh ================================================ DUMPFILE=${DUMPFILE:-$1} EXTRA_PGOPTIONS=${EXTRA_PGOPTIONS:-$2} DUMP_OPTIONS=${DUMP_OPTIONS:-$3} # Override PGOPTIONS to remove verbose output PGOPTIONS="--client-min-messages=warning $EXTRA_PGOPTIONS" export PGOPTIONS echo ${DUMP_OPTIONS} echo $DUMP_OPTIONS echo $(echo $DUMP_OPTIONS) ${PG_BINDIR}/pg_dump -h ${PGHOST} -U ${TEST_ROLE_SUPERUSER} ${DUMP_OPTIONS} -Fp ${TEST_DBNAME} -f ${DUMPFILE} # ${PG_BINDIR}/pg_dump -h ${PGHOST} -U ${TEST_ROLE_SUPERUSER} ${DUMP_OPTIONS} -Fp ${TEST_DBNAME} > /dev/null 2>&1 -f ${DUMPFILE} ${PG_BINDIR}/dropdb -f -h ${PGHOST} -U ${TEST_ROLE_SUPERUSER} ${TEST_DBNAME} ${PG_BINDIR}/createdb -h ${PGHOST} -U ${TEST_ROLE_SUPERUSER} ${TEST_DBNAME} ================================================ FILE: test/sql/utils/pg_dump_aux_restore.sh ================================================ DUMPFILE=${DUMPFILE:-$1} # Override PGOPTIONS to remove verbose output PGOPTIONS='--client-min-messages=warning' export PGOPTIONS # Redirect output to /dev/null to suppress NOTICE ${PG_BINDIR}/pg_restore -h ${PGHOST} -U ${TEST_ROLE_SUPERUSER} -d ${TEST_DBNAME} ${DUMPFILE} > /dev/null 2>&1 ================================================ FILE: test/sql/utils/pg_dump_unprivileged.sh ================================================ export PGOPTIONS PGOPTIONS='--client-min-messages=warning' ${PG_BINDIR}/pg_dump -h ${PGHOST} -U dump_unprivileged dump_unprivileged > /dev/null 2>&1 if [ $? -eq 0 ]; then echo "Database dumped successfully" fi ================================================ FILE: test/sql/utils/test_fatal_command.sh ================================================ DB=$1 COMMAND=$2 ${PG_BINDIR}/psql -X -h ${PGHOST} -U ${TEST_ROLE_SUPERUSER} -d ${DB} --command="${COMMAND}" 2>&1 | grep -v CONTEXT | grep -v "extension script file" ================================================ FILE: test/sql/utils/testsupport.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE SCHEMA IF NOT EXISTS test; GRANT USAGE ON SCHEMA test TO PUBLIC; -- Utility functions to show relation information in tests. These -- functions generate output which is the same across PostgreSQL -- versions. Their usage is preferred over psql's '\d <relation>', -- since that output typically changes across PostgreSQL versions. -- this function is duplicated in test/isolation/specs/multi_transaction_indexing.spec -- if it changes, that copy may need to change as well CREATE OR REPLACE FUNCTION test.show_columns(rel regclass) RETURNS TABLE("Column" name, "Type" text, "NotNull" boolean) LANGUAGE SQL STABLE AS $BODY$ SELECT a.attname, format_type(t.oid, t.typtypmod), a.attnotnull FROM pg_attribute a, pg_type t WHERE a.attrelid = rel AND a.atttypid = t.oid AND a.attnum >= 0 ORDER BY a.attnum; $BODY$; CREATE OR REPLACE FUNCTION test.show_columnsp(pattern text) RETURNS TABLE("Relation" regclass, "Kind" "char", "Column" name, "Column type" text, "NotNull" boolean) LANGUAGE PLPGSQL STABLE AS $BODY$ DECLARE schema_name name = split_part(pattern, '.', 1); table_name name = split_part(pattern, '.', 2); BEGIN IF schema_name = '' OR table_name = '' THEN schema_name := current_schema(); table_name := pattern; END IF; RETURN QUERY SELECT c.oid::regclass, c.relkind, a.attname, format_type(t.oid, t.typtypmod), a.attnotnull FROM pg_class c, pg_attribute a, pg_type t WHERE format('%I.%I', c.relnamespace::regnamespace::name, c.relname) LIKE format('%I.%s', schema_name, table_name) AND a.attrelid = c.oid AND a.atttypid = t.oid AND a.attnum >= 0 ORDER BY c.relname, a.attnum; END $BODY$; -- Extended output about columns analogous to \d+ CREATE OR REPLACE FUNCTION test.show_columns_ext(rel regclass) RETURNS TABLE( "Column" name, "Type" text, "Collation" name, "Nullable" text, "Default" text, "Storage" text, "Stats target" integer, "Description" text ) LANGUAGE SQL STABLE AS $BODY$ SELECT a.attname AS "Column", pg_catalog.format_type(a.atttypid, a.atttypmod) AS "Type", c.collname AS "Collation", CASE WHEN a.attnotnull THEN 'not null' ELSE '' END AS "Nullable", pg_catalog.pg_get_expr(ad.adbin, ad.adrelid) AS "Default", CASE a.attstorage WHEN 'p' THEN 'plain' WHEN 'm' THEN 'main' WHEN 'e' THEN 'external' WHEN 'x' THEN 'extended' ELSE NULL END AS "Storage", a.attstattarget AS "Stats target", d.description AS "Description" FROM pg_catalog.pg_attribute a JOIN pg_catalog.pg_type t ON t.oid = a.atttypid LEFT JOIN pg_catalog.pg_collation c ON a.attcollation = c.oid AND a.attcollation <> 0 LEFT JOIN pg_catalog.pg_attrdef ad ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum LEFT JOIN pg_catalog.pg_description d ON d.objoid = a.attrelid AND d.objsubid = a.attnum WHERE a.attrelid = rel AND a.attnum > 0 AND NOT a.attisdropped ORDER BY a.attnum; $BODY$; CREATE OR REPLACE FUNCTION test.show_indexes(rel regclass) RETURNS TABLE("Index" regclass, "Columns" name[], "Expr" text, "Unique" boolean, "Primary" boolean, "Exclusion" boolean, "Tablespace" name) LANGUAGE SQL STABLE AS $BODY$ SELECT c.oid::regclass, array(SELECT "Column" FROM test.show_columns(i.indexrelid)), pg_get_expr(i.indexprs, c.oid, true), i.indisunique, i.indisprimary, i.indisexclusion, (SELECT t.spcname FROM pg_tablespace t WHERE t.oid = c.reltablespace) FROM pg_class c, pg_index i WHERE c.oid = i.indexrelid AND i.indrelid = rel ORDER BY c.relname; $BODY$; CREATE OR REPLACE FUNCTION test.show_indexespred(rel regclass) RETURNS TABLE("Index" regclass, "Columns" name[], "Expr" text, "Pred" text, "Unique" boolean, "Primary" boolean, "Exclusion" boolean, "Tablespace" name) LANGUAGE SQL STABLE AS $BODY$ SELECT c.oid::regclass, array(SELECT "Column" FROM test.show_columns(i.indexrelid)), pg_get_expr(i.indexprs, i.indrelid, true), pg_get_expr(i.indpred, i.indrelid, true), i.indisunique, i.indisprimary, i.indisexclusion, (SELECT t.spcname FROM pg_tablespace t WHERE t.oid = c.reltablespace) FROM pg_class c, pg_index i WHERE c.oid = i.indexrelid AND i.indrelid = rel ORDER BY c.relname; $BODY$; -- this function is duplicated in test/isolation/specs/multi_transaction_indexing.spec -- if it changes, that copy may need to change as well CREATE OR REPLACE FUNCTION test.show_indexesp(pattern text) RETURNS TABLE("Table" regclass, "Index" regclass, "Columns" name[], "Expr" text, "Unique" boolean, "Primary" boolean, "Exclusion" boolean, "Tablespace" name) LANGUAGE PLPGSQL STABLE AS $BODY$ DECLARE schema_name name = split_part(pattern, '.', 1); table_name name = split_part(pattern, '.', 2); BEGIN IF schema_name = '' OR table_name = '' THEN schema_name := current_schema(); table_name := pattern; END IF; RETURN QUERY SELECT c.oid::regclass, i.indexrelid::regclass, array(SELECT "Column" FROM test.show_columns(i.indexrelid)), pg_get_expr(i.indexprs, c.oid, true), i.indisunique, i.indisprimary, i.indisexclusion, (SELECT t.spcname FROM pg_class cc, pg_tablespace t WHERE cc.oid = i.indexrelid AND t.oid = cc.reltablespace) FROM pg_class c, pg_index i WHERE format('%I.%I', c.relnamespace::regnamespace::name, c.relname) LIKE format('%I.%s', schema_name, table_name) AND c.oid = i.indrelid ORDER BY c.oid, i.indexrelid; END $BODY$; CREATE OR REPLACE FUNCTION test.show_constraints(rel regclass) RETURNS TABLE("Constraint" name, "Type" "char", "Columns" name[], "Index" regclass, "Expr" text, "Deferrable" bool, "Deferred" bool, "Validated" bool) LANGUAGE SQL STABLE AS $BODY$ SELECT c.conname, c.contype, array(SELECT attname FROM pg_attribute a, unnest(conkey) k WHERE a.attrelid = rel AND k = a.attnum), c.conindid::regclass, pg_get_expr(c.conbin, c.conrelid), c.condeferrable, c.condeferred, c.convalidated FROM pg_constraint c WHERE c.conrelid = rel -- to avoid showing not null constraints which are new to PG18 -- https://github.com/postgres/postgres/commit/14e87ffa AND c.conname NOT LIKE '%not_null%' ORDER BY c.conname; $BODY$; CREATE OR REPLACE FUNCTION test.show_constraintsp(pattern text) RETURNS TABLE("Table" regclass, "Constraint" name, "Type" "char", "Columns" name[], "Index" regclass, "Expr" text, "Deferrable" bool, "Deferred" bool, "Validated" bool) LANGUAGE PLPGSQL STABLE AS $BODY$ DECLARE schema_name name = split_part(pattern, '.', 1); table_name name = split_part(pattern, '.', 2); BEGIN IF schema_name = '' OR table_name = '' THEN schema_name := current_schema(); table_name := pattern; END IF; RETURN QUERY SELECT cl.oid::regclass, c.conname, c.contype, array(SELECT attname FROM pg_attribute a, unnest(conkey) k WHERE a.attrelid = cl.oid AND k = a.attnum), c.conindid::regclass, pg_get_expr(c.conbin, c.conrelid), c.condeferrable, c.condeferred, c.convalidated FROM pg_class cl, pg_constraint c WHERE format('%I.%I', cl.relnamespace::regnamespace::name, cl.relname) LIKE format('%I.%s', schema_name, table_name) -- to avoid showing not null constraints which are new to PG18 -- https://github.com/postgres/postgres/commit/14e87ffa AND c.conname NOT LIKE '%not_null%' AND c.conrelid = cl.oid ORDER BY cl.relname, c.conname; END $BODY$; CREATE OR REPLACE FUNCTION test.show_triggers(rel regclass, show_internal boolean = false) RETURNS TABLE("Trigger" name, "Type" smallint, "Function" regproc) LANGUAGE SQL STABLE AS $BODY$ SELECT t.tgname, t.tgtype, t.tgfoid::regproc FROM pg_trigger t WHERE t.tgrelid = rel AND t.tgisinternal = show_internal ORDER BY t.tgname; $BODY$; CREATE OR REPLACE FUNCTION test.show_triggersp(pattern text, show_internal boolean = false) RETURNS TABLE("Table" regclass, "Trigger" name, "Type" smallint, "Function" regproc) LANGUAGE PLPGSQL STABLE AS $BODY$ DECLARE schema_name name = split_part(pattern, '.', 1); table_name name = split_part(pattern, '.', 2); BEGIN IF schema_name = '' OR table_name = '' THEN schema_name := current_schema(); table_name := pattern; END IF; RETURN QUERY SELECT t.tgrelid::regclass, t.tgname, t.tgtype, t.tgfoid::regproc FROM pg_class cl, pg_trigger t WHERE format('%I.%I', cl.relnamespace::regnamespace::name, cl.relname) LIKE format('%I.%s', schema_name, table_name) AND t.tgrelid = cl.oid AND t.tgisinternal = show_internal ORDER BY t.tgrelid, t.tgname; END $BODY$; CREATE OR REPLACE FUNCTION test.show_subtables(rel regclass) RETURNS TABLE("Child" regclass, "Tablespace" name) LANGUAGE SQL STABLE AS $BODY$ SELECT objid::regclass, (SELECT t.spcname FROM pg_tablespace t WHERE t.oid = c.reltablespace) FROM pg_depend d, pg_class c WHERE d.refobjid = rel AND d.deptype = 'n' AND d.classid = 'pg_class'::regclass AND d.objid = c.oid ORDER BY d.refobjid, d.objid; $BODY$; CREATE OR REPLACE FUNCTION test.show_subtablesp(pattern text) RETURNS TABLE("Parent" regclass, "Child" regclass, "Tablespace" name) LANGUAGE PLPGSQL STABLE AS $BODY$ DECLARE schema_name name = split_part(pattern, '.', 1); table_name name = split_part(pattern, '.', 2); BEGIN IF schema_name = '' OR table_name = '' THEN schema_name := current_schema(); table_name := pattern; END IF; RETURN QUERY SELECT refobjid::regclass, objid::regclass, (SELECT t.spcname FROM pg_class cc, pg_tablespace t WHERE cc.oid = d.objid AND t.oid = cc.reltablespace) FROM pg_class c, pg_depend d WHERE format('%I.%I', c.relnamespace::regnamespace::name, c.relname) LIKE format('%I.%s', schema_name, table_name) AND d.refobjid = c.oid AND d.deptype = 'n' AND d.classid = 'pg_class'::regclass ORDER BY d.refobjid, d.objid; END $BODY$; CREATE OR REPLACE FUNCTION test.execute_sql(cmd TEXT) RETURNS TEXT LANGUAGE PLPGSQL AS $BODY$ BEGIN EXECUTE cmd; RETURN cmd; END $BODY$; -- Used to set a deterministic memory setting during tests CREATE OR REPLACE FUNCTION test.set_memory_cache_size(memory_amount text) RETURNS BIGINT AS :MODULE_PATHNAME, 'ts_set_memory_cache_size' LANGUAGE C VOLATILE STRICT; CREATE OR REPLACE FUNCTION test.make_tablespace_path(prefix TEXT, test_name TEXT) RETURNS TEXT LANGUAGE plpgsql AS $BODY$ DECLARE mkdirFlag TEXT := CASE WHEN sysname = 'Windows' THEN '' ELSE '-p ' END FROM _timescaledb_functions.get_os_info(); dirPath TEXT := format('%s%s', prefix, test_name); createDir TEXT := format('mkdir %s%s', mkdirFlag, dirPath); BEGIN EXECUTE format('COPY (SELECT 1) TO PROGRAM %s', quote_literal(createDir)); RETURN dirPath; END; $BODY$; -- Wait for job to execute with success or failure CREATE OR REPLACE FUNCTION test.wait_for_job_to_run(job_param_id INTEGER, expected_runs INTEGER, spins INTEGER=:TEST_SPINWAIT_ITERS) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE r RECORD; BEGIN FOR i in 1..spins LOOP SELECT total_successes, total_failures FROM _timescaledb_internal.bgw_job_stat WHERE job_id=job_param_id INTO r; IF (r.total_failures > 0) THEN RAISE INFO 'wait_for_job_to_run: job execution failed'; RETURN false; ELSEIF (r.total_successes = expected_runs) THEN RETURN true; ELSEIF (r.total_successes > expected_runs) THEN RAISE 'num_runs > expected'; ELSE PERFORM pg_sleep(0.1); END IF; END LOOP; RAISE INFO 'wait_for_job_to_run: timeout after % tries', spins; RETURN false; END $BODY$; -- Wait for job to run or fail CREATE OR REPLACE FUNCTION test.wait_for_job_to_run_or_fail(job_param_id INTEGER, spins INTEGER=:TEST_SPINWAIT_ITERS) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE r RECORD; BEGIN FOR i in 1..spins LOOP SELECT total_runs FROM _timescaledb_internal.bgw_job_stat WHERE job_id=job_param_id INTO r; IF (r.total_runs > 0) THEN RETURN true; ELSE PERFORM pg_sleep(0.1); END IF; END LOOP; RAISE INFO 'wait_for_job_to_run_or_fail: timeout after % tries', spins; RETURN false; END $BODY$; CREATE OR REPLACE VIEW test.extension AS SELECT e.extname AS "Name", e.extversion AS "Version", n.nspname AS "Schema", c.description AS "Description" FROM pg_extension e LEFT JOIN pg_namespace n ON n.oid = e.extnamespace LEFT JOIN pg_description c ON c.objoid = e.oid AND c.classoid = 'pg_extension'::regclass ORDER BY 1; GRANT SELECT ON test.extension TO PUBLIC; -- View to replace \dt commands in tests for consistent output across PostgreSQL versions CREATE OR REPLACE VIEW test.relation AS SELECT n.nspname AS schema, c.relname AS name, CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'f' THEN 'foreign table' WHEN 'p' THEN 'partitioned table' ELSE c.relkind::text END AS type, pg_catalog.pg_get_userbyid(c.relowner) AS owner FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind IN ('r','p','v','m','f','') AND n.nspname <> 'information_schema' AND n.nspname <> 'pg_catalog' AND n.nspname !~ '^pg_toast' ORDER BY 1, 2; GRANT SELECT ON test.relation TO PUBLIC; ================================================ FILE: test/sql/utils/testsupport_init.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. SELECT _timescaledb_functions.stop_background_workers(); -- Cleanup any system job stats that can lead to flaky test DELETE FROM _timescaledb_internal.bgw_job_stat_history WHERE job_id < 1000; DELETE FROM _timescaledb_internal.bgw_job_stat WHERE job_id < 1000; ================================================ FILE: test/sql/uuid.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- -- -- Test "time" partitioning on UUIDv7 -- -- CREATE TABLE uuid_events(id uuid primary key, device int, temp float); \set ON_ERROR_STOP 0 -- Test invalid interval type SELECT create_hypertable('uuid_events', 'id', chunk_time_interval => true); \set ON_ERROR_STOP 1 SELECT create_hypertable('uuid_events', 'id', chunk_time_interval => interval '1 day'); SELECT time_interval FROM timescaledb_information.dimensions WHERE hypertable_name = 'uuid_events'; -- -- Test that inserting boundary values generates the right constraints -- on chunks. -- -- First value with min time: 00000000-0000-7000-8000-000000000000 BEGIN; INSERT INTO uuid_events VALUES ('00000000-0000-7000-8000-000000000000', 1, 1.0); SELECT (test.show_constraints(ch)).* from show_chunks('uuid_events') ch; SELECT uuid_timestamp(id), device, temp FROM uuid_events ORDER BY id; -- Update v7 UUID to a v4 UUID that doesn't violate the chunk's range -- constraint. Currently we don't prevent this "loophole". UPDATE uuid_events SET id = '00000000-0001-4000-8000-000000000000' WHERE id = '00000000-0000-7000-8000-000000000000'; SELECT uuid_timestamp(id), device, temp FROM uuid_events ORDER BY id; -- Update v7 UUID to a v4 that violates the chunk constraint: \set ON_ERROR_STOP 0 UPDATE uuid_events SET id = 'ffff0000-0000-4000-8000-000000000000' WHERE id = '00000000-0001-4000-8000-000000000000'; \set ON_ERROR_STOP 1 ROLLBACK; -- Last value with min time: 00000000-0000-7fff-bfff-ffffffffffff BEGIN; INSERT INTO uuid_events VALUES ('00000000-0000-7fff-bfff-ffffffffffff', 1, 1.0); SELECT (test.show_constraints(ch)).* from show_chunks('uuid_events') ch; ROLLBACK; -- First value with max time: ffffffff-ffff-7000-8000-000000000000 BEGIN; INSERT INTO uuid_events VALUES ('ffffffff-ffff-7000-8000-000000000000', 1, 1.0); SELECT (test.show_constraints(ch)).* from show_chunks('uuid_events') ch; ROLLBACK; -- (Max time with min value) + 1 BEGIN; INSERT INTO uuid_events VALUES ('ffffffff-ffff-7000-8000-000000000001', 1, 1.0); SELECT (test.show_constraints(ch)).* from show_chunks('uuid_events') ch; ROLLBACK; -- Last value with max time: ffffffff-ffff-7fff-bfff-ffffffffffff BEGIN; INSERT INTO uuid_events VALUES ('ffffffff-ffff-7fff-bfff-ffffffffffff', 1, 1.0); SELECT (test.show_constraints(ch)).* from show_chunks('uuid_events') ch; ROLLBACK; -- -- It is possible to generate UUIDs like follows, but the random -- generator used doesn't respect setseed() so used constant UUIDs for -- determinism. -- -- (_timescaledb_functions.uuid_v7_from_timestamptz('2025-01-01 01:00 PST'), 1, 1.0), -- (_timescaledb_functions.uuid_v7_from_timestamptz('2025-01-01 02:00 PST'), 2, 2.0), -- (_timescaledb_functions.uuid_v7_from_timestamptz('2025-01-02 01:00 PST'), 3, 3.0), -- (_timescaledb_functions.uuid_v7_from_timestamptz('2025-01-02 02:00 PST'), 4, 4.0), -- (_timescaledb_functions.uuid_v7_from_timestamptz('2025-01-03 03:00 PST'), 5, 5.0), -- (_timescaledb_functions.uuid_v7_from_timestamptz('2025-01-03 10:00 PST'), 6, 6.0); -- INSERT INTO uuid_events VALUES ('0194214e-cd00-7000-a9a7-63f1416dab45', 2, 2.0), ('01942117-de80-7000-8121-f12b2b69dd96', 1, 1.0), ('0194263e-3a80-7000-8f40-82c987b1bc1f', 3, 3.0), ('01942675-2900-7000-8db1-a98694b18785', 4, 4.0), ('01942bd2-7380-7000-9bc4-5f97443907b8', 5, 5.0), ('01942d52-f900-7000-866e-07d6404d53c1', 6, 6.0); SELECT * FROM show_chunks('uuid_events'); SELECT (test.show_constraints(ch)).* from show_chunks('uuid_events') ch; SELECT id, device, temp FROM uuid_events; SELECT uuid_timestamp(id), device, temp FROM uuid_events; SELECT uuid_timestamp(id), device, temp FROM uuid_events ORDER BY id; SELECT _timescaledb_functions.to_timestamp(range_start) AS range_start, _timescaledb_functions.to_timestamp(range_end) AS range_end FROM _timescaledb_catalog.dimension_slice ds JOIN _timescaledb_catalog.dimension d ON (ds.dimension_id = d.id) JOIN _timescaledb_catalog.hypertable h ON (d.hypertable_id = h.id) WHERE h.table_name = 'uuid_events'; SELECT _timescaledb_functions.to_timestamp(range_start) AS chunk_range_start, _timescaledb_functions.to_timestamp(range_end) AS chunk_range_end FROM _timescaledb_catalog.dimension_slice ds JOIN _timescaledb_catalog.dimension d ON (ds.dimension_id = d.id) JOIN _timescaledb_catalog.hypertable h ON (d.hypertable_id = h.id) WHERE h.table_name = 'uuid_events' LIMIT 1 OFFSET 1 \gset -- Test that chunk exclusion on uuidv7 column works SELECT :'chunk_range_start', to_uuidv7_boundary(:'chunk_range_start'); -- Exclude all but one chunk EXPLAIN (verbose, buffers off, costs off, timing off) SELECT uuid_timestamp(id), device, temp FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_start'); SELECT uuid_timestamp(id), device, temp FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_start') ORDER BY id; -- Exclude only one chunk. Add ordering (DESC) EXPLAIN (verbose, buffers off, costs off, timing off) SELECT uuid_timestamp(id), device, temp FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_end') ORDER BY id DESC; SELECT uuid_timestamp(id), device, temp FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_end') ORDER BY id DESC; SELECT time_bucket('1 day', id) AS day, avg(temp) FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_end') GROUP BY id ORDER BY id DESC; SELECT time_bucket('1 day', uuid_timestamp(id)) AS day, avg(temp) FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_end') GROUP BY id ORDER BY id DESC; -- Bucket with offset SELECT time_bucket('1 day', id, "offset" => '1 week') AS day, avg(temp) FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_end') GROUP BY id ORDER BY id DESC; -- Bucket with origin SELECT time_bucket('1 day', id, timestamptz '2000-01-01 00:00') AS day, avg(temp) FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_end') GROUP BY id ORDER BY id DESC; -- Bucket with time zone SELECT time_bucket('1 day', id, 'Europe/Stockholm') at time zone 'Europe/Stockholm' as day, avg(temp) FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_end') GROUP BY id ORDER BY id DESC; -- Test NULL arguments SELECT time_bucket('1 day', id, 'Europe/Stockholm'::text, NULL::timestamptz) AS day, avg(temp) FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_end') GROUP BY id ORDER BY id DESC; SELECT time_bucket('1 day', id, 'Europe/Stockholm'::text, '2000-01-01 00:00'::timestamptz, NULL::interval) AS day, avg(temp) FROM uuid_events WHERE id < to_uuidv7_boundary(:'chunk_range_end') GROUP BY id ORDER BY id DESC; -- Test UUID time_bucket in WHERE clause. Note that there is currently no chunk -- exclusion when using time_bucket() in the WHERE clause qual. This requires -- special handling of the time_bucket() transform optimizatios for different -- operators. EXPLAIN (COSTS OFF) SELECT time_bucket('1 day', id) AS day, avg(temp) FROM uuid_events WHERE time_bucket('1 day', id) >= :'chunk_range_end' GROUP BY id ORDER BY id DESC; SELECT time_bucket('1 day', id) AS day, avg(temp) FROM uuid_events WHERE time_bucket('1 day', id) >= :'chunk_range_end' GROUP BY id ORDER BY id DESC; EXPLAIN (COSTS OFF) SELECT time_bucket('1 day', id) AS day, avg(temp) FROM uuid_events WHERE time_bucket('1 day', id) > :'chunk_range_start' GROUP BY id ORDER BY id DESC; SELECT time_bucket('1 day', id) AS day, avg(temp) FROM uuid_events WHERE time_bucket('1 day', id) > :'chunk_range_start' GROUP BY id ORDER BY id DESC; EXPLAIN (COSTS OFF) SELECT time_bucket('1 day', id) AS day, avg(temp) FROM uuid_events WHERE time_bucket('1 day', id) < :'chunk_range_end' GROUP BY id ORDER BY id DESC; SELECT time_bucket('1 day', id) AS day, avg(temp) FROM uuid_events WHERE time_bucket('1 day', id) < :'chunk_range_end' GROUP BY id ORDER BY id DESC; EXPLAIN (COSTS OFF) SELECT time_bucket('1 day', id) AS day, avg(temp) FROM uuid_events WHERE time_bucket('1 day', id) <= :'chunk_range_start' GROUP BY id ORDER BY id DESC; SELECT time_bucket('1 day', id) AS day, avg(temp) FROM uuid_events WHERE time_bucket('1 day', id) <= :'chunk_range_start' GROUP BY id ORDER BY id DESC; -- Test time_bucket on non-v7 UUID \set ON_ERROR_STOP 0 SELECT time_bucket('1 day', 'ffff0000-0000-4000-8000-000000000000'::uuid); \set ON_ERROR_STOP 1 CREATE VIEW chunk_ranges AS SELECT chunk_name, range_start, range_end FROM timescaledb_information.chunks WHERE hypertable_name = 'uuid_events'; SELECT * FROM chunk_ranges; SELECT show_chunks('uuid_events', older_than => INTERVAL '1 day'); SELECT show_chunks('uuid_events', older_than => '2025-01-02'); SELECT show_chunks('uuid_events', newer_than => '2025-01-02'); SELECT drop_chunks('uuid_events', older_than => '2025-01-02'); SELECT show_chunks('uuid_events'); -- Insert non-v7 UUIDs \set ON_ERROR_STOP 0 INSERT INTO uuid_events SELECT 'a8961135-cd89-4c4b-aa05-79df642407dd', 5, 5.0; \set ON_ERROR_STOP 1 -- Insert as v7 UUID and later change to non-v7 to show effect on show_chunks() -- and drop_chunks() INSERT INTO uuid_events SELECT 'a8961135-cd89-7000-aa05-79df642407dd', 5, 5.0; SELECT * FROM chunk_ranges; UPDATE uuid_events SET id = 'a8961135-cd89-4c4b-aa05-79df642407dd' WHERE id = 'a8961135-cd89-7000-aa05-79df642407dd'; SELECT show_chunks('uuid_events', newer_than => '2025-01-02'); SELECT drop_chunks('uuid_events', newer_than => '2025-01-02'); SELECT * FROM chunk_ranges; INSERT INTO uuid_events SELECT to_uuidv7(now()), 6, 6.0; SELECT show_chunks('uuid_events', newer_than => INTERVAL '2 months'); SELECT drop_chunks('uuid_events', newer_than => INTERVAL '2 months'); SELECT show_chunks('uuid_events', newer_than => INTERVAL '2 months'); DROP TABLE uuid_events; BEGIN; -- Test UUID partition when using CREATE TABLE ... WITH CREATE TABLE IF NOT EXISTS events ( event_id UUID NOT NULL, entity_id VARCHAR(100) NOT NULL, ts TIMESTAMPTZ NOT NULL, event_type VARCHAR(100) NOT NULL, metadata JSONB, PRIMARY KEY (event_id) ) WITH ( tsdb.hypertable, tsdb.partition_column='event_id', tsdb.chunk_interval='2 hours' ); -- Verify that the chunk time interval is two hours SELECT ((interval_length/1000000)/60)/60 AS hours FROM _timescaledb_catalog.dimension d JOIN _timescaledb_catalog.hypertable h ON (h.id = d.hypertable_id) WHERE h.table_name='events'; ROLLBACK; -- Test a different interval BEGIN; CREATE TABLE IF NOT EXISTS events ( event_id UUID NOT NULL, entity_id VARCHAR(100) NOT NULL, ts TIMESTAMPTZ NOT NULL, event_type VARCHAR(100) NOT NULL, metadata JSONB, PRIMARY KEY (event_id) ) WITH ( tsdb.hypertable, tsdb.partition_column='event_id', tsdb.chunk_interval='2 months' ); -- Verify that the chunk time interval is two hours SELECT (((interval_length/1000000)/60)/60)/24 AS days FROM _timescaledb_catalog.dimension d JOIN _timescaledb_catalog.hypertable h ON (h.id = d.hypertable_id) WHERE h.table_name='events'; ROLLBACK; -- Verify same behavior without CREATE TABLE WITH: BEGIN; CREATE TABLE IF NOT EXISTS events ( event_id UUID NOT NULL, entity_id VARCHAR(100) NOT NULL, ts TIMESTAMPTZ NOT NULL, event_type VARCHAR(100) NOT NULL, metadata JSONB, PRIMARY KEY (event_id) ); SELECT create_hypertable('events', 'event_id', chunk_time_interval => interval '2 hours'); -- Verify that the chunk time interval is two hours SELECT ((interval_length/1000000)/60)/60 AS hours FROM _timescaledb_catalog.dimension d JOIN _timescaledb_catalog.hypertable h ON (h.id = d.hypertable_id) WHERE h.table_name='events'; ROLLBACK; BEGIN; CREATE TABLE IF NOT EXISTS events ( event_id UUID NOT NULL, entity_id VARCHAR(100) NOT NULL, ts TIMESTAMPTZ NOT NULL, event_type VARCHAR(100) NOT NULL, metadata JSONB, PRIMARY KEY (event_id) ); SELECT create_hypertable('events', 'event_id', chunk_time_interval => interval '2 months'); -- Verify that the chunk time interval is two hours SELECT (((interval_length/1000000)/60)/60)/24 AS days FROM _timescaledb_catalog.dimension d JOIN _timescaledb_catalog.hypertable h ON (h.id = d.hypertable_id) WHERE h.table_name='events'; ROLLBACK; ================================================ FILE: test/sql/vacuum.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. CREATE TABLE vacuum_test(time timestamp, temp float); -- create hypertable with three chunks SELECT create_hypertable('vacuum_test', 'time', chunk_time_interval => 2628000000000, create_default_indexes => false); INSERT INTO vacuum_test VALUES ('2017-01-20T16:00:01', 17.5), ('2017-01-21T16:00:01', 19.1), ('2017-04-20T16:00:01', 89.5), ('2017-04-21T16:00:01', 17.1), ('2017-06-20T16:00:01', 18.5), ('2017-06-21T16:00:01', 11.0); -- no stats SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = '_timescaledb_internal' AND tablename LIKE '_hyper_%_chunk' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = 'public' AND tablename LIKE 'vacuum_test' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); VACUUM ANALYZE vacuum_test; -- stats should exist for all three chunks SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = '_timescaledb_internal' AND tablename LIKE '_hyper_%_chunk' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); -- stats should exist on parent hypertable SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = 'public' AND tablename LIKE 'vacuum_test' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); DROP TABLE vacuum_test; --test plain analyze (no_vacuum) CREATE TABLE analyze_test(time timestamp, temp float); SELECT create_hypertable('analyze_test', 'time', chunk_time_interval => 2628000000000, create_default_indexes => false); INSERT INTO analyze_test VALUES ('2017-01-20T16:00:01', 17.5), ('2017-01-21T16:00:01', 19.1), ('2017-04-20T16:00:01', 89.5), ('2017-04-21T16:00:01', 17.1), ('2017-06-20T16:00:01', 18.5), ('2017-06-21T16:00:01', 11.0); -- no stats SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = '_timescaledb_internal' AND tablename LIKE '_hyper_%_chunk' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = 'public' AND tablename LIKE 'analyze_test' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); ANALYZE analyze_test; -- stats should exist for all three chunks SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = '_timescaledb_internal' AND tablename LIKE '_hyper_%_chunk' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); -- stats should exist on parent hypertable SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = 'public' AND tablename LIKE 'analyze_test' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); DROP TABLE analyze_test; -- Run vacuum on a normal (non-hypertable) table CREATE TABLE vacuum_norm(time timestamp, temp float); INSERT INTO vacuum_norm VALUES ('2017-01-20T09:00:01', 17.5), ('2017-01-21T09:00:01', 19.1), ('2017-04-20T09:00:01', 89.5), ('2017-04-21T09:00:01', 17.1), ('2017-06-20T09:00:01', 18.5), ('2017-06-21T09:00:01', 11.0); VACUUM ANALYZE vacuum_norm; DROP TABLE vacuum_norm; --Similar to normal vacuum tests, but PG11 introduced ability to vacuum multiple tables at once, we make sure that works for hypertables as well. CREATE TABLE vacuum_test(time timestamp, temp float); -- create hypertable with three chunks SELECT create_hypertable('vacuum_test', 'time', chunk_time_interval => 2628000000000, create_default_indexes => false); INSERT INTO vacuum_test VALUES ('2017-01-20T16:00:01', 17.5), ('2017-01-21T16:00:01', 19.1), ('2017-04-20T16:00:01', 89.5), ('2017-04-21T16:00:01', 17.1), ('2017-06-20T16:00:01', 18.5), ('2017-06-21T16:00:01', 11.0); CREATE TABLE analyze_test(time timestamp, temp float); SELECT create_hypertable('analyze_test', 'time', chunk_time_interval => 2628000000000, create_default_indexes => false); INSERT INTO analyze_test VALUES ('2017-01-20T16:00:01', 17.5), ('2017-01-21T16:00:01', 19.1), ('2017-04-20T16:00:01', 89.5), ('2017-04-21T16:00:01', 17.1), ('2017-06-20T16:00:01', 18.5), ('2017-06-21T16:00:01', 11.0); CREATE TABLE vacuum_norm(time timestamp, temp float); INSERT INTO vacuum_norm VALUES ('2017-01-20T09:00:01', 17.5), ('2017-01-21T09:00:01', 19.1), ('2017-04-20T09:00:01', 89.5), ('2017-04-21T09:00:01', 17.1), ('2017-06-20T09:00:01', 18.5), ('2017-06-21T09:00:01', 11.0); -- no stats SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = '_timescaledb_internal' AND tablename LIKE '_hyper_%_chunk' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = 'public' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); VACUUM ANALYZE vacuum_norm, vacuum_test, analyze_test; -- stats should exist for all 6 chunks SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = '_timescaledb_internal' AND tablename LIKE '_hyper_%_chunk' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); -- stats should exist on parent hypertable and normal table SELECT tablename, attname, histogram_bounds, n_distinct FROM pg_stats WHERE schemaname = 'public' ORDER BY tablename, attname, array_to_string(histogram_bounds, ','); ================================================ FILE: test/sql/vacuum_parallel.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- PG13 introduced parallel VACUUM functionality. It gets invoked when a table -- has two or more indexes on it. Read up more at -- https://www.postgresql.org/docs/13/sql-vacuum.html#PARALLEL CREATE TABLE vacuum_test(time timestamp NOT NULL, temp1 float, temp2 int); -- create hypertable -- we create chunks in public schema cause otherwise we would need -- elevated privileges to create indexes directly SELECT create_hypertable('vacuum_test', 'time', create_default_indexes => false, associated_schema_name => 'public'); -- parallel vacuum needs the index size to be larger than min_parallel_index_scan_size to kick in SET min_parallel_index_scan_size TO 0; INSERT INTO vacuum_test SELECT TIMESTAMP 'epoch' + (i * INTERVAL '4h'), i, i+1 FROM generate_series(1, 100) as T(i); -- create indexes on the temp columns -- we create indexes manually because otherwise vacuum verbose output -- would be different between 13.2 and 13.3+ -- 13.2 would try to vacuum the parent table index too while 13.3+ wouldn't CREATE INDEX ON _hyper_1_1_chunk(time); CREATE INDEX ON _hyper_1_1_chunk(temp1); CREATE INDEX ON _hyper_1_1_chunk(temp2); CREATE INDEX ON _hyper_1_2_chunk(time); CREATE INDEX ON _hyper_1_2_chunk(temp1); CREATE INDEX ON _hyper_1_2_chunk(temp2); CREATE INDEX ON _hyper_1_3_chunk(time); CREATE INDEX ON _hyper_1_3_chunk(temp1); CREATE INDEX ON _hyper_1_3_chunk(temp2); -- INSERT only will not trigger vacuum on indexes for PG13.3+ UPDATE vacuum_test SET time = time + '1s'::interval, temp1 = random(), temp2 = random(); -- we should see two parallel workers for each chunk VACUUM (PARALLEL 3) vacuum_test; DROP TABLE vacuum_test; ================================================ FILE: test/sql/version.sql ================================================ -- This file and its contents are licensed under the Apache License 2.0. -- Please see the included NOTICE for copyright information and -- LICENSE-APACHE for a copy of the license. -- Test that get_os_info returns 3 x text select pg_typeof(sysname) AS sysname_type,pg_typeof(version) AS version_type,pg_typeof(release) AS release_type from _timescaledb_functions.get_os_info(); ================================================ FILE: test/src/CMakeLists.txt ================================================ set(SOURCES adt_tests.c metadata.c symbol_conflict.c test_compression_settings.c test_bmslist_utils.c test_jsonb_utils.c test_scanner.c test_time_to_internal.c test_time_utils.c test_tss_callbacks.c test_utils.c test_with_clause_parser.c) include(${PROJECT_SOURCE_DIR}/src/build-defs.cmake) add_library(${TESTS_LIB_NAME} OBJECT ${SOURCES}) # Since the test library will be linked into the loadable extension module, it # needs to be compiled as position-independent code (e.g., the -fPIC compiler # flag for GCC) set_target_properties(${TESTS_LIB_NAME} PROPERTIES POSITION_INDEPENDENT_CODE ON) # Set the MODULE_NAME for the symbol conflict test (see symbol_conflict.c) target_compile_definitions(${TESTS_LIB_NAME} PUBLIC MODULE_NAME=timescaledb) target_include_directories(${TESTS_LIB_NAME} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) add_subdirectory(bgw) add_subdirectory(net) if(USE_TELEMETRY) add_subdirectory(telemetry) endif() add_subdirectory(loader) ================================================ FILE: test/src/adt_tests.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <fmgr.h> #include "export.h" #include "test_utils.h" TS_FUNCTION_INFO_V1(ts_test_adts); #define VEC_PREFIX int32 #define VEC_ELEMENT_TYPE int32 #define VEC_DECLARE 1 #define VEC_DEFINE 1 #define VEC_SCOPE static inline #include <adts/vec.h> /* We have to stub this for the unit tests. */ #ifndef CheckCompressedData #define CheckCompressedData(X) Assert(X) #define GLOBAL_MAX_ROWS_PER_COMPRESSION 1015 #endif #include <adts/bit_array.h> static void i32_vec_test(void) { int32_vec *vec = int32_vec_create(CurrentMemoryContext, 0); int i; uint32 old_capacity; for (i = 0; i < 100; i++) int32_vec_append(vec, i); TestAssertInt64Eq(vec->num_elements, 100); if (vec->max_elements < 100) elog(ERROR, "vec capacity %d, should be at least 100", vec->max_elements); for (i = 0; i < 100; i++) TestAssertInt64Eq(*int32_vec_at(vec, i), i); TestAssertPtrEq(int32_vec_last(vec), int32_vec_at(vec, vec->num_elements - 1)); old_capacity = vec->max_elements; int32_vec_delete_range(vec, 30, 19); TestAssertInt64Eq(vec->num_elements, 81); TestAssertInt64Eq(vec->max_elements, old_capacity); for (i = 0; i < 30; i++) TestAssertInt64Eq(*int32_vec_at(vec, i), i); for (; i < 51; i++) TestAssertInt64Eq(*int32_vec_at(vec, i), i + 19); TestAssertPtrEq(int32_vec_last(vec), int32_vec_at(vec, vec->num_elements - 1)); int32_vec_clear(vec); TestAssertInt64Eq(vec->num_elements, 0); TestAssertInt64Eq(vec->max_elements, old_capacity); int32_vec_free_data(vec); TestAssertInt64Eq(vec->num_elements, 0); TestAssertInt64Eq(vec->max_elements, 0); TestAssertPtrEq(vec->data, NULL); /* free_data is idempotent */ int32_vec_free_data(vec); TestAssertInt64Eq(vec->num_elements, 0); TestAssertInt64Eq(vec->max_elements, 0); TestAssertPtrEq(vec->data, NULL); int32_vec_free(vec); } /* including BitArray should give us uint64_vec */ static void uint64_vec_test(void) { uint64_vec vec; int i; uint64_vec_init(&vec, CurrentMemoryContext, 100); for (i = 0; i < 30; i++) uint64_vec_append(&vec, i + 3); TestAssertInt64Eq(vec.num_elements, 30); TestAssertInt64Eq(vec.max_elements, 100); for (i = 0; i < 30; i++) TestAssertInt64Eq(*uint64_vec_at(&vec, i), i + 3); uint64_vec_free_data(&vec); TestAssertInt64Eq(vec.num_elements, 0); TestAssertInt64Eq(vec.max_elements, 0); TestAssertPtrEq(vec.data, NULL); } static void bit_array_test(void) { BitArray bits; BitArrayIterator iter; int i; bit_array_init(&bits, 0); for (i = 0; i < 65; i++) bit_array_append(&bits, i, i); bit_array_append(&bits, 0, 0); bit_array_append(&bits, 0, 0); bit_array_append(&bits, 64, 0x9069060909009090); bit_array_append(&bits, 1, 0); bit_array_append(&bits, 64, ~0x9069060909009090); bit_array_append(&bits, 1, 1); bit_array_iterator_init(&iter, &bits); for (i = 0; i < 65; i++) TestAssertInt64Eq(bit_array_iter_next(&iter, i), i); TestAssertInt64Eq(bit_array_iter_next(&iter, 0), 0); TestAssertInt64Eq(bit_array_iter_next(&iter, 0), 0); TestAssertInt64Eq(bit_array_iter_next(&iter, 64), 0x9069060909009090); TestAssertInt64Eq(bit_array_iter_next(&iter, 1), 0); TestAssertInt64Eq(bit_array_iter_next(&iter, 64), ~0x9069060909009090); TestAssertInt64Eq(bit_array_iter_next(&iter, 1), 1); bit_array_iterator_init_rev(&iter, &bits); TestAssertInt64Eq(bit_array_iter_next_rev(&iter, 1), 1); TestAssertInt64Eq(bit_array_iter_next_rev(&iter, 64), ~0x9069060909009090); TestAssertInt64Eq(bit_array_iter_next_rev(&iter, 1), 0); TestAssertInt64Eq(bit_array_iter_next_rev(&iter, 64), 0x9069060909009090); TestAssertInt64Eq(bit_array_iter_next_rev(&iter, 0), 0); TestAssertInt64Eq(bit_array_iter_next_rev(&iter, 0), 0); for (i = 64; i >= 0; i--) TestAssertInt64Eq(bit_array_iter_next_rev(&iter, i), i); } Datum ts_test_adts(PG_FUNCTION_ARGS) { i32_vec_test(); uint64_vec_test(); bit_array_test(); PG_RETURN_VOID(); } ================================================ FILE: test/src/bgw/CMakeLists.txt ================================================ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/log.c ${CMAKE_CURRENT_SOURCE_DIR}/timer_mock.c ${CMAKE_CURRENT_SOURCE_DIR}/scheduler_mock.c ${CMAKE_CURRENT_SOURCE_DIR}/params.c ${CMAKE_CURRENT_SOURCE_DIR}/test_job_refresh.c ${CMAKE_CURRENT_SOURCE_DIR}/test_job_utils.c) target_sources(${TESTS_LIB_NAME} PRIVATE ${SOURCES}) ================================================ FILE: test/src/bgw/README.md ================================================ # Background Worker Test Infrastructure This directory contains mocks and hooks to enable testing of timescale background workers. There are three main components: a counter-based timer used to test the scheduler in a deterministic manner; a scheduler that inserts test shims to the background worker and understands the tests we will run; and shims to dynamically set time and intercept background worker output. ## Output Background workers started by the test scheduler contain a hook storing all `elog` and `ereport` output in the table ```SQL public.bgw_log( msg_no INT, mock_time BIGINT, application_name TEXT, msg TEXT, ) ``` which must be created in order for tests to check background worker output. `msg_no` contains which message this was in the total-order of all background worker messages sent since the table was created; `mock_time` is the virtualized timestamp ([see that section](## Timer)) at which the message was written; `application_name` the name of the application that wrote the message; `msg` is the messgage string itself. See [`log.c`](log.c) for more detail. (We want to print more data from `ErrorData` at a later date) ## Timer We virtualize the timer to allow deterministic execution by tests. Our timer store a virtual microsecond counter in shared memory, backround processes can read this counter to determine the current time. The scheduler can "wait" on this timer which optionally waits for a process to finish and updates the counter to the waited time. The timer can be reset manually (for instance, to allow multiple tests in one file) with `ts_bgw_params_reset_time`). See [`timer_mock.c`](timer_mock.c) for more detail. ## Configuration Settings and the timer for background worker tests are stored in shared memory. This memory segment must be created with `ts_bgw_params_create` before use. see [params.c](params.c) and [params.h](params.h) for more detail. ================================================ FILE: test/src/bgw/log.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <catalog/namespace.h> #include <postmaster/bgworker.h> #include <storage/proc.h> #include <utils/builtins.h> #include <utils/lsyscache.h> #include <utils/snapmgr.h> #include "log.h" #include "params.h" #include "scanner.h" #include "ts_catalog/catalog.h" #include "utils.h" #include "compat/compat.h" static char *bgw_application_name = "unset"; void ts_bgw_log_set_application_name(char *name) { bgw_application_name = name; } static bool bgw_log_insert_relation(Relation rel, char *msg) { TupleDesc desc = RelationGetDescr(rel); static int32 msg_no = 0; Datum values[4]; bool nulls[4] = { false, false, false }; values[0] = Int32GetDatum(msg_no++); values[1] = Int64GetDatum(ts_params_get()->current_time); values[2] = CStringGetTextDatum(bgw_application_name); values[3] = CStringGetTextDatum(msg); ts_catalog_insert_values(rel, desc, values, nulls); return true; } /* Insert a new entry into public.bgw_log * This table is used for testing as a way for mock background jobs * to insert messages into a log that could then be output into the golden file */ static void bgw_log_insert(char *msg) { Relation rel; PushActiveSnapshot(GetTransactionSnapshot()); Oid log_oid = ts_get_relation_relid("public", "bgw_log", false); rel = table_open(log_oid, RowExclusiveLock); bgw_log_insert_relation(rel, msg); table_close(rel, RowExclusiveLock); PopActiveSnapshot(); } static emit_log_hook_type prev_emit_log_hook = NULL; /* * NOTE: using transactions in emit_log_hook functions is not recommended. * However we rely on this current functionality for our test verifications, * so have to live with it for now. */ static void emit_log_hook_callback(ErrorData *edata) { /* * once proc_exit has started we may no longer be able to start transactions */ if (MyProc == NULL) return; /* * Block signals so we don't lose messages generated during signal * processing if they occur while we are saving this log message (since * emit_log_hook is modified and restored below) */ BackgroundWorkerBlockSignals(); PG_TRY(); { /* * If we do encounter some error writing to our log hook, remove the * hook to prevent potentially infinite recursion where this callback * keeps encountering an error, and it is its own logging callback. We * reinstall the hook when we're successfully done with this function. */ emit_log_hook = NULL; bool started_txn = false; if (!IsTransactionState()) { StartTransactionCommand(); started_txn = true; } bgw_log_insert(edata->message); if (started_txn) CommitTransactionCommand(); if (prev_emit_log_hook != NULL) prev_emit_log_hook(edata); /* Reinstall the hook if log was successful. */ emit_log_hook = emit_log_hook_callback; } PG_CATCH(); { /* If there was an error, rollback what was done before the error */ if (IsTransactionState()) AbortCurrentTransaction(); /* * Reinstall the hook because we are out of the main body of the * function. */ emit_log_hook = emit_log_hook_callback; } PG_END_TRY(); BackgroundWorkerUnblockSignals(); } void ts_register_emit_log_hook() { prev_emit_log_hook = emit_log_hook; emit_log_hook = emit_log_hook_callback; } ================================================ FILE: test/src/bgw/log.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once extern void ts_bgw_log_set_application_name(char *name); extern void ts_register_emit_log_hook(void); ================================================ FILE: test/src/bgw/params.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/relscan.h> #include <access/xact.h> #include <catalog/namespace.h> #include <storage/bufmgr.h> #include <storage/dsm.h> #include <storage/lmgr.h> #include <storage/spin.h> #include <utils/builtins.h> #include <utils/lsyscache.h> #include <utils/rel.h> #include "log.h" #include "params.h" #include "scanner.h" #include "test_utils.h" #include "timer_mock.h" #include "ts_catalog/catalog.h" #include "utils.h" typedef struct FormData_bgw_dsm_handle { /* handle is actually a uint32 */ int64 handle; } FormData_bgw_dsm_handle; typedef struct TestParamsWrapper { TestParams params; slock_t mutex; } TestParamsWrapper; static Oid get_dsm_handle_table_oid() { return ts_get_relation_relid("public", "bgw_dsm_handle_store", false); } static void params_register_dsm_handle(dsm_handle handle) { Relation rel; TableScanDesc scan; HeapTuple tuple; FormData_bgw_dsm_handle *fd; rel = table_open(get_dsm_handle_table_oid(), RowExclusiveLock); scan = table_beginscan(rel, SnapshotSelf, 0, NULL); tuple = heap_copytuple(heap_getnext(scan, ForwardScanDirection)); fd = (FormData_bgw_dsm_handle *) GETSTRUCT(tuple); fd->handle = handle; ts_catalog_update(rel, tuple); heap_freetuple(tuple); table_endscan(scan); table_close(rel, RowExclusiveLock); } static dsm_handle params_load_dsm_handle() { Relation rel; TableScanDesc scan; HeapTuple tuple; FormData_bgw_dsm_handle *fd; dsm_handle handle; rel = table_open(get_dsm_handle_table_oid(), RowExclusiveLock); scan = table_beginscan(rel, SnapshotSelf, 0, NULL); tuple = heap_getnext(scan, ForwardScanDirection); TestAssertTrue(tuple != NULL); tuple = heap_copytuple(tuple); fd = (FormData_bgw_dsm_handle *) GETSTRUCT(tuple); handle = fd->handle; heap_freetuple(tuple); table_endscan(scan); table_close(rel, RowExclusiveLock); return handle; } static dsm_handle params_get_dsm_handle() { static dsm_handle handle = 0; if (handle == 0) handle = params_load_dsm_handle(); return handle; } static TestParamsWrapper * params_open_wrapper(bool *do_close) { dsm_segment *seg; dsm_handle handle = params_get_dsm_handle(); TestParamsWrapper *wrapper; /* * If segment is returned via the mapping then there's no need to call * dsm_detach on it in params_close_wrapper */ seg = dsm_find_mapping(handle); if (seg == NULL) { seg = dsm_attach(handle); if (seg == NULL) elog(ERROR, "got NULL segment in params_open_wrapper"); *do_close = true; } else *do_close = false; TestAssertTrue(seg != NULL); wrapper = dsm_segment_address(seg); TestAssertTrue(wrapper != NULL); return wrapper; }; static void params_close_wrapper(TestParamsWrapper *wrapper) { dsm_segment *seg = dsm_find_mapping(params_get_dsm_handle()); TestAssertTrue(seg != NULL); dsm_detach(seg); } TestParams * ts_params_get() { bool do_close; TestParamsWrapper *wrapper = params_open_wrapper(&do_close); TestParams *res; TestAssertTrue(wrapper != NULL); res = palloc(sizeof(TestParams)); SpinLockAcquire(&wrapper->mutex); memcpy(res, &wrapper->params, sizeof(TestParams)); SpinLockRelease(&wrapper->mutex); if (do_close) params_close_wrapper(wrapper); return res; }; void ts_params_set_time(int64 new_val, bool set_latch) { bool do_close; TestParamsWrapper *wrapper = params_open_wrapper(&do_close); TestAssertTrue(wrapper != NULL); SpinLockAcquire(&wrapper->mutex); wrapper->params.current_time = new_val; SpinLockRelease(&wrapper->mutex); if (set_latch) SetLatch(&wrapper->params.timer_latch); if (do_close) params_close_wrapper(wrapper); } void ts_initialize_timer_latch() { bool do_close; TestParamsWrapper *wrapper = params_open_wrapper(&do_close); TestAssertTrue(wrapper != NULL); SpinLockAcquire(&wrapper->mutex); InitLatch(&wrapper->params.timer_latch); SpinLockRelease(&wrapper->mutex); if (do_close) params_close_wrapper(wrapper); } void ts_reset_and_wait_timer_latch() { bool do_close; TestParamsWrapper *wrapper = params_open_wrapper(&do_close); TestAssertTrue(wrapper != NULL); ResetLatch(&wrapper->params.timer_latch); WaitLatch(&wrapper->params.timer_latch, WL_LATCH_SET | WL_TIMEOUT | WL_POSTMASTER_DEATH, 10000, PG_WAIT_EXTENSION); if (do_close) params_close_wrapper(wrapper); } static void params_set_mock_wait_type(MockWaitType new_val) { bool do_close; TestParamsWrapper *wrapper = params_open_wrapper(&do_close); TestAssertTrue(wrapper != NULL); SpinLockAcquire(&wrapper->mutex); wrapper->params.mock_wait_type = new_val; SpinLockRelease(&wrapper->mutex); if (do_close) params_close_wrapper(wrapper); } TS_FUNCTION_INFO_V1(ts_bgw_params_reset_time); Datum ts_bgw_params_reset_time(PG_FUNCTION_ARGS) { ts_params_set_time(PG_GETARG_INT64(0), PG_GETARG_BOOL(1)); PG_RETURN_VOID(); } TS_FUNCTION_INFO_V1(ts_bgw_params_mock_wait_returns_immediately); Datum ts_bgw_params_mock_wait_returns_immediately(PG_FUNCTION_ARGS) { params_set_mock_wait_type(PG_GETARG_INT32(0)); PG_RETURN_VOID(); } TS_FUNCTION_INFO_V1(ts_bgw_params_create); Datum ts_bgw_params_create(PG_FUNCTION_ARGS) { dsm_segment *seg = dsm_create(sizeof(TestParamsWrapper), 0); TestParamsWrapper *params; TestAssertTrue(seg != NULL); params = dsm_segment_address(seg); *params = (TestParamsWrapper) { .params = { .current_time = 0, }, }; SpinLockInit(¶ms->mutex); params_register_dsm_handle(dsm_segment_handle(seg)); dsm_pin_mapping(seg); dsm_pin_segment(seg); PG_RETURN_VOID(); } TS_FUNCTION_INFO_V1(ts_bgw_params_destroy); Datum ts_bgw_params_destroy(PG_FUNCTION_ARGS) { /* * Removing shared memory segment unpin for now because: * 1) This can fail in EXEC_BACKEND cases. * 2) There's no way to unpin in PG9.6. * 3) The EXEC_BACKEND compile-time flag is not correctly passed down. * 4) This should only affect tests, not actual DB functionality. * #if PG10 && !defined(EXEC_BACKEND) * dsm_unpin_segment(params_get_dsm_handle()); * #endif */ PG_RETURN_VOID(); } ================================================ FILE: test/src/bgw/params.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <storage/latch.h> typedef enum MockWaitType { WAIT_ON_JOB = 0, IMMEDIATELY_SET_UNTIL, WAIT_FOR_OTHER_TO_ADVANCE, WAIT_FOR_STANDARD_WAITLATCH, _MAX_MOCK_WAIT_TYPE } MockWaitType; typedef struct TestParams { Latch timer_latch; int64 current_time; MockWaitType mock_wait_type; } TestParams; extern TestParams *ts_params_get(void); extern void ts_params_set_time(int64 new_val, bool set_latch); extern void ts_initialize_timer_latch(void); extern void ts_reset_and_wait_timer_latch(void); ================================================ FILE: test/src/bgw/scheduler_mock.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <miscadmin.h> #include <pgstat.h> #include <postmaster/bgworker.h> #include <signal.h> #include <storage/ipc.h> #include <storage/latch.h> #include <storage/lmgr.h> #include <storage/lwlock.h> #include <storage/proc.h> #include <storage/shmem.h> #include <utils/builtins.h> #include <utils/errcodes.h> #include <utils/guc.h> #include <utils/jsonb.h> #include <utils/memutils.h> #include <utils/snapmgr.h> #include <utils/timestamp.h> #include "bgw/job.h" #include "bgw/job_stat.h" #include "bgw/scheduler.h" #include "cross_module_fn.h" #include "extension.h" #include "log.h" #include "params.h" #include "test_utils.h" #include "time_bucket.h" #include "timer_mock.h" TS_FUNCTION_INFO_V1(ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish); TS_FUNCTION_INFO_V1(ts_bgw_db_scheduler_test_run); TS_FUNCTION_INFO_V1(ts_bgw_db_scheduler_test_wait_for_scheduler_finish); TS_FUNCTION_INFO_V1(ts_bgw_db_scheduler_test_main); TS_FUNCTION_INFO_V1(ts_bgw_job_execute_test); /* function for testing the correctness of the next_scheduled_slot calculation */ TS_FUNCTION_INFO_V1(ts_test_next_scheduled_execution_slot); typedef enum TestJobType { TEST_JOB_TYPE_JOB_1 = 0, TEST_JOB_TYPE_JOB_2_ERROR, TEST_JOB_TYPE_JOB_3_LONG, TEST_JOB_TYPE_JOB_4, _MAX_TEST_JOB_TYPE } TestJobType; static const char *test_job_type_names[_MAX_TEST_JOB_TYPE] = { [TEST_JOB_TYPE_JOB_1] = "bgw_test_job_1", [TEST_JOB_TYPE_JOB_2_ERROR] = "bgw_test_job_2_error", [TEST_JOB_TYPE_JOB_3_LONG] = "bgw_test_job_3_long", [TEST_JOB_TYPE_JOB_4] = "bgw_test_job_4", }; /* this is copied from the job_stat/ts_get_next_scheduled_execution_slot */ extern Datum ts_test_next_scheduled_execution_slot(PG_FUNCTION_ARGS) { Interval *schedule_interval = PG_GETARG_INTERVAL_P(0); TimestampTz finish_time = PG_GETARG_TIMESTAMPTZ(1); TimestampTz initial_start = PG_GETARG_TIMESTAMPTZ(2); text *timezone = PG_ARGISNULL(3) ? NULL : PG_GETARG_TEXT_PP(3); Datum timebucket_fini, timebucket_init, result; Datum schedint_datum = IntervalPGetDatum(schedule_interval); Interval one_month = { .day = 0, .time = 0, .month = 1, }; if (schedule_interval->month > 0) { if (timezone == NULL) { timebucket_init = DirectFunctionCall2(ts_timestamptz_bucket, schedint_datum, TimestampTzGetDatum(initial_start)); timebucket_fini = DirectFunctionCall2(ts_timestamptz_bucket, schedint_datum, TimestampTzGetDatum(finish_time)); } else { char *tz = text_to_cstring(timezone); timebucket_fini = DirectFunctionCall3(ts_timestamptz_timezone_bucket, schedint_datum, TimestampTzGetDatum(finish_time), CStringGetTextDatum(tz)); timebucket_init = DirectFunctionCall3(ts_timestamptz_timezone_bucket, schedint_datum, TimestampTzGetDatum(initial_start), CStringGetTextDatum(tz)); } /* always the next bucket */ timebucket_fini = DirectFunctionCall2(timestamptz_pl_interval, timebucket_fini, schedint_datum); /* get the number of months between them */ Datum year_init = DirectFunctionCall2(timestamptz_part, CStringGetTextDatum("year"), timebucket_init); Datum year_fini = DirectFunctionCall2(timestamptz_part, CStringGetTextDatum("year"), timebucket_fini); Datum month_init = DirectFunctionCall2(timestamptz_part, CStringGetTextDatum("month"), timebucket_init); Datum month_fini = DirectFunctionCall2(timestamptz_part, CStringGetTextDatum("month"), timebucket_fini); /* convert everything to months */ float8 month_diff = (DatumGetFloat8(year_fini) * 12) + DatumGetFloat8(month_fini) - ((DatumGetFloat8(year_init) * 12) + DatumGetFloat8(month_init)); Datum months_to_add = DirectFunctionCall2(interval_mul, IntervalPGetDatum(&one_month), Float8GetDatum(month_diff)); result = DirectFunctionCall2(timestamptz_pl_interval, TimestampTzGetDatum(initial_start), months_to_add); } else { if (timezone == NULL) { /* it is safe to use the origin in time_bucket calculation */ timebucket_fini = DirectFunctionCall3(ts_timestamptz_bucket, schedint_datum, TimestampTzGetDatum(finish_time), TimestampTzGetDatum(initial_start)); result = timebucket_fini; } else { char *tz = text_to_cstring(timezone); timebucket_fini = DirectFunctionCall4(ts_timestamptz_timezone_bucket, schedint_datum, TimestampTzGetDatum(finish_time), CStringGetTextDatum(tz), TimestampTzGetDatum(initial_start)); result = timebucket_fini; } } while (DatumGetTimestampTz(result) <= finish_time) { result = DirectFunctionCall2(timestamptz_pl_interval, result, schedint_datum); } return result; } extern Datum ts_bgw_db_scheduler_test_main(PG_FUNCTION_ARGS) { Oid db_oid = DatumGetObjectId(MyBgworkerEntry->bgw_main_arg); BgwParams bgw_params; BackgroundWorkerBlockSignals(); /* Setup any signal handlers here */ ts_bgw_scheduler_register_signal_handlers(); BackgroundWorkerUnblockSignals(); ts_bgw_scheduler_setup_callbacks(); memcpy(&bgw_params, MyBgworkerEntry->bgw_extra, sizeof(bgw_params)); elog(NOTICE, "scheduler user id %u", bgw_params.user_oid); elog(NOTICE, "running a test in the background: db=%u ttl=%d", db_oid, bgw_params.ttl); BackgroundWorkerInitializeConnectionByOid(db_oid, bgw_params.user_oid, 0); StartTransactionCommand(); ts_params_get(); ts_initialize_timer_latch(); CommitTransactionCommand(); ts_bgw_log_set_application_name("DB Scheduler"); ts_register_emit_log_hook(); ts_timer_set(&ts_mock_timer); ts_bgw_job_set_job_entrypoint_function_name("ts_bgw_job_execute_test"); pgstat_report_appname("DB Scheduler Test"); ts_bgw_scheduler_setup_mctx(); ts_bgw_scheduler_process(bgw_params.ttl, ts_timer_mock_register_bgw_handle); PG_RETURN_VOID(); } static BackgroundWorkerHandle * start_test_scheduler(int32 ttl, Oid user_oid) { const BgwParams bgw_params = { .bgw_main = "ts_bgw_db_scheduler_test_main", .ttl = ttl, .user_oid = user_oid, }; /* * This is where we would increment the number of bgw used, if we * decide to do so */ ts_bgw_scheduler_setup_mctx(); return ts_bgw_start_worker("ts_bgw_db_scheduler_test_main", &bgw_params); } /* this function will start up a bgw for the scheduler and set the ttl to the given value * (microseconds) */ extern Datum ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(PG_FUNCTION_ARGS) { BackgroundWorkerHandle *worker_handle; pid_t pid; worker_handle = start_test_scheduler(PG_GETARG_INT32(0), GetUserId()); TestAssertTrue(worker_handle != NULL); /* * If RegisterDynamicbackgroundworker fails, worker_handle will be * NULL. Since messages have already been printed in, just exit. */ if (!worker_handle) PG_RETURN_VOID(); BgwHandleStatus status = WaitForBackgroundWorkerStartup(worker_handle, &pid); TestAssertTrue(BGWH_STARTED == status); if (status != BGWH_STARTED) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("bgw not started"))); status = WaitForBackgroundWorkerShutdown(worker_handle); TestAssertTrue(BGWH_STOPPED == status); if (status != BGWH_STOPPED) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("bgw not stopped"))); PG_RETURN_VOID(); } static BackgroundWorkerHandle *current_handle = NULL; extern Datum ts_bgw_db_scheduler_test_run(PG_FUNCTION_ARGS) { pid_t pid; MemoryContext old_ctx; BgwHandleStatus status; old_ctx = MemoryContextSwitchTo(TopMemoryContext); current_handle = start_test_scheduler(PG_GETARG_INT32(0), GetUserId()); MemoryContextSwitchTo(old_ctx); /* * If RegisterDynamicbackgroundworker fails, current_handle will be * NULL. Since messages have already been printed in, just exit. */ if (!current_handle) PG_RETURN_VOID(); status = WaitForBackgroundWorkerStartup(current_handle, &pid); TestAssertTrue(BGWH_STARTED == status); if (status != BGWH_STARTED) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("bgw not started"))); PG_RETURN_VOID(); } extern Datum ts_bgw_db_scheduler_test_wait_for_scheduler_finish(PG_FUNCTION_ARGS) { if (current_handle != NULL) { BgwHandleStatus status = WaitForBackgroundWorkerShutdown(current_handle); TestAssertTrue(BGWH_STOPPED == status); if (status != BGWH_STOPPED) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("bgw not stopped"))); } PG_RETURN_VOID(); } static bool test_job_1() { StartTransactionCommand(); elog(WARNING, "Execute job 1"); CommitTransactionCommand(); return true; } static bool test_job_2_error() { StartTransactionCommand(); elog(WARNING, "Before error job 2"); ereport(ERROR, (errcode(ERRCODE_TRIGGERED_ACTION_EXCEPTION), errmsg("Error job 2"))); elog(WARNING, "After error job 2"); CommitTransactionCommand(); return true; } static void log_terminate_signal(SIGNAL_ARGS) { write_stderr("job got term signal\n"); die(postgres_signal_arg); } TS_FUNCTION_INFO_V1(ts_bgw_test_job_sleep); /* * This function is used for testing removing jobs with * a currently running background job. */ Datum ts_bgw_test_job_sleep(PG_FUNCTION_ARGS) { BackgroundWorkerBlockSignals(); pqsignal(SIGTERM, log_terminate_signal); /* Setup any signal handlers here */ BackgroundWorkerUnblockSignals(); elog(WARNING, "Before sleep"); PopActiveSnapshot(); /* * we commit here so the effect of the elog which is written * to a table with a emit_log_hook is seen by other transactions * to verify the background job started */ CommitTransactionCommand(); StartTransactionCommand(); DirectFunctionCall1(pg_sleep, Float8GetDatum(10)); elog(WARNING, "After sleep"); PG_RETURN_VOID(); } static bool test_job_3_long() { BackgroundWorkerBlockSignals(); pqsignal(SIGTERM, log_terminate_signal); /* Setup any signal handlers here */ BackgroundWorkerUnblockSignals(); elog(WARNING, "Before sleep job 3"); DirectFunctionCall1(pg_sleep, Float8GetDatum(2.0L)); elog(WARNING, "After sleep job 3"); return true; } /* Exactly like job 1, except a wrapper will change its next_start. */ static bool test_job_4(void) { elog(WARNING, "Execute job 4"); return true; } static TestJobType get_test_job_type_from_name(Name job_type_name) { int i; for (i = 0; i < _MAX_TEST_JOB_TYPE; i++) { if (namestrcmp(job_type_name, test_job_type_names[i]) == 0) return i; } return _MAX_TEST_JOB_TYPE; } static bool test_job_dispatcher(BgwJob *job) { ts_register_emit_log_hook(); ts_bgw_log_set_application_name(strdup(NameStr(job->fd.application_name))); StartTransactionCommand(); ts_params_get(); CommitTransactionCommand(); switch (get_test_job_type_from_name(&job->fd.proc_name)) { case TEST_JOB_TYPE_JOB_1: return test_job_1(); case TEST_JOB_TYPE_JOB_2_ERROR: return test_job_2_error(); case TEST_JOB_TYPE_JOB_3_LONG: return test_job_3_long(); case TEST_JOB_TYPE_JOB_4: { /* Set next_start to 200ms */ Interval new_interval = { .time = .2 * USECS_PER_SEC }; return ts_bgw_job_run_and_set_next_start(job, test_job_4, 3, &new_interval, /* atomic */ true, /* mark */ false); } default: return ts_cm_functions->job_execute(job); } return false; } Datum ts_bgw_job_execute_test(PG_FUNCTION_ARGS) { ts_timer_set(&ts_mock_timer); ts_bgw_job_set_scheduler_test_hook(test_job_dispatcher); return ts_bgw_job_entrypoint(fcinfo); } ================================================ FILE: test/src/bgw/test_job_refresh.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <string.h> #include <postgres.h> #include <access/htup_details.h> #include <fmgr.h> #include <funcapi.h> #include <stdlib.h> #include <time.h> #include <utils/memutils.h> #include "compat/compat.h" #include "bgw/scheduler.h" #include "export.h" TS_FUNCTION_INFO_V1(ts_test_job_refresh); static List *cur_scheduled_jobs = NIL; /* Test update_scheduled_jobs_list will correctly update with jobs in bgw_job table. * Call this function after loading up bgw_job table */ Datum ts_test_job_refresh(PG_FUNCTION_ARGS) { FuncCallContext *funcctx; ListCell *lc; if (SRF_IS_FIRSTCALL()) { MemoryContext oldcontext; TupleDesc tupdesc; /* Use top-level memory context to preserve the global static list */ cur_scheduled_jobs = ts_update_scheduled_jobs_list(cur_scheduled_jobs, TopMemoryContext); funcctx = SRF_FIRSTCALL_INIT(); oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); funcctx->user_fctx = list_head(cur_scheduled_jobs); if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context " "that cannot accept type record"))); } funcctx->tuple_desc = BlessTupleDesc(tupdesc); MemoryContextSwitchTo(oldcontext); } funcctx = SRF_PERCALL_SETUP(); lc = (ListCell *) funcctx->user_fctx; if (lc == NULL) SRF_RETURN_DONE(funcctx); else { /* Return the current list_cell and advance ptr */ HeapTuple tuple; Datum *values = palloc(sizeof(*values) * funcctx->tuple_desc->natts); bool *nulls = palloc(sizeof(*nulls) * funcctx->tuple_desc->natts); ts_populate_scheduled_job_tuple(lfirst(lc), values); memset(nulls, 0, sizeof(*nulls) * funcctx->tuple_desc->natts); tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); funcctx->user_fctx = lnext(cur_scheduled_jobs, lc); SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(tuple)); } PG_RETURN_NULL(); } ================================================ FILE: test/src/bgw/test_job_utils.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <fmgr.h> #include <funcapi.h> #include "bgw/job.h" #include "test_utils.h" TS_TEST_FN(ts_test_bgw_job_function_call_string) { int32 job_id = PG_GETARG_INT32(0); BgwJob *job = ts_bgw_job_find(job_id, CurrentMemoryContext, true); const char *stmt = ts_bgw_job_function_call_string(job); PG_RETURN_TEXT_P(cstring_to_text(stmt)); } ================================================ FILE: test/src/bgw/timer_mock.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/relscan.h> #include <access/xact.h> #include <catalog/namespace.h> #include <storage/bufmgr.h> #include <storage/lmgr.h> #include <utils/builtins.h> #include <utils/lsyscache.h> #include <utils/rel.h> #include "annotations.h" #include "bgw/launcher_interface.h" #include "log.h" #include "params.h" #include "scanner.h" #include "timer_mock.h" #include "ts_catalog/catalog.h" static List *bgw_handles = NIL; static bool mock_wait(TimestampTz until); static TimestampTz mock_current_time(void); const Timer ts_mock_timer = { .get_current_timestamp = mock_current_time, .wait = mock_wait, }; void ts_timer_mock_register_bgw_handle(BackgroundWorkerHandle *handle, MemoryContext scheduler_mctx) { elog(WARNING, "[TESTING] Registered new background worker"); MemoryContext old_context = MemoryContextSwitchTo(scheduler_mctx); bgw_handles = lappend(bgw_handles, handle); MemoryContextSwitchTo(old_context); } /* WARNING: mock_wait must _only_ be called from the bgw_scheduler, calling it from a worker will * clobber the timer state */ static bool mock_wait(TimestampTz until) { elog(WARNING, "[TESTING] Wait until " INT64_FORMAT ", started at " INT64_FORMAT, until, ts_params_get()->current_time); ListCell *lc; switch (ts_params_get()->mock_wait_type) { case WAIT_ON_JOB: foreach (lc, bgw_handles) { BackgroundWorkerHandle *bgw_handle = lfirst(lc); WaitForBackgroundWorkerShutdown(bgw_handle); } bgw_handles = NIL; TS_FALLTHROUGH; case IMMEDIATELY_SET_UNTIL: ts_params_set_time(until, false); return true; case WAIT_FOR_OTHER_TO_ADVANCE: { /* Wait for another process to set "next time" */ ts_reset_and_wait_timer_latch(); return true; } case WAIT_FOR_STANDARD_WAITLATCH: ts_get_standard_timer()->wait(until); return true; default: return false; } } static TimestampTz mock_current_time() { return ts_params_get()->current_time; } ================================================ FILE: test/src/bgw/timer_mock.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <postmaster/bgworker.h> #include "bgw/timer.h" extern void ts_timer_mock_register_bgw_handle(BackgroundWorkerHandle *handle, MemoryContext scheduler_mctx); extern const Timer ts_mock_timer; ================================================ FILE: test/src/loader/CMakeLists.txt ================================================ set(SOURCES init.c ${PROJECT_SOURCE_DIR}/src/extension.c ${PROJECT_SOURCE_DIR}/src/guc.c) include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${PROJECT_SOURCE_DIR}/src) add_library(${PROJECT_NAME}-mock-1 MODULE ${SOURCES} config.h) add_library(${PROJECT_NAME}-mock-2 MODULE ${SOURCES} config.h) add_library(${PROJECT_NAME}-mock-3 MODULE ${SOURCES} config.h) # mock-4 will be broken mismatched .so add_library(${PROJECT_NAME}-mock-4 MODULE ${SOURCES} config.h) add_library(${PROJECT_NAME}-mock-5 MODULE ${SOURCES} config.h) add_library(${PROJECT_NAME}-mock-broken MODULE ${SOURCES} config.h) add_library(${PROJECT_NAME}-mock-6 MODULE ${SOURCES} config.h) target_compile_definitions(${PROJECT_NAME}-mock-1 PRIVATE TIMESCALEDB_VERSION_MOD="mock-1" BROKEN=0) target_compile_definitions(${PROJECT_NAME}-mock-2 PRIVATE TIMESCALEDB_VERSION_MOD="mock-2" BROKEN=0) target_compile_definitions(${PROJECT_NAME}-mock-3 PRIVATE TIMESCALEDB_VERSION_MOD="mock-3" BROKEN=0) # mock 4 is intentionally incorrect version mod target_compile_definitions( ${PROJECT_NAME}-mock-4 PRIVATE TIMESCALEDB_VERSION_MOD="mock-4-mismatch" BROKEN=0) target_compile_definitions(${PROJECT_NAME}-mock-5 PRIVATE TIMESCALEDB_VERSION_MOD="mock-5" BROKEN=0) target_compile_definitions( ${PROJECT_NAME}-mock-broken PRIVATE TIMESCALEDB_VERSION_MOD="mock-broken" BROKEN=1) target_compile_definitions(${PROJECT_NAME}-mock-6 PRIVATE TIMESCALEDB_VERSION_MOD="mock-6" BROKEN=0) foreach( MOCK_VERSION mock-1 mock-2 mock-3 mock-4 mock-broken mock-5 mock-6) set_target_properties( ${PROJECT_NAME}-${MOCK_VERSION} PROPERTIES OUTPUT_NAME ${PROJECT_NAME}-${MOCK_VERSION} PREFIX "") install( TARGETS ${PROJECT_NAME}-${MOCK_VERSION} DESTINATION ${PG_PKGLIBDIR} OPTIONAL) endforeach(MOCK_VERSION) add_library(${PROJECT_NAME}_osm-mock-1 MODULE osm_init.c) target_compile_definitions(${PROJECT_NAME}_osm-mock-1 PRIVATE OSM_VERSION_MOD="mock-1") foreach(MOCK_VERSION mock-1) set_target_properties( ${PROJECT_NAME}_osm-${MOCK_VERSION} PROPERTIES OUTPUT_NAME ${PROJECT_NAME}_osm-${MOCK_VERSION} PREFIX "") install( TARGETS ${PROJECT_NAME}_osm-${MOCK_VERSION} DESTINATION ${PG_PKGLIBDIR} OPTIONAL) endforeach(MOCK_VERSION) ================================================ FILE: test/src/loader/config.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ /* Overwrite main config.h so we could inject our own version #s here. */ #define TELEMETRY_DEFAULT TELEMETRY_OFF ================================================ FILE: test/src/loader/init.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <config.h> #ifndef WIN32 #include <access/parallel.h> #endif #include "compat/compat.h" #include "export.h" #include "extension.h" #include <commands/extension.h> #include <miscadmin.h> #include <parser/analyze.h> #include <utils/guc.h> #include <utils/inval.h> #define STR_EXPAND(x) #x #define STR(x) STR_EXPAND(x) #ifdef PG_MODULE_MAGIC PG_MODULE_MAGIC; #endif #if PG16_LT extern void PGDLLEXPORT _PG_init(void); #endif static post_parse_analyze_hook_type prev_post_parse_analyze_hook; bool ts_license_guc_check_hook(char **newval, void **extra, GucSource source); void ts_license_guc_assign_hook(const char *newval, void *extra); TS_FUNCTION_INFO_V1(ts_post_load_init); static void cache_invalidate_callback(Datum arg, Oid relid) { if (ts_extension_is_proxy_table_relid(relid)) ts_extension_invalidate(); } static void post_analyze_hook(ParseState *pstate, Query *query, JumbleState *jstate) { if (ts_extension_is_loaded_and_not_upgrading()) elog(WARNING, "mock post_analyze_hook " STR(TIMESCALEDB_VERSION_MOD)); /* * a symbol needed by IsParallelWorker is not exported on windows so we do * not perform this check */ #ifndef WIN32 if (prev_post_parse_analyze_hook != NULL && !IsParallelWorker()) elog(ERROR, "the extension called with a loader should always have a NULL prev hook"); #endif if (BROKEN && !creating_extension) ereport(ERROR, (errcode(ERRCODE_TRIGGERED_ACTION_EXCEPTION), errmsg("mock broken " STR(TIMESCALEDB_VERSION_MOD)))); } void _PG_init(void) { /* * Check extension_is loaded to catch certain errors such as calls to * functions defined on the wrong extension version */ ts_extension_check_version(TIMESCALEDB_VERSION_MOD); elog(WARNING, "mock init " STR(TIMESCALEDB_VERSION_MOD)); prev_post_parse_analyze_hook = post_parse_analyze_hook; /* * a symbol needed by IsParallelWorker is not exported on windows so we do * not perform this check */ #ifndef WIN32 if (prev_post_parse_analyze_hook != NULL && !IsParallelWorker()) elog(ERROR, "the extension called with a loader should always have a NULL prev hook"); #endif post_parse_analyze_hook = post_analyze_hook; CacheRegisterRelcacheCallback(cache_invalidate_callback, PointerGetDatum(NULL)); } /* mock for extension.c */ void ts_catalog_reset(void); void ts_catalog_reset() { } /* mock for guc.c */ void ts_hypertable_cache_invalidate_callback(void); void ts_hypertable_cache_invalidate_callback(void) { } TS_FUNCTION_INFO_V1(ts_mock_function); Datum ts_mock_function(PG_FUNCTION_ARGS) { elog(WARNING, "mock function call " STR(TIMESCALEDB_VERSION_MOD)); PG_RETURN_VOID(); } TSDLLEXPORT Datum ts_post_load_init(PG_FUNCTION_ARGS) { PG_RETURN_CHAR(0); } bool ts_license_guc_check_hook(char **newval, void **extra, GucSource source) { return true; } void ts_license_guc_assign_hook(const char *newval, void *extra) { } ================================================ FILE: test/src/loader/osm_init.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/namespace.h> #include <tcop/utility.h> #include "compat/compat.h" #include "export.h" #include "loader/lwlocks.h" #ifdef PG_MODULE_MAGIC PG_MODULE_MAGIC; #endif static ProcessUtility_hook_type prev_ProcessUtility_hook; static void osm_process_utility_hook(PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *qc); #if PG16_LT extern void PGDLLEXPORT _PG_init(void); #endif void _PG_init(void) { elog(WARNING, "OSM-%s _PG_init", OSM_VERSION_MOD); void **osm_lock_pointer = find_rendezvous_variable(RENDEZVOUS_OSM_PARALLEL_LWLOCK); if (osm_lock_pointer != NULL) { elog(WARNING, "got lwlock osm lock"); } else { elog(WARNING, "NO lwlock osm lock"); } prev_ProcessUtility_hook = ProcessUtility_hook; ProcessUtility_hook = osm_process_utility_hook; } TS_FUNCTION_INFO_V1(ts_mock_osm); Datum ts_mock_osm(PG_FUNCTION_ARGS) { elog(WARNING, "OSM-%s mock function call", OSM_VERSION_MOD); PG_RETURN_VOID(); } static void osm_process_utility_hook(PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *qc) { if (nodeTag(pstmt->utilityStmt) == T_DropStmt) { DropStmt *stmt = (DropStmt *) pstmt->utilityStmt; if (stmt->removeType == OBJECT_TABLE) { ListCell *lc; foreach (lc, stmt->objects) { RangeVar *relation = makeRangeVarFromNameList(lfirst(lc)); if (relation != NULL) { Oid relid = RangeVarGetRelid(relation, NoLock, true); elog(NOTICE, "OSM-%s got DROP TABLE '%s'", OSM_VERSION_MOD, get_rel_name(relid)); } } } } if (prev_ProcessUtility_hook) prev_ProcessUtility_hook(pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); } ================================================ FILE: test/src/metadata.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <string.h> #include <unistd.h> #include <postgres.h> #include <fmgr.h> #include <utils/builtins.h> #include "export.h" #include "ts_catalog/metadata.h" TS_FUNCTION_INFO_V1(ts_test_uuid); TS_FUNCTION_INFO_V1(ts_test_exported_uuid); TS_FUNCTION_INFO_V1(ts_test_install_timestamp); Datum ts_test_uuid(PG_FUNCTION_ARGS) { PG_RETURN_DATUM(ts_metadata_get_uuid()); } Datum ts_test_exported_uuid(PG_FUNCTION_ARGS) { PG_RETURN_DATUM(ts_metadata_get_exported_uuid()); } Datum ts_test_install_timestamp(PG_FUNCTION_ARGS) { PG_RETURN_DATUM(ts_metadata_get_install_timestamp()); } ================================================ FILE: test/src/net/CMakeLists.txt ================================================ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/test_conn.c ${CMAKE_CURRENT_SOURCE_DIR}/test_http.c ${CMAKE_CURRENT_SOURCE_DIR}/conn_mock.c) target_sources(${TESTS_LIB_NAME} PRIVATE ${SOURCES}) target_include_directories(${TESTS_LIB_NAME} PRIVATE ${PROJECT_SOURCE_DIR}/src/net) ================================================ FILE: test/src/net/conn_mock.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <unistd.h> #include <postgres.h> #include <errno.h> #include <fcntl.h> #include <stdio.h> #include <stdlib.h> #include <time.h> #include "conn_internal.h" #include "conn_mock.h" #define MOCK_MAX_BUF_SIZE 1024 typedef struct MockConnection { Connection conn; char recv_buf[MOCK_MAX_BUF_SIZE]; size_t recv_buf_offset; size_t recv_buf_len; } MockConnection; static int mock_connect(Connection *conn, const char *host, const char *servname, int port) { return 0; } static void mock_close(Connection *conn) { } static ssize_t mock_write(Connection *conn, const char *buf, size_t writelen) { return writelen; } static ssize_t mock_read(Connection *conn, char *buf, size_t readlen) { size_t bytes_to_read = 0; size_t max = readlen; MockConnection *mock = (MockConnection *) conn; if (mock->recv_buf_offset >= mock->recv_buf_len) return 0; if (max >= mock->recv_buf_len - mock->recv_buf_offset) max = mock->recv_buf_len - mock->recv_buf_offset; /* Now read a random amount */ while (bytes_to_read == 0) { bytes_to_read = rand() % (max + 1); } memcpy(buf, mock->recv_buf + mock->recv_buf_offset, bytes_to_read); mock->recv_buf_offset += bytes_to_read; return bytes_to_read; } static int mock_init(Connection *conn) { srand(time(0)); return 0; } static ConnOps mock_ops = { .size = sizeof(MockConnection), .init = mock_init, .connect = mock_connect, .close = mock_close, .write = mock_write, .read = mock_read, }; ssize_t ts_connection_mock_set_recv_buf(Connection *conn, char *buf, size_t buf_len) { MockConnection *mock = (MockConnection *) conn; if (buf_len > MOCK_MAX_BUF_SIZE) return -1; memcpy(mock->recv_buf, buf, buf_len); mock->recv_buf_len = buf_len; return mock->recv_buf_len; } extern void _conn_mock_init(void); extern void _conn_mock_fini(void); void _conn_mock_init(void) { ts_connection_register(CONNECTION_MOCK, &mock_ops); } void _conn_mock_fini(void) { } ================================================ FILE: test/src/net/conn_mock.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <sys/socket.h> typedef struct Connection Connection; extern ssize_t ts_connection_mock_set_recv_buf(Connection *conn, char *buf, size_t buf_len); ================================================ FILE: test/src/net/test_conn.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <string.h> #include <unistd.h> #include <postgres.h> #include <fmgr.h> #include "compat/compat.h" #include "config.h" #include "net/conn.h" #define MAX_RESULT_SIZE 2048 TS_FUNCTION_INFO_V1(ts_test_conn); Datum ts_test_conn(PG_FUNCTION_ARGS) { char response[MAX_RESULT_SIZE]; Connection *conn; int ret; int port = 80; #ifdef TS_USE_OPENSSL int ssl_port = 443; #endif char *host = "httpbin.org"; /* Test connection_init/destroy */ conn = ts_connection_create(CONNECTION_PLAIN); ts_connection_destroy(conn); /* Check pass NULL won't crash */ ts_connection_destroy(NULL); /* Check that delays on the socket are properly handled */ conn = ts_connection_create(CONNECTION_PLAIN); ts_connection_set_timeout_millis(conn, 200); /* This is a brittle assert function because we might not necessarily have */ /* connectivity on the server running this test? */ ret = ts_connection_connect(conn, host, NULL, port); if (ret < 0) elog(ERROR, "%s", ts_connection_get_and_clear_error(conn)); /* should timeout */ ret = ts_connection_read(conn, response, 1); if (ret == 0) elog(ERROR, "Expected timeout"); ts_connection_close(conn); ts_connection_destroy(conn); #ifdef TS_USE_OPENSSL /* Now test ssl_ops */ conn = ts_connection_create(CONNECTION_SSL); ts_connection_set_timeout_millis(conn, 200); ret = ts_connection_connect(conn, host, NULL, ssl_port); if (ret < 0) elog(ERROR, "%s", ts_connection_get_and_clear_error(conn)); ret = ts_connection_read(conn, response, 1); if (ret == 0) elog(ERROR, "Expected timeout"); ts_connection_close(conn); ts_connection_destroy(conn); #endif PG_RETURN_NULL(); } ================================================ FILE: test/src/net/test_http.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <string.h> #include <postgres.h> #include <fmgr.h> #include <stdlib.h> #include <time.h> #include "export.h" #include "net/http.h" #include "test_utils.h" #define MAX_REQUEST_SIZE 4096 /* Tests for auxiliary HttpResponseState functions in http_parsing.h */ static const char *TEST_RESPONSES[] = { "HTTP/1.1 200 OK\r\n" "Content-Type: application/json; charset=utf-8\r\n" "Date: Thu, 12 Jul 2018 18:33:04 GMT\r\n" "ETag: W/\"e-upYEWCL+q6R/++2nWHz5b76hBgo\"\r\n" "Server: nginx " "Vary: Accept-Encoding\r\n" "Content-Length: 14\r\n" "Connection: Close\r\n\r\n" "{\"status\":200}", "HTTP/1.1 200 OK\r\n" "Content-Length: 14\r\n" "Content-Type: application/json; charset=utf-8\r\n" "Date: Thu, 12 Jul 2018 18:33:04 GMT\r\n" "ETag: W/\"e-upYEWCL+q6R/++2nWHz5b76hBgo\"\r\n" "Vary: Accept-Encoding\r\n\r\n" "{\"status\":200}", "HTTP/1.1 200 OK\r\n" "Content-Length: 14\r\n" "Connection: Close\r\n\r\n" "{\"status\":200}", "HTTP/1.1 201 OK\r\n" "Date: Thu, 12 Jul 2018 18:33:04 GMT\r\n" "Content-Length: 14\r\n" "ETag: W/\"e-upYEWCL+q6R/++2nWHz5b76hBgo\"\r\n" "Connection: Close\r\n\r\n" "{\"status\":201}", }; static const char *const BAD_RESPONSES[] = { "HTTP/1.1 200 OK\r\n" "Content-Type: application/json; charset=utf-8\r\n" "Date: Thu, 12 Jul 2018 18:33:04 GMT\r\n" "ETag: W/\"e-upYEWCL+q6R/++2nWHz5b76hBgo\"\r\n" "Connection: Close\r\n" "{\"status\":200}", "Content-Length: 14\r\n" "{\"status\":200}", "Content-Length: 14\r\n" "HTTP/1.1 404 Not Found\r\n" "Connection: Close\r\n\r\n" "{\"status\":404}", NULL }; static size_t TEST_LENGTHS[] = { 14, 14, 14, 14 }; static const char *MESSAGE_BODY[] = { "{\"status\":200}", "{\"status\":200}", "{\"status\":200}", "{\"status\":201}" }; TS_FUNCTION_INFO_V1(ts_test_http_parsing); TS_FUNCTION_INFO_V1(ts_test_http_parsing_full); TS_FUNCTION_INFO_V1(ts_test_http_request_build); static int num_test_strings() { return sizeof(TEST_LENGTHS) / sizeof(TEST_LENGTHS[0]); } /* Check we can successfully parse partial by well-formed HTTP responses */ Datum ts_test_http_parsing(PG_FUNCTION_ARGS) { int num_iterations = PG_GETARG_INT32(0); int i, j; size_t bytes; srand(time(0)); for (j = 0; j < num_iterations; j++) { for (i = 0; i < num_test_strings(); i++) { HttpResponseState *state = ts_http_response_state_create(); bool success; ssize_t bufsize = 0; char *buf; bytes = rand() % (strlen(TEST_RESPONSES[i]) + 1); buf = ts_http_response_state_next_buffer(state, &bufsize); TestAssertTrue(bufsize >= (ssize_t) bytes); /* Copy part of the message into the parsing state */ memcpy(buf, TEST_RESPONSES[i], bytes); /* Now do the parse */ success = ts_http_response_state_parse(state, bytes); TestAssertTrue(success); if (!success) elog(ERROR, "could not parse http state"); success = ts_http_response_state_is_done(state); TestAssertTrue(bytes < strlen(TEST_RESPONSES[i]) ? !success : success); ts_http_response_state_destroy(state); } } PG_RETURN_NULL(); } /* Check we can successfully parse full, well-formed HTTP response AND * successfully find error with full, poorly-formed HTTP responses */ Datum ts_test_http_parsing_full(PG_FUNCTION_ARGS) { int i; size_t bytes; srand(time(0)); for (i = 0; i < num_test_strings(); i++) { HttpResponseState *state = ts_http_response_state_create(); ssize_t bufsize = 0; char *buf; int cmp; buf = ts_http_response_state_next_buffer(state, &bufsize); bytes = strlen(TEST_RESPONSES[i]); TestAssertTrue(bufsize >= (ssize_t) bytes); /* Copy all of the message into the parsing state */ memcpy(buf, TEST_RESPONSES[i], bytes); /* Now do the parse */ TestAssertTrue(ts_http_response_state_parse(state, bytes)); TestAssertTrue(ts_http_response_state_is_done(state)); TestAssertTrue(ts_http_response_state_content_length(state) == TEST_LENGTHS[i]); /* Make sure we read the right message body */ cmp = !strncmp(MESSAGE_BODY[i], ts_http_response_state_body_start(state), ts_http_response_state_content_length(state)); TestAssertTrue(cmp); if (!cmp) elog(ERROR, "bad message"); ts_http_response_state_destroy(state); } /* Now do the bad responses */ for (i = 0; i < 3; i++) { HttpResponseState *state = ts_http_response_state_create(); ssize_t bufsize = 0; char *buf; buf = ts_http_response_state_next_buffer(state, &bufsize); bytes = strlen(BAD_RESPONSES[i]); TestAssertTrue(bufsize >= (ssize_t) bytes); memcpy(buf, BAD_RESPONSES[i], bytes); TestAssertTrue(!ts_http_response_state_parse(state, bytes) || !ts_http_response_state_valid_status(state)); ts_http_response_state_destroy(state); } PG_RETURN_NULL(); } Datum ts_test_http_request_build(PG_FUNCTION_ARGS) { const char *serialized; size_t request_len; const char *expected_response = "GET /v1/alerts HTTP/1.1\r\n" "Host: herp.com\r\nContent-Length: 0\r\n\r\n"; char *host = "herp.com"; HttpRequest *req = ts_http_request_create(HTTP_GET); int cmp_res; ts_http_request_set_uri(req, "/v1/alerts"); ts_http_request_set_version(req, HTTP_VERSION_11); ts_http_request_set_header(req, HTTP_CONTENT_LENGTH, "0"); ts_http_request_set_header(req, HTTP_HOST, host); serialized = ts_http_request_build(req, &request_len); cmp_res = !strncmp(expected_response, serialized, request_len); TestAssertTrue(cmp_res); if (!cmp_res) elog(ERROR, "bad response"); ts_http_request_destroy(req); expected_response = "GET /tmp/path/to/uri HTTP/1.0\r\n" "Content-Length: 0\r\nHost: herp.com\r\nContent-Type: application/json\r\n\r\n"; req = ts_http_request_create(HTTP_GET); ts_http_request_set_uri(req, "/tmp/path/to/uri"); ts_http_request_set_version(req, HTTP_VERSION_10); ts_http_request_set_header(req, HTTP_CONTENT_TYPE, "application/json"); ts_http_request_set_header(req, HTTP_HOST, host); ts_http_request_set_header(req, HTTP_CONTENT_LENGTH, "0"); serialized = ts_http_request_build(req, &request_len); TestAssertTrue(!strncmp(expected_response, serialized, request_len)); ts_http_request_destroy(req); expected_response = "POST /tmp/status/1234 HTTP/1.1\r\n" "Content-Length: 0\r\nHost: herp.com\r\n\r\n"; req = ts_http_request_create(HTTP_POST); ts_http_request_set_uri(req, "/tmp/status/1234"); ts_http_request_set_version(req, HTTP_VERSION_11); ts_http_request_set_header(req, HTTP_HOST, host); ts_http_request_set_header(req, HTTP_CONTENT_LENGTH, "0"); serialized = ts_http_request_build(req, &request_len); TestAssertTrue(!strncmp(expected_response, serialized, request_len)); ts_http_request_destroy(req); /* Check that content-length checking works */ req = ts_http_request_create(HTTP_POST); ts_http_request_set_uri(req, "/tmp/status/1234"); ts_http_request_set_version(req, HTTP_VERSION_11); ts_http_request_set_header(req, HTTP_HOST, host); ts_http_request_set_header(req, HTTP_CONTENT_LENGTH, "9"); TestAssertTrue(!ts_http_request_build(req, &request_len)); ts_http_request_destroy(req); PG_RETURN_NULL(); } ================================================ FILE: test/src/symbol_conflict.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <fmgr.h> #include <utils/builtins.h> #include "export.h" #define STR_EXPAND(x) #x #define STR(x) STR_EXPAND(x) #define FUNC_EXPAND(prefix, name) prefix##_##name #define FUNC(prefix, name) FUNC_EXPAND(prefix, name) /* Function with conflicting name when included in multiple modules */ extern const char *test_symbol_conflict(void); const char * test_symbol_conflict(void) { return "hello from " STR(MODULE_NAME); } TS_FUNCTION_INFO_V1(FUNC(MODULE_NAME, hello)); Datum FUNC(MODULE_NAME, hello)(PG_FUNCTION_ARGS) { PG_RETURN_TEXT_P(cstring_to_text(test_symbol_conflict())); } ================================================ FILE: test/src/telemetry/CMakeLists.txt ================================================ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/test_privacy.c ${CMAKE_CURRENT_SOURCE_DIR}/test_telemetry.c) target_sources(${TESTS_LIB_NAME} PRIVATE ${SOURCES}) ================================================ FILE: test/src/telemetry/test_privacy.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <string.h> #include <unistd.h> #include <postgres.h> #include <fmgr.h> #include <miscadmin.h> #include "compat/compat.h" #include "telemetry/telemetry.h" #include "uuid.h" TS_FUNCTION_INFO_V1(ts_test_privacy); Datum ts_test_privacy(PG_FUNCTION_ARGS) { /* This test should only run when timescaledb.telemetry_level=off */ PG_RETURN_BOOL(ts_telemetry_main("", "", "")); } ================================================ FILE: test/src/telemetry/test_telemetry.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/htup_details.h> #include <fmgr.h> #include <funcapi.h> #include <utils/builtins.h> #include <utils/jsonb.h> #include "compat/compat.h" #include "config.h" #include "export.h" #include "net/http.h" #include "telemetry/telemetry.h" #ifdef TS_DEBUG #include "net/conn_mock.h" #endif #define HTTPS_PORT 443 #define TEST_ENDPOINT "postman-echo.com" /* Since we rely on an external service to test request statuses, we should retry * a few times if we are not getting the correct response. This should reduce * test flakiness. */ #define INVALID_RESPONSE_RETRIES 5 TS_FUNCTION_INFO_V1(ts_test_status); TS_FUNCTION_INFO_V1(ts_test_status_ssl); TS_FUNCTION_INFO_V1(ts_test_status_mock); TS_FUNCTION_INFO_V1(ts_test_telemetry_main_conn); TS_FUNCTION_INFO_V1(ts_test_telemetry); TS_FUNCTION_INFO_V1(ts_test_check_version_response); #ifdef TS_DEBUG static char *test_string; #endif static HttpRequest * build_request(int status) { HttpRequest *req = ts_http_request_create(HTTP_GET); char uri[20]; snprintf(uri, 20, "/status/%d", status); ts_http_request_set_uri(req, uri); ts_http_request_set_version(req, HTTP_VERSION_10); ts_http_request_set_header(req, HTTP_HOST, TEST_ENDPOINT); ts_http_request_set_header(req, HTTP_CONTENT_LENGTH, "0"); return req; } static Datum test_factory(ConnectionType type, int status, char *host, int port) { Connection *conn; HttpRequest *req; HttpResponseState *rsp = NULL; HttpError err; Datum json; conn = ts_connection_create(type); if (conn == NULL) return CStringGetTextDatum("could not initialize a connection"); if (ts_connection_connect(conn, host, NULL, port) < 0) { const char *err_msg = ts_connection_get_and_clear_error(conn); ts_connection_destroy(conn); elog(ERROR, "connection error: %s", err_msg); } #ifdef TS_DEBUG if (type == CONNECTION_MOCK) ts_connection_mock_set_recv_buf(conn, test_string, strlen(test_string)); #endif for (int retries = 0; retries < INVALID_RESPONSE_RETRIES; retries++) { req = build_request(status); rsp = ts_http_response_state_create(); err = ts_http_send_and_recv(conn, req, rsp); ts_http_request_destroy(req); /* We are mocking the connection, no need to retry */ if (type == CONNECTION_MOCK) break; /* Could be a transient HTTP error, lets try again */ if (err != HTTP_ERROR_NONE) continue; /* Got what we want, no need to retry */ if (ts_http_response_state_valid_status(rsp) || ts_http_response_state_status_code(rsp) == status) break; } if (err != HTTP_ERROR_NONE) { ereport(ERROR, (errcode(ERRCODE_IO_ERROR), errmsg("%s", ts_http_strerror(err)))); } ts_connection_destroy(conn); if (!ts_http_response_state_valid_status(rsp)) ereport(ERROR, (errcode(ERRCODE_IO_ERROR), errmsg("endpoint sent back unexpected HTTP status: %d", ts_http_response_state_status_code(rsp)))); json = DirectFunctionCall1(jsonb_in, CStringGetDatum(ts_http_response_state_body_start(rsp))); ts_http_response_state_destroy(rsp); return json; } /* Test ssl_ops */ Datum ts_test_status_ssl(PG_FUNCTION_ARGS) { int status = PG_GETARG_INT32(0); #ifdef TS_USE_OPENSSL return test_factory(CONNECTION_SSL, status, TEST_ENDPOINT, HTTPS_PORT); #else char buf[128] = { '\0' }; if (status / 100 != 2) ereport(ERROR, (errcode(ERRCODE_IO_ERROR), errmsg("endpoint sent back unexpected HTTP status: %d", status))); snprintf(buf, sizeof(buf) - 1, "{\"status\":%d}", status); PG_RETURN_JSONB_P(DatumGetJsonbP(DirectFunctionCall1(jsonb_in, CStringGetDatum(buf)))); #endif } /* Test default_ops */ Datum ts_test_status(PG_FUNCTION_ARGS) { int port = 80; int status = PG_GETARG_INT32(0); PG_RETURN_DATUM(test_factory(CONNECTION_PLAIN, status, TEST_ENDPOINT, port)); } #ifdef TS_DEBUG /* Test mock_ops */ Datum ts_test_status_mock(PG_FUNCTION_ARGS) { int port = 80; text *arg1 = PG_GETARG_TEXT_P(0); test_string = text_to_cstring(arg1); PG_RETURN_DATUM(test_factory(CONNECTION_MOCK, 123, TEST_ENDPOINT, port)); } #endif TS_FUNCTION_INFO_V1(ts_test_validate_server_version); Datum ts_test_validate_server_version(PG_FUNCTION_ARGS) { text *response = PG_GETARG_TEXT_P(0); VersionResult result; if (ts_validate_server_version(text_to_cstring(response), &result)) PG_RETURN_TEXT_P(cstring_to_text(result.versionstr)); PG_RETURN_NULL(); } Datum ts_test_check_version_response(PG_FUNCTION_ARGS) { text *response = PG_GETARG_TEXT_P(0); const char *volatile json = text_to_cstring(response); PG_TRY(); { ts_check_version_response(json); } PG_CATCH(); { /* If the response is malformed, ts_check_version_response() will * throw an error, so we capture the error here. The error message * contains the function pointer, which will vary between test runs, * so we do not re-throw the error here and instead print our own. */ ereport(ERROR, (errcode(ERRCODE_DATA_EXCEPTION), errmsg("malformed telemetry response body"))); } PG_END_TRY(); PG_RETURN_VOID(); } /* Try to get the telemetry function to handle errors. Never connect to the * actual endpoint. Only test cases that will result in connection errors. */ Datum ts_test_telemetry_main_conn(PG_FUNCTION_ARGS) { text *host = PG_GETARG_TEXT_P(0); text *path = PG_GETARG_TEXT_P(1); const char *scheme; #ifdef TS_USE_OPENSSL scheme = "https"; #else scheme = "http"; #endif PG_RETURN_BOOL(ts_telemetry_main(text_to_cstring(host), text_to_cstring(path), scheme)); } Datum ts_test_telemetry(PG_FUNCTION_ARGS) { Connection *conn; ConnectionType conntype; HttpRequest *req; HttpResponseState *rsp; HttpError err; Datum json_body; const char *host = PG_ARGISNULL(0) ? TELEMETRY_HOST : text_to_cstring(PG_GETARG_TEXT_P(0)); const char *servname = PG_ARGISNULL(1) ? "https" : text_to_cstring(PG_GETARG_TEXT_P(1)); int port = PG_ARGISNULL(2) ? 0 : PG_GETARG_INT32(2); int ret; if (PG_NARGS() > 3) elog(ERROR, "invalid number of arguments"); if (strcmp("http", servname) == 0) conntype = CONNECTION_PLAIN; else if (strcmp("https", servname) == 0) conntype = CONNECTION_SSL; else elog(ERROR, "invalid service type '%s'", servname); conn = ts_connection_create(conntype); if (conn == NULL) elog(ERROR, "could not create telemetry connection"); ret = ts_connection_connect(conn, host, servname, port); if (ret < 0) { const char *errstr = ts_connection_get_and_clear_error(conn); ts_connection_destroy(conn); ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not make a connection to %s://%s", servname, host), errdetail("%s", errstr))); } req = ts_build_version_request(host, TELEMETRY_PATH); rsp = ts_http_response_state_create(); err = ts_http_send_and_recv(conn, req, rsp); ts_http_request_destroy(req); ts_connection_destroy(conn); if (err != HTTP_ERROR_NONE) { ts_http_response_state_destroy(rsp); ereport(ERROR, (errcode(ERRCODE_IO_ERROR), errmsg("telemetry error: %s", ts_http_strerror(err)))); } if (!ts_http_response_state_valid_status(rsp)) { ts_http_response_state_destroy(rsp); ereport(ERROR, (errcode(ERRCODE_IO_ERROR), errmsg("telemetry got unexpected HTTP response status: %d", ts_http_response_state_status_code(rsp)))); } json_body = DirectFunctionCall1(jsonb_in, CStringGetDatum(ts_http_response_state_body_start(rsp))); ts_http_response_state_destroy(rsp); PG_RETURN_DATUM(json_body); } ================================================ FILE: test/src/test_bmslist_utils.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include "bmslist_utils.h" #include "test_utils.h" #include <fmgr.h> #include <funcapi.h> #include <ts_catalog/compression_settings.h> static void test_empty_bmslist_contains_items(void) { TsBmsList bmslist = ts_bmslist_create(); int items[] = { 1, 2, 3 }; bool result = ts_bmslist_contains_items(bmslist, items, 3); TestAssertBoolEq(result, false); } static void test_non_empty_bmslist_contains_items(void) { TsBmsList bmslist = ts_bmslist_create(); int items[] = { 1, 2, 3 }; bmslist = ts_bmslist_add_member(bmslist, items, 1); bmslist = ts_bmslist_add_member(bmslist, items, 2); bmslist = ts_bmslist_add_member(bmslist, items, 3); bool result = ts_bmslist_contains_items(bmslist, items, 3); TestAssertBoolEq(result, true); result = ts_bmslist_contains_items(bmslist, items, 2); TestAssertBoolEq(result, true); result = ts_bmslist_contains_items(bmslist, items, 1); TestAssertBoolEq(result, true); items[0] = 4; result = ts_bmslist_contains_items(bmslist, items, 3); TestAssertBoolEq(result, false); result = ts_bmslist_contains_items(bmslist, items, 2); TestAssertBoolEq(result, false); result = ts_bmslist_contains_items(bmslist, items, 1); TestAssertBoolEq(result, false); ts_bmslist_free(bmslist); } TS_TEST_FN(ts_test_bmslist_utils) { test_empty_bmslist_contains_items(); test_non_empty_bmslist_contains_items(); PG_RETURN_VOID(); } ================================================ FILE: test/src/test_compression_settings.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include "foreach_ptr.h" #include <fmgr.h> #include <funcapi.h> #include <ts_catalog/compression_settings.h> #include <utils/jsonb.h> /* Include test_utils.h after all other headers */ #include <test_utils.h> #define TestAssertParsedCompressionSettingsEqCstring(a, b) \ do \ { \ SparseIndexSettings *a_ps = (a); \ Assert(a_ps != NULL); \ const char *a_i = (a) == NULL ? "<null>" : ts_sparse_index_settings_to_cstring(a_ps); \ const char *b_i = (b) == NULL ? "<null>" : (b); \ if (strcmp(a_i, b_i) != 0) \ TestFailure("(%s == %s)", a_i, b_i); \ } while (0) static void test_alter_table_rename_column_effect_jsonb() { const char *jsonb_str = "[{\"type\": \"bloom\", \"column\": \"big1\", \"source\": \"config\"}, " "{\"type\": \"bloom\", \"column\": \"big2\", \"source\": \"config\"}, " "{\"type\": \"bloom\", \"column\": \"value\", \"source\": \"config\"}, " "{\"type\": \"bloom\", \"column\": [\"value\", \"big1\", \"big2\"], \"source\": " "\"config\"}, " "{\"type\": \"bloom\", \"column\": [\"o\", \"big2\"], \"source\": \"config\"}, " "{\"type\": \"minmax\", \"column\": \"ts\", \"source\": \"orderby\"}]"; const char *jsonb_str_expected = "[{\"type\": \"bloom\", \"column\": \"big1\", \"source\": \"config\"}, " "{\"type\": \"bloom\", \"column\": \"xxl\", \"source\": \"config\"}, " "{\"type\": \"bloom\", \"column\": \"value\", \"source\": \"config\"}, " "{\"type\": \"bloom\", \"column\": [\"value\", \"big1\", \"xxl\"], \"source\": " "\"config\"}, " "{\"type\": \"bloom\", \"column\": [\"o\", \"xxl\"], \"source\": \"config\"}, " "{\"type\": \"minmax\", \"column\": \"ts\", \"source\": \"orderby\"}]"; Jsonb *jb = cstring_to_jsonb(jsonb_str); SparseIndexSettings *parsed_settings = ts_convert_to_sparse_index_settings(jb); TestAssertInt64Eq(list_length(parsed_settings->objects), 6); foreach_ptr(SparseIndexSettingsObject, obj, parsed_settings->objects) { Assert(obj != NULL); TestAssertInt64Eq(list_length(obj->pairs), 3); foreach_ptr(SparseIndexSettingsPair, pair, obj->pairs) { Assert(pair != NULL); if (strcmp(pair->key, "column") != 0) { continue; } ListCell *value_cell = NULL; foreach (value_cell, pair->values) { const char *value = (const char *) lfirst(value_cell); Assert(value != NULL); if (strcmp(value, "big2") == 0) { /* Replace the value with the new one, allocate from the parsed settings context */ value_cell->ptr_value = ts_sparse_index_settings_pstrdup(parsed_settings, "xxl"); } } } } Jsonb *result = ts_convert_from_sparse_index_settings(parsed_settings); TestAssertJsonbEqCstring(result, jsonb_str_expected); TestAssertParsedCompressionSettingsEqCstring(parsed_settings, jsonb_str_expected); /* test the per column settings */ List *per_column_settings = ts_get_per_column_compression_settings(parsed_settings); Assert(per_column_settings != NIL); TestAssertInt64Eq(list_length(per_column_settings), 5); PerColumnCompressionSettings *per_column_setting = NULL; { per_column_setting = (PerColumnCompressionSettings *) lfirst(list_head(per_column_settings)); Assert(per_column_setting != NULL); TestAssertCStringEq(per_column_setting->column_name, "big1"); TestAssertInt64Eq(per_column_setting->single_bloom_obj_id, 0); TestAssertInt64Eq(per_column_setting->minmax_obj_id, -1); /* only part of a single composite bloom index */ TestAssertInt64Eq(bms_num_members(per_column_setting->composite_bloom_index_obj_ids), 1); TestAssertBoolEq(bms_is_member(3, per_column_setting->composite_bloom_index_obj_ids), true); } { per_column_setting = (PerColumnCompressionSettings *) lsecond(per_column_settings); Assert(per_column_setting != NULL); TestAssertCStringEq(per_column_setting->column_name, "xxl"); TestAssertInt64Eq(per_column_setting->single_bloom_obj_id, 1); TestAssertInt64Eq(per_column_setting->minmax_obj_id, -1); /* part of two composite bloom indices */ TestAssertInt64Eq(bms_num_members(per_column_setting->composite_bloom_index_obj_ids), 2); TestAssertBoolEq(bms_is_member(3, per_column_setting->composite_bloom_index_obj_ids), true); TestAssertBoolEq(bms_is_member(4, per_column_setting->composite_bloom_index_obj_ids), true); } { per_column_setting = (PerColumnCompressionSettings *) lthird(per_column_settings); Assert(per_column_setting != NULL); TestAssertCStringEq(per_column_setting->column_name, "value"); TestAssertInt64Eq(per_column_setting->single_bloom_obj_id, 2); TestAssertInt64Eq(per_column_setting->minmax_obj_id, -1); /* part of a single composite bloom index */ TestAssertInt64Eq(bms_num_members(per_column_setting->composite_bloom_index_obj_ids), 1); TestAssertBoolEq(bms_is_member(3, per_column_setting->composite_bloom_index_obj_ids), true); } { per_column_setting = (PerColumnCompressionSettings *) lfourth(per_column_settings); Assert(per_column_setting != NULL); TestAssertCStringEq(per_column_setting->column_name, "o"); TestAssertInt64Eq(per_column_setting->single_bloom_obj_id, -1); TestAssertInt64Eq(per_column_setting->minmax_obj_id, -1); /* part of a single composite bloom index */ TestAssertInt64Eq(bms_num_members(per_column_setting->composite_bloom_index_obj_ids), 1); TestAssertBoolEq(bms_is_member(4, per_column_setting->composite_bloom_index_obj_ids), true); } { per_column_setting = (PerColumnCompressionSettings *) lfifth(per_column_settings); Assert(per_column_setting != NULL); TestAssertCStringEq(per_column_setting->column_name, "ts"); TestAssertInt64Eq(per_column_setting->single_bloom_obj_id, -1); TestAssertInt64Eq(per_column_setting->minmax_obj_id, 5); TestAssertPtrEq(per_column_setting->composite_bloom_index_obj_ids, NULL); } pfree(result); ts_free_sparse_index_settings(parsed_settings); pfree(jb); } static void test_alter_table_drop_column_effect_jsonb() { const char *jsonb_str = "[{\"type\": \"bloom\", \"column\": \"big1\", \"source\": \"config\"}, " "{\"type\": \"bloom\", \"column\": \"big2\", \"source\": \"config\"}, " "{\"type\": \"bloom\", \"column\": \"value\", \"source\": \"config\"}, " "{\"type\": \"bloom\", \"column\": [\"value\", \"big1\", \"big2\"], \"source\": " "\"config\"}, " "{\"type\": \"bloom\", \"column\": [\"o\", \"big2\"], \"source\": \"config\"}, " "{\"type\": \"minmax\", \"column\": \"ts\", \"source\": \"orderby\"}]"; const char *jsonb_str_expected = "[{\"type\": \"bloom\", \"column\": \"big1\", \"source\": \"config\"}, " /* DROP: "{\"type\": \"bloom\", \"column\": \"big2\", \"source\": \"config\"}, " */ "{\"type\": \"bloom\", \"column\": \"value\", \"source\": \"config\"}, " /* DROP: "{\"type\": \"bloom\", \"column\": [\"value\", \"big1\", \"big2\"], \"source\": \"config\"}, */ /* DROP: "{\"type\": \"bloom\", \"column\": [\"o\", \"big2\"], \"source\": \"config\"}, " */ "{\"type\": \"minmax\", \"column\": \"ts\", \"source\": \"orderby\"}]"; Jsonb *jb = cstring_to_jsonb(jsonb_str); SparseIndexSettings *parsed_settings = ts_convert_to_sparse_index_settings(jb); TestAssertInt64Eq(list_length(parsed_settings->objects), 6); ListCell *obj_cell = NULL; foreach (obj_cell, parsed_settings->objects) { SparseIndexSettingsObject *obj = (SparseIndexSettingsObject *) lfirst(obj_cell); Assert(obj != NULL); TestAssertInt64Eq(list_length(obj->pairs), 3); bool to_remove = false; foreach_ptr(SparseIndexSettingsPair, pair, obj->pairs) { Assert(pair != NULL); if (strcmp(pair->key, "column") != 0) { continue; } foreach_ptr(const char, value, pair->values) { Assert(value != NULL); if (strcmp(value, "big2") == 0) { to_remove = true; break; } } if (to_remove) { break; } } if (to_remove) { /* Remove the object from the list of objects */ parsed_settings->objects = foreach_delete_current(parsed_settings->objects, obj_cell); } } TestAssertInt64Eq(list_length(parsed_settings->objects), 3); Jsonb *result = ts_convert_from_sparse_index_settings(parsed_settings); TestAssertJsonbEqCstring(result, jsonb_str_expected); TestAssertParsedCompressionSettingsEqCstring(parsed_settings, jsonb_str_expected); ts_free_sparse_index_settings(parsed_settings); pfree(result); pfree(jb); } static void test_convert_to_sparse_index_settings() { { /* Objects with a single pair are converted to SparseIndexSettings */ Jsonb *jb = cstring_to_jsonb("{\"key\": \"value\"}"); SparseIndexSettings *parsed_settings = ts_convert_to_sparse_index_settings(jb); TestAssertInt64Eq(list_length(parsed_settings->objects), 1); TestAssertParsedCompressionSettingsEqCstring(parsed_settings, "[{\"key\": \"value\"}]"); Jsonb *result = ts_convert_from_sparse_index_settings(parsed_settings); TestAssertJsonbEqCstring(result, "[{\"key\": \"value\"}]"); /* per column should be empty because there is no column and no index type */ List *per_column_settings = ts_get_per_column_compression_settings(parsed_settings); TestAssertPtrEq(per_column_settings, NIL); ts_free_sparse_index_settings(parsed_settings); pfree(jb); pfree(result); } { /* Objects with an array value are converted to SparseIndexSettings */ Jsonb *jb = cstring_to_jsonb("{\"key\": [\"value\", \"value2\"]}"); SparseIndexSettings *parsed_settings = ts_convert_to_sparse_index_settings(jb); TestAssertInt64Eq(list_length(parsed_settings->objects), 1); TestAssertParsedCompressionSettingsEqCstring(parsed_settings, "[{\"key\": [\"value\", \"value2\"]}]"); Jsonb *result = ts_convert_from_sparse_index_settings(parsed_settings); TestAssertJsonbEqCstring(result, "[{\"key\": [\"value\", \"value2\"]}]"); ts_free_sparse_index_settings(parsed_settings); pfree(jb); pfree(result); } { /* Objects with multiple pairs are converted to SparseIndexSettings */ Jsonb *jb = cstring_to_jsonb("{\"key\": [\"value\", \"value2\"], \"key2\": \"value3\"}"); SparseIndexSettings *parsed_settings = ts_convert_to_sparse_index_settings(jb); TestAssertInt64Eq(list_length(parsed_settings->objects), 1); TestAssertParsedCompressionSettingsEqCstring(parsed_settings, "[{\"key\": [\"value\", \"value2\"], " "\"key2\": \"value3\"}]"); Jsonb *result = ts_convert_from_sparse_index_settings(parsed_settings); TestAssertJsonbEqCstring(result, "[{\"key\": [\"value\", \"value2\"], \"key2\": \"value3\"}]"); ts_free_sparse_index_settings(parsed_settings); pfree(jb); pfree(result); } { /* Empty objects are converted to NULL */ Jsonb *jb = cstring_to_jsonb("{}"); SparseIndexSettings *parsed_settings = ts_convert_to_sparse_index_settings(jb); TestAssertPtrEq(parsed_settings, NULL); pfree(jb); } { /* Empty arrays are ignored and converted to NULL */ Jsonb *jb = cstring_to_jsonb("[]"); SparseIndexSettings *parsed_settings = ts_convert_to_sparse_index_settings(jb); TestAssertPtrEq(parsed_settings, NULL); pfree(jb); } { /* Empty objects are ignored */ Jsonb *jb = cstring_to_jsonb("[{}, {\"key\": \"value\"}, {}, {}]"); SparseIndexSettings *parsed_settings = ts_convert_to_sparse_index_settings(jb); TestAssertInt64Eq(list_length(parsed_settings->objects), 1); TestAssertParsedCompressionSettingsEqCstring(parsed_settings, "[{\"key\": \"value\"}]"); Jsonb *result = ts_convert_from_sparse_index_settings(parsed_settings); TestAssertJsonbEqCstring(result, "[{\"key\": \"value\"}]"); ts_free_sparse_index_settings(parsed_settings); pfree(jb); pfree(result); } { /* Unexpected nesting of objects return an error */ Jsonb *jb = cstring_to_jsonb("{\"key\": [{\"key2\": \"value2\"}]}"); TestEnsureError(ts_convert_to_sparse_index_settings(jb)); pfree(jb); } { /* Unexpected nesting of objects return an error */ Jsonb *jb = cstring_to_jsonb("{\"key\": {\"key2\": \"value2\"}}"); TestEnsureError(ts_convert_to_sparse_index_settings(jb)); pfree(jb); } { /* Unexpected nesting of objects return an error */ Jsonb *jb = cstring_to_jsonb("{\"key\": [{\"key2\": \"value2\"}, {\"key3\": \"value3\"}]}"); TestEnsureError(ts_convert_to_sparse_index_settings(jb)); pfree(jb); } { /* Unexpected nesting of arrays return an error */ Jsonb *jb = cstring_to_jsonb("{\"key\": [[\"value2\", \"value3\"]]}"); TestEnsureError(ts_convert_to_sparse_index_settings(jb)); pfree(jb); } { /* Unexpected nesting of arrays return an error */ Jsonb *jb = cstring_to_jsonb("[[{\"key\": [\"value2\", \"value3\"]}]]"); TestEnsureError(ts_convert_to_sparse_index_settings(jb)); pfree(jb); } { /* Unexpected nesting of arrays return an error */ Jsonb *jb = cstring_to_jsonb("[{\"key\": [\"value2\", [\"value3\"]]}]"); TestEnsureError(ts_convert_to_sparse_index_settings(jb)); pfree(jb); } } TS_TEST_FN(ts_test_compression_settings) { test_alter_table_rename_column_effect_jsonb(); test_alter_table_drop_column_effect_jsonb(); test_convert_to_sparse_index_settings(); PG_RETURN_VOID(); } ================================================ FILE: test/src/test_jsonb_utils.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include "jsonb_utils.h" #include "test_utils.h" #include "ts_catalog/compression_settings.h" #include "utils/jsonb.h" #include <fmgr.h> #include <funcapi.h> // Declare jsonb_in explicitly extern Datum jsonb_in(PG_FUNCTION_ARGS); const char * jsonb_to_cstring(Jsonb *jsonb) { StringInfoData buf; initStringInfo(&buf); JsonbToCString(&buf, &jsonb->root, 0); return buf.data; } Jsonb * cstring_to_jsonb(const char *cstring) { Datum jsonb_datum = DirectFunctionCall1(jsonb_in, CStringGetDatum(cstring)); Jsonb *jsonb = DatumGetJsonbP(jsonb_datum); return jsonb; } static void test_get_str_field() { { /* Empty JSONB doesn't have the key */ Jsonb *jb = cstring_to_jsonb("{}"); TestAssertCStringEq(ts_jsonb_get_str_field(jb, "key"), NULL); TestAssertJsonbEqCstring(jb, "{}"); pfree(jb); } { /* JSONB with the key, string value */ Jsonb *jb = cstring_to_jsonb("{ \"key\": \"value\" }"); TestAssertCStringEq(ts_jsonb_get_str_field(jb, "key"), "value"); pfree(jb); } { /* JSONB with the key, integer value */ Jsonb *jb = cstring_to_jsonb("{ \"key\": 1 }"); TestAssertCStringEq(ts_jsonb_get_str_field(jb, "key"), "1"); pfree(jb); } { /* JSONB with the key, empty object value */ Jsonb *jb = cstring_to_jsonb("{ \"key\": {} }"); TestAssertCStringEq(ts_jsonb_get_str_field(jb, "key"), "{}"); pfree(jb); } { /* JSONB with the key, array value */ Jsonb *jb = cstring_to_jsonb("{ \"key\": [1, 2, 3] }"); TestAssertCStringEq(ts_jsonb_get_str_field(jb, "key"), "[1, 2, 3]"); pfree(jb); } } static void test_get_bool_field() { { /* JSONB with missing key */ Jsonb *jb = cstring_to_jsonb("{ \"something_else\": {} }"); bool found; TestAssertBoolEq(ts_jsonb_get_bool_field(jb, "key", &found), false); TestAssertBoolEq(found, false); pfree(jb); } { /* JSONB with the key, true value */ Jsonb *jb = cstring_to_jsonb("{ \"key\": true }"); bool found; TestAssertBoolEq(ts_jsonb_get_bool_field(jb, "key", &found), true); TestAssertBoolEq(found, true); pfree(jb); } { /* JSONB with the key, false value */ Jsonb *jb = cstring_to_jsonb("{ \"key\": false }"); bool found; TestAssertBoolEq(ts_jsonb_get_bool_field(jb, "key", &found), false); TestAssertBoolEq(found, true); pfree(jb); } } static void test_has_key_value_str_field() { { /* JSONB with missing key */ Jsonb *jb = cstring_to_jsonb("{ \"something_else\": {} }"); TestAssertBoolEq(ts_jsonb_has_key_value_str_field(jb, "key", "value"), false); pfree(jb); } { /* JSONB with the key, string value */ Jsonb *jb = cstring_to_jsonb("{ \"key\": \"value\" }"); TestAssertBoolEq(ts_jsonb_has_key_value_str_field(jb, "key", "value"), true); pfree(jb); } { /* JSONB with the key, array value, only string key=value pairs should be matched */ Jsonb *jb = cstring_to_jsonb("{ \"key\": [\"value\", \"value2\"] }"); TestAssertBoolEq(ts_jsonb_has_key_value_str_field(jb, "key", "value"), false); pfree(jb); } { /* JSONB with the key, array value, only string key=value pairs should be matched */ Jsonb *jb = cstring_to_jsonb("{ \"key\": [\"value2\", \"value\"] }"); TestAssertBoolEq(ts_jsonb_has_key_value_str_field(jb, "key", "value"), false); pfree(jb); } { /* Key value pair nested in an object */ Jsonb *jb = cstring_to_jsonb("{ \"key\": [\"value\", \"value2\"], \"x\": {\"y\": \"z\", " "\"key\": \"value\"}, \"z\": [{\"key\": \"value\"}] }"); TestAssertBoolEq(ts_jsonb_has_key_value_str_field(jb, "key", "value"), true); TestAssertBoolEq(ts_jsonb_has_key_value_str_field(jb, "y", "z"), true); TestAssertBoolEq(ts_jsonb_has_key_value_str_field(jb, "z", "value"), false); TestAssertBoolEq(ts_jsonb_has_key_value_str_field(jb, "z", "key"), false); pfree(jb); } } static void test_contains_sparse_index_config() { { /* Single column bloom filter */ CompressionSettings settings = { .fd = { .index = cstring_to_jsonb( "{\"type\": \"bloom\", \"column\": \"big1\", \"source\": \"config\"}") } }; TestAssertBoolEq(ts_contains_sparse_index_config(&settings, "big1", "bloom", false /* skip_column_arrays */), true); TestAssertBoolEq(ts_contains_sparse_index_config(&settings, "big1", "bloom", true /* skip_column_arrays */), true); TestAssertBoolEq(ts_contains_sparse_index_config(&settings, "big1", "no_such_type", true /* skip_column_arrays */), false); TestAssertBoolEq(ts_contains_sparse_index_config(&settings, "big1", "no_such_type", false /* skip_column_arrays */), false); TestAssertBoolEq(ts_contains_sparse_index_config(&settings, "non_existent", "bloom", false /* skip_column_arrays */), false); TestAssertBoolEq(ts_contains_sparse_index_config(&settings, "non_existent", "bloom", true /* skip_column_arrays */), false); } { /* Composite bloom filter */ CompressionSettings settings = { .fd = { .index = cstring_to_jsonb( "{\"type\": \"bloom\", \"column\": [\"big1\", " "\"big2\"], \"source\": \"config\"}") } }; TestAssertBoolEq(ts_contains_sparse_index_config(&settings, "big1", "bloom", true /* skip_column_arrays */), false); TestAssertBoolEq(ts_contains_sparse_index_config(&settings, "big1", "bloom", false /* skip_column_arrays */), true); TestAssertBoolEq(ts_contains_sparse_index_config(&settings, "big2", "bloom", true /* skip_column_arrays */), false); TestAssertBoolEq(ts_contains_sparse_index_config(&settings, "big2", "bloom", false /* skip_column_arrays */), true); TestAssertBoolEq(ts_contains_sparse_index_config(&settings, "big1", "no_such_type", true /* skip_column_arrays */), false); TestAssertBoolEq(ts_contains_sparse_index_config(&settings, "big1", "no_such_type", false /* skip_column_arrays */), false); TestAssertBoolEq(ts_contains_sparse_index_config(&settings, "big2", "no_such_type", true /* skip_column_arrays */), false); TestAssertBoolEq(ts_contains_sparse_index_config(&settings, "big2", "no_such_type", false /* skip_column_arrays */), false); TestAssertBoolEq(ts_contains_sparse_index_config(&settings, "non_existent", "bloom", true /* skip_column_arrays */), false); TestAssertBoolEq(ts_contains_sparse_index_config(&settings, "non_existent", "bloom", false /* skip_column_arrays */), false); } } TS_TEST_FN(ts_test_jsonb_utils) { test_get_str_field(); test_get_bool_field(); test_has_key_value_str_field(); test_contains_sparse_index_config(); PG_RETURN_VOID(); } ================================================ FILE: test/src/test_scanner.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include "chunk.h" #include "scan_iterator.h" #include "scanner.h" #include "test_utils.h" TS_TEST_FN(ts_test_scanner) { ScanIterator it; Relation chunkrel; int32 chunk_id[2] = { -1, -1 }; size_t i = 0; /* Test pre-open relation */ it = ts_chunk_scan_iterator_create(CurrentMemoryContext); chunkrel = table_open(it.ctx.table, AccessShareLock); it.ctx.tablerel = chunkrel; /* Explicit start scan to test that we can call it twice without * issue. The loop will also call it */ ts_scan_iterator_start_scan(&it); ts_scanner_foreach(&it) { TupleInfo *ti = ts_scan_iterator_tuple_info(&it); FormData_chunk fd; ts_chunk_formdata_fill(&fd, ti); elog(NOTICE, "1. Scan: \"%s.%s\"", NameStr(fd.schema_name), NameStr(fd.table_name)); if (i < lengthof(chunk_id) && chunk_id[i] == -1) { chunk_id[i] = fd.id; i++; } } ts_scan_iterator_end(&it); /* Add a chunk filter and scan again */ ts_scan_iterator_scan_key_init(&it, Anum_chunk_idx_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(chunk_id[0])); ts_scanner_foreach(&it) { TupleInfo *ti = ts_scan_iterator_tuple_info(&it); FormData_chunk fd; ts_chunk_formdata_fill(&fd, ti); elog(NOTICE, "2. Scan with filter: \"%s.%s\"", NameStr(fd.schema_name), NameStr(fd.table_name)); } /* Rescan */ ts_scan_iterator_scan_key_reset(&it); ts_scan_iterator_scan_key_init(&it, Anum_chunk_idx_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(chunk_id[1])); ts_scan_iterator_rescan(&it); ts_scanner_foreach(&it) { TupleInfo *ti = ts_scan_iterator_tuple_info(&it); FormData_chunk fd; ts_chunk_formdata_fill(&fd, ti); elog(NOTICE, "3. ReScan: \"%s.%s\"", NameStr(fd.schema_name), NameStr(fd.table_name)); } ts_scan_iterator_end(&it); table_close(chunkrel, AccessShareLock); /* Do another scan, but an index scan this time */ it.ctx.tablerel = NULL; it.ctx.index = catalog_get_index(ts_catalog_get(), CHUNK, CHUNK_ID_INDEX); ts_scanner_foreach(&it) { TupleInfo *ti = ts_scan_iterator_tuple_info(&it); FormData_chunk fd; ts_chunk_formdata_fill(&fd, ti); elog(NOTICE, "4. IndexScan: \"%s.%s\"", NameStr(fd.schema_name), NameStr(fd.table_name)); } ts_scan_iterator_close(&it); PG_RETURN_VOID(); } ================================================ FILE: test/src/test_time_to_internal.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_type.h> #include <fmgr.h> #include <utils/date.h> #include "export.h" #include "time_utils.h" #include "utils.h" #include "test_utils.h" TS_FUNCTION_INFO_V1(ts_test_time_to_internal_conversion); TS_FUNCTION_INFO_V1(ts_test_interval_to_internal_conversion); Datum ts_test_time_to_internal_conversion(PG_FUNCTION_ARGS) { int16 i16; int32 i32; int64 i64; /* test integer values */ /* int16 */ for (i16 = -100; i16 < 100; i16++) { TestAssertInt64Eq(i16, ts_time_value_to_internal(Int16GetDatum(i16), INT2OID)); TestAssertInt64Eq(DatumGetInt16(ts_internal_to_time_value(i16, INT2OID)), i16); } TestAssertInt64Eq(PG_INT16_MAX, ts_time_value_to_internal(Int16GetDatum(PG_INT16_MAX), INT2OID)); TestAssertInt64Eq(DatumGetInt16(ts_internal_to_time_value(PG_INT16_MAX, INT2OID)), PG_INT16_MAX); TestAssertInt64Eq(PG_INT16_MIN, ts_time_value_to_internal(Int16GetDatum(PG_INT16_MIN), INT2OID)); TestAssertInt64Eq(DatumGetInt16(ts_internal_to_time_value(PG_INT16_MIN, INT2OID)), PG_INT16_MIN); /* int32 */ for (i32 = -100; i32 < 100; i32++) { TestAssertInt64Eq(i32, ts_time_value_to_internal(Int32GetDatum(i32), INT4OID)); TestAssertInt64Eq(DatumGetInt32(ts_internal_to_time_value(i32, INT4OID)), i32); } TestAssertInt64Eq(PG_INT16_MAX, ts_time_value_to_internal(Int32GetDatum(PG_INT16_MAX), INT4OID)); TestAssertInt64Eq(DatumGetInt32(ts_internal_to_time_value(PG_INT16_MAX, INT4OID)), PG_INT16_MAX); TestAssertInt64Eq(PG_INT32_MAX, ts_time_value_to_internal(Int32GetDatum(PG_INT32_MAX), INT4OID)); TestAssertInt64Eq(DatumGetInt32(ts_internal_to_time_value(PG_INT32_MAX, INT4OID)), PG_INT32_MAX); TestAssertInt64Eq(PG_INT32_MIN, ts_time_value_to_internal(Int32GetDatum(PG_INT32_MIN), INT4OID)); TestAssertInt64Eq(DatumGetInt32(ts_internal_to_time_value(PG_INT32_MIN, INT4OID)), PG_INT32_MIN); /* int64 */ for (i64 = -100; i64 < 100; i64++) { TestAssertInt64Eq(i64, ts_time_value_to_internal(Int64GetDatum(i64), INT8OID)); TestAssertInt64Eq(DatumGetInt64(ts_internal_to_time_value(i64, INT8OID)), i64); } TestAssertInt64Eq(PG_INT16_MIN, ts_time_value_to_internal(Int64GetDatum(PG_INT16_MIN), INT8OID)); TestAssertInt64Eq(DatumGetInt64(ts_internal_to_time_value(PG_INT16_MIN, INT8OID)), PG_INT16_MIN); TestAssertInt64Eq(PG_INT32_MAX, ts_time_value_to_internal(Int64GetDatum(PG_INT32_MAX), INT8OID)); TestAssertInt64Eq(DatumGetInt64(ts_internal_to_time_value(PG_INT32_MAX, INT8OID)), PG_INT32_MAX); TestAssertInt64Eq(PG_INT64_MAX, ts_time_value_to_internal(Int64GetDatum(PG_INT64_MAX), INT8OID)); TestAssertInt64Eq(DatumGetInt64(ts_internal_to_time_value(PG_INT64_MAX, INT8OID)), PG_INT64_MAX); TestAssertInt64Eq(PG_INT64_MIN, ts_time_value_to_internal(Int64GetDatum(PG_INT64_MIN), INT8OID)); TestAssertInt64Eq(DatumGetInt64(ts_internal_to_time_value(PG_INT64_MIN, INT8OID)), PG_INT64_MIN); /* test time values round trip */ /* TIMESTAMP */ for (i64 = -100; i64 < 100; i64++) TestAssertInt64Eq(i64, ts_time_value_to_internal(ts_internal_to_time_value(i64, TIMESTAMPOID), TIMESTAMPOID)); for (i64 = -10000000; i64 < 100000000; i64 += 1000000) TestAssertInt64Eq(i64, ts_time_value_to_internal(ts_internal_to_time_value(i64, TIMESTAMPOID), TIMESTAMPOID)); for (i64 = -1000000000; i64 < 10000000000; i64 += 100000000) TestAssertInt64Eq(i64, ts_time_value_to_internal(ts_internal_to_time_value(i64, TIMESTAMPOID), TIMESTAMPOID)); TestAssertInt64Eq(TS_TIME_NOBEGIN, ts_time_value_to_internal(TimestampGetDatum(DT_NOBEGIN), TIMESTAMPOID)); TestAssertInt64Eq(TS_TIME_NOEND, ts_time_value_to_internal(TimestampGetDatum(DT_NOEND), TIMESTAMPOID)); TestAssertInt64Eq(DT_NOBEGIN, DatumGetTimestamp(ts_internal_to_time_value(PG_INT64_MIN, TIMESTAMPOID))); TestEnsureError(ts_internal_to_time_value(TS_TIMESTAMP_INTERNAL_MIN - 1, TIMESTAMPOID)); TestAssertInt64Eq(TS_TIMESTAMP_INTERNAL_MIN, ts_time_value_to_internal(ts_internal_to_time_value(TS_TIMESTAMP_INTERNAL_MIN, TIMESTAMPOID), TIMESTAMPOID)); TestAssertInt64Eq(DT_NOEND, (ts_time_value_to_internal(ts_internal_to_time_value(PG_INT64_MAX, TIMESTAMPOID), TIMESTAMPOID))); /* TIMESTAMPTZ */ for (i64 = -100; i64 < 100; i64++) TestAssertInt64Eq(i64, ts_time_value_to_internal(ts_internal_to_time_value(i64, TIMESTAMPTZOID), TIMESTAMPTZOID)); for (i64 = -10000000; i64 < 100000000; i64 += 1000000) TestAssertInt64Eq(i64, ts_time_value_to_internal(ts_internal_to_time_value(i64, TIMESTAMPTZOID), TIMESTAMPTZOID)); for (i64 = -1000000000; i64 < 10000000000; i64 += 100000000) TestAssertInt64Eq(i64, ts_time_value_to_internal(ts_internal_to_time_value(i64, TIMESTAMPTZOID), TIMESTAMPTZOID)); TestAssertInt64Eq(TS_TIME_NOBEGIN, ts_time_value_to_internal(TimestampTzGetDatum(DT_NOBEGIN), TIMESTAMPTZOID)); TestAssertInt64Eq(TS_TIME_NOEND, ts_time_value_to_internal(TimestampTzGetDatum(DT_NOEND), TIMESTAMPTZOID)); TestAssertInt64Eq(DT_NOBEGIN, DatumGetTimestampTz(ts_internal_to_time_value(PG_INT64_MIN, TIMESTAMPTZOID))); TestEnsureError(ts_internal_to_time_value(TS_TIMESTAMP_INTERNAL_MIN - 1, TIMESTAMPTZOID)); TestAssertInt64Eq(TS_TIMESTAMP_INTERNAL_MIN, ts_time_value_to_internal(ts_internal_to_time_value(TS_TIMESTAMP_INTERNAL_MIN, TIMESTAMPTZOID), TIMESTAMPTZOID)); TestAssertInt64Eq(DT_NOEND, ts_time_value_to_internal(ts_internal_to_time_value(PG_INT64_MAX, TIMESTAMPTZOID), TIMESTAMPTZOID)); /* DATE */ for (i64 = -100 * USECS_PER_DAY; i64 < 100 * USECS_PER_DAY; i64 += USECS_PER_DAY) TestAssertInt64Eq(i64, ts_time_value_to_internal(ts_internal_to_time_value(i64, DATEOID), DATEOID)); TestAssertInt64Eq(DATEVAL_NOBEGIN, DatumGetDateADT(ts_internal_to_time_value(PG_INT64_MIN, DATEOID))); TestAssertInt64Eq(DATEVAL_NOEND, DatumGetDateADT(ts_internal_to_time_value(PG_INT64_MAX, DATEOID))); TestEnsureError(ts_time_value_to_internal(DateADTGetDatum(DATEVAL_NOBEGIN + 1), DATEOID)); TestEnsureError(ts_time_value_to_internal(DateADTGetDatum(DATEVAL_NOEND - 1), DATEOID)); PG_RETURN_VOID(); }; Datum ts_test_interval_to_internal_conversion(PG_FUNCTION_ARGS) { int16 i16; int32 i32; int64 i64; /* test integer values */ /* int16 */ for (i16 = -100; i16 < 100; i16++) { TestAssertInt64Eq(i16, ts_interval_value_to_internal(Int16GetDatum(i16), INT2OID)); TestAssertInt64Eq(DatumGetInt16(ts_internal_to_interval_value(i16, INT2OID)), i16); } TestAssertInt64Eq(PG_INT16_MAX, ts_interval_value_to_internal(Int16GetDatum(PG_INT16_MAX), INT2OID)); TestAssertInt64Eq(DatumGetInt16(ts_internal_to_interval_value(PG_INT16_MAX, INT2OID)), PG_INT16_MAX); TestAssertInt64Eq(PG_INT16_MIN, ts_interval_value_to_internal(Int16GetDatum(PG_INT16_MIN), INT2OID)); TestAssertInt64Eq(DatumGetInt16(ts_internal_to_interval_value(PG_INT16_MIN, INT2OID)), PG_INT16_MIN); /* int32 */ for (i32 = -100; i32 < 100; i32++) { TestAssertInt64Eq(i32, ts_interval_value_to_internal(Int32GetDatum(i32), INT4OID)); TestAssertInt64Eq(DatumGetInt32(ts_internal_to_interval_value(i32, INT4OID)), i32); } TestAssertInt64Eq(PG_INT16_MAX, ts_interval_value_to_internal(Int32GetDatum(PG_INT16_MAX), INT4OID)); TestAssertInt64Eq(DatumGetInt32(ts_internal_to_interval_value(PG_INT16_MAX, INT4OID)), PG_INT16_MAX); TestAssertInt64Eq(PG_INT32_MAX, ts_interval_value_to_internal(Int32GetDatum(PG_INT32_MAX), INT4OID)); TestAssertInt64Eq(DatumGetInt32(ts_internal_to_interval_value(PG_INT32_MAX, INT4OID)), PG_INT32_MAX); TestAssertInt64Eq(PG_INT32_MIN, ts_interval_value_to_internal(Int32GetDatum(PG_INT32_MIN), INT4OID)); TestAssertInt64Eq(DatumGetInt32(ts_internal_to_interval_value(PG_INT32_MIN, INT4OID)), PG_INT32_MIN); /* int64 */ for (i64 = -100; i64 < 100; i64++) { TestAssertInt64Eq(i64, ts_interval_value_to_internal(Int64GetDatum(i64), INT8OID)); TestAssertInt64Eq(DatumGetInt64(ts_internal_to_interval_value(i64, INT8OID)), i64); } TestAssertInt64Eq(PG_INT16_MIN, ts_interval_value_to_internal(Int64GetDatum(PG_INT16_MIN), INT8OID)); TestAssertInt64Eq(DatumGetInt64(ts_internal_to_interval_value(PG_INT16_MIN, INT8OID)), PG_INT16_MIN); TestAssertInt64Eq(PG_INT32_MAX, ts_interval_value_to_internal(Int64GetDatum(PG_INT32_MAX), INT8OID)); TestAssertInt64Eq(DatumGetInt64(ts_internal_to_interval_value(PG_INT32_MAX, INT8OID)), PG_INT32_MAX); TestAssertInt64Eq(PG_INT64_MAX, ts_interval_value_to_internal(Int64GetDatum(PG_INT64_MAX), INT8OID)); TestAssertInt64Eq(DatumGetInt64(ts_internal_to_interval_value(PG_INT64_MAX, INT8OID)), PG_INT64_MAX); TestAssertInt64Eq(PG_INT64_MIN, ts_interval_value_to_internal(Int64GetDatum(PG_INT64_MIN), INT8OID)); TestAssertInt64Eq(DatumGetInt64(ts_internal_to_interval_value(PG_INT64_MIN, INT8OID)), PG_INT64_MIN); /* INTERVAL */ for (i64 = -100; i64 < 100; i64++) TestAssertInt64Eq(i64, ts_interval_value_to_internal(ts_internal_to_interval_value(i64, INTERVALOID), INTERVALOID)); for (i64 = -10000000; i64 < 100000000; i64 += 1000000) TestAssertInt64Eq(i64, ts_interval_value_to_internal(ts_internal_to_interval_value(i64, INTERVALOID), INTERVALOID)); for (i64 = -1000000000; i64 < 10000000000; i64 += 100000000) TestAssertInt64Eq(i64, ts_interval_value_to_internal(ts_internal_to_interval_value(i64, INTERVALOID), INTERVALOID)); TestAssertInt64Eq(PG_INT64_MIN, ts_interval_value_to_internal(ts_internal_to_interval_value(PG_INT64_MIN, INTERVALOID), INTERVALOID)); TestAssertInt64Eq(PG_INT64_MAX, ts_interval_value_to_internal(ts_internal_to_interval_value(PG_INT64_MAX, INTERVALOID), INTERVALOID)); PG_RETURN_VOID(); } ================================================ FILE: test/src/test_time_utils.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_type.h> #include <fmgr.h> #include <funcapi.h> #include <utils/date.h> #include <time_utils.h> #include <utils.h> #include "test_utils.h" /* * Functions to show TimescaleDB-specific limits of timestamps and dates: */ /* * TIMESTAMP WITH TIME ZONE */ TS_FUNCTION_INFO_V1(ts_timestamptz_pg_min); Datum ts_timestamptz_pg_min(PG_FUNCTION_ARGS) { PG_RETURN_TIMESTAMPTZ(MIN_TIMESTAMP); } TS_FUNCTION_INFO_V1(ts_timestamptz_pg_end); Datum ts_timestamptz_pg_end(PG_FUNCTION_ARGS) { PG_RETURN_TIMESTAMPTZ(END_TIMESTAMP); } TS_FUNCTION_INFO_V1(ts_timestamptz_min); Datum ts_timestamptz_min(PG_FUNCTION_ARGS) { PG_RETURN_TIMESTAMPTZ(TS_TIMESTAMP_MIN); } TS_FUNCTION_INFO_V1(ts_timestamptz_end); Datum ts_timestamptz_end(PG_FUNCTION_ARGS) { PG_RETURN_TIMESTAMPTZ(TS_TIMESTAMP_END); } TS_FUNCTION_INFO_V1(ts_timestamptz_internal_min); Datum ts_timestamptz_internal_min(PG_FUNCTION_ARGS) { PG_RETURN_INT64(TS_TIMESTAMP_INTERNAL_MIN); } TS_FUNCTION_INFO_V1(ts_timestamptz_internal_end); Datum ts_timestamptz_internal_end(PG_FUNCTION_ARGS) { PG_RETURN_INT64(TS_TIMESTAMP_INTERNAL_END); } /* * TIMESTAMP */ TS_FUNCTION_INFO_V1(ts_timestamp_pg_min); Datum ts_timestamp_pg_min(PG_FUNCTION_ARGS) { PG_RETURN_TIMESTAMP(MIN_TIMESTAMP); } TS_FUNCTION_INFO_V1(ts_timestamp_pg_end); Datum ts_timestamp_pg_end(PG_FUNCTION_ARGS) { PG_RETURN_TIMESTAMP(END_TIMESTAMP); } TS_FUNCTION_INFO_V1(ts_timestamp_min); Datum ts_timestamp_min(PG_FUNCTION_ARGS) { PG_RETURN_TIMESTAMP(TS_TIMESTAMP_MIN); } TS_FUNCTION_INFO_V1(ts_timestamp_end); Datum ts_timestamp_end(PG_FUNCTION_ARGS) { PG_RETURN_TIMESTAMP(TS_TIMESTAMP_END); } TS_FUNCTION_INFO_V1(ts_timestamp_internal_min); Datum ts_timestamp_internal_min(PG_FUNCTION_ARGS) { PG_RETURN_INT64(TS_TIMESTAMP_INTERNAL_MIN); } TS_FUNCTION_INFO_V1(ts_timestamp_internal_end); Datum ts_timestamp_internal_end(PG_FUNCTION_ARGS) { PG_RETURN_INT64(TS_TIMESTAMP_INTERNAL_END); } /* * DATE */ TS_FUNCTION_INFO_V1(ts_date_pg_min); Datum ts_date_pg_min(PG_FUNCTION_ARGS) { PG_RETURN_DATEADT(DATETIME_MIN_JULIAN - POSTGRES_EPOCH_JDATE); } TS_FUNCTION_INFO_V1(ts_date_pg_end); Datum ts_date_pg_end(PG_FUNCTION_ARGS) { PG_RETURN_DATEADT(DATE_END_JULIAN - POSTGRES_EPOCH_JDATE); } TS_FUNCTION_INFO_V1(ts_date_min); Datum ts_date_min(PG_FUNCTION_ARGS) { PG_RETURN_DATEADT(TS_DATE_MIN); } TS_FUNCTION_INFO_V1(ts_date_end); Datum ts_date_end(PG_FUNCTION_ARGS) { PG_RETURN_DATEADT(TS_DATE_END); } TS_FUNCTION_INFO_V1(ts_date_internal_min); Datum ts_date_internal_min(PG_FUNCTION_ARGS) { PG_RETURN_INT64(TS_DATE_INTERNAL_MIN); } TS_FUNCTION_INFO_V1(ts_date_internal_end); Datum ts_date_internal_end(PG_FUNCTION_ARGS) { PG_RETURN_INT64(TS_DATE_INTERNAL_END); } TS_FUNCTION_INFO_V1(ts_test_time_utils); Datum ts_test_time_utils(PG_FUNCTION_ARGS) { TestAssertInt64Eq(ts_time_get_min(INT8OID), PG_INT64_MIN); TestAssertInt64Eq(ts_time_get_max(INT8OID), PG_INT64_MAX); TestAssertInt64Eq(ts_time_get_end_or_max(INT8OID), PG_INT64_MAX); TestEnsureError(ts_time_get_end(INT8OID)); TestEnsureError(ts_time_get_nobegin(INT8OID)); TestEnsureError(ts_time_get_noend(INT8OID)); TestAssertInt64Eq(DatumGetInt64(ts_time_datum_get_nobegin_or_min(INT8OID)), PG_INT64_MIN); TestAssertInt64Eq(DatumGetInt64(ts_time_datum_get_min(INT8OID)), PG_INT64_MIN); TestAssertInt64Eq(DatumGetInt64(ts_time_datum_get_max(INT8OID)), PG_INT64_MAX); TestEnsureError(ts_time_datum_get_end(INT8OID)); TestEnsureError(ts_time_datum_get_nobegin(INT8OID)); TestEnsureError(ts_time_datum_get_noend(INT8OID)); TestAssertInt64Eq(ts_time_get_min(INT4OID), PG_INT32_MIN); TestAssertInt64Eq(ts_time_get_max(INT4OID), PG_INT32_MAX); TestAssertInt64Eq(ts_time_get_end_or_max(INT4OID), PG_INT32_MAX); TestEnsureError(ts_time_get_end(INT4OID)); TestEnsureError(ts_time_get_nobegin(INT4OID)); TestEnsureError(ts_time_get_noend(INT4OID)); TestAssertInt64Eq(DatumGetInt32(ts_time_datum_get_nobegin_or_min(INT4OID)), PG_INT32_MIN); TestAssertInt64Eq(DatumGetInt32(ts_time_datum_get_min(INT4OID)), PG_INT32_MIN); TestAssertInt64Eq(DatumGetInt32(ts_time_datum_get_max(INT4OID)), PG_INT32_MAX); TestEnsureError(ts_time_datum_get_end(INT4OID)); TestEnsureError(ts_time_datum_get_nobegin(INT4OID)); TestEnsureError(ts_time_datum_get_noend(INT4OID)); TestAssertInt64Eq(ts_time_get_min(INT2OID), PG_INT16_MIN); TestAssertInt64Eq(ts_time_get_max(INT2OID), PG_INT16_MAX); TestAssertInt64Eq(ts_time_get_end_or_max(INT2OID), PG_INT16_MAX); TestEnsureError(ts_time_get_end(INT2OID)); TestEnsureError(ts_time_get_nobegin(INT2OID)); TestEnsureError(ts_time_get_noend(INT2OID)); TestAssertInt64Eq(DatumGetInt16(ts_time_datum_get_nobegin_or_min(INT2OID)), PG_INT16_MIN); TestAssertInt64Eq(DatumGetInt16(ts_time_datum_get_min(INT2OID)), PG_INT16_MIN); TestAssertInt64Eq(DatumGetInt16(ts_time_datum_get_max(INT2OID)), PG_INT16_MAX); TestEnsureError(ts_time_datum_get_end(INT2OID)); TestEnsureError(ts_time_datum_get_nobegin(INT2OID)); TestEnsureError(ts_time_datum_get_noend(INT2OID)); TestAssertInt64Eq(ts_time_get_min(TIMESTAMPOID), TS_TIMESTAMP_INTERNAL_MIN); TestAssertInt64Eq(ts_time_get_max(TIMESTAMPOID), TS_TIMESTAMP_INTERNAL_MAX); TestAssertInt64Eq(ts_time_get_end(TIMESTAMPOID), TS_TIMESTAMP_INTERNAL_END); TestAssertInt64Eq(ts_time_get_end_or_max(TIMESTAMPOID), TS_TIMESTAMP_INTERNAL_END); TestAssertInt64Eq(ts_time_get_nobegin(TIMESTAMPOID), TS_TIME_NOBEGIN); TestAssertInt64Eq(ts_time_get_noend(TIMESTAMPOID), TS_TIME_NOEND); TestAssertInt64Eq(DatumGetTimestamp(ts_time_datum_get_nobegin_or_min(TIMESTAMPOID)), DT_NOBEGIN); TestAssertInt64Eq(DatumGetTimestamp(ts_time_datum_get_min(TIMESTAMPOID)), TS_TIMESTAMP_MIN); TestAssertInt64Eq(DatumGetTimestamp(ts_time_datum_get_max(TIMESTAMPOID)), TS_TIMESTAMP_MAX); TestAssertInt64Eq(DatumGetTimestamp(ts_time_datum_get_end(TIMESTAMPOID)), TS_TIMESTAMP_END); TestAssertInt64Eq(DatumGetTimestamp(ts_time_datum_get_nobegin(TIMESTAMPOID)), DT_NOBEGIN); TestAssertInt64Eq(DatumGetTimestamp(ts_time_datum_get_noend(TIMESTAMPOID)), DT_NOEND); TestAssertInt64Eq(ts_time_get_min(TIMESTAMPTZOID), TS_TIMESTAMP_INTERNAL_MIN); TestAssertInt64Eq(ts_time_get_max(TIMESTAMPTZOID), TS_TIMESTAMP_INTERNAL_MAX); TestAssertInt64Eq(ts_time_get_end(TIMESTAMPTZOID), TS_TIMESTAMP_INTERNAL_END); TestAssertInt64Eq(ts_time_get_end_or_max(TIMESTAMPTZOID), TS_TIMESTAMP_INTERNAL_END); TestAssertInt64Eq(ts_time_get_nobegin(TIMESTAMPTZOID), TS_TIME_NOBEGIN); TestAssertInt64Eq(ts_time_get_noend(TIMESTAMPTZOID), TS_TIME_NOEND); TestAssertInt64Eq(DatumGetTimestampTz(ts_time_datum_get_nobegin_or_min(TIMESTAMPTZOID)), DT_NOBEGIN); TestAssertInt64Eq(DatumGetTimestampTz(ts_time_datum_get_min(TIMESTAMPTZOID)), TS_TIMESTAMP_MIN); TestAssertInt64Eq(DatumGetTimestampTz(ts_time_datum_get_max(TIMESTAMPTZOID)), TS_TIMESTAMP_MAX); TestAssertInt64Eq(DatumGetTimestampTz(ts_time_datum_get_end(TIMESTAMPTZOID)), TS_TIMESTAMP_END); TestAssertInt64Eq(DatumGetTimestampTz(ts_time_datum_get_nobegin(TIMESTAMPTZOID)), DT_NOBEGIN); TestAssertInt64Eq(DatumGetTimestampTz(ts_time_datum_get_noend(TIMESTAMPTZOID)), DT_NOEND); TestAssertInt64Eq(ts_time_get_min(DATEOID), TS_DATE_INTERNAL_MIN); TestAssertInt64Eq(ts_time_get_max(DATEOID), TS_DATE_INTERNAL_MAX); TestAssertInt64Eq(ts_time_get_end(DATEOID), TS_DATE_INTERNAL_END); TestAssertInt64Eq(ts_time_get_end_or_max(DATEOID), TS_DATE_INTERNAL_END); TestAssertInt64Eq(ts_time_get_nobegin(DATEOID), TS_TIME_NOBEGIN); TestAssertInt64Eq(ts_time_get_noend(DATEOID), TS_TIME_NOEND); TestAssertInt64Eq(DatumGetDateADT(ts_time_datum_get_nobegin_or_min(DATEOID)), DATEVAL_NOBEGIN); TestAssertInt64Eq(DatumGetDateADT(ts_time_datum_get_min(DATEOID)), TS_DATE_MIN); TestAssertInt64Eq(DatumGetDateADT(ts_time_datum_get_max(DATEOID)), TS_DATE_MAX); TestAssertInt64Eq(DatumGetDateADT(ts_time_datum_get_end(DATEOID)), TS_DATE_END); TestAssertInt64Eq(DatumGetDateADT(ts_time_datum_get_nobegin(DATEOID)), DATEVAL_NOBEGIN); TestAssertInt64Eq(DatumGetDateADT(ts_time_datum_get_noend(DATEOID)), DATEVAL_NOEND); /* Test boundary routines with unsupported time type */ TestEnsureError(ts_time_get_min(NUMERICOID)); TestEnsureError(ts_time_get_max(NUMERICOID)); TestEnsureError(ts_time_get_end(NUMERICOID)); TestEnsureError(ts_time_get_nobegin(NUMERICOID)); TestEnsureError(ts_time_get_noend(NUMERICOID)); TestEnsureError(ts_time_datum_get_nobegin_or_min(NUMERICOID)); TestEnsureError(ts_time_datum_get_min(NUMERICOID)); TestEnsureError(ts_time_datum_get_max(NUMERICOID)); TestEnsureError(ts_time_datum_get_end(NUMERICOID)); TestEnsureError(ts_time_datum_get_nobegin(NUMERICOID)); TestEnsureError(ts_time_datum_get_noend(NUMERICOID)); /* Test conversion of min, end, nobegin, and noend between native and * internal (Unix) time */ TestAssertInt64Eq(ts_time_value_to_internal(ts_time_datum_get_min(INT2OID), INT2OID), ts_time_get_min(INT2OID)); TestAssertInt64Eq(ts_time_value_to_internal(ts_time_datum_get_min(INT4OID), INT4OID), ts_time_get_min(INT4OID)); TestAssertInt64Eq(ts_time_value_to_internal(ts_time_datum_get_min(INT8OID), INT8OID), ts_time_get_min(INT8OID)); TestAssertInt64Eq(ts_time_value_to_internal(ts_time_datum_get_min(DATEOID), DATEOID), ts_time_get_min(DATEOID)); TestAssertInt64Eq(ts_time_value_to_internal(ts_time_datum_get_min(TIMESTAMPOID), TIMESTAMPOID), ts_time_get_min(TIMESTAMPOID)); TestAssertInt64Eq(ts_time_value_to_internal(ts_time_datum_get_min(TIMESTAMPTZOID), TIMESTAMPTZOID), ts_time_get_min(TIMESTAMPTZOID)); /* Test saturating addition */ TestAssertInt64Eq(ts_time_saturating_add(ts_time_get_max(INT2OID), 1, INT2OID), ts_time_get_max(INT2OID)); TestAssertInt64Eq(ts_time_saturating_add(ts_time_get_max(INT4OID), 1, INT4OID), ts_time_get_max(INT4OID)); TestAssertInt64Eq(ts_time_saturating_add(ts_time_get_max(INT8OID), 1, INT8OID), ts_time_get_max(INT8OID)); TestAssertInt64Eq(ts_time_saturating_add(ts_time_get_max(DATEOID), 1, DATEOID), ts_time_get_noend(DATEOID)); TestAssertInt64Eq(ts_time_saturating_add(ts_time_get_max(DATEOID), 1, DATEOID), ts_time_get_noend(DATEOID)); TestAssertInt64Eq(ts_time_saturating_add(ts_time_get_max(TIMESTAMPOID), 1, TIMESTAMPOID), ts_time_get_noend(TIMESTAMPOID)); TestAssertInt64Eq(ts_time_saturating_add(ts_time_get_max(TIMESTAMPTZOID), 1, TIMESTAMPOID), ts_time_get_noend(TIMESTAMPOID)); TestAssertInt64Eq(ts_time_saturating_add(ts_time_get_end(DATEOID) - 2, 1, DATEOID), ts_time_get_max(DATEOID)); TestAssertInt64Eq(ts_time_saturating_add(ts_time_get_end(TIMESTAMPOID) - 2, 1, TIMESTAMPOID), ts_time_get_max(TIMESTAMPOID)); TestAssertInt64Eq(ts_time_saturating_add(ts_time_get_end(TIMESTAMPTZOID) - 2, 1, TIMESTAMPOID), ts_time_get_max(TIMESTAMPOID)); TestAssertInt64Eq(ts_time_saturating_add(ts_time_get_min(INT2OID), -1, INT2OID), ts_time_get_min(INT2OID)); TestAssertInt64Eq(ts_time_saturating_add(ts_time_get_min(INT4OID), -1, INT4OID), ts_time_get_min(INT4OID)); TestAssertInt64Eq(ts_time_saturating_add(ts_time_get_min(INT8OID), -1, INT8OID), ts_time_get_min(INT8OID)); TestAssertInt64Eq(ts_time_saturating_add(ts_time_get_min(DATEOID), -1, DATEOID), ts_time_get_nobegin(DATEOID)); TestAssertInt64Eq(ts_time_saturating_add(ts_time_get_min(DATEOID), -1, DATEOID), ts_time_get_nobegin(DATEOID)); TestAssertInt64Eq(ts_time_saturating_add(ts_time_get_min(TIMESTAMPOID), -1, TIMESTAMPOID), ts_time_get_nobegin(TIMESTAMPOID)); TestAssertInt64Eq(ts_time_saturating_add(ts_time_get_min(TIMESTAMPTZOID), -1, TIMESTAMPOID), ts_time_get_nobegin(TIMESTAMPOID)); /* Test saturating subtraction */ TestAssertInt64Eq(ts_time_saturating_sub(ts_time_get_min(INT2OID), 1, INT2OID), ts_time_get_min(INT2OID)); TestAssertInt64Eq(ts_time_saturating_sub(ts_time_get_min(INT4OID), 1, INT4OID), ts_time_get_min(INT4OID)); TestAssertInt64Eq(ts_time_saturating_sub(ts_time_get_min(INT8OID), 1, INT8OID), ts_time_get_min(INT8OID)); TestAssertInt64Eq(ts_time_saturating_sub(ts_time_get_min(DATEOID), 1, DATEOID), ts_time_get_nobegin(DATEOID)); TestAssertInt64Eq(ts_time_saturating_sub(ts_time_get_min(TIMESTAMPOID), 1, TIMESTAMPOID), ts_time_get_nobegin(TIMESTAMPOID)); TestAssertInt64Eq(ts_time_saturating_sub(ts_time_get_min(TIMESTAMPTZOID), 1, TIMESTAMPTZOID), ts_time_get_nobegin(TIMESTAMPTZOID)); TestAssertInt64Eq(ts_time_saturating_sub(ts_time_get_min(DATEOID) + 1, 1, DATEOID), ts_time_get_min(DATEOID)); TestAssertInt64Eq(ts_time_saturating_sub(ts_time_get_min(TIMESTAMPOID) + 1, 1, TIMESTAMPOID), ts_time_get_min(TIMESTAMPOID)); TestAssertInt64Eq(ts_time_saturating_sub(ts_time_get_min(TIMESTAMPTZOID) + 1, 1, TIMESTAMPTZOID), ts_time_get_min(TIMESTAMPTZOID)); TestAssertInt64Eq(ts_time_saturating_sub(ts_time_get_max(INT2OID), -1, INT2OID), ts_time_get_max(INT2OID)); TestAssertInt64Eq(ts_time_saturating_sub(ts_time_get_max(INT4OID), -1, INT4OID), ts_time_get_max(INT4OID)); TestAssertInt64Eq(ts_time_saturating_sub(ts_time_get_max(INT8OID), -1, INT8OID), ts_time_get_max(INT8OID)); TestAssertInt64Eq(ts_time_saturating_sub(ts_time_get_max(DATEOID), -1, DATEOID), ts_time_get_noend(DATEOID)); TestAssertInt64Eq(ts_time_saturating_sub(ts_time_get_max(TIMESTAMPOID), -1, TIMESTAMPOID), ts_time_get_noend(TIMESTAMPOID)); TestAssertInt64Eq(ts_time_saturating_sub(ts_time_get_max(TIMESTAMPTZOID), -1, TIMESTAMPTZOID), ts_time_get_noend(TIMESTAMPTZOID)); PG_RETURN_VOID(); } ================================================ FILE: test/src/test_tss_callbacks.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <fmgr.h> #include <funcapi.h> #include "export.h" #include "tss_callbacks.h" static void test_tss_store_hook(const char *query, int query_location, int query_len, uint64 query_id, uint64 total_time, uint64 rows, const BufferUsage *bufusage, const WalUsage *walusage) { elog(NOTICE, "test_tss_callbacks (mock): query=%s, len=%d, id=" INT64_FORMAT ", rows=" INT64_FORMAT, query, query_len, query_id, rows); } static bool test_tss_enabled_hook(int level) { return true; } TSSCallbacks test_tss_callbacks_v1 = { .version_num = 1, .tss_store_hook = test_tss_store_hook, .tss_enabled_hook_type = test_tss_enabled_hook, }; TS_FUNCTION_INFO_V1(ts_setup_tss_hook_v1); Datum ts_setup_tss_hook_v1(PG_FUNCTION_ARGS) { TSSCallbacks **ptr = (TSSCallbacks **) find_rendezvous_variable(TSS_CALLBACKS_VAR_NAME); *ptr = &test_tss_callbacks_v1; PG_RETURN_NULL(); } TS_FUNCTION_INFO_V1(ts_teardown_tss_hook_v1); Datum ts_teardown_tss_hook_v1(PG_FUNCTION_ARGS) { TSSCallbacks **ptr = (TSSCallbacks **) find_rendezvous_variable(TSS_CALLBACKS_VAR_NAME); *ptr = NULL; PG_RETURN_NULL(); } /* This version will mismatch with the current supported */ TSSCallbacks test_tss_callbacks_v0 = { .version_num = 0, .tss_store_hook = test_tss_store_hook, .tss_enabled_hook_type = test_tss_enabled_hook, }; TS_FUNCTION_INFO_V1(ts_setup_tss_hook_v0); Datum ts_setup_tss_hook_v0(PG_FUNCTION_ARGS) { TSSCallbacks **ptr = (TSSCallbacks **) find_rendezvous_variable(TSS_CALLBACKS_VAR_NAME); *ptr = &test_tss_callbacks_v0; PG_RETURN_NULL(); } ================================================ FILE: test/src/test_utils.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include "test_utils.h" #include <postgres.h> #include <compat/compat.h> #include <commands/dbcommands.h> #include <fmgr.h> #include <miscadmin.h> #include <storage/latch.h> #include <storage/proc.h> #include <storage/procarray.h> #include <utils/builtins.h> #include <utils/elog.h> #include <utils/guc.h> #include <utils/memutils.h> #include "debug_point.h" #include "extension_constants.h" #include "utils.h" TS_FUNCTION_INFO_V1(ts_test_error_injection); TS_FUNCTION_INFO_V1(ts_debug_shippable_error_after_n_rows); TS_FUNCTION_INFO_V1(ts_debug_shippable_fatal_after_n_rows); /* * Test assertion macros. * * Errors are expected since we want to test that the macros work. For each * macro, test one failing and one non-failing condition. The non-failing must * come first since the failing one will abort the function. */ TS_TEST_FN(ts_test_utils_condition) { bool true_value = true; bool false_value = false; TestAssertTrue(true_value == true_value); TestAssertTrue(true_value == false_value); PG_RETURN_VOID(); } TS_TEST_FN(ts_test_utils_int64_eq) { int64 big = 32532978; int64 small = 3242234; TestAssertInt64Eq(big, small); TestAssertInt64Eq(big, big); PG_RETURN_VOID(); } TS_TEST_FN(ts_test_utils_ptr_eq) { bool true_value = true; bool false_value = false; bool *true_ptr = &true_value; bool *false_ptr = &false_value; TestAssertPtrEq(true_ptr, true_ptr); TestAssertPtrEq(true_ptr, false_ptr); PG_RETURN_VOID(); } TS_TEST_FN(ts_test_utils_double_eq) { double big_double = 923423478.3242; double small_double = 324.3; TestAssertDoubleEq(big_double, big_double); TestAssertDoubleEq(big_double, small_double); PG_RETURN_VOID(); } Datum ts_test_error_injection(PG_FUNCTION_ARGS) { text *name = PG_GETARG_TEXT_PP(0); DEBUG_ERROR_INJECTION(text_to_cstring(name)); PG_RETURN_VOID(); } static int transaction_row_counter(void) { static LocalTransactionId last_lxid = 0; static int rows_seen = 0; #if PG17_GE if (last_lxid != MyProc->vxid.lxid) { /* Reset it for each new transaction for predictable results. */ rows_seen = 0; last_lxid = MyProc->vxid.lxid; } #else if (last_lxid != MyProc->lxid) { rows_seen = 0; last_lxid = MyProc->lxid; } #endif return rows_seen++; } static int throw_after_n_rows(int max_rows, int severity) { int rows_seen = transaction_row_counter(); if (max_rows <= rows_seen) { ereport(severity, (errmsg("debug point: requested to error out after %d rows, %d rows seen", max_rows, rows_seen))); } return rows_seen; } Datum ts_debug_shippable_error_after_n_rows(PG_FUNCTION_ARGS) { PG_RETURN_INT32(throw_after_n_rows(PG_GETARG_INT32(0), ERROR)); } Datum ts_debug_shippable_fatal_after_n_rows(PG_FUNCTION_ARGS) { PG_RETURN_INT32(throw_after_n_rows(PG_GETARG_INT32(0), FATAL)); } /* * After how many rows should we error out according to the user-set option. */ static int get_error_after_rows() { int error_after = 7103; /* default is an arbitrary prime */ const char *error_after_option = GetConfigOption(MAKE_EXTOPTION("debug_broken_sendrecv_error_after"), true, false); if (error_after_option) { error_after = pg_strtoint32(error_after_option); } return error_after; } /* * Broken send/receive functions for int4 that throw after an (arbitrarily * chosen prime or configured) number of rows. */ static void broken_sendrecv_throw() { /* * Use ERROR, not FATAL, because PG versions < 14 are unable to report a * FATAL error to the access node before closing the connection, so the test * results would be different. */ (void) throw_after_n_rows(get_error_after_rows(), ERROR); } TS_FUNCTION_INFO_V1(ts_debug_broken_int4recv); Datum ts_debug_broken_int4recv(PG_FUNCTION_ARGS) { broken_sendrecv_throw(); return int4recv(fcinfo); } TS_FUNCTION_INFO_V1(ts_debug_broken_int4send); Datum ts_debug_broken_int4send(PG_FUNCTION_ARGS) { broken_sendrecv_throw(); return int4send(fcinfo); } /* An incorrect int4out that sometimes returns not a number. */ TS_FUNCTION_INFO_V1(ts_debug_incorrect_int4out); Datum ts_debug_incorrect_int4out(PG_FUNCTION_ARGS) { int rows_seen = transaction_row_counter(); if (rows_seen >= get_error_after_rows()) { PG_RETURN_CSTRING("surprise"); } return int4out(fcinfo); } /* Sleeps after a certain number of calls. */ static void ts_debug_sleepy_function() { static LocalTransactionId last_lxid = 0; static int rows_seen = 0; #if PG17_GE if (last_lxid != MyProc->vxid.lxid) { /* Reset it for each new transaction for predictable results. */ rows_seen = 0; last_lxid = MyProc->vxid.lxid; } #else if (last_lxid != MyProc->lxid) { rows_seen = 0; last_lxid = MyProc->lxid; } #endif rows_seen++; if (rows_seen >= 997) { (void) WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, 1000, /* wait_event_info = */ 0); ResetLatch(MyLatch); rows_seen = 0; } } TS_FUNCTION_INFO_V1(ts_debug_sleepy_int4recv); Datum ts_debug_sleepy_int4recv(PG_FUNCTION_ARGS) { ts_debug_sleepy_function(); return int4recv(fcinfo); } TS_FUNCTION_INFO_V1(ts_debug_sleepy_int4send); Datum ts_debug_sleepy_int4send(PG_FUNCTION_ARGS) { ts_debug_sleepy_function(); return int4send(fcinfo); } TS_FUNCTION_INFO_V1(ts_bgw_wait); Datum ts_bgw_wait(PG_FUNCTION_ARGS) { text *datname = PG_GETARG_TEXT_PP(0); /* The timeout is given in seconds, so we compute the number of iterations * necessary to get a coverage of that time */ uint32 iterations = PG_ARGISNULL(1) ? 5 : (PG_GETARG_UINT32(1) + 4) / 5; bool raise_error = PG_ARGISNULL(2) ? true : PG_GETARG_BOOL(2); Oid dboid = get_database_oid(text_to_cstring(datname), false); /* This function contains a timeout of 5 seconds, so we iterate a few * times to make sure that it really has terminated. */ int notherbackends = 0; int npreparedxacts = 0; while (iterations-- > 0) { if (!CountOtherDBBackends(dboid, ¬herbackends, &npreparedxacts)) PG_RETURN_NULL(); ereport(NOTICE, (errmsg("source database \"%s\" is being accessed by other users", text_to_cstring(datname)), errdetail("There are %d other session(s) and %d prepared transaction(s) using the " "database.", notherbackends, npreparedxacts))); } if (raise_error) { ereport(ERROR, (errcode(ERRCODE_OBJECT_IN_USE), errmsg("source database \"%s\" is being accessed by other users", text_to_cstring(datname)), errdetail("There are %d other session(s) and %d prepared transaction(s) using the " "database.", notherbackends, npreparedxacts))); } pg_unreachable(); } /* * Return the number of bytes allocated in a given memory context and its * children. */ TS_FUNCTION_INFO_V1(ts_debug_allocated_bytes); Datum ts_debug_allocated_bytes(PG_FUNCTION_ARGS) { MemoryContext context = NULL; char *context_name = text_to_cstring(PG_GETARG_TEXT_PP(0)); if (strcmp(context_name, "PortalContext") == 0) { context = PortalContext; } else if (strcmp(context_name, "CacheMemoryContext") == 0) { context = CacheMemoryContext; } else if (strcmp(context_name, "TopMemoryContext") == 0) { context = TopMemoryContext; } else { ereport(ERROR, (errmsg("unknown memory context '%s' (search for arbitrary contexts by name is not" "implemented)", context_name))); PG_RETURN_NULL(); } PG_RETURN_UINT64(MemoryContextMemAllocated(context, /* recurse = */ true)); } TS_TEST_FN(ts_test_errdata_to_jsonb) { ErrorData *edata = (ErrorData *) palloc(sizeof(ErrorData)); edata->elevel = ERROR; edata->output_to_server = true; edata->output_to_client = true; edata->hide_stmt = false; edata->hide_ctx = false; edata->filename = "test error filename"; edata->lineno = 123; edata->funcname = "test error function"; edata->domain = "test error domain"; edata->context_domain = "test error context domain"; edata->sqlerrcode = ERRCODE_INVALID_PARAMETER_VALUE; edata->message = "test error message"; edata->detail = "test error detail"; edata->detail_log = "test error detail log"; edata->hint = "test error hint"; edata->context = "test error context"; edata->backtrace = "test error backtrace"; edata->message_id = "test error message id"; edata->schema_name = "test error schema"; edata->table_name = "test error table"; edata->column_name = "test error column"; edata->datatype_name = "test error datatype"; edata->constraint_name = "test error constraint"; edata->cursorpos = 42; edata->internalpos = 42; edata->internalquery = "test error internal query"; edata->saved_errno = 42; NameData proc_schema = { .data = { 0 } }; NameData proc_name = { .data = { 0 } }; namestrcpy(&proc_schema, "proc_schema"); namestrcpy(&proc_name, "proc_name"); Jsonb *out = ts_errdata_to_jsonb(edata, &proc_schema, &proc_name); PG_RETURN_JSONB_P(out); } ================================================ FILE: test/src/test_utils.h ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #pragma once #include <postgres.h> #include <access/xact.h> #include <fmgr.h> #include "export.h" static inline const char * strip_path(const char *filename) { int i = 0, slash = 0; while (filename[i] != '\0') { if (filename[i] == '/' || filename[i] == '\\') slash = i; i++; } return &filename[slash + 1]; } #define TestFailure(fmt, ...) \ do \ { \ elog(WARNING, "TestFailure in %s() at line:%d", __func__, __LINE__); \ elog(ERROR, "TestFailure | " fmt "", ##__VA_ARGS__); \ pg_unreachable(); \ } while (0) #define TestAssertInt64Eq(a, b) \ do \ { \ int64 a_i = (a); \ int64 b_i = (b); \ if (a_i != b_i) \ TestFailure("(%s == %s) [" INT64_FORMAT " == " INT64_FORMAT "]", #a, #b, a_i, b_i); \ } while (0) #define TestAssertPtrEq(a, b) \ do \ { \ void *a_i = (a); \ void *b_i = (b); \ if (a_i != b_i) \ TestFailure("(%s == %s)", #a, #b); \ } while (0) #define TestAssertCStringEq(a, b) \ do \ { \ const char *a_i = (a) == NULL ? "<null>" : (a); \ const char *b_i = (b) == NULL ? "<null>" : (b); \ if (strcmp(a_i, b_i) != 0) \ TestFailure("(%s == %s) [%s == %s]", #a, #b, a_i, b_i); \ } while (0) #define TestAssertDoubleEq(a, b) \ do \ { \ double a_i = (a); \ double b_i = (b); \ if (a_i != b_i) \ TestFailure("(%s == %s) [%f == %f]", #a, #b, a_i, b_i); \ } while (0) #define TestAssertBoolEq(a, b) \ do \ { \ const char *a_i = (a) ? "true" : "false"; \ const char *b_i = (b) ? "true" : "false"; \ bool a_bool = (a); \ bool b_bool = (b); \ if (a_bool != b_bool) \ TestFailure("(%s == %s) [%s == %s]", #a, #b, a_i, b_i); \ } while (0) #define TestEnsureError(a) \ do \ { \ volatile bool this_has_panicked = false; \ MemoryContext oldctx = CurrentMemoryContext; \ BeginInternalSubTransaction("error expected"); \ PG_TRY(); \ { \ (a); \ } \ PG_CATCH(); \ { \ this_has_panicked = true; \ RollbackAndReleaseCurrentSubTransaction(); \ FlushErrorState(); \ } \ PG_END_TRY(); \ MemoryContextSwitchTo(oldctx); \ if (!this_has_panicked) \ { \ elog(ERROR, "failed to panic"); \ } \ } while (0) #define TestAssertTrue(cond) \ do \ { \ if (!(cond)) \ TestFailure("(%s)", #cond); \ } while (0) #define TS_TEST_FN(name) \ TS_FUNCTION_INFO_V1(name); \ Datum name(PG_FUNCTION_ARGS) #ifdef __JSONB_H__ extern const char *jsonb_to_cstring(Jsonb *jsonb); extern Jsonb *cstring_to_jsonb(const char *cstring); #define TestAssertJsonbEqCstring(a, b) \ do \ { \ Jsonb *a_j = (a); \ const char *a_i = (a) == NULL ? "<null>" : jsonb_to_cstring(a_j); \ const char *b_i = (b) == NULL ? "<null>" : (b); \ if (strcmp(a_i, b_i) != 0) \ TestFailure("(%s == %s)", a_i, b_i); \ } while (0) #endif ================================================ FILE: test/src/test_with_clause_parser.c ================================================ /* * This file and its contents are licensed under the Apache License 2.0. * Please see the included NOTICE for copyright information and * LICENSE-APACHE for a copy of the license. */ #include <postgres.h> #include <access/htup_details.h> #include <catalog/pg_type.h> #include <commands/defrem.h> #include <fmgr.h> #include <funcapi.h> #include <postgres_ext.h> #include <utils/array.h> #include <utils/builtins.h> #include <utils/elog.h> #include <utils/lsyscache.h> #include <utils/memutils.h> #include <utils/regproc.h> #include "annotations.h" #include "export.h" #include "test_utils.h" #include "with_clause/with_clause_parser.h" TS_FUNCTION_INFO_V1(ts_sqlstate_raise_in); TS_FUNCTION_INFO_V1(ts_sqlstate_raise_out); /* * Input function that will raise the error code give. This means that you can * trigger an error when reading and converting a string to this type. */ Datum ts_sqlstate_raise_in(PG_FUNCTION_ARGS) { char *code = PG_GETARG_CSTRING(0); if (strlen(code) != 5) ereport(ERROR, errcode(ERRCODE_SYNTAX_ERROR), errmsg("error code \"%s\" was not of length 5", code)); int sqlstate = MAKE_SQLSTATE(code[0], code[1], code[2], code[3], code[4]); ereport(ERROR, errcode(sqlstate), errmsg("raised requested error code \"%s\"", code)); return 0; } /* * Dummy function, we do not store values of this type anywhere. */ Datum ts_sqlstate_raise_out(PG_FUNCTION_ARGS) { PG_RETURN_CSTRING("uninteresting"); } static DefElem * def_elem_from_texts(Datum *texts, int nelems) { DefElem *elem = palloc0(sizeof(*elem)); switch (nelems) { case 1: elem->defname = text_to_cstring(DatumGetTextP(texts[0])); break; case 3: elem->arg = (Node *) makeString(text_to_cstring(DatumGetTextP(texts[2]))); TS_FALLTHROUGH; case 2: elem->defname = text_to_cstring(DatumGetTextP(texts[1])); elem->defnamespace = text_to_cstring(DatumGetTextP(texts[0])); break; default: elog(ERROR, "%d elements invalid for defelem", nelems); } return elem; } static List * def_elems_from_array(ArrayType *with_clause_array) { ArrayMetaState with_clause_meta = { .element_type = TEXTOID }; ArrayIterator with_clause_iter; Datum with_clause_datum; bool with_clause_null; List *def_elems = NIL; get_typlenbyvalalign(with_clause_meta.element_type, &with_clause_meta.typlen, &with_clause_meta.typbyval, &with_clause_meta.typalign); with_clause_iter = array_create_iterator(with_clause_array, 1, &with_clause_meta); while (array_iterate(with_clause_iter, &with_clause_datum, &with_clause_null)) { Datum *with_clause_fields; int with_clause_elems; ArrayType *with_clause = DatumGetArrayTypeP(with_clause_datum); TestAssertTrue(!with_clause_null); deconstruct_array(with_clause, TEXTOID, with_clause_meta.typlen, with_clause_meta.typbyval, with_clause_meta.typalign, &with_clause_fields, NULL, &with_clause_elems); def_elems = lappend(def_elems, def_elem_from_texts(with_clause_fields, with_clause_elems)); } return def_elems; } typedef struct FilteredWithClauses { List *within; List *non_ts_namespace_option; List *without; } FilteredWithClauses; static HeapTuple create_filter_tuple(TupleDesc tuple_desc, DefElem *d, bool within) { Datum *values = palloc0(sizeof(*values) * tuple_desc->natts); bool *nulls = palloc0(sizeof(*nulls) * tuple_desc->natts); TestAssertTrue(tuple_desc->natts >= 4); if (d->defnamespace != NULL) values[0] = CStringGetTextDatum(d->defnamespace); else nulls[0] = true; if (d->defname != NULL) values[1] = CStringGetTextDatum(d->defname); else nulls[1] = true; if (d->arg != NULL) values[2] = CStringGetTextDatum(defGetString(d)); else nulls[2] = true; values[3] = BoolGetDatum(within); return heap_form_tuple(tuple_desc, values, nulls); } TS_TEST_FN(ts_test_with_clause_filter) { FuncCallContext *funcctx; FilteredWithClauses *filtered; MemoryContext oldcontext; if (SRF_IS_FIRSTCALL()) { ArrayType *with_clause_array; TupleDesc tupdesc; List *def_elems; funcctx = SRF_FIRSTCALL_INIT(); oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); with_clause_array = DatumGetArrayTypeP(PG_GETARG_DATUM(0)); if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context " "that cannot accept type record"))); funcctx->tuple_desc = BlessTupleDesc(tupdesc); def_elems = def_elems_from_array(with_clause_array); filtered = palloc(sizeof(*filtered)); filtered->within = NIL; filtered->non_ts_namespace_option = NIL; filtered->without = NIL; ts_with_clause_filter(def_elems, &filtered->within, &filtered->non_ts_namespace_option, &filtered->without); funcctx->user_fctx = filtered; MemoryContextSwitchTo(oldcontext); } funcctx = SRF_PERCALL_SETUP(); filtered = funcctx->user_fctx; if (filtered->within != NIL) { HeapTuple tuple; DefElem *d = linitial(filtered->within); tuple = create_filter_tuple(funcctx->tuple_desc, d, true); filtered->within = list_delete_first(filtered->within); SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(tuple)); } else if (filtered->non_ts_namespace_option != NIL) { HeapTuple tuple; DefElem *d = linitial(filtered->non_ts_namespace_option); tuple = create_filter_tuple(funcctx->tuple_desc, d, true); filtered->non_ts_namespace_option = list_delete_first(filtered->non_ts_namespace_option); SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(tuple)); } else if (filtered->without != NIL) { HeapTuple tuple; DefElem *d = linitial(filtered->without); tuple = create_filter_tuple(funcctx->tuple_desc, d, false); filtered->without = list_delete_first(filtered->without); SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(tuple)); } else SRF_RETURN_DONE(funcctx); } typedef enum TestArgs { TestArgUnimpl = 0, TestArgBool, TestArgInt32, TestArgDefault, TestArgName, TestArgRegclass, TestArgRaise, } TestArgs; static WithClauseDefinition test_args[] = { [TestArgUnimpl] = { .arg_names = {"unimplemented", NULL}, .type_id = InvalidOid, }, [TestArgBool] = { .arg_names = {"bool", NULL}, .type_id = BOOLOID, }, [TestArgInt32] = { .arg_names = {"int32", NULL}, .type_id = INT4OID, }, [TestArgDefault] = { .arg_names = {"default", NULL}, .type_id = INT4OID, .default_val = (Datum)-100 }, [TestArgName] = { .arg_names = {"name", NULL}, .type_id = NAMEOID, }, [TestArgRegclass] = { .arg_names = {"regclass", NULL}, .type_id = REGCLASSOID, }, [TestArgRaise] = { .arg_names = {"sqlstate_raise", NULL}, .type_id = InvalidOid, }, }; typedef struct WithClauseValue { WithClauseResult *parsed; int i; } WithClauseValue; TS_TEST_FN(ts_test_with_clause_parse) { FuncCallContext *funcctx; MemoryContext oldcontext; Datum *values; bool *nulls; HeapTuple tuple; WithClauseValue *result; /* * Look up any missing type ids before using it below to allow * user-defined types. * * Note that this will not look up types we have found in previous calls * of this function. * * We use the slightly more complicated way of calling to_regtype since * that exists on all versions of PostgreSQL. We cannot use regtypein * since that can generate errors and we do not want to deal with that. */ for (unsigned int i = 0; i < TS_ARRAY_LEN(test_args); ++i) { LOCAL_FCINFO(fcinfo_in, 1); Datum result; if (!OidIsValid(test_args[i].type_id)) { InitFunctionCallInfoData(*fcinfo_in, NULL, 1, InvalidOid, NULL, NULL); fcinfo_in->args[0].value = CStringGetTextDatum(test_args[i].arg_names[0]); fcinfo_in->args[0].isnull = false; result = to_regtype(fcinfo_in); if (!fcinfo_in->isnull) test_args[i].type_id = DatumGetObjectId(result); } } if (SRF_IS_FIRSTCALL()) { ArrayType *with_clause_array; List *def_elems; TupleDesc tupdesc; WithClauseResult *parsed; funcctx = SRF_FIRSTCALL_INIT(); oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context " "that cannot accept type record"))); funcctx->tuple_desc = BlessTupleDesc(tupdesc); with_clause_array = DatumGetArrayTypeP(PG_GETARG_DATUM(0)); def_elems = def_elems_from_array(with_clause_array); parsed = ts_with_clauses_parse(def_elems, test_args, TS_ARRAY_LEN(test_args)); result = palloc(sizeof(*result)); result->parsed = parsed; result->i = 0; funcctx->user_fctx = result; MemoryContextSwitchTo(oldcontext); } funcctx = SRF_PERCALL_SETUP(); result = funcctx->user_fctx; if (result == NULL || (size_t) result->i >= TS_ARRAY_LEN(test_args)) SRF_RETURN_DONE(funcctx); values = palloc0(sizeof(*values) * funcctx->tuple_desc->natts); nulls = palloc(sizeof(*nulls) * funcctx->tuple_desc->natts); memset(nulls, true, sizeof(*nulls) * funcctx->tuple_desc->natts); values[0] = CStringGetTextDatum(test_args[result->i].arg_names[0]); nulls[0] = false; if (!result->parsed[result->i].is_default || result->i == TestArgDefault) { values[result->i + 1] = result->parsed[result->i].parsed; nulls[result->i + 1] = false; } tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls); result->i += 1; SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(tuple)); } ================================================ FILE: test/t/001_replication_telemetry.pl ================================================ # This file and its contents are licensed under the Timescale License. # Please see the included NOTICE for copyright information and # LICENSE-TIMESCALE for a copy of the license. use strict; use warnings; use TimescaleNode; use Data::Dumper; use Test::More tests => 4; # This test checks that the extension state is handled correctly # across multiple sessions. Specifically, if the extension state # changes in one session (e.g., the extension is created or dropped), # this should be reflected in other concurrent sessions. # # To test this, we start one background psql session that stays open # for the duration of the tests and then change the extension state # from other sessions. my $node_primary = TimescaleNode->create( 'primary', allows_streaming => 1, auth_extra => [ '--create-role', 'repl_role' ]); my $backup_name = 'my_backup'; # Take backup $node_primary->backup($backup_name); # Create streaming standby linking to primary my $node_standby = PostgreSQL::Test::Cluster->new('standby_1'); $node_standby->init_from_backup($node_primary, $backup_name, has_streaming => 1); $node_standby->start; # Wait for standby to catch up $node_primary->wait_for_catchup($node_standby, 'replay', $node_primary->lsn('insert')); my $result = $node_primary->safe_psql('postgres', "SELECT get_telemetry_report()->'replication'->>'num_wal_senders'"); is($result, qq(1), 'number of wal senders on primary'); $result = $node_primary->safe_psql('postgres', "SELECT get_telemetry_report()->'replication'->>'is_wal_receiver'"); is($result, qq(false), 'primary is wal receiver'); $result = $node_standby->safe_psql('postgres', "SELECT get_telemetry_report()->'replication'->>'num_wal_senders'"); is($result, qq(0), 'number of wal senders on standby'); $result = $node_standby->safe_psql('postgres', "SELECT get_telemetry_report()->'replication'->>'is_wal_receiver'"); is($result, qq(true), 'standby is wal receiver'); done_testing(); 1; ================================================ FILE: test/t/CMakeLists.txt ================================================ if(CMAKE_BUILD_TYPE MATCHES Debug) list(APPEND PROVE_TEST_FILES 001_replication_telemetry.pl) endif(CMAKE_BUILD_TYPE MATCHES Debug) foreach(P_FILE ${PROVE_TEST_FILES}) configure_file(${P_FILE} ${CMAKE_CURRENT_BINARY_DIR}/${P_FILE} COPYONLY) endforeach(P_FILE) ================================================ FILE: test/test-defs.cmake ================================================ set(TEST_ROLE_SUPERUSER super_user) set(TEST_ROLE_DEFAULT_PERM_USER default_perm_user) set(TEST_ROLE_DEFAULT_PERM_USER_2 default_perm_user_2) set(TEST_ROLE_1 test_role_1) set(TEST_ROLE_2 test_role_2) set(TEST_ROLE_3 test_role_3) set(TEST_ROLE_4 test_role_4) set(TEST_ROLE_READ_ONLY test_role_read_only) # TEST_ROLE_2 has password in passfile set(TEST_ROLE_2_PASS pass) # TEST_ROLE_3 does not have password in passfile set(TEST_ROLE_3_PASS pass) # TEST_ROLE_4 does not have password in passfile set(TEST_ROLE_4_PASS pass) set(TEST_INPUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}) set(TEST_OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}) set(TEST_CLUSTER ${TEST_OUTPUT_DIR}/testcluster) # Basic connection info for test instance set(TEST_PGPORT_LOCAL 5432 CACHE STRING "The port of a running PostgreSQL instance") set(TEST_PGHOST localhost CACHE STRING "The hostname of a running PostgreSQL instance") set(TEST_PGUSER ${TEST_ROLE_DEFAULT_PERM_USER} CACHE STRING "The PostgreSQL test user") set(TEST_DBNAME single CACHE STRING "The database name to use for tests") set(TEST_PGPORT_TEMP_INSTANCE 55432 CACHE STRING "The port to run a temporary test PostgreSQL instance on") set(TEST_SCHEDULE ${CMAKE_CURRENT_BINARY_DIR}/test_schedule) set(TEST_SCHEDULE_SHARED ${CMAKE_CURRENT_BINARY_DIR}/shared/test_schedule_shared) set(ISOLATION_TEST_SCHEDULE ${CMAKE_CURRENT_BINARY_DIR}/isolation_test_schedule) set(TEST_PASSFILE ${TEST_OUTPUT_DIR}/pgpass.conf) set(TEST_TIMEOUT 120 CACHE STRING "Timeout in seconds for individual tests") # Windows does not support local connections (unix domain sockets) if(WIN32) set(TEST_HBA_LOCAL "#local") else() set(TEST_HBA_LOCAL "local") endif() # This variable is set differently in CI. We use it to save the logs outside the # tmp instance, because it is deleted by pg_regress on successful test # completion, and we want to run some additional checks on the logs in any case. set(TEST_PG_LOG_DIRECTORY "log" CACHE STRING "Log directory for regression tests") if(USE_TELEMETRY) set(TELEMETRY_DEFAULT_SETTING "timescaledb.telemetry_level=off") else() set(TELEMETRY_DEFAULT_SETTING) endif() configure_file(postgresql.conf.in postgresql.conf) configure_file(${PRIMARY_TEST_DIR}/pgtest.conf.in pgtest.conf) # pgpass file requires chmod 0600 configure_file(${PRIMARY_TEST_DIR}/pgpass.conf.in ${TEST_OUTPUT_DIR}${CMAKE_FILES_DIRECTORY}/pgpass.conf) file( COPY ${TEST_OUTPUT_DIR}${CMAKE_FILES_DIRECTORY}/pgpass.conf DESTINATION ${TEST_OUTPUT_DIR} NO_SOURCE_PERMISSIONS FILE_PERMISSIONS OWNER_READ OWNER_WRITE) set(PG_REGRESS_OPTS_BASE --host=${TEST_PGHOST} --dlpath=${PROJECT_BINARY_DIR}/src) set(PG_REGRESS_OPTS_EXTRA --create-role=${TEST_ROLE_SUPERUSER},${TEST_ROLE_DEFAULT_PERM_USER},${TEST_ROLE_DEFAULT_PERM_USER_2},${TEST_ROLE_1},${TEST_ROLE_2},${TEST_ROLE_3},${TEST_ROLE_4},${TEST_ROLE_READ_ONLY} --dbname=${TEST_DBNAME} --launcher=${PRIMARY_TEST_DIR}/runner.sh) set(PG_REGRESS_SHARED_OPTS_EXTRA --create-role=${TEST_ROLE_SUPERUSER},${TEST_ROLE_DEFAULT_PERM_USER},${TEST_ROLE_DEFAULT_PERM_USER_2} --dbname=${TEST_DBNAME} --launcher=${PRIMARY_TEST_DIR}/runner_shared.sh) set(PG_ISOLATION_REGRESS_OPTS_EXTRA --create-role=${TEST_ROLE_SUPERUSER},${TEST_ROLE_DEFAULT_PERM_USER},${TEST_ROLE_DEFAULT_PERM_USER_2},${TEST_ROLE_1},${TEST_ROLE_2},${TEST_ROLE_3},${TEST_ROLE_READ_ONLY} --dbname=${TEST_DBNAME} --launcher=${PRIMARY_TEST_DIR}/runner_isolation.sh) set(PG_REGRESS_OPTS_INOUT --inputdir=${TEST_INPUT_DIR} --outputdir=${TEST_OUTPUT_DIR}) set(PG_REGRESS_SHARED_OPTS_INOUT --inputdir=${TEST_INPUT_DIR}/shared --outputdir=${TEST_OUTPUT_DIR}/shared --load-extension=timescaledb) set(PG_ISOLATION_REGRESS_OPTS_INOUT --inputdir=${TEST_INPUT_DIR}/isolation --outputdir=${TEST_OUTPUT_DIR}/isolation --load-extension=timescaledb) set(PG_REGRESS_OPTS_TEMP_INSTANCE --port=${TEST_PGPORT_TEMP_INSTANCE} --temp-instance=${TEST_CLUSTER} --temp-config=${TEST_OUTPUT_DIR}/postgresql.conf) set(PG_REGRESS_OPTS_TEMP_INSTANCE_PGTEST --port=${TEST_PGPORT_TEMP_INSTANCE} --temp-instance=${TEST_CLUSTER} --temp-config=${TEST_OUTPUT_DIR}/pgtest.conf) set(PG_REGRESS_OPTS_LOCAL_INSTANCE --port=${TEST_PGPORT_LOCAL}) if(PG_REGRESS) set(PG_REGRESS_ENV PG_BINDIR=${PG_BINDIR} PG_REGRESS=${PG_REGRESS} PG_REGRESS_USE_FAKETIME=1 TEST_DBNAME=${TEST_DBNAME} TEST_INPUT_DIR=${TEST_INPUT_DIR} TEST_OUTPUT_DIR=${TEST_OUTPUT_DIR} TEST_PGHOST=${TEST_PGHOST} TEST_PGUSER=${TEST_PGUSER} TEST_ROLE_1=${TEST_ROLE_1} TEST_ROLE_2=${TEST_ROLE_2} TEST_ROLE_2_PASS=${TEST_ROLE_2_PASS} TEST_ROLE_3=${TEST_ROLE_3} TEST_ROLE_3_PASS=${TEST_ROLE_3_PASS} TEST_ROLE_4_PASS=${TEST_ROLE_4_PASS} TEST_ROLE_DEFAULT_PERM_USER=${TEST_ROLE_DEFAULT_PERM_USER} TEST_ROLE_DEFAULT_PERM_USER_2=${TEST_ROLE_DEFAULT_PERM_USER_2} TEST_ROLE_READ_ONLY=${TEST_ROLE_READ_ONLY} TEST_ROLE_SUPERUSER=${TEST_ROLE_SUPERUSER} TEST_SCHEDULE=${TEST_SCHEDULE}) endif() if(PG_ISOLATION_REGRESS) set(PG_ISOLATION_REGRESS_ENV PG_BINDIR=${PG_BINDIR} TEST_PGUSER=${TEST_PGUSER} TEST_ROLE_SUPERUSER=${TEST_ROLE_SUPERUSER} TEST_ROLE_DEFAULT_PERM_USER=${TEST_ROLE_DEFAULT_PERM_USER} TEST_ROLE_DEFAULT_PERM_USER_2=${TEST_ROLE_DEFAULT_PERM_USER_2} TEST_ROLE_1=${TEST_ROLE_1} TEST_ROLE_2=${TEST_ROLE_2} TEST_ROLE_3=${TEST_ROLE_3} TEST_ROLE_2_PASS=${TEST_2_PASS} TEST_ROLE_3_PASS=${TEST_3_PASS} TEST_ROLE_READ_ONLY=${TEST_ROLE_READ_ONLY} TEST_DBNAME=${TEST_DBNAME} TEST_INPUT_DIR=${TEST_INPUT_DIR} TEST_OUTPUT_DIR=${TEST_OUTPUT_DIR} TEST_SCHEDULE=${ISOLATION_TEST_SCHEDULE} PG_REGRESS=${PG_ISOLATION_REGRESS}) endif() set(TEST_VERSION_SUFFIX ${PG_VERSION_MAJOR}) if(APACHE_ONLY) set(TEST_LICENSE_SUFFIX "oss") else() set(TEST_LICENSE_SUFFIX "tsl") endif() ================================================ FILE: timescaledb.control.in ================================================ # timescaledb extension comment = 'Enables scalable inserts and complex queries for time-series data' default_version = '@PROJECT_VERSION_MOD@' module_pathname = '$libdir/timescaledb-@PROJECT_VERSION_MOD@' #extension cannot be relocatable once installed because it uses multiple schemas and that is forbidden by PG. #(though this extension is relocatable during installation). relocatable = false trusted = true ================================================ FILE: tsl/CMakeLists.txt ================================================ option(CODECOVERAGE "Enable fuzzing of compression using Libfuzzer" OFF) if(COMPRESSION_FUZZING) add_compile_definitions(TS_COMPRESSION_FUZZING=1) endif() # Add the subdirectories add_subdirectory(test) add_subdirectory(src) ================================================ FILE: tsl/LICENSE-TIMESCALE ================================================ TIMESCALE LICENSE AGREEMENT Posted Date: September 24, 2020 PLEASE READ CAREFULLY THIS TIMESCALE LICENSE AGREEMENT ("TSL Agreement"), WHICH CONSTITUTES A LEGALLY BINDING AGREEMENT AND GOVERNS USE OF THE TIMESCALE TIME-SERIES DATABASE SOFTWARE AND RELATED SOFTWARE THAT IS PROVIDED SUBJECT TO THIS TSL AGREEMENT. BY INSTALLING OR USING SUCH SOFTWARE, YOU AGREE THAT YOU HAVE READ AND AGREE TO BE BOUND BY THE TERMS AND CONDITIONS OF THIS TSL AGREEMENT. IF YOU DO NOT AGREE WITH SUCH TERMS AND CONDITIONS, YOU MAY NOT INSTALL OR USE SUCH SOFTWARE. IF YOU ARE INSTALLING OR USING SUCH SOFTWARE ON BEHALF OF A LEGAL ENTITY, YOU REPRESENT AND WARRANT THAT YOU HAVE THE AUTHORITY TO AGREE TO THE TERMS AND CONDITIONS OF THIS TSL AGREEMENT ON BEHALF OF THAT LEGAL ENTITY AND THE RIGHT TO BIND THAT LEGAL ENTITY TO THIS TSL AGREEMENT. This TSL Agreement is entered into by and between Timescale, Inc. ("Timescale") and you or the legal entity on whose behalf you are accepting this TSL Agreement ("You"). 0. BACKGROUND The Timescale time-series database software and related software is offered as "open code" or "source-available" code. This means that all source code of the software is available for inspection and download at https://github.com/timescale. The Timescale software is composed of two major pieces. The first piece (referred to herein as the Timescale Open Source Software, as defined below) is open source software that is licensed under the Apache Version 2.0 license. The second piece (referred to herein as the TSL Licensed Software, as defined below) is all of the Timescale Software other than the Timescale Open Source Software. The TSL Licensed Software may be used under this TSL Agreement without charge. 1. GOVERNING LICENSES 1.1 Source Code. The source code for all Timescale Software is made publicly available by Timescale at https://github.com/timescale. However, different license agreements govern the use of different parts of the Timescale Software source code. The use of Timescale Open Source Software, in both source and executable forms, is governed by the terms of the Apache License Version 2.0, a copy of which is available at https://opensource.org/licenses/Apache-2.0. The use of all other Timescale Software, in both source and executable forms, is governed by this TSL Agreement. 1.2 License Rights to Your Customers. As set forth in Section 2.1 below, the use by Your customers of the Timescale Software as part of any Value Added Products or Services that You distribute will be subject to the most current version of this TSL Agreement. 2. GRANT OF LICENSES 2.1 Grant. Conditioned upon compliance with all of the terms and conditions of this TSL Agreement, Timescale grants to You at no charge the following limited, non-exclusive, non-transferable, fully paid up, worldwide licenses, without the right to grant or authorize sublicenses (except as set forth in Section 2.3): (a) Internal Use. A license to copy, compile, install, and use the Timescale Software and Derivative Works solely for Your own internal business purposes in a manner that does not expose or give access to, directly or indirectly (e.g., via a wrapper), the Timescale Data Definition Interfaces or the Timescale Data Manipulation Interfaces to any person or entity other than You or Your employees and Contractors working on Your behalf. (b) Value Added Products or Services. A license (i) to copy, compile, install, and use the Timescale Software, Derivative Works, or parts thereof to develop and maintain Your Value Added Products or Services, (ii) to utilize (in the case of services) copies of the Timescale Software, Derivative Works, or parts thereof solely as incorporated into or utilized with Your Value Added Products or Services, and (iii) to distribute (in the case of products that are distributed to Your customers) copies of the Timescale Software binaries or of Derivative Works solely in binary form, and both solely as incorporated into or utilized with Your Value Added Products or Services; provided that (1) You notify Your customers that use of such Timescale Software or Derivative Works is subject to this TSL Agreement and You provide to each such customer a copy of the most current version of this TSL Agreement or a URL from which the most current version of this TSL Agreement may be obtained, and (2) the customer is prohibited, either contractually or technically, from defining, redefining, or modifying the database schema or other structural aspects of database objects, such as through use of the Timescale Data Definition Interfaces, in a Timescale Database utilized by such Value Added Products or Services. (c) Distribution of Source Code or Binaries in Standalone Form. Subject to the prohibitions in Section 2.2 below, a license to copy and distribute the Timescale Software source code and binaries solely in unmodified standalone form and subject to the terms and conditions of the most current version of this TSL Agreement. (d) Derivative Works. A license (i) to prepare, compile, and test Derivative Works of the TSL Licensed Software; (ii) to use Derivative Works for Internal Use solely as expressly permitted in Section 2.1(a); (iii) to utilize Derivative Works with Your Value Added Products or Services solely as expressly permitted in Section 2.1(b); (iv) to distribute Derivative Works in binary form with Your Value Added Products or Services solely as expressly permitted in Section 2.1(b); and (v) to distribute Derivative Works back to Timescale under Timescale's Contributor Agreement for potential incorporation into Timescale's maintained code base at its sole discretion. 2.2 Prohibitions. Notwithstanding any other provision in this TSL Agreement, You are prohibited from (i) using any TSL Licensed Software to provide time-sharing services or database-as-a-service services, or to provide any form of software-as-a-service or service offering in which the TSL Licensed Software is offered or made available to third parties to provide time-series database functions or operations, other than as part of Your Value Added Products or Services, or (ii) copying or distributing any TSL Licensed Software for use in any of the foregoing ways. In addition, You agree not to, except as expressly permitted in Section 2.1(d), prepare Derivative Works of any TSL Licensed Software or, except as expressly permitted herein, transfer, sell, rent, lease, sublicense, loan, or otherwise transfer or make available any TSL Licensed Software, whether in source code or binary executable form. 2.3 Affiliates and Contractors. You may permit Your Contractors and Affiliates to exercise the licenses set forth in Section 2.1, provided that such exercise by Contractors must be solely for your benefit and/or the benefit of Your Affiliates, and You shall be responsible for all acts and omissions of such Contractors and Affiliates in connection with such exercise of the licenses, including but not limited to breach of any terms of this TSL Agreement. 2.4 Reservation of Rights. Except as expressly set forth in Section 2.1, no other license or rights to the Timescale Software are granted to You under this TSL Agreement, whether by implication, estoppel, or otherwise. 3. DEFINITIONS In addition to other terms defined elsewhere in this TSL Agreement, the terms below have the following meanings: 3.1 "Affiliate" means, if You are a legal entity, any legal entity that controls, is controlled by, or which is under common control with, You, where "control" means ownership of at least fifty percent (50%) of the outstanding voting shares of the legal entity, or the contractual right to establish policy for, and manage the operations of, the legal entity. 3.2 "Contractor" means a person or entity engaged as a consultant or contractor to perform work on Your behalf, but only to the extent such person or entity is performing such work on Your behalf. 3.3 "Derivative Work" means any modification or enhancement made by You to the TSL Licensed Software, whether in source code, binary executable, intermediate, or other form. 3.4 "Timescale Database" means a time-series database that is created and/or used by the Timescale Software. 3.5 "Timescale Data Definition Interfaces" means SQL commands and other interfaces of the Timescale Software that can be used to define or modify the database schema and other structural aspects of database objects in a Timescale Database, including Data Definition Language (DDL) commands such as CREATE, DROP, ALTER, TRUNCATE, COMMENT, and RENAME. 3.6 "Timescale Data Manipulation Interfaces" means SQL commands and analytical function, procedural, and other types of application programming interfaces or commands, that allow the use, manipulation, and control of data present in a Timescale Database, including Data Manipulation Language (DML) commands such as SELECT, INSERT, UPDATE, and DELETE, Data Control Language (DCL) commands such as GRANT and REVOKE, and Transaction Control Language (TCL) commands such as COMMIT, ROLLBACK, SAVEPOINT, and SET TRANSACTION. 3.7 "Timescale Open Source Software" means those portions of the Timescale Software that Timescale makes publicly available for distribution from time to time as open source software under the terms of the Apache License Version 2.0 or, in some limited instances, under other open source licenses (such as the PostgreSQL license) as identified in the applicable source code files and/or accompanying notices. 3.8 "Timescale Software" means, collectively, all time-series database software and related software made publicly available by Timescale for distribution from time to time, in both source code and binary executable form, which includes the Timescale Open Source Software and the TSL Licensed Software. 3.9 "TSL Licensed Software" means those parts of the Timescale Software other than the Timescale Open Source Software. 3.10 "Value Added Products or Services" means products or services developed by or for You that utilize (for example, as a back-end function or part of a software stack) all or parts of the Timescale Software to provide time-series database storage and operations in support of larger value-added products or services (for example, an IoT platform or vertical-specific application) with respect to which all of the following are true: (i) such value-added products or services are not primarily database storage or operations products or services; (ii) such value-added products or services add substantial value of a different nature to the time-series database storage and operations afforded by the Timescale Software and are the key functions upon which such products or services are offered and marketed; and (iii) users of such Value Added Products or Services are prohibited, either contractually or technically, from defining, redefining, or modifying the database schema or other structural aspects of database objects, such as through use of the Timescale Data Definition Interfaces, in a Timescale Database utilized by such Value Added Products or Services. 4. TERMINATION This TSL Agreement will automatically terminate, whether or not You receive notice of such termination from Timescale, in the event You breach any of its terms or conditions. In accordance with Section 6 below, Timescale shall have no liability for any damage, loss, or expense of any kind, whether consequential, indirect, or direct, suffered or incurred by You arising from or incident to the termination of this TSL Agreement, whether or not Timescale has been advised or is aware of any such potential damage, loss, or expense. 5. DISCLAIMER OF WARRANTIES TO THE MAXIMUM EXTENT PERMITTED UNDER APPLICABLE LAW, ALL TIMESCALE SOFTWARE PROVIDED UNDER THIS TSL AGREEMENT, INCLUDING ALL PORTIONS OF THE TIMESCALE SOFTWARE SUPPLIED ON A TRIAL BASIS, ARE PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND AND TIMESCALE DISCLAIMS ALL SUCH WARRANTIES, WHETHER EXPRESS, STATUTORY, OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF MERCHANTABILITY, TITLE, FITNESS FOR A PARTICULAR PURPOSE, OR NON-INFRINGEMENT, AND ANY IMPLIED WARRANTIES ARISING FROM USAGE OF TRADE, COURSE OF DEALING, OR COURSE OF PERFORMANCE. WITHOUT LIMITING THE FOREGOING, TIMESCALE MAKES NO WARRANTY OR REPRESENTATION AS TO THE RELIABILITY, TIMELINESS, QUALITY, SUITABILITY, PROFITABILITY, SUPPORT, PERFORMANCE, LOSS OF USE OR LOSS OF DATA, AVAILABILITY, OR ACCURACY OF THE TIMESCALE SOFTWARE. YOU ACKNOWLEDGE THAT CHANGES MADE BY TIMESCALE TO THE TIMESCALE SOFTWARE MAY DISRUPT INTEROPERATION WITH YOUR VALUE ADDED PRODUCTS OR SERVICES. TIMESCALE AND ITS LICENSORS DO NOT WARRANT THAT THE TIMESCALE SOFTWARE, OR ANY PORTION THEREOF, IS ERROR FREE OR WILL OPERATE WITHOUT INTERRUPTION, OR THAT ANY VALUE ADDED PRODUCT OR SERVICE INTEROPERATING WITH THE TIMESCALE SOFTWARE WILL NOT EXPERIENCE LOSS OF USE OR LOSS OF DATA. YOU ACKNOWLEDGE THAT IN ENTERING INTO THIS TSL AGREEMENT, YOU HAVE NOT RELIED ON ANY PROMISE, WARRANTY, OR REPRESENTATION NOT EXPRESSLY SET FORTH IN THIS AGREEMENT. 6. LIMITATION OF LIABILITY TO THE MAXIMUM EXTENT PERMITTED UNDER APPLICABLE LAW, IN NO EVENT SHALL TIMESCALE OR ITS LICENSORS BE LIABLE TO YOU OR ANY THIRD PARTY FOR ANY DIRECT OR INDIRECT DAMAGES, INCLUDING BUT NOT LIMITED TO ANY LOSS OF PROFITS OR REVENUE, LOSS OF USE, BUSINESS INTERRUPTION, LOSS OF DATA, COST OF COVER OR SUBSTITUTE GOODS OR SERVICES, OR FOR ANY SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE, OR EXEMPLARY DAMAGES OF ANY KIND, HOWEVER CAUSED, RELATED TO, OR ARISING OUT OF THIS TSL AGREEMENT, ITS TERMINATION OR THE PERFORMANCE OR FAILURE TO PERFORM THIS TSL AGREEMENT, OR THE USE OR INABILITY TO USE THE TIMESCALE SOFTWARE, WHETHER ALLEGED AS A BREACH OF CONTRACT, BREACH OF WARRANTY, TORTIOUS CONDUCT, INCLUDING NEGLIGENCE, OR ANY OTHER LEGAL THEORY, EVEN IF TIMESCALE HAS BEEN ADVISED OR IS AWARE OF THE POSSIBILITY OF SUCH DAMAGES. 7. GENERAL 7.1 Complete Agreement. This TSL Agreement completely and exclusively states the entire agreement of the parties regarding the subject matter hereof and supersedes all prior proposals, agreements, or other communications between the parties, oral or written, regarding such subject matter. 7.2 Modification. This TSL Agreement may be modified by Timescale from time to time, and any such modifications will be effective upon the "Posted Date" set forth at the top of the modified agreement. The modified agreement shall govern any new version of the TSL Licensed Software (and all its constituent source code and binaries) that is officially released as a complete version release by Timescale on or after such Posted Date. Except as set forth in this Section 7.2, this TSL Agreement may not be amended except by a writing executed by both parties. 7.3 Governing Law. This TSL Agreement shall be governed by and construed solely under the laws of the State of New York, without application of any choice of law rules or principles that would lead to the applicability of the law of any other jurisdiction. None of the provisions of either the United Nations Convention on Contracts for the International Sale of Goods or the Uniform Computer Information Transactions Act shall apply. 7.4 Unenforceability. If any provision of this TSL Agreement is held unenforceable, the remaining provisions of this TSL Agreement shall remain in effect and the unenforceable provision shall be replaced by an enforceable provision that best reflects the original intent of the parties. 7.5 Injunctive Relief. You acknowledge that a breach or threatened breach of any provision of this TSL Agreement will cause irreparable harm to Timescale for which damages at law will not provide adequate relief, and Timescale shall therefore be entitled to injunctive relief against such breach or threatened breach without being required to post a bond. 7.6 Assignment. You may not assign this TSL Agreement, including by operation of law in connection with a merger or acquisition or otherwise, in whole or in part, without the prior written consent of Timescale, which Timescale may grant or withhold in its sole and absolute discretion. Any assignment in violation of the preceding sentence is void. 7.7 Independent Contractors. The parties to this TSL Agreement are independent contractors and this TSL Agreement does not establish any relationship of partnership, joint venture, employment, franchise, or agency between the parties. 7.8 U.S. Government Rights. The Timescale Software and related documentation are "Commercial Items", as that term is defined at 48 C.F.R. §2.101, consisting of "Commercial Computer Software" and "Commercial Computer Software Documentation," as such terms are used in 48 C.F.R. §12.212 or 48 C.F.R. §227.7202, as applicable, and are being licensed to U.S. Government end users (a) only as Commercial Items and (b) with only those rights as are granted to all other end users pursuant to the terms and conditions of this TSL Agreement. ================================================ FILE: tsl/README.md ================================================ ## TimescaleDB TSL Library ## The TimescaleDB TSL library is licensed under the [Timescale License](LICENSE-TIMESCALE). - [Continuous Aggregates](src/continuous_aggs/README.md) - [Compression](src/compression/README.md) - [Query optimization for time series](src/nodes/README.md) ================================================ FILE: tsl/src/CMakeLists.txt ================================================ set(SOURCES chunk_api.c chunk.c chunk_merge.c chunk_split.c chunkwise_agg.c init.c planner.c process_utility.c reorder.c) # Add test source code in Debug builds if(CMAKE_BUILD_TYPE MATCHES Debug) set(TS_DEBUG 1) set(DEBUG 1) endif(CMAKE_BUILD_TYPE MATCHES Debug) set(TSL_LIBRARY_NAME ${PROJECT_NAME}-tsl) include(build-defs.cmake) if(CMAKE_BUILD_TYPE MATCHES Debug OR COMPRESSION_FUZZING) add_library(${TSL_LIBRARY_NAME} MODULE ${SOURCES} $<TARGET_OBJECTS:${TSL_TESTS_LIB_NAME}>) else() add_library(${TSL_LIBRARY_NAME} MODULE ${SOURCES}) endif() set_target_properties( ${TSL_LIBRARY_NAME} PROPERTIES OUTPUT_NAME ${TSL_LIBRARY_NAME}-${PROJECT_VERSION_MOD} PREFIX "") target_include_directories(${TSL_LIBRARY_NAME} PRIVATE ${PG_INCLUDEDIR}) target_compile_definitions(${TSL_LIBRARY_NAME} PUBLIC TS_TSL) target_compile_definitions(${TSL_LIBRARY_NAME} PUBLIC TS_SUBMODULE) if(WIN32) target_link_libraries(${TSL_LIBRARY_NAME} ${PG_LIBDIR}/libpq.lib) else() target_link_libraries(${TSL_LIBRARY_NAME} pq) endif() install(TARGETS ${TSL_LIBRARY_NAME} DESTINATION ${PG_PKGLIBDIR}) # if (WIN32) target_link_libraries(${PROJECT_NAME} # ${PROJECT_NAME}-${PROJECT_VERSION_MOD}.lib) endif(WIN32) # We use the UMASH library for hashing in vectorized grouping. If it was not # explicitly disabled already, detect if we can compile it on this platform. if((NOT DEFINED USE_UMASH) OR USE_UMASH) # Check whether we can enable the pclmul instruction required for the UMASH # hashing on amd64. Shouldn't be done if the user has manually specified the # target architecture, no idea how to detect this, but at least we shouldn't # do this when cross-compiling. if(NOT CMAKE_CROSSCOMPILING) check_c_compiler_flag(-mpclmul CC_PCLMUL) if(CC_PCLMUL) add_compile_options(-mpclmul) endif() endif() # The C compiler flags that we add using add_compile_options() are not # automatically used by check_c_source_compiles() because it works in a # separate project, so we have to add them manually. get_directory_property(DIR_COMPILE_OPTIONS_LIST COMPILE_OPTIONS) list(JOIN DIR_COMPILE_OPTIONS_LIST " " DIR_COMPILE_OPTIONS) set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} ${CMAKE_C_FLAGS} ${DIR_COMPILE_OPTIONS} -Werror=implicit-function-declaration" ) check_c_source_compiles( " #if defined(__PCLMUL__) #include <stdint.h> #include <immintrin.h> /* * For some reason, this doesn't compile on our i386 CI, but I also can't detect * it using the standard condition of defined(__x86_64__) && !defined(__ILP32__), * as described at https://wiki.debian.org/X32Port . */ int main() { (void) _mm_cvtsi64_si128((uint64_t) 0); return 0; } #elif defined(__ARM_FEATURE_CRYPTO) /* OK */ int main() { return 0; } #else #error Unsupported platform for UMASH #endif " UMASH_SUPPORTED) unset(CMAKE_REQUIRED_FLAGS) else() set(UMASH_SUPPORTED OFF) endif() option(USE_UMASH "Use the UMASH hash for string and multi-column vectorized grouping" ${UMASH_SUPPORTED}) if(USE_UMASH) if(NOT UMASH_SUPPORTED) message( FATAL_ERROR "UMASH use is requested, but it is not supported in the current configuration" ) endif() add_compile_definitions(TS_USE_UMASH) endif() add_subdirectory(bgw_policy) add_subdirectory(compression) add_subdirectory(continuous_aggs) add_subdirectory(import) add_subdirectory(nodes) ================================================ FILE: tsl/src/README.module.md ================================================ # Submodule Licensing and Initialization # ## Loading and Activation ## We link module loading and activation to the license GUC itself. We have a single GUC, the license, and load submodules based on what capabilities the license enables, i.e., an `apache` license-key does not load this module, while `timescale` key does. This ensures that the loader "does the right thing" with respect to the license, and a user cannot accidentally activate features they aren't licensed to use. The actual loading and activation is done through `check` and `assign` hooks on the license GUC. On `check` we validate the license type and on `assign` we set the capabilities-struct in this module, if needed. The `check` and `assign` functions can be found in [`license_guc.c/h`](/src/license_guc.c) in the Apache-Licensed src. ### Cross License Functions ### To enable binaries which only contain Apache-Licensed code, we dynamically link in Timescale-Licensed code on license activation, and handle all function calls into the module via function pointers. The registry in `ts_cm_functions` of type `CrossModuleFunctions` (declared in [`cross_module_fn.h`](/src/cross_module_fn.h) and defined in [`cross_module_fn.c`](/src/cross_module_fn.c)) stores all of the cross-module functions. To add a new cross-module function you must: - Add a struct member `CrossModuleFunctions.<function name>`. - Add default function to `ts_cm_functions_default` that will be called from the Apache version, usually this function should just call `error_no_default_fn`. **NOTE** Due to function-pointer casting rules, the default function must have the exact same signature as the function pointer; you may _not_ cast another function pointer of another type. - Add the overriding function to `tsl_cm_functions`in `init.c` in this module. To call a cross-module functions use `ts_cm_functions-><function name>(args)`. ================================================ FILE: tsl/src/bgw_policy/CMakeLists.txt ================================================ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/compression_api.c ${CMAKE_CURRENT_SOURCE_DIR}/continuous_aggregate_api.c ${CMAKE_CURRENT_SOURCE_DIR}/process_hyper_inval_api.c ${CMAKE_CURRENT_SOURCE_DIR}/job.c ${CMAKE_CURRENT_SOURCE_DIR}/job_api.c ${CMAKE_CURRENT_SOURCE_DIR}/reorder_api.c ${CMAKE_CURRENT_SOURCE_DIR}/policy_config.c ${CMAKE_CURRENT_SOURCE_DIR}/retention_api.c ${CMAKE_CURRENT_SOURCE_DIR}/policy_utils.c ${CMAKE_CURRENT_SOURCE_DIR}/policies_v2.c) target_sources(${TSL_LIBRARY_NAME} PRIVATE ${SOURCES}) target_include_directories(${TSL_LIBRARY_NAME} PRIVATE ${CMAKE_SOURCE_DIR}) ================================================ FILE: tsl/src/bgw_policy/compression_api.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <fmgr.h> #include <miscadmin.h> #include <utils/builtins.h> #include "compression_api.h" #include "bgw/job.h" #include "bgw/job_stat.h" #include "bgw/timer.h" #include "bgw_policy/continuous_aggregate_api.h" #include "bgw_policy/job.h" #include "bgw_policy/job_api.h" #include "bgw_policy/policies_v2.h" #include "bgw_policy/policy_config.h" #include "errors.h" #include "guc.h" #include "hypertable.h" #include "hypertable_cache.h" #include "jsonb_utils.h" #include "policy_utils.h" #include "utils.h" #include <utils/elog.h> /* Default max runtime is unlimited for compress chunks */ #define DEFAULT_MAX_RUNTIME \ DatumGetIntervalP(DirectFunctionCall3(interval_in, CStringGetDatum("0"), InvalidOid, -1)) /* Default retry period for reorder_jobs is currently 1 hour */ #define DEFAULT_RETRY_PERIOD \ DatumGetIntervalP(DirectFunctionCall3(interval_in, CStringGetDatum("1 hour"), InvalidOid, -1)) /* Default max schedule period for the compression policy is 12 hours. The actual schedule period * will be chunk_interval/2 if the chunk_interval is < 12 hours. */ #define DEFAULT_MAX_SCHEDULE_PERIOD (int64)(12 * 3600 * 1000 * (int64) 1000) static Hypertable *validate_compress_chunks_hypertable(Cache *hcache, Oid user_htoid, bool *is_cagg); int32 policy_compression_get_maxchunks_per_job(const Jsonb *config) { bool found; int32 maxchunks = ts_jsonb_get_int32_field(config, POL_COMPRESSION_CONF_KEY_MAXCHUNKS_TO_COMPRESS, &found); return (found && maxchunks > 0) ? maxchunks : 0; } int64 policy_recompression_get_recompress_after_int(const Jsonb *config) { bool found; int64 compress_after = ts_jsonb_get_int64_field(config, POL_RECOMPRESSION_CONF_KEY_RECOMPRESS_AFTER, &found); if (!found) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not find %s in config for job", POL_RECOMPRESSION_CONF_KEY_RECOMPRESS_AFTER))); return compress_after; } Interval * policy_recompression_get_recompress_after_interval(const Jsonb *config) { Interval *interval = ts_jsonb_get_interval_field(config, POL_RECOMPRESSION_CONF_KEY_RECOMPRESS_AFTER); if (interval == NULL) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not find %s in config for job", POL_RECOMPRESSION_CONF_KEY_RECOMPRESS_AFTER))); return interval; } Datum policy_recompression_proc(PG_FUNCTION_ARGS) { if (PG_NARGS() != 2 || PG_ARGISNULL(0) || PG_ARGISNULL(1)) PG_RETURN_VOID(); ts_feature_flag_check(FEATURE_POLICY); TS_PREVENT_FUNC_IF_READ_ONLY(); policy_recompression_execute(PG_GETARG_INT32(0), PG_GETARG_JSONB_P(1)); PG_RETURN_VOID(); } static void validate_compress_after_type(const Dimension *dim, Oid partitioning_type, Oid compress_after_type) { Oid expected_type = InvalidOid; if (IS_INTEGER_TYPE(partitioning_type)) { Oid now_func = ts_get_integer_now_func(dim, false); if (!IS_INTEGER_TYPE(compress_after_type) && OidIsValid(now_func)) expected_type = partitioning_type; } else if (compress_after_type != INTERVALOID) { expected_type = INTERVALOID; } if (OidIsValid(expected_type)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported compress_after argument type, expected type : %s", format_type_be(expected_type)))); } } Datum policy_compression_check(PG_FUNCTION_ARGS) { PolicyCompressionData policy_data; if (PG_ARGISNULL(0)) { ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("config must not be NULL"))); } policy_compression_read_and_validate_config(PG_GETARG_JSONB_P(0), &policy_data); ts_cache_release(&policy_data.hcache); PG_RETURN_VOID(); } /* compression policies are added to hypertables or continuous aggregates */ Datum policy_compression_add_internal(Oid user_rel_oid, Datum compress_after_datum, Oid compress_after_type, Interval *created_before, Interval *default_schedule_interval, bool user_defined_schedule_interval, bool if_not_exists, bool fixed_schedule, TimestampTz initial_start, const char *timezone) { NameData application_name; NameData proc_name, proc_schema, check_schema, check_name, owner; int32 job_id; Hypertable *hypertable; Cache *hcache; const Dimension *dim; Oid owner_id; bool is_cagg = false; hcache = ts_hypertable_cache_pin(); hypertable = validate_compress_chunks_hypertable(hcache, user_rel_oid, &is_cagg); /* creation time usage not supported with caggs yet */ if (is_cagg && created_before != NULL) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot use \"compress_created_before\" with continuous aggregate \"%s\" ", get_rel_name(user_rel_oid)))); owner_id = ts_hypertable_permissions_check(user_rel_oid, GetUserId()); ts_bgw_job_validate_job_owner(owner_id); /* Make sure that an existing policy doesn't exist on this hypertable */ List *jobs = ts_bgw_job_find_by_proc_and_hypertable_id(POLICY_COMPRESSION_PROC_NAME, FUNCTIONS_SCHEMA_NAME, hypertable->fd.id); dim = hyperspace_get_open_dimension(hypertable->space, 0); Oid partitioning_type = ts_dimension_get_partition_type(dim); if (jobs != NIL) { bool is_equal = false; if (!if_not_exists) { ts_cache_release(&hcache); ereport(ERROR, (errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("columnstore policy already exists for hypertable or continuous " "aggregate \"%s\"", get_rel_name(user_rel_oid)), errhint("Set option \"if_not_exists\" to true to avoid error."))); } Assert(list_length(jobs) == 1); BgwJob *existing = linitial(jobs); if (OidIsValid(compress_after_type)) is_equal = policy_config_check_hypertable_lag_equality(existing->fd.config, POL_COMPRESSION_CONF_KEY_COMPRESS_AFTER, partitioning_type, compress_after_type, compress_after_datum, false /* isnull */); else { Assert(created_before != NULL); is_equal = policy_config_check_hypertable_lag_equality( existing->fd.config, POL_COMPRESSION_CONF_KEY_COMPRESS_CREATED_BEFORE, partitioning_type, INTERVALOID, IntervalPGetDatum(created_before), false /* isnull */); } if (is_equal) { /* If all arguments are the same, do nothing */ ts_cache_release(&hcache); ereport(NOTICE, (errmsg("columnstore policy already exists for hypertable \"%s\", skipping", get_rel_name(user_rel_oid)))); PG_RETURN_INT32(-1); } else { ts_cache_release(&hcache); ereport(WARNING, (errmsg("columnstore policy already exists for hypertable \"%s\"", get_rel_name(user_rel_oid)), errdetail("A policy already exists with different arguments."), errhint("Remove the existing policy before adding a new one."))); PG_RETURN_INT32(-1); } } if (created_before) { Assert(!OidIsValid(compress_after_type)); compress_after_type = INTERVALOID; } if (!is_cagg && IS_INTEGER_TYPE(partitioning_type) && !IS_INTEGER_TYPE(compress_after_type) && created_before == NULL) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid value for parameter %s", POL_COMPRESSION_CONF_KEY_COMPRESS_AFTER), errhint("Integer duration in \"compress_after\" or interval time duration" " in \"compress_created_before\" is required for hypertables with integer " "time dimension."))); if (dim && IS_TIMESTAMP_TYPE(ts_dimension_get_partition_type(dim)) && !user_defined_schedule_interval) { int64 hypertable_schedule_interval = dim->fd.interval_length / 2; /* On hypertables with a small chunk_time_interval, schedule the compression job more often * than DEFAULT_MAX_SCHEDULE_PERIOD */ if (DEFAULT_MAX_SCHEDULE_PERIOD > hypertable_schedule_interval) { default_schedule_interval = DatumGetIntervalP( ts_internal_to_interval_value(hypertable_schedule_interval, INTERVALOID)); } else { default_schedule_interval = DatumGetIntervalP( ts_internal_to_interval_value(DEFAULT_MAX_SCHEDULE_PERIOD, INTERVALOID)); } } /* insert a new job into jobs table */ namestrcpy(&application_name, "Columnstore Policy"); namestrcpy(&proc_name, POLICY_COMPRESSION_PROC_NAME); namestrcpy(&proc_schema, FUNCTIONS_SCHEMA_NAME); namestrcpy(&check_name, POLICY_COMPRESSION_CHECK_NAME); namestrcpy(&check_schema, FUNCTIONS_SCHEMA_NAME); namestrcpy(&owner, GetUserNameFromId(owner_id, false)); JsonbParseState *parse_state = NULL; pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); ts_jsonb_add_int32(parse_state, POLICY_CONFIG_KEY_HYPERTABLE_ID, hypertable->fd.id); validate_compress_after_type(dim, partitioning_type, compress_after_type); switch (compress_after_type) { case INTERVALOID: if (created_before) ts_jsonb_add_interval(parse_state, POL_COMPRESSION_CONF_KEY_COMPRESS_CREATED_BEFORE, created_before); else ts_jsonb_add_interval(parse_state, POL_COMPRESSION_CONF_KEY_COMPRESS_AFTER, DatumGetIntervalP(compress_after_datum)); break; case INT2OID: ts_jsonb_add_int64(parse_state, POL_COMPRESSION_CONF_KEY_COMPRESS_AFTER, DatumGetInt16(compress_after_datum)); break; case INT4OID: ts_jsonb_add_int64(parse_state, POL_COMPRESSION_CONF_KEY_COMPRESS_AFTER, DatumGetInt32(compress_after_datum)); break; case INT8OID: ts_jsonb_add_int64(parse_state, POL_COMPRESSION_CONF_KEY_COMPRESS_AFTER, DatumGetInt64(compress_after_datum)); break; default: ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported datatype for %s: %s", POL_COMPRESSION_CONF_KEY_COMPRESS_AFTER, format_type_be(compress_after_type)))); } JsonbValue *result = pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); Jsonb *config = JsonbValueToJsonb(result); job_id = ts_bgw_job_insert_relation(&application_name, default_schedule_interval, DEFAULT_MAX_RUNTIME, JOB_RETRY_UNLIMITED, DEFAULT_RETRY_PERIOD, &proc_schema, &proc_name, &check_schema, &check_name, owner_id, true, fixed_schedule, hypertable->fd.id, config, initial_start, timezone); if (!TIMESTAMP_NOT_FINITE(initial_start)) { ts_bgw_job_stat_upsert_next_start(job_id, initial_start); } ts_cache_release(&hcache); PG_RETURN_INT32(job_id); } /* compression policies are added to hypertables or continuous aggregates */ Datum policy_compression_add(PG_FUNCTION_ARGS) { /* * The function is not STRICT but we can't allow required args to be NULL * so we need to act like a strict function in those cases */ if (PG_ARGISNULL(0) || PG_ARGISNULL(2)) { ts_feature_flag_check(FEATURE_POLICY); PG_RETURN_NULL(); } Oid user_rel_oid = PG_GETARG_OID(0); Datum compress_after_datum = PG_GETARG_DATUM(1); Oid compress_after_type = PG_ARGISNULL(1) ? InvalidOid : get_fn_expr_argtype(fcinfo->flinfo, 1); bool if_not_exists = PG_GETARG_BOOL(2); bool user_defined_schedule_interval = !(PG_ARGISNULL(3)); Interval *default_schedule_interval = PG_ARGISNULL(3) ? DEFAULT_COMPRESSION_SCHEDULE_INTERVAL : PG_GETARG_INTERVAL_P(3); TimestampTz initial_start = PG_ARGISNULL(4) ? DT_NOBEGIN : PG_GETARG_TIMESTAMPTZ(4); // if not providing initial_start, then we still get the old behavior bool fixed_schedule = !PG_ARGISNULL(4); text *timezone = PG_ARGISNULL(5) ? NULL : PG_GETARG_TEXT_PP(5); char *valid_timezone = NULL; Interval *created_before = PG_GETARG_INTERVAL_P(6); ts_feature_flag_check(FEATURE_POLICY); TS_PREVENT_FUNC_IF_READ_ONLY(); /* compress_after and created_before cannot be specified [or omitted] together */ if ((PG_ARGISNULL(1) && PG_ARGISNULL(6)) || (!PG_ARGISNULL(1) && !PG_ARGISNULL(6))) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg( "need to specify one of \"compress_after\" or \"compress_created_before\""))); /* if users pass in -infinity for initial_start, then use the current_timestamp instead */ if (fixed_schedule) { ts_bgw_job_validate_schedule_interval(default_schedule_interval); if (TIMESTAMP_NOT_FINITE(initial_start)) initial_start = ts_timer_get_current_timestamp(); } if (timezone != NULL) valid_timezone = ts_bgw_job_validate_timezone(PG_GETARG_DATUM(5)); Datum retval; retval = policy_compression_add_internal(user_rel_oid, compress_after_datum, compress_after_type, created_before, default_schedule_interval, user_defined_schedule_interval, if_not_exists, fixed_schedule, initial_start, valid_timezone); return retval; } bool policy_compression_remove_internal(Oid user_rel_oid, bool if_exists) { Hypertable *ht; Cache *hcache; ht = ts_hypertable_cache_get_cache_and_entry(user_rel_oid, CACHE_FLAG_MISSING_OK, &hcache); if (!ht) { const char *view_name = get_rel_name(user_rel_oid); if (!view_name) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("relation is not a hypertable or continuous aggregate"))); else { ContinuousAgg *ca = ts_continuous_agg_find_by_relid(user_rel_oid); if (!ca) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("relation \"%s\" is not a hypertable or continuous aggregate", view_name))); ht = ts_hypertable_get_by_id(ca->data.mat_hypertable_id); } } List *jobs = ts_bgw_job_find_by_proc_and_hypertable_id(POLICY_COMPRESSION_PROC_NAME, FUNCTIONS_SCHEMA_NAME, ht->fd.id); ts_cache_release(&hcache); if (jobs == NIL) { if (!if_exists) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("columnstore policy not found for hypertable \"%s\"", get_rel_name(user_rel_oid)))); else { ereport(NOTICE, (errmsg("columnstore policy not found for hypertable \"%s\", skipping", get_rel_name(user_rel_oid)))); PG_RETURN_BOOL(false); } } ts_hypertable_permissions_check(user_rel_oid, GetUserId()); Assert(list_length(jobs) == 1); BgwJob *job = linitial(jobs); ts_bgw_job_delete_by_id(job->fd.id); PG_RETURN_BOOL(true); } /* remove compression policy from ht or cagg */ Datum policy_compression_remove(PG_FUNCTION_ARGS) { Oid user_rel_oid = PG_GETARG_OID(0); bool if_exists = PG_GETARG_BOOL(1); ts_feature_flag_check(FEATURE_POLICY); TS_PREVENT_FUNC_IF_READ_ONLY(); return policy_compression_remove_internal(user_rel_oid, if_exists); } /* compare cagg job config with compression job config. If there is an overlap, then * throw an error. We do this since we cannot refresh compressed * regions. We do not want cont. aggregate jobs to fail */ /* If this is a cagg, then mark it as a cagg */ static Hypertable * validate_compress_chunks_hypertable(Cache *hcache, Oid user_htoid, bool *is_cagg) { ContinuousAggHypertableStatus status; Hypertable *ht = ts_hypertable_cache_get_entry(hcache, user_htoid, true /* missing_ok */); *is_cagg = false; if (ht != NULL) { if (!TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(ht)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("columnstore not enabled on hypertable \"%s\"", get_rel_name(user_htoid)), errhint("Enable columnstore before adding a columnstore policy."))); } status = ts_continuous_agg_hypertable_status(ht->fd.id); if ((status == HypertableIsMaterialization || status == HypertableIsMaterializationAndRaw)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot add compression policy to materialized hypertable \"%s\" ", get_rel_name(user_htoid)), errhint("Please add the policy to the corresponding continuous aggregate " "instead."))); } } else { /*check if this is a cont aggregate view */ int32 mat_id; bool found; ContinuousAgg *cagg = ts_continuous_agg_find_by_relid(user_htoid); if (cagg == NULL) { ts_cache_release(&hcache); const char *relname = get_rel_name(user_htoid); if (relname) ereport(ERROR, (errcode(ERRCODE_TS_HYPERTABLE_NOT_EXIST), errmsg("\"%s\" is not a hypertable or a continuous aggregate", relname))); else ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("object with id \"%u\" not found", user_htoid))); } *is_cagg = true; mat_id = cagg->data.mat_hypertable_id; ht = ts_hypertable_get_by_id(mat_id); found = policy_refresh_cagg_exists(mat_id); if (!found) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("continuous aggregate policy does not exist for \"%s\"", get_rel_name(user_htoid)), errmsg("setup a refresh policy for \"%s\" before setting up a columnstore " "policy", get_rel_name(user_htoid)))); } if (!TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(ht)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("columnstore not enabled on continuous aggregate \"%s\"", get_rel_name(user_htoid)), errhint("Enable columnstore before adding a columnstore policy."))); } } Assert(ht != NULL); return ht; } ================================================ FILE: tsl/src/bgw_policy/compression_api.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <utils/jsonb.h> #include <utils/timestamp.h> /* User-facing API functions */ extern Datum policy_compression_add(PG_FUNCTION_ARGS); extern Datum policy_compression_remove(PG_FUNCTION_ARGS); extern Datum policy_recompression_proc(PG_FUNCTION_ARGS); extern Datum policy_compression_check(PG_FUNCTION_ARGS); int32 policy_compression_get_hypertable_id(const Jsonb *config); int32 policy_compression_get_maxchunks_per_job(const Jsonb *config); int64 policy_recompression_get_recompress_after_int(const Jsonb *config); Interval *policy_recompression_get_recompress_after_interval(const Jsonb *config); Datum policy_compression_add_internal(Oid user_rel_oid, Datum compress_after_datum, Oid compress_after_type, Interval *created_before, Interval *default_schedule_interval, bool user_defined_schedule_interval, bool if_not_exists, bool fixed_schedule, TimestampTz initial_start, const char *timezone); bool policy_compression_remove_internal(Oid user_rel_oid, bool if_exists); ================================================ FILE: tsl/src/bgw_policy/continuous_aggregate_api.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <common/int128.h> #include <miscadmin.h> #include <parser/parse_coerce.h> #include <utils/acl.h> #include <utils/builtins.h> #include <utils/rangetypes.h> #include "bgw/job.h" #include "bgw/job_stat.h" #include "bgw/timer.h" #include "bgw_policy/continuous_aggregate_api.h" #include "bgw_policy/job.h" #include "bgw_policy/job_api.h" #include "bgw_policy/policies_v2.h" #include "bgw_policy/policy_utils.h" #include "dimension.h" #include "guc.h" #include "jsonb_utils.h" #include "policy_config.h" #include "policy_utils.h" #include "time_utils.h" #include "ts_catalog/continuous_agg.h" /* Default max runtime for a continuous aggregate jobs is unlimited for now */ #define DEFAULT_MAX_RUNTIME \ DatumGetIntervalP(DirectFunctionCall3(interval_in, CStringGetDatum("0"), InvalidOid, -1)) /* Default buckets per batch is 1, which means that the job will refresh 1 bucket at a time */ #define DEFAULT_BUCKETS_PER_BATCH 10 /* Default max batches per execution is 0, which means no limit */ #define DEFAULT_MAX_BATCHES_PER_EXECUTION 0 /* Default refresh newest first is true, which means from newest data to the oldest */ #define DEFAULT_REFRESH_NEWEST_FIRST true int32 policy_continuous_aggregate_get_mat_hypertable_id(const Jsonb *config) { bool found; int32 mat_hypertable_id = ts_jsonb_get_int32_field(config, POL_REFRESH_CONF_KEY_MAT_HYPERTABLE_ID, &found); if (!found) ereport(ERROR, (errcode(ERRCODE_SQL_JSON_MEMBER_NOT_FOUND), errmsg("could not find \"%s\" in config for job", POL_REFRESH_CONF_KEY_MAT_HYPERTABLE_ID))); return mat_hypertable_id; } static int64 get_time_from_interval(const Dimension *dim, Datum interval, Oid type) { Oid partitioning_type = ts_dimension_get_partition_type(dim); if (IS_INTEGER_TYPE(type)) { Oid now_func = ts_get_integer_now_func(dim, true); int64 value = ts_interval_value_to_internal(interval, type); Assert(now_func); return ts_subtract_integer_from_now_saturating(now_func, value, partitioning_type); } else if (type == INTERVALOID) { Datum res = ts_subtract_interval_from_now(DatumGetIntervalP(interval), partitioning_type); return ts_time_value_to_internal(res, partitioning_type); } else elog(ERROR, "unsupported offset type for continuous aggregate policy"); pg_unreachable(); return 0; } static int64 get_time_from_config(const Dimension *dim, const Jsonb *config, const char *json_label, bool *isnull) { Oid partitioning_type = ts_dimension_get_partition_type(dim); *isnull = false; if (IS_INTEGER_TYPE(partitioning_type)) { bool found; int64 interval_val = ts_jsonb_get_int64_field(config, json_label, &found); if (!found) { *isnull = true; return 0; } return get_time_from_interval(dim, Int64GetDatum(interval_val), INT8OID); } else { Interval *interval_val = ts_jsonb_get_interval_field(config, json_label); if (!interval_val) { *isnull = true; return 0; } return get_time_from_interval(dim, IntervalPGetDatum(interval_val), INTERVALOID); } } int64 policy_refresh_cagg_get_refresh_start(const ContinuousAgg *cagg, const Dimension *dim, const Jsonb *config, bool *start_isnull) { int64 res = get_time_from_config(dim, config, POL_REFRESH_CONF_KEY_START_OFFSET, start_isnull); /* interpret NULL as min value for that type */ if (*start_isnull) { Assert(cagg->partition_type == ts_dimension_get_partition_type(dim)); return cagg_get_time_min(cagg); } return res; } int64 policy_refresh_cagg_get_refresh_end(const Dimension *dim, const Jsonb *config, bool *end_isnull) { int64 res = get_time_from_config(dim, config, POL_REFRESH_CONF_KEY_END_OFFSET, end_isnull); if (*end_isnull) return ts_time_get_noend_or_max(ts_dimension_get_partition_type(dim)); return res; } bool policy_refresh_cagg_get_include_tiered_data(const Jsonb *config, bool *isnull) { bool found; bool res = ts_jsonb_get_bool_field(config, POL_REFRESH_CONF_KEY_INCLUDE_TIERED_DATA, &found); *isnull = !found; return res; } int32 policy_refresh_cagg_get_buckets_per_batch(const Jsonb *config) { bool found; int32 res = ts_jsonb_get_int32_field(config, POL_REFRESH_CONF_KEY_BUCKETS_PER_BATCH, &found); if (!found) res = DEFAULT_BUCKETS_PER_BATCH; /* default value */ return res; } int32 policy_refresh_cagg_get_max_batches_per_execution(const Jsonb *config) { bool found; int32 res = ts_jsonb_get_int32_field(config, POL_REFRESH_CONF_KEY_MAX_BATCHES_PER_EXECUTION, &found); if (!found) res = DEFAULT_MAX_BATCHES_PER_EXECUTION; /* default value */ return res; } bool policy_refresh_cagg_get_refresh_newest_first(const Jsonb *config) { bool found; bool res = ts_jsonb_get_bool_field(config, POL_REFRESH_CONF_KEY_REFRESH_NEWEST_FIRST, &found); if (!found) res = DEFAULT_REFRESH_NEWEST_FIRST; /* default value */ return res; } /* returns false if a policy could not be found */ bool policy_refresh_cagg_exists(int32 materialization_id) { Hypertable *mat_ht = ts_hypertable_get_by_id(materialization_id); if (!mat_ht) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("configuration materialization hypertable id %d not found", materialization_id))); List *jobs = ts_bgw_job_find_by_proc_and_hypertable_id(POLICY_REFRESH_CAGG_PROC_NAME, FUNCTIONS_SCHEMA_NAME, materialization_id); if (jobs == NIL) return false; return true; } Datum policy_refresh_cagg_proc(PG_FUNCTION_ARGS) { if (PG_NARGS() != 2 || PG_ARGISNULL(0) || PG_ARGISNULL(1)) PG_RETURN_VOID(); ts_feature_flag_check(FEATURE_POLICY); TS_PREVENT_FUNC_IF_READ_ONLY(); policy_refresh_cagg_execute(PG_GETARG_INT32(0), PG_GETARG_JSONB_P(1)); PG_RETURN_VOID(); } Datum policy_refresh_cagg_check(PG_FUNCTION_ARGS) { if (PG_ARGISNULL(0)) { ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("config must not be NULL"))); } policy_refresh_cagg_read_and_validate_config(PG_GETARG_JSONB_P(0), NULL); PG_RETURN_VOID(); } static void json_add_dim_interval_value(JsonbParseState *parse_state, const char *json_label, Oid dim_type, Datum value) { switch (dim_type) { case INTERVALOID: ts_jsonb_add_interval(parse_state, json_label, DatumGetIntervalP(value)); break; case INT2OID: ts_jsonb_add_int64(parse_state, json_label, DatumGetInt16(value)); break; case INT4OID: ts_jsonb_add_int64(parse_state, json_label, DatumGetInt32(value)); break; case INT8OID: ts_jsonb_add_int64(parse_state, json_label, DatumGetInt64(value)); break; default: ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported interval argument type, expected type : %s", format_type_be(dim_type)))); } } static Datum convert_interval_arg(Oid dim_type, Datum interval, Oid *interval_type, const char *str_msg) { Oid convert_to = dim_type; Datum converted; if (IS_TIMESTAMP_TYPE(dim_type)) convert_to = INTERVALOID; if (*interval_type != convert_to) { if (!can_coerce_type(1, interval_type, &convert_to, COERCION_IMPLICIT)) { if (IS_INTEGER_TYPE(dim_type)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid parameter value for %s", str_msg), errhint("Use time interval of type %s with the continuous aggregate.", format_type_be(dim_type)))); else if (IS_TIMESTAMP_TYPE(dim_type)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid parameter value for %s", str_msg), errhint("Use time interval with a continuous aggregate using " "timestamp-based time " "bucket."))); } } converted = ts_time_datum_convert_arg(interval, interval_type, convert_to); /* For integer types, first convert all types to int64 to get on a common * type. Then check valid time ranges against the partition/dimension * type */ switch (*interval_type) { case INT2OID: converted = Int64GetDatum((int64) DatumGetInt16(converted)); break; case INT4OID: converted = Int64GetDatum((int64) DatumGetInt32(converted)); break; case INT8OID: break; case INTERVALOID: /* For timestamp types, we only support Interval, so nothing further * to do. */ return converted; default: pg_unreachable(); break; } /* Cap at min and max */ if (DatumGetInt64(converted) < ts_time_get_min(dim_type)) converted = ts_time_get_min(dim_type); else if (DatumGetInt64(converted) > ts_time_get_max(dim_type)) converted = ts_time_get_max(dim_type); /* Convert to the desired integer type */ switch (dim_type) { case INT2OID: converted = Int16GetDatum((int16) DatumGetInt64(converted)); break; case INT4OID: converted = Int32GetDatum((int32) DatumGetInt64(converted)); break; case INT8OID: /* Already int64, so nothing to do. */ break; default: pg_unreachable(); break; } *interval_type = dim_type; return converted; } /* * Convert an interval to a 128 integer value. * * Based on PostgreSQL's interval_cmp_value(). */ static inline INT128 interval_to_int128(const Interval *interval) { INT128 span; int64 dayfraction; int64 days; /* * Separate time field into days and dayfraction, then add the month and * day fields to the days part. We cannot overflow int64 days here. */ dayfraction = interval->time % USECS_PER_DAY; days = interval->time / USECS_PER_DAY; days += interval->month * INT64CONST(30); days += interval->day; /* Widen dayfraction to 128 bits */ span = int64_to_int128(dayfraction); /* Scale up days to microseconds, forming a 128-bit product */ int128_add_int64_mul_int64(&span, days, USECS_PER_DAY); return span; } int64 interval_to_int64(Datum interval, Oid type) { switch (type) { case INT2OID: return DatumGetInt16(interval); case INT4OID: return DatumGetInt32(interval); case INT8OID: return DatumGetInt64(interval); case INTERVALOID: { const int64 max = ts_time_get_max(TIMESTAMPTZOID); const int64 min = ts_time_get_min(TIMESTAMPTZOID); INT128 bigres = interval_to_int128(DatumGetIntervalP(interval)); if (int128_compare(bigres, int64_to_int128(max)) >= 0) return max; else if (int128_compare(bigres, int64_to_int128(min)) <= 0) return min; else return int128_to_int64(bigres); } default: break; } ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported interval argument type: %s", format_type_be(type)))); /* Needed to make windows compiler happy */ pg_unreachable(); } /* * Enforce that a policy has a refresh window of at least two buckets to * ensure we materialize at least one bucket each run. * * Why two buckets? Note that the policy probably won't execute at at time * that exactly aligns with a bucket boundary, so a window of one bucket * might not cover a full bucket that we want to materialize: * * Refresh window: [-----) * Materialized buckets: |-----|-----|-----| */ static void validate_window_size(const ContinuousAgg *cagg, const CaggPolicyConfig *config) { int64 start_offset; int64 end_offset; int64 bucket_width; if (config->offset_start.isnull) start_offset = ts_time_get_max(cagg->partition_type); else start_offset = interval_to_int64(config->offset_start.value, config->offset_start.type); if (config->offset_end.isnull) end_offset = ts_time_get_min(cagg->partition_type); else end_offset = interval_to_int64(config->offset_end.value, config->offset_end.type); bucket_width = ts_continuous_agg_bucket_width(cagg->bucket_function); Assert(bucket_width > 0); if (ts_time_saturating_add(end_offset, bucket_width * 2, INT8OID) > start_offset) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("policy refresh window too small"), errdetail("The start and end offsets must cover at least" " two buckets in the valid time range of type \"%s\".", format_type_be(cagg->partition_type)))); } static void parse_offset_arg(const ContinuousAgg *cagg, Oid offset_type, NullableDatum arg, ContinuousAggPolicyOffset *offset) { offset->isnull = arg.isnull; if (!offset->isnull) { offset->value = convert_interval_arg(cagg->partition_type, arg.value, &offset_type, offset->name); offset->type = offset_type; } } static void parse_cagg_policy_config(const ContinuousAgg *cagg, Oid start_offset_type, NullableDatum start_offset, Oid end_offset_type, NullableDatum end_offset, CaggPolicyConfig *config) { MemSet(config, 0, sizeof(CaggPolicyConfig)); config->partition_type = cagg->partition_type; /* This might seem backwards, but since we are dealing with offsets, start * actually translates to max and end to min for maximum window. */ config->offset_start.value = ts_time_datum_get_max(config->partition_type); config->offset_end.value = ts_time_datum_get_min(config->partition_type); config->offset_start.type = config->offset_end.type = IS_TIMESTAMP_TYPE(cagg->partition_type) ? INTERVALOID : cagg->partition_type; config->offset_start.name = POL_REFRESH_CONF_KEY_START_OFFSET; config->offset_end.name = POL_REFRESH_CONF_KEY_END_OFFSET; parse_offset_arg(cagg, start_offset_type, start_offset, &config->offset_start); parse_offset_arg(cagg, end_offset_type, end_offset, &config->offset_end); Assert(config->offset_start.type == config->offset_end.type); validate_window_size(cagg, config); } bool policy_refresh_cagg_check_if_last_policy(PolicyContinuousAggData *policy_data) { ContinuousAgg *cagg = policy_data->cagg; int64 end_offset = policy_data->refresh_window.end; bool end_isnull = policy_data->refresh_window.end_isnull; if (end_isnull) return true; Hypertable *mat_ht = ts_hypertable_get_by_id(cagg->data.mat_hypertable_id); const Dimension *dim = get_open_dimension_for_hypertable(mat_ht, true); List *jobs = ts_bgw_job_find_by_proc_and_hypertable_id(POLICY_REFRESH_CAGG_PROC_NAME, FUNCTIONS_SCHEMA_NAME, cagg->data.mat_hypertable_id); ListCell *lc; /* We need to go through all jobs in order to determine if there is a job which starts after * this one */ foreach (lc, jobs) { BgwJob *job = (BgwJob *) lfirst(lc); bool end_offset_job_isnull; int64 end_offset_job = policy_refresh_cagg_get_refresh_end(dim, job->fd.config, &end_offset_job_isnull); if (end_offset_job_isnull || end_offset < end_offset_job) { return false; } } return true; } /* Ensures the refresh range of the new policy doesn't overlap with an existing one*/ PolicyRefreshOffsetOverlapResult policy_refresh_cagg_check_for_overlaps(ContinuousAgg *cagg, Jsonb *policy_config, int32 existing_job_id) { List *jobs = ts_bgw_job_find_by_proc_and_hypertable_id(POLICY_REFRESH_CAGG_PROC_NAME, FUNCTIONS_SCHEMA_NAME, cagg->data.mat_hypertable_id); PolicyRefreshOffsetOverlapResult overlap_result = POLICY_REFRESH_OFFSET_OVERLAP_NONE; if (jobs == NIL) return overlap_result; Hypertable *mat_ht = ts_hypertable_get_by_id(cagg->data.mat_hypertable_id); const Dimension *dim = get_open_dimension_for_hypertable(mat_ht, true); bool start_offset_isnull, end_offset_isnull; int64 start_offset = policy_refresh_cagg_get_refresh_start(cagg, dim, policy_config, &start_offset_isnull); int64 end_offset = policy_refresh_cagg_get_refresh_end(dim, policy_config, &end_offset_isnull); RangeBound lower = { .val = Int64GetDatum(start_offset), .infinite = start_offset_isnull, .inclusive = true, .lower = true, }; RangeBound upper = { .val = Int64GetDatum(end_offset), .infinite = end_offset_isnull, .inclusive = false, .lower = false, }; TypeCacheEntry *typcache = lookup_type_cache(INT8RANGEOID, TYPECACHE_RANGE_INFO); if (typcache == NULL || typcache->rngelemtype == NULL) elog(ERROR, "cache lookup failed"); RangeType *range = make_range_compat(typcache, &lower, &upper, false, NULL); ListCell *lc; elog(DEBUG1, "start_offset: " INT64_FORMAT ", end_offset: " INT64_FORMAT, start_offset, end_offset); /* We need to go through all jobs in order to determine if there is an existing job with the * exact same offsets */ foreach (lc, jobs) { BgwJob *job = (BgwJob *) lfirst(lc); if (existing_job_id == job->fd.id) { continue; } bool start_offset_job_isnull, end_offset_job_isnull; int64 start_offset_job = policy_refresh_cagg_get_refresh_start(cagg, dim, job->fd.config, &start_offset_job_isnull); int64 end_offset_job = policy_refresh_cagg_get_refresh_end(dim, job->fd.config, &end_offset_job_isnull); RangeBound lower_job = { .val = Int64GetDatum(start_offset_job), .infinite = start_offset_job_isnull, .inclusive = true, .lower = true, }; RangeBound upper_job = { .val = Int64GetDatum(end_offset_job), .infinite = end_offset_job_isnull, .inclusive = false, .lower = false, }; RangeType *range_job = make_range_compat(typcache, &lower_job, &upper_job, false, NULL); elog(DEBUG1, "start_offset_job: " INT64_FORMAT ", end_offset_job: " INT64_FORMAT, start_offset_job, end_offset_job); /* Check if exact same job exists, in which case throw an error or notice depending on * `if_not_exists` */ if (start_offset == start_offset_job && end_offset == end_offset_job) { /* If all arguments are the same, do nothing */ return POLICY_REFRESH_OFFSET_OVERLAP_EQUAL; } /* We need to first check all other jobs to see if there is an exact match, since we prefer * returning an exact match over an overlap, and the list of jobs isn't guaranteed to be * sorted by start/end offset. */ else if (range_overlaps_internal(typcache, range_job, range)) { overlap_result = POLICY_REFRESH_OFFSET_OVERLAP; } } /* We cannot check if the CAgg is hierarchical first and abort early since * we need to respect the if_not_exists parameter that is passed in. * So we check for overlap first, and only if there is no exact match, we block multiple * policies on hierarchical caggs */ if (ContinuousAggIsHierarchical(cagg)) { /* if this is an existing job, it will also be in the list of jobs */ int max_concurrent = existing_job_id ? 1 : 0; if (list_length(jobs) > max_concurrent) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("multiple refresh policies are not supported for hierarchical " "continuous aggregates"))); } return overlap_result; } Datum policy_refresh_cagg_add_internal(Oid cagg_oid, Oid start_offset_type, NullableDatum start_offset, Oid end_offset_type, NullableDatum end_offset, Interval refresh_interval, bool if_not_exists, bool fixed_schedule, TimestampTz initial_start, const char *timezone, NullableDatum include_tiered_data, NullableDatum buckets_per_batch, NullableDatum max_batches_per_execution, NullableDatum refresh_newest_first) { NameData application_name; NameData proc_name, proc_schema, check_name, check_schema, owner; ContinuousAgg *cagg; CaggPolicyConfig policyconf; int32 job_id; Oid owner_id; JsonbParseState *parse_state = NULL; /* Verify that the owner can create a background worker */ owner_id = ts_cagg_permissions_check(cagg_oid, GetUserId()); ts_bgw_job_validate_job_owner(owner_id); cagg = ts_continuous_agg_find_by_relid(cagg_oid); if (!cagg) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("\"%s\" is not a continuous aggregate", get_rel_name(cagg_oid)))); if (!start_offset.isnull) start_offset.isnull = ts_if_offset_is_infinity(start_offset.value, start_offset_type, true /* is_start */); if (!end_offset.isnull) end_offset.isnull = ts_if_offset_is_infinity(end_offset.value, end_offset_type, false /* is_start */); parse_cagg_policy_config(cagg, start_offset_type, start_offset, end_offset_type, end_offset, &policyconf); /* Insert a new job into jobs table */ namestrcpy(&application_name, "Refresh Continuous Aggregate Policy"); namestrcpy(&proc_name, POLICY_REFRESH_CAGG_PROC_NAME); namestrcpy(&proc_schema, FUNCTIONS_SCHEMA_NAME); namestrcpy(&check_name, POLICY_REFRESH_CAGG_CHECK_NAME); namestrcpy(&check_schema, FUNCTIONS_SCHEMA_NAME); namestrcpy(&owner, GetUserNameFromId(owner_id, false)); pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); ts_jsonb_add_int32(parse_state, POL_REFRESH_CONF_KEY_MAT_HYPERTABLE_ID, cagg->data.mat_hypertable_id); if (!policyconf.offset_start.isnull) json_add_dim_interval_value(parse_state, POL_REFRESH_CONF_KEY_START_OFFSET, policyconf.offset_start.type, policyconf.offset_start.value); else ts_jsonb_add_null(parse_state, POL_REFRESH_CONF_KEY_START_OFFSET); if (!policyconf.offset_end.isnull) json_add_dim_interval_value(parse_state, POL_REFRESH_CONF_KEY_END_OFFSET, policyconf.offset_end.type, policyconf.offset_end.value); else ts_jsonb_add_null(parse_state, POL_REFRESH_CONF_KEY_END_OFFSET); if (!include_tiered_data.isnull) ts_jsonb_add_bool(parse_state, POL_REFRESH_CONF_KEY_INCLUDE_TIERED_DATA, include_tiered_data.value); if (!buckets_per_batch.isnull) ts_jsonb_add_int32(parse_state, POL_REFRESH_CONF_KEY_BUCKETS_PER_BATCH, buckets_per_batch.value); if (!max_batches_per_execution.isnull) ts_jsonb_add_int32(parse_state, POL_REFRESH_CONF_KEY_MAX_BATCHES_PER_EXECUTION, max_batches_per_execution.value); if (!refresh_newest_first.isnull) ts_jsonb_add_bool(parse_state, POL_REFRESH_CONF_KEY_REFRESH_NEWEST_FIRST, refresh_newest_first.value); JsonbValue *result = pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); Jsonb *config = JsonbValueToJsonb(result); PolicyRefreshOffsetOverlapResult res = policy_refresh_cagg_check_for_overlaps(cagg, config, 0); switch (res) { case POLICY_REFRESH_OFFSET_OVERLAP_NONE: break; case POLICY_REFRESH_OFFSET_OVERLAP_EQUAL: if (if_not_exists) { ereport(NOTICE, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("continuous aggregate refresh policy already exists for " "\"%s\", skipping", get_rel_name(cagg->relid)), errdetail("A refresh policy with the same start and end offset already " "exists for continuous aggregate \"%s\".", get_rel_name(cagg->relid)))); PG_RETURN_INT32(-1); } ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("continuous aggregate refresh policy already exists for " "\"%s\"", get_rel_name(cagg->relid)), errdetail("A refresh policy with the same start and end offset already exists " "for " "continuous aggregate \"%s\".", get_rel_name(cagg->relid)))); break; case POLICY_REFRESH_OFFSET_OVERLAP: ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("refresh interval overlaps with an existing continuous aggregate " "policy on \"%s\"", get_rel_name(cagg->relid)))); break; } job_id = ts_bgw_job_insert_relation(&application_name, &refresh_interval, DEFAULT_MAX_RUNTIME, JOB_RETRY_UNLIMITED, &refresh_interval, &proc_schema, &proc_name, &check_schema, &check_name, owner_id, true, fixed_schedule, cagg->data.mat_hypertable_id, config, initial_start, timezone); PG_RETURN_INT32(job_id); } Datum policy_refresh_cagg_add(PG_FUNCTION_ARGS) { Oid cagg_oid, start_offset_type, end_offset_type; Interval refresh_interval; bool if_not_exists; NullableDatum start_offset, end_offset; NullableDatum include_tiered_data; NullableDatum buckets_per_batch; NullableDatum max_batches_per_execution; NullableDatum refresh_newest_first; ts_feature_flag_check(FEATURE_POLICY); cagg_oid = PG_GETARG_OID(0); if (PG_ARGISNULL(3)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot use NULL refresh_schedule_interval"))); start_offset_type = get_fn_expr_argtype(fcinfo->flinfo, 1); start_offset.value = PG_GETARG_DATUM(1); start_offset.isnull = PG_ARGISNULL(1); end_offset_type = get_fn_expr_argtype(fcinfo->flinfo, 2); end_offset.value = PG_GETARG_DATUM(2); end_offset.isnull = PG_ARGISNULL(2); refresh_interval = *PG_GETARG_INTERVAL_P(3); if_not_exists = PG_GETARG_BOOL(4); TimestampTz initial_start = PG_ARGISNULL(5) ? DT_NOBEGIN : PG_GETARG_TIMESTAMPTZ(5); bool fixed_schedule = !PG_ARGISNULL(5); text *timezone = PG_ARGISNULL(6) ? NULL : PG_GETARG_TEXT_PP(6); char *valid_timezone = NULL; include_tiered_data.value = PG_GETARG_DATUM(7); include_tiered_data.isnull = PG_ARGISNULL(7); buckets_per_batch.value = PG_GETARG_DATUM(8); buckets_per_batch.isnull = PG_ARGISNULL(8); max_batches_per_execution.value = PG_GETARG_DATUM(9); max_batches_per_execution.isnull = PG_ARGISNULL(9); refresh_newest_first.value = PG_GETARG_DATUM(10); refresh_newest_first.isnull = PG_ARGISNULL(10); Datum retval; /* if users pass in -infinity for initial_start, then use the current_timestamp instead */ if (fixed_schedule) { ts_bgw_job_validate_schedule_interval(&refresh_interval); if (TIMESTAMP_NOT_FINITE(initial_start)) initial_start = ts_timer_get_current_timestamp(); } if (timezone != NULL) valid_timezone = ts_bgw_job_validate_timezone(PG_GETARG_DATUM(6)); retval = policy_refresh_cagg_add_internal(cagg_oid, start_offset_type, start_offset, end_offset_type, end_offset, refresh_interval, if_not_exists, fixed_schedule, initial_start, valid_timezone, include_tiered_data, buckets_per_batch, max_batches_per_execution, refresh_newest_first); if (!TIMESTAMP_NOT_FINITE(initial_start)) { int32 job_id = DatumGetInt32(retval); ts_bgw_job_stat_upsert_next_start(job_id, initial_start); } return retval; } Datum policy_refresh_cagg_remove_internal(Oid cagg_oid, bool if_exists) { int32 mat_htid; ContinuousAgg *cagg = ts_continuous_agg_find_by_relid(cagg_oid); if (!cagg) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("\"%s\" is not a continuous aggregate", get_rel_name(cagg_oid)))); ts_cagg_permissions_check(cagg_oid, GetUserId()); mat_htid = cagg->data.mat_hypertable_id; List *jobs = ts_bgw_job_find_by_proc_and_hypertable_id(POLICY_REFRESH_CAGG_PROC_NAME, FUNCTIONS_SCHEMA_NAME, mat_htid); if (jobs == NIL) { if (!if_exists) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), (errmsg("continuous aggregate policy not found for \"%s\"", get_rel_name(cagg_oid))))); else { ereport(NOTICE, (errmsg("continuous aggregate policy not found for \"%s\", skipping", get_rel_name(cagg_oid)))); PG_RETURN_BOOL(false); } } // Delete all bgw jobs associated with this CAgg ListCell *lc; foreach (lc, jobs) { BgwJob *job = (BgwJob *) lfirst(lc); ts_bgw_job_delete_by_id(job->fd.id); } PG_RETURN_BOOL(true); } Datum policy_refresh_cagg_remove(PG_FUNCTION_ARGS) { Oid cagg_oid = PG_GETARG_OID(0); bool if_not_exists = PG_GETARG_BOOL(1); /* Deprecating this argument */ bool if_exists; /* For backward compatibility, we use IF_NOT_EXISTS when IF_EXISTS is not given */ if_exists = PG_ARGISNULL(2) ? if_not_exists : PG_GETARG_BOOL(2); ts_feature_flag_check(FEATURE_POLICY); (void) policy_refresh_cagg_remove_internal(cagg_oid, if_exists); PG_RETURN_VOID(); } ================================================ FILE: tsl/src/bgw_policy/continuous_aggregate_api.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include "bgw_policy/job.h" #include "dimension.h" #include <continuous_aggs/materialize.h> #include <utils/jsonb.h> typedef enum PolicyRefreshOffsetOverlapResult { POLICY_REFRESH_OFFSET_OVERLAP, /* overlap but not exact */ POLICY_REFRESH_OFFSET_OVERLAP_EQUAL, /* exact match */ POLICY_REFRESH_OFFSET_OVERLAP_NONE, /* no overlap */ } PolicyRefreshOffsetOverlapResult; extern Datum policy_refresh_cagg_add(PG_FUNCTION_ARGS); extern Datum policy_refresh_cagg_proc(PG_FUNCTION_ARGS); extern Datum policy_refresh_cagg_check(PG_FUNCTION_ARGS); extern Datum policy_refresh_cagg_remove(PG_FUNCTION_ARGS); int32 policy_continuous_aggregate_get_mat_hypertable_id(const Jsonb *config); int64 policy_refresh_cagg_get_refresh_start(const ContinuousAgg *cagg, const Dimension *dim, const Jsonb *config, bool *start_isnull); int64 policy_refresh_cagg_get_refresh_end(const Dimension *dim, const Jsonb *config, bool *end_isnull); bool policy_refresh_cagg_get_include_tiered_data(const Jsonb *config, bool *isnull); int32 policy_refresh_cagg_get_buckets_per_batch(const Jsonb *config); int32 policy_refresh_cagg_get_max_batches_per_execution(const Jsonb *config); bool policy_refresh_cagg_get_refresh_newest_first(const Jsonb *config); bool policy_refresh_cagg_exists(int32 materialization_id); Datum policy_refresh_cagg_add_internal( Oid cagg_oid, Oid start_offset_type, NullableDatum start_offset, Oid end_offset_type, NullableDatum end_offset, Interval refresh_interval, bool if_not_exists, bool fixed_schedule, TimestampTz initial_start, const char *timezone, NullableDatum include_tiered_data, NullableDatum buckets_per_batch, NullableDatum max_batches_per_execution, NullableDatum refresh_newest_first); Datum policy_refresh_cagg_remove_internal(Oid cagg_oid, bool if_exists); PolicyRefreshOffsetOverlapResult policy_refresh_cagg_check_for_overlaps(ContinuousAgg *cagg, Jsonb *policy_config, int32 existing_job_id); bool policy_refresh_cagg_check_if_last_policy(PolicyContinuousAggData *policy_data); ================================================ FILE: tsl/src/bgw_policy/job.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include "bgw_policy/policies_v2.h" #include "cache.h" #include <access/xact.h> #include <catalog/namespace.h> #include <catalog/pg_type.h> #include <commands/defrem.h> #include <extension.h> #include <funcapi.h> #include <hypertable_cache.h> #include <nodes/makefuncs.h> #include <nodes/pg_list.h> #include <nodes/primnodes.h> #include <parser/parse_func.h> #include <parser/parser.h> #include <tcop/pquery.h> #include <utils/builtins.h> #include <utils/guc.h> #include <utils/lsyscache.h> #include <utils/portal.h> #include <utils/snapmgr.h> #include <utils/syscache.h> #include <utils/timestamp.h> #include "compat/compat.h" #include "bgw/job.h" #include "bgw/job_stat.h" #include "bgw/timer.h" #include "bgw_policy/chunk_stats.h" #include "bgw_policy/compression_api.h" #include "bgw_policy/continuous_aggregate_api.h" #include "bgw_policy/policy_config.h" #include "bgw_policy/policy_utils.h" #include "bgw_policy/process_hyper_inval_api.h" #include "bgw_policy/reorder_api.h" #include "bgw_policy/retention_api.h" #include "compression/api.h" #include "continuous_aggs/invalidation_threshold.h" #include "continuous_aggs/materialize.h" #include "continuous_aggs/refresh.h" #include "ts_catalog/continuous_agg.h" #ifdef USE_TELEMETRY #include "telemetry/telemetry.h" #endif #include "tsl/src/chunk.h" #include "chunk.h" #include "config.h" #include "dimension.h" #include "dimension_slice.h" #include "guc.h" #include "job.h" #include "jsonb_utils.h" #include "reorder.h" #include "utils.h" #define REORDER_SKIP_RECENT_DIM_SLICES_N 3 static void log_retention_boundary(int elevel, PolicyRetentionData *policy_data, const char *message) { if (OidIsValid(policy_data->boundary_type)) elog(elevel, "%s \"%s\": dropping data %s %s", message, get_rel_name(policy_data->object_relid), policy_data->use_creation_time ? "created before" : "older than", ts_datum_to_string(policy_data->boundary, policy_data->boundary_type)); } static void enable_fast_restart(int32 job_id, const char *job_name) { BgwJobStat *job_stat = ts_bgw_job_stat_find(job_id); if (job_stat != NULL) { /* job might not have a valid last_start if it was not * run by the bgw framework. */ ts_bgw_job_stat_set_next_start(job_id, job_stat->fd.last_start != DT_NOBEGIN ? job_stat->fd.last_start : GetCurrentTransactionStartTimestamp()); } else ts_bgw_job_stat_upsert_next_start(job_id, GetCurrentTransactionStartTimestamp()); elog(DEBUG1, "the %s job is scheduled to run again immediately", job_name); } /* * Returns the ID of a chunk to reorder. Eligible chunks must be at least the * 3rd newest chunk in the hypertable (not entirely exact because we use the number * of dimension slices as a proxy for the number of chunks), * not compressed, not dropped and hasn't been reordered recently. * For this version of automatic reordering, "not reordered * recently" means the chunk has not been reordered at all. This information * is available in the bgw_policy_chunk_stats metadata table. */ static int get_chunk_id_to_reorder(int32 job_id, Hypertable *ht) { const Dimension *time_dimension = hyperspace_get_open_dimension(ht->space, 0); const DimensionSlice *nth_dimension = ts_dimension_slice_nth_latest_slice(time_dimension->fd.id, REORDER_SKIP_RECENT_DIM_SLICES_N); if (!nth_dimension) return -1; Assert(time_dimension != NULL); return ts_dimension_slice_oldest_valid_chunk_for_reorder(job_id, time_dimension->fd.id, BTLessEqualStrategyNumber, nth_dimension->fd.range_start, InvalidStrategy, -1); } /* * returns now() - window as partitioning type datum */ static Datum get_window_boundary(const Dimension *dim, const Jsonb *config, int64 (*int_getter)(const Jsonb *), Interval *(*interval_getter)(const Jsonb *) ) { Oid partitioning_type = ts_dimension_get_partition_type(dim); if (IS_INTEGER_TYPE(partitioning_type)) { Oid now_func = ts_get_integer_now_func(dim, false); /* If "now_func" is provided then we use that for calculating the window. */ if (OidIsValid(now_func)) { int64 res, lag = int_getter(config); res = ts_sub_integer_from_now(lag, partitioning_type, now_func); return Int64GetDatum(res); } else { /* * Otherwise, the interval value can be returned without subtracting it * from now(). */ Interval *lag = interval_getter(config); return IntervalPGetDatum(lag); } } else { Interval *lag = interval_getter(config); /* * For UUID (v7) partitioned hypertables, drop_chunks expects TIMESTAMPTZ * input, so we compute the boundary as TIMESTAMPTZ instead of UUID. */ if (IS_UUID_TYPE(partitioning_type)) partitioning_type = TIMESTAMPTZOID; return ts_subtract_interval_from_now(lag, partitioning_type); } } static List * get_chunk_to_recompress(const Dimension *dim, const Jsonb *config) { Oid partitioning_type = ts_dimension_get_partition_type(dim); Oid boundary_type = partitioning_type; StrategyNumber end_strategy = BTLessStrategyNumber; int32 numchunks = policy_compression_get_maxchunks_per_job(config); /* * For UUID-partitioned hypertables, the boundary is computed as TIMESTAMPTZ * by get_window_boundary, so we need to use TIMESTAMPTZOID for conversion. */ if (IS_UUID_TYPE(partitioning_type)) boundary_type = TIMESTAMPTZOID; Datum boundary = get_window_boundary(dim, config, policy_recompression_get_recompress_after_int, policy_recompression_get_recompress_after_interval); return ts_dimension_slice_get_chunkids_to_compress(dim->fd.id, InvalidStrategy, /*start_strategy*/ -1, /*start_value*/ end_strategy, ts_time_value_to_internal(boundary, boundary_type), false, true, numchunks); } static void check_valid_index(Hypertable *ht, const char *index_name) { Oid index_oid; HeapTuple idxtuple; Form_pg_index index_form; index_oid = ts_get_relation_relid(NameStr(ht->fd.schema_name), (char *) index_name, true); idxtuple = SearchSysCache1(INDEXRELID, ObjectIdGetDatum(index_oid)); if (!HeapTupleIsValid(idxtuple)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("reorder index not found"), errdetail("The index \"%s\" could not be found", index_name))); index_form = (Form_pg_index) GETSTRUCT(idxtuple); if (index_form->indrelid != ht->main_table_relid) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid reorder index"), errhint("The reorder index must by an index on hypertable \"%s\".", NameStr(ht->fd.table_name)))); ReleaseSysCache(idxtuple); } bool policy_reorder_execute(int32 job_id, Jsonb *config) { int chunk_id; Chunk *chunk; PolicyReorderData policy; policy_reorder_read_and_validate_config(config, &policy); /* Find a chunk to reorder in the selected hypertable */ chunk_id = get_chunk_id_to_reorder(job_id, policy.hypertable); if (chunk_id == -1) { elog(NOTICE, "no chunks need reordering for hypertable %s.%s", NameStr(policy.hypertable->fd.schema_name), NameStr(policy.hypertable->fd.table_name)); return true; } /* * NOTE: We pass the Oid of the hypertable's index, and the true reorder * function should translate this to the Oid of the index on the specific * chunk. */ chunk = ts_chunk_get_by_id(chunk_id, false); elog(DEBUG1, "reordering chunk %s.%s", NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name)); reorder_chunk(chunk->table_id, policy.index_relid, false, InvalidOid, InvalidOid, InvalidOid); elog(DEBUG1, "completed reordering chunk %s.%s", NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name)); /* Now update chunk_stats table */ ts_bgw_policy_chunk_stats_record_job_run(job_id, chunk_id, ts_timer_get_current_timestamp()); if (get_chunk_id_to_reorder(job_id, policy.hypertable) != -1) enable_fast_restart(job_id, "reorder"); return true; } void policy_reorder_read_and_validate_config(Jsonb *config, PolicyReorderData *policy) { int32 htid = policy_config_get_hypertable_id(config); Hypertable *ht = ts_hypertable_get_by_id(htid); if (!ht) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("configuration hypertable id %d not found", htid))); const char *index_name = policy_reorder_get_index_name(config); check_valid_index(ht, index_name); if (policy) { policy->hypertable = ht; policy->index_relid = ts_get_relation_relid(NameStr(ht->fd.schema_name), (char *) index_name, false); } } bool policy_retention_execute(int32 job_id, Jsonb *config) { PolicyRetentionData policy_data; bool verbose_log; policy_retention_read_and_validate_config(config, &policy_data); verbose_log = policy_get_verbose_log(config); if (verbose_log) log_retention_boundary(LOG, &policy_data, "applying retention policy to hypertable"); chunk_invoke_drop_chunks(policy_data.object_relid, policy_data.boundary, policy_data.boundary_type, policy_data.use_creation_time); return true; } void policy_retention_read_and_validate_config(Jsonb *config, PolicyRetentionData *policy_data) { Oid object_relid; Hypertable *hypertable; Cache *hcache; const Dimension *open_dim; Datum boundary; Oid boundary_type; ContinuousAgg *cagg; Interval *(*interval_getter)(const Jsonb *); interval_getter = policy_retention_get_drop_after_interval; bool use_creation_time = false; object_relid = ts_hypertable_id_to_relid(policy_config_get_hypertable_id(config), false); hypertable = ts_hypertable_cache_get_cache_and_entry(object_relid, CACHE_FLAG_NONE, &hcache); open_dim = get_open_dimension_for_hypertable(hypertable, false); /* if dim is NULL, then it should be an INTEGER partition with no int_now function */ if (open_dim == NULL) { Oid partition_type; open_dim = hyperspace_get_open_dimension(hypertable->space, 0); partition_type = ts_dimension_get_partition_type(open_dim); if (!IS_INTEGER_TYPE(partition_type)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("incorrect partition type %d. Expected integer", partition_type))); /* if there's no int_now function the boundary is considered as an INTERVAL */ boundary_type = INTERVALOID; interval_getter = policy_retention_get_drop_created_before_interval; use_creation_time = true; } else { boundary_type = ts_dimension_get_partition_type(open_dim); /* * For UUID (v7) partitioned hypertables, drop_chunks expects TIMESTAMPTZ * input (the timestamp is extracted from the UUID internally). We need to * pass the boundary as TIMESTAMPTZ, not as UUID. */ if (IS_UUID_TYPE(boundary_type)) boundary_type = TIMESTAMPTZOID; } boundary = get_window_boundary(open_dim, config, policy_retention_get_drop_after_int, interval_getter); /* We need to do a reverse lookup here since the given hypertable might be a materialized hypertable, and thus need to call drop_chunks on the continuous aggregate instead. */ cagg = ts_continuous_agg_find_by_mat_hypertable_id(hypertable->fd.id, true); if (cagg) { object_relid = ts_get_relation_relid(NameStr(cagg->data.user_view_schema), NameStr(cagg->data.user_view_name), false); } ts_cache_release(&hcache); if (policy_data) { policy_data->object_relid = object_relid; policy_data->boundary = boundary; policy_data->boundary_type = boundary_type; policy_data->use_creation_time = use_creation_time; } } bool policy_refresh_cagg_execute(int32 job_id, Jsonb *config) { PolicyContinuousAggData policy_data; StringInfoData str; initStringInfo(&str); JsonbToCStringIndent(&str, &config->root, VARSIZE(config)); policy_refresh_cagg_read_and_validate_config(config, &policy_data); bool extend_last_bucket = !policy_refresh_cagg_check_if_last_policy(&policy_data); bool enable_osm_reads_old = ts_guc_enable_osm_reads; if (!policy_data.include_tiered_data_isnull) { SetConfigOption("timescaledb.enable_tiered_reads", policy_data.include_tiered_data ? "on" : "off", PGC_USERSET, PGC_S_SESSION); } ContinuousAggRefreshContext context = { .callctx = CAGG_REFRESH_POLICY }; /* Try to split window range into a list of ranges */ List *refresh_window_list = continuous_agg_split_refresh_window(policy_data.cagg, &policy_data.refresh_window, policy_data.buckets_per_batch, policy_data.refresh_newest_first); if (refresh_window_list == NIL) refresh_window_list = lappend(refresh_window_list, &policy_data.refresh_window); else context.callctx = CAGG_REFRESH_POLICY_BATCHED; context.number_of_batches = list_length(refresh_window_list); ListCell *lc; int32 processing_batch = 0; foreach (lc, refresh_window_list) { InternalTimeRange *refresh_window = (InternalTimeRange *) lfirst(lc); elog(DEBUG1, "refreshing continuous aggregate \"%s\" from %s to %s", NameStr(policy_data.cagg->data.user_view_name), ts_internal_to_time_string(refresh_window->start, refresh_window->type), ts_internal_to_time_string(refresh_window->end, refresh_window->type)); context.processing_batch = ++processing_batch; continuous_agg_refresh_internal(policy_data.cagg, refresh_window, context, refresh_window->start_isnull, refresh_window->end_isnull, (context.callctx != CAGG_REFRESH_POLICY_BATCHED), false, /* force */ policy_data.process_hypertable_invalidations, extend_last_bucket); if (processing_batch >= policy_data.max_batches_per_execution && processing_batch < context.number_of_batches && policy_data.max_batches_per_execution > 0) { elog(LOG, "reached maximum number of batches per execution (%d), batches not processed (%d)", policy_data.max_batches_per_execution, context.number_of_batches - processing_batch); break; } } if (!policy_data.include_tiered_data_isnull) { SetConfigOption("timescaledb.enable_tiered_reads", enable_osm_reads_old ? "on" : "off", PGC_USERSET, PGC_S_SESSION); } return true; } void policy_refresh_cagg_read_and_validate_config(Jsonb *config, PolicyContinuousAggData *policy_data) { int32 materialization_id; Hypertable *mat_ht; const Dimension *open_dim; Oid dim_type; int64 refresh_start, refresh_end; int32 buckets_per_batch, max_batches_per_execution; bool start_isnull, end_isnull; bool include_tiered_data, include_tiered_data_isnull; bool refresh_newest_first; materialization_id = policy_continuous_aggregate_get_mat_hypertable_id(config); mat_ht = ts_hypertable_get_by_id(materialization_id); if (!mat_ht) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("configuration materialization hypertable id %d not found", materialization_id))); ContinuousAgg *cagg = ts_continuous_agg_find_by_mat_hypertable_id(materialization_id, false); open_dim = get_open_dimension_for_hypertable(mat_ht, true); dim_type = ts_dimension_get_partition_type(open_dim); refresh_start = policy_refresh_cagg_get_refresh_start(cagg, open_dim, config, &start_isnull); refresh_end = policy_refresh_cagg_get_refresh_end(open_dim, config, &end_isnull); if (refresh_start >= refresh_end) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid refresh window"), errdetail("start_offset: %s, end_offset: %s", ts_internal_to_time_string(refresh_start, dim_type), ts_internal_to_time_string(refresh_end, dim_type)), errhint("The start of the window must be before the end."))); include_tiered_data = policy_refresh_cagg_get_include_tiered_data(config, &include_tiered_data_isnull); buckets_per_batch = policy_refresh_cagg_get_buckets_per_batch(config); if (buckets_per_batch < 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid buckets per batch"), errdetail("buckets_per_batch: %d", buckets_per_batch), errhint("The buckets per batch should be greater than or equal to zero."))); max_batches_per_execution = policy_refresh_cagg_get_max_batches_per_execution(config); if (max_batches_per_execution < 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid max batches per execution"), errdetail("max_batches_per_execution: %d", max_batches_per_execution), errhint( "The max batches per execution should be greater than or equal to zero."))); refresh_newest_first = policy_refresh_cagg_get_refresh_newest_first(config); bool process_hypertable_invalidations_found; bool process_hypertable_invalidations = ts_jsonb_get_bool_field(config, POL_REFRESH_CONF_KEY_PROCESS_HYPERTABLE_INVALIDATIONS, &process_hypertable_invalidations_found); if (policy_data) { policy_data->refresh_window.type = dim_type; policy_data->refresh_window.start = refresh_start; policy_data->refresh_window.start_isnull = start_isnull; policy_data->refresh_window.end = refresh_end; policy_data->refresh_window.end_isnull = end_isnull; policy_data->cagg = cagg; policy_data->include_tiered_data = include_tiered_data; policy_data->include_tiered_data_isnull = include_tiered_data_isnull; policy_data->buckets_per_batch = buckets_per_batch; policy_data->max_batches_per_execution = max_batches_per_execution; policy_data->refresh_newest_first = refresh_newest_first; policy_data->process_hypertable_invalidations = !process_hypertable_invalidations_found || process_hypertable_invalidations; } } /* Read configuration for compression job from config object. */ void policy_compression_read_and_validate_config(Jsonb *config, PolicyCompressionData *policy_data) { Oid table_relid = ts_hypertable_id_to_relid(policy_config_get_hypertable_id(config), false); Cache *hcache; Hypertable *hypertable = ts_hypertable_cache_get_cache_and_entry(table_relid, CACHE_FLAG_NONE, &hcache); if (policy_data) { policy_data->hypertable = hypertable; policy_data->hcache = hcache; } } void policy_recompression_read_and_validate_config(Jsonb *config, PolicyCompressionData *policy_data) { Oid table_relid = ts_hypertable_id_to_relid(policy_config_get_hypertable_id(config), false); Cache *hcache; Hypertable *hypertable = ts_hypertable_cache_get_cache_and_entry(table_relid, CACHE_FLAG_NONE, &hcache); if (policy_data) { policy_data->hypertable = hypertable; policy_data->hcache = hcache; } } bool policy_recompression_execute(int32 job_id, Jsonb *config) { List *chunkid_lst; ListCell *lc; const Dimension *dim; PolicyCompressionData policy_data; bool used_portalcxt = false; MemoryContext saved_cxt, multitxn_cxt; policy_recompression_read_and_validate_config(config, &policy_data); dim = hyperspace_get_open_dimension(policy_data.hypertable->space, 0); /* we want the chunk id list to survive across transactions. So alloc in * a different context */ if (PortalContext) { /*if we have a portal context use that - it will get freed automatically*/ multitxn_cxt = PortalContext; used_portalcxt = true; } else { /* background worker job does not go via usual CALL path, so we do * not have a PortalContext */ multitxn_cxt = AllocSetContextCreate(TopMemoryContext, "CompressionJobCxt", ALLOCSET_DEFAULT_SIZES); } saved_cxt = MemoryContextSwitchTo(multitxn_cxt); chunkid_lst = get_chunk_to_recompress(dim, config); MemoryContextSwitchTo(saved_cxt); if (!chunkid_lst) { elog(NOTICE, "no chunks for hypertable \"%s.%s\" that satisfy recompress chunk policy", NameStr(policy_data.hypertable->fd.schema_name), NameStr(policy_data.hypertable->fd.table_name)); ts_cache_release(&policy_data.hcache); if (!used_portalcxt) MemoryContextDelete(multitxn_cxt); return true; } ts_cache_release(&policy_data.hcache); if (ActiveSnapshotSet()) PopActiveSnapshot(); /* process each chunk in a new transaction */ foreach (lc, chunkid_lst) { CommitTransactionCommand(); StartTransactionCommand(); int32 chunkid = lfirst_int(lc); Chunk *chunk = ts_chunk_get_by_id(chunkid, true); Assert(chunk); if (!ts_chunk_needs_recompression(chunk)) continue; tsl_compress_chunk_wrapper(chunk, true, false); elog(LOG, "completed recompressing chunk \"%s.%s\"", NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name)); } elog(DEBUG1, "job %d completed recompressing chunk", job_id); return true; } void policy_process_hyper_inval_read_and_validate_config(Jsonb *config, PolicyMoveHyperInvalData *policy_data) { int32 hypertable_id = policy_config_get_hypertable_id(config); Oid table_relid = ts_hypertable_id_to_relid(hypertable_id, true); if (!OidIsValid(table_relid)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("configuration hypertable id %d not found", hypertable_id))); Cache *hcache; Hypertable *hypertable = ts_hypertable_cache_get_cache_and_entry(table_relid, CACHE_FLAG_NONE, &hcache); if (policy_data) { policy_data->hypertable = hypertable; policy_data->hcache = hcache; } else { ts_cache_release(&hcache); } } bool policy_process_hyper_inval_execute(int32 job_id, Jsonb *config) { PolicyMoveHyperInvalData policy_data; policy_process_hyper_inval_read_and_validate_config(config, &policy_data); const Dimension *dim = hyperspace_get_open_dimension(policy_data.hypertable->space, 0); Oid dimtype = ts_dimension_get_partition_type(dim); int32 hypertable_id = policy_data.hypertable->fd.id; /* We serialized on the invalidation threshold, so we get and lock it. */ invalidation_threshold_get(hypertable_id); invalidation_process_hypertable_log(hypertable_id, dimtype); ts_cache_release(&policy_data.hcache); return true; } static void job_execute_function(FuncExpr *funcexpr) { bool isnull; EState *estate = CreateExecutorState(); ExprContext *econtext = CreateExprContext(estate); ExprState *es = ExecPrepareExpr((Expr *) funcexpr, estate); ExecEvalExpr(es, econtext, &isnull); FreeExprContext(econtext, true); FreeExecutorState(estate); } static void job_execute_procedure(FuncExpr *funcexpr) { CallStmt *call = makeNode(CallStmt); call->funcexpr = funcexpr; DestReceiver *dest = CreateDestReceiver(DestNone); /* we don't need to create proper param list cause we pass in all arguments as Const */ ParamListInfo params = makeParamList(0); ExecuteCallStmt(call, params, false, dest); } /* * Execute the job. * * This function can be called both from a portal and from a background * worker. */ bool job_execute(BgwJob *job) { Const *arg1, *arg2; bool portal_created = false; char prokind; Oid proc; FuncExpr *funcexpr; MemoryContext parent_ctx = CurrentMemoryContext; StringInfoData query; Portal portal = ActivePortal; /* Check for work_mem setting in config and apply it */ if (job->fd.config) { char *work_mem_setting = ts_jsonb_get_str_field(job->fd.config, "work_mem"); if (work_mem_setting != NULL) { SetConfigOption("work_mem", work_mem_setting, PGC_USERSET, PGC_S_SESSION); } } if (job->fd.config) elog(DEBUG1, "Executing %s with parameters %s", NameStr(job->fd.proc_name), DatumGetCString(DirectFunctionCall1(jsonb_out, JsonbPGetDatum(job->fd.config)))); else elog(DEBUG1, "Executing %s with no parameters", NameStr(job->fd.proc_name)); /* Create a portal if there's no active */ if (!PortalIsValid(portal)) { portal_created = true; portal = CreatePortal("", true, true); portal->visible = false; portal->resowner = CurrentResourceOwner; ActivePortal = portal; PortalContext = portal->portalContext; StartTransactionCommand(); EnsurePortalSnapshotExists(); } #ifdef USE_TELEMETRY /* The telemetry job has a separate code path and since we can reach this * code also when using run_job(), we have a special case here. This will * not be triggered when executed from ts_bgw_job_execute(). */ if (ts_is_telemetry_job(job)) { /* * In the first 12 hours, we want telemetry to ping every * hour. After that initial period, we default to the * schedule_interval listed in the job table. */ Interval one_hour = { .time = 1 * USECS_PER_HOUR }; return ts_bgw_job_run_and_set_next_start(job, ts_telemetry_main_wrapper, TELEMETRY_INITIAL_NUM_RUNS, &one_hour, /* atomic */ false, /* mark */ true); } #endif proc = ts_bgw_job_get_funcid(job); prokind = get_func_prokind(proc); /* * We need to switch back to parent MemoryContext as StartTransactionCommand * switched to CurTransactionContext and this context will be destroyed * on CommitTransactionCommand which may be too short-lived if a policy * has its own transaction handling. */ MemoryContextSwitchTo(parent_ctx); arg1 = makeConst(INT4OID, -1, InvalidOid, 4, Int32GetDatum(job->fd.id), false, true); if (job->fd.config == NULL) arg2 = makeNullConst(JSONBOID, -1, InvalidOid); else arg2 = makeConst(JSONBOID, -1, InvalidOid, -1, JsonbPGetDatum(job->fd.config), false, false); funcexpr = makeFuncExpr(proc, VOIDOID, list_make2(arg1, arg2), InvalidOid, InvalidOid, COERCE_EXPLICIT_CALL); /* Here we create a query string from the function/procedure name that we * are calling. We do not update the status after the execution has * finished since this is wrapped inside the code that starts and stops * any job, not just custom jobs. We just provide more detailed * information here that we are actually calling a specific custom * function. */ initStringInfo(&query); appendStringInfo(&query, "CALL %s.%s()", quote_identifier(NameStr(job->fd.proc_schema)), quote_identifier(NameStr(job->fd.proc_name))); pgstat_report_activity(STATE_RUNNING, query.data); switch (prokind) { case PROKIND_FUNCTION: job_execute_function(funcexpr); break; case PROKIND_PROCEDURE: job_execute_procedure(funcexpr); break; default: ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported function type"))); break; } /* Drop portal if it was created */ if (portal_created) { if (ActiveSnapshotSet()) PopActiveSnapshot(); CommitTransactionCommand(); PortalDrop(portal, false); ActivePortal = NULL; PortalContext = NULL; } return true; } ================================================ FILE: tsl/src/bgw_policy/job.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <utils/jsonb.h> #include <bgw/job.h> #include <hypertable.h> #include "bgw_policy/chunk_stats.h" #include "cache.h" #include "continuous_aggs/materialize.h" /* Add config keys common across job types here */ #define CONFIG_KEY_VERBOSE_LOG "verbose_log" /*used only by retention now*/ typedef struct PolicyMoveHyperInvalData { Hypertable *hypertable; Cache *hcache; } PolicyMoveHyperInvalData; typedef struct PolicyReorderData { Hypertable *hypertable; Oid index_relid; } PolicyReorderData; typedef struct PolicyRetentionData { Oid object_relid; Datum boundary; Oid boundary_type; bool use_creation_time; } PolicyRetentionData; typedef struct PolicyContinuousAggData { InternalTimeRange refresh_window; ContinuousAgg *cagg; bool include_tiered_data; bool include_tiered_data_isnull; int32 buckets_per_batch; int32 max_batches_per_execution; bool refresh_newest_first; bool process_hypertable_invalidations; } PolicyContinuousAggData; typedef struct PolicyCompressionData { Hypertable *hypertable; Cache *hcache; } PolicyCompressionData; /* Reorder function type. Necessary for testing */ typedef void (*reorder_func)(Oid tableOid, Oid indexOid, bool verbose, Oid wait_id, Oid destination_tablespace, Oid index_tablespace); /* Functions exposed only for testing */ extern bool policy_reorder_execute(int32 job_id, Jsonb *config); extern bool policy_retention_execute(int32 job_id, Jsonb *config); extern bool policy_refresh_cagg_execute(int32 job_id, Jsonb *config); extern bool policy_process_hyper_inval_execute(int32 job_id, Jsonb *config); extern bool policy_recompression_execute(int32 job_id, Jsonb *config); extern void policy_reorder_read_and_validate_config(Jsonb *config, PolicyReorderData *policy_data); extern void policy_retention_read_and_validate_config(Jsonb *config, PolicyRetentionData *policy_data); extern void policy_refresh_cagg_read_and_validate_config(Jsonb *config, PolicyContinuousAggData *policy_data); extern void policy_process_hyper_inval_read_and_validate_config(Jsonb *config, PolicyMoveHyperInvalData *policy_data); extern void policy_compression_read_and_validate_config(Jsonb *config, PolicyCompressionData *policy_data); extern void policy_recompression_read_and_validate_config(Jsonb *config, PolicyCompressionData *policy_data); extern bool job_execute(BgwJob *job); ================================================ FILE: tsl/src/bgw_policy/job_api.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <funcapi.h> #include <miscadmin.h> #include <utils/acl.h> #include <utils/builtins.h> #include <parser/parse_func.h> #include <parser/parser.h> #include <bgw/job.h> #include <bgw/job_stat.h> #include "bgw/timer.h" #include "debug_assert.h" #include "hypertable_cache.h" #include "job.h" #include "job_api.h" #include "policies_v2.h" /* Default max runtime for a custom job is unlimited for now */ #define DEFAULT_MAX_RUNTIME 0 /* Default retry period for reorder_jobs is currently 5 minutes */ #define DEFAULT_RETRY_PERIOD (5 * USECS_PER_MINUTE) #define ALTER_JOB_NUM_COLS 13 /* * This function ensures that the check function has the required signature * @param check A valid Oid */ static inline void validate_check_signature(Oid check) { Oid proc = InvalidOid; ObjectWithArgs *object; NameData check_name = { .data = { 0 } }; NameData check_schema = { .data = { 0 } }; namestrcpy(&check_schema, get_namespace_name(get_func_namespace(check))); namestrcpy(&check_name, get_func_name(check)); object = makeNode(ObjectWithArgs); object->objname = list_make2(makeString(NameStr(check_schema)), makeString(NameStr(check_name))); object->objargs = list_make1(SystemTypeName("jsonb")); proc = LookupFuncWithArgs(OBJECT_ROUTINE, object, true); if (!OidIsValid(proc)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("function or procedure %s.%s(config jsonb) not found", NameStr(check_schema), NameStr(check_name)), errhint("The check function's signature must be (config jsonb)."))); } /* * CREATE FUNCTION add_job( * 0 proc REGPROC, * 1 schedule_interval INTERVAL, * 2 config JSONB DEFAULT NULL, * 3 initial_start TIMESTAMPTZ DEFAULT NULL, * 4 scheduled BOOL DEFAULT true * 5 check_config REGPROC DEFAULT NULL * 6 fixed_schedule BOOL DEFAULT TRUE * 7 timezone TEXT DEFAULT NULL * 8 job_name TEXT DEFAULT NULL * ) RETURNS INTEGER */ Datum job_add(PG_FUNCTION_ARGS) { NameData application_name; NameData proc_name; NameData proc_schema; NameData check_name = { .data = { 0 } }; NameData check_schema = { .data = { 0 } }; Interval max_runtime = { .time = DEFAULT_MAX_RUNTIME }; Interval retry_period = { .time = DEFAULT_RETRY_PERIOD }; int32 job_id; char *func_name = NULL; char *check_name_str = NULL; char *valid_timezone = NULL; TimestampTz initial_start = PG_ARGISNULL(3) ? DT_NOBEGIN : PG_GETARG_TIMESTAMPTZ(3); Oid owner = GetUserId(); Oid proc = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); Interval *schedule_interval = PG_ARGISNULL(1) ? NULL : PG_GETARG_INTERVAL_P(1); Jsonb *config = PG_ARGISNULL(2) ? NULL : PG_GETARG_JSONB_P(2); bool scheduled = PG_ARGISNULL(4) ? true : PG_GETARG_BOOL(4); Oid check = PG_ARGISNULL(5) ? InvalidOid : PG_GETARG_OID(5); bool fixed_schedule = PG_ARGISNULL(6) ? true : PG_GETARG_BOOL(6); text *timezone = PG_ARGISNULL(7) ? NULL : PG_GETARG_TEXT_PP(7); /* verify it's a valid timezone */ if (timezone != NULL) valid_timezone = ts_bgw_job_validate_timezone(PG_GETARG_DATUM(7)); char *job_name_str = PG_ARGISNULL(8) ? NULL : text_to_cstring(PG_GETARG_TEXT_PP(8)); TS_PREVENT_FUNC_IF_READ_ONLY(); if (PG_ARGISNULL(0)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("function or procedure cannot be NULL"))); if (NULL == schedule_interval) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("schedule interval cannot be NULL"))); /* for fixed schedules, we use time_bucket in the calculation of next_start Therefore, we cannot allow schedule intervals containing both month and day components */ if (fixed_schedule) ts_bgw_job_validate_schedule_interval(schedule_interval); func_name = get_func_name(proc); if (func_name == NULL) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("function or procedure with OID %u does not exist", proc))); if (object_aclcheck(ProcedureRelationId, proc, owner, ACL_EXECUTE) != ACLCHECK_OK) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permission denied for function \"%s\"", func_name), errhint("Job owner must have EXECUTE privilege on the function."))); if (OidIsValid(check)) { check_name_str = get_func_name(check); if (check_name_str == NULL) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("function with OID %d does not exist", check))); if (object_aclcheck(ProcedureRelationId, check, owner, ACL_EXECUTE) != ACLCHECK_OK) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permission denied for function \"%s\"", check_name_str), errhint("Job owner must have EXECUTE privilege on the function."))); namestrcpy(&check_schema, get_namespace_name(get_func_namespace(check))); namestrcpy(&check_name, check_name_str); } /* if no initial_start was provided for a fixed schedule, use the current time */ if (fixed_schedule && TIMESTAMP_NOT_FINITE(initial_start)) { initial_start = ts_timer_get_current_timestamp(); elog(DEBUG1, "Using current time [%s] as initial start", DatumGetCString( DirectFunctionCall1(timestamptz_out, TimestampTzGetDatum(initial_start)))); } /* Verify that the owner can create a background worker */ ts_bgw_job_validate_job_owner(owner); /* Next, insert a new job into jobs table */ if (job_name_str) namestrcpy(&application_name, job_name_str); else namestrcpy(&application_name, "User-Defined Action"); namestrcpy(&proc_schema, get_namespace_name(get_func_namespace(proc))); namestrcpy(&proc_name, func_name); /* The check exists but may not have the expected signature: (config jsonb) */ if (OidIsValid(check)) validate_check_signature(check); ts_bgw_job_run_config_check(check, 0, config); job_id = ts_bgw_job_insert_relation(&application_name, schedule_interval, &max_runtime, JOB_RETRY_UNLIMITED, &retry_period, &proc_schema, &proc_name, &check_schema, &check_name, owner, scheduled, fixed_schedule, INVALID_HYPERTABLE_ID, config, initial_start, valid_timezone); if (!TIMESTAMP_NOT_FINITE(initial_start)) ts_bgw_job_stat_upsert_next_start(job_id, initial_start); PG_RETURN_INT32(job_id); } static BgwJob * find_job(int32 job_id, bool null_job_id, bool missing_ok) { BgwJob *job; if (null_job_id && !missing_ok) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("job ID cannot be NULL"))); job = ts_bgw_job_find(job_id, CurrentMemoryContext, !missing_ok); if (NULL == job) { Assert(missing_ok); ereport(NOTICE, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("job %d not found, skipping", job_id))); } return job; } /* * CREATE OR REPLACE FUNCTION delete_job(job_id INTEGER) RETURNS VOID */ Datum job_delete(PG_FUNCTION_ARGS) { int32 job_id = PG_GETARG_INT32(0); BgwJob *job; TS_PREVENT_FUNC_IF_READ_ONLY(); job = find_job(job_id, PG_ARGISNULL(0), false); if (!has_privs_of_role(GetUserId(), job->fd.owner)) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("insufficient permissions to delete job owned by \"%s\"", GetUserNameFromId(job->fd.owner, false)))); ts_bgw_job_delete_by_id(job_id); PG_RETURN_VOID(); } /* * CREATE OR REPLACE PROCEDURE run_job(job_id INTEGER) */ Datum job_run(PG_FUNCTION_ARGS) { int32 job_id = PG_GETARG_INT32(0); BgwJob *job = find_job(job_id, PG_ARGISNULL(0), false); ts_bgw_job_permission_check(job, "run"); job_execute(job); PG_RETURN_VOID(); } /* * CREATE OR REPLACE FUNCTION alter_job( * 0 job_id INTEGER, * 1 schedule_interval INTERVAL = NULL, * 2 max_runtime INTERVAL = NULL, * 3 max_retries INTEGER = NULL, * 4 retry_period INTERVAL = NULL, * 5 scheduled BOOL = NULL, * 6 config JSONB = NULL, * 7 next_start TIMESTAMPTZ = NULL * 8 if_exists BOOL = FALSE, * 9 check_config REGPROC = NULL * 10 fixed_schedule BOOL = NULL, * 11 initial_start TIMESTAMPTZ = NULL * 12 timezone TEXT = NULL * 13 job_name TEXT = NULL * ) RETURNS TABLE ( * job_id INTEGER, * schedule_interval INTERVAL, * max_runtime INTERVAL, * max_retries INTEGER, * retry_period INTERVAL, * scheduled BOOL, * config JSONB, * next_start TIMESTAMPTZ * check_config TEXT * fixed_schedule BOOL * initial_start TIMESTAMPTZ * timezone TEXT * job_name TEXT * ) */ Datum job_alter(PG_FUNCTION_ARGS) { BgwJobStat *stat; TupleDesc tupdesc; Datum values[ALTER_JOB_NUM_COLS] = { 0 }; bool nulls[ALTER_JOB_NUM_COLS] = { false }; HeapTuple tuple; TimestampTz next_start; int job_id = PG_GETARG_INT32(0); bool if_exists = PG_GETARG_BOOL(8); BgwJob *job; NameData check_name = { .data = { 0 } }; NameData check_schema = { .data = { 0 } }; Oid check = PG_ARGISNULL(9) ? InvalidOid : PG_GETARG_OID(9); char *check_name_str = NULL; /* Added space for period and NULL */ char schema_qualified_check_name[(2 * NAMEDATALEN) + 2] = { 0 }; bool unregister_check = (!PG_ARGISNULL(9) && !OidIsValid(check)); TimestampTz initial_start = PG_ARGISNULL(11) ? DT_NOBEGIN : PG_GETARG_TIMESTAMPTZ(11); text *timezone = PG_ARGISNULL(12) ? NULL : PG_GETARG_TEXT_PP(12); char *valid_timezone = NULL; /* verify it's a valid timezone */ if (timezone != NULL) valid_timezone = ts_bgw_job_validate_timezone(PG_GETARG_DATUM(12)); TS_PREVENT_FUNC_IF_READ_ONLY(); /* check that caller accepts tuple and abort early if that is not the * case */ if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context " "that cannot accept type record"))); job = find_job(job_id, PG_ARGISNULL(0), if_exists); if (job == NULL) PG_RETURN_NULL(); ts_bgw_job_permission_check(job, "alter"); if (!PG_ARGISNULL(1)) job->fd.schedule_interval = *PG_GETARG_INTERVAL_P(1); if (!PG_ARGISNULL(2)) job->fd.max_runtime = *PG_GETARG_INTERVAL_P(2); if (!PG_ARGISNULL(3)) job->fd.max_retries = PG_GETARG_INT32(3); if (!PG_ARGISNULL(4)) job->fd.retry_period = *PG_GETARG_INTERVAL_P(4); if (!PG_ARGISNULL(5)) job->fd.scheduled = PG_GETARG_BOOL(5); if (!PG_ARGISNULL(6)) job->fd.config = PG_GETARG_JSONB_P(6); if (!PG_ARGISNULL(9)) { if (OidIsValid(check)) { check_name_str = get_func_name(check); if (check_name_str == NULL) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("function with OID %d does not exist", check))); if (object_aclcheck(ProcedureRelationId, check, GetUserId(), ACL_EXECUTE) != ACLCHECK_OK) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permission denied for function \"%s\"", check_name_str), errhint("Job owner must have EXECUTE privilege on the function."))); namestrcpy(&check_schema, get_namespace_name(get_func_namespace(check))); namestrcpy(&check_name, check_name_str); /* The check exists but may not have the expected signature: (config jsonb) */ validate_check_signature(check); namestrcpy(&job->fd.check_schema, NameStr(check_schema)); namestrcpy(&job->fd.check_name, NameStr(check_name)); snprintf(schema_qualified_check_name, sizeof(schema_qualified_check_name) / sizeof(schema_qualified_check_name[0]), "%s.%s", NameStr(check_schema), check_name_str); } } else snprintf(schema_qualified_check_name, sizeof(schema_qualified_check_name) / sizeof(schema_qualified_check_name[0]), "%s.%s", NameStr(job->fd.check_schema), NameStr(job->fd.check_name)); /* * If the CAgg refresh policy job is being altered, then always check for overlap. * There is probably a better place to do this, but we choose to do this here since we need * access to the job_id, which we don't have inside the `policy_check` function called above. */ if (namestrcmp(&job->fd.proc_name, POLICY_REFRESH_CAGG_PROC_NAME) == 0) { int32 materialization_id = policy_continuous_aggregate_get_mat_hypertable_id(job->fd.config); ContinuousAgg *cagg = ts_continuous_agg_find_by_mat_hypertable_id(materialization_id, false); PolicyRefreshOffsetOverlapResult res = policy_refresh_cagg_check_for_overlaps(cagg, job->fd.config, job->fd.id); switch (res) { case POLICY_REFRESH_OFFSET_OVERLAP_NONE: break; case POLICY_REFRESH_OFFSET_OVERLAP_EQUAL: ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("continuous aggregate refresh policy already exists for " "\"%s\"", get_rel_name(cagg->relid)), errdetail("A refresh policy with the same start and end offset already " "exists for " "continuous aggregate \"%s\".", get_rel_name(cagg->relid)))); break; case POLICY_REFRESH_OFFSET_OVERLAP: ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("refresh interval overlaps with an existing continuous aggregate " "policy on \"%s\"", get_rel_name(cagg->relid)))); break; } } if (unregister_check) { NameData empty_namedata = { .data = { 0 } }; namestrcpy(&job->fd.check_schema, NameStr(empty_namedata)); namestrcpy(&job->fd.check_name, NameStr(empty_namedata)); } if (!PG_ARGISNULL(10)) { bool fixed_schedule = PG_GETARG_BOOL(10); /* * initial_start is a required argument for fixed schedules * so we use the current timestamp if it's not provided */ if (fixed_schedule) { if (TIMESTAMP_NOT_FINITE(initial_start)) { initial_start = ts_timer_get_current_timestamp(); elog(NOTICE, "Using current time [%s] as initial start for job %d", DatumGetCString( DirectFunctionCall1(timestamptz_out, TimestampTzGetDatum(initial_start))), job->fd.id); job->fd.initial_start = initial_start; } } job->fd.fixed_schedule = fixed_schedule; } if (!PG_ARGISNULL(11)) { /* user provided +- infinity as initial_start, this is not acceptable */ if (TIMESTAMP_NOT_FINITE(initial_start)) { initial_start = ts_timer_get_current_timestamp(); elog(NOTICE, "Using current time [%s] as initial start for job %d", DatumGetCString( DirectFunctionCall1(timestamptz_out, TimestampTzGetDatum(initial_start))), job->fd.id); } job->fd.initial_start = initial_start; } if (!PG_ARGISNULL(13)) { char app_name[NAMEDATALEN]; int name_len; name_len = snprintf(app_name, NAMEDATALEN, "%s [%d]", text_to_cstring(PG_GETARG_TEXT_PP(13)), job_id); if (name_len >= NAMEDATALEN) ereport(ERROR, (errcode(ERRCODE_NAME_TOO_LONG), errmsg("application name too long."))); namestrcpy(&job->fd.application_name, app_name); } if (valid_timezone != NULL) job->fd.timezone = cstring_to_text(valid_timezone); else job->fd.timezone = NULL; /* it's also possible to alter the fields initial_start and timezone without * specifying fixed_schedule. In that case, update them and also update the * next_start accordingly. * If the job is not on a fixed schedule, then this has no effect on the next_start, * so maybe print a message to the user * that these changes are not really doing anything */ /* this function will also update the next_start if the schedule interval is changed, but I'm not going to rely on this to change stuff */ ts_bgw_job_update_by_id(job_id, job); /* one of the fields below changing necessitates a next_start update */ if (!PG_ARGISNULL(10) || !TIMESTAMP_NOT_FINITE(initial_start) || (valid_timezone != NULL)) { TimestampTz next_start_calculated; if (job->fd.fixed_schedule) { next_start_calculated = ts_get_next_scheduled_execution_slot(job, ts_timer_get_current_timestamp()); ts_bgw_job_stat_update_next_start(job->fd.id, next_start_calculated, false); } else { /* last finish time plus schedule interval */ BgwJobStat *stat = ts_bgw_job_stat_find(job->fd.id); if (stat != NULL) { next_start_calculated = DatumGetTimestampTz( DirectFunctionCall2(timestamptz_pl_interval, TimestampTzGetDatum(stat->fd.last_finish), IntervalPGetDatum(&job->fd.schedule_interval))); /* allow DT_NOBEGIN for next_start here through allow_unset=true in the case that * last_finish is DT_NOBEGIN, * This means the value is counted as unset which is what we want */ ts_bgw_job_stat_update_next_start(job->fd.id, next_start_calculated, true); } } } if (!PG_ARGISNULL(7)) ts_bgw_job_stat_upsert_next_start(job_id, PG_GETARG_TIMESTAMPTZ(7)); stat = ts_bgw_job_stat_find(job_id); if (stat != NULL) next_start = stat->fd.next_start; else next_start = DT_NOBEGIN; tupdesc = BlessTupleDesc(tupdesc); values[0] = Int32GetDatum(job->fd.id); values[1] = IntervalPGetDatum(&job->fd.schedule_interval); values[2] = IntervalPGetDatum(&job->fd.max_runtime); values[3] = Int32GetDatum(job->fd.max_retries); values[4] = IntervalPGetDatum(&job->fd.retry_period); values[5] = BoolGetDatum(job->fd.scheduled); if (job->fd.config == NULL) nulls[6] = true; else values[6] = JsonbPGetDatum(job->fd.config); values[7] = TimestampTzGetDatum(next_start); if (unregister_check) nulls[8] = true; else if (strlen(NameStr(job->fd.check_schema)) > 0) values[8] = CStringGetTextDatum(schema_qualified_check_name); else nulls[8] = true; /* values/nulls[9]: fixed_schedule */ values[9] = job->fd.fixed_schedule; /* values/nulls[10]: initial_start */ if (TIMESTAMP_NOT_FINITE(job->fd.initial_start)) { nulls[10] = true; } else values[10] = TimestampTzGetDatum(job->fd.initial_start); /* values/nulls[11]: timezone */ if (valid_timezone) values[11] = CStringGetTextDatum(valid_timezone); else nulls[11] = true; values[12] = NameGetDatum(&job->fd.application_name); tuple = heap_form_tuple(tupdesc, values, nulls); return HeapTupleGetDatum(tuple); } static Hypertable * get_hypertable_from_oid(Cache **hcache, Oid table_oid) { Hypertable *hypertable = NULL; hypertable = ts_hypertable_cache_get_cache_and_entry(table_oid, CACHE_FLAG_MISSING_OK, hcache); if (!hypertable) { const char *view_name = get_rel_name(table_oid); if (!view_name) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("relation is not a hypertable or continuous aggregate"))); else { ContinuousAgg *ca = ts_continuous_agg_find_by_relid(table_oid); if (!ca) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("relation \"%s\" is not a hypertable or continuous aggregate", view_name))); hypertable = ts_hypertable_get_by_id(ca->data.mat_hypertable_id); } } Assert(hypertable != NULL); return hypertable; } Datum job_alter_set_hypertable_id(PG_FUNCTION_ARGS) { int32 job_id = PG_GETARG_INT32(0); Oid table_oid = PG_GETARG_OID(1); Cache *hcache = NULL; Hypertable *ht = NULL; TS_PREVENT_FUNC_IF_READ_ONLY(); BgwJob *job = find_job(job_id, PG_ARGISNULL(0), false /* missing_ok */); if (job == NULL) PG_RETURN_NULL(); ts_bgw_job_permission_check(job, "alter"); if (!PG_ARGISNULL(1)) { ht = get_hypertable_from_oid(&hcache, table_oid); ts_hypertable_permissions_check(ht->main_table_relid, GetUserId()); } job->fd.hypertable_id = (ht != NULL ? ht->fd.id : 0); ts_bgw_job_update_by_id(job_id, job); if (hcache) ts_cache_release(&hcache); PG_RETURN_INT32(job_id); } ================================================ FILE: tsl/src/bgw_policy/job_api.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> /* Special values for the number of retries of a failed job */ #define JOB_RETRY_UNLIMITED (-1) #define JOB_RETRY_NONE 0 extern Datum job_add(PG_FUNCTION_ARGS); extern Datum job_alter(PG_FUNCTION_ARGS); extern Datum job_delete(PG_FUNCTION_ARGS); extern Datum job_run(PG_FUNCTION_ARGS); extern Datum job_alter_set_hypertable_id(PG_FUNCTION_ARGS); ================================================ FILE: tsl/src/bgw_policy/policies_v2.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <fmgr.h> #include <miscadmin.h> #include <parser/parse_coerce.h> #include <utils/builtins.h> #include <utils/float.h> #include "compat/compat.h" #include "bgw/job.h" #include "bgw_policy/continuous_aggregate_api.h" #include "bgw_policy/job.h" #include "bgw_policy/policies_v2.h" #include "compression_api.h" #include "errors.h" #include "funcapi.h" #include "guc.h" #include "hypertable.h" #include "hypertable_cache.h" #include "jsonb_utils.h" #include "policy_utils.h" #include "utils.h" #if PG16_GE #include "nodes/miscnodes.h" #endif /* Check if the provided argument is infinity */ bool ts_if_offset_is_infinity(Datum arg, Oid argtype, bool is_start) { if (OidIsValid(argtype) && argtype != UNKNOWNOID && argtype != FLOAT8OID) return false; if (argtype != FLOAT8OID) { double val; char *num = DatumGetCString(arg); #if PG16_LT bool have_error = false; val = float8in_internal_opt_error(num, NULL, "double precision", num, &have_error); if (have_error) return false; #else ErrorSaveContext escontext = { .type = T_ErrorSaveContext }; val = float8in_internal(num, NULL, "double precision", num, (Node *) &escontext); if (escontext.error_occurred) return false; #endif arg = Float8GetDatum(val); } float8 result = DatumGetFloat8(arg); return ((result == -get_float8_infinity() && is_start) || (result == get_float8_infinity() && !is_start)); } static void emit_error(const char *err) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("%s", err))); } static int64 offset_to_int64(NullableDatum arg, Oid argtype, Oid partition_type, bool is_start) { if (arg.isnull || ts_if_offset_is_infinity(arg.value, argtype, is_start)) { if (is_start) return ts_time_get_max(partition_type); else return ts_time_get_min(partition_type); } else return interval_to_int64(arg.value, argtype); } /* * Check different conditions if the requested policy parameters * are compatible with each other and then create/update the * provided policies. */ bool validate_and_create_policies(policies_info all_policies, bool if_exists) { int refresh_job_id = 0, compression_job_id = 0, retention_job_id = 0; int64 refresh_interval = 0, compress_after = 0, drop_after = 0, drop_after_HT = 0; int64 start_offset = 0, end_offset = 0, refresh_total_interval = 0; List *jobs = NIL; BgwJob *orig_ht_reten_job = NULL; char *err_gap_refresh = "there are gaps in refresh policy"; char *err_refresh_compress_overlap = "refresh and columnstore policies overlap"; char *err_refresh_reten_overlap = "refresh and retention policies overlap"; char *err_refresh_reten_ht_overlap = "refresh policy of continuous aggregate and retention " "policy of underlying hypertable overlap"; char *err_compress_reten_overlap = "columnstore and retention policies overlap"; jobs = ts_bgw_job_find_by_proc_and_hypertable_id(POLICY_RETENTION_PROC_NAME, FUNCTIONS_SCHEMA_NAME, all_policies.original_HT); if (jobs != NIL) { Assert(list_length(jobs) == 1); orig_ht_reten_job = linitial(jobs); } if (all_policies.refresh) { start_offset = offset_to_int64(all_policies.refresh->start_offset, all_policies.refresh->start_offset_type, all_policies.partition_type, true); end_offset = offset_to_int64(all_policies.refresh->end_offset, all_policies.refresh->end_offset_type, all_policies.partition_type, false); refresh_interval = interval_to_int64(IntervalPGetDatum(&all_policies.refresh->schedule_interval), INTERVALOID); refresh_total_interval = start_offset; if (!IS_INTEGER_TYPE(all_policies.partition_type) && refresh_total_interval != ts_time_get_max(all_policies.partition_type)) refresh_total_interval += refresh_interval; } if (all_policies.compress) compress_after = interval_to_int64(all_policies.compress->compress_after, all_policies.compress->compress_after_type); if (all_policies.retention) drop_after = interval_to_int64(all_policies.retention->drop_after, all_policies.retention->drop_after_type); if (orig_ht_reten_job) { if (IS_INTEGER_TYPE(all_policies.partition_type)) { bool found_drop_after = false; drop_after_HT = ts_jsonb_get_int64_field(orig_ht_reten_job->fd.config, POL_RETENTION_CONF_KEY_DROP_AFTER, &found_drop_after); } else { drop_after_HT = interval_to_int64( IntervalPGetDatum(ts_jsonb_get_interval_field(orig_ht_reten_job->fd.config, POL_RETENTION_CONF_KEY_DROP_AFTER)), INTERVALOID); } } /* Per policy checks */ if (all_policies.refresh && !IS_INTEGER_TYPE(all_policies.partition_type)) { /* * Check if there are any gaps in the refresh policy. The below code is * a little suspect. But since we are planning to do away with the * add_policies/remove_policies APIs there's no need to spend a lot * of time on fixing it below. */ int64 refresh_window_size; if (start_offset == ts_time_get_max(all_policies.partition_type) || end_offset == ts_time_get_min(all_policies.partition_type) || end_offset > start_offset || pg_sub_s64_overflow(start_offset, end_offset, &refresh_window_size)) refresh_window_size = start_offset; /* if refresh_interval is greater than half of refresh_window_size, then there are gaps */ if (refresh_interval > (refresh_window_size / 2)) emit_error(err_gap_refresh); /* Disallow refreshed data to be deleted */ if (orig_ht_reten_job) { if (refresh_total_interval > drop_after_HT) emit_error(err_refresh_reten_ht_overlap); } } /* Cross policy checks */ if (all_policies.refresh && all_policies.compress) { /* Check if refresh policy does not overlap with compression */ if (refresh_total_interval > compress_after) emit_error(err_refresh_compress_overlap); } if (all_policies.refresh && all_policies.retention) { /* Check if refresh policy does not overlap with retention */ if (refresh_total_interval > drop_after) emit_error(err_refresh_reten_overlap); } if (all_policies.retention && all_policies.compress) { /* Check if compression and retention policy overlap */ if (compress_after == drop_after) emit_error(err_compress_reten_overlap); } /* Create policies as required, delete the old ones if coming from alter */ if (all_policies.refresh && all_policies.refresh->create_policy) { NullableDatum include_tiered_data = { .isnull = true }; NullableDatum nbuckets_per_refresh = { .isnull = true }; NullableDatum max_batches_per_execution = { .isnull = true }; NullableDatum refresh_newest_first = { .isnull = true }; if (all_policies.is_alter_policy) policy_refresh_cagg_remove_internal(all_policies.rel_oid, if_exists); refresh_job_id = policy_refresh_cagg_add_internal(all_policies.rel_oid, all_policies.refresh->start_offset_type, all_policies.refresh->start_offset, all_policies.refresh->end_offset_type, all_policies.refresh->end_offset, all_policies.refresh->schedule_interval, false, false, DT_NOBEGIN, NULL, include_tiered_data, nbuckets_per_refresh, max_batches_per_execution, refresh_newest_first); } if (all_policies.compress && all_policies.compress->create_policy) { if (all_policies.is_alter_policy) policy_compression_remove_internal(all_policies.rel_oid, if_exists); compression_job_id = policy_compression_add_internal(all_policies.rel_oid, all_policies.compress->compress_after, all_policies.compress->compress_after_type, NULL, DEFAULT_COMPRESSION_SCHEDULE_INTERVAL, false, if_exists, false, DT_NOBEGIN, NULL); } if (all_policies.retention && all_policies.retention->create_policy) { if (all_policies.is_alter_policy) policy_retention_remove_internal(all_policies.rel_oid, if_exists); retention_job_id = policy_retention_add_internal(all_policies.rel_oid, all_policies.retention->drop_after_type, all_policies.retention->drop_after, NULL, (Interval) DEFAULT_RETENTION_SCHEDULE_INTERVAL, false, false, DT_NOBEGIN, NULL); } return (refresh_job_id || compression_job_id || retention_job_id); } Datum policies_add(PG_FUNCTION_ARGS) { Oid rel_oid; bool if_not_exists; ContinuousAgg *cagg; policies_info all_policies = { .refresh = NULL, .compress = NULL, .retention = NULL }; refresh_policy ref; compression_policy comp; retention_policy ret; ts_feature_flag_check(FEATURE_POLICY); rel_oid = PG_GETARG_OID(0); if_not_exists = PG_GETARG_BOOL(1); cagg = ts_continuous_agg_find_by_relid(rel_oid); if (!cagg) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("\"%s\" is not a continuous aggregate", get_rel_name(rel_oid)))); all_policies.rel_oid = rel_oid; all_policies.is_alter_policy = false; all_policies.original_HT = cagg->data.raw_hypertable_id; all_policies.partition_type = cagg->partition_type; if (!PG_ARGISNULL(2) || !PG_ARGISNULL(3)) { NullableDatum start_offset, end_offset; Oid start_offset_type, end_offset_type; start_offset.isnull = PG_ARGISNULL(2); end_offset.isnull = PG_ARGISNULL(3); start_offset.value = PG_GETARG_DATUM(2); end_offset.value = PG_GETARG_DATUM(3); start_offset_type = get_fn_expr_argtype(fcinfo->flinfo, 2); end_offset_type = get_fn_expr_argtype(fcinfo->flinfo, 3); refresh_policy tmp = { .create_policy = true, .start_offset = start_offset, .end_offset = end_offset, .schedule_interval = *DEFAULT_REFRESH_SCHEDULE_INTERVAL, .start_offset_type = start_offset_type, .end_offset_type = end_offset_type, }; ref = tmp; all_policies.refresh = &ref; } if (!PG_ARGISNULL(4)) { compression_policy tmp = { .create_policy = true, .compress_after = PG_GETARG_DATUM(4), .compress_after_type = get_fn_expr_argtype(fcinfo->flinfo, 4), }; comp = tmp; all_policies.compress = ∁ } if (!PG_ARGISNULL(5)) { retention_policy tmp = { .create_policy = true, .drop_after = PG_GETARG_DATUM(5), .drop_after_type = get_fn_expr_argtype(fcinfo->flinfo, 5) }; ret = tmp; all_policies.retention = &ret; } PG_RETURN_BOOL(validate_and_create_policies(all_policies, if_not_exists)); } Datum policies_remove(PG_FUNCTION_ARGS) { Oid cagg_oid = PG_GETARG_OID(0); ArrayType *policy_array = PG_ARGISNULL(2) ? NULL : PG_GETARG_ARRAYTYPE_P(2); bool if_exists = PG_GETARG_BOOL(1); Datum *policy; int npolicies, failures = 0; int i; bool success = false; ts_feature_flag_check(FEATURE_POLICY); if (policy_array == NULL) PG_RETURN_BOOL(false); deconstruct_array(policy_array, TEXTOID, -1, false, TYPALIGN_INT, &policy, NULL, &npolicies); for (i = 0; i < npolicies; i++) { char *curr_policy = VARDATA(policy[i]); if (pg_strcasecmp(curr_policy, POLICY_REFRESH_CAGG_PROC_NAME) == 0) success = policy_refresh_cagg_remove_internal(cagg_oid, if_exists); else if (pg_strcasecmp(curr_policy, POLICY_COMPRESSION_PROC_NAME) == 0) success = policy_compression_remove_internal(cagg_oid, if_exists); else if (pg_strncasecmp(curr_policy, POLICY_RETENTION_PROC_NAME, strlen(POLICY_RETENTION_PROC_NAME)) == 0) success = policy_retention_remove_internal(cagg_oid, if_exists); else ereport(NOTICE, (errmsg("No relevant policy found"))); if (!success) ++failures; } PG_RETURN_BOOL(success && (0 == failures)); } Datum policies_remove_all(PG_FUNCTION_ARGS) { if (PG_ARGISNULL(0)) PG_RETURN_BOOL(false); Oid cagg_oid = PG_GETARG_OID(0); bool if_exists = PG_GETARG_BOOL(1); List *jobs; ListCell *lc; bool success = if_exists; int failures = 0; ContinuousAgg *cagg = ts_continuous_agg_find_by_relid(cagg_oid); ts_feature_flag_check(FEATURE_POLICY); if (!cagg) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("\"%s\" is not a continuous aggregate", get_rel_name(cagg_oid)))); jobs = ts_bgw_job_find_by_hypertable_id(cagg->data.mat_hypertable_id); foreach (lc, jobs) { BgwJob *job = lfirst(lc); if (namestrcmp(&(job->fd.proc_name), POLICY_REFRESH_CAGG_PROC_NAME) == 0) success = policy_refresh_cagg_remove_internal(cagg_oid, if_exists); else if (namestrcmp(&(job->fd.proc_name), POLICY_COMPRESSION_PROC_NAME) == 0) success = policy_compression_remove_internal(cagg_oid, if_exists); else if (namestrcmp(&(job->fd.proc_name), POLICY_RETENTION_PROC_NAME) == 0) success = policy_retention_remove_internal(cagg_oid, if_exists); else ereport(NOTICE, (errmsg("Ignoring custom job"))); if (!success) ++failures; } PG_RETURN_BOOL(success && (0 == failures)); } Datum policies_alter(PG_FUNCTION_ARGS) { Oid rel_oid = PG_GETARG_OID(0); ContinuousAgg *cagg; List *jobs; ListCell *lc; bool if_exists = false, found, start_found, end_found; policies_info all_policies = { .refresh = NULL, .compress = NULL, .retention = NULL }; refresh_policy ref; compression_policy comp; retention_policy ret; ts_feature_flag_check(FEATURE_POLICY); cagg = ts_continuous_agg_find_by_relid(rel_oid); if (!cagg) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("\"%s\" is not a continuous aggregate", get_rel_name(rel_oid)))); all_policies.is_alter_policy = true; all_policies.rel_oid = rel_oid; all_policies.original_HT = cagg->data.raw_hypertable_id; all_policies.partition_type = cagg->partition_type; jobs = ts_bgw_job_find_by_hypertable_id(cagg->data.mat_hypertable_id); if ((NIL == jobs)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("no jobs found"))); foreach (lc, jobs) { BgwJob *job = lfirst(lc); if (namestrcmp(&(job->fd.proc_name), POLICY_REFRESH_CAGG_PROC_NAME) == 0) { refresh_policy tmp = { .create_policy = false, .end_offset = { 0 }, .end_offset_type = InvalidOid, .start_offset = { 0 }, .start_offset_type = InvalidOid, .schedule_interval = job->fd.schedule_interval }; ref = tmp; all_policies.refresh = &ref; if (IS_INTEGER_TYPE(cagg->partition_type)) { int64 start_value = ts_jsonb_get_int64_field(job->fd.config, POL_REFRESH_CONF_KEY_START_OFFSET, &start_found); int64 end_value = ts_jsonb_get_int64_field(job->fd.config, POL_REFRESH_CONF_KEY_END_OFFSET, &end_found); /* * If there is job then start_offset has to be there because policy is * not created without it. However if found it to be NULL, then we * want to keep it to NULL in this alter command also. */ all_policies.refresh->start_offset.isnull = !start_found; all_policies.refresh->start_offset_type = cagg->partition_type; all_policies.refresh->end_offset.isnull = !end_found; all_policies.refresh->end_offset_type = cagg->partition_type; switch (all_policies.refresh->start_offset_type) { case INT2OID: all_policies.refresh->start_offset.value = Int16GetDatum((int16) start_value); all_policies.refresh->end_offset.value = Int16GetDatum((int16) end_value); break; case INT4OID: all_policies.refresh->start_offset.value = Int32GetDatum((int32) start_value); all_policies.refresh->end_offset.value = Int32GetDatum((int32) end_value); break; case INT8OID: all_policies.refresh->start_offset.value = Int64GetDatum(start_value); all_policies.refresh->end_offset.value = Int64GetDatum(end_value); break; default: Assert(0); } } else { all_policies.refresh->start_offset.value = IntervalPGetDatum( ts_jsonb_get_interval_field(job->fd.config, POL_REFRESH_CONF_KEY_START_OFFSET)); all_policies.refresh->start_offset.isnull = (DatumGetIntervalP(all_policies.refresh->start_offset.value) == NULL); all_policies.refresh->start_offset_type = INTERVALOID; all_policies.refresh->end_offset.value = IntervalPGetDatum( ts_jsonb_get_interval_field(job->fd.config, POL_REFRESH_CONF_KEY_END_OFFSET)); all_policies.refresh->end_offset.isnull = (DatumGetIntervalP(all_policies.refresh->end_offset.value) == NULL); all_policies.refresh->end_offset_type = INTERVALOID; } } else if (namestrcmp(&(job->fd.proc_name), POLICY_COMPRESSION_PROC_NAME) == 0) { compression_policy tmp = { .compress_after = 0, .compress_after_type = InvalidOid, .create_policy = false }; comp = tmp; all_policies.compress = ∁ if (IS_INTEGER_TYPE(cagg->partition_type)) { int64 compress_value = ts_jsonb_get_int64_field(job->fd.config, POL_COMPRESSION_CONF_KEY_COMPRESS_AFTER, &found); all_policies.compress->compress_after_type = cagg->partition_type; switch (all_policies.compress->compress_after_type) { case INT2OID: all_policies.compress->compress_after = Int16GetDatum((int16) compress_value); break; case INT4OID: all_policies.compress->compress_after = Int32GetDatum((int32) compress_value); break; case INT8OID: all_policies.compress->compress_after = Int64GetDatum(compress_value); break; default: Assert(0); } } else { all_policies.compress->compress_after = IntervalPGetDatum( ts_jsonb_get_interval_field(job->fd.config, POL_COMPRESSION_CONF_KEY_COMPRESS_AFTER)); all_policies.compress->compress_after_type = INTERVALOID; } } else if (namestrcmp(&(job->fd.proc_name), POLICY_RETENTION_PROC_NAME) == 0) { retention_policy tmp = { .create_policy = false, .drop_after = 0, .drop_after_type = InvalidOid }; ret = tmp; all_policies.retention = &ret; if (IS_INTEGER_TYPE(cagg->partition_type)) { int64 drop_value = ts_jsonb_get_int64_field(job->fd.config, POL_RETENTION_CONF_KEY_DROP_AFTER, &found); all_policies.retention->drop_after_type = cagg->partition_type; switch (all_policies.retention->drop_after_type) { case INT2OID: all_policies.retention->drop_after = Int16GetDatum((int16) drop_value); break; case INT4OID: all_policies.retention->drop_after = Int32GetDatum((int32) drop_value); break; case INT8OID: all_policies.retention->drop_after = Int64GetDatum(drop_value); break; default: Assert(0); } } else { all_policies.retention->drop_after = IntervalPGetDatum( ts_jsonb_get_interval_field(job->fd.config, POL_RETENTION_CONF_KEY_DROP_AFTER)); all_policies.retention->drop_after_type = INTERVALOID; } } } if (!PG_ARGISNULL(2)) { if (!all_policies.refresh) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("no refresh job found"))); all_policies.refresh->start_offset.value = PG_GETARG_DATUM(2); all_policies.refresh->start_offset_type = get_fn_expr_argtype(fcinfo->flinfo, 2); all_policies.refresh->start_offset.isnull = false; all_policies.refresh->create_policy = true; } if (!PG_ARGISNULL(3)) { if (!all_policies.refresh) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("no refresh job found"))); all_policies.refresh->end_offset.value = PG_GETARG_DATUM(3); all_policies.refresh->end_offset_type = get_fn_expr_argtype(fcinfo->flinfo, 3); all_policies.refresh->end_offset.isnull = false; all_policies.refresh->create_policy = true; } if (!PG_ARGISNULL(4)) { if (!all_policies.compress) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("no compress job found"))); all_policies.compress->compress_after = PG_GETARG_DATUM(4); all_policies.compress->compress_after_type = get_fn_expr_argtype(fcinfo->flinfo, 4); all_policies.compress->create_policy = true; } if (!PG_ARGISNULL(5)) { if (!all_policies.retention) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("no retention job found"))); all_policies.retention->drop_after = PG_GETARG_DATUM(5); all_policies.retention->drop_after_type = get_fn_expr_argtype(fcinfo->flinfo, 5); all_policies.retention->create_policy = true; } PG_RETURN_BOOL(validate_and_create_policies(all_policies, if_exists)); } static void push_to_json(Oid type, JsonbParseState *parse_state, BgwJob *job, char *json_label, char *show_config) { if (IS_INTEGER_TYPE(type)) { bool found; int64 value = ts_jsonb_get_int64_field(job->fd.config, json_label, &found); if (!found) ts_jsonb_add_null(parse_state, show_config); else ts_jsonb_add_int64(parse_state, show_config, value); } else { Interval *value = ts_jsonb_get_interval_field(job->fd.config, json_label); if (value == NULL) ts_jsonb_add_null(parse_state, show_config); else ts_jsonb_add_interval(parse_state, show_config, value); } } Datum policies_show(PG_FUNCTION_ARGS) { Oid rel_oid = PG_GETARG_OID(0); Oid type; ContinuousAgg *cagg; ListCell *lc; FuncCallContext *funcctx; static List *jobs; JsonbParseState *parse_state = NULL; ts_feature_flag_check(FEATURE_POLICY); cagg = ts_continuous_agg_find_by_relid(rel_oid); if (!cagg) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("\"%s\" is not a continuous aggregate", get_rel_name(rel_oid)))); type = IS_TIMESTAMP_TYPE(cagg->partition_type) ? INTERVALOID : cagg->partition_type; pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); if (SRF_IS_FIRSTCALL()) { MemoryContext oldcontext; funcctx = SRF_FIRSTCALL_INIT(); oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); /* Use top-level memory context to preserve the global static list */ jobs = ts_bgw_job_find_by_hypertable_id(cagg->data.mat_hypertable_id); funcctx->user_fctx = list_head(jobs); MemoryContextSwitchTo(oldcontext); } funcctx = SRF_PERCALL_SETUP(); lc = (ListCell *) funcctx->user_fctx; /* * clang doesn't understand that jobs won't be NIL due to the above FIRSTCALL. However * it's also possible that the ts_bgw_job_find_by_hypertable_id function above doesn't * find a job for this hypertable */ if (lc == NULL || jobs == NULL) SRF_RETURN_DONE(funcctx); else { BgwJob *job = lfirst(lc); if (!namestrcmp(&(job->fd.proc_name), POLICY_REFRESH_CAGG_PROC_NAME)) { ts_jsonb_add_str(parse_state, SHOW_POLICY_KEY_POLICY_NAME, POLICY_REFRESH_CAGG_PROC_NAME); push_to_json(type, parse_state, job, POL_REFRESH_CONF_KEY_START_OFFSET, SHOW_POLICY_KEY_REFRESH_START_OFFSET); push_to_json(type, parse_state, job, POL_REFRESH_CONF_KEY_END_OFFSET, SHOW_POLICY_KEY_REFRESH_END_OFFSET); ts_jsonb_add_interval(parse_state, SHOW_POLICY_KEY_REFRESH_INTERVAL, &(job->fd.schedule_interval)); } else if (!namestrcmp(&(job->fd.proc_name), POLICY_COMPRESSION_PROC_NAME)) { ts_jsonb_add_str(parse_state, SHOW_POLICY_KEY_POLICY_NAME, POLICY_COMPRESSION_PROC_NAME); push_to_json(type, parse_state, job, POL_COMPRESSION_CONF_KEY_COMPRESS_AFTER, SHOW_POLICY_KEY_COMPRESS_AFTER); /* POL_COMPRESSION_CONF_KEY_COMPRESS_CREATED_BEFORE not supported with caggs */ ts_jsonb_add_interval(parse_state, SHOW_POLICY_KEY_COMPRESS_INTERVAL, &(job->fd.schedule_interval)); } else if (!namestrcmp(&(job->fd.proc_name), POLICY_RETENTION_PROC_NAME)) { ts_jsonb_add_str(parse_state, SHOW_POLICY_KEY_POLICY_NAME, POLICY_RETENTION_PROC_NAME); push_to_json(type, parse_state, job, POL_RETENTION_CONF_KEY_DROP_AFTER, SHOW_POLICY_KEY_DROP_AFTER); /* POL_RETENTION_CONF_KEY_DROP_CREATED_BEFORE not supported with caggs */ ts_jsonb_add_interval(parse_state, SHOW_POLICY_KEY_RETENTION_INTERVAL, &(job->fd.schedule_interval)); } else ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("\"%s\" unsupported proc", NameStr(job->fd.proc_name)))); JsonbValue *result = pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); funcctx->user_fctx = lnext(jobs, (ListCell *) funcctx->user_fctx); SRF_RETURN_NEXT(funcctx, PointerGetDatum(JsonbValueToJsonb(result))); } PG_RETURN_NULL(); } ================================================ FILE: tsl/src/bgw_policy/policies_v2.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include "compression/api.h" #include "dimension.h" #include <bgw_policy/compression_api.h> #include <bgw_policy/continuous_aggregate_api.h> #include <bgw_policy/retention_api.h> #include <continuous_aggs/materialize.h> #include <utils/jsonb.h> #define POLICY_REFRESH_CAGG_PROC_NAME "policy_refresh_continuous_aggregate" #define POLICY_REFRESH_CAGG_CHECK_NAME "policy_refresh_continuous_aggregate_check" #define POL_REFRESH_CONF_KEY_MAT_HYPERTABLE_ID "mat_hypertable_id" #define POL_REFRESH_CONF_KEY_START_OFFSET "start_offset" #define POL_REFRESH_CONF_KEY_END_OFFSET "end_offset" #define POL_REFRESH_CONF_KEY_INCLUDE_TIERED_DATA "include_tiered_data" #define POL_REFRESH_CONF_KEY_BUCKETS_PER_BATCH "buckets_per_batch" #define POL_REFRESH_CONF_KEY_MAX_BATCHES_PER_EXECUTION "max_batches_per_execution" #define POL_REFRESH_CONF_KEY_REFRESH_NEWEST_FIRST "refresh_newest_first" #define POL_REFRESH_CONF_KEY_PROCESS_HYPERTABLE_INVALIDATIONS "process_hypertable_invalidations" #define POLICY_COMPRESSION_PROC_NAME "policy_compression" #define POLICY_COMPRESSION_CHECK_NAME "policy_compression_check" #define POL_COMPRESSION_CONF_KEY_COMPRESS_AFTER "compress_after" #define POL_COMPRESSION_CONF_KEY_MAXCHUNKS_TO_COMPRESS "maxchunks_to_compress" #define POL_COMPRESSION_CONF_KEY_COMPRESS_CREATED_BEFORE "compress_created_before" #define POLICY_RECOMPRESSION_PROC_NAME "policy_recompression" #define POL_RECOMPRESSION_CONF_KEY_RECOMPRESS_AFTER "recompress_after" #define POLICY_RETENTION_PROC_NAME "policy_retention" #define POLICY_RETENTION_CHECK_NAME "policy_retention_check" #define POL_RETENTION_CONF_KEY_DROP_AFTER "drop_after" #define POL_RETENTION_CONF_KEY_DROP_CREATED_BEFORE "drop_created_before" #define SHOW_POLICY_KEY_POLICY_NAME "policy_name" #define SHOW_POLICY_KEY_REFRESH_INTERVAL "refresh_interval" #define SHOW_POLICY_KEY_REFRESH_START_OFFSET "refresh_start_offset" #define SHOW_POLICY_KEY_REFRESH_END_OFFSET "refresh_end_offset" #define SHOW_POLICY_KEY_COMPRESS_AFTER POL_COMPRESSION_CONF_KEY_COMPRESS_AFTER #define SHOW_POLICY_KEY_COMPRESS_CREATED_BEFORE POL_COMPRESSION_CONF_KEY_COMPRESS_CREATED_BEFORE #define SHOW_POLICY_KEY_COMPRESS_INTERVAL "compress_interval" #define SHOW_POLICY_KEY_DROP_AFTER POL_RETENTION_CONF_KEY_DROP_AFTER #define SHOW_POLICY_KEY_DROP_CREATED_BEFORE POL_RETENTION_CONF_KEY_DROP_CREATED_BEFORE #define SHOW_POLICY_KEY_RETENTION_INTERVAL "retention_interval" #define DEFAULT_RETENTION_SCHEDULE_INTERVAL \ { \ .day = 1 \ } /* * Default scheduled interval for compress jobs = default chunk length. * If this is non-timestamp based hypertable, then default is 1 day */ #define DEFAULT_COMPRESSION_SCHEDULE_INTERVAL \ DatumGetIntervalP(DirectFunctionCall3(interval_in, CStringGetDatum("1 day"), InvalidOid, -1)) #define DEFAULT_REFRESH_SCHEDULE_INTERVAL \ DatumGetIntervalP(DirectFunctionCall3(interval_in, CStringGetDatum("1 hour"), InvalidOid, -1)) extern Datum policies_add(PG_FUNCTION_ARGS); extern Datum policies_remove(PG_FUNCTION_ARGS); extern Datum policies_remove_all(PG_FUNCTION_ARGS); extern Datum policies_alter(PG_FUNCTION_ARGS); extern Datum policies_show(PG_FUNCTION_ARGS); typedef struct CaggPolicyConfig { Oid partition_type; ContinuousAggPolicyOffset offset_start; ContinuousAggPolicyOffset offset_end; } CaggPolicyConfig; typedef struct refresh_policy { Interval schedule_interval; NullableDatum start_offset; NullableDatum end_offset; Oid start_offset_type, end_offset_type; bool create_policy; } refresh_policy; typedef struct compression_policy { Datum compress_after; Oid compress_after_type; bool create_policy; } compression_policy; typedef struct retention_policy { Datum drop_after; Oid drop_after_type; bool create_policy; } retention_policy; typedef struct policies_info { Oid rel_oid; Oid original_HT; Oid partition_type; refresh_policy *refresh; compression_policy *compress; retention_policy *retention; bool is_alter_policy; } policies_info; bool ts_if_offset_is_infinity(Datum arg, Oid argtype, bool is_start); bool validate_and_create_policies(policies_info all_policies, bool if_exists); int64 interval_to_int64(Datum interval, Oid type); ================================================ FILE: tsl/src/bgw_policy/policy_config.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Module with common functions, variables, and constants for working with * policy configurations. */ #include <postgres.h> #include "jsonb_utils.h" #include "policies_v2.h" #include "policy_config.h" int32 policy_config_get_hypertable_id(const Jsonb *config) { bool found; int32 hypertable_id = ts_jsonb_get_int32_field(config, POLICY_CONFIG_KEY_HYPERTABLE_ID, &found); if (!found) ereport(ERROR, (errcode(ERRCODE_SQL_JSON_MEMBER_NOT_FOUND), errmsg("could not find hypertable_id in config for job"))); return hypertable_id; } /* Helper function to compare jsonb label value in the config * with passed in value. * This function is used for labels defined on the hypertable's dimension * Parameters: * config - jsonb config value * label - label we are looking for inside the config * partitioning_type - Oid for hypertable's dimension column * lag_value - value we will compare against the config's * value for the label * lag_type - Oid for lag_value * Returns: * True, if config value is equal to lag_value */ bool policy_config_check_hypertable_lag_equality(Jsonb *config, const char *json_label, Oid partitioning_type, Oid lag_type, Datum lag_datum, bool isnull) { /* * start_offset and end_offset for CAgg policies are allowed to have NULL values * In that case, config_value will be NULL but this is not an error */ bool null_ok = (strcmp(json_label, POL_REFRESH_CONF_KEY_END_OFFSET) == 0 || strcmp(json_label, POL_REFRESH_CONF_KEY_START_OFFSET) == 0); if (IS_INTEGER_TYPE(partitioning_type) && lag_type != INTERVALOID) { bool found; int64 config_value = ts_jsonb_get_int64_field(config, json_label, &found); if (!found && !null_ok) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not find %s in config for existing job", json_label))); if (!found && isnull) return true; if ((!found && !isnull) || (found && isnull)) return false; switch (lag_type) { case INT2OID: return config_value == DatumGetInt16(lag_datum); case INT4OID: return config_value == DatumGetInt32(lag_datum); case INT8OID: return config_value == DatumGetInt64(lag_datum); default: return false; } } else { if (lag_type != INTERVALOID) return false; Interval *config_value = ts_jsonb_get_interval_field(config, json_label); if (config_value == NULL && !null_ok) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not find %s in config for job", json_label))); if (config_value == NULL && isnull) return true; if ((config_value == NULL && !isnull) || (config_value != NULL && isnull)) return false; return DatumGetBool( DirectFunctionCall2(interval_eq, IntervalPGetDatum(config_value), lag_datum)); } } ================================================ FILE: tsl/src/bgw_policy/policy_config.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once /* * Common functions, variables, and constants for working with policy * configurations. */ #include <postgres.h> #include <utils/jsonb.h> #define POLICY_CONFIG_KEY_HYPERTABLE_ID "hypertable_id" extern int32 policy_config_get_hypertable_id(const Jsonb *config); extern bool policy_config_check_hypertable_lag_equality(Jsonb *config, const char *json_label, Oid partitioning_type, Oid lag_type, Datum lag_datum, bool isnull); ================================================ FILE: tsl/src/bgw_policy/policy_utils.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include "policy_utils.h" #include "dimension.h" #include "errors.h" #include "guc.h" #include "hypertable.h" #include "jsonb_utils.h" #include "policies_v2.h" #include "time_utils.h" #include "ts_catalog/continuous_agg.h" #include <utils/builtins.h> const Dimension * get_open_dimension_for_hypertable(const Hypertable *ht, bool fail_if_not_found) { int32 mat_id = ht->fd.id; if (TS_HYPERTABLE_IS_INTERNAL_COMPRESSION_TABLE(ht)) elog(ERROR, "invalid operation on compressed hypertable"); const Dimension *open_dim = hyperspace_get_open_dimension(ht->space, 0); Oid partitioning_type = ts_dimension_get_partition_type(open_dim); if (IS_INTEGER_TYPE(partitioning_type)) { /* if this a materialization hypertable related to cont agg * then need to get the right dimension which has * integer_now function */ open_dim = ts_continuous_agg_find_integer_now_func_by_materialization_id(mat_id); if (open_dim == NULL && fail_if_not_found) { ereport(ERROR, (errcode(ERRCODE_TS_UNEXPECTED), errmsg("missing integer_now function for hypertable \"%s\" ", get_rel_name(ht->main_table_relid)))); } } return open_dim; } bool policy_get_verbose_log(const Jsonb *config) { bool found; bool verbose_log = ts_jsonb_get_bool_field(config, CONFIG_KEY_VERBOSE_LOG, &found); return found ? verbose_log : false; } ================================================ FILE: tsl/src/bgw_policy/policy_utils.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include "job.h" int64 subtract_integer_from_now_internal(int64 interval, Oid time_dim_type, Oid now_func, bool *overflow); const Dimension *get_open_dimension_for_hypertable(const Hypertable *ht, bool fail_if_not_found); bool policy_get_verbose_log(const Jsonb *config); ================================================ FILE: tsl/src/bgw_policy/process_hyper_inval_api.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include "process_hyper_inval_api.h" #include "bgw/job.h" #include "bgw/job_stat.h" #include "bgw/timer.h" #include "bgw_policy/job.h" #include "bgw_policy/job_api.h" #include "guc.h" #include "hypertable.h" #include "jsonb_utils.h" #include "policy_config.h" #include <utils/elog.h> #define DEFAULT_MAX_RUNTIME \ DatumGetIntervalP(DirectFunctionCall3(interval_in, CStringGetDatum("0"), InvalidOid, -1)) Datum policy_process_hyper_inval_proc(PG_FUNCTION_ARGS) { if (PG_NARGS() != 2 || PG_ARGISNULL(0) || PG_ARGISNULL(1)) PG_RETURN_VOID(); ts_feature_flag_check(FEATURE_POLICY); TS_PREVENT_FUNC_IF_READ_ONLY(); policy_process_hyper_inval_execute(PG_GETARG_INT32(0), PG_GETARG_JSONB_P(1)); PG_RETURN_VOID(); } Datum policy_process_hyper_inval_check(PG_FUNCTION_ARGS) { if (PG_ARGISNULL(0)) { ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("config must not be NULL"))); } policy_process_hyper_inval_read_and_validate_config(PG_GETARG_JSONB_P(0), NULL); PG_RETURN_VOID(); } static int32 policy_process_hyper_inval_add_internal(Hypertable *ht, bool if_not_exists, Interval *schedule_interval, Oid owner_id, TimestampTz initial_start, const bool fixed_schedule, char *timezone) { NameData application_name, proc_name, proc_schema, check_schema, check_name, owner; List *jobs = ts_bgw_job_find_by_proc_and_hypertable_id(POLICY_PROCESS_HYPER_INVAL_PROC_NAME, FUNCTIONS_SCHEMA_NAME, ht->fd.id); if (jobs != NIL) { Assert(list_length(jobs) == 1); if (!if_not_exists) ereport(ERROR, errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("move hypertable invalidations policy already exists for \"%s\"", get_rel_name(ht->main_table_relid))); else ereport(NOTICE, errmsg("move hypertable invalidations policy already exists for \"%s\", " "skipping", get_rel_name(ht->main_table_relid))); return -1; } namestrcpy(&application_name, "Move Hypertables Invalidation Policy"); namestrcpy(&proc_name, POLICY_PROCESS_HYPER_INVAL_PROC_NAME); namestrcpy(&proc_schema, FUNCTIONS_SCHEMA_NAME); namestrcpy(&check_name, POLICY_PROCESS_HYPER_INVAL_CHECK_NAME); namestrcpy(&check_schema, FUNCTIONS_SCHEMA_NAME); namestrcpy(&owner, GetUserNameFromId(owner_id, false)); JsonbParseState *parse_state = NULL; pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); ts_jsonb_add_int32(parse_state, POLICY_CONFIG_KEY_HYPERTABLE_ID, ht->fd.id); JsonbValue *result = pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); Jsonb *config = JsonbValueToJsonb(result); return ts_bgw_job_insert_relation(&application_name, schedule_interval, DEFAULT_MAX_RUNTIME, JOB_RETRY_UNLIMITED, schedule_interval, &proc_schema, &proc_name, &check_schema, &check_name, owner_id, true, fixed_schedule, ht->fd.id, config, initial_start, timezone); } Datum policy_process_hyper_inval_add(PG_FUNCTION_ARGS) { Oid ht_oid = PG_GETARG_OID(0); Interval *schedule_interval = PG_GETARG_INTERVAL_P(1); const bool if_not_exists = PG_GETARG_BOOL(2); TimestampTz initial_start = PG_ARGISNULL(3) ? DT_NOBEGIN : PG_GETARG_TIMESTAMPTZ(3); const bool fixed_schedule = !PG_ARGISNULL(3); char *valid_timezone = NULL; ts_feature_flag_check(FEATURE_POLICY); TS_PREVENT_FUNC_IF_READ_ONLY(); if (fixed_schedule) { ts_bgw_job_validate_schedule_interval(schedule_interval); if (TIMESTAMP_NOT_FINITE(initial_start)) initial_start = ts_timer_get_current_timestamp(); } if (!PG_ARGISNULL(4)) valid_timezone = ts_bgw_job_validate_timezone(PG_GETARG_DATUM(4)); Cache *hcache; Hypertable *ht = ts_hypertable_cache_get_cache_and_entry(ht_oid, CACHE_FLAG_NONE, &hcache); /* Check that we have a continuous aggregate attached */ List *caggs = ts_continuous_aggs_find_by_raw_table_id(ht->fd.id); if (list_length(caggs) == 0) ereport(ERROR, errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("\"%s\" does not have an associated continuous aggregate", get_rel_name(ht_oid))); Oid owner_id = ts_hypertable_permissions_check(ht_oid, GetUserId()); ts_bgw_job_validate_job_owner(owner_id); int32 job_id = policy_process_hyper_inval_add_internal(ht, if_not_exists, schedule_interval, owner_id, initial_start, fixed_schedule, valid_timezone); ts_cache_release(&hcache); if (!TIMESTAMP_NOT_FINITE(initial_start)) ts_bgw_job_stat_upsert_next_start(job_id, initial_start); PG_RETURN_INT32(job_id); } static void policy_process_hyper_inval_remove_internal(Oid ht_oid, bool if_exists) { Cache *hcache; Hypertable *ht = ts_hypertable_cache_get_cache_and_entry(ht_oid, CACHE_FLAG_NONE, &hcache); if (!ht) ereport(ERROR, errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("\"%s\" is not a hypertable", get_rel_name(ht_oid))); /* Check permissions */ Oid owner_id = ts_hypertable_permissions_check(ht_oid, GetUserId()); ts_bgw_job_validate_job_owner(owner_id); /* Find jobs */ List *jobs = ts_bgw_job_find_by_proc_and_hypertable_id(POLICY_PROCESS_HYPER_INVAL_PROC_NAME, FUNCTIONS_SCHEMA_NAME, ht->fd.id); if (!jobs && !if_exists) ereport(ERROR, errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("move invalidations policy for \"%s\" not found", get_rel_name(ht_oid))); if (!jobs && if_exists) { ereport(NOTICE, errmsg("move invalidations policy for \"%s\" not found, skipping", get_rel_name(ht_oid))); ts_cache_release(&hcache); return; } Assert(list_length(jobs) == 1); BgwJob *job = linitial(jobs); /* Delete all jobs, not just the first one? */ ts_bgw_job_delete_by_id(job->fd.id); ts_cache_release(&hcache); } Datum policy_process_hyper_inval_remove(PG_FUNCTION_ARGS) { ts_feature_flag_check(FEATURE_POLICY); policy_process_hyper_inval_remove_internal(PG_GETARG_OID(0), PG_GETARG_BOOL(1)); PG_RETURN_VOID(); } ================================================ FILE: tsl/src/bgw_policy/process_hyper_inval_api.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <fmgr.h> #define POLICY_PROCESS_HYPER_INVAL_PROC_NAME "policy_process_hypertable_invalidations" #define POLICY_PROCESS_HYPER_INVAL_CHECK_NAME "policy_process_hypertable_invalidations_check" extern Datum policy_process_hyper_inval_add(PG_FUNCTION_ARGS); extern Datum policy_process_hyper_inval_proc(PG_FUNCTION_ARGS); extern Datum policy_process_hyper_inval_check(PG_FUNCTION_ARGS); extern Datum policy_process_hyper_inval_remove(PG_FUNCTION_ARGS); ================================================ FILE: tsl/src/bgw_policy/reorder_api.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <catalog/namespace.h> #include <catalog/pg_collation.h> #include <catalog/pg_type.h> #include <miscadmin.h> #include <utils/builtins.h> #include <utils/lsyscache.h> #include <utils/syscache.h> #include <utils/timestamp.h> #include <compat/compat.h> #include <dimension.h> #include <hypertable_cache.h> #include <jsonb_utils.h> #include "bgw/job.h" #include "bgw/job_stat.h" #include "bgw/timer.h" #include "bgw_policy/job.h" #include "bgw_policy/job_api.h" #include "bgw_policy/policy_config.h" #include "bgw_policy/reorder_api.h" #include "guc.h" #include "hypertable.h" #include "utils.h" /* * Default scheduled interval for reorder jobs should be 1/2 of the default chunk length. * If no such length is specified for the hypertable, then * the default is 4 days, which is approximately 1/2 of the default chunk size, 7 days. */ #define DEFAULT_SCHEDULE_INTERVAL \ { \ .day = 4 \ } /* Default max runtime for a reorder job is unlimited for now */ #define DEFAULT_MAX_RUNTIME \ DatumGetIntervalP(DirectFunctionCall3(interval_in, CStringGetDatum("0"), InvalidOid, -1)) /* Default retry period for reorder_jobs is currently 5 minutes */ #define DEFAULT_RETRY_PERIOD \ DatumGetIntervalP(DirectFunctionCall3(interval_in, CStringGetDatum("5 min"), InvalidOid, -1)) #define CONFIG_KEY_INDEX_NAME "index_name" #define POLICY_REORDER_PROC_NAME "policy_reorder" #define POLICY_REORDER_CHECK_NAME "policy_reorder_check" char * policy_reorder_get_index_name(const Jsonb *config) { char *index_name = NULL; if (config != NULL) index_name = ts_jsonb_get_str_field(config, CONFIG_KEY_INDEX_NAME); if (index_name == NULL) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not find index_name in config for job"))); return index_name; } static void check_valid_index(Hypertable *ht, Name index_name) { Oid index_oid; HeapTuple idxtuple; Form_pg_index indexForm; index_oid = ts_get_relation_relid(NameStr(ht->fd.schema_name), NameStr(*index_name), true); idxtuple = SearchSysCache1(INDEXRELID, ObjectIdGetDatum(index_oid)); if (!HeapTupleIsValid(idxtuple)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid reorder index"))); indexForm = (Form_pg_index) GETSTRUCT(idxtuple); if (indexForm->indrelid != ht->main_table_relid) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid reorder index"), errhint("The reorder index must by an index on hypertable \"%s\".", NameStr(ht->fd.table_name)))); ReleaseSysCache(idxtuple); } Datum policy_reorder_check(PG_FUNCTION_ARGS) { TS_PREVENT_FUNC_IF_READ_ONLY(); if (PG_ARGISNULL(0)) { ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("config must not be NULL"))); } policy_reorder_read_and_validate_config(PG_GETARG_JSONB_P(0), NULL); PG_RETURN_VOID(); } Datum policy_reorder_proc(PG_FUNCTION_ARGS) { if (PG_NARGS() != 2 || PG_ARGISNULL(0) || PG_ARGISNULL(1)) PG_RETURN_VOID(); ts_feature_flag_check(FEATURE_POLICY); TS_PREVENT_FUNC_IF_READ_ONLY(); policy_reorder_execute(PG_GETARG_INT32(0), PG_GETARG_JSONB_P(1)); PG_RETURN_VOID(); } Datum policy_reorder_add(PG_FUNCTION_ARGS) { /* behave like a strict function */ if (PG_ARGISNULL(0) || PG_ARGISNULL(1) || PG_ARGISNULL(2)) PG_RETURN_NULL(); NameData application_name; NameData proc_name, proc_schema, check_name, check_schema, owner; int32 job_id; const Dimension *dim; Interval schedule_interval = DEFAULT_SCHEDULE_INTERVAL; Oid ht_oid = PG_GETARG_OID(0); Name index_name = PG_GETARG_NAME(1); bool if_not_exists = PG_GETARG_BOOL(2); Cache *hcache; Hypertable *ht; int32 hypertable_id; Oid partitioning_type; Oid owner_id; List *jobs; TimestampTz initial_start = PG_ARGISNULL(3) ? DT_NOBEGIN : PG_GETARG_TIMESTAMPTZ(3); bool fixed_schedule = !PG_ARGISNULL(3); text *timezone = PG_ARGISNULL(4) ? NULL : PG_GETARG_TEXT_PP(4); char *valid_timezone = NULL; ts_feature_flag_check(FEATURE_POLICY); TS_PREVENT_FUNC_IF_READ_ONLY(); if (timezone != NULL) valid_timezone = ts_bgw_job_validate_timezone(PG_GETARG_DATUM(4)); ht = ts_hypertable_cache_get_cache_and_entry(ht_oid, CACHE_FLAG_NONE, &hcache); Assert(ht != NULL); hypertable_id = ht->fd.id; /* First verify that the hypertable corresponds to a valid table */ owner_id = ts_hypertable_permissions_check(ht_oid, GetUserId()); if (TS_HYPERTABLE_IS_INTERNAL_COMPRESSION_TABLE(ht)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot add reorder policy to compressed hypertable \"%s\"", get_rel_name(ht_oid)), errhint("Please add the policy to the corresponding uncompressed hypertable " "instead."))); /* Now verify that the index is an actual index on that hypertable */ check_valid_index(ht, index_name); /* Verify that the hypertable owner can create a background worker */ ts_bgw_job_validate_job_owner(owner_id); /* Make sure that an existing reorder policy doesn't exist on this hypertable */ jobs = ts_bgw_job_find_by_proc_and_hypertable_id(POLICY_REORDER_PROC_NAME, FUNCTIONS_SCHEMA_NAME, ht->fd.id); /* * Try to see if the hypertable has a specified chunk length for the * default schedule interval */ dim = hyperspace_get_open_dimension(ht->space, 0); Assert(dim); partitioning_type = ts_dimension_get_partition_type(dim); if (IS_TIMESTAMP_TYPE(partitioning_type)) { schedule_interval.time = dim->fd.interval_length / 2; schedule_interval.day = 0; schedule_interval.month = 0; } ts_cache_release(&hcache); if (jobs != NIL) { BgwJob *existing = linitial(jobs); Assert(list_length(jobs) == 1); if (!if_not_exists) ereport(ERROR, (errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("reorder policy already exists for hypertable \"%s\"", get_rel_name(ht_oid)))); if (!DatumGetBool(DirectFunctionCall2Coll(nameeq, C_COLLATION_OID, CStringGetDatum(policy_reorder_get_index_name( existing->fd.config)), NameGetDatum(index_name)))) { ereport(WARNING, (errmsg("reorder policy already exists for hypertable \"%s\"", get_rel_name(ht_oid)), errdetail("A policy already exists with different arguments."), errhint("Remove the existing policy before adding a new one."))); PG_RETURN_INT32(-1); } /* If all arguments are the same, do nothing */ ereport(NOTICE, (errmsg("reorder policy already exists on hypertable \"%s\", skipping", get_rel_name(ht_oid)))); PG_RETURN_INT32(-1); } /* if users pass in -infinity for initial_start, then use the current_timestamp instead */ if (fixed_schedule) { ts_bgw_job_validate_schedule_interval(&schedule_interval); if (TIMESTAMP_NOT_FINITE(initial_start)) initial_start = ts_timer_get_current_timestamp(); } /* Next, insert a new job into jobs table */ namestrcpy(&application_name, "Reorder Policy"); namestrcpy(&proc_name, POLICY_REORDER_PROC_NAME); namestrcpy(&proc_schema, FUNCTIONS_SCHEMA_NAME); namestrcpy(&check_name, POLICY_REORDER_CHECK_NAME); namestrcpy(&check_schema, FUNCTIONS_SCHEMA_NAME); namestrcpy(&owner, GetUserNameFromId(owner_id, false)); JsonbParseState *parse_state = NULL; pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); ts_jsonb_add_int32(parse_state, POLICY_CONFIG_KEY_HYPERTABLE_ID, hypertable_id); ts_jsonb_add_str(parse_state, CONFIG_KEY_INDEX_NAME, NameStr(*index_name)); JsonbValue *result = pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); Jsonb *config = JsonbValueToJsonb(result); /* for the reorder policy, we choose a drifting schedule since the user does not control the schedule interval either */ job_id = ts_bgw_job_insert_relation(&application_name, &schedule_interval, DEFAULT_MAX_RUNTIME, JOB_RETRY_UNLIMITED, DEFAULT_RETRY_PERIOD, &proc_schema, &proc_name, &check_schema, &check_name, owner_id, true, fixed_schedule, hypertable_id, config, initial_start, valid_timezone); if (!TIMESTAMP_NOT_FINITE(initial_start)) ts_bgw_job_stat_upsert_next_start(job_id, initial_start); PG_RETURN_INT32(job_id); } Datum policy_reorder_remove(PG_FUNCTION_ARGS) { Oid hypertable_oid = PG_GETARG_OID(0); bool if_exists = PG_GETARG_BOOL(1); Hypertable *ht; Cache *hcache; ts_feature_flag_check(FEATURE_POLICY); TS_PREVENT_FUNC_IF_READ_ONLY(); ht = ts_hypertable_cache_get_cache_and_entry(hypertable_oid, CACHE_FLAG_NONE, &hcache); List *jobs = ts_bgw_job_find_by_proc_and_hypertable_id(POLICY_REORDER_PROC_NAME, FUNCTIONS_SCHEMA_NAME, ht->fd.id); ts_cache_release(&hcache); if (jobs == NIL) { if (!if_exists) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("reorder policy not found for hypertable \"%s\"", get_rel_name(hypertable_oid)))); else { ereport(NOTICE, (errmsg("reorder policy not found for hypertable \"%s\", skipping", get_rel_name(hypertable_oid)))); PG_RETURN_NULL(); } } Assert(list_length(jobs) == 1); BgwJob *job = linitial(jobs); ts_hypertable_permissions_check(hypertable_oid, GetUserId()); ts_bgw_job_delete_by_id(job->fd.id); PG_RETURN_NULL(); } ================================================ FILE: tsl/src/bgw_policy/reorder_api.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> /* User-facing API functions */ extern Datum policy_reorder_add(PG_FUNCTION_ARGS); extern Datum policy_reorder_remove(PG_FUNCTION_ARGS); extern Datum policy_reorder_proc(PG_FUNCTION_ARGS); extern Datum policy_reorder_check(PG_FUNCTION_ARGS); extern char *policy_reorder_get_index_name(const Jsonb *config); ================================================ FILE: tsl/src/bgw_policy/retention_api.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <catalog/pg_type.h> #include <miscadmin.h> #include <utils/builtins.h> #include <utils/lsyscache.h> #include <utils/syscache.h> #include <hypertable_cache.h> #include "bgw/job.h" #include "bgw/job_stat.h" #include "bgw/timer.h" #include "bgw_policy/job.h" #include "bgw_policy/policies_v2.h" #include "bgw_policy/policy_config.h" #include "chunk.h" #include "dimension.h" #include "errors.h" #include "guc.h" #include "hypertable.h" #include "jsonb_utils.h" #include "policy_utils.h" #include "retention_api.h" #include "ts_catalog/continuous_agg.h" #include "utils.h" Datum policy_retention_proc(PG_FUNCTION_ARGS) { if (PG_NARGS() != 2 || PG_ARGISNULL(0) || PG_ARGISNULL(1)) PG_RETURN_VOID(); ts_feature_flag_check(FEATURE_POLICY); TS_PREVENT_FUNC_IF_READ_ONLY(); policy_retention_execute(PG_GETARG_INT32(0), PG_GETARG_JSONB_P(1)); PG_RETURN_VOID(); } Datum policy_retention_check(PG_FUNCTION_ARGS) { TS_PREVENT_FUNC_IF_READ_ONLY(); if (PG_ARGISNULL(0)) { ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("config must not be NULL"))); } policy_retention_read_and_validate_config(PG_GETARG_JSONB_P(0), NULL); PG_RETURN_VOID(); } int64 policy_retention_get_drop_after_int(const Jsonb *config) { bool found; int64 drop_after = ts_jsonb_get_int64_field(config, POL_RETENTION_CONF_KEY_DROP_AFTER, &found); if (!found) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not find %s in config for job", POL_RETENTION_CONF_KEY_DROP_AFTER))); return drop_after; } Interval * policy_retention_get_drop_after_interval(const Jsonb *config) { Interval *interval = ts_jsonb_get_interval_field(config, POL_RETENTION_CONF_KEY_DROP_AFTER); if (interval == NULL) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not find %s in config for job", POL_RETENTION_CONF_KEY_DROP_AFTER))); return interval; } Interval * policy_retention_get_drop_created_before_interval(const Jsonb *config) { Interval *interval = ts_jsonb_get_interval_field(config, POL_RETENTION_CONF_KEY_DROP_CREATED_BEFORE); if (interval == NULL) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not find %s in config for job", POL_RETENTION_CONF_KEY_DROP_CREATED_BEFORE))); return interval; } static Hypertable * validate_drop_chunks_hypertable(Cache *hcache, Oid user_htoid) { ContinuousAggHypertableStatus status; Hypertable *ht = ts_hypertable_cache_get_entry(hcache, user_htoid, true /* missing_ok */); if (ht != NULL) { if (TS_HYPERTABLE_IS_INTERNAL_COMPRESSION_TABLE(ht)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot add retention policy to compressed hypertable \"%s\"", get_rel_name(user_htoid)), errhint("Please add the policy to the corresponding uncompressed hypertable " "instead."))); } status = ts_continuous_agg_hypertable_status(ht->fd.id); if ((status == HypertableIsMaterialization || status == HypertableIsMaterializationAndRaw)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot add retention policy to materialized hypertable \"%s\" ", get_rel_name(user_htoid)), errhint("Please add the policy to the corresponding continuous aggregate " "instead."))); } } else { /*check if this is a cont aggregate view */ int32 mat_id; ContinuousAgg *ca; ca = ts_continuous_agg_find_by_relid(user_htoid); if (ca == NULL) ereport(ERROR, (errcode(ERRCODE_TS_HYPERTABLE_NOT_EXIST), errmsg("\"%s\" is not a hypertable or a continuous aggregate", get_rel_name(user_htoid)))); mat_id = ca->data.mat_hypertable_id; ht = ts_hypertable_get_by_id(mat_id); } Assert(ht != NULL); return ht; } Datum policy_retention_add_internal(Oid ht_oid, Oid window_type, Datum window_datum, Interval *created_before, Interval default_schedule_interval, bool if_not_exists, bool fixed_schedule, TimestampTz initial_start, const char *timezone) { NameData application_name; int32 job_id; Hypertable *hypertable; Cache *hcache; Oid owner_id = ts_hypertable_permissions_check(ht_oid, GetUserId()); Oid partitioning_type; const Dimension *dim; /* Default scheduled interval for drop_chunks jobs is currently 1 day (24 hours) */ /* Default max runtime should not be very long. Right now set to 5 minutes */ Interval default_max_runtime = { .time = 5 * USECS_PER_MINUTE }; /* Default retry period is currently 5 minutes */ Interval default_retry_period = { .time = 5 * USECS_PER_MINUTE }; /* Right now, there is an infinite number of retries for drop_chunks jobs */ int default_max_retries = -1; /* Verify that the hypertable owner can create a background worker */ ts_bgw_job_validate_job_owner(owner_id); /* Make sure that an existing policy doesn't exist on this hypertable */ hcache = ts_hypertable_cache_pin(); hypertable = validate_drop_chunks_hypertable(hcache, ht_oid); dim = hyperspace_get_open_dimension(hypertable->space, 0); partitioning_type = ts_dimension_get_partition_type(dim); List *jobs = ts_bgw_job_find_by_proc_and_hypertable_id(POLICY_RETENTION_PROC_NAME, FUNCTIONS_SCHEMA_NAME, hypertable->fd.id); if (jobs != NIL) { bool is_equal = false; if (!if_not_exists) ereport(ERROR, (errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("retention policy already exists for hypertable \"%s\"", get_rel_name(ht_oid)))); Assert(list_length(jobs) == 1); BgwJob *existing = linitial(jobs); if (OidIsValid(window_type)) is_equal = policy_config_check_hypertable_lag_equality(existing->fd.config, POL_RETENTION_CONF_KEY_DROP_AFTER, partitioning_type, window_type, window_datum, false /* isnull */); else { Assert(created_before != NULL); is_equal = policy_config_check_hypertable_lag_equality( existing->fd.config, POL_RETENTION_CONF_KEY_DROP_CREATED_BEFORE, partitioning_type, INTERVALOID, IntervalPGetDatum(created_before), false /* isnull */); } if (is_equal) { /* If all arguments are the same, do nothing */ ts_cache_release(&hcache); ereport(NOTICE, (errmsg("retention policy already exists for hypertable \"%s\", skipping", get_rel_name(ht_oid)))); PG_RETURN_INT32(-1); } else { ts_cache_release(&hcache); ereport(WARNING, (errmsg("retention policy already exists for hypertable \"%s\"", get_rel_name(ht_oid)), errdetail("A policy already exists with different arguments."), errhint("Remove the existing policy before adding a new one."))); PG_RETURN_INT32(-1); } } if (created_before) { Assert(!OidIsValid(window_type)); window_type = INTERVALOID; } if (IS_INTEGER_TYPE(partitioning_type)) { ContinuousAgg *cagg = ts_continuous_agg_find_by_relid(ht_oid); if ((IS_INTEGER_TYPE(window_type) && cagg == NULL && !OidIsValid(ts_get_integer_now_func(dim, false))) || (!IS_INTEGER_TYPE(window_type) && created_before == NULL)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid value for parameter %s", POL_RETENTION_CONF_KEY_DROP_AFTER), errhint( "Integer duration in \"drop_after\" with valid \"integer_now\" function" " or interval time duration" " in \"drop_created_before\" is required for hypertables with integer " "time dimension."))); } if ((IS_TIMESTAMP_TYPE(partitioning_type) || IS_UUID_TYPE(partitioning_type)) && window_type != INTERVALOID) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid value for parameter %s", POL_RETENTION_CONF_KEY_DROP_AFTER), errhint("Interval time duration is required for hypertable" " with timestamp or UUID time dimension."))); JsonbParseState *parse_state = NULL; pushJsonbValue(&parse_state, WJB_BEGIN_OBJECT, NULL); ts_jsonb_add_int32(parse_state, POLICY_CONFIG_KEY_HYPERTABLE_ID, hypertable->fd.id); switch (window_type) { case INTERVALOID: if (created_before) ts_jsonb_add_interval(parse_state, POL_RETENTION_CONF_KEY_DROP_CREATED_BEFORE, created_before); else ts_jsonb_add_interval(parse_state, POL_RETENTION_CONF_KEY_DROP_AFTER, DatumGetIntervalP(window_datum)); break; case INT2OID: ts_jsonb_add_int64(parse_state, POL_RETENTION_CONF_KEY_DROP_AFTER, DatumGetInt16(window_datum)); break; case INT4OID: ts_jsonb_add_int64(parse_state, POL_RETENTION_CONF_KEY_DROP_AFTER, DatumGetInt32(window_datum)); break; case INT8OID: ts_jsonb_add_int64(parse_state, POL_RETENTION_CONF_KEY_DROP_AFTER, DatumGetInt64(window_datum)); break; default: ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported datatype for %s: %s", POL_RETENTION_CONF_KEY_DROP_AFTER, format_type_be(window_type)))); } JsonbValue *result = pushJsonbValue(&parse_state, WJB_END_OBJECT, NULL); Jsonb *config = JsonbValueToJsonb(result); /* Next, insert a new job into jobs table */ namestrcpy(&application_name, "Retention Policy"); NameData proc_name, proc_schema, check_schema, check_name; namestrcpy(&proc_name, POLICY_RETENTION_PROC_NAME); namestrcpy(&proc_schema, FUNCTIONS_SCHEMA_NAME); namestrcpy(&check_name, POLICY_RETENTION_CHECK_NAME); namestrcpy(&check_schema, FUNCTIONS_SCHEMA_NAME); job_id = ts_bgw_job_insert_relation(&application_name, &default_schedule_interval, &default_max_runtime, default_max_retries, &default_retry_period, &proc_schema, &proc_name, &check_schema, &check_name, owner_id, true, fixed_schedule, hypertable->fd.id, config, initial_start, timezone); ts_cache_release(&hcache); PG_RETURN_INT32(job_id); } Datum policy_retention_add(PG_FUNCTION_ARGS) { /* behave like a strict function */ if (PG_ARGISNULL(0) || PG_ARGISNULL(2)) PG_RETURN_NULL(); Oid ht_oid = PG_GETARG_OID(0); Datum window_datum = PG_GETARG_DATUM(1); bool if_not_exists = PG_GETARG_BOOL(2); Oid window_type = PG_ARGISNULL(1) ? InvalidOid : get_fn_expr_argtype(fcinfo->flinfo, 1); Interval default_schedule_interval = PG_ARGISNULL(3) ? (Interval) DEFAULT_RETENTION_SCHEDULE_INTERVAL : *PG_GETARG_INTERVAL_P(3); TimestampTz initial_start = PG_ARGISNULL(4) ? DT_NOBEGIN : PG_GETARG_TIMESTAMPTZ(4); bool fixed_schedule = !PG_ARGISNULL(4); text *timezone = PG_ARGISNULL(5) ? NULL : PG_GETARG_TEXT_PP(5); char *valid_timezone = NULL; // Interval *created_before = PG_ARGISNULL(6) ? NULL: PG_GETARG_INTERVAL_P(6); Interval *created_before = PG_GETARG_INTERVAL_P(6); ts_feature_flag_check(FEATURE_POLICY); TS_PREVENT_FUNC_IF_READ_ONLY(); Datum retval; /* drop_after and created_before cannot be specified [or omitted] together */ if ((PG_ARGISNULL(1) && PG_ARGISNULL(6)) || (!PG_ARGISNULL(1) && !PG_ARGISNULL(6))) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("need to specify one of \"drop_after\" or \"drop_created_before\""))); /* if users pass in -infinity for initial_start, then use the current_timestamp instead */ if (fixed_schedule) { ts_bgw_job_validate_schedule_interval(&default_schedule_interval); if (TIMESTAMP_NOT_FINITE(initial_start)) initial_start = ts_timer_get_current_timestamp(); } if (timezone != NULL) valid_timezone = ts_bgw_job_validate_timezone(PG_GETARG_DATUM(5)); retval = policy_retention_add_internal(ht_oid, window_type, window_datum, created_before, default_schedule_interval, if_not_exists, fixed_schedule, initial_start, valid_timezone); if (!TIMESTAMP_NOT_FINITE(initial_start)) { int32 job_id = DatumGetInt32(retval); ts_bgw_job_stat_upsert_next_start(job_id, initial_start); } return retval; } Datum policy_retention_remove_internal(Oid table_oid, bool if_exists) { Cache *hcache; Hypertable *hypertable; hypertable = ts_hypertable_cache_get_cache_and_entry(table_oid, CACHE_FLAG_MISSING_OK, &hcache); if (!hypertable) { const char *view_name = get_rel_name(table_oid); if (!view_name) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("relation is not a hypertable or continuous aggregate"))); else { ContinuousAgg *ca = ts_continuous_agg_find_by_relid(table_oid); if (!ca) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("relation \"%s\" is not a hypertable or continuous aggregate", view_name))); hypertable = ts_hypertable_get_by_id(ca->data.mat_hypertable_id); } } Assert(hypertable != NULL); int32 ht_id = hypertable->fd.id; ts_cache_release(&hcache); ts_hypertable_permissions_check(table_oid, GetUserId()); List *jobs = ts_bgw_job_find_by_proc_and_hypertable_id(POLICY_RETENTION_PROC_NAME, FUNCTIONS_SCHEMA_NAME, ht_id); if (jobs == NIL) { if (!if_exists) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("retention policy not found for hypertable \"%s\"", get_rel_name(table_oid)))); else { ereport(NOTICE, (errmsg("retention policy not found for hypertable \"%s\", skipping", get_rel_name(table_oid)))); PG_RETURN_BOOL(false); } } Assert(list_length(jobs) == 1); BgwJob *job = linitial(jobs); ts_bgw_job_delete_by_id(job->fd.id); PG_RETURN_BOOL(true); } Datum policy_retention_remove(PG_FUNCTION_ARGS) { Oid table_oid = PG_GETARG_OID(0); bool if_exists = PG_GETARG_BOOL(1); ts_feature_flag_check(FEATURE_POLICY); TS_PREVENT_FUNC_IF_READ_ONLY(); return policy_retention_remove_internal(table_oid, if_exists); } ================================================ FILE: tsl/src/bgw_policy/retention_api.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> /* User-facing API functions */ extern Datum policy_retention_add(PG_FUNCTION_ARGS); extern Datum policy_retention_proc(PG_FUNCTION_ARGS); extern Datum policy_retention_check(PG_FUNCTION_ARGS); extern Datum policy_retention_remove(PG_FUNCTION_ARGS); int64 policy_retention_get_drop_after_int(const Jsonb *config); Interval *policy_retention_get_drop_after_interval(const Jsonb *config); Interval *policy_retention_get_drop_created_before_interval(const Jsonb *config); Datum policy_retention_add_internal(Oid ht_oid, Oid window_type, Datum window_datum, Interval *created_before, Interval default_schedule_interval, bool if_not_exists, bool fixed_schedule, TimestampTz initial_start, const char *timezone); Datum policy_retention_remove_internal(Oid table_oid, bool if_exists); ================================================ FILE: tsl/src/build-defs.cmake ================================================ # Hide symbols by default in shared libraries set(CMAKE_C_VISIBILITY_PRESET "hidden") if(UNIX) set(CMAKE_C_STANDARD 11) set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -L${PG_LIBDIR}") set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -L${PG_LIBDIR}") set(CMAKE_C_FLAGS "${PG_CFLAGS} ${PG_CPPFLAGS} ${CMAKE_C_FLAGS}") set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -g") endif() if(APPLE) if((${PG_VERSION_MAJOR} GREATER_EQUAL "16")) set(CMAKE_SHARED_MODULE_SUFFIX ".dylib") endif() set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -multiply_defined suppress") set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -multiply_defined suppress -Wl,-undefined,dynamic_lookup -bundle_loader ${PG_BINDIR}/postgres" ) elseif(WIN32) set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} /MANIFEST:NO") set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} /MANIFEST:NO") endif() # PG_LDFLAGS can have strange values if not found, so we just add the flags if # they are defined. if(PG_LDFLAGS) set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${PG_LDFLAGS}") set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${PG_LDFLAGS}") endif() include_directories(${PROJECT_SOURCE_DIR}/src ${PROJECT_SOURCE_DIR}/tsl/src ${PROJECT_BINARY_DIR}/src ${PROJECT_BINARY_DIR}/tsl/src) include_directories(SYSTEM ${PG_INCLUDEDIR_SERVER}) # Only Windows and FreeBSD need the base include/ dir instead of # include/server/, and including both causes problems on Ubuntu where they # frequently get out of sync if(WIN32 OR (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD")) include_directories(SYSTEM ${PG_INCLUDEDIR}) endif() if(WIN32) link_directories(${PROJECT_BINARY_DIR}/src) set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} ${PG_LIBDIR}/postgres.lib ws2_32.lib Version.lib ${PROJECT_NAME}-${PROJECT_VERSION_MOD}.lib" ) set(CMAKE_C_FLAGS "-D_CRT_SECURE_NO_WARNINGS") include_directories(SYSTEM ${PG_INCLUDEDIR_SERVER}/port/win32) if(MSVC) include_directories(SYSTEM ${PG_INCLUDEDIR_SERVER}/port/win32_msvc) endif(MSVC) endif(WIN32) # Name of library with test-specific code set(TSL_TESTS_LIB_NAME ${PROJECT_NAME}-tsl-tests) ================================================ FILE: tsl/src/chunk.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/attnum.h> #include <access/table.h> #include <catalog/catalog.h> #include <catalog/heap.h> #include <catalog/namespace.h> #include <catalog/pg_class.h> #include <catalog/pg_foreign_server.h> #include <catalog/pg_foreign_table.h> #include <fmgr.h> #include <foreign/foreign.h> #include <funcapi.h> #include <miscadmin.h> #include <nodes/lockoptions.h> #include <nodes/makefuncs.h> #include <nodes/nodes.h> #include <nodes/value.h> #include <parser/parse_coerce.h> #include <parser/parse_func.h> #include <utils/acl.h> #include <utils/builtins.h> #include <utils/elog.h> #include <utils/lsyscache.h> #include <utils/memutils.h> #include <utils/palloc.h> #include <utils/rel.h> #include <utils/syscache.h> #include "chunk.h" #include "compression/compression.h" #include "debug_point.h" #include "extension.h" #include "hypertable.h" #include "utils.h" /* Data in a frozen chunk cannot be modified. So any operation * that rewrites data for a frozen chunk will be blocked. * Note that a frozen chunk can still be dropped. */ Datum chunk_freeze_chunk(PG_FUNCTION_ARGS) { Oid chunk_relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); TS_PREVENT_FUNC_IF_READ_ONLY(); Chunk *chunk = ts_chunk_get_by_relid(chunk_relid, true); Assert(chunk != NULL); if (chunk->relkind == RELKIND_FOREIGN_TABLE) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("operation not supported on tiered chunk \"%s\"", get_rel_name(chunk_relid)))); } if (ts_chunk_is_frozen(chunk)) PG_RETURN_BOOL(true); /* get Share lock. will wait for other concurrent transactions that are * modifying the chunk. Does not block SELECTs on the chunk. * Does not block other DDL on the chunk table. */ DEBUG_WAITPOINT("freeze_chunk_before_lock"); LockRelationOid(chunk_relid, ShareLock); bool ret = ts_chunk_set_frozen(chunk); PG_RETURN_BOOL(ret); } Datum chunk_unfreeze_chunk(PG_FUNCTION_ARGS) { Oid chunk_relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); TS_PREVENT_FUNC_IF_READ_ONLY(); Chunk *chunk = ts_chunk_get_by_relid(chunk_relid, true); Assert(chunk != NULL); if (chunk->relkind == RELKIND_FOREIGN_TABLE) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("operation not supported on foreign table \"%s\"", get_rel_name(chunk_relid)))); } if (!ts_chunk_is_frozen(chunk)) PG_RETURN_BOOL(true); /* This is a previously frozen chunk. Only selects are permitted on this chunk. * This changes the status in the catalog to allow previously blocked operations. */ bool ret = ts_chunk_unset_frozen(chunk); PG_RETURN_BOOL(ret); } /* * Invoke drop_chunks via fmgr so that the call can be deparsed and sent to * remote data nodes. * * Given that drop_chunks is an SRF, and has pseudo parameter types, we need * to provide a FuncExpr with type information for the deparser. * * Returns the number of dropped chunks. */ int chunk_invoke_drop_chunks(Oid relid, Datum older_than, Datum older_than_type, bool use_creation_time) { EState *estate; ExprContext *econtext; FuncExpr *fexpr; List *args = NIL; int num_results = 0; SetExprState *state; Oid restype; Oid func_oid; Const *TypeNullCons = makeNullConst(older_than_type, -1, InvalidOid); Const *IntervalVal = makeConst(older_than_type, -1, InvalidOid, get_typlen(older_than_type), older_than, false, get_typbyval(older_than_type)); Const *argarr[DROP_CHUNKS_NARGS] = { makeConst(REGCLASSOID, -1, InvalidOid, sizeof(relid), ObjectIdGetDatum(relid), false, false), TypeNullCons, TypeNullCons, castNode(Const, makeBoolConst(false, true)), TypeNullCons, TypeNullCons }; Oid type_id[DROP_CHUNKS_NARGS] = { REGCLASSOID, ANYOID, ANYOID, BOOLOID, ANYOID, ANYOID }; char *const schema_name = ts_extension_schema_name(); List *const fqn = list_make2(makeString(schema_name), makeString(DROP_CHUNKS_FUNCNAME)); StaticAssertStmt(lengthof(type_id) == lengthof(argarr), "argarr and type_id should have matching lengths"); func_oid = LookupFuncName(fqn, lengthof(type_id), type_id, false); Assert(func_oid); /* LookupFuncName should not return an invalid OID */ /* decide whether to use "older_than" or "drop_created_before" */ if (use_creation_time) argarr[4] = IntervalVal; else argarr[1] = IntervalVal; /* Prepare the function expr with argument list */ get_func_result_type(func_oid, &restype, NULL); for (size_t i = 0; i < lengthof(argarr); i++) args = lappend(args, argarr[i]); fexpr = makeFuncExpr(func_oid, restype, args, InvalidOid, InvalidOid, COERCE_EXPLICIT_CALL); fexpr->funcretset = true; /* Execute the SRF */ estate = CreateExecutorState(); econtext = CreateExprContext(estate); state = ExecInitFunctionResultSet(&fexpr->xpr, econtext, NULL); while (true) { ExprDoneCond isdone; bool isnull; ExecMakeFunctionResultSet(state, econtext, estate->es_query_cxt, &isnull, &isdone); if (isdone == ExprEndResult) break; if (!isnull) num_results++; } /* Cleanup */ FreeExprContext(econtext, false); FreeExecutorState(estate); return num_results; } ================================================ FILE: tsl/src/chunk.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <chunk.h> #include <fmgr.h> extern Datum chunk_freeze_chunk(PG_FUNCTION_ARGS); extern Datum chunk_unfreeze_chunk(PG_FUNCTION_ARGS); extern int chunk_invoke_drop_chunks(Oid relid, Datum older_than, Datum older_than_type, bool use_creation_time); extern Datum chunk_merge_chunks(PG_FUNCTION_ARGS); extern Datum chunk_split_chunk(PG_FUNCTION_ARGS); extern void update_relstats(Relation catrel, Oid relid, BlockNumber num_pages, double ntuples); extern void compute_rel_vacuum_cutoffs(Relation rel, struct VacuumCutoffs *cutoffs); extern void chunk_update_constraints(const Chunk *chunk, const Hypercube *new_cube); ================================================ FILE: tsl/src/chunk_api.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/htup.h> #include <access/htup_details.h> #include <access/multixact.h> #include <access/visibilitymap.h> #include <access/xact.h> #include <catalog/indexing.h> #include <catalog/pg_class.h> #include <catalog/pg_inherits.h> #include <catalog/pg_namespace.h> #include <catalog/pg_operator.h> #include <catalog/pg_type.h> #include <commands/vacuum.h> #include <fmgr.h> #include <funcapi.h> #include <miscadmin.h> #include <nodes/makefuncs.h> #include <storage/lmgr.h> #include <storage/lockdefs.h> #include <utils/array.h> #include <utils/builtins.h> #include <utils/jsonb.h> #include <utils/lsyscache.h> #include <utils/palloc.h> #include <utils/syscache.h> #include "compat/compat.h" #include "chunk.h" #include "chunk_api.h" #include "debug_point.h" #include "error_utils.h" #include "errors.h" #include "hypercube.h" #include "hypertable_cache.h" #include "ts_catalog/array_utils.h" #include "ts_catalog/catalog.h" #include "utils.h" /* * Convert a hypercube to a JSONB value. * * For instance, a two-dimensional hypercube, with dimensions "time" and * "device", might look as follows: * * {"time": [1514419200000000, 1515024000000000], * "device": [-9223372036854775808, 1073741823]} */ static JsonbValue * hypercube_to_jsonb_value(Hypercube *hc, Hyperspace *hs, JsonbParseState **ps) { int i; Assert(hs->num_dimensions == hc->num_slices); pushJsonbValue(ps, WJB_BEGIN_OBJECT, NULL); for (i = 0; i < hc->num_slices; i++) { JsonbValue k, v; char *dim_name = NameStr(hs->dimensions[i].fd.column_name); Datum range_start = DirectFunctionCall1(int8_numeric, Int64GetDatum(hc->slices[i]->fd.range_start)); Datum range_end = DirectFunctionCall1(int8_numeric, Int64GetDatum(hc->slices[i]->fd.range_end)); Assert(hs->dimensions[i].fd.id == hc->slices[i]->fd.dimension_id); k.type = jbvString; k.val.string.len = strlen(dim_name); k.val.string.val = dim_name; pushJsonbValue(ps, WJB_KEY, &k); pushJsonbValue(ps, WJB_BEGIN_ARRAY, NULL); v.type = jbvNumeric; v.val.numeric = DatumGetNumeric(range_start); pushJsonbValue(ps, WJB_ELEM, &v); v.val.numeric = DatumGetNumeric(range_end); pushJsonbValue(ps, WJB_ELEM, &v); pushJsonbValue(ps, WJB_END_ARRAY, NULL); } return pushJsonbValue(ps, WJB_END_OBJECT, NULL); } /* * Create a hypercube from a JSONB object. * * Takes a JSONB object with a hypercube's dimensional constraints and outputs * a Hypercube. The JSONB is the same format as output by * hypercube_to_jsonb_value() above, i.e.: * * {"time": [1514419200000000, 1515024000000000], * "device": [-9223372036854775808, 1073741823]} */ static Hypercube * hypercube_from_jsonb(Jsonb *json, const Hyperspace *hs, const char **parse_error) { JsonbIterator *it; JsonbIteratorToken type; JsonbValue v; Hypercube *hc = NULL; const char *err = NULL; it = JsonbIteratorInit(&json->root); type = JsonbIteratorNext(&it, &v, false); if (type != WJB_BEGIN_OBJECT) { err = "invalid JSON format"; goto out_err; } if (v.val.object.nPairs != hs->num_dimensions) { err = "invalid number of hypercube dimensions"; goto out_err; } hc = ts_hypercube_alloc(hs->num_dimensions); while ((type = JsonbIteratorNext(&it, &v, false))) { int i; const Dimension *dim; int64 range[2]; const char *name; if (type == WJB_END_OBJECT) break; if (type != WJB_KEY) { err = "invalid JSON format"; goto out_err; } name = pnstrdup(v.val.string.val, v.val.string.len); dim = ts_hyperspace_get_dimension_by_name(hs, DIMENSION_TYPE_ANY, name); if (NULL == dim) { err = psprintf("dimension \"%s\" does not exist in hypertable", name); goto out_err; } type = JsonbIteratorNext(&it, &v, false); if (type != WJB_BEGIN_ARRAY) { err = "invalid JSON format"; goto out_err; } if (v.val.array.nElems != 2) { err = psprintf("unexpected number of dimensional bounds for dimension \"%s\"", name); goto out_err; } for (i = 0; i < 2; i++) { type = JsonbIteratorNext(&it, &v, false); if (type != WJB_ELEM) { err = "invalid JSON format"; goto out_err; } if (v.type == jbvString) { if (!IS_TIMESTAMP_TYPE(dim->fd.column_type)) { err = psprintf("constraint for dimension \"%s\" can be string only for date time", name); goto out_err; } char *v_str = (char *) palloc(v.val.string.len + 1); memcpy(v_str, v.val.string.val, v.val.string.len); v_str[v.val.string.len] = '\0'; range[i] = ts_time_value_from_arg(CStringGetDatum(v_str), InvalidOid, dim->fd.column_type, true); } else if (v.type == jbvNumeric) { range[i] = DatumGetInt64( DirectFunctionCall1(numeric_int8, NumericGetDatum(v.val.numeric))); } else { err = psprintf("constraint for dimension \"%s\" should be either numeric or string", name); goto out_err; } } type = JsonbIteratorNext(&it, &v, false); if (type != WJB_END_ARRAY) { err = "invalid JSON format"; goto out_err; } ts_hypercube_add_slice_from_range(hc, dim->fd.id, range[0], range[1]); } out_err: if (NULL != parse_error) *parse_error = err; if (NULL != err) return NULL; return hc; } enum Anum_create_chunk { Anum_create_chunk_id = 1, Anum_create_chunk_hypertable_id, Anum_create_chunk_schema_name, Anum_create_chunk_table_name, Anum_create_chunk_relkind, Anum_create_chunk_slices, Anum_create_chunk_created, _Anum_create_chunk_max, }; #define Natts_create_chunk (_Anum_create_chunk_max - 1) static HeapTuple chunk_form_tuple(Chunk *chunk, Hypertable *ht, TupleDesc tupdesc, bool created) { Datum values[Natts_create_chunk]; bool nulls[Natts_create_chunk] = { false }; JsonbParseState *ps = NULL; JsonbValue *jv = hypercube_to_jsonb_value(chunk->cube, ht->space, &ps); if (NULL == jv) return NULL; values[AttrNumberGetAttrOffset(Anum_create_chunk_id)] = Int32GetDatum(chunk->fd.id); values[AttrNumberGetAttrOffset(Anum_create_chunk_hypertable_id)] = Int32GetDatum(chunk->fd.hypertable_id); values[AttrNumberGetAttrOffset(Anum_create_chunk_schema_name)] = NameGetDatum(&chunk->fd.schema_name); values[AttrNumberGetAttrOffset(Anum_create_chunk_table_name)] = NameGetDatum(&chunk->fd.table_name); values[AttrNumberGetAttrOffset(Anum_create_chunk_relkind)] = CharGetDatum(chunk->relkind); values[AttrNumberGetAttrOffset(Anum_create_chunk_slices)] = JsonbPGetDatum(JsonbValueToJsonb(jv)); values[AttrNumberGetAttrOffset(Anum_create_chunk_created)] = BoolGetDatum(created); return heap_form_tuple(tupdesc, values, nulls); } Datum chunk_show(PG_FUNCTION_ARGS) { Oid chunk_relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); Chunk *chunk = ts_chunk_get_by_relid(chunk_relid, true); Cache *hcache = ts_hypertable_cache_pin(); Hypertable *ht = ts_hypertable_cache_get_entry(hcache, chunk->hypertable_relid, CACHE_FLAG_NONE); TupleDesc tupdesc; HeapTuple tuple; Assert(NULL != chunk); Assert(NULL != ht); if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context " "that cannot accept type record"))); /* * We use the create_chunk tuple for show_chunk, because they only differ * in the created column at the end. That column will not be included here * since it is not part of the tuple descriptor. */ tuple = chunk_form_tuple(chunk, ht, tupdesc, false); ts_cache_release(&hcache); if (NULL == tuple) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not create tuple from chunk"))); PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); } static void check_privileges_for_creating_chunk(Oid hyper_relid) { AclResult acl_result; acl_result = pg_class_aclcheck(hyper_relid, GetUserId(), ACL_INSERT); if (acl_result != ACLCHECK_OK) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permission denied for table \"%s\"", get_rel_name(hyper_relid)), errdetail("Insert privileges required on \"%s\" to create chunks.", get_rel_name(hyper_relid)))); } static Hypercube * get_hypercube_from_slices(Jsonb *slices, const Hypertable *ht) { Hypercube *hc; const char *parse_err; hc = hypercube_from_jsonb(slices, ht->space, &parse_err); if (hc == NULL) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid hypercube for hypertable \"%s\"", get_rel_name(ht->main_table_relid)), errdetail("%s", parse_err))); return hc; } /* * Create a chunk and its metadata. * * This function will create a chunk, either from an existing table or by * creating a new table. If chunk_table_relid is InvalidOid, the chunk table * will be created, otherwise the table referenced by the relid will be * used. The chunk will be associated with the hypertable given by * hypertable_relid. */ Datum chunk_create(PG_FUNCTION_ARGS) { Oid hypertable_relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); Jsonb *slices = PG_ARGISNULL(1) ? NULL : PG_GETARG_JSONB_P(1); const char *schema_name = PG_ARGISNULL(2) ? NULL : PG_GETARG_CSTRING(2); const char *table_name = PG_ARGISNULL(3) ? NULL : PG_GETARG_CSTRING(3); Oid chunk_table_relid = PG_ARGISNULL(4) ? InvalidOid : PG_GETARG_OID(4); Cache *hcache = ts_hypertable_cache_pin(); Hypertable *ht = ts_hypertable_cache_get_entry(hcache, hypertable_relid, CACHE_FLAG_NONE); Hypercube *hc; Chunk *chunk; TupleDesc tupdesc; HeapTuple tuple; bool created; Assert(NULL != ht); Assert(OidIsValid(ht->main_table_relid)); check_privileges_for_creating_chunk(hypertable_relid); if (NULL == slices) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid slices"))); if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context " "that cannot accept type record"))); hc = get_hypercube_from_slices(slices, ht); Assert(NULL != hc); chunk = ts_chunk_find_or_create_without_cuts(ht, hc, schema_name, table_name, chunk_table_relid, &created); Assert(NULL != chunk); tuple = chunk_form_tuple(chunk, ht, tupdesc, created); ts_cache_release(&hcache); if (NULL == tuple) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not create tuple from chunk"))); PG_RETURN_DATUM(HeapTupleGetDatum(tuple)); } /* * Detach a chunk from a hypertable. */ Datum chunk_detach(PG_FUNCTION_ARGS) { Oid chunk_relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); Cache *hcache; Hypertable *ht; Chunk *chunk; Oid ht_rel; TS_PREVENT_FUNC_IF_READ_ONLY(); if (!OidIsValid(chunk_relid)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid chunk relation OID"))); DEBUG_WAITPOINT("chunk_detach_before_lock"); ht_rel = ts_hypertable_id_to_relid(ts_chunk_get_hypertable_id_by_reloid(chunk_relid), true); if (!OidIsValid(ht_rel)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("hypertable not found for the chunk"))); /* Take the same locks taken by PostgreSQL partitioning to be consistent */ LockRelationOid(ht_rel, ShareUpdateExclusiveLock); LockRelationOid(chunk_relid, AccessExclusiveLock); chunk = ts_chunk_get_by_relid(chunk_relid, true); Assert(chunk != NULL); ht = ts_hypertable_cache_get_cache_and_entry(chunk->hypertable_relid, CACHE_FLAG_NONE, &hcache); Assert(ht != NULL); if (!object_ownercheck(RelationRelationId, ht->main_table_relid, GetUserId())) aclcheck_error(ACLCHECK_NOT_OWNER, get_relkind_objtype(get_rel_relkind(ht->main_table_relid)), get_rel_name(ht->main_table_relid)); if (ts_chunk_is_compressed(chunk)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot detach compressed chunk \"%s\"", get_rel_name(chunk_relid)), errhint("Decompress the chunk first."))); if (chunk->fd.osm_chunk) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot detach OSM chunk \"%s\"", get_rel_name(chunk_relid)))); AlterTableCmd cmd = { .type = T_AlterTableCmd, .subtype = AT_DropInherit, .def = (Node *) makeRangeVar(NameStr(ht->fd.schema_name), NameStr(ht->fd.table_name), 0), }; AlterTableStmt stmt = { .type = T_AlterTableStmt, .cmds = list_make1(&cmd), .relation = makeRangeVar(NameStr(ht->fd.schema_name), NameStr(ht->fd.table_name), 0), }; ts_alter_table_with_event_trigger(chunk->table_id, (Node *) &stmt, list_make1(&cmd), false); ts_chunk_detach_by_relid(chunk_relid); ts_cache_release(&hcache); PG_RETURN_VOID(); } /* * Attach an existing relation to a hypertable as a chunk. */ Datum chunk_attach(PG_FUNCTION_ARGS) { Oid ht_relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); Oid chunk_relid = PG_ARGISNULL(1) ? InvalidOid : PG_GETARG_OID(1); Jsonb *slices = PG_ARGISNULL(2) ? NULL : PG_GETARG_JSONB_P(2); Cache *hcache; Hypertable *ht; Hypercube *hc; Chunk PG_USED_FOR_ASSERTS_ONLY *chunk; bool created; TS_PREVENT_FUNC_IF_READ_ONLY(); if (!OidIsValid(chunk_relid)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid chunk relation OID"))); if (!OidIsValid(ht_relid)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid hypertable relation OID"))); if (chunk_relid == ht_relid) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("chunk relation cannot be the same as hypertable relation"))); if (NULL == slices) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid dimension slices argument"), errhint("Provide a json-formatted definition of dimensional constraints for the " "chunk partition."))); DEBUG_WAITPOINT("chunk_attach_before_lock"); /* Take the same locks taken by PostgreSQL partitioning to be consistent */ LockRelationOid(ht_relid, ShareUpdateExclusiveLock); LockRelationOid(chunk_relid, AccessExclusiveLock); /* Only owner is allowed */ if (!object_ownercheck(RelationRelationId, chunk_relid, GetUserId())) aclcheck_error(ACLCHECK_NOT_OWNER, get_relkind_objtype(get_rel_relkind(chunk_relid)), get_rel_name(chunk_relid)); if (is_inheritance_child(chunk_relid)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot attach chunk that is already a child of another table"))); if (ts_is_hypertable(chunk_relid)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot attach hypertable as a chunk"))); /* Check if the table still exists after taking the lock */ if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(chunk_relid))) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("relation with OID %u does not exist", chunk_relid))); check_privileges_for_creating_chunk(ht_relid); ht = ts_hypertable_cache_get_cache_and_entry(ht_relid, CACHE_FLAG_NONE, &hcache); Assert(ht != NULL); hc = get_hypercube_from_slices(slices, ht); Assert(hc != NULL); chunk = ts_chunk_find_or_create_without_cuts(ht, hc, get_namespace_name(get_rel_namespace(chunk_relid)), get_rel_name(chunk_relid), chunk_relid, &created); Assert(chunk != NULL); ts_cache_release(&hcache); PG_RETURN_VOID(); } ================================================ FILE: tsl/src/chunk_api.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <chunk.h> extern Datum chunk_status(PG_FUNCTION_ARGS); extern Datum chunk_show(PG_FUNCTION_ARGS); extern Datum chunk_create(PG_FUNCTION_ARGS); extern Datum chunk_detach(PG_FUNCTION_ARGS); extern Datum chunk_attach(PG_FUNCTION_ARGS); ================================================ FILE: tsl/src/chunk_merge.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/multixact.h> #include <access/xact.h> #include <catalog/catalog.h> #include <catalog/dependency.h> #include <catalog/heap.h> #include <catalog/objectaddress.h> #include <catalog/pg_am.h> #include <catalog/pg_constraint.h> #include <catalog/pg_trigger_d.h> #include <commands/tablecmds.h> #include <commands/trigger.h> #include <executor/spi.h> #include <nodes/makefuncs.h> #include <nodes/parsenodes.h> #include <port.h> #include <storage/block.h> #include <storage/bufmgr.h> #include <storage/itemptr.h> #include <storage/lmgr.h> #include <storage/lockdefs.h> #include <utils/acl.h> #include <utils/elog.h> #include <utils/guc.h> #include <utils/memutils.h> #include <utils/palloc.h> #include <utils/rel.h> #include <utils/relcache.h> #include <utils/snapmgr.h> #include <utils/syscache.h> #include "chunk.h" #include "chunk_index.h" #include "debug_point.h" #include "hypercube.h" #include "import/heapswap.h" #include "ts_catalog/catalog.h" #include "ts_catalog/chunk_rewrite.h" #include "ts_catalog/compression_chunk_size.h" typedef struct RelationMergeInfo { Oid relid; struct VacuumCutoffs cutoffs; FormData_compression_chunk_size ccs; Chunk *chunk; Relation rel; char relpersistence; bool isresult; bool iscompressed_rel; ItemPointerData chunk_rewrite_tid; List *ind_oids_old; List *ind_oids_new; } RelationMergeInfo; typedef struct RelationMergeStats { Oid relid; int32 chunk_id; FormData_compression_chunk_size ccs; BlockNumber num_pages; double reltuples; } RelationMergeStats; static void update_stats_after_merge(const RelationMergeStats *stats) { /* Update table stats */ Relation relRelation = table_open(RelationRelationId, RowExclusiveLock); update_relstats(relRelation, stats->relid, stats->num_pages, stats->reltuples); table_close(relRelation, RowExclusiveLock); /* * Update compression chunk size stats, but only if this is a * non-compressed chunk and at least one of the merged chunks was * compressed. In that case the merged metadata should be non-zero. */ if (stats->ccs.compressed_heap_size > 0) { /* * The result relation should always be compressed because we pick the * first compressed one, if one exists. */ FormData_compression_chunk_size form; memcpy(&form, &stats->ccs, sizeof(form)); ts_compression_chunk_size_update(stats->chunk_id, &form); } } void compute_rel_vacuum_cutoffs(Relation rel, struct VacuumCutoffs *cutoffs) { VacuumParams params; memset(¶ms, 0, sizeof(VacuumParams)); vacuum_get_cutoffs(rel, ¶ms, cutoffs); /* Frozen Id should not go backwards */ TransactionId relfrozenxid = rel->rd_rel->relfrozenxid; if (TransactionIdIsValid(relfrozenxid) && TransactionIdPrecedes(cutoffs->FreezeLimit, relfrozenxid)) cutoffs->FreezeLimit = relfrozenxid; MultiXactId relminmxid = rel->rd_rel->relminmxid; if (MultiXactIdIsValid(relminmxid) && MultiXactIdPrecedes(cutoffs->MultiXactCutoff, relminmxid)) cutoffs->MultiXactCutoff = relminmxid; } static void merge_chunks_finish(Oid new_relid, RelationMergeInfo *relinfos, int nrelids, const RelationMergeStats *stats) { RelationMergeInfo *result_minfo = NULL; for (int i = 0; i < nrelids; i++) { if (relinfos[i].isresult) { result_minfo = &relinfos[i]; break; } } Ensure(result_minfo != NULL, "no chunk to merge into found"); struct VacuumCutoffs *cutoffs = &result_minfo->cutoffs; bool reindex = result_minfo->ind_oids_new == NIL; if (!reindex) { ListCell *lc, *lc2; forboth (lc, result_minfo->ind_oids_old, lc2, result_minfo->ind_oids_new) { Oid ind_old = lfirst_oid(lc); Oid ind_new = lfirst_oid(lc2); Oid mapped_tables[4]; LockRelationOid(ind_old, AccessExclusiveLock); LockRelationOid(ind_new, AccessExclusiveLock); /* Zero out possible results from swapped_relation_files */ memset(mapped_tables, 0, sizeof(mapped_tables)); ts_swap_relation_files(ind_old, ind_new, false, false, true, InvalidTransactionId, InvalidMultiXactId, mapped_tables); } /* The new indexes must be visible for deletion. */ CommandCounterIncrement(); } ts_finish_heap_swap(result_minfo->relid, new_relid, false, /* system catalog */ false /* swap toast by content */, false, /* check constraints */ true, /* internal? */ reindex, cutoffs->FreezeLimit, cutoffs->MultiXactCutoff, result_minfo->relpersistence); update_stats_after_merge(stats); /* Clear the chunk merge mapping for the result relation. The mappings for * non-result relations are deleted when the corresponding chunk is * dropped. Only done in concurrent mode as indicated by a valid TID. */ if (ItemPointerIsValid(&result_minfo->chunk_rewrite_tid)) ts_chunk_rewrite_delete_by_tid(&result_minfo->chunk_rewrite_tid); /* Don't need to drop objects for internal compressed relations, they are * dropped when the main chunk is dropped. */ if (result_minfo->iscompressed_rel) return; if (ts_chunk_is_compressed(result_minfo->chunk)) ts_chunk_set_partial(result_minfo->chunk); Assert(stats->relid == result_minfo->relid); /* * Delete all the merged relations except the result one, since we are * keeping it for the heap swap. */ ObjectAddresses *objects = new_object_addresses(); DEBUG_WAITPOINT("merge_chunks_before_drop"); for (int i = 0; i < nrelids; i++) { RelationMergeInfo *relinfo = &relinfos[i]; Oid relid = relinfo->relid; ObjectAddress object = { .classId = RelationRelationId, .objectId = relid, }; if (!OidIsValid(relid)) continue; if (!relinfo->isresult) { /* Cannot drop if relation is still open */ Assert(relinfo->rel == NULL); if (relinfo->chunk) { const Oid namespaceid = get_rel_namespace(relid); const char *schemaname = get_namespace_name(namespaceid); const char *tablename = get_rel_name(relid); ts_chunk_delete_by_name(schemaname, tablename, DROP_RESTRICT); } add_exact_object_address(&object, objects); } } performMultipleDeletions(objects, DROP_RESTRICT, PERFORM_DELETION_INTERNAL); free_object_addresses(objects); } static int cmp_relations(const void *left, const void *right) { const RelationMergeInfo *linfo = ((RelationMergeInfo *) left); const RelationMergeInfo *rinfo = ((RelationMergeInfo *) right); if (linfo->chunk && rinfo->chunk) { const Hypercube *lcube = linfo->chunk->cube; const Hypercube *rcube = rinfo->chunk->cube; Assert(lcube->num_slices == rcube->num_slices); for (int i = 0; i < lcube->num_slices; i++) { const DimensionSlice *lslice = lcube->slices[i]; const DimensionSlice *rslice = rcube->slices[i]; Assert(lslice->fd.dimension_id == rslice->fd.dimension_id); /* Compare start of range for the dimension */ if (lslice->fd.range_start < rslice->fd.range_start) return -1; if (lslice->fd.range_start > rslice->fd.range_start) return 1; /* If start of range is equal, compare by end of range */ if (lslice->fd.range_end < rslice->fd.range_end) return -1; if (lslice->fd.range_end > rslice->fd.range_end) return 1; } /* Should only reach here if partitioning is equal across all * dimensions. Fall back to comparing relids. */ } return pg_cmp_u32(linfo->relid, rinfo->relid); } /* * Check that the partition boundaries of two chunks align so that a new valid * hypercube can be formed if the chunks are merged. This check assumes that * the hypercubes are sorted so that cube2 "follows" cube1. * * The algorithm is simple and only allows merging along a single dimension in * the same merge. For example, these two cases are mergeable: * * ' ____ * ' |__| * ' |__| * * ' _______ * ' |__|__| * * while these cases are not mergeable: * ' ____ * ' __|__| * ' |__| * * ' ______ * ' |____| * ' |__| * * * The validation can handle merges of many chunks at once if they are * "naively" aligned and this function is called on chunk hypercubes in * "partition order": * * ' _____________ * ' |__|__|__|__| * * However, the validation currently won't accept merges of multiple * dimensions at once: * * ' _____________ * ' |__|__|__|__| * ' |__|__|__|__| * * It also cannot handle complicated merges of multi-dimensional partitioning * schemes like the one below. * * ' _________ * ' |__a____| * ' |_b_|_c_| * * Merging a,b,c, should be possible but the validation currently cannot * handle such cases. Instead, it is necessary to first merge b,c. Then merge * a with the result (b,c) in a separate merge. Note that it is not possible * to merge only a,b or a,c. * * A future, more advanced, validation needs to handle corner-cases like the * one below that has gaps: * * ' _____________ * ' |__|__|__|__| * ' |____| |___| * ' */ static void validate_merge_possible(const Hypercube *cube1, const Hypercube *cube2) { int follow_edges = 0; int equal_edges = 0; Assert(cube1->num_slices == cube2->num_slices); for (int i = 0; i < cube1->num_slices; i++) { const DimensionSlice *slice1 = cube1->slices[i]; const DimensionSlice *slice2 = cube2->slices[i]; if (ts_dimension_slices_equal(slice1, slice2)) equal_edges++; if (slice1->fd.range_end == slice2->fd.range_start) follow_edges++; } if (follow_edges != 1 || (cube1->num_slices - equal_edges) != 1) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create new chunk partition boundaries"), errhint("Try merging chunks that have adjacent partitions."))); } static const ChunkConstraint * get_chunk_constraint_by_slice_id(const ChunkConstraints *ccs, int32 slice_id) { for (int i = 0; i < ccs->num_constraints; i++) { const ChunkConstraint *cc = &ccs->constraints[i]; if (cc->fd.dimension_slice_id == slice_id) return cc; } return NULL; } void chunk_update_constraints(const Chunk *chunk, const Hypercube *new_cube) { Cache *hcache; const Hypertable *ht = ts_hypertable_cache_get_cache_and_entry(chunk->hypertable_relid, CACHE_FLAG_NONE, &hcache); List *new_constraints = NIL; for (int i = 0; i < new_cube->num_slices; i++) { const DimensionSlice *old_slice = chunk->cube->slices[i]; DimensionSlice *new_slice = new_cube->slices[i]; const ChunkConstraint *cc; ScanTupLock tuplock = { .waitpolicy = LockWaitBlock, .lockmode = LockTupleShare, }; /* If nothing changed in this dimension, move on to the next */ if (ts_dimension_slices_equal(old_slice, new_slice)) continue; cc = get_chunk_constraint_by_slice_id(chunk->constraints, old_slice->fd.id); if (cc) { ObjectAddress constrobj = { .classId = ConstraintRelationId, .objectId = get_relation_constraint_oid(chunk->table_id, NameStr(cc->fd.constraint_name), false), }; performDeletion(&constrobj, DROP_RESTRICT, PERFORM_DELETION_INTERNAL); /* Create the new check constraint */ const Dimension *dim = ts_hyperspace_get_dimension_by_id(ht->space, old_slice->fd.dimension_id); Constraint *constr = ts_chunk_constraint_dimensional_create(dim, new_slice, NameStr(cc->fd.constraint_name)); /* Constraint could be NULL, e.g., if the merged chunk covers the * entire range in a space dimension it needs no constraint. */ if (constr != NULL) new_constraints = lappend(new_constraints, constr); } /* Check if there's already a slice with the new range. If so, avoid * inserting a new slice. */ if (!ts_dimension_slice_scan_for_existing(new_slice, &tuplock)) { new_slice->fd.id = -1; ts_dimension_slice_insert(new_slice); /* A new Id should be assigned */ Assert(new_slice->fd.id > 0); } /* Update the chunk constraint to point to the new slice ID */ ts_chunk_constraint_update_slice_id(chunk->fd.id, old_slice->fd.id, new_slice->fd.id); /* Delete the old slice if it is orphaned now */ if (ts_chunk_constraint_scan_by_dimension_slice_id(old_slice->fd.id, NULL, CurrentMemoryContext) == 0) { ts_dimension_slice_delete_by_id(old_slice->fd.id, false); } } /* Add new check constraints, if any */ if (new_constraints != NIL) { /* Adding a constraint should require AccessExclusivelock. It should * already be taken at this point, but specify it to be sure. */ Relation rel = table_open(chunk->table_id, AccessExclusiveLock); AddRelationNewConstraints(rel, NIL /* List *newColDefaults */, new_constraints, false /* allow_merge */, true /* is_local */, false /* is_internal */, NULL /* query string */); table_close(rel, NoLock); } ts_cache_release(&hcache); } static void merge_cubes(Hypercube *merged_cube, const Hypercube *cube) { /* Merge dimension slices */ for (int i = 0; i < cube->num_slices; i++) { const DimensionSlice *slice = cube->slices[i]; DimensionSlice *merged_slice = merged_cube->slices[i]; Assert(slice->fd.dimension_id == merged_slice->fd.dimension_id); if (slice->fd.range_start < merged_slice->fd.range_start) merged_slice->fd.range_start = slice->fd.range_start; if (slice->fd.range_end > merged_slice->fd.range_end) merged_slice->fd.range_end = slice->fd.range_end; } } /* * Use anonymous settings value to disable multidim merges due to a bug in the * routing cache with non-aligned partitions/chunks. */ static bool merge_chunks_multidim_allowed(void) { const char *multidim_merge_enabled = GetConfigOption(MAKE_EXTOPTION("enable_merge_multidim_chunks"), true, false); if (multidim_merge_enabled == NULL) return false; return (pg_strcasecmp("on", multidim_merge_enabled) == 0 || pg_strcasecmp("1", multidim_merge_enabled) == 0 || pg_strcasecmp("true", multidim_merge_enabled) == 0); } #if (PG_VERSION_NUM >= 170000 && PG_VERSION_NUM <= 170002) /* * Workaround for changed behavior in the relation rewrite code that appeared * in PostgreSQL 17.0, but was fixed in 17.3. * * Merge chunks uses the relation rewrite functionality from CLUSTER and * VACUUM FULL. This works for merge because, when writing into a non-empty * relation, new pages are appended while the existing pages remain the * same. In PG17.0, however, that changed so that existing pages in the * relation were zeroed out. The changed behavior was introduced as part of * this commit: * * https://github.com/postgres/postgres/commit/8af256524893987a3e534c6578dd60edfb782a77 * * Fortunately, this was fixed in a follow up commit: * * https://github.com/postgres/postgres/commit/9695835538c2c8e9cd0048028b8c85e1bbf5c79c * * The fix is part of PG 17.3. Howevever, this still leaves PG 17.0 - 17.2 * with different behavior. * * To make the merge chunks code work for the "broken" versions we make PG * believe the first rewrite operation is the size of the fully merged * relation so that we reserve the full space needed and then "append" * backwards into the zeroed space (see illustration below). By doing this, we * ensure that no valid data is zeroed out. The downside of this approach is * that there will be a lot of unnecessary writing of zero pages. Below is an * example of what the rewrite would look like for merging three relations * with one page each. When writing the first relation, PG believes the merged * relation already contains two pages when starting the rewrite. These two * existing pages will be zeroed. When writing the next relation we tell PG * that there is only one existing page in the merged relation, and so forth. * * _____________ * |_0_|_0_|_x_| * _________ * |_0_|_x_| * _____ * |_x_| * * Result: * _____________ * |_x_|_x_|_x_| * */ static BlockNumber merge_rel_nblocks = 0; static BlockNumber *blockoff = NULL; static const TableAmRoutine *old_routine = NULL; static TableAmRoutine routine = {}; /* * TAM relation size function to make PG believe that the merged relation * contains as specific amount of existing data. */ static uint64 pq17_workaround_merge_relation_size(Relation rel, ForkNumber forkNumber) { uint64 nblocks = merge_rel_nblocks; if (forkNumber == MAIN_FORKNUM) return nblocks * BLCKSZ; return old_routine->relation_size(rel, forkNumber); } static inline void pg17_workaround_init(Relation rel, RelationMergeInfo *relinfos, int nrelids) { routine = *rel->rd_tableam; routine.relation_size = pq17_workaround_merge_relation_size; old_routine = rel->rd_tableam; rel->rd_tableam = &routine; blockoff = palloc(sizeof(BlockNumber) * nrelids); uint64 totalblocks = 0; for (int i = 0; i < nrelids; i++) { blockoff[i] = (BlockNumber) totalblocks; if (relinfos[i].rel) { totalblocks += smgrnblocks(RelationGetSmgr(relinfos[i].rel), MAIN_FORKNUM); /* Ensure the offsets don't overflow. For the merge itself, it is * assumed that the write will fail when writing too many blocks */ Ensure(totalblocks <= MaxBlockNumber, "max number of blocks exceeded for merge"); } } } static inline void pg17_workaround_cleanup(Relation rel) { pfree(blockoff); rel->rd_tableam = old_routine; } static inline RelationMergeInfo * get_relmergeinfo(RelationMergeInfo *relinfos, int nrelids, int i) { RelationMergeInfo *relinfo = &relinfos[nrelids - i - 1]; merge_rel_nblocks = blockoff[nrelids - i - 1]; return relinfo; } #else #define pg17_workaround_init(rel, relinfos, nrelids) #define pg17_workaround_cleanup(rel) #define get_relmergeinfo(relinfos, nrelids, i) &(relinfos)[i] #endif /* Update table stats */ void update_relstats(Relation catrel, Oid relid, BlockNumber num_pages, double ntuples) { HeapTuple reltup = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(relid)); if (!HeapTupleIsValid(reltup)) elog(ERROR, "cache lookup failed for relation %u", relid); Form_pg_class relform = (Form_pg_class) GETSTRUCT(reltup); relform->relpages = num_pages; relform->reltuples = ntuples; CatalogTupleUpdate(catrel, &reltup->t_self, reltup); heap_freetuple(reltup); } static double copy_table_data(Relation fromrel, Relation torel, struct VacuumCutoffs *cutoffs, struct VacuumCutoffs *merged_cutoffs) { double num_tuples = 0.0; double tups_vacuumed = 0.0; double tups_recently_dead = 0.0; table_relation_copy_for_cluster(fromrel, torel, NULL, false, cutoffs->OldestXmin, &cutoffs->FreezeLimit, &cutoffs->MultiXactCutoff, &num_tuples, &tups_vacuumed, &tups_recently_dead); elog(LOG, "merged rows from \"%s\" into \"%s\": tuples %lf vacuumed %lf recently dead %lf", RelationGetRelationName(fromrel), RelationGetRelationName(torel), num_tuples, tups_vacuumed, tups_recently_dead); if (TransactionIdPrecedes(merged_cutoffs->FreezeLimit, cutoffs->FreezeLimit)) merged_cutoffs->FreezeLimit = cutoffs->FreezeLimit; if (MultiXactIdPrecedes(merged_cutoffs->MultiXactCutoff, cutoffs->MultiXactCutoff)) merged_cutoffs->MultiXactCutoff = cutoffs->MultiXactCutoff; return num_tuples; } typedef struct SessionLockInfo { LockRelId locktag; LOCKMODE lockmode; } SessionLockInfo; static List * append_rellock(List *rellocks, Relation rel, LOCKMODE lockmode, MemoryContext mcxt) { MemoryContext oldcontext = MemoryContextSwitchTo(mcxt); SessionLockInfo *lockinfo = palloc_object(SessionLockInfo); lockinfo->locktag = rel->rd_lockInfo.lockRelId; lockinfo->lockmode = lockmode; rellocks = lappend(rellocks, lockinfo); MemoryContextSwitchTo(oldcontext); return rellocks; } static Oid merge_relinfos(RelationMergeInfo *relinfos, int nrelids, int mergeindex, LOCKMODE old_heap_lockmode, List **rellocks, RelationMergeStats *stats, MemoryContext merge_mcxt, bool concurrently) { RelationMergeInfo *result_minfo = &relinfos[mergeindex]; Relation result_rel = result_minfo->rel; MemSet(stats, 0, sizeof(RelationMergeStats)); if (result_rel == NULL) return InvalidOid; stats->relid = result_minfo->relid; stats->chunk_id = result_minfo->chunk->fd.id; Oid tablespace = result_rel->rd_rel->reltablespace; struct VacuumCutoffs *merged_cutoffs = &result_minfo->cutoffs; /* Create the transient heap that will receive the re-ordered data */ Oid new_relid = make_new_heap(RelationGetRelid(result_rel), tablespace, result_rel->rd_rel->relam, result_minfo->relpersistence, old_heap_lockmode); Relation new_rel = table_open(new_relid, AccessExclusiveLock); *rellocks = append_rellock(*rellocks, new_rel, AccessExclusiveLock, merge_mcxt); pg17_workaround_init(new_rel, relinfos, nrelids); /* Step 3: write the data from all the rels into a new merged heap */ for (int i = 0; i < nrelids; i++) { RelationMergeInfo *relinfo = get_relmergeinfo(relinfos, nrelids, i); struct VacuumCutoffs *cutoffs_i = &relinfo->cutoffs; double num_tuples = 0.0; if (relinfo->rel) { num_tuples = copy_table_data(relinfo->rel, new_rel, cutoffs_i, merged_cutoffs); stats->reltuples += num_tuples; if (concurrently) { /* * Mark this chunk as being rewritten by adding an entry in the * chunk_rewrite catalog. This allows cleaning up replacement * heaps in case the second transaction fails. * * This should not fail because any conflicting entries were * removed at the start of the merge and a lock on the relation * protects against adding new conflicting entries. However, if * a conflicting entry exists for some reason, adding a new * entry will fail with a unique constraint violation. */ ts_chunk_rewrite_add(relinfo->relid, new_relid); } } /* * Merge compression chunk size stats. * * Simply sum up the stats for all compressed relations that are * merged. Note that we don't add anything for non-compressed * relations that are merged because they don't have stats. This is a * bit weird because the data from uncompressed relations will not be * reflected in the stats of the merged chunk although the data is * part of the chunk. */ stats->ccs.compressed_heap_size += relinfo->ccs.compressed_heap_size; stats->ccs.compressed_toast_size += relinfo->ccs.compressed_toast_size; stats->ccs.compressed_index_size += relinfo->ccs.compressed_index_size; stats->ccs.uncompressed_heap_size += relinfo->ccs.uncompressed_heap_size; stats->ccs.uncompressed_toast_size += relinfo->ccs.uncompressed_toast_size; stats->ccs.uncompressed_index_size += relinfo->ccs.uncompressed_index_size; stats->ccs.numrows_post_compression += relinfo->ccs.numrows_post_compression; stats->ccs.numrows_pre_compression += relinfo->ccs.numrows_pre_compression; stats->ccs.numrows_frozen_immediately += relinfo->ccs.numrows_frozen_immediately; } /* * Rebuild indexes on new heap (if in concurrent mode). In non-concurrent * mode, indexes are rebuilt as part of the heap swap (this is how PG * normally does it). */ if (concurrently) { /* Create versions of the tables indexes for the new table */ result_minfo->ind_oids_new = ts_chunk_index_duplicate(result_rel->rd_id, new_rel->rd_id, &result_minfo->ind_oids_old, InvalidOid); } stats->num_pages = RelationGetNumberOfBlocks(new_rel); pg17_workaround_cleanup(new_rel); /* Now close all relations */ for (int i = 0; i < nrelids; i++) { RelationMergeInfo *relinfo = get_relmergeinfo(relinfos, nrelids, i); /* * Close the relations before the heap swap, but keep the locks until * end of transaction. Note that some relations might be NULL because * not all chunks are compressed. We still maintain a NULL entry in * the the array for the compressed chunk. */ if (relinfo->rel) { table_close(relinfo->rel, NoLock); relinfo->rel = NULL; } } table_close(new_rel, NoLock); return new_relid; } /* * Relock a relation being concurrently merged. * * The locking happens in the second transaction before swapping * the merged heaps. */ static void relock_rel(const Relation hyper_rel, RelationMergeInfo *rmi, LOCKMODE lockmode) { rmi->rel = try_table_open(rmi->relid, lockmode); if (NULL == rmi->rel) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("chunk \"%s\" was removed concurrently", NameStr(rmi->chunk->fd.table_name)))); /* Re-lock toast tables, heap swap expects it */ if (OidIsValid(rmi->rel->rd_rel->reltoastrelid)) LockRelationOid(rmi->rel->rd_rel->reltoastrelid, lockmode); /* Get a lock on the rewrite entry for this merge */ if (!ts_chunk_rewrite_get_with_lock(rmi->relid, NULL, &rmi->chunk_rewrite_tid)) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("chunk rewrite entry for \"%s\" does not exist", RelationGetRelationName(rmi->rel)))); } /* Now that we know the relations still exist, they can be * closed. We just need the locks. */ table_close(rmi->rel, NoLock); rmi->rel = NULL; } static void lock_merged_rels(Oid hyper_relid, RelationMergeInfo *relinfos, RelationMergeInfo *crelinfos, int nrelids, LOCKMODE lockmode) { Relation hyper_rel = try_relation_open(hyper_relid, ShareUpdateExclusiveLock); if (NULL == hyper_rel) { const RelationMergeInfo *rmi = &relinfos[0]; if (NULL != rmi->rel) elog(WARNING, "dangling chunk \"%s\" remains, can't fix", RelationGetRelationName(rmi->rel)); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("hypertable was removed concurrently"))); } for (int i = 0; i < nrelids; i++) { relock_rel(hyper_rel, &relinfos[i], lockmode); if (OidIsValid(crelinfos[i].relid)) relock_rel(hyper_rel, &crelinfos[i], lockmode); } table_close(hyper_rel, NoLock); } /* * Relock the new relation heaps. * * This lock is taken in the second transaction before these heaps * are swapped with the old heaps. */ static void relock_new_rels(Oid new_relid, Oid new_crelid, LOCKMODE lockmode) { /* * Re-lock the new heaps, including any toast tables. */ Relation new_rel = try_table_open(new_relid, lockmode); if (NULL == new_rel) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("new heap was removed concurrently during chunk merge"))); if (OidIsValid(new_rel->rd_rel->reltoastrelid)) { Relation new_toast_rel = table_open(new_rel->rd_rel->reltoastrelid, lockmode); List *indexes = RelationGetIndexList(new_toast_rel); ListCell *lc; foreach (lc, indexes) { Oid indexrelid = lfirst_oid(lc); LockRelationOid(indexrelid, lockmode); } table_close(new_toast_rel, NoLock); } table_close(new_rel, NoLock); if (OidIsValid(new_crelid)) { Relation new_crel = try_table_open(new_crelid, lockmode); if (NULL == new_crel) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("new compressed heap was removed concurrently during chunk merge"))); if (OidIsValid(new_crel->rd_rel->reltoastrelid)) { Relation new_toast_crel = table_open(new_crel->rd_rel->reltoastrelid, lockmode); List *indexes = RelationGetIndexList(new_toast_crel); ListCell *lc; foreach (lc, indexes) { Oid indexrelid = lfirst_oid(lc); LockRelationOid(indexrelid, lockmode); } table_close(new_toast_crel, NoLock); } table_close(new_crel, NoLock); } } /* * Merge N chunk relations into one chunk based on Oids. * * The input chunk relations are ordered according to partition ranges and the * "first" relation in that ordered list will be "kept" to hold the merged * data. The merged chunk will have its partition ranges updated to cover the * ranges of all the merged chunks. * * The merge happens via a heap rewrite, followed by a heap swap, essentially * the same approach implemented by CLUSTER and VACUUM FULL, but applied on * several relations in the same operation (many to one). * * * The heap swap approach handles visibility across all PG isolation levels, * as implemented by the cluster code. * * In the first step, all data from each chunk is written to a temporary heap * (accounting for vacuum, half-dead/visible, and frozen tuples). In the * second step, a heap swap is performed on one of the chunks and all metadata * is rewritten to handle, e.g., new partition ranges. Finally, the old chunks * are dropped, except for the chunk that received the heap swap. * * To be able to merge, the function checks that: * * - all relations are tables (not, e.g,, views) * - all relations use same (or compatible) storage on disk * - all relations are chunks (and not, e.g., foreign/OSM chunks) * * Compressed chunks can be merged, and in that case the non-compressed chunk * and the (internal) compressed chunk are merged in separate * steps. Currently, the merge does not move and recompress data across the * two relations so whatever data was compressed or not compressed prior to * the merge will remain in the same state after the merge. */ Datum chunk_merge_chunks(PG_FUNCTION_ARGS) { ArrayType *chunks_array = PG_ARGISNULL(0) ? NULL : PG_GETARG_ARRAYTYPE_P(0); bool concurrently = (PG_NARGS() > 1 && !PG_ARGISNULL(1)) ? PG_GETARG_BOOL(1) : false; Datum *relids; bool *nulls; int nrelids; RelationMergeInfo *relinfos; RelationMergeInfo *crelinfos; /* For compressed relations */ Oid hypertable_relid = InvalidOid; NameData hypertable_name; int32 hypertable_id = INVALID_HYPERTABLE_ID; Hypercube *merged_cube = NULL; const Hypercube *prev_cube = NULL; int mergeindex = -1; MemoryContext merge_cxt = NULL; List *rellocks = NIL; LOCKMODE lockmode = concurrently ? ExclusiveLock : AccessExclusiveLock; PreventCommandIfReadOnly("merge_chunks"); if (concurrently) PreventInTransactionBlock(true, "merge_chunks"); if (chunks_array == NULL) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("no chunks to merge specified"))); deconstruct_array(chunks_array, REGCLASSOID, sizeof(Oid), true, TYPALIGN_INT, &relids, &nulls, &nrelids); if (nrelids < 2) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("must specify at least two chunks to merge"))); /* Create a private memory context that will survive transaction boundaries */ merge_cxt = AllocSetContextCreate(PortalContext, "MergeChunksConcurrent", ALLOCSET_SMALL_SIZES); /* * The RelationMergeInfos are allocated on the Portal context since they * need to survive across transactions in case of merge with * "concurrently". */ relinfos = MemoryContextAllocZero(merge_cxt, sizeof(struct RelationMergeInfo) * nrelids); crelinfos = MemoryContextAllocZero(merge_cxt, sizeof(struct RelationMergeInfo) * nrelids); /* Sort relids array in order to find duplicates and lock relations in * consistent order to avoid deadlocks. It doesn't matter that we don't * order the nulls array the same since we only care about all relids * being non-null. */ qsort(relids, nrelids, sizeof(Datum), oid_cmp); /* Step 1: Do sanity checks and then prepare to sort rels in consistent order. */ for (int i = 0; i < nrelids; i++) { Oid relid = DatumGetObjectId(relids[i]); RelationMergeInfo *relinfo = &relinfos[i]; Chunk *chunk; Relation rel; ScanTupLock slice_lock = { .lockmode = LockTupleKeyShare, .waitpolicy = LockWaitBlock, .lockflags = TUPLE_LOCK_FLAG_FIND_LAST_VERSION, }; if (nulls[i] || !OidIsValid(relid)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid relation"))); if (i > 0 && DatumGetObjectId(relids[i]) == DatumGetObjectId(relids[i - 1])) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("duplicate relation \"%s\" in merge", get_rel_name(DatumGetObjectId(relids[i]))))); /* Lock the relation before doing other checks that can lock dependent * objects (this can otherwise lead to deadlocks with concurrent * operations). Note that if we take ExclusiveLock here to allow * readers while we are rewriting/merging the relations, the lock * needs to be upgraded to an AccessExclusiveLock later. This can also * lead to deadlocks. * * Ideally, we should probably take locks on all dependent objects as * well, at least on chunk-related objects that will be * dropped. Otherwise, that might also cause deadlocks later. For * example, if doing a concurrent DROP TABLE on one of the chunks will * lead to deadlock because it grabs locks on all dependencies before * dropping. * * However, for now we won't do that because that requires scanning * pg_depends and concurrent operations will probably fail anyway if * we remove the objects. We might as well fail with a deadlock. */ chunk = ts_chunk_get_by_relid_locked(relid, lockmode, &slice_lock, false); if (chunk == NULL) { /* * The relation is not a chunk, or it was deleted. Use * try_relation_open() to figure out which case. It will * automatically check that the relation still exists after the * lock is acquired. We only need AccessShareLock here since we * know the relation isn't a chunk and we only want to generate an * informative error. */ rel = try_relation_open(relid, AccessShareLock); if (rel) { bool is_table = (rel->rd_rel->relkind == RELKIND_RELATION); relation_close(rel, AccessShareLock); if (is_table) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("can only merge hypertable chunks"))); else ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot merge non-table relations"))); } else { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("chunk does not exist"), errdetail("The relation with OID %u might have been removed " "by a concurrent merge or other operation.", relid))); } } if (chunk->fd.osm_chunk) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot merge OSM chunks"))); if (ts_chunk_is_frozen(chunk)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot merge frozen chunk \"%s.%s\" scheduled for tiering", NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name)), errhint("Untier the chunk before merging."))); ChunkRewriteDeleteResult rewrite_result = ts_chunk_rewrite_delete(relid, true); switch (rewrite_result) { case ChunkRewriteOngoing: ereport(ERROR, (errcode(ERRCODE_OBJECT_IN_USE), errmsg("chunk is being merged by another process"))); break; case ChunkRewriteEntryDeleted: case ChunkRewriteEntryDeletedAndTableDropped: case ChunkRewriteEntryDoesNotExist: break; } /* Chunk already locked so we can get the relation directly from the cache */ rel = table_open(relid, NoLock); /* Only owner is allowed to merge */ if (!object_ownercheck(RelationRelationId, relid, GetUserId())) aclcheck_error(ACLCHECK_NOT_OWNER, get_relkind_objtype(rel->rd_rel->relkind), get_rel_name(relid)); /* Lock toast table to prevent it from being concurrently vacuumed */ if (rel->rd_rel->reltoastrelid) LockRelationOid(rel->rd_rel->reltoastrelid, lockmode); /* Add heap relation to the list of locked relations. We need this to * later grab session locks. */ rellocks = append_rellock(rellocks, rel, lockmode, merge_cxt); /* * Check for active uses of the relation in the current transaction, * including open scans and pending AFTER trigger events. */ CheckTableNotInUse(rel, "merge_chunks"); if (!merge_chunks_multidim_allowed()) { Cache *hcache; Hypertable *ht = ts_hypertable_cache_get_cache_and_entry(chunk->hypertable_relid, CACHE_FLAG_NONE, &hcache); Ensure(ht, "missing hypertable for chunk"); if (ht->fd.num_dimensions > 1) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot merge chunk in multi-dimensional hypertable"))); ts_cache_release(&hcache); } /* * Lock also internal compressed relation, if it exists. * * Don't fill in its MergeRelInfo until we sort relations in partition * order below, because the compressed relations need to be in the * same order. */ if (chunk->fd.compressed_chunk_id != INVALID_CHUNK_ID) { Oid crelid = ts_chunk_get_relid(chunk->fd.compressed_chunk_id, false); Relation crel = table_open(crelid, lockmode); rellocks = append_rellock(rellocks, crel, lockmode, merge_cxt); table_close(crel, NoLock); if (mergeindex == -1) mergeindex = i; /* Read compression chunk size stats */ bool found = ts_compression_chunk_size_get(chunk->fd.id, &relinfo->ccs); if (!found) elog(WARNING, "missing compression chunk size stats for compressed chunk \"%s\"", NameStr(chunk->fd.table_name)); } if (hypertable_id == INVALID_HYPERTABLE_ID) { hypertable_id = chunk->fd.hypertable_id; hypertable_relid = chunk->hypertable_relid; namestrcpy(&hypertable_name, get_rel_name(hypertable_relid)); } else if (hypertable_id != chunk->fd.hypertable_id) { Assert(i > 0); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot merge chunks across different hypertables"), errdetail("Chunk \"%s\" is part of hypertable \"%s\" while chunk \"%s\" is " "part of hypertable \"%s\"", get_rel_name(chunk->table_id), get_rel_name(chunk->hypertable_relid), get_rel_name(relinfos[i - 1].chunk->table_id), get_rel_name(relinfos[i - 1].chunk->hypertable_relid)))); } /* * It might not be possible to merge two chunks with different * storage, so better safe than sorry for now. */ Oid amoid = rel->rd_rel->relam; if (amoid != HEAP_TABLE_AM_OID) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("access method \"%s\" is not supported for merge", get_am_name(amoid)))); relinfo->relid = relid; relinfo->rel = rel; relinfo->relpersistence = rel->rd_rel->relpersistence; /* * Make sure the chunk is on the merge_cxt to survive * transaction when merge is concurrent */ MemoryContext old_mcxt = MemoryContextSwitchTo(merge_cxt); relinfo->chunk = ts_chunk_copy(chunk); MemoryContextSwitchTo(old_mcxt); } /* No compressed chunk found, so use index 0 for resulting merged chunk */ if (mergeindex == -1) mergeindex = 0; relinfos[mergeindex].isresult = true; /* Sort rels in partition order (in case of chunks). This is necessary to * validate that a merge is possible. */ qsort(relinfos, nrelids, sizeof(RelationMergeInfo), cmp_relations); /* * Step 2: Check alignment/mergeability and create the merged hypercube * (partition ranges). * * Also, create the final MergeRelationInfo array for any compressed * chunks in the same sort order as the non-compressed ones. */ for (int i = 0; i < nrelids; i++) { const Chunk *chunk = relinfos[i].chunk; Assert(chunk != NULL); if (merged_cube == NULL) { /* * Make sure the chunk is on the merge_cxt to survive * transaction when merge is concurrent */ MemoryContext old_mcxt = MemoryContextSwitchTo(merge_cxt); merged_cube = ts_hypercube_copy(chunk->cube); MemoryContextSwitchTo(old_mcxt); Assert(prev_cube == NULL); } else { Assert(chunk->cube->num_slices == merged_cube->num_slices); Assert(prev_cube != NULL); validate_merge_possible(prev_cube, chunk->cube); merge_cubes(merged_cube, chunk->cube); } prev_cube = chunk->cube; compute_rel_vacuum_cutoffs(relinfos[i].rel, &relinfos[i].cutoffs); /* * Fill in the compressed mergerelinfo array here after final sort of * rels so that the two arrays have the same order. */ if (chunk->fd.compressed_chunk_id != INVALID_CHUNK_ID) { RelationMergeInfo *crelinfo = &crelinfos[i]; Chunk *cchunk = ts_chunk_get_by_id(chunk->fd.compressed_chunk_id, true); /* * Allocate on merge_cxt to survive transaction end in * concurrent mode. */ MemoryContext old_mcxt = MemoryContextSwitchTo(merge_cxt); crelinfo->chunk = ts_chunk_copy(cchunk); MemoryContextSwitchTo(old_mcxt); crelinfo->relid = crelinfo->chunk->table_id; crelinfo->rel = table_open(crelinfo->relid, lockmode); crelinfo->isresult = relinfos[i].isresult; crelinfo->iscompressed_rel = true; crelinfo->relpersistence = crelinfo->rel->rd_rel->relpersistence; compute_rel_vacuum_cutoffs(crelinfos[i].rel, &crelinfos[i].cutoffs); rellocks = append_rellock(rellocks, crelinfo->rel, lockmode, merge_cxt); } /* Need to update the index of the result (merged) relation after * resort */ if (relinfos[i].isresult) mergeindex = i; } DEBUG_WAITPOINT("merge_chunks_before_rewrite"); /* * Step 3: create new heaps and copy all data. * * Now merge all the data into a new temporary heap relation. Do it * separately for the non-compressed and compressed relations. */ RelationMergeStats merge_stats, cmerge_stats; Oid new_relid = merge_relinfos(relinfos, nrelids, mergeindex, lockmode, &rellocks, &merge_stats, merge_cxt, concurrently); Oid new_crelid = merge_relinfos(crelinfos, nrelids, mergeindex, lockmode, &rellocks, &cmerge_stats, merge_cxt, concurrently); /* * From here on we only need the relinfos arrays. */ pfree(relids); pfree(nulls); if (concurrently) { ListCell *lc; /* * In concurrent mode, get a session-level lock on each chunk table to * protect against modifications across transactions. * * A session lock also allows us to release all locks on other * objects, reducing the risk of deadlocks when we upgrade the * ExclusiveLock session lock to an AccessExclusivelock transaction * lock to do the heap swap. */ foreach (lc, rellocks) { SessionLockInfo *lockinfo = (SessionLockInfo *) lfirst(lc); LockRelationIdForSession(&lockinfo->locktag, lockinfo->lockmode); } DEBUG_WAITPOINT("merge_chunks_before_first_commit"); /* * Check if we are being called from another procedure that has an SPI * context. In that case, we need to use SPI calls to start a new * transaction. */ if (SPI_inside_nonatomic_context()) { /* * Commit and retain transaction semantics. The commit_and_chain * call will automatically start a new transaction. */ SPI_commit_and_chain(); } else { PopActiveSnapshot(); CommitTransactionCommand(); StartTransactionCommand(); } DEBUG_WAITPOINT("merge_chunks_after_first_commit"); /* * In new transaction, get a new snapshot and take AccessExclusivelock * on all merge relations. */ PushActiveSnapshot(GetTransactionSnapshot()); lock_merged_rels(hypertable_relid, relinfos, crelinfos, nrelids, AccessExclusiveLock); relock_new_rels(new_relid, new_crelid, AccessExclusiveLock); } else { /* Make new table stats visible */ CommandCounterIncrement(); } /* * Step 4: Finish the merge by swapping relation files. */ DEBUG_WAITPOINT("merge_chunks_before_heap_swap"); merge_chunks_finish(new_relid, relinfos, nrelids, &merge_stats); if (OidIsValid(new_crelid)) merge_chunks_finish(new_crelid, crelinfos, nrelids, &cmerge_stats); /* * Step 5: Update the dimensional metadata and constraints for the chunk * we are keeping. */ if (merged_cube) { RelationMergeInfo *result_minfo = &relinfos[mergeindex]; Assert(result_minfo->chunk); DEBUG_WAITPOINT("merge_chunks_before_constraints"); chunk_update_constraints(result_minfo->chunk, merged_cube); ts_hypercube_free(merged_cube); } /* * Cleanup for concurrent mode. */ if (concurrently) { ListCell *lc; foreach (lc, rellocks) { SessionLockInfo *lockinfo = (SessionLockInfo *) lfirst(lc); UnlockRelationIdForSession(&lockinfo->locktag, lockinfo->lockmode); } PopActiveSnapshot(); } MemoryContextDelete(merge_cxt); DEBUG_ERROR_INJECTION("merge_chunks_fail"); DEBUG_WAITPOINT("merge_chunks_before_exit"); PG_RETURN_VOID(); } ================================================ FILE: tsl/src/chunk_split.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/multixact.h> #include <access/rewriteheap.h> #include <catalog/dependency.h> #include <catalog/heap.h> #include <catalog/indexing.h> #include <catalog/pg_am.h> #include <catalog/pg_collation.h> #include <catalog/pg_constraint.h> #include <commands/tablecmds.h> #include <nodes/lockoptions.h> #include <storage/bufmgr.h> #include <storage/lockdefs.h> #include <utils/acl.h> #include <utils/snapshot.h> #include <utils/syscache.h> #include <math.h> #include "chunk.h" #include "compression/api.h" #include "compression/compression.h" #include "compression/create.h" #include "debug_point.h" #include "hypercube.h" #include "partitioning.h" #include "trigger.h" #include "ts_catalog/array_utils.h" #include "ts_catalog/catalog.h" #include "ts_catalog/compression_chunk_size.h" /* * The split_chunk() procedure currently only supports two-way split. */ #define SPLIT_FACTOR 2 typedef struct SplitContext SplitContext; /* * SplitPointInfo * * Information about point where split happens, including column/dimension and * type we split along. Needed to route tuples to correct result relation. */ typedef struct SplitPoint { const Dimension *dim; int64 point; /* Point at which we split */ /* * Function to route a tuple to a result relation during the split. The * function's implementation is different depending on whether compressed * or non-compressed relations are split. */ HeapTuple (*route_next_tuple)(TupleTableSlot *slot, SplitContext *scontext, int *routing_index); } SplitPoint; /* * CompressedSplitPoint * * Version of SplitPoint for a compressed relation. * * Since tuples are compressed, routing happens on min/max-metadata so column * references are different. */ typedef struct CompressedSplitPoint { SplitPoint base; AttrNumber attnum_min; AttrNumber attnum_max; AttrNumber attnum_count; TupleDesc noncompressed_tupdesc; } CompressedSplitPoint; typedef struct RewriteStats { int64 tuples_written; int64 tuples_alive; int64 tuples_recently_dead; int64 tuples_in_segments; } RewriteStats; /* * RelationWriteState * * State used to rewrite the resulting relations when splitting. Holds * information about attribute mappings in case the relations have different * tuple descriptors (e.g., due to dropped or added columns). */ typedef struct RelationWriteState { BulkInsertState bistate; TupleTableSlot *dstslot; RewriteState rwstate; Relation targetrel; Datum *values; bool *isnull; /* * Tuple mapping is needed in case the old relation has dropped * columns. New relations (as result of split) are "clean" without dropped * columns. The tuple map converts tuples between the source and * destination chunks. */ TupleConversionMap *tupmap; RowCompressor compressor; RewriteStats stats; } RelationWriteState; /* * SplitContext * * Main state for doing a split. */ typedef struct SplitContext { Relation rel; /* Relation/chunk being split */ SplitPoint *sp; struct VacuumCutoffs cutoffs; int split_factor; /* Number of relations to split into */ /* Array of rewrite states used to write the new relations. Size of * split_factor. */ RelationWriteState *rws; int rws_index; /* Index into rsi array indicating currently routed * relation. Set to -1 if no currently routed relation. */ } SplitContext; /* * SplitRelationInfo * * Information about the result relations in a split. Also, information about * the number of tuples written to the relation is returned in the struct. */ typedef struct SplitRelationInfo { Oid relid; /* The relid of the result relation */ int32 chunk_id; /* The corresponding chunk's ID */ bool heap_swap; /* The original relation getting split will receive a heap * swap. New chunks won't get a heap swap since they are * new and not visible to anyone else. */ RewriteStats stats; } SplitRelationInfo; static void relation_split_info_init(RelationWriteState *rws, Relation srcrel, Oid target_relid, struct VacuumCutoffs *cutoffs) { rws->targetrel = table_open(target_relid, AccessExclusiveLock); rws->bistate = GetBulkInsertState(); rws->rwstate = begin_heap_rewrite(srcrel, rws->targetrel, cutoffs->OldestXmin, cutoffs->FreezeLimit, cutoffs->MultiXactCutoff); rws->tupmap = convert_tuples_by_name(RelationGetDescr(srcrel), RelationGetDescr(rws->targetrel)); /* Create tuple slot for new partition. */ rws->dstslot = table_slot_create(rws->targetrel, NULL); ExecStoreAllNullTuple(rws->dstslot); rws->values = (Datum *) palloc0(RelationGetDescr(srcrel)->natts * sizeof(Datum)); rws->isnull = (bool *) palloc0(RelationGetDescr(srcrel)->natts * sizeof(bool)); } static void relation_split_info_cleanup(RelationWriteState *rws, int ti_options) { ExecDropSingleTupleTableSlot(rws->dstslot); FreeBulkInsertState(rws->bistate); table_finish_bulk_insert(rws->targetrel, ti_options); end_heap_rewrite(rws->rwstate); table_close(rws->targetrel, NoLock); pfree(rws->values); pfree(rws->isnull); if (rws->tupmap) free_conversion_map(rws->tupmap); rws->targetrel = NULL; rws->bistate = NULL; rws->dstslot = NULL; rws->tupmap = NULL; rws->values = NULL; rws->isnull = NULL; } /* * Reconstruct and rewrite the given tuple. * * Mostly taken from heapam module. * * When splitting a relation in two, the old relation is retained for one of * the result relations while the other is created new. This might lead to a * situation where the two result relations have different attribute mappings * because the old one could have dropped columns while the new one is "clean" * without dropped columns. Therefore, the rewrite function needs to account * for this when the tuple is rewritten. */ static void reform_and_rewrite_tuple(HeapTuple tuple, Relation srcrel, RelationWriteState *rws) { TupleDesc oldTupDesc = RelationGetDescr(srcrel); TupleDesc newTupDesc = RelationGetDescr(rws->targetrel); HeapTuple tupcopy; if (rws->tupmap) { /* * If this is the "new" relation, the tuple map might be different * from the "source" relation. */ tupcopy = execute_attr_map_tuple(tuple, rws->tupmap); } else { int i; heap_deform_tuple(tuple, oldTupDesc, rws->values, rws->isnull); /* Be sure to null out any dropped columns if this is the "old" * relation. A relation created new doesn't have dropped columns. */ for (i = 0; i < newTupDesc->natts; i++) { if (TupleDescAttr(newTupDesc, i)->attisdropped) rws->isnull[i] = true; } tupcopy = heap_form_tuple(newTupDesc, rws->values, rws->isnull); } /* The heap rewrite module does the rest */ rewrite_heap_tuple(rws->rwstate, tuple, tupcopy); heap_freetuple(tupcopy); } static Datum slot_get_partition_value(TupleTableSlot *slot, AttrNumber attnum, const SplitPoint *sp) { bool isnull = false; Datum value = slot_getattr(slot, attnum, &isnull); /* * Space-partition columns can have NULL values, but we only support * splits on time dimensions at the moment. */ Ensure(!isnull, "unexpected NULL value in partitioning column"); /* * Both time and space dimensions can have partitioning functions, so it * is necessary to always check for a function. */ if (NULL != sp->dim->partitioning) { Oid collation; collation = TupleDescAttr(slot->tts_tupleDescriptor, AttrNumberGetAttrOffset(attnum))->attcollation; value = ts_partitioning_func_apply(sp->dim->partitioning, collation, value); } return value; } /* * Compute the partition/routing index for a tuple. * * Returns 0 or 1 for first or second partition, respectively. */ static int route_tuple(TupleTableSlot *slot, const SplitPoint *sp) { Oid dimtype = ts_dimension_get_partition_type(sp->dim); Datum value = slot_get_partition_value(slot, sp->dim->column_attno, sp); int64 point = ts_time_value_to_internal(value, dimtype); /* * Route to partition based on new boundaries. Only 2-way split is * supported now, so routing is easy. An N-way split requires, e.g., * binary search. */ return (point < sp->point) ? 0 : 1; } /* * Compute the partition/routing index for a compressed tuple. * * Returns 0 or 1 for first or second partition, and -1 if the split point * falls within the given compressed tuple. */ static int route_compressed_tuple(TupleTableSlot *slot, const SplitPoint *sp) { const CompressedSplitPoint *csp = (const CompressedSplitPoint *) sp; Oid dimtype = ts_dimension_get_partition_type(sp->dim); Datum min_value = slot_get_partition_value(slot, csp->attnum_min, sp); Datum max_value = slot_get_partition_value(slot, csp->attnum_max, sp); int64 min_point = ts_time_value_to_internal(min_value, dimtype); int64 max_point = ts_time_value_to_internal(max_value, dimtype); if (max_point < sp->point) return 0; if (min_point >= sp->point) return 1; Assert(min_point < sp->point && max_point >= sp->point); return -1; } /* * Route a tuple to its partition. * * Only a 2-way split is supported at this time. * * For every non-NULL tuple returned, the routing_index will be set to 0 for * the first partition, and 1 for then second. */ static HeapTuple route_next_non_compressed_tuple(TupleTableSlot *slot, SplitContext *scontext, int *routing_index) { if (scontext->rws_index != -1) { scontext->rws_index = -1; return NULL; } scontext->rws_index = route_tuple(slot, scontext->sp); *routing_index = scontext->rws_index; return ExecFetchSlotHeapTuple(slot, false, NULL); } /* * Route a compressed tuple (segment) to its corresponding result partition * for the split. * * If the split point is found to be within the segment, it needs to be split * and sub-segments returned instead. Therefore, this function should be * called in a loop until returning NULL (no sub-segments left). If the * segment is not split, only the original segment is returned. * * For every non-NULL tuple returned, the routing_index will be set to 0 for * the first partition, and 1 for the second. */ static HeapTuple route_next_compressed_tuple(TupleTableSlot *slot, SplitContext *scontext, int *routing_index) { CompressedSplitPoint *csp = (CompressedSplitPoint *) scontext->sp; Assert(scontext->rws_index >= -1 && scontext->rws_index <= scontext->split_factor); if (scontext->rws_index == scontext->split_factor) { /* Nothing more to route for this tuple, so return NULL */ scontext->rws_index = -1; *routing_index = -1; return NULL; } else if (scontext->rws_index >= 0) { /* Segment is being split and recompressed into a sub-segment per * partition. Return the sub-segments until done. */ Assert(scontext->rws_index < scontext->split_factor); RelationWriteState *rws = &scontext->rws[scontext->rws_index]; HeapTuple new_tuple = row_compressor_build_tuple(&rws->compressor); HeapTuple old_tuple = ExecFetchSlotHeapTuple(slot, false, NULL); /* Copy over visibility information from the original segment * tuple. First copy the HeapTupleFields holding the xmin and * xmax. Then copy the infomask which has, among other things, the * frozen flag bits. */ memcpy(&new_tuple->t_data->t_choice.t_heap, &old_tuple->t_data->t_choice.t_heap, sizeof(HeapTupleFields)); new_tuple->t_data->t_infomask &= ~HEAP_XACT_MASK; new_tuple->t_data->t_infomask2 &= ~HEAP2_XACT_MASK; new_tuple->t_data->t_infomask |= old_tuple->t_data->t_infomask & HEAP_XACT_MASK; new_tuple->t_tableOid = RelationGetRelid(rws->targetrel); row_compressor_clear_batch(&rws->compressor, false); rws->stats.tuples_in_segments += rws->compressor.rowcnt_pre_compression; *routing_index = scontext->rws_index; scontext->rws_index++; row_compressor_close(&rws->compressor); return new_tuple; } *routing_index = route_compressed_tuple(slot, scontext->sp); if (*routing_index == -1) { /* * The split point is within the current compressed segment. It needs * to be split across the partitions by decompressing and * recompressing into sub-segments. */ HeapTuple tuple; CompressionSettings *csettings = ts_compression_settings_get_by_compress_relid(RelationGetRelid(scontext->rel)); tuple = ExecFetchSlotHeapTuple(slot, false, NULL); RowDecompressor decompressor = build_decompressor(slot->tts_tupleDescriptor, csp->noncompressed_tupdesc); heap_deform_tuple(tuple, decompressor.in_desc, decompressor.compressed_datums, decompressor.compressed_is_nulls); int nrows = decompress_batch(&decompressor); /* * Initialize a compressor for each new partition. */ for (int i = 0; i < scontext->split_factor; i++) { RelationWriteState *rws = &scontext->rws[i]; row_compressor_init(&rws->compressor, csettings, csp->noncompressed_tupdesc, RelationGetDescr(scontext->rws[i].targetrel)); } /* * Route each decompressed tuple to its corresponding partition's * compressor. */ for (int i = 0; i < nrows; i++) { int routing_index = route_tuple(decompressor.decompressed_slots[i], scontext->sp); Assert(routing_index == 0 || routing_index == 1); RelationWriteState *rws = &scontext->rws[routing_index]; /* * Since we're splitting a segment, the new segments will be * ordered like the original segment. Also, there is no risk of * the segments getting too big since we are only making segments * smaller. */ row_compressor_append_ordered_slot(&rws->compressor, decompressor.decompressed_slots[i]); } row_decompressor_close(&decompressor); scontext->rws_index = 0; /* * Call this function again to return the sub-segments. */ return route_next_compressed_tuple(slot, scontext, routing_index); } /* Update tuple count stats for compressed data */ bool isnull; Datum count = slot_getattr(slot, csp->attnum_count, &isnull); scontext->rws[*routing_index].stats.tuples_in_segments += DatumGetInt32(count); /* * The compressed tuple (segment) can be routed without splitting it. */ Assert(*routing_index >= 0 && *routing_index < scontext->split_factor); scontext->rws_index = scontext->split_factor; return ExecFetchSlotHeapTuple(slot, false, NULL); } static double copy_tuples_for_split(SplitContext *scontext) { Relation srcrel = scontext->rel; TupleTableSlot *srcslot; MemoryContext oldcxt; EState *estate; ExprContext *econtext; TableScanDesc scan; SplitPoint *sp = scontext->sp; estate = CreateExecutorState(); /* Create the tuple slot */ srcslot = table_slot_create(srcrel, NULL); /* * Scan through the rows using SnapshotAny to see everything so that we * can transfer tuples that are deleted or updated but still visible to * concurrent transactions. */ scan = table_beginscan(srcrel, SnapshotAny, 0, NULL); /* * Switch to per-tuple memory context and reset it for each tuple * produced, so we don't leak memory. */ econtext = GetPerTupleExprContext(estate); oldcxt = MemoryContextSwitchTo(GetPerTupleMemoryContext(estate)); /* * Read all the data from the split relation and route the tuples to the * new partitions. Do some vacuuming and cleanup at the same * time. Transfer all visibility information to the new relations. * * Main loop inspired by heapam_relation_copy_for_cluster() used to run * CLUSTER and VACUUM FULL on a table. */ double num_tuples = 0.0; double tups_vacuumed = 0.0; double tups_recently_dead = 0.0; BufferHeapTupleTableSlot *hslot; int routingindex = -1; while (table_scan_getnextslot(scan, ForwardScanDirection, srcslot)) { RelationWriteState *rws = NULL; HeapTuple tuple; Buffer buf; bool isdead; bool isalive = false; CHECK_FOR_INTERRUPTS(); ResetExprContext(econtext); tuple = ExecFetchSlotHeapTuple(srcslot, false, NULL); hslot = (BufferHeapTupleTableSlot *) srcslot; buf = hslot->buffer; LockBuffer(buf, BUFFER_LOCK_SHARE); switch (HeapTupleSatisfiesVacuum(tuple, scontext->cutoffs.OldestXmin, buf)) { case HEAPTUPLE_DEAD: /* Definitely dead */ isdead = true; break; case HEAPTUPLE_RECENTLY_DEAD: tups_recently_dead += 1; isdead = false; break; case HEAPTUPLE_LIVE: /* Live or recently dead, must copy it */ isdead = false; isalive = true; break; case HEAPTUPLE_INSERT_IN_PROGRESS: /* * Since we hold exclusive lock on the relation, normally the * only way to see this is if it was inserted earlier in our * own transaction. Give a warning if this case does not * apply; in any case we better copy it. */ if (!TransactionIdIsCurrentTransactionId(HeapTupleHeaderGetXmin(tuple->t_data))) elog(WARNING, "concurrent insert in progress within table \"%s\"", RelationGetRelationName(srcrel)); /* treat as live */ isdead = false; isalive = true; break; case HEAPTUPLE_DELETE_IN_PROGRESS: /* * Similar situation to INSERT_IN_PROGRESS case. */ if (!TransactionIdIsCurrentTransactionId( HeapTupleHeaderGetUpdateXid(tuple->t_data))) elog(WARNING, "concurrent delete in progress within table \"%s\"", RelationGetRelationName(srcrel)); /* treat as recently dead */ tups_recently_dead += 1; isalive = true; isdead = false; break; default: elog(ERROR, "unexpected HeapTupleSatisfiesVacuum result"); isdead = false; /* keep compiler quiet */ break; } LockBuffer(buf, BUFFER_LOCK_UNLOCK); HeapTuple tuple2; /* * Route the tuple to the matching (new) partition. The routing is * done in a loop because compressed tuple segments might be split * into multiple sub-segment tuples if the split is in the middle of * that segment. */ while ((tuple2 = sp->route_next_tuple(srcslot, scontext, &routingindex))) { Assert(routingindex >= 0 && routingindex < scontext->split_factor); rws = &scontext->rws[routingindex]; if (isdead) { tups_vacuumed += 1; /* heap rewrite module still needs to see it... */ if (rewrite_heap_dead_tuple(rws->rwstate, tuple2)) { /* A previous recently-dead tuple is now known dead */ tups_vacuumed += 1; tups_recently_dead -= 1; } } else { num_tuples++; rws->stats.tuples_written++; if (isalive) rws->stats.tuples_alive++; reform_and_rewrite_tuple(tuple2, srcrel, rws); } } } MemoryContextSwitchTo(oldcxt); const char *nspname = get_namespace_name(RelationGetNamespace(srcrel)); ereport(DEBUG1, (errmsg("\"%s.%s\": found %.0f removable, %.0f nonremovable row versions", nspname, RelationGetRelationName(srcrel), tups_vacuumed, num_tuples), errdetail("%.0f dead row versions cannot be removed yet.", tups_recently_dead))); table_endscan(scan); ExecDropSingleTupleTableSlot(srcslot); FreeExecutorState(estate); return num_tuples; } /* * Split a relation into "split_factor" pieces. */ static void split_relation(Relation rel, SplitPoint *sp, unsigned int split_factor, SplitRelationInfo *split_relations) { char relpersistence = rel->rd_rel->relpersistence; SplitContext scontext = { .rel = rel, .split_factor = split_factor, .rws = palloc0(sizeof(RelationWriteState) * split_factor), .sp = sp, .rws_index = -1, }; compute_rel_vacuum_cutoffs(scontext.rel, &scontext.cutoffs); for (unsigned int i = 0; i < split_factor; i++) { SplitRelationInfo *sri = &split_relations[i]; Oid write_relid = sri->relid; if (sri->heap_swap) { write_relid = make_new_heap(RelationGetRelid(rel), rel->rd_rel->reltablespace, rel->rd_rel->relam, relpersistence, AccessExclusiveLock); } relation_split_info_init(&scontext.rws[i], rel, write_relid, &scontext.cutoffs); } DEBUG_WAITPOINT("split_chunk_before_tuple_routing"); copy_tuples_for_split(&scontext); table_close(rel, NoLock); for (unsigned int i = 0; i < split_factor; i++) { RelationWriteState *rws = &scontext.rws[i]; SplitRelationInfo *sri = &split_relations[i]; ReindexParams reindex_params = { 0 }; int reindex_flags = REINDEX_REL_SUPPRESS_INDEX_USE; Oid write_relid = RelationGetRelid(rws->targetrel); Ensure(relpersistence == RELPERSISTENCE_PERMANENT, "only permanent chunks can be split"); reindex_flags |= REINDEX_REL_FORCE_INDEXES_PERMANENT; /* Save stats before cleaning up rewrite state */ memcpy(&sri->stats, &rws->stats, sizeof(sri->stats)); relation_split_info_cleanup(rws, TABLE_INSERT_SKIP_FSM); /* * Only reindex new chunks. Existing chunk will be reindexed during * the heap swap. */ if (sri->heap_swap) { /* Finally, swap the heap of the chunk that we split so that it only * contains the tuples for its new partition boundaries. AccessExclusive * lock is held during the swap. */ finish_heap_swap(sri->relid, write_relid, false, /* system catalog */ false /* swap toast by content */, true, /* check constraints */ true, /* internal? */ scontext.cutoffs.FreezeLimit, scontext.cutoffs.MultiXactCutoff, relpersistence); } else { /* * Update relfrozenxid and relminmxid for the new chunk. * This is necessary because the heap rewrite preserved tuple * visibility information (xmin/xmax), and the new relation's * relfrozenxid must reflect the freeze limit used during the rewrite. * Without this, VACUUM may find tuples with xmin < relfrozenxid * that aren't frozen, causing "found xmin from before relfrozenxid" errors. */ Relation relRelation = table_open(RelationRelationId, RowExclusiveLock); HeapTuple reltup = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(sri->relid)); if (!HeapTupleIsValid(reltup)) elog(ERROR, "cache lookup failed for relation %u", sri->relid); Form_pg_class relform = (Form_pg_class) GETSTRUCT(reltup); relform->relfrozenxid = scontext.cutoffs.FreezeLimit; relform->relminmxid = scontext.cutoffs.MultiXactCutoff; CatalogTupleUpdate(relRelation, &reltup->t_self, reltup); heap_freetuple(reltup); table_close(relRelation, RowExclusiveLock); reindex_relation_compat(NULL, sri->relid, reindex_flags, &reindex_params); } } pfree(scontext.rws); } static void compute_compression_size_stats_fraction(Form_compression_chunk_size ccs, double fraction) { ccs->compressed_heap_size = (int64) rint((double) ccs->compressed_heap_size * fraction); ccs->uncompressed_heap_size = (int64) rint((double) ccs->uncompressed_heap_size * fraction); ccs->uncompressed_index_size = (int64) rint((double) ccs->uncompressed_index_size * fraction); ccs->compressed_index_size = (int64) rint((double) ccs->compressed_index_size * fraction); ccs->uncompressed_toast_size = (int64) rint((double) ccs->uncompressed_toast_size * fraction); ccs->compressed_toast_size = (int64) rint((double) ccs->compressed_toast_size * fraction); ccs->numrows_frozen_immediately = (int64) rint((double) ccs->numrows_frozen_immediately * fraction); ccs->numrows_pre_compression = (int64) rint((double) ccs->numrows_pre_compression * fraction); ccs->numrows_post_compression = (int64) rint((double) ccs->numrows_post_compression * fraction); } static void update_compression_stats_for_split(const SplitRelationInfo *split_relations, const SplitRelationInfo *compressed_split_relations, int split_factor) { double total_tuples = 0; Assert(split_factor > 1); /* * Set the new chunk status and calculate the total amount of tuples * (compressed and non-compressed), which is used to calculated the * fraction of data each new partition received. */ for (int i = 0; i < split_factor; i++) { const SplitRelationInfo *sri = &split_relations[i]; const SplitRelationInfo *csri = &compressed_split_relations[i]; Chunk *chunk = ts_chunk_get_by_relid(sri->relid, true); if (sri->stats.tuples_written > 0) ts_chunk_set_partial(chunk); else ts_chunk_clear_status(chunk, CHUNK_STATUS_COMPRESSED_PARTIAL); total_tuples += sri->stats.tuples_alive + csri->stats.tuples_in_segments; } /* * Get the existing stats for the original chunk. The stats will be split * across the resulting new chunks. */ FormData_compression_chunk_size ccs; ts_compression_chunk_size_get(split_relations[0].chunk_id, &ccs); for (int i = 0; i < split_factor; i++) { const SplitRelationInfo *sri = &split_relations[i]; const SplitRelationInfo *csri = &compressed_split_relations[i]; FormData_compression_chunk_size new_ccs; /* Calculate the fraction of compressed and non-compressed data received * by the first partition (chunk) */ double fraction = 0.0; if (total_tuples > 0) fraction = (sri->stats.tuples_alive + csri->stats.tuples_in_segments) / total_tuples; memcpy(&new_ccs, &ccs, sizeof(ccs)); compute_compression_size_stats_fraction(&new_ccs, fraction); if (sri->heap_swap) { ts_compression_chunk_size_update(sri->chunk_id, &new_ccs); } else { /* The new partition (chunk) doesn't have stats so create new. */ RelationSize relsize = { .heap_size = new_ccs.uncompressed_heap_size, .index_size = new_ccs.uncompressed_index_size, .toast_size = new_ccs.uncompressed_toast_size, }; RelationSize compressed_relsize = { .heap_size = new_ccs.compressed_heap_size, .index_size = new_ccs.compressed_index_size, .toast_size = new_ccs.compressed_toast_size, }; compression_chunk_size_catalog_insert(sri->chunk_id, &relsize, csri->chunk_id, &compressed_relsize, new_ccs.numrows_pre_compression, new_ccs.numrows_post_compression, new_ccs.numrows_frozen_immediately); } } } /* * Update the chunk stats for the split. Also set the chunk state (partial or * non-partial) and reltuples in pg_class. * * To calculate new compression chunk size stats, the existing stats are * simply split across the result partitions based on the fraction of data * they received. New stats are not calculated since the pre-compression sizes * for the split relations are not known (it would require decompression and * then measuring the disk usage). The advantage of splitting the stats is * that the total size stats is the the same after the split as they were * before the split. */ static void update_chunk_stats_for_split(const SplitRelationInfo *split_relations, const SplitRelationInfo *compressed_split_relations, int split_factor) { if (compressed_split_relations) update_compression_stats_for_split(split_relations, compressed_split_relations, split_factor); /* * Update reltuples in pg_class. The reltuples are normally updated on * reindex, so this update only matters in case of no indexes. */ Relation relRelation = table_open(RelationRelationId, RowExclusiveLock); for (int i = 0; i < split_factor; i++) { const SplitRelationInfo *sri = &split_relations[i]; Relation rel; double ntuples = sri->stats.tuples_alive; rel = table_open(sri->relid, AccessShareLock); update_relstats(relRelation, sri->relid, RelationGetNumberOfBlocks(rel), ntuples); table_close(rel, NoLock); } table_close(relRelation, RowExclusiveLock); } /* * Split a chunk along a given dimension and split point. * * The column/dimension and "split at" point are optional. If these arguments * are not specified, the chunk is split in two equal ranges based on the * primary partitioning column. * * The split is done using the table rewrite approach used by the PostgreSQL * CLUSTER code (also used for VACUUM FULL). It uses the rewrite module to * retain the visibility information of tuples, and also transferring (old) * deleted or updated tuples that are still visible to concurrent transactions * reading an older snapshot. Completely dead tuples are garbage collected. * * The advantage of the rewrite approach is that it is fully MVCC compliant * and ensures the result relations have minimal garbage after the split. Note * that locks don't fully protect against visibility issues since a concurrent * transaction can be pinned to an older snapshot while not (yet) holding any * locks on relations (chunks and hypertables) being split. */ Datum chunk_split_chunk(PG_FUNCTION_ARGS) { Oid relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); const Chunk *chunk; Relation srcrel; ScanTupLock slice_lock = { .lockmode = LockTupleNoKeyExclusive, .waitpolicy = LockWaitBlock, .lockflags = TUPLE_LOCK_FLAG_FIND_LAST_VERSION, }; chunk = ts_chunk_get_by_relid_locked(relid, AccessExclusiveLock, &slice_lock, true); /* Chunk already locked, so use NoLock */ srcrel = table_open(relid, NoLock); if (srcrel->rd_rel->relkind != RELKIND_RELATION) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot split non-table relations"))); Oid amoid = srcrel->rd_rel->relam; if (amoid != HEAP_TABLE_AM_OID) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("access method \"%s\" is not supported for split", get_am_name(amoid)))); /* Only owner is allowed to split */ if (!object_ownercheck(RelationRelationId, relid, GetUserId())) aclcheck_error(ACLCHECK_NOT_OWNER, get_relkind_objtype(srcrel->rd_rel->relkind), get_rel_name(relid)); /* Lock toast table to prevent it from being concurrently vacuumed */ if (srcrel->rd_rel->reltoastrelid) LockRelationOid(srcrel->rd_rel->reltoastrelid, AccessExclusiveLock); /* * Check for active uses of the relation in the current transaction, * including open scans and pending AFTER trigger events. */ CheckTableNotInUse(srcrel, "split_chunk"); if (chunk->fd.osm_chunk) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot split OSM chunks"))); if (ts_chunk_is_frozen(chunk)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot split frozen chunk \"%s.%s\" scheduled for tiering", NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name)), errhint("Untier the chunk before splitting it."))); Cache *hcache; const Hypertable *ht = ts_hypertable_cache_get_cache_and_entry(chunk->hypertable_relid, CACHE_FLAG_NONE, &hcache); const Dimension *dim = hyperspace_get_open_dimension(ht->space, 0); Ensure(dim, "no primary dimension for chunk"); if (ht->fd.num_dimensions > 1) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot split chunk in multi-dimensional hypertable"))); NameData splitdim_name; namestrcpy(&splitdim_name, NameStr(dim->fd.column_name)); Oid splitdim_type = ts_dimension_get_partition_type(dim); Oid splitcolumn_type = dim->fd.column_type; Datum split_at_datum; bool have_split_at = false; /* Check split_at argument */ if (!PG_ARGISNULL(1)) { Oid argtype = get_fn_expr_argtype(fcinfo->flinfo, 1); Datum arg = PG_GETARG_DATUM(1); if (argtype == UNKNOWNOID) { Oid infuncid = InvalidOid; Oid typioparam; getTypeInputInfo(splitdim_type, &infuncid, &typioparam); switch (get_func_nargs(infuncid)) { case 1: /* Functions that take one input argument, e.g., the Date function */ split_at_datum = OidFunctionCall1(infuncid, arg); break; case 3: /* Timestamp functions take three input arguments */ split_at_datum = OidFunctionCall3(infuncid, arg, ObjectIdGetDatum(InvalidOid), Int32GetDatum(-1)); break; default: /* Shouldn't be any time types with other number of args */ Ensure(false, "invalid type for split_at"); pg_unreachable(); } argtype = splitdim_type; } else split_at_datum = arg; if (argtype != splitcolumn_type) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid type '%s' for split_at argument", format_type_be(argtype)), errdetail("The argument type must match the dimension \"%s\"", NameStr(dim->fd.column_name)))); have_split_at = true; } /* Serialize chunk creation around the root hypertable. NOTE: also taken * in ts_chunk_find_or_create_without_cuts() below. */ LockRelationOid(ht->main_table_relid, ShareUpdateExclusiveLock); /* * Find the existing partition slice for the chunk being split. */ DimensionSlice *slice = NULL; Hypercube *new_cube = ts_hypercube_copy(chunk->cube); for (int i = 0; i < new_cube->num_slices; i++) { DimensionSlice *curr_slice = new_cube->slices[i]; if (curr_slice->fd.dimension_id == dim->fd.id) { slice = curr_slice; break; } } Ensure(slice, "no chunk slice for dimension %s", NameStr(dim->fd.column_name)); /* * Pick split point and calculate new ranges. If no split point is given * by the user, then split in the middle. */ int64 interval_range = slice->fd.range_end - slice->fd.range_start; int64 split_at = 0; if (have_split_at) { Datum dim_datum; if (NULL != dim->partitioning) dim_datum = ts_partitioning_func_apply(dim->partitioning, C_COLLATION_OID, split_at_datum); else dim_datum = split_at_datum; split_at = ts_time_value_to_internal(dim_datum, splitdim_type); /* * Check that the split_at value actually produces a valid split. Note * that range_start is inclusive while range_end is non-inclusive. The * split_at value needs to produce partition ranges of at least length * 1. */ if (split_at < (slice->fd.range_start + 1) || split_at > (slice->fd.range_end - 2)) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot split chunk at %s", ts_datum_to_string(dim_datum, splitdim_type)))); } } else split_at = slice->fd.range_start + (interval_range / 2); elog(DEBUG1, "splitting chunk %s at %s", get_rel_name(relid), ts_internal_to_time_string(split_at, splitdim_type)); const CompressionSettings *compress_settings = ts_compression_settings_get(relid); int64 old_end = slice->fd.range_end; /* Update the slice range for the existing chunk */ slice->fd.range_end = split_at; chunk_update_constraints(chunk, new_cube); /* Update the slice for the new chunk */ slice->fd.range_start = split_at; slice->fd.range_end = old_end; slice->fd.id = 0; /* Must set to 0 to mark as new for it to be created */ /* Make updated constraints visible */ CommandCounterIncrement(); /* Reread hypertable after constraints changed */ ts_cache_release(&hcache); ht = ts_hypertable_cache_get_cache_and_entry(chunk->hypertable_relid, CACHE_FLAG_NONE, &hcache); bool created = false; Chunk *new_chunk = ts_chunk_find_or_create_without_cuts(ht, new_cube, NameStr(chunk->fd.schema_name), NULL, InvalidOid, &created); Ensure(created, "could not create chunk for split"); Assert(new_chunk); Chunk *new_compressed_chunk = NULL; if (compress_settings != NULL) { Hypertable *ht_compressed = ts_hypertable_get_by_id(ht->fd.compressed_hypertable_id); new_compressed_chunk = create_compress_chunk(ht_compressed, new_chunk, InvalidOid); ts_trigger_create_all_on_chunk(new_compressed_chunk); ts_chunk_set_compressed_chunk(new_chunk, new_compressed_chunk->fd.id); } CommandCounterIncrement(); DEBUG_WAITPOINT("split_chunk_after_creating_new_chunk"); SplitPoint sp = { .point = split_at, .dim = hyperspace_get_open_dimension(ht->space, 0), .route_next_tuple = route_next_non_compressed_tuple, }; /* * Array of the heap Oids of the resulting relations. Those relations that * will get a heap swap (i.e., the original chunk) has heap_swap set to * true. */ SplitRelationInfo split_relations[SPLIT_FACTOR] = { [0] = { .relid = relid, .chunk_id = chunk->fd.id, .heap_swap = true }, [1] = { .relid = new_chunk->table_id, .chunk_id = new_chunk->fd.id, .heap_swap = false } }; SplitRelationInfo csplit_relations[SPLIT_FACTOR] = {}; SplitRelationInfo *compressed_split_relations = NULL; /* Split and rewrite the compressed relation first, if one exists. */ if (new_compressed_chunk) { int orderby_pos = ts_array_position(compress_settings->fd.orderby, NameStr(splitdim_name)); Ensure(orderby_pos > 0, "primary dimension \"%s\" is not in compression settings", NameStr(splitdim_name)); /* * Get the attribute numbers for the primary dimension's min and max * values in the compressed relation. We'll use these to get the time * range of compressed segments in order to route segments to the * right result chunk. */ const char *min_attname = column_segment_min_name(orderby_pos); const char *max_attname = column_segment_max_name(orderby_pos); CompressedSplitPoint csp = { .base = { .point = split_at, .dim = hyperspace_get_open_dimension(ht->space, 0), .route_next_tuple = route_next_compressed_tuple, }, .attnum_min = get_attnum(compress_settings->fd.compress_relid, min_attname), .attnum_max = get_attnum(compress_settings->fd.compress_relid, max_attname), .attnum_count = get_attnum(compress_settings->fd.compress_relid, COMPRESSION_COLUMN_METADATA_COUNT_NAME), .noncompressed_tupdesc = CreateTupleDescCopy(RelationGetDescr(srcrel)), }; csplit_relations[0] = (SplitRelationInfo){ .relid = compress_settings->fd.compress_relid, .chunk_id = chunk->fd.compressed_chunk_id, .heap_swap = true }; csplit_relations[1] = (SplitRelationInfo){ .relid = new_compressed_chunk->table_id, .chunk_id = new_chunk->fd.compressed_chunk_id, .heap_swap = false }; Relation compressed_rel = table_open(compress_settings->fd.compress_relid, AccessExclusiveLock); compressed_split_relations = csplit_relations; split_relation(compressed_rel, &csp.base, SPLIT_FACTOR, compressed_split_relations); } /* Now split the non-compressed relation */ split_relation(srcrel, &sp, SPLIT_FACTOR, split_relations); ts_cache_release(&hcache); /* Update stats after split is done */ update_chunk_stats_for_split(split_relations, compressed_split_relations, SPLIT_FACTOR); DEBUG_WAITPOINT("split_chunk_at_end"); PG_RETURN_VOID(); } ================================================ FILE: tsl/src/chunkwise_agg.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <optimizer/appendinfo.h> #include <optimizer/cost.h> #include <optimizer/pathnode.h> #include <optimizer/paths.h> #include <optimizer/prep.h> #include <optimizer/tlist.h> #include "chunkwise_agg.h" #include "guc.h" #include "import/planner.h" #include "nodes/chunk_append/chunk_append.h" #include "nodes/columnar_scan/columnar_scan.h" #include "planner.h" /* Helper function to find the first node of the provided type in the pathlist of the relation */ static Node * find_node(const RelOptInfo *relation, NodeTag type) { ListCell *lc; foreach (lc, relation->pathlist) { Node *node = lfirst(lc); if (nodeTag(node) == type) return node; } return NULL; } /* Check if the relation already has a min/max path */ static bool has_min_max_agg_path(const RelOptInfo *relation) { return find_node(relation, T_MinMaxAggPath) != NULL; } /* * Get an an existing aggregation path for the given relation or NULL if no aggregation path exists. */ static AggPath * get_existing_agg_path(const RelOptInfo *relation) { Node *node = find_node(relation, T_AggPath); return node ? castNode(AggPath, node) : NULL; } /* * Get all subpaths from a Append, MergeAppend, or ChunkAppend path */ static void get_subpaths_from_append_path(Path *path, List **subpaths, Path **append, Path **gather) { if (IsA(path, AppendPath)) { AppendPath *append_path = castNode(AppendPath, path); *subpaths = append_path->subpaths; *append = path; return; } if (IsA(path, MergeAppendPath)) { MergeAppendPath *merge_append_path = castNode(MergeAppendPath, path); *subpaths = merge_append_path->subpaths; *append = path; return; } if (ts_is_chunk_append_path(path)) { CustomPath *custom_path = castNode(CustomPath, path); *subpaths = custom_path->custom_paths; *append = path; return; } if (IsA(path, GatherPath)) { *gather = path; get_subpaths_from_append_path(castNode(GatherPath, path)->subpath, subpaths, append, /* gather = */ NULL); return; } if (IsA(path, GatherMergePath)) { *gather = path; get_subpaths_from_append_path(castNode(GatherMergePath, path)->subpath, subpaths, append, /* gather = */ NULL); return; } if (IsA(path, SortPath)) { /* Can see GatherMerge -> Sort -> Partial HashAggregate in parallel plans. */ get_subpaths_from_append_path(castNode(SortPath, path)->subpath, subpaths, append, gather); return; } if (IsA(path, AggPath)) { /* Can see GatherMerge -> Sort -> Partial HashAggregate in parallel plans. */ get_subpaths_from_append_path(castNode(AggPath, path)->subpath, subpaths, append, gather); return; } if (IsA(path, ProjectionPath)) { ProjectionPath *projection = castNode(ProjectionPath, path); get_subpaths_from_append_path(projection->subpath, subpaths, append, gather); return; } /* Aggregation push-down is not supported for other path types so far */ } /* * Copy an AppendPath and set new subpaths. */ static AppendPath * copy_append_path(AppendPath *path, List *subpaths, PathTarget *pathtarget) { AppendPath *newPath = makeNode(AppendPath); memcpy(newPath, path, sizeof(AppendPath)); newPath->subpaths = subpaths; newPath->path.pathtarget = copy_pathtarget(pathtarget); cost_append(newPath); return newPath; } /* * Copy a MergeAppendPath and set new subpaths. */ static MergeAppendPath * copy_merge_append_path(PlannerInfo *root, MergeAppendPath *path, List *subpaths, PathTarget *pathtarget) { MergeAppendPath *newPath = create_merge_append_path(root, path->path.parent, subpaths, path->path.pathkeys, NULL); newPath->path.param_info = path->path.param_info; newPath->path.pathtarget = copy_pathtarget(pathtarget); return newPath; } /* * Copy an append-like path and set new subpaths */ static Path * copy_append_like_path(PlannerInfo *root, Path *path, List *new_subpaths, PathTarget *pathtarget) { if (IsA(path, AppendPath)) { AppendPath *append_path = castNode(AppendPath, path); AppendPath *new_append_path = copy_append_path(append_path, new_subpaths, pathtarget); return &new_append_path->path; } else if (IsA(path, MergeAppendPath)) { MergeAppendPath *merge_append_path = castNode(MergeAppendPath, path); MergeAppendPath *new_merge_append_path = copy_merge_append_path(root, merge_append_path, new_subpaths, pathtarget); return &new_merge_append_path->path; } else if (ts_is_chunk_append_path(path)) { CustomPath *custom_path = castNode(CustomPath, path); ChunkAppendPath *chunk_append_path = (ChunkAppendPath *) custom_path; ChunkAppendPath *new_chunk_append_path = ts_chunk_append_path_copy(chunk_append_path, new_subpaths, pathtarget); return &new_chunk_append_path->cpath.path; } else if (IsA(path, ProjectionPath)) { /* * Projection goes under partial aggregation, so here we can just ignore * it. */ return copy_append_like_path(root, castNode(ProjectionPath, path)->subpath, new_subpaths, pathtarget); } /* Should never happen, already checked by caller */ Ensure(false, "unknown path type"); pg_unreachable(); } /* * Generate a partially sorted aggregated agg path on top of a path */ static AggPath * create_sorted_partial_agg_path(PlannerInfo *root, Path *path, PathTarget *target, double d_num_groups, GroupPathExtraData *extra_data) { Query *parse = root->parse; /* Determine costs for aggregations */ AggClauseCosts *agg_partial_costs = &extra_data->agg_partial_costs; bool is_sorted = pathkeys_contained_in(root->group_pathkeys, path->pathkeys); if (!is_sorted) { path = (Path *) create_sort_path(root, path->parent, path, root->group_pathkeys, -1.0); } AggPath *sorted_agg_path = create_agg_path(root, path->parent, path, target, parse->groupClause ? AGG_SORTED : AGG_PLAIN, AGGSPLIT_INITIAL_SERIAL, #if PG16_LT parse->groupClause, #else root->processed_groupClause, #endif NIL, agg_partial_costs, d_num_groups); return sorted_agg_path; } /* * Generate a partially hashed aggregated add path on top of a path */ static AggPath * create_hashed_partial_agg_path(PlannerInfo *root, Path *path, PathTarget *target, double d_num_groups, GroupPathExtraData *extra_data) { /* Determine costs for aggregations */ AggClauseCosts *agg_partial_costs = &extra_data->agg_partial_costs; AggPath *hash_path = create_agg_path(root, path->parent, path, target, AGG_HASHED, AGGSPLIT_INITIAL_SERIAL, #if PG16_LT root->parse->groupClause, #else root->processed_groupClause, #endif NIL, agg_partial_costs, d_num_groups); return hash_path; } /* * Add partially aggregated subpath */ static void add_partially_aggregated_subpaths(PlannerInfo *root, PathTarget *input_target, PathTarget *partial_grouping_target, double d_num_groups, GroupPathExtraData *extra_data, Path *subpath, List **sorted_paths, List **hashed_paths) { /* Translate targetlist for partition */ AppendRelInfo *appinfo = ts_get_appendrelinfo(root, subpath->parent->relid, false); PathTarget *chunk_grouped_target = copy_pathtarget(partial_grouping_target); chunk_grouped_target->exprs = castNode(List, adjust_appendrel_attrs(root, (Node *) chunk_grouped_target->exprs, /* nappinfos = */ 1, &appinfo)); /* * We might have to project before aggregation. In declarative partitioning * planning, the projection is applied by apply_scanjoin_target_to_path(). */ PathTarget *chunk_target_before_grouping = copy_pathtarget(input_target); chunk_target_before_grouping->exprs = castNode(List, adjust_appendrel_attrs(root, (Node *) chunk_target_before_grouping->exprs, /* nappinfos = */ 1, &appinfo)); /* * Note that we cannot use apply_projection_to_path() here, because it might * modify the targetlist of the projection-capable paths in place, which * would cause a mismatch when these paths are used in another context. * * In case of ColumnarScan path, we can make a copy of it and push the * projection down to it. * * In general, the projection here arises because the pathtarget of the * table scans is determined early based on the reltarget which lists all * used columns in attno order, and the pathtarget before grouping is * computed later and has the grouping columns in front. */ if (ts_is_columnar_scan_path(subpath)) { subpath = (Path *) copy_columnar_scan_path((ColumnarScanPath *) subpath); subpath->pathtarget = chunk_target_before_grouping; } else { subpath = (Path *) create_projection_path(root, subpath->parent, subpath, chunk_target_before_grouping); } if (extra_data->flags & GROUPING_CAN_USE_SORT) { AggPath *agg_path = create_sorted_partial_agg_path(root, subpath, chunk_grouped_target, d_num_groups, extra_data); *sorted_paths = lappend(*sorted_paths, (Path *) agg_path); } if (extra_data->flags & GROUPING_CAN_USE_HASH) { AggPath *agg_path = create_hashed_partial_agg_path(root, subpath, chunk_grouped_target, d_num_groups, extra_data); *hashed_paths = lappend(*hashed_paths, (Path *) agg_path); } } /* * Generate a total aggregation path for partial aggregations. * * The generated paths contain partial aggregations (created by using AGGSPLIT_INITIAL_SERIAL). * These aggregations need to be finished by the caller by adding a node that performs the * AGGSPLIT_FINAL_DESERIAL step. * * The original path can be either parallel or non-parallel aggregation, and the * resulting path will be parallel accordingly. */ static void generate_agg_pushdown_path(PlannerInfo *root, Path *cheapest_total_path, RelOptInfo *input_rel, RelOptInfo *output_rel, RelOptInfo *partially_grouped_rel, PathTarget *grouping_target, PathTarget *partial_grouping_target, double d_num_groups, GroupPathExtraData *extra_data) { /* Get subpaths */ List *subpaths = NIL; Path *top_gather = NULL; Path *top_append = NULL; get_subpaths_from_append_path(cheapest_total_path, &subpaths, &top_append, &top_gather); /* No subpaths available or unsupported append node */ if (subpaths == NIL) { return; } Assert(top_append != NULL); if (list_length(subpaths) < 2) { /* * Doesn't make sense to add per-chunk aggregation paths if there's * only one chunk. */ return; } /* Generate agg paths on top of the append children */ List *sorted_subpaths = NIL; List *hashed_subpaths = NIL; ListCell *lc; foreach (lc, subpaths) { Path *subpath = lfirst(lc); /* Check if we have an append path under an append path (e.g., a partially compressed * chunk. The first append path merges the chunk results. The second append path merges the * uncompressed and the compressed part of the chunk). * * In this case, the partial aggregation needs to be pushed down below the lower * append path. */ List *partially_compressed_paths = NIL; Path *partially_compressed_append = NULL; Path *partially_compressed_gather = NULL; get_subpaths_from_append_path(subpath, &partially_compressed_paths, &partially_compressed_append, &partially_compressed_gather); Assert(partially_compressed_gather == NULL); if (partially_compressed_append != NULL) { List *partially_compressed_sorted = NIL; List *partially_compressed_hashed = NIL; ListCell *lc2; foreach (lc2, partially_compressed_paths) { Path *partially_compressed_path = lfirst(lc2); add_partially_aggregated_subpaths(root, input_rel->reltarget, partial_grouping_target, d_num_groups, extra_data, partially_compressed_path, &partially_compressed_sorted /* Result path */, &partially_compressed_hashed /* Result path */); } if (extra_data->flags & GROUPING_CAN_USE_SORT) { sorted_subpaths = lappend(sorted_subpaths, copy_append_like_path(root, partially_compressed_append, partially_compressed_sorted, partial_grouping_target)); } if (extra_data->flags & GROUPING_CAN_USE_HASH) { hashed_subpaths = lappend(hashed_subpaths, copy_append_like_path(root, partially_compressed_append, partially_compressed_hashed, partial_grouping_target)); } } else { add_partially_aggregated_subpaths(root, input_rel->reltarget, partial_grouping_target, d_num_groups, extra_data, subpath, &sorted_subpaths /* Result paths */, &hashed_subpaths /* Result paths */); } } /* Create new append paths */ if (top_gather == NULL) { /* * The original aggregation plan was non-parallel, so we're creating a * non-parallel plan as well. */ if (sorted_subpaths != NIL) { add_path(partially_grouped_rel, copy_append_like_path(root, top_append, sorted_subpaths, partial_grouping_target)); } if (hashed_subpaths != NIL) { add_path(partially_grouped_rel, copy_append_like_path(root, top_append, hashed_subpaths, partial_grouping_target)); } } else { /* * The cheapest aggregation plan was parallel, so we're creating a * parallel plan as well. */ if (sorted_subpaths != NIL) { add_partial_path(partially_grouped_rel, copy_append_like_path(root, top_append, sorted_subpaths, partial_grouping_target)); } if (hashed_subpaths != NIL) { add_partial_path(partially_grouped_rel, copy_append_like_path(root, top_append, hashed_subpaths, partial_grouping_target)); } } } /* Is the provided path a agg path that uses a sorted or plain agg strategy? */ static bool pg_nodiscard is_path_sorted_or_plain_agg_path(Path *path) { AggPath *agg_path = castNode(AggPath, path); Assert(agg_path->aggstrategy == AGG_SORTED || agg_path->aggstrategy == AGG_PLAIN || agg_path->aggstrategy == AGG_HASHED); return agg_path->aggstrategy == AGG_SORTED || agg_path->aggstrategy == AGG_PLAIN; } /* * Check if this path belongs to a plain or sorted aggregation */ static bool contains_path_plain_or_sorted_agg(Path *path) { List *subpaths = NIL; Path *append = NULL; Path *gather = NULL; get_subpaths_from_append_path(path, &subpaths, &append, &gather); Ensure(subpaths != NIL, "Unable to determine aggregation type"); ListCell *lc; foreach (lc, subpaths) { Path *subpath = lfirst(lc); if (IsA(subpath, AggPath)) return is_path_sorted_or_plain_agg_path(subpath); } /* * No dedicated aggregation nodes found directly underneath the append node. This could be * due to two reasons. * * (1) Only vectorized aggregation is used and we don't have dedicated Aggregation nods. * (2) The query plan uses multi-level appends to keep a certain sorting * - ChunkAppend * - Merge Append * - Agg Chunk 1 * - Agg Chunk 2 * - Merge Append * - Agg Chunk 3 * - Agg Chunk 4 * * in both cases, we use a sorted aggregation node to finalize the partial aggregation and * produce a proper sorting. */ return true; } /* * Replan the aggregation and create a partial aggregation at chunk level and finalize the * aggregation on top of an append node. * * The functionality is inspired by PostgreSQL's create_partitionwise_grouping_paths() function * * Generated aggregation paths: * * Finalize Aggregate * -> Append * -> Partial Aggregation * - Chunk 1 * ... * -> Append of partially compressed chunk 2 * -> Partial Aggregation * -> Scan on uncompressed part of chunk 2 * -> Partial Aggregation * -> Scan on compressed part of chunk 2 * ... * -> Partial Aggregation N * - Chunk N */ void tsl_pushdown_partial_agg(PlannerInfo *root, Hypertable *ht, RelOptInfo *input_rel, RelOptInfo *output_rel, void *extra) { Query *parse = root->parse; /* We are only interested in hypertables */ if (!ht) return; /* Grouping sets are not supported by the partial aggregation pushdown */ if (parse->groupingSets) return; /* Don't replan aggregation if we already have a MinMaxAggPath (e.g., created by * ts_preprocess_first_last_aggregates) */ if (has_min_max_agg_path(output_rel)) return; Assert(extra != NULL); GroupPathExtraData *extra_data = (GroupPathExtraData *) extra; /* Determine the number of groups from the already planned aggregation */ AggPath *existing_agg_path = get_existing_agg_path(output_rel); if (existing_agg_path == NULL) { return; } /* Don't replan aggregation if it contains already partials or non-serializable aggregates */ if (root->hasNonPartialAggs || root->hasNonSerialAggs) return; double d_num_groups = existing_agg_path->numGroups; Assert(d_num_groups > 0); /* Construct partial group agg upper relation */ RelOptInfo *partially_grouped_rel = fetch_upper_rel(root, UPPERREL_PARTIAL_GROUP_AGG, input_rel->relids); partially_grouped_rel->consider_parallel = input_rel->consider_parallel; partially_grouped_rel->consider_startup = input_rel->consider_startup; partially_grouped_rel->reloptkind = input_rel->reloptkind; partially_grouped_rel->serverid = input_rel->serverid; partially_grouped_rel->userid = input_rel->userid; partially_grouped_rel->useridiscurrent = input_rel->useridiscurrent; partially_grouped_rel->fdwroutine = input_rel->fdwroutine; /* Build target list for partial aggregate paths */ PathTarget *grouping_target = output_rel->reltarget; PathTarget *partial_grouping_target = ts_make_partial_grouping_target(root, grouping_target); partially_grouped_rel->reltarget = partial_grouping_target; /* Calculate aggregation costs */ if (!extra_data->partial_costs_set) { /* Init costs */ MemSet(&extra_data->agg_partial_costs, 0, sizeof(AggClauseCosts)); MemSet(&extra_data->agg_final_costs, 0, sizeof(AggClauseCosts)); /* partial phase */ get_agg_clause_costs(root, AGGSPLIT_INITIAL_SERIAL, &extra_data->agg_partial_costs); /* final phase */ get_agg_clause_costs(root, AGGSPLIT_FINAL_DESERIAL, &extra_data->agg_final_costs); extra_data->partial_costs_set = true; } /* * For queries with LIMIT, the aggregated relation can have a path with low * total cost, and a path with low startup cost. We must partialize both, so * loop through the entire pathlist. */ ListCell *lc; foreach (lc, output_rel->pathlist) { Node *path = lfirst(lc); if (!IsA(path, AggPath)) { /* * Shouldn't happen, but here we work with arbitrary paths we don't * control, so it's not an assertion. */ continue; } /* Generate the aggregation pushdown path */ generate_agg_pushdown_path(root, (Path *) path, input_rel, output_rel, partially_grouped_rel, grouping_target, partial_grouping_target, d_num_groups, extra_data); } /* Replan aggregation if we were able to generate partially grouped rel paths */ List *partially_grouped_paths = list_concat(partially_grouped_rel->pathlist, partially_grouped_rel->partial_pathlist); if (partially_grouped_paths == NIL) return; /* Prefer our paths */ output_rel->pathlist = NIL; output_rel->partial_pathlist = NIL; /* * Finalize the created partially aggregated paths by adding a * 'Finalize Aggregate' node on top of them, and adding Sort and Gather * nodes as required. */ AggClauseCosts *agg_final_costs = &extra_data->agg_final_costs; foreach (lc, partially_grouped_paths) { Path *partially_aggregated_path = lfirst(lc); AggStrategy final_strategy; if (contains_path_plain_or_sorted_agg(partially_aggregated_path)) { const bool is_sorted = pathkeys_contained_in(root->group_pathkeys, partially_aggregated_path->pathkeys); if (!is_sorted) { partially_aggregated_path = (Path *) create_sort_path(root, output_rel, partially_aggregated_path, root->group_pathkeys, -1.0); } final_strategy = parse->groupClause ? AGG_SORTED : AGG_PLAIN; } else { final_strategy = AGG_HASHED; } /* * We have to add a Gather or Gather Merge on top of parallel plans. It * goes above the Sort we might have added just before, so that the Sort * is parallelized as well. */ if (partially_aggregated_path->parallel_workers > 0) { double total_groups = partially_aggregated_path->rows * partially_aggregated_path->parallel_workers; if (partially_aggregated_path->pathkeys == NIL) { partially_aggregated_path = (Path *) create_gather_path(root, partially_grouped_rel, partially_aggregated_path, partially_grouped_rel->reltarget, /* required_outer = */ NULL, &total_groups); } else { partially_aggregated_path = (Path *) create_gather_merge_path(root, partially_grouped_rel, partially_aggregated_path, partially_grouped_rel->reltarget, partially_aggregated_path->pathkeys, /* required_outer = */ NULL, &total_groups); } } add_path(output_rel, (Path *) create_agg_path(root, output_rel, partially_aggregated_path, grouping_target, final_strategy, AGGSPLIT_FINAL_DESERIAL, #if PG16_LT parse->groupClause, #else root->processed_groupClause, #endif (List *) parse->havingQual, agg_final_costs, d_num_groups)); } } ================================================ FILE: tsl/src/chunkwise_agg.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/pathnodes.h> #include "export.h" #include "hypertable.h" void tsl_pushdown_partial_agg(PlannerInfo *root, Hypertable *ht, RelOptInfo *input_rel, RelOptInfo *output_rel, void *extra); ================================================ FILE: tsl/src/compression/CMakeLists.txt ================================================ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/api.c ${CMAKE_CURRENT_SOURCE_DIR}/batch_metadata_builder_bloom1.c ${CMAKE_CURRENT_SOURCE_DIR}/batch_metadata_builder_minmax.c ${CMAKE_CURRENT_SOURCE_DIR}/compression.c ${CMAKE_CURRENT_SOURCE_DIR}/compression_dml.c ${CMAKE_CURRENT_SOURCE_DIR}/compression_scankey.c ${CMAKE_CURRENT_SOURCE_DIR}/compression_storage.c ${CMAKE_CURRENT_SOURCE_DIR}/create.c ${CMAKE_CURRENT_SOURCE_DIR}/recompress.c) target_sources(${TSL_LIBRARY_NAME} PRIVATE ${SOURCES}) add_subdirectory(algorithms) ================================================ FILE: tsl/src/compression/README.md ================================================ # Compression Algorithms This is a collection of compression algorithms that are used to compress data of different types. The algorithms are optimized for time-series use-cases; many of them assume that adjacent rows will have "similar" values. ## API Each compression algorithm the API is divided into two parts: a _compressor_ and a _decompression iterator_. The compressor is used to compress new data. - `<algorithm name>_compressor_alloc` - creates the compressor - `<algorithm_name>_compressor_append_null` - appends a null - `<algorithm_name>_compressor_append_value` - appends a non-null value - `<agorithm_name>_compressor_finish` - finalizes the compression and returns the compressed data Data can be read back out using the decompression iterator. An iterator can operate backwards or forwards. There is no random access. The api is - `<algorithm_name>_decompression_iterator_from_datum_<forward|reverse>` - create a new DatumIterator in the forward or reverse direction. - a DatumIterator has a function pointer called `try_next` that returns the next `DecompressResult`. A `DecompressResult` can either be a decompressed value datum, null, or a done marker to indicate that the iterator is done. Each decompression algorithm also contains send and recv function to get the external binary representations. `CompressionAlgorithmDefinition` is a structure that defines function pointers to get forward and reverse iterators as well as send and recv functions. The `definitions` array in `compression.c` contains a `CompressionAlgorithmDefinition` for each compression algorithm. ## Base algorithms The `simple8b rle` algorithm is a building block for many of the compression algorithms. It compresses a series of `uint64` values. It compresses the data by packing the values into the least amount of bits necessary for the magnitude of the int values, using run-length-encoding for large numbers of repeated values, A complete description is in the header file. Note that this is a header-only implementation as performance is paramount here as it is used as a primitive in all the other compression algorithms. ## Compression Algorithms ### DeltaDelta for each integer, it takes the delta-of-deltas with the pervious integer, zigzag encodes this deltadelta, then finally simple8b_rle encodes this zigzagged result. This algorithm performs very well when the magnitude of the delta between adjacent values tends not to vary much, and is optimal for fixed rate-of-change. ### Gorilla `gorilla` encodes floats using the Facebook gorilla algorithm. It stores the compressed xors of adjacent values. It is one of the few simple algorithms that compresses floating point numbers reasonably well. ### Dictionary The dictionary mechanism stores data in two parts: a "dictionary" storing each unique value in the dataset (stored as an array, see below) and simple8b_rle compressed list of indexes into the dictionary, ordered by row. This scheme can store any type of data, but will only be a space improvement if the data set is of relatively low cardinality. ### Array The array "compression" method simply stores the data in an array-like structure and does not actually compress it (though TOAST-based compression can be applied on top). It is the compression mechanism used when no other compression mechanism works. It can store any type of data. ### Bool Compressor The bool compressor is a simple compression algorithm that stores boolean values using the simple8b_rle algorithm only, without any additional processing. During decompression it decompresses the data and stores it in memory as a bitmap. The row based iterators then walk through the bitmap. The bool compressor differs from the other compressors in that it stores the last non-value as a place holder for the null values. This is done to make vectorization easier. ### UUID Compressor The uuid compressor is a compression algorithm that aims at storing UUID v7 values compressed as much as possible by taking advantage of the timestamp values being present in the UUID. The first part of the UUID where the timestamp resides is stored using the delta-delta algorithm. The second part of the UUID is stored without compression, as a sequence of uint64 values. The algorithm checks the cardinality of the values in the compressed batch and based on the cardinality it decides wether it is worth to recompress the batch using the dictionary compression algorithm. In that case it recompresses and stores the UUIDs as a dictionary. # Merging chunks while compressing # ## Setup ## Chunks will be merged during compression if we specify the `compress_chunk_time_interval` parameter. This value will be used to merge chunks adjacent on the time dimension if possible. This allows usage of smaller chunk intervals which are rolled into bigger compressed chunks. ## Operation ## Compression itself is altered by changing the destination compressed chunk from a newly created one to an already existing chunk which satisfies the necessary requirements (is adjacent to the compressed chunk and chunk interval can be increased not to go over compress chunk time interval). After compression completes, catalog is updated by dropping the compressed chunk and increasing the chunk interval of the adjacent chunk to include its time dimension slice. Chunk constraints are updated as necessary. ## Compression setup where time dimension is not the first column on order by ## When merging such chunks, due to the nature of sequence number ordering, we will inherently be left with chunks where the sequence numbers are not correctly ordered. In order to mitigate this issue, chunks are recompressed immediately. This has obvious performance implications which might make merging chunks not optimal for certain setups. # Picking default for `segment_by` and `order_by`. We have two functions to determine the columns for `timescaledb.compress_segmentby` and `timescaledb.compress_orderby` . These functions can be called by the UI to give good defaults. They can also be called internally when a hypertable has compression enabled but no values are provided to specify these options. ## `_timescaledb_functions.get_segmentby_defaults` This function determines a segment-by column to use. It returns a JSONB with the following top-level keys: - columns: an array of column names that should be used for segment by. Right now it always returns a single column. - confidence: a number between 0 and 10 (most confident) indicating how sure we are. - message: a message that should be shown to the user to evaluate the result. The intuition is as follows: we use 3 criterias: - We want to pick an "important" column for querying. We measure "importance", in terms of how early the column comes in an index (i.e. leading columns are very important, others less so). If there are no indexes, all columns will be considered if statistics are populated - The column has many rows for the same column value so that the segments will have many rows. We establish that a column will have many values if (i) it is not a dimension and (ii) either statistics tell us so (via `stadistinct` > 1) or, if statistics aren't populated, we check whether the column is a generated identity or serial column. - When we have multiple qualifying rows, we select the column where rows are spread most evenly across the distinct values. Naturally, statistics give us more confidence that the column has enough rows per segment. In this case we break ties by preferring columns from unique indexes. Otherwise, we prefer columns from non-unique indexes (we are less likely to run into a unique column there). Thus, our preference is based on the whether the column is from a unique or regular index as well as the position of the column in the index. Given these preferences, we think ties happened rarely but will be resolved arbitrarily. One final point: a number of tables don't have any indexed columns that aren't dimensions or serial columns. In this case, we have medium confidence that an empty segment by is correct. ## `_timescaledb_functions.get_orderby_defaults` This function determines which order by columns to use. It returns a JSONB with the following top-level keys: - clauses: an array of column names and sort order key words that shold be used for order by. - confidence: a number between 0 and 10 (most confident) indicating how sure we are. - message: a message that should be shown to the user to evaluate the result. The order by is built in three steps: 1) Use the column order in a unique index (removing the segment_by columns). 2) Add any dimension columns 3) Add the first attribute of any other index (to establish min-max filters on those columns). All non-dimension columns are returned without a sort specifier (thus using `ASC` as default). The dimension columns use `DESC`. ================================================ FILE: tsl/src/compression/algorithms/CMakeLists.txt ================================================ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/array.c ${CMAKE_CURRENT_SOURCE_DIR}/datum_serialize.c ${CMAKE_CURRENT_SOURCE_DIR}/deltadelta.c ${CMAKE_CURRENT_SOURCE_DIR}/dictionary.c ${CMAKE_CURRENT_SOURCE_DIR}/gorilla.c ${CMAKE_CURRENT_SOURCE_DIR}/bool_compress.c ${CMAKE_CURRENT_SOURCE_DIR}/null.c ${CMAKE_CURRENT_SOURCE_DIR}/uuid_compress.c) target_sources(${TSL_LIBRARY_NAME} PRIVATE ${SOURCES}) ================================================ FILE: tsl/src/compression/algorithms/array.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/htup_details.h> #include <access/tupmacs.h> #include <adts/char_vec.h> #include <catalog/namespace.h> #include <catalog/pg_type.h> #include <common/base64.h> #include <funcapi.h> #include <utils/lsyscache.h> #include <utils/syscache.h> #include "array.h" #include "compression/compression.h" #include "datum_serialize.h" #include "guc.h" #include "simple8b_rle.h" #include "simple8b_rle_bitarray.h" #include "simple8b_rle_bitmap.h" #include "compression/arrow_c_data_interface.h" /* A "compressed" array * uint8 has_nulls: 1 iff this has a nulls bitmap stored before the data * Oid element_type: the element stored by this array * simple8b_rle nulls: optional bitmap of nulls within the array * simple8b_rle sizes: the sizes of each data element * char data[]: the elements of the array */ typedef struct ArrayCompressed { CompressedDataHeaderFields; uint8 has_nulls; uint8 padding[6]; Oid element_type; /* 8-byte alignment sentinel for the following fields */ uint64 alignment_sentinel[FLEXIBLE_ARRAY_MEMBER]; } ArrayCompressed; bool array_compressed_has_nulls(const CompressedDataHeader *header) { const ArrayCompressed *ac = (const ArrayCompressed *) header; return ac->has_nulls; } static void pg_attribute_unused() assertions(void) { ArrayCompressed test_val = { .vl_len_ = { 0 } }; Simple8bRleSerialized test_simple8b = { 0 }; /* make sure no padding bytes make it to disk */ StaticAssertStmt(sizeof(ArrayCompressed) == sizeof(test_val.vl_len_) + sizeof(test_val.compression_algorithm) + sizeof(test_val.has_nulls) + sizeof(test_val.padding) + sizeof(test_val.element_type), "ArrayCompressed wrong size"); StaticAssertStmt(sizeof(ArrayCompressed) == 16, "ArrayCompressed wrong size"); /* Note about alignment: the data[] field stores arbitrary Postgres types using store_att_byval * and fetch_att. For this to work the data must be aligned according to the types alignment * parameter (in CREATE TYPE; valid values are 1,2,4,8 bytes). In order to ease implementation, * we simply align the start of data[] on a MAXALIGN (8-byte) boundary. Individual items in the * array are then aligned as specified by the array element type. See top of array.h header in * Postgres source code since it uses the same trick. Thus, we make sure that all fields * before the alignment sentinel are 8-byte aligned, and also that the two Simple8bRleSerialized * elements before the data element are themselves 8-byte aligned as well. */ StaticAssertStmt(offsetof(ArrayCompressed, alignment_sentinel) % MAXIMUM_ALIGNOF == 0, "variable sized data must be 8-byte aligned"); StaticAssertStmt(sizeof(Simple8bRleSerialized) % MAXIMUM_ALIGNOF == 0, "Simple8bRle data must be 8-byte aligned"); StaticAssertStmt(sizeof(test_simple8b.slots[0]) % MAXIMUM_ALIGNOF == 0, "Simple8bRle variable-length slots must be 8-byte aligned"); } typedef struct ArrayCompressedData { Oid element_type; Simple8bRleSerialized *nulls; /* NULL if no nulls */ Simple8bRleSerialized *sizes; const char *data; Size data_len; } ArrayCompressedData; typedef struct ArrayCompressor { Simple8bRleCompressor nulls; Simple8bRleCompressor sizes; char_vec data; Oid type; DatumSerializer *serializer; bool has_nulls; } ArrayCompressor; typedef struct ExtendedCompressor { Compressor base; ArrayCompressor *internal; Oid element_type; } ExtendedCompressor; typedef struct ArrayDecompressionIterator { DecompressionIterator base; Simple8bRleDecompressionIterator nulls; Simple8bRleDecompressionIterator sizes; const char *data; uint32 num_data_bytes; uint32 data_offset; DatumDeserializer *deserializer; bool has_nulls; } ArrayDecompressionIterator; /****************** *** Compressor *** ******************/ static void array_compressor_append_datum(Compressor *compressor, Datum val) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = array_compressor_alloc(extended->element_type); array_compressor_append(extended->internal, val); } static void array_compressor_append_null_value(Compressor *compressor) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = array_compressor_alloc(extended->element_type); array_compressor_append_null(extended->internal); } static bool array_compressor_is_full(Compressor *compressor, Datum val) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = array_compressor_alloc(extended->element_type); Size datum_size_and_align; ArrayCompressor *array_comp = extended->internal; if (datum_serializer_value_may_be_toasted(array_comp->serializer)) val = PointerGetDatum(PG_DETOAST_DATUM_PACKED(val)); datum_size_and_align = datum_get_bytes_size(array_comp->serializer, array_comp->data.num_elements, val) - array_comp->data.num_elements; /* If we can't fit new datum in the max size, we are full */ return (datum_size_and_align + array_comp->data.num_elements) > MAX_ARRAY_COMPRESSOR_SIZE_BYTES; } static void * array_compressor_finish_and_reset(Compressor *compressor) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; void *compressed = array_compressor_finish(extended->internal); pfree(extended->internal); extended->internal = NULL; return compressed; } const Compressor array_compressor = { .append_val = array_compressor_append_datum, .append_null = array_compressor_append_null_value, .is_full = array_compressor_is_full, .finish = array_compressor_finish_and_reset, }; Compressor * array_compressor_for_type(Oid element_type) { ExtendedCompressor *compressor = palloc(sizeof(*compressor)); *compressor = (ExtendedCompressor){ .base = array_compressor, .element_type = element_type, }; return &compressor->base; } ArrayCompressor * array_compressor_alloc(Oid type_to_compress) { ArrayCompressor *compressor = palloc(sizeof(*compressor)); compressor->has_nulls = false; simple8brle_compressor_init(&compressor->nulls); simple8brle_compressor_init(&compressor->sizes); char_vec_init(&compressor->data, CurrentMemoryContext, 0); compressor->type = type_to_compress; compressor->serializer = create_datum_serializer(type_to_compress); return compressor; } void array_compressor_append_null(ArrayCompressor *compressor) { compressor->has_nulls = true; simple8brle_compressor_append(&compressor->nulls, 1); } void array_compressor_append(ArrayCompressor *compressor, Datum val) { Size datum_size_and_align; char *start_ptr; simple8brle_compressor_append(&compressor->nulls, 0); if (datum_serializer_value_may_be_toasted(compressor->serializer)) val = PointerGetDatum(PG_DETOAST_DATUM_PACKED(val)); datum_size_and_align = datum_get_bytes_size(compressor->serializer, compressor->data.num_elements, val) - compressor->data.num_elements; simple8brle_compressor_append(&compressor->sizes, datum_size_and_align); /* datum_to_bytes_and_advance will zero any padding bytes, so we need not do so here */ char_vec_reserve(&compressor->data, datum_size_and_align); start_ptr = compressor->data.data + compressor->data.num_elements; compressor->data.num_elements += datum_size_and_align; datum_to_bytes_and_advance(compressor->serializer, start_ptr, &datum_size_and_align, val); Assert(datum_size_and_align == 0); } typedef struct ArrayCompressorSerializationInfo { Simple8bRleSerialized *sizes; Simple8bRleSerialized *nulls; char_vec data; Size total; } ArrayCompressorSerializationInfo; ArrayCompressorSerializationInfo * array_compressor_get_serialization_info(ArrayCompressor *compressor) { ArrayCompressorSerializationInfo *info = palloc(sizeof(*info)); *info = (ArrayCompressorSerializationInfo){ .sizes = simple8brle_compressor_finish(&compressor->sizes), .nulls = compressor->has_nulls ? simple8brle_compressor_finish(&compressor->nulls) : NULL, .data = compressor->data, .total = 0, }; if (info->nulls != NULL) info->total += simple8brle_serialized_total_size(info->nulls); if (info->sizes != NULL) info->total += simple8brle_serialized_total_size(info->sizes); info->total += compressor->data.num_elements; return info; } Size array_compression_serialization_size(ArrayCompressorSerializationInfo *info) { return info->total; } uint32 array_compression_serialization_num_elements(ArrayCompressorSerializationInfo *info) { CheckCompressedData(info->sizes != NULL); return info->sizes->num_elements; } char * bytes_serialize_array_compressor_and_advance(char *dst, PG_USED_FOR_ASSERTS_ONLY Size dst_size, ArrayCompressorSerializationInfo *info) { uint32 sizes_bytes = simple8brle_serialized_total_size(info->sizes); Assert(dst_size == info->total); if (info->nulls != NULL) { uint32 nulls_bytes = simple8brle_serialized_total_size(info->nulls); Assert(dst_size >= nulls_bytes); dst = bytes_serialize_simple8b_and_advance(dst, nulls_bytes, info->nulls); dst_size -= nulls_bytes; } Assert(dst_size >= sizes_bytes); dst = bytes_serialize_simple8b_and_advance(dst, sizes_bytes, info->sizes); dst_size -= sizes_bytes; Assert(dst_size == info->data.num_elements); memcpy(dst, info->data.data, info->data.num_elements); return dst + info->data.num_elements; } static ArrayCompressed * array_compressed_from_serialization_info(ArrayCompressorSerializationInfo *info, Oid element_type) { char *compressed_data; ArrayCompressed *compressed_array; Size compressed_size = sizeof(ArrayCompressed) + info->total; if (!AllocSizeIsValid(compressed_size)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("compressed size exceeds the maximum allowed (%d)", (int) MaxAllocSize))); compressed_data = palloc0(compressed_size); compressed_array = (ArrayCompressed *) compressed_data; *compressed_array = (ArrayCompressed){ .compression_algorithm = COMPRESSION_ALGORITHM_ARRAY, .has_nulls = info->nulls != NULL, .element_type = element_type, }; SET_VARSIZE(compressed_array->vl_len_, compressed_size); compressed_data += sizeof(ArrayCompressed); compressed_size -= sizeof(ArrayCompressed); bytes_serialize_array_compressor_and_advance(compressed_data, compressed_size, info); return compressed_array; } void * array_compressor_finish(ArrayCompressor *compressor) { ArrayCompressorSerializationInfo *info = array_compressor_get_serialization_info(compressor); if (info->sizes == NULL) return NULL; return array_compressed_from_serialization_info(info, compressor->type); } /****************** *** Decompress *** ******************/ static ArrayCompressedData array_compressed_data_from_bytes(StringInfo serialized_data, Oid element_type, bool has_nulls) { ArrayCompressedData data = { .element_type = element_type }; if (has_nulls) { data.nulls = bytes_deserialize_simple8b_and_advance(serialized_data); } data.sizes = bytes_deserialize_simple8b_and_advance(serialized_data); data.data = serialized_data->data + serialized_data->cursor; data.data_len = serialized_data->len - serialized_data->cursor; return data; } DecompressionIterator * array_decompression_iterator_alloc_forward(StringInfo serialized_data, Oid element_type, bool has_nulls) { ArrayCompressedData data = array_compressed_data_from_bytes(serialized_data, element_type, has_nulls); ArrayDecompressionIterator *iterator = palloc(sizeof(*iterator)); iterator->base.compression_algorithm = COMPRESSION_ALGORITHM_ARRAY; iterator->base.forward = true; iterator->base.element_type = element_type; iterator->base.try_next = array_decompression_iterator_try_next_forward; iterator->has_nulls = data.nulls != NULL; if (iterator->has_nulls) simple8brle_decompression_iterator_init_forward(&iterator->nulls, data.nulls); simple8brle_decompression_iterator_init_forward(&iterator->sizes, data.sizes); iterator->data = data.data; iterator->num_data_bytes = data.data_len; iterator->data_offset = 0; iterator->deserializer = create_datum_deserializer(iterator->base.element_type); return &iterator->base; } DecompressionIterator * tsl_array_decompression_iterator_from_datum_forward(Datum compressed_array, Oid element_type) { void *compressed_data = (void *) PG_DETOAST_DATUM(compressed_array); StringInfoData si = { .data = compressed_data, .len = VARSIZE(compressed_data) }; ArrayCompressed *compressed_array_header = consumeCompressedData(&si, sizeof(ArrayCompressed)); Assert(compressed_array_header->compression_algorithm == COMPRESSION_ALGORITHM_ARRAY); CheckCompressedData(element_type == compressed_array_header->element_type); return array_decompression_iterator_alloc_forward(&si, compressed_array_header->element_type, compressed_array_header->has_nulls == 1); } extern DecompressResult array_decompression_iterator_try_next_forward(DecompressionIterator *general_iter) { Simple8bRleDecompressResult datum_size; ArrayDecompressionIterator *iter; Datum val; const char *start_pointer; Assert(general_iter->compression_algorithm == COMPRESSION_ALGORITHM_ARRAY && general_iter->forward); iter = (ArrayDecompressionIterator *) general_iter; if (iter->has_nulls) { Simple8bRleDecompressResult null = simple8brle_decompression_iterator_try_next_forward(&iter->nulls); if (null.is_done) return (DecompressResult){ .is_done = true, }; if ((null.val & 1) != 0) { return (DecompressResult){ .is_null = true, }; } } datum_size = simple8brle_decompression_iterator_try_next_forward(&iter->sizes); if (datum_size.is_done) return (DecompressResult){ .is_done = true, }; CheckCompressedData(iter->data_offset + datum_size.val <= iter->num_data_bytes); start_pointer = iter->data + iter->data_offset; val = bytes_to_datum_and_advance(iter->deserializer, &start_pointer); iter->data_offset += datum_size.val; CheckCompressedData(iter->data + iter->data_offset == start_pointer); return (DecompressResult){ .val = val, }; } /************************** *** Decompress Reverse *** **************************/ DecompressionIterator * tsl_array_decompression_iterator_from_datum_reverse(Datum compressed_array, Oid element_type) { ArrayCompressed *compressed_array_header; ArrayCompressedData array_compressed_data; ArrayDecompressionIterator *iterator = palloc(sizeof(*iterator)); iterator->base.compression_algorithm = COMPRESSION_ALGORITHM_ARRAY; iterator->base.forward = false; iterator->base.element_type = element_type; iterator->base.try_next = array_decompression_iterator_try_next_reverse; void *compressed_data = PG_DETOAST_DATUM(compressed_array); StringInfoData si = { .data = compressed_data, .len = VARSIZE(compressed_data) }; compressed_array_header = consumeCompressedData(&si, sizeof(ArrayCompressed)); Assert(compressed_array_header->compression_algorithm == COMPRESSION_ALGORITHM_ARRAY); if (element_type != compressed_array_header->element_type) elog(ERROR, "trying to decompress the wrong type"); array_compressed_data = array_compressed_data_from_bytes(&si, compressed_array_header->element_type, compressed_array_header->has_nulls); iterator->has_nulls = array_compressed_data.nulls != NULL; if (iterator->has_nulls) simple8brle_decompression_iterator_init_reverse(&iterator->nulls, array_compressed_data.nulls); simple8brle_decompression_iterator_init_reverse(&iterator->sizes, array_compressed_data.sizes); iterator->data = array_compressed_data.data; iterator->num_data_bytes = array_compressed_data.data_len; iterator->data_offset = iterator->num_data_bytes; iterator->deserializer = create_datum_deserializer(iterator->base.element_type); return &iterator->base; } static ArrowArray *tsl_bool_array_decompress_all(Datum compressed_array, Oid element_type, MemoryContext dest_mctx); static ArrowArray *tsl_text_array_decompress_all(Datum compressed_array, Oid element_type, MemoryContext dest_mctx); static ArrowArray *tsl_uuid_array_decompress_all(Datum compressed_array, Oid element_type, MemoryContext dest_mctx); /* Pass through to the specialized functions below for BOOL and TEXT */ ArrowArray * tsl_array_decompress_all(Datum compressed_array, Oid element_type, MemoryContext dest_mctx) { switch (element_type) { case BOOLOID: return tsl_bool_array_decompress_all(compressed_array, element_type, dest_mctx); case TEXTOID: return tsl_text_array_decompress_all(compressed_array, element_type, dest_mctx); case UUIDOID: return tsl_uuid_array_decompress_all(compressed_array, element_type, dest_mctx); default: elog(ERROR, "unsupported array type %u for bulk decompression", element_type); break; } return NULL; } static ArrowArray * tsl_bool_array_decompress_all(Datum compressed_array, Oid element_type, MemoryContext dest_mctx) { Assert(element_type == BOOLOID); void *compressed_data = PG_DETOAST_DATUM(compressed_array); StringInfoData si = { .data = compressed_data, .len = VARSIZE(compressed_data) }; ArrayCompressed *header = consumeCompressedData(&si, sizeof(ArrayCompressed)); Assert(header->compression_algorithm == COMPRESSION_ALGORITHM_ARRAY); CheckCompressedData(header->element_type == BOOLOID); Simple8bRleSerialized *nulls_serialized = NULL; if (header->has_nulls) { nulls_serialized = bytes_deserialize_simple8b_and_advance(&si); } Simple8bRleSerialized *sizes_serialized = bytes_deserialize_simple8b_and_advance(&si); const uint32 n_notnull = sizes_serialized->num_elements; const uint32 n_total = header->has_nulls ? nulls_serialized->num_elements : n_notnull; const uint32 n_padded_bits = n_total + 63; const uint32 n_padded_bytes = n_padded_bits / 8; uint64 *validity_bitmap = NULL; uint64 *values = MemoryContextAllocZero(dest_mctx, n_padded_bytes); MemoryContext old_context = MemoryContextSwitchTo(dest_mctx); /* Decompress the nulls */ Simple8bRleBitArray validity_bits = simple8brle_bitarray_decompress(nulls_serialized, /* inverted*/ true); validity_bitmap = validity_bits.data; MemoryContextSwitchTo(old_context); /* Decompress the values using the iterator based decompressor */ { int position = 0; DecompressionIterator *iter = tsl_array_decompression_iterator_from_datum_forward(PointerGetDatum(compressed_data), BOOLOID); for (DecompressResult r = array_decompression_iterator_try_next_forward(iter); !r.is_done; r = array_decompression_iterator_try_next_forward(iter)) { if (!r.is_null) { bool data = DatumGetBool(r.val) == true; if (data) { arrow_set_row_validity(values, position, true); } } ++position; } } ArrowArray *result = MemoryContextAllocZero(dest_mctx, sizeof(ArrowArray) + (sizeof(void *) * 2)); const void **buffers = (const void **) &result[1]; buffers[0] = validity_bitmap; buffers[1] = values; result->n_buffers = 2; result->buffers = buffers; result->length = n_total; result->null_count = n_total - n_notnull; return result; } static ArrowArray * tsl_uuid_array_decompress_all(Datum compressed_array, Oid element_type, MemoryContext dest_mctx) { Assert(element_type == UUIDOID); void *compressed_data = PG_DETOAST_DATUM(compressed_array); StringInfoData si = { .data = compressed_data, .len = VARSIZE_ANY(compressed_data) }; ArrayCompressed *header = consumeCompressedData(&si, sizeof(ArrayCompressed)); Assert(header->compression_algorithm == COMPRESSION_ALGORITHM_ARRAY); CheckCompressedData(header->element_type == UUIDOID); Simple8bRleSerialized *nulls_serialized = NULL; if (header->has_nulls) { nulls_serialized = bytes_deserialize_simple8b_and_advance(&si); } Simple8bRleSerialized *sizes_serialized = bytes_deserialize_simple8b_and_advance(&si); const uint32 n_notnull = sizes_serialized->num_elements; /* the nulls_serialized test shouldn't be necessary, but clang needs this to pass without * warnings */ const uint32 n_total = (header->has_nulls && nulls_serialized != NULL) ? nulls_serialized->num_elements : n_notnull; const uint32 n_bytes = n_total * 16; const uint64 *restrict compressed_non_null_values = (const uint64 *) (si.data + si.cursor); CheckCompressedData(n_notnull * 16 <= (uint32) (si.len - si.cursor)); uint64 *restrict validity_bitmap = NULL; uint64 *restrict values = MemoryContextAlloc(dest_mctx, n_bytes); MemoryContext old_context = MemoryContextSwitchTo(dest_mctx); /* Decompress the nulls */ Simple8bRleBitArray validity_bits = simple8brle_bitarray_decompress(nulls_serialized, /* inverted*/ true); validity_bitmap = validity_bits.data; MemoryContextSwitchTo(old_context); /* Check the alignment of compressed_non_null_values */ if (((uintptr_t) compressed_non_null_values % 8) == 0) { int position = 0; for (uint32 i = 0; i < n_total; i++) { if (arrow_row_is_valid(validity_bitmap, i)) { /* Copy the 16 bytes of the UUID with simple assignment, because we know it is * aligned */ values[i * 2] = compressed_non_null_values[position * 2]; values[i * 2 + 1] = compressed_non_null_values[position * 2 + 1]; position++; } } } else { int position = 0; for (uint32 i = 0; i < n_total; i++) { if (arrow_row_is_valid(validity_bitmap, i)) { /* Copy the 16 bytes of the UUID with memcpy */ memcpy(&values[i * 2], &compressed_non_null_values[position * 2], 16); position++; } } } ArrowArray *result = MemoryContextAllocZero(dest_mctx, sizeof(ArrowArray) + (sizeof(void *) * 2)); const void **buffers = (const void **) &result[1]; buffers[0] = validity_bitmap; buffers[1] = values; result->n_buffers = 2; result->buffers = buffers; result->length = n_total; result->null_count = n_total - n_notnull; return result; } #define ELEMENT_TYPE uint32 #include "simple8b_rle_decompress_all.h" #undef ELEMENT_TYPE static ArrowArray * tsl_text_array_decompress_all(Datum compressed_array, Oid element_type, MemoryContext dest_mctx) { Assert(element_type == TEXTOID); void *compressed_data = PG_DETOAST_DATUM(compressed_array); StringInfoData si = { .data = compressed_data, .len = VARSIZE(compressed_data) }; ArrayCompressed *header = consumeCompressedData(&si, sizeof(ArrayCompressed)); Assert(header->compression_algorithm == COMPRESSION_ALGORITHM_ARRAY); CheckCompressedData(header->element_type == TEXTOID); return text_array_decompress_all_serialized_no_header(&si, header->has_nulls, dest_mctx); } ArrowArray * text_array_decompress_all_serialized_no_header(StringInfo si, bool has_nulls, MemoryContext dest_mctx) { Simple8bRleSerialized *nulls_serialized = NULL; if (has_nulls) { nulls_serialized = bytes_deserialize_simple8b_and_advance(si); } Simple8bRleSerialized *sizes_serialized = bytes_deserialize_simple8b_and_advance(si); uint32 n_notnull; const uint32 *sizes = simple8brle_decompress_all_uint32(sizes_serialized, &n_notnull); const uint32 n_total = has_nulls ? nulls_serialized->num_elements : n_notnull; CheckCompressedData(n_total >= n_notnull); uint32 *offsets = (uint32 *) MemoryContextAlloc(dest_mctx, pad_to_multiple(64, sizeof(*offsets) * (n_total + 1))); uint8 *arrow_bodies = (uint8 *) MemoryContextAlloc(dest_mctx, pad_to_multiple(64, si->len - si->cursor)); uint32 offset = 0; for (uint32 i = 0; i < n_notnull; i++) { const void *unaligned = consumeCompressedData(si, sizes[i]); /* * We start reading from the end of previous datum, but this pointer * might be not aligned as required for varlena-4b struct. We have to * align it here. Note that sizes[i] includes the alignment as well in * addition to the varlena size. * * See the corresponding row-by-row code in bytes_to_datum_and_advance(). */ const void *vardata = DatumGetPointer(att_align_pointer(unaligned, TYPALIGN_INT, -1, unaligned)); /* * Check for potentially corrupt varlena headers since we're reading them * directly from compressed data. */ if (VARATT_IS_4B_U(vardata)) { /* * Full varsize must be larger or equal than the header size so that * the calculation of size without header doesn't overflow. */ CheckCompressedData(VARSIZE_4B(vardata) >= VARHDRSZ); } else if (VARATT_IS_1B(vardata)) { /* Can't have a TOAST pointer here. */ CheckCompressedData(!VARATT_IS_1B_E(vardata)); /* * Full varsize must be larger or equal than the header size so that * the calculation of size without header doesn't overflow. */ CheckCompressedData(VARSIZE_1B(vardata) >= VARHDRSZ_SHORT); } else { /* * Can only have an uncompressed datum with 1-byte or 4-byte header * here, no TOAST or compressed data. */ CheckCompressedData(false); } /* * Size of varlena plus alignment must match the size stored in the * sizes array for this element. */ const Datum alignment_bytes = PointerGetDatum(vardata) - PointerGetDatum(unaligned); CheckCompressedData(VARSIZE_ANY(vardata) + alignment_bytes == sizes[i]); const uint32 textlen = VARSIZE_ANY_EXHDR(vardata); memcpy(&arrow_bodies[offset], VARDATA_ANY(vardata), textlen); offsets[i] = offset; CheckCompressedData(offset <= offset + textlen); /* Check for overflow. */ offset += textlen; } offsets[n_notnull] = offset; uint64 *restrict validity_bitmap = NULL; if (has_nulls) { const int validity_bitmap_bytes = sizeof(uint64) * (pad_to_multiple(64, n_total) / 64); validity_bitmap = MemoryContextAlloc(dest_mctx, validity_bitmap_bytes); /* * First, mark all data as valid, we will fill the nulls later if needed. * Note that the validity bitmap size is a multiple of 64 bits. We have to * fill the tail bits with zeros, because the corresponding elements are not * valid. * */ memset(validity_bitmap, 0xFF, validity_bitmap_bytes); if (n_total % 64) { const uint64 tail_mask = ~0ULL >> (64 - n_total % 64); validity_bitmap[n_total / 64] &= tail_mask; } /* * We have decompressed the data with nulls skipped, reshuffle it * according to the nulls bitmap. */ const Simple8bRleBitmap nulls = simple8brle_bitmap_decompress(nulls_serialized); CheckCompressedData(n_notnull + simple8brle_bitmap_num_ones(&nulls) == n_total); int current_notnull_element = n_notnull - 1; for (int i = n_total - 1; i >= 0; i--) { Assert(i >= current_notnull_element); /* * The index of the corresponding offset is higher by one than * the index of the element. The offset[0] is never affected by * this shuffling and is always 0. * Note that unlike the usual null reshuffling in other algorithms, * for offsets, even if all elements are null, the starting offset * is well-defined and we can do this assignment. This case is only * accessible through fuzzing. Through SQL, all-null batches result * in a null compressed value. */ Assert(current_notnull_element + 1 >= 0); offsets[i + 1] = offsets[current_notnull_element + 1]; if (simple8brle_bitmap_get_at(&nulls, i)) { arrow_set_row_validity(validity_bitmap, i, false); } else { Assert(current_notnull_element >= 0); current_notnull_element--; } } Assert(current_notnull_element == -1); } ArrowArray *result = MemoryContextAllocZero(dest_mctx, sizeof(ArrowArray) + (sizeof(void *) * 3)); const void **buffers = (const void **) &result[1]; buffers[0] = validity_bitmap; buffers[1] = offsets; buffers[2] = arrow_bodies; result->n_buffers = 3; result->buffers = buffers; result->length = n_total; result->null_count = n_total - n_notnull; return result; } DecompressResult array_decompression_iterator_try_next_reverse(DecompressionIterator *base_iter) { Simple8bRleDecompressResult datum_size; ArrayDecompressionIterator *iter; Datum val; const char *start_pointer; Assert(base_iter->compression_algorithm == COMPRESSION_ALGORITHM_ARRAY && !base_iter->forward); iter = (ArrayDecompressionIterator *) base_iter; if (iter->has_nulls) { Simple8bRleDecompressResult null = simple8brle_decompression_iterator_try_next_reverse(&iter->nulls); if (null.is_done) return (DecompressResult){ .is_done = true, }; if ((null.val & 1) != 0) { return (DecompressResult){ .is_null = true, }; } } datum_size = simple8brle_decompression_iterator_try_next_reverse(&iter->sizes); if (datum_size.is_done) return (DecompressResult){ .is_done = true, }; Assert((int64) iter->data_offset - (int64) datum_size.val >= 0); iter->data_offset -= datum_size.val; start_pointer = iter->data + iter->data_offset; val = bytes_to_datum_and_advance(iter->deserializer, &start_pointer); return (DecompressResult){ .val = val, }; } /********************* *** send / recv *** *********************/ ArrayCompressorSerializationInfo * array_compressed_data_recv(StringInfo buffer, Oid element_type) { ArrayCompressor *compressor = array_compressor_alloc(element_type); Simple8bRleDecompressionIterator nulls; uint8 has_nulls; DatumDeserializer *deser = create_datum_deserializer(element_type); bool use_binary_recv; uint32 num_elements; uint32 i; has_nulls = pq_getmsgbyte(buffer) != 0; if (has_nulls) simple8brle_decompression_iterator_init_forward(&nulls, simple8brle_serialized_recv(buffer)); use_binary_recv = pq_getmsgbyte(buffer) != 0; /* This is actually the number of not-null elements */ num_elements = pq_getmsgint32(buffer); /* if there are nulls, use that count instead */ if (has_nulls) num_elements = nulls.num_elements; for (i = 0; i < num_elements; i++) { Datum val; if (has_nulls) { Simple8bRleDecompressResult null = simple8brle_decompression_iterator_try_next_forward(&nulls); Assert(!null.is_done); if (null.val) { array_compressor_append_null(compressor); continue; } } val = binary_string_to_datum(deser, use_binary_recv ? BINARY_ENCODING : TEXT_ENCODING, buffer); array_compressor_append(compressor, val); } return array_compressor_get_serialization_info(compressor); } void array_compressed_data_send(StringInfo buffer, const char *_serialized_data, Size _data_size, Oid element_type, bool has_nulls) { DecompressResult datum; DatumSerializer *serializer = create_datum_serializer(element_type); BinaryStringEncoding encoding = datum_serializer_binary_string_encoding(serializer); StringInfoData si = { .data = (char *) _serialized_data, .len = _data_size }; ArrayCompressedData array_compressed_data = array_compressed_data_from_bytes(&si, element_type, has_nulls); si.cursor = 0; DecompressionIterator *data_iter = array_decompression_iterator_alloc_forward(&si, element_type, has_nulls); pq_sendbyte(buffer, array_compressed_data.nulls != NULL); if (array_compressed_data.nulls != NULL) simple8brle_serialized_send(buffer, array_compressed_data.nulls); pq_sendbyte(buffer, encoding == BINARY_ENCODING); /* * we do not send data.sizes because the sizes need not be the same once * deserialized, and we will need to recalculate them on recv. We do need * to send the number of elements, which is always the same as the number * of sizes. */ pq_sendint32(buffer, array_compressed_data.sizes->num_elements); for (datum = array_decompression_iterator_try_next_forward(data_iter); !datum.is_done; datum = array_decompression_iterator_try_next_forward(data_iter)) { if (datum.is_null) continue; datum_append_to_binary_string(serializer, encoding, buffer, datum.val); } } /******************** *** SQL Bindings *** ********************/ Datum array_compressed_recv(StringInfo buffer) { uint8 has_nulls; Oid element_type; has_nulls = pq_getmsgbyte(buffer); CheckCompressedData(has_nulls == 0 || has_nulls == 1); element_type = binary_string_get_type(buffer); ArrayCompressorSerializationInfo *info = array_compressed_data_recv(buffer, element_type); CheckCompressedData(info->sizes != NULL); CheckCompressedData(has_nulls == (info->nulls != NULL)); PG_RETURN_POINTER(array_compressed_from_serialization_info(info, element_type)); } void array_compressed_send(CompressedDataHeader *header, StringInfo buffer) { const char *compressed_data = (char *) header; uint32 data_size; ArrayCompressed *compressed_array_header; Assert(header->compression_algorithm == COMPRESSION_ALGORITHM_ARRAY); compressed_array_header = (ArrayCompressed *) header; compressed_data += sizeof(*compressed_array_header); data_size = VARSIZE(compressed_array_header); data_size -= sizeof(*compressed_array_header); pq_sendbyte(buffer, compressed_array_header->has_nulls == true); type_append_to_binary_string(compressed_array_header->element_type, buffer); array_compressed_data_send(buffer, compressed_data, data_size, compressed_array_header->element_type, compressed_array_header->has_nulls); } extern Datum tsl_array_compressor_append(PG_FUNCTION_ARGS) { ArrayCompressor *compressor = (ArrayCompressor *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0)); MemoryContext agg_context; MemoryContext old_context; if (!AggCheckCallContext(fcinfo, &agg_context)) { /* cannot be called directly because of internal-type argument */ elog(ERROR, "tsl_array_compressor_append called in non-aggregate context"); } old_context = MemoryContextSwitchTo(agg_context); if (compressor == NULL) { Oid type_to_compress = get_fn_expr_argtype(fcinfo->flinfo, 1); compressor = array_compressor_alloc(type_to_compress); } if (PG_ARGISNULL(1)) array_compressor_append_null(compressor); else array_compressor_append(compressor, PG_GETARG_DATUM(1)); MemoryContextSwitchTo(old_context); PG_RETURN_POINTER(compressor); } extern Datum tsl_array_compressor_finish(PG_FUNCTION_ARGS) { ArrayCompressor *compressor = (ArrayCompressor *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0)); void *compressed; if (compressor == NULL) PG_RETURN_NULL(); compressed = array_compressor_finish(compressor); if (compressed == NULL) PG_RETURN_NULL(); PG_RETURN_POINTER(compressed); } ================================================ FILE: tsl/src/compression/algorithms/array.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once /* * The `array` compression method can store any type of data. It simply puts it into an * array-like structure and does not compress it. TOAST-based compression should be applied on top. * * Array compression is are also used as a building block for dictionary compression. */ #include <postgres.h> #include <fmgr.h> #include <utils/memutils.h> #include "compression/compression.h" #define MAX_ARRAY_COMPRESSOR_SIZE_BYTES MaxAllocSize typedef struct StringInfoData StringInfoData; typedef StringInfoData *StringInfo; typedef struct ArrayCompressor ArrayCompressor; typedef struct ArrayCompressed ArrayCompressed; typedef struct ArrayDecompressionIterator ArrayDecompressionIterator; extern const Compressor array_compressor; extern bool array_compressed_has_nulls(const CompressedDataHeader *header); extern Compressor *array_compressor_for_type(Oid element_type); extern ArrayCompressor *array_compressor_alloc(Oid type_to_compress); extern void array_compressor_append_null(ArrayCompressor *compressor); extern void array_compressor_append(ArrayCompressor *compressor, Datum val); extern void *array_compressor_finish(ArrayCompressor *compressor); extern ArrayDecompressionIterator *array_decompression_iterator_alloc(void); extern DecompressionIterator * tsl_array_decompression_iterator_from_datum_forward(Datum compressed_array, Oid element_type); extern DecompressResult array_decompression_iterator_try_next_forward(DecompressionIterator *iter); extern DecompressionIterator * tsl_array_decompression_iterator_from_datum_reverse(Datum compressed_array, Oid element_type); extern DecompressResult array_decompression_iterator_try_next_reverse(DecompressionIterator *iter); /* API for using this as an embedded data structure */ typedef struct ArrayCompressorSerializationInfo ArrayCompressorSerializationInfo; extern ArrayCompressorSerializationInfo * array_compressor_get_serialization_info(ArrayCompressor *compressor); Size array_compression_serialization_size(ArrayCompressorSerializationInfo *info); uint32 array_compression_serialization_num_elements(ArrayCompressorSerializationInfo *info); extern char *bytes_serialize_array_compressor_and_advance(char *dst, Size dst_size, ArrayCompressorSerializationInfo *info); extern DecompressionIterator *array_decompression_iterator_alloc_forward(StringInfo serialized_data, Oid element_type, bool has_nulls); extern ArrayCompressorSerializationInfo *array_compressed_data_recv(StringInfo buffer, Oid element_type); extern void array_compressed_data_send(StringInfo buffer, const char *serialized_data, Size data_size, Oid element_type, bool has_nulls); extern Datum array_compressed_recv(StringInfo buffer); extern void array_compressed_send(CompressedDataHeader *header, StringInfo buffer); extern Datum tsl_array_compressor_append(PG_FUNCTION_ARGS); extern Datum tsl_array_compressor_finish(PG_FUNCTION_ARGS); /* Pass through to the specialized functions below for BOOL and TEXT */ ArrowArray *tsl_array_decompress_all(Datum compressed_array, Oid element_type, MemoryContext dest_mctx); ArrowArray *text_array_decompress_all_serialized_no_header(StringInfo si, bool has_nulls, MemoryContext dest_mctx); #define ARRAY_ALGORITHM_DEFINITION \ { \ .iterator_init_forward = tsl_array_decompression_iterator_from_datum_forward, \ .iterator_init_reverse = tsl_array_decompression_iterator_from_datum_reverse, \ .compressed_data_send = array_compressed_send, \ .compressed_data_recv = array_compressed_recv, \ .compressor_for_type = array_compressor_for_type, \ .compressed_data_storage = TOAST_STORAGE_EXTENDED, \ .decompress_all = tsl_array_decompress_all, \ } ================================================ FILE: tsl/src/compression/algorithms/bool_compress.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include "bool_compress.h" #include "compression/arrow_c_data_interface.h" #include "compression/compression.h" #include "guc.h" #include "simple8b_rle.h" #include "simple8b_rle_bitarray.h" #include "simple8b_rle_bitmap.h" typedef struct BoolCompressed { CompressedDataHeaderFields; uint8 has_nulls; /* 1 if this has a NULLs bitmap after the values, 0 otherwise */ uint8 padding[2]; /* padding added because of Simple8bRleSerialized format */ char values[FLEXIBLE_ARRAY_MEMBER]; } BoolCompressed; typedef struct BoolDecompressionIterator { DecompressionIterator base; Simple8bRleBitmap values; Simple8bRleBitmap validity_bitmap; int32 position; } BoolDecompressionIterator; typedef struct BoolCompressor { Simple8bRleCompressor values; Simple8bRleCompressor validity_bitmap; bool has_nulls; bool last_value; uint32 num_nulls; } BoolCompressor; typedef struct ExtendedCompressor { Compressor base; BoolCompressor *internal; } ExtendedCompressor; /* * Local helpers */ static void bool_compressor_append_bool(Compressor *compressor, Datum val); static void bool_compressor_append_null_value(Compressor *compressor); static void *bool_compressor_finish_and_reset(Compressor *compressor); const Compressor bool_compressor_initializer = { .append_val = bool_compressor_append_bool, .append_null = bool_compressor_append_null_value, .is_full = NULL, .finish = bool_compressor_finish_and_reset, }; static BoolCompressed *bool_compressed_from_parts(Simple8bRleSerialized *values, Simple8bRleSerialized *validity_bitmap); static void decompression_iterator_init(BoolDecompressionIterator *iter, void *compressed, Oid element_type, bool forward); /* * Compressor framework functions and definitions for the bool_compress algorithm. */ extern BoolCompressor * bool_compressor_alloc(void) { BoolCompressor *compressor = palloc0(sizeof(*compressor)); simple8brle_compressor_init(&compressor->values); simple8brle_compressor_init(&compressor->validity_bitmap); return compressor; } extern void bool_compressor_append_null(BoolCompressor *compressor) { /* * We use parallel bitmaps of same size for validity and values, to support * zero-copy decompression into ArrowArray. When an element is null, * the particular value that goes into the values bitmap doesn't matter, so * we add the last seen value, not to break the RLE sequences. */ compressor->has_nulls = true; simple8brle_compressor_append(&compressor->values, compressor->last_value); simple8brle_compressor_append(&compressor->validity_bitmap, 0); compressor->num_nulls++; } extern void bool_compressor_append_value(BoolCompressor *compressor, bool next_val) { compressor->last_value = next_val; simple8brle_compressor_append(&compressor->values, next_val); simple8brle_compressor_append(&compressor->validity_bitmap, 1); } extern void * bool_compressor_finish(BoolCompressor *compressor) { if (compressor == NULL) return NULL; Simple8bRleSerialized *values = simple8brle_compressor_finish(&compressor->values); if (values == NULL) return NULL; if (compressor->num_nulls == compressor->values.num_elements) return NULL; Simple8bRleSerialized *validity_bitmap = simple8brle_compressor_finish(&compressor->validity_bitmap); BoolCompressed *compressed; compressed = bool_compressed_from_parts(values, compressor->has_nulls ? validity_bitmap : NULL); /* When only nulls are present, we can return NULL */ Assert(compressed == NULL || compressed->compression_algorithm == COMPRESSION_ALGORITHM_BOOL); return compressed; } extern bool bool_compressed_has_nulls(const CompressedDataHeader *header) { const BoolCompressed *ddc = (const BoolCompressed *) header; return ddc->has_nulls; } extern DecompressResult bool_decompression_iterator_try_next_forward(DecompressionIterator *iter) { Assert(iter->compression_algorithm == COMPRESSION_ALGORITHM_BOOL && iter->forward); Assert(iter->element_type == BOOLOID); BoolDecompressionIterator *bool_iter = (BoolDecompressionIterator *) iter; if (bool_iter->position >= bool_iter->values.num_elements) return (DecompressResult){ .is_done = true, }; /* check nulls */ if (bool_iter->validity_bitmap.num_elements > 0) { bool is_null = !simple8brle_bitmap_get_at(&bool_iter->validity_bitmap, bool_iter->position); if (is_null) { bool_iter->position++; return (DecompressResult){ .is_null = true, }; } } bool val = simple8brle_bitmap_get_at(&bool_iter->values, bool_iter->position); bool_iter->position++; return (DecompressResult){ .val = BoolGetDatum(val), }; } extern DecompressionIterator * bool_decompression_iterator_from_datum_forward(Datum bool_compressed, Oid element_type) { BoolDecompressionIterator *iterator = palloc(sizeof(*iterator)); CheckCompressedData(DatumGetPointer(bool_compressed) != NULL); decompression_iterator_init(iterator, (void *) PG_DETOAST_DATUM(bool_compressed), element_type, true); return &iterator->base; } extern DecompressResult bool_decompression_iterator_try_next_reverse(DecompressionIterator *iter) { Assert(iter->compression_algorithm == COMPRESSION_ALGORITHM_BOOL && !iter->forward); Assert(iter->element_type == BOOLOID); BoolDecompressionIterator *bool_iter = (BoolDecompressionIterator *) iter; if (bool_iter->position < 0) return (DecompressResult){ .is_done = true, }; /* check nulls */ if (bool_iter->validity_bitmap.num_elements > 0) { bool is_null = !simple8brle_bitmap_get_at(&bool_iter->validity_bitmap, bool_iter->position); if (is_null) { bool_iter->position--; return (DecompressResult){ .is_null = true, }; } } bool val = simple8brle_bitmap_get_at(&bool_iter->values, bool_iter->position); bool_iter->position--; return (DecompressResult){ .val = BoolGetDatum(val), }; } extern DecompressionIterator * bool_decompression_iterator_from_datum_reverse(Datum bool_compressed, Oid element_type) { BoolDecompressionIterator *iterator = palloc(sizeof(*iterator)); CheckCompressedData(DatumGetPointer(bool_compressed) != NULL); decompression_iterator_init(iterator, (void *) PG_DETOAST_DATUM(bool_compressed), element_type, false); return &iterator->base; } extern void bool_compressed_send(CompressedDataHeader *header, StringInfo buffer) { const BoolCompressed *data = (BoolCompressed *) header; Assert(header->compression_algorithm == COMPRESSION_ALGORITHM_BOOL); pq_sendbyte(buffer, data->has_nulls); simple8brle_serialized_send(buffer, (Simple8bRleSerialized *) data->values); if (data->has_nulls) { Simple8bRleSerialized *validity_bitmap = (Simple8bRleSerialized *) (((char *) data->values) + simple8brle_serialized_total_size( (Simple8bRleSerialized *) data->values)); simple8brle_serialized_send(buffer, validity_bitmap); } } extern Datum bool_compressed_recv(StringInfo buffer) { uint8 has_nulls; Simple8bRleSerialized *values; Simple8bRleSerialized *validity_bitmap = NULL; BoolCompressed *compressed; has_nulls = pq_getmsgbyte(buffer); CheckCompressedData(has_nulls == 0 || has_nulls == 1); values = simple8brle_serialized_recv(buffer); if (has_nulls) validity_bitmap = simple8brle_serialized_recv(buffer); compressed = bool_compressed_from_parts(values, validity_bitmap); PG_RETURN_POINTER(compressed); } extern Compressor * bool_compressor_for_type(Oid element_type) { ExtendedCompressor *compressor = palloc(sizeof(*compressor)); switch (element_type) { case BOOLOID: *compressor = (ExtendedCompressor){ .base = bool_compressor_initializer }; return &compressor->base; default: elog(ERROR, "invalid type for bool compressor \"%s\"", format_type_be(element_type)); } pg_unreachable(); } /* * Cross-module functions for the bool_compress algorithm. */ extern Datum tsl_bool_compressor_append(PG_FUNCTION_ARGS) { MemoryContext old_context; MemoryContext agg_context; BoolCompressor *compressor = (BoolCompressor *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0)); if (!AggCheckCallContext(fcinfo, &agg_context)) { /* cannot be called directly because of internal-type argument */ elog(ERROR, "tsl_bool_compressor_append called in non-aggregate context"); } old_context = MemoryContextSwitchTo(agg_context); if (compressor == NULL) { compressor = bool_compressor_alloc(); if (PG_NARGS() > 2) elog(ERROR, "append expects two arguments"); } if (PG_ARGISNULL(1)) bool_compressor_append_null(compressor); else { bool next_val = PG_GETARG_BOOL(1); bool_compressor_append_value(compressor, next_val); } MemoryContextSwitchTo(old_context); PG_RETURN_POINTER(compressor); } extern Datum tsl_bool_compressor_finish(PG_FUNCTION_ARGS) { BoolCompressor *compressor = PG_ARGISNULL(0) ? NULL : (BoolCompressor *) PG_GETARG_POINTER(0); void *compressed; if (compressor == NULL) PG_RETURN_NULL(); compressed = bool_compressor_finish(compressor); if (compressed == NULL) PG_RETURN_NULL(); PG_RETURN_POINTER(compressed); } extern ArrowArray * bool_decompress_all(Datum compressed, Oid element_type, MemoryContext dest_mctx) { MemoryContext old_context; Simple8bRleBitArray value_bits; Simple8bRleBitArray validity_bits; Simple8bRleSerialized *serialized_values = NULL; Simple8bRleSerialized *serialized_validity_bitmap = NULL; ArrowArray *result = NULL; uint64 *validity_bitmap = NULL; uint64 *decompressed_values = NULL; CheckCompressedData(DatumGetPointer(compressed) != NULL); void *detoasted = PG_DETOAST_DATUM(compressed); StringInfoData si = { .data = detoasted, .len = VARSIZE(compressed) }; BoolCompressed *header = consumeCompressedData(&si, sizeof(BoolCompressed)); Assert(header->has_nulls == 0 || header->has_nulls == 1); Assert(element_type == BOOLOID); serialized_values = bytes_deserialize_simple8b_and_advance(&si); const bool has_nulls = header->has_nulls == 1; if (has_nulls) { serialized_validity_bitmap = bytes_deserialize_simple8b_and_advance(&si); } /* Decompress the values directly to bit arrays */ old_context = MemoryContextSwitchTo(dest_mctx); value_bits = simple8brle_bitarray_decompress(serialized_values, /* inverted*/ false); decompressed_values = value_bits.data; validity_bits = simple8brle_bitarray_decompress(serialized_validity_bitmap, /* inverted*/ false); validity_bitmap = validity_bits.data; MemoryContextSwitchTo(old_context); result = MemoryContextAllocZero(dest_mctx, sizeof(ArrowArray) + sizeof(void *) * 2); const void **buffers = (const void **) &result[1]; buffers[0] = validity_bitmap; buffers[1] = decompressed_values; result->n_buffers = 2; result->buffers = buffers; result->length = value_bits.num_elements; result->null_count = has_nulls ? (result->length - validity_bits.num_ones) : 0; return result; } /* * Local helpers */ static void bool_compressor_append_bool(Compressor *compressor, Datum val) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = bool_compressor_alloc(); bool_compressor_append_value(extended->internal, DatumGetBool(val) ? true : false); } static void bool_compressor_append_null_value(Compressor *compressor) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = bool_compressor_alloc(); bool_compressor_append_null(extended->internal); } static void * bool_compressor_finish_and_reset(Compressor *compressor) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; void *compressed = NULL; if (extended != NULL && extended->internal != NULL) { compressed = bool_compressor_finish(extended->internal); pfree(extended->internal); extended->internal = NULL; } return compressed; } static BoolCompressed * bool_compressed_from_parts(Simple8bRleSerialized *values, Simple8bRleSerialized *validity_bitmap) { uint32 validity_bitmap_size = 0; Size compressed_size; char *compressed_data; BoolCompressed *compressed; uint32 num_values = values != NULL ? values->num_elements : 0; uint32 values_size = values != NULL ? simple8brle_serialized_total_size(values) : 0; if (num_values == 0) return NULL; if (validity_bitmap != NULL) validity_bitmap_size = simple8brle_serialized_total_size(validity_bitmap); compressed_size = sizeof(BoolCompressed) + values_size + validity_bitmap_size; if (!AllocSizeIsValid(compressed_size)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("compressed size exceeds the maximum allowed (%d)", (int) MaxAllocSize))); compressed_data = palloc(compressed_size); compressed = (BoolCompressed *) compressed_data; SET_VARSIZE(&compressed->vl_len_, compressed_size); compressed->compression_algorithm = COMPRESSION_ALGORITHM_BOOL; compressed->has_nulls = validity_bitmap_size != 0 ? 1 : 0; compressed_data += sizeof(*compressed); compressed_data = bytes_serialize_simple8b_and_advance(compressed_data, values_size, values); if (compressed->has_nulls == 1 && validity_bitmap != NULL) { CheckCompressedData(validity_bitmap->num_elements == num_values); bytes_serialize_simple8b_and_advance(compressed_data, validity_bitmap_size, validity_bitmap); } return compressed; } static void decompression_iterator_init(BoolDecompressionIterator *iter, void *compressed, Oid element_type, bool forward) { StringInfoData si = { .data = compressed, .len = VARSIZE(compressed) }; BoolCompressed *header = consumeCompressedData(&si, sizeof(BoolCompressed)); Simple8bRleSerialized *values = bytes_deserialize_simple8b_and_advance(&si); Assert(header->has_nulls == 0 || header->has_nulls == 1); Assert(element_type == BOOLOID); const bool has_nulls = header->has_nulls == 1; CheckCompressedData(has_nulls == 0 || has_nulls == 1); *iter = (BoolDecompressionIterator){ .base = { .compression_algorithm = COMPRESSION_ALGORITHM_BOOL, .forward = forward, .element_type = element_type, .try_next = (forward ? bool_decompression_iterator_try_next_forward : bool_decompression_iterator_try_next_reverse) }, .values = { 0 }, .validity_bitmap = { 0 }, .position = 0, }; iter->values = simple8brle_bitmap_decompress(values); if (has_nulls) { Simple8bRleSerialized *validity_bitmap = bytes_deserialize_simple8b_and_advance(&si); iter->validity_bitmap = simple8brle_bitmap_decompress(validity_bitmap); CheckCompressedData(iter->validity_bitmap.num_elements == iter->values.num_elements); } if (!forward) { iter->position = iter->values.num_elements - 1; } } ================================================ FILE: tsl/src/compression/algorithms/bool_compress.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once /* * bool_compress is used to encode boolean values using the simple8b_rle algorithm. * * The bool compressor differs from the other compressors in that it does store a value * even for nulls, which is the last value seen befere the null. With this the bool * compressor always creates a compressed block even for nulls only. * * The boolean compressor represents the boolean values in a batch with two parallel * bitmaps, value bitmap and validity bitmap, like in the Arrow representation. * These bitmaps are compressed with our common bit-packing algorithm. * * The validity bitmap stores a 0 for a null value and a 1 for a non-null value as * required by the Arrow specification. This is the opposite of what the other compression * algorithms do in their nulls bitmaps. */ #include <postgres.h> #include <fmgr.h> #include <lib/stringinfo.h> #include "compression/compression.h" typedef struct BoolCompressor BoolCompressor; typedef struct BoolCompressed BoolCompressed; typedef struct BoolDecompressionIterator BoolDecompressionIterator; /* * Compressor framework functions and definitions for the bool_compress algorithm. */ extern BoolCompressor *bool_compressor_alloc(void); extern void bool_compressor_append_null(BoolCompressor *compressor); extern void bool_compressor_append_value(BoolCompressor *compressor, bool next_val); extern void *bool_compressor_finish(BoolCompressor *compressor); extern bool bool_compressed_has_nulls(const CompressedDataHeader *header); extern DecompressResult bool_decompression_iterator_try_next_forward(DecompressionIterator *iter); extern DecompressionIterator *bool_decompression_iterator_from_datum_forward(Datum bool_compressed, Oid element_type); extern DecompressResult bool_decompression_iterator_try_next_reverse(DecompressionIterator *iter); extern DecompressionIterator *bool_decompression_iterator_from_datum_reverse(Datum bool_compressed, Oid element_type); extern ArrowArray *bool_decompress_all(Datum compressed, Oid element_type, MemoryContext dest_mctx); extern void bool_compressed_send(CompressedDataHeader *header, StringInfo buffer); extern Datum bool_compressed_recv(StringInfo buf); extern Compressor *bool_compressor_for_type(Oid element_type); #define BOOL_COMPRESS_ALGORITHM_DEFINITION \ { \ .iterator_init_forward = bool_decompression_iterator_from_datum_forward, \ .iterator_init_reverse = bool_decompression_iterator_from_datum_reverse, \ .decompress_all = bool_decompress_all, .compressed_data_send = bool_compressed_send, \ .compressed_data_recv = bool_compressed_recv, \ .compressor_for_type = bool_compressor_for_type, \ .compressed_data_storage = TOAST_STORAGE_EXTERNAL, \ } /* * Cross-module functions for the bool_compress algorithm. */ extern Datum tsl_bool_compressor_append(PG_FUNCTION_ARGS); extern Datum tsl_bool_compressor_finish(PG_FUNCTION_ARGS); ================================================ FILE: tsl/src/compression/algorithms/datum_serialize.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/htup_details.h> #include <access/tupmacs.h> #include <catalog/namespace.h> #include <catalog/pg_type.h> #include <libpq/pqformat.h> #include <utils/builtins.h> #include <utils/datum.h> #include <utils/lsyscache.h> #include <utils/sortsupport.h> #include <utils/syscache.h> #include <utils/typcache.h> #include <compat/compat.h> #include "datum_serialize.h" #include "src/utils.h" #include <compression/compression.h> typedef struct DatumSerializer { Oid type_oid; bool type_by_val; int16 type_len; char type_align; char type_storage; Oid type_send; Oid type_out; /* lazy load */ bool send_info_set; FmgrInfo send_flinfo; bool use_binary_send; } DatumSerializer; DatumSerializer * create_datum_serializer(Oid type_oid) { DatumSerializer *res = palloc(sizeof(*res)); /* we use the syscache and not the type cache here b/c we need the * send/recv in/out functions that aren't in type cache */ Form_pg_type type; HeapTuple tup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(type_oid)); if (!HeapTupleIsValid(tup)) elog(ERROR, "cache lookup failed for type %u", type_oid); type = (Form_pg_type) GETSTRUCT(tup); *res = (DatumSerializer){ .type_oid = type_oid, .type_by_val = type->typbyval, .type_len = type->typlen, .type_align = type->typalign, .type_storage = type->typstorage, .type_send = type->typsend, .type_out = type->typoutput, .use_binary_send = OidIsValid(type->typsend), }; ReleaseSysCache(tup); return res; } bool datum_serializer_value_may_be_toasted(DatumSerializer *serializer) { return serializer->type_len == -1; } static inline void load_send_fn(DatumSerializer *serializer) { if (serializer->send_info_set) return; serializer->send_info_set = true; if (serializer->use_binary_send) fmgr_info(serializer->type_send, &serializer->send_flinfo); else fmgr_info(serializer->type_out, &serializer->send_flinfo); } #define TYPE_IS_PACKABLE(typlen, typstorage) ((typlen) == -1 && (typstorage) != 'p') /* Inspired by datum_compute_size in rangetypes.c */ Size datum_get_bytes_size(DatumSerializer *serializer, Size start_offset, Datum val) { Size data_length = start_offset; if (serializer->type_len == -1) { /* varlena */ Pointer ptr = DatumGetPointer(val); if (VARATT_IS_EXTERNAL(ptr)) { /* * Throw error, because we should never get a toasted datum. * Caller should have detoasted it. */ elog(ERROR, "datum should be detoasted before passed to datum_get_bytes_size"); } } if (TYPE_IS_PACKABLE(serializer->type_len, serializer->type_storage) && VARATT_CAN_MAKE_SHORT(DatumGetPointer(val))) { /* * we're anticipating converting to a short varlena header, so adjust * length and don't count any alignment (the case where the Datum is already * in short format is handled by att_align_datum) */ data_length += VARATT_CONVERTED_SHORT_SIZE(DatumGetPointer(val)); } else { data_length = att_align_datum(data_length, serializer->type_align, serializer->type_len, val); data_length = att_addlength_datum(data_length, serializer->type_len, val); } return data_length; } BinaryStringEncoding datum_serializer_binary_string_encoding(DatumSerializer *serializer) { return (serializer->use_binary_send ? BINARY_ENCODING : TEXT_ENCODING); } static void check_allowed_data_len(Size data_length, Size max_size) { if (max_size < data_length) elog(ERROR, "trying to serialize more data than was allocated"); } static inline char * align_and_zero(char *ptr, char type_align, Size *max_size) { char *new_pos = (char *) att_align_nominal(ptr, type_align); if (new_pos != ptr) { Size padding = new_pos - ptr; check_allowed_data_len(padding, *max_size); memset(ptr, 0, padding); *max_size = *max_size - padding; } return new_pos; } /* Inspired by datum_write in rangetypes.c. This reduces the max_size by the data length before * exiting */ char * datum_to_bytes_and_advance(DatumSerializer *serializer, char *start, Size *max_size, Datum datum) { Size data_length; if (serializer->type_by_val) { /* pass-by-value */ start = align_and_zero(start, serializer->type_align, max_size); data_length = serializer->type_len; check_allowed_data_len(data_length, *max_size); /* Data length should be set to something sensible, otherwise an error * will be raised inside store_att_byval, so we assert here to get a * stack. */ Assert(data_length > 0 && data_length <= 8); store_att_byval(start, datum, data_length); } else if (serializer->type_len == -1) { /* varlena */ Pointer val = DatumGetPointer(datum); if (VARATT_IS_EXTERNAL(val)) { /* * Throw error, because we should never get a toast datum. * Caller should have detoasted it. */ elog(ERROR, "datum should be detoasted before passed to datum_to_bytes_and_advance"); data_length = 0; /* keep compiler quiet */ } else if (VARATT_IS_SHORT(val)) { /* no alignment for short varlenas */ data_length = VARSIZE_SHORT(val); check_allowed_data_len(data_length, *max_size); memcpy(start, val, data_length); } else if (TYPE_IS_PACKABLE(serializer->type_len, serializer->type_storage) && VARATT_CAN_MAKE_SHORT(val)) { /* convert to short varlena -- no alignment */ data_length = VARATT_CONVERTED_SHORT_SIZE(val); check_allowed_data_len(data_length, *max_size); SET_VARSIZE_SHORT(start, data_length); memcpy(start + 1, VARDATA(val), data_length - 1); } else { /* full 4-byte header varlena */ start = align_and_zero(start, serializer->type_align, max_size); data_length = VARSIZE(val); check_allowed_data_len(data_length, *max_size); memcpy(start, val, data_length); } } else if (serializer->type_len == -2) { /* cstring ... never needs alignment */ Assert(serializer->type_align == 'c'); data_length = strlen(DatumGetCString(datum)) + 1; check_allowed_data_len(data_length, *max_size); memcpy(start, DatumGetPointer(datum), data_length); } else { /* fixed-length pass-by-reference */ start = align_and_zero(start, serializer->type_align, max_size); Assert(serializer->type_len > 0); data_length = serializer->type_len; check_allowed_data_len(data_length, *max_size); memcpy(start, DatumGetPointer(datum), data_length); } start += data_length; *max_size = *max_size - data_length; return start; } typedef struct DatumDeserializer { bool type_by_val; int16 type_len; char type_align; char type_storage; Oid type_recv; Oid type_in; Oid type_io_param; int32 type_mod; /* lazy load */ bool recv_info_set; FmgrInfo recv_flinfo; bool use_binary_recv; } DatumDeserializer; DatumDeserializer * create_datum_deserializer(Oid type_oid) { DatumDeserializer *res = palloc(sizeof(*res)); /* we use the syscache and not the type cache here b/c we need the * send/recv in/out functions that aren't in type cache */ Form_pg_type type; HeapTuple tup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(type_oid)); if (!HeapTupleIsValid(tup)) elog(ERROR, "cache lookup failed for type %u", type_oid); type = (Form_pg_type) GETSTRUCT(tup); *res = (DatumDeserializer){ .type_by_val = type->typbyval, .type_len = type->typlen, .type_align = type->typalign, .type_storage = type->typstorage, .type_recv = type->typreceive, .type_in = type->typinput, .type_io_param = getTypeIOParam(tup), .type_mod = type->typtypmod, }; ReleaseSysCache(tup); return res; } static inline void load_recv_fn(DatumDeserializer *des, bool use_binary) { if (des->recv_info_set && des->use_binary_recv == use_binary) return; des->recv_info_set = true; des->use_binary_recv = use_binary; if (des->use_binary_recv) fmgr_info(des->type_recv, &des->recv_flinfo); else fmgr_info(des->type_in, &des->recv_flinfo); } /* Loosely based on `range_deserialize` in rangetypes.c */ Datum bytes_to_datum_and_advance(DatumDeserializer *deserializer, const char **ptr) { Datum res; /* att_align_pointer can handle the case where an unaligned short-varlen follows any other * varlen by detecting padding. padding bytes _must always_ be set to 0, while the first byte of * a varlen header is _never_ 0. This means that if the next byte is non-zero, it must be the * start of a short-varlen, otherwise we need to align the pointer. */ *ptr = (Pointer) att_align_pointer(*ptr, deserializer->type_align, deserializer->type_len, *ptr); if (deserializer->type_len == -1) { /* * Check for potentially corrupt varlena headers since we're reading them * directly from compressed data. We can only have a plain datum * with 1-byte or 4-byte header here, no TOAST or compressed data. */ CheckCompressedData(VARATT_IS_4B_U(*ptr) || (VARATT_IS_1B(*ptr) && !VARATT_IS_1B_E(*ptr))); /* * Full varsize must be larger or equal than the header size so that the * calculation of size without header doesn't overflow. */ CheckCompressedData((VARATT_IS_1B(*ptr) && VARSIZE_1B(*ptr) >= VARHDRSZ_SHORT) || (VARSIZE_4B(*ptr) > VARHDRSZ)); } res = ts_fetch_att(*ptr, deserializer->type_by_val, deserializer->type_len); *ptr = att_addlength_pointer(*ptr, deserializer->type_len, *ptr); return res; } void type_append_to_binary_string(Oid type_oid, StringInfo buffer) { Form_pg_type type_tuple; HeapTuple tup = SearchSysCache1(TYPEOID, ObjectIdGetDatum(type_oid)); char *namespace_name; if (!HeapTupleIsValid(tup)) elog(ERROR, "cache lookup failed for type %u", type_oid); type_tuple = (Form_pg_type) GETSTRUCT(tup); namespace_name = get_namespace_name(type_tuple->typnamespace); pq_sendstring(buffer, namespace_name); pq_sendstring(buffer, NameStr(type_tuple->typname)); ReleaseSysCache(tup); } Oid binary_string_get_type(StringInfo buffer) { const char *element_type_namespace = pq_getmsgstring(buffer); const char *element_type_name = pq_getmsgstring(buffer); Oid namespace_oid; Oid type_oid; namespace_oid = LookupExplicitNamespace(element_type_namespace, false); type_oid = GetSysCacheOid2(TYPENAMENSP, Anum_pg_type_oid, PointerGetDatum(element_type_name), ObjectIdGetDatum(namespace_oid)); CheckCompressedData(OidIsValid(type_oid)); return type_oid; } void datum_append_to_binary_string(DatumSerializer *serializer, BinaryStringEncoding encoding, StringInfo buffer, Datum datum) { load_send_fn(serializer); if (encoding == MESSAGE_SPECIFIES_ENCODING) pq_sendbyte(buffer, serializer->use_binary_send); else if (encoding != datum_serializer_binary_string_encoding(serializer)) elog(ERROR, "incorrect encoding chosen in datum_append_to_binary_string"); if (serializer->use_binary_send) { bytea *output = SendFunctionCall(&serializer->send_flinfo, datum); pq_sendint32(buffer, VARSIZE_ANY_EXHDR(output)); pq_sendbytes(buffer, VARDATA(output), VARSIZE_ANY_EXHDR(output)); } else { char *output = OutputFunctionCall(&serializer->send_flinfo, datum); pq_sendstring(buffer, output); } } Datum binary_string_to_datum(DatumDeserializer *deserializer, BinaryStringEncoding encoding, StringInfo buffer) { Datum res; bool use_binary_recv = false; switch (encoding) { case BINARY_ENCODING: use_binary_recv = true; break; case TEXT_ENCODING: use_binary_recv = false; break; case MESSAGE_SPECIFIES_ENCODING: use_binary_recv = pq_getmsgbyte(buffer) != 0; break; } load_recv_fn(deserializer, use_binary_recv); if (use_binary_recv) { uint32 data_size = pq_getmsgint32(buffer); const char *bytes = pq_getmsgbytes(buffer, data_size); StringInfoData d = { .data = (char *) bytes, .len = data_size, .maxlen = data_size, }; res = ReceiveFunctionCall(&deserializer->recv_flinfo, &d, deserializer->type_io_param, deserializer->type_mod); } else { const char *string = pq_getmsgstring(buffer); res = InputFunctionCall(&deserializer->recv_flinfo, (char *) string, deserializer->type_io_param, deserializer->type_mod); } return res; } ================================================ FILE: tsl/src/compression/algorithms/datum_serialize.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <lib/stringinfo.h> /* SERIALIZATION */ typedef struct DatumSerializer DatumSerializer; DatumSerializer *create_datum_serializer(Oid type); bool datum_serializer_value_may_be_toasted(DatumSerializer *serializer); typedef enum { BINARY_ENCODING = 0, TEXT_ENCODING, MESSAGE_SPECIFIES_ENCODING, } BinaryStringEncoding; /* Get the encoding type used by the serializer: either BINARY_ENCODING or TEXT_ENCODING */ BinaryStringEncoding datum_serializer_binary_string_encoding(DatumSerializer *serializer); /* serialize to bytes in memory. */ Size datum_get_bytes_size(DatumSerializer *serializer, Size start_offset, Datum val); char *datum_to_bytes_and_advance(DatumSerializer *serializer, char *start, Size *max_size, Datum datum); /* serialize to a binary string (for send functions) */ void type_append_to_binary_string(Oid type_oid, StringInfo buffer); void datum_append_to_binary_string(DatumSerializer *serializer, BinaryStringEncoding encoding, StringInfo buffer, Datum datum); /* DESERIALIZATION */ typedef struct DatumDeserializer DatumDeserializer; DatumDeserializer *create_datum_deserializer(Oid type); /* deserialization from bytes in memory */ Datum bytes_to_datum_and_advance(DatumDeserializer *deserializer, const char **ptr); /* deserialization from binary strings (for recv functions) */ Datum binary_string_to_datum(DatumDeserializer *deserializer, BinaryStringEncoding encoding, StringInfo buffer); Oid binary_string_get_type(StringInfo buffer); ================================================ FILE: tsl/src/compression/algorithms/deltadelta.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include "deltadelta.h" #include <access/htup_details.h> #include <catalog/pg_aggregate.h> #include <catalog/pg_type.h> #include <common/base64.h> #include <funcapi.h> #include <lib/stringinfo.h> #include <stdbool.h> #include <utils/builtins.h> #include <utils/date.h> #include <utils/lsyscache.h> #include <utils/syscache.h> #include <utils/timestamp.h> #include <utils.h> #include "compression/arrow_c_data_interface.h" #include "compression/compression.h" #include "guc.h" #include "simple8b_rle.h" #include "simple8b_rle_bitmap.h" static uint64 zig_zag_encode(uint64 value); static uint64 zig_zag_decode(uint64 value); typedef struct DeltaDeltaCompressed { CompressedDataHeaderFields; uint8 has_nulls; /* 1 if this has a NULLs bitmap after deltas, 0 otherwise */ uint8 padding[2]; uint64 last_value; uint64 last_delta; char delta_deltas[FLEXIBLE_ARRAY_MEMBER]; } DeltaDeltaCompressed; static void pg_attribute_unused() assertions(void) { DeltaDeltaCompressed test_val = { .vl_len_ = { 0 } }; /* make sure no padding bytes make it to disk */ StaticAssertStmt(sizeof(DeltaDeltaCompressed) == sizeof(test_val.vl_len_) + sizeof(test_val.compression_algorithm) + sizeof(test_val.has_nulls) + sizeof(test_val.padding) + sizeof(test_val.last_value) + sizeof(test_val.last_delta), "DeltaDeltaCompressed wrong size"); StaticAssertStmt(sizeof(DeltaDeltaCompressed) == 24, "DeltaDeltaCompressed wrong size"); } typedef struct DeltaDeltaDecompressionIterator { DecompressionIterator base; uint64 prev_val; uint64 prev_delta; Simple8bRleDecompressionIterator delta_deltas; Simple8bRleDecompressionIterator nulls; bool has_nulls; } DeltaDeltaDecompressionIterator; typedef struct DeltaDeltaCompressor { uint64 prev_val; uint64 prev_delta; Simple8bRleCompressor delta_delta; Simple8bRleCompressor nulls; bool has_nulls; } DeltaDeltaCompressor; typedef struct ExtendedCompressor { Compressor base; DeltaDeltaCompressor *internal; } ExtendedCompressor; bool deltadelta_compressed_has_nulls(const CompressedDataHeader *header) { const DeltaDeltaCompressed *ddc = (const DeltaDeltaCompressed *) header; return ddc->has_nulls; } static void deltadelta_compressor_append_bool(Compressor *compressor, Datum val) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = delta_delta_compressor_alloc(); delta_delta_compressor_append_value(extended->internal, DatumGetBool(val) ? 1 : 0); } static void deltadelta_compressor_append_int16(Compressor *compressor, Datum val) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = delta_delta_compressor_alloc(); delta_delta_compressor_append_value(extended->internal, DatumGetInt16(val)); } static void deltadelta_compressor_append_int32(Compressor *compressor, Datum val) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = delta_delta_compressor_alloc(); delta_delta_compressor_append_value(extended->internal, DatumGetInt32(val)); } static void deltadelta_compressor_append_int64(Compressor *compressor, Datum val) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = delta_delta_compressor_alloc(); delta_delta_compressor_append_value(extended->internal, DatumGetInt64(val)); } static void deltadelta_compressor_append_date(Compressor *compressor, Datum val) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = delta_delta_compressor_alloc(); delta_delta_compressor_append_value(extended->internal, DatumGetDateADT(val)); } static void deltadelta_compressor_append_timestamp(Compressor *compressor, Datum val) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = delta_delta_compressor_alloc(); delta_delta_compressor_append_value(extended->internal, DatumGetTimestamp(val)); } static void deltadelta_compressor_append_timestamptz(Compressor *compressor, Datum val) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = delta_delta_compressor_alloc(); delta_delta_compressor_append_value(extended->internal, DatumGetTimestampTz(val)); } static void deltadelta_compressor_append_null_value(Compressor *compressor) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = delta_delta_compressor_alloc(); delta_delta_compressor_append_null(extended->internal); } static void * deltadelta_compressor_finish_and_reset(Compressor *compressor) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; void *compressed = delta_delta_compressor_finish(extended->internal); pfree(extended->internal); extended->internal = NULL; return compressed; } const Compressor deltadelta_bool_compressor = { .append_val = deltadelta_compressor_append_bool, .append_null = deltadelta_compressor_append_null_value, .is_full = NULL, .finish = deltadelta_compressor_finish_and_reset, }; const Compressor deltadelta_uint16_compressor = { .append_val = deltadelta_compressor_append_int16, .append_null = deltadelta_compressor_append_null_value, .is_full = NULL, .finish = deltadelta_compressor_finish_and_reset, }; const Compressor deltadelta_uint32_compressor = { .append_val = deltadelta_compressor_append_int32, .append_null = deltadelta_compressor_append_null_value, .is_full = NULL, .finish = deltadelta_compressor_finish_and_reset, }; const Compressor deltadelta_uint64_compressor = { .append_val = deltadelta_compressor_append_int64, .append_null = deltadelta_compressor_append_null_value, .is_full = NULL, .finish = deltadelta_compressor_finish_and_reset, }; const Compressor deltadelta_date_compressor = { .append_val = deltadelta_compressor_append_date, .append_null = deltadelta_compressor_append_null_value, .is_full = NULL, .finish = deltadelta_compressor_finish_and_reset, }; const Compressor deltadelta_timestamp_compressor = { .append_val = deltadelta_compressor_append_timestamp, .append_null = deltadelta_compressor_append_null_value, .is_full = NULL, .finish = deltadelta_compressor_finish_and_reset, }; const Compressor deltadelta_timestamptz_compressor = { .append_val = deltadelta_compressor_append_timestamptz, .append_null = deltadelta_compressor_append_null_value, .is_full = NULL, .finish = deltadelta_compressor_finish_and_reset, }; Compressor * delta_delta_compressor_for_type(Oid element_type) { ExtendedCompressor *compressor = palloc(sizeof(*compressor)); switch (element_type) { case BOOLOID: *compressor = (ExtendedCompressor){ .base = deltadelta_bool_compressor }; return &compressor->base; case INT2OID: *compressor = (ExtendedCompressor){ .base = deltadelta_uint16_compressor }; return &compressor->base; case INT4OID: *compressor = (ExtendedCompressor){ .base = deltadelta_uint32_compressor }; return &compressor->base; case INT8OID: *compressor = (ExtendedCompressor){ .base = deltadelta_uint64_compressor }; return &compressor->base; case DATEOID: *compressor = (ExtendedCompressor){ .base = deltadelta_date_compressor }; return &compressor->base; case TIMESTAMPOID: *compressor = (ExtendedCompressor){ .base = deltadelta_timestamp_compressor }; return &compressor->base; case TIMESTAMPTZOID: *compressor = (ExtendedCompressor){ .base = deltadelta_timestamptz_compressor }; return &compressor->base; default: elog(ERROR, "invalid type for delta-delta compressor \"%s\"", format_type_be(element_type)); } pg_unreachable(); } Datum tsl_deltadelta_compressor_append(PG_FUNCTION_ARGS) { MemoryContext old_context; MemoryContext agg_context; DeltaDeltaCompressor *compressor = (DeltaDeltaCompressor *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0)); if (!AggCheckCallContext(fcinfo, &agg_context)) { /* cannot be called directly because of internal-type argument */ elog(ERROR, "tsl_deltadelta_compressor_append called in non-aggregate context"); } old_context = MemoryContextSwitchTo(agg_context); if (compressor == NULL) { compressor = delta_delta_compressor_alloc(); if (PG_NARGS() > 2) elog(ERROR, "append expects two arguments"); } if (PG_ARGISNULL(1)) delta_delta_compressor_append_null(compressor); else { int64 next_val = PG_GETARG_INT64(1); delta_delta_compressor_append_value(compressor, next_val); } MemoryContextSwitchTo(old_context); PG_RETURN_POINTER(compressor); } DeltaDeltaCompressor * delta_delta_compressor_alloc(void) { DeltaDeltaCompressor *compressor = palloc0(sizeof(*compressor)); simple8brle_compressor_init(&compressor->delta_delta); simple8brle_compressor_init(&compressor->nulls); return compressor; } static void * delta_delta_set_header_and_advance(uint64 last_value, uint64 last_delta, bool has_nulls, size_t compressed_size, void *dest) { DeltaDeltaCompressed *compressed = (DeltaDeltaCompressed *) dest; SET_VARSIZE(&compressed->vl_len_, compressed_size); compressed->compression_algorithm = COMPRESSION_ALGORITHM_DELTADELTA; compressed->last_value = last_value; compressed->last_delta = last_delta; compressed->has_nulls = has_nulls ? 1 : 0; compressed->padding[0] = 0; compressed->padding[1] = 0; return (char *) compressed + sizeof(*compressed); } size_t delta_delta_compressor_compressed_size(DeltaDeltaCompressor *compressor, size_t *nulls_size_out) { size_t compressed_size = sizeof(DeltaDeltaCompressed); size_t nulls_size_actual; /* If there are no elements, the compressed size is 0 even if there are nulls */ if (compressor->delta_delta.num_elements == 0) { if (nulls_size_out != NULL) *nulls_size_out = 0; return 0; } compressed_size += simple8brle_compressor_compressed_const_size(&compressor->delta_delta); if (compressor->has_nulls) { nulls_size_actual = simple8brle_compressor_compressed_const_size(&compressor->nulls); compressed_size += nulls_size_actual; if (nulls_size_out != NULL) *nulls_size_out = nulls_size_actual; } else if (nulls_size_out != NULL) *nulls_size_out = 0; return compressed_size; } void * delta_delta_compressor_finish(DeltaDeltaCompressor *compressor) { size_t total_size = delta_delta_compressor_compressed_size(compressor, NULL); char *compressed = NULL; if (total_size == 0) return NULL; compressed = palloc(total_size); delta_delta_compressor_finish_into(compressor, compressed); return compressed; } void * delta_delta_compressor_finish_into(DeltaDeltaCompressor *compressor, void *dest) { size_t data_size; size_t nulls_size; size_t compressed_size; char *result = (char *) dest; /* The compressed size includes the header and the nulls */ compressed_size = delta_delta_compressor_compressed_size(compressor, &nulls_size); if (compressed_size == 0) return dest; /* Check if the data size is valid */ data_size = compressed_size - sizeof(DeltaDeltaCompressed) - nulls_size; Assert(compressed_size > (sizeof(DeltaDeltaCompressed) + nulls_size)); result = delta_delta_set_header_and_advance(compressor->prev_val, compressor->prev_delta, compressor->has_nulls, compressed_size, dest); result = simple8brle_compressor_finish_into(&compressor->delta_delta, result, data_size); if (compressor->has_nulls) { Assert(nulls_size > 0); result = simple8brle_compressor_finish_into(&compressor->nulls, result, nulls_size); } return result; } Datum tsl_deltadelta_compressor_finish(PG_FUNCTION_ARGS) { DeltaDeltaCompressor *compressor = PG_ARGISNULL(0) ? NULL : (DeltaDeltaCompressor *) PG_GETARG_POINTER(0); void *compressed; if (compressor == NULL) PG_RETURN_NULL(); compressed = delta_delta_compressor_finish(compressor); if (compressed == NULL) PG_RETURN_NULL(); PG_RETURN_POINTER(compressed); } void delta_delta_compressor_append_null(DeltaDeltaCompressor *compressor) { compressor->has_nulls = true; simple8brle_compressor_append(&compressor->nulls, 1); } void delta_delta_compressor_append_value(DeltaDeltaCompressor *compressor, int64 next_val) { uint64 delta; uint64 delta_delta; uint64 encoded; /* * We perform all arithmetic using unsigned values due to C's overflow rules: * signed integer overflow is undefined behavior, so if we have a very large delta, * this code is without meaning, while unsigned overflow is 2's complement, so even * very large delta work the same as any other */ /* step 1: delta of deltas */ delta = ((uint64) next_val) - compressor->prev_val; delta_delta = delta - compressor->prev_delta; compressor->prev_val = next_val; compressor->prev_delta = delta; /* step 2: ZigZag encode */ encoded = zig_zag_encode(delta_delta); /* step 3: simple8b/RTE */ simple8brle_compressor_append(&compressor->delta_delta, encoded); simple8brle_compressor_append(&compressor->nulls, 0); } /**********************************************************************************/ /**********************************************************************************/ static void int64_decompression_iterator_init_forward(DeltaDeltaDecompressionIterator *iter, void *compressed, Oid element_type) { StringInfoData si = { .data = compressed, .len = VARSIZE(compressed) }; CheckCompressedData(VARSIZE(compressed) >= sizeof(DeltaDeltaCompressed)); CheckCompressedData(((DeltaDeltaCompressed *) compressed)->compression_algorithm == COMPRESSION_ALGORITHM_DELTADELTA); DeltaDeltaCompressed *header = consumeCompressedData(&si, sizeof(DeltaDeltaCompressed)); Simple8bRleSerialized *deltas = bytes_deserialize_simple8b_and_advance(&si); const bool has_nulls = header->has_nulls == 1; CheckCompressedData(has_nulls == 0 || has_nulls == 1); *iter = (DeltaDeltaDecompressionIterator){ .base = { .compression_algorithm = COMPRESSION_ALGORITHM_DELTADELTA, .forward = true, .element_type = element_type, .try_next = delta_delta_decompression_iterator_try_next_forward, }, .prev_val = 0, .prev_delta = 0, .has_nulls = has_nulls, }; simple8brle_decompression_iterator_init_forward(&iter->delta_deltas, deltas); if (has_nulls) { Simple8bRleSerialized *nulls = bytes_deserialize_simple8b_and_advance(&si); simple8brle_decompression_iterator_init_forward(&iter->nulls, nulls); } } static void int64_decompression_iterator_init_reverse(DeltaDeltaDecompressionIterator *iter, void *compressed, Oid element_type) { StringInfoData si = { .data = compressed, .len = VARSIZE(compressed) }; CheckCompressedData(VARSIZE(compressed) >= sizeof(DeltaDeltaCompressed)); CheckCompressedData(((DeltaDeltaCompressed *) compressed)->compression_algorithm == COMPRESSION_ALGORITHM_DELTADELTA); DeltaDeltaCompressed *header = consumeCompressedData(&si, sizeof(DeltaDeltaCompressed)); Simple8bRleSerialized *deltas = bytes_deserialize_simple8b_and_advance(&si); Assert(header->has_nulls == 0 || header->has_nulls == 1); *iter = (DeltaDeltaDecompressionIterator){ .base = { .compression_algorithm = COMPRESSION_ALGORITHM_DELTADELTA, .forward = false, .element_type = element_type, .try_next = delta_delta_decompression_iterator_try_next_reverse, }, .prev_val = header->last_value, .prev_delta = header->last_delta, .has_nulls = header->has_nulls, }; simple8brle_decompression_iterator_init_reverse(&iter->delta_deltas, deltas); if (header->has_nulls) { Simple8bRleSerialized *nulls = bytes_deserialize_simple8b_and_advance(&si); simple8brle_decompression_iterator_init_reverse(&iter->nulls, nulls); } } static inline DecompressResult convert_from_internal(DecompressResultInternal res_internal, Oid element_type) { if (res_internal.is_done || res_internal.is_null) { return (DecompressResult){ .is_done = res_internal.is_done, .is_null = res_internal.is_null, }; } switch (element_type) { case BOOLOID: return (DecompressResult){ .val = BoolGetDatum(res_internal.val), }; case INT8OID: return (DecompressResult){ .val = Int64GetDatum(res_internal.val), }; case INT4OID: return (DecompressResult){ .val = Int32GetDatum(res_internal.val), }; case INT2OID: return (DecompressResult){ .val = Int16GetDatum(res_internal.val), }; case DATEOID: return (DecompressResult){ .val = DateADTGetDatum(res_internal.val), }; case TIMESTAMPTZOID: return (DecompressResult){ .val = TimestampTzGetDatum(res_internal.val), }; case TIMESTAMPOID: return (DecompressResult){ .val = TimestampGetDatum(res_internal.val), }; default: elog(ERROR, "invalid type requested from deltadelta decompression \"%s\"", format_type_be(element_type)); } pg_unreachable(); } static DecompressResultInternal delta_delta_decompression_iterator_try_next_forward_internal(DeltaDeltaDecompressionIterator *iter) { Simple8bRleDecompressResult result; uint64 delta_delta; /* check for a null value */ if (iter->has_nulls) { Simple8bRleDecompressResult result = simple8brle_decompression_iterator_try_next_forward(&iter->nulls); if (result.is_done) return (DecompressResultInternal){ .is_done = true, }; if (result.val != 0) { CheckCompressedData(result.val == 1); return (DecompressResultInternal){ .is_null = true, }; } } result = simple8brle_decompression_iterator_try_next_forward(&iter->delta_deltas); if (result.is_done) return (DecompressResultInternal){ .is_done = true, }; delta_delta = zig_zag_decode(result.val); iter->prev_delta += delta_delta; iter->prev_val += iter->prev_delta; return (DecompressResultInternal){ .val = iter->prev_val, .is_null = false, .is_done = false, }; } DecompressResult delta_delta_decompression_iterator_try_next_forward(DecompressionIterator *iter) { Assert(iter->compression_algorithm == COMPRESSION_ALGORITHM_DELTADELTA && iter->forward); return convert_from_internal(delta_delta_decompression_iterator_try_next_forward_internal( (DeltaDeltaDecompressionIterator *) iter), iter->element_type); } #define ELEMENT_TYPE uint64 #include "simple8b_rle_decompress_all.h" #undef ELEMENT_TYPE /* Functions for bulk decompression. */ #define ELEMENT_TYPE uint16 #include "deltadelta_impl.c" #undef ELEMENT_TYPE #define ELEMENT_TYPE uint32 #include "deltadelta_impl.c" #undef ELEMENT_TYPE #define ELEMENT_TYPE uint64 #include "deltadelta_impl.c" #undef ELEMENT_TYPE ArrowArray * delta_delta_decompress_all(Datum compressed_data, Oid element_type, MemoryContext dest_mctx) { switch (element_type) { case INT8OID: case TIMESTAMPOID: case TIMESTAMPTZOID: return delta_delta_decompress_all_uint64(compressed_data, dest_mctx); case INT4OID: case DATEOID: return delta_delta_decompress_all_uint32(compressed_data, dest_mctx); case INT2OID: return delta_delta_decompress_all_uint16(compressed_data, dest_mctx); default: elog(ERROR, "type '%s' is not supported for deltadelta decompression", format_type_be(element_type)); pg_unreachable(); } } /* Functions for reverse iterator. */ static DecompressResultInternal delta_delta_decompression_iterator_try_next_reverse_internal(DeltaDeltaDecompressionIterator *iter) { Simple8bRleDecompressResult result; uint64 val; uint64 delta_delta; /* check for a null value */ if (iter->has_nulls) { Simple8bRleDecompressResult result = simple8brle_decompression_iterator_try_next_reverse(&iter->nulls); if (result.is_done) return (DecompressResultInternal){ .is_done = true, }; if (result.val != 0) { Assert(result.val == 1); return (DecompressResultInternal){ .is_null = true, }; } } result = simple8brle_decompression_iterator_try_next_reverse(&iter->delta_deltas); if (result.is_done) return (DecompressResultInternal){ .is_done = true, }; val = iter->prev_val; delta_delta = zig_zag_decode(result.val); iter->prev_val -= iter->prev_delta; iter->prev_delta -= delta_delta; return (DecompressResultInternal){ .val = val, }; } DecompressResult delta_delta_decompression_iterator_try_next_reverse(DecompressionIterator *iter) { Assert(iter->compression_algorithm == COMPRESSION_ALGORITHM_DELTADELTA && !iter->forward); return convert_from_internal(delta_delta_decompression_iterator_try_next_reverse_internal( (DeltaDeltaDecompressionIterator *) iter), iter->element_type); } DecompressionIterator * delta_delta_decompression_iterator_from_datum_forward(Datum deltadelta_compressed, Oid element_type) { DeltaDeltaDecompressionIterator *iterator = palloc(sizeof(*iterator)); int64_decompression_iterator_init_forward(iterator, (void *) PG_DETOAST_DATUM(deltadelta_compressed), element_type); return &iterator->base; } DecompressionIterator * delta_delta_decompression_iterator_from_datum_reverse(Datum deltadelta_compressed, Oid element_type) { DeltaDeltaDecompressionIterator *iterator = palloc(sizeof(*iterator)); int64_decompression_iterator_init_reverse(iterator, (void *) PG_DETOAST_DATUM(deltadelta_compressed), element_type); return &iterator->base; } /**********************************************************************************/ /**********************************************************************************/ void deltadelta_compressed_send(CompressedDataHeader *header, StringInfo buffer) { const DeltaDeltaCompressed *data = (DeltaDeltaCompressed *) header; Assert(header->compression_algorithm == COMPRESSION_ALGORITHM_DELTADELTA); pq_sendbyte(buffer, data->has_nulls); pq_sendint64(buffer, data->last_value); pq_sendint64(buffer, data->last_delta); simple8brle_serialized_send(buffer, (Simple8bRleSerialized *) data->delta_deltas); if (data->has_nulls) { Simple8bRleSerialized *nulls = (Simple8bRleSerialized *) (((char *) data->delta_deltas) + simple8brle_serialized_total_size( (Simple8bRleSerialized *) data->delta_deltas)); simple8brle_serialized_send(buffer, nulls); } } Datum deltadelta_compressed_recv(StringInfo buffer) { uint8 has_nulls; uint64 last_value; uint64 last_delta; Simple8bRleSerialized *delta_delta_values; Simple8bRleSerialized *nulls = NULL; DeltaDeltaCompressed *compressed; void *buf_ptr = NULL; int delta_size = 0; int nulls_size = 0; size_t compressed_size = 0; size_t allocated_size = 2 * sizeof(Simple8bRleSerialized) + sizeof(DeltaDeltaCompressed) + (buffer->len - buffer->cursor); compressed = palloc(allocated_size); has_nulls = pq_getmsgbyte(buffer); CheckCompressedData(has_nulls == 0 || has_nulls == 1); last_value = pq_getmsgint64(buffer); last_delta = pq_getmsgint64(buffer); /* Leave space for the header, but we don't yet know the size of the compressed data */ buf_ptr = (char *) compressed + sizeof(DeltaDeltaCompressed); /* Calculate the size of the delta delta values based on the number of bytes read */ delta_size = buffer->cursor; buf_ptr = simple8brle_serialized_recv_into(buffer, buf_ptr, &delta_delta_values); delta_size = buffer->cursor - delta_size; if (has_nulls) { /* Calculate the size of the nulls based on the number of bytes read */ nulls_size = buffer->cursor; buf_ptr = simple8brle_serialized_recv_into(buffer, buf_ptr, &nulls); nulls_size = buffer->cursor - nulls_size; CheckCompressedData(delta_delta_values->num_elements < nulls->num_elements); } compressed_size = sizeof(DeltaDeltaCompressed) + delta_size + nulls_size; if (!AllocSizeIsValid(compressed_size)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("compressed size exceeds the maximum allowed (%d)", (int) MaxAllocSize))); /* Set header but don't change the buffer pointer */ delta_delta_set_header_and_advance(last_value, last_delta, has_nulls, compressed_size, compressed); PG_RETURN_POINTER(compressed); } /**********************************************************************************/ /**********************************************************************************/ static pg_attribute_always_inline uint64 zig_zag_encode(uint64 value) { // (((uint64)value) << 1) ^ (uint64)(value >> 63); /* since shift is underspecified, we use (value < 0 ? 0xFFFFFFFFFFFFFFFFull : 0) * which compiles to the correct asm, and is well defined */ return (value << 1) ^ (((int64) value) < 0 ? 0xFFFFFFFFFFFFFFFFULL : 0); } static pg_attribute_always_inline uint64 zig_zag_decode(uint64 value) { /* ZigZag turns negative numbers into odd ones, and positive numbers into even ones*/ return (value >> 1) ^ (uint64) - (int64) (value & 1); } ================================================ FILE: tsl/src/compression/algorithms/deltadelta.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once /* * Deltadelta is used to encode integers or integer-like objects (e.g. timestamps). It's input is a * series of integers. first convert that series to a series of delta-of-deltas between * consecutive integers, while storing the first value as well. Given the first value * and the series of delta-of-delta values, it is easy to reconstruct the original series of * integers (assume first delta is 0). * * We now describe how to compress the delta-of-deltas: * First we zigzag encodes the delta-of-deltas * Second, we simple8b_rle encode the zig-zag encoding */ #include <postgres.h> #include <fmgr.h> #include <lib/stringinfo.h> #include "compression/compression.h" typedef struct DeltaDeltaCompressor DeltaDeltaCompressor; typedef struct DeltaDeltaCompressed DeltaDeltaCompressed; typedef struct DeltaDeltaDecompressionIterator DeltaDeltaDecompressionIterator; extern bool deltadelta_compressed_has_nulls(const CompressedDataHeader *header); extern Compressor *delta_delta_compressor_for_type(Oid element_type); extern DeltaDeltaCompressor *delta_delta_compressor_alloc(void); extern void delta_delta_compressor_append_null(DeltaDeltaCompressor *compressor); extern void delta_delta_compressor_append_value(DeltaDeltaCompressor *compressor, int64 next_val); extern void *delta_delta_compressor_finish(DeltaDeltaCompressor *compressor); extern size_t delta_delta_compressor_compressed_size(DeltaDeltaCompressor *compressor, size_t *nulls_size /* out */); extern void *delta_delta_compressor_finish_into(DeltaDeltaCompressor *compressor, void *dest); extern DecompressionIterator * delta_delta_decompression_iterator_from_datum_forward(Datum deltadelta_compressed, Oid element_type); extern DecompressionIterator * delta_delta_decompression_iterator_from_datum_reverse(Datum deltadelta_compressed, Oid element_type); extern DecompressResult delta_delta_decompression_iterator_try_next_forward(DecompressionIterator *iter); extern ArrowArray *delta_delta_decompress_all(Datum compressed_data, Oid element_type, MemoryContext dest_mctx); extern DecompressResult delta_delta_decompression_iterator_try_next_reverse(DecompressionIterator *iter); extern void deltadelta_compressed_send(CompressedDataHeader *header, StringInfo buffer); extern Datum deltadelta_compressed_recv(StringInfo buf); extern Datum tsl_deltadelta_compressor_append(PG_FUNCTION_ARGS); extern Datum tsl_deltadelta_compressor_finish(PG_FUNCTION_ARGS); #define DELTA_DELTA_ALGORITHM_DEFINITION \ { \ .iterator_init_forward = delta_delta_decompression_iterator_from_datum_forward, \ .iterator_init_reverse = delta_delta_decompression_iterator_from_datum_reverse, \ .decompress_all = delta_delta_decompress_all, \ .compressed_data_send = deltadelta_compressed_send, \ .compressed_data_recv = deltadelta_compressed_recv, \ .compressor_for_type = delta_delta_compressor_for_type, \ .compressed_data_storage = TOAST_STORAGE_EXTERNAL, \ } ================================================ FILE: tsl/src/compression/algorithms/deltadelta_impl.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Decompress the entire batch of deltadelta-compressed rows into an Arrow array. * Specialized for each supported data type. */ #define FUNCTION_NAME_HELPER(X, Y) X##_##Y #define FUNCTION_NAME(X, Y) FUNCTION_NAME_HELPER(X, Y) static ArrowArray * FUNCTION_NAME(delta_delta_decompress_all, ELEMENT_TYPE)(Datum compressed, MemoryContext dest_mctx) { StringInfoData si = { .data = DatumGetPointer(compressed), .len = VARSIZE(compressed) }; DeltaDeltaCompressed *header = consumeCompressedData(&si, sizeof(DeltaDeltaCompressed)); Simple8bRleSerialized *deltas_compressed = bytes_deserialize_simple8b_and_advance(&si); const bool has_nulls = header->has_nulls == 1; Assert(header->has_nulls == 0 || header->has_nulls == 1); /* * Can't use element type here because of zig-zag encoding. The deltas are * computed in uint64, so we can get a delta that is actually larger than * the element type. We can't just truncate the delta either, because it * will lead to broken decompression results. The test case is in * test_delta4(). */ uint32 num_deltas; const uint64 *deltas_zigzag = simple8brle_decompress_all_uint64(deltas_compressed, &num_deltas); Simple8bRleBitmap nulls = { 0 }; if (has_nulls) { Simple8bRleSerialized *nulls_compressed = bytes_deserialize_simple8b_and_advance(&si); nulls = simple8brle_bitmap_decompress(nulls_compressed); } /* * Pad the number of elements to multiple of 64 bytes if needed, so that we * can work in 64-byte blocks. */ #define INNER_LOOP_SIZE_LOG2 3 #define INNER_LOOP_SIZE (1 << INNER_LOOP_SIZE_LOG2) const uint32 n_total = has_nulls ? nulls.num_elements : num_deltas; const uint32 n_total_padded = pad_to_multiple(INNER_LOOP_SIZE, n_total); const uint32 n_notnull = num_deltas; const uint32 n_notnull_padded = pad_to_multiple(INNER_LOOP_SIZE, n_notnull); Assert(n_total_padded >= n_total); Assert(n_notnull_padded >= n_notnull); Assert(n_total >= n_notnull); Assert(n_total <= GLOBAL_MAX_ROWS_PER_COMPRESSION); /* * We need additional padding at the end of buffer, because the code that * converts the elements to postgres Datum always reads in 8 bytes. */ const int buffer_bytes = n_total_padded * sizeof(ELEMENT_TYPE) + 8; ELEMENT_TYPE *restrict decompressed_values = MemoryContextAlloc(dest_mctx, buffer_bytes); /* Now fill the data w/o nulls. */ ELEMENT_TYPE current_delta = 0; ELEMENT_TYPE current_element = 0; /* * Manual unrolling speeds up this loop by about 10%. clang vectorizes * the zig_zag_decode part, but not the double-prefix-sum part. * * Also tried using SIMD prefix sum from here twice: * https://en.algorithmica.org/hpc/algorithms/prefix/, it's slower. * * Also tried zig-zag decoding in a separate loop, seems to be slightly * slower, around the noise threshold. */ Assert(n_notnull_padded % INNER_LOOP_SIZE == 0); for (uint32 outer = 0; outer < n_notnull_padded; outer += INNER_LOOP_SIZE) { for (uint32 inner = 0; inner < INNER_LOOP_SIZE; inner++) { current_delta += zig_zag_decode(deltas_zigzag[outer + inner]); current_element += current_delta; decompressed_values[outer + inner] = current_element; } } #undef INNER_LOOP_SIZE_LOG2 #undef INNER_LOOP_SIZE uint64 *restrict validity_bitmap = NULL; if (has_nulls) { /* Now move the data to account for nulls, and fill the validity bitmap. */ const int validity_bitmap_bytes = sizeof(uint64) * ((n_total + 64 - 1) / 64); validity_bitmap = MemoryContextAlloc(dest_mctx, validity_bitmap_bytes); /* * First, mark all data as valid, we will fill the nulls later if needed. * Note that the validity bitmap size is a multiple of 64 bits. We have to * fill the tail bits with zeros, because the corresponding elements are not * valid. * */ memset(validity_bitmap, 0xFF, validity_bitmap_bytes); if (n_total % 64) { const uint64 tail_mask = ~0ULL >> (64 - n_total % 64); validity_bitmap[n_total / 64] &= tail_mask; } /* * The number of not-null elements we have must be consistent with the * nulls bitmap. */ CheckCompressedData(n_notnull + simple8brle_bitmap_num_ones(&nulls) == n_total); int current_notnull_element = n_notnull - 1; for (int i = n_total - 1; i >= 0; i--) { Assert(i >= current_notnull_element); if (simple8brle_bitmap_get_at(&nulls, i)) { arrow_set_row_validity(validity_bitmap, i, false); } else { Assert(current_notnull_element >= 0); decompressed_values[i] = decompressed_values[current_notnull_element]; current_notnull_element--; } } Assert(current_notnull_element == -1); } /* Return the result. */ ArrowArray *result = MemoryContextAllocZero(dest_mctx, sizeof(ArrowArray) + sizeof(void *) * 2); const void **buffers = (const void **) &result[1]; buffers[0] = validity_bitmap; buffers[1] = decompressed_values; result->n_buffers = 2; result->buffers = buffers; result->length = n_total; result->null_count = n_total - n_notnull; return result; } #undef FUNCTION_NAME #undef FUNCTION_NAME_HELPER ================================================ FILE: tsl/src/compression/algorithms/dictionary.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/htup_details.h> #include <access/tupmacs.h> #include <catalog/namespace.h> #include <catalog/pg_aggregate.h> #include <catalog/pg_type.h> #include <common/base64.h> #include <funcapi.h> #include <lib/stringinfo.h> #include <utils/builtins.h> #include <utils/datum.h> #include <utils/lsyscache.h> #include <utils/syscache.h> #include <utils/typcache.h> #include <utils/uuid.h> #include "array.h" #include "compression/arrow_c_data_interface.h" #include "compression/compression.h" #include "datum_serialize.h" #include "dictionary.h" #include "dictionary_hash.h" #include "guc.h" #include "simple8b_rle.h" #include "simple8b_rle_bitarray.h" #include "simple8b_rle_bitmap.h" /* * A compression bitmap is stored as * bool has_nulls * padding * Oid element_type: the element stored by this compressed dictionary * uint32 num_distinct: the number of distinct values * simple8b_rle dictionary indexes: array of mappings from row to index into dictionary items * ArrayCompressed simple8b_rle nulls (optional) ArrayCompressed dictionary items */ typedef struct DictionaryCompressed { CompressedDataHeaderFields; uint8 has_nulls; uint8 padding[2]; Oid element_type; uint32 num_distinct; /* 8-byte alignment sentinel for the following fields */ uint64 alignment_sentinel[FLEXIBLE_ARRAY_MEMBER]; } DictionaryCompressed; bool dictionary_compressed_has_nulls(const CompressedDataHeader *header) { const DictionaryCompressed *dc = (const DictionaryCompressed *) header; return dc->has_nulls; } static void pg_attribute_unused() assertions(void) { DictionaryCompressed test_val; /* make sure no padding bytes make it to disk */ StaticAssertStmt(sizeof(DictionaryCompressed) == sizeof(test_val.vl_len_) + sizeof(test_val.compression_algorithm) + sizeof(test_val.has_nulls) + sizeof(test_val.padding) + sizeof(test_val.element_type) + sizeof(test_val.num_distinct), "CompressedDictionary wrong size"); StaticAssertStmt(sizeof(DictionaryCompressed) == 16, "CompressedDictionary wrong size"); } struct DictionaryDecompressionIterator { DecompressionIterator base; const DictionaryCompressed *compressed; Datum *values; Simple8bRleDecompressionIterator bitmap; Simple8bRleDecompressionIterator nulls; bool has_nulls; }; ////////////////// /// Compressor /// ////////////////// typedef struct DictionaryCompressor { dictionary_hash *dictionary_items; uint32 next_index; uint32 dict_val_size; Oid type; int16 typlen; bool typbyval; char typalign; bool has_nulls; DatumSerializer *serializer; Simple8bRleCompressor dictionary_indexes; Simple8bRleCompressor nulls; } DictionaryCompressor; typedef struct ExtendedCompressor { Compressor base; DictionaryCompressor *internal; Oid element_type; } ExtendedCompressor; static void dictionary_compressor_append_datum(Compressor *compressor, Datum val) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = dictionary_compressor_alloc(extended->element_type); dictionary_compressor_append(extended->internal, val); } static void dictionary_compressor_append_null_value(Compressor *compressor) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = dictionary_compressor_alloc(extended->element_type); dictionary_compressor_append_null(extended->internal); } static bool dictionary_compressor_is_full(Compressor *compressor, Datum val) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = dictionary_compressor_alloc(extended->element_type); Size datum_size_and_align; DictionaryCompressor *dict_comp = (DictionaryCompressor *) extended->internal; if (datum_serializer_value_may_be_toasted(dict_comp->serializer)) val = PointerGetDatum(PG_DETOAST_DATUM_PACKED(val)); datum_size_and_align = datum_get_bytes_size(dict_comp->serializer, dict_comp->dict_val_size, val) - dict_comp->dict_val_size; /* If we can't fit new datum in the max size, we are full */ return (datum_size_and_align + dict_comp->dict_val_size) > MAX_ARRAY_COMPRESSOR_SIZE_BYTES; } static void * dictionary_compressor_finish_and_reset(Compressor *compressor) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; void *compressed = dictionary_compressor_finish(extended->internal); pfree(extended->internal); extended->internal = NULL; return compressed; } const Compressor dictionary_compressor = { .append_val = dictionary_compressor_append_datum, .append_null = dictionary_compressor_append_null_value, .is_full = dictionary_compressor_is_full, .finish = dictionary_compressor_finish_and_reset, }; Compressor * dictionary_compressor_for_type(Oid element_type) { ExtendedCompressor *compressor = palloc(sizeof(*compressor)); *compressor = (ExtendedCompressor){ .base = dictionary_compressor, .element_type = element_type, }; return &compressor->base; } DictionaryCompressor * dictionary_compressor_alloc(Oid type) { DictionaryCompressor *compressor = palloc(sizeof(*compressor)); TypeCacheEntry *tentry = lookup_type_cache(type, TYPECACHE_EQ_OPR_FINFO | TYPECACHE_HASH_PROC_FINFO); compressor->next_index = 0; compressor->dict_val_size = 0; compressor->has_nulls = false; compressor->type = type; compressor->typlen = tentry->typlen; compressor->typbyval = tentry->typbyval; compressor->typalign = tentry->typalign; compressor->dictionary_items = dictionary_hash_alloc(tentry); compressor->serializer = create_datum_serializer(type); simple8brle_compressor_init(&compressor->dictionary_indexes); simple8brle_compressor_init(&compressor->nulls); return compressor; } void dictionary_compressor_append_null(DictionaryCompressor *compressor) { compressor->has_nulls = true; simple8brle_compressor_append(&compressor->nulls, 1); } void dictionary_compressor_append(DictionaryCompressor *compressor, Datum val) { bool found; DictionaryHashItem *dict_item; Assert(compressor != NULL); if (datum_serializer_value_may_be_toasted(compressor->serializer)) val = PointerGetDatum(PG_DETOAST_DATUM_PACKED(val)); dict_item = dictionary_insert(compressor->dictionary_items, val, &found); if (!found) { // per_val->bitmap = roaring_dictionary_create(); dict_item->index = compressor->next_index; dict_item->key = datumCopy(val, compressor->typbyval, compressor->typlen); Assert(compressor->next_index <= INT16_MAX - 1); compressor->next_index += 1; } Size datum_size_and_align = datum_get_bytes_size(compressor->serializer, compressor->dict_val_size, val) - compressor->dict_val_size; compressor->dict_val_size += datum_size_and_align; simple8brle_compressor_append(&compressor->dictionary_indexes, dict_item->index); simple8brle_compressor_append(&compressor->nulls, 0); } typedef struct DictionaryCompressorSerializationInfo { Size bitmaps_size; Size nulls_size; Size dictionary_size; Size total_size; uint32 num_distinct; Simple8bRleSerialized *dictionary_compressed_indexes; Simple8bRleSerialized *compressed_nulls; Datum *value_array; /* same as dictionary_serialization_info just as a regular array */ ArrayCompressorSerializationInfo *dictionary_serialization_info; bool is_all_null; } DictionaryCompressorSerializationInfo; static DictionaryCompressorSerializationInfo compressor_get_serialization_info(DictionaryCompressor *compressor) { Simple8bRleSerialized *dict_indexes = simple8brle_compressor_finish(&compressor->dictionary_indexes); Simple8bRleSerialized *nulls = simple8brle_compressor_finish(&compressor->nulls); dictionary_iterator dictionary_item_iterator; ArrayCompressor *array_comp = array_compressor_alloc(compressor->type); /* the total size is header size + bitmaps size + nulls? + data sizesize */ DictionaryCompressorSerializationInfo sizes = { .dictionary_compressed_indexes = dict_indexes, .compressed_nulls = nulls, .value_array = palloc(compressor->next_index * sizeof(Datum)) }; Size header_size = sizeof(DictionaryCompressed); if (sizes.dictionary_compressed_indexes == NULL) return (DictionaryCompressorSerializationInfo){ .is_all_null = true }; sizes.bitmaps_size = simple8brle_serialized_total_size(dict_indexes); sizes.total_size = MAXALIGN(header_size) + sizes.bitmaps_size; if (compressor->has_nulls) sizes.nulls_size = simple8brle_serialized_total_size(nulls); sizes.total_size += sizes.nulls_size; dictionary_start_iterate(compressor->dictionary_items, &dictionary_item_iterator); sizes.num_distinct = 0; for (DictionaryHashItem *dict_item = dictionary_iterate(compressor->dictionary_items, &dictionary_item_iterator); dict_item != NULL; dict_item = dictionary_iterate(compressor->dictionary_items, &dictionary_item_iterator)) { sizes.value_array[dict_item->index] = dict_item->key; sizes.num_distinct += 1; } for (uint32 i = 0; i < sizes.num_distinct; i++) { array_compressor_append(array_comp, sizes.value_array[i]); } sizes.dictionary_serialization_info = array_compressor_get_serialization_info(array_comp); sizes.dictionary_size = array_compression_serialization_size(sizes.dictionary_serialization_info); sizes.total_size += sizes.dictionary_size; if (!AllocSizeIsValid(sizes.total_size)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("compressed size exceeds the maximum allowed (%d)", (int) MaxAllocSize))); return sizes; } static DictionaryCompressed * dictionary_compressed_from_serialization_info(DictionaryCompressorSerializationInfo sizes, Oid element_type) { char *data = palloc0(sizes.total_size); DictionaryCompressed *bitmap = (DictionaryCompressed *) data; SET_VARSIZE(bitmap->vl_len_, sizes.total_size); bitmap->compression_algorithm = COMPRESSION_ALGORITHM_DICTIONARY; bitmap->element_type = element_type; bitmap->has_nulls = sizes.nulls_size > 0 ? 1 : 0; bitmap->num_distinct = sizes.num_distinct; data = data + sizeof(DictionaryCompressed); data = bytes_serialize_simple8b_and_advance(data, sizes.bitmaps_size, sizes.dictionary_compressed_indexes); if (bitmap->has_nulls) data = bytes_serialize_simple8b_and_advance(data, sizes.nulls_size, sizes.compressed_nulls); data = bytes_serialize_array_compressor_and_advance(data, sizes.dictionary_size, sizes.dictionary_serialization_info); Assert((Size) (data - (char *) bitmap) == sizes.total_size); return bitmap; } static void dictionary_decompression_iterator_init(DictionaryDecompressionIterator *iter, const char *data, bool scan_forward, Oid element_type); /* there are more efficient ways to do this that use * DictionaryCompressorSerializationInfo, but they are not worth implementing * yet */ static ArrayCompressed * dictionary_compressed_to_array_compressed(DictionaryCompressed *compressed) { ArrayCompressor *compressor = array_compressor_alloc(compressed->element_type); DictionaryDecompressionIterator iterator; dictionary_decompression_iterator_init(&iterator, (void *) compressed, true, compressed->element_type); for (DecompressResult res = dictionary_decompression_iterator_try_next_forward(&iterator.base); !res.is_done; res = dictionary_decompression_iterator_try_next_forward(&iterator.base)) { if (res.is_null) array_compressor_append_null(compressor); else array_compressor_append(compressor, res.val); } return array_compressor_finish(compressor); } void * dictionary_compressor_finish(DictionaryCompressor *compressor) { uint64 average_element_size; uint64 expected_array_size; DictionaryCompressed *compressed; DictionaryCompressorSerializationInfo sizes = compressor_get_serialization_info(compressor); if (sizes.is_all_null) return NULL; Assert(0 != sizes.num_distinct); /* calculate what the expected size would have be if we recompressed this as * an array, if this is smaller than the current size, recompress as an array. */ average_element_size = sizes.dictionary_size / sizes.num_distinct; expected_array_size = average_element_size * sizes.dictionary_compressed_indexes->num_elements; compressed = dictionary_compressed_from_serialization_info(sizes, compressor->type); if (expected_array_size < sizes.total_size) return dictionary_compressed_to_array_compressed(compressed); return compressed; } //////////////////// /// Decompressor /// //////////////////// static void dictionary_decompression_iterator_init(DictionaryDecompressionIterator *iter, const char *_data, bool scan_forward, Oid element_type) { StringInfoData si = { .data = (char *) _data, .len = VARSIZE(_data) }; const DictionaryCompressed *bitmap = consumeCompressedData(&si, sizeof(DictionaryCompressed)); Simple8bRleSerialized *s8_bitmap; DecompressionIterator *dictionary_iterator; *iter = (DictionaryDecompressionIterator){ .base = { .compression_algorithm = COMPRESSION_ALGORITHM_DICTIONARY, .forward = scan_forward, .element_type = element_type, .try_next = (scan_forward ? dictionary_decompression_iterator_try_next_forward : dictionary_decompression_iterator_try_next_reverse), }, .compressed = bitmap, .values = palloc(sizeof(Datum) * bitmap->num_distinct), .has_nulls = bitmap->has_nulls == 1, }; s8_bitmap = bytes_deserialize_simple8b_and_advance(&si); if (scan_forward) simple8brle_decompression_iterator_init_forward(&iter->bitmap, s8_bitmap); else simple8brle_decompression_iterator_init_reverse(&iter->bitmap, s8_bitmap); if (iter->has_nulls) { Simple8bRleSerialized *s8_null = bytes_deserialize_simple8b_and_advance(&si); if (scan_forward) simple8brle_decompression_iterator_init_forward(&iter->nulls, s8_null); else simple8brle_decompression_iterator_init_reverse(&iter->nulls, s8_null); } dictionary_iterator = array_decompression_iterator_alloc_forward(&si, bitmap->element_type, /* has_nulls */ false); for (uint32 i = 0; i < bitmap->num_distinct; i++) { DecompressResult res = array_decompression_iterator_try_next_forward(dictionary_iterator); Assert(!res.is_null); Assert(!res.is_done); iter->values[i] = res.val; } Assert(array_decompression_iterator_try_next_forward(dictionary_iterator).is_done); } static ArrowArray *tsl_bool_dictionary_decompress_all(Datum compressed, Oid element_type, MemoryContext dest_mctx); static ArrowArray *tsl_text_dictionary_decompress_all(Datum compressed, Oid element_type, MemoryContext dest_mctx); static ArrowArray *tsl_uuid_dictionary_decompress_all(Datum compressed, Oid element_type, MemoryContext dest_mctx); /* Pass through to the specialized functions below for BOOL, TEXT and UUID */ ArrowArray * tsl_dictionary_decompress_all(Datum compressed, Oid element_type, MemoryContext dest_mctx) { switch (element_type) { case BOOLOID: return tsl_bool_dictionary_decompress_all(compressed, element_type, dest_mctx); case TEXTOID: return tsl_text_dictionary_decompress_all(compressed, element_type, dest_mctx); case UUIDOID: return tsl_uuid_dictionary_decompress_all(compressed, element_type, dest_mctx); default: elog(ERROR, "unsupported dictionary type %u for bulk decompression", element_type); break; } return NULL; } static ArrowArray * tsl_bool_dictionary_decompress_all(Datum compressed, Oid element_type, MemoryContext dest_mctx) { Assert(element_type == BOOLOID); compressed = PointerGetDatum(PG_DETOAST_DATUM(compressed)); StringInfoData si = { .data = DatumGetPointer(compressed), .len = VARSIZE(compressed) }; const DictionaryCompressed *header = consumeCompressedData(&si, sizeof(DictionaryCompressed)); Assert(header->compression_algorithm == COMPRESSION_ALGORITHM_DICTIONARY); CheckCompressedData(header->element_type == BOOLOID); Simple8bRleSerialized *indices_serialized = bytes_deserialize_simple8b_and_advance(&si); Simple8bRleSerialized *nulls_serialized = NULL; if (header->has_nulls) { nulls_serialized = bytes_deserialize_simple8b_and_advance(&si); } const uint32 n_notnull = indices_serialized->num_elements; const uint32 n_total = header->has_nulls ? nulls_serialized->num_elements : n_notnull; const uint32 n_padded_bits = n_total + 63; const uint32 n_padded_bytes = n_padded_bits / 8; uint64 *validity_bitmap = NULL; uint64 *values = MemoryContextAllocZero(dest_mctx, n_padded_bytes); MemoryContext old_context = MemoryContextSwitchTo(dest_mctx); /* Decompress the nulls */ Simple8bRleBitArray validity_bits = simple8brle_bitarray_decompress(nulls_serialized, /* inverted*/ true); validity_bitmap = validity_bits.data; MemoryContextSwitchTo(old_context); if (header->has_nulls) { CheckCompressedData(validity_bits.num_ones == n_notnull); CheckCompressedData(validity_bits.num_elements == n_total); } /* Decompress the values using the iterator based decompressor */ { int position = 0; DecompressionIterator *iter = tsl_dictionary_decompression_iterator_from_datum_forward(compressed, BOOLOID); for (DecompressResult r = dictionary_decompression_iterator_try_next_forward(iter); !r.is_done; r = dictionary_decompression_iterator_try_next_forward(iter)) { if (!r.is_null) { bool data = DatumGetBool(r.val) == true; if (data) { arrow_set_row_validity(values, position, true); } } ++position; } } ArrowArray *result = MemoryContextAllocZero(dest_mctx, sizeof(ArrowArray) + (sizeof(void *) * 2)); const void **buffers = (const void **) &result[1]; buffers[0] = validity_bitmap; buffers[1] = values; result->n_buffers = 2; result->buffers = buffers; result->length = n_total; result->null_count = n_total - n_notnull; return result; } #define ELEMENT_TYPE int16 #include "simple8b_rle_decompress_all.h" #undef ELEMENT_TYPE static ArrowArray * tsl_uuid_dictionary_decompress_all(Datum compressed, Oid element_type, MemoryContext dest_mctx) { Assert(element_type == UUIDOID); compressed = PointerGetDatum(PG_DETOAST_DATUM(compressed)); StringInfoData si = { .data = DatumGetPointer(compressed), .len = VARSIZE_ANY(compressed) }; const DictionaryCompressed *header = consumeCompressedData(&si, sizeof(DictionaryCompressed)); Assert(header->compression_algorithm == COMPRESSION_ALGORITHM_DICTIONARY); CheckCompressedData(header->element_type == UUIDOID); Simple8bRleSerialized *indices_serialized = bytes_deserialize_simple8b_and_advance(&si); Simple8bRleSerialized *nulls_serialized = NULL; if (header->has_nulls) { nulls_serialized = bytes_deserialize_simple8b_and_advance(&si); } const uint32 n_notnull = indices_serialized->num_elements; const uint32 n_total = header->has_nulls ? nulls_serialized->num_elements : n_notnull; const uint32 n_bytes = n_total * 16; uint64 *restrict validity_bitmap = NULL; uint64 *restrict values = MemoryContextAllocZero(dest_mctx, n_bytes); MemoryContext old_context = MemoryContextSwitchTo(dest_mctx); /* Decompress the nulls */ Simple8bRleBitArray validity_bits = simple8brle_bitarray_decompress(nulls_serialized, /* inverted*/ true); validity_bitmap = validity_bits.data; MemoryContextSwitchTo(old_context); if (header->has_nulls) { CheckCompressedData(validity_bits.num_ones == n_notnull); CheckCompressedData(validity_bits.num_elements == n_total); } /* create a context so I can throw away all temp data in one step */ MemoryContext temp_context = AllocSetContextCreate(CurrentMemoryContext, "tsl_uuid_dictionary_decompress_all", ALLOCSET_DEFAULT_SIZES); /* This is the padding requirement of simple8brle_decompress_all. */ const uint32 n_padded = n_total + 63; int16 *restrict indices = MemoryContextAlloc(temp_context, sizeof(int16) * n_padded); const uint32 n_decompressed = simple8brle_decompress_all_buf_int16(indices_serialized, indices, n_padded); CheckCompressedData(n_decompressed == n_notnull); /* Don't care about the sizes stored in the Array, just skip over them. */ Simple8bRleSerialized *sizes_serialized = bytes_deserialize_simple8b_and_advance(&si); CheckCompressedData(sizes_serialized->num_elements == header->num_distinct); /* Verify that the remaining size has enough space for the values */ CheckCompressedData((uint32) ((si.len - si.cursor) / 16) >= header->num_distinct); uint64 *restrict dict_values = (uint64 *) (si.data + si.cursor); { int position = 0; for (uint32 i = 0; i < n_total; ++i) { if (arrow_row_is_valid(validity_bitmap, i)) { int16 idx = indices[position]; /* Check that the dictionary indices that we've just read are not out of bounds. */ CheckCompressedData(idx >= 0 && idx < (int16) header->num_distinct); /* Use assignment as both sides are coming from palloc, so it is guaranteed to be * aligned */ values[i * 2] = dict_values[idx * 2]; values[i * 2 + 1] = dict_values[idx * 2 + 1]; position++; } } } MemoryContextDelete(temp_context); ArrowArray *result = MemoryContextAllocZero(dest_mctx, sizeof(ArrowArray) + (sizeof(void *) * 2)); const void **buffers = (const void **) &result[1]; buffers[0] = validity_bitmap; buffers[1] = values; result->n_buffers = 2; result->buffers = buffers; result->length = n_total; result->null_count = n_total - n_notnull; return result; } static ArrowArray * tsl_text_dictionary_decompress_all(Datum compressed, Oid element_type, MemoryContext dest_mctx) { Assert(element_type == TEXTOID); compressed = PointerGetDatum(PG_DETOAST_DATUM(compressed)); StringInfoData si = { .data = DatumGetPointer(compressed), .len = VARSIZE(compressed) }; const DictionaryCompressed *header = consumeCompressedData(&si, sizeof(DictionaryCompressed)); Assert(header->compression_algorithm == COMPRESSION_ALGORITHM_DICTIONARY); CheckCompressedData(header->element_type == TEXTOID); Simple8bRleSerialized *indices_serialized = bytes_deserialize_simple8b_and_advance(&si); Simple8bRleSerialized *nulls_serialized = NULL; if (header->has_nulls) { nulls_serialized = bytes_deserialize_simple8b_and_advance(&si); } const uint32 n_notnull = indices_serialized->num_elements; const uint32 n_total = header->has_nulls ? nulls_serialized->num_elements : n_notnull; CheckCompressedData(n_total >= n_notnull); const uint32 n_padded = n_total + 63; /* This is the padding requirement of simple8brle_decompress_all. */ int16 *restrict indices = MemoryContextAlloc(dest_mctx, sizeof(int16) * n_padded); const uint32 n_decompressed = simple8brle_decompress_all_buf_int16(indices_serialized, indices, n_padded); CheckCompressedData(n_decompressed == n_notnull); /* Check that the dictionary indices that we've just read are not out of bounds. */ CheckCompressedData(header->num_distinct <= GLOBAL_MAX_ROWS_PER_COMPRESSION); /* We use signed indexes as recommended by the Arrow spec. */ CheckCompressedData(header->num_distinct <= INT16_MAX); bool have_incorrect_index = false; for (uint32 i = 0; i < n_notnull; i++) { have_incorrect_index = have_incorrect_index || indices[i] >= (int16) header->num_distinct; } CheckCompressedData(!have_incorrect_index); /* Decompress the actual values in the dictionary. */ ArrowArray *dict = text_array_decompress_all_serialized_no_header(&si, /* has_nulls = */ false, dest_mctx); CheckCompressedData(header->num_distinct == dict->length); uint64 *restrict validity_bitmap = NULL; if (header->has_nulls) { /* Fill validity and indices of the array elements, reshuffling for nulls if needed. */ const int validity_bitmap_bytes = sizeof(uint64) * pad_to_multiple(64, n_total) / 64; validity_bitmap = MemoryContextAlloc(dest_mctx, validity_bitmap_bytes); /* * First, mark all data as valid, we will fill the nulls later if needed. * Note that the validity bitmap size is a multiple of 64 bits. We have to * fill the tail bits with zeros, because the corresponding elements are not * valid. * */ memset(validity_bitmap, 0xFF, validity_bitmap_bytes); if (n_total % 64) { const uint64 tail_mask = ~0ULL >> (64 - n_total % 64); validity_bitmap[n_total / 64] &= tail_mask; } /* * We have decompressed the data with nulls skipped, reshuffle it * according to the nulls bitmap. */ Simple8bRleBitmap nulls = simple8brle_bitmap_decompress(nulls_serialized); CheckCompressedData(n_notnull + simple8brle_bitmap_num_ones(&nulls) == n_total); /* current_notnull_element needs to go below 0, so use signed type */ int64 current_notnull_element = n_notnull - 1; for (int64 i = n_total - 1; i >= 0; i--) { Assert(i >= current_notnull_element); if (simple8brle_bitmap_get_at(&nulls, i)) { arrow_set_row_validity(validity_bitmap, i, false); indices[i] = 0; } else { Assert(current_notnull_element >= 0); indices[i] = indices[current_notnull_element]; current_notnull_element--; } } Assert(current_notnull_element == -1); } ArrowArray *result = MemoryContextAllocZero(dest_mctx, sizeof(ArrowArray) + (sizeof(void *) * 2)); const void **buffers = (const void **) &result[1]; buffers[0] = validity_bitmap; buffers[1] = indices; result->n_buffers = 2; result->buffers = buffers; result->length = n_total; result->null_count = n_total - n_notnull; result->dictionary = dict; return result; } DecompressionIterator * tsl_dictionary_decompression_iterator_from_datum_forward(Datum dictionary_compressed, Oid element_type) { DictionaryDecompressionIterator *iterator = palloc(sizeof(*iterator)); dictionary_decompression_iterator_init(iterator, (void *) PG_DETOAST_DATUM(dictionary_compressed), true, element_type); return &iterator->base; } DecompressionIterator * tsl_dictionary_decompression_iterator_from_datum_reverse(Datum dictionary_compressed, Oid element_type) { DictionaryDecompressionIterator *iterator = palloc(sizeof(*iterator)); dictionary_decompression_iterator_init(iterator, (void *) PG_DETOAST_DATUM(dictionary_compressed), false, element_type); return &iterator->base; } DecompressResult dictionary_decompression_iterator_try_next_forward(DecompressionIterator *iter_base) { DictionaryDecompressionIterator *iter; Simple8bRleDecompressResult result; Assert(iter_base->compression_algorithm == COMPRESSION_ALGORITHM_DICTIONARY && iter_base->forward); iter = (DictionaryDecompressionIterator *) iter_base; if (iter->has_nulls) { Simple8bRleDecompressResult null = simple8brle_decompression_iterator_try_next_forward(&iter->nulls); if (null.is_done) return (DecompressResult){ .is_done = true, }; if ((null.val & 1) != 0) { return (DecompressResult){ .is_null = true, }; } } result = simple8brle_decompression_iterator_try_next_forward(&iter->bitmap); if (result.is_done) return (DecompressResult){ .is_done = true, }; CheckCompressedData(result.val < iter->compressed->num_distinct); return (DecompressResult){ .val = iter->values[result.val], .is_null = false, .is_done = false, }; } DecompressResult dictionary_decompression_iterator_try_next_reverse(DecompressionIterator *iter_base) { DictionaryDecompressionIterator *iter; Simple8bRleDecompressResult result; Assert(iter_base->compression_algorithm == COMPRESSION_ALGORITHM_DICTIONARY && !iter_base->forward); iter = (DictionaryDecompressionIterator *) iter_base; if (iter->has_nulls) { Simple8bRleDecompressResult null = simple8brle_decompression_iterator_try_next_reverse(&iter->nulls); if (null.is_done) return (DecompressResult){ .is_done = true, }; if ((null.val & 1) != 0) { return (DecompressResult){ .is_null = true, }; } } result = simple8brle_decompression_iterator_try_next_reverse(&iter->bitmap); if (result.is_done) return (DecompressResult){ .is_done = true, }; Assert(result.val < iter->compressed->num_distinct); return (DecompressResult){ .val = iter->values[result.val], .is_null = false, .is_done = false, }; } ///////////////////// /// SQL Functions /// ///////////////////// Datum tsl_dictionary_compressor_append(PG_FUNCTION_ARGS) { DictionaryCompressor *compressor = (DictionaryCompressor *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0)); MemoryContext agg_context; MemoryContext old_context; if (!AggCheckCallContext(fcinfo, &agg_context)) { /* cannot be called directly because of internal-type argument */ elog(ERROR, "tsl_dictionary_compressor_append called in non-aggregate context"); } old_context = MemoryContextSwitchTo(agg_context); if (compressor == NULL) { Oid type_to_compress = get_fn_expr_argtype(fcinfo->flinfo, 1); compressor = dictionary_compressor_alloc(type_to_compress); } if (PG_ARGISNULL(1)) dictionary_compressor_append_null(compressor); else dictionary_compressor_append(compressor, PG_GETARG_DATUM(1)); MemoryContextSwitchTo(old_context); PG_RETURN_POINTER(compressor); } Datum tsl_dictionary_compressor_finish(PG_FUNCTION_ARGS) { DictionaryCompressor *compressor = (DictionaryCompressor *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0)); void *compressed; if (compressor == NULL) PG_RETURN_NULL(); compressed = dictionary_compressor_finish(compressor); if (compressed == NULL) PG_RETURN_NULL(); PG_RETURN_POINTER(compressed); } ///////////////////// /// I/O Functions /// ///////////////////// void dictionary_compressed_send(CompressedDataHeader *header, StringInfo buffer) { uint32 data_size; uint32 size; const DictionaryCompressed *compressed_header; const char *compressed_data; Assert(header->compression_algorithm == COMPRESSION_ALGORITHM_DICTIONARY); compressed_header = (DictionaryCompressed *) header; compressed_data = (char *) compressed_header; compressed_data += sizeof(*compressed_header); data_size = VARSIZE(compressed_header); data_size -= sizeof(*compressed_header); pq_sendbyte(buffer, compressed_header->has_nulls == true); type_append_to_binary_string(compressed_header->element_type, buffer); size = simple8brle_serialized_total_size((void *) compressed_data); simple8brle_serialized_send(buffer, (void *) compressed_data); compressed_data += size; data_size -= size; if (compressed_header->has_nulls) { uint32 size = simple8brle_serialized_total_size((void *) compressed_data); simple8brle_serialized_send(buffer, (void *) compressed_data); compressed_data += size; data_size -= size; } array_compressed_data_send(buffer, compressed_data, data_size, compressed_header->element_type, false); } Datum dictionary_compressed_recv(StringInfo buffer) { DictionaryCompressorSerializationInfo info = { 0 }; uint8 has_nulls; Oid element_type; has_nulls = pq_getmsgbyte(buffer); CheckCompressedData(has_nulls == 0 || has_nulls == 1); element_type = binary_string_get_type(buffer); info.dictionary_compressed_indexes = simple8brle_serialized_recv(buffer); info.bitmaps_size = simple8brle_serialized_total_size(info.dictionary_compressed_indexes); info.total_size = MAXALIGN(sizeof(DictionaryCompressed)) + info.bitmaps_size; if (has_nulls) { info.compressed_nulls = simple8brle_serialized_recv(buffer); info.nulls_size = simple8brle_serialized_total_size(info.compressed_nulls); info.total_size += info.nulls_size; } info.dictionary_serialization_info = array_compressed_data_recv(buffer, element_type); CheckCompressedData(info.dictionary_serialization_info != NULL); info.dictionary_size = array_compression_serialization_size(info.dictionary_serialization_info); info.total_size += info.dictionary_size; info.num_distinct = array_compression_serialization_num_elements(info.dictionary_serialization_info); if (!AllocSizeIsValid(info.total_size)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("compressed size exceeds the maximum allowed (%d)", (int) MaxAllocSize))); return PointerGetDatum(dictionary_compressed_from_serialization_info(info, element_type)); } ================================================ FILE: tsl/src/compression/algorithms/dictionary.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once /* * The Dictionary compressions scheme can store any type of data but is optimized for * low-cardinality data sets. The dictionary of distinct items is stored as an `array` compressed * object. The row->dictionary item mapping is stored as a series of integer-based indexes into the * dictionary array ordered by row number (called dictionary_indexes; compressed using * `simple8b_rle`). */ #include <postgres.h> #include "compression/compression.h" #include <lib/stringinfo.h> #include <fmgr.h> typedef struct DictionaryCompressor DictionaryCompressor; typedef struct DictionaryCompressed DictionaryCompressed; typedef struct DictionaryDecompressionIterator DictionaryDecompressionIterator; extern bool dictionary_compressed_has_nulls(const CompressedDataHeader *header); extern Compressor *dictionary_compressor_for_type(Oid element_type); extern DictionaryCompressor *dictionary_compressor_alloc(Oid type_to_compress); extern void dictionary_compressor_append_null(DictionaryCompressor *compressor); extern void dictionary_compressor_append(DictionaryCompressor *compressor, Datum val); extern void *dictionary_compressor_finish(DictionaryCompressor *compressor); extern DecompressionIterator * tsl_dictionary_decompression_iterator_from_datum_forward(Datum dictionary_compressed, Oid element_type); extern DecompressResult dictionary_decompression_iterator_try_next_forward(DecompressionIterator *iter); extern DecompressionIterator * tsl_dictionary_decompression_iterator_from_datum_reverse(Datum dictionary_compressed, Oid element_type); extern DecompressResult dictionary_decompression_iterator_try_next_reverse(DecompressionIterator *iter); extern void dictionary_compressed_send(CompressedDataHeader *header, StringInfo buffer); extern Datum dictionary_compressed_recv(StringInfo buf); extern Datum tsl_dictionary_compressor_append(PG_FUNCTION_ARGS); extern Datum tsl_dictionary_compressor_finish(PG_FUNCTION_ARGS); /* Pass through to the specialized functions below for BOOL and TEXT */ ArrowArray *tsl_dictionary_decompress_all(Datum compressed, Oid element_type, MemoryContext dest_mctx); #define DICTIONARY_ALGORITHM_DEFINITION \ { \ .iterator_init_forward = tsl_dictionary_decompression_iterator_from_datum_forward, \ .iterator_init_reverse = tsl_dictionary_decompression_iterator_from_datum_reverse, \ .compressed_data_send = dictionary_compressed_send, \ .compressed_data_recv = dictionary_compressed_recv, \ .compressor_for_type = dictionary_compressor_for_type, \ .compressed_data_storage = TOAST_STORAGE_EXTENDED, \ .decompress_all = tsl_dictionary_decompress_all, \ } ================================================ FILE: tsl/src/compression/algorithms/dictionary_hash.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once /* * The Dictionary compressions scheme can store any type of data but is optimized for * low-cardinality data sets. The dictionary of distinct items is stored as an `array` compressed * object. The row->dictionary item mapping is stored as a series of integer-based indexes into the * dictionary array ordered by row number (called dictionary_indexes; compressed using * `simple8b_rle`). */ #include <postgres.h> #include <funcapi.h> #include <utils/typcache.h> #include "compat/compat.h" typedef struct HashMeta { FunctionCallInfo hash_info; FunctionCallInfo eq_info; } HashMeta; typedef struct DictionaryHashItem { Datum key; /* hash entry status */ uint32 hash; uint16 status; uint16 index; } DictionaryHashItem; typedef struct dictionary_hash dictionary_hash; static uint32 datum_hash(dictionary_hash *tb, Datum key); static bool datum_eq(dictionary_hash *tb, Datum a, Datum b); #define SH_PREFIX dictionary #define SH_ELEMENT_TYPE DictionaryHashItem #define SH_KEY_TYPE Datum #define SH_KEY key #define SH_HASH_KEY(tb, key) datum_hash(tb, key) #define SH_EQUAL(tb, a, b) datum_eq(tb, a, b) #define SH_STORE_HASH #define SH_GET_HASH(tb, entry) entry->hash #define SH_SCOPE static inline #define SH_DEFINE #define SH_DECLARE #include "lib/simplehash.h" static uint32 datum_hash(dictionary_hash *tb, Datum key) { HashMeta *meta = (HashMeta *) tb->private_data; FunctionCallInfo fcinfo = meta->hash_info; Datum value; FC_SET_ARG(fcinfo, 0, key); fcinfo->isnull = false; value = FunctionCallInvoke(fcinfo); Assert(!fcinfo->isnull); return DatumGetUInt32(value); } static bool datum_eq(dictionary_hash *tb, Datum a, Datum b) { HashMeta *meta = (HashMeta *) tb->private_data; FunctionCallInfo fcinfo = meta->eq_info; Datum value; FC_SET_ARG(fcinfo, 0, a); FC_SET_ARG(fcinfo, 1, b); fcinfo->isnull = false; value = FunctionCallInvoke(fcinfo); Assert(!fcinfo->isnull); return DatumGetBool(value); } static dictionary_hash * dictionary_hash_alloc(TypeCacheEntry *tentry) { HashMeta *meta = palloc(sizeof(*meta)); Oid collation = InvalidOid; collation = tentry->typcollation; if (tentry->hash_proc_finfo.fn_addr == NULL || tentry->eq_opr_finfo.fn_addr == NULL) elog(ERROR, "invalid type for dictionary compression, type must have both a hash function and " "equality function"); /* May be more correct to get collation defined on the column, which may be different than the * collation defined on the type (what we're currently using). We need to think about * backwards compatibility, and different collations. Should only affect compression ratios * anyway. */ meta->eq_info = HEAP_FCINFO(2); InitFunctionCallInfoData(*meta->eq_info, &tentry->eq_opr_finfo, 2, collation, NULL, NULL); meta->hash_info = HEAP_FCINFO(2); InitFunctionCallInfoData(*meta->hash_info, &tentry->hash_proc_finfo, 1, collation, NULL, NULL); return dictionary_create(CurrentMemoryContext, 10, meta); } ================================================ FILE: tsl/src/compression/algorithms/float_utils.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> static inline uint32 float_get_bits(float in) { uint32 out; StaticAssertStmt(sizeof(float) == sizeof(uint32), "float is not IEEE double wide float"); /* yes, this is the correct way to extract the bits of a floating point number in C */ memcpy(&out, &in, sizeof(uint32)); return out; } static pg_attribute_always_inline float bits_get_float(uint32 bits) { float out; StaticAssertStmt(sizeof(float) == sizeof(uint32), "float is not IEEE double wide float"); /* yes, this is the correct way to extract the bits of a floating point number in C */ memcpy(&out, &bits, sizeof(uint32)); return out; } static inline uint64 double_get_bits(double in) { uint64 out; StaticAssertStmt(sizeof(uint64) == sizeof(double), "double is not IEEE double wide float"); /* yes, this is the correct way to extract the bits of a floating point number in C */ memcpy(&out, &in, sizeof(uint64)); return out; } static pg_attribute_always_inline double bits_get_double(uint64 bits) { double out; StaticAssertStmt(sizeof(uint64) == sizeof(double), "double is not IEEE double wide float"); /* yes, this is the correct way to extract the bits of a floating point number in C */ memcpy(&out, &bits, sizeof(double)); return out; } ================================================ FILE: tsl/src/compression/algorithms/gorilla.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/htup_details.h> #include <catalog/pg_type.h> #include <common/base64.h> #include <funcapi.h> #include <lib/stringinfo.h> #include <libpq/pqformat.h> #include <port/pg_bitutils.h> #include <utils/builtins.h> #include <utils/memutils.h> #include "gorilla.h" #include "compression/arrow_c_data_interface.h" #include "compression/compression.h" #include "float_utils.h" #include "guc.h" #include "simple8b_rle.h" #include "simple8b_rle_bitmap.h" #include "adts/bit_array.h" /* * Gorilla compressed data is stored as * uint16 compression_algorithm: id number for the compression scheme * uint8 has_nulls: 1 if we store a NULLs bitmap after the data, otherwise 0 * uint8 bits_used_in_last_xor_bucket: number of bits used in the last bucket * uint64 last_val: the last double stored, as bits * simple8b_rle tag0: array of first tag bits (as in gorilla), also stores nelems * simple8b_rle tag1: array of second tag bits (as in gorilla) * BitArray leading_zeros: array of leading zeroes before the xor (as in gorilla) * simple8b_rle num_bits_used: number of bits used for each xor (as in gorilla) * BitArray xors: array xor values (as in gorilla) * simple8b_rle nulls: 1 if the value is NULL, else 0 */ typedef struct GorillaCompressed { CompressedDataHeaderFields; uint8 has_nulls; /* we only use one bit for has_nulls, the rest can be reused */ uint8 bits_used_in_last_xor_bucket; uint8 bits_used_in_last_leading_zeros_bucket; uint32 num_leading_zeroes_buckets; uint32 num_xor_buckets; uint64 last_value; } GorillaCompressed; #define BITS_PER_LEADING_ZEROS 6 /* expanded version of the compressed data */ typedef struct CompressedGorillaData { const GorillaCompressed *header; Simple8bRleSerialized *tag0s; Simple8bRleSerialized *tag1s; BitArray leading_zeros; Simple8bRleSerialized *num_bits_used_per_xor; BitArray xors; Simple8bRleSerialized *nulls; /* NULL if no nulls */ } CompressedGorillaData; bool gorilla_compressed_has_nulls(const CompressedDataHeader *header) { const GorillaCompressed *gc = (const GorillaCompressed *) header; return gc->has_nulls; } static void pg_attribute_unused() assertions(void) { GorillaCompressed test_val = { .vl_len_ = { 0 } }; /* make sure no padding bytes make it to disk */ StaticAssertStmt(sizeof(GorillaCompressed) == sizeof(test_val.vl_len_) + sizeof(test_val.compression_algorithm) + sizeof(test_val.has_nulls) + sizeof(test_val.bits_used_in_last_xor_bucket) + sizeof(test_val.bits_used_in_last_leading_zeros_bucket) + sizeof(test_val.num_leading_zeroes_buckets) + sizeof(test_val.num_xor_buckets) + sizeof(test_val.last_value), "Gorilla wrong size"); StaticAssertStmt(sizeof(GorillaCompressed) == 24, "Gorilla wrong size"); } typedef struct GorillaCompressor { // NOTE it is a small win to replace these next two with specialized RLE bitmaps Simple8bRleCompressor tag0s; Simple8bRleCompressor tag1s; BitArray leading_zeros; Simple8bRleCompressor bits_used_per_xor; BitArray xors; Simple8bRleCompressor nulls; uint64 prev_val; uint8 prev_leading_zeroes; uint8 prev_trailing_zeros; bool has_nulls; } GorillaCompressor; typedef struct ExtendedCompressor { Compressor base; GorillaCompressor *internal; } ExtendedCompressor; typedef struct GorillaDecompressionIterator { DecompressionIterator base; CompressedGorillaData gorilla_data; Simple8bRleDecompressionIterator tag0s; Simple8bRleDecompressionIterator tag1s; BitArrayIterator leading_zeros; Simple8bRleDecompressionIterator num_bits_used; BitArrayIterator xors; Simple8bRleDecompressionIterator nulls; uint64 prev_val; uint8 prev_leading_zeroes; uint8 prev_xor_bits_used; bool has_nulls; } GorillaDecompressionIterator; /******************** *** Compressor *** ********************/ static void gorilla_compressor_append_float(Compressor *compressor, Datum val) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; uint64 value = float_get_bits(DatumGetFloat4(val)); if (extended->internal == NULL) extended->internal = gorilla_compressor_alloc(); gorilla_compressor_append_value(extended->internal, value); } static void gorilla_compressor_append_double(Compressor *compressor, Datum val) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; uint64 value = double_get_bits(DatumGetFloat8(val)); if (extended->internal == NULL) extended->internal = gorilla_compressor_alloc(); gorilla_compressor_append_value(extended->internal, value); } static void gorilla_compressor_append_int16(Compressor *compressor, Datum val) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = gorilla_compressor_alloc(); gorilla_compressor_append_value(extended->internal, (uint16) DatumGetInt16(val)); } static void gorilla_compressor_append_int32(Compressor *compressor, Datum val) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = gorilla_compressor_alloc(); gorilla_compressor_append_value(extended->internal, (uint32) DatumGetInt32(val)); } static void gorilla_compressor_append_int64(Compressor *compressor, Datum val) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = gorilla_compressor_alloc(); gorilla_compressor_append_value(extended->internal, DatumGetInt64(val)); } static void gorilla_compressor_append_null_value(Compressor *compressor) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = gorilla_compressor_alloc(); gorilla_compressor_append_null(extended->internal); } static void * gorilla_compressor_finish_and_reset(Compressor *compressor) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; void *compressed = gorilla_compressor_finish(extended->internal); pfree(extended->internal); extended->internal = NULL; return compressed; } const Compressor gorilla_float_compressor = { .append_val = gorilla_compressor_append_float, .append_null = gorilla_compressor_append_null_value, .is_full = NULL, .finish = gorilla_compressor_finish_and_reset, }; const Compressor gorilla_double_compressor = { .append_val = gorilla_compressor_append_double, .append_null = gorilla_compressor_append_null_value, .is_full = NULL, .finish = gorilla_compressor_finish_and_reset, }; const Compressor gorilla_uint16_compressor = { .append_val = gorilla_compressor_append_int16, .append_null = gorilla_compressor_append_null_value, .is_full = NULL, .finish = gorilla_compressor_finish_and_reset, }; const Compressor gorilla_uint32_compressor = { .append_val = gorilla_compressor_append_int32, .append_null = gorilla_compressor_append_null_value, .is_full = NULL, .finish = gorilla_compressor_finish_and_reset, }; const Compressor gorilla_uint64_compressor = { .append_val = gorilla_compressor_append_int64, .append_null = gorilla_compressor_append_null_value, .is_full = NULL, .finish = gorilla_compressor_finish_and_reset, }; Compressor * gorilla_compressor_for_type(Oid element_type) { ExtendedCompressor *compressor = palloc(sizeof(*compressor)); switch (element_type) { case FLOAT4OID: *compressor = (ExtendedCompressor){ .base = gorilla_float_compressor }; return &compressor->base; case FLOAT8OID: *compressor = (ExtendedCompressor){ .base = gorilla_double_compressor }; return &compressor->base; case INT2OID: *compressor = (ExtendedCompressor){ .base = gorilla_uint16_compressor }; return &compressor->base; case INT4OID: *compressor = (ExtendedCompressor){ .base = gorilla_uint32_compressor }; return &compressor->base; case INT8OID: *compressor = (ExtendedCompressor){ .base = gorilla_uint64_compressor }; return &compressor->base; default: elog(ERROR, "invalid type for Gorilla compression \"%s\"", format_type_be(element_type)); } pg_unreachable(); } GorillaCompressor * gorilla_compressor_alloc(void) { GorillaCompressor *compressor = palloc(sizeof(*compressor)); simple8brle_compressor_init(&compressor->tag0s); simple8brle_compressor_init(&compressor->tag1s); /* * The number of leading zeros takes about 5 bits to encode, and changes * maybe every 100 rows, so use this as a conservative estimate. */ bit_array_init(&compressor->leading_zeros, /* expected_bits = */ (GLOBAL_MAX_ROWS_PER_COMPRESSION * 5) / 100); simple8brle_compressor_init(&compressor->bits_used_per_xor); /* * We typically see about 12 bits or 4 decimal digits per row for the "xors" * part in gorilla compression. */ bit_array_init(&compressor->xors, /* expected_bits = */ GLOBAL_MAX_ROWS_PER_COMPRESSION * 12); simple8brle_compressor_init(&compressor->nulls); compressor->has_nulls = false; compressor->prev_leading_zeroes = 0; compressor->prev_trailing_zeros = 0; compressor->prev_val = 0; return compressor; } /* This function is used for testing only. */ Datum tsl_gorilla_compressor_append(PG_FUNCTION_ARGS) { MemoryContext old_context; MemoryContext agg_context; Compressor *compressor = (Compressor *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0)); if (!AggCheckCallContext(fcinfo, &agg_context)) { /* cannot be called directly because of internal-type argument */ elog(ERROR, "tsl_gorilla_compressor_append called in non-aggregate context"); } old_context = MemoryContextSwitchTo(agg_context); if (compressor == NULL) { compressor = gorilla_compressor_for_type(get_fn_expr_argtype(fcinfo->flinfo, 1)); } if (PG_ARGISNULL(1)) compressor->append_null(compressor); else { compressor->append_val(compressor, PG_GETARG_DATUM(1)); } MemoryContextSwitchTo(old_context); PG_RETURN_POINTER(compressor); } /* This function is used for testing only. */ Datum tsl_gorilla_compressor_finish(PG_FUNCTION_ARGS) { Compressor *compressor = (Compressor *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0)); if (compressor == NULL) PG_RETURN_NULL(); void *compressed = compressor->finish(compressor); if (compressed == NULL) PG_RETURN_NULL(); PG_RETURN_POINTER(compressed); } void gorilla_compressor_append_null(GorillaCompressor *compressor) { simple8brle_compressor_append(&compressor->nulls, 1); compressor->has_nulls = true; } void gorilla_compressor_append_value(GorillaCompressor *compressor, uint64 val) { bool has_values; uint64 xor = compressor->prev_val ^ val; simple8brle_compressor_append(&compressor->nulls, 0); /* for the first value we store the bitsize even if the xor is all zeroes, * this ensures that the bits-per-xor isn't empty, and that we can calculate * the remaining offsets correctly. */ has_values = !simple8brle_compressor_is_empty(&compressor->bits_used_per_xor); if (has_values && xor == 0) simple8brle_compressor_append(&compressor->tag0s, 0); else { /* leftmost/rightmost 1 is not well-defined when all the bits in the number * are 0; the C implementations of these functions will ERROR, while the * assembly versions may return any value. We special-case 0 to to use * values for leading and trailing-zeroes that we know will work. */ int leading_zeros = xor != 0 ? 63 - pg_leftmost_one_pos64(xor) : 63; int trailing_zeros = xor != 0 ? pg_rightmost_one_pos64(xor) : 1; /* This can easily get stuck with a bad value for trailing_zeroes, leading to a bad * compressed size. We use a new trailing_zeroes if the delta is too large, but the * threshold (12) was picked in a completely unprincipled manner. * Needs benchmarking to determine an ideal threshold. */ bool reuse_bitsizes = has_values && leading_zeros >= compressor->prev_leading_zeroes && trailing_zeros >= compressor->prev_trailing_zeros && ((leading_zeros - compressor->prev_leading_zeroes) + (trailing_zeros - compressor->prev_trailing_zeros) <= 12); uint8 num_bits_used; simple8brle_compressor_append(&compressor->tag0s, 1); simple8brle_compressor_append(&compressor->tag1s, reuse_bitsizes ? 0 : 1); if (!reuse_bitsizes) { compressor->prev_leading_zeroes = leading_zeros; compressor->prev_trailing_zeros = trailing_zeros; num_bits_used = 64 - (leading_zeros + trailing_zeros); bit_array_append(&compressor->leading_zeros, BITS_PER_LEADING_ZEROS, leading_zeros); simple8brle_compressor_append(&compressor->bits_used_per_xor, num_bits_used); } num_bits_used = 64 - (compressor->prev_leading_zeroes + compressor->prev_trailing_zeros); bit_array_append(&compressor->xors, num_bits_used, xor >> compressor->prev_trailing_zeros); } compressor->prev_val = val; } static GorillaCompressed * compressed_gorilla_data_serialize(CompressedGorillaData *input) { Size tags0s_size = simple8brle_serialized_total_size(input->tag0s); Size tags1s_size = simple8brle_serialized_total_size(input->tag1s); Size leading_zeros_size = bit_array_data_bytes_used(&input->leading_zeros); Size bits_used_per_xor_size = simple8brle_serialized_total_size(input->num_bits_used_per_xor); Size xors_size = bit_array_data_bytes_used(&input->xors); Size nulls_size = 0; Size compressed_size; char *data; GorillaCompressed *compressed; if (input->header->has_nulls) nulls_size = simple8brle_serialized_total_size(input->nulls); compressed_size = sizeof(GorillaCompressed) + tags0s_size + tags1s_size + leading_zeros_size + bits_used_per_xor_size + xors_size; if (input->header->has_nulls) compressed_size += nulls_size; if (!AllocSizeIsValid(compressed_size)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("compressed size exceeds the maximum allowed (%d)", (int) MaxAllocSize))); data = palloc0(compressed_size); compressed = (GorillaCompressed *) data; SET_VARSIZE(&compressed->vl_len_, compressed_size); Assert(compressed_size % 4 == 0); compressed->last_value = input->header->last_value; compressed->compression_algorithm = COMPRESSION_ALGORITHM_GORILLA; compressed->has_nulls = input->header->has_nulls; data += sizeof(GorillaCompressed); data = bytes_serialize_simple8b_and_advance(data, tags0s_size, input->tag0s); data = bytes_serialize_simple8b_and_advance(data, tags1s_size, input->tag1s); data = bytes_store_bit_array_and_advance(data, leading_zeros_size, &input->leading_zeros, &compressed->num_leading_zeroes_buckets, &compressed->bits_used_in_last_leading_zeros_bucket); data = bytes_serialize_simple8b_and_advance(data, bits_used_per_xor_size, input->num_bits_used_per_xor); data = bytes_store_bit_array_and_advance(data, xors_size, &input->xors, &compressed->num_xor_buckets, &compressed->bits_used_in_last_xor_bucket); if (input->header->has_nulls) data = bytes_serialize_simple8b_and_advance(data, nulls_size, input->nulls); return compressed; } void * gorilla_compressor_finish(GorillaCompressor *compressor) { GorillaCompressed header = { .compression_algorithm = COMPRESSION_ALGORITHM_GORILLA, .has_nulls = compressor->has_nulls ? 1 : 0, .last_value = compressor->prev_val, }; CompressedGorillaData data = { .header = &header }; data.tag0s = simple8brle_compressor_finish(&compressor->tag0s); if (data.tag0s == NULL) return NULL; data.tag1s = simple8brle_compressor_finish(&compressor->tag1s); Assert(data.tag1s != NULL); data.leading_zeros = compressor->leading_zeros; /* if all elements in the compressed are the same, there will be no xors, * and thus bits_used_per_xor will be empty. Since we need to store the header * to get the sizing right, we force at least one bits_used_per_xor to be created * in append, above */ data.num_bits_used_per_xor = simple8brle_compressor_finish(&compressor->bits_used_per_xor); Assert(data.num_bits_used_per_xor != NULL); data.xors = compressor->xors; data.nulls = simple8brle_compressor_finish(&compressor->nulls); Assert(compressor->has_nulls || data.nulls != NULL); return compressed_gorilla_data_serialize(&data); } /******************************* *** DecompressionIterator *** *******************************/ inline static void bytes_attach_bit_array_and_advance(BitArray *dst, StringInfo si, uint32 num_buckets, uint8 bits_in_last_bucket) { bit_array_wrap_internal(dst, num_buckets, bits_in_last_bucket, (uint64 *) (si->data + si->cursor)); consumeCompressedData(si, bit_array_data_bytes_used(dst)); } static void compressed_gorilla_data_init_from_stringinfo(CompressedGorillaData *expanded, StringInfo si) { expanded->header = (GorillaCompressed *) consumeCompressedData(si, sizeof(GorillaCompressed)); if (expanded->header->compression_algorithm != COMPRESSION_ALGORITHM_GORILLA) elog(ERROR, "unknown compression algorithm"); bool has_nulls = expanded->header->has_nulls == 1; expanded->tag0s = bytes_deserialize_simple8b_and_advance(si); expanded->tag1s = bytes_deserialize_simple8b_and_advance(si); bytes_attach_bit_array_and_advance(&expanded->leading_zeros, si, expanded->header->num_leading_zeroes_buckets, expanded->header->bits_used_in_last_leading_zeros_bucket); expanded->num_bits_used_per_xor = bytes_deserialize_simple8b_and_advance(si); bytes_attach_bit_array_and_advance(&expanded->xors, si, expanded->header->num_xor_buckets, expanded->header->bits_used_in_last_xor_bucket); if (has_nulls) expanded->nulls = bytes_deserialize_simple8b_and_advance(si); else expanded->nulls = NULL; } static void compressed_gorilla_data_init_from_pointer(CompressedGorillaData *expanded, const GorillaCompressed *compressed) { StringInfoData si = { .data = (char *) compressed, .len = VARSIZE(compressed) }; compressed_gorilla_data_init_from_stringinfo(expanded, &si); } static void compressed_gorilla_data_init_from_datum(CompressedGorillaData *data, Datum gorilla_compressed) { compressed_gorilla_data_init_from_pointer(data, (GorillaCompressed *) PG_DETOAST_DATUM( gorilla_compressed)); } static void gorilla_iterator_init_from_expanded_forward(GorillaDecompressionIterator *iterator, Oid element_type) { iterator->base.compression_algorithm = COMPRESSION_ALGORITHM_GORILLA; iterator->base.forward = true; iterator->base.element_type = element_type; iterator->base.try_next = gorilla_decompression_iterator_try_next_forward; iterator->prev_val = 0; iterator->prev_leading_zeroes = 0; iterator->prev_xor_bits_used = 0; simple8brle_decompression_iterator_init_forward(&iterator->tag0s, iterator->gorilla_data.tag0s); simple8brle_decompression_iterator_init_forward(&iterator->tag1s, iterator->gorilla_data.tag1s); bit_array_iterator_init(&iterator->leading_zeros, &iterator->gorilla_data.leading_zeros); simple8brle_decompression_iterator_init_forward(&iterator->num_bits_used, iterator->gorilla_data.num_bits_used_per_xor); bit_array_iterator_init(&iterator->xors, &iterator->gorilla_data.xors); iterator->has_nulls = iterator->gorilla_data.nulls != NULL; if (iterator->has_nulls) simple8brle_decompression_iterator_init_forward(&iterator->nulls, iterator->gorilla_data.nulls); } DecompressionIterator * gorilla_decompression_iterator_from_datum_forward(Datum gorilla_compressed, Oid element_type) { GorillaDecompressionIterator *iterator = palloc(sizeof(*iterator)); compressed_gorilla_data_init_from_datum(&iterator->gorilla_data, gorilla_compressed); gorilla_iterator_init_from_expanded_forward(iterator, element_type); return &iterator->base; } static inline DecompressResult convert_from_internal(DecompressResultInternal res_internal, Oid element_type) { if (res_internal.is_done || res_internal.is_null) { return (DecompressResult){ .is_done = res_internal.is_done, .is_null = res_internal.is_null, }; } switch (element_type) { case FLOAT8OID: return (DecompressResult){ .val = Float8GetDatum(bits_get_double(res_internal.val)), }; case FLOAT4OID: return (DecompressResult){ .val = Float4GetDatum(bits_get_float(res_internal.val)), }; case INT8OID: return (DecompressResult){ .val = Int64GetDatum(res_internal.val), }; case INT4OID: return (DecompressResult){ .val = Int32GetDatum(res_internal.val), }; case INT2OID: return (DecompressResult){ .val = Int16GetDatum(res_internal.val), }; default: elog(ERROR, "invalid type requested from gorilla decompression"); } pg_unreachable(); } static DecompressResultInternal gorilla_decompression_iterator_try_next_forward_internal(GorillaDecompressionIterator *iter) { Simple8bRleDecompressResult tag0; Simple8bRleDecompressResult tag1; uint64 xor ; if (iter->has_nulls) { Simple8bRleDecompressResult null = simple8brle_decompression_iterator_try_next_forward(&iter->nulls); /* Could slightly improve performance here by not returning a tail of non-null bits */ if (null.is_done) { return (DecompressResultInternal){ .is_done = true, }; } if ((null.val & 1) != 0) { return (DecompressResultInternal){ .is_null = true, }; } } tag0 = simple8brle_decompression_iterator_try_next_forward(&iter->tag0s); /* if we don't have a null bitset, this will determine when we're done */ if (tag0.is_done) { CheckCompressedData(!iter->has_nulls); return (DecompressResultInternal){ .is_done = true, }; } if (tag0.val == 0) { return (DecompressResultInternal){ .val = iter->prev_val, }; } tag1 = simple8brle_decompression_iterator_try_next_forward(&iter->tag1s); CheckCompressedData(!tag1.is_done); if (tag1.val != 0) { Simple8bRleDecompressResult num_xor_bits; /* get new xor sizes */ iter->prev_leading_zeroes = bit_array_iter_next(&iter->leading_zeros, BITS_PER_LEADING_ZEROS); CheckCompressedData(iter->prev_leading_zeroes <= 64); num_xor_bits = simple8brle_decompression_iterator_try_next_forward(&iter->num_bits_used); CheckCompressedData(!num_xor_bits.is_done); iter->prev_xor_bits_used = num_xor_bits.val; CheckCompressedData(iter->prev_xor_bits_used <= 64); /* * More than 64 significant bits don't make sense. Exactly 64 we get for * the first encoded number. */ CheckCompressedData(iter->prev_xor_bits_used + iter->prev_leading_zeroes <= 64); } /* * Zero significant bits would mean that the previous number is repeated, * but this should have been encoded with tag0 = 0. * This also might fail if we haven't seen the tag1 = 1 for the first number * and didn't initialize the bit widths. */ CheckCompressedData(iter->prev_xor_bits_used + iter->prev_leading_zeroes > 0); xor = bit_array_iter_next(&iter->xors, iter->prev_xor_bits_used); xor <<= 64 - (iter->prev_leading_zeroes + iter->prev_xor_bits_used); iter->prev_val ^= xor; return (DecompressResultInternal){ .val = iter->prev_val, }; } DecompressResult gorilla_decompression_iterator_try_next_forward(DecompressionIterator *iter_base) { Assert(iter_base->compression_algorithm == COMPRESSION_ALGORITHM_GORILLA && iter_base->forward); return convert_from_internal(gorilla_decompression_iterator_try_next_forward_internal( (GorillaDecompressionIterator *) iter_base), iter_base->element_type); } /**************************************** *** reversed DecompressionIterator *** ****************************************/ /* * conceptually, the bits from the gorilla algorithm can be thought of like * tag0: 1 1 1 1 1 1 1 1 1 1 1 * tag1: 1 0 0 0 0 1 0 0 0 0 1 * nbits: 0 4 5 3 * xor: 1 2 3 4 5 a b c d e Q * that is, tag1 represents the transition between one value in the number of * leading/used bits arrays, and thus can be transversed in any order, whenever * we see a `1`, we switch from using are current numbers to the "next" in * whichever iteration order we're following. When transversing in reverse order * there is a little subtlety in that we run out of lengths before we run out of * tag1 bits (there's an implicit leading `0`), but at that point we've run out * of values anyway, so it does not matter. */ DecompressionIterator * gorilla_decompression_iterator_from_datum_reverse(Datum gorilla_compressed, Oid element_type) { GorillaDecompressionIterator *iter = palloc(sizeof(*iter)); Simple8bRleDecompressResult num_xor_bits; iter->base.compression_algorithm = COMPRESSION_ALGORITHM_GORILLA; iter->base.forward = false; iter->base.element_type = element_type; iter->base.try_next = gorilla_decompression_iterator_try_next_reverse; compressed_gorilla_data_init_from_datum(&iter->gorilla_data, gorilla_compressed); simple8brle_decompression_iterator_init_reverse(&iter->tag0s, iter->gorilla_data.tag0s); simple8brle_decompression_iterator_init_reverse(&iter->tag1s, iter->gorilla_data.tag1s); bit_array_iterator_init_rev(&iter->leading_zeros, &iter->gorilla_data.leading_zeros); simple8brle_decompression_iterator_init_reverse(&iter->num_bits_used, iter->gorilla_data.num_bits_used_per_xor); bit_array_iterator_init_rev(&iter->xors, &iter->gorilla_data.xors); iter->has_nulls = iter->gorilla_data.nulls != NULL; if (iter->has_nulls) simple8brle_decompression_iterator_init_reverse(&iter->nulls, iter->gorilla_data.nulls); /* we need to know how many bits are used, even if the last value didn't store them */ iter->prev_leading_zeroes = bit_array_iter_next_rev(&iter->leading_zeros, BITS_PER_LEADING_ZEROS); num_xor_bits = simple8brle_decompression_iterator_try_next_reverse(&iter->num_bits_used); Assert(!num_xor_bits.is_done); iter->prev_xor_bits_used = num_xor_bits.val; iter->prev_val = iter->gorilla_data.header->last_value; return &iter->base; } static DecompressResultInternal gorilla_decompression_iterator_try_next_reverse_internal(GorillaDecompressionIterator *iter) { Simple8bRleDecompressResult tag0; Simple8bRleDecompressResult tag1; uint64 val; uint64 xor ; if (iter->has_nulls) { Simple8bRleDecompressResult null = simple8brle_decompression_iterator_try_next_reverse(&iter->nulls); if (null.is_done) return (DecompressResultInternal){ .is_done = true, }; if ((null.val & 1) != 0) { return (DecompressResultInternal){ .is_null = true, }; } } val = iter->prev_val; tag0 = simple8brle_decompression_iterator_try_next_reverse(&iter->tag0s); /* if we don't have a null bitset, this will determine when we're done */ if (tag0.is_done) return (DecompressResultInternal){ .is_done = true, }; if (tag0.val == 0) return (DecompressResultInternal){ .val = val, }; xor = bit_array_iter_next_rev(&iter->xors, iter->prev_xor_bits_used); if (iter->prev_leading_zeroes + iter->prev_xor_bits_used < 64) xor <<= 64 - (iter->prev_leading_zeroes + iter->prev_xor_bits_used); iter->prev_val ^= xor; tag1 = simple8brle_decompression_iterator_try_next_reverse(&iter->tag1s); if (tag1.val != 0) { /* get new xor sizes */ Simple8bRleDecompressResult num_xor_bits = simple8brle_decompression_iterator_try_next_reverse(&iter->num_bits_used); /* there're an implicit leading 0 to num_xor_bits and prev_leading_zeroes, */ if (num_xor_bits.is_done) { iter->prev_xor_bits_used = 0; iter->prev_leading_zeroes = 0; } else { iter->prev_xor_bits_used = num_xor_bits.val; iter->prev_leading_zeroes = bit_array_iter_next_rev(&iter->leading_zeros, BITS_PER_LEADING_ZEROS); } } return (DecompressResultInternal){ .val = val, }; } DecompressResult gorilla_decompression_iterator_try_next_reverse(DecompressionIterator *iter_base) { Assert(iter_base->compression_algorithm == COMPRESSION_ALGORITHM_GORILLA && !iter_base->forward); return convert_from_internal(gorilla_decompression_iterator_try_next_reverse_internal( (GorillaDecompressionIterator *) iter_base), iter_base->element_type); } #define MAX_NUM_LEADING_ZEROS_PADDED_N64 (((GLOBAL_MAX_ROWS_PER_COMPRESSION + 63) / 64) * 64) /* * Decompress packed 6bit values in lanes that contain a round number of both * packed and unpacked bytes -- 4 6-bit values are packed into 3 8-bit values. */ static uint8 * unpack_leading_zeros_array(BitArray *bitarray, uint32 *_n) { #define LANE_INPUTS 3 #define LANE_OUTPUTS 4 StaticAssertExpr(BITS_PER_LEADING_ZEROS * LANE_OUTPUTS == 8 * LANE_INPUTS, "the numbers of input and output lanes do not add up"); /* * We have four bytes of padding after leading zeros, so we don't care if * the reads of final bytes run into them and we unpack some nonsense. This * means we can always work in full lanes. * * We do have to check that the result fits into the maximum number of rows, * because we get the length from user input. */ const uint32 n_bytes_packed = bitarray->buckets.num_elements * sizeof(uint64); const uint32 n_lanes = (n_bytes_packed + LANE_INPUTS - 1) / LANE_INPUTS; const uint32 n_outputs = n_lanes * LANE_OUTPUTS; CheckCompressedData(n_outputs <= MAX_NUM_LEADING_ZEROS_PADDED_N64); uint8 *restrict dest = palloc(n_outputs); for (uint32 lane = 0; lane < n_lanes; lane++) { uint8 *restrict lane_dest = &dest[lane * LANE_OUTPUTS]; const uint8 *lane_src = &((uint8 *) bitarray->buckets.data)[lane * LANE_INPUTS]; for (uint32 output_in_lane = 0; output_in_lane < LANE_OUTPUTS; output_in_lane++) { const int startbit_abs = output_in_lane * BITS_PER_LEADING_ZEROS; const int startbit_rel = startbit_abs % 8; const int offs = 8 - startbit_rel; const uint8 this_input = lane_src[startbit_abs / 8]; const uint8 next_input = lane_src[(startbit_abs + BITS_PER_LEADING_ZEROS - 1) / 8]; uint8 output = this_input >> startbit_rel; output |= ((uint64) next_input) << offs; output &= (1ULL << BITS_PER_LEADING_ZEROS) - 1ULL; lane_dest[output_in_lane] = output; } } #undef LANE_INPUTS #undef LANE_OUTPUTS *_n = n_outputs; return dest; } /* Bulk gorilla decompression, specialized for supported data types. */ #define ELEMENT_TYPE uint8 #include "simple8b_rle_decompress_all.h" #undef ELEMENT_TYPE #define ELEMENT_TYPE uint32 #include "gorilla_impl.c" #undef ELEMENT_TYPE #define ELEMENT_TYPE uint64 #include "gorilla_impl.c" #undef ELEMENT_TYPE ArrowArray * gorilla_decompress_all(Datum datum, Oid element_type, MemoryContext dest_mctx) { CompressedGorillaData gorilla_data; compressed_gorilla_data_init_from_datum(&gorilla_data, datum); switch (element_type) { case FLOAT8OID: return gorilla_decompress_all_uint64(&gorilla_data, dest_mctx); case FLOAT4OID: return gorilla_decompress_all_uint32(&gorilla_data, dest_mctx); default: elog(ERROR, "type '%s' is not supported for gorilla decompression", format_type_be(element_type)); pg_unreachable(); } } /************* *** I/O *** **************/ void gorilla_compressed_send(CompressedDataHeader *header, StringInfo buf) { CompressedGorillaData data; const GorillaCompressed *compressed = (GorillaCompressed *) header; Assert(header->compression_algorithm == COMPRESSION_ALGORITHM_GORILLA); compressed_gorilla_data_init_from_pointer(&data, compressed); pq_sendbyte(buf, data.header->has_nulls); pq_sendint64(buf, data.header->last_value); simple8brle_serialized_send(buf, data.tag0s); simple8brle_serialized_send(buf, data.tag1s); bit_array_send(buf, &data.leading_zeros); simple8brle_serialized_send(buf, data.num_bits_used_per_xor); bit_array_send(buf, &data.xors); if (data.header->has_nulls) simple8brle_serialized_send(buf, data.nulls); } Datum gorilla_compressed_recv(StringInfo buf) { GorillaCompressed header = { .vl_len_ = { 0 } }; CompressedGorillaData data = { .header = &header, }; header.has_nulls = pq_getmsgbyte(buf); CheckCompressedData(header.has_nulls == 0 || header.has_nulls == 1); header.last_value = pq_getmsgint64(buf); data.tag0s = simple8brle_serialized_recv(buf); data.tag1s = simple8brle_serialized_recv(buf); data.leading_zeros = bit_array_recv(buf); data.num_bits_used_per_xor = simple8brle_serialized_recv(buf); data.xors = bit_array_recv(buf); if (header.has_nulls) data.nulls = simple8brle_serialized_recv(buf); PG_RETURN_POINTER(compressed_gorilla_data_serialize(&data)); } ================================================ FILE: tsl/src/compression/algorithms/gorilla.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once /* * The Gorilla algorithm compresses floats and is modeled after the Facebook Gorilla paper: * "Gorilla: A Fast, Scalable, In-Memory Time Series Database" by Tuomas Pelkonen et. al. * * How it works: Given a series of floats, first convert that series to a series of xors between * consecutive floats (as uint64), while storing the first value as well. Given the first value * and the series of xor values, getting the floats back is trivial. So, our goal becomes to * compress the series of xors. * * The logic for compressing xors is as follows: * * The compression depends on the observation that a lot of xors will be mostly 0s, and that the * section of the xor that is non-zero will be similar in consecutive xors. So the algorithm tries * to record only the section of the xors that are non-zero. And records state about which section * of the xor its storing rarely (reusing this information from the previous xor if possible). The * state of which section of the xor is recorded is kept in two variables: leading-zeroes and * number_of_bits_used. Thus we record only number_of_bits_used bits of the xor (shifted * appropriately). * * The algorithm keeps state for the number of leading-zeroes and number_of_bits_used from xor to * xor. A boolean array called tag1 is used to indicate that these state variables need to change * for the next xor. A boolean array called Tag0 is used to indicate that the xor is 0. * * The state is as follows: * * two separate series of boolean bit values called tag0, tag1. (simple8brle compressed) * * a series of 6-bit bit-array values called leading-zeroes (not-compressed) * * a series of 64-bit ints of num_bits_used (simple8brle compressed) * * a series of variable-bit-length bits stored in an bit array called xors (not-compressed) * * Pseudocode: * if the xor == 0: * append a 0 to tag0. * You are done. * else: (xor != 0) * append a 1 to tag0. * Figure out the number of leading-zeroes and number_of_bits_used necessary to store the next * xor. Then, Make a decision whether to reuse the leading-zero and number of bits from the * current state. (the decision is based on whether it's possible to fit the next xor in as well * as a heuristic for whether it's cheaper to switch to use less number_of_bits_used) * * if (reusing previous state) * append 0 to tag1. * append number_of_bits_used bits to the xors bit array. These bits consist * of the next xor shifted by the appropriate amount. * You are done. * else: (you are changing state) * append 1 to tag1 * append the new number of leading-zeroes to the leading-zeroes array * append the new number for number_of_bits_used to the num_bits used array * append new number_of_bits_used bits to the xors bit array. * These bits consist of the next xor shifted by the appropriate amount. * you are done. * */ #include <postgres.h> #include <c.h> #include <fmgr.h> #include <lib/stringinfo.h> #include "compression/compression.h" typedef struct GorillaCompressor GorillaCompressor; typedef struct GorillaCompressed GorillaCompressed; typedef struct GorillaDecompressionIterator GorillaDecompressionIterator; extern bool gorilla_compressed_has_nulls(const CompressedDataHeader *header); extern Compressor *gorilla_compressor_for_type(Oid element_type); extern GorillaCompressor *gorilla_compressor_alloc(void); extern void gorilla_compressor_append_null(GorillaCompressor *compressor); extern void gorilla_compressor_append_value(GorillaCompressor *compressor, uint64 val); extern void *gorilla_compressor_finish(GorillaCompressor *compressor); extern DecompressionIterator * gorilla_decompression_iterator_from_datum_forward(Datum gorilla_compressed, Oid element_type); extern DecompressResult gorilla_decompression_iterator_try_next_forward(DecompressionIterator *iter); extern DecompressionIterator * gorilla_decompression_iterator_from_datum_reverse(Datum gorilla_compressed, Oid element_type); extern DecompressResult gorilla_decompression_iterator_try_next_reverse(DecompressionIterator *iter); extern ArrowArray *gorilla_decompress_all(Datum datum, Oid element_type, MemoryContext dest_mctx); extern void gorilla_compressed_send(CompressedDataHeader *header, StringInfo buffer); extern Datum gorilla_compressed_recv(StringInfo buf); extern Datum tsl_gorilla_compressor_append(PG_FUNCTION_ARGS); extern Datum tsl_gorilla_compressor_finish(PG_FUNCTION_ARGS); #define GORILLA_ALGORITHM_DEFINITION \ { \ .iterator_init_forward = gorilla_decompression_iterator_from_datum_forward, \ .iterator_init_reverse = gorilla_decompression_iterator_from_datum_reverse, \ .decompress_all = gorilla_decompress_all, .compressed_data_send = gorilla_compressed_send, \ .compressed_data_recv = gorilla_compressed_recv, \ .compressor_for_type = gorilla_compressor_for_type, \ .compressed_data_storage = TOAST_STORAGE_EXTERNAL, \ } ================================================ FILE: tsl/src/compression/algorithms/gorilla_impl.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Decompress the entire batch of gorilla-compressed rows into an Arrow array. * Specialized for each supported data type. */ #define FUNCTION_NAME_HELPER(X, Y) X##_##Y #define FUNCTION_NAME(X, Y) FUNCTION_NAME_HELPER(X, Y) static ArrowArray * FUNCTION_NAME(gorilla_decompress_all, ELEMENT_TYPE)(CompressedGorillaData *gorilla_data, MemoryContext dest_mctx) { const bool has_nulls = gorilla_data->nulls != NULL; const uint32 n_total = has_nulls ? gorilla_data->nulls->num_elements : gorilla_data->tag0s->num_elements; CheckCompressedData(n_total <= GLOBAL_MAX_ROWS_PER_COMPRESSION); /* * Pad the number of elements to multiple of 64 bytes if needed, so that we * can work in 64-byte blocks. */ const uint32 n_total_padded = ((n_total * sizeof(ELEMENT_TYPE) + 63) / 64) * 64 / sizeof(ELEMENT_TYPE); Assert(n_total_padded >= n_total); /* * We need additional padding at the end of buffer, because the code that * converts the elements to postres Datum always reads in 8 bytes. */ const int buffer_bytes = n_total_padded * sizeof(ELEMENT_TYPE) + 8; ELEMENT_TYPE *restrict decompressed_values = MemoryContextAlloc(dest_mctx, buffer_bytes); const uint32 n_notnull = gorilla_data->tag0s->num_elements; CheckCompressedData(n_total >= n_notnull); /* Unpack the basic compressed data parts. */ const Simple8bRleBitmap tag0s = simple8brle_bitmap_prefixsums(gorilla_data->tag0s); const Simple8bRleBitmap tag1s = simple8brle_bitmap_prefixsums(gorilla_data->tag1s); BitArray leading_zeros_bitarray = gorilla_data->leading_zeros; BitArrayIterator leading_zeros_iterator; bit_array_iterator_init(&leading_zeros_iterator, &leading_zeros_bitarray); uint32 num_leading_zeros_padded; const uint8 *all_leading_zeros = unpack_leading_zeros_array(&gorilla_data->leading_zeros, &num_leading_zeros_padded); uint32 num_bit_widths; const uint8 *bit_widths = simple8brle_decompress_all_uint8(gorilla_data->num_bits_used_per_xor, &num_bit_widths); BitArray xors_bitarray = gorilla_data->xors; BitArrayIterator xors_iterator; bit_array_iterator_init(&xors_iterator, &xors_bitarray); /* * Now decompress the non-null data. * * 1) unpack only the different elements (tag0 = 1) based on the tag1 array. * * 1a) Sanity check: the number of bit widths we have matches the * number of 1s in the tag1s array. */ CheckCompressedData(simple8brle_bitmap_num_ones(&tag1s) == num_bit_widths); CheckCompressedData(simple8brle_bitmap_num_ones(&tag1s) <= num_leading_zeros_padded); /* * 1b) Sanity check: the first tag1 must be 1, so that we initialize the bit * widths. */ CheckCompressedData(simple8brle_bitmap_prefix_sum(&tag1s, 0) == 1); /* * 1c) Sanity check: can't have more different elements than notnull elements. */ const uint16 n_different = tag1s.num_elements; CheckCompressedData(n_different <= n_notnull); /* * 1d) Unpack. * * Note that the bit widths change often, so there's no sense in * having a fast path for stretches of tag1 == 0. */ ELEMENT_TYPE prev = 0; for (uint16 i = 0; i < n_different; i++) { const uint8 current_xor_bits = bit_widths[simple8brle_bitmap_prefix_sum(&tag1s, i) - 1]; const uint8 current_leading_zeros = all_leading_zeros[simple8brle_bitmap_prefix_sum(&tag1s, i) - 1]; /* * Truncate the shift here not to cause UB on the corrupt data. */ const uint8 shift = (64 - (current_xor_bits + current_leading_zeros)) & 63; const uint64 current_xor = bit_array_iter_next(&xors_iterator, current_xor_bits); prev ^= current_xor << shift; decompressed_values[i] = prev; } /* * 2) Fill out the stretches of repeated elements, encoded with tag0 = 0. * * 2a) Sanity check: number of different elements according to tag0s must be * the same as number of different elements according to tag1s, so that the * current_element doesn't underrun. */ CheckCompressedData(simple8brle_bitmap_num_ones(&tag0s) == n_different); /* * 2b) Sanity check: tag0s[0] == 1 -- the first element of the sequence is * always "different from the previous one". */ CheckCompressedData(simple8brle_bitmap_prefix_sum(&tag0s, 0) == 1); /* * 2b) Fill the repeated elements. */ for (int i = n_notnull - 1; i >= 0; i--) { decompressed_values[i] = decompressed_values[simple8brle_bitmap_prefix_sum(&tag0s, i) - 1]; } uint64 *restrict validity_bitmap = NULL; if (has_nulls) { /* * We have unpacked the non-null data. Now reshuffle it to account for nulls, * and fill the validity bitmap. */ const int validity_bitmap_bytes = sizeof(uint64) * ((n_total + 64 - 1) / 64); validity_bitmap = MemoryContextAlloc(dest_mctx, validity_bitmap_bytes); /* * First, mark all data as valid, we will fill the nulls later if needed. * Note that the validity bitmap size is a multiple of 64 bits. We have to * fill the tail bits with zeros, because the corresponding elements are not * valid. * */ memset(validity_bitmap, 0xFF, validity_bitmap_bytes); if (n_total % 64) { const uint64 tail_mask = ~0ULL >> (64 - n_total % 64); validity_bitmap[n_total / 64] &= tail_mask; } /* * We have decompressed the data with nulls skipped, reshuffle it * according to the nulls bitmap. */ const Simple8bRleBitmap nulls = simple8brle_bitmap_decompress(gorilla_data->nulls); CheckCompressedData(n_notnull + simple8brle_bitmap_num_ones(&nulls) == n_total); int current_notnull_element = n_notnull - 1; for (int i = n_total - 1; i >= 0; i--) { Assert(i >= current_notnull_element); if (simple8brle_bitmap_get_at(&nulls, i)) { arrow_set_row_validity(validity_bitmap, i, false); } else { Assert(current_notnull_element >= 0); decompressed_values[i] = decompressed_values[current_notnull_element]; current_notnull_element--; } } Assert(current_notnull_element == -1); } /* Return the result. */ ArrowArray *result = MemoryContextAllocZero(dest_mctx, sizeof(ArrowArray) + sizeof(void *) * 2); const void **buffers = (const void **) &result[1]; buffers[0] = validity_bitmap; buffers[1] = decompressed_values; result->n_buffers = 2; result->buffers = buffers; result->length = n_total; result->null_count = n_total - n_notnull; return result; } #undef FUNCTION_NAME #undef FUNCTION_NAME_HELPER ================================================ FILE: tsl/src/compression/algorithms/null.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include "null.h" #include "fmgr.h" typedef struct NullCompressed { CompressedDataHeaderFields; } NullCompressed; extern DecompressionIterator * null_decompression_iterator_from_datum_forward(Datum bool_compressed, Oid element_type) { elog(ERROR, "null decompression iterator not implemented"); return NULL; } extern DecompressionIterator * null_decompression_iterator_from_datum_reverse(Datum bool_compressed, Oid element_type) { elog(ERROR, "null decompression iterator not implemented"); return NULL; } extern void null_compressed_send(CompressedDataHeader *header, StringInfo buffer) { elog(ERROR, "null compression doesn't implement send"); } extern Datum null_compressed_recv(StringInfo buffer) { /* Sanity checks for invalid buffer */ if (buffer->len == 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("compressed data is invalid to be a null compressed block"))); if (buffer->data == NULL) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("compressed data is NULL"))); PG_RETURN_POINTER(null_compressor_get_dummy_block()); } extern Compressor * null_compressor_for_type(Oid element_type) { elog(ERROR, "null compressor not implemented"); return NULL; } extern void * null_compressor_get_dummy_block(void) { NullCompressed *compressed = palloc(sizeof(NullCompressed)); Size compressed_size = sizeof(NullCompressed); compressed->compression_algorithm = COMPRESSION_ALGORITHM_NULL; SET_VARSIZE(&compressed->vl_len_, compressed_size); return compressed; } ================================================ FILE: tsl/src/compression/algorithms/null.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once /* * The NULL compression algorithm is a no-op compression algorithm that is only * used to signal that all values in a compressed block are NULLs. The compression * interface functions are only defined to comply with the framework, but they * are not implemented and return an ERROR. Calling these function is a software * bug. */ #include <postgres.h> #include <fmgr.h> #include <lib/stringinfo.h> #include "compression/compression.h" /* * Compressor framework functions and definitions for the null algorithm. */ extern DecompressionIterator *null_decompression_iterator_from_datum_forward(Datum bool_compressed, Oid element_type); extern DecompressionIterator *null_decompression_iterator_from_datum_reverse(Datum bool_compressed, Oid element_type); extern void null_compressed_send(CompressedDataHeader *header, StringInfo buffer); extern Datum null_compressed_recv(StringInfo buffer); extern Compressor *null_compressor_for_type(Oid element_type); extern void *null_compressor_get_dummy_block(void); #define NULL_COMPRESS_ALGORITHM_DEFINITION \ { \ .iterator_init_forward = null_decompression_iterator_from_datum_forward, \ .iterator_init_reverse = null_decompression_iterator_from_datum_reverse, \ .decompress_all = NULL, .compressed_data_send = null_compressed_send, \ .compressed_data_recv = null_compressed_recv, \ .compressor_for_type = null_compressor_for_type, \ .compressed_data_storage = TOAST_STORAGE_EXTERNAL, \ } ================================================ FILE: tsl/src/compression/algorithms/simple8b_rle.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <c.h> #include <fmgr.h> #include <lib/stringinfo.h> #include <libpq/pqformat.h> #include <adts/bit_array.h> #include "compat/compat.h" #include <adts/uint64_vec.h> /* This is defined as a header file as it is expected to be used as a primitive * for "real" compression algorithms, not used directly on SQL data. Also, due to inlining. * * * From Vo Ngoc Anh, Alistair Moffat: Index compression using 64-bit words. Softw., Pract. Exper. * 40(2): 131-147 (2010) * * Simple 8b RLE is a block based encoding/compression scheme for integers. Each block is made up of * one selector and one 64-bit data value. The interpretation of the data value is based on the * selector values. Selectors 1-14 indicate that the data value is a bit packing of integers, where * each integer takes up a constant number of bits. The value of the constant-number-of-bits is set * according to the table below. Selector 15 indicates that the block encodes a single "run" of RLE, * where the data element is a bit packing of the run count and run value. * * * Selector value: 0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 | 15 (RLE) * Integers coded: 0 | 64 32 21 16 12 10 9 8 6 5 4 3 2 1 | up to 2^28 * Bits/integer: 0 | 1 2 3 4 5 6 7 8 10 12 16 21 32 64 | 36 bits * Wasted bits: 0 | 0 0 1 0 4 4 1 0 4 4 0 1 0 0 | N/A * * a 0 selector is currently unused */ /************** Constants *****************/ #define SIMPLE8B_BITSIZE 64 #define SIMPLE8B_MAXCODE 15 #define SIMPLE8B_MINCODE 1 #define SIMPLE8B_RLE_SELECTOR SIMPLE8B_MAXCODE #define SIMPLE8B_RLE_MAX_VALUE_BITS 36 #define SIMPLE8B_RLE_MAX_COUNT_BITS (SIMPLE8B_BITSIZE - SIMPLE8B_RLE_MAX_VALUE_BITS) #define SIMPLE8B_RLE_MAX_VALUE_MASK ((1ULL << SIMPLE8B_RLE_MAX_VALUE_BITS) - 1) #define SIMPLE8B_RLE_MAX_COUNT_MASK ((1ULL << SIMPLE8B_RLE_MAX_COUNT_BITS) - 1) #define SIMPLE8B_BITS_PER_SELECTOR 4 #define SIMPLE8B_SELECTORS_PER_SELECTOR_SLOT 16 #define SIMPLE8B_MAX_BUFFERED 256 /* clang-format off */ /* selector value: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, RLE */ #define SIMPLE8B_NUM_ELEMENTS ((uint8[]){ 0, 64, 32, 21, 16, 12, 10, 9, 8, 6, 5, 4, 3, 2, 1, 0 }) #define SIMPLE8B_BIT_LENGTH ((uint8[]){ 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 16, 21, 32, 64, 36 }) /* Map bit lengths directly to selector values */ #define SIMPLE8B_SELECTOR_FOR_BIT_WIDTH ((uint8[]){ \ /* 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, */ \ 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 10, 10, 11, 11, 11, \ /* 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, */ \ 11, 12, 12, 12, 12, 12, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, \ /* 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, */ \ 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, \ /* 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64 */ \ 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14 \ }) /* clang-format on */ /* The full block selector is used when a value occupies the full 64bit block, see above. */ #define SIMPLE8B_FULL_BLOCK_SELECTOR 14 /******************** *** Public API *** ********************/ typedef struct Simple8bRleSerialized { /* the slots are padded with 0 to fill out the last slot, so there may be up * to 59 extra values stored, to counteract this, we store how many values * there should be on output. * We currently disallow more than 2^32 values per compression, since we're * going to limit the amount of rows stored per-compressed-row anyway. */ uint32 num_elements; /* we store nslots as a uint32 since we'll need to fit this in a varlen, and * we cannot store more than 2^32 bytes anyway */ uint32 num_blocks; uint64 slots[FLEXIBLE_ARRAY_MEMBER]; } Simple8bRleSerialized; static void pg_attribute_unused() simple8brle_size_assertions(void) { Simple8bRleSerialized test_val = { 0 }; /* ensure no padding bits make it to disk */ StaticAssertStmt(sizeof(Simple8bRleSerialized) == sizeof(test_val.num_elements) + sizeof(test_val.num_blocks), "simple8b_rle_oob wrong size"); StaticAssertStmt(sizeof(Simple8bRleSerialized) == 8, "simple8b_rle_oob wrong size"); } typedef struct Simple8bRleBlock { uint64 data; uint32 num_elements_compressed; uint8 selector; } Simple8bRleBlock; typedef struct Simple8bRleBuffer { uint64 data; uint32 repcount; /* The saved_repcount is placed here to help cache locality when we * need to do a partial flush and need to restore the repcount. */ uint32 saved_repcount; } Simple8bRleBuffer; typedef struct Simple8bRleCompressor { /* The total number of elements that have been compressed. */ uint32 num_elements; /* The number of elements that have been buffered in the uncompressed_buffer. */ uint32 num_buffered_elements; /* The last value that has been compressed. */ uint64 last_value; /* The buffer of uncompressed elements. */ Simple8bRleBuffer uncompressed_buffer[SIMPLE8B_MAX_BUFFERED]; BitArray selectors; uint64_vec compressed_data; } Simple8bRleCompressor; typedef struct Simple8bRleDecompressionIterator { BitArray selector_data; BitArrayIterator selectors; Simple8bRleBlock current_block; const uint64 *compressed_data; int32 num_blocks; int32 current_compressed_pos; int32 current_in_compressed_pos; uint32 num_elements; uint32 num_elements_returned; } Simple8bRleDecompressionIterator; typedef struct Simple8bRleDecompressResult { uint64 val; bool is_done; } Simple8bRleDecompressResult; static inline void simple8brle_compressor_init(Simple8bRleCompressor *compressor); static inline Simple8bRleSerialized * simple8brle_compressor_finish(Simple8bRleCompressor *compressor); static inline char *simple8brle_compressor_finish_into(Simple8bRleCompressor *compressor, char *dest, size_t expected_size); static inline void simple8brle_compressor_append(Simple8bRleCompressor *compressor, uint64 val); static inline bool simple8brle_compressor_is_empty(Simple8bRleCompressor *compressor); static inline void simple8brle_decompression_iterator_init_forward(Simple8bRleDecompressionIterator *iter, Simple8bRleSerialized *compressed); static inline void simple8brle_decompression_iterator_init_reverse(Simple8bRleDecompressionIterator *iter, Simple8bRleSerialized *compressed); static pg_attribute_always_inline Simple8bRleDecompressResult simple8brle_decompression_iterator_try_next_forward(Simple8bRleDecompressionIterator *iter); static pg_attribute_always_inline Simple8bRleDecompressResult simple8brle_decompression_iterator_try_next_reverse(Simple8bRleDecompressionIterator *iter); static inline void simple8brle_serialized_send(StringInfo buffer, const Simple8bRleSerialized *data); static inline char *bytes_serialize_simple8b_and_advance(char *dest, size_t expected_size, const Simple8bRleSerialized *data); static inline Simple8bRleSerialized *bytes_deserialize_simple8b_and_advance(StringInfo si); static inline size_t simple8brle_serialized_slot_size(const Simple8bRleSerialized *data); static inline size_t simple8brle_serialized_total_size(const Simple8bRleSerialized *data); /* * Calculate the size of the compressed data with the assumption that all uncompressed * data is flushed and pushed already. */ static inline size_t simple8brle_compressor_compressed_size(const Simple8bRleCompressor *compressor); /* * Calculate the size of the compressed data without modifying the compressor and without * making assumptions about the compressor state. */ static inline size_t simple8brle_compressor_compressed_const_size(const Simple8bRleCompressor *compressor); /********************* *** Private API *** *********************/ typedef struct Simple8bRlePartiallyCompressedData { Simple8bRleBlock block; const uint64 *data; uint32 data_size; } Simple8bRlePartiallyCompressedData; /* compressor */ static inline void simple8brle_compressor_partial_flush(Simple8bRleCompressor *compressor); static inline void simple8brle_compressor_full_flush(Simple8bRleCompressor *compressor); /* block */ static inline Simple8bRleBlock simple8brle_block_create(uint8 selector, uint64 data); static inline uint64 simple8brle_block_get_element(Simple8bRleBlock block, uint32 position_in_value); /* utils */ static inline bool simple8brle_selector_is_rle(uint8 selector); static inline uint64 simple8brle_selector_get_bitmask(uint8 selector); static inline uint32 simple8brle_bits_for_value(uint64 v); static inline uint32 simple8brle_rledata_repeatcount(uint64 rledata); static inline uint64 simple8brle_rledata_value(uint64 rledata); static uint32 simple8brle_num_selector_slots_for_num_blocks(uint32 num_blocks); /******************************* *** Simple8bRleSerialized *** *******************************/ static inline Simple8bRleSerialized * simple8brle_serialized_recv(StringInfo buffer) { uint32 i; uint32 num_elements = pq_getmsgint32(buffer); CheckCompressedData(num_elements <= GLOBAL_MAX_ROWS_PER_COMPRESSION); uint32 num_blocks = pq_getmsgint32(buffer); CheckCompressedData(num_blocks <= GLOBAL_MAX_ROWS_PER_COMPRESSION); uint32 num_selector_slots = simple8brle_num_selector_slots_for_num_blocks(num_blocks); Simple8bRleSerialized *data; Size compressed_size = sizeof(Simple8bRleSerialized) + (num_blocks + num_selector_slots) * sizeof(uint64); if (!AllocSizeIsValid(compressed_size)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("compressed size exceeds the maximum allowed (%d)", (int) MaxAllocSize))); data = palloc(compressed_size); data->num_elements = num_elements; data->num_blocks = num_blocks; for (i = 0; i < num_blocks + num_selector_slots; i++) data->slots[i] = pq_getmsgint64(buffer); return data; } static inline void * simple8brle_serialized_recv_into(StringInfo buffer, void *dest, Simple8bRleSerialized **data_out) { uint32 i; uint32 num_elements = pq_getmsgint32(buffer); CheckCompressedData(num_elements <= GLOBAL_MAX_ROWS_PER_COMPRESSION); uint32 num_blocks = pq_getmsgint32(buffer); CheckCompressedData(num_blocks <= GLOBAL_MAX_ROWS_PER_COMPRESSION); uint32 num_selector_slots = simple8brle_num_selector_slots_for_num_blocks(num_blocks); Size compressed_size = sizeof(Simple8bRleSerialized) + (num_blocks + num_selector_slots) * sizeof(uint64); if (!AllocSizeIsValid(compressed_size)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("compressed size exceeds the maximum allowed (%d)", (int) MaxAllocSize))); *data_out = (Simple8bRleSerialized *) dest; (*data_out)->num_elements = num_elements; (*data_out)->num_blocks = num_blocks; for (i = 0; i < num_blocks + num_selector_slots; i++) (*data_out)->slots[i] = pq_getmsgint64(buffer); return (char *) *data_out + compressed_size; } static void simple8brle_serialized_send(StringInfo buffer, const Simple8bRleSerialized *data) { Assert(NULL != data); uint32 num_selector_slots = simple8brle_num_selector_slots_for_num_blocks(data->num_blocks); uint32 i; pq_sendint32(buffer, data->num_elements); pq_sendint32(buffer, data->num_blocks); for (i = 0; i < data->num_blocks + num_selector_slots; i++) pq_sendint64(buffer, data->slots[i]); } static char * bytes_serialize_simple8b_and_advance(char *dest, size_t expected_size, const Simple8bRleSerialized *data) { size_t size = simple8brle_serialized_total_size(data); if (expected_size != size) elog(ERROR, "the size to serialize does not match simple8brle"); memcpy(dest, data, size); return dest + size; } static Simple8bRleSerialized * bytes_deserialize_simple8b_and_advance(StringInfo si) { Simple8bRleSerialized *serialized = consumeCompressedData(si, sizeof(Simple8bRleSerialized)); consumeCompressedData(si, simple8brle_serialized_slot_size(serialized)); CheckCompressedData(serialized->num_elements <= GLOBAL_MAX_ROWS_PER_COMPRESSION); CheckCompressedData(serialized->num_elements > 0); CheckCompressedData(serialized->num_blocks > 0); CheckCompressedData(serialized->num_elements >= serialized->num_blocks); return serialized; } static size_t simple8brle_serialized_slot_size(const Simple8bRleSerialized *data) { if (data == NULL) return 0; int total_slots = data->num_blocks + simple8brle_num_selector_slots_for_num_blocks(data->num_blocks); CheckCompressedData(total_slots > 0); CheckCompressedData((uint32) total_slots < PG_INT32_MAX / sizeof(uint64)); return total_slots * sizeof(uint64); } static size_t simple8brle_serialized_total_size(const Simple8bRleSerialized *data) { Assert(data != NULL); return sizeof(*data) + simple8brle_serialized_slot_size(data); } /******************************* *** Simple8bRleCompressor *** *******************************/ static void simple8brle_compressor_init(Simple8bRleCompressor *compressor) { *compressor = (Simple8bRleCompressor){ .num_elements = 0, .num_buffered_elements = 0, .last_value = 0, }; /* Allocate the size according to the batch target */ uint64_vec_init(&compressor->compressed_data, CurrentMemoryContext, TARGET_COMPRESSED_BATCH_SIZE); bit_array_init(&compressor->selectors, /* expected_bits = */ (TARGET_COMPRESSED_BATCH_SIZE * SIMPLE8B_BITS_PER_SELECTOR)); } static inline void simple8brle_compressor_partial_flush(Simple8bRleCompressor *compressor) { Assert(compressor->num_buffered_elements > 0); uint8 bit_width = 0; uint64 mask = 0; int16 max_pack = 0; uint8 selector = 0; int32 num_buffered_elements = compressor->num_buffered_elements; int32 i; for (i = 0; i < num_buffered_elements; i++) { Simple8bRleBuffer *restrict buffer_base = &compressor->uncompressed_buffer[i]; Assert(buffer_base[0].repcount > 0); uint8 first_bit_width = simple8brle_bits_for_value(buffer_base[0].data); selector = SIMPLE8B_SELECTOR_FOR_BIT_WIDTH[first_bit_width]; bit_width = SIMPLE8B_BIT_LENGTH[selector]; /* The current element times the bit width is large enough to flush. */ if (bit_width * buffer_base[0].repcount >= SIMPLE8B_BITSIZE) { uint64 val = buffer_base[0].data; uint64 repcount = buffer_base[0].repcount; /* We can only RLE elements if they are small enough to fit in the * data part of the RLE block. */ if (val <= SIMPLE8B_RLE_MAX_VALUE_MASK) { /* Flush the value as RLE */ uint64 rle_block = (repcount << SIMPLE8B_RLE_MAX_VALUE_BITS) | val; bit_array_append(&compressor->selectors, SIMPLE8B_BITS_PER_SELECTOR, SIMPLE8B_RLE_SELECTOR); uint64_vec_append(&compressor->compressed_data, rle_block); continue; } /* Otherwise we need to flush each element as a full 64 bit block */ else { /* Each repeated element needs to be flushed as separate blocks */ for (uint32 k = 0; k < repcount; k++) { bit_array_append(&compressor->selectors, SIMPLE8B_BITS_PER_SELECTOR, SIMPLE8B_FULL_BLOCK_SELECTOR); uint64_vec_append(&compressor->compressed_data, val); } continue; } } int32 num_packed = buffer_base[0].repcount; max_pack = SIMPLE8B_NUM_ELEMENTS[selector]; mask = simple8brle_selector_get_bitmask(selector); for (int32 j = 1; (i + j) < num_buffered_elements && num_packed < max_pack; ++j) { Assert(buffer_base[j].repcount > 0); uint64 val = buffer_base[j].data; while (val > mask) { /* We bumped into a value that doesn't fit, need to expand selector, * but we need to make sure we don't leave gaps in the packed data. */ ++selector; mask = simple8brle_selector_get_bitmask(selector); if (num_packed >= SIMPLE8B_NUM_ELEMENTS[selector]) break; } max_pack = SIMPLE8B_NUM_ELEMENTS[selector]; num_packed += buffer_base[j].repcount; } /* Final calculations after selector is determined */ bit_width = SIMPLE8B_BIT_LENGTH[selector]; int32 num_buffer_taken = 0; uint64 packed_value = 0; num_packed = 0; /* * We need to be smart with the repcounts. The case when the repeated values * can fit an entire block is already handled above. Here we move the values * from the current buffer entry to the packed value, thus we decrease the * repcounts one by one. */ while (num_packed < max_pack && (i + num_buffer_taken) < num_buffered_elements) { Simple8bRleBuffer *restrict current_entry = &buffer_base[num_buffer_taken]; current_entry->saved_repcount = current_entry->repcount; uint64 val = current_entry->data; uint32 entry_repcount = current_entry->repcount; while (entry_repcount > 0 && num_packed < max_pack) { packed_value |= (val & mask) << (num_packed * bit_width); ++num_packed; --entry_repcount; } /* Write back the updated repcount. */ current_entry->repcount = entry_repcount; if (entry_repcount > 0) break; ++num_buffer_taken; } if (num_packed == max_pack) { /* Flush the packed value. */ bit_array_append(&compressor->selectors, SIMPLE8B_BITS_PER_SELECTOR, selector); uint64_vec_append(&compressor->compressed_data, packed_value); } else { /* * If we flushed some values from the uncompressed buffer, but there are * some that didn't fit, we need to move them to the beginning of the buffer. * We need to restore the repcounts to their original values first. */ for (int32 j = 0; j < num_buffer_taken; j++) { buffer_base[j].repcount = buffer_base[j].saved_repcount; } uint32 remaining = num_buffered_elements - i; if (remaining > 0) { memcpy(compressor->uncompressed_buffer, &compressor->uncompressed_buffer[i], remaining * sizeof(Simple8bRleBuffer)); } else { /* If no elements are remaining, we can reset the buffer */ compressor->num_buffered_elements = 0; } /* Update the number of buffered elements */ compressor->num_buffered_elements = remaining; break; } i += (num_buffer_taken - 1); } /* If all elements were processed, reset buffer. */ if (i == num_buffered_elements) { compressor->num_buffered_elements = 0; } } static inline void simple8brle_compressor_full_flush(Simple8bRleCompressor *compressor) { Assert(compressor->num_buffered_elements > 0); uint8 bit_width = 0; uint64 mask = 0; int16 max_pack = 0; uint8 selector = 0; int32 num_buffered_elements = compressor->num_buffered_elements; for (int32 i = 0; i < num_buffered_elements; i++) { Simple8bRleBuffer *restrict buffer_base = &compressor->uncompressed_buffer[i]; Assert(buffer_base[0].repcount > 0); uint8 first_bit_width = simple8brle_bits_for_value(buffer_base[0].data); selector = SIMPLE8B_SELECTOR_FOR_BIT_WIDTH[first_bit_width]; bit_width = SIMPLE8B_BIT_LENGTH[selector]; /* The current element times the bit width is large enough to flush. */ if (bit_width * buffer_base[0].repcount >= SIMPLE8B_BITSIZE) { uint64 val = buffer_base[0].data; uint64 repcount = buffer_base[0].repcount; /* We can only RLE elements if they are small enough to fit in the * data part of the RLE block. */ if (val <= SIMPLE8B_RLE_MAX_VALUE_MASK) { /* Flush the value as RLE */ uint64 rle_block = (repcount << SIMPLE8B_RLE_MAX_VALUE_BITS) | val; bit_array_append(&compressor->selectors, SIMPLE8B_BITS_PER_SELECTOR, SIMPLE8B_RLE_SELECTOR); uint64_vec_append(&compressor->compressed_data, rle_block); continue; } /* Otherwise we need to flush each element as a full 64 bit block */ else { /* Each repeated element needs to be flushed as separate blocks */ for (uint32 k = 0; k < repcount; k++) { bit_array_append(&compressor->selectors, SIMPLE8B_BITS_PER_SELECTOR, SIMPLE8B_FULL_BLOCK_SELECTOR); uint64_vec_append(&compressor->compressed_data, val); } continue; } } int32 num_packed = buffer_base[0].repcount; max_pack = SIMPLE8B_NUM_ELEMENTS[selector]; mask = simple8brle_selector_get_bitmask(selector); for (int32 j = 1; (i + j) < num_buffered_elements && num_packed < max_pack; ++j) { Assert(buffer_base[j].repcount > 0); uint64 val = buffer_base[j].data; while (val > mask) { /* We bumped into a value that doesn't fit, need to expand selector, * but we need to make sure we don't leave gaps in the packed data. */ ++selector; mask = simple8brle_selector_get_bitmask(selector); if (num_packed >= SIMPLE8B_NUM_ELEMENTS[selector]) break; } max_pack = SIMPLE8B_NUM_ELEMENTS[selector]; num_packed += buffer_base[j].repcount; } /* Final calculations after selector is determined */ bit_width = SIMPLE8B_BIT_LENGTH[selector]; int32 num_buffer_taken = 0; uint64 packed_value = 0; num_packed = 0; /* * This is the main difference between the partial and full flush. * Here we don't need to be smart with the repcounts, because we are * flushing a full block no matter what. No elements can remain in the * uncompressed buffer. * */ while (num_packed < max_pack && (i + num_buffer_taken) < num_buffered_elements) { uint64 val = buffer_base[num_buffer_taken].data; while (buffer_base[num_buffer_taken].repcount > 0 && num_packed < max_pack) { packed_value |= (val & mask) << (num_packed * bit_width); ++num_packed; buffer_base[num_buffer_taken].repcount--; } if (buffer_base[num_buffer_taken].repcount > 0) break; ++num_buffer_taken; } /* Flush the packed value */ bit_array_append(&compressor->selectors, SIMPLE8B_BITS_PER_SELECTOR, selector); uint64_vec_append(&compressor->compressed_data, packed_value); i += (num_buffer_taken - 1); } compressor->num_buffered_elements = 0; } static void simple8brle_compressor_append(Simple8bRleCompressor *compressor, uint64 val) { Assert(compressor != NULL); if (unlikely(compressor->num_buffered_elements >= SIMPLE8B_MAX_BUFFERED)) { simple8brle_compressor_partial_flush(compressor); } uint32 num_buffered = compressor->num_buffered_elements; /* Check for RLE against last element. This saves a few cycles instead of looking * at the last buffered entry. */ if (likely(num_buffered > 0)) { if (likely(compressor->last_value == val)) { Simple8bRleBuffer *restrict last_entry = &compressor->uncompressed_buffer[num_buffered - 1]; if (likely(val == last_entry->data)) { /* Increment count, no new buffer entry needed */ last_entry->repcount++; compressor->num_elements++; return; } } } /* New unique value - buffer it. */ Simple8bRleBuffer *restrict new_entry = &compressor->uncompressed_buffer[num_buffered]; new_entry->data = val; new_entry->repcount = 1; compressor->num_buffered_elements = num_buffered + 1; compressor->num_elements++; compressor->last_value = val; } static bool simple8brle_compressor_is_empty(Simple8bRleCompressor *compressor) { return compressor->num_elements == 0; } static size_t simple8brle_compressor_compressed_size(const Simple8bRleCompressor *compressor) { /* we store 16 selectors per selector_slot, and one selector_slot per compressed_data_slot. * use num_compressed_data_slots / 16 + 1 to ensure that rounding doesn't truncate our slots * and that we always have a 0 slot at the end. */ return sizeof(Simple8bRleSerialized) + compressor->compressed_data.num_elements * sizeof(*compressor->compressed_data.data) + bit_array_data_bytes_used(&compressor->selectors); } static size_t simple8brle_compressor_compressed_const_size(const Simple8bRleCompressor *compressor) { /* Allocate temp space where the temp_compressor will put the data. Prefer static * allocation to avoid palloc overhead, since this is only used for size calculation. */ #define TEMP_DATA_SIZE TARGET_COMPRESSED_BATCH_SIZE #define TEMP_SELECTORS_SIZE (TEMP_DATA_SIZE / SIMPLE8B_SELECTORS_PER_SELECTOR_SLOT) uint64 temp_data_static[TEMP_DATA_SIZE]; uint64 temp_selectors_static[TEMP_SELECTORS_SIZE]; uint64 temp_data_count = TEMP_DATA_SIZE; uint64 temp_selectors_count = TEMP_SELECTORS_SIZE; uint64 *temp_data = temp_data_static; uint64 *temp_selectors = temp_selectors_static; bool use_static = true; Simple8bRleCompressor temp_compressor = *compressor; /* Replace the data and selectors with the temp space.*/ temp_compressor.compressed_data.data = temp_data; temp_compressor.compressed_data.num_elements = 0; temp_compressor.compressed_data.max_elements = TEMP_DATA_SIZE; temp_compressor.selectors.buckets.data = temp_selectors; temp_compressor.selectors.buckets.num_elements = 0; temp_compressor.selectors.buckets.max_elements = TEMP_SELECTORS_SIZE; temp_compressor.selectors.bits_used_in_last_bucket = 0; /* If the compressor is empty, we can return 0, similar to finish. */ if (temp_compressor.num_elements == 0) return 0; /* * If the compressor has no uncompressed data, we can use the original size calculation. * Note that after every append, it is guaranteed that we have at least one uncompressed * element. Not having uncompressed elements can only happen if the compressor is empty * or finish was called, so we can use the original size calculation. */ if (compressor->num_buffered_elements == 0) return simple8brle_compressor_compressed_size(compressor); /* Because the buffering allows RLE entries to be stored, the worst case scenario for * the buffer size is determined by the repcounts in the buffered elements. */ uint32 actual_buffered = 0; for (uint32 i = 0; i < compressor->num_buffered_elements; i++) { actual_buffered += compressor->uncompressed_buffer[i].repcount; } if (actual_buffered > temp_data_count) { /* We need to increase the temp buffer and allocate it dynamically. * This can only happen if the TARGET_COMPRESSED_BATCH_SIZE is smaller * than the actual buffered elements, which is unlikely. */ temp_data_count = actual_buffered; temp_data = palloc(temp_data_count * sizeof(uint64)); /* Similarly the selectors */ temp_selectors_count = (actual_buffered + 15) / 16; temp_selectors = palloc(temp_selectors_count * sizeof(uint64)); use_static = false; } /* Flush the compressor to ensure all uncompressed data is compressed into the temp_compressor. */ simple8brle_compressor_full_flush(&temp_compressor); size_t num_data_blocks = compressor->compressed_data.num_elements + temp_compressor.compressed_data.num_elements; /* Add up the size of the compressed data blocks and the header. */ size_t result = sizeof(Simple8bRleSerialized) + num_data_blocks * sizeof(*temp_compressor.compressed_data.data); /* Add up the selector bits. */ size_t selector_bits = num_data_blocks * SIMPLE8B_BITS_PER_SELECTOR; result += ((selector_bits + 63) / 64) * sizeof(uint64); #undef TEMP_DATA_SIZE #undef TEMP_SELECTORS_SIZE if (!use_static) { /* If we allocated dynamically, we need to free the memory. */ pfree(temp_data); pfree(temp_selectors); } return result; } static inline uint32 simple8brle_compressor_num_selectors(Simple8bRleCompressor *compressor) { Assert(bit_array_num_bits(&compressor->selectors) % SIMPLE8B_BITS_PER_SELECTOR == 0); return bit_array_num_bits(&compressor->selectors) / SIMPLE8B_BITS_PER_SELECTOR; } static Simple8bRleSerialized * simple8brle_compressor_finish(Simple8bRleCompressor *compressor) { size_t size_left; size_t selector_size; size_t compressed_size; Simple8bRleSerialized *compressed; uint64 bits; if (compressor->num_elements == 0) return NULL; /* Flush any remaining state */ if (compressor->num_buffered_elements > 0) { simple8brle_compressor_full_flush(compressor); } Assert(compressor->num_buffered_elements == 0); compressed_size = simple8brle_compressor_compressed_size(compressor); /* we use palloc0 despite initializing the entire structure, * to ensure padding bits are zeroed, and that there's a 0 selector at the end. * It would be more efficient to ensure there are no padding bits in the struct, * and initialize everything ourselves */ compressed = palloc0(compressed_size); Assert(bit_array_num_buckets(&compressor->selectors) > 0); Assert(compressor->compressed_data.num_elements > 0); Assert(compressor->compressed_data.num_elements == simple8brle_compressor_num_selectors(compressor)); *compressed = (Simple8bRleSerialized){ .num_elements = compressor->num_elements, .num_blocks = compressor->compressed_data.num_elements, }; size_left = compressed_size - sizeof(*compressed); Assert(size_left >= bit_array_data_bytes_used(&compressor->selectors)); selector_size = bit_array_output(&compressor->selectors, compressed->slots, size_left, &bits); size_left -= selector_size; Assert(size_left == (compressor->compressed_data.num_elements * sizeof(*compressor->compressed_data.data))); Assert(compressor->selectors.buckets.num_elements == simple8brle_num_selector_slots_for_num_blocks(compressor->compressed_data.num_elements)); memcpy(compressed->slots + compressor->selectors.buckets.num_elements, compressor->compressed_data.data, size_left); return compressed; } static char * simple8brle_compressor_finish_into(Simple8bRleCompressor *compressor, char *dest, size_t expected_size) { size_t size_left; size_t selector_size; size_t compressed_size; char *end_ptr; Simple8bRleSerialized *compressed; uint64 bits; if (compressor->num_elements == 0) return NULL; Ensure(dest != NULL, "dest is NULL"); /* Flush any remaining state */ if (compressor->num_buffered_elements > 0) { simple8brle_compressor_full_flush(compressor); } Assert(compressor->num_buffered_elements == 0); compressed_size = simple8brle_compressor_compressed_size(compressor); Ensure(expected_size == compressed_size, "expected_size: %zu, compressed_size: %zu", expected_size, compressed_size); compressed = (Simple8bRleSerialized *) dest; Assert(bit_array_num_buckets(&compressor->selectors) > 0); Assert(compressor->compressed_data.num_elements > 0); Assert(compressor->compressed_data.num_elements == simple8brle_compressor_num_selectors(compressor)); *compressed = (Simple8bRleSerialized){ .num_elements = compressor->num_elements, .num_blocks = compressor->compressed_data.num_elements, }; size_left = compressed_size - sizeof(*compressed); Assert(size_left >= bit_array_data_bytes_used(&compressor->selectors)); selector_size = bit_array_output(&compressor->selectors, compressed->slots, size_left, &bits); size_left -= selector_size; Assert(size_left == (compressor->compressed_data.num_elements * sizeof(*compressor->compressed_data.data))); Assert(compressor->selectors.buckets.num_elements == simple8brle_num_selector_slots_for_num_blocks(compressor->compressed_data.num_elements)); memcpy(compressed->slots + compressor->selectors.buckets.num_elements, compressor->compressed_data.data, size_left); end_ptr = (char *) (compressed->slots + compressor->selectors.buckets.num_elements) + size_left; Assert(end_ptr == dest + expected_size); return end_ptr; } /****************************************** *** Simple8bRleDecompressionIterator *** ******************************************/ static void simple8brle_decompression_iterator_init_common(Simple8bRleDecompressionIterator *iter, Simple8bRleSerialized *compressed) { uint32 num_selector_slots = simple8brle_num_selector_slots_for_num_blocks(compressed->num_blocks); *iter = (Simple8bRleDecompressionIterator){ .compressed_data = compressed->slots + num_selector_slots, .num_blocks = compressed->num_blocks, .current_compressed_pos = 0, .current_in_compressed_pos = 0, .num_elements = compressed->num_elements, .num_elements_returned = 0, }; bit_array_wrap(&iter->selector_data, compressed->slots, compressed->num_blocks * SIMPLE8B_BITS_PER_SELECTOR); } static void simple8brle_decompression_iterator_init_forward(Simple8bRleDecompressionIterator *iter, Simple8bRleSerialized *compressed) { simple8brle_decompression_iterator_init_common(iter, compressed); bit_array_iterator_init(&iter->selectors, &iter->selector_data); } static uint32 simple8brle_decompression_iterator_max_elements(Simple8bRleDecompressionIterator *iter, const Simple8bRleSerialized *compressed) { BitArrayIterator selectors; uint32 max_stored = 0; uint32 i; Assert(compressed->num_blocks > 0); bit_array_iterator_init(&selectors, iter->selectors.array); for (i = 0; i < compressed->num_blocks; i++) { uint8 selector = bit_array_iter_next(&selectors, SIMPLE8B_BITS_PER_SELECTOR); if (selector == 0) elog(ERROR, "invalid selector 0"); if (simple8brle_selector_is_rle(selector) && iter->compressed_data) { Assert(simple8brle_rledata_repeatcount(iter->compressed_data[i]) > 0); max_stored += simple8brle_rledata_repeatcount(iter->compressed_data[i]); } else { Assert(selector < SIMPLE8B_MAXCODE); max_stored += SIMPLE8B_NUM_ELEMENTS[selector]; } } return max_stored; } static void simple8brle_decompression_iterator_init_reverse(Simple8bRleDecompressionIterator *iter, Simple8bRleSerialized *compressed) { int32 skipped_in_last; simple8brle_decompression_iterator_init_common(iter, compressed); bit_array_iterator_init_rev(&iter->selectors, &iter->selector_data); skipped_in_last = simple8brle_decompression_iterator_max_elements(iter, compressed) - compressed->num_elements; Assert(NULL != iter->compressed_data); iter->current_block = simple8brle_block_create(bit_array_iter_next_rev(&iter->selectors, SIMPLE8B_BITS_PER_SELECTOR), iter->compressed_data[compressed->num_blocks - 1]); iter->current_in_compressed_pos = iter->current_block.num_elements_compressed - 1 - skipped_in_last; iter->current_compressed_pos = compressed->num_blocks - 2; return; } /* returning a struct produces noticeably better assembly on x86_64 than returning * is_done and is_null via pointers; it uses two registers instead of any memory reads. * Since it is also easier to read, we prefer it here. */ static Simple8bRleDecompressResult simple8brle_decompression_iterator_try_next_forward(Simple8bRleDecompressionIterator *iter) { uint64 uncompressed; if (iter->num_elements_returned >= iter->num_elements) return (Simple8bRleDecompressResult){ .is_done = true, }; if ((uint32) iter->current_in_compressed_pos >= iter->current_block.num_elements_compressed) { CheckCompressedData(iter->current_compressed_pos < iter->num_blocks); iter->current_block = simple8brle_block_create(bit_array_iter_next(&iter->selectors, SIMPLE8B_BITS_PER_SELECTOR), iter->compressed_data[iter->current_compressed_pos]); CheckCompressedData(iter->current_block.selector != 0); CheckCompressedData(iter->current_block.num_elements_compressed <= GLOBAL_MAX_ROWS_PER_COMPRESSION); iter->current_compressed_pos += 1; iter->current_in_compressed_pos = 0; } uncompressed = simple8brle_block_get_element(iter->current_block, iter->current_in_compressed_pos); iter->num_elements_returned += 1; iter->current_in_compressed_pos += 1; return (Simple8bRleDecompressResult){ .val = uncompressed, }; } static Simple8bRleDecompressResult simple8brle_decompression_iterator_try_next_reverse(Simple8bRleDecompressionIterator *iter) { uint64 uncompressed; if (iter->num_elements_returned >= iter->num_elements) return (Simple8bRleDecompressResult){ .is_done = true, }; if (iter->current_in_compressed_pos < 0) { iter->current_block = simple8brle_block_create(bit_array_iter_next_rev(&iter->selectors, SIMPLE8B_BITS_PER_SELECTOR), iter->compressed_data[iter->current_compressed_pos]); iter->current_in_compressed_pos = iter->current_block.num_elements_compressed - 1; iter->current_compressed_pos -= 1; } uncompressed = simple8brle_block_get_element(iter->current_block, iter->current_in_compressed_pos); iter->num_elements_returned += 1; iter->current_in_compressed_pos -= 1; return (Simple8bRleDecompressResult){ .val = uncompressed, }; } /************************** *** Simple8bRleBlock *** **************************/ static pg_attribute_always_inline Simple8bRleBlock simple8brle_block_create(uint8 selector, uint64 data) { Simple8bRleBlock block = (Simple8bRleBlock){ .selector = selector, .data = data, }; if (simple8brle_selector_is_rle(block.selector)) { block.num_elements_compressed = simple8brle_rledata_repeatcount(block.data); } else { block.num_elements_compressed = SIMPLE8B_NUM_ELEMENTS[block.selector]; } return block; } static inline uint64 simple8brle_block_get_element(Simple8bRleBlock block, uint32 position_in_value) { /* we're using 0 for end-of-stream, but haven't decided what to use it for */ if (block.selector == 0) { elog(ERROR, "end of compressed integer stream"); } else if (simple8brle_selector_is_rle(block.selector)) { /* decode rle-encoded integers */ uint64 repeated_value = simple8brle_rledata_value(block.data); CheckCompressedData(simple8brle_rledata_repeatcount(block.data) > 0); Assert(simple8brle_rledata_repeatcount(block.data) > position_in_value); return repeated_value; } else { uint64 compressed_value = block.data; uint32 bits_per_val = SIMPLE8B_BIT_LENGTH[block.selector]; /* decode bit-packed integers*/ Assert(position_in_value < SIMPLE8B_NUM_ELEMENTS[block.selector]); compressed_value >>= bits_per_val * position_in_value; compressed_value &= simple8brle_selector_get_bitmask(block.selector); return compressed_value; } pg_unreachable(); } /*************************** *** Utility Functions *** ***************************/ static pg_attribute_always_inline bool simple8brle_selector_is_rle(uint8 selector) { return selector == SIMPLE8B_RLE_SELECTOR; } static pg_attribute_always_inline uint32 simple8brle_rledata_repeatcount(uint64 rledata) { return (uint32) ((rledata >> SIMPLE8B_RLE_MAX_VALUE_BITS) & SIMPLE8B_RLE_MAX_COUNT_MASK); } static pg_attribute_always_inline uint64 simple8brle_rledata_value(uint64 rledata) { return rledata & SIMPLE8B_RLE_MAX_VALUE_MASK; } static pg_attribute_always_inline uint64 simple8brle_selector_get_bitmask(uint8 selector) { uint8 bitLen = SIMPLE8B_BIT_LENGTH[selector]; Assert(bitLen != 0); uint64 result = ((~0ULL) >> (64 - bitLen)); return result; } static pg_attribute_always_inline uint32 simple8brle_num_selector_slots_for_num_blocks(uint32 num_blocks) { return (num_blocks / SIMPLE8B_SELECTORS_PER_SELECTOR_SLOT) + (num_blocks % SIMPLE8B_SELECTORS_PER_SELECTOR_SLOT != 0 ? 1 : 0); } #ifdef HAVE_BUILTIN_CLZLL static inline uint32 simple8brle_bits_for_value(uint64 v) { if (v == 0) return 0; return 64 - __builtin_clzll(v); } #else static inline uint32 simple8brle_bits_for_value(uint64 v) { uint32 r = 0; if (v >= (1U << 31)) { v >>= 32; r += 32; } if (v >= (1U << 15)) { v >>= 16; r += 16; } if (v >= (1U << 7)) { v >>= 8; r += 8; } if (v >= (1U << 3)) { v >>= 4; r += 4; } if (v >= (1U << 1)) { v >>= 2; r += 2; } if (v >= (1U << 0)) { v >>= 1; r += 1; } return r; } #endif ================================================ FILE: tsl/src/compression/algorithms/simple8b_rle_bitarray.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include "simple8b_rle.h" /* * This is a specialization of Simple8bRLE decoder for encoded 1 bit values * as they are used to store NULL flags in the compression methods as well as * the values for bool compression. * * Note that in the bool compression we store a validity map instead of a NULL * map, which is the same except the bits are inverted. * * The goal of this decoder is to support the following use cases: * * 1. Decompress the validity map of the bool compression method. * 2. Decompress the values of the bool compression method. * 3. Decompress the NULL map of the other compression methods into a validity * map in the ArrowArray. In this case the bits will be inverted. * * The reason we don't use the Simple8bRleBitmap is that the end result is an * array of bits and not bools. * * The complication comes from the RLE encoding of Simple8b while in the Arrow * validity bitmaps we have a straight array of bits. */ typedef struct Simple8bRleBitArray { uint64 *data; uint32 num_elements; uint32 num_blocks; uint16 num_ones; } Simple8bRleBitArray; static Simple8bRleBitArray simple8brle_bitarray_decompress(Simple8bRleSerialized *compressed, bool inverted) { Simple8bRleBitArray result = { 0 }; if (!compressed) { return result; } CheckCompressedData(compressed->num_elements <= GLOBAL_MAX_ROWS_PER_COMPRESSION); CheckCompressedData(compressed->num_blocks <= GLOBAL_MAX_ROWS_PER_COMPRESSION); const uint32 num_elements = compressed->num_elements; const uint32 num_selector_slots = simple8brle_num_selector_slots_for_num_blocks(compressed->num_blocks); const uint64 *compressed_data = compressed->slots + num_selector_slots; const uint32 num_elements_padded = ((num_elements + 63) / 64 + 1) * 64; const uint32 num_blocks = compressed->num_blocks; result.data = palloc0(num_elements_padded / 64 * sizeof(uint64)); result.num_elements = num_elements; result.num_blocks = num_blocks; uint64 *restrict current_output_ptr = result.data; uint32 decompressed_index = 0; uint32 bit_position = 0; for (uint32 block_index = 0; block_index < num_blocks; block_index++) { const uint32 selector_slot = block_index / SIMPLE8B_SELECTORS_PER_SELECTOR_SLOT; const uint32 selector_pos_in_slot = block_index % SIMPLE8B_SELECTORS_PER_SELECTOR_SLOT; const uint64 slot_value = compressed->slots[selector_slot]; const uint8 selector_shift = selector_pos_in_slot * SIMPLE8B_BITS_PER_SELECTOR; const uint64 selector_mask = 0xFULL << selector_shift; const uint8 selector_value = (slot_value & selector_mask) >> selector_shift; Assert(selector_value < 16); uint64 block_data = compressed_data[block_index]; if (simple8brle_selector_is_rle(selector_value)) { /* * RLE block. */ uint32 repeat_count = simple8brle_rledata_repeatcount(block_data); CheckCompressedData(repeat_count <= GLOBAL_MAX_ROWS_PER_COMPRESSION); /* * We might get an incorrect value from the corrupt data. Explicitly * truncate it to 0/1 in case the bool is not a standard bool type * which would have done it for us. */ const bool repeated_value = simple8brle_rledata_value(block_data) & 1; const bool bit_value = repeated_value ^ inverted; CheckCompressedData(decompressed_index + repeat_count <= num_elements); if (bit_value) { result.num_ones += repeat_count; /* Repeated 'ones' repeat_count times */ if ((repeat_count + bit_position) >= 64) { /* Head: Fill the remaining bits in the current word if not aligned */ if (bit_position > 0) { uint64_t head_bits = 64 - bit_position; uint64_t head_mask = (1ULL << head_bits) - 1; *current_output_ptr |= (head_mask << bit_position); repeat_count -= head_bits; decompressed_index += head_bits; current_output_ptr++; bit_position = 0; } /* Middle: Fill complete words */ uint64_t full_words = repeat_count / 64; for (uint64_t j = 0; j < full_words; j++) { *current_output_ptr = 0xFFFFFFFFFFFFFFFF; current_output_ptr++; } decompressed_index += full_words * 64; repeat_count -= full_words * 64; } /* Tail: Handle remaining bits (less than 64) */ if (repeat_count > 0) { Assert(repeat_count < 64); uint64_t tail_mask = (1ULL << (repeat_count & 63)) - 1; *current_output_ptr |= (tail_mask << bit_position); decompressed_index += repeat_count; bit_position = (bit_position + repeat_count) % 64; if (bit_position == 0) { current_output_ptr++; } else { current_output_ptr = result.data + (decompressed_index / 64); } } } else { decompressed_index += repeat_count; bit_position = decompressed_index % 64; current_output_ptr = result.data + (decompressed_index / 64); } Assert(decompressed_index <= num_elements); } else { /* * Bit-packed block. Since this is a bitmap, this block has 64 bits * packed. The last block might contain less than maximal possible * number of elements, but we have 64 bytes of padding on the right * so we don't care. */ CheckCompressedData(selector_value == 1); Assert(SIMPLE8B_BIT_LENGTH[selector_value] == 1); Assert(SIMPLE8B_NUM_ELEMENTS[selector_value] == 64); /* * We should require at least one element from the block. Previous * blocks might have had incorrect lengths, so this is not an * assertion. */ CheckCompressedData(decompressed_index < num_elements); CheckCompressedData(decompressed_index + 64 < num_elements_padded); /* Have to zero out the unused bits, so that the popcnt works properly. */ const int elements_this_block = Min(64, num_elements - decompressed_index); Assert(elements_this_block <= 64); Assert(elements_this_block > 0); block_data = block_data ^ -(uint64_t) inverted; block_data &= (~0ULL) >> (64 - elements_this_block); if (bit_position == 0) { /* The decoding is on exact 64bit boundaries */ *current_output_ptr = block_data; } else { /* We need to split the word */ uint64_t bits_remaining_in_word = 64 - bit_position; /* First part goes at the end of the current word */ *current_output_ptr |= (block_data << bit_position); /* Second part goes at the beginning of the next word */ *(current_output_ptr + 1) |= block_data >> bits_remaining_in_word; } #ifdef HAVE__BUILTIN_POPCOUNT result.num_ones += __builtin_popcountll(block_data); #else for (uint16 i = 0; i < 64; i++) result.num_ones += ((block_data >> i) & 1); #endif decompressed_index += 64; bit_position = decompressed_index % 64; current_output_ptr = result.data + (decompressed_index / 64); } } /* * We might have unpacked more because we work in full blocks, but at least * we shouldn't have unpacked less. */ CheckCompressedData(decompressed_index >= num_elements); Assert(decompressed_index <= num_elements_padded); /* * Might happen if we have stray ones in the higher unused bits of the last * block. */ CheckCompressedData(result.num_ones <= num_elements); return result; } ================================================ FILE: tsl/src/compression/algorithms/simple8b_rle_bitmap.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once /* * This is a specialization of Simple8bRLE decoder for bitmaps, i.e. where the * elements are only 0 and 1. It also counts the number of ones. */ #include "simple8b_rle.h" typedef struct Simple8bRleBitmap { /* Either the bools or prefix sums, depending on the decompression method. */ const void *data; uint16 num_elements; uint16 num_ones; } Simple8bRleBitmap; pg_attribute_always_inline static bool simple8brle_bitmap_get_at(const Simple8bRleBitmap *bitmap, uint16 i) { /* We have some padding on the right but we shouldn't overrun it. */ Assert(i < ((bitmap->num_elements + 63) / 64 + 1) * 64); return ((const bool *) bitmap->data)[i]; } pg_attribute_always_inline static uint16 simple8brle_bitmap_prefix_sum(const Simple8bRleBitmap *bitmap, uint16 i) { Assert(i < ((bitmap->num_elements + 63) / 64 + 1) * 64); return ((const uint16 *) bitmap->data)[i]; } pg_attribute_always_inline static uint16 simple8brle_bitmap_num_ones(const Simple8bRleBitmap *bitmap) { return bitmap->num_ones; } /* * Calculate prefix sum of bits instead of bitmap itself, because it's more * useful for gorilla decompression. Can be unused by other users of this * header. */ static Simple8bRleBitmap simple8brle_bitmap_prefixsums(Simple8bRleSerialized *compressed) pg_attribute_unused(); static Simple8bRleBitmap simple8brle_bitmap_prefixsums(Simple8bRleSerialized *compressed) { CheckCompressedData(compressed->num_elements <= GLOBAL_MAX_ROWS_PER_COMPRESSION); CheckCompressedData(compressed->num_blocks <= GLOBAL_MAX_ROWS_PER_COMPRESSION); const uint32 num_elements = compressed->num_elements; const uint32 num_selector_slots = simple8brle_num_selector_slots_for_num_blocks(compressed->num_blocks); const uint64 *compressed_data = compressed->slots + num_selector_slots; /* * Pad to next multiple of 64 bytes on the right, so that we can simplify the * decompression loop and the get() function. Note that for get() we need at * least one byte of padding, hence the next multiple. */ const uint32 num_elements_padded = ((num_elements + 63) / 64 + 1) * 64; const uint32 num_blocks = compressed->num_blocks; uint16 *restrict prefix_sums = palloc(sizeof(uint16) * num_elements_padded); uint32 num_ones = 0; uint32 decompressed_index = 0; for (uint32 block_index = 0; block_index < num_blocks; block_index++) { const uint32 selector_slot = block_index / SIMPLE8B_SELECTORS_PER_SELECTOR_SLOT; const uint32 selector_pos_in_slot = block_index % SIMPLE8B_SELECTORS_PER_SELECTOR_SLOT; const uint64 slot_value = compressed->slots[selector_slot]; const uint8 selector_shift = selector_pos_in_slot * SIMPLE8B_BITS_PER_SELECTOR; const uint64 selector_mask = 0xFULL << selector_shift; const uint8 selector_value = (slot_value & selector_mask) >> selector_shift; Assert(selector_value < 16); uint64 block_data = compressed_data[block_index]; if (simple8brle_selector_is_rle(selector_value)) { /* * RLE block. */ const uint32 n_block_values = simple8brle_rledata_repeatcount(block_data); CheckCompressedData(n_block_values <= GLOBAL_MAX_ROWS_PER_COMPRESSION); /* * We might get an incorrect value from the corrupt data. Explicitly * truncate it to 0/1 in case the bool is not a standard bool type * which would have done it for us. */ const bool repeated_value = simple8brle_rledata_value(block_data) & 1; CheckCompressedData(decompressed_index + n_block_values <= num_elements); if (repeated_value) { for (uint32 i = 0; i < n_block_values; i++) { prefix_sums[decompressed_index + i] = num_ones + i + 1; } num_ones += n_block_values; } else { for (uint32 i = 0; i < n_block_values; i++) { prefix_sums[decompressed_index + i] = num_ones; } } decompressed_index += n_block_values; Assert(decompressed_index <= num_elements); } else { /* * Bit-packed block. Since this is a bitmap, this block has 64 bits * packed. The last block might contain less than maximal possible * number of elements, but we have 64 bytes of padding on the right * so we don't care. */ CheckCompressedData(selector_value == 1); Assert(SIMPLE8B_BIT_LENGTH[selector_value] == 1); Assert(SIMPLE8B_NUM_ELEMENTS[selector_value] == 64); /* * We should require at least one element from the block. Previous * blocks might have had incorrect lengths, so this is not an * assertion. */ CheckCompressedData(decompressed_index < num_elements); /* Have to zero out the unused bits, so that the popcnt works properly. */ const int elements_this_block = Min(64, num_elements - decompressed_index); Assert(elements_this_block <= 64); Assert(elements_this_block > 0); block_data &= (~0ULL) >> (64 - elements_this_block); /* * The number of block elements should fit within padding. Previous * blocks might have had incorrect lengths, so this is not an * assertion. */ CheckCompressedData(decompressed_index + 64 < num_elements_padded); #ifdef HAVE__BUILTIN_POPCOUNT for (uint16 i = 0; i < 64; i++) { const uint16 word_prefix_sum = __builtin_popcountll(block_data & ((~0ULL) >> (63 - i))); prefix_sums[decompressed_index + i] = num_ones + word_prefix_sum; } num_ones += __builtin_popcountll(block_data); #else /* * Unfortunately, we have to have this fallback for Windows. */ for (uint16 i = 0; i < 64; i++) { const bool this_bit = (block_data >> i) & 1; num_ones += this_bit; prefix_sums[decompressed_index + i] = num_ones; } #endif decompressed_index += 64; } } /* * We might have unpacked more because we work in full blocks, but at least * we shouldn't have unpacked less. */ CheckCompressedData(decompressed_index >= num_elements); Assert(decompressed_index <= num_elements_padded); /* * Might happen if we have stray ones in the higher unused bits of the last * block. */ CheckCompressedData(num_ones <= num_elements); /* * Check that the number of ones actually fits into the uint16 counters * we're using. */ CheckCompressedData(((uint16) num_ones) == num_ones); Simple8bRleBitmap result = { .data = prefix_sums, .num_elements = num_elements, .num_ones = num_ones, }; return result; } static Simple8bRleBitmap simple8brle_bitmap_decompress(Simple8bRleSerialized *compressed) { CheckCompressedData(compressed->num_elements <= GLOBAL_MAX_ROWS_PER_COMPRESSION); CheckCompressedData(compressed->num_blocks <= GLOBAL_MAX_ROWS_PER_COMPRESSION); const uint32 num_elements = compressed->num_elements; const uint32 num_selector_slots = simple8brle_num_selector_slots_for_num_blocks(compressed->num_blocks); const uint64 *compressed_data = compressed->slots + num_selector_slots; /* * Pad to next multiple of 64 bytes on the right, so that we can simplify the * decompression loop and the get() function. Note that for get() we need at * least one byte of padding, hence the next multiple. */ const uint32 num_elements_padded = ((num_elements + 63) / 64 + 1) * 64; const uint32 num_blocks = compressed->num_blocks; bool *restrict bitmap_bools_ = palloc(sizeof(bool) * num_elements_padded); uint32 num_ones = 0; uint32 decompressed_index = 0; for (uint32 block_index = 0; block_index < num_blocks; block_index++) { const uint32 selector_slot = block_index / SIMPLE8B_SELECTORS_PER_SELECTOR_SLOT; const uint32 selector_pos_in_slot = block_index % SIMPLE8B_SELECTORS_PER_SELECTOR_SLOT; const uint64 slot_value = compressed->slots[selector_slot]; const uint8 selector_shift = selector_pos_in_slot * SIMPLE8B_BITS_PER_SELECTOR; const uint64 selector_mask = 0xFULL << selector_shift; const uint8 selector_value = (slot_value & selector_mask) >> selector_shift; Assert(selector_value < 16); uint64 block_data = compressed_data[block_index]; if (simple8brle_selector_is_rle(selector_value)) { /* * RLE block. */ const uint32 n_block_values = simple8brle_rledata_repeatcount(block_data); CheckCompressedData(n_block_values <= GLOBAL_MAX_ROWS_PER_COMPRESSION); /* * We might get an incorrect value from the corrupt data. Explicitly * truncate it to 0/1 in case the bool is not a standard bool type * which would have done it for us. */ const bool repeated_value = simple8brle_rledata_value(block_data) & 1; CheckCompressedData(decompressed_index + n_block_values <= num_elements); if (repeated_value) { for (uint32 i = 0; i < n_block_values; i++) { bitmap_bools_[decompressed_index + i] = true; } num_ones += n_block_values; } else { for (uint32 i = 0; i < n_block_values; i++) { bitmap_bools_[decompressed_index + i] = false; } } decompressed_index += n_block_values; Assert(decompressed_index <= num_elements); } else { /* * Bit-packed block. Since this is a bitmap, this block has 64 bits * packed. The last block might contain less than maximal possible * number of elements, but we have 64 bytes of padding on the right * so we don't care. */ CheckCompressedData(selector_value == 1); Assert(SIMPLE8B_BIT_LENGTH[selector_value] == 1); Assert(SIMPLE8B_NUM_ELEMENTS[selector_value] == 64); /* * We should require at least one element from the block. Previous * blocks might have had incorrect lengths, so this is not an * assertion. */ CheckCompressedData(decompressed_index < num_elements); /* Have to zero out the unused bits, so that the popcnt works properly. */ const int elements_this_block = Min(64, num_elements - decompressed_index); Assert(elements_this_block <= 64); Assert(elements_this_block > 0); block_data &= (~0ULL) >> (64 - elements_this_block); /* * The number of block elements should fit within padding. Previous * blocks might have had incorrect lengths, so this is not an * assertion. */ CheckCompressedData(decompressed_index + 64 < num_elements_padded); #ifdef HAVE__BUILTIN_POPCOUNT num_ones += __builtin_popcountll(block_data); #endif for (uint16 i = 0; i < 64; i++) { const bool this_bit = (block_data >> i) & 1; bitmap_bools_[decompressed_index + i] = this_bit; #ifndef HAVE__BUILTIN_POPCOUNT num_ones += this_bit; #endif } decompressed_index += 64; } } /* * We might have unpacked more because we work in full blocks, but at least * we shouldn't have unpacked less. */ CheckCompressedData(decompressed_index >= num_elements); Assert(decompressed_index <= num_elements_padded); /* * Might happen if we have stray ones in the higher unused bits of the last * block. */ CheckCompressedData(num_ones <= num_elements); Simple8bRleBitmap result = { .data = bitmap_bools_, .num_elements = num_elements, .num_ones = num_ones, }; /* Sanity check. */ #ifdef USE_ASSERT_CHECKING uint32 num_ones_2 = 0; for (uint32 i = 0; i < num_elements; i++) { num_ones_2 += simple8brle_bitmap_get_at(&result, i); } Assert(num_ones_2 == num_ones); #endif return result; } ================================================ FILE: tsl/src/compression/algorithms/simple8b_rle_decompress_all.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #define FUNCTION_NAME_HELPER(X, Y) X##_##Y #define FUNCTION_NAME(X, Y) FUNCTION_NAME_HELPER(X, Y) /* * Specialization of bulk simple8brle decompression for a data type specified by * ELEMENT_TYPE macro. * * The buffer must have a padding of 63 elements after the last one, because * decompression is performed always in full blocks. */ static uint32 FUNCTION_NAME(simple8brle_decompress_all_buf, ELEMENT_TYPE)(Simple8bRleSerialized *compressed, ELEMENT_TYPE *restrict decompressed_values, uint32 n_buffer_elements) { const uint32 n_total_values = compressed->num_elements; /* * Caller must have allocated a properly sized buffer, see the comment above. */ Assert(n_buffer_elements >= n_total_values + 63); const uint32 num_selector_slots = simple8brle_num_selector_slots_for_num_blocks(compressed->num_blocks); const uint32 num_blocks = compressed->num_blocks; /* * Unpack the selector slots to get the selector values. Best done separately, * so that this loop can be vectorized. */ Assert(num_blocks <= GLOBAL_MAX_ROWS_PER_COMPRESSION); uint8 selector_values[GLOBAL_MAX_ROWS_PER_COMPRESSION]; const uint64 *slots = compressed->slots; for (uint32 block_index = 0; block_index < num_blocks; block_index++) { const uint32 selector_slot = block_index / SIMPLE8B_SELECTORS_PER_SELECTOR_SLOT; const uint32 selector_pos_in_slot = block_index % SIMPLE8B_SELECTORS_PER_SELECTOR_SLOT; const uint64 slot_value = slots[selector_slot]; const uint8 selector_shift = selector_pos_in_slot * SIMPLE8B_BITS_PER_SELECTOR; const uint64 selector_mask = 0xFULL << selector_shift; const uint8 selector_value = (slot_value & selector_mask) >> selector_shift; selector_values[block_index] = selector_value; } /* * Now decompress the individual blocks. */ uint32 decompressed_index = 0; const uint64 *blocks = compressed->slots + num_selector_slots; for (uint32 block_index = 0; block_index < num_blocks; block_index++) { const uint8 selector_value = selector_values[block_index]; const uint64 block_data = blocks[block_index]; /* We don't see RLE blocks so often in the real data, <1% of blocks. */ if (unlikely(simple8brle_selector_is_rle(selector_value))) { const uint16 n_block_values = simple8brle_rledata_repeatcount(block_data); CheckCompressedData(n_block_values <= n_buffer_elements); CheckCompressedData(decompressed_index <= n_buffer_elements - n_block_values); const uint64 repeated_value_raw = simple8brle_rledata_value(block_data); const ELEMENT_TYPE repeated_value_converted = repeated_value_raw; CheckCompressedData(repeated_value_raw == (uint64) repeated_value_converted); for (uint16 i = 0; i < n_block_values; i++) { decompressed_values[decompressed_index + i] = repeated_value_converted; } decompressed_index += n_block_values; } else { /* Bit-packed blocks. Generate separate code for each block type. */ #define UNPACK_BLOCK(X) \ case (X): \ { \ /* \ * Error out it if the bit width is higher than that of the destination \ * type. We could just skip it, but this way the result of e.g. gorilla \ * decompression will be closer to what the row-by-row decompression \ * produces, which is easier for testing. \ */ \ const uint8 bits_per_value = SIMPLE8B_BIT_LENGTH[X]; \ CheckCompressedData(bits_per_value <= sizeof(ELEMENT_TYPE) * 8); \ \ /* \ * The last block might have less values than normal, but we have \ * padding at the end so we can unpack them all always for simpler \ * code. We still have to check if they fit, because the incoming data \ * might be incorrect. \ */ \ const uint16 n_block_values = SIMPLE8B_NUM_ELEMENTS[X]; \ CheckCompressedData(n_block_values <= n_buffer_elements); \ CheckCompressedData(decompressed_index <= n_buffer_elements - n_block_values); \ \ const uint64 bitmask = simple8brle_selector_get_bitmask(X); \ \ for (uint16 i = 0; i < n_block_values; i++) \ { \ const ELEMENT_TYPE value = (block_data >> (bits_per_value * i)) & bitmask; \ decompressed_values[decompressed_index + i] = value; \ } \ decompressed_index += n_block_values; \ break; \ } switch (selector_value) { UNPACK_BLOCK(1); UNPACK_BLOCK(2); UNPACK_BLOCK(3); UNPACK_BLOCK(4); UNPACK_BLOCK(5); UNPACK_BLOCK(6); UNPACK_BLOCK(7); UNPACK_BLOCK(8); UNPACK_BLOCK(9); UNPACK_BLOCK(10); UNPACK_BLOCK(11); UNPACK_BLOCK(12); UNPACK_BLOCK(13); UNPACK_BLOCK(14); default: /* * Can only get 0 here in case the data is corrupt. Doesn't * harm to report it right away, because this loop can't be * vectorized. */ CheckCompressedData(false); } #undef UNPACK_BLOCK } } /* * We can decompress more than expected because we work in full blocks, * but if we decompressed less, this means broken data. Better to report it * not to have an uninitialized tail. */ CheckCompressedData(decompressed_index >= n_total_values); Assert(decompressed_index <= n_buffer_elements); return n_total_values; } /* * The same function as above, but does palloc instead of taking the buffer as * an input. We mark it as possibly unused because it is used not for every * element type we have. */ static ELEMENT_TYPE *FUNCTION_NAME(simple8brle_decompress_all, ELEMENT_TYPE)(Simple8bRleSerialized *compressed, uint32 *n_) pg_attribute_unused(); static ELEMENT_TYPE * FUNCTION_NAME(simple8brle_decompress_all, ELEMENT_TYPE)(Simple8bRleSerialized *compressed, uint32 *n_) { const uint32 n_total_values = compressed->num_elements; Assert(n_total_values <= GLOBAL_MAX_ROWS_PER_COMPRESSION); /* * We need a quite significant padding of 63 elements, not bytes, after the * last element, because we work in Simple8B blocks which can contain up to * 64 elements. */ const uint32 n_buffer_elements = n_total_values + 63; ELEMENT_TYPE *restrict decompressed_values = palloc(sizeof(ELEMENT_TYPE) * n_buffer_elements); *n_ = FUNCTION_NAME(simple8brle_decompress_all_buf, ELEMENT_TYPE)(compressed, decompressed_values, n_buffer_elements); return decompressed_values; } ================================================ FILE: tsl/src/compression/algorithms/uuid_compress.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include "uuid_compress.h" #include "adts/uint64_vec.h" #include "common/hashfn.h" #include "compression/arrow_c_data_interface.h" #include "compression/compression.h" #include "deltadelta.h" #include "dictionary.h" #include "guc.h" #include "lib/hyperloglog.h" #include "null.h" #include "simple8b_rle.h" #include "utils/palloc.h" #ifdef TS_USE_UMASH #include "import/umash.h" #endif typedef union UuidType { uint64 components[2]; pg_uuid_t uuid; } UuidType; typedef enum { UUID_COMPRESS_SUBTYPE_DELTADELTA = 0, UUID_COMPRESS_SUBTYPE_COUNT = 1, /* Must be last */ } UuidCompressSubtype; typedef struct UuidCompressed { CompressedDataHeaderFields; /* this uses 5 bytes */ uint8 subtype; uint16 num_nulls; uint32 timestamp_size; uint32 rand_b_and_variant_size; /* 8-byte alignment sentinel for the following fields */ uint64 alignment_sentinel[FLEXIBLE_ARRAY_MEMBER]; } UuidCompressed; static void pg_attribute_unused() assertions(void) { StaticAssertStmt(sizeof(UuidCompressed) == 16, "UuidCompressed wrong size"); StaticAssertStmt(offsetof(UuidCompressed, alignment_sentinel) % MAXIMUM_ALIGNOF == 0, "variable sized data must be 8-byte aligned"); } typedef struct UuidDecompressionIterator { DecompressionIterator base; int32 position; /* position within the total */ int32 total_elements; /* total number of entries plus nulls */ ArrowArray *uuid_buffer; /* init does a decompress_all to get this */ } UuidDecompressionIterator; /* * HyperLogLog parameters. * * The bit width is set such that the error rate is acceptable and also allocate * little memory. 8 uses 256 bytes and the error rate is 6.5% */ #define HLL_BIT_WIDTH 8 #define HLL_ERROR_RATE 0.065 #define HLL_MIN_CARDINALITY 20 /* * The UUID compressor is using delta delta compression for the first 8 * bytes of the UUID and stores the rest as a uint64_vec. At the same time * it keeps track of the cardinality of values. if the cardinality * indicates that we are better off with the dictionary compressor, we will * recompress it at the end. */ typedef struct UuidCompressor { /* Delta-delta encoding for timestamp, version and rand_a. */ DeltaDeltaCompressor *timestamp; /* We store the rand_b and variant parts together as a uint64_vec * to avoid having to store two separate bitmaps. */ uint64_vec rand_b_and_variant; /* HLL state to estimate the cardinality. This is used to check if * we are better off with recompressing the data as a dictionary. */ hyperLogLogState cardinality; /* Number of nulls in the data. */ uint16 num_nulls; /* Number of elements in the timestamp part. */ uint16 num_values; } UuidCompressor; typedef struct ExtendedCompressor { Compressor base; UuidCompressor *internal; } ExtendedCompressor; /* * Local helpers */ static void uuid_compressor_append_uuid(Compressor *compressor, Datum val); static void uuid_compressor_append_null_value(Compressor *compressor); static void *uuid_compressor_finish_and_reset(Compressor *compressor); static void decompression_iterator_init(UuidDecompressionIterator *iter, void *compressed, Oid element_type, bool forward); const Compressor uuid_compressor_initializer = { .append_val = uuid_compressor_append_uuid, .append_null = uuid_compressor_append_null_value, .is_full = NULL, .finish = uuid_compressor_finish_and_reset, }; /* * Compressor framework functions and definitions for the uuid_compress algorithm. */ extern UuidCompressor * uuid_compressor_alloc(void) { UuidCompressor *compressor = palloc0(sizeof(*compressor)); compressor->timestamp = delta_delta_compressor_alloc(); uint64_vec_init(&compressor->rand_b_and_variant, CurrentMemoryContext, TARGET_COMPRESSED_BATCH_SIZE); initHyperLogLog(&compressor->cardinality, HLL_BIT_WIDTH); return compressor; } extern void uuid_compressor_append_null(UuidCompressor *compressor) { delta_delta_compressor_append_null(compressor->timestamp); compressor->num_nulls++; } #ifdef TS_USE_UMASH static inline uint32 uuid_compress_hash(pg_uuid_t *uuid) { static struct umash_params params = { 0 }; if (params.poly[0][0] == 0) { umash_params_derive(¶ms, 0x12345abcdef67890ULL, NULL); Assert(params.poly[0][0] != 0); } uint64 h = umash_full(¶ms, /* seed = */ ~0ULL, /* which = */ 0, uuid->data, 16); return (uint32) (h ^ (h >> 32)); } #else static inline uint32 uuid_compress_hash(pg_uuid_t *uuid) { return hash_bytes((unsigned char *) uuid->data, sizeof(*uuid)); } #endif extern void uuid_compressor_append_value(UuidCompressor *compressor, pg_uuid_t next_val) { uint64_t components[2]; memcpy(components, next_val.data, sizeof(components)); /* The first component is the timestamp, version and rand_a. */ uint64_t timestamp = pg_ntoh64(components[0]); /* The second part is the rand_b and variant. */ uint64_t rand_b_and_variant = components[1]; delta_delta_compressor_append_value(compressor->timestamp, timestamp); uint64_vec_append(&compressor->rand_b_and_variant, rand_b_and_variant); uint32 h = uuid_compress_hash(&next_val); addHyperLogLog(&compressor->cardinality, h); compressor->num_values++; } static size_t uuid_compressor_estimate_dictionary_storage(UuidCompressor *compressor, size_t nulls_compressed_size) { double cardinality = (double) compressor->rand_b_and_variant.num_elements; double cardinality_and_error = cardinality; /* Don't use HLL if there are too few elements to estimate the cardinality. */ if (cardinality > HLL_MIN_CARDINALITY) { cardinality = estimateHyperLogLog(&compressor->cardinality); cardinality_and_error = cardinality * (1.0 - HLL_ERROR_RATE); } int array_index_bytes = ((int) cardinality_and_error * 5 + 63) / 64 * 8; double estimated_dictionary_storage = /* 16 bytes per values in dictionary/array/values */ cardinality_and_error * 16 + /* a single RLE block for the sizes in dictionary/array/sizes */ 16 + /* no nulls in dictionary/array/nulls */ 0 + /* 5 bits on average for the indexes in dictionary/array/indexes */ array_index_bytes + /* storing nulls is the same as in the delta-delta compressor */ nulls_compressed_size; return estimated_dictionary_storage; } extern void * uuid_compressor_finish(UuidCompressor *compressor) { if (compressor == NULL) return NULL; if (compressor->num_values == 0) return NULL; size_t nulls_compressed_size = 0; size_t timestamp_compressed_size = delta_delta_compressor_compressed_size(compressor->timestamp, &nulls_compressed_size); size_t estimated_dictionary_storage = uuid_compressor_estimate_dictionary_storage(compressor, nulls_compressed_size); size_t rand_b_and_variant_compressed_size = compressor->rand_b_and_variant.num_elements * sizeof(uint64_t); Assert(compressor->rand_b_and_variant.num_elements == compressor->num_values); size_t total_compressed_size = sizeof(UuidCompressed) + timestamp_compressed_size + rand_b_and_variant_compressed_size; /* TODO: this is temporary: to iterate over the delta-delta compressed data * we need to finalize the compression, so even if we knew that the dictionary * compression is better we still need to allocate, finish and memcpy the * entries. This is clearly a waste. To solve this we will need an interface * to iterate over compressed data without finalizing it. */ char *compressed_data = palloc(total_compressed_size); UuidCompressed *compressed = (UuidCompressed *) compressed_data; SET_VARSIZE(&compressed->vl_len_, total_compressed_size); compressed->compression_algorithm = COMPRESSION_ALGORITHM_UUID; compressed->num_nulls = compressor->num_nulls; Ensure(compressed->num_nulls == compressor->num_nulls, "unexpected number of nulls, it doesn't fit into the header"); compressed->subtype = UUID_COMPRESS_SUBTYPE_DELTADELTA; compressed->timestamp_size = timestamp_compressed_size; compressed->rand_b_and_variant_size = rand_b_and_variant_compressed_size; compressed_data += sizeof(*compressed); char *timestamp_compressed_data = compressed_data; compressed_data = delta_delta_compressor_finish_into(compressor->timestamp, timestamp_compressed_data); /* Make sure delta-delta took exactly the size it said it will */ Assert(compressed_data - timestamp_compressed_data == (long) timestamp_compressed_size); memcpy(compressed_data, compressor->rand_b_and_variant.data, rand_b_and_variant_compressed_size); if (total_compressed_size > estimated_dictionary_storage) { /* Recompress as dictionary */ DictionaryCompressor *dict_compressor = dictionary_compressor_alloc(UUIDOID); DecompressionIterator *iter = delta_delta_decompression_iterator_from_datum_forward(PointerGetDatum( timestamp_compressed_data), INT8OID); uint32 value_position = 0; for (DecompressResult r = delta_delta_decompression_iterator_try_next_forward(iter); !r.is_done; r = delta_delta_decompression_iterator_try_next_forward(iter)) { if (r.is_null) { dictionary_compressor_append_null(dict_compressor); } else { UuidType uuid_type; uuid_type.components[0] = pg_hton64(DatumGetInt64(r.val)); uuid_type.components[1] = compressor->rand_b_and_variant.data[value_position]; dictionary_compressor_append(dict_compressor, UUIDPGetDatum(&uuid_type.uuid)); ++value_position; } } void *dict_compressed = dictionary_compressor_finish(dict_compressor); if (VARSIZE(dict_compressed) < total_compressed_size) { /* We are better off with the dictionary compression, inline with the estimated size */ pfree(compressed); compressed = dict_compressed; } else { /* We are better off with the original compression, contrary to the estimated size. * This is OK, as the estimate is probabilistic. */ pfree(dict_compressed); } pfree(dict_compressor); pfree(iter); } return compressed; } extern bool uuid_compressed_has_nulls(const CompressedDataHeader *header) { const UuidCompressed *uc = (const UuidCompressed *) header; return uc->num_nulls > 0; } extern DecompressResult uuid_decompression_iterator_try_next_forward(DecompressionIterator *iter) { Assert(iter->compression_algorithm == COMPRESSION_ALGORITHM_UUID && iter->forward); Assert(iter->element_type == UUIDOID); UuidDecompressionIterator *uuid_iter = (UuidDecompressionIterator *) iter; Assert(uuid_iter->uuid_buffer != NULL); Assert(uuid_iter->uuid_buffer->buffers != NULL); Assert(uuid_iter->uuid_buffer->buffers[1] != NULL); const uint64 *validity_bitmap = uuid_iter->uuid_buffer->buffers[0]; UuidType *uuid_values = (UuidType *) uuid_iter->uuid_buffer->buffers[1]; if (uuid_iter->position >= uuid_iter->total_elements) return (DecompressResult){ .is_done = true, }; /* check nulls */ if (validity_bitmap != NULL) { if (!arrow_row_is_valid(validity_bitmap, uuid_iter->position)) { uuid_iter->position++; return (DecompressResult){ .is_null = true, }; } } UuidType *current_uuid = &uuid_values[uuid_iter->position]; uuid_iter->position++; return (DecompressResult){ .val = PointerGetDatum(current_uuid), }; } extern DecompressionIterator * uuid_decompression_iterator_from_datum_forward(Datum uuid_compressed, Oid element_type) { UuidDecompressionIterator *iterator = palloc0(sizeof(*iterator)); CheckCompressedData(DatumGetPointer(uuid_compressed) != NULL); decompression_iterator_init(iterator, (void *) PG_DETOAST_DATUM(uuid_compressed), element_type, true); return &iterator->base; } extern DecompressResult uuid_decompression_iterator_try_next_reverse(DecompressionIterator *iter) { Assert(iter->compression_algorithm == COMPRESSION_ALGORITHM_UUID && !iter->forward); Assert(iter->element_type == UUIDOID); UuidDecompressionIterator *uuid_iter = (UuidDecompressionIterator *) iter; Assert(uuid_iter->uuid_buffer != NULL); Assert(uuid_iter->uuid_buffer->buffers != NULL); Assert(uuid_iter->uuid_buffer->buffers[1] != NULL); const uint64 *validity_bitmap = uuid_iter->uuid_buffer->buffers[0]; UuidType *uuid_values = (UuidType *) uuid_iter->uuid_buffer->buffers[1]; if (uuid_iter->position < 0) return (DecompressResult){ .is_done = true, }; /* check nulls */ if (validity_bitmap != NULL) { if (!arrow_row_is_valid(validity_bitmap, uuid_iter->position)) { uuid_iter->position--; return (DecompressResult){ .is_null = true, }; } } Assert(uuid_iter->position >= 0); UuidType *current_uuid = &uuid_values[uuid_iter->position]; uuid_iter->position--; return (DecompressResult){ .val = PointerGetDatum(current_uuid), }; } extern DecompressionIterator * uuid_decompression_iterator_from_datum_reverse(Datum uuid_compressed, Oid element_type) { UuidDecompressionIterator *iterator = palloc(sizeof(*iterator)); CheckCompressedData(DatumGetPointer(uuid_compressed) != NULL); decompression_iterator_init(iterator, (void *) PG_DETOAST_DATUM(uuid_compressed), element_type, false); return &iterator->base; } extern void uuid_compressed_send(CompressedDataHeader *header, StringInfo buffer) { const UuidCompressed *data = (UuidCompressed *) header; Assert(header->compression_algorithm == COMPRESSION_ALGORITHM_UUID); pq_sendbyte(buffer, data->subtype); pq_sendint16(buffer, data->num_nulls); pq_sendint32(buffer, data->timestamp_size); pq_sendint32(buffer, data->rand_b_and_variant_size); char *ptr = (char *) data->alignment_sentinel; deltadelta_compressed_send((CompressedDataHeader *) ptr, buffer); ptr += data->timestamp_size; uint64 *rand_b_and_variant = (uint64 *) ptr; uint32 num_elements = data->rand_b_and_variant_size / sizeof(uint64); for (uint32 i = 0; i < num_elements; i++) pq_sendint64(buffer, rand_b_and_variant[i]); } extern Datum uuid_compressed_recv(StringInfo buffer) { size_t total_compressed_sized = 0; uint8 subtype = pq_getmsgbyte(buffer); CheckCompressedData(subtype < UUID_COMPRESS_SUBTYPE_COUNT); uint16 num_nulls = pq_getmsgint(buffer, 2); uint32 timestamp_size = pq_getmsgint32(buffer); uint32 rand_b_and_variant_size = pq_getmsgint32(buffer); /* The timestamp_size must accommodate the delta-delta header (24 bytes) */ CheckCompressedData(timestamp_size > 24); CheckCompressedData(rand_b_and_variant_size > 0); CheckCompressedData(rand_b_and_variant_size % sizeof(uint64) == 0); CheckCompressedData(timestamp_size % sizeof(uint64) == 0); CheckCompressedData(rand_b_and_variant_size / sizeof(uint64) < GLOBAL_MAX_ROWS_PER_COMPRESSION); /* A good enough to catch totally bogus timestamp_size values, the actual limit is slightly * tighter */ CheckCompressedData(timestamp_size < (GLOBAL_MAX_ROWS_PER_COMPRESSION * (sizeof(uint64) + 1))); uint32 total_values = num_nulls + rand_b_and_variant_size / sizeof(uint64); CheckCompressedData(total_values < GLOBAL_MAX_ROWS_PER_COMPRESSION); total_compressed_sized = sizeof(UuidCompressed) + timestamp_size + rand_b_and_variant_size; CheckCompressedData(total_compressed_sized <= MaxAllocSize); char *result = palloc(total_compressed_sized); UuidCompressed *compressed = (UuidCompressed *) result; compressed->subtype = subtype; compressed->num_nulls = num_nulls; compressed->timestamp_size = timestamp_size; compressed->rand_b_and_variant_size = rand_b_and_variant_size; SET_VARSIZE(&compressed->vl_len_, total_compressed_sized); compressed->compression_algorithm = COMPRESSION_ALGORITHM_UUID; Datum delta_delta_compressed = deltadelta_compressed_recv(buffer); size_t delta_delta_compressed_size = VARSIZE(delta_delta_compressed); CheckCompressedData(delta_delta_compressed_size == timestamp_size); memcpy(result + sizeof(UuidCompressed), DatumGetPointer(delta_delta_compressed), delta_delta_compressed_size); uint64 *rand_b_and_variant = (uint64 *) (result + sizeof(UuidCompressed) + delta_delta_compressed_size); uint32 num_elements = rand_b_and_variant_size / sizeof(uint64); for (uint32 i = 0; i < num_elements; i++) rand_b_and_variant[i] = pq_getmsgint64(buffer); PG_RETURN_POINTER(result); } extern Compressor * uuid_compressor_for_type(Oid element_type) { ExtendedCompressor *compressor = palloc(sizeof(*compressor)); switch (element_type) { case UUIDOID: *compressor = (ExtendedCompressor){ .base = uuid_compressor_initializer }; return &compressor->base; default: elog(ERROR, "invalid type for uuid compressor \"%s\"", format_type_be(element_type)); } pg_unreachable(); } /* * Cross-module functions for the uuid_compress algorithm. */ extern Datum tsl_uuid_compressor_append(PG_FUNCTION_ARGS) { MemoryContext old_context; MemoryContext agg_context; UuidCompressor *compressor = (UuidCompressor *) (PG_ARGISNULL(0) ? NULL : PG_GETARG_POINTER(0)); if (!AggCheckCallContext(fcinfo, &agg_context)) { /* cannot be called directly because of internal-type argument */ elog(ERROR, "tsl_uuid_compressor_append called in non-aggregate context"); } old_context = MemoryContextSwitchTo(agg_context); if (compressor == NULL) { compressor = uuid_compressor_alloc(); if (PG_NARGS() > 2) elog(ERROR, "append expects two arguments"); } if (PG_ARGISNULL(1)) uuid_compressor_append_null(compressor); else { pg_uuid_t *uuid = DatumGetUUIDP(PG_GETARG_DATUM(1)); Ensure(uuid != NULL, "invalid UUID"); uuid_compressor_append_value(compressor, *uuid); } MemoryContextSwitchTo(old_context); PG_RETURN_POINTER(compressor); } extern Datum tsl_uuid_compressor_finish(PG_FUNCTION_ARGS) { UuidCompressor *compressor = PG_ARGISNULL(0) ? NULL : (UuidCompressor *) PG_GETARG_POINTER(0); void *compressed; if (compressor == NULL) PG_RETURN_NULL(); compressed = uuid_compressor_finish(compressor); if (compressed == NULL) PG_RETURN_NULL(); PG_RETURN_POINTER(compressed); } extern ArrowArray * uuid_decompress_all(Datum compressed, Oid element_type, MemoryContext dest_mctx) { Assert(element_type == UUIDOID); MemoryContext old_context; ArrowArray *timestamp_array = NULL; CheckCompressedData(DatumGetPointer(compressed) != NULL); void *detoasted = PG_DETOAST_DATUM(compressed); StringInfoData si = { .data = detoasted, .len = VARSIZE_ANY(compressed) }; UuidCompressed *header = consumeCompressedData(&si, sizeof(UuidCompressed)); char *timestamp_compressed_data = NULL; char *rand_b_and_variant_compressed_data; CheckCompressedData(header->compression_algorithm == COMPRESSION_ALGORITHM_UUID); CheckCompressedData(header->subtype == UUID_COMPRESS_SUBTYPE_DELTADELTA); /* The timestamp_size must accommodate the delta-delta header (24 bytes) */ CheckCompressedData(header->timestamp_size > 24); CheckCompressedData(header->rand_b_and_variant_size > 0); CheckCompressedData(header->rand_b_and_variant_size % sizeof(uint64) == 0); CheckCompressedData(header->timestamp_size % sizeof(uint64) == 0); /* A good enough to catch totally bogus timestamp_size values, the actual limit is slightly * tighter */ CheckCompressedData(header->timestamp_size < (GLOBAL_MAX_ROWS_PER_COMPRESSION * (sizeof(uint64) + 1))); timestamp_compressed_data = consumeCompressedData(&si, header->timestamp_size); rand_b_and_variant_compressed_data = consumeCompressedData(&si, header->rand_b_and_variant_size); int32 num_values = (int32) (header->rand_b_and_variant_size / sizeof(uint64)); int32 total_elements = (int32) header->num_nulls + num_values; CheckCompressedData(num_values > 0); CheckCompressedData(num_values < GLOBAL_MAX_ROWS_PER_COMPRESSION); old_context = MemoryContextSwitchTo(dest_mctx); /* Make sure we use a 128bit aligned buffer */ char *unaligned = (char *) palloc((total_elements * sizeof(UuidType)) + 15); UuidType *uuid_buffer = (UuidType *) TYPEALIGN(16, unaligned); /* * Note : the validity bits from the delta delta compression is directly usable * in the result array, but the decompressed ingegers will be replaced * with the UUIDs */ timestamp_array = delta_delta_decompress_all(PointerGetDatum(timestamp_compressed_data), INT8OID, CurrentMemoryContext); CheckCompressedData(timestamp_array->length == total_elements); CheckCompressedData(timestamp_array->null_count == header->num_nulls); uint64 *rand_b_and_variant = (uint64 *) palloc(header->rand_b_and_variant_size); memcpy(rand_b_and_variant, rand_b_and_variant_compressed_data, header->rand_b_and_variant_size); uint64 *validity_bitmap = NULL; uint64 *timestamp_values = (uint64 *) timestamp_array->buffers[1]; if (header->num_nulls > 0) { Assert(timestamp_array->buffers[0] != NULL); validity_bitmap = (uint64 *) timestamp_array->buffers[0]; int value_position = 0; for (int i = 0; i < total_elements; i++) { if (arrow_row_is_valid(validity_bitmap, i)) { Assert(value_position < num_values); uuid_buffer[i].components[0] = pg_ntoh64(timestamp_values[i]); uuid_buffer[i].components[1] = rand_b_and_variant[value_position]; ++value_position; } } } else { for (int i = 0; i < total_elements; i++) { uuid_buffer[i].components[0] = pg_ntoh64(timestamp_values[i]); uuid_buffer[i].components[1] = rand_b_and_variant[i]; } } /* * At this point I combined the uncompressed data in the rand_b_and_variant array * and the freshly decompressed data from the delta delta bulk decompressison. * I now free the temp data. Note that `timestamp_values` is the same pointer * as timestamp_array->buffers[1] , the second buffer in the ArrowArray. * This will be replaced with the uuid array. * */ pfree(rand_b_and_variant); pfree(timestamp_values); MemoryContextSwitchTo(old_context); /* * This is the only fixup needed for the ArrowArray. Everything else * is valid data as generated by delta_delta_decompress_all() */ timestamp_array->buffers[1] = (uint8 *) uuid_buffer; return timestamp_array; } /* * Local helpers */ static void uuid_compressor_append_uuid(Compressor *compressor, Datum val) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = uuid_compressor_alloc(); pg_uuid_t *uuid = DatumGetUUIDP(val); Ensure(uuid != NULL, "invalid UUID"); uuid_compressor_append_value(extended->internal, *uuid); } static void uuid_compressor_append_null_value(Compressor *compressor) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; if (extended->internal == NULL) extended->internal = uuid_compressor_alloc(); uuid_compressor_append_null(extended->internal); } static void * uuid_compressor_finish_and_reset(Compressor *compressor) { ExtendedCompressor *extended = (ExtendedCompressor *) compressor; void *compressed = NULL; if (extended != NULL && extended->internal != NULL) { compressed = uuid_compressor_finish(extended->internal); pfree(extended->internal); extended->internal = NULL; } return compressed; } static void decompression_iterator_init(UuidDecompressionIterator *iter, void *compressed, Oid element_type, bool forward) { Assert(element_type == UUIDOID); ArrowArray *arrow_array = uuid_decompress_all(PointerGetDatum(compressed), element_type, CurrentMemoryContext); int32 total_elements = arrow_array->length; *iter = (UuidDecompressionIterator){ .base = { .compression_algorithm = COMPRESSION_ALGORITHM_UUID, .forward = forward, .element_type = element_type, .try_next = (forward ? uuid_decompression_iterator_try_next_forward : uuid_decompression_iterator_try_next_reverse) }, .position = (forward ? 0 : total_elements - 1), .total_elements = total_elements, .uuid_buffer = arrow_array, }; } static void pg_attribute_unused() silence_unused_warning(void) { simple8brle_serialized_recv(NULL); } ================================================ FILE: tsl/src/compression/algorithms/uuid_compress.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once /* * uuid_compress is used to encode UUID values where there are 4 distinct parts * that are encoded separately. The UUID being encoded is optimised for the v7 * UUID format. The four parts are: * * - timestamp : (delta-delta encoding) * - the version number and variant number concatenated : (simple8b_rle) * - rand_a : is encoded as an array of bits * - rand_b : is encoded as an array of bits */ #include <postgres.h> #include "compression/compression.h" #include <fmgr.h> #include <lib/stringinfo.h> #include <utils/uuid.h> typedef struct UuidCompressor UuidCompressor; typedef struct UuidCompressed UuidCompressed; typedef struct UuidDecompressionIterator UuidDecompressionIterator; /* * Compressor framework functions and definitions for the uuid_compress algorithm. */ extern UuidCompressor *uuid_compressor_alloc(void); extern void uuid_compressor_append_null(UuidCompressor *compressor); extern void uuid_compressor_append_value(UuidCompressor *compressor, pg_uuid_t next_val); extern void *uuid_compressor_finish(UuidCompressor *compressor); extern bool uuid_compressed_has_nulls(const CompressedDataHeader *header); extern DecompressResult uuid_decompression_iterator_try_next_forward(DecompressionIterator *iter); extern DecompressionIterator *uuid_decompression_iterator_from_datum_forward(Datum uuid_compressed, Oid element_type); extern DecompressResult uuid_decompression_iterator_try_next_reverse(DecompressionIterator *iter); extern DecompressionIterator *uuid_decompression_iterator_from_datum_reverse(Datum uuid_compressed, Oid element_type); extern void uuid_compressed_send(CompressedDataHeader *header, StringInfo buffer); extern ArrowArray *uuid_decompress_all(Datum compressed, Oid element_type, MemoryContext dest_mctx); extern Datum uuid_compressed_recv(StringInfo buf); extern Compressor *uuid_compressor_for_type(Oid element_type); #define UUID_COMPRESS_ALGORITHM_DEFINITION \ { \ .iterator_init_forward = uuid_decompression_iterator_from_datum_forward, \ .iterator_init_reverse = uuid_decompression_iterator_from_datum_reverse, \ .decompress_all = uuid_decompress_all, .compressed_data_send = uuid_compressed_send, \ .compressed_data_recv = uuid_compressed_recv, \ .compressor_for_type = uuid_compressor_for_type, \ .compressed_data_storage = TOAST_STORAGE_EXTERNAL, \ } /* * Cross-module functions for the uuid_compress algorithm. */ extern Datum tsl_uuid_compressor_append(PG_FUNCTION_ARGS); extern Datum tsl_uuid_compressor_finish(PG_FUNCTION_ARGS); ================================================ FILE: tsl/src/compression/api.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* This file contains the implementation for SQL utility functions that * compress and decompress chunks */ #include <postgres.h> #include <access/tableam.h> #include <access/xact.h> #include <catalog/dependency.h> #include <catalog/index.h> #include <catalog/indexing.h> #include <catalog/pg_am.h> #include <commands/event_trigger.h> #include <commands/tablecmds.h> #include <commands/trigger.h> #include <libpq-fe.h> #include <miscadmin.h> #include <nodes/makefuncs.h> #include <nodes/parsenodes.h> #include <nodes/pg_list.h> #include <parser/parse_func.h> #include <postgres_ext.h> #include <storage/lmgr.h> #include <storage/lockdefs.h> #include <tcop/utility.h> #include <trigger.h> #include <utils/builtins.h> #include <utils/elog.h> #include <utils/fmgrprotos.h> #include <utils/inval.h> #include "compat/compat.h" #include "annotations.h" #include "api.h" #include "cache.h" #include "chunk.h" #include "compression.h" #include "compression_storage.h" #include "create.h" #include "debug_point.h" #include "error_utils.h" #include "errors.h" #include "hypercube.h" #include "hypertable.h" #include "hypertable_cache.h" #include "nodes/columnar_scan/columnar_scan.h" #include "recompress.h" #include "scan_iterator.h" #include "scanner.h" #include "ts_catalog/array_utils.h" #include "ts_catalog/catalog.h" #include "ts_catalog/chunk_column_stats.h" #include "ts_catalog/compression_chunk_size.h" #include "ts_catalog/compression_settings.h" #include "ts_catalog/continuous_agg.h" #include "utils.h" #include "wal_utils.h" typedef struct CompressChunkCxt { Hypertable *srcht; Chunk *srcht_chunk; /* chunk from srcht */ Hypertable *compress_ht; /*compressed table for srcht */ } CompressChunkCxt; static Oid get_compressed_chunk_index_for_recompression(Chunk *uncompressed_chunk); static Node * create_dummy_query() { RawStmt *query = NULL; query = makeNode(RawStmt); query->stmt = (Node *) makeNode(SelectStmt); return (Node *) query; } void compression_chunk_size_catalog_insert(int32 src_chunk_id, const RelationSize *src_size, int32 compress_chunk_id, const RelationSize *compress_size, int64 rowcnt_pre_compression, int64 rowcnt_post_compression, int64 rowcnt_frozen) { Catalog *catalog = ts_catalog_get(); Relation rel; TupleDesc desc; CatalogSecurityContext sec_ctx; Datum values[Natts_compression_chunk_size]; bool nulls[Natts_compression_chunk_size] = { false }; rel = table_open(catalog_get_table_id(catalog, COMPRESSION_CHUNK_SIZE), RowExclusiveLock); desc = RelationGetDescr(rel); memset(values, 0, sizeof(values)); values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_chunk_id)] = Int32GetDatum(src_chunk_id); values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_chunk_id)] = Int32GetDatum(compress_chunk_id); values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_uncompressed_heap_size)] = Int64GetDatum(src_size->heap_size); values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_uncompressed_toast_size)] = Int64GetDatum(src_size->toast_size); values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_uncompressed_index_size)] = Int64GetDatum(src_size->index_size); values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_heap_size)] = Int64GetDatum(compress_size->heap_size); values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_toast_size)] = Int64GetDatum(compress_size->toast_size); values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_index_size)] = Int64GetDatum(compress_size->index_size); values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_numrows_pre_compression)] = Int64GetDatum(rowcnt_pre_compression); values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_numrows_post_compression)] = Int64GetDatum(rowcnt_post_compression); values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_numrows_frozen_immediately)] = Int64GetDatum(rowcnt_frozen); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_insert_values(rel, desc, values, nulls); ts_catalog_restore_user(&sec_ctx); table_close(rel, RowExclusiveLock); } static int compression_chunk_size_catalog_update_merged(int32 chunk_id, const RelationSize *size, int32 merge_chunk_id, const RelationSize *merge_size, int64 merge_rowcnt_pre_compression, int64 merge_rowcnt_post_compression) { ScanIterator iterator = ts_scan_iterator_create(COMPRESSION_CHUNK_SIZE, RowExclusiveLock, CurrentMemoryContext); bool updated = false; iterator.ctx.index = catalog_get_index(ts_catalog_get(), COMPRESSION_CHUNK_SIZE, COMPRESSION_CHUNK_SIZE_PKEY); ts_scan_iterator_scan_key_init(&iterator, Anum_compression_chunk_size_pkey_chunk_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(chunk_id)); ts_scanner_foreach(&iterator) { Datum values[Natts_compression_chunk_size]; bool replIsnull[Natts_compression_chunk_size] = { false }; bool repl[Natts_compression_chunk_size] = { false }; bool should_free; TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); HeapTuple new_tuple; heap_deform_tuple(tuple, ts_scanner_get_tupledesc(ti), values, replIsnull); /* Increment existing sizes with sizes from uncompressed chunk. */ values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_uncompressed_heap_size)] = Int64GetDatum(size->heap_size + DatumGetInt64(values[AttrNumberGetAttrOffset( Anum_compression_chunk_size_uncompressed_heap_size)])); repl[AttrNumberGetAttrOffset(Anum_compression_chunk_size_uncompressed_heap_size)] = true; values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_uncompressed_toast_size)] = Int64GetDatum(size->toast_size + DatumGetInt64(values[AttrNumberGetAttrOffset( Anum_compression_chunk_size_uncompressed_toast_size)])); repl[AttrNumberGetAttrOffset(Anum_compression_chunk_size_uncompressed_toast_size)] = true; values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_uncompressed_index_size)] = Int64GetDatum(size->index_size + DatumGetInt64(values[AttrNumberGetAttrOffset( Anum_compression_chunk_size_uncompressed_index_size)])); repl[AttrNumberGetAttrOffset(Anum_compression_chunk_size_uncompressed_index_size)] = true; values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_heap_size)] = Int64GetDatum(merge_size->heap_size); repl[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_heap_size)] = true; values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_toast_size)] = Int64GetDatum(merge_size->toast_size); repl[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_toast_size)] = true; values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_index_size)] = Int64GetDatum(merge_size->index_size); repl[AttrNumberGetAttrOffset(Anum_compression_chunk_size_compressed_index_size)] = true; values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_numrows_pre_compression)] = Int64GetDatum(merge_rowcnt_pre_compression + DatumGetInt64(values[AttrNumberGetAttrOffset( Anum_compression_chunk_size_numrows_pre_compression)])); repl[AttrNumberGetAttrOffset(Anum_compression_chunk_size_numrows_pre_compression)] = true; values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_numrows_post_compression)] = Int64GetDatum(merge_rowcnt_post_compression + DatumGetInt64(values[AttrNumberGetAttrOffset( Anum_compression_chunk_size_numrows_post_compression)])); repl[AttrNumberGetAttrOffset(Anum_compression_chunk_size_numrows_post_compression)] = true; new_tuple = heap_modify_tuple(tuple, ts_scanner_get_tupledesc(ti), values, replIsnull, repl); ts_catalog_update(ti->scanrel, new_tuple); heap_freetuple(new_tuple); if (should_free) heap_freetuple(tuple); updated = true; break; } ts_scan_iterator_end(&iterator); ts_scan_iterator_close(&iterator); return updated; } static void get_hypertable_or_cagg_name(Hypertable *ht, Name objname) { ContinuousAggHypertableStatus status = ts_continuous_agg_hypertable_status(ht->fd.id); if (status == HypertableIsNotContinuousAgg || status == HypertableIsRawTable) namestrcpy(objname, NameStr(ht->fd.table_name)); else if (status == HypertableIsMaterialization) { ContinuousAgg *cagg = ts_continuous_agg_find_by_mat_hypertable_id(ht->fd.id, false); namestrcpy(objname, NameStr(cagg->data.user_view_name)); } else { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("unexpected hypertable status for %s %d", NameStr(ht->fd.table_name), status))); } } static void compresschunkcxt_init(CompressChunkCxt *cxt, Cache *hcache, Oid hypertable_relid, Oid chunk_relid) { Hypertable *srcht = ts_hypertable_cache_get_entry(hcache, hypertable_relid, CACHE_FLAG_NONE); Hypertable *compress_ht; Chunk *srcchunk; ts_hypertable_permissions_check(srcht->main_table_relid, GetUserId()); if (!TS_HYPERTABLE_HAS_COMPRESSION_TABLE(srcht)) { NameData cagg_ht_name; get_hypertable_or_cagg_name(srcht, &cagg_ht_name); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("columnstore not enabled on \"%s\"", NameStr(cagg_ht_name)), errdetail("It is not possible to convert chunks to columnstore on a hypertable or" " continuous aggregate that does not have columnstore enabled."), errhint("Enable columnstore using ALTER TABLE/MATERIALIZED VIEW with" " the timescaledb.enable_columnstore option."))); } compress_ht = ts_hypertable_get_by_id(srcht->fd.compressed_hypertable_id); if (compress_ht == NULL) ereport(ERROR, (errcode(ERRCODE_TS_HYPERTABLE_NOT_EXIST), errmsg("missing columnstore-enabled hypertable"))); /* user has to be the owner of the compression table too */ ts_hypertable_permissions_check(compress_ht->main_table_relid, GetUserId()); if (!srcht->space) /* something is wrong */ ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("missing hyperspace for hypertable"))); /* refetch the srcchunk with all attributes filled in */ srcchunk = ts_chunk_get_by_relid(chunk_relid, true); ts_chunk_validate_chunk_status_for_operation(srcchunk, CHUNK_COMPRESS, true); cxt->srcht = srcht; cxt->compress_ht = compress_ht; cxt->srcht_chunk = srcchunk; } static Chunk * find_chunk_to_merge_into(Hypertable *ht, Chunk *current_chunk) { int64 max_chunk_interval, current_chunk_interval = 0, compressed_chunk_interval = 0; Chunk *previous_chunk; Point *p; const Dimension *time_dim = hyperspace_get_open_dimension(ht->space, 0); if (!time_dim || time_dim->fd.compress_interval_length == 0) return NULL; Assert(current_chunk->cube->num_slices > 0); Assert(current_chunk->cube->slices[0]->fd.dimension_id == time_dim->fd.id); max_chunk_interval = time_dim->fd.compress_interval_length; p = ts_point_create(current_chunk->cube->num_slices); /* First coordinate is the time coordinates and we want it to fall into previous chunk * hence we reduce it by 1 */ p->coordinates[p->num_coords++] = current_chunk->cube->slices[0]->fd.range_start - 1; current_chunk_interval = current_chunk->cube->slices[0]->fd.range_end - current_chunk->cube->slices[0]->fd.range_start; for (int i = p->num_coords; i < current_chunk->cube->num_slices; i++) { p->coordinates[p->num_coords++] = current_chunk->cube->slices[i]->fd.range_start; } previous_chunk = ts_hypertable_find_chunk_for_point(ht, p, ExclusiveLock); /* If there is no previous adjacent chunk along the time dimension or * if it hasn't been compressed yet, we can't merge. */ if (!previous_chunk || !OidIsValid(previous_chunk->fd.compressed_chunk_id)) return NULL; Assert(previous_chunk->cube->num_slices > 0); Assert(previous_chunk->cube->slices[0]->fd.dimension_id == time_dim->fd.id); compressed_chunk_interval = previous_chunk->cube->slices[0]->fd.range_end - previous_chunk->cube->slices[0]->fd.range_start; /* If the slices do not match (except on time dimension), we cannot merge the chunks. */ if (previous_chunk->cube->num_slices != current_chunk->cube->num_slices) return NULL; for (int i = 1; i < previous_chunk->cube->num_slices; i++) { if (previous_chunk->cube->slices[i]->fd.id != current_chunk->cube->slices[i]->fd.id) { return NULL; } } /* If the compressed chunk is full, we can't merge any more. */ if (compressed_chunk_interval == 0 || compressed_chunk_interval + current_chunk_interval > max_chunk_interval) return NULL; /* Get reloid of the previous compressed chunk via settings */ CompressionSettings *prev_comp_settings = ts_compression_settings_get(previous_chunk->table_id); CompressionSettings *ht_comp_settings = ts_compression_settings_get(ht->main_table_relid); if (!ts_compression_settings_equal_with_defaults(ht_comp_settings, prev_comp_settings)) return NULL; /* We don't support merging chunks with sequence numbers */ if (get_attnum(prev_comp_settings->fd.compress_relid, COMPRESSION_COLUMN_METADATA_SEQUENCE_NUM_NAME) != InvalidAttrNumber) return NULL; return previous_chunk; } /* Check if compression order is violated by merging in a new chunk * Because data merged in uses higher sequence numbers than any data already in the chunk, * the only way the order is guaranteed can be if we know the data we are merging in would come * after the existing data according to the compression order. This is true if the data being merged * in has timestamps greater than the existing data and the first column in the order by is time * ASC. * * The CompressChunkCxt references the chunk we are merging and mergable_chunk is the chunk we * are merging into. */ static bool check_is_chunk_order_violated_by_merge(CompressChunkCxt *cxt, const Dimension *time_dim, Chunk *mergable_chunk) { const DimensionSlice *mergable_slice = ts_hypercube_get_slice_by_dimension_id(mergable_chunk->cube, time_dim->fd.id); if (!mergable_slice) elog(ERROR, "mergeable chunk has no time dimension slice"); const DimensionSlice *compressed_slice = ts_hypercube_get_slice_by_dimension_id(cxt->srcht_chunk->cube, time_dim->fd.id); if (!compressed_slice) elog(ERROR, "columnstore chunk has no time dimension slice"); /* * Ensure the compressed chunk is AFTER the chunk that * it is being merged into. This is already guaranteed by previous checks. */ Ensure(mergable_slice->fd.range_end == compressed_slice->fd.range_start, "chunk being merged is not after the chunk that is being merged into"); CompressionSettings *ht_settings = ts_compression_settings_get(mergable_chunk->hypertable_relid); char *attname = get_attname(cxt->srcht->main_table_relid, time_dim->column_attno, false); int index = ts_array_position(ht_settings->fd.orderby, attname); /* Primary dimension column should be first compress_orderby column. */ if (index != 1) return true; return false; } static Oid compress_chunk_impl(Oid hypertable_relid, Oid chunk_relid) { Oid result_chunk_id = chunk_relid; CompressChunkCxt cxt = { 0 }; Chunk *compress_ht_chunk, *mergable_chunk; Cache *hcache; RelationSize before_size, after_size; CompressionStats cstat; bool new_compressed_chunk = false; hcache = ts_hypertable_cache_pin(); compresschunkcxt_init(&cxt, hcache, hypertable_relid, chunk_relid); /* acquire locks on src and compress hypertable and src chunk */ ereport(DEBUG1, (errmsg("acquiring locks for converting to columnstore \"%s.%s\"", get_namespace_name(get_rel_namespace(chunk_relid)), get_rel_name(chunk_relid)))); LockRelationOid(cxt.srcht->main_table_relid, AccessShareLock); LockRelationOid(cxt.compress_ht->main_table_relid, AccessShareLock); LockRelationOid(cxt.srcht_chunk->table_id, ExclusiveLock); /* acquire locks on catalog tables to keep till end of txn */ LockRelationOid(catalog_get_table_id(ts_catalog_get(), CHUNK), RowExclusiveLock); ereport(DEBUG1, (errmsg("locks acquired for converting to columnstore \"%s.%s\"", get_namespace_name(get_rel_namespace(chunk_relid)), get_rel_name(chunk_relid)))); DEBUG_WAITPOINT("compress_chunk_impl_start"); /* * Re-read the state of the chunk after all locks have been acquired and ensure * it is still uncompressed. Another process running in parallel might have * already performed the compression while we were waiting for the locks to be * acquired. */ Chunk *chunk_state_after_lock = ts_chunk_get_by_relid(chunk_relid, true); /* Throw error if chunk has invalid status for operation */ ts_chunk_validate_chunk_status_for_operation(chunk_state_after_lock, CHUNK_COMPRESS, true); /* get compression properties for hypertable */ mergable_chunk = find_chunk_to_merge_into(cxt.srcht, cxt.srcht_chunk); if (!mergable_chunk) { /* * Set up a dummy parsetree since we're calling AlterTableInternal * inside create_compress_chunk(). We can use anything here because we * are not calling EventTriggerDDLCommandEnd but we use a parse tree * type that CreateCommandTag can handle to avoid spurious printouts * in the event that EventTriggerDDLCommandEnd is called. */ EventTriggerAlterTableStart(create_dummy_query()); /* create compressed chunk and a new table */ compress_ht_chunk = create_compress_chunk(cxt.compress_ht, cxt.srcht_chunk, InvalidOid); /* Associate compressed chunk with main chunk. */ ts_chunk_set_compressed_chunk(cxt.srcht_chunk, compress_ht_chunk->fd.id); new_compressed_chunk = true; ereport(DEBUG1, (errmsg("new columnstore chunk \"%s.%s\" created", NameStr(compress_ht_chunk->fd.schema_name), NameStr(compress_ht_chunk->fd.table_name)))); EventTriggerAlterTableEnd(); } else { /* use an existing compressed chunk to compress into */ compress_ht_chunk = ts_chunk_get_by_id(mergable_chunk->fd.compressed_chunk_id, true); result_chunk_id = mergable_chunk->table_id; ereport(DEBUG1, (errmsg("merge into existing columnstore chunk \"%s.%s\"", NameStr(compress_ht_chunk->fd.schema_name), NameStr(compress_ht_chunk->fd.table_name)))); } /* Since the compressed relation is created in the same transaction as the tuples that will be * written by the compressor, we can insert the tuple directly in frozen state. This is the same * logic as performed in COPY INSERT FROZEN. * * Note: Tuples inserted with HEAP_INSERT_FROZEN become immediately visible to all transactions * (they violate the MVCC pattern). So, this flag can only be used when creating the compressed * chunk in the same transaction as the compressed tuples are inserted. * * If this isn't the case, then tuples can be seen multiple times by parallel readers - once in * the uncompressed part of the hypertable (since they are not deleted in the transaction) and * once in the compressed part of the hypertable since the MVCC semantic is violated due to the * flag. * * In contrast, when the compressed chunk part is created in the same transaction as the tuples * are written, the compressed chunk (i.e., the catalog entry) becomes visible to other * transactions only after the transaction that performs the compression is committed and * the uncompressed chunk is truncated. */ int insert_options = new_compressed_chunk ? HEAP_INSERT_FROZEN : 0; before_size = ts_relation_size_impl(cxt.srcht_chunk->table_id); cstat = compress_chunk(cxt.srcht_chunk->table_id, compress_ht_chunk->table_id, insert_options); after_size = ts_relation_size_impl(compress_ht_chunk->table_id); if (cxt.srcht->range_space) ts_chunk_column_stats_calculate(cxt.srcht, cxt.srcht_chunk); if (new_compressed_chunk) { compression_chunk_size_catalog_insert(cxt.srcht_chunk->fd.id, &before_size, compress_ht_chunk->fd.id, &after_size, cstat.rowcnt_pre_compression, cstat.rowcnt_post_compression, cstat.rowcnt_frozen); /* Detect and emit warning if poor compression ratio is found */ float compression_ratio = ((float) before_size.total_size / after_size.total_size); float POOR_COMPRESSION_THRESHOLD = 1.0; if (ts_guc_enable_compression_ratio_warnings && compression_ratio < POOR_COMPRESSION_THRESHOLD) ereport(WARNING, errcode(ERRCODE_WARNING), errmsg("poor compression ratio detected for chunk \"%s\"'", get_rel_name(chunk_relid)), errdetail("Chunk \"%s\" has a poor compression ratio: %.2f. Size before " "compression: " INT64_FORMAT " bytes. Size after compression: " INT64_FORMAT " bytes", get_rel_name(chunk_relid), compression_ratio, before_size.total_size, after_size.total_size), errhint("Changing compression settings for \"%s\" can improve compression rate", get_rel_name(hypertable_relid))); } else { compression_chunk_size_catalog_update_merged(mergable_chunk->fd.id, &before_size, compress_ht_chunk->fd.id, &after_size, cstat.rowcnt_pre_compression, cstat.rowcnt_post_compression); const Dimension *time_dim = hyperspace_get_open_dimension(cxt.srcht->space, 0); Assert(time_dim != NULL); bool chunk_unordered = check_is_chunk_order_violated_by_merge(&cxt, time_dim, mergable_chunk); ts_chunk_merge_on_dimension(cxt.srcht, mergable_chunk, cxt.srcht_chunk, time_dim->fd.id); if (chunk_unordered) { ts_chunk_set_unordered(mergable_chunk); tsl_compress_chunk_wrapper(mergable_chunk, true, false); } } ts_cache_release(&hcache); return result_chunk_id; } static void decompress_chunk_impl(Chunk *uncompressed_chunk, bool if_compressed) { Cache *hcache; Hypertable *uncompressed_hypertable = ts_hypertable_cache_get_cache_and_entry(uncompressed_chunk->hypertable_relid, CACHE_FLAG_NONE, &hcache); Hypertable *compressed_hypertable; Chunk *compressed_chunk; ts_hypertable_permissions_check(uncompressed_hypertable->main_table_relid, GetUserId()); if (TS_HYPERTABLE_IS_INTERNAL_COMPRESSION_TABLE(uncompressed_hypertable)) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg( "convert_to_rowstore must not be called on the internal columnstore chunk"))); compressed_hypertable = ts_hypertable_get_by_id(uncompressed_hypertable->fd.compressed_hypertable_id); if (compressed_hypertable == NULL) ereport(ERROR, (errcode(ERRCODE_TS_HYPERTABLE_NOT_EXIST), errmsg("missing columnstore-enabled hypertable"))); if (uncompressed_chunk->fd.hypertable_id != uncompressed_hypertable->fd.id) elog(ERROR, "hypertable and chunk do not match"); if (uncompressed_chunk->fd.compressed_chunk_id == INVALID_CHUNK_ID) { ts_cache_release(&hcache); ereport((if_compressed ? NOTICE : ERROR), (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("chunk \"%s\" is not converted to columnstore", get_rel_name(uncompressed_chunk->table_id)))); return; } write_logical_replication_msg_decompression_start(); ts_chunk_validate_chunk_status_for_operation(uncompressed_chunk, CHUNK_DECOMPRESS, true); compressed_chunk = ts_chunk_get_by_id(uncompressed_chunk->fd.compressed_chunk_id, true); ereport(DEBUG1, (errmsg("acquiring locks for converting to rowstore \"%s.%s\"", NameStr(uncompressed_chunk->fd.schema_name), NameStr(uncompressed_chunk->fd.table_name)))); /* acquire locks on src and compress hypertable and src chunk */ LockRelationOid(uncompressed_hypertable->main_table_relid, AccessShareLock); LockRelationOid(compressed_hypertable->main_table_relid, AccessShareLock); /* * Acquire an ExclusiveLock on the uncompressed and the compressed * chunk (the chunks can still be accessed by reads). * * The lock on the compressed chunk is needed because it gets deleted * after decompression. The lock on the uncompressed chunk is needed * to avoid deadlocks (e.g., caused by later lock upgrades or parallel * started chunk compressions). * * Note: Also the function decompress_chunk() will request an * ExclusiveLock on the compressed and on the uncompressed * chunk. See the comments in function about the concurrency of * operations. */ LockRelationOid(uncompressed_chunk->table_id, ExclusiveLock); LockRelationOid(compressed_chunk->table_id, ExclusiveLock); /* acquire locks on catalog tables to keep till end of txn */ LockRelationOid(catalog_get_table_id(ts_catalog_get(), CHUNK), RowExclusiveLock); ereport(DEBUG1, (errmsg("locks acquired for converting to rowstore \"%s.%s\"", NameStr(uncompressed_chunk->fd.schema_name), NameStr(uncompressed_chunk->fd.table_name)))); DEBUG_WAITPOINT("decompress_chunk_impl_start"); /* * Re-read the state of the chunk after all locks have been acquired and ensure * it is still compressed. Another process running in parallel might have * already performed the decompression while we were waiting for the locks to be * acquired. */ Chunk *chunk_state_after_lock = ts_chunk_get_by_id(uncompressed_chunk->fd.id, true); /* Throw error if chunk has invalid status for operation */ ts_chunk_validate_chunk_status_for_operation(chunk_state_after_lock, CHUNK_DECOMPRESS, true); decompress_chunk(compressed_chunk->table_id, uncompressed_chunk->table_id); /* Delete the compressed chunk */ ts_compression_chunk_size_delete(uncompressed_chunk->fd.id); ts_chunk_clear_compressed_chunk(uncompressed_chunk); ts_compression_settings_delete(uncompressed_chunk->table_id); /* * Lock the compressed chunk that is going to be deleted. At this point, * the reference to the compressed chunk is already removed from the * catalog but we need to block readers from accessing this chunk * until the catalog changes are visible to them. * * Note: Calling performMultipleDeletions in chunk_index_tuple_delete * also requests an AccessExclusiveLock on the compressed_chunk. However, * this call makes the lock on the chunk explicit. */ LockRelationOid(uncompressed_chunk->table_id, AccessExclusiveLock); LockRelationOid(compressed_chunk->table_id, AccessExclusiveLock); ts_chunk_drop(compressed_chunk, DROP_RESTRICT, -1); ts_cache_release(&hcache); write_logical_replication_msg_decompression_end(); } static bool recompress_chunk_impl(Chunk *chunk, Oid *uncompressed_chunk_id, bool recompress) { CompressionSettings *chunk_settings = ts_compression_settings_get(chunk->table_id); bool recompressed = false; if (!chunk_settings || !chunk_settings->fd.orderby) { elog(NOTICE, "in-memory recompression is disabled due to no order by, " "performing decompress/compress on chunk \"%s.%s\"", NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name)); return false; } else if (!get_compressed_chunk_index_for_recompression(chunk)) { elog(NOTICE, "in-memory recompression is disabled due to no compressed chunk index, " "performing decompress/compress on chunk \"%s.%s\"", NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name)); return false; } /* * Choose recompression strategy based on chunk state and settings: * 1. Segmentwise recompression: For partial chunks * 2. In-memory recompression: For other compressed chunks with matching settings * 3. Fallback to decompress/compress: When neither strategy is applicable */ if (ts_chunk_is_partial(chunk)) { if (!ts_guc_enable_segmentwise_recompression) { ereport(NOTICE, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("segmentwise in-memory recompression functionality disabled, " "set timescaledb.enable_segmentwise_recompression to on"))); return false; } *uncompressed_chunk_id = recompress_chunk_segmentwise_impl(chunk); recompressed = true; } else { if (!ts_guc_enable_in_memory_recompression) { ereport(DEBUG1, (errcode(ERRCODE_WARNING), errmsg("in-memory recompression functionality disabled, " "set timescaledb.enable_in_memory_recompression to on"))); return false; } recompressed = recompress_chunk_in_memory_impl(chunk); } return recompressed; } /* * Create a new compressed chunk using existing table with compressed data. * * chunk_relid - non-compressed chunk relid * chunk_table - table containing compressed data */ Datum tsl_create_compressed_chunk(PG_FUNCTION_ARGS) { Oid chunk_relid = PG_GETARG_OID(0); Oid chunk_table = PG_GETARG_OID(1); RelationSize uncompressed_size = { .heap_size = PG_GETARG_INT64(2), .toast_size = PG_GETARG_INT64(3), .index_size = PG_GETARG_INT64(4) }; RelationSize compressed_size = { .heap_size = PG_GETARG_INT64(5), .toast_size = PG_GETARG_INT64(6), .index_size = PG_GETARG_INT64(7) }; int64 numrows_pre_compression = PG_GETARG_INT64(8); int64 numrows_post_compression = PG_GETARG_INT64(9); Chunk *chunk; Chunk *compress_ht_chunk; Cache *hcache; CompressChunkCxt cxt; bool chunk_was_compressed; Assert(!PG_ARGISNULL(0)); Assert(!PG_ARGISNULL(1)); ts_feature_flag_check(FEATURE_HYPERTABLE_COMPRESSION); TS_PREVENT_FUNC_IF_READ_ONLY(); chunk = ts_chunk_get_by_relid(chunk_relid, true); hcache = ts_hypertable_cache_pin(); compresschunkcxt_init(&cxt, hcache, chunk->hypertable_relid, chunk_relid); /* Acquire locks on src and compress hypertable and src chunk */ LockRelationOid(cxt.srcht->main_table_relid, AccessShareLock); LockRelationOid(cxt.compress_ht->main_table_relid, AccessShareLock); LockRelationOid(cxt.srcht_chunk->table_id, ShareLock); /* Acquire locks on catalog tables to keep till end of txn */ LockRelationOid(catalog_get_table_id(ts_catalog_get(), CHUNK), RowExclusiveLock); /* * Set up a dummy parsetree since we're calling AlterTableInternal inside * create_compress_chunk(). We can use anything here because we are not * calling EventTriggerDDLCommandEnd but we use a parse tree type that * CreateCommandTag can handle to avoid spurious printouts. */ EventTriggerAlterTableStart(create_dummy_query()); /* Create compressed chunk using existing table */ compress_ht_chunk = create_compress_chunk(cxt.compress_ht, cxt.srcht_chunk, chunk_table); EventTriggerAlterTableEnd(); /* Insert empty stats to compression_chunk_size */ compression_chunk_size_catalog_insert(cxt.srcht_chunk->fd.id, &uncompressed_size, compress_ht_chunk->fd.id, &compressed_size, numrows_pre_compression, numrows_post_compression, 0); chunk_was_compressed = ts_chunk_is_compressed(cxt.srcht_chunk); ts_chunk_set_compressed_chunk(cxt.srcht_chunk, compress_ht_chunk->fd.id); if (!chunk_was_compressed && ts_table_has_tuples(cxt.srcht_chunk->table_id, AccessShareLock)) { /* The chunk was not compressed before it had the compressed chunk * attached to it, and it contains rows, so we set it to be partial. */ ts_chunk_set_partial(cxt.srcht_chunk); } ts_cache_release(&hcache); PG_RETURN_OID(chunk_relid); } Datum tsl_compress_chunk(PG_FUNCTION_ARGS) { Oid uncompressed_chunk_id = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); bool if_not_compressed = PG_ARGISNULL(1) ? true : PG_GETARG_BOOL(1); bool recompress = PG_ARGISNULL(2) ? false : PG_GETARG_BOOL(2); ts_feature_flag_check(FEATURE_HYPERTABLE_COMPRESSION); TS_PREVENT_FUNC_IF_READ_ONLY(); Chunk *chunk = ts_chunk_get_by_relid(uncompressed_chunk_id, true); uncompressed_chunk_id = tsl_compress_chunk_wrapper(chunk, if_not_compressed, recompress); PG_RETURN_OID(uncompressed_chunk_id); } Oid tsl_compress_chunk_wrapper(Chunk *chunk, bool if_not_compressed, bool recompress) { Oid uncompressed_chunk_id = chunk->table_id; write_logical_replication_msg_compression_start(); if (ts_chunk_needs_compression(chunk)) { uncompressed_chunk_id = compress_chunk_impl(chunk->hypertable_relid, chunk->table_id); } else if (recompress || ts_chunk_needs_recompression(chunk)) { /* Try in-memory recompression first and then fall back to decompress/recompress */ bool recompressed = recompress_chunk_impl(chunk, &uncompressed_chunk_id, recompress); if (!recompressed) { /* TODO: move away from manual decompression/compression */ elog(DEBUG1, "falling back to compress/decompress, performing full " "recompression on chunk \"%s.%s\"", NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name)); decompress_chunk_impl(chunk, false); compress_chunk_impl(chunk->hypertable_relid, chunk->table_id); } } else { ereport((if_not_compressed ? NOTICE : ERROR), (errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("chunk \"%s\" is already converted to columnstore", get_rel_name(chunk->table_id)))); } write_logical_replication_msg_compression_end(); return uncompressed_chunk_id; } Datum tsl_decompress_chunk(PG_FUNCTION_ARGS) { Oid uncompressed_chunk_id = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); bool if_compressed = PG_ARGISNULL(1) ? true : PG_GETARG_BOOL(1); int32 chunk_id; ts_feature_flag_check(FEATURE_HYPERTABLE_COMPRESSION); TS_PREVENT_FUNC_IF_READ_ONLY(); Chunk *uncompressed_chunk = ts_chunk_get_by_relid(uncompressed_chunk_id, true); chunk_id = uncompressed_chunk->fd.id; Hypertable *ht = ts_hypertable_get_by_id(uncompressed_chunk->fd.hypertable_id); ts_hypertable_permissions_check(ht->main_table_relid, GetUserId()); if (!ht->fd.compressed_hypertable_id) ereport(ERROR, (errcode(ERRCODE_TS_HYPERTABLE_NOT_EXIST), errmsg("missing columnstore-enabled hypertable"))); if (!ts_chunk_is_compressed(uncompressed_chunk)) { ereport((if_compressed ? NOTICE : ERROR), (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("chunk \"%s\" is not converted to columnstore", get_rel_name(uncompressed_chunk_id)))); PG_RETURN_NULL(); } else decompress_chunk_impl(uncompressed_chunk, if_compressed); /* * Post decompression regular DML can happen into this chunk. So, we update * chunk_column_stats entries for this chunk to min/max entries now. */ ts_chunk_column_stats_reset_by_chunk_id(chunk_id); PG_RETURN_OID(uncompressed_chunk_id); } static bool can_use_in_memory_rebuild(Chunk *chunk) { CompressionSettings *chunk_settings = ts_compression_settings_get(chunk->table_id); /* check if we can allow in-memory recompression to rebuild columnstore */ if (!chunk_settings || !chunk_settings->fd.orderby) { elog(DEBUG1, "in-memory rebuild columnstore is disabled due to no order by"); return false; } if (!get_compressed_chunk_index_for_recompression(chunk)) { elog(DEBUG1, "in-memory rebuild columnstore is disabled due to no compressed chunk index."); return false; } if (!ts_guc_enable_in_memory_recompression) { elog(DEBUG1, "timescaledb.enable_in_memory_recompression is disabled"); return false; } return true; } Datum tsl_rebuild_columnstore(PG_FUNCTION_ARGS) { Oid chunk_relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); ts_feature_flag_check(FEATURE_HYPERTABLE_COMPRESSION); TS_PREVENT_FUNC_IF_READ_ONLY(); if (!OidIsValid(chunk_relid)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid chunk OID"))); Chunk *chunk = ts_chunk_get_by_relid(chunk_relid, true); if (!ts_chunk_is_compressed(chunk) || ts_chunk_is_frozen(chunk)) { ereport(NOTICE, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("chunk \"%s.%s\" is uncompressed or frozen, skipping", NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name)))); PG_RETURN_VOID(); } /* Try rebuild with in-memory recompression, fall back to decompress/compress if needed */ if (!can_use_in_memory_rebuild(chunk) || !recompress_chunk_in_memory_impl(chunk)) { elog(DEBUG1, "falling back to decompress/compress, performing full " "rebuild on chunk \"%s.%s\"", NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name)); decompress_chunk_impl(chunk, false); compress_chunk_impl(chunk->hypertable_relid, chunk->table_id); } PG_RETURN_VOID(); } /* * This is hacky but it doesn't matter. We just want to check for the existence of such an index * on the compressed chunk. */ extern Datum tsl_get_compressed_chunk_index_for_recompression(PG_FUNCTION_ARGS) { ts_feature_flag_check(FEATURE_HYPERTABLE_COMPRESSION); Oid uncompressed_chunk_id = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); Chunk *uncompressed_chunk = ts_chunk_get_by_relid(uncompressed_chunk_id, true); Oid index_oid = get_compressed_chunk_index_for_recompression(uncompressed_chunk); if (OidIsValid(index_oid)) { PG_RETURN_OID(index_oid); } else PG_RETURN_NULL(); } static Oid get_compressed_chunk_index_for_recompression(Chunk *uncompressed_chunk) { Chunk *compressed_chunk = ts_chunk_get_by_id(uncompressed_chunk->fd.compressed_chunk_id, true); Relation uncompressed_chunk_rel = table_open(uncompressed_chunk->table_id, AccessShareLock); Relation compressed_chunk_rel = table_open(compressed_chunk->table_id, AccessShareLock); CompressionSettings *settings = ts_compression_settings_get(uncompressed_chunk->table_id); CatalogIndexState indstate = CatalogOpenIndexes(compressed_chunk_rel); Oid index_oid = get_compressed_chunk_index(indstate, settings); CatalogCloseIndexes(indstate); table_close(compressed_chunk_rel, NoLock); table_close(uncompressed_chunk_rel, NoLock); return index_oid; } Chunk * tsl_compression_chunk_create(Hypertable *compressed_ht, Chunk *src_chunk) { /* Create a new compressed chunk */ return create_compress_chunk(compressed_ht, src_chunk, InvalidOid); } Datum tsl_estimate_compressed_batch_size(PG_FUNCTION_ARGS) { Oid relid = PG_GETARG_OID(0); ts_feature_flag_check(FEATURE_HYPERTABLE_COMPRESSION); float8 approx_batch_size = (float8) ts_columnar_estimate_compressed_batch_size(relid); PG_RETURN_FLOAT8(approx_batch_size); } ================================================ FILE: tsl/src/compression/api.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <fmgr.h> #include <utils.h> #include "chunk.h" extern Datum tsl_create_compressed_chunk(PG_FUNCTION_ARGS); extern Datum tsl_compress_chunk(PG_FUNCTION_ARGS); extern Datum tsl_decompress_chunk(PG_FUNCTION_ARGS); extern Datum tsl_rebuild_columnstore(PG_FUNCTION_ARGS); extern Oid tsl_compress_chunk_wrapper(Chunk *chunk, bool if_not_compressed, bool recompress); extern Chunk *tsl_compression_chunk_create(Hypertable *compressed_ht, Chunk *src_chunk); extern Datum tsl_get_compressed_chunk_index_for_recompression( PG_FUNCTION_ARGS); // arg is oid of uncompressed chunk extern void compression_chunk_size_catalog_insert(int32 src_chunk_id, const RelationSize *src_size, int32 compress_chunk_id, const RelationSize *compress_size, int64 rowcnt_pre_compression, int64 rowcnt_post_compression, int64 rowcnt_frozen); extern Datum tsl_estimate_compressed_batch_size(PG_FUNCTION_ARGS); ================================================ FILE: tsl/src/compression/arrow_c_data_interface.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once /* * This header describes the Arrow C data interface which is a well-known * standard for in-memory interchange of columnar data. * * https://arrow.apache.org/docs/format/CDataInterface.html * * Citing from the above link: * * Arrow C data interface defines a very small, stable set of C definitions that * can be easily copied in any project’s source code and used for columnar data * interchange in the Arrow format. For non-C/C++ languages and runtimes, it * should be almost as easy to translate the C definitions into the * corresponding C FFI declarations. * * Applications and libraries can therefore work with Arrow memory without * necessarily using Arrow libraries or reinventing the wheel. */ #ifndef ARROW_C_DATA_INTERFACE #define ARROW_C_DATA_INTERFACE #define ARROW_FLAG_DICTIONARY_ORDERED 1 #define ARROW_FLAG_NULLABLE 2 #define ARROW_FLAG_MAP_KEYS_SORTED 4 typedef struct ArrowArray { /* * Mandatory. The logical length of the array (i.e. its number of items). */ int64 length; /* * Mandatory. The number of null items in the array. MAY be -1 if not yet * computed. */ int64 null_count; /* * Mandatory. The logical offset inside the array (i.e. the number of * items from the physical start of the buffers). MUST be 0 or positive. * * Producers MAY specify that they will only produce 0-offset arrays to * ease implementation of consumer code. Consumers MAY decide not to * support non-0-offset arrays, but they should document this limitation. */ int64 offset; /* * Mandatory. The number of physical buffers backing this array. The * number of buffers is a function of the data type, as described in the * Columnar format specification. * * Buffers of children arrays are not included. */ int64 n_buffers; /* * Mandatory. The number of children this type has. */ int64 n_children; /* * Mandatory. A C array of pointers to the start of each physical buffer * backing this array. Each void* pointer is the physical start of a * contiguous buffer. There must be ArrowArray.n_buffers pointers. * * The producer MUST ensure that each contiguous buffer is large enough to * represent length + offset values encoded according to the Columnar * format specification. * * It is recommended, but not required, that the memory addresses of the * buffers be aligned at least according to the type of primitive data * that they contain. Consumers MAY decide not to support unaligned * memory. * * The buffer pointers MAY be null only in two situations: * * - for the null bitmap buffer, if ArrowArray.null_count is 0; * * - for any buffer, if the size in bytes of the corresponding buffer would * be 0. * * Buffers of children arrays are not included. */ const void **buffers; struct ArrowArray **children; struct ArrowArray *dictionary; /* * Mandatory. A pointer to a producer-provided release callback. * * See below for memory management and release callback semantics. */ void (*release)(struct ArrowArray *); /* Opaque producer-specific data */ void *private_data; } ArrowArray; /* * We don't use the schema but have to define it for completeness because we're * defining the ARROW_C_DATA_INTERFACE macro. */ struct ArrowSchema { const char *format; const char *name; const char *metadata; int64 flags; int64 n_children; struct ArrowSchema **children; struct ArrowSchema *dictionary; void (*release)(struct ArrowSchema *); void *private_data; }; /* * The include guard ARROW_C_DATA_INTERFACE is required by the Arrow docs to * avoid redefinition of the Arrow structs in the third-party headers, but the * following functions are not part of Arrow C Data Interface, so they are not * under the guard. We still need some kind of guard for them, so we also have * pragma once above. */ #endif static pg_attribute_always_inline bool arrow_row_is_valid(const uint64 *bitmap, size_t row_number) { if (likely(bitmap == NULL)) { return true; } const size_t qword_index = row_number / 64; const size_t bit_index = row_number % 64; const uint64 mask = 1ull << bit_index; return bitmap[qword_index] & mask; } /* * Same as above but for two bitmaps, this is a typical situation when we have * validity bitmap + filter result. */ static pg_attribute_always_inline bool arrow_row_both_valid(const uint64 *bitmap1, const uint64 *bitmap2, size_t row_number) { if (likely(bitmap1 == NULL)) { return arrow_row_is_valid(bitmap2, row_number); } if (likely(bitmap2 == NULL)) { return arrow_row_is_valid(bitmap1, row_number); } const size_t qword_index = row_number / 64; const size_t bit_index = row_number % 64; const uint64 mask = 1ull << bit_index; return (bitmap1[qword_index] & bitmap2[qword_index]) & mask; } static pg_attribute_always_inline void arrow_set_row_validity(uint64 *bitmap, size_t row_number, bool value) { const size_t qword_index = row_number / 64; const size_t bit_index = row_number % 64; const uint64 mask = 1ull << bit_index; const uint64 new_bit = (value ? 1ull : 0ull) << bit_index; bitmap[qword_index] = (bitmap[qword_index] & ~mask) | new_bit; Assert(arrow_row_is_valid(bitmap, row_number) == value); } /* * Combine the validity bitmaps into the given storage. Can return one of the * input filters if the others are NULL. */ static inline pg_nodiscard const uint64 * arrow_combine_validity(size_t num_words, uint64 *restrict storage, const uint64 *filter1, const uint64 *filter2, const uint64 *filter3) { Assert(num_words != 0); Assert(storage != filter1); Assert(storage != filter2); Assert(storage != filter3); /* * Any and all of the filters can be null. For simplicity, move the non-null * filters to the leading positions. */ const uint64 *tmp; #define SWAP(X, Y) \ tmp = (X); \ (X) = (Y); \ (Y) = tmp; if (filter1 == NULL) { /* * We have at least one NULL that goes to the last position. */ SWAP(filter1, filter3); if (filter1 == NULL) { /* * We have another NULL that goes to the second position. */ SWAP(filter1, filter2); } } else { if (filter2 == NULL) { /* * We have at least one NULL that goes to the last position. */ SWAP(filter2, filter3); } } #undef SWAP Assert(filter2 == NULL || filter1 != NULL); Assert(filter3 == NULL || filter2 != NULL); if (filter2 == NULL) { /* Either have one non-null filter, or all of them are null. */ return filter1; } if (filter3 == NULL) { /* Have two non-null filters. */ for (size_t i = 0; i < num_words; i++) { storage[i] = filter1[i] & filter2[i]; } } else { /* Have three non-null filters. */ for (size_t i = 0; i < num_words; i++) { storage[i] = filter1[i] & filter2[i] & filter3[i]; } } return storage; } /* * Do the &= operation on bitmaps. The right argument can be NULL. */ static inline void arrow_validity_and(int num_words, uint64 *restrict left, const uint64 *right) { if (right == NULL) { return; } for (int i = 0; i < num_words; i++) { left[i] &= right[i]; } } /* * Increase the `source_value` to be an even multiple of `pad_to`. */ static inline uint64 pad_to_multiple(uint64 pad_to, uint64 source_value) { return ((source_value + pad_to - 1) / pad_to) * pad_to; } static inline int arrow_num_valid(const uint64 *bitmap, size_t total_rows) { Assert(total_rows != 0); if (bitmap == NULL) { return total_rows; } uint64 num_valid = 0; #ifdef HAVE__BUILTIN_POPCOUNT const uint64 words = pad_to_multiple(64, total_rows) / 64; for (uint64 i = 0; i < words; i++) { num_valid += __builtin_popcountll(bitmap[i]); } #else for (size_t i = 0; i < total_rows; i++) { num_valid += arrow_row_is_valid(bitmap, i); } #endif return num_valid; } ================================================ FILE: tsl/src/compression/batch_metadata_builder.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include "funcapi.h" /* for PGFunction, FmgrInfo */ typedef struct RowCompressor RowCompressor; enum BatchMetadataBuilderType { METADATA_BUILDER_MINMAX, METADATA_BUILDER_BLOOM1, }; typedef struct BatchMetadataBuilder { void (*update_row)(void *builder, TupleTableSlot *slot); void (*insert_to_compressed_row)(void *builder, RowCompressor *compressor); void (*reset)(void *builder, RowCompressor *compressor); enum BatchMetadataBuilderType builder_type; } BatchMetadataBuilder; BatchMetadataBuilder *batch_metadata_builder_minmax_create(Oid type, Oid collation, AttrNumber attnum, int min_attr_offset, int max_attr_offset); BatchMetadataBuilder *batch_metadata_builder_bloom1_create(int num_columns, const Oid *type_oids, const AttrNumber *attnums, int bloom_attr_offset); /* Hasher interface common to bloom filters, used to compute the hash without updating the bloom * filter */ typedef struct Bloom1Hasher { uint64 (*hash_values)(void *hasher, const NullableDatum *values); int num_columns; } Bloom1Hasher; Bloom1Hasher *bloom1_hasher_create(const Oid *type_oids, int num_columns); /* Shared utilities between metadata builders */ int batch_metadata_builder_bloom1_varlena_size(void); uint64 batch_metadata_builder_bloom1_calculate_hash(PGFunction hash_function, FmgrInfo *finfo, Datum needle); void batch_metadata_builder_bloom1_update_bloom_filter_with_hash(void *varlena_ptr, uint64 hash); void batch_metadata_builder_bloom1_insert_bloom_filter_to_compressed_row(void *bloom_varlena, int16 bloom_attr_offset, RowCompressor *compressor); /* Returns true if the hash is maybe present in a bloom filter, if the bloom filter data is * NULL, it returns true, because we cannot be sure if the hash is present or not. */ extern bool bloom1_contains_hash(Datum bloom_datum, uint64 hash); ================================================ FILE: tsl/src/compression/batch_metadata_builder_bloom1.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include "postgres.h" #include <access/detoast.h> #include <catalog/pg_collation_d.h> #include <common/hashfn.h> #include <funcapi.h> #include <utils/builtins.h> #include <utils/typcache.h> #include <math.h> #include "sparse_index_bloom1.h" #include "arrow_c_data_interface.h" #include "batch_metadata_builder.h" #include "city_combine.h" #include "compression.h" #include "guc.h" #ifdef TS_USE_UMASH #include "import/umash.h" #endif /* * There is a tradeoff between making the bloom filters selective enough but * not too big. Testing on some real words datasets, the optimal value seems to * be about 2% false positives. We also want to be able to reduce the filter to * up to 64 bits, so that it fits in the main table. Hence the optimal number of * hashes. This number actually gives a slightly different false positive rate, * so this is what we ultimately use. * Calculator: https://hur.st/bloomfilter/?p=0.02&m=64 */ #define BLOOM1_FALSE_POSITIVES 0.022 #define BLOOM1_HASHES 6 /* * Limit the bits belonging to the particular elements to a small contiguous * region. This improves memory locality when building the bloom filter. */ #define BLOOM1_BLOCK_BITS 256 /* The NULL marker is used to preserve NULLs in the composite hash. The value is coming from Golden * ratio constant so it is unlikely to degrade the collision resistance of the bloom filter. * The purpose of the NULL marker is two fold: 1) to distinguish NULLs from actual values so * we don't get the same hash for ('foo',NULL) and (NULL, 'foo') and 2) with having a specific * marker, we can support IS NULL predicates on columns in addition to equality. * */ #define NULL_MARKER 0x9E3779B97F4A7C15ULL typedef struct Bloom1HasherInternal { Bloom1Hasher functions; PGFunction hash_functions[MAX_BLOOM_FILTER_COLUMNS]; FmgrInfo *hash_function_finfos[MAX_BLOOM_FILTER_COLUMNS]; } Bloom1HasherInternal; typedef struct Bloom1MetadataBuilder { BatchMetadataBuilder functions; int16 bloom_attr_offset; int allocated_varlena_bytes; struct varlena *bloom_varlena; AttrNumber input_columns[MAX_BLOOM_FILTER_COLUMNS]; Bloom1HasherInternal hasher; } Bloom1MetadataBuilder; static void bloom1_hasher_init(Bloom1HasherInternal *hasher, const Oid *type_oids, int num_columns); /* * Low-bias invertible hash function from this article: * http://web.archive.org/web/20250406022607/https://nullprogram.com/blog/2018/07/31/ */ static inline uint64 bloom1_hash64(uint64 x) { x ^= x >> 32; x *= 0xd6e8feb86659fd93U; x ^= x >> 32; x *= 0xd6e8feb86659fd93U; x ^= x >> 32; return x; } static Datum bloom1_hash_8(PG_FUNCTION_ARGS) { PG_RETURN_UINT64(bloom1_hash64(PG_GETARG_INT64(0))); } static Datum bloom1_hash_4(PG_FUNCTION_ARGS) { PG_RETURN_UINT64(bloom1_hash64(PG_GETARG_INT32(0))); } #ifdef TS_USE_UMASH static struct umash_params * hashing_params() { static struct umash_params params = { 0 }; if (params.poly[0][0] == 0) { umash_params_derive(¶ms, 0x12345abcdef67890ULL, NULL); Assert(params.poly[0][0] != 0); } return ¶ms; } static Datum bloom1_hash_varlena(PG_FUNCTION_ARGS) { struct varlena *needle = PG_DETOAST_DATUM_PACKED(PG_GETARG_DATUM(0)); const int length = VARSIZE_ANY_EXHDR(needle); const char *data = VARDATA_ANY(needle); PG_RETURN_UINT64(umash_full(hashing_params(), /* seed = */ ~0ULL, /* which = */ 0, data, length)); } static Datum bloom1_hash_16(PG_FUNCTION_ARGS) { Datum datum = PG_GETARG_DATUM(0); PG_RETURN_UINT64(umash_full(hashing_params(), /* seed = */ ~0ULL, /* which = */ 0, DatumGetPointer(datum), 16)); } #endif /* * Get the hash function we use for building a bloom filter for a particular * type. Returns NULL if not supported. * The signature of the returned function matches the Postgres extended hashing * functions like hashtextextended(). * It's possible, though impractical, for a hash function to be implemented in a * procedural language, not in C. In this case, we need the proper FmgrInfo to * call it. We fetch it from the type cache. For our custom functions, it is NULL. */ PGFunction bloom1_get_hash_function(Oid type, FmgrInfo **finfo) { *finfo = NULL; /* * By default, we use the Postgres extended hashing functions, so that we * can use bloom filters for any types. * We request also the opfamily info and the equality operator, because * otherwise the Postgres type cache code fails obtusely on types with * improper opclasses. It picks up the btree opclass from a binary compatible * type (see GetDefaultOpClass), then an equality operator from this opclass, * and then refuses to return the hash functions because the hash opclass has * a different equality operator. The problem is that this happens over two * consecutive calls to lookup_type_cache(), so the first invocation of our * function says that we have a hash, and the second says that we don't. */ TypeCacheEntry *entry = lookup_type_cache(type, TYPECACHE_EQ_OPR | TYPECACHE_BTREE_OPFAMILY | TYPECACHE_HASH_EXTENDED_PROC_FINFO); /* * For some types we use our custom hash functions. We only do it for the * builtin Postgres types to be on the safe side, and also simplify the * testing by creating bad hash functions from SQL tests. If you change this, * you might have to change the bad hash testing in compress_bloom_sparse.sql. */ switch (entry->hash_extended_proc) { #ifdef TS_USE_UMASH case F_HASHTEXTEXTENDED: return bloom1_hash_varlena; case F_UUID_HASH_EXTENDED: return bloom1_hash_16; #endif case F_HASHINT8EXTENDED: return bloom1_hash_8; case F_HASHINT4EXTENDED: #if PG18_GE /* * PG18 added a custom hashing function for date type. * For backwards compatibility, we need to continue using * our own custom function which was used for < PG18. * * https://github.com/postgres/postgres/commit/23d0b484 */ case F_HASHDATEEXTENDED: #endif return bloom1_hash_4; default: /* * Use the Postgres hash function. We might require the finfo, for * example for functions defined in procedural languages. */ *finfo = &entry->hash_extended_proc_finfo; return entry->hash_extended_proc_finfo.fn_addr; } } static void bloom1_reset(void *builder_, RowCompressor *compressor) { Bloom1MetadataBuilder *builder = (Bloom1MetadataBuilder *) builder_; Assert(builder->functions.builder_type == METADATA_BUILDER_BLOOM1); struct varlena *bloom = builder->bloom_varlena; memset(bloom, 0, builder->allocated_varlena_bytes); SET_VARSIZE(bloom, builder->allocated_varlena_bytes); compressor->compressed_is_null[builder->bloom_attr_offset] = true; compressor->compressed_values[builder->bloom_attr_offset] = 0; } static char * bloom1_words_buf(struct varlena *bloom) { return VARDATA_ANY(bloom); } static int bloom1_num_bits(const struct varlena *bloom) { return 8 * VARSIZE_ANY_EXHDR(bloom); } void batch_metadata_builder_bloom1_insert_bloom_filter_to_compressed_row(void *bloom_varlena, int16 bloom_attr_offset, RowCompressor *compressor) { struct varlena *bloom = (struct varlena *) bloom_varlena; char *restrict words_buf = bloom1_words_buf(bloom); const int orig_num_bits = bloom1_num_bits(bloom); Assert(orig_num_bits % 8 == 0); Assert(orig_num_bits % 64 == 0); const int orig_bits_set = pg_popcount(words_buf, orig_num_bits / 8); if (unlikely(orig_bits_set == 0 || orig_bits_set == orig_num_bits)) { /* * 1) All elements turned out to be null, don't save the empty filter in * that case. The compressed batch will be compressed using the NULL * compression algorithm, so actually checking the rows will be efficient * enough. * * 2) All bits are set, this filter is useless. Shouldn't really happen, * but technically possible, and the following calculations will * segfault in this case. */ compressor->compressed_is_null[bloom_attr_offset] = true; compressor->compressed_values[bloom_attr_offset] = PointerGetDatum(NULL); return; } /* * Our filters are sized for the maximum expected number of the unique * elements, so in practice they can be very sparse if the actual number of * the unique elements is less. The TOAST compression doesn't handle even * the sparse filters very well. Apply a simple compression technique: split * the filter in half and bitwise OR the halves. Repeat this until we reach * the filter bit length that gives the desired false positive ratio. * The desired filter bit length is given by m1, we will now estimate it * based on the estimated current number of elements in the bloom filter (1) * and the ideal number of elements for a bloom filter of given size (2). * (1) n = log(1 - t/m0) / (k * log(1 - 1/m0)), * (2) n = -m1 * log(1 - p ^ (1/k)) / k. */ const double m0 = orig_num_bits; const double k = BLOOM1_HASHES; const double p = BLOOM1_FALSE_POSITIVES; const double t = orig_bits_set; const double m1 = -log(1 - t / m0) / (log(1 - 1 / m0) * log(1 - pow(p, 1 / k))); /* * Compute powers of two corresponding to the current and desired filter * bit length. */ const int starting_pow2 = ceil(log2(m0)); Assert(pow(2, starting_pow2) == m0); /* We don't want to go under 64 bytes. */ const int final_pow2 = MAX(6, ceil(log2(m1))); Assert(final_pow2 >= 6); /* * Fold filter in half, applying bitwise OR, until we reach the desired * filter bit length. */ for (int current_pow2 = starting_pow2; current_pow2 > final_pow2; current_pow2--) { const int half_words = 1 << (current_pow2 - 3 /* 8-bit byte */ - 1 /* half */); Assert(half_words > 0); const char *words_tail = &words_buf[half_words]; for (int i = 0; i < half_words; i++) { words_buf[i] |= words_tail[i]; } } /* * If we have resized the filter, update the nominal size of the varlena * object. */ if (final_pow2 < starting_pow2) { SET_VARSIZE(bloom, (char *) bloom1_words_buf(bloom) + (1 << (final_pow2 - 3 /* 8-bit byte */)) - (char *) bloom); } Assert(bloom1_num_bits(bloom) % (sizeof(*words_buf) * 8) == 0); Assert(bloom1_num_bits(bloom) % 64 == 0); compressor->compressed_is_null[bloom_attr_offset] = false; compressor->compressed_values[bloom_attr_offset] = PointerGetDatum(bloom); } static void bloom1_insert_to_compressed_row(void *builder_, RowCompressor *compressor) { Bloom1MetadataBuilder *builder = (Bloom1MetadataBuilder *) builder_; batch_metadata_builder_bloom1_insert_bloom_filter_to_compressed_row(builder->bloom_varlena, builder->bloom_attr_offset, compressor); } /* * Call a hash function that uses a postgres "extended hash" signature. */ uint64 batch_metadata_builder_bloom1_calculate_hash(PGFunction hash_function, FmgrInfo *finfo, Datum needle) { LOCAL_FCINFO(hashfcinfo, 2); *hashfcinfo = (FunctionCallInfoBaseData){ 0 }; /* * Our hashing is not collation-sensitive, but the Postgres hashing functions * might refuse to work if the collation is not deterministic, so make them * happy. */ hashfcinfo->fncollation = C_COLLATION_OID; hashfcinfo->nargs = 2; hashfcinfo->args[0].value = needle; hashfcinfo->args[0].isnull = false; /* * Seed. Note that on 32-bit systems it is by-reference. */ const int64 seed = 0; hashfcinfo->args[1].value = Int64GetDatumFast(seed); hashfcinfo->args[1].isnull = false; /* * Needed for hash functions defined in procedural languages, not C. While * unlikely, we shouldn't segfault. The finfo is cached in the type cache. */ hashfcinfo->flinfo = finfo; return DatumGetUInt64(hash_function(hashfcinfo)); } /* * The offset of nth bit we're going to set. */ static inline uint32 bloom1_get_one_offset(uint64 value_hash, uint32 index) { const uint32 low = value_hash & ~(uint32) 0; const uint32 high = (value_hash >> 32) & ~(uint32) 0; /* * Add a quadratic component to lessen degradation in the unlikely case when * 'high' is a multiple of block bits. */ return low + (index * high + index * index) % BLOOM1_BLOCK_BITS; } void batch_metadata_builder_bloom1_update_bloom_filter_with_hash(void *varlena_ptr, uint64 hash) { Assert(varlena_ptr != NULL); struct varlena *bloom_varlena = (struct varlena *) varlena_ptr; char *restrict words_buf = bloom1_words_buf(bloom_varlena); const uint32 num_bits = bloom1_num_bits(bloom_varlena); /* * These calculations are a little inconvenient, but I had to switch to * another buffer word size already, so for now I'm keeping the code generic * relative to this size. */ const uint32 num_word_bits = sizeof(*words_buf) * 8; Assert(num_bits % num_word_bits == 0); const uint32 log2_word_bits = pg_leftmost_one_pos32(num_word_bits); Assert(num_word_bits == (1ULL << log2_word_bits)); const uint32 word_mask = num_word_bits - 1; Assert((word_mask >> num_word_bits) == 0); const uint32 absolute_mask = num_bits - 1; for (int i = 0; i < BLOOM1_HASHES; i++) { const uint32 absolute_bit_index = bloom1_get_one_offset(hash, i) & absolute_mask; const uint32 word_index = absolute_bit_index >> log2_word_bits; const uint32 word_bit_index = absolute_bit_index & word_mask; words_buf[word_index] |= 1ULL << word_bit_index; } } static uint64 bloom1_hash_values(void *hasher_, const NullableDatum *values) { Bloom1HasherInternal *hasher = (Bloom1HasherInternal *) hasher_; int num_columns = hasher->functions.num_columns; uint64 accumulated = 0; if (values[0].isnull) accumulated = NULL_MARKER; else accumulated = batch_metadata_builder_bloom1_calculate_hash(hasher->hash_functions[0], hasher->hash_function_finfos[0], values[0].value); for (int i = 1; i < num_columns; i++) { if (values[i].isnull) { accumulated = city_hash_combine(accumulated, NULL_MARKER); } else { uint64 h = batch_metadata_builder_bloom1_calculate_hash(hasher->hash_functions[i], hasher->hash_function_finfos[i], values[i].value); accumulated = city_hash_combine(accumulated, h); } } return accumulated; } static void bloom1_update_row(void *builder_, TupleTableSlot *slot) { Bloom1MetadataBuilder *builder = (Bloom1MetadataBuilder *) builder_; Bloom1Hasher *hasher = &builder->hasher.functions; int num_columns = hasher->num_columns; NullableDatum values[MAX_BLOOM_FILTER_COLUMNS]; for (int i = 0; i < num_columns; i++) { values[i].value = slot_getattr(slot, builder->input_columns[i], &values[i].isnull); } /* For single-column blooms, skip NULLs to match old bloom1_update_null (no-op) behavior. */ if (num_columns == 1 && values[0].isnull) return; uint64 hash = hasher->hash_values(hasher, values); batch_metadata_builder_bloom1_update_bloom_filter_with_hash(builder->bloom_varlena, hash); } /* * We cache some information across function calls in this context. */ typedef struct Bloom1ContainsContext { Oid element_type; int16 element_typlen; bool element_typbyval; char element_typalign; /* This is per-row, here for convenience. */ struct varlena *current_row_bloom; Bloom1HasherInternal bloom_hasher; } Bloom1ContainsContext; static Bloom1ContainsContext * bloom1_contains_context_prepare(FunctionCallInfo fcinfo, bool use_element_type) { Bloom1ContainsContext *context = (Bloom1ContainsContext *) fcinfo->flinfo->fn_extra; if (context == NULL) { Ensure(PG_NARGS() == 2, "bloom1_contains called with wrong number of arguments"); context = MemoryContextAllocZero(fcinfo->flinfo->fn_mcxt, sizeof(*context)); context->element_type = get_fn_expr_argtype(fcinfo->flinfo, 1); if (use_element_type) { context->element_type = get_element_type(context->element_type); Ensure(OidIsValid(context->element_type), "cannot determine array element type for bloom1_contains_any"); } Oid type_oids[MAX_BLOOM_FILTER_COLUMNS]; int num_columns; if (context->element_type == RECORDOID) { HeapTupleHeader tuple = DatumGetHeapTupleHeader(PG_GETARG_DATUM(1)); Oid tupType = HeapTupleHeaderGetTypeId(tuple); int32 tupTypmod = HeapTupleHeaderGetTypMod(tuple); TupleDesc tupdesc = lookup_rowtype_tupdesc(tupType, tupTypmod); num_columns = tupdesc->natts; if (num_columns > MAX_BLOOM_FILTER_COLUMNS) ereport(ERROR, (errcode(ERRCODE_DATA_EXCEPTION), errmsg("composite bloom filter supports at most %d columns, got %d", MAX_BLOOM_FILTER_COLUMNS, num_columns))); for (int i = 0; i < num_columns; i++) type_oids[i] = TupleDescAttr(tupdesc, i)->atttypid; ReleaseTupleDesc(tupdesc); } else { type_oids[0] = context->element_type; num_columns = 1; } bloom1_hasher_init(&context->bloom_hasher, type_oids, num_columns); get_typlenbyvalalign(context->element_type, &context->element_typlen, &context->element_typbyval, &context->element_typalign); fcinfo->flinfo->fn_extra = context; } if (PG_ARGISNULL(0)) { context->current_row_bloom = NULL; } else { context->current_row_bloom = PG_GETARG_VARLENA_P(0); } return context; } static inline bool bloom1_contains_hash_internal(const char *words_buf, uint32 num_bits, uint64 hash) { Assert(words_buf != NULL); /* Must be a power of two. */ CheckCompressedData(num_bits == (1ULL << pg_leftmost_one_pos32(num_bits))); /* Must be >= 64 bits. */ CheckCompressedData(num_bits >= 64); const uint32 num_word_bits = sizeof(*words_buf) * 8; Assert(num_bits % num_word_bits == 0); const uint32 log2_word_bits = pg_leftmost_one_pos32(num_word_bits); Assert(num_word_bits == (1ULL << log2_word_bits)); const uint32 word_mask = num_word_bits - 1; Assert((word_mask >> num_word_bits) == 0); const uint32 absolute_mask = num_bits - 1; for (int i = 0; i < BLOOM1_HASHES; i++) { const uint32 absolute_bit_index = bloom1_get_one_offset(hash, i) & absolute_mask; const uint32 word_index = absolute_bit_index >> log2_word_bits; const uint32 word_bit_index = absolute_bit_index & word_mask; if ((words_buf[word_index] & (1ULL << word_bit_index)) == 0) { return false; } } return true; } /* * Checks whether the given element can be present in the given bloom filter. * This is what we use in predicate pushdown. The SQL signature is: * _timescaledb_functions.bloom1_contains(bloom1, anyelement) */ Datum bloom1_contains(PG_FUNCTION_ARGS) { /* * A null value cannot match the equality condition, although this probably * should be optimized away by the planner. */ if (PG_ARGISNULL(1)) { PG_RETURN_BOOL(false); } Bloom1ContainsContext *context = bloom1_contains_context_prepare(fcinfo, /* use_element_type = */ false); /* * This function is not strict, because if we don't have a bloom filter, this * means the condition can potentially be true. */ struct varlena *bloom = context->current_row_bloom; if (bloom == NULL) { PG_RETURN_BOOL(true); } uint64 hash = 0; NullableDatum values[MAX_BLOOM_FILTER_COLUMNS]; memset(values, 0, sizeof(values)); if (context->bloom_hasher.functions.num_columns > 1) { HeapTupleHeader tuple = DatumGetHeapTupleHeader(PG_GETARG_DATUM(1)); for (int i = 0; i < context->bloom_hasher.functions.num_columns; i++) values[i].value = GetAttributeByNum(tuple, i + 1, &values[i].isnull); } else { values[0].value = PG_GETARG_DATUM(1); values[0].isnull = false; } hash = bloom1_hash_values(&context->bloom_hasher, values); PG_RETURN_BOOL(bloom1_contains_hash(PointerGetDatum(bloom), hash)); } #define ST_SORT sort_hashes #define ST_ELEMENT_TYPE uint64 #define ST_COMPARE(a, b) ((*(a) > *(b)) - (*(a) < *(b))) #define ST_SCOPE static #define ST_DEFINE #include <lib/sort_template.h> /* * Checks whether any element of the given array can be present in the given * bloom filter. This is used for predicate pushdown for x = any(array[...]). * The SQL signature is: * _timescaledb_functions.bloom1_contains_any(bloom1, anyarray) */ Datum bloom1_contains_any(PG_FUNCTION_ARGS) { Bloom1ContainsContext *context = bloom1_contains_context_prepare(fcinfo, /* use_element_type = */ true); /* * This function is not strict, because if we don't have a bloom filter, this * means the condition can potentially be true. */ struct varlena *bloom = context->current_row_bloom; if (bloom == NULL) { PG_RETURN_BOOL(true); } /* * A null value cannot match the equality condition, although this probably * should be optimized away by the planner. */ if (PG_ARGISNULL(1)) { PG_RETURN_BOOL(false); } int num_items; Datum *items; bool *nulls; deconstruct_array(PG_GETARG_ARRAYTYPE_P(1), context->element_type, context->element_typlen, context->element_typbyval, context->element_typalign, &items, &nulls, &num_items); if (num_items == 0) { PG_RETURN_BOOL(false); } /* * Calculate the per-item base hashes that will be used for computing the * individual bloom filter bit offsets. We can reuse the "items" space to * avoid more allocations, but have to allocate as a fallback on 32-bit * systems. */ #if FLOAT8PASSBYVAL uint64 *item_base_hashes = (uint64 *) items; #else uint64 *item_base_hashes = palloc(sizeof(uint64) * num_items); #endif FmgrInfo *finfo = context->bloom_hasher.hash_function_finfos[0]; PGFunction hash_fn = context->bloom_hasher.hash_functions[0]; int valid = 0; for (int i = 0; i < num_items; i++) { if (nulls[i]) { /* * A null value cannot match the equality condition. */ continue; } item_base_hashes[valid++] = batch_metadata_builder_bloom1_calculate_hash(hash_fn, finfo, items[i]); } if (valid == 0) { /* * No non-null elements. */ PG_RETURN_BOOL(false); } /* * Sort the hashes for cache-friendly probing. */ sort_hashes(item_base_hashes, valid); /* * Get the bloom filter parameters. */ const char *words_buf = bloom1_words_buf(bloom); const uint32 num_bits = bloom1_num_bits(bloom); /* Must be a power of two. */ CheckCompressedData(num_bits == (1ULL << pg_leftmost_one_pos32(num_bits))); /* Must be >= 64 bits. */ CheckCompressedData(num_bits >= 64); const uint32 num_word_bits = sizeof(*words_buf) * 8; Assert(num_bits % num_word_bits == 0); const uint32 log2_word_bits = pg_leftmost_one_pos32(num_word_bits); Assert(num_word_bits == (1ULL << log2_word_bits)); const uint32 word_mask = num_word_bits - 1; Assert((word_mask >> num_word_bits) == 0); const uint32 absolute_mask = num_bits - 1; /* Probe the bloom filter. */ for (int item_index = 0; item_index < valid; item_index++) { const uint64 base_hash = item_base_hashes[item_index]; bool match = true; for (int i = 0; i < BLOOM1_HASHES; i++) { const uint32 absolute_bit_index = bloom1_get_one_offset(base_hash, i) & absolute_mask; const uint32 word_index = absolute_bit_index >> log2_word_bits; const uint32 word_bit_index = absolute_bit_index & word_mask; if ((words_buf[word_index] & (1ULL << word_bit_index)) == 0) { match = false; break; } } if (match) { PG_RETURN_BOOL(true); } } PG_RETURN_BOOL(false); } static int bloom1_varlena_alloc_size(uint32 num_bits) { /* * We are not supposed to go below 64 bits because we work in 64-bit words. */ Assert(num_bits % 64 == 0); Assert(num_bits > 0); /* * We must not go over varlena size limit. */ Assert(num_bits / 8 <= (1ULL << 30) - 1); return VARHDRSZ + num_bits / 8; } int batch_metadata_builder_bloom1_varlena_size(void) { /* * Better make the bloom filter size a power of two, because we compress the * sparse filters using division in half. The formula for the lowest * enclosing power of two is pow(2, floor(log2(x * 2 - 1))). */ const int expected_elements = TARGET_COMPRESSED_BATCH_SIZE * 16; const int lowest_power = pg_leftmost_one_pos32(expected_elements * 2 - 1); /* * The total number of elements must fit into uint32, since that's what we * use for addressing the elements. */ Assert(lowest_power < 32); const int desired_bits = 1ULL << lowest_power; return bloom1_varlena_alloc_size(desired_bits); } static void bloom1_hasher_init(Bloom1HasherInternal *hasher, const Oid *type_oids, int num_columns) { *hasher = (Bloom1HasherInternal){ .functions = (Bloom1Hasher){ .hash_values = bloom1_hash_values, .num_columns = num_columns, }, }; for (int i = 0; i < num_columns; i++) { hasher->hash_functions[i] = bloom1_get_hash_function(type_oids[i], &hasher->hash_function_finfos[i]); if (hasher->hash_functions[i] == NULL) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_FUNCTION), errmsg("the argument type %s lacks an extended hash function", format_type_be(type_oids[i])))); } } Bloom1Hasher * bloom1_hasher_create(const Oid *type_oids, int num_columns) { Bloom1HasherInternal *hasher = palloc(sizeof(*hasher)); bloom1_hasher_init(hasher, type_oids, num_columns); return &hasher->functions; } BatchMetadataBuilder * batch_metadata_builder_bloom1_create(int num_columns, const Oid *type_oids, const AttrNumber *attnums, int bloom_attr_offset) { Assert(num_columns >= 1 && num_columns <= MAX_BLOOM_FILTER_COLUMNS); const int varlena_bytes = batch_metadata_builder_bloom1_varlena_size(); Bloom1MetadataBuilder *builder = palloc(sizeof(*builder)); *builder = (Bloom1MetadataBuilder){ .functions = (BatchMetadataBuilder){ .update_row = bloom1_update_row, .insert_to_compressed_row = bloom1_insert_to_compressed_row, .reset = bloom1_reset, .builder_type = METADATA_BUILDER_BLOOM1, }, .bloom_attr_offset = bloom_attr_offset, .allocated_varlena_bytes = varlena_bytes, }; memcpy(builder->input_columns, attnums, num_columns * sizeof(AttrNumber)); /* Initialize the embedded hasher */ bloom1_hasher_init(&builder->hasher, type_oids, num_columns); /* * Initialize the bloom filter. */ builder->bloom_varlena = palloc0(varlena_bytes); SET_VARSIZE(builder->bloom_varlena, varlena_bytes); return &builder->functions; } #ifndef NDEBUG static int bloom1_estimate_ndistinct(struct varlena *bloom) { const double m = bloom1_num_bits(bloom); const double t = pg_popcount(bloom1_words_buf(bloom), m / 8); const double k = BLOOM1_HASHES; return log(1 - t / m) / (k * log(1 - 1 / m)); } /* * We're slightly modifying this Postgres macro to avoid a warning about signed * vs unsigned comparison. */ #define TS_VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \ ((int) VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) < \ (int) ((toast_pointer).va_rawsize - VARHDRSZ)) TS_FUNCTION_INFO_V1(ts_bloom1_debug_info); /* * A function to output various debugging info about a bloom filter. * * Usage hints in the tests. */ Datum ts_bloom1_debug_info(PG_FUNCTION_ARGS) { /* Build a tuple descriptor for our result type */ TupleDesc tuple_desc; if (get_call_result_type(fcinfo, NULL, &tuple_desc) != TYPEFUNC_COMPOSITE) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context " "that cannot accept type record"))); /* Output columns of this function. */ enum { out_toast_header = 0, out_toasted_bytes, out_compressed_bytes, out_detoasted_bytes, out_bits_total, out_bits_set, out_estimated_elements, _out_columns }; Datum values[_out_columns] = { 0 }; bool nulls[_out_columns] = { 0 }; Datum toasted = PG_GETARG_DATUM(0); values[out_toast_header] = Int32GetDatum(((varattrib_1b *) toasted)->va_header); values[out_toasted_bytes] = Int32GetDatum(VARSIZE_ANY_EXHDR(toasted)); struct varlena *detoasted = PG_DETOAST_DATUM(toasted); values[out_detoasted_bytes] = Int32GetDatum(VARSIZE_ANY_EXHDR(detoasted)); const int bits_total = bloom1_num_bits(detoasted); values[out_bits_total] = Int32GetDatum(bits_total); const char *words = bloom1_words_buf(detoasted); values[out_bits_set] = Int32GetDatum(pg_popcount(words, bits_total / 8)); values[out_estimated_elements] = Int32GetDatum(bloom1_estimate_ndistinct(detoasted)); if (VARATT_IS_EXTERNAL_ONDISK(toasted)) { struct varatt_external toast_pointer; VARATT_EXTERNAL_GET_POINTER(toast_pointer, toasted); if (TS_VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)) { values[out_compressed_bytes] = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer); } else { nulls[out_compressed_bytes] = true; } } else if (VARATT_IS_COMPRESSED(toasted)) { values[out_compressed_bytes] = VARDATA_COMPRESSED_GET_EXTSIZE(toasted); } else { nulls[out_compressed_bytes] = true; } PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tuple_desc, values, nulls))); } TS_FUNCTION_INFO_V1(ts_bloom1_debug_hash); /* * A debug function to inspect the actual hash value used for the bloom filter, * e.g. to find the very even hashes with many low bits equal to zero. */ Datum ts_bloom1_debug_hash(PG_FUNCTION_ARGS) { Oid type_oid = get_fn_expr_argtype(fcinfo->flinfo, 0); FmgrInfo *finfo = NULL; PGFunction fn = bloom1_get_hash_function(type_oid, &finfo); Ensure(fn != NULL, "cannot find our hash function"); Assert(!PG_ARGISNULL(0)); Datum needle = PG_GETARG_DATUM(0); PG_RETURN_UINT64(batch_metadata_builder_bloom1_calculate_hash(fn, finfo, needle)); } TS_FUNCTION_INFO_V1(ts_bloom1_composite_debug_hash); Datum ts_bloom1_composite_debug_hash(PG_FUNCTION_ARGS) { if (PG_ARGISNULL(0)) PG_RETURN_NULL(); HeapTupleHeader tuple = DatumGetHeapTupleHeader(PG_GETARG_DATUM(0)); Oid tupType = HeapTupleHeaderGetTypeId(tuple); int32 tupTypmod = HeapTupleHeaderGetTypMod(tuple); TupleDesc tupdesc = lookup_rowtype_tupdesc(tupType, tupTypmod); int num_fields = tupdesc->natts; if (num_fields < 2 || num_fields > MAX_BLOOM_FILTER_COLUMNS) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("composite bloom requires 2-%d fields, got %d", MAX_BLOOM_FILTER_COLUMNS, num_fields))); Oid type_oids[MAX_BLOOM_FILTER_COLUMNS]; for (int i = 0; i < num_fields; i++) type_oids[i] = TupleDescAttr(tupdesc, i)->atttypid; ReleaseTupleDesc(tupdesc); Bloom1Hasher *hasher = bloom1_hasher_create(type_oids, num_fields); NullableDatum values[MAX_BLOOM_FILTER_COLUMNS]; for (int i = 0; i < num_fields; i++) values[i].value = GetAttributeByNum(tuple, i + 1, &values[i].isnull); uint64 hash = hasher->hash_values(hasher, values); PG_RETURN_INT64((int64) hash); } #endif // #ifndef NDEBUG char const *bloom1_column_prefix = NULL; /* this function will be reused in a later PR when we push down pre-calculated * hash value checks from the planner */ bool bloom1_contains_hash(Datum bloom_datum, uint64 hash) { struct varlena *bloom = DatumGetByteaPP(bloom_datum); if (bloom == NULL) return true; /* No bloom = might match */ const char *words_buf = VARDATA_ANY(bloom); const uint32 num_bits = 8 * VARSIZE_ANY_EXHDR(bloom); /* Validate bloom structure */ CheckCompressedData(num_bits == (1ULL << pg_leftmost_one_pos32(num_bits))); CheckCompressedData(num_bits >= 64); return bloom1_contains_hash_internal(words_buf, num_bits, hash); } ================================================ FILE: tsl/src/compression/batch_metadata_builder_minmax.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <libpq/pqformat.h> #include <utils/builtins.h> #include <utils/datum.h> #include <utils/sortsupport.h> #include <utils/typcache.h> #include "batch_metadata_builder_minmax.h" #include "compression.h" static void minmax_update_row(void *builder_, TupleTableSlot *slot); static void minmax_insert_to_compressed_row(void *builder_, RowCompressor *compressor); static void minmax_reset(void *builder_, RowCompressor *compressor); BatchMetadataBuilder * batch_metadata_builder_minmax_create(Oid type_oid, Oid collation, AttrNumber attnum, int min_attr_offset, int max_attr_offset) { BatchMetadataBuilderMinMax *builder = palloc(sizeof(*builder)); TypeCacheEntry *type = lookup_type_cache(type_oid, TYPECACHE_LT_OPR); if (!OidIsValid(type->lt_opr)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_FUNCTION), errmsg("could not identify an less-than operator for type %s", format_type_be(type_oid)))); *builder = (BatchMetadataBuilderMinMax){ .functions = (BatchMetadataBuilder){ .update_row = minmax_update_row, .insert_to_compressed_row = minmax_insert_to_compressed_row, .reset = minmax_reset, .builder_type = METADATA_BUILDER_MINMAX, }, .type_oid = type_oid, .attnum = attnum, .empty = true, .has_null = false, .type_by_val = type->typbyval, .type_len = type->typlen, .min_metadata_attr_offset = min_attr_offset, .max_metadata_attr_offset = max_attr_offset, }; builder->ssup.ssup_cxt = CurrentMemoryContext; builder->ssup.ssup_collation = collation; builder->ssup.ssup_nulls_first = false; PrepareSortSupportFromOrderingOp(type->lt_opr, &builder->ssup); return &builder->functions; } void minmax_update_row(void *builder_, TupleTableSlot *slot) { BatchMetadataBuilderMinMax *builder = (BatchMetadataBuilderMinMax *) builder_; Assert(builder->functions.builder_type == METADATA_BUILDER_MINMAX); bool is_null; Datum val = slot_getattr(slot, builder->attnum, &is_null); if (is_null) { builder->has_null = true; return; } int cmp; if (builder->empty) { builder->min = datumCopy(val, builder->type_by_val, builder->type_len); builder->max = datumCopy(val, builder->type_by_val, builder->type_len); builder->empty = false; return; } cmp = ApplySortComparator(builder->min, false, val, false, &builder->ssup); if (cmp > 0) { if (!builder->type_by_val) pfree(DatumGetPointer(builder->min)); builder->min = datumCopy(val, builder->type_by_val, builder->type_len); } cmp = ApplySortComparator(builder->max, false, val, false, &builder->ssup); if (cmp < 0) { if (!builder->type_by_val) pfree(DatumGetPointer(builder->max)); builder->max = datumCopy(val, builder->type_by_val, builder->type_len); } } static void minmax_reset(void *builder_, RowCompressor *compressor) { BatchMetadataBuilderMinMax *builder = (BatchMetadataBuilderMinMax *) builder_; if (!builder->empty) { if (!builder->type_by_val) { pfree(DatumGetPointer(builder->min)); pfree(DatumGetPointer(builder->max)); } builder->min = 0; builder->max = 0; } builder->empty = true; builder->has_null = false; compressor->compressed_is_null[builder->max_metadata_attr_offset] = true; compressor->compressed_is_null[builder->min_metadata_attr_offset] = true; compressor->compressed_values[builder->min_metadata_attr_offset] = 0; compressor->compressed_values[builder->max_metadata_attr_offset] = 0; } Datum batch_metadata_builder_minmax_min(void *builder_) { BatchMetadataBuilderMinMax *builder = (BatchMetadataBuilderMinMax *) builder_; if (builder->empty) elog(ERROR, "trying to get min from an empty builder"); if (builder->type_len == -1) { Datum unpacked = PointerGetDatum(PG_DETOAST_DATUM_PACKED(builder->min)); if (builder->min != unpacked) pfree(DatumGetPointer(builder->min)); builder->min = unpacked; } return builder->min; } Datum batch_metadata_builder_minmax_max(void *builder_) { BatchMetadataBuilderMinMax *builder = (BatchMetadataBuilderMinMax *) builder_; if (builder->empty) elog(ERROR, "trying to get max from an empty builder"); if (builder->type_len == -1) { Datum unpacked = PointerGetDatum(PG_DETOAST_DATUM_PACKED(builder->max)); if (builder->max != unpacked) pfree(DatumGetPointer(builder->max)); builder->max = unpacked; } return builder->max; } bool batch_metadata_builder_minmax_empty(void *builder_) { BatchMetadataBuilderMinMax *builder = (BatchMetadataBuilderMinMax *) builder_; return builder->empty; } static void minmax_insert_to_compressed_row(void *builder_, RowCompressor *compressor) { BatchMetadataBuilderMinMax *builder = (BatchMetadataBuilderMinMax *) builder_; Assert(builder->min_metadata_attr_offset >= 0); Assert(builder->max_metadata_attr_offset >= 0); if (!batch_metadata_builder_minmax_empty(builder)) { compressor->compressed_is_null[builder->min_metadata_attr_offset] = false; compressor->compressed_is_null[builder->max_metadata_attr_offset] = false; compressor->compressed_values[builder->min_metadata_attr_offset] = batch_metadata_builder_minmax_min(builder); compressor->compressed_values[builder->max_metadata_attr_offset] = batch_metadata_builder_minmax_max(builder); } else { compressor->compressed_is_null[builder->min_metadata_attr_offset] = true; compressor->compressed_is_null[builder->max_metadata_attr_offset] = true; } } ================================================ FILE: tsl/src/compression/batch_metadata_builder_minmax.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <fmgr.h> #include <lib/stringinfo.h> #include <utils/sortsupport.h> #include "batch_metadata_builder.h" typedef struct BatchMetadataBuilderMinMax { BatchMetadataBuilder functions; Oid type_oid; AttrNumber attnum; bool empty; bool has_null; SortSupportData ssup; bool type_by_val; int16 type_len; Datum min; Datum max; int16 min_metadata_attr_offset; int16 max_metadata_attr_offset; } BatchMetadataBuilderMinMax; typedef struct BatchMetadataBuilderMinMax BatchMetadataBuilderMinMax; typedef struct RowCompressor RowCompressor; /* * This is exposed only for the old unit tests. Ideally they should be replaced * with functional tests inspecting the compressed chunk table, and this * test-only interface should be removed. */ Datum batch_metadata_builder_minmax_min(void *builder_); Datum batch_metadata_builder_minmax_max(void *builder_); bool batch_metadata_builder_minmax_empty(void *builder_); ================================================ FILE: tsl/src/compression/city_combine.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* The code below is taken from the CityHash project: https://github.com/google/cityhash * specifically from here: https://github.com/google/cityhash/blob/master/src/city.h#L101 */ /* * Copyright (c) 2011 Google, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. * * CityHash, by Geoff Pike and Jyrki Alakuijala * * http://code.google.com/p/cityhash/ */ #pragma once #include <stdint.h> static inline uint64_t city_hash_combine(uint64_t accumulated_hash, uint64_t new_hash) { const uint64_t kMul = 0x9ddfea08eb382d69ULL; uint64_t a = (accumulated_hash ^ new_hash) * kMul; a ^= (a >> 47); uint64_t b = (new_hash ^ a) * kMul; b ^= (b >> 47); b *= kMul; return b; } ================================================ FILE: tsl/src/compression/compression.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/attmap.h> #include <access/attnum.h> #include <access/detoast.h> #include <access/skey.h> #include <access/tupdesc.h> #include <catalog/heap.h> #include <catalog/indexing.h> #include <catalog/pg_am.h> #include <common/base64.h> #include <funcapi.h> #include <libpq/pqformat.h> #include <storage/predicate.h> #include <utils/datum.h> #include <utils/elog.h> #include <utils/lsyscache.h> #include <utils/palloc.h> #include <utils/rel.h> #include <utils/snapmgr.h> #include <utils/syscache.h> #include <utils/typcache.h> #include "compat/compat.h" #include "algorithms/array.h" #include "algorithms/bool_compress.h" #include "algorithms/deltadelta.h" #include "algorithms/dictionary.h" #include "algorithms/gorilla.h" #include "algorithms/null.h" #include "algorithms/uuid_compress.h" #include "batch_metadata_builder.h" #include "chunk_insert_state.h" #include "compression.h" #include "compression/sparse_index_bloom1.h" #include "continuous_aggs/insert.h" #include "create.h" #include "custom_type_cache.h" #include "debug_assert.h" #include "debug_point.h" #include "guc.h" #include "nodes/modify_hypertable.h" #include "ts_catalog/array_utils.h" #include "ts_catalog/catalog.h" #include "ts_catalog/compression_settings.h" #include <nodes/columnar_scan/vector_quals.h> /* * Timing parameters for truncate locking heuristics. * These are the same as used by Postgres for truncate locking during lazy vacuum. * https://github.com/postgres/postgres/blob/4a0650d359c5981270039eeb634c3b7427aa0af5/src/backend/access/heap/vacuumlazy.c#L82 */ #define COMPRESS_TRUNCATE_LOCK_WAIT_INTERVAL 50 /* ms */ #define COMPRESS_TRUNCATE_LOCK_TIMEOUT 5000 /* ms */ StaticAssertDecl(GLOBAL_MAX_ROWS_PER_COMPRESSION >= TARGET_COMPRESSED_BATCH_SIZE, "max row numbers must be harmonized"); StaticAssertDecl(GLOBAL_MAX_ROWS_PER_COMPRESSION <= INT16_MAX, "dictionary compression uses signed int16 indexes"); static const CompressionAlgorithmDefinition definitions[_END_COMPRESSION_ALGORITHMS] = { [COMPRESSION_ALGORITHM_ARRAY] = ARRAY_ALGORITHM_DEFINITION, [COMPRESSION_ALGORITHM_DICTIONARY] = DICTIONARY_ALGORITHM_DEFINITION, [COMPRESSION_ALGORITHM_GORILLA] = GORILLA_ALGORITHM_DEFINITION, [COMPRESSION_ALGORITHM_DELTADELTA] = DELTA_DELTA_ALGORITHM_DEFINITION, [COMPRESSION_ALGORITHM_BOOL] = BOOL_COMPRESS_ALGORITHM_DEFINITION, [COMPRESSION_ALGORITHM_NULL] = NULL_COMPRESS_ALGORITHM_DEFINITION, [COMPRESSION_ALGORITHM_UUID] = UUID_COMPRESS_ALGORITHM_DEFINITION, }; static NameData compression_algorithm_name[] = { [_INVALID_COMPRESSION_ALGORITHM] = { "INVALID" }, [COMPRESSION_ALGORITHM_ARRAY] = { "ARRAY" }, [COMPRESSION_ALGORITHM_DICTIONARY] = { "DICTIONARY" }, [COMPRESSION_ALGORITHM_GORILLA] = { "GORILLA" }, [COMPRESSION_ALGORITHM_DELTADELTA] = { "DELTADELTA" }, [COMPRESSION_ALGORITHM_BOOL] = { "BOOL" }, [COMPRESSION_ALGORITHM_NULL] = { "NULL" }, [COMPRESSION_ALGORITHM_UUID] = { "UUID" }, }; Name compression_get_algorithm_name(CompressionAlgorithm alg) { return &compression_algorithm_name[alg]; } static Compressor * compressor_for_type(Oid type) { CompressionAlgorithm algorithm = compression_get_default_algorithm(type); if (algorithm >= _END_COMPRESSION_ALGORITHMS) elog(ERROR, "invalid compression algorithm %d", algorithm); return definitions[algorithm].compressor_for_type(type); } DecompressionInitializer tsl_get_decompression_iterator_init(CompressionAlgorithm algorithm, bool reverse) { if (algorithm >= _END_COMPRESSION_ALGORITHMS) elog(ERROR, "invalid compression algorithm %d", algorithm); if (reverse) return definitions[algorithm].iterator_init_reverse; else return definitions[algorithm].iterator_init_forward; } DecompressAllFunction tsl_get_decompress_all_function(CompressionAlgorithm algorithm, Oid type) { if (algorithm >= _END_COMPRESSION_ALGORITHMS) elog(ERROR, "invalid compression algorithm %d", algorithm); if (type != TEXTOID && type != BOOLOID && type != UUIDOID && (algorithm == COMPRESSION_ALGORITHM_DICTIONARY || algorithm == COMPRESSION_ALGORITHM_ARRAY)) { /* Bulk decompression of array and dictionary is only supported for * text, bool and uuid */ return NULL; } return definitions[algorithm].decompress_all; } static Tuplesortstate *compress_chunk_sort_relation(CompressionSettings *settings, Relation in_rel); static void row_compressor_process_ordered_slot(RowCompressor *row_compressor, TupleTableSlot *slot, BulkWriter *writer); static void row_compressor_update_group(RowCompressor *row_compressor, TupleTableSlot *row); static bool row_compressor_new_row_is_in_new_group(RowCompressor *row_compressor, TupleTableSlot *row); static void create_per_compressed_column(RowDecompressor *decompressor); static void row_compressor_append_row(RowCompressor *row_compressor, TupleTableSlot *row); static void row_compressor_flush(RowCompressor *row_compressor, BulkWriter *writer, bool changed_groups); /******************** ** compress_chunk ** ********************/ static CompressedDataHeader * get_compressed_data_header(Datum data) { CompressedDataHeader *header = (CompressedDataHeader *) PG_DETOAST_DATUM(data); if (header->compression_algorithm >= _END_COMPRESSION_ALGORITHMS) elog(ERROR, "invalid compression algorithm %d", header->compression_algorithm); return header; } /* Truncate the relation WITHOUT applying triggers. This is the * main difference with ExecuteTruncate. Triggers aren't applied * because the data remains, just in compressed form. Also don't * restart sequences. Use the transactional branch through ExecuteTruncate. */ static void truncate_relation(Oid table_oid) { List *fks = heap_truncate_find_FKs(list_make1_oid(table_oid)); /* Take an access exclusive lock now. Note that this may very well * be a lock upgrade. */ Relation rel = table_open(table_oid, AccessExclusiveLock); Oid toast_relid; /* Chunks should never have fks into them, but double check */ if (fks != NIL) elog(ERROR, "found a FK into a chunk while truncating"); CheckTableForSerializableConflictIn(rel); #if PG16_LT RelationSetNewRelfilenode(rel, rel->rd_rel->relpersistence); #else RelationSetNewRelfilenumber(rel, rel->rd_rel->relpersistence); #endif toast_relid = rel->rd_rel->reltoastrelid; table_close(rel, NoLock); if (OidIsValid(toast_relid)) { rel = table_open(toast_relid, AccessExclusiveLock); #if PG16_LT RelationSetNewRelfilenode(rel, rel->rd_rel->relpersistence); #else RelationSetNewRelfilenumber(rel, rel->rd_rel->relpersistence); #endif table_close(rel, NoLock); } ReindexParams params = { 0 }; ReindexParams *options = ¶ms; reindex_relation_compat(NULL, table_oid, REINDEX_REL_PROCESS_TOAST, options); rel = table_open(table_oid, AccessExclusiveLock); CommandCounterIncrement(); table_close(rel, NoLock); } /* Handle the all rows deletion of a given relation */ static void RelationDeleteAllRows(Relation rel, Snapshot snap) { TupleTableSlot *slot = table_slot_create(rel, NULL); TableScanDesc scan = table_beginscan(rel, snap, 0, NULL); while (table_scan_getnextslot(scan, ForwardScanDirection, slot)) { simple_table_tuple_delete(rel, &(slot->tts_tid), snap); } table_endscan(scan); ExecDropSingleTupleTableSlot(slot); } /* * Delete the relation WITHOUT applying triggers. This will be used when * `enable_delete_after_compression = true` instead of truncating the relation. * Also don't restart sequences. */ static void delete_relation_rows(Oid table_oid) { Relation rel = table_open(table_oid, RowExclusiveLock); Snapshot snap = RegisterSnapshot(GetLatestSnapshot()); /* Delete the rows in the table */ RelationDeleteAllRows(rel, snap); /* Delete the rows in the toast table */ if (OidIsValid(rel->rd_rel->reltoastrelid)) { Relation toast_rel = table_open(rel->rd_rel->reltoastrelid, RowExclusiveLock); RelationDeleteAllRows(toast_rel, snap); table_close(toast_rel, NoLock); } table_close(rel, NoLock); UnregisterSnapshot(snap); } /* * Use reltuples as an estimate for the number of rows that will get compressed. This value * might be way off the mark in case analyze hasn't happened in quite a while on this input * chunk. But that's the best guesstimate to start off with. * * We will report progress for every 10% of reltuples compressed. If rel or reltuples is not valid * or it's just too low then we just assume reporting every 100K tuples for now. */ #define RELTUPLES_REPORT_DEFAULT 100000 static int64 calculate_reltuples_to_report(float4 reltuples) { int64 report_reltuples = RELTUPLES_REPORT_DEFAULT; if (reltuples > 0) { report_reltuples = (int64) (0.1 * reltuples); /* either analyze has not been done or table doesn't have a lot of rows */ if (report_reltuples < RELTUPLES_REPORT_DEFAULT) report_reltuples = RELTUPLES_REPORT_DEFAULT; } return report_reltuples; } CompressionStats compress_chunk(Oid in_table, Oid out_table, int insert_options) { int n_keys; ListCell *lc; ScanDirection indexscan_direction = NoMovementScanDirection; Relation matched_index_rel = NULL; TupleTableSlot *slot; IndexScanDesc index_scan; HeapTuple in_table_tp = NULL, index_tp = NULL; Form_pg_attribute in_table_attr_tp, index_attr_tp; CompressionStats cstat; CompressionSettings *settings = ts_compression_settings_get_by_compress_relid(out_table); int64 report_reltuples; /* We want to prevent other compressors from compressing this table, * and we want to prevent INSERTs or UPDATEs which could mess up our compression. * We may as well allow readers to keep reading the uncompressed data while * we are compressing, so we only take an ExclusiveLock instead of AccessExclusive. */ Relation in_rel = table_open(in_table, ExclusiveLock); /* We are _just_ INSERTing into the out_table so in principle we could take * a RowExclusive lock, and let other operations read and write this table * as we work. However, we currently compress each table as a oneshot, so * we're taking the stricter lock to prevent accidents. * * Putting RowExclusiveMode behind a GUC so we can try this out with * rollups during compression. */ int out_rel_lockmode = ExclusiveLock; if (ts_guc_enable_rowlevel_compression_locking) { out_rel_lockmode = RowExclusiveLock; } Relation out_rel = relation_open(out_table, out_rel_lockmode); BulkWriter writer = bulk_writer_build(out_rel, insert_options); /* Sanity check we are dealing with relations */ Ensure(in_rel->rd_rel->relkind == RELKIND_RELATION, "compress_chunk called on non-relation"); Ensure(out_rel->rd_rel->relkind == RELKIND_RELATION, "compress_chunk called on non-relation"); PushActiveSnapshot(GetTransactionSnapshot()); /* Before calling row compressor relation should be segmented and sorted as configured * by compress_segmentby and compress_orderby. * Cost of sorting can be mitigated if we find an existing BTREE index defined for * uncompressed chunk otherwise expensive tuplesort will come into play. * * The following code is trying to find an existing index that * matches the configuration so that we can skip sequential scan and * tuplesort. * */ if (ts_guc_enable_compression_indexscan) { List *in_rel_index_oids = RelationGetIndexList(in_rel); foreach (lc, in_rel_index_oids) { Oid index_oid = lfirst_oid(lc); Relation index_rel = index_open(index_oid, AccessShareLock); IndexInfo *index_info = BuildIndexInfo(index_rel); if (index_info->ii_Predicate != 0) { /* * Can't use partial indexes for compression because they refer * only to a subset of all rows. */ index_close(index_rel, AccessShareLock); continue; } int previous_direction = NoMovementScanDirection; int current_direction = NoMovementScanDirection; n_keys = ts_array_length(settings->fd.segmentby) + ts_array_length(settings->fd.orderby); if (n_keys <= index_info->ii_NumIndexKeyAttrs && index_info->ii_Am == BTREE_AM_OID) { int i; for (i = 0; i < n_keys; i++) { const char *attname; int16 position; bool is_orderby_asc = true; bool is_null_first = false; if (i < ts_array_length(settings->fd.segmentby)) { position = i + 1; attname = ts_array_get_element_text(settings->fd.segmentby, position); } else { position = i - ts_array_length(settings->fd.segmentby) + 1; attname = ts_array_get_element_text(settings->fd.orderby, position); is_orderby_asc = !ts_array_get_element_bool(settings->fd.orderby_desc, position); is_null_first = ts_array_get_element_bool(settings->fd.orderby_nullsfirst, position); } int16 att_num = get_attnum(in_table, attname); int16 option = index_rel->rd_indoption[i]; bool index_orderby_asc = ((option & INDOPTION_DESC) == 0); bool index_null_first = ((option & INDOPTION_NULLS_FIRST) != 0); if (att_num == 0 || index_info->ii_IndexAttrNumbers[i] != att_num) { break; } in_table_tp = SearchSysCacheAttNum(in_table, att_num); if (!HeapTupleIsValid(in_table_tp)) elog(ERROR, "table \"%s\" does not have column \"%s\"", get_rel_name(in_table), attname); index_tp = SearchSysCacheAttNum(index_oid, i + 1); if (!HeapTupleIsValid(index_tp)) elog(ERROR, "index \"%s\" does not have column \"%s\"", get_rel_name(index_oid), attname); in_table_attr_tp = (Form_pg_attribute) GETSTRUCT(in_table_tp); index_attr_tp = (Form_pg_attribute) GETSTRUCT(index_tp); if (index_orderby_asc == is_orderby_asc && index_null_first == is_null_first && in_table_attr_tp->attcollation == index_attr_tp->attcollation) { current_direction = ForwardScanDirection; } else if (index_orderby_asc != is_orderby_asc && index_null_first != is_null_first && in_table_attr_tp->attcollation == index_attr_tp->attcollation) { current_direction = BackwardScanDirection; } else { current_direction = NoMovementScanDirection; break; } ReleaseSysCache(in_table_tp); in_table_tp = NULL; ReleaseSysCache(index_tp); index_tp = NULL; if (previous_direction == NoMovementScanDirection) { previous_direction = current_direction; } else if (previous_direction != current_direction) { break; } } if (n_keys == i && (previous_direction == current_direction && current_direction != NoMovementScanDirection)) { matched_index_rel = index_rel; indexscan_direction = current_direction; break; } else { if (HeapTupleIsValid(in_table_tp)) { ReleaseSysCache(in_table_tp); in_table_tp = NULL; } if (HeapTupleIsValid(index_tp)) { ReleaseSysCache(index_tp); index_tp = NULL; } index_close(index_rel, AccessShareLock); } } else { index_close(index_rel, AccessShareLock); } } } RowCompressor row_compressor; Assert(settings->fd.compress_relid == RelationGetRelid(out_rel)); row_compressor_init(&row_compressor, settings, RelationGetDescr(in_rel), RelationGetDescr(out_rel)); if (matched_index_rel != NULL) { int64 nrows_processed = 0; elog(ts_guc_debug_compression_path_info ? INFO : DEBUG1, "using index \"%s\" to scan rows for converting to columnstore", get_rel_name(matched_index_rel->rd_id)); index_scan = index_beginscan_compat(in_rel, matched_index_rel, GetActiveSnapshot(), NULL, 0, 0); slot = table_slot_create(in_rel, NULL); index_rescan(index_scan, NULL, 0, NULL, 0); report_reltuples = calculate_reltuples_to_report(in_rel->rd_rel->reltuples); while (index_getnext_slot(index_scan, indexscan_direction, slot)) { row_compressor_process_ordered_slot(&row_compressor, slot, &writer); if ((++nrows_processed % report_reltuples) == 0) elog(DEBUG2, "converted " INT64_FORMAT " rows to columnstore from \"%s\"", nrows_processed, RelationGetRelationName(in_rel)); } if (row_compressor.rows_compressed_into_current_value > 0) row_compressor_flush(&row_compressor, &writer, true); elog(DEBUG1, "finished converting " INT64_FORMAT " rows to columnstore from \"%s\"", nrows_processed, RelationGetRelationName(in_rel)); ExecDropSingleTupleTableSlot(slot); index_endscan(index_scan); index_close(matched_index_rel, AccessShareLock); } else { elog(ts_guc_debug_compression_path_info ? INFO : DEBUG1, "using tuplesort to scan rows from \"%s\" for converting to columnstore", RelationGetRelationName(in_rel)); Tuplesortstate *sorted_rel = compress_chunk_sort_relation(settings, in_rel); row_compressor_append_sorted_rows(&row_compressor, sorted_rel, in_rel, &writer); tuplesort_end(sorted_rel); } row_compressor_close(&row_compressor); bulk_writer_close(&writer); if (ts_guc_enable_delete_after_compression) { ereport(NOTICE, (errcode(ERRCODE_WARNING_DEPRECATED_FEATURE), errmsg("timescaledb.enable_delete_after_compression is deprecated and will be " "removed in a future version. Please use " "timescaledb.compress_truncate_behaviour instead."))); delete_relation_rows(in_table); DEBUG_WAITPOINT("compression_done_after_delete_uncompressed"); } else { int lock_retry = 0; switch (ts_guc_compress_truncate_behaviour) { case COMPRESS_TRUNCATE_ONLY: DEBUG_WAITPOINT("compression_done_before_truncate_uncompressed"); truncate_relation(in_table); DEBUG_WAITPOINT("compression_done_after_truncate_uncompressed"); break; case COMPRESS_TRUNCATE_OR_DELETE: DEBUG_WAITPOINT("compression_done_before_truncate_or_delete_uncompressed"); while (true) { if (ConditionalLockRelation(in_rel, AccessExclusiveLock)) { truncate_relation(in_table); break; } /* * Check for interrupts while trying to (re-)acquire the exclusive * lock. */ CHECK_FOR_INTERRUPTS(); if (++lock_retry > (COMPRESS_TRUNCATE_LOCK_TIMEOUT / COMPRESS_TRUNCATE_LOCK_WAIT_INTERVAL)) { /* * We failed to establish the lock in the specified number of * retries. This means we give up truncating and fallback to delete */ delete_relation_rows(in_table); break; } (void) WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, COMPRESS_TRUNCATE_LOCK_WAIT_INTERVAL, WAIT_EVENT_VACUUM_TRUNCATE); ResetLatch(MyLatch); } DEBUG_WAITPOINT("compression_done_after_truncate_or_delete_uncompressed"); break; case COMPRESS_TRUNCATE_DISABLED: delete_relation_rows(in_table); DEBUG_WAITPOINT("compression_done_after_delete_uncompressed"); break; } } table_close(out_rel, NoLock); table_close(in_rel, NoLock); PopActiveSnapshot(); cstat.rowcnt_pre_compression = row_compressor.rowcnt_pre_compression; cstat.rowcnt_post_compression = row_compressor.num_compressed_rows; if ((insert_options & HEAP_INSERT_FROZEN) == HEAP_INSERT_FROZEN) cstat.rowcnt_frozen = row_compressor.num_compressed_rows; else cstat.rowcnt_frozen = 0; return cstat; } Tuplesortstate * compression_create_tuplesort_state(CompressionSettings *settings, Relation rel) { TupleDesc tupdesc = RelationGetDescr(rel); int num_segmentby = ts_array_length(settings->fd.segmentby); int num_orderby = ts_array_length(settings->fd.orderby); int n_keys = num_segmentby + num_orderby; AttrNumber *sort_keys = palloc(sizeof(*sort_keys) * n_keys); Oid *sort_operators = palloc(sizeof(*sort_operators) * n_keys); Oid *sort_collations = palloc(sizeof(*sort_collations) * n_keys); bool *nulls_first = palloc(sizeof(*nulls_first) * n_keys); int n; for (n = 0; n < n_keys; n++) { const char *attname; int position; if (n < num_segmentby) { position = n + 1; attname = ts_array_get_element_text(settings->fd.segmentby, position); } else { position = n - num_segmentby + 1; attname = ts_array_get_element_text(settings->fd.orderby, position); } compress_chunk_populate_sort_info_for_column(settings, RelationGetRelid(rel), attname, &sort_keys[n], &sort_operators[n], &sort_collations[n], &nulls_first[n]); } /* Make a copy of the tuple descriptor so that it is allocated on the same * memory context as the tuple sort instead of pointing into the relcache * entry that could be blown away. */ return tuplesort_begin_heap(CreateTupleDescCopy(tupdesc), n_keys, sort_keys, sort_operators, sort_collations, nulls_first, maintenance_work_mem, NULL, false /*=randomAccess*/); } static Tuplesortstate * compress_chunk_sort_relation(CompressionSettings *settings, Relation in_rel) { PushActiveSnapshot(GetLatestSnapshot()); Tuplesortstate *tuplesortstate; TableScanDesc scan; TupleTableSlot *slot; tuplesortstate = compression_create_tuplesort_state(settings, in_rel); scan = table_beginscan(in_rel, GetActiveSnapshot(), 0, NULL); slot = table_slot_create(in_rel, NULL); while (table_scan_getnextslot(scan, ForwardScanDirection, slot)) { if (!TTS_EMPTY(slot)) { /* This may not be the most efficient way to do things. * Since we use begin_heap() the tuplestore expects tupleslots, * so ISTM that the options are this or maybe putdatum(). */ tuplesort_puttupleslot(tuplesortstate, slot); } } table_endscan(scan); ExecDropSingleTupleTableSlot(slot); tuplesort_performsort(tuplesortstate); PopActiveSnapshot(); return tuplesortstate; } void compress_chunk_populate_sort_info_for_column(const CompressionSettings *settings, Oid table, const char *attname, AttrNumber *att_nums, Oid *sort_operator, Oid *collation, bool *nulls_first) { HeapTuple tp; Form_pg_attribute att_tup; TypeCacheEntry *tentry; tp = SearchSysCacheAttName(table, attname); if (!HeapTupleIsValid(tp)) elog(ERROR, "table \"%s\" does not have column \"%s\"", get_rel_name(table), attname); att_tup = (Form_pg_attribute) GETSTRUCT(tp); /* Other validation checks beyond just existence of a valid comparison operator could be useful */ *att_nums = att_tup->attnum; *collation = att_tup->attcollation; tentry = lookup_type_cache(att_tup->atttypid, TYPECACHE_LT_OPR | TYPECACHE_GT_OPR); if (ts_array_is_member(settings->fd.segmentby, attname)) { *nulls_first = false; *sort_operator = tentry->lt_opr; } else { Assert(ts_array_is_member(settings->fd.orderby, attname)); int position = ts_array_position(settings->fd.orderby, attname); *nulls_first = ts_array_get_element_bool(settings->fd.orderby_nullsfirst, position); if (ts_array_get_element_bool(settings->fd.orderby_desc, position)) *sort_operator = tentry->gt_opr; else *sort_operator = tentry->lt_opr; } if (!OidIsValid(*sort_operator)) elog(ERROR, "no valid sort operator for column \"%s\" of type \"%s\"", attname, format_type_be(att_tup->atttypid)); ReleaseSysCache(tp); } /* * Find segment by index on compressed chunk needed when doing index scans * over compressed data */ Oid get_compressed_chunk_index(ResultRelInfo *resultRelInfo, const CompressionSettings *settings) { int num_segmentby_columns = ts_array_length(settings->fd.segmentby); int num_orderby_columns = ts_array_length(settings->fd.orderby); for (int i = 0; i < resultRelInfo->ri_NumIndices; i++) { bool matches = true; Relation index_relation = resultRelInfo->ri_IndexRelationDescs[i]; IndexInfo *index_info = resultRelInfo->ri_IndexRelationInfo[i]; /* The index must include all segment by columns and at least two metadata columns. * Default index we build includes all segmentby columns and metadata columns (min and max, * in that order) for all orderby columns.*/ if (index_info->ii_NumIndexKeyAttrs != num_segmentby_columns + (num_orderby_columns * 2)) continue; for (int j = 0; j < num_segmentby_columns - 1; j++) { AttrNumber attno = index_relation->rd_index->indkey.values[j]; const char *attname = get_attname(index_relation->rd_index->indrelid, attno, false); if (!ts_array_is_member(settings->fd.segmentby, attname)) { matches = false; break; } } if (!matches) continue; return RelationGetRelid(index_relation); } return InvalidOid; } static void build_column_map(const CompressionSettings *settings, const TupleDesc in_desc, const TupleDesc out_desc, PerColumn **pcolumns, int16 **pmap, List **pmetadata_builders) { Oid compressed_data_type_oid = ts_custom_type_cache_get(CUSTOM_TYPE_COMPRESSED_DATA)->type_oid; PerColumn *columns = palloc0(sizeof(PerColumn) * in_desc->natts); int16 *map = palloc0(sizeof(int16) * in_desc->natts); List *metadata_builders = NIL; SparseIndexSettings *parsed_settings = NULL; if (settings && settings->fd.index) parsed_settings = ts_convert_to_sparse_index_settings(settings->fd.index); if (parsed_settings != NULL && ts_guc_enable_composite_bloom_indexes) { ListCell *lc; foreach (lc, parsed_settings->objects) { SparseIndexSettingsObject *obj = lfirst(lc); List *column_names = ts_get_column_names_from_parsed_object(obj); int num_columns = list_length(column_names); if (num_columns < 2) continue; Oid type_oids[MAX_BLOOM_FILTER_COLUMNS]; AttrNumber attnums[MAX_BLOOM_FILTER_COLUMNS]; int col_idx = 0; ListCell *name_cell; foreach (name_cell, column_names) { const char *col_name = (const char *) lfirst(name_cell); AttrNumber attnum = get_attnum(settings->fd.relid, col_name); Ensure(AttributeNumberIsValid(attnum), "could not find column '%s'", col_name); attnums[col_idx] = attnum; type_oids[col_idx] = get_atttype(settings->fd.relid, attnum); col_idx++; } const char *bloom_col_name = compressed_column_metadata_name_list_v2(bloom1_column_prefix, column_names); AttrNumber bloom_attr_number = get_attnum(settings->fd.compress_relid, bloom_col_name); if (!AttributeNumberIsValid(bloom_attr_number)) continue; int bloom_attr_offset = AttrNumberGetAttrOffset(bloom_attr_number); metadata_builders = lappend(metadata_builders, batch_metadata_builder_bloom1_create(num_columns, type_oids, attnums, bloom_attr_offset)); } } ts_free_sparse_index_settings(parsed_settings); if (settings != NULL && OidIsValid(settings->fd.compress_relid)) { for (int i = 0; i < in_desc->natts; i++) { Form_pg_attribute attr = TupleDescAttr(in_desc, i); if (attr->attisdropped) continue; PerColumn *column = &columns[AttrNumberGetAttrOffset(attr->attnum)]; AttrNumber compressed_colnum = get_attnum(settings->fd.compress_relid, NameStr(attr->attname)); Form_pg_attribute compressed_column_attr = TupleDescAttr(out_desc, AttrNumberGetAttrOffset(compressed_colnum)); map[AttrNumberGetAttrOffset(attr->attnum)] = AttrNumberGetAttrOffset(compressed_colnum); bool is_segmentby = ts_array_is_member(settings->fd.segmentby, NameStr(attr->attname)); bool is_orderby = ts_array_is_member(settings->fd.orderby, NameStr(attr->attname)); if (!is_segmentby) { if (compressed_column_attr->atttypid != compressed_data_type_oid) elog(ERROR, "expected column '%s' to be a compressed data type", NameStr(attr->attname)); AttrNumber segment_min_attr_number = compressed_column_metadata_attno(settings, settings->fd.relid, attr->attnum, settings->fd.compress_relid, "min"); AttrNumber segment_max_attr_number = compressed_column_metadata_attno(settings, settings->fd.relid, attr->attnum, settings->fd.compress_relid, "max"); int16 segment_min_attr_offset = segment_min_attr_number - 1; int16 segment_max_attr_offset = segment_max_attr_number - 1; bool has_minmax_metadata = false; if (segment_min_attr_number != InvalidAttrNumber || segment_max_attr_number != InvalidAttrNumber) { has_minmax_metadata = true; Ensure(segment_min_attr_number != InvalidAttrNumber, "could not find the min metadata column"); Ensure(segment_max_attr_number != InvalidAttrNumber, "could not find the min metadata column"); metadata_builders = lappend(metadata_builders, batch_metadata_builder_minmax_create(attr->atttypid, attr->attcollation, attr->attnum, segment_min_attr_offset, segment_max_attr_offset)); } Ensure(!is_orderby || has_minmax_metadata, "orderby columns must have minmax metadata"); const AttrNumber bloom_attr_number = compressed_column_metadata_attno(settings, settings->fd.relid, attr->attnum, settings->fd.compress_relid, bloom1_column_prefix); if (AttributeNumberIsValid(bloom_attr_number)) { Oid type_oid = attr->atttypid; AttrNumber attnum = attr->attnum; const int bloom_attr_offset = AttrNumberGetAttrOffset(bloom_attr_number); metadata_builders = lappend(metadata_builders, batch_metadata_builder_bloom1_create(1, &type_oid, &attnum, bloom_attr_offset)); } *column = (PerColumn){ .compressor = compressor_for_type(attr->atttypid), .segmentby_column_index = -1, }; } else { if (attr->atttypid != compressed_column_attr->atttypid) elog(ERROR, "expected segment by column \"%s\" to be same type as uncompressed column", NameStr(attr->attname)); int16 index = ts_array_position(settings->fd.segmentby, NameStr(attr->attname)); *column = (PerColumn){ .segment_info = segment_info_new(attr), .segmentby_column_index = index, }; } } } *pcolumns = columns; *pmap = map; *pmetadata_builders = metadata_builders; } /* Check if we contain any compressors which need allocation limit checking */ static bool check_for_limited_size_compressors(PerColumn *pcolumns, int16 natts) { for (int i = 0; i < natts; i++) { if (pcolumns[i].compressor && pcolumns[i].compressor->is_full) return true; } return false; } void tsl_compressor_set_invalidation(RowCompressor *compressor, Hypertable *ht, Oid chunk_relid) { const Dimension *time_dim = hyperspace_get_open_dimension(ht->space, 0); Ensure(time_dim, "Hypertable must have an open dimension"); AttrNumber attnum = get_attnum(chunk_relid, NameStr(time_dim->fd.column_name)); compressor->invalidation = palloc0(sizeof(InvalidationSettings)); compressor->invalidation->hypertable_id = ht->fd.id; compressor->invalidation->chunk_relid = chunk_relid; compressor->invalidation->invalidation_column_offset = AttrNumberGetAttrOffset(attnum); } void tsl_compressor_add_slot(RowCompressor *compressor, BulkWriter *bulk_writer, TupleTableSlot *slot) { if (compressor->sort_state) { tuplesort_puttupleslot(compressor->sort_state, slot); compressor->tuples_to_sort++; if (compressor->tuple_sort_limit && compressor->tuples_to_sort >= compressor->tuple_sort_limit) tsl_compressor_flush(compressor, bulk_writer); } else { row_compressor_process_ordered_slot(compressor, slot, bulk_writer); } } void tsl_compressor_flush(RowCompressor *compressor, BulkWriter *bulk_writer) { if (compressor->sort_state) { if (compressor->tuples_to_sort) { tuplesort_performsort(compressor->sort_state); TupleTableSlot *slot = MakeTupleTableSlot(compressor->in_desc, &TTSOpsMinimalTuple); while (tuplesort_gettupleslot(compressor->sort_state, true /*=forward*/, false /*=copy*/, slot, NULL /*=abbrev*/)) row_compressor_process_ordered_slot(compressor, slot, bulk_writer); if (compressor->rows_compressed_into_current_value > 0) row_compressor_flush(compressor, bulk_writer, true); ExecDropSingleTupleTableSlot(slot); tuplesort_reset(compressor->sort_state); compressor->tuples_to_sort = 0; } } else { if (compressor->rows_compressed_into_current_value > 0) row_compressor_flush(compressor, bulk_writer, false); } } void tsl_compressor_free(RowCompressor *compressor, BulkWriter *bulk_writer) { if (compressor->sort_state) tuplesort_end(compressor->sort_state); if (compressor->invalidation) pfree(compressor->invalidation); tsl_compressor_flush(compressor, bulk_writer); row_compressor_close(compressor); bulk_writer_close(bulk_writer); table_close(bulk_writer->out_rel, NoLock); } /* * Initialize a RowCompressor for compressing tuples * * When `sort` is true, the compressor will buffer all the tuples in a * Tuplesortstate and sort them before flushing to the output relation. */ RowCompressor * tsl_compressor_init(Relation in_rel, BulkWriter **bulk_writer, bool sort, int sort_limit) { RowCompressor *compressor = palloc0(sizeof(RowCompressor)); CompressionSettings *settings = ts_compression_settings_get(in_rel->rd_id); Relation out_rel = table_open(settings->fd.compress_relid, RowExclusiveLock); *bulk_writer = bulk_writer_alloc(out_rel, 0); row_compressor_init(compressor, settings, RelationGetDescr(in_rel), RelationGetDescr(out_rel)); if (sort) { compressor->sort_state = compression_create_tuplesort_state(settings, in_rel); compressor->tuple_sort_limit = sort_limit; } return compressor; } /******************** ** row_compressor ** ********************/ void row_compressor_init(RowCompressor *row_compressor, const CompressionSettings *settings, const TupleDesc noncompressed_tupdesc, const TupleDesc compressed_tupdesc) { Name count_metadata_name = DatumGetName( DirectFunctionCall1(namein, CStringGetDatum(COMPRESSION_COLUMN_METADATA_COUNT_NAME))); AttrNumber count_metadata_column_num = get_attnum(settings->fd.compress_relid, NameStr(*count_metadata_name)); if (count_metadata_column_num == InvalidAttrNumber) elog(ERROR, "missing metadata column '%s' in columnstore table", COMPRESSION_COLUMN_METADATA_COUNT_NAME); *row_compressor = (RowCompressor){ .per_row_ctx = AllocSetContextCreate(CurrentMemoryContext, "compress chunk per-row", ALLOCSET_DEFAULT_SIZES), .in_desc = CreateTupleDescCopyConstr(noncompressed_tupdesc), .out_desc = CreateTupleDescCopyConstr(compressed_tupdesc), .n_input_columns = noncompressed_tupdesc->natts, .count_metadata_column_offset = AttrNumberGetAttrOffset(count_metadata_column_num), .compressed_values = palloc(sizeof(Datum) * compressed_tupdesc->natts), .compressed_is_null = palloc(sizeof(bool) * compressed_tupdesc->natts), .rows_compressed_into_current_value = 0, .rowcnt_pre_compression = 0, .num_compressed_rows = 0, .first_iteration = true, .sort_state = NULL, }; memset(row_compressor->compressed_is_null, 1, sizeof(bool) * compressed_tupdesc->natts); build_column_map(settings, noncompressed_tupdesc, compressed_tupdesc, &row_compressor->per_column, &row_compressor->uncompressed_col_to_compressed_col, &row_compressor->metadata_builders); /* If we have dictionary or array compressors, we have to check compressor size so we don't end * up going over allocation limit */ row_compressor->needs_fullness_check = check_for_limited_size_compressors(row_compressor->per_column, row_compressor->n_input_columns); } void row_compressor_append_sorted_rows(RowCompressor *row_compressor, Tuplesortstate *sorted_rel, Relation in_rel, BulkWriter *writer) { TupleTableSlot *slot = MakeTupleTableSlot(row_compressor->in_desc, &TTSOpsMinimalTuple); int64 nrows_processed = 0; int64 report_reltuples; report_reltuples = calculate_reltuples_to_report(in_rel->rd_rel->reltuples); while (tuplesort_gettupleslot(sorted_rel, true /*=forward*/, false /*=copy*/, slot, NULL /*=abbrev*/)) { row_compressor_process_ordered_slot(row_compressor, slot, writer); if ((++nrows_processed % report_reltuples) == 0) elog(DEBUG2, "compressed " INT64_FORMAT " rows from \"%s\"", nrows_processed, RelationGetRelationName(in_rel)); } if (row_compressor->rows_compressed_into_current_value > 0) row_compressor_flush(row_compressor, writer, true); elog(DEBUG1, "finished compressing " INT64_FORMAT " rows from \"%s\"", nrows_processed, RelationGetRelationName(in_rel)); ExecDropSingleTupleTableSlot(slot); } static bool row_compressor_is_full(RowCompressor *row_compressor, TupleTableSlot *row) { if (row_compressor->rows_compressed_into_current_value >= (uint32) ts_guc_compression_batch_size_limit) return true; if (!ts_guc_compression_enable_compressor_batch_limit) return false; if (!row_compressor->needs_fullness_check) return false; /* Check with every column compressor if they can add the next value to current batch */ int col; for (col = 0; col < row_compressor->n_input_columns; col++) { Compressor *compressor = row_compressor->per_column[col].compressor; bool is_null; Datum val; /* No compressor or the compressor has no check, just skip */ if (compressor == NULL || compressor->is_full == NULL) continue; val = slot_getattr(row, AttrOffsetGetAttrNumber(col), &is_null); if (!is_null) { if (compressor->is_full(compressor, val)) return true; } } return false; } void row_compressor_append_ordered_slot(RowCompressor *row_compressor, TupleTableSlot *slot) { MemoryContext old_ctx; slot_getallattrs(slot); old_ctx = MemoryContextSwitchTo(row_compressor->per_row_ctx); if (row_compressor->first_iteration) { row_compressor_update_group(row_compressor, slot); row_compressor->first_iteration = false; } bool changed_groups = row_compressor_new_row_is_in_new_group(row_compressor, slot); bool compressed_row_is_full = row_compressor_is_full(row_compressor, slot); Ensure(!changed_groups, "row is in different group"); Ensure(!compressed_row_is_full, "batch is full"); row_compressor_append_row(row_compressor, slot); MemoryContextSwitchTo(old_ctx); } static void row_compressor_process_ordered_slot(RowCompressor *row_compressor, TupleTableSlot *slot, BulkWriter *writer) { MemoryContext old_ctx; slot_getallattrs(slot); old_ctx = MemoryContextSwitchTo(row_compressor->per_row_ctx); if (row_compressor->first_iteration) { row_compressor_update_group(row_compressor, slot); row_compressor->first_iteration = false; } bool changed_groups = row_compressor_new_row_is_in_new_group(row_compressor, slot); bool compressed_row_is_full = row_compressor_is_full(row_compressor, slot); if (compressed_row_is_full || changed_groups) { if (row_compressor->rows_compressed_into_current_value > 0) row_compressor_flush(row_compressor, writer, changed_groups); if (changed_groups) row_compressor_update_group(row_compressor, slot); } row_compressor_append_row(row_compressor, slot); MemoryContextSwitchTo(old_ctx); } static void row_compressor_update_group(RowCompressor *row_compressor, TupleTableSlot *row) { int col; /* save original memory context */ const MemoryContext oldcontext = CurrentMemoryContext; Assert(row_compressor->rows_compressed_into_current_value == 0); Assert(row_compressor->n_input_columns <= row->tts_nvalid); MemoryContextSwitchTo(row_compressor->per_row_ctx->parent); for (col = 0; col < row_compressor->n_input_columns; col++) { PerColumn *column = &row_compressor->per_column[col]; Datum val; bool is_null; if (column->segment_info == NULL) continue; Assert(column->compressor == NULL); /* Performance Improvement: We should just use array access here; everything is guaranteed to be fetched */ val = slot_getattr(row, AttrOffsetGetAttrNumber(col), &is_null); segment_info_update(column->segment_info, val, is_null); } /* switch to original memory context */ MemoryContextSwitchTo(oldcontext); } static bool row_compressor_new_row_is_in_new_group(RowCompressor *row_compressor, TupleTableSlot *row) { int col; for (col = 0; col < row_compressor->n_input_columns; col++) { PerColumn *column = &row_compressor->per_column[col]; Datum datum = CharGetDatum(0); bool is_null; if (column->segment_info == NULL) continue; Assert(column->compressor == NULL); datum = slot_getattr(row, AttrOffsetGetAttrNumber(col), &is_null); if (!segment_info_datum_is_in_group(column->segment_info, datum, is_null)) return true; } return false; } void row_compressor_append_row(RowCompressor *row_compressor, TupleTableSlot *row) { int col; for (col = 0; col < row_compressor->n_input_columns; col++) { Compressor *compressor = row_compressor->per_column[col].compressor; bool is_null; Datum val; /* if there is no compressor, this must be a segmenter, so just skip */ if (compressor == NULL) continue; /* Performance Improvement: Since we call getallatts at the beginning, slot_getattr is * useless overhead here, and we should just access the array directly. */ val = slot_getattr(row, AttrOffsetGetAttrNumber(col), &is_null); if (is_null) compressor->append_null(compressor); else compressor->append_val(compressor, val); } ListCell *lc; foreach (lc, row_compressor->metadata_builders) { BatchMetadataBuilder *builder = lfirst(lc); builder->update_row(builder, row); } row_compressor->rows_compressed_into_current_value += 1; } HeapTuple row_compressor_build_tuple(RowCompressor *row_compressor) { MemoryContext old_cxt = MemoryContextSwitchTo(row_compressor->per_row_ctx); for (int col = 0; col < row_compressor->n_input_columns; col++) { PerColumn *column = &row_compressor->per_column[col]; Compressor *compressor; int16 compressed_col; if (column->compressor == NULL && column->segment_info == NULL) continue; compressor = column->compressor; compressed_col = row_compressor->uncompressed_col_to_compressed_col[col]; Assert(compressed_col >= 0); if (compressor != NULL) { void *compressed_data; Assert(column->segment_info == NULL); compressed_data = compressor->finish(compressor); if (compressed_data == NULL) { if (ts_guc_enable_null_compression && row_compressor->rows_compressed_into_current_value > 0) compressed_data = null_compressor_get_dummy_block(); } row_compressor->compressed_is_null[compressed_col] = compressed_data == NULL; if (compressed_data != NULL) row_compressor->compressed_values[compressed_col] = PointerGetDatum(compressed_data); } else if (column->segment_info != NULL) { row_compressor->compressed_values[compressed_col] = column->segment_info->val; row_compressor->compressed_is_null[compressed_col] = column->segment_info->is_null; } } ListCell *lc; foreach (lc, row_compressor->metadata_builders) { BatchMetadataBuilder *builder = (BatchMetadataBuilder *) lfirst(lc); builder->insert_to_compressed_row(builder, row_compressor); } row_compressor->compressed_values[row_compressor->count_metadata_column_offset] = Int32GetDatum(row_compressor->rows_compressed_into_current_value); row_compressor->compressed_is_null[row_compressor->count_metadata_column_offset] = false; MemoryContextSwitchTo(old_cxt); /* Build the tuple on the callers memory context */ return heap_form_tuple(row_compressor->out_desc, row_compressor->compressed_values, row_compressor->compressed_is_null); } void row_compressor_clear_batch(RowCompressor *row_compressor, bool changed_groups) { MemoryContext old_cxt = MemoryContextSwitchTo(row_compressor->per_row_ctx); /* free the compressed values now that we're done with them (the old compressor is freed in * finish()) */ for (int col = 0; col < row_compressor->n_input_columns; col++) { PerColumn *column = &row_compressor->per_column[col]; int16 compressed_col; if (column->compressor == NULL && column->segment_info == NULL) continue; compressed_col = row_compressor->uncompressed_col_to_compressed_col[col]; Assert(compressed_col >= 0); if (row_compressor->compressed_is_null[compressed_col]) continue; /* don't free the segment-bys if we've overflowed the row, we still need them */ if (column->segment_info != NULL && !changed_groups) continue; if (column->compressor != NULL || !column->segment_info->typ_by_val) pfree(DatumGetPointer(row_compressor->compressed_values[compressed_col])); row_compressor->compressed_values[compressed_col] = 0; row_compressor->compressed_is_null[compressed_col] = true; } ListCell *lc; foreach (lc, row_compressor->metadata_builders) { BatchMetadataBuilder *builder = (BatchMetadataBuilder *) lfirst(lc); builder->reset(builder, row_compressor); } row_compressor->rowcnt_pre_compression += row_compressor->rows_compressed_into_current_value; row_compressor->num_compressed_rows++; row_compressor->rows_compressed_into_current_value = 0; MemoryContextSwitchTo(old_cxt); MemoryContextReset(row_compressor->per_row_ctx); } static void row_compressor_flush(RowCompressor *row_compressor, BulkWriter *writer, bool changed_groups) { HeapTuple compressed_tuple = row_compressor_build_tuple(row_compressor); MemoryContext old_cxt = MemoryContextSwitchTo(row_compressor->per_row_ctx); /* invalidate continuous aggregate range */ if (row_compressor->invalidation) { InvalidationSettings *settings = row_compressor->invalidation; AttrNumber dim_attnum = AttrOffsetGetAttrNumber(settings->invalidation_column_offset); BatchMetadataBuilderMinMax *minmax_builder = NULL; ListCell *lc; foreach (lc, row_compressor->metadata_builders) { BatchMetadataBuilder *builder = (BatchMetadataBuilder *) lfirst(lc); if (builder->builder_type == METADATA_BUILDER_MINMAX) { BatchMetadataBuilderMinMax *mm = (BatchMetadataBuilderMinMax *) builder; if (mm->attnum == dim_attnum) { minmax_builder = mm; break; } } } Assert(minmax_builder != NULL); Datum min = row_compressor->compressed_values[minmax_builder->min_metadata_attr_offset]; Datum max = row_compressor->compressed_values[minmax_builder->max_metadata_attr_offset]; int64 start = ts_time_value_to_internal(min, minmax_builder->type_oid); int64 end = ts_time_value_to_internal(max, minmax_builder->type_oid); continuous_agg_invalidate_range(settings->hypertable_id, settings->chunk_relid, start, end); } Assert(writer->bistate != NULL); heap_insert(writer->out_rel, compressed_tuple, writer->mycid, writer->insert_options /*=options*/, writer->bistate); if (writer->indexstate->ri_NumIndices > 0) { ts_catalog_index_insert(writer->indexstate, compressed_tuple); } heap_freetuple(compressed_tuple); if (row_compressor->on_flush) row_compressor->on_flush(row_compressor, row_compressor->rows_compressed_into_current_value); MemoryContextSwitchTo(old_cxt); row_compressor_clear_batch(row_compressor, changed_groups); } void row_compressor_reset(RowCompressor *row_compressor) { row_compressor->first_iteration = true; } void row_compressor_close(RowCompressor *row_compressor) { pfree(row_compressor->compressed_is_null); pfree(row_compressor->compressed_values); pfree(row_compressor->per_column); pfree(row_compressor->uncompressed_col_to_compressed_col); FreeTupleDesc(row_compressor->out_desc); } /****************** ** segment_info ** ******************/ SegmentInfo * segment_info_new(Form_pg_attribute column_attr) { TypeCacheEntry *tce = lookup_type_cache(column_attr->atttypid, TYPECACHE_EQ_OPR_FINFO); if (!OidIsValid(tce->eq_opr_finfo.fn_oid)) elog(ERROR, "no equality function for column \"%s\"", NameStr(column_attr->attname)); SegmentInfo *segment_info = palloc(sizeof(*segment_info)); *segment_info = (SegmentInfo){ .typlen = column_attr->attlen, .typ_by_val = column_attr->attbyval, }; fmgr_info_cxt(tce->eq_opr_finfo.fn_oid, &segment_info->eq_fn, CurrentMemoryContext); segment_info->eq_fcinfo = HEAP_FCINFO(2); segment_info->collation = column_attr->attcollation; InitFunctionCallInfoData(*segment_info->eq_fcinfo, &segment_info->eq_fn /*=Flinfo*/, 2 /*=Nargs*/, column_attr->attcollation /*=Collation*/, NULL, /*=Context*/ NULL /*=ResultInfo*/ ); return segment_info; } void segment_info_update(SegmentInfo *segment_info, Datum val, bool is_null) { segment_info->is_null = is_null; if (is_null) segment_info->val = 0; else segment_info->val = datumCopy(val, segment_info->typ_by_val, segment_info->typlen); } bool segment_info_datum_is_in_group(SegmentInfo *segment_info, Datum datum, bool is_null) { Datum data_is_eq; FunctionCallInfo eq_fcinfo; /* if one of the datums is null and the other isn't, we must be in a new group */ if (segment_info->is_null != is_null) return false; /* they're both null */ if (segment_info->is_null) return true; /* neither is null, call the eq function */ eq_fcinfo = segment_info->eq_fcinfo; FC_SET_ARG(eq_fcinfo, 0, segment_info->val); FC_SET_ARG(eq_fcinfo, 1, datum); data_is_eq = FunctionCallInvoke(eq_fcinfo); if (eq_fcinfo->isnull) return false; return DatumGetBool(data_is_eq); } /* * Build a map from compressed attribute numbers to non-compressed attribute * numbers. */ static AttrMap * build_decompress_attrmap(const TupleDesc noncompressed_desc, const TupleDesc compressed_desc, AttrNumber *count_meta_attnum) { AttrMap *attrMap; int outnatts; int innatts; int i; int nextindesc = -1; outnatts = compressed_desc->natts; innatts = noncompressed_desc->natts; attrMap = make_attrmap(outnatts); for (i = 0; i < outnatts; i++) { Form_pg_attribute outatt = TupleDescAttr(compressed_desc, i); char *attname; int j; if (outatt->attisdropped) continue; attname = NameStr(outatt->attname); if (strcmp(attname, COMPRESSION_COLUMN_METADATA_COUNT_NAME) == 0) { *count_meta_attnum = outatt->attnum; /* No point in mapping this attribute since meta columns are not * present in the non-compressed relation and will not be found * below anyway. */ continue; } else if (strncmp(attname, COMPRESSION_COLUMN_METADATA_PREFIX, strlen(COMPRESSION_COLUMN_METADATA_PREFIX)) == 0) { /* We can skip other meta attributes as well */ continue; } for (j = 0; j < innatts; j++) { Form_pg_attribute inatt; nextindesc++; if (nextindesc >= innatts) nextindesc = 0; inatt = TupleDescAttr(noncompressed_desc, nextindesc); if (inatt->attisdropped) continue; if (strcmp(attname, NameStr(inatt->attname)) == 0) { attrMap->attnums[i] = inatt->attnum; break; } } } return attrMap; } BulkWriter bulk_writer_build(Relation out_rel, int insert_options) { BulkWriter writer = { .out_rel = out_rel, .indexstate = CatalogOpenIndexes(out_rel), .mycid = GetCurrentCommandId(true), .bistate = GetBulkInsertState(), .estate = CreateExecutorState(), .insert_options = insert_options, }; return writer; } BulkWriter * bulk_writer_alloc(Relation out_rel, int insert_options) { BulkWriter *writer = palloc(sizeof(BulkWriter)); writer->out_rel = out_rel; writer->indexstate = CatalogOpenIndexes(out_rel); writer->mycid = GetCurrentCommandId(true); writer->bistate = GetBulkInsertState(); writer->estate = CreateExecutorState(); writer->insert_options = insert_options; return writer; } void bulk_writer_close(BulkWriter *writer) { FreeBulkInsertState(writer->bistate); if (writer->indexstate) CatalogCloseIndexes(writer->indexstate); FreeExecutorState(writer->estate); } /********************** ** decompress_chunk ** **********************/ RowDecompressor build_decompressor(const TupleDesc in_desc, const TupleDesc out_desc) { AttrNumber count_meta_attnum = InvalidAttrNumber; AttrMap *attrmap = build_decompress_attrmap(out_desc, in_desc, &count_meta_attnum); Assert(AttributeNumberIsValid(count_meta_attnum)); /* * Use a value that is lower than the typical target batch size, so that we * properly test the reallocation logic. */ const int default_allocated_slots = 300; RowDecompressor decompressor = { .count_compressed_attindex = AttrNumberGetAttrOffset(count_meta_attnum), .in_desc = CreateTupleDescCopyConstr(in_desc), .out_desc = CreateTupleDescCopyConstr(out_desc), .compressed_datums = palloc(sizeof(Datum) * in_desc->natts), .compressed_is_nulls = palloc(sizeof(bool) * in_desc->natts), /* cache memory used to store the decompressed datums/is_null for form_tuple */ .decompressed_datums = palloc(sizeof(Datum) * out_desc->natts), .decompressed_is_nulls = palloc(sizeof(bool) * out_desc->natts), .per_compressed_row_ctx = AllocSetContextCreate(CurrentMemoryContext, "decompress chunk per-compressed row", ALLOCSET_DEFAULT_SIZES), .decompressed_slots = (TupleTableSlot **) palloc0(sizeof(void *) * default_allocated_slots), .decompressed_slots_capacity = default_allocated_slots, .attrmap = attrmap, }; create_per_compressed_column(&decompressor); /* * We need to make sure decompressed_is_nulls is in a defined state. While this * will get written for normal columns it will not get written for dropped columns * since dropped columns don't exist in the compressed chunk so we initialize * with true here. */ memset(decompressor.decompressed_is_nulls, true, out_desc->natts); detoaster_init(&decompressor.detoaster, CurrentMemoryContext); return decompressor; } void row_decompressor_reset(RowDecompressor *decompressor) { MemoryContextReset(decompressor->per_compressed_row_ctx); decompressor->unprocessed_tuples = 0; decompressor->batches_decompressed = 0; decompressor->tuples_decompressed = 0; } void row_decompressor_close(RowDecompressor *decompressor) { MemoryContextDelete(decompressor->per_compressed_row_ctx); detoaster_close(&decompressor->detoaster); free_attrmap(decompressor->attrmap); FreeTupleDesc(decompressor->in_desc); FreeTupleDesc(decompressor->out_desc); pfree(decompressor->compressed_datums); pfree(decompressor->compressed_is_nulls); pfree(decompressor->decompressed_datums); pfree(decompressor->decompressed_is_nulls); pfree((void *) decompressor->decompressed_slots); pfree(decompressor->per_compressed_cols); } void decompress_chunk(Oid in_table, Oid out_table) { /* * Locks are taken in the order uncompressed table then compressed table * for consistency with compress_chunk. * We are _just_ INSERTing into the out_table so in principle we could take * a RowExclusive lock, and let other operations read and write this table * as we work. However, we currently compress each table as a oneshot, so * we're taking the stricter lock to prevent accidents. * We want to prevent other decompressors from decompressing this table, * and we want to prevent INSERTs or UPDATEs which could mess up our decompression. * We may as well allow readers to keep reading the compressed data while * we are decompressing, so we only take an ExclusiveLock instead of AccessExclusive. */ Relation out_rel = table_open(out_table, ExclusiveLock); Relation in_rel = table_open(in_table, ExclusiveLock); int64 nrows_processed = 0; PushActiveSnapshot(GetLatestSnapshot()); BulkWriter writer = bulk_writer_build(out_rel, 0); RowDecompressor decompressor = build_decompressor(RelationGetDescr(in_rel), RelationGetDescr(out_rel)); TupleTableSlot *slot = table_slot_create(in_rel, NULL); TableScanDesc scan = table_beginscan(in_rel, GetActiveSnapshot(), 0, (ScanKey) NULL); int64 report_reltuples = calculate_reltuples_to_report(in_rel->rd_rel->reltuples); while (table_scan_getnextslot(scan, ForwardScanDirection, slot)) { bool should_free; HeapTuple tuple = ExecFetchSlotHeapTuple(slot, false, &should_free); heap_deform_tuple(tuple, decompressor.in_desc, decompressor.compressed_datums, decompressor.compressed_is_nulls); if (should_free) heap_freetuple(tuple); row_decompressor_decompress_row_to_table(&decompressor, &writer); if ((++nrows_processed % report_reltuples) == 0) elog(DEBUG2, "decompressed " INT64_FORMAT " rows from \"%s\"", nrows_processed, RelationGetRelationName(in_rel)); } elog(DEBUG1, "finished decompressing " INT64_FORMAT " rows from \"%s\"", nrows_processed, RelationGetRelationName(in_rel)); table_endscan(scan); ExecDropSingleTupleTableSlot(slot); row_decompressor_close(&decompressor); bulk_writer_close(&writer); table_close(out_rel, NoLock); table_close(in_rel, NoLock); PopActiveSnapshot(); } static void create_per_compressed_column(RowDecompressor *decompressor) { Oid compressed_data_type_oid = ts_custom_type_cache_get(CUSTOM_TYPE_COMPRESSED_DATA)->type_oid; Assert(OidIsValid(compressed_data_type_oid)); decompressor->per_compressed_cols = palloc(sizeof(*decompressor->per_compressed_cols) * decompressor->in_desc->natts); Assert(OidIsValid(compressed_data_type_oid)); for (int col = 0; col < decompressor->in_desc->natts; col++) { Oid decompressed_type; bool is_compressed; int16 decompressed_column_offset; PerCompressedColumn *per_compressed_col = &decompressor->per_compressed_cols[col]; Form_pg_attribute compressed_attr = TupleDescAttr(decompressor->in_desc, col); char *col_name = NameStr(compressed_attr->attname); /* find the mapping from compressed column to uncompressed column, setting * the index of columns that don't have an uncompressed version * (such as metadata) to -1 * Assumption: column names are the same on compressed and * uncompressed chunk. */ AttrNumber decompressed_colnum = decompressor->attrmap->attnums[col]; if (!AttributeNumberIsValid(decompressed_colnum)) { *per_compressed_col = (PerCompressedColumn){ .decompressed_column_offset = -1, }; continue; } decompressed_column_offset = AttrNumberGetAttrOffset(decompressed_colnum); decompressed_type = TupleDescAttr(decompressor->out_desc, decompressed_column_offset)->atttypid; /* determine if the data is compressed or not */ is_compressed = compressed_attr->atttypid == compressed_data_type_oid; if (!is_compressed && compressed_attr->atttypid != decompressed_type) elog(ERROR, "compressed table type '%s' does not match decompressed table type '%s' for " "segment-by column \"%s\"", format_type_be(compressed_attr->atttypid), format_type_be(decompressed_type), col_name); *per_compressed_col = (PerCompressedColumn){ .decompressed_column_offset = decompressed_column_offset, .is_compressed = is_compressed, .decompressed_type = decompressed_type, }; } } static void init_iterator(RowDecompressor *decompressor, CompressedDataHeader *header, int input_column) { Assert(decompressor->in_desc->natts > input_column); PerCompressedColumn *column_info = &decompressor->per_compressed_cols[input_column]; /* Special compression block with the NULL compression algorithm, * tells that all values in the compressed block are NULLs. */ if (header->compression_algorithm == COMPRESSION_ALGORITHM_NULL) { column_info->iterator = NULL; decompressor->compressed_is_nulls[input_column] = true; decompressor->decompressed_is_nulls[column_info->decompressed_column_offset] = true; return; } column_info->iterator = definitions[header->compression_algorithm] .iterator_init_forward(PointerGetDatum(header), column_info->decompressed_type); } static void init_batch(RowDecompressor *decompressor, AttrNumber *attnos, int num_attnos) { /* * Set segmentbys and compressed columns with default value. */ for (int input_column = 0; input_column < decompressor->in_desc->natts; input_column++) { PerCompressedColumn *column_info = &decompressor->per_compressed_cols[input_column]; const int output_index = column_info->decompressed_column_offset; /* Metadata column. */ if (output_index < 0) { continue; } /* Segmentby column. */ if (!column_info->is_compressed) { decompressor->decompressed_datums[output_index] = decompressor->compressed_datums[input_column]; decompressor->decompressed_is_nulls[output_index] = decompressor->compressed_is_nulls[input_column]; continue; } /* Compressed column with default value. */ if (decompressor->compressed_is_nulls[input_column]) { column_info->iterator = NULL; decompressor->decompressed_datums[output_index] = getmissingattr(decompressor->out_desc, output_index + 1, &decompressor->decompressed_is_nulls[output_index]); continue; } /* Only initialize required columns if specified. */ bool found = num_attnos == 0; for (int i = 0; i < num_attnos; i++) { if (output_index == AttrNumberGetAttrOffset(attnos[i])) { found = true; break; } } if (!found) { column_info->iterator = NULL; continue; } /* Normal compressed column. */ Datum compressed_datum = PointerGetDatum( detoaster_detoast_attr_copy((struct varlena *) DatumGetPointer( decompressor->compressed_datums[input_column]), &decompressor->detoaster, CurrentMemoryContext)); CompressedDataHeader *header = get_compressed_data_header(compressed_datum); init_iterator(decompressor, header, input_column); } } /* * Decompresses the current compressed batch into decompressed_slots, and returns * the number of rows in batch. */ int decompress_batch(RowDecompressor *decompressor) { if (decompressor->unprocessed_tuples) return decompressor->unprocessed_tuples; MemoryContext old_ctx = MemoryContextSwitchTo(decompressor->per_compressed_row_ctx); init_batch(decompressor, NULL, 0); /* * Set the number of batch rows from count metadata column. */ const int n_batch_rows = DatumGetInt32(decompressor->compressed_datums[decompressor->count_compressed_attindex]); CheckCompressedData(n_batch_rows > 0); CheckCompressedData(n_batch_rows <= GLOBAL_MAX_ROWS_PER_COMPRESSION); /* * Ensure decompressed_slots array is large enough for this batch. */ if (n_batch_rows > decompressor->decompressed_slots_capacity) { int new_capacity = decompressor->decompressed_slots_capacity * 2; if (new_capacity > GLOBAL_MAX_ROWS_PER_COMPRESSION) { new_capacity = GLOBAL_MAX_ROWS_PER_COMPRESSION; } if (new_capacity < n_batch_rows) { new_capacity = n_batch_rows; } Assert(new_capacity <= GLOBAL_MAX_ROWS_PER_COMPRESSION); MemoryContextSwitchTo(old_ctx); decompressor->decompressed_slots = (TupleTableSlot **) repalloc(decompressor->decompressed_slots, sizeof(void *) * new_capacity); memset(decompressor->decompressed_slots + decompressor->decompressed_slots_capacity, 0, sizeof(void *) * (new_capacity - decompressor->decompressed_slots_capacity)); decompressor->decompressed_slots_capacity = new_capacity; MemoryContextSwitchTo(decompressor->per_compressed_row_ctx); } /* * Decompress all compressed columns for each row of the batch. */ for (int current_row = 0; current_row < n_batch_rows; current_row++) { for (int col = 0; col < decompressor->in_desc->natts; col++) { PerCompressedColumn *column_info = &decompressor->per_compressed_cols[col]; if (column_info->iterator == NULL) { continue; } Assert(column_info->is_compressed); const int output_index = column_info->decompressed_column_offset; const DecompressResult value = column_info->iterator->try_next(column_info->iterator); CheckCompressedData(!value.is_done); decompressor->decompressed_datums[output_index] = value.val; decompressor->decompressed_is_nulls[output_index] = value.is_null; } /* * Form the heap tuple for this decompressed rows and save it for later * processing. */ if (decompressor->decompressed_slots[current_row] == NULL) { MemoryContextSwitchTo(old_ctx); decompressor->decompressed_slots[current_row] = MakeSingleTupleTableSlot(decompressor->out_desc, &TTSOpsHeapTuple); MemoryContextSwitchTo(decompressor->per_compressed_row_ctx); } else { ExecClearTuple(decompressor->decompressed_slots[current_row]); } TupleTableSlot *decompressed_slot = decompressor->decompressed_slots[current_row]; HeapTuple decompressed_tuple = heap_form_tuple(decompressor->out_desc, decompressor->decompressed_datums, decompressor->decompressed_is_nulls); ExecStoreHeapTuple(decompressed_tuple, decompressed_slot, /* should_free = */ false); } /* * Verify that all other columns have ended, i.e. their length is consistent * with the count metadata column. */ for (int col = 0; col < decompressor->in_desc->natts; col++) { PerCompressedColumn *column_info = &decompressor->per_compressed_cols[col]; if (column_info->iterator == NULL) { continue; } Assert(column_info->is_compressed); const DecompressResult value = column_info->iterator->try_next(column_info->iterator); CheckCompressedData(value.is_done); } MemoryContextSwitchTo(old_ctx); decompressor->batches_decompressed++; decompressor->tuples_decompressed += n_batch_rows; decompressor->unprocessed_tuples = n_batch_rows; return n_batch_rows; } /* * Decompresses a single row from current compressed batch * into decompressed_values and decompressed_is_nulls based on the * attnos provided. * * Returns true if the row was decompressed or false if it finished the batch. */ bool decompress_batch_next_row(RowDecompressor *decompressor, AttrNumber *attnos, int num_attnos) { MemoryContext old_ctx = MemoryContextSwitchTo(decompressor->per_compressed_row_ctx); if (decompressor->unprocessed_tuples > 0) { decompressor->unprocessed_tuples--; if (decompressor->unprocessed_tuples == 0) { MemoryContextSwitchTo(old_ctx); return false; } } else { decompressor->batches_decompressed++; init_batch(decompressor, attnos, num_attnos); /* * Set the number of batch rows from count metadata column. */ decompressor->unprocessed_tuples = DatumGetInt32(decompressor->compressed_datums[decompressor->count_compressed_attindex]); CheckCompressedData(decompressor->unprocessed_tuples > 0); CheckCompressedData(decompressor->unprocessed_tuples <= GLOBAL_MAX_ROWS_PER_COMPRESSION); } for (int col = 0; col < decompressor->in_desc->natts; col++) { PerCompressedColumn *column_info = &decompressor->per_compressed_cols[col]; if (column_info->iterator == NULL) { continue; } Assert(column_info->is_compressed); const int output_index = column_info->decompressed_column_offset; const DecompressResult value = column_info->iterator->try_next(column_info->iterator); Assert(!value.is_done); decompressor->decompressed_datums[output_index] = value.val; decompressor->decompressed_is_nulls[output_index] = value.is_null; } decompressor->tuples_decompressed++; MemoryContextSwitchTo(old_ctx); return true; } /* Decompress single column using vectorized decompression */ ArrowArray * decompress_single_column(RowDecompressor *decompressor, AttrNumber attno, bool *single_value) { int16 target_col = -1; PerCompressedColumn *column_info = NULL; for (int col = 0; col < decompressor->in_desc->natts; col++) { column_info = &decompressor->per_compressed_cols[col]; if (!column_info->is_compressed) continue; if (column_info->decompressed_column_offset == AttrNumberGetAttrOffset(attno)) { target_col = col; break; } } Assert(column_info && target_col > -1); if (decompressor->compressed_is_nulls[target_col]) { /* Compressed column has a default value, handle it by generating * a single-value ArrowArray based on the default value. This will have to * be handled specially because of the assumption that the whole row has * this default value. */ *single_value = true; bool isnull; Datum default_datum = getmissingattr(decompressor->out_desc, attno, &isnull); return make_single_value_arrow(column_info->decompressed_type, default_datum, isnull); } *single_value = false; Datum compressed_datum = PointerGetDatum( detoaster_detoast_attr_copy((struct varlena *) DatumGetPointer( decompressor->compressed_datums[target_col]), &decompressor->detoaster, CurrentMemoryContext)); CompressedDataHeader *header = get_compressed_data_header(compressed_datum); /* Handle NULL compression algorithm */ if (header->compression_algorithm == COMPRESSION_ALGORITHM_NULL) { *single_value = true; return make_single_value_arrow(column_info->decompressed_type, (Datum) NULL, true); } DecompressAllFunction decompress_all = tsl_get_decompress_all_function(header->compression_algorithm, column_info->decompressed_type); Assert(decompress_all); return decompress_all(compressed_datum, column_info->decompressed_type, decompressor->per_compressed_row_ctx); } int row_decompressor_decompress_row_to_table(RowDecompressor *decompressor, BulkWriter *writer) { const int n_batch_rows = decompress_batch(decompressor); MemoryContext old_ctx = MemoryContextSwitchTo(decompressor->per_compressed_row_ctx); /* Insert all decompressed rows into table using the bulk insert API. */ table_multi_insert(writer->out_rel, decompressor->decompressed_slots, n_batch_rows, writer->mycid, /* options = */ 0, writer->bistate); /* * Now, update the indexes. If we have several indexes, we want to first * insert the entire batch into one index, then into another, and so on. * Working with one index at a time gives better data access locality, * which reduces the load on shared buffers cache. * The normal Postgres code inserts each row into all indexes, so to do it * the other way around, we create a temporary ResultRelInfo that only * references one index. Then we loop over indexes, and for each index we * set it to this temporary ResultRelInfo, and insert all rows into this * single index. */ if (writer->indexstate->ri_NumIndices > 0) { ResultRelInfo indexstate_copy = *writer->indexstate; Relation single_index_relation; IndexInfo *single_index_info; indexstate_copy.ri_NumIndices = 1; indexstate_copy.ri_IndexRelationDescs = &single_index_relation; indexstate_copy.ri_IndexRelationInfo = &single_index_info; for (int i = 0; i < writer->indexstate->ri_NumIndices; i++) { single_index_relation = writer->indexstate->ri_IndexRelationDescs[i]; single_index_info = writer->indexstate->ri_IndexRelationInfo[i]; for (int row = 0; row < n_batch_rows; row++) { TupleTableSlot *decompressed_slot = decompressor->decompressed_slots[row]; EState *estate = writer->estate; ExprContext *econtext = GetPerTupleExprContext(estate); /* Arrange for econtext's scan tuple to be the tuple under test */ econtext->ecxt_scantuple = decompressed_slot; ExecInsertIndexTuplesCompat(&indexstate_copy, decompressed_slot, estate, false, false, NULL, NIL, false); } } } MemoryContextSwitchTo(old_ctx); row_decompressor_reset(decompressor); return n_batch_rows; } void row_decompressor_decompress_row_to_tuplesort(RowDecompressor *decompressor, Tuplesortstate *tuplesortstate) { const int n_batch_rows = decompress_batch(decompressor); MemoryContext old_ctx = MemoryContextSwitchTo(decompressor->per_compressed_row_ctx); for (int i = 0; i < n_batch_rows; i++) { tuplesort_puttupleslot(tuplesortstate, decompressor->decompressed_slots[i]); } MemoryContextSwitchTo(old_ctx); row_decompressor_reset(decompressor); } /********************/ /*** SQL Bindings ***/ /********************/ Datum tsl_compressed_data_decompress_forward(PG_FUNCTION_ARGS) { CompressedDataHeader *header; FuncCallContext *funcctx; MemoryContext oldcontext; DecompressionIterator *iter; DecompressResult res; if (PG_ARGISNULL(0)) PG_RETURN_NULL(); if (SRF_IS_FIRSTCALL()) { funcctx = SRF_FIRSTCALL_INIT(); oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); header = get_compressed_data_header(PG_GETARG_DATUM(0)); iter = definitions[header->compression_algorithm] .iterator_init_forward(PointerGetDatum(header), get_fn_expr_argtype(fcinfo->flinfo, 1)); funcctx->user_fctx = iter; MemoryContextSwitchTo(oldcontext); } funcctx = SRF_PERCALL_SETUP(); iter = funcctx->user_fctx; res = iter->try_next(iter); if (res.is_done) SRF_RETURN_DONE(funcctx); if (res.is_null) SRF_RETURN_NEXT_NULL(funcctx); SRF_RETURN_NEXT(funcctx, res.val); } Datum tsl_compressed_data_decompress_reverse(PG_FUNCTION_ARGS) { CompressedDataHeader *header; FuncCallContext *funcctx; MemoryContext oldcontext; DecompressionIterator *iter; DecompressResult res; if (PG_ARGISNULL(0)) PG_RETURN_NULL(); if (SRF_IS_FIRSTCALL()) { funcctx = SRF_FIRSTCALL_INIT(); oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); header = get_compressed_data_header(PG_GETARG_DATUM(0)); iter = definitions[header->compression_algorithm] .iterator_init_reverse(PointerGetDatum(header), get_fn_expr_argtype(fcinfo->flinfo, 1)); funcctx->user_fctx = iter; MemoryContextSwitchTo(oldcontext); } funcctx = SRF_PERCALL_SETUP(); iter = funcctx->user_fctx; res = iter->try_next(iter); if (res.is_done) SRF_RETURN_DONE(funcctx); if (res.is_null) SRF_RETURN_NEXT_NULL(funcctx); SRF_RETURN_NEXT(funcctx, res.val); ; } /* * compressed_data_to_array(compressed_data, element_type) -> anyarray */ Datum tsl_compressed_data_to_array(PG_FUNCTION_ARGS) { Datum compressed_data; Oid element_type; ArrayType *result; if (PG_ARGISNULL(0)) PG_RETURN_NULL(); compressed_data = PG_GETARG_DATUM(0); /* Get element type from the second argument's type */ element_type = get_fn_expr_argtype(fcinfo->flinfo, 1); if (!OidIsValid(element_type)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("could not determine element type"))); /* Initial allocation - will grow as needed */ int capacity = TARGET_COMPRESSED_BATCH_SIZE; int count = 0; Datum *values = palloc(sizeof(Datum) * capacity); bool *nulls = palloc(sizeof(bool) * capacity); /* Get type info for array construction */ int16 typlen; bool typbyval; char typalign; get_typlenbyvalalign(element_type, &typlen, &typbyval, &typalign); CompressedDataHeader *header; DecompressionIterator *iter; DecompressResult res; /* Get compressed data header and validate */ header = get_compressed_data_header(compressed_data); /* Initialize the decompression iterator */ iter = definitions[header->compression_algorithm].iterator_init_forward(PointerGetDatum(header), element_type); /* Iterate through all compressed values */ for (;;) { res = iter->try_next(iter); if (res.is_done) break; /* Grow arrays if needed */ if (count >= capacity) { capacity *= 2; values = repalloc(values, sizeof(Datum) * capacity); nulls = repalloc(nulls, sizeof(bool) * capacity); } values[count] = res.val; nulls[count] = res.is_null; count++; } /* Construct and return the PostgreSQL array */ int dims[1]; int lbs[1]; dims[0] = count; lbs[0] = 1; /* 1-based indexing */ result = construct_md_array(values, nulls, 1, /* ndims */ dims, lbs, element_type, typlen, typbyval, typalign); PG_RETURN_ARRAYTYPE_P(result); } /* * compressed_data_column_size(compressed_data, element_type) -> anyarray */ Datum tsl_compressed_data_column_size(PG_FUNCTION_ARGS) { if (PG_ARGISNULL(0)) PG_RETURN_NULL(); Datum compressed_data = PG_GETARG_DATUM(0); /* Get element type from the second argument's type */ Oid element_type = get_fn_expr_argtype(fcinfo->flinfo, 1); if (!OidIsValid(element_type)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("could not determine element type"))); int16 typlen; bool typbyval pg_attribute_unused(); char typalign; get_typlenbyvalalign(element_type, &typlen, &typbyval, &typalign); CompressedDataHeader *header; DecompressionIterator *iter; DecompressResult res; /* Get compressed data header and validate */ header = get_compressed_data_header(compressed_data); if (header->compression_algorithm == COMPRESSION_ALGORITHM_NULL) { /* All values are NULL, so size is 0 */ PG_RETURN_INT32(0); } /* Initialize the decompression iterator */ iter = definitions[header->compression_algorithm].iterator_init_forward(PointerGetDatum(header), element_type); int32 column_size = 0; /* Iterate through all compressed values */ for (;;) { res = iter->try_next(iter); if (res.is_done) break; /* similar to pg_column_size implementation */ if (!res.is_null) { if (typlen == -1) column_size += toast_datum_size(res.val); else if (typlen == -2) column_size += strlen(DatumGetCString(res.val)) + 1; else column_size += typlen; column_size = att_align_nominal(column_size, typalign); } } PG_RETURN_INT32(column_size); } Datum tsl_compressed_data_send(PG_FUNCTION_ARGS) { CompressedDataHeader *header = get_compressed_data_header(PG_GETARG_DATUM(0)); StringInfoData buf; pq_begintypsend(&buf); pq_sendbyte(&buf, header->compression_algorithm); if (header->compression_algorithm != COMPRESSION_ALGORITHM_NULL) { definitions[header->compression_algorithm].compressed_data_send(header, &buf); } PG_RETURN_BYTEA_P(pq_endtypsend(&buf)); } Datum tsl_compressed_data_recv(PG_FUNCTION_ARGS) { StringInfo buf = (StringInfo) PG_GETARG_POINTER(0); CompressedDataHeader header = { .vl_len_ = { 0 } }; header.compression_algorithm = pq_getmsgbyte(buf); if (header.compression_algorithm >= _END_COMPRESSION_ALGORITHMS) elog(ERROR, "invalid compression algorithm %d", header.compression_algorithm); return definitions[header.compression_algorithm].compressed_data_recv(buf); } extern Datum tsl_compressed_data_in(PG_FUNCTION_ARGS) { const char *input = PG_GETARG_CSTRING(0); size_t input_len = strlen(input); int decoded_len; #if PG18_GE /* With version 18 pointer type changed to uint8 * for better readability. * * https://github.com/postgres/postgres/commit/b28c59a6 */ uint8 *decoded; #else char *decoded; #endif StringInfoData data; Datum result; if (input_len > PG_INT32_MAX) elog(ERROR, "input too long"); decoded_len = pg_b64_dec_len(input_len); decoded = palloc(decoded_len + 1); decoded_len = pg_b64_decode(input, input_len, decoded, decoded_len); if (decoded_len < 0) elog(ERROR, "could not decode base64-encoded compressed data"); decoded[decoded_len] = '\0'; data = (StringInfoData){ .data = (char *) decoded, .len = decoded_len, .maxlen = decoded_len, }; result = DirectFunctionCall1(tsl_compressed_data_recv, PointerGetDatum(&data)); PG_RETURN_DATUM(result); } extern Datum tsl_compressed_data_out(PG_FUNCTION_ARGS) { Datum bytes_data = DirectFunctionCall1(tsl_compressed_data_send, PG_GETARG_DATUM(0)); bytea *bytes = DatumGetByteaP(bytes_data); int raw_len = VARSIZE_ANY_EXHDR(bytes); #if PG18_GE /* With version 18 pointer type changed to uint8 * for better readability. * * https://github.com/postgres/postgres/commit/b28c59a6 */ const uint8 *raw_data = (uint8 *) VARDATA(bytes); #else const char *raw_data = VARDATA(bytes); #endif int encoded_len = pg_b64_enc_len(raw_len); char *encoded = palloc(encoded_len + 1); encoded_len = pg_b64_encode(raw_data, raw_len, encoded, encoded_len); if (encoded_len < 0) elog(ERROR, "could not base64-encode compressed data"); encoded[encoded_len] = '\0'; PG_RETURN_CSTRING(encoded); } /* create_hypertable record attribute numbers */ enum Anum_compressed_info { Anum_compressed_info_algorithm = 1, Anum_compressed_info_has_nulls, _Anum_compressed_info_max, }; #define Natts_compressed_info (_Anum_compressed_info_max - 1) extern Datum tsl_compressed_data_info(PG_FUNCTION_ARGS) { const CompressedDataHeader *header = get_compressed_data_header(PG_GETARG_DATUM(0)); TupleDesc tupdesc; HeapTuple tuple; bool has_nulls = false; if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in " "context that cannot accept type record"))); switch (header->compression_algorithm) { case COMPRESSION_ALGORITHM_GORILLA: has_nulls = gorilla_compressed_has_nulls(header); break; case COMPRESSION_ALGORITHM_DICTIONARY: has_nulls = dictionary_compressed_has_nulls(header); break; case COMPRESSION_ALGORITHM_DELTADELTA: has_nulls = deltadelta_compressed_has_nulls(header); break; case COMPRESSION_ALGORITHM_ARRAY: has_nulls = array_compressed_has_nulls(header); break; case COMPRESSION_ALGORITHM_BOOL: has_nulls = bool_compressed_has_nulls(header); break; case COMPRESSION_ALGORITHM_NULL: has_nulls = true; break; case COMPRESSION_ALGORITHM_UUID: has_nulls = uuid_compressed_has_nulls(header); break; default: elog(ERROR, "unknown compression algorithm %d", header->compression_algorithm); break; } tupdesc = BlessTupleDesc(tupdesc); Datum values[Natts_compressed_info]; bool nulls[Natts_compressed_info] = { false }; values[AttrNumberGetAttrOffset(Anum_compressed_info_algorithm)] = NameGetDatum(compression_get_algorithm_name(header->compression_algorithm)); values[AttrNumberGetAttrOffset(Anum_compressed_info_has_nulls)] = BoolGetDatum(has_nulls); tuple = heap_form_tuple(tupdesc, values, nulls); return HeapTupleGetDatum(tuple); } extern Datum tsl_compressed_data_has_nulls(PG_FUNCTION_ARGS) { const CompressedDataHeader *header = get_compressed_data_header(PG_GETARG_DATUM(0)); bool has_nulls = false; switch (header->compression_algorithm) { case COMPRESSION_ALGORITHM_GORILLA: has_nulls = gorilla_compressed_has_nulls(header); break; case COMPRESSION_ALGORITHM_DICTIONARY: has_nulls = dictionary_compressed_has_nulls(header); break; case COMPRESSION_ALGORITHM_DELTADELTA: has_nulls = deltadelta_compressed_has_nulls(header); break; case COMPRESSION_ALGORITHM_ARRAY: has_nulls = array_compressed_has_nulls(header); break; case COMPRESSION_ALGORITHM_BOOL: has_nulls = bool_compressed_has_nulls(header); break; case COMPRESSION_ALGORITHM_NULL: has_nulls = true; break; case COMPRESSION_ALGORITHM_UUID: has_nulls = uuid_compressed_has_nulls(header); break; default: elog(ERROR, "unknown compression algorithm %d", header->compression_algorithm); break; } return BoolGetDatum(has_nulls); } extern CompressionStorage compression_get_toast_storage(CompressionAlgorithm algorithm) { if (algorithm == _INVALID_COMPRESSION_ALGORITHM || algorithm >= _END_COMPRESSION_ALGORITHMS) elog(ERROR, "invalid compression algorithm %d", algorithm); return definitions[algorithm].compressed_data_storage; } /* * Return a default compression algorithm suitable * for the type. The actual algorithm used for a * type might be different though since the compressor * can deviate from the default. The actual algorithm * used for a specific batch can only be determined * by reading the batch header. */ extern CompressionAlgorithm compression_get_default_algorithm(Oid typeoid) { switch (typeoid) { case INT4OID: case INT2OID: case INT8OID: case DATEOID: case TIMESTAMPOID: case TIMESTAMPTZOID: return COMPRESSION_ALGORITHM_DELTADELTA; case FLOAT4OID: case FLOAT8OID: return COMPRESSION_ALGORITHM_GORILLA; case NUMERICOID: return COMPRESSION_ALGORITHM_ARRAY; case BOOLOID: if (ts_guc_enable_bool_compression) return COMPRESSION_ALGORITHM_BOOL; else return COMPRESSION_ALGORITHM_ARRAY; case UUIDOID: if (ts_guc_enable_uuid_compression) return COMPRESSION_ALGORITHM_UUID; else return COMPRESSION_ALGORITHM_DICTIONARY; default: { /* use dictionary if possible, otherwise use array */ TypeCacheEntry *tentry = lookup_type_cache(typeoid, TYPECACHE_EQ_OPR_FINFO | TYPECACHE_HASH_PROC_FINFO); if (tentry->hash_proc_finfo.fn_addr == NULL || tentry->eq_opr_finfo.fn_addr == NULL) return COMPRESSION_ALGORITHM_ARRAY; return COMPRESSION_ALGORITHM_DICTIONARY; } } } const CompressionAlgorithmDefinition * algorithm_definition(CompressionAlgorithm algo) { Assert(algo > 0 && algo < _END_COMPRESSION_ALGORITHMS); return &definitions[algo]; } ================================================ FILE: tsl/src/compression/compression.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <access/attnum.h> #include <catalog/indexing.h> #include <executor/tuptable.h> #include <fmgr.h> #include <lib/stringinfo.h> #include <nodes/execnodes.h> #include <utils/relcache.h> typedef struct BulkInsertStateData *BulkInsertState; #include "batch_metadata_builder_minmax.h" #include "hypertable.h" #include "nodes/columnar_scan/detoaster.h" #include "ts_catalog/compression_settings.h" /* * Compressed data starts with a specialized varlen type starting with the usual * varlen header, and followed by a version specifying which compression * algorithm was used. This allows us to share the same code across different * SQL datatypes. Currently we only allow 127 versions, as we may want to use * variable-width integer type in the event we have more than a non-trivial * number of compression algorithms. */ #define CompressedDataHeaderFields \ char vl_len_[4]; \ uint8 compression_algorithm typedef struct CompressedDataHeader { CompressedDataHeaderFields; } CompressedDataHeader; /* On 32-bit architectures, 64-bit values are boxed when returned as datums. To avoid this overhead we have this type and corresponding iterators for efficiency. The iterators are private to the compression algorithms for now. */ typedef uint64 DecompressDataInternal; typedef struct DecompressResultInternal { DecompressDataInternal val; bool is_null; bool is_done; } DecompressResultInternal; /* This type returns datums and is used as our main interface */ typedef struct DecompressResult { Datum val; bool is_null; bool is_done; } DecompressResult; typedef struct FormData_hypertable ChunkCompressionSettings; typedef struct Compressor Compressor; struct Compressor { void (*append_null)(Compressor *compressord); void (*append_val)(Compressor *compressor, Datum val); bool (*is_full)(Compressor *compressor, Datum val); void *(*finish)(Compressor *data); }; typedef struct ArrowArray ArrowArray; typedef struct DecompressionIterator { uint8 compression_algorithm; bool forward; Oid element_type; DecompressResult (*try_next)(struct DecompressionIterator *); } DecompressionIterator; typedef struct SegmentInfo { Datum val; FmgrInfo eq_fn; FunctionCallInfo eq_fcinfo; int16 typlen; bool is_null; bool typ_by_val; Oid collation; } SegmentInfo; /* this struct holds information about a segmentby column, * and additionally stores the offset for this column in * the chunk. */ typedef struct CompressedSegmentInfo { SegmentInfo *segment_info; int16 chunk_offset; } CompressedSegmentInfo; typedef struct PerCompressedColumn { Oid decompressed_type; /* the compressor to use for compressed columns, always NULL for segmenters * only use if is_compressed */ DecompressionIterator *iterator; /* is this a compressed column or a segment-by column */ bool is_compressed; /* * the index in the decompressed table of the data -1, * if the data is metadata not found in the decompressed table */ int16 decompressed_column_offset; } PerCompressedColumn; typedef struct BulkWriter { Relation out_rel; CatalogIndexState indexstate; EState *estate; CommandId mycid; BulkInsertState bistate; int insert_options; /* heap insert options */ } BulkWriter; typedef struct RowDecompressor { PerCompressedColumn *per_compressed_cols; int16 count_compressed_attindex; TupleDesc in_desc; TupleDesc out_desc; Datum *compressed_datums; bool *compressed_is_nulls; Datum *decompressed_datums; bool *decompressed_is_nulls; MemoryContext per_compressed_row_ctx; int64 batches_decompressed; int64 tuples_decompressed; TupleTableSlot **decompressed_slots; int decompressed_slots_capacity; int unprocessed_tuples; AttrMap *attrmap; Detoaster detoaster; } RowDecompressor; /* * TOAST_STORAGE_EXTENDED for out of line storage. * TOAST_STORAGE_EXTERNAL for out of line storage + native PG toast compression * used when you want to enable postgres native toast * compression on the output of the compression algorithm. */ typedef enum { TOAST_STORAGE_EXTERNAL, TOAST_STORAGE_EXTENDED } CompressionStorage; typedef DecompressionIterator *(*DecompressionInitializer)(Datum, Oid); typedef ArrowArray *(*DecompressAllFunction)(Datum compressed, Oid element_type, MemoryContext dest_mctx); typedef struct CompressionAlgorithmDefinition { DecompressionInitializer iterator_init_forward; DecompressionInitializer iterator_init_reverse; DecompressAllFunction decompress_all; void (*compressed_data_send)(CompressedDataHeader *, StringInfo); Datum (*compressed_data_recv)(StringInfo); Compressor *(*compressor_for_type)(Oid element_type); CompressionStorage compressed_data_storage; } CompressionAlgorithmDefinition; typedef enum CompressionAlgorithm { /* Not a real algorithm, if this does get used, it's a bug in the code */ _INVALID_COMPRESSION_ALGORITHM = 0, COMPRESSION_ALGORITHM_ARRAY, COMPRESSION_ALGORITHM_DICTIONARY, COMPRESSION_ALGORITHM_GORILLA, COMPRESSION_ALGORITHM_DELTADELTA, COMPRESSION_ALGORITHM_BOOL, COMPRESSION_ALGORITHM_NULL, COMPRESSION_ALGORITHM_UUID, /* When adding an algorithm also add a static assert statement below */ /* end of real values */ _END_COMPRESSION_ALGORITHMS, _MAX_NUM_COMPRESSION_ALGORITHMS = 128, } CompressionAlgorithm; typedef struct CompressionStats { int64 rowcnt_pre_compression; int64 rowcnt_post_compression; int64 rowcnt_frozen; } CompressionStats; typedef struct PerColumn { /* the compressor to use for regular columns, NULL for segmenters */ Compressor *compressor; /* segment info; only used if compressor is NULL */ SegmentInfo *segment_info; int16 segmentby_column_index; } PerColumn; typedef struct InvalidationSettings { int32 hypertable_id; Oid chunk_relid; AttrNumber invalidation_column_offset; } InvalidationSettings; typedef struct RowCompressor { /* memory context reset per-row is stored */ MemoryContext per_row_ctx; /* The descriptor of the uncompressed tuple we're processing */ TupleDesc in_desc; /* The descriptor of the compressed tuple we're generating */ TupleDesc out_desc; /* in theory we could have more input columns than outputted ones, so we store the number of inputs/compressors separately */ int n_input_columns; /* info about each column */ struct PerColumn *per_column; /* do we have to check if compressors can accept more data */ bool needs_fullness_check; /* the order of columns in the compressed data need not match the order in the * uncompressed. This array maps each attribute offset in the uncompressed * data to the corresponding one in the compressed */ int16 *uncompressed_col_to_compressed_col; int16 count_metadata_column_offset; /* for continuous aggregate invalidation */ InvalidationSettings *invalidation; /* the number of uncompressed rows compressed into the current compressed row */ uint32 rows_compressed_into_current_value; /* cached arrays used to build the HeapTuple */ Datum *compressed_values; bool *compressed_is_null; int64 rowcnt_pre_compression; int64 num_compressed_rows; /* flag for checking if we are working on the first tuple */ bool first_iteration; /* Callback called on every flush. The ntuples argument is the number of * tuples flushed. Typically used for progress reporting. */ void (*on_flush)(struct RowCompressor *rowcompress, uint64 ntuples); Tuplesortstate *sort_state; int64 tuples_to_sort; /* number of tuples to sort with tuplesort */ int64 tuple_sort_limit; /* number of tuples to flush the compressor on */ List *metadata_builders; /* List of BatchMetadataBuilder */ } RowCompressor; /* * BatchFilter is used for filtering batches before decompressing. * The columns will either be segmentby columns or the corresponding * metadata columns of orderby columns. */ typedef struct BatchFilter { /* Column which we use for filtering */ NameData column_name; /* Filter operation used */ StrategyNumber strategy; /* Collation to be used by the operator */ Oid collation; /* Operator code used */ RegProcedure opcode; /* Value to compare with */ Const *value; /* IS NULL or IS NOT NULL */ bool is_null_check; bool is_null; bool is_array_op; } BatchFilter; extern Datum tsl_compressed_data_decompress_forward(PG_FUNCTION_ARGS); extern Datum tsl_compressed_data_decompress_reverse(PG_FUNCTION_ARGS); extern Datum tsl_compressed_data_send(PG_FUNCTION_ARGS); extern Datum tsl_compressed_data_recv(PG_FUNCTION_ARGS); extern Datum tsl_compressed_data_in(PG_FUNCTION_ARGS); extern Datum tsl_compressed_data_out(PG_FUNCTION_ARGS); extern Datum tsl_compressed_data_info(PG_FUNCTION_ARGS); extern Datum tsl_compressed_data_has_nulls(PG_FUNCTION_ARGS); extern Datum tsl_compressed_data_column_size(PG_FUNCTION_ARGS); extern Datum tsl_compressed_data_to_array(PG_FUNCTION_ARGS); static void pg_attribute_unused() assert_num_compression_algorithms_sane(void) { /* make sure not too many compression algorithms */ StaticAssertStmt(_END_COMPRESSION_ALGORITHMS <= _MAX_NUM_COMPRESSION_ALGORITHMS, "Too many compression algorithms, make sure a decision on variable-length " "version field has been made."); /* existing indexes that MUST NEVER CHANGE */ StaticAssertStmt(COMPRESSION_ALGORITHM_ARRAY == 1, "algorithm index has changed"); StaticAssertStmt(COMPRESSION_ALGORITHM_DICTIONARY == 2, "algorithm index has changed"); StaticAssertStmt(COMPRESSION_ALGORITHM_GORILLA == 3, "algorithm index has changed"); StaticAssertStmt(COMPRESSION_ALGORITHM_DELTADELTA == 4, "algorithm index has changed"); StaticAssertStmt(COMPRESSION_ALGORITHM_BOOL == 5, "algorithm index has changed"); StaticAssertStmt(COMPRESSION_ALGORITHM_NULL == 6, "algorithm index has changed"); StaticAssertStmt(COMPRESSION_ALGORITHM_UUID == 7, "algorithm index has changed"); /* * This should change when adding a new algorithm after adding the new * algorithm to the assert list above. This statement prevents adding a * new algorithm without updating the asserts above */ StaticAssertStmt(_END_COMPRESSION_ALGORITHMS == 8, "number of algorithms have changed, the asserts should be updated"); } extern Name compression_get_algorithm_name(CompressionAlgorithm alg); extern CompressionStorage compression_get_toast_storage(CompressionAlgorithm algo); extern CompressionAlgorithm compression_get_default_algorithm(Oid typeoid); extern CompressionStats compress_chunk(Oid in_table, Oid out_table, int insert_options); extern void decompress_chunk(Oid in_table, Oid out_table); extern DecompressionIterator *(*tsl_get_decompression_iterator_init( CompressionAlgorithm algorithm, bool reverse))(Datum, Oid element_type); extern DecompressAllFunction tsl_get_decompress_all_function(CompressionAlgorithm algorithm, Oid type); typedef struct Chunk Chunk; typedef struct ChunkInsertState ChunkInsertState; extern void decompress_batches_for_insert(ChunkInsertState *cis, TupleTableSlot *slot); extern void init_decompress_state_for_insert(ChunkInsertState *cis, TupleTableSlot *slot); typedef struct ModifyHypertableState ModifyHypertableState; extern bool decompress_target_segments(ModifyHypertableState *ht_state); /* CompressSingleRowState methods */ struct CompressSingleRowState; typedef struct CompressSingleRowState CompressSingleRowState; extern CompressSingleRowState *compress_row_init(int srcht_id, Relation in_rel, Relation out_rel); extern SegmentInfo *segment_info_new(Form_pg_attribute column_attr); extern bool segment_info_datum_is_in_group(SegmentInfo *segment_info, Datum datum, bool is_null); extern TupleTableSlot *compress_row_exec(CompressSingleRowState *cr, TupleTableSlot *slot); extern void compress_row_end(CompressSingleRowState *cr); extern void compress_row_destroy(CompressSingleRowState *cr); extern int row_decompressor_decompress_row_to_table(RowDecompressor *row_decompressor, BulkWriter *writer); extern void row_decompressor_decompress_row_to_tuplesort(RowDecompressor *row_decompressor, Tuplesortstate *tuplesortstate); extern void compress_chunk_populate_sort_info_for_column(const CompressionSettings *settings, Oid table, const char *attname, AttrNumber *att_nums, Oid *sort_operator, Oid *collation, bool *nulls_first); extern Tuplesortstate *compression_create_tuplesort_state(CompressionSettings *settings, Relation rel); extern void row_compressor_init(RowCompressor *row_compressor, const CompressionSettings *settings, const TupleDesc noncompressed_tupdesc, const TupleDesc compressed_tupdesc); extern RowCompressor *tsl_compressor_init(Relation in_rel, BulkWriter **bulk_writer, bool sort, int tuple_sort_limit); extern void tsl_compressor_set_invalidation(RowCompressor *compressor, Hypertable *ht, Oid chunk_relid); extern void tsl_compressor_add_slot(RowCompressor *compressor, BulkWriter *bulk_writer, TupleTableSlot *slot); extern void tsl_compressor_flush(RowCompressor *compressor, BulkWriter *bulk_writer); extern void tsl_compressor_free(RowCompressor *compressor, BulkWriter *bulk_writer); extern void row_compressor_reset(RowCompressor *row_compressor); extern void row_compressor_close(RowCompressor *row_compressor); extern HeapTuple row_compressor_build_tuple(RowCompressor *row_compressor); extern void row_compressor_clear_batch(RowCompressor *row_compressor, bool changed_groups); extern void row_compressor_append_ordered_slot(RowCompressor *row_compressor, TupleTableSlot *slot); extern void row_compressor_append_sorted_rows(RowCompressor *row_compressor, Tuplesortstate *sorted_rel, Relation in_rel, BulkWriter *writer); extern Oid get_compressed_chunk_index(ResultRelInfo *resultRelInfo, const CompressionSettings *settings); extern void segment_info_update(SegmentInfo *segment_info, Datum val, bool is_null); extern BulkWriter bulk_writer_build(Relation out_rel, int insert_options); extern BulkWriter *bulk_writer_alloc(Relation out_rel, int insert_options); extern void bulk_writer_close(BulkWriter *writer); extern RowDecompressor build_decompressor(const TupleDesc in_desc, const TupleDesc out_desc); extern void row_decompressor_reset(RowDecompressor *decompressor); extern void row_decompressor_close(RowDecompressor *decompressor); extern enum CompressionAlgorithms compress_get_default_algorithm(Oid typeoid); extern int decompress_batch(RowDecompressor *decompressor); extern bool decompress_batch_next_row(RowDecompressor *decompressor, AttrNumber *attnos, int num_attnos); extern ArrowArray *decompress_single_column(RowDecompressor *decompressor, AttrNumber attno, bool *single_value); /* * A convenience macro to throw an error about the corrupted compressed data, if * the argument is false. When fuzzing is enabled, we don't show the message not * to pollute the logs. */ #ifndef TS_COMPRESSION_FUZZING #define CORRUPT_DATA_MESSAGE(X) \ (errmsg("the compressed data is corrupt"), errdetail("%s", X), errcode(ERRCODE_DATA_CORRUPTED)) #else #define CORRUPT_DATA_MESSAGE(X) (errcode(ERRCODE_DATA_CORRUPTED)) #endif #define CheckCompressedData(X) \ if (unlikely(!(X))) \ ereport(ERROR, CORRUPT_DATA_MESSAGE(#X)) inline static void * consumeCompressedData(StringInfo si, int bytes) { CheckCompressedData(bytes >= 0); CheckCompressedData(si->cursor + bytes >= si->cursor); /* Check for overflow. */ CheckCompressedData(si->cursor + bytes <= si->len); void *result = si->data + si->cursor; si->cursor += bytes; return result; } const CompressionAlgorithmDefinition *algorithm_definition(CompressionAlgorithm algo); struct decompress_batches_stats { int64 batches_deleted; int64 batches_decompressed; int64 batches_scanned; int64 batches_checked_by_bloom; int64 batches_pruned_by_bloom; int64 batches_without_bloom; int64 batches_bloom_false_positives; int64 tuples_decompressed; int64 tuples_deleted; int64 batches_filtered_compressed; int64 batches_filtered_decompressed; }; ================================================ FILE: tsl/src/compression/compression_dml.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/genam.h> #include <access/sdir.h> #include <access/tableam.h> #include <access/valid.h> #include <catalog/pg_am.h> #include <nodes/nodeFuncs.h> #include <optimizer/optimizer.h> #include <parser/parse_coerce.h> #include <parser/parse_relation.h> #include <parser/parsetree.h> #include <utils/datum.h> #include <utils/lsyscache.h> #include <utils/relcache.h> #include <utils/snapmgr.h> #include <utils/typcache.h> #include <compat/compat.h> #include "foreach_ptr.h" #include <chunk_insert_state.h> #include <compression/arrow_c_data_interface.h> #include <compression/compression.h> #include <compression/compression_dml.h> #include <compression/create.h> #include <compression/sparse_index_bloom1.h> #include <compression/wal_utils.h> #include <continuous_aggs/insert.h> #include <expression_utils.h> #include <indexing.h> #include <nodes/columnar_scan/vector_dict.h> #include <nodes/columnar_scan/vector_predicates.h> #include <nodes/modify_hypertable.h> #include <ts_catalog/array_utils.h> /* * Context for tracking continuous aggregate invalidation during direct batch delete. * When batches are deleted without decompression, we need to track the * time range covered by deleted batches to properly invalidate any * continuous aggregates. */ typedef struct InvalidationContext { int32 hypertable_id; Oid chunk_relid; Oid time_type_oid; AttrNumber min_time_attno; /* compressed chunk column for time min */ AttrNumber max_time_attno; /* compressed chunk column for time max */ } InvalidationContext; typedef BatchQualSummary(BatchMatcher)(RowDecompressor *decompressor, ScanKeyData *scankeys, int num_scankeys, tuple_filtering_constraints *constraints, bool check_full_match, bool *skip_current_tuple); static struct decompress_batches_stats decompress_batches_scan(Relation in_rel, Relation out_rel, Relation index_rel, Snapshot snapshot, bool *skip_current_tuple, bool delete_only, List *is_nulls, InvalidationContext *invalidation_ctx, CachedDecompressionState *cdst, TupleTableSlot *insert_slot); static BatchQualSummary batch_matches(RowDecompressor *decompressor, ScanKeyData *scankeys, int num_scankeys, tuple_filtering_constraints *constraints, bool check_full_match, bool *skip_current_tuple); static BatchQualSummary batch_matches_vectorized(RowDecompressor *decompressor, ScanKeyData *scankeys, int num_scankeys, tuple_filtering_constraints *constraints, bool check_full_match, bool *skip_current_tuple); static void process_predicates(Chunk *ch, CompressionSettings *settings, List *predicates, ScanKeyData **mem_scankeys, int *num_mem_scankeys, List **heap_filters, List **index_filters, List **is_null, List **bloom_filters); static Relation find_matching_index(Relation comp_chunk_rel, List **index_filters, List **heap_filters); static tuple_filtering_constraints *get_batch_keys_for_unique_constraints(Relation relation); static BatchFilter *make_batchfilter(char *column_name, StrategyNumber strategy, Oid collation, RegProcedure opcode, Const *value, bool is_null_check, bool is_null, bool is_array_op); static void report_error(TM_Result result); static bool key_column_is_null(tuple_filtering_constraints *constraints, Relation chunk_rel, Oid ht_relid, TupleTableSlot *slot); static bool can_delete_without_decompression(ModifyHypertableState *ht_state, CompressionSettings *settings, Chunk *chunk, List *predicates); static bool can_vectorize_constraint_checks(tuple_filtering_constraints *constraints, CompressionSettings *settings, Relation chunk_rel, Oid ht_relid, ScanKeyWithAttnos *mem_scankeys); static void update_scankeys(ScanKeyWithAttnos *scankeys, TupleTableSlot *slot, int null_flags); static void init_upsert_bloom_state(ChunkInsertState *cis); static Bitmapset *get_arbiter_index_attnums(ChunkInsertState *cis); static AttrNumber TupleDescGetAttrNumber(TupleDesc desc, const char *name) { for (int i = 0; i < desc->natts; i++) { if (strcmp(name, NameStr(TupleDescAttr(desc, i)->attname)) == 0) return TupleDescAttr(desc, i)->attnum; } return InvalidAttrNumber; } typedef struct MatchedBloom { char *column_name; Bitmapset *attnums; AttrNumber compressed_attnum; int num_cols; } MatchedBloom; /* * Pre-computed bloom filter check for UPDATE/DELETE batch pruning. * The hash is computed once in process_predicates() and checked * per batch in decompress_batches_scan() via bloom1_contains_hash(). */ typedef struct BloomFilterCheck { AttrNumber bloom_attno; /* attnum of bloom metadata column in compressed chunk */ uint64 hash; /* pre-computed hash of the search value(s) */ int num_columns; /* number of columns in the bloom filter (for sort order) */ } BloomFilterCheck; /* * Comparator for list_sort(): order BloomFilterCheck by column count * in descending order, assuming this reflects the selectivity order. */ static int bloom_filter_check_cmp(const ListCell *a, const ListCell *b) { BloomFilterCheck *ca = lfirst(a); BloomFilterCheck *cb = lfirst(b); /* Descending order: more columns first */ return cb->num_columns - ca->num_columns; } /* * Collects equality predicates during process_predicates() for the * post-loop bloom matching pass. Type OIDs are resolved via * get_atttype() at hash computation time. */ typedef struct EqualityPredicate { AttrNumber attno; /* column attno in uncompressed chunk */ Datum constvalue; /* the constant value from WHERE col = <value> */ } EqualityPredicate; /* * Get arbiter index column attnums from the arbiter index list. */ static Bitmapset * get_arbiter_index_attnums(ChunkInsertState *cis) { Assert(cis != NULL); Assert(cis->result_relation_info != NULL); List *arbiterIndexes = cis->result_relation_info->ri_onConflictArbiterIndexes; if (arbiterIndexes == NIL) return NULL; Oid arbiter_oid = linitial_oid(arbiterIndexes); Relation index_rel = index_open(arbiter_oid, AccessShareLock); Bitmapset *attnums = NULL; for (int i = 0; i < index_rel->rd_index->indnkeyatts; i++) { AttrNumber attno = index_rel->rd_index->indkey.values[i]; if (!AttributeNumberIsValid(attno)) { /* Expression index - can't use bloom optimization */ index_close(index_rel, AccessShareLock); return NULL; } attnums = bms_add_member(attnums, attno); } index_close(index_rel, AccessShareLock); return attnums; } /* * Per-chunk initialization of UPSERT bloom state. Called once per chunk in * init_decompress_state_for_insert(), inside the has_primary_or_unique_index block. The result is * cached in CachedDecompressionState via ChunkInsertState in subspace_store. * * It assumes cdst->compression_settings is already looked up for the chunk. * * Discovers which bloom columns match arbiter index columns, that is, being a subset of the * conflict columns. Builds the mapping from bloom columns to INSERT tuple attnums, and resolves * bloom column names to compressed chunk attnums. The chosen bloom filter is stored in the * CachedDecompressionState struct. */ static void init_upsert_bloom_state(ChunkInsertState *cis) { Bitmapset *conflict_attnums = get_arbiter_index_attnums(cis); CachedDecompressionState *cdst = cis->cached_decompression_state; Assert(cdst != NULL); if (cdst == NULL || conflict_attnums == NULL) return; CompressionSettings *settings = cdst->compression_settings; Assert(settings != NULL); if (settings == NULL || settings->fd.index == NULL) return; Oid compressed_relid = settings->fd.compress_relid; SparseIndexSettings *parsed = ts_convert_to_sparse_index_settings(settings->fd.index); Assert(parsed != NULL); if (parsed == NULL) return; /* Map the bloom column names to hypertable attnums, because the bloom columns * will be built based on the insert tuple attnums which are the hypertable attnums. */ TsBmsList per_column_attnos = ts_resolve_columns_to_attnos_from_parsed_settings(parsed, cis->hypertable_relid); Assert(list_length(per_column_attnos) == list_length(parsed->objects)); Assert(list_length(per_column_attnos) > 0); MatchedBloom best_match = { .num_cols = 0 }; /** Parallel iteration over objects and their resolved attnums. */ ListCell *obj_cell; ListCell *attno_cell; forboth (obj_cell, parsed->objects, attno_cell, per_column_attnos) { SparseIndexSettingsObject *obj = lfirst(obj_cell); Bitmapset *bloom_attnos = lfirst(attno_cell); /* Check if bloom type */ List *type_values = ts_get_values_by_key_from_parsed_object(obj, "type"); if (type_values == NIL || strcmp((char *) linitial(type_values), "bloom") != 0) continue; /* Check if bloom columns are a subset of the conflict columns */ if (!bms_is_subset(bloom_attnos, conflict_attnums)) continue; int num_cols = bms_num_members(bloom_attnos); /* Only keep the best match (most columns) */ if (num_cols <= best_match.num_cols) continue; /* Get column name for this bloom */ List *column_names = ts_get_column_names_from_parsed_object(obj); char *col_name = compressed_column_metadata_name_list_v2(bloom1_column_prefix, column_names); /* Verify bloom column exists in the compressed chunk */ AttrNumber compressed_attnum = get_attnum(compressed_relid, col_name); Assert(AttributeNumberIsValid(compressed_attnum)); if (!AttributeNumberIsValid(compressed_attnum)) continue; /* New best match */ best_match.column_name = col_name; best_match.attnums = bms_copy(bloom_attnos); best_match.compressed_attnum = compressed_attnum; best_match.num_cols = num_cols; } /* Create builder for the best match, having the largest number of columns */ if (best_match.num_cols > 0) { Oid type_oids[MAX_BLOOM_FILTER_COLUMNS]; cdst->bloom_column_name = best_match.column_name; cdst->bloom_insert_attnums = best_match.attnums; cdst->upsert_bloom_attnum = best_match.compressed_attnum; int col_idx = 0; int attnum = -1; while ((attnum = bms_next_member(best_match.attnums, attnum)) >= 0) type_oids[col_idx++] = get_atttype(cis->hypertable_relid, attnum); if (ts_guc_enable_sparse_index_bloom) cdst->bloom_hasher = bloom1_hasher_create(type_oids, best_match.num_cols); } ts_bmslist_free(per_column_attnos); ts_free_sparse_index_settings(parsed); } void init_decompress_state_for_insert(ChunkInsertState *cis, TupleTableSlot *slot) { if (!cis->chunk_compressed || cis->cached_decompression_state != NULL) { /* * If the chunk is not compressed or the decompression state has * already been initialized, there is nothing to do here. */ return; } CachedDecompressionState *cdst = NULL; MemoryContext old_context = MemoryContextSwitchTo(cis->mctx); cdst = palloc0(sizeof(CachedDecompressionState)); cis->cached_decompression_state = cdst; cdst->has_primary_or_unique_index = ts_indexing_relation_has_primary_or_unique_index(cis->rel); if (cdst->has_primary_or_unique_index) { tuple_filtering_constraints *constraints = get_batch_keys_for_unique_constraints(cis->rel); if (constraints->covered) constraints->on_conflict = cis->onConflictAction; cdst->constraints = constraints; CompressionSettings *compression_settings = ts_compression_settings_get(RelationGetRelid(cis->rel)); Assert(compression_settings && OidIsValid(compression_settings->fd.compress_relid)); cdst->compression_settings = compression_settings; Relation in_rel = relation_open(compression_settings->fd.compress_relid, RowExclusiveLock); Bitmapset *columns_with_null_check = NULL; Bitmapset *key_columns = constraints->key_columns; Bitmapset *index_columns = NULL; Relation index_rel = NULL; if (ts_guc_enable_dml_decompression_tuple_filtering) { cdst->mem_scankeys.scankeys = build_mem_scankeys_from_slot(cis->hypertable_relid, compression_settings, cis->rel, constraints, slot, &cdst->mem_scankeys.num_scankeys, &cdst->mem_scankeys.attnos); cdst->constraints->vectorized_filtering = can_vectorize_constraint_checks(constraints, compression_settings, cis->rel, cis->hypertable_relid, &cdst->mem_scankeys); cdst->index_scankeys.scankeys = build_index_scankeys_using_slot(cis->hypertable_relid, in_rel, cis->rel, constraints->key_columns, slot, &index_rel, &index_columns, &cdst->index_scankeys.num_scankeys, &cdst->index_scankeys.attnos); if (cis->onConflictAction != ONCONFLICT_NONE) { init_upsert_bloom_state(cis); } } if (index_rel) { /* * Prepare the heap scan keys for all * key columns not found in the index */ key_columns = bms_difference(constraints->key_columns, index_columns); } cdst->heap_scankeys.scankeys = build_heap_scankeys(cis->hypertable_relid, in_rel, cis->rel, compression_settings, key_columns, &columns_with_null_check, slot, &cdst->heap_scankeys.num_scankeys, &cdst->heap_scankeys.attnos); if (index_rel) { cdst->index_relid = RelationGetRelid(index_rel); columns_with_null_check = NULL; index_close(index_rel, AccessShareLock); } cdst->columns_with_null_check = columns_with_null_check; table_close(in_rel, NoLock); } MemoryContextSwitchTo(old_context); } static void update_scankeys(ScanKeyWithAttnos *scankeys, TupleTableSlot *slot, int null_flags) { if (scankeys->num_scankeys == 0) { return; } for (int i = 0; i < scankeys->num_scankeys; i++) { bool isnull = false; Datum value = slot_getattr(slot, scankeys->attnos[i], &isnull); if (isnull) { scankeys->scankeys[i].sk_flags = null_flags; scankeys->scankeys[i].sk_argument = UnassignedDatum; } else { scankeys->scankeys[i].sk_flags = 0; scankeys->scankeys[i].sk_argument = value; } } } void decompress_batches_for_insert(ChunkInsertState *cis, TupleTableSlot *slot) { /* * This is supposed to be called with the actual tuple that is being * inserted, so it cannot be empty. */ Assert(!TTS_EMPTY(slot)); Relation out_rel = cis->rel; CachedDecompressionState *cdst = cis->cached_decompression_state; Assert(cdst != NULL); if (!cdst->has_primary_or_unique_index) { /* * If there are no unique constraints there is nothing to do here. */ return; } if (!ts_guc_enable_dml_decompression) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("inserting into compressed chunk with unique constraints disabled"), errhint("Set timescaledb.enable_dml_decompression to TRUE."))); if (key_column_is_null(cdst->constraints, cis->rel, cis->hypertable_relid, slot)) { /* When any key column is NULL and NULLs are distinct there is no * decompression to be done as the tuple will not conflict with any * existing tuples. */ return; } Assert(cdst->compression_settings->fd.relid == RelationGetRelid(out_rel)); Relation in_rel = relation_open(cdst->compression_settings->fd.compress_relid, RowExclusiveLock); /* the scan keys used for in memory tests of the decompressed tuples */ bool skip_current_tuple = false; struct decompress_batches_stats stats = { 0 }; Relation index_rel = NULL; if (OidIsValid(cdst->index_relid)) { index_rel = index_open(cdst->index_relid, AccessShareLock); } update_scankeys(&cdst->index_scankeys, slot, SK_ISNULL | SK_SEARCHNULL); update_scankeys(&cdst->heap_scankeys, slot, SK_ISNULL | SK_SEARCHNULL); update_scankeys(&cdst->mem_scankeys, slot, SK_ISNULL); if (ts_guc_debug_compression_path_info) { elog(INFO, "Using %s scan with scan keys: index %d, heap %d, memory %d. ", OidIsValid(cdst->index_relid) ? "index" : "table", cdst->index_scankeys.num_scankeys, cdst->heap_scankeys.num_scankeys, cdst->mem_scankeys.num_scankeys); } /* * Using latest snapshot to scan the heap since we are doing this to build * the index on the uncompressed chunks in order to do speculative insertion * which is always built from all tuples (even in higher levels of isolation). */ PushActiveSnapshot(GetLatestSnapshot()); stats = decompress_batches_scan(in_rel, out_rel, index_rel, GetActiveSnapshot(), &skip_current_tuple, false, NIL, NULL /* no CAgg invalidation for inserts */, cdst, slot); if (index_rel) index_close(index_rel, AccessShareLock); PopActiveSnapshot(); if (skip_current_tuple) { cis->skip_current_tuple = true; } cis->counters->batches_deleted += stats.batches_deleted; cis->counters->batches_filtered_decompressed += stats.batches_filtered_decompressed; cis->counters->batches_decompressed += stats.batches_decompressed; cis->counters->tuples_decompressed += stats.tuples_decompressed; cis->counters->batches_scanned += stats.batches_scanned; cis->counters->batches_checked_by_bloom += stats.batches_checked_by_bloom; cis->counters->batches_pruned_by_bloom += stats.batches_pruned_by_bloom; cis->counters->batches_without_bloom += stats.batches_without_bloom; cis->counters->batches_bloom_false_positives += stats.batches_bloom_false_positives; cis->counters->batches_filtered_compressed += stats.batches_filtered_compressed; CommandCounterIncrement(); table_close(in_rel, NoLock); } /* * This method will: * 1. Evaluate WHERE clauses and check if SEGMENT BY columns * are specified or not. * 2. Build scan keys for SEGMENT BY columns. * 3. Move scanned rows to staging area. * 4. Update catalog table to change status of moved chunk. * * Returns true if it decompresses any data. */ static bool decompress_batches_for_update_delete(ModifyHypertableState *ht_state, Chunk *chunk, List *predicates, EState *estate, bool has_joins) { /* process each chunk with its corresponding predicates */ List *heap_filters = NIL; List *index_filters = NIL; List *is_null = NIL; ListCell *lc = NULL; Relation chunk_rel; Relation comp_chunk_rel; Relation matching_index_rel = NULL; BatchFilter *filter; ScanKeyData *scankeys = NULL; Bitmapset *null_columns = NULL; int num_scankeys = 0; ScanKeyData *index_scankeys = NULL; int num_index_scankeys = 0; struct decompress_batches_stats stats = { 0 }; int num_mem_scankeys = 0; ScanKeyData *mem_scankeys = NULL; List *bloom_filters = NIL; CompressionSettings *settings = ts_compression_settings_get(chunk->table_id); bool delete_only = ht_state->mt->operation == CMD_DELETE && !has_joins && can_delete_without_decompression(ht_state, settings, chunk, predicates); InvalidationContext invalidation_ctx = { 0 }; /* * Set up CAgg invalidation context if we're doing direct batch delete * on a hypertable with continuous aggregates. */ if (delete_only && ht_state->has_continuous_aggregate) { const Dimension *time_dim = hyperspace_get_open_dimension(ht_state->ht->space, 0); AttrNumber chunk_time_attno = get_attnum(chunk->table_id, NameStr(time_dim->fd.column_name)); invalidation_ctx.hypertable_id = ht_state->ht->fd.id; invalidation_ctx.chunk_relid = chunk->table_id; invalidation_ctx.time_type_oid = time_dim->fd.column_type; invalidation_ctx.min_time_attno = compressed_column_metadata_attno(settings, chunk->table_id, chunk_time_attno, settings->fd.compress_relid, "min"); invalidation_ctx.max_time_attno = compressed_column_metadata_attno(settings, chunk->table_id, chunk_time_attno, settings->fd.compress_relid, "max"); } process_predicates(chunk, settings, predicates, &mem_scankeys, &num_mem_scankeys, &heap_filters, &index_filters, &is_null, &bloom_filters); chunk_rel = table_open(chunk->table_id, RowExclusiveLock); comp_chunk_rel = table_open(settings->fd.compress_relid, RowExclusiveLock); if (index_filters) { matching_index_rel = find_matching_index(comp_chunk_rel, &index_filters, &heap_filters); } if (heap_filters) { scankeys = build_update_delete_scankeys(comp_chunk_rel, heap_filters, &num_scankeys, &null_columns, &delete_only); } if (matching_index_rel) { index_scankeys = build_index_scankeys(matching_index_rel, index_filters, &num_index_scankeys); } CachedDecompressionState temp_cdst = { 0 }; temp_cdst.index_scankeys.scankeys = index_scankeys; temp_cdst.index_scankeys.num_scankeys = num_index_scankeys; temp_cdst.heap_scankeys.scankeys = scankeys; temp_cdst.heap_scankeys.num_scankeys = num_scankeys; temp_cdst.mem_scankeys.scankeys = mem_scankeys; temp_cdst.mem_scankeys.num_scankeys = num_mem_scankeys; temp_cdst.constraints = NULL; temp_cdst.columns_with_null_check = null_columns; temp_cdst.bloom_filters = bloom_filters; PushActiveSnapshot(GetTransactionSnapshot()); stats = decompress_batches_scan(comp_chunk_rel, chunk_rel, matching_index_rel, GetActiveSnapshot(), NULL, delete_only, is_null, ht_state->has_continuous_aggregate ? &invalidation_ctx : NULL, &temp_cdst, NULL); /* close the selected index */ if (matching_index_rel) index_close(matching_index_rel, AccessShareLock); PopActiveSnapshot(); /* * tuples from compressed chunk has been decompressed and moved * to staging area, thus mark this chunk as partially compressed */ if (stats.batches_decompressed > 0) ts_chunk_set_partial(chunk); table_close(chunk_rel, NoLock); table_close(comp_chunk_rel, NoLock); foreach (lc, heap_filters) { filter = lfirst(lc); pfree(filter); } foreach (lc, index_filters) { filter = lfirst(lc); pfree(filter); } list_free_deep(bloom_filters); ht_state->batches_deleted += stats.batches_deleted; ht_state->batches_filtered_decompressed += stats.batches_filtered_decompressed; ht_state->batches_decompressed += stats.batches_decompressed; ht_state->tuples_decompressed += stats.tuples_decompressed; ht_state->tuples_deleted += stats.tuples_deleted; ht_state->batches_scanned += stats.batches_scanned; ht_state->batches_checked_by_bloom += stats.batches_checked_by_bloom; ht_state->batches_pruned_by_bloom += stats.batches_pruned_by_bloom; ht_state->batches_without_bloom += stats.batches_without_bloom; ht_state->batches_bloom_false_positives += stats.batches_bloom_false_positives; ht_state->batches_filtered_compressed += stats.batches_filtered_compressed; return stats.batches_decompressed > 0; } typedef struct DecompressBatchScanData { TableScanDesc scan; IndexScanDesc index_scan; } DecompressBatchScanData; typedef struct DecompressBatchScanData *DecompressBatchScanDesc; static DecompressBatchScanDesc decompress_batch_beginscan(Relation in_rel, Relation index_rel, Snapshot snapshot, int num_scankeys, ScanKeyData *scankeys) { DecompressBatchScanDesc scan; scan = (DecompressBatchScanDesc) palloc(sizeof(DecompressBatchScanData)); if (index_rel) { scan->index_scan = index_beginscan_compat(in_rel, index_rel, snapshot, NULL, num_scankeys, 0); index_rescan(scan->index_scan, scankeys, num_scankeys, NULL, 0); scan->scan = NULL; } else { scan->scan = table_beginscan(in_rel, snapshot, num_scankeys, scankeys); scan->index_scan = NULL; } return scan; } static bool decompress_batch_scan_getnext_slot(DecompressBatchScanDesc scan, ScanDirection direction, struct TupleTableSlot *slot) { if (scan == NULL) { return false; } else if (scan->index_scan) { return index_getnext_slot(scan->index_scan, direction, slot); } else if (scan->scan) { return table_scan_getnextslot(scan->scan, direction, slot); } else { return false; } } static void decompress_batch_endscan(DecompressBatchScanDesc scan) { if (scan == NULL) { return; } else if (scan->index_scan) { index_endscan(scan->index_scan); } else if (scan->scan) { table_endscan(scan->scan); } pfree(scan); } /* * This method will: * 1.Scan the index created with SEGMENT BY columns or the entire compressed chunk * 2.Fetch matching rows and decompress the row * 3.Delete this row from compressed chunk * 4.Insert decompressed rows to uncompressed chunk * * Returns whether we decompressed anything. * */ static struct decompress_batches_stats decompress_batches_scan(Relation in_rel, Relation out_rel, Relation index_rel, Snapshot snapshot, bool *skip_current_tuple, bool delete_only, List *is_nulls, InvalidationContext *invalidation_ctx, CachedDecompressionState *cdst, TupleTableSlot *insert_slot) { HeapTuple compressed_tuple; BulkWriter writer; RowDecompressor decompressor; bool decompressor_initialized = false; bool valid = false; TM_Result result; DecompressBatchScanDesc scan = NULL; ScanKeyData *index_scankeys = cdst->index_scankeys.scankeys; int num_index_scankeys = cdst->index_scankeys.num_scankeys; ScanKeyData *heap_scankeys = cdst->heap_scankeys.scankeys; int num_heap_scankeys = cdst->heap_scankeys.num_scankeys; ScanKeyData *mem_scankeys = cdst->mem_scankeys.scankeys; int num_mem_scankeys = cdst->mem_scankeys.num_scankeys; tuple_filtering_constraints *constraints = cdst->constraints; Bitmapset *null_columns = cdst->columns_with_null_check; BatchMatcher *batch_matcher = constraints && constraints->vectorized_filtering ? batch_matches_vectorized : batch_matches; AttrNumber meta_count_attno = InvalidAttrNumber; struct decompress_batches_stats stats = { 0 }; /* TODO: Optimization by reusing the index scan while working on a single chunk */ if (index_rel) { scan = decompress_batch_beginscan(in_rel, index_rel, snapshot, num_index_scankeys, index_scankeys); } else { scan = decompress_batch_beginscan(in_rel, NULL, snapshot, num_heap_scankeys, heap_scankeys); } TupleTableSlot *slot = table_slot_create(in_rel, NULL); while (decompress_batch_scan_getnext_slot(scan, ForwardScanDirection, slot)) { stats.batches_scanned++; /* Deconstruct the tuple */ Assert(slot->tts_ops->get_heap_tuple); compressed_tuple = slot->tts_ops->get_heap_tuple(slot); if (index_rel && num_heap_scankeys) { /* filter tuple based on compress_orderby columns */ valid = false; #if PG16_LT HeapKeyTest(compressed_tuple, RelationGetDescr(in_rel), num_heap_scankeys, heap_scankeys, valid); #else valid = HeapKeyTest(compressed_tuple, RelationGetDescr(in_rel), num_heap_scankeys, heap_scankeys); #endif if (!valid) { stats.batches_filtered_compressed++; continue; } } int attrno = bms_next_member(null_columns, -1); int pos = 0; bool is_null_condition = 0; bool seg_col_is_null = false; bool complete_batch_delete; valid = true; /* * Since the heap scan API does not support SK_SEARCHNULL we have to check * for NULL values manually when those are part of the constraints. */ for (; attrno >= 0; attrno = bms_next_member(null_columns, attrno)) { is_null_condition = is_nulls && list_nth_int(is_nulls, pos); seg_col_is_null = slot_attisnull(slot, attrno); if ((seg_col_is_null && !is_null_condition) || (!seg_col_is_null && is_null_condition)) { /* * if segment by column in the scanned tuple has non null value * and IS NULL is specified, OR segment by column has null value * and IS NOT NULL is specified then skip this tuple */ valid = false; break; } pos++; } if (!valid) { stats.batches_filtered_compressed++; continue; } /* To track false positives */ bool bloom_passed = false; /* * Bloom filter pruning for UPDATE/DELETE. Pre-computed hashes * are checked against bloom metadata via slot_getattr(). */ if (cdst->bloom_filters != NIL) { bool bloom_pruned = false; foreach_ptr(BloomFilterCheck, check, cdst->bloom_filters) { bool isnull; Datum bloom_datum = slot_getattr(slot, check->bloom_attno, &isnull); stats.batches_checked_by_bloom++; if (!isnull && !bloom1_contains_hash(bloom_datum, check->hash)) { bloom_pruned = true; break; } if (isnull) stats.batches_without_bloom++; } if (bloom_pruned) { stats.batches_pruned_by_bloom++; stats.batches_filtered_compressed++; continue; } bloom_passed = true; } if (!decompressor_initialized) { decompressor = build_decompressor(RelationGetDescr(in_rel), RelationGetDescr(out_rel)); decompressor_initialized = true; writer = bulk_writer_build(out_rel, 0); meta_count_attno = TupleDescGetAttrNumber(decompressor.in_desc, COMPRESSION_COLUMN_METADATA_COUNT_NAME); Assert(meta_count_attno != InvalidAttrNumber); } heap_deform_tuple(compressed_tuple, decompressor.in_desc, decompressor.compressed_datums, decompressor.compressed_is_nulls); /* Bloom pre-filtering for UPSERT conflict detection */ if (insert_slot != NULL && cdst->bloom_hasher != NULL) { Datum bloom_datum = decompressor.compressed_datums[AttrNumberGetAttrOffset(cdst->upsert_bloom_attnum)]; bool bloom_isnull = decompressor .compressed_is_nulls[AttrNumberGetAttrOffset(cdst->upsert_bloom_attnum)]; if (!bloom_isnull) { NullableDatum values[MAX_BLOOM_FILTER_COLUMNS]; int col_idx = 0; int attnum = -1; while ((attnum = bms_next_member(cdst->bloom_insert_attnums, attnum)) >= 0) { values[col_idx].value = slot_getattr(insert_slot, attnum, &values[col_idx].isnull); col_idx++; } uint64 hash = cdst->bloom_hasher->hash_values(cdst->bloom_hasher, values); stats.batches_checked_by_bloom++; if (!bloom1_contains_hash(bloom_datum, hash)) { row_decompressor_reset(&decompressor); stats.batches_pruned_by_bloom++; continue; } bloom_passed = true; } else { stats.batches_without_bloom++; } } /* If there are no in-memory quals, all rows pass */ BatchQualSummary summary = AllRowsPass; if (num_mem_scankeys) { summary = batch_matcher(&decompressor, mem_scankeys, num_mem_scankeys, constraints, delete_only, /* need to check full batch for direct DELETEs */ skip_current_tuple); /* If no rows pass, complete batch gets filtered */ if (summary == NoRowsPass) { if (bloom_passed) stats.batches_bloom_false_positives++; row_decompressor_reset(&decompressor); stats.batches_filtered_decompressed++; continue; } } complete_batch_delete = (delete_only && summary == AllRowsPass); row_decompressor_reset(&decompressor); if (skip_current_tuple && *skip_current_tuple) { row_decompressor_close(&decompressor); bulk_writer_close(&writer); decompress_batch_endscan(scan); ExecDropSingleTupleTableSlot(slot); return stats; } if (!complete_batch_delete) { write_logical_replication_msg_decompression_start(); } TM_FailureData tmfd; result = table_tuple_delete(in_rel, &compressed_tuple->t_self, GetCurrentCommandId(true), snapshot, InvalidSnapshot, true, &tmfd, false); /* skip reporting error if isolation level is < Repeatable Read * since somebody decompressed the data concurrently, we need to take * that data into account as well when in Read Committed level */ if (result == TM_Deleted && !IsolationUsesXactSnapshot()) { write_logical_replication_msg_decompression_end(); stats.batches_decompressed++; continue; } if (result != TM_Ok) { write_logical_replication_msg_decompression_end(); row_decompressor_close(&decompressor); bulk_writer_close(&writer); decompress_batch_endscan(scan); report_error(result); return stats; } /* If all rows pass, complete batch can be deleted */ if (complete_batch_delete) { stats.batches_deleted++; stats.tuples_deleted += DatumGetInt32( decompressor.compressed_datums[AttrNumberGetAttrOffset(meta_count_attno)]); /* Track time range for continuous aggregate invalidation if needed */ if (invalidation_ctx) { Datum min_time_datum = decompressor.compressed_datums[AttrNumberGetAttrOffset( invalidation_ctx->min_time_attno)]; Datum max_time_datum = decompressor.compressed_datums[AttrNumberGetAttrOffset( invalidation_ctx->max_time_attno)]; int64 batch_min = ts_time_value_to_internal(min_time_datum, invalidation_ctx->time_type_oid); int64 batch_max = ts_time_value_to_internal(max_time_datum, invalidation_ctx->time_type_oid); continuous_agg_invalidate_range(invalidation_ctx->hypertable_id, invalidation_ctx->chunk_relid, batch_min, batch_max); } } else { stats.tuples_decompressed += row_decompressor_decompress_row_to_table(&decompressor, &writer); stats.batches_decompressed++; write_logical_replication_msg_decompression_end(); } } ExecDropSingleTupleTableSlot(slot); decompress_batch_endscan(scan); if (decompressor_initialized) { row_decompressor_close(&decompressor); bulk_writer_close(&writer); } if (ts_guc_debug_compression_path_info) { elog(INFO, "Number of compressed rows fetched from %s: " INT64_FORMAT ". " "Number of compressed rows filtered%s: " INT64_FORMAT ".", index_rel ? "index" : "table scan", stats.batches_scanned, index_rel ? " by heap filters" : "", stats.batches_filtered_compressed); } return stats; } static BatchQualSummary batch_matches(RowDecompressor *decompressor, ScanKeyData *scankeys, int num_scankeys, tuple_filtering_constraints *constraints, bool check_full_match, bool *skip_current_tuple) { AttrNumber *attnos = palloc0(sizeof(AttrNumber) * num_scankeys); for (int i = 0; i < num_scankeys; i++) { attnos[i] = scankeys[i].sk_attno; } bool next_tuple = decompress_batch_next_row(decompressor, attnos, num_scankeys); ScanKey key; bool match; /* Default values are set like this because of binary operations * used to calculate these flags. */ bool match_any = false; bool match_all = true; while (next_tuple) { match = true; for (int i = 0; i < num_scankeys; i++) { key = &scankeys[i]; if (key->sk_flags & SK_ISNULL) { if (!decompressor->decompressed_is_nulls[AttrNumberGetAttrOffset(key->sk_attno)]) { match = false; break; } continue; } else if (decompressor->decompressed_is_nulls[AttrNumberGetAttrOffset(key->sk_attno)]) { match = false; break; } if (!DatumGetBool( FunctionCall2Coll(&key->sk_func, key->sk_collation, decompressor->decompressed_datums[AttrNumberGetAttrOffset( key->sk_attno)], key->sk_argument))) { match = false; break; } } match_any |= match; match_all &= match; if (match) { match_any = true; if (constraints) { if (constraints->on_conflict == ONCONFLICT_NONE) { ereport(ERROR, (errcode(ERRCODE_UNIQUE_VIOLATION), errmsg("duplicate key value violates unique constraint \"%s\"", get_rel_name(constraints->index_relid)) )); } if (constraints->on_conflict == ONCONFLICT_NOTHING && skip_current_tuple) { *skip_current_tuple = true; } } if (!check_full_match) return SomeRowsPass; } next_tuple = decompress_batch_next_row(decompressor, attnos, num_scankeys); } if (match_all) return AllRowsPass; if (match_any) return SomeRowsPass; return NoRowsPass; } static void apply_validity_bitmap(const ArrowArray *arrow, uint64 *restrict result) { const uint64 *validity = (const uint64 *) arrow->buffers[0]; if (validity) { const size_t n_vector_result_words = (arrow->length + 63) / 64; for (size_t i = 0; i < n_vector_result_words; i++) { result[i] &= validity[i]; } } else { Assert(arrow->null_count == 0); } } static BatchQualSummary batch_matches_vectorized(RowDecompressor *decompressor, ScanKeyData *scankeys, int num_scankeys, tuple_filtering_constraints *constraints, bool check_full_match, bool *skip_current_tuple) { const int n_rows = DatumGetInt32(decompressor->compressed_datums[decompressor->count_compressed_attindex]); const int bitmap_bytes = sizeof(uint64) * ((n_rows + 63) / 64); uint64 *restrict result = MemoryContextAlloc(decompressor->per_compressed_row_ctx, bitmap_bytes); uint64 dict_result[(GLOBAL_MAX_ROWS_PER_COMPRESSION + 63) / 64]; memset(result, 0xFF, bitmap_bytes); bool single_value = false; bool batch_failed = false; /* batch_matches() calls decompress_batch_next_row() which increments * the decompressor's batched_decompressed variable. To match that * behaviour we need to bump it here. */ decompressor->batches_decompressed++; for (int sk = 0; sk < num_scankeys; sk++) { ArrowArray *arrow = decompress_single_column(decompressor, scankeys[sk].sk_attno, &single_value); /* Handle null check */ if (scankeys[sk].sk_flags & SK_ISNULL) { if (single_value) { uint64 single_value_result = 1; vector_nulltest(arrow, IS_NULL, &single_value_result); if (!(single_value_result & 1)) { batch_failed = true; break; } } else { vector_nulltest(arrow, IS_NULL, result); } continue; } VectorPredicate *predicate = get_vector_const_predicate(scankeys[sk].sk_func.fn_oid); if (single_value) { /* * For single-value columns (default values), use a separate bitmap * to avoid corrupting the main result. The predicate and validity * bitmap operate on a 1-element arrow, which would clear bits 1-63 * of result[0] if applied directly. */ uint64 single_value_result = 1; predicate(arrow, scankeys[sk].sk_argument, &single_value_result); apply_validity_bitmap(arrow, &single_value_result); if (!(single_value_result & 1)) { batch_failed = true; break; } continue; } /* Handle non-dictionary compressed data */ if (!arrow->dictionary) { predicate(arrow, scankeys[sk].sk_argument, result); } else { /* Handle dictionary compressed data by decompressing the dictionary * first and then translating the results to actual results */ const size_t dict_rows = arrow->dictionary->length; const size_t dict_result_words = (dict_rows + 63) / 64; memset(dict_result, 0xFF, dict_result_words * 8); predicate(arrow->dictionary, scankeys[sk].sk_argument, dict_result); translate_bitmap_from_dictionary(arrow, dict_result, result); } apply_validity_bitmap(arrow, result); } if (batch_failed) { return NoRowsPass; } BatchQualSummary summary = get_vector_qual_summary(result, n_rows); if (summary != NoRowsPass) { if (constraints) { if (constraints->on_conflict == ONCONFLICT_NONE) { ereport(ERROR, (errcode(ERRCODE_UNIQUE_VIOLATION), errmsg("duplicate key value violates unique constraint \"%s\"", get_rel_name(constraints->index_relid)) )); } if (constraints->on_conflict == ONCONFLICT_NOTHING && skip_current_tuple) { *skip_current_tuple = true; } } return summary; } return summary; } /* * Traverse the plan tree to look for Scan nodes on uncompressed chunks. * Once Scan node is found check if chunk is compressed, if so then * decompress those segments which match the filter conditions if present. */ struct decompress_chunk_context { List *relids; ModifyHypertableState *ht_state; /* indicates decompression actually occurred */ bool batches_decompressed; bool has_joins; }; static bool decompress_chunk_walker(PlanState *ps, struct decompress_chunk_context *ctx); bool decompress_target_segments(ModifyHypertableState *ht_state) { ModifyTableState *ps = linitial_node(ModifyTableState, castNode(CustomScanState, ht_state)->custom_ps); struct decompress_chunk_context ctx = { .ht_state = ht_state, .relids = castNode(ModifyTable, ps->ps.plan)->resultRelations, }; Assert(ctx.relids); decompress_chunk_walker(&ps->ps, &ctx); return ctx.batches_decompressed; } static bool decompress_chunk_walker(PlanState *ps, struct decompress_chunk_context *ctx) { RangeTblEntry *rte = NULL; bool needs_decompression = false; bool should_rescan = false; bool batches_decompressed = false; List *predicates = NIL; Chunk *current_chunk; if (ps == NULL) return false; switch (nodeTag(ps)) { /* Note: IndexOnlyScans will never be selected for target * tables because system columns are necessary in order to modify the * data and those columns cannot be a part of the index */ case T_IndexScanState: { /* Get the index quals on the original table and also include * any filters that are used for filtering heap tuples */ predicates = list_union(((IndexScan *) ps->plan)->indexqualorig, ps->plan->qual); needs_decompression = true; break; } case T_BitmapHeapScanState: predicates = list_union(((BitmapHeapScan *) ps->plan)->bitmapqualorig, ps->plan->qual); needs_decompression = true; should_rescan = true; break; case T_SeqScanState: case T_SampleScanState: case T_TidScanState: case T_TidRangeScanState: { predicates = list_copy(ps->plan->qual); needs_decompression = true; break; } case T_NestLoopState: case T_MergeJoinState: case T_HashJoinState: { ctx->has_joins = true; break; } default: break; } if (needs_decompression) { /* * We are only interested in chunk scans of chunks that are the * target of the DML statement not chunk scan on joined hypertables * even when it is a self join */ int scanrelid = ((Scan *) ps->plan)->scanrelid; if (list_member_int(ctx->relids, scanrelid)) { rte = rt_fetch(scanrelid, ps->state->es_range_table); current_chunk = ts_chunk_get_by_relid(rte->relid, false); if (current_chunk && ts_chunk_is_compressed(current_chunk)) { if (!ts_guc_enable_dml_decompression) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("UPDATE/DELETE is disabled on compressed chunks"), errhint("Set timescaledb.enable_dml_decompression to TRUE."))); batches_decompressed = decompress_batches_for_update_delete(ctx->ht_state, current_chunk, predicates, ps->state, ctx->has_joins); ctx->batches_decompressed |= batches_decompressed; /* This is a workaround specifically for bitmap heap scans: * during node initialization, initialize the scan state with the active snapshot * but since we are inserting data to be modified during the same query, they end up * missing that data by using a snapshot which doesn't account for this decompressed * data. To circumvent this issue, we change the internal scan state to use the * transaction snapshot and execute a rescan so the scan state is set correctly and * includes the new data. * * From PG17 this has changed since the scan state is not initialized with * the node. */ if (should_rescan) { ScanState *ss = ((ScanState *) ps); if (ss && ss->ss_currentScanDesc) { ss->ss_currentScanDesc->rs_snapshot = GetActiveSnapshot(); table_rescan(ss->ss_currentScanDesc, NULL); } } } } } if (predicates) pfree(predicates); return planstate_tree_walker(ps, decompress_chunk_walker, ctx); } /* * For insert into compressed chunks with unique index determine the * columns which can be used for INSERT batch filtering. * The passed in relation is the uncompressed chunk. * * In case of multiple unique indexes we have to return the shared columns. * For expression indexes we ignore the columns with expressions, for partial * indexes we ignore predicate. * */ static tuple_filtering_constraints * get_batch_keys_for_unique_constraints(Relation relation) { tuple_filtering_constraints *constraints = palloc0(sizeof(tuple_filtering_constraints)); constraints->on_conflict = ONCONFLICT_UPDATE; constraints->nullsnotdistinct = false; ListCell *lc; /* Fast path if definitely no indexes */ if (!RelationGetForm(relation)->relhasindex) return constraints; List *indexoidlist = RelationGetIndexList(relation); /* Fall out if no indexes (but relhasindex was set) */ if (indexoidlist == NIL) return constraints; foreach (lc, indexoidlist) { Oid indexOid = lfirst_oid(lc); Relation indexDesc = index_open(indexOid, AccessShareLock); /* * We are only interested in unique indexes. PRIMARY KEY indexes also have * indisunique set to true so we do not need to check for them separately. */ if (!indexDesc->rd_index->indislive || !indexDesc->rd_index->indisvalid || !indexDesc->rd_index->indisunique) { index_close(indexDesc, AccessShareLock); continue; } Bitmapset *idx_attrs = NULL; /* * Collect attributes of current index. * For covering indexes we need to ignore the included columns. */ for (int i = 0; i < indexDesc->rd_index->indnkeyatts; i++) { AttrNumber attno = indexDesc->rd_index->indkey.values[i]; /* We are not interested in expression columns which will have attno = 0 */ if (!attno) continue; Assert(AttrNumberIsForUserDefinedAttr(attno)); idx_attrs = bms_add_member(idx_attrs, attno); } index_close(indexDesc, AccessShareLock); if (!constraints->key_columns) { /* First iteration */ constraints->key_columns = bms_copy(idx_attrs); /* * We only optimize unique constraint checks for non-partial and * non-expression indexes. For partial and expression indexes we * can still do batch filtering, just not make decisions about * constraint violations. */ constraints->covered = indexDesc->rd_indexprs == NIL && indexDesc->rd_indpred == NIL; constraints->index_relid = indexDesc->rd_id; } else { /* more than one unique constraint */ constraints->key_columns = bms_intersect(idx_attrs, constraints->key_columns); constraints->covered = false; } /* If any of the unique indexes have NULLS NOT DISTINCT set, we proceed * with checking the constraints with decompression */ constraints->nullsnotdistinct |= indexDesc->rd_index->indnullsnotdistinct; /* When multiple unique indexes are present, in theory there could be no shared * columns even though that is very unlikely as they will probably at least share * the partitioning columns. But since we are looking at chunk indexes here that * is not guaranteed. */ if (!constraints->key_columns) return constraints; } return constraints; } /* * This method will evaluate the predicates, extract * left and right operands, check if any of the operands * can be used for batch filtering and if so, it will * create a BatchFilter object and add it to the corresponding * list. * Any segmentby filter is put into index_filters list other * filters are put into heap_filters list. */ static void process_predicates(Chunk *ch, CompressionSettings *settings, List *predicates, ScanKeyData **mem_scankeys, int *num_mem_scankeys, List **heap_filters, List **index_filters, List **is_null, List **bloom_filters) { ListCell *lc; if (ts_guc_enable_dml_decompression_tuple_filtering) { *mem_scankeys = palloc0(sizeof(ScanKeyData) * list_length(predicates)); } *num_mem_scankeys = 0; List *eq_preds = NIL; /* * We dont want to forward boundParams from the execution state here * as we dont want to constify join params in the predicates. * Constifying JOIN params would not be safe as we don't redo * this part in rescan. */ PlannerGlobal glob = { .boundParams = NULL }; PlannerInfo root = { .glob = &glob }; foreach (lc, predicates) { Node *node = copyObject(lfirst(lc)); Var *var; Expr *expr; Oid collation, opno; RegProcedure opcode; char *column_name; switch (nodeTag(node)) { case T_OpExpr: { OpExpr *opexpr = castNode(OpExpr, node); collation = opexpr->inputcollid; Const *arg_value; if (!ts_extract_expr_args(&opexpr->xpr, &var, &expr, &opno, &opcode)) continue; if (!IsA(expr, Const)) { expr = (Expr *) estimate_expression_value(&root, (Node *) expr); if (!IsA(expr, Const)) continue; } arg_value = castNode(Const, expr); column_name = get_attname(ch->table_id, var->varattno, false); TypeCacheEntry *tce = lookup_type_cache(var->vartype, TYPECACHE_BTREE_OPFAMILY); int op_strategy = get_op_opfamily_strategy(opno, tce->btree_opf); if (ts_array_is_member(settings->fd.segmentby, column_name)) { switch (op_strategy) { case BTEqualStrategyNumber: case BTLessStrategyNumber: case BTLessEqualStrategyNumber: case BTGreaterStrategyNumber: case BTGreaterEqualStrategyNumber: { /* save segment by column name and its corresponding value specified in * WHERE */ *index_filters = lappend(*index_filters, make_batchfilter(column_name, op_strategy, collation, opcode, arg_value, false, /* is_null_check */ false, /* is_null */ false /* is_array_op */ )); break; } default: *heap_filters = lappend(*heap_filters, make_batchfilter(column_name, op_strategy, collation, opcode, arg_value, false, /* is_null_check */ false, /* is_null */ false /* is_array_op */ )); break; } continue; } /* * Segmentby columns are checked as part of batch scan so no need to redo the check. */ if (ts_guc_enable_dml_decompression_tuple_filtering) { ScanKeyEntryInitialize(&(*mem_scankeys)[(*num_mem_scankeys)++], arg_value->constisnull ? SK_ISNULL : 0, var->varattno, op_strategy, arg_value->consttype, arg_value->constcollid, opcode, arg_value->constisnull ? 0 : arg_value->constvalue); } /* * Collect equality predicates for the post-loop * bloom filter matching pass. */ if (op_strategy == BTEqualStrategyNumber && !arg_value->constisnull && ts_guc_enable_dml_bloom_filter) { EqualityPredicate *ep = palloc(sizeof(EqualityPredicate)); ep->attno = var->varattno; #ifdef USE_FLOAT8_BYVAL ep->constvalue = arg_value->constvalue; #else if (arg_value->constbyval) ep->constvalue = Int64GetDatum(arg_value->constvalue); else ep->constvalue = datumCopy(arg_value->constvalue, arg_value->constbyval, arg_value->constlen); #endif eq_preds = lappend(eq_preds, ep); } int min_attno = compressed_column_metadata_attno(settings, settings->fd.relid, var->varattno, settings->fd.compress_relid, "min"); if (min_attno == InvalidAttrNumber) continue; int max_attno = compressed_column_metadata_attno(settings, ch->table_id, var->varattno, settings->fd.compress_relid, "max"); if (max_attno == InvalidAttrNumber) continue; /* Need both min and max metadata attributes to build heap filters */ switch (op_strategy) { case BTEqualStrategyNumber: { /* orderby col = value implies min <= value and max >= value */ *heap_filters = lappend(*heap_filters, make_batchfilter(get_attname(settings->fd.compress_relid, min_attno, false), BTLessEqualStrategyNumber, collation, opcode, arg_value, false, /* is_null_check */ false, /* is_null */ false /* is_array_op */ )); *heap_filters = lappend(*heap_filters, make_batchfilter(get_attname(settings->fd.compress_relid, max_attno, false), BTGreaterEqualStrategyNumber, collation, opcode, arg_value, false, /* is_null_check */ false, /* is_null */ false /* is_array_op */ )); } break; case BTLessStrategyNumber: case BTLessEqualStrategyNumber: { /* orderby col <[=] value implies min <[=] value */ *heap_filters = lappend(*heap_filters, make_batchfilter(get_attname(settings->fd.compress_relid, min_attno, false), op_strategy, collation, opcode, arg_value, false, /* is_null_check */ false, /* is_null */ false /* is_array_op */ )); } break; case BTGreaterStrategyNumber: case BTGreaterEqualStrategyNumber: { /* orderby col >[=] value implies max >[=] value */ *heap_filters = lappend(*heap_filters, make_batchfilter(get_attname(settings->fd.compress_relid, max_attno, false), op_strategy, collation, opcode, arg_value, false, /* is_null_check */ false, /* is_null */ false /* is_array_op */ )); } break; default: /* Do nothing for unknown operator strategies. */ break; } } break; case T_ScalarArrayOpExpr: { ScalarArrayOpExpr *sa_expr = castNode(ScalarArrayOpExpr, node); if (!ts_extract_expr_args(&sa_expr->xpr, &var, &expr, &opno, &opcode)) continue; if (!IsA(expr, Const)) { expr = (Expr *) estimate_expression_value(&root, (Node *) expr); if (!IsA(expr, Const)) continue; } Const *arg_value = castNode(Const, expr); collation = sa_expr->inputcollid; column_name = get_attname(ch->table_id, var->varattno, false); TypeCacheEntry *tce = lookup_type_cache(var->vartype, TYPECACHE_BTREE_OPFAMILY); int op_strategy = get_op_opfamily_strategy(opno, tce->btree_opf); if (ts_array_is_member(settings->fd.segmentby, column_name)) { switch (op_strategy) { case BTEqualStrategyNumber: case BTLessStrategyNumber: case BTLessEqualStrategyNumber: case BTGreaterStrategyNumber: case BTGreaterEqualStrategyNumber: { /* save segment by column name and its corresponding value specified in * WHERE */ *index_filters = lappend(*index_filters, make_batchfilter(column_name, op_strategy, collation, opcode, arg_value, false, /* is_null_check */ false, /* is_null */ true /* is_array_op */ )); break; } default: *heap_filters = lappend(*heap_filters, make_batchfilter(column_name, op_strategy, collation, opcode, arg_value, false, /* is_null_check */ false, /* is_null */ true /* is_array_op */ )); break; } continue; } break; } case T_NullTest: { NullTest *ntest = (NullTest *) node; if (IsA(ntest->arg, Var)) { var = (Var *) ntest->arg; /* ignore system-defined attributes */ if (var->varattno <= 0) continue; column_name = get_attname(ch->table_id, var->varattno, false); if (ts_array_is_member(settings->fd.segmentby, column_name)) { *index_filters = lappend(*index_filters, make_batchfilter(column_name, InvalidStrategy, InvalidOid, InvalidOid, NULL, true, /* is_null_check */ ntest->nulltesttype == IS_NULL, /* is_null */ false /* is_array_op */ )); if (ntest->nulltesttype == IS_NULL) *is_null = lappend_int(*is_null, 1); else *is_null = lappend_int(*is_null, 0); } /* We cannot optimize filtering decompression using ORDERBY * metadata and null check qualifiers. We could possibly do that by checking the * compressed data in combination with the ORDERBY nulls first setting and * verifying that the first or last tuple of a segment contains a NULL value. * This is left for future optimization */ } } break; default: break; } } /* * Bloom filter matching pass: iterate all bloom configs from * SparseIndexSettings, check which ones are fully covered by * equality predicates, compute hashes, and collect matches. * Results are sorted by selectivity (most columns first). */ if (eq_preds != NIL && settings->fd.index != NULL && ts_guc_enable_dml_bloom_filter) { SparseIndexSettings *parsed = ts_convert_to_sparse_index_settings(settings->fd.index); TsBmsList per_column_attnos = ts_resolve_columns_to_attnos_from_parsed_settings(parsed, ch->table_id); /* Build a Bitmapset of equality predicate attnums for bms_is_subset() */ Bitmapset *eq_pred_attnos = NULL; foreach_ptr(EqualityPredicate, ep, eq_preds) eq_pred_attnos = bms_add_member(eq_pred_attnos, ep->attno); ListCell *obj_cell; ListCell *attno_cell; forboth (obj_cell, parsed->objects, attno_cell, per_column_attnos) { SparseIndexSettingsObject *obj = lfirst(obj_cell); Bitmapset *bloom_attnos = lfirst(attno_cell); /* Check if bloom type */ List *type_values = ts_get_values_by_key_from_parsed_object(obj, "type"); if (type_values == NIL || strcmp((char *) linitial(type_values), "bloom") != 0) continue; int num_columns = bms_num_members(bloom_attnos); /* Skip composite blooms if the GUC is off */ if (num_columns > 1 && !ts_guc_enable_composite_bloom_indexes) continue; /* Check if ALL bloom columns have equality predicates */ if (!bms_is_subset(bloom_attnos, eq_pred_attnos)) continue; /* * All columns covered. Collect type OIDs and values in * ascending attnum order (via bms_next_member) to match * the order used during compression. */ Oid type_oids[MAX_BLOOM_FILTER_COLUMNS]; NullableDatum values[MAX_BLOOM_FILTER_COLUMNS]; int col_idx = 0; int attnum = -1; while ((attnum = bms_next_member(bloom_attnos, attnum)) >= 0) { type_oids[col_idx] = get_atttype(ch->table_id, attnum); /* Find the matching equality predicate for this attnum */ foreach_ptr(EqualityPredicate, ep, eq_preds) { if (ep->attno == attnum) { values[col_idx].value = ep->constvalue; values[col_idx].isnull = false; break; } } col_idx++; } Bloom1Hasher *hasher = bloom1_hasher_create(type_oids, num_columns); uint64 hash = hasher->hash_values(hasher, values); /* Resolve bloom metadata column attno in compressed chunk */ List *column_names = ts_get_column_names_from_parsed_object(obj); char *bloom_col_name = compressed_column_metadata_name_list_v2(bloom1_column_prefix, column_names); AttrNumber bloom_attno = get_attnum(settings->fd.compress_relid, bloom_col_name); if (AttributeNumberIsValid(bloom_attno)) { BloomFilterCheck *check = palloc(sizeof(BloomFilterCheck)); check->bloom_attno = bloom_attno; check->hash = hash; check->num_columns = num_columns; *bloom_filters = lappend(*bloom_filters, check); } pfree(hasher); } bms_free(eq_pred_attnos); ts_bmslist_free(per_column_attnos); ts_free_sparse_index_settings(parsed); list_free_deep(eq_preds); } /* * Sort bloom filters by number of columns (descending) assuming they * are more selective. */ if (list_length(*bloom_filters) > 1) list_sort(*bloom_filters, bloom_filter_check_cmp); } static BatchFilter * make_batchfilter(char *column_name, StrategyNumber strategy, Oid collation, RegProcedure opcode, Const *value, bool is_null_check, bool is_null, bool is_array_op) { BatchFilter *segment_filter = palloc0(sizeof(*segment_filter)); *segment_filter = (BatchFilter){ .strategy = strategy, .collation = collation, .opcode = opcode, .value = value, .is_null_check = is_null_check, .is_null = is_null, .is_array_op = is_array_op, }; namestrcpy(&segment_filter->column_name, column_name); return segment_filter; } /* * A compressed chunk can have multiple indexes. For a given list * of columns in index_filters, find the matching index which has * the most columns based on index_filters and adjust the filters * if necessary. * Return matching index if found else return NULL. * * Note: This method will find the best matching index based on * number of filters it matches. If an index matches all the filters, * it will be chosen. Otherwise, it will try to select the index * which has most matches. If there are multiple indexes have * the same number of matches, it will pick the first one it finds. * For example * for a given condition like "WHERE X = 10 AND Y = 8" * if there are multiple indexes like * 1. index (a,b,c,x) * 2. index (a,x,y) * 3. index (x) * In this case 2nd index is returned. If that one didn't exist, * it would return the 1st index. */ static Relation find_matching_index(Relation comp_chunk_rel, List **index_filters, List **heap_filters) { List *index_oids; ListCell *lc; int total_filters = list_length(*index_filters); int max_match_count = 0; Relation result_rel = NULL; /* get list of indexes defined on compressed chunk */ index_oids = RelationGetIndexList(comp_chunk_rel); foreach (lc, index_oids) { int match_count = 0; Relation index_rel = index_open(lfirst_oid(lc), AccessShareLock); IndexInfo *index_info = BuildIndexInfo(index_rel); /* Can't use partial or expression indexes */ if (index_info->ii_Predicate != NIL || index_info->ii_Expressions != NIL) { index_close(index_rel, AccessShareLock); continue; } /* Can only use Btree indexes */ if (index_info->ii_Am != BTREE_AM_OID) { index_close(index_rel, AccessShareLock); continue; } ListCell *li; foreach (li, *index_filters) { for (int i = 0; i < index_rel->rd_index->indnatts; i++) { AttrNumber attnum = index_rel->rd_index->indkey.values[i]; char *attname = get_attname(RelationGetRelid(comp_chunk_rel), attnum, false); BatchFilter *sf = lfirst(li); /* ensure column exists in index relation */ if (!strcmp(attname, NameStr(sf->column_name))) { match_count++; break; } } } if (match_count == total_filters) { /* found index which has all columns specified in WHERE */ if (result_rel) index_close(result_rel, AccessShareLock); if (ts_guc_debug_compression_path_info) elog(INFO, "Index \"%s\" is used for scan. ", RelationGetRelationName(index_rel)); return index_rel; } if (match_count > max_match_count) { max_match_count = match_count; result_rel = index_rel; continue; } index_close(index_rel, AccessShareLock); } /* No matching index whatsoever */ if (!result_rel) { *heap_filters = list_concat(*heap_filters, *index_filters); *index_filters = list_truncate(*index_filters, 0); return NULL; } /* We found an index which matches partially. * It can be used but we need to transfer the unmatched * filters from index_filters to heap filters. */ for (int i = 0; i < list_length(*index_filters); i++) { BatchFilter *sf = list_nth(*index_filters, i); bool match = false; for (int j = 0; j < result_rel->rd_index->indnatts; j++) { AttrNumber attnum = result_rel->rd_index->indkey.values[j]; char *attname = get_attname(RelationGetRelid(comp_chunk_rel), attnum, false); /* ensure column exists in index relation */ if (!strcmp(attname, NameStr(sf->column_name))) { match = true; break; } } if (!match) { *heap_filters = lappend(*heap_filters, sf); *index_filters = list_delete_nth_cell(*index_filters, i); } } if (ts_guc_debug_compression_path_info) elog(INFO, "Index \"%s\" is used for scan. ", RelationGetRelationName(result_rel)); return result_rel; } static void report_error(TM_Result result) { switch (result) { case TM_Deleted: { if (IsolationUsesXactSnapshot()) { /* For Repeatable Read isolation level report error */ ereport(ERROR, (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("could not serialize access due to concurrent update"))); } } break; /* * If another transaction is updating the compressed data, * we have to abort the transaction to keep consistency. */ case TM_Updated: { elog(ERROR, "tuple concurrently updated"); } break; case TM_Invisible: { elog(ERROR, "attempted to lock invisible tuple"); } break; case TM_Ok: break; default: { elog(ERROR, "unexpected tuple operation result: %d", result); } break; } } /* * If key_columns are including all unique constraint columns and NULLS * are not DISTINCT any NULL value in the key columns allows us to skip * finding matching batches as it will not create a constraint violation. */ static bool key_column_is_null(tuple_filtering_constraints *constraints, Relation chunk_rel, Oid ht_relid, TupleTableSlot *slot) { if (!constraints->covered || constraints->nullsnotdistinct) return false; AttrNumber chunk_attno = -1; while ((chunk_attno = bms_next_member(constraints->key_columns, chunk_attno)) > 0) { /* * slot has the physical layout of the hypertable, so we need to * get the attribute number of the hypertable for the column. */ const NameData *attname = attnumAttName(chunk_rel, chunk_attno); AttrNumber ht_attno = get_attnum(ht_relid, NameStr(*attname)); if (slot_attisnull(slot, ht_attno)) return true; } return false; } static bool can_delete_without_decompression(ModifyHypertableState *ht_state, CompressionSettings *settings, Chunk *chunk, List *predicates) { ListCell *lc; if (!ts_guc_enable_compressed_direct_batch_delete) return false; /* * If there is a RETURNING clause we skip the optimization to delete compressed batches directly */ if (ht_state->mt->returningLists) return false; /* * If there are any DELETE row triggers on the hypertable we skip the optimization * to delete compressed batches directly. */ ModifyTableState *ps = linitial_node(ModifyTableState, castNode(CustomScanState, ht_state)->custom_ps); if (ps->rootResultRelInfo->ri_TrigDesc) { TriggerDesc *trigdesc = ps->rootResultRelInfo->ri_TrigDesc; if (trigdesc->trig_delete_before_row || trigdesc->trig_delete_after_row || trigdesc->trig_delete_instead_row) { return false; } } foreach (lc, predicates) { Node *node = lfirst(lc); Var *var; Expr *arg_value; Oid opno; if (ts_extract_expr_args((Expr *) node, &var, &arg_value, &opno, NULL)) { if (!IsA(arg_value, Const)) { return false; } char *column_name = get_attname(chunk->table_id, var->varattno, false); /* Can do direct DELETE if we are dealing with segmentby columns */ if (ts_array_is_member(settings->fd.segmentby, column_name)) continue; /* Can do direct DELETE if we are using in-memory filtering but * only if we can actually create scankeys for filtering */ if (ts_guc_enable_dml_decompression_tuple_filtering) { switch (nodeTag(node)) { case T_ScalarArrayOpExpr: case T_NullTest: return false; default: continue; } } } return false; } return true; } static bool can_vectorize_constraint_checks(tuple_filtering_constraints *constraints, CompressionSettings *settings, Relation chunk_rel, Oid ht_relid, ScanKeyWithAttnos *mem_scankeys) { AttrNumber chunk_attno = -1; Oid typoid, collid; int32 typmod; if (mem_scankeys == NULL || mem_scankeys->num_scankeys == 0) return false; /* We can only vectorize if a vectorized check is available for all scankeys */ for (int sk = 0; sk < mem_scankeys->num_scankeys; sk++) { /* * Here we cannot check for NULL flags even if that is * handled separately, because this code is called from * the `init_decompress_state_for_insert` which sets the * flag based on the first record to be inserted and the * value may change for the subsequent records. * * The `fn_oid` doesn't get updated so it is valid to check * it here. */ ScanKeyData *scankey = &mem_scankeys->scankeys[sk]; if (get_vector_const_predicate(scankey->sk_func.fn_oid) == NULL) return false; } while ((chunk_attno = bms_next_member(constraints->key_columns, chunk_attno)) > 0) { /* * slot has the physical layout of the hypertable, so we need to * get the attribute number of the hypertable for the column. */ char *attname = get_attname(chunk_rel->rd_id, chunk_attno, false); /* Ignore segmentby columns, they aren't compressed */ if (ts_array_is_member(settings->fd.segmentby, attname)) continue; get_atttypetypmodcoll(chunk_rel->rd_id, chunk_attno, &typoid, &typmod, &collid); /* No bulk decompression function, no vectorized filtering */ if (tsl_get_decompress_all_function(compression_get_default_algorithm(typoid), typoid) == NULL) return false; /* For text types, check for non-deterministic collation which * prevents vectorized filtering */ if (typoid == TEXTOID && OidIsValid(collid) && !get_collation_isdeterministic(collid)) return false; } return true; } ================================================ FILE: tsl/src/compression/compression_dml.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <access/skey.h> #include <nodes/nodes.h> #include "ts_catalog/compression_settings.h" typedef struct tuple_filtering_constraints { /* * All key column heap attribute numbers on uncompressed chunk. * We shouldn't be dealing with system columns so no need to * add/subtract FirstLowInvalidHeapAttributeNumber from these. */ Bitmapset *key_columns; /* * The covered flag is set to true if we have a single constraint that is covered * by all the columns present in the Bitmapset. */ bool covered; /* further fields only valid when covered is true */ OnConflictAction on_conflict; Oid index_relid; /* used for better error messages */ bool nullsnotdistinct; bool vectorized_filtering; } tuple_filtering_constraints; bool slot_key_test(TupleTableSlot *slot, ScanKey skey); ScanKeyData *build_mem_scankeys_from_slot(Oid ht_relid, CompressionSettings *settings, Relation out_rel, tuple_filtering_constraints *constraints, TupleTableSlot *slot, int *num_scankeys, AttrNumber **slot_attnos); ScanKeyData *build_index_scankeys(Relation index_rel, List *index_filters, int *num_scankeys); ScanKeyData *build_index_scankeys_using_slot(Oid hypertable_relid, Relation in_rel, Relation out_rel, Bitmapset *key_columns, TupleTableSlot *slot, Relation *result_index_rel, Bitmapset **index_columns, int *num_scan_keys, AttrNumber **slot_attnos); ScanKeyData *build_heap_scankeys(Oid hypertable_relid, Relation in_rel, Relation out_rel, CompressionSettings *settings, Bitmapset *key_columns, Bitmapset **null_columns, TupleTableSlot *slot, int *num_scankeys, AttrNumber **slot_attnos); ScanKeyData *build_update_delete_scankeys(Relation in_rel, List *heap_filters, int *num_scankeys, Bitmapset **null_columns, bool *delete_only); ================================================ FILE: tsl/src/compression/compression_scankey.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_am.h> #include <parser/parse_coerce.h> #include <parser/parse_relation.h> #include <utils/typcache.h> #include "compression.h" #include "compression_dml.h" #include "create.h" #include "ts_catalog/array_utils.h" static Oid deduce_filter_subtype(BatchFilter *filter, Oid att_typoid); static bool create_segment_filter_scankey(Relation in_rel, char *segment_filter_col_name, StrategyNumber strategy, Oid subtype, Oid opcode, ScanKeyData *scankeys, int *num_scankeys, Bitmapset **null_columns, Datum value, bool is_null_check, bool is_array_op); /* * Test ScanKey against a slot. * * Unlike HeapKeyTest, this function takes into account SK_ISNULL * and works correctly when looking for null values. */ bool slot_key_test(TupleTableSlot *compressed_slot, ScanKey key) { /* No need to get the datum if we are only checking for NULLs */ if (key->sk_flags & SK_ISNULL) { return slot_attisnull(compressed_slot, key->sk_attno); } Datum val; bool is_null; val = slot_getattr(compressed_slot, key->sk_attno, &is_null); if (is_null) return false; return DatumGetBool(FunctionCall2Coll(&key->sk_func, key->sk_collation, val, key->sk_argument)); } /* * Build scankeys for decompressed tuple to check if it is part of the batch. * * The key_columns are the columns of the uncompressed chunk. */ ScanKeyData * build_mem_scankeys_from_slot(Oid ht_relid, CompressionSettings *settings, Relation out_rel, tuple_filtering_constraints *constraints, TupleTableSlot *slot, int *num_scankeys, AttrNumber **slot_attnos) { ScanKeyData *scankeys = NULL; int key_index = 0; TupleDesc out_desc = RelationGetDescr(out_rel); if (bms_is_empty(constraints->key_columns)) { *num_scankeys = key_index; return scankeys; } int max_key_columns = bms_num_members(constraints->key_columns); scankeys = palloc(sizeof(ScanKeyData) * max_key_columns); *slot_attnos = palloc0(sizeof(AttrNumber) * max_key_columns); AttrNumber attno = -1; while ((attno = bms_next_member(constraints->key_columns, attno)) > 0) { bool isnull; /* * slot has the physical layout of the hypertable, so we need to * get the attribute number of the hypertable for the column. */ char *attname = get_attname(out_rel->rd_id, attno, false); /* * We can skip any segmentby columns here since they have already been * checked during batch filtering. */ if (ts_array_is_member(settings->fd.segmentby, attname)) { continue; } AttrNumber ht_attno = get_attnum(ht_relid, attname); Datum value = slot_getattr(slot, ht_attno, &isnull); (*slot_attnos)[key_index] = ht_attno; Oid atttypid = TupleDescAttr(out_desc, AttrNumberGetAttrOffset(attno))->atttypid; TypeCacheEntry *tce = lookup_type_cache(atttypid, TYPECACHE_BTREE_OPFAMILY); /* * Should never happen since the column is part of unique constraint * and should therefore have the required opfamily */ if (!OidIsValid(tce->btree_opf)) elog(ERROR, "no btree opfamily for type \"%s\"", format_type_be(atttypid)); Oid opr = get_opfamily_member(tce->btree_opf, atttypid, atttypid, BTEqualStrategyNumber); /* * Fall back to btree operator input type when it is binary compatible with * the column type and no operator for column type could be found. */ if (!OidIsValid(opr) && IsBinaryCoercible(atttypid, tce->btree_opintype)) { opr = get_opfamily_member(tce->btree_opf, tce->btree_opintype, tce->btree_opintype, BTEqualStrategyNumber); } if (!OidIsValid(opr)) elog(ERROR, "no operator found for type \"%s\"", format_type_be(atttypid)); ScanKeyEntryInitialize(&scankeys[key_index++], isnull ? SK_ISNULL : 0, attno, BTEqualStrategyNumber, atttypid, TupleDescAttr(out_desc, AttrNumberGetAttrOffset(attno)) ->attcollation, get_opcode(opr), isnull ? 0 : value); } *num_scankeys = key_index; return scankeys; } /* * Build scankeys for decompression of specific batches. key_columns references the * columns of the uncompressed chunk. */ ScanKeyData * build_heap_scankeys(Oid hypertable_relid, Relation in_rel, Relation out_rel, CompressionSettings *settings, Bitmapset *key_columns, Bitmapset **null_columns, TupleTableSlot *slot, int *num_scankeys, AttrNumber **slot_attnos) { int key_index = 0; ScanKeyData *scankeys = NULL; if (!bms_is_empty(key_columns)) { int max_key_columns = bms_num_members(key_columns) * 2; scankeys = palloc0(max_key_columns * sizeof(ScanKeyData)); *slot_attnos = palloc0(max_key_columns * sizeof(AttrNumber)); AttrNumber attno = -1; while ((attno = bms_next_member(key_columns, attno)) > 0) { char *attname = get_attname(out_rel->rd_id, attno, false); bool isnull; AttrNumber ht_attno = get_attnum(hypertable_relid, attname); /* * This is a not very precise but easy assertion to detect attno * mismatch at least in some cases. The mismatch might happen if the * hypertable and chunk layout are different because of dropped * columns, and we're using a wrong slot type here. */ PG_USED_FOR_ASSERTS_ONLY Oid ht_atttype = get_atttype(hypertable_relid, ht_attno); PG_USED_FOR_ASSERTS_ONLY Oid slot_atttype = TupleDescAttr(slot->tts_tupleDescriptor, AttrNumberGetAttrOffset(ht_attno)) ->atttypid; Assert(ht_atttype == slot_atttype); Datum value = slot_getattr(slot, ht_attno, &isnull); /* * There are 3 possible scenarios we have to consider * when dealing with columns which are part of unique * constraints. * * 1. Column is segmentby-Column * In this case we can add a single ScanKey with an * equality check for the value. * 2. Column is orderby-Column * In this we can add 2 ScanKeys with range constraints * utilizing batch metadata. * 3. Column is neither segmentby nor orderby * In this case we cannot utilize this column for * batch filtering as the values are compressed and * we have no metadata. */ if (ts_array_is_member(settings->fd.segmentby, attname)) { if (create_segment_filter_scankey(in_rel, attname, BTEqualStrategyNumber, InvalidOid, InvalidOid, scankeys, &key_index, null_columns, value, isnull, false)) { (*slot_attnos)[key_index - 1] = ht_attno; } } if (ts_array_is_member(settings->fd.orderby, attname)) { /* Cannot optimize orderby columns with NULL values since those * are not visible in metadata */ if (isnull) continue; int16 index = ts_array_position(settings->fd.orderby, attname); if (create_segment_filter_scankey(in_rel, column_segment_min_name(index), BTLessEqualStrategyNumber, InvalidOid, InvalidOid, scankeys, &key_index, null_columns, value, false, false /* is_null_check */ )) { (*slot_attnos)[key_index - 1] = ht_attno; } if (create_segment_filter_scankey(in_rel, column_segment_max_name(index), BTGreaterEqualStrategyNumber, InvalidOid, InvalidOid, scankeys, &key_index, null_columns, value, false, false /* is_null_check */ )) { (*slot_attnos)[key_index - 1] = ht_attno; } } } } *num_scankeys = key_index; return scankeys; } /* * This method will build scan keys required to do index * scans on compressed chunks. */ ScanKeyData * build_index_scankeys(Relation index_rel, List *index_filters, int *num_scankeys) { ListCell *lc; BatchFilter *filter = NULL; *num_scankeys = list_length(index_filters); ScanKeyData *scankey = palloc0(sizeof(ScanKeyData) * (*num_scankeys)); int idx = 0; int flags; /* Order scankeys based on index attribute order */ for (int idx_attno = 1; idx_attno <= index_rel->rd_index->indnkeyatts && idx < *num_scankeys; idx_attno++) { AttrNumber attno = index_rel->rd_index->indkey.values[AttrNumberGetAttrOffset(idx_attno)]; char *attname = get_attname(index_rel->rd_index->indrelid, attno, false); Oid typoid = attnumTypeId(index_rel, idx_attno); foreach (lc, index_filters) { filter = lfirst(lc); if (!strcmp(attname, NameStr(filter->column_name))) { flags = 0; if (filter->is_null_check) { flags = SK_ISNULL | (filter->is_null ? SK_SEARCHNULL : SK_SEARCHNOTNULL); } if (filter->is_array_op) { flags |= SK_SEARCHARRAY; } ScanKeyEntryInitialize(&scankey[idx++], flags, idx_attno, filter->strategy, deduce_filter_subtype(filter, typoid), /* subtype */ filter->collation, filter->opcode, filter->value ? filter->value->constvalue : 0); } } } Assert(idx == *num_scankeys); return scankey; } /* This method is used to find matching index on compressed chunk * and build scan keys from the slot data */ ScanKeyData * build_index_scankeys_using_slot(Oid hypertable_relid, Relation in_rel, Relation out_rel, Bitmapset *key_columns, TupleTableSlot *slot, Relation *result_index_rel, Bitmapset **index_columns, int *num_scan_keys, AttrNumber **slot_attnos) { List *index_oids; ListCell *lc; ScanKeyData *scankeys = NULL; /* get list of indexes defined on compressed chunk */ index_oids = RelationGetIndexList(in_rel); *num_scan_keys = 0; foreach (lc, index_oids) { Relation index_rel = index_open(lfirst_oid(lc), AccessShareLock); IndexInfo *index_info = BuildIndexInfo(index_rel); /* Can't use partial or expression indexes */ if (index_info->ii_Predicate != NIL || index_info->ii_Expressions != NIL) { index_close(index_rel, AccessShareLock); continue; } /* Can only use Btree indexes */ if (index_info->ii_Am != BTREE_AM_OID) { index_close(index_rel, AccessShareLock); continue; } /* * Must have at least two attributes, index we are looking for contains * at least one segmentby column and a sequence number. */ if (index_rel->rd_index->indnatts < 2) { index_close(index_rel, AccessShareLock); continue; } scankeys = palloc0((index_rel->rd_index->indnatts) * sizeof(ScanKeyData)); *slot_attnos = palloc0((index_rel->rd_index->indnatts) * sizeof(AttrNumber)); /* * Using only key attributes to exclude covering columns * only interested in filtering here */ for (int i = 0; i < index_rel->rd_index->indnkeyatts; i++) { AttrNumber idx_attnum = AttrOffsetGetAttrNumber(i); AttrNumber in_attnum = index_rel->rd_index->indkey.values[i]; const NameData *attname = attnumAttName(in_rel, in_attnum); AttrNumber column_attno = get_attnum(out_rel->rd_id, NameStr(*attname)); /* Make sure we find columns in key columns in order to select the right index */ if (!bms_is_member(column_attno, key_columns)) { break; } bool isnull; AttrNumber ht_attno = get_attnum(hypertable_relid, NameStr(*attname)); Datum value = slot_getattr(slot, ht_attno, &isnull); (*slot_attnos)[*num_scan_keys] = ht_attno; Oid atttypid = attnumTypeId(index_rel, idx_attnum); TypeCacheEntry *tce = lookup_type_cache(atttypid, TYPECACHE_BTREE_OPFAMILY); if (!OidIsValid(tce->btree_opf)) elog(ERROR, "no btree opfamily for type \"%s\"", format_type_be(atttypid)); Oid opr = get_opfamily_member(tce->btree_opf, atttypid, atttypid, BTEqualStrategyNumber); /* * Fall back to btree operator input type when it is binary compatible with * the column type and no operator for column type could be found. */ if (!OidIsValid(opr) && IsBinaryCoercible(atttypid, tce->btree_opintype)) { opr = get_opfamily_member(tce->btree_opf, tce->btree_opintype, tce->btree_opintype, BTEqualStrategyNumber); } /* No operator could be found so we can't create the scankey. */ if (!OidIsValid(opr)) continue; Oid opcode = get_opcode(opr); Ensure(OidIsValid(opcode), "no opcode found for column operator of a hypertable column"); *index_columns = bms_add_member(*index_columns, column_attno); ScanKeyEntryInitialize(&scankeys[(*num_scan_keys)++], isnull ? SK_ISNULL | SK_SEARCHNULL : 0, /* flags */ idx_attnum, BTEqualStrategyNumber, InvalidOid, /* No strategy subtype. */ attnumCollationId(index_rel, idx_attnum), opcode, isnull ? 0 : value); } if (*num_scan_keys > 0) { *result_index_rel = index_rel; break; } else { index_close(index_rel, AccessShareLock); pfree(scankeys); scankeys = NULL; } } return scankeys; } /* * This method will build scan keys for predicates including * SEGMENT BY column with attribute number from compressed chunk * if condition is like <segmentbycol> = <const value>, else * OUT param null_columns is saved with column attribute number. */ ScanKeyData * build_update_delete_scankeys(Relation in_rel, List *heap_filters, int *num_scankeys, Bitmapset **null_columns, bool *delete_only) { ListCell *lc; BatchFilter *filter; int key_index = 0; ScanKeyData *scankeys = palloc0(heap_filters->length * sizeof(ScanKeyData)); foreach (lc, heap_filters) { filter = lfirst(lc); AttrNumber attno = get_attnum(in_rel->rd_id, NameStr(filter->column_name)); Oid typoid = get_atttype(in_rel->rd_id, attno); if (attno == InvalidAttrNumber) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("column \"%s\" of relation \"%s\" does not exist", NameStr(filter->column_name), RelationGetRelationName(in_rel)))); bool added = create_segment_filter_scankey(in_rel, NameStr(filter->column_name), filter->strategy, deduce_filter_subtype(filter, typoid), filter->opcode, scankeys, &key_index, null_columns, filter->value ? filter->value->constvalue : 0, filter->is_null_check, filter->is_array_op); /* * When we plan to DELETE directly on compressed chunks we * need to ensure all query constraints could be applied * to the compressed scan and disable direct DELETE when * we are skipping filters. */ if (*delete_only && !added) *delete_only = false; } *num_scankeys = key_index; return scankeys; } static bool create_segment_filter_scankey(Relation in_rel, char *segment_filter_col_name, StrategyNumber strategy, Oid subtype, Oid opcode, ScanKeyData *scankeys, int *num_scankeys, Bitmapset **null_columns, Datum value, bool is_null_check, bool is_array_op) { AttrNumber cmp_attno = get_attnum(in_rel->rd_id, segment_filter_col_name); Assert(cmp_attno != InvalidAttrNumber); /* This should never happen but if it does happen, we can't generate a scan key for * the filter column so just skip it */ if (cmp_attno == InvalidAttrNumber) return false; int flags = is_array_op ? SK_SEARCHARRAY : 0; /* * In PG versions <= 14 NULL values are always considered distinct * from other NULL values and therefore NULLABLE multi-columnn * unique constraints might expose unexpected behaviour in the * presence of NULL values. * Since SK_SEARCHNULL is not supported by heap scans we cannot * build a ScanKey for NOT NULL and instead have to do those * checks manually. */ if (is_null_check) { *null_columns = bms_add_member(*null_columns, cmp_attno); return false; } Oid opr; /* * All btree operators will have a valid strategy here. For * non-btree operators e.g. <> we directly take the opcode * here. We could do the same for btree in certain cases * but some filters get transformed to min/max filters and * won't keep the initial opcode so we would need to disambiguate * between them. */ if (strategy == InvalidStrategy) { opr = opcode; } else { Oid atttypid = TupleDescAttr(in_rel->rd_att, AttrNumberGetAttrOffset(cmp_attno))->atttypid; TypeCacheEntry *tce = lookup_type_cache(atttypid, TYPECACHE_BTREE_OPFAMILY); if (!OidIsValid(tce->btree_opf)) elog(ERROR, "no btree opfamily for type \"%s\"", format_type_be(atttypid)); opr = get_opfamily_member(tce->btree_opf, atttypid, atttypid, strategy); /* * Fall back to btree operator input type when it is binary compatible with * the column type and no operator for column type could be found. */ if (!OidIsValid(opr) && IsBinaryCoercible(atttypid, tce->btree_opintype)) { opr = get_opfamily_member(tce->btree_opf, tce->btree_opintype, tce->btree_opintype, strategy); } /* No operator could be found so we can't create the scankey. */ if (!OidIsValid(opr)) return false; opr = get_opcode(opr); } /* We should never end up here but: no opcode, no optimization */ if (!OidIsValid(opr)) return false; ScanKeyEntryInitialize(&scankeys[(*num_scankeys)++], flags, cmp_attno, strategy, subtype, TupleDescAttr(in_rel->rd_att, AttrNumberGetAttrOffset(cmp_attno)) ->attcollation, opr, value); return true; } /* * Get the subtype for an indexscan from the provided filter. We also * need to handle array constants appropriately. */ static Oid deduce_filter_subtype(BatchFilter *filter, Oid att_typoid) { Oid subtype = InvalidOid; if (!filter->value) return InvalidOid; /* * Check if the filter type is different from the att type. If yes, the * subtype needs to be set appropriately. */ if (att_typoid != filter->value->consttype) { /* For an array type get its element type */ if (filter->is_array_op) subtype = get_element_type(filter->value->consttype); else subtype = filter->value->consttype; } return subtype; } ================================================ FILE: tsl/src/compression/compression_storage.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * This file contains functions for manipulating compression related * internal storage objects like creating the underlying tables and * setting storage options. */ #include <postgres.h> #include <access/reloptions.h> #include <access/xact.h> #include <catalog/indexing.h> #include <catalog/objectaccess.h> #include <catalog/objectaddress.h> #include <catalog/pg_class.h> #include <catalog/pg_constraint.h> #include <catalog/toasting.h> #include <commands/tablecmds.h> #include <commands/tablespace.h> #include <nodes/makefuncs.h> #include <utils/builtins.h> #include <utils/lsyscache.h> #include <utils/syscache.h> #include "compression.h" #include "compression_storage.h" #include "create.h" #include "custom_type_cache.h" #include "extension_constants.h" #include "guc.h" #include "hypertable.h" #include "ts_catalog/array_utils.h" #include "ts_catalog/catalog.h" #include "ts_catalog/compression_settings.h" #include "utils.h" #define PRINT_COMPRESSION_TABLE_NAME(buf, prefix, hypertable_id) \ do \ { \ int ret = snprintf(buf, NAMEDATALEN, prefix, hypertable_id); \ if (ret < 0 || ret > NAMEDATALEN) \ { \ ereport(ERROR, \ (errcode(ERRCODE_INTERNAL_ERROR), \ errmsg("bad compression hypertable internal name"))); \ } \ } while (0); static void set_toast_tuple_target_on_chunk(Oid compressed_table_id); static void set_statistics_on_compressed_chunk(Oid compressed_table_id); int32 compression_hypertable_create(Hypertable *ht, Oid owner, Oid tablespace_oid) { ObjectAddress tbladdress; char relnamebuf[NAMEDATALEN]; CatalogSecurityContext sec_ctx; Oid compress_relid; CreateStmt *create; RangeVar *compress_rel; int32 compress_hypertable_id; Assert(!TS_HYPERTABLE_HAS_COMPRESSION_TABLE(ht)); create = makeNode(CreateStmt); create->tableElts = NIL; create->inhRelations = NIL; create->ofTypename = NULL; create->constraints = NIL; create->options = NULL; create->oncommit = ONCOMMIT_NOOP; create->tablespacename = get_tablespace_name(tablespace_oid); create->if_not_exists = false; /* Invalid tablespace_oid <=> NULL tablespace name */ Assert(!OidIsValid(tablespace_oid) == (create->tablespacename == NULL)); /* create the compression table */ /* NewRelationCreateToastTable calls CommandCounterIncrement */ ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); compress_hypertable_id = ts_catalog_table_next_seq_id(ts_catalog_get(), HYPERTABLE); PRINT_COMPRESSION_TABLE_NAME(relnamebuf, "_compressed_hypertable_%d", compress_hypertable_id); compress_rel = makeRangeVar(pstrdup(INTERNAL_SCHEMA_NAME), pstrdup(relnamebuf), -1); create->relation = compress_rel; tbladdress = DefineRelation(create, RELKIND_RELATION, owner, NULL, NULL); CommandCounterIncrement(); compress_relid = tbladdress.objectId; ts_copy_relation_acl(ht->main_table_relid, compress_relid, owner); ts_catalog_restore_user(&sec_ctx); ts_hypertable_create_compressed(compress_relid, compress_hypertable_id); return compress_hypertable_id; } Oid compression_chunk_create(Chunk *src_chunk, Chunk *chunk, List *column_defs, Oid tablespace_oid, CompressionSettings *settings) { ObjectAddress tbladdress; CatalogSecurityContext sec_ctx; Datum toast_options; #if PG18_LT char *validnsps[] = HEAP_RELOPT_NAMESPACES; #else const char *const validnsps[] = HEAP_RELOPT_NAMESPACES; #endif Oid owner = ts_rel_get_owner(chunk->hypertable_relid); CreateStmt *create; RangeVar *compress_rel; create = makeNode(CreateStmt); create->tableElts = column_defs; create->inhRelations = NIL; create->ofTypename = NULL; create->constraints = NIL; create->options = NULL; create->oncommit = ONCOMMIT_NOOP; create->tablespacename = get_tablespace_name(tablespace_oid); create->if_not_exists = false; /* Invalid tablespace_oid <=> NULL tablespace name */ Assert(!OidIsValid(tablespace_oid) == (create->tablespacename == NULL)); /* create the compression table */ /* NewRelationCreateToastTable calls CommandCounterIncrement */ ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); compress_rel = makeRangeVar(NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name), -1); create->relation = compress_rel; /* Inherit the persistence (LOGGED or UNLOGGED) from the uncompressed chunk */ create->relation->relpersistence = get_rel_persistence(src_chunk->table_id); tbladdress = DefineRelation(create, RELKIND_RELATION, owner, NULL, NULL); CommandCounterIncrement(); chunk->table_id = tbladdress.objectId; ts_copy_relation_acl(chunk->hypertable_relid, chunk->table_id, owner); toast_options = transformRelOptions((Datum) 0, create->options, "toast", validnsps, true, false); (void) heap_reloptions(RELKIND_TOASTVALUE, toast_options, true); NewRelationCreateToastTable(chunk->table_id, toast_options); modify_compressed_toast_table_storage(settings, column_defs, chunk->table_id); set_statistics_on_compressed_chunk(chunk->table_id); set_toast_tuple_target_on_chunk(chunk->table_id); ts_catalog_restore_user(&sec_ctx); create_compressed_chunk_indexes(chunk, settings); return chunk->table_id; } static void set_toast_tuple_target_on_chunk(Oid compressed_table_id) { DefElem def_elem = { .type = T_DefElem, .defname = "toast_tuple_target", .arg = (Node *) makeInteger(ts_guc_debug_toast_tuple_target), .defaction = DEFELEM_SET, .location = -1, }; AlterTableCmd cmd = { .type = T_AlterTableCmd, .subtype = AT_SetRelOptions, .def = (Node *) list_make1(&def_elem), }; AlterTableInternal(compressed_table_id, list_make1(&cmd), true); } static void set_statistics_on_compressed_chunk(Oid compressed_table_id) { Relation table_rel = table_open(compressed_table_id, ShareUpdateExclusiveLock); Relation attrelation = table_open(AttributeRelationId, RowExclusiveLock); TupleDesc table_desc = RelationGetDescr(table_rel); Oid compressed_data_type = ts_custom_type_cache_get(CUSTOM_TYPE_COMPRESSED_DATA)->type_oid; for (int i = 0; i < table_desc->natts; i++) { Form_pg_attribute attrtuple; HeapTuple tuple; Form_pg_attribute col_attr = TupleDescAttr(table_desc, i); Datum repl_val[Natts_pg_attribute] = { 0 }; bool repl_null[Natts_pg_attribute] = { false }; bool repl_repl[Natts_pg_attribute] = { false }; /* skip system columns */ if (col_attr->attnum <= 0) continue; tuple = SearchSysCacheCopyAttName(RelationGetRelid(table_rel), NameStr(col_attr->attname)); if (!HeapTupleIsValid(tuple)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_COLUMN), errmsg("column \"%s\" of compressed table \"%s\" does not exist", NameStr(col_attr->attname), RelationGetRelationName(table_rel)))); attrtuple = (Form_pg_attribute) GETSTRUCT(tuple); /* The planner should never look at compressed column statistics because * it will not understand them. Statistics on the other columns, * segmentbys and metadata, are very important, so we increase their * target. */ if (col_attr->atttypid == compressed_data_type) repl_val[AttrNumberGetAttrOffset(Anum_pg_attribute_attstattarget)] = Int16GetDatum(0); else repl_val[AttrNumberGetAttrOffset(Anum_pg_attribute_attstattarget)] = Int16GetDatum(1000); repl_repl[AttrNumberGetAttrOffset(Anum_pg_attribute_attstattarget)] = true; tuple = heap_modify_tuple(tuple, RelationGetDescr(attrelation), repl_val, repl_null, repl_repl); CatalogTupleUpdate(attrelation, &tuple->t_self, tuple); InvokeObjectPostAlterHook(RelationRelationId, RelationGetRelid(table_rel), attrtuple->attnum); heap_freetuple(tuple); } table_close(attrelation, NoLock); table_close(table_rel, NoLock); } /* modify storage attributes for toast table columns attached to the * compression table */ void modify_compressed_toast_table_storage(CompressionSettings *settings, List *coldefs, Oid compress_relid) { ListCell *lc; List *cmds = NIL; Oid compresseddata_oid = ts_custom_type_cache_get(CUSTOM_TYPE_COMPRESSED_DATA)->type_oid; foreach (lc, coldefs) { ColumnDef *cd = lfirst_node(ColumnDef, lc); AttrNumber attno = get_attnum(compress_relid, cd->colname); if (attno != InvalidAttrNumber && get_atttype(compress_relid, attno) == compresseddata_oid) { /* * All columns that pass the datatype check are columns * that are also present in the uncompressed hypertable. * Metadata columns are missing from the uncompressed * hypertable but they do not have compresseddata datatype * and therefore would be skipped. */ attno = get_attnum(settings->fd.relid, cd->colname); Assert(attno != InvalidAttrNumber); Oid typid = get_atttype(settings->fd.relid, attno); CompressionStorage stor = compression_get_toast_storage(compression_get_default_algorithm(typid)); if (stor != TOAST_STORAGE_EXTERNAL) /* external is default storage for toast columns */ { AlterTableCmd *cmd = makeNode(AlterTableCmd); cmd->subtype = AT_SetStorage; cmd->name = pstrdup(cd->colname); Assert(stor == TOAST_STORAGE_EXTENDED); cmd->def = (Node *) makeString("extended"); cmds = lappend(cmds, cmd); } } } if (cmds != NIL) { AlterTableInternal(compress_relid, cmds, false); } } void create_compressed_chunk_indexes(Chunk *chunk, CompressionSettings *settings) { IndexStmt stmt = { .type = T_IndexStmt, .accessMethod = DEFAULT_INDEX_TYPE, .idxname = NULL, .relation = makeRangeVar(NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name), 0), .tableSpace = get_tablespace_name(get_rel_tablespace(chunk->table_id)), }; NameData index_name; ObjectAddress index_addr; HeapTuple index_tuple; List *indexcols = NIL; StringInfoData buf; initStringInfo(&buf); if (settings->fd.segmentby) { Datum datum; bool isnull; ArrayIterator it = array_create_iterator(settings->fd.segmentby, 0, NULL); while (array_iterate(it, &datum, &isnull)) { IndexElem *segment_elem = makeNode(IndexElem); segment_elem->name = TextDatumGetCString(datum); appendStringInfoString(&buf, segment_elem->name); appendStringInfoString(&buf, ", "); indexcols = lappend(indexcols, segment_elem); } } SortByDir ordering; SortByNulls nulls_ordering; StringInfoData orderby_buf; initStringInfo(&orderby_buf); for (int i = 1; i <= ts_array_length(settings->fd.orderby); i++) { resetStringInfo(&orderby_buf); /* Add min metadata column */ IndexElem *orderby_min_elem = makeNode(IndexElem); orderby_min_elem->name = column_segment_min_name(i); if (ts_array_get_element_bool(settings->fd.orderby_desc, i)) { appendStringInfoString(&orderby_buf, " DESC"); ordering = SORTBY_DESC; } else { appendStringInfoString(&orderby_buf, " ASC"); ordering = SORTBY_ASC; } orderby_min_elem->ordering = ordering; if (ts_array_get_element_bool(settings->fd.orderby_nullsfirst, i)) { if (orderby_min_elem->ordering != SORTBY_DESC) { appendStringInfoString(&orderby_buf, " NULLS FIRST"); nulls_ordering = SORTBY_NULLS_FIRST; } else { nulls_ordering = SORTBY_NULLS_DEFAULT; } } else { if (orderby_min_elem->ordering != SORTBY_DESC) { nulls_ordering = SORTBY_NULLS_DEFAULT; } else { appendStringInfoString(&orderby_buf, " NULLS LAST"); nulls_ordering = SORTBY_NULLS_LAST; } } orderby_min_elem->nulls_ordering = nulls_ordering; appendStringInfoString(&buf, orderby_min_elem->name); appendStringInfoString(&buf, orderby_buf.data); appendStringInfoString(&buf, ", "); indexcols = lappend(indexcols, orderby_min_elem); /* Add max metadata column */ IndexElem *orderby_max_elem = makeNode(IndexElem); orderby_max_elem->name = column_segment_max_name(i); orderby_max_elem->ordering = orderby_min_elem->ordering; orderby_max_elem->nulls_ordering = orderby_min_elem->nulls_ordering; appendStringInfoString(&buf, orderby_max_elem->name); appendStringInfoString(&buf, orderby_buf.data); appendStringInfoString(&buf, ", "); indexcols = lappend(indexcols, orderby_max_elem); } stmt.indexParams = indexcols; index_addr = DefineIndexCompat(chunk->table_id, &stmt, InvalidOid, /* IndexRelationId */ InvalidOid, /* parentIndexId */ InvalidOid, /* parentConstraintId */ -1, /* total_parts */ false, /* is_alter_table */ false, /* check_rights */ false, /* check_not_in_use */ false, /* skip_build */ false); /* quiet */ index_tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(index_addr.objectId)); if (!HeapTupleIsValid(index_tuple)) elog(ERROR, "cache lookup failed for index relid %u", index_addr.objectId); index_name = ((Form_pg_class) GETSTRUCT(index_tuple))->relname; elog(DEBUG1, "adding index %s ON %s.%s USING BTREE(%s)", NameStr(index_name), NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name), buf.data); ReleaseSysCache(index_tuple); } ================================================ FILE: tsl/src/compression/compression_storage.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * This file contains functions for manipulating compression related * internal storage objects like creating the underlying tables and * setting storage options. */ #include <postgres.h> #include "chunk.h" #include "hypertable.h" int32 compression_hypertable_create(Hypertable *ht, Oid owner, Oid tablespace_oid); Oid compression_chunk_create(Chunk *src_chunk, Chunk *chunk, List *column_defs, Oid tablespace_oid, CompressionSettings *settings); void modify_compressed_toast_table_storage(CompressionSettings *settings, List *coldefs, Oid compress_relid); void create_compressed_chunk_indexes(Chunk *chunk, CompressionSettings *settings); ================================================ FILE: tsl/src/compression/create.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/heapam.h> #include <access/reloptions.h> #include <access/tupdesc.h> #include <access/xact.h> #include <catalog/index.h> #include <catalog/indexing.h> #include <catalog/objectaccess.h> #include <catalog/pg_am_d.h> #include <catalog/pg_constraint.h> #include <catalog/pg_constraint_d.h> #include <catalog/pg_type.h> #include <catalog/toasting.h> #include <commands/alter.h> #include <commands/defrem.h> #include <commands/tablecmds.h> #include <commands/tablespace.h> #include <common/md5.h> #include <executor/spi.h> #include <miscadmin.h> #include <nodes/makefuncs.h> #include <parser/parse_type.h> #include <storage/lmgr.h> #include <tcop/utility.h> #include <utils/array.h> #include <utils/builtins.h> #include <utils/datum.h> #include <utils/guc.h> #include <utils/rel.h> #include <utils/syscache.h> #include <utils/typcache.h> #include "compat/compat.h" #include "bgw_policy/policies_v2.h" #include "chunk.h" #include "chunk_index.h" #include "compression.h" #include "compression/compression_storage.h" #include "compression/sparse_index_bloom1.h" #include "create.h" #include "custom_type_cache.h" #include "dimension.h" #include "foreach_ptr.h" #include "guc.h" #include "hypertable_cache.h" #include "jsonb_utils.h" #include "trigger.h" #include "ts_catalog/array_utils.h" #include "ts_catalog/catalog.h" #include "ts_catalog/compression_settings.h" #include "ts_catalog/continuous_agg.h" #include "utils.h" #include "with_clause/alter_table_with_clause.h" #include "with_clause/create_table_with_clause.h" #include "bgw_policy/compression_api.h" #ifdef USE_ASSERT_CHECKING static const char *sparse_index_types[] = { "min", "max" }; static bool is_sparse_index_type(const char *type) { for (size_t i = 0; i < sizeof(sparse_index_types) / sizeof(sparse_index_types[0]); i++) { if (strcmp(sparse_index_types[i], type) == 0) { return true; } } if (strcmp(bloom1_column_prefix, type) == 0) { return true; } if (ts_guc_read_legacy_bloom1_v1 && strcmp("bloom1", type) == 0) { return true; } return false; } #endif static void validate_hypertable_for_compression(Hypertable *ht); static List *build_columndefs(CompressionSettings *settings, Oid src_reloid); static ColumnDef *build_columndef_singlecolumn(const char *colname, Oid typid); static void compression_settings_set_manually_for_create(Hypertable *ht, CompressionSettings *settings, WithClauseResult *with_clause_options); static void compression_settings_set_manually_for_alter(Hypertable *ht, CompressionSettings *settings, WithClauseResult *with_clause_options); static void create_default_composite_bloom(IndexInfo *index_info, Hypertable *ht, CompressionSettings *settings, JsonbParseState *parse_state, TsBmsList *sparse_index_columns, bool *has_object); static char * compression_column_segment_metadata_name(const char *type, int16 column_index) { Assert(is_sparse_index_type(type)); char *buf = palloc(sizeof(char) * NAMEDATALEN); Assert(column_index > 0); int ret = snprintf(buf, NAMEDATALEN, COMPRESSION_COLUMN_METADATA_PATTERN_V1, type, column_index); if (ret < 0 || ret > NAMEDATALEN) { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("bad segment metadata column name"))); } return buf; } /* * Validate that compression settings don't exceed PostgreSQL's INDEX_MAX_KEYS limit. * * Compression creates an implicit index on the compressed chunk with: * - 1 index key per segmentby column * - 2 index keys per orderby column (for min/max metadata) */ static void validate_compression_index_key_limit(CompressionSettings *settings) { int num_segmentby_keys = ts_array_length(settings->fd.segmentby); int num_orderby_keys = 2 * ts_array_length(settings->fd.orderby); if ((num_segmentby_keys + num_orderby_keys) > INDEX_MAX_KEYS) ereport(ERROR, (errcode(ERRCODE_TOO_MANY_COLUMNS), errmsg("too many segmentby and orderby columns"), errdetail("Combined segmentby keys (%d) and orderby keys (%d) cannot exceed %d", num_segmentby_keys, num_orderby_keys, INDEX_MAX_KEYS))); } char * column_segment_min_name(int16 column_index) { return compression_column_segment_metadata_name("min", column_index); } char * column_segment_max_name(int16 column_index) { return compression_column_segment_metadata_name("max", column_index); } /* * Get metadata name for a given column name and metadata type, format version 2. * We can't reference the attribute numbers, because they can change after * drop/restore if we had any dropped columns. * We might have to truncate the column names to fit into the NAMEDATALEN here, * in this case we disambiguate them with their md5 hash. */ char * compressed_column_metadata_name_v2(const char *metadata_type, const char **column_names, int num_columns) { Assert(is_sparse_index_type(metadata_type)); Assert(strlen(metadata_type) <= 6); Assert(column_names != NULL); Assert(num_columns > 0); Assert(num_columns <= MAX_BLOOM_FILTER_COLUMNS); int len = 0; StringInfoData buf = { 0 }; initStringInfo(&buf); for (int i = 0; i < num_columns; i++) { Assert(column_names[i] != NULL); #ifdef USE_ASSERT_CHECKING int col_len = strlen(column_names[i]); #endif Assert(col_len > 0 && col_len < NAMEDATALEN); if (i > 0) appendStringInfoChar(&buf, '_'); appendStringInfo(&buf, "%s", column_names[i]); } len = buf.len; /* * We have to fit the name into NAMEDATALEN - 1 which is 63 bytes: * 12 (_ts_meta_v2_) + 6 (metadata_type) + [1 (_) + x (column_name)]x num_columns + 1 (_) + 4 * (hash) = 63; x = 63 - 24 = 39. */ char *result; if (len > 39) { const char *errstr = NULL; char hash[33]; Ensure(pg_md5_hash(buf.data, len, hash, &errstr), "md5 computation failure"); result = psprintf("_ts_meta_v2_%.6s_%.4s_%.39s", metadata_type, hash, buf.data); } else { result = psprintf("_ts_meta_v2_%.6s_%.39s", metadata_type, buf.data); } Assert(strlen(result) < NAMEDATALEN); return result; } char * compressed_column_metadata_name_list_v2(const char *metadata_type, List *column_names_list) { int num_column_names = list_length(column_names_list); Ensure(num_column_names > 0, "list of column names must be non-empty"); Ensure(num_column_names <= MAX_BLOOM_FILTER_COLUMNS, "list of column names must be less than or equal to %d, got %d", MAX_BLOOM_FILTER_COLUMNS, num_column_names); const char *column_names[MAX_BLOOM_FILTER_COLUMNS]; ListCell *cell = NULL; int i = 0; foreach (cell, column_names_list) { column_names[i] = (const char *) lfirst(cell); i++; } return compressed_column_metadata_name_v2(metadata_type, column_names, num_column_names); } int compressed_column_metadata_attno(const CompressionSettings *settings, Oid chunk_reloid, AttrNumber chunk_attno, Oid compressed_reloid, char const *metadata_type) { Assert(is_sparse_index_type(metadata_type)); char *attname = get_attname(chunk_reloid, chunk_attno, /* missing_ok = */ false); int16 orderby_pos = ts_array_position(settings->fd.orderby, attname); if (orderby_pos != 0 && (strcmp(metadata_type, "min") == 0 || strcmp(metadata_type, "max") == 0)) { char *metadata_name = compression_column_segment_metadata_name(metadata_type, orderby_pos); return get_attnum(compressed_reloid, metadata_name); } char *metadata_name = compressed_column_metadata_name_v2(metadata_type, (const char **) &attname, 1); return get_attnum(compressed_reloid, metadata_name); } /* * The heuristic for whether we should use the bloom filter sparse index. */ static bool should_create_bloom_sparse_index(Oid atttypid, TypeCacheEntry *type, Oid src_reloid) { /* * The index must be enabled by the GUC. */ if (!ts_guc_enable_sparse_index_bloom) { return false; } /* * The type must be hashable. For some types we use our own hash functions * which have better characteristics. */ FmgrInfo *finfo = NULL; if (bloom1_get_hash_function(atttypid, &finfo) == NULL) { return false; } /* * For time types, we expect: * 1) range queries, not equality, * 2) correlation with the orderby columns, e.g. creation time correlates * with the update time that is used as orderby. * This makes minmax indexes more suitable than bloom filters. */ if (atttypid == TIMESTAMPTZOID || atttypid == TIMESTAMPOID || atttypid == TIMEOID || atttypid == TIMETZOID || atttypid == DATEOID) { return false; } /* * For fractional arithmetic types, equality queries are unlikely. */ if (atttypid == FLOAT4OID || atttypid == FLOAT8OID || atttypid == NUMERICOID) { return false; } /* * Bloom filters for 1k elements with 2% false positive rate require about * one byte per element, so there's no point in using them for smaller data * types that typically compress to less than that. */ if (type->typlen > 0 && type->typlen < 4) { return false; } return true; } /* * Create a column definition for a sparse index column. The attributes passed is a * List of Form_pg_attribute elements. Min and max indices only use * the first element. Bloom filters may use multiple columns. */ static ColumnDef * create_sparse_index_column_def(List *attributes, const char *metadata_type) { Assert(is_sparse_index_type(metadata_type)); ColumnDef *column_def = NULL; List *column_names = NIL; /* At least one valid attribute must be present */ Assert(attributes != NULL); Assert(list_length(attributes) > 0); Assert(list_length(attributes) <= MAX_BLOOM_FILTER_COLUMNS); const bool is_bloom = strcmp(metadata_type, bloom1_column_prefix) == 0; { /* Populate the column names array */ ListCell *cell = NULL; int i = 0; foreach (cell, attributes) { Form_pg_attribute attr = (Form_pg_attribute) lfirst(cell); Ensure(i < MAX_BLOOM_FILTER_COLUMNS, "too many columns for bloom filter, got %d, max %d, name: %s", i + 1, MAX_BLOOM_FILTER_COLUMNS, NameStr(attr->attname)); column_names = lappend(column_names, NameStr(attr->attname)); i++; } } if (is_bloom) { /* * The types must be hashable. For some types we use our own hash functions * which have better characteristics. */ ListCell *cell = NULL; foreach (cell, attributes) { Form_pg_attribute attr = (Form_pg_attribute) lfirst(cell); FmgrInfo *finfo = NULL; if (bloom1_get_hash_function(attr->atttypid, &finfo) == NULL) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_FUNCTION), errmsg("invalid bloom filter column type %s, name: %s", format_type_be(attr->atttypid), NameStr(attr->attname)), errdetail("Could not identify a hashing function for the type."))); } column_def = makeColumnDef(compressed_column_metadata_name_list_v2(metadata_type, column_names), ts_custom_type_cache_get(CUSTOM_TYPE_BLOOM1)->type_oid, /* typmod = */ -1, /* collation = */ 0); /* * We have our custom compression for bloom filters, and the * result is almost incompressible with lz4 (~2%), so disable it. */ column_def->storage = TYPSTORAGE_EXTERNAL; /* Composite bloom filters are more selective, try to store them inline. */ if (list_length(column_names) > 1) { column_def->storage = TYPSTORAGE_MAIN; } } else /* either min or max */ { Form_pg_attribute attr = (Form_pg_attribute) lfirst(list_head(attributes)); TypeCacheEntry *type = lookup_type_cache(attr->atttypid, TYPECACHE_LT_OPR); /* * a comparison operator if required for min max operations */ if (!OidIsValid(type->lt_opr)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_FUNCTION), errmsg("invalid minmax column type %s", format_type_be(attr->atttypid)), errdetail("Could not identify a less-than operator for the type."))); column_def = makeColumnDef(compressed_column_metadata_name_list_v2(metadata_type, column_names), attr->atttypid, attr->atttypmod, attr->attcollation); if (attr->attstorage != TYPSTORAGE_PLAIN) { column_def->storage = TYPSTORAGE_MAIN; } } return column_def; } /* * return the columndef list for compressed hypertable. * we do this by getting the source hypertable's attrs, * 1. validate the segmentby cols and orderby cols exists in this list and * 2. create the columndefs for the new compressed hypertable * segmentby_cols have same datatype as the original table * all other cols have COMPRESSEDDATA_TYPE type */ static List * build_columndefs(CompressionSettings *settings, Oid src_reloid) { Oid compresseddata_oid = ts_custom_type_cache_get(CUSTOM_TYPE_COMPRESSED_DATA)->type_oid; ArrayType *segmentby = settings->fd.segmentby; List *compressed_column_defs = NIL; List *segmentby_column_defs = NIL; Jsonb *sparse_cfg = settings->fd.index; SparseIndexSettings *parsed_settings = sparse_cfg ? ts_convert_to_sparse_index_settings(sparse_cfg) : NULL; Bitmapset *all_composite_bloom_obj_ids = NULL; List *per_column_settings = ts_get_per_column_compression_settings(parsed_settings); Relation rel = table_open(src_reloid, AccessShareLock); TupleDesc tupdesc = rel->rd_att; int num_sparse_index_objects = parsed_settings != NULL ? list_length(parsed_settings->objects) : 0; List **composite_attr_lists = NULL; if (num_sparse_index_objects > 0) { /* Allocate an array of Lists that contain Form_pg_attribute elements for each sparse index * configuration object. Minmax and single bloom filter configuration objects will have a * single element list. */ composite_attr_lists = palloc0(sizeof(List *) * num_sparse_index_objects); } for (int attoffset = 0; attoffset < tupdesc->natts; attoffset++) { Form_pg_attribute attr = TupleDescAttr(tupdesc, attoffset); if (attr->attisdropped) continue; if (strncmp(NameStr(attr->attname), COMPRESSION_COLUMN_METADATA_PREFIX, strlen(COMPRESSION_COLUMN_METADATA_PREFIX)) == 0) ereport(ERROR, (errcode(ERRCODE_RESERVED_NAME), errmsg("cannot convert tables with reserved column prefix '%s'", COMPRESSION_COLUMN_METADATA_PREFIX))); bool is_segmentby = ts_array_is_member(segmentby, NameStr(attr->attname)); if (is_segmentby) { segmentby_column_defs = lappend(segmentby_column_defs, makeColumnDef(NameStr(attr->attname), attr->atttypid, attr->atttypmod, attr->attcollation)); continue; } PerColumnCompressionSettings *per_column_setting = per_column_settings ? ts_get_per_column_compression_settings_by_column_name(per_column_settings, NameStr(attr->attname)) : NULL; if (per_column_setting != NULL && composite_attr_lists != NULL) { if (per_column_setting->minmax_obj_id != -1 && per_column_setting->minmax_obj_id < num_sparse_index_objects) { /* Minmax index configuration objects will have a single element list */ Assert(list_length(composite_attr_lists[per_column_setting->minmax_obj_id]) == 0); composite_attr_lists[per_column_setting->minmax_obj_id] = lappend(composite_attr_lists[per_column_setting->minmax_obj_id], attr); } if (per_column_setting->single_bloom_obj_id != -1 && per_column_setting->single_bloom_obj_id < num_sparse_index_objects) { /* Single bloom filter configuration objects will have a single element list */ Assert(list_length(composite_attr_lists[per_column_setting->single_bloom_obj_id]) == 0); composite_attr_lists[per_column_setting->single_bloom_obj_id] = lappend(composite_attr_lists[per_column_setting->single_bloom_obj_id], attr); } if (per_column_setting->composite_bloom_index_obj_ids != NULL) { /* The bitmapset tells which sparse index configuration objects the current * column participates in. Iterate over the bitmapset and add an entry * to the composite_attr_lists. */ int i = -1; while ((i = bms_next_member(per_column_setting->composite_bloom_index_obj_ids, i)) >= 0) { composite_attr_lists[i] = lappend(composite_attr_lists[i], attr); } /* capture all composite bloom index objects */ all_composite_bloom_obj_ids = bms_union(all_composite_bloom_obj_ids, per_column_setting->composite_bloom_index_obj_ids); } } /* * This is either an orderby or a normal compressed column. We want to * have metadata for some of them. Put the metadata columns before the * respective compressed column, because they are accessed before * decompression. */ const bool is_orderby = ts_array_is_member(settings->fd.orderby, NameStr(attr->attname)); if (is_orderby) { int index = ts_array_position(settings->fd.orderby, NameStr(attr->attname)); TypeCacheEntry *type = lookup_type_cache(attr->atttypid, TYPECACHE_LT_OPR); /* * We must be able to create the metadata for the orderby columns, * because it is required for sorting. */ if (!OidIsValid(type->lt_opr)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_FUNCTION), errmsg("invalid ordering column type %s", format_type_be(attr->atttypid)), errdetail("Could not identify a less-than operator for the type."))); /* segment_meta min and max columns */ ColumnDef *def = makeColumnDef(column_segment_min_name(index), attr->atttypid, attr->atttypmod, attr->attcollation); def->storage = TYPSTORAGE_PLAIN; compressed_column_defs = lappend(compressed_column_defs, def); def = makeColumnDef(column_segment_max_name(index), attr->atttypid, attr->atttypmod, attr->attcollation); def->storage = TYPSTORAGE_PLAIN; compressed_column_defs = lappend(compressed_column_defs, def); } else if (per_column_setting != NULL && composite_attr_lists != NULL) { /* check sparse index columndefs is applicable */ bool is_bloom = per_column_setting->single_bloom_obj_id != -1; bool is_minmax = per_column_setting->minmax_obj_id != -1; /* * We allow only one sparse index per column. Columns used in the ORDER BY * clause implicitly have a minmax index and adding a bloom filter on them is not * allowed. * * The parser is expected to enforce this constraint earlier, but we check again * here as a safeguard. */ Ensure((!is_bloom || !is_minmax), "Should not create bloom filter for minmax column \"%s\"", NameStr(attr->attname)); /* build sparse index columndefs if applicable */ if (is_bloom) { if (!ts_guc_enable_sparse_index_bloom) { ereport(WARNING, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("Creating bloom sparse index is disabled"), errhint("Either set \"enable_sparse_index_bloom\" to true or remove " "the bloom filter indexes from \"sparse_index\" configuration " "of the hypertable."))); } /* * Add bloom filter sparse index for this column. */ ColumnDef *bloom_column_def = create_sparse_index_column_def(composite_attr_lists[per_column_setting ->single_bloom_obj_id], bloom1_column_prefix); compressed_column_defs = lappend(compressed_column_defs, bloom_column_def); } else if (is_minmax) { /* * Add minmax sparse index for this column. */ ColumnDef *def = create_sparse_index_column_def(composite_attr_lists[per_column_setting ->minmax_obj_id], "min"); compressed_column_defs = lappend(compressed_column_defs, def); def = create_sparse_index_column_def(composite_attr_lists[per_column_setting ->minmax_obj_id], "max"); compressed_column_defs = lappend(compressed_column_defs, def); } } compressed_column_defs = lappend(compressed_column_defs, makeColumnDef(NameStr(attr->attname), compresseddata_oid, /* typmod = */ -1, /* collOid = */ InvalidOid)); } /* add the composite bloom columns */ if (composite_attr_lists != NULL && per_column_settings != NULL) { /* iterate over the all_composite_bloom_obj_ids bitmapset */ int i = -1; while ((i = bms_next_member(all_composite_bloom_obj_ids, i)) >= 0) { Assert(i < num_sparse_index_objects); Assert(composite_attr_lists[i] != NULL); List *attr_list = composite_attr_lists[i]; if (attr_list != NULL) { ColumnDef *def = create_sparse_index_column_def(attr_list, bloom1_column_prefix); compressed_column_defs = lappend(compressed_column_defs, def); } } } /* * Add the metadata columns. Count is always accessed, so put it first. */ List *all_column_defs = list_make1(makeColumnDef(COMPRESSION_COLUMN_METADATA_COUNT_NAME, INT4OID, -1 /* typemod */, 0 /*collation*/)); /* * Then, put all segmentby columns. They are likely to be used in filters * before decompression. */ all_column_defs = list_concat(all_column_defs, segmentby_column_defs); /* * Then, put all the compressed columns. */ all_column_defs = list_concat(all_column_defs, compressed_column_defs); table_close(rel, AccessShareLock); return all_column_defs; } /* use this api for the case when you add a single column to a table that already has * compression setup * such as ALTER TABLE xyz ADD COLUMN ..... */ static ColumnDef * build_columndef_singlecolumn(const char *colname, Oid typid) { Oid compresseddata_oid = ts_custom_type_cache_get(CUSTOM_TYPE_COMPRESSED_DATA)->type_oid; if (strncmp(colname, COMPRESSION_COLUMN_METADATA_PREFIX, strlen(COMPRESSION_COLUMN_METADATA_PREFIX)) == 0) ereport(ERROR, (errcode(ERRCODE_RESERVED_NAME), errmsg("cannot convert tables with reserved column prefix '%s'", COMPRESSION_COLUMN_METADATA_PREFIX))); return makeColumnDef(colname, compresseddata_oid, -1 /*typmod*/, 0 /*collation*/); } /* * Create compress chunk for specific table. * * If table_id is InvalidOid, create a new table. * */ Chunk * create_compress_chunk(Hypertable *compress_ht, Chunk *src_chunk, Oid table_id) { Catalog *catalog = ts_catalog_get(); CatalogSecurityContext sec_ctx; Chunk *compress_chunk; int namelen; Oid tablespace_oid; Assert(compress_ht->space->num_dimensions == 0); /* Create a new catalog entry for chunk based on uncompressed chunk */ ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); compress_chunk = ts_chunk_create_base(ts_catalog_table_next_seq_id(catalog, CHUNK), 0, RELKIND_RELATION); ts_catalog_restore_user(&sec_ctx); compress_chunk->fd.hypertable_id = compress_ht->fd.id; compress_chunk->hypertable_relid = compress_ht->main_table_relid; namestrcpy(&compress_chunk->fd.schema_name, INTERNAL_SCHEMA_NAME); if (OidIsValid(table_id)) { Relation table_rel = table_open(table_id, AccessShareLock); strncpy(NameStr(compress_chunk->fd.table_name), RelationGetRelationName(table_rel), NAMEDATALEN); table_close(table_rel, AccessShareLock); } else { /* Fail if we overflow the name limit */ namelen = snprintf(NameStr(compress_chunk->fd.table_name), NAMEDATALEN, "compress%s_%d_chunk", NameStr(compress_ht->fd.associated_table_prefix), compress_chunk->fd.id); if (namelen >= NAMEDATALEN) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("invalid name \"%s\" for compressed chunk", NameStr(compress_chunk->fd.table_name)), errdetail("The associated table prefix is too long."))); } /* Insert chunk */ ts_chunk_insert_lock(compress_chunk, RowExclusiveLock); /* Create the actual table relation for the chunk * Note that we have to pick the tablespace here as the compressed ht doesn't have dimensions * on which to base this decision. We simply pick the same tablespace as the uncompressed chunk * for now. */ tablespace_oid = get_rel_tablespace(src_chunk->table_id); CompressionSettings *settings = ts_compression_settings_get(src_chunk->hypertable_relid); /* * On hypertables created with CREATE TABLE ... WITH we enable compression * by default but do not create CompressionSettings immediately assuming * that we have more information available when the first compression * is actually triggered allowing us to generate better compression * settings. */ if (!settings) { settings = ts_compression_settings_create(src_chunk->hypertable_relid, InvalidOid, NULL, NULL, NULL, NULL, NULL); } Hypertable *ht = ts_hypertable_get_by_id(src_chunk->fd.hypertable_id); compression_settings_set_defaults(ht, settings, ts_alter_table_with_clause_parse(NIL)); if (OidIsValid(table_id)) compress_chunk->table_id = table_id; else { List *column_defs = build_columndefs(settings, src_chunk->table_id); compress_chunk->table_id = compression_chunk_create(src_chunk, compress_chunk, column_defs, tablespace_oid, settings); } if (!OidIsValid(compress_chunk->table_id)) elog(ERROR, "could not create columnstore chunk table"); /* Materialize current compression settings for this chunk */ ts_compression_settings_materialize(settings, src_chunk->table_id, compress_chunk->table_id); /* if the src chunk is not in the default tablespace, the compressed indexes * should also be in a non-default tablespace. IN the usual case, this is inferred * from the hypertable's and chunk's tablespace info. We do not propagate * attach_tablespace settings to the compressed hypertable. So we have to explicitly * pass the tablespace information here */ ts_chunk_index_create_all(compress_chunk->fd.hypertable_id, compress_chunk->hypertable_relid, compress_chunk->fd.id, compress_chunk->table_id, tablespace_oid); return compress_chunk; } /* Add the hypertable time column to the end of the orderby list if * it's not already in the orderby or segmentby. */ static OrderBySettings add_time_to_order_by_if_not_included(OrderBySettings obs, ArrayType *segmentby, Hypertable *ht) { const Dimension *time_dim; const char *time_col_name; bool found = false; time_dim = hyperspace_get_open_dimension(ht->space, 0); if (!time_dim) return obs; time_col_name = get_attname(ht->main_table_relid, time_dim->column_attno, false); if (ts_array_is_member(obs.orderby, time_col_name)) found = true; if (ts_array_is_member(segmentby, time_col_name)) found = true; if (!found) { /* Add time DESC NULLS FIRST to order by settings */ obs.orderby = ts_array_add_element_text(obs.orderby, pstrdup(time_col_name)); obs.orderby_desc = ts_array_add_element_bool(obs.orderby_desc, true); obs.orderby_nullsfirst = ts_array_add_element_bool(obs.orderby_nullsfirst, true); } return obs; } /* returns list of constraints that need to be cloned on the compressed hypertable * This is limited to foreign key constraints now */ static void validate_existing_constraints(Hypertable *ht, CompressionSettings *settings) { Relation pg_constr; SysScanDesc scan; ScanKeyData scankey; HeapTuple tuple; ArrayType *arr; Assert(ht->main_table_relid == settings->fd.relid); pg_constr = table_open(ConstraintRelationId, AccessShareLock); ScanKeyInit(&scankey, Anum_pg_constraint_conrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(settings->fd.relid)); scan = systable_beginscan(pg_constr, ConstraintRelidTypidNameIndexId, true, NULL, 1, &scankey); while (HeapTupleIsValid(tuple = systable_getnext(scan))) { Form_pg_constraint form = (Form_pg_constraint) GETSTRUCT(tuple); /* * We check primary, unique, and exclusion constraints. */ if (form->contype == CONSTRAINT_CHECK || form->contype == CONSTRAINT_TRIGGER #if PG17_GE || form->contype == CONSTRAINT_NOTNULL /* CONSTRAINT_NOTNULL introduced in PG17, see b0e96f311985 */ #endif ) { continue; } else if (form->contype == CONSTRAINT_EXCLUSION) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("constraint %s is not supported for converting to columnstore", NameStr(form->conname)), errhint("Exclusion constraints are not supported on hypertables that are " "converted to columnstore."))); } else { int j, numkeys; int16 *attnums; bool is_null; /* Extract the conkey array, ie, attnums of PK's columns */ Datum adatum = heap_getattr(tuple, Anum_pg_constraint_conkey, RelationGetDescr(pg_constr), &is_null); if (is_null) { Oid oid = heap_getattr(tuple, Anum_pg_constraint_oid, RelationGetDescr(pg_constr), &is_null); elog(ERROR, "null conkey for constraint %u", oid); } arr = DatumGetArrayTypeP(adatum); /* ensure not toasted */ numkeys = ts_array_length(arr); attnums = (int16 *) ARR_DATA_PTR(arr); for (j = 0; j < numkeys; j++) { const char *attname = get_attname(settings->fd.relid, attnums[j], false); /* is colno a segment-by or order_by column */ if (!form->conindid && (settings->fd.segmentby && settings->fd.orderby) && !ts_array_is_member(settings->fd.segmentby, attname) && !ts_array_is_member(settings->fd.orderby, attname)) ereport(WARNING, (errmsg("column \"%s\" should be used for segmenting or ordering", attname))); } } } systable_endscan(scan); table_close(pg_constr, AccessShareLock); } /* * Validate existing indexes on the hypertable. Note that there can be indexes * that do not have a corresponding constraint. * * We pass in a list of indexes that we should ignore since these are checked * by the constraint checking above. */ static void validate_existing_indexes(Hypertable *ht, CompressionSettings *settings) { Relation pg_index; HeapTuple htup; ScanKeyData skey; SysScanDesc indscan; ScanKeyInit(&skey, Anum_pg_index_indrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(ht->main_table_relid)); pg_index = table_open(IndexRelationId, AccessShareLock); indscan = systable_beginscan(pg_index, IndexIndrelidIndexId, true, NULL, 1, &skey); while (HeapTupleIsValid(htup = systable_getnext(indscan))) { Form_pg_index index = (Form_pg_index) GETSTRUCT(htup); /* We can ignore indexes that are being dropped, invalid indexes, * exclusion indexes, and any indexes checked by the constraint * checking. We can also skip checks below if the index is not a * unique index. */ if (!index->indislive || !index->indisvalid || index->indisexclusion || !index->indisunique) continue; /* Now we check that all columns of the unique index are part of the * segmentby columns. */ for (int i = 0; i < index->indnkeyatts; i++) { int attno = index->indkey.values[i]; if (attno == 0) continue; /* skip check for expression column */ const char *attname = get_attname(ht->main_table_relid, attno, false); if ((settings->fd.segmentby && settings->fd.orderby) && !ts_array_is_member(settings->fd.segmentby, attname) && !ts_array_is_member(settings->fd.orderby, attname)) ereport(WARNING, (errmsg("column \"%s\" should be used for segmenting or ordering", attname))); } } systable_endscan(indscan); table_close(pg_index, AccessShareLock); } static void drop_existing_compression_table(Hypertable *ht) { if (ts_chunk_exists_with_compression(ht->fd.id)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot drop columnstore-enabled hypertable with columnstore chunks"))); Hypertable *compressed = ts_hypertable_get_by_id(ht->fd.compressed_hypertable_id); if (compressed == NULL) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("columnstore-enabled hypertable not found"), errdetail("columnstore was enabled on \"%s\", but its internal" " columnstore hypertable could not be found.", NameStr(ht->fd.table_name)))); /* need to drop the old compressed hypertable in case the segment by columns changed (and * thus the column types of compressed hypertable need to change) */ ts_hypertable_drop(compressed, DROP_RESTRICT); ts_hypertable_unset_compressed(ht); } static bool disable_compression(Hypertable *ht, WithClauseResult *with_clause_options) { if (!TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(ht)) /* compression is not enabled, so just return */ return false; if (ts_chunk_exists_with_compression(ht->fd.id)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot disable columnstore on hypertable with columnstore chunks"))); if (TS_HYPERTABLE_HAS_COMPRESSION_TABLE(ht)) drop_existing_compression_table(ht); else { ts_hypertable_unset_compressed(ht); } ts_compression_settings_delete(ht->main_table_relid); return true; } /* Add column to internal compression table */ static void add_column_to_compression_table(Oid relid, CompressionSettings *settings, ColumnDef *coldef) { AlterTableCmd *addcol_cmd; /* create altertable stmt to add column to the compressed hypertable */ // Assert(TS_HYPERTABLE_IS_INTERNAL_COMPRESSION_TABLE(compress_ht)); addcol_cmd = makeNode(AlterTableCmd); addcol_cmd->subtype = AT_AddColumn; addcol_cmd->def = (Node *) coldef; addcol_cmd->missing_ok = false; /* alter the table and add column */ ts_alter_table_with_event_trigger(relid, NULL, list_make1(addcol_cmd), true); modify_compressed_toast_table_storage(settings, list_make1(coldef), relid); } /* Drop column from internal compression table, drop the bloom filter columns as well and * update the compression settings for the chunk */ static void drop_column_from_compression_table(CompressionSettings *comp_settings, char *name) { Oid relid = comp_settings->fd.compress_relid; AlterTableCmd *cmd; List *cmds = NIL; Jsonb *jb = comp_settings->fd.index; /* create altertable stmt to drop column from the compressed hypertable */ cmd = makeNode(AlterTableCmd); cmd->subtype = AT_DropColumn; cmd->name = name; cmd->missing_ok = true; cmds = list_make1(cmd); if (jb) { SparseIndexSettings *parsed_settings = ts_convert_to_sparse_index_settings(jb); if (parsed_settings) { bool removed_any = false; ListCell *obj_cell = NULL; foreach (obj_cell, parsed_settings->objects) { bool removed = false; const char *bloom_column_name = NULL; SparseIndexSettingsObject *obj = (SparseIndexSettingsObject *) lfirst(obj_cell); foreach_ptr(SparseIndexSettingsPair, pair, obj->pairs) { if (strcmp(pair->key, ts_sparse_index_common_keys[SparseIndexKeyCol]) != 0) { continue; } foreach_ptr(const char, value, pair->values) { if (strcmp(value, name) == 0) { removed = true; Assert(list_length(pair->values) <= MAX_BLOOM_FILTER_COLUMNS); bloom_column_name = compressed_column_metadata_name_list_v2(bloom1_column_prefix, pair->values); Assert(bloom_column_name != NULL); break; } } if (removed) { break; } } /* if the column was removed, we need to remove the object from the list */ if (removed) { removed_any = true; if (bloom_column_name) { cmd = makeNode(AlterTableCmd); cmd->subtype = AT_DropColumn; cmd->name = pstrdup(bloom_column_name); cmd->missing_ok = true; cmds = lappend(cmds, cmd); } parsed_settings->objects = foreach_delete_current(parsed_settings->objects, obj_cell); } } if (removed_any) { jb = ts_convert_from_sparse_index_settings(parsed_settings); comp_settings->fd.index = jb; ts_compression_settings_update(comp_settings); } ts_free_sparse_index_settings(parsed_settings); } } /* alter the table and drop column */ ts_alter_table_with_event_trigger(relid, NULL, cmds, true); } static bool update_compress_chunk_time_interval(Hypertable *ht, WithClauseResult *with_clause_options) { const Dimension *time_dim = hyperspace_get_open_dimension(ht->space, 0); if (!time_dim) return false; Interval *compress_interval = ts_compress_hypertable_parse_chunk_time_interval(with_clause_options, ht); if (!compress_interval) { return false; } int64 compress_interval_usec = ts_interval_value_to_internal(IntervalPGetDatum(compress_interval), INTERVALOID); if (compress_interval_usec % time_dim->fd.interval_length > 0) elog(WARNING, "compress chunk interval is not a multiple of chunk interval, you should use a " "factor of chunk interval to merge as much as possible"); return ts_hypertable_set_compress_interval(ht, compress_interval_usec); } /* * enables compression for the passed in table by * creating a compression hypertable with special properties * Note: caller should check security permissions * * Return true if compression was enabled, false otherwise. * * Steps: * 1. Check existing constraints on the table -> can we support them with compression? * 2. Create internal compression table + mark hypertable as compression enabled * 3. Add catalog entries to hypertable_compression to record compression settings. * 4. Copy constraints to internal compression table */ bool tsl_process_compress_table(Hypertable *ht, WithClauseResult *with_clause_options) { int32 compress_htid; bool compress_disable = !with_clause_options[AlterTableFlagColumnstore].is_default && !DatumGetBool(with_clause_options[AlterTableFlagColumnstore].parsed); CompressionSettings *settings; ts_feature_flag_check(FEATURE_HYPERTABLE_COMPRESSION); validate_hypertable_for_compression(ht); /* Lock the uncompressed ht in exclusive mode and keep till end of txn */ LockRelationOid(ht->main_table_relid, AccessExclusiveLock); /* reload info after lock */ ht = ts_hypertable_get_by_id(ht->fd.id); if (compress_disable) { return disable_compression(ht, with_clause_options); } if (!with_clause_options[AlterTableFlagCompressChunkTimeInterval].is_default) { update_compress_chunk_time_interval(ht, with_clause_options); } settings = ts_compression_settings_get(ht->main_table_relid); if (!settings) { settings = ts_compression_settings_create(ht->main_table_relid, InvalidOid, NULL, NULL, NULL, NULL, NULL); } compression_settings_set_manually_for_alter(ht, settings, with_clause_options); if (!TS_HYPERTABLE_HAS_COMPRESSION_TABLE(ht)) { /* take explicit locks on catalog tables and keep them till end of txn */ LockRelationOid(catalog_get_table_id(ts_catalog_get(), HYPERTABLE), RowExclusiveLock); /* Check if we can create a compressed hypertable with existing * constraints and indexes. */ validate_existing_constraints(ht, settings); validate_existing_indexes(ht, settings); Oid ownerid = ts_rel_get_owner(ht->main_table_relid); Oid tablespace_oid = get_rel_tablespace(ht->main_table_relid); compress_htid = compression_hypertable_create(ht, ownerid, tablespace_oid); ts_hypertable_set_compressed(ht, compress_htid); } /* * Check for suboptimal compressed chunk merging configuration * * When compress_chunk_time_interval is configured to merge chunks during compression the * primary dimension should be the first compress_orderby column otherwise chunk merging will * require decompression. */ Dimension *dim = ts_hyperspace_get_mutable_dimension(ht->space, DIMENSION_TYPE_OPEN, 0); if (dim && dim->fd.compress_interval_length && ts_array_position(settings->fd.orderby, NameStr(dim->fd.column_name)) != 1) { ereport(WARNING, (errcode(ERRCODE_WARNING), errmsg("compress_chunk_time_interval configured and primary dimension not " "first column in compress_orderby"), errhint("consider setting \"%s\" as first compress_orderby column", NameStr(dim->fd.column_name)))); } /* do not release any locks, will get released by xact end */ return true; } /* * Verify uncompressed hypertable is compatible with conpression */ static void validate_hypertable_for_compression(Hypertable *ht) { if (TS_HYPERTABLE_IS_INTERNAL_COMPRESSION_TABLE(ht)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot compress internal columnstore hypertable"))); } /*check row security settings for the table */ if (ts_has_row_security(ht->main_table_relid)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("columnstore cannot be used on table with row security"))); Relation rel = table_open(ht->main_table_relid, AccessShareLock); TupleDesc tupdesc = RelationGetDescr(rel); /* * This is only a rough estimate and the actual row size might be different. * We use this only to show a warning when the row size is close to the * maximum row size. */ Size row_size = MAXALIGN(SizeofHeapTupleHeader); row_size += 8; /* sequence_num */ row_size += 4; /* count */ row_size += 16; /* min/max */ for (int attno = 0; attno < tupdesc->natts; attno++) { Form_pg_attribute attr = TupleDescAttr(tupdesc, attno); if (attr->attisdropped) continue; row_size += 18; /* assume 18 bytes for each compressed column (varlena) */ if (strncmp(NameStr(attr->attname), COMPRESSION_COLUMN_METADATA_PREFIX, strlen(COMPRESSION_COLUMN_METADATA_PREFIX)) == 0) ereport(ERROR, (errcode(ERRCODE_RESERVED_NAME), errmsg("cannot convert tables with reserved column prefix '%s' to columnstore", COMPRESSION_COLUMN_METADATA_PREFIX))); } if (row_size > MaxHeapTupleSize) { ereport(WARNING, (errmsg("compressed row size might exceed maximum row size"), errdetail("Estimated row size of columnstore-enabled hypertable is %zu. This " "exceeds the " "maximum size of %zu and can cause conversion of chunks to columnstore " "to fail.", row_size, MaxHeapTupleSize))); } /* * Check that all triggers are ok for compressed tables. */ Relation pg_trigger = table_open(TriggerRelationId, AccessShareLock); HeapTuple tuple; ScanKeyData key; ScanKeyInit(&key, Anum_pg_trigger_tgrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(ht->main_table_relid)); SysScanDesc scan = systable_beginscan(pg_trigger, TriggerRelidNameIndexId, true, NULL, 1, &key); while (HeapTupleIsValid(tuple = systable_getnext(scan))) { bool oldtable_isnull; Form_pg_trigger trigrec = (Form_pg_trigger) GETSTRUCT(tuple); /* * We currently don't support transition tables for DELETE triggers * on compressed tables because deleting a complete segment will not build a * transition table for the delete. */ fastgetattr(tuple, Anum_pg_trigger_tgoldtable, pg_trigger->rd_att, &oldtable_isnull); if (!oldtable_isnull && !TRIGGER_FOR_ROW(trigrec->tgtype) && TRIGGER_FOR_DELETE(trigrec->tgtype)) ereport(ERROR, errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("DELETE triggers with transition tables not supported")); } systable_endscan(scan); table_close(pg_trigger, AccessShareLock); table_close(rel, AccessShareLock); } /* * Get the default segment by value for a hypertable */ static ArrayType * compression_setting_segmentby_get_default(const Hypertable *ht) { StringInfoData command; StringInfoData result; int res; ArrayType *column_res = NULL; Datum datum; text *message; bool isnull; MemoryContext upper = CurrentMemoryContext; MemoryContext old; int32 confidence = -1; Oid default_segmentby_fn = ts_guc_default_segmentby_fn_oid(); if (!OidIsValid(default_segmentby_fn)) { elog(LOG_SERVER_ONLY, "segment_by default: hypertable=\"%s\" columns=\"\" function: \"\" confidence=-1", get_rel_name(ht->main_table_relid)); return NULL; } /* Lock down search_path */ int save_nestlevel = NewGUCNestLevel(); RestrictSearchPath(); initStringInfo(&command); appendStringInfo(&command, "SELECT " " (SELECT array_agg(x) " " FROM jsonb_array_elements_text(seg_by->'columns') t(x))::text[], " " seg_by->>'message', " " (seg_by->>'confidence')::int " "FROM %s.%s(%d) seg_by", quote_identifier(get_namespace_name(get_func_namespace(default_segmentby_fn))), quote_identifier(get_func_name(default_segmentby_fn)), ht->main_table_relid); if (SPI_connect() != SPI_OK_CONNECT) elog(ERROR, "could not connect to SPI"); res = SPI_execute(command.data, true /* read_only */, 0 /*count*/); if (res < 0) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), (errmsg("could not get the default segment by for a hypertable \"%s\"", get_rel_name(ht->main_table_relid))))); old = MemoryContextSwitchTo(upper); datum = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull); if (!isnull) column_res = DatumGetArrayTypePCopy(datum); MemoryContextSwitchTo(old); datum = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 2, &isnull); if (!isnull) { message = DatumGetTextPP(datum); elog(LOG_SERVER_ONLY, "there was some uncertainty picking the default segment by for the hypertable: %s", text_to_cstring(message)); } datum = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 3, &isnull); if (!isnull) { confidence = DatumGetInt32(datum); } pfree(command.data); /* Reset search path since this can be executed as part of a larger transaction */ AtEOXact_GUC(false, save_nestlevel); res = SPI_finish(); if (res != SPI_OK_FINISH) elog(ERROR, "SPI_finish failed: %s", SPI_result_code_string(res)); initStringInfo(&result); ts_array_append_stringinfo(column_res, &result); elog(LOG_SERVER_ONLY, "segment_by default: hypertable=\"%s\" columns=\"%s\" function: \"%s.%s\" confidence=%d", get_rel_name(ht->main_table_relid), result.data, get_namespace_name(get_func_namespace(default_segmentby_fn)), get_func_name(default_segmentby_fn), confidence); pfree(result.data); return column_res; } /* * Get the default segment by value for a hypertable */ static OrderBySettings compression_setting_orderby_get_default(Hypertable *ht, ArrayType *segmentby) { StringInfoData command; int res; text *column_res = NULL; Datum datum; text *message; bool isnull; MemoryContext upper = CurrentMemoryContext; MemoryContext old; char *orderby; int32 confidence = -1; Oid types[] = { TEXTARRAYOID }; Datum values[] = { PointerGetDatum(segmentby) }; char nulls[] = { segmentby == NULL ? 'n' : 'v' }; Oid orderby_fn = ts_guc_default_orderby_fn_oid(); if (!OidIsValid(orderby_fn)) { /* fallback to original logic */ OrderBySettings obs = (OrderBySettings){ 0 }; obs = add_time_to_order_by_if_not_included(obs, segmentby, ht); elog(LOG_SERVER_ONLY, "order_by default: hypertable=\"%s\" function=\"\" confidence=-1", get_rel_name(ht->main_table_relid)); return obs; } /* Lock down search_path */ int save_nestlevel = NewGUCNestLevel(); RestrictSearchPath(); initStringInfo(&command); appendStringInfo(&command, "SELECT " " (SELECT string_agg(x, ', ') FROM " "jsonb_array_elements_text(seg_by->'clauses') " "t(x))::text, " " seg_by->>'message', " " (seg_by->>'confidence')::int " "FROM %s.%s(%d, coalesce($1, array[]::text[])) seg_by", quote_identifier(get_namespace_name(get_func_namespace(orderby_fn))), quote_identifier(get_func_name(orderby_fn)), ht->main_table_relid); if (SPI_connect() != SPI_OK_CONNECT) elog(ERROR, "could not connect to SPI"); res = SPI_execute_with_args(command.data, 1, types, values, nulls, true /* read_only */, 0 /*count*/); if (res < 0) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), (errmsg("could not get the default order by for a hypertable \"%s\"", get_rel_name(ht->main_table_relid))))); old = MemoryContextSwitchTo(upper); datum = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull); if (!isnull) column_res = DatumGetTextPCopy(datum); MemoryContextSwitchTo(old); datum = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 2, &isnull); if (!isnull) { message = DatumGetTextPP(datum); elog(LOG_SERVER_ONLY, "there was some uncertainty picking the default order by for the hypertable: %s", text_to_cstring(message)); } datum = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 3, &isnull); if (!isnull) { confidence = DatumGetInt32(datum); } /* Reset search path since this can be executed as part of a larger transaction */ AtEOXact_GUC(false, save_nestlevel); pfree(command.data); res = SPI_finish(); if (res != SPI_OK_FINISH) elog(ERROR, "SPI_finish failed: %s", SPI_result_code_string(res)); if (column_res != NULL) orderby = TextDatumGetCString(PointerGetDatum(column_res)); else orderby = ""; elog(LOG_SERVER_ONLY, "order_by default: hypertable=\"%s\" clauses=\"%s\" function=\"%s.%s\" confidence=%d", get_rel_name(ht->main_table_relid), orderby, get_namespace_name(get_func_namespace(orderby_fn)), get_func_name(orderby_fn), confidence); if (*orderby == '\0') { return (OrderBySettings){ 0 }; } return ts_compress_parse_order_collist(orderby, ht); } /* sparse indexes will only be set by default if there was no configuration */ static bool can_set_default_sparse_index(CompressionSettings *settings) { return (settings->fd.index == NULL) || !ts_jsonb_has_key_value_str_field(settings->fd.index, ts_sparse_index_common_keys[SparseIndexKeySource], ts_sparse_index_source_names [_SparseIndexSourceEnumConfig]); } static void create_default_composite_bloom(IndexInfo *index_info, Hypertable *ht, CompressionSettings *settings, JsonbParseState *parse_state, TsBmsList *sparse_index_columns, bool *has_object) { int num_cols = index_info->ii_NumIndexKeyAttrs; /* Allocate bloom config for the number of columns in the index */ BloomFilterConfig bloom_config; bloom_config.base.type = _SparseIndexTypeEnumBloom; bloom_config.base.source = _SparseIndexSourceEnumDefault; bloom_config.columns = palloc0(num_cols * sizeof(SparseIndexColumn)); /* Extract columns, filtering out segmentby columns. * Note: orderby columns are not filtered out here because they can * be in composite bloom filters. */ int valid_columns = 0; /* * The index must be enabled by the GUC. */ if (!ts_guc_enable_sparse_index_bloom) { return; } /* Bitmapset of column attnums */ Bitmapset *attnums_bitmap = NULL; /* Check the total width of the hashable columns */ int total_width = 0; for (int i = 0; i < num_cols; i++) { AttrNumber attno = index_info->ii_IndexAttrNumbers[i]; /* Skip expression indexes */ if (attno == InvalidAttrNumber) continue; char *attname = get_attname(ht->main_table_relid, attno, false); /* Skip segmentby columns but continue processing other columns */ if (ts_array_is_member(settings->fd.segmentby, attname)) continue; Oid atttypid = get_atttype(ht->main_table_relid, attno); /* Check if hashable */ FmgrInfo *finfo = NULL; if (bloom1_get_hash_function(atttypid, &finfo) == NULL) continue; TypeCacheEntry *type = lookup_type_cache(atttypid, TYPECACHE_HASH_EXTENDED_PROC); total_width += (type->typlen > 0 ? type->typlen : 4); /* Equality queries are unlikely for floating-point types, so we skip them. */ if (atttypid == FLOAT4OID || atttypid == FLOAT8OID) continue; /* Add to bloom config */ bloom_config.columns[valid_columns].attnum = attno; bloom_config.columns[valid_columns].name = attname; bloom_config.columns[valid_columns].type = atttypid; valid_columns++; attnums_bitmap = bms_add_member(attnums_bitmap, attno); } /* Need at least 2 valid columns for composite bloom and the total width must be at least 4 * bytes. */ if (valid_columns < 2 || total_width < 4) { pfree(bloom_config.columns); bms_free(attnums_bitmap); return; } /* Check if this exact bloom already exists */ if (ts_bmslist_contains_set(*sparse_index_columns, attnums_bitmap)) { pfree(bloom_config.columns); bms_free(attnums_bitmap); return; } bloom_config.num_columns = valid_columns; /* Column names must be in attnum order for metadata column naming */ qsort(bloom_config.columns, bloom_config.num_columns, sizeof(SparseIndexColumn), ts_qsort_attrnumber_cmp); /* Add the bloom's column set to the list */ *sparse_index_columns = ts_bmslist_add_set(*sparse_index_columns, attnums_bitmap); /* Convert to JSONB and add to array */ ts_convert_sparse_index_config_to_jsonb(parse_state, &bloom_config.base); *has_object = true; pfree(bloom_config.columns); } static Jsonb * compression_setting_sparse_index_get_default(Hypertable *ht, CompressionSettings *settings) { bool has_object = false; TsBmsList sparse_index_columns = ts_bmslist_create(); JsonbParseState *parse_state = NULL; /* * Sparse indexes are only created automatically if they are not set in compression settings */ if (!ts_guc_auto_sparse_indexes || !can_set_default_sparse_index(settings)) return NULL; /* * Check which columns have btree indexes. We will create sparse minmax * indexes for them in compressed chunk. */ Relation rel = table_open(ht->main_table_relid, AccessShareLock); ListCell *lc; List *index_oids = RelationGetIndexList(rel); pushJsonbValue(&parse_state, WJB_BEGIN_ARRAY, NULL); foreach (lc, index_oids) { Oid index_oid = lfirst_oid(lc); Relation index_rel = index_open(index_oid, AccessShareLock); IndexInfo *index_info = BuildIndexInfo(index_rel); index_close(index_rel, NoLock); /* * We want to create the sparse minmax index, if it can satisfy the same * kinds of queries as the uncompressed index. The simplest case is btree * which can satisfy equality and comparison tests, same as sparse minmax. * * If an uncompressed column has an index, we want to create a * sparse index for it as well. A sparse index can't satisfy ordering * queries, but at least we can use a bloom index to satisfy equality * queries. Create it when we have uncompressed index types that can * also satisfy equality. */ if (index_info->ii_Am != BTREE_AM_OID && index_info->ii_Am != HASH_AM_OID && index_info->ii_Am != BRIN_AM_OID) { continue; } int num_cols = index_info->ii_NumIndexKeyAttrs; if (ts_guc_enable_sparse_index_bloom && num_cols >= 2 && num_cols <= MAX_BLOOM_FILTER_COLUMNS) { create_default_composite_bloom(index_info, ht, settings, parse_state, &sparse_index_columns, &has_object); } for (int i = 0; i < num_cols; i++) { char *attname; Oid atttypid; MinmaxIndexColumnConfig minmax_config; BloomFilterConfig bloom_config; SparseIndexConfigBase *config = NULL; TypeCacheEntry *type; const int attno = index_info->ii_IndexAttrNumbers[i]; if (attno == InvalidAttrNumber) { continue; } attname = get_attname(ht->main_table_relid, attno, false); /* do not create sparse index for orderby columns */ if (ts_array_is_member(settings->fd.orderby, attname) || ts_array_is_member(settings->fd.segmentby, attname) || ts_bmslist_contains_items(sparse_index_columns, &attno, 1)) continue; atttypid = get_atttype(ht->main_table_relid, attno); type = lookup_type_cache(atttypid, TYPECACHE_LT_OPR | TYPECACHE_HASH_EXTENDED_PROC); /* construct sparse index config */ if (ts_guc_enable_sparse_index_bloom && should_create_bloom_sparse_index(atttypid, type, ht->main_table_relid)) { config = &bloom_config.base; config->type = _SparseIndexTypeEnumBloom; bloom_config.num_columns = 1; bloom_config.columns = palloc(1 * sizeof(SparseIndexColumn)); bloom_config.columns[0].attnum = attno; bloom_config.columns[0].name = attname; bloom_config.columns[0].type = atttypid; } else if (OidIsValid(type->lt_opr)) { config = &minmax_config.base; config->type = _SparseIndexTypeEnumMinmax; minmax_config.col = attname; } else continue; config->source = _SparseIndexSourceEnumDefault; /* convert to json object */ ts_convert_sparse_index_config_to_jsonb(parse_state, config); sparse_index_columns = ts_bmslist_add_member(sparse_index_columns, &attno, 1); has_object = true; } } table_close(rel, AccessShareLock); ts_bmslist_free(sparse_index_columns); return has_object ? JsonbValueToJsonb(pushJsonbValue(&parse_state, WJB_END_ARRAY, NULL)) : NULL; } void compression_settings_set_defaults(Hypertable *ht, CompressionSettings *settings, WithClauseResult *with_clause_options) { /* orderby arrays should always be in sync either all NULL or none */ Assert( (settings->fd.orderby && settings->fd.orderby_desc && settings->fd.orderby_nullsfirst) || (!settings->fd.orderby && !settings->fd.orderby_desc && !settings->fd.orderby_nullsfirst)); bool add_orderby_sparse_index = false; /* get default settings which will be stored at chunk level */ if (!(settings->fd.orderby) && with_clause_options[AlterTableFlagOrderBy].is_default) { if (!settings->fd.segmentby && with_clause_options[AlterTableFlagSegmentBy].is_default) { settings->fd.segmentby = compression_setting_segmentby_get_default(ht); } settings->fd.index = ts_remove_orderby_sparse_index(settings); OrderBySettings obs = compression_setting_orderby_get_default(ht, settings->fd.segmentby); settings->fd.orderby = obs.orderby; settings->fd.orderby_desc = obs.orderby_desc; settings->fd.orderby_nullsfirst = obs.orderby_nullsfirst; add_orderby_sparse_index = settings->fd.index != NULL; } if (ts_guc_auto_sparse_indexes && can_set_default_sparse_index(settings)) { settings->fd.index = compression_setting_sparse_index_get_default(ht, settings); settings->fd.index = ts_add_orderby_sparse_index(settings); } else if (add_orderby_sparse_index) { settings->fd.index = ts_add_orderby_sparse_index(settings); } /* should always be valid, but call as a sanity check */ validate_compression_index_key_limit(settings); } static void compression_settings_set_manually_for_alter(Hypertable *ht, CompressionSettings *settings, WithClauseResult *with_clause_options) { /* orderby arrays should always be in sync either all NULL or none */ Assert( (settings->fd.orderby && settings->fd.orderby_desc && settings->fd.orderby_nullsfirst) || (!settings->fd.orderby && !settings->fd.orderby_desc && !settings->fd.orderby_nullsfirst)); if (with_clause_options[AlterTableFlagSegmentBy].is_default && with_clause_options[AlterTableFlagOrderBy].is_default && with_clause_options[AlterTableFlagIndex].is_default) return; bool add_orderby_sparse_index = false; if (!with_clause_options[AlterTableFlagSegmentBy].is_default) { settings->fd.segmentby = ts_compress_hypertable_parse_segment_by(with_clause_options[AlterTableFlagSegmentBy], ht); } if (!with_clause_options[AlterTableFlagOrderBy].is_default) { settings->fd.index = ts_remove_orderby_sparse_index(settings); OrderBySettings obs = ts_compress_hypertable_parse_order_by(with_clause_options[AlterTableFlagOrderBy], ht); obs = add_time_to_order_by_if_not_included(obs, settings->fd.segmentby, ht); settings->fd.orderby = obs.orderby; settings->fd.orderby_desc = obs.orderby_desc; settings->fd.orderby_nullsfirst = obs.orderby_nullsfirst; add_orderby_sparse_index = settings->fd.index != NULL; } if (!with_clause_options[AlterTableFlagIndex].is_default) { if (!settings->fd.orderby) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot set sparse index option without orderby option"), errdetail("Either set both options or remove both options to trigger default " "values"))); } settings->fd.index = ts_compress_hypertable_parse_index(with_clause_options[AlterTableFlagIndex], ht); settings->fd.index = ts_add_orderby_sparse_index(settings); } else if (add_orderby_sparse_index) { settings->fd.index = ts_add_orderby_sparse_index(settings); } validate_compression_index_key_limit(settings); /* update manual settings */ ts_compression_settings_update(settings); } static void compression_settings_set_manually_for_create(Hypertable *ht, CompressionSettings *settings, WithClauseResult *with_clause_options) { if (with_clause_options[CreateTableFlagSegmentBy].is_default && with_clause_options[CreateTableFlagOrderBy].is_default && with_clause_options[CreateTableFlagIndex].is_default) return; bool add_orderby_sparse_index = false; if (!with_clause_options[CreateTableFlagSegmentBy].is_default) { settings->fd.segmentby = ts_compress_hypertable_parse_segment_by(with_clause_options[CreateTableFlagSegmentBy], ht); } if (!with_clause_options[CreateTableFlagOrderBy].is_default) { settings->fd.index = ts_remove_orderby_sparse_index(settings); OrderBySettings obs = ts_compress_hypertable_parse_order_by(with_clause_options[CreateTableFlagOrderBy], ht); obs = add_time_to_order_by_if_not_included(obs, settings->fd.segmentby, ht); settings->fd.orderby = obs.orderby; settings->fd.orderby_desc = obs.orderby_desc; settings->fd.orderby_nullsfirst = obs.orderby_nullsfirst; add_orderby_sparse_index = settings->fd.index != NULL; } if (!with_clause_options[CreateTableFlagIndex].is_default) { if (!settings->fd.orderby) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot set sparse index option without orderby option"), errdetail("Either set both options or remove both options to trigger default " "values"))); } settings->fd.index = ts_compress_hypertable_parse_index(with_clause_options[CreateTableFlagIndex], ht); settings->fd.index = ts_add_orderby_sparse_index(settings); } else if (add_orderby_sparse_index) { settings->fd.index = ts_add_orderby_sparse_index(settings); } validate_compression_index_key_limit(settings); /* update manual settings */ ts_compression_settings_update(settings); } /* Add a column to a table that has compression enabled * This function specifically adds the column to the internal compression table. */ void tsl_process_compress_table_add_column(Hypertable *ht, ColumnDef *orig_def) { ts_feature_flag_check(FEATURE_HYPERTABLE_COMPRESSION); if (!TS_HYPERTABLE_HAS_COMPRESSION_TABLE(ht)) { return; } List *chunks = ts_chunk_get_by_hypertable_id(ht->fd.compressed_hypertable_id); ListCell *lc; Oid coloid = LookupTypeNameOid(NULL, orig_def->typeName, false); foreach (lc, chunks) { Chunk *chunk = lfirst(lc); /* don't add column if it already exists */ if (get_attnum(chunk->table_id, orig_def->colname) != InvalidAttrNumber) { return; } ColumnDef *coldef = build_columndef_singlecolumn(orig_def->colname, coloid); CompressionSettings *settings = ts_compression_settings_get_by_compress_relid(chunk->table_id); add_column_to_compression_table(chunk->table_id, settings, coldef); } } /* Drop a column from a table that has compression enabled * This function specifically removes it from the internal compression table * and removes it from metadata. * Removing orderby or segmentby columns is not supported. */ void tsl_process_compress_table_drop_column(Hypertable *ht, char *name) { Assert(TS_HYPERTABLE_HAS_COMPRESSION_TABLE(ht) || TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(ht)); ts_feature_flag_check(FEATURE_HYPERTABLE_COMPRESSION); CompressionSettings *settings = ts_compression_settings_get(ht->main_table_relid); Ensure(settings != NULL, "compression settings not found for hypertable \"%s\"", get_rel_name(ht->main_table_relid)); Jsonb *jb = settings->fd.index; /* check if the column is a segmentby or orderby column */ if (settings && (ts_array_is_member(settings->fd.segmentby, name) || ts_array_is_member(settings->fd.orderby, name))) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot drop orderby or segmentby column from a hypertable with " "columnstore enabled"))); List *chunks = ts_chunk_get_by_hypertable_id(ht->fd.compressed_hypertable_id); ListCell *lc; int num_chunks = list_length(chunks); CompressionSettings **chunk_settings = palloc(sizeof(CompressionSettings *) * num_chunks); int i = 0; foreach (lc, chunks) { Chunk *chunk = lfirst(lc); CompressionSettings *settings = ts_compression_settings_get_by_compress_relid(chunk->table_id); chunk_settings[i++] = settings; if (ts_array_is_member(settings->fd.segmentby, name) || ts_array_is_member(settings->fd.orderby, name)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot drop orderby or segmentby column from a chunk with " "columnstore enabled"))); } if (TS_HYPERTABLE_HAS_COMPRESSION_TABLE(ht)) { for (int i = 0; i < num_chunks; i++) { CompressionSettings *comp_settings = chunk_settings[i]; drop_column_from_compression_table(comp_settings, name); } } /* update the compression settings for the main table */ if (jb) { SparseIndexSettings *parsed_settings = ts_convert_to_sparse_index_settings(jb); if (parsed_settings) { bool removed_any = false; ListCell *obj_cell = NULL; foreach (obj_cell, parsed_settings->objects) { bool removed = false; SparseIndexSettingsObject *obj = (SparseIndexSettingsObject *) lfirst(obj_cell); Assert(obj != NULL); foreach_ptr(SparseIndexSettingsPair, pair, obj->pairs) { if (strcmp(pair->key, ts_sparse_index_common_keys[SparseIndexKeyCol]) != 0) { continue; } foreach_ptr(const char, value, pair->values) { if (strcmp(value, name) == 0) { removed = true; break; } } if (removed) { break; } } /* if the column was removed, we need to remove the object from the list */ if (removed) { removed_any = true; parsed_settings->objects = foreach_delete_current(parsed_settings->objects, obj_cell); } } if (removed_any) { jb = ts_convert_from_sparse_index_settings(parsed_settings); settings->fd.index = jb; ts_compression_settings_update(settings); } ts_free_sparse_index_settings(parsed_settings); } } } /* Rename a column on a hypertable that has compression enabled. * * This function renames the existing column in the internal compression table. * We assume that there is a 1-1 mapping between the original chunk and * compressed chunk column names and that the names are identical. * Also update any metadata associated with the column. */ void tsl_process_compress_table_rename_column(Hypertable *ht, const RenameStmt *stmt) { Assert(stmt->relationType == OBJECT_TABLE && stmt->renameType == OBJECT_COLUMN); Assert(TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(ht)); struct RenameFromTo { char *from; char *to; }; if (strncmp(stmt->newname, COMPRESSION_COLUMN_METADATA_PREFIX, strlen(COMPRESSION_COLUMN_METADATA_PREFIX)) == 0) ereport(ERROR, (errcode(ERRCODE_RESERVED_NAME), errmsg("cannot convert tables with reserved column prefix '%s' to columnstore", COMPRESSION_COLUMN_METADATA_PREFIX))); if (!TS_HYPERTABLE_HAS_COMPRESSION_TABLE(ht)) { return; } RenameStmt *compressed_col_stmt = (RenameStmt *) copyObject(stmt); RenameStmt *compressed_index_stmt = (RenameStmt *) copyObject(stmt); List *chunks = ts_chunk_get_by_hypertable_id(ht->fd.compressed_hypertable_id); CompressionSettings *ht_settings = NULL; ListCell *lc; foreach (lc, chunks) { Chunk *chunk = lfirst(lc); compressed_col_stmt->relation = makeRangeVar(NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name), -1); ExecRenameStmt(compressed_col_stmt); List *rename_from_to = NIL; CompressionSettings *settings = ts_compression_settings_get(chunk->table_id); if (!settings || settings->fd.index == NULL) { /* only lookup ht settings if we haven't already */ if (!ht_settings) { ht_settings = ts_compression_settings_get(ht->main_table_relid); } settings = ht_settings; } /* check the minmax and single bloom index columns no matter what the compression settings * says */ { /* handle minmax index */ struct RenameFromTo *from_to = (struct RenameFromTo *) palloc(sizeof(struct RenameFromTo)); from_to->from = compressed_column_metadata_name_v2("min", (const char **) &stmt->subname, 1); from_to->to = compressed_column_metadata_name_v2("min", (const char **) &stmt->newname, 1); rename_from_to = lappend(rename_from_to, from_to); from_to = (struct RenameFromTo *) palloc(sizeof(struct RenameFromTo)); from_to->from = compressed_column_metadata_name_v2("max", (const char **) &stmt->subname, 1); from_to->to = compressed_column_metadata_name_v2("max", (const char **) &stmt->newname, 1); rename_from_to = lappend(rename_from_to, from_to); } { /* handle single bloom index */ struct RenameFromTo *from_to = (struct RenameFromTo *) palloc(sizeof(struct RenameFromTo)); from_to->from = compressed_column_metadata_name_v2(bloom1_column_prefix, (const char **) &stmt->subname, 1); from_to->to = compressed_column_metadata_name_v2(bloom1_column_prefix, (const char **) &stmt->newname, 1); rename_from_to = lappend(rename_from_to, from_to); } if (settings && settings->fd.index != NULL) { SparseIndexSettings *parsed_settings = ts_convert_to_sparse_index_settings(settings->fd.index); List *per_column_settings = ts_get_per_column_compression_settings(parsed_settings); PerColumnCompressionSettings *per_column_setting = per_column_settings ? ts_get_per_column_compression_settings_by_column_name(per_column_settings, stmt->subname) : NULL; if (per_column_setting != NULL) { if (per_column_setting->composite_bloom_index_obj_ids != NULL) { /* one column may participate in multiple composite bloom indices, so we need to * handle all of them */ int i = -1; struct RenameFromTo *from_to = NULL; while ((i = bms_next_member(per_column_setting->composite_bloom_index_obj_ids, i)) >= 0) { SparseIndexSettingsObject *obj = (SparseIndexSettingsObject *) list_nth(parsed_settings->objects, i); Assert(obj != NULL); List *column_names = ts_get_column_names_from_parsed_object(obj); Assert(column_names != NULL); Assert(list_length(column_names) > 1); Assert(list_length(column_names) <= MAX_BLOOM_FILTER_COLUMNS); char *new_name[MAX_BLOOM_FILTER_COLUMNS] = { NULL }; int j = 0; ListCell *cell = NULL; foreach (cell, column_names) { const char *column_name = (const char *) lfirst(cell); if (strcmp(column_name, stmt->subname) == 0) { new_name[j] = stmt->newname; } else { new_name[j] = pstrdup(column_name); } j++; } /* handle composite bloom index */ from_to = (struct RenameFromTo *) palloc(sizeof(struct RenameFromTo)); from_to->from = compressed_column_metadata_name_list_v2(bloom1_column_prefix, column_names); from_to->to = compressed_column_metadata_name_v2(bloom1_column_prefix, (const char **) new_name, list_length(column_names)); rename_from_to = lappend(rename_from_to, from_to); } } } ts_free_sparse_index_settings(parsed_settings); } compressed_index_stmt->relation = compressed_col_stmt->relation; if (rename_from_to != NULL) { ListCell *cell = NULL; foreach (cell, rename_from_to) { struct RenameFromTo *from_to = (struct RenameFromTo *) lfirst(cell); Assert(from_to != NULL); Assert(from_to->from != NULL); Assert(from_to->to != NULL); if (get_attnum(chunk->table_id, from_to->from) == InvalidAttrNumber) { continue; } compressed_index_stmt->subname = from_to->from; compressed_index_stmt->newname = from_to->to; ExecRenameStmt(compressed_index_stmt); } list_free_deep(rename_from_to); } } } /* * Enables compression for a hypertable without creating initial configuration * * This is used when creating a hypertable with CREATE TABLE ... WITH (timescaledb.hypertable) */ void tsl_columnstore_setup(Hypertable *ht, WithClauseResult *with_clause_options) { LockRelationOid(catalog_get_table_id(ts_catalog_get(), HYPERTABLE), RowExclusiveLock); Oid ownerid = ts_rel_get_owner(ht->main_table_relid); Oid tablespace_oid = get_rel_tablespace(ht->main_table_relid); CompressionSettings *settings = ts_compression_settings_create(ht->main_table_relid, InvalidOid, NULL, NULL, NULL, NULL, NULL); compression_settings_set_manually_for_create(ht, settings, with_clause_options); int compress_htid = compression_hypertable_create(ht, ownerid, tablespace_oid); ts_hypertable_set_compressed(ht, compress_htid); /* Add default compression policy when compression is enabled via CREATE TABLE WITH */ /* Use the chunk interval as the compression interval */ const Dimension *time_dim = hyperspace_get_open_dimension(ht->space, 0); if (time_dim != NULL) { Oid compress_after_type = ts_dimension_get_partition_type(time_dim); Datum compress_after_datum; if (IS_TIMESTAMP_TYPE(compress_after_type) || IS_UUID_TYPE(compress_after_type)) compress_after_type = INTERVALOID; compress_after_datum = ts_internal_to_interval_value(time_dim->fd.interval_length, compress_after_type); policy_compression_add_internal( ht->main_table_relid, compress_after_datum, compress_after_type, NULL, /* created_before */ DEFAULT_COMPRESSION_SCHEDULE_INTERVAL, /* default_schedule_interval */ true, /* user_defined_schedule_interval */ true, /* if_not_exists */ false, /* fixed_schedule */ GetCurrentTimestamp() + USECS_PER_DAY, /* initial_start */ NULL /* timezone */); } } ================================================ FILE: tsl/src/compression/create.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/parsenodes.h> #include "hypertable.h" #include "with_clause/with_clause_parser.h" #define COMPRESSION_COLUMN_METADATA_PREFIX "_ts_meta_" #define COMPRESSION_COLUMN_METADATA_COUNT_NAME COMPRESSION_COLUMN_METADATA_PREFIX "count" #define COMPRESSION_COLUMN_METADATA_SEQUENCE_NUM_NAME \ COMPRESSION_COLUMN_METADATA_PREFIX "sequence_num" #define COMPRESSION_COLUMN_METADATA_PATTERN_V1 "_ts_meta_%s_%d" bool tsl_process_compress_table(Hypertable *ht, WithClauseResult *with_clause_options); void tsl_process_compress_table_add_column(Hypertable *ht, ColumnDef *orig_def); void tsl_process_compress_table_drop_column(Hypertable *ht, char *name); void tsl_process_compress_table_rename_column(Hypertable *ht, const RenameStmt *stmt); Chunk *create_compress_chunk(Hypertable *compress_ht, Chunk *src_chunk, Oid table_id); char *column_segment_min_name(int16 column_index); char *column_segment_max_name(int16 column_index); char *compressed_column_metadata_name_v2(const char *metadata_type, const char **column_names, int num_columns); char *compressed_column_metadata_name_list_v2(const char *metadata_type, List *column_names_list); typedef struct CompressionSettings CompressionSettings; int compressed_column_metadata_attno(const CompressionSettings *settings, Oid chunk_reloid, AttrNumber chunk_attno, Oid compressed_reloid, char const *metadata_type); void tsl_columnstore_setup(Hypertable *ht, WithClauseResult *with_clause_options); void compression_settings_set_defaults(Hypertable *ht, CompressionSettings *settings, WithClauseResult *with_clause_options); ================================================ FILE: tsl/src/compression/recompress.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include "debug_point.h" #include <parser/parse_coerce.h> #include <parser/parse_relation.h> #include <utils/inval.h> #include <utils/lsyscache.h> #include <utils/rel.h> #include <utils/relcache.h> #include <utils/snapmgr.h> #include <utils/syscache.h> #include <utils/typcache.h> #include "api.h" #include "compression.h" #include "compression_dml.h" #include "create.h" #include "debug_assert.h" #include "guc.h" #include "indexing.h" #include "recompress.h" #include "ts_catalog/array_utils.h" #include "ts_catalog/chunk_column_stats.h" #include "ts_catalog/compression_chunk_size.h" #include "ts_catalog/compression_settings.h" #include "with_clause/alter_table_with_clause.h" /* * Timing parameters for spin locking heuristics. * These are the same as used by Postgres for truncate locking during lazy vacuum. * https://github.com/postgres/postgres/blob/4a0650d359c5981270039eeb634c3b7427aa0af5/src/backend/access/heap/vacuumlazy.c#L82 */ #define RECOMPRESS_EXCLUSIVE_LOCK_WAIT_INTERVAL 50 /* ms */ #ifdef TS_DEBUG /* Lock timeout reduced for the sake of faster testing. */ #define RECOMPRESS_EXCLUSIVE_LOCK_TIMEOUT 100 /* ms */ #else #define RECOMPRESS_EXCLUSIVE_LOCK_TIMEOUT 5000 /* ms */ #endif static bool fetch_uncompressed_chunk_into_tuplesort(Tuplesortstate *tuplesortstate, Relation uncompressed_chunk_rel, Snapshot snapshot); static bool delete_tuple_for_recompression(Relation rel, ItemPointer tid, Snapshot snapshot); static void update_current_segment(CompressedSegmentInfo *current_segment, Datum *values, bool *isnulls, int nsegmentby_cols); static void create_segmentby_scankeys(CompressionSettings *settings, Relation index_rel, Relation compressed_chunk_rel, ScanKeyData *index_scankeys); static void create_orderby_scankeys(CompressionSettings *settings, Relation index_rel, Relation compressed_chunk_rel, ScanKeyData *orderby_scankeys); static void update_segmentby_scankeys(Datum *values, bool *isnulls, int num_segmentby, ScanKey index_scankeys); static void update_orderby_scankeys(Datum *values, bool *isnulls, int num_segmentby, int num_orderby, ScanKey orderby_scankeys); static enum Batch_match_result match_tuple_batch(TupleTableSlot *compressed_slot, int num_orderby, ScanKey orderby_scankeys, bool *nulls_first); static bool check_changed_group(CompressedSegmentInfo *current_segment, Datum *values, bool *isnulls, int nsegmentby_cols); static void recompress_segment(Tuplesortstate *tuplesortstate, Relation compressed_chunk_rel, RowCompressor *row_compressor, BulkWriter *writer); static void try_updating_chunk_status(Chunk *uncompressed_chunk, Relation uncompressed_chunk_rel); /* * Recompress an existing chunk by decompressing the batches * that are affected by the addition of newer data. The existing * compressed chunk will not be recreated but modified in place. * * 0 uncompressed_chunk_id REGCLASS * 1 if_not_compressed BOOL = false */ Datum tsl_recompress_chunk_segmentwise(PG_FUNCTION_ARGS) { Oid uncompressed_chunk_id = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); bool if_not_compressed = PG_ARGISNULL(1) ? true : PG_GETARG_BOOL(1); ts_feature_flag_check(FEATURE_HYPERTABLE_COMPRESSION); TS_PREVENT_FUNC_IF_READ_ONLY(); Chunk *chunk = ts_chunk_get_by_relid(uncompressed_chunk_id, true); if (!ts_chunk_is_partial(chunk)) { int elevel = if_not_compressed ? NOTICE : ERROR; ereport(elevel, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("nothing to recompress in chunk %s.%s", NameStr(chunk->fd.schema_name), NameStr(chunk->fd.table_name)))); } else { if (!ts_guc_enable_segmentwise_recompression) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("segmentwise recompression functionality disabled, " "enable it by first setting " "timescaledb.enable_segmentwise_recompression to on"))); } CompressionSettings *settings = ts_compression_settings_get(uncompressed_chunk_id); if (!settings->fd.orderby) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("segmentwise recompression cannot be applied for " "compression with no " "order by"))); } uncompressed_chunk_id = recompress_chunk_segmentwise_impl(chunk); } PG_RETURN_OID(uncompressed_chunk_id); } static RecompressContext * compress_chunk_populate_recompress_ctx(CompressionSettings *settings, Relation uncompressed_chunk_rel, Relation compressed_chunk_rel, Relation index_rel, const bool for_uncompressed) { RecompressContext *recompress_ctx; int n; int position; const char *attname; AttrNumber col_attno; Relation chunk_rel = for_uncompressed ? uncompressed_chunk_rel : compressed_chunk_rel; /* Initialize sort info structure */ recompress_ctx = palloc0(sizeof(RecompressContext)); /* Calculate array sizes */ recompress_ctx->num_segmentby = ts_array_length(settings->fd.segmentby); recompress_ctx->num_orderby = ts_array_length(settings->fd.orderby); recompress_ctx->n_keys = recompress_ctx->num_segmentby + recompress_ctx->num_orderby; /* Allocate arrays */ recompress_ctx->sort_keys = palloc(sizeof(*recompress_ctx->sort_keys) * recompress_ctx->n_keys); recompress_ctx->sort_operators = palloc(sizeof(*recompress_ctx->sort_operators) * recompress_ctx->n_keys); recompress_ctx->sort_collations = palloc(sizeof(*recompress_ctx->sort_collations) * recompress_ctx->n_keys); recompress_ctx->nulls_first = palloc(sizeof(*recompress_ctx->nulls_first) * recompress_ctx->n_keys); recompress_ctx->current_segment = palloc0(sizeof(CompressedSegmentInfo) * recompress_ctx->n_keys); /* Populate sort information for each column */ for (n = 0; n < recompress_ctx->n_keys; n++) { if (n < recompress_ctx->num_segmentby) { position = n + 1; attname = ts_array_get_element_text(settings->fd.segmentby, position); col_attno = get_attnum(chunk_rel->rd_id, attname); recompress_ctx->current_segment[n].chunk_offset = AttrNumberGetAttrOffset(col_attno); recompress_ctx->current_segment[n].segment_info = segment_info_new(TupleDescAttr(RelationGetDescr(chunk_rel), recompress_ctx->current_segment[n].chunk_offset)); } else { position = n - recompress_ctx->num_segmentby + 1; attname = ts_array_get_element_text(settings->fd.orderby, position); col_attno = get_attnum(chunk_rel->rd_id, attname); recompress_ctx->current_segment[n].chunk_offset = AttrNumberGetAttrOffset(col_attno); } compress_chunk_populate_sort_info_for_column(settings, RelationGetRelid(uncompressed_chunk_rel), attname, &recompress_ctx->sort_keys[n], &recompress_ctx->sort_operators[n], &recompress_ctx->sort_collations[n], &recompress_ctx->nulls_first[n]); } /* Allocate scankeys */ recompress_ctx->index_scankeys = palloc(sizeof(ScanKeyData) * recompress_ctx->num_segmentby); recompress_ctx->orderby_scankeys = palloc(sizeof(ScanKeyData) * recompress_ctx->num_orderby * 2); /* Populate scankeys */ create_segmentby_scankeys(settings, index_rel, compressed_chunk_rel, recompress_ctx->index_scankeys); create_orderby_scankeys(settings, index_rel, compressed_chunk_rel, recompress_ctx->orderby_scankeys); return recompress_ctx; } static void free_chunk_recompress_ctx(RecompressContext *recompress_ctx) { if (recompress_ctx == NULL) return; if (recompress_ctx->sort_keys) pfree(recompress_ctx->sort_keys); if (recompress_ctx->sort_operators) pfree(recompress_ctx->sort_operators); if (recompress_ctx->sort_collations) pfree(recompress_ctx->sort_collations); if (recompress_ctx->nulls_first) pfree(recompress_ctx->nulls_first); if (recompress_ctx->current_segment) pfree(recompress_ctx->current_segment); if (recompress_ctx->index_scankeys) pfree(recompress_ctx->index_scankeys); if (recompress_ctx->orderby_scankeys) pfree(recompress_ctx->orderby_scankeys); pfree(recompress_ctx); } Oid recompress_chunk_segmentwise_impl(Chunk *uncompressed_chunk) { Oid uncompressed_chunk_id = uncompressed_chunk->table_id; /* * only proceed if status in (3, 9, 11) * 1: compressed * 2: compressed_unordered * 4: frozen * 8: compressed_partial */ if (!ts_chunk_is_compressed(uncompressed_chunk) && ts_chunk_is_partial(uncompressed_chunk)) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("unexpected chunk status %d in chunk %s.%s", uncompressed_chunk->fd.status, NameStr(uncompressed_chunk->fd.schema_name), NameStr(uncompressed_chunk->fd.table_name)))); /* need it to find the segby cols from the catalog */ Chunk *compressed_chunk = ts_chunk_get_by_id(uncompressed_chunk->fd.compressed_chunk_id, true); CompressionSettings *settings = ts_compression_settings_get(uncompressed_chunk->table_id); /* We should not do segment-wise recompression with empty orderby, see #7748 */ Ensure(settings->fd.orderby, "empty order by, cannot recompress segmentwise"); ereport(DEBUG1, (errmsg("acquiring locks for recompression: \"%s.%s\"", NameStr(uncompressed_chunk->fd.schema_name), NameStr(uncompressed_chunk->fd.table_name)))); LOCKMODE recompression_lockmode = ts_guc_enable_exclusive_locking_recompression ? ExclusiveLock : ShareUpdateExclusiveLock; /* lock both chunks, compressed and uncompressed */ Relation uncompressed_chunk_rel = table_open(uncompressed_chunk->table_id, recompression_lockmode); Relation compressed_chunk_rel = table_open(compressed_chunk->table_id, recompression_lockmode); bool has_unique_constraints = ts_indexing_relation_has_primary_or_unique_index(uncompressed_chunk_rel); int count; LOCKTAG locktag; SET_LOCKTAG_RELATION(locktag, MyDatabaseId, uncompressed_chunk_id); /* * Recompression does not block inserts but it can interfere with * constraint checking since it moves uncompressed tuples from * uncompressed chunk to compressed chunk but the INSERTs check * tuples in the opposite order. * * If there are unique constraints and multiple INSERTs happening at start * we want to just bail out so not to cause wasted work and bloat. */ if (has_unique_constraints) { GetLockConflicts(&locktag, ExclusiveLock, &count); if (count > 1) { elog(WARNING, "skipping recompression of chunk %s.%s due to unique constraints and concurrent " "DML", NameStr(uncompressed_chunk->fd.schema_name), NameStr(uncompressed_chunk->fd.table_name)); table_close(uncompressed_chunk_rel, NoLock); table_close(compressed_chunk_rel, NoLock); PG_RETURN_OID(uncompressed_chunk_id); } } Hypertable *ht = ts_hypertable_get_by_id(uncompressed_chunk->fd.hypertable_id); if (ht->range_space) ts_chunk_column_stats_calculate(ht, uncompressed_chunk); TupleDesc compressed_rel_tupdesc = RelationGetDescr(compressed_chunk_rel); TupleDesc uncompressed_rel_tupdesc = RelationGetDescr(uncompressed_chunk_rel); /******************** row decompressor **************/ RowDecompressor decompressor = build_decompressor(RelationGetDescr(compressed_chunk_rel), RelationGetDescr(uncompressed_chunk_rel)); /********** row compressor *******************/ RowCompressor row_compressor; Assert(settings->fd.compress_relid == RelationGetRelid(compressed_chunk_rel)); row_compressor_init(&row_compressor, settings, RelationGetDescr(uncompressed_chunk_rel), RelationGetDescr(compressed_chunk_rel)); BulkWriter writer = bulk_writer_build(compressed_chunk_rel, 0); Oid index_oid = get_compressed_chunk_index(writer.indexstate, settings); /* For chunks with no segmentby settings, we can still do segmentwise recompression * The entire chunk is treated as a single segment */ elog(ts_guc_debug_compression_path_info ? INFO : DEBUG1, "Using index \"%s\" for recompression", get_rel_name(index_oid)); LOCKMODE index_lockmode = ts_guc_enable_exclusive_locking_recompression ? ExclusiveLock : RowExclusiveLock; Relation index_rel = index_open(index_oid, index_lockmode); ereport(DEBUG1, (errmsg("locks acquired for recompression: \"%s.%s\"", NameStr(uncompressed_chunk->fd.schema_name), NameStr(uncompressed_chunk->fd.table_name)))); /* Need to populate recompress context of an uncompressed chunk */ RecompressContext *recompress_ctx = compress_chunk_populate_recompress_ctx(settings, uncompressed_chunk_rel, compressed_chunk_rel, index_rel, true); /* Used for sorting and iterating over all the uncompressed tuples that have * to be recompressed. These tuples are sorted based on the segmentby and * orderby settings. */ Tuplesortstate *input_tuplesortstate = tuplesort_begin_heap(uncompressed_rel_tupdesc, recompress_ctx->n_keys, recompress_ctx->sort_keys, recompress_ctx->sort_operators, recompress_ctx->sort_collations, recompress_ctx->nulls_first, maintenance_work_mem, NULL, false); /* Used for gathering and resorting the tuples that should be recompressed together. * Since we are working on a per-segment level here, we only need to sort them * based on the orderby settings. */ Tuplesortstate *recompress_tuplesortstate = tuplesort_begin_heap(uncompressed_rel_tupdesc, recompress_ctx->num_orderby, &recompress_ctx->sort_keys[recompress_ctx->num_segmentby], &recompress_ctx->sort_operators[recompress_ctx->num_segmentby], &recompress_ctx->sort_collations[recompress_ctx->num_segmentby], &recompress_ctx->nulls_first[recompress_ctx->num_segmentby], maintenance_work_mem, NULL, false); /************** snapshot ****************************/ Snapshot snapshot = RegisterSnapshot(GetTransactionSnapshot()); TupleTableSlot *uncompressed_slot = MakeTupleTableSlot(uncompressed_rel_tupdesc, &TTSOpsMinimalTuple); TupleTableSlot *compressed_slot = table_slot_create(compressed_chunk_rel, NULL); Datum *values = palloc(sizeof(Datum) * recompress_ctx->n_keys); bool *isnulls = palloc(sizeof(bool) * recompress_ctx->n_keys); HeapTuple compressed_tuple; IndexScanDesc index_scan = index_beginscan_compat(compressed_chunk_rel, index_rel, snapshot, NULL, recompress_ctx->num_segmentby, 0); bool found_tuple = fetch_uncompressed_chunk_into_tuplesort(input_tuplesortstate, uncompressed_chunk_rel, snapshot); if (!found_tuple) goto finish; tuplesort_performsort(input_tuplesortstate); for (found_tuple = tuplesort_gettupleslot(input_tuplesortstate, true /*=forward*/, false /*=copy*/, uncompressed_slot, NULL /*=abbrev*/); found_tuple;) { CHECK_FOR_INTERRUPTS(); for (int i = 0; i < recompress_ctx->n_keys; i++) { values[i] = slot_getattr(uncompressed_slot, AttrOffsetGetAttrNumber( recompress_ctx->current_segment[i].chunk_offset), &isnulls[i]); } update_current_segment(recompress_ctx->current_segment, values, isnulls, recompress_ctx->num_segmentby); /* Build scankeys based on uncompressed tuple values */ update_segmentby_scankeys(values, isnulls, recompress_ctx->num_segmentby, recompress_ctx->index_scankeys); update_orderby_scankeys(values, isnulls, recompress_ctx->num_segmentby, recompress_ctx->num_orderby, recompress_ctx->orderby_scankeys); index_rescan(index_scan, recompress_ctx->index_scankeys, recompress_ctx->num_segmentby, NULL, 0); bool done_with_segment = false; bool tuples_for_recompression = false; enum Batch_match_result result; while (index_getnext_slot(index_scan, ForwardScanDirection, compressed_slot)) { /* Check if the uncompressed tuple is before, inside, or after the compressed batch */ result = match_tuple_batch(compressed_slot, recompress_ctx->num_orderby, recompress_ctx->orderby_scankeys, &recompress_ctx->nulls_first[recompress_ctx->num_segmentby]); /* If the tuple is before the batch, add it for recompression * also keep adding uncompressed tuples while they are: * - any left * - before the current batch * - in the same segment group */ while (result == Tuple_before) { tuples_for_recompression = true; tuplesort_puttupleslot(recompress_tuplesortstate, uncompressed_slot); /* If we happen to hit the end of uncompressed tuples or tuple changed segment group * we are done with the segment group */ found_tuple = tuplesort_gettupleslot(input_tuplesortstate, true /*=forward*/, false /*=copy*/, uncompressed_slot, NULL /*=abbrev*/); if (!found_tuple) { done_with_segment = true; break; } for (int i = 0; i < recompress_ctx->n_keys; i++) { values[i] = slot_getattr(uncompressed_slot, AttrOffsetGetAttrNumber( recompress_ctx->current_segment[i].chunk_offset), &isnulls[i]); } done_with_segment = check_changed_group(recompress_ctx->current_segment, values, isnulls, recompress_ctx->num_segmentby); if (done_with_segment) break; update_orderby_scankeys(values, isnulls, recompress_ctx->num_segmentby, recompress_ctx->num_orderby, recompress_ctx->orderby_scankeys); result = match_tuple_batch(compressed_slot, recompress_ctx->num_orderby, recompress_ctx->orderby_scankeys, &recompress_ctx->nulls_first[recompress_ctx->num_segmentby]); } /* If we are done with segment, recompress everything we have so far * and break out of this segment index scan */ if (done_with_segment) { tuples_for_recompression = false; recompress_segment(recompress_tuplesortstate, uncompressed_chunk_rel, &row_compressor, &writer); break; } /* If the tuple matches the batch, add the batch for recompression */ /* Potential optimization: merge uncompressed tuples and decompressed tuples * into the tuplesortstate since they are both already sorted */ if (result == Tuple_match) { tuples_for_recompression = true; bool should_free; compressed_tuple = ExecFetchSlotHeapTuple(compressed_slot, false, &should_free); heap_deform_tuple(compressed_tuple, compressed_rel_tupdesc, decompressor.compressed_datums, decompressor.compressed_is_nulls); row_decompressor_decompress_row_to_tuplesort(&decompressor, recompress_tuplesortstate); if (!delete_tuple_for_recompression(compressed_chunk_rel, &(compressed_slot->tts_tid), snapshot)) ereport(ERROR, (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("aborting recompression due to concurrent updates on " "compressed data, retrying with next policy run"))); CommandCounterIncrement(); if (should_free) heap_freetuple(compressed_tuple); continue; } /* At this point, tuple is after the batch * If there are tuples added for recompression, do it * and continue to the next batch */ if (tuples_for_recompression) { tuples_for_recompression = false; recompress_segment(recompress_tuplesortstate, uncompressed_chunk_rel, &row_compressor, &writer); } } /* End if we are finished with all uncompressed tuples */ if (!found_tuple) { break; } /* Reset index scan if we are done with with this segment */ if (done_with_segment) { continue; } /* We are done with existing batches for this segment group * Everything after this point goes into new batches * until we hit a new segment group or exhaust the uncompressed tuples */ while (!check_changed_group(recompress_ctx->current_segment, values, isnulls, recompress_ctx->num_segmentby)) { tuples_for_recompression = true; tuplesort_puttupleslot(recompress_tuplesortstate, uncompressed_slot); found_tuple = tuplesort_gettupleslot(input_tuplesortstate, true /*=forward*/, false /*=copy*/, uncompressed_slot, NULL /*=abbrev*/); if (!found_tuple) { tuples_for_recompression = false; recompress_segment(recompress_tuplesortstate, uncompressed_chunk_rel, &row_compressor, &writer); break; } for (int i = 0; i < recompress_ctx->num_segmentby; i++) { values[i] = slot_getattr(uncompressed_slot, AttrOffsetGetAttrNumber( recompress_ctx->current_segment[i].chunk_offset), &isnulls[i]); } } if (tuples_for_recompression) { recompress_segment(recompress_tuplesortstate, uncompressed_chunk_rel, &row_compressor, &writer); } } finish: row_compressor_close(&row_compressor); bulk_writer_close(&writer); ExecDropSingleTupleTableSlot(uncompressed_slot); ExecDropSingleTupleTableSlot(compressed_slot); index_endscan(index_scan); UnregisterSnapshot(snapshot); index_close(index_rel, NoLock); row_decompressor_close(&decompressor); tuplesort_end(input_tuplesortstate); tuplesort_end(recompress_tuplesortstate); free_chunk_recompress_ctx(recompress_ctx); /* If we can quickly upgrade the lock, lets try updating the chunk status to fully * compressed. But we need to check if there are any uncompressed tuples in the * relation since somebody might have inserted new tuples while we were recompressing. */ if (ConditionalLockRelation(uncompressed_chunk_rel, ExclusiveLock)) { try_updating_chunk_status(uncompressed_chunk, uncompressed_chunk_rel); } else if (has_unique_constraints) { /* * This can be problematic since we cannot acquire ExclusiveLock meaning its * possible there are inserts going which need to check unique constraints. * Due to the reverse direction of tuple movement, concurrent recompression * and speculative insertion could potentially cause false negatives during * constraint checking. For now, our best option here is to bail. * * We use a spin lock to wait for the ExclusiveLock or bail out if we can't get it in time. */ int lock_retry = 0; while (true) { if (ConditionalLockRelation(uncompressed_chunk_rel, ExclusiveLock)) { try_updating_chunk_status(uncompressed_chunk, uncompressed_chunk_rel); break; } /* * Check for interrupts while trying to (re-)acquire the exclusive * lock. */ CHECK_FOR_INTERRUPTS(); if (++lock_retry > (RECOMPRESS_EXCLUSIVE_LOCK_TIMEOUT / RECOMPRESS_EXCLUSIVE_LOCK_WAIT_INTERVAL)) { /* * We failed to establish the lock in the specified number of * retries. This means we give up trying to get the exclusive lock are abort the * recompression operation */ ereport(ERROR, (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("aborting recompression due to concurrent DML on uncompressed " "data, retrying with next policy run"))); break; } (void) WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, RECOMPRESS_EXCLUSIVE_LOCK_WAIT_INTERVAL, WAIT_EVENT_VACUUM_TRUNCATE); ResetLatch(MyLatch); DEBUG_WAITPOINT("chunk_recompress_after_latch"); } } table_close(uncompressed_chunk_rel, NoLock); table_close(compressed_chunk_rel, NoLock); PG_RETURN_OID(uncompressed_chunk_id); } /* * perform_recompression expects appropriate permissions and checks have already been done. * Relations must have appropriate locks and the CompressionSettings of compressed_chunk and * new_compressed_chunk should match */ static void perform_recompression(RecompressContext *recompress_ctx, Relation compressed_chunk_rel, Relation uncompressed_chunk_rel, Relation index_rel, CompressionSettings *new_settings, Relation new_compressed_chunk_rel) { RowDecompressor decompressor; Tuplesortstate *tuplesortstate; RowCompressor row_compressor; BulkWriter writer; TupleTableSlot *compressed_slot; bool first_iteration = true; IndexScanDesc index_scan; HeapTuple compressed_tuple; PushActiveSnapshot(GetTransactionSnapshot()); decompressor = build_decompressor(RelationGetDescr(compressed_chunk_rel), RelationGetDescr(uncompressed_chunk_rel)); tuplesortstate = tuplesort_begin_heap(RelationGetDescr(uncompressed_chunk_rel), recompress_ctx->n_keys, recompress_ctx->sort_keys, recompress_ctx->sort_operators, recompress_ctx->sort_collations, recompress_ctx->nulls_first, maintenance_work_mem, NULL, false); row_compressor_init(&row_compressor, new_settings, RelationGetDescr(uncompressed_chunk_rel), RelationGetDescr(new_compressed_chunk_rel)); writer = bulk_writer_build(new_compressed_chunk_rel, 0); compressed_slot = table_slot_create(compressed_chunk_rel, NULL); Datum *values = palloc(sizeof(Datum) * recompress_ctx->num_segmentby); bool *isnulls = palloc(sizeof(bool) * recompress_ctx->num_segmentby); /* * we use the compressed chunk's index to scan so that we get the compressed tuples sorted * by segment-by and order-by minmax */ index_scan = index_beginscan_compat(compressed_chunk_rel, index_rel, GetActiveSnapshot(), NULL, 0, 0); index_scan->xs_want_itup = true; index_rescan(index_scan, NULL, 0, NULL, 0); while (index_getnext_slot(index_scan, ForwardScanDirection, compressed_slot)) { for (int i = 0; i < recompress_ctx->num_segmentby; i++) { values[i] = index_getattr(index_scan->xs_itup, AttrOffsetGetAttrNumber(i), index_scan->xs_itupdesc, &isnulls[i]); } if (first_iteration) { update_current_segment(recompress_ctx->current_segment, values, isnulls, recompress_ctx->num_segmentby); first_iteration = false; } else if (check_changed_group(recompress_ctx->current_segment, values, isnulls, recompress_ctx->num_segmentby)) { recompress_segment(tuplesortstate, uncompressed_chunk_rel, &row_compressor, &writer); update_current_segment(recompress_ctx->current_segment, values, isnulls, recompress_ctx->num_segmentby); } bool should_free; compressed_tuple = ExecFetchSlotHeapTuple(compressed_slot, false, &should_free); heap_deform_tuple(compressed_tuple, RelationGetDescr(compressed_chunk_rel), decompressor.compressed_datums, decompressor.compressed_is_nulls); row_decompressor_decompress_row_to_tuplesort(&decompressor, tuplesortstate); if (should_free) heap_freetuple(compressed_tuple); } recompress_segment(tuplesortstate, uncompressed_chunk_rel, &row_compressor, &writer); row_compressor_close(&row_compressor); bulk_writer_close(&writer); ExecDropSingleTupleTableSlot(compressed_slot); index_endscan(index_scan); row_decompressor_close(&decompressor); tuplesort_end(tuplesortstate); PopActiveSnapshot(); } /* * Perform per segment in-memory recompression of a compressed chunk. */ bool recompress_chunk_in_memory_impl(Chunk *uncompressed_chunk) { if (uncompressed_chunk == NULL) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("chunk cannot be NULL"))); Ensure(ts_guc_enable_in_memory_recompression, "in-memory recompression functionality disabled"); if (!ts_chunk_is_compressed(uncompressed_chunk) || ts_chunk_is_frozen(uncompressed_chunk)) return false; Chunk *compressed_chunk = ts_chunk_get_by_id(uncompressed_chunk->fd.compressed_chunk_id, true); Ensure(compressed_chunk != NULL, "compressed chunk not found for chunk \"%s\"", get_rel_name(uncompressed_chunk->table_id)); CompressionSettings *settings = ts_compression_settings_get(uncompressed_chunk->table_id); Ensure(settings != NULL, "compression settings not found for chunk \"%s\"", get_rel_name(uncompressed_chunk->table_id)); Ensure(settings->fd.orderby, "empty order by, cannot recompress in-memory"); LOCKMODE lockmode = ExclusiveLock; Relation uncompressed_chunk_rel = table_open(uncompressed_chunk->table_id, lockmode); Relation compressed_chunk_rel = table_open(compressed_chunk->table_id, lockmode); /* Check new chunk will have the same compression settings */ Hypertable *ht = ts_hypertable_get_by_id(uncompressed_chunk->fd.hypertable_id); CompressionSettings *check_new_settings = ts_compression_settings_get(uncompressed_chunk->hypertable_relid); compression_settings_set_defaults(ht, check_new_settings, ts_alter_table_with_clause_parse(NIL)); if (!ts_array_equal(settings->fd.segmentby, check_new_settings->fd.segmentby)) { table_close(uncompressed_chunk_rel, lockmode); table_close(compressed_chunk_rel, lockmode); return false; } /* Check that the compressed chunk's index exist. TODO: Add support for this scenario */ CatalogIndexState indstate = CatalogOpenIndexes(compressed_chunk_rel); Oid index_oid = get_compressed_chunk_index(indstate, settings); CatalogCloseIndexes(indstate); if (!OidIsValid(index_oid)) { table_close(uncompressed_chunk_rel, lockmode); table_close(compressed_chunk_rel, lockmode); return false; } Relation index_rel = index_open(index_oid, lockmode); RecompressContext *recompress_ctx = compress_chunk_populate_recompress_ctx(settings, uncompressed_chunk_rel, compressed_chunk_rel, index_rel, false); /* Delete old compression settings before creating new compressed chunk to avoid conflict */ ts_compression_settings_delete(uncompressed_chunk->table_id); Hypertable *compressed_ht = ts_hypertable_get_by_id(ht->fd.compressed_hypertable_id); Chunk *new_compressed_chunk = create_compress_chunk(compressed_ht, uncompressed_chunk, InvalidOid); /* The old compression settings were deleted above to avoid catalog conflicts. */ CompressionSettings *new_settings = ts_compression_settings_get(uncompressed_chunk->table_id); Relation new_compressed_chunk_rel = table_open(new_compressed_chunk->table_id, lockmode); Ensure(ts_compression_settings_equal(new_settings, check_new_settings), "compression settings mismatch during recompression of \"%s.%s\"", NameStr(uncompressed_chunk->fd.schema_name), NameStr(uncompressed_chunk->fd.table_name)); perform_recompression(recompress_ctx, compressed_chunk_rel, uncompressed_chunk_rel, index_rel, new_settings, new_compressed_chunk_rel); free_chunk_recompress_ctx(recompress_ctx); index_close(index_rel, NoLock); table_close(uncompressed_chunk_rel, NoLock); table_close(compressed_chunk_rel, NoLock); table_close(new_compressed_chunk_rel, NoLock); LockRelationOid(uncompressed_chunk->table_id, AccessExclusiveLock); LockRelationOid(compressed_chunk->table_id, AccessExclusiveLock); ts_chunk_drop(compressed_chunk, DROP_RESTRICT, -1); if (ts_chunk_clear_status(uncompressed_chunk, CHUNK_STATUS_COMPRESSED_UNORDERED)) ereport(DEBUG1, (errmsg("cleared chunk status for recompression: \"%s.%s\"", NameStr(uncompressed_chunk->fd.schema_name), NameStr(uncompressed_chunk->fd.table_name)))); ts_chunk_set_compressed_chunk(uncompressed_chunk, new_compressed_chunk->fd.id); /* recompress successful */ return true; } static void update_scankey(ScanKey index_scankey, Datum val, bool is_null) { index_scankey->sk_flags = is_null ? SK_ISNULL | SK_SEARCHNULL : 0; index_scankey->sk_argument = val; } static void update_segmentby_scankeys(Datum *values, bool *isnulls, int num_segmentby, ScanKey index_scankeys) { for (int i = 0; i < num_segmentby; i++) { update_scankey(&index_scankeys[i], values[i], isnulls[i]); } } static void update_orderby_scankeys(Datum *values, bool *isnulls, int num_segmentby, int num_orderby, ScanKey orderby_scankeys) { int min_index, max_index; for (int i = 0; i < num_orderby; i++) { min_index = i * 2; max_index = min_index + 1; update_scankey(&orderby_scankeys[min_index], values[num_segmentby + i], isnulls[num_segmentby + i]); update_scankey(&orderby_scankeys[max_index], values[num_segmentby + i], isnulls[num_segmentby + i]); } } static enum Batch_match_result handle_null_scan(int key_flags, bool nulls_first, enum Batch_match_result result) { if (key_flags & SK_ISNULL) return nulls_first ? Tuple_before : Tuple_after; return result; } static enum Batch_match_result match_tuple_batch(TupleTableSlot *compressed_slot, int num_orderby, ScanKey orderby_scankeys, bool *nulls_first) { ScanKey key; for (int i = 0; i < num_orderby; i++) { key = &orderby_scankeys[i * 2]; if (!slot_key_test(compressed_slot, key)) return handle_null_scan(key->sk_flags, nulls_first[i], Tuple_before); key = &orderby_scankeys[i * 2 + 1]; if (!slot_key_test(compressed_slot, key)) return handle_null_scan(key->sk_flags, nulls_first[i], Tuple_after); } return Tuple_match; } static bool fetch_uncompressed_chunk_into_tuplesort(Tuplesortstate *tuplesortstate, Relation uncompressed_chunk_rel, Snapshot snapshot) { bool matching_exist = false; TableScanDesc scan = table_beginscan(uncompressed_chunk_rel, snapshot, 0, 0); TupleTableSlot *slot = table_slot_create(uncompressed_chunk_rel, NULL); while (table_scan_getnextslot(scan, ForwardScanDirection, slot)) { matching_exist = true; slot_getallattrs(slot); tuplesort_puttupleslot(tuplesortstate, slot); if (!delete_tuple_for_recompression(uncompressed_chunk_rel, &slot->tts_tid, snapshot)) ereport(ERROR, (errcode(ERRCODE_T_R_SERIALIZATION_FAILURE), errmsg("aborting recompression due to concurrent updates on " "uncompressed data, retrying with next policy run"))); } ExecDropSingleTupleTableSlot(slot); table_endscan(scan); return matching_exist; } /* Sort the tuples and recompress them */ static void recompress_segment(Tuplesortstate *tuplesortstate, Relation compressed_chunk_rel, RowCompressor *row_compressor, BulkWriter *writer) { tuplesort_performsort(tuplesortstate); row_compressor_reset(row_compressor); row_compressor_append_sorted_rows(row_compressor, tuplesortstate, compressed_chunk_rel, writer); tuplesort_reset(tuplesortstate); CommandCounterIncrement(); } static void update_current_segment(CompressedSegmentInfo *current_segment, Datum *values, bool *isnulls, int nsegmentby_cols) { for (int i = 0; i < nsegmentby_cols; i++) { /* new segment, need to do per-segment processing */ segment_info_update(current_segment[i].segment_info, values[i], isnulls[i]); } } static bool check_changed_group(CompressedSegmentInfo *current_segment, Datum *values, bool *isnulls, int nsegmentby_cols) { for (int i = 0; i < nsegmentby_cols; i++) { if (!segment_info_datum_is_in_group(current_segment[i].segment_info, values[i], isnulls[i])) return true; } return false; } static void init_scankey(ScanKey sk, AttrNumber attnum, Oid atttypid, Oid attcollid, StrategyNumber strategy) { TypeCacheEntry *tce = lookup_type_cache(atttypid, TYPECACHE_BTREE_OPFAMILY); if (!OidIsValid(tce->btree_opf)) elog(ERROR, "no btree opfamily for type \"%s\"", format_type_be(atttypid)); Oid opr = get_opfamily_member(tce->btree_opf, atttypid, atttypid, strategy); /* * Fall back to btree operator input type when it is binary compatible with * the column type and no operator for column type could be found. */ if (!OidIsValid(opr) && IsBinaryCoercible(atttypid, tce->btree_opintype)) { opr = get_opfamily_member(tce->btree_opf, tce->btree_opintype, tce->btree_opintype, strategy); } if (!OidIsValid(opr)) elog(ERROR, "no operator for type \"%s\"", format_type_be(atttypid)); opr = get_opcode(opr); if (!OidIsValid(opr)) elog(ERROR, "no opcode for type \"%s\"", format_type_be(atttypid)); ScanKeyEntryInitialize(sk, 0, /* flags */ attnum, strategy, InvalidOid, /* No strategy subtype. */ attcollid, opr, UnassignedDatum); } static void create_segmentby_scankeys(CompressionSettings *settings, Relation index_rel, Relation compressed_chunk_rel, ScanKeyData *index_scankeys) { int num_segmentby = ts_array_length(settings->fd.segmentby); for (int i = 0; i < num_segmentby; i++) { AttrNumber idx_attnum = AttrOffsetGetAttrNumber(i); AttrNumber in_attnum = index_rel->rd_index->indkey.values[i]; const NameData PG_USED_FOR_ASSERTS_ONLY *attname = attnumAttName(compressed_chunk_rel, in_attnum); Assert(strcmp(NameStr(*attname), ts_array_get_element_text(settings->fd.segmentby, i + 1)) == 0); init_scankey(&index_scankeys[i], idx_attnum, attnumTypeId(index_rel, idx_attnum), attnumCollationId(index_rel, idx_attnum), BTEqualStrategyNumber); } } static void create_orderby_scankeys(CompressionSettings *settings, Relation index_rel, Relation compressed_chunk_rel, ScanKeyData *orderby_scankeys) { int position; int num_orderby = ts_array_length(settings->fd.orderby); /* Create two scankeys per orderby column, for min and max metadata columns respectively */ for (int i = 0; i < num_orderby * 2; i = i + 2) { position = (i / 2) + 1; AttrNumber first_attno = get_attnum(compressed_chunk_rel->rd_id, column_segment_min_name(position)); StrategyNumber first_strategy = BTLessEqualStrategyNumber; AttrNumber second_attno = get_attnum(compressed_chunk_rel->rd_id, column_segment_max_name(position)); StrategyNumber second_strategy = BTGreaterEqualStrategyNumber; Assert(first_attno != InvalidAttrNumber); Assert(second_attno != InvalidAttrNumber); bool is_desc = ts_array_get_element_bool(settings->fd.orderby_desc, position); /* If we are using DESC order, swap the order of metadata scankeys * since we rely on the order to determine whether a tuple is before or after * the compressed batch and the index is also ordered in that way. */ if (is_desc) { AttrNumber temp_attno = first_attno; StrategyNumber temp_strategy = first_strategy; first_attno = second_attno; first_strategy = second_strategy; second_attno = temp_attno; second_strategy = temp_strategy; } init_scankey(&orderby_scankeys[i], first_attno, attnumTypeId(compressed_chunk_rel, first_attno), attnumCollationId(compressed_chunk_rel, first_attno), first_strategy); init_scankey(&orderby_scankeys[i + 1], second_attno, attnumTypeId(compressed_chunk_rel, second_attno), attnumCollationId(compressed_chunk_rel, second_attno), second_strategy); } } /* Deleting a tuple for recompression if we can. * If there is an unexpected result, we should just abort the operation completely. * There are potential optimizations that can be done here in certain scenarios. */ static bool delete_tuple_for_recompression(Relation rel, ItemPointer tid, Snapshot snapshot) { TM_Result result; TM_FailureData tmfd; result = table_tuple_delete(rel, tid, GetCurrentCommandId(true), snapshot, InvalidSnapshot, true /* for now, just wait for commit/abort, that might let us proceed */ , &tmfd, true /* changingPart */); return result == TM_Ok; } /* Check if we can update the chunk status to fully compressed after segmentwise recompression * We can only do this if there were no concurrent DML operations, so we check to see if there are * any uncompressed tuples in the chunk after compression. * If there aren't, we can update the chunk status * * Note: Caller is expected to have an ExclusiveLock on the uncompressed_chunk */ static void try_updating_chunk_status(Chunk *uncompressed_chunk, Relation uncompressed_chunk_rel) { PushActiveSnapshot(GetLatestSnapshot()); TableScanDesc scan = table_beginscan(uncompressed_chunk_rel, GetActiveSnapshot(), 0, 0); ScanDirection scan_dir = BackwardScanDirection; TupleTableSlot *slot = table_slot_create(uncompressed_chunk_rel, NULL); /* Doing a backwards scan with assumption that newly inserted tuples * are most likely at the end of the heap. */ bool has_tuples = false; if (table_scan_getnextslot(scan, scan_dir, slot)) { has_tuples = true; } ExecDropSingleTupleTableSlot(slot); table_endscan(scan); PopActiveSnapshot(); if (!has_tuples) { /* * Only clear PARTIAL. Segmentwise recompression only processes * segments that have new uncompressed data, so segments without new * data are left as-is. Any overlapping batches in those segments * remain as is, so the UNORDERED flag must be preserved. */ if (ts_chunk_clear_status(uncompressed_chunk, CHUNK_STATUS_COMPRESSED_PARTIAL)) ereport(DEBUG1, (errmsg("cleared chunk status for recompression: \"%s.%s\"", NameStr(uncompressed_chunk->fd.schema_name), NameStr(uncompressed_chunk->fd.table_name)))); /* changed chunk status, so invalidate any plans involving this chunk */ CacheInvalidateRelcacheByRelid(uncompressed_chunk->table_id); } } ================================================ FILE: tsl/src/compression/recompress.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <fmgr.h> #include <utils.h> #include "chunk.h" /* Structure to hold recompression sort information */ typedef struct RecompressContext { int num_segmentby; int num_orderby; int n_keys; AttrNumber *sort_keys; Oid *sort_operators; Oid *sort_collations; bool *nulls_first; CompressedSegmentInfo *current_segment; ScanKeyData *index_scankeys; ScanKeyData *orderby_scankeys; } RecompressContext; extern Datum tsl_recompress_chunk_segmentwise(PG_FUNCTION_ARGS); Oid recompress_chunk_segmentwise_impl(Chunk *chunk); bool recompress_chunk_in_memory_impl(Chunk *uncompressed_chunk); /* Result of matching an uncompressed tuple against a compressed batch */ enum Batch_match_result { Tuple_before = 1, Tuple_match, Tuple_after, _Batch_match_result_max, }; ================================================ FILE: tsl/src/compression/sparse_index_bloom1.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include "postgres.h" Datum bloom1_contains(PG_FUNCTION_ARGS); Datum bloom1_contains_any(PG_FUNCTION_ARGS); PGFunction bloom1_get_hash_function(Oid type, FmgrInfo **finfo); extern char const *bloom1_column_prefix; /* * We used to have two possible hashes depending on the build configuration, * which were incompatible with each other. They both used the "bloom1" column * prefix. This could lead to false negatives if a database was updated to a * different build of the TimescaleDB extension. Now these hashing configuration * use different prefixes. The bloom filter is still constructed according to * the "bloom1" rules. */ #ifdef TS_USE_UMASH #define default_bloom1_column_prefix "bloomh" #else #define default_bloom1_column_prefix "bloomg" #endif ================================================ FILE: tsl/src/compression/wal_utils.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <replication/message.h> #include "guc.h" /* * Utils to insert markers into the WAL log which demarcate compression * and decompression operations. * The primary purpose is to be able to discern between user-driven DML * operations (caused by statements which INSERT/UPDATE/DELETE data), and * compression-driven DML (moving data to/from compressed chunks). */ #define COMPRESSION_MARKER_START "::timescaledb-compression-start" #define COMPRESSION_MARKER_END "::timescaledb-compression-end" #define DECOMPRESSION_MARKER_START "::timescaledb-decompression-start" #define DECOMPRESSION_MARKER_END "::timescaledb-decompression-end" static inline bool is_compression_wal_markers_enabled() { return ts_guc_enable_compression_wal_markers && XLogLogicalInfoActive(); } static inline void write_logical_replication_msg_compression_start() { if (is_compression_wal_markers_enabled()) { LogLogicalMessageCompat(COMPRESSION_MARKER_START, "", 0, true, true); } } static inline void write_logical_replication_msg_compression_end() { if (is_compression_wal_markers_enabled()) { LogLogicalMessageCompat(COMPRESSION_MARKER_END, "", 0, true, true); } } static inline void write_logical_replication_msg_decompression_start() { if (is_compression_wal_markers_enabled()) { LogLogicalMessageCompat(DECOMPRESSION_MARKER_START, "", 0, true, true); } } static inline void write_logical_replication_msg_decompression_end() { if (is_compression_wal_markers_enabled()) { LogLogicalMessageCompat(DECOMPRESSION_MARKER_END, "", 0, true, true); } } ================================================ FILE: tsl/src/continuous_aggs/CMakeLists.txt ================================================ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/common.c ${CMAKE_CURRENT_SOURCE_DIR}/create.c ${CMAKE_CURRENT_SOURCE_DIR}/finalize.c ${CMAKE_CURRENT_SOURCE_DIR}/insert.c ${CMAKE_CURRENT_SOURCE_DIR}/invalidation_threshold.c ${CMAKE_CURRENT_SOURCE_DIR}/invalidation.c ${CMAKE_CURRENT_SOURCE_DIR}/materialize.c ${CMAKE_CURRENT_SOURCE_DIR}/options.c ${CMAKE_CURRENT_SOURCE_DIR}/planner.c ${CMAKE_CURRENT_SOURCE_DIR}/refresh.c ${CMAKE_CURRENT_SOURCE_DIR}/utils.c) target_sources(${TSL_LIBRARY_NAME} PRIVATE ${SOURCES}) ================================================ FILE: tsl/src/continuous_aggs/README.md ================================================ # Continuous Aggregates # A continuous aggregate is a special kind of materialized view for aggregates that can be partially and continuously refreshed, either manually or automated by a policy that runs in the background. Unlike a regular materialized view, a continuous aggregate doesn't require complete re-materialization on every refresh. Instead, it is possible to refresh a subset of the continuous aggregate at relatively low cost, thus enabling continuous aggregation as new data is written or old data is updated and/or backfilled. To enable continuous aggregation, a continuous aggregate stores partial aggregations for every time bucket in an internal hypertable. The advantage of this configuration is that each time bucket can be recomputed individually, without requiring updates to other buckets, and buckets can be combined to form more granular aggregates (e.g., hourly buckets can be combined to form daily buckets). Finalization of the partial buckets happens automatically at query time. Although such finalization gives slightly higher querying times, it is offset by more efficient refreshes that only recompute the buckets that have been "invalidated" by changes in the raw data. A continuous aggregate policy automates the refreshing, allowing the aggregate to stay up-to-date without manual intervention. A policy can be configured to only refresh the most recent data (e.g., just the last hour's worth of data) or ensure that the continuous aggregate is always up-to-date with the underlying source data. Policies that focus on recent data allow older parts of the continuous aggregate to stay the same or be governed by manual refreshes. ## Bookkeeping and Internal State ## TimescaleDB does bookkeeping for each continuous aggregate to know which buckets of the aggregates require refreshing. Whenever a modification happens to the source data, an invalidation for the modified region is written to an invalidation log. However, invalidations are not written after the *invalidation threshold*, which tracks the latest bucket materialized thus far. This threshold allows write amplification to be kept to a minimum by not writing invalidations for "hot" time buckets that are assumed to still have data being written to them. Thus, to store, maintain, and query aggregations, continuous aggregates consist of the following objects: 1. A user view, which queries and finalizes the aggregations and is also the object that users interact with. 2. A partial view, which is used to materialize new data. 3. A direct view, which holds the original query that users specified. 4. An internal materialization hypertable, containing the materialized data as partial aggregates for each time bucket. 5. An invalidation threshold, which is a timestamp that tracks the latest materialization. Invalidations that occur before this timestamp will be logged, while invalidations after it will not be logged. 6. A trigger on the source hypertable that writes invalidations to the hypertable invalidation log at transaction end, based on INSERT, UPDATE, and DELETE statements that mutate the data. 7. A hypertable invalidation log that tracks invalidated regions of data for each hypertable. Entries in this log contain time ranges that need to be re-materialized across all the hypertable's continuous aggregates. 8. A materialization invalidation log. Once a refresh runs on a given continuous aggregate, this log tracks how invalidations from the hypertable invalidation log are processed against the refresh window for the refreshed continuous aggregate. Thus, a single invalidation in the hypertable invalidation log becomes one entry per continuous aggregate in the materialization invalidation log. ## The materialized hypertable ## The materialized hypertable does not store the aggregate's output, but rather the partial aggregate state. For instance, in case of an average, each bucket stores the sum and count in an internal binary form. The partial aggregates are what gives continuous aggregates flexibility; buckets can be individually updated and multiple partial aggregates can be combined to form new partials. Future enhancements may allow aggregating at different time resolutions using the the same underlying continuous aggregate. ## The Invalidation Log and Threshold ## There are two mechanisms for collecting hypertable invalidations: 1. Using ModifyHypertable hooks for INSERT / UPDATE / DELETE 2. Using a hook in our custom COPY implementation ### Invalidation Log Table Mutating transactions must record their mutations in the invalidation log, so that a refresh knows to re-materialize the invalidated range. To reduce the extra writes by invalidations, only one invalidation range (lowest and highest modified value) is written at the end of a mutating transaction. As a result, a refresh might materialize more data than necessary, but the insert incurs a smaller overhead instead. Write amplification is further reduced by never writing invalidations after the invalidation threshold, which can be configured to lag behind the time bucket that sees the most writes. Whenever a refresh occurs across a time range that is newer than the current invalidation threshold, the threshold must first be moved to the end of the refreshed region so that new invalidations are recorded in the region after the refresh. However, mutations in the refreshed region can also happen concurrently with the refresh, so, in order to not lose any invalidations, the invalidation threshold must be moved in its own transaction before the new region is materialized. Thus, every refresh may happen across two transactions; first one that moves the invalidation threshold (if necessary) and a second one that does the actual materialization of new data. The second transaction of the refresh will only materialize regions that are recorded as invalid in the invalidation log. Thus, the initial state of a continuous aggregate is to have an entry in the invalidation log that invalidates the entire range of the aggregate. During the refresh, the log is processed and invalidations are cut against the given refresh window, leaving only invalidation entries that are outside the refresh window. Subsequently, if the refresh window does not match any invalidations, there is nothing to refresh either. ## Distribution of functions across files Each source file has an associated header file that should be included to use functions from the corresponding file. <dl> <dt>`common.c`</dt> <dd>This file contains the functions common in all scenarios of creating a continuous aggregates.</dd> <dt>`create.c`</dt> <dd>This file contains the functions that are directly responsible for the creation of the continuous aggregates, like creating hypertable, catalog_entry, view, etc.</dd> <dt>`finalize.c`</dt> <dd>This file contains the specific functions for the case when continous aggregates are created in old format.</dd> <dt>`materialize.c`</dt> <dd>This file contains the functions directly dealing with the materialization of the continuous aggregates.</dd> <dt>`invalidation.c`</dt> <dd>Functions related to invalidation processing for continuous aggregates.</dd> </dl> ================================================ FILE: tsl/src/continuous_aggs/common.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include "common.h" #include <utils/acl.h> #include <utils/date.h> #include <utils/timestamp.h> #include <utils/uuid.h> #include "extension.h" #include "guc.h" static Const *check_time_bucket_argument(Node *arg, char *position, bool process_checks); static void caggtimebucketinfo_init(ContinuousAggTimeBucketInfo *src, int32 hypertable_id, Oid hypertable_oid, AttrNumber hypertable_partition_colno, Oid hypertable_partition_coltype, int64 hypertable_partition_col_interval, int32 parent_mat_hypertable_id); static void process_additional_timebucket_parameter(ContinuousAggBucketFunction *bf, Const *arg, bool *custom_origin); static void process_timebucket_parameters(FuncExpr *fe, ContinuousAggBucketFunction *bf, bool process_checks, bool is_cagg_create, AttrNumber htpartcolno); static void caggtimebucket_validate(ContinuousAggTimeBucketInfo *tbinfo, List *groupClause, List *targetList, List *rtable, bool is_cagg_create); static bool cagg_query_supported(const Query *query, StringInfo hint, StringInfo detail); static Datum get_bucket_width_datum(ContinuousAggTimeBucketInfo bucket_info); static int64 get_bucket_width(ContinuousAggTimeBucketInfo bucket_info); static FuncExpr *build_conversion_call(Oid type, FuncExpr *boundary); static FuncExpr *build_boundary_call(int32 ht_id, Oid type); static Const *cagg_boundary_make_lower_bound(Oid type); static Node *build_union_query_quals(int32 ht_id, Oid partcoltype, Oid opno, int varno, AttrNumber attno); static RangeTblEntry *makeRangeTblEntry(Query *subquery, const char *aliasname); static bool time_bucket_info_has_fixed_width(const ContinuousAggBucketFunction *bf); #define INTERNAL_TO_DATE_FUNCTION "to_date" #define INTERNAL_TO_TSTZ_FUNCTION "to_timestamp" #define INTERNAL_TO_TS_FUNCTION "to_timestamp_without_timezone" #define BOUNDARY_FUNCTION "cagg_watermark" static Const * check_time_bucket_argument(Node *arg, char *position, bool process_checks) { if (IsA(arg, NamedArgExpr)) arg = (Node *) castNode(NamedArgExpr, arg)->arg; Node *expr = eval_const_expressions(NULL, arg); if (process_checks && !IsA(expr, Const)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("only immutable expressions allowed in time bucket function"), errhint("Use an immutable expression as %s argument to the time bucket function.", position))); return castNode(Const, expr); } /* * Initialize caggtimebucket. */ static void caggtimebucketinfo_init(ContinuousAggTimeBucketInfo *src, int32 hypertable_id, Oid hypertable_oid, AttrNumber hypertable_partition_colno, Oid hypertable_partition_coltype, int64 hypertable_partition_col_interval, int32 parent_mat_hypertable_id) { src->htid = hypertable_id; src->parent_mat_hypertable_id = parent_mat_hypertable_id; src->htoid = hypertable_oid; src->htoidparent = InvalidOid; src->htpartcolno = hypertable_partition_colno; src->htpartcoltype = hypertable_partition_coltype; src->htpartcol_interval_len = hypertable_partition_col_interval; /* Initialize bucket function data structure */ src->bf = palloc0(sizeof(ContinuousAggBucketFunction)); src->bf->bucket_function = InvalidOid; src->bf->bucket_width_type = InvalidOid; /* Time based buckets */ src->bf->bucket_time_width = NULL; /* not specified by default */ src->bf->bucket_time_timezone = NULL; /* not specified by default */ src->bf->bucket_time_offset = NULL; /* not specified by default */ TIMESTAMP_NOBEGIN(src->bf->bucket_time_origin); /* origin is not specified by default */ /* Integer based buckets */ src->bf->bucket_integer_width = 0; /* invalid value */ src->bf->bucket_integer_offset = 0; /* invalid value */ } /* * Check if the supplied OID belongs to a valid bucket function * for continuous aggregates. */ bool function_allowed_in_cagg_definition(Oid funcid) { FuncInfo *finfo = ts_func_cache_get_bucketing_func(funcid); if (finfo == NULL) return false; if (finfo->allowed_in_cagg_definition) return true; return false; } /* * When a view is created (StoreViewQuery), 2 dummy rtable entries corresponding to "old" and * "new" are prepended to the rtable list. We remove these and adjust the varnos to recreate * the user or direct view query. */ void RemoveRangeTableEntries(Query *query) { #if PG16_LT List *rtable = query->rtable; Assert(list_length(rtable) >= 3); rtable = list_delete_first(rtable); query->rtable = list_delete_first(rtable); OffsetVarNodes((Node *) query, -2, 0); Assert(list_length(query->rtable) >= 1); #endif } /* * Extract the final view from the UNION ALL query. * * q1 is the query on the materialization hypertable with the finalize call * q2 is the query on the raw hypertable which was supplied in the initial CREATE VIEW statement * returns q1 from: * SELECT * from ( SELECT * from q1 where <coale_qual> * UNION ALL * SELECT * from q2 where existing_qual and <coale_qual> * where coale_qual is: time < ----> (or >= ) * COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark( <htid>)), * '-infinity'::timestamp with time zone) * The WHERE clause of the final view is removed. */ Query * destroy_union_query(Query *q) { Assert(q->commandType == CMD_SELECT && ((SetOperationStmt *) q->setOperations)->op == SETOP_UNION && ((SetOperationStmt *) q->setOperations)->all == true); /* Get RTE of the left-hand side of UNION ALL. */ RangeTblEntry *rte = linitial(q->rtable); Assert(rte->rtekind == RTE_SUBQUERY); Query *query = copyObject(rte->subquery); /* Delete the WHERE clause from the final view. */ query->jointree->quals = NULL; return query; } /* * Handle additional parameter of the timebucket function such as timezone, offset, or origin */ static void process_additional_timebucket_parameter(ContinuousAggBucketFunction *bf, Const *arg, bool *custom_origin) { char *tz_name; switch (exprType((Node *) arg)) { /* Timezone as text */ case TEXTOID: tz_name = TextDatumGetCString(arg->constvalue); if (!ts_is_valid_timezone_name(tz_name)) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid timezone name \"%s\"", tz_name))); } bf->bucket_time_timezone = tz_name; break; case INTERVALOID: /* Bucket offset as interval */ bf->bucket_time_offset = DatumGetIntervalP(arg->constvalue); break; case DATEOID: /* Bucket origin as Date */ if (!arg->constisnull) bf->bucket_time_origin = date2timestamptz_opt_overflow(DatumGetDateADT(arg->constvalue), NULL); *custom_origin = true; break; case TIMESTAMPOID: /* Bucket origin as Timestamp */ bf->bucket_time_origin = DatumGetTimestamp(arg->constvalue); *custom_origin = true; break; case TIMESTAMPTZOID: /* Bucket origin as TimestampTZ */ bf->bucket_time_origin = DatumGetTimestampTz(arg->constvalue); *custom_origin = true; break; case INT2OID: /* Bucket offset as smallint */ bf->bucket_integer_offset = DatumGetInt16(arg->constvalue); break; case INT4OID: /* Bucket offset as int */ bf->bucket_integer_offset = DatumGetInt32(arg->constvalue); break; case INT8OID: /* Bucket offset as bigint */ bf->bucket_integer_offset = DatumGetInt64(arg->constvalue); break; default: ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("unable to handle time_bucket parameter of type: %s", format_type_be(exprType((Node *) arg))))); pg_unreachable(); } } /* * Process the FuncExpr node to fill the bucket function data structure. The other * parameters are used when `process_check` is true that means we need to raise errors * when invalid parameters are passed to the time bucket function when creating a cagg. */ static void process_timebucket_parameters(FuncExpr *fe, ContinuousAggBucketFunction *bf, bool process_checks, bool is_cagg_create, AttrNumber htpartcolno) { Node *width_arg; Node *col_arg; bool custom_origin = false; TIMESTAMP_NOBEGIN(bf->bucket_time_origin); int nargs; /* Only column allowed : time_bucket('1day', <column> ) */ col_arg = lsecond(fe->args); /* Could be a named argument */ if (IsA(col_arg, NamedArgExpr)) col_arg = (Node *) castNode(NamedArgExpr, col_arg)->arg; if (process_checks && htpartcolno != InvalidAttrNumber && (!(IsA(col_arg, Var)) || castNode(Var, col_arg)->varattno != htpartcolno)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("time bucket function must reference the primary hypertable " "dimension column"))); nargs = list_length(fe->args); Assert(nargs >= 2 && nargs <= 5); /* * Process the third argument of the time bucket function. This could be `timezone`, `offset`, * or `origin`. * * Time bucket function variations with 3 and 5 arguments: * - time_bucket(width SMALLINT, ts SMALLINT, offset SMALLINT) * - time_bucket(width INTEGER, ts INTEGER, offset INTEGER) * - time_bucket(width BIGINT, ts BIGINT, offset BIGINT) * - time_bucket(width INTERVAL, ts DATE, offset INTERVAL) * - time_bucket(width INTERVAL, ts DATE, origin DATE) * - time_bucket(width INTERVAL, ts TIMESTAMPTZ, offset INTERVAL) * - time_bucket(width INTERVAL, ts TIMESTAMPTZ, origin TIMESTAMPTZ) * - time_bucket(width INTERVAL, ts TIMESTAMPTZ, timezone TEXT, origin TIMESTAMPTZ, * offset INTERVAL) * - time_bucket(width INTERVAL, ts TIMESTAMP, offset INTERVAL) * - time_bucket(width INTERVAL, ts TIMESTAMP, origin TIMESTAMP) */ if (nargs >= 3) { Const *arg = check_time_bucket_argument(lthird(fe->args), "third", process_checks); process_additional_timebucket_parameter(bf, arg, &custom_origin); } /* * Process the fourth and fifth arguments of the time bucket function. This could be `origin` or * `offset`. * * Time bucket function variation with 5 arguments: * - time_bucket(width INTERVAL, ts TIMESTAMPTZ, timezone TEXT, origin TIMESTAMPTZ, * offset INTERVAL) */ if (nargs >= 4) { Const *arg = check_time_bucket_argument(lfourth(fe->args), "fourth", process_checks); process_additional_timebucket_parameter(bf, arg, &custom_origin); } if (nargs == 5) { Const *arg = check_time_bucket_argument(lfifth(fe->args), "fifth", process_checks); process_additional_timebucket_parameter(bf, arg, &custom_origin); } if (process_checks && custom_origin && TIMESTAMP_NOT_FINITE(bf->bucket_time_origin)) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid origin value: infinity"))); } /* * We constify width expression here so any immutable expression will be allowed. * Otherwise it would make it harder to create caggs for hypertables with e.g. int8 * partitioning column as int constants default to int4 and so expression would * have a cast and not be a Const. */ width_arg = linitial(fe->args); if (IsA(width_arg, NamedArgExpr)) width_arg = (Node *) castNode(NamedArgExpr, width_arg)->arg; width_arg = eval_const_expressions(NULL, width_arg); if (IsA(width_arg, Const)) { Const *width = castNode(Const, width_arg); bf->bucket_width_type = width->consttype; if (width->constisnull) { if (process_checks && is_cagg_create) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid bucket width for time bucket function"))); } else { if (width->consttype == INTERVALOID) { bf->bucket_time_width = DatumGetIntervalP(width->constvalue); } if (!IS_TIME_BUCKET_INFO_TIME_BASED(bf)) { bf->bucket_integer_width = ts_interval_value_to_internal(width->constvalue, width->consttype); } } } else { if (process_checks) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("only immutable expressions allowed in time bucket function"), errhint("Use an immutable expression as first argument to the time bucket " "function."))); } bf->bucket_function = fe->funcid; bf->bucket_time_based = ts_continuous_agg_bucket_on_interval(bf->bucket_function); bf->bucket_fixed_interval = time_bucket_info_has_fixed_width(bf); } /* * Check if the group-by clauses has exactly 1 time_bucket(.., <col>) where * <col> is the hypertable's partitioning column and other invariants. Then fill * the `bucket_width` and other fields of `tbinfo`. */ static void caggtimebucket_validate(ContinuousAggTimeBucketInfo *tbinfo, List *groupClause, List *targetList, List *rtable, bool is_cagg_create) { ListCell *l; bool found = false; /* Make sure tbinfo was initialized. This assumption is used below. */ Assert(tbinfo->bf->bucket_integer_width == 0); Assert(tbinfo->bf->bucket_time_timezone == NULL); Assert(TIMESTAMP_NOT_FINITE(tbinfo->bf->bucket_time_origin)); List *group_exprs = get_sortgrouplist_exprs(groupClause, targetList); #if PG18_GE /* PG18 introduced RTEs for group clauses so * we can just use rtable to look for GROUP BY expressions. * * https://github.com/postgres/postgres/commit/247dea89 */ List *group_rte_exprs = NIL; foreach (l, rtable) { RangeTblEntry *rte = (RangeTblEntry *) lfirst(l); if (rte->rtekind == RTE_GROUP) group_rte_exprs = list_concat(group_rte_exprs, rte->groupexprs); } group_exprs = group_rte_exprs; #endif foreach (l, group_exprs) { Expr *expr = (Expr *) lfirst(l); if (IsA(expr, FuncExpr)) { FuncExpr *fe = castNode(FuncExpr, expr); /* Filter any non bucketing functions */ FuncInfo *finfo = ts_func_cache_get_bucketing_func(fe->funcid); if (finfo == NULL || !finfo->is_bucketing_func) { continue; } /* Do we have a bucketing function that is not allowed in the CAgg definition? * * This is only validated upon creation. If an older TSDB version has allowed us to use * the function and it's now removed from the list of allowed functions, we should not * error out (e.g., materialized_only setting is changed on a CAgg that uses the * deprecated time_bucket_ng function). */ if (!function_allowed_in_cagg_definition(fe->funcid)) { continue; } if (found) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("continuous aggregate view cannot contain" " multiple time bucket functions"))); else found = true; process_timebucket_parameters(fe, tbinfo->bf, true, is_cagg_create, tbinfo->htpartcolno); } } if (tbinfo->bf->bucket_time_offset != NULL && TIMESTAMP_NOT_FINITE(tbinfo->bf->bucket_time_origin) == false) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("using offset and origin in a time_bucket function at the same time is not " "supported"))); } if (!time_bucket_info_has_fixed_width(tbinfo->bf)) { /* Variable-sized buckets can be used only with intervals. */ Assert(tbinfo->bf->bucket_time_width != NULL); Assert(IS_TIME_BUCKET_INFO_TIME_BASED(tbinfo->bf)); if ((tbinfo->bf->bucket_time_width->month != 0) && ((tbinfo->bf->bucket_time_width->day != 0) || (tbinfo->bf->bucket_time_width->time != 0))) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("invalid interval specified"), errhint("Use either months or days and hours, but not months, days and hours " "together"))); } } if (!found) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("continuous aggregate view must include a valid time bucket function"))); } /* * Check query and extract error details and error hints. * * Returns: * True if the query is supported, false otherwise with hints and errors * added. */ static bool cagg_query_supported(const Query *query, StringInfo hint, StringInfo detail) { if (!query->jointree->fromlist) { appendStringInfoString(hint, "FROM clause missing in the query"); return false; } if (query->commandType != CMD_SELECT) { appendStringInfoString(hint, "Use a SELECT query in the continuous aggregate view."); return false; } if (query->hasWindowFuncs) { if (ts_guc_enable_cagg_window_functions) { elog(WARNING, "window function support is experimental and may result in unexpected results " "depending on the functions used."); } else { appendStringInfoString(detail, "Window function support not enabled."); appendStringInfoString(hint, "Enable experimental window function support by setting " "timescaledb.enable_cagg_window_functions."); return false; } } if (query->hasDistinctOn || query->distinctClause) { appendStringInfoString(detail, "DISTINCT / DISTINCT ON queries are not supported by continuous " "aggregates."); return false; } if (query->limitOffset || query->limitCount) { appendStringInfoString(detail, "LIMIT and LIMIT OFFSET are not supported in queries defining " "continuous aggregates."); appendStringInfoString(hint, "Use LIMIT and LIMIT OFFSET in SELECTS from the continuous " "aggregate view instead."); return false; } if (query->hasRecursive || query->hasSubLinks || query->cteList) { appendStringInfoString(detail, "CTEs and subqueries are not supported by " "continuous aggregates."); return false; } if (query->hasForUpdate || query->hasModifyingCTE) { appendStringInfoString(detail, "Data modification is not allowed in continuous aggregate view " "definitions."); return false; } if (query->hasRowSecurity) { appendStringInfoString(detail, "Row level security is not supported by continuous aggregate " "views."); return false; } if (query->groupingSets) { appendStringInfoString(detail, "GROUP BY GROUPING SETS, ROLLUP and CUBE are not supported by " "continuous aggregates"); appendStringInfoString(hint, "Define multiple continuous aggregates with different grouping " "levels."); return false; } if (query->setOperations) { appendStringInfoString(detail, "UNION, EXCEPT & INTERSECT are not supported by continuous " "aggregates"); return false; } if (!query->groupClause) { /* * Query can have aggregate without group by , so look * for groupClause. */ appendStringInfoString(hint, "Include at least one aggregate function" " and a GROUP BY clause with time bucket."); return false; } return true; /* Query was OK and is supported. */ } static Datum get_bucket_width_datum(ContinuousAggTimeBucketInfo bucket_info) { Datum width = UnassignedDatum; switch (bucket_info.bf->bucket_width_type) { case INT8OID: case INT4OID: case INT2OID: width = ts_internal_to_interval_value(bucket_info.bf->bucket_integer_width, bucket_info.bf->bucket_width_type); break; case INTERVALOID: width = IntervalPGetDatum(bucket_info.bf->bucket_time_width); break; default: Assert(false); } return width; } static int64 get_bucket_width(ContinuousAggTimeBucketInfo bucket_info) { int64 width = 0; /* Calculate the width. */ switch (bucket_info.bf->bucket_width_type) { case INT8OID: case INT4OID: case INT2OID: width = bucket_info.bf->bucket_integer_width; break; case INTERVALOID: { /* * Original interval should not be changed, hence create a local copy * for this check. */ Interval interval = *bucket_info.bf->bucket_time_width; /* * epoch will treat year as 365.25 days. This leads to the unexpected * result that year is not multiple of day or month, which is perceived * as a bug. For that reason, we treat all months as 30 days regardless of year */ if (interval.month && !interval.day && !interval.time) { interval.day = interval.month * DAYS_PER_MONTH; interval.month = 0; } /* Convert Interval to int64 */ width = ts_interval_value_to_internal(IntervalPGetDatum(&interval), INTERVALOID); break; } default: Assert(false); } return width; } ContinuousAggTimeBucketInfo cagg_validate_query(const Query *query, const char *cagg_schema, const char *cagg_name, const bool is_cagg_create) { ContinuousAggTimeBucketInfo bucket_info = { 0 }; ContinuousAggTimeBucketInfo bucket_info_parent = { 0 }; Hypertable *ht = NULL, *ht_parent = NULL; RangeTblEntry *rte = NULL; StringInfoData hint; StringInfoData detail; bool is_hierarchical = false; Query *prev_query = NULL; ContinuousAgg *cagg_parent = NULL; initStringInfo(&hint); initStringInfo(&detail); if (!cagg_query_supported(query, &hint, &detail)) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("invalid continuous aggregate query"), hint.len > 0 ? errhint("%s", hint.data) : 0, detail.len > 0 ? errdetail("%s", detail.data) : 0)); } int num_hypertables = 0; ListCell *lc; foreach (lc, query->rtable) { RangeTblEntry *inner_rte = lfirst_node(RangeTblEntry, lc); if (inner_rte->rtekind == RTE_RELATION) { bool is_hypertable = ts_is_hypertable(inner_rte->relid) || ts_continuous_agg_find_by_relid(inner_rte->relid); if (is_hypertable) { num_hypertables++; if (rte == NULL) rte = copyObject(inner_rte); } if (is_hypertable && inner_rte->inh == false) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("invalid continuous aggregate view"), errdetail( "FROM ONLY on hypertables is not allowed in continuous aggregate."))); } /* Only inner joins are allowed. */ if (inner_rte->jointype != JOIN_INNER && inner_rte->jointype != JOIN_LEFT) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("only INNER or LEFT joins are supported in continuous aggregates"))); /* Subquery only using LATERAL */ if (inner_rte->subquery && !inner_rte->lateral) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("invalid continuous aggregate view"), errdetail("Sub-queries are not supported in FROM clause."))); /* TABLESAMPLE not allowed */ if (inner_rte->tablesample) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("invalid continuous aggregate view"), errdetail("TABLESAMPLE is not supported in continuous aggregate."))); } if (num_hypertables > 1) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("invalid continuous aggregate view"), errdetail("Only one hypertable is allowed in continuous aggregate view."))); if (rte == NULL) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("invalid continuous aggregate view"), errdetail("At least one hypertable should be used in the view definition."))); } const Dimension *part_dimension = NULL; int32 parent_mat_hypertable_id = INVALID_HYPERTABLE_ID; Cache *hcache = ts_hypertable_cache_pin(); if (rte->relkind == RELKIND_RELATION) { ht = ts_hypertable_cache_get_entry(hcache, rte->relid, CACHE_FLAG_MISSING_OK); if (!ht) { ts_cache_release(&hcache); ereport(ERROR, (errcode(ERRCODE_TS_HYPERTABLE_NOT_EXIST), errmsg("table \"%s\" is not a hypertable", get_rel_name(rte->relid)))); } } else { cagg_parent = ts_continuous_agg_find_by_relid(rte->relid); if (!cagg_parent) { ts_cache_release(&hcache); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("invalid continuous aggregate query"), errhint("Continuous aggregate needs to query hypertable or another " "continuous aggregate."))); } parent_mat_hypertable_id = cagg_parent->data.mat_hypertable_id; ht = ts_hypertable_cache_get_entry_by_id(hcache, cagg_parent->data.mat_hypertable_id); /* If parent cagg is hierarchical then we should get the matht otherwise the rawht. */ if (ContinuousAggIsHierarchical(cagg_parent)) ht_parent = ts_hypertable_cache_get_entry_by_id(hcache, cagg_parent->data.mat_hypertable_id); else ht_parent = ts_hypertable_cache_get_entry_by_id(hcache, cagg_parent->data.raw_hypertable_id); /* Get the querydef for the source cagg. */ is_hierarchical = true; prev_query = ts_continuous_agg_get_query(cagg_parent); } /* * Check if user can refresh continuous aggregate * We only check for SELECT on the hypertable here but there * could be other permissions needed depending on the query. * For WITH DATA this is not a problem since we try a refresh * immediately but for WITH NO DATA the refresh might still * fail due to other permissions being needed. */ AclResult aclresult = pg_class_aclcheck(ht->main_table_relid, GetUserId(), ACL_SELECT); if (aclresult != ACLCHECK_OK) { /* User doesn't have permission */ aclcheck_error(aclresult, get_relkind_objtype(get_rel_relkind(ht->main_table_relid)), get_rel_name(ht->main_table_relid)); } if (TS_HYPERTABLE_IS_INTERNAL_COMPRESSION_TABLE(ht)) { ts_cache_release(&hcache); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("hypertable is an internal compressed hypertable"))); } if (rte->relkind == RELKIND_RELATION) { ContinuousAggHypertableStatus status = ts_continuous_agg_hypertable_status(ht->fd.id); /* Prevent create a CAGG over an existing materialization hypertable. */ if (status == HypertableIsMaterialization || status == HypertableIsMaterializationAndRaw) { const ContinuousAgg *cagg = ts_continuous_agg_find_by_mat_hypertable_id(ht->fd.id, false); Assert(cagg != NULL); ts_cache_release(&hcache); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("hypertable is a continuous aggregate materialization table"), errdetail("Materialization hypertable \"%s.%s\".", NameStr(ht->fd.schema_name), NameStr(ht->fd.table_name)), errhint("Do you want to use continuous aggregate \"%s.%s\" instead?", NameStr(cagg->data.user_view_schema), NameStr(cagg->data.user_view_name)))); } } /* Get primary partitioning column information. */ part_dimension = hyperspace_get_open_dimension(ht->space, 0); /* * NOTE: if we ever allow custom partitioning functions we'll need to * change part_dimension->fd.column_type to partitioning_type * below, along with any other fallout. */ if (part_dimension == NULL || part_dimension->partitioning != NULL) { ts_cache_release(&hcache); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("custom partitioning functions not supported" " with continuous aggregates"))); } if (IS_INTEGER_TYPE(ts_dimension_get_partition_type(part_dimension)) && rte->relkind == RELKIND_RELATION) { const char *funcschema = NameStr(part_dimension->fd.integer_now_func_schema); const char *funcname = NameStr(part_dimension->fd.integer_now_func); if (strlen(funcschema) == 0 || strlen(funcname) == 0) { ts_cache_release(&hcache); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("custom time function required on hypertable \"%s\"", get_rel_name(ht->main_table_relid)), errdetail("An integer-based hypertable requires a custom time function to " "support continuous aggregates."), errhint("Set a custom time function on the hypertable."))); } } caggtimebucketinfo_init(&bucket_info, ht->fd.id, ht->main_table_relid, part_dimension->column_attno, part_dimension->fd.column_type, part_dimension->fd.interval_length, parent_mat_hypertable_id); if (is_hierarchical) { const Dimension *part_dimension_parent = hyperspace_get_open_dimension(ht_parent->space, 0); caggtimebucketinfo_init(&bucket_info_parent, ht_parent->fd.id, ht_parent->main_table_relid, part_dimension_parent->column_attno, part_dimension_parent->fd.column_type, part_dimension_parent->fd.interval_length, INVALID_HYPERTABLE_ID); } ts_cache_release(&hcache); /* * We need a GROUP By clause with time_bucket on the partitioning * column of the hypertable */ Assert(query->groupClause); caggtimebucket_validate(&bucket_info, query->groupClause, query->targetList, query->rtable, is_cagg_create); /* Check row security settings for the table. */ if (ts_has_row_security(rte->relid)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create continuous aggregate on hypertable with row security"))); /* At this point, we should have a valid bucket function. Otherwise, we have errored out before. */ Ensure(OidIsValid(bucket_info.bf->bucket_function), "unable to find valid bucket function"); /* Ignore time_bucket_ng in this check, since offset and origin were allowed in the past */ FuncInfo *func_info = ts_func_cache_get_bucketing_func(bucket_info.bf->bucket_function); Ensure(func_info != NULL, "bucket function is not found in function cache"); /* hierarchical cagg validations */ if (is_hierarchical) { int64 bucket_width = 0, bucket_width_parent = 0; bool is_greater_or_equal_than_parent = true, is_multiple_of_parent = true; Assert(prev_query->groupClause); caggtimebucket_validate(&bucket_info_parent, prev_query->groupClause, prev_query->targetList, prev_query->rtable, is_cagg_create); /* Cannot create cagg with fixed bucket on top of variable bucket. */ if (time_bucket_info_has_fixed_width(bucket_info_parent.bf) == false && time_bucket_info_has_fixed_width(bucket_info.bf) == true) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create continuous aggregate with fixed-width bucket on top of " "one using variable-width bucket"), errdetail("Continuous aggregate with a fixed time bucket width (e.g. 61 days) " "cannot be created on top of one using variable time bucket width " "(e.g. 1 month).\n" "The variance can lead to the fixed width one not being a multiple " "of the variable width one."))); } /* Get bucket widths for validation. */ bucket_width = get_bucket_width(bucket_info); bucket_width_parent = get_bucket_width(bucket_info_parent); Assert(bucket_width != 0); Assert(bucket_width_parent != 0); /* Check if the current bucket is greater or equal than the parent. */ is_greater_or_equal_than_parent = (bucket_width >= bucket_width_parent); /* Check if buckets are multiple. */ if (bucket_width_parent != 0) { if (bucket_width_parent > bucket_width && bucket_width != 0) is_multiple_of_parent = ((bucket_width_parent % bucket_width) == 0); else is_multiple_of_parent = ((bucket_width % bucket_width_parent) == 0); } /* Proceed with validation errors. */ if (!is_greater_or_equal_than_parent || !is_multiple_of_parent) { char *message = NULL; /* New bucket should be multiple of the parent. */ if (!is_multiple_of_parent) message = "multiple of"; /* New bucket should be greater than the parent. */ if (!is_greater_or_equal_than_parent) message = "greater or equal than"; ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create continuous aggregate with incompatible bucket width"), errdetail("Time bucket width of \"%s.%s\" [%s] should be %s the time " "bucket width of \"%s.%s\" [%s].", cagg_schema, cagg_name, ts_datum_to_string(get_bucket_width_datum(bucket_info), bucket_info.bf->bucket_width_type), message, NameStr(cagg_parent->data.user_view_schema), NameStr(cagg_parent->data.user_view_name), ts_datum_to_string(get_bucket_width_datum(bucket_info_parent), bucket_info_parent.bf->bucket_width_type)))); } /* Test compatible time origin values */ if (bucket_info.bf->bucket_time_origin != bucket_info_parent.bf->bucket_time_origin) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "cannot create continuous aggregate with different bucket origin values"), errdetail("Time origin of \"%s.%s\" [%s] and \"%s.%s\" [%s] should be the " "same.", cagg_schema, cagg_name, ts_datum_to_string(TimestampTzGetDatum( bucket_info.bf->bucket_time_origin), TIMESTAMPTZOID), NameStr(cagg_parent->data.user_view_schema), NameStr(cagg_parent->data.user_view_name), ts_datum_to_string(TimestampTzGetDatum( bucket_info_parent.bf->bucket_time_origin), TIMESTAMPTZOID)))); } /* Test compatible time offset values */ if (bucket_info.bf->bucket_time_offset != NULL || bucket_info_parent.bf->bucket_time_offset != NULL) { bool bucket_offset_isnull = bucket_info.bf->bucket_time_offset == NULL; bool bucket_offset_parent_isnull = bucket_info_parent.bf->bucket_time_offset == NULL; Datum offset_datum = IntervalPGetDatum(bucket_info.bf->bucket_time_offset); Datum offset_datum_parent = IntervalPGetDatum(bucket_info_parent.bf->bucket_time_offset); bool both_buckets_are_equal = false; bool both_buckets_have_offset = !bucket_offset_isnull && !bucket_offset_parent_isnull; if (both_buckets_have_offset) { both_buckets_are_equal = DatumGetBool( DirectFunctionCall2(interval_eq, offset_datum, offset_datum_parent)); } if (!both_buckets_are_equal) { char *offset = !bucket_offset_isnull ? DatumGetCString(DirectFunctionCall1(interval_out, offset_datum)) : "NULL"; char *offset_parent = !bucket_offset_parent_isnull ? DatumGetCString(DirectFunctionCall1(interval_out, offset_datum_parent)) : "NULL"; ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create continuous aggregate with different bucket offset " "values"), errdetail("Time origin of \"%s.%s\" [%s] and \"%s.%s\" [%s] should be the " "same.", cagg_schema, cagg_name, offset, NameStr(cagg_parent->data.user_view_schema), NameStr(cagg_parent->data.user_view_name), offset_parent))); } } /* Test compatible integer offset values */ if (bucket_info.bf->bucket_integer_offset != bucket_info_parent.bf->bucket_integer_offset) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "cannot create continuous aggregate with different bucket offset values"), errdetail("Integer offset of \"%s.%s\" [" INT64_FORMAT "] and \"%s.%s\" [" INT64_FORMAT "] should be the same.", cagg_schema, cagg_name, bucket_info.bf->bucket_integer_offset, NameStr(cagg_parent->data.user_view_schema), NameStr(cagg_parent->data.user_view_name), bucket_info_parent.bf->bucket_integer_offset))); } } if (is_hierarchical) bucket_info.htoidparent = cagg_parent->relid; return bucket_info; } /* * Get oid of function to convert from our internal representation * to postgres representation. */ Oid cagg_get_boundary_converter_funcoid(Oid typoid) { char *function_name; Oid argtyp[] = { INT8OID }; switch (typoid) { case DATEOID: function_name = INTERNAL_TO_DATE_FUNCTION; break; case TIMESTAMPOID: function_name = INTERNAL_TO_TS_FUNCTION; break; case TIMESTAMPTZOID: function_name = INTERNAL_TO_TSTZ_FUNCTION; break; default: /* * This should never be reached and unsupported datatypes * should be caught at much earlier stages. */ ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("no converter function defined for datatype: %s", format_type_be(typoid)))); pg_unreachable(); } List *func_name = list_make2(makeString(FUNCTIONS_SCHEMA_NAME), makeString(function_name)); Oid converter_oid = LookupFuncName(func_name, lengthof(argtyp), argtyp, false); Assert(OidIsValid(converter_oid)); return converter_oid; } static FuncExpr * build_conversion_call(Oid type, FuncExpr *boundary) { /* * If the partitioning column type is not integer we need to convert * to proper representation. */ switch (type) { case INT2OID: case INT4OID: { /* Since the boundary function returns int8 we need to cast to proper type here. */ Oid cast_oid = ts_get_cast_func(INT8OID, type); return makeFuncExpr(cast_oid, type, list_make1(boundary), InvalidOid, InvalidOid, COERCE_IMPLICIT_CAST); } case INT8OID: /* Nothing to do for int8. */ return boundary; case DATEOID: case TIMESTAMPOID: case TIMESTAMPTZOID: { /* * date/timestamp/timestamptz need to be converted since * we store them differently from postgres format. */ Oid converter_oid = cagg_get_boundary_converter_funcoid(type); return makeFuncExpr(converter_oid, type, list_make1(boundary), InvalidOid, InvalidOid, COERCE_EXPLICIT_CALL); } case UUIDOID: { /* * UUID needs two-step conversion: first convert the internal int8 * representation to timestamptz, then convert timestamptz to a * boundary UUID via to_uuidv7_boundary(). */ Oid tstz_converter_oid = cagg_get_boundary_converter_funcoid(TIMESTAMPTZOID); FuncExpr *tstz_boundary = makeFuncExpr(tstz_converter_oid, TIMESTAMPTZOID, list_make1(boundary), InvalidOid, InvalidOid, COERCE_EXPLICIT_CALL); Oid uuid_argtyp[] = { TIMESTAMPTZOID }; List *uuid_func_name = list_make2(makeString(ts_extension_schema_name()), makeString("to_uuidv7_boundary")); Oid uuid_converter_oid = LookupFuncName(uuid_func_name, lengthof(uuid_argtyp), uuid_argtyp, false); return makeFuncExpr(uuid_converter_oid, UUIDOID, list_make1(tstz_boundary), InvalidOid, InvalidOid, COERCE_EXPLICIT_CALL); } default: /* * All valid types should be handled above, this should * never be reached and error handling at earlier stages * should catch this. */ ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("unsupported datatype for continuous aggregates: %s", format_type_be(type)))); pg_unreachable(); } } /* * Return the Oid of the cagg_watermark function */ Oid get_watermark_function_oid(void) { Oid argtyp[] = { INT4OID }; Oid boundary_func_oid = LookupFuncName(list_make2(makeString(FUNCTIONS_SCHEMA_NAME), makeString(BOUNDARY_FUNCTION)), lengthof(argtyp), argtyp, false); return boundary_func_oid; } /* * Build function call that returns boundary for a hypertable * wrapped in type conversion calls when required. */ static FuncExpr * build_boundary_call(int32 ht_id, Oid type) { FuncExpr *boundary; Oid boundary_func_oid = get_watermark_function_oid(); List *func_args = list_make1(makeConst(INT4OID, -1, InvalidOid, 4, Int32GetDatum(ht_id), false, true)); boundary = makeFuncExpr(boundary_func_oid, INT8OID, func_args, InvalidOid, InvalidOid, COERCE_EXPLICIT_CALL); return build_conversion_call(type, boundary); } /* * Create Const of proper type for lower bound of watermark when * watermark has not been set yet. */ static Const * cagg_boundary_make_lower_bound(Oid type) { Datum value; int16 typlen; bool typbyval; if (type == UUIDOID) { /* * For UUID, create an all-zeros UUID as the lower bound. This is * smaller than any valid UUIDv7 and serves as the "beginning of time" * fallback when the watermark is NULL. */ pg_uuid_t *uuid = (pg_uuid_t *) palloc0(sizeof(pg_uuid_t)); return makeConst(UUIDOID, -1, InvalidOid, UUID_LEN, UUIDPGetDatum(uuid), false, false); } get_typlenbyval(type, &typlen, &typbyval); value = ts_time_datum_get_nobegin_or_min(type); return makeConst(type, -1, InvalidOid, typlen, value, false, typbyval); } static Node * build_union_query_quals(int32 ht_id, Oid partcoltype, Oid opno, int varno, AttrNumber attno) { Var *var = makeVar(varno, attno, partcoltype, -1, InvalidOid, InvalidOid); FuncExpr *boundary = build_boundary_call(ht_id, partcoltype); CoalesceExpr *coalesce = makeNode(CoalesceExpr); coalesce->coalescetype = partcoltype; coalesce->coalescecollid = InvalidOid; coalesce->args = list_make2(boundary, cagg_boundary_make_lower_bound(partcoltype)); return (Node *) make_opclause(opno, BOOLOID, false, (Expr *) var, (Expr *) coalesce, InvalidOid, InvalidOid); } static RangeTblEntry * makeRangeTblEntry(Query *query, const char *aliasname) { RangeTblEntry *rte = makeNode(RangeTblEntry); ListCell *lc; rte->rtekind = RTE_SUBQUERY; rte->relid = InvalidOid; rte->subquery = query; rte->alias = makeAlias(aliasname, NIL); rte->eref = copyObject(rte->alias); foreach (lc, query->targetList) { TargetEntry *tle = lfirst_node(TargetEntry, lc); if (!tle->resjunk) rte->eref->colnames = lappend(rte->eref->colnames, makeString(pstrdup(tle->resname))); } rte->lateral = false; rte->inh = false; /* never true for subqueries */ rte->inFromCl = false; return rte; } /* * Build union query combining the materialized data with data from the raw data hypertable. * * q1 is the query on the materialization hypertable with the finalize call * q2 is the query on the raw hypertable which was supplied in the initial CREATE VIEW statement * returns a query as * SELECT * from ( SELECT * from q1 where <coale_qual> * UNION ALL * SELECT * from q2 where existing_qual and <coale_qual> * where coale_qual is: time < ----> (or >= ) * * COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(<htid>)), * '-infinity'::timestamp with time zone) * * See build_union_query_quals for COALESCE clauses. */ Query * build_union_query(ContinuousAggTimeBucketInfo *tbinfo, int matpartcolno, Query *q1, Query *q2, int materialize_htid) { ListCell *lc1, *lc2; List *col_types = NIL; List *col_typmods = NIL; List *col_collations = NIL; List *tlist = NIL; List *sortClause = NIL; int varno; Node *q2_quals = NULL; Assert(list_length(q1->targetList) <= list_length(q2->targetList)); q1 = copyObject(q1); q2 = copyObject(q2); if (q1->sortClause) sortClause = copyObject(q1->sortClause); /* * For UUID-partitioned hypertables, the materialization table's partition * column is TIMESTAMPTZ (the output of time_bucket on UUID), while the raw * table's partition column is UUID. We need different types and operators * for q1 (materialized data) and q2 (raw data). */ Oid q1_partcoltype = tbinfo->htpartcoltype; Oid q2_partcoltype = tbinfo->htpartcoltype; if (tbinfo->htpartcoltype == UUIDOID) q1_partcoltype = TIMESTAMPTZOID; TypeCacheEntry *tce_q1 = lookup_type_cache(q1_partcoltype, TYPECACHE_LT_OPR); TypeCacheEntry *tce_q2 = lookup_type_cache(q2_partcoltype, TYPECACHE_LT_OPR); varno = list_length(q1->rtable); q1->jointree->quals = build_union_query_quals(materialize_htid, q1_partcoltype, tce_q1->lt_opr, varno, matpartcolno); /* * If there is join in CAgg definition then adjust varno * to get time column from the hypertable in the join. */ varno = list_length(q2->rtable); if (list_length(q2->rtable) > 1) { int nvarno = 1; foreach (lc2, q2->rtable) { RangeTblEntry *rte = lfirst_node(RangeTblEntry, lc2); if (rte->rtekind == RTE_RELATION) { /* look for hypertable or parent hypertable in RangeTableEntry list */ if (rte->relid == tbinfo->htoid || rte->relid == tbinfo->htoidparent) { varno = nvarno; break; } } nvarno++; } } q2_quals = build_union_query_quals(materialize_htid, q2_partcoltype, get_negator(tce_q2->lt_opr), varno, tbinfo->htpartcolno); q2->jointree->quals = make_and_qual(q2->jointree->quals, q2_quals); Query *query = makeNode(Query); SetOperationStmt *setop = makeNode(SetOperationStmt); RangeTblEntry *rte_q1 = makeRangeTblEntry(q1, "*SELECT* 1"); RangeTblEntry *rte_q2 = makeRangeTblEntry(q2, "*SELECT* 2"); RangeTblRef *ref_q1 = makeNode(RangeTblRef); RangeTblRef *ref_q2 = makeNode(RangeTblRef); query->commandType = CMD_SELECT; query->rtable = list_make2(rte_q1, rte_q2); query->setOperations = (Node *) setop; setop->op = SETOP_UNION; setop->all = true; ref_q1->rtindex = 1; ref_q2->rtindex = 2; setop->larg = (Node *) ref_q1; setop->rarg = (Node *) ref_q2; forboth (lc1, q1->targetList, lc2, q2->targetList) { TargetEntry *tle = lfirst_node(TargetEntry, lc1); TargetEntry *tle2 = lfirst_node(TargetEntry, lc2); TargetEntry *tle_union; Var *expr; if (!tle->resjunk) { col_types = lappend_int(col_types, exprType((Node *) tle->expr)); col_typmods = lappend_int(col_typmods, exprTypmod((Node *) tle->expr)); col_collations = lappend_int(col_collations, exprCollation((Node *) tle->expr)); expr = makeVarFromTargetEntry(1, tle); /* * We need to use resname from q2 because that is the query from the * initial CREATE VIEW statement so the VIEW can be updated in place. */ tle_union = makeTargetEntry((Expr *) copyObject(expr), list_length(tlist) + 1, tle2->resname, false); tle_union->resorigtbl = expr->varno; tle_union->resorigcol = expr->varattno; tle_union->ressortgroupref = tle->ressortgroupref; tlist = lappend(tlist, tle_union); } } query->targetList = tlist; query->jointree = makeFromExpr(NIL, NULL); if (sortClause) { query->sortClause = sortClause; } setop->colTypes = col_types; setop->colTypmods = col_typmods; setop->colCollations = col_collations; return query; } /* * Returns true if the time bucket size is fixed */ static bool time_bucket_info_has_fixed_width(const ContinuousAggBucketFunction *bf) { if (!IS_TIME_BUCKET_INFO_TIME_BASED(bf)) { return true; } else { /* Historically, we treat all buckets with timezones as variable. Buckets with only days are * treated as fixed. */ return bf->bucket_time_width->month == 0 && bf->bucket_time_timezone == NULL; } } ContinuousAgg * cagg_get_by_relid_or_fail(const Oid cagg_relid) { ContinuousAgg *cagg; if (!OidIsValid(cagg_relid)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid continuous aggregate"))); cagg = ts_continuous_agg_find_by_relid(cagg_relid); if (NULL == cagg) { const char *relname = get_rel_name(cagg_relid); if (relname == NULL) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), (errmsg("continuous aggregate does not exist")))); else ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), (errmsg("relation \"%s\" is not a continuous aggregate", relname)))); } return cagg; } /* Get time bucket function info based on the view definition */ ContinuousAggBucketFunction * ts_cagg_get_bucket_function_info(Oid view_oid) { Relation view_rel = relation_open(view_oid, AccessShareLock); Query *query = copyObject(get_view_query(view_rel)); relation_close(view_rel, NoLock); Assert(query != NULL); Assert(query->commandType == CMD_SELECT); ContinuousAggBucketFunction *bf = palloc0(sizeof(ContinuousAggBucketFunction)); ListCell *l; foreach (l, query->groupClause) { SortGroupClause *sgc = lfirst_node(SortGroupClause, l); TargetEntry *tle = get_sortgroupclause_tle(sgc, query->targetList); Expr *expr = tle->expr; #if PG18_GE /* PG18 introduced RTEs for group clauses so * we can use rtable to look up GROUP BY expressions. * * https://github.com/postgres/postgres/commit/247dea89 */ if (IsA(expr, Var)) { Var *var = castNode(Var, tle->expr); Assert((int) var->varno <= list_length(query->rtable)); RangeTblEntry *rte = list_nth(query->rtable, var->varno - 1); Assert(rte->rtekind == RTE_GROUP); Assert(var->varattno > 0); Expr *node = list_nth(rte->groupexprs, var->varattno - 1); if (IsA(node, FuncExpr)) expr = node; } #endif if (IsA(expr, FuncExpr)) { FuncExpr *fe = castNode(FuncExpr, expr); /* Filter any non bucketing functions */ FuncInfo *finfo = ts_func_cache_get_bucketing_func(fe->funcid); if (finfo == NULL) continue; Assert(finfo->is_bucketing_func); process_timebucket_parameters(fe, bf, false, false, InvalidAttrNumber); break; } } return bf; } /* * This function is responsible to return a list of column names used in * GROUP BY clause of the cagg query. It behaves a bit different depending * of the type of the Continuous Aggregate. * * Retrieve the "direct view query" and find the GROUP BY clause and * "time_bucket" clause. We use the "direct view query" because in the * "user view query" we removed the re-aggregation in the part that query * the materialization hypertable so we don't have a GROUP BY clause * anymore. * * Get the column name from the GROUP BY clause because all the column * names are the same in all underlying objects (user view, direct view, * partial view and materialization hypertable). */ List * cagg_find_groupingcols(ContinuousAgg *agg, Hypertable *mat_ht) { List *retlist = NIL; ListCell *lc; Query *cagg_view_query = ts_continuous_agg_get_query(agg); Oid mat_relid = mat_ht->main_table_relid; Query *finalize_query; #if PG16_LT /* The view rule has dummy old and new range table entries as the 1st and 2nd entries */ Assert(list_length(cagg_view_query->rtable) >= 2); #endif if (cagg_view_query->setOperations) { /* * This corresponds to the union view. * PG16_LT the 3rd RTE entry has the SELECT 1 query from the union view. * PG16_GE the 1st RTE entry has the SELECT 1 query from the union view */ #if PG16_LT RangeTblEntry *finalize_query_rte = lthird(cagg_view_query->rtable); #else RangeTblEntry *finalize_query_rte = linitial(cagg_view_query->rtable); #endif if (finalize_query_rte->rtekind != RTE_SUBQUERY) ereport(ERROR, (errcode(ERRCODE_TS_UNEXPECTED), errmsg("unexpected rte type for view %d", finalize_query_rte->rtekind))); finalize_query = finalize_query_rte->subquery; } else { finalize_query = cagg_view_query; } foreach (lc, finalize_query->groupClause) { SortGroupClause *cagg_gc = (SortGroupClause *) lfirst(lc); TargetEntry *cagg_tle = get_sortgroupclause_tle(cagg_gc, finalize_query->targetList); /* "resname" is the same as "mat column names" */ if (!cagg_tle->resjunk && cagg_tle->resname) retlist = lappend(retlist, get_attname(mat_relid, cagg_tle->resno, false)); } return retlist; } ================================================ FILE: tsl/src/continuous_aggs/common.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <access/reloptions.h> #include <access/xact.h> #include <catalog/pg_aggregate.h> #include <catalog/pg_type.h> #include <catalog/toasting.h> #include <commands/tablecmds.h> #include <commands/tablespace.h> #include <miscadmin.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <nodes/parsenodes.h> #include <nodes/pg_list.h> #include <optimizer/optimizer.h> #include <parser/parse_func.h> #include <parser/parse_oper.h> #include <parser/parsetree.h> #include <rewrite/rewriteHandler.h> #include <rewrite/rewriteManip.h> #include <utils/builtins.h> #include <utils/syscache.h> #include <utils/typcache.h> #include "errors.h" #include "func_cache.h" #include "hypertable_cache.h" #include "timezones.h" #include "ts_catalog/catalog.h" #include "ts_catalog/continuous_agg.h" #define DEFAULT_MATPARTCOLUMN_NAME "time_partition_col" #define CAGG_INVALIDATION_THRESHOLD_NAME "invalidation threshold watermark" #define CAGG_INVALIDATION_WRONG_GREATEST_VALUE ((int64) -210866803200000001) typedef struct FinalizeQueryInfo { List *final_seltlist; /* select target list for finalize query */ Node *final_havingqual; /* having qual for finalize query */ Query *final_userquery; /* user query used to compute the finalize_query */ } FinalizeQueryInfo; typedef struct MaterializationHypertableColumnInfo { List *matcollist; /* column defns for materialization tbl*/ List *partial_seltlist; /* tlist entries for populating the materialization table columns */ List *partial_grouplist; /* group clauses used for populating the materialization table */ List *mat_groupcolname_list; /* names of columns that are populated by the group-by clause correspond to the partial_grouplist. time_bucket column is not included here: it is the matpartcolname */ int matpartcolno; /*index of partitioning column in matcollist */ char *matpartcolname; /*name of the partition column */ } MaterializationHypertableColumnInfo; typedef struct ContinuousAggTimeBucketInfo { int32 htid; /* hypertable id */ int32 parent_mat_hypertable_id; /* parent materialization hypertable id */ Oid htoid; /* hypertable oid */ Oid htoidparent; /* parent hypertable oid in case of hierarchical */ AttrNumber htpartcolno; /* primary partitioning column of raw hypertable */ /* This should also be the column used by time_bucket */ Oid htpartcoltype; /* The collation type */ int64 htpartcol_interval_len; /* interval length setting for primary partitioning column */ /* General bucket information */ ContinuousAggBucketFunction *bf; } ContinuousAggTimeBucketInfo; typedef enum ContinuousAggRefreshCallContext { CAGG_REFRESH_CREATION, CAGG_REFRESH_WINDOW, CAGG_REFRESH_POLICY, CAGG_REFRESH_POLICY_BATCHED } ContinuousAggRefreshCallContext; typedef struct ContinuousAggRefreshContext { ContinuousAggRefreshCallContext callctx; int32 processing_batch; int32 number_of_batches; } ContinuousAggRefreshContext; #define IS_TIME_BUCKET_INFO_TIME_BASED(bucket_function) \ (bucket_function->bucket_width_type == INTERVALOID) #define CAGG_MAKEQUERY(selquery, srcquery) \ do \ { \ (selquery) = makeNode(Query); \ (selquery)->commandType = CMD_SELECT; \ (selquery)->querySource = (srcquery)->querySource; \ (selquery)->queryId = (srcquery)->queryId; \ (selquery)->canSetTag = (srcquery)->canSetTag; \ (selquery)->utilityStmt = copyObject((srcquery)->utilityStmt); \ (selquery)->resultRelation = 0; \ (selquery)->hasAggs = true; \ (selquery)->hasRowSecurity = false; \ (selquery)->rtable = NULL; \ } while (0); extern ContinuousAggTimeBucketInfo cagg_validate_query(const Query *query, const char *cagg_schema, const char *cagg_name, const bool is_cagg_create); extern Query *destroy_union_query(Query *q); extern void RemoveRangeTableEntries(Query *query); extern Query *build_union_query(ContinuousAggTimeBucketInfo *tbinfo, int matpartcolno, Query *q1, Query *q2, int materialize_htid); extern bool function_allowed_in_cagg_definition(Oid funcid); extern Oid get_watermark_function_oid(void); extern Oid cagg_get_boundary_converter_funcoid(Oid typoid); extern ContinuousAgg *cagg_get_by_relid_or_fail(const Oid cagg_relid); extern List *cagg_find_groupingcols(ContinuousAgg *agg, Hypertable *mat_ht); static inline int64 cagg_get_time_min(const ContinuousAgg *cagg) { if (cagg->bucket_function->bucket_fixed_interval == false) { /* * To determine inscribed/circumscribed refresh window for variable-sized * buckets we should be able to calculate time_bucket(window.begin) and * time_bucket(window.end). This, however, is not possible in general case. * As an example, the minimum date is 4714-11-24 BC, which is before any * reasonable default `origin` value. Thus for variable-sized buckets * instead of minimum date we use -infinity since time_bucket(-infinity) * is well-defined as -infinity. * * For more details see: * - ts_compute_inscribed_bucketed_refresh_window_variable() * - ts_compute_circumscribed_bucketed_refresh_window_variable() */ return ts_time_get_nobegin_or_min(cagg->partition_type); } /* For fixed-sized buckets return min (start of time) */ return ts_time_get_min(cagg->partition_type); } ContinuousAggBucketFunction *ts_cagg_get_bucket_function_info(Oid view_oid); ================================================ FILE: tsl/src/continuous_aggs/create.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * This file contains the code for processing continuous aggregate * DDL statements which are of the form: * * CREATE MATERIALIZED VIEW <name> WITH (ts_continuous = [option] ) * AS <select query> * The entry point for the code is * tsl_process_continuous_agg_viewstmt * The bulk of the code that creates the underlying tables/views etc. is in * cagg_create. */ #include <postgres.h> #include <access/reloptions.h> #include <access/sysattr.h> #include <access/xact.h> #include <access/xlogutils.h> #include <catalog/index.h> #include <catalog/indexing.h> #include <catalog/pg_namespace.h> #include <catalog/pg_type.h> #include <catalog/toasting.h> #include <commands/defrem.h> #include <commands/tablecmds.h> #include <commands/tablespace.h> #include <commands/view.h> #include <executor/spi.h> #include <fmgr.h> #include <miscadmin.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <nodes/nodes.h> #include <nodes/parsenodes.h> #include <nodes/pg_list.h> #include <optimizer/clauses.h> #include <optimizer/optimizer.h> #include <optimizer/prep.h> #include <optimizer/tlist.h> #include <parser/analyze.h> #include <parser/parse_func.h> #include <parser/parse_oper.h> #include <parser/parse_relation.h> #include <parser/parse_type.h> #include <parser/parsetree.h> #include <replication/logical.h> #include <replication/slot.h> #include <storage/lwlocknames.h> #include <utils/acl.h> #include <utils/builtins.h> #include <utils/catcache.h> #include <utils/elog.h> #include <utils/pg_lsn.h> #include <utils/rel.h> #include <utils/resowner.h> #include <utils/ruleutils.h> #include <utils/snapshot.h> #include <utils/syscache.h> #include <utils/typcache.h> #include "common.h" #include "config.h" #include "create.h" #include "finalize.h" #include "invalidation_threshold.h" #include "debug_assert.h" #include "dimension.h" #include "extension_constants.h" #include "guc.h" #include "hypertable.h" #include "hypertable_cache.h" #include "invalidation.h" #include "refresh.h" #include "time_utils.h" #include "ts_catalog/catalog.h" #include "ts_catalog/continuous_agg.h" #include "ts_catalog/continuous_aggs_watermark.h" #include "with_clause/create_materialized_view_with_clause.h" static void create_cagg_catalog_entry(int32 matht_id, int32 rawht_id, const char *user_schema, const char *user_view, const char *partial_schema, const char *partial_view, bool materialized_only, const char *direct_schema, const char *direct_view, const int32 parent_mat_hypertable_id); static void create_bucket_function_catalog_entry(int32 matht_id, Oid bucket_function, const char *bucket_width, const char *origin, const char *offset, const char *timezone, const bool bucket_fixed_width); static void cagg_create_hypertable(int32 hypertable_id, Oid mat_tbloid, const char *matpartcolname, int64 mat_tbltimecol_interval); static void mattablecolumninfo_add_mattable_index(MaterializationHypertableColumnInfo *matcolinfo, Hypertable *ht); static ObjectAddress create_view_for_query(Query *selquery, RangeVar *viewrel); static void fixup_userview_query_tlist(Query *userquery, List *tlist_aliases); static void cagg_create(const CreateTableAsStmt *create_stmt, ViewStmt *stmt, Query *panquery, ContinuousAggTimeBucketInfo *bucket_info, WithClauseResult *with_clause_options); #define MATPARTCOL_INTERVAL_FACTOR 10 static void makeMaterializedTableName(char *buf, const char *prefix, int hypertable_id) { int ret = snprintf(buf, NAMEDATALEN, prefix, hypertable_id); if (ret < 0 || ret > NAMEDATALEN) { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("bad materialization internal name"))); } } /* * Create a entry for the materialization table in table CONTINUOUS_AGGS. */ static void create_cagg_catalog_entry(int32 matht_id, int32 rawht_id, const char *user_schema, const char *user_view, const char *partial_schema, const char *partial_view, bool materialized_only, const char *direct_schema, const char *direct_view, const int32 parent_mat_hypertable_id) { Catalog *catalog = ts_catalog_get(); Relation rel; TupleDesc desc; NameData user_schnm, user_viewnm, partial_schnm, partial_viewnm, direct_schnm, direct_viewnm; Datum values[Natts_continuous_agg]; bool nulls[Natts_continuous_agg] = { false }; CatalogSecurityContext sec_ctx; namestrcpy(&user_schnm, user_schema); namestrcpy(&user_viewnm, user_view); namestrcpy(&partial_schnm, partial_schema); namestrcpy(&partial_viewnm, partial_view); namestrcpy(&direct_schnm, direct_schema); namestrcpy(&direct_viewnm, direct_view); rel = table_open(catalog_get_table_id(catalog, CONTINUOUS_AGG), RowExclusiveLock); desc = RelationGetDescr(rel); memset(values, 0, sizeof(values)); values[AttrNumberGetAttrOffset(Anum_continuous_agg_mat_hypertable_id)] = matht_id; values[AttrNumberGetAttrOffset(Anum_continuous_agg_raw_hypertable_id)] = rawht_id; if (parent_mat_hypertable_id == INVALID_HYPERTABLE_ID) nulls[AttrNumberGetAttrOffset(Anum_continuous_agg_parent_mat_hypertable_id)] = true; else { values[AttrNumberGetAttrOffset(Anum_continuous_agg_parent_mat_hypertable_id)] = parent_mat_hypertable_id; } values[AttrNumberGetAttrOffset(Anum_continuous_agg_user_view_schema)] = NameGetDatum(&user_schnm); values[AttrNumberGetAttrOffset(Anum_continuous_agg_user_view_name)] = NameGetDatum(&user_viewnm); values[AttrNumberGetAttrOffset(Anum_continuous_agg_partial_view_schema)] = NameGetDatum(&partial_schnm); values[AttrNumberGetAttrOffset(Anum_continuous_agg_partial_view_name)] = NameGetDatum(&partial_viewnm); values[AttrNumberGetAttrOffset(Anum_continuous_agg_direct_view_schema)] = NameGetDatum(&direct_schnm); values[AttrNumberGetAttrOffset(Anum_continuous_agg_direct_view_name)] = NameGetDatum(&direct_viewnm); values[AttrNumberGetAttrOffset(Anum_continuous_agg_materialize_only)] = BoolGetDatum(materialized_only); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_insert_values(rel, desc, values, nulls); ts_catalog_restore_user(&sec_ctx); table_close(rel, RowExclusiveLock); } /* * Create a entry for the materialization table in table * CONTINUOUS_AGGS_BUCKET_FUNCTION. */ static void create_bucket_function_catalog_entry(int32 matht_id, Oid bucket_function, const char *bucket_width, const char *bucket_origin, const char *bucket_offset, const char *bucket_timezone, const bool bucket_fixed_width) { Catalog *catalog = ts_catalog_get(); Relation rel; TupleDesc desc; Datum values[Natts_continuous_aggs_bucket_function]; bool nulls[Natts_continuous_aggs_bucket_function] = { false }; CatalogSecurityContext sec_ctx; Assert(OidIsValid(bucket_function)); Assert(bucket_width != NULL); rel = table_open(catalog_get_table_id(catalog, CONTINUOUS_AGGS_BUCKET_FUNCTION), RowExclusiveLock); desc = RelationGetDescr(rel); memset(values, 0, sizeof(values)); /* Hypertable ID */ values[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_mat_hypertable_id)] = matht_id; /* Bucket function */ values[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_function)] = CStringGetTextDatum(format_procedure_qualified(bucket_function)); /* Bucket width */ values[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_width)] = CStringGetTextDatum(bucket_width); /* Bucket origin */ if (bucket_origin != NULL) { values[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_origin)] = CStringGetTextDatum(bucket_origin); } else { nulls[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_origin)] = true; } /* Bucket offset */ if (bucket_offset != NULL) { values[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_offset)] = CStringGetTextDatum(bucket_offset); } else { nulls[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_offset)] = true; } /* Bucket timezone */ if (bucket_timezone != NULL) { values[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_timezone)] = CStringGetTextDatum(bucket_timezone); } else { nulls[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_timezone)] = true; } /* Bucket fixed width */ values[AttrNumberGetAttrOffset(Anum_continuous_aggs_bucket_function_bucket_fixed_width)] = BoolGetDatum(bucket_fixed_width); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_insert_values(rel, desc, values, nulls); ts_catalog_restore_user(&sec_ctx); table_close(rel, RowExclusiveLock); } /* * Create hypertable for the table referred by mat_tbloid * matpartcolname - partition column for hypertable * timecol_interval - is the partitioning column's interval for hypertable partition */ static void cagg_create_hypertable(int32 hypertable_id, Oid mat_tbloid, const char *matpartcolname, int64 mat_tbltimecol_interval) { bool created; int flags = 0; NameData mat_tbltimecol; DimensionInfo *time_dim_info; ChunkSizingInfo *chunk_sizing_info; namestrcpy(&mat_tbltimecol, matpartcolname); time_dim_info = ts_dimension_info_create_open(mat_tbloid, &mat_tbltimecol, Int64GetDatum(mat_tbltimecol_interval), INT8OID, InvalidOid); /* * Ideally would like to change/expand the API so setting the column name manually is * unnecessary, but not high priority. */ chunk_sizing_info = ts_chunk_sizing_info_get_default_disabled(mat_tbloid); chunk_sizing_info->colname = matpartcolname; created = ts_hypertable_create_from_info(mat_tbloid, hypertable_id, flags, time_dim_info, NULL, NULL, NULL, chunk_sizing_info); if (!created) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not create materialization hypertable"))); } /* * Add additional indexes to materialization table for the columns derived from * the group-by column list of the partial select query. * If partial select query has: * GROUP BY timebucket_expr, <grpcol1, grpcol2, grpcol3 ...> * index on mattable is <grpcol1, timebucketcol>, <grpcol2, timebucketcol> ... and so on. * i.e. #indexes =( #grp-cols - 1) */ static void mattablecolumninfo_add_mattable_index(MaterializationHypertableColumnInfo *matcolinfo, Hypertable *ht) { IndexStmt stmt = { .type = T_IndexStmt, .accessMethod = DEFAULT_INDEX_TYPE, .idxname = NULL, .relation = makeRangeVar(NameStr(ht->fd.schema_name), NameStr(ht->fd.table_name), 0), .tableSpace = get_tablespace_name(get_rel_tablespace(ht->main_table_relid)), }; IndexElem timeelem = { .type = T_IndexElem, .name = matcolinfo->matpartcolname, .ordering = SORTBY_DESC }; ListCell *le = NULL; foreach (le, matcolinfo->mat_groupcolname_list) { NameData indxname; ObjectAddress indxaddr; HeapTuple indxtuple; char *grpcolname = (char *) lfirst(le); IndexElem grpelem = { .type = T_IndexElem, .name = grpcolname }; stmt.indexParams = list_make2(&grpelem, &timeelem); indxaddr = DefineIndexCompat(ht->main_table_relid, &stmt, InvalidOid, /* indexRelationId */ InvalidOid, /* parentIndexId */ InvalidOid, /* parentConstraintId */ -1, /* total_parts */ false, /* is_alter_table */ false, /* check_rights */ false, /* check_not_in_use */ false, /* skip_build */ false); /* quiet */ indxtuple = SearchSysCache1(RELOID, ObjectIdGetDatum(indxaddr.objectId)); if (!HeapTupleIsValid(indxtuple)) elog(ERROR, "cache lookup failed for index relid %u", indxaddr.objectId); indxname = ((Form_pg_class) GETSTRUCT(indxtuple))->relname; elog(DEBUG1, "adding index %s ON %s.%s USING BTREE(%s, %s)", NameStr(indxname), NameStr(ht->fd.schema_name), NameStr(ht->fd.table_name), grpcolname, matcolinfo->matpartcolname); ReleaseSysCache(indxtuple); } } /* * Create the materialization hypertable root by faking up a * CREATE TABLE parsetree and passing it to DefineRelation. * Reuse the information from ViewStmt: * Remove the options on the into clause that we will not honour * Modify the relname to ts_internal_<name> * * Parameters: * mat_rel: relation information for the materialization table * bucket_info: bucket information used for setting up the * hypertable partitioning (`chunk_interval_size`). * tablespace_name: Name of the tablespace for the materialization table. * mataddress: return the ObjectAddress * * RETURNS: hypertable id of the materialization table */ static int32 create_materialization_table(MaterializationHypertableColumnInfo *matcolinfo, int32 hypertable_id, RangeVar *mat_rel, ContinuousAggTimeBucketInfo *bucket_info, bool create_addl_index, IntoClause *into, int64 matpartcol_interval, ObjectAddress *mataddress) { Oid uid, saved_uid; int sec_ctx; char *matpartcolname = matcolinfo->matpartcolname; CreateStmt *create; Datum toast_options; #if PG18_LT char *validnsps[] = HEAP_RELOPT_NAMESPACES; #else const char *const validnsps[] = HEAP_RELOPT_NAMESPACES; #endif int32 mat_htid; Oid mat_relid; Cache *hcache; Hypertable *mat_ht = NULL, *orig_ht = NULL; Oid owner = GetUserId(); create = makeNode(CreateStmt); create->relation = mat_rel; create->tableElts = matcolinfo->matcollist; create->inhRelations = NIL; create->ofTypename = NULL; create->constraints = NIL; create->options = NULL; create->oncommit = ONCOMMIT_NOOP; create->tablespacename = into->tableSpaceName; create->accessMethod = into->accessMethod; create->if_not_exists = false; /* Create the materialization table. */ SWITCH_TO_TS_USER(mat_rel->schemaname, uid, saved_uid, sec_ctx); *mataddress = DefineRelation(create, RELKIND_RELATION, owner, NULL, NULL); CommandCounterIncrement(); mat_relid = mataddress->objectId; /* NewRelationCreateToastTable calls CommandCounterIncrement. */ toast_options = transformRelOptions(UnassignedDatum, create->options, "toast", validnsps, true, false); (void) heap_reloptions(RELKIND_TOASTVALUE, toast_options, true); NewRelationCreateToastTable(mat_relid, toast_options); RESTORE_USER(uid, saved_uid, sec_ctx); cagg_create_hypertable(hypertable_id, mat_relid, matpartcolname, matpartcol_interval); /* Retrieve the hypertable id from the cache. */ mat_ht = ts_hypertable_cache_get_cache_and_entry(mat_relid, CACHE_FLAG_NONE, &hcache); mat_htid = mat_ht->fd.id; /* Create additional index on the group-by columns for the materialization table. */ if (create_addl_index) mattablecolumninfo_add_mattable_index(matcolinfo, mat_ht); /* * Initialize the invalidation log for the cagg. Initially, everything is * invalid. Add an infinite invalidation for the continuous * aggregate. This is the initial state of the aggregate before any * refreshes. */ orig_ht = ts_hypertable_cache_get_entry(hcache, bucket_info->htoid, CACHE_FLAG_NONE); continuous_agg_invalidate_mat_ht(orig_ht, mat_ht, TS_TIME_NOBEGIN, TS_TIME_NOEND); ts_cache_release(&hcache); return mat_htid; } /* * Use the userview query to create the partial query to populate * the materialization columns and remove HAVING clause and ORDER BY. */ static Query * get_partial_select_query(MaterializationHypertableColumnInfo *mattblinfo, Query *userview_query) { Query *partial_selquery = NULL; partial_selquery = copyObject(userview_query); /* Partial view should always include the time dimension column */ partial_selquery->targetList = mattblinfo->partial_seltlist; partial_selquery->groupClause = mattblinfo->partial_grouplist; return partial_selquery; } /* * Create a view for the query using the SELECt stmt sqlquery * and view name from RangeVar viewrel. */ static ObjectAddress create_view_for_query(Query *selquery, RangeVar *viewrel) { Oid uid, saved_uid; int sec_ctx; ObjectAddress address; CreateStmt *create; List *selcollist = NIL; Oid owner = GetUserId(); ListCell *lc; foreach (lc, selquery->targetList) { TargetEntry *tle = (TargetEntry *) lfirst(lc); if (!tle->resjunk) { ColumnDef *col = makeColumnDef(tle->resname, exprType((Node *) tle->expr), exprTypmod((Node *) tle->expr), exprCollation((Node *) tle->expr)); selcollist = lappend(selcollist, col); } } create = makeNode(CreateStmt); create->relation = viewrel; create->tableElts = selcollist; create->inhRelations = NIL; create->ofTypename = NULL; create->constraints = NIL; create->options = NULL; create->oncommit = ONCOMMIT_NOOP; create->tablespacename = NULL; create->if_not_exists = false; /* * Create the view. Viewname is in viewrel. */ SWITCH_TO_TS_USER(viewrel->schemaname, uid, saved_uid, sec_ctx); address = DefineRelation(create, RELKIND_VIEW, owner, NULL, NULL); CommandCounterIncrement(); StoreViewQuery(address.objectId, selquery, false); CommandCounterIncrement(); RESTORE_USER(uid, saved_uid, sec_ctx); return address; } /* * Assign aliases to the targetlist in the query according to the * column_names provided in the CREATE VIEW statement. */ static void fixup_userview_query_tlist(Query *userquery, List *tlist_aliases) { if (tlist_aliases != NIL) { ListCell *lc; ListCell *alist_item = list_head(tlist_aliases); foreach (lc, userquery->targetList) { TargetEntry *tle = (TargetEntry *) lfirst(lc); /* Junk columns don't get aliases. */ if (tle->resjunk) continue; tle->resname = pstrdup(strVal(lfirst(alist_item))); alist_item = lnext(tlist_aliases, alist_item); if (alist_item == NULL) break; /* done assigning aliases */ } if (alist_item != NULL) ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("too many column names specified"))); } } /* * Modifies the passed in ViewStmt to do the following * a) Create a hypertable for the continuous agg materialization. * b) create a view that references the underlying * materialization table instead of the original table used in * the CREATE VIEW stmt. * Example: * CREATE VIEW mcagg ... * AS select a, min(b)+max(d) from foo group by a,timebucket(a); * * Step 1. create a materialiation table which stores the partials for the * aggregates and the grouping columns + internal columns. * So we have a table like _materialization_hypertable * with columns: *( a, col1, col2, col3, internal-columns) * where col1 = partialize(min(b)), col2= partialize(max(d)), * col3= timebucket(a)) * * Step 2: Create a view with modified select query * CREATE VIEW mcagg * as * select a, finalize( col1) + finalize(col2)) * from _materialization_hypertable * group by a, col3 * * Step 3: Create a view to populate the materialization table * create view ts_internal_mcagg_view * as * select a, partialize(min(b)), partialize(max(d)), timebucket(a), <internal-columns> * from foo * group by <internal-columns> , a , timebucket(a); * * Notes: ViewStmt->query is the raw parse tree * panquery is the output of running parse_anlayze( ViewStmt->query) * Since 1.7, we support real time aggregation. * If real time aggregation is off i.e. materialized only, the mcagg view is as described in Step 2. * If it is turned on * we build a union query that selects from the internal mat view and the raw hypertable * (see build_union_query for details) * CREATE VIEW mcagg * as * SELECT * from * ( SELECT a, finalize(col1) + finalize(col2) from ts_internal_mcagg_view * ---> query from Step 2 with additional where clause * WHERE timecol < materialization threshold * group by <internal-columns> , a , timebucket(a); * UNION ALL * SELECT a, min(b)+max(d) from foo ---> original view stmt * ----> with additional where clause * WHERE timecol >= materialization threshold * GROUP BY a, time_bucket(a) * ) */ static void cagg_create(const CreateTableAsStmt *create_stmt, ViewStmt *stmt, Query *panquery, ContinuousAggTimeBucketInfo *bucket_info, WithClauseResult *with_clause_options) { ObjectAddress mataddress; char relnamebuf[NAMEDATALEN]; FinalizeQueryInfo finalqinfo; CatalogSecurityContext sec_ctx; bool is_create_mattbl_index; Query *final_selquery; Query *partial_selquery; /* query to populate the mattable*/ Query *orig_userview_query; /* copy of the original user query for dummy view */ Oid nspid; RangeVar *part_rel = NULL, *mat_rel = NULL, *dum_rel = NULL; int32 materialize_hypertable_id; bool materialized_only = DatumGetBool(with_clause_options[CreateMaterializedViewFlagMaterializedOnly].parsed); int64 matpartcol_interval = 0; if (!with_clause_options[CreateMaterializedViewFlagChunkTimeInterval].is_default) { matpartcol_interval = interval_to_usec(DatumGetIntervalP( with_clause_options[CreateMaterializedViewFlagChunkTimeInterval].parsed)); } else { matpartcol_interval = bucket_info->htpartcol_interval_len; /* Apply the factor just for non-Hierachical CAggs */ if (bucket_info->parent_mat_hypertable_id == INVALID_HYPERTABLE_ID) matpartcol_interval *= MATPARTCOL_INTERVAL_FACTOR; } /* * Assign the column_name aliases in CREATE VIEW to the query. * No other modifications to panquery. */ fixup_userview_query_tlist(panquery, stmt->aliases); MaterializationHypertableColumnInfo mattblinfo = { .partial_grouplist = copyObject(panquery->groupClause) }; finalizequery_init(&finalqinfo, panquery, &mattblinfo); /* * Invalidate all options on the stmt before using it * The options are valid only for internal use (ts_continuous). */ stmt->options = NULL; /* * Step 1: create the materialization table. */ ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); materialize_hypertable_id = ts_catalog_table_next_seq_id(ts_catalog_get(), HYPERTABLE); ts_catalog_restore_user(&sec_ctx); makeMaterializedTableName(relnamebuf, "_materialized_hypertable_%d", materialize_hypertable_id); mat_rel = makeRangeVar(pstrdup(INTERNAL_SCHEMA_NAME), pstrdup(relnamebuf), -1); is_create_mattbl_index = DatumGetBool(with_clause_options[CreateMaterializedViewFlagCreateGroupIndexes].parsed); create_materialization_table(&mattblinfo, materialize_hypertable_id, mat_rel, bucket_info, is_create_mattbl_index, create_stmt->into, matpartcol_interval, &mataddress); /* * Step 2: Create view with select finalize from materialization table. */ final_selquery = finalizequery_get_select_query(&finalqinfo, mattblinfo.matcollist, &mataddress, mat_rel->relname); if (!materialized_only) final_selquery = build_union_query(bucket_info, mattblinfo.matpartcolno, final_selquery, panquery, materialize_hypertable_id); /* Copy view acl to materialization hypertable. */ ObjectAddress view_address = create_view_for_query(final_selquery, stmt->view); ts_copy_relation_acl(view_address.objectId, mataddress.objectId, GetUserId()); /* * Step 3: create the internal view with select partialize(..). */ partial_selquery = get_partial_select_query(&mattblinfo, panquery); makeMaterializedTableName(relnamebuf, "_partial_view_%d", materialize_hypertable_id); part_rel = makeRangeVar(pstrdup(INTERNAL_SCHEMA_NAME), pstrdup(relnamebuf), -1); create_view_for_query(partial_selquery, part_rel); /* * Additional miscellaneous steps. */ /* * Create a dummy view to store the user supplied view query. * This is to get PG to display the view correctly without * having to replicate the PG source code for make_viewdef. */ orig_userview_query = copyObject(panquery); makeMaterializedTableName(relnamebuf, "_direct_view_%d", materialize_hypertable_id); dum_rel = makeRangeVar(pstrdup(INTERNAL_SCHEMA_NAME), pstrdup(relnamebuf), -1); create_view_for_query(orig_userview_query, dum_rel); /* Step 4: Add catalog table entry for the objects we just created. */ nspid = RangeVarGetCreationNamespace(stmt->view); create_cagg_catalog_entry(materialize_hypertable_id, bucket_info->htid, get_namespace_name(nspid), /*schema name for user view */ stmt->view->relname, part_rel->schemaname, part_rel->relname, materialized_only, dum_rel->schemaname, dum_rel->relname, bucket_info->parent_mat_hypertable_id); char *bucket_origin = NULL; char *bucket_offset = NULL; char *bucket_width = NULL; if (IS_TIME_BUCKET_INFO_TIME_BASED(bucket_info->bf)) { /* Bucketing on time */ Assert(bucket_info->bf->bucket_time_width != NULL); bucket_width = DatumGetCString( DirectFunctionCall1(interval_out, IntervalPGetDatum(bucket_info->bf->bucket_time_width))); if (!TIMESTAMP_NOT_FINITE(bucket_info->bf->bucket_time_origin)) { bucket_origin = DatumGetCString( DirectFunctionCall1(timestamptz_out, TimestampTzGetDatum(bucket_info->bf->bucket_time_origin))); } if (bucket_info->bf->bucket_time_offset != NULL) { bucket_offset = DatumGetCString( DirectFunctionCall1(interval_out, IntervalPGetDatum(bucket_info->bf->bucket_time_offset))); } } else { /* Bucketing on integers */ bucket_width = palloc0(MAXINT8LEN + 1); pg_lltoa(bucket_info->bf->bucket_integer_width, bucket_width); /* Integer buckets with origin are not supported, so noting to do. */ Assert(bucket_origin == NULL); if (bucket_info->bf->bucket_integer_offset != 0) { bucket_offset = palloc0(MAXINT8LEN + 1); pg_lltoa(bucket_info->bf->bucket_integer_offset, bucket_offset); } } create_bucket_function_catalog_entry(materialize_hypertable_id, bucket_info->bf->bucket_function, bucket_width, bucket_origin, bucket_offset, bucket_info->bf->bucket_time_timezone, bucket_info->bf->bucket_fixed_interval); } DDLResult tsl_process_continuous_agg_viewstmt(Node *node, const char *query_string, void *pstmt, WithClauseResult *with_clause_options) { const CreateTableAsStmt *stmt = castNode(CreateTableAsStmt, node); ContinuousAggTimeBucketInfo timebucket_exprinfo; Oid nspid; ViewStmt viewstmt = { .type = T_ViewStmt, .view = stmt->into->rel, .query = (Node *) stmt->into->viewQuery, .options = stmt->into->options, .aliases = stmt->into->colNames, }; ContinuousAgg *cagg; Hypertable *mat_ht; Oid relid; char *schema_name; ts_feature_flag_check(FEATURE_CAGG); nspid = RangeVarGetCreationNamespace(stmt->into->rel); relid = get_relname_relid(stmt->into->rel->relname, nspid); if (OidIsValid(relid)) { if (stmt->if_not_exists) { ereport(NOTICE, (errcode(ERRCODE_DUPLICATE_TABLE), errmsg("continuous aggregate \"%s\" already exists, skipping", stmt->into->rel->relname))); return DDL_DONE; } else { ereport(ERROR, (errcode(ERRCODE_DUPLICATE_TABLE), errmsg("continuous aggregate \"%s\" already exists", stmt->into->rel->relname), errhint("Drop or rename the existing continuous aggregate first or use " "another name."))); } } if (!with_clause_options[CreateMaterializedViewFlagColumnstore].is_default) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot enable compression while creating a continuous aggregate"), errhint("Use ALTER MATERIALIZED VIEW to enable compression."))); } schema_name = get_namespace_name(nspid); timebucket_exprinfo = cagg_validate_query((Query *) stmt->into->viewQuery, schema_name, stmt->into->rel->relname, true); cagg_create(stmt, &viewstmt, (Query *) stmt->query, &timebucket_exprinfo, with_clause_options); /* Insert the MIN of the time dimension type for the new watermark */ CommandCounterIncrement(); relid = get_relname_relid(stmt->into->rel->relname, nspid); Ensure(OidIsValid(relid), "relation \"%s\".\"%s\" not found", schema_name, stmt->into->rel->relname); cagg = ts_continuous_agg_find_by_relid(relid); Ensure(NULL != cagg, "continuous aggregate \"%s\".\"%s\" not found", schema_name, stmt->into->rel->relname); mat_ht = ts_hypertable_get_by_id(cagg->data.mat_hypertable_id); Ensure(NULL != mat_ht, "materialization hypertable %d not found", cagg->data.mat_hypertable_id); ts_cagg_watermark_insert(mat_ht, 0, true); invalidation_threshold_initialize(cagg); if (!stmt->into->skipData) { InternalTimeRange refresh_window = { .type = InvalidOid, }; /* * We are creating a refresh window here in a similar way to how it's * done in continuous_agg_refresh. We do not call the PG function * directly since we want to be able to suppress the output in that * function and adding a 'verbose' parameter to is not useful for a * user. */ refresh_window.type = cagg->partition_type; /* * To determine inscribed/circumscribed refresh window for variable-sized * buckets we should be able to calculate time_bucket(window.begin) and * time_bucket(window.end). This, however, is not possible in general case. * As an example, the minimum date is 4714-11-24 BC, which is before any * reasonable default `origin` value. Thus for variable-sized buckets * instead of minimum date we use -infinity since time_bucket(-infinity) * is well-defined as -infinity. * * For more details see: * - ts_compute_inscribed_bucketed_refresh_window_variable() * - ts_compute_circumscribed_bucketed_refresh_window_variable() */ refresh_window.start = cagg_get_time_min(cagg); refresh_window.end = ts_time_get_noend_or_max(refresh_window.type); ContinuousAggRefreshContext context = { .callctx = CAGG_REFRESH_CREATION }; continuous_agg_refresh_internal(cagg, &refresh_window, context, true, /* start_isnull */ true, /* end_isnull */ true, /* bucketing_refresh_window */ false, /* force */ true, /* process_hypertable_invalidations */ false /*extend_last_bucket*/); } return DDL_DONE; } /* * Flip the view definition of an existing continuous aggregate from * real-time to materialized-only or vice versa depending on the current state. */ void cagg_flip_realtime_view_definition(ContinuousAgg *agg, Hypertable *mat_ht) { int sec_ctx; Oid uid, saved_uid; Query *result_view_query; /* User view query of the user defined CAGG. */ Oid user_view_oid = ts_get_relation_relid(NameStr(agg->data.user_view_schema), NameStr(agg->data.user_view_name), false); Relation user_view_rel = relation_open(user_view_oid, AccessShareLock); Query *user_query = copyObject(get_view_query(user_view_rel)); /* Keep lock until end of transaction. */ relation_close(user_view_rel, NoLock); RemoveRangeTableEntries(user_query); /* Direct view query of the original user view definition at CAGG creation. */ Oid direct_view_oid = ts_get_relation_relid(NameStr(agg->data.direct_view_schema), NameStr(agg->data.direct_view_name), false); Relation direct_view_rel = relation_open(direct_view_oid, AccessShareLock); Query *direct_query = copyObject(get_view_query(direct_view_rel)); /* Keep lock until end of transaction. */ relation_close(direct_view_rel, NoLock); RemoveRangeTableEntries(direct_query); ContinuousAggTimeBucketInfo timebucket_exprinfo = cagg_validate_query(direct_query, NameStr(agg->data.user_view_schema), NameStr(agg->data.user_view_name), false); /* Flip */ agg->data.materialized_only = !agg->data.materialized_only; if (agg->data.materialized_only) { result_view_query = destroy_union_query(user_query); } else { /* Get primary partitioning column information of time bucketing. */ const Dimension *mat_part_dimension = hyperspace_get_open_dimension(mat_ht->space, 0); result_view_query = build_union_query(&timebucket_exprinfo, mat_part_dimension->column_attno, user_query, direct_query, mat_ht->fd.id); } SWITCH_TO_TS_USER(NameStr(agg->data.user_view_schema), uid, saved_uid, sec_ctx); StoreViewQuery(user_view_oid, result_view_query, true); CommandCounterIncrement(); RESTORE_USER(uid, saved_uid, sec_ctx); } /* * Sync target list column names with the relation's pg_attribute names. */ static void sync_target_list_names(List *targetList, TupleDesc desc) { ListCell *lc; int i = 0; foreach (lc, targetList) { TargetEntry *tle = lfirst_node(TargetEntry, lc); if (tle->resjunk) break; FormData_pg_attribute *attr = TupleDescAttr(desc, i); tle->resname = NameStr(attr->attname); ++i; } } void cagg_rename_view_columns(ContinuousAgg *agg) { int sec_ctx; Oid uid, saved_uid; /* * This function is called from the process_rename start handler after * ExecRenameStmt has already been called on the user view, direct view, * partial view, and materialization table. All pg_attribute entries now * have the new column names. * * PostgreSQL's ExecRenameStmt only renames pg_attribute — it does NOT * update the stored query trees in pg_rewrite. We update the stored * query trees for the direct view and the user view so that subsequent * operations (build_union_query, destroy_union_query) see correct names. */ /* --- Update direct view's stored query --- */ Oid direct_view_oid = ts_get_relation_relid(NameStr(agg->data.direct_view_schema), NameStr(agg->data.direct_view_name), false); Relation direct_view_rel = relation_open(direct_view_oid, AccessShareLock); Query *direct_query = copyObject(get_view_query(direct_view_rel)); RemoveRangeTableEntries(direct_query); sync_target_list_names(direct_query->targetList, RelationGetDescr(direct_view_rel)); SWITCH_TO_TS_USER(NameStr(agg->data.user_view_schema), uid, saved_uid, sec_ctx); StoreViewQuery(direct_view_oid, direct_query, true); CommandCounterIncrement(); RESTORE_USER(uid, saved_uid, sec_ctx); /* * Do not close the relation before StoreViewQuery since it can otherwise * release the memory for attr->attname, causing a segfault. */ relation_close(direct_view_rel, NoLock); /* --- Update user view's stored query --- */ Oid user_view_oid = ts_get_relation_relid(NameStr(agg->data.user_view_schema), NameStr(agg->data.user_view_name), false); Relation user_view_rel = relation_open(user_view_oid, AccessShareLock); Query *user_query = copyObject(get_view_query(user_view_rel)); RemoveRangeTableEntries(user_query); TupleDesc user_desc = RelationGetDescr(user_view_rel); sync_target_list_names(user_query->targetList, user_desc); /* * When materialized_only is false the user view query is a UNION ALL * with subqueries whose target lists also carry column names. Update * those so that destroy_union_query(), which extracts a subquery, * produces a query with current names. */ if (user_query->setOperations) { ListCell *lc; foreach (lc, user_query->rtable) { RangeTblEntry *rte = lfirst_node(RangeTblEntry, lc); if (rte->rtekind == RTE_SUBQUERY && rte->subquery) sync_target_list_names(rte->subquery->targetList, user_desc); } } SWITCH_TO_TS_USER(NameStr(agg->data.user_view_schema), uid, saved_uid, sec_ctx); StoreViewQuery(user_view_oid, user_query, true); CommandCounterIncrement(); RESTORE_USER(uid, saved_uid, sec_ctx); relation_close(user_view_rel, NoLock); } ================================================ FILE: tsl/src/continuous_aggs/create.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include "ts_catalog/continuous_agg.h" #include "with_clause/with_clause_parser.h" #include <process_utility.h> DDLResult tsl_process_continuous_agg_viewstmt(Node *node, const char *query_string, void *pstmt, WithClauseResult *with_clause_options); extern void cagg_flip_realtime_view_definition(ContinuousAgg *agg, Hypertable *mat_ht); extern void cagg_rename_view_columns(ContinuousAgg *agg); ================================================ FILE: tsl/src/continuous_aggs/finalize.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include "finalize.h" #include <parser/parse_relation.h> #include "common.h" #include "create.h" /* Static function prototypes */ static Var *mattablecolumninfo_addentry(MaterializationHypertableColumnInfo *out, Node *input, List *rtable, int original_query_resno, bool *skip_adding); static inline void makeMaterializeColumnName(char *colbuf, const char *type, int original_query_resno, int colno); static inline void makeMaterializeColumnName(char *colbuf, const char *type, int original_query_resno, int colno) { int ret = snprintf(colbuf, NAMEDATALEN, "%s_%d_%d", type, original_query_resno, colno); if (ret < 0 || ret >= NAMEDATALEN) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("bad materialization table column name"))); } /* * Init the finalize query data structure. * Parameters: * orig_query - the original query from user view that is being used as template for the finalize * query tlist_aliases - aliases for the view select list materialization table columns are created * . This will be returned in the mattblinfo * * DO NOT modify orig_query. Make a copy if needed. * SIDE_EFFECT: the data structure in mattblinfo is modified as a side effect by adding new * materialize table columns and partialize exprs. */ void finalizequery_init(FinalizeQueryInfo *inp, Query *orig_query, MaterializationHypertableColumnInfo *mattblinfo) { ListCell *lc; int resno = 1; inp->final_userquery = copyObject(orig_query); inp->final_seltlist = NIL; inp->final_havingqual = NULL; /* * We want all the entries in the targetlist (resjunk or not) * in the materialization table definition so we include group-by/having clause etc. * We have to do 3 things here: * 1) create a column for mat table * 2) partialize_expr to populate it, and * 3) modify the target entry to be a finalize_expr * that selects from the materialization table. */ foreach (lc, orig_query->targetList) { TargetEntry *tle = (TargetEntry *) lfirst(lc); TargetEntry *modte = copyObject(tle); if (!orig_query->sortClause) modte->ressortgroupref = 0; /* * We need columns for non-aggregate targets. * If it is not a resjunk OR appears in the grouping clause. */ if (tle->resjunk == false || tle->ressortgroupref > 0) { Var *var; bool skip_adding = false; var = mattablecolumninfo_addentry(mattblinfo, (Node *) tle, orig_query->rtable, resno, &skip_adding); /* Skip adding this column for finalized form. */ if (skip_adding) { continue; } /* Fix the expression for the target entry. */ modte->expr = (Expr *) var; } /* * Construct the targetlist for the query on the * materialization table. The TL maps 1-1 with the original query: * e.g select a, min(b)+max(d) from foo group by a,timebucket(a); * becomes * select <a-col>, * ts_internal_cagg_final(..b-col ) + ts_internal_cagg_final(..d-col) * from mattbl * group by a-col, timebucket(a-col) */ /* * We copy the modte target entries, resnos should be the same for * final_selquery and origquery. So tleSortGroupReffor the targetentry * can be reused, only table info needs to be modified. */ Assert(modte->resno >= resno); resno++; if (IsA(modte->expr, Var)) { modte->resorigcol = ((Var *) modte->expr)->varattno; } inp->final_seltlist = lappend(inp->final_seltlist, modte); } } /* * Create select query with the finalize aggregates * for the materialization table. * matcollist - column list for mat table * mattbladdress - materialization table ObjectAddress * This is the function responsible for creating the final * structures for selecting from the materialized hypertable * created for the Cagg which is * select * from _timescaldeb_internal._materialized_hypertable_<xxx> */ Query * finalizequery_get_select_query(FinalizeQueryInfo *inp, List *matcollist, ObjectAddress *mattbladdress, char *relname) { Query *final_selquery = NULL; CAGG_MAKEQUERY(final_selquery, inp->final_userquery); final_selquery->hasAggs = false; /* New RangeTblEntry for the materialization hypertable */ RangeTblEntry *rte = makeNode(RangeTblEntry); rte->inFromCl = true; rte->inh = true; rte->rellockmode = 1; rte->eref = makeAlias(relname, NIL); rte->relid = mattbladdress->objectId; rte->rtekind = RTE_RELATION; rte->relkind = RELKIND_RELATION; rte->tablesample = NULL; #if PG16_LT rte->requiredPerms |= ACL_SELECT; rte->insertedCols = NULL; rte->updatedCols = NULL; #else RTEPermissionInfo *perminfo = addRTEPermissionInfo(&final_selquery->rteperminfos, rte); perminfo->selectedCols = NULL; perminfo->relid = mattbladdress->objectId; perminfo->requiredPerms |= ACL_SELECT; perminfo->insertedCols = NULL; perminfo->updatedCols = NULL; #endif /* Aliases for column names for the materialization hypertable. */ ListCell *lc; int attno = 0; foreach (lc, matcollist) { ColumnDef *cdef = lfirst_node(ColumnDef, lc); rte->eref->colnames = lappend(rte->eref->colnames, makeString(cdef->colname)); attno = list_length(rte->eref->colnames) - FirstLowInvalidHeapAttributeNumber; #if PG16_LT rte->selectedCols = bms_add_member(rte->selectedCols, attno); #else perminfo->selectedCols = bms_add_member(perminfo->selectedCols, attno); #endif } /* Fixup targetlist with the correct rel information. */ foreach (lc, inp->final_seltlist) { TargetEntry *tle = lfirst_node(TargetEntry, lc); /* * In case when this is a cagg with joins, the Var from the normal table * already has resorigtbl populated and we need to use that to resolve * the Var. Hence only modify the tle when resorigtbl is unset * which means it is Var of the Hypertable */ if (IsA(tle->expr, Var) && !OidIsValid(tle->resorigtbl)) { tle->resorigtbl = rte->relid; tle->resorigcol = castNode(Var, tle->expr)->varattno; } } RangeTblRef *rtr = makeNode(RangeTblRef); rtr->rtindex = 1; final_selquery->rtable = list_make1(rte); final_selquery->jointree = makeFromExpr(list_make1(rtr), NULL); final_selquery->targetList = inp->final_seltlist; final_selquery->sortClause = inp->final_userquery->sortClause; return final_selquery; } /* * Add Information required to create and populate the materialization table columns * creating a columndef for the materialization table. * * Notes: make sure the materialization table columns do not save * values computed by mutable function. * * Notes on TargetEntry fields: * - (resname != NULL) means it's projected in our case * - (ressortgroupref > 0) means part of GROUP BY, which can be projected or not, depending of the * value of the resjunk * - (resjunk == true) applies for GROUP BY columns that are not projected * */ static Var * mattablecolumninfo_addentry(MaterializationHypertableColumnInfo *out, Node *input, List *rtable, int original_query_resno, bool *skip_adding) { int matcolno = list_length(out->matcollist) + 1; char colbuf[NAMEDATALEN]; char *colname; TargetEntry *part_te = NULL; ColumnDef *col; Var *var; Oid coltype = InvalidOid, colcollation = InvalidOid; int32 coltypmod; *skip_adding = false; if (contain_mutable_functions(input)) { ereport(WARNING, (errmsg("using non-immutable functions in continuous aggregate view may lead to " "inconsistent results on rematerialization"))); } switch (nodeTag(input)) { case T_TargetEntry: { TargetEntry *tle = (TargetEntry *) input; bool timebkt_chk = false; if (IsA(tle->expr, FuncExpr)) timebkt_chk = function_allowed_in_cagg_definition(((FuncExpr *) tle->expr)->funcid); #if PG18_GE /* PG18 introduced RTEs for group clauses so * we use rtable to look up GROUP BY expressions. * * https://github.com/postgres/postgres/commit/247dea89 */ if (IsA(tle->expr, Var)) { Var *var = castNode(Var, tle->expr); Assert((int) var->varno <= list_length(rtable)); RangeTblEntry *rte = list_nth(rtable, var->varno - 1); Assert(var->varattno > 0); /* * Var might not be part of the GROUP BY clause * eg for functionally dependent columns on tables with primary key */ if (rte->rtekind == RTE_GROUP) { Node *node = list_nth(rte->groupexprs, var->varattno - 1); if (IsA(node, FuncExpr)) { if (contain_mutable_functions(node)) { ereport(WARNING, (errmsg("using non-immutable functions in continuous aggregate " "view may lead to " "inconsistent results on rematerialization"))); } FuncExpr *expr = (FuncExpr *) node; timebkt_chk = function_allowed_in_cagg_definition(((FuncExpr *) expr)->funcid); } } } #endif if (tle->resname) colname = pstrdup(tle->resname); else { if (timebkt_chk) colname = DEFAULT_MATPARTCOLUMN_NAME; else { makeMaterializeColumnName(colbuf, "grp", original_query_resno, matcolno); colname = colbuf; /* For finalized form we skip adding extra group by columns. */ *skip_adding = true; } } if (timebkt_chk) { tle->resname = pstrdup(colname); out->matpartcolno = matcolno; out->matpartcolname = pstrdup(colname); } else { /* * Add indexes only for columns that are part of the GROUP BY clause * and for finals form. * We skip adding it because we'll not add the extra group by columns * to the materialization hypertable anymore. */ if (!*skip_adding && tle->ressortgroupref > 0) out->mat_groupcolname_list = lappend(out->mat_groupcolname_list, pstrdup(colname)); } coltype = exprType((Node *) tle->expr); coltypmod = exprTypmod((Node *) tle->expr); colcollation = exprCollation((Node *) tle->expr); col = makeColumnDef(colname, coltype, coltypmod, colcollation); part_te = (TargetEntry *) copyObject(input); /* Keep original resjunk if not time bucket. */ if (timebkt_chk) { /* * Need to project all the partial entries so that * materialization table is filled. */ part_te->resjunk = false; } part_te->resno = matcolno; if (timebkt_chk) { col->is_not_null = true; } if (part_te->resname == NULL) { part_te->resname = pstrdup(colname); } } break; case T_Var: { makeMaterializeColumnName(colbuf, "var", original_query_resno, matcolno); colname = colbuf; coltype = exprType(input); coltypmod = exprTypmod(input); colcollation = exprCollation(input); col = makeColumnDef(colname, coltype, coltypmod, colcollation); part_te = makeTargetEntry((Expr *) input, matcolno, pstrdup(colname), false); /* Need to project all the partial entries so that materialization table is filled. */ part_te->resjunk = false; part_te->resno = matcolno; } break; default: elog(ERROR, "invalid node type %d", nodeTag(input)); break; } Assert(list_length(out->matcollist) <= list_length(out->partial_seltlist)); Assert(col != NULL); Assert(part_te != NULL); if (!*skip_adding) { out->matcollist = lappend(out->matcollist, col); } out->partial_seltlist = lappend(out->partial_seltlist, part_te); var = makeVar(1, matcolno, coltype, coltypmod, colcollation, 0); return var; } ================================================ FILE: tsl/src/continuous_aggs/finalize.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <catalog/pg_aggregate.h> #include <catalog/pg_collation.h> #include <catalog/pg_type.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <nodes/pg_list.h> #include <parser/parse_func.h> #include <utils/builtins.h> #include <utils/regproc.h> #include <utils/syscache.h> #include "common.h" #include "ts_catalog/catalog.h" extern Query *finalize_query_get_select_query(FinalizeQueryInfo *inp, List *matcollist, ObjectAddress *mattbladdress); extern void finalizequery_init(FinalizeQueryInfo *inp, Query *orig_query, MaterializationHypertableColumnInfo *mattblinfo); extern Query *finalizequery_get_select_query(FinalizeQueryInfo *inp, List *matcollist, ObjectAddress *mattbladdress, char *relname); ================================================ FILE: tsl/src/continuous_aggs/insert.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <utils/hsearch.h> #include <utils/snapmgr.h> #include "compat/compat.h" #include "continuous_aggs/insert.h" #include "debug_point.h" #include "guc.h" #include "invalidation.h" #include "partitioning.h" /* * When tuples in a hypertable that has a continuous aggregate are modified, the * lowest modified value and the greatest modified value must be tracked over * the course of a transaction or statement. At the end of the statement these * values will be inserted into the proper cache invalidation log table for * their associated hypertable if they are below the speculative materialization * watermark (or, if in REPEATABLE_READ isolation level or higher, they will be * inserted no matter what as we cannot see if a materialization transaction has * started and moved the watermark during our transaction in that case). * * We accomplish this at the transaction level by keeping a hash table of each * hypertable that has been modified in the transaction and the lowest and * greatest modified values. The hashtable will be updated via ModifyHypertable * for every row that is inserted, updated or deleted. * We use a hashtable because we need to keep track of this on a per hypertable * basis and multiple can have tuples modified during a single transaction. * (And if we move to per-chunk cache-invalidation it makes it even easier). * */ typedef struct ContinuousAggsCacheInvalEntry { Oid chunk_relid; int32 hypertable_id; Dimension hypertable_open_dimension; AttrNumber open_dimension_attno; bool value_is_set; int64 lowest_modified_value; int64 greatest_modified_value; } ContinuousAggsCacheInvalEntry; typedef struct ContinuousAggsCacheHyperInvalThresholdEntry { int32 hypertable_id; int64 watermark; } ContinuousAggsCacheHyperInvalThresholdEntry; static int64 get_lowest_invalidated_time_for_hypertable(int32 hypertable_id); static inline int64 cache_get_lowest_invalidated_time_for_hypertable(int32 hypertable_id); #define CA_CACHE_INVAL_INIT_HTAB_SIZE 64 static HTAB *continuous_aggs_cache_inval_htab = NULL; static HTAB *continuous_aggs_cache_hyper_inval_threshold_htab = NULL; static MemoryContext continuous_aggs_invalidation_mctx = NULL; static inline void cache_inval_entry_init(ContinuousAggsCacheInvalEntry *cache_entry, int32 hypertable_id, Oid chunk_relid); static inline ContinuousAggsCacheInvalEntry *get_cache_inval_entry(int32 hypertable_id, Oid chunk_relid); static void cache_inval_cleanup(void); static void cache_inval_htab_write(void); static void continuous_agg_xact_invalidation_callback(XactEvent event, void *arg); static ScanTupleResult invalidation_tuple_found(TupleInfo *ti, void *min); static void cache_inval_init() { HASHCTL ctl; Assert(continuous_aggs_invalidation_mctx == NULL); continuous_aggs_invalidation_mctx = AllocSetContextCreate(TopTransactionContext, "ContinuousAggsInvalidationCtx", ALLOCSET_DEFAULT_SIZES); memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(ContinuousAggsCacheInvalEntry); ctl.hcxt = continuous_aggs_invalidation_mctx; continuous_aggs_cache_inval_htab = hash_create("TS Continuous Aggs Cache Inval", CA_CACHE_INVAL_INIT_HTAB_SIZE, &ctl, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(int32); ctl.entrysize = sizeof(ContinuousAggsCacheHyperInvalThresholdEntry); ctl.hcxt = continuous_aggs_invalidation_mctx; continuous_aggs_cache_hyper_inval_threshold_htab = hash_create("TS Continuous Aggs Hypertable Invalidation Threshold", CA_CACHE_INVAL_INIT_HTAB_SIZE, &ctl, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); } static void update_cache_from_tuple(ContinuousAggsCacheInvalEntry *cache_entry, HeapTuple tuple, TupleDesc tupdesc) { Datum datum; bool isnull; Oid dimtype; Dimension *d = &cache_entry->hypertable_open_dimension; AttrNumber col = cache_entry->open_dimension_attno; Assert(d->type == DIMENSION_TYPE_OPEN); datum = heap_getattr(tuple, col, tupdesc, &isnull); /* * Even though there are NOT NULL constraints on time columns checking these happens * after invalidation processing so we skip nulls here to allow for normal postgres * error handling for these NULL values. */ if (isnull) return; dimtype = ts_dimension_get_partition_type(d); int64 timeval = ts_time_value_to_internal(datum, dimtype); cache_entry->value_is_set = true; if (timeval < cache_entry->lowest_modified_value) cache_entry->lowest_modified_value = timeval; if (timeval > cache_entry->greatest_modified_value) cache_entry->greatest_modified_value = timeval; } static inline void cache_inval_entry_init(ContinuousAggsCacheInvalEntry *cache_entry, int32 hypertable_id, Oid chunk_relid) { Cache *ht_cache = ts_hypertable_cache_pin(); Hypertable *ht = ts_hypertable_cache_get_entry_by_id(ht_cache, hypertable_id); Ensure(ht, "could not find hypertable with id %d", hypertable_id); const Dimension *open_dim = hyperspace_get_open_dimension(ht->space, 0); Ensure(open_dim, "hypertable %d has no open partitioning dimension", hypertable_id); cache_entry->chunk_relid = chunk_relid; cache_entry->hypertable_id = hypertable_id; cache_entry->hypertable_open_dimension = *open_dim; cache_entry->open_dimension_attno = get_attnum(chunk_relid, NameStr(open_dim->fd.column_name)); cache_entry->value_is_set = false; cache_entry->lowest_modified_value = INVAL_POS_INFINITY; cache_entry->greatest_modified_value = INVAL_NEG_INFINITY; ts_cache_release(&ht_cache); } static inline ContinuousAggsCacheInvalEntry * get_cache_inval_entry(int32 hypertable_id, Oid chunk_relid) { ContinuousAggsCacheInvalEntry *cache_entry; bool found; if (!continuous_aggs_cache_inval_htab) cache_inval_init(); cache_entry = (ContinuousAggsCacheInvalEntry *) hash_search(continuous_aggs_cache_inval_htab, &chunk_relid, HASH_ENTER, &found); if (!found) cache_inval_entry_init(cache_entry, hypertable_id, chunk_relid); return cache_entry; } /* * Used by direct compress invalidation */ void continuous_agg_invalidate_range(int32 hypertable_id, Oid chunk_relid, int64 start, int64 end) { ContinuousAggsCacheInvalEntry *cache_entry = get_cache_inval_entry(hypertable_id, chunk_relid); cache_entry->value_is_set = true; Assert(start <= end); if (start < cache_entry->lowest_modified_value) cache_entry->lowest_modified_value = start; if (end > cache_entry->greatest_modified_value) cache_entry->greatest_modified_value = end; } void continuous_agg_dml_invalidate(int32 hypertable_id, Relation chunk_rel, HeapTuple chunk_tuple, HeapTuple chunk_newtuple, bool update) { ContinuousAggsCacheInvalEntry *cache_entry = get_cache_inval_entry(hypertable_id, chunk_rel->rd_id); update_cache_from_tuple(cache_entry, chunk_tuple, RelationGetDescr(chunk_rel)); if (!update) return; /* on update we need to invalidate the new time value as well as the old one */ update_cache_from_tuple(cache_entry, chunk_newtuple, RelationGetDescr(chunk_rel)); } static inline void cache_inval_entry_write(ContinuousAggsCacheInvalEntry *entry) { int64 liv; if (!entry->value_is_set) return; /* The materialization worker uses a READ COMMITTED isolation level by default. Therefore, if we * use a stronger isolation level, the isolation threshold could update without us seeing the * new value. In order to prevent serialization errors, we always append invalidation entries in * the case when we're using a strong enough isolation level that we won't see the new * threshold. The materializer can handle invalidations that are beyond the threshold * gracefully. */ if (IsolationUsesXactSnapshot()) { invalidation_hyper_log_add_entry(entry->hypertable_id, entry->lowest_modified_value, entry->greatest_modified_value); return; } liv = cache_get_lowest_invalidated_time_for_hypertable(entry->hypertable_id); if (entry->lowest_modified_value < liv) invalidation_hyper_log_add_entry(entry->hypertable_id, entry->lowest_modified_value, entry->greatest_modified_value); }; static void cache_inval_cleanup(void) { Assert(continuous_aggs_cache_inval_htab != NULL); Assert(continuous_aggs_cache_hyper_inval_threshold_htab != NULL); hash_destroy(continuous_aggs_cache_inval_htab); hash_destroy(continuous_aggs_cache_hyper_inval_threshold_htab); MemoryContextDelete(continuous_aggs_invalidation_mctx); continuous_aggs_cache_inval_htab = NULL; continuous_aggs_cache_hyper_inval_threshold_htab = NULL; continuous_aggs_invalidation_mctx = NULL; }; static void cache_inval_htab_write(void) { HASH_SEQ_STATUS hash_seq; ContinuousAggsCacheInvalEntry *current_entry; Catalog *catalog; if (hash_get_num_entries(continuous_aggs_cache_inval_htab) == 0) return; catalog = ts_catalog_get(); /* The invalidation threshold must remain locked until the end of * the transaction to ensure the materializer will see our updates, * so we explicitly lock it here */ LockRelationOid(catalog_get_table_id(catalog, CONTINUOUS_AGGS_INVALIDATION_THRESHOLD), AccessShareLock); hash_seq_init(&hash_seq, continuous_aggs_cache_inval_htab); while ((current_entry = hash_seq_search(&hash_seq)) != NULL) cache_inval_entry_write(current_entry); }; /* * We use TopTransactionContext for our cached invalidations. * We need to make sure cache_inval_cleanup() is always called after cache_inval_htab_write(). * We need this memory context to survive the transaction lifetime so that cache_inval_cleanup() * does not attempt to tear down memory that has already been freed due to a transaction ending. * * The order of operations in postgres can be this: * CallXactCallbacks(XACT_EVENT_PRE_PREPARE); * ... * CallXactCallbacks(XACT_EVENT_PREPARE); * ... * MemoryContextDelete(TopTransactionContext); * * or that: * CallXactCallbacks(XACT_EVENT_PRE_COMMIT); * ... * CallXactCallbacks(XACT_EVENT_COMMIT); * ... * MemoryContextDelete(TopTransactionContext); * * In the case of a 2PC transaction, we need to make sure to apply the invalidations at * XACT_EVENT_PRE_PREPARE time, before TopTransactionContext is torn down by PREPARE TRANSACTION. * Otherwise, we are unable to call cache_inval_cleanup() without corrupting the memory. For * this reason, we also deallocate at XACT_EVENT_PREPARE time. * * For local transactions we apply the invalidations at XACT_EVENT_PRE_COMMIT time. * Similar care is taken of parallel workers and aborting transactions. */ static void continuous_agg_xact_invalidation_callback(XactEvent event, void *arg) { /* Return quickly if we never initialize the hashtable */ if (!continuous_aggs_cache_inval_htab) return; switch (event) { case XACT_EVENT_PRE_PREPARE: case XACT_EVENT_PRE_COMMIT: case XACT_EVENT_PARALLEL_PRE_COMMIT: cache_inval_htab_write(); break; case XACT_EVENT_PREPARE: case XACT_EVENT_COMMIT: case XACT_EVENT_PARALLEL_COMMIT: case XACT_EVENT_ABORT: case XACT_EVENT_PARALLEL_ABORT: cache_inval_cleanup(); break; default: break; } } void _continuous_aggs_cache_inval_init(void) { RegisterXactCallback(continuous_agg_xact_invalidation_callback, NULL); } void _continuous_aggs_cache_inval_fini(void) { UnregisterXactCallback(continuous_agg_xact_invalidation_callback, NULL); } static ScanTupleResult invalidation_tuple_found(TupleInfo *ti, void *min) { bool isnull; Datum watermark = slot_getattr(ti->slot, Anum_continuous_aggs_invalidation_threshold_watermark, &isnull); Assert(!isnull); if (DatumGetInt64(watermark) < *((int64 *) min)) *((int64 *) min) = DatumGetInt64(watermark); DEBUG_WAITPOINT("invalidation_tuple_found_done"); /* * Return SCAN_CONTINUE because we check for multiple tuples as an error * condition. */ return SCAN_CONTINUE; } static int64 get_lowest_invalidated_time_for_hypertable(int32 hypertable_id) { int64 min_val = INVAL_POS_INFINITY; Catalog *catalog = ts_catalog_get(); ScanKeyData scankey[1]; ScannerCtx scanctx; PushActiveSnapshot(GetLatestSnapshot()); ScanKeyInit(&scankey[0], Anum_continuous_aggs_invalidation_threshold_pkey_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); scanctx = (ScannerCtx){ .table = catalog_get_table_id(catalog, CONTINUOUS_AGGS_INVALIDATION_THRESHOLD), .index = catalog_get_index(catalog, CONTINUOUS_AGGS_INVALIDATION_THRESHOLD, CONTINUOUS_AGGS_INVALIDATION_THRESHOLD_PKEY), .nkeys = 1, .scankey = scankey, .tuple_found = &invalidation_tuple_found, .filter = NULL, .data = &min_val, .lockmode = AccessShareLock, .scandirection = ForwardScanDirection, .result_mctx = NULL, /* We need to define a custom snapshot for this scan. The default snapshot (SNAPSHOT_SELF) reads data of all committed transactions, even if they have started after our scan. If a parallel session updates the scanned value and commits during a scan, we end up in a situation where we see the old and the new value. This causes ts_scanner_scan_one() to fail. */ .snapshot = GetActiveSnapshot(), }; /* If we don't find any invalidation threshold watermark, then we've never done any * materialization we'll treat this as if the invalidation timestamp is at min value, since the * first materialization needs to scan the entire table anyway; the invalidations are redundant. */ if (!ts_scanner_scan_one(&scanctx, false, CAGG_INVALIDATION_THRESHOLD_NAME)) min_val = INVAL_NEG_INFINITY; PopActiveSnapshot(); return min_val; } static inline int64 cache_get_lowest_invalidated_time_for_hypertable(int32 hypertable_id) { ContinuousAggsCacheHyperInvalThresholdEntry *hyper_inval_cache_entry; bool found; hyper_inval_cache_entry = (ContinuousAggsCacheHyperInvalThresholdEntry *) hash_search(continuous_aggs_cache_hyper_inval_threshold_htab, &hypertable_id, HASH_ENTER, &found); if (!found) { hyper_inval_cache_entry->hypertable_id = hypertable_id; hyper_inval_cache_entry->watermark = get_lowest_invalidated_time_for_hypertable(hypertable_id); } return hyper_inval_cache_entry->watermark; } ================================================ FILE: tsl/src/continuous_aggs/insert.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> extern void _continuous_aggs_cache_inval_init(void); extern void _continuous_aggs_cache_inval_fini(void); extern void continuous_agg_invalidate_range(int32 hypertable_id, Oid chunk_relid, int64 start, int64 end); extern void continuous_agg_dml_invalidate(int32 hypertable_id, Relation chunk_rel, HeapTuple chunk_tuple, HeapTuple chunk_newtuple, bool update); ================================================ FILE: tsl/src/continuous_aggs/invalidation.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/htup.h> #include <access/htup_details.h> #include <access/xact.h> #include <extension.h> #include <fmgr.h> #include <funcapi.h> #include <hypertable_cache.h> #include <miscadmin.h> #include <nodes/makefuncs.h> #include <nodes/memnodes.h> #include <parser/parse_func.h> #include <scan_iterator.h> #include <scanner.h> #include <storage/lockdefs.h> #include <time_bucket.h> #include <time_utils.h> #include <utils.h> #include <utils/builtins.h> #include <utils/elog.h> #include <utils/lsyscache.h> #include <utils/memutils.h> #include <utils/palloc.h> #include <utils/snapmgr.h> #include <utils/tuplestore.h> #include "cache.h" #include "continuous_aggs/invalidation_threshold.h" #include "continuous_aggs/materialize.h" #include "guc.h" #include "invalidation.h" #include "refresh.h" #include "ts_catalog/catalog.h" #include "ts_catalog/continuous_agg.h" /* * Invalidation processing for continuous aggregates. * * Invalidations track the regions/intervals [start, end] of a continuous * aggregate that are out-of-date relative to the source hypertable on top of * which the aggregate is defined. When a continuous aggregate is out-of-date * across one or more regions, it can be refreshed with a window covering * those regions in order to bring it up-to-date with the source data again. * * Invalidations are generated by mutations on the source hypertable (INSERT, * DELETE, UPDATE, TRUNCATE, drop_chunks, etc.) and are initially written to a * hypertable invalidation log [hypertable_id, start, end]. * * When a continuous aggregate is refreshed, invalidations are moved from the * hypertable invalidation log to a continuous aggregate invalidation log, * where each original entry creates one new entry per continuous aggregate * [cagg_id, start, end]. Thus, if one continuous aggregate is refreshed but * not others, then only the invalidations for the refreshed aggregate are * processed. * * Simplified, invalidations move through the following stages: * * insert_trigger => hypertable_inval_log => cagg_inval_log => refreshing * * Thus, invalidations are generated by mutations and are processed and used * as input for refreshing the a continuous aggregate. * * Invalidations can overlap or be duplicates. Therefore, invalidations are * merged during processing to reduce the number of entries in the logs. This * typically happens during a refresh of a continuous aggregate, which also * cuts invalidations along the refresh window. The cutting will leave some * parts of entries in the invalidation log while the entries that fall * within the refresh window are stored in an invalidation store and used for * refreshing: * * |-------------| refresh window * * |-----| |---| |----| invalidations * * => * * |---| |--| invalidations that remain in the log * * |-| |---| |--| invalidations that are used for refreshing * * The invalidation store will spill to disk in case of many invalidations so * it won't blow up memory usage. If there are no invalidations in the store * after processing, then the continuous aggregate is up-to-date in the region * defined by the refresh window. */ /* * Processing state used while processing the materialization invalidation log * and refreshing the continuous aggregate. */ typedef struct ContinuousAggInvalidationState { const ContinuousAgg *cagg; MemoryContext per_tuple_mctx; Relation cagg_log_rel; Relation cagg_queue_rel; Snapshot snapshot; Tuplestorestate *invalidations; } ContinuousAggInvalidationState; /* * Processing state used while moving invalidations from hypertable * invalidation log to materialization invalidation log. */ typedef struct HypertableInvalidationState { int32 hypertable_id; Oid dimtype; /* Type of the underlying hypertable's bucketed attribute */ const ContinuousAggInfo *all_caggs; MemoryContext per_tuple_mctx; Relation cagg_log_rel; Snapshot snapshot; } HypertableInvalidationState; typedef enum ContinuousAggTableType { HYPER_INVALIDATION_LOG, CAGG_INVALIDATION_LOG, CAGG_MATERIALIZATION_RANGES, } ContinuousAggTableType; static Relation open_cagg_table(ContinuousAggTableType type, LOCKMODE lockmode); static void hypertable_invalidation_scan_init(ScanIterator *iterator, int32 hyper_id, LOCKMODE lockmode); static HeapTuple create_materialization_ranges_tup(TupleDesc tupdesc, int32 cagg_hyper_id, const InternalTimeRange range); static void check_materialization_ranges_overlap(const ContinuousAgg *cagg, const InternalTimeRange refresh_window); static void insert_new_cagg_materialization_ranges(const ContinuousAggInvalidationState *state, const InternalTimeRange refresh_window, int32 cagg_hyper_id); static bool save_invalidation_for_refresh(const ContinuousAggInvalidationState *state, const Invalidation *invalidation); static void set_remainder_after_cut(Invalidation *remainder, int32 hyper_id, int64 lowest_modified_value, int64 greatest_modified_value); static void invalidation_entry_reset(Invalidation *entry); static void invalidation_entry_set_from_hyper_invalidation(Invalidation *entry, const TupleInfo *ti, int32 hyper_id, Oid dimtype, const ContinuousAggBucketFunction *bucket_function); static void invalidation_entry_set_from_cagg_invalidation(Invalidation *entry, const TupleInfo *ti, Oid dimtype, const ContinuousAggBucketFunction *bucket_function); static bool invalidations_can_be_merged(const Invalidation *a, const Invalidation *b); static bool invalidation_entry_try_merge(Invalidation *entry, const Invalidation *newentry); static void insert_new_cagg_invalidation(const HypertableInvalidationState *state, const Invalidation *entry, int32 cagg_hyper_id); static void move_invalidations_from_hyper_to_cagg_log(const HypertableInvalidationState *state); static void cagg_invalidations_scan_by_hypertable_init(ScanIterator *iterator, int32 cagg_hyper_id, LOCKMODE lockmode); static Invalidation cut_cagg_invalidation(const ContinuousAggInvalidationState *state, const InternalTimeRange *refresh_window, const Invalidation *entry); static Invalidation cut_cagg_invalidation_and_compute_remainder( const ContinuousAggInvalidationState *state, const InternalTimeRange *refresh_window, const Invalidation *mergedentry, const Invalidation *current_remainder); static void clear_cagg_invalidations_for_refresh(const ContinuousAggInvalidationState *state, const InternalTimeRange *refresh_window, bool force); static void cagg_invalidation_state_init(ContinuousAggInvalidationState *state, const ContinuousAgg *cagg); static void cagg_invalidation_state_cleanup(const ContinuousAggInvalidationState *state); static Relation open_cagg_table(ContinuousAggTableType type, LOCKMODE lockmode) { static const CatalogTable logmappings[] = { [HYPER_INVALIDATION_LOG] = CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG, [CAGG_INVALIDATION_LOG] = CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG, [CAGG_MATERIALIZATION_RANGES] = CONTINUOUS_AGGS_MATERIALIZATION_RANGES, }; Catalog *catalog = ts_catalog_get(); Oid relid = catalog_get_table_id(catalog, logmappings[type]); return table_open(relid, lockmode); } static void hypertable_invalidation_scan_init(ScanIterator *iterator, int32 hyper_id, LOCKMODE lockmode) { *iterator = ts_scan_iterator_create(CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG, lockmode, CurrentMemoryContext); iterator->ctx.index = catalog_get_index(ts_catalog_get(), CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG, CONTINUOUS_AGGS_HYPERTABLE_INVALIDATION_LOG_IDX); ts_scan_iterator_scan_key_init( iterator, Anum_continuous_aggs_hypertable_invalidation_log_idx_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hyper_id)); } HeapTuple create_invalidation_tup(const TupleDesc tupdesc, int32 cagg_hyper_id, int64 start, int64 end) { Datum values[Natts_continuous_aggs_materialization_invalidation_log] = { 0 }; bool isnull[Natts_continuous_aggs_materialization_invalidation_log] = { false }; values[AttrNumberGetAttrOffset( Anum_continuous_aggs_materialization_invalidation_log_materialization_id)] = Int32GetDatum(cagg_hyper_id); values[AttrNumberGetAttrOffset( Anum_continuous_aggs_materialization_invalidation_log_lowest_modified_value)] = Int64GetDatum(start); values[AttrNumberGetAttrOffset( Anum_continuous_aggs_materialization_invalidation_log_greatest_modified_value)] = Int64GetDatum(end); return heap_form_tuple(tupdesc, values, isnull); } /* * Add an entry to the continuous aggregate invalidation log. */ void invalidation_cagg_log_add_entry(int32 cagg_hyper_id, int64 start, int64 end) { Relation rel = open_cagg_table(CAGG_INVALIDATION_LOG, RowExclusiveLock); CatalogSecurityContext sec_ctx; HeapTuple tuple; Assert(start <= end); tuple = create_invalidation_tup(RelationGetDescr(rel), cagg_hyper_id, start, end); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_insert_only(rel, tuple); ts_catalog_restore_user(&sec_ctx); heap_freetuple(tuple); table_close(rel, NoLock); } void invalidation_hyper_log_add_entry(int32 hyper_id, int64 start, int64 end) { Relation rel = open_cagg_table(HYPER_INVALIDATION_LOG, RowExclusiveLock); CatalogSecurityContext sec_ctx; Datum values[Natts_continuous_aggs_hypertable_invalidation_log]; bool nulls[Natts_continuous_aggs_hypertable_invalidation_log] = { false }; Assert(start <= end); values[AttrNumberGetAttrOffset( Anum_continuous_aggs_hypertable_invalidation_log_hypertable_id)] = Int32GetDatum(hyper_id); values[AttrNumberGetAttrOffset( Anum_continuous_aggs_hypertable_invalidation_log_lowest_modified_value)] = Int64GetDatum(start); values[AttrNumberGetAttrOffset( Anum_continuous_aggs_hypertable_invalidation_log_greatest_modified_value)] = Int64GetDatum(end); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_insert_values(rel, RelationGetDescr(rel), values, nulls); ts_catalog_restore_user(&sec_ctx); table_close(rel, NoLock); elog(DEBUG1, "hypertable log for hypertable %d added entry [" INT64_FORMAT ", " INT64_FORMAT "]", hyper_id, start, end); } /* * Invalidate one or more continuous aggregates. * * Add an invalidation in the given range. The invalidation is added either to * the hypertable invalidation log or the continuous aggregate invalidation * log depending on the type of the given hypertable. If the hypertable is a * "raw" hypertable (i.e., one that has one or more continuous aggregates), the * entry is added to the hypertable invalidation log and will invalidate all * the associated continuous aggregates. If the hypertable is instead an * materialized hypertable, the entry is added to the cagg invalidation log * and only invalidates the continuous aggregate owning that materialized * hypertable. */ void continuous_agg_invalidate_raw_ht(const Hypertable *raw_ht, int64 start, int64 end) { Assert(raw_ht != NULL); invalidation_hyper_log_add_entry(raw_ht->fd.id, start, end); } void continuous_agg_invalidate_mat_ht(const Hypertable *raw_ht, const Hypertable *mat_ht, int64 start, int64 end) { Assert((raw_ht != NULL) && (mat_ht != NULL)); invalidation_cagg_log_add_entry(mat_ht->fd.id, start, end); } static HeapTuple create_materialization_ranges_tup(TupleDesc tupdesc, int32 cagg_hyper_id, const InternalTimeRange range) { Datum values[Natts_continuous_aggs_materialization_ranges] = { 0 }; bool isnull[Natts_continuous_aggs_materialization_ranges] = { false }; values[AttrNumberGetAttrOffset( Anum_continuous_aggs_materialization_ranges_materialization_id)] = Int32GetDatum(cagg_hyper_id); values[AttrNumberGetAttrOffset( Anum_continuous_aggs_materialization_ranges_lowest_modified_value)] = Int64GetDatum(range.start); values[AttrNumberGetAttrOffset( Anum_continuous_aggs_materialization_ranges_greatest_modified_value)] = Int64GetDatum(range.end); return heap_form_tuple(tupdesc, values, isnull); } /* * Check if a new materialization range overlaps with any existing range for the * same materialization_id. two ranges [s1, e1) and [s2, e2) overlap iff * s1 < e2 AND e1 > s2. */ static void check_materialization_ranges_overlap(const ContinuousAgg *cagg, const InternalTimeRange refresh_window) { ScanIterator iterator = ts_scan_iterator_create(CONTINUOUS_AGGS_MATERIALIZATION_RANGES, AccessShareLock, CurrentMemoryContext); iterator.ctx.index = catalog_get_index(ts_catalog_get(), CONTINUOUS_AGGS_MATERIALIZATION_RANGES, CONTINUOUS_AGGS_MATERIALIZATION_RANGES_IDX); ts_scan_iterator_scan_key_init( &iterator, Anum_continuous_aggs_materialization_ranges_idx_materialization_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(cagg->data.mat_hypertable_id)); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); bool isnull; int64 existing_start = DatumGetInt64( slot_getattr(ti->slot, Anum_continuous_aggs_materialization_ranges_lowest_modified_value, &isnull)); Assert(!isnull); int64 existing_end = DatumGetInt64( slot_getattr(ti->slot, Anum_continuous_aggs_materialization_ranges_greatest_modified_value, &isnull)); Assert(!isnull); if (refresh_window.start < existing_end && refresh_window.end > existing_start) { ts_scan_iterator_close(&iterator); ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("materialization range [%s, %s) overlaps with existing range [%s, %s)" " in materialization_ranges table for continuous aggregate " "\"%s\".\"%s\"", ts_internal_to_time_string(refresh_window.start, cagg->partition_type), ts_internal_to_time_string(refresh_window.end, cagg->partition_type), ts_internal_to_time_string(existing_start, cagg->partition_type), ts_internal_to_time_string(existing_end, cagg->partition_type), NameStr(cagg->data.user_view_schema), NameStr(cagg->data.user_view_name)), errdetail("A concurrent refresh is working on this range or a previously" " failed refresh left an overlapping range in the" " materialization_ranges table."))); } } ts_scan_iterator_close(&iterator); } static void insert_new_cagg_materialization_ranges(const ContinuousAggInvalidationState *state, const InternalTimeRange refresh_window, int32 cagg_hyper_id) { CatalogSecurityContext sec_ctx; TupleDesc tupdesc = RelationGetDescr(state->cagg_queue_rel); HeapTuple tuple; check_materialization_ranges_overlap(state->cagg, refresh_window); tuple = create_materialization_ranges_tup(tupdesc, cagg_hyper_id, refresh_window); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_insert_only(state->cagg_queue_rel, tuple); ts_catalog_restore_user(&sec_ctx); heap_freetuple(tuple); } typedef enum InvalidationResult { INVAL_NOMATCH, INVAL_DELETE, INVAL_CUT, } InvalidationResult; static inline bool IsValidInvalidation(const Invalidation *invalidation) { Assert(invalidation->lowest_modified_value <= invalidation->greatest_modified_value); return invalidation->hyper_id != INVALID_HYPERTABLE_ID && invalidation->lowest_modified_value <= invalidation->greatest_modified_value; } static bool save_invalidation_for_refresh(const ContinuousAggInvalidationState *state, const Invalidation *invalidation) { if (!IsValidInvalidation(invalidation)) return false; int32 cagg_hyper_id = state->cagg->data.mat_hypertable_id; TupleDesc tupdesc = RelationGetDescr(state->cagg_log_rel); HeapTuple refresh_tup = create_invalidation_tup(tupdesc, cagg_hyper_id, invalidation->lowest_modified_value, invalidation->greatest_modified_value); tuplestore_puttuple(state->invalidations, refresh_tup); heap_freetuple(refresh_tup); InternalTimeRange refresh_window = { .type = state->cagg->partition_type, .start = invalidation->lowest_modified_value, /* Invalidations are inclusive at the end, while refresh windows aren't, so add one to the end of the invalidated region */ .end = ts_time_saturating_add(invalidation->greatest_modified_value, 1, state->cagg->partition_type), }; InternalTimeRange bucketed_refresh_window = compute_circumscribed_bucketed_refresh_window(state->cagg, &refresh_window, state->cagg->bucket_function); insert_new_cagg_materialization_ranges(state, bucketed_refresh_window, cagg_hyper_id); return true; } static void set_remainder_after_cut(Invalidation *remainder, int32 hyper_id, int64 lowest_modified_value, int64 greatest_modified_value) { MemSet(remainder, 0, sizeof(*remainder)); remainder->hyper_id = hyper_id; remainder->lowest_modified_value = lowest_modified_value; remainder->greatest_modified_value = greatest_modified_value; } /* * Try to cut an invalidation against the refresh window. * * If an invalidation entry overlaps with the refresh window, it needs * additional processing: it is either cut, deleted, or left unmodified. * * The part(s) of the invalidation that are outside the refresh window after * the cut will remain in the log. The part of the invalidation that fits * within the window is returned as the "remainder". * * Note that the refresh window is exclusive in the end while invalidations * are inclusive. */ static InvalidationResult cut_invalidation_along_refresh_window(const ContinuousAggInvalidationState *state, const Invalidation *invalidation, const InternalTimeRange *refresh_window, Invalidation *remainder) { int32 cagg_hyper_id = state->cagg->data.mat_hypertable_id; TupleDesc tupdesc = RelationGetDescr(state->cagg_log_rel); InvalidationResult result = INVAL_NOMATCH; HeapTuple lower = NULL; HeapTuple upper = NULL; Assert(remainder != NULL); /* Entry is completely enclosed by the refresh window */ if (invalidation->lowest_modified_value >= refresh_window->start && invalidation->greatest_modified_value < refresh_window->end) { /* * Entry completely enclosed so can be deleted: * * [---------------) * [+++++] */ result = INVAL_DELETE; set_remainder_after_cut(remainder, cagg_hyper_id, invalidation->lowest_modified_value, invalidation->greatest_modified_value); } else { if (invalidation->lowest_modified_value < refresh_window->start && invalidation->greatest_modified_value >= refresh_window->start) { /* * Need to cut in right end: * * [------) * [++++++] * * [++] */ lower = create_invalidation_tup(tupdesc, cagg_hyper_id, invalidation->lowest_modified_value, refresh_window->start - 1); set_remainder_after_cut(remainder, cagg_hyper_id, refresh_window->start, /* Refresh window not exclusive at end */ MIN(refresh_window->end - 1, invalidation->greatest_modified_value)); result = INVAL_CUT; } if (invalidation->lowest_modified_value < refresh_window->end && invalidation->greatest_modified_value >= refresh_window->end) { /* * If the invalidation is already cut on the left above, the reminder is set and * will be reset here. The assert prevents from losing information from the reminder. */ Assert((result == INVAL_CUT && remainder->lowest_modified_value == refresh_window->start) || result == INVAL_NOMATCH); /* * Need to cut in left end: * * [------) * [++++++++] * * [++++] */ upper = create_invalidation_tup(tupdesc, cagg_hyper_id, refresh_window->end, invalidation->greatest_modified_value); set_remainder_after_cut(remainder, cagg_hyper_id, MAX(invalidation->lowest_modified_value, refresh_window->start), /* Refresh window exclusive at end */ refresh_window->end - 1); result = INVAL_CUT; } } /* Insert any modifications into the cagg invalidation log */ if (result == INVAL_CUT) { CatalogSecurityContext sec_ctx; HeapTuple other_range = NULL; ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); /* We'd like to do one update (unless the TID is not set), and * optionally one insert. We pick one of the tuples for an update, and * the other one will be an insert. */ if (lower || upper) { HeapTuple tup = lower ? lower : upper; other_range = lower ? upper : lower; /* If the TID is set, we are updating an existing tuple, i.e., we * are processing and entry in the cagg log itself. Otherwise, we * are processing the hypertable invalidation log and need to * insert a new entry. */ if (ItemPointerIsValid(&invalidation->tid)) { ItemPointerData tid = invalidation->tid; ts_catalog_update_tid_only(state->cagg_log_rel, &tid, tup); } else ts_catalog_insert_only(state->cagg_log_rel, tup); heap_freetuple(tup); } if (other_range) { ts_catalog_insert_only(state->cagg_log_rel, other_range); heap_freetuple(other_range); } ts_catalog_restore_user(&sec_ctx); } return result; } static void invalidation_entry_reset(Invalidation *entry) { MemSet(entry, 0, sizeof(Invalidation)); } /* * Expand an invalidation to bucket boundaries. * * Since a refresh always materializes full buckets, we can safely expand an * invalidation to bucket boundaries and in the process merge a lot more * invalidations. */ void invalidation_expand_to_bucket_boundaries(Invalidation *inv, Oid time_type_oid, const ContinuousAggBucketFunction *bucket_function) { const int64 time_dimension_min = ts_time_get_min(time_type_oid); const int64 time_dimension_max = ts_time_get_max(time_type_oid); int64 min_bucket_start; int64 max_bucket_end; if (bucket_function->bucket_fixed_interval == false) { ts_compute_circumscribed_bucketed_refresh_window_variable(&inv->lowest_modified_value, &inv->greatest_modified_value, bucket_function); /* ts_compute_circumscribed_bucketed_refresh_window_variable returns the start of the * next bucket as the end (exclusive). Since invalidations are inclusive at both ends, * subtract 1 to get the last value of the current bucket (inclusive). * Don't adjust infinity values. */ if (inv->greatest_modified_value != INVAL_POS_INFINITY && inv->greatest_modified_value != INVAL_NEG_INFINITY) { inv->greatest_modified_value = int64_saturating_sub(inv->greatest_modified_value, 1); } return; } int64 bucket_width = ts_continuous_agg_fixed_bucket_width(bucket_function); Assert(bucket_width > 0); NullableDatum offset = INIT_NULL_DATUM; NullableDatum origin = INIT_NULL_DATUM; fill_bucket_offset_origin(bucket_function, time_type_oid, &offset, &origin); /* Compute the start of the "first" bucket for the type. The min value * must be at the start of the "first" bucket or somewhere in the * bucket. If the min value falls on the exact start of the bucket we are * good. Otherwise, we need to move to the next full bucket. */ min_bucket_start = ts_time_saturating_add(time_dimension_min, bucket_width - 1, time_type_oid); min_bucket_start = ts_time_bucket_by_type(bucket_width, min_bucket_start, time_type_oid); /* Compute the end of the "last" bucket for the time type. Remember that * invalidations are inclusive, so the "greatest" value should be the last * value of the last full bucket. Either the max value is already the last * value of the last bucket, or we need to return the last value of the * previous full bucket. */ max_bucket_end = ts_time_bucket_by_type(bucket_width, time_dimension_max, time_type_oid); /* Check if the max value was already the last value of the last bucket */ if (ts_time_saturating_add(max_bucket_end, bucket_width - 1, time_type_oid) == time_dimension_max) { max_bucket_end = time_dimension_max; } else { /* The last bucket was partial. To get the end of previous bucket, we * need to move one step down from the partial last bucket. */ max_bucket_end = ts_time_saturating_sub(max_bucket_end, 1, time_type_oid); } if (inv->lowest_modified_value < min_bucket_start) /* Below the min bucket, so treat as invalid to -infinity. */ inv->lowest_modified_value = INVAL_NEG_INFINITY; else if (inv->lowest_modified_value > max_bucket_end) /* Above the max bucket, so treat as invalid to +infinity. */ inv->lowest_modified_value = INVAL_POS_INFINITY; else inv->lowest_modified_value = ts_time_bucket_by_type_extended(bucket_width, inv->lowest_modified_value, time_type_oid, offset, origin); if (inv->greatest_modified_value < min_bucket_start) /* Below the min bucket, so treat as invalid to -infinity. */ inv->greatest_modified_value = INVAL_NEG_INFINITY; else if (inv->greatest_modified_value > max_bucket_end) /* Above the max bucket, so treat as invalid to +infinity. */ inv->greatest_modified_value = INVAL_POS_INFINITY; else { inv->greatest_modified_value = ts_time_bucket_by_type_extended(bucket_width, inv->greatest_modified_value, time_type_oid, offset, origin); inv->greatest_modified_value = ts_time_saturating_add(inv->greatest_modified_value, bucket_width - 1, time_type_oid); } } /* * Macro to set an Invalidation from a tuple. The tuple can either have the * format of the hypertable invalidation log or the continuous aggregate * invalidation log (as determined by the type parameter). */ #define INVALIDATION_ENTRY_SET(entry, ti, hypertable_id, type) \ do \ { \ bool should_free; \ HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); \ type form; \ form = (type) GETSTRUCT(tuple); \ (entry)->hyper_id = form->hypertable_id; \ (entry)->lowest_modified_value = form->lowest_modified_value; \ (entry)->greatest_modified_value = form->greatest_modified_value; \ (entry)->is_modified = false; \ ItemPointerCopy(&tuple->t_self, &(entry)->tid); \ \ if (should_free) \ heap_freetuple(tuple); \ } while (0); static void invalidation_entry_set_from_hyper_invalidation(Invalidation *entry, const TupleInfo *ti, int32 hyper_id, Oid dimtype, const ContinuousAggBucketFunction *bucket_function) { INVALIDATION_ENTRY_SET(entry, ti, hypertable_id, Form_continuous_aggs_hypertable_invalidation_log); /* Since hypertable invalidations are moved to the continuous aggregate * invalidation log, a different hypertable ID must be set (the ID of the * materialized hypertable). */ entry->hyper_id = hyper_id; invalidation_expand_to_bucket_boundaries(entry, dimtype, bucket_function); } static void invalidation_entry_set_from_cagg_invalidation(Invalidation *entry, const TupleInfo *ti, Oid dimtype, const ContinuousAggBucketFunction *bucket_function) { INVALIDATION_ENTRY_SET(entry, ti, materialization_id, Form_continuous_aggs_materialization_invalidation_log); /* It isn't strictly necessary to expand the invalidation to bucket * boundaries here since all invalidations were already expanded when * copied from the hypertable invalidation log. However, since * invalidation expansion wasn't implemented in early 2.0.x versions of * the extension, there might be unexpanded entries in the cagg * invalidation log for some users. Therefore we try to expand * invalidation entries also here, although in most cases it would do * nothing. */ invalidation_expand_to_bucket_boundaries(entry, dimtype, bucket_function); } /* * Check if two invalidations can be merged into one. * * Since invalidations are inclusive in both ends, two adjacent invalidations * can be merged. */ static bool invalidations_can_be_merged(const Invalidation *a, const Invalidation *b) { /* To account for adjacency, expand one window 1 step in each * direction. This makes adjacent invalidations overlapping. */ int64 a_start = int64_saturating_sub(a->lowest_modified_value, 1); int64 a_end = int64_saturating_add(a->greatest_modified_value, 1); return a_end >= b->lowest_modified_value && a_start <= b->greatest_modified_value; } /* * Try to merge two invalidations into one. * * Returns true if the invalidations were merged, otherwise false. * * Given that we scan ordered on lowest_modified_value, the previous and * current invalidation can overlap in two ways (generalized): * * |------| * |++++++++| * * |-------------| * |++++++++| * * The closest non-overlapping case is (note that adjacent invalidations can * be merged since they are inclusive in both ends): * * |--| * |++++++++| * */ static bool invalidation_entry_try_merge(Invalidation *entry, const Invalidation *newentry) { if (!IsValidInvalidation(newentry)) return false; /* Quick exit if no overlap */ if (!invalidations_can_be_merged(entry, newentry)) return false; /* Check if the new entry expands beyond the old one (first case above) */ if (entry->greatest_modified_value < newentry->greatest_modified_value) { entry->greatest_modified_value = newentry->greatest_modified_value; entry->is_modified = true; } return true; } static void insert_new_cagg_invalidation(const HypertableInvalidationState *state, const Invalidation *entry, int32 cagg_hyper_id) { CatalogSecurityContext sec_ctx; TupleDesc tupdesc = RelationGetDescr(state->cagg_log_rel); HeapTuple tuple = create_invalidation_tup(tupdesc, cagg_hyper_id, entry->lowest_modified_value, entry->greatest_modified_value); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_insert_only(state->cagg_log_rel, tuple); ts_catalog_restore_user(&sec_ctx); heap_freetuple(tuple); } /* * Process invalidations in the hypertable invalidation log. * * Copy and delete all entries from the hypertable invalidation log. * * Note that each entry gets one copy per continuous aggregate in the cagg * invalidation log (unless it was merged or matched the refresh * window). These copied entries are later used to track invalidations across * refreshes on a per-cagg basis. * * After this function has run, there are no entries left in the hypertable * invalidation log. */ static void move_invalidations_from_hyper_to_cagg_log(const HypertableInvalidationState *state) { const ContinuousAggInfo *all_caggs = state->all_caggs; int32 hyper_id = state->hypertable_id; int32 last_cagg_hyper_id; ListCell *lc1, *lc2; last_cagg_hyper_id = llast_int(all_caggs->mat_hypertable_ids); /* We use a per-tuple memory context in the scan loop since we could be * processing a lot of invalidations (basically an unbounded * amount). Initialize it here by resetting it. */ MemoryContextReset(state->per_tuple_mctx); /* * Looping over all continuous aggregates in the outer loop ensures all * tuples for a specific continuous aggregate is inserted consecutively in * the cagg invalidation log. This creates better locality for scanning * the invalidations later. */ forboth (lc1, all_caggs->mat_hypertable_ids, lc2, all_caggs->bucket_functions) { int32 cagg_hyper_id = lfirst_int(lc1); const ContinuousAggBucketFunction *bucket_function = lfirst(lc2); Invalidation mergedentry; ScanIterator iterator; invalidation_entry_reset(&mergedentry); hypertable_invalidation_scan_init(&iterator, hyper_id, RowExclusiveLock); iterator.ctx.snapshot = state->snapshot; /* Scan all invalidations */ ts_scanner_foreach(&iterator) { TupleInfo *ti; MemoryContext oldmctx; Invalidation logentry; oldmctx = MemoryContextSwitchTo(state->per_tuple_mctx); ti = ts_scan_iterator_tuple_info(&iterator); invalidation_entry_set_from_hyper_invalidation(&logentry, ti, cagg_hyper_id, state->dimtype, bucket_function); if (!IsValidInvalidation(&mergedentry)) { mergedentry = logentry; mergedentry.hyper_id = cagg_hyper_id; } else if (!invalidation_entry_try_merge(&mergedentry, &logentry)) { insert_new_cagg_invalidation(state, &mergedentry, cagg_hyper_id); mergedentry = logentry; } if (cagg_hyper_id == last_cagg_hyper_id) { CatalogSecurityContext sec_ctx; /* The invalidation has been processed for all caggs, so the * only thing left is to delete it from the source hypertable * invalidation log. */ ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_delete_tid_only(ti->scanrel, &logentry.tid); ts_catalog_restore_user(&sec_ctx); } MemoryContextSwitchTo(oldmctx); MemoryContextReset(state->per_tuple_mctx); } ts_scan_iterator_close(&iterator); /* Handle the last merged invalidation */ if (IsValidInvalidation(&mergedentry)) insert_new_cagg_invalidation(state, &mergedentry, cagg_hyper_id); } } static void cagg_invalidations_scan_by_hypertable_init(ScanIterator *iterator, int32 cagg_hyper_id, LOCKMODE lockmode) { *iterator = ts_scan_iterator_create(CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG, lockmode, CurrentMemoryContext); iterator->ctx.index = catalog_get_index(ts_catalog_get(), CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG, CONTINUOUS_AGGS_MATERIALIZATION_INVALIDATION_LOG_IDX); ts_scan_iterator_scan_key_init( iterator, Anum_continuous_aggs_materialization_invalidation_log_idx_materialization_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(cagg_hyper_id)); } /* * Cut an invalidation and return the part, if any, that remains within the * refresh window. */ static Invalidation cut_cagg_invalidation(const ContinuousAggInvalidationState *state, const InternalTimeRange *refresh_window, const Invalidation *entry) { InvalidationResult result; Invalidation remainder; ItemPointerData tid = entry->tid; invalidation_entry_reset(&remainder); result = cut_invalidation_along_refresh_window(state, entry, refresh_window, &remainder); switch (result) { case INVAL_NOMATCH: /* If no cutting was done (i.e., the invalidation was outside the * refresh window), but the invalidation was previously merged * (expanded) with another invalidation, then we still need to * update it. */ if (entry->is_modified) { HeapTuple tuple = create_invalidation_tup(RelationGetDescr(state->cagg_log_rel), entry->hyper_id, entry->lowest_modified_value, entry->greatest_modified_value); ts_catalog_update_tid_only(state->cagg_log_rel, &tid, tuple); heap_freetuple(tuple); } break; case INVAL_DELETE: ts_catalog_delete_tid_only(state->cagg_log_rel, &tid); break; case INVAL_CUT: /* Nothing to do */ break; } return remainder; } static Invalidation cut_cagg_invalidation_and_compute_remainder(const ContinuousAggInvalidationState *state, const InternalTimeRange *refresh_window, const Invalidation *mergedentry, const Invalidation *current_remainder) { Invalidation new_remainder; Invalidation remainder = *current_remainder; /* The previous and current invalidation could not be merged. We * need to cut the prev invalidation against the refresh window */ new_remainder = cut_cagg_invalidation(state, refresh_window, mergedentry); if (!IsValidInvalidation(&remainder)) remainder = new_remainder; else if (IsValidInvalidation(&new_remainder) && !invalidation_entry_try_merge(&remainder, &new_remainder)) { save_invalidation_for_refresh(state, &remainder); remainder = new_remainder; } return remainder; } /* * Clear all cagg invalidations that match a refresh window. * * This function clears all invalidations in the cagg invalidation log that * matches a window, and adds the invalidation segments covered by the window * to the invalidation store (tuple store) in the state argument. The * remaining segments that are added to the invalidation store are regions * that require materialization. * * An invalidation entry that gets processed is either completely enclosed * (covered) by the refresh window, or it partially overlaps. In the former * case, the invalidation entry is removed and for the latter case it is * cut. Thus, an entry can either disappear, reduce in size, or be cut in two. * * Note that the refresh window is inclusive at the start and exclusive at the * end. This function also assumes that invalidations are scanned in order of * lowest_modified_value. */ static void clear_cagg_invalidations_for_refresh(const ContinuousAggInvalidationState *state, const InternalTimeRange *refresh_window, bool force) { ScanIterator iterator; Invalidation mergedentry; Invalidation remainder; invalidation_entry_reset(&mergedentry); invalidation_entry_reset(&remainder); cagg_invalidations_scan_by_hypertable_init(&iterator, state->cagg->data.mat_hypertable_id, RowExclusiveLock); iterator.ctx.data = (void *) &state; iterator.ctx.snapshot = state->snapshot; ScanTupLock scantuplock = { .waitpolicy = LockWaitBlock, .lockmode = LockTupleExclusive, .lockflags = TUPLE_LOCK_FLAG_FIND_LAST_VERSION, }; iterator.ctx.tuplock = &scantuplock; iterator.ctx.flags = SCANNER_F_KEEPLOCK; MemoryContextReset(state->per_tuple_mctx); /* * Force refresh within the entire window. * * At this point the refresh window has already been inscribed to bucket * boundaries by the caller, so [start, end) covers exactly the set of * buckets to materialize. * * Synthesize an invalidation covering [start, end-1] (inclusive) and use * it as the initial remainder. We use end-1 because greatest_modified_value * is inclusive while refresh_window->end is exclusive. * * By seeding the remainder with this forced entry, any cagg invalidation * log entries whose inside parts overlap the window will be merged into it * in the scan loop below rather than being saved as separate entries. * The single merged remainder is then saved once at the end of this function. */ if (force) { remainder.hyper_id = state->cagg->data.mat_hypertable_id; remainder.lowest_modified_value = refresh_window->start; remainder.greatest_modified_value = ts_time_saturating_sub(refresh_window->end, 1, refresh_window->type); remainder.is_modified = false; ItemPointerSetInvalid(&remainder.tid); } /* Process all invalidations for the continuous aggregate */ ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); MemoryContext oldmctx; Invalidation logentry; oldmctx = MemoryContextSwitchTo(state->per_tuple_mctx); invalidation_entry_set_from_cagg_invalidation(&logentry, ti, state->cagg->partition_type, state->cagg->bucket_function); if (!IsValidInvalidation(&mergedentry)) mergedentry = logentry; else if (invalidation_entry_try_merge(&mergedentry, &logentry)) { /* * The previous and current invalidation were merged into * one entry (i.e., they overlapped or were adjacent). */ ts_catalog_delete_tid_only(state->cagg_log_rel, &logentry.tid); } else { remainder = cut_cagg_invalidation_and_compute_remainder(state, refresh_window, &mergedentry, &remainder); mergedentry = logentry; } MemoryContextSwitchTo(oldmctx); MemoryContextReset(state->per_tuple_mctx); } ts_scan_iterator_close(&iterator); /* Handle the last (merged) invalidation */ if (IsValidInvalidation(&mergedentry)) remainder = cut_cagg_invalidation_and_compute_remainder(state, refresh_window, &mergedentry, &remainder); /* Handle the last (merged) remainder */ save_invalidation_for_refresh(state, &remainder); } static void cagg_invalidation_state_init(ContinuousAggInvalidationState *state, const ContinuousAgg *cagg) { state->cagg = cagg; state->cagg_log_rel = open_cagg_table(CAGG_INVALIDATION_LOG, RowExclusiveLock); state->cagg_queue_rel = open_cagg_table(CAGG_MATERIALIZATION_RANGES, RowExclusiveLock); state->per_tuple_mctx = AllocSetContextCreate(CurrentMemoryContext, "Materialization invalidations", ALLOCSET_DEFAULT_SIZES); state->snapshot = RegisterSnapshot(GetTransactionSnapshot()); } static void cagg_invalidation_state_cleanup(const ContinuousAggInvalidationState *state) { table_close(state->cagg_log_rel, NoLock); table_close(state->cagg_queue_rel, NoLock); UnregisterSnapshot(state->snapshot); MemoryContextDelete(state->per_tuple_mctx); } static void hypertable_invalidation_state_init(HypertableInvalidationState *state, int32 hypertable_id, Oid dimtype, const ContinuousAggInfo *all_caggs) { state->hypertable_id = hypertable_id; state->dimtype = dimtype; state->all_caggs = all_caggs; state->cagg_log_rel = open_cagg_table(CAGG_INVALIDATION_LOG, RowExclusiveLock); state->per_tuple_mctx = AllocSetContextCreate(CurrentMemoryContext, "Hypertable invalidations", ALLOCSET_DEFAULT_SIZES); state->snapshot = RegisterSnapshot(GetTransactionSnapshot()); } static void hypertable_invalidation_state_cleanup(const HypertableInvalidationState *state) { table_close(state->cagg_log_rel, NoLock); UnregisterSnapshot(state->snapshot); MemoryContextDelete(state->per_tuple_mctx); } /* * Move invalidations for a single hypertable from hypertable invalidation log * to materialization invalidation log. This will move *all* hypertable * invalidations for the hypertable to the associated continuous aggregates. */ void invalidation_process_hypertable_log(int32 hypertable_id, Oid dimtype) { HypertableInvalidationState state; const ContinuousAggInfo all_caggs = ts_continuous_agg_get_all_caggs_info(hypertable_id); hypertable_invalidation_state_init(&state, hypertable_id, dimtype, &all_caggs); move_invalidations_from_hyper_to_cagg_log(&state); hypertable_invalidation_state_cleanup(&state); } InvalidationStore * invalidation_process_cagg_log(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, long max_materializations, ContinuousAggRefreshContext context, bool force) { ContinuousAggInvalidationState state; InvalidationStore *store = NULL; long count; cagg_invalidation_state_init(&state, cagg); state.invalidations = tuplestore_begin_heap(false, false, work_mem); clear_cagg_invalidations_for_refresh(&state, refresh_window, force); count = tuplestore_tuple_count(state.invalidations); if (count == 0) { tuplestore_end(state.invalidations); } else { store = palloc(sizeof(InvalidationStore)); store->tupstore = state.invalidations; store->tupdesc = CreateTupleDescCopy(RelationGetDescr(state.cagg_log_rel)); } cagg_invalidation_state_cleanup(&state); return store; } void invalidation_store_free(InvalidationStore *store) { FreeTupleDesc(store->tupdesc); tuplestore_end(store->tupstore); pfree(store); } ================================================ FILE: tsl/src/continuous_aggs/invalidation.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include "continuous_aggs/materialize.h" #include "ts_catalog/continuous_agg.h" /* * Invalidation. * * A common representation of an invalidation that works across both the * hypertable invalidation log and the continuous aggregate invalidation log. */ typedef struct Invalidation { int32 hyper_id; int64 lowest_modified_value; int64 greatest_modified_value; bool is_modified; ItemPointerData tid; } Invalidation; #define INVAL_NEG_INFINITY PG_INT64_MIN #define INVAL_POS_INFINITY PG_INT64_MAX typedef struct InvalidationStore { Tuplestorestate *tupstore; TupleDesc tupdesc; } InvalidationStore; typedef struct Hypertable Hypertable; extern void invalidation_cagg_log_add_entry(int32 cagg_hyper_id, int64 start, int64 end); extern void invalidation_hyper_log_add_entry(int32 hyper_id, int64 start, int64 end); extern void continuous_agg_invalidate_raw_ht(const Hypertable *raw_ht, int64 start, int64 end); extern void continuous_agg_invalidate_mat_ht(const Hypertable *raw_ht, const Hypertable *mat_ht, int64 start, int64 end); extern Datum continuous_agg_process_hypertable_invalidations(PG_FUNCTION_ARGS); extern void invalidation_process_hypertable_log(int32 hypertable_id, Oid dimtype); extern InvalidationStore *invalidation_process_cagg_log(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, long max_materializations, ContinuousAggRefreshContext context, bool force); extern void invalidation_store_free(InvalidationStore *store); extern void invalidation_expand_to_bucket_boundaries(Invalidation *inv, Oid time_type_oid, const ContinuousAggBucketFunction *bucket_function); extern HeapTuple create_invalidation_tup(const TupleDesc tupdesc, int32 cagg_hyper_id, int64 start, int64 end); ================================================ FILE: tsl/src/continuous_aggs/invalidation_threshold.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/htup.h> #include <access/htup_details.h> #include <access/tableam.h> #include <access/xact.h> #include <nodes/memnodes.h> #include <storage/lmgr.h> #include <storage/lockdefs.h> #include <utils/builtins.h> #include <utils/memutils.h> #include <utils/snapmgr.h> #include <compat/compat.h> #include "ts_catalog/catalog.h" #include <scan_iterator.h> #include <scanner.h> #include <time_bucket.h> #include <time_utils.h> #include "continuous_aggs/materialize.h" #include "continuous_aggs/refresh.h" #include "debug_point.h" #include "invalidation_threshold.h" #include "ts_catalog/continuous_agg.h" #include <utils.h> /* * Invalidation threshold. * * The invalidation threshold acts as a dampener on a hypertable to make sure * that invalidations written during inserts won't cause too much write * amplification in "hot" regions---typically the "head" of the table. The * presumption is that most inserts happen at recent time intervals, and those * intervals will be invalid until writes move out of them. Therefore, it * isn't worth writing invalidations in that region since it is presumed * out-of-date anyway. Further, although it is possible to refresh a * continuous aggregate in those "hot" regions, it will lead to partially * filled buckets. Thus, refreshing those intervals is discouraged since the * aggregate will be immediately out-of-date until the buckets are filled. The * invalidation threshold is, in other words, used as a marker that lags * behind the head of the hypertable, where invalidations are written before * the threshold but not after it. * * The invalidation threshold is moved forward (and only forward) by refreshes * on continuous aggregates when it covers a window that stretches beyond the * current threshold. The invalidation threshold needs to be moved in its own * transaction, with exclusive access, before the refresh starts to * materialize data. This is to avoid losing any invalidations that occur * between the start of the transaction that moves the threshold and its end * (when the new threshold becomes visible). * * ______________________________________________ * |_______________________________________|_____| recent data * ^ * invalidations written here | no invalidations * | * invalidation threshold * * Transactions that use an isolation level stronger than READ COMMITTED will * not be able to "see" changes to the invalidation threshold that may have * been made while they were running. Therefore, they always create records * in the hypertable invalidation log. See the cache_inval_entry_write() * implementation in tsl/src/continuous_aggs/insert.c */ typedef struct InvalidationThresholdData { const ContinuousAgg *cagg; const InternalTimeRange *refresh_window; int64 computed_invalidation_threshold; } InvalidationThresholdData; typedef struct InvalidationThresholdGetData { int32 hypertable_id; int64 threshold; } InvalidationThresholdGetData; static ScanTupleResult invalidation_threshold_scan_get(TupleInfo *ti, void *const data) { InvalidationThresholdGetData *the_data = (InvalidationThresholdGetData *) data; bool isnull; Datum datum; if (ti->lockresult == TM_Updated || ti->lockresult == TM_Deleted) return SCAN_RESTART_WITH_NEW_SNAPSHOT; Ensure(ti->lockresult == TM_Ok, "unable to lock invalidation threshold tuple for hypertable %d (lock result %d)", the_data->hypertable_id, ti->lockresult); datum = slot_getattr(ti->slot, Anum_continuous_aggs_invalidation_threshold_watermark, &isnull); Ensure(!isnull, "invalidation threshold for hypertable %d is null", the_data->hypertable_id); the_data->threshold = DatumGetInt64(datum); return SCAN_DONE; } static ScanTupleResult invalidation_threshold_scan_update(TupleInfo *ti, void *const data) { DEBUG_WAITPOINT("invalidation_threshold_scan_update_enter"); InvalidationThresholdData *invthresh = (InvalidationThresholdData *) data; /* If the tuple was modified concurrently, retry the operation and use a new snapshot * to see the updated tuple. */ if (ti->lockresult == TM_Updated) return SCAN_RESTART_WITH_NEW_SNAPSHOT; if (ti->lockresult != TM_Ok) { elog(ERROR, "unable to lock invalidation threshold tuple for hypertable %d (lock result %d)", invthresh->cagg->data.raw_hypertable_id, ti->lockresult); pg_unreachable(); } bool isnull; Datum datum = slot_getattr(ti->slot, Anum_continuous_aggs_invalidation_threshold_watermark, &isnull); /* NULL should never happen because we always initialize the threshold with the MIN * value of the partition type */ Ensure(!isnull, "invalidation threshold for hypertable %d is null", invthresh->cagg->data.raw_hypertable_id); int64 current_invalidation_threshold = DatumGetInt64(datum); /* Compute new invalidation threshold. Note that this computation caps the * threshold at the end of the last bucket that holds data in the * underlying hypertable. */ invthresh->computed_invalidation_threshold = invalidation_threshold_compute(invthresh->cagg, invthresh->refresh_window); if (invthresh->computed_invalidation_threshold > current_invalidation_threshold) { bool nulls[Natts_continuous_agg]; Datum values[Natts_continuous_agg]; bool do_replace[Natts_continuous_agg] = { false }; bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); HeapTuple new_tuple; TupleDesc tupdesc = ts_scanner_get_tupledesc(ti); heap_deform_tuple(tuple, tupdesc, values, nulls); do_replace[AttrNumberGetAttrOffset(Anum_continuous_aggs_invalidation_threshold_watermark)] = true; values[AttrNumberGetAttrOffset(Anum_continuous_aggs_invalidation_threshold_watermark)] = Int64GetDatum(invthresh->computed_invalidation_threshold); new_tuple = heap_modify_tuple(tuple, tupdesc, values, nulls, do_replace); ts_catalog_update(ti->scanrel, new_tuple); heap_freetuple(new_tuple); if (should_free) heap_freetuple(tuple); } else { elog(DEBUG1, "hypertable %d existing watermark >= new invalidation threshold " INT64_FORMAT " " INT64_FORMAT, invthresh->cagg->data.raw_hypertable_id, current_invalidation_threshold, invthresh->computed_invalidation_threshold); invthresh->computed_invalidation_threshold = current_invalidation_threshold; } return SCAN_CONTINUE; } /* * Get the invalidation threshold for the hypertable. * * This will also lock the row. */ int64 invalidation_threshold_get(int32 hypertable_id) { InvalidationThresholdGetData data = { .hypertable_id = hypertable_id }; ScanKeyData scankey[1]; Catalog *catalog = ts_catalog_get(); ScanTupLock scantuplock = { .waitpolicy = LockWaitBlock, .lockmode = LockTupleExclusive, }; PushActiveSnapshot(GetLatestSnapshot()); ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, CONTINUOUS_AGGS_INVALIDATION_THRESHOLD), .index = catalog_get_index(catalog, CONTINUOUS_AGGS_INVALIDATION_THRESHOLD, CONTINUOUS_AGGS_INVALIDATION_THRESHOLD_PKEY), .nkeys = 1, .scankey = scankey, .data = &data, .tuple_found = invalidation_threshold_scan_get, .lockmode = RowShareLock, .scandirection = ForwardScanDirection, .result_mctx = CurrentMemoryContext, .tuplock = &scantuplock, .flags = SCANNER_F_KEEPLOCK, .snapshot = GetActiveSnapshot(), }; ScanKeyInit(&scankey[0], Anum_continuous_aggs_invalidation_threshold_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); bool found = ts_scanner_scan_one(&scanctx, false, CAGG_INVALIDATION_THRESHOLD_NAME); Ensure(found, "invalidation threshold for hypertable %d not found", hypertable_id); PopActiveSnapshot(); return data.threshold; } /* * Set a new invalidation threshold. * * The threshold is only updated if the new threshold is greater than the old * one. * * On success, the new threshold is returned, otherwise the existing threshold * is returned instead. */ int64 invalidation_threshold_set_or_get(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window) { bool found = false; ScanKeyData scankey[1]; Catalog *catalog = ts_catalog_get(); ScanTupLock scantuplock = { .waitpolicy = LockWaitBlock, .lockmode = LockTupleExclusive, }; InvalidationThresholdData updatectx = { .cagg = cagg, .refresh_window = refresh_window, }; PushActiveSnapshot(GetLatestSnapshot()); ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, CONTINUOUS_AGGS_INVALIDATION_THRESHOLD), .index = catalog_get_index(catalog, CONTINUOUS_AGGS_INVALIDATION_THRESHOLD, CONTINUOUS_AGGS_INVALIDATION_THRESHOLD_PKEY), .nkeys = 1, .scankey = scankey, .data = &updatectx, .tuple_found = invalidation_threshold_scan_update, .lockmode = RowExclusiveLock, .scandirection = ForwardScanDirection, .result_mctx = CurrentMemoryContext, .tuplock = &scantuplock, .flags = SCANNER_F_KEEPLOCK, /* We update the threshold value using this scanner. Since the scanner uses SnapshotSelf * per default, the updated tuple would become immediately visible to the scanner (the * snapshot includes "changes made by the current command") and ts_scanner_scan_one() * would fail due to the second found tuple. A normal MVCC snapshot is used to prevent * the update is immediately seen by the scanner. */ .snapshot = GetActiveSnapshot(), }; ScanKeyInit(&scankey[0], Anum_continuous_aggs_invalidation_threshold_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(cagg->data.raw_hypertable_id)); found = ts_scanner_scan_one(&scanctx, false, CAGG_INVALIDATION_THRESHOLD_NAME); Ensure(found, "invalidation threshold for hypertable %d not found", cagg->data.raw_hypertable_id); PopActiveSnapshot(); return updatectx.computed_invalidation_threshold; } /* * Compute a new invalidation threshold. * * The new invalidation threshold returned is the end of the given refresh * window, unless it ends at "infinity" in which case the threshold is capped * at the end of the last bucket materialized. */ int64 invalidation_threshold_compute(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window) { bool max_refresh = false; Hypertable *ht = ts_hypertable_get_by_id(cagg->data.raw_hypertable_id); if (IS_TIMESTAMP_TYPE(refresh_window->type)) max_refresh = TS_TIME_IS_END(refresh_window->end, refresh_window->type) || TS_TIME_IS_NOEND(refresh_window->end, refresh_window->type); else max_refresh = TS_TIME_IS_MAX(refresh_window->end, refresh_window->type); if (max_refresh) { bool isnull; int64 maxval = ts_hypertable_get_open_dim_max_value(ht, 0, &isnull); if (isnull) { /* No data in hypertable */ return cagg_get_time_min(cagg); } else { if (cagg->bucket_function->bucket_fixed_interval == false) { return ts_compute_beginning_of_the_next_bucket_variable(maxval, cagg->bucket_function); } int64 bucket_width = ts_continuous_agg_fixed_bucket_width(cagg->bucket_function); Assert(bucket_width > 0); NullableDatum offset = INIT_NULL_DATUM; NullableDatum origin = INIT_NULL_DATUM; fill_bucket_offset_origin(cagg->bucket_function, refresh_window->type, &offset, &origin); int64 bucket_start = ts_time_bucket_by_type_extended(bucket_width, maxval, refresh_window->type, offset, origin); /* Add one bucket to get to the end of the last bucket */ return ts_time_saturating_add(bucket_start, bucket_width, refresh_window->type); } } return refresh_window->end; } /* * Initialize the invalidation threshold. * * The initial value of the invalidation threshold should be the MIN * value for the Continuous Aggregate partition type. */ void invalidation_threshold_initialize(const ContinuousAgg *cagg) { bool found = false; ScanKeyData scankey[1]; Catalog *catalog = ts_catalog_get(); ScannerCtx scanctx = { .table = catalog_get_table_id(catalog, CONTINUOUS_AGGS_INVALIDATION_THRESHOLD), .index = catalog_get_index(catalog, CONTINUOUS_AGGS_INVALIDATION_THRESHOLD, CONTINUOUS_AGGS_INVALIDATION_THRESHOLD_PKEY), .nkeys = 1, .scankey = scankey, .lockmode = ShareUpdateExclusiveLock, .scandirection = ForwardScanDirection, .result_mctx = CurrentMemoryContext, .flags = SCANNER_F_KEEPLOCK, }; ScanKeyInit(&scankey[0], Anum_continuous_aggs_invalidation_threshold_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(cagg->data.raw_hypertable_id)); found = ts_scanner_scan_one(&scanctx, false, CAGG_INVALIDATION_THRESHOLD_NAME); if (!found) { Relation rel = table_open(catalog_get_table_id(catalog, CONTINUOUS_AGGS_INVALIDATION_THRESHOLD), ShareUpdateExclusiveLock); TupleDesc desc = RelationGetDescr(rel); Datum values[Natts_continuous_aggs_invalidation_threshold]; bool nulls[Natts_continuous_aggs_invalidation_threshold] = { false }; CatalogSecurityContext sec_ctx; /* get the MIN value for the partition type */ int64 min_value = cagg_get_time_min(cagg); values[AttrNumberGetAttrOffset(Anum_continuous_aggs_invalidation_threshold_hypertable_id)] = Int32GetDatum(cagg->data.raw_hypertable_id); values[AttrNumberGetAttrOffset(Anum_continuous_aggs_invalidation_threshold_watermark)] = Int64GetDatum(min_value); ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); ts_catalog_insert_values(rel, desc, values, nulls); ts_catalog_restore_user(&sec_ctx); table_close(rel, NoLock); } } ================================================ FILE: tsl/src/continuous_aggs/invalidation_threshold.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> typedef struct InternalTimeRange InternalTimeRange; typedef struct ContinuousAgg ContinuousAgg; typedef struct Hypertable Hypertable; extern int64 invalidation_threshold_get(int32 hypertable_id); extern int64 invalidation_threshold_set_or_get(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window); extern int64 invalidation_threshold_compute(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window); extern void invalidation_threshold_initialize(const ContinuousAgg *cagg); ================================================ FILE: tsl/src/continuous_aggs/materialize.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <executor/spi.h> #include <fmgr.h> #include <lib/stringinfo.h> #include <utils/builtins.h> #include <utils/date.h> #include <utils/guc.h> #include <utils/palloc.h> #include <utils/rel.h> #include <utils/relcache.h> #include <utils/snapmgr.h> #include <utils/timestamp.h> #include "compat/compat.h" #include "debug_assert.h" #include "guc.h" #include "materialize.h" #include "scan_iterator.h" #include "scanner.h" #include "time_utils.h" #include "ts_catalog/array_utils.h" #include "ts_catalog/compression_settings.h" #include "ts_catalog/continuous_agg.h" #include "ts_catalog/continuous_aggs_watermark.h" /********************* * utility functions * *********************/ static TimeRange internal_time_range_to_time_range(InternalTimeRange internal); static Datum internal_to_time_value_or_infinite(int64 internal, Oid time_type, bool *is_infinite_out); static List *cagg_find_aggref_and_var_cols(ContinuousAgg *cagg, Hypertable *mat_ht); static char *build_merge_insert_columns(List *strings, const char *separator, const char *prefix); static char *build_merge_join_clause(List *column_names); static char *build_merge_update_clause(List *column_names); /*************************** * materialization support * ***************************/ typedef enum MaterializationPlanType { PLAN_TYPE_INSERT, PLAN_TYPE_DELETE, PLAN_TYPE_EXISTS, PLAN_TYPE_MERGE, PLAN_TYPE_MERGE_DELETE, PLAN_TYPE_RANGES_SELECT, PLAN_TYPE_RANGES_DELETE, PLAN_TYPE_RANGES_PENDING, _MAX_MATERIALIZATION_PLAN_TYPES } MaterializationPlanType; typedef struct MaterializationContext { Hypertable *mat_ht; const ContinuousAgg *cagg; SchemaAndName partial_view; SchemaAndName materialization_table; NameData *time_column_name; TimeRange materialization_range; InternalTimeRange internal_materialization_range; ItemPointer tupleid; int nargs; } MaterializationContext; typedef char *(*MaterializationCreateStatement)(MaterializationContext *context); typedef struct MaterializationPlan { SPIPlanPtr plan; bool read_only; bool catalog_security_context; int nargs; MaterializationCreateStatement create_statement; const char *error_message; const char *progress_message; } MaterializationPlan; static char *build_order_by_clause(MaterializationContext *context); static char *create_materialization_insert_statement(MaterializationContext *context); static char *create_materialization_delete_statement(MaterializationContext *context); static char *create_materialization_exists_statement(MaterializationContext *context); static char *create_materialization_merge_statement(MaterializationContext *context); static char *create_materialization_merge_delete_statement(MaterializationContext *context); static char *create_materialization_ranges_select_statement(MaterializationContext *context); static char *create_materialization_ranges_delete_statement(MaterializationContext *context); static char *create_materialization_ranges_pending_statement(MaterializationContext *context); static MaterializationPlan materialization_plans[_MAX_MATERIALIZATION_PLAN_TYPES + 1] = { [PLAN_TYPE_INSERT] = { .nargs = 2, .create_statement = create_materialization_insert_statement, .error_message = "could not insert old values into materialization table \"%s.%s\"", .progress_message = "inserted " UINT64_FORMAT " row(s) into materialization table \"%s.%s\"" }, [PLAN_TYPE_DELETE] = { .nargs = 2, .create_statement = create_materialization_delete_statement, .error_message = "could not delete old values from materialization table \"%s.%s\"", .progress_message = "deleted " UINT64_FORMAT " row(s) from materialization table \"%s.%s\"" }, [PLAN_TYPE_EXISTS] = { .read_only = true, .nargs = 2, .create_statement = create_materialization_exists_statement, .error_message = "could not check the materialization table \"%s.%s\"" }, [PLAN_TYPE_MERGE] = { .nargs = 2, .create_statement = create_materialization_merge_statement, .error_message = "could not merge old values into materialization table \"%s.%s\"", .progress_message = "merged " UINT64_FORMAT " row(s) into materialization table \"%s.%s\"" }, [PLAN_TYPE_MERGE_DELETE] = { .nargs = 2, .create_statement = create_materialization_merge_delete_statement, .error_message = "could not delete old values from " "materialization table \"%s.%s\"", .progress_message = "deleted " UINT64_FORMAT " row(s) from materialization table \"%s.%s\"" }, [PLAN_TYPE_RANGES_SELECT] = { .catalog_security_context = true, .nargs = 3, .create_statement = create_materialization_ranges_select_statement, .error_message = "could not select invalidation entries for " "materialization table \"%s.%s\"" }, [PLAN_TYPE_RANGES_DELETE] = { .catalog_security_context = true, .nargs = 1, .create_statement = create_materialization_ranges_delete_statement, .error_message = "could not delete invalidation entries for " "materialization table \"%s.%s\"" }, [PLAN_TYPE_RANGES_PENDING] = { .read_only = true, .nargs = 3, .create_statement = create_materialization_ranges_pending_statement, .error_message = "could not select pending materialization " "ranges \"%s.%s\"" }, }; static Oid *create_materialization_plan_argtypes(MaterializationContext *context, MaterializationPlanType plan_type, int nargs); static MaterializationPlan *create_materialization_plan(MaterializationContext *context, MaterializationPlanType plan_type); static void create_materialization_plan_args(MaterializationContext *context, MaterializationPlanType plan_type, Datum **values, char **nulls); static uint64 execute_materialization_plan(MaterializationContext *context, MaterializationPlanType plan_type); static void free_materialization_plan(MaterializationContext *context, MaterializationPlanType plan_type); static void free_materialization_plans(MaterializationContext *context); static void update_watermark(MaterializationContext *context); static void execute_materializations(MaterializationContext *context); /* API to update materializations from refresh code */ void continuous_agg_update_materialization(Hypertable *mat_ht, const ContinuousAgg *cagg, SchemaAndName partial_view, SchemaAndName materialization_table, const NameData *time_column_name, InternalTimeRange materialization_range) { MaterializationContext context = { .mat_ht = mat_ht, .cagg = cagg, .partial_view = partial_view, .materialization_table = materialization_table, .time_column_name = (NameData *) time_column_name, .materialization_range = internal_time_range_to_time_range(materialization_range), .internal_materialization_range = materialization_range, }; /* Lock down search_path */ int save_nestlevel = NewGUCNestLevel(); RestrictSearchPath(); /* pin the start of new_materialization to the end of new_materialization, * we are not allowed to materialize beyond that point */ if (materialization_range.start > materialization_range.end) materialization_range.start = materialization_range.end; /* Then insert the materializations */ context.materialization_range = internal_time_range_to_time_range(materialization_range); execute_materializations(&context); /* Restore search_path */ AtEOXact_GUC(false, save_nestlevel); } /* API to check for pending materialization ranges */ bool continuous_agg_has_pending_materializations(const ContinuousAgg *cagg, InternalTimeRange materialization_range) { MaterializationContext context = { .cagg = cagg, .internal_materialization_range = materialization_range, }; /* Lock down search_path */ int save_nestlevel = NewGUCNestLevel(); RestrictSearchPath(); if (materialization_range.start > materialization_range.end) materialization_range.start = materialization_range.end; PushActiveSnapshot(GetLatestSnapshot()); bool has_pending_materializations = (execute_materialization_plan(&context, PLAN_TYPE_RANGES_PENDING) > 0); free_materialization_plan(&context, PLAN_TYPE_RANGES_PENDING); PopActiveSnapshot(); /* Restore search_path */ AtEOXact_GUC(false, save_nestlevel); return has_pending_materializations; } static Datum time_range_internal_to_min_time_value(Oid type) { switch (type) { case TIMESTAMPOID: return TimestampGetDatum(DT_NOBEGIN); case TIMESTAMPTZOID: return TimestampTzGetDatum(DT_NOBEGIN); case DATEOID: return DateADTGetDatum(DATEVAL_NOBEGIN); default: return ts_internal_to_time_value(PG_INT64_MIN, type); } } static Datum time_range_internal_to_max_time_value(Oid type) { switch (type) { case TIMESTAMPOID: return TimestampGetDatum(DT_NOEND); case TIMESTAMPTZOID: return TimestampTzGetDatum(DT_NOEND); case DATEOID: return DateADTGetDatum(DATEVAL_NOEND); break; default: return ts_internal_to_time_value(PG_INT64_MAX, type); } } static Datum internal_to_time_value_or_infinite(int64 internal, Oid time_type, bool *is_infinite_out) { /* MIN and MAX can occur due to NULL thresholds, or due to a lack of invalidations. Since our * regular conversion function errors in those cases, and we want to use those as markers for an * open threshold in one direction, we special case this here*/ if (internal == PG_INT64_MIN) { if (is_infinite_out != NULL) *is_infinite_out = true; return time_range_internal_to_min_time_value(time_type); } else if (internal == PG_INT64_MAX) { if (is_infinite_out != NULL) *is_infinite_out = true; return time_range_internal_to_max_time_value(time_type); } else { if (is_infinite_out != NULL) *is_infinite_out = false; return ts_internal_to_time_value(internal, time_type); } } static TimeRange internal_time_range_to_time_range(InternalTimeRange internal) { TimeRange range; range.type = internal.type; range.start = internal_to_time_value_or_infinite(internal.start, internal.type, NULL); range.end = internal_to_time_value_or_infinite(internal.end, internal.type, NULL); return range; } static List * cagg_find_aggref_and_var_cols(ContinuousAgg *cagg, Hypertable *mat_ht) { List *retlist = NIL; ListCell *lc; Query *cagg_view_query = ts_continuous_agg_get_query(cagg); foreach (lc, cagg_view_query->targetList) { TargetEntry *tle = castNode(TargetEntry, lfirst(lc)); if (!tle->resjunk && (tle->ressortgroupref == 0 || get_sortgroupref_clause_noerr(tle->ressortgroupref, cagg_view_query->groupClause) == NULL)) retlist = lappend(retlist, get_attname(mat_ht->main_table_relid, tle->resno, false)); } return retlist; } static char * build_merge_insert_columns(List *strings, const char *separator, const char *prefix) { StringInfoData ret; initStringInfo(&ret); Assert(strings != NIL); ListCell *lc; foreach (lc, strings) { char *grpcol = (char *) lfirst(lc); if (ret.len > 0) appendStringInfoString(&ret, separator); if (prefix) appendStringInfoString(&ret, prefix); appendStringInfoString(&ret, quote_identifier(grpcol)); } elog(DEBUG2, "%s: %s", __func__, ret.data); return ret.data; } static char * build_merge_join_clause(List *column_names) { StringInfoData ret; initStringInfo(&ret); Assert(column_names != NIL); ListCell *lc; foreach (lc, column_names) { char *column = (char *) lfirst(lc); if (ret.len > 0) appendStringInfoString(&ret, " AND "); appendStringInfoString(&ret, "P."); appendStringInfoString(&ret, quote_identifier(column)); appendStringInfoString(&ret, " IS NOT DISTINCT FROM M."); appendStringInfoString(&ret, quote_identifier(column)); } elog(DEBUG2, "%s: %s", __func__, ret.data); return ret.data; } static char * build_merge_update_clause(List *column_names) { StringInfoData ret; initStringInfo(&ret); Assert(column_names != NIL); ListCell *lc; foreach (lc, column_names) { char *column = (char *) lfirst(lc); if (ret.len > 0) appendStringInfoString(&ret, ", "); appendStringInfoString(&ret, quote_identifier(column)); appendStringInfoString(&ret, " = P."); appendStringInfoString(&ret, quote_identifier(column)); } elog(DEBUG2, "%s: %s", __func__, ret.data); return ret.data; } /* Build ORDER BY clause based on segmentby + orderby compression settings */ static char * build_order_by_clause(MaterializationContext *context) { /* Don't build ORDER BY clause if compression is not enabled */ if (!TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(context->mat_ht)) return ""; /* No ORDER BY if no compression */ CompressionSettings *settings = ts_compression_settings_get(context->mat_ht->main_table_relid); int num_segmentby = ts_array_length(settings->fd.segmentby); int num_orderby = ts_array_length(settings->fd.orderby); StringInfo ret = makeStringInfo(); appendStringInfoString(ret, "ORDER BY "); /* process segmentby settings */ for (int i = 1; i <= num_segmentby; i++) { if (i > 1) appendStringInfoString(ret, ", "); appendStringInfoString(ret, quote_identifier( ts_array_get_element_text(settings->fd.segmentby, i))); } /* process orderby settings */ for (int i = 1; i <= num_orderby; i++) { bool is_orderby_desc = ts_array_get_element_bool(settings->fd.orderby_desc, i); bool is_null_first = ts_array_get_element_bool(settings->fd.orderby_nullsfirst, i); if (num_segmentby > 0 || i > 1) appendStringInfoString(ret, ", "); appendStringInfoString(ret, quote_identifier( ts_array_get_element_text(settings->fd.orderby, i))); if (is_orderby_desc) appendStringInfoString(ret, " DESC"); else appendStringInfoString(ret, " ASC"); if (is_null_first) appendStringInfoString(ret, " NULLS FIRST"); else appendStringInfoString(ret, " NULLS LAST"); } elog(DEBUG2, "%s: %s", __func__, ret->data); return ret->data; } static inline bool has_direct_compress_on_cagg_refresh_enabled(MaterializationContext *context) { return ts_guc_enable_direct_compress_on_cagg_refresh && TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(context->mat_ht); } /* Create INSERT statement */ static char * create_materialization_insert_statement(MaterializationContext *context) { /* If direct compress on cagg refresh is enabled, build ORDER BY clause based on segmentby and * orderby settings. This is necessary because we set * `timescaledb.enable_direct_compress_insert_client_sorted=on` in order to send ordered data to * the compressor. */ char *orderby = has_direct_compress_on_cagg_refresh_enabled(context) ? build_order_by_clause(context) : ""; StringInfoData query; initStringInfo(&query); appendStringInfo(&query, "INSERT INTO %s.%s SELECT * FROM %s.%s AS I " "WHERE I.%s >= $1 AND I.%s < $2 %s;", quote_identifier(NameStr(*context->materialization_table.schema)), quote_identifier(NameStr(*context->materialization_table.name)), quote_identifier(NameStr(*context->partial_view.schema)), quote_identifier(NameStr(*context->partial_view.name)), quote_identifier(NameStr(*context->time_column_name)), quote_identifier(NameStr(*context->time_column_name)), orderby); return query.data; } /* Create DELETE statement */ static char * create_materialization_delete_statement(MaterializationContext *context) { StringInfoData query; initStringInfo(&query); appendStringInfo(&query, "DELETE FROM %s.%s AS D " "WHERE D.%s >= $1 AND D.%s < $2;", quote_identifier(NameStr(*context->materialization_table.schema)), quote_identifier(NameStr(*context->materialization_table.name)), quote_identifier(NameStr(*context->time_column_name)), quote_identifier(NameStr(*context->time_column_name))); return query.data; } /* Create SELECT EXISTS statement */ static char * create_materialization_exists_statement(MaterializationContext *context) { StringInfoData query; initStringInfo(&query); appendStringInfo(&query, "SELECT 1 FROM %s.%s AS M " "WHERE M.%s >= $1 AND M.%s < $2 " "LIMIT 1;", quote_identifier(NameStr(*context->materialization_table.schema)), quote_identifier(NameStr(*context->materialization_table.name)), quote_identifier(NameStr(*context->time_column_name)), quote_identifier(NameStr(*context->time_column_name))); return query.data; } /* Create MERGE statement */ static char * create_materialization_merge_statement(MaterializationContext *context) { List *grp_colnames = cagg_find_groupingcols((ContinuousAgg *) context->cagg, context->mat_ht); List *agg_colnames = cagg_find_aggref_and_var_cols((ContinuousAgg *) context->cagg, context->mat_ht); List *all_columns = NIL; /* Concat both lists into a single one*/ all_columns = list_concat(all_columns, grp_colnames); all_columns = list_concat(all_columns, agg_colnames); StringInfoData merge_update; initStringInfo(&merge_update); char *merge_update_clause = build_merge_update_clause(all_columns); /* It make no sense but is possible to create a cagg only with time bucket (without * aggregate functions) */ if (merge_update_clause != NULL) { appendStringInfo(&merge_update, " WHEN MATCHED AND ROW(M.*) IS DISTINCT FROM ROW(P.*) THEN " " UPDATE SET %s ", merge_update_clause); } StringInfoData query; initStringInfo(&query); /* MERGE statement to UPDATE affected buckets and INSERT new ones */ appendStringInfo(&query, "WITH partial AS ( " " SELECT * " " FROM %s.%s " " WHERE %s >= $1 AND %s < $2 " ") " "MERGE INTO %s.%s M " "USING partial P ON %s AND M.%s >= $1 AND M.%s < $2 " " %s " /* UPDATE */ " WHEN NOT MATCHED THEN " " INSERT (%s) VALUES (%s) ", /* partial VIEW */ quote_identifier(NameStr(*context->partial_view.schema)), quote_identifier(NameStr(*context->partial_view.name)), /* partial WHERE */ quote_identifier(NameStr(*context->time_column_name)), quote_identifier(NameStr(*context->time_column_name)), /* materialization hypertable */ quote_identifier(NameStr(*context->materialization_table.schema)), quote_identifier(NameStr(*context->materialization_table.name)), /* MERGE JOIN condition */ build_merge_join_clause(grp_colnames), /* extra MERGE JOIN condition with primary dimension */ quote_identifier(NameStr(*context->time_column_name)), quote_identifier(NameStr(*context->time_column_name)), /* UPDATE */ merge_update.data, /* INSERT */ build_merge_insert_columns(all_columns, ", ", NULL), build_merge_insert_columns(all_columns, ", ", "P.")); return query.data; } /* Create DELETE after MERGE query statement */ static char * create_materialization_merge_delete_statement(MaterializationContext *context) { StringInfoData query; initStringInfo(&query); List *grp_colnames = cagg_find_groupingcols((ContinuousAgg *) context->cagg, context->mat_ht); appendStringInfo(&query, "DELETE " "FROM %s.%s M " "WHERE M.%s >= $1 AND M.%s < $2 " "AND NOT EXISTS (" " SELECT FROM %s.%s P " " WHERE %s AND P.%s >= $1 AND P.%s < $2) ", /* materialization hypertable */ quote_identifier(NameStr(*context->materialization_table.schema)), quote_identifier(NameStr(*context->materialization_table.name)), /* materialization hypertable WHERE */ quote_identifier(NameStr(*context->time_column_name)), quote_identifier(NameStr(*context->time_column_name)), /* partial VIEW */ quote_identifier(NameStr(*context->partial_view.schema)), quote_identifier(NameStr(*context->partial_view.name)), /* MERGE JOIN condition */ build_merge_join_clause(grp_colnames), /* partial WHERE */ quote_identifier(NameStr(*context->time_column_name)), quote_identifier(NameStr(*context->time_column_name))); return query.data; } static char * create_materialization_ranges_select_statement(MaterializationContext *context) { StringInfoData query; initStringInfo(&query); appendStringInfo(&query, "SELECT ctid, lowest_modified_value, greatest_modified_value " "FROM _timescaledb_catalog.continuous_aggs_materialization_ranges " "WHERE materialization_id = $1 " "AND greatest_modified_value >= lowest_modified_value " "AND lowest_modified_value >= $2 " "AND greatest_modified_value <= $3 " "AND pg_catalog.int8range(lowest_modified_value, greatest_modified_value) && " "pg_catalog.int8range($2, $3) " "ORDER BY lowest_modified_value ASC " "LIMIT 1 " "FOR UPDATE SKIP LOCKED "); return query.data; } static char * create_materialization_ranges_delete_statement(MaterializationContext *context) { StringInfoData query; initStringInfo(&query); appendStringInfo(&query, "DELETE " "FROM _timescaledb_catalog.continuous_aggs_materialization_ranges " "WHERE ctid = $1"); return query.data; } static char * create_materialization_ranges_pending_statement(MaterializationContext *context) { StringInfoData query; initStringInfo(&query); appendStringInfo(&query, "SELECT * " "FROM _timescaledb_catalog.continuous_aggs_materialization_ranges " "WHERE materialization_id = $1 " "AND greatest_modified_value >= lowest_modified_value " "AND lowest_modified_value >= $2 " "AND greatest_modified_value <= $3 " "AND pg_catalog.int8range(lowest_modified_value, greatest_modified_value) && " "pg_catalog.int8range($2, $3) " "LIMIT 1 "); return query.data; } static Oid * create_materialization_plan_argtypes(MaterializationContext *context, MaterializationPlanType plan_type, int nargs) { Oid *argtypes = (Oid *) palloc(nargs * sizeof(Oid)); switch (plan_type) { case PLAN_TYPE_RANGES_SELECT: /* 3 arguments */ case PLAN_TYPE_RANGES_PENDING: argtypes[0] = INT4OID; /* materialization_id */ argtypes[1] = INT8OID; argtypes[2] = INT8OID; break; case PLAN_TYPE_RANGES_DELETE: /* 1 argument1 */ argtypes[0] = TIDOID; /* ctid */ break; default: /* 2 arguments */ argtypes[0] = context->materialization_range.type; argtypes[1] = context->materialization_range.type; break; } return argtypes; } static MaterializationPlan * create_materialization_plan(MaterializationContext *context, MaterializationPlanType plan_type) { Assert(plan_type >= PLAN_TYPE_INSERT); Assert(plan_type < _MAX_MATERIALIZATION_PLAN_TYPES); MaterializationPlan *materialization = &materialization_plans[plan_type]; if (materialization->plan == NULL) { char *query = materialization->create_statement(context); Oid *argtypes = create_materialization_plan_argtypes(context, plan_type, materialization->nargs); elog(DEBUG2, "%s: %s", __func__, query); materialization->plan = SPI_prepare(query, materialization->nargs, argtypes); if (materialization->plan == NULL) elog(ERROR, "%s: SPI_prepare failed: %s", __func__, query); SPI_keepplan(materialization->plan); pfree(query); pfree(argtypes); } return materialization; } static void create_materialization_plan_args(MaterializationContext *context, MaterializationPlanType plan_type, Datum **values, char **nulls) { switch (plan_type) { case PLAN_TYPE_RANGES_SELECT: /* 3 arguments */ case PLAN_TYPE_RANGES_PENDING: { /* read the maximum of one bucket before the window start and after the window end to * prevent pickup large pending ranges */ const int64 bucket_width = ts_continuous_agg_bucket_width(context->cagg->bucket_function); const int64 start_adjusted = context->internal_materialization_range.start_isnull ? context->internal_materialization_range.start : ts_time_saturating_sub(context->internal_materialization_range.start, bucket_width, context->cagg->partition_type); const int64 end_adjusted = context->internal_materialization_range.end_isnull ? context->internal_materialization_range.end : ts_time_saturating_add(context->internal_materialization_range.end, bucket_width, context->cagg->partition_type); (*values)[0] = Int32GetDatum(context->cagg->data.mat_hypertable_id); (*values)[1] = Int64GetDatum(start_adjusted); (*values)[2] = Int64GetDatum(end_adjusted); (*nulls)[0] = false; (*nulls)[1] = false; (*nulls)[2] = false; break; } case PLAN_TYPE_RANGES_DELETE: /* 1 argument */ { (*values)[0] = ItemPointerGetDatum(context->tupleid); (*nulls)[0] = false; break; } default: /* 2 arguments */ { (*values)[0] = context->materialization_range.start; (*values)[1] = context->materialization_range.end; (*nulls)[0] = false; (*nulls)[1] = false; break; } } } static uint64 execute_materialization_plan(MaterializationContext *context, MaterializationPlanType plan_type) { MaterializationPlan *materialization = create_materialization_plan(context, plan_type); Datum *values = (Datum *) palloc(materialization->nargs * sizeof(Datum)); char *nulls = (char *) palloc(materialization->nargs * sizeof(char)); create_materialization_plan_args(context, plan_type, &values, &nulls); CatalogSecurityContext sec_ctx; if (materialization->catalog_security_context) ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx); int res = SPI_execute_plan(materialization->plan, values, nulls, materialization->read_only, 0); if (materialization->catalog_security_context) ts_catalog_restore_user(&sec_ctx); if (res < 0) { Ensure(materialization->error_message, "materialization plan error message not set for plan type %d", plan_type); elog(ERROR, materialization->error_message, NameStr(*context->materialization_table.schema), NameStr(*context->materialization_table.name)); } else if (materialization->progress_message) { elog(LOG, materialization->progress_message, SPI_processed, NameStr(*context->materialization_table.schema), NameStr(*context->materialization_table.name)); } if (SPI_processed > 0 && plan_type == PLAN_TYPE_RANGES_SELECT) { bool isnull; Datum dat; Assert(SPI_processed == 1); /* ctid */ dat = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull); context->tupleid = DatumGetItemPointer(dat); /* lowest_modified_value */ dat = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 2, &isnull); context->materialization_range.start = internal_to_time_value_or_infinite(DatumGetInt64(dat), context->materialization_range.type, NULL); /* greatest_modified_value */ dat = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 3, &isnull); context->materialization_range.end = internal_to_time_value_or_infinite(DatumGetInt64(dat), context->materialization_range.type, NULL); } pfree(values); pfree(nulls); return SPI_processed; } static void free_materialization_plan(MaterializationContext *context, MaterializationPlanType plan_type) { MaterializationPlan *materialization = &materialization_plans[plan_type]; if (materialization->plan != NULL) { SPI_freeplan(materialization->plan); materialization->plan = NULL; } } static void free_materialization_plans(MaterializationContext *context) { for (int plan_type = PLAN_TYPE_INSERT; plan_type < _MAX_MATERIALIZATION_PLAN_TYPES; plan_type++) { free_materialization_plan(context, plan_type); } } static void update_watermark(MaterializationContext *context) { int res; StringInfoData command; Oid types[] = { context->materialization_range.type }; Datum values[] = { context->materialization_range.start }; char nulls[] = { false }; initStringInfo(&command); appendStringInfo(&command, "SELECT %s FROM %s.%s AS I " "WHERE I.%s >= $1 " "ORDER BY 1 DESC LIMIT 1;", quote_identifier(NameStr(*context->time_column_name)), quote_identifier(NameStr(*context->materialization_table.schema)), quote_identifier(NameStr(*context->materialization_table.name)), quote_identifier(NameStr(*context->time_column_name))); elog(DEBUG2, "%s: %s", __func__, command.data); res = SPI_execute_with_args(command.data, 1, types, values, nulls, false /* read_only */, 0 /* count */); if (res < 0) elog(ERROR, "%s: could not get the last bucket of the materialized data", __func__); Ensure(SPI_gettypeid(SPI_tuptable->tupdesc, 1) == context->materialization_range.type, "partition types for result (%d) and dimension (%d) do not match", SPI_gettypeid(SPI_tuptable->tupdesc, 1), context->materialization_range.type); if (SPI_processed > 0) { bool isnull; Datum maxdat = SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull); if (!isnull) { int64 watermark = ts_time_value_to_internal(maxdat, context->materialization_range.type); ts_cagg_watermark_update(context->mat_ht, watermark, isnull, false); } } } static void execute_materializations(MaterializationContext *context) { volatile uint64 rows_processed = 0; bool prev_enable_direct_compress_insert = ts_guc_enable_direct_compress_insert; bool prev_enable_direct_compress_insert_client_sorted = ts_guc_enable_direct_compress_insert_client_sorted; if (has_direct_compress_on_cagg_refresh_enabled(context)) { /* Force the direct compress on INSERT */ SetConfigOption("timescaledb.enable_direct_compress_insert", "on", PGC_USERSET, PGC_S_SESSION); SetConfigOption("timescaledb.enable_direct_compress_insert_client_sorted", "on", PGC_USERSET, PGC_S_SESSION); } PG_TRY(); { while (execute_materialization_plan(context, PLAN_TYPE_RANGES_SELECT) > 0) { /* MERGE statement is supported only for non-compressed CAggs */ if (ts_guc_enable_merge_on_cagg_refresh && !TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(context->mat_ht)) { /* Fallback to INSERT materializations if there are no rows to change on it */ if (execute_materialization_plan(context, PLAN_TYPE_EXISTS) == 0) { elog(DEBUG2, "no rows to merge on materialization table \"%s.%s\", falling back to " "INSERT", NameStr(*context->materialization_table.schema), NameStr(*context->materialization_table.name)); rows_processed = execute_materialization_plan(context, PLAN_TYPE_INSERT); } else { rows_processed += execute_materialization_plan(context, PLAN_TYPE_MERGE); rows_processed += execute_materialization_plan(context, PLAN_TYPE_MERGE_DELETE); } } else { rows_processed += execute_materialization_plan(context, PLAN_TYPE_DELETE); rows_processed += execute_materialization_plan(context, PLAN_TYPE_INSERT); } /* Delete the pending range entry */ rows_processed += execute_materialization_plan(context, PLAN_TYPE_RANGES_DELETE); } /* Free all cached plans */ free_materialization_plans(context); } PG_CATCH(); { /* Make sure all cached plans in the session be released before rethrowing the error */ free_materialization_plans(context); PG_RE_THROW(); } PG_END_TRY(); /* Get the max(time_dimension) of the materialized data */ if (rows_processed > 0) { update_watermark(context); } /* Restore previous GUC values */ SetConfigOption("timescaledb.enable_direct_compress_insert", prev_enable_direct_compress_insert ? "on" : "off", PGC_USERSET, PGC_S_SESSION); SetConfigOption("timescaledb.enable_direct_compress_insert_client_sorted", prev_enable_direct_compress_insert_client_sorted ? "on" : "off", PGC_USERSET, PGC_S_SESSION); } ================================================ FILE: tsl/src/continuous_aggs/materialize.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include "common.h" #include "ts_catalog/continuous_agg.h" #include <fmgr.h> #include <nodes/pg_list.h> typedef struct SchemaAndName { Name schema; Name name; } SchemaAndName; /*********************** * Time ranges ***********************/ typedef struct TimeRange { Oid type; Datum start; Datum end; } TimeRange; typedef struct InternalTimeRange { Oid type; int64 start; /* inclusive */ int64 end; /* exclusive */ bool start_isnull; bool end_isnull; } InternalTimeRange; void continuous_agg_update_materialization(Hypertable *mat_ht, const ContinuousAgg *cagg, SchemaAndName partial_view, SchemaAndName materialization_table, const NameData *time_column_name, InternalTimeRange materialization_range); bool continuous_agg_has_pending_materializations(const ContinuousAgg *cagg, InternalTimeRange materialization_range); ================================================ FILE: tsl/src/continuous_aggs/options.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <catalog/namespace.h> #include <commands/view.h> #include <miscadmin.h> #include <nodes/makefuncs.h> #include <optimizer/optimizer.h> #include <rewrite/rewriteManip.h> #include <utils/builtins.h> #include "cache.h" #include "compression/create.h" #include "continuous_aggs/common.h" #include "continuous_aggs/create.h" #include "errors.h" #include "hypertable_cache.h" #include "options.h" #include "scan_iterator.h" #include "ts_catalog/array_utils.h" #include "ts_catalog/continuous_agg.h" #include "with_clause/alter_table_with_clause.h" #include "with_clause/create_materialized_view_with_clause.h" static void cagg_update_materialized_only(ContinuousAgg *agg, bool materialized_only); static List *cagg_get_compression_params(ContinuousAgg *agg, Hypertable *mat_ht, WithClauseResult *with_clause_options); static void cagg_alter_compression(ContinuousAgg *agg, Hypertable *mat_ht, List *compress_defelems); static void cagg_update_materialized_only(ContinuousAgg *agg, bool materialized_only) { ScanIterator iterator = ts_scan_iterator_create(CONTINUOUS_AGG, RowExclusiveLock, CurrentMemoryContext); iterator.ctx.index = catalog_get_index(ts_catalog_get(), CONTINUOUS_AGG, CONTINUOUS_AGG_PKEY); ts_scan_iterator_scan_key_init(&iterator, Anum_continuous_agg_pkey_mat_hypertable_id, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(agg->data.mat_hypertable_id)); ts_scanner_foreach(&iterator) { TupleInfo *ti = ts_scan_iterator_tuple_info(&iterator); bool nulls[Natts_continuous_agg]; Datum values[Natts_continuous_agg]; bool doReplace[Natts_continuous_agg] = { false }; bool should_free; HeapTuple tuple = ts_scan_iterator_fetch_heap_tuple(&iterator, false, &should_free); HeapTuple new_tuple; TupleDesc tupdesc = ts_scan_iterator_tupledesc(&iterator); heap_deform_tuple(tuple, tupdesc, values, nulls); doReplace[AttrNumberGetAttrOffset(Anum_continuous_agg_materialize_only)] = true; values[AttrNumberGetAttrOffset(Anum_continuous_agg_materialize_only)] = BoolGetDatum(materialized_only); new_tuple = heap_modify_tuple(tuple, tupdesc, values, nulls, doReplace); ts_catalog_update(ti->scanrel, new_tuple); heap_freetuple(new_tuple); if (should_free) heap_freetuple(tuple); break; } ts_scan_iterator_close(&iterator); } /* get the compression parameters for cagg. The parameters are * derived from the cagg view definition. * Computes: * compress_orderby = time_bucket column from cagg query followed by remaining grouping columns */ static List * cagg_get_compression_params(ContinuousAgg *agg, Hypertable *mat_ht, WithClauseResult *with_clause_options) { const Dimension *mat_ht_dim = hyperspace_get_open_dimension(mat_ht->space, 0); StringInfoData info; initStringInfo(&info); ArrayType *segmentby_columns = NULL; /* add time column as first entry */ appendStringInfoString(&info, quote_identifier(NameStr(mat_ht_dim->fd.column_name))); if (with_clause_options[AlterTableFlagSegmentBy].parsed) { segmentby_columns = ts_compress_hypertable_parse_segment_by(with_clause_options[AlterTableFlagSegmentBy], mat_ht); } List *grp_colnames = cagg_find_groupingcols(agg, mat_ht); if (grp_colnames) { ListCell *lc; foreach (lc, grp_colnames) { char *grpcol = (char *) lfirst(lc); /* skip time dimension since we put it as first entry */ if (namestrcmp((Name) & (mat_ht_dim->fd.column_name), grpcol) == 0) continue; if (segmentby_columns && ts_array_is_member(segmentby_columns, grpcol)) continue; if (info.len > 0) appendStringInfoString(&info, ","); appendStringInfoString(&info, quote_identifier(grpcol)); } } DefElem *ordby = makeDefElemExtended(EXTENSION_NAMESPACE, "compress_orderby", (Node *) makeString(info.data), DEFELEM_UNSPEC, -1); return list_make1(ordby); } /* forwards compression related changes via an alter statement to the underlying HT */ static void cagg_alter_compression(ContinuousAgg *agg, Hypertable *mat_ht, List *compress_defelems) { Assert(mat_ht != NULL); WithClauseResult *with_clause_options = ts_alter_table_with_clause_parse(compress_defelems); if (with_clause_options[AlterTableFlagColumnstore].parsed) { List *default_compress_defelems = cagg_get_compression_params(agg, mat_ht, with_clause_options); WithClauseResult *default_with_clause_options = ts_alter_table_with_clause_parse(default_compress_defelems); /* Merge defaults if there's any. */ for (int i = 0; i < AlterTableFlagsMax; i++) { if (with_clause_options[i].is_default && !default_with_clause_options[i].is_default) { with_clause_options[i] = default_with_clause_options[i]; elog(NOTICE, "defaulting %s to %s", with_clause_options[i].definition->arg_names[0], ts_with_clause_result_deparse_value(&with_clause_options[i])); } } } tsl_process_compress_table(mat_ht, with_clause_options); } void continuous_agg_update_options(ContinuousAgg *agg, WithClauseResult *with_clause_options) { if (!with_clause_options[CreateMaterializedViewFlagContinuous].is_default) elog(ERROR, "cannot disable continuous aggregates"); if (!with_clause_options[CreateMaterializedViewFlagMaterializedOnly].is_default) { bool materialized_only = DatumGetBool(with_clause_options[CreateMaterializedViewFlagMaterializedOnly].parsed); Cache *hcache = ts_hypertable_cache_pin(); Hypertable *mat_ht = ts_hypertable_cache_get_entry_by_id(hcache, agg->data.mat_hypertable_id); if (materialized_only == agg->data.materialized_only) { /* nothing changed, so just return */ ts_cache_release(&hcache); return; } Assert(mat_ht != NULL); cagg_flip_realtime_view_definition(agg, mat_ht); cagg_update_materialized_only(agg, materialized_only); ts_cache_release(&hcache); } if (!with_clause_options[CreateMaterializedViewFlagChunkTimeInterval].is_default) { Cache *hcache = ts_hypertable_cache_pin(); Hypertable *mat_ht = ts_hypertable_cache_get_entry_by_id(hcache, agg->data.mat_hypertable_id); int64 interval = interval_to_usec(DatumGetIntervalP( with_clause_options[CreateMaterializedViewFlagChunkTimeInterval].parsed)); Dimension *dim = ts_hyperspace_get_mutable_dimension(mat_ht->space, DIMENSION_TYPE_OPEN, 0); ts_dimension_set_chunk_interval(dim, interval); ts_cache_release(&hcache); } List *compression_options = ts_continuous_agg_get_compression_defelems(with_clause_options); if (list_length(compression_options) > 0) { Cache *hcache = ts_hypertable_cache_pin(); Hypertable *mat_ht = ts_hypertable_cache_get_entry_by_id(hcache, agg->data.mat_hypertable_id); Assert(mat_ht != NULL); cagg_alter_compression(agg, mat_ht, compression_options); ts_cache_release(&hcache); } if (!with_clause_options[CreateMaterializedViewFlagCreateGroupIndexes].is_default) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot alter create_group_indexes option for continuous aggregates"))); } } ================================================ FILE: tsl/src/continuous_aggs/options.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include "ts_catalog/continuous_agg.h" #include "with_clause/with_clause_parser.h" extern void continuous_agg_update_options(ContinuousAgg *cagg, WithClauseResult *with_clause_options); ================================================ FILE: tsl/src/continuous_aggs/planner.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <nodes/nodes.h> #include <nodes/pg_list.h> #include <parser/parse_func.h> #include "continuous_aggs/common.h" #include "planner.h" #include "ts_catalog/continuous_aggs_watermark.h" /* * The watermark function of a CAgg query is embedded into further functions. It * has the following structure: * * 1. a coalesce expression * 2. an optional to timestamp conversion function (date, timestamp, timestamptz) * 3. the actual watermark function * * For example: * ... COALESCE(to_timestamp(cagg_watermark(59)), XXX) ... * ... COALESCE(cagg_watermark(59), XXX) ... * * We use the following data structure while walking to the query to analyze the query * and collect references to the needed functions. The data structure contains: * * (a) values (e.g., function Oids) which are needed to analyze the query * (b) values that are changed during walking through the query * (e.g., references to parent functions) * (c) result data like the watermark functions and their parent functions */ typedef struct { /* (a) Values initialized after creating the context */ List *to_timestamp_func_oids; // List of Oids of the timestamp conversion functions /* (b) Values changed while walking through the query */ CoalesceExpr *parent_coalesce_expr; // the current parent coalesce_expr FuncExpr *parent_to_timestamp_func; // the current parent timestamp function /* (c) Result values */ List *watermark_parent_functions; // List of parent functions of a watermark (1) and (2) List *watermark_functions; // List of watermark functions (3) List *relids; // List of used relids by the query bool valid_query; // Is the query valid a valid CAgg query or not } ConstifyWatermarkContext; /* Oid of the watermark function. It can be stored into a static variable because it will not * change over the lifetime of a backend session, so we can lookup it only once. */ static Oid watermark_function_oid = InvalidOid; /* * Walk through the elements of the query and detect the watermark functions and their * parent functions. */ static bool constify_cagg_watermark_walker(Node *node, ConstifyWatermarkContext *context) { if (node == NULL) return false; if (IsA(node, FuncExpr)) { FuncExpr *funcExpr = castNode(FuncExpr, node); /* Handle watermark function */ if (watermark_function_oid == funcExpr->funcid) { /* The watermark function takes exactly one argument */ Assert(list_length(funcExpr->args) == 1); /* No coalesce expression found so far or function parameter is not constant, we are not * interested in this expression */ if (context->parent_coalesce_expr == NULL || !IsA(linitial(funcExpr->args), Const) || (castNode(Const, linitial(funcExpr->args))->constisnull)) { context->valid_query = false; return false; } context->watermark_functions = lappend(context->watermark_functions, funcExpr); /* Only on time based hypertables, we have a to_timestamp function */ if (context->parent_to_timestamp_func != NULL) { /* to_timestamp functions take only one parameter. This should be a reference to our * function */ Assert(linitial(context->parent_to_timestamp_func->args) == node); context->watermark_parent_functions = lappend(context->watermark_parent_functions, context->parent_to_timestamp_func); } else { /* For non int64 partitioned tables, the watermark function is wrapped into a cast * for example: COALESCE((_timescaledb_functions.cagg_watermark(11))::integer, * '-2147483648'::integer)) */ Node *coalesce_arg = linitial(context->parent_coalesce_expr->args); if (coalesce_arg != node) { /* Check if the watermark function is wrapped into a cast function */ if (!IsA(coalesce_arg, FuncExpr) || ((FuncExpr *) coalesce_arg)->args == NIL || linitial(((FuncExpr *) coalesce_arg)->args) != node) { context->valid_query = false; return false; } context->watermark_parent_functions = lappend(context->watermark_parent_functions, coalesce_arg); } else { context->watermark_parent_functions = lappend(context->watermark_parent_functions, context->parent_coalesce_expr); } } } /* Capture the timestamp conversion function */ if (list_member_oid(context->to_timestamp_func_oids, funcExpr->funcid)) { FuncExpr *old_func_expr = context->parent_to_timestamp_func; context->parent_to_timestamp_func = funcExpr; bool result = expression_tree_walker(node, constify_cagg_watermark_walker, context); context->parent_to_timestamp_func = old_func_expr; return result; } } else if (IsA(node, Query)) { /* Recurse into subselects */ Query *query = castNode(Query, node); return query_tree_walker(query, constify_cagg_watermark_walker, context, QTW_EXAMINE_RTES_BEFORE); } else if (IsA(node, CoalesceExpr)) { /* Capture the CoalesceExpr */ CoalesceExpr *parent_coalesce_expr = context->parent_coalesce_expr; context->parent_coalesce_expr = castNode(CoalesceExpr, node); bool result = expression_tree_walker(node, constify_cagg_watermark_walker, context); context->parent_coalesce_expr = parent_coalesce_expr; return result; } else if (IsA(node, RangeTblEntry)) { /* Collect the Oid of the used range tables */ RangeTblEntry *rte = (RangeTblEntry *) node; if (rte->rtekind == RTE_RELATION) { context->relids = list_append_unique_oid(context->relids, rte->relid); } /* allow range_table_walker to continue */ return false; } return expression_tree_walker(node, constify_cagg_watermark_walker, context); } /* * The entry of the watermark HTAB. */ typedef struct WatermarkConstEntry { int32 key; Const *watermark_constant; } WatermarkConstEntry; /* The query can contain multiple watermarks (i.e., two hierarchal real-time CAggs) * We maintain a hash map (hypertable id -> constant) to ensure we use the same constant * for the same watermark across the while query. */ static HTAB *pg_nodiscard init_watermark_map() { struct HASHCTL hctl = { .keysize = sizeof(int32), .entrysize = sizeof(WatermarkConstEntry), .hcxt = CurrentMemoryContext, }; /* Use 4 initial elements to have enough space for normal and hierarchical CAggs */ return hash_create("Watermark const values", 4, &hctl, HASH_ELEM | HASH_CONTEXT | HASH_BLOBS); } /* * Get a constant value for our watermark function. The constant is cached * in a hash map to ensure we use the same constant for invocations of the * watermark function with the same parameter across the whole query. */ static Const * get_watermark_const(HTAB *watermarks, int32 watermark_hypertable_id, List *range_table_oids) { bool found; WatermarkConstEntry *watermark_const = hash_search(watermarks, &watermark_hypertable_id, HASH_ENTER, &found); if (!found) { /* * Check that the argument of the watermark function is also a range table of the query. We * only constify the value when this condition is true. Only in this case, the query will be * removed from the query cache by PostgreSQL when an invalidation for the watermark * hypertable is processed (see CacheInvalidateRelcacheByRelid). */ Oid ht_relid = ts_hypertable_id_to_relid(watermark_hypertable_id, true); if (!OidIsValid(ht_relid)) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid materialized hypertable ID: %d", watermark_hypertable_id))); } /* Given table is not a part of our range tables */ if (!list_member_oid(range_table_oids, ht_relid)) { watermark_const->watermark_constant = NULL; return NULL; } /* Not found, create a new constant */ int64 watermark = ts_cagg_watermark_get(watermark_hypertable_id); Const *const_watermark = makeConst(INT8OID, -1, InvalidOid, sizeof(int64), Int64GetDatum(watermark), false, FLOAT8PASSBYVAL); watermark_const->watermark_constant = const_watermark; } return watermark_const->watermark_constant; } /* * Replace the collected references to the watermark function in the context variable * with constant values. */ static void replace_watermark_with_const(ConstifyWatermarkContext *context) { Assert(context != NULL); Assert(context->valid_query); /* We need to have at least one watermark value */ if (list_length(context->watermark_functions) < 1) return; HTAB *watermarks = init_watermark_map(); /* The list of watermark function should have the same length as the parent functions. In * other words, each watermark function should have exactly one parent function. */ Assert(list_length(context->watermark_parent_functions) == list_length(context->watermark_functions)); /* Iterate over the function parents and the actual watermark functions. Get a * const value for each function and replace the reference to the watermark function * in the function parent. */ ListCell *parent_lc, *watermark_lc; forboth (parent_lc, context->watermark_parent_functions, watermark_lc, context->watermark_functions) { FuncExpr *watermark_function = lfirst(watermark_lc); Assert(watermark_function_oid == watermark_function->funcid); Const *arg = (Const *) linitial(watermark_function->args); int32 watermark_hypertable_id = DatumGetInt32(arg->constvalue); Const *watermark_const = get_watermark_const(watermarks, watermark_hypertable_id, context->relids); /* No constant created, it means the hypertable id used by the watermark function is not a * range table and no invalidations would be processed. So, not replacing the function * invocation. */ if (watermark_const == NULL) continue; /* Replace cagg_watermark FuncExpr node by a Const node */ if (IsA(lfirst(parent_lc), FuncExpr)) { FuncExpr *parent_func_expr = castNode(FuncExpr, lfirst(parent_lc)); linitial(parent_func_expr->args) = (Node *) watermark_const; } else { /* Check that the assumed parent function is our parent function */ CoalesceExpr *parent_coalesce_expr = castNode(CoalesceExpr, lfirst(parent_lc)); linitial(parent_coalesce_expr->args) = (Node *) watermark_const; } } /* Clean up the hash map */ hash_destroy(watermarks); } /* * Constify all references to the CAgg watermark function if the query is a union query on a CAgg */ void constify_cagg_watermark(Query *parse) { if (parse == NULL) return; /* process only SELECT queries */ if (parse->commandType != CMD_SELECT) return; Node *node = (Node *) parse; ConstifyWatermarkContext context = { 0 }; context.valid_query = true; if (!OidIsValid(watermark_function_oid)) { watermark_function_oid = get_watermark_function_oid(); Ensure(OidIsValid(watermark_function_oid), "unable to determine watermark function Oid"); } /* Get Oid of all used timestamp converter functions. * * The watermark function can be invoked by a timestamp conversion function. * For example: to_timestamp(cagg_watermark(XX)). We collect the Oid of all these * converter functions in the list to_timestamp_func_oids. */ context.to_timestamp_func_oids = NIL; context.to_timestamp_func_oids = lappend_oid(context.to_timestamp_func_oids, cagg_get_boundary_converter_funcoid(DATEOID)); context.to_timestamp_func_oids = lappend_oid(context.to_timestamp_func_oids, cagg_get_boundary_converter_funcoid(TIMESTAMPOID)); context.to_timestamp_func_oids = lappend_oid(context.to_timestamp_func_oids, cagg_get_boundary_converter_funcoid(TIMESTAMPTZOID)); /* Walk through the query and collect function information */ constify_cagg_watermark_walker(node, &context); /* Replace watermark functions with const value if the query might belong to the CAgg query */ if (context.valid_query) replace_watermark_with_const(&context); } /* * Push down ORDER BY and LIMIT into subqueries of UNION for realtime * continuous aggregates when sorting by time. * * This is only enabled on PG16 and above because the internal structure is different * in previous versions. */ void cagg_sort_pushdown(Query *parse, int *cursor_opts) { ListCell *lc; /* We dont optimize aggregations on top of caggs for now. */ if (parse->groupClause) return; /* Nothing to do if we have no valid sort clause */ if (list_length(parse->rtable) != 1 || list_length(parse->sortClause) != 1 || !OidIsValid(linitial_node(SortGroupClause, parse->sortClause)->sortop)) return; Cache *cache = ts_hypertable_cache_pin(); foreach (lc, parse->rtable) { RangeTblEntry *rte = lfirst(lc); /* * Realtime cagg view will have 2 rtable entries, one for the materialized data and one for * the not yet materialized data. */ if (rte->rtekind != RTE_SUBQUERY || rte->relkind != RELKIND_VIEW || list_length(rte->subquery->rtable) != 2) continue; ContinuousAgg *cagg = ts_continuous_agg_find_by_relid(rte->relid); /* * This optimization only applies to realtime caggs. */ if (!cagg || cagg->data.materialized_only) continue; Hypertable *ht = ts_hypertable_cache_get_entry_by_id(cache, cagg->data.mat_hypertable_id); Dimension const *dim = hyperspace_get_open_dimension(ht->space, 0); /* We should only encounter hypertables with an open dimension */ if (!dim) continue; SortGroupClause *sort = linitial_node(SortGroupClause, parse->sortClause); TargetEntry *tle = get_sortgroupref_tle(sort->tleSortGroupRef, parse->targetList); /* * We only pushdown ORDER BY when it's single column * ORDER BY on the time column. */ AttrNumber time_col = dim->column_attno; if (!IsA(tle->expr, Var) || castNode(Var, tle->expr)->varattno != time_col) continue; RangeTblEntry *mat_rte = linitial_node(RangeTblEntry, rte->subquery->rtable); RangeTblEntry *rt_rte = lsecond_node(RangeTblEntry, rte->subquery->rtable); mat_rte->subquery->sortClause = list_copy(parse->sortClause); rt_rte->subquery->sortClause = list_copy(parse->sortClause); TargetEntry *mat_tle = list_nth(mat_rte->subquery->targetList, time_col - 1); TargetEntry *rt_tle = list_nth(rt_rte->subquery->targetList, time_col - 1); SortGroupClause *cagg_group = linitial(rt_rte->subquery->groupClause); cagg_group = list_nth(rt_rte->subquery->groupClause, rt_tle->ressortgroupref - 1); cagg_group->sortop = sort->sortop; cagg_group->nulls_first = sort->nulls_first; #if PG18_GE /* Track sort order * https://github.com/postgres/postgres/commit/0d2aa4d4 */ cagg_group->reverse_sort = sort->reverse_sort; #endif linitial_node(SortGroupClause, rt_rte->subquery->sortClause)->tleSortGroupRef = rt_tle->ressortgroupref; mat_tle->ressortgroupref = linitial_node(SortGroupClause, mat_rte->subquery->sortClause)->tleSortGroupRef; Oid placeholder; CompareType strategy; get_ordering_op_properties(sort->sortop, &placeholder, &placeholder, &strategy); /* * If this is DESC order and the sortop is the commutator of the cagg_group sortop, * we can align the sortop of the cagg_group with the sortop of the sort clause, which * will allow us to have the GroupAggregate node to produce the correct order and avoid * having to resort. */ if (strategy == BTGreaterStrategyNumber) { rte->subquery->rtable = list_make2(rt_rte, mat_rte); } /* * We have to prevent parallelism when we do this optimization because * the subplans of the Append have to be processed sequentially. */ *cursor_opts = *cursor_opts & ~CURSOR_OPT_PARALLEL_OK; parse->sortClause = NIL; rte->subquery->sortClause = NIL; } ts_cache_release(&cache); } ================================================ FILE: tsl/src/continuous_aggs/planner.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #ifndef TIMESCALEDB_TSL_CONTINUOUS_AGGS_PLANNER_H #define TIMESCALEDB_TSL_CONTINUOUS_AGGS_PLANNER_H #include "planner/planner.h" void constify_cagg_watermark(Query *parse); void cagg_sort_pushdown(Query *parse, int *cursor_opts); #endif ================================================ FILE: tsl/src/continuous_aggs/refresh.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <executor/spi.h> #include <fmgr.h> #include <miscadmin.h> #include <storage/lmgr.h> #include <utils/acl.h> #include <utils/builtins.h> #include <utils/date.h> #include <utils/fmgrprotos.h> #include <utils/guc.h> #include <utils/lsyscache.h> #include <utils/snapmgr.h> #include "bgw_policy/policies_v2.h" #include "debug_point.h" #include "dimension.h" #include "dimension_slice.h" #include "guc.h" #include "hypertable.h" #include "invalidation.h" #include "invalidation_threshold.h" #include "jsonb_utils.h" #include "materialize.h" #include "process_utility.h" #include "refresh.h" #include "time_bucket.h" #include "time_utils.h" #include "ts_catalog/catalog.h" #include "ts_catalog/continuous_agg.h" #define CAGG_REFRESH_LOG_LEVEL \ (context.callctx == CAGG_REFRESH_POLICY || context.callctx == CAGG_REFRESH_POLICY_BATCHED ? \ LOG : \ DEBUG1) typedef struct ContinuousAggRefreshState { ContinuousAgg cagg; Hypertable *cagg_ht; InternalTimeRange refresh_window; SchemaAndName partial_view; bool bucketing_refresh_window; } ContinuousAggRefreshState; static Hypertable *cagg_get_hypertable_or_fail(int32 hypertable_id); static InternalTimeRange get_largest_bucketed_window(Oid timetype, int64 bucket_width); static InternalTimeRange compute_inscribed_bucketed_refresh_window(const ContinuousAgg *cagg, const InternalTimeRange *const refresh_window, const int64 bucket_width); static void continuous_agg_refresh_init(ContinuousAggRefreshState *refresh, const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, bool bucketing_refresh_window); static void continuous_agg_refresh_execute(const ContinuousAggRefreshState *refresh, const InternalTimeRange *bucketed_refresh_window); static void log_refresh_window(int elevel, const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, ContinuousAggRefreshContext context); static void continuous_agg_refresh_execute_wrapper(const InternalTimeRange *bucketed_refresh_window, const ContinuousAggRefreshContext context, const long iteration, void *arg1_refresh); static void continuous_agg_refresh_with_window(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, const InvalidationStore *invalidations, const ContinuousAggRefreshContext context, bool bucketing_refresh_window); static void emit_up_to_date_notice(const ContinuousAgg *cagg, const ContinuousAggRefreshContext context); static bool process_cagg_invalidations_and_refresh(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, const ContinuousAggRefreshContext context, bool bucketing_refresh_window, bool force); static Hypertable * cagg_get_hypertable_or_fail(int32 hypertable_id) { Hypertable *ht = ts_hypertable_get_by_id(hypertable_id); if (NULL == ht) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("invalid continuous aggregate state"), errdetail("A continuous aggregate references a hypertable that does not exist."))); return ht; } /* * Compute the largest possible bucketed window given the time type and * internal restrictions. * * The largest bucketed window is governed by restrictions set by the type and * internal, TimescaleDB-specific legacy details (see get_max_window above for * further explanation). */ static InternalTimeRange get_largest_bucketed_window(Oid timetype, int64 bucket_width) { InternalTimeRange maxwindow = { .type = timetype, .start = ts_time_get_min(timetype), .end = ts_time_get_end_or_max(timetype), }; InternalTimeRange maxbuckets = { .type = timetype, }; /* For the MIN value, the corresponding bucket either falls on the exact * MIN or it will be below it. Therefore, we add (bucket_width - 1) to * move to the next bucket to be within the allowed range. */ maxwindow.start = ts_time_saturating_add(maxwindow.start, bucket_width - 1, timetype); maxbuckets.start = ts_time_bucket_by_type(bucket_width, maxwindow.start, timetype); maxbuckets.end = ts_time_get_end_or_max(timetype); return maxbuckets; } /* * Adjust the refresh window to align with inscribed buckets, so it includes buckets, which are * fully covered by the refresh window. * * Bucketing refresh window is necessary for a continuous aggregate refresh, which can refresh only * entire buckets. The result of the function is a bucketed window, where its start is at the start * of the first bucket, which is fully inside the refresh window, and its end is at the end of the * last fully covered bucket. * * Example1, the window needs to shrink: * [---------) - given refresh window * .|....|....|....|. - buckets * [----) - inscribed bucketed window * * Example2, the window is already aligned: * [----) - given refresh window * .|....|....|....|. - buckets * [----) - inscribed bucketed window * * This function is called for the continuous aggregate policy and manual refresh. In such case * excluding buckets, which are not fully covered by the refresh window, avoids refreshing a bucket, * where part of its data were dropped by a retention policy. See #2198 for details. */ static InternalTimeRange compute_inscribed_bucketed_refresh_window(const ContinuousAgg *cagg, const InternalTimeRange *const refresh_window, const int64 bucket_width) { Assert(cagg != NULL); Assert(cagg->bucket_function != NULL); InternalTimeRange result = *refresh_window; InternalTimeRange largest_bucketed_window = get_largest_bucketed_window(refresh_window->type, bucket_width); /* Get offset and origin for bucket function */ NullableDatum offset = INIT_NULL_DATUM; NullableDatum origin = INIT_NULL_DATUM; fill_bucket_offset_origin(cagg->bucket_function, refresh_window->type, &offset, &origin); /* Defined offset and origin in one function is not supported */ Assert(offset.isnull == true || origin.isnull == true); if (refresh_window->start <= largest_bucketed_window.start) { result.start = largest_bucketed_window.start; } else { /* The start time needs to be aligned with the first fully enclosed bucket. * So the original window start is moved to next bucket, except if the start is * already aligned with a bucket, thus 1 is subtracted to avoid moving into next * bucket in the aligned case. */ int64 included_bucket = ts_time_saturating_add(refresh_window->start, bucket_width - 1, refresh_window->type); /* Get the start of the included bucket. */ result.start = ts_time_bucket_by_type_extended(bucket_width, included_bucket, refresh_window->type, offset, origin); } if (refresh_window->end >= largest_bucketed_window.end) { result.end = largest_bucketed_window.end; } else { /* The window is reduced to the beginning of the bucket, which contains the exclusive * end of the refresh window. */ result.end = ts_time_bucket_by_type_extended(bucket_width, refresh_window->end, refresh_window->type, offset, origin); } return result; } /* * Get the offset as Datum value of an integer based bucket */ static Datum int_bucket_offset_to_datum(Oid type, const ContinuousAggBucketFunction *bucket_function) { Assert(bucket_function->bucket_time_based == false); switch (type) { case INT2OID: return Int16GetDatum(bucket_function->bucket_integer_offset); case INT4OID: return Int32GetDatum(bucket_function->bucket_integer_offset); case INT8OID: return Int64GetDatum(bucket_function->bucket_integer_offset); default: elog(ERROR, "invalid integer time_bucket type \"%s\"", format_type_be(type)); pg_unreachable(); } } /* * Get a NullableDatum for offset and origin based on the CAgg information */ void fill_bucket_offset_origin(const ContinuousAggBucketFunction *bucket_function, Oid type, NullableDatum *offset, NullableDatum *origin) { Assert(bucket_function != NULL); Assert(offset != NULL); Assert(origin != NULL); Assert(offset->isnull); Assert(origin->isnull); if (bucket_function->bucket_time_based) { if (bucket_function->bucket_time_offset != NULL) { offset->isnull = false; offset->value = IntervalPGetDatum(bucket_function->bucket_time_offset); } if (TIMESTAMP_NOT_FINITE(bucket_function->bucket_time_origin) == false) { origin->isnull = false; if (type == DATEOID) { /* Date was converted into a timestamp in process_additional_timebucket_parameter(), * build a Date again */ origin->value = DirectFunctionCall1(timestamp_date, TimestampGetDatum(bucket_function->bucket_time_origin)); } else { origin->value = TimestampGetDatum(bucket_function->bucket_time_origin); } } } else { if (bucket_function->bucket_integer_offset != 0) { offset->isnull = false; offset->value = int_bucket_offset_to_datum(type, bucket_function); } } } /* * Adjust the refresh window to align with circumscribed buckets, so it includes buckets, which * fully cover the refresh window. * * Bucketing refresh window is necessary for a continuous aggregate refresh, which can refresh only * entire buckets. The result of the function is a bucketed window, where its start is at the start * of a bucket, which contains the start of the refresh window, and its end is at the end of a * bucket, which contains the end of the refresh window. * * Example1, the window needs to expand: * [---------) - given refresh window * .|....|....|....|. - buckets * [--------------) - circumscribed bucketed window * * Example2, the window is already aligned: * [----) - given refresh window * .|....|....|....|. - buckets * [----) - inscribed bucketed window * * This function is called for an invalidation window before refreshing it and after the * invalidation window was adjusted to be fully inside a refresh window. In the case of a * continuous aggregate policy or manual refresh, the refresh window is the inscribed bucketed * window. * * The circumscribed behaviour is also used for a refresh on drop, when the refresh is called during * dropping chunks manually or as part of retention policy. */ InternalTimeRange compute_circumscribed_bucketed_refresh_window(const ContinuousAgg *cagg, const InternalTimeRange *const refresh_window, const ContinuousAggBucketFunction *bucket_function) { Assert(cagg != NULL); Assert(cagg->bucket_function != NULL); if (bucket_function->bucket_fixed_interval == false) { InternalTimeRange result = *refresh_window; ts_compute_circumscribed_bucketed_refresh_window_variable(&result.start, &result.end, bucket_function); return result; } /* Interval is fixed */ int64 bucket_width = ts_continuous_agg_fixed_bucket_width(bucket_function); Assert(bucket_width > 0); InternalTimeRange result = *refresh_window; InternalTimeRange largest_bucketed_window = get_largest_bucketed_window(refresh_window->type, bucket_width); /* Get offset and origin for bucket function */ NullableDatum offset = INIT_NULL_DATUM; NullableDatum origin = INIT_NULL_DATUM; fill_bucket_offset_origin(cagg->bucket_function, refresh_window->type, &offset, &origin); /* Defined offset and origin in one function is not supported */ Assert(offset.isnull == true || origin.isnull == true); if (refresh_window->start <= largest_bucketed_window.start) { result.start = largest_bucketed_window.start; } else { /* For alignment with a bucket, which includes the start of the refresh window, we just * need to get start of the bucket. */ result.start = ts_time_bucket_by_type_extended(bucket_width, refresh_window->start, refresh_window->type, offset, origin); } if (refresh_window->end >= largest_bucketed_window.end) { result.end = largest_bucketed_window.end; } else { int64 exclusive_end; int64 bucketed_end; Assert(refresh_window->end > result.start); /* The end of the window is non-inclusive so subtract one before * bucketing in case we're already at the end of the bucket (we don't * want to add an extra bucket). */ exclusive_end = ts_time_saturating_sub(refresh_window->end, 1, refresh_window->type); bucketed_end = ts_time_bucket_by_type_extended(bucket_width, exclusive_end, refresh_window->type, offset, origin); /* We get the time value for the start of the bucket, so need to add * bucket_width to get the end of it. */ result.end = ts_time_saturating_add(bucketed_end, bucket_width, refresh_window->type); } return result; } /* * Initialize the refresh state for a continuous aggregate. * * The state holds information for executing a refresh of a continuous aggregate. */ static void continuous_agg_refresh_init(ContinuousAggRefreshState *refresh, const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, bool bucketing_refresh_window) { MemSet(refresh, 0, sizeof(*refresh)); refresh->cagg = *cagg; refresh->cagg_ht = cagg_get_hypertable_or_fail(cagg->data.mat_hypertable_id); refresh->refresh_window = *refresh_window; refresh->bucketing_refresh_window = bucketing_refresh_window; refresh->partial_view.schema = &refresh->cagg.data.partial_view_schema; refresh->partial_view.name = &refresh->cagg.data.partial_view_name; } /* * Execute a refresh. * * The refresh will materialize the area given by the refresh window in the * refresh state. */ static void continuous_agg_refresh_execute(const ContinuousAggRefreshState *refresh, const InternalTimeRange *bucketed_refresh_window) { SchemaAndName cagg_hypertable_name = { .schema = &refresh->cagg_ht->fd.schema_name, .name = &refresh->cagg_ht->fd.table_name, }; const Dimension *time_dim = hyperspace_get_open_dimension(refresh->cagg_ht->space, 0); Assert(time_dim != NULL); continuous_agg_update_materialization(refresh->cagg_ht, &refresh->cagg, refresh->partial_view, cagg_hypertable_name, &time_dim->fd.column_name, *bucketed_refresh_window); } static void log_refresh_window(int elevel, const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, ContinuousAggRefreshContext context) { const char *msg = "continuous aggregate refresh (individual invalidation) on"; if (context.callctx == CAGG_REFRESH_POLICY_BATCHED) elog(elevel, "%s \"%s\" in window [ %s, %s ] (batch %d of %d)", msg, NameStr(cagg->data.user_view_name), ts_internal_to_time_string(refresh_window->start, refresh_window->type), ts_internal_to_time_string(refresh_window->end, refresh_window->type), context.processing_batch, context.number_of_batches); else elog(elevel, "%s \"%s\" in window [ %s, %s ]", msg, NameStr(cagg->data.user_view_name), ts_internal_to_time_string(refresh_window->start, refresh_window->type), ts_internal_to_time_string(refresh_window->end, refresh_window->type)); } typedef void (*scan_refresh_ranges_funct_t)(const InternalTimeRange *bucketed_refresh_window, const ContinuousAggRefreshContext context, const long iteration, /* 0 is first range */ void *arg1); static void continuous_agg_refresh_execute_wrapper(const InternalTimeRange *bucketed_refresh_window, const ContinuousAggRefreshContext context, const long iteration, void *arg1_refresh) { const ContinuousAggRefreshState *refresh = (const ContinuousAggRefreshState *) arg1_refresh; (void) iteration; log_refresh_window(CAGG_REFRESH_LOG_LEVEL, &refresh->cagg, bucketed_refresh_window, context); continuous_agg_refresh_execute(refresh, bucketed_refresh_window); } static long continuous_agg_scan_refresh_window_ranges(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, const InvalidationStore *invalidations, const ContinuousAggRefreshContext context, scan_refresh_ranges_funct_t exec_func, void *func_arg1) { TupleTableSlot *slot; long count = 0; ContinuousAggRefreshState *refresh = (ContinuousAggRefreshState *) func_arg1; slot = MakeSingleTupleTableSlot(invalidations->tupdesc, &TTSOpsMinimalTuple); while (tuplestore_gettupleslot(invalidations->tupstore, true /* forward */, false /* copy */, slot)) { bool isnull; Datum start = slot_getattr( slot, Anum_continuous_aggs_materialization_invalidation_log_lowest_modified_value, &isnull); Datum end = slot_getattr( slot, Anum_continuous_aggs_materialization_invalidation_log_greatest_modified_value, &isnull); InternalTimeRange invalidation = { .type = refresh_window->type, .start = DatumGetInt64(start), /* Invalidations are inclusive at the end, while refresh windows * aren't, so add one to the end of the invalidated region */ .end = ts_time_saturating_add(DatumGetInt64(end), 1, refresh_window->type), }; InternalTimeRange bucketed_refresh_window = { .type = invalidation.type, .start = invalidation.start, .end = invalidation.end, }; if (refresh->bucketing_refresh_window) { bucketed_refresh_window = compute_circumscribed_bucketed_refresh_window(cagg, &invalidation, cagg->bucket_function); } (*exec_func)(&bucketed_refresh_window, context, count, func_arg1); count++; } ExecDropSingleTupleTableSlot(slot); return count; } /* * Execute refreshes based on the processed invalidations. * * The given refresh window covers a set of buckets, some of which are * out-of-date (invalid) and some which are up-to-date (valid). Invalid * buckets that are adjacent form larger ranges, as shown below. * * Refresh window: [-----------------------------------------) * Invalid ranges: [-----] [-] [--] [-] [---] * Merged range: [---------------------------) * * The maximum number of individual (non-mergeable) ranges are * #buckets_in_window/2 (i.e., every other bucket is invalid). * * Since it might not be efficient to materialize a lot buckets separately * when there are many invalid (non-adjecent) buckets/ranges, we put a limit * on the number of individual materializations we do. This limit is * determined by the MATERIALIZATIONS_PER_REFRESH_WINDOW setting. * * Thus, if the refresh window covers a large number of buckets, but only a * few of them are invalid, it is likely beneficial to materialized these * separately to avoid materializing a lot of buckets that are already * up-to-date. But if the number of invalid buckets/ranges go above the * threshold, we materialize all of them in one go using the "merged range", * as illustrated above. */ static void continuous_agg_refresh_with_window(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, const InvalidationStore *invalidations, const ContinuousAggRefreshContext context, bool bucketing_refresh_window) { ContinuousAggRefreshState refresh; continuous_agg_refresh_init(&refresh, cagg, refresh_window, bucketing_refresh_window); long count pg_attribute_unused(); count = continuous_agg_scan_refresh_window_ranges(cagg, refresh_window, invalidations, context, continuous_agg_refresh_execute_wrapper, (void *) &refresh /* arg1 */); Assert(count); } #define REFRESH_FUNCTION_NAME "refresh_continuous_aggregate()" /* * Refresh a continuous aggregate across the given window. */ Datum continuous_agg_refresh(PG_FUNCTION_ARGS) { Oid cagg_relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); bool force = PG_ARGISNULL(3) ? false : PG_GETARG_BOOL(3); Jsonb *options = PG_ARGISNULL(4) ? NULL : PG_GETARG_JSONB_P(4); bool process_hypertable_invalidations = true; ContinuousAgg *cagg; InternalTimeRange refresh_window = { .type = InvalidOid, }; ts_feature_flag_check(FEATURE_CAGG); if (options) { bool found; bool value = ts_jsonb_get_bool_field(options, POL_REFRESH_CONF_KEY_PROCESS_HYPERTABLE_INVALIDATIONS, &found); process_hypertable_invalidations = !found || value; } cagg = cagg_get_by_relid_or_fail(cagg_relid); refresh_window.type = cagg->partition_type; if (!PG_ARGISNULL(1)) refresh_window.start = ts_time_value_from_arg(PG_GETARG_DATUM(1), get_fn_expr_argtype(fcinfo->flinfo, 1), refresh_window.type, true); else /* get min time for a cagg depending of the primary partition type */ refresh_window.start = cagg_get_time_min(cagg); if (!PG_ARGISNULL(2)) refresh_window.end = ts_time_value_from_arg(PG_GETARG_DATUM(2), get_fn_expr_argtype(fcinfo->flinfo, 2), refresh_window.type, true); else refresh_window.end = ts_time_get_noend_or_max(refresh_window.type); ContinuousAggRefreshContext context = { .callctx = CAGG_REFRESH_WINDOW }; continuous_agg_refresh_internal(cagg, &refresh_window, context, PG_ARGISNULL(1), PG_ARGISNULL(2), true, force, process_hypertable_invalidations, false /*extend_last_bucket*/); PG_RETURN_VOID(); } static void emit_up_to_date_notice(const ContinuousAgg *cagg, const ContinuousAggRefreshContext context) { switch (context.callctx) { case CAGG_REFRESH_WINDOW: case CAGG_REFRESH_CREATION: elog(NOTICE, "continuous aggregate \"%s\" is already up-to-date", NameStr(cagg->data.user_view_name)); break; case CAGG_REFRESH_POLICY: case CAGG_REFRESH_POLICY_BATCHED: break; } } static bool process_cagg_invalidations_and_refresh(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, const ContinuousAggRefreshContext context, bool bucketing_refresh_window, bool force) { InvalidationStore *invalidations; Oid hyper_relid = ts_hypertable_id_to_relid(cagg->data.mat_hypertable_id, false); /* Lock the continuous aggregate's materialized hypertable to protect against * concurrent invalidation log processing. * * It will produce rows in the `continuous_aggs_materialization_ranges` table * to be materialized later either serially or in parallel for non-overlap * refresh ranges. * * This is supposed to be a short transaction and in the future we can consider * relaxing this lock. */ LockRelationOid(hyper_relid, ShareUpdateExclusiveLock); invalidations = invalidation_process_cagg_log(cagg, refresh_window, ts_guc_cagg_max_individual_materializations, context, force); DEBUG_WAITPOINT("before_process_cagg_invalidations_for_refresh_lock"); SPI_commit_and_chain(); DEBUG_WAITPOINT("after_process_cagg_invalidations_for_refresh_lock"); if (invalidations != NULL) { if (context.callctx == CAGG_REFRESH_CREATION) { Assert(OidIsValid(cagg->relid)); ereport(NOTICE, (errmsg("refreshing continuous aggregate \"%s\"", get_rel_name(cagg->relid)), errhint("Use WITH NO DATA if you do not want to refresh the continuous " "aggregate on creation."))); } continuous_agg_refresh_with_window(cagg, refresh_window, invalidations, context, bucketing_refresh_window); if (invalidations) invalidation_store_free(invalidations); return true; } return false; } void continuous_agg_refresh_internal(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window_arg, const ContinuousAggRefreshContext context, const bool start_isnull, const bool end_isnull, bool bucketing_refresh_window, bool force, bool process_hypertable_invalidations, bool extend_last_bucket) { int32 mat_id = cagg->data.mat_hypertable_id; InternalTimeRange refresh_window = *refresh_window_arg; int64 invalidation_threshold; bool nonatomic = ts_process_utility_is_context_nonatomic(); /* Reset the saved ProcessUtilityContext value promptly before * calling Prevent* checks so the potential unsupported (atomic) * value won't linger there in case of ereport exit. */ ts_process_utility_context_reset(); PreventCommandIfReadOnly(REFRESH_FUNCTION_NAME); /* Prevent running refresh if we're in a transaction block since a refresh * can run two transactions and might take a long time to release locks if * there's a lot to materialize. Strictly, it is optional to prohibit * transaction blocks since there will be only one transaction if the * invalidation threshold needs no update. However, materialization might * still take a long time and it is probably best for consistency to always * prevent transaction blocks. */ PreventInTransactionBlock(nonatomic, REFRESH_FUNCTION_NAME); /* * We don't cagg refresh to fail because of decompression limit. So disable * the decompression limit for the duration of the refresh. */ const char *old_decompression_limit = GetConfigOption("timescaledb.max_tuples_decompressed_per_dml_transaction", false, false); SetConfigOption("timescaledb.max_tuples_decompressed_per_dml_transaction", "0", PGC_USERSET, PGC_S_SESSION); /* Connect to SPI manager due to the underlying SPI calls */ int rc = SPI_connect_ext(SPI_OPT_NONATOMIC); if (rc != SPI_OK_CONNECT) elog(ERROR, "SPI_connect failed: %s", SPI_result_code_string(rc)); /* Lock down search_path */ int save_nestlevel = NewGUCNestLevel(); RestrictSearchPath(); /* Like regular materialized views, require owner to refresh. */ if (!object_ownercheck(RelationRelationId, cagg->relid, GetUserId())) aclcheck_error(ACLCHECK_NOT_OWNER, get_relkind_objtype(get_rel_relkind(cagg->relid)), get_rel_name(cagg->relid)); /* No bucketing when open ended */ if (bucketing_refresh_window && !(start_isnull && end_isnull)) { if (cagg->bucket_function->bucket_fixed_interval == false) { refresh_window = *refresh_window_arg; ts_compute_inscribed_bucketed_refresh_window_variable(&refresh_window.start, &refresh_window.end, cagg->bucket_function); } else { int64 bucket_width = ts_continuous_agg_fixed_bucket_width(cagg->bucket_function); Assert(bucket_width > 0); refresh_window = compute_inscribed_bucketed_refresh_window(cagg, refresh_window_arg, bucket_width); } } if (refresh_window.start >= refresh_window.end) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("refresh window too small"), errdetail("The refresh window must cover at least one bucket of data."), errhint("Align the refresh window with the bucket" " time zone or use at least two buckets."))); /* If there is no other policy defined after this, the inscribed bucket calculated above * is correct. However, in the case of concurrent policies, if this isn't the last * policy defined then we should extend the end of the window to include the partial * bucket. This is done to ensure concurrent policies that are 'adjacent' don't skip a * bucket We don't need to do this when the CAgg is created WITH DATA, or manually * refreshed */ if (extend_last_bucket && !(start_isnull && end_isnull)) { if (cagg->bucket_function->bucket_fixed_interval == false) { refresh_window.end = ts_compute_beginning_of_the_next_bucket_variable(refresh_window.end, cagg->bucket_function); } else { int64 bucket_width = ts_continuous_agg_fixed_bucket_width(cagg->bucket_function); refresh_window.end = ts_time_saturating_add(refresh_window.end, bucket_width - 1, refresh_window.type); } } /* * Perform the refresh across two transactions. * * The first transaction moves the invalidation threshold (if needed) and * copies over invalidations from the hypertable log to the cagg * invalidation log. Doing the threshold and copying as part of the first * transaction ensures that the threshold and new invalidations will be * visible as soon as possible to concurrent refreshes and that we keep * locks for only a short period. * * The second transaction processes the cagg invalidation log and then * performs the actual refresh (materialization of data). This transaction * serializes around a lock on the materialized hypertable for the * continuous aggregate that gets refreshed. */ /* Set the new invalidation threshold. Note that this only updates the * threshold if the new value is greater than the old one. Otherwise, the * existing threshold is returned. */ invalidation_threshold = invalidation_threshold_set_or_get(cagg, &refresh_window); /* We must also cap the refresh window at the invalidation threshold. If * we process invalidations after the threshold, the continuous aggregates * won't be refreshed when the threshold is moved forward in the * future. The invalidation threshold should already be aligned on bucket * boundary. */ if (refresh_window.end > invalidation_threshold) refresh_window.end = invalidation_threshold; /* Capping the end might have made the window 0, or negative, so nothing to refresh in that * case. * * For variable width buckets we use a refresh_window.start value that is lower than the * -infinity value (ts_time_get_nobegin < ts_time_get_min). Therefore, the first check in the * following if statement is not enough. If the invalidation_threshold returns the min_value for * the data type, we end up with [nobegin, min_value] which is an invalid time interval. * Therefore, we have also to check if the invalidation_threshold is defined. If not, no refresh * is needed. */ if ((refresh_window.start >= refresh_window.end) || (IS_TIMESTAMP_TYPE(refresh_window.type) && invalidation_threshold == ts_time_get_min(refresh_window.type))) { emit_up_to_date_notice(cagg, context); /* Restore search_path */ AtEOXact_GUC(false, save_nestlevel); rc = SPI_finish(); if (rc != SPI_OK_FINISH) elog(ERROR, "SPI_finish failed: %s", SPI_result_code_string(rc)); return; } if (process_hypertable_invalidations) { /* * If we are using trigger-based invalidations, we can process the * invalidations for the associated hypertable only and later read the * invalidations for other hypertables, but when using WAL-based * invalidation we need to process all of the hypertables that are * currently using WAL. * * We want to prevent any changes to how invalidations are collected * in the meantime since changing the invalidation collection method * while this is running might cause problems and miss invalidations. * * Concurrency on the replication slot is controlled using some * special sauce in ReplicationSlotAcquire(), which is called inside * pg_logical_slot_get_changes_guts(). * * This will currently generate an error rather than blocking on the * lock, so we need to add a separate lock to ensure a blocking * behaviour. */ invalidation_process_hypertable_log(cagg->data.raw_hypertable_id, refresh_window.type); } /* Commit and Start a new transaction */ SPI_commit_and_chain(); cagg = ts_continuous_agg_find_by_mat_hypertable_id(mat_id, false); bool refreshed = process_cagg_invalidations_and_refresh(cagg, &refresh_window, context, bucketing_refresh_window, force); /* check if we have any pending materializations in our refresh window range, * if so, we need to process them * Note that we use the original refresh window range here, not the one that has been processed * by the refresh function*/ refresh_window = *refresh_window_arg; bool has_pending_materializations = continuous_agg_has_pending_materializations(cagg, refresh_window); if (has_pending_materializations) { ContinuousAggRefreshState refresh; continuous_agg_refresh_init(&refresh, cagg, &refresh_window, bucketing_refresh_window); #ifdef TS_DEBUG elog(NOTICE, "continuous aggregate \"%s\" has pending materializations in window [ %s, %s ]", NameStr(cagg->data.user_view_name), ts_internal_to_time_string(refresh_window.start, refresh_window.type), ts_internal_to_time_string(refresh_window.end, refresh_window.type)); #endif InternalTimeRange invalidation = { .type = refresh_window.type, .start = refresh_window.start, /* Invalidations are inclusive at the end, while refresh windows * aren't, so add one to the end of the invalidated region */ .end = ts_time_saturating_add(refresh_window.end, 1, refresh_window.type), }; InternalTimeRange bucketed_refresh_window = { .type = invalidation.type, .start = invalidation.start, .end = invalidation.end, }; if (bucketing_refresh_window) { bucketed_refresh_window = compute_circumscribed_bucketed_refresh_window(cagg, &invalidation, cagg->bucket_function); } continuous_agg_refresh_execute(&refresh, &bucketed_refresh_window); } if (!refreshed && !has_pending_materializations) emit_up_to_date_notice(cagg, context); DEBUG_WAITPOINT("after_process_cagg_materializations"); /* Restore search_path */ AtEOXact_GUC(false, save_nestlevel); SetConfigOption("timescaledb.max_tuples_decompressed_per_dml_transaction", old_decompression_limit, PGC_USERSET, PGC_S_SESSION); rc = SPI_finish(); if (rc != SPI_OK_FINISH) elog(ERROR, "SPI_finish failed: %s", SPI_result_code_string(rc)); } static void debug_refresh_window(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, const char *msg) { elog(DEBUG1, "%s \"%s\" in window [ %s, %s ] internal [ " INT64_FORMAT ", " INT64_FORMAT " ] minimum [ %s ]", msg, NameStr(cagg->data.user_view_name), ts_internal_to_time_string(refresh_window->start, refresh_window->type), ts_internal_to_time_string(refresh_window->end, refresh_window->type), refresh_window->start, refresh_window->end, ts_datum_to_string(Int64GetDatum(ts_time_get_min(refresh_window->type)), refresh_window->type)); } List * continuous_agg_split_refresh_window(ContinuousAgg *cagg, InternalTimeRange *original_refresh_window, int32 buckets_per_batch, bool refresh_newest_first) { /* Do not produce batches when the number of buckets per batch is zero (disabled) */ if (buckets_per_batch == 0) { return NIL; } InternalTimeRange refresh_window = { .type = original_refresh_window->type, .start = original_refresh_window->start, .start_isnull = original_refresh_window->start_isnull, .end = original_refresh_window->end, .end_isnull = original_refresh_window->end_isnull, }; debug_refresh_window(cagg, &refresh_window, "begin"); const Hypertable *ht = cagg_get_hypertable_or_fail(cagg->data.raw_hypertable_id); const Dimension *time_dim = hyperspace_get_open_dimension(ht->space, 0); /* * Cap the refresh window to the min and max time of the hypertable * * In order to don't produce unnecessary batches we need to check if the start and end of the * refresh window is NULL then get the min/max slice from the original hypertable * */ if (refresh_window.start_isnull) { debug_refresh_window(cagg, &refresh_window, "START IS NULL"); DimensionSlice *slice = ts_dimension_slice_nth_earliest_slice(time_dim->fd.id, 1); /* If still there's no MIN slice range start then return no batches */ if (NULL == slice || TS_TIME_IS_MIN(slice->fd.range_start, refresh_window.type) || TS_TIME_IS_NOBEGIN(slice->fd.range_start, refresh_window.type)) { elog(DEBUG1, "no min slice range start for continuous aggregate \"%s.%s\", falling back to " "single batch processing", NameStr(cagg->data.user_view_schema), NameStr(cagg->data.user_view_name)); return NIL; } refresh_window.start = slice->fd.range_start; refresh_window.start_isnull = false; } if (refresh_window.end_isnull) { debug_refresh_window(cagg, &refresh_window, "END IS NULL"); DimensionSlice *slice = ts_dimension_slice_nth_latest_slice(time_dim->fd.id, 1); /* If still there's no MAX slice range start then return no batches */ if (NULL == slice || TS_TIME_IS_MAX(slice->fd.range_end, refresh_window.type) || TS_TIME_IS_NOEND(slice->fd.range_end, refresh_window.type)) { elog(DEBUG1, "no min slice range start for continuous aggregate \"%s.%s\", falling back to " "single batch processing", NameStr(cagg->data.user_view_schema), NameStr(cagg->data.user_view_name)); return NIL; } refresh_window.end = slice->fd.range_end; refresh_window.end_isnull = false; } /* Compute the inscribed bucket for the capped refresh window range */ const int64 bucket_width = ts_continuous_agg_bucket_width(cagg->bucket_function); if (cagg->bucket_function->bucket_fixed_interval == false) { ts_compute_inscribed_bucketed_refresh_window_variable(&refresh_window.start, &refresh_window.end, cagg->bucket_function); } else { refresh_window = compute_inscribed_bucketed_refresh_window(cagg, &refresh_window, bucket_width); } /* Check if the refresh size is large enough to produce bathes, if not then return no batches */ const int64 refresh_window_size = i64abs(refresh_window.end - refresh_window.start); const int64 batch_size = (bucket_width * buckets_per_batch); if (refresh_window_size <= batch_size) { Oid type = IS_TIMESTAMP_TYPE(refresh_window.type) ? INTERVALOID : refresh_window.type; Datum refresh_size_interval = ts_internal_to_interval_value(refresh_window_size, type); Datum batch_size_interval = ts_internal_to_interval_value(batch_size, type); elog(DEBUG1, "refresh window size (%s) is smaller than or equal to batch size (%s), falling back " "to single batch processing", ts_datum_to_string(refresh_size_interval, type), ts_datum_to_string(batch_size_interval, type)); return NIL; } debug_refresh_window(cagg, &refresh_window, "before produce batches"); /* * Produce the batches to be processed * * The refresh window is split into multiple batches of size `batch_size` each. The batches are * produced in reverse order so that the first range produced is the last range to be processed. * * The batches are produced in reverse order because the most recent data should be the first to * be processed and be visible for the users. * * It takes in account the invalidation logs (hypertable and materialization hypertable) to * avoid producing wholes that have no data to be processed. * * The logic is something like the following: * 1. Get dimension slices from the original hypertables * 2. Get either hypertable and materialization hypertable invalidation logs * 3. Produce the batches in reverse order * 4. Check if the produced batch overlaps either with dimension slices #1 and invalidation logs * #2 * 5. If the batch overlaps with both then it's a valid batch to be processed * 6. If the batch overlaps with only one of them then it's not a valid batch to be processed * 7. If the batch does not overlap with any of them then it's not a valid batch to be processed */ const char *query_str_template = " \ WITH dimension_slices AS ( \ SELECT \ range_start AS start, \ range_end AS end \ FROM \ _timescaledb_catalog.dimension_slice \ JOIN _timescaledb_catalog.dimension ON dimension.id = dimension_slice.dimension_id \ WHERE \ hypertable_id = $1 \ AND dimension_id = $2 \ AND range_end >= range_start \ ORDER BY \ %s \ ), \ invalidation_logs AS ( \ SELECT \ lowest_modified_value, \ greatest_modified_value \ FROM \ _timescaledb_catalog.continuous_aggs_materialization_invalidation_log \ WHERE \ materialization_id = $3 \ AND greatest_modified_value >= lowest_modified_value \ UNION ALL \ SELECT \ pg_catalog.min(lowest_modified_value) AS lowest_modified_value, \ pg_catalog.max(greatest_modified_value) AS greatest_modified_value \ FROM \ _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log \ WHERE \ hypertable_id = $1 \ AND greatest_modified_value >= lowest_modified_value \ ) \ SELECT \ refresh_start AS start, \ LEAST($6::numeric, refresh_start::numeric + $4::numeric)::bigint AS end \ FROM \ pg_catalog.generate_series($5, $6, $4) AS refresh_start \ WHERE \ EXISTS ( \ SELECT FROM dimension_slices \ WHERE \ pg_catalog.int8range(refresh_start, LEAST($6::numeric, refresh_start::numeric + $4::numeric)::bigint) \ OPERATOR(pg_catalog.&&) \ pg_catalog.int8range(dimension_slices.start, dimension_slices.end) \ ) \ AND EXISTS ( \ SELECT FROM \ invalidation_logs \ WHERE \ pg_catalog.int8range(refresh_start, LEAST($6::numeric, refresh_start::numeric + $4::numeric)::bigint) \ OPERATOR(pg_catalog.&&) \ pg_catalog.int8range(lowest_modified_value, greatest_modified_value) \ AND lowest_modified_value IS NOT NULL \ AND (greatest_modified_value IS NOT NULL AND greatest_modified_value != $7) \ ) \ ORDER BY \ refresh_start %s;"; const char *query_str = psprintf(query_str_template, refresh_newest_first ? "range_end DESC" : "range_start ASC", refresh_newest_first ? "DESC" : "ASC"); /* List of InternalTimeRange elements to be returned */ List *refresh_window_list = NIL; /* Prepare for SPI call */ int res; Oid types[] = { INT4OID, INT4OID, INT4OID, INT8OID, INT8OID, INT8OID, INT8OID }; Datum values[] = { Int32GetDatum(ht->fd.id), Int32GetDatum(time_dim->fd.id), Int32GetDatum(cagg->data.mat_hypertable_id), Int64GetDatum(batch_size), Int64GetDatum(refresh_window.start), Int64GetDatum(refresh_window.end), Int64GetDatum(CAGG_INVALIDATION_WRONG_GREATEST_VALUE) }; char nulls[] = { false, false, false, false, false, false, false }; MemoryContext oldcontext = CurrentMemoryContext; if (SPI_connect() != SPI_OK_CONNECT) elog(ERROR, "could not connect to SPI"); /* Lock down search_path */ int save_nestlevel = NewGUCNestLevel(); RestrictSearchPath(); res = SPI_execute_with_args(query_str, 7, types, values, nulls, false /* read_only */, 0 /* count */); if (res < 0) elog(ERROR, "%s: could not produce batches for the policy cagg refresh", __func__); if (SPI_processed == 1) { elog(DEBUG1, "only one batch produced for continuous aggregate \"%s.%s\", falling back to single " "batch processing", NameStr(cagg->data.user_view_schema), NameStr(cagg->data.user_view_name)); /* Restore search_path */ AtEOXact_GUC(false, save_nestlevel); res = SPI_finish(); if (res != SPI_OK_FINISH) elog(ERROR, "SPI_finish failed: %s", SPI_result_code_string(res)); return NIL; } /* Build the batches list */ for (uint64 batch = 0; batch < SPI_processed; batch++) { bool range_start_isnull, range_end_isnull; Datum range_start = SPI_getbinval(SPI_tuptable->vals[batch], SPI_tuptable->tupdesc, 1, &range_start_isnull); Datum range_end = SPI_getbinval(SPI_tuptable->vals[batch], SPI_tuptable->tupdesc, 2, &range_end_isnull); /* We need to allocate the list in the old memory context because here we're in the SPI * context */ MemoryContext saved_context = MemoryContextSwitchTo(oldcontext); InternalTimeRange *range = palloc0(sizeof(InternalTimeRange)); range->start = DatumGetInt64(range_start); range->start_isnull = range_start_isnull; range->end = DatumGetInt64(range_end); range->end_isnull = range_end_isnull; range->type = original_refresh_window->type; /* For variable-length buckets, circumscribe the batch to bucket boundaries. * The batch size calculation uses a 30-day approximation for months, so we need * to expand batches to cover complete buckets.*/ if (cagg->bucket_function->bucket_fixed_interval == false) { ts_compute_circumscribed_bucketed_refresh_window_variable(&range->start, &range->end, cagg->bucket_function); } /* * To make sure that the first range (or last range in case of refreshing from oldest to * newest) is aligned with the end of the refresh window we need to set the end to the * maximum value of the time type if the original refresh window end is NULL. */ if (((batch == 0 && refresh_newest_first) || (batch == (SPI_processed - 1) && !refresh_newest_first)) && original_refresh_window->end_isnull) { range->end = ts_time_get_noend_or_max(range->type); range->end_isnull = true; } /* * To make sure that the last range (or first range in case of refreshing from oldest to * newest) is aligned with the start of the refresh window we need to set the start to the * maximum value of the time type if the original refresh window start is NULL. */ if (((batch == (SPI_processed - 1) && refresh_newest_first) || (batch == 0 && !refresh_newest_first)) && original_refresh_window->start_isnull) { range->start = cagg_get_time_min(cagg); range->start_isnull = true; } refresh_window_list = lappend(refresh_window_list, range); MemoryContextSwitchTo(saved_context); debug_refresh_window(cagg, range, "batch produced"); } /* Restore search_path */ AtEOXact_GUC(false, save_nestlevel); res = SPI_finish(); if (res != SPI_OK_FINISH) elog(ERROR, "SPI_finish failed: %s", SPI_result_code_string(res)); if (refresh_window_list == NIL) { elog(DEBUG1, "no valid batches produced for continuous aggregate \"%s.%s\", falling back to single " "batch processing", NameStr(cagg->data.user_view_schema), NameStr(cagg->data.user_view_name)); } return refresh_window_list; } ================================================ FILE: tsl/src/continuous_aggs/refresh.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <fmgr.h> #include "invalidation.h" #include "materialize.h" #include "ts_catalog/continuous_agg.h" extern Datum continuous_agg_refresh(PG_FUNCTION_ARGS); extern void continuous_agg_refresh_internal(const ContinuousAgg *cagg, const InternalTimeRange *refresh_window, const ContinuousAggRefreshContext context, const bool start_isnull, const bool end_isnull, bool bucketing_refresh_window, bool force, bool process_hypertable_invalidations, bool extend_last_bucket); extern List *continuous_agg_split_refresh_window(ContinuousAgg *cagg, InternalTimeRange *original_refresh_window, int32 buckets_per_batch, bool refresh_newest_first); InternalTimeRange compute_circumscribed_bucketed_refresh_window(const ContinuousAgg *cagg, const InternalTimeRange *const refresh_window, const ContinuousAggBucketFunction *bucket_function); extern void fill_bucket_offset_origin(const ContinuousAggBucketFunction *bucket_function, Oid type, NullableDatum *offset, NullableDatum *origin); ================================================ FILE: tsl/src/continuous_aggs/utils.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <commands/view.h> #include <storage/lmgr.h> #include <utils/acl.h> #include <utils/regproc.h> #include <utils/snapmgr.h> #include <utils/timestamp.h> #include "extension.h" #include "guc.h" #include "time_bucket.h" #include "utils.h" enum { Anum_cagg_validate_query_valid = 1, Anum_cagg_validate_query_error_level, Anum_cagg_validate_query_error_code, Anum_cagg_validate_query_error_message, Anum_cagg_validate_query_error_detail, Anum_cagg_validate_query_error_hint, _Anum_cagg_validate_query_max }; #define Natts_cagg_validate_query (_Anum_cagg_validate_query_max - 1) #define ORIGIN_PARAMETER_NAME "origin" static Datum create_cagg_validate_query_datum(TupleDesc tupdesc, const bool is_valid_query, ErrorData *edata) { NullableDatum datums[Natts_cagg_validate_query] = { { 0 } }; HeapTuple tuple; tupdesc = BlessTupleDesc(tupdesc); ts_datum_set_bool(Anum_cagg_validate_query_valid, datums, is_valid_query, false); ts_datum_set_text_from_cstring(Anum_cagg_validate_query_error_level, datums, edata->elevel > 0 ? error_severity(edata->elevel) : NULL); ts_datum_set_text_from_cstring(Anum_cagg_validate_query_error_code, datums, edata->sqlerrcode > 0 ? unpack_sql_state(edata->sqlerrcode) : NULL); ts_datum_set_text_from_cstring(Anum_cagg_validate_query_error_message, datums, edata->message ? edata->message : NULL); ts_datum_set_text_from_cstring(Anum_cagg_validate_query_error_detail, datums, edata->detail ? edata->detail : NULL); ts_datum_set_text_from_cstring(Anum_cagg_validate_query_error_hint, datums, edata->hint ? edata->hint : NULL); Assert(tupdesc->natts == Natts_cagg_validate_query); tuple = ts_heap_form_tuple(tupdesc, datums); return HeapTupleGetDatum(tuple); } Datum continuous_agg_validate_query(PG_FUNCTION_ARGS) { text *query_text = PG_GETARG_TEXT_P(0); char *sql; volatile bool is_valid_query = false; Datum datum_sql; TupleDesc tupdesc; ErrorData *edata; MemoryContext oldcontext = CurrentMemoryContext; /* Change $1, $2 ... placeholders to NULL constant. This is necessary to make parser happy */ sql = text_to_cstring(query_text); elog(DEBUG1, "sql: %s", sql); datum_sql = CStringGetTextDatum(sql); datum_sql = DirectFunctionCall4Coll(textregexreplace, C_COLLATION_OID, datum_sql, CStringGetTextDatum("\\$[0-9]+"), CStringGetTextDatum("NULL"), CStringGetTextDatum("g")); sql = text_to_cstring(DatumGetTextP(datum_sql)); elog(DEBUG1, "sql: %s", sql); if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) elog(ERROR, "function returning record called in context that cannot accept type record"); PG_TRY(); { List *tree; Node *node; RawStmt *rawstmt; ParseState *pstate; Query *query; edata = (ErrorData *) palloc0(sizeof(ErrorData)); edata->message = NULL; edata->detail = NULL; edata->hint = NULL; tree = pg_parse_query(sql); if (tree == NIL) { edata->elevel = ERROR; edata->sqlerrcode = ERRCODE_INTERNAL_ERROR; edata->message = "failed to parse query"; } else if (list_length(tree) > 1) { edata->elevel = WARNING; edata->sqlerrcode = ERRCODE_FEATURE_NOT_SUPPORTED; edata->message = "multiple statements are not supported"; } else { node = linitial(tree); rawstmt = (RawStmt *) node; pstate = make_parsestate(NULL); Assert(IsA(node, RawStmt)); if (!IsA(rawstmt->stmt, SelectStmt)) { edata->elevel = WARNING; edata->sqlerrcode = ERRCODE_FEATURE_NOT_SUPPORTED; edata->message = "only select statements are supported"; } else { pstate->p_sourcetext = sql; query = transformTopLevelStmt(pstate, rawstmt); free_parsestate(pstate); (void) cagg_validate_query(query, "public", "cagg_validate", false); is_valid_query = true; } } } PG_CATCH(); { MemoryContextSwitchTo(oldcontext); edata = CopyErrorData(); FlushErrorState(); } PG_END_TRY(); PG_RETURN_DATUM(create_cagg_validate_query_datum(tupdesc, is_valid_query, edata)); } /* Get the Oid of the direct view of the CAgg. We cannot use the TimescaleDB internal * functions such as ts_continuous_agg_find_by_mat_hypertable_id() at this point since this * function can be called during an extension upgrade and ts_catalog_get() does not work. */ static Oid get_direct_view_oid(int32 mat_hypertable_id) { RangeVar *ts_cagg = makeRangeVar(CATALOG_SCHEMA_NAME, CONTINUOUS_AGG_TABLE_NAME, -1 /* taken location unknown */); Relation cagg_rel = relation_openrv_extended(ts_cagg, AccessShareLock, /* missing ok */ true); RangeVar *ts_cagg_idx = makeRangeVar(CATALOG_SCHEMA_NAME, TS_CAGG_CATALOG_IDX, -1 /* taken location unknown */); Relation cagg_idx_rel = relation_openrv_extended(ts_cagg_idx, AccessShareLock, /* missing ok */ true); /* Prepare relation scan */ TupleTableSlot *slot = table_slot_create(cagg_rel, NULL); ScanKeyData scankeys[1]; ScanKeyEntryInitialize(&scankeys[0], 0, 1, BTEqualStrategyNumber, InvalidOid, InvalidOid, F_INT4EQ, Int32GetDatum(mat_hypertable_id)); /* Prepare index scan */ PushActiveSnapshot(GetTransactionSnapshot()); IndexScanDesc indexscan = index_beginscan_compat(cagg_rel, cagg_idx_rel, GetActiveSnapshot(), NULL, 1, 0); index_rescan(indexscan, scankeys, 1, NULL, 0); /* Read tuple from relation */ bool got_next_slot = index_getnext_slot(indexscan, ForwardScanDirection, slot); if (!got_next_slot) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid materialized hypertable ID: %d", mat_hypertable_id))); bool is_null = false; /* We use the view_schema and view_name to get data from the system catalog. Even char pointers * are passed to the catalog, it calls namehashfast() internally, which assumes that a char of * NAMEDATALEN is allocated. */ NameData view_schema_name; NameData view_name_name; /* We need to get the attribute names dynamically since this function can be called during an * upgrade and the fixed attribution numbers (i.e., Anum_continuous_agg_direct_view_schema) can * be different. */ AttrNumber direct_view_schema_attr = get_attnum(cagg_rel->rd_id, "direct_view_schema"); Ensure(direct_view_schema_attr != InvalidAttrNumber, "unable to get attribute number for direct_view_schema"); AttrNumber direct_view_name_attr = get_attnum(cagg_rel->rd_id, "direct_view_name"); Ensure(direct_view_name_attr != InvalidAttrNumber, "unable to get attribute number for direct_view_name"); char *view_schema = DatumGetCString(slot_getattr(slot, direct_view_schema_attr, &is_null)); Ensure(!is_null, "unable to get view schema for oid %d", mat_hypertable_id); Assert(view_schema != NULL); namestrcpy(&view_schema_name, view_schema); char *view_name = DatumGetCString(slot_getattr(slot, direct_view_name_attr, &is_null)); Ensure(!is_null, "unable to get view name for oid %d", mat_hypertable_id); Assert(view_name != NULL); namestrcpy(&view_name_name, view_name); got_next_slot = index_getnext_slot(indexscan, ForwardScanDirection, slot); Ensure(!got_next_slot, "found duplicate definitions for CAgg mat_ht %d", mat_hypertable_id); /* End relation scan */ index_endscan(indexscan); ExecDropSingleTupleTableSlot(slot); relation_close(cagg_rel, AccessShareLock); relation_close(cagg_idx_rel, AccessShareLock); PopActiveSnapshot(); /* Get Oid of user view */ Oid direct_view_oid = ts_get_relation_relid(NameStr(view_schema_name), NameStr(view_name_name), false); Assert(OidIsValid(direct_view_oid)); return direct_view_oid; } enum { Anum_cagg_bucket_function_oid = 1, Anum_cagg_bucket_function_width, Anum_cagg_bucket_function_origin, Anum_cagg_bucket_function_offset, Anum_cagg_bucket_function_timezone, Anum_cagg_bucket_function_fixed_width, _Anum_cagg_bucket_function_max }; #define Natts_cagg_bucket_function (_Anum_cagg_bucket_function_max - 1) static Datum create_cagg_get_bucket_function_datum(TupleDesc tupdesc, ContinuousAggBucketFunction *bf) { NullableDatum datums[Natts_cagg_bucket_function] = { { 0 } }; HeapTuple tuple; char *bucket_origin = NULL; char *bucket_offset = NULL; char *bucket_width = NULL; if (IS_TIME_BUCKET_INFO_TIME_BASED(bf)) { /* Bucketing on time */ Assert(bf->bucket_time_width != NULL); bucket_width = DatumGetCString( DirectFunctionCall1(interval_out, IntervalPGetDatum(bf->bucket_time_width))); if (!TIMESTAMP_NOT_FINITE(bf->bucket_time_origin)) { bucket_origin = DatumGetCString( DirectFunctionCall1(timestamptz_out, TimestampTzGetDatum(bf->bucket_time_origin))); } if (bf->bucket_time_offset != NULL) { bucket_offset = DatumGetCString( DirectFunctionCall1(interval_out, IntervalPGetDatum(bf->bucket_time_offset))); } } else { /* Bucketing on integers */ bucket_width = palloc0((MAXINT8LEN + 1) * sizeof(char)); pg_lltoa(bf->bucket_integer_width, bucket_width); /* Integer buckets with origin are not supported, so nothing to do. */ Assert(bucket_origin == NULL); if (bf->bucket_integer_offset != 0) { bucket_offset = palloc0((MAXINT8LEN + 1) * sizeof(char)); pg_lltoa(bf->bucket_integer_offset, bucket_offset); } } tupdesc = BlessTupleDesc(tupdesc); ts_datum_set_objectid(Anum_cagg_bucket_function_oid, datums, bf->bucket_function); ts_datum_set_text_from_cstring(Anum_cagg_bucket_function_width, datums, bucket_width); ts_datum_set_text_from_cstring(Anum_cagg_bucket_function_origin, datums, bucket_origin); ts_datum_set_text_from_cstring(Anum_cagg_bucket_function_offset, datums, bucket_offset); ts_datum_set_text_from_cstring(Anum_cagg_bucket_function_timezone, datums, bf->bucket_time_timezone); ts_datum_set_bool(Anum_cagg_bucket_function_fixed_width, datums, bf->bucket_fixed_interval, false); Assert(tupdesc->natts == Natts_cagg_validate_query); tuple = ts_heap_form_tuple(tupdesc, datums); return HeapTupleGetDatum(tuple); } /* * Get the bucket function information for the given materialized hypertable id. * * When running `cagg_get_bucket_function_info` the function returns the following fields: * - oid: The Oid of the bucket function * - width: The width of the bucket function * - origin: The origin of the bucket function * - offset: The offset of the bucket function * - timezone: The timezone of the bucket function * - fixed_width: Is the bucket width fixed * * When running `cagg_get_bucket_function` the function returns the following fields: * - oid: The Oid of the bucket function */ static Datum cagg_get_bucket_function_datum(int32 mat_hypertable_id, FunctionCallInfo fcinfo) { Oid direct_view_oid = get_direct_view_oid(mat_hypertable_id); TupleDesc tupdesc; if (fcinfo != NULL && get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) elog(ERROR, "function returning record called in context that cannot accept type record"); ContinuousAggBucketFunction *bf = ts_cagg_get_bucket_function_info(direct_view_oid); if (!OidIsValid(bf->bucket_function)) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("time_bucket function not found in CAgg definition for " "mat_ht_id: %d", mat_hypertable_id))); pg_unreachable(); } if (fcinfo != NULL) return create_cagg_get_bucket_function_datum(tupdesc, bf); return ObjectIdGetDatum(bf->bucket_function); } /* * This function returns the `time_bucket` function Oid in the user view definition * of a given materialization hupertable. * * NOTE: this function is deprecated and should be removed in the future, use * `cagg_get_bucket_function_info` instead. */ Datum continuous_agg_get_bucket_function(PG_FUNCTION_ARGS) { /* Return the oid of the bucket function */ PG_RETURN_DATUM(cagg_get_bucket_function_datum(PG_GETARG_INT32(0), NULL)); } /* * This function returns all information about the `time_bucket` function of a given * materialization hypertable using the user view definition stored in Postgres catalog. */ Datum continuous_agg_get_bucket_function_info(PG_FUNCTION_ARGS) { /* Return all bucket function info */ PG_RETURN_DATUM(cagg_get_bucket_function_datum(PG_GETARG_INT32(0), fcinfo)); } Datum continuous_agg_get_grouping_columns(PG_FUNCTION_ARGS) { List *cagg_group_cols = NIL; ListCell *lc = NULL; Oid cagg_relid = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); ArrayBuildState *astate = NULL; ContinuousAgg *cagg = cagg_get_by_relid_or_fail(cagg_relid); Cache *hcache = ts_hypertable_cache_pin(); Hypertable *mat_ht = ts_hypertable_cache_get_entry_by_id(hcache, cagg->data.mat_hypertable_id); Assert(mat_ht != NULL); cagg_group_cols = cagg_find_groupingcols(cagg, mat_ht); foreach (lc, cagg_group_cols) { char *group_col = lfirst(lc); astate = accumArrayResult(astate, CStringGetTextDatum(group_col), false, TEXTOID, CurrentMemoryContext); } ts_cache_release(&hcache); PG_RETURN_DATUM(makeArrayResult(astate, CurrentMemoryContext)); } ================================================ FILE: tsl/src/continuous_aggs/utils.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <catalog/pg_collation.h> #include <funcapi.h> #include <parser/analyze.h> #include <parser/parser.h> #include <tcop/tcopprot.h> #include "compat/compat.h" #include "common.h" extern Datum continuous_agg_validate_query(PG_FUNCTION_ARGS); extern Datum continuous_agg_get_bucket_function(PG_FUNCTION_ARGS); extern Datum continuous_agg_get_bucket_function_info(PG_FUNCTION_ARGS); extern Datum continuous_agg_get_grouping_columns(PG_FUNCTION_ARGS); ================================================ FILE: tsl/src/import/CMakeLists.txt ================================================ set(SOURCES) if(USE_UMASH) list(APPEND SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/umash.c) endif() if(SOURCES) # Disable clang-tidy for imported code add_library(target_no_static_code_analysis OBJECT ${SOURCES}) set_target_properties(target_no_static_code_analysis PROPERTIES C_CLANG_TIDY "") target_sources(${TSL_LIBRARY_NAME} PRIVATE $<TARGET_OBJECTS:target_no_static_code_analysis>) endif() ================================================ FILE: tsl/src/import/ts_like_match.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * This file contains source code that was copied and/or modified from * the PostgreSQL database, which is licensed under the open-source * PostgreSQL License. Please see the NOTICE at the top level * directory for a copy of the PostgreSQL License. * * This is a copy of backend/utils/adt/like_match.c from PG 15.0, git commit sha * 2a7ce2e2ce474504a707ec03e128fde66cfb8b48. * It has one modification: the check_stack_depth() check is moved to happen * before recursion to simplify the non-recursive code path. */ /*-------------------- * Match text and pattern, return LIKE_TRUE, LIKE_FALSE, or LIKE_ABORT. * * LIKE_TRUE: they match * LIKE_FALSE: they don't match * LIKE_ABORT: not only don't they match, but the text is too short. * * If LIKE_ABORT is returned, then no suffix of the text can match the * pattern either, so an upper-level % scan can stop scanning now. *-------------------- */ #ifdef MATCH_LOWER #define GETCHAR(t) MATCH_LOWER(t) #else #define GETCHAR(t) (t) #endif static int MatchText(const char *t, int tlen, const char *p, int plen) { /* Fast path for match-everything pattern */ if (plen == 1 && *p == '%') return LIKE_TRUE; /* * In this loop, we advance by char when matching wildcards (and thus on * recursive entry to this function we are properly char-synced). On other * occasions it is safe to advance by byte, as the text and pattern will * be in lockstep. This allows us to perform all comparisons between the * text and pattern on a byte by byte basis, even for multi-byte * encodings. */ while (tlen > 0 && plen > 0) { if (*p == '\\') { /* Next pattern byte must match literally, whatever it is */ NextByte(p, plen); /* ... and there had better be one, per SQL standard */ if (plen <= 0) ereport(ERROR, (errcode(ERRCODE_INVALID_ESCAPE_SEQUENCE), errmsg("LIKE pattern must not end with escape character"))); if (GETCHAR(*p) != GETCHAR(*t)) return LIKE_FALSE; } else if (*p == '%') { char firstpat; /* * % processing is essentially a search for a text position at * which the remainder of the text matches the remainder of the * pattern, using a recursive call to check each potential match. * * If there are wildcards immediately following the %, we can skip * over them first, using the idea that any sequence of N _'s and * one or more %'s is equivalent to N _'s and one % (ie, it will * match any sequence of at least N text characters). In this way * we will always run the recursive search loop using a pattern * fragment that begins with a literal character-to-match, thereby * not recursing more than we have to. */ NextByte(p, plen); while (plen > 0) { if (*p == '%') NextByte(p, plen); else if (*p == '_') { /* If not enough text left to match the pattern, ABORT */ if (tlen <= 0) return LIKE_ABORT; NextChar(t, tlen); NextByte(p, plen); } else break; /* Reached a non-wildcard pattern char */ } /* * If we're at end of pattern, match: we have a trailing % which * matches any remaining text string. */ if (plen <= 0) return LIKE_TRUE; /* * Otherwise, scan for a text position at which we can match the * rest of the pattern. The first remaining pattern char is known * to be a regular or escaped literal character, so we can compare * the first pattern byte to each text byte to avoid recursing * more than we have to. This fact also guarantees that we don't * have to consider a match to the zero-length substring at the * end of the text. */ if (*p == '\\') { if (plen < 2) ereport(ERROR, (errcode(ERRCODE_INVALID_ESCAPE_SEQUENCE), errmsg("LIKE pattern must not end with escape character"))); firstpat = GETCHAR(p[1]); } else firstpat = GETCHAR(*p); while (tlen > 0) { if (GETCHAR(*t) == firstpat) { /* Since this function recurses, it could be driven to stack overflow */ check_stack_depth(); int matched = MatchText(t, tlen, p, plen); if (matched != LIKE_FALSE) return matched; /* TRUE or ABORT */ } NextChar(t, tlen); } /* * End of text with no match, so no point in trying later places * to start matching this pattern. */ return LIKE_ABORT; } else if (*p == '_') { /* _ matches any single character, and we know there is one */ NextChar(t, tlen); NextByte(p, plen); continue; } else if (GETCHAR(*p) != GETCHAR(*t)) { /* non-wildcard pattern char fails to match text char */ return LIKE_FALSE; } /* * Pattern and text match, so advance. * * It is safe to use NextByte instead of NextChar here, even for * multi-byte character sets, because we are not following immediately * after a wildcard character. If we are in the middle of a multibyte * character, we must already have matched at least one byte of the * character from both text and pattern; so we cannot get out-of-sync * on character boundaries. And we know that no backend-legal * encoding allows ASCII characters such as '%' to appear as non-first * bytes of characters, so we won't mistakenly detect a new wildcard. */ NextByte(t, tlen); NextByte(p, plen); } if (tlen > 0) return LIKE_FALSE; /* end of pattern, but not of text */ /* * End of text, but perhaps not of pattern. Match iff the remaining * pattern can match a zero-length string, ie, it's zero or more %'s. */ while (plen > 0 && *p == '%') NextByte(p, plen); if (plen <= 0) return LIKE_TRUE; /* * End of text with no match, so no point in trying later places to start * matching this pattern. */ return LIKE_ABORT; } /* MatchText() */ #ifdef CHAREQ #undef CHAREQ #endif #undef NextChar #undef CopyAdvChar #undef MatchText #undef GETCHAR #ifdef MATCH_LOWER #undef MATCH_LOWER #endif ================================================ FILE: tsl/src/import/umash.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * This file contains source code that was copied and/or modified from * the UMASH hash implementation at https://github.com/backtrace-labs/umash. * * This is a copy of umash.c, git commit sha * fc4c5b6ca1f06c308e96c43aa080bd766238e092. */ #include "umash.h" /* * UMASH is distributed under the MIT license. * * SPDX-License-Identifier: MIT * * Copyright 2020-2022 Backtrace I/O, Inc. * Copyright 2022 Paul Khuong * Copyright 2022 Dougall Johnson * * Permission is hereby granted, free of charge, to any person * obtaining a copy of this software and associated documentation * files (the "Software"), to deal in the Software without * restriction, including without limitation the rights to use, copy, * modify, merge, publish, distribute, sublicense, and/or sell copies * of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be * included in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #if !defined(UMASH_TEST_ONLY) && !defined(NDEBUG) #define NDEBUG #endif /** * -DUMASH_LONG_INPUTS=0 to disable the routine specialised for long * inputs, and -DUMASH_LONG_INPUTS=1 to enable it. If the variable * isn't defined, we try to probe for `umash_long.inc`: that's where * the long input routines are defined. */ #ifndef UMASH_LONG_INPUTS #ifdef __has_include #if __has_include("umash_long.inc") #define UMASH_LONG_INPUTS 1 #endif /* __has_include() */ #endif /* __has_include */ #ifndef UMASH_LONG_INPUTS #define UMASH_LONG_INPUTS 0 #endif /* !UMASH_LONG_INPUTS */ #endif /* !UMASH_LONG_INPUTS */ /* * Default to dynamically dispatching implementations on x86-64 * (there's nothing to dispatch on aarch64). */ #ifndef UMASH_DYNAMIC_DISPATCH #ifdef __x86_64__ #define UMASH_DYNAMIC_DISPATCH 1 #else #define UMASH_DYNAMIC_DISPATCH 0 #endif #endif /* * Enable inline assembly by default when building with recent GCC or * compatible compilers. It should always be safe to disable this * option, although there may be a performance cost. */ #ifndef UMASH_INLINE_ASM #if defined(__clang__) /* * We need clang 8+ for output flags, and 10+ for relaxed vector * constraints. */ #if __clang_major__ >= 10 #define UMASH_INLINE_ASM 1 #else #define UMASH_INLINE_ASM 0 #endif /* __clang_major__ */ #elif defined(__GNUC__) #if __GNUC__ >= 6 #define UMASH_INLINE_ASM 1 #else #define UMASH_INLINE_ASM 0 #endif /* __GNUC__ */ #else #define UMASH_INLINE_ASM 0 #endif #endif #if defined __has_attribute #if __has_attribute(nonstring) #define TS_NONSTRING __attribute__((nonstring)) #else #define TS_NONSTRING #endif #else #define TS_NONSTRING #endif #include <assert.h> #include <string.h> #ifdef __PCLMUL__ /* If we have access to x86 PCLMUL (and some basic SSE). */ #include <immintrin.h> /* We only use 128-bit vector, as pairs of 64-bit integers. */ typedef __m128i v128; #define V128_ZERO { 0 }; static inline v128 v128_create(uint64_t lo, uint64_t hi) { return _mm_set_epi64x(hi, lo); } /* Shift each 64-bit lane left by one bit. */ static inline v128 v128_shift(v128 x) { return _mm_add_epi64(x, x); } /* Computes the 128-bit carryless product of x and y. */ static inline v128 v128_clmul(uint64_t x, uint64_t y) { return _mm_clmulepi64_si128(_mm_cvtsi64_si128(x), _mm_cvtsi64_si128(y), 0); } /* Computes the 128-bit carryless product of the high and low halves of x. */ static inline v128 v128_clmul_cross(v128 x) { return _mm_clmulepi64_si128(x, x, 1); } #elif defined(__ARM_FEATURE_CRYPTO) #include <arm_neon.h> typedef uint64x2_t v128; #define V128_ZERO { 0 }; static inline v128 v128_create(uint64_t lo, uint64_t hi) { return vcombine_u64(vcreate_u64(lo), vcreate_u64(hi)); } static inline v128 v128_shift(v128 x) { return vshlq_n_u64(x, 1); } static inline v128 v128_clmul(uint64_t x, uint64_t y) { return vreinterpretq_u64_p128(vmull_p64(x, y)); } static inline v128 v128_clmul_cross(v128 x) { v128 swapped = vextq_u64(x, x, 1); #if UMASH_INLINE_ASM /* Keep the result out of GPRs. */ __asm__("" : "+w"(swapped)); #endif return v128_clmul(vgetq_lane_u64(x, 0), vgetq_lane_u64(swapped, 0)); } #else #error \ "Unsupported platform: umash requires CLMUL (-mpclmul) on x86-64, or crypto (-march=...+crypto) extensions on aarch64." #endif /* * #define UMASH_STAP_PROBE=1 to insert probe points in public UMASH * functions. * * This functionality depends on Systemtap's SDT header file. */ #if defined(UMASH_STAP_PROBE) && UMASH_STAP_PROBE #include <sys/sdt.h> #else #define DTRACE_PROBE1(lib, name, a0) #define DTRACE_PROBE2(lib, name, a0, a1) #define DTRACE_PROBE3(lib, name, a0, a1, a2) #define DTRACE_PROBE4(lib, name, a0, a1, a2, a3) #endif /* * #define UMASH_SECTION="special_section" to emit all UMASH symbols * in the `special_section` ELF section. */ #if defined(UMASH_SECTION) && defined(__GNUC__) #define FN __attribute__((__section__(UMASH_SECTION))) #else #define FN #endif /* * Defining UMASH_TEST_ONLY switches to a debug build with internal * symbols exposed. */ #ifdef UMASH_TEST_ONLY #define TEST_DEF FN #include "t/umash_test_only.h" #else #define TEST_DEF static FN #endif #ifdef __GNUC__ #define LIKELY(X) __builtin_expect(!!(X), 1) #define UNLIKELY(X) __builtin_expect(!!(X), 0) #define HOT __attribute__((__hot__)) #define COLD __attribute__((__cold__)) #else #define LIKELY(X) X #define UNLIKELY(X) X #define HOT #define COLD #endif #define ARRAY_SIZE(ARR) (sizeof(ARR) / sizeof(ARR[0])) #define BLOCK_SIZE (sizeof(uint64_t) * UMASH_OH_PARAM_COUNT) /* * We derive independent short hashes by offsetting the constant array * by four u64s. In theory, any positive even number works, but this * is the constant we used in an earlier incarnation, and it works. */ #define OH_SHORT_HASH_SHIFT 4 /* Incremental UMASH consumes 16 bytes at a time. */ #define INCREMENTAL_GRANULARITY 16 /** * Modular arithmetic utilities. * * The code below uses GCC extensions. It should be possible to add * support for other compilers. */ #if !defined(__x86_64__) || !UMASH_INLINE_ASM static inline void mul128(uint64_t x, uint64_t y, uint64_t *hi, uint64_t *lo) { __uint128_t product = x; product *= y; *hi = product >> 64; *lo = product; return; } #else static inline void mul128(uint64_t x, uint64_t y, uint64_t *hi, uint64_t *lo) { uint64_t mulhi, mullo; __asm__("mul %3" : "=a"(mullo), "=d"(mulhi) : "%a"(x), "r"(y) : "cc"); *hi = mulhi; *lo = mullo; return; } #endif TEST_DEF inline uint64_t add_mod_fast(uint64_t x, uint64_t y) { unsigned long long sum; /* If `sum` overflows, `sum + 8` does not. */ return (__builtin_uaddll_overflow(x, y, &sum) ? sum + 8 : sum); } static FN COLD uint64_t add_mod_slow_slow_path(uint64_t sum, uint64_t fixup) { /* Reduce sum, mod 2**64 - 8. */ sum = (sum >= (uint64_t)-8) ? sum + 8 : sum; /* sum < 2**64 - 8, so this doesn't overflow. */ sum += fixup; /* Reduce again. */ sum = (sum >= (uint64_t)-8) ? sum + 8 : sum; return sum; } TEST_DEF inline uint64_t add_mod_slow(uint64_t x, uint64_t y) { unsigned long long sum; uint64_t fixup = 0; /* x + y \equiv sum + fixup */ if (__builtin_uaddll_overflow(x, y, &sum)) fixup = 8; /* * We must ensure `sum + fixup < 2**64 - 8`. * * We want a conditional branch here, but not in the * overflowing add: overflows happen roughly half the time on * pseudorandom inputs, but `sum < 2**64 - 16` is almost * always true, for pseudorandom `sum`. */ if (LIKELY(sum < (uint64_t)-16)) return sum + fixup; #ifdef UMASH_INLINE_ASM /* * Some compilers like to compile the likely branch above with * conditional moves or predication. Insert a compiler barrier * in the slow path here to force a branch. */ __asm__("" : "+r"(sum)); #endif return add_mod_slow_slow_path(sum, fixup); } TEST_DEF inline uint64_t mul_mod_fast(uint64_t m, uint64_t x) { uint64_t hi, lo; mul128(m, x, &hi, &lo); return add_mod_fast(lo, 8 * hi); } TEST_DEF inline uint64_t horner_double_update(uint64_t acc, uint64_t m0, uint64_t m1, uint64_t x, uint64_t y) { acc = add_mod_fast(acc, x); return add_mod_slow(mul_mod_fast(m0, acc), mul_mod_fast(m1, y)); } /** * Salsa20 stream generator, used to derive struct umash_param. * * Slightly prettified version of D. J. Bernstein's public domain NaCL * (version 20110121), without paying any attention to constant time * execution or any other side-channel. */ static inline uint32_t rotate(uint32_t u, int c) { return (u << c) | (u >> (32 - c)); } static inline uint32_t load_littleendian(const void *buf) { uint32_t ret = 0; uint8_t x[4]; memcpy(x, buf, sizeof(x)); for (size_t i = 0; i < 4; i++) ret |= (uint32_t)x[i] << (8 * i); return ret; } static inline void store_littleendian(void *dst, uint32_t u) { for (size_t i = 0; i < 4; i++) { uint8_t lo = u; memcpy(dst, &lo, 1); u >>= 8; dst = (char *)dst + 1; } return; } static FN void core_salsa20(char *out, const uint8_t in[static 16], const uint8_t key[static 32], const uint8_t constant[16]) { enum { ROUNDS = 20 }; uint32_t x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15; uint32_t j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15; j0 = x0 = load_littleendian(constant + 0); j1 = x1 = load_littleendian(key + 0); j2 = x2 = load_littleendian(key + 4); j3 = x3 = load_littleendian(key + 8); j4 = x4 = load_littleendian(key + 12); j5 = x5 = load_littleendian(constant + 4); j6 = x6 = load_littleendian(in + 0); j7 = x7 = load_littleendian(in + 4); j8 = x8 = load_littleendian(in + 8); j9 = x9 = load_littleendian(in + 12); j10 = x10 = load_littleendian(constant + 8); j11 = x11 = load_littleendian(key + 16); j12 = x12 = load_littleendian(key + 20); j13 = x13 = load_littleendian(key + 24); j14 = x14 = load_littleendian(key + 28); j15 = x15 = load_littleendian(constant + 12); for (size_t i = 0; i < ROUNDS; i += 2) { x4 ^= rotate(x0 + x12, 7); x8 ^= rotate(x4 + x0, 9); x12 ^= rotate(x8 + x4, 13); x0 ^= rotate(x12 + x8, 18); x9 ^= rotate(x5 + x1, 7); x13 ^= rotate(x9 + x5, 9); x1 ^= rotate(x13 + x9, 13); x5 ^= rotate(x1 + x13, 18); x14 ^= rotate(x10 + x6, 7); x2 ^= rotate(x14 + x10, 9); x6 ^= rotate(x2 + x14, 13); x10 ^= rotate(x6 + x2, 18); x3 ^= rotate(x15 + x11, 7); x7 ^= rotate(x3 + x15, 9); x11 ^= rotate(x7 + x3, 13); x15 ^= rotate(x11 + x7, 18); x1 ^= rotate(x0 + x3, 7); x2 ^= rotate(x1 + x0, 9); x3 ^= rotate(x2 + x1, 13); x0 ^= rotate(x3 + x2, 18); x6 ^= rotate(x5 + x4, 7); x7 ^= rotate(x6 + x5, 9); x4 ^= rotate(x7 + x6, 13); x5 ^= rotate(x4 + x7, 18); x11 ^= rotate(x10 + x9, 7); x8 ^= rotate(x11 + x10, 9); x9 ^= rotate(x8 + x11, 13); x10 ^= rotate(x9 + x8, 18); x12 ^= rotate(x15 + x14, 7); x13 ^= rotate(x12 + x15, 9); x14 ^= rotate(x13 + x12, 13); x15 ^= rotate(x14 + x13, 18); } x0 += j0; x1 += j1; x2 += j2; x3 += j3; x4 += j4; x5 += j5; x6 += j6; x7 += j7; x8 += j8; x9 += j9; x10 += j10; x11 += j11; x12 += j12; x13 += j13; x14 += j14; x15 += j15; store_littleendian(out + 0, x0); store_littleendian(out + 4, x1); store_littleendian(out + 8, x2); store_littleendian(out + 12, x3); store_littleendian(out + 16, x4); store_littleendian(out + 20, x5); store_littleendian(out + 24, x6); store_littleendian(out + 28, x7); store_littleendian(out + 32, x8); store_littleendian(out + 36, x9); store_littleendian(out + 40, x10); store_littleendian(out + 44, x11); store_littleendian(out + 48, x12); store_littleendian(out + 52, x13); store_littleendian(out + 56, x14); store_littleendian(out + 60, x15); return; } TEST_DEF void salsa20_stream( void *dst, size_t len, const uint8_t nonce[static 8], const uint8_t key[static 32]) { static const uint8_t TS_NONSTRING sigma[16] = "expand 32-byte k"; uint8_t in[16]; if (len == 0) return; memcpy(in, nonce, 8); memset(in + 8, 0, 8); while (len >= 64) { unsigned int u; core_salsa20(dst, in, key, sigma); u = 1; for (size_t i = 8; i < 16; i++) { u += in[i]; in[i] = u; u >>= 8; } dst = (char *)dst + 64; len -= 64; } if (len > 0) { char block[64]; core_salsa20(block, in, key, sigma); memcpy(dst, block, len); } return; } #if defined(UMASH_TEST_ONLY) || UMASH_LONG_INPUTS #include "umash_long.inc" #endif /** * OH block compression. */ TEST_DEF struct umash_oh oh_varblock(const uint64_t *params, uint64_t tag, const void *block, size_t n_bytes) { struct umash_oh ret; v128 acc = V128_ZERO; /* The final block processes `remaining > 0` bytes. */ size_t remaining = 1 + ((n_bytes - 1) % sizeof(v128)); size_t end_full_pairs = (n_bytes - remaining) / sizeof(uint64_t); const void *last_ptr = (const char *)block + n_bytes - sizeof(v128); size_t i; for (i = 0; i < end_full_pairs; i += 2) { v128 x, k; memcpy(&x, block, sizeof(x)); block = (const char *)block + sizeof(x); memcpy(&k, ¶ms[i], sizeof(k)); x ^= k; acc ^= v128_clmul_cross(x); } memcpy(&ret, &acc, sizeof(ret)); /* Compress the final (potentially partial) pair. */ { uint64_t x, y, enh_hi, enh_lo; memcpy(&x, last_ptr, sizeof(x)); last_ptr = (const char *)last_ptr + sizeof(x); memcpy(&y, last_ptr, sizeof(y)); x += params[i]; y += params[i + 1]; mul128(x, y, &enh_hi, &enh_lo); enh_hi += tag; ret.bits[0] ^= enh_lo; ret.bits[1] ^= enh_hi ^ enh_lo; } return ret; } TEST_DEF void oh_varblock_fprint(struct umash_oh dst[static restrict 2], const uint64_t *restrict params, uint64_t tag, const void *restrict block, size_t n_bytes) { v128 acc = V128_ZERO; /* Base umash */ v128 acc_shifted = V128_ZERO; /* Accumulates shifted values */ v128 lrc; /* The final block processes `remaining > 0` bytes. */ size_t remaining = 1 + ((n_bytes - 1) % sizeof(v128)); size_t end_full_pairs = (n_bytes - remaining) / sizeof(uint64_t); const void *last_ptr = (const char *)block + n_bytes - sizeof(v128); size_t i; lrc = v128_create(params[UMASH_OH_PARAM_COUNT], params[UMASH_OH_PARAM_COUNT + 1]); for (i = 0; i < end_full_pairs; i += 2) { v128 x, k; memcpy(&x, block, sizeof(x)); block = (const char *)block + sizeof(x); memcpy(&k, ¶ms[i], sizeof(k)); x ^= k; lrc ^= x; x = v128_clmul_cross(x); acc ^= x; if (i + 2 >= end_full_pairs) break; acc_shifted ^= x; acc_shifted = v128_shift(acc_shifted); } /* * Update the LRC for the last chunk before treating it * specially. */ { v128 x, k; memcpy(&x, last_ptr, sizeof(x)); memcpy(&k, ¶ms[end_full_pairs], sizeof(k)); lrc ^= x ^ k; } acc_shifted ^= acc; acc_shifted = v128_shift(acc_shifted); acc_shifted ^= v128_clmul_cross(lrc); memcpy(&dst[0], &acc, sizeof(dst[0])); memcpy(&dst[1], &acc_shifted, sizeof(dst[1])); { uint64_t x, y, kx, ky, enh_hi, enh_lo; memcpy(&x, last_ptr, sizeof(x)); last_ptr = (const char *)last_ptr + sizeof(x); memcpy(&y, last_ptr, sizeof(y)); kx = x + params[end_full_pairs]; ky = y + params[end_full_pairs + 1]; mul128(kx, ky, &enh_hi, &enh_lo); enh_hi += tag; enh_hi ^= enh_lo; dst[0].bits[0] ^= enh_lo; dst[0].bits[1] ^= enh_hi; dst[1].bits[0] ^= enh_lo; dst[1].bits[1] ^= enh_hi; } return; } /** * Returns `then` if `cond` is true, `otherwise` if false. * * This noise helps compiler emit conditional moves. */ static inline const void * select_ptr(bool cond, const void *then, const void *otherwise) { const char *ret; #if UMASH_INLINE_ASM /* Force strict evaluation of both arguments. */ __asm__("" ::"r"(then), "r"(otherwise)); #endif ret = (cond) ? then : otherwise; #if UMASH_INLINE_ASM /* And also force the result to be materialised with a blackhole. */ __asm__("" : "+r"(ret)); #endif return ret; } /** * Short UMASH (<= 8 bytes). */ TEST_DEF inline uint64_t vec_to_u64(const void *data, size_t n_bytes) { const char zeros[2] = { 0 }; uint32_t hi, lo; /* * If there are at least 4 bytes to read, read the first 4 in * `lo`, and the last 4 in `hi`. This covers the whole range, * since `n_bytes` is at most 8. */ if (LIKELY(n_bytes >= sizeof(lo))) { memcpy(&lo, data, sizeof(lo)); memcpy(&hi, (const char *)data + n_bytes - sizeof(hi), sizeof(hi)); } else { /* 0 <= n_bytes < 4. Decode the size in binary. */ uint16_t word; uint8_t byte; /* * If the size is odd, load the first byte in `byte`; * otherwise, load in a zero. */ memcpy(&byte, select_ptr(n_bytes & 1, data, zeros), 1); lo = byte; /* * If the size is 2 or 3, load the last two bytes in `word`; * otherwise, load in a zero. */ memcpy(&word, select_ptr(n_bytes & 2, (const char *)data + n_bytes - 2, zeros), 2); /* * We have now read `bytes[0 ... n_bytes - 1]` * exactly once without overwriting any data. */ hi = word; } /* * Mix `hi` with the `lo` bits: SplitMix64 seems to have * trouble with the top 4 bits. */ return ((uint64_t)hi << 32) | (lo + hi); } TEST_DEF uint64_t umash_short(const uint64_t *params, uint64_t seed, const void *data, size_t n_bytes) { uint64_t h; seed += params[n_bytes]; h = vec_to_u64(data, n_bytes); h ^= h >> 30; h *= 0xbf58476d1ce4e5b9ULL; h = (h ^ seed) ^ (h >> 27); h *= 0x94d049bb133111ebULL; h ^= h >> 31; return h; } static FN struct umash_fp umash_fp_short(const uint64_t *params, uint64_t seed, const void *data, size_t n_bytes) { struct umash_fp ret; uint64_t h; ret.hash[0] = seed + params[n_bytes]; ret.hash[1] = seed + params[n_bytes + OH_SHORT_HASH_SHIFT]; h = vec_to_u64(data, n_bytes); h ^= h >> 30; h *= 0xbf58476d1ce4e5b9ULL; h ^= h >> 27; #define TAIL(i) \ do { \ ret.hash[i] ^= h; \ ret.hash[i] *= 0x94d049bb133111ebULL; \ ret.hash[i] ^= ret.hash[i] >> 31; \ } while (0) TAIL(0); TAIL(1); #undef TAIL return ret; } /** * Rotates `x` left by `n` bits. */ static inline uint64_t rotl64(uint64_t x, int n) { return (x << n) | (x >> (64 - n)); } TEST_DEF inline uint64_t finalize(uint64_t x) { return (x ^ rotl64(x, 8)) ^ rotl64(x, 33); } TEST_DEF uint64_t umash_medium(const uint64_t multipliers[static 2], const uint64_t *oh, uint64_t seed, const void *data, size_t n_bytes) { uint64_t enh_hi, enh_lo; { uint64_t x, y; memcpy(&x, data, sizeof(x)); memcpy(&y, (const char *)data + n_bytes - sizeof(y), sizeof(y)); x += oh[0]; y += oh[1]; mul128(x, y, &enh_hi, &enh_lo); enh_hi += seed ^ n_bytes; } enh_hi ^= enh_lo; return finalize(horner_double_update( /*acc=*/0, multipliers[0], multipliers[1], enh_lo, enh_hi)); } static FN struct umash_fp umash_fp_medium(const uint64_t multipliers[static 2][2], const uint64_t *oh, uint64_t seed, const void *data, size_t n_bytes) { struct umash_fp ret; const uint64_t offset = seed ^ n_bytes; uint64_t enh_hi, enh_lo; union { v128 v; uint64_t u64[2]; } mixed_lrc; uint64_t lrc[2] = { oh[UMASH_OH_PARAM_COUNT], oh[UMASH_OH_PARAM_COUNT + 1] }; uint64_t x, y; uint64_t a, b; /* Expand the 9-16 bytes to 16. */ memcpy(&x, data, sizeof(x)); memcpy(&y, (const char *)data + n_bytes - sizeof(y), sizeof(y)); a = oh[0]; b = oh[1]; lrc[0] ^= x ^ a; lrc[1] ^= y ^ b; mixed_lrc.v = v128_clmul(lrc[0], lrc[1]); a += x; b += y; mul128(a, b, &enh_hi, &enh_lo); enh_hi += offset; enh_hi ^= enh_lo; ret.hash[0] = finalize(horner_double_update( /*acc=*/0, multipliers[0][0], multipliers[0][1], enh_lo, enh_hi)); ret.hash[1] = finalize(horner_double_update(/*acc=*/0, multipliers[1][0], multipliers[1][1], enh_lo ^ mixed_lrc.u64[0], enh_hi ^ mixed_lrc.u64[1])); return ret; } TEST_DEF uint64_t umash_long(const uint64_t multipliers[static 2], const uint64_t *oh, uint64_t seed, const void *data, size_t n_bytes) { uint64_t acc = 0; /* * umash_long.inc defines this variable when the long input * routine is enabled. */ #ifdef UMASH_MULTIPLE_BLOCKS_THRESHOLD if (UNLIKELY(n_bytes >= UMASH_MULTIPLE_BLOCKS_THRESHOLD)) { size_t n_block = n_bytes / BLOCK_SIZE; const void *remaining; n_bytes %= BLOCK_SIZE; remaining = (const char *)data + (n_block * BLOCK_SIZE); acc = umash_multiple_blocks(acc, multipliers, oh, seed, data, n_block); data = remaining; if (n_bytes == 0) goto finalize; goto last_block; } #else /* Avoid warnings about the unused labels. */ if (0) { goto last_block; goto finalize; } #endif while (n_bytes > BLOCK_SIZE) { struct umash_oh compressed; compressed = oh_varblock(oh, seed, data, BLOCK_SIZE); data = (const char *)data + BLOCK_SIZE; n_bytes -= BLOCK_SIZE; acc = horner_double_update(acc, multipliers[0], multipliers[1], compressed.bits[0], compressed.bits[1]); } last_block: /* Do the final block. */ { struct umash_oh compressed; seed ^= (uint8_t)n_bytes; compressed = oh_varblock(oh, seed, data, n_bytes); acc = horner_double_update(acc, multipliers[0], multipliers[1], compressed.bits[0], compressed.bits[1]); } finalize: return finalize(acc); } TEST_DEF struct umash_fp umash_fp_long(const uint64_t multipliers[static 2][2], const uint64_t *oh, uint64_t seed, const void *data, size_t n_bytes) { struct umash_oh compressed[2]; struct umash_fp ret; uint64_t acc[2] = { 0, 0 }; #ifdef UMASH_MULTIPLE_BLOCKS_THRESHOLD if (UNLIKELY(n_bytes >= UMASH_MULTIPLE_BLOCKS_THRESHOLD)) { struct umash_fp poly = { .hash = { 0, 0 } }; size_t n_block = n_bytes / BLOCK_SIZE; const void *remaining; n_bytes %= BLOCK_SIZE; remaining = (const char *)data + (n_block * BLOCK_SIZE); poly = umash_fprint_multiple_blocks( poly, multipliers, oh, seed, data, n_block); acc[0] = poly.hash[0]; acc[1] = poly.hash[1]; data = remaining; if (n_bytes == 0) goto finalize; goto last_block; } #else /* Avoid warnings about the unused labels. */ if (0) { goto last_block; goto finalize; } #endif while (n_bytes > BLOCK_SIZE) { oh_varblock_fprint(compressed, oh, seed, data, BLOCK_SIZE); #define UPDATE(i) \ acc[i] = horner_double_update(acc[i], multipliers[i][0], multipliers[i][1], \ compressed[i].bits[0], compressed[i].bits[1]) UPDATE(0); UPDATE(1); #undef UPDATE data = (const char *)data + BLOCK_SIZE; n_bytes -= BLOCK_SIZE; } last_block: oh_varblock_fprint(compressed, oh, seed ^ (uint8_t)n_bytes, data, n_bytes); #define FINAL(i) \ do { \ acc[i] = horner_double_update(acc[i], multipliers[i][0], \ multipliers[i][1], compressed[i].bits[0], compressed[i].bits[1]); \ } while (0) FINAL(0); FINAL(1); #undef FINAL finalize: ret.hash[0] = finalize(acc[0]); ret.hash[1] = finalize(acc[1]); return ret; } static FN bool value_is_repeated(const uint64_t *values, size_t n, uint64_t needle) { for (size_t i = 0; i < n; i++) { if (values[i] == needle) return true; } return false; } FN bool umash_params_prepare(struct umash_params *params) { static const uint64_t modulo = (1UL << 61) - 1; /* * The polynomial parameters have two redundant fields (for * the pre-squared multipliers). Use them as our source of * extra entropy if needed. */ uint64_t buf[] = { params->poly[0][0], params->poly[1][0] }; size_t buf_idx = 0; #define GET_RANDOM(DST) \ do { \ if (buf_idx >= ARRAY_SIZE(buf)) \ return false; \ \ (DST) = buf[buf_idx++]; \ } while (0) /* Check the polynomial multipliers: we don't want 0s. */ for (size_t i = 0; i < ARRAY_SIZE(params->poly); i++) { uint64_t f = params->poly[i][1]; while (true) { /* * Zero out bits and use rejection sampling to * guarantee uniformity. */ f &= (1UL << 61) - 1; if (f != 0 && f < modulo) break; GET_RANDOM(f); } /* We can work in 2**64 - 8 and reduce after the fact. */ params->poly[i][0] = mul_mod_fast(f, f) % modulo; params->poly[i][1] = f; } /* Avoid repeated OH noise values. */ for (size_t i = 0; i < ARRAY_SIZE(params->oh); i++) { while (value_is_repeated(params->oh, i, params->oh[i])) GET_RANDOM(params->oh[i]); } return true; } FN void umash_params_derive(struct umash_params *params, uint64_t bits, const void *key) { uint8_t umash_key[32] TS_NONSTRING = "Do not use UMASH VS adversaries."; if (key != NULL) memcpy(umash_key, key, sizeof(umash_key)); while (true) { uint8_t nonce[8]; for (size_t i = 0; i < 8; i++) nonce[i] = bits >> (8 * i); salsa20_stream(params, sizeof(*params), nonce, umash_key); if (umash_params_prepare(params)) return; /* * This should practically never fail, so really * shouldn't happen multiple times. If it does, an * infinite loop is as good as anything else. */ bits++; } } /* * Updates the polynomial state at the end of a block. */ static FN void sink_update_poly(struct umash_sink *sink) { uint64_t oh0, oh1; oh0 = sink->oh_acc.bits[0]; oh1 = sink->oh_acc.bits[1]; sink->poly_state[0].acc = horner_double_update(sink->poly_state[0].acc, sink->poly_state[0].mul[0], sink->poly_state[0].mul[1], oh0, oh1); sink->oh_acc = (struct umash_oh) { .bits = { 0 } }; if (sink->hash_wanted == 0) return; oh0 = sink->oh_twisted.acc.bits[0]; oh1 = sink->oh_twisted.acc.bits[1]; sink->poly_state[1].acc = horner_double_update(sink->poly_state[1].acc, sink->poly_state[1].mul[0], sink->poly_state[1].mul[1], oh0, oh1); sink->oh_twisted = (struct umash_twisted_oh) { .lrc = { sink->oh[UMASH_OH_PARAM_COUNT], sink->oh[UMASH_OH_PARAM_COUNT + 1] } }; return; } /* * Updates the OH state with 16 bytes of data. If `final` is true, we * are definitely consuming the last chunk in the input. */ static FN void sink_consume_buf( struct umash_sink *sink, const char buf[static INCREMENTAL_GRANULARITY], bool final) { const size_t buf_begin = sizeof(sink->buf) - INCREMENTAL_GRANULARITY; const size_t param = sink->oh_iter; const uint64_t k0 = sink->oh[param]; const uint64_t k1 = sink->oh[param + 1]; uint64_t x, y; /* Use GPR loads to avoid forwarding stalls. */ memcpy(&x, buf, sizeof(x)); memcpy(&y, buf + sizeof(x), sizeof(y)); /* All but the last 16-byte chunk of each block goes through PH. */ if (sink->oh_iter < UMASH_OH_PARAM_COUNT - 2 && !final) { v128 acc, h, twisted_acc, prev; uint64_t m0, m1; m0 = x ^ k0; m1 = y ^ k1; memcpy(&acc, &sink->oh_acc, sizeof(acc)); h = v128_clmul(m0, m1); acc ^= h; memcpy(&sink->oh_acc, &acc, sizeof(acc)); if (sink->hash_wanted == 0) goto next; sink->oh_twisted.lrc[0] ^= m0; sink->oh_twisted.lrc[1] ^= m1; memcpy(&twisted_acc, &sink->oh_twisted.acc, sizeof(twisted_acc)); memcpy(&prev, sink->oh_twisted.prev, sizeof(prev)); twisted_acc ^= prev; twisted_acc = v128_shift(twisted_acc); memcpy(&sink->oh_twisted.acc, &twisted_acc, sizeof(twisted_acc)); memcpy(&sink->oh_twisted.prev, &h, sizeof(h)); } else { /* The last chunk is combined with the size tag with ENH. */ uint64_t tag = sink->seed ^ (uint8_t)(sink->block_size + sink->bufsz); uint64_t enh_hi, enh_lo; mul128(x + k0, y + k1, &enh_hi, &enh_lo); enh_hi += tag; enh_hi ^= enh_lo; if (sink->hash_wanted != 0) { union { v128 vec; uint64_t h[2]; } lrc_hash; uint64_t lrc0, lrc1; uint64_t oh0, oh1; uint64_t oh_twisted0, oh_twisted1; lrc0 = sink->oh_twisted.lrc[0] ^ x ^ k0; lrc1 = sink->oh_twisted.lrc[1] ^ y ^ k1; lrc_hash.vec = v128_clmul(lrc0, lrc1); oh_twisted0 = sink->oh_twisted.acc.bits[0]; oh_twisted1 = sink->oh_twisted.acc.bits[1]; oh0 = sink->oh_acc.bits[0]; oh1 = sink->oh_acc.bits[1]; oh0 ^= oh_twisted0; oh0 <<= 1; oh1 ^= oh_twisted1; oh1 <<= 1; oh0 ^= lrc_hash.h[0]; oh1 ^= lrc_hash.h[1]; sink->oh_twisted.acc.bits[0] = oh0 ^ enh_lo; sink->oh_twisted.acc.bits[1] = oh1 ^ enh_hi; } sink->oh_acc.bits[0] ^= enh_lo; sink->oh_acc.bits[1] ^= enh_hi; } next: memmove(&sink->buf, buf, buf_begin); sink->block_size += sink->bufsz; sink->bufsz = 0; sink->oh_iter += 2; if (sink->oh_iter == UMASH_OH_PARAM_COUNT || final) { sink_update_poly(sink); sink->block_size = 0; sink->oh_iter = 0; } return; } /** * Hashes full 256-byte blocks into a sink that just dumped its OH * state in the toplevel polynomial hash and reset the block state. */ static FN size_t block_sink_update(struct umash_sink *sink, const void *data, size_t n_bytes) { size_t consumed = 0; assert(n_bytes >= BLOCK_SIZE); assert(sink->bufsz == 0); assert(sink->block_size == 0); assert(sink->oh_iter == 0); #ifdef UMASH_MULTIPLE_BLOCKS_THRESHOLD if (UNLIKELY(n_bytes > UMASH_MULTIPLE_BLOCKS_THRESHOLD)) { /* * We leave the last block (partial or not) for the * caller: incremental hashing must save some state * at the end of a block. */ size_t n_blocks = (n_bytes - 1) / BLOCK_SIZE; if (sink->hash_wanted != 0) { const uint64_t multipliers[2][2] = { [0][0] = sink->poly_state[0].mul[0], [0][1] = sink->poly_state[0].mul[1], [1][0] = sink->poly_state[1].mul[0], [1][1] = sink->poly_state[1].mul[1], }; struct umash_fp poly = { .hash[0] = sink->poly_state[0].acc, .hash[1] = sink->poly_state[1].acc, }; poly = umash_fprint_multiple_blocks( poly, multipliers, sink->oh, sink->seed, data, n_blocks); sink->poly_state[0].acc = poly.hash[0]; sink->poly_state[1].acc = poly.hash[1]; } else { sink->poly_state[0].acc = umash_multiple_blocks( sink->poly_state[0].acc, sink->poly_state[0].mul, sink->oh, sink->seed, data, n_blocks); } return n_blocks * BLOCK_SIZE; } #endif while (n_bytes > BLOCK_SIZE) { /* * Is this worth unswitching? Not obviously, given * the amount of work in one OH block. */ if (sink->hash_wanted != 0) { struct umash_oh hashes[2]; oh_varblock_fprint( hashes, sink->oh, sink->seed, data, BLOCK_SIZE); sink->oh_acc = hashes[0]; sink->oh_twisted.acc = hashes[1]; } else { sink->oh_acc = oh_varblock(sink->oh, sink->seed, data, BLOCK_SIZE); } sink_update_poly(sink); consumed += BLOCK_SIZE; data = (const char *)data + BLOCK_SIZE; n_bytes -= BLOCK_SIZE; } return consumed; } FN void umash_sink_update(struct umash_sink *sink, const void *data, size_t n_bytes) { const size_t buf_begin = sizeof(sink->buf) - INCREMENTAL_GRANULARITY; size_t remaining = INCREMENTAL_GRANULARITY - sink->bufsz; DTRACE_PROBE4(libumash, umash_sink_update, sink, remaining, data, n_bytes); if (n_bytes < remaining) { memcpy(&sink->buf[buf_begin + sink->bufsz], data, n_bytes); sink->bufsz += n_bytes; return; } memcpy(&sink->buf[buf_begin + sink->bufsz], data, remaining); data = (const char *)data + remaining; n_bytes -= remaining; /* We know we're hashing at least 16 bytes. */ sink->large_umash = true; sink->bufsz = INCREMENTAL_GRANULARITY; /* * We can't compress a 16-byte buffer until we know whether * data is coming: the last 16-byte chunk goes to `NH` instead * of `PH`. We could try to detect when the buffer is the * last chunk in a block and immediately go to `NH`, but it * seems more robust to always let the stores settle before we * read them, just in case the combination is bad for forwarding. */ if (n_bytes == 0) return; sink_consume_buf(sink, sink->buf + buf_begin, /*final=*/false); while (n_bytes > INCREMENTAL_GRANULARITY) { size_t consumed; if (sink->oh_iter == 0 && n_bytes > BLOCK_SIZE) { consumed = block_sink_update(sink, data, n_bytes); assert(consumed >= BLOCK_SIZE); /* * Save the tail of the data we just consumed * in `sink->buf[0 ... buf_begin - 1]`: the * final digest may need those bytes for its * redundant read. */ memcpy(sink->buf, (const char *)data + (consumed - INCREMENTAL_GRANULARITY), buf_begin); } else { consumed = INCREMENTAL_GRANULARITY; sink->bufsz = INCREMENTAL_GRANULARITY; sink_consume_buf(sink, data, /*final=*/false); } n_bytes -= consumed; data = (const char *)data + consumed; } memcpy(&sink->buf[buf_begin], data, n_bytes); sink->bufsz = n_bytes; return; } FN uint64_t umash_full(const struct umash_params *params, uint64_t seed, int which, const void *data, size_t n_bytes) { DTRACE_PROBE4(libumash, umash_full, params, which, data, n_bytes); /* * We don't (yet) implement code that only evaluates the * second hash. We don't currently use that logic, and it's * about to become a bit more complex, so let's just go for a * full fingerprint and take what we need. * * umash_full is also rarely used that way: usually we want * either the main hash, or the full fingerprint. */ if (UNLIKELY(which != 0)) { struct umash_fp fp; fp = umash_fprint(params, seed, data, n_bytes); return fp.hash[1]; } /* * It's not that short inputs are necessarily more likely, but * we want to make sure they fall through correctly to * minimise latency. */ if (LIKELY(n_bytes <= sizeof(v128))) { if (LIKELY(n_bytes <= sizeof(uint64_t))) return umash_short(params->oh, seed, data, n_bytes); return umash_medium(params->poly[0], params->oh, seed, data, n_bytes); } return umash_long(params->poly[0], params->oh, seed, data, n_bytes); } FN struct umash_fp umash_fprint( const struct umash_params *params, uint64_t seed, const void *data, size_t n_bytes) { DTRACE_PROBE3(libumash, umash_fprint, params, data, n_bytes); if (LIKELY(n_bytes <= sizeof(v128))) { if (LIKELY(n_bytes <= sizeof(uint64_t))) return umash_fp_short(params->oh, seed, data, n_bytes); return umash_fp_medium(params->poly, params->oh, seed, data, n_bytes); } return umash_fp_long(params->poly, params->oh, seed, data, n_bytes); } FN void umash_init(struct umash_state *state, const struct umash_params *params, uint64_t seed, int which) { which = (which == 0) ? 0 : 1; DTRACE_PROBE3(libumash, umash_init, state, params, which); state->sink = (struct umash_sink) { .poly_state[0] = { .mul = { params->poly[0][0], params->poly[0][1], }, }, .poly_state[1]= { .mul = { params->poly[1][0], params->poly[1][1], }, }, .oh = params->oh, .hash_wanted = which, .oh_twisted.lrc = { params->oh[UMASH_OH_PARAM_COUNT], params->oh[UMASH_OH_PARAM_COUNT + 1] }, .seed = seed, }; return; } FN void umash_fp_init( struct umash_fp_state *state, const struct umash_params *params, uint64_t seed) { DTRACE_PROBE2(libumash, umash_fp_init, state, params); state->sink = (struct umash_sink) { .poly_state[0] = { .mul = { params->poly[0][0], params->poly[0][1], }, }, .poly_state[1]= { .mul = { params->poly[1][0], params->poly[1][1], }, }, .oh = params->oh, .hash_wanted = 2, .oh_twisted.lrc = { params->oh[UMASH_OH_PARAM_COUNT], params->oh[UMASH_OH_PARAM_COUNT + 1] }, .seed = seed, }; return; } /** * Pumps any last block out of the incremental state. */ static FN void digest_flush(struct umash_sink *sink) { if (sink->bufsz > 0) sink_consume_buf(sink, &sink->buf[sink->bufsz], /*final=*/true); return; } /** * Finalizes a digest out of `sink`'s current state. * * The `sink` must be `digest_flush`ed if it is a `large_umash`. * * @param index 0 to return the first (only, if hashing) value, 1 for the * second independent value for fingerprinting. */ static FN uint64_t digest(const struct umash_sink *sink, int index) { const size_t buf_begin = sizeof(sink->buf) - INCREMENTAL_GRANULARITY; const size_t shift = (index == 0) ? 0 : OH_SHORT_HASH_SHIFT; if (sink->large_umash) return finalize(sink->poly_state[index].acc); if (sink->bufsz <= sizeof(uint64_t)) return umash_short( &sink->oh[shift], sink->seed, &sink->buf[buf_begin], sink->bufsz); return umash_medium(sink->poly_state[index].mul, sink->oh, sink->seed, &sink->buf[buf_begin], sink->bufsz); } static FN struct umash_fp fp_digest_sink(const struct umash_sink *sink) { struct umash_sink copy; struct umash_fp ret; const size_t buf_begin = sizeof(sink->buf) - INCREMENTAL_GRANULARITY; if (sink->large_umash) { copy = *sink; digest_flush(©); sink = © } else if (sink->bufsz <= sizeof(uint64_t)) { return umash_fp_short( sink->oh, sink->seed, &sink->buf[buf_begin], sink->bufsz); } else { const struct umash_params *params; /* * Back out the params struct from our pointer to its * `oh` member. */ params = (const void *)((const char *)sink->oh - __builtin_offsetof(struct umash_params, oh)); return umash_fp_medium(params->poly, sink->oh, sink->seed, &sink->buf[buf_begin], sink->bufsz); } for (size_t i = 0; i < ARRAY_SIZE(ret.hash); i++) ret.hash[i] = digest(sink, i); return ret; } FN uint64_t umash_digest(const struct umash_state *state) { struct umash_sink copy; const struct umash_sink *sink = &state->sink; DTRACE_PROBE1(libumash, umash_digest, state); if (sink->hash_wanted == 1) { struct umash_fp fp; fp = fp_digest_sink(sink); return fp.hash[1]; } if (sink->large_umash) { copy = *sink; digest_flush(©); sink = © } return digest(sink, 0); } FN struct umash_fp umash_fp_digest(const struct umash_fp_state *state) { DTRACE_PROBE1(libumash, umash_fp_digest, state); return fp_digest_sink(&state->sink); } ================================================ FILE: tsl/src/import/umash.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * This file contains source code that was copied and/or modified from * the UMASH hash implementation at https://github.com/backtrace-labs/umash. * * This is a copy of umash.h, git commit sha * fc4c5b6ca1f06c308e96c43aa080bd766238e092. */ /* * UMASH is distributed under the MIT license. * * SPDX-License-Identifier: MIT * * Copyright 2020-2022 Backtrace I/O, Inc. * Copyright 2022 Paul Khuong * Copyright 2022 Dougall Johnson * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sublicense, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be * included in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #ifndef UMASH_H #define UMASH_H #include <stdbool.h> #include <stddef.h> #include <stdint.h> #ifndef TS_USE_UMASH #error "UMASH usage is disabled, but the header is included" #endif /** * # UMASH: a non-cryptographic hash function with collision bounds * * SPDX-License-Identifier: MIT * Copyright 2020-2022 Backtrace I/O, Inc. * Copyright 2022 Paul Khuong * * UMASH is a fast (9-22 ns latency for inputs of 1-64 bytes and 22 * GB/s peak throughput, on a 2.5 GHz Intel 8175M) 64-bit hash * function with mathematically proven collision bounds: it is * [ceil(s / 4096) * 2^{-55}]-almost-universal for inputs of s or * fewer bytes. * * When that's not enough, UMASH can also generate a pair of 64-bit * hashes in a single traversal. The resulting fingerprint reduces * the collision probability to less than [ceil(s / 2^{26})^2 * 2^{-83}]; * the probability that two distinct inputs receive the same * fingerprint is less 2^{-83} for inputs up to 64 MB, and less than * 2^{-70} as long as the inputs are shorter than 5 GB each. This * expectation is taken over the randomly generated `umash_params`. * If an attacker can infer the contents of these parameters, the * bounds do not apply. * * ## Initialisation * * In order to use `UMASH`, one must first generate a `struct * umash_params`; each such param defines a distinct `UMASH` function * (a pair of such functions, in fact). Ideally, one would fill * a struct with random bytes and call`umash_params_prepare`. * * - `umash_params_prepare`: attempts to convert the contents of * randomly filled `struct umash_params` into a valid UMASH * parameter struct (key). When the input consists of uniformly * generated random bytes, the probability of failure is * astronomically small. * * - `umash_params_derive`: deterministically constructs a `struct * umash_params` from a 64-bit seed and an optional 32-byte secret. * The seed and secret are expanded into random bytes with Salsa20; * the resulting `umash_params` should be practically random, as * long the seed or secret are unknown. * * ## Batch hashing and fingerprinting * * Once we have a `struct umash_params`, we can use `umash_full` or * `umash_fprint` like regular hash functions. * * - `umash_full` can compute either of the two UMASH functions * described by a `struct umash_params`. Its `seed` argument will * change the output, but is not associated with any collision * bound. * * - `umash_fprint` computes both `UMASH` functions described by a * `struct umash_params`. `umash_fp::hash[0]` corresponds to * calling `umash_full` with the same arguments and `which = 0`; * `umash_fp::hash[1]` corresponds to `which = 1`. * * ## Incremental hashing and fingerprinting * * We can also compute UMASH values by feeding bytes incrementally. * The result is guaranteed to the same as if we had buffered all the * bytes and called `umash_full` or `umash_fprint`. * * - `umash_init` initialises a `struct umash_state` with the same * parameters one would pass to `umash_full`. * * - `umash_digest` computes the value `umash_full` would return * were it passed the arguments that were given to `umash_init`, * and the bytes "fed" into the `umash_state`. * * - `umash_fp_init` initialises a `struct umash_fp_state` with the * same parameters one would pass to `umash_fprint`. * * - `umash_fp_digest` computes the value `umash_fprint` would return * for the bytes "fed" into the `umash_fp_state`. * * In both cases, one passes a pointer to `struct umash_state::sink` * or `struct umash_fp_state::sink` to callees that wish to feed bytes * into the `umash_state` or `umash_fp_state`. * * - `umash_sink_update` feeds a byte range to the `umash_sink` * initialised by calling `umash_init` or `umash_fp_init`. The sink * does not take ownership of anything and the input bytes may be * overwritten or freed as soon as `umash_sink_update` returns. */ #ifdef __cplusplus extern "C" { #endif enum { UMASH_OH_PARAM_COUNT = 32, UMASH_OH_TWISTING_COUNT = 2 }; /** * A single UMASH params struct stores the parameters for a pair of * independent `UMASH` functions. */ struct umash_params { /* * Each uint64_t[2] array consists of {f^2, f}, where f is a * random multiplier in mod 2**61 - 1. */ uint64_t poly[2][2]; /* * The second (twisted) OH function uses an additional * 128-bit constant stored in the last two elements. */ uint64_t oh[UMASH_OH_PARAM_COUNT + UMASH_OH_TWISTING_COUNT]; }; /** * A fingerprint consists of two independent `UMASH` hash values. */ struct umash_fp { uint64_t hash[2]; }; /** * This struct holds the state for incremental UMASH hashing or * fingerprinting. * * A sink owns no allocation, and simply borrows a pointer to its * `umash_params`. It can be byte-copied to snapshot its state. * * The layout works best with alignment to 64 bytes, but does not * require it. */ struct umash_sink { /* * We incrementally maintain two states when fingerprinting. * When hashing, only the first `poly_state` and `oh_acc` * entries are active. */ struct { uint64_t mul[2]; /* Multiplier, and multiplier^2. */ uint64_t acc; /* Current Horner accumulator. */ } poly_state[2]; /* * We write new bytes to the second half, and keep the previous * 16 byte chunk in the first half. * * We may temporarily have a full 16-byte buffer in the second half: * we must know if the first 16 byte chunk is the first of many, or * the whole input. */ char buf[2 * 16]; /* The next 64 bytes are accessed in the `OH` inner loop. */ /* key->oh. */ const uint64_t *oh; /* oh_iter tracks where we are in the inner loop, times 2. */ uint32_t oh_iter; uint8_t bufsz; /* Write pointer in `buf + 16`. */ uint8_t block_size; /* Current OH block size, excluding `bufsz`. */ bool large_umash; /* True once we definitely have >= 16 bytes. */ /* * 0 if we're computing the first umash, 1 for the second, and * 2 for a fingerprint. * * In practice, we treat 1 and 2 the same (always compute a * full fingerprint), and return only the second half if we * only want that half. */ uint8_t hash_wanted; /* Accumulators for the current OH value. */ struct umash_oh { uint64_t bits[2]; } oh_acc; struct umash_twisted_oh { uint64_t lrc[2]; uint64_t prev[2]; struct umash_oh acc; } oh_twisted; uint64_t seed; }; /** * The `umash_state` struct wraps a sink in a type-safe interface: we * don't want to try and extract a fingerprint from a sink configured * for hashing. */ struct umash_state { struct umash_sink sink; }; /** * Similarly, the `umash_fp_state` struct wraps a sink from which we * should extract a fingerprint. */ struct umash_fp_state { struct umash_sink sink; }; /** * Converts a `umash_params` struct filled with random values into * something usable by the UMASH functions below. * * When it succeeds, this function is idempotent. Failure happens * with probability < 2**-110 is `params` is filled with uniformly * distributed random bits. That's an astronomically unlikely event, * and most likely signals an issue with the caller's (pseudo-)random * number generator. * * @return false on failure, probably because the input was not random. */ bool umash_params_prepare(struct umash_params *params); /** * Deterministically derives a `umash_params` struct from `bits` and * `key`. The `bits` values do not have to be particularly well * distributed, and can be generated sequentially. * * @param key a pointer to exactly 32 secret bytes. NULL will be * replaced with "Do not use UMASH VS adversaries.", the default * UMASH secret. */ void umash_params_derive(struct umash_params *, uint64_t bits, const void *key); /** * Updates a `umash_sink` to take into account `data[0 ... n_bytes)`. */ void umash_sink_update(struct umash_sink *, const void *data, size_t n_bytes); /** * Computes the UMASH hash of `data[0 ... n_bytes)`. * * Randomly generated `param` lead to independent UMASH values and * associated worst-case collision bounds; changing the `seed` comes * with no guarantee. * * @param which 0 to compute the first UMASH defined by `params`, 1 * for the second. */ uint64_t umash_full(const struct umash_params *params, uint64_t seed, int which, const void *data, size_t n_bytes); /** * Computes the UMASH fingerprint of `data[0 ... n_bytes)`. * * Randomly generated `param` lead to independent UMASH values and * associated worst-case collision bounds; changing the `seed` comes * with no guarantee. */ struct umash_fp umash_fprint( const struct umash_params *params, uint64_t seed, const void *data, size_t n_bytes); /** * Prepares a `umash_state` for computing the `which`th UMASH function in * `params`. */ void umash_init( struct umash_state *, const struct umash_params *params, uint64_t seed, int which); /** * Returns the UMASH value for the bytes that have been * `umash_sink_update`d into the state. */ uint64_t umash_digest(const struct umash_state *); /** * Prepares a `umash_fp_state` for computing the UMASH fingerprint in * `params`. */ void umash_fp_init( struct umash_fp_state *, const struct umash_params *params, uint64_t seed); /** * Returns the UMASH fingerprint for the bytes that have been * `umash_sink_update`d into the state. */ struct umash_fp umash_fp_digest(const struct umash_fp_state *); #ifdef __cplusplus } #endif #endif /* !UMASH_H */ ================================================ FILE: tsl/src/init.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <fmgr.h> #include <storage/ipc.h> #include "bgw_policy/compression_api.h" #include "bgw_policy/continuous_aggregate_api.h" #include "bgw_policy/job.h" #include "bgw_policy/job_api.h" #include "bgw_policy/policies_v2.h" #include "bgw_policy/process_hyper_inval_api.h" #include "bgw_policy/reorder_api.h" #include "bgw_policy/retention_api.h" #include "chunk.h" #include "chunk_api.h" #include "compression/algorithms/array.h" #include "compression/algorithms/bool_compress.h" #include "compression/algorithms/deltadelta.h" #include "compression/algorithms/dictionary.h" #include "compression/algorithms/gorilla.h" #include "compression/algorithms/uuid_compress.h" #include "compression/api.h" #include "compression/compression.h" #include "compression/create.h" #include "compression/recompress.h" #include "compression/sparse_index_bloom1.h" #include "continuous_aggs/create.h" #include "continuous_aggs/insert.h" #include "continuous_aggs/invalidation.h" #include "continuous_aggs/options.h" #include "continuous_aggs/refresh.h" #include "continuous_aggs/utils.h" #include "cross_module_fn.h" #include "export.h" #include "hypertable.h" #include "license_guc.h" #include "nodes/columnar_index_scan/columnar_index_scan.h" #include "nodes/columnar_scan/planner.h" #include "nodes/gapfill/gapfill_functions.h" #include "nodes/skip_scan/skip_scan.h" #include "nodes/vector_agg/plan.h" #include "planner.h" #include "process_utility.h" #include "reorder.h" #ifdef PG_MODULE_MAGIC PG_MODULE_MAGIC; #endif #ifdef APACHE_ONLY #error "cannot compile the TSL for ApacheOnly mode" #endif #if PG16_LT extern void PGDLLEXPORT _PG_init(void); #endif /* * Cross module function initialization. * * During module start we set ts_cm_functions to point at the tsl version of the * function registry. * * NOTE: To ensure that your cross-module function has a correct default, you * must also add it to ts_cm_functions_default in cross_module_fn.c in the * Apache codebase. */ CrossModuleFunctions tsl_cm_functions = { .create_upper_paths_hook = tsl_create_upper_paths_hook, .set_rel_pathlist_dml = tsl_set_rel_pathlist_dml, .set_rel_pathlist_query = tsl_set_rel_pathlist_query, .sort_transform_replace_pathkeys = tsl_sort_transform_replace_pathkeys, /* bgw policies */ .policy_compression_add = policy_compression_add, .policy_compression_remove = policy_compression_remove, .policy_recompression_proc = policy_recompression_proc, .policy_compression_check = policy_compression_check, .policy_refresh_cagg_add = policy_refresh_cagg_add, .policy_refresh_cagg_proc = policy_refresh_cagg_proc, .policy_refresh_cagg_check = policy_refresh_cagg_check, .policy_refresh_cagg_remove = policy_refresh_cagg_remove, .policy_process_hyper_inval_add = policy_process_hyper_inval_add, .policy_process_hyper_inval_proc = policy_process_hyper_inval_proc, .policy_process_hyper_inval_check = policy_process_hyper_inval_check, .policy_process_hyper_inval_remove = policy_process_hyper_inval_remove, .policy_reorder_add = policy_reorder_add, .policy_reorder_proc = policy_reorder_proc, .policy_reorder_check = policy_reorder_check, .policy_reorder_remove = policy_reorder_remove, .policy_retention_add = policy_retention_add, .policy_retention_proc = policy_retention_proc, .policy_retention_check = policy_retention_check, .policy_retention_remove = policy_retention_remove, .job_add = job_add, .job_alter = job_alter, .job_alter_set_hypertable_id = job_alter_set_hypertable_id, .job_delete = job_delete, .job_run = job_run, .job_execute = job_execute, /* gapfill */ .gapfill_marker = gapfill_marker, .gapfill_int16_time_bucket = gapfill_int16_time_bucket, .gapfill_int32_time_bucket = gapfill_int32_time_bucket, .gapfill_int64_time_bucket = gapfill_int64_time_bucket, .gapfill_date_time_bucket = gapfill_date_time_bucket, .gapfill_timestamp_time_bucket = gapfill_timestamp_time_bucket, .gapfill_timestamptz_time_bucket = gapfill_timestamptz_time_bucket, .gapfill_timestamptz_timezone_time_bucket = gapfill_timestamptz_timezone_time_bucket, .reorder_chunk = tsl_reorder_chunk, .move_chunk = tsl_move_chunk, .policies_add = policies_add, .policies_remove = policies_remove, .policies_remove_all = policies_remove_all, .policies_alter = policies_alter, .policies_show = policies_show, /* Vectorized queries */ .tsl_postprocess_plan = tsl_postprocess_plan, /* Continuous Aggregates */ .process_cagg_viewstmt = tsl_process_continuous_agg_viewstmt, .continuous_agg_refresh = continuous_agg_refresh, .continuous_agg_invalidate_raw_ht = continuous_agg_invalidate_raw_ht, .continuous_agg_invalidate_mat_ht = continuous_agg_invalidate_mat_ht, .continuous_agg_dml_invalidate = continuous_agg_dml_invalidate, .continuous_agg_update_options = continuous_agg_update_options, .continuous_agg_validate_query = continuous_agg_validate_query, .continuous_agg_get_bucket_function = continuous_agg_get_bucket_function, .continuous_agg_get_bucket_function_info = continuous_agg_get_bucket_function_info, .continuous_agg_get_grouping_columns = continuous_agg_get_grouping_columns, /* Compression */ .compressed_data_decompress_forward = tsl_compressed_data_decompress_forward, .compressed_data_decompress_reverse = tsl_compressed_data_decompress_reverse, .compressed_data_column_size = tsl_compressed_data_column_size, .compressed_data_to_array = tsl_compressed_data_to_array, .compressed_data_send = tsl_compressed_data_send, .compressed_data_recv = tsl_compressed_data_recv, .compressed_data_in = tsl_compressed_data_in, .compressed_data_out = tsl_compressed_data_out, .compressed_data_info = tsl_compressed_data_info, .compressed_data_has_nulls = tsl_compressed_data_has_nulls, .deltadelta_compressor_append = tsl_deltadelta_compressor_append, .deltadelta_compressor_finish = tsl_deltadelta_compressor_finish, .gorilla_compressor_append = tsl_gorilla_compressor_append, .gorilla_compressor_finish = tsl_gorilla_compressor_finish, .dictionary_compressor_append = tsl_dictionary_compressor_append, .dictionary_compressor_finish = tsl_dictionary_compressor_finish, .array_compressor_append = tsl_array_compressor_append, .array_compressor_finish = tsl_array_compressor_finish, .bool_compressor_append = tsl_bool_compressor_append, .bool_compressor_finish = tsl_bool_compressor_finish, .uuid_compressor_append = tsl_uuid_compressor_append, .uuid_compressor_finish = tsl_uuid_compressor_finish, .bloom1_contains = bloom1_contains, .bloom1_contains_any = bloom1_contains_any, .bloom1_get_hash_function = bloom1_get_hash_function, .process_compress_table = tsl_process_compress_table, .process_altertable_cmd = tsl_process_altertable_cmd, .process_rename_cmd = tsl_process_rename_cmd, .compress_chunk = tsl_compress_chunk, .decompress_chunk = tsl_decompress_chunk, .rebuild_columnstore = tsl_rebuild_columnstore, .decompress_batches_for_insert = decompress_batches_for_insert, .init_decompress_state_for_insert = init_decompress_state_for_insert, .decompress_target_segments = decompress_target_segments, .columnstore_setup = tsl_columnstore_setup, .compressor_init = tsl_compressor_init, .compressor_set_invalidation = tsl_compressor_set_invalidation, .compressor_add_slot = tsl_compressor_add_slot, .compressor_flush = tsl_compressor_flush, .compressor_free = tsl_compressor_free, .compression_chunk_create = tsl_compression_chunk_create, .show_chunk = chunk_show, .create_compressed_chunk = tsl_create_compressed_chunk, .create_chunk = chunk_create, .chunk_freeze_chunk = chunk_freeze_chunk, .chunk_unfreeze_chunk = chunk_unfreeze_chunk, .recompress_chunk_segmentwise = tsl_recompress_chunk_segmentwise, .get_compressed_chunk_index_for_recompression = tsl_get_compressed_chunk_index_for_recompression, .preprocess_query_tsl = tsl_preprocess_query, .merge_chunks = chunk_merge_chunks, .split_chunk = chunk_split_chunk, .detach_chunk = chunk_detach, .attach_chunk = chunk_attach, .estimate_compressed_batch_size = tsl_estimate_compressed_batch_size, }; static void ts_module_cleanup_on_pg_exit(int code, Datum arg) { _continuous_aggs_cache_inval_fini(); } TS_FUNCTION_INFO_V1(ts_module_init); /* * Module init function, sets ts_cm_functions to point at tsl_cm_functions */ PGDLLEXPORT Datum ts_module_init(PG_FUNCTION_ARGS) { bool register_proc_exit = PG_GETARG_BOOL(0); ts_cm_functions = &tsl_cm_functions; _continuous_aggs_cache_inval_init(); _columnar_index_scan_init(); _columnar_scan_init(); _skip_scan_init(); _vector_agg_init(); /* Register a cleanup function to be called when the backend exits */ if (register_proc_exit) { on_proc_exit(ts_module_cleanup_on_pg_exit, 0); /* * We also register some GUCs here which are impossible to register in * the Apache module, because the default value is only known in the TSL * module. It is done in this branch to avoid being called multiple * times in the parallel workers. */ /* * The read-only GUC to query the current metadata column prefix used * for bloom filter sparse indexes. It can be different depending on the * hashing schema we use, that is determined at build time. In debug * builds, it can be changed for testing. */ bloom1_column_prefix = default_bloom1_column_prefix; DefineCustomStringVariable(MAKE_EXTOPTION("bloom1_column_prefix"), "bloom filter column prefix", "The prefix used for the metadata columns storing the sparse " "bloom filter indexes.", (char **) &bloom1_column_prefix, default_bloom1_column_prefix, #ifndef NDEBUG PGC_USERSET, #else PGC_INTERNAL, #endif 0, NULL, NULL, NULL); } PG_RETURN_BOOL(true); } /* Informative functions */ PGDLLEXPORT void _PG_init(void) { /* * In a normal backend, we disable loading the tsl until after the main * timescale library is loaded, after which we enable it from the loader. * In parallel workers the restore shared libraries function will load the * libraries itself, and we bypass the loader, so we need to ensure that * timescale is aware it can use the tsl if needed. It is always safe to * do this here, because if we reach this point, we must have already * loaded the tsl, so we no longer need to worry about its load order * relative to the other libraries. */ ts_license_enable_module_loading(); } ================================================ FILE: tsl/src/nodes/CMakeLists.txt ================================================ set(SOURCES) target_sources(${TSL_LIBRARY_NAME} PRIVATE ${SOURCES}) add_subdirectory(columnar_index_scan) add_subdirectory(columnar_scan) add_subdirectory(gapfill) add_subdirectory(skip_scan) add_subdirectory(vector_agg) ================================================ FILE: tsl/src/nodes/README.md ================================================ # TimescaleDB Optimizations TimescaleDB has a number of optimizations to improve performance of query execution. - [Skip scan](skip_scan/README.md) optimize queries involving `DISTINCT` - [Gapfill](gapfill/README.md) supports gapfilling time-series using LOCF and interpolation ================================================ FILE: tsl/src/nodes/columnar_index_scan/CMakeLists.txt ================================================ # Add all *.c to sources in upperlevel directory set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/columnar_index_scan.c ${CMAKE_CURRENT_SOURCE_DIR}/columnar_index_scan_exec.c) target_sources(${TSL_LIBRARY_NAME} PRIVATE ${SOURCES}) ================================================ FILE: tsl/src/nodes/columnar_index_scan/columnar_index_scan.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/sysattr.h> #include <nodes/bitmapset.h> #include <nodes/extensible.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <nodes/plannodes.h> #include <optimizer/optimizer.h> #include <parser/parsetree.h> #include <utils/fmgroids.h> #include <utils/lsyscache.h> #include "columnar_index_scan.h" #include "compression/create.h" #include "expression_utils.h" #include "func_cache.h" #include "guc.h" #include "nodes/columnar_scan/columnar_scan.h" #include "nodes/columnar_scan/planner.h" #include "ts_catalog/array_utils.h" #include "ts_catalog/compression_settings.h" #include "utils.h" static CustomScanMethods columnar_index_scan_plan_methods = { .CustomName = COLUMNAR_INDEX_SCAN_NAME, .CreateCustomScanState = columnar_index_scan_state_create, }; void _columnar_index_scan_init(void) { TryRegisterCustomScanMethods(&columnar_index_scan_plan_methods); } /* * Check if an aggregate function can use compressed chunk sparse index. * * Currently supported aggregates are min, max, first, and last. */ static bool is_supported_aggregate(Aggref *aggref, Var **arg_var_out, const char **meta_type_out) { if (aggref->args == NIL) return false; /* Get the argument - must be a Var (possibly with implicit coercions) */ TargetEntry *arg_te = linitial_node(TargetEntry, aggref->args); Node *arg_expr = strip_implicit_coercions((Node *) arg_te->expr); if (!IsA(arg_expr, Var)) return false; Var *var = castNode(Var, arg_expr); /* Reject system columns except tableoid */ if (var->varattno <= 0 && var->varattno != TableOidAttributeNumber) return false; switch (aggref->aggfnoid) { case F_MIN_ANYARRAY: case F_MIN_ANYENUM: case F_MIN_BPCHAR: #if PG18_GE case F_MIN_BYTEA: #endif case F_MIN_DATE: case F_MIN_FLOAT4: case F_MIN_FLOAT8: case F_MIN_INET: case F_MIN_INT2: case F_MIN_INT4: case F_MIN_INT8: case F_MIN_INTERVAL: case F_MIN_MONEY: case F_MIN_NUMERIC: case F_MIN_OID: case F_MIN_PG_LSN: #if PG18_GE case F_MIN_RECORD: #endif case F_MIN_TEXT: case F_MIN_TID: case F_MIN_TIME: case F_MIN_TIMESTAMP: case F_MIN_TIMESTAMPTZ: case F_MIN_TIMETZ: case F_MIN_XID8: *meta_type_out = "min"; *arg_var_out = var; return true; case F_MAX_ANYARRAY: case F_MAX_ANYENUM: case F_MAX_BPCHAR: #if PG18_GE case F_MAX_BYTEA: #endif case F_MAX_DATE: case F_MAX_FLOAT4: case F_MAX_FLOAT8: case F_MAX_INET: case F_MAX_INT2: case F_MAX_INT4: case F_MAX_INT8: case F_MAX_INTERVAL: case F_MAX_MONEY: case F_MAX_NUMERIC: case F_MAX_OID: case F_MAX_PG_LSN: #if PG18_GE case F_MAX_RECORD: #endif case F_MAX_TEXT: case F_MAX_TID: case F_MAX_TIME: case F_MAX_TIMESTAMP: case F_MAX_TIMESTAMPTZ: case F_MAX_TIMETZ: case F_MAX_XID8: *meta_type_out = "max"; *arg_var_out = var; return true; default: /* Check for first()/last() with both args referencing same column */ if (!OidIsValid(ts_first_func_oid) || !OidIsValid(ts_last_func_oid)) ts_func_cache_init(); if (aggref->aggfnoid == ts_first_func_oid || aggref->aggfnoid == ts_last_func_oid) { if (list_length(aggref->args) != 2) return false; TargetEntry *tle2 = castNode(TargetEntry, lsecond(aggref->args)); Node *arg2_expr = strip_implicit_coercions((Node *) tle2->expr); if (!equal(var, arg2_expr)) return false; *meta_type_out = (aggref->aggfnoid == ts_first_func_oid) ? "min" : "max"; *arg_var_out = var; return true; } break; } return false; } /* * Check if all quals reference only segmentby columns. * * This function is only relevant for partial chunks. For fully compressed * chunks, segmentby quals are pushed to the compressed scan and removed from * plan->qual, so plan->qual is NIL and this function is never called. * For partial chunks, segmentby quals remain on plan->qual because they must * also be checked against the uncompressed portion, but they have already been * pushed to the compressed scan as well. ColumnarIndexScan only reads * compressed metadata, so these quals are already handled. * * Uses the decompression_map and is_segmentby_column lists from the * ColumnarScan's custom_private to build a set of custom_scan_tlist * positions that are segmentby columns, then checks each qual's Vars * against that set. * * Returns false for quals containing no Vars (e.g. constified tableoid), * since those constant expressions are not handled by the compressed scan * and must still be evaluated. In practice these only appear on partial * chunks from constraint exclusion checks (e.g. tableoid comparisons for * hypertable expansion) and should be uncommon. */ static bool quals_only_reference_segmentby(List *quals, CustomScan *cscan) { List *decompression_map = list_nth(cscan->custom_private, DCP_DecompressionMap); List *is_segmentby = list_nth(cscan->custom_private, DCP_IsSegmentbyColumn); Bitmapset *segmentby_positions = NULL; ListCell *lc_map, *lc_seg; forboth (lc_map, decompression_map, lc_seg, is_segmentby) { int tlist_pos = lfirst_int(lc_map); if (lfirst_int(lc_seg) && tlist_pos > 0) segmentby_positions = bms_add_member(segmentby_positions, tlist_pos); } bool result = true; ListCell *lc; foreach (lc, quals) { List *vars = pull_var_clause(lfirst(lc), 0); if (vars == NIL) { result = false; break; } ListCell *lc2; foreach (lc2, vars) { Var *var = lfirst_node(Var, lc2); if (!bms_is_member(var->varattno, segmentby_positions)) { result = false; break; } } if (!result) break; } bms_free(segmentby_positions); return result; } /* * Check if the ColumnarScan's vectorized quals are empty (no filters applied). */ static bool columnar_scan_has_no_vector_quals(CustomScan *cscan) { /* * The vectorized quals are stored in custom_exprs. If the list is empty * or the first element is NIL, there are no vectorized quals. */ if (cscan->custom_exprs == NIL) return true; if (linitial(cscan->custom_exprs) == NIL) return true; return false; } /* * Find the resno in the leaf plan's targetlist that corresponds to a given * compressed chunk attribute number. */ static AttrNumber find_resno_by_compressed_attno(Plan *leaf_plan, AttrNumber compressed_attno) { ListCell *lc; foreach (lc, leaf_plan->targetlist) { TargetEntry *tle = lfirst_node(TargetEntry, lc); if (IsA(tle->expr, Var)) { Var *var = castNode(Var, tle->expr); if (var->varattno == compressed_attno) return tle->resno; } } return InvalidAttrNumber; } /* * Context for validate_entries_walker. */ typedef struct ValidateContext { Oid uncompressed_relid; Oid compressed_relid; Index uncompressed_scanrelid; Index compressed_scanrelid; CompressionSettings *settings; Plan *leaf_plan; List *custom_scan_tlist; List *output_map; AttrNumber next_resno; } ValidateContext; static void add_scan_output(ValidateContext *ctx, AttrNumber child_resno, Index tlist_varno, AttrNumber tlist_attno, Oid col_type, int32 col_typmod, Oid col_collid) { Var *tlist_var = makeVar(tlist_varno, tlist_attno, col_type, col_typmod, col_collid, 0); ctx->custom_scan_tlist = lappend(ctx->custom_scan_tlist, makeTargetEntry((Expr *) tlist_var, ctx->next_resno++, NULL, false)); ctx->output_map = lappend(ctx->output_map, makeInteger(child_resno)); } /* * Expression tree walker that validates Var and Aggref nodes in the resolved * Agg targetlist. Builds custom_scan_tlist and output_map entries in DFS order. * Returns true (abort) if any node cannot be handled. * * Aggrefs are validated but NOT recursed into — their args reference the * child plan being replaced, so we only care about the aggregate itself. */ static bool validate_entries_walker(Node *node, void *context) { ValidateContext *ctx = (ValidateContext *) context; if (node == NULL) return false; if (IsA(node, Var)) { Var *var = (Var *) node; if (var->varattno == TableOidAttributeNumber) { add_scan_output(ctx, TableOidAttributeNumber, ctx->uncompressed_scanrelid, TableOidAttributeNumber, OIDOID, -1, InvalidOid); return false; } if (var->varattno <= 0) return true; char *col_name = get_attname(ctx->uncompressed_relid, var->varattno, false); if (ctx->settings == NULL || !ts_array_is_member(ctx->settings->fd.segmentby, col_name)) return true; AttrNumber compressed_attno = get_attnum(ctx->compressed_relid, col_name); if (compressed_attno == InvalidAttrNumber) return true; AttrNumber child_resno = find_resno_by_compressed_attno(ctx->leaf_plan, compressed_attno); if (child_resno == InvalidAttrNumber) return true; add_scan_output(ctx, child_resno, ctx->uncompressed_scanrelid, var->varattno, var->vartype, var->vartypmod, var->varcollid); return false; } if (IsA(node, Aggref)) { Aggref *aggref = (Aggref *) node; /* No DISTINCT, ORDER BY, or FILTER on the Aggref */ if (aggref->aggdistinct != NIL || aggref->aggorder != NIL || aggref->aggfilter != NULL) return true; if (aggref->aggfnoid == F_COUNT_) { AttrNumber meta_count_attno = get_attnum(ctx->compressed_relid, COMPRESSION_COLUMN_METADATA_COUNT_NAME); if (meta_count_attno == InvalidAttrNumber) return true; AttrNumber child_resno = find_resno_by_compressed_attno(ctx->leaf_plan, meta_count_attno); if (child_resno == InvalidAttrNumber) return true; add_scan_output(ctx, child_resno, ctx->compressed_scanrelid, meta_count_attno, INT4OID, -1, InvalidOid); } else { Var *arg_var = NULL; const char *meta_type = NULL; if (!is_supported_aggregate(aggref, &arg_var, &meta_type)) return true; if (arg_var->varattno == TableOidAttributeNumber) { /* * tableoid is constant within a single chunk scan, so * min(tableoid) = max(tableoid) = tableoid. Output the * tableoid sentinel and let the Agg reduce it. */ add_scan_output(ctx, TableOidAttributeNumber, ctx->uncompressed_scanrelid, TableOidAttributeNumber, OIDOID, -1, InvalidOid); } else { AttrNumber meta_attno = compressed_column_metadata_attno(ctx->settings, ctx->uncompressed_relid, arg_var->varattno, ctx->compressed_relid, meta_type); if (meta_attno == InvalidAttrNumber) return true; AttrNumber child_resno = find_resno_by_compressed_attno(ctx->leaf_plan, meta_attno); if (child_resno == InvalidAttrNumber) return true; add_scan_output(ctx, child_resno, ctx->uncompressed_scanrelid, arg_var->varattno, get_atttype(ctx->compressed_relid, meta_attno), -1, arg_var->varcollid); } } /* Don't recurse into Aggref children */ return false; } return expression_tree_walker(node, validate_entries_walker, context); } /* * Context for rewrite_agg_tlist_mutator. */ typedef struct RewriteContext { Agg *agg; List *custom_scan_tlist; AttrNumber next_resno; } RewriteContext; /* * Expression tree mutator that rewrites Var and Aggref nodes in the Agg's * targetlist to reference ColumnarIndexScan output columns. Type information * for rewritten aggregate arguments is read from the pre-built custom_scan_tlist. */ static Node * rewrite_agg_tlist_mutator(Node *node, void *context) { RewriteContext *ctx = (RewriteContext *) context; if (node == NULL) return NULL; if (IsA(node, Var) && ((Var *) node)->varno == OUTER_VAR) { Var *var = (Var *) node; AttrNumber resno = ctx->next_resno++; /* Update grpColIdx for GROUP BY Vars */ for (int k = 0; k < ctx->agg->numCols; k++) { if (ctx->agg->grpColIdx[k] == var->varattno) ctx->agg->grpColIdx[k] = resno; } return (Node *) makeVar(OUTER_VAR, resno, var->vartype, var->vartypmod, var->varcollid, var->varlevelsup); } if (IsA(node, Aggref)) { Aggref *orig = (Aggref *) node; AttrNumber resno = ctx->next_resno++; if (orig->aggfnoid == F_COUNT_) { /* * Rewrite count(*) → sum(_ts_meta_count). * * sum(int4) uses int4_sum as transition function with INT8 * transition type, same as count(*)'s int8inc. Both use * int8pl as combine function, so the Finalize Agg above * (which still has the original count(*) aggfnoid) can * combine partial states from either without changes. * * For AGGSPLIT_SIMPLE (no Finalize), wrap in COALESCE to * preserve count(*)'s 0-for-empty semantics since sum() * returns NULL for no rows. For AGGSPLIT_INITIAL_SERIAL, * the Finalize's strict combine function (int8pl) and * count's initial value (0) handle NULL partial states. */ Aggref *aggref = copyObject(orig); aggref->aggfnoid = F_SUM_INT4; aggref->aggtranstype = INT8OID; aggref->aggstar = false; aggref->aggargtypes = list_make1_oid(INT4OID); Var *arg_var = makeVar(OUTER_VAR, resno, INT4OID, -1, InvalidOid, 0); aggref->args = list_make1(makeTargetEntry((Expr *) arg_var, 1, NULL, false)); if (ctx->agg->aggsplit == AGGSPLIT_SIMPLE) { CoalesceExpr *coalesce = makeNode(CoalesceExpr); coalesce->coalescetype = aggref->aggtype; coalesce->args = list_make2(aggref, makeConst(INT8OID, -1, InvalidOid, sizeof(int64), Int64GetDatum(0), false, true)); coalesce->location = -1; return (Node *) coalesce; } return (Node *) aggref; } /* min/max/first/last: rewrite args to point to metadata column */ Aggref *aggref = copyObject(orig); TargetEntry *scan_tle = list_nth_node(TargetEntry, ctx->custom_scan_tlist, resno - 1); Var *scan_var = castNode(Var, scan_tle->expr); Var *new_arg = makeVar(OUTER_VAR, resno, scan_var->vartype, scan_var->vartypmod, scan_var->varcollid, 0); TargetEntry *new_arg_te = makeTargetEntry((Expr *) new_arg, 1, NULL, false); if (list_length(aggref->args) == 2) { /* first/last: both args point to same metadata column */ Var *new_arg2 = copyObject(new_arg); TargetEntry *new_arg_te2 = makeTargetEntry((Expr *) new_arg2, 2, NULL, false); aggref->args = list_make2(new_arg_te, new_arg_te2); } else { aggref->args = list_make1(new_arg_te); } return (Node *) aggref; } return expression_tree_mutator(node, rewrite_agg_tlist_mutator, context); } /* * Create a ColumnarIndexScan plan node as a pure scan node that replaces the ColumnarScan * beneath the Agg. The Agg stays in the plan with rewritten Aggrefs: * count(*) → sum(_ts_meta_count) * min(col)/max(col) → same aggfnoid, arg rewritten to metadata column * first(col,col)/last(col,col) → same aggfnoid, both args rewritten * * Expressions that combine aggregates (e.g. 2*count(*), min(x)+max(y)) are * supported — the walker/mutator handles Var and Aggref nodes at any depth. */ static Plan * columnar_index_scan_plan_create(Agg *agg, CustomScan *cscan, List *rtable) { Plan *compressed_scan_subtree = linitial(cscan->custom_plans); Plan *leaf_plan = compressed_scan_subtree; if (IsA(leaf_plan, Sort)) leaf_plan = leaf_plan->lefttree; Scan *compressed_scan = (Scan *) leaf_plan; RangeTblEntry *compressed_rte = rt_fetch(compressed_scan->scanrelid, rtable); Oid compressed_relid = compressed_rte->relid; Oid uncompressed_relid = rt_fetch(cscan->scan.scanrelid, rtable)->relid; CompressionSettings *settings = ts_compression_settings_get(uncompressed_relid); /* * Validation pass: walk the resolved targetlist to validate all Var and * Aggref leaf nodes. Builds custom_scan_tlist and output_map directly. * No modifications to the Agg happen here, so bailing out is safe. */ ValidateContext validate_ctx = { .uncompressed_relid = uncompressed_relid, .compressed_relid = compressed_relid, .uncompressed_scanrelid = cscan->scan.scanrelid, .compressed_scanrelid = compressed_scan->scanrelid, .settings = settings, .leaf_plan = leaf_plan, .custom_scan_tlist = NIL, .output_map = NIL, .next_resno = 1, }; Plan *childplan = agg->plan.lefttree; Node *resolved_targetlist = ts_resolve_outer_special_vars((Node *) agg->plan.targetlist, childplan); if (validate_entries_walker(resolved_targetlist, &validate_ctx)) return NULL; if (agg->plan.qual) { Node *resolved_qual = ts_resolve_outer_special_vars((Node *) agg->plan.qual, childplan); if (validate_entries_walker(resolved_qual, &validate_ctx)) return NULL; } List *custom_scan_tlist = validate_ctx.custom_scan_tlist; List *output_map = validate_ctx.output_map; if (custom_scan_tlist == NIL) return NULL; /* * Rewrite pass: walk the Agg's targetlist with a mutator that rewrites * Var and Aggref nodes to reference ColumnarIndexScan output columns. */ RewriteContext rewrite_ctx = { .agg = agg, .custom_scan_tlist = custom_scan_tlist, .next_resno = 1, }; agg->plan.targetlist = castNode(List, rewrite_agg_tlist_mutator((Node *) agg->plan.targetlist, &rewrite_ctx)); if (agg->plan.qual) agg->plan.qual = castNode(List, rewrite_agg_tlist_mutator((Node *) agg->plan.qual, &rewrite_ctx)); /* Build ColumnarIndexScan CustomScan */ CustomScan *columnar_index_scan = (CustomScan *) makeNode(CustomScan); columnar_index_scan->custom_plans = list_make1(compressed_scan_subtree); columnar_index_scan->methods = &columnar_index_scan_plan_methods; columnar_index_scan->scan.scanrelid = cscan->scan.scanrelid; columnar_index_scan->custom_scan_tlist = custom_scan_tlist; columnar_index_scan->scan.plan.targetlist = ts_build_trivial_custom_output_targetlist(custom_scan_tlist); /* Copy cost/parallel/param fields from the ColumnarScan */ columnar_index_scan->scan.plan.plan_rows = cscan->scan.plan.plan_rows; columnar_index_scan->scan.plan.plan_width = cscan->scan.plan.plan_width; columnar_index_scan->scan.plan.startup_cost = cscan->scan.plan.startup_cost; columnar_index_scan->scan.plan.total_cost = cscan->scan.plan.total_cost; columnar_index_scan->scan.plan.parallel_aware = false; columnar_index_scan->scan.plan.parallel_safe = cscan->scan.plan.parallel_safe; columnar_index_scan->scan.plan.async_capable = false; columnar_index_scan->scan.plan.plan_node_id = cscan->scan.plan.plan_node_id; columnar_index_scan->scan.plan.initPlan = cscan->scan.plan.initPlan; columnar_index_scan->scan.plan.extParam = bms_copy(cscan->scan.plan.extParam); columnar_index_scan->scan.plan.allParam = bms_copy(cscan->scan.plan.allParam); columnar_index_scan->custom_private = list_make1(output_map); /* Set ColumnarIndexScan as the Agg's child */ agg->plan.lefttree = (Plan *) columnar_index_scan; return (Plan *) agg; } /* * Callback for ts_plan_tree_walker: try to insert a ColumnarIndexScan node between an Agg * and its ColumnarScan child. */ static Plan * insert_columnar_index_scan(Plan *plan, void *context) { List *rtable = (List *) context; if (plan->type != T_Agg) return plan; Agg *agg = castNode(Agg, plan); /* * We are looking for the partial step (AGGSPLIT_INITIAL_SERIAL) of * partial/finalize aggregate or non-partial aggregates (AGGSPLIT_SIMPLE). */ if (agg->aggsplit != AGGSPLIT_INITIAL_SERIAL && agg->aggsplit != AGGSPLIT_SIMPLE) return plan; Plan *childplan = agg->plan.lefttree; /* * The child must be a ColumnarScan. */ if (!ts_is_columnar_scan_plan(childplan)) return plan; CustomScan *cscan = castNode(CustomScan, childplan); /* * No Postgres quals on the ColumnarScan, or quals only on segmentby * columns. Segmentby filters are pushed to the compressed scan so * ColumnarIndexScan can skip them. */ if (childplan->qual != NIL && !quals_only_reference_segmentby(childplan->qual, cscan)) return plan; /* * No vectorized quals on the ColumnarScan. */ if (!columnar_scan_has_no_vector_quals(cscan)) return plan; Plan *result = columnar_index_scan_plan_create(agg, cscan, rtable); if (result == NULL) return plan; return result; } /* * Where possible, replace Agg -> ColumnarScan with Agg -> ColumnarIndexScan. * The Agg stays in the plan with rewritten Aggrefs; ColumnarIndexScan replaces * the ColumnarScan as a pure scan node over compressed metadata. */ Plan * try_insert_columnar_index_scan_node(Plan *plan, List *rtable) { return ts_plan_tree_walker(plan, insert_columnar_index_scan, rtable); } ================================================ FILE: tsl/src/nodes/columnar_index_scan/columnar_index_scan.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #define COLUMNAR_INDEX_SCAN_NAME "ColumnarIndexScan" #include <postgres.h> #include <nodes/plannodes.h> extern void _columnar_index_scan_init(void); extern Plan *try_insert_columnar_index_scan_node(Plan *plan, List *rtable); extern Node *columnar_index_scan_state_create(CustomScan *cscan); ================================================ FILE: tsl/src/nodes/columnar_index_scan/columnar_index_scan_exec.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/sysattr.h> #include <executor/executor.h> #include <nodes/extensible.h> #include <nodes/plannodes.h> #include <utils/rel.h> #include "columnar_index_scan.h" typedef struct ColumnarIndexScanState { CustomScanState custom; int num_outputs; AttrNumber *child_resnos; /* one per output column */ } ColumnarIndexScanState; static void columnar_index_scan_begin(CustomScanState *node, EState *estate, int eflags) { ColumnarIndexScanState *state = (ColumnarIndexScanState *) node; CustomScan *cscan = castNode(CustomScan, node->ss.ps.plan); /* * Parse output_map from custom_private: a flat list of Integer values, * one child_resno per output column. */ List *output_map = linitial(cscan->custom_private); int num_outputs = list_length(output_map); state->num_outputs = num_outputs; state->child_resnos = palloc(sizeof(AttrNumber) * num_outputs); ListCell *lc; int i = 0; foreach (lc, output_map) { state->child_resnos[i++] = intVal(lfirst(lc)); } Assert(list_length(cscan->custom_plans) == 1); node->custom_ps = list_make1(ExecInitNode(linitial(cscan->custom_plans), estate, eflags)); } static TupleTableSlot * columnar_index_scan_exec(CustomScanState *node) { ColumnarIndexScanState *state = (ColumnarIndexScanState *) node; PlanState *child_ps = linitial(node->custom_ps); TupleTableSlot *result_slot = node->ss.ps.ps_ResultTupleSlot; ExecClearTuple(result_slot); TupleTableSlot *child_slot = ExecProcNode(child_ps); if (TupIsNull(child_slot)) return NULL; /* * Copy values from the child slot to the output slot using the output_map. * Each output column i gets its value from child_resnos[i]. */ Datum *values = result_slot->tts_values; bool *nulls = result_slot->tts_isnull; for (int i = 0; i < state->num_outputs; i++) { if (state->child_resnos[i] == TableOidAttributeNumber) { values[i] = ObjectIdGetDatum(node->ss.ss_currentRelation->rd_id); nulls[i] = false; } else { values[i] = slot_getattr(child_slot, state->child_resnos[i], &nulls[i]); } } return ExecStoreVirtualTuple(result_slot); } static void columnar_index_scan_end(CustomScanState *node) { ExecEndNode(linitial(node->custom_ps)); } static void columnar_index_scan_rescan(CustomScanState *node) { ExecReScan(linitial(node->custom_ps)); } static struct CustomExecMethods exec_methods = { .CustomName = COLUMNAR_INDEX_SCAN_NAME, .BeginCustomScan = columnar_index_scan_begin, .ExecCustomScan = columnar_index_scan_exec, .EndCustomScan = columnar_index_scan_end, .ReScanCustomScan = columnar_index_scan_rescan, .ExplainCustomScan = NULL, }; Node * columnar_index_scan_state_create(CustomScan *cscan) { ColumnarIndexScanState *state = (ColumnarIndexScanState *) newNode(sizeof(ColumnarIndexScanState), T_CustomScanState); state->custom.methods = &exec_methods; return (Node *) state; } ================================================ FILE: tsl/src/nodes/columnar_scan/CMakeLists.txt ================================================ # Add all *.c to sources in upperlevel directory set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/batch_array.c ${CMAKE_CURRENT_SOURCE_DIR}/batch_queue_heap.c ${CMAKE_CURRENT_SOURCE_DIR}/batch_queue_fifo.c ${CMAKE_CURRENT_SOURCE_DIR}/compressed_batch.c ${CMAKE_CURRENT_SOURCE_DIR}/columnar_scan.c ${CMAKE_CURRENT_SOURCE_DIR}/detoaster.c ${CMAKE_CURRENT_SOURCE_DIR}/exec.c ${CMAKE_CURRENT_SOURCE_DIR}/planner.c ${CMAKE_CURRENT_SOURCE_DIR}/pred_text.c ${CMAKE_CURRENT_SOURCE_DIR}/pred_vector_array.c ${CMAKE_CURRENT_SOURCE_DIR}/qual_pushdown.c ${CMAKE_CURRENT_SOURCE_DIR}/vector_predicates.c) target_sources(${TSL_LIBRARY_NAME} PRIVATE ${SOURCES}) ================================================ FILE: tsl/src/nodes/columnar_scan/batch_array.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include "nodes/columnar_scan/batch_array.h" #include "compression/compression.h" #include "nodes/columnar_scan/compressed_batch.h" /* * Create states to hold information for up to n batches. */ void batch_array_init(BatchArray *array, int nbatches, int ncolumns_per_batch) { Assert(nbatches >= 0); array->n_batch_states = nbatches; array->n_columns_per_batch = ncolumns_per_batch; array->unused_batch_states = bms_add_range(NULL, 0, nbatches - 1); array->n_batch_state_bytes = sizeof(DecompressBatchState) + sizeof(CompressedColumnValues) * ncolumns_per_batch; array->batch_states = palloc0(array->n_batch_state_bytes * nbatches); Assert(bms_num_members(array->unused_batch_states) == array->n_batch_states); } /* * Destroy batch states. */ void batch_array_destroy(BatchArray *array) { for (int i = 0; i < array->n_batch_states; i++) { DecompressBatchState *batch_state = batch_array_get_at(array, i); compressed_batch_destroy(batch_state); } pfree(array->batch_states); array->batch_states = NULL; } /* * Enhance the capacity of existing batch states. */ static void batch_array_enlarge(BatchArray *array, int new_number) { Assert(new_number > array->n_batch_states); /* Request additional memory */ array->batch_states = repalloc(array->batch_states, array->n_batch_state_bytes * new_number); /* Zero out the tail. The batch states are initialized on first use. */ memset(((char *) array->batch_states) + (array->n_batch_state_bytes * array->n_batch_states), 0x0, array->n_batch_state_bytes * (new_number - array->n_batch_states)); /* Register the new states as unused */ array->unused_batch_states = bms_add_range(array->unused_batch_states, array->n_batch_states, new_number - 1); Assert(bms_num_members(array->unused_batch_states) == new_number - array->n_batch_states); /* Update number of available batch states */ array->n_batch_states = new_number; } /* * Mark a DecompressBatchState as unused */ void batch_array_clear_at(BatchArray *array, int batch_index) { Assert(batch_index >= 0); Assert(batch_index < array->n_batch_states); DecompressBatchState *batch_state = batch_array_get_at(array, batch_index); /* Reset batch state */ compressed_batch_discard_tuples(batch_state); array->unused_batch_states = bms_add_member(array->unused_batch_states, batch_index); } void batch_array_clear_all(BatchArray *array) { for (int i = 0; i < array->n_batch_states; i++) batch_array_clear_at(array, i); Assert(bms_num_members(array->unused_batch_states) == array->n_batch_states); } /* * Get the next free and unused batch state and mark as used */ int batch_array_get_unused_slot(BatchArray *array) { if (bms_is_empty(array->unused_batch_states)) batch_array_enlarge(array, array->n_batch_states * 2); Assert(!bms_is_empty(array->unused_batch_states)); int next_unused_batch = bms_next_member(array->unused_batch_states, -1); Assert(next_unused_batch >= 0); Assert(next_unused_batch < array->n_batch_states); Assert(TupIsNull(compressed_batch_current_tuple(batch_array_get_at(array, next_unused_batch)))); array->unused_batch_states = bms_del_member(array->unused_batch_states, next_unused_batch); return next_unused_batch; } ================================================ FILE: tsl/src/nodes/columnar_scan/batch_array.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/bitmapset.h> #include <stdbool.h> /* The value for an invalid batch id */ #define INVALID_BATCH_ID -1 typedef struct BatchArray { /* Batch states */ int n_batch_states; /* Number of batch states */ /* * The batch states. It's void* because they have a variable length * column array, so normal indexing can't be used. Use the batch_array_get_at * accessor instead. */ void *batch_states; int n_batch_state_bytes; int n_columns_per_batch; Bitmapset *unused_batch_states; /* The unused batch states */ } BatchArray; /* * Create states to hold information for up to n batches */ void batch_array_init(BatchArray *array, int nbatches, int ncolumns_per_batch); void batch_array_destroy(BatchArray *array); extern int batch_array_get_unused_slot(BatchArray *array); inline static struct DecompressBatchState * batch_array_get_at(const BatchArray *array, int batch_index) { /* * Since we're accessing batch states through a "char" pointer, use * "restrict" to tell the compiler that it doesn't alias with anything. * Might be important in hot loops. */ return (struct DecompressBatchState *) ((char *restrict) array->batch_states + array->n_batch_state_bytes * batch_index); } extern void batch_array_clear_at(BatchArray *array, int batch_index); extern void batch_array_clear_all(BatchArray *array); inline static bool batch_array_has_active_batches(const BatchArray *array) { return bms_num_members(array->unused_batch_states) != array->n_batch_states; } ================================================ FILE: tsl/src/nodes/columnar_scan/batch_queue.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #ifndef TIMESCALEDB_BATCH_QUEUE_H #define TIMESCALEDB_BATCH_QUEUE_H #include <postgres.h> #include <executor/tuptable.h> #include "decompress_context.h" /* Initial amount of batch states */ #define INITIAL_BATCH_CAPACITY 16 struct BatchQueue; typedef struct BatchQueueFunctions { void (*free)(struct BatchQueue *); bool (*needs_next_batch)(struct BatchQueue *); void (*pop)(struct BatchQueue *, DecompressContext *); void (*push_batch)(struct BatchQueue *, DecompressContext *, TupleTableSlot *); void (*reset)(struct BatchQueue *); TupleTableSlot *(*top_tuple)(struct BatchQueue *); } BatchQueueFunctions; typedef struct BatchQueue { BatchArray batch_array; const BatchQueueFunctions *funcs; } BatchQueue; #include "batch_queue_fifo.h" #include "batch_queue_heap.h" #endif /* TIMESCALEDB_BATCH_QUEUE_H */ ================================================ FILE: tsl/src/nodes/columnar_scan/batch_queue_fifo.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include "batch_queue_fifo.h" #include "batch_array.h" /* * Create a FIFO batch queue. * * Pass the function struct as argument to ensure it is a pointer to the * struct that is inlined at the place this function is called. This is mostly * to be able to later Assert() that the struct pointer is pointing to the * expected function struct. */ BatchQueue * batch_queue_fifo_create(int num_compressed_cols, const BatchQueueFunctions *funcs) { BatchQueue *bq = palloc0(sizeof(BatchQueue)); batch_array_init(&bq->batch_array, INITIAL_BATCH_CAPACITY, num_compressed_cols); bq->funcs = funcs; return bq; } ================================================ FILE: tsl/src/nodes/columnar_scan/batch_queue_fifo.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include "batch_queue.h" #include "compressed_batch.h" static inline void batch_queue_fifo_free(BatchQueue *bq) { batch_array_destroy(&bq->batch_array); pfree(bq); } static inline bool batch_queue_fifo_needs_next_batch(BatchQueue *bq) { return TupIsNull(compressed_batch_current_tuple(batch_array_get_at(&bq->batch_array, 0))); } static inline void batch_queue_fifo_pop(BatchQueue *bq, DecompressContext *dcontext) { DecompressBatchState *batch_state = batch_array_get_at(&bq->batch_array, 0); if (TupIsNull(compressed_batch_current_tuple(batch_state))) { /* Allow this function to be called on the initial empty queue. */ return; } compressed_batch_advance(dcontext, batch_state); } static inline void batch_queue_fifo_push_batch(BatchQueue *bq, DecompressContext *dcontext, TupleTableSlot *compressed_slot) { BatchArray *batch_array = &bq->batch_array; DecompressBatchState *batch_state = batch_array_get_at(batch_array, 0); Assert(TupIsNull(compressed_batch_current_tuple(batch_array_get_at(batch_array, 0)))); compressed_batch_set_compressed_tuple(dcontext, batch_state, compressed_slot); compressed_batch_advance(dcontext, batch_state); } static inline void batch_queue_fifo_reset(BatchQueue *bq) { batch_array_clear_all(&bq->batch_array); } static inline TupleTableSlot * batch_queue_fifo_top_tuple(BatchQueue *bq) { return compressed_batch_current_tuple(batch_array_get_at(&bq->batch_array, 0)); } static const struct BatchQueueFunctions BatchQueueFunctionsFifo = { .free = batch_queue_fifo_free, .needs_next_batch = batch_queue_fifo_needs_next_batch, .pop = batch_queue_fifo_pop, .push_batch = batch_queue_fifo_push_batch, .reset = batch_queue_fifo_reset, .top_tuple = batch_queue_fifo_top_tuple, }; extern BatchQueue *batch_queue_fifo_create(int num_compressed_cols, const BatchQueueFunctions *funcs); ================================================ FILE: tsl/src/nodes/columnar_scan/batch_queue_heap.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <lib/binaryheap.h> #include <nodes/bitmapset.h> #include "compression/compression.h" #include "nodes/columnar_scan/batch_array.h" #include "nodes/columnar_scan/batch_queue.h" #include "nodes/columnar_scan/compressed_batch.h" typedef struct { Datum value; bool null; } HeapEntryColumn; typedef struct BatchQueueHeap { BatchQueue queue; binaryheap *merge_heap; /* Binary heap of slot indices */ /* * Requested sort order of the heap. */ int nkeys; SortSupport sortkeys; /* * This is the actual entries of the heap we're going to compare. We're using * these minimal structures for better memory locality instead of addressing * the entire compressed batches. Would be even better to put them into the * heap inline, but unfortunately the Postgres binary heap doesn't support * this. * * For each batch, we have nkeys of HeapEntryColumn values, which contain * the latest decompressed values. */ HeapEntryColumn *heap_entries; /* * We use this to check when we have to ask for the next input batch. */ TupleTableSlot *last_batch_first_tuple_slot; HeapEntryColumn *last_batch_first_tuple_entry; } BatchQueueHeap; /* * Compare heap entries for two batches. This function is used for comparing the * first tuple of the last batch to the current top tuple, so it is only called * once per input tuple, and optimized specializations for it are less important. */ static int32 compare_entries(HeapEntryColumn *entryA, HeapEntryColumn *entryB, const SortSupport sortkeys, int nkeys) { for (int key = 0; key < nkeys; key++) { int compare = ApplySortComparator(entryA[key].value, entryA[key].null, entryB[key].value, entryB[key].null, &sortkeys[key]); if (compare != 0) { INVERT_COMPARE_RESULT(compare); return compare; } } return 0; } /* * Compare top tuples of two given batch array slots. We support specializations * for comparison of the first tuple, like tuplesort. */ static pg_attribute_always_inline int32 compare_heap_pos_impl(Datum a, Datum b, void *arg, int32 (*apply_first_datum_comparator)(Datum, bool, Datum, bool, SortSupport)) { BatchQueueHeap *queue = (BatchQueueHeap *) arg; PG_USED_FOR_ASSERTS_ONLY BatchArray *batch_array = &queue->queue.batch_array; int batchA = DatumGetInt32(a); Assert(batchA <= batch_array->n_batch_states); int batchB = DatumGetInt32(b); Assert(batchB <= batch_array->n_batch_states); const int nkeys = queue->nkeys; SortSupport sortkeys = queue->sortkeys; HeapEntryColumn *entryA = &queue->heap_entries[batchA * nkeys]; HeapEntryColumn *entryB = &queue->heap_entries[batchB * nkeys]; int compare = apply_first_datum_comparator(entryA[0].value, entryA[0].null, entryB[0].value, entryB[0].null, &sortkeys[0]); if (compare != 0) { INVERT_COMPARE_RESULT(compare); return compare; } for (int key = 1; key < nkeys; key++) { int compare = ApplySortComparator(entryA[key].value, entryA[key].null, entryB[key].value, entryB[key].null, &sortkeys[key]); if (compare != 0) { INVERT_COMPARE_RESULT(compare); return compare; } } return 0; } static int32 compare_heap_pos_generic(Datum a, Datum b, void *arg) { return compare_heap_pos_impl(a, b, arg, ApplySortComparator); } static int32 compare_heap_pos_int32(Datum a, Datum b, void *arg) { return compare_heap_pos_impl(a, b, arg, ApplyInt32SortComparator); } #if SIZEOF_DATUM >= 8 static int32 compare_heap_pos_signed(Datum a, Datum b, void *arg) { return compare_heap_pos_impl(a, b, arg, ApplySignedSortComparator); } #endif /* Add a new datum to the heap and perform an automatic resizing if needed. In contrast to * the binaryheap_add_unordered() function, the capacity of the heap is automatically * increased if needed. */ static pg_nodiscard binaryheap * binaryheap_add_unordered_autoresize(binaryheap *heap, Datum d) { /* Resize heap if needed */ if (heap->bh_size >= heap->bh_space) { heap->bh_space = heap->bh_space * 2; Size new_size = offsetof(binaryheap, bh_nodes) + (sizeof(Datum) * heap->bh_space); heap = (binaryheap *) repalloc(heap, new_size); } /* Insert new element */ binaryheap_add(heap, d); return heap; } static void batch_queue_heap_pop(BatchQueue *bq, DecompressContext *dcontext) { BatchQueueHeap *queue = (BatchQueueHeap *) bq; BatchArray *batch_array = &bq->batch_array; if (binaryheap_empty(queue->merge_heap)) { /* Allow this function to be called on the initial empty heap. */ return; } const int top_batch_index = DatumGetInt32(binaryheap_first(queue->merge_heap)); DecompressBatchState *top_batch = batch_array_get_at(batch_array, top_batch_index); compressed_batch_advance(dcontext, top_batch); TupleTableSlot *top_tuple = compressed_batch_current_tuple(top_batch); if (TupIsNull(top_tuple)) { /* Batch is exhausted, recycle batch_state */ (void) binaryheap_remove_first(queue->merge_heap); batch_array_clear_at(batch_array, top_batch_index); } else { /* * Update the heap entries for this batch with the current decompressed * tuple values. */ for (int key = 0; key < queue->nkeys; key++) { SortSupport sortKey = &queue->sortkeys[key]; const AttrNumber attr = AttrNumberGetAttrOffset(sortKey->ssup_attno); /* * We're working with virtual tuple slots so no need for slot_getattr(). */ Assert(TTS_IS_VIRTUAL(top_tuple)); queue->heap_entries[(top_batch_index * queue->nkeys) + key].value = top_tuple->tts_values[attr]; queue->heap_entries[(top_batch_index * queue->nkeys) + key].null = top_tuple->tts_isnull[attr]; } /* Place this batch on the heap according to its new decompressed tuple. */ binaryheap_replace_first(queue->merge_heap, Int32GetDatum(top_batch_index)); } } static bool batch_queue_heap_needs_next_batch(BatchQueue *_queue) { BatchQueueHeap *queue = (BatchQueueHeap *) _queue; if (binaryheap_empty(queue->merge_heap)) { return true; } const int top_batch_index = DatumGetInt32(binaryheap_first(queue->merge_heap)); const int comparison_result = compare_entries(&queue->heap_entries[queue->nkeys * top_batch_index], queue->last_batch_first_tuple_entry, queue->sortkeys, queue->nkeys); /* * The invariant we have to preserve is that either: * 1) the current top tuple sorts before the first tuple of the last * added batch, * 2) the input has ended. * Since the incoming batches arrive in the order of their first tuple, * if this invariant holds, then the current top tuple is found inside the * heap. * If it doesn't hold, the top tuple might be in the next incoming batches, * and we have to continue adding them. */ return comparison_result <= 0; } static void batch_queue_heap_push_batch(BatchQueue *_queue, DecompressContext *dcontext, TupleTableSlot *compressed_slot) { BatchQueueHeap *queue = (BatchQueueHeap *) _queue; BatchArray *batch_array = &queue->queue.batch_array; Assert(!TupIsNull(compressed_slot)); const int old_size = batch_array->n_batch_states; const int new_batch_index = batch_array_get_unused_slot(batch_array); if (batch_array->n_batch_states != old_size) { queue->heap_entries = repalloc(queue->heap_entries, sizeof(HeapEntryColumn) * queue->nkeys * batch_array->n_batch_states); } DecompressBatchState *batch_state = batch_array_get_at(batch_array, new_batch_index); compressed_batch_set_compressed_tuple(dcontext, batch_state, compressed_slot); compressed_batch_save_first_tuple(dcontext, batch_state, queue->last_batch_first_tuple_slot); /* * Update the heap entries for the first tuple of the last batch. */ for (int key = 0; key < queue->nkeys; key++) { SortSupport sortKey = &queue->sortkeys[key]; const AttrNumber attr = AttrNumberGetAttrOffset(sortKey->ssup_attno); /* * We're working with virtual tuple slots so no need for slot_getattr(). */ Assert(TTS_IS_VIRTUAL(queue->last_batch_first_tuple_slot)); queue->last_batch_first_tuple_entry[key].value = queue->last_batch_first_tuple_slot->tts_values[attr]; queue->last_batch_first_tuple_entry[key].null = queue->last_batch_first_tuple_slot->tts_isnull[attr]; } TupleTableSlot *current_tuple = compressed_batch_current_tuple(batch_state); if (TupIsNull(current_tuple)) { /* Might happen if there are no tuples in the batch that pass the quals. */ batch_array_clear_at(batch_array, new_batch_index); return; } /* * Update the heap entries for this batch with the first decompressed tuple * values. */ for (int key = 0; key < queue->nkeys; key++) { SortSupport sortKey = &queue->sortkeys[key]; const AttrNumber attr = AttrNumberGetAttrOffset(sortKey->ssup_attno); /* * We're working with virtual tuple slots so no need for slot_getattr(). */ Assert(TTS_IS_VIRTUAL(current_tuple)); queue->heap_entries[(new_batch_index * queue->nkeys) + key].value = current_tuple->tts_values[attr]; queue->heap_entries[(new_batch_index * queue->nkeys) + key].null = current_tuple->tts_isnull[attr]; } /* * Put the batch on the heap. */ queue->merge_heap = binaryheap_add_unordered_autoresize(queue->merge_heap, new_batch_index); } static TupleTableSlot * batch_queue_heap_top_tuple(BatchQueue *bq) { BatchQueueHeap *bqh = (BatchQueueHeap *) bq; BatchArray *batch_array = &bq->batch_array; if (binaryheap_empty(bqh->merge_heap)) { return NULL; } const int top_batch_index = DatumGetInt32(binaryheap_first(bqh->merge_heap)); DecompressBatchState *top_batch = batch_array_get_at(batch_array, top_batch_index); TupleTableSlot *top_tuple = compressed_batch_current_tuple(top_batch); Assert(!TupIsNull(top_tuple)); return top_tuple; } static void batch_queue_heap_reset(BatchQueue *bq) { BatchQueueHeap *bqh = (BatchQueueHeap *) bq; binaryheap_reset(bqh->merge_heap); } /* * Free the binary heap. */ static void batch_queue_heap_free(BatchQueue *_queue) { BatchQueueHeap *queue = (BatchQueueHeap *) _queue; BatchArray *batch_array = &queue->queue.batch_array; elog(DEBUG3, "heap has capacity of %d", queue->merge_heap->bh_space); elog(DEBUG3, "created batch states %d", batch_array->n_batch_states); batch_array_clear_all(batch_array); pfree(queue->heap_entries); binaryheap_free(queue->merge_heap); queue->merge_heap = NULL; pfree(queue->sortkeys); ExecDropSingleTupleTableSlot(queue->last_batch_first_tuple_slot); pfree(queue->last_batch_first_tuple_entry); batch_array_destroy(batch_array); pfree(queue); } const struct BatchQueueFunctions BatchQueueFunctionsHeap = { .free = batch_queue_heap_free, .needs_next_batch = batch_queue_heap_needs_next_batch, .pop = batch_queue_heap_pop, .push_batch = batch_queue_heap_push_batch, .reset = batch_queue_heap_reset, .top_tuple = batch_queue_heap_top_tuple, }; static SortSupport build_batch_sorted_merge_info(const List *sortinfo, int *nkeys) { Assert(sortinfo != NULL); List *sort_col_idx = linitial(sortinfo); List *sort_ops = lsecond(sortinfo); List *sort_collations = lthird(sortinfo); List *sort_nulls = lfourth(sortinfo); *nkeys = list_length(linitial((sortinfo))); Assert(list_length(sort_col_idx) == list_length(sort_ops)); Assert(list_length(sort_ops) == list_length(sort_collations)); Assert(list_length(sort_collations) == list_length(sort_nulls)); Assert(*nkeys > 0); SortSupportData *sortkeys = palloc0(sizeof(SortSupportData) * *nkeys); /* Inspired by nodeMergeAppend.c */ for (int i = 0; i < *nkeys; i++) { SortSupportData *sortkey = &sortkeys[i]; sortkey->ssup_cxt = CurrentMemoryContext; sortkey->ssup_collation = list_nth_oid(sort_collations, i); sortkey->ssup_nulls_first = list_nth_oid(sort_nulls, i); sortkey->ssup_attno = list_nth_oid(sort_col_idx, i); /* * It isn't feasible to perform abbreviated key conversion, since * tuples are pulled into mergestate's binary heap as needed. It * would likely be counter-productive to convert tuples into an * abbreviated representation as they're pulled up, so opt out of that * additional optimization entirely. */ sortkey->abbreviate = false; PrepareSortSupportFromOrderingOp(list_nth_oid(sort_ops, i), sortkey); } return sortkeys; } BatchQueue * batch_queue_heap_create(int num_compressed_cols, const List *sortinfo, const TupleDesc result_tupdesc, const BatchQueueFunctions *funcs) { BatchQueueHeap *queue = palloc0(sizeof(BatchQueueHeap)); batch_array_init(&queue->queue.batch_array, INITIAL_BATCH_CAPACITY, num_compressed_cols); queue->sortkeys = build_batch_sorted_merge_info(sortinfo, &queue->nkeys); queue->heap_entries = palloc(sizeof(HeapEntryColumn) * queue->nkeys * INITIAL_BATCH_CAPACITY); /* * Choose a specialization for faster comparison of the first column. This is * the approach that tuplesort uses, see e.g. qsort_tuple_signed(). * The ssup_datum_unsigned_cmp is used only for abbreviated keys which the * batch sorted merge doesn't use, so we use a generic comparator in this * case. */ binaryheap_comparator comparator = compare_heap_pos_generic; if (queue->sortkeys[0].comparator == ssup_datum_int32_cmp) { comparator = compare_heap_pos_int32; } #if SIZEOF_DATUM >= 8 else if (queue->sortkeys[0].comparator == ssup_datum_signed_cmp) { comparator = compare_heap_pos_signed; } #endif queue->merge_heap = binaryheap_allocate(INITIAL_BATCH_CAPACITY, comparator, queue); queue->last_batch_first_tuple_slot = MakeSingleTupleTableSlot(result_tupdesc, &TTSOpsVirtual); queue->last_batch_first_tuple_entry = palloc(sizeof(HeapEntryColumn) * queue->nkeys); queue->queue.funcs = funcs; return &queue->queue; } ================================================ FILE: tsl/src/nodes/columnar_scan/batch_queue_heap.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include "batch_queue.h" extern BatchQueue *batch_queue_heap_create(int num_compressed_cols, const List *sortinfo, const TupleDesc result_tupdesc, const BatchQueueFunctions *funcs); extern const struct BatchQueueFunctions BatchQueueFunctionsHeap; ================================================ FILE: tsl/src/nodes/columnar_scan/columnar_scan.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include "chunk.h" #include "hypertable_cache.h" #include <catalog/pg_operator.h> #include <math.h> #include <miscadmin.h> #include <nodes/bitmapset.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <optimizer/cost.h> #include <optimizer/optimizer.h> #include <optimizer/pathnode.h> #include <optimizer/paths.h> #include <parser/parse_relation.h> #include <parser/parsetree.h> #include <planner/planner.h> #include <storage/lockdefs.h> #include <utils/builtins.h> #include <utils/lsyscache.h> #include <utils/syscache.h> #include <utils/typcache.h> #include <planner.h> #include "compat/compat.h" #include "compression/compression.h" #include "compression/create.h" #include "cross_module_fn.h" #include "custom_type_cache.h" #include "debug_assert.h" #include "import/allpaths.h" #include "import/planner.h" #include "nodes/columnar_scan/columnar_scan.h" #include "nodes/columnar_scan/planner.h" #include "nodes/columnar_scan/qual_pushdown.h" #include "ts_catalog/array_utils.h" #include "utils.h" static CustomPathMethods columnar_scan_path_methods = { .CustomName = "ColumnarScan", .PlanCustomPath = columnar_scan_plan_create, }; typedef struct SortInfo { List *required_compressed_pathkeys; List *required_eq_classes; bool needs_sequence_num; bool use_compressed_sort; /* sort can be pushed below ColumnarScan */ bool use_batch_sorted_merge; bool reverse; List *decompressed_sort_pathkeys; QualCost decompressed_sort_pathkeys_cost; } SortInfo; static RangeTblEntry *columnar_scan_make_rte(Oid compressed_relid, LOCKMODE lockmode, Query *parse); static void create_compressed_scan_paths(PlannerInfo *root, RelOptInfo *compressed_rel, const CompressionInfo *compression_info, const SortInfo *sort_info); static ColumnarScanPath *columnar_scan_path_create(PlannerInfo *root, const CompressionInfo *info, Path *compressed_path); static void columnar_scan_add_plannerinfo(PlannerInfo *root, CompressionInfo *info, const Chunk *chunk, RelOptInfo *chunk_rel, bool needs_sequence_num); static SortInfo build_sortinfo(PlannerInfo *root, const Chunk *chunk, RelOptInfo *chunk_rel, const CompressionInfo *info, List *pathkeys); static Bitmapset *find_const_segmentby(RelOptInfo *chunk_rel, const CompressionInfo *info); static EquivalenceClass * append_ec_for_seqnum(PlannerInfo *root, const CompressionInfo *info, const SortInfo *sort_info, Var *var, Oid sortop, bool nulls_first) { MemoryContext oldcontext = MemoryContextSwitchTo(root->planner_cxt); Oid opfamily, opcintype, equality_op; CompareType strategy; List *opfamilies; EquivalenceClass *newec = makeNode(EquivalenceClass); EquivalenceMember *em = makeNode(EquivalenceMember); /* Find the operator in pg_amop --- failure shouldn't happen */ if (!get_ordering_op_properties(sortop, &opfamily, &opcintype, &strategy)) elog(ERROR, "operator %u is not a valid ordering operator", sortop); /* * EquivalenceClasses need to contain opfamily lists based on the family * membership of mergejoinable equality operators, which could belong to * more than one opfamily. So we have to look up the opfamily's equality * operator and get its membership. */ equality_op = get_opfamily_member(opfamily, opcintype, opcintype, BTEqualStrategyNumber); if (!OidIsValid(equality_op)) /* shouldn't happen */ elog(ERROR, "missing operator %d(%u,%u) in opfamily %u", BTEqualStrategyNumber, opcintype, opcintype, opfamily); opfamilies = get_mergejoin_opfamilies(equality_op); if (!opfamilies) /* certainly should find some */ elog(ERROR, "could not find opfamilies for equality operator %u", equality_op); em->em_expr = (Expr *) var; em->em_relids = bms_make_singleton(info->compressed_rel->relid); #if PG16_LT em->em_nullable_relids = NULL; #endif em->em_is_const = false; em->em_is_child = false; em->em_datatype = INT4OID; newec->ec_opfamilies = list_copy(opfamilies); newec->ec_collation = 0; newec->ec_members = list_make1(em); newec->ec_sources = NIL; newec->ec_derives_list = NIL; newec->ec_relids = bms_make_singleton(info->compressed_rel->relid); newec->ec_has_const = false; newec->ec_has_volatile = false; #if PG16_LT newec->ec_below_outer_join = false; #endif newec->ec_broken = false; newec->ec_sortref = 0; newec->ec_min_security = UINT_MAX; newec->ec_max_security = 0; newec->ec_merged = NULL; info->compressed_rel->eclass_indexes = bms_add_member(info->compressed_rel->eclass_indexes, list_length(root->eq_classes)); root->eq_classes = lappend(root->eq_classes, newec); MemoryContextSwitchTo(oldcontext); return newec; } static EquivalenceClass * append_ec_for_metadata_col(PlannerInfo *root, const CompressionInfo *info, Expr *expr, PathKey *pk, Oid em_datatype) { MemoryContext oldcontext = MemoryContextSwitchTo(root->planner_cxt); EquivalenceMember *em = makeNode(EquivalenceMember); em->em_expr = expr; em->em_relids = bms_make_singleton(info->compressed_rel->relid); em->em_is_const = false; em->em_is_child = false; em->em_datatype = em_datatype; EquivalenceClass *ec = makeNode(EquivalenceClass); ec->ec_opfamilies = pk->pk_eclass->ec_opfamilies; ec->ec_collation = pk->pk_eclass->ec_collation; ec->ec_members = list_make1(em); ec->ec_sources = list_copy(pk->pk_eclass->ec_sources); ec->ec_derives_list = list_copy(pk->pk_eclass->ec_derives_list); ec->ec_relids = bms_make_singleton(info->compressed_rel->relid); ec->ec_has_const = pk->pk_eclass->ec_has_const; ec->ec_has_volatile = pk->pk_eclass->ec_has_volatile; #if PG16_LT ec->ec_below_outer_join = pk->pk_eclass->ec_below_outer_join; #endif ec->ec_broken = pk->pk_eclass->ec_broken; ec->ec_sortref = pk->pk_eclass->ec_sortref; ec->ec_min_security = pk->pk_eclass->ec_min_security; ec->ec_max_security = pk->pk_eclass->ec_max_security; ec->ec_merged = pk->pk_eclass->ec_merged; root->eq_classes = lappend(root->eq_classes, ec); MemoryContextSwitchTo(oldcontext); info->compressed_rel->eclass_indexes = bms_add_member(info->compressed_rel->eclass_indexes, root->eq_classes->length - 1); return ec; } static List * build_compressed_scan_pathkeys(const SortInfo *sort_info, PlannerInfo *root, List *chunk_pathkeys, const CompressionInfo *info) { Var *var; int varattno; List *required_compressed_pathkeys = NIL; ListCell *lc = NULL; PathKey *pk; /* * all segmentby columns need to be prefix of pathkeys * except those with equality constraint in baserestrictinfo */ if (info->num_segmentby_columns > 0) { TimescaleDBPrivate *compressed_fdw_private = (TimescaleDBPrivate *) info->compressed_rel->fdw_private; /* * We don't need any sorting for the segmentby columns that are equated * to a constant. The respective constant ECs are excluded from * canonical pathkeys, so we won't see these columns here. Count them as * seen from the start, so that we arrive at the proper counts of seen * segmentby columns in the end. */ for (lc = list_head(chunk_pathkeys); lc; lc = lnext(chunk_pathkeys, lc)) { PathKey *pk = lfirst(lc); EquivalenceMember *compressed_em = NULL; ListCell *ec_em_pair_cell; foreach (ec_em_pair_cell, compressed_fdw_private->compressed_ec_em_pairs) { List *pair = lfirst(ec_em_pair_cell); if (linitial(pair) == pk->pk_eclass) { compressed_em = lsecond(pair); break; } } /* * We should exit the loop after we've seen all required segmentby * columns. If we haven't seen them all, but the next pathkey * already refers a compressed column, it is a bug. See * build_sortinfo(). */ if (!compressed_em) break; required_compressed_pathkeys = lappend(required_compressed_pathkeys, pk); } } /* * If pathkeys contains non-segmentby columns the rest of the ordering * requirements will be satisfied by ordering by sequence_num. */ if (sort_info->needs_sequence_num) { /* TODO: split up legacy sequence number path and non-sequence number path into dedicated * functions. */ if (info->has_seq_num) { bool nulls_first; Oid sortop; varattno = get_attnum(info->compressed_rte->relid, COMPRESSION_COLUMN_METADATA_SEQUENCE_NUM_NAME); var = makeVar(info->compressed_rel->relid, varattno, INT4OID, -1, InvalidOid, 0); if (sort_info->reverse) { sortop = get_commutator(Int4LessOperator); nulls_first = true; } else { sortop = Int4LessOperator; nulls_first = false; } /* * Create the EquivalenceClass for the sequence number column of this * compressed chunk, so that we can build the PathKey that refers to it. */ EquivalenceClass *ec = append_ec_for_seqnum(root, info, sort_info, var, sortop, nulls_first); /* Find the operator in pg_amop --- failure shouldn't happen. */ Oid opfamily, opcintype; CompareType strategy; if (!get_ordering_op_properties(sortop, &opfamily, &opcintype, &strategy)) elog(ERROR, "operator %u is not a valid ordering operator", sortop); pk = make_canonical_pathkey(root, ec, opfamily, strategy, nulls_first); required_compressed_pathkeys = lappend(required_compressed_pathkeys, pk); } else { /* If there are no segmentby pathkeys, start from the beginning of the list */ if (info->num_segmentby_columns == 0) { lc = list_head(chunk_pathkeys); } Assert(lc != NULL); Expr *expr; char *column_name; for (; lc != NULL; lc = lnext(chunk_pathkeys, lc)) { pk = lfirst(lc); EquivalenceMember *chunk_em = ts_find_em_for_rel(pk->pk_eclass, info->chunk_rel); Assert(chunk_em); expr = chunk_em->em_expr; /* * Use em_datatype from the original equivalence member as the * opcintype. For polymorphic types like anyenum, * canonicalize_ec_expression will not add a RelabelType (it * replaces polymorphic req_type with the concrete type), so we * must explicitly pass the correct em_datatype to the metadata * column EC. */ Oid opcintype = chunk_em->em_datatype; Oid collation = exprCollation((Node *) expr); expr = (Expr *) strip_implicit_coercions((Node *) expr); var = castNode(Var, expr); Assert(var->varattno > 0); column_name = get_attname(info->chunk_rte->relid, var->varattno, false); int16 orderby_index = ts_array_position(info->settings->fd.orderby, column_name); varattno = get_attnum(info->compressed_rte->relid, column_segment_min_name(orderby_index)); Assert(orderby_index != 0); bool orderby_desc = ts_array_get_element_bool(info->settings->fd.orderby_desc, orderby_index); bool orderby_nullsfirst = ts_array_get_element_bool(info->settings->fd.orderby_nullsfirst, orderby_index); bool nulls_first; CompareType strategy; if (sort_info->reverse) { strategy = orderby_desc ? BTLessStrategyNumber : BTGreaterStrategyNumber; nulls_first = !orderby_nullsfirst; } else { strategy = orderby_desc ? BTGreaterStrategyNumber : BTLessStrategyNumber; nulls_first = orderby_nullsfirst; } Var *metadata_var = makeVar(info->compressed_rel->relid, varattno, var->vartype, var->vartypmod, var->varcollid, var->varlevelsup); Expr *min_expr = canonicalize_ec_expression((Expr *) metadata_var, opcintype, collation); EquivalenceClass *min_ec = append_ec_for_metadata_col(root, info, min_expr, pk, opcintype); PathKey *min = make_canonical_pathkey(root, min_ec, pk->pk_opfamily, strategy, nulls_first); required_compressed_pathkeys = lappend(required_compressed_pathkeys, min); varattno = get_attnum(info->compressed_rte->relid, column_segment_max_name(orderby_index)); metadata_var = makeVar(info->compressed_rel->relid, varattno, var->vartype, var->vartypmod, var->varcollid, var->varlevelsup); Expr *max_expr = canonicalize_ec_expression((Expr *) metadata_var, opcintype, collation); EquivalenceClass *max_ec = append_ec_for_metadata_col(root, info, max_expr, pk, opcintype); PathKey *max = make_canonical_pathkey(root, max_ec, pk->pk_opfamily, strategy, nulls_first); required_compressed_pathkeys = lappend(required_compressed_pathkeys, max); } } } return required_compressed_pathkeys; } ColumnarScanPath * copy_columnar_scan_path(ColumnarScanPath *src) { Assert(ts_is_columnar_scan_path(&src->custom_path.path)); ColumnarScanPath *dst = palloc(sizeof(ColumnarScanPath)); memcpy(dst, src, sizeof(ColumnarScanPath)); return dst; } /* * Maps the attno of the min metadata column in the compressed chunk to the * attno of the corresponding max metadata column. Zero if none or not applicable. */ typedef struct SelectivityEstimationContext { AttrNumber *min_to_max; AttrNumber *max_to_min; List *vars; } SelectivityEstimationContext; /* * Collect the Vars referencing the "min" metadata columns into the context->vars. */ static bool min_metadata_vars_collector(Node *orig_node, SelectivityEstimationContext *context) { if (orig_node == NULL) { /* * An expression node can have a NULL field and the mutator will be * still called for it, so we have to handle this. */ return false; } if (!IsA(orig_node, Var)) { /* * Recurse. */ return expression_tree_walker(orig_node, min_metadata_vars_collector, context); } Var *orig_var = castNode(Var, orig_node); if (orig_var->varattno <= 0) { /* * We don't handle special variables. Not sure how it could happen though. */ return false; } AttrNumber replaced_attno = context->min_to_max[orig_var->varattno]; if (replaced_attno == InvalidAttrNumber) { /* * No replacement for this column. */ return false; } context->vars = lappend(context->vars, orig_var); return false; } static void set_compressed_baserel_size_estimates(PlannerInfo *root, RelOptInfo *rel, CompressionInfo *compression_info) { /* * We need some custom selectivity estimation code for the compressed chunk * table, because some pushed down filters require special handling. * * An equality condition can be pushed down to the minmax sparse index * condition, and becomes x_min <= const and const <= x_max. Postgres * treats the part of this condition as independent, which leads to * significant overestimates when x has high cardinality, and therefore * not using the Index Scan. This stems from the fact that Postgres doesn't * know that x_max is always just very slightly more than x_min for the * given compressed batch. * To work around this, temporarily replace all conditions on x_min with * conditions on x_max before feeding them to the Postgres clauselist * selectivity functions. Since the range of x_min to x_max for a given * batch is small relative to the range of x in the entire chunk, this * should not introduce much error, but at the same time allow Postgres to * see the correlation. * * We do this here for the entire baserestrictinfo and not per-rinfo as we * add them during filter pushdown, because the Postgres clauselist * selectivity estimator must see the entire clause list to detect the range * conditions. * * First, build the correspondence of min metadata attno -> max metadata * attno for all minmax metadata. */ const int storage_elements = 2 * (compression_info->compressed_rel->max_attr + 1); AttrNumber *storage = palloc0(storage_elements * sizeof(*storage)); SelectivityEstimationContext context = { .min_to_max = &storage[0], .max_to_min = &storage[compression_info->compressed_rel->max_attr], }; for (int uncompressed_attno = 1; uncompressed_attno <= compression_info->chunk_rel->max_attr; uncompressed_attno++) { if (get_rte_attribute_is_dropped(compression_info->chunk_rte, uncompressed_attno)) { /* Skip the dropped column. */ continue; } const char *attname = get_attname(compression_info->chunk_rte->relid, uncompressed_attno, /* missing_ok = */ false); const int16 orderby_pos = ts_array_position(compression_info->settings->fd.orderby, attname); if (orderby_pos == 0) { /* * This reasoning is only applicable to orderby columns, where each * batch is a thin slice of the entire range of the column. It also does * not have many intersections, because the compressed batches mostly * follow the total order of orderby columns, that is relaxed for the * last orderby columns or unordered chunks.This does not necessarily * hold for non-orderby columns that can also have a sparse index. */ continue; } AttrNumber min_attno = compressed_column_metadata_attno(compression_info->settings, compression_info->chunk_rte->relid, uncompressed_attno, compression_info->compressed_rte->relid, "min"); AttrNumber max_attno = compressed_column_metadata_attno(compression_info->settings, compression_info->chunk_rte->relid, uncompressed_attno, compression_info->compressed_rte->relid, "max"); if (min_attno == InvalidAttrNumber || max_attno == InvalidAttrNumber) { continue; } Assert(&context.min_to_max[min_attno] < &storage[storage_elements]); Assert(&context.max_to_min[max_attno] < &storage[storage_elements]); context.min_to_max[min_attno] = max_attno; context.max_to_min[max_attno] = min_attno; } /* * Then, replace all conditions on min metadata column with conditions on * max metadata column. */ ListCell *lc; foreach (lc, rel->baserestrictinfo) { RestrictInfo *orig_restrictinfo = castNode(RestrictInfo, lfirst(lc)); Node *orig_clause = (Node *) orig_restrictinfo->clause; expression_tree_walker(orig_clause, min_metadata_vars_collector, &context); } /* * Temporarily replace "min" with "max" in-place to save on memory allocations. */ foreach (lc, context.vars) { Var *var = castNode(Var, lfirst(lc)); Assert(var->varattno != InvalidAttrNumber); Assert(context.min_to_max[var->varattno] != InvalidAttrNumber); Assert(context.max_to_min[context.min_to_max[var->varattno]] == var->varattno); var->varattno = context.min_to_max[var->varattno]; } /* * Compute selectivity with the updated filters. */ set_baserel_size_estimates(root, rel); /* * Replace the Vars back. */ foreach (lc, context.vars) { Var *var = castNode(Var, lfirst(lc)); var->varattno = context.max_to_min[var->varattno]; } pfree(storage); } static CompressionInfo * build_compressioninfo(PlannerInfo *root, const Hypertable *ht, const Chunk *chunk, RelOptInfo *chunk_rel) { AppendRelInfo *appinfo; CompressionInfo *info = palloc0(sizeof(CompressionInfo)); info->compresseddata_oid = ts_custom_type_cache_get(CUSTOM_TYPE_COMPRESSED_DATA)->type_oid; info->chunk_rel = chunk_rel; info->chunk_rte = planner_rt_fetch(chunk_rel->relid, root); info->settings = ts_compression_settings_get(chunk->table_id); if (chunk_rel->reloptkind == RELOPT_OTHER_MEMBER_REL) { appinfo = ts_get_appendrelinfo(root, chunk_rel->relid, false); RangeTblEntry *rte = planner_rt_fetch(appinfo->parent_relid, root); if (rte->rtekind == RTE_RELATION) { info->ht_rte = rte; info->ht_rel = root->simple_rel_array[appinfo->parent_relid]; } else { /* In UNION queries referencing chunks directly, the parent rel can be a subquery */ Assert(rte->rtekind == RTE_SUBQUERY); info->single_chunk = true; info->ht_rte = info->chunk_rte; info->ht_rel = info->chunk_rel; } } else { Assert(chunk_rel->reloptkind == RELOPT_BASEREL); info->single_chunk = true; info->ht_rte = info->chunk_rte; info->ht_rel = info->chunk_rel; } info->hypertable_id = ht->fd.id; info->num_orderby_columns = ts_array_length(info->settings->fd.orderby); info->num_segmentby_columns = ts_array_length(info->settings->fd.segmentby); if (info->num_segmentby_columns) { ArrayIterator it = array_create_iterator(info->settings->fd.segmentby, 0, NULL); Datum datum; bool isnull; while (array_iterate(it, &datum, &isnull)) { Ensure(!isnull, "NULL element in catalog array"); AttrNumber chunk_attno = get_attnum(info->chunk_rte->relid, TextDatumGetCString(datum)); info->chunk_segmentby_attnos = bms_add_member(info->chunk_segmentby_attnos, chunk_attno); } } info->has_seq_num = get_attnum(info->settings->fd.compress_relid, COMPRESSION_COLUMN_METADATA_SEQUENCE_NUM_NAME) != InvalidAttrNumber; info->chunk_const_segmentby = find_const_segmentby(chunk_rel, info); /* * If the chunk is member of hypertable expansion or a UNION, find its * parent relation ids. We will use it later to filter out some parameterized * paths. */ if (chunk_rel->reloptkind == RELOPT_OTHER_MEMBER_REL) { info->parent_relids = find_childrel_parents(root, chunk_rel); } info->chunk_status = chunk->fd.status; return info; } /* * Estimate the average count of elements in the compressed batch based on the * Postgres statistics for _ts_meta_count column. * Returns TARGET_COMPRESSED_BATCH_SIZE when no pg_statistic entry exists. */ double ts_columnar_estimate_compressed_batch_size(const Oid relid) { AttrNumber attnum = get_attnum(relid, "_ts_meta_count"); if (attnum == InvalidAttrNumber) return TARGET_COMPRESSED_BATCH_SIZE; /* fetch statistics */ HeapTuple statsTuple = SearchSysCache3(STATRELATTINH, ObjectIdGetDatum(relid), Int16GetDatum(attnum), BoolGetDatum(false)); if (!HeapTupleIsValid(statsTuple)) { return TARGET_COMPRESSED_BATCH_SIZE; } double mcv_sum = 0.0; double mcv_freq = 0.0; /* exact MCV contribution */ AttStatsSlot mcvslot; if (get_attstatsslot(&mcvslot, statsTuple, STATISTIC_KIND_MCV, InvalidOid, ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS)) { for (int i = 0; i < mcvslot.nvalues; i++) { double val = (double) DatumGetInt32(mcvslot.values[i]); double freq = (double) mcvslot.numbers[i]; mcv_sum += val * freq; mcv_freq += freq; } free_attstatsslot(&mcvslot); } double hist_sum = 0.0; /* histogram contribution */ AttStatsSlot histslot; if (get_attstatsslot(&histslot, statsTuple, STATISTIC_KIND_HISTOGRAM, InvalidOid, ATTSTATSSLOT_VALUES)) { int buckets = histslot.nvalues - 1; if (buckets > 0 && mcv_freq < 1.0) { for (int i = 0; i < buckets; i++) { double lo = (double) DatumGetInt32(histslot.values[i]); double hi = (double) DatumGetInt32(histslot.values[i + 1]); hist_sum += (lo + hi) / 2.0; } hist_sum *= (1.0 - mcv_freq) / buckets; } free_attstatsslot(&histslot); } ReleaseSysCache(statsTuple); const double final_result = mcv_sum + hist_sum; if (final_result == 0) { /* * For tables with few rows, the statistics tuple will contain all zero * values. We shouldn't return zero in this case to avoid weird behavior. */ return TARGET_COMPRESSED_BATCH_SIZE; } return final_result; } /* * calculate cost for ColumnarScanPath * * since we have to read whole batch before producing tuple * we put cost of 1 tuple of compressed_scan as startup cost */ static void cost_columnar_scan(PlannerInfo *root, const CompressionInfo *compression_info, Path *path, Path *compressed_path) { /* startup_cost is cost before fetching first tuple */ const double compressed_rows = Max(1, compressed_path->rows); path->startup_cost = compressed_path->startup_cost + (compressed_path->total_cost - compressed_path->startup_cost) / compressed_rows; /* total_cost is cost for fetching all tuples */ path->rows = compressed_path->rows * compression_info->compressed_batch_size; path->total_cost = compressed_path->total_cost + path->rows * cpu_tuple_cost; #if PG18_GE /* PG18 changes the way we handle disabled nodes so we * need to take those into account as well. * * https://github.com/postgres/postgres/commit/e2225346 */ path->disabled_nodes = compressed_path->disabled_nodes; #endif } /* Smoothstep function S1 (the h01 cubic Hermite spline). */ static double smoothstep(double x, double start, double end) { x = (x - start) / (end - start); if (x < 0) { x = 0; } else if (x > 1) { x = 1; } return x * x * (3.0F - 2.0F * x); } /* * If the query 'order by' is prefix of the compression 'order by' (or equal), we can exploit * the ordering of the individual batches to create a total ordered result without resorting * the tuples. This speeds up all queries that use this ordering (because no sort node is * needed). In particular, queries that use a LIMIT are speed-up because only the top elements * of the affected batches needs to be decompressed. Without the optimization, the entire batches * are decompressed, sorted, and then the top elements are taken from the result. * * The idea is to do something similar to the MergeAppend node; a BinaryHeap is used * to merge the per segment by column sorted individual batches into a sorted result. So, we end * up which a data flow which looks as follows: * * ColumnarScan * * Decompress Batch 1 * * Decompress Batch 2 * * Decompress Batch 3 * [....] * * Decompress Batch N * * Using the presorted batches, we are able to open these batches dynamically. If we don't presort * them, we would have to open all batches at the same time. This would be similar to the work the * MergeAppend does, but this is not needed in our case and we could reduce the size of the heap and * the amount of parallel open batches. * * The algorithm works as follows: * * (1) A sort node is placed below the decompress scan node and on top of the scan * on the compressed chunk. This sort node uses the min/max values of the 'order by' * columns from the metadata of the batch to get them into an order which can be * used to merge them. * * [Scan on compressed chunk] -> [Sort on min/max values] -> [Decompress and merge] * * For example, the batches are sorted on the min value of the 'order by' metadata * column: [0, 3] [0, 5] [3, 7] [6, 10] * * (2) The decompress chunk node initializes a binary heap, opens the first batch and * decompresses the first tuple from the batch. The tuple is put on the heap. In addition * the opened batch is marked as the most recent batch (MRB). * * (3) As soon as a tuple is requested from the heap, the following steps are performed: * (3a) If the heap is empty, we are done. * (3b) The top tuple from the heap is taken. It is checked if this tuple is from the * MRB. If this is the case, the next batch is opened, the first tuple is decompressed, * placed on the heap and this batch is marked as MRB. This is repeated until the * top tuple from the heap is not from the MRB. After the top tuple is not from the * MRB, all batches (and one ahead) which might contain the most recent tuple are * opened and placed on the heap. * * In the example above, the first three batches are opened because the first two * batches might contain tuples with a value of 0. * (3c) The top element from the heap is removed, the next tuple from the batch is * decompressed (if present) and placed on the heap. * (3d) The former top tuple of the heap is returned. * * This function calculate the costs for retrieving the decompressed in-order * using a binary heap. */ static void cost_batch_sorted_merge(PlannerInfo *root, const CompressionInfo *compression_info, ColumnarScanPath *dcpath, Path *compressed_path) { Path sort_path; /* dummy for result of cost_sort */ /* * Don't disable the compressed batch sorted merge plan with the enable_sort * GUC. We have a separate GUC for it, and this way you can try to force the * batch sorted merge plan by disabling sort. */ const bool old_enable_sort = enable_sort; enable_sort = true; cost_sort(&sort_path, root, dcpath->required_compressed_pathkeys, #if PG18_GE compressed_path->disabled_nodes, #endif compressed_path->total_cost, compressed_path->rows, compressed_path->pathtarget->width, 0.0, work_mem, -1); enable_sort = old_enable_sort; /* * In compressed batch sorted merge, for each distinct segmentby value we * have to keep the corresponding latest batch open. Estimate the number of * these batches with the usual Postgres estimator for grouping cardinality. */ List *segmentby_groupexprs = NIL; for (int segmentby_attno = bms_next_member(compression_info->chunk_segmentby_attnos, -1); segmentby_attno > 0; segmentby_attno = bms_next_member(compression_info->chunk_segmentby_attnos, segmentby_attno)) { char *colname = get_attname(compression_info->chunk_rte->relid, segmentby_attno, /* missing_ok = */ false); AttrNumber compressed_attno = get_attnum(compression_info->compressed_rte->relid, colname); Ensure(compressed_attno != InvalidAttrNumber, "segmentby column %s not found in compressed chunk %d", colname, compression_info->compressed_rte->relid); Var *var = palloc(sizeof(Var)); *var = (Var){ .xpr.type = T_Var, .varno = compression_info->compressed_rel->relid, .varattno = compressed_attno }; segmentby_groupexprs = lappend(segmentby_groupexprs, var); } const double open_batches_estimated = estimate_num_groups(root, segmentby_groupexprs, dcpath->custom_path.path.rows, NULL, NULL); Assert(open_batches_estimated > 0); /* * We can't have more open batches than the total number of compressed rows, * so clamp it for sanity of the following calculations. */ const double open_batches_clamped = Min(open_batches_estimated, sort_path.rows); /* * Keeping a lot of batches open might use a lot of memory. The batch sorted * merge can't offload anything to disk, so we just penalize it heavily if * we expect it to go over the work_mem. First, estimate the amount of * memory we'll need. We do this on the basis of uncompressed chunk width, * as if we had to materialize entire decompressed batches. This might * be less precise when bulk decompression is not used, because we * materialize only the compressed data which is smaller. But it accounts * for projections, which is probably more important than precision, because * we often read a small subset of columns in analytical queries. The * compressed chunk is never projected so we can't use it for that. */ const double work_mem_bytes = work_mem * 1024.0; const double needed_memory_bytes = open_batches_clamped * compression_info->compressed_batch_size * dcpath->custom_path.path.pathtarget->width; /* * Next, calculate the cost penalty. It is a smooth step, starting at 75% of * work_mem, and ending at 125%. We want to effectively disable this plan * if it doesn't fit into the available memory, so the penalty should be * comparable to disable_cost but still less than it, so that the * manual disables still have priority. */ const double work_mem_penalty = 0.1 * disable_cost * smoothstep(needed_memory_bytes, 0.75 * work_mem_bytes, 1.25 * work_mem_bytes); Assert(work_mem_penalty >= 0); /* * startup_cost is cost before fetching first tuple. Batch sorted merge has * to load at least the number of batches we expect to be open * simultaneously, before it can produce the first row. */ const double sort_path_cost_for_startup = sort_path.startup_cost + ((sort_path.total_cost - sort_path.startup_cost) * (open_batches_clamped / sort_path.rows)); Assert(sort_path_cost_for_startup >= 0); dcpath->custom_path.path.startup_cost = sort_path_cost_for_startup + work_mem_penalty; /* * Finally, to run this path to completion, we have to complete the * underlying sort path, and return all uncompressed rows. Getting one * uncompressed row involves replacing the top row in the heap, which costs * O(log(heap size)). The constant multiplier is found empirically by * benchmarking the queries returning 1 - 1e9 tuples, with segmentby * cardinality 1 to 1e4, and adjusting the cost so that the fastest plan is * used. The "+ 1" under the logarithm is to avoid zero uncompressed row cost * when we expect to have only 1 batch open. */ const double sort_path_cost_rest = sort_path.total_cost - sort_path_cost_for_startup; Assert(sort_path_cost_rest >= 0); const double uncompressed_row_cost = 1.5 * log(open_batches_clamped + 1) * cpu_tuple_cost; Assert(uncompressed_row_cost > 0); dcpath->custom_path.path.total_cost = dcpath->custom_path.path.startup_cost + sort_path_cost_rest + dcpath->custom_path.path.rows * uncompressed_row_cost; #if PG18_GE /* PG18 changes the way we handle disabled nodes so we * need to take those into account as well. * * https://github.com/postgres/postgres/commit/e2225346 */ dcpath->custom_path.path.disabled_nodes = sort_path.disabled_nodes; #endif } /* * This function adds per-chunk sorted paths for compressed chunks if beneficial. This has two * advantages: * * (1) Make ChunkAppend possible. If at least one chunk of a hypertable is uncompressed, PostgreSQL * will generate a MergeAppend path in generate_orderedappend_paths() / create_merge_append_path() * due to the existing pathkeys of the index on the uncompressed chunk. If all chunks are * compressed, no path keys are present and no MergeAppend path is generated by PostgreSQL. In that * case, the ChunkAppend optimization cannot be used because MergeAppend path can be promoted in * ts_chunk_append_path_create(). Adding a sorted path with pathkeys makes ChunkAppend possible for * these queries. * * (2) Sorting on a per-chunk basis and merging / appending these results could be faster than * sorting the whole input. Especially limit queries that use an ORDER BY that is compatible with * the partitioning of the hypertable could be inefficiently executed otherwise. For example, an * expensive query plan with a sort node on top of the append node could be chosen. Due to the sort * node at the high level in the query plan and the missing ChunkAppend node (see (1)), all chunks * are decompressed (instead of only the actually needed ones). * * If existing index pathkeys do not match query pathkeys and sort cannot be pushed down * into compressed index, for example "SELECT * FROM ... ORDER BY (segcol, time DESC, some_col)" if * compressed index is (segcol, time DESC), we should allow SortPath over (ColumnarScan * <- IndexScan) for such cases, i.e. should consider IndexScan compressed paths along with SeqScan * compressed paths. IndexScans with useful index conditions can be cheaper than SeqScans. * * The logic is inspired by PostgreSQL's add_paths_with_pathkeys_for_rel() function. * * Note: This function adds only non-partial paths. In parallel plans PostgreSQL prefers sorting * directly under the gather (merge) node and the per-chunk sorting are not used in parallel plans. * To save planning time, we therefore refrain from adding them. */ static Path * make_chunk_sorted_path(PlannerInfo *root, RelOptInfo *chunk_rel, Path *path, Path *compressed_path, const SortInfo *sort_info) { /* * Don't have a useful sorting after decompression. */ if (sort_info->decompressed_sort_pathkeys == NIL) { return NULL; } Assert(ts_is_columnar_scan_path(path)); /* * We should be given an unsorted ColumnarScan path. */ Assert(path->pathkeys == NIL); /* * Create the sorted path for these useful_pathkeys. Copy the decompress * chunk path because the original can be recycled in add_path, and our * sorted path must be independent. */ ColumnarScanPath *path_copy = copy_columnar_scan_path((ColumnarScanPath *) path); /* * Create the Sort path. */ Path *sorted_path = (Path *) create_sort_path(root, chunk_rel, (Path *) path_copy, sort_info->decompressed_sort_pathkeys, root->limit_tuples); /* Set in "create_sort_path" in PG18GE, have to set separately for PG17LE. * Need to preserve info for sort over parametrized index paths. */ sorted_path->param_info = path->param_info; /* * Now, we need another dumb workaround for Postgres problems. When creating * a sort plan, it performs a linear search of equivalence member of a * pathkey's equivalence class, that matches the sorted relation (see * prepare_sort_from_pathkeys()). This is effectively quadratic in the * number of chunks, and becomes a real CPU sink after we pass 1k chunks. * Try to reflect this in the costs, because in some cases a chunk-wise sort * might be avoided, e.g. Limit 1 over MergeAppend over chunk-wise Sort can * be just as well replaced with a Limit 1 over Sort over Append of chunks, * that is just marginally costlier. * * We can't easily know the number of chunks in the query here, so add some * startup cost that is quadratic in the current chunk index, which * hopefully should be a good enough replacement. */ const int parent_relindex = bms_next_member(chunk_rel->top_parent_relids, -1); if (parent_relindex) { const int chunk_index = chunk_rel->relid - parent_relindex; sorted_path->startup_cost += cpu_operator_cost * chunk_index * chunk_index; sorted_path->total_cost += cpu_operator_cost * chunk_index * chunk_index; } return sorted_path; } static List *build_on_single_compressed_path(PlannerInfo *root, const Chunk *chunk, RelOptInfo *chunk_rel, Path *compressed_path, bool add_uncompressed_part, List *uncompressed_table_pathlist, const SortInfo *sort_info, const CompressionInfo *compression_info); void ts_columnar_scan_generate_paths(PlannerInfo *root, RelOptInfo *chunk_rel, const Hypertable *ht, const Chunk *chunk) { /* * For UPDATE/DELETE commands, the executor decompresses and brings the rows into * the uncompressed chunk. Therefore, it's necessary to add the scan on the * uncompressed portion. */ bool add_uncompressed_part = ts_chunk_is_partial(chunk); if (ts_chunk_is_compressed(chunk) && ts_cm_functions->decompress_target_segments && !add_uncompressed_part) { for (PlannerInfo *proot = root->parent_root; proot != NULL && !add_uncompressed_part; proot = proot->parent_root) { /* * We could additionally check and compare that the relation involved in the subquery * and the DML target relation are one and the same. But these kinds of queries * should be rare. */ if (proot->parse->commandType == CMD_UPDATE || proot->parse->commandType == CMD_DELETE #if PG15_GE || proot->parse->commandType == CMD_MERGE #endif ) { add_uncompressed_part = true; } } } CompressionInfo *compression_info = build_compressioninfo(root, ht, chunk, chunk_rel); /* double check we don't end up here on single chunk queries with ONLY */ Assert(compression_info->chunk_rel->reloptkind == RELOPT_OTHER_MEMBER_REL || (compression_info->chunk_rel->reloptkind == RELOPT_BASEREL && ts_rte_is_marked_for_expansion(compression_info->chunk_rte))); SortInfo sort_info = build_sortinfo(root, chunk, chunk_rel, compression_info, root->query_pathkeys); Assert(chunk->fd.compressed_chunk_id > 0); List *uncompressed_table_pathlist = chunk_rel->pathlist; List *uncompressed_table_parallel_pathlist = chunk_rel->partial_pathlist; chunk_rel->pathlist = NIL; chunk_rel->partial_pathlist = NIL; /* add RangeTblEntry and RelOptInfo for compressed chunk */ columnar_scan_add_plannerinfo(root, compression_info, chunk, chunk_rel, sort_info.needs_sequence_num); if (sort_info.use_compressed_sort) { sort_info.required_compressed_pathkeys = build_compressed_scan_pathkeys(&sort_info, root, root->query_pathkeys, compression_info); } RelOptInfo *compressed_rel = compression_info->compressed_rel; compressed_rel->consider_parallel = chunk_rel->consider_parallel; /* translate chunk_rel->baserestrictinfo */ if (ts_guc_enable_columnar_scan_filter_pushdown) { columnar_scan_filter_pushdown(root, compression_info->settings, chunk_rel, compressed_rel, add_uncompressed_part); } /* * Estimate the size of the compressed chunk table. */ set_compressed_baserel_size_estimates(root, compressed_rel, compression_info); /* * Estimate the size of the compressed batch from Postgres * statistics. */ compression_info->compressed_batch_size = ts_columnar_estimate_compressed_batch_size(compression_info->compressed_rte->relid); /* * Estimate the size of decompressed chunk based on the compressed chunk. * * The tuple estimates derived from pg_class will be empty, so we have to * compute that based on the compressed relation as well. Wrong estimates * there lead to wrong join order choice and wrong low cost for Sort over * Append, and also different MergeAppend costs on Postgres before 17 due to * a bug there. */ const double new_row_estimate = compressed_rel->rows * compression_info->compressed_batch_size; const double new_tuples_estimate = compressed_rel->tuples * compression_info->compressed_batch_size; if (!compression_info->single_chunk) { /* * Adjust the hypertable estimate by the diff of new and old chunk * estimate. */ AppendRelInfo *chunk_info = ts_get_appendrelinfo(root, chunk_rel->relid, false); const Index ht_relid = chunk_info->parent_relid; RelOptInfo *hypertable_rel = root->simple_rel_array[ht_relid]; const double delta = new_row_estimate - chunk_rel->rows; hypertable_rel->rows += delta; /* * For appendrel, set tuples to the same value as rows, * like set_append_rel_size() does. */ hypertable_rel->tuples += delta; } chunk_rel->rows = new_row_estimate; chunk_rel->tuples = new_tuples_estimate; /* * Create the paths for the compressed chunk table. */ create_compressed_scan_paths(root, compressed_rel, compression_info, &sort_info); /* create non-parallel paths */ ListCell *compressed_cell; foreach (compressed_cell, compressed_rel->pathlist) { Path *compressed_path = lfirst(compressed_cell); List *decompressed_paths = build_on_single_compressed_path(root, chunk, chunk_rel, compressed_path, add_uncompressed_part, uncompressed_table_pathlist, &sort_info, compression_info); /* * We want to consider startup costs so that IndexScan is preferred to * sorted SeqScan when we may have a chance to use SkipScan. We consider * startup costs for LIMIT queries, and SkipScan is basically a * "LIMIT 1" query run "ndistinct" times. At this point we don't have * all information to check if SkipScan can be used, but we can narrow * it down. */ if (!chunk_rel->consider_startup && IsA(compressed_path, IndexPath)) { /* Candidate for SELECT DISTINCT SkipScan */ if (list_length(root->distinct_pathkeys) == 1 /* Candidate for DISTINCT aggregate SkipScan */ || (root->numOrderedAggs >= 1 && list_length(root->group_pathkeys) == 1)) { chunk_rel->consider_startup = true; } } /* * Add the paths to the chunk relation. */ ListCell *decompressed_cell; foreach (decompressed_cell, decompressed_paths) { Path *path = lfirst(decompressed_cell); add_path(chunk_rel, path); } } /* create parallel paths */ List *uncompressed_paths_with_parallel = list_concat(uncompressed_table_parallel_pathlist, uncompressed_table_pathlist); foreach (compressed_cell, compressed_rel->partial_pathlist) { Path *compressed_path = lfirst(compressed_cell); /* Partial parameterized paths are not supported */ Assert(bms_is_empty(PATH_REQ_OUTER(compressed_path))); List *decompressed_paths = build_on_single_compressed_path(root, chunk, chunk_rel, compressed_path, add_uncompressed_part, uncompressed_paths_with_parallel, &sort_info, compression_info); /* * Add the paths to the chunk relation. */ ListCell *decompressed_cell; foreach (decompressed_cell, decompressed_paths) { Path *path = lfirst(decompressed_cell); add_partial_path(chunk_rel, path); } } /* the chunk_rel now owns the paths, remove them from the compressed_rel so they can't be freed * if it's planned */ compressed_rel->pathlist = NIL; compressed_rel->partial_pathlist = NIL; /* * Remove the compressed_rel from planner arrays to prevent it from being * referenced again. */ root->simple_rel_array[compressed_rel->relid] = NULL; root->append_rel_array[compressed_rel->relid] = NULL; /* We should never get in the situation with no viable paths. */ Ensure(chunk_rel->pathlist, "could not create decompression path"); } /* * Add various decompression paths that are possible based on the given * compressed path. */ static List * build_on_single_compressed_path(PlannerInfo *root, const Chunk *chunk, RelOptInfo *chunk_rel, Path *compressed_path, bool add_uncompressed_part, List *uncompressed_table_pathlist, const SortInfo *sort_info, const CompressionInfo *compression_info) { /* * We skip any BitmapScan parameterized paths here as supporting * those would require fixing up the internal scan. Since we * currently do not do this BitmapScans would be generated * when we have a parameterized path on a compressed column * that would have invalid references due to our * EquivalenceClasses. */ if (IsA(compressed_path, BitmapHeapPath) && compressed_path->param_info) return NIL; /* * Filter out all paths that try to JOIN the compressed chunk on the * hypertable or the uncompressed chunk * Ideally, we wouldn't create these paths in the first place. * However, create_join_clause code is called by PG while generating paths for the * compressed_rel via generate_implied_equalities_for_column. * create_join_clause ends up creating rinfo's between compressed_rel and ht because * PG does not know that compressed_rel is related to ht in anyway. * The parent-child relationship between chunk_rel and ht is known * to PG and so it does not try to create meaningless rinfos for that case. */ if (compressed_path->param_info != NULL) { if (bms_is_member(chunk_rel->relid, compressed_path->param_info->ppi_req_outer)) return NIL; /* check if this is path made with references between * compressed_rel + hypertable or a nesting subquery. * The latter can happen in the case of UNION queries. see github 2917. This * happens since PG is not aware that the nesting * subquery that references the hypertable is a parent of compressed_rel as well. */ if (bms_overlap(compression_info->parent_relids, compressed_path->param_info->ppi_req_outer)) { return NIL; } } Path *chunk_path_no_sort = (Path *) columnar_scan_path_create(root, compression_info, compressed_path); List *decompressed_paths = list_make1(chunk_path_no_sort); /* * Create a path for the batch sorted merge optimization. This optimization * performs a sorted merge of the involved batches by using a binary heap * and preserving the compression order. This optimization is only * considered if we can't push down the sort to the compressed chunk. If we * can push down the sort, the batches can be directly consumed in this * order and we don't need to use this optimization. */ if (sort_info->use_batch_sorted_merge && ts_guc_enable_decompression_sorted_merge) { Assert(!sort_info->use_compressed_sort); ColumnarScanPath *path_copy = copy_columnar_scan_path((ColumnarScanPath *) chunk_path_no_sort); path_copy->reverse = sort_info->reverse; path_copy->batch_sorted_merge = true; /* * The segment by optimization is only enabled if it can deliver the tuples in the * same order as the query requested it. So, we can just copy the pathkeys of the * query here. */ path_copy->custom_path.path.pathkeys = sort_info->decompressed_sort_pathkeys; cost_batch_sorted_merge(root, compression_info, path_copy, compressed_path); if (ts_guc_debug_require_batch_sorted_merge == DRO_Force) { path_copy->custom_path.path.startup_cost = cpu_tuple_cost; path_copy->custom_path.path.total_cost = 2 * cpu_tuple_cost; } decompressed_paths = lappend(decompressed_paths, path_copy); } else if (ts_guc_debug_require_batch_sorted_merge == DRO_Require || ts_guc_debug_require_batch_sorted_merge == DRO_Force) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("debug: batch sorted merge is required but not possible at planning " "time"))); } /* * If we can push down the sort below the ColumnarScan node, we set the * pathkeys of the decompress node to the decompressed_sort_pathkeys. We * will determine whether to put an actual sort between the decompression * node and the scan during plan creation. */ if (sort_info->use_compressed_sort) { if (pathkeys_contained_in(sort_info->required_compressed_pathkeys, compressed_path->pathkeys)) { /* * The compressed path already has the required ordering. Modify * in place the no-sorting path we just created above. */ ColumnarScanPath *path = (ColumnarScanPath *) chunk_path_no_sort; path->reverse = sort_info->reverse; path->needs_sequence_num = sort_info->needs_sequence_num; path->required_compressed_pathkeys = sort_info->required_compressed_pathkeys; path->custom_path.path.pathkeys = sort_info->decompressed_sort_pathkeys; } else { /* * We must sort the underlying compressed path to get the * required ordering. Make a copy of no-sorting path and modify * it accordingly */ ColumnarScanPath *path_copy = copy_columnar_scan_path((ColumnarScanPath *) chunk_path_no_sort); path_copy->reverse = sort_info->reverse; path_copy->needs_sequence_num = sort_info->needs_sequence_num; path_copy->required_compressed_pathkeys = sort_info->required_compressed_pathkeys; path_copy->custom_path.path.pathkeys = sort_info->decompressed_sort_pathkeys; /* * Add costing for a sort. The standard Postgres pattern is to add the cost during * path creation, but not add the sort path itself, that's done during plan * creation. Examples of this in: create_merge_append_path & * create_merge_append_plan */ Path sort_path; /* dummy for result of cost_sort */ cost_sort(&sort_path, root, sort_info->required_compressed_pathkeys, #if PG18_GE compressed_path->disabled_nodes, #endif compressed_path->total_cost, compressed_path->rows, compressed_path->pathtarget->width, 0.0, work_mem, -1); cost_columnar_scan(root, compression_info, &path_copy->custom_path.path, &sort_path); decompressed_paths = lappend(decompressed_paths, path_copy); } } /* * Also try explicit sort after decompression, if we couldn't push down the * sort. Don't do this for parallel plans, because in this case it is * typically done with Sort under Gather node. This splits the Sort in * per-worker buckets, so splitting the buckets further per-chunk is less * important. */ if (!sort_info->use_compressed_sort && chunk_path_no_sort->parallel_workers == 0) { Path *sort_above_chunk = make_chunk_sorted_path(root, chunk_rel, chunk_path_no_sort, compressed_path, sort_info); if (sort_above_chunk != NULL) { decompressed_paths = lappend(decompressed_paths, sort_above_chunk); } } if (!add_uncompressed_part) { /* * If the chunk has only the compressed part, we're done. */ return decompressed_paths; } /* * This is a partially compressed chunk, we have to combine data from * compressed and uncompressed chunk. */ List *combined_paths = NIL; /* * All decompressed paths we've built have the same parameterization since * we're building on a single compressed path. We only inherit the * parameterization from it and don't add our own. */ Bitmapset *req_outer = PATH_REQ_OUTER(chunk_path_no_sort); /* * Look up the uncompressed chunk paths. We might need an unordered path * (SeqScan) and an ordered path (e.g. IndexScan). */ Path *unordered_uncompressed_path = get_cheapest_path_for_pathkeys(uncompressed_table_pathlist, NIL, req_outer, TOTAL_COST, false); Ensure(unordered_uncompressed_path != NULL, "couldn't find a scan path for uncompressed chunk table"); Path *ordered_uncompressed_path = NULL; if (sort_info->decompressed_sort_pathkeys != NIL) { ordered_uncompressed_path = get_cheapest_path_for_pathkeys(uncompressed_table_pathlist, sort_info->decompressed_sort_pathkeys, req_outer, TOTAL_COST, false); } /* * All children of an append path are required to have the same parameterization * so we reparameterize here when we couldn't get a path with the parameterization * we need. Reparameterization should always succeed here since uncompressed_path * should always be a scan. */ if (!bms_equal(req_outer, PATH_REQ_OUTER(unordered_uncompressed_path))) { unordered_uncompressed_path = reparameterize_path(root, unordered_uncompressed_path, req_outer, 1.0); Ensure(unordered_uncompressed_path != NULL, "couldn't reparameterize a scan path for uncompressed chunk table"); } if (ordered_uncompressed_path != NULL && !bms_equal(req_outer, PATH_REQ_OUTER(ordered_uncompressed_path))) { ordered_uncompressed_path = reparameterize_path(root, ordered_uncompressed_path, req_outer, 1.0); } /* * Create plain Append, potentially parallel. It only makes sense for the * unsorted input paths. */ { const int workers = Max(chunk_path_no_sort->parallel_workers, unordered_uncompressed_path->parallel_workers); List *parallel_paths = NIL; List *sequential_paths = NIL; if (chunk_path_no_sort->parallel_workers > 0) { parallel_paths = lappend(parallel_paths, chunk_path_no_sort); } else { sequential_paths = lappend(sequential_paths, chunk_path_no_sort); } if (unordered_uncompressed_path->parallel_workers > 0) { parallel_paths = lappend(parallel_paths, unordered_uncompressed_path); } else { sequential_paths = lappend(sequential_paths, unordered_uncompressed_path); } Path *plain_append = (Path *) create_append_path(root, chunk_rel, sequential_paths, parallel_paths, /* pathkeys = */ NIL, req_outer, workers, workers > 0, chunk_path_no_sort->rows + unordered_uncompressed_path->rows); combined_paths = lappend(combined_paths, plain_append); } if (sort_info->decompressed_sort_pathkeys == NIL) { /* * No sorting requested, so we're done after creating the plain Append * above. */ return combined_paths; } /* * We require sorting, try MergeAppend. */ Path *uncompressed_path_for_merge = ordered_uncompressed_path; if (uncompressed_path_for_merge == NULL || IsA(uncompressed_path_for_merge, SortPath)) { /* * Don't use explicit Sort as MergeAppend child, because the * MergeAppend adds the required sorting anyway. With the explicit * Sort it still works but performs the pathkey lookups twice, which * leads to planning performance regression. */ uncompressed_path_for_merge = unordered_uncompressed_path; } if (uncompressed_path_for_merge->parallel_workers > 0) { /* * MergeAppend can't be parallel. */ return combined_paths; } /* * For Merge Append, we consider: * 1) explicit sorting over decompressed path, * 2) compressed sort pushdown path, * 3) batch sorted merge path. * We have to make a cost-based decision between them (i.e. batch sorted * merge might be more expensive due to memory requirements). */ ListCell *lc; foreach (lc, decompressed_paths) { Path *decompression_path = lfirst(lc); if (decompression_path->parallel_workers > 0) { /* * MergeAppend can't be parallel. */ continue; } if (decompression_path == chunk_path_no_sort) { /* * We can't use the unsorted decompression path directly because it * doesn't have the sort projection cost workaround. */ continue; } if (!bms_is_empty(chunk_rel->lateral_relids) || !bms_is_empty(req_outer)) { /* * Parametrized MergeAppend paths are not supported. */ continue; } if (IsA(decompression_path, SortPath)) { /* * We have to remove the explicit Sort, otherwise it will lead to * planning time regression because of double call of * prepare_sort_from_pathkeys() in MergeAppend plan creation. Still, * we have to use the copy of ColumnarScan path that we created * for explicit sorting, because it has the sort projection cost * workaround. */ decompression_path = castNode(SortPath, decompression_path)->subpath; } Path *merge_append = (Path *) create_merge_append_path(root, chunk_rel, list_make2(decompression_path, uncompressed_path_for_merge), sort_info->decompressed_sort_pathkeys, req_outer); combined_paths = lappend(combined_paths, merge_append); } return combined_paths; } /* * Add a var for a particular column to the reltarget. attrs_used is a bitmap * of which columns we already have in reltarget. We do not add the columns that * are already there, and update it after adding something. */ static void compressed_reltarget_add_var_for_column(RelOptInfo *compressed_rel, Oid compressed_relid, const char *column_name, Bitmapset **attrs_used) { AttrNumber attnum = get_attnum(compressed_relid, column_name); Assert(attnum > 0); if (bms_is_member(attnum, *attrs_used)) { /* This column is already in reltarget, we don't need duplicates. */ return; } *attrs_used = bms_add_member(*attrs_used, attnum); Oid typid, collid; int32 typmod; get_atttypetypmodcoll(compressed_relid, attnum, &typid, &typmod, &collid); compressed_rel->reltarget->exprs = lappend(compressed_rel->reltarget->exprs, makeVar(compressed_rel->relid, attnum, typid, typmod, collid, 0)); } /* copy over the vars from the chunk_rel->reltarget to the compressed_rel->reltarget * altering the fields that need it */ static void compressed_rel_setup_reltarget(RelOptInfo *compressed_rel, CompressionInfo *info, bool needs_sequence_num) { bool have_whole_row_var = false; Bitmapset *attrs_used = NULL; Oid compressed_relid = info->compressed_rte->relid; /* * We have to decompress three kinds of columns: * 1) output targetlist of the relation, * 2) columns required for the quals (WHERE), * 3) columns required for joins. */ List *exprs = list_copy(info->chunk_rel->reltarget->exprs); ListCell *lc; foreach (lc, info->chunk_rel->baserestrictinfo) { exprs = lappend(exprs, ((RestrictInfo *) lfirst(lc))->clause); } foreach (lc, info->chunk_rel->joininfo) { exprs = lappend(exprs, ((RestrictInfo *) lfirst(lc))->clause); } /* * Now go over the required expressions we prepared above, and add the * required columns to the compressed reltarget. */ info->compressed_rel->reltarget->exprs = NIL; foreach (lc, exprs) { ListCell *lc2; List *chunk_vars = pull_var_clause(lfirst(lc), PVC_RECURSE_PLACEHOLDERS); foreach (lc2, chunk_vars) { char *column_name; Var *chunk_var = castNode(Var, lfirst(lc2)); /* skip vars that aren't from the uncompressed chunk */ if ((Index) chunk_var->varno != info->chunk_rel->relid) { continue; } /* * If there's a system column or whole-row reference, add a whole- * row reference, and we're done. */ if (chunk_var->varattno <= 0) { have_whole_row_var = true; continue; } column_name = get_attname(info->chunk_rte->relid, chunk_var->varattno, false); compressed_reltarget_add_var_for_column(compressed_rel, compressed_relid, column_name, &attrs_used); /* if the column is an orderby, add it's metadata columns too */ int16 index = ts_array_position(info->settings->fd.orderby, column_name); if (index != 0) { compressed_reltarget_add_var_for_column(compressed_rel, compressed_relid, column_segment_min_name(index), &attrs_used); compressed_reltarget_add_var_for_column(compressed_rel, compressed_relid, column_segment_max_name(index), &attrs_used); } } } /* always add the count column */ compressed_reltarget_add_var_for_column(compressed_rel, compressed_relid, COMPRESSION_COLUMN_METADATA_COUNT_NAME, &attrs_used); /* add the sequence number or orderby metadata columns if we try to order by them*/ if (needs_sequence_num) { if (info->has_seq_num) { compressed_reltarget_add_var_for_column(compressed_rel, compressed_relid, COMPRESSION_COLUMN_METADATA_SEQUENCE_NUM_NAME, &attrs_used); } else { for (int i = 1; i <= ts_array_length(info->settings->fd.orderby); i++) { compressed_reltarget_add_var_for_column(compressed_rel, compressed_relid, column_segment_min_name(i), &attrs_used); compressed_reltarget_add_var_for_column(compressed_rel, compressed_relid, column_segment_max_name(i), &attrs_used); } } } /* * It doesn't make sense to request a whole-row var from the compressed * chunk scan. If it is requested, just fetch the rest of columns. The * whole-row var will be created by the projection of ColumnarScan node. */ if (have_whole_row_var) { for (int i = 1; i <= info->chunk_rel->max_attr; i++) { char *column_name = get_attname(info->chunk_rte->relid, i, /* missing_ok = */ false); AttrNumber chunk_attno = get_attnum(info->chunk_rte->relid, column_name); if (chunk_attno == InvalidAttrNumber) { /* Skip the dropped column. */ continue; } AttrNumber compressed_attno = get_attnum(info->compressed_rte->relid, column_name); if (compressed_attno == InvalidAttrNumber) { elog(ERROR, "column '%s' not found in the compressed chunk '%s'", column_name, get_rel_name(info->compressed_rte->relid)); } if (bms_is_member(compressed_attno, attrs_used)) { continue; } compressed_reltarget_add_var_for_column(compressed_rel, compressed_relid, column_name, &attrs_used); } } } static Bitmapset * columnar_scan_adjust_child_relids(Bitmapset *src, int chunk_relid, int compressed_chunk_relid) { Bitmapset *result = NULL; if (src != NULL) { result = bms_copy(src); result = bms_del_member(result, chunk_relid); result = bms_add_member(result, compressed_chunk_relid); } return result; } /* based on adjust_appendrel_attrs_mutator handling of RestrictInfo */ static Node * chunk_joininfo_mutator(Node *node, CompressionInfo *context) { if (node == NULL) return NULL; if (IsA(node, Var)) { Var *var = castNode(Var, node); Var *compress_var = copyObject(var); char *column_name; AttrNumber compressed_attno; if ((Index) var->varno != context->chunk_rel->relid) return (Node *) var; column_name = get_attname(context->chunk_rte->relid, var->varattno, false); compressed_attno = get_attnum(context->compressed_rte->relid, column_name); compress_var->varno = context->compressed_rel->relid; compress_var->varattno = compressed_attno; return (Node *) compress_var; } else if (IsA(node, RestrictInfo)) { RestrictInfo *oldinfo = (RestrictInfo *) node; RestrictInfo *newinfo = makeNode(RestrictInfo); /* Copy all flat-copiable fields */ memcpy(newinfo, oldinfo, sizeof(RestrictInfo)); /* Recursively fix the clause itself */ newinfo->clause = (Expr *) chunk_joininfo_mutator((Node *) oldinfo->clause, context); /* and the modified version, if an OR clause */ newinfo->orclause = (Expr *) chunk_joininfo_mutator((Node *) oldinfo->orclause, context); /* adjust relid sets too */ newinfo->clause_relids = columnar_scan_adjust_child_relids(oldinfo->clause_relids, context->chunk_rel->relid, context->compressed_rel->relid); newinfo->required_relids = columnar_scan_adjust_child_relids(oldinfo->required_relids, context->chunk_rel->relid, context->compressed_rel->relid); newinfo->outer_relids = columnar_scan_adjust_child_relids(oldinfo->outer_relids, context->chunk_rel->relid, context->compressed_rel->relid); #if PG16_LT newinfo->nullable_relids = columnar_scan_adjust_child_relids(oldinfo->nullable_relids, context->chunk_rel->relid, context->compressed_rel->relid); #endif newinfo->left_relids = columnar_scan_adjust_child_relids(oldinfo->left_relids, context->chunk_rel->relid, context->compressed_rel->relid); newinfo->right_relids = columnar_scan_adjust_child_relids(oldinfo->right_relids, context->chunk_rel->relid, context->compressed_rel->relid); newinfo->eval_cost.startup = -1; newinfo->norm_selec = -1; newinfo->outer_selec = -1; newinfo->left_em = NULL; newinfo->right_em = NULL; newinfo->scansel_cache = NIL; newinfo->left_bucketsize = -1; newinfo->right_bucketsize = -1; newinfo->left_mcvfreq = -1; newinfo->right_mcvfreq = -1; return (Node *) newinfo; } return expression_tree_mutator(node, chunk_joininfo_mutator, context); } /* Check if the expression references a compressed column in compressed chunk. */ static bool has_compressed_vars_walker(Node *node, CompressionInfo *info) { if (node == NULL) { return false; } if (IsA(node, Var)) { Var *var = castNode(Var, node); if ((Index) var->varno != info->compressed_rel->relid) { return false; } if (var->varattno <= 0) { /* * Shouldn't see a system var here, might be a whole row var? * In any case, we can't push it down to the compressed scan level. */ return true; } if (bms_is_member(var->varattno, info->compressed_attnos_in_compressed_chunk)) { return true; } return false; } return expression_tree_walker(node, has_compressed_vars_walker, info); } static bool has_compressed_vars(RestrictInfo *ri, CompressionInfo *info) { return expression_tree_walker((Node *) ri->clause, has_compressed_vars_walker, info); } /* translate chunk_rel->joininfo for compressed_rel * this is necessary for create_index_path which gets join clauses from * rel->joininfo and sets up parameterized paths (in rel->ppilist). * ppi_clauses is finally used to add any additional filters on the * indexpath when creating a plan in create_indexscan_plan. * Otherwise we miss additional filters that need to be applied after * the index plan is executed (github issue 1558) */ static void compressed_rel_setup_joininfo(RelOptInfo *compressed_rel, CompressionInfo *info) { RelOptInfo *chunk_rel = info->chunk_rel; ListCell *lc; List *compress_joininfo = NIL; foreach (lc, chunk_rel->joininfo) { RestrictInfo *ri = (RestrictInfo *) lfirst(lc); RestrictInfo *adjusted = (RestrictInfo *) chunk_joininfo_mutator((Node *) ri, info); Assert(IsA(adjusted, RestrictInfo)); if (has_compressed_vars(adjusted, info)) { /* * We can't check clauses that refer to compressed columns during * the compressed scan. */ continue; } compress_joininfo = lappend(compress_joininfo, adjusted); } compressed_rel->joininfo = compress_joininfo; } typedef struct EMCreationContext { Oid uncompressed_relid; Oid compressed_relid; Index uncompressed_relid_idx; Index compressed_relid_idx; CompressionSettings *settings; } EMCreationContext; static Node * create_var_for_compressed_equivalence_member(Var *var, const EMCreationContext *context, const char *attname) { /* based on adjust_appendrel_attrs_mutator */ Assert((Index) var->varno == context->uncompressed_relid_idx); Assert(var->varattno > 0); var = (Var *) copyObject(var); if (var->varlevelsup == 0) { var->varno = context->compressed_relid_idx; var->varattno = get_attnum(context->compressed_relid, attname); var->varnosyn = var->varno; var->varattnosyn = var->varattno; return (Node *) var; } return NULL; } /* This function is inspired by the Postgres add_child_rel_equivalences. */ static bool add_segmentby_to_equivalence_class(PlannerInfo *root, EquivalenceClass *cur_ec, CompressionInfo *info, EMCreationContext *context) { TimescaleDBPrivate *compressed_fdw_private = (TimescaleDBPrivate *) info->compressed_rel->fdw_private; Assert(compressed_fdw_private != NULL); EquivalenceMember *cur_em; #if PG18_GE /* Use specialized iterator to include child ems. * * https://github.com/postgres/postgres/commit/d69d45a5 */ EquivalenceMemberIterator it; setup_eclass_member_iterator(&it, cur_ec, bms_make_singleton(info->chunk_rel->relid)); while ((cur_em = eclass_member_iterator_next(&it)) != NULL) { #else ListCell *lc; foreach (lc, cur_ec->ec_members) { cur_em = (EquivalenceMember *) lfirst(lc); #endif Node *node; Expr *child_expr; Relids new_relids; Var *var; Assert(!bms_overlap(cur_em->em_relids, info->compressed_rel->relids)); /* only consider EquivalenceMembers that are Vars, possibly with RelabelType, of the * uncompressed chunk */ node = strip_implicit_coercions((Node *) cur_em->em_expr); if (!(node && IsA(node, Var))) continue; var = castNode(Var, node); /* * We want to base our equivalence member on the hypertable equivalence * member, not on the uncompressed chunk one. We can't just check for * em_is_child though because the hypertable might be a child itself and not * a top-level EquivalenceMember. This is mostly relevant for PG16+ where * we have to specify a parent for the newly created equivalence member. */ if ((Index) var->varno != info->ht_rel->relid) continue; if (var->varattno <= 0) { /* * We can have equivalence members that refer to special variables, * but these variables can't be segmentby, so we're not interested * in them here. */ continue; } /* given that the em is a var of the uncompressed chunk, the relid of the chunk should * be set on the em */ Assert(bms_is_member(info->ht_rel->relid, cur_em->em_relids)); Assert(OidIsValid(info->ht_rte->relid)); const char *attname = get_attname(info->ht_rte->relid, var->varattno, false); if (!ts_array_is_member(context->settings->fd.segmentby, attname)) continue; child_expr = (Expr *) create_var_for_compressed_equivalence_member(var, context, attname); if (child_expr == NULL) continue; /* #8681: coerce compressed var to current equivalence member type/collation, * in case we dug the "cur_em->em_expr" var from under RelabelTypes */ child_expr = canonicalize_ec_expression(child_expr, cur_em->em_datatype, cur_ec->ec_collation); /* * Transform em_relids to match. Note we do *not* do * pull_varnos(child_expr) here, as for example the * transformation might have substituted a constant, but we * don't want the child member to be marked as constant. */ new_relids = bms_copy(cur_em->em_relids); new_relids = bms_del_member(new_relids, info->ht_rel->relid); new_relids = bms_add_members(new_relids, info->compressed_rel->relids); /* copied from add_eq_member */ { EquivalenceMember *em = makeNode(EquivalenceMember); em->em_expr = child_expr; em->em_relids = new_relids; em->em_is_const = false; em->em_is_child = true; em->em_datatype = cur_em->em_datatype; #if PG16_GE em->em_jdomain = cur_em->em_jdomain; em->em_parent = cur_em; #endif #if PG16_LT /* * For versions less than PG16, transform and set em_nullable_relids similar to * em_relids. Note that this code assumes parent and child relids are singletons. */ Relids new_nullable_relids = cur_em->em_nullable_relids; if (bms_is_member(info->ht_rel->relid, new_nullable_relids)) { new_nullable_relids = bms_copy(new_nullable_relids); new_nullable_relids = bms_del_member(new_nullable_relids, info->ht_rel->relid); new_nullable_relids = bms_add_members(new_nullable_relids, info->compressed_rel->relids); } em->em_nullable_relids = new_nullable_relids; #endif /* * In some cases the new EC member is likely to be accessed soon, so * it would make sense to add it to the front, but we cannot do that * here. If we do that, the compressed chunk EM might get picked as * SortGroupExpr by cost_incremental_sort, and estimate_num_groups * will assert that the rel is simple rel, but it will fail because * the compressed chunk rel is a deadrel. Anyway, it wouldn't make * sense to estimate the group numbers by one append member, * probably Postgres expects to see the parent relation first in the * EMs. */ #if PG18_LT cur_ec->ec_members = lappend(cur_ec->ec_members, em); cur_ec->ec_relids = bms_add_members(cur_ec->ec_relids, info->compressed_rel->relids); #else ts_add_child_eq_member(root, cur_ec, em, info->compressed_rel->relid); #endif /* * Cache the matching EquivalenceClass and EquivalenceMember for * segmentby column for future use, if we want to build a path that * sorts on it. Sorting is defined by PathKeys, which refer to * EquivalenceClasses, so it's a convenient form. */ compressed_fdw_private->compressed_ec_em_pairs = lappend(compressed_fdw_private->compressed_ec_em_pairs, list_make2(cur_ec, em)); return true; } } return false; } static void compressed_rel_setup_equivalence_classes(PlannerInfo *root, CompressionInfo *info) { EMCreationContext context = { .uncompressed_relid = info->ht_rte->relid, .compressed_relid = info->compressed_rte->relid, .uncompressed_relid_idx = info->ht_rel->relid, .compressed_relid_idx = info->compressed_rel->relid, .settings = info->settings, }; Assert(info->chunk_rte->relid != info->compressed_rel->relid); Assert(info->chunk_rel->relid != info->compressed_rel->relid); /* based on add_child_rel_equivalences */ int i = -1; Assert(root->ec_merging_done); /* use chunk rel's eclass_indexes to avoid traversing all * the root's eq_classes */ while ((i = bms_next_member(info->chunk_rel->eclass_indexes, i)) >= 0) { EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); /* * If this EC contains a volatile expression, then generating child * EMs would be downright dangerous, so skip it. We rely on a * volatile EC having only one EM. */ if (cur_ec->ec_has_volatile) continue; /* if the compressed rel is already part of this EC, * we don't need to re-add it */ if (bms_overlap(cur_ec->ec_relids, info->compressed_rel->relids)) continue; bool em_added = add_segmentby_to_equivalence_class(root, cur_ec, info, &context); /* Record this EC index for the compressed rel */ if (em_added) info->compressed_rel->eclass_indexes = bms_add_member(info->compressed_rel->eclass_indexes, i); } info->compressed_rel->has_eclass_joins = info->chunk_rel->has_eclass_joins; } /* * create RangeTblEntry and RelOptInfo for the compressed chunk * and add it to PlannerInfo */ static void columnar_scan_add_plannerinfo(PlannerInfo *root, CompressionInfo *info, const Chunk *chunk, RelOptInfo *chunk_rel, bool needs_sequence_num) { Index compressed_index = root->simple_rel_array_size; /* * Add the compressed chunk to the baserel cache. Note that it belongs to * a different hypertable, the internal compression table. * * Ensure we do not grab a slice lock because that will assign a transaction ID that could * unnecessarily block other operations. */ const Chunk *compressed_chunk = ts_chunk_get_by_relid_locked(info->settings->fd.compress_relid, AccessShareLock, NULL, true); ts_add_baserel_cache_entry_for_chunk(info->settings->fd.compress_relid, ts_planner_get_hypertable(compressed_chunk ->hypertable_relid, CACHE_FLAG_NONE)); expand_planner_arrays(root, 1); info->compressed_rte = columnar_scan_make_rte(info->settings->fd.compress_relid, info->chunk_rte->rellockmode, root->parse); root->simple_rte_array[compressed_index] = info->compressed_rte; root->parse->rtable = lappend(root->parse->rtable, info->compressed_rte); root->simple_rel_array[compressed_index] = NULL; RelOptInfo *compressed_rel = build_simple_rel(root, compressed_index, NULL); #if PG16_GE /* * When initially creating the RTE we add a RTEPerminfo entry for the * RTE but that is only to make build_simple_rel happy. * Asserts in the permission check code will fail with an RTEPerminfo * with no permissions to check so we remove it again here as we don't * want permission checks on the compressed chunks when querying * hypertables with compressed data. */ root->parse->rteperminfos = list_delete_last(root->parse->rteperminfos); info->compressed_rte->perminfoindex = 0; #endif /* github issue :1558 * set up top_parent_relids for this rel as the same as the * original hypertable, otherwise eq classes are not computed correctly * in generate_join_implied_equalities (called by * get_baserel_parampathinfo <- create_index_paths) */ Assert(info->single_chunk || chunk_rel->top_parent_relids != NULL); compressed_rel->top_parent_relids = bms_copy(chunk_rel->top_parent_relids); compressed_rel->lateral_relids = bms_copy(chunk_rel->lateral_relids); root->simple_rel_array[compressed_index] = compressed_rel; info->compressed_rel = compressed_rel; Relation r = table_open(info->compressed_rte->relid, AccessShareLock); for (int i = 0; i < r->rd_att->natts; i++) { Form_pg_attribute attr = TupleDescAttr(r->rd_att, i); if (attr->attisdropped || attr->atttypid != info->compresseddata_oid) continue; info->compressed_attnos_in_compressed_chunk = bms_add_member(info->compressed_attnos_in_compressed_chunk, attr->attnum); } table_close(r, NoLock); compressed_rel_setup_reltarget(compressed_rel, info, needs_sequence_num); compressed_rel_setup_equivalence_classes(root, info); /* translate chunk_rel->joininfo for compressed_rel */ compressed_rel_setup_joininfo(compressed_rel, info); /* * Force parallel plan creation, see compute_parallel_worker(). * This is not compatible with ts_classify_relation(), but on the other hand * the compressed chunk rel shouldn't exist anywhere outside of the * decompression planning, it is removed at the end. * * This is not needed for direct select from a single chunk, in which case * the chunk reloptkind will be RELOPT_BASEREL */ if (chunk_rel->reloptkind == RELOPT_OTHER_MEMBER_REL) { compressed_rel->reloptkind = RELOPT_OTHER_MEMBER_REL; /* * We have to minimally initialize the append relation info for the * compressed chunks, so that the generate_implied_equalities() works. * Only the parent hypertable relindex is needed. */ root->append_rel_array[compressed_rel->relid] = makeNode(AppendRelInfo); root->append_rel_array[compressed_rel->relid]->parent_relid = info->ht_rel->relid; } } static ColumnarScanPath * columnar_scan_path_create(PlannerInfo *root, const CompressionInfo *compression_info, Path *compressed_path) { ColumnarScanPath *path; path = (ColumnarScanPath *) newNode(sizeof(ColumnarScanPath), T_CustomPath); path->info = compression_info; path->custom_path.path.pathtype = T_CustomScan; path->custom_path.path.parent = compression_info->chunk_rel; path->custom_path.path.pathtarget = compression_info->chunk_rel->reltarget; if (compressed_path->param_info != NULL) { /* * Note that we have to separately generate the parameterized path info * for decompressed chunk path. The compressed parameterized path only * checks the clauses on segmentby columns, not on the compressed * columns. */ path->custom_path.path.param_info = get_baserel_parampathinfo(root, compression_info->chunk_rel, compressed_path->param_info->ppi_req_outer); Assert(path->custom_path.path.param_info != NULL); } else { path->custom_path.path.param_info = NULL; } /* * Setting this flags means that Postgres can change the result targetlist * after the plan creation. This node can cope with this because it performs * the usual Postgres projection to produce the result tuple from the scan * tuple. The decompression-specific code works before that, and produces * the scan tuple based on the compressed tuple. The scan tuple descriptor * is based either on custom_scan_tlist or scanrelid, and the decompression * map is based on the compressed tuple, so they are not dependent on the * result targetlist, and we can allow it to be changed later. This allows * us to avoid a separate Result node, for a small performance saving. */ path->custom_path.flags = CUSTOMPATH_SUPPORT_PROJECTION; path->custom_path.methods = &columnar_scan_path_methods; path->batch_sorted_merge = false; /* * ColumnarScan doesn't manage any parallelism itself. */ path->custom_path.path.parallel_aware = false; /* * It can be applied per parallel worker, if its underlying scan is parallel. */ path->custom_path.path.parallel_safe = compressed_path->parallel_safe; path->custom_path.path.parallel_workers = compressed_path->parallel_workers; path->custom_path.custom_paths = list_make1(compressed_path); path->reverse = false; path->chunk_status = compression_info->chunk_status; path->required_compressed_pathkeys = NIL; cost_columnar_scan(root, compression_info, &path->custom_path.path, compressed_path); return path; } /* NOTE: this needs to be called strictly after all restrictinfos have been added * to the compressed rel */ static void create_compressed_scan_paths(PlannerInfo *root, RelOptInfo *compressed_rel, const CompressionInfo *compression_info, const SortInfo *sort_info) { Path *compressed_path; Relids required_outer = compressed_rel->lateral_relids; /* Must have same lateral relids as the chunk hypertable */ Assert(bms_equal(required_outer, compression_info->chunk_rel->lateral_relids)); /* clamp total_table_pages to 10 pages since this is the * minimum estimate for number of pages. * Add the value to any existing estimates */ root->total_table_pages += Max(compressed_rel->pages, 10); /* create non parallel scan path */ compressed_path = create_seqscan_path(root, compressed_rel, required_outer, 0); add_path(compressed_rel, compressed_path); /* * Create parallel seq scan path. * We marked the compressed rel as RELOPT_OTHER_MEMBER_REL when creating it, * so we should get a nonzero number of parallel workers even for small * tables, so that they don't prevent parallelism in the entire append plan. * See compute_parallel_workers(). This also applies to the creation of * index paths below. * * Parameterized rels that depend on an outer rel are not allowed to form partial * sequential scan paths */ if (compressed_rel->consider_parallel && required_outer == NULL) { int parallel_workers = compute_parallel_worker(compressed_rel, compressed_rel->pages, -1, max_parallel_workers_per_gather); if (parallel_workers > 0) { add_partial_path(compressed_rel, create_seqscan_path(root, compressed_rel, NULL, parallel_workers)); } } if (sort_info->use_compressed_sort) { /* * If we can push down sort below decompression we temporarily switch * out root->query_pathkeys to allow matching to pathkeys produces by * decompression */ List *orig_pathkeys = root->query_pathkeys; List *orig_eq_classes = root->eq_classes; Bitmapset *orig_eclass_indexes = compression_info->compressed_rel->eclass_indexes; root->query_pathkeys = sort_info->required_compressed_pathkeys; /* We can optimize iterating over EquivalenceClasses by reducing them to * the subset which are from the compressed chunk. This only works if we don't * have joins based on equivalence classes involved since those * use eclass_indexes which is not valid with this optimization. * * Clauseless joins work fine since they don't rely on eclass_indexes. */ if (!compression_info->chunk_rel->has_eclass_joins) { int i = -1; List *required_eq_classes = NIL; while ((i = bms_next_member(compression_info->compressed_rel->eclass_indexes, i)) >= 0) { EquivalenceClass *cur_ec = (EquivalenceClass *) list_nth(root->eq_classes, i); required_eq_classes = lappend(required_eq_classes, cur_ec); } root->eq_classes = required_eq_classes; compression_info->compressed_rel->eclass_indexes = NULL; } check_index_predicates(root, compressed_rel); create_index_paths(root, compressed_rel); root->query_pathkeys = orig_pathkeys; root->eq_classes = orig_eq_classes; compression_info->compressed_rel->eclass_indexes = orig_eclass_indexes; } else { check_index_predicates(root, compressed_rel); create_index_paths(root, compressed_rel); } } /* * create RangeTblEntry for compressed chunk */ static RangeTblEntry * columnar_scan_make_rte(Oid compressed_relid, LOCKMODE lockmode, Query *parse) { RangeTblEntry *rte = makeNode(RangeTblEntry); Relation r = table_open(compressed_relid, lockmode); int varattno; rte->rtekind = RTE_RELATION; rte->relid = compressed_relid; rte->relkind = r->rd_rel->relkind; rte->rellockmode = lockmode; rte->eref = makeAlias(RelationGetRelationName(r), NULL); /* * inlined from buildRelationAliases() * alias handling has been stripped because we won't * need alias handling at this level */ for (varattno = 0; varattno < r->rd_att->natts; varattno++) { Form_pg_attribute attr = TupleDescAttr(r->rd_att, varattno); /* Always insert an empty string for a dropped column */ const char *attrname = attr->attisdropped ? "" : NameStr(attr->attname); rte->eref->colnames = lappend(rte->eref->colnames, makeString(pstrdup(attrname))); } /* * Drop the rel refcount, but keep the access lock till end of transaction * so that the table can't be deleted or have its schema modified * underneath us. */ table_close(r, NoLock); /* * Set flags and access permissions. * * The initial default on access checks is always check-for-READ-access, * which is the right thing for all except target tables. */ rte->lateral = false; rte->inh = false; rte->inFromCl = false; #if PG16_LT rte->requiredPerms = 0; rte->checkAsUser = InvalidOid; /* not set-uid by default, either */ rte->selectedCols = NULL; rte->insertedCols = NULL; rte->updatedCols = NULL; #else /* Add empty perminfo for the new RTE to make build_simple_rel happy. */ addRTEPermissionInfo(&parse->rteperminfos, rte); #endif return rte; } /* * Find segmentby columns that are equated to a constant by a toplevel * baserestrictinfo. * * This will detect Var = Const and Var = Param and set the corresponding bit * in CompressionInfo->chunk_const_segmentby. */ static Bitmapset * find_const_segmentby(RelOptInfo *chunk_rel, const CompressionInfo *info) { Bitmapset *segmentby_columns = NULL; if (chunk_rel->baserestrictinfo != NIL) { ListCell *lc_ri; foreach (lc_ri, chunk_rel->baserestrictinfo) { RestrictInfo *ri = lfirst(lc_ri); if (IsA(ri->clause, OpExpr) && list_length(castNode(OpExpr, ri->clause)->args) == 2) { OpExpr *op = castNode(OpExpr, ri->clause); Node *lnode, *rnode; Var *var; Expr *other; if (op->opretset) continue; lnode = strip_implicit_coercions(linitial(op->args)); rnode = strip_implicit_coercions(lsecond(op->args)); Assert(lnode && rnode); if (IsA(lnode, Var)) { var = castNode(Var, lnode); other = lsecond(op->args); } else if (IsA(rnode, Var)) { var = castNode(Var, rnode); other = linitial(op->args); } else continue; if ((Index) var->varno != chunk_rel->relid || var->varattno <= 0) continue; if (IsA(other, Const) || IsA(other, Param)) { TypeCacheEntry *tce = lookup_type_cache(var->vartype, TYPECACHE_EQ_OPR); if (op->opno != tce->eq_opr) { /* Issue #9066: check if our OpExpr is still an equality (Var = Const) * for Var and Const of different types */ #if PG18_GE List *opinfos = get_op_index_interpretation(op->opno); #else List *opinfos = get_op_btree_interpretation(op->opno); #endif ListCell *lc; bool equality = false; foreach (lc, opinfos) { #if PG18_GE OpIndexInterpretation *opinfo = (OpIndexInterpretation *) lfirst(lc); if (opinfo->cmptype == COMPARE_EQ) #else OpBtreeInterpretation *opinfo = (OpBtreeInterpretation *) lfirst(lc); if (opinfo->strategy == BTEqualStrategyNumber) #endif { Oid mixed_type_eqop = get_opfamily_member(opinfo->opfamily_id, var->vartype, exprType((Node *) other), BTEqualStrategyNumber); if (op->opno == mixed_type_eqop) { equality = true; break; } } } if (!equality) continue; } if (bms_is_member(var->varattno, info->chunk_segmentby_attnos)) segmentby_columns = bms_add_member(segmentby_columns, var->varattno); } } } } return segmentby_columns; } /* * Returns whether the pathkeys starting at the given offset match the compression * orderby, and whether the order is reverse. */ static bool match_pathkeys_to_compression_orderby(List *pathkeys, List *chunk_em_exprs, int starting_pathkey_offset, const CompressionInfo *compression_info, bool *out_reverse) { int compressed_pk_index = 0; for (int i = starting_pathkey_offset; i < list_length(pathkeys); i++) { compressed_pk_index++; PathKey *pk = list_nth_node(PathKey, pathkeys, i); Node *node = strip_implicit_coercions((Node *) list_nth(chunk_em_exprs, i)); if (node == NULL || !IsA(node, Var)) { return false; } Var *var = castNode(Var, node); if (var->varattno <= 0) { return false; } char *column_name = get_attname(compression_info->chunk_rte->relid, var->varattno, false); int orderby_index = ts_array_position(compression_info->settings->fd.orderby, column_name); if (orderby_index != compressed_pk_index) { return false; } bool orderby_desc = ts_array_get_element_bool(compression_info->settings->fd.orderby_desc, orderby_index); bool orderby_nullsfirst = ts_array_get_element_bool(compression_info->settings->fd.orderby_nullsfirst, orderby_index); /* * In PG18+: pk_cmptype is either COMPARE_LT (for ASC) or COMPARE_GT (for DESC) * For previous PG versions we have compatibility macros to make these new names available. */ bool this_pathkey_reverse = false; if (pk->pk_cmptype == COMPARE_LT) { if (!orderby_desc && orderby_nullsfirst == pk->pk_nulls_first) { this_pathkey_reverse = false; } else if (orderby_desc && orderby_nullsfirst != pk->pk_nulls_first) { this_pathkey_reverse = true; } else { return false; } } else if (pk->pk_cmptype == COMPARE_GT) { if (orderby_desc && orderby_nullsfirst == pk->pk_nulls_first) { this_pathkey_reverse = false; } else if (!orderby_desc && orderby_nullsfirst != pk->pk_nulls_first) { this_pathkey_reverse = true; } else { return false; } } /* * first pathkey match determines if this is forward or backward scan * any further pathkey items need to have same direction */ if (compressed_pk_index == 1) { *out_reverse = this_pathkey_reverse; } else if (this_pathkey_reverse != *out_reverse) { return false; } } return true; } /* * Check if we can push down the sort below the ColumnarScan node and fill * SortInfo accordingly * * The following conditions need to be true for pushdown: * - all segmentby columns need to be prefix of pathkeys or have equality constraint * - the rest of pathkeys needs to match compress_orderby * * If query pathkeys is shorter than segmentby + compress_orderby pushdown can still be done */ static SortInfo build_sortinfo(PlannerInfo *root, const Chunk *chunk, RelOptInfo *chunk_rel, const CompressionInfo *compression_info, List *pathkeys) { Var *var; char *column_name; ListCell *lc; SortInfo sort_info = { 0 }; if (pathkeys == NIL) { return sort_info; } /* * Translate the pathkeys to chunk expressions, creating a List of them * parallel to the pathkeys list, with NULL entries if we didn't find a * match. */ List *chunk_em_exprs = NIL; foreach (lc, pathkeys) { PathKey *pk = lfirst(lc); EquivalenceClass *ec = pk->pk_eclass; Expr *em_expr = NULL; if (!ec->ec_has_volatile) { em_expr = ts_find_em_expr_for_rel(pk->pk_eclass, compression_info->chunk_rel); } chunk_em_exprs = lappend(chunk_em_exprs, em_expr); } Assert(list_length(chunk_em_exprs) == list_length(pathkeys)); /* Find the pathkeys we can use for explicitly sorting after decompression. */ List *sort_pathkey_exprs = NIL; List *sort_pathkeys = NIL; for (int i = 0; i < list_length(chunk_em_exprs); i++) { PathKey *pk = list_nth_node(PathKey, pathkeys, i); Expr *chunk_em_expr = (Expr *) list_nth(chunk_em_exprs, i); if (chunk_em_expr == NULL) { break; } sort_pathkeys = lappend(sort_pathkeys, pk); sort_pathkey_exprs = lappend(sort_pathkey_exprs, chunk_em_expr); } if (sort_pathkeys == NIL) { return sort_info; } sort_info.decompressed_sort_pathkeys = sort_pathkeys; cost_qual_eval(&sort_info.decompressed_sort_pathkeys_cost, sort_pathkey_exprs, root); /* * Next, check if we can push the sort down to the compressed part. * * Batch sorted merge optimization is enabled for unordered chunks * because we do merge the batches at execution time which * only relies that the batches themselves are sorted which is * always the case. */ /* all segmentby columns need to be prefix of pathkeys */ int i = 0; if (compression_info->num_segmentby_columns > 0) { Bitmapset *segmentby_columns; /* * initialize segmentby with equality constraints from baserestrictinfo because * those columns dont need to be prefix of pathkeys */ segmentby_columns = bms_copy(compression_info->chunk_const_segmentby); /* * loop over pathkeys until we find one that is not a segmentby column * we keep looping even if we found all segmentby columns in case a * columns appears both in baserestrictinfo and in ORDER BY clause */ for (i = 0; i < list_length(pathkeys); i++) { Assert(bms_num_members(segmentby_columns) <= compression_info->num_segmentby_columns); Node *node = strip_implicit_coercions((Node *) list_nth(chunk_em_exprs, i)); if (node == NULL || !IsA(node, Var)) break; var = castNode(Var, node); if (var->varattno <= 0) break; column_name = get_attname(compression_info->chunk_rte->relid, var->varattno, false); if (!ts_array_is_member(compression_info->settings->fd.segmentby, column_name)) break; segmentby_columns = bms_add_member(segmentby_columns, var->varattno); } /* * Pathkeys satisfied by sorting the compressed data on segmentby columns. * Can use compressed sort on segmentby cols for unordered chunks as well. */ if (i == list_length(pathkeys)) { sort_info.use_compressed_sort = true; return sort_info; } /* * If pathkeys still has items, but we didn't find all segmentby columns, * we cannot satisfy these pathkeys by sorting the compressed chunk table. */ if (i != list_length(pathkeys) && bms_num_members(segmentby_columns) != compression_info->num_segmentby_columns) { /* * If we didn't have any segmentby columns in pathkeys, try batch sorted merge * instead. */ if (i == 0) { sort_info.use_batch_sorted_merge = match_pathkeys_to_compression_orderby(pathkeys, chunk_em_exprs, /* starting_pathkey_offset = */ 0, compression_info, &sort_info.reverse); } return sort_info; } } /* * Cannot push down sort on non-segmentby columns * if the chunk has batches overlapping on orderby columns */ if (ts_chunk_is_unordered(chunk)) return sort_info; /* * Pathkeys includes columns past segmentby columns, so we need sequence_num * in the targetlist for ordering. */ sort_info.needs_sequence_num = true; /* * loop over the rest of pathkeys * this needs to exactly match the configured compress_orderby */ sort_info.use_compressed_sort = match_pathkeys_to_compression_orderby(pathkeys, chunk_em_exprs, i, compression_info, &sort_info.reverse); return sort_info; } /* Check if the provided path is a ColumnarScanPath */ bool ts_is_columnar_scan_path(Path *path) { return IsA(path, CustomPath) && castNode(CustomPath, path)->methods == &columnar_scan_path_methods; } ================================================ FILE: tsl/src/nodes/columnar_scan/columnar_scan.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/bitmapset.h> #include <nodes/extensible.h> #include "chunk.h" #include "hypertable.h" #include "ts_catalog/compression_settings.h" typedef struct CompressionInfo { RelOptInfo *chunk_rel; RelOptInfo *compressed_rel; RelOptInfo *ht_rel; RangeTblEntry *chunk_rte; RangeTblEntry *compressed_rte; RangeTblEntry *ht_rte; Oid compresseddata_oid; CompressionSettings *settings; int hypertable_id; List *hypertable_compression_info; int num_orderby_columns; int num_segmentby_columns; /* chunk attribute numbers that are segmentby columns */ Bitmapset *chunk_segmentby_attnos; /* * Chunk segmentby attribute numbers that are equated to a constant by a * baserestrictinfo. */ Bitmapset *chunk_const_segmentby; /* compressed chunk attribute numbers for columns that are compressed */ Bitmapset *compressed_attnos_in_compressed_chunk; bool single_chunk; /* query on explicit chunk */ bool has_seq_num; /* legacy sequence number support */ Relids parent_relids; /* relids of the parent hypertable and UNION */ /* Compressed batch size estimated from statistics. */ double compressed_batch_size; int32 chunk_status; } CompressionInfo; typedef struct ColumnarScanPath { CustomPath custom_path; const CompressionInfo *info; List *required_compressed_pathkeys; bool needs_sequence_num; bool reverse; bool batch_sorted_merge; int32 chunk_status; } ColumnarScanPath; void ts_columnar_scan_generate_paths(PlannerInfo *root, RelOptInfo *rel, const Hypertable *ht, const Chunk *chunk); extern bool ts_is_columnar_scan_path(Path *path); extern bool ts_is_columnar_scan_plan(Plan *plan); extern double ts_columnar_estimate_compressed_batch_size(const Oid relid); ColumnarScanPath *copy_columnar_scan_path(ColumnarScanPath *src); ================================================ FILE: tsl/src/nodes/columnar_scan/compressed_batch.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <executor/tuptable.h> #include <nodes/bitmapset.h> #include <utils/builtins.h> #include <utils/date.h> #include <utils/timestamp.h> #include <utils/uuid.h> #include "compression/arrow_c_data_interface.h" #include "compression/compression.h" #include "debug_assert.h" #include "guc.h" #include "nodes/columnar_scan/compressed_batch.h" #include "nodes/columnar_scan/vector_dict.h" #include "nodes/columnar_scan/vector_predicates.h" #include "nodes/columnar_scan/vector_quals.h" /* * Create a single-value ArrowArray of an arithmetic type. This is a specialized * function because arithmetic types have a particular layout of ArrowArrays. */ static ArrowArray * make_single_value_arrow_arithmetic(Oid arithmetic_type, Datum datum, bool isnull) { struct ArrowWithBuffers { ArrowArray arrow; uint64 arrow_buffers_array_storage[2]; uint64 validity_buffer[1]; /* The value buffer has 64-byte padding as required by Arrow. */ uint64 values_buffer[8]; }; struct ArrowWithBuffers *with_buffers = palloc0(sizeof(struct ArrowWithBuffers)); ArrowArray *arrow = &with_buffers->arrow; arrow->length = 1; arrow->buffers = (const void **) with_buffers->arrow_buffers_array_storage; arrow->n_buffers = 2; arrow->buffers[0] = with_buffers->validity_buffer; arrow->buffers[1] = with_buffers->values_buffer; if (isnull) { /* * The validity bitmap was initialized to invalid on allocation, and * the Datum might be invalid if the value is null (important on i386 * where it might be pass-by-reference), so don't read it. */ arrow->null_count = 1; return arrow; } arrow_set_row_validity((uint64 *) arrow->buffers[0], 0, true); #define FOR_TYPE(PGTYPE, CTYPE, FROMDATUM) \ case PGTYPE: \ *((CTYPE *) arrow->buffers[1]) = FROMDATUM(datum); \ break switch (arithmetic_type) { FOR_TYPE(INT8OID, int64, DatumGetInt64); FOR_TYPE(INT4OID, int32, DatumGetInt32); FOR_TYPE(INT2OID, int16, DatumGetInt16); FOR_TYPE(FLOAT8OID, float8, DatumGetFloat8); FOR_TYPE(FLOAT4OID, float4, DatumGetFloat4); FOR_TYPE(TIMESTAMPTZOID, TimestampTz, DatumGetTimestampTz); FOR_TYPE(TIMESTAMPOID, Timestamp, DatumGetTimestamp); FOR_TYPE(DATEOID, DateADT, DatumGetDateADT); FOR_TYPE(UUIDOID, pg_uuid_t, *DatumGetUUIDP); case BOOLOID: arrow_set_row_validity((uint64 *) arrow->buffers[1], 0, DatumGetBool(datum)); break; default: elog(ERROR, "unexpected column type '%s'", format_type_be(arithmetic_type)); pg_unreachable(); } #undef FOR_TYPE return arrow; } /* * Create a single-value ArrowArray of text. This is a specialized function * because the text ArrowArray has a specialized layout. */ static ArrowArray * make_single_value_arrow_text(Datum datum, bool isnull) { struct ArrowWithBuffers { ArrowArray arrow; uint64 arrow_buffers_array_storage[3]; uint64 validity_buffer[1]; uint32 offsets_buffer[2]; /* The value buffer has 64-byte padding as required by Arrow. */ uint64 values_buffer[8]; }; struct ArrowWithBuffers *with_buffers = palloc0(sizeof(struct ArrowWithBuffers)); ArrowArray *arrow = &with_buffers->arrow; arrow->length = 1; arrow->buffers = (const void **) with_buffers->arrow_buffers_array_storage; arrow->n_buffers = 3; arrow->buffers[0] = with_buffers->validity_buffer; arrow->buffers[1] = with_buffers->offsets_buffer; arrow->buffers[2] = with_buffers->values_buffer; if (isnull) { /* * The validity bitmap was initialized to invalid on allocation, and * the Datum might be invalid if the value is null (important on i386 * where it might be pass-by-reference), so don't read it. */ arrow->null_count = 1; return arrow; } arrow_set_row_validity((uint64 *) arrow->buffers[0], 0, true); text *detoasted = PG_DETOAST_DATUM(datum); ((uint32 *) arrow->buffers[1])[1] = VARSIZE_ANY_EXHDR(detoasted); arrow->buffers[2] = VARDATA(detoasted); return arrow; } /* * Create a single value ArrowArray from Postgres Datum. This is used to run * the usual vectorized predicates on compressed columns with default values. */ ArrowArray * make_single_value_arrow(Oid pgtype, Datum datum, bool isnull) { if (pgtype == TEXTOID) { return make_single_value_arrow_text(datum, isnull); } return make_single_value_arrow_arithmetic(pgtype, datum, isnull); } int get_max_varlena_bytes(ArrowArray *text_array) { if (text_array->dictionary != NULL) { Ensure(text_array->dictionary->dictionary == NULL, "got dictionary-encoded dictionary for a text arrow array"); return get_max_varlena_bytes(text_array->dictionary); } int maxbytes = 0; uint32 *offsets = (uint32 *) text_array->buffers[1]; for (int i = 0; i < text_array->length; i++) { const int curbytes = offsets[i + 1] - offsets[i]; if (curbytes > maxbytes) { maxbytes = curbytes; } } return VARHDRSZ + maxbytes; } static void decompress_scalar_column(CompressedColumnValues *column, Datum value, bool isnull) { column->decompression_type = DT_Scalar; column->buffers[0] = DatumGetPointer(BoolGetDatum(isnull)); column->buffers[1] = DatumGetPointer(value); *column->output_isnull = isnull; *column->output_value = value; } static void decompress_column(DecompressContext *dcontext, DecompressBatchState *batch_state, TupleTableSlot *compressed_slot, int i) { CompressionColumnDescription *column_description = &dcontext->compressed_chunk_columns[i]; CompressedColumnValues *column_values = &batch_state->compressed_columns[i]; column_values->arrow = NULL; const int value_bytes = get_typlen(column_description->typid); Assert(value_bytes != 0); bool isnull; Datum value = slot_getattr(compressed_slot, column_description->compressed_scan_attno, &isnull); if (isnull) { /* * The column will have a default value for the entire batch, * set it now. * * We might use a custom targetlist-based scan tuple which has no * default values, so the default values are fetched from the * uncompressed chunk tuple descriptor. */ bool isnull; Datum value = getmissingattr(dcontext->uncompressed_chunk_tdesc, column_description->uncompressed_chunk_attno, &isnull); decompress_scalar_column(column_values, value, isnull); return; } /* Detoast the compressed datum. */ value = PointerGetDatum(detoaster_detoast_attr_copy((struct varlena *) DatumGetPointer(value), &dcontext->detoaster, batch_state->per_batch_context)); CompressedDataHeader *header = (CompressedDataHeader *) value; /* First check if this is a block of NULL values. */ if (header->compression_algorithm == COMPRESSION_ALGORITHM_NULL) { decompress_scalar_column(column_values, (Datum) NULL, /* isnull = */ true); return; } /* Decompress the entire batch if it is supported. */ ArrowArray *arrow = NULL; if (dcontext->enable_bulk_decompression && column_description->bulk_decompression_supported) { if (dcontext->bulk_decompression_context == NULL) { dcontext->bulk_decompression_context = create_bulk_decompression_mctx( MemoryContextGetParent(batch_state->per_batch_context)); } DecompressAllFunction decompress_all = tsl_get_decompress_all_function(header->compression_algorithm, column_description->typid); Assert(decompress_all != NULL); MemoryContext context_before_decompression = MemoryContextSwitchTo(dcontext->bulk_decompression_context); arrow = decompress_all(PointerGetDatum(header), column_description->typid, batch_state->per_batch_context); MemoryContextSwitchTo(context_before_decompression); MemoryContextReset(dcontext->bulk_decompression_context); } if (arrow == NULL) { /* As a fallback, decompress row-by-row. */ column_values->decompression_type = DT_Iterator; MemoryContext old_context = MemoryContextSwitchTo(batch_state->per_batch_context); column_values->buffers[0] = tsl_get_decompression_iterator_init(header->compression_algorithm, dcontext->reverse)(PointerGetDatum(header), column_description->typid); MemoryContextSwitchTo(old_context); return; } /* Should have been filled from the count metadata column. */ Assert(batch_state->total_batch_rows != 0); if (batch_state->total_batch_rows != arrow->length) { elog(ERROR, "compressed column out of sync with batch counter"); } column_values->arrow = arrow; if (value_bytes > 0) { /* Fixed-width column. */ column_values->decompression_type = value_bytes; column_values->buffers[0] = arrow->buffers[0]; column_values->buffers[1] = arrow->buffers[1]; column_values->buffers[2] = NULL; column_values->buffers[3] = NULL; if (column_description->typid == BOOLOID) { /* The bool columns have a dedicated storage format. */ column_values->decompression_type = DT_ArrowBits; } } else { /* * Text column. Pre-allocate memory for its text Datum in the * decompressed scan slot. We can't put direct references to Arrow * memory there, because it doesn't have the varlena headers that * Postgres expects for text. */ const int maxbytes = get_max_varlena_bytes(arrow); *column_values->output_value = PointerGetDatum(MemoryContextAlloc(batch_state->per_batch_context, maxbytes)); /* * Set up the datum conversion based on whether we use the dictionary. */ if (arrow->dictionary == NULL) { column_values->decompression_type = DT_ArrowText; column_values->buffers[0] = arrow->buffers[0]; column_values->buffers[1] = arrow->buffers[1]; column_values->buffers[2] = arrow->buffers[2]; column_values->buffers[3] = NULL; } else { column_values->decompression_type = DT_ArrowTextDict; column_values->buffers[0] = arrow->buffers[0]; column_values->buffers[1] = arrow->dictionary->buffers[1]; column_values->buffers[2] = arrow->dictionary->buffers[2]; column_values->buffers[3] = arrow->buffers[1]; } } } /* * Get the arrow array for the compressed batch via the VectorQualState. * * This is a ColumnarScan-specific implementation of the * VectorQualState->get_arrow_array() function used to interface with the * vector qual code across different scan nodes. */ const ArrowArray * compressed_batch_get_arrow_array(VectorQualState *vqstate, Expr *expr, bool *is_default_value) { CompressedBatchVectorQualState *cbvqstate = (CompressedBatchVectorQualState *) vqstate; DecompressContext *dcontext = cbvqstate->dcontext; DecompressBatchState *batch_state = cbvqstate->batch_state; CompressionColumnDescription *column_description = NULL; TupleTableSlot *compressed_slot = vqstate->slot; const Var *var = castNode(Var, expr); int column_index = 0; for (; column_index < dcontext->num_data_columns; column_index++) { column_description = &dcontext->compressed_chunk_columns[column_index]; if (var->varno == INDEX_VAR) { /* * Reference into custom scan tlist, happens when we are using a * non-default custom scan tuple. */ if (column_description->custom_scan_attno == var->varattno) { break; } } else { /* * Reference into uncompressed chunk tuple. * * Note that this is somewhat redundant, because this branch is * taken when we do not use a custom scan tuple, and in this case * the custom scan attno is the same as the uncompressed chunk attno, * so the above branch would do as well. This difference might * become relevant in the future, if we stop outputting the * columns that are needed only for the vectorized quals. */ if (column_description->uncompressed_chunk_attno == var->varattno) { break; } } } Ensure(column_index < dcontext->num_data_columns, "decompressed column %d not found in batch", var->varattno); Assert(column_description != NULL); Assert(column_description->typid == var->vartype); CompressedColumnValues *column_values = &batch_state->compressed_columns[column_index]; if (column_values->decompression_type == DT_Invalid) { /* * We decompress the compressed columns on demand, so that we can * skip decompressing some columns if the entire batch doesn't pass * the quals. */ decompress_column(dcontext, batch_state, compressed_slot, column_index); Assert(column_values->decompression_type != DT_Invalid); } Ensure(column_values->decompression_type != DT_Iterator, "expected arrow array but got iterator for column index %d", column_index); /* * Prepare to compute the vector predicate. We have to handle the * default values in a special way because they don't produce the usual * decompressed ArrowArrays. */ const ArrowArray *vector = column_values->arrow; if (column_values->arrow == NULL) { /* * The compressed column had a default value. We can't fall back to * the non-vectorized quals now, so build a single-value ArrowArray * with this default value, check if it passes the predicate, and apply * it to the entire batch. */ Assert(column_values->decompression_type == DT_Scalar); /* * We saved the actual default value into the decompressed scan slot * above, so pull it from there. */ vector = make_single_value_arrow(column_description->typid, PointerGetDatum(column_values->buffers[1]), DatumGetBool(PointerGetDatum(column_values->buffers[0]))); /* * We start from an all-valid bitmap, because the predicate is * AND-ed to it. */ *is_default_value = true; } else *is_default_value = false; return vector; } static void compute_plain_qual(VectorQualState *vqstate, TupleTableSlot *slot, Node *qual, uint64 *restrict result) { /* * Some predicates can be evaluated to a Const at run time. */ if (IsA(qual, Const)) { Const *c = castNode(Const, qual); if (c->constisnull || !DatumGetBool(c->constvalue)) { /* * Some predicates are evaluated to a null Const, like a * strict comparison with stable expression that evaluates to null. * No rows pass. */ const size_t n_batch_result_words = (vqstate->num_results + 63) / 64; for (size_t i = 0; i < n_batch_result_words; i++) { result[i] = 0; } } else { /* * This is a constant true qual, every row passes and we can * just ignore it. No idea how it can happen though. */ Assert(false); } return; } /* * For now, we support NullTest, "Var ? Const" predicates, * boolean Variables, the negation of boolean variables * and ScalarArrayOperations. */ List *args = NULL; RegProcedure vector_const_opcode = InvalidOid; ScalarArrayOpExpr *saop = NULL; OpExpr *opexpr = NULL; NullTest *nulltest = NULL; BooleanTest *booltest = NULL; Var *bool_var = NULL; bool negate_bool_var = false; if (IsA(qual, NullTest)) { nulltest = castNode(NullTest, qual); args = list_make1(nulltest->arg); } else if (IsA(qual, ScalarArrayOpExpr)) { saop = castNode(ScalarArrayOpExpr, qual); args = saop->args; vector_const_opcode = get_opcode(saop->opno); } else if (IsA(qual, Var)) { bool_var = castNode(Var, qual); Ensure(bool_var->vartype == BOOLOID, "expected boolean Var"); args = list_make1(bool_var); } else if (IsA(qual, BoolExpr)) { BoolExpr *boolexpr = castNode(BoolExpr, qual); Ensure(boolexpr->boolop == NOT_EXPR, "expected NOT BoolExpr"); Ensure(list_length(boolexpr->args) == 1, "expected one argument in NOT BoolExpr"); Ensure(IsA(linitial(boolexpr->args), Var), "expected Var in NOT BoolExpr"); bool_var = castNode(Var, linitial(boolexpr->args)); Ensure(bool_var->vartype == BOOLOID, "expected boolean Var"); /* * We can vectorize boolean variables like 'COL = false' which is * transformed to BoolExpr(NOT_EXPR, Var). */ negate_bool_var = true; bool_var = castNode(Var, linitial(boolexpr->args)); args = list_make1(bool_var); } else if (IsA(qual, BooleanTest)) { booltest = castNode(BooleanTest, qual); Ensure(IsA(booltest->arg, Var), "expected Var in BooleanTest"); args = list_make1(booltest->arg); } else { Ensure(IsA(qual, OpExpr), "expected OpExpr"); opexpr = castNode(OpExpr, qual); args = opexpr->args; vector_const_opcode = get_opcode(opexpr->opno); } /* * Find the compressed column referred to by the Var. */ Expr *expr = linitial(args); uint64 default_value_predicate_result[1]; uint64 *predicate_result = result; bool default_value = false; const ArrowArray *vector = vqstate->get_arrow_array(vqstate, expr, &default_value); if (default_value) { /* * We start from an all-valid bitmap, because the predicate is * AND-ed to it. */ default_value_predicate_result[0] = 1; predicate_result = default_value_predicate_result; } if (nulltest) { vector_nulltest(vector, nulltest->nulltesttype, predicate_result); } else if (bool_var) { vector_booleantest(vector, (negate_bool_var ? IS_FALSE : IS_TRUE), predicate_result); } else if (booltest) { vector_booleantest(vector, booltest->booltesttype, predicate_result); } else { /* * Find the vector_const predicate. */ VectorPredicate *vector_const_predicate = get_vector_const_predicate(vector_const_opcode); Assert(vector_const_predicate != NULL); Ensure(IsA(lsecond(args), Const), "failed to evaluate runtime constant in vectorized filter"); /* * The vectorizable predicates should be STRICT, so we shouldn't see null * constants here. However, a null constant can still come from a scalar * array expression like x = any(null::int[]). Such a predicate fails * for all rows. */ Const *constnode = castNode(Const, lsecond(args)); Ensure(saop != NULL || !constnode->constisnull, "vectorized predicate called for a null value"); if (constnode->constisnull) { const size_t n_batch_result_words = (vqstate->num_results + 63) / 64; for (size_t i = 0; i < n_batch_result_words; i++) { result[i] = 0; } return; } /* * If the data is dictionary-encoded, we are going to compute the * predicate on dictionary and then translate the results. */ const ArrowArray *vector_nodict = NULL; uint64 *restrict predicate_result_nodict = NULL; uint64 dict_result[(GLOBAL_MAX_ROWS_PER_COMPRESSION + 63) / 64]; if (vector->dictionary) { const size_t dict_rows = vector->dictionary->length; const size_t dict_result_words = (dict_rows + 63) / 64; memset(dict_result, 0xFF, dict_result_words * 8); predicate_result_nodict = dict_result; vector_nodict = vector->dictionary; } else { predicate_result_nodict = predicate_result; vector_nodict = vector; } /* * At last, compute the predicate. */ if (saop) { vector_array_predicate(vector_const_predicate, saop->useOr, vector_nodict, constnode->constvalue, predicate_result_nodict); } else { vector_const_predicate(vector_nodict, constnode->constvalue, predicate_result_nodict); } /* * If the vector is dictionary-encoded, we have just computed the * predicate for dictionary and now have to translate it. */ if (vector->dictionary) { translate_bitmap_from_dictionary(vector, predicate_result_nodict, predicate_result); } /* * Account for nulls which shouldn't pass the predicate. Note that the * vector here might have only one row, in contrast with the number of * rows in the batch, if the column has a default value in this batch. */ const size_t n_vector_result_words = (vector->length + 63) / 64; Assert((predicate_result != default_value_predicate_result) || n_vector_result_words == 1); /* to placate Coverity. */ const uint64 *validity = (const uint64 *) vector->buffers[0]; if (validity) { for (size_t i = 0; i < n_vector_result_words; i++) { predicate_result[i] &= validity[i]; } } else { Assert(vector->null_count == 0); } } /* Translate the result if the column had a default value. */ if (default_value) { if (!(default_value_predicate_result[0] & 1)) { /* * We had a default value for the compressed column, and it * didn't pass the predicate, so the entire batch didn't pass. */ const size_t n_batch_result_words = (vqstate->num_results + 63) / 64; for (size_t i = 0; i < n_batch_result_words; i++) { result[i] = 0; } } } } static void compute_one_qual(VectorQualState *vqstate, TupleTableSlot *compressed_slot, Node *qual, uint64 *restrict result); static void compute_qual_conjunction(VectorQualState *vqstate, TupleTableSlot *compressed_slot, List *quals, uint64 *restrict result) { ListCell *lc; foreach (lc, quals) { compute_one_qual(vqstate, compressed_slot, lfirst(lc), result); if (get_vector_qual_summary(result, vqstate->num_results) == NoRowsPass) { /* * Exit early if no rows pass already. This might allow us to avoid * reading the columns required for the subsequent quals. */ return; } } } static void compute_qual_disjunction(VectorQualState *vqstate, TupleTableSlot *compressed_slot, List *quals, uint64 *restrict result) { const size_t n_rows = vqstate->num_results; const size_t n_result_words = (n_rows + 63) / 64; uint64 *or_result = MemoryContextAlloc(vqstate->per_vector_mcxt, (sizeof(uint64) * n_result_words)); for (size_t i = 0; i < n_result_words; i++) { or_result[i] = 0; } uint64 *one_qual_result = MemoryContextAlloc(vqstate->per_vector_mcxt, (sizeof(uint64) * n_result_words)); ListCell *lc; foreach (lc, quals) { for (size_t i = 0; i < n_result_words; i++) { one_qual_result[i] = (uint64) -1; } compute_one_qual(vqstate, compressed_slot, lfirst(lc), one_qual_result); for (size_t i = 0; i < n_result_words; i++) { or_result[i] |= one_qual_result[i]; } if (get_vector_qual_summary(or_result, n_rows) == AllRowsPass) { /* * We can sometimes avoing reading the columns required for the * rest of conditions if we break out early here. */ return; } } for (size_t i = 0; i < n_result_words; i++) { result[i] &= or_result[i]; } } static void compute_one_qual(VectorQualState *vqstate, TupleTableSlot *compressed_slot, Node *qual, uint64 *restrict result) { if (!IsA(qual, BoolExpr)) { compute_plain_qual(vqstate, compressed_slot, qual, result); return; } BoolExpr *boolexpr = castNode(BoolExpr, qual); if (boolexpr->boolop == AND_EXPR) { compute_qual_conjunction(vqstate, compressed_slot, boolexpr->args, result); return; } /* * Postgres removes NOT for operators we can vectorize, so we don't support * NOT in general, except when the column is boolean and the expression is * like 'Col = false'. In this case, the NOT is present and we can vectorize * it. * * Apart from these cases, only OR is left. */ if (boolexpr->boolop == NOT_EXPR) { if (list_length(boolexpr->args) == 1 && IsA(linitial(boolexpr->args), Var)) { compute_plain_qual(vqstate, compressed_slot, qual, result); return; } } Ensure(boolexpr->boolop == OR_EXPR, "expected OR"); compute_qual_disjunction(vqstate, compressed_slot, boolexpr->args, result); } /* * Compute the vectorized filters. Returns true if we have any passing rows. If not, * it means the entire batch is filtered out, and we use this for further * optimizations. */ BatchQualSummary vector_qual_compute(VectorQualState *vqstate) { /* * Allocate the bitmap that will hold the vectorized qual results. We will * initialize it to all ones and AND the individual quals to it. */ const size_t n_rows = vqstate->num_results; const int bitmap_bytes = sizeof(uint64) * ((n_rows + 63) / 64); vqstate->vector_qual_result = MemoryContextAlloc(vqstate->per_vector_mcxt, bitmap_bytes); memset(vqstate->vector_qual_result, 0xFF, bitmap_bytes); if (n_rows % 64 != 0) { /* * We have to zero out the bits for past-the-end elements in the last * bitmap word. Since all predicates are ANDed to the result bitmap, * we can do it here once instead of doing it in each predicate. */ const uint64 mask = ((uint64) -1) >> (64 - vqstate->num_results % 64); vqstate->vector_qual_result[vqstate->num_results / 64] = mask; } /* * Compute the quals. */ compute_qual_conjunction(vqstate, vqstate->slot, vqstate->vectorized_quals_constified, vqstate->vector_qual_result); return get_vector_qual_summary(vqstate->vector_qual_result, n_rows); } /* * Scrolls the compressed batch to the end, discarding any tuples left in it. * This makes the batch ready to accept the next compressed tuple, but without * de-initializing its expensive reusable parts such as memory context and tuple * slots. This is used when vectorized quals don't pass for the entire batch, * and also in batch array to free the batch state for reuse. */ void compressed_batch_discard_tuples(DecompressBatchState *batch_state) { batch_state->next_batch_row = batch_state->total_batch_rows; batch_state->vector_qual_result = NULL; if (batch_state->per_batch_context != NULL) { ExecClearTuple(&batch_state->decompressed_scan_slot_data.base); MemoryContextReset(batch_state->per_batch_context); } else { /* * Check that we have a valid zero-initialized batch here. */ Assert(IsA(&batch_state->decompressed_scan_slot_data, Invalid)); Assert(batch_state->decompressed_scan_slot_data.base.tts_ops == NULL); } } /* * Initializes the zero-initialized batch state. We do this on demand, because * it involves the creation of memory context and tuple slots, which are * relatively expensive. */ static void compressed_batch_lazy_init(DecompressContext *dcontext, DecompressBatchState *batch_state) { /* Init memory context */ batch_state->per_batch_context = create_per_batch_mctx(dcontext); Assert(batch_state->per_batch_context != NULL); /* Get a reference to the decompressed scan TupleTableSlot */ TupleTableSlot *decompressed_slot = dcontext->custom_scan_slot; /* * This code follows Postgres' MakeTupleTableSlot(). */ TupleTableSlot *slot = &batch_state->decompressed_scan_slot_data.base; Assert(IsA(slot, Invalid)); Assert(slot->tts_ops == NULL); slot->type = T_TupleTableSlot; slot->tts_flags = TTS_FLAG_EMPTY | TTS_FLAG_FIXED; /* * The decompressed slot and the respective tuple descriptor are owned by * DecompressContext and live throughout the entire decompression process, * so here we can reuse a plain pointer to the tuple descriptor without * additional reference counting. */ slot->tts_tupleDescriptor = decompressed_slot->tts_tupleDescriptor; slot->tts_mcxt = CurrentMemoryContext; slot->tts_nvalid = 0; slot->tts_values = palloc0(MAXALIGN(slot->tts_tupleDescriptor->natts * sizeof(Datum)) + MAXALIGN(slot->tts_tupleDescriptor->natts * sizeof(bool))); slot->tts_isnull = (bool *) ((char *) slot->tts_values) + MAXALIGN(slot->tts_tupleDescriptor->natts * sizeof(Datum)); /* * Have to initially set nulls to true, because this is the uncompressed chunk * tuple, and some of its columns might be not even decompressed. The tuple * slot functions will get confused by them, because they expect a non-null * value for attributes not marked as null. */ memset(slot->tts_isnull, true, slot->tts_tupleDescriptor->natts * sizeof(bool)); /* * ColumnarScan produces virtual tuple slots. */ *((const TupleTableSlotOps **) &slot->tts_ops) = &TTSOpsVirtual; slot->tts_ops->init(slot); } /* * Initialize the batch decompression state with the new compressed tuple. */ void compressed_batch_set_compressed_tuple(DecompressContext *dcontext, DecompressBatchState *batch_state, TupleTableSlot *compressed_slot) { Assert(TupIsNull(compressed_batch_current_tuple(batch_state))); /* * The batch states are initialized on demand, because creating the memory * context and the tuple table slots is expensive. */ if (batch_state->per_batch_context == NULL) { compressed_batch_lazy_init(dcontext, batch_state); } TupleTableSlot *decompressed_tuple = compressed_batch_current_tuple(batch_state); Assert(decompressed_tuple != NULL); batch_state->total_batch_rows = 0; batch_state->next_batch_row = 0; MemoryContextReset(batch_state->per_batch_context); for (int i = 0; i < dcontext->num_columns_with_metadata; i++) { CompressionColumnDescription *column_description = &dcontext->compressed_chunk_columns[i]; switch (column_description->type) { case COMPRESSED_COLUMN: { /* * We decompress the compressed columns on demand, so that we can * skip decompressing some columns if the entire batch doesn't pass * the quals. Skip them for now. */ Assert(i < dcontext->num_data_columns); CompressedColumnValues *column_values = &batch_state->compressed_columns[i]; column_values->decompression_type = DT_Invalid; column_values->arrow = NULL; const AttrNumber attr = AttrNumberGetAttrOffset(column_description->custom_scan_attno); column_values->output_value = &decompressed_tuple->tts_values[attr]; column_values->output_isnull = &decompressed_tuple->tts_isnull[attr]; break; } case SEGMENTBY_COLUMN: { /* * A segmentby column is not going to change during one batch, * and our output tuples are read-only, so it's enough to only * save it once per batch, which we do here. */ Assert(i < dcontext->num_data_columns); CompressedColumnValues *column_values = &batch_state->compressed_columns[i]; bool isnull; Datum value = slot_getattr(compressed_slot, column_description->compressed_scan_attno, &isnull); /* * Note that if it's not a by-value type, we should copy it into * the slot context. */ if (!column_description->by_value && !isnull && DatumGetPointer(value) != NULL) { if (column_description->value_bytes < 0) { /* This is a varlena type. */ value = PointerGetDatum( detoaster_detoast_attr_copy((struct varlena *) value, &dcontext->detoaster, batch_state->per_batch_context)); } else { /* This is a fixed-length by-reference type. */ void *tmp = MemoryContextAlloc(batch_state->per_batch_context, column_description->value_bytes); memcpy(tmp, DatumGetPointer(value), column_description->value_bytes); value = PointerGetDatum(tmp); } } const AttrNumber attr = AttrNumberGetAttrOffset(column_description->custom_scan_attno); column_values->output_value = &decompressed_tuple->tts_values[attr]; column_values->output_isnull = &decompressed_tuple->tts_isnull[attr]; decompress_scalar_column(column_values, value, isnull); break; } case COUNT_COLUMN: { bool isnull; Datum value = slot_getattr(compressed_slot, column_description->compressed_scan_attno, &isnull); /* count column should never be NULL */ Assert(!isnull); int count_value = DatumGetInt32(value); if (count_value <= 0) { ereport(ERROR, (errmsg("the compressed data is corrupt: got a segment with length %d", count_value))); } Assert(batch_state->total_batch_rows == 0); CheckCompressedData(count_value <= UINT16_MAX); batch_state->total_batch_rows = count_value; break; } case SEQUENCE_NUM_COLUMN: /* * nothing to do here for sequence number * we only needed this for sorting in node below */ break; } } CompressedBatchVectorQualState cbvqstate = { .vqstate = { .vectorized_quals_constified = dcontext->vectorized_quals_constified, .num_results = batch_state->total_batch_rows, .per_vector_mcxt = batch_state->per_batch_context, .slot = compressed_slot, .get_arrow_array = compressed_batch_get_arrow_array, }, .batch_state = batch_state, .dcontext = dcontext, }; VectorQualState *vqstate = &cbvqstate.vqstate; BatchQualSummary vector_qual_summary = vqstate->vectorized_quals_constified != NIL ? vector_qual_compute(vqstate) : AllRowsPass; batch_state->vector_qual_result = vqstate->vector_qual_result; if (vector_qual_summary == NoRowsPass && !dcontext->batch_sorted_merge) { /* * The entire batch doesn't pass the vectorized quals, so we might be * able to avoid reading and decompressing other columns. Scroll it to * the end. * Note that this optimization can't work with "batch sorted merge", * because the latter always has to read the first row of the batch for * its sorting needs, so it always has to read and decompress all * columns. This can be improved by only decompressing the columns * needed for sorting. */ compressed_batch_discard_tuples(batch_state); InstrCountTuples2(dcontext->ps, 1); InstrCountFiltered1(dcontext->ps, batch_state->total_batch_rows); } else { /* * We have some rows in the batch that pass the vectorized filters, so * we have to decompress the rest of the compressed columns. */ const int num_data_columns = dcontext->num_data_columns; for (int i = 0; i < num_data_columns; i++) { CompressedColumnValues *column_values = &batch_state->compressed_columns[i]; if (column_values->decompression_type == DT_Invalid) { decompress_column(dcontext, batch_state, compressed_slot, i); Assert(column_values->decompression_type != DT_Invalid); } } /* * If all rows pass, no need to test the vector qual for each row. This * is a common case for time range conditions. */ if (vector_qual_summary == AllRowsPass) { batch_state->vector_qual_result = NULL; vqstate->vector_qual_result = NULL; } } } /* * Construct the next tuple in the decompressed scan slot. * Doesn't check the quals. */ static void make_next_tuple(DecompressBatchState *batch_state, uint16 arrow_row, int num_data_columns) { TupleTableSlot *decompressed_scan_slot = &batch_state->decompressed_scan_slot_data.base; Assert(batch_state->total_batch_rows > 0); Assert(batch_state->next_batch_row < batch_state->total_batch_rows); compressed_columns_to_postgres_data(batch_state->compressed_columns, num_data_columns, arrow_row); /* * It's a virtual tuple slot, so no point in clearing/storing it * per each row, we can just update the values in-place. This saves * some CPU. We have to store it after ExecQual returns false (the tuple * didn't pass the filter), or after a new batch. The standard protocol * is to clear and set the tuple slot for each row, but our output tuple * slots are read-only, and the memory is owned by this node, so it is * safe to violate this protocol. */ Assert(TTS_IS_VIRTUAL(decompressed_scan_slot)); if (TTS_EMPTY(decompressed_scan_slot)) { ExecStoreVirtualTuple(decompressed_scan_slot); } } static bool vector_qual(DecompressBatchState *batch_state, uint16 arrow_row) { Assert(batch_state->total_batch_rows > 0); Assert(batch_state->next_batch_row < batch_state->total_batch_rows); if (!batch_state->vector_qual_result) { return true; } return arrow_row_is_valid(batch_state->vector_qual_result, arrow_row); } static bool postgres_qual(DecompressContext *dcontext, DecompressBatchState *batch_state) { TupleTableSlot *decompressed_scan_slot = &batch_state->decompressed_scan_slot_data.base; Assert(IsA(decompressed_scan_slot, TupleTableSlot)); Assert(!TupIsNull(decompressed_scan_slot)); if (dcontext->ps == NULL || dcontext->ps->qual == NULL) { return true; } /* Perform the usual Postgres selection. */ ExprContext *econtext = dcontext->ps->ps_ExprContext; econtext->ecxt_scantuple = decompressed_scan_slot; ResetExprContext(econtext); return ExecQual(dcontext->ps->qual, econtext); } /* * Decompress the next tuple from the batch indicated by batch state. The result is stored * in batch_state->decompressed_scan_slot. The slot will be empty if the batch * is entirely processed. */ void compressed_batch_advance(DecompressContext *dcontext, DecompressBatchState *batch_state) { Assert(batch_state->total_batch_rows > 0); TupleTableSlot *decompressed_scan_slot = &batch_state->decompressed_scan_slot_data.base; const bool reverse = dcontext->reverse; const int num_data_columns = dcontext->num_data_columns; for (; batch_state->next_batch_row < batch_state->total_batch_rows; batch_state->next_batch_row++) { const uint16 output_row = batch_state->next_batch_row; const uint16 arrow_row = unlikely(reverse) ? batch_state->total_batch_rows - 1 - output_row : output_row; if (!vector_qual(batch_state, arrow_row)) { /* * This row doesn't pass the vectorized quals. Advance the iterated * compressed columns if we have any. */ for (int i = 0; i < num_data_columns; i++) { CompressedColumnValues *column_values = &batch_state->compressed_columns[i]; if (column_values->decompression_type == DT_Iterator) { DecompressionIterator *iterator = (DecompressionIterator *) column_values->buffers[0]; iterator->try_next(iterator); } } InstrCountFiltered1(dcontext->ps, 1); continue; } make_next_tuple(batch_state, arrow_row, num_data_columns); if (!postgres_qual(dcontext, batch_state)) { /* * The tuple didn't pass the qual, fetch the next one in the next * iteration. */ InstrCountFiltered1(dcontext->ps, 1); continue; } /* The tuple passed the qual. */ batch_state->next_batch_row++; return; } /* * Reached end of batch. Check that the columns that we're decompressing * row-by-row have also ended. */ Assert(batch_state->next_batch_row == batch_state->total_batch_rows); for (int i = 0; i < num_data_columns; i++) { CompressedColumnValues *column_values = &batch_state->compressed_columns[i]; if (column_values->decompression_type == DT_Iterator) { DecompressionIterator *iterator = (DecompressionIterator *) column_values->buffers[0]; DecompressResult result = iterator->try_next(iterator); if (!result.is_done) { elog(ERROR, "compressed column out of sync with batch counter"); } } } /* Clear old slot state */ ExecClearTuple(decompressed_scan_slot); } /* * Before loading the first matching tuple from the batch, also save the very * first one into the given slot, even if it doesn't pass the quals. This is * needed for batch sorted merge. */ void compressed_batch_save_first_tuple(DecompressContext *dcontext, DecompressBatchState *batch_state, TupleTableSlot *first_tuple_slot) { Assert(batch_state->next_batch_row == 0); Assert(batch_state->total_batch_rows > 0); Assert(TupIsNull(compressed_batch_current_tuple(batch_state))); /* * Check that we have decompressed all columns even if the vector quals * didn't pass for the entire batch. We need them because we're asked * to save the first tuple. This doesn't actually happen yet, because the * vectorized decompression is disabled with sorted merge. */ #ifdef USE_ASSERT_CHECKING const int num_data_columns = dcontext->num_data_columns; for (int i = 0; i < num_data_columns; i++) { CompressedColumnValues *column_values = &batch_state->compressed_columns[i]; Assert(column_values->decompression_type != DT_Invalid); } #endif /* Make the first tuple and save it. */ Assert(batch_state->next_batch_row == 0); const uint16 arrow_row = dcontext->reverse ? batch_state->total_batch_rows - 1 : 0; make_next_tuple(batch_state, arrow_row, dcontext->num_data_columns); ExecCopySlot(first_tuple_slot, &batch_state->decompressed_scan_slot_data.base); /* * Check the quals and advance, so that the batch is in the correct state * for the subsequent calls (matching tuple is in decompressed scan slot). */ const bool qual_passed = vector_qual(batch_state, arrow_row) && postgres_qual(dcontext, batch_state); batch_state->next_batch_row++; if (!qual_passed) { InstrCountFiltered1(dcontext->ps, 1); compressed_batch_advance(dcontext, batch_state); } } /* * Frees all resources used by the compressed batch. * * If the batch is intended to be reused, use compressed_batch_discard_tuples() * instead. */ void compressed_batch_destroy(DecompressBatchState *batch_state) { Assert(batch_state != NULL); if (batch_state->per_batch_context != NULL) { MemoryContextDelete(batch_state->per_batch_context); batch_state->per_batch_context = NULL; } if (batch_state->decompressed_scan_slot_data.base.tts_values != NULL) { /* * Can be separately NULL in the current simplified prototype for * vectorized aggregation, but ideally it should change together with * per-batch context. */ pfree(batch_state->decompressed_scan_slot_data.base.tts_values); batch_state->decompressed_scan_slot_data.base.tts_values = NULL; } } ================================================ FILE: tsl/src/nodes/columnar_scan/compressed_batch.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include "compression/compression.h" #include "nodes/columnar_scan/decompress_context.h" #include "nodes/columnar_scan/vector_quals.h" #include <executor/tuptable.h> typedef struct ArrowArray ArrowArray; /* How to obtain the decompressed datum for individual row. */ typedef enum { /* * The decompressed value is a boolean and is stored as a vector of bits. */ DT_ArrowBits = -5, DT_ArrowTextDict = -4, DT_ArrowText = -3, /* * The decompressed value is already in the decompressed slot. This is used * for segmentby and compressed columns with default value in batch. */ DT_Scalar = -2, DT_Iterator = -1, DT_Invalid = 0, /* * Any positive number is also valid for the decompression type. It means * arrow array of a fixed-size by-value type, with size in bytes given by * the number. */ } DecompressionType; typedef struct CompressedColumnValues { /* How to obtain the decompressed datum for individual row. */ DecompressionType decompression_type; /* Where to put the decompressed datum. */ Datum *output_value; bool *output_isnull; /* * The flattened source buffers for getting the decompressed datum. * Depending on decompression type, they are as follows: * scalar: isnull, value * iterator: iterator * arrow fixed: validity, value * arrow text: validity, uint32* offsets, void* bodies * arrow dict text: validity, uint32* dict offsets, void* dict bodies, int16* indices */ const void *restrict buffers[4]; /* * The source arrow array, if any. We don't use it for building the * individual rows, and use the flattened buffers instead to lessen the * amount of indirections. However, it is used for vectorized filters. */ ArrowArray *arrow; } CompressedColumnValues; /* * All the information needed to decompress a batch. */ typedef struct DecompressBatchState { /* * The slot for the decompressed tuple. * * We embed it into the batch state as the first member (data inheritance), * so that it's easier to pass out to parent nodes, while following the usual * Postgres interface of passing the tuple table slots. * We use &batch_state->decompressed_scan_slot_data.base everywhere where we * need the TupleTableSlot*, and some parent nodes can cast this pointer to * DecompressBatchState* to use our custom interfaces. * * The slot itself follows the TTSVirtualOps tuple slot protocol, because Postgres * expression executor has special fast path for virtual tuples, and we don't * really need the custom tuple slot protocol for anything. One potential use * case for it would be late decompression by implementing custom slot_getattr(). * It was actually implemented and didn't show any benefits in the preliminary * testing, compared to what we already achieve with lazy decompression after * vectorized filters. One reason is that the Postgres expression compiler * can be eager in requesting materialization. For example, it would call * slot_getattr up to the last attribute used by every filter in a qualifier, * before running any qualifiers. This might be possible to configure, but * the area needs more research. * * See the PR #6628 for context. */ VirtualTupleTableSlot decompressed_scan_slot_data; uint16 total_batch_rows; uint16 next_batch_row; MemoryContext per_batch_context; /* * Arrow-style bitmap that says whether the vector quals passed for a given * row. Indexed same as arrow arrays, w/o accounting for the reverse scan * direction. Initialized to all ones, i.e. all rows pass. */ const uint64 *restrict vector_qual_result; /* * This follows DecompressContext.compressed_chunk_columns, but does not * include the trailing metadata columns, but only the leading data columns. * These columns are compressed and segmentby columns, their total number is * given by DecompressContext.num_data_columns. */ CompressedColumnValues compressed_columns[FLEXIBLE_ARRAY_MEMBER]; } DecompressBatchState; extern void compressed_batch_set_compressed_tuple(DecompressContext *dcontext, DecompressBatchState *batch_state, TupleTableSlot *compressed_slot); extern void compressed_batch_advance(DecompressContext *dcontext, DecompressBatchState *batch_state); extern void compressed_batch_save_first_tuple(DecompressContext *dcontext, DecompressBatchState *batch_state, TupleTableSlot *first_tuple_slot); /* * Initialize the batch memory context and bulk decompression context. * * We use Generation context here because the AllocSet has a hardcoded threshold * of 8kB per allocation, after which it allocates directly through malloc. We * want to make the blocks as big as possible, but below the malloc's mmap * threshold. For small queries, these contexts are basically single-shot and * the page faults after an mmap slow them down significantly. The threshold * should be 128 kiB according to the docs, but I'm seeing 64 kiB in testing. * * If bulk decompression is not used, use the default size for batch context. * This reduces memory usage and improves performance with batch sorted merge. */ #define create_bulk_decompression_mctx(parent_mctx) \ GenerationContextCreate(parent_mctx, \ "DecompressBatchState bulk decompression", \ 0, \ 64 * 1024, \ 64 * 1024); #define create_per_batch_mctx(dcontext) \ GenerationContextCreate(CurrentMemoryContext, \ "DecompressBatchState per-batch", \ 0, \ dcontext->enable_bulk_decompression ? 64 * 1024 : 8 * 1024, \ dcontext->enable_bulk_decompression ? 64 * 1024 : 8 * 1024); extern void compressed_batch_destroy(DecompressBatchState *batch_state); extern void compressed_batch_discard_tuples(DecompressBatchState *batch_state); /* * Returns the current decompressed tuple in the compressed batch. */ inline static TupleTableSlot * compressed_batch_current_tuple(DecompressBatchState *batch_state) { if (IsA(&batch_state->decompressed_scan_slot_data, Invalid)) { /* * For convenience, we want a zero-initialized batch to be a valid * "empty" state, but unfortunately a zero-initialized TupleTableSlotData * is not a valid tuple slot, so here we have to work around this mismatch. */ Assert(batch_state->decompressed_scan_slot_data.base.tts_ops == NULL); Assert(batch_state->per_batch_context == NULL); return NULL; } Assert(batch_state->per_batch_context != NULL); return &batch_state->decompressed_scan_slot_data.base; } /* * VectorQualState for a compressed batch used to pass * ColumnarScan-specific data to vector qual functions that are shared * across scan nodes. */ typedef struct CompressedBatchVectorQualState { VectorQualState vqstate; DecompressBatchState *batch_state; DecompressContext *dcontext; } CompressedBatchVectorQualState; const ArrowArray *compressed_batch_get_arrow_array(VectorQualState *vqstate, Expr *expr, bool *is_default_value); int get_max_varlena_bytes(ArrowArray *text_array); inline static void store_text_datum(CompressedColumnValues *column_values, int arrow_row) { const uint32 start = ((uint32 *) column_values->buffers[1])[arrow_row]; const int32 value_bytes = ((uint32 *) column_values->buffers[1])[arrow_row + 1] - start; Assert(value_bytes >= 0); const int total_bytes = value_bytes + VARHDRSZ; Assert(DatumGetPointer(*column_values->output_value) != NULL); SET_VARSIZE(*column_values->output_value, total_bytes); memcpy(VARDATA(*column_values->output_value), &((uint8 *) column_values->buffers[2])[start], value_bytes); } static pg_attribute_always_inline void compressed_columns_to_postgres_data(CompressedColumnValues *columns, int num_data_columns, uint16 arrow_row) { for (int i = 0; i < num_data_columns; i++) { CompressedColumnValues *column_values = &columns[i]; switch ((int) column_values->decompression_type) { case DT_Iterator: { DecompressionIterator *iterator = (DecompressionIterator *) column_values->buffers[0]; DecompressResult result = iterator->try_next(iterator); if (result.is_done) { elog(ERROR, "compressed column out of sync with batch counter"); } *column_values->output_isnull = result.is_null; *column_values->output_value = result.val; break; } #ifndef USE_FLOAT8_BYVAL case 8: #endif case 16: { /* * Fixed-width by-reference type that doesn't fit into a Datum. * For now this only happens for 8-byte types on 32-bit systems, * but eventually we could also use it for bigger by-value types * such as UUID. */ const uint8 value_bytes = column_values->decompression_type; const char *src = column_values->buffers[1]; *column_values->output_value = PointerGetDatum(&src[value_bytes * arrow_row]); *column_values->output_isnull = !arrow_row_is_valid(column_values->buffers[0], arrow_row); break; } case DT_ArrowBits: { /* * The DT_ArrowBits type is a special case, because the value is * stored as an Array of bits. */ *column_values->output_value = BoolGetDatum(arrow_row_is_valid(column_values->buffers[1], arrow_row)); *column_values->output_isnull = !arrow_row_is_valid(column_values->buffers[0], arrow_row); break; } case 2: case 4: #ifdef USE_FLOAT8_BYVAL case 8: #endif { /* * Fixed-width by-value type that fits into a Datum. * * The conversion of Datum to more narrow types will truncate * the higher bytes, so we don't care if we read some garbage * into them, and can always read 8 bytes. These are unaligned * reads, so technically we have to do memcpy. */ const uint8 value_bytes = column_values->decompression_type; Assert(value_bytes <= SIZEOF_DATUM); const char *src = column_values->buffers[1]; memcpy(column_values->output_value, &src[value_bytes * arrow_row], SIZEOF_DATUM); *column_values->output_isnull = !arrow_row_is_valid(column_values->buffers[0], arrow_row); break; } case DT_ArrowText: { store_text_datum(column_values, arrow_row); *column_values->output_isnull = !arrow_row_is_valid(column_values->buffers[0], arrow_row); break; } case DT_ArrowTextDict: { const int16 index = ((int16 *) column_values->buffers[3])[arrow_row]; store_text_datum(column_values, index); *column_values->output_isnull = !arrow_row_is_valid(column_values->buffers[0], arrow_row); break; } default: { /* A compressed column with default value, do nothing. */ Assert(column_values->decompression_type == DT_Scalar); } } } } ================================================ FILE: tsl/src/nodes/columnar_scan/decompress_context.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #ifndef TIMESCALEDB_DECOMPRESS_CONTEXT_H #define TIMESCALEDB_DECOMPRESS_CONTEXT_H #include <postgres.h> #include <access/attnum.h> #include <executor/tuptable.h> #include <nodes/execnodes.h> #include <nodes/pg_list.h> #include "batch_array.h" #include "detoaster.h" typedef enum CompressionColumnType { SEGMENTBY_COLUMN, COMPRESSED_COLUMN, COUNT_COLUMN, SEQUENCE_NUM_COLUMN, } CompressionColumnType; typedef struct CompressionColumnDescription { CompressionColumnType type; Oid typid; int16 value_bytes; bool by_value; /* * Attno of the decompressed column in the scan tuple of ColumnarScan node. * Negative values are special columns that do not have a representation in * the decompressed chunk, but are still used for decompression. The `type` * field is set accordingly for these columns. */ AttrNumber custom_scan_attno; /* * Attno of this column in the uncompressed chunks. We use it to fetch the * default value from the uncompressed chunk tuple descriptor. */ AttrNumber uncompressed_chunk_attno; /* * Attno of the compressed column in the input compressed chunk scan. */ AttrNumber compressed_scan_attno; bool bulk_decompression_supported; } CompressionColumnDescription; typedef struct DecompressContext { /* * Note that this array contains only those columns that are decompressed * (output_attno != 0), and the order is different from the compressed chunk * tuple order: first go the actual data columns, and after that the metadata * columns. */ CompressionColumnDescription *compressed_chunk_columns; /* * This includes all decompressed columns (output_attno != 0), including the * metadata columns. */ int num_columns_with_metadata; /* This excludes the metadata columns. */ int num_data_columns; List *vectorized_quals_constified; bool reverse; bool batch_sorted_merge; /* Batch sorted merge optimization enabled. */ bool enable_bulk_decompression; /* * Scratch space for bulk decompression which might need a lot of temporary * data. */ MemoryContext bulk_decompression_context; TupleTableSlot *custom_scan_slot; /* * The scan tuple descriptor might be different from the uncompressed chunk * one, and it doesn't have the default column values in that case, so we * have to fetch the default values from the uncompressed chunk tuple * descriptor which we store here. */ TupleDesc uncompressed_chunk_tdesc; PlanState *ps; /* Set for filtering and instrumentation */ Detoaster detoaster; int32 chunk_status; } DecompressContext; #endif /* TIMESCALEDB_DECOMPRESS_CONTEXT_H */ ================================================ FILE: tsl/src/nodes/columnar_scan/detoaster.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include "detoaster.h" #include <access/detoast.h> #include <access/genam.h> #include <access/heaptoast.h> #include <access/relscan.h> #include <access/skey.h> #include <access/stratnum.h> #include <access/table.h> #include <access/tableam.h> #include <access/toast_internals.h> #include <utils/expandeddatum.h> #include <utils/fmgroids.h> #include <utils/rel.h> #include <utils/relcache.h> #include <compat/compat.h> #include "debug_assert.h" #include <compression/compression.h> /* We redefine this postgres macro to fix a warning about signed integer comparison. */ #define TS_VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \ (((int32) VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer)) < (toast_pointer).va_rawsize - VARHDRSZ) /* * Fetch a TOAST slice from a heap table. * * This function is a modified copy of heap_fetch_toast_slice(). The difference * is that it holds the open toast relation, index and other intermediate data * for detoasting in the Detoaster struct, to allow them to be reused over many * input tuples. */ static void ts_fetch_toast(Detoaster *detoaster, struct varatt_external *toast_pointer, struct varlena *result) { const Oid valueid = toast_pointer->va_valueid; /* * Open the toast relation and its indexes */ if (detoaster->toastrel == NULL) { MemoryContext old_mctx = MemoryContextSwitchTo(detoaster->mctx); detoaster->toastrel = table_open(toast_pointer->va_toastrelid, AccessShareLock); int num_indexes; Relation *toastidxs; /* Look for the valid index of toast relation */ const int validIndex = toast_open_indexes(detoaster->toastrel, AccessShareLock, &toastidxs, &num_indexes); detoaster->index = toastidxs[validIndex]; for (int i = 0; i < num_indexes; i++) { if (i != validIndex) { index_close(toastidxs[i], AccessShareLock); } } /* Set up a scan key to fetch from the index. */ ScanKeyInit(&detoaster->toastkey, (AttrNumber) 1, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(valueid)); /* Prepare for scan */ #if PG18_GE detoaster->SnapshotToast = *get_toast_snapshot(); #else init_toast_snapshot(&detoaster->SnapshotToast); #endif detoaster->toastscan = systable_beginscan_ordered(detoaster->toastrel, detoaster->index, &detoaster->SnapshotToast, 1, &detoaster->toastkey); MemoryContextSwitchTo(old_mctx); } else { Ensure(detoaster->toastrel->rd_id == toast_pointer->va_toastrelid, "unexpected toast pointer relid %d, expected %d", toast_pointer->va_toastrelid, detoaster->toastrel->rd_id); detoaster->toastkey.sk_argument = ObjectIdGetDatum(valueid); index_rescan(detoaster->toastscan->iscan, &detoaster->toastkey, 1, NULL, 0); } TupleDesc toasttupDesc = detoaster->toastrel->rd_att; /////////////////////////////////////////////// /* * Read the chunks by index * * The index is on (valueid, chunkidx) so they will come in order */ const int32 attrsize = VARATT_EXTERNAL_GET_EXTSIZE(*toast_pointer); const int32 totalchunks = ((attrsize - 1) / TOAST_MAX_CHUNK_SIZE) + 1; const int startchunk = 0; const int endchunk = (attrsize - 1) / TOAST_MAX_CHUNK_SIZE; Assert(endchunk <= totalchunks); HeapTuple ttup; int32 expectedchunk = startchunk; while ((ttup = systable_getnext_ordered(detoaster->toastscan, ForwardScanDirection)) != NULL) { int32 curchunk; Pointer chunk; bool isnull; char *chunkdata; int32 chunksize; int32 expected_size; int32 chcpystrt; int32 chcpyend; /* * Have a chunk, extract the sequence number and the data */ curchunk = DatumGetInt32(fastgetattr(ttup, 2, toasttupDesc, &isnull)); Assert(!isnull); chunk = DatumGetPointer(fastgetattr(ttup, 3, toasttupDesc, &isnull)); Assert(!isnull); if (!VARATT_IS_EXTENDED(chunk)) { chunksize = VARSIZE(chunk) - VARHDRSZ; chunkdata = VARDATA(chunk); } else if (VARATT_IS_SHORT(chunk)) { /* could happen due to heap_form_tuple doing its thing */ chunksize = VARSIZE_SHORT(chunk) - VARHDRSZ_SHORT; chunkdata = VARDATA_SHORT(chunk); } else { /* should never happen */ elog(ERROR, "found toasted toast chunk for toast value %u in %s", valueid, RelationGetRelationName(detoaster->toastrel)); chunksize = 0; /* keep compiler quiet */ chunkdata = NULL; } /* * Some checks on the data we've found */ if (curchunk != expectedchunk) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg_internal("unexpected chunk number %d (expected %d) for toast value %u " "in %s", curchunk, expectedchunk, valueid, RelationGetRelationName(detoaster->toastrel)))); if (curchunk > endchunk) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg_internal("unexpected chunk number %d (out of range %d..%d) for toast " "value %u in %s", curchunk, startchunk, endchunk, valueid, RelationGetRelationName(detoaster->toastrel)))); expected_size = curchunk < totalchunks - 1 ? TOAST_MAX_CHUNK_SIZE : attrsize - ((totalchunks - 1) * TOAST_MAX_CHUNK_SIZE); if (chunksize != expected_size) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg_internal("unexpected chunk size %d (expected %d) in chunk %d of %d for " "toast value %u in %s", chunksize, expected_size, curchunk, totalchunks, valueid, RelationGetRelationName(detoaster->toastrel)))); /* * Copy the data into proper place in our result */ chcpystrt = 0; chcpyend = chunksize - 1; if (curchunk == startchunk) chcpystrt = 0; if (curchunk == endchunk) chcpyend = (attrsize - 1) % TOAST_MAX_CHUNK_SIZE; memcpy(VARDATA(result) + (curchunk * TOAST_MAX_CHUNK_SIZE) + chcpystrt, chunkdata + chcpystrt, (chcpyend - chcpystrt) + 1); expectedchunk++; } /* * Final checks that we successfully fetched the datum */ if (expectedchunk != (endchunk + 1)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg_internal("missing chunk number %d for toast value %u in %s", expectedchunk, valueid, RelationGetRelationName(detoaster->toastrel)))); } /* * The memory context is used to store intermediate data, and is supposed to * live over the calls to detoaster_detoast_attr_copy(). * That function itself can be called in a short-lived memory context. */ void detoaster_init(Detoaster *detoaster, MemoryContext mctx) { detoaster->toastrel = NULL; detoaster->mctx = mctx; } void detoaster_close(Detoaster *detoaster) { /* Close toast table */ if (detoaster->toastrel != NULL) { systable_endscan_ordered(detoaster->toastscan); table_close(detoaster->toastrel, AccessShareLock); index_close(detoaster->index, AccessShareLock); detoaster->toastrel = NULL; detoaster->index = NULL; } } /* * Copy of Postgres' toast_fetch_datum(): Reconstruct an in memory Datum from * the chunks saved in the toast relation. */ static struct varlena * ts_toast_fetch_datum(struct varlena *attr, Detoaster *detoaster, MemoryContext dest_mctx) { struct varlena *result; struct varatt_external toast_pointer; int32 attrsize; if (!VARATT_IS_EXTERNAL_ONDISK(attr)) elog(ERROR, "toast_fetch_datum shouldn't be called for non-ondisk datums"); /* Must copy to access aligned fields */ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); attrsize = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer); result = (struct varlena *) MemoryContextAlloc(dest_mctx, attrsize + VARHDRSZ); if (TS_VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)) SET_VARSIZE_COMPRESSED(result, attrsize + VARHDRSZ); else SET_VARSIZE(result, attrsize + VARHDRSZ); if (attrsize == 0) return result; /* Probably shouldn't happen, but just in * case. */ /* Fetch all chunks */ ts_fetch_toast(detoaster, &toast_pointer, result); return result; } #include <access/toast_compression.h> static struct varlena * ts_toast_decompress_datum(struct varlena *attr) { ToastCompressionId cmid; Assert(VARATT_IS_COMPRESSED(attr)); /* * Fetch the compression method id stored in the compression header and * decompress the data using the appropriate decompression routine. */ cmid = TOAST_COMPRESS_METHOD(attr); switch (cmid) { case TOAST_PGLZ_COMPRESSION_ID: return pglz_decompress_datum(attr); case TOAST_LZ4_COMPRESSION_ID: return lz4_decompress_datum(attr); default: elog(ERROR, "invalid compression method id %d", cmid); return NULL; /* keep compiler quiet */ } } /* * Modification of Postgres' detoast_attr() where we use the stateful Detoaster * and skip some cases that don't occur for the toasted compressed data. Even if * the data is inline and no detoasting is needed, copies it into the destination * memory context. */ struct varlena * detoaster_detoast_attr_copy(struct varlena *attr, Detoaster *detoaster, MemoryContext dest_mctx) { if (!VARATT_IS_EXTENDED(attr)) { /* * This case is unlikely because the compressed data is almost always * toasted and not inline, but we still have to copy the data into the * destination memory context. The source compressed tuple may have * independent unknown lifetime. */ Size len = VARSIZE(attr); struct varlena *result = (struct varlena *) MemoryContextAlloc(dest_mctx, len); memcpy(result, attr, len); return result; } if (VARATT_IS_EXTERNAL_ONDISK(attr)) { /* * This is an externally stored datum --- fetch it back from there. */ attr = ts_toast_fetch_datum(attr, detoaster, dest_mctx); /* If it's compressed, decompress it */ if (VARATT_IS_COMPRESSED(attr)) { struct varlena *tmp = attr; MemoryContext old_context = MemoryContextSwitchTo(dest_mctx); attr = ts_toast_decompress_datum(tmp); MemoryContextSwitchTo(old_context); pfree(tmp); } return attr; } /* * Can't get indirect TOAST here (out-of-line Datum that's stored in memory), * because we're reading from the compressed chunk table. */ Ensure(!VARATT_IS_EXTERNAL_INDIRECT(attr), "got indirect TOAST for compressed data"); /* * Compressed data doesn't have an expanded representation. */ Ensure(!VARATT_IS_EXTERNAL_EXPANDED(attr), "got expanded TOAST for compressed data"); if (VARATT_IS_COMPRESSED(attr)) { /* * This is a compressed value stored inline in the main tuple. It rarely * occurs in practice, because we set a low toast_tuple_target = 128 * for the compressed chunks, but is still technically possible. * * Note that the attr comes from the compressed tuple slot here, so we * don't have to free it unlike the above case of decompression. */ MemoryContext old_context = MemoryContextSwitchTo(dest_mctx); attr = ts_toast_decompress_datum(attr); MemoryContextSwitchTo(old_context); return attr; } /* * The only option left is a short-header varlena --- convert to 4-byte * header format. */ Ensure(VARATT_IS_SHORT(attr), "got unexpected TOAST type for compressed data"); /* * Check that the size of datum is not less than the size of header, which * could lead to data_size of UINT64_MAX. This is possible in case of * TOAST data corruption. Postgres doesn't specifically check for this, * because in any case it will be detected by the subsequent palloc call, * but we do it to silence the Coverity warning. */ CheckCompressedData(VARSIZE_SHORT(attr) >= VARHDRSZ_SHORT); Size data_size = VARSIZE_SHORT(attr) - VARHDRSZ_SHORT; Size new_size = data_size + VARHDRSZ; struct varlena *new_attr; new_attr = (struct varlena *) MemoryContextAlloc(dest_mctx, new_size); SET_VARSIZE(new_attr, new_size); memcpy(VARDATA(new_attr), VARDATA_SHORT(attr), data_size); attr = new_attr; return attr; } ================================================ FILE: tsl/src/nodes/columnar_scan/detoaster.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <access/genam.h> #include <access/relscan.h> #include <access/skey.h> #include <utils/snapshot.h> typedef struct RelationData *Relation; typedef struct Detoaster { MemoryContext mctx; Relation toastrel; Relation index; SnapshotData SnapshotToast; ScanKeyData toastkey; SysScanDesc toastscan; } Detoaster; void detoaster_init(Detoaster *detoaster, MemoryContext mctx); void detoaster_close(Detoaster *detoaster); struct varlena *detoaster_detoast_attr_copy(struct varlena *attr, Detoaster *detoaster, MemoryContext dest_mctx); ================================================ FILE: tsl/src/nodes/columnar_scan/exec.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/sysattr.h> #include <executor/executor.h> #include <miscadmin.h> #include <nodes/bitmapset.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <optimizer/optimizer.h> #include <parser/parsetree.h> #include <rewrite/rewriteManip.h> #include <tcop/tcopprot.h> #include <utils/datum.h> #include <utils/memutils.h> #include <utils/typcache.h> #include "compat/compat.h" #include "compression/arrow_c_data_interface.h" #include "compression/compression.h" #include "guc.h" #include "import/ts_explain.h" #include "nodes/columnar_scan/batch_array.h" #include "nodes/columnar_scan/batch_queue.h" #include "nodes/columnar_scan/columnar_scan.h" #include "nodes/columnar_scan/compressed_batch.h" #include "nodes/columnar_scan/exec.h" #include "nodes/columnar_scan/planner.h" #include "ts_catalog/array_utils.h" #if PG18_GE #include <commands/explain_format.h> #include <commands/explain_state.h> #endif static void columnar_scan_begin(CustomScanState *node, EState *estate, int eflags); static void columnar_scan_end(CustomScanState *node); static void columnar_scan_rescan(CustomScanState *node); static void columnar_scan_explain(CustomScanState *node, List *ancestors, ExplainState *es); static CustomExecMethods columnar_scan_state_methods = { .BeginCustomScan = columnar_scan_begin, .ExecCustomScan = NULL, /* To be determined later. */ .EndCustomScan = columnar_scan_end, .ReScanCustomScan = columnar_scan_rescan, .ExplainCustomScan = columnar_scan_explain, }; /* * Build the sortkeys data structure from the list structure in the * custom_private field of the custom scan. This sort info is used to sort * binary heap used for batch sorted merge. */ Node * columnar_scan_state_create(CustomScan *cscan) { ColumnarScanState *chunk_state; chunk_state = (ColumnarScanState *) newNode(sizeof(ColumnarScanState), T_CustomScanState); chunk_state->exec_methods = columnar_scan_state_methods; chunk_state->csstate.methods = &chunk_state->exec_methods; Assert(IsA(cscan->custom_private, List)); Assert(list_length(cscan->custom_private) == DCP_Count); List *settings = list_nth(cscan->custom_private, DCP_Settings); chunk_state->decompression_map = list_nth(cscan->custom_private, DCP_DecompressionMap); chunk_state->is_segmentby_column = list_nth(cscan->custom_private, DCP_IsSegmentbyColumn); chunk_state->bulk_decompression_column = list_nth(cscan->custom_private, DCP_BulkDecompressionColumn); chunk_state->sortinfo = list_nth(cscan->custom_private, DCP_SortInfo); chunk_state->custom_scan_tlist = cscan->custom_scan_tlist; Assert(IsA(settings, IntList)); Assert(list_length(settings) == DCS_Count); chunk_state->hypertable_id = list_nth_int(settings, DCS_HypertableId); chunk_state->chunk_relid = list_nth_int(settings, DCS_ChunkRelid); chunk_state->decompress_context.reverse = list_nth_int(settings, DCS_Reverse); chunk_state->decompress_context.batch_sorted_merge = list_nth_int(settings, DCS_BatchSortedMerge); chunk_state->decompress_context.chunk_status = list_nth_int(settings, DCS_ChunkStatus); chunk_state->decompress_context.enable_bulk_decompression = list_nth_int(settings, DCS_EnableBulkDecompression); chunk_state->has_row_marks = list_nth_int(settings, DCS_HasRowMarks); Assert(IsA(cscan->custom_exprs, List)); Assert(list_length(cscan->custom_exprs) == 1); chunk_state->vectorized_quals_original = linitial(cscan->custom_exprs); Assert(list_length(chunk_state->decompression_map) == list_length(chunk_state->is_segmentby_column)); return (Node *) chunk_state; } typedef struct ConstifyTableOidContext { Index chunk_index; Oid chunk_relid; bool made_changes; } ConstifyTableOidContext; static Node * constify_tableoid_walker(Node *node, ConstifyTableOidContext *ctx) { if (node == NULL) return NULL; if (IsA(node, Var)) { Var *var = castNode(Var, node); if ((Index) var->varno != ctx->chunk_index) return node; if (var->varattno == TableOidAttributeNumber) { ctx->made_changes = true; return ( Node *) makeConst(OIDOID, -1, InvalidOid, 4, (Datum) ctx->chunk_relid, false, true); } /* * we doublecheck system columns here because projection will * segfault if any system columns get through */ if (var->varattno < SelfItemPointerAttributeNumber) ereport(ERROR, (errcode(ERRCODE_INVALID_COLUMN_REFERENCE), errmsg("transparent decompression only supports tableoid system column"))); return node; } return expression_tree_mutator(node, constify_tableoid_walker, (void *) ctx); } static List * constify_tableoid(List *node, Index chunk_index, Oid chunk_relid) { ConstifyTableOidContext ctx = { .chunk_index = chunk_index, .chunk_relid = chunk_relid, .made_changes = false, }; List *result = (List *) constify_tableoid_walker((Node *) node, &ctx); if (ctx.made_changes) { return result; } return node; } pg_attribute_always_inline static TupleTableSlot * columnar_scan_exec_impl(ColumnarScanState *chunk_state, const BatchQueueFunctions *funcs); static TupleTableSlot * columnar_scan_exec_fifo(CustomScanState *node) { ColumnarScanState *chunk_state = (ColumnarScanState *) node; Assert(!chunk_state->decompress_context.batch_sorted_merge); return columnar_scan_exec_impl(chunk_state, &BatchQueueFunctionsFifo); } static TupleTableSlot * columnar_scan_exec_heap(CustomScanState *node) { ColumnarScanState *chunk_state = (ColumnarScanState *) node; Assert(chunk_state->decompress_context.batch_sorted_merge); return columnar_scan_exec_impl(chunk_state, &BatchQueueFunctionsHeap); } /* * Complete initialization of the supplied CustomScanState. * * Standard fields have been initialized by ExecInitCustomScan, * but any private fields should be initialized here. */ static void columnar_scan_begin(CustomScanState *node, EState *estate, int eflags) { ColumnarScanState *chunk_state = (ColumnarScanState *) node; DecompressContext *dcontext = &chunk_state->decompress_context; CustomScan *cscan = castNode(CustomScan, node->ss.ps.plan); Plan *compressed_scan = linitial(cscan->custom_plans); Assert(list_length(cscan->custom_plans) == 1); PlanState *ps = &node->ss.ps; if (ps->ps_ProjInfo) { /* * if we are projecting we need to constify tableoid references here * because decompressed tuple are virtual tuples and don't have * system columns. * * We do the constify in executor because even after plan creation * our targetlist might still get modified by parent nodes pushing * down targetlist. */ List *tlist = ps->plan->targetlist; List *modified_tlist = constify_tableoid(tlist, cscan->scan.scanrelid, chunk_state->chunk_relid); if (modified_tlist != tlist) { ps->ps_ProjInfo = ExecBuildProjectionInfo(modified_tlist, ps->ps_ExprContext, ps->ps_ResultTupleSlot, ps, node->ss.ss_ScanTupleSlot->tts_tupleDescriptor); } } /* * Sort keys should only be present at the level of this node when batch * sorted merge is used. * In other cases of sort pushdown, sorting is performed by the underlying * compressed scan. */ Assert(dcontext->batch_sorted_merge == true || list_length(chunk_state->sortinfo) == 0); /* * Init the underlying compressed scan. */ node->custom_ps = lappend(node->custom_ps, ExecInitNode(compressed_scan, estate, eflags)); /* * Count the actual data columns we have to decompress, skipping the * metadata columns. We only need the metadata columns when initializing the * compressed batch, so they are not saved in the compressed batch itself, * it tracks only the data columns. We put the metadata columns to the end * of the array to have the same column indexes in compressed batch state * and in decompression context. */ int num_data_columns = 0; int num_columns_with_metadata = 0; ListCell *dest_cell; ListCell *is_segmentby_cell; forboth (dest_cell, chunk_state->decompression_map, is_segmentby_cell, chunk_state->is_segmentby_column) { AttrNumber output_attno = lfirst_int(dest_cell); if (output_attno == 0) { /* We are asked not to decompress this column, skip it. */ continue; } if (output_attno > 0) { /* * Not a metadata column. */ num_data_columns++; } num_columns_with_metadata++; } Assert(num_data_columns <= num_columns_with_metadata); dcontext->num_data_columns = num_data_columns; dcontext->num_columns_with_metadata = num_columns_with_metadata; dcontext->compressed_chunk_columns = palloc0(sizeof(CompressionColumnDescription) * num_columns_with_metadata); dcontext->custom_scan_slot = node->ss.ss_ScanTupleSlot; dcontext->uncompressed_chunk_tdesc = RelationGetDescr(node->ss.ss_currentRelation); dcontext->ps = &node->ss.ps; TupleDesc desc = dcontext->custom_scan_slot->tts_tupleDescriptor; /* * Compressed columns go in front, and the rest go to the back, so we have * separate indices for them. */ int current_compressed = 0; int current_not_compressed = num_data_columns; for (int compressed_index = 0; compressed_index < list_length(chunk_state->decompression_map); compressed_index++) { CompressionColumnDescription column = { .compressed_scan_attno = AttrOffsetGetAttrNumber(compressed_index), .custom_scan_attno = list_nth_int(chunk_state->decompression_map, compressed_index), .bulk_decompression_supported = list_nth_int(chunk_state->bulk_decompression_column, compressed_index) }; if (column.custom_scan_attno == 0) { /* We are asked not to decompress this column, skip it. */ continue; } if (column.custom_scan_attno > 0) { /* normal column that is also present in decompressed chunk */ Form_pg_attribute attribute = TupleDescAttr(desc, AttrNumberGetAttrOffset(column.custom_scan_attno)); column.typid = attribute->atttypid; get_typlenbyval(column.typid, &column.value_bytes, &column.by_value); if (list_nth_int(chunk_state->is_segmentby_column, compressed_index)) column.type = SEGMENTBY_COLUMN; else column.type = COMPRESSED_COLUMN; if (cscan->custom_scan_tlist == NIL) { column.uncompressed_chunk_attno = column.custom_scan_attno; } else { Var *var = castNode(Var, castNode(TargetEntry, list_nth(cscan->custom_scan_tlist, AttrNumberGetAttrOffset(column.custom_scan_attno))) ->expr); column.uncompressed_chunk_attno = var->varattno; } } else { /* metadata columns */ switch (column.custom_scan_attno) { case COLUMNAR_SCAN_COUNT_ID: column.type = COUNT_COLUMN; break; case COLUMNAR_SCAN_SEQUENCE_NUM_ID: column.type = SEQUENCE_NUM_COLUMN; break; default: elog(ERROR, "Invalid column attno \"%d\"", column.custom_scan_attno); break; } } if (column.custom_scan_attno > 0) { /* Data column. */ Assert(current_compressed < num_data_columns); dcontext->compressed_chunk_columns[current_compressed++] = column; } else { /* Metadata column. */ Assert(current_not_compressed < num_columns_with_metadata); dcontext->compressed_chunk_columns[current_not_compressed++] = column; } } Assert(current_compressed == num_data_columns); Assert(current_not_compressed == num_columns_with_metadata); /* * Choose which batch queue we are going to use: heap for batch sorted * merge, and one-element FIFO for normal decompression. */ if (dcontext->batch_sorted_merge) { chunk_state->batch_queue = batch_queue_heap_create(num_data_columns, chunk_state->sortinfo, dcontext->custom_scan_slot->tts_tupleDescriptor, &BatchQueueFunctionsHeap); chunk_state->exec_methods.ExecCustomScan = columnar_scan_exec_heap; } else { chunk_state->batch_queue = batch_queue_fifo_create(num_data_columns, &BatchQueueFunctionsFifo); chunk_state->exec_methods.ExecCustomScan = columnar_scan_exec_fifo; } if ((ts_guc_debug_require_batch_sorted_merge == DRO_Require || ts_guc_debug_require_batch_sorted_merge == DRO_Force) && !dcontext->batch_sorted_merge) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("debug: batch sorted merge is required but not used"))); } if (ts_guc_debug_require_batch_sorted_merge == DRO_Forbid && dcontext->batch_sorted_merge) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("debug: batch sorted merge is used when it is forbidden"))); } /* Constify stable expressions in vectorized predicates. */ PlannerGlobal glob = { .boundParams = node->ss.ps.state->es_param_list_info, }; PlannerInfo root = { .glob = &glob, }; ListCell *lc; foreach (lc, chunk_state->vectorized_quals_original) { Node *constified = estimate_expression_value(&root, (Node *) lfirst(lc)); dcontext->vectorized_quals_constified = lappend(dcontext->vectorized_quals_constified, constified); } detoaster_init(&dcontext->detoaster, CurrentMemoryContext); } /* * The exec function for the ColumnarScan node. It takes the explicit queue * functions pointer as an optimization, to allow these functions to be * inlined in the FIFO case. This is important because this is a part of a * relatively hot loop. */ pg_attribute_always_inline static TupleTableSlot * columnar_scan_exec_impl(ColumnarScanState *chunk_state, const BatchQueueFunctions *bqfuncs) { DecompressContext *dcontext = &chunk_state->decompress_context; BatchQueue *bq = chunk_state->batch_queue; Assert(bq->funcs == bqfuncs); bqfuncs->pop(bq, dcontext); while (bqfuncs->needs_next_batch(bq)) { TupleTableSlot *subslot = ExecProcNode(linitial(chunk_state->csstate.custom_ps)); if (TupIsNull(subslot)) { /* Won't have more compressed tuples. */ break; } bqfuncs->push_batch(bq, dcontext, subslot); } TupleTableSlot *result_slot = bqfuncs->top_tuple(bq); if (TupIsNull(result_slot)) { return NULL; } if (chunk_state->has_row_marks) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("locking compressed tuples is not supported"))); } if (chunk_state->csstate.ss.ps.ps_ProjInfo) { ExprContext *econtext = chunk_state->csstate.ss.ps.ps_ExprContext; econtext->ecxt_scantuple = result_slot; return ExecProject(chunk_state->csstate.ss.ps.ps_ProjInfo); } return result_slot; } static void columnar_scan_rescan(CustomScanState *node) { ColumnarScanState *chunk_state = (ColumnarScanState *) node; BatchQueue *bq = chunk_state->batch_queue; bq->funcs->reset(bq); if (node->ss.ps.chgParam != NULL) UpdateChangedParamSet(linitial(node->custom_ps), node->ss.ps.chgParam); ExecReScan(linitial(node->custom_ps)); } /* End the decompress operation and free the requested resources */ static void columnar_scan_end(CustomScanState *node) { ColumnarScanState *chunk_state = (ColumnarScanState *) node; BatchQueue *bq = chunk_state->batch_queue; bq->funcs->free(bq); ExecEndNode(linitial(node->custom_ps)); detoaster_close(&chunk_state->decompress_context.detoaster); } /* * Output additional information for EXPLAIN of a custom-scan plan node. */ static void columnar_scan_explain(CustomScanState *node, List *ancestors, ExplainState *es) { ColumnarScanState *chunk_state = (ColumnarScanState *) node; DecompressContext *dcontext = &chunk_state->decompress_context; ts_show_scan_qual(chunk_state->vectorized_quals_original, "Vectorized Filter", &node->ss.ps, ancestors, es); if (!node->ss.ps.plan->qual && chunk_state->vectorized_quals_original) { /* * The normal explain won't show this if there are no normal quals but * only the vectorized ones. */ ts_show_instrumentation_count("Rows Removed by Filter", 1, &node->ss.ps, es); } if (es->analyze && es->verbose && (node->ss.ps.instrument->ntuples2 > 0 || es->format != EXPLAIN_FORMAT_TEXT)) { ExplainPropertyFloat("Batches Removed by Filter", NULL, node->ss.ps.instrument->ntuples2, 0, es); } if (es->verbose || es->format != EXPLAIN_FORMAT_TEXT) { /* Display any statuses in addition to COMPRESSED */ if (dcontext->chunk_status > CHUNK_STATUS_COMPRESSED) { StringInfoData status_text; initStringInfo(&status_text); ArrayType *arr = DatumGetArrayTypeP(DirectFunctionCall1(ts_chunk_status_text, Int32GetDatum(dcontext->chunk_status - CHUNK_STATUS_COMPRESSED))); ts_array_append_stringinfo(arr, &status_text); pfree(arr); ExplainPropertyText("Chunk Status", status_text.data, es); } if (dcontext->batch_sorted_merge) { ExplainPropertyBool("Batch Sorted Merge", dcontext->batch_sorted_merge, es); } if (dcontext->reverse) { ExplainPropertyBool("Reverse", dcontext->reverse, es); } if (es->analyze) { ExplainPropertyBool("Bulk Decompression", chunk_state->decompress_context.enable_bulk_decompression, es); } } } ================================================ FILE: tsl/src/nodes/columnar_scan/exec.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include "batch_queue.h" #include "decompress_context.h" #include <nodes/extensible.h> #define COLUMNAR_SCAN_COUNT_ID -9 #define COLUMNAR_SCAN_SEQUENCE_NUM_ID -10 typedef struct ColumnarScanState { CustomScanState csstate; List *decompression_map; List *is_segmentby_column; List *bulk_decompression_column; List *custom_scan_tlist; bool has_row_marks; DecompressContext decompress_context; int hypertable_id; Oid chunk_relid; BatchQueue *batch_queue; CustomExecMethods exec_methods; List *sortinfo; /* * For some predicates, we have more efficient implementation that work on * the entire compressed batch in one go. They go to this list, and the rest * goes into the usual ss.ps.qual. Note that we constify stable functions * in these predicates at execution time, but have to keep the original * version for EXPLAIN. We also need special handling for quals that * evaluate to constant false, hence the flag. */ List *vectorized_quals_original; } ColumnarScanState; extern Node *columnar_scan_state_create(CustomScan *cscan); ================================================ FILE: tsl/src/nodes/columnar_scan/planner.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/sysattr.h> #include <catalog/pg_namespace.h> #include <catalog/pg_operator.h> #include <nodes/bitmapset.h> #include <nodes/extensible.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <nodes/nodes.h> #include <optimizer/cost.h> #include <optimizer/optimizer.h> #include <optimizer/paths.h> #include <optimizer/plancat.h> #include <optimizer/restrictinfo.h> #include <optimizer/tlist.h> #include <parser/parse_relation.h> #include <parser/parsetree.h> #include <utils/builtins.h> #include <utils/typcache.h> #include "compression/compression.h" #include "compression/create.h" #include "custom_type_cache.h" #include "guc.h" #include "import/list.h" #include "import/planner.h" #include "nodes/chunk_append/transform.h" #include "nodes/columnar_scan/columnar_scan.h" #include "nodes/columnar_scan/exec.h" #include "nodes/columnar_scan/planner.h" #include "nodes/columnar_scan/vector_quals.h" #include "nodes/vector_agg/exec.h" #include "ts_catalog/array_utils.h" #include "vector_predicates.h" static CustomScanMethods columnar_scan_plan_methods = { .CustomName = "ColumnarScan", .CreateCustomScanState = columnar_scan_state_create, }; /* Check if the provided plan is a ColumnarScanPlan */ bool ts_is_columnar_scan_plan(Plan *plan) { return IsA(plan, CustomScan) && castNode(CustomScan, plan)->methods == &columnar_scan_plan_methods; } void _columnar_scan_init(void) { TryRegisterCustomScanMethods(&columnar_scan_plan_methods); } static void check_for_system_columns(Bitmapset *attrs_used) { int bit = bms_next_member(attrs_used, -1); if (bit > 0 && bit + FirstLowInvalidHeapAttributeNumber < 0) { /* we support tableoid so skip that */ if (bit == TableOidAttributeNumber - FirstLowInvalidHeapAttributeNumber) bit = bms_next_member(attrs_used, bit); if (bit > 0 && bit + FirstLowInvalidHeapAttributeNumber < 0) ereport(ERROR, (errcode(ERRCODE_INVALID_COLUMN_REFERENCE), errmsg("transparent decompression only supports tableoid system column"))); } } typedef struct { bool bulk_decompression_possible; int custom_scan_attno; } UncompressedColumnInfo; typedef struct { /* Can be negative if it's a metadata column, zero if not decompressed. */ int uncompressed_chunk_attno; bool bulk_decompression_possible; bool is_segmentby; } CompressedColumnInfo; /* * Scratch space for mapping out the decompressed columns. */ typedef struct { PlannerInfo *root; ColumnarScanPath *decompress_path; Bitmapset *uncompressed_attrs_needed; /* * If we produce at least some columns that support bulk decompression. */ bool have_bulk_decompression_columns; /* * Maps the uncompressed chunk attno to the respective column compression * info. This lives only during planning so that we can understand on which * columns we can apply vectorized quals, and which uncompressed attno goes * to which custom scan attno (it's not the same if we're using a custom * scan targetlist). */ UncompressedColumnInfo *uncompressed_attno_info; /* * Maps the compressed chunk attno to the respective column compression info. */ CompressedColumnInfo *compressed_attno_info; /* * We might use a custom scan targetlist for ColumnarScan node if it * allows us to avoid projection. */ List *custom_scan_targetlist; /* * Next, we have basically the same data as the above, but expressed as * several Lists, to allow passing them through the custom plan settings. */ /* * decompression_map maps targetlist entries of the compressed scan to * custom scan attnos. Negative values are metadata columns in the compressed * scan that do not have a representation in the uncompressed chunk, but are * still used for decompression. */ List *decompression_map; /* * This Int list is parallel to the compressed scan targetlist, just like * the above one. The value is true if a given targetlist entry is a * segmentby column, false otherwise. Has the same length as the above list. * We have to use the parallel lists and not a list of structs, because the * Plans have to be copyable by the Postgres _copy functions, and we can't * do that for a custom struct. */ List *is_segmentby_column; /* * Same structure as above, says whether we support bulk decompression for this * column. */ List *bulk_decompression_column; } DecompressionMapContext; static bool * build_vector_attrs_array(const UncompressedColumnInfo *colinfo, const CompressionInfo *info) { const AttrNumber arrlen = info->chunk_rel->max_attr + 1; bool *vector_attrs = palloc(sizeof(bool) * arrlen); for (AttrNumber attno = 0; attno < arrlen; attno++) { vector_attrs[attno] = colinfo[attno].bulk_decompression_possible; } return vector_attrs; } /* * Try to make the custom scan targetlist that follows the order of the * pathtarget. This would allow us to avoid a projection from scan tuple to * output tuple. * Returns NIL if it's not possible, e.g. if there are whole-row variables or * variables that are used for quals but not for output. */ static List * follow_uncompressed_output_tlist(const DecompressionMapContext *context) { List *result = NIL; Bitmapset *uncompressed_attrs_found = NULL; const CompressionInfo *info = context->decompress_path->info; const PathTarget *pathtarget = context->decompress_path->custom_path.path.pathtarget; int custom_scan_attno = 1; for (int i = 0; i < list_length(pathtarget->exprs); i++) { Expr *expr = list_nth(pathtarget->exprs, i); if (!IsA(expr, Var)) { /* * The pathtarget has some non-var expressions, so we won't be able * to build a matching decompressed scan targetlist. */ return NIL; } Var *var = castNode(Var, expr); /* This should produce uncompressed chunk columns. */ Assert((Index) var->varno == info->chunk_rel->relid); const int uncompressed_chunk_attno = var->varattno; if (uncompressed_chunk_attno <= 0) { /* * The pathtarget has some special vars so we won't be able to * build a matching decompressed scan targetlist. */ return NIL; } char *attname = get_attname(info->chunk_rte->relid, uncompressed_chunk_attno, /* missing_ok = */ false); TargetEntry *target_entry = makeTargetEntry((Expr *) copyObject(var), /* resno = */ custom_scan_attno, /* resname = */ attname, /* resjunk = */ false); target_entry->ressortgroupref = pathtarget->sortgrouprefs ? pathtarget->sortgrouprefs[i] : 0; result = lappend(result, target_entry); uncompressed_attrs_found = bms_add_member(uncompressed_attrs_found, uncompressed_chunk_attno - FirstLowInvalidHeapAttributeNumber); custom_scan_attno++; } if (!bms_equal(uncompressed_attrs_found, context->uncompressed_attrs_needed)) { /* * There are some variables that are not in the pathtarget that are used * for quals. We still have to have them in the scan tuple in this case. * Note that while we could possibly relax this at execution time for * vectorized quals, the requirement that the qual var be found in the * scan targetlist is a Postgres one. */ return NIL; } return result; } /* * Given the compressed output targetlist and the bitmapset of the needed * columns, determine which compressed chunk column become which uncompressed * chunk column. * * Note that the uncompressed_attrs_needed bitmap is offset by the * FirstLowInvalidHeapAttributeNumber, similar to RelOptInfo.attr_needed. This * allows to encode the requirement for system columns, which have negative * attnos. */ static void build_decompression_map(DecompressionMapContext *context, List *compressed_output_tlist) { ColumnarScanPath *path = context->decompress_path; const CompressionInfo *info = path->info; /* * Track which normal and metadata columns we were able to find in the * targetlist. */ bool missing_count = true; Bitmapset *uncompressed_attrs_found = NULL; Bitmapset *selectedCols = NULL; #if PG16_LT selectedCols = info->ht_rte->selectedCols; #else if (info->ht_rte->perminfoindex > 0) { RTEPermissionInfo *perminfo = getRTEPermissionInfo(context->root->parse->rteperminfos, info->ht_rte); selectedCols = perminfo->selectedCols; } #endif /* * TODO this way to determine which columns are used is actually wrong, see * https://github.com/timescale/timescaledb/issues/4195#issuecomment-1104238863 * Left as is for now, because changing it uncovers a whole new story with * ctid. */ check_for_system_columns(selectedCols); /* * We allow tableoid system column, it won't be in the targetlist but will * be added at decompression time. Always mark it as found. */ if (bms_is_member(TableOidAttributeNumber - FirstLowInvalidHeapAttributeNumber, context->uncompressed_attrs_needed)) { uncompressed_attrs_found = bms_add_member(uncompressed_attrs_found, TableOidAttributeNumber - FirstLowInvalidHeapAttributeNumber); } ListCell *lc; context->uncompressed_attno_info = palloc0(sizeof(*context->uncompressed_attno_info) * (info->chunk_rel->max_attr + 1)); context->compressed_attno_info = palloc0(sizeof(*context->compressed_attno_info) * (info->compressed_rel->max_attr + 1)); /* * Go over the scan targetlist and determine to which output column each * scan column goes, saving other additional info as we do that. */ context->have_bulk_decompression_columns = false; context->decompression_map = NIL; foreach (lc, compressed_output_tlist) { TargetEntry *target = (TargetEntry *) lfirst(lc); if (!IsA(target->expr, Var)) { elog(ERROR, "compressed scan targetlist entries must be Vars"); } Var *var = castNode(Var, target->expr); Assert((Index) var->varno == info->compressed_rel->relid); AttrNumber compressed_chunk_attno = var->varattno; if (compressed_chunk_attno == InvalidAttrNumber) { /* * We shouldn't have whole-row vars in the compressed scan tlist, * they are going to be built by final projection of ColumnarScan * custom scan. * See compressed_rel_setup_reltarget(). */ elog(ERROR, "compressed scan targetlist must not have whole-row vars"); } const char *column_name = get_attname(info->compressed_rte->relid, compressed_chunk_attno, /* missing_ok = */ false); AttrNumber uncompressed_chunk_attno = get_attnum(info->chunk_rte->relid, column_name); AttrNumber destination_attno = 0; if (uncompressed_chunk_attno != InvalidAttrNumber) { /* * Normal column, not a metadata column. */ Assert(uncompressed_chunk_attno != InvalidAttrNumber); if (bms_is_member(0 - FirstLowInvalidHeapAttributeNumber, context->uncompressed_attrs_needed)) { /* * attno = 0 means whole-row var. Output all the columns. */ destination_attno = uncompressed_chunk_attno; uncompressed_attrs_found = bms_add_member(uncompressed_attrs_found, uncompressed_chunk_attno - FirstLowInvalidHeapAttributeNumber); } else if (bms_is_member(uncompressed_chunk_attno - FirstLowInvalidHeapAttributeNumber, context->uncompressed_attrs_needed)) { destination_attno = uncompressed_chunk_attno; uncompressed_attrs_found = bms_add_member(uncompressed_attrs_found, uncompressed_chunk_attno - FirstLowInvalidHeapAttributeNumber); } } else { /* * Metadata column. * We always need count column, and sometimes a sequence number * column. We don't output them, but use them for decompression, * hence the special negative destination attnos. * The min/max metadata columns are normally not required for output * or decompression, they are used only as filter for the compressed * scan, so we skip them here. */ Assert(strncmp(column_name, COMPRESSION_COLUMN_METADATA_PREFIX, strlen(COMPRESSION_COLUMN_METADATA_PREFIX)) == 0); if (strcmp(column_name, COMPRESSION_COLUMN_METADATA_COUNT_NAME) == 0) { destination_attno = COLUMNAR_SCAN_COUNT_ID; missing_count = false; } } const bool is_segment = ts_array_is_member(info->settings->fd.segmentby, column_name); /* * Determine if we can use bulk decompression for this column. */ Oid typoid = get_atttype(info->chunk_rte->relid, uncompressed_chunk_attno); const bool bulk_decompression_possible = !is_segment && destination_attno > 0 && tsl_get_decompress_all_function(compression_get_default_algorithm(typoid), typoid) != NULL; context->have_bulk_decompression_columns |= bulk_decompression_possible; /* * Save information about decompressed columns in uncompressed chunk * for planning of vectorized filters. */ if (uncompressed_chunk_attno != InvalidAttrNumber) { context->uncompressed_attno_info[uncompressed_chunk_attno] = (UncompressedColumnInfo){ .bulk_decompression_possible = bulk_decompression_possible, .custom_scan_attno = InvalidAttrNumber, }; } context->compressed_attno_info[compressed_chunk_attno] = (CompressedColumnInfo){ .bulk_decompression_possible = bulk_decompression_possible, .uncompressed_chunk_attno = destination_attno, .is_segmentby = is_segment, }; } /* * Check that we have found all the needed columns in the compressed targetlist. * We can't conveniently check that we have all columns for all-row vars, so * skip attno 0 in this check. */ Bitmapset *attrs_not_found = bms_difference(context->uncompressed_attrs_needed, uncompressed_attrs_found); int bit = bms_next_member(attrs_not_found, 0 - FirstLowInvalidHeapAttributeNumber); if (bit >= 0) { elog(ERROR, "column '%s' (%d) not found in the targetlist for compressed chunk '%s'", get_attname(info->chunk_rte->relid, bit + FirstLowInvalidHeapAttributeNumber, /* missing_ok = */ true), bit + FirstLowInvalidHeapAttributeNumber, get_rel_name(info->compressed_rte->relid)); } if (missing_count) { elog(ERROR, "the count column was not found in the compressed targetlist"); } /* * If possible, try to make the custom scan targetlist same as the required * output targetlist, so that we can avoid a projection there. */ context->custom_scan_targetlist = follow_uncompressed_output_tlist(context); if (context->custom_scan_targetlist != NIL) { /* * The decompression will produce a custom scan tuple, set the custom * scan attnos accordingly. */ int custom_scan_attno = 1; foreach (lc, context->custom_scan_targetlist) { const int uncompressed_chunk_attno = castNode(Var, castNode(TargetEntry, lfirst(lc))->expr)->varattno; context->uncompressed_attno_info[uncompressed_chunk_attno].custom_scan_attno = custom_scan_attno; custom_scan_attno++; } } else { /* * The decompression will produce the uncompressed chunk tuple, set the * custom scan attnos accordingly. * Note that we might have dropped columns here, but we can set these * attnos for them just as well, they won't be decompressed anyway * because they are not in the compressed scan output. */ for (int i = 1; i <= info->chunk_rel->max_attr; i++) { UncompressedColumnInfo *uncompressed_info = &context->uncompressed_attno_info[i]; uncompressed_info->custom_scan_attno = i; } } /* * Finally, we have to convert the decompression information we've build * into several lists so that it can be passed through the custom path * settings. */ foreach (lc, compressed_output_tlist) { TargetEntry *target = (TargetEntry *) lfirst(lc); Var *var = castNode(Var, target->expr); Assert((Index) var->varno == info->compressed_rel->relid); const AttrNumber compressed_chunk_attno = var->varattno; Assert(compressed_chunk_attno != InvalidAttrNumber); CompressedColumnInfo *compressed_info = &context->compressed_attno_info[compressed_chunk_attno]; /* * Note that the decompressed custom scan targetlist might follow * neither its output targetlist (when we need more columns for filters) * nor the uncompressed chunk tuple. So here we have to do this * additional conversion. */ int compressed_column_destination; if (compressed_info->uncompressed_chunk_attno <= 0) { compressed_column_destination = compressed_info->uncompressed_chunk_attno; } else { UncompressedColumnInfo *uncompressed_info = &context->uncompressed_attno_info[compressed_info->uncompressed_chunk_attno]; compressed_column_destination = uncompressed_info->custom_scan_attno; } context->decompression_map = lappend_int(context->decompression_map, compressed_column_destination); context->is_segmentby_column = lappend_int(context->is_segmentby_column, compressed_info->is_segmentby); context->bulk_decompression_column = lappend_int(context->bulk_decompression_column, compressed_info->bulk_decompression_possible); } } /* replace vars that reference the compressed table with ones that reference the * uncompressed one. Based on replace_nestloop_params */ static Node * replace_compressed_vars(Node *node, const CompressionInfo *info) { if (node == NULL) return NULL; if (IsA(node, Var)) { Var *var = (Var *) node; Var *new_var; char *colname; /* constify tableoid in quals */ if ((Index) var->varno == info->chunk_rel->relid && var->varattno == TableOidAttributeNumber) return (Node *) makeConst(OIDOID, -1, InvalidOid, 4, (Datum) info->chunk_rte->relid, false, true); /* Upper-level Vars should be long gone at this point */ Assert(var->varlevelsup == 0); /* If not to be replaced, we can just return the Var unmodified */ if ((Index) var->varno != info->compressed_rel->relid) return node; /* Create a decompressed Var to replace the compressed one */ colname = get_attname(info->compressed_rte->relid, var->varattno, false); new_var = makeVar(info->chunk_rel->relid, get_attnum(info->chunk_rte->relid, colname), var->vartype, var->vartypmod, var->varcollid, var->varlevelsup); if (!AttributeNumberIsValid(new_var->varattno)) elog(ERROR, "cannot find column %s on decompressed chunk", colname); /* And return the replacement var */ return (Node *) new_var; } if (IsA(node, PlaceHolderVar)) elog(ERROR, "ignoring placeholders"); return expression_tree_mutator(node, replace_compressed_vars, (void *) info); } /* * Find the resno of the given attribute in the provided target list */ static AttrNumber find_attr_pos_in_tlist(List *targetlist, AttrNumber pos) { ListCell *lc; Assert(targetlist != NIL); Assert(pos > 0 && pos != InvalidAttrNumber); foreach (lc, targetlist) { TargetEntry *target = (TargetEntry *) lfirst(lc); if (!IsA(target->expr, Var)) elog(ERROR, "compressed scan targetlist entries must be Vars"); Var *var = castNode(Var, target->expr); AttrNumber compressed_attno = var->varattno; if (compressed_attno == pos) return target->resno; } elog(ERROR, "Unable to locate var %d in targetlist", pos); pg_unreachable(); } static bool is_not_runtime_constant_walker(Node *node, void *context) { if (node == NULL) { return false; } switch (nodeTag(node)) { case T_Var: case T_PlaceHolderVar: /* * We might want to support these nodes to have vectorizable join * clauses (T_Var) or join clauses referencing a variable that is * above outer join (T_PlaceHolderVar). We don't support them at the * moment. */ return true; case T_Param: /* * We support external query parameters (e.g. from parameterized * prepared statements), because they are constant for the duration * of the query. * * Join and initplan parameters are passed as PARAM_EXEC and require * support in the Rescan functions of the custom scan node. We don't * support them at the moment. */ return castNode(Param, node)->paramkind != PARAM_EXTERN; default: if (check_functions_in_node(node, contains_volatile_functions_checker, /* context = */ NULL)) { return true; } return expression_tree_walker(node, is_not_runtime_constant_walker, /* context = */ NULL); } } /* * Check if the given node is a run-time constant, i.e. it doesn't contain * volatile functions or variables or parameters. This means we can evaluate * it at run time, allowing us to apply the vectorized comparison operators * that have the form "Var op Const". This applies for example to filter * expressions like `time > now() - interval '1 hour'`. * Note that we do the same evaluation when doing run time chunk exclusion, but * there is no good way to pass the evaluated clauses to the underlying nodes * like this ColumnarScan node. * * Similar checks are performed for sparse index pushdown. */ static bool is_not_runtime_constant(Node *node) { bool result = is_not_runtime_constant_walker(node, /* context = */ NULL); return result; } /* * Try to check if the current qual is vectorizable, and if needed make a * commuted copy. If not, return NULL. */ Node * vector_qual_make(Node *qual, const VectorQualInfo *vqinfo) { /* * We can vectorize BoolExpr (AND/OR/NOT). */ if (IsA(qual, BoolExpr)) { BoolExpr *boolexpr = castNode(BoolExpr, qual); if (boolexpr->boolop == NOT_EXPR) { /* * NOT should be removed by Postgres for all operators we can * vectorize (see prepqual.c) except when the where clause is * something like 'COL = false' for bool columns. In this case, we * have to check if it was transformed to BoolExpr(NOT_EXPR, Var) so * we can vectorize it, provided that the column supports bulk * decompression. */ if (list_length(boolexpr->args) == 1 && IsA(linitial(boolexpr->args), Var)) { if (!vqinfo->vector_attrs[castNode(Var, linitial(boolexpr->args))->varattno]) { return NULL; } } else { return NULL; } } bool need_copy = false; List *vectorized_args = NIL; ListCell *lc; foreach (lc, boolexpr->args) { Node *arg = lfirst(lc); Node *vectorized_arg = vector_qual_make(arg, vqinfo); if (vectorized_arg == NULL) { return NULL; } if (vectorized_arg != arg) { need_copy = true; } vectorized_args = lappend(vectorized_args, vectorized_arg); } if (!need_copy) { return (Node *) boolexpr; } BoolExpr *boolexpr_copy = (BoolExpr *) copyObject(boolexpr); boolexpr_copy->args = vectorized_args; return (Node *) boolexpr_copy; } /* * Among the simple predicates, we vectorize some "Var op Const" binary * predicates, scalar array operations with these predicates, boolean variables * and null test. */ NullTest *nulltest = NULL; OpExpr *opexpr = NULL; ScalarArrayOpExpr *saop = NULL; Node *arg1 = NULL; Node *arg2 = NULL; Oid opno = InvalidOid; Var *var = NULL; if (IsA(qual, OpExpr)) { opexpr = castNode(OpExpr, qual); opno = opexpr->opno; if (list_length(opexpr->args) != 2) { return NULL; } arg1 = (Node *) linitial(opexpr->args); arg2 = (Node *) lsecond(opexpr->args); } else if (IsA(qual, ScalarArrayOpExpr)) { saop = castNode(ScalarArrayOpExpr, qual); opno = saop->opno; Assert(list_length(saop->args) == 2); arg1 = (Node *) linitial(saop->args); arg2 = (Node *) lsecond(saop->args); } else if (IsA(qual, NullTest)) { nulltest = castNode(NullTest, qual); arg1 = (Node *) nulltest->arg; } else if (IsA(qual, BooleanTest)) { BooleanTest *booltest = castNode(BooleanTest, qual); if (IsA(booltest->arg, Var)) { var = castNode(Var, booltest->arg); if (!vqinfo->vector_attrs[var->varattno]) { return NULL; } return (Node *) booltest; } else { return NULL; } } else if (IsA(qual, Var) && (castNode(Var, qual))->vartype == BOOLOID) { /* We can vectorize boolean variables if bulk decompression is possible. */ var = castNode(Var, qual); if (!vqinfo->vector_attrs[var->varattno]) { return NULL; } return (Node *) var; } else { return NULL; } if (opexpr && IsA(arg2, Var)) { /* * Try to commute the operator if we have Var on the right. */ opno = get_commutator(opno); if (!OidIsValid(opno)) { return NULL; } opexpr = (OpExpr *) copyObject(opexpr); opexpr->opno = opno; /* * opfuncid is a cache, we can set it to InvalidOid like the * CommuteOpExpr() does. */ opexpr->opfuncid = InvalidOid; opexpr->args = list_make2(arg2, arg1); Node *tmp = arg1; arg1 = arg2; arg2 = tmp; } /* * We can vectorize the operation where the left side is a Var. */ if (!IsA(arg1, Var)) { return NULL; } var = castNode(Var, arg1); if ((Index) var->varno != vqinfo->rti) { /* * We have a Var from other relation (join clause), can't vectorize it * at the moment. */ return NULL; } if (var->varattno <= 0) { /* * Can't vectorize operators with special variables such as whole-row var. */ return NULL; } /* * ExecQual is performed before ExecProject and operates on the decompressed * scan slot, so the qual attnos are the uncompressed chunk attnos. */ if (!vqinfo->vector_attrs[var->varattno]) { /* This column doesn't support bulk decompression. */ return NULL; } if (nulltest) { /* * The checks we've done to this point is all that is required for null * test. */ return (Node *) nulltest; } /* * We can vectorize the operation where the right side is a constant or can * be evaluated to a constant at run time (e.g. contains stable functions). */ Assert(arg2); if (is_not_runtime_constant(arg2)) { return NULL; } Oid opcode = get_opcode(opno); if (!get_vector_const_predicate(opcode)) { return NULL; } if (OidIsValid(var->varcollid) && !get_collation_isdeterministic(var->varcollid)) { /* * Can't vectorize string equality with a nondeterministic collation. * Not sure if we have to check the collation of Const as well, but it * will be known only at planning time. Currently we don't check it at * all. Also this is untested because we don't have nondeterministic * collations in all test configurations. */ return NULL; } if (opexpr) { /* * The checks we've done to this point is all that is required for * OpExpr. */ return (Node *) opexpr; } /* * The only option that is left is a ScalarArrayOpExpr. */ Assert(saop != NULL); /* * The planner can decide to build a hash table. It's still somewhat slower * than our vectorized lookups for array lengths <= 32. */ if (saop->hashfuncid) { if (!IsA(arg2, Const)) { /* * The planner as of PG 17 only uses hashing for plan-time constants, * but double-check. */ return NULL; } Const *c = castNode(Const, arg2); if (c->constisnull) { /* * Shouldn't happen, but not controlled by us. */ return NULL; } Datum arrdatum = c->constvalue; ArrayType *arr = (ArrayType *) DatumGetPointer(arrdatum); const int nitems = ArrayGetNItems(ARR_NDIM(arr), ARR_DIMS(arr)); if (nitems > 32) { return NULL; } } return (Node *) saop; } /* * Find the scan qualifiers that can be vectorized and put them into a separate * list. */ static void find_vectorized_quals(DecompressionMapContext *context, ColumnarScanPath *path, List *qual_list, List **vectorized, List **nonvectorized) { VectorQualInfo vqi = { .maxattno = path->info->chunk_rel->max_attr, .vector_attrs = build_vector_attrs_array(context->uncompressed_attno_info, path->info), .rti = path->info->chunk_rel->relid, }; ListCell *lc; foreach (lc, qual_list) { Node *source_qual = lfirst(lc); /* * We can't vectorize the stable cross-type operators (for example * timestamp > timestamptz), so try to cast the constant to the same * type to convert it to the same-type operator. We do the same thing * for chunk exclusion. */ Node *transformed_comparison = (Node *) ts_transform_cross_datatype_comparison((Expr *) source_qual); Node *vectorized_qual = vector_qual_make(transformed_comparison, &vqi); if (vectorized_qual) { *vectorized = lappend(*vectorized, vectorized_qual); } else { *nonvectorized = lappend(*nonvectorized, source_qual); } } pfree(vqi.vector_attrs); } /* * Copy of the Postgres' static function from createplan.c. * * Some places in this file build Sort nodes that don't have a directly * corresponding Path node. The cost of the sort is, or should have been, * included in the cost of the Path node we're working from, but since it's * not split out, we have to re-figure it using cost_sort(). This is just * to label the Sort node nicely for EXPLAIN. * * limit_tuples is as for cost_sort (in particular, pass -1 if no limit) */ static void ts_label_sort_with_costsize(PlannerInfo *root, Sort *plan, double limit_tuples) { Plan *lefttree = plan->plan.lefttree; Path sort_path; /* dummy for result of cost_sort */ /* * This function shouldn't have to deal with IncrementalSort plans because * they are only created from corresponding Path nodes. */ Assert(IsA(plan, Sort)); cost_sort(&sort_path, root, NIL, #if PG18_GE lefttree->disabled_nodes, #endif lefttree->total_cost, lefttree->plan_rows, lefttree->plan_width, 0.0, work_mem, limit_tuples); plan->plan.startup_cost = sort_path.startup_cost; plan->plan.total_cost = sort_path.total_cost; plan->plan.plan_rows = lefttree->plan_rows; plan->plan.plan_width = lefttree->plan_width; plan->plan.parallel_aware = false; plan->plan.parallel_safe = lefttree->parallel_safe; } /* * Find a variable of the given relation somewhere in the expression tree. * Currently we use this to find the Var argument of time_bucket, when we prepare * the batch sorted merge parameters after using the monotonous sorting transform * optimization. */ static Var * find_var_subexpression(void *expr, Index varno) { List *varlist = pull_var_clause((Node *) expr, 0); if (list_length(varlist) == 1) { Var *var = (Var *) linitial(varlist); if ((Index) var->varno == (Index) varno) { return var; } return NULL; } return NULL; } Plan * columnar_scan_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *path, List *output_targetlist, List *clauses, List *custom_plans) { ColumnarScanPath *dcpath = (ColumnarScanPath *) path; CustomScan *decompress_plan = makeNode(CustomScan); Scan *compressed_scan = linitial(custom_plans); Path *compressed_path = linitial(path->custom_paths); List *settings; ListCell *lc; Assert(list_length(custom_plans) == 1); Assert(list_length(path->custom_paths) == 1); decompress_plan->flags = path->flags; decompress_plan->methods = &columnar_scan_plan_methods; decompress_plan->scan.scanrelid = dcpath->info->chunk_rel->relid; if (IsA(compressed_path, IndexPath)) { /* * Check if any of the decompressed scan clauses are redundant with * the compressed index scan clauses. Note that we can't use * is_redundant_derived_clause() here, because it can't work with * IndexClause's, so we use some custom code based on it. */ IndexPath *ipath = castNode(IndexPath, compressed_path); foreach (lc, clauses) { RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); ListCell *indexclause_cell = NULL; if (rinfo->parent_ec != NULL) { foreach (indexclause_cell, ipath->indexclauses) { IndexClause *indexclause = lfirst(indexclause_cell); RestrictInfo *index_rinfo = indexclause->rinfo; if (index_rinfo->parent_ec == rinfo->parent_ec) { break; } } } if (indexclause_cell != NULL) { /* We already have an index clause derived from same EquivalenceClass. */ continue; } /* * We don't have this clause in the underlying index scan, add it * to the decompressed scan. */ decompress_plan->scan.plan.qual = lappend(decompress_plan->scan.plan.qual, rinfo->clause); } } else { foreach (lc, clauses) { RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); decompress_plan->scan.plan.qual = lappend(decompress_plan->scan.plan.qual, rinfo->clause); } } decompress_plan->scan.plan.qual = (List *) replace_compressed_vars((Node *) decompress_plan->scan.plan.qual, dcpath->info); /* * Try to use a physical tlist if possible. There's no reason to do the * extra work of projecting the result of compressed chunk scan, because * ColumnarScan can choose only the needed columns itself. * Note that Postgres uses the CP_EXACT_TLIST option when planning the child * paths of the Custom path, so we won't automatically get a physical tlist * here. */ bool target_list_compressed_is_physical = false; if (compressed_path->pathtype == T_IndexOnlyScan) { compressed_scan->plan.targetlist = ((IndexPath *) compressed_path)->indexinfo->indextlist; } else { List *physical_tlist = build_physical_tlist(root, dcpath->info->compressed_rel); /* Can be null if the relation has dropped columns. */ if (physical_tlist) { compressed_scan->plan.targetlist = physical_tlist; target_list_compressed_is_physical = true; } } /* * Determine which columns we have to decompress. * output_targetlist is sometimes empty, e.g. for a direct select from * chunk. We have a ProjectionPath above ColumnarScan in this case, and * the targetlist for this path is not built by the planner * (CP_IGNORE_TLIST). This is why we have to examine rel pathtarget. * Looking at the targetlist is not enough, we also have to decompress the * columns participating in quals and in pathkeys. */ Bitmapset *uncompressed_attrs_needed = NULL; pull_varattnos((Node *) decompress_plan->scan.plan.qual, dcpath->info->chunk_rel->relid, &uncompressed_attrs_needed); pull_varattnos((Node *) dcpath->custom_path.path.pathtarget->exprs, dcpath->info->chunk_rel->relid, &uncompressed_attrs_needed); /* * Determine which compressed column goes to which output column. */ DecompressionMapContext context = { .root = root, .decompress_path = dcpath, .uncompressed_attrs_needed = uncompressed_attrs_needed }; build_decompression_map(&context, compressed_scan->plan.targetlist); /* Build heap sort info for batch sorted merge. */ List *sort_options = NIL; if (dcpath->batch_sorted_merge) { /* * 'order by' of the query and the 'order by' of the compressed batches * match, so we will we use a heap to merge the batches. For the heap we * need a compare function that determines the heap order. This function * is constructed here. * * Batch sorted merge is done over the decompressed chunk scan tuple, so * we must match the pathkeys to the decompressed chunk tupdesc. */ int numsortkeys = list_length(dcpath->custom_path.path.pathkeys); List *sort_col_idx = NIL; List *sort_ops = NIL; List *sort_collations = NIL; List *sort_nulls = NIL; /* */ ListCell *lc; foreach (lc, dcpath->custom_path.path.pathkeys) { PathKey *pk = lfirst(lc); EquivalenceClass *ec = pk->pk_eclass; /* * Find the equivalence member that belongs to decompressed relation. */ EquivalenceMember *em; ListCell *membercell = NULL; #if PG18_GE /* In PG18, iterating over child ems requires you to * use child relids with a special iterator. Here we gather * them by collecting them from childmembers array. * * https://github.com/postgres/postgres/commit/d69d45a5 */ EquivalenceMemberIterator it; setup_eclass_member_iterator(&it, ec, dcpath->custom_path.path.parent->relids); while ((em = eclass_member_iterator_next(&it)) != NULL) { /* Setting up so that the check below doesn't complain */ membercell = &list_make_int_cell(1); #else foreach (membercell, ec->ec_members) { em = lfirst(membercell); #endif if (em->em_is_const) { continue; } int em_relid; if (!bms_get_singleton_member(em->em_relids, &em_relid)) { continue; } if ((Index) em_relid != dcpath->info->chunk_rel->relid) { continue; } /* * The equivalence member expression might be a monotonous * expression of the decompressed relation Var, so recurse to * find it. */ Var *var = find_var_subexpression(em->em_expr, em_relid); Ensure(var != NULL, "non-Var pathkey not expected for compressed batch sorted merge"); Assert((Index) var->varno == (Index) em_relid); /* * Convert its varattno which is the varattno of the * uncompressed chunk tuple, to the decompressed scan tuple * varattno. */ const int decompressed_scan_attno = context.uncompressed_attno_info[var->varattno].custom_scan_attno; Assert(decompressed_scan_attno > 0); /* * Look up the correct sort operator from the PathKey's slightly * abstracted representation. */ Oid sortop = get_opfamily_member(pk->pk_opfamily, var->vartype, var->vartype, pk->pk_cmptype); if (!OidIsValid(sortop)) /* should not happen */ elog(ERROR, "missing operator %d(%u,%u) in opfamily %u", pk->pk_cmptype, var->vartype, var->vartype, pk->pk_opfamily); sort_col_idx = lappend_oid(sort_col_idx, decompressed_scan_attno); sort_collations = lappend_oid(sort_collations, var->varcollid); sort_nulls = lappend_oid(sort_nulls, pk->pk_nulls_first); sort_ops = lappend_oid(sort_ops, sortop); break; } Ensure(membercell != NULL, "could not find matching decompressed chunk column for batch sorted merge " "pathkey"); } sort_options = list_make4(sort_col_idx, sort_ops, sort_collations, sort_nulls); /* * Build a sort node for the compressed batches. The sort function is * derived from the sort function of the pathkeys, except that it refers * to the min and max metadata columns of the batches. We have already * verified that the pathkeys match the compression order_by, so this * mapping is possible. */ AttrNumber *sortColIdx = palloc(sizeof(AttrNumber) * numsortkeys); Oid *sortOperators = palloc(sizeof(Oid) * numsortkeys); Oid *collations = palloc(sizeof(Oid) * numsortkeys); bool *nullsFirst = palloc(sizeof(bool) * numsortkeys); for (int i = 0; i < numsortkeys; i++) { Oid sortop = list_nth_oid(sort_ops, i); /* Find the operator in pg_amop --- failure shouldn't happen */ Oid opfamily, opcintype; CompareType strategy; if (!get_ordering_op_properties(list_nth_oid(sort_ops, i), &opfamily, &opcintype, &strategy)) elog(ERROR, "operator %u is not a valid ordering operator", sortOperators[i]); /* * This way to determine the matching metadata column works, because * we have already verified that the pathkeys match the compression * orderby. */ Assert(strategy == BTLessStrategyNumber || strategy == BTGreaterStrategyNumber); char *meta_col_name = strategy == BTLessStrategyNumber ? column_segment_min_name(i + 1) : column_segment_max_name(i + 1); AttrNumber attr_position = get_attnum(dcpath->info->compressed_rte->relid, meta_col_name); if (attr_position == InvalidAttrNumber) elog(ERROR, "couldn't find metadata column \"%s\"", meta_col_name); /* * If the the compressed target list is not based on the layout of * the uncompressed chunk (see comment for physical_tlist above), * adjust the position of the attribute. */ if (target_list_compressed_is_physical) sortColIdx[i] = attr_position; else sortColIdx[i] = find_attr_pos_in_tlist(compressed_scan->plan.targetlist, attr_position); sortOperators[i] = sortop; collations[i] = list_nth_oid(sort_collations, i); nullsFirst[i] = list_nth_oid(sort_nulls, i); } /* Now build the compressed batches sort node */ Sort *sort = ts_make_sort((Plan *) compressed_scan, numsortkeys, sortColIdx, sortOperators, collations, nullsFirst); ts_label_sort_with_costsize(root, sort, /* limit_tuples = */ -1.0); decompress_plan->custom_plans = list_make1(sort); } else { /* * Add a sort if the compressed scan is not ordered appropriately. */ if (!pathkeys_contained_in(dcpath->required_compressed_pathkeys, compressed_path->pathkeys)) { List *compressed_pks = dcpath->required_compressed_pathkeys; Sort *sort = ts_make_sort_from_pathkeys((Plan *) compressed_scan, compressed_pks, bms_make_singleton(compressed_scan->scanrelid)); ts_label_sort_with_costsize(root, sort, /* limit_tuples = */ -1.0); decompress_plan->custom_plans = list_make1(sort); } else { decompress_plan->custom_plans = custom_plans; } } Assert(list_length(custom_plans) == 1); const bool enable_bulk_decompression = !dcpath->batch_sorted_merge && ts_guc_enable_bulk_decompression && context.have_bulk_decompression_columns; /* * For some predicates, we have more efficient implementation that work on * the entire compressed batch in one go. They go to this list, and the rest * goes into the usual scan.plan.qual. */ List *vectorized_quals = NIL; if (enable_bulk_decompression) { List *nonvectorized_quals = NIL; find_vectorized_quals(&context, dcpath, decompress_plan->scan.plan.qual, &vectorized_quals, &nonvectorized_quals); decompress_plan->scan.plan.qual = nonvectorized_quals; } #ifdef TS_DEBUG if (ts_guc_debug_require_vector_qual == DRO_Forbid && list_length(vectorized_quals) > 0) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("debug: encountered vector quals when they are disabled"))); } else if (ts_guc_debug_require_vector_qual == DRO_Require) { if (list_length(decompress_plan->scan.plan.qual) > 0) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("debug: encountered non-vector quals when they are disabled"))); } if (list_length(vectorized_quals) == 0) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("debug: did not encounter vector quals when they are required"))); } } #endif settings = ts_new_list(T_IntList, DCS_Count); lfirst_int(list_nth_cell(settings, DCS_HypertableId)) = dcpath->info->hypertable_id; lfirst_int(list_nth_cell(settings, DCS_ChunkRelid)) = dcpath->info->chunk_rte->relid; lfirst_int(list_nth_cell(settings, DCS_Reverse)) = dcpath->reverse; lfirst_int(list_nth_cell(settings, DCS_BatchSortedMerge)) = dcpath->batch_sorted_merge; lfirst_int(list_nth_cell(settings, DCS_EnableBulkDecompression)) = enable_bulk_decompression; lfirst_int(list_nth_cell(settings, DCS_HasRowMarks)) = root->parse->rowMarks != NIL; lfirst_int(list_nth_cell(settings, DCS_ChunkStatus)) = dcpath->chunk_status; /* * Vectorized quals must go into custom_exprs, because Postgres has to see * them and perform the varno adjustments on them when flattening the * subqueries. */ decompress_plan->custom_exprs = list_make1(vectorized_quals); decompress_plan->custom_private = ts_new_list(T_List, DCP_Count); lfirst(list_nth_cell(decompress_plan->custom_private, DCP_Settings)) = settings; lfirst(list_nth_cell(decompress_plan->custom_private, DCP_DecompressionMap)) = context.decompression_map; lfirst(list_nth_cell(decompress_plan->custom_private, DCP_IsSegmentbyColumn)) = context.is_segmentby_column; lfirst(list_nth_cell(decompress_plan->custom_private, DCP_BulkDecompressionColumn)) = context.bulk_decompression_column; lfirst(list_nth_cell(decompress_plan->custom_private, DCP_SortInfo)) = sort_options; /* * We might be using a custom scan tuple if it allows us to avoid the * projection. Otherwise, this tlist is NIL and we'll be using the * uncompressed tuple as the custom scan tuple. */ decompress_plan->custom_scan_tlist = context.custom_scan_targetlist; /* * Note that we cannot decide here that we require a projection. It is * decided at Path stage, now we must produce the requested targetlist. */ decompress_plan->scan.plan.targetlist = output_targetlist; return &decompress_plan->scan.plan; } ================================================ FILE: tsl/src/nodes/columnar_scan/planner.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> typedef enum { DCS_HypertableId = 0, DCS_ChunkRelid = 1, DCS_Reverse = 2, DCS_BatchSortedMerge = 3, DCS_EnableBulkDecompression = 4, DCS_HasRowMarks = 5, DCS_ChunkStatus = 6, DCS_Count } ColumnarScanSettingsIndex; typedef enum { DCP_Settings = 0, DCP_DecompressionMap = 1, DCP_IsSegmentbyColumn = 2, DCP_BulkDecompressionColumn = 3, DCP_SortInfo = 4, DCP_Count } ColumnarScanPrivateIndex; extern Plan *columnar_scan_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *path, List *output_targetlist, List *clauses, List *custom_plans); extern void _columnar_scan_init(void); ================================================ FILE: tsl/src/nodes/columnar_scan/pred_text.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include "pred_text.h" #include <miscadmin.h> #include "compat/compat.h" #if PG16_GE #include <varatt.h> #endif static void vector_const_text_comparison(const ArrowArray *arrow, const Datum constdatum, bool needequal, uint64 *restrict result) { Assert(!arrow->dictionary); text *consttext = (text *) DatumGetPointer(constdatum); const size_t textlen = VARSIZE_ANY_EXHDR(consttext); const uint8 *cstring = (uint8 *) VARDATA_ANY(consttext); const uint32 *offsets = (uint32 *) arrow->buffers[1]; const uint8 *values = (uint8 *) arrow->buffers[2]; const size_t n = arrow->length; for (size_t outer = 0; outer < n / 64; outer++) { uint64 word = 0; for (size_t inner = 0; inner < 64; inner++) { const size_t row = (outer * 64) + inner; const size_t bit_index = inner; #define INNER_LOOP \ const uint32 start = offsets[row]; \ const uint32 end = offsets[row + 1]; \ Assert(end >= start); \ const uint32 veclen = end - start; \ bool isequal = veclen != textlen ? \ false : \ (strncmp((char *) &values[start], (char *) cstring, textlen) == 0); \ word |= ((uint64) (isequal == needequal)) << bit_index; INNER_LOOP } result[outer] &= word; } if (n % 64) { uint64 word = 0; for (size_t row = (n / 64) * 64; row < n; row++) { const size_t bit_index = row % 64; INNER_LOOP } result[n / 64] &= word; } #undef INNER_LOOP } void vector_const_texteq(const ArrowArray *arrow, const Datum constdatum, uint64 *restrict result) { vector_const_text_comparison(arrow, constdatum, /* needequal = */ true, result); } void vector_const_textne(const ArrowArray *arrow, const Datum constdatum, uint64 *restrict result) { vector_const_text_comparison(arrow, constdatum, /* needequal = */ false, result); } /* * Generate specializations for LIKE functions based on database encoding. This * follows the Postgres code from backend/utils/adt/like.c, version 15.0, * commit sha 2a7ce2e2ce474504a707ec03e128fde66cfb8b48. * The copy of PG code begins here. * ---------------------------------------------------------------------------- */ #define LIKE_TRUE 1 #define LIKE_FALSE 0 #define LIKE_ABORT (-1) /* setup to compile like_match.c for UTF8 encoding, using fast NextChar */ #define NextByte(p, plen) ((p)++, (plen)--) #define NextChar(p, plen) \ do \ { \ (p)++; \ (plen)--; \ } while ((plen) > 0 && (*(p) & 0xC0) == 0x80) #define MatchText UTF8_MatchText #include "import/ts_like_match.c" /* * ---------------------------------------------------------------------------- * The copy of PG code ends here. */ static void vector_const_like_impl(const ArrowArray *arrow, const Datum constdatum, uint64 *restrict result, int (*match)(const char *, int, const char *, int), bool should_match) { Assert(!arrow->dictionary); text *consttext = (text *) DatumGetPointer(constdatum); const size_t textlen = VARSIZE_ANY_EXHDR(consttext); const char *restrict cstring = VARDATA_ANY(consttext); const uint32 *offsets = (uint32 *) arrow->buffers[1]; const char *restrict values = arrow->buffers[2]; const size_t n = arrow->length; for (size_t outer = 0; outer < n / 64; outer++) { uint64 word = 0; for (size_t inner = 0; inner < 64; inner++) { const size_t row = (outer * 64) + inner; const size_t bit_index = inner; /* * The inner loop could have been an inline function, but it would have 5 * parameters and one of them in/out, so a macro probably has better * readability. */ #define INNER_LOOP \ const uint32 start = offsets[row]; \ const uint32 end = offsets[row + 1]; \ Assert(end >= start); \ const uint32 veclen = end - start; \ int result = match(&values[start], veclen, cstring, textlen); \ bool valid = (result == LIKE_TRUE) == should_match; \ word |= ((uint64) valid) << bit_index; INNER_LOOP } result[outer] &= word; } if (n % 64) { uint64 word = 0; for (size_t row = (n / 64) * 64; row < n; row++) { const size_t bit_index = row % 64; INNER_LOOP } result[n / 64] &= word; } #undef INNER_LOOP } void vector_const_textlike_utf8(const ArrowArray *arrow, const Datum constdatum, uint64 *restrict result) { vector_const_like_impl(arrow, constdatum, result, UTF8_MatchText, /* should_match = */ true); } void vector_const_textnlike_utf8(const ArrowArray *arrow, const Datum constdatum, uint64 *restrict result) { vector_const_like_impl(arrow, constdatum, result, UTF8_MatchText, /* should_match = */ false); } ================================================ FILE: tsl/src/nodes/columnar_scan/pred_text.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include "compression/arrow_c_data_interface.h" extern void vector_const_texteq(const ArrowArray *arrow, const Datum constdatum, uint64 *restrict result); extern void vector_const_textne(const ArrowArray *arrow, const Datum constdatum, uint64 *restrict result); extern void vector_const_textlike_utf8(const ArrowArray *arrow, const Datum constdatum, uint64 *restrict result); extern void vector_const_textnlike_utf8(const ArrowArray *arrow, const Datum constdatum, uint64 *restrict result); ================================================ FILE: tsl/src/nodes/columnar_scan/pred_vector_array.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include "compression/arrow_c_data_interface.h" #include "compression/compression.h" #include "guc.h" #include "src/utils.h" #include "vector_predicates.h" /* * Vectorized implementation of ScalarArrayOpExpr. Applies scalar_predicate for * vector and each element of array, combines the result according to "is_or" * flag. Written along the lines of ExecEvalScalarArrayOp(). */ void vector_array_predicate(VectorPredicate *vector_const_predicate, bool is_or, const ArrowArray *vector, Datum array, uint64 *restrict final_result) { const size_t n_rows = vector->length; const size_t result_words = (n_rows + 63) / 64; uint64 *restrict array_result = final_result; /* * For OR, we need an intermediate storage to accumulate the results * from all elements. * For AND, we can apply predicate for each element to the final result. */ uint64 array_result_storage[(GLOBAL_MAX_ROWS_PER_COMPRESSION + 63) / 64]; if (is_or) { array_result = array_result_storage; for (size_t i = 0; i < result_words; i++) { array_result_storage[i] = 0; } } ArrayType *arr = DatumGetArrayTypeP(array); int16 typlen; bool typbyval; char typalign; get_typlenbyvalalign(ARR_ELEMTYPE(arr), &typlen, &typbyval, &typalign); const char *array_data = (const char *) ARR_DATA_PTR(arr); const size_t nitems = ArrayGetNItems(ARR_NDIM(arr), ARR_DIMS(arr)); const uint64 *array_null_bitmap = (uint64 *) ARR_NULLBITMAP(arr); for (size_t array_index = 0; array_index < nitems; array_index++) { if (array_null_bitmap != NULL && !arrow_row_is_valid(array_null_bitmap, array_index)) { /* * This array element is NULL. We can't avoid NULLS when evaluating * the stable functions at run time, so we have to support them. * This is a predicate, not a generic scalar array operation, so * thankfully we return a non-nullable bool. * For ANY: null | true = true, null | false = null, so this means * we can skip the null element and continue evaluation. * For ALL: null & true = null, null & false = false, so this means * that for each row the condition goes to false, and we don't have * to evaluate the next elements. */ if (is_or) { continue; } for (size_t word = 0; word < result_words; word++) { final_result[word] = 0; } return; } Datum constvalue = ts_fetch_att(array_data, typbyval, typlen); array_data = att_addlength_pointer(array_data, typlen, array_data); array_data = (const char *) att_align_nominal(array_data, typalign); /* * For OR, we also need an intermediate storage for predicate result * for each array element, since the predicates AND their result. * * For AND, we can and apply predicate for each array element to the * final result. */ uint64 single_result_storage[(GLOBAL_MAX_ROWS_PER_COMPRESSION + 63) / 64]; uint64 *restrict single_result; if (is_or) { single_result = single_result_storage; for (size_t outer = 0; outer < result_words; outer++) { single_result[outer] = ~0ULL; } } else { single_result = array_result; } vector_const_predicate(vector, constvalue, single_result); if (is_or) { for (size_t outer = 0; outer < result_words; outer++) { array_result[outer] |= single_result[outer]; } } /* * The bitmaps are small, no more than 15 qwords for our maximal * compressed batch size of 1000 rows, so we can check for early exit * after every row. */ BatchQualSummary summary = get_vector_qual_summary(array_result, n_rows); if (summary == (is_or ? AllRowsPass : NoRowsPass)) { return; } } if (is_or) { for (size_t outer = 0; outer < result_words; outer++) { /* * The tail bits corresponding to past-the-end rows when n % 64 != 0 * should be already zeroed out in the final_result. */ final_result[outer] &= array_result[outer]; } } } ================================================ FILE: tsl/src/nodes/columnar_scan/pred_vector_const_arithmetic_all.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Define all supported "vector ? const" predicates for arithmetic types. */ /* int8 functions. */ #define VECTOR_CTYPE int64 #define CONST_CTYPE int64 #define CONST_CONVERSION(X) DatumGetInt64(X) #define PG_PREDICATE(X) \ F_INT8##X: case F_TIMESTAMPTZ_##X: \ case F_TIMESTAMP_##X #include "pred_vector_const_arithmetic_type_pair.c" /* int84 functions. */ #define VECTOR_CTYPE int64 #define CONST_CTYPE int32 #define CONST_CONVERSION(X) DatumGetInt32(X) #define PG_PREDICATE(X) F_INT84##X #include "pred_vector_const_arithmetic_type_pair.c" /* int82 functions. */ #define VECTOR_CTYPE int64 #define CONST_CTYPE int16 #define CONST_CONVERSION(X) DatumGetInt16(X) #define PG_PREDICATE(X) F_INT82##X #include "pred_vector_const_arithmetic_type_pair.c" /* int48 functions. */ #define VECTOR_CTYPE int32 #define CONST_CTYPE int64 #define CONST_CONVERSION(X) DatumGetInt64(X) #define PG_PREDICATE(X) F_INT48##X #include "pred_vector_const_arithmetic_type_pair.c" /* int4 functions. */ #define VECTOR_CTYPE int32 #define CONST_CTYPE int32 #define CONST_CONVERSION(X) DatumGetInt32(X) #define PG_PREDICATE(X) F_INT4##X #include "pred_vector_const_arithmetic_type_pair.c" /* int42 functions. */ #define VECTOR_CTYPE int32 #define CONST_CTYPE int16 #define CONST_CONVERSION(X) DatumGetInt16(X) #define PG_PREDICATE(X) F_INT42##X #include "pred_vector_const_arithmetic_type_pair.c" /* int28 functions. */ #define VECTOR_CTYPE int16 #define CONST_CTYPE int64 #define CONST_CONVERSION(X) DatumGetInt64(X) #define PG_PREDICATE(X) F_INT28##X #include "pred_vector_const_arithmetic_type_pair.c" /* int24 functions. */ #define VECTOR_CTYPE int16 #define CONST_CTYPE int32 #define CONST_CONVERSION(X) DatumGetInt32(X) #define PG_PREDICATE(X) F_INT24##X #include "pred_vector_const_arithmetic_type_pair.c" /* int2 functions. */ #define VECTOR_CTYPE int16 #define CONST_CTYPE int16 #define CONST_CONVERSION(X) DatumGetInt16(X) #define PG_PREDICATE(X) F_INT2##X #include "pred_vector_const_arithmetic_type_pair.c" /* float8 functions. */ #define VECTOR_CTYPE float8 #define CONST_CTYPE float8 #define CONST_CONVERSION(X) DatumGetFloat8(X) #define PG_PREDICATE(X) F_FLOAT8##X #include "pred_vector_const_arithmetic_type_pair.c" /* float84 functions. */ #define VECTOR_CTYPE float8 #define CONST_CTYPE float4 #define CONST_CONVERSION(X) DatumGetFloat4(X) #define PG_PREDICATE(X) F_FLOAT84##X #include "pred_vector_const_arithmetic_type_pair.c" /* float48 functions. */ #define VECTOR_CTYPE float4 #define CONST_CTYPE float8 #define CONST_CONVERSION(X) DatumGetFloat8(X) #define PG_PREDICATE(X) F_FLOAT48##X #include "pred_vector_const_arithmetic_type_pair.c" /* float4 functions. */ #define VECTOR_CTYPE float4 #define CONST_CTYPE float4 #define CONST_CONVERSION(X) DatumGetFloat4(X) #define PG_PREDICATE(X) F_FLOAT4##X #include "pred_vector_const_arithmetic_type_pair.c" /* date functions. */ #define VECTOR_CTYPE DateADT #define CONST_CTYPE DateADT #define CONST_CONVERSION(X) DatumGetDateADT(X) #define PG_PREDICATE(X) F_DATE_##X #include "pred_vector_const_arithmetic_type_pair.c" ================================================ FILE: tsl/src/nodes/columnar_scan/pred_vector_const_arithmetic_single.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Compute a vector-const predicate and AND it to the filter bitmap. * Specialized for particular arithmetic data types and predicate. * Marked as noinline for the ease of debugging. Inlining it shouldn't be * beneficial because it's a big self-contained loop. */ #define PG_PREDICATE_HELPER(X) PG_PREDICATE(X) #define FUNCTION_NAME_HELPER(X, Y, Z) predicate_##X##_##Y##_vector_##Z##_const #define FUNCTION_NAME(X, Y, Z) FUNCTION_NAME_HELPER(X, Y, Z) #ifdef GENERATE_DISPATCH_TABLE case PG_PREDICATE_HELPER(PREDICATE_NAME): return FUNCTION_NAME(PREDICATE_NAME, VECTOR_CTYPE, CONST_CTYPE); #else static pg_noinline void FUNCTION_NAME(PREDICATE_NAME, VECTOR_CTYPE, CONST_CTYPE)(const ArrowArray *arrow, const Datum constdatum, uint64 *restrict result) { const size_t n = arrow->length; /* Now run the predicate itself. */ const CONST_CTYPE constvalue = CONST_CONVERSION(constdatum); const VECTOR_CTYPE *vector = (const VECTOR_CTYPE *) arrow->buffers[1]; for (size_t outer = 0; outer < n / 64; outer++) { /* no need to check the values if the result is already invalid */ if (result[outer] == 0) continue; uint64 word = 0; for (size_t inner = 0; inner < 64; inner++) { const bool valid = PREDICATE_EXPRESSION(vector[outer * 64 + inner], constvalue); word |= ((uint64) valid) << inner; } result[outer] &= word; } if (n % 64) { uint64 tail_word = 0; for (size_t i = (n / 64) * 64; i < n; i++) { const bool valid = PREDICATE_EXPRESSION(vector[i], constvalue); tail_word |= ((uint64) valid) << (i % 64); } result[n / 64] &= tail_word; } } #endif #undef PG_PREDICATE_HELPER #undef FUNCTION_NAME #undef FUNCTION_NAME_HELPER #undef PREDICATE_EXPRESSION #undef PREDICATE_NAME ================================================ FILE: tsl/src/nodes/columnar_scan/pred_vector_const_arithmetic_type_pair.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Vector-const predicates for one pair of arithmetic types. For NaN comparison, * Postgres has its own nonstandard rules different from the IEEE floats. */ #define PREDICATE_NAME GE #define PREDICATE_EXPRESSION(X, Y) (isnan((double) (X)) || (!isnan((double) (Y)) && (X) >= (Y))) #include "pred_vector_const_arithmetic_single.c" #define PREDICATE_NAME LE #define PREDICATE_EXPRESSION(X, Y) (isnan((double) (Y)) || (!isnan((double) (X)) && (X) <= (Y))) #include "pred_vector_const_arithmetic_single.c" #define PREDICATE_NAME LT #define PREDICATE_EXPRESSION(X, Y) (!isnan((double) (X)) && (isnan((double) (Y)) || (X) < (Y))) #include "pred_vector_const_arithmetic_single.c" #define PREDICATE_NAME GT #define PREDICATE_EXPRESSION(X, Y) (!isnan((double) (Y)) && (isnan((double) (X)) || (X) > (Y))) #include "pred_vector_const_arithmetic_single.c" #define PREDICATE_NAME EQ #define PREDICATE_EXPRESSION(X, Y) (isnan((double) (X)) ? isnan((double) (Y)) : ((X) == (Y))) #include "pred_vector_const_arithmetic_single.c" #define PREDICATE_NAME NE #define PREDICATE_EXPRESSION(X, Y) (isnan((double) (X)) ? !isnan((double) (Y)) : ((X) != (Y))) #include "pred_vector_const_arithmetic_single.c" #undef VECTOR_CTYPE #undef CONST_CTYPE #undef CONST_CONVERSION #undef PG_PREDICATE ================================================ FILE: tsl/src/nodes/columnar_scan/qual_pushdown.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <optimizer/optimizer.h> #include <optimizer/restrictinfo.h> #include <parser/parse_func.h> #include <parser/parsetree.h> #include <utils/builtins.h> #include <utils/typcache.h> #include "columnar_scan.h" #include "compression/batch_metadata_builder.h" #include "compression/create.h" #include "compression/sparse_index_bloom1.h" #include "custom_type_cache.h" #include "guc.h" #include "ts_catalog/array_utils.h" #include "qual_pushdown.h" typedef struct QualPushdownContext { RelOptInfo *chunk_rel; RelOptInfo *compressed_rel; RangeTblEntry *chunk_rte; RangeTblEntry *compressed_rte; CompressionSettings *settings; /* * This is actually the result, not the static input context like above, but * there's no way to separate this properly using the expression tree mutator * interface. */ bool can_pushdown; bool needs_recheck; } QualPushdownContext; static QualPushdownContext copy_context(const QualPushdownContext *source) { QualPushdownContext copy; copy = *source; copy.can_pushdown = true; copy.needs_recheck = false; return copy; } static Node *qual_pushdown_mutator(Node *node, QualPushdownContext *context); /* * Result of validating an OpExpr as a potential bloom filter candidate. * Does NOT make decisions about which operand to use. */ typedef struct HashableEqualityInfo { Var *left_var; /* NULL if left is not a Var on chunk_rel */ Var *right_var; /* NULL if right is not a Var on chunk_rel */ Expr *left_expr; /* Original left operand (after unwrapping RelabelType) */ Expr *right_expr; /* Original right operand (after unwrapping RelabelType) */ Oid opno; /* Original operator OID */ bool left_hashable; /* Is operator in left_var's hash opfamily? */ bool right_hashable; /* Is operator in right_var's hash opfamily? */ bool valid; /* Is this a valid hashable equality? */ } HashableEqualityInfo; static HashableEqualityInfo validate_hashable_equality(OpExpr *opexpr, QualPushdownContext *context); static Var *extract_var_for_bloom1(OpExpr *opexpr, QualPushdownContext *context, Expr **value_out, Oid *op_oid_out); static Var *extract_var_for_composite_bloom(OpExpr *opexpr, QualPushdownContext *context, Expr **value_out, Oid *op_oid_out); static void pushdown_composite_blooms(PlannerInfo *root, QualPushdownContext *context); void columnar_scan_filter_pushdown(PlannerInfo *root, CompressionSettings *settings, RelOptInfo *chunk_rel, RelOptInfo *compressed_rel, bool chunk_partial) { ListCell *lc; List *decompress_clauses = NIL; QualPushdownContext base_context = { .chunk_rel = chunk_rel, .compressed_rel = compressed_rel, .chunk_rte = planner_rt_fetch(chunk_rel->relid, root), .compressed_rte = planner_rt_fetch(compressed_rel->relid, root), .settings = settings, }; /* * Collect composite bloom candidates first. * This looks at ALL equality predicates together to find composite bloom matches * and push down the composite bloom filters. */ if (ts_guc_enable_sparse_index_bloom && settings != NULL && settings->fd.index != NULL && ts_guc_enable_composite_bloom_indexes) { pushdown_composite_blooms(root, &base_context); } foreach (lc, chunk_rel->baserestrictinfo) { RestrictInfo *ri = lfirst(lc); QualPushdownContext clause_context = copy_context(&base_context); Node *pushed_down = qual_pushdown_mutator((Node *) ri->clause, &clause_context); if (clause_context.can_pushdown) { /* * We have to call eval_const_expressions after pushing down * the quals, to normalize the bool expressions. Namely, we might add an * AND boolexpr on minmax metadata columns, but the normal form is not * allowed to have nested AND boolexprs. They break some functions like * generate_bitmap_or_paths(). */ pushed_down = eval_const_expressions(root, pushed_down); if (IsA(pushed_down, BoolExpr) && castNode(BoolExpr, pushed_down)->boolop == AND_EXPR) { /* have to separate out and expr into different restrict infos */ ListCell *lc_and; BoolExpr *bool_expr = castNode(BoolExpr, pushed_down); foreach (lc_and, bool_expr->args) { compressed_rel->baserestrictinfo = lappend(compressed_rel->baserestrictinfo, make_simple_restrictinfo(root, lfirst(lc_and))); } } else compressed_rel->baserestrictinfo = lappend(compressed_rel->baserestrictinfo, make_simple_restrictinfo(root, (Expr *) pushed_down)); } /* * We need to check the restriction clause on the decompress node if the clause can't be * pushed down or needs re-checking. */ if (!clause_context.can_pushdown || clause_context.needs_recheck || chunk_partial) { decompress_clauses = lappend(decompress_clauses, ri); } } chunk_rel->baserestrictinfo = decompress_clauses; } static OpExpr * make_segment_meta_opexpr(QualPushdownContext *context, Oid opno, AttrNumber meta_column_attno, Var *uncompressed_var, Expr *compare_to_expr, StrategyNumber strategy) { Var *meta_var = makeVar(context->compressed_rel->relid, meta_column_attno, uncompressed_var->vartype, -1, InvalidOid, 0); return (OpExpr *) make_opclause(opno, BOOLOID, false, (Expr *) meta_var, copyObject(compare_to_expr), InvalidOid, uncompressed_var->varcollid); } static void expr_fetch_minmax_metadata(QualPushdownContext *context, Expr *expr, AttrNumber *min_attno, AttrNumber *max_attno) { *min_attno = InvalidAttrNumber; *max_attno = InvalidAttrNumber; if (!IsA(expr, Var)) return; Var *var = castNode(Var, expr); /* * Not on the chunk we expect. This doesn't really happen because we don't * push down the join quals, only the baserestrictinfo. */ if ((Index) var->varno != context->chunk_rel->relid) return; /* ignore system attributes or whole row references */ if (var->varattno <= 0) return; *min_attno = compressed_column_metadata_attno(context->settings, context->chunk_rte->relid, var->varattno, context->compressed_rte->relid, "min"); *max_attno = compressed_column_metadata_attno(context->settings, context->chunk_rte->relid, var->varattno, context->compressed_rte->relid, "max"); } static void * pushdown_op_to_segment_meta_min_max(QualPushdownContext *context, OpExpr *orig_opexpr) { /* * This always requires rechecking the decompressed data. */ context->needs_recheck = true; List *expr_args = orig_opexpr->args; Assert(list_length(expr_args) == 2); Expr *orig_leftop = linitial(expr_args); Expr *orig_rightop = lsecond(expr_args); if (IsA(orig_leftop, RelabelType)) orig_leftop = ((RelabelType *) orig_leftop)->arg; if (IsA(orig_rightop, RelabelType)) orig_rightop = ((RelabelType *) orig_rightop)->arg; /* Find the side that has var with segment meta set expr to the other side */ Oid op_oid = orig_opexpr->opno; AttrNumber min_attno; AttrNumber max_attno; expr_fetch_minmax_metadata(context, orig_leftop, &min_attno, &max_attno); if (min_attno == InvalidAttrNumber || max_attno == InvalidAttrNumber) { /* No metadata for the left operand, try to commute the operator. */ op_oid = get_commutator(op_oid); Expr *tmp = orig_leftop; orig_leftop = orig_rightop; orig_rightop = tmp; expr_fetch_minmax_metadata(context, orig_leftop, &min_attno, &max_attno); } if (min_attno == InvalidAttrNumber || max_attno == InvalidAttrNumber) { /* No metadata for either operand. */ context->can_pushdown = false; return orig_opexpr; } Var *var_with_segment_meta = castNode(Var, orig_leftop); /* May be able to allow non-strict operations as well. * Next steps: Think through edge cases, either allow and write tests or figure out why we must * block strict operations */ if (!OidIsValid(op_oid) || !op_strict(op_oid)) { context->can_pushdown = false; return orig_opexpr; } /* If the collation to be used by the OP doesn't match the column's collation do not push down * as the materialized min/max value do not match the semantics of what we need here */ Oid op_collation = orig_opexpr->inputcollid; if (var_with_segment_meta->varcollid != op_collation) { context->can_pushdown = false; return orig_opexpr; } TypeCacheEntry *tce = lookup_type_cache(var_with_segment_meta->vartype, TYPECACHE_BTREE_OPFAMILY); const int strategy = get_op_opfamily_strategy(op_oid, tce->btree_opf); if (strategy == InvalidStrategy) { context->can_pushdown = false; return orig_opexpr; } /* * Check if the righthand expression is safe to push down. We cannot combine * it with the original operator if there can be false negatives. */ QualPushdownContext tmp_context = copy_context(context); Expr *pushed_down_rightop = (Expr *) qual_pushdown_mutator((Node *) orig_rightop, &tmp_context); if (!tmp_context.can_pushdown || tmp_context.needs_recheck) { context->can_pushdown = false; return orig_opexpr; } Assert(pushed_down_rightop != NULL); const Oid expr_type_id = exprType((Node *) pushed_down_rightop); switch (strategy) { case BTEqualStrategyNumber: { /* var = expr implies min < expr and max > expr */ Oid opno_le = get_opfamily_member(tce->btree_opf, tce->type_id, expr_type_id, BTLessEqualStrategyNumber); Oid opno_ge = get_opfamily_member(tce->btree_opf, tce->type_id, expr_type_id, BTGreaterEqualStrategyNumber); if (!OidIsValid(opno_le) || !OidIsValid(opno_ge)) { /* * Shouldn't be possible if we managed to create the min/max * sparse index, but defend against catalog corruption. */ context->can_pushdown = false; return orig_opexpr; } return make_andclause( list_make2(make_segment_meta_opexpr(context, opno_le, min_attno, var_with_segment_meta, pushed_down_rightop, BTLessEqualStrategyNumber), make_segment_meta_opexpr(context, opno_ge, max_attno, var_with_segment_meta, pushed_down_rightop, BTGreaterEqualStrategyNumber))); } case BTLessStrategyNumber: case BTLessEqualStrategyNumber: /* var < expr implies min < expr */ { Oid opno = get_opfamily_member(tce->btree_opf, tce->type_id, expr_type_id, strategy); if (!OidIsValid(opno)) { /* * Shouldn't be possible if we managed to create the min/max * sparse index, but defend against catalog corruption. */ context->can_pushdown = false; return orig_opexpr; } return (Expr *) make_segment_meta_opexpr(context, opno, min_attno, var_with_segment_meta, pushed_down_rightop, strategy); } case BTGreaterStrategyNumber: case BTGreaterEqualStrategyNumber: /* var > expr implies max > expr */ { Oid opno = get_opfamily_member(tce->btree_opf, tce->type_id, expr_type_id, strategy); if (!OidIsValid(opno)) { /* * Shouldn't be possible if we managed to create the min/max * sparse index, but defend against catalog corruption. */ context->can_pushdown = false; return orig_opexpr; } return (Expr *) make_segment_meta_opexpr(context, opno, max_attno, var_with_segment_meta, pushed_down_rightop, strategy); } default: context->can_pushdown = false; return orig_opexpr; } } static void expr_fetch_bloom1_metadata(QualPushdownContext *context, Expr *expr, AttrNumber *bloom1_attno) { *bloom1_attno = InvalidAttrNumber; if (!IsA(expr, Var)) return; Var *var = castNode(Var, expr); /* * Not on the chunk we expect. This doesn't really happen because we don't * push down the join quals, only the baserestrictinfo. */ if ((Index) var->varno != context->chunk_rel->relid) return; /* ignore system attributes or whole row references */ if (var->varattno <= 0) return; *bloom1_attno = compressed_column_metadata_attno(context->settings, context->chunk_rte->relid, var->varattno, context->compressed_rte->relid, bloom1_column_prefix); if (*bloom1_attno == InvalidAttrNumber && ts_guc_read_legacy_bloom1_v1) { /* * The version 1 of bloom1 indexes is disabled by default because its * hashing was dependent on build options leading to corrupt indexes, * but can be enabled manually. */ *bloom1_attno = compressed_column_metadata_attno(context->settings, context->chunk_rte->relid, var->varattno, context->compressed_rte->relid, "bloom1"); } } /* * Validate an OpExpr as a hashable equality predicate. * * Does NOT: * - Decide which operand is "column" vs "value" * - Check bloom metadata * - Check collation (Caller's responsibility - depends on which Var is chosen) * - Commute the operator * * DOES validate: * - OpExpr structure * - Var identification on chunk_rel * - Hash operator validity for each Var * * Returns info about both operands with validation flags. * Caller decides which Var to use and validates collation. */ static HashableEqualityInfo validate_hashable_equality(OpExpr *opexpr, QualPushdownContext *context) { Assert(opexpr != NULL); Assert(context != NULL); HashableEqualityInfo info = { 0 }; info.valid = false; info.left_hashable = false; info.right_hashable = false; if (list_length(opexpr->args) != 2) return info; Expr *left = linitial(opexpr->args); Expr *right = lsecond(opexpr->args); /* Unwrap RelabelType */ if (IsA(left, RelabelType)) left = ((RelabelType *) left)->arg; if (IsA(right, RelabelType)) right = ((RelabelType *) right)->arg; info.left_expr = left; info.right_expr = right; info.opno = opexpr->opno; /* Must have valid operator OID */ if (!OidIsValid(info.opno)) return info; /* Identify Vars on our relation and validate hash operator for each */ if (IsA(left, Var)) { Var *left_var = (Var *) left; if ((Index) left_var->varno == context->chunk_rel->relid && left_var->varattno > 0) { info.left_var = left_var; /* Check if operator is hashable equality for this type */ TypeCacheEntry *tce = lookup_type_cache(left_var->vartype, TYPECACHE_HASH_OPFAMILY); if (OidIsValid(tce->hash_opf)) { int strategy = get_op_opfamily_strategy(info.opno, tce->hash_opf); if (strategy == HTEqualStrategyNumber) info.left_hashable = true; } } } if (IsA(right, Var)) { Var *right_var = (Var *) right; if ((Index) right_var->varno == context->chunk_rel->relid && right_var->varattno > 0) { info.right_var = right_var; /* Check if operator is hashable equality for this type */ TypeCacheEntry *tce = lookup_type_cache(right_var->vartype, TYPECACHE_HASH_OPFAMILY); if (OidIsValid(tce->hash_opf)) { int strategy = get_op_opfamily_strategy(info.opno, tce->hash_opf); if (strategy == HTEqualStrategyNumber) info.right_hashable = true; } } } /* Must have at least one Var on our relation */ if (info.left_var == NULL && info.right_var == NULL) return info; /* Must have at least one Var that passes hashable equality check */ if (!info.left_hashable && !info.right_hashable) return info; info.valid = true; return info; } /* * Extract Var for single-column bloom filter pushdown. * Uses bloom metadata presence to decide which operand to use. * * This handles cases like: * - bloom_col = 5 (left has bloom) * - 5 = bloom_col (right has bloom, commute) * - bloom_col = segmentby_col (left has bloom, caller validates segmentby) * - segmentby_col = bloom_col (right has bloom, commute, caller validates) * - bloom_col1 = bloom_col2 (left has bloom, caller validates bloom_col2: FAILS) * * Returns the Var that has single-column bloom metadata, along with * the value expression and (possibly commuted) operator. */ static Var * extract_var_for_bloom1(OpExpr *opexpr, QualPushdownContext *context, Expr **value_out, Oid *op_oid_out) { Assert(value_out != NULL); Assert(op_oid_out != NULL); Assert(opexpr != NULL); Assert(context != NULL); *value_out = NULL; *op_oid_out = InvalidOid; /* Validate the expression */ HashableEqualityInfo info = validate_hashable_equality(opexpr, context); if (!info.valid) return NULL; /* Try to find a Var with bloom metadata that passes hash operator validation. */ Var *chosen_var = NULL; Expr *value_expr_tmp = NULL; Oid op_oid_tmp; AttrNumber bloom1_attno = InvalidAttrNumber; if (info.left_var != NULL && info.left_hashable) { expr_fetch_bloom1_metadata(context, (Expr *) info.left_var, &bloom1_attno); if (bloom1_attno != InvalidAttrNumber) { /* Left has bloom metadata and valid hash operator. */ chosen_var = info.left_var; value_expr_tmp = info.right_expr; op_oid_tmp = info.opno; } } /* If left didn't qualify, try right. */ if (chosen_var == NULL && info.right_var != NULL && info.right_hashable) { expr_fetch_bloom1_metadata(context, (Expr *) info.right_var, &bloom1_attno); if (bloom1_attno != InvalidAttrNumber) { /* Right has bloom metadata and valid hash operator. Need commutation. */ chosen_var = info.right_var; value_expr_tmp = info.left_expr; op_oid_tmp = get_commutator(info.opno); } } if (chosen_var == NULL) { /* No Var with both bloom metadata and valid hash operator */ return NULL; } /* Validate collation for the chosen Var */ Oid op_collation = opexpr->inputcollid; if (chosen_var->varcollid != op_collation) { /* Collation mismatch - bloom filter hash won't match operator hash */ return NULL; } /* Cannot use non-deterministic collations */ if (OidIsValid(op_collation) && !get_collation_isdeterministic(op_collation)) return NULL; *value_out = value_expr_tmp; *op_oid_out = op_oid_tmp; return chosen_var; } static void * pushdown_op_to_segment_meta_bloom1(QualPushdownContext *context, OpExpr *orig_opexpr) { /* * This always requires rechecking the decompressed data. */ context->needs_recheck = true; /* * Use single-column bloom helper to find Var with bloom metadata. * Helper returns first Var with bloom metadata. */ Expr *orig_rightop = NULL; Oid op_oid; Var *var = extract_var_for_bloom1(orig_opexpr, context, &orig_rightop, &op_oid); if (var == NULL) { context->can_pushdown = false; return orig_opexpr; } /* Get bloom metadata. */ AttrNumber bloom1_attno = InvalidAttrNumber; expr_fetch_bloom1_metadata(context, (Expr *) var, &bloom1_attno); Assert(bloom1_attno != InvalidAttrNumber); /* * The hash equality operators are supposed to be strict. */ Assert(op_strict(op_oid)); /* * Check if the righthand expression is safe to push down. We cannot combine * it with the original operator if there can be false negatives. */ QualPushdownContext tmp_context = copy_context(context); Expr *pushed_down_rightop = (Expr *) qual_pushdown_mutator((Node *) orig_rightop, &tmp_context); if (!tmp_context.can_pushdown || tmp_context.needs_recheck) { context->can_pushdown = false; return orig_opexpr; } Assert(pushed_down_rightop != NULL); /* * We can have cross-type equality operator, but in this case the our hashes * or Postgres hashes for the respective types are guaranteed to have the * same result for both types, so we don't need any type conversion here. * The only special case is composite types. The right-hand constant would * have the anonymous type "record" and would be compared polymorphically * at runtime with the record_eq() function. However, this type doesn't have * an extended hash function. Just refuse to work with it. */ const Oid compared_type = exprType((Node *) pushed_down_rightop); if (compared_type == RECORDOID) { context->can_pushdown = false; return orig_opexpr; } /* * var = expr implies bloom1_contains(var_bloom, expr). */ Var *bloom_var = makeVar(context->compressed_rel->relid, bloom1_attno, ts_custom_type_cache_get(CUSTOM_TYPE_BLOOM1)->type_oid, -1, InvalidOid, 0); Oid func = LookupFuncName(list_make2(makeString("_timescaledb_functions"), makeString("bloom1_contains")), /* nargs = */ -1, /* argtypes = */ (void *) -1, /* missing_ok = */ false); return (Expr *) makeFuncExpr(func, BOOLOID, list_make2(bloom_var, pushed_down_rightop), /* funccollid = */ InvalidOid, /* inputcollid = */ InvalidOid, COERCE_EXPLICIT_CALL); } /* * Try to transform x = any(array[]) into bloom1_contains_any(bloom_x, array[]). */ static void * pushdown_saop_bloom1(QualPushdownContext *context, ScalarArrayOpExpr *orig_saop) { /* * This always requires rechecking the decompressed data. */ context->needs_recheck = true; if (!orig_saop->useOr) { context->can_pushdown = false; return orig_saop; } List *expr_args = orig_saop->args; Assert(list_length(expr_args) == 2); Expr *orig_leftop = linitial(expr_args); Expr *orig_rightop = lsecond(expr_args); if (IsA(orig_leftop, RelabelType)) orig_leftop = ((RelabelType *) orig_leftop)->arg; if (IsA(orig_rightop, RelabelType)) orig_rightop = ((RelabelType *) orig_rightop)->arg; /* * For scalar array operation, we expect a var on the left side. */ AttrNumber bloom1_attno = InvalidAttrNumber; expr_fetch_bloom1_metadata(context, orig_leftop, &bloom1_attno); if (bloom1_attno == InvalidAttrNumber) { /* No metadata for left operand. */ context->can_pushdown = false; return orig_saop; } Var *var_with_segment_meta = castNode(Var, orig_leftop); /* * Play it safe and don't push down if the operator collation doesn't match * the column collation. */ Oid op_collation = orig_saop->inputcollid; if (var_with_segment_meta->varcollid != op_collation) { context->can_pushdown = false; return orig_saop; } /* * We cannot use bloom filters for non-deterministic collations. */ if (OidIsValid(op_collation) && !get_collation_isdeterministic(op_collation)) { context->can_pushdown = false; return orig_saop; } /* * We only support hashable equality operators. */ const Oid op_oid = orig_saop->opno; TypeCacheEntry *tce = lookup_type_cache(var_with_segment_meta->vartype, TYPECACHE_HASH_OPFAMILY); const int strategy = get_op_opfamily_strategy(op_oid, tce->hash_opf); if (strategy != HTEqualStrategyNumber) { context->can_pushdown = false; return orig_saop; } /* * The hash equality operators are supposed to be strict. */ Assert(op_strict(op_oid)); /* * Check if the righthand expression is safe to push down. We cannot combine * it with the original operator if there can be false negatives. */ QualPushdownContext tmp_context = copy_context(context); Expr *pushed_down_rightop = (Expr *) qual_pushdown_mutator((Node *) orig_rightop, &tmp_context); if (!tmp_context.can_pushdown || tmp_context.needs_recheck) { context->can_pushdown = false; return orig_saop; } Assert(pushed_down_rightop != NULL); /* * var = any(array) implies bloom1_contains_any(var_bloom, array). */ Var *bloom_var = makeVar(context->compressed_rel->relid, bloom1_attno, ts_custom_type_cache_get(CUSTOM_TYPE_BLOOM1)->type_oid, -1, InvalidOid, 0); Oid func = LookupFuncName(list_make2(makeString("_timescaledb_functions"), makeString("bloom1_contains_any")), /* nargs = */ -1, /* argtypes = */ (void *) -1, /* missing_ok = */ false); return makeFuncExpr(func, BOOLOID, list_make2(bloom_var, pushed_down_rightop), /* funccollid = */ InvalidOid, /* inputcollid = */ InvalidOid, COERCE_EXPLICIT_CALL); } /* * Extract Var for composite bloom filter pushdown. * Uses segmentby membership as a heuristic for Var-to-Var cases. * Does NOT check bloom metadata (composite bloom is checked later by name matching). * * This handles cases like: * - col = 5 (obvious) * - 5 = col (commute) * - col = segmentby_col (prefer non-segmentby col) * - col1 = col2 (prefer non-segmentby, but caller validates value) * * IMPORTANT: For col1 = col2 where both are non-segmentby, returns col1 but * the caller (pushdown_composite_blooms) will reject col2 during value validation. * Composite bloom requires value expressions to be constants, params, or segmentby Vars. * * Returns a Var along with the value expression and (possibly commuted) operator. */ static Var * extract_var_for_composite_bloom(OpExpr *opexpr, QualPushdownContext *context, Expr **value_out, Oid *op_oid_out) { Assert(opexpr != NULL); Assert(context != NULL); Assert(value_out != NULL); Assert(op_oid_out != NULL); *value_out = NULL; *op_oid_out = InvalidOid; /* Validate the expression */ HashableEqualityInfo info = validate_hashable_equality(opexpr, context); if (!info.valid) return NULL; /* Only one side is a Var. */ Var *chosen_var = NULL; Expr *value_expr_tmp = NULL; Oid op_oid_tmp; bool value_is_segmentby = false; if (info.left_var != NULL && info.right_var == NULL) { chosen_var = info.left_var; value_expr_tmp = info.right_expr; op_oid_tmp = info.opno; } else if (info.right_var != NULL && info.left_var == NULL) { chosen_var = info.right_var; value_expr_tmp = info.left_expr; op_oid_tmp = get_commutator(info.opno); } else if (info.left_var != NULL && info.right_var != NULL) { /* * Both are Vars. Prefer non-segmentby Var (segmentby columns cannot * be in bloom filters), but also require valid hash operator. */ bool left_is_segmentby = false; bool right_is_segmentby = false; if (context->settings && context->settings->fd.segmentby) { char *left_attname = get_attname(context->chunk_rte->relid, info.left_var->varattno, false); left_is_segmentby = ts_array_is_member(context->settings->fd.segmentby, left_attname); char *right_attname = get_attname(context->chunk_rte->relid, info.right_var->varattno, false); right_is_segmentby = ts_array_is_member(context->settings->fd.segmentby, right_attname); } /* Try candidates in preference order: non-segmentby+hashable, then segmentby+hashable */ if (!right_is_segmentby && info.right_hashable) { /* Right is non-segmentby and hashable - prefer it */ chosen_var = info.right_var; value_expr_tmp = info.left_expr; op_oid_tmp = get_commutator(info.opno); value_is_segmentby = left_is_segmentby; } else if (!left_is_segmentby && info.left_hashable) { /* Left is non-segmentby and hashable - use it */ chosen_var = info.left_var; value_expr_tmp = info.right_expr; op_oid_tmp = info.opno; value_is_segmentby = right_is_segmentby; } else if (info.left_hashable) { /* Left is hashable (may be segmentby) - use it as fallback */ chosen_var = info.left_var; value_expr_tmp = info.right_expr; op_oid_tmp = info.opno; value_is_segmentby = right_is_segmentby; } else if (info.right_hashable) { /* Right is hashable (may be segmentby) - use it as last resort */ chosen_var = info.right_var; value_expr_tmp = info.left_expr; op_oid_tmp = get_commutator(info.opno); value_is_segmentby = left_is_segmentby; } else { /* Neither Var has valid hash operator */ return NULL; } } /* Validate the chosen Var and value expression */ if (chosen_var != NULL) { /* Check collation */ Oid op_collation = opexpr->inputcollid; if (chosen_var->varcollid != op_collation) { /* Collation mismatch. */ return NULL; } /* Cannot use non-deterministic collations */ if (OidIsValid(op_collation) && !get_collation_isdeterministic(op_collation)) return NULL; /* * Reject non-segmentby Vars in value expression. * For composite bloom, value expressions must be pushable (const, param, or segmentby Var). * Non-segmentby Vars need decompression and cannot be used in bloom checks. * * Note: value_is_segmentby is only set when both sides are Vars (both-Vars case). * In that case, value_expr_tmp is always a Var on chunk_rel. If value_is_segmentby * is false in the both-Vars case, the value Var is non-segmentby and must be rejected. * In the single-Var case, value_expr_tmp is not a Var on chunk_rel, so this check * doesn't apply (value_is_segmentby remains false but value_expr_tmp is not a Var). */ if (IsA(value_expr_tmp, Var)) { Var *value_var = (Var *) value_expr_tmp; /* Only check segmentby for Vars on our relation */ if ((Index) value_var->varno == context->chunk_rel->relid && value_var->varattno > 0 && !value_is_segmentby) { /* Value is a non-segmentby Var on our relation - cannot be pushed */ return NULL; } } *value_out = value_expr_tmp; *op_oid_out = op_oid_tmp; return chosen_var; } return NULL; } /* * Scan baserestrictinfo for composite bloom opportunities. * For each applicable composite bloom, generate bloom1_contains(bloom, ROW(...)). * * Scans predicates first, then parses settings only if needed. */ static void pushdown_composite_blooms(PlannerInfo *root, QualPushdownContext *context) { Assert(root != NULL); Assert(context != NULL); /* We need settings to generate composite blooms. */ CompressionSettings *settings = context->settings; if (settings == NULL || settings->fd.index == NULL) { return; } /* We need at least two baserestrictinfo to have a chance to push down a composite bloom filter. */ if (list_length(context->chunk_rel->baserestrictinfo) < 2) { return; } ListCell *lc; Bitmapset *var_attnos = NULL; /* Build map: chunk_attno -> value_expr for equality predicates. */ /* Note that we may have more than one predicate for the same attribute * (e.g. col1 = 1 AND col1 = 2) which is a contradiction and we should * be able to detect and optimize for this, meaning that this predicate * will always be false. This is a TODO. * * For now, we will just use the last one, which will filter out some * chunks which is better than nothing. */ AttrNumber max_attno = context->chunk_rel->max_attr; Expr **attno_to_value = palloc0((max_attno + 1) * sizeof(Expr *)); /* Determine vars with equality predicates. */ foreach (lc, context->chunk_rel->baserestrictinfo) { RestrictInfo *ri = lfirst_node(RestrictInfo, lc); if (!IsA(ri->clause, OpExpr)) continue; Expr *value = NULL; Oid op_oid = InvalidOid; Var *var = extract_var_for_composite_bloom(castNode(OpExpr, ri->clause), context, &value, &op_oid); if (var != NULL && value != NULL) { var_attnos = bms_add_member(var_attnos, var->varattno); attno_to_value[var->varattno] = value; } } /* Check if not enough vars with equality predicates. */ if (bms_num_members(var_attnos) < 2) return; /* Parse settings to get per-column compression settings. */ SparseIndexSettings *parsed = ts_convert_to_sparse_index_settings(settings->fd.index); if (parsed == NULL) { bms_free(var_attnos); pfree(attno_to_value); return; } /* For each sparse index object, resolve the columns to attribute numbers. */ TsBmsList per_column_attnos = ts_resolve_columns_to_attnos_from_parsed_settings(parsed, context->chunk_rte->relid); /* This bitmap tells which sparse index objects are candidates for composite bloom filters. */ Bitmapset *composite_filter_candidates_ids = NULL; /* Iterate over the resolved columns and check if they match the vars with equality predicates. */ ListCell *attno_cell = NULL; int sparse_index_obj_id = -1; foreach (attno_cell, per_column_attnos) { sparse_index_obj_id++; Bitmapset *attnos = lfirst(attno_cell); /* Only care about sparse indices with at least 2 columns. */ if (bms_num_members(attnos) < 2) { continue; } if (bms_is_subset(attnos, var_attnos)) { /* This sparse index object matches the vars with equality predicates. */ SparseIndexSettingsObject *obj = list_nth(parsed->objects, sparse_index_obj_id); Assert(obj != NULL); if (obj == NULL) { continue; } /* Get the index type from the parsed object. */ List *index_type = ts_get_values_by_key_from_parsed_object(obj, ts_sparse_index_common_keys [SparseIndexKeyType] /* "type" */); Assert(index_type != NIL && list_length(index_type) == 1); if (index_type == NIL || list_length(index_type) != 1) { continue; } /* Check that it is a bloom index. */ const char *index_type_str = lfirst(list_head(index_type)); if (strcmp(index_type_str, ts_sparse_index_type_names[_SparseIndexTypeEnumBloom] /* "bloom" */) != 0) { continue; } Assert(list_length(ts_get_column_names_from_parsed_object(obj)) >= 2); composite_filter_candidates_ids = bms_add_member(composite_filter_candidates_ids, sparse_index_obj_id); } } /* Check if there are composite filter candidates. */ if (bms_is_empty(composite_filter_candidates_ids)) { pfree(attno_to_value); bms_free(var_attnos); ts_bmslist_free(per_column_attnos); ts_free_sparse_index_settings(parsed); return; } /* For each composite filter candidate, build the composite bloom filter. */ int candidate_filter_id = -1; while ((candidate_filter_id = bms_next_member(composite_filter_candidates_ids, candidate_filter_id)) >= 0) { SparseIndexSettingsObject *obj = list_nth(parsed->objects, candidate_filter_id); Assert(obj != NULL); /* The column attnos generated from the parsed object is a list indexed by the object id.*/ Bitmapset *column_attnos = list_nth(per_column_attnos, candidate_filter_id); Assert(bms_num_members(column_attnos) >= 2); /* Iterate over the attnos for the current candidate filter an check if this is a valid * predicate to push down. This is a safety check and hope that the composite bloom filter * will also be valid to push down by induction. */ List *pushed_value_exprs = NIL; bool all_valid = true; int col_attno = -1; while ((col_attno = bms_next_member(column_attnos, col_attno)) >= 0) { Expr *value_expr = attno_to_value[col_attno]; Assert(value_expr != NULL); /* Validate the value expression via qual_pushdown_mutator. */ QualPushdownContext tmp_context = copy_context(context); Expr *pushed_value = (Expr *) qual_pushdown_mutator((Node *) value_expr, &tmp_context); if (!tmp_context.can_pushdown || tmp_context.needs_recheck) { all_valid = false; break; } pushed_value_exprs = lappend(pushed_value_exprs, pushed_value); } if (!all_valid) { continue; } List *column_names = ts_get_column_names_from_parsed_object(obj); Assert(list_length(column_names) >= 2 && list_length(column_names) <= MAX_BLOOM_FILTER_COLUMNS); /* Check if this chunk has the composite bloom column */ char *composite_col_name = compressed_column_metadata_name_list_v2(bloom1_column_prefix, column_names); AttrNumber composite_attno = get_attnum(context->compressed_rte->relid, composite_col_name); pfree(composite_col_name); if (!AttributeNumberIsValid(composite_attno)) { continue; } /* Build bloom1_contains(composite_bloom, ROW(...)) */ Var *bloom_var = makeVar(context->compressed_rel->relid, composite_attno, ts_custom_type_cache_get(CUSTOM_TYPE_BLOOM1)->type_oid, -1, InvalidOid, 0); /* If all pushed-down values are Const, pre-hash at planning time. */ FuncExpr *bloom_check = NULL; /* Build ROW(val1, val2, ...) expression using pushed-down values */ RowExpr *row_expr = makeNode(RowExpr); row_expr->args = pushed_value_exprs; row_expr->row_typeid = RECORDOID; row_expr->row_format = COERCE_IMPLICIT_CAST; row_expr->colnames = NIL; row_expr->location = -1; Oid func_oid = LookupFuncName(list_make2(makeString("_timescaledb_functions"), makeString("bloom1_contains")), -1, (void *) -1, false); bloom_check = makeFuncExpr(func_oid, BOOLOID, list_make2(bloom_var, row_expr), InvalidOid, InvalidOid, COERCE_EXPLICIT_CALL); /* Add to baserestrictinfo. */ context->compressed_rel->baserestrictinfo = lappend(context->compressed_rel->baserestrictinfo, make_simple_restrictinfo(root, (Expr *) bloom_check)); } /* Cleanup */ bms_free(var_attnos); bms_free(composite_filter_candidates_ids); ts_free_sparse_index_settings(parsed); ts_bmslist_free(per_column_attnos); pfree(attno_to_value); } /* * Deconstruct a Const of array type into a list of the array values. */ static List * deconstruct_array_const(Const *array_const) { /* * No way to represent that as a list (NIL is an empty array), so has to be * handled by the caller. */ Assert(!array_const->constisnull); Oid array_type = array_const->consttype; Datum array_datum = array_const->constvalue; Oid element_type = get_element_type(array_type); Assert(OidIsValid(element_type)); int16 typlen; bool typbyval; char typalign; get_typlenbyvalalign(element_type, &typlen, &typbyval, &typalign); int nelems; Datum *elem_values; bool *elem_nulls; deconstruct_array(DatumGetArrayTypeP(array_datum), element_type, typlen, typbyval, typalign, &elem_values, &elem_nulls, &nelems); List *const_list = NIL; for (int i = 0; i < nelems; i++) { Const *elem_const = makeConst(element_type, array_const->consttypmod, array_const->constcollid, typlen, elem_values[i], elem_nulls[i], typbyval); const_list = lappend(const_list, elem_const); } return const_list; } /* * Push down the scalar array operation by transforming it into a series of * OR/AND clauses. */ static Expr * pushdown_saop_boolexpr(QualPushdownContext *context, ScalarArrayOpExpr *saop) { void *scalar_arg = linitial(saop->args); void *array_arg = list_nth(saop->args, 1); List *array_elements; if (IsA(array_arg, Const) && !castNode(Const, array_arg)->constisnull) { array_elements = deconstruct_array_const(castNode(Const, array_arg)); } else if (IsA(array_arg, ArrayExpr)) { array_elements = castNode(ArrayExpr, array_arg)->elements; } else { /* * We can encounter an array-type Param here, and maybe something else. * This function has to deconstruct the array into elements now, so * these types of array argument are not suitable. */ context->can_pushdown = false; return (Expr *) saop; } /* * This will be the operation on the scalar value and an individual array * element. */ OpExpr *opexpr = makeNode(OpExpr); opexpr->opno = saop->opno; opexpr->opfuncid = saop->opfuncid; opexpr->opresulttype = BOOLOID; opexpr->inputcollid = saop->inputcollid; /* * Try to apply the above operation for each array element. */ List *pushed_down_ops = NIL; ListCell *lc; foreach (lc, array_elements) { opexpr->args = list_make2(scalar_arg, lfirst(lc)); QualPushdownContext tmp_context = copy_context(context); void *transformed = qual_pushdown_mutator((Node *) opexpr, &tmp_context); /* * If the scalar array operation uses AND, it's correct and useful to * push down the check only for some array elements. * * For OR, we must be able to push down the checks for every element. */ if (!tmp_context.can_pushdown) { if (saop->useOr) { context->can_pushdown = false; return (Expr *) saop; } /* * If we pushed down the clause only partially, we have to mark that * it needs rechecking, even when the individual parts don't. */ context->needs_recheck = true; continue; } context->needs_recheck |= tmp_context.needs_recheck; pushed_down_ops = lappend(pushed_down_ops, transformed); } /* * We can have no pushed down clauses if: * 1) we had an AND scalar array operation, but failed to push down every * individual clause. * 2) we had an empty array argument, apparently it's not simplified by * Postgres' eval_const_expressions(). */ if (pushed_down_ops == NIL) { context->can_pushdown = false; return (Expr *) saop; } if (list_length(pushed_down_ops) == 1) return linitial(pushed_down_ops); if (saop->useOr) { return make_orclause(pushed_down_ops); } else { return make_andclause(pushed_down_ops); } } static bool contain_volatile_functions_checker(Oid func_id, void *context) { return (func_volatile(func_id) == PROVOLATILE_VOLATILE); } /* * Push down the given expression node. * * This is used as a mutator for expression_tree_mutator(). * * We return the original node if we cannot push it down, to be consistent with * the expression_tree_mutator behavior. The caller must check * context.can_pushdown. */ static Node * qual_pushdown_mutator(Node *orig_node, QualPushdownContext *context) { if (orig_node == NULL) { /* * An expression node can have a NULL field and the mutator will be * still called for it, so we have to handle this. */ return NULL; } if (!context->can_pushdown) { /* * Stop early if we already know we can't push down this filter. */ return orig_node; } if (check_functions_in_node(orig_node, contain_volatile_functions_checker, /* context = */ NULL)) { /* pushdown is not safe for volatile expressions */ context->can_pushdown = false; return orig_node; } switch (nodeTag(orig_node)) { case T_Var: { Var *var = castNode(Var, orig_node); Assert((Index) var->varno == context->chunk_rel->relid); if (var->varattno <= 0) { /* Can't do this for system columns such as whole-row var. */ context->can_pushdown = false; return orig_node; } char *attname = get_attname(context->chunk_rte->relid, var->varattno, false); /* we can only push down quals for segmentby columns */ if (!ts_array_is_member(context->settings->fd.segmentby, attname)) { context->can_pushdown = false; return orig_node; } var = copyObject(var); var->varno = context->compressed_rel->relid; var->varattno = get_attnum(context->compressed_rte->relid, attname); return (Node *) var; } case T_OpExpr: { OpExpr *opexpr = (OpExpr *) orig_node; /* * It might be possible to push down the OpExpr as is, if it * references only the segmentby columns. Check this case first. * * Note that we can't push down the entire operator if we pushed * down both sides inexactly, i.e. they require recheck. This means * we can have false positives there, and combining false positives * with the original operator could lead to false negatives, which * would be a bug. Consider for example (x = 1) = (y = 1) in case * where both sides are false, but there's a false posistive for the * pushed down version of the left side but not the right side. */ QualPushdownContext tmp_context = copy_context(context); void *pushed_down = expression_tree_mutator((Node *) orig_node, qual_pushdown_mutator, &tmp_context); if (tmp_context.can_pushdown && !tmp_context.needs_recheck) { return pushed_down; } if (opexpr->opresulttype != BOOLOID) { /* * The following pushdown options only support operators that * return bool. */ context->can_pushdown = false; return orig_node; } if (list_length(opexpr->args) != 2) { /* * The following pushdown options only support operators with * two operands. */ context->can_pushdown = false; return orig_node; } /* * Try bloom1 sparse index. */ if (ts_guc_enable_sparse_index_bloom) { tmp_context = copy_context(context); pushed_down = pushdown_op_to_segment_meta_bloom1(&tmp_context, opexpr); if (tmp_context.can_pushdown) { context->needs_recheck |= tmp_context.needs_recheck; return pushed_down; } } /* * Try minmax sparse index. */ tmp_context = copy_context(context); pushed_down = pushdown_op_to_segment_meta_min_max(&tmp_context, opexpr); if (tmp_context.can_pushdown) { context->needs_recheck |= tmp_context.needs_recheck; return pushed_down; } /* * No other options to push down the OpExpr. */ context->can_pushdown = false; return orig_node; } case T_ScalarArrayOpExpr: { /* * It can be possible to push down the scalar array operation as is, * if it references only the segmentby columns. Check this case * first. * * See the comment for OpExpr about needs_recheck handling. */ QualPushdownContext tmp_context = copy_context(context); void *pushed_down = expression_tree_mutator((Node *) orig_node, qual_pushdown_mutator, &tmp_context); if (tmp_context.can_pushdown && !tmp_context.needs_recheck) { return pushed_down; } ScalarArrayOpExpr *saop = castNode(ScalarArrayOpExpr, orig_node); /* * Try to transform x = any(array[]) into * bloom1_contains_any(bloom_x, array[]). */ if (ts_guc_enable_sparse_index_bloom) { tmp_context = *context; pushed_down = pushdown_saop_bloom1(&tmp_context, saop); if (tmp_context.can_pushdown) { context->needs_recheck |= tmp_context.needs_recheck; return pushed_down; } } /* * Generic code for scalar array operation pushdown that transforms * them into a series of OR/AND clauses. */ tmp_context = *context; pushed_down = pushdown_saop_boolexpr(&tmp_context, saop); if (tmp_context.can_pushdown) { context->needs_recheck |= tmp_context.needs_recheck; return pushed_down; } /* * No other ways to push it down, so consider it failed. */ context->can_pushdown = false; return orig_node; } case T_BoolExpr: { BoolExpr *orig_boolexpr = castNode(BoolExpr, orig_node); List *pushed_down_args = NIL; ListCell *lc; foreach (lc, orig_boolexpr->args) { QualPushdownContext tmp_context = *context; void *pushed_down = qual_pushdown_mutator(lfirst(lc), &tmp_context); /* * If the bool operation uses AND, it's correct and useful to * push down only some arguments. * * For OR, we must be able to push down every argument. */ if (!tmp_context.can_pushdown) { if (orig_boolexpr->boolop != AND_EXPR) { context->can_pushdown = false; return orig_node; } /* * If we pushed down the expression only partially, it means * we'll have to recheck it even if individual parts don't * require rechecking. */ context->needs_recheck = true; continue; } context->needs_recheck |= tmp_context.needs_recheck; pushed_down_args = lappend(pushed_down_args, pushed_down); } /* * We might have no pushed down arguments if we had an AND bool * operation, but failed to push down every individual argument. */ if (pushed_down_args == NIL) { context->can_pushdown = false; return orig_node; } BoolExpr *boolexpr_copy = makeNode(BoolExpr); *boolexpr_copy = *orig_boolexpr; boolexpr_copy->args = pushed_down_args; return (Node *) boolexpr_copy; } /* * These nodes do not influence the pushdown by themselves, so we * recurse. */ case T_FuncExpr: case T_CoerceViaIO: case T_RelabelType: case T_List: case T_Const: case T_NullTest: case T_Param: case T_SQLValueFunction: case T_CaseExpr: case T_CaseWhen: case T_ArrayExpr: { Node *pushed_down = expression_tree_mutator((Node *) orig_node, qual_pushdown_mutator, context); return pushed_down; } /* * We don't know how to work with other nodes. */ default: context->can_pushdown = false; return orig_node; } } ================================================ FILE: tsl/src/nodes/columnar_scan/qual_pushdown.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> void columnar_scan_filter_pushdown(PlannerInfo *root, CompressionSettings *settings, RelOptInfo *chunk_rel, RelOptInfo *compressed_rel, bool chunk_partial); ================================================ FILE: tsl/src/nodes/columnar_scan/vector_dict.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include "compression/arrow_c_data_interface.h" /* * When we have a dictionary-encoded Arrow Array, and have run a predicate on * the dictionary, this function is used to translate the dictionary predicate * result to the final predicate result. */ static void translate_bitmap_from_dictionary(const ArrowArray *arrow, const uint64 *dict_result, uint64 *restrict final_result) { Assert(arrow->dictionary != NULL); const size_t n = arrow->length; const int16 *indices = (int16 *) arrow->buffers[1]; for (size_t outer = 0; outer < n / 64; outer++) { uint64 word = 0; for (size_t inner = 0; inner < 64; inner++) { const size_t row = (outer * 64) + inner; const size_t bit_index = inner; #define INNER_LOOP \ const int16 index = indices[row]; \ const bool valid = arrow_row_is_valid(dict_result, index); \ word |= ((uint64) valid) << bit_index; INNER_LOOP } final_result[outer] &= word; } if (n % 64) { uint64 word = 0; for (size_t row = (n / 64) * 64; row < n; row++) { const size_t bit_index = row % 64; INNER_LOOP } final_result[n / 64] &= word; } #undef INNER_LOOP } ================================================ FILE: tsl/src/nodes/columnar_scan/vector_predicates.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Functions for working with vectorized predicates. */ #include <postgres.h> #include <mb/pg_wchar.h> #include <utils/date.h> #include <utils/fmgroids.h> #include <utils/uuid.h> #include "compression/arrow_c_data_interface.h" #include "vector_predicates.h" #include "compat/compat.h" #include "compression/compression.h" #include "debug_assert.h" /* * We include all implementations of vector-const predicates here. No separate * declarations for them to reduce the amount of macro template magic. */ #include "pred_vector_const_arithmetic_all.c" #include "pred_text.h" /* * Look up the vectorized implementation for a Postgres predicate, specified by * its Oid in pg_proc. Note that this Oid is different from the opcode. */ VectorPredicate * get_vector_const_predicate(Oid pg_predicate) { switch (pg_predicate) { #define GENERATE_DISPATCH_TABLE #include "pred_vector_const_arithmetic_all.c" #undef GENERATE_DISPATCH_TABLE case F_TEXTEQ: return vector_const_texteq; case F_TEXTNE: return vector_const_textne; case F_BOOLEQ: return vector_booleq; case F_UUID_EQ: return vector_uuideq; case F_UUID_NE: return vector_uuidne; default: /* * More checks below, this branch is to placate the static analyzers. */ break; } if (GetDatabaseEncoding() == PG_UTF8) { /* We have some simple LIKE vectorization for case-sensitive UTF8. */ switch (pg_predicate) { case F_TEXTLIKE: return vector_const_textlike_utf8; case F_TEXTNLIKE: return vector_const_textnlike_utf8; default: /* * This branch is to placate the static analyzers. */ break; } } return NULL; } void vector_nulltest(const ArrowArray *arrow, int test_type, uint64 *restrict result) { const bool should_be_null = test_type == IS_NULL; const uint16 bitmap_words = (arrow->length + 63) / 64; const uint64 *validity = (const uint64 *) arrow->buffers[0]; for (uint16 i = 0; i < bitmap_words; i++) { const uint64 validity_word = validity != NULL ? validity[i] : ~0ULL; if (should_be_null) { result[i] &= ~validity_word; } else { result[i] &= validity_word; } } } void vector_booleq(const ArrowArray *arrow, Datum arg, uint64 *restrict result) { bool check_val = DatumGetBool(arg); if (check_val) { vector_booleantest(arrow, IS_TRUE, result); } else { vector_booleantest(arrow, IS_FALSE, result); } } typedef union UUIDBuffer { uint64 components[2]; pg_uuid_t uuid; #ifdef HAVE_INT128 int128 i128; #endif } UUIDBuffer; #define VECTOR_CTYPE UUIDBuffer #define CONST_CTYPE UUIDBuffer #define CONST_CONVERSION(X) *((UUIDBuffer *) DatumGetPointer(X)) #define PREDICATE_NAME NE #ifdef HAVE_INT128 #define PREDICATE_EXPRESSION(X, Y) (((X).i128) != ((Y).i128)) #else #define PREDICATE_EXPRESSION(X, Y) \ ((X.components[0]) != (Y.components[0]) || (X.components[1]) != (Y.components[1])) #endif #include "pred_vector_const_arithmetic_single.c" void vector_uuidne(const ArrowArray *arrow, Datum arg, uint64 *restrict result) { pg_uuid_t *uuid = DatumGetUUIDP(arg); /* * No assumptions are being made about the alignment of the argument UUID, * so we copy the values to a local variable. This is because uuid is defined as typalign 'c'. */ UUIDBuffer arg_values; memcpy(&arg_values.uuid, uuid, sizeof(pg_uuid_t)); predicate_NE_UUIDBuffer_vector_UUIDBuffer_const(arrow, PointerGetDatum(&arg_values), result); } #undef VECTOR_CTYPE #undef CONST_CTYPE #undef CONST_CONVERSION #undef PG_PREDICATE #define VECTOR_CTYPE UUIDBuffer #define CONST_CTYPE UUIDBuffer #define CONST_CONVERSION(X) *((UUIDBuffer *) DatumGetPointer(X)) #define PREDICATE_NAME EQ #ifdef HAVE_INT128 #define PREDICATE_EXPRESSION(X, Y) (((X).i128) == ((Y).i128)) #else #define PREDICATE_EXPRESSION(X, Y) \ ((X.components[0]) == (Y.components[0]) && (X.components[1]) == (Y.components[1])) #endif #include "pred_vector_const_arithmetic_single.c" void vector_uuideq(const ArrowArray *arrow, Datum arg, uint64 *restrict result) { pg_uuid_t *uuid = DatumGetUUIDP(arg); /* * No assumptions are being made about the alignment of the argument UUID, * so we copy the values to a local variable. This is because uuid is defined as typalign 'c'. */ UUIDBuffer arg_values; memcpy(&arg_values.uuid, uuid, sizeof(pg_uuid_t)); predicate_EQ_UUIDBuffer_vector_UUIDBuffer_const(arrow, PointerGetDatum(&arg_values), result); } #undef VECTOR_CTYPE #undef CONST_CTYPE #undef CONST_CONVERSION #undef PG_PREDICATE void vector_booleantest(const ArrowArray *arrow, int test_type, uint64 *restrict result) { const uint16 bitmap_words = (arrow->length + 63) / 64; const uint64 *restrict validity = (const uint64 *) arrow->buffers[0]; const uint64 *restrict values = (const uint64 *) arrow->buffers[1]; switch (test_type) { case IS_TRUE: { if (validity) { for (uint16 i = 0; i < bitmap_words; i++) { result[i] &= validity[i] & values[i]; } } else { for (uint16 i = 0; i < bitmap_words; i++) { result[i] &= values[i]; } } break; } case IS_NOT_TRUE: { if (validity) { for (uint16 i = 0; i < bitmap_words; i++) { result[i] &= (~validity[i] | ~values[i]); } } else { for (uint16 i = 0; i < bitmap_words; i++) { result[i] &= ~values[i]; } } break; } case IS_FALSE: { if (validity) { for (uint16 i = 0; i < bitmap_words; i++) { result[i] &= validity[i] & ~values[i]; } } else { for (uint16 i = 0; i < bitmap_words; i++) { result[i] &= ~values[i]; } } break; } case IS_NOT_FALSE: { if (validity) { for (uint16 i = 0; i < bitmap_words; i++) { result[i] &= (~validity[i] | values[i]); } } else { for (uint16 i = 0; i < bitmap_words; i++) { result[i] &= values[i]; } } break; } case IS_UNKNOWN: { if (validity) { for (uint16 i = 0; i < bitmap_words; i++) { result[i] &= ~validity[i]; } } else { /* No validity, so all rows are valid and all result rows are filtered out. */ memset(result, 0, bitmap_words * sizeof(uint64)); } break; } case IS_NOT_UNKNOWN: { if (validity) { for (uint16 i = 0; i < bitmap_words; i++) { result[i] &= validity[i]; } } break; } default: Assert(false); break; } } ================================================ FILE: tsl/src/nodes/columnar_scan/vector_predicates.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Functions for working with vectorized predicates. */ #pragma once typedef void(VectorPredicate)(const ArrowArray *, Datum, uint64 *restrict); VectorPredicate *get_vector_const_predicate(Oid pg_predicate); void vector_array_predicate(VectorPredicate *vector_const_predicate, bool is_or, const ArrowArray *vector, Datum array, uint64 *restrict final_result); void vector_nulltest(const ArrowArray *arrow, int test_type, uint64 *restrict result); /* this implements the vectorized BooleanTest, where NULLs are handled in a special way: * for example IS_NOT_TRUE(NULL) is true and IS_NOT_FALSE(NULL) is true, plus there are * NULL test types, like IS_UNKNOWN and IS_NOT_UNKNOWN. */ void vector_booleantest(const ArrowArray *arrow, int test_type, uint64 *restrict result); void vector_booleq(const ArrowArray *arrow, Datum arg, uint64 *restrict result); void vector_uuideq(const ArrowArray *arrow, Datum arg, uint64 *restrict result); void vector_uuidne(const ArrowArray *arrow, Datum arg, uint64 *restrict result); typedef enum BatchQualSummary { AllRowsPass, NoRowsPass, SomeRowsPass } BatchQualSummary; static pg_attribute_always_inline BatchQualSummary get_vector_qual_summary(const uint64 *qual_result, size_t n_rows) { bool any_rows_pass = false; bool all_rows_pass = true; for (size_t i = 0; i < n_rows / 64; i++) { any_rows_pass = any_rows_pass || (qual_result[i] != 0); all_rows_pass = all_rows_pass && (~qual_result[i] == 0); } if (n_rows % 64 != 0) { const uint64 last_word_mask = ~0ULL >> (64 - n_rows % 64); any_rows_pass |= (qual_result[n_rows / 64] & last_word_mask) != 0; all_rows_pass &= ((~qual_result[n_rows / 64]) & last_word_mask) == 0; } Assert(!(all_rows_pass && !any_rows_pass)); if (!any_rows_pass) { return NoRowsPass; } if (all_rows_pass) { return AllRowsPass; } return SomeRowsPass; } ================================================ FILE: tsl/src/nodes/columnar_scan/vector_quals.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <executor/tuptable.h> #include <nodes/primnodes.h> #include "compression/arrow_c_data_interface.h" #include "vector_predicates.h" /* * VectorQualInfo provides planner time information for extracting * vectorizable quals from regular quals. */ typedef struct VectorQualInfo { /* * The range-table index of the relation to compute vectorized quals * for. */ Index rti; bool reverse; /* * Arrays indexed by uncompressed attno indicating whether an * attribute/column is a vectorizable type and/or a segmentby attribute. * * Note: array lengths are maxattno + 1. */ bool *vector_attrs; bool *segmentby_attrs; /* Max attribute number found in arrays above */ AttrNumber maxattno; } VectorQualInfo; /* * VectorQualState keeps the necessary state needed for the computation of * vectorized filters in scan nodes. */ typedef struct VectorQualState { List *vectorized_quals_constified; uint16 num_results; uint64 *vector_qual_result; MemoryContext per_vector_mcxt; TupleTableSlot *slot; /* * Interface function to be provided by scan node. * * Given a (compressed) tuple/slot, and a column reference (Var), get the * corresponding arrow array. * * Scan-node specific context data can be provided by wrapping this struct * in a larger one. */ const ArrowArray *(*get_arrow_array)(struct VectorQualState *vqstate, Expr *expr, bool *is_default_value); } VectorQualState; extern Node *vector_qual_make(Node *qual, const VectorQualInfo *vqinfo); extern BatchQualSummary vector_qual_compute(VectorQualState *vqstate); extern ArrowArray *make_single_value_arrow(Oid pgtype, Datum datum, bool isnull); ================================================ FILE: tsl/src/nodes/gapfill/CMakeLists.txt ================================================ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/gapfill_functions.c ${CMAKE_CURRENT_SOURCE_DIR}/gapfill_plan.c ${CMAKE_CURRENT_SOURCE_DIR}/gapfill_exec.c ${CMAKE_CURRENT_SOURCE_DIR}/locf.c ${CMAKE_CURRENT_SOURCE_DIR}/interpolate.c) target_sources(${TSL_LIBRARY_NAME} PRIVATE ${SOURCES}) ================================================ FILE: tsl/src/nodes/gapfill/README.md ================================================ # Gap filling This module implements first level support for gap fill queries, including support for LOCF (last observation carried forward) and interpolation, without requiring to join against `generate_series`. This makes it easier to join timeseries with different or irregular sampling intervals. ## Design This introduces a new gapfill customscan node that is inserted above the aggregation node of a query. The node will inject tuples for time intervals without data. The node requires data to be sorted by time, but it will inject sort nodes in the plan to ensure data is sorted correctly if the query order does not match the required order. The time_bucket_gapfill functions only serves to trigger injecting the gapfill customscan node in the planner all the tuple injecting happens in the gapfill node and time_bucket_gapfill just calls plain time_bucket. The locf and interpolate function calls serve as markers in the plan to trigger locf or interpolate behaviour. In the targetlist of the gapfill node those functions will be toplevel function calls. The gapfill state transitions are described in gapfill_internal.h ## Usage Gapfill query ``` SELECT time_bucket_gapfill(1,time,0,6) AS time, min(value) AS value FROM (values (0,1),(5,6)) v(time,value) GROUP BY 1 ORDER BY 1; ``` Gapfill query with LOCF ``` SELECT time_bucket_gapfill(1,time,0,6) AS time, locf(min(value)) AS value FROM (values (0,1),(5,6)) v(time,value) GROUP BY 1 ORDER BY 1; ``` Gapfill query with interpolation ``` SELECT time_bucket_gapfill(1,time,0,6) AS time, interpolate(min(value)) AS value FROM (values (0,1),(5,6)) v(time,value) GROUP BY 1 ORDER BY 1; ``` ================================================ FILE: tsl/src/nodes/gapfill/gapfill.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/pathnodes.h> #include <nodes/primnodes.h> #define GAPFILL_FUNCTION "time_bucket_gapfill" #define GAPFILL_LOCF_FUNCTION "locf" #define GAPFILL_INTERPOLATE_FUNCTION "interpolate" /* * Indices into CustomScan->custom_private for GapFill node. */ typedef enum GapfillPrivateIndex { GFP_GapfillFunc = 0, /* FuncExpr: time_bucket_gapfill call */ GFP_GroupClause = 1, /* List: parse->groupClause */ GFP_JoinTree = 2, /* FromExpr: parse->jointree */ GFP_Args = 3, /* List: gapfill function arguments */ GFP_Count } GapfillPrivateIndex; void plan_add_gapfill(PlannerInfo *root, RelOptInfo *group_rel); void gapfill_adjust_window_targetlist(PlannerInfo *root, RelOptInfo *input_rel, RelOptInfo *output_rel); typedef struct GapFillPath { CustomPath cpath; FuncExpr *func; /* time_bucket_gapfill function call */ } GapFillPath; ================================================ FILE: tsl/src/nodes/gapfill/gapfill_exec.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/attnum.h> #include <access/htup_details.h> #include <c.h> #include <catalog/pg_cast.h> #include <catalog/pg_collation.h> #include <catalog/pg_type.h> #include <miscadmin.h> #include <nodes/extensible.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <nodes/primnodes.h> #include <optimizer/clauses.h> #include <optimizer/optimizer.h> #include <utils/builtins.h> #include <utils/date.h> #include <utils/datum.h> #include <utils/lsyscache.h> #include <utils/memutils.h> #include <utils/syscache.h> #include <utils/timestamp.h> #include <utils/typcache.h> #include <compat/compat.h> #include "gapfill.h" #include "gapfill_internal.h" #include "interpolate.h" #include "locf.h" #include "time_bucket.h" #include <annotations.h> typedef enum GapFillBoundary { GAPFILL_START, GAPFILL_END, } GapFillBoundary; typedef union GapFillColumnStateUnion { GapFillColumnState *base; GapFillGroupColumnState *group; GapFillInterpolateColumnState *interpolate; GapFillLocfColumnState *locf; } GapFillColumnStateUnion; #define foreach_column(column, index, state) \ Assert((state)->ncolumns > 0); \ for ((index) = 0, (column) = (state)->columns[index]; \ (index) < (state)->ncolumns && ((column) = (state)->columns[index], true); \ (index)++) static void gapfill_begin(CustomScanState *node, EState *estate, int eflags); static void gapfill_end(CustomScanState *node); static void gapfill_rescan(CustomScanState *node); static TupleTableSlot *gapfill_exec(CustomScanState *node); static void gapfill_state_reset_group(GapFillState *state, TupleTableSlot *slot); static TupleTableSlot *gapfill_state_gaptuple_create(GapFillState *state, int64 time); static bool gapfill_state_is_new_group(GapFillState *state, TupleTableSlot *slot); static void gapfill_state_set_next(GapFillState *state, TupleTableSlot *subslot); static TupleTableSlot *gapfill_state_return_subplan_slot(GapFillState *state); static TupleTableSlot *gapfill_fetch_next_tuple(GapFillState *state); static void gapfill_state_initialize_columns(GapFillState *state, List *exec_tlist); static GapFillColumnState *gapfill_column_state_create(GapFillColumnType ctype, Oid typeid); static bool gapfill_is_group_column(GapFillState *state, TargetEntry *tle); static TargetEntry *gapfill_get_fixed_agg_expr_column(GapFillState *state, TargetEntry *tle); static CustomExecMethods gapfill_state_methods = { .BeginCustomScan = gapfill_begin, .ExecCustomScan = gapfill_exec, .EndCustomScan = gapfill_end, .ReScanCustomScan = gapfill_rescan, }; /* * convert Datum to int64 according to type * internally we store all times as int64 in the * same format postgres does */ int64 gapfill_datum_get_internal(Datum value, Oid type) { switch (type) { case INT2OID: return DatumGetInt16(value); case DATEOID: case INT4OID: return DatumGetInt32(value); case TIMESTAMPOID: case TIMESTAMPTZOID: case INT8OID: return DatumGetInt64(value); default: /* * should never happen since time_bucket_gapfill is not defined * for other datatypes */ ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported datatype for time_bucket_gapfill: %s", format_type_be(type)))); pg_unreachable(); break; } } /* * convert int64 to Datum according to type * internally we store all times as int64 in the * same format postgres does */ static inline Datum gapfill_internal_get_datum(int64 value, Oid type) { switch (type) { case INT2OID: return Int16GetDatum(value); case DATEOID: case INT4OID: return Int32GetDatum(value); case TIMESTAMPOID: case TIMESTAMPTZOID: case INT8OID: return Int64GetDatum(value); default: /* * should never happen since time_bucket_gapfill is not defined * for other datatypes */ Assert(false); return Int64GetDatum(0); } } static Expr * get_start_arg(GapFillState *state) { if (!state->have_timezone) return lthird(state->args); else return lfourth(state->args); } static Expr * get_finish_arg(GapFillState *state) { if (!state->have_timezone) return lfourth(state->args); else return lfifth(state->args); } static Expr * get_timezone_arg(GapFillState *state) { Assert(state->have_timezone); return lthird(state->args); } static inline int64 gapfill_period_get_internal(Oid timetype, Oid argtype, Datum arg, Interval **interval) { switch (timetype) { case DATEOID: case TIMESTAMPOID: case TIMESTAMPTZOID: Assert(INTERVALOID == argtype); Interval *interval_arg = DatumGetIntervalP(arg); if (interval_arg->time < 0 || interval_arg->day < 0 || interval_arg->month < 0 || interval_arg->time + interval_arg->day + interval_arg->month == 0) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid time_bucket_gapfill argument: bucket_width must be " "greater than 0"))); } *interval = interval_arg; return 0; break; case INT2OID: Assert(INT2OID == argtype); return DatumGetInt16(arg); case INT4OID: Assert(INT4OID == argtype); return DatumGetInt32(arg); case INT8OID: Assert(INT8OID == argtype); return DatumGetInt64(arg); default: /* * should never happen since time_bucket_gapfill is not defined * for other datatypes */ ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported datatype for time_bucket_gapfill: %s", format_type_be(timetype)))); pg_unreachable(); break; } } /* * Create a GapFill node from this plan. This is the full execution * state that replaces the plan node as the plan moves from planning to * execution. */ Node * gapfill_state_create(CustomScan *cscan) { GapFillState *state = (GapFillState *) newNode(sizeof(GapFillState), T_CustomScanState); state->csstate.methods = &gapfill_state_methods; state->subplan = linitial(cscan->custom_plans); state->args = list_nth(cscan->custom_private, GFP_Args); state->have_timezone = list_length(state->args) == 5; return (Node *) state; } static bool is_const_null(Expr *expr) { return IsA(expr, Const) && castNode(Const, expr)->constisnull; } /* * lookup cast func oid in pg_cast * * throws an error if no cast can be found */ static Oid get_cast_func(Oid source, Oid target) { Oid result = InvalidOid; HeapTuple casttup; casttup = SearchSysCache2(CASTSOURCETARGET, ObjectIdGetDatum(source), ObjectIdGetDatum(target)); if (HeapTupleIsValid(casttup)) { Form_pg_cast castform = (Form_pg_cast) GETSTRUCT(casttup); result = castform->castfunc; ReleaseSysCache(casttup); } if (!OidIsValid(result)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("could not find cast from %s to %s", format_type_be(source), format_type_be(target)))); return result; } /* * returns true if v1 and v2 reference the same object */ static bool var_equal(Var *v1, Var *v2) { return v1->varno == v2->varno && v1->varattno == v2->varattno && v1->vartype == v2->vartype; } static bool is_simple_expr_walker(Node *node, void *context) { if (node == NULL) return false; /* * since expression_tree_walker does early exit on true * logic is reverted and return value of true means expression * is not simple, this is reverted in parent */ switch (nodeTag(node)) { /* * whitelist expression types we deem safe to execute in a * separate expression context */ case T_Const: case T_FuncExpr: case T_NamedArgExpr: case T_OpExpr: case T_DistinctExpr: case T_NullIfExpr: case T_ScalarArrayOpExpr: case T_BoolExpr: case T_CoerceViaIO: case T_CaseExpr: case T_CaseWhen: break; case T_Param: if (castNode(Param, node)->paramkind != PARAM_EXTERN) return true; break; default: return true; } return expression_tree_walker(node, is_simple_expr_walker, context); } /* * check if expression is simple expression and contains only simple * subexpressions */ static bool is_simple_expr(Expr *node) { /* * since expression_tree_walker does early exit on true and we use that to * skip processing on first non-simple expression we invert return value * from expression_tree_walker here */ return !is_simple_expr_walker((Node *) node, NULL); } /* * align a value with the bucket boundary * even though we use int64 as our internal representation we cannot call * ts_int64_bucket here because int variants of time_bucket align differently * then non-int variants because the bucket start is on monday for the latter */ static int64 align_with_time_bucket(GapFillState *state, Expr *expr) { CustomScan *cscan = castNode(CustomScan, state->csstate.ss.ps.plan); FuncExpr *time_bucket = copyObject(list_nth(cscan->custom_private, GFP_GapfillFunc)); Datum value; bool isnull; if (!is_simple_expr(expr)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg( "invalid time_bucket_gapfill argument: start must be a simple expression"))); if (state->have_timezone) { if (IsA(get_timezone_arg(state), Const) && castNode(Const, get_timezone_arg(state))->constisnull) { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid time_bucket_gapfill argument: timezone cannot be NULL"))); } time_bucket->args = list_make3(linitial(time_bucket->args), expr, lthird(time_bucket->args)); } else { time_bucket->args = list_make2(linitial(time_bucket->args), expr); } value = gapfill_exec_expr(state, state->scanslot, (Expr *) time_bucket, &isnull); /* start expression must not evaluate to NULL */ if (isnull) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid time_bucket_gapfill argument: start cannot be NULL"), errhint("Specify start and finish as arguments or in the WHERE clause."))); return gapfill_datum_get_internal(value, state->gapfill_typid); } static int64 get_boundary_expr_value(GapFillState *state, GapFillBoundary boundary, Expr *expr) { Datum arg_value; bool isnull; /* * add an explicit cast here if types do not match */ if (exprType((Node *) expr) != state->gapfill_typid) { Oid cast_oid = get_cast_func(exprType((Node *) expr), state->gapfill_typid); expr = (Expr *) makeFuncExpr(cast_oid, state->gapfill_typid, list_make1(expr), InvalidOid, InvalidOid, 0); } arg_value = gapfill_exec_expr(state, state->scanslot, expr, &isnull); if (isnull) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid time_bucket_gapfill argument: %s cannot be NULL", boundary == GAPFILL_START ? "start" : "finish"), errhint("Specify start and finish as arguments or in the WHERE clause."))); return gapfill_datum_get_internal(arg_value, state->gapfill_typid); } typedef struct CollectBoundaryContext { List *quals; Var *ts_var; } CollectBoundaryContext; /* * expression references our gapfill time column and could be * a boundary expression, more thorough check is in * infer_gapfill_boundary */ static bool is_boundary_expr(Node *node, CollectBoundaryContext *context) { OpExpr *op; Node *left, *right; if (!IsA(node, OpExpr)) return false; op = castNode(OpExpr, node); if (op->args->length != 2) return false; left = linitial(op->args); right = llast(op->args); /* Var OP Var is not useful here because we are not yet at a point * where we could evaluate them */ if (IsA(left, Var) && IsA(right, Var)) return false; if (IsA(left, Var) && var_equal(castNode(Var, left), context->ts_var)) return true; if (IsA(right, Var) && var_equal(castNode(Var, right), context->ts_var)) return true; return false; } static bool collect_boundary_walker(Node *node, CollectBoundaryContext *context) { Node *quals = NULL; if (node == NULL) return false; if (IsA(node, FromExpr)) { quals = castNode(FromExpr, node)->quals; } else if (IsA(node, JoinExpr)) { JoinExpr *j = castNode(JoinExpr, node); /* don't descend into outer join */ if (IS_OUTER_JOIN(j->jointype)) return false; quals = j->quals; } if (quals) { ListCell *lc; foreach (lc, castNode(List, quals)) { if (is_boundary_expr(lfirst(lc), context)) context->quals = lappend(context->quals, lfirst(lc)); } } return expression_tree_walker(node, collect_boundary_walker, context); } /* * traverse jointree to look for expressions referencing * the time column of our gapfill call */ static List * collect_boundary_expressions(Node *node, Var *ts_var) { CollectBoundaryContext context = { .quals = NIL, .ts_var = ts_var }; collect_boundary_walker(node, &context); return context.quals; } static int64 infer_gapfill_boundary(GapFillState *state, GapFillBoundary boundary) { CustomScan *cscan = castNode(CustomScan, state->csstate.ss.ps.plan); FuncExpr *func = list_nth(cscan->custom_private, GFP_GapfillFunc); FromExpr *jt = list_nth(cscan->custom_private, GFP_JoinTree); ListCell *lc; Var *ts_var; TypeCacheEntry *tce = lookup_type_cache(state->gapfill_typid, TYPECACHE_BTREE_OPFAMILY); int strategy; Oid lefttype, righttype; List *quals; int64 boundary_value = 0; bool boundary_found = false; /* * if the second argument to time_bucket_gapfill is not a column reference * we cannot match WHERE clause to the time column */ if (!IsA(lsecond(func->args), Var)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid time_bucket_gapfill argument: ts needs to refer to a single " "column if no start or finish is supplied"), errhint("Specify start and finish as arguments or in the WHERE clause."))); ts_var = castNode(Var, lsecond(func->args)); quals = collect_boundary_expressions((Node *) jt, ts_var); foreach (lc, quals) { OpExpr *opexpr = lfirst_node(OpExpr, lc); Var *var; Expr *expr; Oid op; int64 value; if (IsA(linitial(opexpr->args), Var)) { var = linitial(opexpr->args); expr = lsecond(opexpr->args); op = opexpr->opno; } else if (IsA(lsecond(opexpr->args), Var)) { var = lsecond(opexpr->args); expr = linitial(opexpr->args); op = get_commutator(opexpr->opno); } else { /* collect_boundary_expressions has filtered those out already */ Assert(false); continue; } if (!op_in_opfamily(op, tce->btree_opf)) continue; /* * only allow simple expressions because Params have not been set up * at this stage and Vars will not work either because we execute in * separate execution context */ if (!is_simple_expr(expr) || !var_equal(ts_var, var)) continue; get_op_opfamily_properties(op, tce->btree_opf, false, &strategy, &lefttype, &righttype); if (boundary == GAPFILL_START && strategy != BTGreaterStrategyNumber && strategy != BTGreaterEqualStrategyNumber) continue; if (boundary == GAPFILL_END && strategy != BTLessStrategyNumber && strategy != BTLessEqualStrategyNumber) continue; value = get_boundary_expr_value(state, boundary, expr); /* * if the boundary expression operator does not match the operator * used by the gapfill node we adjust the value by 1 here * * the operators for the gapfill node are >= for start and < for end * column > value becomes start >= value + 1 column <= value becomes * end < value + 1 */ if (strategy == BTGreaterStrategyNumber || strategy == BTLessEqualStrategyNumber) value += 1; if (!boundary_found) { boundary_found = true; boundary_value = value; } else { if (boundary == GAPFILL_START) boundary_value = Max(boundary_value, value); else boundary_value = Min(boundary_value, value); } } if (boundary_found) return boundary_value; ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("missing time_bucket_gapfill argument: could not infer %s from WHERE clause", boundary == GAPFILL_START ? "start" : "finish"), errhint("Specify start and finish as arguments or in the WHERE clause."))); pg_unreachable(); } static Const * make_const_value_for_gapfill_internal(Oid typid, int64 value) { TypeCacheEntry *tce = lookup_type_cache(typid, 0); Datum d = gapfill_internal_get_datum(value, typid); return makeConst(typid, -1, InvalidOid, tce->typlen, d, false, tce->typbyval); } static void gapfill_advance_timestamp(GapFillState *state) { Datum next; switch (state->gapfill_typid) { case DATEOID: next = DirectFunctionCall2(date_pl_interval, DateADTGetDatum(state->gapfill_start), IntervalPGetDatum(state->next_offset)); next = DirectFunctionCall1(timestamp_date, next); state->next_timestamp = DatumGetDateADT(next); break; case TIMESTAMPOID: next = DirectFunctionCall2(timestamp_pl_interval, TimestampGetDatum(state->gapfill_start), IntervalPGetDatum(state->next_offset)); state->next_timestamp = DatumGetTimestamp(next); break; case TIMESTAMPTZOID: /* * To be consistent with time_bucket we do UTC bucketing unless * a different timezone got explicitly passed to the function * and we are bucketing by non-fixed intervals. */ if (state->have_timezone && (state->next_offset->day != 0 || state->next_offset->month != 0)) { bool isnull; /* TODO: optimize by constifying and caching the datum if possible */ Datum tzname = gapfill_exec_expr(state, state->scanslot, get_timezone_arg(state), &isnull); Assert(!isnull); /* Convert to local timestamp */ next = DirectFunctionCall2(timestamptz_zone, tzname, TimestampTzGetDatum(state->gapfill_start)); /* Add interval */ next = DirectFunctionCall2(timestamp_pl_interval, next, IntervalPGetDatum(state->next_offset)); /* Convert back to specified timezone */ next = DirectFunctionCall2(timestamp_zone, tzname, next); } else { next = DirectFunctionCall2(timestamp_pl_interval, TimestampTzGetDatum(state->gapfill_start), IntervalPGetDatum(state->next_offset)); } state->next_timestamp = DatumGetTimestampTz(next); break; default: state->next_timestamp += state->gapfill_period; break; } /* Advance the interval offset if necessary */ if (state->gapfill_interval) { Datum tspan = DirectFunctionCall2(interval_pl, IntervalPGetDatum(state->gapfill_interval), IntervalPGetDatum(state->next_offset)); state->next_offset = DatumGetIntervalP(tspan); } } /* * Initialize the scan state */ static void gapfill_begin(CustomScanState *node, EState *estate, int eflags) { GapFillState *state = (GapFillState *) node; CustomScan *cscan = castNode(CustomScan, state->csstate.ss.ps.plan); /* * this is the time_bucket_gapfill call from the plan which is used to * extract arguments and to align gapfill_start */ FuncExpr *func = list_nth(cscan->custom_private, GFP_GapfillFunc); TupleDesc tupledesc = state->csstate.ss.ps.ps_ResultTupleSlot->tts_tupleDescriptor; List *targetlist = copyObject(state->csstate.ss.ps.plan->targetlist); bool isnull; Datum arg_value; state->gapfill_typid = func->funcresulttype; state->state = FETCHED_NONE; state->subslot = MakeSingleTupleTableSlot(tupledesc, &TTSOpsVirtual); state->scanslot = MakeSingleTupleTableSlot(tupledesc, &TTSOpsVirtual); /* bucket_width */ if (!is_simple_expr(linitial(state->args))) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid time_bucket_gapfill argument: bucket_width must be a simple " "expression"))); arg_value = gapfill_exec_expr(state, NULL, linitial(state->args), &isnull); if (isnull) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid time_bucket_gapfill argument: bucket_width cannot be NULL"))); state->gapfill_period = gapfill_period_get_internal(func->funcresulttype, exprType(linitial(state->args)), arg_value, &state->gapfill_interval); /* * this would error when trying to align start and stop to bucket_width as well below * but checking this explicitly here will make a nicer error message */ if (state->gapfill_period <= 0 && !state->gapfill_interval) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg( "invalid time_bucket_gapfill argument: bucket_width must be greater than 0"))); /* * check if gapfill start was left out so we have to infer from WHERE * clause */ if (is_const_null(get_start_arg(state))) { int64 start = infer_gapfill_boundary(state, GAPFILL_START); Const *expr = make_const_value_for_gapfill_internal(state->gapfill_typid, start); state->gapfill_start = align_with_time_bucket(state, (Expr *) expr); } else { /* * pass gapfill start through time_bucket so it is aligned with bucket * start */ state->gapfill_start = align_with_time_bucket(state, get_start_arg(state)); } state->next_timestamp = state->gapfill_start; state->next_offset = state->gapfill_interval; /* gap fill end */ if (is_const_null(get_finish_arg(state))) state->gapfill_end = infer_gapfill_boundary(state, GAPFILL_END); else { if (!is_simple_expr(get_finish_arg(state))) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid time_bucket_gapfill argument: finish must be a simple " "expression"))); arg_value = gapfill_exec_expr(state, NULL, get_finish_arg(state), &isnull); /* * the default value for finish is NULL but this is checked above, * when a non-Const is passed here that evaluates to NULL we bail */ if (isnull) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid time_bucket_gapfill argument: finish cannot be NULL"), errhint("Specify start and finish as arguments or in the WHERE clause."))); state->gapfill_end = gapfill_datum_get_internal(arg_value, func->funcresulttype); } gapfill_state_initialize_columns(state, targetlist); /* * Build ProjectionInfo that will be used for gap filled tuples only. */ state->pi = ExecBuildProjectionInfo(targetlist, state->csstate.ss.ps.ps_ExprContext, MakeSingleTupleTableSlot(tupledesc, &TTSOpsVirtual), &state->csstate.ss.ps, NULL); state->csstate.custom_ps = list_make1(ExecInitNode(state->subplan, estate, eflags)); } /* * This is the main loop of the node it is called whenever the upper node * wants to consume a new tuple. Returning NULL signals that the tuples * are exhausted. All gapfill state transitions happen in this function. */ static TupleTableSlot * gapfill_exec(CustomScanState *node) { GapFillState *state = (GapFillState *) node; TupleTableSlot *slot = NULL; while (true) { CHECK_FOR_INTERRUPTS(); /* fetch next tuple from subplan */ if (FETCHED_NONE == state->state) { slot = gapfill_fetch_next_tuple(state); if (slot) { if (state->multigroup && gapfill_state_is_new_group(state, slot)) state->state = FETCHED_NEXT_GROUP; else state->state = FETCHED_ONE; gapfill_state_set_next(state, slot); } else { /* * if GROUP BY has non time_bucket_gapfill columns but the * query has not initialized the groups there is nothing we * can do here */ if (state->multigroup && !state->groups_initialized) return NULL; else state->state = FETCHED_LAST; } } /* return any subplan tuples before gapfill_start */ if (FETCHED_ONE == state->state && state->subslot_time < state->gapfill_start) { state->state = FETCHED_NONE; return gapfill_state_return_subplan_slot(state); } /* if we have tuple from subplan check if it needs to be inserted now */ if (FETCHED_ONE == state->state && state->subslot_time == state->next_timestamp) { state->state = FETCHED_NONE; gapfill_advance_timestamp(state); return gapfill_state_return_subplan_slot(state); } /* if we are within gapfill boundaries we need to insert tuple */ if (state->next_timestamp < state->gapfill_end) { Assert(state->state != FETCHED_NONE); slot = gapfill_state_gaptuple_create(state, state->next_timestamp); gapfill_advance_timestamp(state); return slot; } /* return any remaining subplan tuples after gapfill_end */ if (FETCHED_ONE == state->state) { state->state = FETCHED_NONE; return gapfill_state_return_subplan_slot(state); } /* * Done with current group, prepare for next */ if (FETCHED_NEXT_GROUP == state->state) { state->state = FETCHED_ONE; state->next_timestamp = state->gapfill_start; gapfill_state_reset_group(state, state->subslot); continue; } return NULL; } } static void gapfill_end(CustomScanState *node) { if (node->custom_ps != NIL) { ExecEndNode(linitial(node->custom_ps)); } } static void gapfill_rescan(CustomScanState *node) { GapFillState *state = (GapFillState *) node; if (node->custom_ps != NIL) { if (node->ss.ps.chgParam != NULL) UpdateChangedParamSet(linitial(node->custom_ps), node->ss.ps.chgParam); ExecReScan(linitial(node->custom_ps)); } state->state = FETCHED_NONE; state->next_timestamp = state->gapfill_start; state->next_offset = state->gapfill_interval; if (state->multigroup) state->groups_initialized = false; /* Reset column states for locf and interpolate */ for (int i = 0; i < state->ncolumns; i++) { GapFillColumnState *column = state->columns[i]; switch (column->ctype) { case LOCF_COLUMN: gapfill_locf_group_change((GapFillLocfColumnState *) column); break; case INTERPOLATE_COLUMN: { GapFillInterpolateColumnState *ic = (GapFillInterpolateColumnState *) column; ic->prev.isnull = true; ic->next.isnull = true; break; } case GROUP_COLUMN: case DERIVED_COLUMN: { GapFillGroupColumnState *gc = (GapFillGroupColumnState *) column; gc->isnull = true; break; } default: break; } } } static void gapfill_state_reset_group(GapFillState *state, TupleTableSlot *slot) { GapFillColumnStateUnion column; int i; Datum value; bool isnull; foreach_column(column.base, i, state) { value = slot_getattr(slot, AttrOffsetGetAttrNumber(i), &isnull); switch (column.base->ctype) { case INTERPOLATE_COLUMN: gapfill_interpolate_group_change(column.interpolate, state->subslot_time, value, isnull); break; case LOCF_COLUMN: gapfill_locf_group_change(column.locf); break; case GROUP_COLUMN: case DERIVED_COLUMN: column.group->isnull = isnull; if (!isnull) column.group->value = datumCopy(value, column.base->typbyval, column.base->typlen); break; default: break; } } state->next_offset = state->gapfill_interval; } /* * Create generated tuple according to column state */ static TupleTableSlot * gapfill_state_gaptuple_create(GapFillState *state, int64 time) { TupleTableSlot *slot = state->scanslot; GapFillColumnStateUnion column; int i; ExecClearTuple(slot); /* * we need to fill in group columns first because locf and interpolation * might reference those columns when doing out of bounds lookup */ foreach_column(column.base, i, state) { switch (column.base->ctype) { case TIME_COLUMN: slot->tts_values[i] = gapfill_internal_get_datum(time, state->gapfill_typid); slot->tts_isnull[i] = false; break; case GROUP_COLUMN: case DERIVED_COLUMN: slot->tts_values[i] = column.group->value; slot->tts_isnull[i] = column.group->isnull; break; case NULL_COLUMN: slot->tts_isnull[i] = true; break; default: break; } } /* * mark slot as containing data so it can be used in locf and interpolate * lookup expressions */ ExecStoreVirtualTuple(slot); foreach_column(column.base, i, state) { switch (column.base->ctype) { case LOCF_COLUMN: /* We may execute lookup expression over a generated tuple which fills the gap */ gapfill_locf_calculate(column.locf, state, slot, time, &slot->tts_values[i], &slot->tts_isnull[i]); break; case INTERPOLATE_COLUMN: gapfill_interpolate_calculate(column.interpolate, state, time, &slot->tts_values[i], &slot->tts_isnull[i]); break; default: break; } } ResetExprContext(state->pi->pi_exprContext); state->pi->pi_exprContext->ecxt_scantuple = slot; return ExecProject(state->pi); } /* * Returns true if tuple in the TupleTableSlot belongs to the next * aggregation group */ static bool gapfill_state_is_new_group(GapFillState *state, TupleTableSlot *slot) { GapFillColumnStateUnion column; int i; Datum value; bool isnull; /* groups not initialized yet */ if (!state->groups_initialized) { state->groups_initialized = true; gapfill_state_reset_group(state, slot); return false; } foreach_column(column.base, i, state) { if (column.base->ctype == GROUP_COLUMN) { value = slot_getattr(slot, AttrOffsetGetAttrNumber(i), &isnull); if (isnull && column.group->isnull) continue; if (isnull != column.group->isnull) return true; /* We need to use FunctionCall2Coll here since equality comparison * functions can try to access flinfo (see arrayfuncs.c). */ if (!DatumGetBool(FunctionCall2Coll(&column.group->eq_func, column.group->collation, value, column.group->value))) return true; } } return false; } /* * Returns subslot tuple and adjusts column state accordingly */ static TupleTableSlot * gapfill_state_return_subplan_slot(GapFillState *state) { GapFillColumnStateUnion column; CustomScanState *node = castNode(CustomScanState, state); int i; Datum value; bool isnull; foreach_column(column.base, i, state) { switch (column.base->ctype) { case LOCF_COLUMN: value = slot_getattr(state->subslot, AttrOffsetGetAttrNumber(i), &isnull); /* We may execute lookup expression over an input tuple from the subplan to override * NULL value when NULLs are treated as missing. Use the correct tuple for the * purpose. */ if (isnull && column.locf->treat_null_as_missing) gapfill_locf_calculate(column.locf, state, state->subslot, state->subslot_time, &state->subslot->tts_values[i], &state->subslot->tts_isnull[i]); else gapfill_locf_tuple_returned(column.locf, value, isnull); break; case INTERPOLATE_COLUMN: value = slot_getattr(state->subslot, AttrOffsetGetAttrNumber(i), &isnull); gapfill_interpolate_tuple_returned(column.interpolate, state->subslot_time, value, isnull); break; default: break; } } if (node->ss.ps.ps_ProjInfo) { ExprContext *econtext = node->ss.ps.ps_ExprContext; ResetExprContext(econtext); econtext->ecxt_scantuple = state->subslot; return ExecProject(node->ss.ps.ps_ProjInfo); } return state->subslot; } static void gapfill_state_set_next(GapFillState *state, TupleTableSlot *subslot) { GapFillColumnStateUnion column; int i; Datum value; bool isnull; /* * if this tuple is for next group we dont update column state yet * updating of column state happens in gapfill_state_reset_group instead */ if (FETCHED_NEXT_GROUP == state->state) return; foreach_column(column.base, i, state) { /* nothing to do here for locf */ if (INTERPOLATE_COLUMN == column.base->ctype) { value = slot_getattr(subslot, AttrOffsetGetAttrNumber(i), &isnull); gapfill_interpolate_tuple_fetched(column.interpolate, state->subslot_time, value, isnull); } } } static TupleTableSlot * gapfill_fetch_next_tuple(GapFillState *state) { Datum time_value; bool isnull; PlanState *subplan = linitial(castNode(CustomScanState, state)->custom_ps); TupleTableSlot *subslot = ExecProcNode(subplan); if (TupIsNull(subslot)) return NULL; /* we cannot simply treat an arbitrary source slot as virtual, * instead we must copy the data into our own slot in order to be able to * modify it */ ExecCopySlot(state->subslot, subslot); time_value = slot_getattr(subslot, AttrOffsetGetAttrNumber(state->time_index), &isnull); if (isnull) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid time_bucket_gapfill argument: ts cannot be NULL"))); state->subslot_time = gapfill_datum_get_internal(time_value, state->gapfill_typid); return state->subslot; } /* * Initialize column meta data */ static void gapfill_state_initialize_columns(GapFillState *state, List *exec_tlist) { TupleDesc tupledesc = state->csstate.ss.ps.ps_ResultTupleSlot->tts_tupleDescriptor; CustomScan *cscan = castNode(CustomScan, state->csstate.ss.ps.plan); TargetEntry *tle; Expr *expr; int i; state->ncolumns = tupledesc->natts; state->columns = (GapFillColumnState **) palloc(state->ncolumns * sizeof(GapFillColumnState *)); for (i = 0; i < state->ncolumns; i++) { tle = list_nth(cscan->custom_scan_tlist, i); expr = tle->expr; if (tle->ressortgroupref && gapfill_is_group_column(state, tle)) { /* * if there is time_bucket_gapfill function call this is our time * column */ if (IsA(expr, FuncExpr) && strncmp(get_func_name(castNode(FuncExpr, expr)->funcid), GAPFILL_FUNCTION, NAMEDATALEN) == 0) { state->columns[i] = gapfill_column_state_create(TIME_COLUMN, TupleDescAttr(tupledesc, i)->atttypid); state->time_index = i; continue; } /* otherwise this is a normal group column */ state->columns[i] = gapfill_column_state_create(GROUP_COLUMN, TupleDescAttr(tupledesc, i)->atttypid); state->multigroup = true; state->groups_initialized = false; continue; } else if (IsA(expr, FuncExpr)) { /* locf and interpolate will be toplevel function calls in the gapfill node */ if (strncmp(get_func_name(castNode(FuncExpr, expr)->funcid), GAPFILL_LOCF_FUNCTION, NAMEDATALEN) == 0) { state->columns[i] = gapfill_column_state_create(LOCF_COLUMN, TupleDescAttr(tupledesc, i)->atttypid); gapfill_locf_initialize((GapFillLocfColumnState *) state->columns[i], state, (FuncExpr *) expr); continue; } if (strncmp(get_func_name(castNode(FuncExpr, expr)->funcid), GAPFILL_INTERPOLATE_FUNCTION, NAMEDATALEN) == 0) { state->columns[i] = gapfill_column_state_create(INTERPOLATE_COLUMN, TupleDescAttr(tupledesc, i)->atttypid); gapfill_interpolate_initialize((GapFillInterpolateColumnState *) state->columns[i], state, (FuncExpr *) expr); continue; } } /* * any column that does not have an aggregation function and is not * an explicit GROUP BY column has to be derived from a GROUP BY * column so we treat those similar to GROUP BY column for gapfill * purposes. */ bool column_contains_aggs = contain_agg_clause((Node *) expr); if (!column_contains_aggs && contain_var_clause((Node *) expr)) { state->columns[i] = gapfill_column_state_create(DERIVED_COLUMN, TupleDescAttr(tupledesc, i)->atttypid); state->multigroup = true; state->groups_initialized = false; continue; } /* * For every column with Aggrefs we take the original expression tree from the * subplan and replace Aggref nodes with Const NULL nodes. This is * necessary because the expression might be evaluated below the * aggregation so we need to pull up expression from subplan into * projection for gapfilled tuples so expressions like COALESCE work * correctly for gapfilled tuples. */ if (column_contains_aggs) { TargetEntry *agg_expr_tle = gapfill_get_fixed_agg_expr_column(state, tle); Assert(agg_expr_tle); Node *entry = copyObject((Node *) agg_expr_tle); /* Fix for #4894 when we have expressions like (agg + group_expr): * after getting fixed entry where aggs are replaced with NULLs * and group expressions are replaced with exec group columns, * check whether this column contains group columns and needs to be DERIVED or NULL. */ if (contain_var_clause(entry)) { state->columns[i] = gapfill_column_state_create(DERIVED_COLUMN, TupleDescAttr(tupledesc, i)->atttypid); state->multigroup = true; state->groups_initialized = false; } else state->columns[i] = gapfill_column_state_create(NULL_COLUMN, TupleDescAttr(tupledesc, i)->atttypid); lfirst(list_nth_cell(exec_tlist, i)) = entry; continue; } /* column with no special action from gap fill node */ state->columns[i] = gapfill_column_state_create(NULL_COLUMN, TupleDescAttr(tupledesc, i)->atttypid); } } /* * Create GapFillColumnState object, set proper type and fill in datatype information */ static GapFillColumnState * gapfill_column_state_create(GapFillColumnType ctype, Oid typeid) { TypeCacheEntry *tce; int tc_flags = 0; GapFillColumnState *column; size_t size; switch (ctype) { case GROUP_COLUMN: tc_flags |= TYPECACHE_EQ_OPR; TS_FALLTHROUGH; case DERIVED_COLUMN: size = sizeof(GapFillGroupColumnState); break; case LOCF_COLUMN: size = sizeof(GapFillLocfColumnState); break; case INTERPOLATE_COLUMN: size = sizeof(GapFillInterpolateColumnState); break; default: size = sizeof(GapFillColumnState); break; } tce = lookup_type_cache(typeid, tc_flags); column = palloc0(size); column->ctype = ctype; column->typid = tce->type_id; column->typbyval = tce->typbyval; column->typlen = tce->typlen; if (ctype == GROUP_COLUMN) { GapFillGroupColumnState *gcolumn = (GapFillGroupColumnState *) column; Oid eq_opr_func = get_opcode(tce->eq_opr); fmgr_info_cxt(eq_opr_func, &gcolumn->eq_func, CurrentMemoryContext); gcolumn->collation = tce->typcollation; } return column; } /* * check if the target entry is a GROUP BY column, we need * this check because ressortgroupref will be nonzero for * ORDER BY and GROUP BY columns but we are only interested * in actual GROUP BY columns */ static bool gapfill_is_group_column(GapFillState *state, TargetEntry *tle) { ListCell *lc; CustomScan *cscan = castNode(CustomScan, state->csstate.ss.ps.plan); List *groups = list_nth(cscan->custom_private, GFP_GroupClause); foreach (lc, groups) { if (tle->ressortgroupref == ((SortGroupClause *) lfirst(lc))->tleSortGroupRef) return true; } return false; } /* * If the target entry contains an aggregate, it has been fixed in "custom_exprs" * so that the aggregate is replaced with NULL * and any group expressions are replaced with exec group vars. * We will get the fixed aggregate expression here and use it in exec tlist. */ static TargetEntry * gapfill_get_fixed_agg_expr_column(GapFillState *state, TargetEntry *tle) { ListCell *lc; CustomScan *cscan = castNode(CustomScan, state->csstate.ss.ps.plan); List *mutated_agg_exprs_list = castNode(List, cscan->custom_exprs); Assert(list_length(mutated_agg_exprs_list) == 1); List *mutated_agg_exprs = castNode(List, linitial(mutated_agg_exprs_list)); foreach (lc, mutated_agg_exprs) { TargetEntry *mutated_agg_expr_tle = castNode(TargetEntry, lfirst(lc)); if (tle->resno == mutated_agg_expr_tle->resno) return mutated_agg_expr_tle; } return NULL; } /* * Execute expression and return result of expression */ Datum gapfill_exec_expr(GapFillState *state, TupleTableSlot *ecxt_slot, Expr *expr, bool *isnull) { ExprState *exprstate = ExecInitExpr(expr, &state->csstate.ss.ps); ExprContext *exprcontext = GetPerTupleExprContext(state->csstate.ss.ps.state); exprcontext->ecxt_scantuple = ecxt_slot; return ExecEvalExprSwitchContext(exprstate, exprcontext, isnull); } /* * Adjust attribute number of all Var nodes in an expression to have the * proper index into the gap filled tuple. This is necessary to make column * references in correlated subqueries in lookup queries work. */ Expr * gapfill_adjust_varnos(GapFillState *state, Expr *expr) { ListCell *lc_var, *lc_tle; List *vars = pull_var_clause((Node *) expr, 0); List *tlist = castNode(CustomScan, state->csstate.ss.ps.plan)->custom_scan_tlist; foreach (lc_var, vars) { Var *var = lfirst(lc_var); foreach (lc_tle, tlist) { TargetEntry *tle = lfirst(lc_tle); /* * subqueries in aggregate queries can only reference columns so * we only need to look for targetlist toplevel column references */ if (IsA(tle->expr, Var) && castNode(Var, tle->expr)->varattno == var->varattno) { var->varattno = tle->resno; break; } } } return expr; } ================================================ FILE: tsl/src/nodes/gapfill/gapfill_functions.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <fmgr.h> #include <utils/lsyscache.h> #include "gapfill_functions.h" #include "time_bucket.h" Datum gapfill_marker(PG_FUNCTION_ARGS) { if (PG_ARGISNULL(0)) PG_RETURN_NULL(); else PG_RETURN_DATUM(PG_GETARG_DATUM(0)); } #define GAPFILL_TIMEBUCKET_WRAPPER(datatype) \ Datum gapfill_##datatype##_time_bucket(PG_FUNCTION_ARGS) \ { \ /* \ * since time_bucket is STRICT and time_bucket_gapfill \ * is not we need to add explicit checks for NULL here \ */ \ if (PG_ARGISNULL(0) || PG_ARGISNULL(1)) \ PG_RETURN_NULL(); \ return DirectFunctionCall2(ts_##datatype##_bucket, \ PG_GETARG_DATUM(0), \ PG_GETARG_DATUM(1)); \ } GAPFILL_TIMEBUCKET_WRAPPER(int16); GAPFILL_TIMEBUCKET_WRAPPER(int32); GAPFILL_TIMEBUCKET_WRAPPER(int64); GAPFILL_TIMEBUCKET_WRAPPER(date); GAPFILL_TIMEBUCKET_WRAPPER(timestamp); GAPFILL_TIMEBUCKET_WRAPPER(timestamptz); Datum gapfill_timestamptz_timezone_time_bucket(PG_FUNCTION_ARGS) { /* * since time_bucket is STRICT and time_bucket_gapfill * is not we need to add explicit checks for NULL here */ if (PG_ARGISNULL(0) || PG_ARGISNULL(1) || PG_ARGISNULL(2)) PG_RETURN_NULL(); return DirectFunctionCall3(ts_timestamptz_timezone_bucket, PG_GETARG_DATUM(0), PG_GETARG_DATUM(1), PG_GETARG_DATUM(2)); } ================================================ FILE: tsl/src/nodes/gapfill/gapfill_functions.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once /* * Functions used by gapfill which are exported to SQL. * Shell functions for these are defined in src/gapfill.c */ #include <postgres.h> #include <fmgr.h> extern Datum gapfill_marker(PG_FUNCTION_ARGS); extern Datum gapfill_int16_time_bucket(PG_FUNCTION_ARGS); extern Datum gapfill_int32_time_bucket(PG_FUNCTION_ARGS); extern Datum gapfill_int64_time_bucket(PG_FUNCTION_ARGS); extern Datum gapfill_timestamp_time_bucket(PG_FUNCTION_ARGS); extern Datum gapfill_timestamptz_time_bucket(PG_FUNCTION_ARGS); extern Datum gapfill_timestamptz_timezone_time_bucket(PG_FUNCTION_ARGS); extern Datum gapfill_date_time_bucket(PG_FUNCTION_ARGS); ================================================ FILE: tsl/src/nodes/gapfill/gapfill_internal.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <datatype/timestamp.h> #include <nodes/execnodes.h> /* * GapFillFetchState describes the state of subslot in GapFillState: * FETCHED_NONE: no tuple in subslot * FETCHED_ONE: valid tuple in subslot * FETCHED_LAST: no tuple in subslot and no more tuples from subplan * FETCHED_NEXT_GROUP: tuple in subslot belongs to next aggregation group * * The start state is FETCHED_NONE and the end state is FETCHED_LAST * * State transition with single group by time * * no tuple returned * FETCHED_NONE --> fetch_next_tuple -------------------> FETCHED_LAST * ^ | * | | tuple found * | | * | v * └----------FETCHED_ONE * tuple returned * * State transition with multiple groups * * no tuple returned * FETCHED_NONE --> fetch_next_tuple -------------------> FETCHED_LAST * ^ | * | | tuple found * | v * | check_group_changed -------> FETCHED_NEXT_GROUP * | | yes | * | | no | * | | | * | v | * └----------FETCHED_ONE <----------------- * tuple returned */ typedef enum GapFillFetchState { FETCHED_NONE, FETCHED_ONE, FETCHED_NEXT_GROUP, FETCHED_LAST, } GapFillFetchState; /* * NULL_COLUMN: column with no special action from gapfill e.g. min(value) * TIME_COLUMN: column with time_bucket_gapfill call * GROUP_COLUMN: any column appearing in GROUP BY clause * DERIVED_COLUMN: column not appearing in GROUP BY but dependent on GROUP BY column * LOCF_COLUMN: column with locf call * INTERPOLATE_COLUMN: column with interpolate call */ typedef enum GapFillColumnType { NULL_COLUMN, TIME_COLUMN, GROUP_COLUMN, DERIVED_COLUMN, LOCF_COLUMN, INTERPOLATE_COLUMN } GapFillColumnType; typedef struct GapFillColumnState { GapFillColumnType ctype; Oid typid; bool typbyval; int16 typlen; } GapFillColumnState; typedef struct GapFillGroupColumnState { GapFillColumnState base; Datum value; bool isnull; Oid collation; FmgrInfo eq_func; } GapFillGroupColumnState; typedef struct GapFillState { CustomScanState csstate; Plan *subplan; Oid gapfill_typid; /* arguments of the gapfill function call */ List *args; bool have_timezone; int64 gapfill_start; int64 gapfill_end; /* bucket width for fixed-size buckets */ int64 gapfill_period; /* bucket width when bucketing by month */ Interval *gapfill_interval; int64 next_timestamp; /* interval offset for next_timestamp from gapfill_start */ Interval *next_offset; int64 subslot_time; /* time of tuple in subslot */ int time_index; /* position of time column */ TupleTableSlot *subslot; /* TupleTableSlot storing data from subplan */ bool multigroup; /* multiple groupings */ bool groups_initialized; int ncolumns; GapFillColumnState **columns; ProjectionInfo *pi; TupleTableSlot *scanslot; GapFillFetchState state; } GapFillState; Node *gapfill_state_create(CustomScan *); Expr *gapfill_adjust_varnos(GapFillState *state, Expr *expr); Datum gapfill_exec_expr(GapFillState *state, TupleTableSlot *, Expr *expr, bool *isnull); int64 gapfill_datum_get_internal(Datum, Oid); ================================================ FILE: tsl/src/nodes/gapfill/gapfill_plan.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <nodes/execnodes.h> #include <nodes/extensible.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <optimizer/clauses.h> #include <optimizer/optimizer.h> #include <optimizer/pathnode.h> #include <optimizer/paths.h> #include <optimizer/tlist.h> #include <parser/parse_func.h> #include <utils/lsyscache.h> #include "compat/compat.h" #include "gapfill.h" #include "gapfill_internal.h" #include "import/list.h" #include "utils.h" static CustomScanMethods gapfill_plan_methods = { .CustomName = "GapFill", .CreateCustomScanState = gapfill_state_create, }; typedef struct gapfill_walker_context { union { Node *node; Expr *expr; FuncExpr *func; WindowFunc *window; } call; int count; } gapfill_walker_context; /* * Replace Aggref with const NULL */ static Node * gapfill_aggref_mutator(Node *node, void *context) { if (node == NULL) return NULL; if (IsA(node, Aggref)) return (Node *) makeConst(((Aggref *) node)->aggtype, -1, InvalidOid, -2, UnassignedDatum, true, false); return expression_tree_mutator(node, gapfill_aggref_mutator, context); } /* * Check if an expression is NOT a runtime constant. * */ static bool contains_nonconstant_walker(Node *node, void *context) { if (node == NULL) return false; if (IsA(node, Var)) { return true; } if (IsA(node, SubLink)) { return true; } if (IsA(node, PlaceHolderVar)) { return true; } if (IsA(node, Param)) { Param *param = castNode(Param, node); /* PARAM_EXTERN are external query parameters, constant for query duration */ return param->paramkind != PARAM_EXTERN; } if (check_functions_in_node(node, contains_volatile_functions_checker, /* context = */ NULL)) { return true; } return expression_tree_walker(node, contains_nonconstant_walker, context); } static bool contains_nonconstant_expr(Node *node) { return contains_nonconstant_walker(node, NULL); } /* * FuncExpr is time_bucket_gapfill function call */ static inline bool is_gapfill_function_call(FuncExpr *call) { char *func_name = get_func_name(call->funcid); return strncmp(func_name, GAPFILL_FUNCTION, NAMEDATALEN) == 0; } /* * FuncExpr is locf or interpolate function call */ static inline bool is_marker_function_call(FuncExpr *call) { char *func_name = get_func_name(call->funcid); return strncmp(func_name, GAPFILL_LOCF_FUNCTION, NAMEDATALEN) == 0 || strncmp(func_name, GAPFILL_INTERPOLATE_FUNCTION, NAMEDATALEN) == 0; } /* * Find time_bucket_gapfill function call */ static bool gapfill_function_walker(Node *node, gapfill_walker_context *context) { if (node == NULL) return false; if (IsA(node, FuncExpr) && is_gapfill_function_call(castNode(FuncExpr, node))) { context->call.node = node; context->count++; } return expression_tree_walker(node, gapfill_function_walker, context); } /* * Find locf/interpolate function call */ static bool marker_function_walker(Node *node, gapfill_walker_context *context) { if (node == NULL) return false; if (IsA(node, FuncExpr) && is_marker_function_call(castNode(FuncExpr, node))) { context->call.node = node; context->count++; } return expression_tree_walker(node, marker_function_walker, context); } /* * Find window function calls */ static bool window_function_walker(Node *node, gapfill_walker_context *context) { if (node == NULL) return false; if (IsA(node, WindowFunc)) { context->call.node = node; context->count++; } return expression_tree_walker(node, window_function_walker, context); } /* * check if ordering matches the order we need: * all groups need to be part of order * pathkeys must consist of group elements only * last element of pathkeys needs to be time_bucket_gapfill ASC */ static bool gapfill_correct_order(PlannerInfo *root, Path *subpath, FuncExpr *func) { int num_groupby_pathkeys; #if PG16_LT num_groupby_pathkeys = list_length(root->group_pathkeys); #else /* In PG16 group_pathkeys can contain additional pathkeys * used for optimization on ordered aggregates. * We only want to deal with group by elements only here. */ num_groupby_pathkeys = root->num_groupby_pathkeys; #endif if (list_length(subpath->pathkeys) != num_groupby_pathkeys) return false; if (list_length(subpath->pathkeys) > 0) { PathKey *pk = llast(subpath->pathkeys); EquivalenceMember *em = linitial(pk->pk_eclass->ec_members); /* time_bucket_gapfill is last element */ if (pk->pk_cmptype == COMPARE_LT && IsA(em->em_expr, FuncExpr) && ((FuncExpr *) em->em_expr)->funcid == func->funcid) { int i; /* check all groupby pathkeys are part of subpath pathkeys */ for (i = 0; i < num_groupby_pathkeys; i++) { if (!list_member(subpath->pathkeys, list_nth(root->group_pathkeys, i))) return false; } return true; } } return false; } /* Create a gapfill plan node in the form of a CustomScan node. The * purpose of this plan node is to insert tuples for missing groups. * * Note that CustomScan nodes cannot be extended (by struct embedding) because * they might be copied, therefore we pass any extra info in the custom_private * field. * * The gapfill plan takes the original Agg node and imposes itself on top of the * Agg node. During execution, the gapfill node will produce the new tuples. */ static Plan * gapfill_plan_create(PlannerInfo *root, RelOptInfo *rel, CustomPath *path, List *tlist, List *clauses, List *custom_plans) { GapFillPath *gfpath = (GapFillPath *) path; CustomScan *cscan = makeNode(CustomScan); List *args = list_copy(gfpath->func->args); cscan->scan.scanrelid = 0; cscan->scan.plan.targetlist = tlist; cscan->custom_plans = custom_plans; cscan->custom_scan_tlist = tlist; /* When we have original target entries like (agg + group_expr) * we will replace agg with NULL and put resulting expression into exec-fixed "targetlist", * but we need to fix "group_expr" to refer to exec targetlist group column. * Only then we can safely put (NULL + group_column_exec) entry into exec-fixed targetlist. */ List *mutated_agg_exprs = NIL; if (contain_agg_clause((Node *) tlist)) { TargetEntry *tle; ListCell *lc; foreach (lc, tlist) { tle = lfirst(lc); if (contain_agg_clause((Node *) tle)) { Node *entry = copyObject((Node *) tle); entry = gapfill_aggref_mutator(entry, NULL); mutated_agg_exprs = lappend(mutated_agg_exprs, entry); } } } cscan->custom_exprs = list_make1(mutated_agg_exprs); cscan->flags = path->flags; cscan->methods = &gapfill_plan_methods; cscan->custom_private = ts_new_list(T_List, GFP_Count); lfirst(list_nth_cell(cscan->custom_private, GFP_GapfillFunc)) = gfpath->func; lfirst(list_nth_cell(cscan->custom_private, GFP_GroupClause)) = root->parse->groupClause; lfirst(list_nth_cell(cscan->custom_private, GFP_JoinTree)) = root->parse->jointree; lfirst(list_nth_cell(cscan->custom_private, GFP_Args)) = args; return &cscan->scan.plan; } static CustomPathMethods gapfill_path_methods = { .CustomName = "GapFill", .PlanCustomPath = gapfill_plan_create, }; static bool gapfill_expression_walker(Expr *node, bool (*walker)(Node *, gapfill_walker_context *), gapfill_walker_context *context) { context->count = 0; context->call.node = NULL; return (*walker)((Node *) node, context); } /* * Build expression lists for the gapfill node and the node below. * All marker functions will be top-level function calls in the * resulting gapfill node targetlist and will not be included in * the subpath expression list */ static void gapfill_build_pathtarget(PathTarget *pt_upper, PathTarget *pt_path, PathTarget *pt_subpath) { ListCell *lc; int i = -1; foreach (lc, pt_upper->exprs) { Expr *expr = lfirst(lc); gapfill_walker_context context; i++; /* check for locf/interpolate calls */ gapfill_expression_walker(expr, marker_function_walker, &context); if (context.count > 1) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("multiple interpolate/locf function calls per resultset column not " "supported"))); if (context.count == 1) { /* * marker needs to be toplevel for now unless we have a projection capable * node above gapfill node */ if (expr != context.call.expr && !contain_window_function((Node *) expr)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("%s must be toplevel function call", get_func_name(context.call.func->funcid)))); /* if there is an aggregation it needs to be a child of the marker function */ if (contain_agg_clause((Node *) expr) && !contain_agg_clause(linitial(context.call.func->args))) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("aggregate functions must be below %s", get_func_name(context.call.func->funcid)))); if (contain_window_function(context.call.node)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("window functions must not be below %s", get_func_name(context.call.func->funcid)))); add_column_to_pathtarget(pt_path, context.call.expr, pt_upper->sortgrouprefs[i]); add_column_to_pathtarget(pt_subpath, linitial(context.call.func->args), pt_upper->sortgrouprefs[i]); continue; } /* check for plain window function calls without locf/interpolate */ gapfill_expression_walker(expr, window_function_walker, &context); if (context.count > 1) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("multiple window function calls per column not supported"))); if (context.count == 1) { /* * window functions without arguments like rank() don't need to * appear in the target list below WindowAgg node */ if (context.call.window->args != NIL) { ListCell *lc_arg; /* * check arguments past first argument dont have Vars */ for (lc_arg = lnext(context.call.window->args, list_head(context.call.window->args)); lc_arg != NULL; lc_arg = lnext(context.call.window->args, lc_arg)) { if (contain_var_clause(lfirst(lc_arg))) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("window functions with multiple column " "references not supported"))); } if (contain_var_clause(linitial(context.call.window->args))) { add_column_to_pathtarget(pt_path, linitial(context.call.window->args), pt_upper->sortgrouprefs[i]); add_column_to_pathtarget(pt_subpath, linitial(context.call.window->args), pt_upper->sortgrouprefs[i]); } } } else { /* * no locf/interpolate or window functions found so we can * use expression verbatim */ add_column_to_pathtarget(pt_path, expr, pt_upper->sortgrouprefs[i]); add_column_to_pathtarget(pt_subpath, expr, pt_upper->sortgrouprefs[i]); } } } /* * Create a Gapfill Path node. * * The gap fill node needs rows to be sorted by time ASC * so we insert sort paths if the query order does not match * that */ static Path * gapfill_path_create(PlannerInfo *root, Path *subpath, FuncExpr *func) { GapFillPath *path; path = (GapFillPath *) newNode(sizeof(GapFillPath), T_CustomPath); path->cpath.path.pathtype = T_CustomScan; path->cpath.methods = &gapfill_path_methods; /* * parallel_safe must be false because it is not safe to execute this node * in parallel, but it is safe for child nodes to be parallel */ Assert(!path->cpath.path.parallel_safe); path->cpath.path.rows = subpath->rows; path->cpath.path.parent = subpath->parent; path->cpath.path.param_info = subpath->param_info; path->cpath.flags = 0; path->cpath.path.pathkeys = subpath->pathkeys; path->cpath.path.pathtarget = create_empty_pathtarget(); subpath->pathtarget = create_empty_pathtarget(); gapfill_build_pathtarget(root->upper_targets[UPPERREL_FINAL], path->cpath.path.pathtarget, subpath->pathtarget); if (!gapfill_correct_order(root, subpath, func)) { List *new_order = NIL; PathKey *pk_func = NULL; int num_groupby_pathkeys; #if PG16_LT num_groupby_pathkeys = list_length(root->group_pathkeys); #else /* In PG16 group_pathkeys can contain additional pathkeys * used for optimization on ordered aggregates. * We only want to deal with group by elements only here. */ num_groupby_pathkeys = root->num_groupby_pathkeys; #endif int i; /* subpath does not have correct order */ for (i = 0; i < num_groupby_pathkeys; i++) { PathKey *pk = list_nth(root->group_pathkeys, i); EquivalenceMember *em = linitial(pk->pk_eclass->ec_members); if (!pk_func && IsA(em->em_expr, FuncExpr) && ((FuncExpr *) em->em_expr)->funcid == func->funcid) { if (pk->pk_cmptype == COMPARE_LT) pk_func = pk; else pk_func = make_canonical_pathkey(root, pk->pk_eclass, pk->pk_opfamily, BTLessStrategyNumber, pk->pk_nulls_first); } else new_order = lappend(new_order, pk); } if (!pk_func) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("no top level time_bucket_gapfill in group by clause"))); new_order = lappend(new_order, pk_func); subpath = (Path *) create_sort_path(root, subpath->parent, subpath, new_order, root->limit_tuples); } path->cpath.path.startup_cost = subpath->startup_cost; path->cpath.path.total_cost = subpath->total_cost; path->cpath.path.pathkeys = subpath->pathkeys; path->cpath.custom_paths = list_make1(subpath); path->func = func; return &path->cpath.path; } /* * Prepend GapFill node to every group_rel path. * The implementation assumes that TimescaleDB planning hook is called only once * per grouping. */ void plan_add_gapfill(PlannerInfo *root, RelOptInfo *group_rel) { ListCell *lc; Query *parse = root->parse; gapfill_walker_context context = { .call.node = NULL, .count = 0 }; if (CMD_SELECT != parse->commandType || parse->groupClause == NIL) return; /* * Look for time_bucket_gapfill function call in the target list, which * will succeed on every call to plan_add_gapfill, thus it will lead to * incorrect query plan if plan_add_gapfill is called more than once per * grouping. */ gapfill_function_walker((Node *) parse->targetList, &context); if (context.count == 0) return; if (context.count > 1) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("multiple time_bucket_gapfill calls not allowed"))); /* * Check for non-constant timezone parameter. Gapfill needs a consistent * timezone to generate gap timestamps, so column references and * subqueries are not supported. * * The timezone variant has 5 arguments after PostgreSQL fills in * defaults: (bucket_width, ts, timezone, start, finish). The * non-timezone variant has 4: (bucket_width, ts, start, finish). */ FuncExpr *func = context.call.func; int nargs = list_length(func->args); if (nargs == 5) { Expr *tz_arg = lthird(func->args); if (contains_nonconstant_expr((Node *) tz_arg)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("time_bucket_gapfill does not support non-constant timezone"), errhint("Use a constant timezone value."))); } List *copy = group_rel->pathlist; group_rel->pathlist = NIL; group_rel->cheapest_total_path = NULL; group_rel->cheapest_startup_path = NULL; group_rel->cheapest_unique_path = NULL; /* * cheapest_parameterized_paths will be rebuilt by set_cheapest() * after this hook returns. We must not delete ppilist as it contains * ParamPathInfo entries needed for parameterized paths (e.g. LATERAL). */ list_free(group_rel->cheapest_parameterized_paths); group_rel->cheapest_parameterized_paths = NULL; foreach (lc, copy) { add_path(group_rel, gapfill_path_create(root, lfirst(lc), context.call.func)); } list_free(copy); } static inline bool is_gapfill_path(Path *path) { return IsA(path, CustomPath) && castNode(CustomPath, path)->methods == &gapfill_path_methods; } /* * Since we construct the targetlist for the gapfill node from the * final targetlist we need to adjust any intermediate targetlists * between toplevel window agg node and gapfill node. This adjustment * is only necessary if multiple WindowAgg nodes are present. * In that case we need to adjust the targetlists of nodes between * toplevel WindowAgg node and Gapfill node * * Gapfill plan with multiple WindowAgg nodes: * * WindowAgg * -> WindowAgg * -> Custom Scan (GapFill) * -> Sort * Sort Key: (time_bucket_gapfill(1, "time")) * -> HashAggregate * Group Key: time_bucket_gapfill(1, "time") * -> Seq Scan on metrics_int * */ void gapfill_adjust_window_targetlist(PlannerInfo *root, RelOptInfo *input_rel, RelOptInfo *output_rel) { ListCell *lc; if (!is_gapfill_path(linitial(input_rel->pathlist))) return; foreach (lc, output_rel->pathlist) { WindowAggPath *toppath = lfirst(lc); /* * the toplevel WindowAggPath has the highest index. If winref is * 1 we only have one WindowAggPath if its greater then 1 then there * are multiple WindowAgg nodes. * * we skip toplevel WindowAggPath because targetlist of toplevel WindowAggPath * is our starting point for building gapfill targetlist so we don't need to * adjust the toplevel targetlist */ if (IsA(toppath, WindowAggPath) && toppath->winclause->winref > 1) { WindowAggPath *path; for (path = (WindowAggPath *) toppath->subpath; IsA(path, WindowAggPath); path = (WindowAggPath *) path->subpath) { PathTarget *pt_top = toppath->path.pathtarget; PathTarget *pt; ListCell *lc_expr; int i = -1; pt = create_empty_pathtarget(); /* * for each child we build targetlist based on top path * targetlist */ foreach (lc_expr, pt_top->exprs) { gapfill_walker_context context; i++; gapfill_expression_walker(lfirst(lc_expr), window_function_walker, &context); /* * we error out on multiple window functions per resultset column * when building gapfill node targetlist so we only assert here */ Assert(context.count <= 1); if (context.count == 1) { if (context.call.window->winref <= path->winclause->winref) /* * window function of current level or below * so we can put in verbatim */ add_column_to_pathtarget(pt, lfirst(lc_expr), pt_top->sortgrouprefs[i]); else if (context.call.window->args != NIL) { ListCell *lc_arg; if (list_length(context.call.window->args) > 1) /* * check arguments past first argument dont have Vars */ for (lc_arg = lnext(context.call.window->args, list_head(context.call.window->args)); lc_arg != NULL; lc_arg = lnext(context.call.window->args, lc_arg)) { if (contain_var_clause(lfirst(lc_arg))) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("window functions with multiple column " "references not supported"))); } if (contain_var_clause(linitial(context.call.window->args))) add_column_to_pathtarget(pt, linitial(context.call.window->args), pt_top->sortgrouprefs[i]); } } else add_column_to_pathtarget(pt, lfirst(lc_expr), pt_top->sortgrouprefs[i]); } path->path.pathtarget = pt; } } } } ================================================ FILE: tsl/src/nodes/gapfill/interpolate.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/htup_details.h> #include <catalog/pg_type.h> #include <utils/builtins.h> #include <utils/datum.h> #include <utils/numeric.h> #include <utils/typcache.h> #include "compat/compat.h" #include "gapfill_internal.h" #include "interpolate.h" #define INTERPOLATE(x, x0, x1, y0, y1) (((y0) * ((x1) - (x)) + (y1) * ((x) - (x0))) / ((x1) - (x0))) /* * gapfill_interpolate_initialize gets called when plan is initialized for every interpolate column */ void gapfill_interpolate_initialize(GapFillInterpolateColumnState *interpolate, GapFillState *state, FuncExpr *function) { interpolate->prev.isnull = true; interpolate->next.isnull = true; if (list_length(function->args) > 1) interpolate->lookup_before = gapfill_adjust_varnos(state, lsecond(function->args)); if (list_length(function->args) > 2) interpolate->lookup_after = gapfill_adjust_varnos(state, lthird(function->args)); } /* * gapfill_interpolate_group_change gets called when a new aggregation group becomes active */ void gapfill_interpolate_group_change(GapFillInterpolateColumnState *column, int64 time, Datum value, bool isnull) { column->prev.isnull = true; column->next.isnull = isnull; if (!isnull) { column->next.time = time; column->next.value = datumCopy(value, column->base.typbyval, column->base.typlen); } } /* * gapfill_interpolate_tuple_fetched gets called when a new tuple is fetched from subplan */ void gapfill_interpolate_tuple_fetched(GapFillInterpolateColumnState *column, int64 time, Datum value, bool isnull) { column->next.isnull = isnull; if (!isnull) { column->next.time = time; column->next.value = datumCopy(value, column->base.typbyval, column->base.typlen); } } /* * gapfill_interpolate_tuple_returned gets called when subplan tuple is returned */ void gapfill_interpolate_tuple_returned(GapFillInterpolateColumnState *column, int64 time, Datum value, bool isnull) { column->next.isnull = true; column->prev.isnull = isnull; if (!isnull) { column->prev.time = time; column->prev.value = datumCopy(value, column->base.typbyval, column->base.typlen); } } /* * Do out of bounds lookup for interpolation */ static void gapfill_fetch_sample(GapFillState *state, GapFillInterpolateColumnState *column, GapFillInterpolateSample *sample, Expr *lookup) { HeapTupleHeader th; HeapTupleData tuple; TupleDesc tupdesc; Datum value; bool isnull; Datum datum = gapfill_exec_expr(state, state->scanslot, lookup, &isnull); if (isnull) { sample->isnull = true; return; } th = DatumGetHeapTupleHeader(datum); if (HeapTupleHeaderGetNatts(th) != 2) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("interpolate RECORD arguments must have 2 elements"))); /* Extract type information from the tuple itself */ Assert(RECORDOID == HeapTupleHeaderGetTypeId(th)); tupdesc = lookup_rowtype_tupdesc(HeapTupleHeaderGetTypeId(th), HeapTupleHeaderGetTypMod(th)); /* Build a temporary HeapTuple control structure */ tuple.t_len = HeapTupleHeaderGetDatumLength(th); ItemPointerSetInvalid(&(tuple.t_self)); tuple.t_tableOid = InvalidOid; tuple.t_data = th; /* check first element in record matches timestamp datatype */ if (TupleDescAttr(tupdesc, 0)->atttypid != state->columns[state->time_index]->typid) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("first argument of interpolate returned record must match used timestamp " "datatype"), errdetail("Returned type %s does not match expected type %s.", format_type_be(TupleDescAttr(tupdesc, 0)->atttypid), format_type_be(column->base.typid)))); /* check second element in record matches interpolate datatype */ if (TupleDescAttr(tupdesc, 1)->atttypid != column->base.typid) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("second argument of interpolate returned record must match used " "interpolate datatype"), errdetail("Returned type %s does not match expected type %s.", format_type_be(TupleDescAttr(tupdesc, 1)->atttypid), format_type_be(column->base.typid)))); value = heap_getattr(&tuple, 1, tupdesc, &sample->isnull); if (!sample->isnull) { sample->time = gapfill_datum_get_internal(value, state->gapfill_typid); value = heap_getattr(&tuple, 2, tupdesc, &sample->isnull); if (!sample->isnull) sample->value = datumCopy(value, column->base.typbyval, column->base.typlen); } ReleaseTupleDesc(tupdesc); } /* Calculate the interpolation using numerics, returning the result as a numeric datum */ static Datum interpolate_numeric(int64 x_i, int64 x0_i, int64 x1_i, Datum y0, Datum y1) { Datum x0 = DirectFunctionCall1(int8_numeric, Int64GetDatum(x0_i)); Datum x1 = DirectFunctionCall1(int8_numeric, Int64GetDatum(x1_i)); Datum x = DirectFunctionCall1(int8_numeric, Int64GetDatum(x_i)); Datum x1_sub_x = DirectFunctionCall2(numeric_sub, x1, x); Datum x_sub_x0 = DirectFunctionCall2(numeric_sub, x, x0); Datum y0_mul_x1_sub_x = DirectFunctionCall2(numeric_mul, y0, x1_sub_x); Datum y1_mul_x_sub_x0 = DirectFunctionCall2(numeric_mul, y1, x_sub_x0); Datum numerator = DirectFunctionCall2(numeric_add, y0_mul_x1_sub_x, y1_mul_x_sub_x0); Datum denominator = DirectFunctionCall2(numeric_sub, x1, x0); return DirectFunctionCall2(numeric_div, numerator, denominator); } /* * gapfill_interpolate_calculate gets called for every gapfilled tuple to calculate values * * Calculate linear interpolation value * y = (y0(x1-x) + y1(x-x0))/(x1-x0) */ void gapfill_interpolate_calculate(GapFillInterpolateColumnState *column, GapFillState *state, int64 time, Datum *value, bool *isnull) { int64 x, x0, x1; Datum y0, y1; /* only evaluate expr for first tuple */ if (column->prev.isnull && column->lookup_before && time == state->gapfill_start) gapfill_fetch_sample(state, column, &column->prev, column->lookup_before); if (column->next.isnull && column->lookup_after && (FETCHED_LAST == state->state || FETCHED_NEXT_GROUP == state->state)) gapfill_fetch_sample(state, column, &column->next, column->lookup_after); *isnull = column->prev.isnull || column->next.isnull; if (*isnull) return; y0 = column->prev.value; y1 = column->next.value; x = time; x0 = column->prev.time; x1 = column->next.time; switch (column->base.typid) { /* All integer types must use numeric-based interpolation calculations since they are * multiplied by int64 and this could cause an overflow. numerics also interpolate better * because the answer is rounded and not truncated. We can't use float8 because that doesn't handle really big ints exactly. We can't use the Postgres INT128 implementation because it doesn't support division. */ case INT2OID: *value = DirectFunctionCall1(numeric_int2, interpolate_numeric(x, x0, x1, DirectFunctionCall1(int2_numeric, y0), DirectFunctionCall1(int2_numeric, y1))); break; case INT4OID: *value = DirectFunctionCall1(numeric_int4, interpolate_numeric(x, x0, x1, DirectFunctionCall1(int4_numeric, y0), DirectFunctionCall1(int4_numeric, y1))); break; case INT8OID: *value = DirectFunctionCall1(numeric_int8, interpolate_numeric(x, x0, x1, DirectFunctionCall1(int8_numeric, y0), DirectFunctionCall1(int8_numeric, y1))); break; case FLOAT4OID: /* Shortcircuit calculation when y0 == y1 for float because otherwise * output will be unstable for certain values due to float rounding. */ if (DatumGetFloat4(y0) == DatumGetFloat4(y1)) *value = y0; else *value = Float4GetDatum(INTERPOLATE(x, x0, x1, DatumGetFloat4(y0), DatumGetFloat4(y1))); break; case FLOAT8OID: /* Shortcircuit calculation when y0 == y1 for float because otherwise * output will be unstable for certain values due to float rounding. */ if (DatumGetFloat8(y0) == DatumGetFloat8(y1)) *value = y0; else *value = Float8GetDatum(INTERPOLATE(x, x0, x1, DatumGetFloat8(y0), DatumGetFloat8(y1))); break; default: /* * should never happen since interpolate is not defined for other * datatypes */ ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("unsupported datatype for interpolate: %s", format_type_be(column->base.typid)))); pg_unreachable(); break; } } ================================================ FILE: tsl/src/nodes/gapfill/interpolate.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include "gapfill_internal.h" typedef struct GapFillInterpolateSample { int64 time; Datum value; bool isnull; } GapFillInterpolateSample; typedef struct GapFillInterpolateColumnState { GapFillColumnState base; Expr *lookup_before; Expr *lookup_after; GapFillInterpolateSample prev; GapFillInterpolateSample next; } GapFillInterpolateColumnState; void gapfill_interpolate_initialize(GapFillInterpolateColumnState *, GapFillState *, FuncExpr *); void gapfill_interpolate_group_change(GapFillInterpolateColumnState *, int64, Datum, bool); void gapfill_interpolate_tuple_fetched(GapFillInterpolateColumnState *, int64, Datum, bool); void gapfill_interpolate_tuple_returned(GapFillInterpolateColumnState *, int64, Datum, bool); void gapfill_interpolate_calculate(GapFillInterpolateColumnState *, GapFillState *, int64, Datum *, bool *); ================================================ FILE: tsl/src/nodes/gapfill/locf.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_type.h> #include <utils/datum.h> #include "gapfill_internal.h" #include "locf.h" /* * gapfill_locf_initialize gets called when plan is initialized for every locf column */ void gapfill_locf_initialize(GapFillLocfColumnState *locf, GapFillState *state, FuncExpr *function) { locf->isnull = true; /* check if out of boundary lookup expression was supplied */ if (list_length(function->args) > 1) locf->lookup_last = gapfill_adjust_varnos(state, lsecond(function->args)); /* check if treat_null_as_missing was supplied */ if (list_length(function->args) > 2) { Const *treat_null_as_missing = lthird(function->args); if (!IsA(treat_null_as_missing, Const) || treat_null_as_missing->consttype != BOOLOID) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg( "invalid locf argument: treat_null_as_missing must be a BOOL literal"))); if (!treat_null_as_missing->constisnull) locf->treat_null_as_missing = DatumGetBool(treat_null_as_missing->constvalue); } } /* * gapfill_locf_group_change gets called when a new aggregation group becomes active */ void gapfill_locf_group_change(GapFillLocfColumnState *locf) { locf->isnull = true; } /* * gapfill_locf_tuple_returned gets called when subplan tuple is returned */ void gapfill_locf_tuple_returned(GapFillLocfColumnState *locf, Datum value, bool isnull) { locf->isnull = isnull; if (!isnull) locf->value = datumCopy(value, locf->base.typbyval, locf->base.typlen); } /* * gapfill_locf_calculate gets called for every gapfilled tuple to calculate values */ void gapfill_locf_calculate(GapFillLocfColumnState *locf, GapFillState *state, TupleTableSlot *ecxt_slot, int64 time, Datum *value, bool *isnull) { /* only evaluate expr for first tuple */ if (locf->isnull && locf->lookup_last && time == state->gapfill_start) locf->value = gapfill_exec_expr(state, ecxt_slot, locf->lookup_last, &locf->isnull); *value = locf->value; *isnull = locf->isnull; } ================================================ FILE: tsl/src/nodes/gapfill/locf.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include "gapfill_internal.h" typedef struct GapFillLocfColumnState { GapFillColumnState base; Expr *lookup_last; Datum value; bool isnull; bool treat_null_as_missing; } GapFillLocfColumnState; void gapfill_locf_initialize(GapFillLocfColumnState *, GapFillState *, FuncExpr *); void gapfill_locf_group_change(GapFillLocfColumnState *); void gapfill_locf_tuple_returned(GapFillLocfColumnState *, Datum, bool); void gapfill_locf_calculate(GapFillLocfColumnState *, GapFillState *, TupleTableSlot *, int64, Datum *, bool *); ================================================ FILE: tsl/src/nodes/skip_scan/CMakeLists.txt ================================================ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/planner.c ${CMAKE_CURRENT_SOURCE_DIR}/exec.c) target_sources(${TSL_LIBRARY_NAME} PRIVATE ${SOURCES}) ================================================ FILE: tsl/src/nodes/skip_scan/README.md ================================================ # SkipScan # This module implements SkipScan; an optimization for `SELECT DISTINCT ON`. Usually for `SELECT DISTINCT ON` Postgres will plan either a `UNIQUE` over a sorted path, or some form of aggregate. In either case, it needs to scan the entire table, even in cases where there are only a few unique values. A skip scan optimizes this case when we have an ordered index. Instead of scanning the entire table and deduplicating after, the scan remembers the last value returned, and searches the index for the next value after that one. This means that for a table with `k` keys, with `u` distinct values, a skip scan runs in time `u * log(k)` as opposed to scanning then deduplicating, which takes time `k`. We can write the number of unique values `u` as of function of `k` by dividing by the number of repeats `r` i.e. `u = k/r` this means that a skip scan will be faster if each key is repeated more than a logarithmic number of times, i.e. if `r > log(k)` then `u * log(k) < k/log(k) * log(k) < k`. ## Implementation ## We plan our skip scan with a tree something like ```SQL Custom Scan (SkipScan) on table -> Index Scan using table_key_idx on table Index Cond: (key > NULL) ``` After each iteration through the `SkipScan` we replace the `key > NULL` with a `key > [next value we are returning]` and restart the underlying `IndexScan`. There are some subtleties around `NULL` handling, see the source file for more detail. ## Planning Heuristics ## To plan our SkipScan we look for a compatible plan, for instance ```SQL Unique -> Index Scan ``` or ```SQL Unique -> Merge Append -> Index Scan ... ``` given such a plan, we know the index is sorted in an order with the distinct key(s) first, so we can add quals to the `IndexScan` representing the previous key returned, and thus skip over the repeated values. The `Unique` node tells us which columns are relevant. We use this to create plans that look like ```SQL Unique -> Custom Scan (SkipScan) on skip_scan -> Index Scan using skip_scan_dev_name_idx on skip_scan ``` or ```SQL Unique -> Merge Append Sort Key: _hyper_2_1_chunk.dev_name -> Custom Scan (SkipScan) on _hyper_2_1_chunk -> Index Scan using _hyper_2_1_chunk_idx on _hyper_2_1_chunk -> Custom Scan (SkipScan) on _hyper_2_2_chunk -> Index Scan using _hyper_2_2_chunk_idx on _hyper_2_2_chunk ``` respectively. While we could remove the top-level Unique node for the single chunk/normal table case we keep it so we don't need to support projection as postgres won't modify the SkipScan targetlist that way. ## Postgres-Native Skip Scan ## Upstream postgres is also working on a skip scan implementation, see e.g. https://commitfest.postgresql.org/32/1741/ As when this document was first written, it is not yet merged. Their strategy involves integrating this functionality into the btree searching code, and will be available in PG15 at the earliest. The two implementations should not interfere with eachother. ================================================ FILE: tsl/src/nodes/skip_scan/exec.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * SkipScan is an optimized form of SELECT DISTINCT ON (column) * Conceptually, a SkipScan is a regular IndexScan with an additional skip-qual like * WHERE column > [previous value of column] * * Implementing this qual is complicated by two factors: * 1. The first time through the SkipScan there is no previous value for the * DISTINCT column. * 2. NULL values don't behave nicely with ordering operators. * * To get around these issues, we have to special case those two cases. All in * all, the SkipScan's state machine evolves according to the following flowchart * * start * | * +================================+ * | search for NULL if NULLS FIRST | * +================================+ * | * v * +=====================+ +==============================+ * | search for non-NULL |--found-->| search for values after prev | * +=====================+ value +==============================+ * | | * v v * +================================+ * | search for NULL if NULLS LAST | * +================================+ * | * v * /===========\ * | DONE | * \===========/ * * * * N-key SkipScan needs to do 2^N null check stages when using the above scheme, * made even more complicated with having to change searches for previous keys. * * So we made a decision to support multikey SkipScan in NOT NULL mode only. * * For N-key SkipScan we search with these predicates when current key = K: * (key_1 = prev_1),...,(key_K > prev_K),(key_K+1 IS NOT NULL)...(key_N IS NOT NULL) * * As all skip keys are NOT NULL, "IS NOT NULL" fetches the tuple with no previous value. * * We start the search with K=1 i.e. with these predicates: * (key_1 IS NOT NULL),...,(key_N IS NOT NULL). * * When a tuple is fetched we set K=N as we can fill all previous values, search is now: * (key_1 = prev_1),...,(key_N > prev_N) * * When no tuple is fetched and K>1 we can relax the search and move to previous key (K-1): * (key_1 = prev_1),...,(key_K-1 > prev_K-1),(key_K IS NOT NULL)...(key_N IS NOT NULL) * * When no tuple is fetched and K=1, we are done. * * Multikey SkipScan flowchart: * start (K=1) * | +---------+ * | | | * v v | * +=================================+ | * | search for NOT NULL after K | | * +=================================+ | * | | | * | found value | | * v | | * +==============================+ | | * | search for values after prev | | | * +==============================+ | | * | | | * | no value | | * v v | * +======================+ | * | K=1 | K>1 | * v v | * /===========\ +=========+ | * | DONE | | K = K-1 |---+ * \===========/ +=========+ * */ #include <postgres.h> #include <access/genam.h> #include <access/nbtree.h> #include <nodes/extensible.h> #include <nodes/pg_list.h> #include <utils/datum.h> #include "guc.h" #include "nodes/columnar_scan/columnar_scan.h" #include "nodes/columnar_scan/exec.h" #include "nodes/skip_scan/skip_scan.h" typedef enum SkipScanStage { SS_BEGIN = 0, SS_NULLS_FIRST, SS_NOT_NULL, SS_VALUES, SS_NULLS_LAST, SS_PREV_KEY, SS_END, } SkipScanStage; typedef struct SkipKeyData { ScanKey skip_key; /* Comparison value filled in at runtime */ Datum prev_datum; bool prev_is_null; /* Info about the type we are performing DISTINCT on */ bool distinct_by_val; int distinct_col_attnum; int distinct_typ_len; int sk_attno; SkipKeyNullStatus nulls; } SkipKeyData; typedef struct SkipScanState { CustomScanState cscan_state; IndexScanDesc *scan_desc; MemoryContext ctx; /* Interior Index(Only)Scan the SkipScan runs over */ ScanState *idx; /* Pointers into the Index(Only)Scan */ int *num_scan_keys; ScanKey *scan_keys; int num_skip_keys; SkipKeyData *skip_keys; /* Skip key with ">" qual, coming after "=" skip quals for multikey SkipScan */ int current_key; /* For Multikey SkipScan we keep copies of "sk_func" for "=" and ">" for keys 1..N-1 * to be swapped during execution. */ FmgrInfo *eq_funcs; /* Will be filled after IndexScan scankeys have been initialized */ FmgrInfo *comp_funcs; StrategyNumber *comp_strategies; SkipScanStage stage; /* rescan required before getting next tuple */ bool needs_rescan; /* child_plan node is the input for skip scan plan node. * if skip scan is directly over index scan, child_plan = idx_scan * if skip scan is over compressed chunk, * idx_scan = compressed index scan, * child_plan = decompressed input into skip scan */ Plan *child_plan; void *idx_scan; } SkipScanState; static bool has_nulls_first(SkipScanState *state); static bool has_nulls_last(SkipScanState *state); static void skip_scan_rescan_index(SkipScanState *state); static void skip_scan_switch_stage(SkipScanState *state, SkipScanStage new_stage); static void skip_scan_begin(CustomScanState *node, EState *estate, int eflags) { SkipScanState *state = (SkipScanState *) node; state->ctx = AllocSetContextCreate(estate->es_query_cxt, "skipscan", ALLOCSET_DEFAULT_SIZES); node->custom_ps = list_make1((ScanState *) ExecInitNode(state->child_plan, estate, eflags)); ScanState *child_state = linitial(node->custom_ps); if (state->child_plan == state->idx_scan) { state->idx = child_state; } else if (IsA(child_state, CustomScanState)) { Assert(ts_is_columnar_scan_plan(state->child_plan)); state->idx = linitial(castNode(CustomScanState, child_state)->custom_ps); } else elog(ERROR, "unknown subscan type in SkipScan"); if (IsA(state->idx_scan, IndexScan)) { IndexScanState *idx = castNode(IndexScanState, state->idx); state->scan_keys = &idx->iss_ScanKeys; state->num_scan_keys = &idx->iss_NumScanKeys; state->scan_desc = &idx->iss_ScanDesc; } else if (IsA(state->idx_scan, IndexOnlyScan)) { IndexOnlyScanState *idx = castNode(IndexOnlyScanState, state->idx); state->scan_keys = &idx->ioss_ScanKeys; state->num_scan_keys = &idx->ioss_NumScanKeys; state->scan_desc = &idx->ioss_ScanDesc; } else elog(ERROR, "unknown subscan type in SkipScan"); /* scankeys are not setup for explain only */ if (eflags & EXEC_FLAG_EXPLAIN_ONLY) return; /* find position of our skip key * skip key is put as first key for the respective column in sort_indexquals */ ScanKey scankeydata = *state->scan_keys; int j = 0; for (int i = 0; i < *state->num_scan_keys; i++) { if (scankeydata[i].sk_flags == SK_ISNULL && scankeydata[i].sk_attno == state->skip_keys[j].sk_attno) { SkipKeyData *skipkeydata = &state->skip_keys[j++]; skipkeydata->skip_key = &scankeydata[i]; /* Set up ">" sk_func swaps for skip keys 1..N-1 */ if (j < state->num_skip_keys) { state->comp_strategies[j - 1] = scankeydata[i].sk_strategy; fmgr_info_copy(&state->comp_funcs[j - 1], &scankeydata[i].sk_func, CurrentMemoryContext); } if (j == state->num_skip_keys) break; } } if (j < state->num_skip_keys) elog(ERROR, "ScanKey for skip qual not found"); /* when we fetch the 1st tuple we update all skip keys from 0 to N */ state->current_key = 0; } static bool has_nulls_first(SkipScanState *state) { return state->skip_keys[0].nulls == SK_NULLS_FIRST; } static bool has_nulls_last(SkipScanState *state) { return state->skip_keys[0].nulls == SK_NULLS_LAST; } static void skip_scan_rescan_index(SkipScanState *state) { /* if the scan in the child scan has not been * setup yet which is true before the first tuple * has been retrieved from child scan we cannot * trigger rescan but since the child scan * has not been initialized it will pick up * any ScanKey changes we did */ if (*state->scan_desc) { index_rescan(*state->scan_desc, *state->scan_keys, *state->num_scan_keys, NULL /*orderbys*/, 0 /*norderbys*/); /* Discard current compressed index tuple as we are ready to move to the next compressed * tuple via SkipScan */ ScanState *child = linitial(state->cscan_state.custom_ps); if (ts_is_columnar_scan_plan(state->child_plan)) { ColumnarScanState *ds = (ColumnarScanState *) child; TupleTableSlot *slot = ds->batch_queue->funcs->top_tuple(ds->batch_queue); if (slot) { compressed_batch_discard_tuples((DecompressBatchState *) slot); } } } state->needs_rescan = false; } /* * Update skip scankey flags according to stage */ static void skip_scan_switch_stage(SkipScanState *state, SkipScanStage new_stage) { Assert(new_stage > state->stage || state->num_skip_keys > 1); switch (new_stage) { case SS_NOT_NULL: for (int i = 0; i < state->num_skip_keys; i++) { state->skip_keys[i].skip_key->sk_flags = SK_ISNULL | SK_SEARCHNOTNULL; state->skip_keys[i].skip_key->sk_argument = 0; } state->current_key = 0; state->needs_rescan = true; break; case SS_PREV_KEY: /* Done searching with ">" for this key: set this key to NOT NULL i.e. any value, * set previous "=" key to search with ">". */ state->skip_keys[state->current_key].skip_key->sk_flags = SK_ISNULL | SK_SEARCHNOTNULL; state->current_key--; state->skip_keys[state->current_key].skip_key->sk_flags = 0; fmgr_info_copy(&state->skip_keys[state->current_key].skip_key->sk_func, &state->comp_funcs[state->current_key], CurrentMemoryContext); state->skip_keys[state->current_key].skip_key->sk_strategy = state->comp_strategies[state->current_key]; state->needs_rescan = true; break; case SS_VALUES: for (int i = 0; i < state->num_skip_keys; i++) { state->skip_keys[i].skip_key->sk_flags = 0; /* reset all ">" back to "=" from the current key to N-1 */ if (i >= state->current_key && i < state->num_skip_keys - 1) { fmgr_info_copy(&state->skip_keys[i].skip_key->sk_func, &state->eq_funcs[i], CurrentMemoryContext); state->skip_keys[i].skip_key->sk_strategy = BTEqualStrategyNumber; } } state->current_key = state->num_skip_keys - 1; state->needs_rescan = true; break; case SS_NULLS_LAST: case SS_NULLS_FIRST: state->skip_keys[0].skip_key->sk_flags = SK_ISNULL | SK_SEARCHNULL; state->skip_keys[0].skip_key->sk_argument = 0; state->needs_rescan = true; break; case SS_BEGIN: case SS_END: break; } state->stage = new_stage; } static void skip_scan_update_key(SkipScanState *state, TupleTableSlot *slot) { for (int i = state->current_key; i < state->num_skip_keys; i++) { if (!state->skip_keys[i].prev_is_null && !state->skip_keys[i].distinct_by_val) { Assert(state->stage == SS_VALUES || state->num_skip_keys > 1); pfree(DatumGetPointer(state->skip_keys[i].prev_datum)); } MemoryContext old_ctx = MemoryContextSwitchTo(state->ctx); state->skip_keys[i].prev_datum = slot_getattr(slot, state->skip_keys[i].distinct_col_attnum, &state->skip_keys[i].prev_is_null); if (state->skip_keys[i].prev_is_null) { state->skip_keys[i].skip_key->sk_flags = SK_ISNULL; state->skip_keys[i].skip_key->sk_argument = 0; } else { state->skip_keys[i].prev_datum = datumCopy(state->skip_keys[i].prev_datum, state->skip_keys[i].distinct_by_val, state->skip_keys[i].distinct_typ_len); state->skip_keys[i].skip_key->sk_argument = state->skip_keys[i].prev_datum; } MemoryContextSwitchTo(old_ctx); } /* we need to do a rescan whenever we modify the ScanKey */ state->needs_rescan = true; } static TupleTableSlot * skip_scan_exec(CustomScanState *node) { SkipScanState *state = (SkipScanState *) node; TupleTableSlot *result; ScanState *child_state; /* * We are not supporting projection here since no plan * we generate will need it as our SkipScan node will * always be below Unique need so our targetlist * will not get modified by postgres. */ Assert(!node->ss.ps.ps_ProjInfo); while (true) { if (state->needs_rescan) skip_scan_rescan_index(state); switch (state->stage) { case SS_BEGIN: if (has_nulls_first(state)) skip_scan_switch_stage(state, SS_NULLS_FIRST); else skip_scan_switch_stage(state, SS_NOT_NULL); break; case SS_NULLS_FIRST: child_state = linitial(state->cscan_state.custom_ps); result = child_state->ps.ExecProcNode(&child_state->ps); /* * if we found a NULL value we return it, otherwise * we restart the scan looking for non-NULL */ skip_scan_switch_stage(state, SS_NOT_NULL); if (!TupIsNull(result)) return result; break; case SS_NOT_NULL: case SS_PREV_KEY: case SS_VALUES: child_state = linitial(state->cscan_state.custom_ps); result = child_state->ps.ExecProcNode(&child_state->ps); if (!TupIsNull(result)) { /* * if we found a tuple we update the skip scan key * and look for values greater than the value we just * found. If this is the first non-NULL value we * also switch stage to look for values greater than * that in subsequent calls. */ skip_scan_update_key(state, result); if (state->stage == SS_NOT_NULL || state->stage == SS_PREV_KEY) skip_scan_switch_stage(state, SS_VALUES); return result; } else { /* * if there are no more values that satisfy * the skip constraint we are either done * for NULLS FIRST ordering or need to check * for NULLs if we have NULLS LAST ordering * * Or we can move back one key for multikey SkipScan to relax the search, * i.e. make current key NOT NULL (any value) and change previous search from * "=" to ">" */ if (has_nulls_last(state)) skip_scan_switch_stage(state, SS_NULLS_LAST); else if (state->current_key > 0) skip_scan_switch_stage(state, SS_PREV_KEY); else skip_scan_switch_stage(state, SS_END); } break; case SS_NULLS_LAST: child_state = linitial(state->cscan_state.custom_ps); result = child_state->ps.ExecProcNode(&child_state->ps); skip_scan_switch_stage(state, SS_END); return result; break; case SS_END: return NULL; break; } } } static void skip_scan_end(CustomScanState *node) { SkipScanState *state = (SkipScanState *) node; ScanState *child_state = linitial(state->cscan_state.custom_ps); ExecEndNode(&child_state->ps); } static void skip_scan_rescan(CustomScanState *node) { SkipScanState *state = (SkipScanState *) node; /* reset stage so we can assert in skip_scan_switch_stage that stage always moves forward */ state->stage = SS_BEGIN; /* Switching state here instead of in the main loop * means we dont have to call skip_scan_rescan_index * as ExecReScan on the child scan takes care of that. */ if (has_nulls_first(state)) skip_scan_switch_stage(state, SS_NULLS_FIRST); else skip_scan_switch_stage(state, SS_NOT_NULL); for (int i = 0; i < state->num_skip_keys; i++) { state->skip_keys[i].prev_is_null = true; state->skip_keys[i].prev_datum = 0; } state->needs_rescan = false; ScanState *child_state = linitial(state->cscan_state.custom_ps); ExecReScan(&child_state->ps); MemoryContextReset(state->ctx); } static CustomExecMethods skip_scan_state_methods = { .CustomName = "SkipScanState", .BeginCustomScan = skip_scan_begin, .EndCustomScan = skip_scan_end, .ExecCustomScan = skip_scan_exec, .ReScanCustomScan = skip_scan_rescan, }; Node * tsl_skip_scan_state_create(CustomScan *cscan) { SkipScanState *state = (SkipScanState *) newNode(sizeof(SkipScanState), T_CustomScanState); state->child_plan = linitial(cscan->custom_plans); if (ts_is_columnar_scan_plan(state->child_plan)) { CustomScan *csplan = castNode(CustomScan, state->child_plan); state->idx_scan = linitial(csplan->custom_plans); } else { state->idx_scan = state->child_plan; } state->stage = SS_BEGIN; /* set up N skipkeyinfos for N skip keys */ List *skinfos = (List *) linitial(cscan->custom_private); state->num_skip_keys = list_length(skinfos); state->skip_keys = palloc(sizeof(SkipKeyData) * state->num_skip_keys); ListCell *lc; int i = 0; foreach (lc, skinfos) { List *skipkeyinfo = (List *) lfirst(lc); state->skip_keys[i].distinct_col_attnum = list_nth_int(skipkeyinfo, SK_DistinctColAttno); state->skip_keys[i].distinct_by_val = list_nth_int(skipkeyinfo, SK_DistinctByVal); state->skip_keys[i].distinct_typ_len = list_nth_int(skipkeyinfo, SK_DistinctTypeLen); state->skip_keys[i].nulls = list_nth_int(skipkeyinfo, SK_NullStatus); Assert(state->num_skip_keys == 1 || state->skip_keys[i].nulls == SK_NOT_NULL); state->skip_keys[i].sk_attno = list_nth_int(skipkeyinfo, SK_IndexKeyAttno); state->skip_keys[i].prev_is_null = true; i++; } state->eq_funcs = NULL; state->comp_funcs = NULL; state->comp_strategies = NULL; /* set up N-1 equality ops for N skip keys if N>1 */ if (state->num_skip_keys > 1) { /* Should have a list of N-1 equality op Oids for N skip keys if N>1 */ Assert(list_length(cscan->custom_private) == 2); List *eqoids = (List *) lsecond(cscan->custom_private); state->eq_funcs = palloc(sizeof(FmgrInfo) * (state->num_skip_keys - 1)); state->comp_funcs = palloc(sizeof(FmgrInfo) * (state->num_skip_keys - 1)); state->comp_strategies = palloc(sizeof(StrategyNumber) * (state->num_skip_keys - 1)); int i = 0; /* Set up "=" sk_funcs for keys 1..N-1 */ foreach (lc, eqoids) { Oid eqoid = lfirst_oid(lc); Assert(OidIsValid(eqoid)); fmgr_info(eqoid, &state->eq_funcs[i++]); } Assert(i == state->num_skip_keys - 1); } state->cscan_state.methods = &skip_scan_state_methods; return (Node *) state; } ================================================ FILE: tsl/src/nodes/skip_scan/planner.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/sysattr.h> #include <nodes/extensible.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <nodes/pathnodes.h> #include <optimizer/clauses.h> #include <optimizer/cost.h> #include <optimizer/optimizer.h> #include <optimizer/pathnode.h> #include <optimizer/paths.h> #include <optimizer/planmain.h> #include <optimizer/prep.h> #include <optimizer/restrictinfo.h> #include <optimizer/tlist.h> #include <parser/parse_coerce.h> #include <parser/parsetree.h> #include <rewrite/rewriteManip.h> #include <utils/syscache.h> #include <utils/typcache.h> #include "compat/compat.h" #include "guc.h" #include "nodes/chunk_append/chunk_append.h" #include "nodes/columnar_scan/columnar_scan.h" #include "nodes/constraint_aware_append/constraint_aware_append.h" #include "nodes/skip_scan/skip_scan.h" #include <import/planner.h> #include <math.h> typedef struct SkipKeyInfo { /* Index clause which we'll use to skip past elements we've already seen */ RestrictInfo *skip_clause; /* Is this key guaranteed to be not null? */ bool notnull; /* attribute number of the distinct column on the table/chunk which provides comparison value * for Skip qual */ AttrNumber distinct_attno; /* attribute number of the Skip qual comparison column on the indexed table/chunk * "indexed_column_attno = distinct_attno" for (SkipScan <- Index Scan) scenario, * it can be different for (SkipScan <- ColumnarScan <- compressed Index Scan) scenario, * in that case "indexed_column_attno" is the attribute number of the compressed chunk column * corresponding to the distinct column "distinct_attno" on the decompressed chunk consumed by * SkipScan */ AttrNumber indexed_column_attno; /* The column offset on the index we are calling DISTINCT on */ AttrNumber scankey_attno; int distinct_typ_len; bool distinct_by_val; /* InvalidOid for the last skip key, always invalid for one-key SkipScan * For N-key SkipScan default quals are (sk1 = p1), (sk2 = p2), .. (sk_n > p_n), * we'll switch to (sk_i > p_i) when no more values for (sk_i+1 > p_i+1), * so we will store "=" along with ">" comparator for keys 1..N-1. */ Oid eqcomp; } SkipKeyInfo; typedef struct SkipScanPath { CustomPath cpath; IndexPath *index_path; /* List of skip column attributes for each skip key */ List *skipkeyinfo; /* Vars referencing the distinct columns on the relation */ List *dvars; } SkipScanPath; typedef struct DistinctPathInfo { UpperRelationKind stage; /* What kind of Upper distinct path we are dealing with */ Path *unique_path; /* If not NULL, valid Upper distinct path */ List * distinct_expr; /* If not NULL, list of valid distinct expressions for Upper distinct path */ } DistinctPathInfo; static int get_idx_key(IndexOptInfo *idxinfo, AttrNumber attno); static List *sort_indexquals(IndexOptInfo *indexinfo, List *quals); static OpExpr *fix_indexqual(IndexOptInfo *index, RestrictInfo *rinfo, AttrNumber scankey_attno); static bool build_skip_qual(PlannerInfo *root, SkipKeyInfo *skinfo, IndexPath *index_path, Var *var, bool build_eqop); static List *build_subpath(PlannerInfo *root, List *subpaths, DistinctPathInfo *dpinfo, List *top_pathkeys); static Var *get_distinct_var(PlannerInfo *root, Expr *tlexpr, IndexPath *index_path, Path *child_path, SkipKeyInfo *skinfo); static TargetEntry *tlist_member_match_var(Var *var, List *targetlist); /************************** * SkipScan Plan Creation * **************************/ static CustomScanMethods skip_scan_plan_methods = { .CustomName = "SkipScan", .CreateCustomScanState = tsl_skip_scan_state_create, }; void _skip_scan_init(void) { TryRegisterCustomScanMethods(&skip_scan_plan_methods); } static Plan * setup_index_plan(CustomScan *skip_plan, Plan *child_plan) { Plan *plan = child_plan; if (IsA(child_plan, IndexScan)) { skip_plan->scan = castNode(IndexScan, child_plan)->scan; } else if (IsA(child_plan, IndexOnlyScan)) { skip_plan->scan = castNode(IndexOnlyScan, child_plan)->scan; } else if (ts_is_columnar_scan_plan(child_plan)) { CustomScan *csplan = castNode(CustomScan, plan); skip_plan->scan = csplan->scan; plan = linitial(csplan->custom_plans); } else elog(ERROR, "unsupported subplan type for SkipScan: %s", ts_get_node_name((Node *) child_plan)); return plan; } static Plan * skip_scan_plan_create(PlannerInfo *root, RelOptInfo *relopt, CustomPath *best_path, List *tlist, List *clauses, List *custom_plans) { SkipScanPath *path = (SkipScanPath *) best_path; CustomScan *skip_plan = makeNode(CustomScan); IndexPath *index_path = path->index_path; Plan *child_plan = linitial(custom_plans); Plan *plan = setup_index_plan(skip_plan, child_plan); skip_plan->scan.plan.targetlist = tlist; skip_plan->custom_scan_tlist = list_copy(tlist); skip_plan->scan.plan.qual = NIL; skip_plan->scan.plan.type = T_CustomScan; skip_plan->methods = &skip_scan_plan_methods; skip_plan->custom_plans = custom_plans; /* Setup for SkipScan debug info */ StringInfoData debuginfo; RangeTblEntry *indexed_rte = NULL; char *sep = ""; if (ts_guc_debug_skip_scan_info) { initStringInfo(&debuginfo); RelOptInfo *indexed_rel = index_path->path.parent; indexed_rte = planner_rt_fetch(indexed_rel->relid, root); Oid indrelid = InvalidOid; if (IsA(plan, IndexScan)) { IndexScan *idx_plan = castNode(IndexScan, plan); indrelid = idx_plan->indexid; } else if (IsA(plan, IndexOnlyScan)) { IndexOnlyScan *idx_plan = castNode(IndexOnlyScan, plan); indrelid = idx_plan->indexid; } appendStringInfo(&debuginfo, "SkipScan used on %s(", get_rel_name(indrelid)); } ListCell *lc, *lv; /* List of N-1 equality op Oids for N-key skipscan, stays NIL for one-key skipscan */ List *eqcomps = NIL; /* List of N skipkeyinfo Int lists for N-key skipscan */ List *skinfos = NIL; forboth (lc, path->skipkeyinfo, lv, path->dvars) { SkipKeyInfo *skinfo = (SkipKeyInfo *) lfirst(lc); Var *dvar = castNode(Var, lfirst(lv)); OpExpr *op = fix_indexqual(index_path->indexinfo, skinfo->skip_clause, skinfo->scankey_attno); if (OidIsValid(skinfo->eqcomp)) eqcomps = lappend_oid(eqcomps, skinfo->eqcomp); if (IsA(plan, IndexScan)) { IndexScan *idx_plan = castNode(IndexScan, plan); /* we prepend skip qual here so sort_indexquals will put it as first qual for that * column */ idx_plan->indexqual = sort_indexquals(index_path->indexinfo, lcons(op, idx_plan->indexqual)); } else if (IsA(plan, IndexOnlyScan)) { IndexOnlyScan *idx_plan = castNode(IndexOnlyScan, plan); /* we prepend skip qual here so sort_indexquals will put it as first qual for that * column */ idx_plan->indexqual = sort_indexquals(index_path->indexinfo, lcons(op, idx_plan->indexqual)); } else elog(ERROR, "unsupported subplan type for SkipScan: %s", ts_get_node_name((Node *) plan)); /* get position of distinct column in tuples produced by child scan */ TargetEntry *tle = tlist_member_match_var(dvar, child_plan->targetlist); SkipKeyNullStatus sknulls; if (skinfo->notnull) sknulls = SK_NOT_NULL; else { bool nulls_first = index_path->indexinfo->nulls_first[skinfo->scankey_attno - 1]; if (index_path->indexscandir == BackwardScanDirection) nulls_first = !nulls_first; sknulls = (nulls_first ? SK_NULLS_FIRST : SK_NULLS_LAST); } skinfos = lappend(skinfos, list_make5_int(tle->resno, skinfo->distinct_by_val, skinfo->distinct_typ_len, sknulls, skinfo->scankey_attno)); /* Debug info about skip key */ if (ts_guc_debug_skip_scan_info) { char *attname = get_attname(indexed_rte->relid, skinfo->indexed_column_attno, false); char *sknullstext; switch (sknulls) { case SK_NOT_NULL: sknullstext = "NOT NULL"; break; case SK_NULLS_FIRST: sknullstext = "NULLS FIRST"; break; case SK_NULLS_LAST: sknullstext = "NULLS LAST"; break; default: Assert(false); } appendStringInfo(&debuginfo, "%s%s %s", sep, attname, sknullstext); sep = ", "; } } if (ts_guc_debug_skip_scan_info) { appendStringInfoString(&debuginfo, ")"); elog(INFO, "%s", debuginfo.data); } skip_plan->custom_private = lappend(skip_plan->custom_private, skinfos); /* Don't need equality ops for one-key skipscan */ if (eqcomps != NIL) { Assert(list_length(skinfos) > 1); skip_plan->custom_private = lappend(skip_plan->custom_private, eqcomps); } return &skip_plan->scan.plan; } /************************* * SkipScanPath Creation * *************************/ static CustomPathMethods skip_scan_path_methods = { .CustomName = "SkipScanPath", .PlanCustomPath = skip_scan_plan_create, }; #if PG16_GE typedef struct FindAggrefsContext { List *aggrefs; /* all non-nested Aggrefs found in a node */ } FindAggrefsContext; static bool find_aggrefs_walker(Node *node, FindAggrefsContext *context) { if (node == NULL) return false; if (IsA(node, Aggref)) { context->aggrefs = lappend(context->aggrefs, node); /* don't recurse inside Aggrefs */ return false; } return expression_tree_walker(node, find_aggrefs_walker, context); } #endif static Expr * get_distint_clause_expr(PlannerInfo *root, SortGroupClause *distinct_clause) { Node *expr = get_sortgroupclause_expr(distinct_clause, root->parse->targetList); /* we ignore any columns that can be constified to allow for cases like DISTINCT 'abc', * column */ if (IsA(estimate_expression_value(root, expr), Const)) return NULL; /* We ignore binary-compatible relabeling */ Expr *tlexpr = (Expr *) expr; while (tlexpr && IsA(tlexpr, RelabelType)) tlexpr = ((RelabelType *) tlexpr)->arg; if (!IsA(tlexpr, Var)) return NULL; return tlexpr; } /* We can get upper path Distinct expression once for upper path, * rather than repeat this check for each child path of an upper path input */ static List * get_upper_distinct_expr(PlannerInfo *root, UpperRelationKind stage) { ListCell *lc; Expr *tlexpr = NULL; List *result = NULL; if (stage == UPPERREL_DISTINCT && root->parse->distinctClause) { /* Obtain Distinct key from the target list, we ruled out numkeys > 1 cases before. * Examples of queries with 1 Distinct key but multiple target entries: * SELECT dev, dev FROM t; SELECT 1, dev FROM t; SELECT dev, time FROM t WHERE time = 100; */ SortGroupClause *distinct_clause = NULL; #if PG16_GE foreach (lc, root->processed_distinctClause) { distinct_clause = (SortGroupClause *) lfirst(lc); tlexpr = get_distint_clause_expr(root, distinct_clause); if (tlexpr) result = lappend(result, tlexpr); else return NULL; } #else if (root->distinct_pathkeys) { foreach (lc, root->distinct_pathkeys) { PathKey *pathkey = (PathKey *) lfirst(lc); if (pathkey->pk_eclass->ec_sortref) { foreach (lc, root->parse->distinctClause) { SortGroupClause *clause = lfirst_node(SortGroupClause, lc); if (clause->tleSortGroupRef == pathkey->pk_eclass->ec_sortref) { distinct_clause = clause; break; } } if (!distinct_clause) return NULL; } /* We can get PathKey with ec_sortref = 0 in PG15 * when False filter is not pushed into a relation with distinct column (i.e. it's * on top of a join), so need to support this case in PG15 */ else return NULL; tlexpr = get_distint_clause_expr(root, distinct_clause); if (tlexpr) result = lappend(result, tlexpr); else return NULL; } } /* In PG16+ we use LIMIT instead of UpperUniquePath for (numkeys = 0), * but in PG15- we would still create UpperUniquePath for (numkeys = 0), so handle this case * here */ else { foreach (lc, root->parse->distinctClause) { distinct_clause = lfirst_node(SortGroupClause, lc); tlexpr = get_distint_clause_expr(root, distinct_clause); if (tlexpr) result = lappend(result, tlexpr); else return NULL; } } #endif } #if PG16_GE else if (stage == UPPERREL_GROUP_AGG) { /* Find all non-nested Aggrefs in the query target list */ FindAggrefsContext agg_ctx = { .aggrefs = NULL }; find_aggrefs_walker((Node *) root->parse->targetList, &agg_ctx); foreach (lc, agg_ctx.aggrefs) { Aggref *agg = lfirst_node(Aggref, lc); /* Only distinct aggs with 1 sorted argument are eligible*/ if (agg->aggdistinct && agg->aggpresorted && list_length(agg->args) == 1) { TargetEntry *tle = (TargetEntry *) linitial(agg->args); Expr *expr = tle->expr; /* We ignore binary-compatible relabeling */ while (expr && IsA(expr, RelabelType)) expr = ((RelabelType *) expr)->arg; /* Distinct agg over a Const is OK */ if (IsA(estimate_expression_value(root, (Node *) expr), Const)) continue; /* Don't support no-var arguments */ if (!IsA(expr, Var)) return NULL; /* Don't support multiple distinct aggs over different columns */ if (tlexpr && !tlist_member_match_var((Var *) tlexpr, agg->args)) return NULL; /* If Distinct agg path has a groupby column, it needs to match Distinct agg column */ if (root->processed_groupClause) { /* Should have bailed out on gby exprs > 1 earlier * Only 1-key SkipScan is supported for distinct aggregates */ Assert(list_length(root->processed_groupClause) == 1); SortGroupClause *sortcl = (SortGroupClause *) linitial(root->processed_groupClause); Expr *gbykey = (Expr *) get_sortgroupclause_expr(sortcl, root->processed_tlist); if (!equal(gbykey, expr)) return NULL; } /* Found a valid distinct agg over a valid Var */ if (!tlexpr) { tlexpr = expr; result = lappend(result, tlexpr); } } else { return NULL; } } } #endif return result; } static void obtain_upper_distinct_path(PlannerInfo *root, RelOptInfo *output_rel, DistinctPathInfo *dpinfo) { ListCell *lc; /* * look for Unique Path so we dont have to repeat some of * the calculations done by postgres and can also assume * that the DISTINCT clause is eligible for sort based * DISTINCT */ if (dpinfo->stage == UPPERREL_DISTINCT) { if (!ts_guc_enable_skip_scan) return; foreach (lc, output_rel->pathlist) { if (IsA(lfirst(lc), UpperUniquePath)) { UpperUniquePath *unique = (UpperUniquePath *) lfirst_node(UpperUniquePath, lc); /* We can handle DISTINCT on more than one key if all keys are guaranteed not-nulls. * To do so, we break down the SkipScan into subproblems: first * find the minimal tuple then for each prefix find all unique suffix * tuples. For instance, if we are searching over (int, int), we would * first find (0, 0) then find (0, N) for all N in the domain, then * find (1, N), then (2, N), etc */ if (!ts_guc_enable_multikey_skip_scan && unique->numkeys > 1) return; #if PG16_GE /* since PG16+ we no longer create UpperUniquePath with 0 numkeys, * we create LIMIT path instead, so shouldn't be here with 0 numkeys */ Assert(unique->numkeys >= 1); #endif dpinfo->unique_path = (Path *) unique; break; } } } /* Sorted inputs for Distinct aggs weren't supported until PG16 */ #if PG16_GE /* Look for Aggpath with eligible Distinct aggregates */ else if (dpinfo->stage == UPPERREL_GROUP_AGG) { if (!ts_guc_enable_skip_scan_for_distinct_aggregates) return; /* Cannot apply SkipScan to distinct aggregates with more than one key */ if (list_length(root->group_pathkeys) > 1) return; foreach (lc, output_rel->pathlist) { if (IsA(lfirst(lc), AggPath)) { AggPath *unique = (AggPath *) lfirst_node(AggPath, lc); /* If Distinct agg path has a group key, it must match Distinct aggregate input sort * key, otherwise cannot apply SkipScan */ if (unique->path.pathkeys && !pathkeys_contained_in(unique->path.pathkeys, unique->subpath->pathkeys)) { return; } dpinfo->unique_path = (Path *) lfirst_node(AggPath, lc); break; } } } #endif else return; if (!dpinfo->unique_path) return; /* Check if we have valid distinct expression to source from the underlying index */ dpinfo->distinct_expr = get_upper_distinct_expr(root, dpinfo->stage); if (!dpinfo->distinct_expr) { dpinfo->unique_path = NULL; return; } /* Need to make a copy of the unique path here because add_path() in the * pathlist loop below might prune it if the new unique path * (SkipScanPath) dominates the old one. When the unique path is pruned, * the pointer will no longer be valid in the next iteration of the * pathlist loop. Fortunately, the Path object is not deeply freed, so a * shallow copy is enough. */ if (dpinfo->stage == UPPERREL_DISTINCT) { UpperUniquePath *unique = makeNode(UpperUniquePath); memcpy(unique, lfirst_node(UpperUniquePath, lc), sizeof(UpperUniquePath)); dpinfo->unique_path = (Path *) unique; } else if (dpinfo->stage == UPPERREL_GROUP_AGG) { AggPath *dist_agg_path = makeNode(AggPath); memcpy(dist_agg_path, lfirst_node(AggPath, lc), sizeof(AggPath)); dpinfo->unique_path = (Path *) dist_agg_path; } } static SkipScanPath *skip_scan_path_create(PlannerInfo *root, Path *child_path, DistinctPathInfo *dpinfo); /* * Create SkipScan paths based on existing Unique paths. * For a Unique path on a simple relation like the following * * Unique * -> Index Scan using skip_scan_dev_name_idx on skip_scan * * a SkipScan path like this will be created: * * Unique * -> Custom Scan (SkipScan) on skip_scan * -> Index Scan using skip_scan_dev_name_idx on skip_scan * * For a Unique path on a hypertable with multiple chunks like the following * * Unique * -> Merge Append * Sort Key: _hyper_2_1_chunk.dev_name * -> Index Scan using _hyper_2_1_chunk_idx on _hyper_2_1_chunk * -> Index Scan using _hyper_2_2_chunk_idx on _hyper_2_2_chunk * * a SkipScan path like this will be created: * * Unique * -> Merge Append * Sort Key: _hyper_2_1_chunk.dev_name * -> Custom Scan (SkipScan) on _hyper_2_1_chunk * -> Index Scan using _hyper_2_1_chunk_idx on _hyper_2_1_chunk * -> Custom Scan (SkipScan) on _hyper_2_2_chunk * -> Index Scan using _hyper_2_2_chunk_idx on _hyper_2_2_chunk */ void tsl_skip_scan_paths_add(PlannerInfo *root, RelOptInfo *input_rel, RelOptInfo *output_rel, UpperRelationKind stage) { DistinctPathInfo dpinfo = { .stage = stage, .unique_path = NULL, .distinct_expr = NULL, }; obtain_upper_distinct_path(root, output_rel, &dpinfo); if (!dpinfo.unique_path) return; Assert(IsA(dpinfo.unique_path, UpperUniquePath) || IsA(dpinfo.unique_path, AggPath)); ListCell *lc; foreach (lc, input_rel->pathlist) { bool has_caa = false; Path *subpath = lfirst(lc); List *top_pathkeys = NULL; /* Unique path has to be sorted on at least DISTINCT ON key */ if (IsA(dpinfo.unique_path, UpperUniquePath)) { if (!pathkeys_contained_in(dpinfo.unique_path->pathkeys, subpath->pathkeys)) continue; } /* AggPath with distinct aggs may not be sorted, but the input into distinct aggs needs to * be sorted */ else if (IsA(dpinfo.unique_path, AggPath)) { if (!subpath->pathkeys || !pathkeys_contained_in(dpinfo.unique_path->pathkeys, subpath->pathkeys)) continue; /* Need to check sortedness for inputs of Distinct aggs, so we'll keep track of the * input pathkeys */ top_pathkeys = subpath->pathkeys; } /* If path is a ProjectionPath we strip it off for processing * but also add a ProjectionPath on top of the SKipScanPaths * later. */ ProjectionPath *proj = NULL; if (IsA(subpath, ProjectionPath)) { proj = castNode(ProjectionPath, subpath); subpath = proj->subpath; } /* Path might be wrapped in a ConstraintAwareAppendPath if this * is a MergeAppend that could benefit from runtime exclusion. * We treat this similar to ProjectionPath and add it back * later */ if (ts_is_constraint_aware_append_path(subpath)) { subpath = linitial(castNode(CustomPath, subpath)->custom_paths); Assert(IsA(subpath, MergeAppendPath)); has_caa = true; } if (IsA(subpath, IndexPath) || ts_is_columnar_scan_path(subpath)) { subpath = (Path *) skip_scan_path_create(root, subpath, &dpinfo); if (!subpath) continue; } else if (IsA(subpath, MergeAppendPath)) { MergeAppendPath *merge_path = castNode(MergeAppendPath, subpath); List *new_paths = build_subpath(root, merge_path->subpaths, &dpinfo, top_pathkeys); /* build_subpath returns NULL when no SkipScanPath was created */ if (!new_paths) continue; subpath = (Path *) create_merge_append_path(root, merge_path->path.parent, new_paths, merge_path->path.pathkeys, NULL); subpath->pathtarget = copy_pathtarget(merge_path->path.pathtarget); } /* We may have Append over one input which will be removed from the plan later. * Consider it when it is sorted correctly. #7778 */ else if (IsA(subpath, AppendPath)) { AppendPath *append_path = castNode(AppendPath, subpath); if (list_length(append_path->subpaths) > 1) continue; List *new_paths = build_subpath(root, append_path->subpaths, &dpinfo, top_pathkeys); /* build_subpath returns NULL when no SkipScanPath was created */ if (!new_paths) continue; subpath = (Path *) create_append_path(root, append_path->path.parent, new_paths, NULL, append_path->path.pathkeys, NULL, append_path->path.parallel_workers, append_path->path.parallel_aware, -1); subpath->pathtarget = copy_pathtarget(append_path->path.pathtarget); } else if (ts_is_chunk_append_path(subpath)) { ChunkAppendPath *ca = (ChunkAppendPath *) subpath; List *new_paths = build_subpath(root, ca->cpath.custom_paths, &dpinfo, top_pathkeys); /* ChunkAppend should never be wrapped in ConstraintAwareAppendPath */ Assert(!has_caa); /* build_subpath returns NULL when no SkipScanPath was created */ if (!new_paths) continue; /* We copy the existing ChunkAppendPath here because we don't have all the * information used for creating the original one and we don't want to * duplicate all the checks done when creating the original one. */ subpath = (Path *) ts_chunk_append_path_copy(ca, new_paths, ca->cpath.path.pathtarget); } else { continue; } /* add ConstraintAwareAppendPath if the original path had one */ if (has_caa) subpath = ts_constraint_aware_append_path_create(root, subpath); Path *new_unique = NULL; if (IsA(dpinfo.unique_path, UpperUniquePath)) { UpperUniquePath *unique = (UpperUniquePath *) dpinfo.unique_path; new_unique = (Path *) create_upper_unique_path(root, output_rel, subpath, unique->numkeys, unique->path.rows); new_unique->pathtarget = unique->path.pathtarget; if (proj) new_unique = (Path *) create_projection_path(root, output_rel, new_unique, copy_pathtarget(new_unique->pathtarget)); } else if (IsA(dpinfo.unique_path, AggPath)) { AggPath *dist_agg_path = (AggPath *) dpinfo.unique_path; if (proj) { proj->subpath = subpath; subpath = (Path *) proj; } AggClauseCosts agg_costs; MemSet(&agg_costs, 0, sizeof(AggClauseCosts)); get_agg_clause_costs(root, dist_agg_path->aggsplit, &agg_costs); new_unique = (Path *) create_agg_path(root, output_rel, subpath, dist_agg_path->path.pathtarget, dist_agg_path->aggstrategy, dist_agg_path->aggsplit, dist_agg_path->groupClause, dist_agg_path->qual, (const AggClauseCosts *) &agg_costs, dist_agg_path->numGroups); } add_path(output_rel, new_unique); } } #if PG17_LT static bool attr_is_notnull(Oid relid, AttrNumber attno) { HeapTuple tp = SearchSysCache2(ATTNUM, ObjectIdGetDatum(relid), Int16GetDatum(attno)); if (!HeapTupleIsValid(tp)) return false; Form_pg_attribute att_tup = (Form_pg_attribute) GETSTRUCT(tp); bool result = att_tup->attnotnull; ReleaseSysCache(tp); return result; } #endif /* Check if skip key is guaranteed not-null */ static void check_notnull_skipkey(SkipKeyInfo *skinfo, Path *child_path, IndexPath *index_path) { ListCell *l; /* Quickly look through index clauses on this skip key */ foreach (l, index_path->indexclauses) { IndexClause *ic = (IndexClause *) lfirst(l); /* index quals are ordered by indexcol, nothing to see if we've passed our indexcol */ if (ic->indexcol > skinfo->scankey_attno - 1) break; /* We may have row comparison with skip key not being a leading col, * like (col, skipcol) > (3, 5), but it can allow NULL skipcols to pass if (col>3) is true, * so for row comparisons we will only look at leading "indexcol" and not at "indexcols". */ if (ic->indexcol == skinfo->scankey_attno - 1) { /* Any simple index qual but "isNull" filters out nulls, * including "lossy" index quals extracted from index clauses. */ ListCell *lc; foreach (lc, ic->indexquals) { RestrictInfo *iqual = (RestrictInfo *) lfirst(lc); if (!(IsA(iqual->clause, NullTest) && ((NullTest *) iqual->clause)->nulltesttype == IS_NULL)) { skinfo->notnull = true; return; } } } } /* Otherwise look at all non-indexqual index filters on the key (like (key+1)>5) to see if they * filter out NULLs */ RelOptInfo *indexed_rel = index_path->path.parent; foreach (l, index_path->indexinfo->indrestrictinfo) { RestrictInfo *ri = castNode(RestrictInfo, lfirst(l)); Bitmapset *clause_attnos = NULL; pull_varattnos((Node *) ri->clause, indexed_rel->relid, &clause_attnos); if (bms_is_member(skinfo->indexed_column_attno - FirstLowInvalidHeapAttributeNumber, clause_attnos)) { if (!contain_nonstrict_functions((Node *) ri->clause)) { skinfo->notnull = true; return; } } } /* Failing that, look at filters not pushed down into index (like col1+col2>1) to see if they * filter out NULLs */ RelOptInfo *child_rel = child_path->parent; foreach (l, child_rel->baserestrictinfo) { RestrictInfo *ri = castNode(RestrictInfo, lfirst(l)); Bitmapset *clause_attnos = NULL; pull_varattnos((Node *) ri->clause, child_rel->relid, &clause_attnos); if (bms_is_member(skinfo->distinct_attno - FirstLowInvalidHeapAttributeNumber, clause_attnos)) { if (!contain_nonstrict_functions((Node *) ri->clause)) { skinfo->notnull = true; return; } } } } static IndexPath * get_compressed_index_path(ColumnarScanPath *dcpath) { Path *compressed_path = linitial(dcpath->custom_path.custom_paths); if (IsA(compressed_path, IndexPath)) { IndexPath *index_path = castNode(IndexPath, compressed_path); if (!pathkeys_contained_in(dcpath->required_compressed_pathkeys, compressed_path->pathkeys)) return NULL; return index_path; } return NULL; } static SkipScanPath * skip_scan_path_create(PlannerInfo *root, Path *child_path, DistinctPathInfo *dpinfo) { IndexPath *index_path = NULL; if (IsA(child_path, IndexPath)) { index_path = castNode(IndexPath, child_path); } else if (ts_is_columnar_scan_path(child_path)) { if (!ts_guc_enable_compressed_skip_scan) return NULL; ColumnarScanPath *dcpath = (ColumnarScanPath *) child_path; index_path = get_compressed_index_path(dcpath); } if (!index_path) return NULL; /* cannot use SkipScan with non-orderable index or IndexPath without pathkeys */ if (!index_path->path.pathkeys || !index_path->indexinfo->sortopfamily) return NULL; /* orderbyops are not compatible with skipscan */ if (index_path->indexorderbys != NIL) return NULL; SkipScanPath *skip_scan_path = (SkipScanPath *) newNode(sizeof(SkipScanPath), T_CustomPath); skip_scan_path->cpath.path.pathtype = T_CustomScan; skip_scan_path->cpath.path.pathkeys = child_path->pathkeys; skip_scan_path->cpath.path.pathtarget = child_path->pathtarget; skip_scan_path->cpath.path.param_info = child_path->param_info; skip_scan_path->cpath.path.parent = child_path->parent; skip_scan_path->cpath.custom_paths = list_make1(child_path); skip_scan_path->cpath.methods = &skip_scan_path_methods; /* While add_path may pfree paths with higher costs * it will never free IndexPaths and only ever do a shallow * free so reusing the IndexPath here is safe. */ skip_scan_path->index_path = index_path; ListCell *lc; int sk_no = 0; int num_skipkeys = list_length(dpinfo->distinct_expr); foreach (lc, dpinfo->distinct_expr) { Expr *dexpr = (Expr *) lfirst(lc); /* Placeholder for skip key attributes */ SkipKeyInfo *skinfo = palloc(sizeof(SkipKeyInfo)); Var *dvar = get_distinct_var(root, dexpr, index_path, child_path, skinfo); if (!dvar) { pfree(skinfo); return NULL; } /* build skip qual this may fail if we cannot look up the operator */ if (!build_skip_qual(root, skinfo, index_path, dvar, (++sk_no) < num_skipkeys)) { pfree(skinfo); return NULL; } if (!skinfo->notnull) check_notnull_skipkey(skinfo, child_path, index_path); /* Multikey SkipScan is only supported in not-null mode */ if (!skinfo->notnull && num_skipkeys > 1) return NULL; skip_scan_path->dvars = lappend(skip_scan_path->dvars, dvar); skip_scan_path->skipkeyinfo = lappend(skip_scan_path->skipkeyinfo, skinfo); } /* We have valid SkipScanPath: now we can cost it */ double startup = child_path->startup_cost; double total = child_path->total_cost; double rows = child_path->rows; double indexscan_rows = index_path->path.rows; /* Also true for SkipScan over compressed chunks as can't have more distinct segmentby values * than number of batches */ int ndistinct = indexscan_rows; /* For SELECT DISTINCT path, #rows can cap "ndistinct", * but for Distinct aggregates #rows = 1 usually, i.e. we can't cap "ndistinct" in this case. */ if (dpinfo->stage == UPPERREL_DISTINCT) ndistinct = Min(ndistinct, dpinfo->unique_path->rows); /* If we are on a chunk rather than on a PG table, we want to get "ndistinct" for this chunk, * as Unique path rows may combine rows from each chunk and may not represent a true * "ndistinct". Consider a hypertable with 1000 chunks, each chunk has the same 1 distinct * value, Unique path will add them up and we will get "ndistinct" = 1000 instead of 1. If * Unique path has "ndistinct=1" we can't go any smaller so will just accept this number. */ if (ndistinct > 1) { ndistinct = Max(1, floor(estimate_num_groups(root, skip_scan_path->dvars, ndistinct, NULL, NULL))); } skip_scan_path->cpath.path.rows = ndistinct; /* Addressing #8107: filters on the indexed data which are not index quals * will require sequential scanning of indexed tuples until finding a tuple passing the filter. * For some highly selective filters it may mean scanning a lot of tuples, sometimes the entire * input. SeqScan may perform better than IndexScan for such filters. We need to account for * such filters in the cost model i.e. cost the number of tuples to scan before passing the * filter. */ List *clauses_needing_scan = NULL; /* If a filter is not pushed down into compessed indexed data, it's a filter for which we will * need to scan and decompress until filter is passed */ if ((Path *) index_path != child_path) clauses_needing_scan = child_path->parent->baserestrictinfo; else { /* For uncompressed index data we need a finer check for which filters on the indexed data * are index quals or not */ ListCell *lc; foreach (lc, index_path->indexinfo->indrestrictinfo) { RestrictInfo *ri = (RestrictInfo *) lfirst(lc); bool match_found = false; ListCell *l; foreach (l, index_path->indexclauses) { IndexClause *ic = (IndexClause *) lfirst(l); if (ri == ic->rinfo) { match_found = true; break; } } /* This is a filter which is not an index qual: will have to scan indexed data until * this filter is passed */ if (!match_found) clauses_needing_scan = lappend(clauses_needing_scan, ri); } } /* Heuristic for accounting for previous key shifts in multikey SkipScan * In general, we will have (k_1*k_2*..*k_N + k_1*..*k_N-1 + ... + k_1) shifts * where numdistinct = k_1*k_2*..*k_N and for each key "k_i" we have "k_i+1" distinct values. * * Worst case scenario is when k_1 = numdistinct and the rest =1, then we'll have (numdistinct * * N) shifts. If it's uniform for each key i.e. numdistinct = k^N we'll have less than * (numdistinct * 2) shifts. We pick something in the middle to avoid evaluating k_i for each * key. */ int numkeys_multiplier = 1.0 + (num_skipkeys - 1) * 0.5; /* We calculate SkipScan cost as ndistinct * startup_cost + (ndistinct/rows) * total_cost * ndistinct * startup_cost is to account for the rescans we have to do and since startup * cost for indexes does not include page access cost we add a fraction of the total cost * accounting for the number of rows we expect to fetch. * If the row estimate for the scan is 1 we assume that the estimate got clamped to 1 * and no rows would be returned by this scan and this chunk will most likely be excluded * by runtime exclusion. Otherwise the cost for this path would be highly inflated due * to (ndistinct / rows) * total leading to SkipScan not being chosen for queries on * hypertables with a lot of excluded chunks. * * This is the cost of (SkipScan <- IndexScan) scenario */ if (clauses_needing_scan == NULL && (Path *) index_path == child_path) { skip_scan_path->cpath.path.startup_cost = startup; if (indexscan_rows > 1) skip_scan_path->cpath.path.total_cost = ndistinct * startup * numkeys_multiplier + (ndistinct / rows) * total; else skip_scan_path->cpath.path.total_cost = startup; } /* For (SkipScan <- ColumnarScan <- compressed IndexScan) scenario * we will estimate cost as (ndistinct * costs( child_path LIMIT 1 OFFSET x)) * i.e. as if we computed "ndistinct" LIMIT 1 queries on the "child_path" after initial setup. * If there is no qual above IndexScan, then OFFSET=0 (we don't need to scan tuples to pass qual * before returning the 1st tuple), otherwise OFFSET = (1 / qual_selectivity - 1), i.e. we have * to skip OFFSET tuples until we get the one which passes the qual. */ else { int64 offset_until_qual_pass = 0; if (clauses_needing_scan != NULL) { /* Avoid division by qual_selectivity = 0.0 */ Selectivity qual_selectivity = Max(1.0 / (rows + 1), clauselist_selectivity(root, clauses_needing_scan, 0, JOIN_INNER, NULL)); offset_until_qual_pass = Max(0, floor(1 / qual_selectivity)); } adjust_limit_rows_costs(&rows, &startup, &total, offset_until_qual_pass, 1); skip_scan_path->cpath.path.startup_cost = startup; if (indexscan_rows > 1) skip_scan_path->cpath.path.total_cost = startup + (total - startup) * ndistinct * numkeys_multiplier; else skip_scan_path->cpath.path.total_cost = startup; } /* Finally, adjust SkipScan run costs with GUC multiplier (1.0 by default), to give users more * control over choosing SkipScan */ skip_scan_path->cpath.path.total_cost = startup + (skip_scan_path->cpath.path.total_cost - startup) * ts_guc_skip_scan_run_cost_multiplier; return skip_scan_path; } /* Extract the Var to use for the SkipScan and do attno mapping if required. */ static Var * get_distinct_var(PlannerInfo *root, Expr *tlexpr, IndexPath *index_path, Path *child_path, SkipKeyInfo *skinfo) { RelOptInfo *rel = child_path->parent; RelOptInfo *indexed_rel = index_path->path.parent; Assert(tlexpr && IsA(tlexpr, Var)); Var *var = castNode(Var, tlexpr); RangeTblEntry *ht_rte = planner_rt_fetch(var->varno, root); /* check whether a skip var is declared NOT NULL * it's enough to check hypertable for NOT NULL * as NOT NULL constraint will be propagated to and checked on all chunks */ #if PG17_LT skinfo->notnull = attr_is_notnull(ht_rte->relid, var->varattno); #else RelOptInfo *baserel = ((Index) var->varno == rel->relid ? rel : rel->parent); skinfo->notnull = bms_is_member(var->varattno, baserel->notnullattnums); #endif /* If we are dealing with a hypertable Var extracted from distinctClause will point to * the parent hypertable while the IndexPath will be on a Chunk. * For a normal PG table they point to the same relation and we are done here. */ if ((Index) var->varno == rel->relid) { /* Get attribute number for distinct column on a normal PG table */ skinfo->indexed_column_attno = var->varattno; return var; } RangeTblEntry *chunk_rte = planner_rt_fetch(rel->relid, root); RangeTblEntry *indexed_rte = (indexed_rel == rel ? chunk_rte : planner_rt_fetch(indexed_rel->relid, root)); /* Check for hypertable */ if (!ts_is_hypertable(ht_rte->relid) || !bms_is_member(var->varno, rel->top_parent_relids)) return NULL; char *attname = get_attname(ht_rte->relid, var->varattno, false); var = copyObject(var); var->varattno = get_attnum(chunk_rte->relid, attname); /* Get attribute number for distinct column on a compressed chunk */ if (ts_is_columnar_scan_path(child_path)) { /* distinct column has to be a segmentby column */ ColumnarScanPath *dcpath = (ColumnarScanPath *) child_path; if (!bms_is_member(var->varattno, dcpath->info->chunk_segmentby_attnos)) { return NULL; } skinfo->indexed_column_attno = get_attnum(indexed_rte->relid, attname); } /* Get attribute number for distinct column on an uncompressed chunk */ else { skinfo->indexed_column_attno = var->varattno; } var->varno = rel->relid; return var; } /* * Creates SkipScanPath for each path of subpaths that is an IndexPath * If no subpath can be changed to SkipScanPath returns NULL * otherwise returns list of new paths */ static List * build_subpath(PlannerInfo *root, List *subpaths, DistinctPathInfo *dpinfo, List *top_pathkeys) { bool has_skip_path = false; List *new_paths = NIL; ListCell *lc; foreach (lc, subpaths) { Path *child = lfirst(lc); if (IsA(child, IndexPath) || ts_is_columnar_scan_path(child)) { if (top_pathkeys && !pathkeys_contained_in(top_pathkeys, child->pathkeys)) continue; SkipScanPath *skip_path = skip_scan_path_create(root, child, dpinfo); if (skip_path) { child = (Path *) skip_path; has_skip_path = true; } } new_paths = lappend(new_paths, child); } if (!has_skip_path && new_paths) { pfree(new_paths); return NIL; } return new_paths; } static bool build_skip_qual(PlannerInfo *root, SkipKeyInfo *skinfo, IndexPath *index_path, Var *var, bool build_eqop) { IndexOptInfo *info = index_path->indexinfo; Oid column_type = exprType((Node *) var); Oid column_collation = get_typcollation(column_type); TypeCacheEntry *tce = lookup_type_cache(column_type, 0); bool need_coerce = false; /* * Skipscan is not applicable for the following case: * We might have a path with an index that produces the correct pathkeys for the target ordering * without actually including all the columns of the ORDER BY. If the path uses an index that * does not include the distinct column, we cannot use it for skipscan and have to discard this * path from skipscan generation. This happens, for instance, when we have an order by clause * (like ORDER BY a, b) with constraints in the WHERE clause (like WHERE a = <constant>) . "a" * can now be removed from the Pathkeys (since it is a constant) and the query can be satisfied * by using an index on just column "b". * * Example query: * SELECT DISTINCT ON (a) * FROM test WHERE a in (2) ORDER BY a ASC, time DESC; * Since a is always 2 due to the WHERE clause we can create the correct ordering for the * ORDER BY with an index that does not include the a column and only includes the time column. */ int idx_key = get_idx_key(index_path->indexinfo, skinfo->indexed_column_attno); if (idx_key < 0) return false; /* sk_attno of the skip qual */ skinfo->scankey_attno = idx_key + 1; skinfo->distinct_attno = var->varattno; skinfo->distinct_by_val = tce->typbyval; skinfo->distinct_typ_len = tce->typlen; int16 strategy = info->reverse_sort[idx_key] ? BTLessStrategyNumber : BTGreaterStrategyNumber; if (index_path->indexscandir == BackwardScanDirection) { strategy = (strategy == BTLessStrategyNumber) ? BTGreaterStrategyNumber : BTLessStrategyNumber; } Oid opcintype = info->opcintype[idx_key]; Oid comparator = get_opfamily_member(info->sortopfamily[idx_key], column_type, column_type, strategy); Oid eqop = InvalidOid; if (build_eqop) eqop = get_opfamily_member(info->sortopfamily[idx_key], column_type, column_type, BTEqualStrategyNumber); /* If there is no exact operator match for the column type we have here check * if we can coerce to the type of the operator class. */ if (!OidIsValid(comparator)) { if (IsBinaryCoercible(column_type, opcintype)) { comparator = get_opfamily_member(info->sortopfamily[idx_key], opcintype, opcintype, strategy); if (!OidIsValid(comparator)) return false; if (build_eqop) eqop = get_opfamily_member(info->sortopfamily[idx_key], opcintype, opcintype, BTEqualStrategyNumber); need_coerce = true; } else return false; /* cannot use this index */ } Const *prev_val = makeNullConst(need_coerce ? opcintype : column_type, -1, column_collation); Expr *current_val = (Expr *) makeVar(info->rel->relid /*varno*/, skinfo->indexed_column_attno /*varattno*/, column_type /*vartype*/, -1 /*vartypmod*/, column_collation /*varcollid*/, 0 /*varlevelsup*/); if (need_coerce) { CoerceViaIO *coerce = makeNode(CoerceViaIO); coerce->arg = current_val; coerce->resulttype = opcintype; coerce->resultcollid = column_collation; coerce->coerceformat = COERCE_IMPLICIT_CAST; coerce->location = -1; current_val = &coerce->xpr; } Expr *comparison_expr = make_opclause(comparator, BOOLOID /*opresulttype*/, false /*opretset*/, current_val /*leftop*/, &prev_val->xpr /*rightop*/, InvalidOid /*opcollid*/, info->indexcollations[idx_key] /*inputcollid*/); set_opfuncid(castNode(OpExpr, comparison_expr)); skinfo->eqcomp = (build_eqop ? get_opcode(eqop) : InvalidOid); skinfo->skip_clause = make_simple_restrictinfo(root, comparison_expr); return true; } static int get_idx_key(IndexOptInfo *idxinfo, AttrNumber attno) { for (int i = 0; i < idxinfo->nkeycolumns; i++) { if (attno == idxinfo->indexkeys[i]) return i; } return -1; } /* Sort quals according to index column order. * ScanKeys need to be sorted by the position of the index column * they are referencing but since we don't want to adjust actual * ScanKey array we presort qual list when creating plan. */ static List * sort_indexquals(IndexOptInfo *indexinfo, List *quals) { List *indexclauses[INDEX_MAX_KEYS] = { 0 }; List *ordered_list = NIL; ListCell *lc; int i; foreach (lc, quals) { Bitmapset *bms = NULL; pull_varattnos(lfirst(lc), INDEX_VAR, &bms); Assert(bms_num_members(bms) >= 1); i = bms_next_member(bms, -1) + FirstLowInvalidHeapAttributeNumber - 1; indexclauses[i] = lappend(indexclauses[i], lfirst(lc)); } for (i = 0; i < indexinfo->nkeycolumns; i++) { if (indexclauses[i] != NIL) ordered_list = list_concat(ordered_list, indexclauses[i]); } return ordered_list; } static OpExpr * fix_indexqual(IndexOptInfo *index, RestrictInfo *rinfo, AttrNumber scankey_attno) { /* technically our placeholder col > NULL is unsatisfiable, and in some instances * the planner will realize this and use is as an excuse to remove other quals. * in order to prevent this, we prepare this qual ourselves. */ /* fix_indexqual_references */ OpExpr *op = copyObject(castNode(OpExpr, rinfo->clause)); Assert(list_length(op->args) == 2); Assert(bms_equal(rinfo->left_relids, index->rel->relids)); /* fix_indexqual_operand */ Assert(index->indexkeys[scankey_attno - 1] != 0); Var *node = linitial_node(Var, pull_var_clause(linitial(op->args), 0)); Assert((Index) ((Var *) node)->varno == index->rel->relid && ((Var *) node)->varattno == index->indexkeys[scankey_attno - 1]); Var *result = (Var *) copyObject(node); result->varno = INDEX_VAR; result->varattno = scankey_attno; linitial(op->args) = result; return op; } /* * tlist_member_match_var * Same as tlist_member, except that we match the provided Var on the basis * of varno/varattno/varlevelsup/vartype only, rather than full equal(). * * This is needed in some cases where we can't be sure of an exact typmod * match. For safety, though, we insist on vartype match. * * static function copied from src/backend/optimizer/util/tlist.c */ static TargetEntry * tlist_member_match_var(Var *var, List *targetlist) { ListCell *temp; foreach (temp, targetlist) { TargetEntry *tlentry = (TargetEntry *) lfirst(temp); Var *tlvar = (Var *) tlentry->expr; if (!tlvar || !IsA(tlvar, Var)) continue; if (var->varno == tlvar->varno && var->varattno == tlvar->varattno && var->varlevelsup == tlvar->varlevelsup && var->vartype == tlvar->vartype) return tlentry; } return NULL; } ================================================ FILE: tsl/src/nodes/skip_scan/skip_scan.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/plannodes.h> typedef enum SkipKeyNullStatus { SK_NOT_NULL = 0, SK_NULLS_FIRST, SK_NULLS_LAST } SkipKeyNullStatus; typedef enum { SK_DistinctColAttno = 0, SK_DistinctByVal = 1, SK_DistinctTypeLen = 2, SK_NullStatus = 3, SK_IndexKeyAttno = 4 } SkipScanPrivateIndex; extern void tsl_skip_scan_paths_add(PlannerInfo *root, RelOptInfo *input_rel, RelOptInfo *output_rel, UpperRelationKind stage); extern Node *tsl_skip_scan_state_create(CustomScan *cscan); extern void _skip_scan_init(void); ================================================ FILE: tsl/src/nodes/vector_agg/CMakeLists.txt ================================================ add_subdirectory(function) add_subdirectory(hashing) set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/exec.c ${CMAKE_CURRENT_SOURCE_DIR}/grouping_policy_batch.c ${CMAKE_CURRENT_SOURCE_DIR}/grouping_policy_hash.c ${CMAKE_CURRENT_SOURCE_DIR}/plan.c ${CMAKE_CURRENT_SOURCE_DIR}/plan_columnar_scan.c) target_sources(${TSL_LIBRARY_NAME} PRIVATE ${SOURCES}) ================================================ FILE: tsl/src/nodes/vector_agg/exec.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <commands/explain.h> #include <executor/executor.h> #include <executor/tuptable.h> #include <fmgr.h> #include <funcapi.h> #include <nodes/extensible.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <nodes/pg_list.h> #include <optimizer/optimizer.h> #include "nodes/vector_agg/exec.h" #include "compat/compat.h" #include "compression/arrow_c_data_interface.h" #include "nodes/columnar_scan/columnar_scan.h" #include "nodes/columnar_scan/compressed_batch.h" #include "nodes/columnar_scan/exec.h" #include "nodes/columnar_scan/vector_quals.h" #include "nodes/vector_agg.h" #include "nodes/vector_agg/filter_word_iterator.h" #include "nodes/vector_agg/plan.h" #include "nodes/vector_agg/vector_slot.h" #if PG18_GE #include "commands/explain_format.h" #include "commands/explain_state.h" #endif static CompressedBatchVectorQualState compressed_batch_init_vector_quals(DecompressContext *dcontext, List *quals, TupleTableSlot *slot); static int get_input_offset(const DecompressContext *dcontext, const Var *var) { const CompressionColumnDescription *value_column_description = NULL; for (int i = 0; i < dcontext->num_data_columns; i++) { const CompressionColumnDescription *current_column = &dcontext->compressed_chunk_columns[i]; if (current_column->uncompressed_chunk_attno == var->varattno) { value_column_description = current_column; break; } } Ensure(value_column_description != NULL, "aggregated compressed column not found"); Assert(value_column_description->type == COMPRESSED_COLUMN || value_column_description->type == SEGMENTBY_COLUMN); const int index = value_column_description - dcontext->compressed_chunk_columns; return index; } /* * Workspace for converting the results of a Postgres function into a columnar * format. */ typedef struct { DecompressionType type; uint64 *restrict validity; int allocated_body_bytes; uint8 *restrict body_buffer; uint32 *restrict offset_buffer; uint32 current_offset; } ColumnarResult; static void columnar_result_init_for_type(ColumnarResult *columnar_result, DecompressBatchState const *batch_state, Oid typeoid) { int16 typlen; bool typbyval; get_typlenbyval(typeoid, &typlen, &typbyval); if (typeoid == BOOLOID) { columnar_result->type = DT_ArrowBits; } else if (typlen == -1) { columnar_result->type = DT_ArrowText; } else { Assert(typlen > 0); columnar_result->type = typlen; } const int nrows = batch_state->total_batch_rows; const size_t num_validity_words = (nrows + 63) / 64; if (columnar_result->type == DT_ArrowBits) { columnar_result->allocated_body_bytes = sizeof(uint64) * num_validity_words; } else if (columnar_result->type == DT_ArrowText) { /* * Arrow variable-length types require n + 1 offsets to store the end * position of the last element. Pad to 64 bytes per Arrow spec. */ columnar_result->offset_buffer = MemoryContextAllocZero(batch_state->per_batch_context, pad_to_multiple(64, sizeof(*columnar_result->offset_buffer) * (nrows + 1))); columnar_result->allocated_body_bytes = pad_to_multiple(64, 10); } else { Assert(columnar_result->type > 0); columnar_result->allocated_body_bytes = pad_to_multiple(64, 1 + columnar_result->type * nrows); } columnar_result->body_buffer = MemoryContextAllocZero(batch_state->per_batch_context, columnar_result->allocated_body_bytes); } static pg_attribute_always_inline void columnar_result_set_row(ColumnarResult *columnar_result, DecompressBatchState const *batch_state, int row, Datum datum, bool isnull) { const int nrows = batch_state->total_batch_rows; Assert(row < nrows); if (isnull) { if (columnar_result->validity == NULL) { const int num_validity_words = (nrows + 63) / 64; columnar_result->validity = MemoryContextAlloc(batch_state->per_batch_context, num_validity_words * sizeof(*columnar_result->validity)); memset(columnar_result->validity, -1, num_validity_words * sizeof(*columnar_result->validity)); if (nrows % 64 != 0) { const uint64 tail_mask = ~0ULL >> (64 - nrows % 64); columnar_result->validity[nrows / 64] &= tail_mask; } } arrow_set_row_validity(columnar_result->validity, row, false); return; } switch ((int) columnar_result->type) { case DT_ArrowBits: { arrow_set_row_validity((uint64 *restrict) columnar_result->body_buffer, row, DatumGetBool(datum)); break; } case DT_ArrowText: { const int result_bytes = VARSIZE_ANY_EXHDR(datum); const int required_body_bytes = pad_to_multiple(64, columnar_result->current_offset + result_bytes); if (required_body_bytes > columnar_result->allocated_body_bytes) { /* * We reallocate based on how many rows in the batch we have * left, not to overshoot too much. At the same time, we * shouldn't reallocate too often either. The parameters were * tuned manually on a few real data sets until this balance * looked somewhat acceptable. */ const int new_body_bytes = required_body_bytes * Min(10, Max(1.2, 1.2 * nrows / ((float) row + 1))) + 1; Assert(new_body_bytes >= required_body_bytes); columnar_result->body_buffer = repalloc(columnar_result->body_buffer, new_body_bytes); columnar_result->allocated_body_bytes = new_body_bytes; } memcpy(&columnar_result->body_buffer[columnar_result->current_offset], VARDATA_ANY(datum), result_bytes); columnar_result->offset_buffer[row] = columnar_result->current_offset; columnar_result->current_offset += result_bytes; break; } case 2: case 4: #ifdef USE_FLOAT8_BYVAL case 8: #endif memcpy(row * columnar_result->type + (uint8 *restrict) columnar_result->body_buffer, &datum, sizeof(Datum)); break; #ifndef USE_FLOAT8_BYVAL case 8: #endif case 16: memcpy(row * columnar_result->type + (uint8 *restrict) columnar_result->body_buffer, DatumGetPointer(datum), columnar_result->type); break; default: elog(ERROR, "wrong arrow result type %d", columnar_result->type); } } static CompressedColumnValues columnar_result_finalize(ColumnarResult *columnar_result, DecompressBatchState const *batch_state) { const int nrows = batch_state->total_batch_rows; ArrowArray *arrow_result = NULL; if (columnar_result->type == DT_ArrowBits) { arrow_result = MemoryContextAllocZero(batch_state->per_batch_context, sizeof(ArrowArray) + 2 * sizeof(void *)); arrow_result->buffers = (void *) &arrow_result[1]; arrow_result->buffers[1] = columnar_result->body_buffer; } else if (columnar_result->type == DT_ArrowText) { columnar_result->offset_buffer[nrows] = columnar_result->current_offset; arrow_result = MemoryContextAllocZero(batch_state->per_batch_context, sizeof(ArrowArray) + 3 * sizeof(void *)); arrow_result->buffers = (void *) &arrow_result[1]; arrow_result->buffers[1] = columnar_result->offset_buffer; arrow_result->buffers[2] = columnar_result->body_buffer; } else { Assert(columnar_result->type > 0); arrow_result = MemoryContextAllocZero(batch_state->per_batch_context, sizeof(ArrowArray) + 2 * sizeof(void *)); arrow_result->buffers = (void *) &arrow_result[1]; arrow_result->buffers[1] = columnar_result->body_buffer; } arrow_result->length = nrows; arrow_result->buffers[0] = columnar_result->validity; arrow_result->null_count = arrow_result->length - arrow_num_valid(arrow_result->buffers[0], nrows); CompressedColumnValues result = { .decompression_type = columnar_result->type, .buffers = { arrow_result->buffers[0], arrow_result->buffers[1], columnar_result->type == DT_ArrowText ? arrow_result->buffers[2] : NULL }, .arrow = arrow_result, }; return result; } static pg_noinline CompressedColumnValues vector_slot_evaluate_function(DecompressContext *dcontext, TupleTableSlot *slot, uint64 const *filter, List *args, Oid funcoid, Oid inputcollid) { const DecompressBatchState *batch_state = (const DecompressBatchState *) slot; const int nargs = list_length(args); FmgrInfo flinfo; fmgr_info(funcoid, &flinfo); FunctionCallInfo fcinfo = palloc0(SizeForFunctionCallInfo(nargs)); InitFunctionCallInfoData(*fcinfo, &flinfo, nargs, inputcollid, NULL, NULL); CompressedColumnValues *arg_values = palloc0(nargs * sizeof(*arg_values)); bool have_null_bitmap = false; bool have_null_scalars = false; ListCell *lc; foreach (lc, args) { const int i = foreach_current_index(lc); CompressedColumnValues arg_value = vector_slot_evaluate_expression(dcontext, slot, filter, lfirst(lc)); Ensure(arg_value.decompression_type != DT_Invalid, "got DT_Invalid for argument %d", i); have_null_bitmap = (arg_value.arrow != NULL && arg_value.arrow->null_count > 0) || have_null_bitmap; arg_value.output_value = &fcinfo->args[i].value; arg_value.output_isnull = &fcinfo->args[i].isnull; if (arg_value.decompression_type == DT_ArrowText || arg_value.decompression_type == DT_ArrowTextDict) { const int maxbytes = get_max_varlena_bytes(arg_value.arrow); *arg_value.output_value = PointerGetDatum(MemoryContextAlloc(batch_state->per_batch_context, maxbytes)); } else if (arg_value.decompression_type == DT_Scalar) { /* * The values of the scalar columns have to be stored once at * initialization, they won't be updated per-row. */ *arg_value.output_value = PointerGetDatum(arg_value.buffers[1]); *arg_value.output_isnull = DatumGetBool(PointerGetDatum(arg_value.buffers[0])); have_null_scalars = *arg_value.output_isnull || have_null_scalars; } arg_values[i] = arg_value; } /* * We only evaluate strict functions, so if we have a scalar null argument, * return a scalar null. */ if (have_null_scalars) { pfree(fcinfo); pfree(arg_values); return (CompressedColumnValues){ .decompression_type = DT_Scalar, .buffers[0] = DatumGetPointer(BoolGetDatum(true)) }; } /* * Our Postgres function is strict, so we should avoid calling it on null * inputs. */ const int nrows = batch_state->total_batch_rows; const size_t num_validity_words = (nrows + 63) / 64; uint64 *input_validity = NULL; if (have_null_bitmap || filter != NULL) { uint64 *restrict combined_validity = MemoryContextAlloc(batch_state->per_batch_context, sizeof(*combined_validity) * num_validity_words); memset(combined_validity, -1, num_validity_words * sizeof(*combined_validity)); arrow_validity_and(num_validity_words, combined_validity, filter); for (int i = 0; i < nargs; i++) { arrow_validity_and(num_validity_words, combined_validity, arg_values[i].buffers[0]); } input_validity = combined_validity; } /* * Call the Postgres function on every row. Here as well, we have to deal * with very selective filters and avoid evaluating the functions on long * consecutive ranges of filtered out rows, to improve the performance. */ ColumnarResult columnar_result = { 0 }; columnar_result_init_for_type(&columnar_result, batch_state, get_func_rettype(funcoid)); MemoryContext function_call_context = AllocSetContextCreate(CurrentMemoryContext, "bulk function call", ALLOCSET_DEFAULT_SIZES); MemoryContext old = MemoryContextSwitchTo(function_call_context); FilterWordIterator iter = filter_word_iterator_init(nrows, input_validity); int last_processed_row = 0; for (;;) { /* * The Arrow format requires the offsets to monotonically increase even * for the invalid rows. */ if (columnar_result.offset_buffer != NULL) { for (int row = last_processed_row; row < iter.start_row; row++) { columnar_result.offset_buffer[row] = columnar_result.current_offset; } } if (!filter_word_iterator_is_valid(&iter)) { break; } for (int row = iter.start_row; row < iter.end_row; row++) { /* * The Arrow format requires the offsets to monotonically increase even * for the invalid rows. */ if (columnar_result.offset_buffer != NULL) { columnar_result.offset_buffer[row] = columnar_result.current_offset; } /* * Do not evaluate the function on null inputs because it is strict. */ if (!arrow_row_is_valid(input_validity, row)) { continue; } compressed_columns_to_postgres_data(arg_values, nargs, row); const Datum datum = FunctionCallInvoke(fcinfo); /* * A strict function can still return a null for a non-null argument. */ const bool isnull = fcinfo->isnull; columnar_result_set_row(&columnar_result, batch_state, row, datum, isnull); MemoryContextReset(function_call_context); } last_processed_row = iter.end_row; filter_word_iterator_advance(&iter); } MemoryContextSwitchTo(old); MemoryContextDelete(function_call_context); /* * Figure out the validity bitmap of the result rows. Besides the null * inputs, the function itself can return nulls for some rows. */ if (columnar_result.validity != NULL) { arrow_validity_and(num_validity_words, columnar_result.validity, input_validity); } else { columnar_result.validity = (uint64 *) input_validity; } pfree(fcinfo); pfree(arg_values); return columnar_result_finalize(&columnar_result, batch_state); } /* * Return the arrow array or the datum (in case of single scalar value) for a * given expression as a CompressedColumnValues struct. */ CompressedColumnValues vector_slot_evaluate_expression(DecompressContext *dcontext, TupleTableSlot *slot, uint64 const *filter, const Expr *argument) { const DecompressBatchState *batch_state = (const DecompressBatchState *) slot; switch (((Node *) argument)->type) { case T_Const: { const Const *c = (const Const *) argument; CompressedColumnValues result = { .decompression_type = DT_Scalar, .buffers[1] = DatumGetPointer(c->constvalue), .buffers[0] = DatumGetPointer(BoolGetDatum(c->constisnull)) }; return result; } case T_Var: { const Var *var = (const Var *) argument; const uint16 offset = get_input_offset(dcontext, var); const CompressedColumnValues *values = &batch_state->compressed_columns[offset]; Ensure(values->decompression_type != DT_Invalid, "got DT_Invalid decompression type at offset %d", offset); return *values; } case T_OpExpr: { const OpExpr *o = (const OpExpr *) argument; return vector_slot_evaluate_function(dcontext, slot, filter, o->args, o->opfuncid, o->inputcollid); } case T_FuncExpr: { const FuncExpr *f = (const FuncExpr *) argument; return vector_slot_evaluate_function(dcontext, slot, filter, f->args, f->funcid, f->inputcollid); } default: Ensure(false, "wrong node type %s for vector expression", ts_get_node_name((Node *) argument)); return (CompressedColumnValues){ .decompression_type = DT_Invalid }; } } static void vector_agg_begin(CustomScanState *node, EState *estate, int eflags) { CustomScan *cscan = castNode(CustomScan, node->ss.ps.plan); node->custom_ps = lappend(node->custom_ps, ExecInitNode(linitial(cscan->custom_plans), estate, eflags)); VectorAggState *vector_agg_state = (VectorAggState *) node; vector_agg_state->input_ended = false; /* * Set up the helper structures used to evaluate stable expressions in * vectorized FILTER clauses. */ PlannerGlobal glob = { .boundParams = node->ss.ps.state->es_param_list_info, }; PlannerInfo root = { .glob = &glob, }; /* * The aggregated targetlist with Aggrefs is in the custom scan targetlist * of the custom scan node that is performing the vectorized aggregation. * We do this to avoid projections at this node, because the postgres * projection functions complain when they see an Aggref in a custom * node output targetlist. * The output targetlist, in turn, consists of just the INDEX_VAR references * into the custom_scan_tlist. * Now, iterate through the aggregated targetlist to collect aggregates and * output grouping columns. */ List *aggregated_tlist = castNode(CustomScan, vector_agg_state->custom.ss.ps.plan)->custom_scan_tlist; const int tlist_length = list_length(aggregated_tlist); /* * First, count how many grouping columns and aggregate functions we have. */ int agg_functions_counter = 0; int grouping_column_counter = 0; for (int i = 0; i < tlist_length; i++) { TargetEntry *tlentry = list_nth_node(TargetEntry, aggregated_tlist, i); if (IsA(tlentry->expr, Aggref)) { agg_functions_counter++; } else { /* This is a grouping column. */ grouping_column_counter++; } } Assert(agg_functions_counter + grouping_column_counter == tlist_length); /* * Allocate the storage for definitions of aggregate function and grouping * columns. */ vector_agg_state->num_agg_defs = agg_functions_counter; vector_agg_state->agg_defs = palloc0(sizeof(*vector_agg_state->agg_defs) * vector_agg_state->num_agg_defs); vector_agg_state->num_grouping_columns = grouping_column_counter; vector_agg_state->grouping_columns = palloc0(sizeof(*vector_agg_state->grouping_columns) * vector_agg_state->num_grouping_columns); /* * Loop through the aggregated targetlist again and fill the definitions. */ agg_functions_counter = 0; grouping_column_counter = 0; for (int i = 0; i < tlist_length; i++) { TargetEntry *tlentry = list_nth_node(TargetEntry, aggregated_tlist, i); if (IsA(tlentry->expr, Aggref)) { /* This is an aggregate function. */ VectorAggDef *def = &vector_agg_state->agg_defs[agg_functions_counter++]; def->output_offset = i; Aggref *aggref = castNode(Aggref, tlentry->expr); VectorAggFunctions *func = get_vector_aggregate(aggref->aggfnoid, aggref->inputcollid); Assert(func != NULL); def->func = *func; if (list_length(aggref->args) > 0) { Assert(list_length(aggref->args) == 1); /* The aggregate should be a partial aggregate */ Assert(aggref->aggsplit == AGGSPLIT_INITIAL_SERIAL); def->argument = castNode(TargetEntry, linitial(aggref->args))->expr; } else { def->argument = NULL; } if (aggref->aggfilter != NULL) { Node *constified = estimate_expression_value(&root, (Node *) aggref->aggfilter); def->filter_clauses = list_make1(constified); } } else { /* This is a grouping column. */ GroupingColumn *col = &vector_agg_state->grouping_columns[grouping_column_counter++]; col->expr = tlentry->expr; col->output_offset = i; TupleDesc tdesc = NULL; Oid type = InvalidOid; TypeFuncClass type_class = get_expr_result_type((Node *) tlentry->expr, &type, &tdesc); Ensure(type_class == TYPEFUNC_SCALAR, "wrong grouping column type class %d", type_class); get_typlenbyval(type, &col->value_bytes, &col->by_value); } } /* * Create the grouping policy chosen at plan time. */ const VectorAggGroupingType grouping_type = intVal(list_nth(cscan->custom_private, VASI_GroupingType)); if (grouping_type == VAGT_Batch) { /* * Per-batch grouping. */ vector_agg_state->grouping = create_grouping_policy_batch(vector_agg_state->num_agg_defs, vector_agg_state->agg_defs, vector_agg_state->num_grouping_columns, vector_agg_state->grouping_columns); } else { /* * Hash grouping. */ vector_agg_state->grouping = create_grouping_policy_hash(vector_agg_state->num_agg_defs, vector_agg_state->agg_defs, vector_agg_state->num_grouping_columns, vector_agg_state->grouping_columns, grouping_type); } } static void vector_agg_end(CustomScanState *node) { ExecEndNode(linitial(node->custom_ps)); } static void vector_agg_rescan(CustomScanState *node) { if (node->ss.ps.chgParam != NULL) UpdateChangedParamSet(linitial(node->custom_ps), node->ss.ps.chgParam); ExecReScan(linitial(node->custom_ps)); VectorAggState *state = (VectorAggState *) node; state->input_ended = false; state->grouping->gp_reset(state->grouping); } /* * Get the next slot to aggregate for a compressed batch. * * Implements "get next slot" on top of ColumnarScan. Note that compressed * tuples are read directly from the ColumnarScan child node, which means * that the processing normally done in ColumnarScan is actually done here * (batch processing and filtering). * * Returns an TupleTableSlot that implements a compressed batch. */ static TupleTableSlot * compressed_batch_get_next_slot(VectorAggState *vector_agg_state) { ColumnarScanState *decompress_state = (ColumnarScanState *) linitial(vector_agg_state->custom.custom_ps); DecompressContext *dcontext = &decompress_state->decompress_context; BatchQueue *batch_queue = decompress_state->batch_queue; DecompressBatchState *batch_state = batch_array_get_at(&batch_queue->batch_array, 0); do { /* * We discard the previous compressed batch here and not earlier, * because the grouping column values returned by the batch grouping * policy are owned by the compressed batch memory context. This is done * to avoid generic value copying in the grouping policy to simplify its * code. */ compressed_batch_discard_tuples(batch_state); TupleTableSlot *compressed_slot = ExecProcNode(linitial(decompress_state->csstate.custom_ps)); if (TupIsNull(compressed_slot)) { vector_agg_state->input_ended = true; return NULL; } if (dcontext->ps->instrument) { /* * Ensure proper EXPLAIN output for the underlying ColumnarScan * node. * * This value is normally updated by InstrStopNode(), and is * required so that the calculations in InstrEndLoop() run properly. * We have to call it manually because we run the underlying * ColumnarScan manually and not as a normal Postgres node. */ dcontext->ps->instrument->running = true; } compressed_batch_set_compressed_tuple(dcontext, batch_state, compressed_slot); /* If the entire batch is filtered out, then immediately read the next * one */ } while (batch_state->next_batch_row >= batch_state->total_batch_rows); /* * Count rows filtered out by vectorized filters for EXPLAIN. Normally * this is done in tuple-by-tuple interface of ColumnarScan, so that * it doesn't say it filtered out more rows that were returned (e.g. * with LIMIT). Here we always work in full batches. The batches that * were fully filtered out, and their rows, were already counted in * compressed_batch_set_compressed_tuple(). */ const int not_filtered_rows = arrow_num_valid(batch_state->vector_qual_result, batch_state->total_batch_rows); InstrCountFiltered1(dcontext->ps, batch_state->total_batch_rows - not_filtered_rows); if (dcontext->ps->instrument) { /* * Ensure proper EXPLAIN output for the underlying ColumnarScan * node. * * This value is normally updated by InstrStopNode(), and is * required so that the calculations in InstrEndLoop() run properly. * We have to call it manually because we run the underlying * ColumnarScan manually and not as a normal Postgres node. */ dcontext->ps->instrument->tuplecount += not_filtered_rows; } return &batch_state->decompressed_scan_slot_data.base; } /* * Initialize vector quals for a compressed batch. * * Used to implement vectorized aggregate function filter clause. */ static CompressedBatchVectorQualState compressed_batch_init_vector_quals(DecompressContext *dcontext, List *quals, TupleTableSlot *slot) { DecompressBatchState *batch_state = (DecompressBatchState *) slot; return (CompressedBatchVectorQualState) { .vqstate = { .vectorized_quals_constified = quals, .num_results = batch_state->total_batch_rows, .per_vector_mcxt = batch_state->per_batch_context, .slot = slot, .get_arrow_array = compressed_batch_get_arrow_array, }, .batch_state = batch_state, .dcontext = dcontext, }; } static TupleTableSlot * vector_agg_exec(CustomScanState *node) { VectorAggState *vector_agg_state = (VectorAggState *) node; ColumnarScanState *decompress_state = (ColumnarScanState *) linitial(vector_agg_state->custom.custom_ps); DecompressContext *dcontext = &decompress_state->decompress_context; ExprContext *econtext = node->ss.ps.ps_ExprContext; ResetExprContext(econtext); TupleTableSlot *aggregated_slot = vector_agg_state->custom.ss.ps.ps_ResultTupleSlot; ExecClearTuple(aggregated_slot); /* * If we have more partial aggregation results, continue returning them. */ GroupingPolicy *grouping = vector_agg_state->grouping; MemoryContext old_context = MemoryContextSwitchTo(econtext->ecxt_per_tuple_memory); bool have_partial = grouping->gp_do_emit(grouping, aggregated_slot); MemoryContextSwitchTo(old_context); if (have_partial) { /* The grouping policy produced a partial aggregation result. */ return ExecStoreVirtualTuple(aggregated_slot); } /* * Have no more partial aggregation results but might still have input. * Reset the grouping policy and start a new cycle of partial aggregation. */ grouping->gp_reset(grouping); /* * If the partial aggregation results have ended, and the input has ended, * we're done. */ if (vector_agg_state->input_ended) { return NULL; } /* * Now we loop through the input compressed tuples, until they end or until * the grouping policy asks us to emit partials. */ while (!grouping->gp_should_emit(grouping)) { /* * Get the next slot to aggregate. It will be either a compressed * batch or an arrow tuple table slot. Both hold arrow arrays of data * that can be vectorized. */ TupleTableSlot *slot = vector_agg_state->get_next_slot(vector_agg_state); /* * Exit if there is no more data. Note that it is not possible to do * the standard TupIsNull() check here because the compressed batch's * implementation of TupleTableSlot never clears the empty flag bit * (TTS_EMPTY), so it will always look empty. Therefore, look at the * "input_ended" flag instead. */ if (vector_agg_state->input_ended) break; /* * Compute the vectorized filters for the aggregate function FILTER * clauses. */ const int naggs = vector_agg_state->num_agg_defs; for (int i = 0; i < naggs; i++) { VectorAggDef *agg_def = &vector_agg_state->agg_defs[i]; uint64 *filter_clause_result = NULL; if (agg_def->filter_clauses != NIL) { CompressedBatchVectorQualState vqstate = compressed_batch_init_vector_quals(dcontext, agg_def->filter_clauses, slot); if (vector_qual_compute(&vqstate.vqstate) != AllRowsPass) { filter_clause_result = vqstate.vqstate.vector_qual_result; } } DecompressBatchState *batch_state = (DecompressBatchState *) slot; if (filter_clause_result != NULL) { const int num_validity_words = (batch_state->total_batch_rows + 63) / 64; arrow_validity_and(num_validity_words, filter_clause_result, batch_state->vector_qual_result); agg_def->effective_batch_filter = filter_clause_result; } else { agg_def->effective_batch_filter = batch_state->vector_qual_result; } } /* * Finally, pass the compressed batch to the grouping policy. */ grouping->gp_add_batch(grouping, dcontext, slot); } /* * If we have partial aggregation results, start returning them. */ old_context = MemoryContextSwitchTo(econtext->ecxt_per_tuple_memory); have_partial = grouping->gp_do_emit(grouping, aggregated_slot); MemoryContextSwitchTo(old_context); if (have_partial) { /* Have partial aggregation results. */ return ExecStoreVirtualTuple(aggregated_slot); } if (vector_agg_state->input_ended) { /* * Have no partial aggregation results and the input has ended, so we're * done. We can get here only if we had no input at all, otherwise the * grouping policy would have produced some partials above. */ return NULL; } /* * We cannot get here. This would mean we still have input, and the * grouping policy asked us to stop but couldn't produce any partials. */ Assert(false); pg_unreachable(); return NULL; } static void vector_agg_explain(CustomScanState *node, List *ancestors, ExplainState *es) { VectorAggState *state = (VectorAggState *) node; if (es->verbose || es->format != EXPLAIN_FORMAT_TEXT) { ExplainPropertyText("Grouping Policy", state->grouping->gp_explain(state->grouping), es); } } static struct CustomExecMethods exec_methods = { .CustomName = VECTOR_AGG_NODE_NAME, .BeginCustomScan = vector_agg_begin, .ExecCustomScan = vector_agg_exec, .EndCustomScan = vector_agg_end, .ReScanCustomScan = vector_agg_rescan, .ExplainCustomScan = vector_agg_explain, }; Node * vector_agg_state_create(CustomScan *cscan) { VectorAggState *state = (VectorAggState *) newNode(sizeof(VectorAggState), T_CustomScanState); Assert(ts_is_columnar_scan_plan((Plan *) linitial(cscan->custom_plans))); state->custom.methods = &exec_methods; /* * Initialize VectorAggState to process vector slots from different * subnodes. * * When the child is ColumnarScan, VectorAgg doesn't read the slot from * the child node. Instead, it bypasses ColumnarScan and reads * compressed tuples directly from the grandchild. It therefore needs to * handle batch decompression and vectorized qual filtering itself, in its * own "get next slot" implementation. * * The vector qual init functions are needed to implement vectorized * aggregate function FILTER clauses for arrow tuple table slots and * compressed batches, respectively. */ state->get_next_slot = compressed_batch_get_next_slot; return (Node *) state; } ================================================ FILE: tsl/src/nodes/vector_agg/exec.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include "nodes/columnar_scan/compressed_batch.h" #include <nodes/execnodes.h> #include "function/functions.h" #include "grouping_policy.h" typedef struct VectorAggDef { VectorAggFunctions func; Expr *argument; int output_offset; List *filter_clauses; /* * This filter bitmap ANDs the batch filter and the aggregate function * FILTER clause, if present. */ uint64 const *effective_batch_filter; } VectorAggDef; typedef struct GroupingColumn { Expr *expr; int output_offset; int16 value_bytes; bool by_value; } GroupingColumn; typedef struct VectorAggState { CustomScanState custom; int num_agg_defs; VectorAggDef *agg_defs; int num_grouping_columns; GroupingColumn *grouping_columns; /* * We can't call the underlying scan after it has ended, or it will be * restarted. This is the behavior of Postgres heap scans. So we have to * track whether it has ended to avoid this. */ bool input_ended; GroupingPolicy *grouping; /* * Function for getting the next slot from the child node depending on * child node type. */ TupleTableSlot *(*get_next_slot)(struct VectorAggState *vector_agg_state); } VectorAggState; extern Node *vector_agg_state_create(CustomScan *cscan); ================================================ FILE: tsl/src/nodes/vector_agg/filter_word_iterator.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once /* * Iterator that skips the bitmap words that are fully zero. It helps us do less * work when most of the batch rows are filtered out. */ typedef struct { const int nrows; const int past_the_end_word; uint64 const *const filter; int start_word; int end_word; int start_row; int end_row; } FilterWordIterator; inline static void filter_word_iterator_advance(FilterWordIterator *iter) { if (iter->filter == NULL) { iter->start_word = iter->end_word; iter->end_word = iter->past_the_end_word; iter->start_row = iter->end_row; iter->end_row = iter->nrows; return; } /* * Skip the bitmap words which are zero. */ for (iter->start_word = iter->end_word; iter->start_word < iter->past_the_end_word && iter->filter[iter->start_word] == 0; iter->start_word++) ; if (iter->start_word >= iter->past_the_end_word) { /* * Finished. The start_row shouldn't be used because the iterator is * invalid now, but set it to past-the-end for consistency. */ iter->start_row = iter->nrows; return; } /* * Collect the consecutive bitmap words which are nonzero. */ for (iter->end_word = iter->start_word + 1; iter->end_word < iter->past_the_end_word && iter->filter[iter->end_word] != 0; iter->end_word++) ; Assert(iter->end_word > iter->start_word); /* * Now we have the [start, end] range of bitmap words that are * nonzero. * * Determine starting and ending rows, also skipping the starting * and trailing zero bits at the ends of the range. */ iter->start_row = iter->start_word * 64 + pg_rightmost_one_pos64(iter->filter[iter->start_word]); Assert(iter->start_row <= iter->nrows); /* * The bits for past-the-end rows must be set to zero, so this * calculation should yield no more than n. */ iter->end_row = (iter->end_word - 1) * 64 + pg_leftmost_one_pos64(iter->filter[iter->end_word - 1]) + 1; Assert(iter->end_row <= iter->nrows); } inline static FilterWordIterator filter_word_iterator_init(int nrows, uint64 const *filter) { FilterWordIterator iter = { .nrows = nrows, .past_the_end_word = (nrows - 1) / 64 + 1, .filter = filter }; filter_word_iterator_advance(&iter); return iter; } inline static bool filter_word_iterator_is_valid(FilterWordIterator const *iter) { return iter->start_word < iter->past_the_end_word; } ================================================ FILE: tsl/src/nodes/vector_agg/function/CMakeLists.txt ================================================ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/functions.c ${CMAKE_CURRENT_SOURCE_DIR}/minmax_templates.c ${CMAKE_CURRENT_SOURCE_DIR}/int24_sum_templates.c ${CMAKE_CURRENT_SOURCE_DIR}/sum_float_templates.c ${CMAKE_CURRENT_SOURCE_DIR}/float48_accum_templates.c ${CMAKE_CURRENT_SOURCE_DIR}/int24_avg_accum_templates.c ${CMAKE_CURRENT_SOURCE_DIR}/int128_accum_templates.c) target_sources(${TSL_LIBRARY_NAME} PRIVATE ${SOURCES}) ================================================ FILE: tsl/src/nodes/vector_agg/function/agg_many_vector_helper.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * A generic implementation of adding the given batch to many aggregate function * states with given offsets. Used for hash aggregation, and builds on the * FUNCTION_NAME(one) function, which adds one passing non-null row to the given * aggregate function state. */ static pg_attribute_always_inline void FUNCTION_NAME(many_vector_impl)(void *restrict agg_states, const uint32 *offsets, const uint64 *filter, int start_row, int end_row, const ArrowArray *vector, MemoryContext agg_extra_mctx) { FUNCTION_NAME(state) *restrict states = (FUNCTION_NAME(state) *) agg_states; const CTYPE *values = vector->buffers[1]; MemoryContext old = MemoryContextSwitchTo(agg_extra_mctx); for (int row = start_row; row < end_row; row++) { const CTYPE value = values[row]; FUNCTION_NAME(state) *restrict state = &states[offsets[row]]; if (arrow_row_is_valid(filter, row)) { Assert(offsets[row] != 0); FUNCTION_NAME(one)(state, value); } } MemoryContextSwitchTo(old); } static pg_noinline void FUNCTION_NAME(many_vector_all_valid)(void *restrict agg_states, const uint32 *offsets, int start_row, int end_row, const ArrowArray *vector, MemoryContext agg_extra_mctx) { FUNCTION_NAME(many_vector_impl) (agg_states, offsets, NULL, start_row, end_row, vector, agg_extra_mctx); } static void FUNCTION_NAME(many_vector)(void *restrict agg_states, const uint32 *offsets, const uint64 *filter, int start_row, int end_row, const ArrowArray *vector, MemoryContext agg_extra_mctx) { if (filter == NULL) { FUNCTION_NAME(many_vector_all_valid) (agg_states, offsets, start_row, end_row, vector, agg_extra_mctx); } else { FUNCTION_NAME(many_vector_impl) (agg_states, offsets, filter, start_row, end_row, vector, agg_extra_mctx); } } ================================================ FILE: tsl/src/nodes/vector_agg/function/agg_scalar_helper.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * A generic function for aggregating a constant input. We use a very simple * implementation here, because aggregating a segmentby column or a column with * default value is a relatively rare case, but it requires a fully custom * implementation otherwise. */ static void FUNCTION_NAME(scalar)(void *agg_state, Datum constvalue, bool constisnull, int n, MemoryContext agg_extra_mctx) { if (constisnull) { return; } const CTYPE value = DATUM_TO_CTYPE(constvalue); MemoryContext old = MemoryContextSwitchTo(agg_extra_mctx); for (int i = 0; i < n; i++) { FUNCTION_NAME(one)(agg_state, value); } MemoryContextSwitchTo(old); } ================================================ FILE: tsl/src/nodes/vector_agg/function/agg_vector_validity_helper.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Generate a separate implementation of aggregating an ArrowArray for the * common cases where we have no nulls and/or all rows pass the filter. It * avoids branches so can be more easily vectorized. */ static pg_attribute_always_inline void FUNCTION_NAME(vector_impl_arrow)(void *agg_state, const ArrowArray *vector, const uint64 *filter, MemoryContext agg_extra_mctx) { const int n = vector->length; const CTYPE *values = vector->buffers[1]; FUNCTION_NAME(vector_impl)(agg_state, n, values, filter, agg_extra_mctx); } static pg_noinline void FUNCTION_NAME(vector_all_valid)(void *agg_state, const ArrowArray *vector, MemoryContext agg_extra_mctx) { FUNCTION_NAME(vector_impl_arrow)(agg_state, vector, NULL, agg_extra_mctx); } static pg_noinline void FUNCTION_NAME(vector_one_validity)(void *agg_state, const ArrowArray *vector, const uint64 *filter, MemoryContext agg_extra_mctx) { FUNCTION_NAME(vector_impl_arrow)(agg_state, vector, filter, agg_extra_mctx); } static void FUNCTION_NAME(vector)(void *agg_state, const ArrowArray *vector, const uint64 *filter, MemoryContext agg_extra_mctx) { if (filter == NULL) { /* All rows are valid and we don't have to check any validity bitmaps. */ FUNCTION_NAME(vector_all_valid)(agg_state, vector, agg_extra_mctx); } else { /* Have to check only one combined validity bitmap. */ FUNCTION_NAME(vector_one_validity)(agg_state, vector, filter, agg_extra_mctx); } } ================================================ FILE: tsl/src/nodes/vector_agg/function/float48_accum_single.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Vectorized implementation of a Postgres float{4,8}_accum() transition * function for a single type. They use the same Youngs-Cramer state, but for * AVG we can skip calculating the Sxx variable. */ #ifdef GENERATE_DISPATCH_TABLE /* * Forward declaration for the vectorized aggregate function definition. */ extern VectorAggFunctions FUNCTION_NAME(argdef); /* * Helper macros to generate the cases for the given argument type. We support * different aggregate functions based on whether we calculate the Sxx variable. */ #ifdef NEED_SXX #define ACCUM_CASE_HELPER(PG_TYPE) \ case F_STDDEV_##PG_TYPE: \ case F_STDDEV_SAMP_##PG_TYPE: \ case F_STDDEV_POP_##PG_TYPE: \ case F_VARIANCE_##PG_TYPE: \ case F_VAR_SAMP_##PG_TYPE: \ case F_VAR_POP_##PG_TYPE: #else #define ACCUM_CASE_HELPER(PG_TYPE) case F_AVG_##PG_TYPE: #endif #define ACCUM_CASE(PG_TYPE) ACCUM_CASE_HELPER(PG_TYPE) /* * The actual case label. */ ACCUM_CASE(PG_TYPE) return &FUNCTION_NAME(argdef); #else /* * State of Youngs-Cramer algorithm, see the comments for float8_accum() * Postgres function. */ typedef struct { double N; double Sx; #ifdef NEED_SXX double Sxx; #endif } FUNCTION_NAME(state); static void FUNCTION_NAME(init)(void *restrict agg_states, int n) { FUNCTION_NAME(state) *states = (FUNCTION_NAME(state) *) agg_states; for (int i = 0; i < n; i++) { states[i] = (FUNCTION_NAME(state)){ 0 }; } } static void FUNCTION_NAME(emit)(void *agg_state, Datum *out_result, bool *out_isnull) { FUNCTION_NAME(state) *state = (FUNCTION_NAME(state) *) agg_state; const size_t nbytes = 3 * sizeof(float8) + ARR_OVERHEAD_NONULLS(/* ndims = */ 1); ArrayType *result = palloc(nbytes); SET_VARSIZE(result, nbytes); result->ndim = 1; result->dataoffset = 0; result->elemtype = FLOAT8OID; ARR_DIMS(result)[0] = 3; ARR_LBOUND(result)[0] = 1; /* * The array elements are stored by value, regardless of if the float8 * itself is by-value on this platform. */ ((float8 *) ARR_DATA_PTR(result))[0] = state->N; ((float8 *) ARR_DATA_PTR(result))[1] = state->Sx; ((float8 *) ARR_DATA_PTR(result))[2] = /* * Sxx should be NaN if any of the inputs are infinite or NaN. This is * checked by float8_combine even if it's not used for the actual * calculations. */ 0. * state->Sx #ifdef NEED_SXX + state->Sxx #endif ; *out_result = PointerGetDatum(result); *out_isnull = false; } /* * Youngs-Cramer update for rows after the first. */ static pg_attribute_always_inline void FUNCTION_NAME(update)(const uint64 *filter, const CTYPE *values, int row, double *N, double *Sx #ifdef NEED_SXX , double *Sxx #endif ) { const CTYPE newval = values[row]; if (!arrow_row_is_valid(filter, row)) { return; } /* * This code follows the Postgres float8_accum() transition function, see * the comments there. */ const double newN = *N + 1.0; const double newSx = *Sx + newval; #ifdef NEED_SXX Assert(*N > 0.0); const double tmp = newval * newN - newSx; *Sxx += tmp * tmp / (*N * newN); #endif *N = newN; *Sx = newSx; } /* * Combine two Youngs-Cramer states following the float8_combine() function. */ static pg_attribute_always_inline void FUNCTION_NAME(combine)(double *inout_N, double *inout_Sx, #ifdef NEED_SXX double *inout_Sxx, #endif double N2, double Sx2 #ifdef NEED_SXX , double Sxx2 #endif ) { const double N1 = *inout_N; const double Sx1 = *inout_Sx; #ifdef NEED_SXX const double Sxx1 = *inout_Sxx; #endif if (unlikely(N1 == 0)) { *inout_N = N2; *inout_Sx = Sx2; #ifdef NEED_SXX *inout_Sxx = Sxx2; #endif return; } if (unlikely(N2 == 0)) { *inout_N = N1; *inout_Sx = Sx1; #ifdef NEED_SXX *inout_Sxx = Sxx1; #endif return; } const double combinedN = N1 + N2; const double combinedSx = Sx1 + Sx2; #ifdef NEED_SXX const double tmp = Sx1 / N1 - Sx2 / N2; const double combinedSxx = Sxx1 + Sxx2 + N1 * N2 * tmp * tmp / combinedN; #endif *inout_N = combinedN; *inout_Sx = combinedSx; #ifdef NEED_SXX *inout_Sxx = combinedSxx; #endif } #ifdef NEED_SXX #define UPDATE(filter, values, row, N, Sx, Sxx) \ FUNCTION_NAME(update)(filter, values, row, N, Sx, Sxx) #define COMBINE(inout_N, inout_Sx, inout_Sxx, N2, Sx2, Sxx2) \ FUNCTION_NAME(combine)(inout_N, inout_Sx, inout_Sxx, N2, Sx2, Sxx2) #else #define UPDATE(filter, values, row, N, Sx, Sxx) FUNCTION_NAME(update)(filter, values, row, N, Sx) #define COMBINE(inout_N, inout_Sx, inout_Sxx, N2, Sx2, Sxx2) \ FUNCTION_NAME(combine)(inout_N, inout_Sx, N2, Sx2) #endif static pg_attribute_always_inline void FUNCTION_NAME(vector_impl)(void *agg_state, size_t n, const CTYPE *values, const uint64 *filter, MemoryContext agg_extra_mctx) { /* * Vector registers can be up to 512 bits wide. */ #define UNROLL_SIZE ((int) (512 / 8 / sizeof(CTYPE))) /* * Each inner iteration works with its own accumulators to avoid data * dependencies. */ double Narray[UNROLL_SIZE] = { 0 }; double Sxarray[UNROLL_SIZE] = { 0 }; #ifdef NEED_SXX double Sxxarray[UNROLL_SIZE] = { 0 }; #endif size_t row = 0; #ifdef NEED_SXX /* * Initialize each state with the first matching row. We do this separately * to make the actual update function branchless, namely the computation of * Sxx which works differently for the first row. */ for (size_t inner = 0; inner < UNROLL_SIZE; inner++) { for (; row < n; row++) { const CTYPE newval = values[row]; if (arrow_row_is_valid(filter, row)) { Narray[inner] = 1; Sxarray[inner] = newval; Sxxarray[inner] = 0 * newval; row++; break; } } } /* * Scroll to the row that is a multiple of UNROLL_SIZE. This is the correct * row at which to enter the unrolled loop below. */ for (size_t inner = row % UNROLL_SIZE; inner > 0 && inner < UNROLL_SIZE && row < n; inner++, row++) { UPDATE(filter, values, row, &Narray[inner], &Sxarray[inner], &Sxxarray[inner]); } #endif /* * Unrolled loop. */ Assert(row % UNROLL_SIZE == 0 || row == n); for (; row < UNROLL_SIZE * (n / UNROLL_SIZE); row += UNROLL_SIZE) { for (size_t inner = 0; inner < UNROLL_SIZE; inner++) { UPDATE(filter, values, row + inner, &Narray[inner], &Sxarray[inner], &Sxxarray[inner]); } } /* * Process the odd tail. */ for (; row < n; row++) { const size_t inner = row % UNROLL_SIZE; UPDATE(filter, values, row, &Narray[inner], &Sxarray[inner], &Sxxarray[inner]); } /* * Merge all intermediate states into the first one. */ for (int i = 1; i < UNROLL_SIZE; i++) { COMBINE(&Narray[0], &Sxarray[0], &Sxxarray[0], Narray[i], Sxarray[i], Sxxarray[i]); } #undef UNROLL_SIZE /* * Merge the total computed state into the aggregate function state. */ FUNCTION_NAME(state) *state = (FUNCTION_NAME(state) *) agg_state; COMBINE(&state->N, &state->Sx, &state->Sxx, Narray[0], Sxarray[0], Sxxarray[0]); } static pg_attribute_always_inline void FUNCTION_NAME(one)(void *restrict agg_state, const CTYPE value) { FUNCTION_NAME(state) *state = (FUNCTION_NAME(state) *) agg_state; /* * This code follows the Postgres float8_accum() transition function, see * the comments there. */ const double newN = state->N + 1.0; const double newSx = state->Sx + value; #ifdef NEED_SXX if (state->N > 0.0) { const double tmp = value * newN - newSx; state->Sxx += tmp * tmp / (state->N * newN); } else { state->Sxx = 0 * value; } #endif state->N = newN; state->Sx = newSx; } #include "agg_many_vector_helper.c" #include "agg_scalar_helper.c" #include "agg_vector_validity_helper.c" VectorAggFunctions FUNCTION_NAME(argdef) = { .state_bytes = sizeof(FUNCTION_NAME(state)), .agg_init = FUNCTION_NAME(init), .agg_emit = FUNCTION_NAME(emit), .agg_scalar = FUNCTION_NAME(scalar), .agg_vector = FUNCTION_NAME(vector), .agg_many_vector = FUNCTION_NAME(many_vector), }; #undef UPDATE #undef COMBINE #endif #undef PG_TYPE #undef CTYPE #undef DATUM_TO_CTYPE #undef CTYPE_TO_DATUM #undef ACCUM_CASE #undef ACCUM_CASE_HELPER ================================================ FILE: tsl/src/nodes/vector_agg/function/float48_accum_templates.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Common parts for aggregate functions that use the float{4,8}_accum transition. */ #include <postgres.h> #include <catalog/pg_type_d.h> #include <utils/array.h> #include <utils/float.h> #include <utils/fmgroids.h> #include <utils/fmgrprotos.h> #include "functions.h" #include "template_helper.h" #include <compression/arrow_c_data_interface.h> #ifndef GENERATE_DISPATCH_TABLE #endif /* * Templated parts for vectorized avg(float). */ #define AGG_NAME accum_no_squares #include "float48_accum_types.c" /* * Templated parts for vectorized functions that use the Sxx state (stddev etc). */ #define AGG_NAME accum_with_squares #define NEED_SXX #include "float48_accum_types.c" ================================================ FILE: tsl/src/nodes/vector_agg/function/float48_accum_types.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Functions handled by *accum() aggregate functions states, implementation for * all types. They use the same Youngs-Cramer state, but for AVG we can skip * calculating the Sxx variable. */ #define PG_TYPE FLOAT4 #define CTYPE float #define CTYPE_TO_DATUM Float4GetDatum #define DATUM_TO_CTYPE DatumGetFloat4 #include "float48_accum_single.c" #define PG_TYPE FLOAT8 #define CTYPE double #define CTYPE_TO_DATUM Float8GetDatum #define DATUM_TO_CTYPE DatumGetFloat8 #include "float48_accum_single.c" #undef AGG_NAME #undef NEED_SXX ================================================ FILE: tsl/src/nodes/vector_agg/function/functions.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <limits.h> #include <postgres.h> #include <common/int.h> #include <utils/date.h> #include <utils/float.h> #include <utils/fmgroids.h> #include <utils/fmgrprotos.h> #include "functions.h" #include "compat/compat.h" /* * For PG18+: provide the old lc_collate_is_c() API using pg_locale_t flags. */ #if PG18_GE #include "utils/pg_locale.h" static inline bool lc_collate_is_c(Oid collation) { return pg_newlocale_from_collation(collation)->collate_is_c; } #else #include <utils/pg_locale.h> #endif /* * Aggregate function count(*). */ typedef struct { int64 count; } CountState; static void count_init(void *restrict agg_states, int n) { CountState *states = (CountState *) agg_states; for (int i = 0; i < n; i++) { states[i].count = 0; } } static void count_emit(void *agg_state, Datum *out_result, bool *out_isnull) { CountState *state = (CountState *) agg_state; *out_result = Int64GetDatum(state->count); *out_isnull = false; } static void count_star_scalar(void *agg_state, Datum constvalue, bool constisnull, int n, MemoryContext agg_extra_mctx) { CountState *state = (CountState *) agg_state; state->count += n; } static pg_attribute_always_inline void count_star_many_scalar_impl(void *restrict agg_states, const uint32 *offsets, const uint64 *filter, int start_row, int end_row, Datum constvalue, bool constisnull, MemoryContext agg_extra_mctx) { CountState *states = (CountState *) agg_states; for (int row = start_row; row < end_row; row++) { if (arrow_row_is_valid(filter, row)) { states[offsets[row]].count++; } } } static pg_noinline void count_star_many_scalar_nofilter(void *restrict agg_states, const uint32 *offsets, int start_row, int end_row, Datum constvalue, bool constisnull, MemoryContext agg_extra_mctx) { count_star_many_scalar_impl(agg_states, offsets, NULL, start_row, end_row, constvalue, constisnull, agg_extra_mctx); } static void count_star_many_scalar(void *restrict agg_states, const uint32 *offsets, const uint64 *filter, int start_row, int end_row, Datum constvalue, bool constisnull, MemoryContext agg_extra_mctx) { if (filter == NULL) { count_star_many_scalar_nofilter(agg_states, offsets, start_row, end_row, constvalue, constisnull, agg_extra_mctx); } else { count_star_many_scalar_impl(agg_states, offsets, filter, start_row, end_row, constvalue, constisnull, agg_extra_mctx); } } VectorAggFunctions count_star_agg = { .state_bytes = sizeof(CountState), .agg_init = count_init, .agg_scalar = count_star_scalar, .agg_emit = count_emit, .agg_many_scalar = count_star_many_scalar, }; /* * Aggregate function count(x). */ static void count_any_scalar(void *agg_state, Datum constvalue, bool constisnull, int n, MemoryContext agg_extra_mctx) { if (constisnull) { return; } CountState *state = (CountState *) agg_state; state->count += n; } static void count_any_vector(void *agg_state, const ArrowArray *vector, const uint64 *filter, MemoryContext agg_extra_mctx) { CountState *state = (CountState *) agg_state; const int n = vector->length; /* First, process the full words. */ for (int i = 0; i < n / 64; i++) { const uint64 filter_word = filter ? filter[i] : ~0ULL; #ifdef HAVE__BUILTIN_POPCOUNT state->count += __builtin_popcountll(filter_word); #else /* * Unfortunately, we have to have this fallback for Windows. */ for (uint16 i = 0; i < 64; i++) { const bool this_bit = (filter_word >> i) & 1; state->count += this_bit; } #endif } /* * The tail word needs special handling because not all rows there are valid * (some are past-the-end) even when the bitmap is null. */ for (int i = 64 * (n / 64); i < n; i++) { state->count += arrow_row_is_valid(filter, i); } } static void count_any_many_vector(void *restrict agg_states, const uint32 *offsets, const uint64 *filter, int start_row, int end_row, const ArrowArray *vector, MemoryContext agg_extra_mctx) { for (int row = start_row; row < end_row; row++) { CountState *state = (offsets[row] + (CountState *) agg_states); if (arrow_row_is_valid(filter, row)) { state->count++; } } } VectorAggFunctions count_any_agg = { .state_bytes = sizeof(CountState), .agg_init = count_init, .agg_emit = count_emit, .agg_scalar = count_any_scalar, .agg_vector = count_any_vector, .agg_many_vector = count_any_many_vector, }; /* * Return the vector aggregate definition corresponding to the given * PG aggregate function Oid and collation. * * The collation parameter is used for text aggregates (min/max) which use * memcmp for comparison. This only produces correct results for C collation, * so we cannot use vectorized aggregation for non-C collations. */ VectorAggFunctions * get_vector_aggregate(Oid aggfnoid, Oid collation) { switch (aggfnoid) { case F_COUNT_: return &count_star_agg; case F_COUNT_ANY: return &count_any_agg; #define GENERATE_DISPATCH_TABLE 1 #include "float48_accum_templates.c" #include "int128_accum_templates.c" #include "int24_avg_accum_templates.c" #include "int24_sum_templates.c" #include "minmax_templates.c" #include "sum_float_templates.c" #undef GENERATE_DISPATCH_TABLE default: return NULL; } } ================================================ FILE: tsl/src/nodes/vector_agg/function/functions.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <compression/arrow_c_data_interface.h> /* * Function table for a vectorized implementation of an aggregate function. * * One state of aggregate function corresponds to one set of rows that are * supposed to be grouped together (e.g. for one grouping key). * * There are functions for adding a compressed batch to one aggregate function * state (no grouping keys), and to multiple aggregate function states laid out * contiguously in memory. */ typedef struct { /* Size of the aggregate function state. */ size_t state_bytes; /* * Initialize the n aggregate function states stored contiguously at the * given pointer. */ void (*agg_init)(void *restrict agg_states, int n); /* Aggregate a given arrow array. */ void (*agg_vector)(void *restrict agg_state, const ArrowArray *vector, const uint64 *filter, MemoryContext agg_extra_mctx); /* Aggregate a scalar value, like segmentby or column with default value. */ void (*agg_scalar)(void *restrict agg_state, Datum constvalue, bool constisnull, int n, MemoryContext agg_extra_mctx); /* * Add the rows of the given arrow array to aggregate function states given * by the respective offsets. */ void (*agg_many_vector)(void *restrict agg_states, const uint32 *offsets, const uint64 *filter, int start_row, int end_row, const ArrowArray *vector, MemoryContext agg_extra_mctx); /* * Same as above, but for a scalar argument. This is mostly important for * count(*) and can be NULL. */ void (*agg_many_scalar)(void *restrict agg_states, const uint32 *offsets, const uint64 *filter, int start_row, int end_row, Datum constvalue, bool constisnull, MemoryContext agg_extra_mctx); /* Emit a partial aggregation result. */ void (*agg_emit)(void *restrict agg_state, Datum *out_result, bool *out_isnull); } VectorAggFunctions; VectorAggFunctions *get_vector_aggregate(Oid aggfnoid, Oid collation); ================================================ FILE: tsl/src/nodes/vector_agg/function/int128_accum_single.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Vectorized implementation of a single transition function that uses the * Int128AggState transition state. Note that the serialization format differs * based on whether the sum of squares is needed. */ #ifdef GENERATE_DISPATCH_TABLE extern VectorAggFunctions FUNCTION_NAME(argdef); AGG_CASES return &FUNCTION_NAME(argdef); #else typedef struct { int64 N; int128 sumX; #ifdef NEED_SUMX2 int128 sumX2; #endif } FUNCTION_NAME(state); static void FUNCTION_NAME(init)(void *restrict agg_states, int n) { FUNCTION_NAME(state) *states = (FUNCTION_NAME(state) *) agg_states; for (int i = 0; i < n; i++) { states[i].N = 0; states[i].sumX = 0; #ifdef NEED_SUMX2 states[i].sumX2 = 0; #endif } } static void FUNCTION_NAME(emit)(void *agg_state, Datum *out_result, bool *out_isnull) { FUNCTION_NAME(state) *state = (FUNCTION_NAME(state) *) agg_state; PgInt128AggState result = { .N = state->N, .sumX = state->sumX, #ifdef NEED_SUMX2 .sumX2 = state->sumX2, #endif }; /* * The serialization functions insist on being called in aggregate context, * but thankfully don't use it in any way so we can use this dummy. */ AggState agg_context = { .ss.ps.type = T_AggState }; LOCAL_FCINFO(fcinfo, 1); InitFunctionCallInfoData(*fcinfo, NULL, 1, InvalidOid, (Node *) &agg_context, NULL); fcinfo->args[0].value = PointerGetDatum(&result); fcinfo->args[0].isnull = false; #ifdef NEED_SUMX2 *out_result = numeric_poly_serialize(fcinfo); #else *out_result = int8_avg_serialize(fcinfo); #endif *out_isnull = false; } static pg_attribute_always_inline void FUNCTION_NAME(vector_impl)(void *agg_state, int n, const CTYPE *values, const uint64 *filter, MemoryContext agg_extra_mctx) { int64 N = 0; int128 sumX = 0; #ifdef NEED_SUMX2 int128 sumX2 = 0; #endif for (int row = 0; row < n; row++) { const bool row_ok = arrow_row_is_valid(filter, row); const CTYPE value = values[row]; N += row_ok; sumX += value * row_ok; #ifdef NEED_SUMX2 sumX2 += ((int128) value) * ((int128) value) * row_ok; #endif } FUNCTION_NAME(state) *state = (FUNCTION_NAME(state) *) agg_state; state->N += N; state->sumX += sumX; #ifdef NEED_SUMX2 state->sumX2 += sumX2; #endif } static pg_attribute_always_inline void FUNCTION_NAME(one)(void *restrict agg_state, const CTYPE value) { FUNCTION_NAME(state) *state = (FUNCTION_NAME(state) *) agg_state; state->N++; state->sumX += value; #ifdef NEED_SUMX2 state->sumX2 += ((int128) value) * ((int128) value); #endif } #include "agg_many_vector_helper.c" #include "agg_scalar_helper.c" #include "agg_vector_validity_helper.c" VectorAggFunctions FUNCTION_NAME(argdef) = { .state_bytes = sizeof(FUNCTION_NAME(state)), .agg_init = FUNCTION_NAME(init), .agg_emit = FUNCTION_NAME(emit), .agg_scalar = FUNCTION_NAME(scalar), .agg_vector = FUNCTION_NAME(vector), .agg_many_vector = FUNCTION_NAME(many_vector), }; #endif #undef PG_TYPE #undef CTYPE #undef DATUM_TO_CTYPE #undef AGG_CASES #undef AGG_NAME #undef NEED_SUMX2 ================================================ FILE: tsl/src/nodes/vector_agg/function/int128_accum_templates.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Vectorized transition functions that use the Int128AggState transition state. */ #include <postgres.h> #include <nodes/execnodes.h> #include <utils/fmgroids.h> #include <utils/fmgrprotos.h> #include "functions.h" #include "template_helper.h" #include <compression/arrow_c_data_interface.h> #ifdef HAVE_INT128 #ifndef GENERATE_DISPATCH_TABLE /* * The PG aggregation state that we have to serialize. Copied from numeric.c. */ typedef struct { bool calcSumX2; /* if true, calculate sumX2 */ int64 N; /* count of processed numbers */ int128 sumX; /* sum of processed numbers */ int128 sumX2; /* sum of squares of processed numbers */ } PgInt128AggState; #endif /* * Vectorized implementation of int8_avg_accum() function. */ #define AGG_NAME accum_no_squares #define AGG_CASES \ case PG_AGG_OID_HELPER(SUM, PG_TYPE): \ case PG_AGG_OID_HELPER(AVG, PG_TYPE): #define PG_TYPE INT8 #define CTYPE int64 #define DATUM_TO_CTYPE DatumGetInt64 #include "int128_accum_single.c" /* * Vectorized implementation of int2_accum() function. */ #define NEED_SUMX2 #define AGG_NAME accum_with_squares #define AGG_CASES \ case PG_AGG_OID_HELPER(STDDEV, PG_TYPE): \ case PG_AGG_OID_HELPER(STDDEV_SAMP, PG_TYPE): \ case PG_AGG_OID_HELPER(STDDEV_POP, PG_TYPE): \ case PG_AGG_OID_HELPER(VARIANCE, PG_TYPE): \ case PG_AGG_OID_HELPER(VAR_SAMP, PG_TYPE): \ case PG_AGG_OID_HELPER(VAR_POP, PG_TYPE): #define PG_TYPE INT2 #define CTYPE int16 #define DATUM_TO_CTYPE DatumGetInt16 #include "int128_accum_single.c" /* * Vectorized implementation of int4_accum() function. */ #define NEED_SUMX2 #define AGG_NAME accum_with_squares #define AGG_CASES \ case PG_AGG_OID_HELPER(STDDEV, PG_TYPE): \ case PG_AGG_OID_HELPER(STDDEV_SAMP, PG_TYPE): \ case PG_AGG_OID_HELPER(STDDEV_POP, PG_TYPE): \ case PG_AGG_OID_HELPER(VARIANCE, PG_TYPE): \ case PG_AGG_OID_HELPER(VAR_SAMP, PG_TYPE): \ case PG_AGG_OID_HELPER(VAR_POP, PG_TYPE): #define PG_TYPE INT4 #define CTYPE int32 #define DATUM_TO_CTYPE DatumGetInt32 #include "int128_accum_single.c" #endif ================================================ FILE: tsl/src/nodes/vector_agg/function/int24_avg_accum_single.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #ifdef GENERATE_DISPATCH_TABLE extern VectorAggFunctions FUNCTION_NAME(argdef); case PG_AGG_OID_HELPER(AGG_NAME, PG_TYPE): return &FUNCTION_NAME(argdef); #else static pg_attribute_always_inline void FUNCTION_NAME(vector_impl)(void *agg_state, int n, const CTYPE *values, const uint64 *filter, MemoryContext agg_extra_mctx) { int64 batch_count = 0; int64 batch_sum = 0; for (int row = 0; row < n; row++) { const bool row_ok = arrow_row_is_valid(filter, row); batch_count += row_ok; batch_sum += values[row] * row_ok; } Int24AvgAccumState *state = (Int24AvgAccumState *) agg_state; state->count += batch_count; state->sum += batch_sum; } typedef Int24AvgAccumState FUNCTION_NAME(state); static pg_attribute_always_inline void FUNCTION_NAME(one)(void *restrict agg_state, const CTYPE value) { FUNCTION_NAME(state) *state = (FUNCTION_NAME(state) *) agg_state; state->count++; state->sum += value; } #include "agg_many_vector_helper.c" #include "agg_scalar_helper.c" #include "agg_vector_validity_helper.c" VectorAggFunctions FUNCTION_NAME(argdef) = { .state_bytes = sizeof(Int24AvgAccumState), .agg_init = int24_avg_accum_init, .agg_emit = int24_avg_accum_emit, .agg_scalar = FUNCTION_NAME(scalar), .agg_vector = FUNCTION_NAME(vector), .agg_many_vector = FUNCTION_NAME(many_vector), }; #endif #undef PG_TYPE #undef CTYPE #undef DATUM_TO_CTYPE ================================================ FILE: tsl/src/nodes/vector_agg/function/int24_avg_accum_templates.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Vectorized implementation for int{2,4}_avg_accum transition functions. */ #include <postgres.h> #include <catalog/pg_type_d.h> #include <utils/array.h> #include <utils/fmgroids.h> #include <utils/fmgrprotos.h> #include "functions.h" #include "template_helper.h" #include <compression/arrow_c_data_interface.h> #ifndef GENERATE_DISPATCH_TABLE typedef struct { int64 count; int64 sum; } Int24AvgAccumState; static void int24_avg_accum_init(void *restrict agg_states, int n) { Int24AvgAccumState *states = (Int24AvgAccumState *) agg_states; for (int i = 0; i < n; i++) { states[i].count = 0; states[i].sum = 0; } } static void int24_avg_accum_emit(void *agg_state, Datum *out_result, bool *out_isnull) { Int24AvgAccumState *state = (Int24AvgAccumState *) agg_state; const size_t nbytes = 2 * sizeof(int64) + ARR_OVERHEAD_NONULLS(/* ndims = */ 1); ArrayType *result = palloc(nbytes); SET_VARSIZE(result, nbytes); result->ndim = 1; result->dataoffset = 0; result->elemtype = INT8OID; ARR_DIMS(result)[0] = 2; ARR_LBOUND(result)[0] = 1; /* * The array elements are stored by value, regardless of if the int8 itself * is by-value on this platform. */ ((int64 *) ARR_DATA_PTR(result))[0] = state->count; ((int64 *) ARR_DATA_PTR(result))[1] = state->sum; *out_result = PointerGetDatum(result); *out_isnull = false; } #endif #define AGG_NAME AVG #define PG_TYPE INT2 #define CTYPE int16 #define DATUM_TO_CTYPE DatumGetInt16 #include "int24_avg_accum_single.c" #define PG_TYPE INT4 #define CTYPE int32 #define DATUM_TO_CTYPE DatumGetInt32 #include "int24_avg_accum_single.c" #undef AGG_NAME ================================================ FILE: tsl/src/nodes/vector_agg/function/int24_sum_single.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #ifdef GENERATE_DISPATCH_TABLE extern VectorAggFunctions FUNCTION_NAME(argdef); case PG_AGG_OID_HELPER(AGG_NAME, PG_TYPE): return &FUNCTION_NAME(argdef); #else static pg_attribute_always_inline void FUNCTION_NAME(vector_impl)(void *agg_state, int n, const CTYPE *values, const uint64 *filter, MemoryContext agg_extra_mctx) { Int24SumState *state = (Int24SumState *) agg_state; /* * We accumulate the sum as int64, so we can sum INT_MAX = 2^31 - 1 * at least 2^31 times without incurring an overflow of the int64 * accumulator. The same is true for negative numbers. The * compressed batch size is currently capped at 1000 rows, but even * if it's changed in the future, it's unlikely that we support * batches larger than 65536 rows, not to mention 2^31. Therefore, * we don't need to check for overflows within the loop, which would * slow down the calculation. */ Assert(n <= INT_MAX); /* * Note that we use a simplest loop here, there are many possibilities of * optimizing this function (for example, this loop is not unrolled by * clang-16). */ int64 batch_sum = 0; bool have_result = false; for (int row = 0; row < n; row++) { const bool row_ok = arrow_row_is_valid(filter, row); batch_sum += values[row] * row_ok; have_result = have_result || row_ok; } if (unlikely(pg_add_s64_overflow(state->result, batch_sum, &state->result))) { ereport(ERROR, (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), errmsg("bigint out of range"))); } state->isvalid = state->isvalid || have_result; } static pg_attribute_always_inline void FUNCTION_NAME(one)(void *restrict agg_state, const CTYPE value) { Int24SumState *state = (Int24SumState *) agg_state; state->result += value; state->isvalid = true; } typedef Int24SumState FUNCTION_NAME(state); #include "agg_many_vector_helper.c" #include "agg_scalar_helper.c" #include "agg_vector_validity_helper.c" VectorAggFunctions FUNCTION_NAME(argdef) = { .state_bytes = sizeof(Int24SumState), .agg_init = int_sum_init, .agg_emit = int_sum_emit, .agg_scalar = FUNCTION_NAME(scalar), .agg_vector = FUNCTION_NAME(vector), .agg_many_vector = FUNCTION_NAME(many_vector), }; #endif #undef PG_TYPE #undef CTYPE #undef DATUM_TO_CTYPE #undef CTYPE_TO_DATUM ================================================ FILE: tsl/src/nodes/vector_agg/function/int24_sum_templates.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Common parts for vectorized sum(int). */ #include <limits.h> #include <postgres.h> #include <common/int.h> #include <utils/fmgroids.h> #include <utils/fmgrprotos.h> #include "functions.h" #include "template_helper.h" #include <compression/arrow_c_data_interface.h> #ifndef GENERATE_DISPATCH_TABLE typedef struct { int64 result; bool isvalid; } Int24SumState; static void int_sum_init(void *restrict agg_states, int n) { Int24SumState *states = (Int24SumState *) agg_states; for (int i = 0; i < n; i++) { states[i].result = 0; states[i].isvalid = false; } } static void int_sum_emit(void *agg_state, Datum *out_result, bool *out_isnull) { Int24SumState *state = (Int24SumState *) agg_state; *out_result = Int64GetDatum(state->result); *out_isnull = !state->isvalid; } #endif /* * Templated parts for vectorized sum(int). */ #define AGG_NAME SUM #define PG_TYPE INT4 #define CTYPE int32 #define DATUM_TO_CTYPE DatumGetInt32 #include "int24_sum_single.c" #define PG_TYPE INT2 #define CTYPE int16 #define DATUM_TO_CTYPE DatumGetInt16 #include "int24_sum_single.c" #undef AGG_NAME ================================================ FILE: tsl/src/nodes/vector_agg/function/minmax_arithmetic_single.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #ifdef GENERATE_DISPATCH_TABLE extern VectorAggFunctions FUNCTION_NAME(argdef); case PG_AGG_OID_HELPER(AGG_NAME, PG_TYPE): return &FUNCTION_NAME(argdef); #else static pg_attribute_always_inline void FUNCTION_NAME(vector_impl)(void *agg_state, int n, const CTYPE *values, const uint64 *filter, MemoryContext agg_extra_mctx) { MinMaxState *state = (MinMaxState *) agg_state; CTYPE outer_result = state->isvalid ? DATUM_TO_CTYPE(state->value) : 0; bool outer_isvalid = state->isvalid; for (int row = 0; row < n; row++) { const CTYPE new_value = values[row]; const bool new_value_ok = arrow_row_is_valid(filter, row); /* * Note that we have to properly handle NaNs and Infinities for floats. */ const bool do_replace = new_value_ok && (unlikely(!outer_isvalid) || PREDICATE(outer_result, new_value)); outer_result = do_replace ? new_value : outer_result; outer_isvalid = outer_isvalid || do_replace; } state->isvalid = outer_isvalid; /* Note that float8 Datum is by-reference on 32-bit systems. */ MemoryContext old = MemoryContextSwitchTo(agg_extra_mctx); state->value = CTYPE_TO_DATUM(outer_result); MemoryContextSwitchTo(old); } typedef MinMaxState FUNCTION_NAME(state); static pg_attribute_always_inline void FUNCTION_NAME(one)(void *restrict agg_state, const CTYPE value) { FUNCTION_NAME(state) *state = (FUNCTION_NAME(state) *) agg_state; if (!state->isvalid || PREDICATE(DATUM_TO_CTYPE(state->value), value) || isnan((double) value)) { /* * Note that float8 Datum is by-reference on 32-bit systems, and this * function is called in the extra aggregate data memory context. */ state->value = CTYPE_TO_DATUM(value); state->isvalid = true; } } #include "agg_many_vector_helper.c" #include "agg_scalar_helper.c" #include "agg_vector_validity_helper.c" VectorAggFunctions FUNCTION_NAME(argdef) = { .state_bytes = sizeof(MinMaxState), .agg_init = minmax_init, .agg_emit = minmax_emit, .agg_scalar = FUNCTION_NAME(scalar), .agg_vector = FUNCTION_NAME(vector), .agg_many_vector = FUNCTION_NAME(many_vector), }; #endif #undef PG_TYPE #undef CTYPE #undef DATUM_TO_CTYPE #undef CTYPE_TO_DATUM ================================================ FILE: tsl/src/nodes/vector_agg/function/minmax_arithmetic_types.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #define PG_TYPE INT2 #define CTYPE int16 #define DATUM_TO_CTYPE DatumGetInt16 #define CTYPE_TO_DATUM Int16GetDatum #include "minmax_arithmetic_single.c" #define PG_TYPE INT4 #define CTYPE int32 #define DATUM_TO_CTYPE DatumGetInt32 #define CTYPE_TO_DATUM Int32GetDatum #include "minmax_arithmetic_single.c" #define PG_TYPE INT8 #define CTYPE int64 #define DATUM_TO_CTYPE DatumGetInt64 #define CTYPE_TO_DATUM Int64GetDatum #include "minmax_arithmetic_single.c" #define PG_TYPE FLOAT4 #define CTYPE float #define DATUM_TO_CTYPE DatumGetFloat4 #define CTYPE_TO_DATUM Float4GetDatum #include "minmax_arithmetic_single.c" #define PG_TYPE FLOAT8 #define CTYPE double #define DATUM_TO_CTYPE DatumGetFloat8 #define CTYPE_TO_DATUM Float8GetDatum #include "minmax_arithmetic_single.c" #define PG_TYPE TIMESTAMP #define CTYPE Timestamp #define DATUM_TO_CTYPE DatumGetTimestamp #define CTYPE_TO_DATUM TimestampGetDatum #include "minmax_arithmetic_single.c" #define PG_TYPE TIMESTAMPTZ #define CTYPE TimestampTz #define DATUM_TO_CTYPE DatumGetTimestampTz #define CTYPE_TO_DATUM TimestampTzGetDatum #include "minmax_arithmetic_single.c" #define PG_TYPE DATE #define CTYPE DateADT #define DATUM_TO_CTYPE DatumGetDateADT #define CTYPE_TO_DATUM DateADTGetDatum #include "minmax_arithmetic_single.c" #undef PREDICATE #undef AGG_NAME ================================================ FILE: tsl/src/nodes/vector_agg/function/minmax_templates.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <port/pg_bitutils.h> #include <utils/date.h> #include <utils/fmgroids.h> #include <utils/fmgrprotos.h> #include <utils/timestamp.h> #include "functions.h" #include "template_helper.h" #include <compression/arrow_c_data_interface.h> #include "compat/compat.h" #if PG16_GE #include <varatt.h> #endif /* * Common parts for vectorized min(), max(). */ #ifndef GENERATE_DISPATCH_TABLE typedef struct { bool isvalid; Datum value; } MinMaxState; static void minmax_init(void *restrict agg_states, int n) { MinMaxState *states = (MinMaxState *) agg_states; for (int i = 0; i < n; i++) { states[i].isvalid = false; states[i].value = 0; } } static void minmax_emit(void *agg_state, Datum *out_result, bool *out_isnull) { MinMaxState *state = (MinMaxState *) agg_state; *out_result = state->value; *out_isnull = !state->isvalid; } #endif /* * Templated parts for vectorized min(), max(). * * NaN handled similar to equivalent PG functions. */ #define AGG_NAME MIN #define PREDICATE(CURRENT, NEW) \ (unlikely(!isnan((double) (NEW))) && (isnan((double) (CURRENT)) || (CURRENT) > (NEW))) #include "minmax_arithmetic_types.c" #define AGG_NAME MAX #define PREDICATE(CURRENT, NEW) \ (unlikely(!isnan((double) (CURRENT))) && (isnan((double) (NEW)) || (CURRENT) < (NEW))) #include "minmax_arithmetic_types.c" #ifndef GENERATE_DISPATCH_TABLE /* * Common parts for vectorized min(text), max(text). */ typedef struct { uint32 capacity; struct varlena *data; } MinMaxBytesState; typedef struct BytesView { const uint8 *data; uint32 len; } BytesView; static void minmax_bytes_init(void *restrict agg_states, int n) { MinMaxBytesState *restrict states = (MinMaxBytesState *) agg_states; for (int i = 0; i < n; i++) { states[i].capacity = 0; states[i].data = NULL; } } static void minmax_bytes_emit(void *agg_state, Datum *out_result, bool *out_isnull) { MinMaxBytesState *state = (MinMaxBytesState *) agg_state; *out_isnull = state->capacity == 0; *out_result = PointerGetDatum(state->data); } #endif /* * Templated parts for vectorized min(text), max(text). */ #define PG_TYPE TEXT #define AGG_NAME MIN #define PREDICATE(CURRENT, NEW) ((CURRENT) > (NEW)) #include "minmax_text.c" #define PG_TYPE TEXT #define AGG_NAME MAX #define PREDICATE(CURRENT, NEW) ((CURRENT) < (NEW)) #include "minmax_text.c" #undef AGG_NAME ================================================ FILE: tsl/src/nodes/vector_agg/function/minmax_text.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #ifdef GENERATE_DISPATCH_TABLE extern VectorAggFunctions FUNCTION_NAME(argdef); case PG_AGG_OID_HELPER(AGG_NAME, PG_TYPE): /* * Text min/max uses memcmp for comparison, which only produces correct * ordering for C collation. For non-C collations, return NULL to fall * back to the Postgres aggregation. */ if (lc_collate_is_c(collation)) return &FUNCTION_NAME(argdef); return NULL; #else typedef MinMaxBytesState FUNCTION_NAME(state); static pg_attribute_always_inline void FUNCTION_NAME(one)(void *restrict agg_state, const BytesView new_value) { FUNCTION_NAME(state) *restrict state = (FUNCTION_NAME(state) *) agg_state; /* * If current value is null, we replace it with the new value, otherwise we * have to check the predicate. */ bool replace = state->capacity == 0; if (likely(!replace)) { const uint32 current_len = VARSIZE(state->data) - VARHDRSZ; const int result = memcmp(VARDATA(state->data), new_value.data, Min(current_len, new_value.len)); if (result == 0) { replace = PREDICATE(current_len, new_value.len); } else { replace = PREDICATE(result, 0); } } if (replace) { const uint32 new_vardata_bytes = new_value.len + VARHDRSZ; if (new_vardata_bytes > state->capacity) { /* * Reallocate to closest power of two to amortize the costs. Varlena * is limited to 2^30 - 1 bytes. */ Assert(new_vardata_bytes < INT32_MAX / 2); const int lowest_power = pg_leftmost_one_pos32(new_vardata_bytes * 2 - 1); const int new_capacity = 1ULL << lowest_power; state->data = palloc(new_capacity); state->capacity = new_capacity; } SET_VARSIZE(state->data, new_vardata_bytes); memcpy(VARDATA(state->data), new_value.data, new_value.len); Assert(state->capacity > 0); Assert(VARSIZE(state->data) <= state->capacity); } } static void FUNCTION_NAME(vector)(void *agg_state, const ArrowArray *arrow, const uint64 *filter, MemoryContext agg_extra_mctx) { FUNCTION_NAME(state) *restrict state = (FUNCTION_NAME(state) *) agg_state; const int16 *body_offset_indexes = arrow->dictionary ? arrow->buffers[1] : NULL; const uint8 *bodies = arrow->dictionary ? arrow->dictionary->buffers[2] : arrow->buffers[2]; const uint32 *body_offsets = arrow->dictionary ? arrow->dictionary->buffers[1] : arrow->buffers[1]; const int n = arrow->length; MemoryContext old = MemoryContextSwitchTo(agg_extra_mctx); for (int row = 0; row < n; row++) { const int body_offset_index = body_offset_indexes == NULL ? row : body_offset_indexes[row]; const int body_offset = body_offsets[body_offset_index]; const int body_bytes = body_offsets[body_offset_index + 1] - body_offset; const BytesView value = { .data = &bodies[body_offset], .len = body_bytes }; if (arrow_row_is_valid(filter, row)) { FUNCTION_NAME(one)(state, value); } } MemoryContextSwitchTo(old); } static void FUNCTION_NAME(many_vector)(void *restrict agg_states, const uint32 *state_indices, const uint64 *filter, int start_row, int end_row, const ArrowArray *arrow, MemoryContext agg_extra_mctx) { FUNCTION_NAME(state) *restrict states = (FUNCTION_NAME(state) *) agg_states; const int16 *body_offset_indexes = arrow->dictionary ? arrow->buffers[1] : NULL; const uint8 *bodies = arrow->dictionary ? arrow->dictionary->buffers[2] : arrow->buffers[2]; const uint32 *body_offsets = arrow->dictionary ? arrow->dictionary->buffers[1] : arrow->buffers[1]; MemoryContext old = MemoryContextSwitchTo(agg_extra_mctx); for (int row = start_row; row < end_row; row++) { FUNCTION_NAME(state) *restrict state = &states[state_indices[row]]; const int body_offset_index = body_offset_indexes == NULL ? row : body_offset_indexes[row]; const int body_offset = body_offsets[body_offset_index]; const int body_bytes = body_offsets[body_offset_index + 1] - body_offset; const BytesView value = { .data = &bodies[body_offset], .len = body_bytes }; if (arrow_row_is_valid(filter, row)) { Assert(state_indices[row] != 0); FUNCTION_NAME(one)(state, value); } } MemoryContextSwitchTo(old); } static void FUNCTION_NAME(scalar)(void *agg_state, Datum constvalue, bool constisnull, int n, MemoryContext agg_extra_mctx) { if (constisnull) { return; } BytesView value = { .data = (const uint8 *) VARDATA_ANY(constvalue), .len = VARSIZE_ANY_EXHDR(constvalue) }; MemoryContext old = MemoryContextSwitchTo(agg_extra_mctx); FUNCTION_NAME(one)(agg_state, value); MemoryContextSwitchTo(old); } VectorAggFunctions FUNCTION_NAME(argdef) = { .state_bytes = sizeof(MinMaxBytesState), .agg_init = minmax_bytes_init, .agg_emit = minmax_bytes_emit, .agg_scalar = FUNCTION_NAME(scalar), .agg_vector = FUNCTION_NAME(vector), .agg_many_vector = FUNCTION_NAME(many_vector), }; #endif #undef PG_TYPE #undef CTYPE #undef DATUM_TO_CTYPE #undef CTYPE_TO_DATUM #undef PREDICATE #undef AGG_NAME ================================================ FILE: tsl/src/nodes/vector_agg/function/sum_float_single.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #ifdef GENERATE_DISPATCH_TABLE extern VectorAggFunctions FUNCTION_NAME(argdef); case PG_AGG_OID_HELPER(AGG_NAME, PG_TYPE): return &FUNCTION_NAME(argdef); #else StaticAssertDecl(sizeof(CTYPE) == sizeof(MASKTYPE), "CTYPE and MASKTYPE must be the same size"); static void FUNCTION_NAME(emit)(void *agg_state, Datum *out_result, bool *out_isnull) { FloatSumState *state = (FloatSumState *) agg_state; const CTYPE result_casted = state->result; *out_result = CTYPE_TO_DATUM(result_casted); *out_isnull = !state->isvalid; } static pg_attribute_always_inline void FUNCTION_NAME(vector_impl)(void *agg_state, int n, const CTYPE *values, const uint64 *filter, MemoryContext agg_extra_mctx) { /* * Vector registers can be up to 512 bits wide. */ #define UNROLL_SIZE ((int) (512 / 8 / sizeof(CTYPE))) bool have_result_accu[UNROLL_SIZE] = { 0 }; double sum_accu[UNROLL_SIZE] = { 0 }; for (int outer = 0; outer < UNROLL_SIZE * (n / UNROLL_SIZE); outer += UNROLL_SIZE) { for (int inner = 0; inner < UNROLL_SIZE; inner++) { const int row = outer + inner; double *dest = &sum_accu[inner]; bool *have_result = &have_result_accu[inner]; /* * We're using a trick with bitmasking the numbers that don't * pass the filter, to allow for branchless code generation. This is * analogous to integer version where we just multiply the integers * by bool, but for floats we can't use multiplication because of * infinities and NaNs. */ #define INNER_LOOP \ const bool row_valid = arrow_row_is_valid(filter, row); \ union \ { \ CTYPE f; \ MASKTYPE m; \ } u = { .f = values[row] }; \ u.m &= row_valid ? ~(MASKTYPE) 0 : (MASKTYPE) 0; \ *dest += u.f; \ *have_result = *have_result || row_valid; INNER_LOOP } } for (int row = UNROLL_SIZE * (n / UNROLL_SIZE); row < n; row++) { double *dest = &sum_accu[0]; bool *have_result = &have_result_accu[0]; INNER_LOOP } for (int i = 1; i < UNROLL_SIZE; i++) { sum_accu[0] += sum_accu[i]; have_result_accu[0] = have_result_accu[0] || have_result_accu[i]; } #undef UNROLL_SIZE #undef INNER_LOOP FloatSumState *state = (FloatSumState *) agg_state; state->isvalid = state->isvalid || have_result_accu[0]; state->result += sum_accu[0]; } typedef FloatSumState FUNCTION_NAME(state); static pg_attribute_always_inline void FUNCTION_NAME(one)(void *restrict agg_state, const CTYPE value) { FUNCTION_NAME(state) *state = (FUNCTION_NAME(state) *) agg_state; state->isvalid = true; state->result += value; } #include "agg_many_vector_helper.c" #include "agg_scalar_helper.c" #include "agg_vector_validity_helper.c" VectorAggFunctions FUNCTION_NAME(argdef) = { .state_bytes = sizeof(FloatSumState), .agg_init = float_sum_init, .agg_emit = FUNCTION_NAME(emit), .agg_scalar = FUNCTION_NAME(scalar), .agg_vector = FUNCTION_NAME(vector), .agg_many_vector = FUNCTION_NAME(many_vector), }; #endif #undef PG_TYPE #undef CTYPE #undef MASKTYPE #undef DATUM_TO_CTYPE #undef CTYPE_TO_DATUM ================================================ FILE: tsl/src/nodes/vector_agg/function/sum_float_templates.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Common parts for vectorized sum(float). */ #include <postgres.h> #include "functions.h" #include "template_helper.h" #include <compression/arrow_c_data_interface.h> #include <utils/fmgroids.h> #include <utils/fmgrprotos.h> #ifndef GENERATE_DISPATCH_TABLE typedef struct { double result; bool isvalid; } FloatSumState; static void float_sum_init(void *restrict agg_states, int n) { FloatSumState *states = (FloatSumState *) agg_states; for (int i = 0; i < n; i++) { states[i].result = 0; states[i].isvalid = false; } } #endif /* * Templated parts for vectorized sum(float). */ #define AGG_NAME SUM #define PG_TYPE FLOAT4 #define CTYPE float #define MASKTYPE uint32 #define CTYPE_TO_DATUM Float4GetDatum #define DATUM_TO_CTYPE DatumGetFloat4 #include "sum_float_single.c" #define PG_TYPE FLOAT8 #define CTYPE double #define MASKTYPE uint64 #define CTYPE_TO_DATUM Float8GetDatum #define DATUM_TO_CTYPE DatumGetFloat8 #include "sum_float_single.c" #undef AGG_NAME ================================================ FILE: tsl/src/nodes/vector_agg/function/template_helper.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #define PG_AGG_OID(AGG_NAME, ARGUMENT_TYPE) F_##AGG_NAME##_##ARGUMENT_TYPE #define PG_AGG_OID_HELPER(X, Y) PG_AGG_OID(X, Y) #define FUNCTION_NAME_HELPER2(X, Y, Z) X##_##Y##_##Z #define FUNCTION_NAME_HELPER(X, Y, Z) FUNCTION_NAME_HELPER2(X, Y, Z) #define FUNCTION_NAME(Z) FUNCTION_NAME_HELPER(AGG_NAME, PG_TYPE, Z) ================================================ FILE: tsl/src/nodes/vector_agg/grouping_policy.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <executor/tuptable.h> typedef struct GroupingPolicy GroupingPolicy; typedef struct DecompressContext DecompressContext; typedef struct TupleTableSlot TupleTableSlot; typedef struct VectorAggDef VectorAggDef; typedef struct GroupingColumn GroupingColumn; /* * This is a common interface for grouping policies which define how the rows * are grouped for aggregation -- e.g. there can be an implementation for no * grouping, grouping by compression segmentby columns, grouping over sorted * input (GroupAggregate), grouping using a hash table, and so on. */ typedef struct GroupingPolicy { /* * Used for rescans in the Postgres sense. */ void (*gp_reset)(GroupingPolicy *gp); /* * Aggregate a single compressed batch. */ void (*gp_add_batch)(GroupingPolicy *gp, DecompressContext *dcontext, TupleTableSlot *vector_slot); /* * Is a partial aggregation result ready? */ bool (*gp_should_emit)(GroupingPolicy *gp); /* * Emit a partial aggregation result into the result slot. */ bool (*gp_do_emit)(GroupingPolicy *gp, TupleTableSlot *aggregated_slot); /* * Destroy the grouping policy. */ void (*gp_destroy)(GroupingPolicy *gp); /* * Description of this grouping policy for the EXPLAIN output. */ char *(*gp_explain)(GroupingPolicy *gp); } GroupingPolicy; /* * The various types of grouping we might use, as determined at planning time. * The hashed subtypes are all implemented by hash grouping policy. */ typedef enum { VAGT_Invalid, VAGT_Batch, VAGT_HashSingleFixed2, VAGT_HashSingleFixed4, VAGT_HashSingleFixed8, VAGT_HashSingleText, VAGT_HashSerialized, } VectorAggGroupingType; extern GroupingPolicy *create_grouping_policy_batch(int num_agg_defs, VectorAggDef *agg_defs, int num_grouping_columns, GroupingColumn *grouping_columns); extern GroupingPolicy *create_grouping_policy_hash(int num_agg_defs, VectorAggDef *agg_defs, int num_grouping_columns, GroupingColumn *grouping_columns, VectorAggGroupingType grouping_type); ================================================ FILE: tsl/src/nodes/vector_agg/grouping_policy_batch.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * This grouping policy aggregates entire compressed batches. It can be used to * aggregate with no grouping, or to produce partial aggregates per each batch * to group by segmentby columns. */ #include <postgres.h> #include <access/attnum.h> #include <executor/tuptable.h> #include <nodes/pg_list.h> #include "grouping_policy.h" #include "nodes/vector_agg/exec.h" #include "nodes/vector_agg/vector_slot.h" typedef struct { GroupingPolicy funcs; int num_agg_defs; VectorAggDef *agg_defs; /* * Temporary storage for combined bitmap of batch filter and aggregate * argument validity. */ uint64 *tmp_filter; uint64 num_tmp_filter_words; void **agg_states; int num_grouping_columns; GroupingColumn *grouping_columns; Datum *output_grouping_values; bool *output_grouping_isnull; bool have_results; /* * A memory context for aggregate functions to allocate additional data, * i.e. if they store strings or float8 datum on 32-bit systems, or they * have variable-length state like the exact distinct function or the * statistical sketches. * Valid until the grouping policy is reset. */ MemoryContext agg_extra_mctx; } GroupingPolicyBatch; static const GroupingPolicy grouping_policy_batch_functions; GroupingPolicy * create_grouping_policy_batch(int num_agg_defs, VectorAggDef *agg_defs, int num_grouping_columns, GroupingColumn *output_grouping_columns) { GroupingPolicyBatch *policy = palloc0(sizeof(GroupingPolicyBatch)); policy->funcs = grouping_policy_batch_functions; policy->num_grouping_columns = num_grouping_columns; policy->grouping_columns = output_grouping_columns; policy->num_agg_defs = num_agg_defs; policy->agg_defs = agg_defs; policy->agg_extra_mctx = AllocSetContextCreate(CurrentMemoryContext, "agg extra", ALLOCSET_DEFAULT_SIZES); policy->agg_states = (void **) palloc(sizeof(*policy->agg_states) * policy->num_agg_defs); for (int i = 0; i < policy->num_agg_defs; i++) { VectorAggDef *agg_def = &policy->agg_defs[i]; policy->agg_states[i] = palloc(agg_def->func.state_bytes); } policy->output_grouping_values = (Datum *) palloc0(MAXALIGN(num_grouping_columns * sizeof(Datum)) + MAXALIGN(num_grouping_columns * sizeof(bool))); policy->output_grouping_isnull = (bool *) ((char *) policy->output_grouping_values + MAXALIGN(num_grouping_columns * sizeof(Datum))); return &policy->funcs; } static void gp_batch_reset(GroupingPolicy *obj) { GroupingPolicyBatch *policy = (GroupingPolicyBatch *) obj; MemoryContextReset(policy->agg_extra_mctx); const int naggs = policy->num_agg_defs; for (int i = 0; i < naggs; i++) { VectorAggDef *agg_def = &policy->agg_defs[i]; void *agg_state = policy->agg_states[i]; agg_def->func.agg_init(agg_state, 1); } const int ngrp = policy->num_grouping_columns; for (int i = 0; i < ngrp; i++) { policy->output_grouping_values[i] = 0; policy->output_grouping_isnull[i] = true; } policy->have_results = false; } static void compute_single_aggregate(GroupingPolicyBatch *policy, DecompressContext *dcontext, TupleTableSlot *vector_slot, VectorAggDef *agg_def, void *agg_state, MemoryContext agg_extra_mctx) { /* * We have functions with one argument, and one function with no arguments * (count(*)). Collect the arguments. */ const ArrowArray *arg_arrow = NULL; const uint64 *arg_validity_bitmap = NULL; Datum arg_datum = 0; bool arg_isnull = true; if (agg_def->argument != NULL) { const CompressedColumnValues values = vector_slot_evaluate_expression(dcontext, vector_slot, agg_def->effective_batch_filter, agg_def->argument); Assert(values.decompression_type != DT_Invalid); Ensure(values.decompression_type != DT_Iterator, "expected arrow array but got iterator"); if (values.arrow != NULL) { arg_arrow = values.arrow; arg_validity_bitmap = values.buffers[0]; } else { Assert(values.decompression_type == DT_Scalar); arg_isnull = DatumGetBool(PointerGetDatum(values.buffers[0])); arg_datum = PointerGetDatum(values.buffers[1]); } } /* * Compute the combined validity bitmap that includes the argument validity. */ DecompressBatchState *batch_state = (DecompressBatchState *) vector_slot; const size_t num_words = (batch_state->total_batch_rows + 63) / 64; const uint64 *combined_validity = arrow_combine_validity(num_words, policy->tmp_filter, agg_def->effective_batch_filter, arg_validity_bitmap, NULL); /* * Now call the function. */ if (arg_arrow != NULL) { /* Arrow argument. */ agg_def->func.agg_vector(agg_state, arg_arrow, combined_validity, agg_extra_mctx); } else { /* * Scalar argument, or count(*). Have to also count the valid rows in * the batch. * * The batches that are fully filtered out by vectorized quals should * have been skipped by the caller, but we also have to check for the * case when no rows match the aggregate FILTER clause. */ const int n = arrow_num_valid(combined_validity, batch_state->total_batch_rows); if (n > 0) { agg_def->func.agg_scalar(agg_state, arg_datum, arg_isnull, n, agg_extra_mctx); } } } static void gp_batch_add_batch(GroupingPolicy *gp, DecompressContext *dcontext, TupleTableSlot *vector_slot) { GroupingPolicyBatch *policy = (GroupingPolicyBatch *) gp; uint16 total_batch_rows = 0; const uint64 *vector_qual_result = vector_slot_get_qual_result(vector_slot, &total_batch_rows); /* * Allocate the temporary filter array for computing the combined results of * batch filter, aggregate filter and column validity. */ const size_t num_words = (total_batch_rows + 63) / 64; if (num_words > policy->num_tmp_filter_words) { const size_t new_words = (num_words * 2) + 1; if (policy->tmp_filter != NULL) { pfree(policy->tmp_filter); } policy->tmp_filter = palloc(sizeof(*policy->tmp_filter) * new_words); policy->num_tmp_filter_words = new_words; } /* * Compute the aggregates. */ const int naggs = policy->num_agg_defs; for (int i = 0; i < naggs; i++) { VectorAggDef *agg_def = &policy->agg_defs[i]; void *agg_state = policy->agg_states[i]; compute_single_aggregate(policy, dcontext, vector_slot, agg_def, agg_state, policy->agg_extra_mctx); } /* * Save the values of the grouping columns. */ const int ngrp = policy->num_grouping_columns; for (int i = 0; i < ngrp; i++) { GroupingColumn *col = &policy->grouping_columns[i]; Assert(col->output_offset >= 0); const CompressedColumnValues values = vector_slot_evaluate_expression(dcontext, vector_slot, vector_qual_result, col->expr); Assert(values.decompression_type == DT_Scalar); /* * By sheer luck, we can avoid generically copying the Datum here, * because if we have any output grouping columns in this policy, it * means we're grouping by segmentby, and these values will be valid * until the next call to the vector agg node. */ policy->output_grouping_values[i] = PointerGetDatum(values.buffers[1]); policy->output_grouping_isnull[i] = DatumGetBool(PointerGetDatum(values.buffers[0])); } policy->have_results = true; } static bool gp_batch_should_emit(GroupingPolicy *gp) { GroupingPolicyBatch *policy = (GroupingPolicyBatch *) gp; /* * If we're grouping by segmentby columns, we have to output partials for * every batch. */ return policy->num_grouping_columns > 0 && policy->have_results; } static bool gp_batch_do_emit(GroupingPolicy *gp, TupleTableSlot *aggregated_slot) { GroupingPolicyBatch *policy = (GroupingPolicyBatch *) gp; if (!policy->have_results) { return false; } const int naggs = policy->num_agg_defs; for (int i = 0; i < naggs; i++) { VectorAggDef *agg_def = &policy->agg_defs[i]; void *agg_state = policy->agg_states[i]; agg_def->func.agg_emit(agg_state, &aggregated_slot->tts_values[agg_def->output_offset], &aggregated_slot->tts_isnull[agg_def->output_offset]); } const int ngrp = policy->num_grouping_columns; for (int i = 0; i < ngrp; i++) { GroupingColumn *col = &policy->grouping_columns[i]; Assert(col->output_offset >= 0); aggregated_slot->tts_values[col->output_offset] = policy->output_grouping_values[i]; aggregated_slot->tts_isnull[col->output_offset] = policy->output_grouping_isnull[i]; } /* * We only have one partial aggregation result for this policy. */ policy->have_results = false; return true; } static char * gp_batch_explain(GroupingPolicy *gp) { GroupingPolicyBatch *policy = (GroupingPolicyBatch *) gp; /* * If we're grouping by segmentby columns, we have to output partials for * every batch. */ return policy->num_grouping_columns > 0 ? "per compressed batch" : "all compressed batches"; } static const GroupingPolicy grouping_policy_batch_functions = { .gp_reset = gp_batch_reset, .gp_add_batch = gp_batch_add_batch, .gp_should_emit = gp_batch_should_emit, .gp_do_emit = gp_batch_do_emit, .gp_explain = gp_batch_explain, }; ================================================ FILE: tsl/src/nodes/vector_agg/grouping_policy_hash.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * This grouping policy groups the rows using a hash table. Currently it only * supports a single fixed-size by-value compressed column that fits into a Datum. */ #include <postgres.h> #include <access/attnum.h> #include <access/tupdesc.h> #include <executor/tuptable.h> #include <nodes/pg_list.h> #include "grouping_policy.h" #include "guc.h" #include "nodes/vector_agg/exec.h" #include "nodes/vector_agg/filter_word_iterator.h" #include "nodes/vector_agg/vector_slot.h" #include "grouping_policy_hash.h" #ifdef USE_FLOAT8_BYVAL #define DEBUG_LOG(MSG, ...) elog(DEBUG3, MSG, __VA_ARGS__) #else /* * On 32-bit platforms we'd have to use the cross-platform int width printf * specifiers which are really unreadable. */ #define DEBUG_LOG(...) #endif extern HashingStrategy single_fixed_2_strategy; extern HashingStrategy single_fixed_4_strategy; extern HashingStrategy single_fixed_8_strategy; #ifdef TS_USE_UMASH extern HashingStrategy single_text_strategy; extern HashingStrategy serialized_strategy; #endif static const GroupingPolicy grouping_policy_hash_functions; GroupingPolicy * create_grouping_policy_hash(int num_agg_defs, VectorAggDef *agg_defs, int num_grouping_columns, GroupingColumn *grouping_columns, VectorAggGroupingType grouping_type) { GroupingPolicyHash *policy = palloc0(sizeof(GroupingPolicyHash)); policy->funcs = grouping_policy_hash_functions; policy->num_grouping_columns = num_grouping_columns; policy->grouping_columns = grouping_columns; policy->agg_extra_mctx = AllocSetContextCreate(CurrentMemoryContext, "agg extra", ALLOCSET_DEFAULT_SIZES); /* * This should match the expected grouping cardinality. Here we use a value * noticeably lower than the batch size, so that the reallocation logic is * triggered in more cases and better tested. */ policy->num_allocated_per_key_agg_states = 300; policy->num_agg_defs = num_agg_defs; policy->agg_defs = agg_defs; policy->per_agg_per_key_states = (void **) palloc(sizeof(*policy->per_agg_per_key_states) * policy->num_agg_defs); for (int i = 0; i < policy->num_agg_defs; i++) { const VectorAggDef *agg_def = &policy->agg_defs[i]; policy->per_agg_per_key_states[i] = palloc(agg_def->func.state_bytes * policy->num_allocated_per_key_agg_states); } policy->current_batch_grouping_column_values = palloc(sizeof(CompressedColumnValues) * num_grouping_columns); switch (grouping_type) { #ifdef TS_USE_UMASH case VAGT_HashSerialized: policy->hashing = serialized_strategy; break; case VAGT_HashSingleText: policy->hashing = single_text_strategy; break; #endif case VAGT_HashSingleFixed8: policy->hashing = single_fixed_8_strategy; break; case VAGT_HashSingleFixed4: policy->hashing = single_fixed_4_strategy; break; case VAGT_HashSingleFixed2: policy->hashing = single_fixed_2_strategy; break; default: Ensure(false, "failed to determine the hashing strategy"); break; } policy->hashing.key_body_mctx = policy->agg_extra_mctx; policy->hashing.init(&policy->hashing, policy); return &policy->funcs; } static void gp_hash_reset(GroupingPolicy *obj) { GroupingPolicyHash *policy = (GroupingPolicyHash *) obj; MemoryContextReset(policy->agg_extra_mctx); policy->returning_results = false; policy->hashing.reset(&policy->hashing); policy->stat_input_valid_rows = 0; policy->stat_input_total_rows = 0; policy->stat_bulk_filtered_rows = 0; policy->stat_consecutive_keys = 0; } static void compute_single_aggregate(GroupingPolicyHash *policy, DecompressContext *dcontext, TupleTableSlot *vector_slot, const VectorAggDef *agg_def, void *agg_states) { const uint32 *offsets = policy->key_index_for_row; MemoryContext agg_extra_mctx = policy->agg_extra_mctx; /* * We have functions with one argument, and one function with no arguments * (count(*)). Collect the arguments. */ const ArrowArray *arg_arrow = NULL; const uint64 *arg_validity_bitmap = NULL; Datum arg_datum = 0; bool arg_isnull = true; if (agg_def->argument != NULL) { const CompressedColumnValues values = vector_slot_evaluate_expression(dcontext, vector_slot, agg_def->effective_batch_filter, agg_def->argument); Assert(values.decompression_type != DT_Invalid); Ensure(values.decompression_type != DT_Iterator, "expected arrow array but got iterator"); if (values.arrow != NULL) { arg_arrow = values.arrow; arg_validity_bitmap = values.buffers[0]; } else { Assert(values.decompression_type == DT_Scalar); arg_isnull = DatumGetBool(PointerGetDatum(values.buffers[0])); arg_datum = PointerGetDatum(values.buffers[1]); } } /* * Compute the combined validity bitmap that includes the argument validity. */ DecompressBatchState *batch_state = (DecompressBatchState *) vector_slot; const size_t num_words = (batch_state->total_batch_rows + 63) / 64; const uint64 *combined_validity = arrow_combine_validity(num_words, policy->tmp_filter, agg_def->effective_batch_filter, arg_validity_bitmap, NULL); /* * Now call the function, skipping the sequences of rows that didn't pass * the filters. */ for (FilterWordIterator iter = filter_word_iterator_init(batch_state->total_batch_rows, combined_validity); filter_word_iterator_is_valid(&iter); filter_word_iterator_advance(&iter)) { if (arg_arrow != NULL) { /* Arrow argument. */ agg_def->func.agg_many_vector(agg_states, offsets, combined_validity, iter.start_row, iter.end_row, arg_arrow, agg_extra_mctx); } else { /* * Scalar argument, or count(*). The latter has an optimized * implementation. */ if (agg_def->func.agg_many_scalar != NULL) { agg_def->func.agg_many_scalar(agg_states, offsets, combined_validity, iter.start_row, iter.end_row, arg_datum, arg_isnull, agg_extra_mctx); } else { for (int i = iter.start_row; i < iter.end_row; i++) { if (!arrow_row_is_valid(combined_validity, i)) { continue; } void *state = (offsets[i] * agg_def->func.state_bytes + (char *) agg_states); agg_def->func.agg_scalar(state, arg_datum, arg_isnull, 1, agg_extra_mctx); } } } } } static void gp_hash_add_batch(GroupingPolicy *gp, DecompressContext *dcontext, TupleTableSlot *vector_slot) { GroupingPolicyHash *policy = (GroupingPolicyHash *) gp; uint16 nrows; const uint64 *restrict filter = vector_slot_get_qual_result(vector_slot, &nrows); Assert(!policy->returning_results); /* * Initialize the array for storing the aggregate state offsets corresponding * to a given batch row. We don't need the offsets for the previous batch * that are currently stored there, so we don't need to use repalloc. */ if ((size_t) nrows > policy->num_key_index_for_row) { if (policy->key_index_for_row != NULL) { pfree(policy->key_index_for_row); } policy->num_key_index_for_row = nrows; policy->key_index_for_row = palloc(sizeof(policy->key_index_for_row[0]) * policy->num_key_index_for_row); } memset(policy->key_index_for_row, 0, nrows * sizeof(policy->key_index_for_row[0])); /* * Allocate the temporary filter array for computing the combined results of * batch filter, aggregate filter and column validity. */ const size_t num_words = (nrows + 63) / 64; if (num_words > policy->num_tmp_filter_words) { policy->tmp_filter = palloc(sizeof(*policy->tmp_filter) * (num_words * 2 + 1)); policy->num_tmp_filter_words = (num_words * 2 + 1); } /* * Arrange the input compressed columns in the order of grouping columns. */ for (int i = 0; i < policy->num_grouping_columns; i++) { const GroupingColumn *def = &policy->grouping_columns[i]; policy->current_batch_grouping_column_values[i] = vector_slot_evaluate_expression(dcontext, vector_slot, filter, def->expr); } /* * Call the per-batch initialization function of the hashing strategy. */ policy->hashing.prepare_for_batch(policy, vector_slot); /* * Remember which grouping keys have already existed, and which we * have to initialize. State index zero is invalid. */ const uint32 last_initialized_key_index = policy->hashing.last_used_key_index; Assert(last_initialized_key_index <= policy->num_allocated_per_key_agg_states); /* * Add the grouping keys to the hash table, skipping the sequences * of rows that are filtered out by the batch filter. */ int stats_matched_rows = 0; for (FilterWordIterator iter = filter_word_iterator_init(nrows, filter); filter_word_iterator_is_valid(&iter); filter_word_iterator_advance(&iter)) { stats_matched_rows += iter.end_row - iter.start_row; Assert((size_t) iter.end_row <= policy->num_key_index_for_row); policy->hashing.fill_offsets(policy, vector_slot, iter.start_row, iter.end_row); } policy->stat_input_total_rows += nrows; policy->stat_input_valid_rows += arrow_num_valid(filter, nrows); policy->stat_bulk_filtered_rows += nrows - stats_matched_rows; /* * Process the aggregate function states. We are processing single aggregate * function for the entire batch to improve the memory locality. */ const uint64 new_aggstate_rows = Max(policy->hashing.last_used_key_index + 1, policy->num_allocated_per_key_agg_states * 2 + 1); const int num_fns = policy->num_agg_defs; for (int agg_index = 0; agg_index < num_fns; agg_index++) { const VectorAggDef *agg_def = &policy->agg_defs[agg_index]; /* * If we added new keys for this batch, initialize the states for these * keys for this aggregate function. */ if (policy->hashing.last_used_key_index > last_initialized_key_index) { /* * If the aggregate function states don't fit into the existing * storage, reallocate it. We will record the allocated size later, * and before that, the allocation needs to be done for every * aggregate function. */ if (policy->hashing.last_used_key_index >= policy->num_allocated_per_key_agg_states) { policy->per_agg_per_key_states[agg_index] = repalloc(policy->per_agg_per_key_states[agg_index], new_aggstate_rows * agg_def->func.state_bytes); } void *first_uninitialized_state = agg_def->func.state_bytes * (last_initialized_key_index + 1) + (char *) policy->per_agg_per_key_states[agg_index]; agg_def->func.agg_init(first_uninitialized_state, policy->hashing.last_used_key_index - last_initialized_key_index); } /* * Add this batch to the states of this aggregate function. */ compute_single_aggregate(policy, dcontext, vector_slot, agg_def, policy->per_agg_per_key_states[agg_index]); } /* * If we got new grouping keys in this batch, this means we had to * reallocate the aggregate function states for them, and now have to record * the new allocated size. */ if (policy->hashing.last_used_key_index >= policy->num_allocated_per_key_agg_states) { Assert(new_aggstate_rows > policy->num_allocated_per_key_agg_states); policy->num_allocated_per_key_agg_states = new_aggstate_rows; } } static bool gp_hash_should_emit(GroupingPolicy *gp) { GroupingPolicyHash *policy = (GroupingPolicyHash *) gp; if (policy->hashing.last_used_key_index > UINT32_MAX - GLOBAL_MAX_ROWS_PER_COMPRESSION) { /* * The max valid key index is UINT32_MAX, so we have to spill if the next * batch can possibly lead to key index overflow. */ return true; } /* * Don't grow the hash table cardinality too much, otherwise we become bound * by memory reads. In general, when this first stage of grouping doesn't * significantly reduce the cardinality, it becomes pure overhead and the * work will be done by the final Postgres aggregation, so we should bail * out early here. */ return policy->hashing.get_size_bytes(&policy->hashing) > 512 * 1024; } static bool gp_hash_do_emit(GroupingPolicy *gp, TupleTableSlot *aggregated_slot) { GroupingPolicyHash *policy = (GroupingPolicyHash *) gp; if (!policy->returning_results) { policy->returning_results = true; policy->last_returned_key = 1; const float keys = policy->hashing.last_used_key_index; if (keys > 0) { DEBUG_LOG("spill after " UINT64_FORMAT " input, " UINT64_FORMAT " valid, " UINT64_FORMAT " bulk filtered, " UINT64_FORMAT " cons, %.0f keys, " "%f ratio, %ld curctx bytes, %ld aggstate bytes", policy->stat_input_total_rows, policy->stat_input_valid_rows, policy->stat_bulk_filtered_rows, policy->stat_consecutive_keys, keys, policy->stat_input_valid_rows / keys, MemoryContextMemAllocated(CurrentMemoryContext, false), MemoryContextMemAllocated(policy->agg_extra_mctx, false)); } } else { policy->last_returned_key++; } const uint32 current_key = policy->last_returned_key; const uint32 keys_end = policy->hashing.last_used_key_index + 1; if (current_key >= keys_end) { policy->returning_results = false; return false; } const int naggs = policy->num_agg_defs; for (int i = 0; i < naggs; i++) { const VectorAggDef *agg_def = &policy->agg_defs[i]; void *agg_states = policy->per_agg_per_key_states[i]; void *agg_state = current_key * agg_def->func.state_bytes + (char *) agg_states; agg_def->func.agg_emit(agg_state, &aggregated_slot->tts_values[agg_def->output_offset], &aggregated_slot->tts_isnull[agg_def->output_offset]); } policy->hashing.emit_key(policy, current_key, aggregated_slot); DEBUG_PRINT("%p: output key index %d\n", policy, current_key); return true; } static char * gp_hash_explain(GroupingPolicy *gp) { GroupingPolicyHash *policy = (GroupingPolicyHash *) gp; return psprintf("hashed with %s key", policy->hashing.explain_name); } static const GroupingPolicy grouping_policy_hash_functions = { .gp_reset = gp_hash_reset, .gp_add_batch = gp_hash_add_batch, .gp_should_emit = gp_hash_should_emit, .gp_do_emit = gp_hash_do_emit, .gp_explain = gp_hash_explain, }; ================================================ FILE: tsl/src/nodes/vector_agg/grouping_policy_hash.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <nodes/pg_list.h> #include "grouping_policy.h" #include "nodes/columnar_scan/compressed_batch.h" #include "hashing/hashing_strategy.h" typedef struct GroupingPolicyHash GroupingPolicyHash; /* * Hash grouping policy. * * The grouping and aggregation is performed as follows: * * 0) The grouping policy keeps track of the unique grouping keys seen in * the input rows, and the states of aggregate functions for each key. This * spans multiple input compressed batches, and is reset after the partial * aggregation results are emitted. * * 1) For each row of the new compressed batch, we obtain an index that * uniquely identifies its grouping key. This is done by matching the row's * grouping columns to the hash table recording the unique grouping keys and * their respective indexes. It is performed in bulk for all rows of the batch, * to improve memory locality. The details of this are managed by the hashing * strategy. * * 2) The key indexes are used to locate the aggregate function states * corresponding to a given row's key, and update it. This is done in bulk for all * rows of the batch, and for each aggregate function separately, to generate * simpler and potentially vectorizable code, and improve memory locality. * * 3) After the input has ended, or if the memory limit is reached, the partial * results are emitted into the output slot. This is done in the order of unique * grouping key indexes, thereby preserving the incoming key order. This * guarantees that this policy works correctly even in a Partial GroupAggregate * node, even though it's not optimal performance-wise. We only support the * direct order of records in batch though, not reverse. This is checked at * planning time. */ typedef struct GroupingPolicyHash { /* * We're using data inheritance from the GroupingPolicy. */ GroupingPolicy funcs; /* * Aggregate function definitions. */ int num_agg_defs; const VectorAggDef *restrict agg_defs; /* * Grouping column definitions. */ int num_grouping_columns; const GroupingColumn *restrict grouping_columns; /* * The values of the grouping columns picked from the compressed batch and * arranged in the order of grouping column definitions. */ CompressedColumnValues *restrict current_batch_grouping_column_values; /* * Hashing strategy that is responsible for mapping the rows to the unique * indexes of their grouping keys. */ HashingStrategy hashing; /* * Temporary storage of unique indexes of keys corresponding to a given row * of the compressed batch that is currently being aggregated. We keep it in * the policy because it is potentially too big to keep on stack, and we * don't want to reallocate it for each batch. */ uint32 *restrict key_index_for_row; uint64 num_key_index_for_row; /* * The temporary filter bitmap we use to combine the results of the * vectorized filters in WHERE, validity of the aggregate function argument, * and the aggregate FILTER clause. It is then used by the aggregate * function implementation to filter out the rows that don't pass. */ uint64 *tmp_filter; uint64 num_tmp_filter_words; /* * Aggregate function states. Each element is an array of states for the * respective function from agg_defs. These arrays are indexed by the unique * grouping key indexes. The key index 0 is invalid, so the corresponding * states are unused. * The states of each aggregate function are stored separately and * contiguously, to achieve better memory locality when updating them. */ void **per_agg_per_key_states; uint64 num_allocated_per_key_agg_states; /* * A memory context for aggregate functions to allocate additional data, * i.e. if they store strings or float8 datum on 32-bit systems. Valid until * the grouping policy is reset. */ MemoryContext agg_extra_mctx; /* * Whether we are in the mode of returning the partial aggregation results. * If we are, track the index of the last returned grouping key. */ bool returning_results; uint32 last_returned_key; /* * Some statistics for debugging. */ uint64 stat_input_total_rows; uint64 stat_input_valid_rows; uint64 stat_bulk_filtered_rows; uint64 stat_consecutive_keys; } GroupingPolicyHash; // #define DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__) #ifndef DEBUG_PRINT #define DEBUG_PRINT(...) #endif ================================================ FILE: tsl/src/nodes/vector_agg/hashing/CMakeLists.txt ================================================ set(SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/hash_strategy_single_fixed_2.c ${CMAKE_CURRENT_SOURCE_DIR}/hash_strategy_single_fixed_4.c ${CMAKE_CURRENT_SOURCE_DIR}/hash_strategy_single_fixed_8.c ${CMAKE_CURRENT_SOURCE_DIR}/hash_strategy_common.c) if(USE_UMASH) list(APPEND SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/hash_strategy_single_text.c ${CMAKE_CURRENT_SOURCE_DIR}/hash_strategy_serialized.c) endif() target_sources(${TSL_LIBRARY_NAME} PRIVATE ${SOURCES}) ================================================ FILE: tsl/src/nodes/vector_agg/hashing/batch_hashing_params.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include "nodes/vector_agg/grouping_policy_hash.h" #include "nodes/vector_agg/vector_slot.h" /* * The data required to map the rows of the given compressed batch to the unique * indexes of grouping keys, using a hash table. */ typedef struct BatchHashingParams { const uint64 *batch_filter; CompressedColumnValues single_grouping_column; int num_grouping_columns; const CompressedColumnValues *grouping_column_values; GroupingPolicyHash *policy; HashingStrategy *restrict hashing; uint32 *restrict result_key_indexes; } BatchHashingParams; static pg_attribute_always_inline BatchHashingParams build_batch_hashing_params(GroupingPolicyHash *policy, TupleTableSlot *vector_slot) { uint16 nrows; BatchHashingParams params = { .policy = policy, .hashing = &policy->hashing, .batch_filter = vector_slot_get_qual_result(vector_slot, &nrows), .num_grouping_columns = policy->num_grouping_columns, .grouping_column_values = policy->current_batch_grouping_column_values, .result_key_indexes = policy->key_index_for_row, }; Assert(policy->num_grouping_columns > 0); if (policy->num_grouping_columns == 1) { params.single_grouping_column = policy->current_batch_grouping_column_values[0]; } return params; } ================================================ FILE: tsl/src/nodes/vector_agg/hashing/hash64.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once /* * We can use crc32 as a hash function, it has bad properties but takes only one * cycle, which is why it is sometimes used in the existing hash table * implementations. When we don't have the crc32 instruction, use the SplitMix64 * finalizer. */ static pg_attribute_always_inline uint64 hash64_splitmix(uint64 x) { x ^= x >> 30; x *= 0xbf58476d1ce4e5b9U; x ^= x >> 27; x *= 0x94d049bb133111ebU; x ^= x >> 31; return x; } #ifdef USE_SSE42_CRC32C #include <nmmintrin.h> static pg_attribute_always_inline uint64 hash64_crc(uint64 x) { return _mm_crc32_u64(~0ULL, x); } #define HASH64 hash64_crc #else #define HASH64 hash64_splitmix #endif ================================================ FILE: tsl/src/nodes/vector_agg/hashing/hash_strategy_common.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include "hashing_strategy.h" #include "nodes/vector_agg/exec.h" #include "nodes/vector_agg/grouping_policy_hash.h" /* * Allocate enough storage for keys, given that each row of the new compressed * batch might turn out to be a new grouping key. We do this separately to avoid * allocations in the hot loop that fills the hash table. */ void hash_strategy_output_key_alloc(GroupingPolicyHash *policy, uint16 nrows) { HashingStrategy *hashing = &policy->hashing; const uint32 num_possible_keys = hashing->last_used_key_index + 1 + nrows; if (num_possible_keys > hashing->num_allocated_output_keys) { hashing->num_allocated_output_keys = num_possible_keys * 2 + 1; const size_t new_bytes = sizeof(Datum) * hashing->num_allocated_output_keys; if (hashing->output_keys == NULL) { hashing->output_keys = palloc(new_bytes); } else { hashing->output_keys = repalloc(hashing->output_keys, new_bytes); } } } /* * Emit a single-column grouping key with the given index into the aggregated * slot. */ void hash_strategy_output_key_single_emit(GroupingPolicyHash *policy, uint32 current_key, TupleTableSlot *aggregated_slot) { HashingStrategy *hashing = &policy->hashing; Assert(policy->num_grouping_columns == 1); const GroupingColumn *col = &policy->grouping_columns[0]; aggregated_slot->tts_values[col->output_offset] = hashing->output_keys[current_key]; aggregated_slot->tts_isnull[col->output_offset] = current_key == hashing->null_key_index; } ================================================ FILE: tsl/src/nodes/vector_agg/hashing/hash_strategy_impl.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include "batch_hashing_params.h" #include "nodes/vector_agg/vector_slot.h" /* * The hash table maps the value of the grouping key to its unique index. * We don't store any extra information here, because we're accessing the memory * of the hash table randomly, and want it to be as small as possible to fit the * caches. */ typedef struct FUNCTION_NAME(entry) { /* Key index 0 is invalid. */ uint32 key_index; uint8 status; HASH_TABLE_KEY_TYPE hash_table_key; } FUNCTION_NAME(entry); #define SH_PREFIX KEY_VARIANT #define SH_ELEMENT_TYPE FUNCTION_NAME(entry) #define SH_KEY_TYPE HASH_TABLE_KEY_TYPE #define SH_KEY hash_table_key #define SH_HASH_KEY(tb, key) KEY_HASH(key) #define SH_EQUAL(tb, a, b) KEY_EQUAL(a, b) #define SH_SCOPE static inline #define SH_DECLARE #define SH_DEFINE #include <lib/simplehash.h> struct FUNCTION_NAME(hash); static uint64 FUNCTION_NAME(get_size_bytes)(HashingStrategy *hashing) { struct FUNCTION_NAME(hash) *hash = (struct FUNCTION_NAME(hash) *) hashing->table; return hash->members * sizeof(FUNCTION_NAME(entry)); } static void FUNCTION_NAME(hash_strategy_init)(HashingStrategy *hashing, GroupingPolicyHash *policy) { hashing->table = FUNCTION_NAME(create)(CurrentMemoryContext, policy->num_allocated_per_key_agg_states, NULL); FUNCTION_NAME(key_hashing_init)(hashing); } static void FUNCTION_NAME(hash_strategy_reset)(HashingStrategy *hashing) { struct FUNCTION_NAME(hash) *table = (struct FUNCTION_NAME(hash) *) hashing->table; FUNCTION_NAME(reset)(table); hashing->last_used_key_index = 0; hashing->null_key_index = 0; /* * Have to reset this because it's in the key body context which is also * reset here. */ hashing->tmp_key_storage = NULL; hashing->num_tmp_key_storage_bytes = 0; } static void FUNCTION_NAME(hash_strategy_prepare_for_batch)(GroupingPolicyHash *policy, TupleTableSlot *vector_slot) { uint16 nrows = 0; vector_slot_get_qual_result(vector_slot, &nrows); hash_strategy_output_key_alloc(policy, nrows); FUNCTION_NAME(key_hashing_prepare_for_batch)(policy, vector_slot); } /* * Fill the unique key indexes for all rows of the batch, using a hash table. */ static pg_attribute_always_inline void FUNCTION_NAME(fill_offsets_impl)(BatchHashingParams params, int start_row, int end_row) { HashingStrategy *restrict hashing = params.hashing; uint32 *restrict indexes = params.result_key_indexes; struct FUNCTION_NAME(hash) *restrict table = hashing->table; HASH_TABLE_KEY_TYPE prev_hash_table_key = { 0 }; uint32 previous_key_index = 0; for (int row = start_row; row < end_row; row++) { if (!arrow_row_is_valid(params.batch_filter, row)) { /* The row doesn't pass the filter. */ DEBUG_PRINT("%p: row %d doesn't pass batch filter\n", hashing, row); continue; } /* * Get the key for the given row. For some hashing strategies, the key * that is used for the hash table is different from actual values of * the grouping columns, termed "output key" here. */ bool key_valid = false; OUTPUT_KEY_TYPE output_key = { 0 }; HASH_TABLE_KEY_TYPE hash_table_key = { 0 }; FUNCTION_NAME(key_hashing_get_key)(params, row, &output_key, &hash_table_key, &key_valid); if (unlikely(!key_valid)) { /* The key is null. */ if (hashing->null_key_index == 0) { hashing->null_key_index = ++hashing->last_used_key_index; } indexes[row] = hashing->null_key_index; DEBUG_PRINT("%p: row %d null key index %d\n", hashing, row, hashing->null_key_index); continue; } if (likely(previous_key_index != 0) && KEY_EQUAL(hash_table_key, prev_hash_table_key)) { /* * In real data sets, we often see consecutive rows with the * same value of a grouping column, so checking for this case * improves performance. For multi-column keys, this is unlikely, * but we currently often have suboptimal plans that use this policy * as a GroupAggregate, so we still use this as an easy optimization * for that case. */ indexes[row] = previous_key_index; #ifndef NDEBUG params.policy->stat_consecutive_keys++; #endif DEBUG_PRINT("%p: row %d consecutive key index %d\n", hashing, row, previous_key_index); continue; } /* * Find the key using the hash table. */ bool found = false; FUNCTION_NAME(entry) *restrict entry = FUNCTION_NAME(insert)(table, hash_table_key, &found); if (!found) { /* * New key, have to store it persistently. */ const uint32 index = ++hashing->last_used_key_index; entry->key_index = index; FUNCTION_NAME(key_hashing_store_new)(hashing, index, output_key); DEBUG_PRINT("%p: row %d new key index %d\n", hashing, row, index); } else { DEBUG_PRINT("%p: row %d old key index %d\n", hashing, row, entry->key_index); } indexes[row] = entry->key_index; previous_key_index = entry->key_index; prev_hash_table_key = entry->hash_table_key; } } static void FUNCTION_NAME(fill_offsets)(GroupingPolicyHash *policy, TupleTableSlot *vector_slot, int start_row, int end_row) { Assert((size_t) end_row <= policy->num_key_index_for_row); BatchHashingParams params = build_batch_hashing_params(policy, vector_slot); FUNCTION_NAME(fill_offsets_impl)(params, start_row, end_row); } HashingStrategy FUNCTION_NAME(strategy) = { .emit_key = FUNCTION_NAME(emit_key), .explain_name = EXPLAIN_NAME, .fill_offsets = FUNCTION_NAME(fill_offsets), .get_size_bytes = FUNCTION_NAME(get_size_bytes), .init = FUNCTION_NAME(hash_strategy_init), .prepare_for_batch = FUNCTION_NAME(hash_strategy_prepare_for_batch), .reset = FUNCTION_NAME(hash_strategy_reset), }; #undef EXPLAIN_NAME #undef KEY_VARIANT #undef KEY_EQUAL #undef OUTPUT_KEY_TYPE #undef HASH_TABLE_KEY_TYPE #undef USE_DICT_HASHING ================================================ FILE: tsl/src/nodes/vector_agg/hashing/hash_strategy_impl_single_fixed_key.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Key handling function for a single fixed-size grouping key. */ #include "batch_hashing_params.h" static void FUNCTION_NAME(key_hashing_init)(HashingStrategy *hashing) { } static void FUNCTION_NAME(key_hashing_prepare_for_batch)(GroupingPolicyHash *policy, TupleTableSlot *vector_slot) { } static pg_attribute_always_inline void FUNCTION_NAME(key_hashing_get_key)(BatchHashingParams params, int row, void *restrict output_key_ptr, void *restrict hash_table_key_ptr, bool *restrict valid) { OUTPUT_KEY_TYPE *restrict output_key = (OUTPUT_KEY_TYPE *) output_key_ptr; HASH_TABLE_KEY_TYPE *restrict hash_table_key = (HASH_TABLE_KEY_TYPE *) hash_table_key_ptr; if (unlikely(params.single_grouping_column.decompression_type == DT_Scalar)) { *output_key = DATUM_TO_OUTPUT_KEY(PointerGetDatum(params.single_grouping_column.buffers[1])); *valid = !DatumGetBool(PointerGetDatum(params.single_grouping_column.buffers[0])); } else if (params.single_grouping_column.decompression_type == sizeof(OUTPUT_KEY_TYPE)) { const OUTPUT_KEY_TYPE *values = params.single_grouping_column.buffers[1]; *valid = arrow_row_is_valid(params.single_grouping_column.buffers[0], row); *output_key = values[row]; } else { pg_unreachable(); } /* * For the fixed-size hash grouping, we use the output key as the hash table * key as well. */ *hash_table_key = *output_key; } static pg_attribute_always_inline void FUNCTION_NAME(key_hashing_store_new)(HashingStrategy *restrict hashing, uint32 new_key_index, OUTPUT_KEY_TYPE output_key) { hashing->output_keys[new_key_index] = OUTPUT_KEY_TO_DATUM(output_key); } static void FUNCTION_NAME(emit_key)(GroupingPolicyHash *policy, uint32 current_key, TupleTableSlot *aggregated_slot) { hash_strategy_output_key_single_emit(policy, current_key, aggregated_slot); } #undef DATUM_TO_OUTPUT_KEY #undef OUTPUT_KEY_TO_DATUM ================================================ FILE: tsl/src/nodes/vector_agg/hashing/hash_strategy_serialized.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Implementation of column hashing for multiple serialized columns. */ #include <postgres.h> #include <common/hashfn.h> #include "compression/arrow_c_data_interface.h" #include "nodes/columnar_scan/compressed_batch.h" #include "nodes/vector_agg/exec.h" #include "nodes/vector_agg/grouping_policy_hash.h" #include "template_helper.h" #include "batch_hashing_params.h" #include "umash_fingerprint_key.h" #define EXPLAIN_NAME "serialized" #define KEY_VARIANT serialized #define OUTPUT_KEY_TYPE text * static void serialized_key_hashing_init(HashingStrategy *hashing) { hashing->umash_params = umash_key_hashing_init(); } static void serialized_key_hashing_prepare_for_batch(GroupingPolicyHash *policy, TupleTableSlot *vector_slot) { } static pg_attribute_always_inline bool byte_bitmap_row_is_valid(const uint8 *bitmap, size_t row_number) { const size_t byte_index = row_number / 8; const size_t bit_index = row_number % 8; const uint8 mask = ((uint8) 1) << bit_index; return bitmap[byte_index] & mask; } static pg_attribute_always_inline void byte_bitmap_set_row_validity(uint8 *bitmap, size_t row_number, bool value) { const size_t byte_index = row_number / 8; const size_t bit_index = row_number % 8; const uint8 mask = ((uint8) 1) << bit_index; const uint8 new_bit = ((uint8) value) << bit_index; bitmap[byte_index] = (bitmap[byte_index] & ~mask) | new_bit; Assert(byte_bitmap_row_is_valid(bitmap, row_number) == value); } static pg_attribute_always_inline void serialized_key_hashing_get_key(BatchHashingParams params, int row, void *restrict output_key_ptr, void *restrict hash_table_key_ptr, bool *restrict valid) { HashingStrategy *hashing = params.hashing; text **restrict output_key = (text **) output_key_ptr; HASH_TABLE_KEY_TYPE *restrict hash_table_key = (HASH_TABLE_KEY_TYPE *) hash_table_key_ptr; const int num_columns = params.num_grouping_columns; const size_t bitmap_bytes = (num_columns + 7) / 8; /* * Loop through the grouping columns to determine the length of the key. We * need that to allocate memory to store it. * * The key has the null bitmap at the beginning. */ size_t num_bytes = bitmap_bytes; for (int column_index = 0; column_index < num_columns; column_index++) { const CompressedColumnValues *column_values = ¶ms.grouping_column_values[column_index]; if (column_values->decompression_type == DT_Scalar) { if (!DatumGetBool(PointerGetDatum(column_values->buffers[0]))) { const GroupingColumn *def = ¶ms.policy->grouping_columns[column_index]; if (def->value_bytes > 0) { num_bytes += def->value_bytes; } else { /* * The default value always has a long varlena header, but * we are going to use short if it fits. */ const int32 value_bytes = VARSIZE_ANY_EXHDR(column_values->buffers[1]); if (value_bytes + VARHDRSZ_SHORT <= VARATT_SHORT_MAX) { /* Short varlena, unaligned. */ const int total_bytes = value_bytes + VARHDRSZ_SHORT; num_bytes += total_bytes; } else { /* Long varlena, requires alignment. */ const int total_bytes = value_bytes + VARHDRSZ; num_bytes = TYPEALIGN(4, num_bytes) + total_bytes; } } } continue; } const bool is_valid = arrow_row_is_valid(column_values->buffers[0], row); if (!is_valid) { continue; } if (column_values->decompression_type > 0) { num_bytes += column_values->decompression_type; continue; } if (column_values->decompression_type == DT_ArrowBits) { num_bytes += 1; continue; } Assert(column_values->decompression_type == DT_ArrowText || column_values->decompression_type == DT_ArrowTextDict); Assert((column_values->decompression_type == DT_ArrowTextDict) == (column_values->buffers[3] != NULL)); const uint32 data_row = (column_values->decompression_type == DT_ArrowTextDict) ? ((int16 *) column_values->buffers[3])[row] : row; const uint32 start = ((uint32 *) column_values->buffers[1])[data_row]; const int32 value_bytes = ((uint32 *) column_values->buffers[1])[data_row + 1] - start; if (value_bytes + VARHDRSZ_SHORT <= VARATT_SHORT_MAX) { /* Short varlena, unaligned. */ const int total_bytes = value_bytes + VARHDRSZ_SHORT; num_bytes += total_bytes; } else { /* Long varlena, requires alignment. */ const int total_bytes = value_bytes + VARHDRSZ; num_bytes = TYPEALIGN(4, num_bytes) + total_bytes; } } /* * The key has short or long varlena header. This is a little tricky, we * decide the header length after we have counted all the columns, but we * put it at the beginning. Technically it could change the length because * of the alignment. In practice, we only use alignment by 4 bytes for long * varlena strings, and if we have at least one long varlena string column, * the key is also going to use the long varlena header which is 4 bytes, so * the alignment is not affected. If we use the short varlena header for the * key, it necessarily means that there were no long varlena columns and * therefore no alignment is needed. */ const bool key_uses_short_header = num_bytes + VARHDRSZ_SHORT <= VARATT_SHORT_MAX; num_bytes += key_uses_short_header ? VARHDRSZ_SHORT : VARHDRSZ; /* * Use temporary storage for the new key, reallocate if it's too small. */ if (num_bytes > hashing->num_tmp_key_storage_bytes) { if (hashing->tmp_key_storage != NULL) { pfree(hashing->tmp_key_storage); } hashing->tmp_key_storage = MemoryContextAlloc(hashing->key_body_mctx, num_bytes); hashing->num_tmp_key_storage_bytes = num_bytes; } uint8 *restrict serialized_key_storage = hashing->tmp_key_storage; /* * Build the actual grouping key. */ uint32 offset = 0; offset += key_uses_short_header ? VARHDRSZ_SHORT : VARHDRSZ; /* * We must always save the validity bitmap, even when there are no * null words, so that the key is uniquely deserializable. Otherwise a key * with some nulls might collide with a key with no nulls. */ uint8 *restrict serialized_key_validity_bitmap = &serialized_key_storage[offset]; offset += bitmap_bytes; /* * Loop through the grouping columns again and add their values to the * grouping key. */ for (int column_index = 0; column_index < num_columns; column_index++) { const CompressedColumnValues *column_values = ¶ms.grouping_column_values[column_index]; if (column_values->decompression_type == DT_Scalar) { const bool is_valid = !DatumGetBool(PointerGetDatum(column_values->buffers[0])); Datum value = PointerGetDatum(column_values->buffers[1]); byte_bitmap_set_row_validity(serialized_key_validity_bitmap, column_index, is_valid); if (is_valid) { const GroupingColumn *def = ¶ms.policy->grouping_columns[column_index]; if (def->by_value) { memcpy(&serialized_key_storage[offset], &value, def->value_bytes); offset += def->value_bytes; } else if (def->value_bytes > 0) { memcpy(&serialized_key_storage[offset], DatumGetPointer(value), def->value_bytes); offset += def->value_bytes; } else { /* * The default value always has a long varlena header, but * we are going to use short if it fits. */ const int32 value_bytes = VARSIZE_ANY_EXHDR(value); if (value_bytes + VARHDRSZ_SHORT <= VARATT_SHORT_MAX) { /* Short varlena, no alignment. */ const int32 total_bytes = value_bytes + VARHDRSZ_SHORT; SET_VARSIZE_SHORT(&serialized_key_storage[offset], total_bytes); offset += VARHDRSZ_SHORT; } else { /* Long varlena, requires alignment. Zero out the alignment bytes. */ memset(&serialized_key_storage[offset], 0, 4); offset = TYPEALIGN(4, offset); const int32 total_bytes = value_bytes + VARHDRSZ; SET_VARSIZE(&serialized_key_storage[offset], total_bytes); offset += VARHDRSZ; } memcpy(&serialized_key_storage[offset], VARDATA_ANY(value), value_bytes); offset += value_bytes; } } continue; } const bool is_valid = arrow_row_is_valid(column_values->buffers[0], row); byte_bitmap_set_row_validity(serialized_key_validity_bitmap, column_index, is_valid); if (!is_valid) { continue; } if (column_values->decompression_type > 0) { Assert(offset <= UINT_MAX - column_values->decompression_type); switch ((int) column_values->decompression_type) { case 2: memcpy(&serialized_key_storage[offset], row + (int16 *) column_values->buffers[1], 2); break; case 4: memcpy(&serialized_key_storage[offset], row + (int32 *) column_values->buffers[1], 4); break; case 8: memcpy(&serialized_key_storage[offset], row + (int64 *) column_values->buffers[1], 8); break; case 16: memcpy(&serialized_key_storage[offset], (row * 2) + (int64 *) column_values->buffers[1], 16); break; default: pg_unreachable(); break; } offset += column_values->decompression_type; continue; } if (column_values->decompression_type == DT_ArrowBits) { serialized_key_storage[offset] = arrow_row_is_valid(column_values->buffers[1], row); offset += 1; continue; } Assert(column_values->decompression_type == DT_ArrowText || column_values->decompression_type == DT_ArrowTextDict); const uint32 data_row = column_values->decompression_type == DT_ArrowTextDict ? ((int16 *) column_values->buffers[3])[row] : row; const uint32 start = ((uint32 *) column_values->buffers[1])[data_row]; const int32 value_bytes = ((uint32 *) column_values->buffers[1])[data_row + 1] - start; if (value_bytes + VARHDRSZ_SHORT <= VARATT_SHORT_MAX) { /* Short varlena, unaligned. */ const int32 total_bytes = value_bytes + VARHDRSZ_SHORT; SET_VARSIZE_SHORT(&serialized_key_storage[offset], total_bytes); offset += VARHDRSZ_SHORT; } else { /* Long varlena, requires alignment. Zero out the alignment bytes. */ memset(&serialized_key_storage[offset], 0, 4); offset = TYPEALIGN(4, offset); const int32 total_bytes = value_bytes + VARHDRSZ; SET_VARSIZE(&serialized_key_storage[offset], total_bytes); offset += VARHDRSZ; } memcpy(&serialized_key_storage[offset], &((uint8 *) column_values->buffers[2])[start], value_bytes); offset += value_bytes; } Assert(offset == num_bytes); if (key_uses_short_header) { SET_VARSIZE_SHORT(serialized_key_storage, offset); } else { SET_VARSIZE(serialized_key_storage, offset); } DEBUG_PRINT("key is %d bytes: ", offset); for (size_t i = 0; i < offset; i++) { DEBUG_PRINT("%.2x.", serialized_key_storage[i]); } DEBUG_PRINT("\n"); *output_key = (text *) serialized_key_storage; Assert(VARSIZE_ANY(*output_key) == num_bytes); /* * The multi-column key is always considered non-null, and the null flags * for the individual columns are stored in a bitmap that is part of the * key. */ *valid = true; const struct umash_fp fp = umash_fprint(params.hashing->umash_params, /* seed = */ ~0ULL, serialized_key_storage, num_bytes); *hash_table_key = umash_fingerprint_get_key(fp); } static pg_attribute_always_inline void serialized_key_hashing_store_new(HashingStrategy *restrict hashing, uint32 new_key_index, text *output_key) { /* * We will store this key so we have to consume the temporary storage that * was used for it. The subsequent keys will need to allocate new memory. */ Assert(hashing->tmp_key_storage == (void *) output_key); hashing->tmp_key_storage = NULL; hashing->num_tmp_key_storage_bytes = 0; hashing->output_keys[new_key_index] = PointerGetDatum(output_key); } static void serialized_emit_key(GroupingPolicyHash *policy, uint32 current_key, TupleTableSlot *aggregated_slot) { const HashingStrategy *hashing = &policy->hashing; const int num_key_columns = policy->num_grouping_columns; const Datum serialized_key_datum = hashing->output_keys[current_key]; const uint8 *serialized_key = (const uint8 *) VARDATA_ANY(serialized_key_datum); PG_USED_FOR_ASSERTS_ONLY const int key_data_bytes = VARSIZE_ANY_EXHDR(serialized_key_datum); const uint8 *restrict ptr = serialized_key; /* * We have the column validity bitmap at the beginning of the key. */ const int bitmap_bytes = (num_key_columns + 7) / 8; Assert(bitmap_bytes <= key_data_bytes); const uint8 *restrict key_validity_bitmap = serialized_key; ptr += bitmap_bytes; DEBUG_PRINT("emit key #%d, with header %ld without %d bytes: ", current_key, VARSIZE_ANY(serialized_key_datum), key_data_bytes); for (size_t i = 0; i < VARSIZE_ANY(serialized_key_datum); i++) { DEBUG_PRINT("%.2x.", ((const uint8 *) serialized_key_datum)[i]); } DEBUG_PRINT("\n"); for (int column_index = 0; column_index < num_key_columns; column_index++) { const GroupingColumn *col = &policy->grouping_columns[column_index]; const bool isnull = !byte_bitmap_row_is_valid(key_validity_bitmap, column_index); aggregated_slot->tts_isnull[col->output_offset] = isnull; if (isnull) { continue; } Datum *output = &aggregated_slot->tts_values[col->output_offset]; if (col->value_bytes > 0) { if (col->by_value) { Assert(col->by_value); Assert((size_t) col->value_bytes <= sizeof(Datum)); *output = 0; memcpy(output, ptr, col->value_bytes); } else { *output = PointerGetDatum(ptr); } ptr += col->value_bytes; } else { Assert(col->value_bytes == -1); Assert(!col->by_value); if (VARATT_IS_SHORT(ptr)) { *output = PointerGetDatum(ptr); ptr += VARSIZE_SHORT(ptr); } else { ptr = (const uint8 *) TYPEALIGN(4, ptr); *output = PointerGetDatum(ptr); ptr += VARSIZE(ptr); } } } Assert(ptr == serialized_key + key_data_bytes); } #include "hash_strategy_impl.c" ================================================ FILE: tsl/src/nodes/vector_agg/hashing/hash_strategy_single_fixed_2.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Implementation of column hashing for a single fixed size 2-byte column. */ #include <postgres.h> #include "compression/arrow_c_data_interface.h" #include "hash64.h" #include "nodes/columnar_scan/compressed_batch.h" #include "nodes/vector_agg/exec.h" #include "nodes/vector_agg/grouping_policy_hash.h" #include "template_helper.h" #define EXPLAIN_NAME "single 2-byte" #define KEY_VARIANT single_fixed_2 #define OUTPUT_KEY_TYPE int16 #define HASH_TABLE_KEY_TYPE OUTPUT_KEY_TYPE #define DATUM_TO_OUTPUT_KEY DatumGetInt16 #define OUTPUT_KEY_TO_DATUM Int16GetDatum #include "hash_strategy_impl_single_fixed_key.c" #define KEY_EQUAL(a, b) a == b #define KEY_HASH(X) HASH64(X) #include "hash_strategy_impl.c" ================================================ FILE: tsl/src/nodes/vector_agg/hashing/hash_strategy_single_fixed_4.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Implementation of column hashing for a single fixed size 4-byte column. */ #include <postgres.h> #include "compression/arrow_c_data_interface.h" #include "hash64.h" #include "nodes/columnar_scan/compressed_batch.h" #include "nodes/vector_agg/exec.h" #include "nodes/vector_agg/grouping_policy_hash.h" #include "template_helper.h" #define EXPLAIN_NAME "single 4-byte" #define KEY_VARIANT single_fixed_4 #define OUTPUT_KEY_TYPE int32 #define HASH_TABLE_KEY_TYPE int32 #define DATUM_TO_OUTPUT_KEY DatumGetInt32 #define OUTPUT_KEY_TO_DATUM Int32GetDatum #include "hash_strategy_impl_single_fixed_key.c" #define KEY_EQUAL(a, b) a == b #define KEY_HASH(X) HASH64(X) #include "hash_strategy_impl.c" ================================================ FILE: tsl/src/nodes/vector_agg/hashing/hash_strategy_single_fixed_8.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Implementation of column hashing for a single fixed size 8-byte column. */ #include <postgres.h> #include "compression/arrow_c_data_interface.h" #include "hash64.h" #include "nodes/columnar_scan/compressed_batch.h" #include "nodes/vector_agg/exec.h" #include "nodes/vector_agg/grouping_policy_hash.h" #include "template_helper.h" #define EXPLAIN_NAME "single 8-byte" #define KEY_VARIANT single_fixed_8 #define OUTPUT_KEY_TYPE int64 #define HASH_TABLE_KEY_TYPE int64 #define DATUM_TO_OUTPUT_KEY DatumGetInt64 #define OUTPUT_KEY_TO_DATUM Int64GetDatum #include "hash_strategy_impl_single_fixed_key.c" #define KEY_EQUAL(a, b) a == b #define KEY_HASH(X) HASH64(X) #include "hash_strategy_impl.c" ================================================ FILE: tsl/src/nodes/vector_agg/hashing/hash_strategy_single_text.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * Implementation of column hashing for a single text column. */ #include <postgres.h> #include <common/hashfn.h> #include "compression/arrow_c_data_interface.h" #include "nodes/columnar_scan/compressed_batch.h" #include "nodes/vector_agg/exec.h" #include "nodes/vector_agg/grouping_policy_hash.h" #include "template_helper.h" #include "batch_hashing_params.h" #include "umash_fingerprint_key.h" #define EXPLAIN_NAME "single text" #define KEY_VARIANT single_text #define OUTPUT_KEY_TYPE BytesView static void single_text_key_hashing_init(HashingStrategy *hashing) { hashing->umash_params = umash_key_hashing_init(); } typedef struct BytesView { const uint8 *data; uint32 len; } BytesView; static BytesView get_bytes_view(CompressedColumnValues *column_values, int arrow_row) { const uint32 start = ((uint32 *) column_values->buffers[1])[arrow_row]; const int32 value_bytes = ((uint32 *) column_values->buffers[1])[arrow_row + 1] - start; Assert(value_bytes >= 0); return (BytesView){ .len = value_bytes, .data = &((uint8 *) column_values->buffers[2])[start] }; } static pg_attribute_always_inline void single_text_key_hashing_get_key(BatchHashingParams params, int row, void *restrict output_key_ptr, void *restrict hash_table_key_ptr, bool *restrict valid) { Assert(params.policy->num_grouping_columns == 1); BytesView *restrict output_key = (BytesView *) output_key_ptr; HASH_TABLE_KEY_TYPE *restrict hash_table_key = (HASH_TABLE_KEY_TYPE *) hash_table_key_ptr; if (unlikely(params.single_grouping_column.decompression_type == DT_Scalar)) { *valid = !DatumGetBool(PointerGetDatum(params.single_grouping_column.buffers[0])); if (*valid) { output_key->len = VARSIZE_ANY_EXHDR(params.single_grouping_column.buffers[1]); output_key->data = (const uint8 *) VARDATA_ANY(params.single_grouping_column.buffers[1]); } else { output_key->len = 0; output_key->data = NULL; } } else if (params.single_grouping_column.decompression_type == DT_ArrowText) { /* * The Arrow format requires the offsets to be monotonically increasing * even for null rows, so this is safe. */ *output_key = get_bytes_view(¶ms.single_grouping_column, row); *valid = arrow_row_is_valid(params.single_grouping_column.buffers[0], row); } else if (params.single_grouping_column.decompression_type == DT_ArrowTextDict) { const int16 index = ((int16 *) params.single_grouping_column.buffers[3])[row]; *output_key = get_bytes_view(¶ms.single_grouping_column, index); *valid = arrow_row_is_valid(params.single_grouping_column.buffers[0], row); } else { pg_unreachable(); } DEBUG_PRINT("%p consider key row %d key index %d is %d bytes: ", params.hashing, row, params.hashing->last_used_key_index + 1, output_key->len); for (size_t i = 0; i < output_key->len; i++) { DEBUG_PRINT("%.2x.", output_key->data[i]); } DEBUG_PRINT("\n"); const struct umash_fp fp = umash_fprint(params.policy->hashing.umash_params, /* seed = */ ~0ULL, output_key->data, output_key->len); *hash_table_key = umash_fingerprint_get_key(fp); } static pg_attribute_always_inline void single_text_key_hashing_store_new(HashingStrategy *restrict hashing, uint32 new_key_index, BytesView output_key) { const int total_bytes = output_key.len + VARHDRSZ; text *restrict stored = (text *) MemoryContextAlloc(hashing->key_body_mctx, total_bytes); SET_VARSIZE(stored, total_bytes); memcpy(VARDATA(stored), output_key.data, output_key.len); hashing->output_keys[new_key_index] = PointerGetDatum(stored); } /* * We use the standard single-key key output functions. */ static void single_text_emit_key(GroupingPolicyHash *policy, uint32 current_key, TupleTableSlot *aggregated_slot) { return hash_strategy_output_key_single_emit(policy, current_key, aggregated_slot); } static void single_text_key_hashing_prepare_for_batch(GroupingPolicyHash *policy, TupleTableSlot *vector_slot) { } #include "hash_strategy_impl.c" ================================================ FILE: tsl/src/nodes/vector_agg/hashing/hashing_strategy.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> typedef struct GroupingPolicyHash GroupingPolicyHash; typedef struct HashingStrategy HashingStrategy; typedef struct DecompressBatchState DecompressBatchState; typedef struct TupleTableSlot TupleTableSlot; /* * The hashing strategy manages the details of how the grouping keys are stored * in a hash table. */ typedef struct HashingStrategy { char *explain_name; void (*init)(HashingStrategy *hashing, GroupingPolicyHash *policy); void (*reset)(HashingStrategy *hashing); uint64 (*get_size_bytes)(HashingStrategy *hashing); void (*prepare_for_batch)(GroupingPolicyHash *policy, TupleTableSlot *vector_slot); void (*fill_offsets)(GroupingPolicyHash *policy, TupleTableSlot *vector_slot, int start_row, int end_row); void (*emit_key)(GroupingPolicyHash *policy, uint32 current_key, TupleTableSlot *aggregated_slot); /* * The hash table we use for grouping. It matches each grouping key to its * unique integer index. */ void *table; /* * For each unique grouping key, we store the values of the grouping columns. * This is stored separately from hash table keys, because they might not * have the full column values, and also storing them contiguously here * leads to better memory access patterns when emitting the results. * The details of the key storage are managed by the hashing strategy. The * by-reference keys can use a separate memory context for dense storage. */ Datum *restrict output_keys; uint64 num_allocated_output_keys; MemoryContext key_body_mctx; /* * The last used index of an unique grouping key. Key index 0 is invalid. */ uint32 last_used_key_index; /* * In single-column grouping, we store the null key outside of the hash * table, and its index is given by this value. Key index 0 is invalid. * This is done to avoid having an "is null" flag in the hash table entries, * to reduce the hash table size. */ uint32 null_key_index; #ifdef TS_USE_UMASH /* * UMASH fingerprinting parameters. */ struct umash_params *umash_params; #endif /* * Temporary key storages. Some hashing strategies need to put the key in a * separate memory area, we don't want to alloc/free it on each row. */ uint8 *tmp_key_storage; uint64 num_tmp_key_storage_bytes; } HashingStrategy; void hash_strategy_output_key_alloc(GroupingPolicyHash *policy, uint16 nrows); void hash_strategy_output_key_single_emit(GroupingPolicyHash *policy, uint32 current_key, TupleTableSlot *aggregated_slot); ================================================ FILE: tsl/src/nodes/vector_agg/hashing/template_helper.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #define FUNCTION_NAME_HELPER2(X, Y) X##_##Y #define FUNCTION_NAME_HELPER(X, Y) FUNCTION_NAME_HELPER2(X, Y) #define FUNCTION_NAME(Y) FUNCTION_NAME_HELPER(KEY_VARIANT, Y) ================================================ FILE: tsl/src/nodes/vector_agg/hashing/umash_fingerprint_key.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once /* * Helpers to use the umash fingerprint as a hash table key in our hashing * strategies for vectorized grouping. */ #include "import/umash.h" /* * The struct is packed so that the hash table entry fits into 16 * bytes with the uint32 key index that goes before. */ struct umash_fingerprint_key { uint32 hash; uint64 rest; } pg_attribute_packed(); #define HASH_TABLE_KEY_TYPE struct umash_fingerprint_key #define KEY_HASH(X) (X.hash) #define KEY_EQUAL(a, b) (a.hash == b.hash && a.rest == b.rest) static inline struct umash_fingerprint_key umash_fingerprint_get_key(struct umash_fp fp) { const struct umash_fingerprint_key key = { .hash = fp.hash[0] & (~(uint32) 0), .rest = fp.hash[1], }; return key; } static inline struct umash_params * umash_key_hashing_init() { struct umash_params *params = palloc0(sizeof(struct umash_params)); umash_params_derive(params, 0xabcdef1234567890ull, NULL); return params; } ================================================ FILE: tsl/src/nodes/vector_agg/plan.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/attnum.h> #include <commands/explain.h> #include <executor/executor.h> #include <funcapi.h> #include <nodes/extensible.h> #include <nodes/makefuncs.h> #include <nodes/nodeFuncs.h> #include <nodes/plannodes.h> #include <optimizer/planner.h> #include <parser/parsetree.h> #include <utils/fmgroids.h> #include "plan.h" #include "exec.h" #include "expression_utils.h" #include "import/list.h" #include "nodes/chunk_append/chunk_append.h" #include "nodes/columnar_scan/columnar_scan.h" #include "nodes/columnar_scan/vector_quals.h" #include "nodes/modify_hypertable.h" #include "nodes/vector_agg.h" #include "utils.h" static struct CustomScanMethods scan_methods = { .CustomName = VECTOR_AGG_NODE_NAME, .CreateCustomScanState = vector_agg_state_create }; void _vector_agg_init(void) { TryRegisterCustomScanMethods(&scan_methods); } bool ts_is_vector_agg_plan(Plan *plan) { return IsA(plan, CustomScan) && castNode(CustomScan, plan)->methods == &scan_methods; } /* * Create a vectorized aggregation node to replace the given partial aggregation * node. */ static Plan * vector_agg_plan_create(Plan *childplan, Agg *agg, List *resolved_targetlist, VectorAggGroupingType grouping_type) { CustomScan *vector_agg = (CustomScan *) makeNode(CustomScan); vector_agg->custom_plans = list_make1(childplan); vector_agg->methods = &scan_methods; vector_agg->custom_scan_tlist = resolved_targetlist; /* * Note that this is being called from the post-planning hook, and therefore * after set_plan_refs(). The meaning of output targetlists is different from * the previous planning stages, and they contain special varnos referencing * the scan targetlists. */ vector_agg->scan.plan.targetlist = ts_build_trivial_custom_output_targetlist(vector_agg->custom_scan_tlist); /* * Copy the costs from the normal aggregation node, so that they show up in * the EXPLAIN output. They are not used for any other purposes, because * this hook is called after the planning is finished. */ vector_agg->scan.plan.plan_rows = agg->plan.plan_rows; vector_agg->scan.plan.plan_width = agg->plan.plan_width; vector_agg->scan.plan.startup_cost = agg->plan.startup_cost; vector_agg->scan.plan.total_cost = agg->plan.total_cost; vector_agg->scan.plan.parallel_aware = false; vector_agg->scan.plan.parallel_safe = childplan->parallel_safe; vector_agg->scan.plan.async_capable = false; vector_agg->scan.plan.plan_node_id = agg->plan.plan_node_id; Assert(agg->plan.qual == NIL); vector_agg->scan.plan.initPlan = agg->plan.initPlan; vector_agg->scan.plan.extParam = bms_copy(agg->plan.extParam); vector_agg->scan.plan.allParam = bms_copy(agg->plan.allParam); vector_agg->custom_private = ts_new_list(T_List, VASI_Count); lfirst(list_nth_cell(vector_agg->custom_private, VASI_GroupingType)) = makeInteger(grouping_type); return (Plan *) vector_agg; } /* * Whether we have an in-memory columnar representation for a given type. */ static bool is_vector_type(Oid typeoid) { switch (typeoid) { case BOOLOID: case FLOAT4OID: case FLOAT8OID: case INT2OID: case INT4OID: case INT8OID: case TEXTOID: case TIMESTAMPOID: case TIMESTAMPTZOID: case DATEOID: case UUIDOID: case INTERVALOID: return true; default: return false; } } static bool is_vector_expr(const VectorQualInfo *vqinfo, Expr *expr); /* * Whether we can evaluate this function as part of the columnar pipeline. */ static bool is_vector_function(const VectorQualInfo *vqinfo, List *args, Oid funcoid, Oid resulttype, Oid inputcollid) { if (!is_vector_type(resulttype)) { return false; } ListCell *lc; foreach (lc, args) { if (!is_vector_expr(vqinfo, (Expr *) lfirst(lc))) { return false; } } if (!func_strict(funcoid)) { return false; } if (func_volatile(funcoid) == PROVOLATILE_VOLATILE) { return false; } return true; } /* * Whether the expression can be used for vectorized processing: must be a Var * that refers to either a bulk-decompressed or a segmentby column. */ static bool is_vector_expr(const VectorQualInfo *vqinfo, Expr *expr) { /* * Skip NULLs for uniform handling of the optional nodes. */ if (expr == NULL) { return true; } switch (((Node *) expr)->type) { case T_Const: { Const *c = (Const *) expr; return is_vector_type(c->consttype); } case T_FuncExpr: { /* Can vectorize some functions! */ FuncExpr *f = castNode(FuncExpr, expr); return is_vector_function(vqinfo, f->args, f->funcid, f->funcresulttype, f->inputcollid); } case T_OpExpr: { OpExpr *o = castNode(OpExpr, expr); return is_vector_function(vqinfo, o->args, o->opfuncid, o->opresulttype, o->inputcollid); } case T_Var: { Var *var = castNode(Var, expr); if (var->varattno <= 0) { /* Can't work with special attributes like tableoid. */ return false; } Assert(var->varattno <= vqinfo->maxattno); const bool is_vector = vqinfo->vector_attrs && vqinfo->vector_attrs[var->varattno]; /* * The segmentby columns are considered vectorizable, but their type might not actually * have a columnar representation. Theoretically this can work because they are always * represented as DT_Scalar, but in practice this is poorly tested and of limited * utility, so we consider such columns not to be vectorizable at the moment. */ return is_vector && is_vector_type(var->vartype); } default: return false; } } /* * Whether we can vectorize this particular aggregate. */ static bool can_vectorize_aggref(const VectorQualInfo *vqi, Aggref *aggref) { if (aggref->aggdirectargs != NIL) { /* Can't process ordered-set aggregates with direct arguments. */ return false; } if (aggref->aggorder != NIL) { /* Can't process aggregates with an ORDER BY clause. */ return false; } if (aggref->aggdistinct != NIL) { /* Can't process aggregates with DISTINCT clause. */ return false; } if (aggref->aggfilter != NULL) { /* Can process aggregates with filter clause if it's vectorizable. */ Node *aggfilter_vectorized = vector_qual_make((Node *) aggref->aggfilter, vqi); if (aggfilter_vectorized == NULL) { return false; } aggref->aggfilter = (Expr *) aggfilter_vectorized; } if (get_vector_aggregate(aggref->aggfnoid, aggref->inputcollid) == NULL) { /* * We don't have a vectorized implementation for this particular * aggregate function. */ return false; } if (aggref->args == NIL) { /* This must be count(*), we can vectorize it. */ return true; } /* The function must have one argument, check it. */ Assert(list_length(aggref->args) == 1); TargetEntry *argument = castNode(TargetEntry, linitial(aggref->args)); return is_vector_expr(vqi, argument->expr); } /* * What vectorized grouping strategy we can use for the given grouping columns. */ static VectorAggGroupingType get_vectorized_grouping_type(const VectorQualInfo *vqinfo, Agg *agg, List *resolved_targetlist) { /* * The Agg->numCols value can be less than the number of the non-aggregated * vars in the aggregated targetlist, if some of them are equated to a * constant. This behavior started with PG 16. This case is not very * important, so we treat all non-aggregated columns as grouping columns to * keep the vectorized aggregation node simple. */ int num_grouping_columns = 0; bool all_segmentby = true; Oid single_grouping_var_type = InvalidOid; int16 typlen = 0; bool typbyval = false; ListCell *lc; foreach (lc, resolved_targetlist) { TargetEntry *target_entry = lfirst_node(TargetEntry, lc); if (IsA(target_entry->expr, Aggref)) { continue; } num_grouping_columns++; if (!is_vector_expr(vqinfo, target_entry->expr)) { return VAGT_Invalid; } /* * Detect whether we're only grouping by segmentby columns, in which * case we can use the whole-batch grouping strategy. Probably this * could be extended to allow arbitrary expressions referencing only the * segmentby columns. */ if (IsA(target_entry->expr, Var)) { Var *var = castNode(Var, target_entry->expr); all_segmentby &= vqinfo->segmentby_attrs[var->varattno]; } else { all_segmentby = false; } /* * If we have a single grouping column, record it for the additional * checks later. */ if (num_grouping_columns != 1) { continue; } TupleDesc tdesc = NULL; TypeFuncClass type_class = get_expr_result_type((Node *) target_entry->expr, &single_grouping_var_type, &tdesc); if (type_class != TYPEFUNC_SCALAR) { continue; } get_typlenbyval(single_grouping_var_type, &typlen, &typbyval); Ensure(typlen != 0, "invalid zero typlen for type %d", single_grouping_var_type); } Assert(num_grouping_columns >= agg->numCols); /* * We support vectorized aggregation without grouping. */ if (num_grouping_columns == 0) { return VAGT_Batch; } /* * We support grouping by any number of columns if all of them are segmentby. */ if (all_segmentby) { return VAGT_Batch; } /* * We support hashed vectorized grouping by one fixed-size by-value * compressed column. * We can use our hash table for GroupAggregate as well, because it preserves * the input order of the keys, but only for the direct order, not reverse. */ if (num_grouping_columns == 1 && typlen != 0) { if (typbyval) { switch (typlen) { case 1: #ifdef TS_USE_UMASH Assert(single_grouping_var_type == BOOLOID); return VAGT_HashSerialized; #else return VAGT_Invalid; #endif case 2: return VAGT_HashSingleFixed2; case 4: return VAGT_HashSingleFixed4; case 8: return VAGT_HashSingleFixed8; default: Ensure(false, "invalid fixed size %d of a vector type", typlen); break; } } #ifdef TS_USE_UMASH /* * We also have the UUID type which is by-reference and has a * columnar in-memory representation, but no specialized single-column * vectorized grouping support. It can use the serialized grouping * strategy. */ else if (single_grouping_var_type == TEXTOID) { return VAGT_HashSingleText; } #endif } #ifdef TS_USE_UMASH /* * Use hashing of serialized keys when we have many grouping columns. */ return VAGT_HashSerialized; #else return VAGT_Invalid; #endif } typedef struct HasVectorAggContext { bool has_agg; bool has_vector_agg; } HasVectorAggContext; static Plan * has_vector_agg(Plan *plan, void *context) { HasVectorAggContext *ctx = (HasVectorAggContext *) context; if (IsA(plan, Agg)) { ctx->has_agg = true; } else if (ts_is_vector_agg_plan(plan)) { ctx->has_vector_agg = true; } return plan; } /* * Whether we have a vectorized aggregation node and any aggregate node at all * in the plan tree. This is used for testing. */ bool has_vector_agg_node(Plan *plan, bool *has_some_agg) { HasVectorAggContext context = { .has_agg = false, .has_vector_agg = false }; ts_plan_tree_walker(plan, has_vector_agg, &context); *has_some_agg = context.has_agg; return context.has_vector_agg; } /* * Check if a VectorAgg is possible on top of the given child plan. * * If the child plan is compatible, also initialize the VectorQualInfo struct * for aggregation FILTER clauses. * * Returns true if the scan node is a supported child, otherwise false. */ static bool vectoragg_plan_possible(Plan *childplan, VectorQualInfo *vqi) { if (!IsA(childplan, CustomScan)) return false; if (childplan->qual != NIL) { /* Can't do vectorized aggregation if we have Postgres quals. */ return false; } if (ts_is_columnar_scan_plan(childplan)) { vectoragg_plan_columnar_scan(childplan, vqi); return true; } return false; } static Node * mark_partial_aggref_mutator(Node *node, void *context) { if (node == NULL) return NULL; if (IsA(node, Aggref)) { mark_partial_aggref(castNode(Aggref, node), AGGSPLIT_INITIAL_SERIAL); return node; } return expression_tree_mutator(node, mark_partial_aggref_mutator, context); } typedef struct MakeFinalizeAggContext { Agg *agg; List *vector_agg_targetlist; } MakeFinalizeAggContext; static Node * make_finalize_agg_mutator(Node *node, void *context) { if (node == NULL) return NULL; if (IsA(node, TargetEntry)) { TargetEntry *tle = castNode(TargetEntry, node); MakeFinalizeAggContext *ctx = (MakeFinalizeAggContext *) context; if (IsA(tle->expr, Var)) { Var *var = castNode(Var, tle->expr); Assert(var->varno == OUTER_VAR); AttrNumber old_attno = var->varattno; var->varattno = tle->resno; for (int k = 0; k < ctx->agg->numCols; k++) { if (ctx->agg->grpColIdx[k] == old_attno) ctx->agg->grpColIdx[k] = tle->resno; } return node; } if (IsA(tle->expr, Aggref)) { Aggref *aggref = castNode(Aggref, tle->expr); /* * Look up the VectorAgg output type for this column, which is * the transition type set by mark_partial_aggref above. */ TargetEntry *vag_tle = list_nth(ctx->vector_agg_targetlist, tle->resno - 1); Oid var_type = exprType((Node *) vag_tle->expr); mark_partial_aggref(aggref, AGGSPLIT_FINAL_DESERIAL); Var *var = makeVar(OUTER_VAR, tle->resno, var_type, -1, aggref->aggcollid, 0); aggref->args = list_make1(makeTargetEntry((Expr *) var, 1, NULL, false)); return node; } } return expression_tree_mutator(node, make_finalize_agg_mutator, context); } static Plan *insert_vector_agg(Plan *plan, void *context); Plan * try_insert_vector_agg_node(Plan *plan) { return ts_plan_tree_walker(plan, insert_vector_agg, NULL); } static Plan * insert_vector_agg(Plan *plan, void *context) { if (!IsA(plan, Agg)) { return plan; } Agg *agg = castNode(Agg, plan); if (agg->aggsplit != AGGSPLIT_INITIAL_SERIAL && agg->aggsplit != AGGSPLIT_SIMPLE) { /* Can only vectorize partial or non-partial aggregation node. */ return plan; } if (agg->groupingSets != NIL) { /* No GROUPING SETS support. */ return plan; } if (agg->plan.qual != NIL) { /* * No HAVING support. Probably we can't have it in this node in any case, * because we only replace the partial aggregation nodes which can't * check the HAVING clause. */ return plan; } if (agg->plan.lefttree == NULL) { /* * Not sure what this would mean, but check for it just to be on the * safe side because we can effectively see any possible plan here. */ return plan; } Plan *childplan = agg->plan.lefttree; VectorQualInfo vqi; MemSet(&vqi, 0, sizeof(VectorQualInfo)); /* * Build supplementary info to determine whether we can vectorize the * aggregate FILTER clauses. */ if (!vectoragg_plan_possible(childplan, &vqi)) { /* Not a compatible vectoragg child node */ return plan; } /* * To make it easier to examine the variables participating in the aggregation, * the subsequent checks are performed on the aggregated targetlist with * all variables resolved to uncompressed chunk variables. */ List *resolved_targetlist = castNode(List, ts_resolve_outer_special_vars((Node *) agg->plan.targetlist, childplan)); const VectorAggGroupingType grouping_type = get_vectorized_grouping_type(&vqi, agg, resolved_targetlist); if (grouping_type == VAGT_Invalid) { /* The grouping is not vectorizable. */ return plan; } /* * The hash grouping strategies do not preserve the input key order when the * reverse ordering is requested, so in this case they cannot work in * GroupAggregate mode. */ if (grouping_type != VAGT_Batch && agg->aggstrategy != AGG_HASHED) { if (vqi.reverse) { return plan; } } /* Now check the output targetlist. */ ListCell *lc; foreach (lc, resolved_targetlist) { TargetEntry *target_entry = lfirst_node(TargetEntry, lc); if (IsA(target_entry->expr, Aggref)) { Aggref *aggref = castNode(Aggref, target_entry->expr); if (!can_vectorize_aggref(&vqi, aggref)) { /* Aggregate function not vectorizable. */ return plan; } } } /* * Finally, all requirements are satisfied and we can vectorize this * aggregation node. */ Plan *vector_agg_plan = vector_agg_plan_create(childplan, agg, resolved_targetlist, grouping_type); if (agg->aggsplit == AGGSPLIT_SIMPLE) { /* * Convert a non-partial aggregation into a two-phase partial + finalize * aggregation with VectorAgg performing the partial step. */ CustomScan *vector_agg = castNode(CustomScan, vector_agg_plan); vector_agg->custom_scan_tlist = (List *) expression_tree_mutator((Node *) vector_agg->custom_scan_tlist, mark_partial_aggref_mutator, NULL); /* * Rebuild the plan output targetlist to reflect the updated types. * VectorAgg returns ps_ResultTupleSlot whose TupleDesc is derived from * plan.targetlist, so it must match the actual partial aggregate output * types for correct tuple materialization on all platforms. */ vector_agg->scan.plan.targetlist = ts_build_trivial_custom_output_targetlist(vector_agg->custom_scan_tlist); /* * Set up the parent Agg to finalize the partial results from VectorAgg. */ agg->aggsplit = AGGSPLIT_FINAL_DESERIAL; agg->plan.lefttree = vector_agg_plan; MakeFinalizeAggContext finalize_ctx = { .agg = agg, .vector_agg_targetlist = vector_agg->scan.plan.targetlist, }; agg->plan.targetlist = (List *) expression_tree_mutator((Node *) agg->plan.targetlist, make_finalize_agg_mutator, &finalize_ctx); agg->plan.qual = (List *) expression_tree_mutator((Node *) agg->plan.qual, make_finalize_agg_mutator, &finalize_ctx); return (Plan *) agg; } return vector_agg_plan; } ================================================ FILE: tsl/src/nodes/vector_agg/plan.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <nodes/plannodes.h> #include <utils/relcache.h> #include "nodes/columnar_scan/vector_quals.h" /* * The indexes of settings that we have to pass through the custom_private list. */ typedef enum { VASI_GroupingType = 0, VASI_Count } VectorAggSettingsIndex; extern void _vector_agg_init(void); extern void vectoragg_plan_columnar_scan(Plan *childplan, VectorQualInfo *vqi); Plan *try_insert_vector_agg_node(Plan *plan); bool has_vector_agg_node(Plan *plan, bool *has_some_agg); bool ts_is_vector_agg_plan(Plan *plan); ================================================ FILE: tsl/src/nodes/vector_agg/plan_columnar_scan.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <nodes/pathnodes.h> #include <nodes/plannodes.h> #include "nodes/columnar_scan/planner.h" #include "plan.h" /* * Whether the given compressed column index corresponds to a vector variable. */ static bool is_vector_compressed_column(const CustomScan *custom, int compressed_column_index, bool *out_is_segmentby) { List *bulk_decompression_column = list_nth(custom->custom_private, DCP_BulkDecompressionColumn); const bool bulk_decompression_enabled_for_column = list_nth_int(bulk_decompression_column, compressed_column_index); /* * Bulk decompression can be disabled for all columns in the ColumnarScan * node settings, we can't do vectorized aggregation for compressed columns * in that case. For segmentby columns it's still possible. */ List *settings = linitial(custom->custom_private); const bool bulk_decompression_enabled_globally = list_nth_int(settings, DCS_EnableBulkDecompression); /* * Check if this column is a segmentby. */ List *is_segmentby_column = list_nth(custom->custom_private, DCP_IsSegmentbyColumn); const bool is_segmentby = list_nth_int(is_segmentby_column, compressed_column_index); if (out_is_segmentby) { *out_is_segmentby = is_segmentby; } /* * We support vectorized aggregation either for segmentby columns or for * columns with bulk decompression enabled. */ if (!is_segmentby && !(bulk_decompression_enabled_for_column && bulk_decompression_enabled_globally)) { /* Vectorized aggregation not possible for this particular column. */ return false; } return true; } /* * Map the custom scan attribute number to the uncompressed chunk attribute * number. */ static int custom_scan_to_uncompressed_chunk_attno(List *custom_scan_tlist, int custom_scan_attno) { if (custom_scan_tlist == NIL) { return custom_scan_attno; } Var *var = castNode(Var, castNode(TargetEntry, list_nth(custom_scan_tlist, AttrNumberGetAttrOffset(custom_scan_attno))) ->expr); return var->varattno; } void vectoragg_plan_columnar_scan(Plan *childplan, VectorQualInfo *vqi) { const CustomScan *custom = castNode(CustomScan, childplan); vqi->rti = custom->scan.scanrelid; /* * Now, we have to translate the decompressed varno into the compressed * column index, to check if the column supports bulk decompression. */ List *decompression_map = list_nth(custom->custom_private, DCP_DecompressionMap); /* * There's no easy way to determine maximum attribute number for uncompressed * chunk at this stage, so we'll have to go through all the compressed columns * for this. */ int maxattno = 0; for (int compressed_column_index = 0; compressed_column_index < list_length(decompression_map); compressed_column_index++) { const int custom_scan_attno = list_nth_int(decompression_map, compressed_column_index); if (custom_scan_attno <= 0) { continue; } const int uncompressed_chunk_attno = custom_scan_to_uncompressed_chunk_attno(custom->custom_scan_tlist, custom_scan_attno); if (uncompressed_chunk_attno > maxattno) { maxattno = uncompressed_chunk_attno; } } vqi->maxattno = maxattno; vqi->vector_attrs = (bool *) palloc0(sizeof(bool) * (maxattno + 1)); vqi->segmentby_attrs = (bool *) palloc0(sizeof(bool) * (maxattno + 1)); for (int compressed_column_index = 0; compressed_column_index < list_length(decompression_map); compressed_column_index++) { const int custom_scan_attno = list_nth_int(decompression_map, compressed_column_index); if (custom_scan_attno <= 0) { continue; } const int uncompressed_chunk_attno = custom_scan_to_uncompressed_chunk_attno(custom->custom_scan_tlist, custom_scan_attno); vqi->vector_attrs[uncompressed_chunk_attno] = is_vector_compressed_column(custom, compressed_column_index, &vqi->segmentby_attrs[uncompressed_chunk_attno]); } List *settings = linitial(custom->custom_private); vqi->reverse = list_nth_int(settings, DCS_Reverse); } ================================================ FILE: tsl/src/nodes/vector_agg/vector_slot.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <access/attnum.h> #include <access/tupdesc.h> #include <nodes/columnar_scan/compressed_batch.h> /* * Vector slot functions. * * These functions provide a common interface for arrow slots and compressed * batches. * */ /* * Get the result vectorized filter bitmap. */ static inline const uint64 * vector_slot_get_qual_result(const TupleTableSlot *slot, uint16 *num_rows) { const DecompressBatchState *batch_state = (const DecompressBatchState *) slot; *num_rows = batch_state->total_batch_rows; return batch_state->vector_qual_result; } /* * Return the arrow array or the datum (in case of single scalar value) for a * given attribute as a CompressedColumnValues struct. */ CompressedColumnValues vector_slot_evaluate_expression(DecompressContext *dcontext, TupleTableSlot *slot, uint64 const *filter, const Expr *argument); ================================================ FILE: tsl/src/planner.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <catalog/pg_trigger.h> #include <commands/extension.h> #include <foreign/fdwapi.h> #include <nodes/nodeFuncs.h> #include <nodes/parsenodes.h> #include <optimizer/pathnode.h> #include <optimizer/paths.h> #include <parser/parsetree.h> #include "compat/compat.h" #include "chunk.h" #include "chunkwise_agg.h" #include "continuous_aggs/planner.h" #include "guc.h" #include "hypertable.h" #include "nodes/columnar_index_scan/columnar_index_scan.h" #include "nodes/columnar_scan/columnar_scan.h" #include "nodes/gapfill/gapfill.h" #include "nodes/skip_scan/skip_scan.h" #include "nodes/vector_agg/plan.h" #include "planner.h" #include <math.h> #define OSM_EXTENSION_NAME "timescaledb_osm" static bool involves_hypertable(PlannerInfo *root, RelOptInfo *parent) { for (int relid = bms_next_member(parent->relids, -1); relid > 0; relid = bms_next_member(parent->relids, relid)) { Hypertable *ht; RelOptInfo *child = root->simple_rel_array[relid]; /* * RelOptInfo can be null here for join RTEs on PG >= 16. This doesn't * matter because we'll have all the baserels in relids bitmap as well. */ if (child != NULL && ts_classify_relation(root, child, &ht) == TS_REL_HYPERTABLE) { return true; } } return false; } void tsl_create_upper_paths_hook(PlannerInfo *root, UpperRelationKind stage, RelOptInfo *input_rel, RelOptInfo *output_rel, TsRelType input_reltype, Hypertable *ht, void *extra) { switch (stage) { case UPPERREL_GROUP_AGG: if (input_reltype != TS_REL_HYPERTABLE_CHILD) { plan_add_gapfill(root, output_rel); } if (ts_guc_enable_chunkwise_aggregation && input_rel != NULL && !IS_DUMMY_REL(input_rel) && output_rel != NULL && involves_hypertable(root, input_rel)) { tsl_pushdown_partial_agg(root, ht, input_rel, output_rel, extra); } if (root->numOrderedAggs && !IS_DUMMY_REL(input_rel) && output_rel != NULL) { tsl_skip_scan_paths_add(root, input_rel, output_rel, stage); } break; case UPPERREL_WINDOW: if (IsA(linitial(input_rel->pathlist), CustomPath)) gapfill_adjust_window_targetlist(root, input_rel, output_rel); break; case UPPERREL_DISTINCT: tsl_skip_scan_paths_add(root, input_rel, output_rel, stage); break; default: break; } } /* * Check if a chunk should be decompressed via a ColumnarScan plan. * * Check first that it is a compressed chunk. Then, decompress unless it is * SELECT * FROM ONLY <chunk>. We check if it is the ONLY case by calling * ts_rte_is_marked_for_expansion. Respecting ONLY here is important to not * break postgres tools like pg_dump. */ static inline bool use_columnar_scan(const RelOptInfo *rel, const RangeTblEntry *rte, const Chunk *chunk) { if (!ts_guc_enable_columnarscan) return false; /* Check that the chunk is actually compressed */ return chunk->fd.compressed_chunk_id != INVALID_CHUNK_ID && /* Check that it is _not_ SELECT FROM ONLY <chunk> */ (rel->reloptkind != RELOPT_BASEREL || ts_rte_is_marked_for_expansion(rte)); } void tsl_set_rel_pathlist_query(PlannerInfo *root, RelOptInfo *rel, Index rti, RangeTblEntry *rte, Hypertable *ht) { /* Only interested in queries on relations that are part of hypertables * with compression enabled, so quick exit if not this case. */ if (ht == NULL || !TS_HYPERTABLE_HAS_COMPRESSION_TABLE(ht)) return; /* * For a chunk, we can get here via a query on the hypertable that expands * to the chunk or by direct query on the chunk. In the former case, * reloptkind will be RELOPT_OTHER_MEMBER_REL (nember of hypertable) or in * the latter case reloptkind will be RELOPT_BASEREL (standalone rel). * * These two cases are checked in ts_planner_chunk_fetch(). */ const Chunk *chunk = ts_planner_chunk_fetch(root, rel); if (chunk == NULL) return; if (use_columnar_scan(rel, rte, chunk)) { ts_columnar_scan_generate_paths(root, rel, ht, chunk); } } void tsl_set_rel_pathlist_dml(PlannerInfo *root, RelOptInfo *rel, Index rti, RangeTblEntry *rte, Hypertable *ht) { /* * We do not support MERGE command with UPDATE/DELETE merge actions on * compressed hypertables, because Custom Scan (ModifyHypertable) node is * not generated in the plan for MERGE command on compressed hypertables */ if (ht != NULL && TS_HYPERTABLE_HAS_COMPRESSION_TABLE(ht)) { if (root->parse->commandType == CMD_MERGE) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("The MERGE command with UPDATE/DELETE merge actions is not support on " "compressed hypertables"))); #if !PG17_GE /* * PG16 and earlier: Remove BitmapHeapScan paths for DML on partial chunks. * * On PG16, BitmapHeapScan eagerly initializes its heap scan descriptor * with the original snapshot during plan initialization. When we * decompress rows and call CommandCounterIncrement(), the stale * snapshot cannot see the newly decompressed rows. * * This bug only affects partial chunks because: * - Fully compressed chunks have rel->indexlist = NIL (set in * timescaledb_get_relation_info_hook), so BitmapHeapScan is not * available anyway. * - Partial chunks have indexes available, so BitmapHeapScan can be * chosen, and decompression will add rows that it cannot see. * * PG17+ fixed this via commit 1577081e961 which lazily initializes * the scan descriptor in BitmapHeapNext(), using the current * estate->es_snapshot after CommandCounterIncrement(). * * IMPORTANT: PostgreSQL's add_path() prunes dominated paths. If * BitmapHeapPath has lower cost than SeqScan (common with adjusted * cost parameters), SeqScan may have been pruned from the pathlist. * If removing BitmapHeapPath would leave no paths, we must add a * another path as fallback to ensure a valid plan exists. */ const Chunk *chunk = ts_planner_chunk_fetch(root, rel); if (chunk && ts_chunk_is_partial(chunk)) { ListCell *lc; List *filtered_paths = NIL; foreach (lc, rel->pathlist) { Path *path = lfirst(lc); if (!IsA(path, BitmapHeapPath)) filtered_paths = lappend(filtered_paths, path); } /* * If removing BitmapHeapPath left us with no paths, try to add * alternative scan paths. This can happen when BitmapHeapPath * dominated and pruned other paths due to cost calculations. * * Prefer IndexScan if available, fall back to SeqScan. */ if (filtered_paths == NIL && rel->pathlist != NIL) { /* * Try to create index paths. create_index_paths() adds paths * to rel->pathlist, but it also creates BitmapHeapPath entries * which we must filter out again. */ rel->pathlist = NIL; /* Clear the BitmapHeapPath */ create_index_paths(root, rel); /* Filter out any BitmapHeapPath that create_index_paths added */ foreach (lc, rel->pathlist) { Path *path = lfirst(lc); if (!IsA(path, BitmapHeapPath)) filtered_paths = lappend(filtered_paths, path); } /* * If no non-bitmap index paths were created (e.g., enable_indexscan=off), * add SeqScan as the final fallback. */ if (filtered_paths == NIL) { Relids required_outer = rel->lateral_relids; Path *seqpath = create_seqscan_path(root, rel, required_outer, 0); filtered_paths = lappend(filtered_paths, seqpath); } } rel->pathlist = filtered_paths; /* Also filter partial_pathlist for parallel plans */ filtered_paths = NIL; foreach (lc, rel->partial_pathlist) { Path *path = lfirst(lc); if (!IsA(path, BitmapHeapPath)) filtered_paths = lappend(filtered_paths, path); } rel->partial_pathlist = filtered_paths; } #endif /* !PG17_GE */ } } /* * Run preprocess query optimizations */ void tsl_preprocess_query(Query *parse, int *cursor_opts) { Assert(parse != NULL); /* Check if constification of watermark values is enabled */ if (ts_guc_enable_cagg_watermark_constify) { constify_cagg_watermark(parse); } #if PG16_GE /* Push down ORDER BY and LIMIT for realtime cagg (PG16+ only) */ if (ts_guc_enable_cagg_sort_pushdown) { cagg_sort_pushdown(parse, cursor_opts); } #endif } /* * Replaces pathkeys in tsl-specific custom path types during sort transformation. * * This hook is called from ts_sort_transform_replace_pathkeys() in sort_transform.c * after the basic pathkey replacement has been performed. It handles tsl-specific * path types (such as ColumnarScan) that contain additional pathkey fields beyond * the standard path.pathkeys field. */ void tsl_sort_transform_replace_pathkeys(void *path, List *transformed_pathkeys, List *original_pathkeys) { if (!path) return; if (ts_is_columnar_scan_path(path)) { ColumnarScanPath *dcpath = (ColumnarScanPath *) path; if (compare_pathkeys(dcpath->required_compressed_pathkeys, transformed_pathkeys) == PATHKEYS_EQUAL) { dcpath->required_compressed_pathkeys = original_pathkeys; } } } /* * Run plan postprocessing optimizations. */ void tsl_postprocess_plan(PlannedStmt *stmt) { if (ts_guc_enable_columnarindexscan) { stmt->planTree = try_insert_columnar_index_scan_node(stmt->planTree, stmt->rtable); stmt->subplans = (List *) try_insert_columnar_index_scan_node((Plan *) stmt->subplans, stmt->rtable); } if (ts_guc_enable_vectorized_aggregation) { stmt->planTree = try_insert_vector_agg_node(stmt->planTree); stmt->subplans = (List *) try_insert_vector_agg_node((Plan *) stmt->subplans); } #ifdef TS_DEBUG if (ts_guc_debug_require_vector_agg != DRO_Allow) { bool has_some_agg = false; const bool has_vector_partial_agg = has_vector_agg_node(stmt->planTree, &has_some_agg); /* * For convenience of using this in the tests, we don't complain about * queries that don't have aggregation at all. */ if (has_some_agg) { if (!has_vector_partial_agg && ts_guc_debug_require_vector_agg == DRO_Require) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("vectorized aggregation node not found when required by the " "debug_require_vector_agg GUC"))); } if (has_vector_partial_agg && ts_guc_debug_require_vector_agg == DRO_Forbid) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("vectorized aggregation node found when forbidden by the " "debug_require_vector_agg GUC"))); } } } #endif } ================================================ FILE: tsl/src/planner.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> #include <optimizer/planner.h> #include "hypertable.h" #include <planner/planner.h> void tsl_create_upper_paths_hook(PlannerInfo *, UpperRelationKind, RelOptInfo *, RelOptInfo *, TsRelType, Hypertable *, void *); void tsl_set_rel_pathlist_query(PlannerInfo *, RelOptInfo *, Index, RangeTblEntry *, Hypertable *); void tsl_set_rel_pathlist_dml(PlannerInfo *, RelOptInfo *, Index, RangeTblEntry *, Hypertable *); void tsl_sort_transform_replace_pathkeys(void *path, List *transformed_pathkeys, List *original_pathkeys); void tsl_preprocess_query(Query *parse, int *cursor_opts); void tsl_postprocess_plan(PlannedStmt *stmt); ================================================ FILE: tsl/src/process_utility.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #include <postgres.h> #include <access/xact.h> #include <catalog/namespace.h> #include <catalog/pg_trigger.h> #include <commands/event_trigger.h> #include <commands/tablecmds.h> #include <nodes/makefuncs.h> #include <nodes/nodes.h> #include <nodes/parsenodes.h> #include <storage/lockdefs.h> #include "compression/create.h" #include "continuous_aggs/create.h" #include "guc.h" #include "hypertable_cache.h" #include "process_utility.h" #include "ts_catalog/continuous_agg.h" /* AlterTableCmds that need tsl side processing invoke this function * we only process AddColumn command right now. */ void tsl_process_altertable_cmd(Hypertable *ht, const AlterTableCmd *cmd) { switch (cmd->subtype) { case AT_AddColumn: #if PG16_LT case AT_AddColumnRecurse: #endif if (TS_HYPERTABLE_HAS_COMPRESSION_TABLE(ht) || TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(ht)) { ColumnDef *orig_coldef = castNode(ColumnDef, cmd->def); tsl_process_compress_table_add_column(ht, orig_coldef); } break; case AT_DropColumn: #if PG16_LT case AT_DropColumnRecurse: #endif if (TS_HYPERTABLE_HAS_COMPRESSION_TABLE(ht) || TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(ht)) { tsl_process_compress_table_drop_column(ht, cmd->name); } break; default: break; } } void tsl_process_rename_cmd(Oid relid, Cache *hcache, const RenameStmt *stmt) { if (stmt->renameType == OBJECT_COLUMN) { /* * process_rename_column() always sets relid to the materialization * hypertable before calling us, so the cache lookup always succeeds. */ Hypertable *ht = ts_hypertable_cache_get_entry(hcache, relid, CACHE_FLAG_MISSING_OK); if (ht) { ContinuousAgg *cagg = ts_continuous_agg_find_by_mat_hypertable_id(ht->fd.id, true); if (cagg) cagg_rename_view_columns(cagg); } if (ht && (TS_HYPERTABLE_HAS_COMPRESSION_TABLE(ht) || TS_HYPERTABLE_HAS_COMPRESSION_ENABLED(ht))) { tsl_process_compress_table_rename_column(ht, stmt); } } } ================================================ FILE: tsl/src/process_utility.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <commands/event_trigger.h> #include <process_utility.h> extern void tsl_process_altertable_cmd(Hypertable *ht, const AlterTableCmd *cmd); extern void tsl_process_rename_cmd(Oid relid, Cache *hcache, const RenameStmt *stmt); extern DDLResult tsl_ddl_command_start(ProcessUtilityArgs *args); extern void tsl_ddl_command_end(EventTriggerData *command); ================================================ FILE: tsl/src/reorder.c ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ /* * This file contains source code that was copied and/or modified from the * PostgreSQL database, which is licensed under the open-source PostgreSQL * License. Please see the NOTICE at the top level directory for a copy of * the PostgreSQL License. */ /* see postgres commit ab5e9caa4a3ec4765348a0482e88edcf3f6aab4a */ #include <postgres.h> #include <access/amapi.h> #include <access/multixact.h> #include <access/relscan.h> #include <access/rewriteheap.h> #include <access/transam.h> #include <access/xact.h> #include <access/xlog.h> #include <catalog/catalog.h> #include <catalog/dependency.h> #include <catalog/heap.h> #include <catalog/index.h> #include <catalog/namespace.h> #include <catalog/objectaccess.h> #include <catalog/pg_am.h> #include <catalog/pg_authid.h> #include <catalog/pg_tablespace_d.h> #include <catalog/toasting.h> #include <commands/cluster.h> #include <commands/tablecmds.h> #include <commands/tablespace.h> #include <commands/vacuum.h> #include <executor/spi.h> #include <miscadmin.h> #include <nodes/pg_list.h> #include <optimizer/planner.h> #include <storage/bufmgr.h> #include <storage/lmgr.h> #include <storage/lockdefs.h> #include <storage/predicate.h> #include <storage/smgr.h> #include <tcop/tcopprot.h> #include <utils/acl.h> #include <utils/builtins.h> #include <utils/fmgroids.h> #include <utils/guc.h> #include <utils/inval.h> #include <utils/lsyscache.h> #include <utils/memutils.h> #include <utils/pg_rusage.h> #include <utils/relmapper.h> #include <utils/snapmgr.h> #include <utils/syscache.h> #include <utils/tuplesort.h> #include <access/toast_internals.h> #include "chunk.h" #include "chunk_index.h" #include "hypertable_cache.h" #include "import/heapswap.h" #include "indexing.h" #include "reorder.h" static void reorder_rel(Oid tableOid, Oid indexOid, bool verbose, Oid wait_id, Oid destination_tablespace, Oid index_tablespace); #define REORDER_ACCESS_EXCLUSIVE_DEADLOCK_TIMEOUT "101000" static void rebuild_relation(Relation OldHeap, Oid indexOid, bool verbose, Oid wait_id, Oid destination_tablespace, Oid index_tablespace); static void copy_heap_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex, bool verbose, bool *pSwapToastByContent, TransactionId *pFreezeXid, MultiXactId *pCutoffMulti); static Oid chunk_get_reorder_index(Hypertable *ht, Chunk *chunk, Oid index_relid); Datum tsl_reorder_chunk(PG_FUNCTION_ARGS) { Oid chunk_id = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); Oid index_id = PG_ARGISNULL(1) ? InvalidOid : PG_GETARG_OID(1); bool verbose = PG_ARGISNULL(2) ? false : PG_GETARG_BOOL(2); /* used for debugging purposes only see finish_heap_swaps */ Oid wait_id = PG_NARGS() < 4 || PG_ARGISNULL(3) ? InvalidOid : PG_GETARG_OID(3); /* * Allow reorder in transactions for testing purposes only */ if (!OidIsValid(wait_id)) PreventInTransactionBlock(true, "reorder"); reorder_chunk(chunk_id, index_id, verbose, wait_id, InvalidOid, InvalidOid); PG_RETURN_VOID(); } Datum tsl_move_chunk(PG_FUNCTION_ARGS) { Oid chunk_id = PG_ARGISNULL(0) ? InvalidOid : PG_GETARG_OID(0); Oid destination_tablespace = PG_ARGISNULL(1) ? InvalidOid : get_tablespace_oid(PG_GETARG_NAME(1)->data, false); Oid index_destination_tablespace = PG_ARGISNULL(2) ? InvalidOid : get_tablespace_oid(PG_GETARG_NAME(2)->data, false); Oid index_id = PG_ARGISNULL(3) ? InvalidOid : PG_GETARG_OID(3); bool verbose = PG_ARGISNULL(4) ? false : PG_GETARG_BOOL(4); Chunk *chunk; /* used for debugging purposes only see finish_heap_swaps */ Oid wait_id = PG_NARGS() < 6 || PG_ARGISNULL(5) ? InvalidOid : PG_GETARG_OID(5); /* * Allow move in transactions for testing purposes only */ if (!OidIsValid(wait_id)) PreventInTransactionBlock(true, "move"); /* * Index_destination_tablespace is currently a required parameter in order * to avoid situations where there is ambiguity about where indexes should * be placed based on where the index was created and the new tablespace * (and avoid interactions with multi-tablespace hypertable functionality). * Eventually we may want to offer an option to keep indexes in the * tablespace of their parent if it is specified. */ if (!OidIsValid(chunk_id) || !OidIsValid(destination_tablespace) || !OidIsValid(index_destination_tablespace)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("valid chunk, destination_tablespace, and index_destination_tablespaces " "are required"))); chunk = ts_chunk_get_by_relid(chunk_id, false); if (NULL == chunk) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("\"%s\" is not a chunk", get_rel_name(chunk_id)))); if (ts_chunk_contains_compressed_data(chunk)) { Chunk *chunk_parent = ts_chunk_get_compressed_chunk_parent(chunk); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot directly move internal columnstore data"), errdetail("Chunk \"%s\" contains columnstore data for chunk \"%s\" and cannot be " "moved directly.", get_rel_name(chunk_id), get_rel_name(chunk_parent->table_id)), errhint("Moving chunk \"%s\" will also move the columnstore data.", get_rel_name(chunk_parent->table_id)))); } /* If chunk is compressed move it by altering tablespace on both chunks */ if (OidIsValid(chunk->fd.compressed_chunk_id)) { Chunk *compressed_chunk = ts_chunk_get_by_id(chunk->fd.compressed_chunk_id, true); AlterTableCmd cmd = { .type = T_AlterTableCmd, .subtype = AT_SetTableSpace, .name = get_tablespace_name(destination_tablespace) }; if (OidIsValid(index_id)) ereport(NOTICE, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("ignoring index parameter"), errdetail("Chunk will not be reordered as it has columnstore data."))); ts_alter_table_with_event_trigger(chunk_id, fcinfo->context, list_make1(&cmd), false); ts_alter_table_with_event_trigger(compressed_chunk->table_id, fcinfo->context, list_make1(&cmd), false); /* move indexes on original and compressed chunk */ ts_chunk_index_move_all(chunk_id, index_destination_tablespace); ts_chunk_index_move_all(compressed_chunk->table_id, index_destination_tablespace); } else { reorder_chunk(chunk_id, index_id, verbose, wait_id, destination_tablespace, index_destination_tablespace); } PG_RETURN_VOID(); } void reorder_chunk(Oid chunk_id, Oid index_id, bool verbose, Oid wait_id, Oid destination_tablespace, Oid index_tablespace) { Chunk *chunk; Cache *hcache; Hypertable *ht; if (!OidIsValid(chunk_id)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("must provide a valid chunk to cluster"))); chunk = ts_chunk_get_by_relid(chunk_id, false); if (NULL == chunk) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("\"%s\" is not a chunk", get_rel_name(chunk_id)))); ht = ts_hypertable_cache_get_cache_and_entry(chunk->hypertable_relid, CACHE_FLAG_NONE, &hcache); /* Our check gives better error messages, but keep the original one too. */ ts_hypertable_permissions_check(ht->main_table_relid, GetUserId()); if (!object_ownercheck(RelationRelationId, ht->main_table_relid, GetUserId())) { Oid main_table_relid = ht->main_table_relid; ts_cache_release(&hcache); aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_TABLE, get_rel_name(main_table_relid)); } Oid index_relid = chunk_get_reorder_index(ht, chunk, index_id); if (!OidIsValid(index_relid)) { ts_cache_release(&hcache); if (OidIsValid(index_id)) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("\"%s\" is not a valid clustering index for table \"%s\"", get_rel_name(index_id), get_rel_name(chunk_id)))); else ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("there is no previously clustered index for table \"%s\"", get_rel_name(chunk_id)))); } if (OidIsValid(destination_tablespace) && destination_tablespace != MyDatabaseTableSpace) { AclResult aclresult; aclresult = object_aclcheck(TableSpaceRelationId, destination_tablespace, GetUserId(), ACL_CREATE); if (aclresult != ACLCHECK_OK) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permission denied for tablespace \"%s\"", get_tablespace_name(destination_tablespace)))); ; } if (OidIsValid(index_tablespace) && index_tablespace != MyDatabaseTableSpace) { AclResult aclresult; aclresult = object_aclcheck(TableSpaceRelationId, index_tablespace, GetUserId(), ACL_CREATE); if (aclresult != ACLCHECK_OK) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("permission denied for tablespace \"%s\"", get_tablespace_name(index_tablespace)))); } /* * We must mark each chunk index as clustered before calling reorder_rel() * because it expects indexes that need to be rechecked (due to new * transaction) to already have that mark set */ ts_chunk_index_mark_clustered(chunk->table_id, index_relid); reorder_rel(chunk->table_id, index_relid, verbose, wait_id, destination_tablespace, index_tablespace); ts_cache_release(&hcache); } static bool index_belongs_to_relation(Oid relid, Oid index_oid) { HeapTuple tuple; Form_pg_index rd_index; bool result; tuple = SearchSysCache1(INDEXRELID, ObjectIdGetDatum(index_oid)); if (!HeapTupleIsValid(tuple)) return false; rd_index = (Form_pg_index) GETSTRUCT(tuple); result = rd_index->indrelid == relid; ReleaseSysCache(tuple); return result; } /* * Find the index to reorder a chunk on based on a possibly NULL indexname * returns InvalidOid if no such index is found */ static Oid chunk_get_reorder_index(Hypertable *ht, Chunk *chunk, Oid index_relid) { /* * Index search order: 1. Explicitly named index 2. Chunk cluster index 3. * - index belongs to the chunk * - index belongs to the hypertable * - any clustered index on the chunk * - any clustered index on the hypertable */ if (OidIsValid(index_relid)) { if (index_belongs_to_relation(chunk->table_id, index_relid)) return index_relid; if (index_belongs_to_relation(ht->main_table_relid, index_relid)) { Relation chunk_rel = table_open(chunk->table_id, AccessShareLock); Oid chunk_index_oid = ts_chunk_index_get_by_hypertable_indexrelid(chunk_rel, index_relid); table_close(chunk_rel, NoLock); return chunk_index_oid; } return InvalidOid; } index_relid = ts_indexing_find_clustered_index(chunk->table_id); if (OidIsValid(index_relid)) return index_relid; index_relid = ts_indexing_find_clustered_index(ht->main_table_relid); if (OidIsValid(index_relid)) { Relation chunk_rel = table_open(chunk->table_id, AccessShareLock); Oid chunk_index_oid = ts_chunk_index_get_by_hypertable_indexrelid(chunk_rel, index_relid); table_close(chunk_rel, NoLock); return chunk_index_oid; } return InvalidOid; } /* The following functions are based on their equivalents in postgres's cluster.c */ /* * reorder_rel * * This clusters the table by creating a new, clustered table and * swapping the relfilenodes of the new table and the old table, so * the OID of the original table is preserved. * * Indexes are rebuilt in the same manner. */ static void reorder_rel(Oid tableOid, Oid indexOid, bool verbose, Oid wait_id, Oid destination_tablespace, Oid index_tablespace) { Relation OldHeap; HeapTuple tuple; Form_pg_index indexForm; if (!OidIsValid(indexOid)) elog(ERROR, "Reorder must specify an index."); /* Check for user-requested abort. */ CHECK_FOR_INTERRUPTS(); /* * We grab exclusive access to the target rel and index for the duration * of the transaction. (This is redundant for the single-transaction * case, since cluster() already did it.) The index lock is taken inside * check_index_is_clusterable. */ OldHeap = try_relation_open(tableOid, ExclusiveLock); /* If the table has gone away, we can skip processing it */ if (!OldHeap) { ereport(WARNING, (errcode(ERRCODE_WARNING), errmsg("table disappeared during reorder"))); return; } /* * Since we may open a new transaction for each relation, we have to check * that the relation still is what we think it is. */ /* Check that the user still owns the relation */ if (!object_ownercheck(RelationRelationId, tableOid, GetUserId())) { relation_close(OldHeap, ExclusiveLock); ereport(WARNING, (errcode(ERRCODE_WARNING), errmsg("ownership changed during reorder"))); return; } if (IsSystemRelation(OldHeap)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot reorder a system relation"))); if (OldHeap->rd_rel->relpersistence != RELPERSISTENCE_PERMANENT) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("can only reorder a permanent table"))); /* We do not allow reordering on shared catalogs. */ if (OldHeap->rd_rel->relisshared) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot reorder a shared catalog"))); if (OldHeap->rd_rel->relkind != RELKIND_RELATION) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("can only reorder a relation"))); /* * Check that the index still exists */ if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexOid))) { ereport(WARNING, (errcode(ERRCODE_WARNING), errmsg("index disappeared during reorder"))); relation_close(OldHeap, ExclusiveLock); return; } /* * Check that the index is still the one with indisclustered set. */ tuple = SearchSysCache1(INDEXRELID, ObjectIdGetDatum(indexOid)); if (!HeapTupleIsValid(tuple)) /* probably can't happen */ { ereport(WARNING, (errcode(ERRCODE_WARNING), errmsg("invalid index heap during reorder"))); relation_close(OldHeap, ExclusiveLock); return; } indexForm = (Form_pg_index) GETSTRUCT(tuple); /* * We always mark indexes as clustered when we intercept a cluster * command, if it's not marked as such here, something has gone wrong */ if (!indexForm->indisclustered) ereport(ERROR, (errcode(ERRCODE_ASSERT_FAILURE), errmsg("invalid index heap during reorder"))); ReleaseSysCache(tuple); /* * Also check for active uses of the relation in the current transaction, * including open scans and pending AFTER trigger events. */ CheckTableNotInUse(OldHeap, "CLUSTER"); /* Check heap and index are valid to cluster on */ check_index_is_clusterable(OldHeap, indexOid, ExclusiveLock); /* rebuild_relation does all the dirty work */ rebuild_relation(OldHeap, indexOid, verbose, wait_id, destination_tablespace, index_tablespace); /* NB: rebuild_relation does table_close() on OldHeap */ } static void reorder_finish_heap_swaps(Oid OIDOldHeap, Oid OIDNewHeap, char relpersistence, List *old_index_oids, List *new_index_oids, bool swap_toast_by_content, bool is_internal, TransactionId frozenXid, MultiXactId cutoffMulti, Oid wait_id) { ListCell *old_index_cell; ListCell *new_index_cell; Oid mapped_tables[4]; int config_change; #ifdef DEBUG /* * For debug purposes we serialize against wait_id if it exists, this * allows us to "pause" reorder immediately before swapping in the new * table */ if (OidIsValid(wait_id)) { Relation waiter = table_open(wait_id, AccessExclusiveLock); table_close(waiter, AccessExclusiveLock); } #endif /* * There's a risk of deadlock if some other process is also trying to * upgrade their lock in the same manner as us, at this time. Since our * transaction has performed a large amount of work, and only needs to be * run once per chunk, we do not want to abort it due to this deadlock. To * prevent abort we set our `deadlock_timeout` to a large value in the * expectation that the other process will timeout and abort first. * Currently we set `deadlock_timeout` to 1 hour, as this should be longer * than any other normal process, while still allowing the system to make * progress in the event of a real deadlock. As this is the last lock we * grab, and the setting is local to our transaction we do not bother * changing the guc back. */ config_change = set_config_option("deadlock_timeout", REORDER_ACCESS_EXCLUSIVE_DEADLOCK_TIMEOUT, PGC_SUSET, PGC_S_SESSION, GUC_ACTION_LOCAL, true, 0, false); if (config_change == 0) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("deadlock_timeout guc does not exist."))); else if (config_change < 0) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("could not set deadlock_timeout guc."))); /* Upgrade to an AccessExclusiveLock for the heap swap */ LockRelationOid(OIDOldHeap, AccessExclusiveLock); /* Swap the contents of the indexes */ Assert(list_length(old_index_oids) == list_length(new_index_oids)); forboth (old_index_cell, old_index_oids, new_index_cell, new_index_oids) { Oid old_index_oid = lfirst_oid(old_index_cell); Oid new_index_oid = lfirst_oid(new_index_cell); ts_swap_relation_files(old_index_oid, new_index_oid, false, swap_toast_by_content, true, frozenXid, cutoffMulti, mapped_tables); } /* Old indexes must be visible for deletion */ CommandCounterIncrement(); /* * Swap the physical files of the target and transient tables, then * rebuild the target's indexes and throw away the transient table. */ ts_finish_heap_swap(OIDOldHeap, OIDNewHeap, false, swap_toast_by_content, true, true, true, frozenXid, cutoffMulti, relpersistence); } /* * rebuild_relation: rebuild an existing relation in index or physical order * * OldHeap: table to rebuild --- must be opened and exclusive-locked! * indexOid: index to cluster by, or InvalidOid to rewrite in physical order. * * NB: this routine closes OldHeap at the right time; caller should not. */ static void rebuild_relation(Relation OldHeap, Oid indexOid, bool verbose, Oid wait_id, Oid destination_tablespace, Oid index_tablespace) { Oid tableOid = RelationGetRelid(OldHeap); Oid tableSpace = OidIsValid(destination_tablespace) ? destination_tablespace : OldHeap->rd_rel->reltablespace; Oid OIDNewHeap; List *old_index_oids; List *new_index_oids; char relpersistence; bool swap_toast_by_content; TransactionId frozenXid; MultiXactId cutoffMulti; /* Mark the correct index as clustered */ mark_index_clustered(OldHeap, indexOid, true); /* Remember info about rel before closing OldHeap */ relpersistence = OldHeap->rd_rel->relpersistence; /* Close relcache entry, but keep lock until transaction commit */ table_close(OldHeap, NoLock); /* Create the transient table that will receive the re-ordered data */ OIDNewHeap = make_new_heap(tableOid, tableSpace, OldHeap->rd_rel->relam, relpersistence, ExclusiveLock); /* Copy the heap data into the new table in the desired order */ copy_heap_data(OIDNewHeap, tableOid, indexOid, verbose, &swap_toast_by_content, &frozenXid, &cutoffMulti); /* Create versions of the tables indexes for the new table */ new_index_oids = ts_chunk_index_duplicate(tableOid, OIDNewHeap, &old_index_oids, index_tablespace); reorder_finish_heap_swaps(tableOid, OIDNewHeap, relpersistence, old_index_oids, new_index_oids, swap_toast_by_content, true, frozenXid, cutoffMulti, wait_id); } /* * Do the physical copying of heap data. * * There are three output parameters: * *pSwapToastByContent is set true if toast tables must be swapped by content. * *pFreezeXid receives the TransactionId used as freeze cutoff point. * *pCutoffMulti receives the MultiXactId used as a cutoff point. */ static void copy_heap_data(Oid OIDNewHeap, Oid OIDOldHeap, Oid OIDOldIndex, bool verbose, bool *pSwapToastByContent, TransactionId *pFreezeXid, MultiXactId *pCutoffMulti) { Relation NewHeap, OldHeap, OldIndex; Relation relRelation; HeapTuple reltup; Form_pg_class relform; TupleDesc PG_USED_FOR_ASSERTS_ONLY oldTupDesc; TupleDesc newTupDesc; int natts; Datum *values; bool *isnull; bool use_sort; double num_tuples = 0, tups_vacuumed = 0, tups_recently_dead = 0; BlockNumber num_pages; int elevel = verbose ? INFO : DEBUG2; PGRUsage ru0; pg_rusage_init(&ru0); /* * Open the relations we need. */ NewHeap = table_open(OIDNewHeap, AccessExclusiveLock); OldHeap = table_open(OIDOldHeap, ExclusiveLock); if (OidIsValid(OIDOldIndex)) OldIndex = index_open(OIDOldIndex, ExclusiveLock); else OldIndex = NULL; /* * Their tuple descriptors should be exactly alike, but here we only need * assume that they have the same number of columns. */ oldTupDesc = RelationGetDescr(OldHeap); newTupDesc = RelationGetDescr(NewHeap); Assert(newTupDesc->natts == oldTupDesc->natts); /* Preallocate values/isnull arrays */ natts = newTupDesc->natts; values = (Datum *) palloc(natts * sizeof(Datum)); isnull = (bool *) palloc(natts * sizeof(bool)); /* * If the OldHeap has a toast table, get lock on the toast table to keep * it from being vacuumed. This is needed because autovacuum processes * toast tables independently of their main tables, with no lock on the * latter. If an autovacuum were to start on the toast table after we * compute our OldestXmin below, it would use a later OldestXmin, and then * possibly remove as DEAD toast tuples belonging to main tuples we think * are only RECENTLY_DEAD. Then we'd fail while trying to copy those * tuples. * * We don't need to open the toast relation here, just lock it. The lock * will be held till end of transaction. */ if (OldHeap->rd_rel->reltoastrelid) LockRelationOid(OldHeap->rd_rel->reltoastrelid, ExclusiveLock); /* use_wal off requires smgr_targblock be initially invalid */ Assert(RelationGetTargetBlock(NewHeap) == InvalidBlockNumber); /* * If both tables have TOAST tables, perform toast swap by content. It is * possible that the old table has a toast table but the new one doesn't, * if toastable columns have been dropped. In that case we have to do * swap by links. This is okay because swap by content is only essential * for system catalogs, and we don't support schema changes for them. */ if (OldHeap->rd_rel->reltoastrelid && NewHeap->rd_rel->reltoastrelid) { *pSwapToastByContent = true; /* * When doing swap by content, any toast pointers written into NewHeap * must use the old toast table's OID, because that's where the toast * data will eventually be found. Set this up by setting rd_toastoid. * This also tells toast_save_datum() to preserve the toast value * OIDs, which we want so as not to invalidate toast pointers in * system catalog caches, and to avoid making multiple copies of a * single toast value. * * Note that we must hold NewHeap open until we are done writing data, * since the relcache will not guarantee to remember this setting once * the relation is closed. Also, this technique depends on the fact * that no one will try to read from the NewHeap until after we've * finished writing it and swapping the rels --- otherwise they could * follow the toast pointers to the wrong place. (It would actually * work for values copied over from the old toast table, but not for * any values that we toast which were previously not toasted.) */ NewHeap->rd_toastoid = OldHeap->rd_rel->reltoastrelid; } else *pSwapToastByContent = false; /* * Compute xids used to freeze and weed out dead tuples and multixacts. * Since we're going to rewrite the whole table anyway, there's no reason * not to be aggressive about this. */ struct VacuumCutoffs cutoffs; VacuumParams params; memset(¶ms, 0, sizeof(VacuumParams)); vacuum_get_cutoffs(OldHeap, ¶ms, &cutoffs); /* * FreezeXid will become the table's new relfrozenxid, and that mustn't go * backwards, so take the max. */ { TransactionId relfrozenxid = OldHeap->rd_rel->relfrozenxid; if (TransactionIdIsValid(relfrozenxid) && TransactionIdPrecedes(cutoffs.FreezeLimit, relfrozenxid)) cutoffs.FreezeLimit = relfrozenxid; } /* * MultiXactCutoff, similarly, shouldn't go backwards either. */ { MultiXactId relminmxid = OldHeap->rd_rel->relminmxid; if (MultiXactIdIsValid(relminmxid) && MultiXactIdPrecedes(cutoffs.MultiXactCutoff, relminmxid)) cutoffs.MultiXactCutoff = relminmxid; } /* return selected values to caller */ *pFreezeXid = cutoffs.FreezeLimit; *pCutoffMulti = cutoffs.MultiXactCutoff; /* * We know how to use a sort to duplicate the ordering of a btree index, * and will use seqscan-and-sort for that. Otherwise, always use an * indexscan for other indexes or plain seqscan if no index is supplied. */ if (OldIndex != NULL && OldIndex->rd_rel->relam == BTREE_AM_OID) use_sort = true; else use_sort = false; /* Log what we're doing */ if (OldIndex != NULL && !use_sort) ereport(elevel, (errmsg("reordering \"%s.%s\" using index scan on \"%s\"", get_namespace_name(RelationGetNamespace(OldHeap)), RelationGetRelationName(OldHeap), RelationGetRelationName(OldIndex)))); else if (use_sort) ereport(elevel, (errmsg("reordering \"%s.%s\" using sequential scan and sort", get_namespace_name(RelationGetNamespace(OldHeap)), RelationGetRelationName(OldHeap)))); else ereport(ERROR, (errmsg("tried to use a reorder without an index \"%s.%s\"", get_namespace_name(RelationGetNamespace(OldHeap)), RelationGetRelationName(OldHeap)))); table_relation_copy_for_cluster(OldHeap, NewHeap, OldIndex, use_sort, cutoffs.OldestXmin, &cutoffs.FreezeLimit, &cutoffs.MultiXactCutoff, &num_tuples, &tups_vacuumed, &tups_recently_dead); /* Reset rd_toastoid just to be tidy --- it shouldn't be looked at again */ NewHeap->rd_toastoid = InvalidOid; num_pages = RelationGetNumberOfBlocks(NewHeap); /* Log what we did */ ereport(elevel, (errmsg("\"%s\": found %.0f removable, %.0f nonremovable row versions in %u pages", RelationGetRelationName(OldHeap), tups_vacuumed, num_tuples, RelationGetNumberOfBlocks(OldHeap)), errdetail("%.0f dead row versions cannot be removed yet.\n" "%s.", tups_recently_dead, pg_rusage_show(&ru0)))); /* Clean up */ pfree(values); pfree(isnull); if (OldIndex != NULL) index_close(OldIndex, NoLock); table_close(OldHeap, NoLock); table_close(NewHeap, NoLock); /* Update pg_class to reflect the correct values of pages and tuples. */ relRelation = table_open(RelationRelationId, RowExclusiveLock); reltup = SearchSysCacheCopy1(RELOID, ObjectIdGetDatum(OIDNewHeap)); if (!HeapTupleIsValid(reltup)) elog(ERROR, "cache lookup failed for relation %u", OIDNewHeap); relform = (Form_pg_class) GETSTRUCT(reltup); relform->relpages = num_pages; relform->reltuples = num_tuples; /* Don't update the stats for pg_class. See swap_relation_files. */ Assert(OIDOldHeap != RelationRelationId); CacheInvalidateRelcacheByTuple(reltup); /* Clean up. */ heap_freetuple(reltup); table_close(relRelation, RowExclusiveLock); /* Make the update visible */ CommandCounterIncrement(); } ================================================ FILE: tsl/src/reorder.h ================================================ /* * This file and its contents are licensed under the Timescale License. * Please see the included NOTICE for copyright information and * LICENSE-TIMESCALE for a copy of the license. */ #pragma once #include <postgres.h> extern Datum tsl_reorder_chunk(PG_FUNCTION_ARGS); extern Datum tsl_move_chunk(PG_FUNCTION_ARGS); extern void reorder_chunk(Oid chunk_id, Oid index_id, bool verbose, Oid wait_id, Oid destination_tablespace, Oid index_tablespace); ================================================ FILE: tsl/test/.gitignore ================================================ results/ dump/ regression.diffs regression.out unit/testoutputs.tmp ================================================ FILE: tsl/test/CMakeLists.txt ================================================ include("${PRIMARY_TEST_DIR}/test-defs.cmake") set(_local_install_checks) set(_install_checks) # No checks for REGRESS_CHECKS needed here since all the checks are done in the # parent CMakeLists.txt. if(PG_REGRESS) # This custom target executes one command for each test configuration. It # might be possible to automatically generate this custom target, but for now # the configurations are hard-coded. add_custom_target( regresscheck-t COMMAND ${CMAKE_COMMAND} -E env ${PG_REGRESS_ENV} EXE_DIR=${CMAKE_CURRENT_SOURCE_DIR} TEST_SCHEDULE=${TEST_SCHEDULE} TEST_TIMEOUT=${TEST_TIMEOUT} TEST_PGPORT=${TEST_PGPORT_TEMP_INSTANCE} ${PRIMARY_TEST_DIR}/pg_regress.sh ${PG_REGRESS_OPTS_BASE} ${PG_REGRESS_OPTS_EXTRA} ${PG_REGRESS_OPTS_INOUT} ${PG_REGRESS_OPTS_TEMP_INSTANCE} --temp-config=${TEST_OUTPUT_DIR}/postgresql.conf USES_TERMINAL) add_custom_target( regresscheck-t-rerun COMMAND ${PRIMARY_TEST_DIR}/ci_rerun.sh regresscheck-t USES_TERMINAL) add_custom_target( regresschecklocal-t COMMAND ${CMAKE_COMMAND} -E env ${PG_REGRESS_ENV} EXE_DIR=${CMAKE_CURRENT_SOURCE_DIR} TEST_PGPORT=${TEST_PGPORT_LOCAL} TEST_SCHEDULE=${TEST_SCHEDULE} TEST_TIMEOUT=${TEST_TIMEOUT} ${PRIMARY_TEST_DIR}/pg_regress.sh ${PG_REGRESS_OPTS_BASE} ${PG_REGRESS_OPTS_EXTRA} ${PG_REGRESS_OPTS_INOUT} ${PG_REGRESS_OPTS_LOCAL_INSTANCE} USES_TERMINAL) list(APPEND _local_install_checks regresschecklocal-t) list(APPEND _install_checks regresscheck-t) add_custom_target( regresscheck-shared COMMAND ${CMAKE_COMMAND} -E env ${PG_REGRESS_ENV} EXE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/shared TEST_SCHEDULE=${TEST_SCHEDULE_SHARED} TEST_TIMEOUT=${TEST_TIMEOUT} TEST_PGPORT=${TEST_PGPORT_TEMP_INSTANCE} ${PRIMARY_TEST_DIR}/pg_regress.sh ${PG_REGRESS_OPTS_BASE} ${PG_REGRESS_SHARED_OPTS_EXTRA} ${PG_REGRESS_SHARED_OPTS_INOUT} ${PG_REGRESS_OPTS_TEMP_INSTANCE} --temp-config=${TEST_OUTPUT_DIR}/postgresql.conf USES_TERMINAL) add_custom_target( regresscheck-shared-rerun COMMAND ${PRIMARY_TEST_DIR}/ci_rerun.sh regresscheck-shared USES_TERMINAL) add_custom_target( regresschecklocal-shared COMMAND ${CMAKE_COMMAND} -E env ${PG_REGRESS_ENV} EXE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/shared TEST_SCHEDULE=${TEST_SCHEDULE_SHARED} TEST_TIMEOUT=${TEST_TIMEOUT} TEST_PGPORT=${TEST_PGPORT_LOCAL} ${PRIMARY_TEST_DIR}/pg_regress.sh ${PG_REGRESS_OPTS_BASE} ${PG_REGRESS_SHARED_OPTS_EXTRA} ${PG_REGRESS_SHARED_OPTS_INOUT} ${PG_REGRESS_OPTS_LOCAL_INSTANCE} USES_TERMINAL) list(APPEND _install_checks regresscheck-shared) list(APPEND _local_install_checks regresschecklocal-shared) elseif(REQUIRE_ALL_TESTS) message( FATAL_ERROR "All tests were required but 'pg_regress' could not be found") endif() if(TAP_CHECKS) add_custom_target( provecheck-t COMMAND rm -rf ${CMAKE_CURRENT_BINARY_DIR}/tmp_check COMMAND CONFDIR=${CMAKE_BINARY_DIR}/tsl/test PATH="${PG_BINDIR}:$ENV{PATH}" PG_REGRESS=${PG_REGRESS} SRC_DIR=${PG_SOURCE_DIR} CM_SRC_DIR=${CMAKE_SOURCE_DIR} PG_LIBDIR=${PG_LIBDIR} PG_VERSION_MAJOR=${PG_VERSION_MAJOR} ${PRIMARY_TEST_DIR}/pg_prove.sh USES_TERMINAL) list(APPEND _install_checks provecheck-t) elseif(REQUIRE_ALL_TESTS) message( FATAL_ERROR "All tests were required but TAP_CHECKS was off (see previous messages why)" ) endif() if(PG_ISOLATION_REGRESS) add_custom_target( isolationcheck-t COMMAND ${CMAKE_COMMAND} -E env ${PG_ISOLATION_REGRESS_ENV} EXE_DIR=${CMAKE_CURRENT_SOURCE_DIR} SPECS_DIR=${CMAKE_CURRENT_BINARY_DIR}/isolation/specs TEST_PGPORT=${TEST_PGPORT_TEMP_INSTANCE} ${PRIMARY_TEST_DIR}/pg_regress.sh ${PG_REGRESS_OPTS_BASE} ${PG_ISOLATION_REGRESS_OPTS_EXTRA} ${PG_ISOLATION_REGRESS_OPTS_INOUT} ${PG_REGRESS_OPTS_TEMP_INSTANCE} --temp-config=${TEST_OUTPUT_DIR}/postgresql.conf USES_TERMINAL) add_custom_target( isolationcheck-t-rerun COMMAND ${PRIMARY_TEST_DIR}/ci_rerun.sh isolationcheck-t USES_TERMINAL) add_custom_target( isolationchecklocal-t COMMAND ${CMAKE_COMMAND} -E env ${PG_ISOLATION_REGRESS_ENV} EXE_DIR=${CMAKE_CURRENT_SOURCE_DIR} SPECS_DIR=${CMAKE_CURRENT_BINARY_DIR}/isolation/specs TEST_PGPORT=${TEST_PGPORT_LOCAL} ${PRIMARY_TEST_DIR}/pg_regress.sh ${PG_REGRESS_OPTS_BASE} ${PG_ISOLATION_REGRESS_OPTS_EXTRA} ${PG_ISOLATION_REGRESS_OPTS_INOUT} ${PG_REGRESS_OPTS_LOCAL_INSTANCE} USES_TERMINAL) list(APPEND _local_install_checks isolationchecklocal-t) list(APPEND _install_checks isolationcheck-t) elseif(REQUIRE_ALL_TESTS) message( FATAL_ERROR "All tests were required but 'pg_isolation_regress' could not be found") endif() add_subdirectory(shared) add_subdirectory(sql) add_subdirectory(isolation) add_subdirectory(t) # installchecklocal tests against an existing postgres instance if(_local_install_checks) add_custom_target(installchecklocal-t DEPENDS ${_local_install_checks}) add_dependencies(installchecklocal installchecklocal-t) endif() if(_install_checks) add_custom_target(installcheck-t DEPENDS ${_install_checks}) add_dependencies(installcheck installcheck-t) endif() if(CMAKE_BUILD_TYPE MATCHES Debug OR COMPRESSION_FUZZING) add_subdirectory(src) endif() ================================================ FILE: tsl/test/expected/agg_partials_pushdown.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set PREFIX 'EXPLAIN (analyze, verbose, buffers off, costs off, timing off, summary off)' -- Make parallel plans predictable SET max_parallel_workers_per_gather = 1; SET parallel_leader_participation = off; CREATE TABLE testtable(filter_1 int, filler_2 int, filler_3 int, time timestamptz NOT NULL, device_id int, v0 int, v1 int, v2 float, v3 float); SELECT create_hypertable('testtable', 'time'); create_hypertable ------------------------ (1,public,testtable,t) ALTER TABLE testtable SET (timescaledb.compress, timescaledb.compress_orderby='time DESC', timescaledb.compress_segmentby='device_id'); INSERT INTO testtable(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id+1, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-01 0:00:00+0'::timestamptz,'2000-01-10 23:55:00+0','1day') gtime(time), generate_series(1,5,1) gdevice(device_id); SELECT compress_chunk(c) FROM show_chunks('testtable') c; compress_chunk ---------------------------------------- _timescaledb_internal._hyper_1_1_chunk _timescaledb_internal._hyper_1_2_chunk ANALYZE testtable; -- Pushdown aggregation to the chunk level SELECT count(*), sum(v0), sum(v1), sum(v2), sum(v3) FROM testtable WHERE time >= '2000-01-01 00:00:00+0' AND time <= '2000-02-01 00:00:00+0'; count | sum | sum | sum | sum -------+-----+-----+-----+----- 50 | 200 | 250 | 175 | :PREFIX SELECT count(*), sum(v0), sum(v1), sum(v2), sum(v3) FROM testtable WHERE time >= '2000-01-01 00:00:00+0' AND time <= '2000-02-01 00:00:00+0'; --- QUERY PLAN --- Finalize Aggregate (actual rows=1.00 loops=1) Output: count(*), sum(testtable.v0), sum(testtable.v1), sum(testtable.v2), sum(testtable.v3) -> Append (actual rows=2.00 loops=1) -> Custom Scan (VectorAgg) (actual rows=1.00 loops=1) Output: (PARTIAL count(*)), (PARTIAL sum(_hyper_1_1_chunk.v0)), (PARTIAL sum(_hyper_1_1_chunk.v1)), (PARTIAL sum(_hyper_1_1_chunk.v2)), (PARTIAL sum(_hyper_1_1_chunk.v3)) Grouping Policy: all compressed batches -> Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_1_1_chunk (actual rows=25.00 loops=1) Output: _hyper_1_1_chunk.v0, _hyper_1_1_chunk.v1, _hyper_1_1_chunk.v2, _hyper_1_1_chunk.v3 Vectorized Filter: ((_hyper_1_1_chunk."time" >= 'Fri Dec 31 16:00:00 1999 PST'::timestamp with time zone) AND (_hyper_1_1_chunk."time" <= 'Mon Jan 31 16:00:00 2000 PST'::timestamp with time zone)) Bulk Decompression: true -> Seq Scan on _timescaledb_internal.compress_hyper_2_3_chunk (actual rows=5.00 loops=1) Output: compress_hyper_2_3_chunk._ts_meta_count, compress_hyper_2_3_chunk.device_id, compress_hyper_2_3_chunk.filter_1, compress_hyper_2_3_chunk.filler_2, compress_hyper_2_3_chunk.filler_3, compress_hyper_2_3_chunk._ts_meta_min_1, compress_hyper_2_3_chunk._ts_meta_max_1, compress_hyper_2_3_chunk."time", compress_hyper_2_3_chunk.v0, compress_hyper_2_3_chunk.v1, compress_hyper_2_3_chunk.v2, compress_hyper_2_3_chunk.v3 Filter: ((compress_hyper_2_3_chunk._ts_meta_max_1 >= 'Fri Dec 31 16:00:00 1999 PST'::timestamp with time zone) AND (compress_hyper_2_3_chunk._ts_meta_min_1 <= 'Mon Jan 31 16:00:00 2000 PST'::timestamp with time zone)) -> Custom Scan (VectorAgg) (actual rows=1.00 loops=1) Output: (PARTIAL count(*)), (PARTIAL sum(_hyper_1_2_chunk.v0)), (PARTIAL sum(_hyper_1_2_chunk.v1)), (PARTIAL sum(_hyper_1_2_chunk.v2)), (PARTIAL sum(_hyper_1_2_chunk.v3)) Grouping Policy: all compressed batches -> Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_1_2_chunk (actual rows=25.00 loops=1) Output: _hyper_1_2_chunk.v0, _hyper_1_2_chunk.v1, _hyper_1_2_chunk.v2, _hyper_1_2_chunk.v3 Bulk Decompression: true -> Seq Scan on _timescaledb_internal.compress_hyper_2_4_chunk (actual rows=5.00 loops=1) Output: compress_hyper_2_4_chunk._ts_meta_count, compress_hyper_2_4_chunk.device_id, compress_hyper_2_4_chunk.filter_1, compress_hyper_2_4_chunk.filler_2, compress_hyper_2_4_chunk.filler_3, compress_hyper_2_4_chunk._ts_meta_min_1, compress_hyper_2_4_chunk._ts_meta_max_1, compress_hyper_2_4_chunk."time", compress_hyper_2_4_chunk.v0, compress_hyper_2_4_chunk.v1, compress_hyper_2_4_chunk.v2, compress_hyper_2_4_chunk.v3 -- Create partially compressed chunk INSERT INTO testtable(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id+1, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-01 0:00:00+0'::timestamptz,'2000-01-10 23:55:00+0','1day') gtime(time), generate_series(1,5,1) gdevice(device_id); ANALYZE testtable; -- Pushdown aggregation to the chunk level SELECT count(*), sum(v0), sum(v1), sum(v2), sum(v3) FROM testtable WHERE time >= '2000-01-01 00:00:00+0' AND time <= '2000-02-01 00:00:00+0'; count | sum | sum | sum | sum -------+-----+-----+-----+----- 100 | 400 | 500 | 350 | :PREFIX SELECT count(*), sum(v0), sum(v1), sum(v2), sum(v3) FROM testtable WHERE time >= '2000-01-01 00:00:00+0' AND time <= '2000-02-01 00:00:00+0'; --- QUERY PLAN --- Finalize Aggregate (actual rows=1.00 loops=1) Output: count(*), sum(testtable.v0), sum(testtable.v1), sum(testtable.v2), sum(testtable.v3) -> Append (actual rows=4.00 loops=1) -> Custom Scan (VectorAgg) (actual rows=1.00 loops=1) Output: (PARTIAL count(*)), (PARTIAL sum(_hyper_1_1_chunk.v0)), (PARTIAL sum(_hyper_1_1_chunk.v1)), (PARTIAL sum(_hyper_1_1_chunk.v2)), (PARTIAL sum(_hyper_1_1_chunk.v3)) Grouping Policy: all compressed batches -> Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_1_1_chunk (actual rows=25.00 loops=1) Output: _hyper_1_1_chunk.v0, _hyper_1_1_chunk.v1, _hyper_1_1_chunk.v2, _hyper_1_1_chunk.v3 Vectorized Filter: ((_hyper_1_1_chunk."time" >= 'Fri Dec 31 16:00:00 1999 PST'::timestamp with time zone) AND (_hyper_1_1_chunk."time" <= 'Mon Jan 31 16:00:00 2000 PST'::timestamp with time zone)) Chunk Status: PARTIAL Bulk Decompression: true -> Seq Scan on _timescaledb_internal.compress_hyper_2_3_chunk (actual rows=5.00 loops=1) Output: compress_hyper_2_3_chunk._ts_meta_count, compress_hyper_2_3_chunk.device_id, compress_hyper_2_3_chunk.filter_1, compress_hyper_2_3_chunk.filler_2, compress_hyper_2_3_chunk.filler_3, compress_hyper_2_3_chunk._ts_meta_min_1, compress_hyper_2_3_chunk._ts_meta_max_1, compress_hyper_2_3_chunk."time", compress_hyper_2_3_chunk.v0, compress_hyper_2_3_chunk.v1, compress_hyper_2_3_chunk.v2, compress_hyper_2_3_chunk.v3 Filter: ((compress_hyper_2_3_chunk._ts_meta_max_1 >= 'Fri Dec 31 16:00:00 1999 PST'::timestamp with time zone) AND (compress_hyper_2_3_chunk._ts_meta_min_1 <= 'Mon Jan 31 16:00:00 2000 PST'::timestamp with time zone)) -> Partial Aggregate (actual rows=1.00 loops=1) Output: PARTIAL count(*), PARTIAL sum(_hyper_1_1_chunk.v0), PARTIAL sum(_hyper_1_1_chunk.v1), PARTIAL sum(_hyper_1_1_chunk.v2), PARTIAL sum(_hyper_1_1_chunk.v3) -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk (actual rows=25.00 loops=1) Output: _hyper_1_1_chunk.v0, _hyper_1_1_chunk.v1, _hyper_1_1_chunk.v2, _hyper_1_1_chunk.v3 Filter: ((_hyper_1_1_chunk."time" >= 'Fri Dec 31 16:00:00 1999 PST'::timestamp with time zone) AND (_hyper_1_1_chunk."time" <= 'Mon Jan 31 16:00:00 2000 PST'::timestamp with time zone)) -> Custom Scan (VectorAgg) (actual rows=1.00 loops=1) Output: (PARTIAL count(*)), (PARTIAL sum(_hyper_1_2_chunk.v0)), (PARTIAL sum(_hyper_1_2_chunk.v1)), (PARTIAL sum(_hyper_1_2_chunk.v2)), (PARTIAL sum(_hyper_1_2_chunk.v3)) Grouping Policy: all compressed batches -> Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_1_2_chunk (actual rows=25.00 loops=1) Output: _hyper_1_2_chunk.v0, _hyper_1_2_chunk.v1, _hyper_1_2_chunk.v2, _hyper_1_2_chunk.v3 Chunk Status: PARTIAL Bulk Decompression: true -> Seq Scan on _timescaledb_internal.compress_hyper_2_4_chunk (actual rows=5.00 loops=1) Output: compress_hyper_2_4_chunk._ts_meta_count, compress_hyper_2_4_chunk.device_id, compress_hyper_2_4_chunk.filter_1, compress_hyper_2_4_chunk.filler_2, compress_hyper_2_4_chunk.filler_3, compress_hyper_2_4_chunk._ts_meta_min_1, compress_hyper_2_4_chunk._ts_meta_max_1, compress_hyper_2_4_chunk."time", compress_hyper_2_4_chunk.v0, compress_hyper_2_4_chunk.v1, compress_hyper_2_4_chunk.v2, compress_hyper_2_4_chunk.v3 -> Partial Aggregate (actual rows=1.00 loops=1) Output: PARTIAL count(*), PARTIAL sum(_hyper_1_2_chunk.v0), PARTIAL sum(_hyper_1_2_chunk.v1), PARTIAL sum(_hyper_1_2_chunk.v2), PARTIAL sum(_hyper_1_2_chunk.v3) -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk (actual rows=25.00 loops=1) Output: _hyper_1_2_chunk.v0, _hyper_1_2_chunk.v1, _hyper_1_2_chunk.v2, _hyper_1_2_chunk.v3 -- Same query using chunk append SELECT count(*), sum(v0), sum(v1), sum(v2), sum(v3) FROM testtable WHERE time >= '2000-01-01 00:00:00+0'::text::timestamptz AND time <= '2000-02-01 00:00:00+0'; count | sum | sum | sum | sum -------+-----+-----+-----+----- 100 | 400 | 500 | 350 | :PREFIX SELECT count(*), sum(v0), sum(v1), sum(v2), sum(v3) FROM testtable WHERE time >= '2000-01-01 00:00:00+0'::text::timestamptz AND time <= '2000-02-01 00:00:00+0'; --- QUERY PLAN --- Finalize Aggregate (actual rows=1.00 loops=1) Output: count(*), sum(testtable.v0), sum(testtable.v1), sum(testtable.v2), sum(testtable.v3) -> Custom Scan (ChunkAppend) on public.testtable (actual rows=4.00 loops=1) Output: (PARTIAL count(*)), (PARTIAL sum(testtable.v0)), (PARTIAL sum(testtable.v1)), (PARTIAL sum(testtable.v2)), (PARTIAL sum(testtable.v3)) Startup Exclusion: true Runtime Exclusion: false Chunks excluded during startup: 0 -> Custom Scan (VectorAgg) (actual rows=1.00 loops=1) Output: (PARTIAL count(*)), (PARTIAL sum(_hyper_1_1_chunk.v0)), (PARTIAL sum(_hyper_1_1_chunk.v1)), (PARTIAL sum(_hyper_1_1_chunk.v2)), (PARTIAL sum(_hyper_1_1_chunk.v3)) Grouping Policy: all compressed batches -> Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_1_1_chunk (actual rows=25.00 loops=1) Output: _hyper_1_1_chunk.v0, _hyper_1_1_chunk.v1, _hyper_1_1_chunk.v2, _hyper_1_1_chunk.v3 Vectorized Filter: (_hyper_1_1_chunk."time" >= ('2000-01-01 00:00:00+0'::cstring)::timestamp with time zone) Chunk Status: PARTIAL Bulk Decompression: true -> Seq Scan on _timescaledb_internal.compress_hyper_2_3_chunk (actual rows=5.00 loops=1) Output: compress_hyper_2_3_chunk._ts_meta_count, compress_hyper_2_3_chunk.device_id, compress_hyper_2_3_chunk.filter_1, compress_hyper_2_3_chunk.filler_2, compress_hyper_2_3_chunk.filler_3, compress_hyper_2_3_chunk._ts_meta_min_1, compress_hyper_2_3_chunk._ts_meta_max_1, compress_hyper_2_3_chunk."time", compress_hyper_2_3_chunk.v0, compress_hyper_2_3_chunk.v1, compress_hyper_2_3_chunk.v2, compress_hyper_2_3_chunk.v3 Filter: (compress_hyper_2_3_chunk._ts_meta_max_1 >= ('2000-01-01 00:00:00+0'::cstring)::timestamp with time zone) -> Partial Aggregate (actual rows=1.00 loops=1) Output: PARTIAL count(*), PARTIAL sum(_hyper_1_1_chunk.v0), PARTIAL sum(_hyper_1_1_chunk.v1), PARTIAL sum(_hyper_1_1_chunk.v2), PARTIAL sum(_hyper_1_1_chunk.v3) -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk (actual rows=25.00 loops=1) Output: _hyper_1_1_chunk.v0, _hyper_1_1_chunk.v1, _hyper_1_1_chunk.v2, _hyper_1_1_chunk.v3 Filter: (_hyper_1_1_chunk."time" >= ('2000-01-01 00:00:00+0'::cstring)::timestamp with time zone) -> Custom Scan (VectorAgg) (actual rows=1.00 loops=1) Output: (PARTIAL count(*)), (PARTIAL sum(_hyper_1_2_chunk.v0)), (PARTIAL sum(_hyper_1_2_chunk.v1)), (PARTIAL sum(_hyper_1_2_chunk.v2)), (PARTIAL sum(_hyper_1_2_chunk.v3)) Grouping Policy: all compressed batches -> Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_1_2_chunk (actual rows=25.00 loops=1) Output: _hyper_1_2_chunk.v0, _hyper_1_2_chunk.v1, _hyper_1_2_chunk.v2, _hyper_1_2_chunk.v3 Vectorized Filter: (_hyper_1_2_chunk."time" >= ('2000-01-01 00:00:00+0'::cstring)::timestamp with time zone) Chunk Status: PARTIAL Bulk Decompression: true -> Seq Scan on _timescaledb_internal.compress_hyper_2_4_chunk (actual rows=5.00 loops=1) Output: compress_hyper_2_4_chunk._ts_meta_count, compress_hyper_2_4_chunk.device_id, compress_hyper_2_4_chunk.filter_1, compress_hyper_2_4_chunk.filler_2, compress_hyper_2_4_chunk.filler_3, compress_hyper_2_4_chunk._ts_meta_min_1, compress_hyper_2_4_chunk._ts_meta_max_1, compress_hyper_2_4_chunk."time", compress_hyper_2_4_chunk.v0, compress_hyper_2_4_chunk.v1, compress_hyper_2_4_chunk.v2, compress_hyper_2_4_chunk.v3 Filter: (compress_hyper_2_4_chunk._ts_meta_max_1 >= ('2000-01-01 00:00:00+0'::cstring)::timestamp with time zone) -> Partial Aggregate (actual rows=1.00 loops=1) Output: PARTIAL count(*), PARTIAL sum(_hyper_1_2_chunk.v0), PARTIAL sum(_hyper_1_2_chunk.v1), PARTIAL sum(_hyper_1_2_chunk.v2), PARTIAL sum(_hyper_1_2_chunk.v3) -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk (actual rows=25.00 loops=1) Output: _hyper_1_2_chunk.v0, _hyper_1_2_chunk.v1, _hyper_1_2_chunk.v2, _hyper_1_2_chunk.v3 Filter: (_hyper_1_2_chunk."time" >= ('2000-01-01 00:00:00+0'::cstring)::timestamp with time zone) -- Perform chunk append startup chunk exclusion - issue 6282 :PREFIX SELECT count(*), sum(v0), sum(v1), sum(v2), sum(v3) FROM testtable WHERE time >= '2000-01-09 00:00:00+0'::text::timestamptz AND time <= '2000-02-01 00:00:00+0'::text::timestamptz; --- QUERY PLAN --- Finalize Aggregate (actual rows=1.00 loops=1) Output: count(*), sum(testtable.v0), sum(testtable.v1), sum(testtable.v2), sum(testtable.v3) -> Custom Scan (ChunkAppend) on public.testtable (actual rows=2.00 loops=1) Output: (PARTIAL count(*)), (PARTIAL sum(testtable.v0)), (PARTIAL sum(testtable.v1)), (PARTIAL sum(testtable.v2)), (PARTIAL sum(testtable.v3)) Startup Exclusion: true Runtime Exclusion: false Chunks excluded during startup: 2 -> Custom Scan (VectorAgg) (actual rows=1.00 loops=1) Output: (PARTIAL count(*)), (PARTIAL sum(_hyper_1_2_chunk.v0)), (PARTIAL sum(_hyper_1_2_chunk.v1)), (PARTIAL sum(_hyper_1_2_chunk.v2)), (PARTIAL sum(_hyper_1_2_chunk.v3)) Grouping Policy: all compressed batches -> Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_1_2_chunk (actual rows=10.00 loops=1) Output: _hyper_1_2_chunk.v0, _hyper_1_2_chunk.v1, _hyper_1_2_chunk.v2, _hyper_1_2_chunk.v3 Vectorized Filter: ((_hyper_1_2_chunk."time" >= ('2000-01-09 00:00:00+0'::cstring)::timestamp with time zone) AND (_hyper_1_2_chunk."time" <= ('2000-02-01 00:00:00+0'::cstring)::timestamp with time zone)) Rows Removed by Filter: 15 Chunk Status: PARTIAL Bulk Decompression: true -> Seq Scan on _timescaledb_internal.compress_hyper_2_4_chunk (actual rows=5.00 loops=1) Output: compress_hyper_2_4_chunk._ts_meta_count, compress_hyper_2_4_chunk.device_id, compress_hyper_2_4_chunk.filter_1, compress_hyper_2_4_chunk.filler_2, compress_hyper_2_4_chunk.filler_3, compress_hyper_2_4_chunk._ts_meta_min_1, compress_hyper_2_4_chunk._ts_meta_max_1, compress_hyper_2_4_chunk."time", compress_hyper_2_4_chunk.v0, compress_hyper_2_4_chunk.v1, compress_hyper_2_4_chunk.v2, compress_hyper_2_4_chunk.v3 Filter: ((compress_hyper_2_4_chunk._ts_meta_max_1 >= ('2000-01-09 00:00:00+0'::cstring)::timestamp with time zone) AND (compress_hyper_2_4_chunk._ts_meta_min_1 <= ('2000-02-01 00:00:00+0'::cstring)::timestamp with time zone)) -> Partial Aggregate (actual rows=1.00 loops=1) Output: PARTIAL count(*), PARTIAL sum(_hyper_1_2_chunk.v0), PARTIAL sum(_hyper_1_2_chunk.v1), PARTIAL sum(_hyper_1_2_chunk.v2), PARTIAL sum(_hyper_1_2_chunk.v3) -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk (actual rows=10.00 loops=1) Output: _hyper_1_2_chunk.v0, _hyper_1_2_chunk.v1, _hyper_1_2_chunk.v2, _hyper_1_2_chunk.v3 Filter: ((_hyper_1_2_chunk."time" >= ('2000-01-09 00:00:00+0'::cstring)::timestamp with time zone) AND (_hyper_1_2_chunk."time" <= ('2000-02-01 00:00:00+0'::cstring)::timestamp with time zone)) Rows Removed by Filter: 15 -- Force plain / sorted aggregation SET enable_hashagg = OFF; SELECT count(*), sum(v0), sum(v1), sum(v2), sum(v3) FROM testtable WHERE time >= '2000-01-01 00:00:00+0'::text::timestamptz AND time <= '2000-02-01 00:00:00+0'; count | sum | sum | sum | sum -------+-----+-----+-----+----- 100 | 400 | 500 | 350 | :PREFIX SELECT count(*), sum(v0), sum(v1), sum(v2), sum(v3) FROM testtable WHERE time >= '2000-01-01 00:00:00+0'::text::timestamptz AND time <= '2000-02-01 00:00:00+0'; --- QUERY PLAN --- Finalize Aggregate (actual rows=1.00 loops=1) Output: count(*), sum(testtable.v0), sum(testtable.v1), sum(testtable.v2), sum(testtable.v3) -> Custom Scan (ChunkAppend) on public.testtable (actual rows=4.00 loops=1) Output: (PARTIAL count(*)), (PARTIAL sum(testtable.v0)), (PARTIAL sum(testtable.v1)), (PARTIAL sum(testtable.v2)), (PARTIAL sum(testtable.v3)) Startup Exclusion: true Runtime Exclusion: false Chunks excluded during startup: 0 -> Custom Scan (VectorAgg) (actual rows=1.00 loops=1) Output: (PARTIAL count(*)), (PARTIAL sum(_hyper_1_1_chunk.v0)), (PARTIAL sum(_hyper_1_1_chunk.v1)), (PARTIAL sum(_hyper_1_1_chunk.v2)), (PARTIAL sum(_hyper_1_1_chunk.v3)) Grouping Policy: all compressed batches -> Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_1_1_chunk (actual rows=25.00 loops=1) Output: _hyper_1_1_chunk.v0, _hyper_1_1_chunk.v1, _hyper_1_1_chunk.v2, _hyper_1_1_chunk.v3 Vectorized Filter: (_hyper_1_1_chunk."time" >= ('2000-01-01 00:00:00+0'::cstring)::timestamp with time zone) Chunk Status: PARTIAL Bulk Decompression: true -> Seq Scan on _timescaledb_internal.compress_hyper_2_3_chunk (actual rows=5.00 loops=1) Output: compress_hyper_2_3_chunk._ts_meta_count, compress_hyper_2_3_chunk.device_id, compress_hyper_2_3_chunk.filter_1, compress_hyper_2_3_chunk.filler_2, compress_hyper_2_3_chunk.filler_3, compress_hyper_2_3_chunk._ts_meta_min_1, compress_hyper_2_3_chunk._ts_meta_max_1, compress_hyper_2_3_chunk."time", compress_hyper_2_3_chunk.v0, compress_hyper_2_3_chunk.v1, compress_hyper_2_3_chunk.v2, compress_hyper_2_3_chunk.v3 Filter: (compress_hyper_2_3_chunk._ts_meta_max_1 >= ('2000-01-01 00:00:00+0'::cstring)::timestamp with time zone) -> Partial Aggregate (actual rows=1.00 loops=1) Output: PARTIAL count(*), PARTIAL sum(_hyper_1_1_chunk.v0), PARTIAL sum(_hyper_1_1_chunk.v1), PARTIAL sum(_hyper_1_1_chunk.v2), PARTIAL sum(_hyper_1_1_chunk.v3) -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk (actual rows=25.00 loops=1) Output: _hyper_1_1_chunk.v0, _hyper_1_1_chunk.v1, _hyper_1_1_chunk.v2, _hyper_1_1_chunk.v3 Filter: (_hyper_1_1_chunk."time" >= ('2000-01-01 00:00:00+0'::cstring)::timestamp with time zone) -> Custom Scan (VectorAgg) (actual rows=1.00 loops=1) Output: (PARTIAL count(*)), (PARTIAL sum(_hyper_1_2_chunk.v0)), (PARTIAL sum(_hyper_1_2_chunk.v1)), (PARTIAL sum(_hyper_1_2_chunk.v2)), (PARTIAL sum(_hyper_1_2_chunk.v3)) Grouping Policy: all compressed batches -> Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_1_2_chunk (actual rows=25.00 loops=1) Output: _hyper_1_2_chunk.v0, _hyper_1_2_chunk.v1, _hyper_1_2_chunk.v2, _hyper_1_2_chunk.v3 Vectorized Filter: (_hyper_1_2_chunk."time" >= ('2000-01-01 00:00:00+0'::cstring)::timestamp with time zone) Chunk Status: PARTIAL Bulk Decompression: true -> Seq Scan on _timescaledb_internal.compress_hyper_2_4_chunk (actual rows=5.00 loops=1) Output: compress_hyper_2_4_chunk._ts_meta_count, compress_hyper_2_4_chunk.device_id, compress_hyper_2_4_chunk.filter_1, compress_hyper_2_4_chunk.filler_2, compress_hyper_2_4_chunk.filler_3, compress_hyper_2_4_chunk._ts_meta_min_1, compress_hyper_2_4_chunk._ts_meta_max_1, compress_hyper_2_4_chunk."time", compress_hyper_2_4_chunk.v0, compress_hyper_2_4_chunk.v1, compress_hyper_2_4_chunk.v2, compress_hyper_2_4_chunk.v3 Filter: (compress_hyper_2_4_chunk._ts_meta_max_1 >= ('2000-01-01 00:00:00+0'::cstring)::timestamp with time zone) -> Partial Aggregate (actual rows=1.00 loops=1) Output: PARTIAL count(*), PARTIAL sum(_hyper_1_2_chunk.v0), PARTIAL sum(_hyper_1_2_chunk.v1), PARTIAL sum(_hyper_1_2_chunk.v2), PARTIAL sum(_hyper_1_2_chunk.v3) -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk (actual rows=25.00 loops=1) Output: _hyper_1_2_chunk.v0, _hyper_1_2_chunk.v1, _hyper_1_2_chunk.v2, _hyper_1_2_chunk.v3 Filter: (_hyper_1_2_chunk."time" >= ('2000-01-01 00:00:00+0'::cstring)::timestamp with time zone) RESET enable_hashagg; -- Check chunk exclusion for index scans SET enable_seqscan = OFF; SELECT count(*), sum(v0), sum(v1), sum(v2), sum(v3) FROM testtable WHERE time >= '2000-01-09 00:00:00+0'::text::timestamptz AND time <= '2000-02-01 00:00:00+0'::text::timestamptz; count | sum | sum | sum | sum -------+-----+-----+-----+----- 20 | 80 | 100 | 70 | :PREFIX SELECT count(*), sum(v0), sum(v1), sum(v2), sum(v3) FROM testtable WHERE time >= '2000-01-09 00:00:00+0'::text::timestamptz AND time <= '2000-02-01 00:00:00+0'::text::timestamptz; --- QUERY PLAN --- Finalize Aggregate (actual rows=1.00 loops=1) Output: count(*), sum(testtable.v0), sum(testtable.v1), sum(testtable.v2), sum(testtable.v3) -> Custom Scan (ChunkAppend) on public.testtable (actual rows=2.00 loops=1) Output: (PARTIAL count(*)), (PARTIAL sum(testtable.v0)), (PARTIAL sum(testtable.v1)), (PARTIAL sum(testtable.v2)), (PARTIAL sum(testtable.v3)) Startup Exclusion: true Runtime Exclusion: false Chunks excluded during startup: 2 -> Custom Scan (VectorAgg) (actual rows=1.00 loops=1) Output: (PARTIAL count(*)), (PARTIAL sum(_hyper_1_2_chunk.v0)), (PARTIAL sum(_hyper_1_2_chunk.v1)), (PARTIAL sum(_hyper_1_2_chunk.v2)), (PARTIAL sum(_hyper_1_2_chunk.v3)) Grouping Policy: all compressed batches -> Custom Scan (ColumnarScan) on _timescaledb_internal._hyper_1_2_chunk (actual rows=10.00 loops=1) Output: _hyper_1_2_chunk.v0, _hyper_1_2_chunk.v1, _hyper_1_2_chunk.v2, _hyper_1_2_chunk.v3 Vectorized Filter: ((_hyper_1_2_chunk."time" >= ('2000-01-09 00:00:00+0'::cstring)::timestamp with time zone) AND (_hyper_1_2_chunk."time" <= ('2000-02-01 00:00:00+0'::cstring)::timestamp with time zone)) Rows Removed by Filter: 15 Chunk Status: PARTIAL Bulk Decompression: true -> Index Scan using compress_hyper_2_4_chunk_device_id__ts_meta_min_1__ts_meta__idx on _timescaledb_internal.compress_hyper_2_4_chunk (actual rows=5.00 loops=1) Output: compress_hyper_2_4_chunk._ts_meta_count, compress_hyper_2_4_chunk.device_id, compress_hyper_2_4_chunk.filter_1, compress_hyper_2_4_chunk.filler_2, compress_hyper_2_4_chunk.filler_3, compress_hyper_2_4_chunk._ts_meta_min_1, compress_hyper_2_4_chunk._ts_meta_max_1, compress_hyper_2_4_chunk."time", compress_hyper_2_4_chunk.v0, compress_hyper_2_4_chunk.v1, compress_hyper_2_4_chunk.v2, compress_hyper_2_4_chunk.v3 Index Cond: ((compress_hyper_2_4_chunk._ts_meta_min_1 <= ('2000-02-01 00:00:00+0'::cstring)::timestamp with time zone) AND (compress_hyper_2_4_chunk._ts_meta_max_1 >= ('2000-01-09 00:00:00+0'::cstring)::timestamp with time zone)) -> Partial Aggregate (actual rows=1.00 loops=1) Output: PARTIAL count(*), PARTIAL sum(_hyper_1_2_chunk.v0), PARTIAL sum(_hyper_1_2_chunk.v1), PARTIAL sum(_hyper_1_2_chunk.v2), PARTIAL sum(_hyper_1_2_chunk.v3) -> Index Scan using _hyper_1_2_chunk_testtable_time_idx on _timescaledb_internal._hyper_1_2_chunk (actual rows=10.00 loops=1) Output: _hyper_1_2_chunk.v0, _hyper_1_2_chunk.v1, _hyper_1_2_chunk.v2, _hyper_1_2_chunk.v3 Index Cond: ((_hyper_1_2_chunk."time" >= ('2000-01-09 00:00:00+0'::cstring)::timestamp with time zone) AND (_hyper_1_2_chunk."time" <= ('2000-02-01 00:00:00+0'::cstring)::timestamp with time zone)) RESET enable_seqscan; -- Check Append Node under ChunkAppend RESET enable_hashagg; RESET timescaledb.enable_chunkwise_aggregation; CREATE TABLE testtable2 ( timecustom BIGINT NOT NULL, device_id TEXT NOT NULL, series_0 DOUBLE PRECISION NULL, series_1 DOUBLE PRECISION NULL, series_2 DOUBLE PRECISION NULL, series_bool BOOLEAN NULL ); CREATE INDEX ON testtable2 (timeCustom DESC NULLS LAST, device_id); SELECT * FROM create_hypertable('testtable2', 'timecustom', 'device_id', number_partitions => 2, chunk_time_interval=>_timescaledb_functions.interval_to_usec('1 month')); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 3 | public | testtable2 | t INSERT INTO testtable2 VALUES (1257894000000000000, 'dev1', 1.5, 1, 2, true), (1257894000000000000, 'dev1', 1.5, 2, NULL, NULL), (1257894000000001000, 'dev1', 2.5, 3, NULL, NULL), (1257894001000000000, 'dev1', 3.5, 4, NULL, NULL), (1257897600000000000, 'dev1', 4.5, 5, NULL, false), (1257894002000000000, 'dev1', 5.5, 6, NULL, true), (1257894002000000000, 'dev1', 5.5, 7, NULL, false); INSERT INTO testtable2(timeCustom, device_id, series_0, series_1) VALUES (1257987600000000000, 'dev1', 1.5, 1), (1257987600000000000, 'dev1', 1.5, 2), (1257894000000000000, 'dev2', 1.5, 1), (1257894002000000000, 'dev1', 2.5, 3); SELECT timeCustom t, min(series_0) FROM PUBLIC.testtable2 GROUP BY t ORDER BY t DESC NULLS LAST limit 2; t | min ---------------------+----- 1257987600000000000 | 1.5 1257897600000000000 | 4.5 :PREFIX SELECT timeCustom t, min(series_0) FROM PUBLIC.testtable2 GROUP BY t ORDER BY t DESC NULLS LAST limit 2; --- QUERY PLAN --- Limit (actual rows=2.00 loops=1) Output: testtable2.timecustom, (min(testtable2.series_0)) -> Finalize GroupAggregate (actual rows=2.00 loops=1) Output: testtable2.timecustom, min(testtable2.series_0) Group Key: testtable2.timecustom -> Custom Scan (ChunkAppend) on public.testtable2 (actual rows=3.00 loops=1) Output: testtable2.timecustom, (PARTIAL min(testtable2.series_0)) Order: testtable2.timecustom DESC NULLS LAST Startup Exclusion: false Runtime Exclusion: false -> Partial GroupAggregate (actual rows=1.00 loops=1) Output: _hyper_3_7_chunk.timecustom, PARTIAL min(_hyper_3_7_chunk.series_0) Group Key: _hyper_3_7_chunk.timecustom -> Index Scan using _hyper_3_7_chunk_testtable2_timecustom_device_id_idx on _timescaledb_internal._hyper_3_7_chunk (actual rows=2.00 loops=1) Output: _hyper_3_7_chunk.timecustom, _hyper_3_7_chunk.series_0 -> Partial GroupAggregate (actual rows=1.00 loops=1) Output: _hyper_3_6_chunk.timecustom, PARTIAL min(_hyper_3_6_chunk.series_0) Group Key: _hyper_3_6_chunk.timecustom -> Index Scan using _hyper_3_6_chunk_testtable2_timecustom_device_id_idx on _timescaledb_internal._hyper_3_6_chunk (actual rows=1.00 loops=1) Output: _hyper_3_6_chunk.timecustom, _hyper_3_6_chunk.series_0 -> Merge Append (actual rows=1.00 loops=1) Sort Key: testtable2.timecustom DESC NULLS LAST -> Partial GroupAggregate (actual rows=1.00 loops=1) Output: _hyper_3_8_chunk.timecustom, PARTIAL min(_hyper_3_8_chunk.series_0) Group Key: _hyper_3_8_chunk.timecustom -> Index Scan using _hyper_3_8_chunk_testtable2_timecustom_device_id_idx on _timescaledb_internal._hyper_3_8_chunk (actual rows=1.00 loops=1) Output: _hyper_3_8_chunk.timecustom, _hyper_3_8_chunk.series_0 -> Partial GroupAggregate (actual rows=1.00 loops=1) Output: _hyper_3_5_chunk.timecustom, PARTIAL min(_hyper_3_5_chunk.series_0) Group Key: _hyper_3_5_chunk.timecustom -> Index Scan using _hyper_3_5_chunk_testtable2_timecustom_device_id_idx on _timescaledb_internal._hyper_3_5_chunk (actual rows=4.00 loops=1) Output: _hyper_3_5_chunk.timecustom, _hyper_3_5_chunk.series_0 -- Force parallel query SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false); set_config ------------ on SET parallel_setup_cost = 0; SET parallel_tuple_cost = 0; SELECT timeCustom t, min(series_0) FROM PUBLIC.testtable2 GROUP BY t ORDER BY t DESC NULLS LAST limit 2; t | min ---------------------+----- 1257987600000000000 | 1.5 1257897600000000000 | 4.5 :PREFIX SELECT timeCustom t, min(series_0) FROM PUBLIC.testtable2 GROUP BY t ORDER BY t DESC NULLS LAST limit 2; --- QUERY PLAN --- Gather (actual rows=2.00 loops=1) Output: testtable2.timecustom, (min(testtable2.series_0)) Workers Planned: 1 Workers Launched: 1 Single Copy: true -> Limit (actual rows=2.00 loops=1) Output: testtable2.timecustom, (min(testtable2.series_0)) Worker 0: actual rows=2.00 loops=1 -> Finalize GroupAggregate (actual rows=2.00 loops=1) Output: testtable2.timecustom, min(testtable2.series_0) Group Key: testtable2.timecustom Worker 0: actual rows=2.00 loops=1 -> Custom Scan (ChunkAppend) on public.testtable2 (actual rows=3.00 loops=1) Output: testtable2.timecustom, (PARTIAL min(testtable2.series_0)) Order: testtable2.timecustom DESC NULLS LAST Startup Exclusion: false Runtime Exclusion: false Worker 0: actual rows=3.00 loops=1 -> Partial GroupAggregate (actual rows=1.00 loops=1) Output: _hyper_3_7_chunk.timecustom, PARTIAL min(_hyper_3_7_chunk.series_0) Group Key: _hyper_3_7_chunk.timecustom Worker 0: actual rows=1.00 loops=1 -> Index Scan using _hyper_3_7_chunk_testtable2_timecustom_device_id_idx on _timescaledb_internal._hyper_3_7_chunk (actual rows=2.00 loops=1) Output: _hyper_3_7_chunk.timecustom, _hyper_3_7_chunk.series_0 Worker 0: actual rows=2.00 loops=1 -> Partial GroupAggregate (actual rows=1.00 loops=1) Output: _hyper_3_6_chunk.timecustom, PARTIAL min(_hyper_3_6_chunk.series_0) Group Key: _hyper_3_6_chunk.timecustom Worker 0: actual rows=1.00 loops=1 -> Index Scan using _hyper_3_6_chunk_testtable2_timecustom_device_id_idx on _timescaledb_internal._hyper_3_6_chunk (actual rows=1.00 loops=1) Output: _hyper_3_6_chunk.timecustom, _hyper_3_6_chunk.series_0 Worker 0: actual rows=1.00 loops=1 -> Merge Append (actual rows=1.00 loops=1) Sort Key: testtable2.timecustom DESC NULLS LAST Worker 0: actual rows=1.00 loops=1 -> Partial GroupAggregate (actual rows=1.00 loops=1) Output: _hyper_3_8_chunk.timecustom, PARTIAL min(_hyper_3_8_chunk.series_0) Group Key: _hyper_3_8_chunk.timecustom Worker 0: actual rows=1.00 loops=1 -> Index Scan using _hyper_3_8_chunk_testtable2_timecustom_device_id_idx on _timescaledb_internal._hyper_3_8_chunk (actual rows=1.00 loops=1) Output: _hyper_3_8_chunk.timecustom, _hyper_3_8_chunk.series_0 Worker 0: actual rows=1.00 loops=1 -> Partial GroupAggregate (actual rows=1.00 loops=1) Output: _hyper_3_5_chunk.timecustom, PARTIAL min(_hyper_3_5_chunk.series_0) Group Key: _hyper_3_5_chunk.timecustom Worker 0: actual rows=1.00 loops=1 -> Index Scan using _hyper_3_5_chunk_testtable2_timecustom_device_id_idx on _timescaledb_internal._hyper_3_5_chunk (actual rows=4.00 loops=1) Output: _hyper_3_5_chunk.timecustom, _hyper_3_5_chunk.series_0 Worker 0: actual rows=4.00 loops=1 -- Test that we don't process groupingSets :PREFIX SELECT timeCustom t, min(series_0) FROM PUBLIC.testtable2 GROUP BY ROLLUP(t); --- QUERY PLAN --- Gather (actual rows=7.00 loops=1) Output: testtable2.timecustom, (min(testtable2.series_0)) Workers Planned: 1 Workers Launched: 1 Single Copy: true -> MixedAggregate (actual rows=7.00 loops=1) Output: testtable2.timecustom, min(testtable2.series_0) Hash Key: testtable2.timecustom Group Key: () Worker 0: actual rows=7.00 loops=1 -> Append (actual rows=11.00 loops=1) Worker 0: actual rows=11.00 loops=1 -> Seq Scan on _timescaledb_internal._hyper_3_5_chunk (actual rows=7.00 loops=1) Output: _hyper_3_5_chunk.timecustom, _hyper_3_5_chunk.series_0 Worker 0: actual rows=7.00 loops=1 -> Seq Scan on _timescaledb_internal._hyper_3_6_chunk (actual rows=1.00 loops=1) Output: _hyper_3_6_chunk.timecustom, _hyper_3_6_chunk.series_0 Worker 0: actual rows=1.00 loops=1 -> Seq Scan on _timescaledb_internal._hyper_3_7_chunk (actual rows=2.00 loops=1) Output: _hyper_3_7_chunk.timecustom, _hyper_3_7_chunk.series_0 Worker 0: actual rows=2.00 loops=1 -> Seq Scan on _timescaledb_internal._hyper_3_8_chunk (actual rows=1.00 loops=1) Output: _hyper_3_8_chunk.timecustom, _hyper_3_8_chunk.series_0 Worker 0: actual rows=1.00 loops=1 -- Check parallel fallback into a non-partial aggregation SET timescaledb.enable_chunkwise_aggregation = OFF; SET enable_hashagg = OFF; SELECT timeCustom t, min(series_0) FROM PUBLIC.testtable2 GROUP BY t ORDER BY t DESC NULLS LAST limit 2; t | min ---------------------+----- 1257987600000000000 | 1.5 1257897600000000000 | 4.5 :PREFIX SELECT timeCustom t, min(series_0) FROM PUBLIC.testtable2 GROUP BY t ORDER BY t DESC NULLS LAST limit 2; --- QUERY PLAN --- Gather (actual rows=2.00 loops=1) Output: testtable2.timecustom, (min(testtable2.series_0)) Workers Planned: 1 Workers Launched: 1 Single Copy: true -> Limit (actual rows=2.00 loops=1) Output: testtable2.timecustom, (min(testtable2.series_0)) Worker 0: actual rows=2.00 loops=1 -> GroupAggregate (actual rows=2.00 loops=1) Output: testtable2.timecustom, min(testtable2.series_0) Group Key: testtable2.timecustom Worker 0: actual rows=2.00 loops=1 -> Custom Scan (ChunkAppend) on public.testtable2 (actual rows=4.00 loops=1) Output: testtable2.timecustom, testtable2.series_0 Order: testtable2.timecustom DESC NULLS LAST Startup Exclusion: false Runtime Exclusion: false Worker 0: actual rows=4.00 loops=1 -> Index Scan using _hyper_3_7_chunk_testtable2_timecustom_device_id_idx on _timescaledb_internal._hyper_3_7_chunk (actual rows=2.00 loops=1) Output: _hyper_3_7_chunk.timecustom, _hyper_3_7_chunk.series_0 Worker 0: actual rows=2.00 loops=1 -> Index Scan using _hyper_3_6_chunk_testtable2_timecustom_device_id_idx on _timescaledb_internal._hyper_3_6_chunk (actual rows=1.00 loops=1) Output: _hyper_3_6_chunk.timecustom, _hyper_3_6_chunk.series_0 Worker 0: actual rows=1.00 loops=1 -> Merge Append (actual rows=1.00 loops=1) Sort Key: testtable2.timecustom DESC NULLS LAST Worker 0: actual rows=1.00 loops=1 -> Index Scan using _hyper_3_8_chunk_testtable2_timecustom_device_id_idx on _timescaledb_internal._hyper_3_8_chunk (actual rows=1.00 loops=1) Output: _hyper_3_8_chunk.timecustom, _hyper_3_8_chunk.series_0 Worker 0: actual rows=1.00 loops=1 -> Index Scan using _hyper_3_5_chunk_testtable2_timecustom_device_id_idx on _timescaledb_internal._hyper_3_5_chunk (actual rows=1.00 loops=1) Output: _hyper_3_5_chunk.timecustom, _hyper_3_5_chunk.series_0 Worker 0: actual rows=1.00 loops=1 RESET timescaledb.enable_chunkwise_aggregation; RESET enable_hashagg; -- Test aggregation pushdown with MergeAppend node CREATE TABLE merge_append_test (start_time timestamptz, sensor_id int, cluster varchar (253), cost_recommendation_memory numeric); SELECT * FROM create_hypertable('merge_append_test', 'start_time'); WARNING: column type "character varying" used for "cluster" does not follow best practices hypertable_id | schema_name | table_name | created ---------------+-------------+-------------------+--------- 4 | public | merge_append_test | t CREATE INDEX merge_append_test_sensorid ON merge_append_test USING btree (start_time, sensor_id); INSERT INTO merge_append_test SELECT date_series, 1, 'production-1', random() * 100 FROM generate_series('2023-10-01 00:00:00', '2023-12-01 00:00:00', INTERVAL '1 hour') AS date_series ; INSERT INTO merge_append_test SELECT date_series, sensor_id, 'production-2', random() * 100 FROM generate_series('2023-10-01 00:00:00', '2023-12-01 00:00:00', INTERVAL '1 hour') AS date_series, generate_series(1, 100, 1) AS sensor_id ; ANALYZE merge_append_test; SET enable_seqscan = off; SET random_page_cost = 0; SET cpu_operator_cost = 0; SET enable_hashagg = off; RESET parallel_setup_cost; RESET parallel_tuple_cost; SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END, 'off', false); set_config ------------ off :PREFIX SELECT start_time, sensor_id, SUM(cost_recommendation_memory) FROM merge_append_test WHERE start_time >= '2023-11-27 00:00:00Z' AND start_time <= '2023-12-01 00:00:00Z' AND sensor_id < 10 AND CLUSTER = 'production-2' GROUP BY 1, 2; --- QUERY PLAN --- Finalize GroupAggregate (actual rows=873.00 loops=1) Output: merge_append_test.start_time, merge_append_test.sensor_id, sum(merge_append_test.cost_recommendation_memory) Group Key: merge_append_test.start_time, merge_append_test.sensor_id -> Merge Append (actual rows=873.00 loops=1) Sort Key: merge_append_test.start_time, merge_append_test.sensor_id -> Partial GroupAggregate (actual rows=648.00 loops=1) Output: _hyper_4_17_chunk.start_time, _hyper_4_17_chunk.sensor_id, PARTIAL sum(_hyper_4_17_chunk.cost_recommendation_memory) Group Key: _hyper_4_17_chunk.start_time, _hyper_4_17_chunk.sensor_id -> Index Scan using _hyper_4_17_chunk_merge_append_test_sensorid on _timescaledb_internal._hyper_4_17_chunk (actual rows=648.00 loops=1) Output: _hyper_4_17_chunk.start_time, _hyper_4_17_chunk.sensor_id, _hyper_4_17_chunk.cost_recommendation_memory Index Cond: ((_hyper_4_17_chunk.start_time >= 'Sun Nov 26 16:00:00 2023 PST'::timestamp with time zone) AND (_hyper_4_17_chunk.start_time <= 'Thu Nov 30 16:00:00 2023 PST'::timestamp with time zone) AND (_hyper_4_17_chunk.sensor_id < 10)) Filter: ((_hyper_4_17_chunk.cluster)::text = 'production-2'::text) Rows Removed by Filter: 72 -> Partial GroupAggregate (actual rows=225.00 loops=1) Output: _hyper_4_18_chunk.start_time, _hyper_4_18_chunk.sensor_id, PARTIAL sum(_hyper_4_18_chunk.cost_recommendation_memory) Group Key: _hyper_4_18_chunk.start_time, _hyper_4_18_chunk.sensor_id -> Index Scan using _hyper_4_18_chunk_merge_append_test_sensorid on _timescaledb_internal._hyper_4_18_chunk (actual rows=225.00 loops=1) Output: _hyper_4_18_chunk.start_time, _hyper_4_18_chunk.sensor_id, _hyper_4_18_chunk.cost_recommendation_memory Index Cond: ((_hyper_4_18_chunk.start_time >= 'Sun Nov 26 16:00:00 2023 PST'::timestamp with time zone) AND (_hyper_4_18_chunk.start_time <= 'Thu Nov 30 16:00:00 2023 PST'::timestamp with time zone) AND (_hyper_4_18_chunk.sensor_id < 10)) Filter: ((_hyper_4_18_chunk.cluster)::text = 'production-2'::text) Rows Removed by Filter: 25 RESET enable_seqscan; RESET random_page_cost; RESET cpu_operator_cost; RESET enable_hashagg; ================================================ FILE: tsl/test/expected/attach_chunk.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER GRANT CREATE ON DATABASE :"TEST_DBNAME" TO :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; -- Create test tables and hypertables (reusing from detach_chunk test) CREATE TABLE devices(id int PRIMARY KEY); INSERT INTO devices VALUES (1), (2), (3); CREATE TABLE attach_test(id int, time timestamptz not null, device int, temp float); CREATE INDEX attach_test_device_idx ON attach_test (device); ALTER TABLE attach_test ADD CONSTRAINT attach_test_temp_check CHECK (temp > 0), ADD CONSTRAINT attach_test_device_fkey FOREIGN KEY (device) REFERENCES devices(id), ADD CONSTRAINT attach_test_id_time_unique UNIQUE (id, time); SELECT * FROM create_hypertable('attach_test', 'time', 'id', 2); hypertable_id | schema_name | table_name | created ---------------+-------------+-------------+--------- 1 | public | attach_test | t CREATE TABLE attach_test_ref ( id int PRIMARY KEY, ref_id int, ref_time timestamptz, FOREIGN KEY (ref_id, ref_time) REFERENCES attach_test(id, time) ); INSERT INTO attach_test VALUES (1, '2025-06-01 05:00:00+3', 1, 23.4), (2, '2025-06-15 05:00:00+3', 2, 24.5), (3, '2025-06-30 05:00:00+3', 3, 25.6); -- Get chunk information for testing SELECT chunk_id AS "CHUNK_ID", hypertable_id AS "HYPERTABLE_ID", schema_name AS "CHUNK_SCHEMA", table_name AS "CHUNK_TABLE", schema_name || '.' || table_name AS "CHUNK_NAME", slices AS "CHUNK_SLICES" FROM _timescaledb_functions.show_chunk((SELECT show_chunks('attach_test') LIMIT 1)); \gset CHUNK_ID | HYPERTABLE_ID | CHUNK_SCHEMA | CHUNK_TABLE | CHUNK_NAME | CHUNK_SLICES ----------+---------------+-----------------------+------------------+----------------------------------------+------------------------------------------------------------------------------------------ 1 | 1 | _timescaledb_internal | _hyper_1_1_chunk | _timescaledb_internal._hyper_1_1_chunk | {"id": [-9223372036854775808, 1073741823], "time": [1748476800000000, 1749081600000000]} -- Successful attachment CREATE TABLE regular_table_to_attach(id int, time timestamptz not null, device int, temp float); CREATE INDEX regular_table_device_idx ON regular_table_to_attach (device); ALTER TABLE regular_table_to_attach ADD CONSTRAINT attach_test_temp_check CHECK (temp > 0), ADD CONSTRAINT pre_attach_temp_check CHECK (temp < 100), ADD CONSTRAINT pre_attach_id_time_unique UNIQUE (id, time); INSERT INTO regular_table_to_attach VALUES (10, '2025-07-05 13:00:00+3', 2, 27.8); -- Attach it as a chunk CALL attach_chunk('attach_test', 'regular_table_to_attach', '{"time": ["2025-07-01 05:00:00+3", "2025-07-07 05:00:00+3"], "id": [-9223372036854775808, 1073741823]}'); -- Verify data is accessible through the hypertable SELECT count(*) > 0 FROM attach_test WHERE time >= '2025-07-5'::timestamptz; ?column? ---------- t -- Check the chunk and related metadata are set up correctly SELECT id AS "ATTACHED_CHUNK" FROM _timescaledb_catalog.chunk WHERE hypertable_id = :'HYPERTABLE_ID' AND table_name = 'regular_table_to_attach'; \gset ATTACHED_CHUNK ---------------- 4 -- Verify that each constraint/index on a auto-created chunk is also present on the attached chunk SELECT hypertable_constraint_name FROM _timescaledb_catalog.chunk_constraint WHERE chunk_id = :'CHUNK_ID' EXCEPT SELECT hypertable_constraint_name FROM _timescaledb_catalog.chunk_constraint WHERE chunk_id = :'ATTACHED_CHUNK'; hypertable_constraint_name ---------------------------- SELECT * FROM test.show_indexesp('regular_table_to_attach'); Table | Index | Columns | Expr | Unique | Primary | Exclusion | Tablespace -------------------------+----------------------------------------------+-----------+------+--------+---------+-----------+------------ regular_table_to_attach | regular_table_device_idx | {device} | | f | f | f | regular_table_to_attach | pre_attach_id_time_unique | {id,time} | | t | f | f | regular_table_to_attach | regular_table_to_attach_attach_test_time_idx | {time} | | f | f | f | SELECT * FROM _timescaledb_catalog.dimension_slice ds JOIN _timescaledb_catalog.chunk_constraint cc ON ds.id = cc.dimension_slice_id WHERE cc.chunk_id = :'ATTACHED_CHUNK'; id | dimension_id | range_start | range_end | chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----+--------------+----------------------+------------------+----------+--------------------+-----------------+---------------------------- 2 | 2 | -9223372036854775808 | 1073741823 | 4 | 2 | constraint_2 | 6 | 1 | 1751335200000000 | 1751853600000000 | 4 | 6 | constraint_6 | -- Verify that the chunk is a child of the hypertable SELECT count(*) > 0 FROM pg_inherits WHERE inhrelid = 'regular_table_to_attach'::regclass::oid AND inhparent = 'attach_test'::regclass::oid; ?column? ---------- t -- Verify foreign key references work SELECT count(*) > 0 FROM pg_constraint WHERE contype = 'f' AND confrelid = 'regular_table_to_attach'::regclass::oid; ?column? ---------- t SELECT count(*) > 0 FROM pg_constraint WHERE contype = 'f' AND conrelid = 'regular_table_to_attach'::regclass::oid; ?column? ---------- t -- Verify data is routed to the correct chunk SELECT count(*) FROM attach_test WHERE time > '2025-07-01'::timestamptz; count ------- 1 INSERT INTO attach_test VALUES (5, '2025-07-4 05:00:00+3', 2, 19.5); SELECT count(*) FROM regular_table_to_attach; count ------- 2 -- Detach and re-attach a chunk CALL detach_chunk(:'CHUNK_NAME'); CALL attach_chunk('attach_test', :'CHUNK_NAME', :'CHUNK_SLICES'); -- Store the new chunk id SELECT chunk_id AS "CHUNK_ID" FROM _timescaledb_functions.show_chunk(:'CHUNK_NAME'); \gset CHUNK_ID ---------- 5 -- Verify it's re-attached SELECT count(*) > 0 FROM pg_inherits WHERE inhrelid = :'CHUNK_NAME'::regclass::oid; ?column? ---------- t -- Verify constraints and indexes are restored SELECT * FROM _timescaledb_catalog.chunk_constraint WHERE chunk_id = :'CHUNK_ID'; chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----------+--------------------+--------------------------------+---------------------------- 5 | | 1_2_attach_test_id_time_unique | attach_test_id_time_unique 5 | | 5_8_attach_test_device_fkey | attach_test_device_fkey 5 | 2 | constraint_2 | 5 | 7 | constraint_7 | SELECT * FROM _timescaledb_catalog.dimension_slice ds JOIN _timescaledb_catalog.chunk_constraint cc ON ds.id = cc.dimension_slice_id WHERE cc.chunk_id = :'CHUNK_ID'; id | dimension_id | range_start | range_end | chunk_id | dimension_slice_id | constraint_name | hypertable_constraint_name ----+--------------+----------------------+------------------+----------+--------------------+-----------------+---------------------------- 2 | 2 | -9223372036854775808 | 1073741823 | 5 | 2 | constraint_2 | 7 | 1 | 1748476800000000 | 1749081600000000 | 5 | 7 | constraint_7 | -- Attach a chunk to another hypertable with a different dimension CALL detach_chunk('regular_table_to_attach'); CREATE TABLE hypertable_with_different_dimension(id int, time timestamptz not null, device int, temp float); SELECT * FROM create_hypertable('hypertable_with_different_dimension', 'time'); hypertable_id | schema_name | table_name | created ---------------+-------------+-------------------------------------+--------- 2 | public | hypertable_with_different_dimension | t CALL attach_chunk('hypertable_with_different_dimension', 'regular_table_to_attach', '{"time": ["2025-07-01 05:00:00+3", "2025-07-07 05:00:00+3"]}'); CALL detach_chunk('regular_table_to_attach'); CREATE TABLE not_a_hypertable(id int, time timestamptz not null, device int, temp float); -- Error cases \set ON_ERROR_STOP 0 CALL attach_chunk('attach_test', 'nonexistent_table', '{"time": ["2025-07-01 05:00:00+3", "2025-07-07 05:00:00+3"], "id": [-9223372036854775808, 1073741823]}'); ERROR: relation "nonexistent_table" does not exist at character 34 CALL attach_chunk('not_a_hypertable', 'regular_table_to_attach', '{"time": ["2025-07-01 05:00:00+3", "2025-07-07 05:00:00+3"], "id": [-9223372036854775808, 1073741823]}'); ERROR: table "not_a_hypertable" is not a hypertable CALL attach_chunk('nonexistent_hypertable', 'regular_table_to_attach', '{"time": ["2025-07-01 05:00:00+3", "2025-07-07 05:00:00+3"], "id": [-9223372036854775808, 1073741823]}'); ERROR: relation "nonexistent_hypertable" does not exist at character 19 CALL attach_chunk(98765, 'regular_table_to_attach', '{"time": ["2025-07-01 05:00:00+3", "2025-07-07 05:00:00+3"], "id": [-9223372036854775808, 1073741823]}'); ERROR: relation with OID 98765 does not exist CALL attach_chunk(0, 'regular_table_to_attach', '{"time": ["2025-07-01 05:00:00+3", "2025-07-07 05:00:00+3"], "id": [-9223372036854775808, 1073741823]}'); ERROR: invalid hypertable relation OID -- invalid json format CALL attach_chunk('attach_test', 'regular_table_to_attach', '"time": ["2025-07-01 05:00:00+3", "2025-07-07 05:00:00+3"], "id": [-9223372036854775808, 1073741823]'); ERROR: invalid input syntax for type json at character 61 -- incorrect dimension information CALL attach_chunk('attach_test', 'regular_table_to_attach', '{"time": ["2025123-07-01 05:00:00+3", "2025-07-07 05:00:00+3"], "id": [-9223372036854775808, 1073741823]}'); ERROR: timestamp out of range: "2025123-07-01 05:00:00+3" CALL attach_chunk('attach_test', 'regular_table_to_attach', '{"time": ["2025-07-01 05:00:00+3", "2025-07-07 05:00:00+3"], "incorrect_key": [-9223372036854775808, 1073741823]}'); ERROR: invalid hypercube for hypertable "attach_test" CALL attach_chunk('attach_test', 'regular_table_to_attach', NULL); ERROR: invalid dimension slices argument CALL attach_chunk('attach_test', 'regular_table_to_attach', :'CHUNK_SLICES'); ERROR: chunk creation failed due to collision CALL attach_chunk('attach_test', :'CHUNK_NAME', :'CHUNK_SLICES'); ERROR: cannot attach chunk that is already a child of another table -- Attach a chunk of another hypertable CALL attach_chunk('hypertable_with_different_dimension', :'CHUNK_NAME', :'CHUNK_SLICES'); ERROR: cannot attach chunk that is already a child of another table -- Try to attach a table that's already a child of another table CREATE TABLE parent_table(id int); CREATE TABLE child_table(time timestamptz not null, device int, temp float) INHERITS (parent_table); CALL attach_chunk('attach_test', 'child_table', '{"time": ["2025-07-01 05:00:00+3", "2025-07-07 05:00:00+3"], "id": [-9223372036854775808, 1073741823]}'); ERROR: cannot attach chunk that is already a child of another table -- Try to attach a table with incompatible types CREATE TABLE incompatible_table(id int, time timestamptz not null, device text, temp float); CALL attach_chunk('attach_test', 'incompatible_table', '{"time": ["2025-07-01 05:00:00+3", "2025-07-07 05:00:00+3"], "id": [-9223372036854775808, 1073741823]}'); ERROR: child table "incompatible_table" has different type for column "device" -- Try to attach a table with missing columns CREATE TABLE missing_col_table(id int, time timestamptz not null); CALL attach_chunk('attach_test', 'missing_col_table', '{"time": ["2025-07-01 05:00:00+3", "2025-07-07 05:00:00+3"], "id": [-9223372036854775808, 1073741823]}'); ERROR: child table is missing column "device" -- Try to attach a table with extra columns CREATE TABLE extra_col_table(id int, time timestamptz not null, device int, temp float, extra_col int); CALL attach_chunk('attach_test', 'extra_col_table', '{"time": ["2025-07-01 05:00:00+3", "2025-07-07 05:00:00+3"], "id": [-9223372036854775808, 1073741823]}'); ERROR: table "extra_col_table" contains column "extra_col" not found in parent "attach_test" -- Attach by non-owner is not allowed set role :ROLE_1; CALL attach_chunk('attach_test', 'regular_table_to_attach', '{"time": ["2025-07-01 05:00:00+3", "2025-07-07 05:00:00+3"], "id": [-9223372036854775808, 1073741823]}'); ERROR: must be owner of table regular_table_to_attach set role :ROLE_DEFAULT_PERM_USER; CALL attach_chunk('attach_test', 'regular_table_to_attach', '{"time": ["2025-06-01 05:00:00+3", "2025-06-07 05:00:00+3"], "id": [-9223372036854775808, 1073741823]}'); ERROR: chunk creation failed due to collision -- Attach hypertable as a chunk CALL attach_chunk('attach_test', 'hypertable_with_different_dimension', '{"time": ["2025-07-01 05:00:00+3", "2025-07-07 05:00:00+3"], "id": [-9223372036854775808, 1073741823]}'); ERROR: cannot attach hypertable as a chunk \set ON_ERROR_STOP 1 -- Test rollback behavior SELECT count(*) AS "PRE_ROLLBACK_CHUNKS" FROM _timescaledb_catalog.chunk WHERE hypertable_id = :'HYPERTABLE_ID'; \gset PRE_ROLLBACK_CHUNKS --------------------- 3 BEGIN; CREATE TABLE rollback_test_table(id int, time timestamptz not null, device int, temp float); ALTER TABLE rollback_test_table ADD CONSTRAINT attach_test_temp_check CHECK (temp > 0); CALL attach_chunk('attach_test', 'rollback_test_table', '{"time": ["2025-07-01 05:00:00+3", "2025-07-07 05:00:00+3"], "id": [-9223372036854775808, 1073741823]}'); ROLLBACK; SELECT count(*) = :'PRE_ROLLBACK_CHUNKS' FROM _timescaledb_catalog.chunk WHERE hypertable_id = :'HYPERTABLE_ID'; ?column? ---------- t CALL attach_chunk('attach_test', 'regular_table_to_attach', '{"time": ["2025-07-01 05:00:00", "2025-07-07 05:00:00"], "id": [-9223372036854775808, 1073741823]}'); CALL detach_chunk('regular_table_to_attach'); CALL attach_chunk('attach_test', 'regular_table_to_attach', '{"time": ["2025-07-01", "2025-07-07"], "id": [-9223372036854775808, 1073741823]}'); CALL detach_chunk('regular_table_to_attach'); CALL attach_chunk('attach_test', 'regular_table_to_attach', '{"time": ["Tue July 1 05:00:00 2025 GMT+3", "Mon July 7 05:00:00 2025 GMT+3"], "id": [-9223372036854775808, 1073741823]}'); CALL detach_chunk('regular_table_to_attach'); CALL attach_chunk('attach_test', 'regular_table_to_attach', '{"time": [1751356800000000, 1751875200000000], "id": [-9223372036854775808, 1073741823]}'); CALL detach_chunk('regular_table_to_attach'); -- Attach a table with rows violating chunk constraints INSERT INTO regular_table_to_attach VALUES (4, '2024-07-05 13:00:00+3', 2, 27.8); \set ON_ERROR_STOP 0 CALL attach_chunk('attach_test', 'regular_table_to_attach', '{"time": ["2025-07-01 05:00:00+3", "2025-07-07 05:00:00+3"], "id": [-9223372036854775808, 1073741823]}'); ERROR: dimension constraint for column "time" violated by some row \set ON_ERROR_STOP 1 -- Clean up DROP TABLE regular_table_to_attach; DROP TABLE attach_test_ref; DROP TABLE attach_test; DROP TABLE devices CASCADE; ================================================ FILE: tsl/test/expected/bgw_custom.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. CREATE TABLE custom_log(job_id int, args jsonb, extra text, runner NAME DEFAULT CURRENT_ROLE); CREATE OR REPLACE FUNCTION custom_func(jobid int, args jsonb) RETURNS VOID LANGUAGE SQL AS $$ INSERT INTO custom_log VALUES($1, $2, 'custom_func'); $$; CREATE OR REPLACE FUNCTION custom_func_definer(jobid int, args jsonb) RETURNS VOID LANGUAGE SQL AS $$ INSERT INTO custom_log VALUES($1, $2, 'security definer'); $$ SECURITY DEFINER; CREATE OR REPLACE PROCEDURE custom_proc(job_id int, args jsonb) LANGUAGE SQL AS $$ INSERT INTO custom_log VALUES($1, $2, 'custom_proc'); $$; -- procedure with transaction handling CREATE OR REPLACE PROCEDURE custom_proc2(job_id int, args jsonb) LANGUAGE PLPGSQL AS $$ BEGIN INSERT INTO custom_log VALUES($1, $2, 'custom_proc2 1 COMMIT ' || (args->>'type')); COMMIT; INSERT INTO custom_log VALUES($1, $2, 'custom_proc2 2 ROLLBACK ' || (args->>'type')); ROLLBACK; INSERT INTO custom_log VALUES($1, $2, 'custom_proc2 3 COMMIT ' || (args->>'type')); COMMIT; END $$; \set ON_ERROR_STOP 0 -- test bad input SELECT add_job(NULL, '1h'); ERROR: function or procedure cannot be NULL SELECT add_job(0, '1h'); ERROR: function or procedure with OID 0 does not exist -- this will return an error about Oid 4294967295 -- while regproc is unsigned int postgres has an implicit cast from int to regproc SELECT add_job(-1, '1h'); ERROR: function or procedure with OID 4294967295 does not exist SELECT add_job('invalid_func', '1h'); ERROR: function "invalid_func" does not exist at character 16 SELECT add_job('custom_func', NULL); ERROR: schedule interval cannot be NULL SELECT add_job('custom_func', 'invalid interval'); ERROR: invalid input syntax for type interval: "invalid interval" at character 31 SELECT add_job('custom_func', '1h', job_name := 'this_is_a_really_really_really_long_application_name_to_overflow'); ERROR: application name too long. \set ON_ERROR_STOP 1 select '2000-01-01 00:00:00+00' as time_zero \gset SELECT add_job('custom_func','1h', config:='{"type":"function"}'::jsonb, initial_start => :'time_zero'::TIMESTAMPTZ); add_job --------- 1001 SELECT add_job('custom_proc','1h', config:='{"type":"procedure"}'::jsonb, initial_start => :'time_zero'::TIMESTAMPTZ); add_job --------- 1002 SELECT add_job('custom_proc2','1h', config:= '{"type":"procedure"}'::jsonb, initial_start => :'time_zero'::TIMESTAMPTZ); add_job --------- 1003 SELECT add_job('custom_func', '1h', config:='{"type":"function"}'::jsonb, initial_start => :'time_zero'::TIMESTAMPTZ); add_job --------- 1004 SELECT add_job('custom_func_definer', '1h', config:='{"type":"function"}'::jsonb, initial_start => :'time_zero'::TIMESTAMPTZ, job_name := 'custom_job_name'); add_job --------- 1005 -- exclude internal jobs SELECT * FROM timescaledb_information.jobs WHERE job_id >= 1000 ORDER BY 1; job_id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | config | next_start | initial_start | hypertable_schema | hypertable_name | check_schema | check_name --------+----------------------------+-------------------+-------------+-------------+--------------+-------------+---------------------+-------------------+-----------+----------------+-----------------------+------------------------------+------------------------------+-------------------+-----------------+--------------+------------ 1001 | User-Defined Action [1001] | @ 1 hour | @ 0 | -1 | @ 5 mins | public | custom_func | default_perm_user | t | t | {"type": "function"} | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | | | | 1002 | User-Defined Action [1002] | @ 1 hour | @ 0 | -1 | @ 5 mins | public | custom_proc | default_perm_user | t | t | {"type": "procedure"} | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | | | | 1003 | User-Defined Action [1003] | @ 1 hour | @ 0 | -1 | @ 5 mins | public | custom_proc2 | default_perm_user | t | t | {"type": "procedure"} | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | | | | 1004 | User-Defined Action [1004] | @ 1 hour | @ 0 | -1 | @ 5 mins | public | custom_func | default_perm_user | t | t | {"type": "function"} | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | | | | 1005 | custom_job_name [1005] | @ 1 hour | @ 0 | -1 | @ 5 mins | public | custom_func_definer | default_perm_user | t | t | {"type": "function"} | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | | | | SELECT count(*) FROM _timescaledb_catalog.bgw_job WHERE config->>'type' IN ('procedure', 'function'); count ------- 5 \set ON_ERROR_STOP 0 -- test bad input CALL run_job(NULL); ERROR: job ID cannot be NULL CALL run_job(-1); ERROR: job -1 not found \set ON_ERROR_STOP 1 CALL run_job(1001); CALL run_job(1002); CALL run_job(1003); CALL run_job(1004); CALL run_job(1005); SELECT * FROM custom_log ORDER BY job_id, extra; job_id | args | extra | runner --------+-----------------------+---------------------------------+------------------- 1001 | {"type": "function"} | custom_func | default_perm_user 1002 | {"type": "procedure"} | custom_proc | default_perm_user 1003 | {"type": "procedure"} | custom_proc2 1 COMMIT procedure | default_perm_user 1003 | {"type": "procedure"} | custom_proc2 3 COMMIT procedure | default_perm_user 1004 | {"type": "function"} | custom_func | default_perm_user 1005 | {"type": "function"} | security definer | default_perm_user \set ON_ERROR_STOP 0 -- test bad input SELECT delete_job(NULL); delete_job ------------ SELECT delete_job(-1); ERROR: job -1 not found \set ON_ERROR_STOP 1 -- We keep job 1001 for some additional checks. SELECT delete_job(1002); delete_job ------------ SELECT delete_job(1003); delete_job ------------ SELECT delete_job(1004); delete_job ------------ SELECT delete_job(1005); delete_job ------------ -- check jobs got removed SELECT count(*) FROM timescaledb_information.jobs WHERE job_id >= 1002; count ------- 0 \c :TEST_DBNAME :ROLE_SUPERUSER -- create a new job with longer id SELECT nextval('_timescaledb_catalog.bgw_job_id_seq') as nextval \gset SELECT setval('_timescaledb_catalog.bgw_job_id_seq', 2147483647, false); setval ------------ 2147483647 SELECT add_job('custom_func', '1h', config:='{"type":"function"}'::jsonb, job_name := 'custom_job_name'); add_job ------------ 2147483647 \set ON_ERROR_STOP 0 -- test bad input SELECT alter_job(NULL, if_exists => false); ERROR: job ID cannot be NULL SELECT alter_job(-1, if_exists => false); ERROR: job -1 not found SELECT alter_job(1001, job_name => 'this_is_a_really_really_really_long_application_name_to_overflow'); ERROR: application name too long. SELECT alter_job(2147483647, job_name => 'this_is_a_really_really_really_long_application_name_to_overflow'); ERROR: application name too long. \set ON_ERROR_STOP 1 -- test bad input but don't fail SELECT alter_job(NULL, if_exists => true); NOTICE: job 0 not found, skipping alter_job ----------- SELECT alter_job(-1, if_exists => true); NOTICE: job -1 not found, skipping alter_job ----------- -- test altering job with NULL config SELECT job_id FROM alter_job(1001,scheduled:=false); job_id -------- 1001 SELECT scheduled, config FROM timescaledb_information.jobs WHERE job_id = 1001; scheduled | config -----------+---------------------- f | {"type": "function"} -- test updating job settings SELECT job_id FROM alter_job(1001,config:='{"test":"test"}'); job_id -------- 1001 SELECT scheduled, config FROM timescaledb_information.jobs WHERE job_id = 1001; scheduled | config -----------+------------------ f | {"test": "test"} SELECT job_id FROM alter_job(1001,scheduled:=true); job_id -------- 1001 SELECT scheduled, config FROM timescaledb_information.jobs WHERE job_id = 1001; scheduled | config -----------+------------------ t | {"test": "test"} SELECT job_id FROM alter_job(1001,scheduled:=false); job_id -------- 1001 SELECT scheduled, config FROM timescaledb_information.jobs WHERE job_id = 1001; scheduled | config -----------+------------------ f | {"test": "test"} -- test updating the job name SELECT job_id, application_name FROM alter_job(1001,job_name:='custom_name_2'); job_id | application_name --------+---------------------- 1001 | custom_name_2 [1001] SELECT job_id, application_name FROM alter_job(2147483647,job_name:='short_name_to_fit'); job_id | application_name ------------+-------------------------------- 2147483647 | short_name_to_fit [2147483647] SELECT application_name FROM timescaledb_information.jobs WHERE job_id >= 1001; application_name -------------------------------- custom_name_2 [1001] short_name_to_fit [2147483647] -- Done with jobs now, so remove it. SELECT delete_job(1001); delete_job ------------ SELECT delete_job(2147483647); delete_job ------------ -- reset the sequence to its previous value SELECT setval('_timescaledb_catalog.bgw_job_id_seq', :nextval, false); setval -------- 1006 --test for #2793 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- background workers are disabled, so the job will not run -- SELECT add_job( proc=>'custom_func', schedule_interval=>'1h', initial_start =>'2018-01-01 10:00:00-05') AS job_id_1 \gset SELECT job_id, next_start, scheduled, schedule_interval FROM timescaledb_information.jobs WHERE job_id > 1001; job_id | next_start | scheduled | schedule_interval --------+------------------------------+-----------+------------------- 1006 | Mon Jan 01 07:00:00 2018 PST | t | @ 1 hour \x SELECT * FROM timescaledb_information.job_stats WHERE job_id > 1001; -[ RECORD 1 ]----------+----------------------------- hypertable_schema | hypertable_name | job_id | 1006 last_run_started_at | -infinity last_successful_finish | -infinity last_run_status | job_status | Scheduled last_run_duration | next_start | Mon Jan 01 07:00:00 2018 PST total_runs | 0 total_successes | 0 total_failures | 0 \x SELECT delete_job(:job_id_1); delete_job ------------ -- tests for #3545 TRUNCATE custom_log; -- Nested procedure call CREATE OR REPLACE PROCEDURE custom_proc_nested(job_id int, args jsonb) LANGUAGE PLPGSQL AS $$ BEGIN INSERT INTO custom_log VALUES($1, $2, 'custom_proc_nested 1 COMMIT'); COMMIT; INSERT INTO custom_log VALUES($1, $2, 'custom_proc_nested 2 ROLLBACK'); ROLLBACK; INSERT INTO custom_log VALUES($1, $2, 'custom_proc_nested 3 COMMIT'); COMMIT; END $$; CREATE OR REPLACE PROCEDURE custom_proc3(job_id int, args jsonb) LANGUAGE PLPGSQL AS $$ BEGIN CALL custom_proc_nested(job_id, args); END $$; CREATE OR REPLACE PROCEDURE custom_proc4(job_id int, args jsonb) LANGUAGE PLPGSQL AS $$ BEGIN INSERT INTO custom_log VALUES($1, $2, 'custom_proc4 1 COMMIT'); COMMIT; INSERT INTO custom_log VALUES($1, $2, 'custom_proc4 2 ROLLBACK'); ROLLBACK; RAISE EXCEPTION 'forced exception'; INSERT INTO custom_log VALUES($1, $2, 'custom_proc4 3 ABORT'); COMMIT; END $$; CREATE OR REPLACE PROCEDURE custom_proc5(job_id int, args jsonb) LANGUAGE PLPGSQL AS $$ BEGIN CALL refresh_continuous_aggregate('conditions_summary_daily', '2021-08-01 00:00', '2021-08-31 00:00'); END $$; -- Remove any default jobs, e.g., telemetry \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE _timescaledb_catalog.bgw_job RESTART IDENTITY CASCADE; NOTICE: truncate cascades to table "bgw_job_stat" NOTICE: truncate cascades to table "bgw_policy_chunk_stats" \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT add_job('custom_proc2', '1h', config := '{"type":"procedure"}'::jsonb, initial_start := now()) AS job_id_1 \gset SELECT add_job('custom_proc3', '1h', config := '{"type":"procedure"}'::jsonb, initial_start := now()) AS job_id_2 \gset \c :TEST_DBNAME :ROLE_SUPERUSER -- Start Background Workers SELECT _timescaledb_functions.start_background_workers(); start_background_workers -------------------------- t -- Wait for jobs SELECT test.wait_for_job_to_run(:job_id_1, 1); wait_for_job_to_run --------------------- t SELECT test.wait_for_job_to_run(:job_id_2, 1); wait_for_job_to_run --------------------- t -- Check results SELECT * FROM custom_log ORDER BY job_id, extra; job_id | args | extra | runner --------+-----------------------+---------------------------------+------------------- 1000 | {"type": "procedure"} | custom_proc2 1 COMMIT procedure | default_perm_user 1000 | {"type": "procedure"} | custom_proc2 3 COMMIT procedure | default_perm_user 1001 | {"type": "procedure"} | custom_proc_nested 1 COMMIT | default_perm_user 1001 | {"type": "procedure"} | custom_proc_nested 3 COMMIT | default_perm_user -- Delete previous jobs SELECT delete_job(:job_id_1); delete_job ------------ SELECT delete_job(:job_id_2); delete_job ------------ TRUNCATE custom_log; -- Forced Exception SELECT add_job('custom_proc4', '1h', config := '{"type":"procedure"}'::jsonb, initial_start := now()) AS job_id_3 \gset SELECT _timescaledb_functions.restart_background_workers(); restart_background_workers ---------------------------- t SELECT test.wait_for_job_to_run(:job_id_3, 1); INFO: wait_for_job_to_run: job execution failed wait_for_job_to_run --------------------- f -- Check results SELECT * FROM custom_log ORDER BY job_id, extra; job_id | args | extra | runner --------+-----------------------+-----------------------+------------ 1002 | {"type": "procedure"} | custom_proc4 1 COMMIT | super_user -- Delete previous jobs SELECT delete_job(:job_id_3); delete_job ------------ CREATE TABLE conditions ( time TIMESTAMP NOT NULL, location TEXT NOT NULL, location2 char(10) NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL ) WITH (autovacuum_enabled = FALSE); SELECT create_hypertable('conditions', 'time', chunk_time_interval := '15 days'::interval); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable ------------------------- (1,public,conditions,t) ALTER TABLE conditions SET ( timescaledb.compress, timescaledb.compress_segmentby = 'location', timescaledb.compress_orderby = 'time' ); INSERT INTO conditions SELECT generate_series('2021-08-01 00:00'::timestamp, '2021-08-31 00:00'::timestamp, '1 day'), 'POR', 'klick', 55, 75; -- Chunk compress stats SELECT * FROM _timescaledb_internal.compressed_chunk_stats ORDER BY chunk_name; hypertable_schema | hypertable_name | chunk_schema | chunk_name | compression_status | uncompressed_heap_size | uncompressed_index_size | uncompressed_toast_size | uncompressed_total_size | compressed_heap_size | compressed_index_size | compressed_toast_size | compressed_total_size -------------------+-----------------+-----------------------+------------------+--------------------+------------------------+-------------------------+-------------------------+-------------------------+----------------------+-----------------------+-----------------------+----------------------- public | conditions | _timescaledb_internal | _hyper_1_1_chunk | Uncompressed | | | | | | | | public | conditions | _timescaledb_internal | _hyper_1_2_chunk | Uncompressed | | | | | | | | public | conditions | _timescaledb_internal | _hyper_1_3_chunk | Uncompressed | | | | | | | | -- Compression policy SELECT add_compression_policy('conditions', interval '1 day') AS job_id_4 \gset SELECT _timescaledb_functions.restart_background_workers(); restart_background_workers ---------------------------- t SELECT test.wait_for_job_to_run(:job_id_4, 1); wait_for_job_to_run --------------------- t -- Chunk compress stats SELECT * FROM _timescaledb_internal.compressed_chunk_stats ORDER BY chunk_name; hypertable_schema | hypertable_name | chunk_schema | chunk_name | compression_status | uncompressed_heap_size | uncompressed_index_size | uncompressed_toast_size | uncompressed_total_size | compressed_heap_size | compressed_index_size | compressed_toast_size | compressed_total_size -------------------+-----------------+-----------------------+------------------+--------------------+------------------------+-------------------------+-------------------------+-------------------------+----------------------+-----------------------+-----------------------+----------------------- public | conditions | _timescaledb_internal | _hyper_1_1_chunk | Compressed | 8192 | 16384 | 8192 | 32768 | 16384 | 16384 | 8192 | 40960 public | conditions | _timescaledb_internal | _hyper_1_2_chunk | Compressed | 8192 | 16384 | 8192 | 32768 | 16384 | 16384 | 8192 | 40960 public | conditions | _timescaledb_internal | _hyper_1_3_chunk | Compressed | 8192 | 16384 | 8192 | 32768 | 16384 | 16384 | 8192 | 40960 --TEST compression job after inserting data into previously compressed chunk INSERT INTO conditions SELECT generate_series('2021-08-01 00:00'::timestamp, '2021-08-31 00:00'::timestamp, '1 day'), 'NYC', 'nycity', 40, 40; SELECT id, table_name, status from _timescaledb_catalog.chunk where hypertable_id = (select id from _timescaledb_catalog.hypertable where table_name = 'conditions') order by id; id | table_name | status ----+------------------+-------- 1 | _hyper_1_1_chunk | 9 2 | _hyper_1_2_chunk | 9 3 | _hyper_1_3_chunk | 9 --running job second time, wait for it to complete select t.schedule_interval FROM alter_job(:job_id_4, next_start=> now() ) t; schedule_interval ------------------- @ 12 hours SELECT _timescaledb_functions.restart_background_workers(); restart_background_workers ---------------------------- t SELECT test.wait_for_job_to_run(:job_id_4, 2); wait_for_job_to_run --------------------- t SELECT id, table_name, status from _timescaledb_catalog.chunk where hypertable_id = (select id from _timescaledb_catalog.hypertable where table_name = 'conditions') order by id; id | table_name | status ----+------------------+-------- 1 | _hyper_1_1_chunk | 1 2 | _hyper_1_2_chunk | 1 3 | _hyper_1_3_chunk | 1 -- Drop the compression job SELECT delete_job(:job_id_4); delete_job ------------ -- Decompress chunks before create the cagg SELECT decompress_chunk(c) FROM show_chunks('conditions') c; decompress_chunk ---------------------------------------- _timescaledb_internal._hyper_1_1_chunk _timescaledb_internal._hyper_1_2_chunk _timescaledb_internal._hyper_1_3_chunk -- TEST Continuous Aggregate job CREATE MATERIALIZED VIEW conditions_summary_daily WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT location, time_bucket(INTERVAL '1 day', time) AS bucket, AVG(temperature), MAX(temperature), MIN(temperature) FROM conditions GROUP BY location, bucket WITH NO DATA; -- Refresh Continous Aggregate by Job SELECT add_job('custom_proc5', '1h', config := '{"type":"procedure"}'::jsonb, initial_start := now()) AS job_id_5 \gset SELECT _timescaledb_functions.restart_background_workers(); restart_background_workers ---------------------------- t SELECT test.wait_for_job_to_run(:job_id_5, 1); wait_for_job_to_run --------------------- t SELECT count(*) FROM conditions_summary_daily; count ------- 62 -- TESTs for alter_job_set_hypertable_id API SELECT _timescaledb_functions.alter_job_set_hypertable_id( :job_id_5, NULL); alter_job_set_hypertable_id ----------------------------- 1004 SELECT id, proc_name, hypertable_id FROM _timescaledb_catalog.bgw_job WHERE id = :job_id_5; id | proc_name | hypertable_id ------+--------------+--------------- 1004 | custom_proc5 | -- error case, try to associate with a PG relation \set ON_ERROR_STOP 0 SELECT _timescaledb_functions.alter_job_set_hypertable_id( :job_id_5, 'custom_log'); ERROR: relation "custom_log" is not a hypertable or continuous aggregate \set ON_ERROR_STOP 1 -- TEST associate the cagg with the job SELECT _timescaledb_functions.alter_job_set_hypertable_id( :job_id_5, 'conditions_summary_daily'::regclass); alter_job_set_hypertable_id ----------------------------- 1004 SELECT id, proc_name, hypertable_id FROM _timescaledb_catalog.bgw_job WHERE id = :job_id_5; id | proc_name | hypertable_id ------+--------------+--------------- 1004 | custom_proc5 | 3 --verify that job is dropped when cagg is dropped DROP MATERIALIZED VIEW conditions_summary_daily; NOTICE: drop cascades to table _timescaledb_internal._hyper_3_7_chunk SELECT id, proc_name, hypertable_id FROM _timescaledb_catalog.bgw_job WHERE id = :job_id_5; id | proc_name | hypertable_id ----+-----------+--------------- -- Cleanup DROP TABLE conditions; DROP TABLE custom_log; -- Stop Background Workers SELECT _timescaledb_functions.stop_background_workers(); stop_background_workers ------------------------- t SELECT _timescaledb_functions.restart_background_workers(); restart_background_workers ---------------------------- t \set ON_ERROR_STOP 0 -- add test for custom jobs with custom check functions -- create the functions/procedures to be used as checking functions CREATE OR REPLACE PROCEDURE test_config_check_proc(config jsonb) LANGUAGE PLPGSQL AS $$ DECLARE drop_after interval; BEGIN SELECT jsonb_object_field_text (config, 'drop_after')::interval INTO STRICT drop_after; IF drop_after IS NULL THEN RAISE EXCEPTION 'Config must be not NULL and have drop_after'; END IF ; END $$; CREATE OR REPLACE FUNCTION test_config_check_func(config jsonb) RETURNS VOID AS $$ DECLARE drop_after interval; BEGIN IF config IS NULL THEN RETURN; END IF; SELECT jsonb_object_field_text (config, 'drop_after')::interval INTO STRICT drop_after; IF drop_after IS NULL THEN RAISE EXCEPTION 'Config can be NULL but must have drop_after if not'; END IF ; END $$ LANGUAGE PLPGSQL; -- step 2, create a procedure to run as a custom job CREATE OR REPLACE PROCEDURE test_proc_with_check(job_id int, config jsonb) LANGUAGE PLPGSQL AS $$ BEGIN RAISE NOTICE 'Will only print this if config passes checks, my config is %', config; END $$; -- step 3, add the job with the config check function passed as argument -- test procedures, should get an unsupported error select add_job('test_proc_with_check', '5 secs', config => '{}', check_config => 'test_config_check_proc'::regproc); ERROR: unsupported function type -- test functions select add_job('test_proc_with_check', '5 secs', config => '{}', check_config => 'test_config_check_func'::regproc); ERROR: Config can be NULL but must have drop_after if not select add_job('test_proc_with_check', '5 secs', config => NULL, check_config => 'test_config_check_func'::regproc); add_job --------- 1005 select add_job('test_proc_with_check', '5 secs', config => '{"drop_after": "chicken"}', check_config => 'test_config_check_func'::regproc); ERROR: invalid input syntax for type interval: "chicken" select add_job('test_proc_with_check', '5 secs', config => '{"drop_after": "2 weeks"}', check_config => 'test_config_check_func'::regproc) as job_with_func_check_id \gset --- test alter_job select alter_job(:job_with_func_check_id, config => '{"drop_after":"chicken"}'); ERROR: invalid input syntax for type interval: "chicken" select config from alter_job(:job_with_func_check_id, config => '{"drop_after":"5 years"}'); config --------------------------- {"drop_after": "5 years"} -- test that jobs with an incorrect check function signature will not be registered -- these are all incorrect function signatures CREATE OR REPLACE FUNCTION test_config_check_func_0args() RETURNS VOID AS $$ BEGIN RAISE NOTICE 'I take no arguments and will validate anything you give me!'; END $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION test_config_check_func_2args(config jsonb, intarg int) RETURNS VOID AS $$ BEGIN RAISE NOTICE 'I take two arguments (jsonb, int) and I should fail to run!'; END $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION test_config_check_func_intarg(config int) RETURNS VOID AS $$ BEGIN RAISE NOTICE 'I take one argument which is an integer and I should fail to run!'; END $$ LANGUAGE PLPGSQL; -- -- this should fail, it has an incorrect check function select add_job('test_proc_with_check', '5 secs', config => '{}', check_config => 'test_config_check_func_0args'::regproc); ERROR: function or procedure public.test_config_check_func_0args(config jsonb) not found -- -- so should this select add_job('test_proc_with_check', '5 secs', config => '{}', check_config => 'test_config_check_func_2args'::regproc); ERROR: function or procedure public.test_config_check_func_2args(config jsonb) not found -- and this select add_job('test_proc_with_check', '5 secs', config => '{}', check_config => 'test_config_check_func_intarg'::regproc); ERROR: function or procedure public.test_config_check_func_intarg(config jsonb) not found -- and this fails as it calls a nonexistent function select add_job('test_proc_with_check', '5 secs', config => '{}', check_config => 'test_nonexistent_check_func'::regproc); ERROR: function "test_nonexistent_check_func" does not exist at character 82 -- when called with a valid check function and a NULL config no check should occur CREATE OR REPLACE FUNCTION test_config_check_func(config jsonb) RETURNS VOID AS $$ BEGIN RAISE NOTICE 'This message will get printed for both NULL and not NULL config'; END $$ LANGUAGE PLPGSQL; SET client_min_messages = NOTICE; -- check done for both NULL and non-NULL config select add_job('test_proc_with_check', '5 secs', config => NULL, check_config => 'test_config_check_func'::regproc); NOTICE: This message will get printed for both NULL and not NULL config add_job --------- 1007 -- check done select add_job('test_proc_with_check', '5 secs', config => '{}', check_config => 'test_config_check_func'::regproc) as job_id \gset NOTICE: This message will get printed for both NULL and not NULL config -- check function not returning void CREATE OR REPLACE FUNCTION test_config_check_func_returns_int(config jsonb) RETURNS INT AS $$ BEGIN raise notice 'I print a message, and then I return least(1,2)'; RETURN LEAST(1, 2); END $$ LANGUAGE PLPGSQL; select add_job('test_proc_with_check', '5 secs', config => '{}', check_config => 'test_config_check_func_returns_int'::regproc, initial_start => :'time_zero'::timestamptz) as job_id_int \gset NOTICE: I print a message, and then I return least(1,2) -- rename the check function and then call alter_job to register the new name ALTER FUNCTION test_config_check_func RENAME TO renamed_func; select job_id, schedule_interval, config, check_config from alter_job(:job_id, check_config => 'renamed_func'::regproc, schedule_interval => '1 hour'); NOTICE: This message will get printed for both NULL and not NULL config job_id | schedule_interval | config | check_config --------+-------------------+--------+--------------------- 1008 | @ 1 hour | {} | public.renamed_func -- run alter again, should get a config check select job_id, schedule_interval, config, check_config from alter_job(:job_id, config => '{}'); NOTICE: This message will get printed for both NULL and not NULL config job_id | schedule_interval | config | check_config --------+-------------------+--------+--------------------- 1008 | @ 1 hour | {} | public.renamed_func -- drop the registered check function, verify that alter_job will work and print a warning that -- the check is being skipped due to the check function missing DROP FUNCTION test_config_check_func_returns_int; select job_id, schedule_interval, config, check_config from alter_job(:job_id_int, config => '{"field":"value"}'); WARNING: function public.test_config_check_func_returns_int(config jsonb) not found, skipping config validation for job 1009 job_id | schedule_interval | config | check_config --------+-------------------+--------------------+------------------------------------------- 1009 | @ 5 secs | {"field": "value"} | public.test_config_check_func_returns_int -- do not drop the current check function but register a new one CREATE OR REPLACE FUNCTION substitute_check_func(config jsonb) RETURNS VOID AS $$ BEGIN RAISE NOTICE 'This message is a substitute of the previously printed one'; END $$ LANGUAGE PLPGSQL; -- register the new check select job_id, schedule_interval, config, check_config from alter_job(:job_id, check_config => 'substitute_check_func'); NOTICE: This message is a substitute of the previously printed one job_id | schedule_interval | config | check_config --------+-------------------+--------+------------------------------ 1008 | @ 1 hour | {} | public.substitute_check_func select job_id, schedule_interval, config, check_config from alter_job(:job_id, config => '{}'); NOTICE: This message is a substitute of the previously printed one job_id | schedule_interval | config | check_config --------+-------------------+--------+------------------------------ 1008 | @ 1 hour | {} | public.substitute_check_func RESET client_min_messages; -- test an oid that doesn't exist select add_job('test_proc_with_check', '5 secs', config => '{}', check_config => 17424217::regproc); ERROR: function with OID 17424217 does not exist \c :TEST_DBNAME :ROLE_SUPERUSER -- test a function with insufficient privileges create schema test_schema; create role user_noexec with login; grant usage on schema test_schema to user_noexec; CREATE OR REPLACE FUNCTION test_schema.test_config_check_func_privileges(config jsonb) RETURNS VOID AS $$ BEGIN RAISE NOTICE 'This message will only get printed if privileges suffice'; END $$ LANGUAGE PLPGSQL; revoke execute on function test_schema.test_config_check_func_privileges from public; -- verify the user doesn't have execute permissions on the function select has_function_privilege('user_noexec', 'test_schema.test_config_check_func_privileges(jsonb)', 'execute'); has_function_privilege ------------------------ f \c :TEST_DBNAME user_noexec -- user_noexec should not have exec permissions on this function select add_job('test_proc_with_check', '5 secs', config => '{}', check_config => 'test_schema.test_config_check_func_privileges'::regproc); ERROR: permission denied for function "test_config_check_func_privileges" \c :TEST_DBNAME :ROLE_SUPERUSER -- check that alter_job rejects a check function with invalid signature select add_job('test_proc_with_check', '5 secs', config => '{}', check_config => 'renamed_func', initial_start => :'time_zero'::timestamptz) as job_id_alter \gset NOTICE: This message will get printed for both NULL and not NULL config select job_id, schedule_interval, config, check_config from alter_job(:job_id_alter, check_config => 'test_config_check_func_0args'); ERROR: function or procedure public.test_config_check_func_0args(config jsonb) not found select job_id, schedule_interval, config, check_config from alter_job(:job_id_alter); NOTICE: This message will get printed for both NULL and not NULL config job_id | schedule_interval | config | check_config --------+-------------------+--------+--------------------- 1010 | @ 5 secs | {} | public.renamed_func -- test that we can unregister the check function select job_id, schedule_interval, config, check_config from alter_job(:job_id_alter, check_config => 0); job_id | schedule_interval | config | check_config --------+-------------------+--------+-------------- 1010 | @ 5 secs | {} | -- no message printed now select job_id, schedule_interval, config, check_config from alter_job(:job_id_alter, config => '{}'); job_id | schedule_interval | config | check_config --------+-------------------+--------+-------------- 1010 | @ 5 secs | {} | -- test the case where we have a background job that registers jobs with a check fn CREATE OR REPLACE PROCEDURE add_scheduled_jobs_with_check(job_id int, config jsonb) LANGUAGE PLPGSQL AS $$ BEGIN perform add_job('test_proc_with_check', schedule_interval => '10 secs', config => '{}', check_config => 'renamed_func'); END $$; select add_job('add_scheduled_jobs_with_check', schedule_interval => '1 hour') as last_job_id \gset -- wait for enough time SELECT _timescaledb_functions.restart_background_workers(); restart_background_workers ---------------------------- t SELECT test.wait_for_job_to_run(:last_job_id, 1); wait_for_job_to_run --------------------- t select total_runs, total_successes, last_run_status from timescaledb_information.job_stats where job_id = :last_job_id; total_runs | total_successes | last_run_status ------------+-----------------+----------------- 1 | 1 | Success -- test coverage for alter_job -- registering an invalid oid select alter_job(:job_id_alter, check_config => 123456789::regproc); ERROR: function with OID 123456789 does not exist -- registering a function with insufficient privileges \c :TEST_DBNAME user_noexec select * from add_job('test_proc_with_check', '5 secs', config => '{}') as job_id_owner \gset select * from alter_job(:job_id_owner, check_config => 'test_schema.test_config_check_func_privileges'::regproc); ERROR: permission denied for function "test_config_check_func_privileges" \c :TEST_DBNAME :ROLE_SUPERUSER DROP SCHEMA test_schema CASCADE; NOTICE: drop cascades to function test_schema.test_config_check_func_privileges(jsonb) -- Delete all jobs with that owner before we can drop the user. DELETE FROM _timescaledb_catalog.bgw_job WHERE owner = 'user_noexec'::regrole; DROP ROLE user_noexec; -- test with aggregate check proc create function jsonb_add (j1 jsonb, j2 jsonb) returns jsonb AS $$ BEGIN RETURN j1 || j2; END $$ LANGUAGE PLPGSQL; CREATE AGGREGATE sum_jsb (jsonb) ( sfunc = jsonb_add, stype = jsonb, initcond = '{}' ); -- for test coverage, check unsupported aggregate type select add_job('test_proc_with_check', '5 secs', config => '{}', check_config => 'sum_jsb'::regproc); ERROR: unsupported function type -- Cleanup jobs TRUNCATE _timescaledb_catalog.bgw_job CASCADE; NOTICE: truncate cascades to table "bgw_job_stat" NOTICE: truncate cascades to table "bgw_policy_chunk_stats" -- github issue 4610 CREATE TABLE sensor_data ( time timestamptz not null, sensor_id integer not null, cpu double precision null, temperature double precision null ); SELECT FROM create_hypertable('sensor_data','time'); -- SELECT '2022-10-06 00:00:00+00' as start_date_sd \gset INSERT INTO sensor_data SELECT time + (INTERVAL '1 minute' * random()) AS time, sensor_id, random() AS cpu, random()* 100 AS temperature FROM generate_series(:'start_date_sd'::timestamptz - INTERVAL '1 months', :'start_date_sd'::timestamptz - INTERVAL '1 week', INTERVAL '30 minute') AS g1(time), generate_series(1, 50, 1 ) AS g2(sensor_id) ORDER BY time; -- enable compression ALTER TABLE sensor_data SET (timescaledb.compress, timescaledb.compress_orderby = 'time DESC'); -- create new chunks INSERT INTO sensor_data SELECT time + (INTERVAL '1 minute' * random()) AS time, sensor_id, random() AS cpu, random()* 100 AS temperature FROM generate_series(:'start_date_sd'::timestamptz - INTERVAL '2 months', :'start_date_sd'::timestamptz - INTERVAL '2 week', INTERVAL '60 minute') AS g1(time), generate_series(1, 30, 1 ) AS g2(sensor_id) ORDER BY time; -- get the name of a new uncompressed chunk SELECT chunk_name AS new_uncompressed_chunk_name FROM timescaledb_information.chunks WHERE hypertable_name = 'sensor_data' AND NOT is_compressed LIMIT 1 \gset -- change compression status so that this chunk is skipped when policy is run update _timescaledb_catalog.chunk set status=3 where table_name = :'new_uncompressed_chunk_name'; -- add new compression policy job SELECT add_compression_policy('sensor_data', INTERVAL '1' minute) AS compressjob_id \gset -- set recompress to true SELECT alter_job(id,config:=jsonb_set(config,'{recompress}', 'true')) FROM _timescaledb_catalog.bgw_job WHERE id = :compressjob_id; alter_job --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1014,"@ 12 hours","@ 0",-1,"@ 1 hour",t,"{""recompress"": true, ""hypertable_id"": 4, ""compress_after"": ""@ 1 min""}",-infinity,_timescaledb_functions.policy_compression_check,f,,,"Columnstore Policy [1014]") -- verify that there are other uncompressed new chunks that need to be compressed SELECT count(*) > 1 FROM timescaledb_information.chunks WHERE hypertable_name = 'sensor_data' AND NOT is_compressed; ?column? ---------- t -- disable notice/warning as the new_uncompressed_chunk_name -- is dynamic and it will be printed in those messages. SET client_min_messages TO ERROR; CALL run_job(:compressjob_id); ERROR: columnstore policy failure SET client_min_messages TO NOTICE; -- check compression status is not changed for the chunk whose status was manually updated SELECT status FROM _timescaledb_catalog.chunk where table_name = :'new_uncompressed_chunk_name'; status -------- 3 -- confirm all the other new chunks are now compressed despite -- facing an error when trying to compress :'new_uncompressed_chunk_name' SELECT count(*) = 0 FROM timescaledb_information.chunks WHERE hypertable_name = 'sensor_data' AND NOT is_compressed; ?column? ---------- t -- cleanup SELECT _timescaledb_functions.stop_background_workers(); stop_background_workers ------------------------- t DROP TABLE sensor_data; SELECT _timescaledb_functions.restart_background_workers(); restart_background_workers ---------------------------- t -- Github issue #5537 -- Proc that waits until the given job enters the expected state CREATE OR REPLACE PROCEDURE wait_for_job_status(job_param_id INTEGER, expected_status TEXT, spins INTEGER=:TEST_SPINWAIT_ITERS) LANGUAGE PLPGSQL AS $$ DECLARE jobstatus TEXT; BEGIN FOR i in 1..spins LOOP SELECT job_status FROM timescaledb_information.job_stats WHERE job_id = job_param_id INTO jobstatus; IF jobstatus = expected_status THEN RETURN; END IF; PERFORM pg_sleep(0.1); ROLLBACK; END LOOP; RAISE EXCEPTION 'wait_for_job_status(%): timeout after % tries', job_param_id, spins; END; $$; -- Proc that sleeps for 1m - to keep the test jobs in running state CREATE OR REPLACE PROCEDURE proc_that_sleeps(job_id INT, config JSONB) LANGUAGE PLPGSQL AS $$ BEGIN PERFORM pg_sleep(60); END $$; -- create new jobs and ensure that the second one gets scheduled -- before the first one by adjusting the initial_start values SELECT add_job('proc_that_sleeps', '1h', initial_start => now()::timestamptz + interval '2s') AS job_id_1 \gset SELECT add_job('proc_that_sleeps', '1h', initial_start => now()::timestamptz - interval '2s') AS job_id_2 \gset SELECT _timescaledb_functions.restart_background_workers(); restart_background_workers ---------------------------- t -- wait for the jobs to start running job_2 will start running first CALL wait_for_job_status(:job_id_2, 'Running'); CALL wait_for_job_status(:job_id_1, 'Running'); -- add a new job and wait for it to start SELECT add_job('proc_that_sleeps', '1h') AS job_id_3 \gset CALL wait_for_job_status(:job_id_3, 'Running'); -- verify that none of the jobs crashed SELECT job_id, job_status, next_start, total_runs, total_successes, total_failures FROM timescaledb_information.job_stats WHERE job_id IN (:job_id_1, :job_id_2, :job_id_3) ORDER BY job_id; job_id | job_status | next_start | total_runs | total_successes | total_failures --------+------------+------------+------------+-----------------+---------------- 1015 | Running | -infinity | 1 | 0 | 0 1016 | Running | -infinity | 1 | 0 | 0 1017 | Running | -infinity | 1 | 0 | 0 SELECT job_id, err_message FROM timescaledb_information.job_errors WHERE job_id IN (:job_id_1, :job_id_2, :job_id_3); job_id | err_message --------+------------- -- cleanup SELECT _timescaledb_functions.stop_background_workers(); stop_background_workers ------------------------- t CALL wait_for_job_status(:job_id_1, 'Scheduled'); CALL wait_for_job_status(:job_id_2, 'Scheduled'); CALL wait_for_job_status(:job_id_3, 'Scheduled'); SELECT delete_job(:job_id_1); delete_job ------------ SELECT delete_job(:job_id_2); delete_job ------------ SELECT delete_job(:job_id_3); delete_job ------------ CREATE OR REPLACE FUNCTION ts_test_bgw_job_function_call_string(job_id INTEGER) RETURNS text AS :MODULE_PATHNAME LANGUAGE C STABLE STRICT; \set ON_ERROR_STOP 0 SELECT ts_test_bgw_job_function_call_string(999999); ERROR: job 999999 not found \set ON_ERROR_STOP 1 SELECT add_job('custom_func', '1h') AS job_func \gset SELECT add_job('custom_proc', '1h') AS job_proc \gset SELECT ts_test_bgw_job_function_call_string(:job_func); ts_test_bgw_job_function_call_string ----------------------------------------- SELECT public.custom_func('1018', NULL) SELECT ts_test_bgw_job_function_call_string(:job_proc); ts_test_bgw_job_function_call_string --------------------------------------- CALL public.custom_proc('1019', NULL) SELECT delete_job(:job_func); delete_job ------------ SELECT delete_job(:job_proc); delete_job ------------ SELECT add_job('custom_func', '1h', config => '{"type":"function"}'::jsonb) AS job_func \gset SELECT add_job('custom_proc', '1h', config => '{"type":"procedure"}'::jsonb) AS job_proc \gset SELECT ts_test_bgw_job_function_call_string(:job_func); ts_test_bgw_job_function_call_string ----------------------------------------------------------- SELECT public.custom_func('1020', '{"type": "function"}') SELECT ts_test_bgw_job_function_call_string(:job_proc); ts_test_bgw_job_function_call_string ---------------------------------------------------------- CALL public.custom_proc('1021', '{"type": "procedure"}') -- Remove the procedure and let's check it fallingback to PROKIND_FUNCTION DROP PROCEDURE custom_proc(jobid int, args jsonb); SELECT ts_test_bgw_job_function_call_string(:job_proc); ts_test_bgw_job_function_call_string ------------------------------------------------------------ SELECT public.custom_proc('1021', '{"type": "procedure"}') \set ON_ERROR_STOP 0 -- Mess with pg catalog to don't identify the PROKIND BEGIN; UPDATE pg_catalog.pg_proc SET prokind = 'X' WHERE oid = 'custom_func(int,jsonb)'::regprocedure; SELECT ts_test_bgw_job_function_call_string(:job_func); ERROR: unsupported function type: X ROLLBACK; \set ON_ERROR_STOP 1 SELECT delete_job(:job_func); delete_job ------------ SELECT delete_job(:job_proc); delete_job ------------ -- Test work_mem config option in job execution CREATE TABLE work_mem_log(job_id int, work_mem_value text); CREATE OR REPLACE PROCEDURE log_work_mem(job_id int, config jsonb) LANGUAGE PLPGSQL AS $$ BEGIN INSERT INTO work_mem_log VALUES(job_id, current_setting('work_mem')); END $$; -- Add job with work_mem in config SELECT add_job('log_work_mem', '1h', config => '{"work_mem": "123MB"}'::jsonb) AS job_work_mem \gset -- Run the job - work_mem should be set to 123MB before execution CALL run_job(:job_work_mem); -- Verify work_mem was set during job execution SELECT * FROM work_mem_log; job_id | work_mem_value --------+---------------- 1022 | 123MB -- Cleanup SELECT delete_job(:job_work_mem); delete_job ------------ DROP TABLE work_mem_log; DROP PROCEDURE log_work_mem; ================================================ FILE: tsl/test/expected/bgw_db_scheduler.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- -- Setup -- \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(timeout INT = -1, mock_start_time INT = 0) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_run(timeout INT = -1, mock_start_time INT = 0) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_wait_for_scheduler_finish() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_create() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_destroy() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_test_job_sleep(job_id INT, config JSONB) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_reset_time(set_time BIGINT = 0, wait BOOLEAN = false) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION insert_job( application_name NAME, job_type NAME, schedule_interval INTERVAL, max_runtime INTERVAL, retry_period INTERVAL, owner regrole DEFAULT CURRENT_ROLE::regrole, scheduled BOOL DEFAULT true, fixed_schedule BOOL DEFAULT false ) RETURNS INT LANGUAGE SQL SECURITY DEFINER AS $$ INSERT INTO _timescaledb_catalog.bgw_job(application_name,schedule_interval,max_runtime,max_retries, retry_period,proc_name,proc_schema,owner,scheduled,fixed_schedule) VALUES($1,$3,$4,5,$5,$2,'public',$6,$7,$8) RETURNING id; $$; CREATE OR REPLACE FUNCTION test_toggle_scheduled(job_id INTEGER) RETURNS VOID LANGUAGE SQL SECURITY DEFINER AS $$ UPDATE _timescaledb_catalog.bgw_job SET scheduled = NOT scheduled WHERE id = $1; $$; \set WAIT_ON_JOB 0 \set IMMEDIATELY_SET_UNTIL 1 \set WAIT_FOR_OTHER_TO_ADVANCE 2 \set WAIT_FOR_STANDARD_WAITLATCH 3 CREATE OR REPLACE FUNCTION ts_bgw_params_mock_wait_returns_immediately(new_val INTEGER) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE FUNCTION get_application_pid(app_name TEXT) RETURNS INTEGER LANGUAGE SQL AS $BODY$ SELECT pid FROM pg_stat_activity WHERE application_name = app_name; $BODY$; CREATE FUNCTION wait_application_pid(app_name TEXT, wait_for_start BOOLEAN = true) RETURNS INTEGER LANGUAGE PLPGSQL AS $BODY$ DECLARE r INTEGER; BEGIN --wait up to a second checking each 100ms FOR i in 1..10 LOOP SELECT get_application_pid(app_name) INTO r; IF (wait_for_start AND r IS NULL) OR (NOT wait_for_start AND r IS NOT NULL) THEN PERFORM pg_sleep(0.1); PERFORM pg_stat_clear_snapshot(); ELSE RETURN r; END IF; END LOOP; RETURN NULL; END $BODY$; CREATE FUNCTION wait_for_logentry(job_id INTEGER) RETURNS TEXT LANGUAGE PLPGSQL AS $BODY$ DECLARE app_name TEXT; message TEXT; BEGIN SELECT application_name INTO app_name FROM _timescaledb_catalog.bgw_job WHERE id = job_id; --wait up to a second checking each 100ms FOR i in 1..10 LOOP SELECT msg INTO message FROM bgw_log WHERE application_name = app_name ORDER BY msg_no DESC LIMIT 1; IF FOUND THEN RETURN message; END IF; PERFORM pg_sleep(0.1); PERFORM pg_stat_clear_snapshot(); END LOOP; RETURN NULL; END $BODY$; -- Remove any default jobs, e.g., telemetry DELETE FROM _timescaledb_catalog.bgw_job WHERE TRUE; TRUNCATE _timescaledb_internal.bgw_job_stat; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE public.bgw_log( msg_no INT, mock_time BIGINT, application_name TEXT, msg TEXT ); CREATE VIEW sorted_bgw_log AS SELECT msg_no, application_name, regexp_replace(regexp_replace(msg, '(Wait until|started at|execution time) [0-9]+(\.[0-9]+)?', '\1 (RANDOM)', 'g'), 'background worker "[^"]+"','connection') AS msg FROM bgw_log ORDER BY mock_time, application_name COLLATE "C", msg_no; CREATE TABLE public.bgw_dsm_handle_store( handle BIGINT ); INSERT INTO public.bgw_dsm_handle_store VALUES (0); SELECT ts_bgw_params_create(); ts_bgw_params_create ---------------------- -- -- Test running the scheduler with no jobs -- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(50); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ -- empty -- turn on extended display to make the many fields of the table easier to parse \x on SELECT * FROM _timescaledb_internal.bgw_job_stat; \x off -- empty SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) -- -- Test running the scheduler with a job marked as unscheduled -- TRUNCATE bgw_log; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- SELECT insert_job('unscheduled', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s',scheduled:= false); insert_job ------------ 1000 SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(50); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ -- empty \x on SELECT * FROM _timescaledb_internal.bgw_job_stat; \x off SELECT * FROM timescaledb_information.job_stats; hypertable_schema | hypertable_name | job_id | last_run_started_at | last_successful_finish | last_run_status | job_status | last_run_duration | next_start | total_runs | total_successes | total_failures -------------------+-----------------+--------+---------------------+------------------------+-----------------+------------+-------------------+------------+------------+-----------------+---------------- | | 1000 | | | | Paused | | | | | SELECT test_toggle_scheduled(1000); test_toggle_scheduled ----------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(50); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ \x on SELECT * FROM _timescaledb_internal.bgw_job_stat; -[ RECORD 1 ]-----------+-------------------------------- job_id | 1000 last_start | Fri Dec 31 16:00:00.05 1999 PST last_finish | Fri Dec 31 16:00:00.05 1999 PST next_start | Fri Dec 31 16:00:00.15 1999 PST last_successful_finish | Fri Dec 31 16:00:00.05 1999 PST last_run_success | t total_runs | 1 total_duration | @ 0 total_duration_failures | @ 0 total_successes | 1 total_failures | 0 total_crashes | 0 consecutive_failures | 0 consecutive_crashes | 0 flags | 0 \x off SELECT * FROM timescaledb_information.job_stats; hypertable_schema | hypertable_name | job_id | last_run_started_at | last_successful_finish | last_run_status | job_status | last_run_duration | next_start | total_runs | total_successes | total_failures -------------------+-----------------+--------+---------------------------------+---------------------------------+-----------------+------------+-------------------+---------------------------------+------------+-----------------+---------------- | | 1000 | Fri Dec 31 16:00:00.05 1999 PST | Fri Dec 31 16:00:00.05 1999 PST | Success | Scheduled | | Fri Dec 31 16:00:00.15 1999 PST | 1 | 1 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | unscheduled | Execute job 1 SELECT delete_job(1000); delete_job ------------ -- -- test deleting job also terminates running jobs -- SELECT add_job('ts_bgw_test_job_sleep','1h') AS job_id \gset SELECT ts_bgw_db_scheduler_test_run(); ts_bgw_db_scheduler_test_run ------------------------------ SELECT wait_for_logentry(:job_id); wait_for_logentry ------------------- Before sleep SELECT application_name FROM pg_stat_activity WHERE application_name LIKE 'User-Defined Action%'; application_name ---------------------------- User-Defined Action [1001] \x on SELECT job_id, job_status FROM timescaledb_information.job_stats; -[ RECORD 1 ]------- job_id | 1001 job_status | Running -- Showing non-volatile information from pg_stat_activity for -- debugging purposes. Information schema above reads from this view. SELECT datname, usename, application_name, state, query, wait_event_type, wait_event FROM pg_stat_activity WHERE application_name LIKE 'User-Defined Action%'; -[ RECORD 1 ]----+------------------------------------ datname | db_bgw_db_scheduler usename | default_perm_user application_name | User-Defined Action [1001] state | active query | CALL public.ts_bgw_test_job_sleep() wait_event_type | Timeout wait_event | PgSleep \x off SELECT delete_job(:job_id); delete_job ------------ SELECT wait_application_pid('User-Defined Action [' || :job_id || ']', false); wait_application_pid ---------------------- SELECT application_name FROM pg_stat_activity WHERE application_name LIKE 'User-Defined Action%'; application_name ------------------ -- wait for scheduler finish SELECT ts_bgw_db_scheduler_test_wait_for_scheduler_finish(); ts_bgw_db_scheduler_test_wait_for_scheduler_finish ---------------------------------------------------- -- -- Test running a normal job -- \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; ALTER SEQUENCE _timescaledb_catalog.bgw_job_id_seq RESTART; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- SELECT insert_job('test_job_1', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'); insert_job ------------ 1000 select * from _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ------+------------------+-------------------+-----------------+-------------+--------------+-------------+----------------+------------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- 1000 | test_job_1 | @ 0.1 secs | @ 1 min 40 secs | 5 | @ 1 sec | public | bgw_test_job_1 | super_user | t | f | | | | | | \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --Tests that the scheduler start a job right away if it's the first time and there is no job_stat entry for it SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, next_start, last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | next_start | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+--------------------------------+------------------------------+------------------+------------+-----------------+----------------+--------------- 1000 | Fri Dec 31 16:00:00.1 1999 PST | Fri Dec 31 16:00:00 1999 PST | t | 1 | 1 | 0 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_1 | Execute job 1 --Test that the scheduler will not run job again if not enough time has passed SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25, 25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1000 | t | 1 | 1 | 0 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_1 | Execute job 1 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) --After enough time has passed the scheduler will run the job again SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(100, 50); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, next_start, last_finish, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | next_start | last_finish | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+--------------------------------+--------------------------------+------------------+------------+-----------------+----------------+--------------- 1000 | Fri Dec 31 16:00:00.2 1999 PST | Fri Dec 31 16:00:00.1 1999 PST | t | 2 | 2 | 0 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_1 | Execute job 1 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_1 | Execute job 1 --Now it runs it one more time SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(120, 100); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, next_start, last_finish, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | next_start | last_finish | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+--------------------------------+--------------------------------+------------------+------------+-----------------+----------------+--------------- 1000 | Fri Dec 31 16:00:00.3 1999 PST | Fri Dec 31 16:00:00.2 1999 PST | t | 3 | 3 | 0 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_1 | Execute job 1 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_1 | Execute job 1 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_1 | Execute job 1 -- -- Test what happens when running a job that throws an error -- \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; TRUNCATE _timescaledb_internal.bgw_job_stat; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- DELETE FROM _timescaledb_catalog.bgw_job; SELECT insert_job('test_job_2', 'bgw_test_job_2_error', INTERVAL '100ms', INTERVAL '100s', INTERVAL '100ms') AS test_job_2_id \gset \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --Run the first time and error SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1001 | f | 1 | 0 | 1 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 SELECT last_finish, last_successful_finish, last_run_success FROM _timescaledb_internal.bgw_job_stat; last_finish | last_successful_finish | last_run_success ------------------------------+------------------------+------------------ Fri Dec 31 16:00:00 1999 PST | -infinity | f --Scheduler runs the job again, sees another error, and increases the wait time SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(125); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1001 | f | 2 | 0 | 2 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 --The job runs and fails again a few more times increasing the wait time each time. SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(225); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1001 | f | 3 | 0 | 3 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(425); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1001 | f | 4 | 0 | 4 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 --Once the wait time reaches 500ms it stops increasion SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(525); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1001 | f | 5 | 0 | 5 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+----------------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 reached max_retries after 5 consecutive failures 2 | test_job_2 | job 1001 threw an error 3 | test_job_2 | Error job 2 -- Get status of failing job `test_job_2` to check it reached `max_retries` and -- the new `job_status` now is `Paused` SELECT job_id, last_run_status, job_status, total_runs, total_successes, total_failures FROM timescaledb_information.job_stats WHERE job_id = :test_job_2_id; job_id | last_run_status | job_status | total_runs | total_successes | total_failures --------+-----------------+------------+------------+-----------------+---------------- 1001 | Failed | Paused | 5 | 0 | 5 -- Alter job to be rescheduled and run it again \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; SELECT scheduled FROM alter_job(:test_job_2_id, scheduled => true) AS discard; scheduled ----------- t \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(525); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat WHERE job_id = :test_job_2_id; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1001 | f | 6 | 0 | 6 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+----------------------------------------------------------- 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 reached max_retries after 6 consecutive failures 2 | test_job_2 | job 1001 threw an error 3 | test_job_2 | Error job 2 -- -- Test timeout logic -- \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; TRUNCATE _timescaledb_internal.bgw_job_stat; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- DELETE FROM _timescaledb_catalog.bgw_job; --set timeout lower than job length SELECT insert_job('test_job_3_long', 'bgw_test_job_3_long', INTERVAL '5000ms', INTERVAL '20ms', INTERVAL '50ms'); insert_job ------------ 1002 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT ts_bgw_params_mock_wait_returns_immediately(:IMMEDIATELY_SET_UNTIL); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- --Test that the scheduler kills a job that takes too long SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(200); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes, consecutive_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes | consecutive_crashes --------+------------------+------------+-----------------+----------------+---------------+--------------------- 1002 | f | 1 | 0 | 1 | 0 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 2 | DB Scheduler | terminating connection due to timeout 3 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 4 | DB Scheduler | job 1002 failed --Check that the scheduler does not kill a job with infinite timeout \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; TRUNCATE _timescaledb_internal.bgw_job_stat; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- DELETE FROM _timescaledb_catalog.bgw_job; --set timeout to 0 SELECT insert_job('test_job_3_long', 'bgw_test_job_3_long', INTERVAL '5000ms', INTERVAL '0', INTERVAL '10ms'); insert_job ------------ 1003 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(550); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes, consecutive_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes | consecutive_crashes --------+------------------+------------+-----------------+----------------+---------------+--------------------- 1003 | t | 1 | 1 | 0 | 0 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_3_long | Before sleep job 3 1 | test_job_3_long | After sleep job 3 SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_ON_JOB); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- -- -- Test signal handling -- --Test sending a SIGTERM to a job \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- TRUNCATE _timescaledb_internal.bgw_job_stat; DELETE FROM _timescaledb_catalog.bgw_job; SELECT insert_job('test_job_3_long', 'bgw_test_job_3_long', INTERVAL '5000ms', INTERVAL '100s', INTERVAL '500ms'); insert_job ------------ 1004 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --escalated priv needed for access to pg_stat_activity \c :TEST_DBNAME :ROLE_SUPERUSER SELECT ts_bgw_db_scheduler_test_run(300); ts_bgw_db_scheduler_test_run ------------------------------ SELECT pg_terminate_backend(wait_application_pid('test_job_3_long')); pg_terminate_backend ---------------------- t SELECT ts_bgw_db_scheduler_test_wait_for_scheduler_finish(); ts_bgw_db_scheduler_test_wait_for_scheduler_finish ---------------------------------------------------- SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+----------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_3_long | Before sleep job 3 1 | test_job_3_long | terminating connection due to administrator command 2 | DB Scheduler | job 1004 failed SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1004 | f | 1 | 0 | 1 | 0 -- Test that the job is able to run again and succeed SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(900); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1004 | t | 2 | 1 | 1 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+----------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_3_long | Before sleep job 3 1 | test_job_3_long | terminating connection due to administrator command 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 2 | DB Scheduler | job 1004 failed 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_3_long | Before sleep job 3 1 | test_job_3_long | After sleep job 3 --Test sending a SIGHUP to a job \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- TRUNCATE _timescaledb_internal.bgw_job_stat; DELETE FROM _timescaledb_catalog.bgw_job; SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_FOR_STANDARD_WAITLATCH); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- SELECT ts_bgw_db_scheduler_test_run(-1); ts_bgw_db_scheduler_test_run ------------------------------ SHOW timescaledb.shutdown_bgw_scheduler; timescaledb.shutdown_bgw_scheduler ------------------------------------ off ALTER SYSTEM SET timescaledb.shutdown_bgw_scheduler TO 'on'; SELECT pg_reload_conf(); pg_reload_conf ---------------- t \c :TEST_DBNAME :ROLE_SUPERUSER SHOW timescaledb.shutdown_bgw_scheduler; timescaledb.shutdown_bgw_scheduler ------------------------------------ on SELECT ts_bgw_db_scheduler_test_wait_for_scheduler_finish(); ts_bgw_db_scheduler_test_wait_for_scheduler_finish ---------------------------------------------------- -- The number of scheduler restarts is not deterministic during [..]_wait_for_scheduler_finish(). -- Therefore, we filter these messages to get a deterministic test output. SELECT * FROM sorted_bgw_log WHERE msg NOT LIKE '[TESTING] Wait until%'; msg_no | application_name | msg --------+------------------+----------------------------------------------- 0 | DB Scheduler | bgw scheduler stopped due to shutdown_bgw guc ALTER SYSTEM RESET timescaledb.shutdown_bgw_scheduler; SELECT pg_reload_conf(); pg_reload_conf ---------------- t \c :TEST_DBNAME :ROLE_SUPERUSER SHOW timescaledb.shutdown_bgw_scheduler; timescaledb.shutdown_bgw_scheduler ------------------------------------ off SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_ON_JOB); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- --Test that sending SIGTERM to scheduler terminates the jobs as well \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; TRUNCATE _timescaledb_internal.bgw_job_stat; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- DELETE FROM _timescaledb_catalog.bgw_job; SELECT insert_job('test_job_3_long', 'bgw_test_job_3_long', INTERVAL '5000ms', INTERVAL '100s', INTERVAL '10ms'); insert_job ------------ 1005 SELECT ts_bgw_db_scheduler_test_run(500); ts_bgw_db_scheduler_test_run ------------------------------ SELECT wait_application_pid('test_job_3_long') IS NOT NULL ; ?column? ---------- t SELECT pg_terminate_backend(wait_application_pid('DB Scheduler Test')); pg_terminate_backend ---------------------- t SELECT ts_bgw_db_scheduler_test_wait_for_scheduler_finish(); ts_bgw_db_scheduler_test_wait_for_scheduler_finish ---------------------------------------------------- SELECT job_id, last_finish, last_run_success, total_runs, total_successes, total_failures, total_crashes, consecutive_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_finish | last_run_success | total_runs | total_successes | total_failures | total_crashes | consecutive_crashes --------+-------------+------------------+------------+-----------------+----------------+---------------+--------------------- 1005 | -infinity | f | 1 | 0 | 0 | 1 | 1 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+----------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 2 | DB Scheduler | terminating connection due to administrator command 0 | test_job_3_long | Before sleep job 3 1 | test_job_3_long | terminating connection due to administrator command --After a SIGTERM to scheduler and jobs, the jobs are considered crashed and there is a imposed wait of 5 min before a job can be run. --See that there is no run again because of the crash-imposed wait (not run with the 10ms retry_period) SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(500); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_finish, next_start, last_run_success, total_runs, total_successes, total_failures, total_crashes, consecutive_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_finish | next_start | last_run_success | total_runs | total_successes | total_failures | total_crashes | consecutive_crashes --------+-------------+------------+------------------+------------+-----------------+----------------+---------------+--------------------- 1005 | -infinity | -infinity | f | 1 | 0 | 0 | 1 | 1 --But after the 5 min period the job is again run SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(400000); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_finish, next_start, last_run_success, total_runs, total_successes, total_failures, total_crashes, consecutive_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_finish | next_start | last_run_success | total_runs | total_successes | total_failures | total_crashes | consecutive_crashes --------+--------------------------------+--------------------------------+------------------+------------+-----------------+----------------+---------------+--------------------- 1005 | Fri Dec 31 16:05:00.5 1999 PST | Fri Dec 31 16:05:05.5 1999 PST | t | 2 | 1 | 0 | 1 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+----------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 2 | DB Scheduler | terminating connection due to administrator command 0 | test_job_3_long | Before sleep job 3 1 | test_job_3_long | terminating connection due to administrator command 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_3_long | Before sleep job 3 1 | test_job_3_long | After sleep job 3 CREATE FUNCTION wait_for_timer_to_run(started_at INTEGER, spins INTEGER=:TEST_SPINWAIT_ITERS) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE num_runs INTEGER; message TEXT; BEGIN select format('[TESTING] Wait until %%, started at %s', started_at) into message; FOR i in 1..spins LOOP SELECT COUNT(*) from bgw_log where msg LIKE message INTO num_runs; if (num_runs > 0) THEN RETURN true; ELSE PERFORM pg_sleep(0.1); END IF; END LOOP; RETURN false; END $BODY$; CREATE FUNCTION wait_for_job_3_to_finish(runs INTEGER, spins INTEGER=:TEST_SPINWAIT_ITERS) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE num_runs INTEGER; BEGIN FOR i in 1..spins LOOP SELECT COUNT(*) from bgw_log where msg='After sleep job 3' INTO num_runs; if (num_runs = runs) THEN RETURN true; ELSE PERFORM pg_sleep(0.1); END IF; END LOOP; RETURN false; END $BODY$; -- -- Test starting more jobs than availlable workers -- \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; TRUNCATE _timescaledb_internal.bgw_job_stat; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- DELETE FROM _timescaledb_catalog.bgw_job; SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_FOR_OTHER_TO_ADVANCE); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- --Our normal limit is 8 jobs (1 already taken up by the launcher, we don't register the test scheduler) --so start 8 workers. Make the schedule_INTERVAL long and the retry period short so that the --retries happen within the scheduler run time but everything only runs once. SELECT insert_job('test_job_3_long_1', 'bgw_test_job_3_long', INTERVAL '5000ms', INTERVAL '100s', INTERVAL '10ms'), insert_job('test_job_3_long_2', 'bgw_test_job_3_long', INTERVAL '5000ms', INTERVAL '100s', INTERVAL '10ms'), insert_job('test_job_3_long_3', 'bgw_test_job_3_long', INTERVAL '5000ms', INTERVAL '100s', INTERVAL '10ms'), insert_job('test_job_3_long_4', 'bgw_test_job_3_long', INTERVAL '5000ms', INTERVAL '100s', INTERVAL '10ms'), insert_job('test_job_3_long_5', 'bgw_test_job_3_long', INTERVAL '5000ms', INTERVAL '100s', INTERVAL '10ms'), insert_job('test_job_3_long_6', 'bgw_test_job_3_long', INTERVAL '5000ms', INTERVAL '100s', INTERVAL '10ms'); insert_job | insert_job | insert_job | insert_job | insert_job | insert_job ------------+------------+------------+------------+------------+------------ 1006 | 1007 | 1008 | 1009 | 1010 | 1011 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT ts_bgw_db_scheduler_test_run(25000); --quit at second 25 ts_bgw_db_scheduler_test_run ------------------------------ --the first 7 jobs will run right away, but not the last one SELECT wait_for_timer_to_run(0); wait_for_timer_to_run ----------------------- t SELECT wait_for_job_3_to_finish(6); wait_for_job_3_to_finish -------------------------- t SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes, consecutive_crashes FROM _timescaledb_internal.bgw_job_stat ORDER BY job_id; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes | consecutive_crashes --------+------------------+------------+-----------------+----------------+---------------+--------------------- 1006 | t | 1 | 1 | 0 | 0 | 0 1007 | t | 1 | 1 | 0 | 0 | 0 1008 | t | 1 | 1 | 0 | 0 | 0 1009 | t | 1 | 1 | 0 | 0 | 0 1010 | t | 1 | 1 | 0 | 0 | 0 1011 | t | 1 | 1 | 0 | 0 | 0 SELECT ts_bgw_params_reset_time(30000000, true); --set to second 30, which causes a quit. ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_wait_for_scheduler_finish(); ts_bgw_db_scheduler_test_wait_for_scheduler_finish ---------------------------------------------------- --should have all 8 runs, all with success runs SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes, consecutive_crashes FROM _timescaledb_internal.bgw_job_stat ORDER BY job_id; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes | consecutive_crashes --------+------------------+------------+-----------------+----------------+---------------+--------------------- 1006 | t | 1 | 1 | 0 | 0 | 0 1007 | t | 1 | 1 | 0 | 0 | 0 1008 | t | 1 | 1 | 0 | 0 | 0 1009 | t | 1 | 1 | 0 | 0 | 0 1010 | t | 1 | 1 | 0 | 0 | 0 1011 | t | 1 | 1 | 0 | 0 | 0 SELECT * FROM sorted_bgw_log WHERE application_name = 'DB Scheduler' ORDER BY application_name, msg_no; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Registered new background worker 3 | DB Scheduler | [TESTING] Registered new background worker 4 | DB Scheduler | [TESTING] Registered new background worker 5 | DB Scheduler | [TESTING] Registered new background worker 6 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) SELECT ts_bgw_params_destroy(); ts_bgw_params_destroy ----------------------- -- -- Test setting next_start time within a job -- \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; TRUNCATE _timescaledb_internal.bgw_job_stat; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_ON_JOB); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- DELETE FROM _timescaledb_catalog.bgw_job; SELECT insert_job('test_job_4', 'bgw_test_job_4', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'); insert_job ------------ 1012 select * from _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ------+------------------+-------------------+-----------------+-------------+--------------+-------------+----------------+------------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- 1012 | test_job_4 | @ 0.1 secs | @ 1 min 40 secs | 5 | @ 1 sec | public | bgw_test_job_4 | super_user | t | f | | | | | | \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- Now run and make sure next_start is 200ms away, not 100ms SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1012 | t | 1 | 1 | 0 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_4 | Execute job 4 -- Now just make sure that the job actually runs in 200ms SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(200); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ -- Print next_start and last_finish explicitly, instead of the difference, to make sure the times have changed -- since the last run SELECT job_id, next_start, last_finish, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | next_start | last_finish | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+--------------------------------+--------------------------------+------------------+------------+-----------------+----------------+--------------- 1012 | Fri Dec 31 16:00:00.4 1999 PST | Fri Dec 31 16:00:00.2 1999 PST | t | 2 | 2 | 0 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_4 | Execute job 4 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_4 | Execute job 4 -- Test updating jobs list TRUNCATE bgw_log; \set ON_ERROR_STOP 0 SELECT _timescaledb_functions.stop_background_workers(); ERROR: must be superuser to stop background workers SELECT _timescaledb_functions.restart_background_workers(); ERROR: must be superuser to restart background workers SELECT _timescaledb_functions.start_background_workers(); ERROR: must be superuser to start background workers \set ON_ERROR_STOP 1 \c :TEST_DBNAME :ROLE_SUPERUSER SELECT _timescaledb_functions.stop_background_workers(); stop_background_workers ------------------------- t CREATE OR REPLACE FUNCTION ts_test_job_refresh() RETURNS TABLE( id INTEGER, application_name NAME, schedule_interval INTERVAL, max_runtime INTERVAL, max_retries INT, retry_period INTERVAL, next_start TIMESTAMPTZ, timeout_at TIMESTAMPTZ, reserved_worker BOOLEAN, may_next_mark_end BOOLEAN ) AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE FUNCTION verify_refresh_correct() RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE num_jobs INTEGER; num_jobs_in_list INTEGER; BEGIN SELECT COUNT(*) from _timescaledb_catalog.bgw_job INTO num_jobs; select COUNT(*) from ts_test_job_refresh() JOIN _timescaledb_catalog.bgw_job USING (id,application_name,schedule_interval,max_runtime,max_retries,retry_period) INTO num_jobs_in_list; IF (num_jobs = num_jobs_in_list) THEN RETURN true; END IF; RETURN false; END $BODY$; CREATE FUNCTION wait_for_job_1_to_run(runs INTEGER, spins INTEGER=:TEST_SPINWAIT_ITERS) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE num_runs INTEGER; BEGIN FOR i in 1..spins LOOP SELECT COUNT(*) from bgw_log where msg='Execute job 1' INTO num_runs; if (num_runs = runs) THEN RETURN true; ELSE PERFORM pg_sleep(0.1); END IF; END LOOP; RETURN false; END $BODY$; select * from verify_refresh_correct(); verify_refresh_correct ------------------------ t -- Should return the same table select * from verify_refresh_correct(); verify_refresh_correct ------------------------ t DELETE FROM _timescaledb_catalog.bgw_job; -- Make sure jobs list is empty select count(*) from ts_test_job_refresh(); count ------- 0 SELECT insert_job('test_1', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'), insert_job('test_2', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'), insert_job('test_3', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'); insert_job | insert_job | insert_job ------------+------------+------------ 1013 | 1014 | 1015 select * from verify_refresh_correct(); verify_refresh_correct ------------------------ t DELETE from _timescaledb_catalog.bgw_job where application_name='test_2'; SELECT insert_job('test_4', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'); insert_job ------------ 1016 select * from verify_refresh_correct(); verify_refresh_correct ------------------------ t DELETE FROM _timescaledb_catalog.bgw_job; SELECT insert_job('test_10', 'test_10', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'); insert_job ------------ 1017 select * from verify_refresh_correct(); verify_refresh_correct ------------------------ t -- Should be idempotent select * from verify_refresh_correct(); verify_refresh_correct ------------------------ t DELETE FROM _timescaledb_catalog.bgw_job; SELECT insert_job('another', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'), insert_job('another1', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'), insert_job('another2', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'), insert_job('another3', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'), insert_job('another4', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'); insert_job | insert_job | insert_job | insert_job | insert_job ------------+------------+------------+------------+------------ 1018 | 1019 | 1020 | 1021 | 1022 select * from verify_refresh_correct(); verify_refresh_correct ------------------------ t DELETE FROM _timescaledb_catalog.bgw_job where application_name='another' OR application_name='another3'; SELECT insert_job('blah', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'); insert_job ------------ 1023 select * from verify_refresh_correct(); verify_refresh_correct ------------------------ t -- Now test a real scheduler-mock running in a loop and updating the list of jobs TRUNCATE _timescaledb_internal.bgw_job_stat; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- DELETE FROM _timescaledb_catalog.bgw_job; SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_FOR_OTHER_TO_ADVANCE); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- SELECT ts_bgw_db_scheduler_test_run(500); ts_bgw_db_scheduler_test_run ------------------------------ -- Wait for scheduler to start up SELECT wait_for_timer_to_run(0); wait_for_timer_to_run ----------------------- t SELECT insert_job('another', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s') AS job_id \gset -- call alter_job to trigger cache invalidation SELECT alter_job(:job_id,scheduled:=true); alter_job ----------------------------------------------------------------------------- (1024,"@ 0.1 secs","@ 1 min 40 secs",5,"@ 1 sec",t,,-infinity,,f,,,another) SELECT ts_bgw_params_reset_time(50000, true); ts_bgw_params_reset_time -------------------------- SELECT wait_for_timer_to_run(50000); wait_for_timer_to_run ----------------------- t SELECT wait_for_job_1_to_run(1); wait_for_job_1_to_run ----------------------- t SELECT ts_bgw_params_reset_time(150000, true); ts_bgw_params_reset_time -------------------------- SELECT wait_for_timer_to_run(150000); wait_for_timer_to_run ----------------------- t SELECT wait_for_job_1_to_run(2); wait_for_job_1_to_run ----------------------- t \x on select * from _timescaledb_internal.bgw_job_stat; -[ RECORD 1 ]-----------+-------------------------------- job_id | 1024 last_start | Fri Dec 31 16:00:00.15 1999 PST last_finish | Fri Dec 31 16:00:00.15 1999 PST next_start | Fri Dec 31 16:00:00.25 1999 PST last_successful_finish | Fri Dec 31 16:00:00.15 1999 PST last_run_success | t total_runs | 2 total_duration | @ 0 total_duration_failures | @ 0 total_successes | 2 total_failures | 0 total_crashes | 0 consecutive_failures | 0 consecutive_crashes | 0 flags | 0 \x off SELECT delete_job(x.id) FROM (select * from _timescaledb_catalog.bgw_job) x; delete_job ------------ -- test null handling in delete_job SELECT delete_job(NULL); delete_job ------------ SELECT ts_bgw_params_reset_time(200000, true); ts_bgw_params_reset_time -------------------------- SELECT wait_for_timer_to_run(200000); wait_for_timer_to_run ----------------------- t -- In the next time interval, nothing should be run because scheduler should have an empty list SELECT ts_bgw_params_reset_time(300000, true); ts_bgw_params_reset_time -------------------------- SELECT wait_for_timer_to_run(300000); wait_for_timer_to_run ----------------------- t -- Same for this time interval SELECT ts_bgw_params_reset_time(400000, true); ts_bgw_params_reset_time -------------------------- SELECT wait_for_timer_to_run(400000); wait_for_timer_to_run ----------------------- t -- Now add a new job and make sure it gets run before the scheduler dies SELECT insert_job('new_job', 'bgw_test_job_1', INTERVAL '10ms', INTERVAL '100s', INTERVAL '1s') AS job_id \gset -- call alter_job to trigger cache invalidation SELECT alter_job(:job_id,scheduled:=true); alter_job ------------------------------------------------------------------------------ (1025,"@ 0.01 secs","@ 1 min 40 secs",5,"@ 1 sec",t,,-infinity,,f,,,new_job) SELECT ts_bgw_params_reset_time(450000, true); ts_bgw_params_reset_time -------------------------- -- New job should be run once, for a total of 3 runs of this job in the log SELECT wait_for_job_1_to_run(3); wait_for_job_1_to_run ----------------------- t -- New job should be run again SELECT ts_bgw_params_reset_time(480000, true); ts_bgw_params_reset_time -------------------------- SELECT wait_for_job_1_to_run(4); wait_for_job_1_to_run ----------------------- t SELECT ts_bgw_params_reset_time(500000, true); ts_bgw_params_reset_time -------------------------- SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | another | Execute job 1 3 | DB Scheduler | [TESTING] Registered new background worker 4 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | another | Execute job 1 5 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 6 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 7 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 8 | DB Scheduler | [TESTING] Registered new background worker 9 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | new_job | Execute job 1 10 | DB Scheduler | [TESTING] Registered new background worker 11 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | new_job | Execute job 1 \x on SELECT * FROM _timescaledb_internal.bgw_job_stat; -[ RECORD 1 ]-----------+-------------------------------- job_id | 1025 last_start | Fri Dec 31 16:00:00.48 1999 PST last_finish | Fri Dec 31 16:00:00.48 1999 PST next_start | Fri Dec 31 16:00:00.49 1999 PST last_successful_finish | Fri Dec 31 16:00:00.48 1999 PST last_run_success | t total_runs | 2 total_duration | @ 0 total_duration_failures | @ 0 total_successes | 2 total_failures | 0 total_crashes | 0 consecutive_failures | 0 consecutive_crashes | 0 flags | 0 \x off -- -- Test without retry -- \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; TRUNCATE _timescaledb_internal.bgw_job_stat; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_ON_JOB); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- DELETE FROM _timescaledb_catalog.bgw_job; INSERT INTO _timescaledb_catalog.bgw_job(application_name, schedule_interval, max_runtime, max_retries, retry_period, proc_schema, proc_name) VALUES('bgw_test_job_2_error', INTERVAL '5000ms', INTERVAL '20ms', 0, INTERVAL '20ms', 'public', 'bgw_test_job_2_error') RETURNING id; id ------ 1026 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- Run the first time SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1026 | f | 1 | 0 | 1 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+----------------------+----------------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | bgw_test_job_2_error | job 1026 reached max_retries after 1 consecutive failures 2 | bgw_test_job_2_error | job 1026 threw an error 3 | bgw_test_job_2_error | Error job 2 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) SELECT last_finish, last_successful_finish, last_run_success FROM _timescaledb_internal.bgw_job_stat; last_finish | last_successful_finish | last_run_success ------------------------------+------------------------+------------------ Fri Dec 31 16:00:00 1999 PST | -infinity | f -- Run the second time SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(100, 50); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1026 | f | 1 | 0 | 1 | 0 -- We increase the mock time a lot to ensure the job does not get restarted. However, the amount of scheduler sleep/wakeup cycles -- is not deterministic. Therefore, we filter these messages to get a deterministic test output. SELECT * FROM sorted_bgw_log WHERE msg NOT LIKE '[TESTING] Wait until%'; msg_no | application_name | msg --------+----------------------+----------------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | bgw_test_job_2_error | job 1026 reached max_retries after 1 consecutive failures 2 | bgw_test_job_2_error | job 1026 threw an error 3 | bgw_test_job_2_error | Error job 2 SELECT last_finish, last_successful_finish, last_run_success FROM _timescaledb_internal.bgw_job_stat; last_finish | last_successful_finish | last_run_success ------------------------------+------------------------+------------------ Fri Dec 31 16:00:00 1999 PST | -infinity | f -- clean up jobs \c :TEST_DBNAME :ROLE_SUPERUSER SELECT _timescaledb_functions.stop_background_workers(); stop_background_workers ------------------------- t ================================================ FILE: tsl/test/expected/bgw_db_scheduler_fixed.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- -- Setup -- \c :TEST_DBNAME :ROLE_SUPERUSER SET timezone TO PST8PDT; -- this mock_start_time doesnt seem to be used anywhere CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(timeout INT = -1, mock_start_time INT = 0) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_run(timeout INT = -1, mock_start_time INT = 0) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_wait_for_scheduler_finish() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_create() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_destroy() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_test_job_sleep(job_id INT, config JSONB) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_reset_time(set_time BIGINT = 0, wait BOOLEAN = false) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; -- we use insert_job instead of add_job because we want to be able to set and use max_retries, max_runtime, retry_period which are not part of the add_job api CREATE OR REPLACE FUNCTION insert_job( application_name NAME, job_type NAME, schedule_interval INTERVAL, max_runtime INTERVAL, retry_period INTERVAL, owner regrole DEFAULT pg_catalog.quote_ident(current_role)::regrole, scheduled BOOL DEFAULT true, fixed_schedule BOOL DEFAULT true ) RETURNS INT LANGUAGE SQL SECURITY DEFINER AS $$ INSERT INTO _timescaledb_catalog.bgw_job(application_name,schedule_interval,max_runtime,max_retries, retry_period,proc_name,proc_schema,owner,scheduled,fixed_schedule,initial_start) VALUES($1,$3,$4,5,$5,$2,'public',$6,$7,$8,'2000-01-01 00:00:00+00'::timestamptz) RETURNING id; $$; CREATE OR REPLACE FUNCTION test_toggle_scheduled(job_id INTEGER) RETURNS VOID LANGUAGE SQL SECURITY DEFINER AS $$ UPDATE _timescaledb_catalog.bgw_job SET scheduled = NOT scheduled WHERE id = $1; $$; \set WAIT_ON_JOB 0 \set IMMEDIATELY_SET_UNTIL 1 \set WAIT_FOR_OTHER_TO_ADVANCE 2 \set WAIT_FOR_STANDARD_WAITLATCH 3 -- simply sets the wait type CREATE OR REPLACE FUNCTION ts_bgw_params_mock_wait_returns_immediately(new_val INTEGER) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE FUNCTION get_application_pid(app_name TEXT) RETURNS INTEGER LANGUAGE SQL AS $BODY$ SELECT pid FROM pg_stat_activity WHERE application_name = app_name; $BODY$; CREATE FUNCTION wait_application_pid(app_name TEXT, wait_for_start BOOLEAN = true) RETURNS INTEGER LANGUAGE PLPGSQL AS $BODY$ DECLARE r INTEGER; BEGIN --wait up to a second checking each 100ms FOR i in 1..10 LOOP SELECT get_application_pid(app_name) INTO r; IF (wait_for_start AND r IS NULL) OR (NOT wait_for_start AND r IS NOT NULL) THEN PERFORM pg_sleep(0.1); PERFORM pg_stat_clear_snapshot(); ELSE RETURN r; END IF; END LOOP; RETURN NULL; END $BODY$; CREATE FUNCTION wait_for_logentry(job_id INTEGER) RETURNS TEXT LANGUAGE PLPGSQL AS $BODY$ DECLARE app_name TEXT; message TEXT; BEGIN SELECT application_name INTO app_name FROM _timescaledb_catalog.bgw_job WHERE id = job_id; --wait up to a second checking each 100ms FOR i in 1..10 LOOP SELECT msg INTO message FROM bgw_log WHERE application_name = app_name ORDER BY msg_no DESC LIMIT 1; IF FOUND THEN RETURN message; END IF; PERFORM pg_sleep(0.1); PERFORM pg_stat_clear_snapshot(); END LOOP; RETURN NULL; END $BODY$; -- Remove any default jobs, e.g., telemetry DELETE FROM _timescaledb_catalog.bgw_job WHERE TRUE; TRUNCATE _timescaledb_internal.bgw_job_stat; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SET timezone TO PST8PDT; CREATE TABLE public.bgw_log( msg_no INT, mock_time BIGINT, application_name TEXT, msg TEXT ); CREATE VIEW sorted_bgw_log AS SELECT msg_no, application_name, regexp_replace(regexp_replace(msg, '(Wait until|started at|execution time) [0-9]+(\.[0-9]+)?', '\1 (RANDOM)', 'g'), 'background worker "[^"]+"','connection') AS msg FROM bgw_log ORDER BY mock_time, application_name COLLATE "C", msg_no; CREATE TABLE public.bgw_dsm_handle_store( handle BIGINT ); INSERT INTO public.bgw_dsm_handle_store VALUES (0); SELECT ts_bgw_params_create(); ts_bgw_params_create ---------------------- -- -- Test running the scheduler with no jobs -- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(50); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ -- empty SELECT * FROM _timescaledb_internal.bgw_job_stat; job_id | last_start | last_finish | next_start | last_successful_finish | last_run_success | total_runs | total_duration | total_duration_failures | total_successes | total_failures | total_crashes | consecutive_failures | consecutive_crashes | flags --------+------------+-------------+------------+------------------------+------------------+------------+----------------+-------------------------+-----------------+----------------+---------------+----------------------+---------------------+------- -- empty SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) -- -- Test running the scheduler with a job marked as unscheduled -- TRUNCATE bgw_log; -- this function sets the counter (microseconds) that corresponds to the current time to the -- given value (defalut 0, and the default for setting the latch is false) SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- SELECT insert_job('unscheduled', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s',scheduled:= false); insert_job ------------ 1000 SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(50); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ -- empty SELECT * FROM _timescaledb_internal.bgw_job_stat; job_id | last_start | last_finish | next_start | last_successful_finish | last_run_success | total_runs | total_duration | total_duration_failures | total_successes | total_failures | total_crashes | consecutive_failures | consecutive_crashes | flags --------+------------+-------------+------------+------------------------+------------------+------------+----------------+-------------------------+-----------------+----------------+---------------+----------------------+---------------------+------- SELECT * FROM timescaledb_information.job_stats; hypertable_schema | hypertable_name | job_id | last_run_started_at | last_successful_finish | last_run_status | job_status | last_run_duration | next_start | total_runs | total_successes | total_failures -------------------+-----------------+--------+---------------------+------------------------+-----------------+------------+-------------------+------------+------------+-----------------+---------------- | | 1000 | | | | Paused | | | | | SELECT test_toggle_scheduled(1000); test_toggle_scheduled ----------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(50); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM _timescaledb_internal.bgw_job_stat; job_id | last_start | last_finish | next_start | last_successful_finish | last_run_success | total_runs | total_duration | total_duration_failures | total_successes | total_failures | total_crashes | consecutive_failures | consecutive_crashes | flags --------+---------------------------------+---------------------------------+--------------------------------+---------------------------------+------------------+------------+----------------+-------------------------+-----------------+----------------+---------------+----------------------+---------------------+------- 1000 | Fri Dec 31 16:00:00.05 1999 PST | Fri Dec 31 16:00:00.05 1999 PST | Fri Dec 31 16:00:00.1 1999 PST | Fri Dec 31 16:00:00.05 1999 PST | t | 1 | @ 0 | @ 0 | 1 | 0 | 0 | 0 | 0 | 0 SELECT * FROM timescaledb_information.job_stats; hypertable_schema | hypertable_name | job_id | last_run_started_at | last_successful_finish | last_run_status | job_status | last_run_duration | next_start | total_runs | total_successes | total_failures -------------------+-----------------+--------+---------------------------------+---------------------------------+-----------------+------------+-------------------+--------------------------------+------------+-----------------+---------------- | | 1000 | Fri Dec 31 16:00:00.05 1999 PST | Fri Dec 31 16:00:00.05 1999 PST | Success | Scheduled | | Fri Dec 31 16:00:00.1 1999 PST | 1 | 1 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | unscheduled | Execute job 1 SELECT delete_job(1000); delete_job ------------ -- -- test deleting job also terminates running jobs -- SELECT add_job('ts_bgw_test_job_sleep','1h') AS job_id \gset SELECT ts_bgw_db_scheduler_test_run(); ts_bgw_db_scheduler_test_run ------------------------------ SELECT wait_for_logentry(:job_id); wait_for_logentry ------------------- Before sleep SELECT application_name FROM pg_stat_activity WHERE application_name LIKE 'User-Defined Action%'; application_name ---------------------------- User-Defined Action [1001] \x on SELECT job_id, job_status FROM timescaledb_information.job_stats; -[ RECORD 1 ]------- job_id | 1001 job_status | Running -- Showing non-volatile information from pg_stat_activity for -- debugging purposes. Information schema above reads from this view. SELECT datname, usename, application_name, state, query, wait_event_type, wait_event FROM pg_stat_activity WHERE application_name LIKE 'User-Defined Action%'; -[ RECORD 1 ]----+------------------------------------ datname | db_bgw_db_scheduler_fixed usename | default_perm_user application_name | User-Defined Action [1001] state | active query | CALL public.ts_bgw_test_job_sleep() wait_event_type | Timeout wait_event | PgSleep \x off SELECT delete_job(:job_id); delete_job ------------ SELECT wait_application_pid('User-Defined Action [' || :job_id || ']', false); wait_application_pid ---------------------- SELECT application_name FROM pg_stat_activity WHERE application_name LIKE 'User-Defined Action%'; application_name ------------------ -- -- Test running a normal job -- \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; ALTER SEQUENCE _timescaledb_catalog.bgw_job_id_seq RESTART; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- SELECT insert_job('test_job_1', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'); insert_job ------------ 1000 select * from _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ------+------------------+-------------------+-----------------+-------------+--------------+-------------+----------------+------------+-----------+----------------+------------------------------+---------------+--------+--------------+------------+---------- 1000 | test_job_1 | @ 0.1 secs | @ 1 min 40 secs | 5 | @ 1 sec | public | bgw_test_job_1 | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | | | | | \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SET timezone TO PST8PDT; --Tests that the scheduler start a job right away if it's the first time and there is no job_stat entry for it SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, next_start, last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | next_start | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+--------------------------------+------------------------------+------------------+------------+-----------------+----------------+--------------- 1000 | Fri Dec 31 16:00:00.1 1999 PST | Fri Dec 31 16:00:00 1999 PST | t | 1 | 1 | 0 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_1 | Execute job 1 --Test that the scheduler will not run job again if not enough time has passed SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25, 25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1000 | t | 1 | 1 | 0 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_1 | Execute job 1 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) --After enough time has passed the scheduler will run the job again SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(100, 50); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, next_start, last_finish, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | next_start | last_finish | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+--------------------------------+--------------------------------+------------------+------------+-----------------+----------------+--------------- 1000 | Fri Dec 31 16:00:00.2 1999 PST | Fri Dec 31 16:00:00.1 1999 PST | t | 2 | 2 | 0 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_1 | Execute job 1 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_1 | Execute job 1 --Now it runs it one more time SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(120, 100); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, next_start, last_finish, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | next_start | last_finish | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+--------------------------------+--------------------------------+------------------+------------+-----------------+----------------+--------------- 1000 | Fri Dec 31 16:00:00.3 1999 PST | Fri Dec 31 16:00:00.2 1999 PST | t | 3 | 3 | 0 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_1 | Execute job 1 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_1 | Execute job 1 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_1 | Execute job 1 -- -- Test what happens when running a job that throws an error -- \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; TRUNCATE _timescaledb_internal.bgw_job_stat; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- DELETE FROM _timescaledb_catalog.bgw_job; -- schedule_interval, max_runtime, retry_period SELECT insert_job('test_job_2', 'bgw_test_job_2_error', INTERVAL '800ms', INTERVAL '100s', INTERVAL '200ms'); insert_job ------------ 1001 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SET timezone TO PST8PDT; --Run the first time and error SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1001 | f | 1 | 0 | 1 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 SELECT last_finish, last_successful_finish, last_run_success FROM _timescaledb_internal.bgw_job_stat; last_finish | last_successful_finish | last_run_success ------------------------------+------------------------+------------------ Fri Dec 31 16:00:00 1999 PST | -infinity | f -- what we aim to verify here is the following: -- 1. that the job is run again on its next scheduled slot, if the next_start calculated based -- on failure count would surpass it -- the next_start on failure is calculated by adding failure_count * retry_period to finish time -- maximum backoff is 5 * schedule interval, but for a fixed job, if we surpass the next_scheduled_slot -- for it this way, then we execute again at the next scheduled slot instead --Scheduler runs the job again, sees another error, and increases the wait time -- this retry time is before the next scheduled execution, so the job is allowed to retry before then SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(225); -- will see 2 failures now ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1001 | f | 2 | 0 | 2 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 -- as we have a job on a fixed_schedule, the next_start will not be more than the next scheduled slot -- If the calculated next_start is more than the next scheduled execution slot, then -- we will execute again at the next scheduled slot. -- again this is before the next scheduled slot so the job retries before then SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(425); -- will see 3 failures now ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1001 | f | 3 | 0 | 3 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 -- will see 4 failures now SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(625); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1001 | f | 4 | 0 | 4 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 -- will see 5 failures now because job executes again on its next scheduled slot (800ms after its initial start, which is 0) SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(825); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1001 | f | 5 | 0 | 5 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+----------------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 threw an error 2 | test_job_2 | Error job 2 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | test_job_2 | job 1001 reached max_retries after 5 consecutive failures 2 | test_job_2 | job 1001 threw an error 3 | test_job_2 | Error job 2 -- Get status of failing job `test_job_2` to check it reached `max_retries` and -- the new `job_status` now is `Paused` SELECT job_id, last_run_status, job_status, total_runs, total_successes, total_failures FROM timescaledb_information.job_stats WHERE job_id = 1001; job_id | last_run_status | job_status | total_runs | total_successes | total_failures --------+-----------------+------------+------------+-----------------+---------------- 1001 | Failed | Paused | 5 | 0 | 5 -- Alter job to be rescheduled and run it again \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; SELECT scheduled FROM alter_job(1001, scheduled => true) AS discard; scheduled ----------- t \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(825); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat WHERE job_id = 1001; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1001 | f | 6 | 0 | 6 | 0 -- -- Test timeout logic -- \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; TRUNCATE _timescaledb_internal.bgw_job_stat; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- DELETE FROM _timescaledb_catalog.bgw_job; --set timeout lower than job length SELECT insert_job('test_job_3_long', 'bgw_test_job_3_long', INTERVAL '5000ms', INTERVAL '20ms', INTERVAL '50ms'); insert_job ------------ 1002 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT ts_bgw_params_mock_wait_returns_immediately(:IMMEDIATELY_SET_UNTIL); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- --Test that the scheduler kills a job that takes too long SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(200); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes, consecutive_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes | consecutive_crashes --------+------------------+------------+-----------------+----------------+---------------+--------------------- 1002 | f | 1 | 0 | 1 | 0 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 2 | DB Scheduler | terminating connection due to timeout 3 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 4 | DB Scheduler | job 1002 failed --Check that the scheduler does not kill a job with infinite timeout \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; TRUNCATE _timescaledb_internal.bgw_job_stat; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- DELETE FROM _timescaledb_catalog.bgw_job; --set timeout to 0 SELECT insert_job('test_job_3_long', 'bgw_test_job_3_long', INTERVAL '5000ms', INTERVAL '0', INTERVAL '10ms'); insert_job ------------ 1003 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(550); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes, consecutive_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes | consecutive_crashes --------+------------------+------------+-----------------+----------------+---------------+--------------------- 1003 | t | 1 | 1 | 0 | 0 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_3_long | Before sleep job 3 1 | test_job_3_long | After sleep job 3 SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_ON_JOB); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- -- -- Test signal handling -- --Test sending a SIGTERM to a job \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- TRUNCATE _timescaledb_internal.bgw_job_stat; DELETE FROM _timescaledb_catalog.bgw_job; SELECT insert_job('test_job_3_long', 'bgw_test_job_3_long', INTERVAL '5000ms', INTERVAL '100s', INTERVAL '500ms'); insert_job ------------ 1004 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --escalated priv needed for access to pg_stat_activity \c :TEST_DBNAME :ROLE_SUPERUSER SELECT ts_bgw_db_scheduler_test_run(300); ts_bgw_db_scheduler_test_run ------------------------------ SELECT pg_terminate_backend(wait_application_pid('test_job_3_long')); pg_terminate_backend ---------------------- t SELECT ts_bgw_db_scheduler_test_wait_for_scheduler_finish(); ts_bgw_db_scheduler_test_wait_for_scheduler_finish ---------------------------------------------------- SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+----------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_3_long | Before sleep job 3 1 | test_job_3_long | terminating connection due to administrator command 2 | DB Scheduler | job 1004 failed SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1004 | f | 1 | 0 | 1 | 0 -- Test that the job is able to run again and succeed SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(900); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1004 | t | 2 | 1 | 1 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+----------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_3_long | Before sleep job 3 1 | test_job_3_long | terminating connection due to administrator command 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 2 | DB Scheduler | job 1004 failed 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_3_long | Before sleep job 3 1 | test_job_3_long | After sleep job 3 --Test sending a SIGHUP to a job \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- TRUNCATE _timescaledb_internal.bgw_job_stat; DELETE FROM _timescaledb_catalog.bgw_job; SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_FOR_STANDARD_WAITLATCH); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- SELECT ts_bgw_db_scheduler_test_run(-1); ts_bgw_db_scheduler_test_run ------------------------------ SHOW timescaledb.shutdown_bgw_scheduler; timescaledb.shutdown_bgw_scheduler ------------------------------------ off ALTER SYSTEM SET timescaledb.shutdown_bgw_scheduler TO 'on'; SELECT pg_reload_conf(); pg_reload_conf ---------------- t \c :TEST_DBNAME :ROLE_SUPERUSER SHOW timescaledb.shutdown_bgw_scheduler; timescaledb.shutdown_bgw_scheduler ------------------------------------ on SELECT ts_bgw_db_scheduler_test_wait_for_scheduler_finish(); ts_bgw_db_scheduler_test_wait_for_scheduler_finish ---------------------------------------------------- SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+----------------------------------------------- 0 | DB Scheduler | bgw scheduler stopped due to shutdown_bgw guc ALTER SYSTEM RESET timescaledb.shutdown_bgw_scheduler; SELECT pg_reload_conf(); pg_reload_conf ---------------- t \c :TEST_DBNAME :ROLE_SUPERUSER SHOW timescaledb.shutdown_bgw_scheduler; timescaledb.shutdown_bgw_scheduler ------------------------------------ off SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_ON_JOB); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- --Test that sending SIGTERM to scheduler terminates the jobs as well \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; TRUNCATE _timescaledb_internal.bgw_job_stat; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- DELETE FROM _timescaledb_catalog.bgw_job; SELECT insert_job('test_job_3_long', 'bgw_test_job_3_long', INTERVAL '5000ms', INTERVAL '100s', INTERVAL '10ms'); insert_job ------------ 1005 SELECT ts_bgw_db_scheduler_test_run(500); ts_bgw_db_scheduler_test_run ------------------------------ SELECT wait_application_pid('test_job_3_long') IS NOT NULL ; ?column? ---------- t SELECT pg_terminate_backend(wait_application_pid('DB Scheduler Test')); pg_terminate_backend ---------------------- t SELECT ts_bgw_db_scheduler_test_wait_for_scheduler_finish(); ts_bgw_db_scheduler_test_wait_for_scheduler_finish ---------------------------------------------------- SELECT job_id, last_finish, last_run_success, total_runs, total_successes, total_failures, total_crashes, consecutive_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_finish | last_run_success | total_runs | total_successes | total_failures | total_crashes | consecutive_crashes --------+-------------+------------------+------------+-----------------+----------------+---------------+--------------------- 1005 | -infinity | f | 1 | 0 | 0 | 1 | 1 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+----------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 2 | DB Scheduler | terminating connection due to administrator command 0 | test_job_3_long | Before sleep job 3 1 | test_job_3_long | terminating connection due to administrator command --After a SIGTERM to scheduler and jobs, the jobs are considered crashed and there is a imposed wait of 5 min before a job can be run. --See that there is no run again because of the crash-imposed wait (not run with the 10ms retry_period) SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(500); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_finish, next_start, last_run_success, total_runs, total_successes, total_failures, total_crashes, consecutive_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_finish | next_start | last_run_success | total_runs | total_successes | total_failures | total_crashes | consecutive_crashes --------+-------------+------------+------------------+------------+-----------------+----------------+---------------+--------------------- 1005 | -infinity | -infinity | f | 1 | 0 | 0 | 1 | 1 --But after the 5 min period the job is again run SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(400000); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_finish, next_start, last_run_success, total_runs, total_successes, total_failures, total_crashes, consecutive_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_finish | next_start | last_run_success | total_runs | total_successes | total_failures | total_crashes | consecutive_crashes --------+--------------------------------+------------------------------+------------------+------------+-----------------+----------------+---------------+--------------------- 1005 | Fri Dec 31 16:05:00.5 1999 PST | Fri Dec 31 16:05:05 1999 PST | t | 2 | 1 | 0 | 1 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+----------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 2 | DB Scheduler | terminating connection due to administrator command 0 | test_job_3_long | Before sleep job 3 1 | test_job_3_long | terminating connection due to administrator command 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_3_long | Before sleep job 3 1 | test_job_3_long | After sleep job 3 CREATE FUNCTION wait_for_timer_to_run(started_at INTEGER, spins INTEGER=:TEST_SPINWAIT_ITERS) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE num_runs INTEGER; message TEXT; BEGIN select format('[TESTING] Wait until %%, started at %s', started_at) into message; FOR i in 1..spins LOOP SELECT COUNT(*) from bgw_log where msg LIKE message INTO num_runs; if (num_runs > 0) THEN RETURN true; ELSE PERFORM pg_sleep(0.1); END IF; END LOOP; RETURN false; END $BODY$; CREATE FUNCTION wait_for_job_3_to_finish(runs INTEGER, spins INTEGER=:TEST_SPINWAIT_ITERS) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE num_runs INTEGER; BEGIN FOR i in 1..spins LOOP SELECT COUNT(*) from bgw_log where msg='After sleep job 3' INTO num_runs; if (num_runs = runs) THEN RETURN true; ELSE PERFORM pg_sleep(0.1); END IF; END LOOP; RETURN false; END $BODY$; -- -- Test starting more jobs than availlable workers -- \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; TRUNCATE _timescaledb_internal.bgw_job_stat; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- DELETE FROM _timescaledb_catalog.bgw_job; SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_FOR_OTHER_TO_ADVANCE); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- --Our normal limit is 8 jobs (1 already taken up by the launcher, we don't register the test scheduler) --so start 8 workers. Make the schedule_INTERVAL long and the retry period short so that the --retries happen within the scheduler run time but everything only runs once. SELECT insert_job('test_job_3_long_1', 'bgw_test_job_3_long', INTERVAL '5000ms', INTERVAL '100s', INTERVAL '10ms'), insert_job('test_job_3_long_2', 'bgw_test_job_3_long', INTERVAL '5000ms', INTERVAL '100s', INTERVAL '10ms'), insert_job('test_job_3_long_3', 'bgw_test_job_3_long', INTERVAL '5000ms', INTERVAL '100s', INTERVAL '10ms'), insert_job('test_job_3_long_4', 'bgw_test_job_3_long', INTERVAL '5000ms', INTERVAL '100s', INTERVAL '10ms'), insert_job('test_job_3_long_5', 'bgw_test_job_3_long', INTERVAL '5000ms', INTERVAL '100s', INTERVAL '10ms'), insert_job('test_job_3_long_6', 'bgw_test_job_3_long', INTERVAL '5000ms', INTERVAL '100s', INTERVAL '10ms'); insert_job | insert_job | insert_job | insert_job | insert_job | insert_job ------------+------------+------------+------------+------------+------------ 1006 | 1007 | 1008 | 1009 | 1010 | 1011 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT ts_bgw_db_scheduler_test_run(25000); --quit at second 25 ts_bgw_db_scheduler_test_run ------------------------------ --the first 7 jobs will run right away, but not the last one SELECT wait_for_timer_to_run(0); wait_for_timer_to_run ----------------------- t SELECT wait_for_job_3_to_finish(6); wait_for_job_3_to_finish -------------------------- t SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes, consecutive_crashes FROM _timescaledb_internal.bgw_job_stat ORDER BY job_id; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes | consecutive_crashes --------+------------------+------------+-----------------+----------------+---------------+--------------------- 1006 | t | 1 | 1 | 0 | 0 | 0 1007 | t | 1 | 1 | 0 | 0 | 0 1008 | t | 1 | 1 | 0 | 0 | 0 1009 | t | 1 | 1 | 0 | 0 | 0 1010 | t | 1 | 1 | 0 | 0 | 0 1011 | t | 1 | 1 | 0 | 0 | 0 SELECT ts_bgw_params_reset_time(30000000, true); --set to second 30, which causes a quit. ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_wait_for_scheduler_finish(); ts_bgw_db_scheduler_test_wait_for_scheduler_finish ---------------------------------------------------- --should have all 8 runs, all with success runs SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes, consecutive_crashes FROM _timescaledb_internal.bgw_job_stat ORDER BY job_id; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes | consecutive_crashes --------+------------------+------------+-----------------+----------------+---------------+--------------------- 1006 | t | 1 | 1 | 0 | 0 | 0 1007 | t | 1 | 1 | 0 | 0 | 0 1008 | t | 1 | 1 | 0 | 0 | 0 1009 | t | 1 | 1 | 0 | 0 | 0 1010 | t | 1 | 1 | 0 | 0 | 0 1011 | t | 1 | 1 | 0 | 0 | 0 SELECT * FROM sorted_bgw_log WHERE application_name = 'DB Scheduler' ORDER BY application_name, msg_no; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Registered new background worker 3 | DB Scheduler | [TESTING] Registered new background worker 4 | DB Scheduler | [TESTING] Registered new background worker 5 | DB Scheduler | [TESTING] Registered new background worker 6 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) SELECT ts_bgw_params_destroy(); ts_bgw_params_destroy ----------------------- -- -- Test setting next_start time within a job -- \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; TRUNCATE _timescaledb_internal.bgw_job_stat; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_ON_JOB); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- DELETE FROM _timescaledb_catalog.bgw_job; SELECT insert_job('test_job_4', 'bgw_test_job_4', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'); insert_job ------------ 1012 select * from _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ------+------------------+-------------------+-----------------+-------------+--------------+-------------+----------------+------------+-----------+----------------+------------------------------+---------------+--------+--------------+------------+---------- 1012 | test_job_4 | @ 0.1 secs | @ 1 min 40 secs | 5 | @ 1 sec | public | bgw_test_job_4 | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | | | | | \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- Now run and make sure next_start is 200ms away, not 100ms SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1012 | t | 1 | 1 | 0 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_4 | Execute job 4 -- Now just make sure that the job actually runs in 200ms SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(200); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ -- Print next_start and last_finish explicitly, instead of the difference, to make sure the times have changed -- since the last run SELECT job_id, next_start, last_finish, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat; job_id | next_start | last_finish | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+--------------------------------+--------------------------------+------------------+------------+-----------------+----------------+--------------- 1012 | Fri Dec 31 16:00:00.4 1999 PST | Fri Dec 31 16:00:00.2 1999 PST | t | 2 | 2 | 0 | 0 SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Registered new background worker 1 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_4 | Execute job 4 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_4 | Execute job 4 -- Test updating jobs list TRUNCATE bgw_log; \set ON_ERROR_STOP 0 SELECT _timescaledb_functions.stop_background_workers(); ERROR: must be superuser to stop background workers SELECT _timescaledb_functions.restart_background_workers(); ERROR: must be superuser to restart background workers SELECT _timescaledb_functions.start_background_workers(); ERROR: must be superuser to start background workers \set ON_ERROR_STOP 1 \c :TEST_DBNAME :ROLE_SUPERUSER SELECT _timescaledb_functions.stop_background_workers(); stop_background_workers ------------------------- t SET timezone TO PST8PDT; CREATE OR REPLACE FUNCTION ts_test_job_refresh() RETURNS TABLE( id INTEGER, application_name NAME, schedule_interval INTERVAL, max_runtime INTERVAL, max_retries INT, retry_period INTERVAL, next_start TIMESTAMPTZ, timeout_at TIMESTAMPTZ, reserved_worker BOOLEAN, may_next_mark_end BOOLEAN ) AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE FUNCTION verify_refresh_correct() RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE num_jobs INTEGER; num_jobs_in_list INTEGER; BEGIN SELECT COUNT(*) from _timescaledb_catalog.bgw_job INTO num_jobs; select COUNT(*) from ts_test_job_refresh() JOIN _timescaledb_catalog.bgw_job USING (id,application_name,schedule_interval,max_runtime,max_retries,retry_period) INTO num_jobs_in_list; IF (num_jobs = num_jobs_in_list) THEN RETURN true; END IF; RETURN false; END $BODY$; CREATE FUNCTION wait_for_job_1_to_run(runs INTEGER, spins INTEGER=:TEST_SPINWAIT_ITERS) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE num_runs INTEGER; BEGIN FOR i in 1..spins LOOP SELECT COUNT(*) from bgw_log where msg='Execute job 1' INTO num_runs; if (num_runs = runs) THEN RETURN true; ELSE PERFORM pg_sleep(0.1); END IF; END LOOP; RETURN false; END $BODY$; select * from verify_refresh_correct(); verify_refresh_correct ------------------------ t -- Should return the same table select * from verify_refresh_correct(); verify_refresh_correct ------------------------ t DELETE FROM _timescaledb_catalog.bgw_job; -- Make sure jobs list is empty select count(*) from ts_test_job_refresh(); count ------- 0 SELECT insert_job('test_1', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'), insert_job('test_2', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'), insert_job('test_3', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'); insert_job | insert_job | insert_job ------------+------------+------------ 1013 | 1014 | 1015 select * from verify_refresh_correct(); verify_refresh_correct ------------------------ t DELETE from _timescaledb_catalog.bgw_job where application_name='test_2'; SELECT insert_job('test_4', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'); insert_job ------------ 1016 select * from verify_refresh_correct(); verify_refresh_correct ------------------------ t DELETE FROM _timescaledb_catalog.bgw_job; SELECT insert_job('test_10', 'test_10', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'); insert_job ------------ 1017 select * from verify_refresh_correct(); verify_refresh_correct ------------------------ t -- Should be idempotent select * from verify_refresh_correct(); verify_refresh_correct ------------------------ t DELETE FROM _timescaledb_catalog.bgw_job; SELECT insert_job('another', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'), insert_job('another1', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'), insert_job('another2', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'), insert_job('another3', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'), insert_job('another4', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'); insert_job | insert_job | insert_job | insert_job | insert_job ------------+------------+------------+------------+------------ 1018 | 1019 | 1020 | 1021 | 1022 select * from verify_refresh_correct(); verify_refresh_correct ------------------------ t DELETE FROM _timescaledb_catalog.bgw_job where application_name='another' OR application_name='another3'; SELECT insert_job('blah', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s'); insert_job ------------ 1023 select * from verify_refresh_correct(); verify_refresh_correct ------------------------ t -- Now test a real scheduler-mock running in a loop and updating the list of jobs TRUNCATE _timescaledb_internal.bgw_job_stat; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- DELETE FROM _timescaledb_catalog.bgw_job; SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_FOR_OTHER_TO_ADVANCE); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- SELECT ts_bgw_db_scheduler_test_run(500); ts_bgw_db_scheduler_test_run ------------------------------ -- Wait for scheduler to start up SELECT wait_for_timer_to_run(0); wait_for_timer_to_run ----------------------- t SELECT insert_job('another', 'bgw_test_job_1', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s') AS job_id \gset -- call alter_job to trigger cache invalidation SELECT alter_job(:job_id,scheduled:=true); alter_job ----------------------------------------------------------------------------------------------------------- (1024,"@ 0.1 secs","@ 1 min 40 secs",5,"@ 1 sec",t,,-infinity,,t,"Fri Dec 31 16:00:00 1999 PST",,another) SELECT ts_bgw_params_reset_time(50000, true); ts_bgw_params_reset_time -------------------------- SELECT wait_for_timer_to_run(50000); wait_for_timer_to_run ----------------------- t SELECT wait_for_job_1_to_run(1); wait_for_job_1_to_run ----------------------- t SELECT ts_bgw_params_reset_time(150000, true); ts_bgw_params_reset_time -------------------------- SELECT wait_for_timer_to_run(150000); wait_for_timer_to_run ----------------------- t SELECT wait_for_job_1_to_run(2); wait_for_job_1_to_run ----------------------- t select * from _timescaledb_internal.bgw_job_stat; job_id | last_start | last_finish | next_start | last_successful_finish | last_run_success | total_runs | total_duration | total_duration_failures | total_successes | total_failures | total_crashes | consecutive_failures | consecutive_crashes | flags --------+---------------------------------+---------------------------------+--------------------------------+---------------------------------+------------------+------------+----------------+-------------------------+-----------------+----------------+---------------+----------------------+---------------------+------- 1024 | Fri Dec 31 16:00:00.15 1999 PST | Fri Dec 31 16:00:00.15 1999 PST | Fri Dec 31 16:00:00.2 1999 PST | Fri Dec 31 16:00:00.15 1999 PST | t | 2 | @ 0 | @ 0 | 2 | 0 | 0 | 0 | 0 | 0 SELECT delete_job(x.id) FROM (select * from _timescaledb_catalog.bgw_job) x; delete_job ------------ -- test null handling in delete_job SELECT delete_job(NULL); delete_job ------------ SELECT ts_bgw_params_reset_time(200000, true); ts_bgw_params_reset_time -------------------------- SELECT wait_for_timer_to_run(200000); wait_for_timer_to_run ----------------------- t -- In the next time interval, nothing should be run because scheduler should have an empty list SELECT ts_bgw_params_reset_time(300000, true); ts_bgw_params_reset_time -------------------------- SELECT wait_for_timer_to_run(300000); wait_for_timer_to_run ----------------------- t -- Same for this time interval SELECT ts_bgw_params_reset_time(400000, true); ts_bgw_params_reset_time -------------------------- SELECT wait_for_timer_to_run(400000); wait_for_timer_to_run ----------------------- t -- Now add a new job and make sure it gets run before the scheduler dies SELECT insert_job('new_job', 'bgw_test_job_1', INTERVAL '10ms', INTERVAL '100s', INTERVAL '1s') AS job_id \gset -- call alter_job to trigger cache invalidation SELECT alter_job(:job_id,scheduled:=true); alter_job ------------------------------------------------------------------------------------------------------------ (1025,"@ 0.01 secs","@ 1 min 40 secs",5,"@ 1 sec",t,,-infinity,,t,"Fri Dec 31 16:00:00 1999 PST",,new_job) SELECT ts_bgw_params_reset_time(450000, true); ts_bgw_params_reset_time -------------------------- -- New job should be run once, for a total of 3 runs of this job in the log SELECT wait_for_job_1_to_run(3); wait_for_job_1_to_run ----------------------- t -- New job should be run again SELECT ts_bgw_params_reset_time(480000, true); ts_bgw_params_reset_time -------------------------- SELECT wait_for_job_1_to_run(4); wait_for_job_1_to_run ----------------------- t SELECT ts_bgw_params_reset_time(500000, true); ts_bgw_params_reset_time -------------------------- SELECT * FROM sorted_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 1 | DB Scheduler | [TESTING] Registered new background worker 2 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | another | Execute job 1 3 | DB Scheduler | [TESTING] Registered new background worker 4 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | another | Execute job 1 5 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 6 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 7 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 8 | DB Scheduler | [TESTING] Registered new background worker 9 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | new_job | Execute job 1 10 | DB Scheduler | [TESTING] Registered new background worker 11 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | new_job | Execute job 1 SELECT * FROM _timescaledb_internal.bgw_job_stat; job_id | last_start | last_finish | next_start | last_successful_finish | last_run_success | total_runs | total_duration | total_duration_failures | total_successes | total_failures | total_crashes | consecutive_failures | consecutive_crashes | flags --------+---------------------------------+---------------------------------+---------------------------------+---------------------------------+------------------+------------+----------------+-------------------------+-----------------+----------------+---------------+----------------------+---------------------+------- 1025 | Fri Dec 31 16:00:00.48 1999 PST | Fri Dec 31 16:00:00.48 1999 PST | Fri Dec 31 16:00:00.49 1999 PST | Fri Dec 31 16:00:00.48 1999 PST | t | 2 | @ 0 | @ 0 | 2 | 0 | 0 | 0 | 0 | 0 -- clean up jobs SELECT _timescaledb_functions.stop_background_workers(); stop_background_workers ------------------------- t select delete_job(:job_id); delete_job ------------ -- test the new API with all its parameters: with timezone, without timezone TRUNCATE bgw_log; TRUNCATE bgw_dsm_handle_store; INSERT INTO public.bgw_dsm_handle_store VALUES (0); SELECT ts_bgw_params_create(); ts_bgw_params_create ---------------------- CREATE TABLE test_table_scheduler ( time timestamptz not null, a int, b int ); select '2000-01-01 00:00:00+00' as init \gset select create_hypertable('test_table_scheduler', 'time', chunk_time_interval => interval '1 month'); create_hypertable ----------------------------------- (1,public,test_table_scheduler,t) INSERT INTO test_table_scheduler values (now() - interval '10 years', 1, 1), (now() - interval '8 years', 1, 1), (now() - interval '6 years', 1, 1), (now() - interval '4 years', 1, 1), (now() - interval '2 years', 1, 1), (now() - interval '1 years', 2, 2), (now() - interval '6 months', 3, 3), (now() - interval '3 months', 4, 4); CREATE MATERIALIZED VIEW cagg_scheduler(time, avg_a) WITH (timescaledb.continuous) AS SELECT time_bucket('1 month', time), avg(a) FROM test_table_scheduler GROUP BY time_bucket('1 month', time) WITH NO DATA; SELECT set_chunk_time_interval('_timescaledb_internal._materialized_hypertable_2', interval '36500 days'); set_chunk_time_interval ------------------------- select show_chunks('test_table_scheduler'); show_chunks ---------------------------------------- _timescaledb_internal._hyper_1_1_chunk _timescaledb_internal._hyper_1_2_chunk _timescaledb_internal._hyper_1_3_chunk _timescaledb_internal._hyper_1_4_chunk _timescaledb_internal._hyper_1_5_chunk _timescaledb_internal._hyper_1_6_chunk _timescaledb_internal._hyper_1_7_chunk _timescaledb_internal._hyper_1_8_chunk alter table test_table_scheduler set (timescaledb.compress, timescaledb.compress_orderby = 'time DESC'); select add_retention_policy('test_table_scheduler', interval '2 year', initial_start => :'init'::timestamptz, timezone => 'Europe/Berlin'); add_retention_policy ---------------------- 1026 select add_compression_policy('test_table_scheduler', interval '1 year', initial_start => :'init'::timestamptz, timezone => 'Europe/Berlin'); add_compression_policy ------------------------ 1027 select add_continuous_aggregate_policy('cagg_scheduler', interval '1 year', interval '2 months', interval '3 weeks', initial_start => :'init'::timestamptz + interval '5 ms', timezone => 'Europe/Athens'); add_continuous_aggregate_policy --------------------------------- 1028 select * from _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ------+--------------------------------------------+-------------------+-------------+-------------+--------------+------------------------+-------------------------------------+------------+-----------+----------------+----------------------------------+---------------+--------------------------------------------------------------------------------+------------------------+-------------------------------------------+--------------- 1026 | Retention Policy [1026] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_functions | policy_retention | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"drop_after": "@ 2 years", "hypertable_id": 1} | _timescaledb_functions | policy_retention_check | Europe/Berlin 1027 | Columnstore Policy [1027] | @ 12 hours | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"hypertable_id": 1, "compress_after": "@ 1 year"} | _timescaledb_functions | policy_compression_check | Europe/Berlin 1028 | Refresh Continuous Aggregate Policy [1028] | @ 21 days | @ 0 | -1 | @ 21 days | _timescaledb_functions | policy_refresh_continuous_aggregate | super_user | t | t | Fri Dec 31 16:00:00.005 1999 PST | 2 | {"end_offset": "@ 2 mons", "start_offset": "@ 1 year", "mat_hypertable_id": 2} | _timescaledb_functions | policy_refresh_continuous_aggregate_check | Europe/Athens -- now wait for scheduler to run the policies SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * from _timescaledb_internal.bgw_job_stat; job_id | last_start | last_finish | next_start | last_successful_finish | last_run_success | total_runs | total_duration | total_duration_failures | total_successes | total_failures | total_crashes | consecutive_failures | consecutive_crashes | flags --------+----------------------------------+----------------------------------+----------------------------------+----------------------------------+------------------+------------+----------------+-------------------------+-----------------+----------------+---------------+----------------------+---------------------+------- 1026 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 16:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST | t | 1 | @ 0 | @ 0 | 1 | 0 | 0 | 0 | 0 | 0 1027 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 04:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST | t | 1 | @ 0 | @ 0 | 1 | 0 | 0 | 0 | 0 | 0 1028 | Fri Dec 31 16:00:00.005 1999 PST | Fri Dec 31 16:00:00.005 1999 PST | Fri Jan 21 16:00:00.005 2000 PST | Fri Dec 31 16:00:00.005 1999 PST | t | 1 | @ 0 | @ 0 | 1 | 0 | 0 | 0 | 0 | 0 SELECT show_chunks('test_table_scheduler'); show_chunks ---------------------------------------- _timescaledb_internal._hyper_1_5_chunk _timescaledb_internal._hyper_1_6_chunk _timescaledb_internal._hyper_1_7_chunk _timescaledb_internal._hyper_1_8_chunk select hypertable_schema, hypertable_name, chunk_schema, chunk_name, is_compressed from timescaledb_information.chunks ; hypertable_schema | hypertable_name | chunk_schema | chunk_name | is_compressed -----------------------+----------------------------+-----------------------+-------------------+--------------- public | test_table_scheduler | _timescaledb_internal | _hyper_1_5_chunk | t public | test_table_scheduler | _timescaledb_internal | _hyper_1_6_chunk | f public | test_table_scheduler | _timescaledb_internal | _hyper_1_7_chunk | f public | test_table_scheduler | _timescaledb_internal | _hyper_1_8_chunk | f _timescaledb_internal | _materialized_hypertable_2 | _timescaledb_internal | _hyper_2_10_chunk | f select avg_a from cagg_scheduler ORDER BY 1; avg_a -------------------- 3.0000000000000000 4.0000000000000000 -- test the API for add_job too create or replace procedure job_test_fixed(jobid int, config jsonb) language plpgsql as $$ begin raise NOTICE 'this is job_test_fixed'; end $$; \set ON_ERROR_STOP 0 select add_job('job_test_fixed', interval '7 months', initial_start => :'init'::timestamptz + interval '10 ms'); add_job --------- 1029 select add_job('job_test_fixed', interval '7 months', initial_start => :'init'::timestamptz + interval '10 ms', timezone => 'Europe/Athens'); add_job --------- 1030 -- this will fail because the timezone has a bad value select add_job('job_test_fixed', interval '8 weeks', timezone => 'EuRoPe/AmEriCa'); ERROR: time zone "EuRoPe/AmEriCa" not recognized select add_reorder_policy('test_table_scheduler','test_table_scheduler_time_idx', initial_start => :'init'::timestamptz + interval '15 ms', timezone => 'Europe/Berlin'); add_reorder_policy -------------------- 1031 SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ select * from _timescaledb_catalog.bgw_job order by id; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ------+--------------------------------------------+-------------------+-------------+-------------+--------------+------------------------+-------------------------------------+------------+-----------+----------------+----------------------------------+---------------+--------------------------------------------------------------------------------+------------------------+-------------------------------------------+--------------- 1026 | Retention Policy [1026] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_functions | policy_retention | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"drop_after": "@ 2 years", "hypertable_id": 1} | _timescaledb_functions | policy_retention_check | Europe/Berlin 1027 | Columnstore Policy [1027] | @ 12 hours | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | super_user | t | t | Fri Dec 31 16:00:00 1999 PST | 1 | {"hypertable_id": 1, "compress_after": "@ 1 year"} | _timescaledb_functions | policy_compression_check | Europe/Berlin 1028 | Refresh Continuous Aggregate Policy [1028] | @ 21 days | @ 0 | -1 | @ 21 days | _timescaledb_functions | policy_refresh_continuous_aggregate | super_user | t | t | Fri Dec 31 16:00:00.005 1999 PST | 2 | {"end_offset": "@ 2 mons", "start_offset": "@ 1 year", "mat_hypertable_id": 2} | _timescaledb_functions | policy_refresh_continuous_aggregate_check | Europe/Athens 1029 | User-Defined Action [1029] | @ 7 mons | @ 0 | -1 | @ 5 mins | public | job_test_fixed | super_user | t | t | Fri Dec 31 16:00:00.01 1999 PST | | | | | 1030 | User-Defined Action [1030] | @ 7 mons | @ 0 | -1 | @ 5 mins | public | job_test_fixed | super_user | t | t | Fri Dec 31 16:00:00.01 1999 PST | | | | | Europe/Athens 1031 | Reorder Policy [1031] | @ 360 hours | @ 0 | -1 | @ 5 mins | _timescaledb_functions | policy_reorder | super_user | t | t | Fri Dec 31 16:00:00.015 1999 PST | 1 | {"index_name": "test_table_scheduler_time_idx", "hypertable_id": 1} | _timescaledb_functions | policy_reorder_check | Europe/Berlin SELECT job_id, date_trunc('second',last_start) AS last_start, date_trunc('second',last_finish) AS last_finish, date_trunc('second',next_start) AS next_start, date_trunc('second',last_successful_finish) as last_successful_finish FROM _timescaledb_internal.bgw_job_stat ORDER BY job_id; job_id | last_start | last_finish | next_start | last_successful_finish --------+------------------------------+------------------------------+------------------------------+------------------------------ 1026 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 16:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST 1027 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 01 04:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST 1028 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Fri Jan 21 16:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST 1029 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Mon Jul 31 16:00:00 2000 PDT | Fri Dec 31 16:00:00 1999 PST 1030 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Mon Jul 31 16:00:00 2000 PDT | Fri Dec 31 16:00:00 1999 PST 1031 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | Sat Jan 15 16:00:00 2000 PST | Fri Dec 31 16:00:00 1999 PST -- test ability to switch from one type of schedule to another CREATE OR REPLACE PROCEDURE job_test(jobid int, config jsonb) language plpgsql as $$ BEGIN PERFORM pg_sleep(0.5); END $$; SELECT add_job('job_test', '8 min', fixed_schedule => false) AS jobid_drifting_1 \gset SELECT add_job('job_test', '8 min', fixed_schedule => false) AS jobid_drifting_2 \gset SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT last_finish AS finish_time_drifting_1 FROM _timescaledb_internal.bgw_job_stat WHERE job_id = :jobid_drifting_1 \gset -- job is on fixed schedule, so changing the timezone and initial start, has no effect on its next start, -- which should be 8 min after the finish time SELECT next_start AS next_start_drifting_1 FROM alter_job(:jobid_drifting_1, schedule_interval => interval '10 min', timezone => 'Europe/Athens') \gset SELECT :'next_start_drifting_1'::timestamptz - :'finish_time_drifting_1'::timestamptz as diff_interval; diff_interval --------------- @ 10 mins -- this will print a notice about using the current time as initial start -- suppress the notice though as it will lead to flaky tests set client_min_messages = 'warning'; SELECT next_start, initial_start FROM alter_job(:jobid_drifting_1, schedule_interval => interval '10 min', fixed_schedule => true, initial_start => '-infinity') \gset -- should be 10 min SELECT :'next_start'::timestamptz - :'initial_start'::timestamptz; ?column? ----------- @ 10 mins -- if job is not on fixed schedule, and we change it to fixed schedule, then user should also provide initial_start. -- if they don't, a notice is printed that we're using current time as initial start SELECT next_start, initial_start FROM alter_job(:jobid_drifting_2, schedule_interval => interval '10 min', fixed_schedule => true) \gset SELECT :'next_start'::timestamptz - :'initial_start'::timestamptz; ?column? ----------- @ 10 mins reset client_min_messages; -- jobs starting with fixed schedules SELECT add_job('job_test', '1 month', initial_start => '2000-01-01 00:03') as jobid_fixed_1 \gset -- wait for the job to run, then check its next_start: (3 minutes = 180mil microseconds) SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(180000005); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT last_finish, next_start from _timescaledb_internal.bgw_job_stat where job_id = :jobid_fixed_1; last_finish | next_start ------------------------------+------------------------------ Sat Jan 01 00:03:00 2000 PST | Tue Feb 01 00:03:00 2000 PST SELECT date_part('hour',next_start)::integer, date_part('minute',next_start)::integer, date_part('second',next_start)::integer FROM alter_job(:jobid_fixed_1, initial_start => '2020-01-01 04:00'); date_part | date_part | date_part -----------+-----------+----------- 4 | 0 | 0 SELECT date_part('hour',next_start)::integer, date_part('minute',next_start)::integer, date_part('second',next_start)::integer FROM _timescaledb_internal.bgw_job_stat WHERE job_id = :jobid_fixed_1; date_part | date_part | date_part -----------+-----------+----------- 4 | 0 | 0 -- go from fixed_schedule to drifting schedule SELECT ts_bgw_params_destroy(); ts_bgw_params_destroy ----------------------- SELECT ts_bgw_params_create(); ts_bgw_params_create ---------------------- SELECT add_job('job_test', '30 sec', initial_start => '2000-01-01 00:00:23') as jobid_fixed_2 \gset -- wait for the job to run, check the next_start, once it's finished, switch to drifting schedule and -- check the next_start again -- wait for 30 seconds to pass SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(30000025); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ select * from _timescaledb_internal.bgw_job_stat WHERE job_id = :jobid_fixed_2; job_id | last_start | last_finish | next_start | last_successful_finish | last_run_success | total_runs | total_duration | total_duration_failures | total_successes | total_failures | total_crashes | consecutive_failures | consecutive_crashes | flags --------+------------------------------+------------------------------+------------------------------+------------------------------+------------------+------------+----------------+-------------------------+-----------------+----------------+---------------+----------------------+---------------------+------- 1035 | Sat Jan 01 00:00:23 2000 PST | Sat Jan 01 00:00:23 2000 PST | Sat Jan 01 00:00:53 2000 PST | Sat Jan 01 00:00:23 2000 PST | t | 1 | @ 0 | @ 0 | 1 | 0 | 0 | 0 | 0 | 0 UPDATE _timescaledb_internal.bgw_job_stat SET last_finish = last_finish + interval '10 sec', last_successful_finish = last_successful_finish + interval '10 sec' WHERE job_id = :jobid_fixed_2; -- next_start is unchanged SELECT * FROM _timescaledb_internal.bgw_job_stat WHERE job_id = :jobid_fixed_2; job_id | last_start | last_finish | next_start | last_successful_finish | last_run_success | total_runs | total_duration | total_duration_failures | total_successes | total_failures | total_crashes | consecutive_failures | consecutive_crashes | flags --------+------------------------------+------------------------------+------------------------------+------------------------------+------------------+------------+----------------+-------------------------+-----------------+----------------+---------------+----------------------+---------------------+------- 1035 | Sat Jan 01 00:00:23 2000 PST | Sat Jan 01 00:00:33 2000 PST | Sat Jan 01 00:00:53 2000 PST | Sat Jan 01 00:00:33 2000 PST | t | 1 | @ 0 | @ 0 | 1 | 0 | 0 | 0 | 0 | 0 -- next start is now updated SELECT alter_job(:jobid_fixed_2, fixed_schedule => false); alter_job ------------------------------------------------------------------------------------------------------------------------------------------ (1035,"@ 30 secs","@ 0",-1,"@ 5 mins",t,,"Sat Jan 01 00:01:03 2000 PST",,f,"Sat Jan 01 00:00:23 2000 PST",,"User-Defined Action [1035]") ================================================ FILE: tsl/test/expected/bgw_job_ddl.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- Test for DDL-like functionality CREATE VIEW my_jobs AS SELECT proc_schema, proc_name, owner FROM _timescaledb_catalog.bgw_job WHERE id >= 1000 ORDER BY proc_schema, proc_name, owner; GRANT SELECT ON my_jobs TO PUBLIC; \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION insert_job( application_name NAME, job_type NAME, schedule_interval INTERVAL, max_runtime INTERVAL, retry_period INTERVAL, owner regrole DEFAULT CURRENT_ROLE::regrole, scheduled BOOL DEFAULT true, fixed_schedule BOOL DEFAULT false ) RETURNS INT LANGUAGE SQL SECURITY DEFINER AS $$ INSERT INTO _timescaledb_catalog.bgw_job(application_name,schedule_interval,max_runtime,max_retries, retry_period,proc_name,proc_schema,owner,scheduled,fixed_schedule) VALUES($1,$3,$4,5,$5,$2,'public',$6,$7,$8) RETURNING id; $$; CREATE PROCEDURE more_magic(job_id INT, config jsonb) LANGUAGE plpgsql AS $$ BEGIN RAISE NOTICE 'done'; END; $$; CREATE PROCEDURE some_magic(job_id INT, config jsonb) LANGUAGE plpgsql AS $$ BEGIN RAISE NOTICE 'done'; END; $$; CREATE USER another_user; SET ROLE another_user; SELECT insert_job('one_job', 'some_magic', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s') AS job_id_2 \gset SELECT insert_job('another_one', 'more_magic', INTERVAL '100ms', INTERVAL '100s', INTERVAL '1s') AS job_id \gset SELECT * FROM my_jobs; proc_schema | proc_name | owner -------------+------------+-------------- public | more_magic | another_user public | some_magic | another_user -- Test that reassigning to another user privileges does not work for -- a normal user. We test both users with superuser privileges and -- default permissions. \set ON_ERROR_STOP 0 REASSIGN OWNED BY another_user TO :ROLE_SUPERUSER; ERROR: permission denied to reassign objects REASSIGN OWNED BY another_user TO :ROLE_DEFAULT_PERM_USER; ERROR: permission denied to reassign objects \set ON_ERROR_STOP 1 RESET ROLE; -- Test that renaming a user changes keeps the job assigned to that user. ALTER USER another_user RENAME TO renamed_user; SELECT * FROM my_jobs; proc_schema | proc_name | owner -------------+------------+-------------- public | more_magic | renamed_user public | some_magic | renamed_user -- Test that renaming the procedure also modifies the entry in the -- jobs table. ALTER PROCEDURE more_magic RENAME TO magic; SELECT * FROM my_jobs; proc_schema | proc_name | owner -------------+------------+-------------- public | magic | renamed_user public | some_magic | renamed_user -- Test that modifying the schema also modifies the entry in the jobs -- table. CREATE SCHEMA frugal; ALTER PROCEDURE magic SET SCHEMA frugal; ALTER PROCEDURE some_magic SET SCHEMA frugal; SELECT * FROM my_jobs; proc_schema | proc_name | owner -------------+------------+-------------- frugal | magic | renamed_user frugal | some_magic | renamed_user -- Test that renaming the schema will rename the procedure schema START TRANSACTION; ALTER SCHEMA frugal RENAME TO wicked; SELECT * FROM my_jobs; proc_schema | proc_name | owner -------------+------------+-------------- wicked | magic | renamed_user wicked | some_magic | renamed_user ROLLBACK; \set VERBOSITY default \set ON_ERROR_STOP 0 SELECT * FROM my_jobs; proc_schema | proc_name | owner -------------+------------+-------------- frugal | magic | renamed_user frugal | some_magic | renamed_user -- Test that dropping a user owning a job fails. DROP USER renamed_user; ERROR: role "renamed_user" cannot be dropped because some objects depend on it DETAIL: owner of job 1001 -- Test that dropping the procedure fails since there is a background -- job using it. DROP PROCEDURE frugal.magic; ERROR: cannot drop frugal.magic because background job 1001 depends on it HINT: Use delete_job() to drop the job first. -- Test that re-assigning objects owned by an unknown user still fails REASSIGN OWNED BY renamed_user, unknown_user TO :ROLE_DEFAULT_PERM_USER; ERROR: role "unknown_user" does not exist -- Test that dropping the schema without CASCADE will error out DROP SCHEMA frugal; ERROR: cannot drop schema frugal because other objects depend on it DETAIL: function frugal.magic(integer,jsonb) depends on schema frugal function frugal.some_magic(integer,jsonb) depends on schema frugal HINT: Use DROP ... CASCADE to drop the dependent objects too. \set ON_ERROR_STOP 1 -- Test that reassigning the owned job actually changes the owner of -- the job. START TRANSACTION; REASSIGN OWNED BY renamed_user TO :ROLE_DEFAULT_PERM_USER; SELECT * FROM my_jobs; proc_schema | proc_name | owner -------------+------------+------------------- frugal | magic | default_perm_user frugal | some_magic | default_perm_user ROLLBACK; -- Test that reassigning to postgres works REASSIGN OWNED BY renamed_user TO :ROLE_SUPERUSER; SELECT * FROM my_jobs; proc_schema | proc_name | owner -------------+------------+------------ frugal | magic | super_user frugal | some_magic | super_user -- Dropping the user now should work. DROP USER renamed_user; -- Dropping using Cascade should work and remove the background worker -- entry as well. START TRANSACTION; SELECT * FROM my_jobs; proc_schema | proc_name | owner -------------+------------+------------ frugal | magic | super_user frugal | some_magic | super_user DROP PROCEDURE frugal.magic CASCADE; NOTICE: drop cascades to job 1001 SELECT * FROM my_jobs; proc_schema | proc_name | owner -------------+------------+------------ frugal | some_magic | super_user ROLLBACK; DELETE FROM _timescaledb_catalog.bgw_job WHERE id = :job_id; -- We should be able to drop the procedure without CASCADE now since -- it is not used by any job. DROP PROCEDURE frugal.magic; -- We should be able to drop the schema with CASCADE despite -- containing a procedure used by a background worker, but this should -- remove the job from the background worker table. SELECT * FROM my_jobs; proc_schema | proc_name | owner -------------+------------+------------ frugal | some_magic | super_user DROP SCHEMA frugal CASCADE; NOTICE: drop cascades to job 1000 NOTICE: drop cascades to function frugal.some_magic(integer,jsonb) SELECT * FROM my_jobs; proc_schema | proc_name | owner -------------+-----------+------- ================================================ FILE: tsl/test/expected/bgw_job_stat_history.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER ALTER DATABASE :TEST_DBNAME SET timezone TO 'UTC'; \c CREATE PROCEDURE custom_job_ok(job_id int, config jsonb) LANGUAGE PLPGSQL AS $$ BEGIN RAISE INFO 'custom_job'; END $$; CREATE PROCEDURE custom_job_error(job_id int, config jsonb) LANGUAGE PLPGSQL AS $$ BEGIN PERFORM 1/0; END $$; CREATE VIEW job_history_summary AS SELECT job_id, succeeded, count(*) AS record_count FROM _timescaledb_internal.bgw_job_stat_history GROUP BY job_id, succeeded; CREATE VIEW recent_job_history_summary AS SELECT job_id, succeeded, count(*) AS record_count FROM _timescaledb_internal.bgw_job_stat_history WHERE execution_finish > now() - interval '30 days' GROUP BY job_id, succeeded; -- Do not log all jobs, only FAILED executions SHOW timescaledb.enable_job_execution_logging; timescaledb.enable_job_execution_logging ------------------------------------------ off SELECT add_job('custom_job_ok', schedule_interval => interval '1 hour', initial_start := now()) AS job_id_1 \gset SELECT add_job('custom_job_error', schedule_interval => interval '1 hour', initial_start := now()) AS job_id_2 \gset -- Start Background Workers SELECT _timescaledb_functions.start_background_workers(); start_background_workers -------------------------- t SELECT test.wait_for_job_to_run(:job_id_1, 1); wait_for_job_to_run --------------------- t SELECT test.wait_for_job_to_run(:job_id_2, 1); INFO: wait_for_job_to_run: job execution failed wait_for_job_to_run --------------------- f -- only 1 failure SELECT count(*), succeeded FROM timescaledb_information.job_history WHERE job_id >= 1000 GROUP BY 2 ORDER BY 2; count | succeeded -------+----------- 1 | f SELECT proc_schema, proc_name, sqlerrcode, err_message FROM timescaledb_information.job_history WHERE job_id >= 1000 AND succeeded IS FALSE; proc_schema | proc_name | sqlerrcode | err_message -------------+------------------+------------+------------------ public | custom_job_error | 22012 | division by zero -- Check current jobs status SELECT job_id, job_status, total_runs, total_successes, total_failures FROM timescaledb_information.job_stats WHERE job_id >= 1000 ORDER BY job_id; job_id | job_status | total_runs | total_successes | total_failures --------+------------+------------+-----------------+---------------- 1000 | Scheduled | 1 | 1 | 0 1001 | Scheduled | 1 | 0 | 1 -- Log all executions ALTER SYSTEM SET timescaledb.enable_job_execution_logging TO ON; SELECT pg_reload_conf(); pg_reload_conf ---------------- t -- Reconnect to make sure the GUC is set \c :TEST_DBNAME :ROLE_SUPERUSER SELECT scheduled FROM alter_job(:job_id_1, next_start => now()); scheduled ----------- t SELECT scheduled FROM alter_job(:job_id_2, next_start => now()); scheduled ----------- t SELECT _timescaledb_functions.restart_background_workers(); restart_background_workers ---------------------------- t SELECT test.wait_for_job_to_run(:job_id_1, 2); wait_for_job_to_run --------------------- t SELECT test.wait_for_job_to_run(:job_id_2, 2); INFO: wait_for_job_to_run: job execution failed wait_for_job_to_run --------------------- f -- 1 succeeded 2 failures SELECT count(*), succeeded FROM timescaledb_information.job_history WHERE job_id >= 1000 GROUP BY 2 ORDER BY 2; count | succeeded -------+----------- 2 | f 1 | t -- Check current jobs status SELECT job_id, job_status, total_runs, total_successes, total_failures FROM timescaledb_information.job_stats WHERE job_id >= 1000 ORDER BY job_id; job_id | job_status | total_runs | total_successes | total_failures --------+------------+------------+-----------------+---------------- 1000 | Scheduled | 2 | 2 | 0 1001 | Scheduled | 2 | 0 | 2 -- Check config changes over time SELECT scheduled FROM alter_job(:job_id_1, config => '{"foo": 1}'::jsonb); scheduled ----------- t SELECT scheduled FROM alter_job(:job_id_2, config => '{"bar": 1}'::jsonb); scheduled ----------- t SELECT scheduled FROM alter_job(:job_id_1, next_start => now()); scheduled ----------- t SELECT scheduled FROM alter_job(:job_id_2, next_start => now()); scheduled ----------- t SELECT _timescaledb_functions.restart_background_workers(); restart_background_workers ---------------------------- t SELECT test.wait_for_job_to_run(:job_id_1, 3); wait_for_job_to_run --------------------- t SELECT test.wait_for_job_to_run(:job_id_2, 3); INFO: wait_for_job_to_run: job execution failed wait_for_job_to_run --------------------- f -- Check job execution history SELECT job_id, pid IS NOT NULL AS pid, proc_schema, proc_name, succeeded, config, sqlerrcode, err_message FROM timescaledb_information.job_history WHERE job_id >= 1000 ORDER BY id, job_id; job_id | pid | proc_schema | proc_name | succeeded | config | sqlerrcode | err_message --------+-----+-------------+------------------+-----------+------------+------------+------------------ 1001 | t | public | custom_job_error | f | | 22012 | division by zero 1000 | t | public | custom_job_ok | t | | | 1001 | t | public | custom_job_error | f | | 22012 | division by zero 1000 | t | public | custom_job_ok | t | {"foo": 1} | | 1001 | t | public | custom_job_error | f | {"bar": 1} | 22012 | division by zero -- Changing the config of one job SELECT scheduled FROM alter_job(:job_id_1, config => '{"foo": 2, "bar": 1}'::jsonb); scheduled ----------- t SELECT scheduled FROM alter_job(:job_id_1, next_start => now()); scheduled ----------- t SELECT _timescaledb_functions.restart_background_workers(); restart_background_workers ---------------------------- t SELECT test.wait_for_job_to_run(:job_id_1, 4); wait_for_job_to_run --------------------- t -- Check job execution history SELECT job_id, pid IS NOT NULL AS pid, proc_schema, proc_name, succeeded, config, sqlerrcode, err_message FROM timescaledb_information.job_history WHERE job_id = :job_id_1 ORDER BY id; job_id | pid | proc_schema | proc_name | succeeded | config | sqlerrcode | err_message --------+-----+-------------+---------------+-----------+----------------------+------------+------------- 1000 | t | public | custom_job_ok | t | | | 1000 | t | public | custom_job_ok | t | {"foo": 1} | | 1000 | t | public | custom_job_ok | t | {"bar": 1, "foo": 2} | | -- Change the job procedure to alter the job configuration during the execution CREATE OR REPLACE PROCEDURE custom_job_ok(job_id int, config jsonb) LANGUAGE PLPGSQL AS $$ BEGIN RAISE INFO 'custom_job'; PERFORM alter_job(job_id, config => '{"config_changed_by_job_execution": 1}'::jsonb); END $$; -- Run the job SELECT scheduled FROM alter_job(:job_id_1, next_start => now()); scheduled ----------- t SELECT _timescaledb_functions.restart_background_workers(); restart_background_workers ---------------------------- t SELECT test.wait_for_job_to_run(:job_id_1, 5); wait_for_job_to_run --------------------- t -- Check job execution history SELECT job_id, pid IS NOT NULL AS pid, proc_schema, proc_name, succeeded, config, sqlerrcode, err_message FROM timescaledb_information.job_history WHERE job_id = :job_id_1 ORDER BY id; job_id | pid | proc_schema | proc_name | succeeded | config | sqlerrcode | err_message --------+-----+-------------+---------------+-----------+----------------------------------------+------------+------------- 1000 | t | public | custom_job_ok | t | | | 1000 | t | public | custom_job_ok | t | {"foo": 1} | | 1000 | t | public | custom_job_ok | t | {"bar": 1, "foo": 2} | | 1000 | t | public | custom_job_ok | t | {"config_changed_by_job_execution": 1} | | -- Change the job procedure to alter the job configuration during the execution CREATE OR REPLACE PROCEDURE custom_job_ok(job_id int, config jsonb) LANGUAGE PLPGSQL AS $$ BEGIN RAISE INFO 'custom_job'; PERFORM alter_job(job_id, config => '{"change_not_logged": 1}'::jsonb); COMMIT; PERFORM alter_job(job_id, config => '{"only_last_change_is_logged": 1}'::jsonb); COMMIT; END $$; -- Run the job SELECT scheduled FROM alter_job(:job_id_1, next_start => now()); scheduled ----------- t SELECT _timescaledb_functions.restart_background_workers(); restart_background_workers ---------------------------- t SELECT test.wait_for_job_to_run(:job_id_1, 6); wait_for_job_to_run --------------------- t -- Check job execution history SELECT job_id, pid IS NOT NULL AS pid, proc_schema, proc_name, succeeded, config, sqlerrcode, err_message FROM timescaledb_information.job_history WHERE job_id = :job_id_1 ORDER BY id; job_id | pid | proc_schema | proc_name | succeeded | config | sqlerrcode | err_message --------+-----+-------------+---------------+-----------+----------------------------------------+------------+------------- 1000 | t | public | custom_job_ok | t | | | 1000 | t | public | custom_job_ok | t | {"foo": 1} | | 1000 | t | public | custom_job_ok | t | {"bar": 1, "foo": 2} | | 1000 | t | public | custom_job_ok | t | {"config_changed_by_job_execution": 1} | | 1000 | t | public | custom_job_ok | t | {"only_last_change_is_logged": 1} | | -- Alter other information about the job CREATE PROCEDURE custom_job_alter(job_id int, config jsonb) LANGUAGE PLPGSQL AS $$ BEGIN RAISE LOG 'custom_job_alter'; END $$; SELECT add_job('custom_job_alter', schedule_interval => interval '1 hour', initial_start := now()) AS job_id_3 \gset SELECT _timescaledb_functions.restart_background_workers(); restart_background_workers ---------------------------- t SELECT test.wait_for_job_to_run(:job_id_3, 1); wait_for_job_to_run --------------------- t SELECT timezone, fixed_schedule, config, schedule_interval FROM alter_job(:job_id_3, timezone => 'America/Sao_Paulo', fixed_schedule => false, config => '{"key": "value"}'::jsonb, schedule_interval => interval '10 min', next_start => now()); timezone | fixed_schedule | config | schedule_interval -------------------+----------------+------------------+------------------- America/Sao_Paulo | f | {"key": "value"} | @ 10 mins SELECT _timescaledb_functions.restart_background_workers(); restart_background_workers ---------------------------- t SELECT test.wait_for_job_to_run(:job_id_3, 2); wait_for_job_to_run --------------------- t -- Should return two executions, the second will show the changed values SELECT job_id, succeeded, data->'job'->>'timezone' AS timezone, data->'job'->>'fixed_schedule' AS fixed_schedule, data->'job'->>'schedule_interval' AS schedule_interval, data->'job'->'config' AS config FROM _timescaledb_internal.bgw_job_stat_history WHERE job_id = :job_id_3 ORDER BY id; job_id | succeeded | timezone | fixed_schedule | schedule_interval | config --------+-----------+-------------------+----------------+-------------------+------------------ 1002 | t | | true | 01:00:00 | 1002 | t | America/Sao_Paulo | false | 00:10:00 | {"key": "value"} SELECT delete_job(:job_id_1); delete_job ------------ SELECT delete_job(:job_id_2); delete_job ------------ SELECT delete_job(:job_id_3); delete_job ------------ ALTER SYSTEM RESET timescaledb.enable_job_execution_logging; SELECT pg_reload_conf(); pg_reload_conf ---------------- t \c :TEST_DBNAME :ROLE_SUPERUSER -- The GUC is PGC_SIGHUP context so only ALTER SYSTEM is allowed \set ON_ERROR_STOP 0 SHOW timescaledb.enable_job_execution_logging; timescaledb.enable_job_execution_logging ------------------------------------------ off SET timescaledb.enable_job_execution_logging TO OFF; ERROR: parameter "timescaledb.enable_job_execution_logging" cannot be changed now SHOW timescaledb.enable_job_execution_logging; timescaledb.enable_job_execution_logging ------------------------------------------ off ALTER DATABASE :TEST_DBNAME SET timescaledb.enable_job_execution_logging TO ON; ERROR: parameter "timescaledb.enable_job_execution_logging" cannot be changed now SHOW timescaledb.enable_job_execution_logging; timescaledb.enable_job_execution_logging ------------------------------------------ off \set ON_ERROR_STOP 1 SELECT _timescaledb_functions.stop_background_workers(); stop_background_workers ------------------------- t -- Test bgw_job_stat_history retention job -- Alter the drop_after interval to be fixed (30 days) to ensure tests are deterministic SELECT config AS config FROM _timescaledb_catalog.bgw_job WHERE id = 3 \gset SELECT config FROM alter_job(3, config => jsonb_set(:'config', '{drop_after}', '"30 days"')); config ---------------------------------------------------------------------------------------- {"drop_after": "30 days", "max_failures_per_job": 1000, "max_successes_per_job": 1000} -- These configuration should fail since they are not valid. \set ON_ERROR_STOP 0 SELECT config FROM alter_job(3, config => :'config'::jsonb - 'drop_after'); ERROR: drop_after interval not provided SELECT config FROM alter_job(3, config => :'config'::jsonb - 'max_successes_per_job'); ERROR: max_successes_per_job not provided SELECT config FROM alter_job(3, config => :'config'::jsonb - 'max_failures_per_job'); ERROR: max_failures_per_job not provided SELECT config FROM alter_job(3, config => jsonb_set(:'config', '{max_successes_per_job}', '0')); ERROR: max_successes_per_job has to be at least 10 SELECT config FROM alter_job(3, config => jsonb_set(:'config', '{max_failures_per_job}', '0')); ERROR: max_failures_per_job has to be at least 10 SELECT config FROM alter_job(3, config => jsonb_set(:'config', '{max_successes_per_job}', '"none"')); ERROR: invalid input syntax for type integer: "none" SELECT config FROM alter_job(3, config => jsonb_set(:'config', '{max_failures_per_job}', '"none"')); ERROR: invalid input syntax for type integer: "none" \set ON_ERROR_STOP 1 -- Test 1 TRUNCATE _timescaledb_internal.bgw_job_stat_history; -- Insert test data: jobs every 15 minutes from 3 months ago to today -- Each job runs for 5 minutes (job_id=100, pid=12345) -- Fix NOW to ensure the tests are deterministic SET timezone TO 'UTC'; INSERT INTO _timescaledb_internal.bgw_job_stat_history (job_id, pid, succeeded, execution_start, execution_finish, data) SELECT 100 as job_id, 12345 as pid, true as succeeded, ts as execution_start, ts + interval '5 minutes' as execution_finish, '{}'::jsonb as data FROM generate_series(now() - interval '90 days', now(), interval '15 minutes') as ts; -- Check data after insertion select * from job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- 100 | t | 8641 -- Test the retention job (job id 3) CALL run_job(3); -- Check data after retention SELECT * FROM job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- 100 | t | 1000 -- Verify only recent records remain SELECT * FROM recent_job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- 100 | t | 1000 -- Cleanup TRUNCATE _timescaledb_internal.bgw_job_stat_history; -- Test 2: Empty table (no job history) CALL run_job(3); SELECT * FROM job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- -- Verify only recent records remain SELECT * FROM recent_job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- -- Test 3: Odd number of entries (5 entries) INSERT INTO _timescaledb_internal.bgw_job_stat_history (job_id, pid, succeeded, execution_start, execution_finish, data) VALUES (301, 3001, true, now() - interval '60 days', now() - interval '60 days' + interval '5 minutes', '{}'), (302, 3002, true, now() - interval '6 weeks', now() - interval '6 weeks' + interval '5 minutes', '{}'), (303, 3003, true, now() - interval '30 days', now() - interval '30 days' + interval '5 minutes', '{}'), (301, 3001, true, now() - interval '2 weeks', now() - interval '2 weeks' + interval '5 minutes', '{}'), (304, 3004, true, now() - interval '1 week', now() - interval '1 week' + interval '5 minutes', '{}'); SELECT * FROM job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- 301 | t | 2 302 | t | 1 303 | t | 1 304 | t | 1 CALL run_job(3); SELECT * FROM job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- 301 | t | 1 303 | t | 1 304 | t | 1 -- Verify only recent records remain SELECT * FROM recent_job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- 301 | t | 1 303 | t | 1 304 | t | 1 TRUNCATE _timescaledb_internal.bgw_job_stat_history; -- Test 4: Even number of entries (6 entries) INSERT INTO _timescaledb_internal.bgw_job_stat_history (job_id, pid, succeeded, execution_start, execution_finish, data) VALUES (401, 4001, true, now() - interval '90 days', now() - interval '90 days' + interval '5 minutes', '{}'), (402, 4002, true, now() - interval '60 days', now() - interval '60 days' + interval '5 minutes', '{}'), (403, 4003, true, now() - interval '6 weeks', now() - interval '6 weeks' + interval '5 minutes', '{}'), (401, 4001, true, now() - interval '30 days', now() - interval '30 days' + interval '5 minutes', '{}'), (404, 4004, true, now() - interval '2 weeks', now() - interval '2 weeks' + interval '5 minutes', '{}'), (402, 4002, true, now() - interval '1 week', now() - interval '1 week' + interval '5 minutes', '{}'); SELECT * FROM job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- 403 | t | 1 401 | t | 2 404 | t | 1 402 | t | 2 CALL run_job(3); SELECT * FROM job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- 401 | t | 1 404 | t | 1 402 | t | 1 -- Verify only recent records remain SELECT * FROM recent_job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- 401 | t | 1 404 | t | 1 402 | t | 1 TRUNCATE _timescaledb_internal.bgw_job_stat_history; -- Test 5: Missing middle job id (gaps in sequence) INSERT INTO _timescaledb_internal.bgw_job_stat_history (job_id, pid, succeeded, execution_start, execution_finish, data) SELECT 501 + (row_number() over () % 3) as job_id, 5001 + (row_number() over () % 3) as pid, true as succeeded, ts as execution_start, ts + interval '5 minutes' as execution_finish, '{}'::jsonb as data FROM generate_series(now() - interval '60 days', now() - interval '1 week', interval '1 week') as ts; -- Delete some records to create gaps DELETE FROM _timescaledb_internal.bgw_job_stat_history WHERE id IN (SELECT id FROM _timescaledb_internal.bgw_job_stat_history ORDER BY id LIMIT 2 OFFSET 2); SELECT * FROM job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- 503 | t | 3 502 | t | 2 501 | t | 1 CALL run_job(3); SELECT * FROM job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- 503 | t | 1 502 | t | 1 501 | t | 1 -- Verify only recent records remain SELECT * FROM recent_job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- 503 | t | 1 502 | t | 1 501 | t | 1 TRUNCATE _timescaledb_internal.bgw_job_stat_history; -- Test 6: All records older than retention period INSERT INTO _timescaledb_internal.bgw_job_stat_history (job_id, pid, succeeded, execution_start, execution_finish, data) VALUES (601, 6001, true, now() - interval '90 days', now() - interval '90 days' + interval '5 minutes', '{}'), (602, 6002, true, now() - interval '60 days', now() - interval '60 days' + interval '5 minutes', '{}'), (601, 6001, true, now() - interval '6 weeks', now() - interval '6 weeks' + interval '5 minutes', '{}'); SELECT * FROM job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- 601 | t | 2 602 | t | 1 CALL run_job(3); SELECT * FROM job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- 601 | t | 2 602 | t | 1 -- Verify only recent records remain SELECT * FROM recent_job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- TRUNCATE _timescaledb_internal.bgw_job_stat_history; -- Test 7: No records older than retention period INSERT INTO _timescaledb_internal.bgw_job_stat_history (job_id, pid, succeeded, execution_start, execution_finish, data) VALUES (701, 7001, true, now() - interval '1 week', now() - interval '1 week' + interval '7 minutes', '{}'), (702, 7002, true, now() - interval '6 days', now() - interval '6 days' + interval '7 minutes', '{}'), (703, 7003, true, now() - interval '7 days', now() - interval '7 days' + interval '7 minutes', '{}'), (701, 7001, true, now() - interval '4 days', now() - interval '4 days' + interval '7 minutes', '{}'); SELECT * FROM job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- 701 | t | 2 702 | t | 1 703 | t | 1 CALL run_job(3); SELECT * FROM job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- 701 | t | 2 702 | t | 1 703 | t | 1 -- Verify only recent records remain SELECT * FROM recent_job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- 701 | t | 2 702 | t | 1 703 | t | 1 TRUNCATE _timescaledb_internal.bgw_job_stat_history; -- Test 8: No records older than retention period and not more than -- the expected number of successes and failures per job. SELECT config AS config FROM _timescaledb_catalog.bgw_job WHERE id = 3 \gset SELECT config FROM alter_job(3, config => jsonb_set(jsonb_set(:'config', '{max_failures_per_job}', '15'), '{max_successes_per_job}', '10')); config ------------------------------------------------------------------------------------ {"drop_after": "30 days", "max_failures_per_job": 15, "max_successes_per_job": 10} INSERT INTO _timescaledb_internal.bgw_job_stat_history (job_id, pid, succeeded, execution_start, execution_finish, data) VALUES (803, 7003, true, now() - interval '7 days', now() - interval '7 days' + interval '7 minutes', '{}'), (801, 7001, true, now() - interval '4 days', now() - interval '4 days' + interval '7 minutes', '{}'); INSERT INTO _timescaledb_internal.bgw_job_stat_history(job_id, pid, succeeded, execution_start, execution_finish, data) SELECT 801, 7001, true, now() - format('%s hour', hours)::interval, now() - interval '1 week' + interval '7 minutes', '{}' FROM generate_series(1,20) hours; INSERT INTO _timescaledb_internal.bgw_job_stat_history(job_id, pid, succeeded, execution_start, execution_finish, data) SELECT 802, 7001, false, now() - format('%s minutes', hours)::interval, now() - interval '6 days' + interval '7 minutes', '{}' FROM generate_series(1,20) hours; SELECT * FROM job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- 801 | t | 21 803 | t | 1 802 | f | 20 CALL run_job(3); SELECT * FROM job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- 801 | t | 10 803 | t | 1 802 | f | 15 -- Verify only recent records remain SELECT * FROM recent_job_history_summary; job_id | succeeded | record_count --------+-----------+-------------- 801 | t | 10 803 | t | 1 802 | f | 15 -- Cleanup TRUNCATE _timescaledb_internal.bgw_job_stat_history; -- Test that lock_timeout can be configured SELECT config FROM alter_job(3, config => jsonb_set(:'config', '{lock_timeout}', '"1s"')); config -------------------------------------------------------------------------------------------------------------- {"drop_after": "30 days", "lock_timeout": "1s", "max_failures_per_job": 1000, "max_successes_per_job": 1000} CALL run_job(3); -- Test the job_history_bsearch function directly as well -- It returns the first element where execution_finish >= search_point or NULL if no such element exists \set NOW '2025-08-15 12:34:00' -- No elements in table SELECT _timescaledb_functions.job_history_bsearch(:'NOW'::timestamptz - interval '1 month'); job_history_bsearch --------------------- -- Single element INSERT INTO _timescaledb_internal.bgw_job_stat_history (id, job_id, pid, succeeded, execution_start, execution_finish, data) VALUES (5, 601, 6001, true, :'NOW'::timestamptz - interval '2 weeks', :'NOW'::timestamptz - interval '2 weeks' + interval '5 minutes', '{}'); -- Return the single element SELECT _timescaledb_functions.job_history_bsearch(:'NOW'::timestamptz - interval '1 month'); job_history_bsearch --------------------- 5 -- Return NULL SELECT _timescaledb_functions.job_history_bsearch(:'NOW'::timestamptz - interval '1 week'); job_history_bsearch --------------------- TRUNCATE _timescaledb_internal.bgw_job_stat_history; -- Two elements INSERT INTO _timescaledb_internal.bgw_job_stat_history (id, job_id, pid, succeeded, execution_start, execution_finish, data) VALUES (5, 701, 7001, true, :'NOW'::timestamptz - interval '3 weeks', :'NOW'::timestamptz - interval '3 weeks' + interval '5 minutes', '{}'), (6, 702, 7002, true, :'NOW'::timestamptz - interval '1 week', :'NOW'::timestamptz - interval '1 week' + interval '5 minutes', '{}'); -- Returns the first element SELECT _timescaledb_functions.job_history_bsearch(:'NOW'::timestamptz - interval '1 month'); job_history_bsearch --------------------- 5 -- Returns the second element SELECT _timescaledb_functions.job_history_bsearch(:'NOW'::timestamptz - interval '2 weeks'); job_history_bsearch --------------------- 6 -- Returns NULL SELECT _timescaledb_functions.job_history_bsearch(:'NOW'::timestamptz - interval '3 days'); job_history_bsearch --------------------- TRUNCATE _timescaledb_internal.bgw_job_stat_history; -- Odd number of elements INSERT INTO _timescaledb_internal.bgw_job_stat_history (id, job_id, pid, succeeded, execution_start, execution_finish, data) VALUES (5, 801, 8001, true, :'NOW'::timestamptz - interval '5 weeks', :'NOW'::timestamptz - interval '5 weeks' + interval '5 minutes', '{}'), (6, 802, 8002, true, :'NOW'::timestamptz - interval '4 weeks', :'NOW'::timestamptz - interval '4 weeks' + interval '5 minutes', '{}'), (7, 803, 8003, true, :'NOW'::timestamptz - interval '3 weeks', :'NOW'::timestamptz - interval '3 weeks' + interval '5 minutes', '{}'), (8, 804, 8004, true, :'NOW'::timestamptz - interval '2 weeks', :'NOW'::timestamptz - interval '2 weeks' + interval '5 minutes', '{}'), (9, 805, 8005, true, :'NOW'::timestamptz - interval '1 week', :'NOW'::timestamptz - interval '1 week' + interval '5 minutes', '{}'); -- Returns the first element SELECT _timescaledb_functions.job_history_bsearch(:'NOW'::timestamptz - interval '6 weeks'); job_history_bsearch --------------------- 5 -- Returns the middle element SELECT _timescaledb_functions.job_history_bsearch(:'NOW'::timestamptz - interval '3 weeks'); job_history_bsearch --------------------- 7 -- Returns one after the middle element SELECT _timescaledb_functions.job_history_bsearch(:'NOW'::timestamptz - interval '2 weeks 3 days'); job_history_bsearch --------------------- 8 -- Returns NULL SELECT _timescaledb_functions.job_history_bsearch(:'NOW'::timestamptz - interval '2 days'); job_history_bsearch --------------------- TRUNCATE _timescaledb_internal.bgw_job_stat_history; -- Even number of elements INSERT INTO _timescaledb_internal.bgw_job_stat_history (id, job_id, pid, succeeded, execution_start, execution_finish, data) VALUES (5, 902, 9002, true, :'NOW'::timestamptz - interval '5 weeks', :'NOW'::timestamptz - interval '5 weeks' + interval '5 minutes', '{}'), (6, 903, 9003, true, :'NOW'::timestamptz - interval '4 weeks', :'NOW'::timestamptz - interval '4 weeks' + interval '5 minutes', '{}'), (7, 904, 9004, true, :'NOW'::timestamptz - interval '3 weeks', :'NOW'::timestamptz - interval '3 weeks' + interval '5 minutes', '{}'), (8, 905, 9005, true, :'NOW'::timestamptz - interval '2 weeks', :'NOW'::timestamptz - interval '2 weeks' + interval '5 minutes', '{}'), (9, 906, 9006, true, :'NOW'::timestamptz - interval '1 week', :'NOW'::timestamptz - interval '1 week' + interval '5 minutes', '{}'), (10, 907, 9007, true, :'NOW'::timestamptz - interval '3 days', :'NOW'::timestamptz - interval '3 days' + interval '5 minutes', '{}'); -- Returns the first element SELECT _timescaledb_functions.job_history_bsearch(:'NOW'::timestamptz - interval '6 weeks'); job_history_bsearch --------------------- 5 -- Returns the middle element SELECT _timescaledb_functions.job_history_bsearch(:'NOW'::timestamptz - interval '3 weeks'); job_history_bsearch --------------------- 7 -- Returns one after the middle element SELECT _timescaledb_functions.job_history_bsearch(:'NOW'::timestamptz - interval '2 weeks 3 days'); job_history_bsearch --------------------- 8 -- Returns NULL SELECT _timescaledb_functions.job_history_bsearch(:'NOW'::timestamptz - interval '2 days'); job_history_bsearch --------------------- TRUNCATE _timescaledb_internal.bgw_job_stat_history; -- With gaps in id INSERT INTO _timescaledb_internal.bgw_job_stat_history (id, job_id, pid, succeeded, execution_start, execution_finish, data) VALUES (10, 1001, 10001, true, :'NOW'::timestamptz - interval '5 weeks', :'NOW'::timestamptz - interval '5 weeks' + interval '5 minutes', '{}'), (11, 1002, 10002, true, :'NOW'::timestamptz - interval '4 weeks', :'NOW'::timestamptz - interval '4 weeks' + interval '5 minutes', '{}'), (13, 1003, 10003, true, :'NOW'::timestamptz - interval '3 weeks', :'NOW'::timestamptz - interval '3 weeks' + interval '5 minutes', '{}'), (15, 1004, 10004, true, :'NOW'::timestamptz - interval '2 weeks', :'NOW'::timestamptz - interval '2 weeks' + interval '5 minutes', '{}'), (16, 1005, 10005, true, :'NOW'::timestamptz - interval '1 week', :'NOW'::timestamptz - interval '1 week' + interval '5 minutes', '{}'); -- Returns id before the gap SELECT _timescaledb_functions.job_history_bsearch(:'NOW'::timestamptz - interval '3 weeks 3 days') AS result_gap_trigger1; result_gap_trigger1 --------------------- 13 -- Returns id after the gap SELECT _timescaledb_functions.job_history_bsearch(:'NOW'::timestamptz - interval '2 weeks 3 days') AS result_gap_trigger2; result_gap_trigger2 --------------------- 15 -- Returns second element SELECT _timescaledb_functions.job_history_bsearch(:'NOW'::timestamptz - interval '4 weeks 3 days') AS result_gap_trigger3; result_gap_trigger3 --------------------- 11 -- Final cleanup TRUNCATE _timescaledb_internal.bgw_job_stat_history; ================================================ FILE: tsl/test/expected/bgw_job_stat_history_errors.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER \set client_min_messages TO NOTICE; create or replace procedure job_fail(jobid int, config jsonb) language plpgsql as $$ begin perform pg_sleep(2); raise exception 'raising an exception'; end $$; -- very simple case: job that raises an exception select add_job('job_fail', '4 minutes') as jobf_id \gset -- test jobs that try to update concurrently CREATE TABLE custom_log ( a int, b int, msg text ); insert into custom_log values (0, 0, 'msg0'); ALTER SYSTEM SET DEFAULT_TRANSACTION_ISOLATION TO 'serializable'; SELECT pg_reload_conf(); pg_reload_conf ---------------- t -- Reconnect to make sure the GUC is set \c :TEST_DBNAME :ROLE_SUPERUSER -- test a concurrent update CREATE OR REPLACE PROCEDURE custom_proc1(jobid int, config jsonb) LANGUAGE PLPGSQL AS $$ BEGIN UPDATE custom_log set msg = 'msg1' where msg = 'msg0'; perform pg_sleep(5); COMMIT; END $$; CREATE OR REPLACE PROCEDURE custom_proc2(jobid int, config jsonb) LANGUAGE PLPGSQL AS $$ BEGIN UPDATE custom_log set msg = 'msg2' where msg = 'msg0'; COMMIT; END $$; select add_job('custom_proc1', '2 min', initial_start => now()); add_job --------- 1001 -- to make sure custom_log is first updated by custom_proc_1 select add_job('custom_proc2', '2 min', initial_start => now() + interval '2 seconds'); add_job --------- 1002 SELECT _timescaledb_functions.start_background_workers(); start_background_workers -------------------------- t -- enough time to for job_fail to fail select pg_sleep(5); pg_sleep ---------- select job_id, data->'job'->>'proc_name' as proc_name, data->'error_data'->>'message' as err_message, data->'error_data'->>'sqlerrcode' as sqlerrcode from _timescaledb_internal.bgw_job_stat_history where job_id = :jobf_id and succeeded is false; job_id | proc_name | err_message | sqlerrcode --------+-----------+----------------------+------------ 1000 | job_fail | raising an exception | P0001 select delete_job(:jobf_id); delete_job ------------ select pg_sleep(5); pg_sleep ---------- -- exclude internal jobs select job_id, data->'job'->>'proc_name' as proc_name, data->'error_data'->>'message' as err_message, data->'error_data'->>'sqlerrcode' as sqlerrcode from _timescaledb_internal.bgw_job_stat_history WHERE job_id >= 1000 and succeeded is false; job_id | proc_name | err_message | sqlerrcode --------+--------------+-----------------------------------------------------+------------ 1000 | job_fail | raising an exception | P0001 1002 | custom_proc2 | could not serialize access due to concurrent update | 40001 ALTER SYSTEM RESET DEFAULT_TRANSACTION_ISOLATION; SELECT pg_reload_conf(); pg_reload_conf ---------------- t -- Reconnect to make sure the GUC is set \c :TEST_DBNAME :ROLE_SUPERUSER -- test the retention job SELECT next_start FROM alter_job(3, next_start => '2060-01-01 00:00:00+00'::timestamptz); next_start ------------------------------ Wed Dec 31 16:00:00 2059 PST DELETE FROM _timescaledb_internal.bgw_job_stat_history; INSERT INTO _timescaledb_internal.bgw_job_stat_history(job_id, pid, succeeded, execution_start, execution_finish, data) VALUES (123, 12345, false, '2000-01-01 00:00:00+00'::timestamptz, '2000-01-01 00:00:10+00'::timestamptz, '{}'), (456, 45678, false, '2000-01-01 00:00:20+00'::timestamptz, '2000-01-01 00:00:40+00'::timestamptz, '{}'), -- not older than a month (123, 23456, false, '2050-01-01 00:00:00+00'::timestamptz, '2050-01-01 00:00:10+00'::timestamptz, '{}'); -- 3 rows in the table before policy runs SELECT job_id, pid, succeeded, execution_start, execution_finish, data FROM _timescaledb_internal.bgw_job_stat_history WHERE succeeded IS FALSE; job_id | pid | succeeded | execution_start | execution_finish | data --------+-------+-----------+------------------------------+------------------------------+------ 123 | 12345 | f | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:10 1999 PST | {} 456 | 45678 | f | Fri Dec 31 16:00:20 1999 PST | Fri Dec 31 16:00:40 1999 PST | {} 123 | 23456 | f | Fri Dec 31 16:00:00 2049 PST | Fri Dec 31 16:00:10 2049 PST | {} -- drop all job_stats for the retention job DELETE FROM _timescaledb_internal.bgw_job_stat WHERE job_id = 3; SELECT FROM alter_job(3, next_start => now()); -- SELECT _timescaledb_functions.restart_background_workers(); restart_background_workers ---------------------------- t SELECT test.wait_for_job_to_run(3, 1); wait_for_job_to_run --------------------- t -- only the last row remains SELECT job_id, pid, succeeded, execution_start, execution_finish, data FROM _timescaledb_internal.bgw_job_stat_history WHERE succeeded IS FALSE; job_id | pid | succeeded | execution_start | execution_finish | data --------+-------+-----------+------------------------------+------------------------------+------ 123 | 23456 | f | Fri Dec 31 16:00:00 2049 PST | Fri Dec 31 16:00:10 2049 PST | {} -- test failure when starting jobs \c :TEST_DBNAME :ROLE_SUPERUSER SELECT _timescaledb_functions.stop_background_workers(); stop_background_workers ------------------------- t -- Job didn't finish yet and Crash detected DELETE FROM _timescaledb_internal.bgw_job_stat_history; INSERT INTO _timescaledb_internal.bgw_job_stat_history(job_id, pid, succeeded, execution_start, execution_finish, data) VALUES (1, NULL, NULL, '2000-01-01 00:00:00+00'::timestamptz, NULL, '{}'), -- Crash server detected (2, 2222, false, '2000-01-01 00:00:00+00'::timestamptz, NULL, '{}'), -- Didn't finished yet (3, 3333, false, '2000-01-01 00:00:00+00'::timestamptz, '2000-01-01 01:00:00+00'::timestamptz, '{}'), -- Finish with ERROR (4, 4444, true, '2000-01-01 00:00:00+00'::timestamptz, '2000-01-01 01:00:00+00'::timestamptz, '{}'); -- Finish with SUCCESS SELECT job_id, pid, succeeded, start_time, finish_time, config, err_message FROM timescaledb_information.job_history ORDER BY job_id; job_id | pid | succeeded | start_time | finish_time | config | err_message --------+------+-----------+------------------------------+------------------------------+--------+------------------------------------- 1 | | | Fri Dec 31 16:00:00 1999 PST | | | job crash detected, see server logs 2 | 2222 | f | Fri Dec 31 16:00:00 1999 PST | | | 3 | 3333 | f | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 17:00:00 1999 PST | | 4 | 4444 | t | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 17:00:00 1999 PST | | SELECT job_id, pid, start_time, finish_time, err_message FROM timescaledb_information.job_errors ORDER BY job_id; job_id | pid | start_time | finish_time | err_message --------+------+------------------------------+------------------------------+------------------------------------- 1 | | Fri Dec 31 16:00:00 1999 PST | | job crash detected, see server logs 2 | 2222 | Fri Dec 31 16:00:00 1999 PST | | 3 | 3333 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 17:00:00 1999 PST | DELETE FROM _timescaledb_internal.bgw_job_stat; DELETE FROM _timescaledb_internal.bgw_job_stat_history; DELETE FROM _timescaledb_catalog.bgw_job CASCADE; SELECT _timescaledb_functions.start_background_workers(); start_background_workers -------------------------- t \set VERBOSITY default -- Setup Jobs DO $TEST$ DECLARE stmt TEXT; njobs INT := 30; BEGIN RAISE INFO 'Creating % jobs', njobs; FOR stmt IN SELECT format('CREATE PROCEDURE custom_job%s(job_id int, config jsonb) LANGUAGE PLPGSQL AS $$ BEGIN PERFORM pg_sleep(1); END; $$', i) FROM generate_series(1, njobs) AS i LOOP EXECUTE stmt; END LOOP; RAISE INFO 'Scheduling % jobs', njobs; PERFORM add_job(format('custom_job%s', i)::regproc, schedule_interval => interval '1 hour', initial_start := now()) FROM generate_series(1, njobs) AS i; END; $TEST$; INFO: Creating 30 jobs INFO: Scheduling 30 jobs SELECT _timescaledb_functions.restart_background_workers(); restart_background_workers ---------------------------- t -- Wait for jobs to run DO $TEST$ DECLARE njobs INT := 30; BEGIN RAISE INFO 'Waiting for the % jobs to run', njobs; SET LOCAL client_min_messages TO WARNING; PERFORM test.wait_for_job_to_run_or_fail(id) FROM _timescaledb_catalog.bgw_job WHERE id >= 1000; END; $TEST$; INFO: Waiting for the 30 jobs to run SELECT count(*) > 0 FROM timescaledb_information.job_history WHERE succeeded IS FALSE AND err_message ~ 'failed to start job'; ?column? ---------- t SELECT count(*) > 0 FROM timescaledb_information.job_errors WHERE err_message ~ 'failed to start job'; ?column? ---------- t \set VERBOSITY terse \c :TEST_DBNAME :ROLE_SUPERUSER SELECT _timescaledb_functions.stop_background_workers(); stop_background_workers ------------------------- t ================================================ FILE: tsl/test/expected/bgw_job_stat_history_errors_permissions.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER -- Table to update concurrently to generate error message CREATE TABLE my_table (a int, b int); INSERT INTO my_table VALUES (0, 0); GRANT ALL ON my_table TO PUBLIC; ALTER SYSTEM SET DEFAULT_TRANSACTION_ISOLATION TO 'serializable'; SELECT pg_reload_conf(); pg_reload_conf ---------------- t \c :TEST_DBNAME :ROLE_SUPERUSER SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE OR REPLACE PROCEDURE job_fail(jobid int, config jsonb) AS $$ BEGIN RAISE EXCEPTION 'raising an exception'; END $$ LANGUAGE plpgsql; SELECT add_job('job_fail', '4 minutes', initial_start => now()) as job_fail_id \gset CREATE OR REPLACE PROCEDURE custom_proc1(jobid int, config jsonb) LANGUAGE PLPGSQL AS $$ BEGIN UPDATE my_table SET b = 1 WHERE a = 0; PERFORM pg_sleep(5); COMMIT; END $$; SELECT add_job('custom_proc1', '2 min', initial_start => now()) as custom_proc1_id \gset SET ROLE :ROLE_DEFAULT_PERM_USER_2; CREATE OR REPLACE PROCEDURE custom_proc2(jobid int, config jsonb) LANGUAGE PLPGSQL AS $$ BEGIN UPDATE my_table SET b = 2 WHERE a = 0; PERFORM pg_sleep(5); COMMIT; END $$; -- to make sure custom_log is first updated by custom_proc_1 select add_job('custom_proc2', '2 min', initial_start => now() + interval '1 seconds') as custom_proc2_id \gset SET ROLE :ROLE_SUPERUSER; SELECT _timescaledb_functions.start_background_workers(); start_background_workers -------------------------- t SELECT pg_sleep(6); pg_sleep ---------- \d timescaledb_information.job_errors View "timescaledb_information.job_errors" Column | Type | Collation | Nullable | Default -------------+--------------------------+-----------+----------+--------- job_id | integer | | | proc_schema | text | | | proc_name | text | | | pid | integer | | | start_time | timestamp with time zone | | | finish_time | timestamp with time zone | | | sqlerrcode | text | | | err_message | text | | | -- We add a few entries without a matching job id, so that we get a -- null owner. Note that the second entry does not have a message -- defined, so it will print a standardized message assuming that the -- job crashed. \set start '2000-01-01 00:00:00+00' \set finish '2000-01-01 00:00:10+00' INSERT INTO _timescaledb_internal.bgw_job_stat_history(job_id, pid, succeeded, execution_start, execution_finish, data) VALUES (11111, 12345, false, :'start'::timestamptz, :'finish'::timestamptz, '{"error_data": {"message": "not an error"}}'), (22222, 45678, false, :'start'::timestamptz, NULL, '{}'), -- Started and didn't finished yet (33333, NULL, NULL, :'start'::timestamptz, NULL, NULL); -- Crash detected cause not assigned an PID -- We check the log as different users and should only see what we -- have permissions to see. We only bother about jobs at 1000 or -- larger since the standard jobs are flaky. SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT job_id, proc_schema, proc_name, sqlerrcode, err_message FROM timescaledb_information.job_errors WHERE job_id >= 1000 ORDER BY job_id; job_id | proc_schema | proc_name | sqlerrcode | err_message --------+-------------+--------------+------------+----------------------------------------------------- 1000 | public | job_fail | P0001 | raising an exception 1002 | public | custom_proc2 | 40001 | could not serialize access due to concurrent update 11111 | | | | not an error 22222 | | | | SET ROLE :ROLE_DEFAULT_PERM_USER_2; SELECT job_id, proc_schema, proc_name, sqlerrcode, err_message FROM timescaledb_information.job_errors WHERE job_id >= 1000 ORDER BY job_id; job_id | proc_schema | proc_name | sqlerrcode | err_message --------+-------------+--------------+------------+----------------------------------------------------- 1000 | public | job_fail | P0001 | raising an exception 1002 | public | custom_proc2 | 40001 | could not serialize access due to concurrent update 11111 | | | | not an error 22222 | | | | SET ROLE :ROLE_SUPERUSER; SELECT job_id, proc_schema, proc_name, sqlerrcode, err_message FROM timescaledb_information.job_errors WHERE job_id >= 1000 ORDER BY job_id; job_id | proc_schema | proc_name | sqlerrcode | err_message --------+-------------+--------------+------------+----------------------------------------------------- 1000 | public | job_fail | P0001 | raising an exception 1002 | public | custom_proc2 | 40001 | could not serialize access due to concurrent update 11111 | | | | not an error 22222 | | | | 33333 | | | | job crash detected, see server logs SELECT delete_job(:custom_proc2_id); delete_job ------------ SELECT delete_job(:custom_proc1_id); delete_job ------------ SELECT delete_job(:job_fail_id); delete_job ------------ ALTER SYSTEM RESET DEFAULT_TRANSACTION_ISOLATION; SELECT pg_reload_conf(); pg_reload_conf ---------------- t \c :TEST_DBNAME :ROLE_SUPERUSER SELECT _timescaledb_functions.stop_background_workers(); stop_background_workers ------------------------- t ================================================ FILE: tsl/test/expected/bgw_policy.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. CREATE OR REPLACE FUNCTION reorder_called(chunk_id INT) RETURNS BOOL AS $$ SELECT indisclustered FROM _timescaledb_catalog.chunk ch JOIN pg_index i on indrelid = format('%I.%I',ch.schema_name,ch.table_name)::regclass WHERE ch.id = chunk_id; $$ LANGUAGE SQL; CREATE TABLE test_table(time timestamptz, chunk_id int); SELECT create_hypertable('test_table', 'time'); create_hypertable ------------------------- (1,public,test_table,t) -- These inserts should create 5 different chunks INSERT INTO test_table VALUES (now() - INTERVAL '3 weeks', 1); INSERT INTO test_table VALUES (now(), 2); INSERT INTO test_table VALUES (now() - INTERVAL '5 months', 3); INSERT INTO test_table VALUES (now() - INTERVAL '3 months', 4); INSERT INTO test_table VALUES (now() - INTERVAL '8 months', 5); SELECT COUNT(*) FROM _timescaledb_catalog.chunk as c, _timescaledb_catalog.hypertable as ht where c.hypertable_id = ht.id and ht.table_name='test_table'; count ------- 5 -- Make sure reorder correctly selects chunks to reorder -- by starting with oldest chunks select add_reorder_policy('test_table', 'test_table_time_idx') as reorder_job_id \gset select * from _timescaledb_catalog.bgw_job WHERE id >= 1000 ORDER BY id; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ------+-----------------------+-------------------+-------------+-------------+--------------+------------------------+----------------+-------------------+-----------+----------------+---------------+---------------+-----------------------------------------------------------+------------------------+----------------------+---------- 1000 | Reorder Policy [1000] | @ 84 hours | @ 0 | -1 | @ 5 mins | _timescaledb_functions | policy_reorder | default_perm_user | t | f | | 1 | {"index_name": "test_table_time_idx", "hypertable_id": 1} | _timescaledb_functions | policy_reorder_check | select job_id, chunk_id, num_times_job_run from _timescaledb_internal.bgw_policy_chunk_stats; job_id | chunk_id | num_times_job_run --------+----------+------------------- -- Make a manual calls to reorder: make sure the correct chunk is called -- Chunk 5 should be first CALL run_job(:reorder_job_id); select job_id, chunk_id, num_times_job_run from _timescaledb_internal.bgw_policy_chunk_stats; job_id | chunk_id | num_times_job_run --------+----------+------------------- 1000 | 5 | 1 -- Confirm that reorder was called on the correct chunk Oid SELECT reorder_called(5); reorder_called ---------------- t -- Chunk 3 is next CALL run_job(:reorder_job_id); select job_id, chunk_id, num_times_job_run from _timescaledb_internal.bgw_policy_chunk_stats; job_id | chunk_id | num_times_job_run --------+----------+------------------- 1000 | 5 | 1 1000 | 3 | 1 SELECT reorder_called(3); reorder_called ---------------- t -- Chunk 4 is next CALL run_job(:reorder_job_id); select job_id, chunk_id, num_times_job_run from _timescaledb_internal.bgw_policy_chunk_stats; job_id | chunk_id | num_times_job_run --------+----------+------------------- 1000 | 5 | 1 1000 | 3 | 1 1000 | 4 | 1 SELECT reorder_called(4); reorder_called ---------------- t -- The following calls should not reorder any chunk, because they're all too new CALL run_job(:reorder_job_id); NOTICE: no chunks need reordering for hypertable public.test_table select job_id, chunk_id, num_times_job_run from _timescaledb_internal.bgw_policy_chunk_stats; job_id | chunk_id | num_times_job_run --------+----------+------------------- 1000 | 5 | 1 1000 | 3 | 1 1000 | 4 | 1 CALL run_job(:reorder_job_id); NOTICE: no chunks need reordering for hypertable public.test_table select job_id, chunk_id, num_times_job_run from _timescaledb_internal.bgw_policy_chunk_stats; job_id | chunk_id | num_times_job_run --------+----------+------------------- 1000 | 5 | 1 1000 | 3 | 1 1000 | 4 | 1 INSERT INTO test_table VALUES (now() - INTERVAL '7 days', 6); -- This call should reorder chunk 1 CALL run_job(:reorder_job_id); select job_id, chunk_id, num_times_job_run from _timescaledb_internal.bgw_policy_chunk_stats; job_id | chunk_id | num_times_job_run --------+----------+------------------- 1000 | 5 | 1 1000 | 3 | 1 1000 | 4 | 1 1000 | 1 | 1 SELECT reorder_called(1); reorder_called ---------------- t -- Should not reorder anything, because all chunks are too new CALL run_job(:reorder_job_id); NOTICE: no chunks need reordering for hypertable public.test_table select job_id, chunk_id, num_times_job_run from _timescaledb_internal.bgw_policy_chunk_stats; job_id | chunk_id | num_times_job_run --------+----------+------------------- 1000 | 5 | 1 1000 | 3 | 1 1000 | 4 | 1 1000 | 1 | 1 select remove_reorder_policy('test_table'); remove_reorder_policy ----------------------- -- Now do drop_chunks test select add_retention_policy('test_table', INTERVAL '4 months', true) as drop_chunks_job_id \gset SELECT count(*) FROM _timescaledb_catalog.chunk as c, _timescaledb_catalog.hypertable as ht where c.hypertable_id = ht.id and ht.table_name='test_table'; count ------- 6 -- Now simulate drop_chunks running automatically by calling it explicitly CALL run_job(:drop_chunks_job_id); -- Should have 4 chunks left SELECT count(*) FROM _timescaledb_catalog.chunk as c, _timescaledb_catalog.hypertable as ht where c.hypertable_id = ht.id and ht.table_name='test_table' \gset before_ select :before_count=4; ?column? ---------- t -- Make sure this second call does nothing CALL run_job(:drop_chunks_job_id); SELECT count(*) FROM _timescaledb_catalog.chunk as c, _timescaledb_catalog.hypertable as ht where c.hypertable_id = ht.id and ht.table_name='test_table' \gset after_ -- Should be true select :before_count=:after_count; ?column? ---------- t INSERT INTO test_table VALUES (now() - INTERVAL '2 weeks', 1); SELECT count(*) FROM _timescaledb_catalog.chunk as c, _timescaledb_catalog.hypertable as ht where c.hypertable_id = ht.id and ht.table_name='test_table' \gset before_ -- This call should also do nothing CALL run_job(:drop_chunks_job_id); SELECT count(*) FROM _timescaledb_catalog.chunk as c, _timescaledb_catalog.hypertable as ht where c.hypertable_id = ht.id and ht.table_name='test_table' \gset after_ -- Should be true select :before_count=:after_count; ?column? ---------- t select remove_retention_policy('test_table'); remove_retention_policy ------------------------- -- Now test reorder chunk selection when there is space partitioning TRUNCATE test_table; SELECT add_dimension('public.test_table', 'chunk_id', 2); add_dimension ---------------------------------- (2,public,test_table,chunk_id,t) INSERT INTO test_table VALUES (now() - INTERVAL '3 weeks', 1); INSERT INTO test_table VALUES (now(), 2); INSERT INTO test_table VALUES (now() - INTERVAL '5 months', 3); INSERT INTO test_table VALUES (now() - INTERVAL '3 months', 4); INSERT INTO test_table VALUES (now() - INTERVAL '3 months', -4); INSERT INTO test_table VALUES (now() - INTERVAL '8 months', 5); INSERT INTO test_table VALUES (now() - INTERVAL '8 months', -5); select add_reorder_policy('test_table', 'test_table_time_idx') as reorder_job_id \gset -- Should be nothing in the chunk_stats table select count(*) from _timescaledb_internal.bgw_policy_chunk_stats where job_id=:reorder_job_id; count ------- 0 -- Make a manual calls to reorder: make sure the correct (oldest) chunk is called select chunk_id from _timescaledb_catalog.dimension_slice as ds, _timescaledb_catalog.chunk_constraint as cc where ds.dimension_id=1 and ds.id=cc.dimension_slice_id ORDER BY ds.range_start LIMIT 1 \gset oldest_ CALL run_job(:reorder_job_id); select job_id, chunk_id, num_times_job_run from _timescaledb_internal.bgw_policy_chunk_stats where job_id=:reorder_job_id and chunk_id=:oldest_chunk_id; job_id | chunk_id | num_times_job_run --------+----------+------------------- 1002 | 13 | 1 -- Confirm that reorder was called on the correct chunk Oid SELECT reorder_called(:oldest_chunk_id); reorder_called ---------------- t -- Now run reorder again and pick the next oldest chunk select cc.chunk_id from _timescaledb_catalog.dimension_slice as ds, _timescaledb_catalog.chunk_constraint as cc where ds.dimension_id=1 and ds.id=cc.dimension_slice_id and cc.chunk_id NOT IN (select chunk_id from _timescaledb_internal.bgw_policy_chunk_stats) ORDER BY ds.range_start LIMIT 1 \gset oldest_ CALL run_job(:reorder_job_id); select job_id, chunk_id, num_times_job_run from _timescaledb_internal.bgw_policy_chunk_stats where job_id=:reorder_job_id and chunk_id=:oldest_chunk_id; job_id | chunk_id | num_times_job_run --------+----------+------------------- 1002 | 14 | 1 -- Confirm that reorder was called on the correct chunk Oid SELECT reorder_called(:oldest_chunk_id); reorder_called ---------------- t -- Again select cc.chunk_id from _timescaledb_catalog.dimension_slice as ds, _timescaledb_catalog.chunk_constraint as cc where ds.dimension_id=1 and ds.id=cc.dimension_slice_id and cc.chunk_id NOT IN (select chunk_id from _timescaledb_internal.bgw_policy_chunk_stats) ORDER BY ds.range_start LIMIT 1 \gset oldest_ CALL run_job(:reorder_job_id); select job_id, chunk_id, num_times_job_run from _timescaledb_internal.bgw_policy_chunk_stats where job_id=:reorder_job_id and chunk_id=:oldest_chunk_id; job_id | chunk_id | num_times_job_run --------+----------+------------------- 1002 | 10 | 1 SELECT reorder_called(:oldest_chunk_id); reorder_called ---------------- t -- Again select cc.chunk_id from _timescaledb_catalog.dimension_slice as ds, _timescaledb_catalog.chunk_constraint as cc where ds.dimension_id=1 and ds.id=cc.dimension_slice_id and cc.chunk_id NOT IN (select chunk_id from _timescaledb_internal.bgw_policy_chunk_stats) ORDER BY ds.range_start LIMIT 1 \gset oldest_ CALL run_job(:reorder_job_id); select job_id, chunk_id, num_times_job_run from _timescaledb_internal.bgw_policy_chunk_stats where job_id=:reorder_job_id and chunk_id=:oldest_chunk_id; job_id | chunk_id | num_times_job_run --------+----------+------------------- 1002 | 11 | 1 SELECT reorder_called(:oldest_chunk_id); reorder_called ---------------- t -- Again select cc.chunk_id from _timescaledb_catalog.dimension_slice as ds, _timescaledb_catalog.chunk_constraint as cc where ds.dimension_id=1 and ds.id=cc.dimension_slice_id and cc.chunk_id NOT IN (select chunk_id from _timescaledb_internal.bgw_policy_chunk_stats) ORDER BY ds.range_start LIMIT 1 \gset oldest_ CALL run_job(:reorder_job_id); select job_id, chunk_id, num_times_job_run from _timescaledb_internal.bgw_policy_chunk_stats where job_id=:reorder_job_id and chunk_id=:oldest_chunk_id; job_id | chunk_id | num_times_job_run --------+----------+------------------- 1002 | 12 | 1 SELECT reorder_called(:oldest_chunk_id); reorder_called ---------------- t -- Ran out of chunks, so should be a noop CALL run_job(:reorder_job_id); NOTICE: no chunks need reordering for hypertable public.test_table -- Corner case: when there are no recent-enough chunks to reorder, -- DO NOT reorder any new chunks created by space partitioning. -- We only want to reorder when new dimension_slices on time are created. INSERT INTO test_table VALUES (now() - INTERVAL '5 months', -5); INSERT INTO test_table VALUES (now() - INTERVAL '3 weeks', -5); INSERT INTO test_table VALUES (now(), -25); -- Should be noop CALL run_job(:reorder_job_id); NOTICE: no chunks need reordering for hypertable public.test_table -- But if we create a new time dimension, reorder it INSERT INTO test_table VALUES (now() - INTERVAL '1 year', 1); select cc.chunk_id from _timescaledb_catalog.dimension_slice as ds, _timescaledb_catalog.chunk_constraint as cc where ds.dimension_id=1 and ds.id=cc.dimension_slice_id and cc.chunk_id NOT IN (select chunk_id from _timescaledb_internal.bgw_policy_chunk_stats) ORDER BY ds.range_start LIMIT 1 \gset oldest_ CALL run_job(:reorder_job_id); select job_id, chunk_id, num_times_job_run from _timescaledb_internal.bgw_policy_chunk_stats where job_id=:reorder_job_id and chunk_id=:oldest_chunk_id; job_id | chunk_id | num_times_job_run --------+----------+------------------- 1002 | 16 | 1 SELECT reorder_called(:oldest_chunk_id); reorder_called ---------------- t -- Should be noop again CALL run_job(:reorder_job_id); NOTICE: no chunks need reordering for hypertable public.test_table CREATE TABLE test_table_int(time bigint, junk int); CREATE TABLE test_overflow_smallint(time smallint, junk int); SELECT create_hypertable('test_table_int', 'time', chunk_time_interval => 1); create_hypertable ----------------------------- (2,public,test_table_int,t) SELECT create_hypertable('test_overflow_smallint', 'time', chunk_time_interval => 1); create_hypertable ------------------------------------- (3,public,test_overflow_smallint,t) create or replace function dummy_now() returns BIGINT LANGUAGE SQL IMMUTABLE as 'SELECT 1::BIGINT'; create or replace function dummy_now2() returns BIGINT LANGUAGE SQL IMMUTABLE as 'SELECT 2::BIGINT'; create or replace function overflow_now() returns SMALLINT LANGUAGE SQL IMMUTABLE as 'SELECT 32767::SMALLINT'; CREATE TABLE test_table_perm(time timestamp PRIMARY KEY); SELECT create_hypertable('test_table_perm', 'time', chunk_time_interval => 1); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices WARNING: unexpected interval: smaller than one second create_hypertable ------------------------------ (4,public,test_table_perm,t) \set ON_ERROR_STOP 0 -- we cannot add a drop_chunks policy on a table whose open dimension is not time and no now_func is set select add_retention_policy('test_table_int', INTERVAL '4 months', true); ERROR: invalid value for parameter drop_after \set ON_ERROR_STOP 1 INSERT INTO test_table_int VALUES (-2, -2), (-1, -1), (0,0), (1, 1), (2, 2), (3, 3); \c :TEST_DBNAME :ROLE_SUPERUSER; CREATE USER unprivileged; \c :TEST_DBNAME unprivileged -- should fail as the user has no permissions on the table \set ON_ERROR_STOP 0 select set_integer_now_func('test_table_int', 'dummy_now'); ERROR: must be owner of hypertable "test_table_int" \set ON_ERROR_STOP 1 \c :TEST_DBNAME :ROLE_SUPERUSER; DROP USER unprivileged; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER select set_integer_now_func('test_table_int', 'dummy_now'); set_integer_now_func ---------------------- select * from test_table_int; time | junk ------+------ -2 | -2 -1 | -1 0 | 0 1 | 1 2 | 2 3 | 3 SELECT count(*) FROM _timescaledb_catalog.chunk as c, _timescaledb_catalog.hypertable as ht where c.hypertable_id = ht.id and ht.table_name='test_table_int'; count ------- 6 select add_retention_policy('test_table_int', 1, true) as drop_chunks_job_id \gset -- Now simulate drop_chunks running automatically by calling it explicitly CALL run_job(:drop_chunks_job_id); select * from test_table_int; time | junk ------+------ 0 | 0 1 | 1 2 | 2 3 | 3 -- Should have 4 chunks left SELECT count(*) FROM _timescaledb_catalog.chunk as c, _timescaledb_catalog.hypertable as ht where c.hypertable_id = ht.id and ht.table_name='test_table_int' \gset before_ select :before_count=4; ?column? ---------- t -- Make sure this second call does nothing CALL run_job(:drop_chunks_job_id); SELECT count(*) FROM _timescaledb_catalog.chunk as c, _timescaledb_catalog.hypertable as ht where c.hypertable_id = ht.id and ht.table_name='test_table_int' \gset after_ -- Should be true select :before_count=:after_count; ?column? ---------- t INSERT INTO test_table_int VALUES (42, 42); SELECT count(*) FROM _timescaledb_catalog.chunk as c, _timescaledb_catalog.hypertable as ht where c.hypertable_id = ht.id and ht.table_name='test_table_int' \gset before_ -- This call should also do nothing CALL run_job(:drop_chunks_job_id); SELECT count(*) FROM _timescaledb_catalog.chunk as c, _timescaledb_catalog.hypertable as ht where c.hypertable_id = ht.id and ht.table_name='test_table_int' \gset after_ -- Should be true select :before_count=:after_count; ?column? ---------- t INSERT INTO test_table_int VALUES (-1, -1); SELECT count(*) FROM _timescaledb_catalog.chunk as c, _timescaledb_catalog.hypertable as ht where c.hypertable_id = ht.id and ht.table_name='test_table_int' \gset add_one_ select :before_count+1=:add_one_count; ?column? ---------- t CALL run_job(:drop_chunks_job_id); SELECT count(*) FROM _timescaledb_catalog.chunk as c, _timescaledb_catalog.hypertable as ht where c.hypertable_id = ht.id and ht.table_name='test_table_int' \gset after_ -- (-1,-1) was in droping range so it should be dropped by background job select :before_count=:after_count; ?column? ---------- t select set_integer_now_func('test_table_int', 'dummy_now2', replace_if_exists=>true); set_integer_now_func ---------------------- select * from test_table_int; time | junk ------+------ 0 | 0 1 | 1 2 | 2 3 | 3 42 | 42 CALL run_job(:drop_chunks_job_id); -- added one to now() so time entry with value 0 should be dropped now SELECT count(*) FROM _timescaledb_catalog.chunk as c, _timescaledb_catalog.hypertable as ht where c.hypertable_id = ht.id and ht.table_name='test_table_int' \gset after_ select :before_count=:after_count+1; ?column? ---------- t select * from test_table_int; time | junk ------+------ 1 | 1 2 | 2 3 | 3 42 | 42 -- make the now() function invalid -- returns INT and not BIGINT drop function dummy_now2(); create or replace function dummy_now2() returns INT LANGUAGE SQL IMMUTABLE as 'SELECT 2::INT'; \set ON_ERROR_STOP 0 CALL run_job(:drop_chunks_job_id); ERROR: invalid integer_now function \set ON_ERROR_STOP 1 -- test the expected use case of set_integer_now_func create function nowstamp() returns bigint language sql STABLE as 'SELECT extract(epoch from now())::BIGINT'; select set_integer_now_func('test_table_int', 'nowstamp', replace_if_exists=>true); set_integer_now_func ---------------------- CALL run_job(:drop_chunks_job_id); SELECT count(*) FROM _timescaledb_catalog.chunk as c, _timescaledb_catalog.hypertable as ht where c.hypertable_id = ht.id and ht.table_name='test_table_int' \gset after_ select :after_count=0; ?column? ---------- t -- test the case when now()-interval overflows select set_integer_now_func('test_overflow_smallint', 'overflow_now'); set_integer_now_func ---------------------- select add_retention_policy('test_overflow_smallint', -2) as drop_chunks_job_id \gset \set ON_ERROR_STOP 0 CALL run_job(:drop_chunks_job_id); ERROR: integer time overflow \set ON_ERROR_STOP 1 -- test the case when partitioning function and now function are set. create table part_time_now_func(time float8, temp float8); create or replace function time_partfunc(unixtime float8) returns bigint language plpgsql immutable as $body$ declare retval bigint; begin retval := unixtime::bigint; raise notice 'time value for % is %', unixtime, retval; return retval; end $body$; create or replace function dummy_now() returns bigint language sql immutable as 'select 2::bigint'; select create_hypertable('part_time_now_func', 'time', time_partitioning_func => 'time_partfunc', chunk_time_interval=>1); create_hypertable --------------------------------- (5,public,part_time_now_func,t) insert into part_time_now_func values (1.1, 23.4), (2.2, 22.3), (3.3, 42.3); NOTICE: time value for 1.1 is 1 NOTICE: time value for 1.1 is 1 NOTICE: time value for 1.1 is 1 NOTICE: time value for 1.1 is 1 NOTICE: time value for 2.2 is 2 NOTICE: time value for 2.2 is 2 NOTICE: time value for 2.2 is 2 NOTICE: time value for 2.2 is 2 NOTICE: time value for 3.3 is 3 NOTICE: time value for 3.3 is 3 NOTICE: time value for 3.3 is 3 NOTICE: time value for 3.3 is 3 select * from part_time_now_func; time | temp ------+------ 1.1 | 23.4 2.2 | 22.3 3.3 | 42.3 select set_integer_now_func('part_time_now_func', 'dummy_now'); set_integer_now_func ---------------------- select add_retention_policy('part_time_now_func', 0) as drop_chunks_job_id \gset CALL run_job(:drop_chunks_job_id); select * from part_time_now_func; time | temp ------+------ 2.2 | 22.3 3.3 | 42.3 select remove_retention_policy('part_time_now_func'); remove_retention_policy ------------------------- \c :TEST_DBNAME :ROLE_SUPERUSER alter function dummy_now() rename to dummy_now_renamed; alter schema public rename to new_public; select * from _timescaledb_catalog.dimension; id | hypertable_id | column_name | column_type | aligned | num_slices | partitioning_func_schema | partitioning_func | interval_length | compress_interval_length | integer_now_func_schema | integer_now_func ----+---------------+-------------+-----------------------------+---------+------------+--------------------------+--------------------+-----------------+--------------------------+-------------------------+------------------ 1 | 1 | time | timestamp with time zone | t | | | | 604800000000 | | | 2 | 1 | chunk_id | integer | f | 2 | _timescaledb_functions | get_partition_hash | | | | 5 | 4 | time | timestamp without time zone | t | | | | 1 | | | 6 | 5 | time | double precision | t | | new_public | time_partfunc | 1 | | new_public | dummy_now 3 | 2 | time | bigint | t | | | | 1 | | new_public | nowstamp 4 | 3 | time | smallint | t | | | | 1 | | new_public | overflow_now alter schema new_public rename to public; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 -- test that the behavior is strict when providing NULL required arguments create table test_strict (time timestamptz not null, a int, b int); select create_hypertable('test_strict', 'time'); create_hypertable -------------------------- (6,public,test_strict,t) \set ON_ERROR_STOP 0 select add_reorder_policy('test_table_perm', 'test_table_perm_pkey'); ERROR: must be owner of hypertable "test_table_perm" select remove_reorder_policy('test_table'); ERROR: must be owner of hypertable "test_table" select add_retention_policy('test_table_perm', INTERVAL '4 months', true); ERROR: must be owner of hypertable "test_table_perm" select remove_retention_policy('test_table'); ERROR: must be owner of hypertable "test_table" select add_retention_policy('test_strict', drop_after => NULL); ERROR: need to specify one of "drop_after" or "drop_created_before" \set ON_ERROR_STOP 1 -- Check the number of non-internal policies SELECT proc_name, count(*) FROM _timescaledb_catalog.bgw_job WHERE id >= 1000 GROUP BY proc_name; proc_name | count ------------------+------- policy_reorder | 1 policy_retention | 2 -- test retention with null arguments select add_retention_policy(NULL, NULL); add_retention_policy ---------------------- select add_retention_policy(NULL, drop_after => interval '2 days'); add_retention_policy ---------------------- -- this is an optional argument select add_retention_policy('test_strict', drop_after => interval '2 days', if_not_exists => NULL); add_retention_policy ---------------------- select add_retention_policy('test_strict', interval '2 days', schedule_interval => NULL); add_retention_policy ---------------------- 1006 -- test compression with null arguments alter table test_strict set (timescaledb.compress); select add_compression_policy(NULL, compress_after => NULL); add_compression_policy ------------------------ select add_compression_policy('test_strict', INTERVAL '2 weeks', if_not_exists => NULL); add_compression_policy ------------------------ select add_compression_policy('test_strict', INTERVAL '2 weeks', schedule_interval => NULL); add_compression_policy ------------------------ 1007 -- test that we get the default schedule_interval if nothing is specified create table test_missing_schedint (time timestamptz not null, a int, b int); select create_hypertable('test_missing_schedint', 'time', chunk_time_interval=> '31days'::interval); create_hypertable ------------------------------------ (8,public,test_missing_schedint,t) -- we expect shedule_interval to be 1 day select add_retention_policy('test_missing_schedint', interval '2 weeks') as retenion_id_missing_schedint \gset -- we expect schedule_interval to be chunk_time_interval/2 for timestamptz time alter table test_missing_schedint set (timescaledb.compress); select add_compression_policy('test_missing_schedint', interval '60 days') as compression_id_missing_schedint \gset -- we expect schedule_interval to be 1 day for int time create table test_missing_schedint_integer (time int not null, a int, b int); -- 10 days interval select create_hypertable('test_missing_schedint_integer', 'time', chunk_time_interval => 864000000); create_hypertable --------------------------------------------- (10,public,test_missing_schedint_integer,t) alter table test_missing_schedint_integer set (timescaledb.compress); select add_compression_policy('test_missing_schedint_integer', BIGINT '600000') as compression_id_integer \gset select * from _timescaledb_catalog.bgw_job where id in (:retenion_id_missing_schedint, :compression_id_missing_schedint, :compression_id_integer); id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ------+---------------------------+-------------------+-------------+-------------+--------------+------------------------+--------------------+---------------------+-----------+----------------+---------------+---------------+-----------------------------------------------------+------------------------+--------------------------+---------- 1008 | Retention Policy [1008] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_functions | policy_retention | default_perm_user_2 | t | f | | 8 | {"drop_after": "@ 14 days", "hypertable_id": 8} | _timescaledb_functions | policy_retention_check | 1009 | Columnstore Policy [1009] | @ 12 hours | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | default_perm_user_2 | t | f | | 8 | {"hypertable_id": 8, "compress_after": "@ 60 days"} | _timescaledb_functions | policy_compression_check | 1010 | Columnstore Policy [1010] | @ 1 day | @ 0 | -1 | @ 1 hour | _timescaledb_functions | policy_compression | default_perm_user_2 | t | f | | 10 | {"hypertable_id": 10, "compress_after": 600000} | _timescaledb_functions | policy_compression_check | -- test policy check functions with NULL args \set ON_ERROR_STOP 0 select add_compression_policy('test_strict', compress_after => NULL); ERROR: need to specify one of "compress_after" or "compress_created_before" SELECT _timescaledb_functions.policy_compression_check(NULL); ERROR: config must not be NULL SELECT _timescaledb_functions.policy_refresh_continuous_aggregate_check(NULL); ERROR: config must not be NULL SELECT _timescaledb_functions.policy_reorder_check(NULL); ERROR: config must not be NULL SELECT _timescaledb_functions.policy_retention_check(NULL); ERROR: config must not be NULL \set ON_ERROR_STOP 1 --TEST check if alias for bgw_job table works SELECT * from _timescaledb_config.bgw_job WHERE id = :retenion_id_missing_schedint ORDER BY id; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ------+-------------------------+-------------------+-------------+-------------+--------------+------------------------+------------------+---------------------+-----------+----------------+---------------+---------------+-------------------------------------------------+------------------------+------------------------+---------- 1008 | Retention Policy [1008] | @ 1 day | @ 5 mins | -1 | @ 5 mins | _timescaledb_functions | policy_retention | default_perm_user_2 | t | f | | 8 | {"drop_after": "@ 14 days", "hypertable_id": 8} | _timescaledb_functions | policy_retention_check | ================================================ FILE: tsl/test/expected/bgw_reorder_drop_chunks.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- -- Setup -- \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(timeout INT = -1) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_run(timeout INT = -1, mock_start_time INT = 0) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_wait_for_scheduler_finish() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_create() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_destroy() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_reset_time(set_time BIGINT = 0, wait BOOLEAN = false) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; \set WAIT_ON_JOB 0 \set IMMEDIATELY_SET_UNTIL 1 \set WAIT_FOR_OTHER_TO_ADVANCE 2 -- Remove any default jobs, e.g., telemetry DELETE FROM _timescaledb_catalog.bgw_job; TRUNCATE _timescaledb_internal.bgw_job_stat; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE public.bgw_log( msg_no INT, mock_time BIGINT, application_name TEXT, msg TEXT ); CREATE VIEW sorted_bgw_log AS SELECT msg_no, mock_time, application_name, regexp_replace(CASE WHEN length(msg) > 80 THEN substring(msg, 1, 80) || '...' ELSE msg END, '(execution time) [0-9]+(\.[0-9]+)?', '\1 (RANDOM)', 'g') AS msg FROM bgw_log ORDER BY mock_time, application_name COLLATE "C", msg_no; CREATE TABLE public.bgw_dsm_handle_store( handle BIGINT ); INSERT INTO public.bgw_dsm_handle_store VALUES (0); SELECT ts_bgw_params_create(); ts_bgw_params_create ---------------------- SELECT * FROM _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- SELECT * FROM timescaledb_information.job_stats; hypertable_schema | hypertable_name | job_id | last_run_started_at | last_successful_finish | last_run_status | job_status | last_run_duration | next_start | total_runs | total_successes | total_failures -------------------+-----------------+--------+---------------------+------------------------+-----------------+------------+-------------------+------------+------------+-----------------+---------------- \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER ------------------------------ -- test reorder policy runs -- ------------------------------ CREATE TABLE test_reorder_table(time int, chunk_id int); SELECT create_hypertable('test_reorder_table', 'time', chunk_time_interval => 1); create_hypertable --------------------------------- (1,public,test_reorder_table,t) -- These inserts should create 5 different chunks INSERT INTO test_reorder_table VALUES (1, 1); INSERT INTO test_reorder_table VALUES (2, 2); INSERT INTO test_reorder_table VALUES (3, 3); INSERT INTO test_reorder_table VALUES (4, 4); INSERT INTO test_reorder_table VALUES (5, 5); SELECT COUNT(*) FROM _timescaledb_catalog.chunk as c, _timescaledb_catalog.hypertable as ht where c.hypertable_id = ht.id and ht.table_name='test_reorder_table'; count ------- 5 SELECT count(*) FROM _timescaledb_catalog.bgw_job WHERE proc_schema = '_timescaledb_functions' AND proc_name = 'policy_reorder'; count ------- 0 select add_reorder_policy('test_reorder_table', 'test_reorder_table_time_idx') as reorder_job_id \gset SELECT count(*) FROM _timescaledb_catalog.bgw_job WHERE proc_schema = '_timescaledb_functions' AND proc_name = 'policy_reorder'; count ------- 1 -- job was created SELECT * FROM timescaledb_information.jobs WHERE job_id=:reorder_job_id; job_id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | config | next_start | initial_start | hypertable_schema | hypertable_name | check_schema | check_name --------+-----------------------+-------------------+-------------+-------------+--------------+------------------------+----------------+-------------------+-----------+----------------+-------------------------------------------------------------------+------------+---------------+-------------------+--------------------+------------------------+---------------------- 1000 | Reorder Policy [1000] | @ 4 days | @ 0 | -1 | @ 5 mins | _timescaledb_functions | policy_reorder | default_perm_user | t | f | {"index_name": "test_reorder_table_time_idx", "hypertable_id": 1} | | | public | test_reorder_table | _timescaledb_functions | policy_reorder_check -- no stats SELECT job_id, next_start, last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat ORDER BY job_id; job_id | next_start | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------+------------+------------------+------------+-----------------+----------------+--------------- -- nothing clustered SELECT indexrelid::regclass, indisclustered FROM pg_index WHERE indisclustered = true ORDER BY 1; indexrelid | indisclustered ------------+---------------- -- run first time SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+------------------+-------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until 25000, started at 0 SELECT * FROM timescaledb_information.jobs WHERE job_id=:reorder_job_id; job_id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | config | next_start | initial_start | hypertable_schema | hypertable_name | check_schema | check_name --------+-----------------------+-------------------+-------------+-------------+--------------+------------------------+----------------+-------------------+-----------+----------------+-------------------------------------------------------------------+------------------------------+---------------+-------------------+--------------------+------------------------+---------------------- 1000 | Reorder Policy [1000] | @ 4 days | @ 0 | -1 | @ 5 mins | _timescaledb_functions | policy_reorder | default_perm_user | t | f | {"index_name": "test_reorder_table_time_idx", "hypertable_id": 1} | Fri Dec 31 16:00:00 1999 PST | | public | test_reorder_table | _timescaledb_functions | policy_reorder_check -- job ran once, successfully SELECT job_id, next_start, last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:reorder_job_id; job_id | next_start | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------------------+------------------------------+------------------+------------+-----------------+----------------+--------------- 1000 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | t | 1 | 1 | 0 | 0 -- first chunk reordered SELECT indexrelid::regclass, indisclustered FROM pg_index WHERE indisclustered = true ORDER BY 1; indexrelid | indisclustered --------------------------------------------------------------------+---------------- _timescaledb_internal._hyper_1_1_chunk_test_reorder_table_time_idx | t -- second call to scheduler should immediately run reorder again, due to catchup SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+------------------+---------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until 25000, started at 0 0 | 25000 | DB Scheduler | [TESTING] Registered new background worker 1 | 25000 | DB Scheduler | [TESTING] Wait until 50000, started at 25000 SELECT * FROM timescaledb_information.jobs WHERE job_id=:reorder_job_id; job_id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | config | next_start | initial_start | hypertable_schema | hypertable_name | check_schema | check_name --------+-----------------------+-------------------+-------------+-------------+--------------+------------------------+----------------+-------------------+-----------+----------------+-------------------------------------------------------------------+----------------------------------+---------------+-------------------+--------------------+------------------------+---------------------- 1000 | Reorder Policy [1000] | @ 4 days | @ 0 | -1 | @ 5 mins | _timescaledb_functions | policy_reorder | default_perm_user | t | f | {"index_name": "test_reorder_table_time_idx", "hypertable_id": 1} | Fri Dec 31 16:00:00.025 1999 PST | | public | test_reorder_table | _timescaledb_functions | policy_reorder_check -- two runs SELECT job_id, next_start, last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:reorder_job_id; job_id | next_start | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+----------------------------------+----------------------------------+------------------+------------+-----------------+----------------+--------------- 1000 | Fri Dec 31 16:00:00.025 1999 PST | Fri Dec 31 16:00:00.025 1999 PST | t | 2 | 2 | 0 | 0 -- two chunks clustered SELECT indexrelid::regclass, indisclustered FROM pg_index WHERE indisclustered = true ORDER BY 1; indexrelid | indisclustered --------------------------------------------------------------------+---------------- _timescaledb_internal._hyper_1_1_chunk_test_reorder_table_time_idx | t _timescaledb_internal._hyper_1_2_chunk_test_reorder_table_time_idx | t -- third call to scheduler should immediately run reorder again, due to catchup SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(50); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ -- job info is gone SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+------------------+----------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until 25000, started at 0 0 | 25000 | DB Scheduler | [TESTING] Registered new background worker 1 | 25000 | DB Scheduler | [TESTING] Wait until 50000, started at 25000 0 | 50000 | DB Scheduler | [TESTING] Registered new background worker 1 | 50000 | DB Scheduler | [TESTING] Wait until 100000, started at 50000 SELECT * FROM timescaledb_information.jobs WHERE job_id=:reorder_job_id; job_id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | config | next_start | initial_start | hypertable_schema | hypertable_name | check_schema | check_name --------+-----------------------+-------------------+-------------+-------------+--------------+------------------------+----------------+-------------------+-----------+----------------+-------------------------------------------------------------------+---------------------------------+---------------+-------------------+--------------------+------------------------+---------------------- 1000 | Reorder Policy [1000] | @ 4 days | @ 0 | -1 | @ 5 mins | _timescaledb_functions | policy_reorder | default_perm_user | t | f | {"index_name": "test_reorder_table_time_idx", "hypertable_id": 1} | Tue Jan 04 16:00:00.05 2000 PST | | public | test_reorder_table | _timescaledb_functions | policy_reorder_check SELECT * FROM _timescaledb_internal.bgw_job_stat where job_id=:reorder_job_id; job_id | last_start | last_finish | next_start | last_successful_finish | last_run_success | total_runs | total_duration | total_duration_failures | total_successes | total_failures | total_crashes | consecutive_failures | consecutive_crashes | flags --------+---------------------------------+---------------------------------+---------------------------------+---------------------------------+------------------+------------+----------------+-------------------------+-----------------+----------------+---------------+----------------------+---------------------+------- 1000 | Fri Dec 31 16:00:00.05 1999 PST | Fri Dec 31 16:00:00.05 1999 PST | Tue Jan 04 16:00:00.05 2000 PST | Fri Dec 31 16:00:00.05 1999 PST | t | 3 | @ 0 | @ 0 | 3 | 0 | 0 | 0 | 0 | 0 -- three chunks clustered SELECT indexrelid::regclass, indisclustered FROM pg_index WHERE indisclustered = true ORDER BY 1; indexrelid | indisclustered --------------------------------------------------------------------+---------------- _timescaledb_internal._hyper_1_1_chunk_test_reorder_table_time_idx | t _timescaledb_internal._hyper_1_2_chunk_test_reorder_table_time_idx | t _timescaledb_internal._hyper_1_3_chunk_test_reorder_table_time_idx | t -- running is a nop SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(100); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+------------------+------------------------------------------------ 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until 25000, started at 0 0 | 25000 | DB Scheduler | [TESTING] Registered new background worker 1 | 25000 | DB Scheduler | [TESTING] Wait until 50000, started at 25000 0 | 50000 | DB Scheduler | [TESTING] Registered new background worker 1 | 50000 | DB Scheduler | [TESTING] Wait until 100000, started at 50000 0 | 100000 | DB Scheduler | [TESTING] Wait until 200000, started at 100000 SELECT * FROM timescaledb_information.jobs WHERE job_id=:reorder_job_id; job_id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | config | next_start | initial_start | hypertable_schema | hypertable_name | check_schema | check_name --------+-----------------------+-------------------+-------------+-------------+--------------+------------------------+----------------+-------------------+-----------+----------------+-------------------------------------------------------------------+---------------------------------+---------------+-------------------+--------------------+------------------------+---------------------- 1000 | Reorder Policy [1000] | @ 4 days | @ 0 | -1 | @ 5 mins | _timescaledb_functions | policy_reorder | default_perm_user | t | f | {"index_name": "test_reorder_table_time_idx", "hypertable_id": 1} | Tue Jan 04 16:00:00.05 2000 PST | | public | test_reorder_table | _timescaledb_functions | policy_reorder_check SELECT * FROM _timescaledb_internal.bgw_job_stat where job_id=:reorder_job_id; job_id | last_start | last_finish | next_start | last_successful_finish | last_run_success | total_runs | total_duration | total_duration_failures | total_successes | total_failures | total_crashes | consecutive_failures | consecutive_crashes | flags --------+---------------------------------+---------------------------------+---------------------------------+---------------------------------+------------------+------------+----------------+-------------------------+-----------------+----------------+---------------+----------------------+---------------------+------- 1000 | Fri Dec 31 16:00:00.05 1999 PST | Fri Dec 31 16:00:00.05 1999 PST | Tue Jan 04 16:00:00.05 2000 PST | Fri Dec 31 16:00:00.05 1999 PST | t | 3 | @ 0 | @ 0 | 3 | 0 | 0 | 0 | 0 | 0 -- still have 3 chunks clustered SELECT indexrelid::regclass, indisclustered FROM pg_index WHERE indisclustered = true ORDER BY 1; indexrelid | indisclustered --------------------------------------------------------------------+---------------- _timescaledb_internal._hyper_1_1_chunk_test_reorder_table_time_idx | t _timescaledb_internal._hyper_1_2_chunk_test_reorder_table_time_idx | t _timescaledb_internal._hyper_1_3_chunk_test_reorder_table_time_idx | t --check that views work correctly SELECT * FROM timescaledb_information.job_stats; hypertable_schema | hypertable_name | job_id | last_run_started_at | last_successful_finish | last_run_status | job_status | last_run_duration | next_start | total_runs | total_successes | total_failures -------------------+--------------------+--------+---------------------------------+---------------------------------+-----------------+------------+-------------------+---------------------------------+------------+-----------------+---------------- public | test_reorder_table | 1000 | Fri Dec 31 16:00:00.05 1999 PST | Fri Dec 31 16:00:00.05 1999 PST | Success | Scheduled | | Tue Jan 04 16:00:00.05 2000 PST | 3 | 3 | 0 -- test deleting the policy SELECT remove_reorder_policy('test_reorder_table'); remove_reorder_policy ----------------------- SELECT * FROM timescaledb_information.jobs WHERE job_id=:reorder_job_id; job_id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | config | next_start | initial_start | hypertable_schema | hypertable_name | check_schema | check_name --------+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+--------+------------+---------------+-------------------+-----------------+--------------+------------ SELECT job_id, next_start, last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:reorder_job_id; job_id | next_start | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------+------------+------------------+------------+-----------------+----------------+--------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(125); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+------------------+------------------------------------------------ 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until 25000, started at 0 0 | 25000 | DB Scheduler | [TESTING] Registered new background worker 1 | 25000 | DB Scheduler | [TESTING] Wait until 50000, started at 25000 0 | 50000 | DB Scheduler | [TESTING] Registered new background worker 1 | 50000 | DB Scheduler | [TESTING] Wait until 100000, started at 50000 0 | 100000 | DB Scheduler | [TESTING] Wait until 200000, started at 100000 0 | 200000 | DB Scheduler | [TESTING] Wait until 325000, started at 200000 SELECT * FROM timescaledb_information.jobs WHERE job_id=:reorder_job_id; job_id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | config | next_start | initial_start | hypertable_schema | hypertable_name | check_schema | check_name --------+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+--------+------------+---------------+-------------------+-----------------+--------------+------------ -- still only 3 chunks clustered SELECT indexrelid::regclass, indisclustered FROM pg_index WHERE indisclustered = true ORDER BY 1; indexrelid | indisclustered --------------------------------------------------------------------+---------------- _timescaledb_internal._hyper_1_1_chunk_test_reorder_table_time_idx | t _timescaledb_internal._hyper_1_2_chunk_test_reorder_table_time_idx | t _timescaledb_internal._hyper_1_3_chunk_test_reorder_table_time_idx | t \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; TRUNCATE _timescaledb_internal.bgw_job_stat; DELETE FROM _timescaledb_catalog.bgw_job; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER ----------------------------------- -- test drop chunks policy runs -- ----------------------------------- CREATE TABLE test_drop_chunks_table(time timestamptz, drop_order int); SELECT create_hypertable('test_drop_chunks_table', 'time', chunk_time_interval => INTERVAL '1 week'); create_hypertable ------------------------------------- (2,public,test_drop_chunks_table,t) -- These inserts should create 5 different chunks INSERT INTO test_drop_chunks_table VALUES (now() - INTERVAL '2 month', 4); INSERT INTO test_drop_chunks_table VALUES (now(), 5); INSERT INTO test_drop_chunks_table VALUES (now() - INTERVAL '6 months', 2); INSERT INTO test_drop_chunks_table VALUES (now() - INTERVAL '4 months', 3); INSERT INTO test_drop_chunks_table VALUES (now() - INTERVAL '8 months', 1); SELECT show_chunks('test_drop_chunks_table'); show_chunks ----------------------------------------- _timescaledb_internal._hyper_2_6_chunk _timescaledb_internal._hyper_2_7_chunk _timescaledb_internal._hyper_2_8_chunk _timescaledb_internal._hyper_2_9_chunk _timescaledb_internal._hyper_2_10_chunk SELECT COUNT(*) FROM _timescaledb_catalog.chunk as c, _timescaledb_catalog.hypertable as ht where c.hypertable_id = ht.id and ht.table_name='test_drop_chunks_table'; count ------- 5 SELECT count(*) FROM _timescaledb_catalog.bgw_job WHERE proc_schema = '_timescaledb_functions' AND proc_name = 'policy_retention'; count ------- 0 SELECT add_retention_policy('test_drop_chunks_table', INTERVAL '4 months') as drop_chunks_job_id \gset SELECT count(*) FROM _timescaledb_catalog.bgw_job WHERE proc_schema = '_timescaledb_functions' AND proc_name = 'policy_retention'; count ------- 1 SELECT alter_job(:drop_chunks_job_id, schedule_interval => INTERVAL '1 second'); alter_job ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ (1001,"@ 1 sec","@ 5 mins",-1,"@ 5 mins",t,"{""drop_after"": ""@ 4 mons"", ""hypertable_id"": 2}",-infinity,_timescaledb_functions.policy_retention_check,f,,,"Retention Policy [1001]") SELECT * FROM timescaledb_information.jobs WHERE job_id=:drop_chunks_job_id; job_id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | config | next_start | initial_start | hypertable_schema | hypertable_name | check_schema | check_name --------+-------------------------+-------------------+-------------+-------------+--------------+------------------------+------------------+-------------------+-----------+----------------+------------------------------------------------+------------+---------------+-------------------+------------------------+------------------------+------------------------ 1001 | Retention Policy [1001] | @ 1 sec | @ 5 mins | -1 | @ 5 mins | _timescaledb_functions | policy_retention | default_perm_user | t | f | {"drop_after": "@ 4 mons", "hypertable_id": 2} | | | public | test_drop_chunks_table | _timescaledb_functions | policy_retention_check -- no stats SELECT job_id, next_start, last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat WHERE job_id=:drop_chunks_job_id; job_id | next_start | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------+------------+------------------+------------+-----------------+----------------+--------------- -- all chunks are there SELECT show_chunks('test_drop_chunks_table'); show_chunks ----------------------------------------- _timescaledb_internal._hyper_2_6_chunk _timescaledb_internal._hyper_2_7_chunk _timescaledb_internal._hyper_2_8_chunk _timescaledb_internal._hyper_2_9_chunk _timescaledb_internal._hyper_2_10_chunk -- run first time SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+------------------+-------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until 25000, started at 0 SELECT * FROM timescaledb_information.jobs WHERE job_id=:drop_chunks_job_id; job_id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | config | next_start | initial_start | hypertable_schema | hypertable_name | check_schema | check_name --------+-------------------------+-------------------+-------------+-------------+--------------+------------------------+------------------+-------------------+-----------+----------------+------------------------------------------------+------------------------------+---------------+-------------------+------------------------+------------------------+------------------------ 1001 | Retention Policy [1001] | @ 1 sec | @ 5 mins | -1 | @ 5 mins | _timescaledb_functions | policy_retention | default_perm_user | t | f | {"drop_after": "@ 4 mons", "hypertable_id": 2} | Fri Dec 31 16:00:01 1999 PST | | public | test_drop_chunks_table | _timescaledb_functions | policy_retention_check -- job ran once, successfully SELECT job_id, time_bucket('1m',next_start) AS next_start, time_bucket('1m',last_finish) as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:drop_chunks_job_id; job_id | next_start | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------------------+------------------------------+------------------+------------+-----------------+----------------+--------------- 1001 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | t | 1 | 1 | 0 | 0 -- chunks 8 and 10 dropped SELECT show_chunks('test_drop_chunks_table'); show_chunks ---------------------------------------- _timescaledb_internal._hyper_2_6_chunk _timescaledb_internal._hyper_2_7_chunk _timescaledb_internal._hyper_2_9_chunk -- job doesn't run again immediately SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+------------------+---------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until 25000, started at 0 0 | 25000 | DB Scheduler | [TESTING] Wait until 50000, started at 25000 SELECT * FROM timescaledb_information.jobs WHERE job_id=:drop_chunks_job_id; job_id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | config | next_start | initial_start | hypertable_schema | hypertable_name | check_schema | check_name --------+-------------------------+-------------------+-------------+-------------+--------------+------------------------+------------------+-------------------+-----------+----------------+------------------------------------------------+------------------------------+---------------+-------------------+------------------------+------------------------+------------------------ 1001 | Retention Policy [1001] | @ 1 sec | @ 5 mins | -1 | @ 5 mins | _timescaledb_functions | policy_retention | default_perm_user | t | f | {"drop_after": "@ 4 mons", "hypertable_id": 2} | Fri Dec 31 16:00:01 1999 PST | | public | test_drop_chunks_table | _timescaledb_functions | policy_retention_check -- still only 1 run SELECT job_id, time_bucket('1m',next_start) AS next_start, time_bucket('1m',last_finish) as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:drop_chunks_job_id; job_id | next_start | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------------------+------------------------------+------------------+------------+-----------------+----------------+--------------- 1001 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | t | 1 | 1 | 0 | 0 -- same chunks SELECT show_chunks('test_drop_chunks_table'); show_chunks ---------------------------------------- _timescaledb_internal._hyper_2_6_chunk _timescaledb_internal._hyper_2_7_chunk _timescaledb_internal._hyper_2_9_chunk -- a new chunk older than the drop date will be dropped INSERT INTO test_drop_chunks_table VALUES (now() - INTERVAL '12 months', 0); SELECT show_chunks('test_drop_chunks_table'); show_chunks ----------------------------------------- _timescaledb_internal._hyper_2_6_chunk _timescaledb_internal._hyper_2_7_chunk _timescaledb_internal._hyper_2_9_chunk _timescaledb_internal._hyper_2_11_chunk SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(10000); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+------------------+--------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until 25000, started at 0 0 | 25000 | DB Scheduler | [TESTING] Wait until 50000, started at 25000 0 | 50000 | DB Scheduler | [TESTING] Wait until 1000000, started at 50000 1 | 1000000 | DB Scheduler | [TESTING] Registered new background worker 2 | 1000000 | DB Scheduler | [TESTING] Wait until 10050000, started at 1000000 SELECT * FROM timescaledb_information.jobs WHERE job_id=:drop_chunks_job_id; job_id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | config | next_start | initial_start | hypertable_schema | hypertable_name | check_schema | check_name --------+-------------------------+-------------------+-------------+-------------+--------------+------------------------+------------------+-------------------+-----------+----------------+------------------------------------------------+------------------------------+---------------+-------------------+------------------------+------------------------+------------------------ 1001 | Retention Policy [1001] | @ 1 sec | @ 5 mins | -1 | @ 5 mins | _timescaledb_functions | policy_retention | default_perm_user | t | f | {"drop_after": "@ 4 mons", "hypertable_id": 2} | Fri Dec 31 16:00:02 1999 PST | | public | test_drop_chunks_table | _timescaledb_functions | policy_retention_check -- 2 runs SELECT job_id, time_bucket('1m',next_start) AS next_start, time_bucket('1m',last_finish) as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:drop_chunks_job_id; job_id | next_start | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------------------+------------------------------+------------------+------------+-----------------+----------------+--------------- 1001 | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | t | 2 | 2 | 0 | 0 SELECT show_chunks('test_drop_chunks_table'); show_chunks ---------------------------------------- _timescaledb_internal._hyper_2_6_chunk _timescaledb_internal._hyper_2_7_chunk _timescaledb_internal._hyper_2_9_chunk --test that views work SELECT * FROM timescaledb_information.job_stats; hypertable_schema | hypertable_name | job_id | last_run_started_at | last_successful_finish | last_run_status | job_status | last_run_duration | next_start | total_runs | total_successes | total_failures -------------------+------------------------+--------+------------------------------+------------------------------+-----------------+------------+-------------------+------------------------------+------------+-----------------+---------------- public | test_drop_chunks_table | 1001 | Fri Dec 31 16:00:01 1999 PST | Fri Dec 31 16:00:01 1999 PST | Success | Scheduled | | Fri Dec 31 16:00:02 1999 PST | 2 | 2 | 0 -- Test that add_retention_policy also works with timestamp (without time zone) and date types -- and that the policy execution is being logged -- Test for date CREATE TABLE test_drop_chunks_table_date(time date, drop_order int); SELECT create_hypertable('test_drop_chunks_table_date', 'time', chunk_time_interval => INTERVAL '1 week'); create_hypertable ------------------------------------------ (3,public,test_drop_chunks_table_date,t) INSERT INTO test_drop_chunks_table_date VALUES (now() - INTERVAL '2 month', 4); INSERT INTO test_drop_chunks_table_date VALUES (now(), 5); INSERT INTO test_drop_chunks_table_date VALUES (now() - INTERVAL '6 months', 2); INSERT INTO test_drop_chunks_table_date VALUES (now() - INTERVAL '4 months', 3); INSERT INTO test_drop_chunks_table_date VALUES (now() - INTERVAL '8 months', 1); -- Clear the job stats and reset timer, this will also clear the bgw_log \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; TRUNCATE _timescaledb_internal.bgw_job_stat; DELETE FROM _timescaledb_catalog.bgw_job; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- Test for timestamp CREATE TABLE test_drop_chunks_table_tsntz(time date, drop_order int); SELECT create_hypertable('test_drop_chunks_table_tsntz', 'time', chunk_time_interval => INTERVAL '1 week'); create_hypertable ------------------------------------------- (4,public,test_drop_chunks_table_tsntz,t) INSERT INTO test_drop_chunks_table_tsntz VALUES (now() - INTERVAL '2 month', 4); INSERT INTO test_drop_chunks_table_tsntz VALUES (now(), 5); INSERT INTO test_drop_chunks_table_tsntz VALUES (now() - INTERVAL '6 months', 2); INSERT INTO test_drop_chunks_table_tsntz VALUES (now() - INTERVAL '4 months', 3); INSERT INTO test_drop_chunks_table_tsntz VALUES (now() - INTERVAL '8 months', 1); -- Add retention policies for both tables SELECT add_retention_policy('test_drop_chunks_table_date', INTERVAL '4 months') as drop_chunks_date_job_id \gset SELECT add_retention_policy('test_drop_chunks_table_tsntz', INTERVAL '4 months') as drop_chunks_tsntz_job_id \gset -- Test that retention policy is being logged SELECT alter_job(id,config:=jsonb_set(config,'{verbose_log}', 'true')) FROM _timescaledb_catalog.bgw_job WHERE id = :drop_chunks_date_job_id; alter_job ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1002,"@ 1 day","@ 5 mins",-1,"@ 5 mins",t,"{""drop_after"": ""@ 4 mons"", ""verbose_log"": true, ""hypertable_id"": 3}",-infinity,_timescaledb_functions.policy_retention_check,f,,,"Retention Policy [1002]") SELECT alter_job(id,config:=jsonb_set(config,'{verbose_log}', 'true')) FROM _timescaledb_catalog.bgw_job WHERE id = :drop_chunks_tsntz_job_id; alter_job ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1003,"@ 1 day","@ 5 mins",-1,"@ 5 mins",t,"{""drop_after"": ""@ 4 mons"", ""verbose_log"": true, ""hypertable_id"": 4}",-infinity,_timescaledb_functions.policy_retention_check,f,,,"Retention Policy [1003]") SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(1000); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+-------------------------+------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Registered new background worker 2 | 0 | DB Scheduler | [TESTING] Wait until 1000000, started at 0 0 | 0 | Retention Policy [1002] | applying retention policy to hypertable "test_drop_chunks_table_date": dropping ... 0 | 0 | Retention Policy [1003] | applying retention policy to hypertable "test_drop_chunks_table_tsntz": dropping... -- test the schedule_interval parameter for policies CREATE TABLE test_schedint(time timestamptz, a int, b int); select create_hypertable('test_schedint', 'time'); create_hypertable ---------------------------- (5,public,test_schedint,t) insert into test_schedint values (now(), 1, 2), (now() + interval '2 seconds', 2, 3); -- test the retention policy select add_retention_policy('test_schedint', interval '2 months', schedule_interval => '30 seconds') as polret_schedint \gset -- wait for a bit more than "schedule_interval" seconds, then verify the policy has run twice select ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(1000); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ select total_runs, total_successes, total_failures from timescaledb_information.job_stats where job_id = :polret_schedint; total_runs | total_successes | total_failures ------------+-----------------+---------------- 1 | 1 | 0 select ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(30000); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ select total_runs, total_successes, total_failures from timescaledb_information.job_stats where job_id = :polret_schedint; total_runs | total_successes | total_failures ------------+-----------------+---------------- 2 | 2 | 0 -- if we wait another 30s, we should see 3 runs of the job select ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(30000); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ select total_runs, total_successes, total_failures from timescaledb_information.job_stats where job_id = :polret_schedint; total_runs | total_successes | total_failures ------------+-----------------+---------------- 3 | 3 | 0 -- test the compression policy alter table test_schedint set (timescaledb.compress); select add_compression_policy('test_schedint', interval '3 weeks', schedule_interval => '40 seconds') as polcomp_schedint \gset select ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(1000); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ select total_runs, total_successes, total_failures from timescaledb_information.job_stats where job_id = :polcomp_schedint; total_runs | total_successes | total_failures ------------+-----------------+---------------- 1 | 1 | 0 select ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(40000); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ select total_runs, total_successes, total_failures from timescaledb_information.job_stats where job_id = :polcomp_schedint; total_runs | total_successes | total_failures ------------+-----------------+---------------- 2 | 2 | 0 -- if we wait another 40s, we should see 3 runs of the job select ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(40000); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ select total_runs, total_successes, total_failures from timescaledb_information.job_stats where job_id = :polcomp_schedint; total_runs | total_successes | total_failures ------------+-----------------+---------------- 3 | 3 | 0 ================================================ FILE: tsl/test/expected/bgw_scheduler_control.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE FUNCTION ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(INT, INT) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE FUNCTION ts_bgw_params_create() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE FUNCTION ts_bgw_params_destroy() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE FUNCTION ts_bgw_params_reset_time(set_time BIGINT, wait BOOLEAN) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; ALTER DATABASE :TEST_DBNAME OWNER TO :ROLE_DEFAULT_PERM_USER; GRANT EXECUTE ON FUNCTION pg_reload_conf TO :ROLE_DEFAULT_PERM_USER; GRANT ALTER SYSTEM, SET ON PARAMETER timescaledb.bgw_log_level TO :ROLE_DEFAULT_PERM_USER; -- These are needed to set up the test scheduler CREATE TABLE public.bgw_dsm_handle_store(handle BIGINT); INSERT INTO public.bgw_dsm_handle_store VALUES (0); SELECT ts_bgw_params_create(); ts_bgw_params_create ---------------------- -- Test scheduler automatically writes to this table by name, so -- create it. CREATE TABLE public.bgw_log( msg_no INT, mock_time BIGINT, application_name TEXT, msg TEXT ); CREATE VIEW cleaned_bgw_log AS SELECT msg_no, application_name, regexp_replace(regexp_replace(msg, '(Wait until|started at|execution time|database) [0-9]+(\.[0-9]+)?', '\1 (RANDOM)', 'g'), 'background worker "[^"]+"','connection') AS msg FROM bgw_log ORDER BY mock_time, application_name COLLATE "C", msg_no; -- Remove all default jobs DELETE FROM _timescaledb_catalog.bgw_job WHERE TRUE; TRUNCATE _timescaledb_internal.bgw_job_stat; -- -- Set bgw log level and reload config. -- -- Debug messages should be in log now which it wasn't before. -- -- We change user to make sure that granting SET and ALTER SYSTEM -- privileges to the default user actually works. SET ROLE :ROLE_DEFAULT_PERM_USER; ALTER DATABASE :TEST_DBNAME SET timescaledb.bgw_log_level = 'DEBUG1'; SELECT pg_reload_conf(); pg_reload_conf ---------------- t RESET ROLE; SELECT ts_bgw_params_reset_time(0, false); ts_bgw_params_reset_time -------------------------- INSERT INTO _timescaledb_catalog.bgw_job( application_name, schedule_interval, max_runtime, max_retries, retry_period, proc_schema, proc_name, owner, scheduled, fixed_schedule ) VALUES ( 'test_job_1b', --application_name INTERVAL '100ms', --schedule_interval INTERVAL '100s', --max_runtime 5, --max_retries INTERVAL '1s', --retry_period 'public', --proc_schema 'bgw_test_job_1', --proc_name CURRENT_ROLE::regrole, --owner TRUE, --scheduled FALSE --fixed_schedule ) RETURNING id AS job_id \gset SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25, 0); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM cleaned_bgw_log; msg_no | application_name | msg --------+------------------+------------------------------------------------------------------------- 0 | DB Scheduler | extension state changed: unknown to created 1 | DB Scheduler | database scheduler for database (RANDOM) starting 2 | DB Scheduler | launching job 1000 "test_job_1b" 3 | DB Scheduler | [TESTING] Registered new background worker 4 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | test_job_1b | Execute job 1 1 | test_job_1b | job 1000 (test_job_1b) exiting with success: execution time (RANDOM) ms 5 | DB Scheduler | scheduler for database (RANDOM) exiting with exit status 0 -- We test that we can set it to FATAL, which removed LOG level -- entries from the log. ALTER DATABASE :TEST_DBNAME SET timescaledb.bgw_log_level = 'FATAL'; SELECT pg_reload_conf(); pg_reload_conf ---------------- t \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; SELECT ts_bgw_params_reset_time(0, false); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25, 0); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM cleaned_bgw_log; msg_no | application_name | msg --------+------------------+----- -- We test that we can set it to ERROR. ALTER DATABASE :TEST_DBNAME SET timescaledb.bgw_log_level = 'ERROR'; SELECT pg_reload_conf(); pg_reload_conf ---------------- t \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; SELECT ts_bgw_params_reset_time(0, false); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25, 0); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM cleaned_bgw_log; msg_no | application_name | msg --------+------------------+----- -- Reset the log level and check that normal entries are showing up -- again. ALTER DATABASE :TEST_DBNAME RESET timescaledb.bgw_log_level; SELECT pg_reload_conf(); pg_reload_conf ---------------- t \c :TEST_DBNAME :ROLE_SUPERUSER TRUNCATE bgw_log; SELECT ts_bgw_params_reset_time(0, false); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25, 0); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM cleaned_bgw_log; msg_no | application_name | msg --------+------------------+---------------------------------------------------- 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) SELECT delete_job(:job_id); delete_job ------------ SET ROLE :ROLE_DEFAULT_PERM_USER; -- Make sure we can set the variable using ALTER SYSTEM using the -- previous grants. We don't bother about checking that it has an -- effect here since we already knows it works from the above code. ALTER SYSTEM SET timescaledb.bgw_log_level TO 'DEBUG2'; ALTER SYSTEM RESET timescaledb.bgw_log_level; ================================================ FILE: tsl/test/expected/bgw_scheduler_restart.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE VIEW tsdb_bgw AS SELECT datname, pid, backend_type, application_name FROM pg_stat_activity WHERE backend_type LIKE '%TimescaleDB%' ORDER BY datname, backend_type, application_name; -- Wait for at least one background worker matching pattern to have -- started. CREATE PROCEDURE wait_for_some_started( min_time double precision, timeout double precision, pattern text ) AS $$ DECLARE backend_count int; BEGIN FOR i IN 0..(timeout / min_time)::int LOOP PERFORM pg_sleep(min_time); SELECT count(*) INTO backend_count FROM tsdb_bgw WHERE backend_type LIKE pattern; IF backend_count > 0 THEN RETURN; END IF; END LOOP; RAISE EXCEPTION 'backend matching % did not start before timeout', pattern; END; $$ LANGUAGE plpgsql; -- Wait for the number of background workers matching pattern to be -- zero. CREATE PROCEDURE wait_for_all_stopped( min_time double precision, timeout double precision, pattern text ) AS $$ DECLARE backend_count int; BEGIN FOR i IN 0..(timeout / min_time)::int LOOP PERFORM pg_sleep(min_time); SELECT count(*) INTO backend_count FROM tsdb_bgw WHERE backend_type LIKE pattern; IF backend_count = 0 THEN RETURN; END IF; END LOOP; RAISE EXCEPTION 'backend matching % did not start before timeout', pattern; END; $$ LANGUAGE plpgsql; CREATE PROCEDURE ts_terminate_launcher() AS $$ SELECT pg_terminate_backend(pid) FROM tsdb_bgw WHERE backend_type LIKE '%Launcher%'; $$ LANGUAGE SQL; -- Show the default scheduler restart time SHOW timescaledb.bgw_scheduler_restart_time; timescaledb.bgw_scheduler_restart_time ---------------------------------------- -1 -- Test that it cannot be set to something between -1 and 10 \set ON_ERROR_STOP 0 \set VERBOSITY default ALTER SYSTEM SET timescaledb.bgw_scheduler_restart_time TO '5s'; ERROR: invalid value for parameter "timescaledb.bgw_scheduler_restart_time": 5 DETAIL: Scheduler restart time must be be either -1 or at least 10 seconds. \set VERBOSITY terse \set ON_ERROR_STOP 1 -- Set scheduler restart time to a lower value to make the test a -- little faster. ALTER SYSTEM SET timescaledb.bgw_scheduler_restart_time TO '10s'; ALTER SYSTEM SET timescaledb.debug_bgw_scheduler_exit_status TO 1; SELECT pg_reload_conf(); pg_reload_conf ---------------- t -- Reconnect and check the restart time to make sure that it is -- correct. \c :TEST_DBNAME :ROLE_SUPERUSER SHOW timescaledb.bgw_scheduler_restart_time; timescaledb.bgw_scheduler_restart_time ---------------------------------------- 10s -- Launcher is running, so we need to restart it for the scheduler -- restart time to take effect. SELECT datname, application_name FROM tsdb_bgw; datname | application_name ---------+---------------------------------------- | TimescaleDB Background Worker Launcher CALL ts_terminate_launcher(); -- It will restart automatically, but we wait for it to start. CALL wait_for_some_started(1, 50, '%Launcher%'); -- Make sure that the new value of the scheduler restart time is -- correct or the rest of the tests will fail. SHOW timescaledb.bgw_scheduler_restart_time; timescaledb.bgw_scheduler_restart_time ---------------------------------------- 10s -- Verify that launcher is running. If it is not, the rest of the test -- will fail. SELECT datname, application_name FROM tsdb_bgw; datname | application_name --------------------------+----------------------------------------- db_bgw_scheduler_restart | TimescaleDB Background Worker Scheduler | TimescaleDB Background Worker Launcher -- Now we can start the background workers. SELECT _timescaledb_functions.start_background_workers(); start_background_workers -------------------------- t -- They should start immediately, but let's wait for them to start. CALL wait_for_some_started(1, 50, '%Scheduler%'); -- Check that the schedulers are running. If they are not, the rest of -- the test is meaningless. SELECT datname, application_name FROM tsdb_bgw; datname | application_name --------------------------+----------------------------------------- db_bgw_scheduler_restart | TimescaleDB Background Worker Scheduler | TimescaleDB Background Worker Launcher -- Kill the schedulers and check that they restart. SELECT pg_terminate_backend(pid) FROM tsdb_bgw WHERE datname = :'TEST_DBNAME' AND backend_type LIKE '%Scheduler%'; pg_terminate_backend ---------------------- t -- Wait for scheduler to exit, they should exit immediately. CALL wait_for_all_stopped(1, 50, '%Scheduler%'); -- Check that the schedulers really exited. SELECT datname, application_name FROM tsdb_bgw; datname | application_name ---------+---------------------------------------- | TimescaleDB Background Worker Launcher -- Wait for scheduler to restart. CALL wait_for_some_started(10, 100, '%Scheduler%'); -- Make sure that the launcher and schedulers are running. Otherwise -- the test will fail. SELECT datname, application_name FROM tsdb_bgw; datname | application_name --------------------------+----------------------------------------- db_bgw_scheduler_restart | TimescaleDB Background Worker Scheduler | TimescaleDB Background Worker Launcher -- Now, we had a previous bug where killing the launcher at this point -- would leave the schedulers running (because the launcher did not -- have a handle for them) and when launcher is restarting, it would -- start more schedulers, leaving two schedulers per database. -- Get the PID of the launcher to be able to compare it after the restart SELECT pid AS orig_pid FROM tsdb_bgw WHERE backend_type LIKE '%Launcher%' \gset -- Kill the launcher. Since there are new restarted schedulers, the -- handle could not be used to terminate them, and they would be left -- running. CALL ts_terminate_launcher(); -- Launcher will restart immediately, but we wait one second to give -- it a chance to start. CALL wait_for_some_started(1, 50, '%Launcher%'); -- Check that the launcher is running and that there are exactly one -- scheduler per database. Here the old schedulers are killed, so it -- will be schedulers with a different PID than the ones before the -- launcher was killed, but we are not showing this here. SELECT (pid != :orig_pid) AS different_pid, datname, application_name FROM tsdb_bgw; different_pid | datname | application_name ---------------+--------------------------+----------------------------------------- t | db_bgw_scheduler_restart | TimescaleDB Background Worker Scheduler t | | TimescaleDB Background Worker Launcher ALTER SYSTEM RESET timescaledb.bgw_scheduler_restart_time; ALTER SYSTEM RESET timescaledb.debug_bgw_scheduler_exit_status; SELECT pg_reload_conf(); pg_reload_conf ---------------- t SELECT _timescaledb_functions.stop_background_workers(); stop_background_workers ------------------------- t -- We need to restart the launcher as well to read the reset -- configuration or it will affect other tests. CALL ts_terminate_launcher(); ================================================ FILE: tsl/test/expected/bgw_security.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set ROLE_ADMIN :TEST_DBNAME _admin \c :TEST_DBNAME :ROLE_SUPERUSER CREATE ROLE :ROLE_ADMIN LOGIN; GRANT :ROLE_ADMIN TO :ROLE_DEFAULT_PERM_USER; \c :TEST_DBNAME :ROLE_SUPERUSER CREATE TABLE custom_log (ts integer, msg text); GRANT ALL ON custom_log TO PUBLIC; CREATE PROCEDURE custom_job(integer, jsonb) AS $$ INSERT INTO custom_log values($1, 'custom_job'); $$ LANGUAGE SQL; SET ROLE :ROLE_ADMIN; SELECT add_job('custom_job', '1h') AS job_id \gset RESET ROLE; SELECT id, proc_name, owner FROM _timescaledb_catalog.bgw_job WHERE id = :job_id; id | proc_name | owner ------+------------+----------------------- 1000 | custom_job | db_bgw_security_admin \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 -- We should fail to execute and delete the job since we do not own it -- or belong to the group that owns it. \set ON_ERROR_STOP 0 CALL run_job(:job_id); ERROR: insufficient permissions to run job 1000 SELECT delete_job(:job_id); ERROR: insufficient permissions to delete job owned by "db_bgw_security_admin" \set ON_ERROR_STOP 1 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER -- This should succeed since the role belongs to the job owner group. CALL run_job(:job_id); -- This should succeed since we belong to the owners role. SELECT delete_job(:job_id); delete_job ------------ \c :TEST_DBNAME :ROLE_SUPERUSER DROP ROLE :ROLE_ADMIN; ================================================ FILE: tsl/test/expected/bgw_telemetry.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER -- Check that we can use run_job() with the telemetry job, so first -- locate the job id for it (should be 1, but who knows, and it is not -- important for this test). SELECT id AS job_id FROM _timescaledb_catalog.bgw_job WHERE proc_schema = '_timescaledb_functions' AND proc_name = 'policy_telemetry' \gset -- It should be possible to run it twice and running it should change -- the last_finish time. Since job_stats can be empty to start with, -- we run it once first to populate job_stats. CALL run_job(:job_id); SELECT last_finish AS last_finish FROM _timescaledb_internal.bgw_job_stat WHERE job_id = :job_id \gset SELECT pg_sleep(1); pg_sleep ---------- CALL run_job(:job_id); SELECT last_finish > :'last_finish' AS job_executed, last_run_success FROM _timescaledb_internal.bgw_job_stat WHERE job_id = :job_id; job_executed | last_run_success --------------+------------------ t | f -- Running it as the default user should fail since they do not own -- the job. This should be the case also for the telemetry job, which -- is a little special. \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER \set ON_ERROR_STOP 0 CALL run_job(:job_id); ERROR: insufficient permissions to run job 1 \set ON_ERROR_STOP 1 ================================================ FILE: tsl/test/expected/cagg-15.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- initialize the bgw mock state to prevent the materialization workers from running \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION ts_bgw_params_create() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION test.continuous_aggs_find_view(cagg REGCLASS) RETURNS VOID AS :TSL_MODULE_PATHNAME, 'ts_test_continuous_agg_find_by_view_name' LANGUAGE C; \set WAIT_ON_JOB 0 \set IMMEDIATELY_SET_UNTIL 1 \set WAIT_FOR_OTHER_TO_ADVANCE 2 -- remove any default jobs, e.g., telemetry so bgw_job isn't polluted DELETE FROM _timescaledb_catalog.bgw_job; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT * FROM _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- --TEST1 --- --basic test with count create table foo (a integer, b integer, c integer); select table_name from create_hypertable('foo', 'a', chunk_time_interval=> 10); table_name ------------ foo insert into foo values( 3 , 16 , 20); insert into foo values( 1 , 10 , 20); insert into foo values( 1 , 11 , 20); insert into foo values( 1 , 12 , 20); insert into foo values( 1 , 13 , 20); insert into foo values( 1 , 14 , 20); insert into foo values( 2 , 14 , 20); insert into foo values( 2 , 15 , 20); insert into foo values( 2 , 16 , 20); CREATE OR REPLACE FUNCTION integer_now_foo() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(a), 0) FROM foo $$; SELECT set_integer_now_func('foo', 'integer_now_foo'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW mat_m1(a, countb) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select a, count(b) from foo group by time_bucket(1, a), a WITH NO DATA; SELECT add_continuous_aggregate_policy('mat_m1', NULL, 2::integer, '12 h'::interval) AS job_id \gset SELECT * FROM _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ------+--------------------------------------------+-------------------+-------------+-------------+--------------+------------------------+-------------------------------------+-------------------+-----------+----------------+---------------+---------------+-----------------------------------------------------------------+------------------------+-------------------------------------------+---------- 1000 | Refresh Continuous Aggregate Policy [1000] | @ 12 hours | @ 0 | -1 | @ 12 hours | _timescaledb_functions | policy_refresh_continuous_aggregate | default_perm_user | t | f | | 2 | {"end_offset": 2, "start_offset": null, "mat_hypertable_id": 2} | _timescaledb_functions | policy_refresh_continuous_aggregate_check | SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select a, count(b), time_bucket(1, a) from foo group by time_bucket(1, a) , a ; select * from mat_m1 order by a ; a | countb ---+-------- 1 | 5 2 | 3 3 | 1 --check triggers on user hypertable -- SET ROLE :ROLE_SUPERUSER; select tgname, tgtype, tgenabled , relname from pg_trigger, pg_class where tgrelid = pg_class.oid and pg_class.relname like 'foo' order by tgname; tgname | tgtype | tgenabled | relname --------+--------+-----------+--------- SET ROLE :ROLE_DEFAULT_PERM_USER; -- TEST2 --- DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to table _timescaledb_internal._hyper_2_2_chunk SHOW enable_partitionwise_aggregate; enable_partitionwise_aggregate -------------------------------- off SET enable_partitionwise_aggregate = on; SELECT * FROM _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions insert into conditions values ( '2010-01-01 09:00:00-08', 'SFO', 55, 45); insert into conditions values ( '2010-01-02 09:00:00-08', 'por', 100, 100); insert into conditions values ( '2010-01-02 09:00:00-08', 'SFO', 65, 45); insert into conditions values ( '2010-01-02 09:00:00-08', 'NYC', 65, 45); insert into conditions values ( '2018-11-01 09:00:00-08', 'NYC', 45, 35); insert into conditions values ( '2018-11-02 09:00:00-08', 'NYC', 35, 15); CREATE MATERIALIZED VIEW mat_m1( timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1day', timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket('1day', timec) WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset -- Materialized hypertable for mat_m1 should not be visible in the -- hypertables view: SELECT hypertable_schema, hypertable_name FROM timescaledb_information.hypertables ORDER BY 1,2; hypertable_schema | hypertable_name -------------------+----------------- public | conditions public | foo SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select time_bucket('1day', timec), min(location), sum(temperature), sum(humidity) from conditions group by time_bucket('1day', timec) ; SET ROLE :ROLE_DEFAULT_PERM_USER; --should have same results -- select timec, minl, sumt, sumh from mat_m1 order by timec; timec | minl | sumt | sumh ------------------------------+------+------+------ Thu Dec 31 16:00:00 2009 PST | SFO | 55 | 45 Fri Jan 01 16:00:00 2010 PST | NYC | 230 | 190 Wed Oct 31 17:00:00 2018 PDT | NYC | 45 | 35 Thu Nov 01 17:00:00 2018 PDT | NYC | 35 | 15 select time_bucket('1day', timec), min(location), sum(temperature), sum(humidity) from conditions group by time_bucket('1day', timec) order by 1; time_bucket | min | sum | sum ------------------------------+-----+-----+----- Thu Dec 31 16:00:00 2009 PST | SFO | 55 | 45 Fri Jan 01 16:00:00 2010 PST | NYC | 230 | 190 Wed Oct 31 17:00:00 2018 PDT | NYC | 45 | 35 Thu Nov 01 17:00:00 2018 PDT | NYC | 35 | 15 SET enable_partitionwise_aggregate = off; -- TEST3 -- -- drop on table conditions should cascade to materialized mat_v1 drop table conditions cascade; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions insert into conditions values ( '2010-01-01 09:00:00-08', 'SFO', 55, 45); insert into conditions values ( '2010-01-02 09:00:00-08', 'por', 100, 100); insert into conditions values ( '2010-01-02 09:00:00-08', 'NYC', 65, 45); insert into conditions values ( '2010-01-02 09:00:00-08', 'SFO', 65, 45); insert into conditions values ( '2010-01-03 09:00:00-08', 'NYC', 45, 55); insert into conditions values ( '2010-01-05 09:00:00-08', 'SFO', 75, 100); insert into conditions values ( '2018-11-01 09:00:00-08', 'NYC', 45, 35); insert into conditions values ( '2018-11-02 09:00:00-08', 'NYC', 35, 15); insert into conditions values ( '2018-11-03 09:00:00-08', 'NYC', 35, 25); CREATE MATERIALIZED VIEW mat_m1( timec, minl, sumth, stddevh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select time_bucket('1week', timec), min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) ; SET ROLE :ROLE_DEFAULT_PERM_USER; --should have same results -- select timec, minl, sumth, stddevh from mat_m1 order by timec; timec | minl | sumth | stddevh ------------------------------+------+-------+------------------ Sun Dec 27 16:00:00 2009 PST | NYC | 620 | 23.8746727726266 Sun Jan 03 16:00:00 2010 PST | SFO | 175 | Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 select time_bucket('1week', timec) , min(location), sum(temperature)+ sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) order by time_bucket('1week', timec); time_bucket | min | ?column? | stddev ------------------------------+-----+----------+------------------ Sun Dec 27 16:00:00 2009 PST | NYC | 620 | 23.8746727726266 Sun Jan 03 16:00:00 2010 PST | SFO | 175 | Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 -- TEST4 -- --materialized view with group by clause + expression in SELECT -- use previous data from conditions --drop only the view. -- apply where clause on result of mat_m1 -- DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW mat_m1( timec, minl, sumth, stddevh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions where location = 'NYC' group by time_bucket('1week', timec) WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select time_bucket('1week', timec), min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions where location = 'NYC' group by time_bucket('1week', timec) ; SET ROLE :ROLE_DEFAULT_PERM_USER; --should have same results -- select timec, minl, sumth, stddevh from mat_m1 where stddevh is not null order by timec; timec | minl | sumth | stddevh ------------------------------+------+-------+------------------ Sun Dec 27 16:00:00 2009 PST | NYC | 210 | 7.07106781186548 Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 select time_bucket('1week', timec) , min(location), sum(temperature)+ sum(humidity), stddev(humidity) from conditions where location = 'NYC' group by time_bucket('1week', timec) order by time_bucket('1week', timec); time_bucket | min | ?column? | stddev ------------------------------+-----+----------+------------------ Sun Dec 27 16:00:00 2009 PST | NYC | 210 | 7.07106781186548 Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 -- TEST5 -- ---------test with having clause ---------------------- DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects create materialized view mat_m1( timec, minl, sumth, stddevh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having stddev(humidity) is not null WITH NO DATA; ; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select time_bucket('1week', timec), min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having stddev(humidity) is not null; SET ROLE :ROLE_DEFAULT_PERM_USER; -- should have same results -- select * from mat_m1 order by sumth; timec | minl | sumth | stddevh ------------------------------+------+-------+------------------ Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 Sun Dec 27 16:00:00 2009 PST | NYC | 620 | 23.8746727726266 select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having stddev(humidity) is not null order by sum(temperature)+sum(humidity); time_bucket | min | ?column? | stddev ------------------------------+-----+----------+------------------ Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 Sun Dec 27 16:00:00 2009 PST | NYC | 620 | 23.8746727726266 -- TEST6 -- --group by with more than 1 group column -- having clause with a mix of columns from select list + others drop table conditions cascade; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp numeric NULL, highp numeric null ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions insert into conditions select generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'POR', 55, 75, 40, 70; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'NYC', 35, 45, 50, 40; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-15 00:00'::timestamp, '1 day'), 'LA', 73, 55, 71, 28; --naming with AS clauses CREATE MATERIALIZED VIEW mat_naming WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec) as bucket, location as loc, sum(temperature)+sum(humidity) as sumth, stddev(humidity) from conditions group by bucket, loc having min(location) >= 'NYC' and avg(temperature) > 20 WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_naming' \gset select attnum , attname from pg_attribute where attnum > 0 and attrelid = (Select oid from pg_class where relname like :'MAT_TABLE_NAME') order by attnum, attname; attnum | attname --------+--------- 1 | bucket 2 | loc 3 | sumth 4 | stddev DROP MATERIALIZED VIEW mat_naming; --naming with default names CREATE MATERIALIZED VIEW mat_naming WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec), location, sum(temperature)+sum(humidity) as sumth, stddev(humidity) from conditions group by 1,2 having min(location) >= 'NYC' and avg(temperature) > 20 WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_naming' \gset select attnum , attname from pg_attribute where attnum > 0 and attrelid = (Select oid from pg_class where relname like :'MAT_TABLE_NAME') order by attnum, attname; attnum | attname --------+------------- 1 | time_bucket 2 | location 3 | sumth 4 | stddev DROP MATERIALIZED VIEW mat_naming; --naming with view col names CREATE MATERIALIZED VIEW mat_naming(bucket, loc, sum_t_h, stdd) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec), location, sum(temperature)+sum(humidity), stddev(humidity) from conditions group by 1,2 having min(location) >= 'NYC' and avg(temperature) > 20 WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_naming' \gset select attnum , attname from pg_attribute where attnum > 0 and attrelid = (Select oid from pg_class where relname like :'MAT_TABLE_NAME') order by attnum, attname; attnum | attname --------+--------- 1 | bucket 2 | loc 3 | sum_t_h 4 | stdd DROP MATERIALIZED VIEW mat_naming; CREATE MATERIALIZED VIEW mat_m1(timec, minl, sumth, stddevh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having min(location) >= 'NYC' and avg(temperature) > 20 WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset select attnum , attname from pg_attribute where attnum > 0 and attrelid = (Select oid from pg_class where relname like :'MAT_TABLE_NAME') order by attnum, attname; attnum | attname --------+--------- 1 | timec 2 | minl 3 | sumth 4 | stddevh SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select time_bucket('1week', timec), min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having min(location) >= 'NYC' and avg(temperature) > 20; SET ROLE :ROLE_DEFAULT_PERM_USER; --should have same results -- select timec, minl, sumth, stddevh from mat_m1 order by timec, minl; timec | minl | sumth | stddevh ------------------------------+------+-------+------------------ Sun Dec 16 16:00:00 2018 PST | NYC | 1470 | 15.5662356498831 Sun Dec 23 16:00:00 2018 PST | NYC | 1470 | 15.5662356498831 Sun Dec 30 16:00:00 2018 PST | NYC | 210 | 21.2132034355964 select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having min(location) >= 'NYC' and avg(temperature) > 20 and avg(lowp) > 10 order by time_bucket('1week', timec), min(location); time_bucket | min | ?column? | stddev ------------------------------+-----+----------+------------------ Sun Dec 16 16:00:00 2018 PST | NYC | 1470 | 15.5662356498831 Sun Dec 23 16:00:00 2018 PST | NYC | 1470 | 15.5662356498831 Sun Dec 30 16:00:00 2018 PST | NYC | 210 | 21.2132034355964 --check view defintion in information views select view_name, view_definition from timescaledb_information.continuous_aggregates where view_name::text like 'mat_m1'; view_name | view_definition -----------+----------------------------------------------------------------------------------------------------------------- mat_m1 | SELECT time_bucket('@ 7 days'::interval, conditions.timec) AS timec, + | min(conditions.location) AS minl, + | (sum(conditions.temperature) + sum(conditions.humidity)) AS sumth, + | stddev(conditions.humidity) AS stddevh + | FROM conditions + | GROUP BY (time_bucket('@ 7 days'::interval, conditions.timec)) + | HAVING ((min(conditions.location) >= 'NYC'::text) AND (avg(conditions.temperature) > (20)::double precision)); --TEST6 -- select from internal view SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select * from :"PART_VIEW_SCHEMA".:"PART_VIEW_NAME"; SET ROLE :ROLE_DEFAULT_PERM_USER; --lets drop the view and check DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to table _timescaledb_internal._hyper_13_24_chunk drop table conditions; CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions insert into conditions select generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'POR', 55, 75, 40, 70, NULL; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'NYC', 35, 45, 50, 40, NULL; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-15 00:00'::timestamp, '1 day'), 'LA', 73, 55, NULL, 28, NULL; SELECT $$ select time_bucket('1week', timec) , min(location) as col1, sum(temperature)+sum(humidity) as col2, stddev(humidity) as col3, min(allnull) as col4 from conditions group by time_bucket('1week', timec) having min(location) >= 'NYC' and avg(temperature) > 20 $$ AS "QUERY" \gset \set ECHO errors psql:include/cont_agg_equal.sql:8: NOTICE: materialized view "mat_test" does not exist, skipping ?column? | count ---------------------------------------------------------------+------- Number of rows different between view and original (expect 0) | 0 SELECT $$ select time_bucket('1week', timec), location, sum(temperature)+sum(humidity) as col2, stddev(humidity) as col3, min(allnull) as col4 from conditions group by location, time_bucket('1week', timec) $$ AS "QUERY" \gset \set ECHO errors psql:include/cont_agg_equal.sql:8: NOTICE: drop cascades to table _timescaledb_internal._hyper_15_34_chunk ?column? | count ---------------------------------------------------------------+------- Number of rows different between view and original (expect 0) | 0 --TEST7 -- drop tests for view and hypertable --DROP tests \set ON_ERROR_STOP 0 SELECT h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA", direct_view_name as "DIR_VIEW_NAME", direct_view_schema as "DIR_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_test' \gset DROP TABLE :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME"; ERROR: cannot drop table _timescaledb_internal._materialized_hypertable_16 because other objects depend on it DROP VIEW :"PART_VIEW_SCHEMA".:"PART_VIEW_NAME"; ERROR: cannot drop the partial/direct view because it is required by a continuous aggregate DROP VIEW :"DIR_VIEW_SCHEMA".:"DIR_VIEW_NAME"; ERROR: cannot drop the partial/direct view because it is required by a continuous aggregate \set ON_ERROR_STOP 1 --catalog entry still there; SELECT count(*) FROM _timescaledb_catalog.continuous_agg ca WHERE user_view_name = 'mat_test'; count ------- 1 --mat table, user_view, direct view and partial view all there select count(*) from pg_class where relname = :'PART_VIEW_NAME'; count ------- 1 select count(*) from pg_class where relname = :'MAT_TABLE_NAME'; count ------- 1 select count(*) from pg_class where relname = :'DIR_VIEW_NAME'; count ------- 1 select count(*) from pg_class where relname = 'mat_test'; count ------- 1 DROP MATERIALIZED VIEW mat_test; NOTICE: drop cascades to 2 other objects --catalog entry should be gone SELECT count(*) FROM _timescaledb_catalog.continuous_agg ca WHERE user_view_name = 'mat_test'; count ------- 0 --mat table, user_view, direct view and partial view all gone select count(*) from pg_class where relname = :'PART_VIEW_NAME'; count ------- 0 select count(*) from pg_class where relname = :'MAT_TABLE_NAME'; count ------- 0 select count(*) from pg_class where relname = :'DIR_VIEW_NAME'; count ------- 0 select count(*) from pg_class where relname = 'mat_test'; count ------- 0 --test dropping raw table DROP TABLE conditions; CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions --no data in hyper table on purpose so that CASCADE is not required because of chunks CREATE MATERIALIZED VIEW mat_drop_test(timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1day', timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket('1day', timec) WITH NO DATA; \set ON_ERROR_STOP 0 DROP TABLE conditions; ERROR: cannot drop table conditions because other objects depend on it \set ON_ERROR_STOP 1 --insert data now insert into conditions select generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'POR', 55, 75, 40, 70, NULL; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'NYC', 35, 45, 50, 40, NULL; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-15 00:00'::timestamp, '1 day'), 'LA', 73, 55, NULL, 28, NULL; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_drop_test' \gset SET client_min_messages TO NOTICE; CALL refresh_continuous_aggregate('mat_drop_test', NULL, NULL); --force invalidation insert into conditions select generate_series('2017-11-01 00:00'::timestamp, '2017-12-15 00:00'::timestamp, '1 day'), 'LA', 73, 55, NULL, 28, NULL; select count(*) from _timescaledb_catalog.continuous_aggs_invalidation_threshold; count ------- 1 select count(*) from _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log; count ------- 8 DROP TABLE conditions CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects --catalog entry should be gone SELECT count(*) FROM _timescaledb_catalog.continuous_agg ca WHERE user_view_name = 'mat_drop_test'; count ------- 0 select count(*) from _timescaledb_catalog.continuous_aggs_invalidation_threshold; count ------- 0 select count(*) from _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log; count ------- 0 select count(*) from _timescaledb_catalog.continuous_aggs_materialization_invalidation_log; count ------- 0 SELECT * FROM _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- --mat table, user_view, and partial view all gone select count(*) from pg_class where relname = :'PART_VIEW_NAME'; count ------- 0 select count(*) from pg_class where relname = :'MAT_TABLE_NAME'; count ------- 0 select count(*) from pg_class where relname = 'mat_drop_test'; count ------- 0 --TEST With options CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions CREATE MATERIALIZED VIEW mat_with_test(timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1day', timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket('1day', timec), location, humidity, temperature WITH NO DATA; SELECT add_continuous_aggregate_policy('mat_with_test', NULL, '5 h'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1001 SELECT alter_job(id, schedule_interval => '1h') FROM _timescaledb_catalog.bgw_job; alter_job ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ (1001,"@ 1 hour","@ 0",-1,"@ 12 hours",t,"{""end_offset"": ""@ 5 hours"", ""start_offset"": null, ""mat_hypertable_id"": 20}",-infinity,_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1001]") SELECT schedule_interval FROM _timescaledb_catalog.bgw_job; schedule_interval ------------------- @ 1 hour SELECT alter_job(id, schedule_interval => '2h') FROM _timescaledb_catalog.bgw_job; alter_job ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1001,"@ 2 hours","@ 0",-1,"@ 12 hours",t,"{""end_offset"": ""@ 5 hours"", ""start_offset"": null, ""mat_hypertable_id"": 20}",-infinity,_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1001]") SELECT schedule_interval FROM _timescaledb_catalog.bgw_job; schedule_interval ------------------- @ 2 hours select indexname, indexdef from pg_indexes where tablename = (SELECT h.table_name FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_with_test') order by indexname; indexname | indexdef ---------------------------------------+---------------------------------------------------------------------------------------------------------------------------------- _materialized_hypertable_20_timec_idx | CREATE INDEX _materialized_hypertable_20_timec_idx ON _timescaledb_internal._materialized_hypertable_20 USING btree (timec DESC) DROP MATERIALIZED VIEW mat_with_test; --no additional indexes CREATE MATERIALIZED VIEW mat_with_test(timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.create_group_indexes=false) as select time_bucket('1day', timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket('1day', timec), location, humidity, temperature WITH NO DATA; select indexname, indexdef from pg_indexes where tablename = (SELECT h.table_name FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_with_test'); indexname | indexdef ---------------------------------------+---------------------------------------------------------------------------------------------------------------------------------- _materialized_hypertable_21_timec_idx | CREATE INDEX _materialized_hypertable_21_timec_idx ON _timescaledb_internal._materialized_hypertable_21 USING btree (timec DESC) DROP TABLE conditions CASCADE; NOTICE: drop cascades to 2 other objects --test WITH using a hypertable with an integer time dimension CREATE TABLE conditions ( timec INT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec', chunk_time_interval=> 100); table_name ------------ conditions CREATE OR REPLACE FUNCTION integer_now_conditions() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now_conditions'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW mat_with_test(timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket(100, timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket(100, timec) WITH NO DATA; SELECT add_continuous_aggregate_policy('mat_with_test', NULL, 500::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1002 SELECT alter_job(id, schedule_interval => '2h') FROM _timescaledb_catalog.bgw_job; alter_job --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1002,"@ 2 hours","@ 0",-1,"@ 12 hours",t,"{""end_offset"": 500, ""start_offset"": null, ""mat_hypertable_id"": 23}",-infinity,_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1002]") SELECT schedule_interval FROM _timescaledb_catalog.bgw_job; schedule_interval ------------------- @ 2 hours DROP TABLE conditions CASCADE; NOTICE: drop cascades to 2 other objects --test space partitions CREATE TABLE space_table ( time BIGINT, dev BIGINT, data BIGINT ); SELECT create_hypertable( 'space_table', 'time', chunk_time_interval => 10, partitioning_column => 'dev', number_partitions => 3); create_hypertable --------------------------- (24,public,space_table,t) CREATE OR REPLACE FUNCTION integer_now_space_table() returns BIGINT LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), BIGINT '0') FROM space_table $$; SELECT set_integer_now_func('space_table', 'integer_now_space_table'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW space_view WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('4', time), COUNT(data) FROM space_table GROUP BY 1 WITH NO DATA; INSERT INTO space_table VALUES (0, 1, 1), (0, 2, 1), (1, 1, 1), (1, 2, 1), (10, 1, 1), (10, 2, 1), (11, 1, 1), (11, 2, 1); SELECT h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA", direct_view_name as "DIR_VIEW_NAME", direct_view_schema as "DIR_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'space_view' \gset SELECT * FROM :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" ORDER BY time_bucket; time_bucket | count -------------+------- CALL refresh_continuous_aggregate('space_view', NULL, NULL); SELECT * FROM space_view ORDER BY 1; time_bucket | count -------------+------- 0 | 4 8 | 4 SELECT * FROM :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" ORDER BY time_bucket; time_bucket | count -------------+------- 0 | 4 8 | 4 INSERT INTO space_table VALUES (3, 2, 1); CALL refresh_continuous_aggregate('space_view', NULL, NULL); SELECT * FROM space_view ORDER BY 1; time_bucket | count -------------+------- 0 | 5 8 | 4 SELECT * FROM :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" ORDER BY time_bucket; time_bucket | count -------------+------- 0 | 5 8 | 4 INSERT INTO space_table VALUES (2, 3, 1); CALL refresh_continuous_aggregate('space_view', NULL, NULL); SELECT * FROM space_view ORDER BY 1; time_bucket | count -------------+------- 0 | 6 8 | 4 SELECT * FROM :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" ORDER BY time_bucket; time_bucket | count -------------+------- 0 | 6 8 | 4 DROP TABLE space_table CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_25_60_chunk -- -- TEST FINALIZEFUNC_EXTRA -- -- create special aggregate to test ffunc_extra -- Raise warning with the actual type being passed in CREATE OR REPLACE FUNCTION fake_ffunc(a int8, b int, c int, d int, x anyelement) RETURNS anyelement AS $$ BEGIN RAISE WARNING 'type % %', pg_typeof(d), pg_typeof(x); RETURN x; END; $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION fake_sfunc(a int8, b int, c int, d int, x anyelement) RETURNS int8 AS $$ BEGIN RETURN b; END; $$ LANGUAGE plpgsql; CREATE AGGREGATE aggregate_to_test_ffunc_extra(int, int, int, anyelement) ( SFUNC = fake_sfunc, STYPE = int8, COMBINEFUNC = int8pl, FINALFUNC = fake_ffunc, PARALLEL = SAFE, FINALFUNC_EXTRA ); CREATE TABLE conditions ( timec INT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec', chunk_time_interval=> 100); table_name ------------ conditions CREATE OR REPLACE FUNCTION integer_now_conditions() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now_conditions'); set_integer_now_func ---------------------- insert into conditions select generate_series(0, 200, 10), 'POR', 55, 75, 40, 70, NULL; CREATE MATERIALIZED VIEW mat_ffunc_test WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket(100, timec), aggregate_to_test_ffunc_extra(timec, 1, 3, 'test'::text) from conditions group by time_bucket(100, timec); NOTICE: refreshing continuous aggregate "mat_ffunc_test" WARNING: type integer text WARNING: type integer text WARNING: type integer text SELECT * FROM mat_ffunc_test ORDER BY time_bucket; time_bucket | aggregate_to_test_ffunc_extra -------------+------------------------------- 0 | 100 | 200 | DROP MATERIALIZED view mat_ffunc_test; NOTICE: drop cascades to table _timescaledb_internal._hyper_27_65_chunk CREATE MATERIALIZED VIEW mat_ffunc_test WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket(100, timec), aggregate_to_test_ffunc_extra(timec, 4, 5, bigint '123') from conditions group by time_bucket(100, timec); NOTICE: refreshing continuous aggregate "mat_ffunc_test" WARNING: type integer bigint WARNING: type integer bigint WARNING: type integer bigint SELECT * FROM mat_ffunc_test ORDER BY time_bucket; time_bucket | aggregate_to_test_ffunc_extra -------------+------------------------------- 0 | 100 | 200 | --refresh mat view test when time_bucket is not projected -- DROP MATERIALIZED VIEW mat_ffunc_test; NOTICE: drop cascades to table _timescaledb_internal._hyper_28_66_chunk CREATE MATERIALIZED VIEW mat_refresh_test WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select location, max(humidity) from conditions group by time_bucket(100, timec), location WITH NO DATA; insert into conditions select generate_series(0, 50, 10), 'NYC', 55, 75, 40, 70, NULL; CALL refresh_continuous_aggregate('mat_refresh_test', NULL, NULL); SELECT * FROM mat_refresh_test order by 1,2 ; location | max ----------+----- NYC | 75 POR | 75 POR | 75 POR | 75 -- test for bug when group by is not in project list CREATE MATERIALIZED VIEW conditions_grpby_view with (timescaledb.continuous, timescaledb.materialized_only=false) as select time_bucket(100, timec), sum(humidity) from conditions group by time_bucket(100, timec), location; NOTICE: refreshing continuous aggregate "conditions_grpby_view" select * from conditions_grpby_view order by 1, 2; time_bucket | sum -------------+----- 0 | 450 0 | 750 100 | 750 200 | 75 CREATE MATERIALIZED VIEW conditions_grpby_view2 with (timescaledb.continuous, timescaledb.materialized_only=false) as select time_bucket(100, timec), sum(humidity) from conditions group by time_bucket(100, timec), location having avg(temperature) > 0; NOTICE: refreshing continuous aggregate "conditions_grpby_view2" select * from conditions_grpby_view2 order by 1, 2; time_bucket | sum -------------+----- 0 | 450 0 | 750 100 | 750 200 | 75 -- Test internal functions for continuous aggregates SELECT test.continuous_aggs_find_view('mat_refresh_test'); continuous_aggs_find_view --------------------------- -- Test pseudotype/enum handling CREATE TYPE status_enum AS ENUM ( 'red', 'yellow', 'green' ); CREATE TABLE cagg_types ( time TIMESTAMPTZ NOT NULL, status status_enum, names NAME[], floats FLOAT[] ); SELECT table_name FROM create_hypertable('cagg_types', 'time'); table_name ------------ cagg_types INSERT INTO cagg_types SELECT '2000-01-01', 'yellow', '{foo,bar,baz}', '{1,2.5,3}'; CREATE MATERIALIZED VIEW mat_types WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1d', time), min(status) AS status, max(names) AS names, min(floats) AS floats FROM cagg_types GROUP BY 1; NOTICE: refreshing continuous aggregate "mat_types" CALL refresh_continuous_aggregate('mat_types',NULL,NULL); NOTICE: continuous aggregate "mat_types" is already up-to-date SELECT * FROM mat_types; time_bucket | status | names | floats ------------------------------+--------+---------------+----------- Fri Dec 31 16:00:00 1999 PST | yellow | {foo,bar,baz} | {1,2.5,3} ------------------------------------------------------------------------------------- -- Test issue #2616 where cagg view contains an experssion with several aggregates in CREATE TABLE water_consumption ( sensor_id integer NOT NULL, timestamp timestamp(0) NOT NULL, water_index integer ); SELECT create_hypertable('water_consumption', 'timestamp', 'sensor_id', 2); WARNING: column type "timestamp without time zone" used for "timestamp" does not follow best practices create_hypertable --------------------------------- (34,public,water_consumption,t) INSERT INTO public.water_consumption (sensor_id, timestamp, water_index) VALUES (1, '2010-11-03 09:42:30', 1030), (1, '2010-11-03 09:42:40', 1032), (1, '2010-11-03 09:42:50', 1035), (1, '2010-11-03 09:43:30', 1040), (1, '2010-11-03 09:43:40', 1045), (1, '2010-11-03 09:43:50', 1050), (1, '2010-11-03 09:44:30', 1052), (1, '2010-11-03 09:44:40', 1057), (1, '2010-11-03 09:44:50', 1060), (1, '2010-11-03 09:45:30', 1063), (1, '2010-11-03 09:45:40', 1067), (1, '2010-11-03 09:45:50', 1070); -- The test with the view originally reported in the issue. CREATE MATERIALIZED VIEW water_consumption_aggregation_minute WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT sensor_id, time_bucket(INTERVAL '1 minute', timestamp) + '1 minute' AS timestamp, (max(water_index) - min(water_index)) AS water_consumption FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) WITH NO DATA; CALL refresh_continuous_aggregate('water_consumption_aggregation_minute', NULL, NULL); -- The results of the view and the query over hypertable should be the same SELECT * FROM water_consumption_aggregation_minute ORDER BY water_consumption; sensor_id | timestamp | water_consumption -----------+--------------------------+------------------- 1 | Wed Nov 03 09:43:00 2010 | 5 1 | Wed Nov 03 09:46:00 2010 | 7 1 | Wed Nov 03 09:45:00 2010 | 8 1 | Wed Nov 03 09:44:00 2010 | 10 SELECT sensor_id, time_bucket(INTERVAL '1 minute', timestamp) + '1 minute' AS timestamp, (max(water_index) - min(water_index)) AS water_consumption FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) ORDER BY water_consumption; sensor_id | timestamp | water_consumption -----------+--------------------------+------------------- 1 | Wed Nov 03 09:43:00 2010 | 5 1 | Wed Nov 03 09:46:00 2010 | 7 1 | Wed Nov 03 09:45:00 2010 | 8 1 | Wed Nov 03 09:44:00 2010 | 10 -- Simplified test, where the view doesn't contain all group by clauses CREATE MATERIALIZED VIEW water_consumption_no_select_bucket WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT sensor_id, (max(water_index) - min(water_index)) AS water_consumption FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) WITH NO DATA; CALL refresh_continuous_aggregate('water_consumption_no_select_bucket', NULL, NULL); -- The results of the view and the query over hypertable should be the same SELECT * FROM water_consumption_no_select_bucket ORDER BY water_consumption; sensor_id | water_consumption -----------+------------------- 1 | 5 1 | 7 1 | 8 1 | 10 SELECT sensor_id, (max(water_index) - min(water_index)) AS water_consumption FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) ORDER BY water_consumption; sensor_id | water_consumption -----------+------------------- 1 | 5 1 | 7 1 | 8 1 | 10 -- The test with SELECT matching GROUP BY and placing aggregate expression not the last CREATE MATERIALIZED VIEW water_consumption_aggregation_no_addition WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT sensor_id, (max(water_index) - min(water_index)) AS water_consumption, time_bucket(INTERVAL '1 minute', timestamp) AS timestamp FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) WITH NO DATA; CALL refresh_continuous_aggregate('water_consumption_aggregation_no_addition', NULL, NULL); -- The results of the view and the query over hypertable should be the same SELECT * FROM water_consumption_aggregation_no_addition ORDER BY water_consumption; sensor_id | water_consumption | timestamp -----------+-------------------+-------------------------- 1 | 5 | Wed Nov 03 09:42:00 2010 1 | 7 | Wed Nov 03 09:45:00 2010 1 | 8 | Wed Nov 03 09:44:00 2010 1 | 10 | Wed Nov 03 09:43:00 2010 SELECT sensor_id, (max(water_index) - min(water_index)) AS water_consumption, time_bucket(INTERVAL '1 minute', timestamp) AS timestamp FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) ORDER BY water_consumption; sensor_id | water_consumption | timestamp -----------+-------------------+-------------------------- 1 | 5 | Wed Nov 03 09:42:00 2010 1 | 7 | Wed Nov 03 09:45:00 2010 1 | 8 | Wed Nov 03 09:44:00 2010 1 | 10 | Wed Nov 03 09:43:00 2010 DROP TABLE water_consumption CASCADE; NOTICE: drop cascades to 6 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_35_73_chunk NOTICE: drop cascades to table _timescaledb_internal._hyper_36_74_chunk NOTICE: drop cascades to table _timescaledb_internal._hyper_37_75_chunk ---- --- github issue 2655 --- create table raw_data(time timestamptz, search_query text, cnt integer, cnt2 integer); select create_hypertable('raw_data','time', chunk_time_interval=>'15 days'::interval); create_hypertable ------------------------ (38,public,raw_data,t) insert into raw_data select '2000-01-01','Q1'; --having has exprs that appear in select CREATE MATERIALIZED VIEW search_query_count_1m WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT search_query,count(search_query) as count, time_bucket(INTERVAL '1 minute', time) AS bucket FROM raw_data WHERE search_query is not null AND LENGTH(TRIM(both from search_query))>0 GROUP BY search_query, bucket HAVING count(search_query) > 3 OR sum(cnt) > 1; NOTICE: refreshing continuous aggregate "search_query_count_1m" --having has aggregates + grp by columns that appear in select CREATE MATERIALIZED VIEW search_query_count_2 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT search_query,count(search_query) as count, sum(cnt), time_bucket(INTERVAL '1 minute', time) AS bucket FROM raw_data WHERE search_query is not null AND LENGTH(TRIM(both from search_query))>0 GROUP BY search_query, bucket HAVING count(search_query) > 3 OR sum(cnt) > 1 OR ( sum(cnt) + count(cnt)) > 1 AND search_query = 'Q1'; NOTICE: refreshing continuous aggregate "search_query_count_2" CREATE MATERIALIZED VIEW search_query_count_3 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT search_query,count(search_query) as count, sum(cnt), time_bucket(INTERVAL '1 minute', time) AS bucket FROM raw_data WHERE search_query is not null AND LENGTH(TRIM(both from search_query))>0 GROUP BY cnt +cnt2 , bucket, search_query HAVING cnt + cnt2 + sum(cnt) > 2 or count(cnt2) > 10; NOTICE: refreshing continuous aggregate "search_query_count_3" insert into raw_data select '2000-01-01 00:00+0','Q1', 1, 100; insert into raw_data select '2000-01-01 00:00+0','Q1', 2, 200; insert into raw_data select '2000-01-01 00:00+0','Q1', 3, 300; insert into raw_data select '2000-01-02 00:00+0','Q2', 10, 10; insert into raw_data select '2000-01-02 00:00+0','Q2', 20, 20; CALL refresh_continuous_aggregate('search_query_count_1m', NULL, NULL); SELECT * FROM search_query_count_1m ORDER BY 1, 2; search_query | count | bucket --------------+-------+------------------------------ Q1 | 3 | Fri Dec 31 16:00:00 1999 PST Q2 | 2 | Sat Jan 01 16:00:00 2000 PST --only 1 of these should appear in the result insert into raw_data select '2000-01-02 00:00+0','Q3', 0, 0; insert into raw_data select '2000-01-03 00:00+0','Q4', 20, 20; CALL refresh_continuous_aggregate('search_query_count_1m', NULL, NULL); SELECT * FROM search_query_count_1m ORDER BY 1, 2; search_query | count | bucket --------------+-------+------------------------------ Q1 | 3 | Fri Dec 31 16:00:00 1999 PST Q2 | 2 | Sat Jan 01 16:00:00 2000 PST Q4 | 1 | Sun Jan 02 16:00:00 2000 PST --refresh search_query_count_2--- CALL refresh_continuous_aggregate('search_query_count_2', NULL, NULL); SELECT * FROM search_query_count_2 ORDER BY 1, 2; search_query | count | sum | bucket --------------+-------+-----+------------------------------ Q1 | 3 | 6 | Fri Dec 31 16:00:00 1999 PST Q2 | 2 | 30 | Sat Jan 01 16:00:00 2000 PST Q4 | 1 | 20 | Sun Jan 02 16:00:00 2000 PST --refresh search_query_count_3--- CALL refresh_continuous_aggregate('search_query_count_3', NULL, NULL); SELECT * FROM search_query_count_3 ORDER BY 1, 2, 3; search_query | count | sum | bucket --------------+-------+-----+------------------------------ Q1 | 1 | 1 | Fri Dec 31 16:00:00 1999 PST Q1 | 1 | 2 | Fri Dec 31 16:00:00 1999 PST Q1 | 1 | 3 | Fri Dec 31 16:00:00 1999 PST Q2 | 1 | 10 | Sat Jan 01 16:00:00 2000 PST Q2 | 1 | 20 | Sat Jan 01 16:00:00 2000 PST Q4 | 1 | 20 | Sun Jan 02 16:00:00 2000 PST --- TEST enable compression on continuous aggregates CREATE VIEW cagg_compression_status as SELECT ca.mat_hypertable_id AS mat_htid, ca.user_view_name AS cagg_name , h.schema_name AS mat_schema_name, h.table_name AS mat_table_name, ca.materialized_only FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) ; SELECT mat_htid AS "MAT_HTID" , mat_schema_name || '.' || mat_table_name AS "MAT_HTNAME" , mat_table_name AS "MAT_TABLE_NAME" FROM cagg_compression_status WHERE cagg_name = 'search_query_count_3' \gset ALTER MATERIALIZED VIEW search_query_count_3 SET (timescaledb.compress = 'true'); NOTICE: defaulting compress_orderby to bucket,search_query SELECT cagg_name, mat_table_name FROM cagg_compression_status where cagg_name = 'search_query_count_3'; cagg_name | mat_table_name ----------------------+----------------------------- search_query_count_3 | _materialized_hypertable_41 \x SELECT * FROM timescaledb_information.compression_settings WHERE hypertable_name = :'MAT_TABLE_NAME'; -[ RECORD 1 ]----------+---------------------------- hypertable_schema | _timescaledb_internal hypertable_name | _materialized_hypertable_41 attname | bucket segmentby_column_index | orderby_column_index | 1 orderby_asc | t orderby_nullsfirst | f -[ RECORD 2 ]----------+---------------------------- hypertable_schema | _timescaledb_internal hypertable_name | _materialized_hypertable_41 attname | search_query segmentby_column_index | orderby_column_index | 2 orderby_asc | t orderby_nullsfirst | f \x SELECT compress_chunk(ch) FROM show_chunks('search_query_count_3') ch; compress_chunk ------------------------------------------ _timescaledb_internal._hyper_41_79_chunk SELECT * from search_query_count_3 ORDER BY 1, 2, 3; search_query | count | sum | bucket --------------+-------+-----+------------------------------ Q1 | 1 | 1 | Fri Dec 31 16:00:00 1999 PST Q1 | 1 | 2 | Fri Dec 31 16:00:00 1999 PST Q1 | 1 | 3 | Fri Dec 31 16:00:00 1999 PST Q2 | 1 | 10 | Sat Jan 01 16:00:00 2000 PST Q2 | 1 | 20 | Sat Jan 01 16:00:00 2000 PST Q4 | 1 | 20 | Sun Jan 02 16:00:00 2000 PST -- insert into a new region of the hypertable and then refresh the cagg -- (note we still do not support refreshes into existing regions. -- cagg chunks do not map 1-1 to hypertabl regions. They encompass -- more data -- ). insert into raw_data select '2000-05-01 00:00+0','Q3', 0, 0; -- On PG >= 14 the refresh test below will pass because we added support for UPDATE/DELETE on compressed chunks in PR #5339 \set ON_ERROR_STOP 0 CALL refresh_continuous_aggregate('search_query_count_3', NULL, '2000-06-01 00:00+0'::timestamptz); CALL refresh_continuous_aggregate('search_query_count_3', '2000-05-01 00:00+0'::timestamptz, '2000-06-01 00:00+0'::timestamptz); NOTICE: continuous aggregate "search_query_count_3" is already up-to-date \set ON_ERROR_STOP 1 --insert row insert into raw_data select '2001-05-10 00:00+0','Q3', 100, 100; --this should succeed since it does not refresh any compressed regions in the cagg CALL refresh_continuous_aggregate('search_query_count_3', '2001-05-01 00:00+0'::timestamptz, '2001-06-01 00:00+0'::timestamptz); --verify watermark and check that chunks are compressed SELECT _timescaledb_functions.to_timestamp(w) FROM _timescaledb_functions.cagg_watermark(:'MAT_HTID') w; to_timestamp ------------------------------ Wed May 09 17:01:00 2001 PDT SELECT chunk_name, range_start, range_end, is_compressed FROM timescaledb_information.chunks WHERE hypertable_name = :'MAT_TABLE_NAME' ORDER BY 1; chunk_name | range_start | range_end | is_compressed --------------------+------------------------------+------------------------------+--------------- _hyper_41_79_chunk | Fri Dec 24 16:00:00 1999 PST | Mon May 22 17:00:00 2000 PDT | t _hyper_41_83_chunk | Sun Mar 18 16:00:00 2001 PST | Wed Aug 15 17:00:00 2001 PDT | f SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id = :'MAT_HTID' ORDER BY 1, 2,3; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- 41 | -9223372036854775808 | -210866803200000001 41 | 959817600000000 | 988675199999999 41 | 991353600000000 | 9223372036854775807 SELECT * from search_query_count_3 WHERE bucket > '2001-01-01' ORDER BY 1, 2, 3; search_query | count | sum | bucket --------------+-------+-----+------------------------------ Q3 | 1 | 100 | Wed May 09 17:00:00 2001 PDT --now disable compression , will error out -- \set ON_ERROR_STOP 0 ALTER MATERIALIZED VIEW search_query_count_3 SET (timescaledb.compress = 'false'); ERROR: cannot disable columnstore on hypertable with columnstore chunks \set ON_ERROR_STOP 1 SELECT decompress_chunk(format('%I.%I', schema_name, table_name)) FROM _timescaledb_catalog.chunk WHERE hypertable_id = :'MAT_HTID' and status = 1; decompress_chunk ------------------------------------------ _timescaledb_internal._hyper_41_79_chunk --disable compression on cagg after decompressing all chunks-- ALTER MATERIALIZED VIEW search_query_count_3 SET (timescaledb.compress = 'false'); SELECT cagg_name, mat_table_name FROM cagg_compression_status where cagg_name = 'search_query_count_3'; cagg_name | mat_table_name ----------------------+----------------------------- search_query_count_3 | _materialized_hypertable_41 SELECT view_name, materialized_only, compression_enabled FROM timescaledb_information.continuous_aggregates where view_name = 'search_query_count_3'; view_name | materialized_only | compression_enabled ----------------------+-------------------+--------------------- search_query_count_3 | f | f -- TEST caggs on table with more columns than in the cagg view defn -- CREATE TABLE test_morecols ( time TIMESTAMPTZ NOT NULL, val1 INTEGER, val2 INTEGER, val3 INTEGER, val4 INTEGER, val5 INTEGER, val6 INTEGER, val7 INTEGER, val8 INTEGER); SELECT create_hypertable('test_morecols', 'time', chunk_time_interval=> '7 days'::interval); create_hypertable ----------------------------- (43,public,test_morecols,t) INSERT INTO test_morecols SELECT generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 55, 75, 40, 70, NULL, 100, 200, 200; CREATE MATERIALIZED VIEW test_morecols_cagg with (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('30 days',time), avg(val1), count(val2) FROM test_morecols GROUP BY 1; NOTICE: refreshing continuous aggregate "test_morecols_cagg" ALTER MATERIALIZED VIEW test_morecols_cagg SET (timescaledb.compress='true'); NOTICE: defaulting compress_orderby to time_bucket SELECT compress_chunk(ch) FROM show_chunks('test_morecols_cagg') ch; compress_chunk ------------------------------------------ _timescaledb_internal._hyper_44_89_chunk SELECT * FROM test_morecols_cagg ORDER BY time_bucket; time_bucket | avg | count ------------------------------+---------------------+------- Fri Nov 23 16:00:00 2018 PST | 55.0000000000000000 | 23 Sun Dec 23 16:00:00 2018 PST | 55.0000000000000000 | 8 SELECT view_name, materialized_only, compression_enabled FROM timescaledb_information.continuous_aggregates where view_name = 'test_morecols_cagg'; view_name | materialized_only | compression_enabled --------------------+-------------------+--------------------- test_morecols_cagg | f | t --should keep compressed option, modify only materialized -- ALTER MATERIALIZED VIEW test_morecols_cagg SET (timescaledb.materialized_only='true'); SELECT view_name, materialized_only, compression_enabled FROM timescaledb_information.continuous_aggregates where view_name = 'test_morecols_cagg'; view_name | materialized_only | compression_enabled --------------------+-------------------+--------------------- test_morecols_cagg | t | t CREATE TABLE issue3248(filler_1 int, filler_2 int, filler_3 int, time timestamptz NOT NULL, device_id int, v0 int, v1 int, v2 float, v3 float); CREATE INDEX ON issue3248(time DESC); CREATE INDEX ON issue3248(device_id,time DESC); SELECT create_hypertable('issue3248','time',create_default_indexes:=false); create_hypertable ------------------------- (46,public,issue3248,t) ALTER TABLE issue3248 DROP COLUMN filler_1; INSERT INTO issue3248(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id+1, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-01 0:00:00+0'::timestamptz,'2000-01-05 23:55:00+0','8h') gtime(time), generate_series(1,5,1) gdevice(device_id); ALTER TABLE issue3248 DROP COLUMN filler_2; INSERT INTO issue3248(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id-1, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-06 0:00:00+0'::timestamptz,'2000-01-12 23:55:00+0','8h') gtime(time), generate_series(1,5,1) gdevice(device_id); ALTER TABLE issue3248 DROP COLUMN filler_3; INSERT INTO issue3248(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-13 0:00:00+0'::timestamptz,'2000-01-19 23:55:00+0','8h') gtime(time), generate_series(1,5,1) gdevice(device_id); ANALYZE issue3248; CREATE materialized view issue3248_cagg WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h',time), device_id, min(v0), max(v1), avg(v2) FROM issue3248 GROUP BY 1,2; NOTICE: refreshing continuous aggregate "issue3248_cagg" SELECT FROM issue3248 AS m, LATERAL(SELECT m FROM issue3248_cagg WHERE avg IS NULL LIMIT 1) AS lat; -- -- test that option create_group_indexes is taken into account CREATE TABLE test_group_idx ( time timestamptz, symbol int, value numeric ); select create_hypertable('test_group_idx', 'time'); create_hypertable ------------------------------ (48,public,test_group_idx,t) insert into test_group_idx select t, round(random()*10), random()*5 from generate_series('2020-01-01', '2020-02-25', INTERVAL '12 hours') t; create materialized view cagg_index_true with (timescaledb.continuous, timescaledb.materialized_only=false, timescaledb.create_group_indexes=true) as select time_bucket('1 day', "time") as bucket, sum(value), symbol from test_group_idx group by bucket, symbol; NOTICE: refreshing continuous aggregate "cagg_index_true" create materialized view cagg_index_false with (timescaledb.continuous, timescaledb.materialized_only=false, timescaledb.create_group_indexes=false) as select time_bucket('1 day', "time") as bucket, sum(value), symbol from test_group_idx group by bucket, symbol; NOTICE: refreshing continuous aggregate "cagg_index_false" create materialized view cagg_index_default with (timescaledb.continuous, timescaledb.materialized_only=false) as select time_bucket('1 day', "time") as bucket, sum(value), symbol from test_group_idx group by bucket, symbol; NOTICE: refreshing continuous aggregate "cagg_index_default" -- see corresponding materialization_hypertables select view_name, materialization_hypertable_name from timescaledb_information.continuous_aggregates ca where view_name like 'cagg_index_%' ORDER BY view_name; view_name | materialization_hypertable_name --------------------+--------------------------------- cagg_index_default | _materialized_hypertable_51 cagg_index_false | _materialized_hypertable_50 cagg_index_true | _materialized_hypertable_49 -- now make sure a group index has been created when explicitly asked for \x on select i.* from pg_indexes i join pg_class c on schemaname = relnamespace::regnamespace::text and tablename = relname where tablename in (select materialization_hypertable_name from timescaledb_information.continuous_aggregates where view_name like 'cagg_index_%') order by tablename, indexname; -[ RECORD 1 ]------------------------------------------------------------------------------------------------------------------------------------------------- schemaname | _timescaledb_internal tablename | _materialized_hypertable_49 indexname | _materialized_hypertable_49_bucket_idx tablespace | indexdef | CREATE INDEX _materialized_hypertable_49_bucket_idx ON _timescaledb_internal._materialized_hypertable_49 USING btree (bucket DESC) -[ RECORD 2 ]------------------------------------------------------------------------------------------------------------------------------------------------- schemaname | _timescaledb_internal tablename | _materialized_hypertable_49 indexname | _materialized_hypertable_49_symbol_bucket_idx tablespace | indexdef | CREATE INDEX _materialized_hypertable_49_symbol_bucket_idx ON _timescaledb_internal._materialized_hypertable_49 USING btree (symbol, bucket DESC) -[ RECORD 3 ]------------------------------------------------------------------------------------------------------------------------------------------------- schemaname | _timescaledb_internal tablename | _materialized_hypertable_50 indexname | _materialized_hypertable_50_bucket_idx tablespace | indexdef | CREATE INDEX _materialized_hypertable_50_bucket_idx ON _timescaledb_internal._materialized_hypertable_50 USING btree (bucket DESC) -[ RECORD 4 ]------------------------------------------------------------------------------------------------------------------------------------------------- schemaname | _timescaledb_internal tablename | _materialized_hypertable_51 indexname | _materialized_hypertable_51_bucket_idx tablespace | indexdef | CREATE INDEX _materialized_hypertable_51_bucket_idx ON _timescaledb_internal._materialized_hypertable_51 USING btree (bucket DESC) -[ RECORD 5 ]------------------------------------------------------------------------------------------------------------------------------------------------- schemaname | _timescaledb_internal tablename | _materialized_hypertable_51 indexname | _materialized_hypertable_51_symbol_bucket_idx tablespace | indexdef | CREATE INDEX _materialized_hypertable_51_symbol_bucket_idx ON _timescaledb_internal._materialized_hypertable_51 USING btree (symbol, bucket DESC) \x off -- -- TESTs for removing old CAggs restrictions -- DROP TABLE conditions CASCADE; NOTICE: drop cascades to 8 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_29_67_chunk NOTICE: drop cascades to table _timescaledb_internal._hyper_30_68_chunk NOTICE: drop cascades to table _timescaledb_internal._hyper_31_69_chunk CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL ); SELECT create_hypertable('conditions', 'timec'); create_hypertable -------------------------- (52,public,conditions,t) INSERT INTO conditions VALUES ('2010-01-01 09:00:00-08', 'SFO', 55, 45), ('2010-01-02 09:00:00-08', 'por', 100, 100), ('2010-01-02 09:00:00-08', 'NYC', 65, 45), ('2010-01-02 09:00:00-08', 'SFO', 65, 45), ('2010-01-03 09:00:00-08', 'NYC', 45, 55), ('2010-01-05 09:00:00-08', 'SFO', 75, 100), ('2018-11-01 09:00:00-08', 'NYC', 45, 35), ('2018-11-02 09:00:00-08', 'NYC', 35, 15), ('2018-11-03 09:00:00-08', 'NYC', 35, 25); -- aggregate with DISTINCT CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), COUNT(location), SUM(DISTINCT temperature) FROM conditions GROUP BY time_bucket('1week', timec), location; NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1, 2, 3; time_bucket | count | sum ------------------------------+-------+----- Sun Dec 27 16:00:00 2009 PST | 1 | 100 Sun Dec 27 16:00:00 2009 PST | 2 | 110 Sun Dec 27 16:00:00 2009 PST | 2 | 120 Sun Jan 03 16:00:00 2010 PST | 1 | 75 Sun Oct 28 17:00:00 2018 PDT | 3 | 80 -- aggregate with FILTER DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), SUM(temperature) FILTER (WHERE humidity > 60) FROM conditions GROUP BY time_bucket('1week', timec), location; NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1, 2; time_bucket | sum ------------------------------+----- Sun Dec 27 16:00:00 2009 PST | 100 Sun Dec 27 16:00:00 2009 PST | Sun Dec 27 16:00:00 2009 PST | Sun Jan 03 16:00:00 2010 PST | 75 Sun Oct 28 17:00:00 2018 PDT | -- aggregate with filter in having clause DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), MAX(temperature) FROM conditions GROUP BY time_bucket('1week', timec), location HAVING SUM(temperature) FILTER (WHERE humidity > 40) > 50; NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1, 2; time_bucket | max ------------------------------+----- Sun Dec 27 16:00:00 2009 PST | 65 Sun Dec 27 16:00:00 2009 PST | 65 Sun Dec 27 16:00:00 2009 PST | 100 Sun Jan 03 16:00:00 2010 PST | 75 -- ordered set aggr DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to table _timescaledb_internal._hyper_55_116_chunk CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), MODE() WITHIN GROUP(ORDER BY humidity) FROM conditions GROUP BY time_bucket('1week', timec); NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1; time_bucket | mode ------------------------------+------ Sun Dec 27 16:00:00 2009 PST | 45 Sun Jan 03 16:00:00 2010 PST | 100 Sun Oct 28 17:00:00 2018 PDT | 15 -- hypothetical-set aggr DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), RANK(60) WITHIN GROUP (ORDER BY humidity), DENSE_RANK(60) WITHIN GROUP (ORDER BY humidity), PERCENT_RANK(60) WITHIN GROUP (ORDER BY humidity) FROM conditions GROUP BY time_bucket('1week', timec); NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1; time_bucket | rank | dense_rank | percent_rank ------------------------------+------+------------+-------------- Sun Dec 27 16:00:00 2009 PST | 5 | 3 | 0.8 Sun Jan 03 16:00:00 2010 PST | 1 | 1 | 0 Sun Oct 28 17:00:00 2018 PDT | 4 | 4 | 1 -- userdefined aggregate without combine function DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE AGGREGATE newavg ( sfunc = int4_avg_accum, basetype = int4, stype = _int8, finalfunc = int8_avg, initcond1 = '{0,0}' ); CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT SUM(humidity), round(newavg(temperature::int4)) FROM conditions GROUP BY time_bucket('1week', timec), location ORDER BY 1,2; NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1, 2; sum | round -----+------- 75 | 38 90 | 60 100 | 55 100 | 75 100 | 100 -- ORDER BY in the view definition DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), COUNT(location), SUM(temperature) FROM conditions GROUP BY time_bucket('1week', timec) ORDER BY sum DESC; NOTICE: refreshing continuous aggregate "mat_m1" -- CAgg definition for realtime SELECT pg_get_viewdef('mat_m1',true); pg_get_viewdef ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ( SELECT _materialized_hypertable_59.time_bucket, + _materialized_hypertable_59.count, + _materialized_hypertable_59.sum + FROM _timescaledb_internal._materialized_hypertable_59 + WHERE _materialized_hypertable_59.time_bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(59)), '-infinity'::timestamp with time zone)+ ORDER BY _materialized_hypertable_59.sum DESC) + UNION ALL + ( SELECT time_bucket('@ 7 days'::interval, conditions.timec) AS time_bucket, + count(conditions.location) AS count, + sum(conditions.temperature) AS sum + FROM conditions + WHERE conditions.timec >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(59)), '-infinity'::timestamp with time zone) + GROUP BY (time_bucket('@ 7 days'::interval, conditions.timec)) + ORDER BY (sum(conditions.temperature)) DESC) + ORDER BY 3 DESC; -- Ordered result SELECT * FROM mat_m1; time_bucket | count | sum ------------------------------+-------+----- Sun Dec 27 16:00:00 2009 PST | 5 | 330 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Jan 03 16:00:00 2010 PST | 1 | 75 -- Insert new data and query again to make sure we produce ordered data INSERT INTO conditions VALUES ('2018-11-10 09:00:00-08', 'SFO', 10, 10); SELECT * FROM mat_m1; time_bucket | count | sum ------------------------------+-------+----- Sun Dec 27 16:00:00 2009 PST | 5 | 330 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Jan 03 16:00:00 2010 PST | 1 | 75 Sun Nov 04 16:00:00 2018 PST | 1 | 10 -- This new row will change the order again INSERT INTO conditions VALUES ('2018-11-11 09:00:00-08', 'SFO', 400, 400); SELECT * FROM mat_m1; time_bucket | count | sum ------------------------------+-------+----- Sun Nov 04 16:00:00 2018 PST | 2 | 410 Sun Dec 27 16:00:00 2009 PST | 5 | 330 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Jan 03 16:00:00 2010 PST | 1 | 75 -- Merge Append EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM mat_m1; --- QUERY PLAN --- Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Index Scan Backward using _hyper_59_123_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_123_chunk -> Index Scan Backward using _hyper_59_124_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_124_chunk Index Cond: (time_bucket < 'Sun Nov 04 16:00:00 2018 PST'::timestamp with time zone) -> Sort Sort Key: (sum(conditions.temperature)) DESC -> Finalize HashAggregate Group Key: (time_bucket('@ 7 days'::interval, conditions.timec)) -> Append -> Partial HashAggregate Group Key: time_bucket('@ 7 days'::interval, _hyper_52_111_chunk.timec) -> Index Scan Backward using _hyper_52_111_chunk_conditions_timec_idx on _hyper_52_111_chunk Index Cond: (timec >= 'Sun Nov 04 16:00:00 2018 PST'::timestamp with time zone) -> Partial HashAggregate Group Key: time_bucket('@ 7 days'::interval, _hyper_52_125_chunk.timec) -> Seq Scan on _hyper_52_125_chunk -- Ordering by another column SELECT * FROM mat_m1 ORDER BY count; time_bucket | count | sum ------------------------------+-------+----- Sun Jan 03 16:00:00 2010 PST | 1 | 75 Sun Nov 04 16:00:00 2018 PST | 2 | 410 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Dec 27 16:00:00 2009 PST | 5 | 330 EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM mat_m1 ORDER BY count; --- QUERY PLAN --- Sort Sort Key: _materialized_hypertable_59.count -> Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Index Scan Backward using _hyper_59_123_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_123_chunk -> Index Scan Backward using _hyper_59_124_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_124_chunk Index Cond: (time_bucket < 'Sun Nov 04 16:00:00 2018 PST'::timestamp with time zone) -> Sort Sort Key: (sum(conditions.temperature)) DESC -> Finalize HashAggregate Group Key: (time_bucket('@ 7 days'::interval, conditions.timec)) -> Append -> Partial HashAggregate Group Key: time_bucket('@ 7 days'::interval, _hyper_52_111_chunk.timec) -> Index Scan Backward using _hyper_52_111_chunk_conditions_timec_idx on _hyper_52_111_chunk Index Cond: (timec >= 'Sun Nov 04 16:00:00 2018 PST'::timestamp with time zone) -> Partial HashAggregate Group Key: time_bucket('@ 7 days'::interval, _hyper_52_125_chunk.timec) -> Seq Scan on _hyper_52_125_chunk -- Change the type of cagg ALTER MATERIALIZED VIEW mat_m1 SET (timescaledb.materialized_only=true); -- CAgg definition for materialized only SELECT pg_get_viewdef('mat_m1',true); pg_get_viewdef ----------------------------------------------------------- SELECT _materialized_hypertable_59.time_bucket, + _materialized_hypertable_59.count, + _materialized_hypertable_59.sum + FROM _timescaledb_internal._materialized_hypertable_59+ ORDER BY _materialized_hypertable_59.sum DESC; -- Now the query will show only the materialized data, without last two -- records inserted into the original hypertable (last two insers above) SELECT * FROM mat_m1; time_bucket | count | sum ------------------------------+-------+----- Sun Dec 27 16:00:00 2009 PST | 5 | 330 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Jan 03 16:00:00 2010 PST | 1 | 75 -- Merge Append EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM mat_m1; --- QUERY PLAN --- Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Index Scan Backward using _hyper_59_123_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_123_chunk -> Index Scan Backward using _hyper_59_124_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_124_chunk -- Ordering by another column SELECT * FROM mat_m1 ORDER BY count; time_bucket | count | sum ------------------------------+-------+----- Sun Jan 03 16:00:00 2010 PST | 1 | 75 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Dec 27 16:00:00 2009 PST | 5 | 330 EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM mat_m1 ORDER BY count; --- QUERY PLAN --- Sort Sort Key: _materialized_hypertable_59.count -> Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Index Scan Backward using _hyper_59_123_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_123_chunk -> Index Scan Backward using _hyper_59_124_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_124_chunk SELECT h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset -- Invalidate old region and refresh again DELETE FROM conditions WHERE timec < '2010-01-05 09:00:00-08'; CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); -- Querying the cagg produce ordered records as expected SELECT * FROM mat_m1; time_bucket | count | sum ------------------------------+-------+----- Sun Nov 04 16:00:00 2018 PST | 2 | 410 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Jan 03 16:00:00 2010 PST | 1 | 75 -- Querying direct the materialization hypertable doesn't -- produce ordered records SELECT * FROM :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME"; time_bucket | count | sum ------------------------------+-------+----- Sun Jan 03 16:00:00 2010 PST | 1 | 75 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Nov 04 16:00:00 2018 PST | 2 | 410 DROP TABLE conditions CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL ); SELECT table_name FROM create_hypertable('conditions', 'timec'); table_name ------------ conditions INSERT INTO conditions VALUES ('2010-01-01 09:00:00-08', 'SFO', 55, 45), ('2010-01-02 09:00:00-08', 'por', 100, 100), ('2010-01-02 09:00:00-08', 'SFO', 65, 45), ('2010-01-02 09:00:00-08', 'NYC', 65, 45), ('2018-11-01 09:00:00-08', 'NYC', 45, 35), ('2018-11-02 09:00:00-08', 'NYC', 35, 15); CREATE MATERIALIZED VIEW conditions_summary_new(timec, minl, sumt, sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1day', timec), min(location), sum(temperature), sum(humidity) FROM conditions GROUP BY time_bucket('1day', timec) WITH NO DATA; \x ON SELECT * FROM timescaledb_information.continuous_aggregates WHERE view_name = 'conditions_summary_new'; -[ RECORD 1 ]---------------------+--------------------------------------------------------------------- hypertable_schema | public hypertable_name | conditions view_schema | public view_name | conditions_summary_new view_owner | default_perm_user materialized_only | t compression_enabled | f materialization_hypertable_schema | _timescaledb_internal materialization_hypertable_name | _materialized_hypertable_61 view_definition | SELECT time_bucket('@ 1 day'::interval, conditions.timec) AS timec,+ | min(conditions.location) AS minl, + | sum(conditions.temperature) AS sumt, + | sum(conditions.humidity) AS sumh + | FROM conditions + | GROUP BY (time_bucket('@ 1 day'::interval, conditions.timec)); \x OFF CALL refresh_continuous_aggregate('conditions_summary_new', NULL, NULL); -- Check and compare number of returned rows SELECT count(*) FROM conditions_summary_new; count ------- 4 -- Parallel planning test for realtime Continuous Aggregate DROP TABLE conditions CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, temperature DOUBLE PRECISION NULL ); SELECT table_name FROM create_hypertable('conditions', 'timec'); table_name ------------ conditions INSERT INTO conditions SELECT t, 10 FROM generate_series('2023-01-01 00:00-03'::timestamptz, '2023-12-31 23:59-03'::timestamptz, '1 hour'::interval) AS t; CREATE MATERIALIZED VIEW conditions_daily WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1 day', timec), SUM(temperature) FROM conditions GROUP BY 1 ORDER BY 2 DESC; NOTICE: refreshing continuous aggregate "conditions_daily" SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false); set_config ------------ on SET max_parallel_workers_per_gather = 4; SET parallel_setup_cost = 0; SET parallel_tuple_cost = 0; -- Parallel planning EXPLAIN (BUFFERS OFF, COSTS OFF, TIMING OFF) SELECT * FROM conditions_daily WHERE time_bucket >= '2023-07-01'; --- QUERY PLAN --- Merge Append Sort Key: _materialized_hypertable_63.sum DESC -> Gather Merge Workers Planned: 2 -> Sort Sort Key: _materialized_hypertable_63.sum DESC -> Parallel Append -> Parallel Index Scan using _hyper_63_185_chunk__materialized_hypertable_63_time_bucket_idx on _hyper_63_185_chunk Index Cond: ((time_bucket < 'Mon Jan 01 16:00:00 2024 PST'::timestamp with time zone) AND (time_bucket >= 'Sat Jul 01 00:00:00 2023 PDT'::timestamp with time zone)) -> Parallel Index Scan using _hyper_63_187_chunk__materialized_hypertable_63_time_bucket_idx on _hyper_63_187_chunk Index Cond: ((time_bucket < 'Mon Jan 01 16:00:00 2024 PST'::timestamp with time zone) AND (time_bucket >= 'Sat Jul 01 00:00:00 2023 PDT'::timestamp with time zone)) -> Parallel Seq Scan on _hyper_63_184_chunk -> Sort Sort Key: (sum(_hyper_62_182_chunk.temperature)) DESC -> HashAggregate Group Key: (time_bucket('@ 1 day'::interval, _hyper_62_182_chunk.timec)) -> Gather Workers Planned: 1 -> Result -> Parallel Index Scan Backward using _hyper_62_182_chunk_conditions_timec_idx on _hyper_62_182_chunk Index Cond: ((timec >= 'Mon Jan 01 16:00:00 2024 PST'::timestamp with time zone) AND (timec >= 'Sat Jul 01 00:00:00 2023 PDT'::timestamp with time zone)) Filter: (time_bucket('@ 1 day'::interval, timec) >= 'Sat Jul 01 00:00:00 2023 PDT'::timestamp with time zone) ================================================ FILE: tsl/test/expected/cagg-16.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- initialize the bgw mock state to prevent the materialization workers from running \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION ts_bgw_params_create() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION test.continuous_aggs_find_view(cagg REGCLASS) RETURNS VOID AS :TSL_MODULE_PATHNAME, 'ts_test_continuous_agg_find_by_view_name' LANGUAGE C; \set WAIT_ON_JOB 0 \set IMMEDIATELY_SET_UNTIL 1 \set WAIT_FOR_OTHER_TO_ADVANCE 2 -- remove any default jobs, e.g., telemetry so bgw_job isn't polluted DELETE FROM _timescaledb_catalog.bgw_job; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT * FROM _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- --TEST1 --- --basic test with count create table foo (a integer, b integer, c integer); select table_name from create_hypertable('foo', 'a', chunk_time_interval=> 10); table_name ------------ foo insert into foo values( 3 , 16 , 20); insert into foo values( 1 , 10 , 20); insert into foo values( 1 , 11 , 20); insert into foo values( 1 , 12 , 20); insert into foo values( 1 , 13 , 20); insert into foo values( 1 , 14 , 20); insert into foo values( 2 , 14 , 20); insert into foo values( 2 , 15 , 20); insert into foo values( 2 , 16 , 20); CREATE OR REPLACE FUNCTION integer_now_foo() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(a), 0) FROM foo $$; SELECT set_integer_now_func('foo', 'integer_now_foo'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW mat_m1(a, countb) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select a, count(b) from foo group by time_bucket(1, a), a WITH NO DATA; SELECT add_continuous_aggregate_policy('mat_m1', NULL, 2::integer, '12 h'::interval) AS job_id \gset SELECT * FROM _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ------+--------------------------------------------+-------------------+-------------+-------------+--------------+------------------------+-------------------------------------+-------------------+-----------+----------------+---------------+---------------+-----------------------------------------------------------------+------------------------+-------------------------------------------+---------- 1000 | Refresh Continuous Aggregate Policy [1000] | @ 12 hours | @ 0 | -1 | @ 12 hours | _timescaledb_functions | policy_refresh_continuous_aggregate | default_perm_user | t | f | | 2 | {"end_offset": 2, "start_offset": null, "mat_hypertable_id": 2} | _timescaledb_functions | policy_refresh_continuous_aggregate_check | SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select a, count(b), time_bucket(1, a) from foo group by time_bucket(1, a) , a ; select * from mat_m1 order by a ; a | countb ---+-------- 1 | 5 2 | 3 3 | 1 --check triggers on user hypertable -- SET ROLE :ROLE_SUPERUSER; select tgname, tgtype, tgenabled , relname from pg_trigger, pg_class where tgrelid = pg_class.oid and pg_class.relname like 'foo' order by tgname; tgname | tgtype | tgenabled | relname --------+--------+-----------+--------- SET ROLE :ROLE_DEFAULT_PERM_USER; -- TEST2 --- DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to table _timescaledb_internal._hyper_2_2_chunk SHOW enable_partitionwise_aggregate; enable_partitionwise_aggregate -------------------------------- off SET enable_partitionwise_aggregate = on; SELECT * FROM _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions insert into conditions values ( '2010-01-01 09:00:00-08', 'SFO', 55, 45); insert into conditions values ( '2010-01-02 09:00:00-08', 'por', 100, 100); insert into conditions values ( '2010-01-02 09:00:00-08', 'SFO', 65, 45); insert into conditions values ( '2010-01-02 09:00:00-08', 'NYC', 65, 45); insert into conditions values ( '2018-11-01 09:00:00-08', 'NYC', 45, 35); insert into conditions values ( '2018-11-02 09:00:00-08', 'NYC', 35, 15); CREATE MATERIALIZED VIEW mat_m1( timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1day', timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket('1day', timec) WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset -- Materialized hypertable for mat_m1 should not be visible in the -- hypertables view: SELECT hypertable_schema, hypertable_name FROM timescaledb_information.hypertables ORDER BY 1,2; hypertable_schema | hypertable_name -------------------+----------------- public | conditions public | foo SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select time_bucket('1day', timec), min(location), sum(temperature), sum(humidity) from conditions group by time_bucket('1day', timec) ; SET ROLE :ROLE_DEFAULT_PERM_USER; --should have same results -- select timec, minl, sumt, sumh from mat_m1 order by timec; timec | minl | sumt | sumh ------------------------------+------+------+------ Thu Dec 31 16:00:00 2009 PST | SFO | 55 | 45 Fri Jan 01 16:00:00 2010 PST | NYC | 230 | 190 Wed Oct 31 17:00:00 2018 PDT | NYC | 45 | 35 Thu Nov 01 17:00:00 2018 PDT | NYC | 35 | 15 select time_bucket('1day', timec), min(location), sum(temperature), sum(humidity) from conditions group by time_bucket('1day', timec) order by 1; time_bucket | min | sum | sum ------------------------------+-----+-----+----- Thu Dec 31 16:00:00 2009 PST | SFO | 55 | 45 Fri Jan 01 16:00:00 2010 PST | NYC | 230 | 190 Wed Oct 31 17:00:00 2018 PDT | NYC | 45 | 35 Thu Nov 01 17:00:00 2018 PDT | NYC | 35 | 15 SET enable_partitionwise_aggregate = off; -- TEST3 -- -- drop on table conditions should cascade to materialized mat_v1 drop table conditions cascade; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions insert into conditions values ( '2010-01-01 09:00:00-08', 'SFO', 55, 45); insert into conditions values ( '2010-01-02 09:00:00-08', 'por', 100, 100); insert into conditions values ( '2010-01-02 09:00:00-08', 'NYC', 65, 45); insert into conditions values ( '2010-01-02 09:00:00-08', 'SFO', 65, 45); insert into conditions values ( '2010-01-03 09:00:00-08', 'NYC', 45, 55); insert into conditions values ( '2010-01-05 09:00:00-08', 'SFO', 75, 100); insert into conditions values ( '2018-11-01 09:00:00-08', 'NYC', 45, 35); insert into conditions values ( '2018-11-02 09:00:00-08', 'NYC', 35, 15); insert into conditions values ( '2018-11-03 09:00:00-08', 'NYC', 35, 25); CREATE MATERIALIZED VIEW mat_m1( timec, minl, sumth, stddevh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select time_bucket('1week', timec), min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) ; SET ROLE :ROLE_DEFAULT_PERM_USER; --should have same results -- select timec, minl, sumth, stddevh from mat_m1 order by timec; timec | minl | sumth | stddevh ------------------------------+------+-------+------------------ Sun Dec 27 16:00:00 2009 PST | NYC | 620 | 23.8746727726266 Sun Jan 03 16:00:00 2010 PST | SFO | 175 | Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 select time_bucket('1week', timec) , min(location), sum(temperature)+ sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) order by time_bucket('1week', timec); time_bucket | min | ?column? | stddev ------------------------------+-----+----------+------------------ Sun Dec 27 16:00:00 2009 PST | NYC | 620 | 23.8746727726266 Sun Jan 03 16:00:00 2010 PST | SFO | 175 | Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 -- TEST4 -- --materialized view with group by clause + expression in SELECT -- use previous data from conditions --drop only the view. -- apply where clause on result of mat_m1 -- DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW mat_m1( timec, minl, sumth, stddevh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions where location = 'NYC' group by time_bucket('1week', timec) WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select time_bucket('1week', timec), min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions where location = 'NYC' group by time_bucket('1week', timec) ; SET ROLE :ROLE_DEFAULT_PERM_USER; --should have same results -- select timec, minl, sumth, stddevh from mat_m1 where stddevh is not null order by timec; timec | minl | sumth | stddevh ------------------------------+------+-------+------------------ Sun Dec 27 16:00:00 2009 PST | NYC | 210 | 7.07106781186548 Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 select time_bucket('1week', timec) , min(location), sum(temperature)+ sum(humidity), stddev(humidity) from conditions where location = 'NYC' group by time_bucket('1week', timec) order by time_bucket('1week', timec); time_bucket | min | ?column? | stddev ------------------------------+-----+----------+------------------ Sun Dec 27 16:00:00 2009 PST | NYC | 210 | 7.07106781186548 Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 -- TEST5 -- ---------test with having clause ---------------------- DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects create materialized view mat_m1( timec, minl, sumth, stddevh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having stddev(humidity) is not null WITH NO DATA; ; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select time_bucket('1week', timec), min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having stddev(humidity) is not null; SET ROLE :ROLE_DEFAULT_PERM_USER; -- should have same results -- select * from mat_m1 order by sumth; timec | minl | sumth | stddevh ------------------------------+------+-------+------------------ Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 Sun Dec 27 16:00:00 2009 PST | NYC | 620 | 23.8746727726266 select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having stddev(humidity) is not null order by sum(temperature)+sum(humidity); time_bucket | min | ?column? | stddev ------------------------------+-----+----------+------------------ Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 Sun Dec 27 16:00:00 2009 PST | NYC | 620 | 23.8746727726266 -- TEST6 -- --group by with more than 1 group column -- having clause with a mix of columns from select list + others drop table conditions cascade; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp numeric NULL, highp numeric null ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions insert into conditions select generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'POR', 55, 75, 40, 70; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'NYC', 35, 45, 50, 40; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-15 00:00'::timestamp, '1 day'), 'LA', 73, 55, 71, 28; --naming with AS clauses CREATE MATERIALIZED VIEW mat_naming WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec) as bucket, location as loc, sum(temperature)+sum(humidity) as sumth, stddev(humidity) from conditions group by bucket, loc having min(location) >= 'NYC' and avg(temperature) > 20 WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_naming' \gset select attnum , attname from pg_attribute where attnum > 0 and attrelid = (Select oid from pg_class where relname like :'MAT_TABLE_NAME') order by attnum, attname; attnum | attname --------+--------- 1 | bucket 2 | loc 3 | sumth 4 | stddev DROP MATERIALIZED VIEW mat_naming; --naming with default names CREATE MATERIALIZED VIEW mat_naming WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec), location, sum(temperature)+sum(humidity) as sumth, stddev(humidity) from conditions group by 1,2 having min(location) >= 'NYC' and avg(temperature) > 20 WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_naming' \gset select attnum , attname from pg_attribute where attnum > 0 and attrelid = (Select oid from pg_class where relname like :'MAT_TABLE_NAME') order by attnum, attname; attnum | attname --------+------------- 1 | time_bucket 2 | location 3 | sumth 4 | stddev DROP MATERIALIZED VIEW mat_naming; --naming with view col names CREATE MATERIALIZED VIEW mat_naming(bucket, loc, sum_t_h, stdd) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec), location, sum(temperature)+sum(humidity), stddev(humidity) from conditions group by 1,2 having min(location) >= 'NYC' and avg(temperature) > 20 WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_naming' \gset select attnum , attname from pg_attribute where attnum > 0 and attrelid = (Select oid from pg_class where relname like :'MAT_TABLE_NAME') order by attnum, attname; attnum | attname --------+--------- 1 | bucket 2 | loc 3 | sum_t_h 4 | stdd DROP MATERIALIZED VIEW mat_naming; CREATE MATERIALIZED VIEW mat_m1(timec, minl, sumth, stddevh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having min(location) >= 'NYC' and avg(temperature) > 20 WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset select attnum , attname from pg_attribute where attnum > 0 and attrelid = (Select oid from pg_class where relname like :'MAT_TABLE_NAME') order by attnum, attname; attnum | attname --------+--------- 1 | timec 2 | minl 3 | sumth 4 | stddevh SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select time_bucket('1week', timec), min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having min(location) >= 'NYC' and avg(temperature) > 20; SET ROLE :ROLE_DEFAULT_PERM_USER; --should have same results -- select timec, minl, sumth, stddevh from mat_m1 order by timec, minl; timec | minl | sumth | stddevh ------------------------------+------+-------+------------------ Sun Dec 16 16:00:00 2018 PST | NYC | 1470 | 15.5662356498831 Sun Dec 23 16:00:00 2018 PST | NYC | 1470 | 15.5662356498831 Sun Dec 30 16:00:00 2018 PST | NYC | 210 | 21.2132034355964 select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having min(location) >= 'NYC' and avg(temperature) > 20 and avg(lowp) > 10 order by time_bucket('1week', timec), min(location); time_bucket | min | ?column? | stddev ------------------------------+-----+----------+------------------ Sun Dec 16 16:00:00 2018 PST | NYC | 1470 | 15.5662356498831 Sun Dec 23 16:00:00 2018 PST | NYC | 1470 | 15.5662356498831 Sun Dec 30 16:00:00 2018 PST | NYC | 210 | 21.2132034355964 --check view defintion in information views select view_name, view_definition from timescaledb_information.continuous_aggregates where view_name::text like 'mat_m1'; view_name | view_definition -----------+------------------------------------------------------------------------------------------- mat_m1 | SELECT time_bucket('@ 7 days'::interval, timec) AS timec, + | min(location) AS minl, + | (sum(temperature) + sum(humidity)) AS sumth, + | stddev(humidity) AS stddevh + | FROM conditions + | GROUP BY (time_bucket('@ 7 days'::interval, timec)) + | HAVING ((min(location) >= 'NYC'::text) AND (avg(temperature) > (20)::double precision)); --TEST6 -- select from internal view SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select * from :"PART_VIEW_SCHEMA".:"PART_VIEW_NAME"; SET ROLE :ROLE_DEFAULT_PERM_USER; --lets drop the view and check DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to table _timescaledb_internal._hyper_13_24_chunk drop table conditions; CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions insert into conditions select generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'POR', 55, 75, 40, 70, NULL; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'NYC', 35, 45, 50, 40, NULL; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-15 00:00'::timestamp, '1 day'), 'LA', 73, 55, NULL, 28, NULL; SELECT $$ select time_bucket('1week', timec) , min(location) as col1, sum(temperature)+sum(humidity) as col2, stddev(humidity) as col3, min(allnull) as col4 from conditions group by time_bucket('1week', timec) having min(location) >= 'NYC' and avg(temperature) > 20 $$ AS "QUERY" \gset \set ECHO errors psql:include/cont_agg_equal.sql:8: NOTICE: materialized view "mat_test" does not exist, skipping ?column? | count ---------------------------------------------------------------+------- Number of rows different between view and original (expect 0) | 0 SELECT $$ select time_bucket('1week', timec), location, sum(temperature)+sum(humidity) as col2, stddev(humidity) as col3, min(allnull) as col4 from conditions group by location, time_bucket('1week', timec) $$ AS "QUERY" \gset \set ECHO errors psql:include/cont_agg_equal.sql:8: NOTICE: drop cascades to table _timescaledb_internal._hyper_15_34_chunk ?column? | count ---------------------------------------------------------------+------- Number of rows different between view and original (expect 0) | 0 --TEST7 -- drop tests for view and hypertable --DROP tests \set ON_ERROR_STOP 0 SELECT h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA", direct_view_name as "DIR_VIEW_NAME", direct_view_schema as "DIR_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_test' \gset DROP TABLE :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME"; ERROR: cannot drop table _timescaledb_internal._materialized_hypertable_16 because other objects depend on it DROP VIEW :"PART_VIEW_SCHEMA".:"PART_VIEW_NAME"; ERROR: cannot drop the partial/direct view because it is required by a continuous aggregate DROP VIEW :"DIR_VIEW_SCHEMA".:"DIR_VIEW_NAME"; ERROR: cannot drop the partial/direct view because it is required by a continuous aggregate \set ON_ERROR_STOP 1 --catalog entry still there; SELECT count(*) FROM _timescaledb_catalog.continuous_agg ca WHERE user_view_name = 'mat_test'; count ------- 1 --mat table, user_view, direct view and partial view all there select count(*) from pg_class where relname = :'PART_VIEW_NAME'; count ------- 1 select count(*) from pg_class where relname = :'MAT_TABLE_NAME'; count ------- 1 select count(*) from pg_class where relname = :'DIR_VIEW_NAME'; count ------- 1 select count(*) from pg_class where relname = 'mat_test'; count ------- 1 DROP MATERIALIZED VIEW mat_test; NOTICE: drop cascades to 2 other objects --catalog entry should be gone SELECT count(*) FROM _timescaledb_catalog.continuous_agg ca WHERE user_view_name = 'mat_test'; count ------- 0 --mat table, user_view, direct view and partial view all gone select count(*) from pg_class where relname = :'PART_VIEW_NAME'; count ------- 0 select count(*) from pg_class where relname = :'MAT_TABLE_NAME'; count ------- 0 select count(*) from pg_class where relname = :'DIR_VIEW_NAME'; count ------- 0 select count(*) from pg_class where relname = 'mat_test'; count ------- 0 --test dropping raw table DROP TABLE conditions; CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions --no data in hyper table on purpose so that CASCADE is not required because of chunks CREATE MATERIALIZED VIEW mat_drop_test(timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1day', timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket('1day', timec) WITH NO DATA; \set ON_ERROR_STOP 0 DROP TABLE conditions; ERROR: cannot drop table conditions because other objects depend on it \set ON_ERROR_STOP 1 --insert data now insert into conditions select generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'POR', 55, 75, 40, 70, NULL; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'NYC', 35, 45, 50, 40, NULL; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-15 00:00'::timestamp, '1 day'), 'LA', 73, 55, NULL, 28, NULL; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_drop_test' \gset SET client_min_messages TO NOTICE; CALL refresh_continuous_aggregate('mat_drop_test', NULL, NULL); --force invalidation insert into conditions select generate_series('2017-11-01 00:00'::timestamp, '2017-12-15 00:00'::timestamp, '1 day'), 'LA', 73, 55, NULL, 28, NULL; select count(*) from _timescaledb_catalog.continuous_aggs_invalidation_threshold; count ------- 1 select count(*) from _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log; count ------- 8 DROP TABLE conditions CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects --catalog entry should be gone SELECT count(*) FROM _timescaledb_catalog.continuous_agg ca WHERE user_view_name = 'mat_drop_test'; count ------- 0 select count(*) from _timescaledb_catalog.continuous_aggs_invalidation_threshold; count ------- 0 select count(*) from _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log; count ------- 0 select count(*) from _timescaledb_catalog.continuous_aggs_materialization_invalidation_log; count ------- 0 SELECT * FROM _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- --mat table, user_view, and partial view all gone select count(*) from pg_class where relname = :'PART_VIEW_NAME'; count ------- 0 select count(*) from pg_class where relname = :'MAT_TABLE_NAME'; count ------- 0 select count(*) from pg_class where relname = 'mat_drop_test'; count ------- 0 --TEST With options CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions CREATE MATERIALIZED VIEW mat_with_test(timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1day', timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket('1day', timec), location, humidity, temperature WITH NO DATA; SELECT add_continuous_aggregate_policy('mat_with_test', NULL, '5 h'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1001 SELECT alter_job(id, schedule_interval => '1h') FROM _timescaledb_catalog.bgw_job; alter_job ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ (1001,"@ 1 hour","@ 0",-1,"@ 12 hours",t,"{""end_offset"": ""@ 5 hours"", ""start_offset"": null, ""mat_hypertable_id"": 20}",-infinity,_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1001]") SELECT schedule_interval FROM _timescaledb_catalog.bgw_job; schedule_interval ------------------- @ 1 hour SELECT alter_job(id, schedule_interval => '2h') FROM _timescaledb_catalog.bgw_job; alter_job ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1001,"@ 2 hours","@ 0",-1,"@ 12 hours",t,"{""end_offset"": ""@ 5 hours"", ""start_offset"": null, ""mat_hypertable_id"": 20}",-infinity,_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1001]") SELECT schedule_interval FROM _timescaledb_catalog.bgw_job; schedule_interval ------------------- @ 2 hours select indexname, indexdef from pg_indexes where tablename = (SELECT h.table_name FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_with_test') order by indexname; indexname | indexdef ---------------------------------------+---------------------------------------------------------------------------------------------------------------------------------- _materialized_hypertable_20_timec_idx | CREATE INDEX _materialized_hypertable_20_timec_idx ON _timescaledb_internal._materialized_hypertable_20 USING btree (timec DESC) DROP MATERIALIZED VIEW mat_with_test; --no additional indexes CREATE MATERIALIZED VIEW mat_with_test(timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.create_group_indexes=false) as select time_bucket('1day', timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket('1day', timec), location, humidity, temperature WITH NO DATA; select indexname, indexdef from pg_indexes where tablename = (SELECT h.table_name FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_with_test'); indexname | indexdef ---------------------------------------+---------------------------------------------------------------------------------------------------------------------------------- _materialized_hypertable_21_timec_idx | CREATE INDEX _materialized_hypertable_21_timec_idx ON _timescaledb_internal._materialized_hypertable_21 USING btree (timec DESC) DROP TABLE conditions CASCADE; NOTICE: drop cascades to 2 other objects --test WITH using a hypertable with an integer time dimension CREATE TABLE conditions ( timec INT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec', chunk_time_interval=> 100); table_name ------------ conditions CREATE OR REPLACE FUNCTION integer_now_conditions() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now_conditions'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW mat_with_test(timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket(100, timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket(100, timec) WITH NO DATA; SELECT add_continuous_aggregate_policy('mat_with_test', NULL, 500::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1002 SELECT alter_job(id, schedule_interval => '2h') FROM _timescaledb_catalog.bgw_job; alter_job --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1002,"@ 2 hours","@ 0",-1,"@ 12 hours",t,"{""end_offset"": 500, ""start_offset"": null, ""mat_hypertable_id"": 23}",-infinity,_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1002]") SELECT schedule_interval FROM _timescaledb_catalog.bgw_job; schedule_interval ------------------- @ 2 hours DROP TABLE conditions CASCADE; NOTICE: drop cascades to 2 other objects --test space partitions CREATE TABLE space_table ( time BIGINT, dev BIGINT, data BIGINT ); SELECT create_hypertable( 'space_table', 'time', chunk_time_interval => 10, partitioning_column => 'dev', number_partitions => 3); create_hypertable --------------------------- (24,public,space_table,t) CREATE OR REPLACE FUNCTION integer_now_space_table() returns BIGINT LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), BIGINT '0') FROM space_table $$; SELECT set_integer_now_func('space_table', 'integer_now_space_table'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW space_view WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('4', time), COUNT(data) FROM space_table GROUP BY 1 WITH NO DATA; INSERT INTO space_table VALUES (0, 1, 1), (0, 2, 1), (1, 1, 1), (1, 2, 1), (10, 1, 1), (10, 2, 1), (11, 1, 1), (11, 2, 1); SELECT h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA", direct_view_name as "DIR_VIEW_NAME", direct_view_schema as "DIR_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'space_view' \gset SELECT * FROM :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" ORDER BY time_bucket; time_bucket | count -------------+------- CALL refresh_continuous_aggregate('space_view', NULL, NULL); SELECT * FROM space_view ORDER BY 1; time_bucket | count -------------+------- 0 | 4 8 | 4 SELECT * FROM :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" ORDER BY time_bucket; time_bucket | count -------------+------- 0 | 4 8 | 4 INSERT INTO space_table VALUES (3, 2, 1); CALL refresh_continuous_aggregate('space_view', NULL, NULL); SELECT * FROM space_view ORDER BY 1; time_bucket | count -------------+------- 0 | 5 8 | 4 SELECT * FROM :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" ORDER BY time_bucket; time_bucket | count -------------+------- 0 | 5 8 | 4 INSERT INTO space_table VALUES (2, 3, 1); CALL refresh_continuous_aggregate('space_view', NULL, NULL); SELECT * FROM space_view ORDER BY 1; time_bucket | count -------------+------- 0 | 6 8 | 4 SELECT * FROM :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" ORDER BY time_bucket; time_bucket | count -------------+------- 0 | 6 8 | 4 DROP TABLE space_table CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_25_60_chunk -- -- TEST FINALIZEFUNC_EXTRA -- -- create special aggregate to test ffunc_extra -- Raise warning with the actual type being passed in CREATE OR REPLACE FUNCTION fake_ffunc(a int8, b int, c int, d int, x anyelement) RETURNS anyelement AS $$ BEGIN RAISE WARNING 'type % %', pg_typeof(d), pg_typeof(x); RETURN x; END; $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION fake_sfunc(a int8, b int, c int, d int, x anyelement) RETURNS int8 AS $$ BEGIN RETURN b; END; $$ LANGUAGE plpgsql; CREATE AGGREGATE aggregate_to_test_ffunc_extra(int, int, int, anyelement) ( SFUNC = fake_sfunc, STYPE = int8, COMBINEFUNC = int8pl, FINALFUNC = fake_ffunc, PARALLEL = SAFE, FINALFUNC_EXTRA ); CREATE TABLE conditions ( timec INT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec', chunk_time_interval=> 100); table_name ------------ conditions CREATE OR REPLACE FUNCTION integer_now_conditions() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now_conditions'); set_integer_now_func ---------------------- insert into conditions select generate_series(0, 200, 10), 'POR', 55, 75, 40, 70, NULL; CREATE MATERIALIZED VIEW mat_ffunc_test WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket(100, timec), aggregate_to_test_ffunc_extra(timec, 1, 3, 'test'::text) from conditions group by time_bucket(100, timec); NOTICE: refreshing continuous aggregate "mat_ffunc_test" WARNING: type integer text WARNING: type integer text WARNING: type integer text SELECT * FROM mat_ffunc_test ORDER BY time_bucket; time_bucket | aggregate_to_test_ffunc_extra -------------+------------------------------- 0 | 100 | 200 | DROP MATERIALIZED view mat_ffunc_test; NOTICE: drop cascades to table _timescaledb_internal._hyper_27_65_chunk CREATE MATERIALIZED VIEW mat_ffunc_test WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket(100, timec), aggregate_to_test_ffunc_extra(timec, 4, 5, bigint '123') from conditions group by time_bucket(100, timec); NOTICE: refreshing continuous aggregate "mat_ffunc_test" WARNING: type integer bigint WARNING: type integer bigint WARNING: type integer bigint SELECT * FROM mat_ffunc_test ORDER BY time_bucket; time_bucket | aggregate_to_test_ffunc_extra -------------+------------------------------- 0 | 100 | 200 | --refresh mat view test when time_bucket is not projected -- DROP MATERIALIZED VIEW mat_ffunc_test; NOTICE: drop cascades to table _timescaledb_internal._hyper_28_66_chunk CREATE MATERIALIZED VIEW mat_refresh_test WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select location, max(humidity) from conditions group by time_bucket(100, timec), location WITH NO DATA; insert into conditions select generate_series(0, 50, 10), 'NYC', 55, 75, 40, 70, NULL; CALL refresh_continuous_aggregate('mat_refresh_test', NULL, NULL); SELECT * FROM mat_refresh_test order by 1,2 ; location | max ----------+----- NYC | 75 POR | 75 POR | 75 POR | 75 -- test for bug when group by is not in project list CREATE MATERIALIZED VIEW conditions_grpby_view with (timescaledb.continuous, timescaledb.materialized_only=false) as select time_bucket(100, timec), sum(humidity) from conditions group by time_bucket(100, timec), location; NOTICE: refreshing continuous aggregate "conditions_grpby_view" select * from conditions_grpby_view order by 1, 2; time_bucket | sum -------------+----- 0 | 450 0 | 750 100 | 750 200 | 75 CREATE MATERIALIZED VIEW conditions_grpby_view2 with (timescaledb.continuous, timescaledb.materialized_only=false) as select time_bucket(100, timec), sum(humidity) from conditions group by time_bucket(100, timec), location having avg(temperature) > 0; NOTICE: refreshing continuous aggregate "conditions_grpby_view2" select * from conditions_grpby_view2 order by 1, 2; time_bucket | sum -------------+----- 0 | 450 0 | 750 100 | 750 200 | 75 -- Test internal functions for continuous aggregates SELECT test.continuous_aggs_find_view('mat_refresh_test'); continuous_aggs_find_view --------------------------- -- Test pseudotype/enum handling CREATE TYPE status_enum AS ENUM ( 'red', 'yellow', 'green' ); CREATE TABLE cagg_types ( time TIMESTAMPTZ NOT NULL, status status_enum, names NAME[], floats FLOAT[] ); SELECT table_name FROM create_hypertable('cagg_types', 'time'); table_name ------------ cagg_types INSERT INTO cagg_types SELECT '2000-01-01', 'yellow', '{foo,bar,baz}', '{1,2.5,3}'; CREATE MATERIALIZED VIEW mat_types WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1d', time), min(status) AS status, max(names) AS names, min(floats) AS floats FROM cagg_types GROUP BY 1; NOTICE: refreshing continuous aggregate "mat_types" CALL refresh_continuous_aggregate('mat_types',NULL,NULL); NOTICE: continuous aggregate "mat_types" is already up-to-date SELECT * FROM mat_types; time_bucket | status | names | floats ------------------------------+--------+---------------+----------- Fri Dec 31 16:00:00 1999 PST | yellow | {foo,bar,baz} | {1,2.5,3} ------------------------------------------------------------------------------------- -- Test issue #2616 where cagg view contains an experssion with several aggregates in CREATE TABLE water_consumption ( sensor_id integer NOT NULL, timestamp timestamp(0) NOT NULL, water_index integer ); SELECT create_hypertable('water_consumption', 'timestamp', 'sensor_id', 2); WARNING: column type "timestamp without time zone" used for "timestamp" does not follow best practices create_hypertable --------------------------------- (34,public,water_consumption,t) INSERT INTO public.water_consumption (sensor_id, timestamp, water_index) VALUES (1, '2010-11-03 09:42:30', 1030), (1, '2010-11-03 09:42:40', 1032), (1, '2010-11-03 09:42:50', 1035), (1, '2010-11-03 09:43:30', 1040), (1, '2010-11-03 09:43:40', 1045), (1, '2010-11-03 09:43:50', 1050), (1, '2010-11-03 09:44:30', 1052), (1, '2010-11-03 09:44:40', 1057), (1, '2010-11-03 09:44:50', 1060), (1, '2010-11-03 09:45:30', 1063), (1, '2010-11-03 09:45:40', 1067), (1, '2010-11-03 09:45:50', 1070); -- The test with the view originally reported in the issue. CREATE MATERIALIZED VIEW water_consumption_aggregation_minute WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT sensor_id, time_bucket(INTERVAL '1 minute', timestamp) + '1 minute' AS timestamp, (max(water_index) - min(water_index)) AS water_consumption FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) WITH NO DATA; CALL refresh_continuous_aggregate('water_consumption_aggregation_minute', NULL, NULL); -- The results of the view and the query over hypertable should be the same SELECT * FROM water_consumption_aggregation_minute ORDER BY water_consumption; sensor_id | timestamp | water_consumption -----------+--------------------------+------------------- 1 | Wed Nov 03 09:43:00 2010 | 5 1 | Wed Nov 03 09:46:00 2010 | 7 1 | Wed Nov 03 09:45:00 2010 | 8 1 | Wed Nov 03 09:44:00 2010 | 10 SELECT sensor_id, time_bucket(INTERVAL '1 minute', timestamp) + '1 minute' AS timestamp, (max(water_index) - min(water_index)) AS water_consumption FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) ORDER BY water_consumption; sensor_id | timestamp | water_consumption -----------+--------------------------+------------------- 1 | Wed Nov 03 09:43:00 2010 | 5 1 | Wed Nov 03 09:46:00 2010 | 7 1 | Wed Nov 03 09:45:00 2010 | 8 1 | Wed Nov 03 09:44:00 2010 | 10 -- Simplified test, where the view doesn't contain all group by clauses CREATE MATERIALIZED VIEW water_consumption_no_select_bucket WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT sensor_id, (max(water_index) - min(water_index)) AS water_consumption FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) WITH NO DATA; CALL refresh_continuous_aggregate('water_consumption_no_select_bucket', NULL, NULL); -- The results of the view and the query over hypertable should be the same SELECT * FROM water_consumption_no_select_bucket ORDER BY water_consumption; sensor_id | water_consumption -----------+------------------- 1 | 5 1 | 7 1 | 8 1 | 10 SELECT sensor_id, (max(water_index) - min(water_index)) AS water_consumption FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) ORDER BY water_consumption; sensor_id | water_consumption -----------+------------------- 1 | 5 1 | 7 1 | 8 1 | 10 -- The test with SELECT matching GROUP BY and placing aggregate expression not the last CREATE MATERIALIZED VIEW water_consumption_aggregation_no_addition WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT sensor_id, (max(water_index) - min(water_index)) AS water_consumption, time_bucket(INTERVAL '1 minute', timestamp) AS timestamp FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) WITH NO DATA; CALL refresh_continuous_aggregate('water_consumption_aggregation_no_addition', NULL, NULL); -- The results of the view and the query over hypertable should be the same SELECT * FROM water_consumption_aggregation_no_addition ORDER BY water_consumption; sensor_id | water_consumption | timestamp -----------+-------------------+-------------------------- 1 | 5 | Wed Nov 03 09:42:00 2010 1 | 7 | Wed Nov 03 09:45:00 2010 1 | 8 | Wed Nov 03 09:44:00 2010 1 | 10 | Wed Nov 03 09:43:00 2010 SELECT sensor_id, (max(water_index) - min(water_index)) AS water_consumption, time_bucket(INTERVAL '1 minute', timestamp) AS timestamp FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) ORDER BY water_consumption; sensor_id | water_consumption | timestamp -----------+-------------------+-------------------------- 1 | 5 | Wed Nov 03 09:42:00 2010 1 | 7 | Wed Nov 03 09:45:00 2010 1 | 8 | Wed Nov 03 09:44:00 2010 1 | 10 | Wed Nov 03 09:43:00 2010 DROP TABLE water_consumption CASCADE; NOTICE: drop cascades to 6 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_35_73_chunk NOTICE: drop cascades to table _timescaledb_internal._hyper_36_74_chunk NOTICE: drop cascades to table _timescaledb_internal._hyper_37_75_chunk ---- --- github issue 2655 --- create table raw_data(time timestamptz, search_query text, cnt integer, cnt2 integer); select create_hypertable('raw_data','time', chunk_time_interval=>'15 days'::interval); create_hypertable ------------------------ (38,public,raw_data,t) insert into raw_data select '2000-01-01','Q1'; --having has exprs that appear in select CREATE MATERIALIZED VIEW search_query_count_1m WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT search_query,count(search_query) as count, time_bucket(INTERVAL '1 minute', time) AS bucket FROM raw_data WHERE search_query is not null AND LENGTH(TRIM(both from search_query))>0 GROUP BY search_query, bucket HAVING count(search_query) > 3 OR sum(cnt) > 1; NOTICE: refreshing continuous aggregate "search_query_count_1m" --having has aggregates + grp by columns that appear in select CREATE MATERIALIZED VIEW search_query_count_2 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT search_query,count(search_query) as count, sum(cnt), time_bucket(INTERVAL '1 minute', time) AS bucket FROM raw_data WHERE search_query is not null AND LENGTH(TRIM(both from search_query))>0 GROUP BY search_query, bucket HAVING count(search_query) > 3 OR sum(cnt) > 1 OR ( sum(cnt) + count(cnt)) > 1 AND search_query = 'Q1'; NOTICE: refreshing continuous aggregate "search_query_count_2" CREATE MATERIALIZED VIEW search_query_count_3 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT search_query,count(search_query) as count, sum(cnt), time_bucket(INTERVAL '1 minute', time) AS bucket FROM raw_data WHERE search_query is not null AND LENGTH(TRIM(both from search_query))>0 GROUP BY cnt +cnt2 , bucket, search_query HAVING cnt + cnt2 + sum(cnt) > 2 or count(cnt2) > 10; NOTICE: refreshing continuous aggregate "search_query_count_3" insert into raw_data select '2000-01-01 00:00+0','Q1', 1, 100; insert into raw_data select '2000-01-01 00:00+0','Q1', 2, 200; insert into raw_data select '2000-01-01 00:00+0','Q1', 3, 300; insert into raw_data select '2000-01-02 00:00+0','Q2', 10, 10; insert into raw_data select '2000-01-02 00:00+0','Q2', 20, 20; CALL refresh_continuous_aggregate('search_query_count_1m', NULL, NULL); SELECT * FROM search_query_count_1m ORDER BY 1, 2; search_query | count | bucket --------------+-------+------------------------------ Q1 | 3 | Fri Dec 31 16:00:00 1999 PST Q2 | 2 | Sat Jan 01 16:00:00 2000 PST --only 1 of these should appear in the result insert into raw_data select '2000-01-02 00:00+0','Q3', 0, 0; insert into raw_data select '2000-01-03 00:00+0','Q4', 20, 20; CALL refresh_continuous_aggregate('search_query_count_1m', NULL, NULL); SELECT * FROM search_query_count_1m ORDER BY 1, 2; search_query | count | bucket --------------+-------+------------------------------ Q1 | 3 | Fri Dec 31 16:00:00 1999 PST Q2 | 2 | Sat Jan 01 16:00:00 2000 PST Q4 | 1 | Sun Jan 02 16:00:00 2000 PST --refresh search_query_count_2--- CALL refresh_continuous_aggregate('search_query_count_2', NULL, NULL); SELECT * FROM search_query_count_2 ORDER BY 1, 2; search_query | count | sum | bucket --------------+-------+-----+------------------------------ Q1 | 3 | 6 | Fri Dec 31 16:00:00 1999 PST Q2 | 2 | 30 | Sat Jan 01 16:00:00 2000 PST Q4 | 1 | 20 | Sun Jan 02 16:00:00 2000 PST --refresh search_query_count_3--- CALL refresh_continuous_aggregate('search_query_count_3', NULL, NULL); SELECT * FROM search_query_count_3 ORDER BY 1, 2, 3; search_query | count | sum | bucket --------------+-------+-----+------------------------------ Q1 | 1 | 1 | Fri Dec 31 16:00:00 1999 PST Q1 | 1 | 2 | Fri Dec 31 16:00:00 1999 PST Q1 | 1 | 3 | Fri Dec 31 16:00:00 1999 PST Q2 | 1 | 10 | Sat Jan 01 16:00:00 2000 PST Q2 | 1 | 20 | Sat Jan 01 16:00:00 2000 PST Q4 | 1 | 20 | Sun Jan 02 16:00:00 2000 PST --- TEST enable compression on continuous aggregates CREATE VIEW cagg_compression_status as SELECT ca.mat_hypertable_id AS mat_htid, ca.user_view_name AS cagg_name , h.schema_name AS mat_schema_name, h.table_name AS mat_table_name, ca.materialized_only FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) ; SELECT mat_htid AS "MAT_HTID" , mat_schema_name || '.' || mat_table_name AS "MAT_HTNAME" , mat_table_name AS "MAT_TABLE_NAME" FROM cagg_compression_status WHERE cagg_name = 'search_query_count_3' \gset ALTER MATERIALIZED VIEW search_query_count_3 SET (timescaledb.compress = 'true'); NOTICE: defaulting compress_orderby to bucket,search_query SELECT cagg_name, mat_table_name FROM cagg_compression_status where cagg_name = 'search_query_count_3'; cagg_name | mat_table_name ----------------------+----------------------------- search_query_count_3 | _materialized_hypertable_41 \x SELECT * FROM timescaledb_information.compression_settings WHERE hypertable_name = :'MAT_TABLE_NAME'; -[ RECORD 1 ]----------+---------------------------- hypertable_schema | _timescaledb_internal hypertable_name | _materialized_hypertable_41 attname | bucket segmentby_column_index | orderby_column_index | 1 orderby_asc | t orderby_nullsfirst | f -[ RECORD 2 ]----------+---------------------------- hypertable_schema | _timescaledb_internal hypertable_name | _materialized_hypertable_41 attname | search_query segmentby_column_index | orderby_column_index | 2 orderby_asc | t orderby_nullsfirst | f \x SELECT compress_chunk(ch) FROM show_chunks('search_query_count_3') ch; compress_chunk ------------------------------------------ _timescaledb_internal._hyper_41_79_chunk SELECT * from search_query_count_3 ORDER BY 1, 2, 3; search_query | count | sum | bucket --------------+-------+-----+------------------------------ Q1 | 1 | 1 | Fri Dec 31 16:00:00 1999 PST Q1 | 1 | 2 | Fri Dec 31 16:00:00 1999 PST Q1 | 1 | 3 | Fri Dec 31 16:00:00 1999 PST Q2 | 1 | 10 | Sat Jan 01 16:00:00 2000 PST Q2 | 1 | 20 | Sat Jan 01 16:00:00 2000 PST Q4 | 1 | 20 | Sun Jan 02 16:00:00 2000 PST -- insert into a new region of the hypertable and then refresh the cagg -- (note we still do not support refreshes into existing regions. -- cagg chunks do not map 1-1 to hypertabl regions. They encompass -- more data -- ). insert into raw_data select '2000-05-01 00:00+0','Q3', 0, 0; -- On PG >= 14 the refresh test below will pass because we added support for UPDATE/DELETE on compressed chunks in PR #5339 \set ON_ERROR_STOP 0 CALL refresh_continuous_aggregate('search_query_count_3', NULL, '2000-06-01 00:00+0'::timestamptz); CALL refresh_continuous_aggregate('search_query_count_3', '2000-05-01 00:00+0'::timestamptz, '2000-06-01 00:00+0'::timestamptz); NOTICE: continuous aggregate "search_query_count_3" is already up-to-date \set ON_ERROR_STOP 1 --insert row insert into raw_data select '2001-05-10 00:00+0','Q3', 100, 100; --this should succeed since it does not refresh any compressed regions in the cagg CALL refresh_continuous_aggregate('search_query_count_3', '2001-05-01 00:00+0'::timestamptz, '2001-06-01 00:00+0'::timestamptz); --verify watermark and check that chunks are compressed SELECT _timescaledb_functions.to_timestamp(w) FROM _timescaledb_functions.cagg_watermark(:'MAT_HTID') w; to_timestamp ------------------------------ Wed May 09 17:01:00 2001 PDT SELECT chunk_name, range_start, range_end, is_compressed FROM timescaledb_information.chunks WHERE hypertable_name = :'MAT_TABLE_NAME' ORDER BY 1; chunk_name | range_start | range_end | is_compressed --------------------+------------------------------+------------------------------+--------------- _hyper_41_79_chunk | Fri Dec 24 16:00:00 1999 PST | Mon May 22 17:00:00 2000 PDT | t _hyper_41_83_chunk | Sun Mar 18 16:00:00 2001 PST | Wed Aug 15 17:00:00 2001 PDT | f SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id = :'MAT_HTID' ORDER BY 1, 2,3; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- 41 | -9223372036854775808 | -210866803200000001 41 | 959817600000000 | 988675199999999 41 | 991353600000000 | 9223372036854775807 SELECT * from search_query_count_3 WHERE bucket > '2001-01-01' ORDER BY 1, 2, 3; search_query | count | sum | bucket --------------+-------+-----+------------------------------ Q3 | 1 | 100 | Wed May 09 17:00:00 2001 PDT --now disable compression , will error out -- \set ON_ERROR_STOP 0 ALTER MATERIALIZED VIEW search_query_count_3 SET (timescaledb.compress = 'false'); ERROR: cannot disable columnstore on hypertable with columnstore chunks \set ON_ERROR_STOP 1 SELECT decompress_chunk(format('%I.%I', schema_name, table_name)) FROM _timescaledb_catalog.chunk WHERE hypertable_id = :'MAT_HTID' and status = 1; decompress_chunk ------------------------------------------ _timescaledb_internal._hyper_41_79_chunk --disable compression on cagg after decompressing all chunks-- ALTER MATERIALIZED VIEW search_query_count_3 SET (timescaledb.compress = 'false'); SELECT cagg_name, mat_table_name FROM cagg_compression_status where cagg_name = 'search_query_count_3'; cagg_name | mat_table_name ----------------------+----------------------------- search_query_count_3 | _materialized_hypertable_41 SELECT view_name, materialized_only, compression_enabled FROM timescaledb_information.continuous_aggregates where view_name = 'search_query_count_3'; view_name | materialized_only | compression_enabled ----------------------+-------------------+--------------------- search_query_count_3 | f | f -- TEST caggs on table with more columns than in the cagg view defn -- CREATE TABLE test_morecols ( time TIMESTAMPTZ NOT NULL, val1 INTEGER, val2 INTEGER, val3 INTEGER, val4 INTEGER, val5 INTEGER, val6 INTEGER, val7 INTEGER, val8 INTEGER); SELECT create_hypertable('test_morecols', 'time', chunk_time_interval=> '7 days'::interval); create_hypertable ----------------------------- (43,public,test_morecols,t) INSERT INTO test_morecols SELECT generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 55, 75, 40, 70, NULL, 100, 200, 200; CREATE MATERIALIZED VIEW test_morecols_cagg with (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('30 days',time), avg(val1), count(val2) FROM test_morecols GROUP BY 1; NOTICE: refreshing continuous aggregate "test_morecols_cagg" ALTER MATERIALIZED VIEW test_morecols_cagg SET (timescaledb.compress='true'); NOTICE: defaulting compress_orderby to time_bucket SELECT compress_chunk(ch) FROM show_chunks('test_morecols_cagg') ch; compress_chunk ------------------------------------------ _timescaledb_internal._hyper_44_89_chunk SELECT * FROM test_morecols_cagg ORDER BY time_bucket; time_bucket | avg | count ------------------------------+---------------------+------- Fri Nov 23 16:00:00 2018 PST | 55.0000000000000000 | 23 Sun Dec 23 16:00:00 2018 PST | 55.0000000000000000 | 8 SELECT view_name, materialized_only, compression_enabled FROM timescaledb_information.continuous_aggregates where view_name = 'test_morecols_cagg'; view_name | materialized_only | compression_enabled --------------------+-------------------+--------------------- test_morecols_cagg | f | t --should keep compressed option, modify only materialized -- ALTER MATERIALIZED VIEW test_morecols_cagg SET (timescaledb.materialized_only='true'); SELECT view_name, materialized_only, compression_enabled FROM timescaledb_information.continuous_aggregates where view_name = 'test_morecols_cagg'; view_name | materialized_only | compression_enabled --------------------+-------------------+--------------------- test_morecols_cagg | t | t CREATE TABLE issue3248(filler_1 int, filler_2 int, filler_3 int, time timestamptz NOT NULL, device_id int, v0 int, v1 int, v2 float, v3 float); CREATE INDEX ON issue3248(time DESC); CREATE INDEX ON issue3248(device_id,time DESC); SELECT create_hypertable('issue3248','time',create_default_indexes:=false); create_hypertable ------------------------- (46,public,issue3248,t) ALTER TABLE issue3248 DROP COLUMN filler_1; INSERT INTO issue3248(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id+1, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-01 0:00:00+0'::timestamptz,'2000-01-05 23:55:00+0','8h') gtime(time), generate_series(1,5,1) gdevice(device_id); ALTER TABLE issue3248 DROP COLUMN filler_2; INSERT INTO issue3248(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id-1, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-06 0:00:00+0'::timestamptz,'2000-01-12 23:55:00+0','8h') gtime(time), generate_series(1,5,1) gdevice(device_id); ALTER TABLE issue3248 DROP COLUMN filler_3; INSERT INTO issue3248(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-13 0:00:00+0'::timestamptz,'2000-01-19 23:55:00+0','8h') gtime(time), generate_series(1,5,1) gdevice(device_id); ANALYZE issue3248; CREATE materialized view issue3248_cagg WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h',time), device_id, min(v0), max(v1), avg(v2) FROM issue3248 GROUP BY 1,2; NOTICE: refreshing continuous aggregate "issue3248_cagg" SELECT FROM issue3248 AS m, LATERAL(SELECT m FROM issue3248_cagg WHERE avg IS NULL LIMIT 1) AS lat; -- -- test that option create_group_indexes is taken into account CREATE TABLE test_group_idx ( time timestamptz, symbol int, value numeric ); select create_hypertable('test_group_idx', 'time'); create_hypertable ------------------------------ (48,public,test_group_idx,t) insert into test_group_idx select t, round(random()*10), random()*5 from generate_series('2020-01-01', '2020-02-25', INTERVAL '12 hours') t; create materialized view cagg_index_true with (timescaledb.continuous, timescaledb.materialized_only=false, timescaledb.create_group_indexes=true) as select time_bucket('1 day', "time") as bucket, sum(value), symbol from test_group_idx group by bucket, symbol; NOTICE: refreshing continuous aggregate "cagg_index_true" create materialized view cagg_index_false with (timescaledb.continuous, timescaledb.materialized_only=false, timescaledb.create_group_indexes=false) as select time_bucket('1 day', "time") as bucket, sum(value), symbol from test_group_idx group by bucket, symbol; NOTICE: refreshing continuous aggregate "cagg_index_false" create materialized view cagg_index_default with (timescaledb.continuous, timescaledb.materialized_only=false) as select time_bucket('1 day', "time") as bucket, sum(value), symbol from test_group_idx group by bucket, symbol; NOTICE: refreshing continuous aggregate "cagg_index_default" -- see corresponding materialization_hypertables select view_name, materialization_hypertable_name from timescaledb_information.continuous_aggregates ca where view_name like 'cagg_index_%' ORDER BY view_name; view_name | materialization_hypertable_name --------------------+--------------------------------- cagg_index_default | _materialized_hypertable_51 cagg_index_false | _materialized_hypertable_50 cagg_index_true | _materialized_hypertable_49 -- now make sure a group index has been created when explicitly asked for \x on select i.* from pg_indexes i join pg_class c on schemaname = relnamespace::regnamespace::text and tablename = relname where tablename in (select materialization_hypertable_name from timescaledb_information.continuous_aggregates where view_name like 'cagg_index_%') order by tablename, indexname; -[ RECORD 1 ]------------------------------------------------------------------------------------------------------------------------------------------------- schemaname | _timescaledb_internal tablename | _materialized_hypertable_49 indexname | _materialized_hypertable_49_bucket_idx tablespace | indexdef | CREATE INDEX _materialized_hypertable_49_bucket_idx ON _timescaledb_internal._materialized_hypertable_49 USING btree (bucket DESC) -[ RECORD 2 ]------------------------------------------------------------------------------------------------------------------------------------------------- schemaname | _timescaledb_internal tablename | _materialized_hypertable_49 indexname | _materialized_hypertable_49_symbol_bucket_idx tablespace | indexdef | CREATE INDEX _materialized_hypertable_49_symbol_bucket_idx ON _timescaledb_internal._materialized_hypertable_49 USING btree (symbol, bucket DESC) -[ RECORD 3 ]------------------------------------------------------------------------------------------------------------------------------------------------- schemaname | _timescaledb_internal tablename | _materialized_hypertable_50 indexname | _materialized_hypertable_50_bucket_idx tablespace | indexdef | CREATE INDEX _materialized_hypertable_50_bucket_idx ON _timescaledb_internal._materialized_hypertable_50 USING btree (bucket DESC) -[ RECORD 4 ]------------------------------------------------------------------------------------------------------------------------------------------------- schemaname | _timescaledb_internal tablename | _materialized_hypertable_51 indexname | _materialized_hypertable_51_bucket_idx tablespace | indexdef | CREATE INDEX _materialized_hypertable_51_bucket_idx ON _timescaledb_internal._materialized_hypertable_51 USING btree (bucket DESC) -[ RECORD 5 ]------------------------------------------------------------------------------------------------------------------------------------------------- schemaname | _timescaledb_internal tablename | _materialized_hypertable_51 indexname | _materialized_hypertable_51_symbol_bucket_idx tablespace | indexdef | CREATE INDEX _materialized_hypertable_51_symbol_bucket_idx ON _timescaledb_internal._materialized_hypertable_51 USING btree (symbol, bucket DESC) \x off -- -- TESTs for removing old CAggs restrictions -- DROP TABLE conditions CASCADE; NOTICE: drop cascades to 8 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_29_67_chunk NOTICE: drop cascades to table _timescaledb_internal._hyper_30_68_chunk NOTICE: drop cascades to table _timescaledb_internal._hyper_31_69_chunk CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL ); SELECT create_hypertable('conditions', 'timec'); create_hypertable -------------------------- (52,public,conditions,t) INSERT INTO conditions VALUES ('2010-01-01 09:00:00-08', 'SFO', 55, 45), ('2010-01-02 09:00:00-08', 'por', 100, 100), ('2010-01-02 09:00:00-08', 'NYC', 65, 45), ('2010-01-02 09:00:00-08', 'SFO', 65, 45), ('2010-01-03 09:00:00-08', 'NYC', 45, 55), ('2010-01-05 09:00:00-08', 'SFO', 75, 100), ('2018-11-01 09:00:00-08', 'NYC', 45, 35), ('2018-11-02 09:00:00-08', 'NYC', 35, 15), ('2018-11-03 09:00:00-08', 'NYC', 35, 25); -- aggregate with DISTINCT CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), COUNT(location), SUM(DISTINCT temperature) FROM conditions GROUP BY time_bucket('1week', timec), location; NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1, 2, 3; time_bucket | count | sum ------------------------------+-------+----- Sun Dec 27 16:00:00 2009 PST | 1 | 100 Sun Dec 27 16:00:00 2009 PST | 2 | 110 Sun Dec 27 16:00:00 2009 PST | 2 | 120 Sun Jan 03 16:00:00 2010 PST | 1 | 75 Sun Oct 28 17:00:00 2018 PDT | 3 | 80 -- aggregate with FILTER DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), SUM(temperature) FILTER (WHERE humidity > 60) FROM conditions GROUP BY time_bucket('1week', timec), location; NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1, 2; time_bucket | sum ------------------------------+----- Sun Dec 27 16:00:00 2009 PST | 100 Sun Dec 27 16:00:00 2009 PST | Sun Dec 27 16:00:00 2009 PST | Sun Jan 03 16:00:00 2010 PST | 75 Sun Oct 28 17:00:00 2018 PDT | -- aggregate with filter in having clause DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), MAX(temperature) FROM conditions GROUP BY time_bucket('1week', timec), location HAVING SUM(temperature) FILTER (WHERE humidity > 40) > 50; NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1, 2; time_bucket | max ------------------------------+----- Sun Dec 27 16:00:00 2009 PST | 65 Sun Dec 27 16:00:00 2009 PST | 65 Sun Dec 27 16:00:00 2009 PST | 100 Sun Jan 03 16:00:00 2010 PST | 75 -- ordered set aggr DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to table _timescaledb_internal._hyper_55_116_chunk CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), MODE() WITHIN GROUP(ORDER BY humidity) FROM conditions GROUP BY time_bucket('1week', timec); NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1; time_bucket | mode ------------------------------+------ Sun Dec 27 16:00:00 2009 PST | 45 Sun Jan 03 16:00:00 2010 PST | 100 Sun Oct 28 17:00:00 2018 PDT | 15 -- hypothetical-set aggr DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), RANK(60) WITHIN GROUP (ORDER BY humidity), DENSE_RANK(60) WITHIN GROUP (ORDER BY humidity), PERCENT_RANK(60) WITHIN GROUP (ORDER BY humidity) FROM conditions GROUP BY time_bucket('1week', timec); NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1; time_bucket | rank | dense_rank | percent_rank ------------------------------+------+------------+-------------- Sun Dec 27 16:00:00 2009 PST | 5 | 3 | 0.8 Sun Jan 03 16:00:00 2010 PST | 1 | 1 | 0 Sun Oct 28 17:00:00 2018 PDT | 4 | 4 | 1 -- userdefined aggregate without combine function DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE AGGREGATE newavg ( sfunc = int4_avg_accum, basetype = int4, stype = _int8, finalfunc = int8_avg, initcond1 = '{0,0}' ); CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT SUM(humidity), round(newavg(temperature::int4)) FROM conditions GROUP BY time_bucket('1week', timec), location ORDER BY 1,2; NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1, 2; sum | round -----+------- 75 | 38 90 | 60 100 | 55 100 | 75 100 | 100 -- ORDER BY in the view definition DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), COUNT(location), SUM(temperature) FROM conditions GROUP BY time_bucket('1week', timec) ORDER BY sum DESC; NOTICE: refreshing continuous aggregate "mat_m1" -- CAgg definition for realtime SELECT pg_get_viewdef('mat_m1',true); pg_get_viewdef ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ( SELECT _materialized_hypertable_59.time_bucket, + _materialized_hypertable_59.count, + _materialized_hypertable_59.sum + FROM _timescaledb_internal._materialized_hypertable_59 + WHERE _materialized_hypertable_59.time_bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(59)), '-infinity'::timestamp with time zone)+ ORDER BY _materialized_hypertable_59.sum DESC) + UNION ALL + ( SELECT time_bucket('@ 7 days'::interval, conditions.timec) AS time_bucket, + count(conditions.location) AS count, + sum(conditions.temperature) AS sum + FROM conditions + WHERE conditions.timec >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(59)), '-infinity'::timestamp with time zone) + GROUP BY (time_bucket('@ 7 days'::interval, conditions.timec)) + ORDER BY (sum(conditions.temperature)) DESC) + ORDER BY 3 DESC; -- Ordered result SELECT * FROM mat_m1; time_bucket | count | sum ------------------------------+-------+----- Sun Dec 27 16:00:00 2009 PST | 5 | 330 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Jan 03 16:00:00 2010 PST | 1 | 75 -- Insert new data and query again to make sure we produce ordered data INSERT INTO conditions VALUES ('2018-11-10 09:00:00-08', 'SFO', 10, 10); SELECT * FROM mat_m1; time_bucket | count | sum ------------------------------+-------+----- Sun Dec 27 16:00:00 2009 PST | 5 | 330 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Jan 03 16:00:00 2010 PST | 1 | 75 Sun Nov 04 16:00:00 2018 PST | 1 | 10 -- This new row will change the order again INSERT INTO conditions VALUES ('2018-11-11 09:00:00-08', 'SFO', 400, 400); SELECT * FROM mat_m1; time_bucket | count | sum ------------------------------+-------+----- Sun Nov 04 16:00:00 2018 PST | 2 | 410 Sun Dec 27 16:00:00 2009 PST | 5 | 330 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Jan 03 16:00:00 2010 PST | 1 | 75 -- Merge Append EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM mat_m1; --- QUERY PLAN --- Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Index Scan Backward using _hyper_59_123_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_123_chunk -> Index Scan Backward using _hyper_59_124_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_124_chunk Index Cond: (time_bucket < 'Sun Nov 04 16:00:00 2018 PST'::timestamp with time zone) -> Sort Sort Key: (sum(conditions.temperature)) DESC -> Finalize HashAggregate Group Key: (time_bucket('@ 7 days'::interval, conditions.timec)) -> Append -> Partial HashAggregate Group Key: time_bucket('@ 7 days'::interval, _hyper_52_111_chunk.timec) -> Index Scan Backward using _hyper_52_111_chunk_conditions_timec_idx on _hyper_52_111_chunk Index Cond: (timec >= 'Sun Nov 04 16:00:00 2018 PST'::timestamp with time zone) -> Partial HashAggregate Group Key: time_bucket('@ 7 days'::interval, _hyper_52_125_chunk.timec) -> Seq Scan on _hyper_52_125_chunk -- Ordering by another column SELECT * FROM mat_m1 ORDER BY count; time_bucket | count | sum ------------------------------+-------+----- Sun Jan 03 16:00:00 2010 PST | 1 | 75 Sun Nov 04 16:00:00 2018 PST | 2 | 410 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Dec 27 16:00:00 2009 PST | 5 | 330 EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM mat_m1 ORDER BY count; --- QUERY PLAN --- Sort Sort Key: _materialized_hypertable_59.count -> Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Index Scan Backward using _hyper_59_123_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_123_chunk -> Index Scan Backward using _hyper_59_124_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_124_chunk Index Cond: (time_bucket < 'Sun Nov 04 16:00:00 2018 PST'::timestamp with time zone) -> Sort Sort Key: (sum(conditions.temperature)) DESC -> Finalize HashAggregate Group Key: (time_bucket('@ 7 days'::interval, conditions.timec)) -> Append -> Partial HashAggregate Group Key: time_bucket('@ 7 days'::interval, _hyper_52_111_chunk.timec) -> Index Scan Backward using _hyper_52_111_chunk_conditions_timec_idx on _hyper_52_111_chunk Index Cond: (timec >= 'Sun Nov 04 16:00:00 2018 PST'::timestamp with time zone) -> Partial HashAggregate Group Key: time_bucket('@ 7 days'::interval, _hyper_52_125_chunk.timec) -> Seq Scan on _hyper_52_125_chunk -- Change the type of cagg ALTER MATERIALIZED VIEW mat_m1 SET (timescaledb.materialized_only=true); -- CAgg definition for materialized only SELECT pg_get_viewdef('mat_m1',true); pg_get_viewdef ----------------------------------------------------------- SELECT time_bucket, + count, + sum + FROM _timescaledb_internal._materialized_hypertable_59+ ORDER BY sum DESC; -- Now the query will show only the materialized data, without last two -- records inserted into the original hypertable (last two insers above) SELECT * FROM mat_m1; time_bucket | count | sum ------------------------------+-------+----- Sun Dec 27 16:00:00 2009 PST | 5 | 330 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Jan 03 16:00:00 2010 PST | 1 | 75 -- Merge Append EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM mat_m1; --- QUERY PLAN --- Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Index Scan Backward using _hyper_59_123_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_123_chunk -> Index Scan Backward using _hyper_59_124_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_124_chunk -- Ordering by another column SELECT * FROM mat_m1 ORDER BY count; time_bucket | count | sum ------------------------------+-------+----- Sun Jan 03 16:00:00 2010 PST | 1 | 75 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Dec 27 16:00:00 2009 PST | 5 | 330 EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM mat_m1 ORDER BY count; --- QUERY PLAN --- Sort Sort Key: _materialized_hypertable_59.count -> Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Index Scan Backward using _hyper_59_123_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_123_chunk -> Index Scan Backward using _hyper_59_124_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_124_chunk SELECT h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset -- Invalidate old region and refresh again DELETE FROM conditions WHERE timec < '2010-01-05 09:00:00-08'; CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); -- Querying the cagg produce ordered records as expected SELECT * FROM mat_m1; time_bucket | count | sum ------------------------------+-------+----- Sun Nov 04 16:00:00 2018 PST | 2 | 410 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Jan 03 16:00:00 2010 PST | 1 | 75 -- Querying direct the materialization hypertable doesn't -- produce ordered records SELECT * FROM :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME"; time_bucket | count | sum ------------------------------+-------+----- Sun Jan 03 16:00:00 2010 PST | 1 | 75 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Nov 04 16:00:00 2018 PST | 2 | 410 DROP TABLE conditions CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL ); SELECT table_name FROM create_hypertable('conditions', 'timec'); table_name ------------ conditions INSERT INTO conditions VALUES ('2010-01-01 09:00:00-08', 'SFO', 55, 45), ('2010-01-02 09:00:00-08', 'por', 100, 100), ('2010-01-02 09:00:00-08', 'SFO', 65, 45), ('2010-01-02 09:00:00-08', 'NYC', 65, 45), ('2018-11-01 09:00:00-08', 'NYC', 45, 35), ('2018-11-02 09:00:00-08', 'NYC', 35, 15); CREATE MATERIALIZED VIEW conditions_summary_new(timec, minl, sumt, sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1day', timec), min(location), sum(temperature), sum(humidity) FROM conditions GROUP BY time_bucket('1day', timec) WITH NO DATA; \x ON SELECT * FROM timescaledb_information.continuous_aggregates WHERE view_name = 'conditions_summary_new'; -[ RECORD 1 ]---------------------+---------------------------------------------------------- hypertable_schema | public hypertable_name | conditions view_schema | public view_name | conditions_summary_new view_owner | default_perm_user materialized_only | t compression_enabled | f materialization_hypertable_schema | _timescaledb_internal materialization_hypertable_name | _materialized_hypertable_61 view_definition | SELECT time_bucket('@ 1 day'::interval, timec) AS timec,+ | min(location) AS minl, + | sum(temperature) AS sumt, + | sum(humidity) AS sumh + | FROM conditions + | GROUP BY (time_bucket('@ 1 day'::interval, timec)); \x OFF CALL refresh_continuous_aggregate('conditions_summary_new', NULL, NULL); -- Check and compare number of returned rows SELECT count(*) FROM conditions_summary_new; count ------- 4 -- Parallel planning test for realtime Continuous Aggregate DROP TABLE conditions CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, temperature DOUBLE PRECISION NULL ); SELECT table_name FROM create_hypertable('conditions', 'timec'); table_name ------------ conditions INSERT INTO conditions SELECT t, 10 FROM generate_series('2023-01-01 00:00-03'::timestamptz, '2023-12-31 23:59-03'::timestamptz, '1 hour'::interval) AS t; CREATE MATERIALIZED VIEW conditions_daily WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1 day', timec), SUM(temperature) FROM conditions GROUP BY 1 ORDER BY 2 DESC; NOTICE: refreshing continuous aggregate "conditions_daily" SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false); set_config ------------ on SET max_parallel_workers_per_gather = 4; SET parallel_setup_cost = 0; SET parallel_tuple_cost = 0; -- Parallel planning EXPLAIN (BUFFERS OFF, COSTS OFF, TIMING OFF) SELECT * FROM conditions_daily WHERE time_bucket >= '2023-07-01'; --- QUERY PLAN --- Merge Append Sort Key: _materialized_hypertable_63.sum DESC -> Gather Merge Workers Planned: 2 -> Sort Sort Key: _materialized_hypertable_63.sum DESC -> Parallel Append -> Parallel Index Scan using _hyper_63_185_chunk__materialized_hypertable_63_time_bucket_idx on _hyper_63_185_chunk Index Cond: ((time_bucket < 'Mon Jan 01 16:00:00 2024 PST'::timestamp with time zone) AND (time_bucket >= 'Sat Jul 01 00:00:00 2023 PDT'::timestamp with time zone)) -> Parallel Index Scan using _hyper_63_187_chunk__materialized_hypertable_63_time_bucket_idx on _hyper_63_187_chunk Index Cond: ((time_bucket < 'Mon Jan 01 16:00:00 2024 PST'::timestamp with time zone) AND (time_bucket >= 'Sat Jul 01 00:00:00 2023 PDT'::timestamp with time zone)) -> Parallel Seq Scan on _hyper_63_184_chunk -> Sort Sort Key: (sum(_hyper_62_182_chunk.temperature)) DESC -> HashAggregate Group Key: (time_bucket('@ 1 day'::interval, _hyper_62_182_chunk.timec)) -> Gather Workers Planned: 1 -> Result -> Parallel Index Scan Backward using _hyper_62_182_chunk_conditions_timec_idx on _hyper_62_182_chunk Index Cond: ((timec >= 'Mon Jan 01 16:00:00 2024 PST'::timestamp with time zone) AND (timec >= 'Sat Jul 01 00:00:00 2023 PDT'::timestamp with time zone)) Filter: (time_bucket('@ 1 day'::interval, timec) >= 'Sat Jul 01 00:00:00 2023 PDT'::timestamp with time zone) ================================================ FILE: tsl/test/expected/cagg-17.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- initialize the bgw mock state to prevent the materialization workers from running \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION ts_bgw_params_create() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION test.continuous_aggs_find_view(cagg REGCLASS) RETURNS VOID AS :TSL_MODULE_PATHNAME, 'ts_test_continuous_agg_find_by_view_name' LANGUAGE C; \set WAIT_ON_JOB 0 \set IMMEDIATELY_SET_UNTIL 1 \set WAIT_FOR_OTHER_TO_ADVANCE 2 -- remove any default jobs, e.g., telemetry so bgw_job isn't polluted DELETE FROM _timescaledb_catalog.bgw_job; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT * FROM _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- --TEST1 --- --basic test with count create table foo (a integer, b integer, c integer); select table_name from create_hypertable('foo', 'a', chunk_time_interval=> 10); table_name ------------ foo insert into foo values( 3 , 16 , 20); insert into foo values( 1 , 10 , 20); insert into foo values( 1 , 11 , 20); insert into foo values( 1 , 12 , 20); insert into foo values( 1 , 13 , 20); insert into foo values( 1 , 14 , 20); insert into foo values( 2 , 14 , 20); insert into foo values( 2 , 15 , 20); insert into foo values( 2 , 16 , 20); CREATE OR REPLACE FUNCTION integer_now_foo() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(a), 0) FROM foo $$; SELECT set_integer_now_func('foo', 'integer_now_foo'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW mat_m1(a, countb) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select a, count(b) from foo group by time_bucket(1, a), a WITH NO DATA; SELECT add_continuous_aggregate_policy('mat_m1', NULL, 2::integer, '12 h'::interval) AS job_id \gset SELECT * FROM _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ------+--------------------------------------------+-------------------+-------------+-------------+--------------+------------------------+-------------------------------------+-------------------+-----------+----------------+---------------+---------------+-----------------------------------------------------------------+------------------------+-------------------------------------------+---------- 1000 | Refresh Continuous Aggregate Policy [1000] | @ 12 hours | @ 0 | -1 | @ 12 hours | _timescaledb_functions | policy_refresh_continuous_aggregate | default_perm_user | t | f | | 2 | {"end_offset": 2, "start_offset": null, "mat_hypertable_id": 2} | _timescaledb_functions | policy_refresh_continuous_aggregate_check | SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select a, count(b), time_bucket(1, a) from foo group by time_bucket(1, a) , a ; select * from mat_m1 order by a ; a | countb ---+-------- 1 | 5 2 | 3 3 | 1 --check triggers on user hypertable -- SET ROLE :ROLE_SUPERUSER; select tgname, tgtype, tgenabled , relname from pg_trigger, pg_class where tgrelid = pg_class.oid and pg_class.relname like 'foo' order by tgname; tgname | tgtype | tgenabled | relname --------+--------+-----------+--------- SET ROLE :ROLE_DEFAULT_PERM_USER; -- TEST2 --- DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to table _timescaledb_internal._hyper_2_2_chunk SHOW enable_partitionwise_aggregate; enable_partitionwise_aggregate -------------------------------- off SET enable_partitionwise_aggregate = on; SELECT * FROM _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions insert into conditions values ( '2010-01-01 09:00:00-08', 'SFO', 55, 45); insert into conditions values ( '2010-01-02 09:00:00-08', 'por', 100, 100); insert into conditions values ( '2010-01-02 09:00:00-08', 'SFO', 65, 45); insert into conditions values ( '2010-01-02 09:00:00-08', 'NYC', 65, 45); insert into conditions values ( '2018-11-01 09:00:00-08', 'NYC', 45, 35); insert into conditions values ( '2018-11-02 09:00:00-08', 'NYC', 35, 15); CREATE MATERIALIZED VIEW mat_m1( timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1day', timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket('1day', timec) WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset -- Materialized hypertable for mat_m1 should not be visible in the -- hypertables view: SELECT hypertable_schema, hypertable_name FROM timescaledb_information.hypertables ORDER BY 1,2; hypertable_schema | hypertable_name -------------------+----------------- public | conditions public | foo SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select time_bucket('1day', timec), min(location), sum(temperature), sum(humidity) from conditions group by time_bucket('1day', timec) ; SET ROLE :ROLE_DEFAULT_PERM_USER; --should have same results -- select timec, minl, sumt, sumh from mat_m1 order by timec; timec | minl | sumt | sumh ------------------------------+------+------+------ Thu Dec 31 16:00:00 2009 PST | SFO | 55 | 45 Fri Jan 01 16:00:00 2010 PST | NYC | 230 | 190 Wed Oct 31 17:00:00 2018 PDT | NYC | 45 | 35 Thu Nov 01 17:00:00 2018 PDT | NYC | 35 | 15 select time_bucket('1day', timec), min(location), sum(temperature), sum(humidity) from conditions group by time_bucket('1day', timec) order by 1; time_bucket | min | sum | sum ------------------------------+-----+-----+----- Thu Dec 31 16:00:00 2009 PST | SFO | 55 | 45 Fri Jan 01 16:00:00 2010 PST | NYC | 230 | 190 Wed Oct 31 17:00:00 2018 PDT | NYC | 45 | 35 Thu Nov 01 17:00:00 2018 PDT | NYC | 35 | 15 SET enable_partitionwise_aggregate = off; -- TEST3 -- -- drop on table conditions should cascade to materialized mat_v1 drop table conditions cascade; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions insert into conditions values ( '2010-01-01 09:00:00-08', 'SFO', 55, 45); insert into conditions values ( '2010-01-02 09:00:00-08', 'por', 100, 100); insert into conditions values ( '2010-01-02 09:00:00-08', 'NYC', 65, 45); insert into conditions values ( '2010-01-02 09:00:00-08', 'SFO', 65, 45); insert into conditions values ( '2010-01-03 09:00:00-08', 'NYC', 45, 55); insert into conditions values ( '2010-01-05 09:00:00-08', 'SFO', 75, 100); insert into conditions values ( '2018-11-01 09:00:00-08', 'NYC', 45, 35); insert into conditions values ( '2018-11-02 09:00:00-08', 'NYC', 35, 15); insert into conditions values ( '2018-11-03 09:00:00-08', 'NYC', 35, 25); CREATE MATERIALIZED VIEW mat_m1( timec, minl, sumth, stddevh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select time_bucket('1week', timec), min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) ; SET ROLE :ROLE_DEFAULT_PERM_USER; --should have same results -- select timec, minl, sumth, stddevh from mat_m1 order by timec; timec | minl | sumth | stddevh ------------------------------+------+-------+------------------ Sun Dec 27 16:00:00 2009 PST | NYC | 620 | 23.8746727726266 Sun Jan 03 16:00:00 2010 PST | SFO | 175 | Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 select time_bucket('1week', timec) , min(location), sum(temperature)+ sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) order by time_bucket('1week', timec); time_bucket | min | ?column? | stddev ------------------------------+-----+----------+------------------ Sun Dec 27 16:00:00 2009 PST | NYC | 620 | 23.8746727726266 Sun Jan 03 16:00:00 2010 PST | SFO | 175 | Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 -- TEST4 -- --materialized view with group by clause + expression in SELECT -- use previous data from conditions --drop only the view. -- apply where clause on result of mat_m1 -- DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW mat_m1( timec, minl, sumth, stddevh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions where location = 'NYC' group by time_bucket('1week', timec) WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select time_bucket('1week', timec), min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions where location = 'NYC' group by time_bucket('1week', timec) ; SET ROLE :ROLE_DEFAULT_PERM_USER; --should have same results -- select timec, minl, sumth, stddevh from mat_m1 where stddevh is not null order by timec; timec | minl | sumth | stddevh ------------------------------+------+-------+------------------ Sun Dec 27 16:00:00 2009 PST | NYC | 210 | 7.07106781186548 Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 select time_bucket('1week', timec) , min(location), sum(temperature)+ sum(humidity), stddev(humidity) from conditions where location = 'NYC' group by time_bucket('1week', timec) order by time_bucket('1week', timec); time_bucket | min | ?column? | stddev ------------------------------+-----+----------+------------------ Sun Dec 27 16:00:00 2009 PST | NYC | 210 | 7.07106781186548 Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 -- TEST5 -- ---------test with having clause ---------------------- DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects create materialized view mat_m1( timec, minl, sumth, stddevh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having stddev(humidity) is not null WITH NO DATA; ; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select time_bucket('1week', timec), min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having stddev(humidity) is not null; SET ROLE :ROLE_DEFAULT_PERM_USER; -- should have same results -- select * from mat_m1 order by sumth; timec | minl | sumth | stddevh ------------------------------+------+-------+------------------ Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 Sun Dec 27 16:00:00 2009 PST | NYC | 620 | 23.8746727726266 select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having stddev(humidity) is not null order by sum(temperature)+sum(humidity); time_bucket | min | ?column? | stddev ------------------------------+-----+----------+------------------ Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 Sun Dec 27 16:00:00 2009 PST | NYC | 620 | 23.8746727726266 -- TEST6 -- --group by with more than 1 group column -- having clause with a mix of columns from select list + others drop table conditions cascade; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp numeric NULL, highp numeric null ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions insert into conditions select generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'POR', 55, 75, 40, 70; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'NYC', 35, 45, 50, 40; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-15 00:00'::timestamp, '1 day'), 'LA', 73, 55, 71, 28; --naming with AS clauses CREATE MATERIALIZED VIEW mat_naming WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec) as bucket, location as loc, sum(temperature)+sum(humidity) as sumth, stddev(humidity) from conditions group by bucket, loc having min(location) >= 'NYC' and avg(temperature) > 20 WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_naming' \gset select attnum , attname from pg_attribute where attnum > 0 and attrelid = (Select oid from pg_class where relname like :'MAT_TABLE_NAME') order by attnum, attname; attnum | attname --------+--------- 1 | bucket 2 | loc 3 | sumth 4 | stddev DROP MATERIALIZED VIEW mat_naming; --naming with default names CREATE MATERIALIZED VIEW mat_naming WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec), location, sum(temperature)+sum(humidity) as sumth, stddev(humidity) from conditions group by 1,2 having min(location) >= 'NYC' and avg(temperature) > 20 WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_naming' \gset select attnum , attname from pg_attribute where attnum > 0 and attrelid = (Select oid from pg_class where relname like :'MAT_TABLE_NAME') order by attnum, attname; attnum | attname --------+------------- 1 | time_bucket 2 | location 3 | sumth 4 | stddev DROP MATERIALIZED VIEW mat_naming; --naming with view col names CREATE MATERIALIZED VIEW mat_naming(bucket, loc, sum_t_h, stdd) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec), location, sum(temperature)+sum(humidity), stddev(humidity) from conditions group by 1,2 having min(location) >= 'NYC' and avg(temperature) > 20 WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_naming' \gset select attnum , attname from pg_attribute where attnum > 0 and attrelid = (Select oid from pg_class where relname like :'MAT_TABLE_NAME') order by attnum, attname; attnum | attname --------+--------- 1 | bucket 2 | loc 3 | sum_t_h 4 | stdd DROP MATERIALIZED VIEW mat_naming; CREATE MATERIALIZED VIEW mat_m1(timec, minl, sumth, stddevh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having min(location) >= 'NYC' and avg(temperature) > 20 WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset select attnum , attname from pg_attribute where attnum > 0 and attrelid = (Select oid from pg_class where relname like :'MAT_TABLE_NAME') order by attnum, attname; attnum | attname --------+--------- 1 | timec 2 | minl 3 | sumth 4 | stddevh SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select time_bucket('1week', timec), min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having min(location) >= 'NYC' and avg(temperature) > 20; SET ROLE :ROLE_DEFAULT_PERM_USER; --should have same results -- select timec, minl, sumth, stddevh from mat_m1 order by timec, minl; timec | minl | sumth | stddevh ------------------------------+------+-------+------------------ Sun Dec 16 16:00:00 2018 PST | NYC | 1470 | 15.5662356498831 Sun Dec 23 16:00:00 2018 PST | NYC | 1470 | 15.5662356498831 Sun Dec 30 16:00:00 2018 PST | NYC | 210 | 21.2132034355964 select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having min(location) >= 'NYC' and avg(temperature) > 20 and avg(lowp) > 10 order by time_bucket('1week', timec), min(location); time_bucket | min | ?column? | stddev ------------------------------+-----+----------+------------------ Sun Dec 16 16:00:00 2018 PST | NYC | 1470 | 15.5662356498831 Sun Dec 23 16:00:00 2018 PST | NYC | 1470 | 15.5662356498831 Sun Dec 30 16:00:00 2018 PST | NYC | 210 | 21.2132034355964 --check view defintion in information views select view_name, view_definition from timescaledb_information.continuous_aggregates where view_name::text like 'mat_m1'; view_name | view_definition -----------+------------------------------------------------------------------------------------------- mat_m1 | SELECT time_bucket('@ 7 days'::interval, timec) AS timec, + | min(location) AS minl, + | (sum(temperature) + sum(humidity)) AS sumth, + | stddev(humidity) AS stddevh + | FROM conditions + | GROUP BY (time_bucket('@ 7 days'::interval, timec)) + | HAVING ((min(location) >= 'NYC'::text) AND (avg(temperature) > (20)::double precision)); --TEST6 -- select from internal view SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select * from :"PART_VIEW_SCHEMA".:"PART_VIEW_NAME"; SET ROLE :ROLE_DEFAULT_PERM_USER; --lets drop the view and check DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to table _timescaledb_internal._hyper_13_24_chunk drop table conditions; CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions insert into conditions select generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'POR', 55, 75, 40, 70, NULL; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'NYC', 35, 45, 50, 40, NULL; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-15 00:00'::timestamp, '1 day'), 'LA', 73, 55, NULL, 28, NULL; SELECT $$ select time_bucket('1week', timec) , min(location) as col1, sum(temperature)+sum(humidity) as col2, stddev(humidity) as col3, min(allnull) as col4 from conditions group by time_bucket('1week', timec) having min(location) >= 'NYC' and avg(temperature) > 20 $$ AS "QUERY" \gset \set ECHO errors psql:include/cont_agg_equal.sql:8: NOTICE: materialized view "mat_test" does not exist, skipping ?column? | count ---------------------------------------------------------------+------- Number of rows different between view and original (expect 0) | 0 SELECT $$ select time_bucket('1week', timec), location, sum(temperature)+sum(humidity) as col2, stddev(humidity) as col3, min(allnull) as col4 from conditions group by location, time_bucket('1week', timec) $$ AS "QUERY" \gset \set ECHO errors psql:include/cont_agg_equal.sql:8: NOTICE: drop cascades to table _timescaledb_internal._hyper_15_34_chunk ?column? | count ---------------------------------------------------------------+------- Number of rows different between view and original (expect 0) | 0 --TEST7 -- drop tests for view and hypertable --DROP tests \set ON_ERROR_STOP 0 SELECT h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA", direct_view_name as "DIR_VIEW_NAME", direct_view_schema as "DIR_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_test' \gset DROP TABLE :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME"; ERROR: cannot drop table _timescaledb_internal._materialized_hypertable_16 because other objects depend on it DROP VIEW :"PART_VIEW_SCHEMA".:"PART_VIEW_NAME"; ERROR: cannot drop the partial/direct view because it is required by a continuous aggregate DROP VIEW :"DIR_VIEW_SCHEMA".:"DIR_VIEW_NAME"; ERROR: cannot drop the partial/direct view because it is required by a continuous aggregate \set ON_ERROR_STOP 1 --catalog entry still there; SELECT count(*) FROM _timescaledb_catalog.continuous_agg ca WHERE user_view_name = 'mat_test'; count ------- 1 --mat table, user_view, direct view and partial view all there select count(*) from pg_class where relname = :'PART_VIEW_NAME'; count ------- 1 select count(*) from pg_class where relname = :'MAT_TABLE_NAME'; count ------- 1 select count(*) from pg_class where relname = :'DIR_VIEW_NAME'; count ------- 1 select count(*) from pg_class where relname = 'mat_test'; count ------- 1 DROP MATERIALIZED VIEW mat_test; NOTICE: drop cascades to 2 other objects --catalog entry should be gone SELECT count(*) FROM _timescaledb_catalog.continuous_agg ca WHERE user_view_name = 'mat_test'; count ------- 0 --mat table, user_view, direct view and partial view all gone select count(*) from pg_class where relname = :'PART_VIEW_NAME'; count ------- 0 select count(*) from pg_class where relname = :'MAT_TABLE_NAME'; count ------- 0 select count(*) from pg_class where relname = :'DIR_VIEW_NAME'; count ------- 0 select count(*) from pg_class where relname = 'mat_test'; count ------- 0 --test dropping raw table DROP TABLE conditions; CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions --no data in hyper table on purpose so that CASCADE is not required because of chunks CREATE MATERIALIZED VIEW mat_drop_test(timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1day', timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket('1day', timec) WITH NO DATA; \set ON_ERROR_STOP 0 DROP TABLE conditions; ERROR: cannot drop table conditions because other objects depend on it \set ON_ERROR_STOP 1 --insert data now insert into conditions select generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'POR', 55, 75, 40, 70, NULL; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'NYC', 35, 45, 50, 40, NULL; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-15 00:00'::timestamp, '1 day'), 'LA', 73, 55, NULL, 28, NULL; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_drop_test' \gset SET client_min_messages TO NOTICE; CALL refresh_continuous_aggregate('mat_drop_test', NULL, NULL); --force invalidation insert into conditions select generate_series('2017-11-01 00:00'::timestamp, '2017-12-15 00:00'::timestamp, '1 day'), 'LA', 73, 55, NULL, 28, NULL; select count(*) from _timescaledb_catalog.continuous_aggs_invalidation_threshold; count ------- 1 select count(*) from _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log; count ------- 8 DROP TABLE conditions CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects --catalog entry should be gone SELECT count(*) FROM _timescaledb_catalog.continuous_agg ca WHERE user_view_name = 'mat_drop_test'; count ------- 0 select count(*) from _timescaledb_catalog.continuous_aggs_invalidation_threshold; count ------- 0 select count(*) from _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log; count ------- 0 select count(*) from _timescaledb_catalog.continuous_aggs_materialization_invalidation_log; count ------- 0 SELECT * FROM _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- --mat table, user_view, and partial view all gone select count(*) from pg_class where relname = :'PART_VIEW_NAME'; count ------- 0 select count(*) from pg_class where relname = :'MAT_TABLE_NAME'; count ------- 0 select count(*) from pg_class where relname = 'mat_drop_test'; count ------- 0 --TEST With options CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions CREATE MATERIALIZED VIEW mat_with_test(timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1day', timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket('1day', timec), location, humidity, temperature WITH NO DATA; SELECT add_continuous_aggregate_policy('mat_with_test', NULL, '5 h'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1001 SELECT alter_job(id, schedule_interval => '1h') FROM _timescaledb_catalog.bgw_job; alter_job ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ (1001,"@ 1 hour","@ 0",-1,"@ 12 hours",t,"{""end_offset"": ""@ 5 hours"", ""start_offset"": null, ""mat_hypertable_id"": 20}",-infinity,_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1001]") SELECT schedule_interval FROM _timescaledb_catalog.bgw_job; schedule_interval ------------------- @ 1 hour SELECT alter_job(id, schedule_interval => '2h') FROM _timescaledb_catalog.bgw_job; alter_job ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1001,"@ 2 hours","@ 0",-1,"@ 12 hours",t,"{""end_offset"": ""@ 5 hours"", ""start_offset"": null, ""mat_hypertable_id"": 20}",-infinity,_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1001]") SELECT schedule_interval FROM _timescaledb_catalog.bgw_job; schedule_interval ------------------- @ 2 hours select indexname, indexdef from pg_indexes where tablename = (SELECT h.table_name FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_with_test') order by indexname; indexname | indexdef ---------------------------------------+---------------------------------------------------------------------------------------------------------------------------------- _materialized_hypertable_20_timec_idx | CREATE INDEX _materialized_hypertable_20_timec_idx ON _timescaledb_internal._materialized_hypertable_20 USING btree (timec DESC) DROP MATERIALIZED VIEW mat_with_test; --no additional indexes CREATE MATERIALIZED VIEW mat_with_test(timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.create_group_indexes=false) as select time_bucket('1day', timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket('1day', timec), location, humidity, temperature WITH NO DATA; select indexname, indexdef from pg_indexes where tablename = (SELECT h.table_name FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_with_test'); indexname | indexdef ---------------------------------------+---------------------------------------------------------------------------------------------------------------------------------- _materialized_hypertable_21_timec_idx | CREATE INDEX _materialized_hypertable_21_timec_idx ON _timescaledb_internal._materialized_hypertable_21 USING btree (timec DESC) DROP TABLE conditions CASCADE; NOTICE: drop cascades to 2 other objects --test WITH using a hypertable with an integer time dimension CREATE TABLE conditions ( timec INT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec', chunk_time_interval=> 100); table_name ------------ conditions CREATE OR REPLACE FUNCTION integer_now_conditions() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now_conditions'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW mat_with_test(timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket(100, timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket(100, timec) WITH NO DATA; SELECT add_continuous_aggregate_policy('mat_with_test', NULL, 500::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1002 SELECT alter_job(id, schedule_interval => '2h') FROM _timescaledb_catalog.bgw_job; alter_job --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1002,"@ 2 hours","@ 0",-1,"@ 12 hours",t,"{""end_offset"": 500, ""start_offset"": null, ""mat_hypertable_id"": 23}",-infinity,_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1002]") SELECT schedule_interval FROM _timescaledb_catalog.bgw_job; schedule_interval ------------------- @ 2 hours DROP TABLE conditions CASCADE; NOTICE: drop cascades to 2 other objects --test space partitions CREATE TABLE space_table ( time BIGINT, dev BIGINT, data BIGINT ); SELECT create_hypertable( 'space_table', 'time', chunk_time_interval => 10, partitioning_column => 'dev', number_partitions => 3); create_hypertable --------------------------- (24,public,space_table,t) CREATE OR REPLACE FUNCTION integer_now_space_table() returns BIGINT LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), BIGINT '0') FROM space_table $$; SELECT set_integer_now_func('space_table', 'integer_now_space_table'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW space_view WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('4', time), COUNT(data) FROM space_table GROUP BY 1 WITH NO DATA; INSERT INTO space_table VALUES (0, 1, 1), (0, 2, 1), (1, 1, 1), (1, 2, 1), (10, 1, 1), (10, 2, 1), (11, 1, 1), (11, 2, 1); SELECT h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA", direct_view_name as "DIR_VIEW_NAME", direct_view_schema as "DIR_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'space_view' \gset SELECT * FROM :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" ORDER BY time_bucket; time_bucket | count -------------+------- CALL refresh_continuous_aggregate('space_view', NULL, NULL); SELECT * FROM space_view ORDER BY 1; time_bucket | count -------------+------- 0 | 4 8 | 4 SELECT * FROM :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" ORDER BY time_bucket; time_bucket | count -------------+------- 0 | 4 8 | 4 INSERT INTO space_table VALUES (3, 2, 1); CALL refresh_continuous_aggregate('space_view', NULL, NULL); SELECT * FROM space_view ORDER BY 1; time_bucket | count -------------+------- 0 | 5 8 | 4 SELECT * FROM :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" ORDER BY time_bucket; time_bucket | count -------------+------- 0 | 5 8 | 4 INSERT INTO space_table VALUES (2, 3, 1); CALL refresh_continuous_aggregate('space_view', NULL, NULL); SELECT * FROM space_view ORDER BY 1; time_bucket | count -------------+------- 0 | 6 8 | 4 SELECT * FROM :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" ORDER BY time_bucket; time_bucket | count -------------+------- 0 | 6 8 | 4 DROP TABLE space_table CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_25_60_chunk -- -- TEST FINALIZEFUNC_EXTRA -- -- create special aggregate to test ffunc_extra -- Raise warning with the actual type being passed in CREATE OR REPLACE FUNCTION fake_ffunc(a int8, b int, c int, d int, x anyelement) RETURNS anyelement AS $$ BEGIN RAISE WARNING 'type % %', pg_typeof(d), pg_typeof(x); RETURN x; END; $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION fake_sfunc(a int8, b int, c int, d int, x anyelement) RETURNS int8 AS $$ BEGIN RETURN b; END; $$ LANGUAGE plpgsql; CREATE AGGREGATE aggregate_to_test_ffunc_extra(int, int, int, anyelement) ( SFUNC = fake_sfunc, STYPE = int8, COMBINEFUNC = int8pl, FINALFUNC = fake_ffunc, PARALLEL = SAFE, FINALFUNC_EXTRA ); CREATE TABLE conditions ( timec INT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec', chunk_time_interval=> 100); table_name ------------ conditions CREATE OR REPLACE FUNCTION integer_now_conditions() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now_conditions'); set_integer_now_func ---------------------- insert into conditions select generate_series(0, 200, 10), 'POR', 55, 75, 40, 70, NULL; CREATE MATERIALIZED VIEW mat_ffunc_test WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket(100, timec), aggregate_to_test_ffunc_extra(timec, 1, 3, 'test'::text) from conditions group by time_bucket(100, timec); NOTICE: refreshing continuous aggregate "mat_ffunc_test" WARNING: type integer text WARNING: type integer text WARNING: type integer text SELECT * FROM mat_ffunc_test ORDER BY time_bucket; time_bucket | aggregate_to_test_ffunc_extra -------------+------------------------------- 0 | 100 | 200 | DROP MATERIALIZED view mat_ffunc_test; NOTICE: drop cascades to table _timescaledb_internal._hyper_27_65_chunk CREATE MATERIALIZED VIEW mat_ffunc_test WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket(100, timec), aggregate_to_test_ffunc_extra(timec, 4, 5, bigint '123') from conditions group by time_bucket(100, timec); NOTICE: refreshing continuous aggregate "mat_ffunc_test" WARNING: type integer bigint WARNING: type integer bigint WARNING: type integer bigint SELECT * FROM mat_ffunc_test ORDER BY time_bucket; time_bucket | aggregate_to_test_ffunc_extra -------------+------------------------------- 0 | 100 | 200 | --refresh mat view test when time_bucket is not projected -- DROP MATERIALIZED VIEW mat_ffunc_test; NOTICE: drop cascades to table _timescaledb_internal._hyper_28_66_chunk CREATE MATERIALIZED VIEW mat_refresh_test WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select location, max(humidity) from conditions group by time_bucket(100, timec), location WITH NO DATA; insert into conditions select generate_series(0, 50, 10), 'NYC', 55, 75, 40, 70, NULL; CALL refresh_continuous_aggregate('mat_refresh_test', NULL, NULL); SELECT * FROM mat_refresh_test order by 1,2 ; location | max ----------+----- NYC | 75 POR | 75 POR | 75 POR | 75 -- test for bug when group by is not in project list CREATE MATERIALIZED VIEW conditions_grpby_view with (timescaledb.continuous, timescaledb.materialized_only=false) as select time_bucket(100, timec), sum(humidity) from conditions group by time_bucket(100, timec), location; NOTICE: refreshing continuous aggregate "conditions_grpby_view" select * from conditions_grpby_view order by 1, 2; time_bucket | sum -------------+----- 0 | 450 0 | 750 100 | 750 200 | 75 CREATE MATERIALIZED VIEW conditions_grpby_view2 with (timescaledb.continuous, timescaledb.materialized_only=false) as select time_bucket(100, timec), sum(humidity) from conditions group by time_bucket(100, timec), location having avg(temperature) > 0; NOTICE: refreshing continuous aggregate "conditions_grpby_view2" select * from conditions_grpby_view2 order by 1, 2; time_bucket | sum -------------+----- 0 | 450 0 | 750 100 | 750 200 | 75 -- Test internal functions for continuous aggregates SELECT test.continuous_aggs_find_view('mat_refresh_test'); continuous_aggs_find_view --------------------------- -- Test pseudotype/enum handling CREATE TYPE status_enum AS ENUM ( 'red', 'yellow', 'green' ); CREATE TABLE cagg_types ( time TIMESTAMPTZ NOT NULL, status status_enum, names NAME[], floats FLOAT[] ); SELECT table_name FROM create_hypertable('cagg_types', 'time'); table_name ------------ cagg_types INSERT INTO cagg_types SELECT '2000-01-01', 'yellow', '{foo,bar,baz}', '{1,2.5,3}'; CREATE MATERIALIZED VIEW mat_types WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1d', time), min(status) AS status, max(names) AS names, min(floats) AS floats FROM cagg_types GROUP BY 1; NOTICE: refreshing continuous aggregate "mat_types" CALL refresh_continuous_aggregate('mat_types',NULL,NULL); NOTICE: continuous aggregate "mat_types" is already up-to-date SELECT * FROM mat_types; time_bucket | status | names | floats ------------------------------+--------+---------------+----------- Fri Dec 31 16:00:00 1999 PST | yellow | {foo,bar,baz} | {1,2.5,3} ------------------------------------------------------------------------------------- -- Test issue #2616 where cagg view contains an experssion with several aggregates in CREATE TABLE water_consumption ( sensor_id integer NOT NULL, timestamp timestamp(0) NOT NULL, water_index integer ); SELECT create_hypertable('water_consumption', 'timestamp', 'sensor_id', 2); WARNING: column type "timestamp without time zone" used for "timestamp" does not follow best practices create_hypertable --------------------------------- (34,public,water_consumption,t) INSERT INTO public.water_consumption (sensor_id, timestamp, water_index) VALUES (1, '2010-11-03 09:42:30', 1030), (1, '2010-11-03 09:42:40', 1032), (1, '2010-11-03 09:42:50', 1035), (1, '2010-11-03 09:43:30', 1040), (1, '2010-11-03 09:43:40', 1045), (1, '2010-11-03 09:43:50', 1050), (1, '2010-11-03 09:44:30', 1052), (1, '2010-11-03 09:44:40', 1057), (1, '2010-11-03 09:44:50', 1060), (1, '2010-11-03 09:45:30', 1063), (1, '2010-11-03 09:45:40', 1067), (1, '2010-11-03 09:45:50', 1070); -- The test with the view originally reported in the issue. CREATE MATERIALIZED VIEW water_consumption_aggregation_minute WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT sensor_id, time_bucket(INTERVAL '1 minute', timestamp) + '1 minute' AS timestamp, (max(water_index) - min(water_index)) AS water_consumption FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) WITH NO DATA; CALL refresh_continuous_aggregate('water_consumption_aggregation_minute', NULL, NULL); -- The results of the view and the query over hypertable should be the same SELECT * FROM water_consumption_aggregation_minute ORDER BY water_consumption; sensor_id | timestamp | water_consumption -----------+--------------------------+------------------- 1 | Wed Nov 03 09:43:00 2010 | 5 1 | Wed Nov 03 09:46:00 2010 | 7 1 | Wed Nov 03 09:45:00 2010 | 8 1 | Wed Nov 03 09:44:00 2010 | 10 SELECT sensor_id, time_bucket(INTERVAL '1 minute', timestamp) + '1 minute' AS timestamp, (max(water_index) - min(water_index)) AS water_consumption FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) ORDER BY water_consumption; sensor_id | timestamp | water_consumption -----------+--------------------------+------------------- 1 | Wed Nov 03 09:43:00 2010 | 5 1 | Wed Nov 03 09:46:00 2010 | 7 1 | Wed Nov 03 09:45:00 2010 | 8 1 | Wed Nov 03 09:44:00 2010 | 10 -- Simplified test, where the view doesn't contain all group by clauses CREATE MATERIALIZED VIEW water_consumption_no_select_bucket WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT sensor_id, (max(water_index) - min(water_index)) AS water_consumption FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) WITH NO DATA; CALL refresh_continuous_aggregate('water_consumption_no_select_bucket', NULL, NULL); -- The results of the view and the query over hypertable should be the same SELECT * FROM water_consumption_no_select_bucket ORDER BY water_consumption; sensor_id | water_consumption -----------+------------------- 1 | 5 1 | 7 1 | 8 1 | 10 SELECT sensor_id, (max(water_index) - min(water_index)) AS water_consumption FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) ORDER BY water_consumption; sensor_id | water_consumption -----------+------------------- 1 | 5 1 | 7 1 | 8 1 | 10 -- The test with SELECT matching GROUP BY and placing aggregate expression not the last CREATE MATERIALIZED VIEW water_consumption_aggregation_no_addition WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT sensor_id, (max(water_index) - min(water_index)) AS water_consumption, time_bucket(INTERVAL '1 minute', timestamp) AS timestamp FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) WITH NO DATA; CALL refresh_continuous_aggregate('water_consumption_aggregation_no_addition', NULL, NULL); -- The results of the view and the query over hypertable should be the same SELECT * FROM water_consumption_aggregation_no_addition ORDER BY water_consumption; sensor_id | water_consumption | timestamp -----------+-------------------+-------------------------- 1 | 5 | Wed Nov 03 09:42:00 2010 1 | 7 | Wed Nov 03 09:45:00 2010 1 | 8 | Wed Nov 03 09:44:00 2010 1 | 10 | Wed Nov 03 09:43:00 2010 SELECT sensor_id, (max(water_index) - min(water_index)) AS water_consumption, time_bucket(INTERVAL '1 minute', timestamp) AS timestamp FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) ORDER BY water_consumption; sensor_id | water_consumption | timestamp -----------+-------------------+-------------------------- 1 | 5 | Wed Nov 03 09:42:00 2010 1 | 7 | Wed Nov 03 09:45:00 2010 1 | 8 | Wed Nov 03 09:44:00 2010 1 | 10 | Wed Nov 03 09:43:00 2010 DROP TABLE water_consumption CASCADE; NOTICE: drop cascades to 6 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_35_73_chunk NOTICE: drop cascades to table _timescaledb_internal._hyper_36_74_chunk NOTICE: drop cascades to table _timescaledb_internal._hyper_37_75_chunk ---- --- github issue 2655 --- create table raw_data(time timestamptz, search_query text, cnt integer, cnt2 integer); select create_hypertable('raw_data','time', chunk_time_interval=>'15 days'::interval); create_hypertable ------------------------ (38,public,raw_data,t) insert into raw_data select '2000-01-01','Q1'; --having has exprs that appear in select CREATE MATERIALIZED VIEW search_query_count_1m WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT search_query,count(search_query) as count, time_bucket(INTERVAL '1 minute', time) AS bucket FROM raw_data WHERE search_query is not null AND LENGTH(TRIM(both from search_query))>0 GROUP BY search_query, bucket HAVING count(search_query) > 3 OR sum(cnt) > 1; NOTICE: refreshing continuous aggregate "search_query_count_1m" --having has aggregates + grp by columns that appear in select CREATE MATERIALIZED VIEW search_query_count_2 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT search_query,count(search_query) as count, sum(cnt), time_bucket(INTERVAL '1 minute', time) AS bucket FROM raw_data WHERE search_query is not null AND LENGTH(TRIM(both from search_query))>0 GROUP BY search_query, bucket HAVING count(search_query) > 3 OR sum(cnt) > 1 OR ( sum(cnt) + count(cnt)) > 1 AND search_query = 'Q1'; NOTICE: refreshing continuous aggregate "search_query_count_2" CREATE MATERIALIZED VIEW search_query_count_3 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT search_query,count(search_query) as count, sum(cnt), time_bucket(INTERVAL '1 minute', time) AS bucket FROM raw_data WHERE search_query is not null AND LENGTH(TRIM(both from search_query))>0 GROUP BY cnt +cnt2 , bucket, search_query HAVING cnt + cnt2 + sum(cnt) > 2 or count(cnt2) > 10; NOTICE: refreshing continuous aggregate "search_query_count_3" insert into raw_data select '2000-01-01 00:00+0','Q1', 1, 100; insert into raw_data select '2000-01-01 00:00+0','Q1', 2, 200; insert into raw_data select '2000-01-01 00:00+0','Q1', 3, 300; insert into raw_data select '2000-01-02 00:00+0','Q2', 10, 10; insert into raw_data select '2000-01-02 00:00+0','Q2', 20, 20; CALL refresh_continuous_aggregate('search_query_count_1m', NULL, NULL); SELECT * FROM search_query_count_1m ORDER BY 1, 2; search_query | count | bucket --------------+-------+------------------------------ Q1 | 3 | Fri Dec 31 16:00:00 1999 PST Q2 | 2 | Sat Jan 01 16:00:00 2000 PST --only 1 of these should appear in the result insert into raw_data select '2000-01-02 00:00+0','Q3', 0, 0; insert into raw_data select '2000-01-03 00:00+0','Q4', 20, 20; CALL refresh_continuous_aggregate('search_query_count_1m', NULL, NULL); SELECT * FROM search_query_count_1m ORDER BY 1, 2; search_query | count | bucket --------------+-------+------------------------------ Q1 | 3 | Fri Dec 31 16:00:00 1999 PST Q2 | 2 | Sat Jan 01 16:00:00 2000 PST Q4 | 1 | Sun Jan 02 16:00:00 2000 PST --refresh search_query_count_2--- CALL refresh_continuous_aggregate('search_query_count_2', NULL, NULL); SELECT * FROM search_query_count_2 ORDER BY 1, 2; search_query | count | sum | bucket --------------+-------+-----+------------------------------ Q1 | 3 | 6 | Fri Dec 31 16:00:00 1999 PST Q2 | 2 | 30 | Sat Jan 01 16:00:00 2000 PST Q4 | 1 | 20 | Sun Jan 02 16:00:00 2000 PST --refresh search_query_count_3--- CALL refresh_continuous_aggregate('search_query_count_3', NULL, NULL); SELECT * FROM search_query_count_3 ORDER BY 1, 2, 3; search_query | count | sum | bucket --------------+-------+-----+------------------------------ Q1 | 1 | 1 | Fri Dec 31 16:00:00 1999 PST Q1 | 1 | 2 | Fri Dec 31 16:00:00 1999 PST Q1 | 1 | 3 | Fri Dec 31 16:00:00 1999 PST Q2 | 1 | 10 | Sat Jan 01 16:00:00 2000 PST Q2 | 1 | 20 | Sat Jan 01 16:00:00 2000 PST Q4 | 1 | 20 | Sun Jan 02 16:00:00 2000 PST --- TEST enable compression on continuous aggregates CREATE VIEW cagg_compression_status as SELECT ca.mat_hypertable_id AS mat_htid, ca.user_view_name AS cagg_name , h.schema_name AS mat_schema_name, h.table_name AS mat_table_name, ca.materialized_only FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) ; SELECT mat_htid AS "MAT_HTID" , mat_schema_name || '.' || mat_table_name AS "MAT_HTNAME" , mat_table_name AS "MAT_TABLE_NAME" FROM cagg_compression_status WHERE cagg_name = 'search_query_count_3' \gset ALTER MATERIALIZED VIEW search_query_count_3 SET (timescaledb.compress = 'true'); NOTICE: defaulting compress_orderby to bucket,search_query SELECT cagg_name, mat_table_name FROM cagg_compression_status where cagg_name = 'search_query_count_3'; cagg_name | mat_table_name ----------------------+----------------------------- search_query_count_3 | _materialized_hypertable_41 \x SELECT * FROM timescaledb_information.compression_settings WHERE hypertable_name = :'MAT_TABLE_NAME'; -[ RECORD 1 ]----------+---------------------------- hypertable_schema | _timescaledb_internal hypertable_name | _materialized_hypertable_41 attname | bucket segmentby_column_index | orderby_column_index | 1 orderby_asc | t orderby_nullsfirst | f -[ RECORD 2 ]----------+---------------------------- hypertable_schema | _timescaledb_internal hypertable_name | _materialized_hypertable_41 attname | search_query segmentby_column_index | orderby_column_index | 2 orderby_asc | t orderby_nullsfirst | f \x SELECT compress_chunk(ch) FROM show_chunks('search_query_count_3') ch; compress_chunk ------------------------------------------ _timescaledb_internal._hyper_41_79_chunk SELECT * from search_query_count_3 ORDER BY 1, 2, 3; search_query | count | sum | bucket --------------+-------+-----+------------------------------ Q1 | 1 | 1 | Fri Dec 31 16:00:00 1999 PST Q1 | 1 | 2 | Fri Dec 31 16:00:00 1999 PST Q1 | 1 | 3 | Fri Dec 31 16:00:00 1999 PST Q2 | 1 | 10 | Sat Jan 01 16:00:00 2000 PST Q2 | 1 | 20 | Sat Jan 01 16:00:00 2000 PST Q4 | 1 | 20 | Sun Jan 02 16:00:00 2000 PST -- insert into a new region of the hypertable and then refresh the cagg -- (note we still do not support refreshes into existing regions. -- cagg chunks do not map 1-1 to hypertabl regions. They encompass -- more data -- ). insert into raw_data select '2000-05-01 00:00+0','Q3', 0, 0; -- On PG >= 14 the refresh test below will pass because we added support for UPDATE/DELETE on compressed chunks in PR #5339 \set ON_ERROR_STOP 0 CALL refresh_continuous_aggregate('search_query_count_3', NULL, '2000-06-01 00:00+0'::timestamptz); CALL refresh_continuous_aggregate('search_query_count_3', '2000-05-01 00:00+0'::timestamptz, '2000-06-01 00:00+0'::timestamptz); NOTICE: continuous aggregate "search_query_count_3" is already up-to-date \set ON_ERROR_STOP 1 --insert row insert into raw_data select '2001-05-10 00:00+0','Q3', 100, 100; --this should succeed since it does not refresh any compressed regions in the cagg CALL refresh_continuous_aggregate('search_query_count_3', '2001-05-01 00:00+0'::timestamptz, '2001-06-01 00:00+0'::timestamptz); --verify watermark and check that chunks are compressed SELECT _timescaledb_functions.to_timestamp(w) FROM _timescaledb_functions.cagg_watermark(:'MAT_HTID') w; to_timestamp ------------------------------ Wed May 09 17:01:00 2001 PDT SELECT chunk_name, range_start, range_end, is_compressed FROM timescaledb_information.chunks WHERE hypertable_name = :'MAT_TABLE_NAME' ORDER BY 1; chunk_name | range_start | range_end | is_compressed --------------------+------------------------------+------------------------------+--------------- _hyper_41_79_chunk | Fri Dec 24 16:00:00 1999 PST | Mon May 22 17:00:00 2000 PDT | t _hyper_41_83_chunk | Sun Mar 18 16:00:00 2001 PST | Wed Aug 15 17:00:00 2001 PDT | f SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id = :'MAT_HTID' ORDER BY 1, 2,3; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- 41 | -9223372036854775808 | -210866803200000001 41 | 959817600000000 | 988675199999999 41 | 991353600000000 | 9223372036854775807 SELECT * from search_query_count_3 WHERE bucket > '2001-01-01' ORDER BY 1, 2, 3; search_query | count | sum | bucket --------------+-------+-----+------------------------------ Q3 | 1 | 100 | Wed May 09 17:00:00 2001 PDT --now disable compression , will error out -- \set ON_ERROR_STOP 0 ALTER MATERIALIZED VIEW search_query_count_3 SET (timescaledb.compress = 'false'); ERROR: cannot disable columnstore on hypertable with columnstore chunks \set ON_ERROR_STOP 1 SELECT decompress_chunk(format('%I.%I', schema_name, table_name)) FROM _timescaledb_catalog.chunk WHERE hypertable_id = :'MAT_HTID' and status = 1; decompress_chunk ------------------------------------------ _timescaledb_internal._hyper_41_79_chunk --disable compression on cagg after decompressing all chunks-- ALTER MATERIALIZED VIEW search_query_count_3 SET (timescaledb.compress = 'false'); SELECT cagg_name, mat_table_name FROM cagg_compression_status where cagg_name = 'search_query_count_3'; cagg_name | mat_table_name ----------------------+----------------------------- search_query_count_3 | _materialized_hypertable_41 SELECT view_name, materialized_only, compression_enabled FROM timescaledb_information.continuous_aggregates where view_name = 'search_query_count_3'; view_name | materialized_only | compression_enabled ----------------------+-------------------+--------------------- search_query_count_3 | f | f -- TEST caggs on table with more columns than in the cagg view defn -- CREATE TABLE test_morecols ( time TIMESTAMPTZ NOT NULL, val1 INTEGER, val2 INTEGER, val3 INTEGER, val4 INTEGER, val5 INTEGER, val6 INTEGER, val7 INTEGER, val8 INTEGER); SELECT create_hypertable('test_morecols', 'time', chunk_time_interval=> '7 days'::interval); create_hypertable ----------------------------- (43,public,test_morecols,t) INSERT INTO test_morecols SELECT generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 55, 75, 40, 70, NULL, 100, 200, 200; CREATE MATERIALIZED VIEW test_morecols_cagg with (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('30 days',time), avg(val1), count(val2) FROM test_morecols GROUP BY 1; NOTICE: refreshing continuous aggregate "test_morecols_cagg" ALTER MATERIALIZED VIEW test_morecols_cagg SET (timescaledb.compress='true'); NOTICE: defaulting compress_orderby to time_bucket SELECT compress_chunk(ch) FROM show_chunks('test_morecols_cagg') ch; compress_chunk ------------------------------------------ _timescaledb_internal._hyper_44_89_chunk SELECT * FROM test_morecols_cagg ORDER BY time_bucket; time_bucket | avg | count ------------------------------+---------------------+------- Fri Nov 23 16:00:00 2018 PST | 55.0000000000000000 | 23 Sun Dec 23 16:00:00 2018 PST | 55.0000000000000000 | 8 SELECT view_name, materialized_only, compression_enabled FROM timescaledb_information.continuous_aggregates where view_name = 'test_morecols_cagg'; view_name | materialized_only | compression_enabled --------------------+-------------------+--------------------- test_morecols_cagg | f | t --should keep compressed option, modify only materialized -- ALTER MATERIALIZED VIEW test_morecols_cagg SET (timescaledb.materialized_only='true'); SELECT view_name, materialized_only, compression_enabled FROM timescaledb_information.continuous_aggregates where view_name = 'test_morecols_cagg'; view_name | materialized_only | compression_enabled --------------------+-------------------+--------------------- test_morecols_cagg | t | t CREATE TABLE issue3248(filler_1 int, filler_2 int, filler_3 int, time timestamptz NOT NULL, device_id int, v0 int, v1 int, v2 float, v3 float); CREATE INDEX ON issue3248(time DESC); CREATE INDEX ON issue3248(device_id,time DESC); SELECT create_hypertable('issue3248','time',create_default_indexes:=false); create_hypertable ------------------------- (46,public,issue3248,t) ALTER TABLE issue3248 DROP COLUMN filler_1; INSERT INTO issue3248(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id+1, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-01 0:00:00+0'::timestamptz,'2000-01-05 23:55:00+0','8h') gtime(time), generate_series(1,5,1) gdevice(device_id); ALTER TABLE issue3248 DROP COLUMN filler_2; INSERT INTO issue3248(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id-1, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-06 0:00:00+0'::timestamptz,'2000-01-12 23:55:00+0','8h') gtime(time), generate_series(1,5,1) gdevice(device_id); ALTER TABLE issue3248 DROP COLUMN filler_3; INSERT INTO issue3248(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-13 0:00:00+0'::timestamptz,'2000-01-19 23:55:00+0','8h') gtime(time), generate_series(1,5,1) gdevice(device_id); ANALYZE issue3248; CREATE materialized view issue3248_cagg WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h',time), device_id, min(v0), max(v1), avg(v2) FROM issue3248 GROUP BY 1,2; NOTICE: refreshing continuous aggregate "issue3248_cagg" SELECT FROM issue3248 AS m, LATERAL(SELECT m FROM issue3248_cagg WHERE avg IS NULL LIMIT 1) AS lat; -- -- test that option create_group_indexes is taken into account CREATE TABLE test_group_idx ( time timestamptz, symbol int, value numeric ); select create_hypertable('test_group_idx', 'time'); create_hypertable ------------------------------ (48,public,test_group_idx,t) insert into test_group_idx select t, round(random()*10), random()*5 from generate_series('2020-01-01', '2020-02-25', INTERVAL '12 hours') t; create materialized view cagg_index_true with (timescaledb.continuous, timescaledb.materialized_only=false, timescaledb.create_group_indexes=true) as select time_bucket('1 day', "time") as bucket, sum(value), symbol from test_group_idx group by bucket, symbol; NOTICE: refreshing continuous aggregate "cagg_index_true" create materialized view cagg_index_false with (timescaledb.continuous, timescaledb.materialized_only=false, timescaledb.create_group_indexes=false) as select time_bucket('1 day', "time") as bucket, sum(value), symbol from test_group_idx group by bucket, symbol; NOTICE: refreshing continuous aggregate "cagg_index_false" create materialized view cagg_index_default with (timescaledb.continuous, timescaledb.materialized_only=false) as select time_bucket('1 day', "time") as bucket, sum(value), symbol from test_group_idx group by bucket, symbol; NOTICE: refreshing continuous aggregate "cagg_index_default" -- see corresponding materialization_hypertables select view_name, materialization_hypertable_name from timescaledb_information.continuous_aggregates ca where view_name like 'cagg_index_%' ORDER BY view_name; view_name | materialization_hypertable_name --------------------+--------------------------------- cagg_index_default | _materialized_hypertable_51 cagg_index_false | _materialized_hypertable_50 cagg_index_true | _materialized_hypertable_49 -- now make sure a group index has been created when explicitly asked for \x on select i.* from pg_indexes i join pg_class c on schemaname = relnamespace::regnamespace::text and tablename = relname where tablename in (select materialization_hypertable_name from timescaledb_information.continuous_aggregates where view_name like 'cagg_index_%') order by tablename, indexname; -[ RECORD 1 ]------------------------------------------------------------------------------------------------------------------------------------------------- schemaname | _timescaledb_internal tablename | _materialized_hypertable_49 indexname | _materialized_hypertable_49_bucket_idx tablespace | indexdef | CREATE INDEX _materialized_hypertable_49_bucket_idx ON _timescaledb_internal._materialized_hypertable_49 USING btree (bucket DESC) -[ RECORD 2 ]------------------------------------------------------------------------------------------------------------------------------------------------- schemaname | _timescaledb_internal tablename | _materialized_hypertable_49 indexname | _materialized_hypertable_49_symbol_bucket_idx tablespace | indexdef | CREATE INDEX _materialized_hypertable_49_symbol_bucket_idx ON _timescaledb_internal._materialized_hypertable_49 USING btree (symbol, bucket DESC) -[ RECORD 3 ]------------------------------------------------------------------------------------------------------------------------------------------------- schemaname | _timescaledb_internal tablename | _materialized_hypertable_50 indexname | _materialized_hypertable_50_bucket_idx tablespace | indexdef | CREATE INDEX _materialized_hypertable_50_bucket_idx ON _timescaledb_internal._materialized_hypertable_50 USING btree (bucket DESC) -[ RECORD 4 ]------------------------------------------------------------------------------------------------------------------------------------------------- schemaname | _timescaledb_internal tablename | _materialized_hypertable_51 indexname | _materialized_hypertable_51_bucket_idx tablespace | indexdef | CREATE INDEX _materialized_hypertable_51_bucket_idx ON _timescaledb_internal._materialized_hypertable_51 USING btree (bucket DESC) -[ RECORD 5 ]------------------------------------------------------------------------------------------------------------------------------------------------- schemaname | _timescaledb_internal tablename | _materialized_hypertable_51 indexname | _materialized_hypertable_51_symbol_bucket_idx tablespace | indexdef | CREATE INDEX _materialized_hypertable_51_symbol_bucket_idx ON _timescaledb_internal._materialized_hypertable_51 USING btree (symbol, bucket DESC) \x off -- -- TESTs for removing old CAggs restrictions -- DROP TABLE conditions CASCADE; NOTICE: drop cascades to 8 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_29_67_chunk NOTICE: drop cascades to table _timescaledb_internal._hyper_30_68_chunk NOTICE: drop cascades to table _timescaledb_internal._hyper_31_69_chunk CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL ); SELECT create_hypertable('conditions', 'timec'); create_hypertable -------------------------- (52,public,conditions,t) INSERT INTO conditions VALUES ('2010-01-01 09:00:00-08', 'SFO', 55, 45), ('2010-01-02 09:00:00-08', 'por', 100, 100), ('2010-01-02 09:00:00-08', 'NYC', 65, 45), ('2010-01-02 09:00:00-08', 'SFO', 65, 45), ('2010-01-03 09:00:00-08', 'NYC', 45, 55), ('2010-01-05 09:00:00-08', 'SFO', 75, 100), ('2018-11-01 09:00:00-08', 'NYC', 45, 35), ('2018-11-02 09:00:00-08', 'NYC', 35, 15), ('2018-11-03 09:00:00-08', 'NYC', 35, 25); -- aggregate with DISTINCT CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), COUNT(location), SUM(DISTINCT temperature) FROM conditions GROUP BY time_bucket('1week', timec), location; NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1, 2, 3; time_bucket | count | sum ------------------------------+-------+----- Sun Dec 27 16:00:00 2009 PST | 1 | 100 Sun Dec 27 16:00:00 2009 PST | 2 | 110 Sun Dec 27 16:00:00 2009 PST | 2 | 120 Sun Jan 03 16:00:00 2010 PST | 1 | 75 Sun Oct 28 17:00:00 2018 PDT | 3 | 80 -- aggregate with FILTER DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), SUM(temperature) FILTER (WHERE humidity > 60) FROM conditions GROUP BY time_bucket('1week', timec), location; NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1, 2; time_bucket | sum ------------------------------+----- Sun Dec 27 16:00:00 2009 PST | 100 Sun Dec 27 16:00:00 2009 PST | Sun Dec 27 16:00:00 2009 PST | Sun Jan 03 16:00:00 2010 PST | 75 Sun Oct 28 17:00:00 2018 PDT | -- aggregate with filter in having clause DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), MAX(temperature) FROM conditions GROUP BY time_bucket('1week', timec), location HAVING SUM(temperature) FILTER (WHERE humidity > 40) > 50; NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1, 2; time_bucket | max ------------------------------+----- Sun Dec 27 16:00:00 2009 PST | 65 Sun Dec 27 16:00:00 2009 PST | 65 Sun Dec 27 16:00:00 2009 PST | 100 Sun Jan 03 16:00:00 2010 PST | 75 -- ordered set aggr DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to table _timescaledb_internal._hyper_55_116_chunk CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), MODE() WITHIN GROUP(ORDER BY humidity) FROM conditions GROUP BY time_bucket('1week', timec); NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1; time_bucket | mode ------------------------------+------ Sun Dec 27 16:00:00 2009 PST | 45 Sun Jan 03 16:00:00 2010 PST | 100 Sun Oct 28 17:00:00 2018 PDT | 15 -- hypothetical-set aggr DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), RANK(60) WITHIN GROUP (ORDER BY humidity), DENSE_RANK(60) WITHIN GROUP (ORDER BY humidity), PERCENT_RANK(60) WITHIN GROUP (ORDER BY humidity) FROM conditions GROUP BY time_bucket('1week', timec); NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1; time_bucket | rank | dense_rank | percent_rank ------------------------------+------+------------+-------------- Sun Dec 27 16:00:00 2009 PST | 5 | 3 | 0.8 Sun Jan 03 16:00:00 2010 PST | 1 | 1 | 0 Sun Oct 28 17:00:00 2018 PDT | 4 | 4 | 1 -- userdefined aggregate without combine function DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE AGGREGATE newavg ( sfunc = int4_avg_accum, basetype = int4, stype = _int8, finalfunc = int8_avg, initcond1 = '{0,0}' ); CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT SUM(humidity), round(newavg(temperature::int4)) FROM conditions GROUP BY time_bucket('1week', timec), location ORDER BY 1,2; NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1, 2; sum | round -----+------- 75 | 38 90 | 60 100 | 55 100 | 75 100 | 100 -- ORDER BY in the view definition DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), COUNT(location), SUM(temperature) FROM conditions GROUP BY time_bucket('1week', timec) ORDER BY sum DESC; NOTICE: refreshing continuous aggregate "mat_m1" -- CAgg definition for realtime SELECT pg_get_viewdef('mat_m1',true); pg_get_viewdef ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ( SELECT _materialized_hypertable_59.time_bucket, + _materialized_hypertable_59.count, + _materialized_hypertable_59.sum + FROM _timescaledb_internal._materialized_hypertable_59 + WHERE _materialized_hypertable_59.time_bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(59)), '-infinity'::timestamp with time zone)+ ORDER BY _materialized_hypertable_59.sum DESC) + UNION ALL + ( SELECT time_bucket('@ 7 days'::interval, conditions.timec) AS time_bucket, + count(conditions.location) AS count, + sum(conditions.temperature) AS sum + FROM conditions + WHERE conditions.timec >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(59)), '-infinity'::timestamp with time zone) + GROUP BY (time_bucket('@ 7 days'::interval, conditions.timec)) + ORDER BY (sum(conditions.temperature)) DESC) + ORDER BY 3 DESC; -- Ordered result SELECT * FROM mat_m1; time_bucket | count | sum ------------------------------+-------+----- Sun Dec 27 16:00:00 2009 PST | 5 | 330 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Jan 03 16:00:00 2010 PST | 1 | 75 -- Insert new data and query again to make sure we produce ordered data INSERT INTO conditions VALUES ('2018-11-10 09:00:00-08', 'SFO', 10, 10); SELECT * FROM mat_m1; time_bucket | count | sum ------------------------------+-------+----- Sun Dec 27 16:00:00 2009 PST | 5 | 330 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Jan 03 16:00:00 2010 PST | 1 | 75 Sun Nov 04 16:00:00 2018 PST | 1 | 10 -- This new row will change the order again INSERT INTO conditions VALUES ('2018-11-11 09:00:00-08', 'SFO', 400, 400); SELECT * FROM mat_m1; time_bucket | count | sum ------------------------------+-------+----- Sun Nov 04 16:00:00 2018 PST | 2 | 410 Sun Dec 27 16:00:00 2009 PST | 5 | 330 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Jan 03 16:00:00 2010 PST | 1 | 75 -- Merge Append EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM mat_m1; --- QUERY PLAN --- Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Index Scan Backward using _hyper_59_123_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_123_chunk -> Index Scan Backward using _hyper_59_124_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_124_chunk Index Cond: (time_bucket < 'Sun Nov 04 16:00:00 2018 PST'::timestamp with time zone) -> Sort Sort Key: (sum(conditions.temperature)) DESC -> Finalize HashAggregate Group Key: (time_bucket('@ 7 days'::interval, conditions.timec)) -> Append -> Partial HashAggregate Group Key: time_bucket('@ 7 days'::interval, _hyper_52_111_chunk.timec) -> Index Scan Backward using _hyper_52_111_chunk_conditions_timec_idx on _hyper_52_111_chunk Index Cond: (timec >= 'Sun Nov 04 16:00:00 2018 PST'::timestamp with time zone) -> Partial HashAggregate Group Key: time_bucket('@ 7 days'::interval, _hyper_52_125_chunk.timec) -> Seq Scan on _hyper_52_125_chunk -- Ordering by another column SELECT * FROM mat_m1 ORDER BY count; time_bucket | count | sum ------------------------------+-------+----- Sun Jan 03 16:00:00 2010 PST | 1 | 75 Sun Nov 04 16:00:00 2018 PST | 2 | 410 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Dec 27 16:00:00 2009 PST | 5 | 330 EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM mat_m1 ORDER BY count; --- QUERY PLAN --- Sort Sort Key: _materialized_hypertable_59.count -> Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Index Scan Backward using _hyper_59_123_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_123_chunk -> Index Scan Backward using _hyper_59_124_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_124_chunk Index Cond: (time_bucket < 'Sun Nov 04 16:00:00 2018 PST'::timestamp with time zone) -> Sort Sort Key: (sum(conditions.temperature)) DESC -> Finalize HashAggregate Group Key: (time_bucket('@ 7 days'::interval, conditions.timec)) -> Append -> Partial HashAggregate Group Key: time_bucket('@ 7 days'::interval, _hyper_52_111_chunk.timec) -> Index Scan Backward using _hyper_52_111_chunk_conditions_timec_idx on _hyper_52_111_chunk Index Cond: (timec >= 'Sun Nov 04 16:00:00 2018 PST'::timestamp with time zone) -> Partial HashAggregate Group Key: time_bucket('@ 7 days'::interval, _hyper_52_125_chunk.timec) -> Seq Scan on _hyper_52_125_chunk -- Change the type of cagg ALTER MATERIALIZED VIEW mat_m1 SET (timescaledb.materialized_only=true); -- CAgg definition for materialized only SELECT pg_get_viewdef('mat_m1',true); pg_get_viewdef ----------------------------------------------------------- SELECT time_bucket, + count, + sum + FROM _timescaledb_internal._materialized_hypertable_59+ ORDER BY sum DESC; -- Now the query will show only the materialized data, without last two -- records inserted into the original hypertable (last two insers above) SELECT * FROM mat_m1; time_bucket | count | sum ------------------------------+-------+----- Sun Dec 27 16:00:00 2009 PST | 5 | 330 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Jan 03 16:00:00 2010 PST | 1 | 75 -- Merge Append EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM mat_m1; --- QUERY PLAN --- Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Index Scan Backward using _hyper_59_123_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_123_chunk -> Index Scan Backward using _hyper_59_124_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_124_chunk -- Ordering by another column SELECT * FROM mat_m1 ORDER BY count; time_bucket | count | sum ------------------------------+-------+----- Sun Jan 03 16:00:00 2010 PST | 1 | 75 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Dec 27 16:00:00 2009 PST | 5 | 330 EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM mat_m1 ORDER BY count; --- QUERY PLAN --- Sort Sort Key: _materialized_hypertable_59.count -> Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Index Scan Backward using _hyper_59_123_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_123_chunk -> Index Scan Backward using _hyper_59_124_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_124_chunk SELECT h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset -- Invalidate old region and refresh again DELETE FROM conditions WHERE timec < '2010-01-05 09:00:00-08'; CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); -- Querying the cagg produce ordered records as expected SELECT * FROM mat_m1; time_bucket | count | sum ------------------------------+-------+----- Sun Nov 04 16:00:00 2018 PST | 2 | 410 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Jan 03 16:00:00 2010 PST | 1 | 75 -- Querying direct the materialization hypertable doesn't -- produce ordered records SELECT * FROM :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME"; time_bucket | count | sum ------------------------------+-------+----- Sun Jan 03 16:00:00 2010 PST | 1 | 75 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Nov 04 16:00:00 2018 PST | 2 | 410 DROP TABLE conditions CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL ); SELECT table_name FROM create_hypertable('conditions', 'timec'); table_name ------------ conditions INSERT INTO conditions VALUES ('2010-01-01 09:00:00-08', 'SFO', 55, 45), ('2010-01-02 09:00:00-08', 'por', 100, 100), ('2010-01-02 09:00:00-08', 'SFO', 65, 45), ('2010-01-02 09:00:00-08', 'NYC', 65, 45), ('2018-11-01 09:00:00-08', 'NYC', 45, 35), ('2018-11-02 09:00:00-08', 'NYC', 35, 15); CREATE MATERIALIZED VIEW conditions_summary_new(timec, minl, sumt, sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1day', timec), min(location), sum(temperature), sum(humidity) FROM conditions GROUP BY time_bucket('1day', timec) WITH NO DATA; \x ON SELECT * FROM timescaledb_information.continuous_aggregates WHERE view_name = 'conditions_summary_new'; -[ RECORD 1 ]---------------------+---------------------------------------------------------- hypertable_schema | public hypertable_name | conditions view_schema | public view_name | conditions_summary_new view_owner | default_perm_user materialized_only | t compression_enabled | f materialization_hypertable_schema | _timescaledb_internal materialization_hypertable_name | _materialized_hypertable_61 view_definition | SELECT time_bucket('@ 1 day'::interval, timec) AS timec,+ | min(location) AS minl, + | sum(temperature) AS sumt, + | sum(humidity) AS sumh + | FROM conditions + | GROUP BY (time_bucket('@ 1 day'::interval, timec)); \x OFF CALL refresh_continuous_aggregate('conditions_summary_new', NULL, NULL); -- Check and compare number of returned rows SELECT count(*) FROM conditions_summary_new; count ------- 4 -- Parallel planning test for realtime Continuous Aggregate DROP TABLE conditions CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, temperature DOUBLE PRECISION NULL ); SELECT table_name FROM create_hypertable('conditions', 'timec'); table_name ------------ conditions INSERT INTO conditions SELECT t, 10 FROM generate_series('2023-01-01 00:00-03'::timestamptz, '2023-12-31 23:59-03'::timestamptz, '1 hour'::interval) AS t; CREATE MATERIALIZED VIEW conditions_daily WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1 day', timec), SUM(temperature) FROM conditions GROUP BY 1 ORDER BY 2 DESC; NOTICE: refreshing continuous aggregate "conditions_daily" SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false); set_config ------------ on SET max_parallel_workers_per_gather = 4; SET parallel_setup_cost = 0; SET parallel_tuple_cost = 0; -- Parallel planning EXPLAIN (BUFFERS OFF, COSTS OFF, TIMING OFF) SELECT * FROM conditions_daily WHERE time_bucket >= '2023-07-01'; --- QUERY PLAN --- Merge Append Sort Key: _materialized_hypertable_63.sum DESC -> Gather Merge Workers Planned: 2 -> Sort Sort Key: _materialized_hypertable_63.sum DESC -> Parallel Append -> Parallel Index Scan using _hyper_63_185_chunk__materialized_hypertable_63_time_bucket_idx on _hyper_63_185_chunk Index Cond: ((time_bucket < 'Mon Jan 01 16:00:00 2024 PST'::timestamp with time zone) AND (time_bucket >= 'Sat Jul 01 00:00:00 2023 PDT'::timestamp with time zone)) -> Parallel Index Scan using _hyper_63_187_chunk__materialized_hypertable_63_time_bucket_idx on _hyper_63_187_chunk Index Cond: ((time_bucket < 'Mon Jan 01 16:00:00 2024 PST'::timestamp with time zone) AND (time_bucket >= 'Sat Jul 01 00:00:00 2023 PDT'::timestamp with time zone)) -> Parallel Seq Scan on _hyper_63_184_chunk -> Sort Sort Key: (sum(_hyper_62_182_chunk.temperature)) DESC -> HashAggregate Group Key: (time_bucket('@ 1 day'::interval, _hyper_62_182_chunk.timec)) -> Gather Workers Planned: 1 -> Result -> Parallel Index Scan Backward using _hyper_62_182_chunk_conditions_timec_idx on _hyper_62_182_chunk Index Cond: ((timec >= 'Mon Jan 01 16:00:00 2024 PST'::timestamp with time zone) AND (timec >= 'Sat Jul 01 00:00:00 2023 PDT'::timestamp with time zone)) Filter: (time_bucket('@ 1 day'::interval, timec) >= 'Sat Jul 01 00:00:00 2023 PDT'::timestamp with time zone) ================================================ FILE: tsl/test/expected/cagg-18.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- initialize the bgw mock state to prevent the materialization workers from running \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION ts_bgw_params_create() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION test.continuous_aggs_find_view(cagg REGCLASS) RETURNS VOID AS :TSL_MODULE_PATHNAME, 'ts_test_continuous_agg_find_by_view_name' LANGUAGE C; \set WAIT_ON_JOB 0 \set IMMEDIATELY_SET_UNTIL 1 \set WAIT_FOR_OTHER_TO_ADVANCE 2 -- remove any default jobs, e.g., telemetry so bgw_job isn't polluted DELETE FROM _timescaledb_catalog.bgw_job; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT * FROM _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- --TEST1 --- --basic test with count create table foo (a integer, b integer, c integer); select table_name from create_hypertable('foo', 'a', chunk_time_interval=> 10); table_name ------------ foo insert into foo values( 3 , 16 , 20); insert into foo values( 1 , 10 , 20); insert into foo values( 1 , 11 , 20); insert into foo values( 1 , 12 , 20); insert into foo values( 1 , 13 , 20); insert into foo values( 1 , 14 , 20); insert into foo values( 2 , 14 , 20); insert into foo values( 2 , 15 , 20); insert into foo values( 2 , 16 , 20); CREATE OR REPLACE FUNCTION integer_now_foo() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(a), 0) FROM foo $$; SELECT set_integer_now_func('foo', 'integer_now_foo'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW mat_m1(a, countb) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select a, count(b) from foo group by time_bucket(1, a), a WITH NO DATA; SELECT add_continuous_aggregate_policy('mat_m1', NULL, 2::integer, '12 h'::interval) AS job_id \gset SELECT * FROM _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ------+--------------------------------------------+-------------------+-------------+-------------+--------------+------------------------+-------------------------------------+-------------------+-----------+----------------+---------------+---------------+-----------------------------------------------------------------+------------------------+-------------------------------------------+---------- 1000 | Refresh Continuous Aggregate Policy [1000] | @ 12 hours | @ 0 | -1 | @ 12 hours | _timescaledb_functions | policy_refresh_continuous_aggregate | default_perm_user | t | f | | 2 | {"end_offset": 2, "start_offset": null, "mat_hypertable_id": 2} | _timescaledb_functions | policy_refresh_continuous_aggregate_check | SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select a, count(b), time_bucket(1, a) from foo group by time_bucket(1, a) , a ; select * from mat_m1 order by a ; a | countb ---+-------- 1 | 5 2 | 3 3 | 1 --check triggers on user hypertable -- SET ROLE :ROLE_SUPERUSER; select tgname, tgtype, tgenabled , relname from pg_trigger, pg_class where tgrelid = pg_class.oid and pg_class.relname like 'foo' order by tgname; tgname | tgtype | tgenabled | relname --------+--------+-----------+--------- SET ROLE :ROLE_DEFAULT_PERM_USER; -- TEST2 --- DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to table _timescaledb_internal._hyper_2_2_chunk SHOW enable_partitionwise_aggregate; enable_partitionwise_aggregate -------------------------------- off SET enable_partitionwise_aggregate = on; SELECT * FROM _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions insert into conditions values ( '2010-01-01 09:00:00-08', 'SFO', 55, 45); insert into conditions values ( '2010-01-02 09:00:00-08', 'por', 100, 100); insert into conditions values ( '2010-01-02 09:00:00-08', 'SFO', 65, 45); insert into conditions values ( '2010-01-02 09:00:00-08', 'NYC', 65, 45); insert into conditions values ( '2018-11-01 09:00:00-08', 'NYC', 45, 35); insert into conditions values ( '2018-11-02 09:00:00-08', 'NYC', 35, 15); CREATE MATERIALIZED VIEW mat_m1( timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1day', timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket('1day', timec) WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset -- Materialized hypertable for mat_m1 should not be visible in the -- hypertables view: SELECT hypertable_schema, hypertable_name FROM timescaledb_information.hypertables ORDER BY 1,2; hypertable_schema | hypertable_name -------------------+----------------- public | conditions public | foo SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select time_bucket('1day', timec), min(location), sum(temperature), sum(humidity) from conditions group by time_bucket('1day', timec) ; SET ROLE :ROLE_DEFAULT_PERM_USER; --should have same results -- select timec, minl, sumt, sumh from mat_m1 order by timec; timec | minl | sumt | sumh ------------------------------+------+------+------ Thu Dec 31 16:00:00 2009 PST | SFO | 55 | 45 Fri Jan 01 16:00:00 2010 PST | NYC | 230 | 190 Wed Oct 31 17:00:00 2018 PDT | NYC | 45 | 35 Thu Nov 01 17:00:00 2018 PDT | NYC | 35 | 15 select time_bucket('1day', timec), min(location), sum(temperature), sum(humidity) from conditions group by time_bucket('1day', timec) order by 1; time_bucket | min | sum | sum ------------------------------+-----+-----+----- Thu Dec 31 16:00:00 2009 PST | SFO | 55 | 45 Fri Jan 01 16:00:00 2010 PST | NYC | 230 | 190 Wed Oct 31 17:00:00 2018 PDT | NYC | 45 | 35 Thu Nov 01 17:00:00 2018 PDT | NYC | 35 | 15 SET enable_partitionwise_aggregate = off; -- TEST3 -- -- drop on table conditions should cascade to materialized mat_v1 drop table conditions cascade; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions insert into conditions values ( '2010-01-01 09:00:00-08', 'SFO', 55, 45); insert into conditions values ( '2010-01-02 09:00:00-08', 'por', 100, 100); insert into conditions values ( '2010-01-02 09:00:00-08', 'NYC', 65, 45); insert into conditions values ( '2010-01-02 09:00:00-08', 'SFO', 65, 45); insert into conditions values ( '2010-01-03 09:00:00-08', 'NYC', 45, 55); insert into conditions values ( '2010-01-05 09:00:00-08', 'SFO', 75, 100); insert into conditions values ( '2018-11-01 09:00:00-08', 'NYC', 45, 35); insert into conditions values ( '2018-11-02 09:00:00-08', 'NYC', 35, 15); insert into conditions values ( '2018-11-03 09:00:00-08', 'NYC', 35, 25); CREATE MATERIALIZED VIEW mat_m1( timec, minl, sumth, stddevh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select time_bucket('1week', timec), min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) ; SET ROLE :ROLE_DEFAULT_PERM_USER; --should have same results -- select timec, minl, sumth, stddevh from mat_m1 order by timec; timec | minl | sumth | stddevh ------------------------------+------+-------+------------------ Sun Dec 27 16:00:00 2009 PST | NYC | 620 | 23.8746727726266 Sun Jan 03 16:00:00 2010 PST | SFO | 175 | Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 select time_bucket('1week', timec) , min(location), sum(temperature)+ sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) order by time_bucket('1week', timec); time_bucket | min | ?column? | stddev ------------------------------+-----+----------+------------------ Sun Dec 27 16:00:00 2009 PST | NYC | 620 | 23.8746727726266 Sun Jan 03 16:00:00 2010 PST | SFO | 175 | Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 -- TEST4 -- --materialized view with group by clause + expression in SELECT -- use previous data from conditions --drop only the view. -- apply where clause on result of mat_m1 -- DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW mat_m1( timec, minl, sumth, stddevh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions where location = 'NYC' group by time_bucket('1week', timec) WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select time_bucket('1week', timec), min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions where location = 'NYC' group by time_bucket('1week', timec) ; SET ROLE :ROLE_DEFAULT_PERM_USER; --should have same results -- select timec, minl, sumth, stddevh from mat_m1 where stddevh is not null order by timec; timec | minl | sumth | stddevh ------------------------------+------+-------+------------------ Sun Dec 27 16:00:00 2009 PST | NYC | 210 | 7.07106781186548 Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 select time_bucket('1week', timec) , min(location), sum(temperature)+ sum(humidity), stddev(humidity) from conditions where location = 'NYC' group by time_bucket('1week', timec) order by time_bucket('1week', timec); time_bucket | min | ?column? | stddev ------------------------------+-----+----------+------------------ Sun Dec 27 16:00:00 2009 PST | NYC | 210 | 7.07106781186548 Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 -- TEST5 -- ---------test with having clause ---------------------- DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects create materialized view mat_m1( timec, minl, sumth, stddevh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having stddev(humidity) is not null WITH NO DATA; ; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select time_bucket('1week', timec), min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having stddev(humidity) is not null; SET ROLE :ROLE_DEFAULT_PERM_USER; -- should have same results -- select * from mat_m1 order by sumth; timec | minl | sumth | stddevh ------------------------------+------+-------+------------------ Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 Sun Dec 27 16:00:00 2009 PST | NYC | 620 | 23.8746727726266 select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having stddev(humidity) is not null order by sum(temperature)+sum(humidity); time_bucket | min | ?column? | stddev ------------------------------+-----+----------+------------------ Sun Oct 28 17:00:00 2018 PDT | NYC | 190 | 10 Sun Dec 27 16:00:00 2009 PST | NYC | 620 | 23.8746727726266 -- TEST6 -- --group by with more than 1 group column -- having clause with a mix of columns from select list + others drop table conditions cascade; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp numeric NULL, highp numeric null ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions insert into conditions select generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'POR', 55, 75, 40, 70; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'NYC', 35, 45, 50, 40; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-15 00:00'::timestamp, '1 day'), 'LA', 73, 55, 71, 28; --naming with AS clauses CREATE MATERIALIZED VIEW mat_naming WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec) as bucket, location as loc, sum(temperature)+sum(humidity) as sumth, stddev(humidity) from conditions group by bucket, loc having min(location) >= 'NYC' and avg(temperature) > 20 WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_naming' \gset select attnum , attname from pg_attribute where attnum > 0 and attrelid = (Select oid from pg_class where relname like :'MAT_TABLE_NAME') order by attnum, attname; attnum | attname --------+--------- 1 | bucket 2 | loc 3 | sumth 4 | stddev DROP MATERIALIZED VIEW mat_naming; --naming with default names CREATE MATERIALIZED VIEW mat_naming WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec), location, sum(temperature)+sum(humidity) as sumth, stddev(humidity) from conditions group by 1,2 having min(location) >= 'NYC' and avg(temperature) > 20 WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_naming' \gset select attnum , attname from pg_attribute where attnum > 0 and attrelid = (Select oid from pg_class where relname like :'MAT_TABLE_NAME') order by attnum, attname; attnum | attname --------+------------- 1 | time_bucket 2 | location 3 | sumth 4 | stddev DROP MATERIALIZED VIEW mat_naming; --naming with view col names CREATE MATERIALIZED VIEW mat_naming(bucket, loc, sum_t_h, stdd) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec), location, sum(temperature)+sum(humidity), stddev(humidity) from conditions group by 1,2 having min(location) >= 'NYC' and avg(temperature) > 20 WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_naming' \gset select attnum , attname from pg_attribute where attnum > 0 and attrelid = (Select oid from pg_class where relname like :'MAT_TABLE_NAME') order by attnum, attname; attnum | attname --------+--------- 1 | bucket 2 | loc 3 | sum_t_h 4 | stdd DROP MATERIALIZED VIEW mat_naming; CREATE MATERIALIZED VIEW mat_m1(timec, minl, sumth, stddevh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having min(location) >= 'NYC' and avg(temperature) > 20 WITH NO DATA; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset select attnum , attname from pg_attribute where attnum > 0 and attrelid = (Select oid from pg_class where relname like :'MAT_TABLE_NAME') order by attnum, attname; attnum | attname --------+--------- 1 | timec 2 | minl 3 | sumth 4 | stddevh SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select time_bucket('1week', timec), min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having min(location) >= 'NYC' and avg(temperature) > 20; SET ROLE :ROLE_DEFAULT_PERM_USER; --should have same results -- select timec, minl, sumth, stddevh from mat_m1 order by timec, minl; timec | minl | sumth | stddevh ------------------------------+------+-------+------------------ Sun Dec 16 16:00:00 2018 PST | NYC | 1470 | 15.5662356498831 Sun Dec 23 16:00:00 2018 PST | NYC | 1470 | 15.5662356498831 Sun Dec 30 16:00:00 2018 PST | NYC | 210 | 21.2132034355964 select time_bucket('1week', timec) , min(location), sum(temperature)+sum(humidity), stddev(humidity) from conditions group by time_bucket('1week', timec) having min(location) >= 'NYC' and avg(temperature) > 20 and avg(lowp) > 10 order by time_bucket('1week', timec), min(location); time_bucket | min | ?column? | stddev ------------------------------+-----+----------+------------------ Sun Dec 16 16:00:00 2018 PST | NYC | 1470 | 15.5662356498831 Sun Dec 23 16:00:00 2018 PST | NYC | 1470 | 15.5662356498831 Sun Dec 30 16:00:00 2018 PST | NYC | 210 | 21.2132034355964 --check view defintion in information views select view_name, view_definition from timescaledb_information.continuous_aggregates where view_name::text like 'mat_m1'; view_name | view_definition -----------+------------------------------------------------------------------------------------------- mat_m1 | SELECT time_bucket('@ 7 days'::interval, timec) AS timec, + | min(location) AS minl, + | (sum(temperature) + sum(humidity)) AS sumth, + | stddev(humidity) AS stddevh + | FROM conditions + | GROUP BY (time_bucket('@ 7 days'::interval, timec)) + | HAVING ((min(location) >= 'NYC'::text) AND (avg(temperature) > (20)::double precision)); --TEST6 -- select from internal view SET ROLE :ROLE_SUPERUSER; insert into :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" select * from :"PART_VIEW_SCHEMA".:"PART_VIEW_NAME"; SET ROLE :ROLE_DEFAULT_PERM_USER; --lets drop the view and check DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to table _timescaledb_internal._hyper_13_24_chunk drop table conditions; CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions insert into conditions select generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'POR', 55, 75, 40, 70, NULL; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'NYC', 35, 45, 50, 40, NULL; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-15 00:00'::timestamp, '1 day'), 'LA', 73, 55, NULL, 28, NULL; SELECT $$ select time_bucket('1week', timec) , min(location) as col1, sum(temperature)+sum(humidity) as col2, stddev(humidity) as col3, min(allnull) as col4 from conditions group by time_bucket('1week', timec) having min(location) >= 'NYC' and avg(temperature) > 20 $$ AS "QUERY" \gset \set ECHO errors psql:include/cont_agg_equal.sql:8: NOTICE: materialized view "mat_test" does not exist, skipping ?column? | count ---------------------------------------------------------------+------- Number of rows different between view and original (expect 0) | 0 SELECT $$ select time_bucket('1week', timec), location, sum(temperature)+sum(humidity) as col2, stddev(humidity) as col3, min(allnull) as col4 from conditions group by location, time_bucket('1week', timec) $$ AS "QUERY" \gset \set ECHO errors psql:include/cont_agg_equal.sql:8: NOTICE: drop cascades to table _timescaledb_internal._hyper_15_34_chunk ?column? | count ---------------------------------------------------------------+------- Number of rows different between view and original (expect 0) | 0 --TEST7 -- drop tests for view and hypertable --DROP tests \set ON_ERROR_STOP 0 SELECT h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA", direct_view_name as "DIR_VIEW_NAME", direct_view_schema as "DIR_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_test' \gset DROP TABLE :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME"; ERROR: cannot drop table _timescaledb_internal._materialized_hypertable_16 because other objects depend on it DROP VIEW :"PART_VIEW_SCHEMA".:"PART_VIEW_NAME"; ERROR: cannot drop the partial/direct view because it is required by a continuous aggregate DROP VIEW :"DIR_VIEW_SCHEMA".:"DIR_VIEW_NAME"; ERROR: cannot drop the partial/direct view because it is required by a continuous aggregate \set ON_ERROR_STOP 1 --catalog entry still there; SELECT count(*) FROM _timescaledb_catalog.continuous_agg ca WHERE user_view_name = 'mat_test'; count ------- 1 --mat table, user_view, direct view and partial view all there select count(*) from pg_class where relname = :'PART_VIEW_NAME'; count ------- 1 select count(*) from pg_class where relname = :'MAT_TABLE_NAME'; count ------- 1 select count(*) from pg_class where relname = :'DIR_VIEW_NAME'; count ------- 1 select count(*) from pg_class where relname = 'mat_test'; count ------- 1 DROP MATERIALIZED VIEW mat_test; NOTICE: drop cascades to 2 other objects --catalog entry should be gone SELECT count(*) FROM _timescaledb_catalog.continuous_agg ca WHERE user_view_name = 'mat_test'; count ------- 0 --mat table, user_view, direct view and partial view all gone select count(*) from pg_class where relname = :'PART_VIEW_NAME'; count ------- 0 select count(*) from pg_class where relname = :'MAT_TABLE_NAME'; count ------- 0 select count(*) from pg_class where relname = :'DIR_VIEW_NAME'; count ------- 0 select count(*) from pg_class where relname = 'mat_test'; count ------- 0 --test dropping raw table DROP TABLE conditions; CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions --no data in hyper table on purpose so that CASCADE is not required because of chunks CREATE MATERIALIZED VIEW mat_drop_test(timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1day', timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket('1day', timec) WITH NO DATA; \set ON_ERROR_STOP 0 DROP TABLE conditions; ERROR: cannot drop table conditions because other objects depend on it \set ON_ERROR_STOP 1 --insert data now insert into conditions select generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'POR', 55, 75, 40, 70, NULL; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'NYC', 35, 45, 50, 40, NULL; insert into conditions select generate_series('2018-11-01 00:00'::timestamp, '2018-12-15 00:00'::timestamp, '1 day'), 'LA', 73, 55, NULL, 28, NULL; SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_drop_test' \gset SET client_min_messages TO NOTICE; CALL refresh_continuous_aggregate('mat_drop_test', NULL, NULL); --force invalidation insert into conditions select generate_series('2017-11-01 00:00'::timestamp, '2017-12-15 00:00'::timestamp, '1 day'), 'LA', 73, 55, NULL, 28, NULL; select count(*) from _timescaledb_catalog.continuous_aggs_invalidation_threshold; count ------- 1 select count(*) from _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log; count ------- 8 DROP TABLE conditions CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects --catalog entry should be gone SELECT count(*) FROM _timescaledb_catalog.continuous_agg ca WHERE user_view_name = 'mat_drop_test'; count ------- 0 select count(*) from _timescaledb_catalog.continuous_aggs_invalidation_threshold; count ------- 0 select count(*) from _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log; count ------- 0 select count(*) from _timescaledb_catalog.continuous_aggs_materialization_invalidation_log; count ------- 0 SELECT * FROM _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- --mat table, user_view, and partial view all gone select count(*) from pg_class where relname = :'PART_VIEW_NAME'; count ------- 0 select count(*) from pg_class where relname = :'MAT_TABLE_NAME'; count ------- 0 select count(*) from pg_class where relname = 'mat_drop_test'; count ------- 0 --TEST With options CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions CREATE MATERIALIZED VIEW mat_with_test(timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket('1day', timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket('1day', timec), location, humidity, temperature WITH NO DATA; SELECT add_continuous_aggregate_policy('mat_with_test', NULL, '5 h'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1001 SELECT alter_job(id, schedule_interval => '1h') FROM _timescaledb_catalog.bgw_job; alter_job ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ (1001,"@ 1 hour","@ 0",-1,"@ 12 hours",t,"{""end_offset"": ""@ 5 hours"", ""start_offset"": null, ""mat_hypertable_id"": 20}",-infinity,_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1001]") SELECT schedule_interval FROM _timescaledb_catalog.bgw_job; schedule_interval ------------------- @ 1 hour SELECT alter_job(id, schedule_interval => '2h') FROM _timescaledb_catalog.bgw_job; alter_job ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1001,"@ 2 hours","@ 0",-1,"@ 12 hours",t,"{""end_offset"": ""@ 5 hours"", ""start_offset"": null, ""mat_hypertable_id"": 20}",-infinity,_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1001]") SELECT schedule_interval FROM _timescaledb_catalog.bgw_job; schedule_interval ------------------- @ 2 hours select indexname, indexdef from pg_indexes where tablename = (SELECT h.table_name FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_with_test') order by indexname; indexname | indexdef ---------------------------------------+---------------------------------------------------------------------------------------------------------------------------------- _materialized_hypertable_20_timec_idx | CREATE INDEX _materialized_hypertable_20_timec_idx ON _timescaledb_internal._materialized_hypertable_20 USING btree (timec DESC) DROP MATERIALIZED VIEW mat_with_test; --no additional indexes CREATE MATERIALIZED VIEW mat_with_test(timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true, timescaledb.create_group_indexes=false) as select time_bucket('1day', timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket('1day', timec), location, humidity, temperature WITH NO DATA; select indexname, indexdef from pg_indexes where tablename = (SELECT h.table_name FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_with_test'); indexname | indexdef ---------------------------------------+---------------------------------------------------------------------------------------------------------------------------------- _materialized_hypertable_21_timec_idx | CREATE INDEX _materialized_hypertable_21_timec_idx ON _timescaledb_internal._materialized_hypertable_21 USING btree (timec DESC) DROP TABLE conditions CASCADE; NOTICE: drop cascades to 2 other objects --test WITH using a hypertable with an integer time dimension CREATE TABLE conditions ( timec INT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec', chunk_time_interval=> 100); table_name ------------ conditions CREATE OR REPLACE FUNCTION integer_now_conditions() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now_conditions'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW mat_with_test(timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket(100, timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket(100, timec) WITH NO DATA; SELECT add_continuous_aggregate_policy('mat_with_test', NULL, 500::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1002 SELECT alter_job(id, schedule_interval => '2h') FROM _timescaledb_catalog.bgw_job; alter_job --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1002,"@ 2 hours","@ 0",-1,"@ 12 hours",t,"{""end_offset"": 500, ""start_offset"": null, ""mat_hypertable_id"": 23}",-infinity,_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1002]") SELECT schedule_interval FROM _timescaledb_catalog.bgw_job; schedule_interval ------------------- @ 2 hours DROP TABLE conditions CASCADE; NOTICE: drop cascades to 2 other objects --test space partitions CREATE TABLE space_table ( time BIGINT, dev BIGINT, data BIGINT ); SELECT create_hypertable( 'space_table', 'time', chunk_time_interval => 10, partitioning_column => 'dev', number_partitions => 3); create_hypertable --------------------------- (24,public,space_table,t) CREATE OR REPLACE FUNCTION integer_now_space_table() returns BIGINT LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), BIGINT '0') FROM space_table $$; SELECT set_integer_now_func('space_table', 'integer_now_space_table'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW space_view WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('4', time), COUNT(data) FROM space_table GROUP BY 1 WITH NO DATA; INSERT INTO space_table VALUES (0, 1, 1), (0, 2, 1), (1, 1, 1), (1, 2, 1), (10, 1, 1), (10, 2, 1), (11, 1, 1), (11, 2, 1); SELECT h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA", direct_view_name as "DIR_VIEW_NAME", direct_view_schema as "DIR_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'space_view' \gset SELECT * FROM :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" ORDER BY time_bucket; time_bucket | count -------------+------- CALL refresh_continuous_aggregate('space_view', NULL, NULL); SELECT * FROM space_view ORDER BY 1; time_bucket | count -------------+------- 0 | 4 8 | 4 SELECT * FROM :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" ORDER BY time_bucket; time_bucket | count -------------+------- 0 | 4 8 | 4 INSERT INTO space_table VALUES (3, 2, 1); CALL refresh_continuous_aggregate('space_view', NULL, NULL); SELECT * FROM space_view ORDER BY 1; time_bucket | count -------------+------- 0 | 5 8 | 4 SELECT * FROM :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" ORDER BY time_bucket; time_bucket | count -------------+------- 0 | 5 8 | 4 INSERT INTO space_table VALUES (2, 3, 1); CALL refresh_continuous_aggregate('space_view', NULL, NULL); SELECT * FROM space_view ORDER BY 1; time_bucket | count -------------+------- 0 | 6 8 | 4 SELECT * FROM :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME" ORDER BY time_bucket; time_bucket | count -------------+------- 0 | 6 8 | 4 DROP TABLE space_table CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_25_60_chunk -- -- TEST FINALIZEFUNC_EXTRA -- -- create special aggregate to test ffunc_extra -- Raise warning with the actual type being passed in CREATE OR REPLACE FUNCTION fake_ffunc(a int8, b int, c int, d int, x anyelement) RETURNS anyelement AS $$ BEGIN RAISE WARNING 'type % %', pg_typeof(d), pg_typeof(x); RETURN x; END; $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION fake_sfunc(a int8, b int, c int, d int, x anyelement) RETURNS int8 AS $$ BEGIN RETURN b; END; $$ LANGUAGE plpgsql; CREATE AGGREGATE aggregate_to_test_ffunc_extra(int, int, int, anyelement) ( SFUNC = fake_sfunc, STYPE = int8, COMBINEFUNC = int8pl, FINALFUNC = fake_ffunc, PARALLEL = SAFE, FINALFUNC_EXTRA ); CREATE TABLE conditions ( timec INT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec', chunk_time_interval=> 100); table_name ------------ conditions CREATE OR REPLACE FUNCTION integer_now_conditions() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now_conditions'); set_integer_now_func ---------------------- insert into conditions select generate_series(0, 200, 10), 'POR', 55, 75, 40, 70, NULL; CREATE MATERIALIZED VIEW mat_ffunc_test WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket(100, timec), aggregate_to_test_ffunc_extra(timec, 1, 3, 'test'::text) from conditions group by time_bucket(100, timec); NOTICE: refreshing continuous aggregate "mat_ffunc_test" WARNING: type integer text WARNING: type integer text WARNING: type integer text SELECT * FROM mat_ffunc_test ORDER BY time_bucket; time_bucket | aggregate_to_test_ffunc_extra -------------+------------------------------- 0 | 100 | 200 | DROP MATERIALIZED view mat_ffunc_test; NOTICE: drop cascades to table _timescaledb_internal._hyper_27_65_chunk CREATE MATERIALIZED VIEW mat_ffunc_test WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select time_bucket(100, timec), aggregate_to_test_ffunc_extra(timec, 4, 5, bigint '123') from conditions group by time_bucket(100, timec); NOTICE: refreshing continuous aggregate "mat_ffunc_test" WARNING: type integer bigint WARNING: type integer bigint WARNING: type integer bigint SELECT * FROM mat_ffunc_test ORDER BY time_bucket; time_bucket | aggregate_to_test_ffunc_extra -------------+------------------------------- 0 | 100 | 200 | --refresh mat view test when time_bucket is not projected -- DROP MATERIALIZED VIEW mat_ffunc_test; NOTICE: drop cascades to table _timescaledb_internal._hyper_28_66_chunk CREATE MATERIALIZED VIEW mat_refresh_test WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select location, max(humidity) from conditions group by time_bucket(100, timec), location WITH NO DATA; insert into conditions select generate_series(0, 50, 10), 'NYC', 55, 75, 40, 70, NULL; CALL refresh_continuous_aggregate('mat_refresh_test', NULL, NULL); SELECT * FROM mat_refresh_test order by 1,2 ; location | max ----------+----- NYC | 75 POR | 75 POR | 75 POR | 75 -- test for bug when group by is not in project list CREATE MATERIALIZED VIEW conditions_grpby_view with (timescaledb.continuous, timescaledb.materialized_only=false) as select time_bucket(100, timec), sum(humidity) from conditions group by time_bucket(100, timec), location; NOTICE: refreshing continuous aggregate "conditions_grpby_view" select * from conditions_grpby_view order by 1, 2; time_bucket | sum -------------+----- 0 | 450 0 | 750 100 | 750 200 | 75 CREATE MATERIALIZED VIEW conditions_grpby_view2 with (timescaledb.continuous, timescaledb.materialized_only=false) as select time_bucket(100, timec), sum(humidity) from conditions group by time_bucket(100, timec), location having avg(temperature) > 0; NOTICE: refreshing continuous aggregate "conditions_grpby_view2" select * from conditions_grpby_view2 order by 1, 2; time_bucket | sum -------------+----- 0 | 450 0 | 750 100 | 750 200 | 75 -- Test internal functions for continuous aggregates SELECT test.continuous_aggs_find_view('mat_refresh_test'); continuous_aggs_find_view --------------------------- -- Test pseudotype/enum handling CREATE TYPE status_enum AS ENUM ( 'red', 'yellow', 'green' ); CREATE TABLE cagg_types ( time TIMESTAMPTZ NOT NULL, status status_enum, names NAME[], floats FLOAT[] ); SELECT table_name FROM create_hypertable('cagg_types', 'time'); table_name ------------ cagg_types INSERT INTO cagg_types SELECT '2000-01-01', 'yellow', '{foo,bar,baz}', '{1,2.5,3}'; CREATE MATERIALIZED VIEW mat_types WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1d', time), min(status) AS status, max(names) AS names, min(floats) AS floats FROM cagg_types GROUP BY 1; NOTICE: refreshing continuous aggregate "mat_types" CALL refresh_continuous_aggregate('mat_types',NULL,NULL); NOTICE: continuous aggregate "mat_types" is already up-to-date SELECT * FROM mat_types; time_bucket | status | names | floats ------------------------------+--------+---------------+----------- Fri Dec 31 16:00:00 1999 PST | yellow | {foo,bar,baz} | {1,2.5,3} ------------------------------------------------------------------------------------- -- Test issue #2616 where cagg view contains an experssion with several aggregates in CREATE TABLE water_consumption ( sensor_id integer NOT NULL, timestamp timestamp(0) NOT NULL, water_index integer ); SELECT create_hypertable('water_consumption', 'timestamp', 'sensor_id', 2); WARNING: column type "timestamp without time zone" used for "timestamp" does not follow best practices create_hypertable --------------------------------- (34,public,water_consumption,t) INSERT INTO public.water_consumption (sensor_id, timestamp, water_index) VALUES (1, '2010-11-03 09:42:30', 1030), (1, '2010-11-03 09:42:40', 1032), (1, '2010-11-03 09:42:50', 1035), (1, '2010-11-03 09:43:30', 1040), (1, '2010-11-03 09:43:40', 1045), (1, '2010-11-03 09:43:50', 1050), (1, '2010-11-03 09:44:30', 1052), (1, '2010-11-03 09:44:40', 1057), (1, '2010-11-03 09:44:50', 1060), (1, '2010-11-03 09:45:30', 1063), (1, '2010-11-03 09:45:40', 1067), (1, '2010-11-03 09:45:50', 1070); -- The test with the view originally reported in the issue. CREATE MATERIALIZED VIEW water_consumption_aggregation_minute WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT sensor_id, time_bucket(INTERVAL '1 minute', timestamp) + '1 minute' AS timestamp, (max(water_index) - min(water_index)) AS water_consumption FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) WITH NO DATA; CALL refresh_continuous_aggregate('water_consumption_aggregation_minute', NULL, NULL); -- The results of the view and the query over hypertable should be the same SELECT * FROM water_consumption_aggregation_minute ORDER BY water_consumption; sensor_id | timestamp | water_consumption -----------+--------------------------+------------------- 1 | Wed Nov 03 09:43:00 2010 | 5 1 | Wed Nov 03 09:46:00 2010 | 7 1 | Wed Nov 03 09:45:00 2010 | 8 1 | Wed Nov 03 09:44:00 2010 | 10 SELECT sensor_id, time_bucket(INTERVAL '1 minute', timestamp) + '1 minute' AS timestamp, (max(water_index) - min(water_index)) AS water_consumption FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) ORDER BY water_consumption; sensor_id | timestamp | water_consumption -----------+--------------------------+------------------- 1 | Wed Nov 03 09:43:00 2010 | 5 1 | Wed Nov 03 09:46:00 2010 | 7 1 | Wed Nov 03 09:45:00 2010 | 8 1 | Wed Nov 03 09:44:00 2010 | 10 -- Simplified test, where the view doesn't contain all group by clauses CREATE MATERIALIZED VIEW water_consumption_no_select_bucket WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT sensor_id, (max(water_index) - min(water_index)) AS water_consumption FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) WITH NO DATA; CALL refresh_continuous_aggregate('water_consumption_no_select_bucket', NULL, NULL); -- The results of the view and the query over hypertable should be the same SELECT * FROM water_consumption_no_select_bucket ORDER BY water_consumption; sensor_id | water_consumption -----------+------------------- 1 | 5 1 | 7 1 | 8 1 | 10 SELECT sensor_id, (max(water_index) - min(water_index)) AS water_consumption FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) ORDER BY water_consumption; sensor_id | water_consumption -----------+------------------- 1 | 5 1 | 7 1 | 8 1 | 10 -- The test with SELECT matching GROUP BY and placing aggregate expression not the last CREATE MATERIALIZED VIEW water_consumption_aggregation_no_addition WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT sensor_id, (max(water_index) - min(water_index)) AS water_consumption, time_bucket(INTERVAL '1 minute', timestamp) AS timestamp FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) WITH NO DATA; CALL refresh_continuous_aggregate('water_consumption_aggregation_no_addition', NULL, NULL); -- The results of the view and the query over hypertable should be the same SELECT * FROM water_consumption_aggregation_no_addition ORDER BY water_consumption; sensor_id | water_consumption | timestamp -----------+-------------------+-------------------------- 1 | 5 | Wed Nov 03 09:42:00 2010 1 | 7 | Wed Nov 03 09:45:00 2010 1 | 8 | Wed Nov 03 09:44:00 2010 1 | 10 | Wed Nov 03 09:43:00 2010 SELECT sensor_id, (max(water_index) - min(water_index)) AS water_consumption, time_bucket(INTERVAL '1 minute', timestamp) AS timestamp FROM water_consumption GROUP BY sensor_id, time_bucket(INTERVAL '1 minute', timestamp) ORDER BY water_consumption; sensor_id | water_consumption | timestamp -----------+-------------------+-------------------------- 1 | 5 | Wed Nov 03 09:42:00 2010 1 | 7 | Wed Nov 03 09:45:00 2010 1 | 8 | Wed Nov 03 09:44:00 2010 1 | 10 | Wed Nov 03 09:43:00 2010 DROP TABLE water_consumption CASCADE; NOTICE: drop cascades to 6 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_35_73_chunk NOTICE: drop cascades to table _timescaledb_internal._hyper_36_74_chunk NOTICE: drop cascades to table _timescaledb_internal._hyper_37_75_chunk ---- --- github issue 2655 --- create table raw_data(time timestamptz, search_query text, cnt integer, cnt2 integer); select create_hypertable('raw_data','time', chunk_time_interval=>'15 days'::interval); create_hypertable ------------------------ (38,public,raw_data,t) insert into raw_data select '2000-01-01','Q1'; --having has exprs that appear in select CREATE MATERIALIZED VIEW search_query_count_1m WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT search_query,count(search_query) as count, time_bucket(INTERVAL '1 minute', time) AS bucket FROM raw_data WHERE search_query is not null AND LENGTH(TRIM(both from search_query))>0 GROUP BY search_query, bucket HAVING count(search_query) > 3 OR sum(cnt) > 1; NOTICE: refreshing continuous aggregate "search_query_count_1m" --having has aggregates + grp by columns that appear in select CREATE MATERIALIZED VIEW search_query_count_2 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT search_query,count(search_query) as count, sum(cnt), time_bucket(INTERVAL '1 minute', time) AS bucket FROM raw_data WHERE search_query is not null AND LENGTH(TRIM(both from search_query))>0 GROUP BY search_query, bucket HAVING count(search_query) > 3 OR sum(cnt) > 1 OR ( sum(cnt) + count(cnt)) > 1 AND search_query = 'Q1'; NOTICE: refreshing continuous aggregate "search_query_count_2" CREATE MATERIALIZED VIEW search_query_count_3 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT search_query,count(search_query) as count, sum(cnt), time_bucket(INTERVAL '1 minute', time) AS bucket FROM raw_data WHERE search_query is not null AND LENGTH(TRIM(both from search_query))>0 GROUP BY cnt +cnt2 , bucket, search_query HAVING cnt + cnt2 + sum(cnt) > 2 or count(cnt2) > 10; NOTICE: refreshing continuous aggregate "search_query_count_3" insert into raw_data select '2000-01-01 00:00+0','Q1', 1, 100; insert into raw_data select '2000-01-01 00:00+0','Q1', 2, 200; insert into raw_data select '2000-01-01 00:00+0','Q1', 3, 300; insert into raw_data select '2000-01-02 00:00+0','Q2', 10, 10; insert into raw_data select '2000-01-02 00:00+0','Q2', 20, 20; CALL refresh_continuous_aggregate('search_query_count_1m', NULL, NULL); SELECT * FROM search_query_count_1m ORDER BY 1, 2; search_query | count | bucket --------------+-------+------------------------------ Q1 | 3 | Fri Dec 31 16:00:00 1999 PST Q2 | 2 | Sat Jan 01 16:00:00 2000 PST --only 1 of these should appear in the result insert into raw_data select '2000-01-02 00:00+0','Q3', 0, 0; insert into raw_data select '2000-01-03 00:00+0','Q4', 20, 20; CALL refresh_continuous_aggregate('search_query_count_1m', NULL, NULL); SELECT * FROM search_query_count_1m ORDER BY 1, 2; search_query | count | bucket --------------+-------+------------------------------ Q1 | 3 | Fri Dec 31 16:00:00 1999 PST Q2 | 2 | Sat Jan 01 16:00:00 2000 PST Q4 | 1 | Sun Jan 02 16:00:00 2000 PST --refresh search_query_count_2--- CALL refresh_continuous_aggregate('search_query_count_2', NULL, NULL); SELECT * FROM search_query_count_2 ORDER BY 1, 2; search_query | count | sum | bucket --------------+-------+-----+------------------------------ Q1 | 3 | 6 | Fri Dec 31 16:00:00 1999 PST Q2 | 2 | 30 | Sat Jan 01 16:00:00 2000 PST Q4 | 1 | 20 | Sun Jan 02 16:00:00 2000 PST --refresh search_query_count_3--- CALL refresh_continuous_aggregate('search_query_count_3', NULL, NULL); SELECT * FROM search_query_count_3 ORDER BY 1, 2, 3; search_query | count | sum | bucket --------------+-------+-----+------------------------------ Q1 | 1 | 1 | Fri Dec 31 16:00:00 1999 PST Q1 | 1 | 2 | Fri Dec 31 16:00:00 1999 PST Q1 | 1 | 3 | Fri Dec 31 16:00:00 1999 PST Q2 | 1 | 10 | Sat Jan 01 16:00:00 2000 PST Q2 | 1 | 20 | Sat Jan 01 16:00:00 2000 PST Q4 | 1 | 20 | Sun Jan 02 16:00:00 2000 PST --- TEST enable compression on continuous aggregates CREATE VIEW cagg_compression_status as SELECT ca.mat_hypertable_id AS mat_htid, ca.user_view_name AS cagg_name , h.schema_name AS mat_schema_name, h.table_name AS mat_table_name, ca.materialized_only FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) ; SELECT mat_htid AS "MAT_HTID" , mat_schema_name || '.' || mat_table_name AS "MAT_HTNAME" , mat_table_name AS "MAT_TABLE_NAME" FROM cagg_compression_status WHERE cagg_name = 'search_query_count_3' \gset ALTER MATERIALIZED VIEW search_query_count_3 SET (timescaledb.compress = 'true'); NOTICE: defaulting compress_orderby to bucket,search_query SELECT cagg_name, mat_table_name FROM cagg_compression_status where cagg_name = 'search_query_count_3'; cagg_name | mat_table_name ----------------------+----------------------------- search_query_count_3 | _materialized_hypertable_41 \x SELECT * FROM timescaledb_information.compression_settings WHERE hypertable_name = :'MAT_TABLE_NAME'; -[ RECORD 1 ]----------+---------------------------- hypertable_schema | _timescaledb_internal hypertable_name | _materialized_hypertable_41 attname | bucket segmentby_column_index | orderby_column_index | 1 orderby_asc | t orderby_nullsfirst | f -[ RECORD 2 ]----------+---------------------------- hypertable_schema | _timescaledb_internal hypertable_name | _materialized_hypertable_41 attname | search_query segmentby_column_index | orderby_column_index | 2 orderby_asc | t orderby_nullsfirst | f \x SELECT compress_chunk(ch) FROM show_chunks('search_query_count_3') ch; compress_chunk ------------------------------------------ _timescaledb_internal._hyper_41_79_chunk SELECT * from search_query_count_3 ORDER BY 1, 2, 3; search_query | count | sum | bucket --------------+-------+-----+------------------------------ Q1 | 1 | 1 | Fri Dec 31 16:00:00 1999 PST Q1 | 1 | 2 | Fri Dec 31 16:00:00 1999 PST Q1 | 1 | 3 | Fri Dec 31 16:00:00 1999 PST Q2 | 1 | 10 | Sat Jan 01 16:00:00 2000 PST Q2 | 1 | 20 | Sat Jan 01 16:00:00 2000 PST Q4 | 1 | 20 | Sun Jan 02 16:00:00 2000 PST -- insert into a new region of the hypertable and then refresh the cagg -- (note we still do not support refreshes into existing regions. -- cagg chunks do not map 1-1 to hypertabl regions. They encompass -- more data -- ). insert into raw_data select '2000-05-01 00:00+0','Q3', 0, 0; -- On PG >= 14 the refresh test below will pass because we added support for UPDATE/DELETE on compressed chunks in PR #5339 \set ON_ERROR_STOP 0 CALL refresh_continuous_aggregate('search_query_count_3', NULL, '2000-06-01 00:00+0'::timestamptz); CALL refresh_continuous_aggregate('search_query_count_3', '2000-05-01 00:00+0'::timestamptz, '2000-06-01 00:00+0'::timestamptz); NOTICE: continuous aggregate "search_query_count_3" is already up-to-date \set ON_ERROR_STOP 1 --insert row insert into raw_data select '2001-05-10 00:00+0','Q3', 100, 100; --this should succeed since it does not refresh any compressed regions in the cagg CALL refresh_continuous_aggregate('search_query_count_3', '2001-05-01 00:00+0'::timestamptz, '2001-06-01 00:00+0'::timestamptz); --verify watermark and check that chunks are compressed SELECT _timescaledb_functions.to_timestamp(w) FROM _timescaledb_functions.cagg_watermark(:'MAT_HTID') w; to_timestamp ------------------------------ Wed May 09 17:01:00 2001 PDT SELECT chunk_name, range_start, range_end, is_compressed FROM timescaledb_information.chunks WHERE hypertable_name = :'MAT_TABLE_NAME' ORDER BY 1; chunk_name | range_start | range_end | is_compressed --------------------+------------------------------+------------------------------+--------------- _hyper_41_79_chunk | Fri Dec 24 16:00:00 1999 PST | Mon May 22 17:00:00 2000 PDT | t _hyper_41_83_chunk | Sun Mar 18 16:00:00 2001 PST | Wed Aug 15 17:00:00 2001 PDT | f SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id = :'MAT_HTID' ORDER BY 1, 2,3; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- 41 | -9223372036854775808 | -210866803200000001 41 | 959817600000000 | 988675199999999 41 | 991353600000000 | 9223372036854775807 SELECT * from search_query_count_3 WHERE bucket > '2001-01-01' ORDER BY 1, 2, 3; search_query | count | sum | bucket --------------+-------+-----+------------------------------ Q3 | 1 | 100 | Wed May 09 17:00:00 2001 PDT --now disable compression , will error out -- \set ON_ERROR_STOP 0 ALTER MATERIALIZED VIEW search_query_count_3 SET (timescaledb.compress = 'false'); ERROR: cannot disable columnstore on hypertable with columnstore chunks \set ON_ERROR_STOP 1 SELECT decompress_chunk(format('%I.%I', schema_name, table_name)) FROM _timescaledb_catalog.chunk WHERE hypertable_id = :'MAT_HTID' and status = 1; decompress_chunk ------------------------------------------ _timescaledb_internal._hyper_41_79_chunk --disable compression on cagg after decompressing all chunks-- ALTER MATERIALIZED VIEW search_query_count_3 SET (timescaledb.compress = 'false'); SELECT cagg_name, mat_table_name FROM cagg_compression_status where cagg_name = 'search_query_count_3'; cagg_name | mat_table_name ----------------------+----------------------------- search_query_count_3 | _materialized_hypertable_41 SELECT view_name, materialized_only, compression_enabled FROM timescaledb_information.continuous_aggregates where view_name = 'search_query_count_3'; view_name | materialized_only | compression_enabled ----------------------+-------------------+--------------------- search_query_count_3 | f | f -- TEST caggs on table with more columns than in the cagg view defn -- CREATE TABLE test_morecols ( time TIMESTAMPTZ NOT NULL, val1 INTEGER, val2 INTEGER, val3 INTEGER, val4 INTEGER, val5 INTEGER, val6 INTEGER, val7 INTEGER, val8 INTEGER); SELECT create_hypertable('test_morecols', 'time', chunk_time_interval=> '7 days'::interval); create_hypertable ----------------------------- (43,public,test_morecols,t) INSERT INTO test_morecols SELECT generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 55, 75, 40, 70, NULL, 100, 200, 200; CREATE MATERIALIZED VIEW test_morecols_cagg with (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('30 days',time), avg(val1), count(val2) FROM test_morecols GROUP BY 1; NOTICE: refreshing continuous aggregate "test_morecols_cagg" ALTER MATERIALIZED VIEW test_morecols_cagg SET (timescaledb.compress='true'); NOTICE: defaulting compress_orderby to time_bucket SELECT compress_chunk(ch) FROM show_chunks('test_morecols_cagg') ch; compress_chunk ------------------------------------------ _timescaledb_internal._hyper_44_89_chunk SELECT * FROM test_morecols_cagg ORDER BY time_bucket; time_bucket | avg | count ------------------------------+---------------------+------- Fri Nov 23 16:00:00 2018 PST | 55.0000000000000000 | 23 Sun Dec 23 16:00:00 2018 PST | 55.0000000000000000 | 8 SELECT view_name, materialized_only, compression_enabled FROM timescaledb_information.continuous_aggregates where view_name = 'test_morecols_cagg'; view_name | materialized_only | compression_enabled --------------------+-------------------+--------------------- test_morecols_cagg | f | t --should keep compressed option, modify only materialized -- ALTER MATERIALIZED VIEW test_morecols_cagg SET (timescaledb.materialized_only='true'); SELECT view_name, materialized_only, compression_enabled FROM timescaledb_information.continuous_aggregates where view_name = 'test_morecols_cagg'; view_name | materialized_only | compression_enabled --------------------+-------------------+--------------------- test_morecols_cagg | t | t CREATE TABLE issue3248(filler_1 int, filler_2 int, filler_3 int, time timestamptz NOT NULL, device_id int, v0 int, v1 int, v2 float, v3 float); CREATE INDEX ON issue3248(time DESC); CREATE INDEX ON issue3248(device_id,time DESC); SELECT create_hypertable('issue3248','time',create_default_indexes:=false); create_hypertable ------------------------- (46,public,issue3248,t) ALTER TABLE issue3248 DROP COLUMN filler_1; INSERT INTO issue3248(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id+1, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-01 0:00:00+0'::timestamptz,'2000-01-05 23:55:00+0','8h') gtime(time), generate_series(1,5,1) gdevice(device_id); ALTER TABLE issue3248 DROP COLUMN filler_2; INSERT INTO issue3248(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id-1, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-06 0:00:00+0'::timestamptz,'2000-01-12 23:55:00+0','8h') gtime(time), generate_series(1,5,1) gdevice(device_id); ALTER TABLE issue3248 DROP COLUMN filler_3; INSERT INTO issue3248(time,device_id,v0,v1,v2,v3) SELECT time, device_id, device_id, device_id + 2, device_id + 0.5, NULL FROM generate_series('2000-01-13 0:00:00+0'::timestamptz,'2000-01-19 23:55:00+0','8h') gtime(time), generate_series(1,5,1) gdevice(device_id); ANALYZE issue3248; CREATE materialized view issue3248_cagg WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h',time), device_id, min(v0), max(v1), avg(v2) FROM issue3248 GROUP BY 1,2; NOTICE: refreshing continuous aggregate "issue3248_cagg" SELECT FROM issue3248 AS m, LATERAL(SELECT m FROM issue3248_cagg WHERE avg IS NULL LIMIT 1) AS lat; -- -- test that option create_group_indexes is taken into account CREATE TABLE test_group_idx ( time timestamptz, symbol int, value numeric ); select create_hypertable('test_group_idx', 'time'); create_hypertable ------------------------------ (48,public,test_group_idx,t) insert into test_group_idx select t, round(random()*10), random()*5 from generate_series('2020-01-01', '2020-02-25', INTERVAL '12 hours') t; create materialized view cagg_index_true with (timescaledb.continuous, timescaledb.materialized_only=false, timescaledb.create_group_indexes=true) as select time_bucket('1 day', "time") as bucket, sum(value), symbol from test_group_idx group by bucket, symbol; NOTICE: refreshing continuous aggregate "cagg_index_true" create materialized view cagg_index_false with (timescaledb.continuous, timescaledb.materialized_only=false, timescaledb.create_group_indexes=false) as select time_bucket('1 day', "time") as bucket, sum(value), symbol from test_group_idx group by bucket, symbol; NOTICE: refreshing continuous aggregate "cagg_index_false" create materialized view cagg_index_default with (timescaledb.continuous, timescaledb.materialized_only=false) as select time_bucket('1 day', "time") as bucket, sum(value), symbol from test_group_idx group by bucket, symbol; NOTICE: refreshing continuous aggregate "cagg_index_default" -- see corresponding materialization_hypertables select view_name, materialization_hypertable_name from timescaledb_information.continuous_aggregates ca where view_name like 'cagg_index_%' ORDER BY view_name; view_name | materialization_hypertable_name --------------------+--------------------------------- cagg_index_default | _materialized_hypertable_51 cagg_index_false | _materialized_hypertable_50 cagg_index_true | _materialized_hypertable_49 -- now make sure a group index has been created when explicitly asked for \x on select i.* from pg_indexes i join pg_class c on schemaname = relnamespace::regnamespace::text and tablename = relname where tablename in (select materialization_hypertable_name from timescaledb_information.continuous_aggregates where view_name like 'cagg_index_%') order by tablename, indexname; -[ RECORD 1 ]------------------------------------------------------------------------------------------------------------------------------------------------- schemaname | _timescaledb_internal tablename | _materialized_hypertable_49 indexname | _materialized_hypertable_49_bucket_idx tablespace | indexdef | CREATE INDEX _materialized_hypertable_49_bucket_idx ON _timescaledb_internal._materialized_hypertable_49 USING btree (bucket DESC) -[ RECORD 2 ]------------------------------------------------------------------------------------------------------------------------------------------------- schemaname | _timescaledb_internal tablename | _materialized_hypertable_49 indexname | _materialized_hypertable_49_symbol_bucket_idx tablespace | indexdef | CREATE INDEX _materialized_hypertable_49_symbol_bucket_idx ON _timescaledb_internal._materialized_hypertable_49 USING btree (symbol, bucket DESC) -[ RECORD 3 ]------------------------------------------------------------------------------------------------------------------------------------------------- schemaname | _timescaledb_internal tablename | _materialized_hypertable_50 indexname | _materialized_hypertable_50_bucket_idx tablespace | indexdef | CREATE INDEX _materialized_hypertable_50_bucket_idx ON _timescaledb_internal._materialized_hypertable_50 USING btree (bucket DESC) -[ RECORD 4 ]------------------------------------------------------------------------------------------------------------------------------------------------- schemaname | _timescaledb_internal tablename | _materialized_hypertable_51 indexname | _materialized_hypertable_51_bucket_idx tablespace | indexdef | CREATE INDEX _materialized_hypertable_51_bucket_idx ON _timescaledb_internal._materialized_hypertable_51 USING btree (bucket DESC) -[ RECORD 5 ]------------------------------------------------------------------------------------------------------------------------------------------------- schemaname | _timescaledb_internal tablename | _materialized_hypertable_51 indexname | _materialized_hypertable_51_symbol_bucket_idx tablespace | indexdef | CREATE INDEX _materialized_hypertable_51_symbol_bucket_idx ON _timescaledb_internal._materialized_hypertable_51 USING btree (symbol, bucket DESC) \x off -- -- TESTs for removing old CAggs restrictions -- DROP TABLE conditions CASCADE; NOTICE: drop cascades to 8 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_29_67_chunk NOTICE: drop cascades to table _timescaledb_internal._hyper_30_68_chunk NOTICE: drop cascades to table _timescaledb_internal._hyper_31_69_chunk CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL ); SELECT create_hypertable('conditions', 'timec'); create_hypertable -------------------------- (52,public,conditions,t) INSERT INTO conditions VALUES ('2010-01-01 09:00:00-08', 'SFO', 55, 45), ('2010-01-02 09:00:00-08', 'por', 100, 100), ('2010-01-02 09:00:00-08', 'NYC', 65, 45), ('2010-01-02 09:00:00-08', 'SFO', 65, 45), ('2010-01-03 09:00:00-08', 'NYC', 45, 55), ('2010-01-05 09:00:00-08', 'SFO', 75, 100), ('2018-11-01 09:00:00-08', 'NYC', 45, 35), ('2018-11-02 09:00:00-08', 'NYC', 35, 15), ('2018-11-03 09:00:00-08', 'NYC', 35, 25); -- aggregate with DISTINCT CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), COUNT(location), SUM(DISTINCT temperature) FROM conditions GROUP BY time_bucket('1week', timec), location; NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1, 2, 3; time_bucket | count | sum ------------------------------+-------+----- Sun Dec 27 16:00:00 2009 PST | 1 | 100 Sun Dec 27 16:00:00 2009 PST | 2 | 110 Sun Dec 27 16:00:00 2009 PST | 2 | 120 Sun Jan 03 16:00:00 2010 PST | 1 | 75 Sun Oct 28 17:00:00 2018 PDT | 3 | 80 -- aggregate with FILTER DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), SUM(temperature) FILTER (WHERE humidity > 60) FROM conditions GROUP BY time_bucket('1week', timec), location; NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1, 2; time_bucket | sum ------------------------------+----- Sun Dec 27 16:00:00 2009 PST | 100 Sun Dec 27 16:00:00 2009 PST | Sun Dec 27 16:00:00 2009 PST | Sun Jan 03 16:00:00 2010 PST | 75 Sun Oct 28 17:00:00 2018 PDT | -- aggregate with filter in having clause DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), MAX(temperature) FROM conditions GROUP BY time_bucket('1week', timec), location HAVING SUM(temperature) FILTER (WHERE humidity > 40) > 50; NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1, 2; time_bucket | max ------------------------------+----- Sun Dec 27 16:00:00 2009 PST | 65 Sun Dec 27 16:00:00 2009 PST | 65 Sun Dec 27 16:00:00 2009 PST | 100 Sun Jan 03 16:00:00 2010 PST | 75 -- ordered set aggr DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to table _timescaledb_internal._hyper_55_116_chunk CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), MODE() WITHIN GROUP(ORDER BY humidity) FROM conditions GROUP BY time_bucket('1week', timec); NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1; time_bucket | mode ------------------------------+------ Sun Dec 27 16:00:00 2009 PST | 45 Sun Jan 03 16:00:00 2010 PST | 100 Sun Oct 28 17:00:00 2018 PDT | 15 -- hypothetical-set aggr DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), RANK(60) WITHIN GROUP (ORDER BY humidity), DENSE_RANK(60) WITHIN GROUP (ORDER BY humidity), PERCENT_RANK(60) WITHIN GROUP (ORDER BY humidity) FROM conditions GROUP BY time_bucket('1week', timec); NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1; time_bucket | rank | dense_rank | percent_rank ------------------------------+------+------------+-------------- Sun Dec 27 16:00:00 2009 PST | 5 | 3 | 0.8 Sun Jan 03 16:00:00 2010 PST | 1 | 1 | 0 Sun Oct 28 17:00:00 2018 PDT | 4 | 4 | 1 -- userdefined aggregate without combine function DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE AGGREGATE newavg ( sfunc = int4_avg_accum, basetype = int4, stype = _int8, finalfunc = int8_avg, initcond1 = '{0,0}' ); CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT SUM(humidity), round(newavg(temperature::int4)) FROM conditions GROUP BY time_bucket('1week', timec), location ORDER BY 1,2; NOTICE: refreshing continuous aggregate "mat_m1" SELECT * FROM mat_m1 ORDER BY 1, 2; sum | round -----+------- 75 | 38 90 | 60 100 | 55 100 | 75 100 | 100 -- ORDER BY in the view definition DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1week', timec), COUNT(location), SUM(temperature) FROM conditions GROUP BY time_bucket('1week', timec) ORDER BY sum DESC; NOTICE: refreshing continuous aggregate "mat_m1" -- CAgg definition for realtime SELECT pg_get_viewdef('mat_m1',true); pg_get_viewdef ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ( SELECT _materialized_hypertable_59.time_bucket, + _materialized_hypertable_59.count, + _materialized_hypertable_59.sum + FROM _timescaledb_internal._materialized_hypertable_59 + WHERE _materialized_hypertable_59.time_bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(59)), '-infinity'::timestamp with time zone)+ ORDER BY _materialized_hypertable_59.sum DESC) + UNION ALL + ( SELECT time_bucket('@ 7 days'::interval, conditions.timec) AS time_bucket, + count(conditions.location) AS count, + sum(conditions.temperature) AS sum + FROM conditions + WHERE conditions.timec >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(59)), '-infinity'::timestamp with time zone) + GROUP BY (time_bucket('@ 7 days'::interval, conditions.timec)) + ORDER BY (sum(conditions.temperature)) DESC) + ORDER BY 3 DESC; -- Ordered result SELECT * FROM mat_m1; time_bucket | count | sum ------------------------------+-------+----- Sun Dec 27 16:00:00 2009 PST | 5 | 330 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Jan 03 16:00:00 2010 PST | 1 | 75 -- Insert new data and query again to make sure we produce ordered data INSERT INTO conditions VALUES ('2018-11-10 09:00:00-08', 'SFO', 10, 10); SELECT * FROM mat_m1; time_bucket | count | sum ------------------------------+-------+----- Sun Dec 27 16:00:00 2009 PST | 5 | 330 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Jan 03 16:00:00 2010 PST | 1 | 75 Sun Nov 04 16:00:00 2018 PST | 1 | 10 -- This new row will change the order again INSERT INTO conditions VALUES ('2018-11-11 09:00:00-08', 'SFO', 400, 400); SELECT * FROM mat_m1; time_bucket | count | sum ------------------------------+-------+----- Sun Nov 04 16:00:00 2018 PST | 2 | 410 Sun Dec 27 16:00:00 2009 PST | 5 | 330 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Jan 03 16:00:00 2010 PST | 1 | 75 -- Merge Append EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM mat_m1; --- QUERY PLAN --- Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Index Scan Backward using _hyper_59_123_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_123_chunk -> Index Scan Backward using _hyper_59_124_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_124_chunk Index Cond: (time_bucket < 'Sun Nov 04 16:00:00 2018 PST'::timestamp with time zone) -> Sort Sort Key: (sum(conditions.temperature)) DESC -> Finalize HashAggregate Group Key: (time_bucket('@ 7 days'::interval, conditions.timec)) -> Append -> Partial HashAggregate Group Key: time_bucket('@ 7 days'::interval, _hyper_52_111_chunk.timec) -> Index Scan Backward using _hyper_52_111_chunk_conditions_timec_idx on _hyper_52_111_chunk Index Cond: (timec >= 'Sun Nov 04 16:00:00 2018 PST'::timestamp with time zone) -> Partial HashAggregate Group Key: time_bucket('@ 7 days'::interval, _hyper_52_125_chunk.timec) -> Seq Scan on _hyper_52_125_chunk -- Ordering by another column SELECT * FROM mat_m1 ORDER BY count; time_bucket | count | sum ------------------------------+-------+----- Sun Jan 03 16:00:00 2010 PST | 1 | 75 Sun Nov 04 16:00:00 2018 PST | 2 | 410 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Dec 27 16:00:00 2009 PST | 5 | 330 EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM mat_m1 ORDER BY count; --- QUERY PLAN --- Sort Sort Key: _materialized_hypertable_59.count -> Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Index Scan Backward using _hyper_59_123_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_123_chunk -> Index Scan Backward using _hyper_59_124_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_124_chunk Index Cond: (time_bucket < 'Sun Nov 04 16:00:00 2018 PST'::timestamp with time zone) -> Sort Sort Key: (sum(conditions.temperature)) DESC -> Finalize HashAggregate Group Key: (time_bucket('@ 7 days'::interval, conditions.timec)) -> Append -> Partial HashAggregate Group Key: time_bucket('@ 7 days'::interval, _hyper_52_111_chunk.timec) -> Index Scan Backward using _hyper_52_111_chunk_conditions_timec_idx on _hyper_52_111_chunk Index Cond: (timec >= 'Sun Nov 04 16:00:00 2018 PST'::timestamp with time zone) -> Partial HashAggregate Group Key: time_bucket('@ 7 days'::interval, _hyper_52_125_chunk.timec) -> Seq Scan on _hyper_52_125_chunk -- Change the type of cagg ALTER MATERIALIZED VIEW mat_m1 SET (timescaledb.materialized_only=true); -- CAgg definition for materialized only SELECT pg_get_viewdef('mat_m1',true); pg_get_viewdef ----------------------------------------------------------- SELECT time_bucket, + count, + sum + FROM _timescaledb_internal._materialized_hypertable_59+ ORDER BY sum DESC; -- Now the query will show only the materialized data, without last two -- records inserted into the original hypertable (last two insers above) SELECT * FROM mat_m1; time_bucket | count | sum ------------------------------+-------+----- Sun Dec 27 16:00:00 2009 PST | 5 | 330 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Jan 03 16:00:00 2010 PST | 1 | 75 -- Merge Append EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM mat_m1; --- QUERY PLAN --- Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Index Scan Backward using _hyper_59_123_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_123_chunk -> Index Scan Backward using _hyper_59_124_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_124_chunk -- Ordering by another column SELECT * FROM mat_m1 ORDER BY count; time_bucket | count | sum ------------------------------+-------+----- Sun Jan 03 16:00:00 2010 PST | 1 | 75 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Dec 27 16:00:00 2009 PST | 5 | 330 EXPLAIN (BUFFERS OFF, COSTS OFF) SELECT * FROM mat_m1 ORDER BY count; --- QUERY PLAN --- Sort Sort Key: _materialized_hypertable_59.count -> Merge Append Sort Key: _materialized_hypertable_59.sum DESC -> Index Scan Backward using _hyper_59_123_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_123_chunk -> Index Scan Backward using _hyper_59_124_chunk__materialized_hypertable_59_sum_time_bucket on _hyper_59_124_chunk SELECT h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_m1' \gset -- Invalidate old region and refresh again DELETE FROM conditions WHERE timec < '2010-01-05 09:00:00-08'; CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); -- Querying the cagg produce ordered records as expected SELECT * FROM mat_m1; time_bucket | count | sum ------------------------------+-------+----- Sun Nov 04 16:00:00 2018 PST | 2 | 410 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Jan 03 16:00:00 2010 PST | 1 | 75 -- Querying direct the materialization hypertable doesn't -- produce ordered records SELECT * FROM :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME"; time_bucket | count | sum ------------------------------+-------+----- Sun Jan 03 16:00:00 2010 PST | 1 | 75 Sun Oct 28 17:00:00 2018 PDT | 3 | 115 Sun Nov 04 16:00:00 2018 PST | 2 | 410 DROP TABLE conditions CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL ); SELECT table_name FROM create_hypertable('conditions', 'timec'); table_name ------------ conditions INSERT INTO conditions VALUES ('2010-01-01 09:00:00-08', 'SFO', 55, 45), ('2010-01-02 09:00:00-08', 'por', 100, 100), ('2010-01-02 09:00:00-08', 'SFO', 65, 45), ('2010-01-02 09:00:00-08', 'NYC', 65, 45), ('2018-11-01 09:00:00-08', 'NYC', 45, 35), ('2018-11-02 09:00:00-08', 'NYC', 35, 15); CREATE MATERIALIZED VIEW conditions_summary_new(timec, minl, sumt, sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1day', timec), min(location), sum(temperature), sum(humidity) FROM conditions GROUP BY time_bucket('1day', timec) WITH NO DATA; \x ON SELECT * FROM timescaledb_information.continuous_aggregates WHERE view_name = 'conditions_summary_new'; -[ RECORD 1 ]---------------------+---------------------------------------------------------- hypertable_schema | public hypertable_name | conditions view_schema | public view_name | conditions_summary_new view_owner | default_perm_user materialized_only | t compression_enabled | f materialization_hypertable_schema | _timescaledb_internal materialization_hypertable_name | _materialized_hypertable_61 view_definition | SELECT time_bucket('@ 1 day'::interval, timec) AS timec,+ | min(location) AS minl, + | sum(temperature) AS sumt, + | sum(humidity) AS sumh + | FROM conditions + | GROUP BY (time_bucket('@ 1 day'::interval, timec)); \x OFF CALL refresh_continuous_aggregate('conditions_summary_new', NULL, NULL); -- Check and compare number of returned rows SELECT count(*) FROM conditions_summary_new; count ------- 4 -- Parallel planning test for realtime Continuous Aggregate DROP TABLE conditions CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, temperature DOUBLE PRECISION NULL ); SELECT table_name FROM create_hypertable('conditions', 'timec'); table_name ------------ conditions INSERT INTO conditions SELECT t, 10 FROM generate_series('2023-01-01 00:00-03'::timestamptz, '2023-12-31 23:59-03'::timestamptz, '1 hour'::interval) AS t; CREATE MATERIALIZED VIEW conditions_daily WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1 day', timec), SUM(temperature) FROM conditions GROUP BY 1 ORDER BY 2 DESC; NOTICE: refreshing continuous aggregate "conditions_daily" SELECT set_config(CASE WHEN current_setting('server_version_num')::int < 160000 THEN 'force_parallel_mode' ELSE 'debug_parallel_query' END,'on', false); set_config ------------ on SET max_parallel_workers_per_gather = 4; SET parallel_setup_cost = 0; SET parallel_tuple_cost = 0; -- Parallel planning EXPLAIN (BUFFERS OFF, COSTS OFF, TIMING OFF) SELECT * FROM conditions_daily WHERE time_bucket >= '2023-07-01'; --- QUERY PLAN --- Merge Append Sort Key: _materialized_hypertable_63.sum DESC -> Gather Merge Workers Planned: 2 -> Sort Sort Key: _materialized_hypertable_63.sum DESC -> Parallel Append -> Parallel Index Scan using _hyper_63_185_chunk__materialized_hypertable_63_time_bucket_idx on _hyper_63_185_chunk Index Cond: ((time_bucket < 'Mon Jan 01 16:00:00 2024 PST'::timestamp with time zone) AND (time_bucket >= 'Sat Jul 01 00:00:00 2023 PDT'::timestamp with time zone)) -> Parallel Index Scan using _hyper_63_187_chunk__materialized_hypertable_63_time_bucket_idx on _hyper_63_187_chunk Index Cond: ((time_bucket < 'Mon Jan 01 16:00:00 2024 PST'::timestamp with time zone) AND (time_bucket >= 'Sat Jul 01 00:00:00 2023 PDT'::timestamp with time zone)) -> Parallel Seq Scan on _hyper_63_184_chunk -> Sort Sort Key: (sum(_hyper_62_182_chunk.temperature)) DESC -> HashAggregate Group Key: (time_bucket('@ 1 day'::interval, _hyper_62_182_chunk.timec)) -> Gather Workers Planned: 1 -> Result -> Parallel Index Scan Backward using _hyper_62_182_chunk_conditions_timec_idx on _hyper_62_182_chunk Index Cond: ((timec >= 'Mon Jan 01 16:00:00 2024 PST'::timestamp with time zone) AND (timec >= 'Sat Jul 01 00:00:00 2023 PDT'::timestamp with time zone)) Filter: (time_bucket('@ 1 day'::interval, timec) >= 'Sat Jul 01 00:00:00 2023 PDT'::timestamp with time zone) ================================================ FILE: tsl/test/expected/cagg_bgw-15.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- -- Setup -- \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(timeout INT = -1, mock_start_time INT = 0) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_run(timeout INT = -1, mock_start_time INT = 0) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_wait_for_scheduler_finish() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_create() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_destroy() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_reset_time(set_time BIGINT = 0, wait BOOLEAN = false) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; --test that this all works under the community license ALTER DATABASE :TEST_DBNAME SET timescaledb.license_key='Community'; --create a function with no permissions to execute CREATE FUNCTION get_constant_no_perms() RETURNS INTEGER LANGUAGE SQL IMMUTABLE AS $BODY$ SELECT 10; $BODY$; REVOKE EXECUTE ON FUNCTION get_constant_no_perms() FROM PUBLIC; \set WAIT_ON_JOB 0 \set IMMEDIATELY_SET_UNTIL 1 \set WAIT_FOR_OTHER_TO_ADVANCE 2 CREATE OR REPLACE FUNCTION ts_bgw_params_mock_wait_returns_immediately(new_val INTEGER) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; -- Remove any default jobs, e.g., telemetry DELETE FROM _timescaledb_catalog.bgw_job WHERE TRUE; TRUNCATE _timescaledb_internal.bgw_job_stat; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE public.bgw_log( msg_no INT, mock_time BIGINT, application_name TEXT, msg TEXT ); CREATE VIEW sorted_bgw_log AS SELECT msg_no, mock_time, application_name, regexp_replace(regexp_replace(msg, '(Wait until|started at|execution time) [0-9]+(\.[0-9]+)?', '\1 (RANDOM)', 'g'), 'background worker "[^"]+"','connection') AS msg FROM bgw_log ORDER BY mock_time, application_name COLLATE "C", msg_no; CREATE TABLE public.bgw_dsm_handle_store( handle BIGINT ); INSERT INTO public.bgw_dsm_handle_store VALUES (0); SELECT ts_bgw_params_create(); ts_bgw_params_create ---------------------- SELECT * FROM _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- SELECT * FROM timescaledb_information.job_stats; hypertable_schema | hypertable_name | job_id | last_run_started_at | last_successful_finish | last_run_status | job_status | last_run_duration | next_start | total_runs | total_successes | total_failures -------------------+-----------------+--------+---------------------+------------------------+-----------------+------------+-------------------+------------+------------+-----------------+---------------- SELECT * FROM _timescaledb_catalog.continuous_agg; mat_hypertable_id | raw_hypertable_id | parent_mat_hypertable_id | user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name | materialized_only -------------------+-------------------+--------------------------+------------------+----------------+---------------------+-------------------+--------------------+------------------+------------------- -- though user on access node has required GRANTS, this will propagate GRANTS to the connected data nodes GRANT CREATE ON SCHEMA public TO :ROLE_DEFAULT_PERM_USER; WARNING: no privileges were granted for "public" \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE test_continuous_agg_table(time int, data int); SELECT create_hypertable('test_continuous_agg_table', 'time', chunk_time_interval => 10); create_hypertable ---------------------------------------- (1,public,test_continuous_agg_table,t) CREATE OR REPLACE FUNCTION integer_now_test() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), 0) FROM test_continuous_agg_table $$; SELECT set_integer_now_func('test_continuous_agg_table', 'integer_now_test'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW test_continuous_agg_view WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time), SUM(data) as value FROM test_continuous_agg_table GROUP BY 1 WITH NO DATA; SELECT add_continuous_aggregate_policy('test_continuous_agg_view', NULL, 4::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1000 SELECT id as raw_table_id FROM _timescaledb_catalog.hypertable WHERE table_name='test_continuous_agg_table' \gset -- min distance from end should be 1 SELECT mat_hypertable_id, user_view_schema, user_view_name FROM _timescaledb_catalog.continuous_agg; mat_hypertable_id | user_view_schema | user_view_name -------------------+------------------+-------------------------- 2 | public | test_continuous_agg_view SELECT mat_hypertable_id, bucket_width FROM _timescaledb_catalog.continuous_aggs_bucket_function; mat_hypertable_id | bucket_width -------------------+-------------- 2 | 2 SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg \gset SELECT id AS job_id FROM _timescaledb_catalog.bgw_job where hypertable_id=:mat_hypertable_id \gset -- job was created SELECT * FROM _timescaledb_catalog.bgw_job where hypertable_id=:mat_hypertable_id; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ------+--------------------------------------------+-------------------+-------------+-------------+--------------+------------------------+-------------------------------------+-------------------+-----------+----------------+---------------+---------------+-----------------------------------------------------------------+------------------------+-------------------------------------------+---------- 1000 | Refresh Continuous Aggregate Policy [1000] | @ 12 hours | @ 0 | -1 | @ 12 hours | _timescaledb_functions | policy_refresh_continuous_aggregate | default_perm_user | t | f | | 2 | {"end_offset": 4, "start_offset": null, "mat_hypertable_id": 2} | _timescaledb_functions | policy_refresh_continuous_aggregate_check | -- create 10 time buckets INSERT INTO test_continuous_agg_table SELECT i, i FROM (SELECT generate_series(0, 10) as i) AS j; -- no stats SELECT job_id, next_start, last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat ORDER BY job_id; job_id | next_start | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------+------------+------------------+------------+-----------------+----------------+--------------- -- no data in view SELECT * FROM test_continuous_agg_view ORDER BY 1; time_bucket | value -------------+------- -- run first time SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+--------------------------------------------+------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1000] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -2147483648, 6 ] 1 | 0 | Refresh Continuous Aggregate Policy [1000] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 2 | 0 | Refresh Continuous Aggregate Policy [1000] | inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" SELECT * FROM _timescaledb_catalog.bgw_job where id=:job_id; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ------+--------------------------------------------+-------------------+-------------+-------------+--------------+------------------------+-------------------------------------+-------------------+-----------+----------------+---------------+---------------+-----------------------------------------------------------------+------------------------+-------------------------------------------+---------- 1000 | Refresh Continuous Aggregate Policy [1000] | @ 12 hours | @ 0 | -1 | @ 12 hours | _timescaledb_functions | policy_refresh_continuous_aggregate | default_perm_user | t | f | | 2 | {"end_offset": 4, "start_offset": null, "mat_hypertable_id": 2} | _timescaledb_functions | policy_refresh_continuous_aggregate_check | -- job ran once, successfully SELECT job_id, next_start-last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------+------------------+------------+-----------------+----------------+--------------- 1000 | @ 12 hours | t | 1 | 1 | 0 | 0 --clear log for next run of scheduler. TRUNCATE public.bgw_log; CREATE FUNCTION wait_for_timer_to_run(started_at INTEGER, spins INTEGER=:TEST_SPINWAIT_ITERS) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE num_runs INTEGER; message TEXT; BEGIN select format('[TESTING] Wait until %%, started at %s', started_at) into message; FOR i in 1..spins LOOP SELECT COUNT(*) from bgw_log where msg LIKE message INTO num_runs; if (num_runs > 0) THEN RETURN true; ELSE PERFORM pg_sleep(0.1); END IF; END LOOP; RETURN false; END $BODY$; --make sure there is 1 job to start with SELECT test.wait_for_job_to_run(:job_id, 1); wait_for_job_to_run --------------------- t SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_FOR_OTHER_TO_ADVANCE); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- --start the scheduler on 0 time SELECT ts_bgw_params_reset_time(0, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run(extract(epoch from interval '24 hour')::int * 1000, 0); ts_bgw_db_scheduler_test_run ------------------------------ SELECT wait_for_timer_to_run(0); wait_for_timer_to_run ----------------------- t --advance to 12:00 so that it runs one more time; now we know the --scheduler has loaded up the job with the old schedule_interval SELECT ts_bgw_params_reset_time(extract(epoch from interval '12 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- SELECT test.wait_for_job_to_run(:job_id, 2); wait_for_job_to_run --------------------- t --advance clock 1us to make the scheduler realize the job is done SELECT ts_bgw_params_reset_time((extract(epoch from interval '12 hour')::bigint * 1000000)+1, true); ts_bgw_params_reset_time -------------------------- --alter the refresh interval and check if next_start is altered SELECT alter_job(:job_id, schedule_interval => '1m', retry_period => '1m'); alter_job ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1000,"@ 1 min","@ 0",-1,"@ 1 min",t,"{""end_offset"": 4, ""start_offset"": null, ""mat_hypertable_id"": 2}","Sat Jan 01 04:01:00 2000 PST",_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1000]") SELECT job_id, next_start - last_finish as until_next, total_runs FROM _timescaledb_internal.bgw_job_stat WHERE job_id=:job_id;; job_id | until_next | total_runs --------+------------+------------ 1000 | @ 1 min | 2 --advance to 12:02, job should have run at 12:01 SELECT ts_bgw_params_reset_time((extract(epoch from interval '12 hour')::bigint * 1000000)+(extract(epoch from interval '2 minute')::bigint * 1000000), true); ts_bgw_params_reset_time -------------------------- SELECT test.wait_for_job_to_run(:job_id, 3); wait_for_job_to_run --------------------- t --next run in 1 minute SELECT job_id, next_start-last_finish as until_next, total_runs FROM _timescaledb_internal.bgw_job_stat WHERE job_id=:job_id; job_id | until_next | total_runs --------+------------+------------ 1000 | @ 1 min | 3 --change next run to be after 30s instead SELECT (next_start - '30s'::interval) AS "NEW_NEXT_START" FROM _timescaledb_internal.bgw_job_stat WHERE job_id=:job_id \gset SELECT alter_job(:job_id, next_start => :'NEW_NEXT_START'); alter_job ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1000,"@ 1 min","@ 0",-1,"@ 1 min",t,"{""end_offset"": 4, ""start_offset"": null, ""mat_hypertable_id"": 2}","Sat Jan 01 04:02:30 2000 PST",_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1000]") SELECT ts_bgw_params_reset_time((extract(epoch from interval '12 hour')::bigint * 1000000)+(extract(epoch from interval '2 minute 30 seconds')::bigint * 1000000), true); ts_bgw_params_reset_time -------------------------- SELECT test.wait_for_job_to_run(:job_id, 4); wait_for_job_to_run --------------------- t --advance clock to quit scheduler SELECT ts_bgw_params_reset_time(extract(epoch from interval '25 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- select ts_bgw_db_scheduler_test_wait_for_scheduler_finish(); ts_bgw_db_scheduler_test_wait_for_scheduler_finish ---------------------------------------------------- SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_ON_JOB); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- TRUNCATE public.bgw_log; -- data before 8 SELECT * FROM test_continuous_agg_view ORDER BY 1; time_bucket | value -------------+------- 0 | 1 2 | 5 4 | 9 -- invalidations test by running job multiple times SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- DROP MATERIALIZED VIEW test_continuous_agg_view; NOTICE: drop cascades to table _timescaledb_internal._hyper_2_3_chunk CREATE MATERIALIZED VIEW test_continuous_agg_view WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time), SUM(data) as value FROM test_continuous_agg_table GROUP BY 1 WITH NO DATA; SELECT add_continuous_aggregate_policy('test_continuous_agg_view', 100::integer, -2::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1001 SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg \gset SELECT id AS job_id FROM _timescaledb_catalog.bgw_job WHERE hypertable_id=:mat_hypertable_id \gset SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+--------------------------------------------+--------------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 10, 12 ] (batch 1 of 2) 1 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 3 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -10, 10 ] (batch 2 of 2) 4 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 5 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" -- job ran once, successfully SELECT job_id, last_finish - next_start as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+----------------+------------------+------------+-----------------+----------------+--------------- 1001 | @ 12 hours ago | t | 1 | 1 | 0 | 0 -- should have refreshed everything we have so far SELECT * FROM test_continuous_agg_view ORDER BY 1; time_bucket | value -------------+------- 0 | 1 2 | 5 4 | 9 6 | 13 8 | 17 10 | 10 -- invalidate some data UPDATE test_continuous_agg_table SET data = 11 WHERE time = 6; --advance time by 12h so that job runs one more time SELECT ts_bgw_params_reset_time(extract(epoch from interval '12 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25, 25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-------------+--------------------------------------------+--------------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 10, 12 ] (batch 1 of 2) 1 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 3 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -10, 10 ] (batch 2 of 2) 4 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 5 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 0 | 43200000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 43200000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -90, -10 ] 1 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | inserted 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 3 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 6, 8 ] 4 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 5 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" SELECT job_id, next_start - last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------+------------------+------------+-----------------+----------------+--------------- 1001 | @ 12 hours | t | 2 | 2 | 0 | 0 -- should have updated data for time=6 SELECT * FROM test_continuous_agg_view ORDER BY 1; time_bucket | value -------------+------- 0 | 1 2 | 5 4 | 9 6 | 18 8 | 17 10 | 10 \x on --check the information views -- select view_name, view_owner, materialization_hypertable_schema, materialization_hypertable_name from timescaledb_information.continuous_aggregates where view_name::text like '%test_continuous_agg_view'; -[ RECORD 1 ]---------------------+--------------------------- view_name | test_continuous_agg_view view_owner | default_perm_user materialization_hypertable_schema | _timescaledb_internal materialization_hypertable_name | _materialized_hypertable_3 select view_name, view_definition from timescaledb_information.continuous_aggregates where view_name::text like '%test_continuous_agg_view'; -[ RECORD 1 ]---+------------------------------------------------------------------------- view_name | test_continuous_agg_view view_definition | SELECT time_bucket(2, test_continuous_agg_table."time") AS time_bucket,+ | sum(test_continuous_agg_table.data) AS value + | FROM test_continuous_agg_table + | GROUP BY (time_bucket(2, test_continuous_agg_table."time")); select job_status, last_run_duration from timescaledb_information.job_stats ps, timescaledb_information.continuous_aggregates cagg where cagg.view_name::text like '%test_continuous_agg_view' and cagg.materialization_hypertable_name = ps.hypertable_name; -[ RECORD 1 ]-----+---------- job_status | Scheduled last_run_duration | \x off -- test merged refresh (change data in two chunks) UPDATE test_continuous_agg_table SET data = 11; --advance time by 1day so that job runs one more time SELECT ts_bgw_params_reset_time(extract(epoch from interval '1day')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(50, 50); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-------------+--------------------------------------------+--------------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 10, 12 ] (batch 1 of 2) 1 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 3 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -10, 10 ] (batch 2 of 2) 4 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 5 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 0 | 43200000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 43200000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -90, -10 ] 1 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | inserted 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 3 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 6, 8 ] 4 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 5 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 0 | 86400000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 86400000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 86400000000 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 0, 12 ] 1 | 86400000000 | Refresh Continuous Aggregate Policy [1001] | deleted 6 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 86400000000 | Refresh Continuous Aggregate Policy [1001] | inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" DROP MATERIALIZED VIEW test_continuous_agg_view; NOTICE: drop cascades to table _timescaledb_internal._hyper_3_4_chunk --create a view with a function that it has no permission to execute CREATE MATERIALIZED VIEW test_continuous_agg_view WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time), SUM(data) as value, get_constant_no_perms() FROM test_continuous_agg_table GROUP BY 1 WITH NO DATA; SELECT add_continuous_aggregate_policy('test_continuous_agg_view', 100::integer, -2::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1002 SELECT id AS job_id FROM _timescaledb_catalog.bgw_job ORDER BY id desc limit 1 \gset SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ -- job fails SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1002 | f | 1 | 0 | 1 | 0 DROP MATERIALIZED VIEW test_continuous_agg_view; --advance clock to quit scheduler SELECT ts_bgw_params_reset_time(extract(epoch from interval '25 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- select ts_bgw_db_scheduler_test_wait_for_scheduler_finish(); ts_bgw_db_scheduler_test_wait_for_scheduler_finish ---------------------------------------------------- SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_ON_JOB); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- --clear log for next run of the scheduler TRUNCATE public.bgw_log; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- -- -- Test creating continuous aggregate with a user that is the non-owner of the raw table -- CREATE TABLE test_continuous_agg_table_w_grant(time int, data int); SELECT create_hypertable('test_continuous_agg_table_w_grant', 'time', chunk_time_interval => 10); create_hypertable ------------------------------------------------ (5,public,test_continuous_agg_table_w_grant,t) CREATE OR REPLACE FUNCTION integer_now_test1() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), 0) FROM test_continuous_agg_table_w_grant $$; SELECT set_integer_now_func('test_continuous_agg_table_w_grant', 'integer_now_test1'); set_integer_now_func ---------------------- GRANT SELECT, TRIGGER ON test_continuous_agg_table_w_grant TO public; INSERT INTO test_continuous_agg_table_w_grant SELECT 1 , 1; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 -- make sure view can be created CREATE MATERIALIZED VIEW test_continuous_agg_view_user_2 WITH ( timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time), SUM(data) as value FROM test_continuous_agg_table_w_grant GROUP BY 1 WITH NO DATA; SELECT add_continuous_aggregate_policy('test_continuous_agg_view_user_2', NULL, -2::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1003 SELECT id AS job_id FROM _timescaledb_catalog.bgw_job ORDER BY id desc limit 1 \gset SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT id, owner FROM _timescaledb_catalog.bgw_job WHERE id = :job_id ; id | owner ------+--------------------- 1003 | default_perm_user_2 SELECT job_id, next_start - last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------+------------------+------------+-----------------+----------------+--------------- 1003 | @ 12 hours | t | 1 | 1 | 0 | 0 --view is populated SELECT * FROM test_continuous_agg_view_user_2 ORDER BY 1; time_bucket | value -------------+------- 0 | 1 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --revoke permissions from the continuous agg view owner to select from raw table --no further updates to cont agg should happen REVOKE SELECT ON test_continuous_agg_table_w_grant FROM public; --add new data to table INSERT INTO test_continuous_agg_table_w_grant VALUES(5,1); \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 --advance time by 12h so that job tries to run one more time SELECT ts_bgw_params_reset_time(extract(epoch from interval '12 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25, 25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ --should show a failing execution because no longer has permissions (due to lack of permission on partial view owner's part) SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1003 | f | 2 | 1 | 1 | 0 --view was NOT updated; but the old stuff is still there SELECT * FROM test_continuous_agg_view_user_2; time_bucket | value -------------+------- 0 | 1 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT * from sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-------------+--------------------------------------------+-------------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1003] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view_user_2" in window [ -2147483648, 2 ] 1 | 0 | Refresh Continuous Aggregate Policy [1003] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" 2 | 0 | Refresh Continuous Aggregate Policy [1003] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" 0 | 43200000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 43200000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 43200000000 | Refresh Continuous Aggregate Policy [1003] | job 1003 threw an error 1 | 43200000000 | Refresh Continuous Aggregate Policy [1003] | permission denied for table test_continuous_agg_table_w_grant -- Count the number of continuous aggregate policies SELECT count(*) FROM _timescaledb_catalog.bgw_job WHERE proc_schema = '_timescaledb_functions' AND proc_name = 'policy_refresh_continuous_aggregate'; count ------- 1 ================================================ FILE: tsl/test/expected/cagg_bgw-16.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- -- Setup -- \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(timeout INT = -1, mock_start_time INT = 0) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_run(timeout INT = -1, mock_start_time INT = 0) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_wait_for_scheduler_finish() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_create() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_destroy() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_reset_time(set_time BIGINT = 0, wait BOOLEAN = false) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; --test that this all works under the community license ALTER DATABASE :TEST_DBNAME SET timescaledb.license_key='Community'; --create a function with no permissions to execute CREATE FUNCTION get_constant_no_perms() RETURNS INTEGER LANGUAGE SQL IMMUTABLE AS $BODY$ SELECT 10; $BODY$; REVOKE EXECUTE ON FUNCTION get_constant_no_perms() FROM PUBLIC; \set WAIT_ON_JOB 0 \set IMMEDIATELY_SET_UNTIL 1 \set WAIT_FOR_OTHER_TO_ADVANCE 2 CREATE OR REPLACE FUNCTION ts_bgw_params_mock_wait_returns_immediately(new_val INTEGER) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; -- Remove any default jobs, e.g., telemetry DELETE FROM _timescaledb_catalog.bgw_job WHERE TRUE; TRUNCATE _timescaledb_internal.bgw_job_stat; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE public.bgw_log( msg_no INT, mock_time BIGINT, application_name TEXT, msg TEXT ); CREATE VIEW sorted_bgw_log AS SELECT msg_no, mock_time, application_name, regexp_replace(regexp_replace(msg, '(Wait until|started at|execution time) [0-9]+(\.[0-9]+)?', '\1 (RANDOM)', 'g'), 'background worker "[^"]+"','connection') AS msg FROM bgw_log ORDER BY mock_time, application_name COLLATE "C", msg_no; CREATE TABLE public.bgw_dsm_handle_store( handle BIGINT ); INSERT INTO public.bgw_dsm_handle_store VALUES (0); SELECT ts_bgw_params_create(); ts_bgw_params_create ---------------------- SELECT * FROM _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- SELECT * FROM timescaledb_information.job_stats; hypertable_schema | hypertable_name | job_id | last_run_started_at | last_successful_finish | last_run_status | job_status | last_run_duration | next_start | total_runs | total_successes | total_failures -------------------+-----------------+--------+---------------------+------------------------+-----------------+------------+-------------------+------------+------------+-----------------+---------------- SELECT * FROM _timescaledb_catalog.continuous_agg; mat_hypertable_id | raw_hypertable_id | parent_mat_hypertable_id | user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name | materialized_only -------------------+-------------------+--------------------------+------------------+----------------+---------------------+-------------------+--------------------+------------------+------------------- -- though user on access node has required GRANTS, this will propagate GRANTS to the connected data nodes GRANT CREATE ON SCHEMA public TO :ROLE_DEFAULT_PERM_USER; WARNING: no privileges were granted for "public" \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE test_continuous_agg_table(time int, data int); SELECT create_hypertable('test_continuous_agg_table', 'time', chunk_time_interval => 10); create_hypertable ---------------------------------------- (1,public,test_continuous_agg_table,t) CREATE OR REPLACE FUNCTION integer_now_test() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), 0) FROM test_continuous_agg_table $$; SELECT set_integer_now_func('test_continuous_agg_table', 'integer_now_test'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW test_continuous_agg_view WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time), SUM(data) as value FROM test_continuous_agg_table GROUP BY 1 WITH NO DATA; SELECT add_continuous_aggregate_policy('test_continuous_agg_view', NULL, 4::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1000 SELECT id as raw_table_id FROM _timescaledb_catalog.hypertable WHERE table_name='test_continuous_agg_table' \gset -- min distance from end should be 1 SELECT mat_hypertable_id, user_view_schema, user_view_name FROM _timescaledb_catalog.continuous_agg; mat_hypertable_id | user_view_schema | user_view_name -------------------+------------------+-------------------------- 2 | public | test_continuous_agg_view SELECT mat_hypertable_id, bucket_width FROM _timescaledb_catalog.continuous_aggs_bucket_function; mat_hypertable_id | bucket_width -------------------+-------------- 2 | 2 SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg \gset SELECT id AS job_id FROM _timescaledb_catalog.bgw_job where hypertable_id=:mat_hypertable_id \gset -- job was created SELECT * FROM _timescaledb_catalog.bgw_job where hypertable_id=:mat_hypertable_id; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ------+--------------------------------------------+-------------------+-------------+-------------+--------------+------------------------+-------------------------------------+-------------------+-----------+----------------+---------------+---------------+-----------------------------------------------------------------+------------------------+-------------------------------------------+---------- 1000 | Refresh Continuous Aggregate Policy [1000] | @ 12 hours | @ 0 | -1 | @ 12 hours | _timescaledb_functions | policy_refresh_continuous_aggregate | default_perm_user | t | f | | 2 | {"end_offset": 4, "start_offset": null, "mat_hypertable_id": 2} | _timescaledb_functions | policy_refresh_continuous_aggregate_check | -- create 10 time buckets INSERT INTO test_continuous_agg_table SELECT i, i FROM (SELECT generate_series(0, 10) as i) AS j; -- no stats SELECT job_id, next_start, last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat ORDER BY job_id; job_id | next_start | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------+------------+------------------+------------+-----------------+----------------+--------------- -- no data in view SELECT * FROM test_continuous_agg_view ORDER BY 1; time_bucket | value -------------+------- -- run first time SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+--------------------------------------------+------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1000] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -2147483648, 6 ] 1 | 0 | Refresh Continuous Aggregate Policy [1000] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 2 | 0 | Refresh Continuous Aggregate Policy [1000] | inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" SELECT * FROM _timescaledb_catalog.bgw_job where id=:job_id; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ------+--------------------------------------------+-------------------+-------------+-------------+--------------+------------------------+-------------------------------------+-------------------+-----------+----------------+---------------+---------------+-----------------------------------------------------------------+------------------------+-------------------------------------------+---------- 1000 | Refresh Continuous Aggregate Policy [1000] | @ 12 hours | @ 0 | -1 | @ 12 hours | _timescaledb_functions | policy_refresh_continuous_aggregate | default_perm_user | t | f | | 2 | {"end_offset": 4, "start_offset": null, "mat_hypertable_id": 2} | _timescaledb_functions | policy_refresh_continuous_aggregate_check | -- job ran once, successfully SELECT job_id, next_start-last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------+------------------+------------+-----------------+----------------+--------------- 1000 | @ 12 hours | t | 1 | 1 | 0 | 0 --clear log for next run of scheduler. TRUNCATE public.bgw_log; CREATE FUNCTION wait_for_timer_to_run(started_at INTEGER, spins INTEGER=:TEST_SPINWAIT_ITERS) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE num_runs INTEGER; message TEXT; BEGIN select format('[TESTING] Wait until %%, started at %s', started_at) into message; FOR i in 1..spins LOOP SELECT COUNT(*) from bgw_log where msg LIKE message INTO num_runs; if (num_runs > 0) THEN RETURN true; ELSE PERFORM pg_sleep(0.1); END IF; END LOOP; RETURN false; END $BODY$; --make sure there is 1 job to start with SELECT test.wait_for_job_to_run(:job_id, 1); wait_for_job_to_run --------------------- t SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_FOR_OTHER_TO_ADVANCE); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- --start the scheduler on 0 time SELECT ts_bgw_params_reset_time(0, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run(extract(epoch from interval '24 hour')::int * 1000, 0); ts_bgw_db_scheduler_test_run ------------------------------ SELECT wait_for_timer_to_run(0); wait_for_timer_to_run ----------------------- t --advance to 12:00 so that it runs one more time; now we know the --scheduler has loaded up the job with the old schedule_interval SELECT ts_bgw_params_reset_time(extract(epoch from interval '12 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- SELECT test.wait_for_job_to_run(:job_id, 2); wait_for_job_to_run --------------------- t --advance clock 1us to make the scheduler realize the job is done SELECT ts_bgw_params_reset_time((extract(epoch from interval '12 hour')::bigint * 1000000)+1, true); ts_bgw_params_reset_time -------------------------- --alter the refresh interval and check if next_start is altered SELECT alter_job(:job_id, schedule_interval => '1m', retry_period => '1m'); alter_job ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1000,"@ 1 min","@ 0",-1,"@ 1 min",t,"{""end_offset"": 4, ""start_offset"": null, ""mat_hypertable_id"": 2}","Sat Jan 01 04:01:00 2000 PST",_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1000]") SELECT job_id, next_start - last_finish as until_next, total_runs FROM _timescaledb_internal.bgw_job_stat WHERE job_id=:job_id;; job_id | until_next | total_runs --------+------------+------------ 1000 | @ 1 min | 2 --advance to 12:02, job should have run at 12:01 SELECT ts_bgw_params_reset_time((extract(epoch from interval '12 hour')::bigint * 1000000)+(extract(epoch from interval '2 minute')::bigint * 1000000), true); ts_bgw_params_reset_time -------------------------- SELECT test.wait_for_job_to_run(:job_id, 3); wait_for_job_to_run --------------------- t --next run in 1 minute SELECT job_id, next_start-last_finish as until_next, total_runs FROM _timescaledb_internal.bgw_job_stat WHERE job_id=:job_id; job_id | until_next | total_runs --------+------------+------------ 1000 | @ 1 min | 3 --change next run to be after 30s instead SELECT (next_start - '30s'::interval) AS "NEW_NEXT_START" FROM _timescaledb_internal.bgw_job_stat WHERE job_id=:job_id \gset SELECT alter_job(:job_id, next_start => :'NEW_NEXT_START'); alter_job ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1000,"@ 1 min","@ 0",-1,"@ 1 min",t,"{""end_offset"": 4, ""start_offset"": null, ""mat_hypertable_id"": 2}","Sat Jan 01 04:02:30 2000 PST",_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1000]") SELECT ts_bgw_params_reset_time((extract(epoch from interval '12 hour')::bigint * 1000000)+(extract(epoch from interval '2 minute 30 seconds')::bigint * 1000000), true); ts_bgw_params_reset_time -------------------------- SELECT test.wait_for_job_to_run(:job_id, 4); wait_for_job_to_run --------------------- t --advance clock to quit scheduler SELECT ts_bgw_params_reset_time(extract(epoch from interval '25 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- select ts_bgw_db_scheduler_test_wait_for_scheduler_finish(); ts_bgw_db_scheduler_test_wait_for_scheduler_finish ---------------------------------------------------- SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_ON_JOB); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- TRUNCATE public.bgw_log; -- data before 8 SELECT * FROM test_continuous_agg_view ORDER BY 1; time_bucket | value -------------+------- 0 | 1 2 | 5 4 | 9 -- invalidations test by running job multiple times SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- DROP MATERIALIZED VIEW test_continuous_agg_view; NOTICE: drop cascades to table _timescaledb_internal._hyper_2_3_chunk CREATE MATERIALIZED VIEW test_continuous_agg_view WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time), SUM(data) as value FROM test_continuous_agg_table GROUP BY 1 WITH NO DATA; SELECT add_continuous_aggregate_policy('test_continuous_agg_view', 100::integer, -2::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1001 SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg \gset SELECT id AS job_id FROM _timescaledb_catalog.bgw_job WHERE hypertable_id=:mat_hypertable_id \gset SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+--------------------------------------------+--------------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 10, 12 ] (batch 1 of 2) 1 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 3 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -10, 10 ] (batch 2 of 2) 4 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 5 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" -- job ran once, successfully SELECT job_id, last_finish - next_start as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+----------------+------------------+------------+-----------------+----------------+--------------- 1001 | @ 12 hours ago | t | 1 | 1 | 0 | 0 -- should have refreshed everything we have so far SELECT * FROM test_continuous_agg_view ORDER BY 1; time_bucket | value -------------+------- 0 | 1 2 | 5 4 | 9 6 | 13 8 | 17 10 | 10 -- invalidate some data UPDATE test_continuous_agg_table SET data = 11 WHERE time = 6; --advance time by 12h so that job runs one more time SELECT ts_bgw_params_reset_time(extract(epoch from interval '12 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25, 25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-------------+--------------------------------------------+--------------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 10, 12 ] (batch 1 of 2) 1 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 3 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -10, 10 ] (batch 2 of 2) 4 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 5 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 0 | 43200000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 43200000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -90, -10 ] 1 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | inserted 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 3 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 6, 8 ] 4 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 5 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" SELECT job_id, next_start - last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------+------------------+------------+-----------------+----------------+--------------- 1001 | @ 12 hours | t | 2 | 2 | 0 | 0 -- should have updated data for time=6 SELECT * FROM test_continuous_agg_view ORDER BY 1; time_bucket | value -------------+------- 0 | 1 2 | 5 4 | 9 6 | 18 8 | 17 10 | 10 \x on --check the information views -- select view_name, view_owner, materialization_hypertable_schema, materialization_hypertable_name from timescaledb_information.continuous_aggregates where view_name::text like '%test_continuous_agg_view'; -[ RECORD 1 ]---------------------+--------------------------- view_name | test_continuous_agg_view view_owner | default_perm_user materialization_hypertable_schema | _timescaledb_internal materialization_hypertable_name | _materialized_hypertable_3 select view_name, view_definition from timescaledb_information.continuous_aggregates where view_name::text like '%test_continuous_agg_view'; -[ RECORD 1 ]---+----------------------------------------------- view_name | test_continuous_agg_view view_definition | SELECT time_bucket(2, "time") AS time_bucket,+ | sum(data) AS value + | FROM test_continuous_agg_table + | GROUP BY (time_bucket(2, "time")); select job_status, last_run_duration from timescaledb_information.job_stats ps, timescaledb_information.continuous_aggregates cagg where cagg.view_name::text like '%test_continuous_agg_view' and cagg.materialization_hypertable_name = ps.hypertable_name; -[ RECORD 1 ]-----+---------- job_status | Scheduled last_run_duration | \x off -- test merged refresh (change data in two chunks) UPDATE test_continuous_agg_table SET data = 11; --advance time by 1day so that job runs one more time SELECT ts_bgw_params_reset_time(extract(epoch from interval '1day')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(50, 50); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-------------+--------------------------------------------+--------------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 10, 12 ] (batch 1 of 2) 1 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 3 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -10, 10 ] (batch 2 of 2) 4 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 5 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 0 | 43200000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 43200000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -90, -10 ] 1 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | inserted 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 3 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 6, 8 ] 4 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 5 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 0 | 86400000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 86400000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 86400000000 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 0, 12 ] 1 | 86400000000 | Refresh Continuous Aggregate Policy [1001] | deleted 6 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 86400000000 | Refresh Continuous Aggregate Policy [1001] | inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" DROP MATERIALIZED VIEW test_continuous_agg_view; NOTICE: drop cascades to table _timescaledb_internal._hyper_3_4_chunk --create a view with a function that it has no permission to execute CREATE MATERIALIZED VIEW test_continuous_agg_view WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time), SUM(data) as value, get_constant_no_perms() FROM test_continuous_agg_table GROUP BY 1 WITH NO DATA; SELECT add_continuous_aggregate_policy('test_continuous_agg_view', 100::integer, -2::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1002 SELECT id AS job_id FROM _timescaledb_catalog.bgw_job ORDER BY id desc limit 1 \gset SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ -- job fails SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1002 | f | 1 | 0 | 1 | 0 DROP MATERIALIZED VIEW test_continuous_agg_view; --advance clock to quit scheduler SELECT ts_bgw_params_reset_time(extract(epoch from interval '25 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- select ts_bgw_db_scheduler_test_wait_for_scheduler_finish(); ts_bgw_db_scheduler_test_wait_for_scheduler_finish ---------------------------------------------------- SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_ON_JOB); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- --clear log for next run of the scheduler TRUNCATE public.bgw_log; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- -- -- Test creating continuous aggregate with a user that is the non-owner of the raw table -- CREATE TABLE test_continuous_agg_table_w_grant(time int, data int); SELECT create_hypertable('test_continuous_agg_table_w_grant', 'time', chunk_time_interval => 10); create_hypertable ------------------------------------------------ (5,public,test_continuous_agg_table_w_grant,t) CREATE OR REPLACE FUNCTION integer_now_test1() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), 0) FROM test_continuous_agg_table_w_grant $$; SELECT set_integer_now_func('test_continuous_agg_table_w_grant', 'integer_now_test1'); set_integer_now_func ---------------------- GRANT SELECT, TRIGGER ON test_continuous_agg_table_w_grant TO public; INSERT INTO test_continuous_agg_table_w_grant SELECT 1 , 1; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 -- make sure view can be created CREATE MATERIALIZED VIEW test_continuous_agg_view_user_2 WITH ( timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time), SUM(data) as value FROM test_continuous_agg_table_w_grant GROUP BY 1 WITH NO DATA; SELECT add_continuous_aggregate_policy('test_continuous_agg_view_user_2', NULL, -2::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1003 SELECT id AS job_id FROM _timescaledb_catalog.bgw_job ORDER BY id desc limit 1 \gset SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT id, owner FROM _timescaledb_catalog.bgw_job WHERE id = :job_id ; id | owner ------+--------------------- 1003 | default_perm_user_2 SELECT job_id, next_start - last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------+------------------+------------+-----------------+----------------+--------------- 1003 | @ 12 hours | t | 1 | 1 | 0 | 0 --view is populated SELECT * FROM test_continuous_agg_view_user_2 ORDER BY 1; time_bucket | value -------------+------- 0 | 1 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --revoke permissions from the continuous agg view owner to select from raw table --no further updates to cont agg should happen REVOKE SELECT ON test_continuous_agg_table_w_grant FROM public; --add new data to table INSERT INTO test_continuous_agg_table_w_grant VALUES(5,1); \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 --advance time by 12h so that job tries to run one more time SELECT ts_bgw_params_reset_time(extract(epoch from interval '12 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25, 25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ --should show a failing execution because no longer has permissions (due to lack of permission on partial view owner's part) SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1003 | f | 2 | 1 | 1 | 0 --view was NOT updated; but the old stuff is still there SELECT * FROM test_continuous_agg_view_user_2; time_bucket | value -------------+------- 0 | 1 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT * from sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-------------+--------------------------------------------+-------------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1003] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view_user_2" in window [ -2147483648, 2 ] 1 | 0 | Refresh Continuous Aggregate Policy [1003] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" 2 | 0 | Refresh Continuous Aggregate Policy [1003] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" 0 | 43200000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 43200000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 43200000000 | Refresh Continuous Aggregate Policy [1003] | job 1003 threw an error 1 | 43200000000 | Refresh Continuous Aggregate Policy [1003] | permission denied for table test_continuous_agg_table_w_grant -- Count the number of continuous aggregate policies SELECT count(*) FROM _timescaledb_catalog.bgw_job WHERE proc_schema = '_timescaledb_functions' AND proc_name = 'policy_refresh_continuous_aggregate'; count ------- 1 ================================================ FILE: tsl/test/expected/cagg_bgw-17.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- -- Setup -- \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(timeout INT = -1, mock_start_time INT = 0) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_run(timeout INT = -1, mock_start_time INT = 0) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_wait_for_scheduler_finish() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_create() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_destroy() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_reset_time(set_time BIGINT = 0, wait BOOLEAN = false) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; --test that this all works under the community license ALTER DATABASE :TEST_DBNAME SET timescaledb.license_key='Community'; --create a function with no permissions to execute CREATE FUNCTION get_constant_no_perms() RETURNS INTEGER LANGUAGE SQL IMMUTABLE AS $BODY$ SELECT 10; $BODY$; REVOKE EXECUTE ON FUNCTION get_constant_no_perms() FROM PUBLIC; \set WAIT_ON_JOB 0 \set IMMEDIATELY_SET_UNTIL 1 \set WAIT_FOR_OTHER_TO_ADVANCE 2 CREATE OR REPLACE FUNCTION ts_bgw_params_mock_wait_returns_immediately(new_val INTEGER) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; -- Remove any default jobs, e.g., telemetry DELETE FROM _timescaledb_catalog.bgw_job WHERE TRUE; TRUNCATE _timescaledb_internal.bgw_job_stat; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE public.bgw_log( msg_no INT, mock_time BIGINT, application_name TEXT, msg TEXT ); CREATE VIEW sorted_bgw_log AS SELECT msg_no, mock_time, application_name, regexp_replace(regexp_replace(msg, '(Wait until|started at|execution time) [0-9]+(\.[0-9]+)?', '\1 (RANDOM)', 'g'), 'background worker "[^"]+"','connection') AS msg FROM bgw_log ORDER BY mock_time, application_name COLLATE "C", msg_no; CREATE TABLE public.bgw_dsm_handle_store( handle BIGINT ); INSERT INTO public.bgw_dsm_handle_store VALUES (0); SELECT ts_bgw_params_create(); ts_bgw_params_create ---------------------- SELECT * FROM _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- SELECT * FROM timescaledb_information.job_stats; hypertable_schema | hypertable_name | job_id | last_run_started_at | last_successful_finish | last_run_status | job_status | last_run_duration | next_start | total_runs | total_successes | total_failures -------------------+-----------------+--------+---------------------+------------------------+-----------------+------------+-------------------+------------+------------+-----------------+---------------- SELECT * FROM _timescaledb_catalog.continuous_agg; mat_hypertable_id | raw_hypertable_id | parent_mat_hypertable_id | user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name | materialized_only -------------------+-------------------+--------------------------+------------------+----------------+---------------------+-------------------+--------------------+------------------+------------------- -- though user on access node has required GRANTS, this will propagate GRANTS to the connected data nodes GRANT CREATE ON SCHEMA public TO :ROLE_DEFAULT_PERM_USER; WARNING: no privileges were granted for "public" \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE test_continuous_agg_table(time int, data int); SELECT create_hypertable('test_continuous_agg_table', 'time', chunk_time_interval => 10); create_hypertable ---------------------------------------- (1,public,test_continuous_agg_table,t) CREATE OR REPLACE FUNCTION integer_now_test() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), 0) FROM test_continuous_agg_table $$; SELECT set_integer_now_func('test_continuous_agg_table', 'integer_now_test'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW test_continuous_agg_view WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time), SUM(data) as value FROM test_continuous_agg_table GROUP BY 1 WITH NO DATA; SELECT add_continuous_aggregate_policy('test_continuous_agg_view', NULL, 4::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1000 SELECT id as raw_table_id FROM _timescaledb_catalog.hypertable WHERE table_name='test_continuous_agg_table' \gset -- min distance from end should be 1 SELECT mat_hypertable_id, user_view_schema, user_view_name FROM _timescaledb_catalog.continuous_agg; mat_hypertable_id | user_view_schema | user_view_name -------------------+------------------+-------------------------- 2 | public | test_continuous_agg_view SELECT mat_hypertable_id, bucket_width FROM _timescaledb_catalog.continuous_aggs_bucket_function; mat_hypertable_id | bucket_width -------------------+-------------- 2 | 2 SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg \gset SELECT id AS job_id FROM _timescaledb_catalog.bgw_job where hypertable_id=:mat_hypertable_id \gset -- job was created SELECT * FROM _timescaledb_catalog.bgw_job where hypertable_id=:mat_hypertable_id; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ------+--------------------------------------------+-------------------+-------------+-------------+--------------+------------------------+-------------------------------------+-------------------+-----------+----------------+---------------+---------------+-----------------------------------------------------------------+------------------------+-------------------------------------------+---------- 1000 | Refresh Continuous Aggregate Policy [1000] | @ 12 hours | @ 0 | -1 | @ 12 hours | _timescaledb_functions | policy_refresh_continuous_aggregate | default_perm_user | t | f | | 2 | {"end_offset": 4, "start_offset": null, "mat_hypertable_id": 2} | _timescaledb_functions | policy_refresh_continuous_aggregate_check | -- create 10 time buckets INSERT INTO test_continuous_agg_table SELECT i, i FROM (SELECT generate_series(0, 10) as i) AS j; -- no stats SELECT job_id, next_start, last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat ORDER BY job_id; job_id | next_start | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------+------------+------------------+------------+-----------------+----------------+--------------- -- no data in view SELECT * FROM test_continuous_agg_view ORDER BY 1; time_bucket | value -------------+------- -- run first time SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+--------------------------------------------+------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1000] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -2147483648, 6 ] 1 | 0 | Refresh Continuous Aggregate Policy [1000] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 2 | 0 | Refresh Continuous Aggregate Policy [1000] | inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" SELECT * FROM _timescaledb_catalog.bgw_job where id=:job_id; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ------+--------------------------------------------+-------------------+-------------+-------------+--------------+------------------------+-------------------------------------+-------------------+-----------+----------------+---------------+---------------+-----------------------------------------------------------------+------------------------+-------------------------------------------+---------- 1000 | Refresh Continuous Aggregate Policy [1000] | @ 12 hours | @ 0 | -1 | @ 12 hours | _timescaledb_functions | policy_refresh_continuous_aggregate | default_perm_user | t | f | | 2 | {"end_offset": 4, "start_offset": null, "mat_hypertable_id": 2} | _timescaledb_functions | policy_refresh_continuous_aggregate_check | -- job ran once, successfully SELECT job_id, next_start-last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------+------------------+------------+-----------------+----------------+--------------- 1000 | @ 12 hours | t | 1 | 1 | 0 | 0 --clear log for next run of scheduler. TRUNCATE public.bgw_log; CREATE FUNCTION wait_for_timer_to_run(started_at INTEGER, spins INTEGER=:TEST_SPINWAIT_ITERS) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE num_runs INTEGER; message TEXT; BEGIN select format('[TESTING] Wait until %%, started at %s', started_at) into message; FOR i in 1..spins LOOP SELECT COUNT(*) from bgw_log where msg LIKE message INTO num_runs; if (num_runs > 0) THEN RETURN true; ELSE PERFORM pg_sleep(0.1); END IF; END LOOP; RETURN false; END $BODY$; --make sure there is 1 job to start with SELECT test.wait_for_job_to_run(:job_id, 1); wait_for_job_to_run --------------------- t SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_FOR_OTHER_TO_ADVANCE); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- --start the scheduler on 0 time SELECT ts_bgw_params_reset_time(0, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run(extract(epoch from interval '24 hour')::int * 1000, 0); ts_bgw_db_scheduler_test_run ------------------------------ SELECT wait_for_timer_to_run(0); wait_for_timer_to_run ----------------------- t --advance to 12:00 so that it runs one more time; now we know the --scheduler has loaded up the job with the old schedule_interval SELECT ts_bgw_params_reset_time(extract(epoch from interval '12 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- SELECT test.wait_for_job_to_run(:job_id, 2); wait_for_job_to_run --------------------- t --advance clock 1us to make the scheduler realize the job is done SELECT ts_bgw_params_reset_time((extract(epoch from interval '12 hour')::bigint * 1000000)+1, true); ts_bgw_params_reset_time -------------------------- --alter the refresh interval and check if next_start is altered SELECT alter_job(:job_id, schedule_interval => '1m', retry_period => '1m'); alter_job ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1000,"@ 1 min","@ 0",-1,"@ 1 min",t,"{""end_offset"": 4, ""start_offset"": null, ""mat_hypertable_id"": 2}","Sat Jan 01 04:01:00 2000 PST",_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1000]") SELECT job_id, next_start - last_finish as until_next, total_runs FROM _timescaledb_internal.bgw_job_stat WHERE job_id=:job_id;; job_id | until_next | total_runs --------+------------+------------ 1000 | @ 1 min | 2 --advance to 12:02, job should have run at 12:01 SELECT ts_bgw_params_reset_time((extract(epoch from interval '12 hour')::bigint * 1000000)+(extract(epoch from interval '2 minute')::bigint * 1000000), true); ts_bgw_params_reset_time -------------------------- SELECT test.wait_for_job_to_run(:job_id, 3); wait_for_job_to_run --------------------- t --next run in 1 minute SELECT job_id, next_start-last_finish as until_next, total_runs FROM _timescaledb_internal.bgw_job_stat WHERE job_id=:job_id; job_id | until_next | total_runs --------+------------+------------ 1000 | @ 1 min | 3 --change next run to be after 30s instead SELECT (next_start - '30s'::interval) AS "NEW_NEXT_START" FROM _timescaledb_internal.bgw_job_stat WHERE job_id=:job_id \gset SELECT alter_job(:job_id, next_start => :'NEW_NEXT_START'); alter_job ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1000,"@ 1 min","@ 0",-1,"@ 1 min",t,"{""end_offset"": 4, ""start_offset"": null, ""mat_hypertable_id"": 2}","Sat Jan 01 04:02:30 2000 PST",_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1000]") SELECT ts_bgw_params_reset_time((extract(epoch from interval '12 hour')::bigint * 1000000)+(extract(epoch from interval '2 minute 30 seconds')::bigint * 1000000), true); ts_bgw_params_reset_time -------------------------- SELECT test.wait_for_job_to_run(:job_id, 4); wait_for_job_to_run --------------------- t --advance clock to quit scheduler SELECT ts_bgw_params_reset_time(extract(epoch from interval '25 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- select ts_bgw_db_scheduler_test_wait_for_scheduler_finish(); ts_bgw_db_scheduler_test_wait_for_scheduler_finish ---------------------------------------------------- SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_ON_JOB); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- TRUNCATE public.bgw_log; -- data before 8 SELECT * FROM test_continuous_agg_view ORDER BY 1; time_bucket | value -------------+------- 0 | 1 2 | 5 4 | 9 -- invalidations test by running job multiple times SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- DROP MATERIALIZED VIEW test_continuous_agg_view; NOTICE: drop cascades to table _timescaledb_internal._hyper_2_3_chunk CREATE MATERIALIZED VIEW test_continuous_agg_view WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time), SUM(data) as value FROM test_continuous_agg_table GROUP BY 1 WITH NO DATA; SELECT add_continuous_aggregate_policy('test_continuous_agg_view', 100::integer, -2::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1001 SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg \gset SELECT id AS job_id FROM _timescaledb_catalog.bgw_job WHERE hypertable_id=:mat_hypertable_id \gset SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+--------------------------------------------+--------------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 10, 12 ] (batch 1 of 2) 1 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 3 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -10, 10 ] (batch 2 of 2) 4 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 5 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" -- job ran once, successfully SELECT job_id, last_finish - next_start as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+----------------+------------------+------------+-----------------+----------------+--------------- 1001 | @ 12 hours ago | t | 1 | 1 | 0 | 0 -- should have refreshed everything we have so far SELECT * FROM test_continuous_agg_view ORDER BY 1; time_bucket | value -------------+------- 0 | 1 2 | 5 4 | 9 6 | 13 8 | 17 10 | 10 -- invalidate some data UPDATE test_continuous_agg_table SET data = 11 WHERE time = 6; --advance time by 12h so that job runs one more time SELECT ts_bgw_params_reset_time(extract(epoch from interval '12 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25, 25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-------------+--------------------------------------------+--------------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 10, 12 ] (batch 1 of 2) 1 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 3 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -10, 10 ] (batch 2 of 2) 4 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 5 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 0 | 43200000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 43200000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -90, -10 ] 1 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | inserted 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 3 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 6, 8 ] 4 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 5 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" SELECT job_id, next_start - last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------+------------------+------------+-----------------+----------------+--------------- 1001 | @ 12 hours | t | 2 | 2 | 0 | 0 -- should have updated data for time=6 SELECT * FROM test_continuous_agg_view ORDER BY 1; time_bucket | value -------------+------- 0 | 1 2 | 5 4 | 9 6 | 18 8 | 17 10 | 10 \x on --check the information views -- select view_name, view_owner, materialization_hypertable_schema, materialization_hypertable_name from timescaledb_information.continuous_aggregates where view_name::text like '%test_continuous_agg_view'; -[ RECORD 1 ]---------------------+--------------------------- view_name | test_continuous_agg_view view_owner | default_perm_user materialization_hypertable_schema | _timescaledb_internal materialization_hypertable_name | _materialized_hypertable_3 select view_name, view_definition from timescaledb_information.continuous_aggregates where view_name::text like '%test_continuous_agg_view'; -[ RECORD 1 ]---+----------------------------------------------- view_name | test_continuous_agg_view view_definition | SELECT time_bucket(2, "time") AS time_bucket,+ | sum(data) AS value + | FROM test_continuous_agg_table + | GROUP BY (time_bucket(2, "time")); select job_status, last_run_duration from timescaledb_information.job_stats ps, timescaledb_information.continuous_aggregates cagg where cagg.view_name::text like '%test_continuous_agg_view' and cagg.materialization_hypertable_name = ps.hypertable_name; -[ RECORD 1 ]-----+---------- job_status | Scheduled last_run_duration | \x off -- test merged refresh (change data in two chunks) UPDATE test_continuous_agg_table SET data = 11; --advance time by 1day so that job runs one more time SELECT ts_bgw_params_reset_time(extract(epoch from interval '1day')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(50, 50); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-------------+--------------------------------------------+--------------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 10, 12 ] (batch 1 of 2) 1 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 3 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -10, 10 ] (batch 2 of 2) 4 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 5 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 0 | 43200000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 43200000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -90, -10 ] 1 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | inserted 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 3 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 6, 8 ] 4 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 5 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 0 | 86400000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 86400000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 86400000000 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 0, 12 ] 1 | 86400000000 | Refresh Continuous Aggregate Policy [1001] | deleted 6 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 86400000000 | Refresh Continuous Aggregate Policy [1001] | inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" DROP MATERIALIZED VIEW test_continuous_agg_view; NOTICE: drop cascades to table _timescaledb_internal._hyper_3_4_chunk --create a view with a function that it has no permission to execute CREATE MATERIALIZED VIEW test_continuous_agg_view WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time), SUM(data) as value, get_constant_no_perms() FROM test_continuous_agg_table GROUP BY 1 WITH NO DATA; SELECT add_continuous_aggregate_policy('test_continuous_agg_view', 100::integer, -2::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1002 SELECT id AS job_id FROM _timescaledb_catalog.bgw_job ORDER BY id desc limit 1 \gset SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ -- job fails SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1002 | f | 1 | 0 | 1 | 0 DROP MATERIALIZED VIEW test_continuous_agg_view; --advance clock to quit scheduler SELECT ts_bgw_params_reset_time(extract(epoch from interval '25 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- select ts_bgw_db_scheduler_test_wait_for_scheduler_finish(); ts_bgw_db_scheduler_test_wait_for_scheduler_finish ---------------------------------------------------- SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_ON_JOB); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- --clear log for next run of the scheduler TRUNCATE public.bgw_log; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- -- -- Test creating continuous aggregate with a user that is the non-owner of the raw table -- CREATE TABLE test_continuous_agg_table_w_grant(time int, data int); SELECT create_hypertable('test_continuous_agg_table_w_grant', 'time', chunk_time_interval => 10); create_hypertable ------------------------------------------------ (5,public,test_continuous_agg_table_w_grant,t) CREATE OR REPLACE FUNCTION integer_now_test1() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), 0) FROM test_continuous_agg_table_w_grant $$; SELECT set_integer_now_func('test_continuous_agg_table_w_grant', 'integer_now_test1'); set_integer_now_func ---------------------- GRANT SELECT, TRIGGER ON test_continuous_agg_table_w_grant TO public; INSERT INTO test_continuous_agg_table_w_grant SELECT 1 , 1; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 -- make sure view can be created CREATE MATERIALIZED VIEW test_continuous_agg_view_user_2 WITH ( timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time), SUM(data) as value FROM test_continuous_agg_table_w_grant GROUP BY 1 WITH NO DATA; SELECT add_continuous_aggregate_policy('test_continuous_agg_view_user_2', NULL, -2::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1003 SELECT id AS job_id FROM _timescaledb_catalog.bgw_job ORDER BY id desc limit 1 \gset SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT id, owner FROM _timescaledb_catalog.bgw_job WHERE id = :job_id ; id | owner ------+--------------------- 1003 | default_perm_user_2 SELECT job_id, next_start - last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------+------------------+------------+-----------------+----------------+--------------- 1003 | @ 12 hours | t | 1 | 1 | 0 | 0 --view is populated SELECT * FROM test_continuous_agg_view_user_2 ORDER BY 1; time_bucket | value -------------+------- 0 | 1 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --revoke permissions from the continuous agg view owner to select from raw table --no further updates to cont agg should happen REVOKE SELECT ON test_continuous_agg_table_w_grant FROM public; --add new data to table INSERT INTO test_continuous_agg_table_w_grant VALUES(5,1); \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 --advance time by 12h so that job tries to run one more time SELECT ts_bgw_params_reset_time(extract(epoch from interval '12 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25, 25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ --should show a failing execution because no longer has permissions (due to lack of permission on partial view owner's part) SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1003 | f | 2 | 1 | 1 | 0 --view was NOT updated; but the old stuff is still there SELECT * FROM test_continuous_agg_view_user_2; time_bucket | value -------------+------- 0 | 1 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT * from sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-------------+--------------------------------------------+-------------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1003] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view_user_2" in window [ -2147483648, 2 ] 1 | 0 | Refresh Continuous Aggregate Policy [1003] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" 2 | 0 | Refresh Continuous Aggregate Policy [1003] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" 0 | 43200000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 43200000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 43200000000 | Refresh Continuous Aggregate Policy [1003] | job 1003 threw an error 1 | 43200000000 | Refresh Continuous Aggregate Policy [1003] | permission denied for table test_continuous_agg_table_w_grant -- Count the number of continuous aggregate policies SELECT count(*) FROM _timescaledb_catalog.bgw_job WHERE proc_schema = '_timescaledb_functions' AND proc_name = 'policy_refresh_continuous_aggregate'; count ------- 1 ================================================ FILE: tsl/test/expected/cagg_bgw-18.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- -- Setup -- \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(timeout INT = -1, mock_start_time INT = 0) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_run(timeout INT = -1, mock_start_time INT = 0) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_wait_for_scheduler_finish() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_create() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_destroy() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_reset_time(set_time BIGINT = 0, wait BOOLEAN = false) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; --test that this all works under the community license ALTER DATABASE :TEST_DBNAME SET timescaledb.license_key='Community'; --create a function with no permissions to execute CREATE FUNCTION get_constant_no_perms() RETURNS INTEGER LANGUAGE SQL IMMUTABLE AS $BODY$ SELECT 10; $BODY$; REVOKE EXECUTE ON FUNCTION get_constant_no_perms() FROM PUBLIC; \set WAIT_ON_JOB 0 \set IMMEDIATELY_SET_UNTIL 1 \set WAIT_FOR_OTHER_TO_ADVANCE 2 CREATE OR REPLACE FUNCTION ts_bgw_params_mock_wait_returns_immediately(new_val INTEGER) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; -- Remove any default jobs, e.g., telemetry DELETE FROM _timescaledb_catalog.bgw_job WHERE TRUE; TRUNCATE _timescaledb_internal.bgw_job_stat; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE public.bgw_log( msg_no INT, mock_time BIGINT, application_name TEXT, msg TEXT ); CREATE VIEW sorted_bgw_log AS SELECT msg_no, mock_time, application_name, regexp_replace(regexp_replace(msg, '(Wait until|started at|execution time) [0-9]+(\.[0-9]+)?', '\1 (RANDOM)', 'g'), 'background worker "[^"]+"','connection') AS msg FROM bgw_log ORDER BY mock_time, application_name COLLATE "C", msg_no; CREATE TABLE public.bgw_dsm_handle_store( handle BIGINT ); INSERT INTO public.bgw_dsm_handle_store VALUES (0); SELECT ts_bgw_params_create(); ts_bgw_params_create ---------------------- SELECT * FROM _timescaledb_catalog.bgw_job; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ----+------------------+-------------------+-------------+-------------+--------------+-------------+-----------+-------+-----------+----------------+---------------+---------------+--------+--------------+------------+---------- SELECT * FROM timescaledb_information.job_stats; hypertable_schema | hypertable_name | job_id | last_run_started_at | last_successful_finish | last_run_status | job_status | last_run_duration | next_start | total_runs | total_successes | total_failures -------------------+-----------------+--------+---------------------+------------------------+-----------------+------------+-------------------+------------+------------+-----------------+---------------- SELECT * FROM _timescaledb_catalog.continuous_agg; mat_hypertable_id | raw_hypertable_id | parent_mat_hypertable_id | user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name | materialized_only -------------------+-------------------+--------------------------+------------------+----------------+---------------------+-------------------+--------------------+------------------+------------------- -- though user on access node has required GRANTS, this will propagate GRANTS to the connected data nodes GRANT CREATE ON SCHEMA public TO :ROLE_DEFAULT_PERM_USER; WARNING: no privileges were granted for "public" \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE test_continuous_agg_table(time int, data int); SELECT create_hypertable('test_continuous_agg_table', 'time', chunk_time_interval => 10); create_hypertable ---------------------------------------- (1,public,test_continuous_agg_table,t) CREATE OR REPLACE FUNCTION integer_now_test() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), 0) FROM test_continuous_agg_table $$; SELECT set_integer_now_func('test_continuous_agg_table', 'integer_now_test'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW test_continuous_agg_view WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time), SUM(data) as value FROM test_continuous_agg_table GROUP BY 1 WITH NO DATA; SELECT add_continuous_aggregate_policy('test_continuous_agg_view', NULL, 4::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1000 SELECT id as raw_table_id FROM _timescaledb_catalog.hypertable WHERE table_name='test_continuous_agg_table' \gset -- min distance from end should be 1 SELECT mat_hypertable_id, user_view_schema, user_view_name FROM _timescaledb_catalog.continuous_agg; mat_hypertable_id | user_view_schema | user_view_name -------------------+------------------+-------------------------- 2 | public | test_continuous_agg_view SELECT mat_hypertable_id, bucket_width FROM _timescaledb_catalog.continuous_aggs_bucket_function; mat_hypertable_id | bucket_width -------------------+-------------- 2 | 2 SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg \gset SELECT id AS job_id FROM _timescaledb_catalog.bgw_job where hypertable_id=:mat_hypertable_id \gset -- job was created SELECT * FROM _timescaledb_catalog.bgw_job where hypertable_id=:mat_hypertable_id; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ------+--------------------------------------------+-------------------+-------------+-------------+--------------+------------------------+-------------------------------------+-------------------+-----------+----------------+---------------+---------------+-----------------------------------------------------------------+------------------------+-------------------------------------------+---------- 1000 | Refresh Continuous Aggregate Policy [1000] | @ 12 hours | @ 0 | -1 | @ 12 hours | _timescaledb_functions | policy_refresh_continuous_aggregate | default_perm_user | t | f | | 2 | {"end_offset": 4, "start_offset": null, "mat_hypertable_id": 2} | _timescaledb_functions | policy_refresh_continuous_aggregate_check | -- create 10 time buckets INSERT INTO test_continuous_agg_table SELECT i, i FROM (SELECT generate_series(0, 10) as i) AS j; -- no stats SELECT job_id, next_start, last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat ORDER BY job_id; job_id | next_start | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------+------------+------------------+------------+-----------------+----------------+--------------- -- no data in view SELECT * FROM test_continuous_agg_view ORDER BY 1; time_bucket | value -------------+------- -- run first time SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+--------------------------------------------+------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1000] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -2147483648, 6 ] 1 | 0 | Refresh Continuous Aggregate Policy [1000] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 2 | 0 | Refresh Continuous Aggregate Policy [1000] | inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" SELECT * FROM _timescaledb_catalog.bgw_job where id=:job_id; id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | initial_start | hypertable_id | config | check_schema | check_name | timezone ------+--------------------------------------------+-------------------+-------------+-------------+--------------+------------------------+-------------------------------------+-------------------+-----------+----------------+---------------+---------------+-----------------------------------------------------------------+------------------------+-------------------------------------------+---------- 1000 | Refresh Continuous Aggregate Policy [1000] | @ 12 hours | @ 0 | -1 | @ 12 hours | _timescaledb_functions | policy_refresh_continuous_aggregate | default_perm_user | t | f | | 2 | {"end_offset": 4, "start_offset": null, "mat_hypertable_id": 2} | _timescaledb_functions | policy_refresh_continuous_aggregate_check | -- job ran once, successfully SELECT job_id, next_start-last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------+------------------+------------+-----------------+----------------+--------------- 1000 | @ 12 hours | t | 1 | 1 | 0 | 0 --clear log for next run of scheduler. TRUNCATE public.bgw_log; CREATE FUNCTION wait_for_timer_to_run(started_at INTEGER, spins INTEGER=:TEST_SPINWAIT_ITERS) RETURNS BOOLEAN LANGUAGE PLPGSQL AS $BODY$ DECLARE num_runs INTEGER; message TEXT; BEGIN select format('[TESTING] Wait until %%, started at %s', started_at) into message; FOR i in 1..spins LOOP SELECT COUNT(*) from bgw_log where msg LIKE message INTO num_runs; if (num_runs > 0) THEN RETURN true; ELSE PERFORM pg_sleep(0.1); END IF; END LOOP; RETURN false; END $BODY$; --make sure there is 1 job to start with SELECT test.wait_for_job_to_run(:job_id, 1); wait_for_job_to_run --------------------- t SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_FOR_OTHER_TO_ADVANCE); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- --start the scheduler on 0 time SELECT ts_bgw_params_reset_time(0, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run(extract(epoch from interval '24 hour')::int * 1000, 0); ts_bgw_db_scheduler_test_run ------------------------------ SELECT wait_for_timer_to_run(0); wait_for_timer_to_run ----------------------- t --advance to 12:00 so that it runs one more time; now we know the --scheduler has loaded up the job with the old schedule_interval SELECT ts_bgw_params_reset_time(extract(epoch from interval '12 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- SELECT test.wait_for_job_to_run(:job_id, 2); wait_for_job_to_run --------------------- t --advance clock 1us to make the scheduler realize the job is done SELECT ts_bgw_params_reset_time((extract(epoch from interval '12 hour')::bigint * 1000000)+1, true); ts_bgw_params_reset_time -------------------------- --alter the refresh interval and check if next_start is altered SELECT alter_job(:job_id, schedule_interval => '1m', retry_period => '1m'); alter_job ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1000,"@ 1 min","@ 0",-1,"@ 1 min",t,"{""end_offset"": 4, ""start_offset"": null, ""mat_hypertable_id"": 2}","Sat Jan 01 04:01:00 2000 PST",_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1000]") SELECT job_id, next_start - last_finish as until_next, total_runs FROM _timescaledb_internal.bgw_job_stat WHERE job_id=:job_id;; job_id | until_next | total_runs --------+------------+------------ 1000 | @ 1 min | 2 --advance to 12:02, job should have run at 12:01 SELECT ts_bgw_params_reset_time((extract(epoch from interval '12 hour')::bigint * 1000000)+(extract(epoch from interval '2 minute')::bigint * 1000000), true); ts_bgw_params_reset_time -------------------------- SELECT test.wait_for_job_to_run(:job_id, 3); wait_for_job_to_run --------------------- t --next run in 1 minute SELECT job_id, next_start-last_finish as until_next, total_runs FROM _timescaledb_internal.bgw_job_stat WHERE job_id=:job_id; job_id | until_next | total_runs --------+------------+------------ 1000 | @ 1 min | 3 --change next run to be after 30s instead SELECT (next_start - '30s'::interval) AS "NEW_NEXT_START" FROM _timescaledb_internal.bgw_job_stat WHERE job_id=:job_id \gset SELECT alter_job(:job_id, next_start => :'NEW_NEXT_START'); alter_job ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1000,"@ 1 min","@ 0",-1,"@ 1 min",t,"{""end_offset"": 4, ""start_offset"": null, ""mat_hypertable_id"": 2}","Sat Jan 01 04:02:30 2000 PST",_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1000]") SELECT ts_bgw_params_reset_time((extract(epoch from interval '12 hour')::bigint * 1000000)+(extract(epoch from interval '2 minute 30 seconds')::bigint * 1000000), true); ts_bgw_params_reset_time -------------------------- SELECT test.wait_for_job_to_run(:job_id, 4); wait_for_job_to_run --------------------- t --advance clock to quit scheduler SELECT ts_bgw_params_reset_time(extract(epoch from interval '25 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- select ts_bgw_db_scheduler_test_wait_for_scheduler_finish(); ts_bgw_db_scheduler_test_wait_for_scheduler_finish ---------------------------------------------------- SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_ON_JOB); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- TRUNCATE public.bgw_log; -- data before 8 SELECT * FROM test_continuous_agg_view ORDER BY 1; time_bucket | value -------------+------- 0 | 1 2 | 5 4 | 9 -- invalidations test by running job multiple times SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- DROP MATERIALIZED VIEW test_continuous_agg_view; NOTICE: drop cascades to table _timescaledb_internal._hyper_2_3_chunk CREATE MATERIALIZED VIEW test_continuous_agg_view WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time), SUM(data) as value FROM test_continuous_agg_table GROUP BY 1 WITH NO DATA; SELECT add_continuous_aggregate_policy('test_continuous_agg_view', 100::integer, -2::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1001 SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg \gset SELECT id AS job_id FROM _timescaledb_catalog.bgw_job WHERE hypertable_id=:mat_hypertable_id \gset SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+--------------------------------------------+--------------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 10, 12 ] (batch 1 of 2) 1 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 3 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -10, 10 ] (batch 2 of 2) 4 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 5 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" -- job ran once, successfully SELECT job_id, last_finish - next_start as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+----------------+------------------+------------+-----------------+----------------+--------------- 1001 | @ 12 hours ago | t | 1 | 1 | 0 | 0 -- should have refreshed everything we have so far SELECT * FROM test_continuous_agg_view ORDER BY 1; time_bucket | value -------------+------- 0 | 1 2 | 5 4 | 9 6 | 13 8 | 17 10 | 10 -- invalidate some data UPDATE test_continuous_agg_table SET data = 11 WHERE time = 6; --advance time by 12h so that job runs one more time SELECT ts_bgw_params_reset_time(extract(epoch from interval '12 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25, 25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-------------+--------------------------------------------+--------------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 10, 12 ] (batch 1 of 2) 1 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 3 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -10, 10 ] (batch 2 of 2) 4 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 5 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 0 | 43200000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 43200000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -90, -10 ] 1 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | inserted 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 3 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 6, 8 ] 4 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 5 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" SELECT job_id, next_start - last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------+------------------+------------+-----------------+----------------+--------------- 1001 | @ 12 hours | t | 2 | 2 | 0 | 0 -- should have updated data for time=6 SELECT * FROM test_continuous_agg_view ORDER BY 1; time_bucket | value -------------+------- 0 | 1 2 | 5 4 | 9 6 | 18 8 | 17 10 | 10 \x on --check the information views -- select view_name, view_owner, materialization_hypertable_schema, materialization_hypertable_name from timescaledb_information.continuous_aggregates where view_name::text like '%test_continuous_agg_view'; -[ RECORD 1 ]---------------------+--------------------------- view_name | test_continuous_agg_view view_owner | default_perm_user materialization_hypertable_schema | _timescaledb_internal materialization_hypertable_name | _materialized_hypertable_3 select view_name, view_definition from timescaledb_information.continuous_aggregates where view_name::text like '%test_continuous_agg_view'; -[ RECORD 1 ]---+----------------------------------------------- view_name | test_continuous_agg_view view_definition | SELECT time_bucket(2, "time") AS time_bucket,+ | sum(data) AS value + | FROM test_continuous_agg_table + | GROUP BY (time_bucket(2, "time")); select job_status, last_run_duration from timescaledb_information.job_stats ps, timescaledb_information.continuous_aggregates cagg where cagg.view_name::text like '%test_continuous_agg_view' and cagg.materialization_hypertable_name = ps.hypertable_name; -[ RECORD 1 ]-----+---------- job_status | Scheduled last_run_duration | \x off -- test merged refresh (change data in two chunks) UPDATE test_continuous_agg_table SET data = 11; --advance time by 1day so that job runs one more time SELECT ts_bgw_params_reset_time(extract(epoch from interval '1day')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(50, 50); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-------------+--------------------------------------------+--------------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 10, 12 ] (batch 1 of 2) 1 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 3 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -10, 10 ] (batch 2 of 2) 4 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 5 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 0 | 43200000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 43200000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ -90, -10 ] 1 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | inserted 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 3 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 6, 8 ] 4 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 5 | 43200000000 | Refresh Continuous Aggregate Policy [1001] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" 0 | 86400000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 86400000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 86400000000 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view" in window [ 0, 12 ] 1 | 86400000000 | Refresh Continuous Aggregate Policy [1001] | deleted 6 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 86400000000 | Refresh Continuous Aggregate Policy [1001] | inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" DROP MATERIALIZED VIEW test_continuous_agg_view; NOTICE: drop cascades to table _timescaledb_internal._hyper_3_4_chunk --create a view with a function that it has no permission to execute CREATE MATERIALIZED VIEW test_continuous_agg_view WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time), SUM(data) as value, get_constant_no_perms() FROM test_continuous_agg_table GROUP BY 1 WITH NO DATA; SELECT add_continuous_aggregate_policy('test_continuous_agg_view', 100::integer, -2::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1002 SELECT id AS job_id FROM _timescaledb_catalog.bgw_job ORDER BY id desc limit 1 \gset SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ -- job fails SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1002 | f | 1 | 0 | 1 | 0 DROP MATERIALIZED VIEW test_continuous_agg_view; --advance clock to quit scheduler SELECT ts_bgw_params_reset_time(extract(epoch from interval '25 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- select ts_bgw_db_scheduler_test_wait_for_scheduler_finish(); ts_bgw_db_scheduler_test_wait_for_scheduler_finish ---------------------------------------------------- SELECT ts_bgw_params_mock_wait_returns_immediately(:WAIT_ON_JOB); ts_bgw_params_mock_wait_returns_immediately --------------------------------------------- --clear log for next run of the scheduler TRUNCATE public.bgw_log; SELECT ts_bgw_params_reset_time(); ts_bgw_params_reset_time -------------------------- -- -- Test creating continuous aggregate with a user that is the non-owner of the raw table -- CREATE TABLE test_continuous_agg_table_w_grant(time int, data int); SELECT create_hypertable('test_continuous_agg_table_w_grant', 'time', chunk_time_interval => 10); create_hypertable ------------------------------------------------ (5,public,test_continuous_agg_table_w_grant,t) CREATE OR REPLACE FUNCTION integer_now_test1() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), 0) FROM test_continuous_agg_table_w_grant $$; SELECT set_integer_now_func('test_continuous_agg_table_w_grant', 'integer_now_test1'); set_integer_now_func ---------------------- GRANT SELECT, TRIGGER ON test_continuous_agg_table_w_grant TO public; INSERT INTO test_continuous_agg_table_w_grant SELECT 1 , 1; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 -- make sure view can be created CREATE MATERIALIZED VIEW test_continuous_agg_view_user_2 WITH ( timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time), SUM(data) as value FROM test_continuous_agg_table_w_grant GROUP BY 1 WITH NO DATA; SELECT add_continuous_aggregate_policy('test_continuous_agg_view_user_2', NULL, -2::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1003 SELECT id AS job_id FROM _timescaledb_catalog.bgw_job ORDER BY id desc limit 1 \gset SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT id, owner FROM _timescaledb_catalog.bgw_job WHERE id = :job_id ; id | owner ------+--------------------- 1003 | default_perm_user_2 SELECT job_id, next_start - last_finish as until_next, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | until_next | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------+------------------+------------+-----------------+----------------+--------------- 1003 | @ 12 hours | t | 1 | 1 | 0 | 0 --view is populated SELECT * FROM test_continuous_agg_view_user_2 ORDER BY 1; time_bucket | value -------------+------- 0 | 1 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --revoke permissions from the continuous agg view owner to select from raw table --no further updates to cont agg should happen REVOKE SELECT ON test_continuous_agg_table_w_grant FROM public; --add new data to table INSERT INTO test_continuous_agg_table_w_grant VALUES(5,1); \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 --advance time by 12h so that job tries to run one more time SELECT ts_bgw_params_reset_time(extract(epoch from interval '12 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25, 25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ --should show a failing execution because no longer has permissions (due to lack of permission on partial view owner's part) SELECT job_id, last_run_success, total_runs, total_successes, total_failures, total_crashes FROM _timescaledb_internal.bgw_job_stat where job_id=:job_id; job_id | last_run_success | total_runs | total_successes | total_failures | total_crashes --------+------------------+------------+-----------------+----------------+--------------- 1003 | f | 2 | 1 | 1 | 0 --view was NOT updated; but the old stuff is still there SELECT * FROM test_continuous_agg_view_user_2; time_bucket | value -------------+------- 0 | 1 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT * from sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-------------+--------------------------------------------+-------------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1003] | continuous aggregate refresh (individual invalidation) on "test_continuous_agg_view_user_2" in window [ -2147483648, 2 ] 1 | 0 | Refresh Continuous Aggregate Policy [1003] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" 2 | 0 | Refresh Continuous Aggregate Policy [1003] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" 0 | 43200000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 43200000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 43200000000 | Refresh Continuous Aggregate Policy [1003] | job 1003 threw an error 1 | 43200000000 | Refresh Continuous Aggregate Policy [1003] | permission denied for table test_continuous_agg_table_w_grant -- Count the number of continuous aggregate policies SELECT count(*) FROM _timescaledb_catalog.bgw_job WHERE proc_schema = '_timescaledb_functions' AND proc_name = 'policy_refresh_continuous_aggregate'; count ------- 1 ================================================ FILE: tsl/test/expected/cagg_bgw_drop_chunks.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- -- Setup for testing bgw jobs --- -- \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(timeout INT = -1) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_create() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; \set WAIT_ON_JOB 0 \set IMMEDIATELY_SET_UNTIL 1 \set WAIT_FOR_OTHER_TO_ADVANCE 2 -- Remove any default jobs, e.g., telemetry DELETE FROM _timescaledb_catalog.bgw_job WHERE TRUE; TRUNCATE _timescaledb_internal.bgw_job_stat; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE public.bgw_log( msg_no INT, mock_time BIGINT, application_name TEXT, msg TEXT ); CREATE VIEW sorted_bgw_log AS SELECT * FROM bgw_log ORDER BY mock_time, application_name COLLATE "C", msg_no; CREATE TABLE public.bgw_dsm_handle_store( handle BIGINT ); INSERT INTO public.bgw_dsm_handle_store VALUES (0); SELECT ts_bgw_params_create(); ts_bgw_params_create ---------------------- ----------------------------------- -- test drop chunks policy runs for materialized hypertables created for -- cont. aggregates ----------------------------------- \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE drop_chunks_table(time BIGINT, data INTEGER); SELECT hypertable_id AS drop_chunks_table_nid FROM create_hypertable('drop_chunks_table', 'time', chunk_time_interval => 1) \gset CREATE OR REPLACE FUNCTION integer_now_test2() returns bigint LANGUAGE SQL STABLE as $$ SELECT 40::bigint $$; SELECT set_integer_now_func('drop_chunks_table', 'integer_now_test2'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW drop_chunks_view1 WITH (timescaledb.continuous) AS SELECT time_bucket('5', time), max(data) FROM drop_chunks_table GROUP BY 1 ORDER BY 1 WITH NO DATA; --raw hypertable will have 40 chunks and the mat. hypertable will have 2 and 4 -- chunks respectively SELECT set_chunk_time_interval('_timescaledb_internal._materialized_hypertable_2', 10); set_chunk_time_interval ------------------------- \set ON_ERROR_STOP 0 INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(1, 39) AS i; \set ON_ERROR_STOP 1 CALL refresh_continuous_aggregate('drop_chunks_view1', NULL, NULL); --TEST1 specify drop chunks policy on mat. hypertable by -- directly does not work \set ON_ERROR_STOP 0 SELECT add_retention_policy( '_timescaledb_internal._materialized_hypertable_2', drop_after => -50) as drop_chunks_job_id1 \gset ERROR: cannot add retention policy to materialized hypertable "_materialized_hypertable_2" \set ON_ERROR_STOP 1 --TEST2 specify drop chunks policy on cont. aggregate -- integer_now func on raw hypertable is used by the drop -- chunks policy SELECT hypertable_id, table_name, integer_now_func FROM _timescaledb_catalog.dimension d, _timescaledb_catalog.hypertable ht WHERE ht.id = d.hypertable_id; hypertable_id | table_name | integer_now_func ---------------+----------------------------+------------------- 1 | drop_chunks_table | integer_now_test2 2 | _materialized_hypertable_2 | SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = '_materialized_hypertable_2' ORDER BY range_start_integer; chunk_name | range_start_integer | range_end_integer -------------------+---------------------+------------------- _hyper_2_40_chunk | 0 | 10 _hyper_2_41_chunk | 10 | 20 _hyper_2_42_chunk | 20 | 30 _hyper_2_43_chunk | 30 | 40 SELECT add_retention_policy( 'drop_chunks_view1', drop_after => 10) as drop_chunks_job_id1 \gset SELECT alter_job(:drop_chunks_job_id1, schedule_interval => INTERVAL '1 second'); alter_job -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1000,"@ 1 sec","@ 5 mins",-1,"@ 5 mins",t,"{""drop_after"": 10, ""hypertable_id"": 2}",-infinity,_timescaledb_functions.policy_retention_check,f,,,"Retention Policy [1000]") SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(2000000); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT count(c) from show_chunks('drop_chunks_view1') as c ; count ------- 1 SELECT remove_retention_policy('drop_chunks_view1'); remove_retention_policy ------------------------- \set ON_ERROR_STOP 0 SELECT remove_retention_policy('unknown'); ERROR: relation "unknown" does not exist at character 32 SELECT remove_retention_policy('1'); ERROR: relation is not a hypertable or continuous aggregate \set ON_ERROR_STOP 1 ================================================ FILE: tsl/test/expected/cagg_ddl-15.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- Set this variable to avoid using a hard-coded path each time query -- results are compared \set QUERY_RESULT_TEST_EQUAL_RELPATH '../../../../test/sql/include/query_result_test_equal.sql' SET ROLE :ROLE_DEFAULT_PERM_USER; --DDL commands on continuous aggregates CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature integer NULL, humidity DOUBLE PRECISION NULL, timemeasure TIMESTAMPTZ, timeinterval INTERVAL ); SELECT table_name FROM create_hypertable('conditions', 'timec'); table_name ------------ conditions -- schema tests \c :TEST_DBNAME :ROLE_SUPERUSER SET timezone TO 'UTC+8'; -- drop if the tablespace1 and/or tablespace2 exists SET client_min_messages TO error; DROP TABLESPACE IF EXISTS tablespace1; DROP TABLESPACE IF EXISTS tablespace2; RESET client_min_messages; CREATE TABLESPACE tablespace1 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE1_PATH; CREATE TABLESPACE tablespace2 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE2_PATH; CREATE SCHEMA rename_schema; GRANT ALL ON SCHEMA rename_schema TO :ROLE_DEFAULT_PERM_USER; CREATE SCHEMA test_schema AUTHORIZATION :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE TABLE foo(time TIMESTAMPTZ NOT NULL, data INTEGER); SELECT create_hypertable('foo', 'time'); create_hypertable ------------------- (2,public,foo,t) CREATE MATERIALIZED VIEW rename_test_old WITH ( timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1week', time), COUNT(data) FROM foo GROUP BY 1 WITH NO DATA; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+-----------------+-----------------------+------------------- public | rename_test_old | _timescaledb_internal | _partial_view_3 ALTER TABLE rename_test_old RENAME TO rename_test; ALTER TABLE rename_test SET SCHEMA test_schema; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+-----------------------+------------------- test_schema | rename_test | _timescaledb_internal | _partial_view_3 ALTER MATERIALIZED VIEW test_schema.rename_test SET SCHEMA rename_schema; DROP SCHEMA test_schema; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+-----------------------+------------------- rename_schema | rename_test | _timescaledb_internal | _partial_view_3 SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA", direct_view_name as "DIR_VIEW_NAME", direct_view_schema as "DIR_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'rename_test' \gset RESET ROLE; SELECT current_user; current_user -------------- super_user ALTER VIEW :"PART_VIEW_SCHEMA".:"PART_VIEW_NAME" SET SCHEMA public; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+---------------------+------------------- rename_schema | rename_test | public | _partial_view_3 --alter direct view schema SELECT user_view_schema, user_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | direct_view_schema | direct_view_name ------------------+----------------+-----------------------+------------------ rename_schema | rename_test | _timescaledb_internal | _direct_view_3 RESET ROLE; SELECT current_user; current_user -------------- super_user ALTER VIEW :"DIR_VIEW_SCHEMA".:"DIR_VIEW_NAME" SET SCHEMA public; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name ------------------+----------------+---------------------+-------------------+--------------------+------------------ rename_schema | rename_test | public | _partial_view_3 | public | _direct_view_3 RESET ROLE; SELECT current_user; current_user -------------- super_user ALTER SCHEMA rename_schema RENAME TO new_name_schema; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name ------------------+----------------+---------------------+-------------------+--------------------+------------------ new_name_schema | rename_test | public | _partial_view_3 | public | _direct_view_3 ALTER VIEW :"PART_VIEW_NAME" SET SCHEMA new_name_schema; ALTER VIEW :"DIR_VIEW_NAME" SET SCHEMA new_name_schema; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name ------------------+----------------+---------------------+-------------------+--------------------+------------------ new_name_schema | rename_test | new_name_schema | _partial_view_3 | new_name_schema | _direct_view_3 RESET ROLE; SELECT current_user; current_user -------------- super_user ALTER SCHEMA new_name_schema RENAME TO foo_name_schema; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+---------------------+------------------- foo_name_schema | rename_test | foo_name_schema | _partial_view_3 ALTER MATERIALIZED VIEW foo_name_schema.rename_test SET SCHEMA public; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+---------------------+------------------- public | rename_test | foo_name_schema | _partial_view_3 RESET ROLE; SELECT current_user; current_user -------------- super_user ALTER SCHEMA foo_name_schema RENAME TO rename_schema; SET ROLE :ROLE_DEFAULT_PERM_USER; SET client_min_messages TO NOTICE; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+---------------------+------------------- public | rename_test | rename_schema | _partial_view_3 ALTER MATERIALIZED VIEW rename_test RENAME TO rename_c_aggregate; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+--------------------+---------------------+------------------- public | rename_c_aggregate | rename_schema | _partial_view_3 SELECT * FROM rename_c_aggregate; time_bucket | count -------------+------- ALTER VIEW rename_schema.:"PART_VIEW_NAME" RENAME TO partial_view; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name ------------------+--------------------+---------------------+-------------------+--------------------+------------------ public | rename_c_aggregate | rename_schema | partial_view | rename_schema | _direct_view_3 --rename direct view ALTER VIEW rename_schema.:"DIR_VIEW_NAME" RENAME TO direct_view; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name ------------------+--------------------+---------------------+-------------------+--------------------+------------------ public | rename_c_aggregate | rename_schema | partial_view | rename_schema | direct_view -- drop_chunks tests DROP TABLE conditions CASCADE; DROP TABLE foo CASCADE; NOTICE: drop cascades to 2 other objects CREATE TABLE drop_chunks_table(time BIGINT NOT NULL, data INTEGER); SELECT hypertable_id AS drop_chunks_table_id FROM create_hypertable('drop_chunks_table', 'time', chunk_time_interval => 10) \gset CREATE OR REPLACE FUNCTION integer_now_test() returns bigint LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), bigint '0') FROM drop_chunks_table $$; SELECT set_integer_now_func('drop_chunks_table', 'integer_now_test'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW drop_chunks_view WITH ( timescaledb.continuous, timescaledb.materialized_only=true ) AS SELECT time_bucket('5', time), COUNT(data) FROM drop_chunks_table GROUP BY 1 WITH NO DATA; SELECT format('%I.%I', schema_name, table_name) AS drop_chunks_mat_table, schema_name AS drop_chunks_mat_schema, table_name AS drop_chunks_mat_table_name FROM _timescaledb_catalog.hypertable, _timescaledb_catalog.continuous_agg WHERE _timescaledb_catalog.continuous_agg.raw_hypertable_id = :drop_chunks_table_id AND _timescaledb_catalog.hypertable.id = _timescaledb_catalog.continuous_agg.mat_hypertable_id \gset -- create 3 chunks, with 3 time bucket INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(0, 29) AS i; -- Only refresh up to bucket 15 initially. Matches the old refresh -- behavior that didn't materialize everything CALL refresh_continuous_aggregate('drop_chunks_view', 0, 15); SELECT count(c) FROM show_chunks('drop_chunks_table') AS c; count ------- 3 SELECT count(c) FROM show_chunks('drop_chunks_view') AS c; count ------- 1 SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | count -------------+------- 0 | 5 5 | 5 10 | 5 -- cannot drop directly from the materialization table without specifying -- cont. aggregate view name explicitly \set ON_ERROR_STOP 0 SELECT drop_chunks(:'drop_chunks_mat_table', newer_than => -20, verbose => true); ERROR: operation not supported on materialized hypertable \set ON_ERROR_STOP 1 SELECT count(c) FROM show_chunks('drop_chunks_table') AS c; count ------- 3 SELECT count(c) FROM show_chunks('drop_chunks_view') AS c; count ------- 1 SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | count -------------+------- 0 | 5 5 | 5 10 | 5 -- drop chunks when the chunksize and time_bucket aren't aligned DROP TABLE drop_chunks_table CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_5_4_chunk CREATE TABLE drop_chunks_table_u(time BIGINT NOT NULL, data INTEGER); SELECT hypertable_id AS drop_chunks_table_u_id FROM create_hypertable('drop_chunks_table_u', 'time', chunk_time_interval => 7) \gset CREATE OR REPLACE FUNCTION integer_now_test1() returns bigint LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), bigint '0') FROM drop_chunks_table_u $$; SELECT set_integer_now_func('drop_chunks_table_u', 'integer_now_test1'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW drop_chunks_view WITH ( timescaledb.continuous, timescaledb.materialized_only=true ) AS SELECT time_bucket('3', time), COUNT(data) FROM drop_chunks_table_u GROUP BY 1 WITH NO DATA; SELECT format('%I.%I', schema_name, table_name) AS drop_chunks_mat_table_u, schema_name AS drop_chunks_mat_schema, table_name AS drop_chunks_mat_table_u_name FROM _timescaledb_catalog.hypertable, _timescaledb_catalog.continuous_agg WHERE _timescaledb_catalog.continuous_agg.raw_hypertable_id = :drop_chunks_table_u_id AND _timescaledb_catalog.hypertable.id = _timescaledb_catalog.continuous_agg.mat_hypertable_id \gset -- create 3 chunks, with 3 time bucket INSERT INTO drop_chunks_table_u SELECT i, i FROM generate_series(0, 21) AS i; -- Refresh up to bucket 15 to match old materializer behavior CALL refresh_continuous_aggregate('drop_chunks_view', 0, 15); SELECT count(c) FROM show_chunks('drop_chunks_table_u') AS c; count ------- 4 SELECT count(c) FROM show_chunks('drop_chunks_view') AS c; count ------- 1 SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | count -------------+------- 0 | 3 3 | 3 6 | 3 9 | 3 12 | 3 -- TRUNCATE test -- Can truncate regular hypertables that have caggs TRUNCATE drop_chunks_table_u; \set ON_ERROR_STOP 0 -- Can't truncate materialized hypertables directly TRUNCATE :drop_chunks_mat_table_u; ERROR: cannot TRUNCATE a hypertable underlying a continuous aggregate \set ON_ERROR_STOP 1 -- Check that we don't interfere with TRUNCATE of normal table and -- partitioned table CREATE TABLE truncate (value int); INSERT INTO truncate VALUES (1), (2); TRUNCATE truncate; SELECT * FROM truncate; value ------- CREATE TABLE truncate_partitioned (value int) PARTITION BY RANGE(value); CREATE TABLE truncate_p1 PARTITION OF truncate_partitioned FOR VALUES FROM (1) TO (3); INSERT INTO truncate_partitioned VALUES (1), (2); TRUNCATE truncate_partitioned; SELECT * FROM truncate_partitioned; value ------- -- ALTER TABLE tests \set ON_ERROR_STOP 0 -- test a variety of ALTER TABLE statements ALTER TABLE :drop_chunks_mat_table_u RENAME time_bucket TO bad_name; ERROR: renaming columns on materialization tables is not supported ALTER TABLE :drop_chunks_mat_table_u ADD UNIQUE(time_bucket); ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u SET UNLOGGED; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u ENABLE ROW LEVEL SECURITY; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u ADD COLUMN fizzle INTEGER; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u DROP COLUMN time_bucket; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u ALTER COLUMN time_bucket DROP NOT NULL; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u ALTER COLUMN time_bucket SET DEFAULT 1; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u ALTER COLUMN time_bucket SET STORAGE EXTERNAL; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u DISABLE TRIGGER ALL; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u SET TABLESPACE foo; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u NOT OF; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u OWNER TO CURRENT_USER; ERROR: operation not supported on materialization tables \set ON_ERROR_STOP 1 ALTER TABLE :drop_chunks_mat_table_u SET SCHEMA public; ALTER TABLE :drop_chunks_mat_table_u_name RENAME TO new_name; SET ROLE :ROLE_DEFAULT_PERM_USER; SET client_min_messages TO NOTICE; SELECT * FROM new_name ORDER BY 1; time_bucket | count -------------+------- 0 | 3 3 | 3 6 | 3 9 | 3 12 | 3 SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | count -------------+------- 0 | 3 3 | 3 6 | 3 9 | 3 12 | 3 \set ON_ERROR_STOP 0 -- no continuous aggregates on a continuous aggregate materialization table CREATE MATERIALIZED VIEW new_name_view WITH ( timescaledb.continuous, timescaledb.materialized_only=true ) AS SELECT time_bucket('6', time_bucket), COUNT("count") FROM new_name GROUP BY 1 WITH NO DATA; ERROR: hypertable is a continuous aggregate materialization table \set ON_ERROR_STOP 1 CREATE TABLE metrics(time timestamptz NOT NULL, device_id int, v1 float, v2 float); SELECT create_hypertable('metrics','time'); create_hypertable ---------------------- (8,public,metrics,t) INSERT INTO metrics SELECT generate_series('2000-01-01'::timestamptz,'2000-01-10','1m'),1,0.25,0.75; -- check expressions in view definition CREATE MATERIALIZED VIEW cagg_expr WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1d', time) AS time, 'Const'::text AS Const, 4.3::numeric AS "numeric", first(metrics,time), CASE WHEN true THEN 'foo' ELSE 'bar' END, COALESCE(NULL,'coalesce'), avg(v1) + avg(v2) AS avg1, avg(v1+v2) AS avg2 FROM metrics GROUP BY 1 WITH NO DATA; CALL refresh_continuous_aggregate('cagg_expr', NULL, NULL); SELECT * FROM cagg_expr ORDER BY time LIMIT 5; time | const | numeric | first | case | coalesce | avg1 | avg2 ------------------------------+-------+---------+----------------------------------------------+------+----------+------+------ Fri Dec 31 16:00:00 1999 UTC | Const | 4.3 | ("Sat Jan 01 00:00:00 2000 UTC",1,0.25,0.75) | foo | coalesce | 1 | 1 Sat Jan 01 16:00:00 2000 UTC | Const | 4.3 | ("Sat Jan 01 16:00:00 2000 UTC",1,0.25,0.75) | foo | coalesce | 1 | 1 Sun Jan 02 16:00:00 2000 UTC | Const | 4.3 | ("Sun Jan 02 16:00:00 2000 UTC",1,0.25,0.75) | foo | coalesce | 1 | 1 Mon Jan 03 16:00:00 2000 UTC | Const | 4.3 | ("Mon Jan 03 16:00:00 2000 UTC",1,0.25,0.75) | foo | coalesce | 1 | 1 Tue Jan 04 16:00:00 2000 UTC | Const | 4.3 | ("Tue Jan 04 16:00:00 2000 UTC",1,0.25,0.75) | foo | coalesce | 1 | 1 --test materialization of invalidation before drop DROP TABLE IF EXISTS drop_chunks_table CASCADE; NOTICE: table "drop_chunks_table" does not exist, skipping DROP TABLE IF EXISTS drop_chunks_table_u CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_7_9_chunk CREATE TABLE drop_chunks_table(time BIGINT NOT NULL, data INTEGER); SELECT hypertable_id AS drop_chunks_table_nid FROM create_hypertable('drop_chunks_table', 'time', chunk_time_interval => 10) \gset CREATE OR REPLACE FUNCTION integer_now_test2() returns bigint LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), bigint '0') FROM drop_chunks_table $$; SELECT set_integer_now_func('drop_chunks_table', 'integer_now_test2'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW drop_chunks_view WITH ( timescaledb.continuous, timescaledb.materialized_only=true ) AS SELECT time_bucket('5', time), max(data) FROM drop_chunks_table GROUP BY 1 WITH NO DATA; INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(0, 20) AS i; --dropping chunks will process the invalidations SELECT drop_chunks('drop_chunks_table', older_than => (integer_now_test2()-9)); drop_chunks ------------------------------------------ _timescaledb_internal._hyper_10_13_chunk SELECT * FROM drop_chunks_table ORDER BY time ASC limit 1; time | data ------+------ 10 | 10 INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(20, 35) AS i; CALL refresh_continuous_aggregate('drop_chunks_view', 10, 40); --this will be seen after the drop its within the invalidation window and will be dropped INSERT INTO drop_chunks_table VALUES (26, 100); --this will not be processed by the drop since chunk 30-39 is not dropped but will be seen after refresh --shows that the drop doesn't do more work than necessary INSERT INTO drop_chunks_table VALUES (31, 200); --move the time up to 39 INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(35, 39) AS i; --the chunks and ranges we have thus far SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table'; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_10_14_chunk | 10 | 20 _hyper_10_15_chunk | 20 | 30 _hyper_10_16_chunk | 30 | 40 --the invalidation on 25 not yet seen SELECT * FROM drop_chunks_view ORDER BY time_bucket DESC; time_bucket | max -------------+----- 35 | 35 30 | 34 25 | 29 20 | 24 15 | 19 10 | 14 --refresh to process the invalidations and then drop CALL refresh_continuous_aggregate('drop_chunks_view', NULL, (integer_now_test2()-9)); SELECT drop_chunks('drop_chunks_table', older_than => (integer_now_test2()-9)); drop_chunks ------------------------------------------ _timescaledb_internal._hyper_10_14_chunk _timescaledb_internal._hyper_10_15_chunk --new values on 25 now seen in view SELECT * FROM drop_chunks_view ORDER BY time_bucket DESC; time_bucket | max -------------+----- 35 | 35 30 | 34 25 | 100 20 | 24 15 | 19 10 | 14 --earliest datapoint now in table SELECT * FROM drop_chunks_table ORDER BY time ASC limit 1; time | data ------+------ 30 | 30 --still see data in the view SELECT * FROM drop_chunks_view WHERE time_bucket < (integer_now_test2()-9) ORDER BY time_bucket DESC; time_bucket | max -------------+----- 25 | 100 20 | 24 15 | 19 10 | 14 --no data but covers dropped chunks SELECT * FROM drop_chunks_table WHERE time < (integer_now_test2()-9) ORDER BY time DESC; time | data ------+------ --recreate the dropped chunk INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(0, 20) AS i; --see data from recreated region SELECT * FROM drop_chunks_table WHERE time < (integer_now_test2()-9) ORDER BY time DESC; time | data ------+------ 20 | 20 19 | 19 18 | 18 17 | 17 16 | 16 15 | 15 14 | 14 13 | 13 12 | 12 11 | 11 10 | 10 9 | 9 8 | 8 7 | 7 6 | 6 5 | 5 4 | 4 3 | 3 2 | 2 1 | 1 0 | 0 --should show chunk with old name and old ranges SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table' ORDER BY range_start_integer; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_10_18_chunk | 0 | 10 _hyper_10_19_chunk | 10 | 20 _hyper_10_20_chunk | 20 | 30 _hyper_10_16_chunk | 30 | 40 --We dropped everything up to the bucket starting at 30 and then --inserted new data up to and including time 20. Therefore, the --dropped data should stay the same as long as we only refresh --buckets that have non-dropped data. CALL refresh_continuous_aggregate('drop_chunks_view', 30, 40); SELECT * FROM drop_chunks_view ORDER BY time_bucket DESC; time_bucket | max -------------+----- 35 | 39 30 | 200 25 | 100 20 | 24 15 | 19 10 | 14 SELECT format('%I.%I', schema_name, table_name) AS drop_chunks_mat_tablen, schema_name AS drop_chunks_mat_schema, table_name AS drop_chunks_mat_table_name FROM _timescaledb_catalog.hypertable, _timescaledb_catalog.continuous_agg WHERE _timescaledb_catalog.continuous_agg.raw_hypertable_id = :drop_chunks_table_nid AND _timescaledb_catalog.hypertable.id = _timescaledb_catalog.continuous_agg.mat_hypertable_id \gset -- TEST drop chunks from continuous aggregates by specifying view name SELECT drop_chunks('drop_chunks_view', newer_than => -20, verbose => true); INFO: dropping chunk _timescaledb_internal._hyper_11_17_chunk drop_chunks ------------------------------------------ _timescaledb_internal._hyper_11_17_chunk -- Test that we cannot drop chunks when specifying materialized -- hypertable INSERT INTO drop_chunks_table SELECT generate_series(45, 55), 500; CALL refresh_continuous_aggregate('drop_chunks_view', 45, 55); SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = :'drop_chunks_mat_table_name' ORDER BY range_start_integer; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_11_23_chunk | 0 | 100 \set ON_ERROR_STOP 0 \set VERBOSITY default SELECT drop_chunks(:'drop_chunks_mat_tablen', older_than => 60); ERROR: operation not supported on materialized hypertable DETAIL: Hypertable "_materialized_hypertable_11" is a materialized hypertable. HINT: Try the operation on the continuous aggregate instead. \set VERBOSITY terse \set ON_ERROR_STOP 1 ----------------------------------------------------------------- -- Test that refresh_continuous_aggregate on chunk will refresh, -- but only in the regions covered by the show chunks. ----------------------------------------------------------------- SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table' ORDER BY 2,3; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_10_18_chunk | 0 | 10 _hyper_10_19_chunk | 10 | 20 _hyper_10_20_chunk | 20 | 30 _hyper_10_16_chunk | 30 | 40 _hyper_10_21_chunk | 40 | 50 _hyper_10_22_chunk | 50 | 60 -- Pick the second chunk as the one to drop WITH numbered_chunks AS ( SELECT row_number() OVER (ORDER BY range_start_integer), chunk_schema, chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table' ORDER BY 1 ) SELECT format('%I.%I', chunk_schema, chunk_name) AS chunk_to_drop, range_start_integer, range_end_integer FROM numbered_chunks WHERE row_number = 2 \gset -- There's data in the table for the chunk/range we will drop SELECT * FROM drop_chunks_table WHERE time >= :range_start_integer AND time < :range_end_integer ORDER BY 1; time | data ------+------ 10 | 10 11 | 11 12 | 12 13 | 13 14 | 14 15 | 15 16 | 16 17 | 17 18 | 18 19 | 19 -- Make sure there is also data in the continuous aggregate -- CARE: -- Note that this behaviour of dropping the materialization table chunks and expecting a refresh -- that overlaps that time range to NOT update those chunks is undefined. CALL refresh_continuous_aggregate('drop_chunks_view', 0, 50); SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | max -------------+----- 0 | 4 5 | 9 10 | 14 15 | 19 20 | 20 45 | 500 50 | 500 -- Drop the second chunk, to leave a gap in the data DROP TABLE :chunk_to_drop; -- Verify that the second chunk is dropped SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table' ORDER BY 2,3; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_10_18_chunk | 0 | 10 _hyper_10_20_chunk | 20 | 30 _hyper_10_16_chunk | 30 | 40 _hyper_10_21_chunk | 40 | 50 _hyper_10_22_chunk | 50 | 60 -- Data is no longer in the table but still in the view SELECT * FROM drop_chunks_table WHERE time >= :range_start_integer AND time < :range_end_integer ORDER BY 1; time | data ------+------ SELECT * FROM drop_chunks_view WHERE time_bucket >= :range_start_integer AND time_bucket < :range_end_integer ORDER BY 1; time_bucket | max -------------+----- 10 | 14 15 | 19 -- Insert a large value in one of the chunks that will be dropped INSERT INTO drop_chunks_table VALUES (:range_start_integer-1, 100); -- Now refresh and drop the two adjecent chunks CALL refresh_continuous_aggregate('drop_chunks_view', NULL, 30); SELECT drop_chunks('drop_chunks_table', older_than=>30); drop_chunks ------------------------------------------ _timescaledb_internal._hyper_10_18_chunk _timescaledb_internal._hyper_10_20_chunk -- Verify that the chunks are dropped SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table' ORDER BY 2,3; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_10_16_chunk | 30 | 40 _hyper_10_21_chunk | 40 | 50 _hyper_10_22_chunk | 50 | 60 -- The continuous aggregate should be refreshed in the regions covered -- by the dropped chunks, but not in the "gap" region, i.e., the -- region of the chunk that was dropped via DROP TABLE. SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | max -------------+----- 0 | 4 5 | 100 20 | 20 45 | 500 50 | 500 -- Now refresh in the region of the first two dropped chunks CALL refresh_continuous_aggregate('drop_chunks_view', 0, :range_end_integer); -- Aggregate data in the refreshed range should no longer exist since -- the underlying data was dropped. SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | max -------------+----- 20 | 20 45 | 500 50 | 500 -------------------------------------------------------------------- -- Check that we can create a materialized table in a tablespace. We -- create one with tablespace and one without and compare them. CREATE VIEW cagg_info AS WITH caggs AS ( SELECT format('%I.%I', user_view_schema, user_view_name)::regclass AS user_view, format('%I.%I', direct_view_schema, direct_view_name)::regclass AS direct_view, format('%I.%I', partial_view_schema, partial_view_name)::regclass AS partial_view, format('%I.%I', ht.schema_name, ht.table_name)::regclass AS mat_relid FROM _timescaledb_catalog.hypertable ht, _timescaledb_catalog.continuous_agg cagg WHERE ht.id = cagg.mat_hypertable_id ) SELECT user_view, pg_get_userbyid(relowner) AS user_view_owner, relname AS mat_table, (SELECT pg_get_userbyid(relowner) FROM pg_class WHERE oid = mat_relid) AS mat_table_owner, direct_view, (SELECT pg_get_userbyid(relowner) FROM pg_class WHERE oid = direct_view) AS direct_view_owner, partial_view, (SELECT pg_get_userbyid(relowner) FROM pg_class WHERE oid = partial_view) AS partial_view_owner, (SELECT spcname FROM pg_tablespace WHERE oid = reltablespace) AS tablespace FROM pg_class JOIN caggs ON pg_class.oid = caggs.mat_relid; GRANT SELECT ON cagg_info TO PUBLIC; CREATE VIEW chunk_info AS SELECT ht.schema_name, ht.table_name, relname AS chunk_name, (SELECT spcname FROM pg_tablespace WHERE oid = reltablespace) AS tablespace FROM pg_class c, _timescaledb_catalog.hypertable ht, _timescaledb_catalog.chunk ch WHERE ch.table_name = c.relname AND ht.id = ch.hypertable_id; CREATE TABLE whatever(time BIGINT NOT NULL, data INTEGER); SELECT hypertable_id AS whatever_nid FROM create_hypertable('whatever', 'time', chunk_time_interval => 10) \gset SELECT set_integer_now_func('whatever', 'integer_now_test'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW whatever_view_1 WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('5', time), COUNT(data) FROM whatever GROUP BY 1 WITH NO DATA; CREATE MATERIALIZED VIEW whatever_view_2 WITH (timescaledb.continuous, timescaledb.materialized_only=true) TABLESPACE tablespace1 AS SELECT time_bucket('5', time), COUNT(data) FROM whatever GROUP BY 1 WITH NO DATA; INSERT INTO whatever SELECT i, i FROM generate_series(0, 29) AS i; CALL refresh_continuous_aggregate('whatever_view_1', NULL, NULL); CALL refresh_continuous_aggregate('whatever_view_2', NULL, NULL); SELECT user_view, mat_table, cagg_info.tablespace AS mat_tablespace, chunk_name, chunk_info.tablespace AS chunk_tablespace FROM cagg_info, chunk_info WHERE mat_table::text = table_name AND user_view::text LIKE 'whatever_view%'; user_view | mat_table | mat_tablespace | chunk_name | chunk_tablespace -----------------+-----------------------------+----------------+--------------------+------------------ whatever_view_1 | _materialized_hypertable_13 | | _hyper_13_27_chunk | whatever_view_2 | _materialized_hypertable_14 | tablespace1 | _hyper_14_28_chunk | tablespace1 ALTER MATERIALIZED VIEW whatever_view_1 SET TABLESPACE tablespace2; SELECT user_view, mat_table, cagg_info.tablespace AS mat_tablespace, chunk_name, chunk_info.tablespace AS chunk_tablespace FROM cagg_info, chunk_info WHERE mat_table::text = table_name AND user_view::text LIKE 'whatever_view%'; user_view | mat_table | mat_tablespace | chunk_name | chunk_tablespace -----------------+-----------------------------+----------------+--------------------+------------------ whatever_view_1 | _materialized_hypertable_13 | tablespace2 | _hyper_13_27_chunk | tablespace2 whatever_view_2 | _materialized_hypertable_14 | tablespace1 | _hyper_14_28_chunk | tablespace1 DROP MATERIALIZED VIEW whatever_view_1; NOTICE: drop cascades to table _timescaledb_internal._hyper_13_27_chunk DROP MATERIALIZED VIEW whatever_view_2; NOTICE: drop cascades to table _timescaledb_internal._hyper_14_28_chunk -- test bucket width expressions on integer hypertables CREATE TABLE metrics_int2 ( time int2 NOT NULL, device_id int, v1 float, v2 float ); CREATE TABLE metrics_int4 ( time int4 NOT NULL, device_id int, v1 float, v2 float ); CREATE TABLE metrics_int8 ( time int8 NOT NULL, device_id int, v1 float, v2 float ); SELECT create_hypertable (('metrics_' || dt)::regclass, 'time', chunk_time_interval => 10) FROM ( VALUES ('int2'), ('int4'), ('int8')) v (dt); create_hypertable ---------------------------- (15,public,metrics_int2,t) (16,public,metrics_int4,t) (17,public,metrics_int8,t) CREATE OR REPLACE FUNCTION int2_now () RETURNS int2 LANGUAGE SQL STABLE AS $$ SELECT 10::int2 $$; CREATE OR REPLACE FUNCTION int4_now () RETURNS int4 LANGUAGE SQL STABLE AS $$ SELECT 10::int4 $$; CREATE OR REPLACE FUNCTION int8_now () RETURNS int8 LANGUAGE SQL STABLE AS $$ SELECT 10::int8 $$; SELECT set_integer_now_func (('metrics_' || dt)::regclass, (dt || '_now')::regproc) FROM ( VALUES ('int2'), ('int4'), ('int8')) v (dt); set_integer_now_func ---------------------- -- width expression for int2 hypertables CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1::smallint, time) FROM metrics_int2 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1::smallint + 2::smallint, time) FROM metrics_int2 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; -- width expression for int4 hypertables CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1, time) FROM metrics_int4 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1 + 2, time) FROM metrics_int4 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; -- width expression for int8 hypertables CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1, time) FROM metrics_int8 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1 + 2, time) FROM metrics_int8 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; \set ON_ERROR_STOP 0 -- non-immutable expresions should be rejected CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(extract(year FROM now())::smallint, time) FROM metrics_int2 GROUP BY 1; ERROR: only immutable expressions allowed in time bucket function CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(extract(year FROM now())::int, time) FROM metrics_int4 GROUP BY 1; ERROR: only immutable expressions allowed in time bucket function CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(extract(year FROM now())::int, time) FROM metrics_int8 GROUP BY 1; ERROR: only immutable expressions allowed in time bucket function \set ON_ERROR_STOP 1 -- Test various ALTER MATERIALIZED VIEW statements. SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE MATERIALIZED VIEW owner_check WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1 + 2, time) FROM metrics_int8 GROUP BY 1 WITH NO DATA; \x on SELECT * FROM cagg_info WHERE user_view::text = 'owner_check'; -[ RECORD 1 ]------+--------------------------------------- user_view | owner_check user_view_owner | default_perm_user mat_table | _materialized_hypertable_24 mat_table_owner | default_perm_user direct_view | _timescaledb_internal._direct_view_24 direct_view_owner | default_perm_user partial_view | _timescaledb_internal._partial_view_24 partial_view_owner | default_perm_user tablespace | \x off -- This should not work since the target user has the wrong role, but -- we test that the normal checks are done when changing the owner. \set ON_ERROR_STOP 0 ALTER MATERIALIZED VIEW owner_check OWNER TO :ROLE_1; ERROR: must be member of role "test_role_1" \set ON_ERROR_STOP 1 -- Superuser can always change owner SET ROLE :ROLE_SUPERUSER; -- Add a refresh policy before changing owner to verify job owner is propagated SELECT add_continuous_aggregate_policy('owner_check', NULL, 1::int8, '1 day'::interval) AS cagg_job_id \gset SELECT owner FROM _timescaledb_config.bgw_job WHERE id = :cagg_job_id; owner ------------------- default_perm_user ALTER MATERIALIZED VIEW owner_check OWNER TO :ROLE_1; \x on SELECT * FROM cagg_info WHERE user_view::text = 'owner_check'; -[ RECORD 1 ]------+--------------------------------------- user_view | owner_check user_view_owner | test_role_1 mat_table | _materialized_hypertable_24 mat_table_owner | test_role_1 direct_view | _timescaledb_internal._direct_view_24 direct_view_owner | test_role_1 partial_view | _timescaledb_internal._partial_view_24 partial_view_owner | test_role_1 tablespace | \x off -- make sure policy job owner is propagated SELECT owner FROM _timescaledb_config.bgw_job WHERE id = :cagg_job_id; owner ------------- test_role_1 SELECT remove_continuous_aggregate_policy('owner_check'); remove_continuous_aggregate_policy ------------------------------------ -- -- Test drop continuous aggregate cases -- -- Issue: #2608 -- CREATE OR REPLACE FUNCTION test_int_now() RETURNS INT LANGUAGE SQL STABLE AS $BODY$ SELECT 50; $BODY$; CREATE TABLE conditionsnm(time_int INT NOT NULL, device INT, value FLOAT); SELECT create_hypertable('conditionsnm', 'time_int', chunk_time_interval => 10); create_hypertable ---------------------------- (25,public,conditionsnm,t) SELECT set_integer_now_func('conditionsnm', 'test_int_now'); set_integer_now_func ---------------------- INSERT INTO conditionsnm SELECT time_val, time_val % 4, 3.14 FROM generate_series(0,100,1) AS time_val; -- Case 1: DROP CREATE MATERIALIZED VIEW conditionsnm_4 WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(7, time_int) as bucket, SUM(value), COUNT(value) FROM conditionsnm GROUP BY bucket WITH DATA; NOTICE: refreshing continuous aggregate "conditionsnm_4" DROP materialized view conditionsnm_4; NOTICE: drop cascades to table _timescaledb_internal._hyper_26_40_chunk -- Case 2: DROP CASCADE should have similar behaviour as DROP CREATE MATERIALIZED VIEW conditionsnm_4 WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(7, time_int) as bucket, SUM(value), COUNT(value) FROM conditionsnm GROUP BY bucket WITH DATA; NOTICE: refreshing continuous aggregate "conditionsnm_4" DROP materialized view conditionsnm_4 CASCADE; NOTICE: drop cascades to table _timescaledb_internal._hyper_27_41_chunk -- Case 3: require CASCADE in case of dependent object CREATE MATERIALIZED VIEW conditionsnm_4 WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(7, time_int) as bucket, SUM(value), COUNT(value) FROM conditionsnm GROUP BY bucket WITH DATA; NOTICE: refreshing continuous aggregate "conditionsnm_4" CREATE VIEW see_cagg as select * from conditionsnm_4; \set ON_ERROR_STOP 0 DROP MATERIALIZED VIEW conditionsnm_4; ERROR: cannot drop view conditionsnm_4 because other objects depend on it \set ON_ERROR_STOP 1 -- Case 4: DROP CASCADE with dependency DROP MATERIALIZED VIEW conditionsnm_4 CASCADE; NOTICE: drop cascades to view see_cagg NOTICE: drop cascades to table _timescaledb_internal._hyper_28_42_chunk -- Test DROP SCHEMA CASCADE with continuous aggregates -- -- Issue: #2350 -- -- Case 1: DROP SCHEMA CASCADE CREATE SCHEMA test_schema; CREATE TABLE test_schema.telemetry_raw ( ts TIMESTAMP WITH TIME ZONE NOT NULL, value DOUBLE PRECISION ); SELECT create_hypertable('test_schema.telemetry_raw', 'ts'); create_hypertable ---------------------------------- (29,test_schema,telemetry_raw,t) CREATE MATERIALIZED VIEW test_schema.telemetry_1s WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(INTERVAL '1s', ts) AS ts_1s, avg(value) FROM test_schema.telemetry_raw GROUP BY ts_1s WITH NO DATA; SELECT ca.raw_hypertable_id, h.schema_name, h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) WHERE user_view_name = 'telemetry_1s'; raw_hypertable_id | schema_name | MAT_TABLE_NAME | PART_VIEW_NAME | partial_view_schema -------------------+-----------------------+-----------------------------+------------------+----------------------- 29 | _timescaledb_internal | _materialized_hypertable_30 | _partial_view_30 | _timescaledb_internal \gset DROP SCHEMA test_schema CASCADE; NOTICE: drop cascades to 4 other objects SELECT count(*) FROM pg_class WHERE relname = :'MAT_TABLE_NAME'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = :'PART_VIEW_NAME'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = 'telemetry_1s'; count ------- 0 SELECT count(*) FROM pg_namespace WHERE nspname = 'test_schema'; count ------- 0 -- Case 2: DROP SCHEMA CASCADE with multiple caggs CREATE SCHEMA test_schema; CREATE TABLE test_schema.telemetry_raw ( ts TIMESTAMP WITH TIME ZONE NOT NULL, value DOUBLE PRECISION ); SELECT create_hypertable('test_schema.telemetry_raw', 'ts'); create_hypertable ---------------------------------- (31,test_schema,telemetry_raw,t) CREATE MATERIALIZED VIEW test_schema.cagg1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(INTERVAL '1s', ts) AS ts_1s, avg(value) FROM test_schema.telemetry_raw GROUP BY ts_1s WITH NO DATA; CREATE MATERIALIZED VIEW test_schema.cagg2 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(INTERVAL '1s', ts) AS ts_1s, avg(value) FROM test_schema.telemetry_raw GROUP BY ts_1s WITH NO DATA; SELECT ca.raw_hypertable_id, h.schema_name, h.table_name AS "MAT_TABLE_NAME1", partial_view_name as "PART_VIEW_NAME1", partial_view_schema FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) WHERE user_view_name = 'cagg1'; raw_hypertable_id | schema_name | MAT_TABLE_NAME1 | PART_VIEW_NAME1 | partial_view_schema -------------------+-----------------------+-----------------------------+------------------+----------------------- 31 | _timescaledb_internal | _materialized_hypertable_32 | _partial_view_32 | _timescaledb_internal \gset SELECT ca.raw_hypertable_id, h.schema_name, h.table_name AS "MAT_TABLE_NAME2", partial_view_name as "PART_VIEW_NAME2", partial_view_schema FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) WHERE user_view_name = 'cagg2'; raw_hypertable_id | schema_name | MAT_TABLE_NAME2 | PART_VIEW_NAME2 | partial_view_schema -------------------+-----------------------+-----------------------------+------------------+----------------------- 31 | _timescaledb_internal | _materialized_hypertable_33 | _partial_view_33 | _timescaledb_internal \gset DROP SCHEMA test_schema CASCADE; NOTICE: drop cascades to 7 other objects SELECT count(*) FROM pg_class WHERE relname = :'MAT_TABLE_NAME1'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = :'PART_VIEW_NAME1'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = 'cagg1'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = :'MAT_TABLE_NAME2'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = :'PART_VIEW_NAME2'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = 'cagg2'; count ------- 0 SELECT count(*) FROM pg_namespace WHERE nspname = 'test_schema'; count ------- 0 DROP TABLESPACE tablespace1; DROP TABLESPACE tablespace2; -- Check that we can rename a column of a materialized view and still -- rebuild it after (#3051, #3405) CREATE TABLE conditions ( time TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL ); SELECT create_hypertable('conditions', 'time'); create_hypertable -------------------------- (34,public,conditions,t) INSERT INTO conditions VALUES ( '2018-01-01 09:20:00-08', 'SFO', 55); INSERT INTO conditions VALUES ( '2018-01-02 09:30:00-08', 'por', 100); INSERT INTO conditions VALUES ( '2018-01-02 09:20:00-08', 'SFO', 65); INSERT INTO conditions VALUES ( '2018-01-02 09:10:00-08', 'NYC', 65); INSERT INTO conditions VALUES ( '2018-11-01 09:20:00-08', 'NYC', 45); INSERT INTO conditions VALUES ( '2018-11-01 10:40:00-08', 'NYC', 55); INSERT INTO conditions VALUES ( '2018-11-01 11:50:00-08', 'NYC', 65); INSERT INTO conditions VALUES ( '2018-11-01 12:10:00-08', 'NYC', 75); INSERT INTO conditions VALUES ( '2018-11-01 13:10:00-08', 'NYC', 85); INSERT INTO conditions VALUES ( '2018-11-02 09:20:00-08', 'NYC', 10); INSERT INTO conditions VALUES ( '2018-11-02 10:30:00-08', 'NYC', 20); CREATE MATERIALIZED VIEW conditions_daily WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT location, time_bucket(INTERVAL '1 day', time) AS bucket, AVG(temperature) FROM conditions GROUP BY location, bucket WITH NO DATA; CREATE MATERIALIZED VIEW conditions_weekly WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT location, time_bucket(INTERVAL '7 day', bucket) AS bucket, AVG(avg) FROM conditions_daily GROUP BY 1, 2 WITH NO DATA; SELECT format('%I.%I', '_timescaledb_internal', h.table_name) AS "MAT_TABLE_NAME", format('%I.%I', '_timescaledb_internal', partial_view_name) AS "PART_VIEW_NAME", format('%I.%I', '_timescaledb_internal', direct_view_name) AS "DIRECT_VIEW_NAME" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) WHERE user_view_name = 'conditions_daily' \gset -- Show both the columns and the view definitions to see that -- references are correct in the view as well. SELECT * FROM test.show_columns('conditions_daily'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f bucket | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'DIRECT_VIEW_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f bucket | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'PART_VIEW_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f bucket | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'MAT_TABLE_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f bucket | timestamp with time zone | t avg | double precision | f ALTER MATERIALIZED VIEW conditions_daily RENAME COLUMN bucket to "time"; -- Show both the columns and the view definitions to see that -- references are correct in the view as well. SELECT * FROM test.show_columns(' conditions_daily'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'DIRECT_VIEW_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'PART_VIEW_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'MAT_TABLE_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | t avg | double precision | f -- This will rebuild the materialized view and should succeed. ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only = false); -- Refresh the continuous aggregate to check that it works after the -- rename. \set VERBOSITY verbose CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); \set VERBOSITY terse -- Rename another column after the flip and verify toggling back and -- forth still works. This exercises the rename when the user view -- already has a UNION ALL query (materialized_only = false). ALTER MATERIALIZED VIEW conditions_daily RENAME COLUMN avg TO average; SELECT * FROM test.show_columns('conditions_daily'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | f average | double precision | f SELECT * FROM test.show_columns(:'DIRECT_VIEW_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | f average | double precision | f SELECT * FROM test.show_columns(:'MAT_TABLE_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | t average | double precision | f ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only = true); ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only = false); -- Verify data is still accessible after multiple renames and toggles. SELECT * FROM conditions_daily ORDER BY location COLLATE "C", time; location | time | average ----------+------------------------------+--------- NYC | Mon Jan 01 16:00:00 2018 UTC | 65 NYC | Wed Oct 31 16:00:00 2018 UTC | 65 NYC | Thu Nov 01 16:00:00 2018 UTC | 15 SFO | Sun Dec 31 16:00:00 2017 UTC | 55 SFO | Mon Jan 01 16:00:00 2018 UTC | 65 por | Mon Jan 01 16:00:00 2018 UTC | 100 -- check hierarchical continuous aggregate still works after renames and toggles on the underlying cagg ALTER MATERIALIZED VIEW conditions_weekly SET (timescaledb.materialized_only = false); ALTER MATERIALIZED VIEW conditions_weekly SET (timescaledb.materialized_only = true); SELECT * FROM conditions_weekly ORDER BY location COLLATE "C", bucket; location | bucket | avg ----------+--------+----- -- Verify that direct rename on the materialization hypertable is blocked. \set ON_ERROR_STOP 0 ALTER TABLE :MAT_TABLE_NAME RENAME COLUMN average TO avg; ERROR: renaming columns on materialization tables is not supported \set ON_ERROR_STOP 1 -- Rename back so subsequent tests that reference "avg" still work. ALTER MATERIALIZED VIEW conditions_daily RENAME COLUMN average TO avg; -- -- Indexes on continuous aggregate -- \set ON_ERROR_STOP 0 -- unique indexes are not supported CREATE UNIQUE INDEX index_unique_error ON conditions_daily ("time", location); ERROR: continuous aggregates do not support UNIQUE indexes -- concurrently index creation not supported CREATE INDEX CONCURRENTLY index_concurrently_avg ON conditions_daily (avg); ERROR: hypertables do not support concurrent index creation \set ON_ERROR_STOP 1 CREATE INDEX index_avg ON conditions_daily (avg); CREATE INDEX index_avg_only ON ONLY conditions_daily (avg); CREATE INDEX index_avg_include ON conditions_daily (avg) INCLUDE (location); CREATE INDEX index_avg_expr ON conditions_daily ((avg + 1)); CREATE INDEX index_avg_location_sfo ON conditions_daily (avg) WHERE location = 'SFO'; CREATE INDEX index_avg_expr_location_sfo ON conditions_daily ((avg + 2)) WHERE location = 'SFO'; SELECT * FROM test.show_indexespred(:'MAT_TABLE_NAME'); Index | Columns | Expr | Pred | Unique | Primary | Exclusion | Tablespace -----------------------------------------------------------------------+-------------------+---------------------------+------------------------+--------+---------+-----------+------------ _timescaledb_internal._materialized_hypertable_35_bucket_idx | {bucket} | | | f | f | f | _timescaledb_internal._materialized_hypertable_35_location_bucket_idx | {location,bucket} | | | f | f | f | _timescaledb_internal.index_avg | {avg} | | | f | f | f | _timescaledb_internal.index_avg_expr | {expr} | avg + 1::double precision | | f | f | f | _timescaledb_internal.index_avg_expr_location_sfo | {expr} | avg + 2::double precision | location = 'SFO'::text | f | f | f | _timescaledb_internal.index_avg_include | {avg,location} | | | f | f | f | _timescaledb_internal.index_avg_location_sfo | {avg} | | location = 'SFO'::text | f | f | f | _timescaledb_internal.index_avg_only | {avg} | | | f | f | f | -- #3696 assertion failure when referencing columns not present in result CREATE TABLE i3696(time timestamptz NOT NULL, search_query text, cnt integer, cnt2 integer); SELECT table_name FROM create_hypertable('i3696','time'); table_name ------------ i3696 CREATE MATERIALIZED VIEW i3696_cagg1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT search_query,count(search_query) as count, sum(cnt), time_bucket(INTERVAL '1 minute', time) AS bucket FROM i3696 GROUP BY cnt +cnt2 , bucket, search_query; NOTICE: continuous aggregate "i3696_cagg1" is already up-to-date ALTER MATERIALIZED VIEW i3696_cagg1 SET (timescaledb.materialized_only = 'true'); CREATE MATERIALIZED VIEW i3696_cagg2 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT search_query,count(search_query) as count, sum(cnt), time_bucket(INTERVAL '1 minute', time) AS bucket FROM i3696 GROUP BY cnt + cnt2, bucket, search_query HAVING cnt + cnt2 + sum(cnt) > 2 or count(cnt2) > 10; NOTICE: continuous aggregate "i3696_cagg2" is already up-to-date ALTER MATERIALIZED VIEW i3696_cagg2 SET (timescaledb.materialized_only = 'true'); --TEST test with multiple settings on continuous aggregates -- -- test for materialized_only + compress combinations (real time aggs enabled initially) CREATE TABLE test_setting(time timestamptz not null, val numeric); SELECT create_hypertable('test_setting', 'time'); create_hypertable ---------------------------- (40,public,test_setting,t) CREATE MATERIALIZED VIEW test_setting_cagg with (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h',time), avg(val), count(*) FROM test_setting GROUP BY 1; NOTICE: continuous aggregate "test_setting_cagg" is already up-to-date INSERT INTO test_setting SELECT generate_series( '2020-01-10 8:00'::timestamp, '2020-01-30 10:00+00'::timestamptz, '1 day'::interval), 10.0; CALL refresh_continuous_aggregate('test_setting_cagg', NULL, '2020-05-30 10:00+00'::timestamptz); SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 --this row is not in the materialized result --- INSERT INTO test_setting VALUES( '2020-11-01', 20); --try out 2 settings here -- ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'true', timescaledb.compress='true'); NOTICE: defaulting compress_orderby to time_bucket SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | t | t --real time aggs is off now , should return 20 -- SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 --now set it back to false -- ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'false', timescaledb.compress='true'); NOTICE: defaulting compress_orderby to time_bucket SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | t | f --count should return additional data since we have real time aggs on SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 21 ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'true', timescaledb.compress='false'); SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | f | t --real time aggs is off now , should return 20 -- SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'false', timescaledb.compress='false'); SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | f | f --count should return additional data since we have real time aggs on SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 21 DELETE FROM test_setting WHERE val = 20; --TEST test with multiple settings on continuous aggregates with real time aggregates turned off initially -- -- test for materialized_only + compress combinations (real time aggs enabled initially) DROP MATERIALIZED VIEW test_setting_cagg; NOTICE: drop cascades to table _timescaledb_internal._hyper_41_50_chunk CREATE MATERIALIZED VIEW test_setting_cagg with (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT time_bucket('1h',time), avg(val), count(*) FROM test_setting GROUP BY 1; NOTICE: refreshing continuous aggregate "test_setting_cagg" CALL refresh_continuous_aggregate('test_setting_cagg', NULL, '2020-05-30 10:00+00'::timestamptz); SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 --this row is not in the materialized result --- INSERT INTO test_setting VALUES( '2020-11-01', 20); --try out 2 settings here -- ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'false', timescaledb.compress='true'); NOTICE: defaulting compress_orderby to time_bucket SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | t | f --count should return additional data since we have real time aggs on SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 21 --now set it back to false -- ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'true', timescaledb.compress='true'); NOTICE: defaulting compress_orderby to time_bucket SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | t | t --real time aggs is off now , should return 20 -- SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'false', timescaledb.compress='false'); SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | f | f --count should return additional data since we have real time aggs on SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 21 ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'true', timescaledb.compress='false'); SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | f | t --real time aggs is off now , should return 20 -- SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 -- END TEST with multiple settings -- Test View Target Entries that contain both aggrefs and Vars in the same expression CREATE TABLE transactions ( "time" timestamp with time zone NOT NULL, dummy1 integer, dummy2 integer, dummy3 integer, dummy4 integer, dummy5 integer, amount integer, fiat_value integer ); SELECT create_hypertable('transactions', 'time'); create_hypertable ---------------------------- (45,public,transactions,t) INSERT INTO transactions VALUES ( '2018-01-01 09:20:00-08', 0, 0, 0, 0, 0, 1, 10); INSERT INTO transactions VALUES ( '2018-01-02 09:30:00-08', 0, 0, 0, 0, 0, -1, 10); INSERT INTO transactions VALUES ( '2018-01-02 09:20:00-08', 0, 0, 0, 0, 0, -1, 10); INSERT INTO transactions VALUES ( '2018-01-02 09:10:00-08', 0, 0, 0, 0, 0, -1, 10); INSERT INTO transactions VALUES ( '2018-11-01 09:20:00-08', 0, 0, 0, 0, 0, 1, 10); INSERT INTO transactions VALUES ( '2018-11-01 10:40:00-08', 0, 0, 0, 0, 0, 1, 10); INSERT INTO transactions VALUES ( '2018-11-01 11:50:00-08', 0, 0, 0, 0, 0, 1, 10); INSERT INTO transactions VALUES ( '2018-11-01 12:10:00-08', 0, 0, 0, 0, 0, -1, 10); INSERT INTO transactions VALUES ( '2018-11-01 13:10:00-08', 0, 0, 0, 0, 0, -1, 10); INSERT INTO transactions VALUES ( '2018-11-02 09:20:00-08', 0, 0, 0, 0, 0, 1, 10); INSERT INTO transactions VALUES ( '2018-11-02 10:30:00-08', 0, 0, 0, 0, 0, -1, 10); CREATE materialized view cashflows( bucket, amount, cashflow, cashflow2 ) WITH ( timescaledb.continuous, timescaledb.materialized_only = true ) AS SELECT time_bucket ('1 day', time) AS bucket, amount, CASE WHEN amount < 0 THEN (0 - sum(fiat_value)) ELSE sum(fiat_value) END AS cashflow, amount + sum(fiat_value) FROM transactions GROUP BY bucket, amount; NOTICE: refreshing continuous aggregate "cashflows" SELECT h.table_name AS "MAT_TABLE_NAME", partial_view_name AS "PART_VIEW_NAME", direct_view_name AS "DIRECT_VIEW_NAME" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) WHERE user_view_name = 'cashflows' \gset -- Show both the columns and the view definitions to see that -- references are correct in the view as well. \d+ "_timescaledb_internal".:"DIRECT_VIEW_NAME" View "_timescaledb_internal._direct_view_46" Column | Type | Collation | Nullable | Default | Storage | Description -----------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | amount | integer | | | | plain | cashflow | bigint | | | | plain | cashflow2 | bigint | | | | plain | View definition: SELECT time_bucket('@ 1 day'::interval, transactions."time") AS bucket, transactions.amount, CASE WHEN transactions.amount < 0 THEN 0 - sum(transactions.fiat_value) ELSE sum(transactions.fiat_value) END AS cashflow, transactions.amount + sum(transactions.fiat_value) AS cashflow2 FROM transactions GROUP BY (time_bucket('@ 1 day'::interval, transactions."time")), transactions.amount; \d+ "_timescaledb_internal".:"PART_VIEW_NAME" View "_timescaledb_internal._partial_view_46" Column | Type | Collation | Nullable | Default | Storage | Description -----------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | amount | integer | | | | plain | cashflow | bigint | | | | plain | cashflow2 | bigint | | | | plain | View definition: SELECT time_bucket('@ 1 day'::interval, transactions."time") AS bucket, transactions.amount, CASE WHEN transactions.amount < 0 THEN 0 - sum(transactions.fiat_value) ELSE sum(transactions.fiat_value) END AS cashflow, transactions.amount + sum(transactions.fiat_value) AS cashflow2 FROM transactions GROUP BY (time_bucket('@ 1 day'::interval, transactions."time")), transactions.amount; \d+ "_timescaledb_internal".:"MAT_TABLE_NAME" Table "_timescaledb_internal._materialized_hypertable_46" Column | Type | Collation | Nullable | Default | Storage | Stats target | Description -----------+--------------------------+-----------+----------+---------+---------+--------------+------------- bucket | timestamp with time zone | | not null | | plain | | amount | integer | | | | plain | | cashflow | bigint | | | | plain | | cashflow2 | bigint | | | | plain | | Indexes: "_materialized_hypertable_46_amount_bucket_idx" btree (amount, bucket DESC) "_materialized_hypertable_46_bucket_idx" btree (bucket DESC) Child tables: _timescaledb_internal._hyper_46_55_chunk, _timescaledb_internal._hyper_46_56_chunk \d+ 'cashflows' View "public.cashflows" Column | Type | Collation | Nullable | Default | Storage | Description -----------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | amount | integer | | | | plain | cashflow | bigint | | | | plain | cashflow2 | bigint | | | | plain | View definition: SELECT _materialized_hypertable_46.bucket, _materialized_hypertable_46.amount, _materialized_hypertable_46.cashflow, _materialized_hypertable_46.cashflow2 FROM _timescaledb_internal._materialized_hypertable_46; SELECT * FROM cashflows ORDER BY cashflows; bucket | amount | cashflow | cashflow2 ------------------------------+--------+----------+----------- Sun Dec 31 16:00:00 2017 UTC | 1 | 10 | 11 Mon Jan 01 16:00:00 2018 UTC | -1 | -30 | 29 Wed Oct 31 16:00:00 2018 UTC | -1 | -20 | 19 Wed Oct 31 16:00:00 2018 UTC | 1 | 30 | 31 Thu Nov 01 16:00:00 2018 UTC | -1 | -10 | 9 Thu Nov 01 16:00:00 2018 UTC | 1 | 10 | 11 -- test cagg creation with named arguments in time_bucket -- note that positional arguments cannot follow named arguments -- 1. test named origin -- 2. test named timezone -- 3. test named ts -- 4. test named bucket width -- named origin CREATE MATERIALIZED VIEW cagg_named_origin WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h', time, 'UTC', origin => '2001-01-03 01:23:45') AS bucket, avg(amount) as avg_amount FROM transactions GROUP BY 1 WITH NO DATA; -- named timezone CREATE MATERIALIZED VIEW cagg_named_tz_origin WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h', time, timezone => 'UTC', origin => '2001-01-03 01:23:45') AS bucket, avg(amount) as avg_amount FROM transactions GROUP BY 1 WITH NO DATA; -- named ts CREATE MATERIALIZED VIEW cagg_named_ts_tz_origin WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h', ts => time, timezone => 'UTC', origin => '2001-01-03 01:23:45') AS bucket, avg(amount) as avg_amount FROM transactions GROUP BY 1 WITH NO DATA; -- named bucket width CREATE MATERIALIZED VIEW cagg_named_all WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(bucket_width => '1h', ts => time, timezone => 'UTC', origin => '2001-01-03 01:23:45') AS bucket, avg(amount) as avg_amount FROM transactions GROUP BY 1 WITH NO DATA; -- Refreshing from the beginning (NULL) of a CAGG with variable time bucket and -- using an INTERVAL for the end timestamp (issue #5534) CREATE MATERIALIZED VIEW transactions_montly WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT time_bucket(INTERVAL '1 month', time) AS bucket, SUM(fiat_value), MAX(fiat_value), MIN(fiat_value) FROM transactions GROUP BY 1 WITH NO DATA; -- No rows SELECT * FROM transactions_montly ORDER BY bucket; bucket | sum | max | min --------+-----+-----+----- -- Refresh from beginning of the CAGG for 1 month CALL refresh_continuous_aggregate('transactions_montly', NULL, INTERVAL '1 month'); SELECT * FROM transactions_montly ORDER BY bucket; bucket | sum | max | min ------------------------------+-----+-----+----- Sun Dec 31 16:00:00 2017 UTC | 40 | 10 | 10 Wed Oct 31 16:00:00 2018 UTC | 70 | 10 | 10 TRUNCATE transactions_montly; -- Partial refresh the CAGG from beginning to an specific timestamp CALL refresh_continuous_aggregate('transactions_montly', NULL, '2018-11-01 11:50:00-08'::timestamptz); SELECT * FROM transactions_montly ORDER BY bucket; bucket | sum | max | min ------------------------------+-----+-----+----- Sun Dec 31 16:00:00 2017 UTC | 40 | 10 | 10 -- Full refresh the CAGG CALL refresh_continuous_aggregate('transactions_montly', NULL, NULL); SELECT * FROM transactions_montly ORDER BY bucket; bucket | sum | max | min ------------------------------+-----+-----+----- Sun Dec 31 16:00:00 2017 UTC | 40 | 10 | 10 Wed Oct 31 16:00:00 2018 UTC | 70 | 10 | 10 -- Check set_chunk_time_interval on continuous aggregate CREATE MATERIALIZED VIEW cagg_set_chunk_time_interval WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(INTERVAL '1 month', time) AS bucket, SUM(fiat_value), MAX(fiat_value), MIN(fiat_value) FROM transactions GROUP BY 1 WITH NO DATA; SELECT set_chunk_time_interval('cagg_set_chunk_time_interval', chunk_time_interval => interval '1 month'); set_chunk_time_interval ------------------------- CALL refresh_continuous_aggregate('cagg_set_chunk_time_interval', NULL, NULL); SELECT _timescaledb_functions.to_interval(d.interval_length) = interval '1 month' FROM _timescaledb_catalog.dimension d RIGHT JOIN _timescaledb_catalog.continuous_agg ca ON ca.user_view_name = 'cagg_set_chunk_time_interval' WHERE d.hypertable_id = ca.mat_hypertable_id; ?column? ---------- t -- Since #6077 CAggs are materialized only by default DROP TABLE conditions CASCADE; NOTICE: drop cascades to 5 other objects NOTICE: drop cascades to 2 other objects CREATE TABLE conditions ( time TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL ); SELECT create_hypertable('conditions', 'time'); create_hypertable -------------------------- (53,public,conditions,t) INSERT INTO conditions VALUES ( '2018-01-01 09:20:00-08', 'SFO', 55); INSERT INTO conditions VALUES ( '2018-01-02 09:30:00-08', 'POR', 100); INSERT INTO conditions VALUES ( '2018-01-02 09:20:00-08', 'SFO', 65); INSERT INTO conditions VALUES ( '2018-01-02 09:10:00-08', 'NYC', 65); INSERT INTO conditions VALUES ( '2018-11-01 09:20:00-08', 'NYC', 45); INSERT INTO conditions VALUES ( '2018-11-01 10:40:00-08', 'NYC', 55); INSERT INTO conditions VALUES ( '2018-11-01 11:50:00-08', 'NYC', 65); INSERT INTO conditions VALUES ( '2018-11-01 12:10:00-08', 'NYC', 75); INSERT INTO conditions VALUES ( '2018-11-01 13:10:00-08', 'NYC', 85); INSERT INTO conditions VALUES ( '2018-11-02 09:20:00-08', 'NYC', 10); INSERT INTO conditions VALUES ( '2018-11-02 10:30:00-08', 'NYC', 20); CREATE MATERIALIZED VIEW conditions_daily WITH (timescaledb.continuous) AS SELECT location, time_bucket(INTERVAL '1 day', time) AS bucket, AVG(temperature) FROM conditions GROUP BY location, bucket WITH NO DATA; \d+ conditions_daily View "public.conditions_daily" Column | Type | Collation | Nullable | Default | Storage | Description ----------+--------------------------+-----------+----------+---------+----------+------------- location | text | | | | extended | bucket | timestamp with time zone | | | | plain | avg | double precision | | | | plain | View definition: SELECT _materialized_hypertable_54.location, _materialized_hypertable_54.bucket, _materialized_hypertable_54.avg FROM _timescaledb_internal._materialized_hypertable_54; -- Should return NO ROWS SELECT * FROM conditions_daily ORDER BY bucket, location; location | bucket | avg ----------+--------+----- ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only=false); \d+ conditions_daily View "public.conditions_daily" Column | Type | Collation | Nullable | Default | Storage | Description ----------+--------------------------+-----------+----------+---------+----------+------------- location | text | | | | extended | bucket | timestamp with time zone | | | | plain | avg | double precision | | | | plain | View definition: SELECT _materialized_hypertable_54.location, _materialized_hypertable_54.bucket, _materialized_hypertable_54.avg FROM _timescaledb_internal._materialized_hypertable_54 WHERE _materialized_hypertable_54.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(54)), '-infinity'::timestamp with time zone) UNION ALL SELECT conditions.location, time_bucket('@ 1 day'::interval, conditions."time") AS bucket, avg(conditions.temperature) AS avg FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(54)), '-infinity'::timestamp with time zone) GROUP BY conditions.location, (time_bucket('@ 1 day'::interval, conditions."time")); -- Should return ROWS because now it is realtime SELECT * FROM conditions_daily ORDER BY bucket, location; location | bucket | avg ----------+------------------------------+----- SFO | Sun Dec 31 16:00:00 2017 UTC | 55 NYC | Mon Jan 01 16:00:00 2018 UTC | 65 POR | Mon Jan 01 16:00:00 2018 UTC | 100 SFO | Mon Jan 01 16:00:00 2018 UTC | 65 NYC | Wed Oct 31 16:00:00 2018 UTC | 65 NYC | Thu Nov 01 16:00:00 2018 UTC | 15 -- Should return ROWS because we refreshed it ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only=true); \d+ conditions_daily View "public.conditions_daily" Column | Type | Collation | Nullable | Default | Storage | Description ----------+--------------------------+-----------+----------+---------+----------+------------- location | text | | | | extended | bucket | timestamp with time zone | | | | plain | avg | double precision | | | | plain | View definition: SELECT _materialized_hypertable_54.location, _materialized_hypertable_54.bucket, _materialized_hypertable_54.avg FROM _timescaledb_internal._materialized_hypertable_54; CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); SELECT * FROM conditions_daily ORDER BY bucket, location; location | bucket | avg ----------+------------------------------+----- SFO | Sun Dec 31 16:00:00 2017 UTC | 55 NYC | Mon Jan 01 16:00:00 2018 UTC | 65 POR | Mon Jan 01 16:00:00 2018 UTC | 100 SFO | Mon Jan 01 16:00:00 2018 UTC | 65 NYC | Wed Oct 31 16:00:00 2018 UTC | 65 NYC | Thu Nov 01 16:00:00 2018 UTC | 15 -- Test TRUNCATE over a Realtime CAgg DROP MATERIALIZED VIEW conditions_daily; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW conditions_daily WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT location, time_bucket(INTERVAL '1 day', time) AS bucket, AVG(temperature) FROM conditions GROUP BY location, bucket WITH NO DATA; SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'conditions_daily' \gset -- Check the current watermark for an empty CAgg SELECT _timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)) AS watermak_empty_cagg; watermak_empty_cagg --------------------------------- Sun Nov 23 16:00:00 4714 UTC BC -- Refresh the CAGG CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); -- Check the watermark after the refresh and before truncate the CAgg SELECT _timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)) AS watermak_before; watermak_before ------------------------------ Fri Nov 02 16:00:00 2018 UTC -- Exists chunks before truncate the cagg (> 0) SELECT count(*) FROM show_chunks('conditions_daily'); count ------- 2 -- Truncate the given CAgg, it should reset the watermark to the empty state TRUNCATE conditions_daily; -- No chunks remains after truncate the cagg (= 0) SELECT count(*) FROM show_chunks('conditions_daily'); count ------- 0 -- Watermark should be reseted SELECT _timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)) AS watermak_after; watermak_after --------------------------------- Sun Nov 23 16:00:00 4714 UTC BC -- Should return ROWS because the watermark was reseted by the TRUNCATE SELECT * FROM conditions_daily ORDER BY bucket, location; location | bucket | avg ----------+------------------------------+----- SFO | Sun Dec 31 16:00:00 2017 UTC | 55 NYC | Mon Jan 01 16:00:00 2018 UTC | 65 POR | Mon Jan 01 16:00:00 2018 UTC | 100 SFO | Mon Jan 01 16:00:00 2018 UTC | 65 NYC | Wed Oct 31 16:00:00 2018 UTC | 65 NYC | Thu Nov 01 16:00:00 2018 UTC | 15 -- check compression settings are cleaned up when deleting a cagg with compression CREATE TABLE cagg_cleanup(time timestamptz not null); SELECT table_name FROM create_hypertable('cagg_cleanup','time'); table_name -------------- cagg_cleanup INSERT INTO cagg_cleanup SELECT '2020-01-01'; CREATE MATERIALIZED VIEW cagg1 WITH (timescaledb.continuous) AS SELECT time_bucket('1h',time) FROM cagg_cleanup GROUP BY 1; NOTICE: refreshing continuous aggregate "cagg1" ALTER MATERIALIZED VIEW cagg1 SET (timescaledb.compress); NOTICE: defaulting compress_orderby to time_bucket SELECT count(compress_chunk(ch)) FROM show_chunks('cagg1') ch; count ------- 1 DROP MATERIALIZED VIEW cagg1; NOTICE: drop cascades to table _timescaledb_internal._hyper_57_70_chunk SELECT * FROM _timescaledb_catalog.compression_settings; relid | compress_relid | segmentby | orderby | orderby_desc | orderby_nullsfirst | index -------+----------------+-----------+---------+--------------+--------------------+------- -- test WITH namespace alias CREATE TABLE with_alias(time timestamptz not null); CREATE MATERIALIZED VIEW cagg_alias WITH (tsdb.continuous, tsdb.materialized_only=false) AS SELECT time_bucket(INTERVAL '1 day', time) FROM conditions GROUP BY 1 WITH NO DATA; ALTER MATERIALIZED VIEW cagg_alias SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_alias SET (tsdb.materialized_only=false); DROP MATERIALIZED VIEW cagg_alias; -- test SET chunk_time_interval CREATE MATERIALIZED VIEW cagg_set WITH (tsdb.continuous, tsdb.chunk_interval='1day') AS SELECT time_bucket(INTERVAL '1 day', time) AS cagg_interval_setter FROM conditions GROUP BY 1 WITH NO DATA; SELECT column_name, time_interval FROM timescaledb_information.dimensions WHERE column_name='cagg_interval_setter'; column_name | time_interval ----------------------+--------------- cagg_interval_setter | @ 1 day ALTER MATERIALIZED VIEW cagg_set SET (tsdb.chunk_interval='23 day'); SELECT column_name, time_interval FROM timescaledb_information.dimensions WHERE column_name='cagg_interval_setter'; column_name | time_interval ----------------------+--------------- cagg_interval_setter | @ 23 days ALTER MATERIALIZED VIEW cagg_set SET (tsdb.chunk_interval='6 month'); SELECT column_name, time_interval FROM timescaledb_information.dimensions WHERE column_name='cagg_interval_setter'; column_name | time_interval ----------------------+--------------- cagg_interval_setter | @ 180 days ALTER MATERIALIZED VIEW cagg_set SET (tsdb.chunk_interval='1 year'); SELECT column_name, time_interval FROM timescaledb_information.dimensions WHERE column_name='cagg_interval_setter'; column_name | time_interval ----------------------+--------------- cagg_interval_setter | @ 360 days -- test cagg with stable functions CREATE MATERIALIZED VIEW cagg_stable WITH (tsdb.continuous) AS SELECT sum(temperature), max(time + INTERVAL '1h') FROM conditions GROUP BY time_bucket('1week', time), location; WARNING: using non-immutable functions in continuous aggregate view may lead to inconsistent results on rematerialization NOTICE: refreshing continuous aggregate "cagg_stable" SELECT * FROM cagg_stable t ORDER BY t; sum | max -----+------------------------------ 65 | Tue Jan 02 10:10:00 2018 UTC 100 | Tue Jan 02 10:30:00 2018 UTC 120 | Tue Jan 02 10:20:00 2018 UTC 355 | Fri Nov 02 11:30:00 2018 UTC --aggregate without combine function but stable function CREATE MATERIALIZED VIEW cagg_json_agg WITH (tsdb.continuous, tsdb.materialized_only=false) AS SELECT json_agg(location) from conditions group by time_bucket('1week', time), location WITH NO DATA; WARNING: using non-immutable functions in continuous aggregate view may lead to inconsistent results on rematerialization CREATE FUNCTION test_stablefunc(int) RETURNS int LANGUAGE 'sql' STABLE AS 'SELECT $1 + 10'; CREATE MATERIALIZED VIEW cagg_stable2 WITH (tsdb.continuous) AS SELECT sum(test_stablefunc(temperature::int)), min(location) FROM conditions GROUP BY time_bucket('1week', time) WITH NO DATA; WARNING: using non-immutable functions in continuous aggregate view may lead to inconsistent results on rematerialization CREATE MATERIALIZED VIEW cagg_stable3 WITH (tsdb.continuous) AS SELECT sum(temperature), min(location) FROM conditions GROUP BY time_bucket('1week', time), test_stablefunc(temperature::int) WITH NO DATA; WARNING: using non-immutable functions in continuous aggregate view may lead to inconsistent results on rematerialization -- test window functions in caggs -- first do sanity check that we error without the GUC \set ON_ERROR_STOP 0 CREATE MATERIALIZED VIEW cagg_window WITH (tsdb.continuous) AS SELECT time_bucket('1week', time), rank() OVER (PARTITION BY time_bucket('1 week',time)) FROM conditions GROUP BY 1; ERROR: invalid continuous aggregate query \set ON_ERROR_STOP 1 SET timescaledb.enable_cagg_window_functions TO on; CREATE MATERIALIZED VIEW cagg_window_1 WITH (tsdb.continuous) AS SELECT time_bucket('1week', time), rank() OVER (PARTITION BY time_bucket('1 week',time)) FROM conditions GROUP BY 1; WARNING: window function support is experimental and may result in unexpected results depending on the functions used. NOTICE: refreshing continuous aggregate "cagg_window_1" CREATE MATERIALIZED VIEW cagg_window_2 WITH (tsdb.continuous) AS SELECT time_bucket('1week', time), rank() OVER (PARTITION BY time_bucket('1 week',time), location) FROM conditions GROUP BY 1, location; WARNING: window function support is experimental and may result in unexpected results depending on the functions used. NOTICE: refreshing continuous aggregate "cagg_window_2" CREATE MATERIALIZED VIEW cagg_window_3 WITH (tsdb.continuous) AS SELECT time_bucket('1week', time), rank() OVER (PARTITION BY time_bucket('1 week',time)) FROM conditions GROUP BY 1, location; WARNING: window function support is experimental and may result in unexpected results depending on the functions used. NOTICE: refreshing continuous aggregate "cagg_window_3" CREATE MATERIALIZED VIEW cagg_window_4 WITH (tsdb.continuous) AS SELECT time_bucket('1week', time), rank() OVER w FROM conditions GROUP BY 1, location WINDOW w AS (PARTITION BY time_bucket('1 week',time)); WARNING: window function support is experimental and may result in unexpected results depending on the functions used. NOTICE: refreshing continuous aggregate "cagg_window_4" -- test setting chunk_interval on a cagg CREATE MATERIALIZED VIEW cagg_chunk_interval WITH (tsdb.continuous, tsdb.chunk_interval='1000 day') AS SELECT time_bucket('1 week', time) FROM conditions GROUP BY 1 WITH NO DATA; SELECT time_interval from timescaledb_information.continuous_aggregates cagg INNER JOIN timescaledb_information.dimensions dim ON cagg.materialization_hypertable_name = dim.hypertable_name WHERE view_name='cagg_chunk_interval'; time_interval --------------- @ 1000 days ALTER MATERIALIZED VIEW cagg_chunk_interval SET (tsdb.chunk_interval='110 day'); SELECT time_interval from timescaledb_information.continuous_aggregates cagg INNER JOIN timescaledb_information.dimensions dim ON cagg.materialization_hypertable_name = dim.hypertable_name WHERE view_name='cagg_chunk_interval'; time_interval --------------- @ 110 days -- test columnstore options CREATE MATERIALIZED VIEW columnstore_options WITH (tsdb.continuous, tsdb.chunk_interval='1 day') AS SELECT time_bucket('1 day', time) FROM conditions GROUP BY 1 WITH NO DATA; SELECT column_name, compress_interval_length from _timescaledb_catalog.dimension where column_name='time_bucket' ORDER BY id DESC LIMIT 1; column_name | compress_interval_length -------------+-------------------------- time_bucket | ALTER MATERIALIZED VIEW columnstore_options SET (tsdb.columnstore, tsdb.chunk_interval='1 day', tsdb.orderby='time_bucket DESC', tsdb.compress_chunk_interval='2 day'); SELECT column_name, compress_interval_length from _timescaledb_catalog.dimension where column_name='time_bucket' ORDER BY id DESC LIMIT 1; column_name | compress_interval_length -------------+-------------------------- time_bucket | 172800000000 ALTER MATERIALIZED VIEW columnstore_options SET (tsdb.compress_chunk_interval='3 day'); SELECT column_name, compress_interval_length from _timescaledb_catalog.dimension where column_name='time_bucket' ORDER BY id DESC LIMIT 1; column_name | compress_interval_length -------------+-------------------------- time_bucket | 259200000000 ALTER MATERIALIZED VIEW columnstore_options SET (tsdb.compress_chunk_time_interval='4 day'); SELECT column_name, compress_interval_length from _timescaledb_catalog.dimension where column_name='time_bucket' ORDER BY id DESC LIMIT 1; column_name | compress_interval_length -------------+-------------------------- time_bucket | 345600000000 -- test set returning functions in caggs CREATE TABLE kpis_raw (time TIMESTAMP NOT NULL, value INTEGER, groups TEXT[]) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column INSERT INTO kpis_raw (time, value, groups) VALUES ('2025-01-01', 10, '{group1,group2,group3}'), ('2025-01-02', 20, '{group1,group4}'), ('2025-02-01', 10, '{group1,group3}'), ('2025-02-01', 20, '{group1,group4}'); CREATE MATERIALIZED VIEW kpis_cagg WITH (tsdb.continuous) AS SELECT time_bucket('7 day', time) AS bucket, count(*) AS number_of_records, avg(value) AS average, unnest(groups) AS kpi_group FROM kpis_raw GROUP BY bucket, kpi_group; NOTICE: refreshing continuous aggregate "kpis_cagg" SELECT * FROM kpis_cagg ORDER BY bucket, kpi_group; bucket | number_of_records | average | kpi_group --------------------------+-------------------+---------------------+----------- Mon Dec 30 00:00:00 2024 | 2 | 15.0000000000000000 | group1 Mon Dec 30 00:00:00 2024 | 1 | 10.0000000000000000 | group2 Mon Dec 30 00:00:00 2024 | 1 | 10.0000000000000000 | group3 Mon Dec 30 00:00:00 2024 | 1 | 20.0000000000000000 | group4 Mon Jan 27 00:00:00 2025 | 2 | 15.0000000000000000 | group1 Mon Jan 27 00:00:00 2025 | 1 | 10.0000000000000000 | group3 Mon Jan 27 00:00:00 2025 | 1 | 20.0000000000000000 | group4 --TEST for caggs with non timescaledb namespace options --non timescaledb namespace options can be set via ALTER \set ON_ERROR_STOP 0 -- will error out CREATE MATERIALIZED VIEW ht_try_weekly WITH (timescaledb.continuous, tigerlake.newoption = true) AS SELECT time_bucket(interval '1 week', time) AS ts_bucket, avg(value) FROM kpis_raw GROUP BY 1; ERROR: non "timescaledb" namespace options can be set only via ALTER --caught by Postgres now ALTER MATERIALIZED VIEW kpis_cagg SET (tigerlake.newoption = true); ERROR: "kpis_cagg" is not a materialized view \set ON_ERROR_STOP 1 -- TEST that cached alter stmt still works (see PR 8739) -- test DDL inside function CREATE TABLE hypertab_ddl( ts timestamp, a integer) WITH (timescaledb.hypertable); NOTICE: using column "ts" as partitioning column CREATE OR REPLACE FUNCTION ddl_function() RETURNS VOID LANGUAGE PLPGSQL AS $$ BEGIN DROP MATERIALIZED VIEW IF EXISTS cagg_hypertab_ddl; CREATE MATERIALIZED VIEW cagg_hypertab_ddl WITH (timescaledb.continuous) AS SELECT time_bucket( '1 day'::interval, ts), COUNT(*) FROM hypertab_ddl GROUP BY 1 WITH NO DATA; END $$; SELECT ddl_function(); NOTICE: materialized view "cagg_hypertab_ddl" does not exist, skipping ddl_function -------------- SELECT view_name from timescaledb_information.continuous_aggregates WHERE hypertable_name='hypertab_ddl'; view_name ------------------- cagg_hypertab_ddl SELECT ddl_function(); ddl_function -------------- SELECT view_name from timescaledb_information.continuous_aggregates WHERE hypertable_name='hypertab_ddl'; view_name ------------------- cagg_hypertab_ddl -- TEST continuous aggregate with functionally dependent columns -- name column depends on id which is in GROUP BY so name doesnt have to be CREATE table sensor(id int PRIMARY KEY, name text); CREATE TABLE sensordata(time timestamptz,id int, value float) WITH (timescaledb.hypertable); NOTICE: using column "time" as partitioning column CREATE MATERIALIZED VIEW cagg_sensordata WITH (tsdb.continuous) AS SELECT s.id, s.name,time_bucket('15 minutes', sd.time) as bucket, avg(sd.value) FROM sensordata sd JOIN sensor s USING(id) GROUP BY s.id, bucket WITH NO DATA; SELECT * FROM cagg_sensordata; id | name | bucket | avg ----+------+--------+----- ================================================ FILE: tsl/test/expected/cagg_ddl-16.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- Set this variable to avoid using a hard-coded path each time query -- results are compared \set QUERY_RESULT_TEST_EQUAL_RELPATH '../../../../test/sql/include/query_result_test_equal.sql' SET ROLE :ROLE_DEFAULT_PERM_USER; --DDL commands on continuous aggregates CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature integer NULL, humidity DOUBLE PRECISION NULL, timemeasure TIMESTAMPTZ, timeinterval INTERVAL ); SELECT table_name FROM create_hypertable('conditions', 'timec'); table_name ------------ conditions -- schema tests \c :TEST_DBNAME :ROLE_SUPERUSER SET timezone TO 'UTC+8'; -- drop if the tablespace1 and/or tablespace2 exists SET client_min_messages TO error; DROP TABLESPACE IF EXISTS tablespace1; DROP TABLESPACE IF EXISTS tablespace2; RESET client_min_messages; CREATE TABLESPACE tablespace1 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE1_PATH; CREATE TABLESPACE tablespace2 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE2_PATH; CREATE SCHEMA rename_schema; GRANT ALL ON SCHEMA rename_schema TO :ROLE_DEFAULT_PERM_USER; CREATE SCHEMA test_schema AUTHORIZATION :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE TABLE foo(time TIMESTAMPTZ NOT NULL, data INTEGER); SELECT create_hypertable('foo', 'time'); create_hypertable ------------------- (2,public,foo,t) CREATE MATERIALIZED VIEW rename_test_old WITH ( timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1week', time), COUNT(data) FROM foo GROUP BY 1 WITH NO DATA; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+-----------------+-----------------------+------------------- public | rename_test_old | _timescaledb_internal | _partial_view_3 ALTER TABLE rename_test_old RENAME TO rename_test; ALTER TABLE rename_test SET SCHEMA test_schema; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+-----------------------+------------------- test_schema | rename_test | _timescaledb_internal | _partial_view_3 ALTER MATERIALIZED VIEW test_schema.rename_test SET SCHEMA rename_schema; DROP SCHEMA test_schema; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+-----------------------+------------------- rename_schema | rename_test | _timescaledb_internal | _partial_view_3 SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA", direct_view_name as "DIR_VIEW_NAME", direct_view_schema as "DIR_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'rename_test' \gset RESET ROLE; SELECT current_user; current_user -------------- super_user ALTER VIEW :"PART_VIEW_SCHEMA".:"PART_VIEW_NAME" SET SCHEMA public; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+---------------------+------------------- rename_schema | rename_test | public | _partial_view_3 --alter direct view schema SELECT user_view_schema, user_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | direct_view_schema | direct_view_name ------------------+----------------+-----------------------+------------------ rename_schema | rename_test | _timescaledb_internal | _direct_view_3 RESET ROLE; SELECT current_user; current_user -------------- super_user ALTER VIEW :"DIR_VIEW_SCHEMA".:"DIR_VIEW_NAME" SET SCHEMA public; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name ------------------+----------------+---------------------+-------------------+--------------------+------------------ rename_schema | rename_test | public | _partial_view_3 | public | _direct_view_3 RESET ROLE; SELECT current_user; current_user -------------- super_user ALTER SCHEMA rename_schema RENAME TO new_name_schema; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name ------------------+----------------+---------------------+-------------------+--------------------+------------------ new_name_schema | rename_test | public | _partial_view_3 | public | _direct_view_3 ALTER VIEW :"PART_VIEW_NAME" SET SCHEMA new_name_schema; ALTER VIEW :"DIR_VIEW_NAME" SET SCHEMA new_name_schema; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name ------------------+----------------+---------------------+-------------------+--------------------+------------------ new_name_schema | rename_test | new_name_schema | _partial_view_3 | new_name_schema | _direct_view_3 RESET ROLE; SELECT current_user; current_user -------------- super_user ALTER SCHEMA new_name_schema RENAME TO foo_name_schema; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+---------------------+------------------- foo_name_schema | rename_test | foo_name_schema | _partial_view_3 ALTER MATERIALIZED VIEW foo_name_schema.rename_test SET SCHEMA public; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+---------------------+------------------- public | rename_test | foo_name_schema | _partial_view_3 RESET ROLE; SELECT current_user; current_user -------------- super_user ALTER SCHEMA foo_name_schema RENAME TO rename_schema; SET ROLE :ROLE_DEFAULT_PERM_USER; SET client_min_messages TO NOTICE; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+---------------------+------------------- public | rename_test | rename_schema | _partial_view_3 ALTER MATERIALIZED VIEW rename_test RENAME TO rename_c_aggregate; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+--------------------+---------------------+------------------- public | rename_c_aggregate | rename_schema | _partial_view_3 SELECT * FROM rename_c_aggregate; time_bucket | count -------------+------- ALTER VIEW rename_schema.:"PART_VIEW_NAME" RENAME TO partial_view; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name ------------------+--------------------+---------------------+-------------------+--------------------+------------------ public | rename_c_aggregate | rename_schema | partial_view | rename_schema | _direct_view_3 --rename direct view ALTER VIEW rename_schema.:"DIR_VIEW_NAME" RENAME TO direct_view; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name ------------------+--------------------+---------------------+-------------------+--------------------+------------------ public | rename_c_aggregate | rename_schema | partial_view | rename_schema | direct_view -- drop_chunks tests DROP TABLE conditions CASCADE; DROP TABLE foo CASCADE; NOTICE: drop cascades to 2 other objects CREATE TABLE drop_chunks_table(time BIGINT NOT NULL, data INTEGER); SELECT hypertable_id AS drop_chunks_table_id FROM create_hypertable('drop_chunks_table', 'time', chunk_time_interval => 10) \gset CREATE OR REPLACE FUNCTION integer_now_test() returns bigint LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), bigint '0') FROM drop_chunks_table $$; SELECT set_integer_now_func('drop_chunks_table', 'integer_now_test'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW drop_chunks_view WITH ( timescaledb.continuous, timescaledb.materialized_only=true ) AS SELECT time_bucket('5', time), COUNT(data) FROM drop_chunks_table GROUP BY 1 WITH NO DATA; SELECT format('%I.%I', schema_name, table_name) AS drop_chunks_mat_table, schema_name AS drop_chunks_mat_schema, table_name AS drop_chunks_mat_table_name FROM _timescaledb_catalog.hypertable, _timescaledb_catalog.continuous_agg WHERE _timescaledb_catalog.continuous_agg.raw_hypertable_id = :drop_chunks_table_id AND _timescaledb_catalog.hypertable.id = _timescaledb_catalog.continuous_agg.mat_hypertable_id \gset -- create 3 chunks, with 3 time bucket INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(0, 29) AS i; -- Only refresh up to bucket 15 initially. Matches the old refresh -- behavior that didn't materialize everything CALL refresh_continuous_aggregate('drop_chunks_view', 0, 15); SELECT count(c) FROM show_chunks('drop_chunks_table') AS c; count ------- 3 SELECT count(c) FROM show_chunks('drop_chunks_view') AS c; count ------- 1 SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | count -------------+------- 0 | 5 5 | 5 10 | 5 -- cannot drop directly from the materialization table without specifying -- cont. aggregate view name explicitly \set ON_ERROR_STOP 0 SELECT drop_chunks(:'drop_chunks_mat_table', newer_than => -20, verbose => true); ERROR: operation not supported on materialized hypertable \set ON_ERROR_STOP 1 SELECT count(c) FROM show_chunks('drop_chunks_table') AS c; count ------- 3 SELECT count(c) FROM show_chunks('drop_chunks_view') AS c; count ------- 1 SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | count -------------+------- 0 | 5 5 | 5 10 | 5 -- drop chunks when the chunksize and time_bucket aren't aligned DROP TABLE drop_chunks_table CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_5_4_chunk CREATE TABLE drop_chunks_table_u(time BIGINT NOT NULL, data INTEGER); SELECT hypertable_id AS drop_chunks_table_u_id FROM create_hypertable('drop_chunks_table_u', 'time', chunk_time_interval => 7) \gset CREATE OR REPLACE FUNCTION integer_now_test1() returns bigint LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), bigint '0') FROM drop_chunks_table_u $$; SELECT set_integer_now_func('drop_chunks_table_u', 'integer_now_test1'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW drop_chunks_view WITH ( timescaledb.continuous, timescaledb.materialized_only=true ) AS SELECT time_bucket('3', time), COUNT(data) FROM drop_chunks_table_u GROUP BY 1 WITH NO DATA; SELECT format('%I.%I', schema_name, table_name) AS drop_chunks_mat_table_u, schema_name AS drop_chunks_mat_schema, table_name AS drop_chunks_mat_table_u_name FROM _timescaledb_catalog.hypertable, _timescaledb_catalog.continuous_agg WHERE _timescaledb_catalog.continuous_agg.raw_hypertable_id = :drop_chunks_table_u_id AND _timescaledb_catalog.hypertable.id = _timescaledb_catalog.continuous_agg.mat_hypertable_id \gset -- create 3 chunks, with 3 time bucket INSERT INTO drop_chunks_table_u SELECT i, i FROM generate_series(0, 21) AS i; -- Refresh up to bucket 15 to match old materializer behavior CALL refresh_continuous_aggregate('drop_chunks_view', 0, 15); SELECT count(c) FROM show_chunks('drop_chunks_table_u') AS c; count ------- 4 SELECT count(c) FROM show_chunks('drop_chunks_view') AS c; count ------- 1 SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | count -------------+------- 0 | 3 3 | 3 6 | 3 9 | 3 12 | 3 -- TRUNCATE test -- Can truncate regular hypertables that have caggs TRUNCATE drop_chunks_table_u; \set ON_ERROR_STOP 0 -- Can't truncate materialized hypertables directly TRUNCATE :drop_chunks_mat_table_u; ERROR: cannot TRUNCATE a hypertable underlying a continuous aggregate \set ON_ERROR_STOP 1 -- Check that we don't interfere with TRUNCATE of normal table and -- partitioned table CREATE TABLE truncate (value int); INSERT INTO truncate VALUES (1), (2); TRUNCATE truncate; SELECT * FROM truncate; value ------- CREATE TABLE truncate_partitioned (value int) PARTITION BY RANGE(value); CREATE TABLE truncate_p1 PARTITION OF truncate_partitioned FOR VALUES FROM (1) TO (3); INSERT INTO truncate_partitioned VALUES (1), (2); TRUNCATE truncate_partitioned; SELECT * FROM truncate_partitioned; value ------- -- ALTER TABLE tests \set ON_ERROR_STOP 0 -- test a variety of ALTER TABLE statements ALTER TABLE :drop_chunks_mat_table_u RENAME time_bucket TO bad_name; ERROR: renaming columns on materialization tables is not supported ALTER TABLE :drop_chunks_mat_table_u ADD UNIQUE(time_bucket); ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u SET UNLOGGED; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u ENABLE ROW LEVEL SECURITY; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u ADD COLUMN fizzle INTEGER; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u DROP COLUMN time_bucket; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u ALTER COLUMN time_bucket DROP NOT NULL; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u ALTER COLUMN time_bucket SET DEFAULT 1; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u ALTER COLUMN time_bucket SET STORAGE EXTERNAL; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u DISABLE TRIGGER ALL; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u SET TABLESPACE foo; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u NOT OF; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u OWNER TO CURRENT_USER; ERROR: operation not supported on materialization tables \set ON_ERROR_STOP 1 ALTER TABLE :drop_chunks_mat_table_u SET SCHEMA public; ALTER TABLE :drop_chunks_mat_table_u_name RENAME TO new_name; SET ROLE :ROLE_DEFAULT_PERM_USER; SET client_min_messages TO NOTICE; SELECT * FROM new_name ORDER BY 1; time_bucket | count -------------+------- 0 | 3 3 | 3 6 | 3 9 | 3 12 | 3 SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | count -------------+------- 0 | 3 3 | 3 6 | 3 9 | 3 12 | 3 \set ON_ERROR_STOP 0 -- no continuous aggregates on a continuous aggregate materialization table CREATE MATERIALIZED VIEW new_name_view WITH ( timescaledb.continuous, timescaledb.materialized_only=true ) AS SELECT time_bucket('6', time_bucket), COUNT("count") FROM new_name GROUP BY 1 WITH NO DATA; ERROR: hypertable is a continuous aggregate materialization table \set ON_ERROR_STOP 1 CREATE TABLE metrics(time timestamptz NOT NULL, device_id int, v1 float, v2 float); SELECT create_hypertable('metrics','time'); create_hypertable ---------------------- (8,public,metrics,t) INSERT INTO metrics SELECT generate_series('2000-01-01'::timestamptz,'2000-01-10','1m'),1,0.25,0.75; -- check expressions in view definition CREATE MATERIALIZED VIEW cagg_expr WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1d', time) AS time, 'Const'::text AS Const, 4.3::numeric AS "numeric", first(metrics,time), CASE WHEN true THEN 'foo' ELSE 'bar' END, COALESCE(NULL,'coalesce'), avg(v1) + avg(v2) AS avg1, avg(v1+v2) AS avg2 FROM metrics GROUP BY 1 WITH NO DATA; CALL refresh_continuous_aggregate('cagg_expr', NULL, NULL); SELECT * FROM cagg_expr ORDER BY time LIMIT 5; time | const | numeric | first | case | coalesce | avg1 | avg2 ------------------------------+-------+---------+----------------------------------------------+------+----------+------+------ Fri Dec 31 16:00:00 1999 UTC | Const | 4.3 | ("Sat Jan 01 00:00:00 2000 UTC",1,0.25,0.75) | foo | coalesce | 1 | 1 Sat Jan 01 16:00:00 2000 UTC | Const | 4.3 | ("Sat Jan 01 16:00:00 2000 UTC",1,0.25,0.75) | foo | coalesce | 1 | 1 Sun Jan 02 16:00:00 2000 UTC | Const | 4.3 | ("Sun Jan 02 16:00:00 2000 UTC",1,0.25,0.75) | foo | coalesce | 1 | 1 Mon Jan 03 16:00:00 2000 UTC | Const | 4.3 | ("Mon Jan 03 16:00:00 2000 UTC",1,0.25,0.75) | foo | coalesce | 1 | 1 Tue Jan 04 16:00:00 2000 UTC | Const | 4.3 | ("Tue Jan 04 16:00:00 2000 UTC",1,0.25,0.75) | foo | coalesce | 1 | 1 --test materialization of invalidation before drop DROP TABLE IF EXISTS drop_chunks_table CASCADE; NOTICE: table "drop_chunks_table" does not exist, skipping DROP TABLE IF EXISTS drop_chunks_table_u CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_7_9_chunk CREATE TABLE drop_chunks_table(time BIGINT NOT NULL, data INTEGER); SELECT hypertable_id AS drop_chunks_table_nid FROM create_hypertable('drop_chunks_table', 'time', chunk_time_interval => 10) \gset CREATE OR REPLACE FUNCTION integer_now_test2() returns bigint LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), bigint '0') FROM drop_chunks_table $$; SELECT set_integer_now_func('drop_chunks_table', 'integer_now_test2'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW drop_chunks_view WITH ( timescaledb.continuous, timescaledb.materialized_only=true ) AS SELECT time_bucket('5', time), max(data) FROM drop_chunks_table GROUP BY 1 WITH NO DATA; INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(0, 20) AS i; --dropping chunks will process the invalidations SELECT drop_chunks('drop_chunks_table', older_than => (integer_now_test2()-9)); drop_chunks ------------------------------------------ _timescaledb_internal._hyper_10_13_chunk SELECT * FROM drop_chunks_table ORDER BY time ASC limit 1; time | data ------+------ 10 | 10 INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(20, 35) AS i; CALL refresh_continuous_aggregate('drop_chunks_view', 10, 40); --this will be seen after the drop its within the invalidation window and will be dropped INSERT INTO drop_chunks_table VALUES (26, 100); --this will not be processed by the drop since chunk 30-39 is not dropped but will be seen after refresh --shows that the drop doesn't do more work than necessary INSERT INTO drop_chunks_table VALUES (31, 200); --move the time up to 39 INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(35, 39) AS i; --the chunks and ranges we have thus far SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table'; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_10_14_chunk | 10 | 20 _hyper_10_15_chunk | 20 | 30 _hyper_10_16_chunk | 30 | 40 --the invalidation on 25 not yet seen SELECT * FROM drop_chunks_view ORDER BY time_bucket DESC; time_bucket | max -------------+----- 35 | 35 30 | 34 25 | 29 20 | 24 15 | 19 10 | 14 --refresh to process the invalidations and then drop CALL refresh_continuous_aggregate('drop_chunks_view', NULL, (integer_now_test2()-9)); SELECT drop_chunks('drop_chunks_table', older_than => (integer_now_test2()-9)); drop_chunks ------------------------------------------ _timescaledb_internal._hyper_10_14_chunk _timescaledb_internal._hyper_10_15_chunk --new values on 25 now seen in view SELECT * FROM drop_chunks_view ORDER BY time_bucket DESC; time_bucket | max -------------+----- 35 | 35 30 | 34 25 | 100 20 | 24 15 | 19 10 | 14 --earliest datapoint now in table SELECT * FROM drop_chunks_table ORDER BY time ASC limit 1; time | data ------+------ 30 | 30 --still see data in the view SELECT * FROM drop_chunks_view WHERE time_bucket < (integer_now_test2()-9) ORDER BY time_bucket DESC; time_bucket | max -------------+----- 25 | 100 20 | 24 15 | 19 10 | 14 --no data but covers dropped chunks SELECT * FROM drop_chunks_table WHERE time < (integer_now_test2()-9) ORDER BY time DESC; time | data ------+------ --recreate the dropped chunk INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(0, 20) AS i; --see data from recreated region SELECT * FROM drop_chunks_table WHERE time < (integer_now_test2()-9) ORDER BY time DESC; time | data ------+------ 20 | 20 19 | 19 18 | 18 17 | 17 16 | 16 15 | 15 14 | 14 13 | 13 12 | 12 11 | 11 10 | 10 9 | 9 8 | 8 7 | 7 6 | 6 5 | 5 4 | 4 3 | 3 2 | 2 1 | 1 0 | 0 --should show chunk with old name and old ranges SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table' ORDER BY range_start_integer; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_10_18_chunk | 0 | 10 _hyper_10_19_chunk | 10 | 20 _hyper_10_20_chunk | 20 | 30 _hyper_10_16_chunk | 30 | 40 --We dropped everything up to the bucket starting at 30 and then --inserted new data up to and including time 20. Therefore, the --dropped data should stay the same as long as we only refresh --buckets that have non-dropped data. CALL refresh_continuous_aggregate('drop_chunks_view', 30, 40); SELECT * FROM drop_chunks_view ORDER BY time_bucket DESC; time_bucket | max -------------+----- 35 | 39 30 | 200 25 | 100 20 | 24 15 | 19 10 | 14 SELECT format('%I.%I', schema_name, table_name) AS drop_chunks_mat_tablen, schema_name AS drop_chunks_mat_schema, table_name AS drop_chunks_mat_table_name FROM _timescaledb_catalog.hypertable, _timescaledb_catalog.continuous_agg WHERE _timescaledb_catalog.continuous_agg.raw_hypertable_id = :drop_chunks_table_nid AND _timescaledb_catalog.hypertable.id = _timescaledb_catalog.continuous_agg.mat_hypertable_id \gset -- TEST drop chunks from continuous aggregates by specifying view name SELECT drop_chunks('drop_chunks_view', newer_than => -20, verbose => true); INFO: dropping chunk _timescaledb_internal._hyper_11_17_chunk drop_chunks ------------------------------------------ _timescaledb_internal._hyper_11_17_chunk -- Test that we cannot drop chunks when specifying materialized -- hypertable INSERT INTO drop_chunks_table SELECT generate_series(45, 55), 500; CALL refresh_continuous_aggregate('drop_chunks_view', 45, 55); SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = :'drop_chunks_mat_table_name' ORDER BY range_start_integer; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_11_23_chunk | 0 | 100 \set ON_ERROR_STOP 0 \set VERBOSITY default SELECT drop_chunks(:'drop_chunks_mat_tablen', older_than => 60); ERROR: operation not supported on materialized hypertable DETAIL: Hypertable "_materialized_hypertable_11" is a materialized hypertable. HINT: Try the operation on the continuous aggregate instead. \set VERBOSITY terse \set ON_ERROR_STOP 1 ----------------------------------------------------------------- -- Test that refresh_continuous_aggregate on chunk will refresh, -- but only in the regions covered by the show chunks. ----------------------------------------------------------------- SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table' ORDER BY 2,3; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_10_18_chunk | 0 | 10 _hyper_10_19_chunk | 10 | 20 _hyper_10_20_chunk | 20 | 30 _hyper_10_16_chunk | 30 | 40 _hyper_10_21_chunk | 40 | 50 _hyper_10_22_chunk | 50 | 60 -- Pick the second chunk as the one to drop WITH numbered_chunks AS ( SELECT row_number() OVER (ORDER BY range_start_integer), chunk_schema, chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table' ORDER BY 1 ) SELECT format('%I.%I', chunk_schema, chunk_name) AS chunk_to_drop, range_start_integer, range_end_integer FROM numbered_chunks WHERE row_number = 2 \gset -- There's data in the table for the chunk/range we will drop SELECT * FROM drop_chunks_table WHERE time >= :range_start_integer AND time < :range_end_integer ORDER BY 1; time | data ------+------ 10 | 10 11 | 11 12 | 12 13 | 13 14 | 14 15 | 15 16 | 16 17 | 17 18 | 18 19 | 19 -- Make sure there is also data in the continuous aggregate -- CARE: -- Note that this behaviour of dropping the materialization table chunks and expecting a refresh -- that overlaps that time range to NOT update those chunks is undefined. CALL refresh_continuous_aggregate('drop_chunks_view', 0, 50); SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | max -------------+----- 0 | 4 5 | 9 10 | 14 15 | 19 20 | 20 45 | 500 50 | 500 -- Drop the second chunk, to leave a gap in the data DROP TABLE :chunk_to_drop; -- Verify that the second chunk is dropped SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table' ORDER BY 2,3; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_10_18_chunk | 0 | 10 _hyper_10_20_chunk | 20 | 30 _hyper_10_16_chunk | 30 | 40 _hyper_10_21_chunk | 40 | 50 _hyper_10_22_chunk | 50 | 60 -- Data is no longer in the table but still in the view SELECT * FROM drop_chunks_table WHERE time >= :range_start_integer AND time < :range_end_integer ORDER BY 1; time | data ------+------ SELECT * FROM drop_chunks_view WHERE time_bucket >= :range_start_integer AND time_bucket < :range_end_integer ORDER BY 1; time_bucket | max -------------+----- 10 | 14 15 | 19 -- Insert a large value in one of the chunks that will be dropped INSERT INTO drop_chunks_table VALUES (:range_start_integer-1, 100); -- Now refresh and drop the two adjecent chunks CALL refresh_continuous_aggregate('drop_chunks_view', NULL, 30); SELECT drop_chunks('drop_chunks_table', older_than=>30); drop_chunks ------------------------------------------ _timescaledb_internal._hyper_10_18_chunk _timescaledb_internal._hyper_10_20_chunk -- Verify that the chunks are dropped SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table' ORDER BY 2,3; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_10_16_chunk | 30 | 40 _hyper_10_21_chunk | 40 | 50 _hyper_10_22_chunk | 50 | 60 -- The continuous aggregate should be refreshed in the regions covered -- by the dropped chunks, but not in the "gap" region, i.e., the -- region of the chunk that was dropped via DROP TABLE. SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | max -------------+----- 0 | 4 5 | 100 20 | 20 45 | 500 50 | 500 -- Now refresh in the region of the first two dropped chunks CALL refresh_continuous_aggregate('drop_chunks_view', 0, :range_end_integer); -- Aggregate data in the refreshed range should no longer exist since -- the underlying data was dropped. SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | max -------------+----- 20 | 20 45 | 500 50 | 500 -------------------------------------------------------------------- -- Check that we can create a materialized table in a tablespace. We -- create one with tablespace and one without and compare them. CREATE VIEW cagg_info AS WITH caggs AS ( SELECT format('%I.%I', user_view_schema, user_view_name)::regclass AS user_view, format('%I.%I', direct_view_schema, direct_view_name)::regclass AS direct_view, format('%I.%I', partial_view_schema, partial_view_name)::regclass AS partial_view, format('%I.%I', ht.schema_name, ht.table_name)::regclass AS mat_relid FROM _timescaledb_catalog.hypertable ht, _timescaledb_catalog.continuous_agg cagg WHERE ht.id = cagg.mat_hypertable_id ) SELECT user_view, pg_get_userbyid(relowner) AS user_view_owner, relname AS mat_table, (SELECT pg_get_userbyid(relowner) FROM pg_class WHERE oid = mat_relid) AS mat_table_owner, direct_view, (SELECT pg_get_userbyid(relowner) FROM pg_class WHERE oid = direct_view) AS direct_view_owner, partial_view, (SELECT pg_get_userbyid(relowner) FROM pg_class WHERE oid = partial_view) AS partial_view_owner, (SELECT spcname FROM pg_tablespace WHERE oid = reltablespace) AS tablespace FROM pg_class JOIN caggs ON pg_class.oid = caggs.mat_relid; GRANT SELECT ON cagg_info TO PUBLIC; CREATE VIEW chunk_info AS SELECT ht.schema_name, ht.table_name, relname AS chunk_name, (SELECT spcname FROM pg_tablespace WHERE oid = reltablespace) AS tablespace FROM pg_class c, _timescaledb_catalog.hypertable ht, _timescaledb_catalog.chunk ch WHERE ch.table_name = c.relname AND ht.id = ch.hypertable_id; CREATE TABLE whatever(time BIGINT NOT NULL, data INTEGER); SELECT hypertable_id AS whatever_nid FROM create_hypertable('whatever', 'time', chunk_time_interval => 10) \gset SELECT set_integer_now_func('whatever', 'integer_now_test'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW whatever_view_1 WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('5', time), COUNT(data) FROM whatever GROUP BY 1 WITH NO DATA; CREATE MATERIALIZED VIEW whatever_view_2 WITH (timescaledb.continuous, timescaledb.materialized_only=true) TABLESPACE tablespace1 AS SELECT time_bucket('5', time), COUNT(data) FROM whatever GROUP BY 1 WITH NO DATA; INSERT INTO whatever SELECT i, i FROM generate_series(0, 29) AS i; CALL refresh_continuous_aggregate('whatever_view_1', NULL, NULL); CALL refresh_continuous_aggregate('whatever_view_2', NULL, NULL); SELECT user_view, mat_table, cagg_info.tablespace AS mat_tablespace, chunk_name, chunk_info.tablespace AS chunk_tablespace FROM cagg_info, chunk_info WHERE mat_table::text = table_name AND user_view::text LIKE 'whatever_view%'; user_view | mat_table | mat_tablespace | chunk_name | chunk_tablespace -----------------+-----------------------------+----------------+--------------------+------------------ whatever_view_1 | _materialized_hypertable_13 | | _hyper_13_27_chunk | whatever_view_2 | _materialized_hypertable_14 | tablespace1 | _hyper_14_28_chunk | tablespace1 ALTER MATERIALIZED VIEW whatever_view_1 SET TABLESPACE tablespace2; SELECT user_view, mat_table, cagg_info.tablespace AS mat_tablespace, chunk_name, chunk_info.tablespace AS chunk_tablespace FROM cagg_info, chunk_info WHERE mat_table::text = table_name AND user_view::text LIKE 'whatever_view%'; user_view | mat_table | mat_tablespace | chunk_name | chunk_tablespace -----------------+-----------------------------+----------------+--------------------+------------------ whatever_view_1 | _materialized_hypertable_13 | tablespace2 | _hyper_13_27_chunk | tablespace2 whatever_view_2 | _materialized_hypertable_14 | tablespace1 | _hyper_14_28_chunk | tablespace1 DROP MATERIALIZED VIEW whatever_view_1; NOTICE: drop cascades to table _timescaledb_internal._hyper_13_27_chunk DROP MATERIALIZED VIEW whatever_view_2; NOTICE: drop cascades to table _timescaledb_internal._hyper_14_28_chunk -- test bucket width expressions on integer hypertables CREATE TABLE metrics_int2 ( time int2 NOT NULL, device_id int, v1 float, v2 float ); CREATE TABLE metrics_int4 ( time int4 NOT NULL, device_id int, v1 float, v2 float ); CREATE TABLE metrics_int8 ( time int8 NOT NULL, device_id int, v1 float, v2 float ); SELECT create_hypertable (('metrics_' || dt)::regclass, 'time', chunk_time_interval => 10) FROM ( VALUES ('int2'), ('int4'), ('int8')) v (dt); create_hypertable ---------------------------- (15,public,metrics_int2,t) (16,public,metrics_int4,t) (17,public,metrics_int8,t) CREATE OR REPLACE FUNCTION int2_now () RETURNS int2 LANGUAGE SQL STABLE AS $$ SELECT 10::int2 $$; CREATE OR REPLACE FUNCTION int4_now () RETURNS int4 LANGUAGE SQL STABLE AS $$ SELECT 10::int4 $$; CREATE OR REPLACE FUNCTION int8_now () RETURNS int8 LANGUAGE SQL STABLE AS $$ SELECT 10::int8 $$; SELECT set_integer_now_func (('metrics_' || dt)::regclass, (dt || '_now')::regproc) FROM ( VALUES ('int2'), ('int4'), ('int8')) v (dt); set_integer_now_func ---------------------- -- width expression for int2 hypertables CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1::smallint, time) FROM metrics_int2 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1::smallint + 2::smallint, time) FROM metrics_int2 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; -- width expression for int4 hypertables CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1, time) FROM metrics_int4 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1 + 2, time) FROM metrics_int4 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; -- width expression for int8 hypertables CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1, time) FROM metrics_int8 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1 + 2, time) FROM metrics_int8 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; \set ON_ERROR_STOP 0 -- non-immutable expresions should be rejected CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(extract(year FROM now())::smallint, time) FROM metrics_int2 GROUP BY 1; ERROR: only immutable expressions allowed in time bucket function CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(extract(year FROM now())::int, time) FROM metrics_int4 GROUP BY 1; ERROR: only immutable expressions allowed in time bucket function CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(extract(year FROM now())::int, time) FROM metrics_int8 GROUP BY 1; ERROR: only immutable expressions allowed in time bucket function \set ON_ERROR_STOP 1 -- Test various ALTER MATERIALIZED VIEW statements. SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE MATERIALIZED VIEW owner_check WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1 + 2, time) FROM metrics_int8 GROUP BY 1 WITH NO DATA; \x on SELECT * FROM cagg_info WHERE user_view::text = 'owner_check'; -[ RECORD 1 ]------+--------------------------------------- user_view | owner_check user_view_owner | default_perm_user mat_table | _materialized_hypertable_24 mat_table_owner | default_perm_user direct_view | _timescaledb_internal._direct_view_24 direct_view_owner | default_perm_user partial_view | _timescaledb_internal._partial_view_24 partial_view_owner | default_perm_user tablespace | \x off -- This should not work since the target user has the wrong role, but -- we test that the normal checks are done when changing the owner. \set ON_ERROR_STOP 0 ALTER MATERIALIZED VIEW owner_check OWNER TO :ROLE_1; ERROR: must be able to SET ROLE "test_role_1" \set ON_ERROR_STOP 1 -- Superuser can always change owner SET ROLE :ROLE_SUPERUSER; -- Add a refresh policy before changing owner to verify job owner is propagated SELECT add_continuous_aggregate_policy('owner_check', NULL, 1::int8, '1 day'::interval) AS cagg_job_id \gset SELECT owner FROM _timescaledb_config.bgw_job WHERE id = :cagg_job_id; owner ------------------- default_perm_user ALTER MATERIALIZED VIEW owner_check OWNER TO :ROLE_1; \x on SELECT * FROM cagg_info WHERE user_view::text = 'owner_check'; -[ RECORD 1 ]------+--------------------------------------- user_view | owner_check user_view_owner | test_role_1 mat_table | _materialized_hypertable_24 mat_table_owner | test_role_1 direct_view | _timescaledb_internal._direct_view_24 direct_view_owner | test_role_1 partial_view | _timescaledb_internal._partial_view_24 partial_view_owner | test_role_1 tablespace | \x off -- make sure policy job owner is propagated SELECT owner FROM _timescaledb_config.bgw_job WHERE id = :cagg_job_id; owner ------------- test_role_1 SELECT remove_continuous_aggregate_policy('owner_check'); remove_continuous_aggregate_policy ------------------------------------ -- -- Test drop continuous aggregate cases -- -- Issue: #2608 -- CREATE OR REPLACE FUNCTION test_int_now() RETURNS INT LANGUAGE SQL STABLE AS $BODY$ SELECT 50; $BODY$; CREATE TABLE conditionsnm(time_int INT NOT NULL, device INT, value FLOAT); SELECT create_hypertable('conditionsnm', 'time_int', chunk_time_interval => 10); create_hypertable ---------------------------- (25,public,conditionsnm,t) SELECT set_integer_now_func('conditionsnm', 'test_int_now'); set_integer_now_func ---------------------- INSERT INTO conditionsnm SELECT time_val, time_val % 4, 3.14 FROM generate_series(0,100,1) AS time_val; -- Case 1: DROP CREATE MATERIALIZED VIEW conditionsnm_4 WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(7, time_int) as bucket, SUM(value), COUNT(value) FROM conditionsnm GROUP BY bucket WITH DATA; NOTICE: refreshing continuous aggregate "conditionsnm_4" DROP materialized view conditionsnm_4; NOTICE: drop cascades to table _timescaledb_internal._hyper_26_40_chunk -- Case 2: DROP CASCADE should have similar behaviour as DROP CREATE MATERIALIZED VIEW conditionsnm_4 WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(7, time_int) as bucket, SUM(value), COUNT(value) FROM conditionsnm GROUP BY bucket WITH DATA; NOTICE: refreshing continuous aggregate "conditionsnm_4" DROP materialized view conditionsnm_4 CASCADE; NOTICE: drop cascades to table _timescaledb_internal._hyper_27_41_chunk -- Case 3: require CASCADE in case of dependent object CREATE MATERIALIZED VIEW conditionsnm_4 WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(7, time_int) as bucket, SUM(value), COUNT(value) FROM conditionsnm GROUP BY bucket WITH DATA; NOTICE: refreshing continuous aggregate "conditionsnm_4" CREATE VIEW see_cagg as select * from conditionsnm_4; \set ON_ERROR_STOP 0 DROP MATERIALIZED VIEW conditionsnm_4; ERROR: cannot drop view conditionsnm_4 because other objects depend on it \set ON_ERROR_STOP 1 -- Case 4: DROP CASCADE with dependency DROP MATERIALIZED VIEW conditionsnm_4 CASCADE; NOTICE: drop cascades to view see_cagg NOTICE: drop cascades to table _timescaledb_internal._hyper_28_42_chunk -- Test DROP SCHEMA CASCADE with continuous aggregates -- -- Issue: #2350 -- -- Case 1: DROP SCHEMA CASCADE CREATE SCHEMA test_schema; CREATE TABLE test_schema.telemetry_raw ( ts TIMESTAMP WITH TIME ZONE NOT NULL, value DOUBLE PRECISION ); SELECT create_hypertable('test_schema.telemetry_raw', 'ts'); create_hypertable ---------------------------------- (29,test_schema,telemetry_raw,t) CREATE MATERIALIZED VIEW test_schema.telemetry_1s WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(INTERVAL '1s', ts) AS ts_1s, avg(value) FROM test_schema.telemetry_raw GROUP BY ts_1s WITH NO DATA; SELECT ca.raw_hypertable_id, h.schema_name, h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) WHERE user_view_name = 'telemetry_1s'; raw_hypertable_id | schema_name | MAT_TABLE_NAME | PART_VIEW_NAME | partial_view_schema -------------------+-----------------------+-----------------------------+------------------+----------------------- 29 | _timescaledb_internal | _materialized_hypertable_30 | _partial_view_30 | _timescaledb_internal \gset DROP SCHEMA test_schema CASCADE; NOTICE: drop cascades to 4 other objects SELECT count(*) FROM pg_class WHERE relname = :'MAT_TABLE_NAME'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = :'PART_VIEW_NAME'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = 'telemetry_1s'; count ------- 0 SELECT count(*) FROM pg_namespace WHERE nspname = 'test_schema'; count ------- 0 -- Case 2: DROP SCHEMA CASCADE with multiple caggs CREATE SCHEMA test_schema; CREATE TABLE test_schema.telemetry_raw ( ts TIMESTAMP WITH TIME ZONE NOT NULL, value DOUBLE PRECISION ); SELECT create_hypertable('test_schema.telemetry_raw', 'ts'); create_hypertable ---------------------------------- (31,test_schema,telemetry_raw,t) CREATE MATERIALIZED VIEW test_schema.cagg1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(INTERVAL '1s', ts) AS ts_1s, avg(value) FROM test_schema.telemetry_raw GROUP BY ts_1s WITH NO DATA; CREATE MATERIALIZED VIEW test_schema.cagg2 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(INTERVAL '1s', ts) AS ts_1s, avg(value) FROM test_schema.telemetry_raw GROUP BY ts_1s WITH NO DATA; SELECT ca.raw_hypertable_id, h.schema_name, h.table_name AS "MAT_TABLE_NAME1", partial_view_name as "PART_VIEW_NAME1", partial_view_schema FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) WHERE user_view_name = 'cagg1'; raw_hypertable_id | schema_name | MAT_TABLE_NAME1 | PART_VIEW_NAME1 | partial_view_schema -------------------+-----------------------+-----------------------------+------------------+----------------------- 31 | _timescaledb_internal | _materialized_hypertable_32 | _partial_view_32 | _timescaledb_internal \gset SELECT ca.raw_hypertable_id, h.schema_name, h.table_name AS "MAT_TABLE_NAME2", partial_view_name as "PART_VIEW_NAME2", partial_view_schema FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) WHERE user_view_name = 'cagg2'; raw_hypertable_id | schema_name | MAT_TABLE_NAME2 | PART_VIEW_NAME2 | partial_view_schema -------------------+-----------------------+-----------------------------+------------------+----------------------- 31 | _timescaledb_internal | _materialized_hypertable_33 | _partial_view_33 | _timescaledb_internal \gset DROP SCHEMA test_schema CASCADE; NOTICE: drop cascades to 7 other objects SELECT count(*) FROM pg_class WHERE relname = :'MAT_TABLE_NAME1'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = :'PART_VIEW_NAME1'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = 'cagg1'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = :'MAT_TABLE_NAME2'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = :'PART_VIEW_NAME2'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = 'cagg2'; count ------- 0 SELECT count(*) FROM pg_namespace WHERE nspname = 'test_schema'; count ------- 0 DROP TABLESPACE tablespace1; DROP TABLESPACE tablespace2; -- Check that we can rename a column of a materialized view and still -- rebuild it after (#3051, #3405) CREATE TABLE conditions ( time TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL ); SELECT create_hypertable('conditions', 'time'); create_hypertable -------------------------- (34,public,conditions,t) INSERT INTO conditions VALUES ( '2018-01-01 09:20:00-08', 'SFO', 55); INSERT INTO conditions VALUES ( '2018-01-02 09:30:00-08', 'por', 100); INSERT INTO conditions VALUES ( '2018-01-02 09:20:00-08', 'SFO', 65); INSERT INTO conditions VALUES ( '2018-01-02 09:10:00-08', 'NYC', 65); INSERT INTO conditions VALUES ( '2018-11-01 09:20:00-08', 'NYC', 45); INSERT INTO conditions VALUES ( '2018-11-01 10:40:00-08', 'NYC', 55); INSERT INTO conditions VALUES ( '2018-11-01 11:50:00-08', 'NYC', 65); INSERT INTO conditions VALUES ( '2018-11-01 12:10:00-08', 'NYC', 75); INSERT INTO conditions VALUES ( '2018-11-01 13:10:00-08', 'NYC', 85); INSERT INTO conditions VALUES ( '2018-11-02 09:20:00-08', 'NYC', 10); INSERT INTO conditions VALUES ( '2018-11-02 10:30:00-08', 'NYC', 20); CREATE MATERIALIZED VIEW conditions_daily WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT location, time_bucket(INTERVAL '1 day', time) AS bucket, AVG(temperature) FROM conditions GROUP BY location, bucket WITH NO DATA; CREATE MATERIALIZED VIEW conditions_weekly WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT location, time_bucket(INTERVAL '7 day', bucket) AS bucket, AVG(avg) FROM conditions_daily GROUP BY 1, 2 WITH NO DATA; SELECT format('%I.%I', '_timescaledb_internal', h.table_name) AS "MAT_TABLE_NAME", format('%I.%I', '_timescaledb_internal', partial_view_name) AS "PART_VIEW_NAME", format('%I.%I', '_timescaledb_internal', direct_view_name) AS "DIRECT_VIEW_NAME" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) WHERE user_view_name = 'conditions_daily' \gset -- Show both the columns and the view definitions to see that -- references are correct in the view as well. SELECT * FROM test.show_columns('conditions_daily'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f bucket | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'DIRECT_VIEW_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f bucket | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'PART_VIEW_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f bucket | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'MAT_TABLE_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f bucket | timestamp with time zone | t avg | double precision | f ALTER MATERIALIZED VIEW conditions_daily RENAME COLUMN bucket to "time"; -- Show both the columns and the view definitions to see that -- references are correct in the view as well. SELECT * FROM test.show_columns(' conditions_daily'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'DIRECT_VIEW_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'PART_VIEW_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'MAT_TABLE_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | t avg | double precision | f -- This will rebuild the materialized view and should succeed. ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only = false); -- Refresh the continuous aggregate to check that it works after the -- rename. \set VERBOSITY verbose CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); \set VERBOSITY terse -- Rename another column after the flip and verify toggling back and -- forth still works. This exercises the rename when the user view -- already has a UNION ALL query (materialized_only = false). ALTER MATERIALIZED VIEW conditions_daily RENAME COLUMN avg TO average; SELECT * FROM test.show_columns('conditions_daily'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | f average | double precision | f SELECT * FROM test.show_columns(:'DIRECT_VIEW_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | f average | double precision | f SELECT * FROM test.show_columns(:'MAT_TABLE_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | t average | double precision | f ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only = true); ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only = false); -- Verify data is still accessible after multiple renames and toggles. SELECT * FROM conditions_daily ORDER BY location COLLATE "C", time; location | time | average ----------+------------------------------+--------- NYC | Mon Jan 01 16:00:00 2018 UTC | 65 NYC | Wed Oct 31 16:00:00 2018 UTC | 65 NYC | Thu Nov 01 16:00:00 2018 UTC | 15 SFO | Sun Dec 31 16:00:00 2017 UTC | 55 SFO | Mon Jan 01 16:00:00 2018 UTC | 65 por | Mon Jan 01 16:00:00 2018 UTC | 100 -- check hierarchical continuous aggregate still works after renames and toggles on the underlying cagg ALTER MATERIALIZED VIEW conditions_weekly SET (timescaledb.materialized_only = false); ALTER MATERIALIZED VIEW conditions_weekly SET (timescaledb.materialized_only = true); SELECT * FROM conditions_weekly ORDER BY location COLLATE "C", bucket; location | bucket | avg ----------+--------+----- -- Verify that direct rename on the materialization hypertable is blocked. \set ON_ERROR_STOP 0 ALTER TABLE :MAT_TABLE_NAME RENAME COLUMN average TO avg; ERROR: renaming columns on materialization tables is not supported \set ON_ERROR_STOP 1 -- Rename back so subsequent tests that reference "avg" still work. ALTER MATERIALIZED VIEW conditions_daily RENAME COLUMN average TO avg; -- -- Indexes on continuous aggregate -- \set ON_ERROR_STOP 0 -- unique indexes are not supported CREATE UNIQUE INDEX index_unique_error ON conditions_daily ("time", location); ERROR: continuous aggregates do not support UNIQUE indexes -- concurrently index creation not supported CREATE INDEX CONCURRENTLY index_concurrently_avg ON conditions_daily (avg); ERROR: hypertables do not support concurrent index creation \set ON_ERROR_STOP 1 CREATE INDEX index_avg ON conditions_daily (avg); CREATE INDEX index_avg_only ON ONLY conditions_daily (avg); CREATE INDEX index_avg_include ON conditions_daily (avg) INCLUDE (location); CREATE INDEX index_avg_expr ON conditions_daily ((avg + 1)); CREATE INDEX index_avg_location_sfo ON conditions_daily (avg) WHERE location = 'SFO'; CREATE INDEX index_avg_expr_location_sfo ON conditions_daily ((avg + 2)) WHERE location = 'SFO'; SELECT * FROM test.show_indexespred(:'MAT_TABLE_NAME'); Index | Columns | Expr | Pred | Unique | Primary | Exclusion | Tablespace -----------------------------------------------------------------------+-------------------+---------------------------+------------------------+--------+---------+-----------+------------ _timescaledb_internal._materialized_hypertable_35_bucket_idx | {bucket} | | | f | f | f | _timescaledb_internal._materialized_hypertable_35_location_bucket_idx | {location,bucket} | | | f | f | f | _timescaledb_internal.index_avg | {avg} | | | f | f | f | _timescaledb_internal.index_avg_expr | {expr} | avg + 1::double precision | | f | f | f | _timescaledb_internal.index_avg_expr_location_sfo | {expr} | avg + 2::double precision | location = 'SFO'::text | f | f | f | _timescaledb_internal.index_avg_include | {avg,location} | | | f | f | f | _timescaledb_internal.index_avg_location_sfo | {avg} | | location = 'SFO'::text | f | f | f | _timescaledb_internal.index_avg_only | {avg} | | | f | f | f | -- #3696 assertion failure when referencing columns not present in result CREATE TABLE i3696(time timestamptz NOT NULL, search_query text, cnt integer, cnt2 integer); SELECT table_name FROM create_hypertable('i3696','time'); table_name ------------ i3696 CREATE MATERIALIZED VIEW i3696_cagg1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT search_query,count(search_query) as count, sum(cnt), time_bucket(INTERVAL '1 minute', time) AS bucket FROM i3696 GROUP BY cnt +cnt2 , bucket, search_query; NOTICE: continuous aggregate "i3696_cagg1" is already up-to-date ALTER MATERIALIZED VIEW i3696_cagg1 SET (timescaledb.materialized_only = 'true'); CREATE MATERIALIZED VIEW i3696_cagg2 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT search_query,count(search_query) as count, sum(cnt), time_bucket(INTERVAL '1 minute', time) AS bucket FROM i3696 GROUP BY cnt + cnt2, bucket, search_query HAVING cnt + cnt2 + sum(cnt) > 2 or count(cnt2) > 10; NOTICE: continuous aggregate "i3696_cagg2" is already up-to-date ALTER MATERIALIZED VIEW i3696_cagg2 SET (timescaledb.materialized_only = 'true'); --TEST test with multiple settings on continuous aggregates -- -- test for materialized_only + compress combinations (real time aggs enabled initially) CREATE TABLE test_setting(time timestamptz not null, val numeric); SELECT create_hypertable('test_setting', 'time'); create_hypertable ---------------------------- (40,public,test_setting,t) CREATE MATERIALIZED VIEW test_setting_cagg with (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h',time), avg(val), count(*) FROM test_setting GROUP BY 1; NOTICE: continuous aggregate "test_setting_cagg" is already up-to-date INSERT INTO test_setting SELECT generate_series( '2020-01-10 8:00'::timestamp, '2020-01-30 10:00+00'::timestamptz, '1 day'::interval), 10.0; CALL refresh_continuous_aggregate('test_setting_cagg', NULL, '2020-05-30 10:00+00'::timestamptz); SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 --this row is not in the materialized result --- INSERT INTO test_setting VALUES( '2020-11-01', 20); --try out 2 settings here -- ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'true', timescaledb.compress='true'); NOTICE: defaulting compress_orderby to time_bucket SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | t | t --real time aggs is off now , should return 20 -- SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 --now set it back to false -- ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'false', timescaledb.compress='true'); NOTICE: defaulting compress_orderby to time_bucket SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | t | f --count should return additional data since we have real time aggs on SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 21 ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'true', timescaledb.compress='false'); SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | f | t --real time aggs is off now , should return 20 -- SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'false', timescaledb.compress='false'); SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | f | f --count should return additional data since we have real time aggs on SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 21 DELETE FROM test_setting WHERE val = 20; --TEST test with multiple settings on continuous aggregates with real time aggregates turned off initially -- -- test for materialized_only + compress combinations (real time aggs enabled initially) DROP MATERIALIZED VIEW test_setting_cagg; NOTICE: drop cascades to table _timescaledb_internal._hyper_41_50_chunk CREATE MATERIALIZED VIEW test_setting_cagg with (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT time_bucket('1h',time), avg(val), count(*) FROM test_setting GROUP BY 1; NOTICE: refreshing continuous aggregate "test_setting_cagg" CALL refresh_continuous_aggregate('test_setting_cagg', NULL, '2020-05-30 10:00+00'::timestamptz); SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 --this row is not in the materialized result --- INSERT INTO test_setting VALUES( '2020-11-01', 20); --try out 2 settings here -- ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'false', timescaledb.compress='true'); NOTICE: defaulting compress_orderby to time_bucket SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | t | f --count should return additional data since we have real time aggs on SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 21 --now set it back to false -- ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'true', timescaledb.compress='true'); NOTICE: defaulting compress_orderby to time_bucket SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | t | t --real time aggs is off now , should return 20 -- SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'false', timescaledb.compress='false'); SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | f | f --count should return additional data since we have real time aggs on SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 21 ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'true', timescaledb.compress='false'); SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | f | t --real time aggs is off now , should return 20 -- SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 -- END TEST with multiple settings -- Test View Target Entries that contain both aggrefs and Vars in the same expression CREATE TABLE transactions ( "time" timestamp with time zone NOT NULL, dummy1 integer, dummy2 integer, dummy3 integer, dummy4 integer, dummy5 integer, amount integer, fiat_value integer ); SELECT create_hypertable('transactions', 'time'); create_hypertable ---------------------------- (45,public,transactions,t) INSERT INTO transactions VALUES ( '2018-01-01 09:20:00-08', 0, 0, 0, 0, 0, 1, 10); INSERT INTO transactions VALUES ( '2018-01-02 09:30:00-08', 0, 0, 0, 0, 0, -1, 10); INSERT INTO transactions VALUES ( '2018-01-02 09:20:00-08', 0, 0, 0, 0, 0, -1, 10); INSERT INTO transactions VALUES ( '2018-01-02 09:10:00-08', 0, 0, 0, 0, 0, -1, 10); INSERT INTO transactions VALUES ( '2018-11-01 09:20:00-08', 0, 0, 0, 0, 0, 1, 10); INSERT INTO transactions VALUES ( '2018-11-01 10:40:00-08', 0, 0, 0, 0, 0, 1, 10); INSERT INTO transactions VALUES ( '2018-11-01 11:50:00-08', 0, 0, 0, 0, 0, 1, 10); INSERT INTO transactions VALUES ( '2018-11-01 12:10:00-08', 0, 0, 0, 0, 0, -1, 10); INSERT INTO transactions VALUES ( '2018-11-01 13:10:00-08', 0, 0, 0, 0, 0, -1, 10); INSERT INTO transactions VALUES ( '2018-11-02 09:20:00-08', 0, 0, 0, 0, 0, 1, 10); INSERT INTO transactions VALUES ( '2018-11-02 10:30:00-08', 0, 0, 0, 0, 0, -1, 10); CREATE materialized view cashflows( bucket, amount, cashflow, cashflow2 ) WITH ( timescaledb.continuous, timescaledb.materialized_only = true ) AS SELECT time_bucket ('1 day', time) AS bucket, amount, CASE WHEN amount < 0 THEN (0 - sum(fiat_value)) ELSE sum(fiat_value) END AS cashflow, amount + sum(fiat_value) FROM transactions GROUP BY bucket, amount; NOTICE: refreshing continuous aggregate "cashflows" SELECT h.table_name AS "MAT_TABLE_NAME", partial_view_name AS "PART_VIEW_NAME", direct_view_name AS "DIRECT_VIEW_NAME" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) WHERE user_view_name = 'cashflows' \gset -- Show both the columns and the view definitions to see that -- references are correct in the view as well. \d+ "_timescaledb_internal".:"DIRECT_VIEW_NAME" View "_timescaledb_internal._direct_view_46" Column | Type | Collation | Nullable | Default | Storage | Description -----------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | amount | integer | | | | plain | cashflow | bigint | | | | plain | cashflow2 | bigint | | | | plain | View definition: SELECT time_bucket('@ 1 day'::interval, "time") AS bucket, amount, CASE WHEN amount < 0 THEN 0 - sum(fiat_value) ELSE sum(fiat_value) END AS cashflow, amount + sum(fiat_value) AS cashflow2 FROM transactions GROUP BY (time_bucket('@ 1 day'::interval, "time")), amount; \d+ "_timescaledb_internal".:"PART_VIEW_NAME" View "_timescaledb_internal._partial_view_46" Column | Type | Collation | Nullable | Default | Storage | Description -----------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | amount | integer | | | | plain | cashflow | bigint | | | | plain | cashflow2 | bigint | | | | plain | View definition: SELECT time_bucket('@ 1 day'::interval, "time") AS bucket, amount, CASE WHEN amount < 0 THEN 0 - sum(fiat_value) ELSE sum(fiat_value) END AS cashflow, amount + sum(fiat_value) AS cashflow2 FROM transactions GROUP BY (time_bucket('@ 1 day'::interval, "time")), amount; \d+ "_timescaledb_internal".:"MAT_TABLE_NAME" Table "_timescaledb_internal._materialized_hypertable_46" Column | Type | Collation | Nullable | Default | Storage | Stats target | Description -----------+--------------------------+-----------+----------+---------+---------+--------------+------------- bucket | timestamp with time zone | | not null | | plain | | amount | integer | | | | plain | | cashflow | bigint | | | | plain | | cashflow2 | bigint | | | | plain | | Indexes: "_materialized_hypertable_46_amount_bucket_idx" btree (amount, bucket DESC) "_materialized_hypertable_46_bucket_idx" btree (bucket DESC) Child tables: _timescaledb_internal._hyper_46_55_chunk, _timescaledb_internal._hyper_46_56_chunk \d+ 'cashflows' View "public.cashflows" Column | Type | Collation | Nullable | Default | Storage | Description -----------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | amount | integer | | | | plain | cashflow | bigint | | | | plain | cashflow2 | bigint | | | | plain | View definition: SELECT bucket, amount, cashflow, cashflow2 FROM _timescaledb_internal._materialized_hypertable_46; SELECT * FROM cashflows ORDER BY cashflows; bucket | amount | cashflow | cashflow2 ------------------------------+--------+----------+----------- Sun Dec 31 16:00:00 2017 UTC | 1 | 10 | 11 Mon Jan 01 16:00:00 2018 UTC | -1 | -30 | 29 Wed Oct 31 16:00:00 2018 UTC | -1 | -20 | 19 Wed Oct 31 16:00:00 2018 UTC | 1 | 30 | 31 Thu Nov 01 16:00:00 2018 UTC | -1 | -10 | 9 Thu Nov 01 16:00:00 2018 UTC | 1 | 10 | 11 -- test cagg creation with named arguments in time_bucket -- note that positional arguments cannot follow named arguments -- 1. test named origin -- 2. test named timezone -- 3. test named ts -- 4. test named bucket width -- named origin CREATE MATERIALIZED VIEW cagg_named_origin WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h', time, 'UTC', origin => '2001-01-03 01:23:45') AS bucket, avg(amount) as avg_amount FROM transactions GROUP BY 1 WITH NO DATA; -- named timezone CREATE MATERIALIZED VIEW cagg_named_tz_origin WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h', time, timezone => 'UTC', origin => '2001-01-03 01:23:45') AS bucket, avg(amount) as avg_amount FROM transactions GROUP BY 1 WITH NO DATA; -- named ts CREATE MATERIALIZED VIEW cagg_named_ts_tz_origin WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h', ts => time, timezone => 'UTC', origin => '2001-01-03 01:23:45') AS bucket, avg(amount) as avg_amount FROM transactions GROUP BY 1 WITH NO DATA; -- named bucket width CREATE MATERIALIZED VIEW cagg_named_all WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(bucket_width => '1h', ts => time, timezone => 'UTC', origin => '2001-01-03 01:23:45') AS bucket, avg(amount) as avg_amount FROM transactions GROUP BY 1 WITH NO DATA; -- Refreshing from the beginning (NULL) of a CAGG with variable time bucket and -- using an INTERVAL for the end timestamp (issue #5534) CREATE MATERIALIZED VIEW transactions_montly WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT time_bucket(INTERVAL '1 month', time) AS bucket, SUM(fiat_value), MAX(fiat_value), MIN(fiat_value) FROM transactions GROUP BY 1 WITH NO DATA; -- No rows SELECT * FROM transactions_montly ORDER BY bucket; bucket | sum | max | min --------+-----+-----+----- -- Refresh from beginning of the CAGG for 1 month CALL refresh_continuous_aggregate('transactions_montly', NULL, INTERVAL '1 month'); SELECT * FROM transactions_montly ORDER BY bucket; bucket | sum | max | min ------------------------------+-----+-----+----- Sun Dec 31 16:00:00 2017 UTC | 40 | 10 | 10 Wed Oct 31 16:00:00 2018 UTC | 70 | 10 | 10 TRUNCATE transactions_montly; -- Partial refresh the CAGG from beginning to an specific timestamp CALL refresh_continuous_aggregate('transactions_montly', NULL, '2018-11-01 11:50:00-08'::timestamptz); SELECT * FROM transactions_montly ORDER BY bucket; bucket | sum | max | min ------------------------------+-----+-----+----- Sun Dec 31 16:00:00 2017 UTC | 40 | 10 | 10 -- Full refresh the CAGG CALL refresh_continuous_aggregate('transactions_montly', NULL, NULL); SELECT * FROM transactions_montly ORDER BY bucket; bucket | sum | max | min ------------------------------+-----+-----+----- Sun Dec 31 16:00:00 2017 UTC | 40 | 10 | 10 Wed Oct 31 16:00:00 2018 UTC | 70 | 10 | 10 -- Check set_chunk_time_interval on continuous aggregate CREATE MATERIALIZED VIEW cagg_set_chunk_time_interval WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(INTERVAL '1 month', time) AS bucket, SUM(fiat_value), MAX(fiat_value), MIN(fiat_value) FROM transactions GROUP BY 1 WITH NO DATA; SELECT set_chunk_time_interval('cagg_set_chunk_time_interval', chunk_time_interval => interval '1 month'); set_chunk_time_interval ------------------------- CALL refresh_continuous_aggregate('cagg_set_chunk_time_interval', NULL, NULL); SELECT _timescaledb_functions.to_interval(d.interval_length) = interval '1 month' FROM _timescaledb_catalog.dimension d RIGHT JOIN _timescaledb_catalog.continuous_agg ca ON ca.user_view_name = 'cagg_set_chunk_time_interval' WHERE d.hypertable_id = ca.mat_hypertable_id; ?column? ---------- t -- Since #6077 CAggs are materialized only by default DROP TABLE conditions CASCADE; NOTICE: drop cascades to 5 other objects NOTICE: drop cascades to 2 other objects CREATE TABLE conditions ( time TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL ); SELECT create_hypertable('conditions', 'time'); create_hypertable -------------------------- (53,public,conditions,t) INSERT INTO conditions VALUES ( '2018-01-01 09:20:00-08', 'SFO', 55); INSERT INTO conditions VALUES ( '2018-01-02 09:30:00-08', 'POR', 100); INSERT INTO conditions VALUES ( '2018-01-02 09:20:00-08', 'SFO', 65); INSERT INTO conditions VALUES ( '2018-01-02 09:10:00-08', 'NYC', 65); INSERT INTO conditions VALUES ( '2018-11-01 09:20:00-08', 'NYC', 45); INSERT INTO conditions VALUES ( '2018-11-01 10:40:00-08', 'NYC', 55); INSERT INTO conditions VALUES ( '2018-11-01 11:50:00-08', 'NYC', 65); INSERT INTO conditions VALUES ( '2018-11-01 12:10:00-08', 'NYC', 75); INSERT INTO conditions VALUES ( '2018-11-01 13:10:00-08', 'NYC', 85); INSERT INTO conditions VALUES ( '2018-11-02 09:20:00-08', 'NYC', 10); INSERT INTO conditions VALUES ( '2018-11-02 10:30:00-08', 'NYC', 20); CREATE MATERIALIZED VIEW conditions_daily WITH (timescaledb.continuous) AS SELECT location, time_bucket(INTERVAL '1 day', time) AS bucket, AVG(temperature) FROM conditions GROUP BY location, bucket WITH NO DATA; \d+ conditions_daily View "public.conditions_daily" Column | Type | Collation | Nullable | Default | Storage | Description ----------+--------------------------+-----------+----------+---------+----------+------------- location | text | | | | extended | bucket | timestamp with time zone | | | | plain | avg | double precision | | | | plain | View definition: SELECT location, bucket, avg FROM _timescaledb_internal._materialized_hypertable_54; -- Should return NO ROWS SELECT * FROM conditions_daily ORDER BY bucket, location; location | bucket | avg ----------+--------+----- ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only=false); \d+ conditions_daily View "public.conditions_daily" Column | Type | Collation | Nullable | Default | Storage | Description ----------+--------------------------+-----------+----------+---------+----------+------------- location | text | | | | extended | bucket | timestamp with time zone | | | | plain | avg | double precision | | | | plain | View definition: SELECT _materialized_hypertable_54.location, _materialized_hypertable_54.bucket, _materialized_hypertable_54.avg FROM _timescaledb_internal._materialized_hypertable_54 WHERE _materialized_hypertable_54.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(54)), '-infinity'::timestamp with time zone) UNION ALL SELECT conditions.location, time_bucket('@ 1 day'::interval, conditions."time") AS bucket, avg(conditions.temperature) AS avg FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(54)), '-infinity'::timestamp with time zone) GROUP BY conditions.location, (time_bucket('@ 1 day'::interval, conditions."time")); -- Should return ROWS because now it is realtime SELECT * FROM conditions_daily ORDER BY bucket, location; location | bucket | avg ----------+------------------------------+----- SFO | Sun Dec 31 16:00:00 2017 UTC | 55 NYC | Mon Jan 01 16:00:00 2018 UTC | 65 POR | Mon Jan 01 16:00:00 2018 UTC | 100 SFO | Mon Jan 01 16:00:00 2018 UTC | 65 NYC | Wed Oct 31 16:00:00 2018 UTC | 65 NYC | Thu Nov 01 16:00:00 2018 UTC | 15 -- Should return ROWS because we refreshed it ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only=true); \d+ conditions_daily View "public.conditions_daily" Column | Type | Collation | Nullable | Default | Storage | Description ----------+--------------------------+-----------+----------+---------+----------+------------- location | text | | | | extended | bucket | timestamp with time zone | | | | plain | avg | double precision | | | | plain | View definition: SELECT location, bucket, avg FROM _timescaledb_internal._materialized_hypertable_54; CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); SELECT * FROM conditions_daily ORDER BY bucket, location; location | bucket | avg ----------+------------------------------+----- SFO | Sun Dec 31 16:00:00 2017 UTC | 55 NYC | Mon Jan 01 16:00:00 2018 UTC | 65 POR | Mon Jan 01 16:00:00 2018 UTC | 100 SFO | Mon Jan 01 16:00:00 2018 UTC | 65 NYC | Wed Oct 31 16:00:00 2018 UTC | 65 NYC | Thu Nov 01 16:00:00 2018 UTC | 15 -- Test TRUNCATE over a Realtime CAgg DROP MATERIALIZED VIEW conditions_daily; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW conditions_daily WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT location, time_bucket(INTERVAL '1 day', time) AS bucket, AVG(temperature) FROM conditions GROUP BY location, bucket WITH NO DATA; SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'conditions_daily' \gset -- Check the current watermark for an empty CAgg SELECT _timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)) AS watermak_empty_cagg; watermak_empty_cagg --------------------------------- Sun Nov 23 16:00:00 4714 UTC BC -- Refresh the CAGG CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); -- Check the watermark after the refresh and before truncate the CAgg SELECT _timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)) AS watermak_before; watermak_before ------------------------------ Fri Nov 02 16:00:00 2018 UTC -- Exists chunks before truncate the cagg (> 0) SELECT count(*) FROM show_chunks('conditions_daily'); count ------- 2 -- Truncate the given CAgg, it should reset the watermark to the empty state TRUNCATE conditions_daily; -- No chunks remains after truncate the cagg (= 0) SELECT count(*) FROM show_chunks('conditions_daily'); count ------- 0 -- Watermark should be reseted SELECT _timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)) AS watermak_after; watermak_after --------------------------------- Sun Nov 23 16:00:00 4714 UTC BC -- Should return ROWS because the watermark was reseted by the TRUNCATE SELECT * FROM conditions_daily ORDER BY bucket, location; location | bucket | avg ----------+------------------------------+----- SFO | Sun Dec 31 16:00:00 2017 UTC | 55 NYC | Mon Jan 01 16:00:00 2018 UTC | 65 POR | Mon Jan 01 16:00:00 2018 UTC | 100 SFO | Mon Jan 01 16:00:00 2018 UTC | 65 NYC | Wed Oct 31 16:00:00 2018 UTC | 65 NYC | Thu Nov 01 16:00:00 2018 UTC | 15 -- check compression settings are cleaned up when deleting a cagg with compression CREATE TABLE cagg_cleanup(time timestamptz not null); SELECT table_name FROM create_hypertable('cagg_cleanup','time'); table_name -------------- cagg_cleanup INSERT INTO cagg_cleanup SELECT '2020-01-01'; CREATE MATERIALIZED VIEW cagg1 WITH (timescaledb.continuous) AS SELECT time_bucket('1h',time) FROM cagg_cleanup GROUP BY 1; NOTICE: refreshing continuous aggregate "cagg1" ALTER MATERIALIZED VIEW cagg1 SET (timescaledb.compress); NOTICE: defaulting compress_orderby to time_bucket SELECT count(compress_chunk(ch)) FROM show_chunks('cagg1') ch; count ------- 1 DROP MATERIALIZED VIEW cagg1; NOTICE: drop cascades to table _timescaledb_internal._hyper_57_70_chunk SELECT * FROM _timescaledb_catalog.compression_settings; relid | compress_relid | segmentby | orderby | orderby_desc | orderby_nullsfirst | index -------+----------------+-----------+---------+--------------+--------------------+------- -- test WITH namespace alias CREATE TABLE with_alias(time timestamptz not null); CREATE MATERIALIZED VIEW cagg_alias WITH (tsdb.continuous, tsdb.materialized_only=false) AS SELECT time_bucket(INTERVAL '1 day', time) FROM conditions GROUP BY 1 WITH NO DATA; ALTER MATERIALIZED VIEW cagg_alias SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_alias SET (tsdb.materialized_only=false); DROP MATERIALIZED VIEW cagg_alias; -- test SET chunk_time_interval CREATE MATERIALIZED VIEW cagg_set WITH (tsdb.continuous, tsdb.chunk_interval='1day') AS SELECT time_bucket(INTERVAL '1 day', time) AS cagg_interval_setter FROM conditions GROUP BY 1 WITH NO DATA; SELECT column_name, time_interval FROM timescaledb_information.dimensions WHERE column_name='cagg_interval_setter'; column_name | time_interval ----------------------+--------------- cagg_interval_setter | @ 1 day ALTER MATERIALIZED VIEW cagg_set SET (tsdb.chunk_interval='23 day'); SELECT column_name, time_interval FROM timescaledb_information.dimensions WHERE column_name='cagg_interval_setter'; column_name | time_interval ----------------------+--------------- cagg_interval_setter | @ 23 days ALTER MATERIALIZED VIEW cagg_set SET (tsdb.chunk_interval='6 month'); SELECT column_name, time_interval FROM timescaledb_information.dimensions WHERE column_name='cagg_interval_setter'; column_name | time_interval ----------------------+--------------- cagg_interval_setter | @ 180 days ALTER MATERIALIZED VIEW cagg_set SET (tsdb.chunk_interval='1 year'); SELECT column_name, time_interval FROM timescaledb_information.dimensions WHERE column_name='cagg_interval_setter'; column_name | time_interval ----------------------+--------------- cagg_interval_setter | @ 360 days -- test cagg with stable functions CREATE MATERIALIZED VIEW cagg_stable WITH (tsdb.continuous) AS SELECT sum(temperature), max(time + INTERVAL '1h') FROM conditions GROUP BY time_bucket('1week', time), location; WARNING: using non-immutable functions in continuous aggregate view may lead to inconsistent results on rematerialization NOTICE: refreshing continuous aggregate "cagg_stable" SELECT * FROM cagg_stable t ORDER BY t; sum | max -----+------------------------------ 65 | Tue Jan 02 10:10:00 2018 UTC 100 | Tue Jan 02 10:30:00 2018 UTC 120 | Tue Jan 02 10:20:00 2018 UTC 355 | Fri Nov 02 11:30:00 2018 UTC --aggregate without combine function but stable function CREATE MATERIALIZED VIEW cagg_json_agg WITH (tsdb.continuous, tsdb.materialized_only=false) AS SELECT json_agg(location) from conditions group by time_bucket('1week', time), location WITH NO DATA; WARNING: using non-immutable functions in continuous aggregate view may lead to inconsistent results on rematerialization CREATE FUNCTION test_stablefunc(int) RETURNS int LANGUAGE 'sql' STABLE AS 'SELECT $1 + 10'; CREATE MATERIALIZED VIEW cagg_stable2 WITH (tsdb.continuous) AS SELECT sum(test_stablefunc(temperature::int)), min(location) FROM conditions GROUP BY time_bucket('1week', time) WITH NO DATA; WARNING: using non-immutable functions in continuous aggregate view may lead to inconsistent results on rematerialization CREATE MATERIALIZED VIEW cagg_stable3 WITH (tsdb.continuous) AS SELECT sum(temperature), min(location) FROM conditions GROUP BY time_bucket('1week', time), test_stablefunc(temperature::int) WITH NO DATA; WARNING: using non-immutable functions in continuous aggregate view may lead to inconsistent results on rematerialization -- test window functions in caggs -- first do sanity check that we error without the GUC \set ON_ERROR_STOP 0 CREATE MATERIALIZED VIEW cagg_window WITH (tsdb.continuous) AS SELECT time_bucket('1week', time), rank() OVER (PARTITION BY time_bucket('1 week',time)) FROM conditions GROUP BY 1; ERROR: invalid continuous aggregate query \set ON_ERROR_STOP 1 SET timescaledb.enable_cagg_window_functions TO on; CREATE MATERIALIZED VIEW cagg_window_1 WITH (tsdb.continuous) AS SELECT time_bucket('1week', time), rank() OVER (PARTITION BY time_bucket('1 week',time)) FROM conditions GROUP BY 1; WARNING: window function support is experimental and may result in unexpected results depending on the functions used. NOTICE: refreshing continuous aggregate "cagg_window_1" CREATE MATERIALIZED VIEW cagg_window_2 WITH (tsdb.continuous) AS SELECT time_bucket('1week', time), rank() OVER (PARTITION BY time_bucket('1 week',time), location) FROM conditions GROUP BY 1, location; WARNING: window function support is experimental and may result in unexpected results depending on the functions used. NOTICE: refreshing continuous aggregate "cagg_window_2" CREATE MATERIALIZED VIEW cagg_window_3 WITH (tsdb.continuous) AS SELECT time_bucket('1week', time), rank() OVER (PARTITION BY time_bucket('1 week',time)) FROM conditions GROUP BY 1, location; WARNING: window function support is experimental and may result in unexpected results depending on the functions used. NOTICE: refreshing continuous aggregate "cagg_window_3" CREATE MATERIALIZED VIEW cagg_window_4 WITH (tsdb.continuous) AS SELECT time_bucket('1week', time), rank() OVER w FROM conditions GROUP BY 1, location WINDOW w AS (PARTITION BY time_bucket('1 week',time)); WARNING: window function support is experimental and may result in unexpected results depending on the functions used. NOTICE: refreshing continuous aggregate "cagg_window_4" -- test setting chunk_interval on a cagg CREATE MATERIALIZED VIEW cagg_chunk_interval WITH (tsdb.continuous, tsdb.chunk_interval='1000 day') AS SELECT time_bucket('1 week', time) FROM conditions GROUP BY 1 WITH NO DATA; SELECT time_interval from timescaledb_information.continuous_aggregates cagg INNER JOIN timescaledb_information.dimensions dim ON cagg.materialization_hypertable_name = dim.hypertable_name WHERE view_name='cagg_chunk_interval'; time_interval --------------- @ 1000 days ALTER MATERIALIZED VIEW cagg_chunk_interval SET (tsdb.chunk_interval='110 day'); SELECT time_interval from timescaledb_information.continuous_aggregates cagg INNER JOIN timescaledb_information.dimensions dim ON cagg.materialization_hypertable_name = dim.hypertable_name WHERE view_name='cagg_chunk_interval'; time_interval --------------- @ 110 days -- test columnstore options CREATE MATERIALIZED VIEW columnstore_options WITH (tsdb.continuous, tsdb.chunk_interval='1 day') AS SELECT time_bucket('1 day', time) FROM conditions GROUP BY 1 WITH NO DATA; SELECT column_name, compress_interval_length from _timescaledb_catalog.dimension where column_name='time_bucket' ORDER BY id DESC LIMIT 1; column_name | compress_interval_length -------------+-------------------------- time_bucket | ALTER MATERIALIZED VIEW columnstore_options SET (tsdb.columnstore, tsdb.chunk_interval='1 day', tsdb.orderby='time_bucket DESC', tsdb.compress_chunk_interval='2 day'); SELECT column_name, compress_interval_length from _timescaledb_catalog.dimension where column_name='time_bucket' ORDER BY id DESC LIMIT 1; column_name | compress_interval_length -------------+-------------------------- time_bucket | 172800000000 ALTER MATERIALIZED VIEW columnstore_options SET (tsdb.compress_chunk_interval='3 day'); SELECT column_name, compress_interval_length from _timescaledb_catalog.dimension where column_name='time_bucket' ORDER BY id DESC LIMIT 1; column_name | compress_interval_length -------------+-------------------------- time_bucket | 259200000000 ALTER MATERIALIZED VIEW columnstore_options SET (tsdb.compress_chunk_time_interval='4 day'); SELECT column_name, compress_interval_length from _timescaledb_catalog.dimension where column_name='time_bucket' ORDER BY id DESC LIMIT 1; column_name | compress_interval_length -------------+-------------------------- time_bucket | 345600000000 -- test set returning functions in caggs CREATE TABLE kpis_raw (time TIMESTAMP NOT NULL, value INTEGER, groups TEXT[]) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column INSERT INTO kpis_raw (time, value, groups) VALUES ('2025-01-01', 10, '{group1,group2,group3}'), ('2025-01-02', 20, '{group1,group4}'), ('2025-02-01', 10, '{group1,group3}'), ('2025-02-01', 20, '{group1,group4}'); CREATE MATERIALIZED VIEW kpis_cagg WITH (tsdb.continuous) AS SELECT time_bucket('7 day', time) AS bucket, count(*) AS number_of_records, avg(value) AS average, unnest(groups) AS kpi_group FROM kpis_raw GROUP BY bucket, kpi_group; NOTICE: refreshing continuous aggregate "kpis_cagg" SELECT * FROM kpis_cagg ORDER BY bucket, kpi_group; bucket | number_of_records | average | kpi_group --------------------------+-------------------+---------------------+----------- Mon Dec 30 00:00:00 2024 | 2 | 15.0000000000000000 | group1 Mon Dec 30 00:00:00 2024 | 1 | 10.0000000000000000 | group2 Mon Dec 30 00:00:00 2024 | 1 | 10.0000000000000000 | group3 Mon Dec 30 00:00:00 2024 | 1 | 20.0000000000000000 | group4 Mon Jan 27 00:00:00 2025 | 2 | 15.0000000000000000 | group1 Mon Jan 27 00:00:00 2025 | 1 | 10.0000000000000000 | group3 Mon Jan 27 00:00:00 2025 | 1 | 20.0000000000000000 | group4 --TEST for caggs with non timescaledb namespace options --non timescaledb namespace options can be set via ALTER \set ON_ERROR_STOP 0 -- will error out CREATE MATERIALIZED VIEW ht_try_weekly WITH (timescaledb.continuous, tigerlake.newoption = true) AS SELECT time_bucket(interval '1 week', time) AS ts_bucket, avg(value) FROM kpis_raw GROUP BY 1; ERROR: non "timescaledb" namespace options can be set only via ALTER --caught by Postgres now ALTER MATERIALIZED VIEW kpis_cagg SET (tigerlake.newoption = true); ERROR: "kpis_cagg" is not a materialized view \set ON_ERROR_STOP 1 -- TEST that cached alter stmt still works (see PR 8739) -- test DDL inside function CREATE TABLE hypertab_ddl( ts timestamp, a integer) WITH (timescaledb.hypertable); NOTICE: using column "ts" as partitioning column CREATE OR REPLACE FUNCTION ddl_function() RETURNS VOID LANGUAGE PLPGSQL AS $$ BEGIN DROP MATERIALIZED VIEW IF EXISTS cagg_hypertab_ddl; CREATE MATERIALIZED VIEW cagg_hypertab_ddl WITH (timescaledb.continuous) AS SELECT time_bucket( '1 day'::interval, ts), COUNT(*) FROM hypertab_ddl GROUP BY 1 WITH NO DATA; END $$; SELECT ddl_function(); NOTICE: materialized view "cagg_hypertab_ddl" does not exist, skipping ddl_function -------------- SELECT view_name from timescaledb_information.continuous_aggregates WHERE hypertable_name='hypertab_ddl'; view_name ------------------- cagg_hypertab_ddl SELECT ddl_function(); ddl_function -------------- SELECT view_name from timescaledb_information.continuous_aggregates WHERE hypertable_name='hypertab_ddl'; view_name ------------------- cagg_hypertab_ddl -- TEST continuous aggregate with functionally dependent columns -- name column depends on id which is in GROUP BY so name doesnt have to be CREATE table sensor(id int PRIMARY KEY, name text); CREATE TABLE sensordata(time timestamptz,id int, value float) WITH (timescaledb.hypertable); NOTICE: using column "time" as partitioning column CREATE MATERIALIZED VIEW cagg_sensordata WITH (tsdb.continuous) AS SELECT s.id, s.name,time_bucket('15 minutes', sd.time) as bucket, avg(sd.value) FROM sensordata sd JOIN sensor s USING(id) GROUP BY s.id, bucket WITH NO DATA; SELECT * FROM cagg_sensordata; id | name | bucket | avg ----+------+--------+----- ================================================ FILE: tsl/test/expected/cagg_ddl-17.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- Set this variable to avoid using a hard-coded path each time query -- results are compared \set QUERY_RESULT_TEST_EQUAL_RELPATH '../../../../test/sql/include/query_result_test_equal.sql' SET ROLE :ROLE_DEFAULT_PERM_USER; --DDL commands on continuous aggregates CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature integer NULL, humidity DOUBLE PRECISION NULL, timemeasure TIMESTAMPTZ, timeinterval INTERVAL ); SELECT table_name FROM create_hypertable('conditions', 'timec'); table_name ------------ conditions -- schema tests \c :TEST_DBNAME :ROLE_SUPERUSER SET timezone TO 'UTC+8'; -- drop if the tablespace1 and/or tablespace2 exists SET client_min_messages TO error; DROP TABLESPACE IF EXISTS tablespace1; DROP TABLESPACE IF EXISTS tablespace2; RESET client_min_messages; CREATE TABLESPACE tablespace1 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE1_PATH; CREATE TABLESPACE tablespace2 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE2_PATH; CREATE SCHEMA rename_schema; GRANT ALL ON SCHEMA rename_schema TO :ROLE_DEFAULT_PERM_USER; CREATE SCHEMA test_schema AUTHORIZATION :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE TABLE foo(time TIMESTAMPTZ NOT NULL, data INTEGER); SELECT create_hypertable('foo', 'time'); create_hypertable ------------------- (2,public,foo,t) CREATE MATERIALIZED VIEW rename_test_old WITH ( timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1week', time), COUNT(data) FROM foo GROUP BY 1 WITH NO DATA; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+-----------------+-----------------------+------------------- public | rename_test_old | _timescaledb_internal | _partial_view_3 ALTER TABLE rename_test_old RENAME TO rename_test; ALTER TABLE rename_test SET SCHEMA test_schema; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+-----------------------+------------------- test_schema | rename_test | _timescaledb_internal | _partial_view_3 ALTER MATERIALIZED VIEW test_schema.rename_test SET SCHEMA rename_schema; DROP SCHEMA test_schema; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+-----------------------+------------------- rename_schema | rename_test | _timescaledb_internal | _partial_view_3 SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA", direct_view_name as "DIR_VIEW_NAME", direct_view_schema as "DIR_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'rename_test' \gset RESET ROLE; SELECT current_user; current_user -------------- super_user ALTER VIEW :"PART_VIEW_SCHEMA".:"PART_VIEW_NAME" SET SCHEMA public; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+---------------------+------------------- rename_schema | rename_test | public | _partial_view_3 --alter direct view schema SELECT user_view_schema, user_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | direct_view_schema | direct_view_name ------------------+----------------+-----------------------+------------------ rename_schema | rename_test | _timescaledb_internal | _direct_view_3 RESET ROLE; SELECT current_user; current_user -------------- super_user ALTER VIEW :"DIR_VIEW_SCHEMA".:"DIR_VIEW_NAME" SET SCHEMA public; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name ------------------+----------------+---------------------+-------------------+--------------------+------------------ rename_schema | rename_test | public | _partial_view_3 | public | _direct_view_3 RESET ROLE; SELECT current_user; current_user -------------- super_user ALTER SCHEMA rename_schema RENAME TO new_name_schema; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name ------------------+----------------+---------------------+-------------------+--------------------+------------------ new_name_schema | rename_test | public | _partial_view_3 | public | _direct_view_3 ALTER VIEW :"PART_VIEW_NAME" SET SCHEMA new_name_schema; ALTER VIEW :"DIR_VIEW_NAME" SET SCHEMA new_name_schema; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name ------------------+----------------+---------------------+-------------------+--------------------+------------------ new_name_schema | rename_test | new_name_schema | _partial_view_3 | new_name_schema | _direct_view_3 RESET ROLE; SELECT current_user; current_user -------------- super_user ALTER SCHEMA new_name_schema RENAME TO foo_name_schema; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+---------------------+------------------- foo_name_schema | rename_test | foo_name_schema | _partial_view_3 ALTER MATERIALIZED VIEW foo_name_schema.rename_test SET SCHEMA public; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+---------------------+------------------- public | rename_test | foo_name_schema | _partial_view_3 RESET ROLE; SELECT current_user; current_user -------------- super_user ALTER SCHEMA foo_name_schema RENAME TO rename_schema; SET ROLE :ROLE_DEFAULT_PERM_USER; SET client_min_messages TO NOTICE; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+---------------------+------------------- public | rename_test | rename_schema | _partial_view_3 ALTER MATERIALIZED VIEW rename_test RENAME TO rename_c_aggregate; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+--------------------+---------------------+------------------- public | rename_c_aggregate | rename_schema | _partial_view_3 SELECT * FROM rename_c_aggregate; time_bucket | count -------------+------- ALTER VIEW rename_schema.:"PART_VIEW_NAME" RENAME TO partial_view; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name ------------------+--------------------+---------------------+-------------------+--------------------+------------------ public | rename_c_aggregate | rename_schema | partial_view | rename_schema | _direct_view_3 --rename direct view ALTER VIEW rename_schema.:"DIR_VIEW_NAME" RENAME TO direct_view; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name ------------------+--------------------+---------------------+-------------------+--------------------+------------------ public | rename_c_aggregate | rename_schema | partial_view | rename_schema | direct_view -- drop_chunks tests DROP TABLE conditions CASCADE; DROP TABLE foo CASCADE; NOTICE: drop cascades to 2 other objects CREATE TABLE drop_chunks_table(time BIGINT NOT NULL, data INTEGER); SELECT hypertable_id AS drop_chunks_table_id FROM create_hypertable('drop_chunks_table', 'time', chunk_time_interval => 10) \gset CREATE OR REPLACE FUNCTION integer_now_test() returns bigint LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), bigint '0') FROM drop_chunks_table $$; SELECT set_integer_now_func('drop_chunks_table', 'integer_now_test'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW drop_chunks_view WITH ( timescaledb.continuous, timescaledb.materialized_only=true ) AS SELECT time_bucket('5', time), COUNT(data) FROM drop_chunks_table GROUP BY 1 WITH NO DATA; SELECT format('%I.%I', schema_name, table_name) AS drop_chunks_mat_table, schema_name AS drop_chunks_mat_schema, table_name AS drop_chunks_mat_table_name FROM _timescaledb_catalog.hypertable, _timescaledb_catalog.continuous_agg WHERE _timescaledb_catalog.continuous_agg.raw_hypertable_id = :drop_chunks_table_id AND _timescaledb_catalog.hypertable.id = _timescaledb_catalog.continuous_agg.mat_hypertable_id \gset -- create 3 chunks, with 3 time bucket INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(0, 29) AS i; -- Only refresh up to bucket 15 initially. Matches the old refresh -- behavior that didn't materialize everything CALL refresh_continuous_aggregate('drop_chunks_view', 0, 15); SELECT count(c) FROM show_chunks('drop_chunks_table') AS c; count ------- 3 SELECT count(c) FROM show_chunks('drop_chunks_view') AS c; count ------- 1 SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | count -------------+------- 0 | 5 5 | 5 10 | 5 -- cannot drop directly from the materialization table without specifying -- cont. aggregate view name explicitly \set ON_ERROR_STOP 0 SELECT drop_chunks(:'drop_chunks_mat_table', newer_than => -20, verbose => true); ERROR: operation not supported on materialized hypertable \set ON_ERROR_STOP 1 SELECT count(c) FROM show_chunks('drop_chunks_table') AS c; count ------- 3 SELECT count(c) FROM show_chunks('drop_chunks_view') AS c; count ------- 1 SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | count -------------+------- 0 | 5 5 | 5 10 | 5 -- drop chunks when the chunksize and time_bucket aren't aligned DROP TABLE drop_chunks_table CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_5_4_chunk CREATE TABLE drop_chunks_table_u(time BIGINT NOT NULL, data INTEGER); SELECT hypertable_id AS drop_chunks_table_u_id FROM create_hypertable('drop_chunks_table_u', 'time', chunk_time_interval => 7) \gset CREATE OR REPLACE FUNCTION integer_now_test1() returns bigint LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), bigint '0') FROM drop_chunks_table_u $$; SELECT set_integer_now_func('drop_chunks_table_u', 'integer_now_test1'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW drop_chunks_view WITH ( timescaledb.continuous, timescaledb.materialized_only=true ) AS SELECT time_bucket('3', time), COUNT(data) FROM drop_chunks_table_u GROUP BY 1 WITH NO DATA; SELECT format('%I.%I', schema_name, table_name) AS drop_chunks_mat_table_u, schema_name AS drop_chunks_mat_schema, table_name AS drop_chunks_mat_table_u_name FROM _timescaledb_catalog.hypertable, _timescaledb_catalog.continuous_agg WHERE _timescaledb_catalog.continuous_agg.raw_hypertable_id = :drop_chunks_table_u_id AND _timescaledb_catalog.hypertable.id = _timescaledb_catalog.continuous_agg.mat_hypertable_id \gset -- create 3 chunks, with 3 time bucket INSERT INTO drop_chunks_table_u SELECT i, i FROM generate_series(0, 21) AS i; -- Refresh up to bucket 15 to match old materializer behavior CALL refresh_continuous_aggregate('drop_chunks_view', 0, 15); SELECT count(c) FROM show_chunks('drop_chunks_table_u') AS c; count ------- 4 SELECT count(c) FROM show_chunks('drop_chunks_view') AS c; count ------- 1 SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | count -------------+------- 0 | 3 3 | 3 6 | 3 9 | 3 12 | 3 -- TRUNCATE test -- Can truncate regular hypertables that have caggs TRUNCATE drop_chunks_table_u; \set ON_ERROR_STOP 0 -- Can't truncate materialized hypertables directly TRUNCATE :drop_chunks_mat_table_u; ERROR: cannot TRUNCATE a hypertable underlying a continuous aggregate \set ON_ERROR_STOP 1 -- Check that we don't interfere with TRUNCATE of normal table and -- partitioned table CREATE TABLE truncate (value int); INSERT INTO truncate VALUES (1), (2); TRUNCATE truncate; SELECT * FROM truncate; value ------- CREATE TABLE truncate_partitioned (value int) PARTITION BY RANGE(value); CREATE TABLE truncate_p1 PARTITION OF truncate_partitioned FOR VALUES FROM (1) TO (3); INSERT INTO truncate_partitioned VALUES (1), (2); TRUNCATE truncate_partitioned; SELECT * FROM truncate_partitioned; value ------- -- ALTER TABLE tests \set ON_ERROR_STOP 0 -- test a variety of ALTER TABLE statements ALTER TABLE :drop_chunks_mat_table_u RENAME time_bucket TO bad_name; ERROR: renaming columns on materialization tables is not supported ALTER TABLE :drop_chunks_mat_table_u ADD UNIQUE(time_bucket); ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u SET UNLOGGED; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u ENABLE ROW LEVEL SECURITY; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u ADD COLUMN fizzle INTEGER; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u DROP COLUMN time_bucket; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u ALTER COLUMN time_bucket DROP NOT NULL; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u ALTER COLUMN time_bucket SET DEFAULT 1; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u ALTER COLUMN time_bucket SET STORAGE EXTERNAL; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u DISABLE TRIGGER ALL; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u SET TABLESPACE foo; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u NOT OF; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u OWNER TO CURRENT_USER; ERROR: operation not supported on materialization tables \set ON_ERROR_STOP 1 ALTER TABLE :drop_chunks_mat_table_u SET SCHEMA public; ALTER TABLE :drop_chunks_mat_table_u_name RENAME TO new_name; SET ROLE :ROLE_DEFAULT_PERM_USER; SET client_min_messages TO NOTICE; SELECT * FROM new_name ORDER BY 1; time_bucket | count -------------+------- 0 | 3 3 | 3 6 | 3 9 | 3 12 | 3 SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | count -------------+------- 0 | 3 3 | 3 6 | 3 9 | 3 12 | 3 \set ON_ERROR_STOP 0 -- no continuous aggregates on a continuous aggregate materialization table CREATE MATERIALIZED VIEW new_name_view WITH ( timescaledb.continuous, timescaledb.materialized_only=true ) AS SELECT time_bucket('6', time_bucket), COUNT("count") FROM new_name GROUP BY 1 WITH NO DATA; ERROR: hypertable is a continuous aggregate materialization table \set ON_ERROR_STOP 1 CREATE TABLE metrics(time timestamptz NOT NULL, device_id int, v1 float, v2 float); SELECT create_hypertable('metrics','time'); create_hypertable ---------------------- (8,public,metrics,t) INSERT INTO metrics SELECT generate_series('2000-01-01'::timestamptz,'2000-01-10','1m'),1,0.25,0.75; -- check expressions in view definition CREATE MATERIALIZED VIEW cagg_expr WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1d', time) AS time, 'Const'::text AS Const, 4.3::numeric AS "numeric", first(metrics,time), CASE WHEN true THEN 'foo' ELSE 'bar' END, COALESCE(NULL,'coalesce'), avg(v1) + avg(v2) AS avg1, avg(v1+v2) AS avg2 FROM metrics GROUP BY 1 WITH NO DATA; CALL refresh_continuous_aggregate('cagg_expr', NULL, NULL); SELECT * FROM cagg_expr ORDER BY time LIMIT 5; time | const | numeric | first | case | coalesce | avg1 | avg2 ------------------------------+-------+---------+----------------------------------------------+------+----------+------+------ Fri Dec 31 16:00:00 1999 UTC | Const | 4.3 | ("Sat Jan 01 00:00:00 2000 UTC",1,0.25,0.75) | foo | coalesce | 1 | 1 Sat Jan 01 16:00:00 2000 UTC | Const | 4.3 | ("Sat Jan 01 16:00:00 2000 UTC",1,0.25,0.75) | foo | coalesce | 1 | 1 Sun Jan 02 16:00:00 2000 UTC | Const | 4.3 | ("Sun Jan 02 16:00:00 2000 UTC",1,0.25,0.75) | foo | coalesce | 1 | 1 Mon Jan 03 16:00:00 2000 UTC | Const | 4.3 | ("Mon Jan 03 16:00:00 2000 UTC",1,0.25,0.75) | foo | coalesce | 1 | 1 Tue Jan 04 16:00:00 2000 UTC | Const | 4.3 | ("Tue Jan 04 16:00:00 2000 UTC",1,0.25,0.75) | foo | coalesce | 1 | 1 --test materialization of invalidation before drop DROP TABLE IF EXISTS drop_chunks_table CASCADE; NOTICE: table "drop_chunks_table" does not exist, skipping DROP TABLE IF EXISTS drop_chunks_table_u CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_7_9_chunk CREATE TABLE drop_chunks_table(time BIGINT NOT NULL, data INTEGER); SELECT hypertable_id AS drop_chunks_table_nid FROM create_hypertable('drop_chunks_table', 'time', chunk_time_interval => 10) \gset CREATE OR REPLACE FUNCTION integer_now_test2() returns bigint LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), bigint '0') FROM drop_chunks_table $$; SELECT set_integer_now_func('drop_chunks_table', 'integer_now_test2'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW drop_chunks_view WITH ( timescaledb.continuous, timescaledb.materialized_only=true ) AS SELECT time_bucket('5', time), max(data) FROM drop_chunks_table GROUP BY 1 WITH NO DATA; INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(0, 20) AS i; --dropping chunks will process the invalidations SELECT drop_chunks('drop_chunks_table', older_than => (integer_now_test2()-9)); drop_chunks ------------------------------------------ _timescaledb_internal._hyper_10_13_chunk SELECT * FROM drop_chunks_table ORDER BY time ASC limit 1; time | data ------+------ 10 | 10 INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(20, 35) AS i; CALL refresh_continuous_aggregate('drop_chunks_view', 10, 40); --this will be seen after the drop its within the invalidation window and will be dropped INSERT INTO drop_chunks_table VALUES (26, 100); --this will not be processed by the drop since chunk 30-39 is not dropped but will be seen after refresh --shows that the drop doesn't do more work than necessary INSERT INTO drop_chunks_table VALUES (31, 200); --move the time up to 39 INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(35, 39) AS i; --the chunks and ranges we have thus far SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table'; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_10_14_chunk | 10 | 20 _hyper_10_15_chunk | 20 | 30 _hyper_10_16_chunk | 30 | 40 --the invalidation on 25 not yet seen SELECT * FROM drop_chunks_view ORDER BY time_bucket DESC; time_bucket | max -------------+----- 35 | 35 30 | 34 25 | 29 20 | 24 15 | 19 10 | 14 --refresh to process the invalidations and then drop CALL refresh_continuous_aggregate('drop_chunks_view', NULL, (integer_now_test2()-9)); SELECT drop_chunks('drop_chunks_table', older_than => (integer_now_test2()-9)); drop_chunks ------------------------------------------ _timescaledb_internal._hyper_10_14_chunk _timescaledb_internal._hyper_10_15_chunk --new values on 25 now seen in view SELECT * FROM drop_chunks_view ORDER BY time_bucket DESC; time_bucket | max -------------+----- 35 | 35 30 | 34 25 | 100 20 | 24 15 | 19 10 | 14 --earliest datapoint now in table SELECT * FROM drop_chunks_table ORDER BY time ASC limit 1; time | data ------+------ 30 | 30 --still see data in the view SELECT * FROM drop_chunks_view WHERE time_bucket < (integer_now_test2()-9) ORDER BY time_bucket DESC; time_bucket | max -------------+----- 25 | 100 20 | 24 15 | 19 10 | 14 --no data but covers dropped chunks SELECT * FROM drop_chunks_table WHERE time < (integer_now_test2()-9) ORDER BY time DESC; time | data ------+------ --recreate the dropped chunk INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(0, 20) AS i; --see data from recreated region SELECT * FROM drop_chunks_table WHERE time < (integer_now_test2()-9) ORDER BY time DESC; time | data ------+------ 20 | 20 19 | 19 18 | 18 17 | 17 16 | 16 15 | 15 14 | 14 13 | 13 12 | 12 11 | 11 10 | 10 9 | 9 8 | 8 7 | 7 6 | 6 5 | 5 4 | 4 3 | 3 2 | 2 1 | 1 0 | 0 --should show chunk with old name and old ranges SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table' ORDER BY range_start_integer; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_10_18_chunk | 0 | 10 _hyper_10_19_chunk | 10 | 20 _hyper_10_20_chunk | 20 | 30 _hyper_10_16_chunk | 30 | 40 --We dropped everything up to the bucket starting at 30 and then --inserted new data up to and including time 20. Therefore, the --dropped data should stay the same as long as we only refresh --buckets that have non-dropped data. CALL refresh_continuous_aggregate('drop_chunks_view', 30, 40); SELECT * FROM drop_chunks_view ORDER BY time_bucket DESC; time_bucket | max -------------+----- 35 | 39 30 | 200 25 | 100 20 | 24 15 | 19 10 | 14 SELECT format('%I.%I', schema_name, table_name) AS drop_chunks_mat_tablen, schema_name AS drop_chunks_mat_schema, table_name AS drop_chunks_mat_table_name FROM _timescaledb_catalog.hypertable, _timescaledb_catalog.continuous_agg WHERE _timescaledb_catalog.continuous_agg.raw_hypertable_id = :drop_chunks_table_nid AND _timescaledb_catalog.hypertable.id = _timescaledb_catalog.continuous_agg.mat_hypertable_id \gset -- TEST drop chunks from continuous aggregates by specifying view name SELECT drop_chunks('drop_chunks_view', newer_than => -20, verbose => true); INFO: dropping chunk _timescaledb_internal._hyper_11_17_chunk drop_chunks ------------------------------------------ _timescaledb_internal._hyper_11_17_chunk -- Test that we cannot drop chunks when specifying materialized -- hypertable INSERT INTO drop_chunks_table SELECT generate_series(45, 55), 500; CALL refresh_continuous_aggregate('drop_chunks_view', 45, 55); SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = :'drop_chunks_mat_table_name' ORDER BY range_start_integer; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_11_23_chunk | 0 | 100 \set ON_ERROR_STOP 0 \set VERBOSITY default SELECT drop_chunks(:'drop_chunks_mat_tablen', older_than => 60); ERROR: operation not supported on materialized hypertable DETAIL: Hypertable "_materialized_hypertable_11" is a materialized hypertable. HINT: Try the operation on the continuous aggregate instead. \set VERBOSITY terse \set ON_ERROR_STOP 1 ----------------------------------------------------------------- -- Test that refresh_continuous_aggregate on chunk will refresh, -- but only in the regions covered by the show chunks. ----------------------------------------------------------------- SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table' ORDER BY 2,3; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_10_18_chunk | 0 | 10 _hyper_10_19_chunk | 10 | 20 _hyper_10_20_chunk | 20 | 30 _hyper_10_16_chunk | 30 | 40 _hyper_10_21_chunk | 40 | 50 _hyper_10_22_chunk | 50 | 60 -- Pick the second chunk as the one to drop WITH numbered_chunks AS ( SELECT row_number() OVER (ORDER BY range_start_integer), chunk_schema, chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table' ORDER BY 1 ) SELECT format('%I.%I', chunk_schema, chunk_name) AS chunk_to_drop, range_start_integer, range_end_integer FROM numbered_chunks WHERE row_number = 2 \gset -- There's data in the table for the chunk/range we will drop SELECT * FROM drop_chunks_table WHERE time >= :range_start_integer AND time < :range_end_integer ORDER BY 1; time | data ------+------ 10 | 10 11 | 11 12 | 12 13 | 13 14 | 14 15 | 15 16 | 16 17 | 17 18 | 18 19 | 19 -- Make sure there is also data in the continuous aggregate -- CARE: -- Note that this behaviour of dropping the materialization table chunks and expecting a refresh -- that overlaps that time range to NOT update those chunks is undefined. CALL refresh_continuous_aggregate('drop_chunks_view', 0, 50); SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | max -------------+----- 0 | 4 5 | 9 10 | 14 15 | 19 20 | 20 45 | 500 50 | 500 -- Drop the second chunk, to leave a gap in the data DROP TABLE :chunk_to_drop; -- Verify that the second chunk is dropped SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table' ORDER BY 2,3; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_10_18_chunk | 0 | 10 _hyper_10_20_chunk | 20 | 30 _hyper_10_16_chunk | 30 | 40 _hyper_10_21_chunk | 40 | 50 _hyper_10_22_chunk | 50 | 60 -- Data is no longer in the table but still in the view SELECT * FROM drop_chunks_table WHERE time >= :range_start_integer AND time < :range_end_integer ORDER BY 1; time | data ------+------ SELECT * FROM drop_chunks_view WHERE time_bucket >= :range_start_integer AND time_bucket < :range_end_integer ORDER BY 1; time_bucket | max -------------+----- 10 | 14 15 | 19 -- Insert a large value in one of the chunks that will be dropped INSERT INTO drop_chunks_table VALUES (:range_start_integer-1, 100); -- Now refresh and drop the two adjecent chunks CALL refresh_continuous_aggregate('drop_chunks_view', NULL, 30); SELECT drop_chunks('drop_chunks_table', older_than=>30); drop_chunks ------------------------------------------ _timescaledb_internal._hyper_10_18_chunk _timescaledb_internal._hyper_10_20_chunk -- Verify that the chunks are dropped SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table' ORDER BY 2,3; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_10_16_chunk | 30 | 40 _hyper_10_21_chunk | 40 | 50 _hyper_10_22_chunk | 50 | 60 -- The continuous aggregate should be refreshed in the regions covered -- by the dropped chunks, but not in the "gap" region, i.e., the -- region of the chunk that was dropped via DROP TABLE. SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | max -------------+----- 0 | 4 5 | 100 20 | 20 45 | 500 50 | 500 -- Now refresh in the region of the first two dropped chunks CALL refresh_continuous_aggregate('drop_chunks_view', 0, :range_end_integer); -- Aggregate data in the refreshed range should no longer exist since -- the underlying data was dropped. SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | max -------------+----- 20 | 20 45 | 500 50 | 500 -------------------------------------------------------------------- -- Check that we can create a materialized table in a tablespace. We -- create one with tablespace and one without and compare them. CREATE VIEW cagg_info AS WITH caggs AS ( SELECT format('%I.%I', user_view_schema, user_view_name)::regclass AS user_view, format('%I.%I', direct_view_schema, direct_view_name)::regclass AS direct_view, format('%I.%I', partial_view_schema, partial_view_name)::regclass AS partial_view, format('%I.%I', ht.schema_name, ht.table_name)::regclass AS mat_relid FROM _timescaledb_catalog.hypertable ht, _timescaledb_catalog.continuous_agg cagg WHERE ht.id = cagg.mat_hypertable_id ) SELECT user_view, pg_get_userbyid(relowner) AS user_view_owner, relname AS mat_table, (SELECT pg_get_userbyid(relowner) FROM pg_class WHERE oid = mat_relid) AS mat_table_owner, direct_view, (SELECT pg_get_userbyid(relowner) FROM pg_class WHERE oid = direct_view) AS direct_view_owner, partial_view, (SELECT pg_get_userbyid(relowner) FROM pg_class WHERE oid = partial_view) AS partial_view_owner, (SELECT spcname FROM pg_tablespace WHERE oid = reltablespace) AS tablespace FROM pg_class JOIN caggs ON pg_class.oid = caggs.mat_relid; GRANT SELECT ON cagg_info TO PUBLIC; CREATE VIEW chunk_info AS SELECT ht.schema_name, ht.table_name, relname AS chunk_name, (SELECT spcname FROM pg_tablespace WHERE oid = reltablespace) AS tablespace FROM pg_class c, _timescaledb_catalog.hypertable ht, _timescaledb_catalog.chunk ch WHERE ch.table_name = c.relname AND ht.id = ch.hypertable_id; CREATE TABLE whatever(time BIGINT NOT NULL, data INTEGER); SELECT hypertable_id AS whatever_nid FROM create_hypertable('whatever', 'time', chunk_time_interval => 10) \gset SELECT set_integer_now_func('whatever', 'integer_now_test'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW whatever_view_1 WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('5', time), COUNT(data) FROM whatever GROUP BY 1 WITH NO DATA; CREATE MATERIALIZED VIEW whatever_view_2 WITH (timescaledb.continuous, timescaledb.materialized_only=true) TABLESPACE tablespace1 AS SELECT time_bucket('5', time), COUNT(data) FROM whatever GROUP BY 1 WITH NO DATA; INSERT INTO whatever SELECT i, i FROM generate_series(0, 29) AS i; CALL refresh_continuous_aggregate('whatever_view_1', NULL, NULL); CALL refresh_continuous_aggregate('whatever_view_2', NULL, NULL); SELECT user_view, mat_table, cagg_info.tablespace AS mat_tablespace, chunk_name, chunk_info.tablespace AS chunk_tablespace FROM cagg_info, chunk_info WHERE mat_table::text = table_name AND user_view::text LIKE 'whatever_view%'; user_view | mat_table | mat_tablespace | chunk_name | chunk_tablespace -----------------+-----------------------------+----------------+--------------------+------------------ whatever_view_1 | _materialized_hypertable_13 | | _hyper_13_27_chunk | whatever_view_2 | _materialized_hypertable_14 | tablespace1 | _hyper_14_28_chunk | tablespace1 ALTER MATERIALIZED VIEW whatever_view_1 SET TABLESPACE tablespace2; SELECT user_view, mat_table, cagg_info.tablespace AS mat_tablespace, chunk_name, chunk_info.tablespace AS chunk_tablespace FROM cagg_info, chunk_info WHERE mat_table::text = table_name AND user_view::text LIKE 'whatever_view%'; user_view | mat_table | mat_tablespace | chunk_name | chunk_tablespace -----------------+-----------------------------+----------------+--------------------+------------------ whatever_view_1 | _materialized_hypertable_13 | tablespace2 | _hyper_13_27_chunk | tablespace2 whatever_view_2 | _materialized_hypertable_14 | tablespace1 | _hyper_14_28_chunk | tablespace1 DROP MATERIALIZED VIEW whatever_view_1; NOTICE: drop cascades to table _timescaledb_internal._hyper_13_27_chunk DROP MATERIALIZED VIEW whatever_view_2; NOTICE: drop cascades to table _timescaledb_internal._hyper_14_28_chunk -- test bucket width expressions on integer hypertables CREATE TABLE metrics_int2 ( time int2 NOT NULL, device_id int, v1 float, v2 float ); CREATE TABLE metrics_int4 ( time int4 NOT NULL, device_id int, v1 float, v2 float ); CREATE TABLE metrics_int8 ( time int8 NOT NULL, device_id int, v1 float, v2 float ); SELECT create_hypertable (('metrics_' || dt)::regclass, 'time', chunk_time_interval => 10) FROM ( VALUES ('int2'), ('int4'), ('int8')) v (dt); create_hypertable ---------------------------- (15,public,metrics_int2,t) (16,public,metrics_int4,t) (17,public,metrics_int8,t) CREATE OR REPLACE FUNCTION int2_now () RETURNS int2 LANGUAGE SQL STABLE AS $$ SELECT 10::int2 $$; CREATE OR REPLACE FUNCTION int4_now () RETURNS int4 LANGUAGE SQL STABLE AS $$ SELECT 10::int4 $$; CREATE OR REPLACE FUNCTION int8_now () RETURNS int8 LANGUAGE SQL STABLE AS $$ SELECT 10::int8 $$; SELECT set_integer_now_func (('metrics_' || dt)::regclass, (dt || '_now')::regproc) FROM ( VALUES ('int2'), ('int4'), ('int8')) v (dt); set_integer_now_func ---------------------- -- width expression for int2 hypertables CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1::smallint, time) FROM metrics_int2 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1::smallint + 2::smallint, time) FROM metrics_int2 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; -- width expression for int4 hypertables CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1, time) FROM metrics_int4 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1 + 2, time) FROM metrics_int4 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; -- width expression for int8 hypertables CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1, time) FROM metrics_int8 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1 + 2, time) FROM metrics_int8 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; \set ON_ERROR_STOP 0 -- non-immutable expresions should be rejected CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(extract(year FROM now())::smallint, time) FROM metrics_int2 GROUP BY 1; ERROR: only immutable expressions allowed in time bucket function CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(extract(year FROM now())::int, time) FROM metrics_int4 GROUP BY 1; ERROR: only immutable expressions allowed in time bucket function CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(extract(year FROM now())::int, time) FROM metrics_int8 GROUP BY 1; ERROR: only immutable expressions allowed in time bucket function \set ON_ERROR_STOP 1 -- Test various ALTER MATERIALIZED VIEW statements. SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE MATERIALIZED VIEW owner_check WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1 + 2, time) FROM metrics_int8 GROUP BY 1 WITH NO DATA; \x on SELECT * FROM cagg_info WHERE user_view::text = 'owner_check'; -[ RECORD 1 ]------+--------------------------------------- user_view | owner_check user_view_owner | default_perm_user mat_table | _materialized_hypertable_24 mat_table_owner | default_perm_user direct_view | _timescaledb_internal._direct_view_24 direct_view_owner | default_perm_user partial_view | _timescaledb_internal._partial_view_24 partial_view_owner | default_perm_user tablespace | \x off -- This should not work since the target user has the wrong role, but -- we test that the normal checks are done when changing the owner. \set ON_ERROR_STOP 0 ALTER MATERIALIZED VIEW owner_check OWNER TO :ROLE_1; ERROR: must be able to SET ROLE "test_role_1" \set ON_ERROR_STOP 1 -- Superuser can always change owner SET ROLE :ROLE_SUPERUSER; -- Add a refresh policy before changing owner to verify job owner is propagated SELECT add_continuous_aggregate_policy('owner_check', NULL, 1::int8, '1 day'::interval) AS cagg_job_id \gset SELECT owner FROM _timescaledb_config.bgw_job WHERE id = :cagg_job_id; owner ------------------- default_perm_user ALTER MATERIALIZED VIEW owner_check OWNER TO :ROLE_1; \x on SELECT * FROM cagg_info WHERE user_view::text = 'owner_check'; -[ RECORD 1 ]------+--------------------------------------- user_view | owner_check user_view_owner | test_role_1 mat_table | _materialized_hypertable_24 mat_table_owner | test_role_1 direct_view | _timescaledb_internal._direct_view_24 direct_view_owner | test_role_1 partial_view | _timescaledb_internal._partial_view_24 partial_view_owner | test_role_1 tablespace | \x off -- make sure policy job owner is propagated SELECT owner FROM _timescaledb_config.bgw_job WHERE id = :cagg_job_id; owner ------------- test_role_1 SELECT remove_continuous_aggregate_policy('owner_check'); remove_continuous_aggregate_policy ------------------------------------ -- -- Test drop continuous aggregate cases -- -- Issue: #2608 -- CREATE OR REPLACE FUNCTION test_int_now() RETURNS INT LANGUAGE SQL STABLE AS $BODY$ SELECT 50; $BODY$; CREATE TABLE conditionsnm(time_int INT NOT NULL, device INT, value FLOAT); SELECT create_hypertable('conditionsnm', 'time_int', chunk_time_interval => 10); create_hypertable ---------------------------- (25,public,conditionsnm,t) SELECT set_integer_now_func('conditionsnm', 'test_int_now'); set_integer_now_func ---------------------- INSERT INTO conditionsnm SELECT time_val, time_val % 4, 3.14 FROM generate_series(0,100,1) AS time_val; -- Case 1: DROP CREATE MATERIALIZED VIEW conditionsnm_4 WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(7, time_int) as bucket, SUM(value), COUNT(value) FROM conditionsnm GROUP BY bucket WITH DATA; NOTICE: refreshing continuous aggregate "conditionsnm_4" DROP materialized view conditionsnm_4; NOTICE: drop cascades to table _timescaledb_internal._hyper_26_40_chunk -- Case 2: DROP CASCADE should have similar behaviour as DROP CREATE MATERIALIZED VIEW conditionsnm_4 WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(7, time_int) as bucket, SUM(value), COUNT(value) FROM conditionsnm GROUP BY bucket WITH DATA; NOTICE: refreshing continuous aggregate "conditionsnm_4" DROP materialized view conditionsnm_4 CASCADE; NOTICE: drop cascades to table _timescaledb_internal._hyper_27_41_chunk -- Case 3: require CASCADE in case of dependent object CREATE MATERIALIZED VIEW conditionsnm_4 WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(7, time_int) as bucket, SUM(value), COUNT(value) FROM conditionsnm GROUP BY bucket WITH DATA; NOTICE: refreshing continuous aggregate "conditionsnm_4" CREATE VIEW see_cagg as select * from conditionsnm_4; \set ON_ERROR_STOP 0 DROP MATERIALIZED VIEW conditionsnm_4; ERROR: cannot drop view conditionsnm_4 because other objects depend on it \set ON_ERROR_STOP 1 -- Case 4: DROP CASCADE with dependency DROP MATERIALIZED VIEW conditionsnm_4 CASCADE; NOTICE: drop cascades to view see_cagg NOTICE: drop cascades to table _timescaledb_internal._hyper_28_42_chunk -- Test DROP SCHEMA CASCADE with continuous aggregates -- -- Issue: #2350 -- -- Case 1: DROP SCHEMA CASCADE CREATE SCHEMA test_schema; CREATE TABLE test_schema.telemetry_raw ( ts TIMESTAMP WITH TIME ZONE NOT NULL, value DOUBLE PRECISION ); SELECT create_hypertable('test_schema.telemetry_raw', 'ts'); create_hypertable ---------------------------------- (29,test_schema,telemetry_raw,t) CREATE MATERIALIZED VIEW test_schema.telemetry_1s WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(INTERVAL '1s', ts) AS ts_1s, avg(value) FROM test_schema.telemetry_raw GROUP BY ts_1s WITH NO DATA; SELECT ca.raw_hypertable_id, h.schema_name, h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) WHERE user_view_name = 'telemetry_1s'; raw_hypertable_id | schema_name | MAT_TABLE_NAME | PART_VIEW_NAME | partial_view_schema -------------------+-----------------------+-----------------------------+------------------+----------------------- 29 | _timescaledb_internal | _materialized_hypertable_30 | _partial_view_30 | _timescaledb_internal \gset DROP SCHEMA test_schema CASCADE; NOTICE: drop cascades to 4 other objects SELECT count(*) FROM pg_class WHERE relname = :'MAT_TABLE_NAME'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = :'PART_VIEW_NAME'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = 'telemetry_1s'; count ------- 0 SELECT count(*) FROM pg_namespace WHERE nspname = 'test_schema'; count ------- 0 -- Case 2: DROP SCHEMA CASCADE with multiple caggs CREATE SCHEMA test_schema; CREATE TABLE test_schema.telemetry_raw ( ts TIMESTAMP WITH TIME ZONE NOT NULL, value DOUBLE PRECISION ); SELECT create_hypertable('test_schema.telemetry_raw', 'ts'); create_hypertable ---------------------------------- (31,test_schema,telemetry_raw,t) CREATE MATERIALIZED VIEW test_schema.cagg1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(INTERVAL '1s', ts) AS ts_1s, avg(value) FROM test_schema.telemetry_raw GROUP BY ts_1s WITH NO DATA; CREATE MATERIALIZED VIEW test_schema.cagg2 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(INTERVAL '1s', ts) AS ts_1s, avg(value) FROM test_schema.telemetry_raw GROUP BY ts_1s WITH NO DATA; SELECT ca.raw_hypertable_id, h.schema_name, h.table_name AS "MAT_TABLE_NAME1", partial_view_name as "PART_VIEW_NAME1", partial_view_schema FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) WHERE user_view_name = 'cagg1'; raw_hypertable_id | schema_name | MAT_TABLE_NAME1 | PART_VIEW_NAME1 | partial_view_schema -------------------+-----------------------+-----------------------------+------------------+----------------------- 31 | _timescaledb_internal | _materialized_hypertable_32 | _partial_view_32 | _timescaledb_internal \gset SELECT ca.raw_hypertable_id, h.schema_name, h.table_name AS "MAT_TABLE_NAME2", partial_view_name as "PART_VIEW_NAME2", partial_view_schema FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) WHERE user_view_name = 'cagg2'; raw_hypertable_id | schema_name | MAT_TABLE_NAME2 | PART_VIEW_NAME2 | partial_view_schema -------------------+-----------------------+-----------------------------+------------------+----------------------- 31 | _timescaledb_internal | _materialized_hypertable_33 | _partial_view_33 | _timescaledb_internal \gset DROP SCHEMA test_schema CASCADE; NOTICE: drop cascades to 7 other objects SELECT count(*) FROM pg_class WHERE relname = :'MAT_TABLE_NAME1'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = :'PART_VIEW_NAME1'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = 'cagg1'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = :'MAT_TABLE_NAME2'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = :'PART_VIEW_NAME2'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = 'cagg2'; count ------- 0 SELECT count(*) FROM pg_namespace WHERE nspname = 'test_schema'; count ------- 0 DROP TABLESPACE tablespace1; DROP TABLESPACE tablespace2; -- Check that we can rename a column of a materialized view and still -- rebuild it after (#3051, #3405) CREATE TABLE conditions ( time TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL ); SELECT create_hypertable('conditions', 'time'); create_hypertable -------------------------- (34,public,conditions,t) INSERT INTO conditions VALUES ( '2018-01-01 09:20:00-08', 'SFO', 55); INSERT INTO conditions VALUES ( '2018-01-02 09:30:00-08', 'por', 100); INSERT INTO conditions VALUES ( '2018-01-02 09:20:00-08', 'SFO', 65); INSERT INTO conditions VALUES ( '2018-01-02 09:10:00-08', 'NYC', 65); INSERT INTO conditions VALUES ( '2018-11-01 09:20:00-08', 'NYC', 45); INSERT INTO conditions VALUES ( '2018-11-01 10:40:00-08', 'NYC', 55); INSERT INTO conditions VALUES ( '2018-11-01 11:50:00-08', 'NYC', 65); INSERT INTO conditions VALUES ( '2018-11-01 12:10:00-08', 'NYC', 75); INSERT INTO conditions VALUES ( '2018-11-01 13:10:00-08', 'NYC', 85); INSERT INTO conditions VALUES ( '2018-11-02 09:20:00-08', 'NYC', 10); INSERT INTO conditions VALUES ( '2018-11-02 10:30:00-08', 'NYC', 20); CREATE MATERIALIZED VIEW conditions_daily WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT location, time_bucket(INTERVAL '1 day', time) AS bucket, AVG(temperature) FROM conditions GROUP BY location, bucket WITH NO DATA; CREATE MATERIALIZED VIEW conditions_weekly WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT location, time_bucket(INTERVAL '7 day', bucket) AS bucket, AVG(avg) FROM conditions_daily GROUP BY 1, 2 WITH NO DATA; SELECT format('%I.%I', '_timescaledb_internal', h.table_name) AS "MAT_TABLE_NAME", format('%I.%I', '_timescaledb_internal', partial_view_name) AS "PART_VIEW_NAME", format('%I.%I', '_timescaledb_internal', direct_view_name) AS "DIRECT_VIEW_NAME" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) WHERE user_view_name = 'conditions_daily' \gset -- Show both the columns and the view definitions to see that -- references are correct in the view as well. SELECT * FROM test.show_columns('conditions_daily'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f bucket | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'DIRECT_VIEW_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f bucket | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'PART_VIEW_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f bucket | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'MAT_TABLE_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f bucket | timestamp with time zone | t avg | double precision | f ALTER MATERIALIZED VIEW conditions_daily RENAME COLUMN bucket to "time"; -- Show both the columns and the view definitions to see that -- references are correct in the view as well. SELECT * FROM test.show_columns(' conditions_daily'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'DIRECT_VIEW_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'PART_VIEW_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'MAT_TABLE_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | t avg | double precision | f -- This will rebuild the materialized view and should succeed. ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only = false); -- Refresh the continuous aggregate to check that it works after the -- rename. \set VERBOSITY verbose CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); \set VERBOSITY terse -- Rename another column after the flip and verify toggling back and -- forth still works. This exercises the rename when the user view -- already has a UNION ALL query (materialized_only = false). ALTER MATERIALIZED VIEW conditions_daily RENAME COLUMN avg TO average; SELECT * FROM test.show_columns('conditions_daily'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | f average | double precision | f SELECT * FROM test.show_columns(:'DIRECT_VIEW_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | f average | double precision | f SELECT * FROM test.show_columns(:'MAT_TABLE_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | t average | double precision | f ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only = true); ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only = false); -- Verify data is still accessible after multiple renames and toggles. SELECT * FROM conditions_daily ORDER BY location COLLATE "C", time; location | time | average ----------+------------------------------+--------- NYC | Mon Jan 01 16:00:00 2018 UTC | 65 NYC | Wed Oct 31 16:00:00 2018 UTC | 65 NYC | Thu Nov 01 16:00:00 2018 UTC | 15 SFO | Sun Dec 31 16:00:00 2017 UTC | 55 SFO | Mon Jan 01 16:00:00 2018 UTC | 65 por | Mon Jan 01 16:00:00 2018 UTC | 100 -- check hierarchical continuous aggregate still works after renames and toggles on the underlying cagg ALTER MATERIALIZED VIEW conditions_weekly SET (timescaledb.materialized_only = false); ALTER MATERIALIZED VIEW conditions_weekly SET (timescaledb.materialized_only = true); SELECT * FROM conditions_weekly ORDER BY location COLLATE "C", bucket; location | bucket | avg ----------+--------+----- -- Verify that direct rename on the materialization hypertable is blocked. \set ON_ERROR_STOP 0 ALTER TABLE :MAT_TABLE_NAME RENAME COLUMN average TO avg; ERROR: renaming columns on materialization tables is not supported \set ON_ERROR_STOP 1 -- Rename back so subsequent tests that reference "avg" still work. ALTER MATERIALIZED VIEW conditions_daily RENAME COLUMN average TO avg; -- -- Indexes on continuous aggregate -- \set ON_ERROR_STOP 0 -- unique indexes are not supported CREATE UNIQUE INDEX index_unique_error ON conditions_daily ("time", location); ERROR: continuous aggregates do not support UNIQUE indexes -- concurrently index creation not supported CREATE INDEX CONCURRENTLY index_concurrently_avg ON conditions_daily (avg); ERROR: hypertables do not support concurrent index creation \set ON_ERROR_STOP 1 CREATE INDEX index_avg ON conditions_daily (avg); CREATE INDEX index_avg_only ON ONLY conditions_daily (avg); CREATE INDEX index_avg_include ON conditions_daily (avg) INCLUDE (location); CREATE INDEX index_avg_expr ON conditions_daily ((avg + 1)); CREATE INDEX index_avg_location_sfo ON conditions_daily (avg) WHERE location = 'SFO'; CREATE INDEX index_avg_expr_location_sfo ON conditions_daily ((avg + 2)) WHERE location = 'SFO'; SELECT * FROM test.show_indexespred(:'MAT_TABLE_NAME'); Index | Columns | Expr | Pred | Unique | Primary | Exclusion | Tablespace -----------------------------------------------------------------------+-------------------+---------------------------+------------------------+--------+---------+-----------+------------ _timescaledb_internal._materialized_hypertable_35_bucket_idx | {bucket} | | | f | f | f | _timescaledb_internal._materialized_hypertable_35_location_bucket_idx | {location,bucket} | | | f | f | f | _timescaledb_internal.index_avg | {avg} | | | f | f | f | _timescaledb_internal.index_avg_expr | {expr} | avg + 1::double precision | | f | f | f | _timescaledb_internal.index_avg_expr_location_sfo | {expr} | avg + 2::double precision | location = 'SFO'::text | f | f | f | _timescaledb_internal.index_avg_include | {avg,location} | | | f | f | f | _timescaledb_internal.index_avg_location_sfo | {avg} | | location = 'SFO'::text | f | f | f | _timescaledb_internal.index_avg_only | {avg} | | | f | f | f | -- #3696 assertion failure when referencing columns not present in result CREATE TABLE i3696(time timestamptz NOT NULL, search_query text, cnt integer, cnt2 integer); SELECT table_name FROM create_hypertable('i3696','time'); table_name ------------ i3696 CREATE MATERIALIZED VIEW i3696_cagg1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT search_query,count(search_query) as count, sum(cnt), time_bucket(INTERVAL '1 minute', time) AS bucket FROM i3696 GROUP BY cnt +cnt2 , bucket, search_query; NOTICE: continuous aggregate "i3696_cagg1" is already up-to-date ALTER MATERIALIZED VIEW i3696_cagg1 SET (timescaledb.materialized_only = 'true'); CREATE MATERIALIZED VIEW i3696_cagg2 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT search_query,count(search_query) as count, sum(cnt), time_bucket(INTERVAL '1 minute', time) AS bucket FROM i3696 GROUP BY cnt + cnt2, bucket, search_query HAVING cnt + cnt2 + sum(cnt) > 2 or count(cnt2) > 10; NOTICE: continuous aggregate "i3696_cagg2" is already up-to-date ALTER MATERIALIZED VIEW i3696_cagg2 SET (timescaledb.materialized_only = 'true'); --TEST test with multiple settings on continuous aggregates -- -- test for materialized_only + compress combinations (real time aggs enabled initially) CREATE TABLE test_setting(time timestamptz not null, val numeric); SELECT create_hypertable('test_setting', 'time'); create_hypertable ---------------------------- (40,public,test_setting,t) CREATE MATERIALIZED VIEW test_setting_cagg with (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h',time), avg(val), count(*) FROM test_setting GROUP BY 1; NOTICE: continuous aggregate "test_setting_cagg" is already up-to-date INSERT INTO test_setting SELECT generate_series( '2020-01-10 8:00'::timestamp, '2020-01-30 10:00+00'::timestamptz, '1 day'::interval), 10.0; CALL refresh_continuous_aggregate('test_setting_cagg', NULL, '2020-05-30 10:00+00'::timestamptz); SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 --this row is not in the materialized result --- INSERT INTO test_setting VALUES( '2020-11-01', 20); --try out 2 settings here -- ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'true', timescaledb.compress='true'); NOTICE: defaulting compress_orderby to time_bucket SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | t | t --real time aggs is off now , should return 20 -- SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 --now set it back to false -- ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'false', timescaledb.compress='true'); NOTICE: defaulting compress_orderby to time_bucket SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | t | f --count should return additional data since we have real time aggs on SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 21 ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'true', timescaledb.compress='false'); SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | f | t --real time aggs is off now , should return 20 -- SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'false', timescaledb.compress='false'); SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | f | f --count should return additional data since we have real time aggs on SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 21 DELETE FROM test_setting WHERE val = 20; --TEST test with multiple settings on continuous aggregates with real time aggregates turned off initially -- -- test for materialized_only + compress combinations (real time aggs enabled initially) DROP MATERIALIZED VIEW test_setting_cagg; NOTICE: drop cascades to table _timescaledb_internal._hyper_41_50_chunk CREATE MATERIALIZED VIEW test_setting_cagg with (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT time_bucket('1h',time), avg(val), count(*) FROM test_setting GROUP BY 1; NOTICE: refreshing continuous aggregate "test_setting_cagg" CALL refresh_continuous_aggregate('test_setting_cagg', NULL, '2020-05-30 10:00+00'::timestamptz); SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 --this row is not in the materialized result --- INSERT INTO test_setting VALUES( '2020-11-01', 20); --try out 2 settings here -- ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'false', timescaledb.compress='true'); NOTICE: defaulting compress_orderby to time_bucket SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | t | f --count should return additional data since we have real time aggs on SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 21 --now set it back to false -- ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'true', timescaledb.compress='true'); NOTICE: defaulting compress_orderby to time_bucket SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | t | t --real time aggs is off now , should return 20 -- SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'false', timescaledb.compress='false'); SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | f | f --count should return additional data since we have real time aggs on SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 21 ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'true', timescaledb.compress='false'); SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | f | t --real time aggs is off now , should return 20 -- SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 -- END TEST with multiple settings -- Test View Target Entries that contain both aggrefs and Vars in the same expression CREATE TABLE transactions ( "time" timestamp with time zone NOT NULL, dummy1 integer, dummy2 integer, dummy3 integer, dummy4 integer, dummy5 integer, amount integer, fiat_value integer ); SELECT create_hypertable('transactions', 'time'); create_hypertable ---------------------------- (45,public,transactions,t) INSERT INTO transactions VALUES ( '2018-01-01 09:20:00-08', 0, 0, 0, 0, 0, 1, 10); INSERT INTO transactions VALUES ( '2018-01-02 09:30:00-08', 0, 0, 0, 0, 0, -1, 10); INSERT INTO transactions VALUES ( '2018-01-02 09:20:00-08', 0, 0, 0, 0, 0, -1, 10); INSERT INTO transactions VALUES ( '2018-01-02 09:10:00-08', 0, 0, 0, 0, 0, -1, 10); INSERT INTO transactions VALUES ( '2018-11-01 09:20:00-08', 0, 0, 0, 0, 0, 1, 10); INSERT INTO transactions VALUES ( '2018-11-01 10:40:00-08', 0, 0, 0, 0, 0, 1, 10); INSERT INTO transactions VALUES ( '2018-11-01 11:50:00-08', 0, 0, 0, 0, 0, 1, 10); INSERT INTO transactions VALUES ( '2018-11-01 12:10:00-08', 0, 0, 0, 0, 0, -1, 10); INSERT INTO transactions VALUES ( '2018-11-01 13:10:00-08', 0, 0, 0, 0, 0, -1, 10); INSERT INTO transactions VALUES ( '2018-11-02 09:20:00-08', 0, 0, 0, 0, 0, 1, 10); INSERT INTO transactions VALUES ( '2018-11-02 10:30:00-08', 0, 0, 0, 0, 0, -1, 10); CREATE materialized view cashflows( bucket, amount, cashflow, cashflow2 ) WITH ( timescaledb.continuous, timescaledb.materialized_only = true ) AS SELECT time_bucket ('1 day', time) AS bucket, amount, CASE WHEN amount < 0 THEN (0 - sum(fiat_value)) ELSE sum(fiat_value) END AS cashflow, amount + sum(fiat_value) FROM transactions GROUP BY bucket, amount; NOTICE: refreshing continuous aggregate "cashflows" SELECT h.table_name AS "MAT_TABLE_NAME", partial_view_name AS "PART_VIEW_NAME", direct_view_name AS "DIRECT_VIEW_NAME" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) WHERE user_view_name = 'cashflows' \gset -- Show both the columns and the view definitions to see that -- references are correct in the view as well. \d+ "_timescaledb_internal".:"DIRECT_VIEW_NAME" View "_timescaledb_internal._direct_view_46" Column | Type | Collation | Nullable | Default | Storage | Description -----------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | amount | integer | | | | plain | cashflow | bigint | | | | plain | cashflow2 | bigint | | | | plain | View definition: SELECT time_bucket('@ 1 day'::interval, "time") AS bucket, amount, CASE WHEN amount < 0 THEN 0 - sum(fiat_value) ELSE sum(fiat_value) END AS cashflow, amount + sum(fiat_value) AS cashflow2 FROM transactions GROUP BY (time_bucket('@ 1 day'::interval, "time")), amount; \d+ "_timescaledb_internal".:"PART_VIEW_NAME" View "_timescaledb_internal._partial_view_46" Column | Type | Collation | Nullable | Default | Storage | Description -----------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | amount | integer | | | | plain | cashflow | bigint | | | | plain | cashflow2 | bigint | | | | plain | View definition: SELECT time_bucket('@ 1 day'::interval, "time") AS bucket, amount, CASE WHEN amount < 0 THEN 0 - sum(fiat_value) ELSE sum(fiat_value) END AS cashflow, amount + sum(fiat_value) AS cashflow2 FROM transactions GROUP BY (time_bucket('@ 1 day'::interval, "time")), amount; \d+ "_timescaledb_internal".:"MAT_TABLE_NAME" Table "_timescaledb_internal._materialized_hypertable_46" Column | Type | Collation | Nullable | Default | Storage | Stats target | Description -----------+--------------------------+-----------+----------+---------+---------+--------------+------------- bucket | timestamp with time zone | | not null | | plain | | amount | integer | | | | plain | | cashflow | bigint | | | | plain | | cashflow2 | bigint | | | | plain | | Indexes: "_materialized_hypertable_46_amount_bucket_idx" btree (amount, bucket DESC) "_materialized_hypertable_46_bucket_idx" btree (bucket DESC) Child tables: _timescaledb_internal._hyper_46_55_chunk, _timescaledb_internal._hyper_46_56_chunk \d+ 'cashflows' View "public.cashflows" Column | Type | Collation | Nullable | Default | Storage | Description -----------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | amount | integer | | | | plain | cashflow | bigint | | | | plain | cashflow2 | bigint | | | | plain | View definition: SELECT bucket, amount, cashflow, cashflow2 FROM _timescaledb_internal._materialized_hypertable_46; SELECT * FROM cashflows ORDER BY cashflows; bucket | amount | cashflow | cashflow2 ------------------------------+--------+----------+----------- Sun Dec 31 16:00:00 2017 UTC | 1 | 10 | 11 Mon Jan 01 16:00:00 2018 UTC | -1 | -30 | 29 Wed Oct 31 16:00:00 2018 UTC | -1 | -20 | 19 Wed Oct 31 16:00:00 2018 UTC | 1 | 30 | 31 Thu Nov 01 16:00:00 2018 UTC | -1 | -10 | 9 Thu Nov 01 16:00:00 2018 UTC | 1 | 10 | 11 -- test cagg creation with named arguments in time_bucket -- note that positional arguments cannot follow named arguments -- 1. test named origin -- 2. test named timezone -- 3. test named ts -- 4. test named bucket width -- named origin CREATE MATERIALIZED VIEW cagg_named_origin WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h', time, 'UTC', origin => '2001-01-03 01:23:45') AS bucket, avg(amount) as avg_amount FROM transactions GROUP BY 1 WITH NO DATA; -- named timezone CREATE MATERIALIZED VIEW cagg_named_tz_origin WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h', time, timezone => 'UTC', origin => '2001-01-03 01:23:45') AS bucket, avg(amount) as avg_amount FROM transactions GROUP BY 1 WITH NO DATA; -- named ts CREATE MATERIALIZED VIEW cagg_named_ts_tz_origin WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h', ts => time, timezone => 'UTC', origin => '2001-01-03 01:23:45') AS bucket, avg(amount) as avg_amount FROM transactions GROUP BY 1 WITH NO DATA; -- named bucket width CREATE MATERIALIZED VIEW cagg_named_all WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(bucket_width => '1h', ts => time, timezone => 'UTC', origin => '2001-01-03 01:23:45') AS bucket, avg(amount) as avg_amount FROM transactions GROUP BY 1 WITH NO DATA; -- Refreshing from the beginning (NULL) of a CAGG with variable time bucket and -- using an INTERVAL for the end timestamp (issue #5534) CREATE MATERIALIZED VIEW transactions_montly WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT time_bucket(INTERVAL '1 month', time) AS bucket, SUM(fiat_value), MAX(fiat_value), MIN(fiat_value) FROM transactions GROUP BY 1 WITH NO DATA; -- No rows SELECT * FROM transactions_montly ORDER BY bucket; bucket | sum | max | min --------+-----+-----+----- -- Refresh from beginning of the CAGG for 1 month CALL refresh_continuous_aggregate('transactions_montly', NULL, INTERVAL '1 month'); SELECT * FROM transactions_montly ORDER BY bucket; bucket | sum | max | min ------------------------------+-----+-----+----- Sun Dec 31 16:00:00 2017 UTC | 40 | 10 | 10 Wed Oct 31 16:00:00 2018 UTC | 70 | 10 | 10 TRUNCATE transactions_montly; -- Partial refresh the CAGG from beginning to an specific timestamp CALL refresh_continuous_aggregate('transactions_montly', NULL, '2018-11-01 11:50:00-08'::timestamptz); SELECT * FROM transactions_montly ORDER BY bucket; bucket | sum | max | min ------------------------------+-----+-----+----- Sun Dec 31 16:00:00 2017 UTC | 40 | 10 | 10 -- Full refresh the CAGG CALL refresh_continuous_aggregate('transactions_montly', NULL, NULL); SELECT * FROM transactions_montly ORDER BY bucket; bucket | sum | max | min ------------------------------+-----+-----+----- Sun Dec 31 16:00:00 2017 UTC | 40 | 10 | 10 Wed Oct 31 16:00:00 2018 UTC | 70 | 10 | 10 -- Check set_chunk_time_interval on continuous aggregate CREATE MATERIALIZED VIEW cagg_set_chunk_time_interval WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(INTERVAL '1 month', time) AS bucket, SUM(fiat_value), MAX(fiat_value), MIN(fiat_value) FROM transactions GROUP BY 1 WITH NO DATA; SELECT set_chunk_time_interval('cagg_set_chunk_time_interval', chunk_time_interval => interval '1 month'); set_chunk_time_interval ------------------------- CALL refresh_continuous_aggregate('cagg_set_chunk_time_interval', NULL, NULL); SELECT _timescaledb_functions.to_interval(d.interval_length) = interval '1 month' FROM _timescaledb_catalog.dimension d RIGHT JOIN _timescaledb_catalog.continuous_agg ca ON ca.user_view_name = 'cagg_set_chunk_time_interval' WHERE d.hypertable_id = ca.mat_hypertable_id; ?column? ---------- t -- Since #6077 CAggs are materialized only by default DROP TABLE conditions CASCADE; NOTICE: drop cascades to 5 other objects NOTICE: drop cascades to 2 other objects CREATE TABLE conditions ( time TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL ); SELECT create_hypertable('conditions', 'time'); create_hypertable -------------------------- (53,public,conditions,t) INSERT INTO conditions VALUES ( '2018-01-01 09:20:00-08', 'SFO', 55); INSERT INTO conditions VALUES ( '2018-01-02 09:30:00-08', 'POR', 100); INSERT INTO conditions VALUES ( '2018-01-02 09:20:00-08', 'SFO', 65); INSERT INTO conditions VALUES ( '2018-01-02 09:10:00-08', 'NYC', 65); INSERT INTO conditions VALUES ( '2018-11-01 09:20:00-08', 'NYC', 45); INSERT INTO conditions VALUES ( '2018-11-01 10:40:00-08', 'NYC', 55); INSERT INTO conditions VALUES ( '2018-11-01 11:50:00-08', 'NYC', 65); INSERT INTO conditions VALUES ( '2018-11-01 12:10:00-08', 'NYC', 75); INSERT INTO conditions VALUES ( '2018-11-01 13:10:00-08', 'NYC', 85); INSERT INTO conditions VALUES ( '2018-11-02 09:20:00-08', 'NYC', 10); INSERT INTO conditions VALUES ( '2018-11-02 10:30:00-08', 'NYC', 20); CREATE MATERIALIZED VIEW conditions_daily WITH (timescaledb.continuous) AS SELECT location, time_bucket(INTERVAL '1 day', time) AS bucket, AVG(temperature) FROM conditions GROUP BY location, bucket WITH NO DATA; \d+ conditions_daily View "public.conditions_daily" Column | Type | Collation | Nullable | Default | Storage | Description ----------+--------------------------+-----------+----------+---------+----------+------------- location | text | | | | extended | bucket | timestamp with time zone | | | | plain | avg | double precision | | | | plain | View definition: SELECT location, bucket, avg FROM _timescaledb_internal._materialized_hypertable_54; -- Should return NO ROWS SELECT * FROM conditions_daily ORDER BY bucket, location; location | bucket | avg ----------+--------+----- ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only=false); \d+ conditions_daily View "public.conditions_daily" Column | Type | Collation | Nullable | Default | Storage | Description ----------+--------------------------+-----------+----------+---------+----------+------------- location | text | | | | extended | bucket | timestamp with time zone | | | | plain | avg | double precision | | | | plain | View definition: SELECT _materialized_hypertable_54.location, _materialized_hypertable_54.bucket, _materialized_hypertable_54.avg FROM _timescaledb_internal._materialized_hypertable_54 WHERE _materialized_hypertable_54.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(54)), '-infinity'::timestamp with time zone) UNION ALL SELECT conditions.location, time_bucket('@ 1 day'::interval, conditions."time") AS bucket, avg(conditions.temperature) AS avg FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(54)), '-infinity'::timestamp with time zone) GROUP BY conditions.location, (time_bucket('@ 1 day'::interval, conditions."time")); -- Should return ROWS because now it is realtime SELECT * FROM conditions_daily ORDER BY bucket, location; location | bucket | avg ----------+------------------------------+----- SFO | Sun Dec 31 16:00:00 2017 UTC | 55 NYC | Mon Jan 01 16:00:00 2018 UTC | 65 POR | Mon Jan 01 16:00:00 2018 UTC | 100 SFO | Mon Jan 01 16:00:00 2018 UTC | 65 NYC | Wed Oct 31 16:00:00 2018 UTC | 65 NYC | Thu Nov 01 16:00:00 2018 UTC | 15 -- Should return ROWS because we refreshed it ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only=true); \d+ conditions_daily View "public.conditions_daily" Column | Type | Collation | Nullable | Default | Storage | Description ----------+--------------------------+-----------+----------+---------+----------+------------- location | text | | | | extended | bucket | timestamp with time zone | | | | plain | avg | double precision | | | | plain | View definition: SELECT location, bucket, avg FROM _timescaledb_internal._materialized_hypertable_54; CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); SELECT * FROM conditions_daily ORDER BY bucket, location; location | bucket | avg ----------+------------------------------+----- SFO | Sun Dec 31 16:00:00 2017 UTC | 55 NYC | Mon Jan 01 16:00:00 2018 UTC | 65 POR | Mon Jan 01 16:00:00 2018 UTC | 100 SFO | Mon Jan 01 16:00:00 2018 UTC | 65 NYC | Wed Oct 31 16:00:00 2018 UTC | 65 NYC | Thu Nov 01 16:00:00 2018 UTC | 15 -- Test TRUNCATE over a Realtime CAgg DROP MATERIALIZED VIEW conditions_daily; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW conditions_daily WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT location, time_bucket(INTERVAL '1 day', time) AS bucket, AVG(temperature) FROM conditions GROUP BY location, bucket WITH NO DATA; SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'conditions_daily' \gset -- Check the current watermark for an empty CAgg SELECT _timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)) AS watermak_empty_cagg; watermak_empty_cagg --------------------------------- Sun Nov 23 16:00:00 4714 UTC BC -- Refresh the CAGG CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); -- Check the watermark after the refresh and before truncate the CAgg SELECT _timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)) AS watermak_before; watermak_before ------------------------------ Fri Nov 02 16:00:00 2018 UTC -- Exists chunks before truncate the cagg (> 0) SELECT count(*) FROM show_chunks('conditions_daily'); count ------- 2 -- Truncate the given CAgg, it should reset the watermark to the empty state TRUNCATE conditions_daily; -- No chunks remains after truncate the cagg (= 0) SELECT count(*) FROM show_chunks('conditions_daily'); count ------- 0 -- Watermark should be reseted SELECT _timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)) AS watermak_after; watermak_after --------------------------------- Sun Nov 23 16:00:00 4714 UTC BC -- Should return ROWS because the watermark was reseted by the TRUNCATE SELECT * FROM conditions_daily ORDER BY bucket, location; location | bucket | avg ----------+------------------------------+----- SFO | Sun Dec 31 16:00:00 2017 UTC | 55 NYC | Mon Jan 01 16:00:00 2018 UTC | 65 POR | Mon Jan 01 16:00:00 2018 UTC | 100 SFO | Mon Jan 01 16:00:00 2018 UTC | 65 NYC | Wed Oct 31 16:00:00 2018 UTC | 65 NYC | Thu Nov 01 16:00:00 2018 UTC | 15 -- check compression settings are cleaned up when deleting a cagg with compression CREATE TABLE cagg_cleanup(time timestamptz not null); SELECT table_name FROM create_hypertable('cagg_cleanup','time'); table_name -------------- cagg_cleanup INSERT INTO cagg_cleanup SELECT '2020-01-01'; CREATE MATERIALIZED VIEW cagg1 WITH (timescaledb.continuous) AS SELECT time_bucket('1h',time) FROM cagg_cleanup GROUP BY 1; NOTICE: refreshing continuous aggregate "cagg1" ALTER MATERIALIZED VIEW cagg1 SET (timescaledb.compress); NOTICE: defaulting compress_orderby to time_bucket SELECT count(compress_chunk(ch)) FROM show_chunks('cagg1') ch; count ------- 1 DROP MATERIALIZED VIEW cagg1; NOTICE: drop cascades to table _timescaledb_internal._hyper_57_70_chunk SELECT * FROM _timescaledb_catalog.compression_settings; relid | compress_relid | segmentby | orderby | orderby_desc | orderby_nullsfirst | index -------+----------------+-----------+---------+--------------+--------------------+------- -- test WITH namespace alias CREATE TABLE with_alias(time timestamptz not null); CREATE MATERIALIZED VIEW cagg_alias WITH (tsdb.continuous, tsdb.materialized_only=false) AS SELECT time_bucket(INTERVAL '1 day', time) FROM conditions GROUP BY 1 WITH NO DATA; ALTER MATERIALIZED VIEW cagg_alias SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_alias SET (tsdb.materialized_only=false); DROP MATERIALIZED VIEW cagg_alias; -- test SET chunk_time_interval CREATE MATERIALIZED VIEW cagg_set WITH (tsdb.continuous, tsdb.chunk_interval='1day') AS SELECT time_bucket(INTERVAL '1 day', time) AS cagg_interval_setter FROM conditions GROUP BY 1 WITH NO DATA; SELECT column_name, time_interval FROM timescaledb_information.dimensions WHERE column_name='cagg_interval_setter'; column_name | time_interval ----------------------+--------------- cagg_interval_setter | @ 1 day ALTER MATERIALIZED VIEW cagg_set SET (tsdb.chunk_interval='23 day'); SELECT column_name, time_interval FROM timescaledb_information.dimensions WHERE column_name='cagg_interval_setter'; column_name | time_interval ----------------------+--------------- cagg_interval_setter | @ 23 days ALTER MATERIALIZED VIEW cagg_set SET (tsdb.chunk_interval='6 month'); SELECT column_name, time_interval FROM timescaledb_information.dimensions WHERE column_name='cagg_interval_setter'; column_name | time_interval ----------------------+--------------- cagg_interval_setter | @ 180 days ALTER MATERIALIZED VIEW cagg_set SET (tsdb.chunk_interval='1 year'); SELECT column_name, time_interval FROM timescaledb_information.dimensions WHERE column_name='cagg_interval_setter'; column_name | time_interval ----------------------+--------------- cagg_interval_setter | @ 360 days -- test cagg with stable functions CREATE MATERIALIZED VIEW cagg_stable WITH (tsdb.continuous) AS SELECT sum(temperature), max(time + INTERVAL '1h') FROM conditions GROUP BY time_bucket('1week', time), location; WARNING: using non-immutable functions in continuous aggregate view may lead to inconsistent results on rematerialization NOTICE: refreshing continuous aggregate "cagg_stable" SELECT * FROM cagg_stable t ORDER BY t; sum | max -----+------------------------------ 65 | Tue Jan 02 10:10:00 2018 UTC 100 | Tue Jan 02 10:30:00 2018 UTC 120 | Tue Jan 02 10:20:00 2018 UTC 355 | Fri Nov 02 11:30:00 2018 UTC --aggregate without combine function but stable function CREATE MATERIALIZED VIEW cagg_json_agg WITH (tsdb.continuous, tsdb.materialized_only=false) AS SELECT json_agg(location) from conditions group by time_bucket('1week', time), location WITH NO DATA; WARNING: using non-immutable functions in continuous aggregate view may lead to inconsistent results on rematerialization CREATE FUNCTION test_stablefunc(int) RETURNS int LANGUAGE 'sql' STABLE AS 'SELECT $1 + 10'; CREATE MATERIALIZED VIEW cagg_stable2 WITH (tsdb.continuous) AS SELECT sum(test_stablefunc(temperature::int)), min(location) FROM conditions GROUP BY time_bucket('1week', time) WITH NO DATA; WARNING: using non-immutable functions in continuous aggregate view may lead to inconsistent results on rematerialization CREATE MATERIALIZED VIEW cagg_stable3 WITH (tsdb.continuous) AS SELECT sum(temperature), min(location) FROM conditions GROUP BY time_bucket('1week', time), test_stablefunc(temperature::int) WITH NO DATA; WARNING: using non-immutable functions in continuous aggregate view may lead to inconsistent results on rematerialization -- test window functions in caggs -- first do sanity check that we error without the GUC \set ON_ERROR_STOP 0 CREATE MATERIALIZED VIEW cagg_window WITH (tsdb.continuous) AS SELECT time_bucket('1week', time), rank() OVER (PARTITION BY time_bucket('1 week',time)) FROM conditions GROUP BY 1; ERROR: invalid continuous aggregate query \set ON_ERROR_STOP 1 SET timescaledb.enable_cagg_window_functions TO on; CREATE MATERIALIZED VIEW cagg_window_1 WITH (tsdb.continuous) AS SELECT time_bucket('1week', time), rank() OVER (PARTITION BY time_bucket('1 week',time)) FROM conditions GROUP BY 1; WARNING: window function support is experimental and may result in unexpected results depending on the functions used. NOTICE: refreshing continuous aggregate "cagg_window_1" CREATE MATERIALIZED VIEW cagg_window_2 WITH (tsdb.continuous) AS SELECT time_bucket('1week', time), rank() OVER (PARTITION BY time_bucket('1 week',time), location) FROM conditions GROUP BY 1, location; WARNING: window function support is experimental and may result in unexpected results depending on the functions used. NOTICE: refreshing continuous aggregate "cagg_window_2" CREATE MATERIALIZED VIEW cagg_window_3 WITH (tsdb.continuous) AS SELECT time_bucket('1week', time), rank() OVER (PARTITION BY time_bucket('1 week',time)) FROM conditions GROUP BY 1, location; WARNING: window function support is experimental and may result in unexpected results depending on the functions used. NOTICE: refreshing continuous aggregate "cagg_window_3" CREATE MATERIALIZED VIEW cagg_window_4 WITH (tsdb.continuous) AS SELECT time_bucket('1week', time), rank() OVER w FROM conditions GROUP BY 1, location WINDOW w AS (PARTITION BY time_bucket('1 week',time)); WARNING: window function support is experimental and may result in unexpected results depending on the functions used. NOTICE: refreshing continuous aggregate "cagg_window_4" -- test setting chunk_interval on a cagg CREATE MATERIALIZED VIEW cagg_chunk_interval WITH (tsdb.continuous, tsdb.chunk_interval='1000 day') AS SELECT time_bucket('1 week', time) FROM conditions GROUP BY 1 WITH NO DATA; SELECT time_interval from timescaledb_information.continuous_aggregates cagg INNER JOIN timescaledb_information.dimensions dim ON cagg.materialization_hypertable_name = dim.hypertable_name WHERE view_name='cagg_chunk_interval'; time_interval --------------- @ 1000 days ALTER MATERIALIZED VIEW cagg_chunk_interval SET (tsdb.chunk_interval='110 day'); SELECT time_interval from timescaledb_information.continuous_aggregates cagg INNER JOIN timescaledb_information.dimensions dim ON cagg.materialization_hypertable_name = dim.hypertable_name WHERE view_name='cagg_chunk_interval'; time_interval --------------- @ 110 days -- test columnstore options CREATE MATERIALIZED VIEW columnstore_options WITH (tsdb.continuous, tsdb.chunk_interval='1 day') AS SELECT time_bucket('1 day', time) FROM conditions GROUP BY 1 WITH NO DATA; SELECT column_name, compress_interval_length from _timescaledb_catalog.dimension where column_name='time_bucket' ORDER BY id DESC LIMIT 1; column_name | compress_interval_length -------------+-------------------------- time_bucket | ALTER MATERIALIZED VIEW columnstore_options SET (tsdb.columnstore, tsdb.chunk_interval='1 day', tsdb.orderby='time_bucket DESC', tsdb.compress_chunk_interval='2 day'); SELECT column_name, compress_interval_length from _timescaledb_catalog.dimension where column_name='time_bucket' ORDER BY id DESC LIMIT 1; column_name | compress_interval_length -------------+-------------------------- time_bucket | 172800000000 ALTER MATERIALIZED VIEW columnstore_options SET (tsdb.compress_chunk_interval='3 day'); SELECT column_name, compress_interval_length from _timescaledb_catalog.dimension where column_name='time_bucket' ORDER BY id DESC LIMIT 1; column_name | compress_interval_length -------------+-------------------------- time_bucket | 259200000000 ALTER MATERIALIZED VIEW columnstore_options SET (tsdb.compress_chunk_time_interval='4 day'); SELECT column_name, compress_interval_length from _timescaledb_catalog.dimension where column_name='time_bucket' ORDER BY id DESC LIMIT 1; column_name | compress_interval_length -------------+-------------------------- time_bucket | 345600000000 -- test set returning functions in caggs CREATE TABLE kpis_raw (time TIMESTAMP NOT NULL, value INTEGER, groups TEXT[]) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column INSERT INTO kpis_raw (time, value, groups) VALUES ('2025-01-01', 10, '{group1,group2,group3}'), ('2025-01-02', 20, '{group1,group4}'), ('2025-02-01', 10, '{group1,group3}'), ('2025-02-01', 20, '{group1,group4}'); CREATE MATERIALIZED VIEW kpis_cagg WITH (tsdb.continuous) AS SELECT time_bucket('7 day', time) AS bucket, count(*) AS number_of_records, avg(value) AS average, unnest(groups) AS kpi_group FROM kpis_raw GROUP BY bucket, kpi_group; NOTICE: refreshing continuous aggregate "kpis_cagg" SELECT * FROM kpis_cagg ORDER BY bucket, kpi_group; bucket | number_of_records | average | kpi_group --------------------------+-------------------+---------------------+----------- Mon Dec 30 00:00:00 2024 | 2 | 15.0000000000000000 | group1 Mon Dec 30 00:00:00 2024 | 1 | 10.0000000000000000 | group2 Mon Dec 30 00:00:00 2024 | 1 | 10.0000000000000000 | group3 Mon Dec 30 00:00:00 2024 | 1 | 20.0000000000000000 | group4 Mon Jan 27 00:00:00 2025 | 2 | 15.0000000000000000 | group1 Mon Jan 27 00:00:00 2025 | 1 | 10.0000000000000000 | group3 Mon Jan 27 00:00:00 2025 | 1 | 20.0000000000000000 | group4 --TEST for caggs with non timescaledb namespace options --non timescaledb namespace options can be set via ALTER \set ON_ERROR_STOP 0 -- will error out CREATE MATERIALIZED VIEW ht_try_weekly WITH (timescaledb.continuous, tigerlake.newoption = true) AS SELECT time_bucket(interval '1 week', time) AS ts_bucket, avg(value) FROM kpis_raw GROUP BY 1; ERROR: non "timescaledb" namespace options can be set only via ALTER --caught by Postgres now ALTER MATERIALIZED VIEW kpis_cagg SET (tigerlake.newoption = true); ERROR: "kpis_cagg" is not a materialized view \set ON_ERROR_STOP 1 -- TEST that cached alter stmt still works (see PR 8739) -- test DDL inside function CREATE TABLE hypertab_ddl( ts timestamp, a integer) WITH (timescaledb.hypertable); NOTICE: using column "ts" as partitioning column CREATE OR REPLACE FUNCTION ddl_function() RETURNS VOID LANGUAGE PLPGSQL AS $$ BEGIN DROP MATERIALIZED VIEW IF EXISTS cagg_hypertab_ddl; CREATE MATERIALIZED VIEW cagg_hypertab_ddl WITH (timescaledb.continuous) AS SELECT time_bucket( '1 day'::interval, ts), COUNT(*) FROM hypertab_ddl GROUP BY 1 WITH NO DATA; END $$; SELECT ddl_function(); NOTICE: materialized view "cagg_hypertab_ddl" does not exist, skipping ddl_function -------------- SELECT view_name from timescaledb_information.continuous_aggregates WHERE hypertable_name='hypertab_ddl'; view_name ------------------- cagg_hypertab_ddl SELECT ddl_function(); ddl_function -------------- SELECT view_name from timescaledb_information.continuous_aggregates WHERE hypertable_name='hypertab_ddl'; view_name ------------------- cagg_hypertab_ddl -- TEST continuous aggregate with functionally dependent columns -- name column depends on id which is in GROUP BY so name doesnt have to be CREATE table sensor(id int PRIMARY KEY, name text); CREATE TABLE sensordata(time timestamptz,id int, value float) WITH (timescaledb.hypertable); NOTICE: using column "time" as partitioning column CREATE MATERIALIZED VIEW cagg_sensordata WITH (tsdb.continuous) AS SELECT s.id, s.name,time_bucket('15 minutes', sd.time) as bucket, avg(sd.value) FROM sensordata sd JOIN sensor s USING(id) GROUP BY s.id, bucket WITH NO DATA; SELECT * FROM cagg_sensordata; id | name | bucket | avg ----+------+--------+----- ================================================ FILE: tsl/test/expected/cagg_ddl-18.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- Set this variable to avoid using a hard-coded path each time query -- results are compared \set QUERY_RESULT_TEST_EQUAL_RELPATH '../../../../test/sql/include/query_result_test_equal.sql' SET ROLE :ROLE_DEFAULT_PERM_USER; --DDL commands on continuous aggregates CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature integer NULL, humidity DOUBLE PRECISION NULL, timemeasure TIMESTAMPTZ, timeinterval INTERVAL ); SELECT table_name FROM create_hypertable('conditions', 'timec'); table_name ------------ conditions -- schema tests \c :TEST_DBNAME :ROLE_SUPERUSER SET timezone TO 'UTC+8'; -- drop if the tablespace1 and/or tablespace2 exists SET client_min_messages TO error; DROP TABLESPACE IF EXISTS tablespace1; DROP TABLESPACE IF EXISTS tablespace2; RESET client_min_messages; CREATE TABLESPACE tablespace1 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE1_PATH; CREATE TABLESPACE tablespace2 OWNER :ROLE_DEFAULT_PERM_USER LOCATION :TEST_TABLESPACE2_PATH; CREATE SCHEMA rename_schema; GRANT ALL ON SCHEMA rename_schema TO :ROLE_DEFAULT_PERM_USER; CREATE SCHEMA test_schema AUTHORIZATION :ROLE_DEFAULT_PERM_USER; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE TABLE foo(time TIMESTAMPTZ NOT NULL, data INTEGER); SELECT create_hypertable('foo', 'time'); create_hypertable ------------------- (2,public,foo,t) CREATE MATERIALIZED VIEW rename_test_old WITH ( timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1week', time), COUNT(data) FROM foo GROUP BY 1 WITH NO DATA; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+-----------------+-----------------------+------------------- public | rename_test_old | _timescaledb_internal | _partial_view_3 ALTER TABLE rename_test_old RENAME TO rename_test; ALTER TABLE rename_test SET SCHEMA test_schema; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+-----------------------+------------------- test_schema | rename_test | _timescaledb_internal | _partial_view_3 ALTER MATERIALIZED VIEW test_schema.rename_test SET SCHEMA rename_schema; DROP SCHEMA test_schema; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+-----------------------+------------------- rename_schema | rename_test | _timescaledb_internal | _partial_view_3 SELECT ca.raw_hypertable_id as "RAW_HYPERTABLE_ID", h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA", direct_view_name as "DIR_VIEW_NAME", direct_view_schema as "DIR_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'rename_test' \gset RESET ROLE; SELECT current_user; current_user -------------- super_user ALTER VIEW :"PART_VIEW_SCHEMA".:"PART_VIEW_NAME" SET SCHEMA public; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+---------------------+------------------- rename_schema | rename_test | public | _partial_view_3 --alter direct view schema SELECT user_view_schema, user_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | direct_view_schema | direct_view_name ------------------+----------------+-----------------------+------------------ rename_schema | rename_test | _timescaledb_internal | _direct_view_3 RESET ROLE; SELECT current_user; current_user -------------- super_user ALTER VIEW :"DIR_VIEW_SCHEMA".:"DIR_VIEW_NAME" SET SCHEMA public; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name ------------------+----------------+---------------------+-------------------+--------------------+------------------ rename_schema | rename_test | public | _partial_view_3 | public | _direct_view_3 RESET ROLE; SELECT current_user; current_user -------------- super_user ALTER SCHEMA rename_schema RENAME TO new_name_schema; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name ------------------+----------------+---------------------+-------------------+--------------------+------------------ new_name_schema | rename_test | public | _partial_view_3 | public | _direct_view_3 ALTER VIEW :"PART_VIEW_NAME" SET SCHEMA new_name_schema; ALTER VIEW :"DIR_VIEW_NAME" SET SCHEMA new_name_schema; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name ------------------+----------------+---------------------+-------------------+--------------------+------------------ new_name_schema | rename_test | new_name_schema | _partial_view_3 | new_name_schema | _direct_view_3 RESET ROLE; SELECT current_user; current_user -------------- super_user ALTER SCHEMA new_name_schema RENAME TO foo_name_schema; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+---------------------+------------------- foo_name_schema | rename_test | foo_name_schema | _partial_view_3 ALTER MATERIALIZED VIEW foo_name_schema.rename_test SET SCHEMA public; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+---------------------+------------------- public | rename_test | foo_name_schema | _partial_view_3 RESET ROLE; SELECT current_user; current_user -------------- super_user ALTER SCHEMA foo_name_schema RENAME TO rename_schema; SET ROLE :ROLE_DEFAULT_PERM_USER; SET client_min_messages TO NOTICE; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+----------------+---------------------+------------------- public | rename_test | rename_schema | _partial_view_3 ALTER MATERIALIZED VIEW rename_test RENAME TO rename_c_aggregate; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name ------------------+--------------------+---------------------+------------------- public | rename_c_aggregate | rename_schema | _partial_view_3 SELECT * FROM rename_c_aggregate; time_bucket | count -------------+------- ALTER VIEW rename_schema.:"PART_VIEW_NAME" RENAME TO partial_view; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name ------------------+--------------------+---------------------+-------------------+--------------------+------------------ public | rename_c_aggregate | rename_schema | partial_view | rename_schema | _direct_view_3 --rename direct view ALTER VIEW rename_schema.:"DIR_VIEW_NAME" RENAME TO direct_view; SELECT user_view_schema, user_view_name, partial_view_schema, partial_view_name, direct_view_schema, direct_view_name FROM _timescaledb_catalog.continuous_agg; user_view_schema | user_view_name | partial_view_schema | partial_view_name | direct_view_schema | direct_view_name ------------------+--------------------+---------------------+-------------------+--------------------+------------------ public | rename_c_aggregate | rename_schema | partial_view | rename_schema | direct_view -- drop_chunks tests DROP TABLE conditions CASCADE; DROP TABLE foo CASCADE; NOTICE: drop cascades to 2 other objects CREATE TABLE drop_chunks_table(time BIGINT NOT NULL, data INTEGER); SELECT hypertable_id AS drop_chunks_table_id FROM create_hypertable('drop_chunks_table', 'time', chunk_time_interval => 10) \gset CREATE OR REPLACE FUNCTION integer_now_test() returns bigint LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), bigint '0') FROM drop_chunks_table $$; SELECT set_integer_now_func('drop_chunks_table', 'integer_now_test'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW drop_chunks_view WITH ( timescaledb.continuous, timescaledb.materialized_only=true ) AS SELECT time_bucket('5', time), COUNT(data) FROM drop_chunks_table GROUP BY 1 WITH NO DATA; SELECT format('%I.%I', schema_name, table_name) AS drop_chunks_mat_table, schema_name AS drop_chunks_mat_schema, table_name AS drop_chunks_mat_table_name FROM _timescaledb_catalog.hypertable, _timescaledb_catalog.continuous_agg WHERE _timescaledb_catalog.continuous_agg.raw_hypertable_id = :drop_chunks_table_id AND _timescaledb_catalog.hypertable.id = _timescaledb_catalog.continuous_agg.mat_hypertable_id \gset -- create 3 chunks, with 3 time bucket INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(0, 29) AS i; -- Only refresh up to bucket 15 initially. Matches the old refresh -- behavior that didn't materialize everything CALL refresh_continuous_aggregate('drop_chunks_view', 0, 15); SELECT count(c) FROM show_chunks('drop_chunks_table') AS c; count ------- 3 SELECT count(c) FROM show_chunks('drop_chunks_view') AS c; count ------- 1 SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | count -------------+------- 0 | 5 5 | 5 10 | 5 -- cannot drop directly from the materialization table without specifying -- cont. aggregate view name explicitly \set ON_ERROR_STOP 0 SELECT drop_chunks(:'drop_chunks_mat_table', newer_than => -20, verbose => true); ERROR: operation not supported on materialized hypertable \set ON_ERROR_STOP 1 SELECT count(c) FROM show_chunks('drop_chunks_table') AS c; count ------- 3 SELECT count(c) FROM show_chunks('drop_chunks_view') AS c; count ------- 1 SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | count -------------+------- 0 | 5 5 | 5 10 | 5 -- drop chunks when the chunksize and time_bucket aren't aligned DROP TABLE drop_chunks_table CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_5_4_chunk CREATE TABLE drop_chunks_table_u(time BIGINT NOT NULL, data INTEGER); SELECT hypertable_id AS drop_chunks_table_u_id FROM create_hypertable('drop_chunks_table_u', 'time', chunk_time_interval => 7) \gset CREATE OR REPLACE FUNCTION integer_now_test1() returns bigint LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), bigint '0') FROM drop_chunks_table_u $$; SELECT set_integer_now_func('drop_chunks_table_u', 'integer_now_test1'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW drop_chunks_view WITH ( timescaledb.continuous, timescaledb.materialized_only=true ) AS SELECT time_bucket('3', time), COUNT(data) FROM drop_chunks_table_u GROUP BY 1 WITH NO DATA; SELECT format('%I.%I', schema_name, table_name) AS drop_chunks_mat_table_u, schema_name AS drop_chunks_mat_schema, table_name AS drop_chunks_mat_table_u_name FROM _timescaledb_catalog.hypertable, _timescaledb_catalog.continuous_agg WHERE _timescaledb_catalog.continuous_agg.raw_hypertable_id = :drop_chunks_table_u_id AND _timescaledb_catalog.hypertable.id = _timescaledb_catalog.continuous_agg.mat_hypertable_id \gset -- create 3 chunks, with 3 time bucket INSERT INTO drop_chunks_table_u SELECT i, i FROM generate_series(0, 21) AS i; -- Refresh up to bucket 15 to match old materializer behavior CALL refresh_continuous_aggregate('drop_chunks_view', 0, 15); SELECT count(c) FROM show_chunks('drop_chunks_table_u') AS c; count ------- 4 SELECT count(c) FROM show_chunks('drop_chunks_view') AS c; count ------- 1 SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | count -------------+------- 0 | 3 3 | 3 6 | 3 9 | 3 12 | 3 -- TRUNCATE test -- Can truncate regular hypertables that have caggs TRUNCATE drop_chunks_table_u; \set ON_ERROR_STOP 0 -- Can't truncate materialized hypertables directly TRUNCATE :drop_chunks_mat_table_u; ERROR: cannot TRUNCATE a hypertable underlying a continuous aggregate \set ON_ERROR_STOP 1 -- Check that we don't interfere with TRUNCATE of normal table and -- partitioned table CREATE TABLE truncate (value int); INSERT INTO truncate VALUES (1), (2); TRUNCATE truncate; SELECT * FROM truncate; value ------- CREATE TABLE truncate_partitioned (value int) PARTITION BY RANGE(value); CREATE TABLE truncate_p1 PARTITION OF truncate_partitioned FOR VALUES FROM (1) TO (3); INSERT INTO truncate_partitioned VALUES (1), (2); TRUNCATE truncate_partitioned; SELECT * FROM truncate_partitioned; value ------- -- ALTER TABLE tests \set ON_ERROR_STOP 0 -- test a variety of ALTER TABLE statements ALTER TABLE :drop_chunks_mat_table_u RENAME time_bucket TO bad_name; ERROR: renaming columns on materialization tables is not supported ALTER TABLE :drop_chunks_mat_table_u ADD UNIQUE(time_bucket); ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u SET UNLOGGED; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u ENABLE ROW LEVEL SECURITY; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u ADD COLUMN fizzle INTEGER; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u DROP COLUMN time_bucket; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u ALTER COLUMN time_bucket DROP NOT NULL; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u ALTER COLUMN time_bucket SET DEFAULT 1; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u ALTER COLUMN time_bucket SET STORAGE EXTERNAL; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u DISABLE TRIGGER ALL; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u SET TABLESPACE foo; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u NOT OF; ERROR: operation not supported on materialization tables ALTER TABLE :drop_chunks_mat_table_u OWNER TO CURRENT_USER; ERROR: operation not supported on materialization tables \set ON_ERROR_STOP 1 ALTER TABLE :drop_chunks_mat_table_u SET SCHEMA public; ALTER TABLE :drop_chunks_mat_table_u_name RENAME TO new_name; SET ROLE :ROLE_DEFAULT_PERM_USER; SET client_min_messages TO NOTICE; SELECT * FROM new_name ORDER BY 1; time_bucket | count -------------+------- 0 | 3 3 | 3 6 | 3 9 | 3 12 | 3 SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | count -------------+------- 0 | 3 3 | 3 6 | 3 9 | 3 12 | 3 \set ON_ERROR_STOP 0 -- no continuous aggregates on a continuous aggregate materialization table CREATE MATERIALIZED VIEW new_name_view WITH ( timescaledb.continuous, timescaledb.materialized_only=true ) AS SELECT time_bucket('6', time_bucket), COUNT("count") FROM new_name GROUP BY 1 WITH NO DATA; ERROR: hypertable is a continuous aggregate materialization table \set ON_ERROR_STOP 1 CREATE TABLE metrics(time timestamptz NOT NULL, device_id int, v1 float, v2 float); SELECT create_hypertable('metrics','time'); create_hypertable ---------------------- (8,public,metrics,t) INSERT INTO metrics SELECT generate_series('2000-01-01'::timestamptz,'2000-01-10','1m'),1,0.25,0.75; -- check expressions in view definition CREATE MATERIALIZED VIEW cagg_expr WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1d', time) AS time, 'Const'::text AS Const, 4.3::numeric AS "numeric", first(metrics,time), CASE WHEN true THEN 'foo' ELSE 'bar' END, COALESCE(NULL,'coalesce'), avg(v1) + avg(v2) AS avg1, avg(v1+v2) AS avg2 FROM metrics GROUP BY 1 WITH NO DATA; CALL refresh_continuous_aggregate('cagg_expr', NULL, NULL); SELECT * FROM cagg_expr ORDER BY time LIMIT 5; time | const | numeric | first | case | coalesce | avg1 | avg2 ------------------------------+-------+---------+----------------------------------------------+------+----------+------+------ Fri Dec 31 16:00:00 1999 UTC | Const | 4.3 | ("Sat Jan 01 00:00:00 2000 UTC",1,0.25,0.75) | foo | coalesce | 1 | 1 Sat Jan 01 16:00:00 2000 UTC | Const | 4.3 | ("Sat Jan 01 16:00:00 2000 UTC",1,0.25,0.75) | foo | coalesce | 1 | 1 Sun Jan 02 16:00:00 2000 UTC | Const | 4.3 | ("Sun Jan 02 16:00:00 2000 UTC",1,0.25,0.75) | foo | coalesce | 1 | 1 Mon Jan 03 16:00:00 2000 UTC | Const | 4.3 | ("Mon Jan 03 16:00:00 2000 UTC",1,0.25,0.75) | foo | coalesce | 1 | 1 Tue Jan 04 16:00:00 2000 UTC | Const | 4.3 | ("Tue Jan 04 16:00:00 2000 UTC",1,0.25,0.75) | foo | coalesce | 1 | 1 --test materialization of invalidation before drop DROP TABLE IF EXISTS drop_chunks_table CASCADE; NOTICE: table "drop_chunks_table" does not exist, skipping DROP TABLE IF EXISTS drop_chunks_table_u CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_7_9_chunk CREATE TABLE drop_chunks_table(time BIGINT NOT NULL, data INTEGER); SELECT hypertable_id AS drop_chunks_table_nid FROM create_hypertable('drop_chunks_table', 'time', chunk_time_interval => 10) \gset CREATE OR REPLACE FUNCTION integer_now_test2() returns bigint LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), bigint '0') FROM drop_chunks_table $$; SELECT set_integer_now_func('drop_chunks_table', 'integer_now_test2'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW drop_chunks_view WITH ( timescaledb.continuous, timescaledb.materialized_only=true ) AS SELECT time_bucket('5', time), max(data) FROM drop_chunks_table GROUP BY 1 WITH NO DATA; INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(0, 20) AS i; --dropping chunks will process the invalidations SELECT drop_chunks('drop_chunks_table', older_than => (integer_now_test2()-9)); drop_chunks ------------------------------------------ _timescaledb_internal._hyper_10_13_chunk SELECT * FROM drop_chunks_table ORDER BY time ASC limit 1; time | data ------+------ 10 | 10 INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(20, 35) AS i; CALL refresh_continuous_aggregate('drop_chunks_view', 10, 40); --this will be seen after the drop its within the invalidation window and will be dropped INSERT INTO drop_chunks_table VALUES (26, 100); --this will not be processed by the drop since chunk 30-39 is not dropped but will be seen after refresh --shows that the drop doesn't do more work than necessary INSERT INTO drop_chunks_table VALUES (31, 200); --move the time up to 39 INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(35, 39) AS i; --the chunks and ranges we have thus far SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table'; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_10_14_chunk | 10 | 20 _hyper_10_15_chunk | 20 | 30 _hyper_10_16_chunk | 30 | 40 --the invalidation on 25 not yet seen SELECT * FROM drop_chunks_view ORDER BY time_bucket DESC; time_bucket | max -------------+----- 35 | 35 30 | 34 25 | 29 20 | 24 15 | 19 10 | 14 --refresh to process the invalidations and then drop CALL refresh_continuous_aggregate('drop_chunks_view', NULL, (integer_now_test2()-9)); SELECT drop_chunks('drop_chunks_table', older_than => (integer_now_test2()-9)); drop_chunks ------------------------------------------ _timescaledb_internal._hyper_10_14_chunk _timescaledb_internal._hyper_10_15_chunk --new values on 25 now seen in view SELECT * FROM drop_chunks_view ORDER BY time_bucket DESC; time_bucket | max -------------+----- 35 | 35 30 | 34 25 | 100 20 | 24 15 | 19 10 | 14 --earliest datapoint now in table SELECT * FROM drop_chunks_table ORDER BY time ASC limit 1; time | data ------+------ 30 | 30 --still see data in the view SELECT * FROM drop_chunks_view WHERE time_bucket < (integer_now_test2()-9) ORDER BY time_bucket DESC; time_bucket | max -------------+----- 25 | 100 20 | 24 15 | 19 10 | 14 --no data but covers dropped chunks SELECT * FROM drop_chunks_table WHERE time < (integer_now_test2()-9) ORDER BY time DESC; time | data ------+------ --recreate the dropped chunk INSERT INTO drop_chunks_table SELECT i, i FROM generate_series(0, 20) AS i; --see data from recreated region SELECT * FROM drop_chunks_table WHERE time < (integer_now_test2()-9) ORDER BY time DESC; time | data ------+------ 20 | 20 19 | 19 18 | 18 17 | 17 16 | 16 15 | 15 14 | 14 13 | 13 12 | 12 11 | 11 10 | 10 9 | 9 8 | 8 7 | 7 6 | 6 5 | 5 4 | 4 3 | 3 2 | 2 1 | 1 0 | 0 --should show chunk with old name and old ranges SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table' ORDER BY range_start_integer; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_10_18_chunk | 0 | 10 _hyper_10_19_chunk | 10 | 20 _hyper_10_20_chunk | 20 | 30 _hyper_10_16_chunk | 30 | 40 --We dropped everything up to the bucket starting at 30 and then --inserted new data up to and including time 20. Therefore, the --dropped data should stay the same as long as we only refresh --buckets that have non-dropped data. CALL refresh_continuous_aggregate('drop_chunks_view', 30, 40); SELECT * FROM drop_chunks_view ORDER BY time_bucket DESC; time_bucket | max -------------+----- 35 | 39 30 | 200 25 | 100 20 | 24 15 | 19 10 | 14 SELECT format('%I.%I', schema_name, table_name) AS drop_chunks_mat_tablen, schema_name AS drop_chunks_mat_schema, table_name AS drop_chunks_mat_table_name FROM _timescaledb_catalog.hypertable, _timescaledb_catalog.continuous_agg WHERE _timescaledb_catalog.continuous_agg.raw_hypertable_id = :drop_chunks_table_nid AND _timescaledb_catalog.hypertable.id = _timescaledb_catalog.continuous_agg.mat_hypertable_id \gset -- TEST drop chunks from continuous aggregates by specifying view name SELECT drop_chunks('drop_chunks_view', newer_than => -20, verbose => true); INFO: dropping chunk _timescaledb_internal._hyper_11_17_chunk drop_chunks ------------------------------------------ _timescaledb_internal._hyper_11_17_chunk -- Test that we cannot drop chunks when specifying materialized -- hypertable INSERT INTO drop_chunks_table SELECT generate_series(45, 55), 500; CALL refresh_continuous_aggregate('drop_chunks_view', 45, 55); SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = :'drop_chunks_mat_table_name' ORDER BY range_start_integer; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_11_23_chunk | 0 | 100 \set ON_ERROR_STOP 0 \set VERBOSITY default SELECT drop_chunks(:'drop_chunks_mat_tablen', older_than => 60); ERROR: operation not supported on materialized hypertable DETAIL: Hypertable "_materialized_hypertable_11" is a materialized hypertable. HINT: Try the operation on the continuous aggregate instead. \set VERBOSITY terse \set ON_ERROR_STOP 1 ----------------------------------------------------------------- -- Test that refresh_continuous_aggregate on chunk will refresh, -- but only in the regions covered by the show chunks. ----------------------------------------------------------------- SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table' ORDER BY 2,3; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_10_18_chunk | 0 | 10 _hyper_10_19_chunk | 10 | 20 _hyper_10_20_chunk | 20 | 30 _hyper_10_16_chunk | 30 | 40 _hyper_10_21_chunk | 40 | 50 _hyper_10_22_chunk | 50 | 60 -- Pick the second chunk as the one to drop WITH numbered_chunks AS ( SELECT row_number() OVER (ORDER BY range_start_integer), chunk_schema, chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table' ORDER BY 1 ) SELECT format('%I.%I', chunk_schema, chunk_name) AS chunk_to_drop, range_start_integer, range_end_integer FROM numbered_chunks WHERE row_number = 2 \gset -- There's data in the table for the chunk/range we will drop SELECT * FROM drop_chunks_table WHERE time >= :range_start_integer AND time < :range_end_integer ORDER BY 1; time | data ------+------ 10 | 10 11 | 11 12 | 12 13 | 13 14 | 14 15 | 15 16 | 16 17 | 17 18 | 18 19 | 19 -- Make sure there is also data in the continuous aggregate -- CARE: -- Note that this behaviour of dropping the materialization table chunks and expecting a refresh -- that overlaps that time range to NOT update those chunks is undefined. CALL refresh_continuous_aggregate('drop_chunks_view', 0, 50); SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | max -------------+----- 0 | 4 5 | 9 10 | 14 15 | 19 20 | 20 45 | 500 50 | 500 -- Drop the second chunk, to leave a gap in the data DROP TABLE :chunk_to_drop; -- Verify that the second chunk is dropped SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table' ORDER BY 2,3; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_10_18_chunk | 0 | 10 _hyper_10_20_chunk | 20 | 30 _hyper_10_16_chunk | 30 | 40 _hyper_10_21_chunk | 40 | 50 _hyper_10_22_chunk | 50 | 60 -- Data is no longer in the table but still in the view SELECT * FROM drop_chunks_table WHERE time >= :range_start_integer AND time < :range_end_integer ORDER BY 1; time | data ------+------ SELECT * FROM drop_chunks_view WHERE time_bucket >= :range_start_integer AND time_bucket < :range_end_integer ORDER BY 1; time_bucket | max -------------+----- 10 | 14 15 | 19 -- Insert a large value in one of the chunks that will be dropped INSERT INTO drop_chunks_table VALUES (:range_start_integer-1, 100); -- Now refresh and drop the two adjecent chunks CALL refresh_continuous_aggregate('drop_chunks_view', NULL, 30); SELECT drop_chunks('drop_chunks_table', older_than=>30); drop_chunks ------------------------------------------ _timescaledb_internal._hyper_10_18_chunk _timescaledb_internal._hyper_10_20_chunk -- Verify that the chunks are dropped SELECT chunk_name, range_start_integer, range_end_integer FROM timescaledb_information.chunks WHERE hypertable_name = 'drop_chunks_table' ORDER BY 2,3; chunk_name | range_start_integer | range_end_integer --------------------+---------------------+------------------- _hyper_10_16_chunk | 30 | 40 _hyper_10_21_chunk | 40 | 50 _hyper_10_22_chunk | 50 | 60 -- The continuous aggregate should be refreshed in the regions covered -- by the dropped chunks, but not in the "gap" region, i.e., the -- region of the chunk that was dropped via DROP TABLE. SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | max -------------+----- 0 | 4 5 | 100 20 | 20 45 | 500 50 | 500 -- Now refresh in the region of the first two dropped chunks CALL refresh_continuous_aggregate('drop_chunks_view', 0, :range_end_integer); -- Aggregate data in the refreshed range should no longer exist since -- the underlying data was dropped. SELECT * FROM drop_chunks_view ORDER BY 1; time_bucket | max -------------+----- 20 | 20 45 | 500 50 | 500 -------------------------------------------------------------------- -- Check that we can create a materialized table in a tablespace. We -- create one with tablespace and one without and compare them. CREATE VIEW cagg_info AS WITH caggs AS ( SELECT format('%I.%I', user_view_schema, user_view_name)::regclass AS user_view, format('%I.%I', direct_view_schema, direct_view_name)::regclass AS direct_view, format('%I.%I', partial_view_schema, partial_view_name)::regclass AS partial_view, format('%I.%I', ht.schema_name, ht.table_name)::regclass AS mat_relid FROM _timescaledb_catalog.hypertable ht, _timescaledb_catalog.continuous_agg cagg WHERE ht.id = cagg.mat_hypertable_id ) SELECT user_view, pg_get_userbyid(relowner) AS user_view_owner, relname AS mat_table, (SELECT pg_get_userbyid(relowner) FROM pg_class WHERE oid = mat_relid) AS mat_table_owner, direct_view, (SELECT pg_get_userbyid(relowner) FROM pg_class WHERE oid = direct_view) AS direct_view_owner, partial_view, (SELECT pg_get_userbyid(relowner) FROM pg_class WHERE oid = partial_view) AS partial_view_owner, (SELECT spcname FROM pg_tablespace WHERE oid = reltablespace) AS tablespace FROM pg_class JOIN caggs ON pg_class.oid = caggs.mat_relid; GRANT SELECT ON cagg_info TO PUBLIC; CREATE VIEW chunk_info AS SELECT ht.schema_name, ht.table_name, relname AS chunk_name, (SELECT spcname FROM pg_tablespace WHERE oid = reltablespace) AS tablespace FROM pg_class c, _timescaledb_catalog.hypertable ht, _timescaledb_catalog.chunk ch WHERE ch.table_name = c.relname AND ht.id = ch.hypertable_id; CREATE TABLE whatever(time BIGINT NOT NULL, data INTEGER); SELECT hypertable_id AS whatever_nid FROM create_hypertable('whatever', 'time', chunk_time_interval => 10) \gset SELECT set_integer_now_func('whatever', 'integer_now_test'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW whatever_view_1 WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('5', time), COUNT(data) FROM whatever GROUP BY 1 WITH NO DATA; CREATE MATERIALIZED VIEW whatever_view_2 WITH (timescaledb.continuous, timescaledb.materialized_only=true) TABLESPACE tablespace1 AS SELECT time_bucket('5', time), COUNT(data) FROM whatever GROUP BY 1 WITH NO DATA; INSERT INTO whatever SELECT i, i FROM generate_series(0, 29) AS i; CALL refresh_continuous_aggregate('whatever_view_1', NULL, NULL); CALL refresh_continuous_aggregate('whatever_view_2', NULL, NULL); SELECT user_view, mat_table, cagg_info.tablespace AS mat_tablespace, chunk_name, chunk_info.tablespace AS chunk_tablespace FROM cagg_info, chunk_info WHERE mat_table::text = table_name AND user_view::text LIKE 'whatever_view%'; user_view | mat_table | mat_tablespace | chunk_name | chunk_tablespace -----------------+-----------------------------+----------------+--------------------+------------------ whatever_view_1 | _materialized_hypertable_13 | | _hyper_13_27_chunk | whatever_view_2 | _materialized_hypertable_14 | tablespace1 | _hyper_14_28_chunk | tablespace1 ALTER MATERIALIZED VIEW whatever_view_1 SET TABLESPACE tablespace2; SELECT user_view, mat_table, cagg_info.tablespace AS mat_tablespace, chunk_name, chunk_info.tablespace AS chunk_tablespace FROM cagg_info, chunk_info WHERE mat_table::text = table_name AND user_view::text LIKE 'whatever_view%'; user_view | mat_table | mat_tablespace | chunk_name | chunk_tablespace -----------------+-----------------------------+----------------+--------------------+------------------ whatever_view_1 | _materialized_hypertable_13 | tablespace2 | _hyper_13_27_chunk | tablespace2 whatever_view_2 | _materialized_hypertable_14 | tablespace1 | _hyper_14_28_chunk | tablespace1 DROP MATERIALIZED VIEW whatever_view_1; NOTICE: drop cascades to table _timescaledb_internal._hyper_13_27_chunk DROP MATERIALIZED VIEW whatever_view_2; NOTICE: drop cascades to table _timescaledb_internal._hyper_14_28_chunk -- test bucket width expressions on integer hypertables CREATE TABLE metrics_int2 ( time int2 NOT NULL, device_id int, v1 float, v2 float ); CREATE TABLE metrics_int4 ( time int4 NOT NULL, device_id int, v1 float, v2 float ); CREATE TABLE metrics_int8 ( time int8 NOT NULL, device_id int, v1 float, v2 float ); SELECT create_hypertable (('metrics_' || dt)::regclass, 'time', chunk_time_interval => 10) FROM ( VALUES ('int2'), ('int4'), ('int8')) v (dt); create_hypertable ---------------------------- (15,public,metrics_int2,t) (16,public,metrics_int4,t) (17,public,metrics_int8,t) CREATE OR REPLACE FUNCTION int2_now () RETURNS int2 LANGUAGE SQL STABLE AS $$ SELECT 10::int2 $$; CREATE OR REPLACE FUNCTION int4_now () RETURNS int4 LANGUAGE SQL STABLE AS $$ SELECT 10::int4 $$; CREATE OR REPLACE FUNCTION int8_now () RETURNS int8 LANGUAGE SQL STABLE AS $$ SELECT 10::int8 $$; SELECT set_integer_now_func (('metrics_' || dt)::regclass, (dt || '_now')::regproc) FROM ( VALUES ('int2'), ('int4'), ('int8')) v (dt); set_integer_now_func ---------------------- -- width expression for int2 hypertables CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1::smallint, time) FROM metrics_int2 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1::smallint + 2::smallint, time) FROM metrics_int2 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; -- width expression for int4 hypertables CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1, time) FROM metrics_int4 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1 + 2, time) FROM metrics_int4 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; -- width expression for int8 hypertables CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1, time) FROM metrics_int8 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1 + 2, time) FROM metrics_int8 GROUP BY 1; NOTICE: continuous aggregate "width_expr" is already up-to-date DROP MATERIALIZED VIEW width_expr; \set ON_ERROR_STOP 0 -- non-immutable expresions should be rejected CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(extract(year FROM now())::smallint, time) FROM metrics_int2 GROUP BY 1; ERROR: only immutable expressions allowed in time bucket function CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(extract(year FROM now())::int, time) FROM metrics_int4 GROUP BY 1; ERROR: only immutable expressions allowed in time bucket function CREATE MATERIALIZED VIEW width_expr WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(extract(year FROM now())::int, time) FROM metrics_int8 GROUP BY 1; ERROR: only immutable expressions allowed in time bucket function \set ON_ERROR_STOP 1 -- Test various ALTER MATERIALIZED VIEW statements. SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE MATERIALIZED VIEW owner_check WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(1 + 2, time) FROM metrics_int8 GROUP BY 1 WITH NO DATA; \x on SELECT * FROM cagg_info WHERE user_view::text = 'owner_check'; -[ RECORD 1 ]------+--------------------------------------- user_view | owner_check user_view_owner | default_perm_user mat_table | _materialized_hypertable_24 mat_table_owner | default_perm_user direct_view | _timescaledb_internal._direct_view_24 direct_view_owner | default_perm_user partial_view | _timescaledb_internal._partial_view_24 partial_view_owner | default_perm_user tablespace | \x off -- This should not work since the target user has the wrong role, but -- we test that the normal checks are done when changing the owner. \set ON_ERROR_STOP 0 ALTER MATERIALIZED VIEW owner_check OWNER TO :ROLE_1; ERROR: must be able to SET ROLE "test_role_1" \set ON_ERROR_STOP 1 -- Superuser can always change owner SET ROLE :ROLE_SUPERUSER; -- Add a refresh policy before changing owner to verify job owner is propagated SELECT add_continuous_aggregate_policy('owner_check', NULL, 1::int8, '1 day'::interval) AS cagg_job_id \gset SELECT owner FROM _timescaledb_config.bgw_job WHERE id = :cagg_job_id; owner ------------------- default_perm_user ALTER MATERIALIZED VIEW owner_check OWNER TO :ROLE_1; \x on SELECT * FROM cagg_info WHERE user_view::text = 'owner_check'; -[ RECORD 1 ]------+--------------------------------------- user_view | owner_check user_view_owner | test_role_1 mat_table | _materialized_hypertable_24 mat_table_owner | test_role_1 direct_view | _timescaledb_internal._direct_view_24 direct_view_owner | test_role_1 partial_view | _timescaledb_internal._partial_view_24 partial_view_owner | test_role_1 tablespace | \x off -- make sure policy job owner is propagated SELECT owner FROM _timescaledb_config.bgw_job WHERE id = :cagg_job_id; owner ------------- test_role_1 SELECT remove_continuous_aggregate_policy('owner_check'); remove_continuous_aggregate_policy ------------------------------------ -- -- Test drop continuous aggregate cases -- -- Issue: #2608 -- CREATE OR REPLACE FUNCTION test_int_now() RETURNS INT LANGUAGE SQL STABLE AS $BODY$ SELECT 50; $BODY$; CREATE TABLE conditionsnm(time_int INT NOT NULL, device INT, value FLOAT); SELECT create_hypertable('conditionsnm', 'time_int', chunk_time_interval => 10); create_hypertable ---------------------------- (25,public,conditionsnm,t) SELECT set_integer_now_func('conditionsnm', 'test_int_now'); set_integer_now_func ---------------------- INSERT INTO conditionsnm SELECT time_val, time_val % 4, 3.14 FROM generate_series(0,100,1) AS time_val; -- Case 1: DROP CREATE MATERIALIZED VIEW conditionsnm_4 WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(7, time_int) as bucket, SUM(value), COUNT(value) FROM conditionsnm GROUP BY bucket WITH DATA; NOTICE: refreshing continuous aggregate "conditionsnm_4" DROP materialized view conditionsnm_4; NOTICE: drop cascades to table _timescaledb_internal._hyper_26_40_chunk -- Case 2: DROP CASCADE should have similar behaviour as DROP CREATE MATERIALIZED VIEW conditionsnm_4 WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(7, time_int) as bucket, SUM(value), COUNT(value) FROM conditionsnm GROUP BY bucket WITH DATA; NOTICE: refreshing continuous aggregate "conditionsnm_4" DROP materialized view conditionsnm_4 CASCADE; NOTICE: drop cascades to table _timescaledb_internal._hyper_27_41_chunk -- Case 3: require CASCADE in case of dependent object CREATE MATERIALIZED VIEW conditionsnm_4 WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(7, time_int) as bucket, SUM(value), COUNT(value) FROM conditionsnm GROUP BY bucket WITH DATA; NOTICE: refreshing continuous aggregate "conditionsnm_4" CREATE VIEW see_cagg as select * from conditionsnm_4; \set ON_ERROR_STOP 0 DROP MATERIALIZED VIEW conditionsnm_4; ERROR: cannot drop view conditionsnm_4 because other objects depend on it \set ON_ERROR_STOP 1 -- Case 4: DROP CASCADE with dependency DROP MATERIALIZED VIEW conditionsnm_4 CASCADE; NOTICE: drop cascades to view see_cagg NOTICE: drop cascades to table _timescaledb_internal._hyper_28_42_chunk -- Test DROP SCHEMA CASCADE with continuous aggregates -- -- Issue: #2350 -- -- Case 1: DROP SCHEMA CASCADE CREATE SCHEMA test_schema; CREATE TABLE test_schema.telemetry_raw ( ts TIMESTAMP WITH TIME ZONE NOT NULL, value DOUBLE PRECISION ); SELECT create_hypertable('test_schema.telemetry_raw', 'ts'); create_hypertable ---------------------------------- (29,test_schema,telemetry_raw,t) CREATE MATERIALIZED VIEW test_schema.telemetry_1s WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(INTERVAL '1s', ts) AS ts_1s, avg(value) FROM test_schema.telemetry_raw GROUP BY ts_1s WITH NO DATA; SELECT ca.raw_hypertable_id, h.schema_name, h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) WHERE user_view_name = 'telemetry_1s'; raw_hypertable_id | schema_name | MAT_TABLE_NAME | PART_VIEW_NAME | partial_view_schema -------------------+-----------------------+-----------------------------+------------------+----------------------- 29 | _timescaledb_internal | _materialized_hypertable_30 | _partial_view_30 | _timescaledb_internal \gset DROP SCHEMA test_schema CASCADE; NOTICE: drop cascades to 4 other objects SELECT count(*) FROM pg_class WHERE relname = :'MAT_TABLE_NAME'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = :'PART_VIEW_NAME'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = 'telemetry_1s'; count ------- 0 SELECT count(*) FROM pg_namespace WHERE nspname = 'test_schema'; count ------- 0 -- Case 2: DROP SCHEMA CASCADE with multiple caggs CREATE SCHEMA test_schema; CREATE TABLE test_schema.telemetry_raw ( ts TIMESTAMP WITH TIME ZONE NOT NULL, value DOUBLE PRECISION ); SELECT create_hypertable('test_schema.telemetry_raw', 'ts'); create_hypertable ---------------------------------- (31,test_schema,telemetry_raw,t) CREATE MATERIALIZED VIEW test_schema.cagg1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(INTERVAL '1s', ts) AS ts_1s, avg(value) FROM test_schema.telemetry_raw GROUP BY ts_1s WITH NO DATA; CREATE MATERIALIZED VIEW test_schema.cagg2 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(INTERVAL '1s', ts) AS ts_1s, avg(value) FROM test_schema.telemetry_raw GROUP BY ts_1s WITH NO DATA; SELECT ca.raw_hypertable_id, h.schema_name, h.table_name AS "MAT_TABLE_NAME1", partial_view_name as "PART_VIEW_NAME1", partial_view_schema FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) WHERE user_view_name = 'cagg1'; raw_hypertable_id | schema_name | MAT_TABLE_NAME1 | PART_VIEW_NAME1 | partial_view_schema -------------------+-----------------------+-----------------------------+------------------+----------------------- 31 | _timescaledb_internal | _materialized_hypertable_32 | _partial_view_32 | _timescaledb_internal \gset SELECT ca.raw_hypertable_id, h.schema_name, h.table_name AS "MAT_TABLE_NAME2", partial_view_name as "PART_VIEW_NAME2", partial_view_schema FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) WHERE user_view_name = 'cagg2'; raw_hypertable_id | schema_name | MAT_TABLE_NAME2 | PART_VIEW_NAME2 | partial_view_schema -------------------+-----------------------+-----------------------------+------------------+----------------------- 31 | _timescaledb_internal | _materialized_hypertable_33 | _partial_view_33 | _timescaledb_internal \gset DROP SCHEMA test_schema CASCADE; NOTICE: drop cascades to 7 other objects SELECT count(*) FROM pg_class WHERE relname = :'MAT_TABLE_NAME1'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = :'PART_VIEW_NAME1'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = 'cagg1'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = :'MAT_TABLE_NAME2'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = :'PART_VIEW_NAME2'; count ------- 0 SELECT count(*) FROM pg_class WHERE relname = 'cagg2'; count ------- 0 SELECT count(*) FROM pg_namespace WHERE nspname = 'test_schema'; count ------- 0 DROP TABLESPACE tablespace1; DROP TABLESPACE tablespace2; -- Check that we can rename a column of a materialized view and still -- rebuild it after (#3051, #3405) CREATE TABLE conditions ( time TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL ); SELECT create_hypertable('conditions', 'time'); create_hypertable -------------------------- (34,public,conditions,t) INSERT INTO conditions VALUES ( '2018-01-01 09:20:00-08', 'SFO', 55); INSERT INTO conditions VALUES ( '2018-01-02 09:30:00-08', 'por', 100); INSERT INTO conditions VALUES ( '2018-01-02 09:20:00-08', 'SFO', 65); INSERT INTO conditions VALUES ( '2018-01-02 09:10:00-08', 'NYC', 65); INSERT INTO conditions VALUES ( '2018-11-01 09:20:00-08', 'NYC', 45); INSERT INTO conditions VALUES ( '2018-11-01 10:40:00-08', 'NYC', 55); INSERT INTO conditions VALUES ( '2018-11-01 11:50:00-08', 'NYC', 65); INSERT INTO conditions VALUES ( '2018-11-01 12:10:00-08', 'NYC', 75); INSERT INTO conditions VALUES ( '2018-11-01 13:10:00-08', 'NYC', 85); INSERT INTO conditions VALUES ( '2018-11-02 09:20:00-08', 'NYC', 10); INSERT INTO conditions VALUES ( '2018-11-02 10:30:00-08', 'NYC', 20); CREATE MATERIALIZED VIEW conditions_daily WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT location, time_bucket(INTERVAL '1 day', time) AS bucket, AVG(temperature) FROM conditions GROUP BY location, bucket WITH NO DATA; CREATE MATERIALIZED VIEW conditions_weekly WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT location, time_bucket(INTERVAL '7 day', bucket) AS bucket, AVG(avg) FROM conditions_daily GROUP BY 1, 2 WITH NO DATA; SELECT format('%I.%I', '_timescaledb_internal', h.table_name) AS "MAT_TABLE_NAME", format('%I.%I', '_timescaledb_internal', partial_view_name) AS "PART_VIEW_NAME", format('%I.%I', '_timescaledb_internal', direct_view_name) AS "DIRECT_VIEW_NAME" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) WHERE user_view_name = 'conditions_daily' \gset -- Show both the columns and the view definitions to see that -- references are correct in the view as well. SELECT * FROM test.show_columns('conditions_daily'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f bucket | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'DIRECT_VIEW_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f bucket | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'PART_VIEW_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f bucket | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'MAT_TABLE_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f bucket | timestamp with time zone | t avg | double precision | f ALTER MATERIALIZED VIEW conditions_daily RENAME COLUMN bucket to "time"; -- Show both the columns and the view definitions to see that -- references are correct in the view as well. SELECT * FROM test.show_columns(' conditions_daily'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'DIRECT_VIEW_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'PART_VIEW_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | f avg | double precision | f SELECT * FROM test.show_columns(:'MAT_TABLE_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | t avg | double precision | f -- This will rebuild the materialized view and should succeed. ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only = false); -- Refresh the continuous aggregate to check that it works after the -- rename. \set VERBOSITY verbose CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); \set VERBOSITY terse -- Rename another column after the flip and verify toggling back and -- forth still works. This exercises the rename when the user view -- already has a UNION ALL query (materialized_only = false). ALTER MATERIALIZED VIEW conditions_daily RENAME COLUMN avg TO average; SELECT * FROM test.show_columns('conditions_daily'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | f average | double precision | f SELECT * FROM test.show_columns(:'DIRECT_VIEW_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | f average | double precision | f SELECT * FROM test.show_columns(:'MAT_TABLE_NAME'); Column | Type | NotNull ----------+--------------------------+--------- location | text | f time | timestamp with time zone | t average | double precision | f ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only = true); ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only = false); -- Verify data is still accessible after multiple renames and toggles. SELECT * FROM conditions_daily ORDER BY location COLLATE "C", time; location | time | average ----------+------------------------------+--------- NYC | Mon Jan 01 16:00:00 2018 UTC | 65 NYC | Wed Oct 31 16:00:00 2018 UTC | 65 NYC | Thu Nov 01 16:00:00 2018 UTC | 15 SFO | Sun Dec 31 16:00:00 2017 UTC | 55 SFO | Mon Jan 01 16:00:00 2018 UTC | 65 por | Mon Jan 01 16:00:00 2018 UTC | 100 -- check hierarchical continuous aggregate still works after renames and toggles on the underlying cagg ALTER MATERIALIZED VIEW conditions_weekly SET (timescaledb.materialized_only = false); ALTER MATERIALIZED VIEW conditions_weekly SET (timescaledb.materialized_only = true); SELECT * FROM conditions_weekly ORDER BY location COLLATE "C", bucket; location | bucket | avg ----------+--------+----- -- Verify that direct rename on the materialization hypertable is blocked. \set ON_ERROR_STOP 0 ALTER TABLE :MAT_TABLE_NAME RENAME COLUMN average TO avg; ERROR: renaming columns on materialization tables is not supported \set ON_ERROR_STOP 1 -- Rename back so subsequent tests that reference "avg" still work. ALTER MATERIALIZED VIEW conditions_daily RENAME COLUMN average TO avg; -- -- Indexes on continuous aggregate -- \set ON_ERROR_STOP 0 -- unique indexes are not supported CREATE UNIQUE INDEX index_unique_error ON conditions_daily ("time", location); ERROR: continuous aggregates do not support UNIQUE indexes -- concurrently index creation not supported CREATE INDEX CONCURRENTLY index_concurrently_avg ON conditions_daily (avg); ERROR: hypertables do not support concurrent index creation \set ON_ERROR_STOP 1 CREATE INDEX index_avg ON conditions_daily (avg); CREATE INDEX index_avg_only ON ONLY conditions_daily (avg); CREATE INDEX index_avg_include ON conditions_daily (avg) INCLUDE (location); CREATE INDEX index_avg_expr ON conditions_daily ((avg + 1)); CREATE INDEX index_avg_location_sfo ON conditions_daily (avg) WHERE location = 'SFO'; CREATE INDEX index_avg_expr_location_sfo ON conditions_daily ((avg + 2)) WHERE location = 'SFO'; SELECT * FROM test.show_indexespred(:'MAT_TABLE_NAME'); Index | Columns | Expr | Pred | Unique | Primary | Exclusion | Tablespace -----------------------------------------------------------------------+-------------------+---------------------------+------------------------+--------+---------+-----------+------------ _timescaledb_internal._materialized_hypertable_35_bucket_idx | {bucket} | | | f | f | f | _timescaledb_internal._materialized_hypertable_35_location_bucket_idx | {location,bucket} | | | f | f | f | _timescaledb_internal.index_avg | {avg} | | | f | f | f | _timescaledb_internal.index_avg_expr | {expr} | avg + 1::double precision | | f | f | f | _timescaledb_internal.index_avg_expr_location_sfo | {expr} | avg + 2::double precision | location = 'SFO'::text | f | f | f | _timescaledb_internal.index_avg_include | {avg,location} | | | f | f | f | _timescaledb_internal.index_avg_location_sfo | {avg} | | location = 'SFO'::text | f | f | f | _timescaledb_internal.index_avg_only | {avg} | | | f | f | f | -- #3696 assertion failure when referencing columns not present in result CREATE TABLE i3696(time timestamptz NOT NULL, search_query text, cnt integer, cnt2 integer); SELECT table_name FROM create_hypertable('i3696','time'); table_name ------------ i3696 CREATE MATERIALIZED VIEW i3696_cagg1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT search_query,count(search_query) as count, sum(cnt), time_bucket(INTERVAL '1 minute', time) AS bucket FROM i3696 GROUP BY cnt +cnt2 , bucket, search_query; NOTICE: continuous aggregate "i3696_cagg1" is already up-to-date ALTER MATERIALIZED VIEW i3696_cagg1 SET (timescaledb.materialized_only = 'true'); CREATE MATERIALIZED VIEW i3696_cagg2 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT search_query,count(search_query) as count, sum(cnt), time_bucket(INTERVAL '1 minute', time) AS bucket FROM i3696 GROUP BY cnt + cnt2, bucket, search_query HAVING cnt + cnt2 + sum(cnt) > 2 or count(cnt2) > 10; NOTICE: continuous aggregate "i3696_cagg2" is already up-to-date ALTER MATERIALIZED VIEW i3696_cagg2 SET (timescaledb.materialized_only = 'true'); --TEST test with multiple settings on continuous aggregates -- -- test for materialized_only + compress combinations (real time aggs enabled initially) CREATE TABLE test_setting(time timestamptz not null, val numeric); SELECT create_hypertable('test_setting', 'time'); create_hypertable ---------------------------- (40,public,test_setting,t) CREATE MATERIALIZED VIEW test_setting_cagg with (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h',time), avg(val), count(*) FROM test_setting GROUP BY 1; NOTICE: continuous aggregate "test_setting_cagg" is already up-to-date INSERT INTO test_setting SELECT generate_series( '2020-01-10 8:00'::timestamp, '2020-01-30 10:00+00'::timestamptz, '1 day'::interval), 10.0; CALL refresh_continuous_aggregate('test_setting_cagg', NULL, '2020-05-30 10:00+00'::timestamptz); SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 --this row is not in the materialized result --- INSERT INTO test_setting VALUES( '2020-11-01', 20); --try out 2 settings here -- ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'true', timescaledb.compress='true'); NOTICE: defaulting compress_orderby to time_bucket SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | t | t --real time aggs is off now , should return 20 -- SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 --now set it back to false -- ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'false', timescaledb.compress='true'); NOTICE: defaulting compress_orderby to time_bucket SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | t | f --count should return additional data since we have real time aggs on SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 21 ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'true', timescaledb.compress='false'); SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | f | t --real time aggs is off now , should return 20 -- SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'false', timescaledb.compress='false'); SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | f | f --count should return additional data since we have real time aggs on SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 21 DELETE FROM test_setting WHERE val = 20; --TEST test with multiple settings on continuous aggregates with real time aggregates turned off initially -- -- test for materialized_only + compress combinations (real time aggs enabled initially) DROP MATERIALIZED VIEW test_setting_cagg; NOTICE: drop cascades to table _timescaledb_internal._hyper_41_50_chunk CREATE MATERIALIZED VIEW test_setting_cagg with (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT time_bucket('1h',time), avg(val), count(*) FROM test_setting GROUP BY 1; NOTICE: refreshing continuous aggregate "test_setting_cagg" CALL refresh_continuous_aggregate('test_setting_cagg', NULL, '2020-05-30 10:00+00'::timestamptz); SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 --this row is not in the materialized result --- INSERT INTO test_setting VALUES( '2020-11-01', 20); --try out 2 settings here -- ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'false', timescaledb.compress='true'); NOTICE: defaulting compress_orderby to time_bucket SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | t | f --count should return additional data since we have real time aggs on SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 21 --now set it back to false -- ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'true', timescaledb.compress='true'); NOTICE: defaulting compress_orderby to time_bucket SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | t | t --real time aggs is off now , should return 20 -- SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'false', timescaledb.compress='false'); SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | f | f --count should return additional data since we have real time aggs on SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 21 ALTER MATERIALIZED VIEW test_setting_cagg SET (timescaledb.materialized_only = 'true', timescaledb.compress='false'); SELECT view_name, compression_enabled, materialized_only FROM timescaledb_information.continuous_aggregates where view_name = 'test_setting_cagg'; view_name | compression_enabled | materialized_only -------------------+---------------------+------------------- test_setting_cagg | f | t --real time aggs is off now , should return 20 -- SELECT count(*) from test_setting_cagg ORDER BY 1; count ------- 20 -- END TEST with multiple settings -- Test View Target Entries that contain both aggrefs and Vars in the same expression CREATE TABLE transactions ( "time" timestamp with time zone NOT NULL, dummy1 integer, dummy2 integer, dummy3 integer, dummy4 integer, dummy5 integer, amount integer, fiat_value integer ); SELECT create_hypertable('transactions', 'time'); create_hypertable ---------------------------- (45,public,transactions,t) INSERT INTO transactions VALUES ( '2018-01-01 09:20:00-08', 0, 0, 0, 0, 0, 1, 10); INSERT INTO transactions VALUES ( '2018-01-02 09:30:00-08', 0, 0, 0, 0, 0, -1, 10); INSERT INTO transactions VALUES ( '2018-01-02 09:20:00-08', 0, 0, 0, 0, 0, -1, 10); INSERT INTO transactions VALUES ( '2018-01-02 09:10:00-08', 0, 0, 0, 0, 0, -1, 10); INSERT INTO transactions VALUES ( '2018-11-01 09:20:00-08', 0, 0, 0, 0, 0, 1, 10); INSERT INTO transactions VALUES ( '2018-11-01 10:40:00-08', 0, 0, 0, 0, 0, 1, 10); INSERT INTO transactions VALUES ( '2018-11-01 11:50:00-08', 0, 0, 0, 0, 0, 1, 10); INSERT INTO transactions VALUES ( '2018-11-01 12:10:00-08', 0, 0, 0, 0, 0, -1, 10); INSERT INTO transactions VALUES ( '2018-11-01 13:10:00-08', 0, 0, 0, 0, 0, -1, 10); INSERT INTO transactions VALUES ( '2018-11-02 09:20:00-08', 0, 0, 0, 0, 0, 1, 10); INSERT INTO transactions VALUES ( '2018-11-02 10:30:00-08', 0, 0, 0, 0, 0, -1, 10); CREATE materialized view cashflows( bucket, amount, cashflow, cashflow2 ) WITH ( timescaledb.continuous, timescaledb.materialized_only = true ) AS SELECT time_bucket ('1 day', time) AS bucket, amount, CASE WHEN amount < 0 THEN (0 - sum(fiat_value)) ELSE sum(fiat_value) END AS cashflow, amount + sum(fiat_value) FROM transactions GROUP BY bucket, amount; NOTICE: refreshing continuous aggregate "cashflows" SELECT h.table_name AS "MAT_TABLE_NAME", partial_view_name AS "PART_VIEW_NAME", direct_view_name AS "DIRECT_VIEW_NAME" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON (h.id = ca.mat_hypertable_id) WHERE user_view_name = 'cashflows' \gset -- Show both the columns and the view definitions to see that -- references are correct in the view as well. \d+ "_timescaledb_internal".:"DIRECT_VIEW_NAME" View "_timescaledb_internal._direct_view_46" Column | Type | Collation | Nullable | Default | Storage | Description -----------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | amount | integer | | | | plain | cashflow | bigint | | | | plain | cashflow2 | bigint | | | | plain | View definition: SELECT time_bucket('@ 1 day'::interval, "time") AS bucket, amount, CASE WHEN amount < 0 THEN 0 - sum(fiat_value) ELSE sum(fiat_value) END AS cashflow, amount + sum(fiat_value) AS cashflow2 FROM transactions GROUP BY (time_bucket('@ 1 day'::interval, "time")), amount; \d+ "_timescaledb_internal".:"PART_VIEW_NAME" View "_timescaledb_internal._partial_view_46" Column | Type | Collation | Nullable | Default | Storage | Description -----------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | amount | integer | | | | plain | cashflow | bigint | | | | plain | cashflow2 | bigint | | | | plain | View definition: SELECT time_bucket('@ 1 day'::interval, "time") AS bucket, amount, CASE WHEN amount < 0 THEN 0 - sum(fiat_value) ELSE sum(fiat_value) END AS cashflow, amount + sum(fiat_value) AS cashflow2 FROM transactions GROUP BY (time_bucket('@ 1 day'::interval, "time")), amount; \d+ "_timescaledb_internal".:"MAT_TABLE_NAME" Table "_timescaledb_internal._materialized_hypertable_46" Column | Type | Collation | Nullable | Default | Storage | Stats target | Description -----------+--------------------------+-----------+----------+---------+---------+--------------+------------- bucket | timestamp with time zone | | not null | | plain | | amount | integer | | | | plain | | cashflow | bigint | | | | plain | | cashflow2 | bigint | | | | plain | | Indexes: "_materialized_hypertable_46_amount_bucket_idx" btree (amount, bucket DESC) "_materialized_hypertable_46_bucket_idx" btree (bucket DESC) Child tables: _timescaledb_internal._hyper_46_55_chunk, _timescaledb_internal._hyper_46_56_chunk \d+ 'cashflows' View "public.cashflows" Column | Type | Collation | Nullable | Default | Storage | Description -----------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | amount | integer | | | | plain | cashflow | bigint | | | | plain | cashflow2 | bigint | | | | plain | View definition: SELECT bucket, amount, cashflow, cashflow2 FROM _timescaledb_internal._materialized_hypertable_46; SELECT * FROM cashflows ORDER BY cashflows; bucket | amount | cashflow | cashflow2 ------------------------------+--------+----------+----------- Sun Dec 31 16:00:00 2017 UTC | 1 | 10 | 11 Mon Jan 01 16:00:00 2018 UTC | -1 | -30 | 29 Wed Oct 31 16:00:00 2018 UTC | -1 | -20 | 19 Wed Oct 31 16:00:00 2018 UTC | 1 | 30 | 31 Thu Nov 01 16:00:00 2018 UTC | -1 | -10 | 9 Thu Nov 01 16:00:00 2018 UTC | 1 | 10 | 11 -- test cagg creation with named arguments in time_bucket -- note that positional arguments cannot follow named arguments -- 1. test named origin -- 2. test named timezone -- 3. test named ts -- 4. test named bucket width -- named origin CREATE MATERIALIZED VIEW cagg_named_origin WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h', time, 'UTC', origin => '2001-01-03 01:23:45') AS bucket, avg(amount) as avg_amount FROM transactions GROUP BY 1 WITH NO DATA; -- named timezone CREATE MATERIALIZED VIEW cagg_named_tz_origin WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h', time, timezone => 'UTC', origin => '2001-01-03 01:23:45') AS bucket, avg(amount) as avg_amount FROM transactions GROUP BY 1 WITH NO DATA; -- named ts CREATE MATERIALIZED VIEW cagg_named_ts_tz_origin WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h', ts => time, timezone => 'UTC', origin => '2001-01-03 01:23:45') AS bucket, avg(amount) as avg_amount FROM transactions GROUP BY 1 WITH NO DATA; -- named bucket width CREATE MATERIALIZED VIEW cagg_named_all WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(bucket_width => '1h', ts => time, timezone => 'UTC', origin => '2001-01-03 01:23:45') AS bucket, avg(amount) as avg_amount FROM transactions GROUP BY 1 WITH NO DATA; -- Refreshing from the beginning (NULL) of a CAGG with variable time bucket and -- using an INTERVAL for the end timestamp (issue #5534) CREATE MATERIALIZED VIEW transactions_montly WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT time_bucket(INTERVAL '1 month', time) AS bucket, SUM(fiat_value), MAX(fiat_value), MIN(fiat_value) FROM transactions GROUP BY 1 WITH NO DATA; -- No rows SELECT * FROM transactions_montly ORDER BY bucket; bucket | sum | max | min --------+-----+-----+----- -- Refresh from beginning of the CAGG for 1 month CALL refresh_continuous_aggregate('transactions_montly', NULL, INTERVAL '1 month'); SELECT * FROM transactions_montly ORDER BY bucket; bucket | sum | max | min ------------------------------+-----+-----+----- Sun Dec 31 16:00:00 2017 UTC | 40 | 10 | 10 Wed Oct 31 16:00:00 2018 UTC | 70 | 10 | 10 TRUNCATE transactions_montly; -- Partial refresh the CAGG from beginning to an specific timestamp CALL refresh_continuous_aggregate('transactions_montly', NULL, '2018-11-01 11:50:00-08'::timestamptz); SELECT * FROM transactions_montly ORDER BY bucket; bucket | sum | max | min ------------------------------+-----+-----+----- Sun Dec 31 16:00:00 2017 UTC | 40 | 10 | 10 -- Full refresh the CAGG CALL refresh_continuous_aggregate('transactions_montly', NULL, NULL); SELECT * FROM transactions_montly ORDER BY bucket; bucket | sum | max | min ------------------------------+-----+-----+----- Sun Dec 31 16:00:00 2017 UTC | 40 | 10 | 10 Wed Oct 31 16:00:00 2018 UTC | 70 | 10 | 10 -- Check set_chunk_time_interval on continuous aggregate CREATE MATERIALIZED VIEW cagg_set_chunk_time_interval WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(INTERVAL '1 month', time) AS bucket, SUM(fiat_value), MAX(fiat_value), MIN(fiat_value) FROM transactions GROUP BY 1 WITH NO DATA; SELECT set_chunk_time_interval('cagg_set_chunk_time_interval', chunk_time_interval => interval '1 month'); set_chunk_time_interval ------------------------- CALL refresh_continuous_aggregate('cagg_set_chunk_time_interval', NULL, NULL); SELECT _timescaledb_functions.to_interval(d.interval_length) = interval '1 month' FROM _timescaledb_catalog.dimension d RIGHT JOIN _timescaledb_catalog.continuous_agg ca ON ca.user_view_name = 'cagg_set_chunk_time_interval' WHERE d.hypertable_id = ca.mat_hypertable_id; ?column? ---------- t -- Since #6077 CAggs are materialized only by default DROP TABLE conditions CASCADE; NOTICE: drop cascades to 5 other objects NOTICE: drop cascades to 2 other objects CREATE TABLE conditions ( time TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL ); SELECT create_hypertable('conditions', 'time'); create_hypertable -------------------------- (53,public,conditions,t) INSERT INTO conditions VALUES ( '2018-01-01 09:20:00-08', 'SFO', 55); INSERT INTO conditions VALUES ( '2018-01-02 09:30:00-08', 'POR', 100); INSERT INTO conditions VALUES ( '2018-01-02 09:20:00-08', 'SFO', 65); INSERT INTO conditions VALUES ( '2018-01-02 09:10:00-08', 'NYC', 65); INSERT INTO conditions VALUES ( '2018-11-01 09:20:00-08', 'NYC', 45); INSERT INTO conditions VALUES ( '2018-11-01 10:40:00-08', 'NYC', 55); INSERT INTO conditions VALUES ( '2018-11-01 11:50:00-08', 'NYC', 65); INSERT INTO conditions VALUES ( '2018-11-01 12:10:00-08', 'NYC', 75); INSERT INTO conditions VALUES ( '2018-11-01 13:10:00-08', 'NYC', 85); INSERT INTO conditions VALUES ( '2018-11-02 09:20:00-08', 'NYC', 10); INSERT INTO conditions VALUES ( '2018-11-02 10:30:00-08', 'NYC', 20); CREATE MATERIALIZED VIEW conditions_daily WITH (timescaledb.continuous) AS SELECT location, time_bucket(INTERVAL '1 day', time) AS bucket, AVG(temperature) FROM conditions GROUP BY location, bucket WITH NO DATA; \d+ conditions_daily View "public.conditions_daily" Column | Type | Collation | Nullable | Default | Storage | Description ----------+--------------------------+-----------+----------+---------+----------+------------- location | text | | | | extended | bucket | timestamp with time zone | | | | plain | avg | double precision | | | | plain | View definition: SELECT location, bucket, avg FROM _timescaledb_internal._materialized_hypertable_54; -- Should return NO ROWS SELECT * FROM conditions_daily ORDER BY bucket, location; location | bucket | avg ----------+--------+----- ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only=false); \d+ conditions_daily View "public.conditions_daily" Column | Type | Collation | Nullable | Default | Storage | Description ----------+--------------------------+-----------+----------+---------+----------+------------- location | text | | | | extended | bucket | timestamp with time zone | | | | plain | avg | double precision | | | | plain | View definition: SELECT _materialized_hypertable_54.location, _materialized_hypertable_54.bucket, _materialized_hypertable_54.avg FROM _timescaledb_internal._materialized_hypertable_54 WHERE _materialized_hypertable_54.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(54)), '-infinity'::timestamp with time zone) UNION ALL SELECT conditions.location, time_bucket('@ 1 day'::interval, conditions."time") AS bucket, avg(conditions.temperature) AS avg FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(54)), '-infinity'::timestamp with time zone) GROUP BY conditions.location, (time_bucket('@ 1 day'::interval, conditions."time")); -- Should return ROWS because now it is realtime SELECT * FROM conditions_daily ORDER BY bucket, location; location | bucket | avg ----------+------------------------------+----- SFO | Sun Dec 31 16:00:00 2017 UTC | 55 NYC | Mon Jan 01 16:00:00 2018 UTC | 65 POR | Mon Jan 01 16:00:00 2018 UTC | 100 SFO | Mon Jan 01 16:00:00 2018 UTC | 65 NYC | Wed Oct 31 16:00:00 2018 UTC | 65 NYC | Thu Nov 01 16:00:00 2018 UTC | 15 -- Should return ROWS because we refreshed it ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.materialized_only=true); \d+ conditions_daily View "public.conditions_daily" Column | Type | Collation | Nullable | Default | Storage | Description ----------+--------------------------+-----------+----------+---------+----------+------------- location | text | | | | extended | bucket | timestamp with time zone | | | | plain | avg | double precision | | | | plain | View definition: SELECT location, bucket, avg FROM _timescaledb_internal._materialized_hypertable_54; CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); SELECT * FROM conditions_daily ORDER BY bucket, location; location | bucket | avg ----------+------------------------------+----- SFO | Sun Dec 31 16:00:00 2017 UTC | 55 NYC | Mon Jan 01 16:00:00 2018 UTC | 65 POR | Mon Jan 01 16:00:00 2018 UTC | 100 SFO | Mon Jan 01 16:00:00 2018 UTC | 65 NYC | Wed Oct 31 16:00:00 2018 UTC | 65 NYC | Thu Nov 01 16:00:00 2018 UTC | 15 -- Test TRUNCATE over a Realtime CAgg DROP MATERIALIZED VIEW conditions_daily; NOTICE: drop cascades to 2 other objects CREATE MATERIALIZED VIEW conditions_daily WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT location, time_bucket(INTERVAL '1 day', time) AS bucket, AVG(temperature) FROM conditions GROUP BY location, bucket WITH NO DATA; SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'conditions_daily' \gset -- Check the current watermark for an empty CAgg SELECT _timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)) AS watermak_empty_cagg; watermak_empty_cagg --------------------------------- Sun Nov 23 16:00:00 4714 UTC BC -- Refresh the CAGG CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); -- Check the watermark after the refresh and before truncate the CAgg SELECT _timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)) AS watermak_before; watermak_before ------------------------------ Fri Nov 02 16:00:00 2018 UTC -- Exists chunks before truncate the cagg (> 0) SELECT count(*) FROM show_chunks('conditions_daily'); count ------- 2 -- Truncate the given CAgg, it should reset the watermark to the empty state TRUNCATE conditions_daily; -- No chunks remains after truncate the cagg (= 0) SELECT count(*) FROM show_chunks('conditions_daily'); count ------- 0 -- Watermark should be reseted SELECT _timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_hypertable_id)) AS watermak_after; watermak_after --------------------------------- Sun Nov 23 16:00:00 4714 UTC BC -- Should return ROWS because the watermark was reseted by the TRUNCATE SELECT * FROM conditions_daily ORDER BY bucket, location; location | bucket | avg ----------+------------------------------+----- SFO | Sun Dec 31 16:00:00 2017 UTC | 55 NYC | Mon Jan 01 16:00:00 2018 UTC | 65 POR | Mon Jan 01 16:00:00 2018 UTC | 100 SFO | Mon Jan 01 16:00:00 2018 UTC | 65 NYC | Wed Oct 31 16:00:00 2018 UTC | 65 NYC | Thu Nov 01 16:00:00 2018 UTC | 15 -- check compression settings are cleaned up when deleting a cagg with compression CREATE TABLE cagg_cleanup(time timestamptz not null); SELECT table_name FROM create_hypertable('cagg_cleanup','time'); table_name -------------- cagg_cleanup INSERT INTO cagg_cleanup SELECT '2020-01-01'; CREATE MATERIALIZED VIEW cagg1 WITH (timescaledb.continuous) AS SELECT time_bucket('1h',time) FROM cagg_cleanup GROUP BY 1; NOTICE: refreshing continuous aggregate "cagg1" ALTER MATERIALIZED VIEW cagg1 SET (timescaledb.compress); NOTICE: defaulting compress_orderby to time_bucket SELECT count(compress_chunk(ch)) FROM show_chunks('cagg1') ch; count ------- 1 DROP MATERIALIZED VIEW cagg1; NOTICE: drop cascades to table _timescaledb_internal._hyper_57_70_chunk SELECT * FROM _timescaledb_catalog.compression_settings; relid | compress_relid | segmentby | orderby | orderby_desc | orderby_nullsfirst | index -------+----------------+-----------+---------+--------------+--------------------+------- -- test WITH namespace alias CREATE TABLE with_alias(time timestamptz not null); CREATE MATERIALIZED VIEW cagg_alias WITH (tsdb.continuous, tsdb.materialized_only=false) AS SELECT time_bucket(INTERVAL '1 day', time) FROM conditions GROUP BY 1 WITH NO DATA; ALTER MATERIALIZED VIEW cagg_alias SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_alias SET (tsdb.materialized_only=false); DROP MATERIALIZED VIEW cagg_alias; -- test SET chunk_time_interval CREATE MATERIALIZED VIEW cagg_set WITH (tsdb.continuous, tsdb.chunk_interval='1day') AS SELECT time_bucket(INTERVAL '1 day', time) AS cagg_interval_setter FROM conditions GROUP BY 1 WITH NO DATA; SELECT column_name, time_interval FROM timescaledb_information.dimensions WHERE column_name='cagg_interval_setter'; column_name | time_interval ----------------------+--------------- cagg_interval_setter | @ 1 day ALTER MATERIALIZED VIEW cagg_set SET (tsdb.chunk_interval='23 day'); SELECT column_name, time_interval FROM timescaledb_information.dimensions WHERE column_name='cagg_interval_setter'; column_name | time_interval ----------------------+--------------- cagg_interval_setter | @ 23 days ALTER MATERIALIZED VIEW cagg_set SET (tsdb.chunk_interval='6 month'); SELECT column_name, time_interval FROM timescaledb_information.dimensions WHERE column_name='cagg_interval_setter'; column_name | time_interval ----------------------+--------------- cagg_interval_setter | @ 180 days ALTER MATERIALIZED VIEW cagg_set SET (tsdb.chunk_interval='1 year'); SELECT column_name, time_interval FROM timescaledb_information.dimensions WHERE column_name='cagg_interval_setter'; column_name | time_interval ----------------------+--------------- cagg_interval_setter | @ 360 days -- test cagg with stable functions CREATE MATERIALIZED VIEW cagg_stable WITH (tsdb.continuous) AS SELECT sum(temperature), max(time + INTERVAL '1h') FROM conditions GROUP BY time_bucket('1week', time), location; WARNING: using non-immutable functions in continuous aggregate view may lead to inconsistent results on rematerialization NOTICE: refreshing continuous aggregate "cagg_stable" SELECT * FROM cagg_stable t ORDER BY t; sum | max -----+------------------------------ 65 | Tue Jan 02 10:10:00 2018 UTC 100 | Tue Jan 02 10:30:00 2018 UTC 120 | Tue Jan 02 10:20:00 2018 UTC 355 | Fri Nov 02 11:30:00 2018 UTC --aggregate without combine function but stable function CREATE MATERIALIZED VIEW cagg_json_agg WITH (tsdb.continuous, tsdb.materialized_only=false) AS SELECT json_agg(location) from conditions group by time_bucket('1week', time), location WITH NO DATA; WARNING: using non-immutable functions in continuous aggregate view may lead to inconsistent results on rematerialization CREATE FUNCTION test_stablefunc(int) RETURNS int LANGUAGE 'sql' STABLE AS 'SELECT $1 + 10'; CREATE MATERIALIZED VIEW cagg_stable2 WITH (tsdb.continuous) AS SELECT sum(test_stablefunc(temperature::int)), min(location) FROM conditions GROUP BY time_bucket('1week', time) WITH NO DATA; WARNING: using non-immutable functions in continuous aggregate view may lead to inconsistent results on rematerialization CREATE MATERIALIZED VIEW cagg_stable3 WITH (tsdb.continuous) AS SELECT sum(temperature), min(location) FROM conditions GROUP BY time_bucket('1week', time), test_stablefunc(temperature::int) WITH NO DATA; WARNING: using non-immutable functions in continuous aggregate view may lead to inconsistent results on rematerialization -- test window functions in caggs -- first do sanity check that we error without the GUC \set ON_ERROR_STOP 0 CREATE MATERIALIZED VIEW cagg_window WITH (tsdb.continuous) AS SELECT time_bucket('1week', time), rank() OVER (PARTITION BY time_bucket('1 week',time)) FROM conditions GROUP BY 1; ERROR: invalid continuous aggregate query \set ON_ERROR_STOP 1 SET timescaledb.enable_cagg_window_functions TO on; CREATE MATERIALIZED VIEW cagg_window_1 WITH (tsdb.continuous) AS SELECT time_bucket('1week', time), rank() OVER (PARTITION BY time_bucket('1 week',time)) FROM conditions GROUP BY 1; WARNING: window function support is experimental and may result in unexpected results depending on the functions used. NOTICE: refreshing continuous aggregate "cagg_window_1" CREATE MATERIALIZED VIEW cagg_window_2 WITH (tsdb.continuous) AS SELECT time_bucket('1week', time), rank() OVER (PARTITION BY time_bucket('1 week',time), location) FROM conditions GROUP BY 1, location; WARNING: window function support is experimental and may result in unexpected results depending on the functions used. NOTICE: refreshing continuous aggregate "cagg_window_2" CREATE MATERIALIZED VIEW cagg_window_3 WITH (tsdb.continuous) AS SELECT time_bucket('1week', time), rank() OVER (PARTITION BY time_bucket('1 week',time)) FROM conditions GROUP BY 1, location; WARNING: window function support is experimental and may result in unexpected results depending on the functions used. NOTICE: refreshing continuous aggregate "cagg_window_3" CREATE MATERIALIZED VIEW cagg_window_4 WITH (tsdb.continuous) AS SELECT time_bucket('1week', time), rank() OVER w FROM conditions GROUP BY 1, location WINDOW w AS (PARTITION BY time_bucket('1 week',time)); WARNING: window function support is experimental and may result in unexpected results depending on the functions used. NOTICE: refreshing continuous aggregate "cagg_window_4" -- test setting chunk_interval on a cagg CREATE MATERIALIZED VIEW cagg_chunk_interval WITH (tsdb.continuous, tsdb.chunk_interval='1000 day') AS SELECT time_bucket('1 week', time) FROM conditions GROUP BY 1 WITH NO DATA; SELECT time_interval from timescaledb_information.continuous_aggregates cagg INNER JOIN timescaledb_information.dimensions dim ON cagg.materialization_hypertable_name = dim.hypertable_name WHERE view_name='cagg_chunk_interval'; time_interval --------------- @ 1000 days ALTER MATERIALIZED VIEW cagg_chunk_interval SET (tsdb.chunk_interval='110 day'); SELECT time_interval from timescaledb_information.continuous_aggregates cagg INNER JOIN timescaledb_information.dimensions dim ON cagg.materialization_hypertable_name = dim.hypertable_name WHERE view_name='cagg_chunk_interval'; time_interval --------------- @ 110 days -- test columnstore options CREATE MATERIALIZED VIEW columnstore_options WITH (tsdb.continuous, tsdb.chunk_interval='1 day') AS SELECT time_bucket('1 day', time) FROM conditions GROUP BY 1 WITH NO DATA; SELECT column_name, compress_interval_length from _timescaledb_catalog.dimension where column_name='time_bucket' ORDER BY id DESC LIMIT 1; column_name | compress_interval_length -------------+-------------------------- time_bucket | ALTER MATERIALIZED VIEW columnstore_options SET (tsdb.columnstore, tsdb.chunk_interval='1 day', tsdb.orderby='time_bucket DESC', tsdb.compress_chunk_interval='2 day'); SELECT column_name, compress_interval_length from _timescaledb_catalog.dimension where column_name='time_bucket' ORDER BY id DESC LIMIT 1; column_name | compress_interval_length -------------+-------------------------- time_bucket | 172800000000 ALTER MATERIALIZED VIEW columnstore_options SET (tsdb.compress_chunk_interval='3 day'); SELECT column_name, compress_interval_length from _timescaledb_catalog.dimension where column_name='time_bucket' ORDER BY id DESC LIMIT 1; column_name | compress_interval_length -------------+-------------------------- time_bucket | 259200000000 ALTER MATERIALIZED VIEW columnstore_options SET (tsdb.compress_chunk_time_interval='4 day'); SELECT column_name, compress_interval_length from _timescaledb_catalog.dimension where column_name='time_bucket' ORDER BY id DESC LIMIT 1; column_name | compress_interval_length -------------+-------------------------- time_bucket | 345600000000 -- test set returning functions in caggs CREATE TABLE kpis_raw (time TIMESTAMP NOT NULL, value INTEGER, groups TEXT[]) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column INSERT INTO kpis_raw (time, value, groups) VALUES ('2025-01-01', 10, '{group1,group2,group3}'), ('2025-01-02', 20, '{group1,group4}'), ('2025-02-01', 10, '{group1,group3}'), ('2025-02-01', 20, '{group1,group4}'); CREATE MATERIALIZED VIEW kpis_cagg WITH (tsdb.continuous) AS SELECT time_bucket('7 day', time) AS bucket, count(*) AS number_of_records, avg(value) AS average, unnest(groups) AS kpi_group FROM kpis_raw GROUP BY bucket, kpi_group; NOTICE: refreshing continuous aggregate "kpis_cagg" SELECT * FROM kpis_cagg ORDER BY bucket, kpi_group; bucket | number_of_records | average | kpi_group --------------------------+-------------------+---------------------+----------- Mon Dec 30 00:00:00 2024 | 2 | 15.0000000000000000 | group1 Mon Dec 30 00:00:00 2024 | 1 | 10.0000000000000000 | group2 Mon Dec 30 00:00:00 2024 | 1 | 10.0000000000000000 | group3 Mon Dec 30 00:00:00 2024 | 1 | 20.0000000000000000 | group4 Mon Jan 27 00:00:00 2025 | 2 | 15.0000000000000000 | group1 Mon Jan 27 00:00:00 2025 | 1 | 10.0000000000000000 | group3 Mon Jan 27 00:00:00 2025 | 1 | 20.0000000000000000 | group4 --TEST for caggs with non timescaledb namespace options --non timescaledb namespace options can be set via ALTER \set ON_ERROR_STOP 0 -- will error out CREATE MATERIALIZED VIEW ht_try_weekly WITH (timescaledb.continuous, tigerlake.newoption = true) AS SELECT time_bucket(interval '1 week', time) AS ts_bucket, avg(value) FROM kpis_raw GROUP BY 1; ERROR: non "timescaledb" namespace options can be set only via ALTER --caught by Postgres now ALTER MATERIALIZED VIEW kpis_cagg SET (tigerlake.newoption = true); ERROR: "kpis_cagg" is not a materialized view \set ON_ERROR_STOP 1 -- TEST that cached alter stmt still works (see PR 8739) -- test DDL inside function CREATE TABLE hypertab_ddl( ts timestamp, a integer) WITH (timescaledb.hypertable); NOTICE: using column "ts" as partitioning column CREATE OR REPLACE FUNCTION ddl_function() RETURNS VOID LANGUAGE PLPGSQL AS $$ BEGIN DROP MATERIALIZED VIEW IF EXISTS cagg_hypertab_ddl; CREATE MATERIALIZED VIEW cagg_hypertab_ddl WITH (timescaledb.continuous) AS SELECT time_bucket( '1 day'::interval, ts), COUNT(*) FROM hypertab_ddl GROUP BY 1 WITH NO DATA; END $$; SELECT ddl_function(); NOTICE: materialized view "cagg_hypertab_ddl" does not exist, skipping ddl_function -------------- SELECT view_name from timescaledb_information.continuous_aggregates WHERE hypertable_name='hypertab_ddl'; view_name ------------------- cagg_hypertab_ddl SELECT ddl_function(); ddl_function -------------- SELECT view_name from timescaledb_information.continuous_aggregates WHERE hypertable_name='hypertab_ddl'; view_name ------------------- cagg_hypertab_ddl -- TEST continuous aggregate with functionally dependent columns -- name column depends on id which is in GROUP BY so name doesnt have to be CREATE table sensor(id int PRIMARY KEY, name text); CREATE TABLE sensordata(time timestamptz,id int, value float) WITH (timescaledb.hypertable); NOTICE: using column "time" as partitioning column CREATE MATERIALIZED VIEW cagg_sensordata WITH (tsdb.continuous) AS SELECT s.id, s.name,time_bucket('15 minutes', sd.time) as bucket, avg(sd.value) FROM sensordata sd JOIN sensor s USING(id) GROUP BY s.id, bucket WITH NO DATA; SELECT * FROM cagg_sensordata; id | name | bucket | avg ----+------+--------+----- ================================================ FILE: tsl/test/expected/cagg_deprecated_bucket_ng.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. CREATE TABLE conditions( time timestamptz NOT NULL, city text NOT NULL, temperature INT NOT NULL); SELECT create_hypertable( 'conditions', 'time', chunk_time_interval => INTERVAL '1 day' ); create_hypertable ------------------------- (1,public,conditions,t) -- Ensure no CAgg using time_bucket_ng can be created \set ON_ERROR_STOP 0 -- Regular CAgg CREATE MATERIALIZED VIEW conditions_summary_weekly WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT city, timescaledb_experimental.time_bucket_ng('7 days', time, 'UTC') AS bucket, MIN(temperature), MAX(temperature) FROM conditions GROUP BY city, bucket WITH NO DATA; ERROR: experimental bucket functions are not supported inside a CAgg definition -- CAgg with origin CREATE MATERIALIZED VIEW conditions_summary_weekly WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT city, timescaledb_experimental.time_bucket_ng('7 days', time, '2024-01-16 18:00:00+00') AS bucket, MIN(temperature), MAX(temperature) FROM conditions GROUP BY city, bucket WITH NO DATA; ERROR: experimental bucket functions are not supported inside a CAgg definition -- CAgg with origin and timezone CREATE MATERIALIZED VIEW conditions_summary_weekly WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT city, timescaledb_experimental.time_bucket_ng('7 days', time, '2024-01-16 18:00:00+00', 'UTC') AS bucket, MIN(temperature), MAX(temperature) FROM conditions GROUP BY city, bucket WITH NO DATA; ERROR: experimental bucket functions are not supported inside a CAgg definition \set ON_ERROR_STOP 1 ================================================ FILE: tsl/test/expected/cagg_direct_compress.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. SET timezone TO PST8PDT; CREATE TABLE conditions ( time TIMESTAMP WITH TIME ZONE NOT NULL, device_id TEXT, location_id INTEGER, temperature NUMERIC, humidity NUMERIC ) WITH ( timescaledb.hypertable, timescaledb.chunk_interval = '1 month' ); NOTICE: using column "time" as partitioning column INSERT INTO conditions SELECT t, d::text, d, 1, 1 FROM generate_series('2025-12-15 00:00:00+00'::timestamptz - interval '1 year', '2025-12-15 00:00:00+00'::timestamptz, interval '1 hour') AS t, generate_series(1, 10) AS d; CREATE MATERIALIZED VIEW conditions_hourly WITH (timescaledb.continuous) AS SELECT time_bucket(INTERVAL '1 hour', time) AS bucket, device_id, MAX(temperature), MIN(temperature), COUNT(*) FROM conditions GROUP BY 1, 2 WITH NO DATA; CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; chunk_status_text ------------------- {} -- Enable columnstore TRUNCATE conditions_hourly; ALTER MATERIALIZED VIEW conditions_hourly SET (timescaledb.compress); NOTICE: defaulting compress_orderby to bucket,device_id -- Enable direct compress on cagg refresh SET timescaledb.enable_direct_compress_on_cagg_refresh TO on; CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; chunk_status_text ------------------- {COMPRESSED} -- Backfill data and refresh again WITHOUT direct compress INSERT INTO conditions SELECT t, d::text, 1, 1 FROM generate_series('2025-12-15 00:00:00+00'::timestamptz - interval '1 year', '2025-12-15 00:00:00+00'::timestamptz, interval '1 hour') AS t, generate_series(1, 10) AS d; SET timescaledb.enable_direct_compress_on_cagg_refresh TO off; CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; chunk_status_text ---------------------- {COMPRESSED,PARTIAL} -- Recompress all uncompressed chunks SELECT compress_chunk(show_chunks('conditions_hourly'), recompress => true) IS NOT NULL AS compress; compress ---------- t t t SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; chunk_status_text ------------------- {COMPRESSED} -- Backfill data and refresh again WITH direct compress INSERT INTO conditions SELECT t, d::text, 1, 1 FROM generate_series('2025-12-15 00:00:00+00'::timestamptz - interval '1 year', '2025-12-15 00:00:00+00'::timestamptz, interval '1 hour') AS t, generate_series(1, 10) AS d; SET timescaledb.enable_direct_compress_on_cagg_refresh TO on; CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; chunk_status_text ------------------- {COMPRESSED} -- Cleanup TRUNCATE conditions; TRUNCATE conditions_hourly; -- Hierarchical CAgg tests CREATE MATERIALIZED VIEW conditions_daily WITH (timescaledb.continuous) AS SELECT time_bucket(INTERVAL '1 day', bucket) AS bucket, device_id, MAX(max), MIN(min), SUM(count) AS count FROM conditions_hourly GROUP BY 1, 2 WITH NO DATA; ALTER MATERIALIZED VIEW conditions_daily SET (timescaledb.compress); NOTICE: defaulting compress_orderby to bucket,device_id INSERT INTO conditions SELECT t, d::text, 1, 1 FROM generate_series('2025-12-15 00:00:00+00'::timestamptz - interval '1 year', '2025-12-15 00:00:00+00'::timestamptz, interval '1 hour') AS t, generate_series(1, 10) AS d; SET timescaledb.enable_direct_compress_on_cagg_refresh TO on; -- Refresh the base CAgg CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; chunk_status_text ------------------- {COMPRESSED} -- Refresh the hierarchical CAgg CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_daily') chunk; chunk_status_text ------------------- {COMPRESSED} -- Produce some invalidations for the base CAgg INSERT INTO conditions SELECT t, d::text, 1, 1 FROM generate_series('2025-12-15 00:00:00+00'::timestamptz - interval '1 year', '2025-12-15 00:00:00+00'::timestamptz, interval '1 hour') AS t, generate_series(1, 10) AS d; -- Refresh the base CAgg CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_hourly') chunk; chunk_status_text ------------------- {COMPRESSED} -- Refreshing again the base CAgg is a no-op since everything is up to date CALL refresh_continuous_aggregate('conditions_hourly', NULL, NULL); NOTICE: continuous aggregate "conditions_hourly" is already up-to-date -- Refresh the hierarchical CAgg with invalidations procuded by the base CAgg CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_daily') chunk; chunk_status_text ------------------- {COMPRESSED} -- Refreshing again the hierarchical CAgg is a no-op since everything is up to date CALL refresh_continuous_aggregate('conditions_daily', NULL, NULL); NOTICE: continuous aggregate "conditions_daily" is already up-to-date -- Tests with custom segmentby and orderby CREATE MATERIALIZED VIEW conditions_weekly WITH (timescaledb.continuous) AS SELECT time_bucket(INTERVAL '1 hour', time) AS bucket, location_id, device_id, MAX(temperature), MIN(temperature), COUNT(*) FROM conditions GROUP BY 1, 2, 3 WITH NO DATA; ALTER MATERIALIZED VIEW conditions_weekly SET (timescaledb.compress_segmentby = 'device_id, location_id', timescaledb.compress_orderby = 'max, min, bucket DESC'); CALL refresh_continuous_aggregate('conditions_weekly', NULL, NULL); SELECT DISTINCT _timescaledb_functions.chunk_status_text(chunk) FROM show_chunks('conditions_weekly') chunk; chunk_status_text ------------------- {COMPRESSED} RESET timescaledb.enable_direct_compress_on_cagg_refresh; ================================================ FILE: tsl/test/expected/cagg_drop_chunks.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER SET timezone TO PST8PDT; -- -- Check that drop chunks with a unique constraint works as expected. -- CREATE TABLE clients ( id SERIAL PRIMARY KEY, name TEXT NOT NULL, UNIQUE(name) ); CREATE TABLE records ( time TIMESTAMPTZ NOT NULL, clientId INT NOT NULL REFERENCES clients(id), value DOUBLE PRECISION, UNIQUE(time, clientId) ); SELECT * FROM create_hypertable('records', 'time', chunk_time_interval => INTERVAL '1h'); hypertable_id | schema_name | table_name | created ---------------+-------------+------------+--------- 1 | public | records | t CREATE MATERIALIZED VIEW records_monthly WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1d', time) as bucket, clientId, avg(value) as value_avg, max(value)-min(value) as value_spread FROM records GROUP BY bucket, clientId WITH NO DATA; INSERT INTO clients(name) VALUES ('test-client'); INSERT INTO records SELECT generate_series('2000-03-01'::timestamptz,'2000-04-01','1 day'),1,3.14; SELECT * FROM records_monthly ORDER BY bucket, clientId; bucket | clientid | value_avg | value_spread ------------------------------+----------+-----------+-------------- Tue Feb 29 16:00:00 2000 PST | 1 | 3.14 | 0 Wed Mar 01 16:00:00 2000 PST | 1 | 3.14 | 0 Thu Mar 02 16:00:00 2000 PST | 1 | 3.14 | 0 Fri Mar 03 16:00:00 2000 PST | 1 | 3.14 | 0 Sat Mar 04 16:00:00 2000 PST | 1 | 3.14 | 0 Sun Mar 05 16:00:00 2000 PST | 1 | 3.14 | 0 Mon Mar 06 16:00:00 2000 PST | 1 | 3.14 | 0 Tue Mar 07 16:00:00 2000 PST | 1 | 3.14 | 0 Wed Mar 08 16:00:00 2000 PST | 1 | 3.14 | 0 Thu Mar 09 16:00:00 2000 PST | 1 | 3.14 | 0 Fri Mar 10 16:00:00 2000 PST | 1 | 3.14 | 0 Sat Mar 11 16:00:00 2000 PST | 1 | 3.14 | 0 Sun Mar 12 16:00:00 2000 PST | 1 | 3.14 | 0 Mon Mar 13 16:00:00 2000 PST | 1 | 3.14 | 0 Tue Mar 14 16:00:00 2000 PST | 1 | 3.14 | 0 Wed Mar 15 16:00:00 2000 PST | 1 | 3.14 | 0 Thu Mar 16 16:00:00 2000 PST | 1 | 3.14 | 0 Fri Mar 17 16:00:00 2000 PST | 1 | 3.14 | 0 Sat Mar 18 16:00:00 2000 PST | 1 | 3.14 | 0 Sun Mar 19 16:00:00 2000 PST | 1 | 3.14 | 0 Mon Mar 20 16:00:00 2000 PST | 1 | 3.14 | 0 Tue Mar 21 16:00:00 2000 PST | 1 | 3.14 | 0 Wed Mar 22 16:00:00 2000 PST | 1 | 3.14 | 0 Thu Mar 23 16:00:00 2000 PST | 1 | 3.14 | 0 Fri Mar 24 16:00:00 2000 PST | 1 | 3.14 | 0 Sat Mar 25 16:00:00 2000 PST | 1 | 3.14 | 0 Sun Mar 26 16:00:00 2000 PST | 1 | 3.14 | 0 Mon Mar 27 16:00:00 2000 PST | 1 | 3.14 | 0 Tue Mar 28 16:00:00 2000 PST | 1 | 3.14 | 0 Wed Mar 29 16:00:00 2000 PST | 1 | 3.14 | 0 Thu Mar 30 16:00:00 2000 PST | 1 | 3.14 | 0 Fri Mar 31 16:00:00 2000 PST | 1 | 3.14 | 0 SELECT chunk_name, range_start, range_end FROM timescaledb_information.chunks WHERE hypertable_name = 'records_monthly' ORDER BY range_start; chunk_name | range_start | range_end ------------+-------------+----------- SELECT chunk_name, range_start, range_end FROM timescaledb_information.chunks WHERE hypertable_name = 'records' ORDER BY range_start; chunk_name | range_start | range_end -------------------+------------------------------+------------------------------ _hyper_1_1_chunk | Wed Mar 01 00:00:00 2000 PST | Wed Mar 01 01:00:00 2000 PST _hyper_1_2_chunk | Thu Mar 02 00:00:00 2000 PST | Thu Mar 02 01:00:00 2000 PST _hyper_1_3_chunk | Fri Mar 03 00:00:00 2000 PST | Fri Mar 03 01:00:00 2000 PST _hyper_1_4_chunk | Sat Mar 04 00:00:00 2000 PST | Sat Mar 04 01:00:00 2000 PST _hyper_1_5_chunk | Sun Mar 05 00:00:00 2000 PST | Sun Mar 05 01:00:00 2000 PST _hyper_1_6_chunk | Mon Mar 06 00:00:00 2000 PST | Mon Mar 06 01:00:00 2000 PST _hyper_1_7_chunk | Tue Mar 07 00:00:00 2000 PST | Tue Mar 07 01:00:00 2000 PST _hyper_1_8_chunk | Wed Mar 08 00:00:00 2000 PST | Wed Mar 08 01:00:00 2000 PST _hyper_1_9_chunk | Thu Mar 09 00:00:00 2000 PST | Thu Mar 09 01:00:00 2000 PST _hyper_1_10_chunk | Fri Mar 10 00:00:00 2000 PST | Fri Mar 10 01:00:00 2000 PST _hyper_1_11_chunk | Sat Mar 11 00:00:00 2000 PST | Sat Mar 11 01:00:00 2000 PST _hyper_1_12_chunk | Sun Mar 12 00:00:00 2000 PST | Sun Mar 12 01:00:00 2000 PST _hyper_1_13_chunk | Mon Mar 13 00:00:00 2000 PST | Mon Mar 13 01:00:00 2000 PST _hyper_1_14_chunk | Tue Mar 14 00:00:00 2000 PST | Tue Mar 14 01:00:00 2000 PST _hyper_1_15_chunk | Wed Mar 15 00:00:00 2000 PST | Wed Mar 15 01:00:00 2000 PST _hyper_1_16_chunk | Thu Mar 16 00:00:00 2000 PST | Thu Mar 16 01:00:00 2000 PST _hyper_1_17_chunk | Fri Mar 17 00:00:00 2000 PST | Fri Mar 17 01:00:00 2000 PST _hyper_1_18_chunk | Sat Mar 18 00:00:00 2000 PST | Sat Mar 18 01:00:00 2000 PST _hyper_1_19_chunk | Sun Mar 19 00:00:00 2000 PST | Sun Mar 19 01:00:00 2000 PST _hyper_1_20_chunk | Mon Mar 20 00:00:00 2000 PST | Mon Mar 20 01:00:00 2000 PST _hyper_1_21_chunk | Tue Mar 21 00:00:00 2000 PST | Tue Mar 21 01:00:00 2000 PST _hyper_1_22_chunk | Wed Mar 22 00:00:00 2000 PST | Wed Mar 22 01:00:00 2000 PST _hyper_1_23_chunk | Thu Mar 23 00:00:00 2000 PST | Thu Mar 23 01:00:00 2000 PST _hyper_1_24_chunk | Fri Mar 24 00:00:00 2000 PST | Fri Mar 24 01:00:00 2000 PST _hyper_1_25_chunk | Sat Mar 25 00:00:00 2000 PST | Sat Mar 25 01:00:00 2000 PST _hyper_1_26_chunk | Sun Mar 26 00:00:00 2000 PST | Sun Mar 26 01:00:00 2000 PST _hyper_1_27_chunk | Mon Mar 27 00:00:00 2000 PST | Mon Mar 27 01:00:00 2000 PST _hyper_1_28_chunk | Tue Mar 28 00:00:00 2000 PST | Tue Mar 28 01:00:00 2000 PST _hyper_1_29_chunk | Wed Mar 29 00:00:00 2000 PST | Wed Mar 29 01:00:00 2000 PST _hyper_1_30_chunk | Thu Mar 30 00:00:00 2000 PST | Thu Mar 30 01:00:00 2000 PST _hyper_1_31_chunk | Fri Mar 31 00:00:00 2000 PST | Fri Mar 31 01:00:00 2000 PST _hyper_1_32_chunk | Sat Apr 01 00:00:00 2000 PST | Sat Apr 01 01:00:00 2000 PST CALL refresh_continuous_aggregate('records_monthly', NULL, NULL); \set VERBOSITY default SELECT drop_chunks('records', '2000-03-16'::timestamptz); drop_chunks ----------------------------------------- _timescaledb_internal._hyper_1_1_chunk _timescaledb_internal._hyper_1_2_chunk _timescaledb_internal._hyper_1_3_chunk _timescaledb_internal._hyper_1_4_chunk _timescaledb_internal._hyper_1_5_chunk _timescaledb_internal._hyper_1_6_chunk _timescaledb_internal._hyper_1_7_chunk _timescaledb_internal._hyper_1_8_chunk _timescaledb_internal._hyper_1_9_chunk _timescaledb_internal._hyper_1_10_chunk _timescaledb_internal._hyper_1_11_chunk _timescaledb_internal._hyper_1_12_chunk _timescaledb_internal._hyper_1_13_chunk _timescaledb_internal._hyper_1_14_chunk _timescaledb_internal._hyper_1_15_chunk \set VERBOSITY terse DROP MATERIALIZED VIEW records_monthly; NOTICE: drop cascades to 32 other objects DROP TABLE records; DROP TABLE clients; \set VERBOSITY default CREATE PROCEDURE refresh_cagg_by_chunk_range(_cagg REGCLASS, _hypertable REGCLASS, _older_than INTEGER) AS $$ DECLARE _r RECORD; BEGIN WITH _chunks AS ( SELECT relname, nspname FROM show_chunks(_hypertable, _older_than) AS relid JOIN pg_catalog.pg_class ON pg_class.oid = relid AND pg_class.relkind = 'r' JOIN pg_catalog.pg_namespace ON pg_namespace.oid = pg_class.relnamespace ) SELECT MIN(range_start) AS range_start, MAX(range_end) AS range_end INTO _r FROM _chunks JOIN _timescaledb_catalog.chunk ON chunk.schema_name = _chunks.nspname AND chunk.table_name = _chunks.relname JOIN _timescaledb_catalog.chunk_constraint ON chunk_id = chunk.id JOIN _timescaledb_catalog.dimension_slice ON dimension_slice.id = dimension_slice_id; RAISE INFO 'range_start=% range_end=%', _r.range_start::int, _r.range_end::int; CALL refresh_continuous_aggregate(_cagg, _r.range_start::int, _r.range_end::int + 1); END; $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION test_int_now() returns INT LANGUAGE SQL STABLE as $$ SELECT 125 $$; CREATE TABLE conditions(time_int INT NOT NULL, value FLOAT); SELECT create_hypertable('conditions', 'time_int', chunk_time_interval => 4); create_hypertable ------------------------- (3,public,conditions,t) INSERT INTO conditions SELECT time_val, 1 FROM generate_series(0, 19, 1) AS time_val; SELECT set_integer_now_func('conditions', 'test_int_now'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW conditions_2 WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(2, time_int) as bucket, SUM(value), COUNT(value) FROM conditions GROUP BY bucket WITH DATA; NOTICE: refreshing continuous aggregate "conditions_2" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. SELECT * FROM conditions_2 ORDER BY bucket; bucket | sum | count --------+-----+------- 0 | 2 | 2 2 | 2 | 2 4 | 2 | 2 6 | 2 | 2 8 | 2 | 2 10 | 2 | 2 12 | 2 | 2 14 | 2 | 2 16 | 2 | 2 18 | 2 | 2 UPDATE conditions SET value = 4.00 WHERE time_int = 0; UPDATE conditions SET value = 4.00 WHERE time_int = 6; CALL refresh_cagg_by_chunk_range('conditions_2', 'conditions', 4); INFO: range_start=0 range_end=4 SELECT drop_chunks('conditions', 4); drop_chunks ----------------------------------------- _timescaledb_internal._hyper_3_65_chunk SELECT * FROM conditions_2 ORDER BY bucket; bucket | sum | count --------+-----+------- 0 | 5 | 2 2 | 2 | 2 4 | 2 | 2 6 | 2 | 2 8 | 2 | 2 10 | 2 | 2 12 | 2 | 2 14 | 2 | 2 16 | 2 | 2 18 | 2 | 2 CALL refresh_cagg_by_chunk_range('conditions_2', 'conditions', 8); INFO: range_start=4 range_end=8 SELECT * FROM conditions_2 ORDER BY bucket; bucket | sum | count --------+-----+------- 0 | 5 | 2 2 | 2 | 2 4 | 2 | 2 6 | 5 | 2 8 | 2 | 2 10 | 2 | 2 12 | 2 | 2 14 | 2 | 2 16 | 2 | 2 18 | 2 | 2 UPDATE conditions SET value = 4.00 WHERE time_int = 19; SELECT drop_chunks('conditions', 8); drop_chunks ----------------------------------------- _timescaledb_internal._hyper_3_66_chunk CALL refresh_cagg_by_chunk_range('conditions_2', 'conditions', 12); INFO: range_start=8 range_end=12 SELECT * FROM conditions_2 ORDER BY bucket; bucket | sum | count --------+-----+------- 0 | 5 | 2 2 | 2 | 2 4 | 2 | 2 6 | 5 | 2 8 | 2 | 2 10 | 2 | 2 12 | 2 | 2 14 | 2 | 2 16 | 2 | 2 18 | 2 | 2 CALL refresh_cagg_by_chunk_range('conditions_2', 'conditions', NULL); INFO: range_start=8 range_end=20 SELECT * FROM conditions_2 ORDER BY bucket; bucket | sum | count --------+-----+------- 0 | 5 | 2 2 | 2 | 2 4 | 2 | 2 6 | 5 | 2 8 | 2 | 2 10 | 2 | 2 12 | 2 | 2 14 | 2 | 2 16 | 2 | 2 18 | 5 | 2 DROP PROCEDURE refresh_cagg_by_chunk_range(REGCLASS, REGCLASS, INTEGER); ================================================ FILE: tsl/test/expected/cagg_dump.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. SET timezone TO PST8PDT; CREATE TYPE custom_type AS (high int, low int); CREATE TABLE conditions_before ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null, highlow custom_type null, bit_int smallint, good_life boolean ); SELECT table_name FROM create_hypertable( 'conditions_before', 'timec'); table_name ------------------- conditions_before INSERT INTO conditions_before SELECT generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'POR', 55, 75, 40, 70, NULL, (1,2)::custom_type, 2, true; INSERT INTO conditions_before SELECT generate_series('2018-11-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day'), 'NYC', 35, 45, 50, 40, NULL, (3,4)::custom_type, 4, false; INSERT INTO conditions_before SELECT generate_series('2018-11-01 00:00'::timestamp, '2018-12-15 00:00'::timestamp, '1 day'), 'LA', 73, 55, NULL, 28, NULL, NULL, 8, true; CREATE TABLE conditions_after (like conditions_before including all); SELECT table_name FROM create_hypertable( 'conditions_after', 'timec'); table_name ------------------ conditions_after INSERT INTO conditions_after SELECT * FROM conditions_before; SELECT $$ SELECT time_bucket('1week', timec) as bucket, location, min(allnull) as min_allnull, max(temperature) as max_temp, sum(temperature)+sum(humidity) as agg_sum_expr, avg(humidity), stddev(humidity), bit_and(bit_int), bit_or(bit_int), bool_and(good_life), every(temperature > 0), bool_or(good_life), count(*) as count_rows, count(temperature) as count_temp, count(allnull) as count_zero, corr(temperature, humidity), covar_pop(temperature, humidity), covar_samp(temperature, humidity), regr_avgx(temperature, humidity), regr_avgy(temperature, humidity), regr_count(temperature, humidity), regr_intercept(temperature, humidity), regr_r2(temperature, humidity), regr_slope(temperature, humidity), regr_sxx(temperature, humidity), regr_sxy(temperature, humidity), regr_syy(temperature, humidity), stddev(temperature) as stddev_temp, stddev_pop(temperature), stddev_samp(temperature), variance(temperature), var_pop(temperature), var_samp(temperature), last(temperature, timec) as last_temp, last(highlow, timec) as last_hl, first(highlow, timec) as first_hl, histogram(temperature, 0, 100, 5) FROM TABLE GROUP BY bucket, location HAVING min(location) >= 'NYC' and avg(temperature) > 20 $$ AS "QUERY_TEMPLATE" \gset SELECT replace(:'QUERY_TEMPLATE', 'TABLE', 'conditions_before') AS "QUERY_BEFORE", replace(:'QUERY_TEMPLATE', 'TABLE', 'conditions_after') AS "QUERY_AFTER" \gset DROP MATERIALIZED VIEW IF EXISTS mat_test; NOTICE: materialized view "mat_test" does not exist, skipping --materialize this VIEW before dump this tests --that the partial state survives the dump intact CREATE MATERIALIZED VIEW mat_before WITH (timescaledb.continuous) AS :QUERY_BEFORE WITH NO DATA; --materialize this VIEW after dump this tests --that the partialize VIEW and refresh mechanics --survives the dump intact CREATE MATERIALIZED VIEW mat_after WITH (timescaledb.continuous) AS :QUERY_AFTER WITH NO DATA; --materialize mat_before SET client_min_messages TO NOTICE; CALL refresh_continuous_aggregate('mat_before', NULL, NULL); SELECT count(*) FROM conditions_before; count ------- 137 SELECT count(*) FROM conditions_after; count ------- 137 SELECT h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_VIEW_name as "PART_VIEW_NAME", partial_VIEW_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_VIEW_name = 'mat_before' \gset SELECT count(*) FROM conditions_before; count ------- 137 SELECT count(*) FROM conditions_after; count ------- 137 --dump & restore \c postgres :ROLE_SUPERUSER \! utils/pg_dump_aux_dump.sh dump/pg_dump.sql \c :TEST_DBNAME SET client_min_messages = ERROR; CREATE EXTENSION timescaledb CASCADE; RESET client_min_messages; --\! cp dump/pg_dump.sql /tmp/dump.sql SELECT timescaledb_pre_restore(); timescaledb_pre_restore ------------------------- t \! utils/pg_dump_aux_restore.sh dump/pg_dump.sql SELECT timescaledb_post_restore(); timescaledb_post_restore -------------------------- t SELECT _timescaledb_functions.stop_background_workers(); stop_background_workers ------------------------- t \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SET timezone TO PST8PDT; --make sure the appropriate DROP are still blocked. \set ON_ERROR_STOP 0 DROP table :"MAT_SCHEMA_NAME".:"MAT_TABLE_NAME"; ERROR: cannot drop table _timescaledb_internal._materialized_hypertable_3 because other objects depend on it DROP VIEW :"PART_VIEW_SCHEMA".:"PART_VIEW_NAME"; ERROR: cannot drop the partial/direct view because it is required by a continuous aggregate \set ON_ERROR_STOP 1 --materialize mat_after CALL refresh_continuous_aggregate('mat_after', NULL, NULL); SELECT count(*) FROM mat_after; count ------- 16 --compare results SELECT count(*) FROM conditions_before; count ------- 137 SELECT count(*) FROM conditions_after; count ------- 137 \set VIEW_NAME mat_before \set QUERY :QUERY_BEFORE \set ECHO errors description | view_name | count ---------------------------------------------------------------+------------+------- Number of rows different between view and original (expect 0) | mat_before | 0 \set VIEW_NAME mat_after \set QUERY :QUERY_AFTER \set ECHO errors description | view_name | count ---------------------------------------------------------------+-----------+------- Number of rows different between view and original (expect 0) | mat_after | 0 DROP MATERIALIZED VIEW mat_after; NOTICE: drop cascades to 2 other objects DROP MATERIALIZED VIEW mat_before; NOTICE: drop cascades to 2 other objects ================================================ FILE: tsl/test/expected/cagg_errors.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set ON_ERROR_STOP 0 \set VERBOSITY default SET timezone TO PST8PDT; --negative tests for query validation create table mat_t1( a integer, b integer,c TEXT); CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature integer NULL, humidity DOUBLE PRECISION NULL, timemeasure TIMESTAMPTZ, timeinterval INTERVAL ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false, timescaledb.myfill = 1) as select location , min(temperature) from conditions group by time_bucket('1d', timec), location WITH NO DATA; ERROR: unrecognized parameter "timescaledb.myfill" HINT: Valid timescaledb parameters are: continuous, create_group_indexes, materialized_only, columnstore, chunk_interval, segmentby, orderby, compress_chunk_interval --valid PG option CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false, check_option = LOCAL ) as select * from conditions , mat_t1 WITH NO DATA; ERROR: unsupported combination of storage parameters DETAIL: A continuous aggregate does not support standard storage parameters. HINT: Use only parameters with the "timescaledb." prefix when creating a continuous aggregate. --non-hypertable CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) as select a, count(*) from mat_t1 group by a WITH NO DATA; ERROR: invalid continuous aggregate view DETAIL: At least one hypertable should be used in the view definition. -- no group by CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) as select count(*) from conditions WITH NO DATA; ERROR: invalid continuous aggregate query HINT: Include at least one aggregate function and a GROUP BY clause with time bucket. -- no time_bucket in group by CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) as select count(*) from conditions group by location WITH NO DATA; ERROR: continuous aggregate view must include a valid time bucket function -- with valid query in a CTE CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS with m1 as ( Select location, count(*) from conditions group by time_bucket('1week', timec) , location) select * from m1 WITH NO DATA; ERROR: invalid continuous aggregate query DETAIL: CTEs and subqueries are not supported by continuous aggregates. --with DISTINCT ON CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) as select distinct on ( location ) count(*) from conditions group by location, time_bucket('1week', timec) WITH NO DATA; ERROR: invalid continuous aggregate query DETAIL: DISTINCT / DISTINCT ON queries are not supported by continuous aggregates. -- time_bucket on non partitioning column of hypertable CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS Select max(temperature) from conditions group by time_bucket('1week', timemeasure) , location WITH NO DATA; ERROR: time bucket function must reference the primary hypertable dimension column --time_bucket on expression CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS Select max(temperature) from conditions group by time_bucket('1week', timec+ '10 minutes'::interval) , location WITH NO DATA; ERROR: time bucket function must reference the primary hypertable dimension column --multiple time_bucket functions CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS Select max(temperature) from conditions group by time_bucket('1week', timec) , time_bucket('1month', timec), location WITH NO DATA; ERROR: continuous aggregate view cannot contain multiple time bucket functions --time_bucket using non-const for first argument CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS Select max(temperature) from conditions group by time_bucket( timeinterval, timec) , location WITH NO DATA; ERROR: only immutable expressions allowed in time bucket function HINT: Use an immutable expression as first argument to the time bucket function. --window function CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS Select avg(temperature) over( order by humidity) from conditions WITH NO DATA; ERROR: invalid continuous aggregate query DETAIL: Window function support not enabled. HINT: Enable experimental window function support by setting timescaledb.enable_cagg_window_functions. -- using subqueries CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS Select sum(humidity), avg(temperature::int4) from ( select humidity, temperature, location, timec from conditions ) q group by time_bucket('1week', timec) , location WITH NO DATA; ERROR: invalid continuous aggregate view DETAIL: Sub-queries are not supported in FROM clause. CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS select * from ( Select sum(humidity), avg(temperature::int4) from conditions group by time_bucket('1week', timec) , location ) q WITH NO DATA; ERROR: invalid continuous aggregate query HINT: Include at least one aggregate function and a GROUP BY clause with time bucket. --using limit /limit offset CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS Select sum(humidity), avg(temperature::int4) from conditions group by time_bucket('1week', timec) , location limit 10 WITH NO DATA; ERROR: invalid continuous aggregate query DETAIL: LIMIT and LIMIT OFFSET are not supported in queries defining continuous aggregates. HINT: Use LIMIT and LIMIT OFFSET in SELECTS from the continuous aggregate view instead. CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS Select sum(humidity), avg(temperature::int4) from conditions group by time_bucket('1week', timec) , location offset 10 WITH NO DATA; ERROR: invalid continuous aggregate query DETAIL: LIMIT and LIMIT OFFSET are not supported in queries defining continuous aggregates. HINT: Use LIMIT and LIMIT OFFSET in SELECTS from the continuous aggregate view instead. --using FETCH CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS Select sum(humidity), avg(temperature::int4) from conditions group by time_bucket('1week', timec) , location fetch first 10 rows only WITH NO DATA; ERROR: invalid continuous aggregate query DETAIL: LIMIT and LIMIT OFFSET are not supported in queries defining continuous aggregates. HINT: Use LIMIT and LIMIT OFFSET in SELECTS from the continuous aggregate view instead. --using locking clauses FOR clause --all should be disabled. we cannot guarntee locks on the hypertable CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS Select sum(humidity), avg(temperature::int4) from conditions group by time_bucket('1week', timec) , location FOR KEY SHARE WITH NO DATA; ERROR: FOR KEY SHARE is not allowed with GROUP BY clause CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS Select sum(humidity), avg(temperature::int4) from conditions group by time_bucket('1week', timec) , location FOR SHARE WITH NO DATA; ERROR: FOR SHARE is not allowed with GROUP BY clause CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS Select sum(humidity), avg(temperature::int4) from conditions group by time_bucket('1week', timec) , location FOR UPDATE WITH NO DATA; ERROR: FOR UPDATE is not allowed with GROUP BY clause CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS Select sum(humidity), avg(temperature::int4) from conditions group by time_bucket('1week', timec) , location FOR NO KEY UPDATE WITH NO DATA; ERROR: FOR NO KEY UPDATE is not allowed with GROUP BY clause --tablesample clause CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS Select sum(humidity), avg(temperature::int4) from conditions tablesample bernoulli(0.2) group by time_bucket('1week', timec) , location WITH NO DATA; ERROR: invalid continuous aggregate view DETAIL: TABLESAMPLE is not supported in continuous aggregate. -- ONLY in from clause CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS Select sum(humidity), avg(temperature::int4) from ONLY conditions group by time_bucket('1week', timec) , location WITH NO DATA; ERROR: invalid continuous aggregate view DETAIL: FROM ONLY on hypertables is not allowed in continuous aggregate. --grouping sets and variants CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS Select sum(humidity), avg(temperature::int4) from conditions group by grouping sets(time_bucket('1week', timec) , location ) WITH NO DATA; ERROR: invalid continuous aggregate query DETAIL: GROUP BY GROUPING SETS, ROLLUP and CUBE are not supported by continuous aggregates HINT: Define multiple continuous aggregates with different grouping levels. CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS Select sum(humidity), avg(temperature::int4) from conditions group by rollup(time_bucket('1week', timec) , location ) WITH NO DATA; ERROR: invalid continuous aggregate query DETAIL: GROUP BY GROUPING SETS, ROLLUP and CUBE are not supported by continuous aggregates HINT: Define multiple continuous aggregates with different grouping levels. -- Should use CREATE MATERIALIZED VIEW to create continuous aggregates CREATE VIEW continuous_aggs_errors_tbl1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1 week', timec) FROM conditions GROUP BY time_bucket('1 week', timec); ERROR: cannot create continuous aggregate with CREATE VIEW HINT: Use CREATE MATERIALIZED VIEW to create a continuous aggregate. -- invalid `bucket_width` for `time_bucket` function CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(NULL, timec), sum(temperature), min(location) FROM conditions GROUP BY 1 WITH NO DATA; ERROR: invalid bucket width for time bucket function -- row security on table create table rowsec_tab( a bigint, b integer, c integer); select table_name from create_hypertable( 'rowsec_tab', 'a', chunk_time_interval=>10); table_name ------------ rowsec_tab CREATE OR REPLACE FUNCTION integer_now_test() returns bigint LANGUAGE SQL STABLE as $$ SELECT coalesce(max(a), 0)::bigint FROM rowsec_tab $$; SELECT set_integer_now_func('rowsec_tab', 'integer_now_test'); set_integer_now_func ---------------------- alter table rowsec_tab ENABLE ROW LEVEL SECURITY; create policy rowsec_tab_allview ON rowsec_tab FOR SELECT USING(true); CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS Select sum( b), min(c) from rowsec_tab group by time_bucket('1', a) WITH NO DATA; ERROR: cannot create continuous aggregate on hypertable with row security drop table conditions cascade; --negative tests for WITH options CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions create materialized view mat_with_test( timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=false) as select time_bucket('1day', timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket('1day', timec) WITH NO DATA; SELECT h.schema_name AS "MAT_SCHEMA_NAME", h.table_name AS "MAT_TABLE_NAME", partial_view_name as "PART_VIEW_NAME", partial_view_schema as "PART_VIEW_SCHEMA" FROM _timescaledb_catalog.continuous_agg ca INNER JOIN _timescaledb_catalog.hypertable h ON(h.id = ca.mat_hypertable_id) WHERE user_view_name = 'mat_with_test' \gset \set ON_ERROR_STOP 0 -- triggers not allowed on continuous aggregate CREATE OR REPLACE FUNCTION not_allowed() RETURNS trigger AS $$ BEGIN RETURN NEW; END; $$ LANGUAGE plpgsql; CREATE TRIGGER not_allowed_trigger INSTEAD OF INSERT ON mat_with_test FOR EACH ROW EXECUTE FUNCTION not_allowed(); ERROR: triggers are not supported on continuous aggregate ALTER MATERIALIZED VIEW mat_with_test SET(timescaledb.create_group_indexes = 'false'); ERROR: cannot alter create_group_indexes option for continuous aggregates ALTER MATERIALIZED VIEW mat_with_test SET(timescaledb.create_group_indexes = 'true'); ERROR: cannot alter create_group_indexes option for continuous aggregates ALTER MATERIALIZED VIEW mat_with_test ALTER timec DROP default; ERROR: cannot alter only SET options of a continuous aggregate \set ON_ERROR_STOP 1 \set VERBOSITY terse DROP TABLE conditions CASCADE; NOTICE: drop cascades to 3 other objects --test WITH using a hypertable with an integer time dimension CREATE TABLE conditions ( timec SMALLINT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec', chunk_time_interval=> 100); table_name ------------ conditions CREATE OR REPLACE FUNCTION integer_now_test_s() returns smallint LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0)::smallint FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now_test_s'); set_integer_now_func ---------------------- \set ON_ERROR_STOP 0 create materialized view mat_with_test( timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=false) as select time_bucket(100, timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket(100, timec) WITH NO DATA; ERROR: time bucket function must reference the primary hypertable dimension column create materialized view mat_with_test( timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=false) as select time_bucket(100, timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket(100, timec) WITH NO DATA; ERROR: time bucket function must reference the primary hypertable dimension column ALTER TABLE conditions ALTER timec type int; create materialized view mat_with_test( timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=false) as select time_bucket(100, timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket(100, timec) WITH NO DATA; \set ON_ERROR_STOP 1 DROP TABLE conditions cascade; NOTICE: drop cascades to 3 other objects CREATE TABLE conditions ( timec BIGINT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec', chunk_time_interval=> 100); table_name ------------ conditions CREATE OR REPLACE FUNCTION integer_now_test_b() returns bigint LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0)::bigint FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now_test_b'); set_integer_now_func ---------------------- create materialized view mat_with_test( timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=false) as select time_bucket(BIGINT '100', timec), min(location), sum(temperature),sum(humidity) from conditions group by 1 WITH NO DATA; -- custom time partition functions are not supported with invalidations CREATE FUNCTION text_part_func(TEXT) RETURNS BIGINT AS $$ SELECT length($1)::BIGINT $$ LANGUAGE SQL IMMUTABLE; CREATE TABLE text_time(time TEXT); SELECT create_hypertable('text_time', 'time', chunk_time_interval => 10, time_partitioning_func => 'text_part_func'); create_hypertable ------------------------ (9,public,text_time,t) \set VERBOSITY default \set ON_ERROR_STOP 0 CREATE MATERIALIZED VIEW text_view WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('5', text_part_func(time)), COUNT(time) FROM text_time GROUP BY 1 WITH NO DATA; ERROR: custom partitioning functions not supported with continuous aggregates \set ON_ERROR_STOP 1 -- Check that we get an error when mixing normal materialized views -- and continuous aggregates. CREATE MATERIALIZED VIEW normal_mat_view AS SELECT time_bucket('5', text_part_func(time)), COUNT(time) FROM text_time GROUP BY 1 WITH NO DATA; \set VERBOSITY terse \set ON_ERROR_STOP 0 DROP MATERIALIZED VIEW normal_mat_view, mat_with_test; ERROR: mixing continuous aggregates and other objects not allowed \set ON_ERROR_STOP 1 DROP TABLE text_time CASCADE; NOTICE: drop cascades to materialized view normal_mat_view CREATE TABLE measurements (time TIMESTAMPTZ NOT NULL, device INT, value FLOAT); SELECT create_hypertable('measurements', 'time'); create_hypertable ---------------------------- (10,public,measurements,t) INSERT INTO measurements VALUES ('2019-03-04 13:30', 1, 1.3); -- Add a continuous aggregate on the measurements table and a policy -- to be able to test error cases for the add_job function. CREATE MATERIALIZED VIEW measurements_summary WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1 day', time), COUNT(time) FROM measurements GROUP BY 1 WITH NO DATA; -- First test that add_job checks the config. It is currently possible -- to add non-custom jobs using the add_job function so we need to -- test that the function actually checks the config parameters. These -- should all generate errors, for different reasons... \set ON_ERROR_STOP 0 -- ... this one because it is missing a field. SELECT add_job( '_timescaledb_functions.policy_refresh_continuous_aggregate'::regproc, '1 hour'::interval, check_config => '_timescaledb_functions.policy_refresh_continuous_aggregate_check'::regproc, config => '{"end_offset": null, "start_offset": null}'); ERROR: could not find "mat_hypertable_id" in config for job -- ... this one because it has a bad value for start_offset SELECT add_job( '_timescaledb_functions.policy_refresh_continuous_aggregate'::regproc, '1 hour'::interval, check_config => '_timescaledb_functions.policy_refresh_continuous_aggregate_check'::regproc, config => '{"end_offset": null, "start_offset": "1 fortnight", "mat_hypertable_id": 11}'); ERROR: invalid input syntax for type interval: "1 fortnight" -- ... this one because it has a bad value for end_offset SELECT add_job( '_timescaledb_functions.policy_refresh_continuous_aggregate'::regproc, '1 hour'::interval, check_config => '_timescaledb_functions.policy_refresh_continuous_aggregate_check'::regproc, config => '{"end_offset": "chicken", "start_offset": null, "mat_hypertable_id": 11}'); ERROR: invalid input syntax for type interval: "chicken" \set ON_ERROR_STOP 1 SELECT add_continuous_aggregate_policy('measurements_summary', NULL, NULL, '1 h'::interval) AS job_id \gset \x on SELECT * FROM _timescaledb_catalog.bgw_job WHERE id = :job_id; -[ RECORD 1 ]-----+-------------------------------------------------------------------- id | 1000 application_name | Refresh Continuous Aggregate Policy [1000] schedule_interval | @ 1 hour max_runtime | @ 0 max_retries | -1 retry_period | @ 1 hour proc_schema | _timescaledb_functions proc_name | policy_refresh_continuous_aggregate owner | default_perm_user scheduled | t fixed_schedule | f initial_start | hypertable_id | 11 config | {"end_offset": null, "start_offset": null, "mat_hypertable_id": 11} check_schema | _timescaledb_functions check_name | policy_refresh_continuous_aggregate_check timezone | \x off -- These are all weird values for the parameters for the continuous -- aggregate jobs and should generate an error. Since the config will -- be replaced, we will also generate error for missing arguments. \set ON_ERROR_STOP 0 SELECT alter_job(:job_id, config => '{"end_offset": "1 week", "start_offset": "2 fortnights"}'); ERROR: could not find "mat_hypertable_id" in config for job SELECT alter_job(:job_id, config => '{"mat_hypertable_id": 11, "end_offset": "chicken", "start_offset": "1 fortnights"}'); ERROR: invalid input syntax for type interval: "1 fortnights" SELECT alter_job(:job_id, config => '{"mat_hypertable_id": 11, "end_offset": "chicken", "start_offset": "1 week"}'); ERROR: invalid input syntax for type interval: "chicken" \set ON_ERROR_STOP 1 DROP TABLE measurements CASCADE; NOTICE: drop cascades to 3 other objects DROP TABLE conditions CASCADE; NOTICE: drop cascades to 3 other objects -- test handling of invalid mat_hypertable_id create table i2980(time timestamptz not null); select create_hypertable('i2980','time'); create_hypertable --------------------- (12,public,i2980,t) create materialized view i2980_cagg with (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h',time), avg(7) FROM i2980 GROUP BY 1; NOTICE: continuous aggregate "i2980_cagg" is already up-to-date select add_continuous_aggregate_policy('i2980_cagg',NULL,NULL,'4h') AS job_id \gset \set ON_ERROR_STOP 0 select alter_job(:job_id,config:='{"end_offset": null, "start_offset": null, "mat_hypertable_id": 1000}'); ERROR: invalid materialized hypertable ID: 1000 --test creating continuous aggregate with compression enabled -- CREATE MATERIALIZED VIEW i2980_cagg2 with (timescaledb.continuous, timescaledb.materialized_only=false, timescaledb.compress) AS SELECT time_bucket('1h',time), avg(7) FROM i2980 GROUP BY 1; ERROR: cannot enable compression while creating a continuous aggregate --this one succeeds CREATE MATERIALIZED VIEW i2980_cagg2 with (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h',time) as bucket, avg(7) FROM i2980 GROUP BY 1; NOTICE: continuous aggregate "i2980_cagg2" is already up-to-date --now enable compression with invalid parameters ALTER MATERIALIZED VIEW i2980_cagg2 SET ( timescaledb.compress, timescaledb.compress_segmentby = 'bucket'); NOTICE: defaulting compress_orderby to bucket ERROR: cannot use column "bucket" for both ordering and segmenting ALTER MATERIALIZED VIEW i2980_cagg2 SET ( timescaledb.compress, timescaledb.compress_orderby = 'bucket'); --enable compression and test re-enabling compression ALTER MATERIALIZED VIEW i2980_cagg2 SET ( timescaledb.compress); NOTICE: defaulting compress_orderby to bucket insert into i2980 select now(); call refresh_continuous_aggregate('i2980_cagg2', NULL, NULL); SELECT compress_chunk(ch) FROM show_chunks('i2980_cagg2') ch; compress_chunk ----------------------------------------- _timescaledb_internal._hyper_14_3_chunk ALTER MATERIALIZED VIEW i2980_cagg2 SET ( timescaledb.compress = 'false'); ERROR: cannot disable columnstore on hypertable with columnstore chunks ALTER MATERIALIZED VIEW i2980_cagg2 SET ( timescaledb.compress = 'true'); NOTICE: defaulting compress_orderby to bucket ALTER MATERIALIZED VIEW i2980_cagg2 SET ( timescaledb.compress, timescaledb.compress_segmentby = 'bucket'); NOTICE: defaulting compress_orderby to bucket ERROR: cannot use column "bucket" for both ordering and segmenting --Errors with compression policy on caggs-- select add_continuous_aggregate_policy('i2980_cagg2', interval '10 day', interval '2 day' ,'4h') AS job_id ; job_id -------- 1002 SELECT add_compression_policy('i2980_cagg', '8 day'::interval); ERROR: columnstore not enabled on continuous aggregate "i2980_cagg" ALTER MATERIALIZED VIEW i2980_cagg SET ( timescaledb.compress ); NOTICE: defaulting compress_orderby to time_bucket SELECT add_continuous_aggregate_policy('i2980_cagg2', '10 day'::interval, '6 day'::interval); ERROR: function add_continuous_aggregate_policy(unknown, interval, interval) does not exist at character 8 SELECT add_compression_policy('i2980_cagg2', '3'::integer); ERROR: unsupported compress_after argument type, expected type : interval SELECT add_compression_policy('i2980_cagg2', 13::integer); ERROR: unsupported compress_after argument type, expected type : interval SELECT materialization_hypertable_schema || '.' || materialization_hypertable_name AS "MAT_TABLE_NAME" FROM timescaledb_information.continuous_aggregates WHERE view_name = 'i2980_cagg2' \gset SELECT add_compression_policy( :'MAT_TABLE_NAME', 13::integer); ERROR: cannot add compression policy to materialized hypertable "_materialized_hypertable_14" --TEST compressing cagg chunks without enabling compression SELECT count(*) FROM (select decompress_chunk(ch) FROM show_chunks('i2980_cagg2') ch ) q; count ------- 1 ALTER MATERIALIZED VIEW i2980_cagg2 SET (timescaledb.compress = 'false'); SELECT compress_chunk(ch) FROM show_chunks('i2980_cagg2') ch; ERROR: columnstore not enabled on "i2980_cagg2" -- test error handling when trying to create cagg on internal hypertable CREATE TABLE comp_ht_test(time timestamptz NOT NULL); SELECT table_name FROM create_hypertable('comp_ht_test','time'); table_name -------------- comp_ht_test ALTER TABLE comp_ht_test SET (timescaledb.compress); SELECT format('%I.%I', ht.schema_name, ht.table_name) AS "INTERNALTABLE" FROM _timescaledb_catalog.hypertable ht INNER JOIN _timescaledb_catalog.hypertable uncompress ON (ht.id = uncompress.compressed_hypertable_id AND uncompress.table_name = 'comp_ht_test') \gset CREATE MATERIALIZED VIEW cagg1 WITH(timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h',now()) FROM :INTERNALTABLE GROUP BY 1; ERROR: hypertable is an internal compressed hypertable --TEST ht + cagg, do not enable compression on ht and try to compress chunk on ht. --Check error handling for this case SELECT compress_chunk(ch) FROM show_chunks('i2980') ch; ERROR: columnstore not enabled on "i2980" -- cagg on normal view should error out CREATE VIEW v1 AS SELECT now() AS time; CREATE MATERIALIZED VIEW cagg1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h',time) FROM v1 GROUP BY 1; ERROR: invalid continuous aggregate view -- cagg on normal view should error out CREATE MATERIALIZED VIEW matv1 AS SELECT now() AS time; CREATE MATERIALIZED VIEW cagg1 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1h',time) FROM matv1 GROUP BY 1; ERROR: invalid continuous aggregate view -- No FROM clause in CAGG definition CREATE MATERIALIZED VIEW cagg1 with (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT 1 GROUP BY 1 WITH NO DATA; ERROR: invalid continuous aggregate query ================================================ FILE: tsl/test/expected/cagg_exp_monthly.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. CREATE TABLE conditions( day DATE NOT NULL, city text NOT NULL, temperature INT NOT NULL); SELECT create_hypertable( 'conditions', 'day', chunk_time_interval => INTERVAL '1 day' ); create_hypertable ------------------------- (1,public,conditions,t) INSERT INTO conditions (day, city, temperature) VALUES ('2021-06-14', 'Moscow', 26), ('2021-06-15', 'Moscow', 22), ('2021-06-16', 'Moscow', 24), ('2021-06-17', 'Moscow', 24), ('2021-06-18', 'Moscow', 27), ('2021-06-19', 'Moscow', 28), ('2021-06-20', 'Moscow', 30), ('2021-06-21', 'Moscow', 31), ('2021-06-22', 'Moscow', 34), ('2021-06-23', 'Moscow', 34), ('2021-06-24', 'Moscow', 34), ('2021-06-25', 'Moscow', 32), ('2021-06-26', 'Moscow', 32), ('2021-06-27', 'Moscow', 31); -- Check that buckets like '1 month 15 days' (fixed-sized + variable-sized) are not allowed \set ON_ERROR_STOP 0 -- timebucket_ng is deprecated and can not be used in new CAggs anymore. -- However, using this GUC the restriction can be lifted in debug builds -- to ensure the functionality can be tested. SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_summary WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 month 15 days', day) AS bucket, MIN(temperature), MAX(temperature) FROM conditions GROUP BY city, bucket WITH NO DATA; ERROR: invalid interval specified \set ON_ERROR_STOP 1 -- Make sure it's possible to create an empty cagg (WITH NO DATA) and -- that all the information about the bucketing function will be saved -- to the TS catalog. CREATE MATERIALIZED VIEW conditions_summary WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 month', day) AS bucket, MIN(temperature), MAX(temperature) FROM conditions GROUP BY city, bucket WITH NO DATA; -- Reset GUC to check if the CAgg would also work in release builds RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT mat_hypertable_id AS cagg_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'conditions_summary' \gset SELECT raw_hypertable_id AS ht_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'conditions_summary' \gset \pset null <NULL> SELECT * FROM _timescaledb_catalog.continuous_aggs_bucket_function WHERE mat_hypertable_id = :cagg_id; mat_hypertable_id | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width -------------------+-------------------------------------------------------------------+--------------+---------------+---------------+-----------------+-------------------- 2 | timescaledb_experimental.time_bucket_ng(interval,pg_catalog.date) | @ 1 mon | <NULL> | <NULL> | <NULL> | f \pset null "" -- Check that the saved invalidation threshold is -infinity SELECT _timescaledb_functions.to_timestamp(watermark) FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id = :ht_id; to_timestamp -------------- -infinity -- Make sure truncating of the refresh window works \set ON_ERROR_STOP 0 CALL refresh_continuous_aggregate('conditions_summary', '2021-07-02', '2021-07-12'); ERROR: refresh window too small \set ON_ERROR_STOP 1 -- Make sure refreshing works CALL refresh_continuous_aggregate('conditions_summary', '2021-06-01', '2021-07-01'); SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary ORDER by month, city; city | month | min | max --------+------------+-----+----- Moscow | 2021-06-01 | 22 | 34 -- Make sure larger refresh window is fine too CALL refresh_continuous_aggregate('conditions_summary', '2021-03-01', '2021-07-01'); SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary ORDER by month, city; city | month | min | max --------+------------+-----+----- Moscow | 2021-06-01 | 22 | 34 -- Special check for "invalid or missing information about the bucketing -- function" code path \c :TEST_DBNAME :ROLE_SUPERUSER CREATE TEMPORARY TABLE restore_table ( LIKE _timescaledb_catalog.continuous_aggs_bucket_function ); INSERT INTO restore_table SELECT * FROM _timescaledb_catalog.continuous_aggs_bucket_function; DELETE FROM _timescaledb_catalog.continuous_aggs_bucket_function; \set ON_ERROR_STOP 0 -- should fail with "invalid or missing information..." CALL refresh_continuous_aggregate('conditions_summary', '2021-06-01', '2021-07-01'); ERROR: invalid or missing information about the bucketing function for cagg \set ON_ERROR_STOP 1 INSERT INTO _timescaledb_catalog.continuous_aggs_bucket_function SELECT * FROM restore_table; DROP TABLE restore_table; -- should execute successfully CALL refresh_continuous_aggregate('conditions_summary', '2021-06-01', '2021-07-01'); NOTICE: continuous aggregate "conditions_summary" is already up-to-date SET ROLE :ROLE_DEFAULT_PERM_USER; -- Check the invalidation threshold SELECT _timescaledb_functions.to_timestamp(watermark) at time zone 'UTC' FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id = :ht_id; timezone -------------------------- Thu Jul 01 00:00:00 2021 -- Add some dummy data for two more months and call refresh (no invalidations test case) INSERT INTO conditions (day, city, temperature) SELECT ts :: date, city, row_number() OVER () FROM generate_series('2021-07-01' :: date, '2021-08-31', '1 day') as ts, unnest(array['Moscow', 'Berlin']) as city; -- Double check generated data SELECT to_char(day, 'YYYY-MM-DD'), city, temperature FROM conditions WHERE day >= '2021-07-01' ORDER BY city DESC, day; to_char | city | temperature ------------+--------+------------- 2021-07-01 | Moscow | 1 2021-07-02 | Moscow | 2 2021-07-03 | Moscow | 3 2021-07-04 | Moscow | 4 2021-07-05 | Moscow | 5 2021-07-06 | Moscow | 6 2021-07-07 | Moscow | 7 2021-07-08 | Moscow | 8 2021-07-09 | Moscow | 9 2021-07-10 | Moscow | 10 2021-07-11 | Moscow | 11 2021-07-12 | Moscow | 12 2021-07-13 | Moscow | 13 2021-07-14 | Moscow | 14 2021-07-15 | Moscow | 15 2021-07-16 | Moscow | 16 2021-07-17 | Moscow | 17 2021-07-18 | Moscow | 18 2021-07-19 | Moscow | 19 2021-07-20 | Moscow | 20 2021-07-21 | Moscow | 21 2021-07-22 | Moscow | 22 2021-07-23 | Moscow | 23 2021-07-24 | Moscow | 24 2021-07-25 | Moscow | 25 2021-07-26 | Moscow | 26 2021-07-27 | Moscow | 27 2021-07-28 | Moscow | 28 2021-07-29 | Moscow | 29 2021-07-30 | Moscow | 30 2021-07-31 | Moscow | 31 2021-08-01 | Moscow | 32 2021-08-02 | Moscow | 33 2021-08-03 | Moscow | 34 2021-08-04 | Moscow | 35 2021-08-05 | Moscow | 36 2021-08-06 | Moscow | 37 2021-08-07 | Moscow | 38 2021-08-08 | Moscow | 39 2021-08-09 | Moscow | 40 2021-08-10 | Moscow | 41 2021-08-11 | Moscow | 42 2021-08-12 | Moscow | 43 2021-08-13 | Moscow | 44 2021-08-14 | Moscow | 45 2021-08-15 | Moscow | 46 2021-08-16 | Moscow | 47 2021-08-17 | Moscow | 48 2021-08-18 | Moscow | 49 2021-08-19 | Moscow | 50 2021-08-20 | Moscow | 51 2021-08-21 | Moscow | 52 2021-08-22 | Moscow | 53 2021-08-23 | Moscow | 54 2021-08-24 | Moscow | 55 2021-08-25 | Moscow | 56 2021-08-26 | Moscow | 57 2021-08-27 | Moscow | 58 2021-08-28 | Moscow | 59 2021-08-29 | Moscow | 60 2021-08-30 | Moscow | 61 2021-08-31 | Moscow | 62 2021-07-01 | Berlin | 63 2021-07-02 | Berlin | 64 2021-07-03 | Berlin | 65 2021-07-04 | Berlin | 66 2021-07-05 | Berlin | 67 2021-07-06 | Berlin | 68 2021-07-07 | Berlin | 69 2021-07-08 | Berlin | 70 2021-07-09 | Berlin | 71 2021-07-10 | Berlin | 72 2021-07-11 | Berlin | 73 2021-07-12 | Berlin | 74 2021-07-13 | Berlin | 75 2021-07-14 | Berlin | 76 2021-07-15 | Berlin | 77 2021-07-16 | Berlin | 78 2021-07-17 | Berlin | 79 2021-07-18 | Berlin | 80 2021-07-19 | Berlin | 81 2021-07-20 | Berlin | 82 2021-07-21 | Berlin | 83 2021-07-22 | Berlin | 84 2021-07-23 | Berlin | 85 2021-07-24 | Berlin | 86 2021-07-25 | Berlin | 87 2021-07-26 | Berlin | 88 2021-07-27 | Berlin | 89 2021-07-28 | Berlin | 90 2021-07-29 | Berlin | 91 2021-07-30 | Berlin | 92 2021-07-31 | Berlin | 93 2021-08-01 | Berlin | 94 2021-08-02 | Berlin | 95 2021-08-03 | Berlin | 96 2021-08-04 | Berlin | 97 2021-08-05 | Berlin | 98 2021-08-06 | Berlin | 99 2021-08-07 | Berlin | 100 2021-08-08 | Berlin | 101 2021-08-09 | Berlin | 102 2021-08-10 | Berlin | 103 2021-08-11 | Berlin | 104 2021-08-12 | Berlin | 105 2021-08-13 | Berlin | 106 2021-08-14 | Berlin | 107 2021-08-15 | Berlin | 108 2021-08-16 | Berlin | 109 2021-08-17 | Berlin | 110 2021-08-18 | Berlin | 111 2021-08-19 | Berlin | 112 2021-08-20 | Berlin | 113 2021-08-21 | Berlin | 114 2021-08-22 | Berlin | 115 2021-08-23 | Berlin | 116 2021-08-24 | Berlin | 117 2021-08-25 | Berlin | 118 2021-08-26 | Berlin | 119 2021-08-27 | Berlin | 120 2021-08-28 | Berlin | 121 2021-08-29 | Berlin | 122 2021-08-30 | Berlin | 123 2021-08-31 | Berlin | 124 -- Make sure the invalidation threshold was unaffected SELECT _timescaledb_functions.to_timestamp(watermark) at time zone 'UTC' FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id = :ht_id; timezone -------------------------- Thu Jul 01 00:00:00 2021 -- Make sure the invalidation log is empty SELECT _timescaledb_functions.to_timestamp(lowest_modified_value) AS lowest, _timescaledb_functions.to_timestamp(greatest_modified_value) AS greatest FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log WHERE hypertable_id = :ht_id; lowest | greatest --------+---------- -- Call refresh CALL refresh_continuous_aggregate('conditions_summary', '2021-06-15', '2021-09-15'); SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary ORDER by month, city; city | month | min | max --------+------------+-----+----- Moscow | 2021-06-01 | 22 | 34 Berlin | 2021-07-01 | 63 | 93 Moscow | 2021-07-01 | 1 | 31 Berlin | 2021-08-01 | 94 | 124 Moscow | 2021-08-01 | 32 | 62 -- Make sure the invalidation threshold has changed SELECT _timescaledb_functions.to_timestamp(watermark) at time zone 'UTC' FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id = :ht_id; timezone -------------------------- Wed Sep 01 00:00:00 2021 -- Make sure the catalog is cleaned up when the cagg is dropped DROP MATERIALIZED VIEW conditions_summary; NOTICE: drop cascades to 3 other objects SELECT count(*) FROM _timescaledb_catalog.continuous_agg WHERE mat_hypertable_id = :cagg_id; count ------- 0 SELECT count(*) FROM _timescaledb_catalog.continuous_aggs_bucket_function WHERE mat_hypertable_id = :cagg_id; count ------- 0 -- Re-create cagg, this time WITH DATA SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_summary WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 month', day) AS bucket, MIN(temperature), MAX(temperature) FROM conditions GROUP BY city, bucket; NOTICE: refreshing continuous aggregate "conditions_summary" RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; -- Make sure cagg was filled SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary ORDER by month, city; city | month | min | max --------+------------+-----+----- Moscow | 2021-06-01 | 22 | 34 Berlin | 2021-07-01 | 63 | 93 Moscow | 2021-07-01 | 1 | 31 Berlin | 2021-08-01 | 94 | 124 Moscow | 2021-08-01 | 32 | 62 -- Check the invalidation. -- Step 1/2. Insert some more data , do a refresh and make sure that the -- invalidation log is empty. INSERT INTO conditions (day, city, temperature) SELECT ts :: date, city, row_number() OVER () FROM generate_series('2021-09-01' :: date, '2021-09-15', '1 day') as ts, unnest(array['Moscow', 'Berlin']) as city; CALL refresh_continuous_aggregate('conditions_summary', '2021-09-01', '2021-10-01'); SELECT _timescaledb_functions.to_timestamp(lowest_modified_value) AS lowest, _timescaledb_functions.to_timestamp(greatest_modified_value) AS greatest FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log WHERE hypertable_id = :ht_id; lowest | greatest --------+---------- SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary ORDER by month, city; city | month | min | max --------+------------+-----+----- Moscow | 2021-06-01 | 22 | 34 Berlin | 2021-07-01 | 63 | 93 Moscow | 2021-07-01 | 1 | 31 Berlin | 2021-08-01 | 94 | 124 Moscow | 2021-08-01 | 32 | 62 Berlin | 2021-09-01 | 16 | 30 Moscow | 2021-09-01 | 1 | 15 -- Step 2/2. Add more data below the invalidation threshold, make sure that the -- invalidation log is not empty, that do a refresh. INSERT INTO conditions (day, city, temperature) SELECT ts :: date, city, (CASE WHEN city = 'Moscow' THEN -40 ELSE 40 END) FROM generate_series('2021-09-16' :: date, '2021-09-30', '1 day') as ts, unnest(array['Moscow', 'Berlin']) as city; SELECT _timescaledb_functions.to_timestamp(lowest_modified_value) at time zone 'UTC' AS lowest, _timescaledb_functions.to_timestamp(greatest_modified_value) at time zone 'UTC' AS greatest FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log WHERE hypertable_id = :ht_id; lowest | greatest --------------------------+-------------------------- Thu Sep 16 00:00:00 2021 | Thu Sep 16 00:00:00 2021 Fri Sep 17 00:00:00 2021 | Fri Sep 17 00:00:00 2021 Sat Sep 18 00:00:00 2021 | Sat Sep 18 00:00:00 2021 Sun Sep 19 00:00:00 2021 | Sun Sep 19 00:00:00 2021 Mon Sep 20 00:00:00 2021 | Mon Sep 20 00:00:00 2021 Tue Sep 21 00:00:00 2021 | Tue Sep 21 00:00:00 2021 Wed Sep 22 00:00:00 2021 | Wed Sep 22 00:00:00 2021 Thu Sep 23 00:00:00 2021 | Thu Sep 23 00:00:00 2021 Fri Sep 24 00:00:00 2021 | Fri Sep 24 00:00:00 2021 Sat Sep 25 00:00:00 2021 | Sat Sep 25 00:00:00 2021 Sun Sep 26 00:00:00 2021 | Sun Sep 26 00:00:00 2021 Mon Sep 27 00:00:00 2021 | Mon Sep 27 00:00:00 2021 Tue Sep 28 00:00:00 2021 | Tue Sep 28 00:00:00 2021 Wed Sep 29 00:00:00 2021 | Wed Sep 29 00:00:00 2021 Thu Sep 30 00:00:00 2021 | Thu Sep 30 00:00:00 2021 CALL refresh_continuous_aggregate('conditions_summary', '2021-09-01', '2021-10-01'); SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary ORDER by month, city; city | month | min | max --------+------------+-----+----- Moscow | 2021-06-01 | 22 | 34 Berlin | 2021-07-01 | 63 | 93 Moscow | 2021-07-01 | 1 | 31 Berlin | 2021-08-01 | 94 | 124 Moscow | 2021-08-01 | 32 | 62 Berlin | 2021-09-01 | 16 | 40 Moscow | 2021-09-01 | -40 | 15 SELECT _timescaledb_functions.to_timestamp(lowest_modified_value) AS lowest, _timescaledb_functions.to_timestamp(greatest_modified_value) AS greatest FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log WHERE hypertable_id = :ht_id; lowest | greatest --------+---------- -- Create a real-time aggregate DROP MATERIALIZED VIEW conditions_summary; NOTICE: drop cascades to 4 other objects SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_summary WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 month', day) AS bucket, MIN(temperature), MAX(temperature) FROM conditions GROUP BY city, bucket; NOTICE: refreshing continuous aggregate "conditions_summary" RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary ORDER by month, city; city | month | min | max --------+------------+-----+----- Moscow | 2021-06-01 | 22 | 34 Berlin | 2021-07-01 | 63 | 93 Moscow | 2021-07-01 | 1 | 31 Berlin | 2021-08-01 | 94 | 124 Moscow | 2021-08-01 | 32 | 62 Berlin | 2021-09-01 | 16 | 40 Moscow | 2021-09-01 | -40 | 15 -- Add some data to the hypertable and make sure they are visible in the cagg INSERT INTO conditions (day, city, temperature) VALUES ('2021-10-01', 'Moscow', 1), ('2021-10-02', 'Moscow', 2), ('2021-10-03', 'Moscow', 3), ('2021-10-04', 'Moscow', 4), ('2021-10-01', 'Berlin', 5), ('2021-10-02', 'Berlin', 6), ('2021-10-03', 'Berlin', 7), ('2021-10-04', 'Berlin', 8); SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary ORDER by month, city; city | month | min | max --------+------------+-----+----- Moscow | 2021-06-01 | 22 | 34 Berlin | 2021-07-01 | 63 | 93 Moscow | 2021-07-01 | 1 | 31 Berlin | 2021-08-01 | 94 | 124 Moscow | 2021-08-01 | 32 | 62 Berlin | 2021-09-01 | 16 | 40 Moscow | 2021-09-01 | -40 | 15 Berlin | 2021-10-01 | 5 | 8 Moscow | 2021-10-01 | 1 | 4 -- Refresh the cagg and make sure that the result of SELECT query didn't change CALL refresh_continuous_aggregate('conditions_summary', '2021-10-01', '2021-11-01'); SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary ORDER by month, city; city | month | min | max --------+------------+-----+----- Moscow | 2021-06-01 | 22 | 34 Berlin | 2021-07-01 | 63 | 93 Moscow | 2021-07-01 | 1 | 31 Berlin | 2021-08-01 | 94 | 124 Moscow | 2021-08-01 | 32 | 62 Berlin | 2021-09-01 | 16 | 40 Moscow | 2021-09-01 | -40 | 15 Berlin | 2021-10-01 | 5 | 8 Moscow | 2021-10-01 | 1 | 4 -- Add some more data, enable compression, compress the chunks and repeat the test INSERT INTO conditions (day, city, temperature) VALUES ('2021-11-01', 'Moscow', 11), ('2021-11-02', 'Moscow', 12), ('2021-11-03', 'Moscow', 13), ('2021-11-04', 'Moscow', 14), ('2021-11-01', 'Berlin', 15), ('2021-11-02', 'Berlin', 16), ('2021-11-03', 'Berlin', 17), ('2021-11-04', 'Berlin', 18); ALTER TABLE conditions SET ( timescaledb.compress, timescaledb.compress_segmentby = 'city' ); SELECT compress_chunk(ch) FROM show_chunks('conditions') AS ch; compress_chunk ------------------------------------------ _timescaledb_internal._hyper_1_1_chunk _timescaledb_internal._hyper_1_2_chunk _timescaledb_internal._hyper_1_3_chunk _timescaledb_internal._hyper_1_4_chunk _timescaledb_internal._hyper_1_5_chunk _timescaledb_internal._hyper_1_6_chunk _timescaledb_internal._hyper_1_7_chunk _timescaledb_internal._hyper_1_8_chunk _timescaledb_internal._hyper_1_9_chunk _timescaledb_internal._hyper_1_10_chunk _timescaledb_internal._hyper_1_11_chunk _timescaledb_internal._hyper_1_12_chunk _timescaledb_internal._hyper_1_13_chunk _timescaledb_internal._hyper_1_14_chunk _timescaledb_internal._hyper_1_16_chunk _timescaledb_internal._hyper_1_17_chunk _timescaledb_internal._hyper_1_18_chunk _timescaledb_internal._hyper_1_19_chunk _timescaledb_internal._hyper_1_20_chunk _timescaledb_internal._hyper_1_21_chunk _timescaledb_internal._hyper_1_22_chunk _timescaledb_internal._hyper_1_23_chunk _timescaledb_internal._hyper_1_24_chunk _timescaledb_internal._hyper_1_25_chunk _timescaledb_internal._hyper_1_26_chunk _timescaledb_internal._hyper_1_27_chunk _timescaledb_internal._hyper_1_28_chunk _timescaledb_internal._hyper_1_29_chunk _timescaledb_internal._hyper_1_30_chunk _timescaledb_internal._hyper_1_31_chunk _timescaledb_internal._hyper_1_32_chunk _timescaledb_internal._hyper_1_33_chunk _timescaledb_internal._hyper_1_34_chunk _timescaledb_internal._hyper_1_35_chunk _timescaledb_internal._hyper_1_36_chunk _timescaledb_internal._hyper_1_37_chunk _timescaledb_internal._hyper_1_38_chunk _timescaledb_internal._hyper_1_39_chunk _timescaledb_internal._hyper_1_40_chunk _timescaledb_internal._hyper_1_41_chunk _timescaledb_internal._hyper_1_42_chunk _timescaledb_internal._hyper_1_43_chunk _timescaledb_internal._hyper_1_44_chunk _timescaledb_internal._hyper_1_45_chunk _timescaledb_internal._hyper_1_46_chunk _timescaledb_internal._hyper_1_47_chunk _timescaledb_internal._hyper_1_48_chunk _timescaledb_internal._hyper_1_49_chunk _timescaledb_internal._hyper_1_50_chunk _timescaledb_internal._hyper_1_51_chunk _timescaledb_internal._hyper_1_52_chunk _timescaledb_internal._hyper_1_53_chunk _timescaledb_internal._hyper_1_54_chunk _timescaledb_internal._hyper_1_55_chunk _timescaledb_internal._hyper_1_56_chunk _timescaledb_internal._hyper_1_57_chunk _timescaledb_internal._hyper_1_58_chunk _timescaledb_internal._hyper_1_59_chunk _timescaledb_internal._hyper_1_60_chunk _timescaledb_internal._hyper_1_61_chunk _timescaledb_internal._hyper_1_62_chunk _timescaledb_internal._hyper_1_63_chunk _timescaledb_internal._hyper_1_64_chunk _timescaledb_internal._hyper_1_65_chunk _timescaledb_internal._hyper_1_66_chunk _timescaledb_internal._hyper_1_67_chunk _timescaledb_internal._hyper_1_68_chunk _timescaledb_internal._hyper_1_69_chunk _timescaledb_internal._hyper_1_70_chunk _timescaledb_internal._hyper_1_71_chunk _timescaledb_internal._hyper_1_72_chunk _timescaledb_internal._hyper_1_73_chunk _timescaledb_internal._hyper_1_74_chunk _timescaledb_internal._hyper_1_75_chunk _timescaledb_internal._hyper_1_76_chunk _timescaledb_internal._hyper_1_77_chunk _timescaledb_internal._hyper_1_83_chunk _timescaledb_internal._hyper_1_84_chunk _timescaledb_internal._hyper_1_85_chunk _timescaledb_internal._hyper_1_86_chunk _timescaledb_internal._hyper_1_87_chunk _timescaledb_internal._hyper_1_88_chunk _timescaledb_internal._hyper_1_89_chunk _timescaledb_internal._hyper_1_90_chunk _timescaledb_internal._hyper_1_91_chunk _timescaledb_internal._hyper_1_92_chunk _timescaledb_internal._hyper_1_93_chunk _timescaledb_internal._hyper_1_94_chunk _timescaledb_internal._hyper_1_95_chunk _timescaledb_internal._hyper_1_96_chunk _timescaledb_internal._hyper_1_97_chunk _timescaledb_internal._hyper_1_99_chunk _timescaledb_internal._hyper_1_100_chunk _timescaledb_internal._hyper_1_101_chunk _timescaledb_internal._hyper_1_102_chunk _timescaledb_internal._hyper_1_103_chunk _timescaledb_internal._hyper_1_104_chunk _timescaledb_internal._hyper_1_105_chunk _timescaledb_internal._hyper_1_106_chunk _timescaledb_internal._hyper_1_107_chunk _timescaledb_internal._hyper_1_108_chunk _timescaledb_internal._hyper_1_109_chunk _timescaledb_internal._hyper_1_110_chunk _timescaledb_internal._hyper_1_111_chunk _timescaledb_internal._hyper_1_112_chunk _timescaledb_internal._hyper_1_113_chunk _timescaledb_internal._hyper_1_118_chunk _timescaledb_internal._hyper_1_119_chunk _timescaledb_internal._hyper_1_120_chunk _timescaledb_internal._hyper_1_121_chunk _timescaledb_internal._hyper_1_123_chunk _timescaledb_internal._hyper_1_124_chunk _timescaledb_internal._hyper_1_125_chunk _timescaledb_internal._hyper_1_126_chunk -- Data for 2021-11 is seen because the cagg is real-time SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary ORDER by month, city; city | month | min | max --------+------------+-----+----- Moscow | 2021-06-01 | 22 | 34 Berlin | 2021-07-01 | 63 | 93 Moscow | 2021-07-01 | 1 | 31 Berlin | 2021-08-01 | 94 | 124 Moscow | 2021-08-01 | 32 | 62 Berlin | 2021-09-01 | 16 | 40 Moscow | 2021-09-01 | -40 | 15 Berlin | 2021-10-01 | 5 | 8 Moscow | 2021-10-01 | 1 | 4 Berlin | 2021-11-01 | 15 | 18 Moscow | 2021-11-01 | 11 | 14 CALL refresh_continuous_aggregate('conditions_summary', '2021-11-01', '2021-12-01'); -- Data for 2021-11 is seen because the cagg was refreshed SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary ORDER by month, city; city | month | min | max --------+------------+-----+----- Moscow | 2021-06-01 | 22 | 34 Berlin | 2021-07-01 | 63 | 93 Moscow | 2021-07-01 | 1 | 31 Berlin | 2021-08-01 | 94 | 124 Moscow | 2021-08-01 | 32 | 62 Berlin | 2021-09-01 | 16 | 40 Moscow | 2021-09-01 | -40 | 15 Berlin | 2021-10-01 | 5 | 8 Moscow | 2021-10-01 | 1 | 4 Berlin | 2021-11-01 | 15 | 18 Moscow | 2021-11-01 | 11 | 14 -- Test N-months buckets where N in 2,3,4,5,6,12,13 on a relatively large table -- This also tests the case when a single hypertable has multiple caggs. CREATE TABLE conditions_large( day DATE NOT NULL, temperature INT NOT NULL); SELECT create_hypertable( 'conditions_large', 'day', chunk_time_interval => INTERVAL '1 month' ); create_hypertable ------------------------------- (6,public,conditions_large,t) INSERT INTO conditions_large(day, temperature) SELECT ts, date_part('month', ts)*100 + date_part('day', ts) FROM generate_series('2010-01-01' :: date, '2020-01-01' :: date - interval '1 day', '1 day') as ts; SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_large_2m WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT timescaledb_experimental.time_bucket_ng('2 months', day) AS bucket, MIN(temperature), MAX(temperature) FROM conditions_large GROUP BY bucket; NOTICE: refreshing continuous aggregate "conditions_large_2m" RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT * FROM conditions_large_2m ORDER BY bucket; bucket | min | max ------------+------+------ 01-01-2010 | 101 | 228 03-01-2010 | 301 | 430 05-01-2010 | 501 | 630 07-01-2010 | 701 | 831 09-01-2010 | 901 | 1031 11-01-2010 | 1101 | 1231 01-01-2011 | 101 | 228 03-01-2011 | 301 | 430 05-01-2011 | 501 | 630 07-01-2011 | 701 | 831 09-01-2011 | 901 | 1031 11-01-2011 | 1101 | 1231 01-01-2012 | 101 | 229 03-01-2012 | 301 | 430 05-01-2012 | 501 | 630 07-01-2012 | 701 | 831 09-01-2012 | 901 | 1031 11-01-2012 | 1101 | 1231 01-01-2013 | 101 | 228 03-01-2013 | 301 | 430 05-01-2013 | 501 | 630 07-01-2013 | 701 | 831 09-01-2013 | 901 | 1031 11-01-2013 | 1101 | 1231 01-01-2014 | 101 | 228 03-01-2014 | 301 | 430 05-01-2014 | 501 | 630 07-01-2014 | 701 | 831 09-01-2014 | 901 | 1031 11-01-2014 | 1101 | 1231 01-01-2015 | 101 | 228 03-01-2015 | 301 | 430 05-01-2015 | 501 | 630 07-01-2015 | 701 | 831 09-01-2015 | 901 | 1031 11-01-2015 | 1101 | 1231 01-01-2016 | 101 | 229 03-01-2016 | 301 | 430 05-01-2016 | 501 | 630 07-01-2016 | 701 | 831 09-01-2016 | 901 | 1031 11-01-2016 | 1101 | 1231 01-01-2017 | 101 | 228 03-01-2017 | 301 | 430 05-01-2017 | 501 | 630 07-01-2017 | 701 | 831 09-01-2017 | 901 | 1031 11-01-2017 | 1101 | 1231 01-01-2018 | 101 | 228 03-01-2018 | 301 | 430 05-01-2018 | 501 | 630 07-01-2018 | 701 | 831 09-01-2018 | 901 | 1031 11-01-2018 | 1101 | 1231 01-01-2019 | 101 | 228 03-01-2019 | 301 | 430 05-01-2019 | 501 | 630 07-01-2019 | 701 | 831 09-01-2019 | 901 | 1031 11-01-2019 | 1101 | 1231 SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_large_3m WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT timescaledb_experimental.time_bucket_ng('3 months', day) AS bucket, MIN(temperature), MAX(temperature) FROM conditions_large GROUP BY bucket; NOTICE: refreshing continuous aggregate "conditions_large_3m" RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT * FROM conditions_large_3m ORDER BY bucket; bucket | min | max ------------+------+------ 01-01-2010 | 101 | 331 04-01-2010 | 401 | 630 07-01-2010 | 701 | 930 10-01-2010 | 1001 | 1231 01-01-2011 | 101 | 331 04-01-2011 | 401 | 630 07-01-2011 | 701 | 930 10-01-2011 | 1001 | 1231 01-01-2012 | 101 | 331 04-01-2012 | 401 | 630 07-01-2012 | 701 | 930 10-01-2012 | 1001 | 1231 01-01-2013 | 101 | 331 04-01-2013 | 401 | 630 07-01-2013 | 701 | 930 10-01-2013 | 1001 | 1231 01-01-2014 | 101 | 331 04-01-2014 | 401 | 630 07-01-2014 | 701 | 930 10-01-2014 | 1001 | 1231 01-01-2015 | 101 | 331 04-01-2015 | 401 | 630 07-01-2015 | 701 | 930 10-01-2015 | 1001 | 1231 01-01-2016 | 101 | 331 04-01-2016 | 401 | 630 07-01-2016 | 701 | 930 10-01-2016 | 1001 | 1231 01-01-2017 | 101 | 331 04-01-2017 | 401 | 630 07-01-2017 | 701 | 930 10-01-2017 | 1001 | 1231 01-01-2018 | 101 | 331 04-01-2018 | 401 | 630 07-01-2018 | 701 | 930 10-01-2018 | 1001 | 1231 01-01-2019 | 101 | 331 04-01-2019 | 401 | 630 07-01-2019 | 701 | 930 10-01-2019 | 1001 | 1231 SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_large_4m WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT timescaledb_experimental.time_bucket_ng('4 months', day) AS bucket, MIN(temperature), MAX(temperature) FROM conditions_large GROUP BY bucket; NOTICE: refreshing continuous aggregate "conditions_large_4m" RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT * FROM conditions_large_4m ORDER BY bucket; bucket | min | max ------------+-----+------ 01-01-2010 | 101 | 430 05-01-2010 | 501 | 831 09-01-2010 | 901 | 1231 01-01-2011 | 101 | 430 05-01-2011 | 501 | 831 09-01-2011 | 901 | 1231 01-01-2012 | 101 | 430 05-01-2012 | 501 | 831 09-01-2012 | 901 | 1231 01-01-2013 | 101 | 430 05-01-2013 | 501 | 831 09-01-2013 | 901 | 1231 01-01-2014 | 101 | 430 05-01-2014 | 501 | 831 09-01-2014 | 901 | 1231 01-01-2015 | 101 | 430 05-01-2015 | 501 | 831 09-01-2015 | 901 | 1231 01-01-2016 | 101 | 430 05-01-2016 | 501 | 831 09-01-2016 | 901 | 1231 01-01-2017 | 101 | 430 05-01-2017 | 501 | 831 09-01-2017 | 901 | 1231 01-01-2018 | 101 | 430 05-01-2018 | 501 | 831 09-01-2018 | 901 | 1231 01-01-2019 | 101 | 430 05-01-2019 | 501 | 831 09-01-2019 | 901 | 1231 SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_large_5m WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT timescaledb_experimental.time_bucket_ng('5 months', day) AS bucket, MIN(temperature), MAX(temperature) FROM conditions_large GROUP BY bucket; NOTICE: refreshing continuous aggregate "conditions_large_5m" RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT * FROM conditions_large_5m ORDER BY bucket; bucket | min | max ------------+-----+------ 01-01-2010 | 101 | 531 06-01-2010 | 601 | 1031 11-01-2010 | 101 | 1231 04-01-2011 | 401 | 831 09-01-2011 | 101 | 1231 02-01-2012 | 201 | 630 07-01-2012 | 701 | 1130 12-01-2012 | 101 | 1231 05-01-2013 | 501 | 930 10-01-2013 | 101 | 1231 03-01-2014 | 301 | 731 08-01-2014 | 801 | 1231 01-01-2015 | 101 | 531 06-01-2015 | 601 | 1031 11-01-2015 | 101 | 1231 04-01-2016 | 401 | 831 09-01-2016 | 101 | 1231 02-01-2017 | 201 | 630 07-01-2017 | 701 | 1130 12-01-2017 | 101 | 1231 05-01-2018 | 501 | 930 10-01-2018 | 101 | 1231 03-01-2019 | 301 | 731 08-01-2019 | 801 | 1231 SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_large_6m WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT timescaledb_experimental.time_bucket_ng('6 months', day) AS bucket, MIN(temperature), MAX(temperature) FROM conditions_large GROUP BY bucket; NOTICE: refreshing continuous aggregate "conditions_large_6m" RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT * FROM conditions_large_6m ORDER BY bucket; bucket | min | max ------------+-----+------ 01-01-2010 | 101 | 630 07-01-2010 | 701 | 1231 01-01-2011 | 101 | 630 07-01-2011 | 701 | 1231 01-01-2012 | 101 | 630 07-01-2012 | 701 | 1231 01-01-2013 | 101 | 630 07-01-2013 | 701 | 1231 01-01-2014 | 101 | 630 07-01-2014 | 701 | 1231 01-01-2015 | 101 | 630 07-01-2015 | 701 | 1231 01-01-2016 | 101 | 630 07-01-2016 | 701 | 1231 01-01-2017 | 101 | 630 07-01-2017 | 701 | 1231 01-01-2018 | 101 | 630 07-01-2018 | 701 | 1231 01-01-2019 | 101 | 630 07-01-2019 | 701 | 1231 SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_large_1y WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT timescaledb_experimental.time_bucket_ng('1 year', day) AS bucket, MIN(temperature), MAX(temperature) FROM conditions_large GROUP BY bucket; NOTICE: refreshing continuous aggregate "conditions_large_1y" RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT * FROM conditions_large_1y ORDER BY bucket; bucket | min | max ------------+-----+------ 01-01-2010 | 101 | 1231 01-01-2011 | 101 | 1231 01-01-2012 | 101 | 1231 01-01-2013 | 101 | 1231 01-01-2014 | 101 | 1231 01-01-2015 | 101 | 1231 01-01-2016 | 101 | 1231 01-01-2017 | 101 | 1231 01-01-2018 | 101 | 1231 01-01-2019 | 101 | 1231 SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_large_1y1m WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT timescaledb_experimental.time_bucket_ng('1 year 1 month', day) AS bucket, MIN(temperature), MAX(temperature) FROM conditions_large GROUP BY bucket; NOTICE: refreshing continuous aggregate "conditions_large_1y1m" RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT * FROM conditions_large_1y1m ORDER BY bucket; bucket | min | max ------------+-----+------ 10-01-2009 | 101 | 1031 11-01-2010 | 101 | 1231 12-01-2011 | 101 | 1231 01-01-2013 | 101 | 1231 02-01-2014 | 101 | 1231 03-01-2015 | 101 | 1231 04-01-2016 | 101 | 1231 05-01-2017 | 101 | 1231 06-01-2018 | 101 | 1231 07-01-2019 | 701 | 1231 -- Trigger merged refresh to check corresponding code path as well DROP MATERIALIZED VIEW conditions_large_1y; NOTICE: drop cascades to 10 other objects SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_large_1y WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT timescaledb_experimental.time_bucket_ng('1 year', day) AS bucket, MIN(temperature), MAX(temperature) FROM conditions_large GROUP BY bucket; NOTICE: refreshing continuous aggregate "conditions_large_1y" RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT * FROM conditions_large_1y ORDER BY bucket; bucket | min | max ------------+-----+------ 01-01-2010 | 101 | 1231 01-01-2011 | 101 | 1231 01-01-2012 | 101 | 1231 01-01-2013 | 101 | 1231 01-01-2014 | 101 | 1231 01-01-2015 | 101 | 1231 01-01-2016 | 101 | 1231 01-01-2017 | 101 | 1231 01-01-2018 | 101 | 1231 01-01-2019 | 101 | 1231 INSERT INTO conditions_large(day, temperature) SELECT ts, date_part('month', ts)*100 + date_part('day', ts) FROM generate_series('2020-01-01' :: date, '2021-01-01' :: date - interval '1 day', '1 day') as ts; CALL refresh_continuous_aggregate('conditions_large_1y', '2020-01-01', '2021-01-01'); SELECT * FROM conditions_large_1y ORDER BY bucket; bucket | min | max ------------+-----+------ 01-01-2010 | 101 | 1231 01-01-2011 | 101 | 1231 01-01-2012 | 101 | 1231 01-01-2013 | 101 | 1231 01-01-2014 | 101 | 1231 01-01-2015 | 101 | 1231 01-01-2016 | 101 | 1231 01-01-2017 | 101 | 1231 01-01-2018 | 101 | 1231 01-01-2019 | 101 | 1231 01-01-2020 | 101 | 1231 -- Test the specific code path of creating a CAGG on top of empty hypertable. CREATE TABLE conditions_empty( day DATE NOT NULL, city text NOT NULL, temperature INT NOT NULL); SELECT create_hypertable( 'conditions_empty', 'day', chunk_time_interval => INTERVAL '1 day' ); create_hypertable -------------------------------- (15,public,conditions_empty,t) SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_summary_empty WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 month', day) AS bucket, MIN(temperature), MAX(temperature) FROM conditions_empty GROUP BY city, bucket; NOTICE: continuous aggregate "conditions_summary_empty" is already up-to-date RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary_empty ORDER by month, city; city | month | min | max ------+-------+-----+----- -- The test above changes the record that gets added to the invalidation log -- for an empty table. Make sure it doesn't have any unintended side-effects -- and the refreshing works as expected. INSERT INTO conditions_empty (day, city, temperature) VALUES ('2021-06-14', 'Moscow', 26), ('2021-06-15', 'Moscow', 22), ('2021-06-16', 'Moscow', 24), ('2021-06-17', 'Moscow', 24), ('2021-06-18', 'Moscow', 27), ('2021-06-19', 'Moscow', 28), ('2021-06-20', 'Moscow', 30), ('2021-06-21', 'Moscow', 31), ('2021-06-22', 'Moscow', 34), ('2021-06-23', 'Moscow', 34), ('2021-06-24', 'Moscow', 34), ('2021-06-25', 'Moscow', 32), ('2021-06-26', 'Moscow', 32), ('2021-06-27', 'Moscow', 31); CALL refresh_continuous_aggregate('conditions_summary_empty', '2021-06-01', '2021-07-01'); SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary_empty ORDER by month, city; city | month | min | max --------+------------+-----+----- Moscow | 2021-06-01 | 22 | 34 -- Make sure add_continuous_aggregate_policy() works CREATE TABLE conditions_policy( day DATE NOT NULL, city text NOT NULL, temperature INT NOT NULL); SELECT create_hypertable( 'conditions_policy', 'day', chunk_time_interval => INTERVAL '1 day' ); create_hypertable --------------------------------- (17,public,conditions_policy,t) INSERT INTO conditions_policy (day, city, temperature) VALUES ('2021-06-14', 'Moscow', 26), ('2021-06-15', 'Moscow', 22), ('2021-06-16', 'Moscow', 24), ('2021-06-17', 'Moscow', 24), ('2021-06-18', 'Moscow', 27), ('2021-06-19', 'Moscow', 28), ('2021-06-20', 'Moscow', 30), ('2021-06-21', 'Moscow', 31), ('2021-06-22', 'Moscow', 34), ('2021-06-23', 'Moscow', 34), ('2021-06-24', 'Moscow', 34), ('2021-06-25', 'Moscow', 32), ('2021-06-26', 'Moscow', 32), ('2021-06-27', 'Moscow', 31); SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_summary_policy WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 month', day) AS bucket, MIN(temperature), MAX(temperature) FROM conditions_policy GROUP BY city, bucket; NOTICE: refreshing continuous aggregate "conditions_summary_policy" RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT * FROM conditions_summary_policy; city | bucket | min | max --------+------------+-----+----- Moscow | 06-01-2021 | 22 | 34 \set ON_ERROR_STOP 0 -- Check for "policy refresh window too small" error SELECT add_continuous_aggregate_policy('conditions_summary_policy', -- Historically, 1 month is just a synonym to 30 days here. -- See interval_to_int64() and interval_to_int128(). start_offset => INTERVAL '2 months', end_offset => INTERVAL '1 day', schedule_interval => INTERVAL '1 hour'); ERROR: policy refresh window too small \set ON_ERROR_STOP 1 SELECT add_continuous_aggregate_policy('conditions_summary_policy', start_offset => INTERVAL '65 days', end_offset => INTERVAL '1 day', schedule_interval => INTERVAL '1 hour') AS job_id \gset SELECT delete_job(:job_id); delete_job ------------ SELECT add_continuous_aggregate_policy('conditions_summary_policy', start_offset => INTERVAL '2 months', end_offset => INTERVAL '0 months', schedule_interval => INTERVAL '1 hour') AS job_id \gset SELECT delete_job(:job_id); delete_job ------------ \set ON_ERROR_STOP 0 SELECT add_continuous_aggregate_policy('conditions_summary_policy', start_offset => INTERVAL '2 months', end_offset => INTERVAL '1 months', schedule_interval => INTERVAL '1 hour'); ERROR: policy refresh window too small \set ON_ERROR_STOP 1 SELECT add_continuous_aggregate_policy('conditions_summary_policy', start_offset => INTERVAL '3 months', end_offset => INTERVAL '1 months', schedule_interval => INTERVAL '1 hour'); add_continuous_aggregate_policy --------------------------------- 1002 ================================================ FILE: tsl/test/expected/cagg_exp_next_gen.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- Make sure experimental immutable function with 2 arguments can be used in caggs. -- Functions with 3 arguments and/or stable functions are currently not supported in caggs. CREATE TABLE conditions( day DATE NOT NULL, city text NOT NULL, temperature INT NOT NULL); SELECT create_hypertable( 'conditions', 'day', chunk_time_interval => INTERVAL '1 day' ); create_hypertable ------------------------- (1,public,conditions,t) INSERT INTO conditions (day, city, temperature) VALUES ('2021-06-14', 'Moscow', 26), ('2021-06-15', 'Moscow', 22), ('2021-06-16', 'Moscow', 24), ('2021-06-17', 'Moscow', 24), ('2021-06-18', 'Moscow', 27), ('2021-06-19', 'Moscow', 28), ('2021-06-20', 'Moscow', 30), ('2021-06-21', 'Moscow', 31), ('2021-06-22', 'Moscow', 34), ('2021-06-23', 'Moscow', 34), ('2021-06-24', 'Moscow', 34), ('2021-06-25', 'Moscow', 32), ('2021-06-26', 'Moscow', 32), ('2021-06-27', 'Moscow', 31); -- timebucket_ng is deprecated and can not be used in new CAggs anymore. -- However, using this GUC the restriction can be lifted in debug builds -- to ensure the functionality can be tested. SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_summary_weekly WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT city, timescaledb_experimental.time_bucket_ng('7 days', day) AS bucket, MIN(temperature), MAX(temperature) FROM conditions GROUP BY city, bucket; NOTICE: refreshing continuous aggregate "conditions_summary_weekly" -- Reset GUC to check if the CAgg would also work in release builds RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT to_char(bucket, 'YYYY-MM-DD'), city, min, max FROM conditions_summary_weekly ORDER BY bucket; to_char | city | min | max ------------+--------+-----+----- 2021-06-12 | Moscow | 22 | 27 2021-06-19 | Moscow | 28 | 34 2021-06-26 | Moscow | 31 | 32 DROP TABLE conditions CASCADE; NOTICE: drop cascades to 3 other objects NOTICE: drop cascades to 2 other objects -- Make sure seconds, minutes, and hours can be used with caggs ('origin' is not -- currently supported in caggs). CREATE TABLE conditions( tstamp TIMESTAMP NOT NULL, city text NOT NULL, temperature INT NOT NULL); SELECT create_hypertable( 'conditions', 'tstamp', chunk_time_interval => INTERVAL '1 day' ); WARNING: column type "timestamp without time zone" used for "tstamp" does not follow best practices create_hypertable ------------------------- (3,public,conditions,t) INSERT INTO conditions (tstamp, city, temperature) VALUES ('2021-06-14 12:30:00', 'Moscow', 26), ('2021-06-14 12:30:10', 'Moscow', 22), ('2021-06-14 12:30:20', 'Moscow', 24), ('2021-06-14 12:30:30', 'Moscow', 24), ('2021-06-14 12:30:40', 'Moscow', 27), ('2021-06-14 12:30:50', 'Moscow', 28), ('2021-06-14 12:31:10', 'Moscow', 30), ('2021-06-14 12:31:20', 'Moscow', 31), ('2021-06-14 12:31:30', 'Moscow', 34), ('2021-06-14 12:31:40', 'Moscow', 34), ('2021-06-14 12:31:50', 'Moscow', 34), ('2021-06-14 12:32:00', 'Moscow', 32), ('2021-06-14 12:32:10', 'Moscow', 32), ('2021-06-14 12:32:20', 'Moscow', 31); SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_summary_30sec WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT city, timescaledb_experimental.time_bucket_ng('30 seconds', tstamp) AS bucket, MIN(temperature), MAX(temperature) FROM conditions GROUP BY city, bucket; NOTICE: refreshing continuous aggregate "conditions_summary_30sec" CREATE MATERIALIZED VIEW conditions_summary_1min WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 minute', tstamp) AS bucket, MIN(temperature), MAX(temperature) FROM conditions GROUP BY city, bucket; NOTICE: refreshing continuous aggregate "conditions_summary_1min" CREATE MATERIALIZED VIEW conditions_summary_1hour WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 hour', tstamp) AS bucket, MIN(temperature), MAX(temperature) FROM conditions GROUP BY city, bucket; NOTICE: refreshing continuous aggregate "conditions_summary_1hour" RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT city, to_char(bucket, 'YYYY-MM-DD HH:mi:ss'), min, max FROM conditions_summary_30sec ORDER BY bucket; city | to_char | min | max --------+---------------------+-----+----- Moscow | 2021-06-14 12:30:00 | 22 | 26 Moscow | 2021-06-14 12:30:30 | 24 | 28 Moscow | 2021-06-14 12:31:00 | 30 | 31 Moscow | 2021-06-14 12:31:30 | 34 | 34 Moscow | 2021-06-14 12:32:00 | 31 | 32 SELECT city, to_char(bucket, 'YYYY-MM-DD HH:mi:ss'), min, max FROM conditions_summary_1min ORDER BY bucket; city | to_char | min | max --------+---------------------+-----+----- Moscow | 2021-06-14 12:30:00 | 22 | 28 Moscow | 2021-06-14 12:31:00 | 30 | 34 Moscow | 2021-06-14 12:32:00 | 31 | 32 SELECT city, to_char(bucket, 'YYYY-MM-DD HH:mi:ss'), min, max FROM conditions_summary_1hour ORDER BY bucket; city | to_char | min | max --------+---------------------+-----+----- Moscow | 2021-06-14 12:00:00 | 22 | 34 DROP TABLE conditions CASCADE; NOTICE: drop cascades to 9 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_4_18_chunk NOTICE: drop cascades to table _timescaledb_internal._hyper_5_19_chunk NOTICE: drop cascades to table _timescaledb_internal._hyper_6_20_chunk -- Experimental functions using different schema for installation than PUBLIC \c :TEST_DBNAME :ROLE_SUPERUSER \set TEST_DBNAME_2 :TEST_DBNAME _2 CREATE DATABASE :TEST_DBNAME_2; \c :TEST_DBNAME_2 :ROLE_SUPERUSER CREATE SCHEMA test1; SET client_min_messages TO ERROR; CREATE EXTENSION timescaledb SCHEMA test1; CREATE TABLE conditions( tstamp TIMESTAMP NOT NULL, city text NOT NULL, temperature INT NOT NULL); SELECT test1.create_hypertable( 'conditions', 'tstamp', chunk_time_interval => INTERVAL '1 day' ); create_hypertable ------------------------- (1,public,conditions,t) SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_summary_monthly WITH (timescaledb.continuous) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 month', tstamp) AS bucket, MIN(temperature), MAX(temperature) FROM conditions GROUP BY city, bucket WITH NO DATA; RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; CREATE MATERIALIZED VIEW conditions_summary_yearly WITH (timescaledb.continuous) AS SELECT city, test1.time_bucket('1 year', tstamp) AS bucket, MIN(temperature), MAX(temperature) FROM conditions GROUP BY city, bucket WITH NO DATA; SELECT bucket_func, bucket_width, bucket_origin, bucket_timezone, bucket_fixed_width FROM _timescaledb_catalog.continuous_aggs_bucket_function ORDER BY 1; bucket_func | bucket_width | bucket_origin | bucket_timezone | bucket_fixed_width -------------------------------------------------------------------------------+--------------+---------------+-----------------+-------------------- test1.time_bucket(interval,timestamp without time zone) | @ 1 year | | | f timescaledb_experimental.time_bucket_ng(interval,timestamp without time zone) | @ 1 mon | | | f -- Try to toggle realtime feature on existing CAgg using timescaledb_experimental.time_bucket_ng ALTER MATERIALIZED VIEW conditions_summary_monthly SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW conditions_summary_monthly SET (timescaledb.materialized_only=true); \c :TEST_DBNAME :ROLE_SUPERUSER DROP DATABASE :TEST_DBNAME_2 WITH (FORCE); ================================================ FILE: tsl/test/expected/cagg_exp_origin.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. CREATE TABLE conditions( day DATE NOT NULL, city text NOT NULL, temperature INT NOT NULL); SELECT create_hypertable( 'conditions', 'day', chunk_time_interval => INTERVAL '1 day' ); create_hypertable ------------------------- (1,public,conditions,t) INSERT INTO conditions (day, city, temperature) VALUES ('2021-06-14', 'Moscow', 26), ('2021-06-15', 'Moscow', 22), ('2021-06-16', 'Moscow', 24), ('2021-06-17', 'Moscow', 24), ('2021-06-18', 'Moscow', 27), ('2021-06-19', 'Moscow', 28), ('2021-06-20', 'Moscow', 30), ('2021-06-21', 'Moscow', 31), ('2021-06-22', 'Moscow', 34), ('2021-06-23', 'Moscow', 34), ('2021-06-24', 'Moscow', 34), ('2021-06-25', 'Moscow', 32), ('2021-06-26', 'Moscow', 32), ('2021-06-27', 'Moscow', 31); \set ON_ERROR_STOP 0 -- timebucket_ng is deprecated and can not be used in new CAggs anymore. -- However, using this GUC the restriction can be lifted in debug builds -- to ensure the functionality can be tested. SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; -- Make sure 'infinity' can't be specified as an origin CREATE MATERIALIZED VIEW conditions_summary_weekly WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT city, timescaledb_experimental.time_bucket_ng('7 days', day, 'infinity' :: date) AS bucket, MIN(temperature), MAX(temperature) FROM conditions GROUP BY city, bucket WITH NO DATA; ERROR: invalid origin value: infinity -- Make sure buckets like '1 months 15 days" (fixed+variable-sized) are not allowed CREATE MATERIALIZED VIEW conditions_summary_weekly WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 month 15 days', day, '2021-06-01') AS bucket, MIN(temperature), MAX(temperature) FROM conditions GROUP BY city, bucket WITH NO DATA; ERROR: invalid interval specified \set ON_ERROR_STOP 1 CREATE MATERIALIZED VIEW conditions_summary_weekly WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT city, timescaledb_experimental.time_bucket_ng('7 days', day, '2000-01-03' :: date) AS bucket, MIN(temperature), MAX(temperature) FROM conditions GROUP BY city, bucket WITH NO DATA; -- Reset GUC to check if the CAgg would also work in release builds RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT to_char(bucket, 'YYYY-MM-DD'), city, min, max FROM conditions_summary_weekly ORDER BY bucket; to_char | city | min | max ---------+------+-----+----- SELECT mat_hypertable_id AS cagg_id, raw_hypertable_id AS ht_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'conditions_summary_weekly' \gset -- Make sure truncating of the refresh window works \set ON_ERROR_STOP 0 CALL refresh_continuous_aggregate('conditions_summary_weekly', '2021-06-14', '2021-06-20'); ERROR: refresh window too small \set ON_ERROR_STOP 1 -- Make sure refreshing works CALL refresh_continuous_aggregate('conditions_summary_weekly', '2021-06-14', '2021-06-21'); SELECT city, to_char(bucket, 'YYYY-MM-DD') AS week, min, max FROM conditions_summary_weekly ORDER BY week, city; city | week | min | max --------+------------+-----+----- Moscow | 2021-06-14 | 22 | 30 -- Check the invalidation threshold SELECT _timescaledb_functions.to_timestamp(watermark) at time zone 'UTC' FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id = :ht_id; timezone -------------------------- Mon Jun 21 00:00:00 2021 -- Add some dummy data for two more weeks and call refresh (no invalidations test case) INSERT INTO conditions (day, city, temperature) SELECT ts :: date, city, row_number() OVER () FROM generate_series('2021-06-28' :: date, '2021-07-11', '1 day') as ts, unnest(array['Moscow', 'Berlin']) as city; -- Double check generated data SELECT to_char(day, 'YYYY-MM-DD'), city, temperature FROM conditions WHERE day >= '2021-06-28' ORDER BY city DESC, day; to_char | city | temperature ------------+--------+------------- 2021-06-28 | Moscow | 1 2021-06-29 | Moscow | 2 2021-06-30 | Moscow | 3 2021-07-01 | Moscow | 4 2021-07-02 | Moscow | 5 2021-07-03 | Moscow | 6 2021-07-04 | Moscow | 7 2021-07-05 | Moscow | 8 2021-07-06 | Moscow | 9 2021-07-07 | Moscow | 10 2021-07-08 | Moscow | 11 2021-07-09 | Moscow | 12 2021-07-10 | Moscow | 13 2021-07-11 | Moscow | 14 2021-06-28 | Berlin | 15 2021-06-29 | Berlin | 16 2021-06-30 | Berlin | 17 2021-07-01 | Berlin | 18 2021-07-02 | Berlin | 19 2021-07-03 | Berlin | 20 2021-07-04 | Berlin | 21 2021-07-05 | Berlin | 22 2021-07-06 | Berlin | 23 2021-07-07 | Berlin | 24 2021-07-08 | Berlin | 25 2021-07-09 | Berlin | 26 2021-07-10 | Berlin | 27 2021-07-11 | Berlin | 28 -- Make sure the invalidation threshold was unaffected SELECT _timescaledb_functions.to_timestamp(watermark) at time zone 'UTC' FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id = :ht_id; timezone -------------------------- Mon Jun 21 00:00:00 2021 -- Make sure the invalidation log is empty SELECT _timescaledb_functions.to_timestamp(lowest_modified_value) AS lowest, _timescaledb_functions.to_timestamp(greatest_modified_value) AS greatest FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log WHERE hypertable_id = :ht_id; lowest | greatest --------+---------- -- Call refresh CALL refresh_continuous_aggregate('conditions_summary_weekly', '2021-06-28', '2021-07-12'); SELECT city, to_char(bucket, 'YYYY-MM-DD') AS week, min, max FROM conditions_summary_weekly ORDER BY week, city; city | week | min | max --------+------------+-----+----- Moscow | 2021-06-14 | 22 | 30 Berlin | 2021-06-28 | 15 | 21 Moscow | 2021-06-28 | 1 | 7 Berlin | 2021-07-05 | 22 | 28 Moscow | 2021-07-05 | 8 | 14 -- Make sure the invalidation threshold has changed SELECT _timescaledb_functions.to_timestamp(watermark) at time zone 'UTC' FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id = :ht_id; timezone -------------------------- Mon Jul 12 00:00:00 2021 -- Check if CREATE MATERIALIZED VIEW ... WITH DATA works. -- Use monthly buckets this time and specify June 2000 as an origin. SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_summary_monthly WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 month', day, '2000-06-01' :: date) AS bucket, MIN(temperature), MAX(temperature) FROM conditions GROUP BY city, bucket; NOTICE: refreshing continuous aggregate "conditions_summary_monthly" RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary_monthly ORDER BY month, city; city | month | min | max --------+------------+-----+----- Berlin | 2021-06-01 | 15 | 17 Moscow | 2021-06-01 | 1 | 34 Berlin | 2021-07-01 | 18 | 28 Moscow | 2021-07-01 | 4 | 14 -- Check the invalidation. -- Step 1/2. Insert some more data , do a refresh and make sure that the -- invalidation log is empty. INSERT INTO conditions (day, city, temperature) SELECT ts :: date, city, row_number() OVER () FROM generate_series('2021-09-01' :: date, '2021-09-15', '1 day') as ts, unnest(array['Moscow', 'Berlin']) as city; CALL refresh_continuous_aggregate('conditions_summary_monthly', '2021-09-01', '2021-10-01'); SELECT _timescaledb_functions.to_timestamp(lowest_modified_value) AS lowest, _timescaledb_functions.to_timestamp(greatest_modified_value) AS greatest FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log WHERE hypertable_id = :ht_id; lowest | greatest --------+---------- SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary_monthly ORDER BY month, city; city | month | min | max --------+------------+-----+----- Berlin | 2021-06-01 | 15 | 17 Moscow | 2021-06-01 | 1 | 34 Berlin | 2021-07-01 | 18 | 28 Moscow | 2021-07-01 | 4 | 14 Berlin | 2021-09-01 | 16 | 30 Moscow | 2021-09-01 | 1 | 15 -- Step 2/2. Add more data below the invalidation threshold, make sure that the -- invalidation log is not empty, then do a refresh. INSERT INTO conditions (day, city, temperature) SELECT ts :: date, city, (CASE WHEN city = 'Moscow' THEN -40 ELSE 40 END) FROM generate_series('2021-09-16' :: date, '2021-09-30', '1 day') as ts, unnest(array['Moscow', 'Berlin']) as city; SELECT _timescaledb_functions.to_timestamp(lowest_modified_value) at time zone 'UTC' AS lowest, _timescaledb_functions.to_timestamp(greatest_modified_value) at time zone 'UTC' AS greatest FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log WHERE hypertable_id = :ht_id; lowest | greatest --------------------------+-------------------------- Thu Sep 16 00:00:00 2021 | Thu Sep 16 00:00:00 2021 Fri Sep 17 00:00:00 2021 | Fri Sep 17 00:00:00 2021 Sat Sep 18 00:00:00 2021 | Sat Sep 18 00:00:00 2021 Sun Sep 19 00:00:00 2021 | Sun Sep 19 00:00:00 2021 Mon Sep 20 00:00:00 2021 | Mon Sep 20 00:00:00 2021 Tue Sep 21 00:00:00 2021 | Tue Sep 21 00:00:00 2021 Wed Sep 22 00:00:00 2021 | Wed Sep 22 00:00:00 2021 Thu Sep 23 00:00:00 2021 | Thu Sep 23 00:00:00 2021 Fri Sep 24 00:00:00 2021 | Fri Sep 24 00:00:00 2021 Sat Sep 25 00:00:00 2021 | Sat Sep 25 00:00:00 2021 Sun Sep 26 00:00:00 2021 | Sun Sep 26 00:00:00 2021 Mon Sep 27 00:00:00 2021 | Mon Sep 27 00:00:00 2021 Tue Sep 28 00:00:00 2021 | Tue Sep 28 00:00:00 2021 Wed Sep 29 00:00:00 2021 | Wed Sep 29 00:00:00 2021 Thu Sep 30 00:00:00 2021 | Thu Sep 30 00:00:00 2021 CALL refresh_continuous_aggregate('conditions_summary_monthly', '2021-09-01', '2021-10-01'); SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary_monthly ORDER BY month, city; city | month | min | max --------+------------+-----+----- Berlin | 2021-06-01 | 15 | 17 Moscow | 2021-06-01 | 1 | 34 Berlin | 2021-07-01 | 18 | 28 Moscow | 2021-07-01 | 4 | 14 Berlin | 2021-09-01 | 16 | 40 Moscow | 2021-09-01 | -40 | 15 SELECT _timescaledb_functions.to_timestamp(lowest_modified_value) AS lowest, _timescaledb_functions.to_timestamp(greatest_modified_value) AS greatest FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log WHERE hypertable_id = :ht_id; lowest | greatest --------+---------- -- Create a real-time aggregate with custom origin - June 2000 SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_summary_rt WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 month', day, '2000-06-01' :: date) AS bucket, MIN(temperature), MAX(temperature) FROM conditions GROUP BY city, bucket; NOTICE: refreshing continuous aggregate "conditions_summary_rt" RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary_rt ORDER BY month, city; city | month | min | max --------+------------+-----+----- Berlin | 2021-06-01 | 15 | 17 Moscow | 2021-06-01 | 1 | 34 Berlin | 2021-07-01 | 18 | 28 Moscow | 2021-07-01 | 4 | 14 Berlin | 2021-09-01 | 16 | 40 Moscow | 2021-09-01 | -40 | 15 -- Add some data to the hypertable and make sure it is visible in the cagg INSERT INTO conditions (day, city, temperature) VALUES ('2021-10-01', 'Moscow', 1), ('2021-10-02', 'Moscow', 2), ('2021-10-03', 'Moscow', 3), ('2021-10-04', 'Moscow', 4), ('2021-10-01', 'Berlin', 5), ('2021-10-02', 'Berlin', 6), ('2021-10-03', 'Berlin', 7), ('2021-10-04', 'Berlin', 8); SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary_rt ORDER BY month, city; city | month | min | max --------+------------+-----+----- Berlin | 2021-06-01 | 15 | 17 Moscow | 2021-06-01 | 1 | 34 Berlin | 2021-07-01 | 18 | 28 Moscow | 2021-07-01 | 4 | 14 Berlin | 2021-09-01 | 16 | 40 Moscow | 2021-09-01 | -40 | 15 Berlin | 2021-10-01 | 5 | 8 Moscow | 2021-10-01 | 1 | 4 -- Refresh the cagg and make sure that the result of SELECT query didn't change CALL refresh_continuous_aggregate('conditions_summary_rt', '2021-10-01', '2021-11-01'); SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary_rt ORDER BY month, city; city | month | min | max --------+------------+-----+----- Berlin | 2021-06-01 | 15 | 17 Moscow | 2021-06-01 | 1 | 34 Berlin | 2021-07-01 | 18 | 28 Moscow | 2021-07-01 | 4 | 14 Berlin | 2021-09-01 | 16 | 40 Moscow | 2021-09-01 | -40 | 15 Berlin | 2021-10-01 | 5 | 8 Moscow | 2021-10-01 | 1 | 4 -- Add some more data, enable compression, compress the chunks and repeat the test INSERT INTO conditions (day, city, temperature) VALUES ('2021-11-01', 'Moscow', 11), ('2021-11-02', 'Moscow', 12), ('2021-11-03', 'Moscow', 13), ('2021-11-04', 'Moscow', 14), ('2021-11-01', 'Berlin', 15), ('2021-11-02', 'Berlin', 16), ('2021-11-03', 'Berlin', 17), ('2021-11-04', 'Berlin', 18); ALTER TABLE conditions SET ( timescaledb.compress, timescaledb.compress_segmentby = 'city' ); SELECT compress_chunk(ch) FROM show_chunks('conditions') AS ch; compress_chunk ----------------------------------------- _timescaledb_internal._hyper_1_1_chunk _timescaledb_internal._hyper_1_2_chunk _timescaledb_internal._hyper_1_3_chunk _timescaledb_internal._hyper_1_4_chunk _timescaledb_internal._hyper_1_5_chunk _timescaledb_internal._hyper_1_6_chunk _timescaledb_internal._hyper_1_7_chunk _timescaledb_internal._hyper_1_8_chunk _timescaledb_internal._hyper_1_9_chunk _timescaledb_internal._hyper_1_10_chunk _timescaledb_internal._hyper_1_11_chunk _timescaledb_internal._hyper_1_12_chunk _timescaledb_internal._hyper_1_13_chunk _timescaledb_internal._hyper_1_14_chunk _timescaledb_internal._hyper_1_16_chunk _timescaledb_internal._hyper_1_17_chunk _timescaledb_internal._hyper_1_18_chunk _timescaledb_internal._hyper_1_19_chunk _timescaledb_internal._hyper_1_20_chunk _timescaledb_internal._hyper_1_21_chunk _timescaledb_internal._hyper_1_22_chunk _timescaledb_internal._hyper_1_23_chunk _timescaledb_internal._hyper_1_24_chunk _timescaledb_internal._hyper_1_25_chunk _timescaledb_internal._hyper_1_26_chunk _timescaledb_internal._hyper_1_27_chunk _timescaledb_internal._hyper_1_28_chunk _timescaledb_internal._hyper_1_29_chunk _timescaledb_internal._hyper_1_34_chunk _timescaledb_internal._hyper_1_35_chunk _timescaledb_internal._hyper_1_36_chunk _timescaledb_internal._hyper_1_37_chunk _timescaledb_internal._hyper_1_38_chunk _timescaledb_internal._hyper_1_39_chunk _timescaledb_internal._hyper_1_40_chunk _timescaledb_internal._hyper_1_41_chunk _timescaledb_internal._hyper_1_42_chunk _timescaledb_internal._hyper_1_43_chunk _timescaledb_internal._hyper_1_44_chunk _timescaledb_internal._hyper_1_45_chunk _timescaledb_internal._hyper_1_46_chunk _timescaledb_internal._hyper_1_47_chunk _timescaledb_internal._hyper_1_48_chunk _timescaledb_internal._hyper_1_50_chunk _timescaledb_internal._hyper_1_51_chunk _timescaledb_internal._hyper_1_52_chunk _timescaledb_internal._hyper_1_53_chunk _timescaledb_internal._hyper_1_54_chunk _timescaledb_internal._hyper_1_55_chunk _timescaledb_internal._hyper_1_56_chunk _timescaledb_internal._hyper_1_57_chunk _timescaledb_internal._hyper_1_58_chunk _timescaledb_internal._hyper_1_59_chunk _timescaledb_internal._hyper_1_60_chunk _timescaledb_internal._hyper_1_61_chunk _timescaledb_internal._hyper_1_62_chunk _timescaledb_internal._hyper_1_63_chunk _timescaledb_internal._hyper_1_64_chunk _timescaledb_internal._hyper_1_68_chunk _timescaledb_internal._hyper_1_69_chunk _timescaledb_internal._hyper_1_70_chunk _timescaledb_internal._hyper_1_71_chunk _timescaledb_internal._hyper_1_73_chunk _timescaledb_internal._hyper_1_74_chunk _timescaledb_internal._hyper_1_75_chunk _timescaledb_internal._hyper_1_76_chunk -- Data for 2021-11 is seen because the cagg is real-time SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary_rt ORDER BY month, city; city | month | min | max --------+------------+-----+----- Berlin | 2021-06-01 | 15 | 17 Moscow | 2021-06-01 | 1 | 34 Berlin | 2021-07-01 | 18 | 28 Moscow | 2021-07-01 | 4 | 14 Berlin | 2021-09-01 | 16 | 40 Moscow | 2021-09-01 | -40 | 15 Berlin | 2021-10-01 | 5 | 8 Moscow | 2021-10-01 | 1 | 4 Berlin | 2021-11-01 | 15 | 18 Moscow | 2021-11-01 | 11 | 14 CALL refresh_continuous_aggregate('conditions_summary_rt', '2021-11-01', '2021-12-01'); -- Data for 2021-11 is seen because the cagg was refreshed SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary_rt ORDER BY month, city; city | month | min | max --------+------------+-----+----- Berlin | 2021-06-01 | 15 | 17 Moscow | 2021-06-01 | 1 | 34 Berlin | 2021-07-01 | 18 | 28 Moscow | 2021-07-01 | 4 | 14 Berlin | 2021-09-01 | 16 | 40 Moscow | 2021-09-01 | -40 | 15 Berlin | 2021-10-01 | 5 | 8 Moscow | 2021-10-01 | 1 | 4 Berlin | 2021-11-01 | 15 | 18 Moscow | 2021-11-01 | 11 | 14 -- Clean up DROP TABLE conditions CASCADE; NOTICE: drop cascades to 7 other objects NOTICE: drop cascades to 3 other objects NOTICE: drop cascades to 3 other objects NOTICE: drop cascades to 5 other objects -- Test the specific code path of creating a CAGG on top of empty hypertable. CREATE TABLE conditions_empty( day DATE NOT NULL, city text NOT NULL, temperature INT NOT NULL); SELECT create_hypertable( 'conditions_empty', 'day', chunk_time_interval => INTERVAL '1 day' ); create_hypertable ------------------------------- (6,public,conditions_empty,t) SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_summary_empty WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 month', day, '2005-02-01') AS bucket, MIN(temperature), MAX(temperature) FROM conditions_empty GROUP BY city, bucket; NOTICE: continuous aggregate "conditions_summary_empty" is already up-to-date RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary_empty ORDER BY month, city; city | month | min | max ------+-------+-----+----- -- The test above changes the record that gets added to the invalidation log -- for an empty table. Make sure it doesn't have any unintended side-effects -- and the refreshing works as expected. INSERT INTO conditions_empty (day, city, temperature) VALUES ('2021-06-14', 'Moscow', 26), ('2021-06-15', 'Moscow', 22), ('2021-06-16', 'Moscow', 24), ('2021-06-17', 'Moscow', 24), ('2021-06-18', 'Moscow', 27), ('2021-06-19', 'Moscow', 28), ('2021-06-20', 'Moscow', 30), ('2021-06-21', 'Moscow', 31), ('2021-06-22', 'Moscow', 34), ('2021-06-23', 'Moscow', 34), ('2021-06-24', 'Moscow', 34), ('2021-06-25', 'Moscow', 32), ('2021-06-26', 'Moscow', 32), ('2021-06-27', 'Moscow', 31); CALL refresh_continuous_aggregate('conditions_summary_empty', '2021-06-01', '2021-07-01'); SELECT city, to_char(bucket, 'YYYY-MM-DD') AS month, min, max FROM conditions_summary_empty ORDER BY month, city; city | month | min | max --------+------------+-----+----- Moscow | 2021-06-01 | 22 | 34 -- Clean up DROP TABLE conditions_empty CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_7_158_chunk -- Make sure add_continuous_aggregate_policy() works CREATE TABLE conditions_policy( day DATE NOT NULL, city text NOT NULL, temperature INT NOT NULL); SELECT create_hypertable( 'conditions_policy', 'day', chunk_time_interval => INTERVAL '1 day' ); create_hypertable -------------------------------- (8,public,conditions_policy,t) INSERT INTO conditions_policy (day, city, temperature) VALUES ('2021-06-14', 'Moscow', 26), ('2021-06-15', 'Moscow', 22), ('2021-06-16', 'Moscow', 24), ('2021-06-17', 'Moscow', 24), ('2021-06-18', 'Moscow', 27), ('2021-06-19', 'Moscow', 28), ('2021-06-20', 'Moscow', 30), ('2021-06-21', 'Moscow', 31), ('2021-06-22', 'Moscow', 34), ('2021-06-23', 'Moscow', 34), ('2021-06-24', 'Moscow', 34), ('2021-06-25', 'Moscow', 32), ('2021-06-26', 'Moscow', 32), ('2021-06-27', 'Moscow', 31); SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_summary_policy WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 month', day, '2005-03-01') AS bucket, MIN(temperature), MAX(temperature) FROM conditions_policy GROUP BY city, bucket; NOTICE: refreshing continuous aggregate "conditions_summary_policy" RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT * FROM conditions_summary_policy; city | bucket | min | max --------+------------+-----+----- Moscow | 06-01-2021 | 22 | 34 \set ON_ERROR_STOP 0 -- Check for "policy refresh window too small" error SELECT add_continuous_aggregate_policy('conditions_summary_policy', -- Historically, 1 month is just a synonym to 30 days here. -- See interval_to_int64() and interval_to_int128(). start_offset => INTERVAL '2 months', end_offset => INTERVAL '1 day', schedule_interval => INTERVAL '1 hour'); ERROR: policy refresh window too small \set ON_ERROR_STOP 1 SELECT add_continuous_aggregate_policy('conditions_summary_policy', start_offset => INTERVAL '65 days', end_offset => INTERVAL '1 day', schedule_interval => INTERVAL '1 hour'); add_continuous_aggregate_policy --------------------------------- 1000 -- Clean up DROP TABLE conditions_policy CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_9_173_chunk -- Make sure CAGGs with custom origin work for timestamp type CREATE TABLE conditions_timestamp( tstamp TIMESTAMP NOT NULL, city TEXT NOT NULL, temperature INT NOT NULL); SELECT create_hypertable( 'conditions_timestamp', 'tstamp', chunk_time_interval => INTERVAL '1 day' ); WARNING: column type "timestamp without time zone" used for "tstamp" does not follow best practices create_hypertable ------------------------------------ (10,public,conditions_timestamp,t) SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_summary_timestamp WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT city, timescaledb_experimental.time_bucket_ng('12 hours', tstamp, '2000-06-01 12:00:00') AS bucket, MIN(temperature), MAX(temperature) FROM conditions_timestamp GROUP BY city, bucket; NOTICE: continuous aggregate "conditions_summary_timestamp" is already up-to-date RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT city, to_char(bucket, 'YYYY-MM-DD HH24:MI:SS') AS b, min, max FROM conditions_summary_timestamp ORDER BY b, city; city | b | min | max ------+---+-----+----- -- Add some data to the hypertable and make sure it is visible in the cagg INSERT INTO conditions_timestamp(tstamp, city, temperature) SELECT ts, city, (CASE WHEN city = 'Moscow' THEN 20000 ELSE 10000 END) + date_part('day', ts)*100 + date_part('hour', ts) FROM generate_series('2010-01-01 00:00:00' :: timestamp, '2010-01-02 00:00:00' :: timestamp - interval '1 hour', '1 hour') as ts, unnest(array['Moscow', 'Berlin']) as city; SELECT city, to_char(bucket, 'YYYY-MM-DD HH24:MI:SS') AS b, min, max FROM conditions_summary_timestamp ORDER BY b, city; city | b | min | max --------+---------------------+-------+------- Berlin | 2010-01-01 00:00:00 | 10100 | 10111 Moscow | 2010-01-01 00:00:00 | 20100 | 20111 Berlin | 2010-01-01 12:00:00 | 10112 | 10123 Moscow | 2010-01-01 12:00:00 | 20112 | 20123 -- Refresh the cagg and make sure that the result of SELECT query didn't change CALL refresh_continuous_aggregate('conditions_summary_timestamp', '2010-01-01 00:00:00', '2010-01-02 00:00:00'); SELECT city, to_char(bucket, 'YYYY-MM-DD HH24:MI:SS') AS b, min, max FROM conditions_summary_timestamp ORDER BY b, city; city | b | min | max --------+---------------------+-------+------- Berlin | 2010-01-01 00:00:00 | 10100 | 10111 Moscow | 2010-01-01 00:00:00 | 20100 | 20111 Berlin | 2010-01-01 12:00:00 | 10112 | 10123 Moscow | 2010-01-01 12:00:00 | 20112 | 20123 -- Add some more data, enable compression, compress the chunks and repeat the test INSERT INTO conditions_timestamp(tstamp, city, temperature) SELECT ts, city, (CASE WHEN city = 'Moscow' THEN 20000 ELSE 10000 END) + date_part('day', ts)*100 + date_part('hour', ts) FROM generate_series('2010-01-02 00:00:00' :: timestamp, '2010-01-03 00:00:00' :: timestamp - interval '1 hour', '1 hour') as ts, unnest(array['Moscow', 'Berlin']) as city; ALTER TABLE conditions_timestamp SET ( timescaledb.compress, timescaledb.compress_segmentby = 'city' ); SELECT compress_chunk(ch) FROM show_chunks('conditions_timestamp') AS ch; compress_chunk ------------------------------------------- _timescaledb_internal._hyper_10_174_chunk _timescaledb_internal._hyper_10_176_chunk -- New data is seen because the cagg is real-time SELECT city, to_char(bucket, 'YYYY-MM-DD HH24:MI:SS') AS b, min, max FROM conditions_summary_timestamp ORDER BY b, city; city | b | min | max --------+---------------------+-------+------- Berlin | 2010-01-01 00:00:00 | 10100 | 10111 Moscow | 2010-01-01 00:00:00 | 20100 | 20111 Berlin | 2010-01-01 12:00:00 | 10112 | 10123 Moscow | 2010-01-01 12:00:00 | 20112 | 20123 Berlin | 2010-01-02 00:00:00 | 10200 | 10211 Moscow | 2010-01-02 00:00:00 | 20200 | 20211 Berlin | 2010-01-02 12:00:00 | 10212 | 10223 Moscow | 2010-01-02 12:00:00 | 20212 | 20223 CALL refresh_continuous_aggregate('conditions_summary_timestamp', '2010-01-02 00:00:00', '2010-01-03 00:00:00'); -- New data is seen because the cagg was refreshed SELECT city, to_char(bucket, 'YYYY-MM-DD HH24:MI:SS') AS b, min, max FROM conditions_summary_timestamp ORDER BY b, city; city | b | min | max --------+---------------------+-------+------- Berlin | 2010-01-01 00:00:00 | 10100 | 10111 Moscow | 2010-01-01 00:00:00 | 20100 | 20111 Berlin | 2010-01-01 12:00:00 | 10112 | 10123 Moscow | 2010-01-01 12:00:00 | 20112 | 20123 Berlin | 2010-01-02 00:00:00 | 10200 | 10211 Moscow | 2010-01-02 00:00:00 | 20200 | 20211 Berlin | 2010-01-02 12:00:00 | 10212 | 10223 Moscow | 2010-01-02 12:00:00 | 20212 | 20223 -- Add a refresh policy SELECT add_continuous_aggregate_policy('conditions_summary_timestamp', start_offset => INTERVAL '25 hours', end_offset => INTERVAL '1 hour', schedule_interval => INTERVAL '30 minutes'); add_continuous_aggregate_policy --------------------------------- 1001 -- Clean up DROP TABLE conditions_timestamp CASCADE; NOTICE: drop cascades to 3 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_11_175_chunk -- Make sure CAGGs with custom origin work for timestamptz type CREATE TABLE conditions_timestamptz( tstamp TIMESTAMPTZ NOT NULL, city TEXT NOT NULL, temperature INT NOT NULL); SELECT create_hypertable( 'conditions_timestamptz', 'tstamp', chunk_time_interval => INTERVAL '1 day' ); create_hypertable -------------------------------------- (13,public,conditions_timestamptz,t) -- Add some data to the hypertable and make sure it is visible in the cagg INSERT INTO conditions_timestamptz(tstamp, city, temperature) SELECT ts, city, (CASE WHEN city = 'Moscow' THEN 20000 ELSE 10000 END) + date_part('day', ts at time zone 'MSK')*100 + date_part('hour', ts at time zone 'MSK') FROM generate_series('2022-01-01 00:00:00 MSK' :: timestamptz, '2022-01-02 00:00:00 MSK' :: timestamptz - interval '1 hour', '1 hour') as ts, unnest(array['Moscow', 'Berlin']) as city; \set ON_ERROR_STOP 0 SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; -- For monthly buckets origin should be the first day of the month in given timezone -- 2020-06-02 00:00:00 MSK == 2020-06-01 21:00:00 UTC CREATE MATERIALIZED VIEW conditions_summary_timestamptz WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 month', tstamp, '2020-06-02 00:00:00 MSK', 'Europe/Moscow') AS bucket, MIN(temperature), MAX(temperature) FROM conditions_timestamptz GROUP BY city, bucket; ERROR: origin must be the first day of the month -- Make sure buckets like '1 months 15 days" (fixed+variable-sized) are not allowed CREATE MATERIALIZED VIEW conditions_summary_timestamptz WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 month 15 days', tstamp, '2020-06-01 00:00:00 MSK', 'Europe/Moscow') AS bucket, MIN(temperature), MAX(temperature) FROM conditions_timestamptz GROUP BY city, bucket; ERROR: invalid interval specified \set ON_ERROR_STOP 1 CREATE MATERIALIZED VIEW conditions_summary_timestamptz WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT city, timescaledb_experimental.time_bucket_ng('12 hours', tstamp, '2020-06-01 12:00:00 MSK', 'Europe/Moscow') AS bucket, MIN(temperature), MAX(temperature) FROM conditions_timestamptz GROUP BY city, bucket; NOTICE: refreshing continuous aggregate "conditions_summary_timestamptz" RESET timescaledb.debug_allow_cagg_with_deprecated_func; -- Make sure the origin is saved in the catalog table SELECT mat_hypertable_id AS cagg_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'conditions_summary_timestamptz' \gset SELECT bucket_func, bucket_width, bucket_origin, bucket_timezone, bucket_fixed_width FROM _timescaledb_catalog.continuous_aggs_bucket_function WHERE mat_hypertable_id = :cagg_id; bucket_func | bucket_width | bucket_origin | bucket_timezone | bucket_fixed_width ---------------------------------------------------------------------------------------------------------------------+--------------+------------------------------+-----------------+-------------------- timescaledb_experimental.time_bucket_ng(interval,timestamp with time zone,timestamp with time zone,pg_catalog.text) | @ 12 hours | Mon Jun 01 02:00:00 2020 PDT | Europe/Moscow | f SELECT city, to_char(bucket at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') AS b, min, max FROM conditions_summary_timestamptz ORDER BY b, city; city | b | min | max --------+---------------------+-------+------- Berlin | 2022-01-01 00:00:00 | 10100 | 10111 Moscow | 2022-01-01 00:00:00 | 20100 | 20111 Berlin | 2022-01-01 12:00:00 | 10112 | 10123 Moscow | 2022-01-01 12:00:00 | 20112 | 20123 -- Check the data SELECT to_char(tstamp at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') AS ts, city, temperature FROM conditions_timestamptz ORDER BY ts, city; ts | city | temperature ---------------------+--------+------------- 2022-01-01 00:00:00 | Berlin | 10100 2022-01-01 00:00:00 | Moscow | 20100 2022-01-01 01:00:00 | Berlin | 10101 2022-01-01 01:00:00 | Moscow | 20101 2022-01-01 02:00:00 | Berlin | 10102 2022-01-01 02:00:00 | Moscow | 20102 2022-01-01 03:00:00 | Berlin | 10103 2022-01-01 03:00:00 | Moscow | 20103 2022-01-01 04:00:00 | Berlin | 10104 2022-01-01 04:00:00 | Moscow | 20104 2022-01-01 05:00:00 | Berlin | 10105 2022-01-01 05:00:00 | Moscow | 20105 2022-01-01 06:00:00 | Berlin | 10106 2022-01-01 06:00:00 | Moscow | 20106 2022-01-01 07:00:00 | Berlin | 10107 2022-01-01 07:00:00 | Moscow | 20107 2022-01-01 08:00:00 | Berlin | 10108 2022-01-01 08:00:00 | Moscow | 20108 2022-01-01 09:00:00 | Berlin | 10109 2022-01-01 09:00:00 | Moscow | 20109 2022-01-01 10:00:00 | Berlin | 10110 2022-01-01 10:00:00 | Moscow | 20110 2022-01-01 11:00:00 | Berlin | 10111 2022-01-01 11:00:00 | Moscow | 20111 2022-01-01 12:00:00 | Berlin | 10112 2022-01-01 12:00:00 | Moscow | 20112 2022-01-01 13:00:00 | Berlin | 10113 2022-01-01 13:00:00 | Moscow | 20113 2022-01-01 14:00:00 | Berlin | 10114 2022-01-01 14:00:00 | Moscow | 20114 2022-01-01 15:00:00 | Berlin | 10115 2022-01-01 15:00:00 | Moscow | 20115 2022-01-01 16:00:00 | Berlin | 10116 2022-01-01 16:00:00 | Moscow | 20116 2022-01-01 17:00:00 | Berlin | 10117 2022-01-01 17:00:00 | Moscow | 20117 2022-01-01 18:00:00 | Berlin | 10118 2022-01-01 18:00:00 | Moscow | 20118 2022-01-01 19:00:00 | Berlin | 10119 2022-01-01 19:00:00 | Moscow | 20119 2022-01-01 20:00:00 | Berlin | 10120 2022-01-01 20:00:00 | Moscow | 20120 2022-01-01 21:00:00 | Berlin | 10121 2022-01-01 21:00:00 | Moscow | 20121 2022-01-01 22:00:00 | Berlin | 10122 2022-01-01 22:00:00 | Moscow | 20122 2022-01-01 23:00:00 | Berlin | 10123 2022-01-01 23:00:00 | Moscow | 20123 SELECT city, to_char(bucket at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') AS b, min, max FROM conditions_summary_timestamptz ORDER BY b, city; city | b | min | max --------+---------------------+-------+------- Berlin | 2022-01-01 00:00:00 | 10100 | 10111 Moscow | 2022-01-01 00:00:00 | 20100 | 20111 Berlin | 2022-01-01 12:00:00 | 10112 | 10123 Moscow | 2022-01-01 12:00:00 | 20112 | 20123 -- Refresh the cagg and make sure that the result of SELECT query didn't change CALL refresh_continuous_aggregate('conditions_summary_timestamptz', '2022-01-01 00:00:00 MSK', '2022-01-02 00:00:00 MSK'); NOTICE: continuous aggregate "conditions_summary_timestamptz" is already up-to-date SELECT city, to_char(bucket at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') AS b, min, max FROM conditions_summary_timestamptz ORDER BY b, city; city | b | min | max --------+---------------------+-------+------- Berlin | 2022-01-01 00:00:00 | 10100 | 10111 Moscow | 2022-01-01 00:00:00 | 20100 | 20111 Berlin | 2022-01-01 12:00:00 | 10112 | 10123 Moscow | 2022-01-01 12:00:00 | 20112 | 20123 -- Add some more data, enable compression, compress the chunks and repeat the test INSERT INTO conditions_timestamptz(tstamp, city, temperature) SELECT ts, city, (CASE WHEN city = 'Moscow' THEN 20000 ELSE 10000 END) + date_part('day', ts at time zone 'MSK')*100 + date_part('hour', ts at time zone 'MSK') FROM generate_series('2022-01-02 00:00:00 MSK' :: timestamptz, '2022-01-03 00:00:00 MSK' :: timestamptz - interval '1 hour', '1 hour') as ts, unnest(array['Moscow', 'Berlin']) as city; ALTER TABLE conditions_timestamptz SET ( timescaledb.compress, timescaledb.compress_segmentby = 'city' ); SELECT compress_chunk(ch) FROM show_chunks('conditions_timestamptz') AS ch; compress_chunk ------------------------------------------- _timescaledb_internal._hyper_13_179_chunk _timescaledb_internal._hyper_13_180_chunk _timescaledb_internal._hyper_13_182_chunk -- New data is seen because the cagg is real-time SELECT city, to_char(bucket at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') AS b, min, max FROM conditions_summary_timestamptz ORDER BY b, city; city | b | min | max --------+---------------------+-------+------- Berlin | 2022-01-01 00:00:00 | 10100 | 10111 Moscow | 2022-01-01 00:00:00 | 20100 | 20111 Berlin | 2022-01-01 12:00:00 | 10112 | 10123 Moscow | 2022-01-01 12:00:00 | 20112 | 20123 Berlin | 2022-01-02 00:00:00 | 10200 | 10211 Moscow | 2022-01-02 00:00:00 | 20200 | 20211 Berlin | 2022-01-02 12:00:00 | 10212 | 10223 Moscow | 2022-01-02 12:00:00 | 20212 | 20223 CALL refresh_continuous_aggregate('conditions_summary_timestamptz', '2022-01-02 00:00:00 MSK', '2022-01-03 00:00:00 MSK'); -- New data is seen because the cagg was refreshed SELECT city, to_char(bucket at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') AS b, min, max FROM conditions_summary_timestamptz ORDER BY b, city; city | b | min | max --------+---------------------+-------+------- Berlin | 2022-01-01 00:00:00 | 10100 | 10111 Moscow | 2022-01-01 00:00:00 | 20100 | 20111 Berlin | 2022-01-01 12:00:00 | 10112 | 10123 Moscow | 2022-01-01 12:00:00 | 20112 | 20123 Berlin | 2022-01-02 00:00:00 | 10200 | 10211 Moscow | 2022-01-02 00:00:00 | 20200 | 20211 Berlin | 2022-01-02 12:00:00 | 10212 | 10223 Moscow | 2022-01-02 12:00:00 | 20212 | 20223 -- Add a refresh policy SELECT add_continuous_aggregate_policy('conditions_summary_timestamptz', start_offset => INTERVAL '25 hours', end_offset => INTERVAL '1 hour', schedule_interval => INTERVAL '30 minutes'); add_continuous_aggregate_policy --------------------------------- 1002 -- Clean up DROP TABLE conditions_timestamptz CASCADE; NOTICE: drop cascades to 3 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_15_181_chunk ================================================ FILE: tsl/test/expected/cagg_exp_timezone.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- Make sure that timezone can't be used for types other that timestamptz CREATE TABLE conditions( day timestamp NOT NULL, -- not timestamptz! city text NOT NULL, temperature INT NOT NULL); SELECT create_hypertable( 'conditions', 'day', chunk_time_interval => INTERVAL '1 day' ); WARNING: column type "timestamp without time zone" used for "day" does not follow best practices create_hypertable ------------------------- (1,public,conditions,t) \set ON_ERROR_STOP 0 -- timebucket_ng is deprecated and can not be used in new CAggs anymore. -- However, using this GUC the restriction can be lifted in debug builds -- to ensure the functionality can be tested. SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 month', day, 'Europe/Moscow') AS bucket, MIN(temperature), MAX(temperature) FROM conditions GROUP BY city, bucket WITH NO DATA; ERROR: invalid input syntax for type timestamp: "Europe/Moscow" at character 178 -- Reset GUC to check if the CAgg would also work in release builds RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; \set ON_ERROR_STOP 1 DROP TABLE conditions CASCADE; CREATE TABLE conditions_tz( day timestamptz NOT NULL, city text NOT NULL, temperature INT NOT NULL); SELECT create_hypertable( 'conditions_tz', 'day', chunk_time_interval => INTERVAL '1 day' ); create_hypertable ---------------------------- (2,public,conditions_tz,t) INSERT INTO conditions_tz (day, city, temperature) VALUES ('2021-06-14 00:00:00 MSK', 'Moscow', 26), ('2021-06-15 00:00:00 MSK', 'Moscow', 22), ('2021-06-16 00:00:00 MSK', 'Moscow', 24), ('2021-06-17 00:00:00 MSK', 'Moscow', 24), ('2021-06-18 00:00:00 MSK', 'Moscow', 27), ('2021-06-19 00:00:00 MSK', 'Moscow', 28), ('2021-06-20 00:00:00 MSK', 'Moscow', 30), ('2021-06-21 00:00:00 MSK', 'Moscow', 31), ('2021-06-22 00:00:00 MSK', 'Moscow', 34), ('2021-06-23 00:00:00 MSK', 'Moscow', 34), ('2021-06-24 00:00:00 MSK', 'Moscow', 34), ('2021-06-25 00:00:00 MSK', 'Moscow', 32), ('2021-06-26 00:00:00 MSK', 'Moscow', 32), ('2021-06-27 00:00:00 MSK', 'Moscow', 31); \set ON_ERROR_STOP 0 -- Check that the name of the timezone is validated SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_summary_tz WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 month', day, 'Europe/Ololondon') AS bucket, MIN(temperature), MAX(temperature) FROM conditions_tz GROUP BY city, bucket WITH NO DATA; ERROR: invalid timezone name "Europe/Ololondon" -- Check that buckets like '1 month 15 days' (fixed-sized + variable-sized) are not allowed CREATE MATERIALIZED VIEW conditions_summary_tz WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 month 15 days', day, 'MSK') AS bucket, MIN(temperature), MAX(temperature) FROM conditions_tz GROUP BY city, bucket WITH NO DATA; ERROR: invalid interval specified -- Check that only immutable expressions can be used as a timezone CREATE MATERIALIZED VIEW conditions_summary_tz WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 day', day, city) AS bucket, MIN(temperature), MAX(temperature) FROM conditions_tz GROUP BY city, bucket WITH NO DATA; ERROR: only immutable expressions allowed in time bucket function \set ON_ERROR_STOP 1 -- Make sure it's possible to create an empty cagg (WITH NO DATA) and -- that all the information about the bucketing function will be saved -- to the TS catalog. CREATE MATERIALIZED VIEW conditions_summary_tz WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 month', day, 'MSK') AS bucket, MIN(temperature), MAX(temperature) FROM conditions_tz GROUP BY city, bucket WITH NO DATA; RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT mat_hypertable_id AS cagg_id_tz, raw_hypertable_id AS ht_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'conditions_summary_tz' \gset -- Make sure the timezone is saved in the catalog table SELECT bucket_func, bucket_width, bucket_origin, bucket_timezone, bucket_fixed_width FROM _timescaledb_catalog.continuous_aggs_bucket_function WHERE mat_hypertable_id = :cagg_id_tz; bucket_func | bucket_width | bucket_origin | bucket_timezone | bucket_fixed_width --------------------------------------------------------------------------------------------+--------------+---------------+-----------------+-------------------- timescaledb_experimental.time_bucket_ng(interval,timestamp with time zone,pg_catalog.text) | @ 1 mon | | MSK | f -- Make sure that buckets with specified timezone are always treated as -- variable-sized, even if the interval is fixed (i.e. days and/or hours) SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_summary_1w WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT city, timescaledb_experimental.time_bucket_ng('7 days', day, 'MSK') AS bucket, MIN(temperature), MAX(temperature) FROM conditions_tz GROUP BY city, bucket WITH NO DATA; RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT mat_hypertable_id AS cagg_id_1w FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'conditions_summary_1w' \gset -- Make sure the timezone is saved in the catalog table SELECT bucket_func, bucket_width, bucket_origin, bucket_timezone, bucket_fixed_width FROM _timescaledb_catalog.continuous_aggs_bucket_function WHERE mat_hypertable_id = :cagg_id_1w; bucket_func | bucket_width | bucket_origin | bucket_timezone | bucket_fixed_width --------------------------------------------------------------------------------------------+--------------+---------------+-----------------+-------------------- timescaledb_experimental.time_bucket_ng(interval,timestamp with time zone,pg_catalog.text) | @ 7 days | | MSK | f -- Check the invalidation threshold is -infinity SELECT _timescaledb_functions.to_timestamp(watermark) at time zone 'MSK' FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id = :ht_id; timezone ----------- -infinity -- Make sure the invalidation log is empty SELECT to_char(_timescaledb_functions.to_timestamp(lowest_modified_value) at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') AS lowest, to_char(_timescaledb_functions.to_timestamp(greatest_modified_value) at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') AS greatest FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log WHERE hypertable_id = :ht_id; lowest | greatest --------+---------- -- Make sure truncating of the refresh window works \set ON_ERROR_STOP 0 CALL refresh_continuous_aggregate('conditions_summary_tz', '2021-07-02 MSK', '2021-07-12 MSK'); ERROR: refresh window too small CALL refresh_continuous_aggregate('conditions_summary_1w', '2021-07-02 MSK', '2021-07-05 MSK'); ERROR: refresh window too small \set ON_ERROR_STOP 1 -- Make sure refreshing works CALL refresh_continuous_aggregate('conditions_summary_tz', '2021-06-01 MSK', '2021-07-01 MSK'); SELECT city, to_char(bucket at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') as month, min, max FROM conditions_summary_tz ORDER by month, city; city | month | min | max --------+---------------------+-----+----- Moscow | 2021-06-01 00:00:00 | 22 | 34 -- Check the invalidation threshold SELECT to_char(_timescaledb_functions.to_timestamp(watermark) at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id = :ht_id; to_char --------------------- 2021-07-01 00:00:00 -- The default origin is Saturday. Here we do refresh only for two full weeks -- in June in order to keep the invalidation threshold as it is. CALL refresh_continuous_aggregate('conditions_summary_1w', '2021-06-12 MSK', '2021-06-26 MSK'); SELECT city, to_char(bucket at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') as week, min, max FROM conditions_summary_1w ORDER by week, city; city | week | min | max --------+---------------------+-----+----- Moscow | 2021-06-12 00:00:00 | 22 | 27 Moscow | 2021-06-19 00:00:00 | 28 | 34 -- Check the invalidation threshold SELECT to_char(_timescaledb_functions.to_timestamp(watermark) at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id = :ht_id; to_char --------------------- 2021-07-01 00:00:00 -- Make sure creating CAGGs without NO DATA works the same way SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_summary_tz2 WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 month', day, 'MSK') AS bucket, MIN(temperature), MAX(temperature) FROM conditions_tz GROUP BY city, bucket; NOTICE: refreshing continuous aggregate "conditions_summary_tz2" RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT city, to_char(bucket at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') as month, min, max FROM conditions_summary_tz2 ORDER by month, city; city | month | min | max --------+---------------------+-----+----- Moscow | 2021-06-01 00:00:00 | 22 | 34 -- Add some dummy data for two more months (no invalidations test case) INSERT INTO conditions_tz (day, city, temperature) SELECT ts, city, row_number() OVER () FROM generate_series('2021-07-01 MSK' :: timestamptz, '2021-08-31 MSK', '3 day') as ts, unnest(array['Moscow', 'Berlin']) as city; -- Double check the generated data SELECT to_char(day at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS'), city, temperature FROM conditions_tz WHERE day >= '2021-07-01 MSK' ORDER BY city DESC, day; to_char | city | temperature ---------------------+--------+------------- 2021-07-01 00:00:00 | Moscow | 1 2021-07-04 00:00:00 | Moscow | 2 2021-07-07 00:00:00 | Moscow | 3 2021-07-10 00:00:00 | Moscow | 4 2021-07-13 00:00:00 | Moscow | 5 2021-07-16 00:00:00 | Moscow | 6 2021-07-19 00:00:00 | Moscow | 7 2021-07-22 00:00:00 | Moscow | 8 2021-07-25 00:00:00 | Moscow | 9 2021-07-28 00:00:00 | Moscow | 10 2021-07-31 00:00:00 | Moscow | 11 2021-08-03 00:00:00 | Moscow | 12 2021-08-06 00:00:00 | Moscow | 13 2021-08-09 00:00:00 | Moscow | 14 2021-08-12 00:00:00 | Moscow | 15 2021-08-15 00:00:00 | Moscow | 16 2021-08-18 00:00:00 | Moscow | 17 2021-08-21 00:00:00 | Moscow | 18 2021-08-24 00:00:00 | Moscow | 19 2021-08-27 00:00:00 | Moscow | 20 2021-08-30 00:00:00 | Moscow | 21 2021-07-01 00:00:00 | Berlin | 22 2021-07-04 00:00:00 | Berlin | 23 2021-07-07 00:00:00 | Berlin | 24 2021-07-10 00:00:00 | Berlin | 25 2021-07-13 00:00:00 | Berlin | 26 2021-07-16 00:00:00 | Berlin | 27 2021-07-19 00:00:00 | Berlin | 28 2021-07-22 00:00:00 | Berlin | 29 2021-07-25 00:00:00 | Berlin | 30 2021-07-28 00:00:00 | Berlin | 31 2021-07-31 00:00:00 | Berlin | 32 2021-08-03 00:00:00 | Berlin | 33 2021-08-06 00:00:00 | Berlin | 34 2021-08-09 00:00:00 | Berlin | 35 2021-08-12 00:00:00 | Berlin | 36 2021-08-15 00:00:00 | Berlin | 37 2021-08-18 00:00:00 | Berlin | 38 2021-08-21 00:00:00 | Berlin | 39 2021-08-24 00:00:00 | Berlin | 40 2021-08-27 00:00:00 | Berlin | 41 2021-08-30 00:00:00 | Berlin | 42 -- Make sure the invalidation threshold was unaffected SELECT to_char(_timescaledb_functions.to_timestamp(watermark) at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id = :ht_id; to_char --------------------- 2021-07-01 00:00:00 -- Make sure the invalidation log is still empty SELECT to_char(_timescaledb_functions.to_timestamp(lowest_modified_value) at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') AS lowest, to_char(_timescaledb_functions.to_timestamp(greatest_modified_value) at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') AS greatest FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log WHERE hypertable_id = :ht_id; lowest | greatest --------+---------- -- Call refresh for two full buckets: 2021-07-01 and 2021-08-01 CALL refresh_continuous_aggregate('conditions_summary_tz', '2021-06-15 MSK', '2021-09-15 MSK'); SELECT city, to_char(bucket at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') as month, min, max FROM conditions_summary_tz ORDER by month, city; city | month | min | max --------+---------------------+-----+----- Moscow | 2021-06-01 00:00:00 | 22 | 34 Berlin | 2021-07-01 00:00:00 | 22 | 32 Moscow | 2021-07-01 00:00:00 | 1 | 11 Berlin | 2021-08-01 00:00:00 | 33 | 42 Moscow | 2021-08-01 00:00:00 | 12 | 21 -- Make sure the invalidation threshold has changed SELECT to_char(_timescaledb_functions.to_timestamp(watermark) at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id = :ht_id; to_char --------------------- 2021-09-01 00:00:00 -- Make sure the invalidation log is still empty SELECT to_char(_timescaledb_functions.to_timestamp(lowest_modified_value) at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') AS lowest, to_char(_timescaledb_functions.to_timestamp(greatest_modified_value) at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') AS greatest FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log WHERE hypertable_id = :ht_id; lowest | greatest --------+---------- -- Add more data below the invalidation threshold, make sure that the -- invalidation log is not empty, then do a refresh. INSERT INTO conditions_tz (day, city, temperature) SELECT ts :: timestamptz, city, (CASE WHEN city = 'Moscow' THEN -100 ELSE 100 END) FROM generate_series('2021-08-16 MSK' :: timestamptz, '2021-08-30 MSK', '1 day') as ts, unnest(array['Moscow', 'Berlin']) as city; SELECT to_char(_timescaledb_functions.to_timestamp(lowest_modified_value) at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') AS lowest, to_char(_timescaledb_functions.to_timestamp(greatest_modified_value) at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') AS greatest FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log WHERE hypertable_id = :ht_id; lowest | greatest ---------------------+--------------------- 2021-08-16 00:00:00 | 2021-08-16 00:00:00 2021-08-17 00:00:00 | 2021-08-17 00:00:00 2021-08-18 00:00:00 | 2021-08-18 00:00:00 2021-08-19 00:00:00 | 2021-08-19 00:00:00 2021-08-20 00:00:00 | 2021-08-20 00:00:00 2021-08-21 00:00:00 | 2021-08-21 00:00:00 2021-08-22 00:00:00 | 2021-08-22 00:00:00 2021-08-23 00:00:00 | 2021-08-23 00:00:00 2021-08-24 00:00:00 | 2021-08-24 00:00:00 2021-08-25 00:00:00 | 2021-08-25 00:00:00 2021-08-26 00:00:00 | 2021-08-26 00:00:00 2021-08-27 00:00:00 | 2021-08-27 00:00:00 2021-08-28 00:00:00 | 2021-08-28 00:00:00 2021-08-29 00:00:00 | 2021-08-29 00:00:00 2021-08-30 00:00:00 | 2021-08-30 00:00:00 CALL refresh_continuous_aggregate('conditions_summary_tz', '2021-08-01 MSK', '2021-09-01 MSK'); SELECT city, to_char(bucket at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') as month, min, max FROM conditions_summary_tz ORDER by month, city; city | month | min | max --------+---------------------+------+----- Moscow | 2021-06-01 00:00:00 | 22 | 34 Berlin | 2021-07-01 00:00:00 | 22 | 32 Moscow | 2021-07-01 00:00:00 | 1 | 11 Berlin | 2021-08-01 00:00:00 | 33 | 100 Moscow | 2021-08-01 00:00:00 | -100 | 21 SELECT to_char(_timescaledb_functions.to_timestamp(lowest_modified_value) at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') AS lowest, to_char(_timescaledb_functions.to_timestamp(greatest_modified_value) at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') AS greatest FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log WHERE hypertable_id = :ht_id; lowest | greatest --------+---------- -- Clean up DROP MATERIALIZED VIEW conditions_summary_tz; NOTICE: drop cascades to 3 other objects DROP MATERIALIZED VIEW conditions_summary_1w; NOTICE: drop cascades to 2 other objects -- Create a real-time aggregate SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_summary_tz WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 month', day, 'MSK') AS bucket, MIN(temperature), MAX(temperature) FROM conditions_tz GROUP BY city, bucket; NOTICE: refreshing continuous aggregate "conditions_summary_tz" RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT city, to_char(bucket at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') as month, min, max FROM conditions_summary_tz ORDER by month, city; city | month | min | max --------+---------------------+------+----- Moscow | 2021-06-01 00:00:00 | 22 | 34 Berlin | 2021-07-01 00:00:00 | 22 | 32 Moscow | 2021-07-01 00:00:00 | 1 | 11 Berlin | 2021-08-01 00:00:00 | 33 | 100 Moscow | 2021-08-01 00:00:00 | -100 | 21 -- Add some data to the hypertable and make sure they are visible in the cagg INSERT INTO conditions_tz (day, city, temperature) VALUES ('2021-10-01 00:00:00 MSK', 'Moscow', 1), ('2021-10-02 00:00:00 MSK', 'Moscow', 2), ('2021-10-03 00:00:00 MSK', 'Moscow', 3), ('2021-10-04 00:00:00 MSK', 'Moscow', 4), ('2021-10-01 00:00:00 MSK', 'Berlin', 5), ('2021-10-02 00:00:00 MSK', 'Berlin', 6), ('2021-10-03 00:00:00 MSK', 'Berlin', 7), ('2021-10-04 00:00:00 MSK', 'Berlin', 8); SELECT city, to_char(bucket at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') as month, min, max FROM conditions_summary_tz ORDER by month, city; city | month | min | max --------+---------------------+------+----- Moscow | 2021-06-01 00:00:00 | 22 | 34 Berlin | 2021-07-01 00:00:00 | 22 | 32 Moscow | 2021-07-01 00:00:00 | 1 | 11 Berlin | 2021-08-01 00:00:00 | 33 | 100 Moscow | 2021-08-01 00:00:00 | -100 | 21 Berlin | 2021-10-01 00:00:00 | 5 | 8 Moscow | 2021-10-01 00:00:00 | 1 | 4 -- Refresh the cagg and make sure that the result of SELECT query didn't change CALL refresh_continuous_aggregate('conditions_summary_tz', '2021-10-01 00:00:00 MSK', '2021-11-01 00:00:00 MSK'); SELECT city, to_char(bucket at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') as month, min, max FROM conditions_summary_tz ORDER by month, city; city | month | min | max --------+---------------------+------+----- Moscow | 2021-06-01 00:00:00 | 22 | 34 Berlin | 2021-07-01 00:00:00 | 22 | 32 Moscow | 2021-07-01 00:00:00 | 1 | 11 Berlin | 2021-08-01 00:00:00 | 33 | 100 Moscow | 2021-08-01 00:00:00 | -100 | 21 Berlin | 2021-10-01 00:00:00 | 5 | 8 Moscow | 2021-10-01 00:00:00 | 1 | 4 -- Add some more data, enable compression, compress the chunks and repeat the test INSERT INTO conditions_tz (day, city, temperature) VALUES ('2021-11-01 00:00:00 MSK', 'Moscow', 11), ('2021-11-02 00:00:00 MSK', 'Moscow', 12), ('2021-11-03 00:00:00 MSK', 'Moscow', 13), ('2021-11-04 00:00:00 MSK', 'Moscow', 14), ('2021-11-01 00:00:00 MSK', 'Berlin', 15), ('2021-11-02 00:00:00 MSK', 'Berlin', 16), ('2021-11-03 00:00:00 MSK', 'Berlin', 17), ('2021-11-04 00:00:00 MSK', 'Berlin', 18); ALTER TABLE conditions_tz SET ( timescaledb.compress, timescaledb.compress_segmentby = 'city' ); SELECT compress_chunk(ch) FROM show_chunks('conditions_tz') AS ch; compress_chunk ----------------------------------------- _timescaledb_internal._hyper_2_1_chunk _timescaledb_internal._hyper_2_2_chunk _timescaledb_internal._hyper_2_3_chunk _timescaledb_internal._hyper_2_4_chunk _timescaledb_internal._hyper_2_5_chunk _timescaledb_internal._hyper_2_6_chunk _timescaledb_internal._hyper_2_7_chunk _timescaledb_internal._hyper_2_8_chunk _timescaledb_internal._hyper_2_9_chunk _timescaledb_internal._hyper_2_10_chunk _timescaledb_internal._hyper_2_11_chunk _timescaledb_internal._hyper_2_12_chunk _timescaledb_internal._hyper_2_13_chunk _timescaledb_internal._hyper_2_14_chunk _timescaledb_internal._hyper_2_19_chunk _timescaledb_internal._hyper_2_20_chunk _timescaledb_internal._hyper_2_21_chunk _timescaledb_internal._hyper_2_22_chunk _timescaledb_internal._hyper_2_23_chunk _timescaledb_internal._hyper_2_24_chunk _timescaledb_internal._hyper_2_25_chunk _timescaledb_internal._hyper_2_26_chunk _timescaledb_internal._hyper_2_27_chunk _timescaledb_internal._hyper_2_28_chunk _timescaledb_internal._hyper_2_29_chunk _timescaledb_internal._hyper_2_30_chunk _timescaledb_internal._hyper_2_31_chunk _timescaledb_internal._hyper_2_32_chunk _timescaledb_internal._hyper_2_33_chunk _timescaledb_internal._hyper_2_34_chunk _timescaledb_internal._hyper_2_35_chunk _timescaledb_internal._hyper_2_36_chunk _timescaledb_internal._hyper_2_37_chunk _timescaledb_internal._hyper_2_38_chunk _timescaledb_internal._hyper_2_39_chunk _timescaledb_internal._hyper_2_42_chunk _timescaledb_internal._hyper_2_43_chunk _timescaledb_internal._hyper_2_44_chunk _timescaledb_internal._hyper_2_45_chunk _timescaledb_internal._hyper_2_46_chunk _timescaledb_internal._hyper_2_47_chunk _timescaledb_internal._hyper_2_48_chunk _timescaledb_internal._hyper_2_49_chunk _timescaledb_internal._hyper_2_50_chunk _timescaledb_internal._hyper_2_51_chunk _timescaledb_internal._hyper_2_55_chunk _timescaledb_internal._hyper_2_56_chunk _timescaledb_internal._hyper_2_57_chunk _timescaledb_internal._hyper_2_58_chunk _timescaledb_internal._hyper_2_60_chunk _timescaledb_internal._hyper_2_61_chunk _timescaledb_internal._hyper_2_62_chunk _timescaledb_internal._hyper_2_63_chunk -- Data for 2021-11 is seen because the cagg is real-time SELECT city, to_char(bucket at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') as month, min, max FROM conditions_summary_tz ORDER by month, city; city | month | min | max --------+---------------------+------+----- Moscow | 2021-06-01 00:00:00 | 22 | 34 Berlin | 2021-07-01 00:00:00 | 22 | 32 Moscow | 2021-07-01 00:00:00 | 1 | 11 Berlin | 2021-08-01 00:00:00 | 33 | 100 Moscow | 2021-08-01 00:00:00 | -100 | 21 Berlin | 2021-10-01 00:00:00 | 5 | 8 Moscow | 2021-10-01 00:00:00 | 1 | 4 Berlin | 2021-11-01 00:00:00 | 15 | 18 Moscow | 2021-11-01 00:00:00 | 11 | 14 CALL refresh_continuous_aggregate('conditions_summary_tz', '2021-11-01 00:00:00 MSK', '2021-12-01 00:00:00 MSK'); -- Data for 2021-11 is seen because the cagg was refreshed SELECT city, to_char(bucket at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') as month, min, max FROM conditions_summary_tz ORDER by month, city; city | month | min | max --------+---------------------+------+----- Moscow | 2021-06-01 00:00:00 | 22 | 34 Berlin | 2021-07-01 00:00:00 | 22 | 32 Moscow | 2021-07-01 00:00:00 | 1 | 11 Berlin | 2021-08-01 00:00:00 | 33 | 100 Moscow | 2021-08-01 00:00:00 | -100 | 21 Berlin | 2021-10-01 00:00:00 | 5 | 8 Moscow | 2021-10-01 00:00:00 | 1 | 4 Berlin | 2021-11-01 00:00:00 | 15 | 18 Moscow | 2021-11-01 00:00:00 | 11 | 14 -- Test for some more cases: single CAGG per HT, creating CAGG on top of an -- empty HT, buckets other than 1 month. CREATE TABLE conditions2( day timestamptz NOT NULL, city text NOT NULL, temperature INT NOT NULL); SELECT create_hypertable( 'conditions2', 'day', chunk_time_interval => INTERVAL '1 day' ); create_hypertable -------------------------- (8,public,conditions2,t) -- Create a real-time aggregate on top of empty HT SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions2_summary WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT city, timescaledb_experimental.time_bucket_ng('7 days', day, 'MSK') AS bucket, MIN(temperature), MAX(temperature) FROM conditions2 GROUP BY city, bucket; NOTICE: continuous aggregate "conditions2_summary" is already up-to-date RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; INSERT INTO conditions2 (day, city, temperature) VALUES ('2021-06-14 00:00:00 MSK', 'Moscow', 26), ('2021-06-15 00:00:00 MSK', 'Moscow', 22), ('2021-06-16 00:00:00 MSK', 'Moscow', 24), ('2021-06-17 00:00:00 MSK', 'Moscow', 24), ('2021-06-18 00:00:00 MSK', 'Moscow', 27), ('2021-06-19 00:00:00 MSK', 'Moscow', 28), ('2021-06-20 00:00:00 MSK', 'Moscow', 30), ('2021-06-21 00:00:00 MSK', 'Moscow', 31), ('2021-06-22 00:00:00 MSK', 'Moscow', 34), ('2021-06-23 00:00:00 MSK', 'Moscow', 34), ('2021-06-24 00:00:00 MSK', 'Moscow', 34), ('2021-06-25 00:00:00 MSK', 'Moscow', 32), ('2021-06-26 00:00:00 MSK', 'Moscow', 32), ('2021-06-27 00:00:00 MSK', 'Moscow', 31); -- All data should be seen for a real-time aggregate SELECT city, to_char(bucket at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') as month, min, max FROM conditions2_summary ORDER by month, city; city | month | min | max --------+---------------------+-----+----- Moscow | 2021-06-12 00:00:00 | 22 | 27 Moscow | 2021-06-19 00:00:00 | 28 | 34 Moscow | 2021-06-26 00:00:00 | 31 | 32 -- Refresh should work CALL refresh_continuous_aggregate('conditions2_summary', '2021-06-12 MSK', '2021-07-03 MSK'); SELECT city, to_char(bucket at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') as month, min, max FROM conditions2_summary ORDER by month, city; city | month | min | max --------+---------------------+-----+----- Moscow | 2021-06-12 00:00:00 | 22 | 27 Moscow | 2021-06-19 00:00:00 | 28 | 34 Moscow | 2021-06-26 00:00:00 | 31 | 32 -- New data should be seen INSERT INTO conditions2 (day, city, temperature) VALUES ('2021-09-30 00:00:00 MSK', 'Moscow', 0), ('2021-10-01 00:00:00 MSK', 'Moscow', 1), ('2021-10-02 00:00:00 MSK', 'Moscow', 2), ('2021-10-03 00:00:00 MSK', 'Moscow', 3), ('2021-10-04 00:00:00 MSK', 'Moscow', 4), ('2021-09-30 00:00:00 MSK', 'Berlin', 5), ('2021-10-01 00:00:00 MSK', 'Berlin', 6), ('2021-10-02 00:00:00 MSK', 'Berlin', 7), ('2021-10-03 00:00:00 MSK', 'Berlin', 8), ('2021-10-04 00:00:00 MSK', 'Berlin', 9); SELECT city, to_char(bucket at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') as month, min, max FROM conditions2_summary ORDER by month, city; city | month | min | max --------+---------------------+-----+----- Moscow | 2021-06-12 00:00:00 | 22 | 27 Moscow | 2021-06-19 00:00:00 | 28 | 34 Moscow | 2021-06-26 00:00:00 | 31 | 32 Berlin | 2021-09-25 00:00:00 | 5 | 6 Moscow | 2021-09-25 00:00:00 | 0 | 1 Berlin | 2021-10-02 00:00:00 | 7 | 9 Moscow | 2021-10-02 00:00:00 | 2 | 4 -- Make sure add_continuous_aggregate_policy() works CREATE TABLE conditions_policy( day TIMESTAMPTZ NOT NULL, city text NOT NULL, temperature INT NOT NULL); SELECT create_hypertable( 'conditions_policy', 'day', chunk_time_interval => INTERVAL '1 day' ); create_hypertable --------------------------------- (10,public,conditions_policy,t) INSERT INTO conditions_policy (day, city, temperature) VALUES ('2021-06-14 00:00:00 MSK', 'Moscow', 26), ('2021-06-14 10:00:00 MSK', 'Moscow', 22), ('2021-06-14 20:00:00 MSK', 'Moscow', 24), ('2021-06-15 00:00:00 MSK', 'Moscow', 24), ('2021-06-15 10:00:00 MSK', 'Moscow', 27), ('2021-06-15 20:00:00 MSK', 'Moscow', 28), ('2021-06-16 00:00:00 MSK', 'Moscow', 30), ('2021-06-16 10:00:00 MSK', 'Moscow', 31), ('2021-06-16 20:00:00 MSK', 'Moscow', 34), ('2021-06-17 00:00:00 MSK', 'Moscow', 34), ('2021-06-17 10:00:00 MSK', 'Moscow', 34), ('2021-06-17 20:00:00 MSK', 'Moscow', 32), ('2021-06-18 00:00:00 MSK', 'Moscow', 32), ('2021-06-18 10:00:00 MSK', 'Moscow', 31), ('2021-06-18 20:00:00 MSK', 'Moscow', 26); SET timescaledb.debug_allow_cagg_with_deprecated_funcs = true; CREATE MATERIALIZED VIEW conditions_summary_policy WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT city, timescaledb_experimental.time_bucket_ng('1 day', day, 'Europe/Moscow') AS bucket, MIN(temperature), MAX(temperature) FROM conditions_policy GROUP BY city, bucket; NOTICE: refreshing continuous aggregate "conditions_summary_policy" RESET timescaledb.debug_allow_cagg_with_deprecated_funcs; SELECT city, to_char(bucket at time zone 'MSK', 'YYYY-MM-DD HH24:MI:SS') as month, min, max FROM conditions_summary_policy ORDER by month, city; city | month | min | max --------+---------------------+-----+----- Moscow | 2021-06-14 00:00:00 | 22 | 26 Moscow | 2021-06-15 00:00:00 | 24 | 28 Moscow | 2021-06-16 00:00:00 | 30 | 34 Moscow | 2021-06-17 00:00:00 | 32 | 34 Moscow | 2021-06-18 00:00:00 | 26 | 32 \set ON_ERROR_STOP 0 -- Check for "policy refresh window too small" error SELECT add_continuous_aggregate_policy('conditions_summary_policy', start_offset => INTERVAL '2 days 23 hours', end_offset => INTERVAL '1 day', schedule_interval => INTERVAL '1 hour'); ERROR: policy refresh window too small \set ON_ERROR_STOP 1 SELECT add_continuous_aggregate_policy('conditions_summary_policy', start_offset => INTERVAL '3 days', end_offset => INTERVAL '1 day', schedule_interval => INTERVAL '1 hour'); add_continuous_aggregate_policy --------------------------------- 1000 ================================================ FILE: tsl/test/expected/cagg_invalidation.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- Disable background workers since we are testing manual refresh \c :TEST_DBNAME :ROLE_SUPERUSER SELECT _timescaledb_functions.stop_background_workers(); stop_background_workers ------------------------- t SET ROLE :ROLE_DEFAULT_PERM_USER; SET datestyle TO 'ISO, YMD'; SET timezone TO 'UTC'; CREATE VIEW hypertable_invalidation_thresholds AS SELECT format('%I.%I', ht.schema_name, ht.table_name)::regclass AS hypertable, watermark AS threshold FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold JOIN _timescaledb_catalog.hypertable ht ON hypertable_id = ht.id ORDER BY 1; CREATE TABLE conditions (time bigint NOT NULL, device int, temp float); SELECT create_hypertable('conditions', 'time', chunk_time_interval => 10); create_hypertable ------------------------- (1,public,conditions,t) CREATE TABLE measurements (time int NOT NULL, device int, temp float); SELECT create_hypertable('measurements', 'time', chunk_time_interval => 10); create_hypertable --------------------------- (2,public,measurements,t) CREATE OR REPLACE FUNCTION bigint_now() RETURNS bigint LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM conditions $$; CREATE OR REPLACE FUNCTION int_now() RETURNS int LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM measurements $$; SELECT set_integer_now_func('conditions', 'bigint_now'); set_integer_now_func ---------------------- SELECT set_integer_now_func('measurements', 'int_now'); set_integer_now_func ---------------------- INSERT INTO conditions SELECT t, ceil(abs(timestamp_hash(to_timestamp(t)::timestamp))%4)::int, abs(timestamp_hash(to_timestamp(t)::timestamp))%40 FROM generate_series(1, 100, 1) t; CREATE TABLE temp AS SELECT * FROM conditions; INSERT INTO measurements SELECT * FROM temp; -- Show the most recent data SELECT * FROM conditions ORDER BY time DESC, device LIMIT 10; time | device | temp ------+--------+------ 100 | 0 | 8 99 | 1 | 5 98 | 2 | 26 97 | 2 | 10 96 | 2 | 34 95 | 2 | 30 94 | 3 | 31 93 | 0 | 4 92 | 0 | 32 91 | 3 | 15 -- Create two continuous aggregates on the same hypertable to test -- that invalidations are handled correctly across both of them. CREATE MATERIALIZED VIEW cond_10 WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(BIGINT '10', time) AS bucket, device, avg(temp) AS avg_temp FROM conditions GROUP BY 1,2 WITH NO DATA; CREATE MATERIALIZED VIEW cond_20 WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(BIGINT '20', time) AS bucket, device, avg(temp) AS avg_temp FROM conditions GROUP BY 1,2 WITH NO DATA; CREATE MATERIALIZED VIEW measure_10 WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(10, time) AS bucket, device, avg(temp) AS avg_temp FROM measurements GROUP BY 1,2 WITH NO DATA; -- There should be three continuous aggregates, two on one hypertable -- and one on the other: SELECT mat_hypertable_id, raw_hypertable_id, user_view_name FROM _timescaledb_catalog.continuous_agg; mat_hypertable_id | raw_hypertable_id | user_view_name -------------------+-------------------+---------------- 3 | 1 | cond_10 4 | 1 | cond_20 5 | 2 | measure_10 -- The continuous aggregates should be empty SELECT * FROM cond_10 ORDER BY 1 DESC, 2; bucket | device | avg_temp --------+--------+---------- SELECT * FROM cond_20 ORDER BY 1 DESC, 2; bucket | device | avg_temp --------+--------+---------- SELECT * FROM measure_10 ORDER BY 1 DESC, 2; bucket | device | avg_temp --------+--------+---------- CREATE OR REPLACE FUNCTION get_hyper_invals() RETURNS TABLE ( "hyper_id" INT, "start" BIGINT, "end" BIGINT ) LANGUAGE SQL VOLATILE AS $$ SELECT hypertable_id, lowest_modified_value, greatest_modified_value FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log ORDER BY 1,2,3 $$; CREATE OR REPLACE FUNCTION get_cagg_invals() RETURNS TABLE ( "cagg_id" INT, "start" BIGINT, "end" BIGINT ) LANGUAGE SQL VOLATILE AS $$ SELECT materialization_id, lowest_modified_value, greatest_modified_value FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log ORDER BY 1,2,3 $$; CREATE VIEW hyper_invals AS SELECT * FROM get_hyper_invals(); CREATE VIEW cagg_invals AS SELECT * FROM get_cagg_invals(); -- Must refresh to move the invalidation threshold, or no -- invalidations will be generated. Initially, threshold is the -- MIN of the time dimension data type: SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold ORDER BY 1,2; hypertable_id | watermark ---------------+---------------------- 1 | -9223372036854775808 2 | -2147483648 -- There should be only "infinite" invalidations in the cagg -- invalidation log: SELECT * FROM cagg_invals; cagg_id | start | end ---------+----------------------+--------------------- 3 | -9223372036854775808 | 9223372036854775807 4 | -9223372036854775808 | 9223372036854775807 5 | -9223372036854775808 | 9223372036854775807 -- Now refresh up to 50 without the first bucket, and the threshold should be updated accordingly: CALL refresh_continuous_aggregate('cond_10', 1, 50); SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold ORDER BY 1,2; hypertable_id | watermark ---------------+------------- 1 | 50 2 | -2147483648 -- Invalidations should be cleared inside the refresh window: SELECT * FROM cagg_invals; cagg_id | start | end ---------+----------------------+--------------------- 3 | -9223372036854775808 | 9 3 | 50 | 9223372036854775807 4 | -9223372036854775808 | 9223372036854775807 5 | -9223372036854775808 | 9223372036854775807 -- Refresh up to 50 from the beginning CALL refresh_continuous_aggregate('cond_10', 0, 50); SELECT * FROM cagg_invals; cagg_id | start | end ---------+----------------------+--------------------- 3 | -9223372036854775808 | -1 3 | 50 | 9223372036854775807 4 | -9223372036854775808 | 9223372036854775807 5 | -9223372036854775808 | 9223372036854775807 -- Refreshing below the threshold does not move it: CALL refresh_continuous_aggregate('cond_10', 20, 49); NOTICE: continuous aggregate "cond_10" is already up-to-date SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold ORDER BY 1,2; hypertable_id | watermark ---------------+------------- 1 | 50 2 | -2147483648 -- Nothing changes with invalidations either since the region was -- already refreshed and no new invalidations have been generated: SELECT * FROM cagg_invals; cagg_id | start | end ---------+----------------------+--------------------- 3 | -9223372036854775808 | -1 3 | 50 | 9223372036854775807 4 | -9223372036854775808 | 9223372036854775807 5 | -9223372036854775808 | 9223372036854775807 -- Refreshing measure_10 moves the threshold only for the other hypertable: CALL refresh_continuous_aggregate('measure_10', 0, 30); SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold ORDER BY 1,2; hypertable_id | watermark ---------------+----------- 1 | 50 2 | 30 SELECT * FROM cagg_invals; cagg_id | start | end ---------+----------------------+--------------------- 3 | -9223372036854775808 | -1 3 | 50 | 9223372036854775807 4 | -9223372036854775808 | 9223372036854775807 5 | -9223372036854775808 | -1 5 | 30 | 9223372036854775807 -- Refresh on the second continuous aggregate, cond_20, on the first -- hypertable moves the same threshold as when refreshing cond_10: CALL refresh_continuous_aggregate('cond_20', 60, 100); SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold ORDER BY 1,2; hypertable_id | watermark ---------------+----------- 1 | 100 2 | 30 SELECT * FROM cagg_invals; cagg_id | start | end ---------+----------------------+--------------------- 3 | -9223372036854775808 | -1 3 | 50 | 9223372036854775807 4 | -9223372036854775808 | 59 4 | 100 | 9223372036854775807 5 | -9223372036854775808 | -1 5 | 30 | 9223372036854775807 -- There should be no hypertable invalidations initially: SELECT * FROM hyper_invals; hyper_id | start | end ----------+-------+----- SELECT * FROM cagg_invals; cagg_id | start | end ---------+----------------------+--------------------- 3 | -9223372036854775808 | -1 3 | 50 | 9223372036854775807 4 | -9223372036854775808 | 59 4 | 100 | 9223372036854775807 5 | -9223372036854775808 | -1 5 | 30 | 9223372036854775807 -- Create invalidations across different ranges. Some of these should -- be deleted and others cut in different ways when a refresh is -- run. Note that the refresh window is inclusive in the start of the -- window but exclusive at the end. -- Entries that should be left unmodified: INSERT INTO conditions VALUES (10, 4, 23.7); INSERT INTO conditions VALUES (10, 5, 23.8), (19, 3, 23.6); INSERT INTO conditions VALUES (60, 3, 23.7), (70, 4, 23.7); -- Should see some invaliations in the hypertable invalidation log: SELECT * FROM hyper_invals; hyper_id | start | end ----------+-------+----- 1 | 10 | 10 1 | 10 | 19 1 | 60 | 60 1 | 70 | 70 -- Generate some invalidations for the other hypertable INSERT INTO measurements VALUES (20, 4, 23.7); INSERT INTO measurements VALUES (30, 5, 23.8), (80, 3, 23.6); -- Should now see invalidations for both hypertables SELECT * FROM hyper_invals; hyper_id | start | end ----------+-------+----- 1 | 10 | 10 1 | 10 | 19 1 | 60 | 60 1 | 70 | 70 2 | 20 | 20 -- First refresh a window where we don't have any invalidations. This -- allows us to see only the copying of the invalidations to the per -- cagg log without additional processing. CALL refresh_continuous_aggregate('cond_10', 20, 60); -- Invalidation threshold remains at 100: SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold ORDER BY 1,2; hypertable_id | watermark ---------------+----------- 1 | 100 2 | 30 -- Invalidations should be moved from the hypertable invalidation log -- to the continuous aggregate log, but only for the hypertable that -- the refreshed aggregate belongs to: SELECT * FROM hyper_invals; hyper_id | start | end ----------+-------+----- 2 | 20 | 20 SELECT * FROM cagg_invals; cagg_id | start | end ---------+----------------------+--------------------- 3 | -9223372036854775808 | -1 3 | 10 | 19 3 | 60 | 9223372036854775807 4 | -9223372036854775808 | 59 4 | 0 | 19 4 | 60 | 79 4 | 100 | 9223372036854775807 5 | -9223372036854775808 | -1 5 | 30 | 9223372036854775807 -- Now add more invalidations to test a refresh that overlaps with them. -- Entries that should be deleted: INSERT INTO conditions VALUES (30, 1, 23.4), (59, 1, 23.4); INSERT INTO conditions VALUES (20, 1, 23.4), (30, 1, 23.4); -- Entries that should be cut to the right, leaving an invalidation to -- the left of the refresh window: INSERT INTO conditions VALUES (1, 4, 23.7), (25, 1, 23.4); INSERT INTO conditions VALUES (19, 4, 23.7), (59, 1, 23.4); -- Entries that should be cut to the left and right, leaving two -- invalidation entries on each side of the refresh window: INSERT INTO conditions VALUES (2, 2, 23.5), (60, 1, 23.4); INSERT INTO conditions VALUES (3, 2, 23.5), (80, 1, 23.4); -- Entries that should be cut to the left, leaving an invalidation to -- the right of the refresh window: INSERT INTO conditions VALUES (60, 3, 23.6), (90, 3, 23.6); INSERT INTO conditions VALUES (20, 5, 23.8), (100, 3, 23.6); -- New invalidations in the hypertable invalidation log: SELECT * FROM hyper_invals; hyper_id | start | end ----------+-------+----- 1 | 1 | 1 1 | 2 | 2 1 | 3 | 3 1 | 19 | 19 1 | 20 | 20 1 | 20 | 20 1 | 25 | 25 1 | 30 | 30 1 | 30 | 30 1 | 59 | 59 1 | 59 | 59 1 | 60 | 60 1 | 60 | 60 1 | 80 | 80 1 | 90 | 90 2 | 20 | 20 -- But nothing has yet changed in the cagg invalidation log: SELECT * FROM cagg_invals; cagg_id | start | end ---------+----------------------+--------------------- 3 | -9223372036854775808 | -1 3 | 10 | 19 3 | 60 | 9223372036854775807 4 | -9223372036854775808 | 59 4 | 0 | 19 4 | 60 | 79 4 | 100 | 9223372036854775807 5 | -9223372036854775808 | -1 5 | 30 | 9223372036854775807 -- Refresh to process invalidations for daily temperature: CALL refresh_continuous_aggregate('cond_10', 20, 60); -- Invalidations should be moved from the hypertable invalidation log -- to the continuous aggregate log. SELECT * FROM hyper_invals; hyper_id | start | end ----------+-------+----- 2 | 20 | 20 -- Only the cond_10 cagg should have its entries cut: SELECT * FROM cagg_invals; cagg_id | start | end ---------+----------------------+--------------------- 3 | -9223372036854775808 | 19 3 | 60 | 9223372036854775807 4 | -9223372036854775808 | 59 4 | 0 | 19 4 | 0 | 99 4 | 60 | 79 4 | 100 | 9223372036854775807 5 | -9223372036854775808 | -1 5 | 30 | 9223372036854775807 -- Refresh also cond_20: CALL refresh_continuous_aggregate('cond_20', 20, 60); -- The cond_20 cagg should also have its entries cut: SELECT * FROM cagg_invals; cagg_id | start | end ---------+----------------------+--------------------- 3 | -9223372036854775808 | 19 3 | 60 | 9223372036854775807 4 | -9223372036854775808 | 19 4 | 60 | 9223372036854775807 5 | -9223372036854775808 | -1 5 | 30 | 9223372036854775807 -- Refresh cond_10 to completely remove an invalidation: CALL refresh_continuous_aggregate('cond_10', 0, 20); -- The 1-19 invalidation should be deleted: SELECT * FROM cagg_invals; cagg_id | start | end ---------+----------------------+--------------------- 3 | -9223372036854775808 | -1 3 | 60 | 9223372036854775807 4 | -9223372036854775808 | 19 4 | 60 | 9223372036854775807 5 | -9223372036854775808 | -1 5 | 30 | 9223372036854775807 -- Clear everything between 0 and 100 to make way for new -- invalidations CALL refresh_continuous_aggregate('cond_10', 0, 100); -- Test refreshing with non-overlapping invalidations INSERT INTO conditions VALUES (20, 1, 23.4), (25, 1, 23.4); INSERT INTO conditions VALUES (30, 1, 23.4), (46, 1, 23.4); CALL refresh_continuous_aggregate('cond_10', 1, 40); SELECT * FROM cagg_invals; cagg_id | start | end ---------+----------------------+--------------------- 3 | -9223372036854775808 | -1 3 | 40 | 49 3 | 100 | 9223372036854775807 4 | -9223372036854775808 | 19 4 | 20 | 59 4 | 60 | 9223372036854775807 5 | -9223372036854775808 | -1 5 | 30 | 9223372036854775807 -- Refresh whithout cutting (in area where there are no -- invalidations). Merging of overlapping entries should still happen: INSERT INTO conditions VALUES (15, 1, 23.4), (42, 1, 23.4); CALL refresh_continuous_aggregate('cond_10', 90, 100); NOTICE: continuous aggregate "cond_10" is already up-to-date SELECT * FROM cagg_invals; cagg_id | start | end ---------+----------------------+--------------------- 3 | -9223372036854775808 | -1 3 | 10 | 19 3 | 40 | 49 3 | 100 | 9223372036854775807 4 | -9223372036854775808 | 19 4 | 0 | 19 4 | 20 | 59 4 | 40 | 59 4 | 60 | 9223372036854775807 5 | -9223372036854775808 | -1 5 | 30 | 9223372036854775807 -- Test max refresh window CALL refresh_continuous_aggregate('cond_10', NULL, NULL); SELECT * FROM cagg_invals; cagg_id | start | end ---------+----------------------+--------------------- 3 | 110 | 9223372036854775807 4 | -9223372036854775808 | 19 4 | 0 | 19 4 | 20 | 59 4 | 40 | 59 4 | 60 | 9223372036854775807 5 | -9223372036854775808 | -1 5 | 30 | 9223372036854775807 SELECT * FROM hyper_invals; hyper_id | start | end ----------+-------+----- 2 | 20 | 20 -- Pick the first chunk of conditions to TRUNCATE SELECT show_chunks AS chunk_to_truncate FROM show_chunks('conditions') ORDER BY 1 LIMIT 1 \gset -- Show the data before truncating one of the chunks SELECT * FROM :chunk_to_truncate ORDER BY 1; time | device | temp ------+--------+------ 1 | 4 | 23.7 1 | 0 | 16 2 | 2 | 23.5 2 | 1 | 25 3 | 2 | 23.5 3 | 0 | 20 4 | 2 | 10 5 | 2 | 26 6 | 1 | 13 7 | 3 | 35 8 | 1 | 37 9 | 3 | 7 -- Truncate one chunk TRUNCATE TABLE :chunk_to_truncate; -- Should see new invalidation entries for conditions SELECT * FROM hyper_invals; hyper_id | start | end ----------+-------+----- 1 | 0 | 10 2 | 20 | 20 -- TRUNCATE the hypertable to invalidate all its continuous aggregates TRUNCATE conditions; -- Now empty SELECT * FROM conditions; time | device | temp ------+--------+------ -- Should see an infinite invalidation entry for conditions SELECT * FROM hyper_invals; hyper_id | start | end ----------+----------------------+--------------------- 1 | -9223372036854775808 | 9223372036854775807 1 | 0 | 10 2 | 20 | 20 -- Aggregates still hold data SELECT * FROM cond_10 ORDER BY 1,2 LIMIT 5; bucket | device | avg_temp --------+--------+---------- 0 | 0 | 18 0 | 1 | 25 0 | 2 | 20.75 0 | 3 | 21 0 | 4 | 23.7 SELECT * FROM cond_20 ORDER BY 1,2 LIMIT 5; bucket | device | avg_temp --------+--------+------------------ 20 | 0 | 18.2857142857143 20 | 1 | 23.5142857142857 20 | 2 | 26 20 | 3 | 23 20 | 5 | 23.8 CALL refresh_continuous_aggregate('cond_10', NULL, NULL); CALL refresh_continuous_aggregate('cond_20', NULL, NULL); -- Both should now be empty after refresh SELECT * FROM cond_10 ORDER BY 1,2; bucket | device | avg_temp --------+--------+---------- SELECT * FROM cond_20 ORDER BY 1,2; bucket | device | avg_temp --------+--------+---------- -- Insert new data again and refresh INSERT INTO conditions VALUES (1, 1, 23.4), (4, 3, 14.3), (5, 1, 13.6), (6, 2, 17.9), (12, 1, 18.3), (19, 3, 28.2), (10, 3, 22.3), (11, 2, 34.9), (15, 2, 45.6), (21, 1, 15.3), (22, 2, 12.3), (29, 3, 16.3); CALL refresh_continuous_aggregate('cond_10', NULL, NULL); CALL refresh_continuous_aggregate('cond_20', NULL, NULL); -- Should now hold data again SELECT * FROM cond_10 ORDER BY 1,2; bucket | device | avg_temp --------+--------+---------- 0 | 1 | 18.5 0 | 2 | 17.9 0 | 3 | 14.3 10 | 1 | 18.3 10 | 2 | 40.25 10 | 3 | 25.25 20 | 1 | 15.3 20 | 2 | 12.3 20 | 3 | 16.3 SELECT * FROM cond_20 ORDER BY 1,2; bucket | device | avg_temp --------+--------+------------------ 0 | 1 | 18.4333333333333 0 | 2 | 32.8 0 | 3 | 21.6 20 | 1 | 15.3 20 | 2 | 12.3 20 | 3 | 16.3 -- Truncate one of the aggregates, but first test that we block -- TRUNCATE ONLY \set ON_ERROR_STOP 0 TRUNCATE ONLY cond_20; ERROR: cannot truncate only a continuous aggregate \set ON_ERROR_STOP 1 TRUNCATE cond_20; -- Should now be empty SELECT * FROM cond_20 ORDER BY 1,2; bucket | device | avg_temp --------+--------+---------- -- Other aggregate is not affected SELECT * FROM cond_10 ORDER BY 1,2; bucket | device | avg_temp --------+--------+---------- 0 | 1 | 18.5 0 | 2 | 17.9 0 | 3 | 14.3 10 | 1 | 18.3 10 | 2 | 40.25 10 | 3 | 25.25 20 | 1 | 15.3 20 | 2 | 12.3 20 | 3 | 16.3 -- Refresh again to bring data back CALL refresh_continuous_aggregate('cond_20', NULL, NULL); -- The aggregate should be populated again SELECT * FROM cond_20 ORDER BY 1,2; bucket | device | avg_temp --------+--------+------------------ 0 | 1 | 18.4333333333333 0 | 2 | 32.8 0 | 3 | 21.6 20 | 1 | 15.3 20 | 2 | 12.3 20 | 3 | 16.3 ------------------------------------------------------- -- Test corner cases against a minimal bucket aggregate ------------------------------------------------------- -- First, clear the table and aggregate TRUNCATE conditions; SELECT * FROM conditions; time | device | temp ------+--------+------ CALL refresh_continuous_aggregate('cond_10', NULL, NULL); SELECT * FROM cond_10 ORDER BY 1,2; bucket | device | avg_temp --------+--------+---------- CREATE MATERIALIZED VIEW cond_1 WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(BIGINT '1', time) AS bucket, device, avg(temp) AS avg_temp FROM conditions GROUP BY 1,2 WITH NO DATA; SELECT mat_hypertable_id AS cond_1_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'cond_1' \gset -- Test invalidations with bucket size 1 INSERT INTO conditions VALUES (0, 1, 1.0); SELECT * FROM hyper_invals; hyper_id | start | end ----------+-------+----- 1 | 0 | 0 2 | 20 | 20 -- Refreshing around the bucket should not update the aggregate CALL refresh_continuous_aggregate('cond_1', -1, 0); SELECT * FROM cond_1 ORDER BY 1,2; bucket | device | avg_temp --------+--------+---------- CALL refresh_continuous_aggregate('cond_1', 1, 2); SELECT * FROM cond_1 ORDER BY 1,2; bucket | device | avg_temp --------+--------+---------- -- Refresh only the invalidated bucket CALL refresh_continuous_aggregate('cond_1', 0, 1); SELECT * FROM cagg_invals WHERE cagg_id = :cond_1_id; cagg_id | start | end ---------+----------------------+--------------------- 6 | -9223372036854775808 | -2 6 | 2 | 9223372036854775807 SELECT * FROM cond_1 ORDER BY 1,2; bucket | device | avg_temp --------+--------+---------- 0 | 1 | 1 -- Refresh 1 extra bucket on the left INSERT INTO conditions VALUES (0, 1, 2.0); CALL refresh_continuous_aggregate('cond_1', -1, 1); SELECT * FROM cond_1 ORDER BY 1,2; bucket | device | avg_temp --------+--------+---------- 0 | 1 | 1.5 -- Refresh 1 extra bucket on the right INSERT INTO conditions VALUES (0, 1, 3.0); CALL refresh_continuous_aggregate('cond_1', 0, 2); SELECT * FROM cond_1 ORDER BY 1,2; bucket | device | avg_temp --------+--------+---------- 0 | 1 | 2 -- Refresh 1 extra bucket on each side INSERT INTO conditions VALUES (0, 1, 4.0); CALL refresh_continuous_aggregate('cond_1', -1, 2); SELECT * FROM cond_1 ORDER BY 1,2; bucket | device | avg_temp --------+--------+---------- 0 | 1 | 2.5 -- Clear to reset aggregate TRUNCATE conditions; CALL refresh_continuous_aggregate('cond_1', NULL, NULL); -- Test invalidation of size 2 INSERT INTO conditions VALUES (0, 1, 1.0), (1, 1, 2.0); -- Refresh one bucket at a time CALL refresh_continuous_aggregate('cond_1', 0, 1); SELECT * FROM cond_1 ORDER BY 1,2; bucket | device | avg_temp --------+--------+---------- 0 | 1 | 1 CALL refresh_continuous_aggregate('cond_1', 1, 2); SELECT * FROM cond_1 ORDER BY 1,2; bucket | device | avg_temp --------+--------+---------- 0 | 1 | 1 1 | 1 | 2 -- Repeat the same thing but refresh the whole invalidation at once TRUNCATE conditions; CALL refresh_continuous_aggregate('cond_1', NULL, NULL); INSERT INTO conditions VALUES (0, 1, 1.0), (1, 1, 2.0); CALL refresh_continuous_aggregate('cond_1', 0, 2); SELECT * FROM cond_1 ORDER BY 1,2; bucket | device | avg_temp --------+--------+---------- 0 | 1 | 1 1 | 1 | 2 -- Test invalidation of size 3 TRUNCATE conditions; CALL refresh_continuous_aggregate('cond_1', NULL, NULL); INSERT INTO conditions VALUES (0, 1, 1.0), (1, 1, 2.0), (2, 1, 3.0); -- Invalidation extends beyond the refresh window on both ends CALL refresh_continuous_aggregate('cond_1', 1, 2); SELECT * FROM cond_1 ORDER BY 1,2; bucket | device | avg_temp --------+--------+---------- 1 | 1 | 2 -- Should leave one invalidation on each side of the refresh window SELECT * FROM cagg_invals WHERE cagg_id = :cond_1_id; cagg_id | start | end ---------+-------+--------------------- 6 | 0 | 0 6 | 2 | 2 6 | 110 | 9223372036854775807 -- Refresh the two remaining invalidations CALL refresh_continuous_aggregate('cond_1', 0, 1); SELECT * FROM cond_1 ORDER BY 1,2; bucket | device | avg_temp --------+--------+---------- 0 | 1 | 1 1 | 1 | 2 CALL refresh_continuous_aggregate('cond_1', 2, 3); SELECT * FROM cond_1 ORDER BY 1,2; bucket | device | avg_temp --------+--------+---------- 0 | 1 | 1 1 | 1 | 2 2 | 1 | 3 -- Clear and repeat but instead refresh the whole range in one go. The -- result should be the same as the three partial refreshes. Use -- DELETE instead of TRUNCATE to clear this time. DELETE FROM conditions; CALL refresh_continuous_aggregate('cond_1', NULL, NULL); INSERT INTO conditions VALUES (0, 1, 1.0), (1, 1, 2.0), (2, 1, 3.0); CALL refresh_continuous_aggregate('cond_1', 0, 3); SELECT * FROM cond_1 ORDER BY 1,2; bucket | device | avg_temp --------+--------+---------- 0 | 1 | 1 1 | 1 | 2 2 | 1 | 3 ---------------------------------------------- -- Test that invalidation threshold is capped ---------------------------------------------- CREATE table threshold_test (time int, value int); SELECT create_hypertable('threshold_test', 'time', chunk_time_interval => 4); create_hypertable ----------------------------- (7,public,threshold_test,t) SELECT set_integer_now_func('threshold_test', 'int_now'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW thresh_2 WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(2, time) AS bucket, max(value) AS max FROM threshold_test GROUP BY 1 WITH NO DATA; SELECT raw_hypertable_id AS thresh_hyper_id, mat_hypertable_id AS thresh_cagg_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'thresh_2' \gset -- There's no invalidation threshold initially SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id = :thresh_hyper_id ORDER BY 1,2; hypertable_id | watermark ---------------+------------- 7 | -2147483648 -- Test that threshold is initilized to min value when there's no data -- and we specify an infinite end. Note that the min value may differ -- depending on time type. CALL refresh_continuous_aggregate('thresh_2', 0, NULL); NOTICE: continuous aggregate "thresh_2" is already up-to-date SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id = :thresh_hyper_id ORDER BY 1,2; hypertable_id | watermark ---------------+------------- 7 | -2147483648 INSERT INTO threshold_test SELECT v, v FROM generate_series(1, 10) v; CALL refresh_continuous_aggregate('thresh_2', 0, 5); -- Threshold should move to end of the last refreshed bucket, which is -- the last bucket fully included in the window, i.e., the window -- shrinks to end of previous bucket. SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id = :thresh_hyper_id ORDER BY 1,2; hypertable_id | watermark ---------------+----------- 7 | 4 -- Refresh where both the start and end of the window is above the -- max data value CALL refresh_continuous_aggregate('thresh_2', 14, NULL); NOTICE: continuous aggregate "thresh_2" is already up-to-date SELECT watermark AS thresh_hyper_id_watermark FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id = :thresh_hyper_id \gset -- Refresh where we start from the current watermark to infinity CALL refresh_continuous_aggregate('thresh_2', :thresh_hyper_id_watermark, NULL); NOTICE: continuous aggregate "thresh_2" is already up-to-date -- Now refresh with max end of the window to test that the -- invalidation threshold is capped at the last bucket of data CALL refresh_continuous_aggregate('thresh_2', 0, NULL); SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id = :thresh_hyper_id ORDER BY 1,2; hypertable_id | watermark ---------------+----------- 7 | 12 -- Should not have processed invalidations beyond the invalidation -- threshold. SELECT * FROM cagg_invals WHERE cagg_id = :thresh_cagg_id; cagg_id | start | end ---------+----------------------+--------------------- 8 | -9223372036854775808 | -1 8 | 12 | 9223372036854775807 -- Check that things are properly materialized SELECT * FROM thresh_2 ORDER BY 1; bucket | max --------+----- 0 | 1 2 | 3 4 | 5 6 | 7 8 | 9 10 | 10 -- Delete the last data SELECT show_chunks AS chunk_to_drop FROM show_chunks('threshold_test') ORDER BY 1 DESC LIMIT 1 \gset DELETE FROM threshold_test WHERE time > 6; -- The last data in the hypertable is gone SELECT time_bucket(2, time) AS bucket, max(value) AS max FROM threshold_test GROUP BY 1 ORDER BY 1; bucket | max --------+----- 0 | 1 2 | 3 4 | 5 6 | 6 -- The aggregate still holds data SELECT * FROM thresh_2 ORDER BY 1; bucket | max --------+----- 0 | 1 2 | 3 4 | 5 6 | 7 8 | 9 10 | 10 -- Refresh the aggregate to bring it up-to-date CALL refresh_continuous_aggregate('thresh_2', 0, NULL); -- Data also gone from the aggregate SELECT * FROM thresh_2 ORDER BY 1; bucket | max --------+----- 0 | 1 2 | 3 4 | 5 6 | 6 -- The invalidation threshold remains the same SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id = :thresh_hyper_id ORDER BY 1,2; hypertable_id | watermark ---------------+----------- 7 | 12 -- Insert new data beyond the invalidation threshold to move it -- forward INSERT INTO threshold_test SELECT v, v FROM generate_series(7, 15) v; CALL refresh_continuous_aggregate('thresh_2', 0, NULL); -- Aggregate now updated to reflect newly aggregated data SELECT * FROM thresh_2 ORDER BY 1; bucket | max --------+----- 0 | 1 2 | 3 4 | 5 6 | 7 8 | 9 10 | 11 12 | 13 14 | 15 -- The invalidation threshold should have moved forward to the end of -- the new data SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id = :thresh_hyper_id ORDER BY 1,2; hypertable_id | watermark ---------------+----------- 7 | 16 -- The aggregate remains invalid beyond the invalidation threshold SELECT * FROM cagg_invals WHERE cagg_id = :thresh_cagg_id; cagg_id | start | end ---------+----------------------+--------------------- 8 | -9223372036854775808 | -1 8 | 16 | 9223372036854775807 ---------------------------------------------------------------------- -- Test that dropping a chunk invalidates the dropped region. First -- create another chunk so that we have two chunks. One of the chunks -- will be dropped. --------------------------------------------------------------------- INSERT INTO conditions VALUES (10, 1, 10.0); -- Chunks currently associated with the hypertable SELECT show_chunks AS chunk_to_drop FROM show_chunks('conditions'); chunk_to_drop ----------------------------------------- _timescaledb_internal._hyper_1_35_chunk _timescaledb_internal._hyper_1_41_chunk -- Pick the first one to drop SELECT show_chunks AS chunk_to_drop FROM show_chunks('conditions') ORDER BY 1 LIMIT 1 \gset -- Show the data before dropping one of the chunks SELECT * FROM conditions ORDER BY 1,2; time | device | temp ------+--------+------ 0 | 1 | 1 1 | 1 | 2 2 | 1 | 3 10 | 1 | 10 -- Drop one chunk DROP TABLE :chunk_to_drop; -- The chunk's data no longer exists in the hypertable SELECT * FROM conditions ORDER BY 1,2; time | device | temp ------+--------+------ 10 | 1 | 10 -- Aggregate still remains in continuous aggregate, however SELECT * FROM cond_1 ORDER BY 1,2; bucket | device | avg_temp --------+--------+---------- 0 | 1 | 1 1 | 1 | 2 2 | 1 | 3 -- Refresh the continuous aggregate to make the dropped data be -- reflected in the aggregate CALL refresh_continuous_aggregate('cond_1', NULL, NULL); -- Aggregate now up-to-date with the source hypertable SELECT * FROM cond_1 ORDER BY 1,2; bucket | device | avg_temp --------+--------+---------- 10 | 1 | 10 -- Test that adjacent invalidations are merged INSERT INTO conditions VALUES(1, 1, 1.0), (2, 1, 2.0); INSERT INTO conditions VALUES(3, 1, 1.0); INSERT INTO conditions VALUES(4, 1, 1.0); INSERT INTO conditions VALUES(6, 1, 1.0); CALL refresh_continuous_aggregate('cond_1', 10, NULL); NOTICE: continuous aggregate "cond_1" is already up-to-date SELECT * FROM cagg_invals WHERE cagg_id = :cond_1_id; cagg_id | start | end ---------+-------+--------------------- 6 | 1 | 4 6 | 6 | 6 6 | 110 | 9223372036854775807 --------------------------------------------------------------------- -- Test that single timestamp invalidations are expanded to buckets, -- and adjacent buckets merged. --------------------------------------------------------------------- -- First clear invalidations in a range: CALL refresh_continuous_aggregate('cond_10', -20, 60); -- The following three should be merged to one range 0-29 INSERT INTO conditions VALUES (5, 1, 1.0); INSERT INTO conditions VALUES (15, 1, 1.0); INSERT INTO conditions VALUES (25, 1, 1.0); -- The last one should not merge with the others INSERT INTO conditions VALUES (40, 1, 1.0); -- Refresh to process invalidations, but outside the range of -- invalidations we inserted so that we don't clear them. CALL refresh_continuous_aggregate('cond_10', 50, 60); NOTICE: continuous aggregate "cond_10" is already up-to-date SELECT mat_hypertable_id AS cond_10_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'cond_10' \gset SELECT * FROM cagg_invals WHERE cagg_id = :cond_10_id; cagg_id | start | end ---------+----------------------+--------------------- 3 | -9223372036854775808 | -21 3 | 0 | 29 3 | 40 | 49 3 | 60 | 9223372036854775807 -- should trigger two individual refreshes CALL refresh_continuous_aggregate('cond_10', 0, 200); -- Insert into every second bucket INSERT INTO conditions VALUES (20, 1, 1.0); INSERT INTO conditions VALUES (40, 1, 1.0); INSERT INTO conditions VALUES (60, 1, 1.0); INSERT INTO conditions VALUES (80, 1, 1.0); INSERT INTO conditions VALUES (100, 1, 1.0); INSERT INTO conditions VALUES (120, 1, 1.0); INSERT INTO conditions VALUES (140, 1, 1.0); CALL refresh_continuous_aggregate('cond_10', 0, 200); -- Test refresh with undefined invalidation threshold and variable sized buckets CREATE TABLE timestamp_ht ( time timestamptz NOT NULL, value float ); SELECT create_hypertable('timestamp_ht', 'time'); create_hypertable --------------------------- (9,public,timestamp_ht,t) CREATE MATERIALIZED VIEW temperature_4h WITH (timescaledb.continuous) AS SELECT time_bucket('4 hour', time), avg(value) FROM timestamp_ht GROUP BY 1 ORDER BY 1; NOTICE: continuous aggregate "temperature_4h" is already up-to-date -- We also treat time_buckets with an hourly interval that uses a time-zone -- as a variable see caggtimebucket_validate(). CREATE MATERIALIZED VIEW temperature_4h_2 WITH (timescaledb.continuous) AS SELECT time_bucket('4 hour', time, 'Europe/Berlin') AS bucket_4h, avg(value) AS average FROM timestamp_ht GROUP BY 1 ORDER BY 1; NOTICE: continuous aggregate "temperature_4h_2" is already up-to-date CREATE MATERIALIZED VIEW temperature_1month WITH (timescaledb.continuous) AS SELECT time_bucket('1 month', time), avg(value) FROM timestamp_ht GROUP BY 1 ORDER BY 1; NOTICE: continuous aggregate "temperature_1month" is already up-to-date CREATE MATERIALIZED VIEW temperature_1month_ts WITH (timescaledb.continuous) AS SELECT time_bucket('1 month', time, 'Europe/Berlin'), avg(value) FROM timestamp_ht GROUP BY 1 ORDER BY 1; NOTICE: continuous aggregate "temperature_1month_ts" is already up-to-date CREATE MATERIALIZED VIEW temperature_1month_hierarchical WITH (timescaledb.continuous) AS SELECT time_bucket('1 month', bucket_4h), avg(average) FROM temperature_4h_2 GROUP BY 1 ORDER BY 1; NOTICE: continuous aggregate "temperature_1month_hierarchical" is already up-to-date CREATE MATERIALIZED VIEW temperature_1month_hierarchical_ts WITH (timescaledb.continuous) AS SELECT time_bucket('1 month', bucket_4h, 'Europe/Berlin'), avg(average) FROM temperature_4h_2 GROUP BY 1 ORDER BY 1; NOTICE: continuous aggregate "temperature_1month_hierarchical_ts" is already up-to-date --------------------------------------------------------------------- --- Issue 5474 --------------------------------------------------------------------- CREATE TABLE i5474 ( time timestamptz NOT NULL, sensor_id integer NOT NULL, cpu double precision NOT NULL, temperature double precision NOT NULL); SELECT create_hypertable('i5474','time'); create_hypertable --------------------- (16,public,i5474,t) CREATE MATERIALIZED VIEW i5474_summary_daily WITH (timescaledb.continuous) AS SELECT time_bucket('1 hour', time, 'AWST') AS bucket, sensor_id, avg(cpu) AS avg_cpu FROM i5474 GROUP BY bucket, sensor_id; NOTICE: continuous aggregate "i5474_summary_daily" is already up-to-date SELECT add_continuous_aggregate_policy('i5474_summary_daily', start_offset => NULL, end_offset => INTERVAL '10 MINUTES', schedule_interval => INTERVAL '1 MINUTE' ) new_job_id \gset -- Check that start_offset = NULL is handled properly by the refresh job... CALL run_job(:new_job_id); -- ...and the CAgg can be refreshed afterward CALL refresh_continuous_aggregate('i5474_summary_daily', NULL, '2023-03-21 05:00:00+00'); NOTICE: continuous aggregate "i5474_summary_daily" is already up-to-date INSERT INTO i5474 (time, sensor_id, cpu, temperature) VALUES ('2000-01-01 05:00:00+00', 1, 1.0, 1.0); CALL refresh_continuous_aggregate('i5474_summary_daily', NULL, '2023-01-01 01:00:00+00'); -- CAgg should be up-to-date now CALL refresh_continuous_aggregate('i5474_summary_daily', NULL, '2023-01-01 01:00:00+00'); NOTICE: continuous aggregate "i5474_summary_daily" is already up-to-date -- -- Test the invalidation move function -- -- Make sure to move the threshold for the insertions we are going to -- do. CALL refresh_continuous_aggregate('measure_10', 0, 200); SELECT * FROM hypertable_invalidation_thresholds WHERE hypertable IN ('conditions'::regclass, 'measurements'::regclass); hypertable | threshold --------------+----------- conditions | 200 measurements | 200 SELECT * FROM hyper_invals; hyper_id | start | end ----------+-------+----- -- Save away the contents of some materialized views so that we can -- check that they are not updated when we move invalidations. SELECT * INTO saved_measure_10 FROM measure_10; SELECT * INTO saved_cond_10 FROM cond_10; -- Generate some invalidations for the hypertables INSERT INTO conditions VALUES (110, 14, 23.7); INSERT INTO conditions VALUES (110, 15, 23.8), (119, 3, 23.6); INSERT INTO conditions VALUES (160, 13, 23.7), (170, 4, 23.7); INSERT INTO measurements VALUES (120, 14, 23.7); INSERT INTO measurements VALUES (130, 15, 23.8), (180, 3, 23.6); -- test direct compress insert invalidation CREATE TABLE direct_compress_insert(time timestamptz) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column INSERT INTO direct_compress_insert SELECT '2025-01-01'; CREATE MATERIALIZED VIEW cagg_insert WITH (tsdb.continuous) AS SELECT time_bucket('1day', time) FROM direct_compress_insert GROUP BY 1; NOTICE: refreshing continuous aggregate "cagg_insert" SET timescaledb.enable_direct_compress_insert = true; EXPLAIN (analyze,buffers off,costs off,timing off,summary off) INSERT INTO direct_compress_insert SELECT '2024-01-01'::timestamptz + format('%sm',i)::interval FROM generate_series(1,1000) g(i); --- QUERY PLAN --- Custom Scan (ModifyHypertable) (actual rows=0.00 loops=1) Direct Compress: true -> Insert on direct_compress_insert (actual rows=0.00 loops=1) -> Function Scan on generate_series g (actual rows=1000.00 loops=1) EXPLAIN (analyze,buffers off,costs off,timing off,summary off) INSERT INTO direct_compress_insert SELECT '2024-01-01'::timestamptz - format('%sm',i)::interval FROM generate_series(1,1000) g(i); --- QUERY PLAN --- Custom Scan (ModifyHypertable) (actual rows=0.00 loops=1) Direct Compress: true -> Insert on direct_compress_insert (actual rows=0.00 loops=1) -> Function Scan on generate_series g (actual rows=1000.00 loops=1) -- should have 2 entries SELECT _timescaledb_functions.to_timestamp(lowest_modified_value) start, _timescaledb_functions.to_timestamp(greatest_modified_value) end from _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log WHERE hypertable_id = 18 ORDER BY 1,2; start | end ------------------------+------------------------ 2023-12-31 07:20:00+00 | 2023-12-31 23:59:00+00 2024-01-01 00:01:00+00 | 2024-01-01 16:40:00+00 EXPLAIN (analyze,buffers off,costs off,timing off,summary off) INSERT INTO direct_compress_insert SELECT '2023-12-31'::timestamptz + format('%sm',i)::interval FROM generate_series(1,2000) g(i); --- QUERY PLAN --- Custom Scan (ModifyHypertable) (actual rows=0.00 loops=1) Direct Compress: true -> Insert on direct_compress_insert (actual rows=0.00 loops=1) -> Function Scan on generate_series g (actual rows=2000.00 loops=1) -- should have 3 entries SELECT _timescaledb_functions.to_timestamp(lowest_modified_value) start, _timescaledb_functions.to_timestamp(greatest_modified_value) end from _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log WHERE hypertable_id = 18 ORDER BY 1,2; start | end ------------------------+------------------------ 2023-12-31 00:01:00+00 | 2024-01-01 09:20:00+00 2023-12-31 07:20:00+00 | 2023-12-31 23:59:00+00 2024-01-01 00:01:00+00 | 2024-01-01 16:40:00+00 -- should have 1 uncompressed and 1 compressed chunk EXPLAIN (costs off,timing off,summary off) SELECT FROM direct_compress_insert; --- QUERY PLAN --- Append -> Seq Scan on _hyper_18_59_chunk -> Custom Scan (ColumnarScan) on _hyper_18_61_chunk -> Seq Scan on compress_hyper_19_62_chunk RESET timescaledb.enable_direct_compress_insert; -- test direct compress copy invalidation CREATE TABLE direct_compress_copy(time timestamptz) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column INSERT INTO direct_compress_copy SELECT '2025-01-01'; CREATE MATERIALIZED VIEW cagg_copy WITH (tsdb.continuous) AS SELECT time_bucket('1day', time) FROM direct_compress_copy GROUP BY 1; NOTICE: refreshing continuous aggregate "cagg_copy" SET timescaledb.enable_direct_compress_copy = true; COPY direct_compress_copy FROM STDIN; -- should have 1 entries now SELECT _timescaledb_functions.to_timestamp(lowest_modified_value) start, _timescaledb_functions.to_timestamp(greatest_modified_value) end from _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log WHERE hypertable_id = 21 ORDER BY 1,2; start | end ------------------------+------------------------ 2023-01-01 00:00:00+00 | 2023-01-03 00:00:00+00 COPY direct_compress_copy FROM STDIN; -- should have 2 entries now SELECT _timescaledb_functions.to_timestamp(lowest_modified_value) start, _timescaledb_functions.to_timestamp(greatest_modified_value) end from _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log WHERE hypertable_id = 21 ORDER BY 1,2; start | end ------------------------+------------------------ 2022-12-31 00:00:00+00 | 2023-01-03 00:00:00+00 2023-01-01 00:00:00+00 | 2023-01-03 00:00:00+00 -- range spanning multiple chunks COPY direct_compress_copy FROM STDIN; -- should have 3 entries now SELECT _timescaledb_functions.to_timestamp(lowest_modified_value) start, _timescaledb_functions.to_timestamp(greatest_modified_value) end from _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log WHERE hypertable_id = 21 ORDER BY 1,2; start | end ------------------------+------------------------ 2022-01-01 00:00:00+00 | 2022-01-01 00:00:00+00 2022-02-28 00:00:00+00 | 2022-02-28 00:00:00+00 2022-12-31 00:00:00+00 | 2023-01-03 00:00:00+00 2023-01-01 00:00:00+00 | 2023-01-03 00:00:00+00 -- should have 1 uncompressed and 3 compressed chunk EXPLAIN (costs off,timing off,summary off) SELECT FROM direct_compress_copy; --- QUERY PLAN --- Append -> Seq Scan on _hyper_21_63_chunk -> Custom Scan (ColumnarScan) on _hyper_21_65_chunk -> Seq Scan on compress_hyper_22_66_chunk -> Custom Scan (ColumnarScan) on _hyper_21_67_chunk -> Seq Scan on compress_hyper_22_68_chunk -> Custom Scan (ColumnarScan) on _hyper_21_69_chunk -> Seq Scan on compress_hyper_22_70_chunk RESET timescaledb.enable_direct_compress_copy; -- test direct compress invalidation with custom partitioning function (not supported atm) CREATE OR REPLACE FUNCTION f_month(timestamptz) returns int language sql AS $$ SELECT 12 * extract(year from $1) + extract(month from $1);$$ immutable; CREATE TABLE part_cagg (time timestamptz); SELECT create_hypertable('part_cagg', 'time', time_partitioning_func => 'f_month', chunk_time_interval => 1); create_hypertable ------------------------- (24,public,part_cagg,t) \set ON_ERROR_STOP 0 CREATE MATERIALIZED VIEW part_cagg1 WITH (tsdb.continuous) AS SELECT time_bucket('1day', time) FROM part_cagg GROUP BY 1; ERROR: custom partitioning functions not supported with continuous aggregates \set ON_ERROR_STOP 1 -- test UPDATE invalidation CREATE TABLE inval_update(time timestamptz) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column INSERT INTO inval_update SELECT '2025-01-01'; CREATE MATERIALIZED VIEW cagg_inval_update WITH (tsdb.continuous) AS SELECT time_bucket('1day', time) FROM inval_update GROUP BY 1; NOTICE: refreshing continuous aggregate "cagg_inval_update" -- check setting to NULL is handled gracefully \set ON_ERROR_STOP 0 UPDATE inval_update SET time = NULL WHERE time = '2025-01-01'; ERROR: null value in column "time" of relation "_hyper_25_71_chunk" violates not-null constraint \set ON_ERROR_STOP 1 UPDATE inval_update SET time = '2025-01-01 00:00:23' WHERE time = '2025-01-01'; -- should have 1 entries now SELECT _timescaledb_functions.to_timestamp(lowest_modified_value) start, _timescaledb_functions.to_timestamp(greatest_modified_value) end from _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log WHERE hypertable_id = 25 ORDER BY 1,2; start | end ------------------------+------------------------ 2025-01-01 00:00:00+00 | 2025-01-01 00:00:23+00 ------------------------------------------------------------------------------------------ --Test that invalidation's greatest_modified value are handle correctly for variable bucket ------------------------------------------------------------------------------------------- CREATE TABLE test_data ( time TIMESTAMPTZ NOT NULL, value INT ); SELECT public.create_hypertable( relation => 'test_data', time_column_name => 'time', chunk_time_interval => interval '1 months' ); create_hypertable ------------------------- (28,public,test_data,t) -- Insert initial data INSERT INTO test_data SELECT time, 1 FROM generate_series('2024-01-01'::timestamptz, '2024-12-31'::timestamptz, '1 day'::interval) time; -- Create continuous aggregate with variable bucket CREATE MATERIALIZED VIEW test_cagg WITH (timescaledb.continuous) AS SELECT time_bucket('1 month'::interval, time) AS bucket, count(*) as count FROM test_data GROUP BY bucket WITH NO DATA; --create continuous aggregate with fixed bucket with offset CREATE MATERIALIZED VIEW test_cagg_1d_offset WITH (timescaledb.continuous) AS SELECT time_bucket('1 day'::interval, time, "offset" => INTERVAL '18 hours') AS bucket, count(*) as count FROM test_data GROUP BY bucket WITH NO DATA; SET timezone = 'UTC'; --Do the first refresh and check materialization invalidation log call refresh_continuous_aggregate ('test_cagg','2023-12-29 15:00:00', '2026-01-28 15:00:00'); call refresh_continuous_aggregate ('test_cagg_1d_offset','2023-12-29 18:00:00', '2024-01-01 18:00:00'); SELECT materialization_id, _timescaledb_functions.to_timestamp(lowest_modified_value) as low, _timescaledb_functions.to_timestamp(greatest_modified_value) as high FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'test_cagg') ORDER BY low; materialization_id | low | high --------------------+------------------------+------------------------------- 29 | -infinity | 2023-12-31 23:59:59.999999+00 29 | 2026-01-01 00:00:00+00 | infinity SELECT CASE WHEN lowest_modified_value <= _timescaledb_functions.get_internal_time_min('timestamptz'::regtype) THEN '-infinity'::timestamptz ELSE _timescaledb_functions.to_timestamp(lowest_modified_value) END AS low, CASE WHEN greatest_modified_value >= _timescaledb_functions.get_internal_time_max('timestamptz'::regtype) THEN 'infinity'::timestamptz ELSE _timescaledb_functions.to_timestamp(greatest_modified_value) END AS high FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id = ( SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'test_cagg_1d_offset' ) ORDER BY lowest_modified_value, greatest_modified_value; low | high ------------------------+------------------------------- -infinity | 2023-12-29 17:59:59.999999+00 2024-01-01 18:00:00+00 | infinity --now do the same refresh again, it should say the cagg is already up to date CALL refresh_continuous_aggregate ('test_cagg','2023-12-29 15:00:00', '2026-01-28 15:00:00'); NOTICE: continuous aggregate "test_cagg" is already up-to-date CALL refresh_continuous_aggregate ('test_cagg','2023-12-29 15:00:00', '2026-01-28 15:00:00'); NOTICE: continuous aggregate "test_cagg" is already up-to-date --Insert data to test that invalidation is moved correctly from hypertable invalidation log -- to materialization invalidation log INSERT INTO test_data SELECT time, 2 FROM generate_series('2024-01-01'::timestamptz, '2024-12-31'::timestamptz, '10 day'::interval) time; --Refresh some first part of the updated range. The range show up in the materialization log --should end with the last timestamp of the bucket (i.e., has the .999999 at the end), --rather than the start of the next bucket CALL refresh_continuous_aggregate ('test_cagg','2023-12-29 15:00:00', '2024-03-15 15:00:00'); SELECT materialization_id, _timescaledb_functions.to_timestamp(lowest_modified_value) as low, _timescaledb_functions.to_timestamp(greatest_modified_value) as high FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'test_cagg') ORDER BY low; materialization_id | low | high --------------------+------------------------+------------------------------- 29 | -infinity | 2023-12-31 23:59:59.999999+00 29 | 2024-03-01 00:00:00+00 | 2024-12-31 23:59:59.999999+00 29 | 2026-01-01 00:00:00+00 | infinity --should see the invalidation ranges with offset (i.e,boundary at the 18hour) SELECT CASE WHEN lowest_modified_value <= _timescaledb_functions.get_internal_time_min('timestamptz'::regtype) THEN '-infinity'::timestamptz ELSE _timescaledb_functions.to_timestamp(lowest_modified_value) END AS low, CASE WHEN greatest_modified_value >= _timescaledb_functions.get_internal_time_max('timestamptz'::regtype) THEN 'infinity'::timestamptz ELSE _timescaledb_functions.to_timestamp(greatest_modified_value) END AS high FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id = ( SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'test_cagg_1d_offset' ) ORDER BY lowest_modified_value, greatest_modified_value; low | high ------------------------+------------------------------- -infinity | 2023-12-29 17:59:59.999999+00 2023-12-31 18:00:00+00 | 2024-01-11 17:59:59.999999+00 2024-01-01 18:00:00+00 | infinity 2024-01-20 18:00:00+00 | 2024-02-10 17:59:59.999999+00 2024-02-19 18:00:00+00 | 2024-03-11 17:59:59.999999+00 2024-03-20 18:00:00+00 | 2024-04-10 17:59:59.999999+00 2024-04-19 18:00:00+00 | 2024-05-10 17:59:59.999999+00 2024-05-19 18:00:00+00 | 2024-06-09 17:59:59.999999+00 2024-06-18 18:00:00+00 | 2024-07-09 17:59:59.999999+00 2024-07-18 18:00:00+00 | 2024-08-08 17:59:59.999999+00 2024-08-17 18:00:00+00 | 2024-09-07 17:59:59.999999+00 2024-09-16 18:00:00+00 | 2024-10-07 17:59:59.999999+00 2024-10-16 18:00:00+00 | 2024-11-06 17:59:59.999999+00 2024-11-15 18:00:00+00 | 2024-12-06 17:59:59.999999+00 2024-12-15 18:00:00+00 | 2024-12-26 17:59:59.999999+00 --test that offset was accounted for in invalidation threshold when refresh the offset cagg to NULL SELECT _timescaledb_functions.to_timestamp(watermark) as invalidation_threshold FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id IN ( SELECT raw_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'test_cagg_1d_offset'); invalidation_threshold ------------------------ 2026-01-01 00:00:00+00 INSERT INTO test_data values ('2026-01-05 00:00:00', 1); CALL refresh_continuous_aggregate ('test_cagg_1d_offset','2023-12-29 15:00:00', NULL); --should be at the 18th hour SELECT _timescaledb_functions.to_timestamp(watermark) as invalidation_threshold FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id IN ( SELECT raw_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'test_cagg_1d_offset'); invalidation_threshold ------------------------ 2026-01-05 18:00:00+00 --clean up DROP TABLE test_data CASCADE; NOTICE: drop cascades to 4 other objects NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 3 other objects RESET timezone; ================================================ FILE: tsl/test/expected/cagg_invalidation_variable_bucket.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- Tests for continuous aggregate invalidation with variable-sized buckets \c :TEST_DBNAME :ROLE_SUPERUSER SET ROLE :ROLE_DEFAULT_PERM_USER; SET datestyle TO 'ISO, YMD'; SET timezone TO 'UTC'; CREATE VIEW hyper_inval_log AS SELECT ht.schema_name || '.' || ht.table_name AS hypertable, _timescaledb_functions.to_timestamp(lowest_modified_value) AS inval_start, _timescaledb_functions.to_timestamp(greatest_modified_value) AS inval_end FROM _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log hil JOIN _timescaledb_catalog.hypertable ht ON ht.id = hil.hypertable_id ORDER BY 1, 2, 3; CREATE VIEW cagg_inval_log AS SELECT ca.user_view_name AS cagg_name, _timescaledb_functions.to_timestamp(mil.lowest_modified_value) AS inval_start, _timescaledb_functions.to_timestamp(mil.greatest_modified_value) AS inval_end FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log mil JOIN _timescaledb_catalog.continuous_agg ca ON ca.mat_hypertable_id = mil.materialization_id ORDER BY 1, 2, 3; ----------------------------------------------------------------------- -- SECTION 1: Monthly buckets with varying month lengths -- Tests that invalidations are correctly processed for variable-width -- buckets. ----------------------------------------------------------------------- CREATE TABLE monthly_data ( time TIMESTAMPTZ NOT NULL, device INT, value FLOAT ); SELECT create_hypertable('monthly_data', 'time', chunk_time_interval => INTERVAL '1 month'); create_hypertable --------------------------- (1,public,monthly_data,t) -- Create a 1-month bucket cagg CREATE MATERIALIZED VIEW cagg_monthly WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT time_bucket('1 month'::interval, time) AS bucket, device, count(*) AS cnt FROM monthly_data GROUP BY 1, 2 WITH NO DATA; -- Insert data spanning 12 months of 2024 (leap year) INSERT INTO monthly_data SELECT ts, 1, extract(epoch FROM ts)::int % 100 FROM generate_series('2024-01-01 00:00:00'::timestamptz, '2024-12-31 23:59:59'::timestamptz, '1 day'::interval) ts; CALL refresh_continuous_aggregate('cagg_monthly', '2024-01-01 00:00:00', '2025-01-01 00:00:00'); -- Verify data is materialized SELECT bucket, cnt FROM cagg_monthly ORDER BY bucket; bucket | cnt ------------------------+----- 2024-01-01 00:00:00+00 | 31 2024-02-01 00:00:00+00 | 29 2024-03-01 00:00:00+00 | 31 2024-04-01 00:00:00+00 | 30 2024-05-01 00:00:00+00 | 31 2024-06-01 00:00:00+00 | 30 2024-07-01 00:00:00+00 | 31 2024-08-01 00:00:00+00 | 31 2024-09-01 00:00:00+00 | 30 2024-10-01 00:00:00+00 | 31 2024-11-01 00:00:00+00 | 30 2024-12-01 00:00:00+00 | 31 -- No invalidations should remain after full refresh SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_monthly'; cagg_name | inval_start | inval_end --------------+------------------------+------------------------------- cagg_monthly | -infinity | 2023-12-31 23:59:59.999999+00 cagg_monthly | 2025-01-01 00:00:00+00 | infinity ----------------------------------------------------------------------- -- Test 1a: Invalidation in February (28/29 day month) of a leap year -- February 2024 has 29 days. ----------------------------------------------------------------------- INSERT INTO monthly_data VALUES ('2024-02-15 12:00:00', 1, 999.0); SELECT * FROM hyper_inval_log; hypertable | inval_start | inval_end ---------------------+------------------------+------------------------ public.monthly_data | 2024-02-15 12:00:00+00 | 2024-02-15 12:00:00+00 -- Refresh only February CALL refresh_continuous_aggregate('cagg_monthly', '2024-02-01 00:00:00', '2024-03-01 00:00:00'); SELECT bucket, cnt FROM cagg_monthly WHERE bucket = '2024-02-01'; bucket | cnt ------------------------+----- 2024-02-01 00:00:00+00 | 30 -- No invalidation should remain for February SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_monthly'; cagg_name | inval_start | inval_end --------------+------------------------+------------------------------- cagg_monthly | -infinity | 2023-12-31 23:59:59.999999+00 cagg_monthly | 2025-01-01 00:00:00+00 | infinity ----------------------------------------------------------------------- -- Test 1b: Invalidation at the exact boundary between Feb 29 and Mar 1 ----------------------------------------------------------------------- -- Insert at the very last moment of Feb 29 INSERT INTO monthly_data VALUES ('2024-02-29 23:59:59.999999', 1, 888.0); -- Insert at the very first moment of Mar 1 INSERT INTO monthly_data VALUES ('2024-03-01 00:00:00', 1, 777.0); SELECT * FROM hyper_inval_log; hypertable | inval_start | inval_end ---------------------+-------------------------------+------------------------------- public.monthly_data | 2024-02-29 23:59:59.999999+00 | 2024-02-29 23:59:59.999999+00 public.monthly_data | 2024-03-01 00:00:00+00 | 2024-03-01 00:00:00+00 -- Refresh February only CALL refresh_continuous_aggregate('cagg_monthly', '2024-02-01 00:00:00', '2024-03-01 00:00:00'); -- The remaining invalidation should only cover March SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_monthly'; cagg_name | inval_start | inval_end --------------+------------------------+------------------------------- cagg_monthly | -infinity | 2023-12-31 23:59:59.999999+00 cagg_monthly | 2024-03-01 00:00:00+00 | 2024-03-31 23:59:59.999999+00 cagg_monthly | 2025-01-01 00:00:00+00 | infinity -- Now refresh March CALL refresh_continuous_aggregate('cagg_monthly', '2024-03-01 00:00:00', '2024-04-01 00:00:00'); -- No invalidations should remain SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_monthly'; cagg_name | inval_start | inval_end --------------+------------------------+------------------------------- cagg_monthly | -infinity | 2023-12-31 23:59:59.999999+00 cagg_monthly | 2025-01-01 00:00:00+00 | infinity ----------------------------------------------------------------------- -- Test 1c: Invalidation spanning multiple months of different lengths ----------------------------------------------------------------------- -- Insert one value in each month INSERT INTO monthly_data VALUES ('2024-02-29 23:59:59', 1, 100.0); -- 29-day INSERT INTO monthly_data VALUES ('2024-03-31 12:00:00', 1, 200.0); -- 31-day INSERT INTO monthly_data VALUES ('2024-04-30 23:59:59', 1, 300.0); -- 30-day -- Refresh with a window that partially covers all three months. CALL refresh_continuous_aggregate('cagg_monthly', '2024-02-15 00:00:00', '2024-04-15 00:00:00'); SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_monthly'; cagg_name | inval_start | inval_end --------------+------------------------+------------------------------- cagg_monthly | -infinity | 2023-12-31 23:59:59.999999+00 cagg_monthly | 2024-02-01 00:00:00+00 | 2024-02-29 23:59:59.999999+00 cagg_monthly | 2024-04-01 00:00:00+00 | 2024-04-30 23:59:59.999999+00 cagg_monthly | 2025-01-01 00:00:00+00 | infinity -- Refresh the whole window to clear all invalidations CALL refresh_continuous_aggregate('cagg_monthly', '2024-02-01 00:00:00', '2024-05-01 00:00:00'); SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_monthly'; cagg_name | inval_start | inval_end --------------+------------------------+------------------------------- cagg_monthly | -infinity | 2023-12-31 23:59:59.999999+00 cagg_monthly | 2025-01-01 00:00:00+00 | infinity ----------------------------------------------------------------------- -- Test 1d: Non-leap year February (28 days) ----------------------------------------------------------------------- INSERT INTO monthly_data SELECT ts, 2, 50.0 FROM generate_series('2025-02-01 00:00:00'::timestamptz, '2025-02-28 23:59:59'::timestamptz, '1 day'::interval) ts; CALL refresh_continuous_aggregate('cagg_monthly', '2025-02-01 00:00:00', '2025-03-01 00:00:00'); -- Verify Feb 2025 bucket has correct number of days SELECT bucket, cnt FROM cagg_monthly WHERE device = 2 AND bucket = '2025-02-01 00:00:00'; bucket | cnt ------------------------+----- 2025-02-01 00:00:00+00 | 28 -- Insert at Feb 28 boundary INSERT INTO monthly_data VALUES ('2025-02-28 23:59:59.999999', 2, 999.0); CALL refresh_continuous_aggregate('cagg_monthly', '2025-02-01 00:00:00', '2025-03-01 00:00:00'); SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_monthly'; cagg_name | inval_start | inval_end --------------+------------------------+------------------------------- cagg_monthly | -infinity | 2023-12-31 23:59:59.999999+00 cagg_monthly | 2025-01-01 00:00:00+00 | 2025-01-31 23:59:59.999999+00 cagg_monthly | 2025-03-01 00:00:00+00 | infinity ----------------------------------------------------------------------- -- SECTION 2: Yearly buckets with leap year crossing -- Tests year-length variability (365 vs 366 days) and the -- 30-day x 12 = 360-day approximation in bucket_width. ----------------------------------------------------------------------- CREATE TABLE yearly_data ( time TIMESTAMPTZ NOT NULL, value FLOAT ); SELECT create_hypertable('yearly_data', 'time', chunk_time_interval => INTERVAL '1 year'); create_hypertable -------------------------- (3,public,yearly_data,t) CREATE MATERIALIZED VIEW cagg_yearly WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT time_bucket('1 year'::interval, time) AS bucket, count(*) AS cnt FROM yearly_data GROUP BY 1 WITH NO DATA; INSERT INTO yearly_data SELECT ts, extract(epoch FROM ts)::int % 1000 FROM generate_series('2024-01-01 00:00:00'::timestamptz, '2025-12-31 00:00:00'::timestamptz, '1 day'::interval) ts; -- Verify each year bucket has the right number of rows CALL refresh_continuous_aggregate('cagg_yearly', '2024-01-01 00:00:00', '2026-01-01 00:00:00'); SELECT bucket, cnt FROM cagg_yearly ORDER BY bucket; bucket | cnt ------------------------+----- 2024-01-01 00:00:00+00 | 366 2025-01-01 00:00:00+00 | 365 ----------------------------------------------------------------------- -- Test 2a: Invalidation crossing year boundary ----------------------------------------------------------------------- INSERT INTO yearly_data VALUES ('2023-12-31 23:59:59.999999', 1111.0); INSERT INTO yearly_data VALUES ('2024-01-01 00:00:00', 2222.0); -- Check that both years are invalidated SELECT * FROM hyper_inval_log; hypertable | inval_start | inval_end --------------------+-------------------------------+------------------------------- public.yearly_data | 2023-12-31 23:59:59.999999+00 | 2023-12-31 23:59:59.999999+00 public.yearly_data | 2024-01-01 00:00:00+00 | 2024-01-01 00:00:00+00 -- Refresh only 2023 - should leave 2024 invalidation in the log CALL refresh_continuous_aggregate('cagg_yearly', '2023-01-01 00:00:00', '2024-01-01 00:00:00'); SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_yearly'; cagg_name | inval_start | inval_end -------------+------------------------+------------------------------- cagg_yearly | -infinity | 2022-12-31 23:59:59.999999+00 cagg_yearly | 2024-01-01 00:00:00+00 | 2024-12-31 23:59:59.999999+00 cagg_yearly | 2026-01-01 00:00:00+00 | infinity CALL refresh_continuous_aggregate('cagg_yearly', '2024-01-01 00:00:00', '2025-01-01 00:00:00'); SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_yearly'; cagg_name | inval_start | inval_end -------------+------------------------+------------------------------- cagg_yearly | -infinity | 2022-12-31 23:59:59.999999+00 cagg_yearly | 2026-01-01 00:00:00+00 | infinity ----------------------------------------------------------------------- -- SECTION 3: DST transitions with timezone-aware monthly buckets -- Tests that bucket boundaries are correct during spring-forward -- and fall-back DST changes. ----------------------------------------------------------------------- SET timezone TO 'Europe/Berlin'; CREATE TABLE dst_data ( time TIMESTAMPTZ NOT NULL, value FLOAT ); SELECT create_hypertable('dst_data', 'time', chunk_time_interval => INTERVAL '1 month'); create_hypertable ----------------------- (5,public,dst_data,t) -- Daily bucket with Europe/Berlin timezone (DST transitions in March and October) CREATE MATERIALIZED VIEW cagg_dst_daily WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT time_bucket('1 day'::interval, time, 'Europe/Berlin') AS bucket, count(*) AS cnt FROM dst_data GROUP BY 1 WITH NO DATA; -- Insert data around March 2025 DST spring-forward (Mar 30, 2025 at 2:00 AM Europe/Berlin) INSERT INTO dst_data SELECT ts, 1.0 FROM generate_series('2025-03-30 00:00:00'::timestamptz, '2025-03-31 23:59:59.999999'::timestamptz, '1 hour'::interval) ts; CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-03-01 00:00:00', '2025-05-01 00:00:00'); -- March 30 should have 23 hours SELECT bucket, cnt FROM cagg_dst_daily ORDER BY bucket; bucket | cnt ------------------------+----- 2025-03-30 00:00:00+01 | 23 2025-03-31 00:00:00+02 | 24 ----------------------------------------------------------------------- -- Test 3a: Fall-back DST transition (October 2025) -- Oct 26, 2025 at 3:00 AM Europe/Berlin becomes 2:00 AM (repeated hour) ----------------------------------------------------------------------- INSERT INTO dst_data SELECT ts, 2.0 FROM generate_series('2025-10-26 00:00:00'::timestamptz, '2025-10-27 23:59:59.999999'::timestamptz, '1 hour'::interval) ts; -- Wide window to cover all DST-shifted buckets CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-10-01 00:00:00', '2026-12-01 00:00:00'); -- October bucket should have extra hour (25-hour day on Oct 26) SELECT bucket, cnt FROM cagg_dst_daily WHERE bucket >= '2025-10-01 00:00:00' AND bucket < '2026-01-01 00:00:00' ORDER BY bucket; bucket | cnt ------------------------+----- 2025-10-26 00:00:00+02 | 25 2025-10-27 00:00:00+01 | 24 -- Insert near the fall-back boundary INSERT INTO dst_data VALUES ('2025-10-26 01:00:00', 888.0); -- 2:00 AM Europe/Berlin (after fall-back) INSERT INTO dst_data VALUES ('2025-10-26 00:30:00', 777.0); -- 2:30 AM Europe/Berlin (before fall-back) CALL refresh_continuous_aggregate('cagg_dst_daily', '2025-09-01 00:00:00', '2026-02-01 00:00:00'); SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_dst_daily'; cagg_name | inval_start | inval_end ----------------+------------------------+------------------------------- cagg_dst_daily | -infinity | 2025-02-28 23:59:59.999999+01 cagg_dst_daily | 2025-05-01 00:00:00+02 | 2025-08-31 23:59:59.999999+02 cagg_dst_daily | 2026-12-01 00:00:00+01 | infinity SET timezone TO 'UTC'; ----------------------------------------------------------------------- -- SECTION 4: Two-month buckets -- Tests 2-month intervals where pairs of months have different totals: -- Jan+Feb: 59-60 days, Mar+Apr: 61, May+Jun: 61, Jul+Aug: 62, -- Sep+Oct: 61, Nov+Dec: 61 ----------------------------------------------------------------------- CREATE TABLE bimonthly_data ( time TIMESTAMPTZ NOT NULL, value INT ); SELECT create_hypertable('bimonthly_data', 'time', chunk_time_interval => INTERVAL '1 month'); create_hypertable ----------------------------- (7,public,bimonthly_data,t) CREATE MATERIALIZED VIEW cagg_bimonthly WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT time_bucket('2 months'::interval, time) AS bucket, count(*) AS cnt FROM bimonthly_data GROUP BY 1 WITH NO DATA; INSERT INTO bimonthly_data SELECT ts, 1 FROM generate_series('2025-01-01 00:00:00'::timestamptz, '2025-12-31 00:00:00'::timestamptz, '1 day'::interval) ts; CALL refresh_continuous_aggregate('cagg_bimonthly', '2025-01-01 00:00:00', '2025-12-31 00:00:00'); SELECT bucket, cnt FROM cagg_bimonthly ORDER BY bucket; bucket | cnt ------------------------+----- 2025-01-01 00:00:00+00 | 59 2025-03-01 00:00:00+00 | 61 2025-05-01 00:00:00+00 | 61 2025-07-01 00:00:00+00 | 62 2025-09-01 00:00:00+00 | 61 ----------------------------------------------------------------------- -- Test 4a: Invalidation at the boundary between 2-month buckets -- (Feb 29 / Mar 1 boundary in leap year, also the JanFeb/MarApr bucket boundary) ----------------------------------------------------------------------- INSERT INTO bimonthly_data VALUES ('2024-02-29 23:59:59.999999', 999); INSERT INTO bimonthly_data VALUES ('2024-03-01 00:00:00', 888); -- Refresh only the Jan-Feb bucket CALL refresh_continuous_aggregate('cagg_bimonthly', '2024-01-01 00:00:00', '2024-03-01 00:00:00'); -- Mar-Apr invalidation should remain SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_bimonthly'; cagg_name | inval_start | inval_end ----------------+------------------------+------------------------------- cagg_bimonthly | -infinity | 2023-12-31 23:59:59.999999+00 cagg_bimonthly | 2024-03-01 00:00:00+00 | 2024-12-31 23:59:59.999999+00 cagg_bimonthly | 2025-11-01 00:00:00+00 | infinity -- Refresh Mar-Apr CALL refresh_continuous_aggregate('cagg_bimonthly', '2024-03-01 00:00:00', '2024-05-01 00:00:00'); SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_bimonthly'; cagg_name | inval_start | inval_end ----------------+------------------------+------------------------------- cagg_bimonthly | -infinity | 2023-12-31 23:59:59.999999+00 cagg_bimonthly | 2024-05-01 00:00:00+00 | 2024-12-31 23:59:59.999999+00 cagg_bimonthly | 2025-11-01 00:00:00+00 | infinity ----------------------------------------------------------------------- -- SECTION 5: Variable-width buckets with offset -- Tests that invalidations are correctly processed when variable-width -- buckets are shifted by an offset. ----------------------------------------------------------------------- CREATE TABLE offset_data ( time TIMESTAMPTZ NOT NULL, device INT, value FLOAT ); SELECT create_hypertable('offset_data', 'time', chunk_time_interval => INTERVAL '1 month'); create_hypertable -------------------------- (9,public,offset_data,t) -- Create a 1-month bucket cagg with a 15-day offset CREATE MATERIALIZED VIEW cagg_month_offset WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT time_bucket('1 month'::interval, time, "offset" := INTERVAL '15 days') AS bucket, device, count(*) AS cnt FROM offset_data GROUP BY 1, 2 WITH NO DATA; -- Insert data spanning 12 months of 2024 (leap year) INSERT INTO offset_data SELECT ts, 1, extract(epoch FROM ts)::int % 100 FROM generate_series('2024-01-01 00:00:00'::timestamptz, '2024-12-31 23:59:59'::timestamptz, '1 day'::interval) ts; CALL refresh_continuous_aggregate('cagg_month_offset', '2024-01-01 00:00:00', '2025-01-01 00:00:00'); -- Buckets should be shifted to the 16th of each month: SELECT bucket, cnt FROM cagg_month_offset ORDER BY bucket; bucket | cnt ------------------------+----- 2024-01-16 00:00:00+00 | 31 2024-02-16 00:00:00+00 | 29 2024-03-16 00:00:00+00 | 31 2024-04-16 00:00:00+00 | 30 2024-05-16 00:00:00+00 | 31 2024-06-16 00:00:00+00 | 30 2024-07-16 00:00:00+00 | 31 2024-08-16 00:00:00+00 | 31 2024-09-16 00:00:00+00 | 30 2024-10-16 00:00:00+00 | 31 2024-11-16 00:00:00+00 | 30 -- No invalidations should remain for the refreshed range SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_month_offset'; cagg_name | inval_start | inval_end -------------------+------------------------+------------------------------- cagg_month_offset | -infinity | 2024-01-15 23:59:59.999999+00 cagg_month_offset | 2024-12-16 00:00:00+00 | infinity ----------------------------------------------------------------------- -- Test 5a: Invalidation at an offset bucket boundary (Feb 16) ----------------------------------------------------------------------- -- Insert just before the bucket boundary INSERT INTO offset_data VALUES ('2024-02-15 23:59:59.999999', 1, 888.0); -- Insert at the bucket boundary INSERT INTO offset_data VALUES ('2024-02-16 00:00:00', 1, 777.0); SELECT * FROM hyper_inval_log; hypertable | inval_start | inval_end --------------------+-------------------------------+------------------------------- public.offset_data | 2024-02-15 23:59:59.999999+00 | 2024-02-15 23:59:59.999999+00 public.offset_data | 2024-02-16 00:00:00+00 | 2024-02-16 00:00:00+00 CALL refresh_continuous_aggregate('cagg_month_offset', '2024-01-01 00:00:00', '2024-04-01 00:00:00'); SELECT bucket, cnt FROM cagg_month_offset WHERE bucket >= '2024-01-16 00:00:00' AND bucket <= '2024-02-16 00:00:00' ORDER BY bucket; bucket | cnt ------------------------+----- 2024-01-16 00:00:00+00 | 32 2024-02-16 00:00:00+00 | 30 SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_month_offset'; cagg_name | inval_start | inval_end -------------------+------------------------+------------------------------- cagg_month_offset | -infinity | 2024-01-15 23:59:59.999999+00 cagg_month_offset | 2024-12-16 00:00:00+00 | infinity ----------------------------------------------------------------------- -- Test 5b: Partial refresh leaves correct invalidation with offset ----------------------------------------------------------------------- -- Insert data across two offset bucket boundaries INSERT INTO offset_data VALUES ('2024-05-15 12:00:00', 1, 100.0); -- in Apr 16 - May 16 bucket INSERT INTO offset_data VALUES ('2024-06-20 12:00:00', 1, 200.0); -- in Jun 16 - Jul 16 bucket -- Refresh only May CALL refresh_continuous_aggregate('cagg_month_offset', '2024-04-01 00:00:00', '2024-06-01 00:00:00'); -- Jun 16 - Jul 16 invalidation should remain SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_month_offset'; cagg_name | inval_start | inval_end -------------------+------------------------+------------------------------- cagg_month_offset | -infinity | 2024-01-15 23:59:59.999999+00 cagg_month_offset | 2024-06-16 00:00:00+00 | 2024-07-15 23:59:59.999999+00 cagg_month_offset | 2024-12-16 00:00:00+00 | infinity CALL refresh_continuous_aggregate('cagg_month_offset', '2024-06-01 00:00:00', '2024-08-01 00:00:00'); SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_month_offset'; cagg_name | inval_start | inval_end -------------------+------------------------+------------------------------- cagg_month_offset | -infinity | 2024-01-15 23:59:59.999999+00 cagg_month_offset | 2024-12-16 00:00:00+00 | infinity ----------------------------------------------------------------------- -- Test 5d: Offset with timezone and variable-width bucket ----------------------------------------------------------------------- SET timezone TO 'Europe/Berlin'; CREATE TABLE offset_tz_data ( time TIMESTAMPTZ NOT NULL, value FLOAT ); SELECT create_hypertable('offset_tz_data', 'time', chunk_time_interval => INTERVAL '1 month'); create_hypertable ------------------------------ (11,public,offset_tz_data,t) CREATE MATERIALIZED VIEW cagg_offset_tz WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT time_bucket('1 day'::interval, time, 'Europe/Berlin', "offset" := INTERVAL '2 hour') AS bucket, count(*) AS cnt FROM offset_tz_data GROUP BY 1 WITH NO DATA; -- Insert data around the DST spring-forward (Mar 30, 2025 at 2:00 AM Europe/Berlin) INSERT INTO offset_tz_data SELECT ts, 1.0 FROM generate_series('2025-03-28 00:00:00'::timestamptz, '2025-04-01 23:59:59'::timestamptz, '1 hour'::interval) ts; CALL refresh_continuous_aggregate('cagg_offset_tz', '2025-03-01 00:00:00', '2025-05-01 00:00:00'); SELECT bucket, cnt FROM cagg_offset_tz ORDER BY bucket; bucket | cnt ------------------------+----- 2025-03-27 02:00:00+01 | 2 2025-03-28 02:00:00+01 | 24 2025-03-29 02:00:00+01 | 24 2025-03-30 03:00:00+02 | 23 2025-03-31 02:00:00+02 | 24 2025-04-01 02:00:00+02 | 22 INSERT INTO offset_tz_data VALUES ('2025-03-30 01:59:59.999999', 888.0); INSERT INTO offset_tz_data VALUES ('2025-03-30 02:00:00', 777.0); CALL refresh_continuous_aggregate('cagg_offset_tz', '2025-03-01 00:00:00', '2025-06-01 00:00:00'); SELECT bucket, cnt FROM cagg_offset_tz ORDER BY bucket; bucket | cnt ------------------------+----- 2025-03-27 02:00:00+01 | 2 2025-03-28 02:00:00+01 | 24 2025-03-29 02:00:00+01 | 25 2025-03-30 03:00:00+02 | 24 2025-03-31 02:00:00+02 | 24 2025-04-01 02:00:00+02 | 22 SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_offset_tz'; cagg_name | inval_start | inval_end ----------------+------------------------+------------------------------- cagg_offset_tz | -infinity | 2025-03-01 01:59:59.999999+01 cagg_offset_tz | 2025-05-31 02:00:00+02 | infinity SET timezone TO 'UTC'; ----------------------------------------------------------------------- -- SECTION 6: Variable-width buckets with custom origin -- Tests that invalidations are correctly processed when variable-width -- buckets use a custom origin. ----------------------------------------------------------------------- CREATE TABLE origin_data ( time TIMESTAMPTZ NOT NULL, device INT, value FLOAT ); SELECT create_hypertable('origin_data', 'time', chunk_time_interval => INTERVAL '1 month'); create_hypertable --------------------------- (13,public,origin_data,t) -- 2-month buckets with origin in Feb: shifts pairs from Jan+Feb/Mar+Apr to Feb+Mar/Apr+May CREATE MATERIALIZED VIEW cagg_bimonth_origin WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT time_bucket('2 months'::interval, time, '2025-02-01 00:00:00+00'::timestamptz) AS bucket, device, count(*) AS cnt FROM origin_data GROUP BY 1, 2 WITH NO DATA; -- Insert data spanning 12 months of 2024 (leap year) INSERT INTO origin_data SELECT ts, 1, extract(epoch FROM ts)::int % 100 FROM generate_series('2025-01-01 00:00:00'::timestamptz, '2025-12-31 23:59:59'::timestamptz, '1 day'::interval) ts; CALL refresh_continuous_aggregate('cagg_bimonth_origin', '2025-01-01 00:00:00', '2026-01-01 00:00:00'); SELECT bucket, cnt FROM cagg_bimonth_origin ORDER BY bucket; bucket | cnt ------------------------+----- 2025-02-01 00:00:00+00 | 59 2025-04-01 00:00:00+00 | 61 2025-06-01 00:00:00+00 | 61 2025-08-01 00:00:00+00 | 61 2025-10-01 00:00:00+00 | 61 SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_bimonth_origin'; cagg_name | inval_start | inval_end ---------------------+------------------------+------------------------------- cagg_bimonth_origin | -infinity | 2025-01-31 23:59:59.999999+00 cagg_bimonth_origin | 2025-12-01 00:00:00+00 | infinity ----------------------------------------------------------------------- -- Test 6a: Invalidation at the origin-shifted bucket boundary (Apr 1) ----------------------------------------------------------------------- -- Insert at the shifted boundary INSERT INTO origin_data VALUES ('2025-03-31 23:59:59.999999', 1, 888.0); INSERT INTO origin_data VALUES ('2025-04-01 00:00:00', 1, 777.0); SELECT * FROM hyper_inval_log; hypertable | inval_start | inval_end --------------------+-------------------------------+------------------------------- public.origin_data | 2025-03-31 23:59:59.999999+00 | 2025-03-31 23:59:59.999999+00 public.origin_data | 2025-04-01 00:00:00+00 | 2025-04-01 00:00:00+00 CALL refresh_continuous_aggregate('cagg_bimonth_origin', '2025-02-01 00:00:00', '2025-06-01 00:00:00'); -- Verify Feb+Mar and Apr+May buckets are updated SELECT bucket, cnt FROM cagg_bimonth_origin WHERE bucket IN ('2025-02-01 00:00:00', '2025-04-01 00:00:00') ORDER BY bucket; bucket | cnt ------------------------+----- 2025-02-01 00:00:00+00 | 60 2025-04-01 00:00:00+00 | 62 SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_bimonth_origin'; cagg_name | inval_start | inval_end ---------------------+------------------------+------------------------------- cagg_bimonth_origin | -infinity | 2025-01-31 23:59:59.999999+00 cagg_bimonth_origin | 2025-12-01 00:00:00+00 | infinity ----------------------------------------------------------------------- -- Test 6b: Partial refresh leaves correct invalidation with origin ----------------------------------------------------------------------- -- Insert across two origin-shifted bucket boundaries INSERT INTO origin_data VALUES ('2025-05-20 12:00:00', 1, 100.0); -- in Apr+May bucket INSERT INTO origin_data VALUES ('2025-07-10 12:00:00', 1, 200.0); -- in Jun+Jul bucket -- Refresh only the Apr-May window CALL refresh_continuous_aggregate('cagg_bimonth_origin', '2025-04-01 00:00:00', '2025-06-01 00:00:00'); -- Jun+Jul invalidation should remain SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_bimonth_origin'; cagg_name | inval_start | inval_end ---------------------+------------------------+------------------------------- cagg_bimonth_origin | -infinity | 2025-01-31 23:59:59.999999+00 cagg_bimonth_origin | 2025-06-01 00:00:00+00 | 2025-07-31 23:59:59.999999+00 cagg_bimonth_origin | 2025-12-01 00:00:00+00 | infinity CALL refresh_continuous_aggregate('cagg_bimonth_origin', '2025-06-01 00:00:00', '2025-09-01 00:00:00'); SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_bimonth_origin'; cagg_name | inval_start | inval_end ---------------------+------------------------+------------------------------- cagg_bimonth_origin | -infinity | 2025-01-31 23:59:59.999999+00 cagg_bimonth_origin | 2025-12-01 00:00:00+00 | infinity ----------------------------------------------------------------------- -- Test 6d: Origin with timezone and variable-width bucket -- 2-month bucket with origin in Feb and Europe/Berlin timezone, -- covering the DST spring-forward transition. ----------------------------------------------------------------------- SET timezone TO 'Europe/Berlin'; CREATE TABLE origin_tz_data ( time TIMESTAMPTZ NOT NULL, value FLOAT ); SELECT create_hypertable('origin_tz_data', 'time', chunk_time_interval => INTERVAL '1 month'); create_hypertable ------------------------------ (15,public,origin_tz_data,t) CREATE MATERIALIZED VIEW cagg_origin_tz WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT time_bucket('1 day'::interval, time, 'Europe/Berlin', origin := '2025-02-01 05:00:00 Europe/Berlin'::timestamptz) AS bucket, count(*) AS cnt FROM origin_tz_data GROUP BY 1 WITH NO DATA; -- DST transition on Mar 30 in Europe/Berlin INSERT INTO origin_tz_data SELECT ts, 1.0 FROM generate_series('2025-03-29 00:00:00'::timestamptz, '2025-03-31 23:59:59'::timestamptz, '1 hour'::interval) ts; CALL refresh_continuous_aggregate('cagg_origin_tz', '2025-02-01 00:00:00', '2025-04-01 00:00:00'); SELECT bucket, cnt FROM cagg_origin_tz ORDER BY bucket; bucket | cnt ------------------------+----- 2025-03-28 05:00:00+01 | 5 2025-03-29 05:00:00+01 | 23 2025-03-30 05:00:00+02 | 24 SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_origin_tz'; cagg_name | inval_start | inval_end ----------------+------------------------+------------------------------- cagg_origin_tz | -infinity | 2025-02-01 04:59:59.999999+01 cagg_origin_tz | 2025-03-31 05:00:00+02 | infinity SET timezone TO 'UTC'; ----------------------------------------------------------------------- -- SECTION 8: Hierarchical continuous aggregates with variable-width buckets -- Tests invalidation propagation through a two-level hierarchy: -- Level 1: 1-month buckets on raw hypertable -- Level 2: 3-month (quarterly) buckets on Level 1 ----------------------------------------------------------------------- CREATE TABLE hier_data ( time TIMESTAMPTZ NOT NULL, value FLOAT ); SELECT create_hypertable('hier_data', 'time', chunk_time_interval => INTERVAL '1 month'); create_hypertable ------------------------- (17,public,hier_data,t) CREATE MATERIALIZED VIEW cagg_hier_monthly WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT time_bucket('1 month'::interval, time) AS bucket, count(*) AS cnt FROM hier_data GROUP BY 1 WITH NO DATA; CREATE MATERIALIZED VIEW cagg_hier_quarterly WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT time_bucket('3 months'::interval, bucket) AS bucket, sum(cnt) AS cnt FROM cagg_hier_monthly GROUP BY 1 WITH NO DATA; INSERT INTO hier_data SELECT ts, 1.0 FROM generate_series('2025-01-01 00:00:00'::timestamptz, '2025-12-31 23:59:59'::timestamptz, '1 day'::interval) ts; CALL refresh_continuous_aggregate('cagg_hier_monthly', '2025-01-01 00:00:00', '2026-01-01 00:00:00'); CALL refresh_continuous_aggregate('cagg_hier_quarterly', '2025-01-01 00:00:00', '2026-01-01 00:00:00'); SELECT bucket, cnt FROM cagg_hier_monthly ORDER BY bucket; bucket | cnt ------------------------+----- 2025-01-01 00:00:00+00 | 31 2025-02-01 00:00:00+00 | 28 2025-03-01 00:00:00+00 | 31 2025-04-01 00:00:00+00 | 30 2025-05-01 00:00:00+00 | 31 2025-06-01 00:00:00+00 | 30 2025-07-01 00:00:00+00 | 31 2025-08-01 00:00:00+00 | 31 2025-09-01 00:00:00+00 | 30 2025-10-01 00:00:00+00 | 31 2025-11-01 00:00:00+00 | 30 2025-12-01 00:00:00+00 | 31 SELECT bucket, cnt FROM cagg_hier_quarterly ORDER BY bucket; bucket | cnt ------------------------+----- 2025-01-01 00:00:00+00 | 90 2025-04-01 00:00:00+00 | 91 2025-07-01 00:00:00+00 | 92 2025-10-01 00:00:00+00 | 92 SELECT * FROM cagg_inval_log WHERE cagg_name IN ('cagg_hier_monthly', 'cagg_hier_quarterly') ORDER BY 1, 2; cagg_name | inval_start | inval_end ---------------------+------------------------+------------------------------- cagg_hier_monthly | -infinity | 2024-12-31 23:59:59.999999+00 cagg_hier_monthly | 2026-01-01 00:00:00+00 | infinity cagg_hier_quarterly | -infinity | 2024-12-31 23:59:59.999999+00 cagg_hier_quarterly | 2026-01-01 00:00:00+00 | infinity ----------------------------------------------------------------------- -- Test 8a: Insert into base, refresh Level 1 only, then Level 2 -- Verifies that refreshing Level 1 creates invalidation in Level 2 -- and that Level 2 refresh picks up the change. ----------------------------------------------------------------------- INSERT INTO hier_data VALUES ('2025-02-15 12:00:00', 999.0); CALL refresh_continuous_aggregate('cagg_hier_monthly', '2025-02-01 00:00:00', '2025-03-01 00:00:00'); SELECT bucket, cnt FROM cagg_hier_monthly WHERE bucket = '2025-02-01 00:00:00'; bucket | cnt ------------------------+----- 2025-02-01 00:00:00+00 | 29 SELECT * FROM hyper_inval_log; hypertable | inval_start | inval_end ---------------------------------------------------+------------------------+------------------------ _timescaledb_internal._materialized_hypertable_18 | 2025-02-01 00:00:00+00 | 2025-02-01 00:00:00+00 SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_hier_quarterly'; cagg_name | inval_start | inval_end ---------------------+------------------------+------------------------------- cagg_hier_quarterly | -infinity | 2024-12-31 23:59:59.999999+00 cagg_hier_quarterly | 2026-01-01 00:00:00+00 | infinity -- Refresh Level 2 for Q1 CALL refresh_continuous_aggregate('cagg_hier_quarterly', '2025-01-01 00:00:00', '2025-04-01 00:00:00'); SELECT bucket, cnt FROM cagg_hier_quarterly WHERE bucket = '2025-01-01 00:00:00'; bucket | cnt ------------------------+----- 2025-01-01 00:00:00+00 | 91 SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_hier_quarterly'; cagg_name | inval_start | inval_end ---------------------+------------------------+------------------------------- cagg_hier_quarterly | -infinity | 2024-12-31 23:59:59.999999+00 cagg_hier_quarterly | 2026-01-01 00:00:00+00 | infinity ----------------------------------------------------------------------- -- Test 8b: Invalidation at the quarter boundary (Mar 31 / Apr 1) -- Mar 31 is in Q1, Apr 1 is in Q2. Refreshing Level 1 modifies both -- March and April in the mat table, so Level 2 should update both -- Q1 and Q2. ----------------------------------------------------------------------- INSERT INTO hier_data VALUES ('2025-03-31 23:59:59.999999', 888.0); INSERT INTO hier_data VALUES ('2025-04-01 00:00:00', 777.0); -- Refresh Level 1 for March and April CALL refresh_continuous_aggregate('cagg_hier_monthly', '2025-03-01 00:00:00', '2025-05-01 00:00:00'); SELECT bucket, cnt FROM cagg_hier_monthly WHERE bucket IN ('2025-03-01 00:00:00', '2025-04-01 00:00:00') ORDER BY bucket; bucket | cnt ------------------------+----- 2025-03-01 00:00:00+00 | 32 2025-04-01 00:00:00+00 | 31 -- Refresh Level 2 with a wide window covering both Q1 and Q2 CALL refresh_continuous_aggregate('cagg_hier_quarterly', '2025-01-01 00:00:00', '2025-07-01 00:00:00'); SELECT bucket, cnt FROM cagg_hier_quarterly WHERE bucket IN ('2025-01-01 00:00:00', '2025-04-01 00:00:00') ORDER BY bucket; bucket | cnt ------------------------+----- 2025-01-01 00:00:00+00 | 92 2025-04-01 00:00:00+00 | 91 SELECT * FROM cagg_inval_log WHERE cagg_name = 'cagg_hier_quarterly'; cagg_name | inval_start | inval_end ---------------------+------------------------+------------------------------- cagg_hier_quarterly | -infinity | 2024-12-31 23:59:59.999999+00 cagg_hier_quarterly | 2026-01-01 00:00:00+00 | infinity DROP MATERIALIZED VIEW cagg_hier_quarterly; NOTICE: drop cascades to 2 other objects DROP TABLE monthly_data CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 3 other objects DROP TABLE yearly_data CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_4_22_chunk DROP TABLE dst_data CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_6_24_chunk DROP TABLE bimonthly_data CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 3 other objects DROP TABLE offset_data CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects DROP TABLE offset_tz_data CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_12_59_chunk DROP TABLE origin_data CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_14_73_chunk DROP TABLE origin_tz_data CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_16_75_chunk DROP TABLE hier_data CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 3 other objects DROP VIEW hyper_inval_log; DROP VIEW cagg_inval_log; ================================================ FILE: tsl/test/expected/cagg_joins.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set VERBOSITY default SET timezone TO PST8PDT; CREATE TABLE conditions( day TIMESTAMPTZ NOT NULL, city text NOT NULL, temperature INT NOT NULL, device_id int NOT NULL ); SELECT table_name FROM create_hypertable('conditions', 'day', chunk_time_interval => INTERVAL '1 day'); table_name ------------ conditions INSERT INTO conditions (day, city, temperature, device_id) VALUES ('2021-06-14', 'Moscow', 26,1), ('2021-06-15', 'Berlin', 22,2), ('2021-06-16', 'Stockholm', 24,3), ('2021-06-17', 'London', 24,4), ('2021-06-18', 'London', 27,4), ('2021-06-19', 'Moscow', 28,4), ('2021-06-20', 'Moscow', 30,1), ('2021-06-21', 'Berlin', 31,1), ('2021-06-22', 'Stockholm', 34,1), ('2021-06-23', 'Stockholm', 34,2), ('2021-06-24', 'Moscow', 34,2), ('2021-06-25', 'London', 32,3), ('2021-06-26', 'Moscow', 32,3), ('2021-06-27', 'Moscow', 31,3); CREATE TABLE conditions_dup AS SELECT * FROM conditions; SELECT table_name FROM create_hypertable('conditions_dup', 'day', chunk_time_interval => INTERVAL '1 day', migrate_data => true); NOTICE: migrating data to chunks DETAIL: Migration might take a while depending on the amount of data. table_name ---------------- conditions_dup CREATE TABLE devices ( device_id int not null, name text, location text); INSERT INTO devices values (1, 'thermo_1', 'Moscow'), (2, 'thermo_2', 'Berlin'),(3, 'thermo_3', 'London'),(4, 'thermo_4', 'Stockholm'); CREATE TABLE location (location_id INTEGER, name TEXT); INSERT INTO location VALUES (1, 'Moscow'), (2, 'Berlin'), (3, 'London'), (4, 'Stockholm'); CREATE TABLE devices_dup AS SELECT * FROM devices; CREATE VIEW devices_view AS SELECT * FROM devices; -- Cagg with inner join + realtime aggregate CREATE MATERIALIZED VIEW cagg_realtime WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM conditions, devices WHERE conditions.device_id = devices.device_id GROUP BY name, bucket ORDER BY bucket; NOTICE: refreshing continuous aggregate "cagg_realtime" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. \d+ cagg_realtime View "public.cagg_realtime" Column | Type | Collation | Nullable | Default | Storage | Description --------+--------------------------+-----------+----------+---------+----------+------------- bucket | timestamp with time zone | | | | plain | avg | numeric | | | | main | name | text | | | | extended | View definition: ( SELECT _materialized_hypertable_3.bucket, _materialized_hypertable_3.avg, _materialized_hypertable_3.name FROM _timescaledb_internal._materialized_hypertable_3 WHERE _materialized_hypertable_3.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(3)), '-infinity'::timestamp with time zone) ORDER BY _materialized_hypertable_3.bucket) UNION ALL ( SELECT time_bucket('@ 1 day'::interval, conditions.day) AS bucket, avg(conditions.temperature) AS avg, devices.name FROM conditions, devices WHERE conditions.device_id = devices.device_id AND conditions.day >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(3)), '-infinity'::timestamp with time zone) GROUP BY devices.name, (time_bucket('@ 1 day'::interval, conditions.day)) ORDER BY (time_bucket('@ 1 day'::interval, conditions.day))) ORDER BY 1; SELECT * FROM cagg_realtime ORDER BY bucket, name; bucket | avg | name ------------------------------+---------------------+---------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 | thermo_1 Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 | thermo_2 Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 | thermo_3 Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 | thermo_4 Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 | thermo_4 Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 | thermo_4 Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 | thermo_1 Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 | thermo_1 Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 | thermo_3 INSERT INTO conditions (day, city, temperature, device_id) VALUES ('2021-06-30', 'Moscow', 28, 3); SELECT * FROM cagg_realtime ORDER BY bucket, name; bucket | avg | name ------------------------------+---------------------+---------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 | thermo_1 Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 | thermo_2 Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 | thermo_3 Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 | thermo_4 Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 | thermo_4 Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 | thermo_4 Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 | thermo_1 Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 | thermo_1 Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 | thermo_3 Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 -- Cagg with inner join + realtime aggregate + JOIN clause CREATE MATERIALIZED VIEW cagg_realtime_join WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM conditions JOIN devices ON conditions.device_id = devices.device_id GROUP BY name, bucket ORDER BY bucket; NOTICE: refreshing continuous aggregate "cagg_realtime_join" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. \d+ cagg_realtime_join View "public.cagg_realtime_join" Column | Type | Collation | Nullable | Default | Storage | Description --------+--------------------------+-----------+----------+---------+----------+------------- bucket | timestamp with time zone | | | | plain | avg | numeric | | | | main | name | text | | | | extended | View definition: ( SELECT _materialized_hypertable_4.bucket, _materialized_hypertable_4.avg, _materialized_hypertable_4.name FROM _timescaledb_internal._materialized_hypertable_4 WHERE _materialized_hypertable_4.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(4)), '-infinity'::timestamp with time zone) ORDER BY _materialized_hypertable_4.bucket) UNION ALL ( SELECT time_bucket('@ 1 day'::interval, conditions.day) AS bucket, avg(conditions.temperature) AS avg, devices.name FROM conditions JOIN devices ON conditions.device_id = devices.device_id WHERE conditions.day >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(4)), '-infinity'::timestamp with time zone) GROUP BY devices.name, (time_bucket('@ 1 day'::interval, conditions.day)) ORDER BY (time_bucket('@ 1 day'::interval, conditions.day))) ORDER BY 1; SELECT * FROM cagg_realtime_join ORDER BY bucket, name; bucket | avg | name ------------------------------+---------------------+---------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 | thermo_1 Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 | thermo_2 Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 | thermo_3 Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 | thermo_4 Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 | thermo_4 Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 | thermo_4 Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 | thermo_1 Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 | thermo_1 Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 | thermo_3 Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 INSERT INTO conditions (day, city, temperature, device_id) VALUES ('2021-06-30', 'Moscow', 28, 3); SELECT * FROM cagg_realtime_join ORDER BY bucket, name; bucket | avg | name ------------------------------+---------------------+---------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 | thermo_1 Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 | thermo_2 Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 | thermo_3 Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 | thermo_4 Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 | thermo_4 Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 | thermo_4 Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 | thermo_1 Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 | thermo_1 Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 | thermo_3 Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 -- Cagg with inner join + realtime aggregate + USING clause CREATE MATERIALIZED VIEW cagg_realtime_using WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM conditions JOIN devices USING (device_id) GROUP BY name, bucket ORDER BY bucket; NOTICE: refreshing continuous aggregate "cagg_realtime_using" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. \d+ cagg_realtime_using View "public.cagg_realtime_using" Column | Type | Collation | Nullable | Default | Storage | Description --------+--------------------------+-----------+----------+---------+----------+------------- bucket | timestamp with time zone | | | | plain | avg | numeric | | | | main | name | text | | | | extended | View definition: ( SELECT _materialized_hypertable_5.bucket, _materialized_hypertable_5.avg, _materialized_hypertable_5.name FROM _timescaledb_internal._materialized_hypertable_5 WHERE _materialized_hypertable_5.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(5)), '-infinity'::timestamp with time zone) ORDER BY _materialized_hypertable_5.bucket) UNION ALL ( SELECT time_bucket('@ 1 day'::interval, conditions.day) AS bucket, avg(conditions.temperature) AS avg, devices.name FROM conditions JOIN devices USING (device_id) WHERE conditions.day >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(5)), '-infinity'::timestamp with time zone) GROUP BY devices.name, (time_bucket('@ 1 day'::interval, conditions.day)) ORDER BY (time_bucket('@ 1 day'::interval, conditions.day))) ORDER BY 1; SELECT * FROM cagg_realtime_using ORDER BY bucket; bucket | avg | name ------------------------------+---------------------+---------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 | thermo_1 Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 | thermo_2 Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 | thermo_3 Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 | thermo_4 Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 | thermo_4 Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 | thermo_4 Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 | thermo_1 Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 | thermo_1 Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 | thermo_3 Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 INSERT INTO conditions (day, city, temperature, device_id) VALUES ('2021-06-30', 'Moscow', 28, 3); SELECT * FROM cagg_realtime_using ORDER BY bucket, name; bucket | avg | name ------------------------------+---------------------+---------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 | thermo_1 Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 | thermo_2 Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 | thermo_3 Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 | thermo_4 Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 | thermo_4 Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 | thermo_4 Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 | thermo_1 Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 | thermo_1 Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 | thermo_3 Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 -- Reorder tables in FROM clause -- Cagg with inner join + realtime aggregate CREATE MATERIALIZED VIEW cagg_realtime_reorder WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM devices, conditions WHERE conditions.device_id = devices.device_id GROUP BY name, bucket ORDER BY bucket; NOTICE: refreshing continuous aggregate "cagg_realtime_reorder" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. \d+ cagg_realtime_reorder View "public.cagg_realtime_reorder" Column | Type | Collation | Nullable | Default | Storage | Description --------+--------------------------+-----------+----------+---------+----------+------------- bucket | timestamp with time zone | | | | plain | avg | numeric | | | | main | name | text | | | | extended | View definition: ( SELECT _materialized_hypertable_6.bucket, _materialized_hypertable_6.avg, _materialized_hypertable_6.name FROM _timescaledb_internal._materialized_hypertable_6 WHERE _materialized_hypertable_6.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(6)), '-infinity'::timestamp with time zone) ORDER BY _materialized_hypertable_6.bucket) UNION ALL ( SELECT time_bucket('@ 1 day'::interval, conditions.day) AS bucket, avg(conditions.temperature) AS avg, devices.name FROM devices, conditions WHERE conditions.device_id = devices.device_id AND conditions.day >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(6)), '-infinity'::timestamp with time zone) GROUP BY devices.name, (time_bucket('@ 1 day'::interval, conditions.day)) ORDER BY (time_bucket('@ 1 day'::interval, conditions.day))) ORDER BY 1; SELECT * FROM cagg_realtime_reorder ORDER BY bucket, name; bucket | avg | name ------------------------------+---------------------+---------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 | thermo_1 Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 | thermo_2 Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 | thermo_3 Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 | thermo_4 Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 | thermo_4 Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 | thermo_4 Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 | thermo_1 Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 | thermo_1 Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 | thermo_3 Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 INSERT INTO conditions (day, city, temperature, device_id) VALUES ('2021-07-01', 'Moscow', 28, 3); SELECT * FROM cagg_realtime_reorder ORDER BY bucket, name; bucket | avg | name ------------------------------+---------------------+---------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 | thermo_1 Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 | thermo_2 Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 | thermo_3 Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 | thermo_4 Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 | thermo_4 Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 | thermo_4 Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 | thermo_1 Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 | thermo_1 Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 | thermo_3 Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 Wed Jun 30 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 -- Reorder tables in FROM clause -- Cagg with inner join + realtime aggregate + JOIN clause CREATE MATERIALIZED VIEW cagg_realtime_reorder_join WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM devices JOIN conditions ON conditions.device_id = devices.device_id GROUP BY name, bucket ORDER BY bucket; NOTICE: refreshing continuous aggregate "cagg_realtime_reorder_join" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. \d+ cagg_realtime_reorder_join View "public.cagg_realtime_reorder_join" Column | Type | Collation | Nullable | Default | Storage | Description --------+--------------------------+-----------+----------+---------+----------+------------- bucket | timestamp with time zone | | | | plain | avg | numeric | | | | main | name | text | | | | extended | View definition: ( SELECT _materialized_hypertable_7.bucket, _materialized_hypertable_7.avg, _materialized_hypertable_7.name FROM _timescaledb_internal._materialized_hypertable_7 WHERE _materialized_hypertable_7.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(7)), '-infinity'::timestamp with time zone) ORDER BY _materialized_hypertable_7.bucket) UNION ALL ( SELECT time_bucket('@ 1 day'::interval, conditions.day) AS bucket, avg(conditions.temperature) AS avg, devices.name FROM devices JOIN conditions ON conditions.device_id = devices.device_id WHERE conditions.day >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(7)), '-infinity'::timestamp with time zone) GROUP BY devices.name, (time_bucket('@ 1 day'::interval, conditions.day)) ORDER BY (time_bucket('@ 1 day'::interval, conditions.day))) ORDER BY 1; SELECT * FROM cagg_realtime_reorder_join ORDER BY bucket, name; bucket | avg | name ------------------------------+---------------------+---------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 | thermo_1 Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 | thermo_2 Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 | thermo_3 Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 | thermo_4 Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 | thermo_4 Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 | thermo_4 Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 | thermo_1 Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 | thermo_1 Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 | thermo_3 Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 Wed Jun 30 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 INSERT INTO conditions (day, city, temperature, device_id) VALUES ('2021-07-01', 'Moscow', 28, 3); SELECT * FROM cagg_realtime_reorder_join; bucket | avg | name ------------------------------+---------------------+---------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 | thermo_1 Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 | thermo_2 Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 | thermo_3 Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 | thermo_4 Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 | thermo_4 Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 | thermo_4 Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 | thermo_1 Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 | thermo_1 Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 | thermo_3 Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 Wed Jun 30 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 -- Reorder tables in FROM clause -- Cagg with inner join + realtime aggregate + USING clause CREATE MATERIALIZED VIEW cagg_realtime_reorder_using WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM devices JOIN conditions USING (device_id) GROUP BY name, bucket ORDER BY bucket; NOTICE: refreshing continuous aggregate "cagg_realtime_reorder_using" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. \d+ cagg_realtime_reorder_using View "public.cagg_realtime_reorder_using" Column | Type | Collation | Nullable | Default | Storage | Description --------+--------------------------+-----------+----------+---------+----------+------------- bucket | timestamp with time zone | | | | plain | avg | numeric | | | | main | name | text | | | | extended | View definition: ( SELECT _materialized_hypertable_8.bucket, _materialized_hypertable_8.avg, _materialized_hypertable_8.name FROM _timescaledb_internal._materialized_hypertable_8 WHERE _materialized_hypertable_8.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone) ORDER BY _materialized_hypertable_8.bucket) UNION ALL ( SELECT time_bucket('@ 1 day'::interval, conditions.day) AS bucket, avg(conditions.temperature) AS avg, devices.name FROM devices JOIN conditions USING (device_id) WHERE conditions.day >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(8)), '-infinity'::timestamp with time zone) GROUP BY devices.name, (time_bucket('@ 1 day'::interval, conditions.day)) ORDER BY (time_bucket('@ 1 day'::interval, conditions.day))) ORDER BY 1; SELECT * FROM cagg_realtime_reorder_using ORDER BY bucket, name; bucket | avg | name ------------------------------+---------------------+---------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 | thermo_1 Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 | thermo_2 Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 | thermo_3 Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 | thermo_4 Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 | thermo_4 Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 | thermo_4 Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 | thermo_1 Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 | thermo_1 Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 | thermo_3 Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 Wed Jun 30 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 INSERT INTO conditions (day, city, temperature, device_id) VALUES ('2021-07-01', 'Moscow', 28, 3); SELECT * FROM cagg_realtime_reorder_using ORDER BY bucket, name; bucket | avg | name ------------------------------+---------------------+---------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 | thermo_1 Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 | thermo_2 Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 | thermo_3 Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 | thermo_4 Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 | thermo_4 Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 | thermo_4 Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 | thermo_1 Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 | thermo_1 Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 | thermo_3 Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 Wed Jun 30 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 -- Cagg with inner joins - realtime aggregate CREATE MATERIALIZED VIEW cagg WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name, devices.device_id AS thermo_id FROM conditions, devices WHERE conditions.device_id = devices.device_id GROUP BY bucket, name, thermo_id ORDER BY bucket; NOTICE: refreshing continuous aggregate "cagg" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. SELECT * FROM cagg ORDER BY bucket, name, thermo_id; bucket | avg | name | thermo_id ------------------------------+---------------------+----------+----------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 | thermo_1 | 1 Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 | thermo_2 | 2 Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 | thermo_3 | 3 Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 | thermo_4 | 4 Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 | thermo_4 | 4 Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 | thermo_4 | 4 Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 | 1 Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 | thermo_1 | 1 Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 | thermo_1 | 1 Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 | 2 Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 | 2 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 | 3 Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 | 3 Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 | thermo_3 | 3 Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 | 3 Wed Jun 30 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 | 3 -- Cagg with inner joins - realtime aggregate + JOIN clause CREATE MATERIALIZED VIEW cagg_join WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM conditions JOIN devices ON conditions.device_id = devices.device_id GROUP BY bucket,name ORDER BY bucket; NOTICE: refreshing continuous aggregate "cagg_join" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. SELECT * FROM cagg_join ORDER BY bucket, name; bucket | avg | name ------------------------------+---------------------+---------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 | thermo_1 Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 | thermo_2 Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 | thermo_3 Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 | thermo_4 Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 | thermo_4 Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 | thermo_4 Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 | thermo_1 Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 | thermo_1 Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 | thermo_3 Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 Wed Jun 30 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 -- Cagg with inner joins - realtime aggregate + USING clause CREATE MATERIALIZED VIEW cagg_using WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM conditions JOIN devices USING (device_id) GROUP BY name, bucket ORDER BY bucket; NOTICE: refreshing continuous aggregate "cagg_using" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. SELECT * FROM cagg_using; bucket | avg | name ------------------------------+---------------------+---------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 | thermo_1 Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 | thermo_2 Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 | thermo_3 Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 | thermo_4 Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 | thermo_4 Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 | thermo_4 Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 | thermo_1 Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 | thermo_1 Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 | thermo_3 Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 Wed Jun 30 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 -- Reorder tables in FROM clause -- Cagg with inner joins - realtime aggregate CREATE MATERIALIZED VIEW cagg_reorder WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM devices, conditions WHERE conditions.device_id = devices.device_id GROUP BY bucket, name ORDER BY bucket; NOTICE: refreshing continuous aggregate "cagg_reorder" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. SELECT * FROM cagg_reorder ORDER BY bucket, name; bucket | avg | name ------------------------------+---------------------+---------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 | thermo_1 Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 | thermo_2 Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 | thermo_3 Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 | thermo_4 Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 | thermo_4 Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 | thermo_4 Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 | thermo_1 Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 | thermo_1 Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 | thermo_3 Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 Wed Jun 30 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 -- Cagg with inner joins - realtime aggregate + JOIN clause CREATE MATERIALIZED VIEW cagg_reorder_join WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM devices JOIN conditions ON conditions.device_id = devices.device_id GROUP BY name, bucket ORDER BY bucket; NOTICE: refreshing continuous aggregate "cagg_reorder_join" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. SELECT * FROM cagg_reorder_join; bucket | avg | name ------------------------------+---------------------+---------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 | thermo_1 Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 | thermo_2 Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 | thermo_3 Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 | thermo_4 Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 | thermo_4 Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 | thermo_4 Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 | thermo_1 Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 | thermo_1 Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 | thermo_3 Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 Wed Jun 30 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 -- Cagg with inner joins - realtime aggregate + USING clause CREATE MATERIALIZED VIEW cagg_reorder_using WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM devices JOIN conditions USING (device_id) GROUP BY name, bucket ORDER BY bucket; NOTICE: refreshing continuous aggregate "cagg_reorder_using" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. SELECT * FROM cagg_reorder_using; bucket | avg | name ------------------------------+---------------------+---------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 | thermo_1 Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 | thermo_2 Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 | thermo_3 Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 | thermo_4 Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 | thermo_4 Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 | thermo_4 Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 | thermo_1 Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 | thermo_1 Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 | thermo_3 Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 Wed Jun 30 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 -- Cagg join with another table using FROM ONLY CREATE MATERIALIZED VIEW cagg_from_only WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM ONLY devices JOIN conditions USING (device_id) GROUP BY name, bucket ORDER BY bucket; NOTICE: refreshing continuous aggregate "cagg_from_only" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. SELECT * FROM cagg_from_only; bucket | avg | name ------------------------------+---------------------+---------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 | thermo_1 Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 | thermo_2 Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 | thermo_3 Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 | thermo_4 Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 | thermo_4 Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 | thermo_4 Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 | thermo_1 Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 | thermo_1 Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 | thermo_3 Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 Wed Jun 30 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 -- Create CAgg with join and additional WHERE conditions CREATE MATERIALIZED VIEW cagg_more_conds WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM conditions JOIN devices ON conditions.device_id = devices.device_id WHERE conditions.city = devices.location AND conditions.temperature > 28 GROUP BY name, bucket; NOTICE: refreshing continuous aggregate "cagg_more_conds" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. SELECT * FROM cagg_more_conds ORDER BY bucket; bucket | avg | name ------------------------------+---------------------+---------- Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 -- Cagg with more conditions and USING clause CREATE MATERIALIZED VIEW cagg_more_conds_using WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM conditions JOIN devices USING (device_id) WHERE conditions.city = devices.location AND conditions.temperature > 28 GROUP BY name, bucket; NOTICE: refreshing continuous aggregate "cagg_more_conds_using" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. SELECT * FROM cagg_more_conds_using ORDER BY bucket; bucket | avg | name ------------------------------+---------------------+---------- Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 -- Hierarchical CAgg with join CREATE MATERIALIZED VIEW cagg_on_cagg WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(INTERVAL '1 day', bucket) AS bucket, SUM(avg) AS temperature FROM cagg, devices WHERE devices.device_id = cagg.thermo_id GROUP BY 1; NOTICE: refreshing continuous aggregate "cagg_on_cagg" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. SELECT * FROM cagg_on_cagg ORDER BY bucket; bucket | temperature ------------------------------+--------------------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 Wed Jun 30 17:00:00 2021 PDT | 28.0000000000000000 DROP MATERIALIZED VIEW cagg_on_cagg CASCADE; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to table _timescaledb_internal._hyper_18_61_chunk drop cascades to table _timescaledb_internal._hyper_18_62_chunk -- Nested CAgg over a CAgg with JOIN clause CREATE MATERIALIZED VIEW cagg_on_cagg_join WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(INTERVAL '1 day', bucket) AS bucket, SUM(avg) AS temperature FROM cagg JOIN devices ON devices.device_id = cagg.thermo_id GROUP BY 1; NOTICE: refreshing continuous aggregate "cagg_on_cagg_join" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. SELECT * FROM cagg_on_cagg_join ORDER BY bucket; bucket | temperature ------------------------------+--------------------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 Wed Jun 30 17:00:00 2021 PDT | 28.0000000000000000 DROP MATERIALIZED VIEW cagg_on_cagg_join CASCADE; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to table _timescaledb_internal._hyper_19_63_chunk drop cascades to table _timescaledb_internal._hyper_19_64_chunk -- Create CAgg with join and ORDER BY CREATE MATERIALIZED VIEW cagg_ordered WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM conditions JOIN devices ON conditions.device_id = devices.device_id WHERE conditions.city = devices.location AND conditions.temperature > 28 GROUP BY name, bucket ORDER BY name; NOTICE: refreshing continuous aggregate "cagg_ordered" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. SELECT * FROM cagg_ordered ORDER BY bucket; bucket | avg | name ------------------------------+---------------------+---------- Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 CREATE MATERIALIZED VIEW cagg_ordered_2 WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM conditions JOIN devices ON conditions.device_id = devices.device_id WHERE conditions.city = devices.location AND conditions.temperature > 28 GROUP BY name, bucket ORDER BY name DESC; NOTICE: refreshing continuous aggregate "cagg_ordered_2" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. SELECT * FROM cagg_ordered_2 ORDER BY bucket; bucket | avg | name ------------------------------+---------------------+---------- Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 CREATE MATERIALIZED VIEW cagg_ordered_3 WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM conditions JOIN devices ON conditions.device_id = devices.device_id WHERE conditions.city = devices.location AND conditions.temperature > 28 GROUP BY name, bucket ORDER BY name, bucket; NOTICE: refreshing continuous aggregate "cagg_ordered_3" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. SELECT * FROM cagg_ordered_3 ORDER BY bucket; bucket | avg | name ------------------------------+---------------------+---------- Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 CREATE MATERIALIZED VIEW cagg_cagg WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), devices.device_id device_id, name FROM conditions, devices WHERE conditions.device_id = devices.device_id GROUP BY name, bucket, devices.device_id; NOTICE: refreshing continuous aggregate "cagg_cagg" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. --Join between cagg and normal table CREATE MATERIALIZED VIEW cagg_nested WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(INTERVAL '1 day', cagg.bucket) AS bucket, devices.name FROM cagg_cagg cagg, devices WHERE cagg.device_id = devices.device_id GROUP BY 1,2; NOTICE: refreshing continuous aggregate "cagg_nested" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. DROP MATERIALIZED VIEW cagg_nested CASCADE; NOTICE: drop cascades to 2 other objects DETAIL: drop cascades to table _timescaledb_internal._hyper_24_73_chunk drop cascades to table _timescaledb_internal._hyper_24_74_chunk -- CAgg with multiple join conditions without JOIN clause CREATE MATERIALIZED VIEW cagg_more_joins_conds WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), MAX(temperature), MIN(temperature), name FROM conditions JOIN devices ON conditions.device_id = devices.device_id AND conditions.city = devices.location AND conditions.temperature > 28 GROUP BY name, bucket; NOTICE: refreshing continuous aggregate "cagg_more_joins_conds" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. -- With LATERAL multiple tables in new format CREATE TABLE mat_t1(a integer, b integer, c TEXT); CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, temperature, count(*) from conditions, LATERAL (SELECT * FROM mat_t1 WHERE mat_t1.a = conditions.temperature) q GROUP BY bucket, temperature; NOTICE: refreshing continuous aggregate "mat_m1" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. -- Joining a hypertable and view CREATE MATERIALIZED VIEW cagg_view WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), devices_view.device_id, name FROM conditions, devices_view WHERE conditions.device_id = devices_view.device_id GROUP BY name, bucket, devices_view.device_id; NOTICE: refreshing continuous aggregate "cagg_view" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. CREATE TABLE cities(name text, currency text); INSERT INTO cities VALUES ('Berlin', 'EUR'), ('London', 'PND'); --Error out when FROM clause has sub selects \set ON_ERROR_STOP 0 CREATE MATERIALIZED VIEW conditions_summary_subselect WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM conditions JOIN (SELECT * FROM devices WHERE location in ( SELECT name FROM cities WHERE currency = 'EUR')) dev ON conditions.device_id = dev.device_id GROUP BY name, bucket; ERROR: invalid continuous aggregate view DETAIL: Sub-queries are not supported in FROM clause. --Error out when WHERE clause has sub selects CREATE MATERIALIZED VIEW conditions_summary_subselect WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM conditions, devices WHERE conditions.city IN ( SELECT location FROM devices WHERE location in ( SELECT name FROM cities WHERE currency = 'EUR')) AND conditions.device_id = devices.device_id GROUP BY name, bucket; ERROR: invalid continuous aggregate query DETAIL: CTEs and subqueries are not supported by continuous aggregates. CREATE MATERIALIZED VIEW conditions_summary_subselect WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM conditions JOIN devices USING(device_id) WHERE conditions.city IN ( SELECT location FROM devices WHERE location in ( SELECT name FROM cities WHERE currency = 'EUR')) GROUP BY name, bucket; ERROR: invalid continuous aggregate query DETAIL: CTEs and subqueries are not supported by continuous aggregates. CREATE MATERIALIZED VIEW conditions_summary_subselect WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM conditions JOIN devices ON conditions.device_id = devices.device_id WHERE conditions.city IN ( SELECT location FROM devices WHERE location in ( SELECT name FROM cities WHERE currency = 'EUR')) GROUP BY name, bucket; ERROR: invalid continuous aggregate query DETAIL: CTEs and subqueries are not supported by continuous aggregates. DROP TABLE cities CASCADE; --Error out when join is between two hypertables CREATE MATERIALIZED VIEW cagg_ht WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(INTERVAL '1 day', conditions.day) AS bucket, AVG(conditions.temperature) FROM conditions, conditions_dup WHERE conditions.device_id = conditions_dup.device_id GROUP BY bucket; ERROR: invalid continuous aggregate view DETAIL: Only one hypertable is allowed in continuous aggregate view. --Error out when join is between two normal tables CREATE MATERIALIZED VIEW cagg_nt WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT AVG(devices.device_id), devices.name, devices.location FROM devices, devices_dup WHERE devices.device_id = devices_dup.device_id GROUP BY devices.name, devices.location; ERROR: invalid continuous aggregate view DETAIL: At least one hypertable should be used in the view definition. \set ON_ERROR_STOP 1 -- JOIN with non-equality condition CREATE MATERIALIZED VIEW cagg_unequal1 WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM conditions, devices WHERE conditions.device_id <> devices.device_id GROUP BY name, bucket; NOTICE: refreshing continuous aggregate "cagg_unequal1" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. -- JOIN with non-equality condition CREATE MATERIALIZED VIEW cagg_unequal2 WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM conditions, devices WHERE conditions.device_id = devices.device_id AND conditions.city like '%cow*' GROUP BY name, bucket; NOTICE: refreshing continuous aggregate "cagg_unequal2" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. -- JOIN with non-equality condition CREATE MATERIALIZED VIEW cagg_unequal3 WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM conditions JOIN devices ON conditions.device_id = devices.device_id OR conditions.city LIKE '%cow*' GROUP BY name, bucket; NOTICE: refreshing continuous aggregate "cagg_unequal3" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. -- Error out when join type is not INNER or LEFT \set ON_ERROR_STOP 0 CREATE MATERIALIZED VIEW cagg_outer WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM conditions FULL JOIN devices ON conditions.device_id = devices.device_id GROUP BY name, bucket; ERROR: only INNER or LEFT joins are supported in continuous aggregates CREATE MATERIALIZED VIEW cagg_right WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM conditions RIGHT JOIN devices ON conditions.device_id = devices.device_id GROUP BY name, bucket; ERROR: only INNER or LEFT joins are supported in continuous aggregates \set ON_ERROR_STOP 1 -- LEFT JOIN is allowed CREATE MATERIALIZED VIEW cagg_left_join WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(INTERVAL '1 day', day) AS bucket, AVG(temperature), name FROM conditions LEFT JOIN devices ON conditions.device_id = devices.device_id GROUP BY name, bucket; NOTICE: refreshing continuous aggregate "cagg_left_join" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. -- Error out for join between cagg and hypertable \set ON_ERROR_STOP 0 CREATE MATERIALIZED VIEW cagg_nested_ht WITH (timescaledb.continuous, timescaledb.materialized_only = TRUE) AS SELECT time_bucket(INTERVAL '1 day', cagg.bucket) AS bucket, cagg.name, conditions.temperature FROM cagg_cagg cagg, conditions WHERE cagg.device_id = conditions.device_id GROUP BY 1,2,3; ERROR: invalid continuous aggregate view DETAIL: Only one hypertable is allowed in continuous aggregate view. \set ON_ERROR_STOP 1 -- Multiple JOINS are supported CREATE MATERIALIZED VIEW conditions_by_day WITH (timescaledb.continuous) AS SELECT time_bucket(INTERVAL '1 day', conditions.day) AS bucket, AVG(conditions.temperature), devices.name AS device, location.name AS location FROM conditions JOIN devices ON conditions.device_id = devices.device_id JOIN location ON location.name = devices.location GROUP BY bucket, devices.name, location.name; NOTICE: refreshing continuous aggregate "conditions_by_day" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. SELECT * FROM conditions_by_day ORDER BY bucket, device, location; bucket | avg | device | location ------------------------------+---------------------+----------+----------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 | thermo_1 | Moscow Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 | thermo_2 | Berlin Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 | thermo_3 | London Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 | thermo_4 | Stockholm Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 | thermo_4 | Stockholm Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 | thermo_4 | Stockholm Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 | Moscow Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 | thermo_1 | Moscow Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 | thermo_1 | Moscow Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 | Berlin Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 | Berlin Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 | London Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 | London Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 | thermo_3 | London Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 | London Wed Jun 30 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 | London ALTER MATERIALIZED VIEW conditions_by_day SET (timescaledb.materialized_only = FALSE); -- Insert one more row on conditions and check the result (should have one more row) INSERT INTO conditions (day, city, temperature, device_id) VALUES ('2024-07-01', 'Moscow', 28, 3); SELECT * FROM conditions_by_day ORDER BY bucket, device, location; bucket | avg | device | location ------------------------------+---------------------+----------+----------- Sun Jun 13 17:00:00 2021 PDT | 26.0000000000000000 | thermo_1 | Moscow Mon Jun 14 17:00:00 2021 PDT | 22.0000000000000000 | thermo_2 | Berlin Tue Jun 15 17:00:00 2021 PDT | 24.0000000000000000 | thermo_3 | London Wed Jun 16 17:00:00 2021 PDT | 24.0000000000000000 | thermo_4 | Stockholm Thu Jun 17 17:00:00 2021 PDT | 27.0000000000000000 | thermo_4 | Stockholm Fri Jun 18 17:00:00 2021 PDT | 28.0000000000000000 | thermo_4 | Stockholm Sat Jun 19 17:00:00 2021 PDT | 30.0000000000000000 | thermo_1 | Moscow Sun Jun 20 17:00:00 2021 PDT | 31.0000000000000000 | thermo_1 | Moscow Mon Jun 21 17:00:00 2021 PDT | 34.0000000000000000 | thermo_1 | Moscow Tue Jun 22 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 | Berlin Wed Jun 23 17:00:00 2021 PDT | 34.0000000000000000 | thermo_2 | Berlin Thu Jun 24 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 | London Fri Jun 25 17:00:00 2021 PDT | 32.0000000000000000 | thermo_3 | London Sat Jun 26 17:00:00 2021 PDT | 31.0000000000000000 | thermo_3 | London Tue Jun 29 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 | London Wed Jun 30 17:00:00 2021 PDT | 28.0000000000000000 | thermo_3 | London Sun Jun 30 17:00:00 2024 PDT | 28.0000000000000000 | thermo_3 | London -- JOIN with a foreign table \c :TEST_DBNAME :ROLE_SUPERUSER SELECT current_setting('port') AS "PGPORT", current_database() AS "PGDATABASE" \gset CREATE EXTENSION postgres_fdw; CREATE SERVER loopback FOREIGN DATA WRAPPER postgres_fdw OPTIONS (host 'localhost', dbname :'PGDATABASE', port :'PGPORT'); CREATE USER MAPPING FOR :ROLE_DEFAULT_PERM_USER SERVER loopback OPTIONS (user :'ROLE_DEFAULT_PERM_USER', password 'nopassword'); ALTER USER MAPPING FOR :ROLE_DEFAULT_PERM_USER SERVER loopback OPTIONS (ADD password_required 'false'); GRANT USAGE ON FOREIGN SERVER loopback TO :ROLE_DEFAULT_PERM_USER; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER; SET timezone TO PST8PDT; CREATE FOREIGN TABLE devices_fdw ( device_id int not null, name text, location text ) SERVER loopback OPTIONS (table_name 'devices'); CREATE MATERIALIZED VIEW conditions_fdw WITH (timescaledb.continuous) AS SELECT time_bucket(INTERVAL '1 day', conditions.day) AS bucket, AVG(conditions.temperature), devices.name AS device FROM conditions JOIN devices_fdw AS devices ON conditions.device_id = devices.device_id GROUP BY bucket, devices.name; NOTICE: refreshing continuous aggregate "conditions_fdw" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. \set VERBOSITY terse SET client_min_messages TO WARNING; DROP TABLE conditions CASCADE; DROP TABLE devices CASCADE; DROP TABLE conditions_dup CASCADE; DROP TABLE devices_dup CASCADE; RESET client_min_messages; \set VERBOSITY default -- SDC #1859 CREATE TABLE conditions( time TIMESTAMPTZ NOT NULL, value FLOAT8 NOT NULL, device_id int NOT NULL ); SELECT table_name FROM create_hypertable('conditions', 'time', chunk_time_interval => INTERVAL '1 day'); table_name ------------ conditions INSERT INTO conditions (time, value, device_id) SELECT t, 1, 1 FROM generate_series('2024-01-01 00:00:00-00'::timestamptz, '2024-12-31 00:00:00-00'::timestamptz, '1 hour'::interval) AS t; CREATE TABLE devices (device_id int not null, name text, location text); INSERT INTO devices values (1, 'thermo_1', 'Moscow'), (2, 'thermo_2', 'Berlin'),(3, 'thermo_3', 'London'),(4, 'thermo_4', 'Stockholm'); CREATE MATERIALIZED VIEW cagg_realtime WITH (timescaledb.continuous, timescaledb.materialized_only = FALSE) AS SELECT time_bucket(INTERVAL '1 day', time) AS bucket, MAX(value), MIN(value), AVG(value), devices.name, devices.location FROM conditions JOIN devices ON conditions.device_id = devices.device_id GROUP BY name, location, bucket WITH NO DATA; \c :TEST_DBNAME :ROLE_SUPERUSER VACUUM ANALYZE; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT a.* FROM cagg_realtime a WHERE a.location = 'Moscow' ORDER BY bucket LIMIT 2; bucket | max | min | avg | name | location ------------------------------+-----+-----+-----+----------+---------- Sun Dec 31 16:00:00 2023 PST | 1 | 1 | 1 | thermo_1 | Moscow Mon Jan 01 16:00:00 2024 PST | 1 | 1 | 1 | thermo_1 | Moscow \set VERBOSITY terse SET client_min_messages TO WARNING; DROP TABLE conditions CASCADE; DROP TABLE devices CASCADE; RESET client_min_messages; ================================================ FILE: tsl/test/expected/cagg_multi.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER SET ROLE :ROLE_DEFAULT_PERM_USER; SET client_min_messages TO NOTICE; CREATE TABLE continuous_agg_test(timeval integer, col1 integer, col2 integer); select create_hypertable('continuous_agg_test', 'timeval', chunk_time_interval=> 2); create_hypertable ---------------------------------- (1,public,continuous_agg_test,t) CREATE OR REPLACE FUNCTION integer_now_test1() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timeval), 0) FROM continuous_agg_test $$; SELECT set_integer_now_func('continuous_agg_test', 'integer_now_test1'); set_integer_now_func ---------------------- INSERT INTO continuous_agg_test VALUES (10, - 4, 1), (11, - 3, 5), (12, - 3, 7), (13, - 3, 9), (14,-4, 11), (15, -4, 22), (16, -4, 23); -- TEST for multiple continuous aggs --- invalidations are picked up by both caggs CREATE MATERIALIZED VIEW cagg_1( timed, cnt ) WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket( 2, timeval), COUNT(col1) FROM continuous_agg_test GROUP BY 1 WITH NO DATA; CREATE MATERIALIZED VIEW cagg_2( timed, grp, maxval) WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(2, timeval), col1, max(col2) FROM continuous_agg_test GROUP BY 1, 2 WITH NO DATA; select view_name, view_owner, materialization_hypertable_name from timescaledb_information.continuous_aggregates ORDER BY 1; view_name | view_owner | materialization_hypertable_name -----------+-------------------+--------------------------------- cagg_1 | default_perm_user | _materialized_hypertable_2 cagg_2 | default_perm_user | _materialized_hypertable_3 --TEST1: cagg_1 is materialized, not cagg_2. CALL refresh_continuous_aggregate('cagg_1', NULL, NULL); select * from cagg_1 order by 1; timed | cnt -------+----- 10 | 2 12 | 2 14 | 2 16 | 1 SELECT time_bucket(2, timeval), COUNT(col1) as value FROM continuous_agg_test GROUP BY 1 order by 1; time_bucket | value -------------+------- 10 | 2 12 | 2 14 | 2 16 | 1 -- check that cagg_2 not materialized select * from cagg_2 order by 1,2; timed | grp | maxval -------+-----+-------- CALL refresh_continuous_aggregate('cagg_2', NULL, NULL); select * from cagg_2 order by 1,2; timed | grp | maxval -------+-----+-------- 10 | -4 | 1 10 | -3 | 5 12 | -3 | 9 14 | -4 | 22 16 | -4 | 23 SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold; hypertable_id | watermark ---------------+----------- 1 | 18 --TEST2: cagg_2 gets invalidations after cagg_1's refresh --will trigger invalidations INSERT INTO continuous_agg_test VALUES (10, -4, 10), (11, - 3, 50), (11, - 3, 70), (10, - 4, 10); SELECT * FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold; hypertable_id | watermark ---------------+----------- 1 | 18 CALL refresh_continuous_aggregate('cagg_1', NULL, NULL); select * from cagg_1 order by 1; timed | cnt -------+----- 10 | 6 12 | 2 14 | 2 16 | 1 SELECT time_bucket(2, timeval), COUNT(col1) as value FROM continuous_agg_test GROUP BY 1 order by 1; time_bucket | value -------------+------- 10 | 6 12 | 2 14 | 2 16 | 1 -- are the invalidations picked up here? select * from cagg_2 order by 1, 2; timed | grp | maxval -------+-----+-------- 10 | -4 | 1 10 | -3 | 5 12 | -3 | 9 14 | -4 | 22 16 | -4 | 23 SELECT time_bucket(2, timeval), col1, max(col2) FROM continuous_agg_test GROUP BY 1, 2 order by 1,2 ; time_bucket | col1 | max -------------+------+----- 10 | -4 | 10 10 | -3 | 70 12 | -3 | 9 14 | -4 | 22 16 | -4 | 23 CALL refresh_continuous_aggregate('cagg_2', NULL, NULL); select * from cagg_2 order by 1, 2; timed | grp | maxval -------+-----+-------- 10 | -4 | 10 10 | -3 | 70 12 | -3 | 9 14 | -4 | 22 16 | -4 | 23 --TEST3: invalidations left over by cagg_1 are dropped --trigger another invalidation INSERT INTO continuous_agg_test VALUES (10, -4, 1000); select count(*) from _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log; count ------- 1 select count(*) from _timescaledb_catalog.continuous_aggs_materialization_invalidation_log; count ------- 4 CALL refresh_continuous_aggregate('cagg_1', NULL, NULL); select count(*) from _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log; count ------- 0 select count(*) from _timescaledb_catalog.continuous_aggs_materialization_invalidation_log; count ------- 5 --now drop cagg_1, should still have materialization_invalidation_log DROP MATERIALIZED VIEW cagg_1; NOTICE: drop cascades to table _timescaledb_internal._hyper_2_5_chunk select count(*) from _timescaledb_catalog.continuous_aggs_materialization_invalidation_log; count ------- 3 --cagg_2 still exists select view_name from timescaledb_information.continuous_aggregates; view_name ----------- cagg_2 drop table continuous_agg_test cascade; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to table _timescaledb_internal._hyper_3_6_chunk select count(*) from _timescaledb_catalog.continuous_aggs_materialization_invalidation_log; count ------- 0 select view_name from timescaledb_information.continuous_aggregates; view_name ----------- --TEST4: invalidations that are copied over by cagg1 are not deleted by cagg2 refresh if -- they do not meet materialization invalidation threshold for cagg2. CREATE TABLE continuous_agg_test(timeval integer, col1 integer, col2 integer); select create_hypertable('continuous_agg_test', 'timeval', chunk_time_interval=> 2); create_hypertable ---------------------------------- (4,public,continuous_agg_test,t) CREATE OR REPLACE FUNCTION integer_now_test1() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timeval), 0) FROM continuous_agg_test $$; SELECT set_integer_now_func('continuous_agg_test', 'integer_now_test1'); set_integer_now_func ---------------------- INSERT INTO continuous_agg_test VALUES (10, -4, 1), (11, -3, 5), (12, -3, 7), (13, -3, 9), (14, -4, 11), (15, -4, 22), (16, -4, 23); CREATE MATERIALIZED VIEW cagg_1( timed, cnt ) WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT time_bucket( 2, timeval), COUNT(col1) FROM continuous_agg_test GROUP BY 1 WITH DATA; NOTICE: refreshing continuous aggregate "cagg_1" CREATE MATERIALIZED VIEW cagg_2( timed, maxval) WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT time_bucket(2, timeval), max(col2) FROM continuous_agg_test GROUP BY 1 WITH NO DATA; --cagg_1 already refreshed on creation select * from cagg_1 order by 1; timed | cnt -------+----- 10 | 2 12 | 2 14 | 2 16 | 1 --partially refresh cagg_2 to 14 to test inserts that won't be seen in --one of the caggs CALL refresh_continuous_aggregate('cagg_2', NULL, 14); select * from cagg_2 order by 1; timed | maxval -------+-------- 10 | 5 12 | 9 --this insert will be processed only by cagg_1 and copied over to cagg_2 insert into continuous_agg_test values( 14, -2, 100); CALL refresh_continuous_aggregate('cagg_1', NULL, NULL); select * from cagg_1 order by 1; timed | cnt -------+----- 10 | 2 12 | 2 14 | 3 16 | 1 CALL refresh_continuous_aggregate('cagg_2', NULL, 14); NOTICE: continuous aggregate "cagg_2" is already up-to-date select * from cagg_2 order by 1; timed | maxval -------+-------- 10 | 5 12 | 9 SET ROLE :ROLE_SUPERUSER; select * from _timescaledb_catalog.continuous_aggs_invalidation_threshold order by 1; hypertable_id | watermark ---------------+----------- 4 | 18 select * from _timescaledb_catalog.continuous_aggs_materialization_invalidation_log order by 1; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- 5 | -9223372036854775808 | -2147483649 5 | 18 | 9223372036854775807 6 | -9223372036854775808 | -2147483649 6 | 14 | 9223372036854775807 SET ROLE :ROLE_DEFAULT_PERM_USER; --this insert will be processed only by cagg_1 and cagg_2 will process the previous --one insert into continuous_agg_test values( 18, -2, 200); CALL refresh_continuous_aggregate('cagg_1', NULL, NULL); select * from cagg_1 order by 1; timed | cnt -------+----- 10 | 2 12 | 2 14 | 3 16 | 1 18 | 1 -- Now make changes visible up to 16 CALL refresh_continuous_aggregate('cagg_2', NULL, 16); select * from cagg_2 order by 1; timed | maxval -------+-------- 10 | 5 12 | 9 14 | 100 --TEST5 2 inserts with the same value can be copied over to materialization invalidation log insert into continuous_agg_test values( 18, -2, 100); insert into continuous_agg_test values( 18, -2, 100); select * from _timescaledb_catalog.continuous_aggs_hypertable_invalidation_log order by 1; hypertable_id | lowest_modified_value | greatest_modified_value ---------------+-----------------------+------------------------- 4 | 18 | 18 4 | 18 | 18 CALL refresh_continuous_aggregate('cagg_1', NULL, NULL); select * from cagg_1 where timed = 18 ; timed | cnt -------+----- 18 | 3 --copied over for cagg_2 to process later? select * from _timescaledb_catalog.continuous_aggs_materialization_invalidation_log order by 1; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- 5 | -9223372036854775808 | -2147483649 5 | 20 | 9223372036854775807 6 | -9223372036854775808 | -2147483649 6 | 16 | 9223372036854775807 6 | 18 | 19 DROP MATERIALIZED VIEW cagg_1; NOTICE: drop cascades to table _timescaledb_internal._hyper_5_11_chunk DROP MATERIALIZED VIEW cagg_2; NOTICE: drop cascades to table _timescaledb_internal._hyper_6_12_chunk ----TEST7 multiple continuous aggregates with real time aggregates test---- create table foo (a integer, b integer, c integer); select table_name FROM create_hypertable('foo', 'a', chunk_time_interval=> 10); table_name ------------ foo INSERT into foo values( 1 , 10 , 20); INSERT into foo values( 1 , 11 , 20); INSERT into foo values( 1 , 12 , 20); INSERT into foo values( 1 , 13 , 20); INSERT into foo values( 1 , 14 , 20); INSERT into foo values( 5 , 14 , 20); INSERT into foo values( 5 , 15 , 20); INSERT into foo values( 5 , 16 , 20); INSERT into foo values( 20 , 16 , 20); INSERT into foo values( 20 , 26 , 20); INSERT into foo values( 20 , 16 , 20); INSERT into foo values( 21 , 15 , 30); INSERT into foo values( 21 , 15 , 30); INSERT into foo values( 21 , 15 , 30); INSERT into foo values( 45 , 14 , 70); CREATE OR REPLACE FUNCTION integer_now_foo() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(a), 0) FROM foo $$; SELECT set_integer_now_func('foo', 'integer_now_foo'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW mat_m1(a, countb) WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(10, a), count(*) FROM foo GROUP BY time_bucket(10, a) WITH NO DATA; CREATE MATERIALIZED VIEW mat_m2(a, countb) WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket(5, a), count(*) FROM foo GROUP BY time_bucket(5, a) WITH NO DATA; select view_name, materialized_only from timescaledb_information.continuous_aggregates WHERE view_name::text like 'mat_m%' order by view_name; view_name | materialized_only -----------+------------------- mat_m1 | f mat_m2 | f CALL refresh_continuous_aggregate('mat_m1', NULL, 35); CALL refresh_continuous_aggregate('mat_m2', NULL, NULL); -- the results from the view should match the direct query SELECT * from mat_m1 order by 1; a | countb ----+-------- 0 | 8 20 | 6 40 | 1 SELECT time_bucket(5, a), count(*) FROM foo GROUP BY time_bucket(5, a) ORDER BY 1; time_bucket | count -------------+------- 0 | 5 5 | 3 20 | 6 45 | 1 SELECT * from mat_m2 order by 1; a | countb ----+-------- 0 | 5 5 | 3 20 | 6 45 | 1 SELECT time_bucket(10, a), count(*) FROM foo GROUP BY time_bucket(10, a) ORDER BY 1; time_bucket | count -------------+------- 0 | 8 20 | 6 40 | 1 ================================================ FILE: tsl/test/expected/cagg_on_cagg.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- Global test variables \set IS_TIME_DIMENSION_WITH_TIMEZONE_1ST FALSE \set IS_TIME_DIMENSION_WITH_TIMEZONE_2TH FALSE \set IS_JOIN FALSE \set INTERVAL_TEST FALSE -- ######################################################## -- ## INTEGER data type tests -- ######################################################## -- Current test variables \set IS_TIME_DIMENSION FALSE \set TIME_DIMENSION_DATATYPE INTEGER \set CAGG_NAME_1ST_LEVEL conditions_summary_1_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2_5 \set CAGG_NAME_3TH_LEVEL conditions_summary_3_10 -- -- Run common tests for INTEGER -- \set BUCKET_WIDTH_1ST 'INTEGER \'1\'' \set BUCKET_WIDTH_2TH 'INTEGER \'5\'' \set BUCKET_WIDTH_3TH 'INTEGER \'10\'' -- Different order of time dimension in raw ht \set IS_DEFAULT_COLUMN_ORDER FALSE \ir include/cagg_on_cagg_setup.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGGs on CAGGs tests SET ROLE :ROLE_DEFAULT_PERM_USER; DROP TABLE IF EXISTS conditions CASCADE; psql:include/cagg_on_cagg_setup.sql:8: NOTICE: table "conditions" does not exist, skipping \if :IS_DEFAULT_COLUMN_ORDER CREATE TABLE conditions ( time :TIME_DIMENSION_DATATYPE NOT NULL, temperature NUMERIC, device_id INT ); \else CREATE TABLE conditions ( temperature NUMERIC, time :TIME_DIMENSION_DATATYPE NOT NULL, device_id INT ); \endif \if :IS_JOIN DROP TABLE IF EXISTS devices CASCADE; CREATE TABLE devices ( device_id int not null, name text, location text); INSERT INTO devices values (1, 'thermo_1', 'Moscow'), (2, 'thermo_2', 'Berlin'),(3, 'thermo_3', 'London'),(4, 'thermo_4', 'Stockholm'); \endif \if :IS_TIME_DIMENSION SELECT table_name FROM create_hypertable('conditions', 'time'); \else SELECT table_name FROM create_hypertable('conditions', 'time', chunk_time_interval => 10); table_name ------------ conditions \endif \if :IS_TIME_DIMENSION INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 01:00:00-00', 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-02 01:00:00-00', 20, 3); \else CREATE OR REPLACE FUNCTION integer_now() RETURNS :TIME_DIMENSION_DATATYPE LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now'); set_integer_now_func ---------------------- INSERT INTO conditions ("time", temperature, device_id) VALUES (1, 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES (2, 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES (5, 20, 3); \endif \ir include/cagg_on_cagg_common.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGG on hypertable (1st level) CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, SUM(temperature) AS temperature, device_id FROM conditions GROUP BY 1, 3 ORDER BY 1, 2, 3 WITH NO DATA; -- CAGG on CAGG (2th level) CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , devices.device_id , devices.name FROM :CAGG_NAME_1ST_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_1ST_LEVEL.device_id GROUP BY 1, 3, 4 ORDER BY 1, 2, 3, 4 \else FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- CAGG on CAGG (3th level) CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , :CAGG_NAME_2TH_LEVEL.device_id , :CAGG_NAME_2TH_LEVEL.name , devices.location FROM :CAGG_NAME_2TH_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_2TH_LEVEL.device_id GROUP BY 1, 3, 4, 5 ORDER BY 1, 2, 3, 4, 5 \else FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- Check chunk_interval \if :IS_TIME_DIMENSION SELECT h.table_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; \else SELECT h.table_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; name | chunk_interval -------------------------+---------------- conditions | 10 conditions_summary_1_1 | 100 conditions_summary_2_5 | 100 conditions_summary_3_10 | 100 \endif -- No data because the CAGGs are just for materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 5 | 2 5 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- 0 | 15 5 | 20 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- 0 | 35 -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 5 | 2 5 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- 0 | 15 5 | 20 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- 0 | 35 \if :IS_TIME_DIMENSION -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-01 01:00:00-00'::timestamptz, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-03 01:00:00-00'::timestamptz, 2); \else -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES (2, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES (10, 2); \endif -- No changes SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 5 | 2 5 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- 0 | 15 5 | 20 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- 0 | 35 -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime changes, just new region SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 5 | 2 5 | 20 | 3 10 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- 0 | 15 5 | 20 10 | 2 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- 0 | 35 10 | 2 -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- All changes are materialized SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 2 | 2 | 5 | 2 5 | 20 | 3 10 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- 0 | 17 5 | 20 10 | 2 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- 0 | 37 10 | 2 -- TRUNCATE tests TRUNCATE :CAGG_NAME_2TH_LEVEL; -- This full refresh will remove all the data from the 3TH level cagg CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Should return no rows SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- -- If we have all the data in the bottom levels caggs we can rebuild CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Now we have all the data SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- 0 | 17 5 | 20 10 | 2 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- 0 | 37 10 | 2 -- DROP tests \set ON_ERROR_STOP 0 -- should error because it depends of other CAGGs DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:172: ERROR: cannot drop view conditions_summary_1_1 because other objects depend on it DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:173: ERROR: cannot drop view conditions_summary_2_5 because other objects depend on it CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:174: NOTICE: continuous aggregate "conditions_summary_1_1" is already up-to-date CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:175: NOTICE: continuous aggregate "conditions_summary_2_5" is already up-to-date \set ON_ERROR_STOP 1 -- DROP the 3TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:179: NOTICE: drop cascades to table _timescaledb_internal._hyper_4_4_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:182: ERROR: relation "conditions_summary_3_10" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs TRUNCATE :CAGG_NAME_2TH_LEVEL,:CAGG_NAME_1ST_LEVEL; CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 2 | 2 | 5 | 2 5 | 20 | 3 10 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- -- DROP the 2TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:196: ERROR: relation "conditions_summary_2_5" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 2 | 2 | 5 | 2 5 | 20 | 3 10 | 2 | -- DROP the first CAGG should work DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:203: NOTICE: drop cascades to table _timescaledb_internal._hyper_2_7_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:206: ERROR: relation "conditions_summary_1_1" does not exist at character 15 \set ON_ERROR_STOP 1 -- Default tests \set IS_DEFAULT_COLUMN_ORDER TRUE \ir include/cagg_on_cagg_setup.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGGs on CAGGs tests SET ROLE :ROLE_DEFAULT_PERM_USER; DROP TABLE IF EXISTS conditions CASCADE; \if :IS_DEFAULT_COLUMN_ORDER CREATE TABLE conditions ( time :TIME_DIMENSION_DATATYPE NOT NULL, temperature NUMERIC, device_id INT ); \else CREATE TABLE conditions ( temperature NUMERIC, time :TIME_DIMENSION_DATATYPE NOT NULL, device_id INT ); \endif \if :IS_JOIN DROP TABLE IF EXISTS devices CASCADE; CREATE TABLE devices ( device_id int not null, name text, location text); INSERT INTO devices values (1, 'thermo_1', 'Moscow'), (2, 'thermo_2', 'Berlin'),(3, 'thermo_3', 'London'),(4, 'thermo_4', 'Stockholm'); \endif \if :IS_TIME_DIMENSION SELECT table_name FROM create_hypertable('conditions', 'time'); \else SELECT table_name FROM create_hypertable('conditions', 'time', chunk_time_interval => 10); table_name ------------ conditions \endif \if :IS_TIME_DIMENSION INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 01:00:00-00', 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-02 01:00:00-00', 20, 3); \else CREATE OR REPLACE FUNCTION integer_now() RETURNS :TIME_DIMENSION_DATATYPE LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now'); set_integer_now_func ---------------------- INSERT INTO conditions ("time", temperature, device_id) VALUES (1, 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES (2, 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES (5, 20, 3); \endif \ir include/cagg_on_cagg_common.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGG on hypertable (1st level) CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, SUM(temperature) AS temperature, device_id FROM conditions GROUP BY 1, 3 ORDER BY 1, 2, 3 WITH NO DATA; -- CAGG on CAGG (2th level) CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , devices.device_id , devices.name FROM :CAGG_NAME_1ST_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_1ST_LEVEL.device_id GROUP BY 1, 3, 4 ORDER BY 1, 2, 3, 4 \else FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- CAGG on CAGG (3th level) CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , :CAGG_NAME_2TH_LEVEL.device_id , :CAGG_NAME_2TH_LEVEL.name , devices.location FROM :CAGG_NAME_2TH_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_2TH_LEVEL.device_id GROUP BY 1, 3, 4, 5 ORDER BY 1, 2, 3, 4, 5 \else FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- Check chunk_interval \if :IS_TIME_DIMENSION SELECT h.table_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; \else SELECT h.table_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; name | chunk_interval -------------------------+---------------- conditions | 10 conditions_summary_1_1 | 100 conditions_summary_2_5 | 100 conditions_summary_3_10 | 100 \endif -- No data because the CAGGs are just for materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 5 | 2 5 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- 0 | 15 5 | 20 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- 0 | 35 -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 5 | 2 5 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- 0 | 15 5 | 20 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- 0 | 35 \if :IS_TIME_DIMENSION -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-01 01:00:00-00'::timestamptz, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-03 01:00:00-00'::timestamptz, 2); \else -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES (2, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES (10, 2); \endif -- No changes SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 5 | 2 5 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- 0 | 15 5 | 20 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- 0 | 35 -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime changes, just new region SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 5 | 2 5 | 20 | 3 10 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- 0 | 15 5 | 20 10 | 2 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- 0 | 35 10 | 2 -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- All changes are materialized SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 2 | 2 | 5 | 2 5 | 20 | 3 10 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- 0 | 17 5 | 20 10 | 2 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- 0 | 37 10 | 2 -- TRUNCATE tests TRUNCATE :CAGG_NAME_2TH_LEVEL; -- This full refresh will remove all the data from the 3TH level cagg CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Should return no rows SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- -- If we have all the data in the bottom levels caggs we can rebuild CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Now we have all the data SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- 0 | 17 5 | 20 10 | 2 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- 0 | 37 10 | 2 -- DROP tests \set ON_ERROR_STOP 0 -- should error because it depends of other CAGGs DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:172: ERROR: cannot drop view conditions_summary_1_1 because other objects depend on it DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:173: ERROR: cannot drop view conditions_summary_2_5 because other objects depend on it CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:174: NOTICE: continuous aggregate "conditions_summary_1_1" is already up-to-date CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:175: NOTICE: continuous aggregate "conditions_summary_2_5" is already up-to-date \set ON_ERROR_STOP 1 -- DROP the 3TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:179: NOTICE: drop cascades to table _timescaledb_internal._hyper_8_11_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:182: ERROR: relation "conditions_summary_3_10" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs TRUNCATE :CAGG_NAME_2TH_LEVEL,:CAGG_NAME_1ST_LEVEL; CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 2 | 2 | 5 | 2 5 | 20 | 3 10 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- -- DROP the 2TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:196: ERROR: relation "conditions_summary_2_5" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 2 | 2 | 5 | 2 5 | 20 | 3 10 | 2 | -- DROP the first CAGG should work DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:203: NOTICE: drop cascades to table _timescaledb_internal._hyper_6_14_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:206: ERROR: relation "conditions_summary_1_1" does not exist at character 15 \set ON_ERROR_STOP 1 -- -- Validation test for non-multiple bucket sizes -- \set BUCKET_WIDTH_1ST 'INTEGER \'2\'' \set BUCKET_WIDTH_2TH 'INTEGER \'5\'' \set WARNING_MESSAGE '-- SHOULD ERROR because non-multiple bucket sizes' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+---------+-----------+----------+---------+---------+------------- bucket | integer | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_9.bucket, _materialized_hypertable_9.temperature FROM _timescaledb_internal._materialized_hypertable_9 WHERE _materialized_hypertable_9.bucket < COALESCE(_timescaledb_functions.cagg_watermark(9)::integer, '-2147483648'::integer) UNION ALL SELECT time_bucket(2, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.cagg_watermark(9)::integer, '-2147483648'::integer) GROUP BY (time_bucket(2, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD ERROR because non-multiple bucket sizes CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with incompatible bucket width DETAIL: Time bucket width of "public.conditions_summary_2" [5] should be multiple of the time bucket width of "public.conditions_summary_1" [2]. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- Validation test for equal bucket sizes -- \set BUCKET_WIDTH_1ST 'INTEGER \'2\'' \set BUCKET_WIDTH_2TH 'INTEGER \'2\'' \set WARNING_MESSAGE 'SHOULD WORK because new bucket should be greater than previous' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+---------+-----------+----------+---------+---------+------------- bucket | integer | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_10.bucket, _materialized_hypertable_10.temperature FROM _timescaledb_internal._materialized_hypertable_10 WHERE _materialized_hypertable_10.bucket < COALESCE(_timescaledb_functions.cagg_watermark(10)::integer, '-2147483648'::integer) UNION ALL SELECT time_bucket(2, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.cagg_watermark(10)::integer, '-2147483648'::integer) GROUP BY (time_bucket(2, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE SHOULD WORK because new bucket should be greater than previous CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+---------+-----------+----------+---------+---------+------------- bucket | integer | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_11.bucket, _materialized_hypertable_11.temperature FROM _timescaledb_internal._materialized_hypertable_11 WHERE _materialized_hypertable_11.bucket < COALESCE(_timescaledb_functions.cagg_watermark(11)::integer, '-2147483648'::integer) UNION ALL SELECT time_bucket(2, conditions_summary_1.bucket) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.cagg_watermark(11)::integer, '-2147483648'::integer) GROUP BY (time_bucket(2, conditions_summary_1.bucket)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- Validation test for bucket size less than source -- \set BUCKET_WIDTH_1ST 'INTEGER \'4\'' \set BUCKET_WIDTH_2TH 'INTEGER \'2\'' \set WARNING_MESSAGE '-- SHOULD ERROR because new bucket should be greater than previous' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+---------+-----------+----------+---------+---------+------------- bucket | integer | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_12.bucket, _materialized_hypertable_12.temperature FROM _timescaledb_internal._materialized_hypertable_12 WHERE _materialized_hypertable_12.bucket < COALESCE(_timescaledb_functions.cagg_watermark(12)::integer, '-2147483648'::integer) UNION ALL SELECT time_bucket(4, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.cagg_watermark(12)::integer, '-2147483648'::integer) GROUP BY (time_bucket(4, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD ERROR because new bucket should be greater than previous CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with incompatible bucket width DETAIL: Time bucket width of "public.conditions_summary_2" [2] should be greater or equal than the time bucket width of "public.conditions_summary_1" [4]. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- ######################################################## -- ## TIMESTAMP data type tests -- ######################################################## -- Current test variables \set IS_TIME_DIMENSION TRUE \set TIME_DIMENSION_DATATYPE TIMESTAMP \set CAGG_NAME_1ST_LEVEL conditions_summary_1_hourly \set CAGG_NAME_2TH_LEVEL conditions_summary_2_daily \set CAGG_NAME_3TH_LEVEL conditions_summary_3_weekly SET timezone TO 'UTC'; -- -- Run common tests for TIMESTAMP -- \set BUCKET_WIDTH_1ST 'INTERVAL \'1 hour\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 day\'' \set BUCKET_WIDTH_3TH 'INTERVAL \'1 week\'' -- Different order of time dimension in raw ht \set IS_DEFAULT_COLUMN_ORDER FALSE \ir include/cagg_on_cagg_setup.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGGs on CAGGs tests SET ROLE :ROLE_DEFAULT_PERM_USER; DROP TABLE IF EXISTS conditions CASCADE; \if :IS_DEFAULT_COLUMN_ORDER CREATE TABLE conditions ( time :TIME_DIMENSION_DATATYPE NOT NULL, temperature NUMERIC, device_id INT ); \else CREATE TABLE conditions ( temperature NUMERIC, time :TIME_DIMENSION_DATATYPE NOT NULL, device_id INT ); \endif \if :IS_JOIN DROP TABLE IF EXISTS devices CASCADE; CREATE TABLE devices ( device_id int not null, name text, location text); INSERT INTO devices values (1, 'thermo_1', 'Moscow'), (2, 'thermo_2', 'Berlin'),(3, 'thermo_3', 'London'),(4, 'thermo_4', 'Stockholm'); \endif \if :IS_TIME_DIMENSION SELECT table_name FROM create_hypertable('conditions', 'time'); psql:include/cagg_on_cagg_setup.sql:30: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices table_name ------------ conditions \else SELECT table_name FROM create_hypertable('conditions', 'time', chunk_time_interval => 10); \endif \if :IS_TIME_DIMENSION INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 01:00:00-00', 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-02 01:00:00-00', 20, 3); \else CREATE OR REPLACE FUNCTION integer_now() RETURNS :TIME_DIMENSION_DATATYPE LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now'); INSERT INTO conditions ("time", temperature, device_id) VALUES (1, 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES (2, 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES (5, 20, 3); \endif \ir include/cagg_on_cagg_common.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGG on hypertable (1st level) CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, SUM(temperature) AS temperature, device_id FROM conditions GROUP BY 1, 3 ORDER BY 1, 2, 3 WITH NO DATA; -- CAGG on CAGG (2th level) CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , devices.device_id , devices.name FROM :CAGG_NAME_1ST_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_1ST_LEVEL.device_id GROUP BY 1, 3, 4 ORDER BY 1, 2, 3, 4 \else FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- CAGG on CAGG (3th level) CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , :CAGG_NAME_2TH_LEVEL.device_id , :CAGG_NAME_2TH_LEVEL.name , devices.location FROM :CAGG_NAME_2TH_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_2TH_LEVEL.device_id GROUP BY 1, 3, 4, 5 ORDER BY 1, 2, 3, 4, 5 \else FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- Check chunk_interval \if :IS_TIME_DIMENSION SELECT h.table_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; name | chunk_interval -----------------------------+---------------- conditions | @ 7 days conditions_summary_1_hourly | @ 70 days conditions_summary_2_daily | @ 70 days conditions_summary_3_weekly | @ 70 days \else SELECT h.table_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; \endif -- No data because the CAGGs are just for materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------------------------+------------- Sat Jan 01 00:00:00 2022 | 15 Sun Jan 02 00:00:00 2022 | 20 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------------------------+------------- Mon Dec 27 00:00:00 2021 | 35 -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------------------------+------------- Sat Jan 01 00:00:00 2022 | 15 Sun Jan 02 00:00:00 2022 | 20 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------------------------+------------- Mon Dec 27 00:00:00 2021 | 35 \if :IS_TIME_DIMENSION -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-01 01:00:00-00'::timestamptz, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-03 01:00:00-00'::timestamptz, 2); \else -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES (2, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES (10, 2); \endif -- No changes SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------------------------+------------- Sat Jan 01 00:00:00 2022 | 15 Sun Jan 02 00:00:00 2022 | 20 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------------------------+------------- Mon Dec 27 00:00:00 2021 | 35 -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime changes, just new region SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 Mon Jan 03 01:00:00 2022 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------------------------+------------- Sat Jan 01 00:00:00 2022 | 15 Sun Jan 02 00:00:00 2022 | 20 Mon Jan 03 00:00:00 2022 | 2 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------------------------+------------- Mon Dec 27 00:00:00 2021 | 35 Mon Jan 03 00:00:00 2022 | 2 -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- All changes are materialized SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 2 | Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 Mon Jan 03 01:00:00 2022 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------------------------+------------- Sat Jan 01 00:00:00 2022 | 17 Sun Jan 02 00:00:00 2022 | 20 Mon Jan 03 00:00:00 2022 | 2 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------------------------+------------- Mon Dec 27 00:00:00 2021 | 37 Mon Jan 03 00:00:00 2022 | 2 -- TRUNCATE tests TRUNCATE :CAGG_NAME_2TH_LEVEL; -- This full refresh will remove all the data from the 3TH level cagg CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Should return no rows SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- -- If we have all the data in the bottom levels caggs we can rebuild CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Now we have all the data SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------------------------+------------- Sat Jan 01 00:00:00 2022 | 17 Sun Jan 02 00:00:00 2022 | 20 Mon Jan 03 00:00:00 2022 | 2 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------------------------+------------- Mon Dec 27 00:00:00 2021 | 37 Mon Jan 03 00:00:00 2022 | 2 -- DROP tests \set ON_ERROR_STOP 0 -- should error because it depends of other CAGGs DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:172: ERROR: cannot drop view conditions_summary_1_hourly because other objects depend on it DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:173: ERROR: cannot drop view conditions_summary_2_daily because other objects depend on it CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:174: NOTICE: continuous aggregate "conditions_summary_1_hourly" is already up-to-date CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:175: NOTICE: continuous aggregate "conditions_summary_2_daily" is already up-to-date \set ON_ERROR_STOP 1 -- DROP the 3TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:179: NOTICE: drop cascades to table _timescaledb_internal._hyper_16_18_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:182: ERROR: relation "conditions_summary_3_weekly" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs TRUNCATE :CAGG_NAME_2TH_LEVEL,:CAGG_NAME_1ST_LEVEL; CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 2 | Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 Mon Jan 03 01:00:00 2022 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- -- DROP the 2TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:196: ERROR: relation "conditions_summary_2_daily" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 2 | Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 Mon Jan 03 01:00:00 2022 | 2 | -- DROP the first CAGG should work DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:203: NOTICE: drop cascades to table _timescaledb_internal._hyper_14_20_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:206: ERROR: relation "conditions_summary_1_hourly" does not exist at character 15 \set ON_ERROR_STOP 1 -- Default tests \set IS_DEFAULT_COLUMN_ORDER TRUE \ir include/cagg_on_cagg_setup.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGGs on CAGGs tests SET ROLE :ROLE_DEFAULT_PERM_USER; DROP TABLE IF EXISTS conditions CASCADE; \if :IS_DEFAULT_COLUMN_ORDER CREATE TABLE conditions ( time :TIME_DIMENSION_DATATYPE NOT NULL, temperature NUMERIC, device_id INT ); \else CREATE TABLE conditions ( temperature NUMERIC, time :TIME_DIMENSION_DATATYPE NOT NULL, device_id INT ); \endif \if :IS_JOIN DROP TABLE IF EXISTS devices CASCADE; CREATE TABLE devices ( device_id int not null, name text, location text); INSERT INTO devices values (1, 'thermo_1', 'Moscow'), (2, 'thermo_2', 'Berlin'),(3, 'thermo_3', 'London'),(4, 'thermo_4', 'Stockholm'); \endif \if :IS_TIME_DIMENSION SELECT table_name FROM create_hypertable('conditions', 'time'); psql:include/cagg_on_cagg_setup.sql:30: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices table_name ------------ conditions \else SELECT table_name FROM create_hypertable('conditions', 'time', chunk_time_interval => 10); \endif \if :IS_TIME_DIMENSION INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 01:00:00-00', 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-02 01:00:00-00', 20, 3); \else CREATE OR REPLACE FUNCTION integer_now() RETURNS :TIME_DIMENSION_DATATYPE LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now'); INSERT INTO conditions ("time", temperature, device_id) VALUES (1, 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES (2, 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES (5, 20, 3); \endif \ir include/cagg_on_cagg_common.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGG on hypertable (1st level) CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, SUM(temperature) AS temperature, device_id FROM conditions GROUP BY 1, 3 ORDER BY 1, 2, 3 WITH NO DATA; -- CAGG on CAGG (2th level) CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , devices.device_id , devices.name FROM :CAGG_NAME_1ST_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_1ST_LEVEL.device_id GROUP BY 1, 3, 4 ORDER BY 1, 2, 3, 4 \else FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- CAGG on CAGG (3th level) CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , :CAGG_NAME_2TH_LEVEL.device_id , :CAGG_NAME_2TH_LEVEL.name , devices.location FROM :CAGG_NAME_2TH_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_2TH_LEVEL.device_id GROUP BY 1, 3, 4, 5 ORDER BY 1, 2, 3, 4, 5 \else FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- Check chunk_interval \if :IS_TIME_DIMENSION SELECT h.table_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; name | chunk_interval -----------------------------+---------------- conditions | @ 7 days conditions_summary_1_hourly | @ 70 days conditions_summary_2_daily | @ 70 days conditions_summary_3_weekly | @ 70 days \else SELECT h.table_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; \endif -- No data because the CAGGs are just for materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------------------------+------------- Sat Jan 01 00:00:00 2022 | 15 Sun Jan 02 00:00:00 2022 | 20 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------------------------+------------- Mon Dec 27 00:00:00 2021 | 35 -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------------------------+------------- Sat Jan 01 00:00:00 2022 | 15 Sun Jan 02 00:00:00 2022 | 20 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------------------------+------------- Mon Dec 27 00:00:00 2021 | 35 \if :IS_TIME_DIMENSION -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-01 01:00:00-00'::timestamptz, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-03 01:00:00-00'::timestamptz, 2); \else -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES (2, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES (10, 2); \endif -- No changes SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------------------------+------------- Sat Jan 01 00:00:00 2022 | 15 Sun Jan 02 00:00:00 2022 | 20 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------------------------+------------- Mon Dec 27 00:00:00 2021 | 35 -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime changes, just new region SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 Mon Jan 03 01:00:00 2022 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------------------------+------------- Sat Jan 01 00:00:00 2022 | 15 Sun Jan 02 00:00:00 2022 | 20 Mon Jan 03 00:00:00 2022 | 2 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------------------------+------------- Mon Dec 27 00:00:00 2021 | 35 Mon Jan 03 00:00:00 2022 | 2 -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- All changes are materialized SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 2 | Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 Mon Jan 03 01:00:00 2022 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------------------------+------------- Sat Jan 01 00:00:00 2022 | 17 Sun Jan 02 00:00:00 2022 | 20 Mon Jan 03 00:00:00 2022 | 2 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------------------------+------------- Mon Dec 27 00:00:00 2021 | 37 Mon Jan 03 00:00:00 2022 | 2 -- TRUNCATE tests TRUNCATE :CAGG_NAME_2TH_LEVEL; -- This full refresh will remove all the data from the 3TH level cagg CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Should return no rows SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- -- If we have all the data in the bottom levels caggs we can rebuild CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Now we have all the data SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------------------------+------------- Sat Jan 01 00:00:00 2022 | 17 Sun Jan 02 00:00:00 2022 | 20 Mon Jan 03 00:00:00 2022 | 2 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------------------------+------------- Mon Dec 27 00:00:00 2021 | 37 Mon Jan 03 00:00:00 2022 | 2 -- DROP tests \set ON_ERROR_STOP 0 -- should error because it depends of other CAGGs DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:172: ERROR: cannot drop view conditions_summary_1_hourly because other objects depend on it DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:173: ERROR: cannot drop view conditions_summary_2_daily because other objects depend on it CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:174: NOTICE: continuous aggregate "conditions_summary_1_hourly" is already up-to-date CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:175: NOTICE: continuous aggregate "conditions_summary_2_daily" is already up-to-date \set ON_ERROR_STOP 1 -- DROP the 3TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:179: NOTICE: drop cascades to table _timescaledb_internal._hyper_20_24_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:182: ERROR: relation "conditions_summary_3_weekly" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs TRUNCATE :CAGG_NAME_2TH_LEVEL,:CAGG_NAME_1ST_LEVEL; CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 2 | Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 Mon Jan 03 01:00:00 2022 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- -- DROP the 2TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:196: ERROR: relation "conditions_summary_2_daily" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 2 | Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 Mon Jan 03 01:00:00 2022 | 2 | -- DROP the first CAGG should work DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:203: NOTICE: drop cascades to table _timescaledb_internal._hyper_18_26_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:206: ERROR: relation "conditions_summary_1_hourly" does not exist at character 15 \set ON_ERROR_STOP 1 -- -- Validation test for variable bucket on top of fixed bucket -- \set BUCKET_WIDTH_1ST 'INTERVAL \'1 month\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'60 days\'' \set WARNING_MESSAGE '-- SHOULD ERROR because is not allowed variable-size bucket on top of fixed-size bucket' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+-----------------------------+-----------+----------+---------+---------+------------- bucket | timestamp without time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_21.bucket, _materialized_hypertable_21.temperature FROM _timescaledb_internal._materialized_hypertable_21 WHERE _materialized_hypertable_21.bucket < COALESCE(_timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(21)), '-infinity'::timestamp without time zone) UNION ALL SELECT time_bucket('@ 1 mon'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(21)), '-infinity'::timestamp without time zone) GROUP BY (time_bucket('@ 1 mon'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD ERROR because is not allowed variable-size bucket on top of fixed-size bucket CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with fixed-width bucket on top of one using variable-width bucket DETAIL: Continuous aggregate with a fixed time bucket width (e.g. 61 days) cannot be created on top of one using variable time bucket width (e.g. 1 month). The variance can lead to the fixed width one not being a multiple of the variable width one. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- Validation test for non-multiple bucket sizes -- \set BUCKET_WIDTH_1ST 'INTERVAL \'2 hours\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'3 hours\'' \set WARNING_MESSAGE '-- SHOULD ERROR because non-multiple bucket sizes' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+-----------------------------+-----------+----------+---------+---------+------------- bucket | timestamp without time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_22.bucket, _materialized_hypertable_22.temperature FROM _timescaledb_internal._materialized_hypertable_22 WHERE _materialized_hypertable_22.bucket < COALESCE(_timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(22)), '-infinity'::timestamp without time zone) UNION ALL SELECT time_bucket('@ 2 hours'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(22)), '-infinity'::timestamp without time zone) GROUP BY (time_bucket('@ 2 hours'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD ERROR because non-multiple bucket sizes CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with incompatible bucket width DETAIL: Time bucket width of "public.conditions_summary_2" [@ 3 hours] should be multiple of the time bucket width of "public.conditions_summary_1" [@ 2 hours]. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- Validation test for equal bucket sizes -- \set BUCKET_WIDTH_1ST 'INTERVAL \'1 hour\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 hour\'' \set WARNING_MESSAGE 'SHOULD WORK because new bucket should be greater than previous' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+-----------------------------+-----------+----------+---------+---------+------------- bucket | timestamp without time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_23.bucket, _materialized_hypertable_23.temperature FROM _timescaledb_internal._materialized_hypertable_23 WHERE _materialized_hypertable_23.bucket < COALESCE(_timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(23)), '-infinity'::timestamp without time zone) UNION ALL SELECT time_bucket('@ 1 hour'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(23)), '-infinity'::timestamp without time zone) GROUP BY (time_bucket('@ 1 hour'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE SHOULD WORK because new bucket should be greater than previous CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+-----------------------------+-----------+----------+---------+---------+------------- bucket | timestamp without time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_24.bucket, _materialized_hypertable_24.temperature FROM _timescaledb_internal._materialized_hypertable_24 WHERE _materialized_hypertable_24.bucket < COALESCE(_timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(24)), '-infinity'::timestamp without time zone) UNION ALL SELECT time_bucket('@ 1 hour'::interval, conditions_summary_1.bucket) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(24)), '-infinity'::timestamp without time zone) GROUP BY (time_bucket('@ 1 hour'::interval, conditions_summary_1.bucket)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- Validation test for bucket size less than source -- \set BUCKET_WIDTH_1ST 'INTERVAL \'2 hours\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 hour\'' \set WARNING_MESSAGE '-- SHOULD ERROR because new bucket should be greater than previous' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+-----------------------------+-----------+----------+---------+---------+------------- bucket | timestamp without time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_25.bucket, _materialized_hypertable_25.temperature FROM _timescaledb_internal._materialized_hypertable_25 WHERE _materialized_hypertable_25.bucket < COALESCE(_timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(25)), '-infinity'::timestamp without time zone) UNION ALL SELECT time_bucket('@ 2 hours'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(25)), '-infinity'::timestamp without time zone) GROUP BY (time_bucket('@ 2 hours'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD ERROR because new bucket should be greater than previous CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with incompatible bucket width DETAIL: Time bucket width of "public.conditions_summary_2" [@ 1 hour] should be greater or equal than the time bucket width of "public.conditions_summary_1" [@ 2 hours]. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- ######################################################## -- ## TIMESTAMPTZ data type tests -- ######################################################## -- Current test variables \set IS_TIME_DIMENSION TRUE \set TIME_DIMENSION_DATATYPE TIMESTAMPTZ \set CAGG_NAME_1ST_LEVEL conditions_summary_1_hourly \set CAGG_NAME_2TH_LEVEL conditions_summary_2_daily \set CAGG_NAME_3TH_LEVEL conditions_summary_3_weekly SET timezone TO 'UTC'; -- -- Run common tests for TIMESTAMPTZ -- \set BUCKET_WIDTH_1ST 'INTERVAL \'1 hour\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 day\'' \set BUCKET_WIDTH_3TH 'INTERVAL \'1 week\'' -- Different order of time dimension in raw ht \set IS_DEFAULT_COLUMN_ORDER FALSE \ir include/cagg_on_cagg_setup.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGGs on CAGGs tests SET ROLE :ROLE_DEFAULT_PERM_USER; DROP TABLE IF EXISTS conditions CASCADE; \if :IS_DEFAULT_COLUMN_ORDER CREATE TABLE conditions ( time :TIME_DIMENSION_DATATYPE NOT NULL, temperature NUMERIC, device_id INT ); \else CREATE TABLE conditions ( temperature NUMERIC, time :TIME_DIMENSION_DATATYPE NOT NULL, device_id INT ); \endif \if :IS_JOIN DROP TABLE IF EXISTS devices CASCADE; CREATE TABLE devices ( device_id int not null, name text, location text); INSERT INTO devices values (1, 'thermo_1', 'Moscow'), (2, 'thermo_2', 'Berlin'),(3, 'thermo_3', 'London'),(4, 'thermo_4', 'Stockholm'); \endif \if :IS_TIME_DIMENSION SELECT table_name FROM create_hypertable('conditions', 'time'); table_name ------------ conditions \else SELECT table_name FROM create_hypertable('conditions', 'time', chunk_time_interval => 10); \endif \if :IS_TIME_DIMENSION INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 01:00:00-00', 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-02 01:00:00-00', 20, 3); \else CREATE OR REPLACE FUNCTION integer_now() RETURNS :TIME_DIMENSION_DATATYPE LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now'); INSERT INTO conditions ("time", temperature, device_id) VALUES (1, 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES (2, 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES (5, 20, 3); \endif \ir include/cagg_on_cagg_common.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGG on hypertable (1st level) CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, SUM(temperature) AS temperature, device_id FROM conditions GROUP BY 1, 3 ORDER BY 1, 2, 3 WITH NO DATA; -- CAGG on CAGG (2th level) CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , devices.device_id , devices.name FROM :CAGG_NAME_1ST_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_1ST_LEVEL.device_id GROUP BY 1, 3, 4 ORDER BY 1, 2, 3, 4 \else FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- CAGG on CAGG (3th level) CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , :CAGG_NAME_2TH_LEVEL.device_id , :CAGG_NAME_2TH_LEVEL.name , devices.location FROM :CAGG_NAME_2TH_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_2TH_LEVEL.device_id GROUP BY 1, 3, 4, 5 ORDER BY 1, 2, 3, 4, 5 \else FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- Check chunk_interval \if :IS_TIME_DIMENSION SELECT h.table_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; name | chunk_interval -----------------------------+---------------- conditions | @ 7 days conditions_summary_1_hourly | @ 70 days conditions_summary_2_daily | @ 70 days conditions_summary_3_weekly | @ 70 days \else SELECT h.table_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; \endif -- No data because the CAGGs are just for materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature ------------------------------+------------- Sat Jan 01 00:00:00 2022 UTC | 15 Sun Jan 02 00:00:00 2022 UTC | 20 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature ------------------------------+------------- Mon Dec 27 00:00:00 2021 UTC | 35 -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature ------------------------------+------------- Sat Jan 01 00:00:00 2022 UTC | 15 Sun Jan 02 00:00:00 2022 UTC | 20 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature ------------------------------+------------- Mon Dec 27 00:00:00 2021 UTC | 35 \if :IS_TIME_DIMENSION -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-01 01:00:00-00'::timestamptz, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-03 01:00:00-00'::timestamptz, 2); \else -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES (2, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES (10, 2); \endif -- No changes SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature ------------------------------+------------- Sat Jan 01 00:00:00 2022 UTC | 15 Sun Jan 02 00:00:00 2022 UTC | 20 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature ------------------------------+------------- Mon Dec 27 00:00:00 2021 UTC | 35 -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime changes, just new region SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 Mon Jan 03 01:00:00 2022 UTC | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature ------------------------------+------------- Sat Jan 01 00:00:00 2022 UTC | 15 Sun Jan 02 00:00:00 2022 UTC | 20 Mon Jan 03 00:00:00 2022 UTC | 2 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature ------------------------------+------------- Mon Dec 27 00:00:00 2021 UTC | 35 Mon Jan 03 00:00:00 2022 UTC | 2 -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- All changes are materialized SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 2 | Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 Mon Jan 03 01:00:00 2022 UTC | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature ------------------------------+------------- Sat Jan 01 00:00:00 2022 UTC | 17 Sun Jan 02 00:00:00 2022 UTC | 20 Mon Jan 03 00:00:00 2022 UTC | 2 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature ------------------------------+------------- Mon Dec 27 00:00:00 2021 UTC | 37 Mon Jan 03 00:00:00 2022 UTC | 2 -- TRUNCATE tests TRUNCATE :CAGG_NAME_2TH_LEVEL; -- This full refresh will remove all the data from the 3TH level cagg CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Should return no rows SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- -- If we have all the data in the bottom levels caggs we can rebuild CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Now we have all the data SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature ------------------------------+------------- Sat Jan 01 00:00:00 2022 UTC | 17 Sun Jan 02 00:00:00 2022 UTC | 20 Mon Jan 03 00:00:00 2022 UTC | 2 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature ------------------------------+------------- Mon Dec 27 00:00:00 2021 UTC | 37 Mon Jan 03 00:00:00 2022 UTC | 2 -- DROP tests \set ON_ERROR_STOP 0 -- should error because it depends of other CAGGs DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:172: ERROR: cannot drop view conditions_summary_1_hourly because other objects depend on it DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:173: ERROR: cannot drop view conditions_summary_2_daily because other objects depend on it CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:174: NOTICE: continuous aggregate "conditions_summary_1_hourly" is already up-to-date CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:175: NOTICE: continuous aggregate "conditions_summary_2_daily" is already up-to-date \set ON_ERROR_STOP 1 -- DROP the 3TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:179: NOTICE: drop cascades to table _timescaledb_internal._hyper_29_30_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:182: ERROR: relation "conditions_summary_3_weekly" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs TRUNCATE :CAGG_NAME_2TH_LEVEL,:CAGG_NAME_1ST_LEVEL; CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 2 | Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 Mon Jan 03 01:00:00 2022 UTC | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- -- DROP the 2TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:196: ERROR: relation "conditions_summary_2_daily" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 2 | Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 Mon Jan 03 01:00:00 2022 UTC | 2 | -- DROP the first CAGG should work DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:203: NOTICE: drop cascades to table _timescaledb_internal._hyper_27_32_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:206: ERROR: relation "conditions_summary_1_hourly" does not exist at character 15 \set ON_ERROR_STOP 1 -- Default tests \set IS_DEFAULT_COLUMN_ORDER TRUE \ir include/cagg_on_cagg_setup.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGGs on CAGGs tests SET ROLE :ROLE_DEFAULT_PERM_USER; DROP TABLE IF EXISTS conditions CASCADE; \if :IS_DEFAULT_COLUMN_ORDER CREATE TABLE conditions ( time :TIME_DIMENSION_DATATYPE NOT NULL, temperature NUMERIC, device_id INT ); \else CREATE TABLE conditions ( temperature NUMERIC, time :TIME_DIMENSION_DATATYPE NOT NULL, device_id INT ); \endif \if :IS_JOIN DROP TABLE IF EXISTS devices CASCADE; CREATE TABLE devices ( device_id int not null, name text, location text); INSERT INTO devices values (1, 'thermo_1', 'Moscow'), (2, 'thermo_2', 'Berlin'),(3, 'thermo_3', 'London'),(4, 'thermo_4', 'Stockholm'); \endif \if :IS_TIME_DIMENSION SELECT table_name FROM create_hypertable('conditions', 'time'); table_name ------------ conditions \else SELECT table_name FROM create_hypertable('conditions', 'time', chunk_time_interval => 10); \endif \if :IS_TIME_DIMENSION INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 01:00:00-00', 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-02 01:00:00-00', 20, 3); \else CREATE OR REPLACE FUNCTION integer_now() RETURNS :TIME_DIMENSION_DATATYPE LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now'); INSERT INTO conditions ("time", temperature, device_id) VALUES (1, 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES (2, 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES (5, 20, 3); \endif \ir include/cagg_on_cagg_common.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGG on hypertable (1st level) CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, SUM(temperature) AS temperature, device_id FROM conditions GROUP BY 1, 3 ORDER BY 1, 2, 3 WITH NO DATA; -- CAGG on CAGG (2th level) CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , devices.device_id , devices.name FROM :CAGG_NAME_1ST_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_1ST_LEVEL.device_id GROUP BY 1, 3, 4 ORDER BY 1, 2, 3, 4 \else FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- CAGG on CAGG (3th level) CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , :CAGG_NAME_2TH_LEVEL.device_id , :CAGG_NAME_2TH_LEVEL.name , devices.location FROM :CAGG_NAME_2TH_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_2TH_LEVEL.device_id GROUP BY 1, 3, 4, 5 ORDER BY 1, 2, 3, 4, 5 \else FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- Check chunk_interval \if :IS_TIME_DIMENSION SELECT h.table_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; name | chunk_interval -----------------------------+---------------- conditions | @ 7 days conditions_summary_1_hourly | @ 70 days conditions_summary_2_daily | @ 70 days conditions_summary_3_weekly | @ 70 days \else SELECT h.table_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; \endif -- No data because the CAGGs are just for materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature ------------------------------+------------- Sat Jan 01 00:00:00 2022 UTC | 15 Sun Jan 02 00:00:00 2022 UTC | 20 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature ------------------------------+------------- Mon Dec 27 00:00:00 2021 UTC | 35 -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature ------------------------------+------------- Sat Jan 01 00:00:00 2022 UTC | 15 Sun Jan 02 00:00:00 2022 UTC | 20 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature ------------------------------+------------- Mon Dec 27 00:00:00 2021 UTC | 35 \if :IS_TIME_DIMENSION -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-01 01:00:00-00'::timestamptz, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-03 01:00:00-00'::timestamptz, 2); \else -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES (2, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES (10, 2); \endif -- No changes SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature ------------------------------+------------- Sat Jan 01 00:00:00 2022 UTC | 15 Sun Jan 02 00:00:00 2022 UTC | 20 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature ------------------------------+------------- Mon Dec 27 00:00:00 2021 UTC | 35 -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime changes, just new region SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 Mon Jan 03 01:00:00 2022 UTC | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature ------------------------------+------------- Sat Jan 01 00:00:00 2022 UTC | 15 Sun Jan 02 00:00:00 2022 UTC | 20 Mon Jan 03 00:00:00 2022 UTC | 2 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature ------------------------------+------------- Mon Dec 27 00:00:00 2021 UTC | 35 Mon Jan 03 00:00:00 2022 UTC | 2 -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- All changes are materialized SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 2 | Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 Mon Jan 03 01:00:00 2022 UTC | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature ------------------------------+------------- Sat Jan 01 00:00:00 2022 UTC | 17 Sun Jan 02 00:00:00 2022 UTC | 20 Mon Jan 03 00:00:00 2022 UTC | 2 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature ------------------------------+------------- Mon Dec 27 00:00:00 2021 UTC | 37 Mon Jan 03 00:00:00 2022 UTC | 2 -- TRUNCATE tests TRUNCATE :CAGG_NAME_2TH_LEVEL; -- This full refresh will remove all the data from the 3TH level cagg CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Should return no rows SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature --------+------------- -- If we have all the data in the bottom levels caggs we can rebuild CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Now we have all the data SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature ------------------------------+------------- Sat Jan 01 00:00:00 2022 UTC | 17 Sun Jan 02 00:00:00 2022 UTC | 20 Mon Jan 03 00:00:00 2022 UTC | 2 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature ------------------------------+------------- Mon Dec 27 00:00:00 2021 UTC | 37 Mon Jan 03 00:00:00 2022 UTC | 2 -- DROP tests \set ON_ERROR_STOP 0 -- should error because it depends of other CAGGs DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:172: ERROR: cannot drop view conditions_summary_1_hourly because other objects depend on it DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:173: ERROR: cannot drop view conditions_summary_2_daily because other objects depend on it CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:174: NOTICE: continuous aggregate "conditions_summary_1_hourly" is already up-to-date CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:175: NOTICE: continuous aggregate "conditions_summary_2_daily" is already up-to-date \set ON_ERROR_STOP 1 -- DROP the 3TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:179: NOTICE: drop cascades to table _timescaledb_internal._hyper_33_36_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:182: ERROR: relation "conditions_summary_3_weekly" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs TRUNCATE :CAGG_NAME_2TH_LEVEL,:CAGG_NAME_1ST_LEVEL; CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 2 | Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 Mon Jan 03 01:00:00 2022 UTC | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature --------+------------- -- DROP the 2TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:196: ERROR: relation "conditions_summary_2_daily" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 2 | Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 Mon Jan 03 01:00:00 2022 UTC | 2 | -- DROP the first CAGG should work DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:203: NOTICE: drop cascades to table _timescaledb_internal._hyper_31_38_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:206: ERROR: relation "conditions_summary_1_hourly" does not exist at character 15 \set ON_ERROR_STOP 1 -- -- Validation test for variable bucket on top of fixed bucket -- \set BUCKET_WIDTH_1ST 'INTERVAL \'1 month\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'60 days\'' \set WARNING_MESSAGE '-- SHOULD ERROR because is not allowed variable-size bucket on top of fixed-size bucket' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_34.bucket, _materialized_hypertable_34.temperature FROM _timescaledb_internal._materialized_hypertable_34 WHERE _materialized_hypertable_34.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(34)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 mon'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(34)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 mon'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD ERROR because is not allowed variable-size bucket on top of fixed-size bucket CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with fixed-width bucket on top of one using variable-width bucket DETAIL: Continuous aggregate with a fixed time bucket width (e.g. 61 days) cannot be created on top of one using variable time bucket width (e.g. 1 month). The variance can lead to the fixed width one not being a multiple of the variable width one. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- Validation test for non-multiple bucket sizes -- \set BUCKET_WIDTH_1ST 'INTERVAL \'2 hours\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'3 hours\'' \set WARNING_MESSAGE '-- SHOULD ERROR because non-multiple bucket sizes' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_35.bucket, _materialized_hypertable_35.temperature FROM _timescaledb_internal._materialized_hypertable_35 WHERE _materialized_hypertable_35.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(35)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 2 hours'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(35)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 2 hours'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD ERROR because non-multiple bucket sizes CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with incompatible bucket width DETAIL: Time bucket width of "public.conditions_summary_2" [@ 3 hours] should be multiple of the time bucket width of "public.conditions_summary_1" [@ 2 hours]. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- Validation test for equal bucket sizes -- \set BUCKET_WIDTH_1ST 'INTERVAL \'1 hour\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 hour\'' \set WARNING_MESSAGE 'SHOULD WORK because new bucket should be greater than previous' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_36.bucket, _materialized_hypertable_36.temperature FROM _timescaledb_internal._materialized_hypertable_36 WHERE _materialized_hypertable_36.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(36)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 hour'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(36)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 hour'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE SHOULD WORK because new bucket should be greater than previous CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_37.bucket, _materialized_hypertable_37.temperature FROM _timescaledb_internal._materialized_hypertable_37 WHERE _materialized_hypertable_37.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(37)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 hour'::interval, conditions_summary_1.bucket) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(37)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 hour'::interval, conditions_summary_1.bucket)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- Validation test for bucket size less than source -- \set BUCKET_WIDTH_1ST 'INTERVAL \'2 hours\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 hour\'' \set WARNING_MESSAGE '-- SHOULD ERROR because new bucket should be greater than previous' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_38.bucket, _materialized_hypertable_38.temperature FROM _timescaledb_internal._materialized_hypertable_38 WHERE _materialized_hypertable_38.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(38)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 2 hours'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(38)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 2 hours'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD ERROR because new bucket should be greater than previous CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with incompatible bucket width DETAIL: Time bucket width of "public.conditions_summary_2" [@ 1 hour] should be greater or equal than the time bucket width of "public.conditions_summary_1" [@ 2 hours]. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- Validations using time bucket with timezone (ref issue #5126) -- \set TIME_DIMENSION_DATATYPE TIMESTAMPTZ \set IS_TIME_DIMENSION_WITH_TIMEZONE_1ST TRUE \set IS_TIME_DIMENSION_WITH_TIMEZONE_2TH TRUE \set CAGG_NAME_1ST_LEVEL conditions_summary_1_5m \set CAGG_NAME_2TH_LEVEL conditions_summary_2_1h \set BUCKET_TZNAME_1ST 'US/Pacific' \set BUCKET_TZNAME_2TH 'US/Pacific' \set BUCKET_WIDTH_1ST 'INTERVAL \'5 minutes\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 hour\'' \set WARNING_MESSAGE '-- SHOULD WORK' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_39.bucket, _materialized_hypertable_39.temperature FROM _timescaledb_internal._materialized_hypertable_39 WHERE _materialized_hypertable_39.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(39)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 5 mins'::interval, conditions."time", 'US/Pacific'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(39)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 5 mins'::interval, conditions."time", 'US/Pacific'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_40.bucket, _materialized_hypertable_40.temperature FROM _timescaledb_internal._materialized_hypertable_40 WHERE _materialized_hypertable_40.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(40)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 hour'::interval, conditions_summary_1.bucket, 'US/Pacific'::text) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(40)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 hour'::interval, conditions_summary_1.bucket, 'US/Pacific'::text)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; \set BUCKET_WIDTH_1ST 'INTERVAL \'5 minutes\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'16 minutes\'' \set WARNING_MESSAGE '-- SHOULD ERROR because non-multiple bucket sizes' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_41.bucket, _materialized_hypertable_41.temperature FROM _timescaledb_internal._materialized_hypertable_41 WHERE _materialized_hypertable_41.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(41)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 5 mins'::interval, conditions."time", 'US/Pacific'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(41)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 5 mins'::interval, conditions."time", 'US/Pacific'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD ERROR because non-multiple bucket sizes CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with incompatible bucket width DETAIL: Time bucket width of "public.conditions_summary_2" [@ 16 mins] should be multiple of the time bucket width of "public.conditions_summary_1" [@ 5 mins]. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- Variable bucket size with the same timezones -- \set BUCKET_TZNAME_1ST 'UTC' \set BUCKET_TZNAME_2TH 'UTC' \set BUCKET_WIDTH_1ST 'INTERVAL \'1 day\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 month\'' \set WARNING_MESSAGE '-- SHOULD WORK' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_42.bucket, _materialized_hypertable_42.temperature FROM _timescaledb_internal._materialized_hypertable_42 WHERE _materialized_hypertable_42.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(42)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions."time", 'UTC'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(42)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions."time", 'UTC'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_43.bucket, _materialized_hypertable_43.temperature FROM _timescaledb_internal._materialized_hypertable_43 WHERE _materialized_hypertable_43.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(43)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 mon'::interval, conditions_summary_1.bucket, 'UTC'::text) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(43)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 mon'::interval, conditions_summary_1.bucket, 'UTC'::text)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; --#Bugfix 5734 #1 \set INTERVAL_TEST TRUE \set BUCKET_WIDTH_1ST 'INTERVAL \'1 hour\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 day\'' \set BUCKET_WIDTH_3TH 'INTERVAL \'1 month\'' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_44.bucket, _materialized_hypertable_44.temperature FROM _timescaledb_internal._materialized_hypertable_44 WHERE _materialized_hypertable_44.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(44)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 hour'::interval, conditions."time", 'UTC'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(44)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 hour'::interval, conditions."time", 'UTC'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_45.bucket, _materialized_hypertable_45.temperature FROM _timescaledb_internal._materialized_hypertable_45 WHERE _materialized_hypertable_45.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(45)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions_summary_1.bucket, 'UTC'::text) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(45)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions_summary_1.bucket, 'UTC'::text)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; psql:include/cagg_on_cagg_validations.sql:71: NOTICE: refreshing continuous aggregate "conditions_summary_3" \d+ :CAGG_NAME_3TH_LEVEL View "public.conditions_summary_3" Column | Type | Collation | Nullable | Default | Storage | Description --------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | View definition: SELECT _materialized_hypertable_46.bucket FROM _timescaledb_internal._materialized_hypertable_46 WHERE _materialized_hypertable_46.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(46)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 mon'::interval, conditions_summary_2.bucket, 'UTC'::text) AS bucket FROM conditions_summary_2 WHERE conditions_summary_2.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(46)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 mon'::interval, conditions_summary_2.bucket, 'UTC'::text)); --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; bucket ------------------------------ Sat Jan 01 00:00:00 2022 UTC DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:79: NOTICE: drop cascades to table _timescaledb_internal._hyper_46_43_chunk DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: drop cascades to table _timescaledb_internal._hyper_45_42_chunk DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_validations.sql:87: NOTICE: drop cascades to table _timescaledb_internal._hyper_44_41_chunk \set INTERVAL_TEST FALSE -- -- Variable bucket size with different timezones -- \set BUCKET_TZNAME_1ST 'US/Pacific' \set BUCKET_TZNAME_2TH 'UTC' \set BUCKET_WIDTH_1ST 'INTERVAL \'1 day\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 month\'' \set WARNING_MESSAGE '-- SHOULD WORK' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_47.bucket, _materialized_hypertable_47.temperature FROM _timescaledb_internal._materialized_hypertable_47 WHERE _materialized_hypertable_47.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(47)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions."time", 'US/Pacific'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(47)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions."time", 'US/Pacific'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_48.bucket, _materialized_hypertable_48.temperature FROM _timescaledb_internal._materialized_hypertable_48 WHERE _materialized_hypertable_48.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(48)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 mon'::interval, conditions_summary_1.bucket, 'UTC'::text) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(48)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 mon'::interval, conditions_summary_1.bucket, 'UTC'::text)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; --#Bugfix 5734 #2 \set INTERVAL_TEST TRUE \set BUCKET_WIDTH_1ST 'INTERVAL \'1 hour\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 day\'' \set BUCKET_WIDTH_3TH 'INTERVAL \'1 month\'' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_49.bucket, _materialized_hypertable_49.temperature FROM _timescaledb_internal._materialized_hypertable_49 WHERE _materialized_hypertable_49.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(49)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 hour'::interval, conditions."time", 'US/Pacific'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(49)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 hour'::interval, conditions."time", 'US/Pacific'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_50.bucket, _materialized_hypertable_50.temperature FROM _timescaledb_internal._materialized_hypertable_50 WHERE _materialized_hypertable_50.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(50)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions_summary_1.bucket, 'UTC'::text) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(50)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions_summary_1.bucket, 'UTC'::text)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; psql:include/cagg_on_cagg_validations.sql:71: NOTICE: refreshing continuous aggregate "conditions_summary_3" \d+ :CAGG_NAME_3TH_LEVEL View "public.conditions_summary_3" Column | Type | Collation | Nullable | Default | Storage | Description --------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | View definition: SELECT _materialized_hypertable_51.bucket FROM _timescaledb_internal._materialized_hypertable_51 WHERE _materialized_hypertable_51.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(51)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 mon'::interval, conditions_summary_2.bucket, 'UTC'::text) AS bucket FROM conditions_summary_2 WHERE conditions_summary_2.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(51)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 mon'::interval, conditions_summary_2.bucket, 'UTC'::text)); --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; bucket ------------------------------ Sat Jan 01 00:00:00 2022 UTC DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:79: NOTICE: drop cascades to table _timescaledb_internal._hyper_51_46_chunk DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: drop cascades to table _timescaledb_internal._hyper_50_45_chunk DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_validations.sql:87: NOTICE: drop cascades to table _timescaledb_internal._hyper_49_44_chunk \set INTERVAL_TEST FALSE -- -- TZ bucket on top of non-TZ bucket -- \set IS_TIME_DIMENSION_WITH_TIMEZONE_1ST FALSE \set IS_TIME_DIMENSION_WITH_TIMEZONE_2TH TRUE \set BUCKET_TZNAME_2TH 'UTC' \set BUCKET_WIDTH_1ST 'INTERVAL \'1 day\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 month\'' \set WARNING_MESSAGE '-- SHOULD WORK' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_52.bucket, _materialized_hypertable_52.temperature FROM _timescaledb_internal._materialized_hypertable_52 WHERE _materialized_hypertable_52.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(52)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(52)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_53.bucket, _materialized_hypertable_53.temperature FROM _timescaledb_internal._materialized_hypertable_53 WHERE _materialized_hypertable_53.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(53)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 mon'::interval, conditions_summary_1.bucket, 'UTC'::text) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(53)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 mon'::interval, conditions_summary_1.bucket, 'UTC'::text)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; --#Bugfix 5734 #3 \set INTERVAL_TEST TRUE \set BUCKET_WIDTH_1ST 'INTERVAL \'1 hour\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 day\'' \set BUCKET_WIDTH_3TH 'INTERVAL \'1 month\'' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_54.bucket, _materialized_hypertable_54.temperature FROM _timescaledb_internal._materialized_hypertable_54 WHERE _materialized_hypertable_54.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(54)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 hour'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(54)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 hour'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_55.bucket, _materialized_hypertable_55.temperature FROM _timescaledb_internal._materialized_hypertable_55 WHERE _materialized_hypertable_55.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(55)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions_summary_1.bucket, 'UTC'::text) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(55)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions_summary_1.bucket, 'UTC'::text)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; psql:include/cagg_on_cagg_validations.sql:71: NOTICE: refreshing continuous aggregate "conditions_summary_3" \d+ :CAGG_NAME_3TH_LEVEL View "public.conditions_summary_3" Column | Type | Collation | Nullable | Default | Storage | Description --------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | View definition: SELECT _materialized_hypertable_56.bucket FROM _timescaledb_internal._materialized_hypertable_56 WHERE _materialized_hypertable_56.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(56)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 mon'::interval, conditions_summary_2.bucket, 'UTC'::text) AS bucket FROM conditions_summary_2 WHERE conditions_summary_2.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(56)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 mon'::interval, conditions_summary_2.bucket, 'UTC'::text)); --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; bucket ------------------------------ Sat Jan 01 00:00:00 2022 UTC DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:79: NOTICE: drop cascades to table _timescaledb_internal._hyper_56_49_chunk DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: drop cascades to table _timescaledb_internal._hyper_55_48_chunk DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_validations.sql:87: NOTICE: drop cascades to table _timescaledb_internal._hyper_54_47_chunk \set INTERVAL_TEST FALSE -- -- non-TZ bucket on top of TZ bucket -- \set IS_TIME_DIMENSION_WITH_TIMEZONE_1ST TRUE \set IS_TIME_DIMENSION_WITH_TIMEZONE_2TH FALSE \set BUCKET_TZNAME_1ST 'UTC' \set BUCKET_WIDTH_1ST 'INTERVAL \'1 day\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 month\'' \set WARNING_MESSAGE '-- SHOULD WORK' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_57.bucket, _materialized_hypertable_57.temperature FROM _timescaledb_internal._materialized_hypertable_57 WHERE _materialized_hypertable_57.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(57)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions."time", 'UTC'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(57)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions."time", 'UTC'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_58.bucket, _materialized_hypertable_58.temperature FROM _timescaledb_internal._materialized_hypertable_58 WHERE _materialized_hypertable_58.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(58)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 mon'::interval, conditions_summary_1.bucket) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(58)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 mon'::interval, conditions_summary_1.bucket)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- test some intuitive intervals that should work but -- were not working due to unix epochs -- validation test for 1 year on top of one day -- validation test for 1 year on top of 1 month -- validation test for 1 year on top of 1 week -- bug report 5231 \set BUCKET_WIDTH_1ST 'INTERVAL \'1 day\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 year\'' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_59.bucket, _materialized_hypertable_59.temperature FROM _timescaledb_internal._materialized_hypertable_59 WHERE _materialized_hypertable_59.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(59)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions."time", 'UTC'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(59)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions."time", 'UTC'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_60.bucket, _materialized_hypertable_60.temperature FROM _timescaledb_internal._materialized_hypertable_60 WHERE _materialized_hypertable_60.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(60)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 year'::interval, conditions_summary_1.bucket) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(60)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 year'::interval, conditions_summary_1.bucket)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; \set BUCKET_WIDTH_1ST 'INTERVAL \'1 day\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'3 month\'' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_61.bucket, _materialized_hypertable_61.temperature FROM _timescaledb_internal._materialized_hypertable_61 WHERE _materialized_hypertable_61.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(61)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions."time", 'UTC'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(61)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions."time", 'UTC'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_62.bucket, _materialized_hypertable_62.temperature FROM _timescaledb_internal._materialized_hypertable_62 WHERE _materialized_hypertable_62.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(62)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 3 mons'::interval, conditions_summary_1.bucket) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(62)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 3 mons'::interval, conditions_summary_1.bucket)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; \set BUCKET_WIDTH_1ST 'INTERVAL \'1 month\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 year\'' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_63.bucket, _materialized_hypertable_63.temperature FROM _timescaledb_internal._materialized_hypertable_63 WHERE _materialized_hypertable_63.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(63)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 mon'::interval, conditions."time", 'UTC'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(63)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 mon'::interval, conditions."time", 'UTC'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_64.bucket, _materialized_hypertable_64.temperature FROM _timescaledb_internal._materialized_hypertable_64 WHERE _materialized_hypertable_64.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(64)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 year'::interval, conditions_summary_1.bucket) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(64)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 year'::interval, conditions_summary_1.bucket)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; \set BUCKET_WIDTH_1ST 'INTERVAL \'1 week\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 year\'' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_65.bucket, _materialized_hypertable_65.temperature FROM _timescaledb_internal._materialized_hypertable_65 WHERE _materialized_hypertable_65.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(65)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 7 days'::interval, conditions."time", 'UTC'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(65)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 7 days'::interval, conditions."time", 'UTC'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with incompatible bucket width DETAIL: Time bucket width of "public.conditions_summary_2" [@ 1 year] should be multiple of the time bucket width of "public.conditions_summary_1" [@ 7 days]. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; \set BUCKET_WIDTH_1ST 'INTERVAL \'1 week\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 month\'' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_66.bucket, _materialized_hypertable_66.temperature FROM _timescaledb_internal._materialized_hypertable_66 WHERE _materialized_hypertable_66.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(66)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 7 days'::interval, conditions."time", 'UTC'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(66)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 7 days'::interval, conditions."time", 'UTC'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with incompatible bucket width DETAIL: Time bucket width of "public.conditions_summary_2" [@ 1 mon] should be multiple of the time bucket width of "public.conditions_summary_1" [@ 7 days]. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- bug report 5277 \set IS_TIME_DIMENSION_WITH_TIMEZONE_1ST FALSE \set IS_TIME_DIMENSION_WITH_TIMEZONE_2TH FALSE -- epoch plus cast to int would compute a bucket width of 0 for parent \set BUCKET_WIDTH_1ST 'INTERVAL \'146 ms\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1168 ms\'' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_67.bucket, _materialized_hypertable_67.temperature FROM _timescaledb_internal._materialized_hypertable_67 WHERE _materialized_hypertable_67.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(67)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 0.146 secs'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(67)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 0.146 secs'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_68.bucket, _materialized_hypertable_68.temperature FROM _timescaledb_internal._materialized_hypertable_68 WHERE _materialized_hypertable_68.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(68)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1.168 secs'::interval, conditions_summary_1.bucket) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(68)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1.168 secs'::interval, conditions_summary_1.bucket)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; \set BUCKET_WIDTH_1ST 'INTERVAL \'9344 ms\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'74752 ms\'' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_69.bucket, _materialized_hypertable_69.temperature FROM _timescaledb_internal._materialized_hypertable_69 WHERE _materialized_hypertable_69.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(69)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 9.344 secs'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(69)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 9.344 secs'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_70.bucket, _materialized_hypertable_70.temperature FROM _timescaledb_internal._materialized_hypertable_70 WHERE _materialized_hypertable_70.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(70)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 min 14.752 secs'::interval, conditions_summary_1.bucket) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(70)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 min 14.752 secs'::interval, conditions_summary_1.bucket)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; \set BUCKET_WIDTH_1ST 'INTERVAL \'74752 ms\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'598016 ms\'' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_71.bucket, _materialized_hypertable_71.temperature FROM _timescaledb_internal._materialized_hypertable_71 WHERE _materialized_hypertable_71.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(71)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 min 14.752 secs'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(71)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 min 14.752 secs'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_72.bucket, _materialized_hypertable_72.temperature FROM _timescaledb_internal._materialized_hypertable_72 WHERE _materialized_hypertable_72.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(72)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 9 mins 58.016 secs'::interval, conditions_summary_1.bucket) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(72)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 9 mins 58.016 secs'::interval, conditions_summary_1.bucket)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- test microseconds - should pass \set BUCKET_WIDTH_1ST 'INTERVAL \'146 usec\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1168 usec\'' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_73.bucket, _materialized_hypertable_73.temperature FROM _timescaledb_internal._materialized_hypertable_73 WHERE _materialized_hypertable_73.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(73)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 0.000146 secs'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(73)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 0.000146 secs'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_74.bucket, _materialized_hypertable_74.temperature FROM _timescaledb_internal._materialized_hypertable_74 WHERE _materialized_hypertable_74.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(74)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 0.001168 secs'::interval, conditions_summary_1.bucket) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(74)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 0.001168 secs'::interval, conditions_summary_1.bucket)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- test microseconds - SHOULD FAIL \set BUCKET_WIDTH_1ST 'INTERVAL \'146 usec\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1160 usec\'' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_75.bucket, _materialized_hypertable_75.temperature FROM _timescaledb_internal._materialized_hypertable_75 WHERE _materialized_hypertable_75.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(75)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 0.000146 secs'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(75)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 0.000146 secs'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with incompatible bucket width DETAIL: Time bucket width of "public.conditions_summary_2" [@ 0.00116 secs] should be multiple of the time bucket width of "public.conditions_summary_1" [@ 0.000146 secs]. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; ================================================ FILE: tsl/test/expected/cagg_on_cagg_joins.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- Global test variables \set IS_TIME_DIMENSION_WITH_TIMEZONE_1ST FALSE \set IS_TIME_DIMENSION_WITH_TIMEZONE_2TH FALSE \set IS_JOIN TRUE \set INTERVAL_TEST FALSE -- ######################################################## -- ## INTEGER data type tests -- ######################################################## -- Current test variables \set IS_TIME_DIMENSION FALSE \set TIME_DIMENSION_DATATYPE INTEGER \set CAGG_NAME_1ST_LEVEL conditions_summary_1_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2_5 \set CAGG_NAME_3TH_LEVEL conditions_summary_3_10 -- -- Run common tests for INTEGER -- \set BUCKET_WIDTH_1ST 'INTEGER \'1\'' \set BUCKET_WIDTH_2TH 'INTEGER \'5\'' \set BUCKET_WIDTH_3TH 'INTEGER \'10\'' -- Different order of time dimension in raw ht \set IS_DEFAULT_COLUMN_ORDER FALSE \ir include/cagg_on_cagg_setup.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGGs on CAGGs tests SET ROLE :ROLE_DEFAULT_PERM_USER; DROP TABLE IF EXISTS conditions CASCADE; psql:include/cagg_on_cagg_setup.sql:8: NOTICE: table "conditions" does not exist, skipping \if :IS_DEFAULT_COLUMN_ORDER CREATE TABLE conditions ( time :TIME_DIMENSION_DATATYPE NOT NULL, temperature NUMERIC, device_id INT ); \else CREATE TABLE conditions ( temperature NUMERIC, time :TIME_DIMENSION_DATATYPE NOT NULL, device_id INT ); \endif \if :IS_JOIN DROP TABLE IF EXISTS devices CASCADE; psql:include/cagg_on_cagg_setup.sql:24: NOTICE: table "devices" does not exist, skipping CREATE TABLE devices ( device_id int not null, name text, location text); INSERT INTO devices values (1, 'thermo_1', 'Moscow'), (2, 'thermo_2', 'Berlin'),(3, 'thermo_3', 'London'),(4, 'thermo_4', 'Stockholm'); \endif \if :IS_TIME_DIMENSION SELECT table_name FROM create_hypertable('conditions', 'time'); \else SELECT table_name FROM create_hypertable('conditions', 'time', chunk_time_interval => 10); table_name ------------ conditions \endif \if :IS_TIME_DIMENSION INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 01:00:00-00', 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-02 01:00:00-00', 20, 3); \else CREATE OR REPLACE FUNCTION integer_now() RETURNS :TIME_DIMENSION_DATATYPE LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now'); set_integer_now_func ---------------------- INSERT INTO conditions ("time", temperature, device_id) VALUES (1, 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES (2, 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES (5, 20, 3); \endif \ir include/cagg_on_cagg_common.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGG on hypertable (1st level) CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, SUM(temperature) AS temperature, device_id FROM conditions GROUP BY 1, 3 ORDER BY 1, 2, 3 WITH NO DATA; -- CAGG on CAGG (2th level) CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , devices.device_id , devices.name FROM :CAGG_NAME_1ST_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_1ST_LEVEL.device_id GROUP BY 1, 3, 4 ORDER BY 1, 2, 3, 4 \else FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- CAGG on CAGG (3th level) CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , :CAGG_NAME_2TH_LEVEL.device_id , :CAGG_NAME_2TH_LEVEL.name , devices.location FROM :CAGG_NAME_2TH_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_2TH_LEVEL.device_id GROUP BY 1, 3, 4, 5 ORDER BY 1, 2, 3, 4, 5 \else FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- Check chunk_interval \if :IS_TIME_DIMENSION SELECT h.table_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; \else SELECT h.table_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; name | chunk_interval -------------------------+---------------- conditions | 10 conditions_summary_1_1 | 100 conditions_summary_2_5 | 100 conditions_summary_3_10 | 100 \endif -- No data because the CAGGs are just for materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+------ SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+------+---------- -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 5 | 2 5 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+---------- 0 | 5 | 2 | thermo_2 0 | 10 | 1 | thermo_1 5 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+----------+---------- 0 | 5 | 2 | thermo_2 | Berlin 0 | 10 | 1 | thermo_1 | Moscow 0 | 20 | 3 | thermo_3 | London -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 5 | 2 5 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+---------- 0 | 5 | 2 | thermo_2 0 | 10 | 1 | thermo_1 5 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+----------+---------- 0 | 5 | 2 | thermo_2 | Berlin 0 | 10 | 1 | thermo_1 | Moscow 0 | 20 | 3 | thermo_3 | London \if :IS_TIME_DIMENSION -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-01 01:00:00-00'::timestamptz, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-03 01:00:00-00'::timestamptz, 2); \else -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES (2, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES (10, 2); \endif -- No changes SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 5 | 2 5 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+---------- 0 | 5 | 2 | thermo_2 0 | 10 | 1 | thermo_1 5 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+----------+---------- 0 | 5 | 2 | thermo_2 | Berlin 0 | 10 | 1 | thermo_1 | Moscow 0 | 20 | 3 | thermo_3 | London -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime changes, just new region SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 5 | 2 5 | 20 | 3 10 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+---------- 0 | 5 | 2 | thermo_2 0 | 10 | 1 | thermo_1 5 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+----------+---------- 0 | 5 | 2 | thermo_2 | Berlin 0 | 10 | 1 | thermo_1 | Moscow 0 | 20 | 3 | thermo_3 | London -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- All changes are materialized SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 2 | 2 | 5 | 2 5 | 20 | 3 10 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+---------- 0 | 5 | 2 | thermo_2 0 | 10 | 1 | thermo_1 5 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+----------+---------- 0 | 5 | 2 | thermo_2 | Berlin 0 | 10 | 1 | thermo_1 | Moscow 0 | 20 | 3 | thermo_3 | London -- TRUNCATE tests TRUNCATE :CAGG_NAME_2TH_LEVEL; -- This full refresh will remove all the data from the 3TH level cagg CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Should return no rows SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+------ SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+------+---------- -- If we have all the data in the bottom levels caggs we can rebuild CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Now we have all the data SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+---------- 0 | 5 | 2 | thermo_2 0 | 10 | 1 | thermo_1 5 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+----------+---------- 0 | 5 | 2 | thermo_2 | Berlin 0 | 10 | 1 | thermo_1 | Moscow 0 | 20 | 3 | thermo_3 | London -- DROP tests \set ON_ERROR_STOP 0 -- should error because it depends of other CAGGs DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:172: ERROR: cannot drop view conditions_summary_1_1 because other objects depend on it DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:173: ERROR: cannot drop view conditions_summary_2_5 because other objects depend on it CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:174: NOTICE: continuous aggregate "conditions_summary_1_1" is already up-to-date CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:175: NOTICE: continuous aggregate "conditions_summary_2_5" is already up-to-date \set ON_ERROR_STOP 1 -- DROP the 3TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:179: NOTICE: drop cascades to table _timescaledb_internal._hyper_4_4_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:182: ERROR: relation "conditions_summary_3_10" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs TRUNCATE :CAGG_NAME_2TH_LEVEL,:CAGG_NAME_1ST_LEVEL; CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 2 | 2 | 5 | 2 5 | 20 | 3 10 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+------ -- DROP the 2TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:196: ERROR: relation "conditions_summary_2_5" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 2 | 2 | 5 | 2 5 | 20 | 3 10 | 2 | -- DROP the first CAGG should work DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:203: NOTICE: drop cascades to table _timescaledb_internal._hyper_2_7_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:206: ERROR: relation "conditions_summary_1_1" does not exist at character 15 \set ON_ERROR_STOP 1 -- Default tests \set ON_ERROR_STOP 0 \set IS_DEFAULT_COLUMN_ORDER TRUE \ir include/cagg_on_cagg_setup.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGGs on CAGGs tests SET ROLE :ROLE_DEFAULT_PERM_USER; DROP TABLE IF EXISTS conditions CASCADE; \if :IS_DEFAULT_COLUMN_ORDER CREATE TABLE conditions ( time :TIME_DIMENSION_DATATYPE NOT NULL, temperature NUMERIC, device_id INT ); \else CREATE TABLE conditions ( temperature NUMERIC, time :TIME_DIMENSION_DATATYPE NOT NULL, device_id INT ); \endif \if :IS_JOIN DROP TABLE IF EXISTS devices CASCADE; CREATE TABLE devices ( device_id int not null, name text, location text); INSERT INTO devices values (1, 'thermo_1', 'Moscow'), (2, 'thermo_2', 'Berlin'),(3, 'thermo_3', 'London'),(4, 'thermo_4', 'Stockholm'); \endif \if :IS_TIME_DIMENSION SELECT table_name FROM create_hypertable('conditions', 'time'); \else SELECT table_name FROM create_hypertable('conditions', 'time', chunk_time_interval => 10); table_name ------------ conditions \endif \if :IS_TIME_DIMENSION INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 01:00:00-00', 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-02 01:00:00-00', 20, 3); \else CREATE OR REPLACE FUNCTION integer_now() RETURNS :TIME_DIMENSION_DATATYPE LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now'); set_integer_now_func ---------------------- INSERT INTO conditions ("time", temperature, device_id) VALUES (1, 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES (2, 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES (5, 20, 3); \endif \ir include/cagg_on_cagg_common.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGG on hypertable (1st level) CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, SUM(temperature) AS temperature, device_id FROM conditions GROUP BY 1, 3 ORDER BY 1, 2, 3 WITH NO DATA; -- CAGG on CAGG (2th level) CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , devices.device_id , devices.name FROM :CAGG_NAME_1ST_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_1ST_LEVEL.device_id GROUP BY 1, 3, 4 ORDER BY 1, 2, 3, 4 \else FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- CAGG on CAGG (3th level) CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , :CAGG_NAME_2TH_LEVEL.device_id , :CAGG_NAME_2TH_LEVEL.name , devices.location FROM :CAGG_NAME_2TH_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_2TH_LEVEL.device_id GROUP BY 1, 3, 4, 5 ORDER BY 1, 2, 3, 4, 5 \else FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- Check chunk_interval \if :IS_TIME_DIMENSION SELECT h.table_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; \else SELECT h.table_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; name | chunk_interval -------------------------+---------------- conditions | 10 conditions_summary_1_1 | 100 conditions_summary_2_5 | 100 conditions_summary_3_10 | 100 \endif -- No data because the CAGGs are just for materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+------ SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+------+---------- -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 5 | 2 5 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+---------- 0 | 5 | 2 | thermo_2 0 | 10 | 1 | thermo_1 5 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+----------+---------- 0 | 5 | 2 | thermo_2 | Berlin 0 | 10 | 1 | thermo_1 | Moscow 0 | 20 | 3 | thermo_3 | London -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 5 | 2 5 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+---------- 0 | 5 | 2 | thermo_2 0 | 10 | 1 | thermo_1 5 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+----------+---------- 0 | 5 | 2 | thermo_2 | Berlin 0 | 10 | 1 | thermo_1 | Moscow 0 | 20 | 3 | thermo_3 | London \if :IS_TIME_DIMENSION -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-01 01:00:00-00'::timestamptz, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-03 01:00:00-00'::timestamptz, 2); \else -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES (2, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES (10, 2); \endif -- No changes SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 5 | 2 5 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+---------- 0 | 5 | 2 | thermo_2 0 | 10 | 1 | thermo_1 5 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+----------+---------- 0 | 5 | 2 | thermo_2 | Berlin 0 | 10 | 1 | thermo_1 | Moscow 0 | 20 | 3 | thermo_3 | London -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime changes, just new region SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 5 | 2 5 | 20 | 3 10 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+---------- 0 | 5 | 2 | thermo_2 0 | 10 | 1 | thermo_1 5 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+----------+---------- 0 | 5 | 2 | thermo_2 | Berlin 0 | 10 | 1 | thermo_1 | Moscow 0 | 20 | 3 | thermo_3 | London -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- All changes are materialized SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 2 | 2 | 5 | 2 5 | 20 | 3 10 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+---------- 0 | 5 | 2 | thermo_2 0 | 10 | 1 | thermo_1 5 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+----------+---------- 0 | 5 | 2 | thermo_2 | Berlin 0 | 10 | 1 | thermo_1 | Moscow 0 | 20 | 3 | thermo_3 | London -- TRUNCATE tests TRUNCATE :CAGG_NAME_2TH_LEVEL; -- This full refresh will remove all the data from the 3TH level cagg CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Should return no rows SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+------ SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+------+---------- -- If we have all the data in the bottom levels caggs we can rebuild CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Now we have all the data SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+---------- 0 | 5 | 2 | thermo_2 0 | 10 | 1 | thermo_1 5 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+----------+---------- 0 | 5 | 2 | thermo_2 | Berlin 0 | 10 | 1 | thermo_1 | Moscow 0 | 20 | 3 | thermo_3 | London -- DROP tests \set ON_ERROR_STOP 0 -- should error because it depends of other CAGGs DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:172: ERROR: cannot drop view conditions_summary_1_1 because other objects depend on it DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:173: ERROR: cannot drop view conditions_summary_2_5 because other objects depend on it CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:174: NOTICE: continuous aggregate "conditions_summary_1_1" is already up-to-date CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:175: NOTICE: continuous aggregate "conditions_summary_2_5" is already up-to-date \set ON_ERROR_STOP 1 -- DROP the 3TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:179: NOTICE: drop cascades to table _timescaledb_internal._hyper_8_11_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:182: ERROR: relation "conditions_summary_3_10" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs TRUNCATE :CAGG_NAME_2TH_LEVEL,:CAGG_NAME_1ST_LEVEL; CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 2 | 2 | 5 | 2 5 | 20 | 3 10 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+------ -- DROP the 2TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:196: ERROR: relation "conditions_summary_2_5" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- 1 | 10 | 1 2 | 2 | 2 | 5 | 2 5 | 20 | 3 10 | 2 | -- DROP the first CAGG should work DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:203: NOTICE: drop cascades to table _timescaledb_internal._hyper_6_14_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:206: ERROR: relation "conditions_summary_1_1" does not exist at character 15 \set ON_ERROR_STOP 1 -- -- Validation test for non-multiple bucket sizes -- \set ON_ERROR_STOP 0 \set BUCKET_WIDTH_1ST 'INTEGER \'2\'' \set BUCKET_WIDTH_2TH 'INTEGER \'5\'' \set WARNING_MESSAGE '-- SHOULD ERROR because non-multiple bucket sizes' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+---------+-----------+----------+---------+---------+------------- bucket | integer | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_9.bucket, _materialized_hypertable_9.temperature FROM _timescaledb_internal._materialized_hypertable_9 WHERE _materialized_hypertable_9.bucket < COALESCE(_timescaledb_functions.cagg_watermark(9)::integer, '-2147483648'::integer) UNION ALL SELECT time_bucket(2, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.cagg_watermark(9)::integer, '-2147483648'::integer) GROUP BY (time_bucket(2, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD ERROR because non-multiple bucket sizes CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with incompatible bucket width DETAIL: Time bucket width of "public.conditions_summary_2" [5] should be multiple of the time bucket width of "public.conditions_summary_1" [2]. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- Validation test for equal bucket sizes -- \set ON_ERROR_STOP 0 \set BUCKET_WIDTH_1ST 'INTEGER \'2\'' \set BUCKET_WIDTH_2TH 'INTEGER \'2\'' \set WARNING_MESSAGE 'SHOULD WORK because new bucket should be greater than previous' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+---------+-----------+----------+---------+---------+------------- bucket | integer | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_10.bucket, _materialized_hypertable_10.temperature FROM _timescaledb_internal._materialized_hypertable_10 WHERE _materialized_hypertable_10.bucket < COALESCE(_timescaledb_functions.cagg_watermark(10)::integer, '-2147483648'::integer) UNION ALL SELECT time_bucket(2, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.cagg_watermark(10)::integer, '-2147483648'::integer) GROUP BY (time_bucket(2, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE SHOULD WORK because new bucket should be greater than previous CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+---------+-----------+----------+---------+---------+------------- bucket | integer | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_11.bucket, _materialized_hypertable_11.temperature FROM _timescaledb_internal._materialized_hypertable_11 WHERE _materialized_hypertable_11.bucket < COALESCE(_timescaledb_functions.cagg_watermark(11)::integer, '-2147483648'::integer) UNION ALL SELECT time_bucket(2, conditions_summary_1.bucket) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.cagg_watermark(11)::integer, '-2147483648'::integer) GROUP BY (time_bucket(2, conditions_summary_1.bucket)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- Validation test for bucket size less than source -- \set ON_ERROR_STOP 0 \set BUCKET_WIDTH_1ST 'INTEGER \'4\'' \set BUCKET_WIDTH_2TH 'INTEGER \'2\'' \set WARNING_MESSAGE '-- SHOULD ERROR because new bucket should be greater than previous' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+---------+-----------+----------+---------+---------+------------- bucket | integer | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_12.bucket, _materialized_hypertable_12.temperature FROM _timescaledb_internal._materialized_hypertable_12 WHERE _materialized_hypertable_12.bucket < COALESCE(_timescaledb_functions.cagg_watermark(12)::integer, '-2147483648'::integer) UNION ALL SELECT time_bucket(4, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.cagg_watermark(12)::integer, '-2147483648'::integer) GROUP BY (time_bucket(4, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD ERROR because new bucket should be greater than previous CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with incompatible bucket width DETAIL: Time bucket width of "public.conditions_summary_2" [2] should be greater or equal than the time bucket width of "public.conditions_summary_1" [4]. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- ######################################################## -- ## TIMESTAMP data type tests -- ######################################################## -- Current test variables \set IS_TIME_DIMENSION TRUE \set TIME_DIMENSION_DATATYPE TIMESTAMP \set CAGG_NAME_1ST_LEVEL conditions_summary_1_hourly \set CAGG_NAME_2TH_LEVEL conditions_summary_2_daily \set CAGG_NAME_3TH_LEVEL conditions_summary_3_weekly \set IS_JOIN TRUE SET timezone TO 'UTC'; -- -- Run common tests for TIMESTAMP -- \set BUCKET_WIDTH_1ST 'INTERVAL \'1 hour\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 day\'' \set BUCKET_WIDTH_3TH 'INTERVAL \'1 week\'' -- Different order of time dimension in raw ht \set IS_DEFAULT_COLUMN_ORDER FALSE \ir include/cagg_on_cagg_setup.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGGs on CAGGs tests SET ROLE :ROLE_DEFAULT_PERM_USER; DROP TABLE IF EXISTS conditions CASCADE; \if :IS_DEFAULT_COLUMN_ORDER CREATE TABLE conditions ( time :TIME_DIMENSION_DATATYPE NOT NULL, temperature NUMERIC, device_id INT ); \else CREATE TABLE conditions ( temperature NUMERIC, time :TIME_DIMENSION_DATATYPE NOT NULL, device_id INT ); \endif \if :IS_JOIN DROP TABLE IF EXISTS devices CASCADE; CREATE TABLE devices ( device_id int not null, name text, location text); INSERT INTO devices values (1, 'thermo_1', 'Moscow'), (2, 'thermo_2', 'Berlin'),(3, 'thermo_3', 'London'),(4, 'thermo_4', 'Stockholm'); \endif \if :IS_TIME_DIMENSION SELECT table_name FROM create_hypertable('conditions', 'time'); psql:include/cagg_on_cagg_setup.sql:30: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices table_name ------------ conditions \else SELECT table_name FROM create_hypertable('conditions', 'time', chunk_time_interval => 10); \endif \if :IS_TIME_DIMENSION INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 01:00:00-00', 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-02 01:00:00-00', 20, 3); \else CREATE OR REPLACE FUNCTION integer_now() RETURNS :TIME_DIMENSION_DATATYPE LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now'); INSERT INTO conditions ("time", temperature, device_id) VALUES (1, 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES (2, 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES (5, 20, 3); \endif \ir include/cagg_on_cagg_common.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGG on hypertable (1st level) CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, SUM(temperature) AS temperature, device_id FROM conditions GROUP BY 1, 3 ORDER BY 1, 2, 3 WITH NO DATA; -- CAGG on CAGG (2th level) CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , devices.device_id , devices.name FROM :CAGG_NAME_1ST_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_1ST_LEVEL.device_id GROUP BY 1, 3, 4 ORDER BY 1, 2, 3, 4 \else FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- CAGG on CAGG (3th level) CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , :CAGG_NAME_2TH_LEVEL.device_id , :CAGG_NAME_2TH_LEVEL.name , devices.location FROM :CAGG_NAME_2TH_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_2TH_LEVEL.device_id GROUP BY 1, 3, 4, 5 ORDER BY 1, 2, 3, 4, 5 \else FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- Check chunk_interval \if :IS_TIME_DIMENSION SELECT h.table_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; name | chunk_interval -----------------------------+---------------- conditions | @ 7 days conditions_summary_1_hourly | @ 70 days conditions_summary_2_daily | @ 70 days conditions_summary_3_weekly | @ 70 days \else SELECT h.table_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; \endif -- No data because the CAGGs are just for materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+------ SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+------+---------- -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 | 20 | 3 | thermo_3 | London -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 | 20 | 3 | thermo_3 | London \if :IS_TIME_DIMENSION -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-01 01:00:00-00'::timestamptz, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-03 01:00:00-00'::timestamptz, 2); \else -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES (2, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES (10, 2); \endif -- No changes SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 | 20 | 3 | thermo_3 | London -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime changes, just new region SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 Mon Jan 03 01:00:00 2022 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 | 20 | 3 | thermo_3 | London -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- All changes are materialized SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 2 | Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 Mon Jan 03 01:00:00 2022 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 | 20 | 3 | thermo_3 | London -- TRUNCATE tests TRUNCATE :CAGG_NAME_2TH_LEVEL; -- This full refresh will remove all the data from the 3TH level cagg CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Should return no rows SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+------ SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+------+---------- -- If we have all the data in the bottom levels caggs we can rebuild CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Now we have all the data SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 | 20 | 3 | thermo_3 | London -- DROP tests \set ON_ERROR_STOP 0 -- should error because it depends of other CAGGs DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:172: ERROR: cannot drop view conditions_summary_1_hourly because other objects depend on it DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:173: ERROR: cannot drop view conditions_summary_2_daily because other objects depend on it CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:174: NOTICE: continuous aggregate "conditions_summary_1_hourly" is already up-to-date CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:175: NOTICE: continuous aggregate "conditions_summary_2_daily" is already up-to-date \set ON_ERROR_STOP 1 -- DROP the 3TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:179: NOTICE: drop cascades to table _timescaledb_internal._hyper_16_18_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:182: ERROR: relation "conditions_summary_3_weekly" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs TRUNCATE :CAGG_NAME_2TH_LEVEL,:CAGG_NAME_1ST_LEVEL; CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 2 | Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 Mon Jan 03 01:00:00 2022 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+------ -- DROP the 2TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:196: ERROR: relation "conditions_summary_2_daily" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 2 | Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 Mon Jan 03 01:00:00 2022 | 2 | -- DROP the first CAGG should work DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:203: NOTICE: drop cascades to table _timescaledb_internal._hyper_14_20_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:206: ERROR: relation "conditions_summary_1_hourly" does not exist at character 15 \set ON_ERROR_STOP 1 -- Default tests \set IS_DEFAULT_COLUMN_ORDER TRUE \ir include/cagg_on_cagg_setup.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGGs on CAGGs tests SET ROLE :ROLE_DEFAULT_PERM_USER; DROP TABLE IF EXISTS conditions CASCADE; \if :IS_DEFAULT_COLUMN_ORDER CREATE TABLE conditions ( time :TIME_DIMENSION_DATATYPE NOT NULL, temperature NUMERIC, device_id INT ); \else CREATE TABLE conditions ( temperature NUMERIC, time :TIME_DIMENSION_DATATYPE NOT NULL, device_id INT ); \endif \if :IS_JOIN DROP TABLE IF EXISTS devices CASCADE; CREATE TABLE devices ( device_id int not null, name text, location text); INSERT INTO devices values (1, 'thermo_1', 'Moscow'), (2, 'thermo_2', 'Berlin'),(3, 'thermo_3', 'London'),(4, 'thermo_4', 'Stockholm'); \endif \if :IS_TIME_DIMENSION SELECT table_name FROM create_hypertable('conditions', 'time'); psql:include/cagg_on_cagg_setup.sql:30: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices table_name ------------ conditions \else SELECT table_name FROM create_hypertable('conditions', 'time', chunk_time_interval => 10); \endif \if :IS_TIME_DIMENSION INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 01:00:00-00', 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-02 01:00:00-00', 20, 3); \else CREATE OR REPLACE FUNCTION integer_now() RETURNS :TIME_DIMENSION_DATATYPE LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now'); INSERT INTO conditions ("time", temperature, device_id) VALUES (1, 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES (2, 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES (5, 20, 3); \endif \ir include/cagg_on_cagg_common.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGG on hypertable (1st level) CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, SUM(temperature) AS temperature, device_id FROM conditions GROUP BY 1, 3 ORDER BY 1, 2, 3 WITH NO DATA; -- CAGG on CAGG (2th level) CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , devices.device_id , devices.name FROM :CAGG_NAME_1ST_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_1ST_LEVEL.device_id GROUP BY 1, 3, 4 ORDER BY 1, 2, 3, 4 \else FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- CAGG on CAGG (3th level) CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , :CAGG_NAME_2TH_LEVEL.device_id , :CAGG_NAME_2TH_LEVEL.name , devices.location FROM :CAGG_NAME_2TH_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_2TH_LEVEL.device_id GROUP BY 1, 3, 4, 5 ORDER BY 1, 2, 3, 4, 5 \else FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- Check chunk_interval \if :IS_TIME_DIMENSION SELECT h.table_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; name | chunk_interval -----------------------------+---------------- conditions | @ 7 days conditions_summary_1_hourly | @ 70 days conditions_summary_2_daily | @ 70 days conditions_summary_3_weekly | @ 70 days \else SELECT h.table_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; \endif -- No data because the CAGGs are just for materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+------ SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+------+---------- -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 | 20 | 3 | thermo_3 | London -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 | 20 | 3 | thermo_3 | London \if :IS_TIME_DIMENSION -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-01 01:00:00-00'::timestamptz, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-03 01:00:00-00'::timestamptz, 2); \else -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES (2, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES (10, 2); \endif -- No changes SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 | 20 | 3 | thermo_3 | London -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime changes, just new region SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 Mon Jan 03 01:00:00 2022 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 | 20 | 3 | thermo_3 | London -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- All changes are materialized SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 2 | Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 Mon Jan 03 01:00:00 2022 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 | 20 | 3 | thermo_3 | London -- TRUNCATE tests TRUNCATE :CAGG_NAME_2TH_LEVEL; -- This full refresh will remove all the data from the 3TH level cagg CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Should return no rows SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+------ SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+------+---------- -- If we have all the data in the bottom levels caggs we can rebuild CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Now we have all the data SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 | 20 | 3 | thermo_3 | London -- DROP tests \set ON_ERROR_STOP 0 -- should error because it depends of other CAGGs DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:172: ERROR: cannot drop view conditions_summary_1_hourly because other objects depend on it DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:173: ERROR: cannot drop view conditions_summary_2_daily because other objects depend on it CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:174: NOTICE: continuous aggregate "conditions_summary_1_hourly" is already up-to-date CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:175: NOTICE: continuous aggregate "conditions_summary_2_daily" is already up-to-date \set ON_ERROR_STOP 1 -- DROP the 3TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:179: NOTICE: drop cascades to table _timescaledb_internal._hyper_20_24_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:182: ERROR: relation "conditions_summary_3_weekly" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs TRUNCATE :CAGG_NAME_2TH_LEVEL,:CAGG_NAME_1ST_LEVEL; CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 2 | Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 Mon Jan 03 01:00:00 2022 | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+------ -- DROP the 2TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:196: ERROR: relation "conditions_summary_2_daily" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 | 10 | 1 Sat Jan 01 01:00:00 2022 | 2 | Sat Jan 01 01:00:00 2022 | 5 | 2 Sun Jan 02 01:00:00 2022 | 20 | 3 Mon Jan 03 01:00:00 2022 | 2 | -- DROP the first CAGG should work DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:203: NOTICE: drop cascades to table _timescaledb_internal._hyper_18_26_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:206: ERROR: relation "conditions_summary_1_hourly" does not exist at character 15 \set ON_ERROR_STOP 1 -- -- Validation test for variable bucket on top of fixed bucket -- \set ON_ERROR_STOP 0 \set BUCKET_WIDTH_1ST 'INTERVAL \'1 month\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'60 days\'' \set WARNING_MESSAGE '-- SHOULD ERROR because is not allowed variable-size bucket on top of fixed-size bucket' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+-----------------------------+-----------+----------+---------+---------+------------- bucket | timestamp without time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_21.bucket, _materialized_hypertable_21.temperature FROM _timescaledb_internal._materialized_hypertable_21 WHERE _materialized_hypertable_21.bucket < COALESCE(_timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(21)), '-infinity'::timestamp without time zone) UNION ALL SELECT time_bucket('@ 1 mon'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(21)), '-infinity'::timestamp without time zone) GROUP BY (time_bucket('@ 1 mon'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD ERROR because is not allowed variable-size bucket on top of fixed-size bucket CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with fixed-width bucket on top of one using variable-width bucket DETAIL: Continuous aggregate with a fixed time bucket width (e.g. 61 days) cannot be created on top of one using variable time bucket width (e.g. 1 month). The variance can lead to the fixed width one not being a multiple of the variable width one. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- Validation test for non-multiple bucket sizes -- \set ON_ERROR_STOP 0 \set BUCKET_WIDTH_1ST 'INTERVAL \'2 hours\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'3 hours\'' \set WARNING_MESSAGE '-- SHOULD ERROR because non-multiple bucket sizes' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+-----------------------------+-----------+----------+---------+---------+------------- bucket | timestamp without time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_22.bucket, _materialized_hypertable_22.temperature FROM _timescaledb_internal._materialized_hypertable_22 WHERE _materialized_hypertable_22.bucket < COALESCE(_timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(22)), '-infinity'::timestamp without time zone) UNION ALL SELECT time_bucket('@ 2 hours'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(22)), '-infinity'::timestamp without time zone) GROUP BY (time_bucket('@ 2 hours'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD ERROR because non-multiple bucket sizes CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with incompatible bucket width DETAIL: Time bucket width of "public.conditions_summary_2" [@ 3 hours] should be multiple of the time bucket width of "public.conditions_summary_1" [@ 2 hours]. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- Validation test for equal bucket sizes -- \set ON_ERROR_STOP 0 \set BUCKET_WIDTH_1ST 'INTERVAL \'1 hour\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 hour\'' \set WARNING_MESSAGE 'SHOULD WORK because new bucket should be greater than previous' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+-----------------------------+-----------+----------+---------+---------+------------- bucket | timestamp without time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_23.bucket, _materialized_hypertable_23.temperature FROM _timescaledb_internal._materialized_hypertable_23 WHERE _materialized_hypertable_23.bucket < COALESCE(_timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(23)), '-infinity'::timestamp without time zone) UNION ALL SELECT time_bucket('@ 1 hour'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(23)), '-infinity'::timestamp without time zone) GROUP BY (time_bucket('@ 1 hour'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE SHOULD WORK because new bucket should be greater than previous CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+-----------------------------+-----------+----------+---------+---------+------------- bucket | timestamp without time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_24.bucket, _materialized_hypertable_24.temperature FROM _timescaledb_internal._materialized_hypertable_24 WHERE _materialized_hypertable_24.bucket < COALESCE(_timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(24)), '-infinity'::timestamp without time zone) UNION ALL SELECT time_bucket('@ 1 hour'::interval, conditions_summary_1.bucket) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(24)), '-infinity'::timestamp without time zone) GROUP BY (time_bucket('@ 1 hour'::interval, conditions_summary_1.bucket)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- Validation test for bucket size less than source -- \set ON_ERROR_STOP 0 \set BUCKET_WIDTH_1ST 'INTERVAL \'2 hours\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 hour\'' \set WARNING_MESSAGE '-- SHOULD ERROR because new bucket should be greater than previous' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+-----------------------------+-----------+----------+---------+---------+------------- bucket | timestamp without time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_25.bucket, _materialized_hypertable_25.temperature FROM _timescaledb_internal._materialized_hypertable_25 WHERE _materialized_hypertable_25.bucket < COALESCE(_timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(25)), '-infinity'::timestamp without time zone) UNION ALL SELECT time_bucket('@ 2 hours'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp_without_timezone(_timescaledb_functions.cagg_watermark(25)), '-infinity'::timestamp without time zone) GROUP BY (time_bucket('@ 2 hours'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD ERROR because new bucket should be greater than previous CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with incompatible bucket width DETAIL: Time bucket width of "public.conditions_summary_2" [@ 1 hour] should be greater or equal than the time bucket width of "public.conditions_summary_1" [@ 2 hours]. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- ######################################################## -- ## TIMESTAMPTZ data type tests -- ######################################################## -- Current test variables \set IS_TIME_DIMENSION TRUE \set TIME_DIMENSION_DATATYPE TIMESTAMPTZ \set CAGG_NAME_1ST_LEVEL conditions_summary_1_hourly \set CAGG_NAME_2TH_LEVEL conditions_summary_2_daily \set CAGG_NAME_3TH_LEVEL conditions_summary_3_weekly SET timezone TO 'UTC'; -- -- Run common tests for TIMESTAMPTZ -- \set BUCKET_WIDTH_1ST 'INTERVAL \'1 hour\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 day\'' \set BUCKET_WIDTH_3TH 'INTERVAL \'1 week\'' -- Different order of time dimension in raw ht \set ON_ERROR_STOP 0 \set IS_DEFAULT_COLUMN_ORDER FALSE \ir include/cagg_on_cagg_setup.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGGs on CAGGs tests SET ROLE :ROLE_DEFAULT_PERM_USER; DROP TABLE IF EXISTS conditions CASCADE; \if :IS_DEFAULT_COLUMN_ORDER CREATE TABLE conditions ( time :TIME_DIMENSION_DATATYPE NOT NULL, temperature NUMERIC, device_id INT ); \else CREATE TABLE conditions ( temperature NUMERIC, time :TIME_DIMENSION_DATATYPE NOT NULL, device_id INT ); \endif \if :IS_JOIN DROP TABLE IF EXISTS devices CASCADE; CREATE TABLE devices ( device_id int not null, name text, location text); INSERT INTO devices values (1, 'thermo_1', 'Moscow'), (2, 'thermo_2', 'Berlin'),(3, 'thermo_3', 'London'),(4, 'thermo_4', 'Stockholm'); \endif \if :IS_TIME_DIMENSION SELECT table_name FROM create_hypertable('conditions', 'time'); table_name ------------ conditions \else SELECT table_name FROM create_hypertable('conditions', 'time', chunk_time_interval => 10); \endif \if :IS_TIME_DIMENSION INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 01:00:00-00', 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-02 01:00:00-00', 20, 3); \else CREATE OR REPLACE FUNCTION integer_now() RETURNS :TIME_DIMENSION_DATATYPE LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now'); INSERT INTO conditions ("time", temperature, device_id) VALUES (1, 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES (2, 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES (5, 20, 3); \endif \ir include/cagg_on_cagg_common.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGG on hypertable (1st level) CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, SUM(temperature) AS temperature, device_id FROM conditions GROUP BY 1, 3 ORDER BY 1, 2, 3 WITH NO DATA; -- CAGG on CAGG (2th level) CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , devices.device_id , devices.name FROM :CAGG_NAME_1ST_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_1ST_LEVEL.device_id GROUP BY 1, 3, 4 ORDER BY 1, 2, 3, 4 \else FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- CAGG on CAGG (3th level) CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , :CAGG_NAME_2TH_LEVEL.device_id , :CAGG_NAME_2TH_LEVEL.name , devices.location FROM :CAGG_NAME_2TH_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_2TH_LEVEL.device_id GROUP BY 1, 3, 4, 5 ORDER BY 1, 2, 3, 4, 5 \else FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- Check chunk_interval \if :IS_TIME_DIMENSION SELECT h.table_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; name | chunk_interval -----------------------------+---------------- conditions | @ 7 days conditions_summary_1_hourly | @ 70 days conditions_summary_2_daily | @ 70 days conditions_summary_3_weekly | @ 70 days \else SELECT h.table_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; \endif -- No data because the CAGGs are just for materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+------ SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+------+---------- -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name ------------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 UTC | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 UTC | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 UTC | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location ------------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 UTC | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 UTC | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 UTC | 20 | 3 | thermo_3 | London -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name ------------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 UTC | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 UTC | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 UTC | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location ------------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 UTC | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 UTC | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 UTC | 20 | 3 | thermo_3 | London \if :IS_TIME_DIMENSION -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-01 01:00:00-00'::timestamptz, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-03 01:00:00-00'::timestamptz, 2); \else -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES (2, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES (10, 2); \endif -- No changes SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name ------------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 UTC | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 UTC | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 UTC | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location ------------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 UTC | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 UTC | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 UTC | 20 | 3 | thermo_3 | London -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime changes, just new region SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 Mon Jan 03 01:00:00 2022 UTC | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name ------------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 UTC | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 UTC | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 UTC | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location ------------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 UTC | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 UTC | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 UTC | 20 | 3 | thermo_3 | London -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- All changes are materialized SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 2 | Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 Mon Jan 03 01:00:00 2022 UTC | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name ------------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 UTC | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 UTC | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 UTC | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location ------------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 UTC | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 UTC | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 UTC | 20 | 3 | thermo_3 | London -- TRUNCATE tests TRUNCATE :CAGG_NAME_2TH_LEVEL; -- This full refresh will remove all the data from the 3TH level cagg CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Should return no rows SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+------ SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+------+---------- -- If we have all the data in the bottom levels caggs we can rebuild CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Now we have all the data SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name ------------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 UTC | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 UTC | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 UTC | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location ------------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 UTC | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 UTC | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 UTC | 20 | 3 | thermo_3 | London -- DROP tests \set ON_ERROR_STOP 0 -- should error because it depends of other CAGGs DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:172: ERROR: cannot drop view conditions_summary_1_hourly because other objects depend on it DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:173: ERROR: cannot drop view conditions_summary_2_daily because other objects depend on it CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:174: NOTICE: continuous aggregate "conditions_summary_1_hourly" is already up-to-date CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:175: NOTICE: continuous aggregate "conditions_summary_2_daily" is already up-to-date \set ON_ERROR_STOP 1 -- DROP the 3TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:179: NOTICE: drop cascades to table _timescaledb_internal._hyper_29_30_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:182: ERROR: relation "conditions_summary_3_weekly" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs TRUNCATE :CAGG_NAME_2TH_LEVEL,:CAGG_NAME_1ST_LEVEL; CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 2 | Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 Mon Jan 03 01:00:00 2022 UTC | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+------ -- DROP the 2TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:196: ERROR: relation "conditions_summary_2_daily" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 2 | Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 Mon Jan 03 01:00:00 2022 UTC | 2 | -- DROP the first CAGG should work DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:203: NOTICE: drop cascades to table _timescaledb_internal._hyper_27_32_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:206: ERROR: relation "conditions_summary_1_hourly" does not exist at character 15 \set ON_ERROR_STOP 1 -- Default tests \set ON_ERROR_STOP 0 \set IS_DEFAULT_COLUMN_ORDER TRUE \ir include/cagg_on_cagg_setup.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGGs on CAGGs tests SET ROLE :ROLE_DEFAULT_PERM_USER; DROP TABLE IF EXISTS conditions CASCADE; \if :IS_DEFAULT_COLUMN_ORDER CREATE TABLE conditions ( time :TIME_DIMENSION_DATATYPE NOT NULL, temperature NUMERIC, device_id INT ); \else CREATE TABLE conditions ( temperature NUMERIC, time :TIME_DIMENSION_DATATYPE NOT NULL, device_id INT ); \endif \if :IS_JOIN DROP TABLE IF EXISTS devices CASCADE; CREATE TABLE devices ( device_id int not null, name text, location text); INSERT INTO devices values (1, 'thermo_1', 'Moscow'), (2, 'thermo_2', 'Berlin'),(3, 'thermo_3', 'London'),(4, 'thermo_4', 'Stockholm'); \endif \if :IS_TIME_DIMENSION SELECT table_name FROM create_hypertable('conditions', 'time'); table_name ------------ conditions \else SELECT table_name FROM create_hypertable('conditions', 'time', chunk_time_interval => 10); \endif \if :IS_TIME_DIMENSION INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 01:00:00-00', 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-02 01:00:00-00', 20, 3); \else CREATE OR REPLACE FUNCTION integer_now() RETURNS :TIME_DIMENSION_DATATYPE LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now'); INSERT INTO conditions ("time", temperature, device_id) VALUES (1, 10, 1); INSERT INTO conditions ("time", temperature, device_id) VALUES (2, 5, 2); INSERT INTO conditions ("time", temperature, device_id) VALUES (5, 20, 3); \endif \ir include/cagg_on_cagg_common.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- CAGG on hypertable (1st level) CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, SUM(temperature) AS temperature, device_id FROM conditions GROUP BY 1, 3 ORDER BY 1, 2, 3 WITH NO DATA; -- CAGG on CAGG (2th level) CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , devices.device_id , devices.name FROM :CAGG_NAME_1ST_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_1ST_LEVEL.device_id GROUP BY 1, 3, 4 ORDER BY 1, 2, 3, 4 \else FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- CAGG on CAGG (3th level) CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket, SUM(temperature) AS temperature \if :IS_JOIN , :CAGG_NAME_2TH_LEVEL.device_id , :CAGG_NAME_2TH_LEVEL.name , devices.location FROM :CAGG_NAME_2TH_LEVEL JOIN devices ON devices.device_id = :CAGG_NAME_2TH_LEVEL.device_id GROUP BY 1, 3, 4, 5 ORDER BY 1, 2, 3, 4, 5 \else FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 ORDER BY 1, 2 \endif WITH NO DATA; -- Check chunk_interval \if :IS_TIME_DIMENSION SELECT h.table_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, _timescaledb_functions.to_interval(d.interval_length) AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; name | chunk_interval -----------------------------+---------------- conditions | @ 7 days conditions_summary_1_hourly | @ 70 days conditions_summary_2_daily | @ 70 days conditions_summary_3_weekly | @ 70 days \else SELECT h.table_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.hypertable h LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = h.id WHERE h.table_name = 'conditions' UNION ALL SELECT c.user_view_name AS name, d.interval_length AS chunk_interval FROM _timescaledb_catalog.continuous_agg c LEFT JOIN _timescaledb_catalog.dimension d on d.hypertable_id = c.mat_hypertable_id WHERE c.user_view_name IN (:'CAGG_NAME_1ST_LEVEL', :'CAGG_NAME_2TH_LEVEL', :'CAGG_NAME_3TH_LEVEL') ORDER BY 1, 2; \endif -- No data because the CAGGs are just for materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id --------+-------------+----------- SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+------ SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+------+---------- -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name ------------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 UTC | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 UTC | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 UTC | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location ------------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 UTC | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 UTC | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 UTC | 20 | 3 | thermo_3 | London -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Materialized data SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name ------------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 UTC | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 UTC | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 UTC | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location ------------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 UTC | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 UTC | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 UTC | 20 | 3 | thermo_3 | London \if :IS_TIME_DIMENSION -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-01 01:00:00-00'::timestamptz, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES ('2022-01-03 01:00:00-00'::timestamptz, 2); \else -- Invalidate an old region INSERT INTO conditions ("time", temperature) VALUES (2, 2); -- New region INSERT INTO conditions ("time", temperature) VALUES (10, 2); \endif -- No changes SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name ------------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 UTC | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 UTC | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 UTC | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location ------------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 UTC | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 UTC | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 UTC | 20 | 3 | thermo_3 | London -- Turn CAGGs into Realtime ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=false); -- Realtime changes, just new region SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 Mon Jan 03 01:00:00 2022 UTC | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name ------------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 UTC | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 UTC | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 UTC | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location ------------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 UTC | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 UTC | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 UTC | 20 | 3 | thermo_3 | London -- Turn CAGGs into materialized only again ALTER MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL SET (timescaledb.materialized_only=true); -- Refresh all data CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- All changes are materialized SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 2 | Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 Mon Jan 03 01:00:00 2022 UTC | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name ------------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 UTC | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 UTC | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 UTC | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location ------------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 UTC | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 UTC | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 UTC | 20 | 3 | thermo_3 | London -- TRUNCATE tests TRUNCATE :CAGG_NAME_2TH_LEVEL; -- This full refresh will remove all the data from the 3TH level cagg CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Should return no rows SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+------ SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location --------+-------------+-----------+------+---------- -- If we have all the data in the bottom levels caggs we can rebuild CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_3TH_LEVEL', NULL, NULL); -- Now we have all the data SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name ------------------------------+-------------+-----------+---------- Sat Jan 01 00:00:00 2022 UTC | 5 | 2 | thermo_2 Sat Jan 01 00:00:00 2022 UTC | 10 | 1 | thermo_1 Sun Jan 02 00:00:00 2022 UTC | 20 | 3 | thermo_3 SELECT * FROM :CAGG_NAME_3TH_LEVEL; bucket | temperature | device_id | name | location ------------------------------+-------------+-----------+----------+---------- Mon Dec 27 00:00:00 2021 UTC | 5 | 2 | thermo_2 | Berlin Mon Dec 27 00:00:00 2021 UTC | 10 | 1 | thermo_1 | Moscow Mon Dec 27 00:00:00 2021 UTC | 20 | 3 | thermo_3 | London -- DROP tests \set ON_ERROR_STOP 0 -- should error because it depends of other CAGGs DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:172: ERROR: cannot drop view conditions_summary_1_hourly because other objects depend on it DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:173: ERROR: cannot drop view conditions_summary_2_daily because other objects depend on it CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:174: NOTICE: continuous aggregate "conditions_summary_1_hourly" is already up-to-date CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); psql:include/cagg_on_cagg_common.sql:175: NOTICE: continuous aggregate "conditions_summary_2_daily" is already up-to-date \set ON_ERROR_STOP 1 -- DROP the 3TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:179: NOTICE: drop cascades to table _timescaledb_internal._hyper_33_36_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_3TH_LEVEL; psql:include/cagg_on_cagg_common.sql:182: ERROR: relation "conditions_summary_3_weekly" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs TRUNCATE :CAGG_NAME_2TH_LEVEL,:CAGG_NAME_1ST_LEVEL; CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 2 | Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 Mon Jan 03 01:00:00 2022 UTC | 2 | SELECT * FROM :CAGG_NAME_2TH_LEVEL; bucket | temperature | device_id | name --------+-------------+-----------+------ -- DROP the 2TH level CAGG don't affect others DROP MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL; \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_common.sql:196: ERROR: relation "conditions_summary_2_daily" does not exist at character 15 \set ON_ERROR_STOP 1 -- should work because dropping the top level CAGG -- don't affect the down level CAGGs SELECT * FROM :CAGG_NAME_1ST_LEVEL; bucket | temperature | device_id ------------------------------+-------------+----------- Sat Jan 01 00:00:00 2022 UTC | 10 | 1 Sat Jan 01 01:00:00 2022 UTC | 2 | Sat Jan 01 01:00:00 2022 UTC | 5 | 2 Sun Jan 02 01:00:00 2022 UTC | 20 | 3 Mon Jan 03 01:00:00 2022 UTC | 2 | -- DROP the first CAGG should work DROP MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:203: NOTICE: drop cascades to table _timescaledb_internal._hyper_31_38_chunk \set ON_ERROR_STOP 0 -- should error because it was dropped SELECT * FROM :CAGG_NAME_1ST_LEVEL; psql:include/cagg_on_cagg_common.sql:206: ERROR: relation "conditions_summary_1_hourly" does not exist at character 15 \set ON_ERROR_STOP 1 -- -- Validation test for variable bucket on top of fixed bucket -- \set ON_ERROR_STOP 0 \set BUCKET_WIDTH_1ST 'INTERVAL \'1 month\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'60 days\'' \set WARNING_MESSAGE '-- SHOULD ERROR because is not allowed variable-size bucket on top of fixed-size bucket' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_34.bucket, _materialized_hypertable_34.temperature FROM _timescaledb_internal._materialized_hypertable_34 WHERE _materialized_hypertable_34.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(34)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 mon'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(34)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 mon'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD ERROR because is not allowed variable-size bucket on top of fixed-size bucket CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with fixed-width bucket on top of one using variable-width bucket DETAIL: Continuous aggregate with a fixed time bucket width (e.g. 61 days) cannot be created on top of one using variable time bucket width (e.g. 1 month). The variance can lead to the fixed width one not being a multiple of the variable width one. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- Validation test for non-multiple bucket sizes -- \set ON_ERROR_STOP 0 \set BUCKET_WIDTH_1ST 'INTERVAL \'2 hours\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'3 hours\'' \set WARNING_MESSAGE '-- SHOULD ERROR because non-multiple bucket sizes' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_35.bucket, _materialized_hypertable_35.temperature FROM _timescaledb_internal._materialized_hypertable_35 WHERE _materialized_hypertable_35.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(35)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 2 hours'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(35)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 2 hours'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD ERROR because non-multiple bucket sizes CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with incompatible bucket width DETAIL: Time bucket width of "public.conditions_summary_2" [@ 3 hours] should be multiple of the time bucket width of "public.conditions_summary_1" [@ 2 hours]. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- Validation test for equal bucket sizes -- \set ON_ERROR_STOP 0 \set BUCKET_WIDTH_1ST 'INTERVAL \'1 hour\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 hour\'' \set WARNING_MESSAGE 'SHOULD WORK because new bucket should be greater than previous' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_36.bucket, _materialized_hypertable_36.temperature FROM _timescaledb_internal._materialized_hypertable_36 WHERE _materialized_hypertable_36.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(36)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 hour'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(36)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 hour'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE SHOULD WORK because new bucket should be greater than previous CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_37.bucket, _materialized_hypertable_37.temperature FROM _timescaledb_internal._materialized_hypertable_37 WHERE _materialized_hypertable_37.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(37)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 hour'::interval, conditions_summary_1.bucket) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(37)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 hour'::interval, conditions_summary_1.bucket)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- Validation test for bucket size less than source -- \set ON_ERROR_STOP 0 \set BUCKET_WIDTH_1ST 'INTERVAL \'2 hours\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 hour\'' \set WARNING_MESSAGE '-- SHOULD ERROR because new bucket should be greater than previous' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_38.bucket, _materialized_hypertable_38.temperature FROM _timescaledb_internal._materialized_hypertable_38 WHERE _materialized_hypertable_38.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(38)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 2 hours'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(38)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 2 hours'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD ERROR because new bucket should be greater than previous CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with incompatible bucket width DETAIL: Time bucket width of "public.conditions_summary_2" [@ 1 hour] should be greater or equal than the time bucket width of "public.conditions_summary_1" [@ 2 hours]. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- Validations using time bucket with timezone (ref issue #5126) -- \set ON_ERROR_STOP 0 \set TIME_DIMENSION_DATATYPE TIMESTAMPTZ \set IS_TIME_DIMENSION_WITH_TIMEZONE_1ST TRUE \set IS_TIME_DIMENSION_WITH_TIMEZONE_2TH TRUE \set CAGG_NAME_1ST_LEVEL conditions_summary_1_5m \set CAGG_NAME_2TH_LEVEL conditions_summary_2_1h \set BUCKET_TZNAME_1ST 'US/Pacific' \set BUCKET_TZNAME_2TH 'US/Pacific' \set BUCKET_WIDTH_1ST 'INTERVAL \'5 minutes\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 hour\'' \set WARNING_MESSAGE '-- SHOULD WORK' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_39.bucket, _materialized_hypertable_39.temperature FROM _timescaledb_internal._materialized_hypertable_39 WHERE _materialized_hypertable_39.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(39)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 5 mins'::interval, conditions."time", 'US/Pacific'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(39)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 5 mins'::interval, conditions."time", 'US/Pacific'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_40.bucket, _materialized_hypertable_40.temperature FROM _timescaledb_internal._materialized_hypertable_40 WHERE _materialized_hypertable_40.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(40)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 hour'::interval, conditions_summary_1.bucket, 'US/Pacific'::text) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(40)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 hour'::interval, conditions_summary_1.bucket, 'US/Pacific'::text)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; \set BUCKET_WIDTH_1ST 'INTERVAL \'5 minutes\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'16 minutes\'' \set WARNING_MESSAGE '-- SHOULD ERROR because non-multiple bucket sizes' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_41.bucket, _materialized_hypertable_41.temperature FROM _timescaledb_internal._materialized_hypertable_41 WHERE _materialized_hypertable_41.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(41)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 5 mins'::interval, conditions."time", 'US/Pacific'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(41)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 5 mins'::interval, conditions."time", 'US/Pacific'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD ERROR because non-multiple bucket sizes CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with incompatible bucket width DETAIL: Time bucket width of "public.conditions_summary_2" [@ 16 mins] should be multiple of the time bucket width of "public.conditions_summary_1" [@ 5 mins]. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- Variable bucket size with the same timezones -- \set BUCKET_TZNAME_1ST 'UTC' \set BUCKET_TZNAME_2TH 'UTC' \set BUCKET_WIDTH_1ST 'INTERVAL \'1 day\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 month\'' \set WARNING_MESSAGE '-- SHOULD WORK' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_42.bucket, _materialized_hypertable_42.temperature FROM _timescaledb_internal._materialized_hypertable_42 WHERE _materialized_hypertable_42.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(42)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions."time", 'UTC'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(42)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions."time", 'UTC'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_43.bucket, _materialized_hypertable_43.temperature FROM _timescaledb_internal._materialized_hypertable_43 WHERE _materialized_hypertable_43.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(43)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 mon'::interval, conditions_summary_1.bucket, 'UTC'::text) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(43)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 mon'::interval, conditions_summary_1.bucket, 'UTC'::text)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- Variable bucket size with different timezones -- \set BUCKET_TZNAME_1ST 'US/Pacific' \set BUCKET_TZNAME_2TH 'UTC' \set BUCKET_WIDTH_1ST 'INTERVAL \'1 day\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 month\'' \set WARNING_MESSAGE '-- SHOULD WORK' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_44.bucket, _materialized_hypertable_44.temperature FROM _timescaledb_internal._materialized_hypertable_44 WHERE _materialized_hypertable_44.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(44)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions."time", 'US/Pacific'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(44)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions."time", 'US/Pacific'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_45.bucket, _materialized_hypertable_45.temperature FROM _timescaledb_internal._materialized_hypertable_45 WHERE _materialized_hypertable_45.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(45)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 mon'::interval, conditions_summary_1.bucket, 'UTC'::text) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(45)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 mon'::interval, conditions_summary_1.bucket, 'UTC'::text)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- TZ bucket on top of non-TZ bucket -- \set IS_TIME_DIMENSION_WITH_TIMEZONE_1ST FALSE \set IS_TIME_DIMENSION_WITH_TIMEZONE_2TH TRUE \set BUCKET_TZNAME_2TH 'UTC' \set BUCKET_WIDTH_1ST 'INTERVAL \'1 day\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 month\'' \set WARNING_MESSAGE '-- SHOULD WORK' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_46.bucket, _materialized_hypertable_46.temperature FROM _timescaledb_internal._materialized_hypertable_46 WHERE _materialized_hypertable_46.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(46)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions."time") AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(46)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions."time")); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_47.bucket, _materialized_hypertable_47.temperature FROM _timescaledb_internal._materialized_hypertable_47 WHERE _materialized_hypertable_47.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(47)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 mon'::interval, conditions_summary_1.bucket, 'UTC'::text) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(47)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 mon'::interval, conditions_summary_1.bucket, 'UTC'::text)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- -- non-TZ bucket on top of TZ bucket -- \set IS_TIME_DIMENSION_WITH_TIMEZONE_1ST TRUE \set IS_TIME_DIMENSION_WITH_TIMEZONE_2TH FALSE \set BUCKET_TZNAME_1ST 'UTC' \set BUCKET_WIDTH_1ST 'INTERVAL \'1 day\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 month\'' \set WARNING_MESSAGE '-- SHOULD WORK' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_48.bucket, _materialized_hypertable_48.temperature FROM _timescaledb_internal._materialized_hypertable_48 WHERE _materialized_hypertable_48.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(48)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions."time", 'UTC'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(48)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions."time", 'UTC'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_49.bucket, _materialized_hypertable_49.temperature FROM _timescaledb_internal._materialized_hypertable_49 WHERE _materialized_hypertable_49.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(49)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 mon'::interval, conditions_summary_1.bucket) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(49)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 mon'::interval, conditions_summary_1.bucket)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; -- test some intuitive intervals that should work but -- were not working due to unix epochs -- validation test for 1 year on top of one day -- validation test for 1 year on top of 1 month -- validation test for 1 year on top of 1 week -- bug report 5231 \set BUCKET_WIDTH_1ST 'INTERVAL \'1 day\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 year\'' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_50.bucket, _materialized_hypertable_50.temperature FROM _timescaledb_internal._materialized_hypertable_50 WHERE _materialized_hypertable_50.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(50)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions."time", 'UTC'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(50)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions."time", 'UTC'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_51.bucket, _materialized_hypertable_51.temperature FROM _timescaledb_internal._materialized_hypertable_51 WHERE _materialized_hypertable_51.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(51)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 year'::interval, conditions_summary_1.bucket) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(51)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 year'::interval, conditions_summary_1.bucket)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; \set BUCKET_WIDTH_1ST 'INTERVAL \'1 day\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'3 month\'' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_52.bucket, _materialized_hypertable_52.temperature FROM _timescaledb_internal._materialized_hypertable_52 WHERE _materialized_hypertable_52.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(52)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 day'::interval, conditions."time", 'UTC'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(52)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 day'::interval, conditions."time", 'UTC'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_53.bucket, _materialized_hypertable_53.temperature FROM _timescaledb_internal._materialized_hypertable_53 WHERE _materialized_hypertable_53.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(53)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 3 mons'::interval, conditions_summary_1.bucket) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(53)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 3 mons'::interval, conditions_summary_1.bucket)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; \set BUCKET_WIDTH_1ST 'INTERVAL \'1 month\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 year\'' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_54.bucket, _materialized_hypertable_54.temperature FROM _timescaledb_internal._materialized_hypertable_54 WHERE _materialized_hypertable_54.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(54)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 mon'::interval, conditions."time", 'UTC'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(54)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 mon'::interval, conditions."time", 'UTC'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_2TH_LEVEL View "public.conditions_summary_2" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_55.bucket, _materialized_hypertable_55.temperature FROM _timescaledb_internal._materialized_hypertable_55 WHERE _materialized_hypertable_55.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(55)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 1 year'::interval, conditions_summary_1.bucket) AS bucket, sum(conditions_summary_1.temperature) AS temperature FROM conditions_summary_1 WHERE conditions_summary_1.bucket >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(55)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 1 year'::interval, conditions_summary_1.bucket)); \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; \set BUCKET_WIDTH_1ST 'INTERVAL \'1 week\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 year\'' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_56.bucket, _materialized_hypertable_56.temperature FROM _timescaledb_internal._materialized_hypertable_56 WHERE _materialized_hypertable_56.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(56)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 7 days'::interval, conditions."time", 'UTC'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(56)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 7 days'::interval, conditions."time", 'UTC'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with incompatible bucket width DETAIL: Time bucket width of "public.conditions_summary_2" [@ 1 year] should be multiple of the time bucket width of "public.conditions_summary_1" [@ 7 days]. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; \set BUCKET_WIDTH_1ST 'INTERVAL \'1 week\'' \set BUCKET_WIDTH_2TH 'INTERVAL \'1 month\'' \ir include/cagg_on_cagg_validations.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \set CAGG_NAME_1ST_LEVEL conditions_summary_1 \set CAGG_NAME_2TH_LEVEL conditions_summary_2 \set CAGG_NAME_3TH_LEVEL conditions_summary_3 -- -- CAGG on hypertable (1st level) -- CREATE MATERIALIZED VIEW :CAGG_NAME_1ST_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_1ST time_bucket(:BUCKET_WIDTH_1ST, "time", :'BUCKET_TZNAME_1ST') AS bucket, \else time_bucket(:BUCKET_WIDTH_1ST, "time") AS bucket, \endif SUM(temperature) AS temperature FROM conditions GROUP BY 1 WITH NO DATA; \d+ :CAGG_NAME_1ST_LEVEL View "public.conditions_summary_1" Column | Type | Collation | Nullable | Default | Storage | Description -------------+--------------------------+-----------+----------+---------+---------+------------- bucket | timestamp with time zone | | | | plain | temperature | numeric | | | | main | View definition: SELECT _materialized_hypertable_57.bucket, _materialized_hypertable_57.temperature FROM _timescaledb_internal._materialized_hypertable_57 WHERE _materialized_hypertable_57.bucket < COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(57)), '-infinity'::timestamp with time zone) UNION ALL SELECT time_bucket('@ 7 days'::interval, conditions."time", 'UTC'::text) AS bucket, sum(conditions.temperature) AS temperature FROM conditions WHERE conditions."time" >= COALESCE(_timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(57)), '-infinity'::timestamp with time zone) GROUP BY (time_bucket('@ 7 days'::interval, conditions."time", 'UTC'::text)); -- -- CAGG on CAGG (2th level) -- \set VERBOSITY default \set ON_ERROR_STOP 0 \echo :WARNING_MESSAGE -- SHOULD WORK CREATE MATERIALIZED VIEW :CAGG_NAME_2TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_2TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket, \else time_bucket(:BUCKET_WIDTH_2TH, "bucket") AS bucket, \endif SUM(temperature) AS temperature FROM :CAGG_NAME_1ST_LEVEL GROUP BY 1 WITH NO DATA; psql:include/cagg_on_cagg_validations.sql:44: ERROR: cannot create continuous aggregate with incompatible bucket width DETAIL: Time bucket width of "public.conditions_summary_2" [@ 1 mon] should be multiple of the time bucket width of "public.conditions_summary_1" [@ 7 days]. \d+ :CAGG_NAME_2TH_LEVEL \set ON_ERROR_STOP 1 \set VERBOSITY terse -- Check for incorrect CAGGs \if :INTERVAL_TEST INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-01 00:00:00-00', 10, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-15 01:00:00-00', 20, 4); INSERT INTO conditions ("time", temperature, device_id) VALUES ('2022-01-31 01:00:00-00', 30, 4); CALL refresh_continuous_aggregate(:'CAGG_NAME_1ST_LEVEL', NULL, NULL); CALL refresh_continuous_aggregate(:'CAGG_NAME_2TH_LEVEL', NULL, NULL); CREATE MATERIALIZED VIEW :CAGG_NAME_3TH_LEVEL WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT \if :IS_TIME_DIMENSION_WITH_TIMEZONE_2TH time_bucket(:BUCKET_WIDTH_3TH, "bucket", :'BUCKET_TZNAME_2TH') AS bucket \else time_bucket(:BUCKET_WIDTH_3TH, "bucket") AS bucket \endif FROM :CAGG_NAME_2TH_LEVEL GROUP BY 1 WITH DATA; \d+ :CAGG_NAME_3TH_LEVEL --There should never be dulpicates in the output of the following query SELECT * from :CAGG_NAME_3TH_LEVEL; DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_3TH_LEVEL; DELETE FROM conditions WHERE device_id = 4; \endif -- -- Cleanup -- DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_2TH_LEVEL; psql:include/cagg_on_cagg_validations.sql:86: NOTICE: materialized view "conditions_summary_2" does not exist, skipping DROP MATERIALIZED VIEW IF EXISTS :CAGG_NAME_1ST_LEVEL; ================================================ FILE: tsl/test/expected/cagg_permissions-15.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- initialize the bgw mock state to prevent the materialization workers from running \c :TEST_DBNAME :ROLE_SUPERUSER -- remove any default jobs, e.g., telemetry so bgw_job isn't polluted DELETE FROM _timescaledb_catalog.bgw_job WHERE TRUE; CREATE VIEW cagg_info AS WITH caggs AS ( SELECT format('%I.%I', user_view_schema, user_view_name)::regclass AS user_view, format('%I.%I', direct_view_schema, direct_view_name)::regclass AS direct_view, format('%I.%I', partial_view_schema, partial_view_name)::regclass AS partial_view, format('%I.%I', ht.schema_name, ht.table_name)::regclass AS mat_relid FROM _timescaledb_catalog.hypertable ht, _timescaledb_catalog.continuous_agg cagg WHERE ht.id = cagg.mat_hypertable_id ) SELECT user_view, (SELECT relacl FROM pg_class WHERE oid = user_view) AS user_view_perm, relname AS mat_table, (relacl) AS mat_table_perm, direct_view, (SELECT relacl FROM pg_class WHERE oid = direct_view) AS direct_view_perm, partial_view, (SELECT relacl FROM pg_class WHERE oid = partial_view) AS partial_view_perm FROM pg_class JOIN caggs ON pg_class.oid = caggs.mat_relid; GRANT SELECT ON cagg_info TO PUBLIC; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE conditions ( timec INT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec', chunk_time_interval=> 100); table_name ------------ conditions CREATE OR REPLACE FUNCTION integer_now_test1() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now_test1'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW mat_refresh_test WITH (timescaledb.continuous, timescaledb.materialized_only=false) as select location, max(humidity) from conditions group by time_bucket(100, timec), location WITH NO DATA; -- Manually create index on CAgg CREATE INDEX cagg_idx on mat_refresh_test(location); \c :TEST_DBNAME :ROLE_SUPERUSER CREATE USER not_priv; \c :TEST_DBNAME not_priv -- A user with no ownership on the Cagg cannot create index on it. -- This should fail \set ON_ERROR_STOP 0 CREATE INDEX cagg_idx on mat_refresh_test(humidity); ERROR: must be owner of hypertable "_materialized_hypertable_2" \set ON_ERROR_STOP 1 \c :TEST_DBNAME :ROLE_SUPERUSER DROP USER not_priv; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT add_continuous_aggregate_policy('mat_refresh_test', NULL, -200::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1000 insert into conditions select generate_series(0, 50, 10), 'NYC', 55, 75, 40, 70, NULL; CALL refresh_continuous_aggregate(' mat_refresh_test', NULL, NULL); SELECT id as cagg_job_id FROM _timescaledb_catalog.bgw_job order by id desc limit 1 \gset SELECT format('%I.%I', materialization_hypertable_schema, materialization_hypertable_name ) as materialization_hypertable FROM timescaledb_information.continuous_aggregates WHERE view_name = 'mat_refresh_test' \gset SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'mat_refresh_test' \gset SELECT schema_name as mat_chunk_schema, table_name as mat_chunk_table FROM _timescaledb_catalog.chunk WHERE hypertable_id = :mat_hypertable_id ORDER BY id desc LIMIT 1 \gset CREATE TABLE conditions_for_perm_check ( timec INT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable('conditions_for_perm_check', 'timec', chunk_time_interval=> 100); table_name --------------------------- conditions_for_perm_check CREATE OR REPLACE FUNCTION integer_now_test2() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0) FROM conditions_for_perm_check $$; SELECT set_integer_now_func('conditions_for_perm_check', 'integer_now_test2'); set_integer_now_func ---------------------- CREATE TABLE conditions_for_perm_check_w_grant ( timec INT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable('conditions_for_perm_check_w_grant', 'timec', chunk_time_interval=> 100); table_name ----------------------------------- conditions_for_perm_check_w_grant CREATE OR REPLACE FUNCTION integer_now_test3() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0) FROM conditions_for_perm_check_w_grant $$; SELECT set_integer_now_func('conditions_for_perm_check_w_grant', 'integer_now_test3'); set_integer_now_func ---------------------- GRANT SELECT, TRIGGER ON conditions_for_perm_check_w_grant TO public; insert into conditions_for_perm_check_w_grant select generate_series(0, 30, 10), 'POR', 55, 75, 40, 70, NULL; --need both select and trigger permissions to create a materialized view on top of it. GRANT SELECT, TRIGGER ON conditions_for_perm_check_w_grant TO public; \c :TEST_DBNAME :ROLE_SUPERUSER create schema custom_schema; CREATE FUNCTION get_constant() RETURNS INTEGER LANGUAGE SQL IMMUTABLE AS $BODY$ SELECT 10; $BODY$; REVOKE EXECUTE ON FUNCTION get_constant() FROM PUBLIC; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 \set ON_ERROR_STOP 0 select from alter_job(:cagg_job_id, max_runtime => NULL); ERROR: insufficient permissions to alter job 1000 --make sure that commands fail ALTER MATERIALIZED VIEW mat_refresh_test SET(timescaledb.materialized_only = true); ERROR: must be owner of continuous aggregate "mat_refresh_test" DROP MATERIALIZED VIEW mat_refresh_test; ERROR: must be owner of view mat_refresh_test CALL refresh_continuous_aggregate('mat_refresh_test', NULL, NULL); ERROR: must be owner of view mat_refresh_test SELECT * FROM mat_refresh_test; ERROR: permission denied for view mat_refresh_test -- Test permissions also when the watermark is not constified and the ACL checks -- in ts_continuous_agg_watermark are executed SET timescaledb.enable_cagg_watermark_constify = OFF; SELECT * FROM mat_refresh_test; ERROR: permission denied for view mat_refresh_test RESET timescaledb.enable_cagg_watermark_constify; SELECT * FROM :materialization_hypertable; ERROR: permission denied for table _materialized_hypertable_2 SELECT * FROM :"mat_chunk_schema".:"mat_chunk_table"; ERROR: permission denied for table _hyper_2_2_chunk --cannot create a mat view without select and trigger grants CREATE MATERIALIZED VIEW mat_perm_view_test WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select location, max(humidity) from conditions_for_perm_check group by time_bucket(100, timec), location WITH NO DATA; ERROR: permission denied for table conditions_for_perm_check --cannot create mat view in a schema without create privileges CREATE MATERIALIZED VIEW custom_schema.mat_perm_view_test WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select location, max(humidity) from conditions_for_perm_check_w_grant group by time_bucket(100, timec), location WITH NO DATA; ERROR: permission denied for schema custom_schema --cannot use a function without EXECUTE privileges --you can create a VIEW but cannot refresh it CREATE MATERIALIZED VIEW mat_perm_view_test WITH ( timescaledb.continuous, timescaledb.materialized_only=true) as select location, max(humidity), get_constant() from conditions_for_perm_check_w_grant group by time_bucket(100, timec), location WITH NO DATA; --this should fail CALL refresh_continuous_aggregate('mat_perm_view_test', NULL, NULL); ERROR: permission denied for function get_constant DROP MATERIALIZED VIEW mat_perm_view_test; \set ON_ERROR_STOP 1 --can create a mat view on something with select and trigger grants CREATE MATERIALIZED VIEW mat_perm_view_test WITH ( timescaledb.continuous, timescaledb.materialized_only=true) as select location, max(humidity) from conditions_for_perm_check_w_grant group by time_bucket(100, timec), location WITH NO DATA; CALL refresh_continuous_aggregate('mat_perm_view_test', NULL, NULL); SELECT * FROM mat_perm_view_test; location | max ----------+----- POR | 75 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --revoke select permissions from role with mat view REVOKE SELECT ON conditions_for_perm_check_w_grant FROM public; insert into conditions_for_perm_check_w_grant select generate_series(100, 130, 10), 'POR', 65, 85, 30, 90, NULL; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 \set ON_ERROR_STOP 0 --refresh mat view should now fail due to lack of permissions CALL refresh_continuous_aggregate('mat_perm_view_test', NULL, NULL); ERROR: permission denied for table conditions_for_perm_check_w_grant \set ON_ERROR_STOP 1 --but the old data will still be there SELECT * FROM mat_perm_view_test; location | max ----------+----- POR | 75 \set VERBOSITY default -- Test that grants and revokes are propagated to the implementation -- objects, that is, the user view, the partial view, the direct view, -- and the materialization table. \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE devices ( time TIMESTAMPTZ NOT NULL, device INT, temp DOUBLE PRECISION NULL, PRIMARY KEY(time, device) ); SELECT create_hypertable('devices', 'time'); create_hypertable ---------------------- (8,public,devices,t) GRANT SELECT, TRIGGER ON devices TO :ROLE_DEFAULT_PERM_USER_2; INSERT INTO devices SELECT time, (random() * 30)::int, random() * 80 FROM generate_series('2020-02-01 00:00:00'::timestamptz, '2020-03-01 00:00:00', '1 hour') AS time; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 CREATE MATERIALIZED VIEW devices_summary WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1 day', time) AS bucket, device, MAX(temp) FROM devices GROUP BY bucket, device WITH NO DATA; \x on SELECT * FROM cagg_info WHERE user_view::text = 'devices_summary'; -[ RECORD 1 ]-----+-------------------------------------- user_view | devices_summary user_view_perm | mat_table | _materialized_hypertable_9 mat_table_perm | direct_view | _timescaledb_internal._direct_view_9 direct_view_perm | partial_view | _timescaledb_internal._partial_view_9 partial_view_perm | GRANT ALL ON devices_summary TO :ROLE_DEFAULT_PERM_USER; SELECT * FROM cagg_info WHERE user_view::text = 'devices_summary'; -[ RECORD 1 ]-----+------------------------------------------------------------------------------------------------ user_view | devices_summary user_view_perm | {default_perm_user_2=arwdDxt/default_perm_user_2,default_perm_user=arwdDxt/default_perm_user_2} mat_table | _materialized_hypertable_9 mat_table_perm | {default_perm_user_2=arwdDxt/default_perm_user_2,default_perm_user=arwdDxt/default_perm_user_2} direct_view | _timescaledb_internal._direct_view_9 direct_view_perm | {default_perm_user_2=arwdDxt/default_perm_user_2,default_perm_user=arwdDxt/default_perm_user_2} partial_view | _timescaledb_internal._partial_view_9 partial_view_perm | {default_perm_user_2=arwdDxt/default_perm_user_2,default_perm_user=arwdDxt/default_perm_user_2} REVOKE SELECT, UPDATE ON devices_summary FROM :ROLE_DEFAULT_PERM_USER; SELECT * FROM cagg_info WHERE user_view::text = 'devices_summary'; -[ RECORD 1 ]-----+---------------------------------------------------------------------------------------------- user_view | devices_summary user_view_perm | {default_perm_user_2=arwdDxt/default_perm_user_2,default_perm_user=adDxt/default_perm_user_2} mat_table | _materialized_hypertable_9 mat_table_perm | {default_perm_user_2=arwdDxt/default_perm_user_2,default_perm_user=adDxt/default_perm_user_2} direct_view | _timescaledb_internal._direct_view_9 direct_view_perm | {default_perm_user_2=arwdDxt/default_perm_user_2,default_perm_user=adDxt/default_perm_user_2} partial_view | _timescaledb_internal._partial_view_9 partial_view_perm | {default_perm_user_2=arwdDxt/default_perm_user_2,default_perm_user=adDxt/default_perm_user_2} \x off -- Check for default privilege permissions get propagated to the materialization hypertable \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA test_default_privileges; GRANT USAGE ON SCHEMA "test_default_privileges" TO :ROLE_DEFAULT_PERM_USER; ALTER DEFAULT PRIVILEGES IN SCHEMA "test_default_privileges" GRANT SELECT ON TABLES TO :ROLE_DEFAULT_PERM_USER; CREATE TABLE test_default_privileges.devices ( time TIMESTAMPTZ NOT NULL, device INT, temp DOUBLE PRECISION NULL, PRIMARY KEY(time, device) ); SELECT create_hypertable('test_default_privileges.devices', 'time'); create_hypertable ---------------------------------------- (10,test_default_privileges,devices,t) CREATE MATERIALIZED VIEW test_default_privileges.devices_summary WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1 day', time) AS bucket, device, MAX(temp) FROM test_default_privileges.devices GROUP BY bucket, device WITH NO DATA; -- check if user view perms have been propagated to the mat-ht SELECT user_view_perm IS NOT DISTINCT FROM mat_table_perm FROM cagg_info WHERE user_view = 'test_default_privileges.devices_summary'::regclass; ?column? ---------- t ================================================ FILE: tsl/test/expected/cagg_permissions-16.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- initialize the bgw mock state to prevent the materialization workers from running \c :TEST_DBNAME :ROLE_SUPERUSER -- remove any default jobs, e.g., telemetry so bgw_job isn't polluted DELETE FROM _timescaledb_catalog.bgw_job WHERE TRUE; CREATE VIEW cagg_info AS WITH caggs AS ( SELECT format('%I.%I', user_view_schema, user_view_name)::regclass AS user_view, format('%I.%I', direct_view_schema, direct_view_name)::regclass AS direct_view, format('%I.%I', partial_view_schema, partial_view_name)::regclass AS partial_view, format('%I.%I', ht.schema_name, ht.table_name)::regclass AS mat_relid FROM _timescaledb_catalog.hypertable ht, _timescaledb_catalog.continuous_agg cagg WHERE ht.id = cagg.mat_hypertable_id ) SELECT user_view, (SELECT relacl FROM pg_class WHERE oid = user_view) AS user_view_perm, relname AS mat_table, (relacl) AS mat_table_perm, direct_view, (SELECT relacl FROM pg_class WHERE oid = direct_view) AS direct_view_perm, partial_view, (SELECT relacl FROM pg_class WHERE oid = partial_view) AS partial_view_perm FROM pg_class JOIN caggs ON pg_class.oid = caggs.mat_relid; GRANT SELECT ON cagg_info TO PUBLIC; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE conditions ( timec INT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec', chunk_time_interval=> 100); table_name ------------ conditions CREATE OR REPLACE FUNCTION integer_now_test1() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now_test1'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW mat_refresh_test WITH (timescaledb.continuous, timescaledb.materialized_only=false) as select location, max(humidity) from conditions group by time_bucket(100, timec), location WITH NO DATA; -- Manually create index on CAgg CREATE INDEX cagg_idx on mat_refresh_test(location); \c :TEST_DBNAME :ROLE_SUPERUSER CREATE USER not_priv; \c :TEST_DBNAME not_priv -- A user with no ownership on the Cagg cannot create index on it. -- This should fail \set ON_ERROR_STOP 0 CREATE INDEX cagg_idx on mat_refresh_test(humidity); ERROR: must be owner of hypertable "_materialized_hypertable_2" \set ON_ERROR_STOP 1 \c :TEST_DBNAME :ROLE_SUPERUSER DROP USER not_priv; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT add_continuous_aggregate_policy('mat_refresh_test', NULL, -200::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1000 insert into conditions select generate_series(0, 50, 10), 'NYC', 55, 75, 40, 70, NULL; CALL refresh_continuous_aggregate(' mat_refresh_test', NULL, NULL); SELECT id as cagg_job_id FROM _timescaledb_catalog.bgw_job order by id desc limit 1 \gset SELECT format('%I.%I', materialization_hypertable_schema, materialization_hypertable_name ) as materialization_hypertable FROM timescaledb_information.continuous_aggregates WHERE view_name = 'mat_refresh_test' \gset SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'mat_refresh_test' \gset SELECT schema_name as mat_chunk_schema, table_name as mat_chunk_table FROM _timescaledb_catalog.chunk WHERE hypertable_id = :mat_hypertable_id ORDER BY id desc LIMIT 1 \gset CREATE TABLE conditions_for_perm_check ( timec INT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable('conditions_for_perm_check', 'timec', chunk_time_interval=> 100); table_name --------------------------- conditions_for_perm_check CREATE OR REPLACE FUNCTION integer_now_test2() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0) FROM conditions_for_perm_check $$; SELECT set_integer_now_func('conditions_for_perm_check', 'integer_now_test2'); set_integer_now_func ---------------------- CREATE TABLE conditions_for_perm_check_w_grant ( timec INT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable('conditions_for_perm_check_w_grant', 'timec', chunk_time_interval=> 100); table_name ----------------------------------- conditions_for_perm_check_w_grant CREATE OR REPLACE FUNCTION integer_now_test3() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0) FROM conditions_for_perm_check_w_grant $$; SELECT set_integer_now_func('conditions_for_perm_check_w_grant', 'integer_now_test3'); set_integer_now_func ---------------------- GRANT SELECT, TRIGGER ON conditions_for_perm_check_w_grant TO public; insert into conditions_for_perm_check_w_grant select generate_series(0, 30, 10), 'POR', 55, 75, 40, 70, NULL; --need both select and trigger permissions to create a materialized view on top of it. GRANT SELECT, TRIGGER ON conditions_for_perm_check_w_grant TO public; \c :TEST_DBNAME :ROLE_SUPERUSER create schema custom_schema; CREATE FUNCTION get_constant() RETURNS INTEGER LANGUAGE SQL IMMUTABLE AS $BODY$ SELECT 10; $BODY$; REVOKE EXECUTE ON FUNCTION get_constant() FROM PUBLIC; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 \set ON_ERROR_STOP 0 select from alter_job(:cagg_job_id, max_runtime => NULL); ERROR: insufficient permissions to alter job 1000 --make sure that commands fail ALTER MATERIALIZED VIEW mat_refresh_test SET(timescaledb.materialized_only = true); ERROR: must be owner of continuous aggregate "mat_refresh_test" DROP MATERIALIZED VIEW mat_refresh_test; ERROR: must be owner of view mat_refresh_test CALL refresh_continuous_aggregate('mat_refresh_test', NULL, NULL); ERROR: must be owner of view mat_refresh_test SELECT * FROM mat_refresh_test; ERROR: permission denied for view mat_refresh_test -- Test permissions also when the watermark is not constified and the ACL checks -- in ts_continuous_agg_watermark are executed SET timescaledb.enable_cagg_watermark_constify = OFF; SELECT * FROM mat_refresh_test; ERROR: permission denied for view mat_refresh_test RESET timescaledb.enable_cagg_watermark_constify; SELECT * FROM :materialization_hypertable; ERROR: permission denied for table _materialized_hypertable_2 SELECT * FROM :"mat_chunk_schema".:"mat_chunk_table"; ERROR: permission denied for table _hyper_2_2_chunk --cannot create a mat view without select and trigger grants CREATE MATERIALIZED VIEW mat_perm_view_test WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select location, max(humidity) from conditions_for_perm_check group by time_bucket(100, timec), location WITH NO DATA; ERROR: permission denied for table conditions_for_perm_check --cannot create mat view in a schema without create privileges CREATE MATERIALIZED VIEW custom_schema.mat_perm_view_test WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select location, max(humidity) from conditions_for_perm_check_w_grant group by time_bucket(100, timec), location WITH NO DATA; ERROR: permission denied for schema custom_schema --cannot use a function without EXECUTE privileges --you can create a VIEW but cannot refresh it CREATE MATERIALIZED VIEW mat_perm_view_test WITH ( timescaledb.continuous, timescaledb.materialized_only=true) as select location, max(humidity), get_constant() from conditions_for_perm_check_w_grant group by time_bucket(100, timec), location WITH NO DATA; --this should fail CALL refresh_continuous_aggregate('mat_perm_view_test', NULL, NULL); ERROR: permission denied for function get_constant DROP MATERIALIZED VIEW mat_perm_view_test; \set ON_ERROR_STOP 1 --can create a mat view on something with select and trigger grants CREATE MATERIALIZED VIEW mat_perm_view_test WITH ( timescaledb.continuous, timescaledb.materialized_only=true) as select location, max(humidity) from conditions_for_perm_check_w_grant group by time_bucket(100, timec), location WITH NO DATA; CALL refresh_continuous_aggregate('mat_perm_view_test', NULL, NULL); SELECT * FROM mat_perm_view_test; location | max ----------+----- POR | 75 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --revoke select permissions from role with mat view REVOKE SELECT ON conditions_for_perm_check_w_grant FROM public; insert into conditions_for_perm_check_w_grant select generate_series(100, 130, 10), 'POR', 65, 85, 30, 90, NULL; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 \set ON_ERROR_STOP 0 --refresh mat view should now fail due to lack of permissions CALL refresh_continuous_aggregate('mat_perm_view_test', NULL, NULL); ERROR: permission denied for table conditions_for_perm_check_w_grant \set ON_ERROR_STOP 1 --but the old data will still be there SELECT * FROM mat_perm_view_test; location | max ----------+----- POR | 75 \set VERBOSITY default -- Test that grants and revokes are propagated to the implementation -- objects, that is, the user view, the partial view, the direct view, -- and the materialization table. \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE devices ( time TIMESTAMPTZ NOT NULL, device INT, temp DOUBLE PRECISION NULL, PRIMARY KEY(time, device) ); SELECT create_hypertable('devices', 'time'); create_hypertable ---------------------- (8,public,devices,t) GRANT SELECT, TRIGGER ON devices TO :ROLE_DEFAULT_PERM_USER_2; INSERT INTO devices SELECT time, (random() * 30)::int, random() * 80 FROM generate_series('2020-02-01 00:00:00'::timestamptz, '2020-03-01 00:00:00', '1 hour') AS time; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 CREATE MATERIALIZED VIEW devices_summary WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1 day', time) AS bucket, device, MAX(temp) FROM devices GROUP BY bucket, device WITH NO DATA; \x on SELECT * FROM cagg_info WHERE user_view::text = 'devices_summary'; -[ RECORD 1 ]-----+-------------------------------------- user_view | devices_summary user_view_perm | mat_table | _materialized_hypertable_9 mat_table_perm | direct_view | _timescaledb_internal._direct_view_9 direct_view_perm | partial_view | _timescaledb_internal._partial_view_9 partial_view_perm | GRANT ALL ON devices_summary TO :ROLE_DEFAULT_PERM_USER; SELECT * FROM cagg_info WHERE user_view::text = 'devices_summary'; -[ RECORD 1 ]-----+------------------------------------------------------------------------------------------------ user_view | devices_summary user_view_perm | {default_perm_user_2=arwdDxt/default_perm_user_2,default_perm_user=arwdDxt/default_perm_user_2} mat_table | _materialized_hypertable_9 mat_table_perm | {default_perm_user_2=arwdDxt/default_perm_user_2,default_perm_user=arwdDxt/default_perm_user_2} direct_view | _timescaledb_internal._direct_view_9 direct_view_perm | {default_perm_user_2=arwdDxt/default_perm_user_2,default_perm_user=arwdDxt/default_perm_user_2} partial_view | _timescaledb_internal._partial_view_9 partial_view_perm | {default_perm_user_2=arwdDxt/default_perm_user_2,default_perm_user=arwdDxt/default_perm_user_2} REVOKE SELECT, UPDATE ON devices_summary FROM :ROLE_DEFAULT_PERM_USER; SELECT * FROM cagg_info WHERE user_view::text = 'devices_summary'; -[ RECORD 1 ]-----+---------------------------------------------------------------------------------------------- user_view | devices_summary user_view_perm | {default_perm_user_2=arwdDxt/default_perm_user_2,default_perm_user=adDxt/default_perm_user_2} mat_table | _materialized_hypertable_9 mat_table_perm | {default_perm_user_2=arwdDxt/default_perm_user_2,default_perm_user=adDxt/default_perm_user_2} direct_view | _timescaledb_internal._direct_view_9 direct_view_perm | {default_perm_user_2=arwdDxt/default_perm_user_2,default_perm_user=adDxt/default_perm_user_2} partial_view | _timescaledb_internal._partial_view_9 partial_view_perm | {default_perm_user_2=arwdDxt/default_perm_user_2,default_perm_user=adDxt/default_perm_user_2} \x off -- Check for default privilege permissions get propagated to the materialization hypertable \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA test_default_privileges; GRANT USAGE ON SCHEMA "test_default_privileges" TO :ROLE_DEFAULT_PERM_USER; ALTER DEFAULT PRIVILEGES IN SCHEMA "test_default_privileges" GRANT SELECT ON TABLES TO :ROLE_DEFAULT_PERM_USER; CREATE TABLE test_default_privileges.devices ( time TIMESTAMPTZ NOT NULL, device INT, temp DOUBLE PRECISION NULL, PRIMARY KEY(time, device) ); SELECT create_hypertable('test_default_privileges.devices', 'time'); create_hypertable ---------------------------------------- (10,test_default_privileges,devices,t) CREATE MATERIALIZED VIEW test_default_privileges.devices_summary WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1 day', time) AS bucket, device, MAX(temp) FROM test_default_privileges.devices GROUP BY bucket, device WITH NO DATA; -- check if user view perms have been propagated to the mat-ht SELECT user_view_perm IS NOT DISTINCT FROM mat_table_perm FROM cagg_info WHERE user_view = 'test_default_privileges.devices_summary'::regclass; ?column? ---------- t ================================================ FILE: tsl/test/expected/cagg_permissions-17.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- initialize the bgw mock state to prevent the materialization workers from running \c :TEST_DBNAME :ROLE_SUPERUSER -- remove any default jobs, e.g., telemetry so bgw_job isn't polluted DELETE FROM _timescaledb_catalog.bgw_job WHERE TRUE; CREATE VIEW cagg_info AS WITH caggs AS ( SELECT format('%I.%I', user_view_schema, user_view_name)::regclass AS user_view, format('%I.%I', direct_view_schema, direct_view_name)::regclass AS direct_view, format('%I.%I', partial_view_schema, partial_view_name)::regclass AS partial_view, format('%I.%I', ht.schema_name, ht.table_name)::regclass AS mat_relid FROM _timescaledb_catalog.hypertable ht, _timescaledb_catalog.continuous_agg cagg WHERE ht.id = cagg.mat_hypertable_id ) SELECT user_view, (SELECT relacl FROM pg_class WHERE oid = user_view) AS user_view_perm, relname AS mat_table, (relacl) AS mat_table_perm, direct_view, (SELECT relacl FROM pg_class WHERE oid = direct_view) AS direct_view_perm, partial_view, (SELECT relacl FROM pg_class WHERE oid = partial_view) AS partial_view_perm FROM pg_class JOIN caggs ON pg_class.oid = caggs.mat_relid; GRANT SELECT ON cagg_info TO PUBLIC; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE conditions ( timec INT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec', chunk_time_interval=> 100); table_name ------------ conditions CREATE OR REPLACE FUNCTION integer_now_test1() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now_test1'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW mat_refresh_test WITH (timescaledb.continuous, timescaledb.materialized_only=false) as select location, max(humidity) from conditions group by time_bucket(100, timec), location WITH NO DATA; -- Manually create index on CAgg CREATE INDEX cagg_idx on mat_refresh_test(location); \c :TEST_DBNAME :ROLE_SUPERUSER CREATE USER not_priv; \c :TEST_DBNAME not_priv -- A user with no ownership on the Cagg cannot create index on it. -- This should fail \set ON_ERROR_STOP 0 CREATE INDEX cagg_idx on mat_refresh_test(humidity); ERROR: must be owner of hypertable "_materialized_hypertable_2" \set ON_ERROR_STOP 1 \c :TEST_DBNAME :ROLE_SUPERUSER DROP USER not_priv; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT add_continuous_aggregate_policy('mat_refresh_test', NULL, -200::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1000 insert into conditions select generate_series(0, 50, 10), 'NYC', 55, 75, 40, 70, NULL; CALL refresh_continuous_aggregate(' mat_refresh_test', NULL, NULL); SELECT id as cagg_job_id FROM _timescaledb_catalog.bgw_job order by id desc limit 1 \gset SELECT format('%I.%I', materialization_hypertable_schema, materialization_hypertable_name ) as materialization_hypertable FROM timescaledb_information.continuous_aggregates WHERE view_name = 'mat_refresh_test' \gset SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'mat_refresh_test' \gset SELECT schema_name as mat_chunk_schema, table_name as mat_chunk_table FROM _timescaledb_catalog.chunk WHERE hypertable_id = :mat_hypertable_id ORDER BY id desc LIMIT 1 \gset CREATE TABLE conditions_for_perm_check ( timec INT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable('conditions_for_perm_check', 'timec', chunk_time_interval=> 100); table_name --------------------------- conditions_for_perm_check CREATE OR REPLACE FUNCTION integer_now_test2() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0) FROM conditions_for_perm_check $$; SELECT set_integer_now_func('conditions_for_perm_check', 'integer_now_test2'); set_integer_now_func ---------------------- CREATE TABLE conditions_for_perm_check_w_grant ( timec INT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable('conditions_for_perm_check_w_grant', 'timec', chunk_time_interval=> 100); table_name ----------------------------------- conditions_for_perm_check_w_grant CREATE OR REPLACE FUNCTION integer_now_test3() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0) FROM conditions_for_perm_check_w_grant $$; SELECT set_integer_now_func('conditions_for_perm_check_w_grant', 'integer_now_test3'); set_integer_now_func ---------------------- GRANT SELECT, TRIGGER ON conditions_for_perm_check_w_grant TO public; insert into conditions_for_perm_check_w_grant select generate_series(0, 30, 10), 'POR', 55, 75, 40, 70, NULL; --need both select and trigger permissions to create a materialized view on top of it. GRANT SELECT, TRIGGER ON conditions_for_perm_check_w_grant TO public; \c :TEST_DBNAME :ROLE_SUPERUSER create schema custom_schema; CREATE FUNCTION get_constant() RETURNS INTEGER LANGUAGE SQL IMMUTABLE AS $BODY$ SELECT 10; $BODY$; REVOKE EXECUTE ON FUNCTION get_constant() FROM PUBLIC; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 \set ON_ERROR_STOP 0 select from alter_job(:cagg_job_id, max_runtime => NULL); ERROR: insufficient permissions to alter job 1000 --make sure that commands fail ALTER MATERIALIZED VIEW mat_refresh_test SET(timescaledb.materialized_only = true); ERROR: must be owner of continuous aggregate "mat_refresh_test" DROP MATERIALIZED VIEW mat_refresh_test; ERROR: must be owner of view mat_refresh_test CALL refresh_continuous_aggregate('mat_refresh_test', NULL, NULL); ERROR: must be owner of view mat_refresh_test SELECT * FROM mat_refresh_test; ERROR: permission denied for view mat_refresh_test -- Test permissions also when the watermark is not constified and the ACL checks -- in ts_continuous_agg_watermark are executed SET timescaledb.enable_cagg_watermark_constify = OFF; SELECT * FROM mat_refresh_test; ERROR: permission denied for view mat_refresh_test RESET timescaledb.enable_cagg_watermark_constify; SELECT * FROM :materialization_hypertable; ERROR: permission denied for table _materialized_hypertable_2 SELECT * FROM :"mat_chunk_schema".:"mat_chunk_table"; ERROR: permission denied for table _hyper_2_2_chunk --cannot create a mat view without select and trigger grants CREATE MATERIALIZED VIEW mat_perm_view_test WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select location, max(humidity) from conditions_for_perm_check group by time_bucket(100, timec), location WITH NO DATA; ERROR: permission denied for table conditions_for_perm_check --cannot create mat view in a schema without create privileges CREATE MATERIALIZED VIEW custom_schema.mat_perm_view_test WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select location, max(humidity) from conditions_for_perm_check_w_grant group by time_bucket(100, timec), location WITH NO DATA; ERROR: permission denied for schema custom_schema --cannot use a function without EXECUTE privileges --you can create a VIEW but cannot refresh it CREATE MATERIALIZED VIEW mat_perm_view_test WITH ( timescaledb.continuous, timescaledb.materialized_only=true) as select location, max(humidity), get_constant() from conditions_for_perm_check_w_grant group by time_bucket(100, timec), location WITH NO DATA; --this should fail CALL refresh_continuous_aggregate('mat_perm_view_test', NULL, NULL); ERROR: permission denied for function get_constant DROP MATERIALIZED VIEW mat_perm_view_test; \set ON_ERROR_STOP 1 --can create a mat view on something with select and trigger grants CREATE MATERIALIZED VIEW mat_perm_view_test WITH ( timescaledb.continuous, timescaledb.materialized_only=true) as select location, max(humidity) from conditions_for_perm_check_w_grant group by time_bucket(100, timec), location WITH NO DATA; CALL refresh_continuous_aggregate('mat_perm_view_test', NULL, NULL); SELECT * FROM mat_perm_view_test; location | max ----------+----- POR | 75 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --revoke select permissions from role with mat view REVOKE SELECT ON conditions_for_perm_check_w_grant FROM public; insert into conditions_for_perm_check_w_grant select generate_series(100, 130, 10), 'POR', 65, 85, 30, 90, NULL; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 \set ON_ERROR_STOP 0 --refresh mat view should now fail due to lack of permissions CALL refresh_continuous_aggregate('mat_perm_view_test', NULL, NULL); ERROR: permission denied for table conditions_for_perm_check_w_grant \set ON_ERROR_STOP 1 --but the old data will still be there SELECT * FROM mat_perm_view_test; location | max ----------+----- POR | 75 \set VERBOSITY default -- Test that grants and revokes are propagated to the implementation -- objects, that is, the user view, the partial view, the direct view, -- and the materialization table. \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE devices ( time TIMESTAMPTZ NOT NULL, device INT, temp DOUBLE PRECISION NULL, PRIMARY KEY(time, device) ); SELECT create_hypertable('devices', 'time'); create_hypertable ---------------------- (8,public,devices,t) GRANT SELECT, TRIGGER ON devices TO :ROLE_DEFAULT_PERM_USER_2; INSERT INTO devices SELECT time, (random() * 30)::int, random() * 80 FROM generate_series('2020-02-01 00:00:00'::timestamptz, '2020-03-01 00:00:00', '1 hour') AS time; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 CREATE MATERIALIZED VIEW devices_summary WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1 day', time) AS bucket, device, MAX(temp) FROM devices GROUP BY bucket, device WITH NO DATA; \x on SELECT * FROM cagg_info WHERE user_view::text = 'devices_summary'; -[ RECORD 1 ]-----+-------------------------------------- user_view | devices_summary user_view_perm | mat_table | _materialized_hypertable_9 mat_table_perm | direct_view | _timescaledb_internal._direct_view_9 direct_view_perm | partial_view | _timescaledb_internal._partial_view_9 partial_view_perm | GRANT ALL ON devices_summary TO :ROLE_DEFAULT_PERM_USER; SELECT * FROM cagg_info WHERE user_view::text = 'devices_summary'; -[ RECORD 1 ]-----+-------------------------------------------------------------------------------------------------- user_view | devices_summary user_view_perm | {default_perm_user_2=arwdDxtm/default_perm_user_2,default_perm_user=arwdDxtm/default_perm_user_2} mat_table | _materialized_hypertable_9 mat_table_perm | {default_perm_user_2=arwdDxtm/default_perm_user_2,default_perm_user=arwdDxtm/default_perm_user_2} direct_view | _timescaledb_internal._direct_view_9 direct_view_perm | {default_perm_user_2=arwdDxtm/default_perm_user_2,default_perm_user=arwdDxtm/default_perm_user_2} partial_view | _timescaledb_internal._partial_view_9 partial_view_perm | {default_perm_user_2=arwdDxtm/default_perm_user_2,default_perm_user=arwdDxtm/default_perm_user_2} REVOKE SELECT, UPDATE ON devices_summary FROM :ROLE_DEFAULT_PERM_USER; SELECT * FROM cagg_info WHERE user_view::text = 'devices_summary'; -[ RECORD 1 ]-----+------------------------------------------------------------------------------------------------ user_view | devices_summary user_view_perm | {default_perm_user_2=arwdDxtm/default_perm_user_2,default_perm_user=adDxtm/default_perm_user_2} mat_table | _materialized_hypertable_9 mat_table_perm | {default_perm_user_2=arwdDxtm/default_perm_user_2,default_perm_user=adDxtm/default_perm_user_2} direct_view | _timescaledb_internal._direct_view_9 direct_view_perm | {default_perm_user_2=arwdDxtm/default_perm_user_2,default_perm_user=adDxtm/default_perm_user_2} partial_view | _timescaledb_internal._partial_view_9 partial_view_perm | {default_perm_user_2=arwdDxtm/default_perm_user_2,default_perm_user=adDxtm/default_perm_user_2} \x off -- Check for default privilege permissions get propagated to the materialization hypertable \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA test_default_privileges; GRANT USAGE ON SCHEMA "test_default_privileges" TO :ROLE_DEFAULT_PERM_USER; ALTER DEFAULT PRIVILEGES IN SCHEMA "test_default_privileges" GRANT SELECT ON TABLES TO :ROLE_DEFAULT_PERM_USER; CREATE TABLE test_default_privileges.devices ( time TIMESTAMPTZ NOT NULL, device INT, temp DOUBLE PRECISION NULL, PRIMARY KEY(time, device) ); SELECT create_hypertable('test_default_privileges.devices', 'time'); create_hypertable ---------------------------------------- (10,test_default_privileges,devices,t) CREATE MATERIALIZED VIEW test_default_privileges.devices_summary WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1 day', time) AS bucket, device, MAX(temp) FROM test_default_privileges.devices GROUP BY bucket, device WITH NO DATA; -- check if user view perms have been propagated to the mat-ht SELECT user_view_perm IS NOT DISTINCT FROM mat_table_perm FROM cagg_info WHERE user_view = 'test_default_privileges.devices_summary'::regclass; ?column? ---------- t ================================================ FILE: tsl/test/expected/cagg_permissions-18.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- initialize the bgw mock state to prevent the materialization workers from running \c :TEST_DBNAME :ROLE_SUPERUSER -- remove any default jobs, e.g., telemetry so bgw_job isn't polluted DELETE FROM _timescaledb_catalog.bgw_job WHERE TRUE; CREATE VIEW cagg_info AS WITH caggs AS ( SELECT format('%I.%I', user_view_schema, user_view_name)::regclass AS user_view, format('%I.%I', direct_view_schema, direct_view_name)::regclass AS direct_view, format('%I.%I', partial_view_schema, partial_view_name)::regclass AS partial_view, format('%I.%I', ht.schema_name, ht.table_name)::regclass AS mat_relid FROM _timescaledb_catalog.hypertable ht, _timescaledb_catalog.continuous_agg cagg WHERE ht.id = cagg.mat_hypertable_id ) SELECT user_view, (SELECT relacl FROM pg_class WHERE oid = user_view) AS user_view_perm, relname AS mat_table, (relacl) AS mat_table_perm, direct_view, (SELECT relacl FROM pg_class WHERE oid = direct_view) AS direct_view_perm, partial_view, (SELECT relacl FROM pg_class WHERE oid = partial_view) AS partial_view_perm FROM pg_class JOIN caggs ON pg_class.oid = caggs.mat_relid; GRANT SELECT ON cagg_info TO PUBLIC; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE conditions ( timec INT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable( 'conditions', 'timec', chunk_time_interval=> 100); table_name ------------ conditions CREATE OR REPLACE FUNCTION integer_now_test1() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0) FROM conditions $$; SELECT set_integer_now_func('conditions', 'integer_now_test1'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW mat_refresh_test WITH (timescaledb.continuous, timescaledb.materialized_only=false) as select location, max(humidity) from conditions group by time_bucket(100, timec), location WITH NO DATA; -- Manually create index on CAgg CREATE INDEX cagg_idx on mat_refresh_test(location); \c :TEST_DBNAME :ROLE_SUPERUSER CREATE USER not_priv; \c :TEST_DBNAME not_priv -- A user with no ownership on the Cagg cannot create index on it. -- This should fail \set ON_ERROR_STOP 0 CREATE INDEX cagg_idx on mat_refresh_test(humidity); ERROR: must be owner of hypertable "_materialized_hypertable_2" \set ON_ERROR_STOP 1 \c :TEST_DBNAME :ROLE_SUPERUSER DROP USER not_priv; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER SELECT add_continuous_aggregate_policy('mat_refresh_test', NULL, -200::integer, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1000 insert into conditions select generate_series(0, 50, 10), 'NYC', 55, 75, 40, 70, NULL; CALL refresh_continuous_aggregate(' mat_refresh_test', NULL, NULL); SELECT id as cagg_job_id FROM _timescaledb_catalog.bgw_job order by id desc limit 1 \gset SELECT format('%I.%I', materialization_hypertable_schema, materialization_hypertable_name ) as materialization_hypertable FROM timescaledb_information.continuous_aggregates WHERE view_name = 'mat_refresh_test' \gset SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'mat_refresh_test' \gset SELECT schema_name as mat_chunk_schema, table_name as mat_chunk_table FROM _timescaledb_catalog.chunk WHERE hypertable_id = :mat_hypertable_id ORDER BY id desc LIMIT 1 \gset CREATE TABLE conditions_for_perm_check ( timec INT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable('conditions_for_perm_check', 'timec', chunk_time_interval=> 100); table_name --------------------------- conditions_for_perm_check CREATE OR REPLACE FUNCTION integer_now_test2() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0) FROM conditions_for_perm_check $$; SELECT set_integer_now_func('conditions_for_perm_check', 'integer_now_test2'); set_integer_now_func ---------------------- CREATE TABLE conditions_for_perm_check_w_grant ( timec INT NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL, lowp double precision NULL, highp double precision null, allnull double precision null ); select table_name from create_hypertable('conditions_for_perm_check_w_grant', 'timec', chunk_time_interval=> 100); table_name ----------------------------------- conditions_for_perm_check_w_grant CREATE OR REPLACE FUNCTION integer_now_test3() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(timec), 0) FROM conditions_for_perm_check_w_grant $$; SELECT set_integer_now_func('conditions_for_perm_check_w_grant', 'integer_now_test3'); set_integer_now_func ---------------------- GRANT SELECT, TRIGGER ON conditions_for_perm_check_w_grant TO public; insert into conditions_for_perm_check_w_grant select generate_series(0, 30, 10), 'POR', 55, 75, 40, 70, NULL; --need both select and trigger permissions to create a materialized view on top of it. GRANT SELECT, TRIGGER ON conditions_for_perm_check_w_grant TO public; \c :TEST_DBNAME :ROLE_SUPERUSER create schema custom_schema; CREATE FUNCTION get_constant() RETURNS INTEGER LANGUAGE SQL IMMUTABLE AS $BODY$ SELECT 10; $BODY$; REVOKE EXECUTE ON FUNCTION get_constant() FROM PUBLIC; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 \set ON_ERROR_STOP 0 select from alter_job(:cagg_job_id, max_runtime => NULL); ERROR: insufficient permissions to alter job 1000 --make sure that commands fail ALTER MATERIALIZED VIEW mat_refresh_test SET(timescaledb.materialized_only = true); ERROR: must be owner of continuous aggregate "mat_refresh_test" DROP MATERIALIZED VIEW mat_refresh_test; ERROR: must be owner of view mat_refresh_test CALL refresh_continuous_aggregate('mat_refresh_test', NULL, NULL); ERROR: must be owner of view mat_refresh_test SELECT * FROM mat_refresh_test; ERROR: permission denied for view mat_refresh_test -- Test permissions also when the watermark is not constified and the ACL checks -- in ts_continuous_agg_watermark are executed SET timescaledb.enable_cagg_watermark_constify = OFF; SELECT * FROM mat_refresh_test; ERROR: permission denied for view mat_refresh_test RESET timescaledb.enable_cagg_watermark_constify; SELECT * FROM :materialization_hypertable; ERROR: permission denied for table _materialized_hypertable_2 SELECT * FROM :"mat_chunk_schema".:"mat_chunk_table"; ERROR: permission denied for table _hyper_2_2_chunk --cannot create a mat view without select and trigger grants CREATE MATERIALIZED VIEW mat_perm_view_test WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select location, max(humidity) from conditions_for_perm_check group by time_bucket(100, timec), location WITH NO DATA; ERROR: permission denied for table conditions_for_perm_check --cannot create mat view in a schema without create privileges CREATE MATERIALIZED VIEW custom_schema.mat_perm_view_test WITH (timescaledb.continuous, timescaledb.materialized_only=true) as select location, max(humidity) from conditions_for_perm_check_w_grant group by time_bucket(100, timec), location WITH NO DATA; ERROR: permission denied for schema custom_schema --cannot use a function without EXECUTE privileges --you can create a VIEW but cannot refresh it CREATE MATERIALIZED VIEW mat_perm_view_test WITH ( timescaledb.continuous, timescaledb.materialized_only=true) as select location, max(humidity), get_constant() from conditions_for_perm_check_w_grant group by time_bucket(100, timec), location WITH NO DATA; --this should fail CALL refresh_continuous_aggregate('mat_perm_view_test', NULL, NULL); ERROR: permission denied for function get_constant DROP MATERIALIZED VIEW mat_perm_view_test; \set ON_ERROR_STOP 1 --can create a mat view on something with select and trigger grants CREATE MATERIALIZED VIEW mat_perm_view_test WITH ( timescaledb.continuous, timescaledb.materialized_only=true) as select location, max(humidity) from conditions_for_perm_check_w_grant group by time_bucket(100, timec), location WITH NO DATA; CALL refresh_continuous_aggregate('mat_perm_view_test', NULL, NULL); SELECT * FROM mat_perm_view_test; location | max ----------+----- POR | 75 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER --revoke select permissions from role with mat view REVOKE SELECT ON conditions_for_perm_check_w_grant FROM public; insert into conditions_for_perm_check_w_grant select generate_series(100, 130, 10), 'POR', 65, 85, 30, 90, NULL; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 \set ON_ERROR_STOP 0 --refresh mat view should now fail due to lack of permissions CALL refresh_continuous_aggregate('mat_perm_view_test', NULL, NULL); ERROR: permission denied for table conditions_for_perm_check_w_grant \set ON_ERROR_STOP 1 --but the old data will still be there SELECT * FROM mat_perm_view_test; location | max ----------+----- POR | 75 \set VERBOSITY default -- Test that grants and revokes are propagated to the implementation -- objects, that is, the user view, the partial view, the direct view, -- and the materialization table. \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER CREATE TABLE devices ( time TIMESTAMPTZ NOT NULL, device INT, temp DOUBLE PRECISION NULL, PRIMARY KEY(time, device) ); SELECT create_hypertable('devices', 'time'); create_hypertable ---------------------- (8,public,devices,t) GRANT SELECT, TRIGGER ON devices TO :ROLE_DEFAULT_PERM_USER_2; INSERT INTO devices SELECT time, (random() * 30)::int, random() * 80 FROM generate_series('2020-02-01 00:00:00'::timestamptz, '2020-03-01 00:00:00', '1 hour') AS time; \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER_2 CREATE MATERIALIZED VIEW devices_summary WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1 day', time) AS bucket, device, MAX(temp) FROM devices GROUP BY bucket, device WITH NO DATA; \x on SELECT * FROM cagg_info WHERE user_view::text = 'devices_summary'; -[ RECORD 1 ]-----+-------------------------------------- user_view | devices_summary user_view_perm | mat_table | _materialized_hypertable_9 mat_table_perm | direct_view | _timescaledb_internal._direct_view_9 direct_view_perm | partial_view | _timescaledb_internal._partial_view_9 partial_view_perm | GRANT ALL ON devices_summary TO :ROLE_DEFAULT_PERM_USER; SELECT * FROM cagg_info WHERE user_view::text = 'devices_summary'; -[ RECORD 1 ]-----+-------------------------------------------------------------------------------------------------- user_view | devices_summary user_view_perm | {default_perm_user_2=arwdDxtm/default_perm_user_2,default_perm_user=arwdDxtm/default_perm_user_2} mat_table | _materialized_hypertable_9 mat_table_perm | {default_perm_user_2=arwdDxtm/default_perm_user_2,default_perm_user=arwdDxtm/default_perm_user_2} direct_view | _timescaledb_internal._direct_view_9 direct_view_perm | {default_perm_user_2=arwdDxtm/default_perm_user_2,default_perm_user=arwdDxtm/default_perm_user_2} partial_view | _timescaledb_internal._partial_view_9 partial_view_perm | {default_perm_user_2=arwdDxtm/default_perm_user_2,default_perm_user=arwdDxtm/default_perm_user_2} REVOKE SELECT, UPDATE ON devices_summary FROM :ROLE_DEFAULT_PERM_USER; SELECT * FROM cagg_info WHERE user_view::text = 'devices_summary'; -[ RECORD 1 ]-----+------------------------------------------------------------------------------------------------ user_view | devices_summary user_view_perm | {default_perm_user_2=arwdDxtm/default_perm_user_2,default_perm_user=adDxtm/default_perm_user_2} mat_table | _materialized_hypertable_9 mat_table_perm | {default_perm_user_2=arwdDxtm/default_perm_user_2,default_perm_user=adDxtm/default_perm_user_2} direct_view | _timescaledb_internal._direct_view_9 direct_view_perm | {default_perm_user_2=arwdDxtm/default_perm_user_2,default_perm_user=adDxtm/default_perm_user_2} partial_view | _timescaledb_internal._partial_view_9 partial_view_perm | {default_perm_user_2=arwdDxtm/default_perm_user_2,default_perm_user=adDxtm/default_perm_user_2} \x off -- Check for default privilege permissions get propagated to the materialization hypertable \c :TEST_DBNAME :ROLE_SUPERUSER CREATE SCHEMA test_default_privileges; GRANT USAGE ON SCHEMA "test_default_privileges" TO :ROLE_DEFAULT_PERM_USER; ALTER DEFAULT PRIVILEGES IN SCHEMA "test_default_privileges" GRANT SELECT ON TABLES TO :ROLE_DEFAULT_PERM_USER; CREATE TABLE test_default_privileges.devices ( time TIMESTAMPTZ NOT NULL, device INT, temp DOUBLE PRECISION NULL, PRIMARY KEY(time, device) ); SELECT create_hypertable('test_default_privileges.devices', 'time'); create_hypertable ---------------------------------------- (10,test_default_privileges,devices,t) CREATE MATERIALIZED VIEW test_default_privileges.devices_summary WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('1 day', time) AS bucket, device, MAX(temp) FROM test_default_privileges.devices GROUP BY bucket, device WITH NO DATA; -- check if user view perms have been propagated to the mat-ht SELECT user_view_perm IS NOT DISTINCT FROM mat_table_perm FROM cagg_info WHERE user_view = 'test_default_privileges.devices_summary'::regclass; ?column? ---------- t ================================================ FILE: tsl/test/expected/cagg_planning.out ================================================ [File too large to display: 46.7 KB] ================================================ FILE: tsl/test/expected/cagg_policy.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- test add and remove refresh policy apis SET ROLE :ROLE_DEFAULT_PERM_USER; --TEST1 --- --basic test with count CREATE TABLE int_tab (a integer, b integer, c integer); SELECT table_name FROM create_hypertable('int_tab', 'a', chunk_time_interval=> 10); table_name ------------ int_tab INSERT INTO int_tab VALUES( 3 , 16 , 20); INSERT INTO int_tab VALUES( 1 , 10 , 20); INSERT INTO int_tab VALUES( 1 , 11 , 20); INSERT INTO int_tab VALUES( 1 , 12 , 20); INSERT INTO int_tab VALUES( 1 , 13 , 20); INSERT INTO int_tab VALUES( 1 , 14 , 20); INSERT INTO int_tab VALUES( 2 , 14 , 20); INSERT INTO int_tab VALUES( 2 , 15 , 20); INSERT INTO int_tab VALUES( 2 , 16 , 20); CREATE OR REPLACE FUNCTION integer_now_int_tab() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(a), 0) FROM int_tab $$; SELECT set_integer_now_func('int_tab', 'integer_now_int_tab'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW mat_m1( a, countb ) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as SELECT a, count(b) FROM int_tab GROUP BY time_bucket(1, a), a WITH NO DATA; \c :TEST_DBNAME :ROLE_SUPERUSER SET timezone TO PST8PDT; DELETE FROM _timescaledb_catalog.bgw_job WHERE TRUE; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT count(*) FROM _timescaledb_catalog.bgw_job; count ------- 0 \set ON_ERROR_STOP 0 \set VERBOSITY default -- Test 1 step policy for integer type buckets ALTER materialized view mat_m1 set (timescaledb.compress = true); NOTICE: defaulting compress_orderby to time_partition_col,a -- No policy is added if one errors out SELECT timescaledb_experimental.add_policies('mat_m1', refresh_start_offset => 1, refresh_end_offset => 10, compress_after => 11, drop_after => 20); ERROR: policy refresh window too small DETAIL: The start and end offsets must cover at least two buckets in the valid time range of type "integer". SELECT timescaledb_experimental.show_policies('mat_m1'); show_policies --------------- -- All policies are added in one step SELECT timescaledb_experimental.add_policies('mat_m1', refresh_start_offset => 10, refresh_end_offset => 1, compress_after => 11, drop_after => 20); add_policies -------------- t SELECT timescaledb_experimental.show_policies('mat_m1'); show_policies --------------------------------------------------------------------------------------------------------------------------------------------- {"policy_name": "policy_compression", "compress_after": 11, "compress_interval": "@ 1 day"} {"policy_name": "policy_refresh_continuous_aggregate", "refresh_interval": "@ 1 hour", "refresh_end_offset": 1, "refresh_start_offset": 10} {"drop_after": 20, "policy_name": "policy_retention", "retention_interval": "@ 1 day"} --Test coverage: new view for policies on CAggs SELECT * FROM timescaledb_experimental.policies ORDER BY relation_name, proc_name; relation_name | relation_schema | schedule_interval | proc_schema | proc_name | config | hypertable_schema | hypertable_name ---------------+-----------------+-------------------+------------------------+-------------------------------------+---------------------------------------------------------------+-----------------------+---------------------------- mat_m1 | public | @ 1 day | _timescaledb_functions | policy_compression | {"hypertable_id": 2, "compress_after": 11} | _timescaledb_internal | _materialized_hypertable_2 mat_m1 | public | @ 1 hour | _timescaledb_functions | policy_refresh_continuous_aggregate | {"end_offset": 1, "start_offset": 10, "mat_hypertable_id": 2} | _timescaledb_internal | _materialized_hypertable_2 mat_m1 | public | @ 1 day | _timescaledb_functions | policy_retention | {"drop_after": 20, "hypertable_id": 2} | _timescaledb_internal | _materialized_hypertable_2 --Test coverage: new view for policies only shows the policies for CAggs SELECT add_retention_policy('int_tab', 20); add_retention_policy ---------------------- 1003 SELECT * FROM timescaledb_experimental.policies ORDER BY relation_name, proc_name; relation_name | relation_schema | schedule_interval | proc_schema | proc_name | config | hypertable_schema | hypertable_name ---------------+-----------------+-------------------+------------------------+-------------------------------------+---------------------------------------------------------------+-----------------------+---------------------------- mat_m1 | public | @ 1 day | _timescaledb_functions | policy_compression | {"hypertable_id": 2, "compress_after": 11} | _timescaledb_internal | _materialized_hypertable_2 mat_m1 | public | @ 1 hour | _timescaledb_functions | policy_refresh_continuous_aggregate | {"end_offset": 1, "start_offset": 10, "mat_hypertable_id": 2} | _timescaledb_internal | _materialized_hypertable_2 mat_m1 | public | @ 1 day | _timescaledb_functions | policy_retention | {"drop_after": 20, "hypertable_id": 2} | _timescaledb_internal | _materialized_hypertable_2 SELECT remove_retention_policy('int_tab'); remove_retention_policy ------------------------- -- Test for duplicated policies (issue #5492) CREATE MATERIALIZED VIEW mat_m2( a, sumb ) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as SELECT a, sum(b) FROM int_tab GROUP BY time_bucket(1, a), a WITH NO DATA; -- add refresh policy SELECT timescaledb_experimental.add_policies('mat_m2', refresh_start_offset => 10, refresh_end_offset => 1); add_policies -------------- t SELECT timescaledb_experimental.show_policies('mat_m2'); show_policies --------------------------------------------------------------------------------------------------------------------------------------------- {"policy_name": "policy_refresh_continuous_aggregate", "refresh_interval": "@ 1 hour", "refresh_end_offset": 1, "refresh_start_offset": 10} -- check for only one refresh policy for each cagg SELECT * FROM timescaledb_experimental.policies WHERE proc_name ~ 'refresh' ORDER BY relation_name, proc_name; relation_name | relation_schema | schedule_interval | proc_schema | proc_name | config | hypertable_schema | hypertable_name ---------------+-----------------+-------------------+------------------------+-------------------------------------+---------------------------------------------------------------+-----------------------+---------------------------- mat_m1 | public | @ 1 hour | _timescaledb_functions | policy_refresh_continuous_aggregate | {"end_offset": 1, "start_offset": 10, "mat_hypertable_id": 2} | _timescaledb_internal | _materialized_hypertable_2 mat_m2 | public | @ 1 hour | _timescaledb_functions | policy_refresh_continuous_aggregate | {"end_offset": 1, "start_offset": 10, "mat_hypertable_id": 4} | _timescaledb_internal | _materialized_hypertable_4 SELECT timescaledb_experimental.remove_all_policies('mat_m2'); remove_all_policies --------------------- t DROP MATERIALIZED VIEW mat_m2; -- Alter policies SELECT timescaledb_experimental.alter_policies('mat_m1', refresh_start_offset => 11, compress_after=>13, drop_after => 25); alter_policies ---------------- t SELECT timescaledb_experimental.show_policies('mat_m1'); show_policies --------------------------------------------------------------------------------------------------------------------------------------------- {"policy_name": "policy_compression", "compress_after": 13, "compress_interval": "@ 1 day"} {"policy_name": "policy_refresh_continuous_aggregate", "refresh_interval": "@ 1 hour", "refresh_end_offset": 1, "refresh_start_offset": 11} {"drop_after": 25, "policy_name": "policy_retention", "retention_interval": "@ 1 day"} -- Remove one or more policy SELECT timescaledb_experimental.remove_policies('mat_m1', false, 'policy_refresh_continuous_aggregate', 'policy_compression'); remove_policies ----------------- t SELECT timescaledb_experimental.show_policies('mat_m1'); show_policies ---------------------------------------------------------------------------------------- {"drop_after": 25, "policy_name": "policy_retention", "retention_interval": "@ 1 day"} -- Add one policy SELECT timescaledb_experimental.add_policies('mat_m1', refresh_start_offset => 10, refresh_end_offset => 1); add_policies -------------- t SELECT timescaledb_experimental.show_policies('mat_m1'); show_policies --------------------------------------------------------------------------------------------------------------------------------------------- {"policy_name": "policy_refresh_continuous_aggregate", "refresh_interval": "@ 1 hour", "refresh_end_offset": 1, "refresh_start_offset": 10} {"drop_after": 25, "policy_name": "policy_retention", "retention_interval": "@ 1 day"} -- Remove all policies SELECT timescaledb_experimental.remove_policies('mat_m1', false, 'policy_refresh_continuous_aggregate', 'policy_retention'); remove_policies ----------------- t SELECT timescaledb_experimental.show_policies('mat_m1'); show_policies --------------- --Cross policy checks --refresh and compression policy overlap SELECT timescaledb_experimental.add_policies('mat_m1', refresh_start_offset => 12, refresh_end_offset => 1, compress_after=>11); ERROR: refresh and columnstore policies overlap --refresh and retention policy overlap SELECT timescaledb_experimental.add_policies('mat_m1', refresh_start_offset => 12, refresh_end_offset => 1, drop_after=>11); ERROR: refresh and retention policies overlap --compression and retention policy overlap SELECT timescaledb_experimental.add_policies('mat_m1', compress_after => 10, drop_after => 10); ERROR: columnstore and retention policies overlap -- Alter non existent policies SELECT timescaledb_experimental.alter_policies('mat_m1', refresh_start_offset => 12, compress_after=>11, drop_after => 15); ERROR: no jobs found ALTER materialized view mat_m1 set (timescaledb.compress = false); SELECT add_continuous_aggregate_policy('int_tab', '1 day'::interval, 10 , '1 h'::interval); ERROR: "int_tab" is not a continuous aggregate SELECT add_continuous_aggregate_policy('mat_m1', '1 day'::interval, 10 , '1 h'::interval); ERROR: invalid parameter value for start_offset HINT: Use time interval of type integer with the continuous aggregate. SELECT add_continuous_aggregate_policy('mat_m1', '1 day'::interval, 10 ); ERROR: function add_continuous_aggregate_policy(unknown, interval, integer) does not exist LINE 1: SELECT add_continuous_aggregate_policy('mat_m1', '1 day'::in... ^ HINT: No function matches the given name and argument types. You might need to add explicit type casts. SELECT add_continuous_aggregate_policy('mat_m1', 10, '1 day'::interval, '1 h'::interval); ERROR: invalid parameter value for end_offset HINT: Use time interval of type integer with the continuous aggregate. --start_interval < end_interval SELECT add_continuous_aggregate_policy('mat_m1', 5, 10, '1h'::interval); ERROR: policy refresh window too small DETAIL: The start and end offsets must cover at least two buckets in the valid time range of type "integer". --refresh window less than two buckets SELECT add_continuous_aggregate_policy('mat_m1', 11, 10, '1h'::interval); ERROR: policy refresh window too small DETAIL: The start and end offsets must cover at least two buckets in the valid time range of type "integer". SELECT add_continuous_aggregate_policy('mat_m1', 20, 10, '1h'::interval) as job_id \gset --adding again should warn/error SELECT add_continuous_aggregate_policy('mat_m1', 20, 10, '1h'::interval, if_not_exists=>false); ERROR: continuous aggregate refresh policy already exists for "mat_m1" DETAIL: A refresh policy with the same start and end offset already exists for continuous aggregate "mat_m1". SELECT add_continuous_aggregate_policy('mat_m1', 20, 15, '1h'::interval, if_not_exists=>true); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT add_continuous_aggregate_policy('mat_m1', 20, 10, '1h'::interval, if_not_exists=>true); NOTICE: continuous aggregate refresh policy already exists for "mat_m1", skipping DETAIL: A refresh policy with the same start and end offset already exists for continuous aggregate "mat_m1". add_continuous_aggregate_policy --------------------------------- -1 -- modify config and try to add, should error out SELECT config FROM _timescaledb_catalog.bgw_job where id = :job_id; config ---------------------------------------------------------------- {"end_offset": 10, "start_offset": 20, "mat_hypertable_id": 2} SELECT hypertable_id as mat_id FROM _timescaledb_catalog.bgw_job where id = :job_id \gset \set VERBOSITY terse \set ON_ERROR_STOP 1 \c :TEST_DBNAME :ROLE_SUPERUSER SET timezone TO PST8PDT; UPDATE _timescaledb_catalog.bgw_job SET config = jsonb_build_object('mat_hypertable_id', :mat_id) WHERE id = :job_id; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT config FROM _timescaledb_catalog.bgw_job where id = :job_id; config -------------------------- {"mat_hypertable_id": 2} \set ON_ERROR_STOP 0 \set VERBOSITY default SELECT add_continuous_aggregate_policy('mat_m1', 20, 10, '1h'::interval, if_not_exists=>true); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('int_tab'); ERROR: "int_tab" is not a continuous aggregate SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ -- add with NULL offset, readd with NULL offset SELECT add_continuous_aggregate_policy('mat_m1', 20, NULL, '1h'::interval, if_not_exists=>true); add_continuous_aggregate_policy --------------------------------- 1010 SELECT add_continuous_aggregate_policy('mat_m1', 20, NULL, '1h'::interval, if_not_exists=>true); -- same param values, so we get a NOTICE NOTICE: continuous aggregate refresh policy already exists for "mat_m1", skipping DETAIL: A refresh policy with the same start and end offset already exists for continuous aggregate "mat_m1". add_continuous_aggregate_policy --------------------------------- -1 SELECT add_continuous_aggregate_policy('mat_m1', NULL, NULL, '1h'::interval, if_not_exists=>true); -- different values, so we get a WARNING ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', NULL, 20, '1h'::interval, if_not_exists=>true); add_continuous_aggregate_policy --------------------------------- 1011 SELECT add_continuous_aggregate_policy('mat_m1', NULL, 20, '1h'::interval, if_not_exists=>true); NOTICE: continuous aggregate refresh policy already exists for "mat_m1", skipping DETAIL: A refresh policy with the same start and end offset already exists for continuous aggregate "mat_m1". add_continuous_aggregate_policy --------------------------------- -1 SELECT add_continuous_aggregate_policy('mat_m1', NULL, NULL, '1h'::interval, if_not_exists=>true); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ --this one will fail SELECT remove_continuous_aggregate_policy('mat_m1'); ERROR: continuous aggregate policy not found for "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1', if_not_exists=>true); NOTICE: continuous aggregate policy not found for "mat_m1", skipping remove_continuous_aggregate_policy ------------------------------------ --now try to add a policy as a different user than the one that created the cagg --should fail SET ROLE :ROLE_DEFAULT_PERM_USER_2; SELECT add_continuous_aggregate_policy('mat_m1', 20, 10, '1h'::interval) as job_id ; ERROR: must be owner of continuous aggregate "mat_m1" \set VERBOSITY terse \set ON_ERROR_STOP 1 SET ROLE :ROLE_DEFAULT_PERM_USER; DROP MATERIALIZED VIEW mat_m1; --- code coverage tests : add policy for timestamp and date based table --- CREATE TABLE continuous_agg_max_mat_date(time DATE); SELECT create_hypertable('continuous_agg_max_mat_date', 'time'); create_hypertable ------------------------------------------ (5,public,continuous_agg_max_mat_date,t) CREATE MATERIALIZED VIEW max_mat_view_date WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('7 days', time) FROM continuous_agg_max_mat_date GROUP BY 1 WITH NO DATA; \set ON_ERROR_STOP 0 \set VERBOSITY default -- Test 1 step policy for timestamp type buckets ALTER materialized view max_mat_view_date set (timescaledb.compress = true); NOTICE: defaulting compress_orderby to time_bucket -- Only works for cagg SELECT timescaledb_experimental.add_policies('continuous_agg_max_mat_date', refresh_start_offset => '1 day'::interval, refresh_end_offset => '2 day'::interval, compress_after => '20 days'::interval, drop_after => '25 days'::interval); ERROR: "continuous_agg_max_mat_date" is not a continuous aggregate SELECT timescaledb_experimental.show_policies('continuous_agg_max_mat_date'); ERROR: "continuous_agg_max_mat_date" is not a continuous aggregate SELECT timescaledb_experimental.alter_policies('continuous_agg_max_mat_date', compress_after=>'16 days'::interval); ERROR: "continuous_agg_max_mat_date" is not a continuous aggregate SELECT timescaledb_experimental.remove_policies('continuous_agg_max_mat_date', false, 'policy_refresh_continuous_aggregate'); ERROR: "continuous_agg_max_mat_date" is not a continuous aggregate -- No policy is added if one errors out SELECT timescaledb_experimental.add_policies('max_mat_view_date', refresh_start_offset => '1 day'::interval, refresh_end_offset => '2 day'::interval, compress_after => '20 days'::interval, drop_after => '25 days'::interval); ERROR: policy refresh window too small DETAIL: The start and end offsets must cover at least two buckets in the valid time range of type "date". SELECT timescaledb_experimental.show_policies('max_mat_view_date'); show_policies --------------- -- Create open ended refresh_policy SELECT timescaledb_experimental.add_policies('max_mat_view_date', refresh_end_offset => '2 day'::interval); add_policies -------------- t SELECT timescaledb_experimental.show_policies('max_mat_view_date'); show_policies -------------------------------------------------------------------------------------------------------------------------------------------------------- {"policy_name": "policy_refresh_continuous_aggregate", "refresh_interval": "@ 1 hour", "refresh_end_offset": "@ 2 days", "refresh_start_offset": null} SELECT timescaledb_experimental.remove_policies('max_mat_view_date', false, 'policy_refresh_continuous_aggregate'); remove_policies ----------------- t SELECT timescaledb_experimental.add_policies('max_mat_view_date', refresh_end_offset => '2 day'::interval, refresh_start_offset=>'-infinity'); add_policies -------------- t SELECT timescaledb_experimental.show_policies('max_mat_view_date'); show_policies -------------------------------------------------------------------------------------------------------------------------------------------------------- {"policy_name": "policy_refresh_continuous_aggregate", "refresh_interval": "@ 1 hour", "refresh_end_offset": "@ 2 days", "refresh_start_offset": null} SELECT timescaledb_experimental.remove_policies('max_mat_view_date', false, 'policy_refresh_continuous_aggregate'); remove_policies ----------------- t SELECT timescaledb_experimental.add_policies('max_mat_view_date', refresh_start_offset => '2 day'::interval); add_policies -------------- t SELECT timescaledb_experimental.show_policies('max_mat_view_date'); show_policies -------------------------------------------------------------------------------------------------------------------------------------------------------- {"policy_name": "policy_refresh_continuous_aggregate", "refresh_interval": "@ 1 hour", "refresh_end_offset": null, "refresh_start_offset": "@ 2 days"} SELECT timescaledb_experimental.remove_policies('max_mat_view_date', false, 'policy_refresh_continuous_aggregate'); remove_policies ----------------- t SELECT timescaledb_experimental.add_policies('max_mat_view_date', refresh_start_offset => '2 day'::interval, refresh_end_offset=>'infinity'); add_policies -------------- t SELECT timescaledb_experimental.show_policies('max_mat_view_date'); show_policies -------------------------------------------------------------------------------------------------------------------------------------------------------- {"policy_name": "policy_refresh_continuous_aggregate", "refresh_interval": "@ 1 hour", "refresh_end_offset": null, "refresh_start_offset": "@ 2 days"} SELECT timescaledb_experimental.remove_policies('max_mat_view_date', false, 'policy_refresh_continuous_aggregate'); remove_policies ----------------- t -- Open ended at both sides, for code coverage SELECT timescaledb_experimental.add_policies('max_mat_view_date', refresh_end_offset => 'infinity', refresh_start_offset => '-infinity'); add_policies -------------- t SELECT timescaledb_experimental.show_policies('max_mat_view_date'); show_policies -------------------------------------------------------------------------------------------------------------------------------------------------- {"policy_name": "policy_refresh_continuous_aggregate", "refresh_interval": "@ 1 hour", "refresh_end_offset": null, "refresh_start_offset": null} SELECT timescaledb_experimental.remove_policies('max_mat_view_date', false, 'policy_refresh_continuous_aggregate'); remove_policies ----------------- t -- All policies are added in one step SELECT timescaledb_experimental.add_policies('max_mat_view_date', refresh_start_offset => '15 days'::interval, refresh_end_offset => '1 day'::interval, compress_after => '20 days'::interval, drop_after => '25 days'::interval); add_policies -------------- t SELECT timescaledb_experimental.show_policies('max_mat_view_date'); show_policies -------------------------------------------------------------------------------------------------------------------------------------------------------------- {"policy_name": "policy_compression", "compress_after": "@ 20 days", "compress_interval": "@ 12 hours"} {"policy_name": "policy_refresh_continuous_aggregate", "refresh_interval": "@ 1 hour", "refresh_end_offset": "@ 1 day", "refresh_start_offset": "@ 15 days"} {"drop_after": "@ 25 days", "policy_name": "policy_retention", "retention_interval": "@ 1 day"} -- Alter policies SELECT timescaledb_experimental.alter_policies('max_mat_view_date', refresh_start_offset => '16 days'::interval, compress_after=>'26 days'::interval, drop_after => '40 days'::interval); alter_policies ---------------- t SELECT timescaledb_experimental.show_policies('max_mat_view_date'); show_policies -------------------------------------------------------------------------------------------------------------------------------------------------------------- {"policy_name": "policy_compression", "compress_after": "@ 26 days", "compress_interval": "@ 12 hours"} {"policy_name": "policy_refresh_continuous_aggregate", "refresh_interval": "@ 1 hour", "refresh_end_offset": "@ 1 day", "refresh_start_offset": "@ 16 days"} {"drop_after": "@ 40 days", "policy_name": "policy_retention", "retention_interval": "@ 1 day"} --Alter refresh_policy to make it open ended SELECT timescaledb_experimental.remove_policies('max_mat_view_date', false, 'policy_retention', 'policy_compression'); remove_policies ----------------- t SELECT timescaledb_experimental.alter_policies('max_mat_view_date', refresh_start_offset =>'-infinity'); alter_policies ---------------- t SELECT timescaledb_experimental.show_policies('max_mat_view_date'); show_policies ------------------------------------------------------------------------------------------------------------------------------------------------------- {"policy_name": "policy_refresh_continuous_aggregate", "refresh_interval": "@ 1 hour", "refresh_end_offset": "@ 1 day", "refresh_start_offset": null} SELECT timescaledb_experimental.alter_policies('max_mat_view_date', refresh_end_offset =>'infinity', refresh_start_offset =>'5 days'::interval); alter_policies ---------------- t SELECT timescaledb_experimental.show_policies('max_mat_view_date'); show_policies -------------------------------------------------------------------------------------------------------------------------------------------------------- {"policy_name": "policy_refresh_continuous_aggregate", "refresh_interval": "@ 1 hour", "refresh_end_offset": null, "refresh_start_offset": "@ 5 days"} --Cross policy checks -- Refresh and compression policies overlap SELECT timescaledb_experimental.add_policies('max_mat_view_date', compress_after => '20 days'::interval, drop_after => '25 days'::interval); add_policies -------------- t SELECT timescaledb_experimental.alter_policies('max_mat_view_date', compress_after=> '4 days'::interval); ERROR: refresh and columnstore policies overlap SELECT timescaledb_experimental.show_policies('max_mat_view_date'); show_policies -------------------------------------------------------------------------------------------------------------------------------------------------------- {"policy_name": "policy_compression", "compress_after": "@ 20 days", "compress_interval": "@ 12 hours"} {"policy_name": "policy_refresh_continuous_aggregate", "refresh_interval": "@ 1 hour", "refresh_end_offset": null, "refresh_start_offset": "@ 5 days"} {"drop_after": "@ 25 days", "policy_name": "policy_retention", "retention_interval": "@ 1 day"} -- Refresh and retention policies overlap SELECT timescaledb_experimental.alter_policies('max_mat_view_date', refresh_start_offset =>'5 days'::interval, drop_after=> '4 days'::interval); ERROR: refresh and retention policies overlap SELECT timescaledb_experimental.show_policies('max_mat_view_date'); show_policies -------------------------------------------------------------------------------------------------------------------------------------------------------- {"policy_name": "policy_compression", "compress_after": "@ 20 days", "compress_interval": "@ 12 hours"} {"policy_name": "policy_refresh_continuous_aggregate", "refresh_interval": "@ 1 hour", "refresh_end_offset": null, "refresh_start_offset": "@ 5 days"} {"drop_after": "@ 25 days", "policy_name": "policy_retention", "retention_interval": "@ 1 day"} --Do not allow refreshed data to be deleted SELECT add_retention_policy('continuous_agg_max_mat_date', '25 days'::interval); add_retention_policy ---------------------- 1027 SELECT timescaledb_experimental.alter_policies('max_mat_view_date', refresh_start_offset =>'25 days'::interval); ERROR: refresh policy of continuous aggregate and retention policy of underlying hypertable overlap SELECT remove_retention_policy('continuous_agg_max_mat_date'); remove_retention_policy ------------------------- -- Remove one or more policy -- Code coverage: no policy names provided SELECT timescaledb_experimental.remove_policies('max_mat_view_date', false); remove_policies ----------------- f -- Code coverage: incorrect name of policy SELECT timescaledb_experimental.remove_policies('max_mat_view_date', false, 'refresh_policy'); NOTICE: No relevant policy found remove_policies ----------------- f SELECT timescaledb_experimental.remove_policies('max_mat_view_date', false, 'policy_refresh_continuous_aggregate', 'policy_compression'); remove_policies ----------------- t SELECT timescaledb_experimental.show_policies('max_mat_view_date'); show_policies ------------------------------------------------------------------------------------------------- {"drop_after": "@ 25 days", "policy_name": "policy_retention", "retention_interval": "@ 1 day"} -- Add one policy SELECT timescaledb_experimental.add_policies('max_mat_view_date', refresh_start_offset => '15 day'::interval, refresh_end_offset => '1 day'::interval); add_policies -------------- t SELECT timescaledb_experimental.show_policies('max_mat_view_date'); show_policies -------------------------------------------------------------------------------------------------------------------------------------------------------------- {"policy_name": "policy_refresh_continuous_aggregate", "refresh_interval": "@ 1 hour", "refresh_end_offset": "@ 1 day", "refresh_start_offset": "@ 15 days"} {"drop_after": "@ 25 days", "policy_name": "policy_retention", "retention_interval": "@ 1 day"} -- Remove all policies SELECT * FROM timescaledb_experimental.policies ORDER BY relation_name, proc_name; relation_name | relation_schema | schedule_interval | proc_schema | proc_name | config | hypertable_schema | hypertable_name -------------------+-----------------+-------------------+------------------------+-------------------------------------+--------------------------------------------------------------------------------+-----------------------+---------------------------- max_mat_view_date | public | @ 1 hour | _timescaledb_functions | policy_refresh_continuous_aggregate | {"end_offset": "@ 1 day", "start_offset": "@ 15 days", "mat_hypertable_id": 6} | _timescaledb_internal | _materialized_hypertable_6 max_mat_view_date | public | @ 1 day | _timescaledb_functions | policy_retention | {"drop_after": "@ 25 days", "hypertable_id": 6} | _timescaledb_internal | _materialized_hypertable_6 SELECT timescaledb_experimental.remove_all_policies(NULL); -- should fail remove_all_policies --------------------- f SELECT timescaledb_experimental.remove_all_policies('continuous_agg_max_mat_date'); -- should fail ERROR: "continuous_agg_max_mat_date" is not a continuous aggregate SELECT timescaledb_experimental.remove_all_policies('max_mat_view_date', false); remove_all_policies --------------------- t SELECT timescaledb_experimental.remove_all_policies('max_mat_view_date', false); -- should fail remove_all_policies --------------------- f CREATE OR REPLACE FUNCTION custom_func(jobid int, args jsonb) RETURNS RECORD LANGUAGE SQL AS $$ VALUES($1, $2, 'custom_func'); $$; -- inject custom job SELECT add_job('custom_func','1h', config:='{"type":"function"}'::jsonb, initial_start => '2000-01-01 00:00:00+00'::timestamptz) AS job_id \gset SELECT _timescaledb_functions.alter_job_set_hypertable_id( :job_id, 'max_mat_view_date'::regclass); alter_job_set_hypertable_id ----------------------------- 1029 SELECT * FROM timescaledb_information.jobs WHERE job_id != 1 ORDER BY 1; job_id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | fixed_schedule | config | next_start | initial_start | hypertable_schema | hypertable_name | check_schema | check_name --------+----------------------------+-------------------+-------------+-------------+--------------+-------------+-------------+-------------------+-----------+----------------+----------------------+------------------------------+------------------------------+-------------------+-------------------+--------------+------------ 1029 | User-Defined Action [1029] | @ 1 hour | @ 0 | -1 | @ 5 mins | public | custom_func | default_perm_user | t | t | {"type": "function"} | Fri Dec 31 16:00:00 1999 PST | Fri Dec 31 16:00:00 1999 PST | public | max_mat_view_date | | SELECT timescaledb_experimental.remove_all_policies('max_mat_view_date', true); -- ignore custom job NOTICE: Ignoring custom job remove_all_policies --------------------- t SELECT delete_job(:job_id); delete_job ------------ DROP FUNCTION custom_func; SELECT timescaledb_experimental.show_policies('max_mat_view_date'); show_policies --------------- ALTER materialized view max_mat_view_date set (timescaledb.compress = false); SELECT add_continuous_aggregate_policy('max_mat_view_date', '2 days', 10, '1 day'::interval); ERROR: invalid parameter value for end_offset HINT: Use time interval with a continuous aggregate using timestamp-based time bucket. --start_interval < end_interval SELECT add_continuous_aggregate_policy('max_mat_view_date', '1 day'::interval, '2 days'::interval , '1 day'::interval) ; ERROR: policy refresh window too small DETAIL: The start and end offsets must cover at least two buckets in the valid time range of type "date". --interval less than two buckets SELECT add_continuous_aggregate_policy('max_mat_view_date', '7 days', '1 day', '1 day'::interval); ERROR: policy refresh window too small DETAIL: The start and end offsets must cover at least two buckets in the valid time range of type "date". SELECT add_continuous_aggregate_policy('max_mat_view_date', '14 days', '1 day', '1 day'::interval); ERROR: policy refresh window too small DETAIL: The start and end offsets must cover at least two buckets in the valid time range of type "date". SELECT add_continuous_aggregate_policy('max_mat_view_date', '13 days', '-10 hours', '1 day'::interval); ERROR: policy refresh window too small DETAIL: The start and end offsets must cover at least two buckets in the valid time range of type "date". \set VERBOSITY terse \set ON_ERROR_STOP 1 -- Negative start offset gives two bucket window: SELECT add_continuous_aggregate_policy('max_mat_view_date', '13 days', '-1 day', '1 day'::interval); add_continuous_aggregate_policy --------------------------------- 1030 SELECT remove_continuous_aggregate_policy('max_mat_view_date'); remove_continuous_aggregate_policy ------------------------------------ -- Both offsets NULL: SELECT add_continuous_aggregate_policy('max_mat_view_date', NULL, NULL, '1 day'::interval); add_continuous_aggregate_policy --------------------------------- 1031 SELECT remove_continuous_aggregate_policy('max_mat_view_date'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('max_mat_view_date', '15 days', '1 day', '1 day'::interval) as job_id \gset SELECT config FROM _timescaledb_catalog.bgw_job WHERE id = :job_id; config -------------------------------------------------------------------------------- {"end_offset": "@ 1 day", "start_offset": "@ 15 days", "mat_hypertable_id": 6} INSERT INTO continuous_agg_max_mat_date SELECT generate_series('2019-09-01'::date, '2019-09-10'::date, '1 day'); --- to prevent NOTICES set message level to warning SET client_min_messages TO warning; CALL run_job(:job_id); RESET client_min_messages; DROP MATERIALIZED VIEW max_mat_view_date; CREATE TABLE continuous_agg_timestamp(time TIMESTAMP); SELECT create_hypertable('continuous_agg_timestamp', 'time'); WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable --------------------------------------- (8,public,continuous_agg_timestamp,t) CREATE MATERIALIZED VIEW max_mat_view_timestamp WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('7 days', time) FROM continuous_agg_timestamp GROUP BY 1 WITH NO DATA; --the start offset overflows the smallest time value, but is capped at --the min value SELECT add_continuous_aggregate_policy('max_mat_view_timestamp', '1000000 years', '1 day' , '1 h'::interval); add_continuous_aggregate_policy --------------------------------- 1033 SELECT remove_continuous_aggregate_policy('max_mat_view_timestamp'); remove_continuous_aggregate_policy ------------------------------------ \set ON_ERROR_STOP 0 \set VERBOSITY default --start and end offset capped at the lowest time value, which means --zero size window SELECT add_continuous_aggregate_policy('max_mat_view_timestamp', '1000000 years', '900000 years' , '1 h'::interval); ERROR: policy refresh window too small DETAIL: The start and end offsets must cover at least two buckets in the valid time range of type "timestamp without time zone". SELECT add_continuous_aggregate_policy('max_mat_view_timestamp', '301 days', '10 months' , '1 h'::interval); ERROR: policy refresh window too small DETAIL: The start and end offsets must cover at least two buckets in the valid time range of type "timestamp without time zone". \set VERBOSITY terse \set ON_ERROR_STOP 1 SELECT add_continuous_aggregate_policy('max_mat_view_timestamp', '15 days', '1 h'::interval , '1 h'::interval) as job_id \gset --- to prevent NOTICES set message level to warning SET client_min_messages TO warning; CALL run_job(:job_id); RESET client_min_messages ; SELECT config FROM _timescaledb_catalog.bgw_job WHERE id = :job_id; config --------------------------------------------------------------------------------- {"end_offset": "@ 1 hour", "start_offset": "@ 15 days", "mat_hypertable_id": 9} \c :TEST_DBNAME :ROLE_SUPERUSER SET timezone TO PST8PDT; UPDATE _timescaledb_catalog.bgw_job SET config = jsonb_build_object('mat_hypertable_id', :mat_id) WHERE id = :job_id; SET ROLE :ROLE_DEFAULT_PERM_USER; SELECT config FROM _timescaledb_catalog.bgw_job where id = :job_id; config -------------------------- {"mat_hypertable_id": 2} \set ON_ERROR_STOP 0 SELECT add_continuous_aggregate_policy('max_mat_view_timestamp', '15 day', '1 day', '1h'::interval, if_not_exists=>true); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "max_mat_view_timestamp" SELECT add_continuous_aggregate_policy('max_mat_view_timestamp', 'xyz', '1 day', '1h'::interval, if_not_exists=>true); ERROR: invalid input syntax for type interval: "xyz" \set ON_ERROR_STOP 1 DROP MATERIALIZED VIEW max_mat_view_timestamp; --smallint table CREATE TABLE smallint_tab (a smallint); SELECT table_name FROM create_hypertable('smallint_tab', 'a', chunk_time_interval=> 10); table_name -------------- smallint_tab CREATE OR REPLACE FUNCTION integer_now_smallint_tab() returns smallint LANGUAGE SQL STABLE as $$ SELECT coalesce(max(a)::smallint, 0::smallint) FROM smallint_tab ; $$; SELECT set_integer_now_func('smallint_tab', 'integer_now_smallint_tab'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW mat_smallint( a, countb ) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as SELECT time_bucket( SMALLINT '1', a) , count(*) FROM smallint_tab GROUP BY 1 WITH NO DATA; \set ON_ERROR_STOP 0 \set VERBOSITY default -- Test 1 step policy for smallint type buckets ALTER materialized view mat_smallint set (timescaledb.compress = true); NOTICE: defaulting compress_orderby to a -- All policies are added in one step SELECT timescaledb_experimental.add_policies('mat_smallint', refresh_start_offset => 10::smallint, refresh_end_offset => 1::smallint, compress_after => 11::smallint, drop_after => 20::smallint); add_policies -------------- t SELECT timescaledb_experimental.show_policies('mat_smallint'); show_policies --------------------------------------------------------------------------------------------------------------------------------------------- {"policy_name": "policy_compression", "compress_after": 11, "compress_interval": "@ 1 day"} {"policy_name": "policy_refresh_continuous_aggregate", "refresh_interval": "@ 1 hour", "refresh_end_offset": 1, "refresh_start_offset": 10} {"drop_after": 20, "policy_name": "policy_retention", "retention_interval": "@ 1 day"} -- Alter policies SELECT timescaledb_experimental.alter_policies('mat_smallint', refresh_start_offset => 11::smallint, compress_after=>13::smallint, drop_after => 25::smallint); alter_policies ---------------- t SELECT timescaledb_experimental.show_policies('mat_smallint'); show_policies --------------------------------------------------------------------------------------------------------------------------------------------- {"policy_name": "policy_compression", "compress_after": 13, "compress_interval": "@ 1 day"} {"policy_name": "policy_refresh_continuous_aggregate", "refresh_interval": "@ 1 hour", "refresh_end_offset": 1, "refresh_start_offset": 11} {"drop_after": 25, "policy_name": "policy_retention", "retention_interval": "@ 1 day"} SELECT timescaledb_experimental.remove_all_policies('mat_smallint', false); remove_all_policies --------------------- t ALTER materialized view mat_smallint set (timescaledb.compress = false); SELECT add_continuous_aggregate_policy('mat_smallint', 15, 0 , '1 h'::interval); ERROR: invalid parameter value for start_offset HINT: Use time interval of type smallint with the continuous aggregate. SELECT add_continuous_aggregate_policy('mat_smallint', 98898::smallint , 0::smallint, '1 h'::interval); ERROR: smallint out of range SELECT add_continuous_aggregate_policy('mat_smallint', 5::smallint, 10::smallint , '1 h'::interval) as job_id \gset ERROR: policy refresh window too small DETAIL: The start and end offsets must cover at least two buckets in the valid time range of type "smallint". \set VERBOSITY terse \set ON_ERROR_STOP 1 SELECT add_continuous_aggregate_policy('mat_smallint', 15::smallint, 0::smallint , '1 h'::interval) as job_id \gset INSERT INTO smallint_tab VALUES(5); INSERT INTO smallint_tab VALUES(10); INSERT INTO smallint_tab VALUES(20); CALL run_job(:job_id); SELECT * FROM mat_smallint ORDER BY 1; a | countb ----+-------- 5 | 1 10 | 1 --remove all the data-- TRUNCATE table smallint_tab; CALL refresh_continuous_aggregate('mat_smallint', NULL, NULL); SELECT * FROM mat_smallint ORDER BY 1; a | countb ---+-------- -- Case 1: overflow by subtracting from PG_INT16_MIN --overflow start_interval, end_interval [-32768, -32768) SELECT remove_continuous_aggregate_policy('mat_smallint'); remove_continuous_aggregate_policy ------------------------------------ INSERT INTO smallint_tab VALUES( -32768 ); SELECT integer_now_smallint_tab(); integer_now_smallint_tab -------------------------- -32768 SELECT add_continuous_aggregate_policy('mat_smallint', 10::smallint, 5::smallint , '1 h'::interval) as job_id \gset \set ON_ERROR_STOP 0 CALL run_job(:job_id); ERROR: invalid refresh window \set ON_ERROR_STOP 1 SELECT * FROM mat_smallint ORDER BY 1; a | countb ---+-------- -- overflow start_interval. now this runs as range is capped [-32768, -32765) INSERT INTO smallint_tab VALUES( -32760 ); SELECT maxval, maxval - 10, maxval -5 FROM integer_now_smallint_tab() as maxval; maxval | ?column? | ?column? --------+----------+---------- -32760 | -32770 | -32765 CALL run_job(:job_id); SELECT * FROM mat_smallint ORDER BY 1; a | countb --------+-------- -32768 | 1 --remove all the data-- TRUNCATE table smallint_tab; CALL refresh_continuous_aggregate('mat_smallint', NULL, NULL); SELECT * FROM mat_smallint ORDER BY 1; a | countb ---+-------- -- Case 2: overflow by subtracting from PG_INT16_MAX --overflow start and end . will fail as range is [32767, 32767] SELECT remove_continuous_aggregate_policy('mat_smallint'); remove_continuous_aggregate_policy ------------------------------------ INSERT INTO smallint_tab VALUES( 32766 ); INSERT INTO smallint_tab VALUES( 32767 ); SELECT maxval, maxval - (-1), maxval - (-2) FROM integer_now_smallint_tab() as maxval; maxval | ?column? | ?column? --------+----------+---------- 32767 | 32768 | 32769 SELECT add_continuous_aggregate_policy('mat_smallint', -1::smallint, -3::smallint , '1 h'::interval) as job_id \gset \set ON_ERROR_STOP 0 CALL run_job(:job_id); ERROR: invalid refresh window \set ON_ERROR_STOP 1 SELECT * FROM mat_smallint ORDER BY 1; a | countb ---+-------- SELECT remove_continuous_aggregate_policy('mat_smallint'); remove_continuous_aggregate_policy ------------------------------------ --overflow end . will work range is [32765, 32767) SELECT maxval, maxval - (1), maxval - (-2) FROM integer_now_smallint_tab() as maxval; maxval | ?column? | ?column? --------+----------+---------- 32767 | 32766 | 32769 SELECT add_continuous_aggregate_policy('mat_smallint', 1::smallint, -3::smallint , '1 h'::interval) as job_id \gset \set ON_ERROR_STOP 0 CALL run_job(:job_id); SELECT * FROM mat_smallint ORDER BY 1; a | countb -------+-------- 32766 | 1 -- tests for interval argument conversions -- \set ON_ERROR_STOP 0 SELECT add_continuous_aggregate_policy('mat_smallint', 15, 10, '1h'::interval, if_not_exists=>true); ERROR: invalid parameter value for start_offset SELECT add_continuous_aggregate_policy('mat_smallint', '15', 10, '1h'::interval, if_not_exists=>true); ERROR: invalid parameter value for end_offset SELECT add_continuous_aggregate_policy('mat_smallint', '15', '10', '1h'::interval, if_not_exists=>true); add_continuous_aggregate_policy --------------------------------- 1045 \set ON_ERROR_STOP 1 --bigint table CREATE TABLE bigint_tab (a bigint); SELECT table_name FROM create_hypertable('bigint_tab', 'a', chunk_time_interval=> 10); table_name ------------ bigint_tab CREATE OR REPLACE FUNCTION integer_now_bigint_tab() returns bigint LANGUAGE SQL STABLE as $$ SELECT 20::bigint $$; SELECT set_integer_now_func('bigint_tab', 'integer_now_bigint_tab'); set_integer_now_func ---------------------- CREATE MATERIALIZED VIEW mat_bigint( a, countb ) WITH (timescaledb.continuous, timescaledb.materialized_only=true) as SELECT time_bucket( BIGINT '1', a) , count(*) FROM bigint_tab GROUP BY 1 WITH NO DATA; -- Test 1 step policy for bigint type buckets ALTER materialized view mat_bigint set (timescaledb.compress = true); NOTICE: defaulting compress_orderby to a -- All policies are added in one step SELECT timescaledb_experimental.add_policies('mat_bigint', refresh_start_offset => 10::bigint, refresh_end_offset => 1::bigint, compress_after => 11::bigint, drop_after => 20::bigint); add_policies -------------- t SELECT timescaledb_experimental.show_policies('mat_bigint'); show_policies --------------------------------------------------------------------------------------------------------------------------------------------- {"policy_name": "policy_compression", "compress_after": 11, "compress_interval": "@ 1 day"} {"policy_name": "policy_refresh_continuous_aggregate", "refresh_interval": "@ 1 hour", "refresh_end_offset": 1, "refresh_start_offset": 10} {"drop_after": 20, "policy_name": "policy_retention", "retention_interval": "@ 1 day"} -- Alter policies SELECT timescaledb_experimental.alter_policies('mat_bigint', refresh_start_offset => 11::bigint, compress_after=>13::bigint, drop_after => 25::bigint); alter_policies ---------------- t SELECT timescaledb_experimental.show_policies('mat_bigint'); show_policies --------------------------------------------------------------------------------------------------------------------------------------------- {"policy_name": "policy_compression", "compress_after": 13, "compress_interval": "@ 1 day"} {"policy_name": "policy_refresh_continuous_aggregate", "refresh_interval": "@ 1 hour", "refresh_end_offset": 1, "refresh_start_offset": 11} {"drop_after": 25, "policy_name": "policy_retention", "retention_interval": "@ 1 day"} SELECT timescaledb_experimental.remove_all_policies('mat_bigint', false); remove_all_policies --------------------- t ALTER materialized view mat_bigint set (timescaledb.compress = false); \set ON_ERROR_STOP 0 SELECT add_continuous_aggregate_policy('mat_bigint', 5::bigint, 10::bigint , '1 h'::interval) ; ERROR: policy refresh window too small \set ON_ERROR_STOP 1 SELECT add_continuous_aggregate_policy('mat_bigint', 15::bigint, 0::bigint , '1 h'::interval) as job_mid \gset INSERT INTO bigint_tab VALUES(5); INSERT INTO bigint_tab VALUES(10); INSERT INTO bigint_tab VALUES(20); CALL run_job(:job_mid); SELECT * FROM mat_bigint ORDER BY 1; a | countb ----+-------- 5 | 1 10 | 1 -- test NULL for end SELECT remove_continuous_aggregate_policy('mat_bigint'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_bigint', 1::smallint, NULL , '1 h'::interval) as job_id \gset INSERT INTO bigint_tab VALUES(500); CALL run_job(:job_id); SELECT * FROM mat_bigint WHERE a>100 ORDER BY 1; a | countb -----+-------- 500 | 1 ALTER MATERIALIZED VIEW mat_bigint SET (timescaledb.compress); NOTICE: defaulting compress_orderby to a ALTER MATERIALIZED VIEW mat_smallint SET (timescaledb.compress); NOTICE: defaulting compress_orderby to a -- With immutable compressed chunks, these policies would fail by overlapping the refresh window SELECT add_compression_policy('mat_smallint', -4::smallint); add_compression_policy ------------------------ 1054 SELECT remove_compression_policy('mat_smallint'); remove_compression_policy --------------------------- t SELECT add_compression_policy('mat_bigint', 0::bigint); add_compression_policy ------------------------ 1055 SELECT remove_compression_policy('mat_bigint'); remove_compression_policy --------------------------- t -- End previous limitation tests SELECT add_compression_policy('mat_smallint', 5::smallint); add_compression_policy ------------------------ 1056 SELECT add_compression_policy('mat_bigint', 20::bigint); add_compression_policy ------------------------ 1057 -- end of coverage tests --TEST continuous aggregate + compression policy on caggs CREATE TABLE metrics ( time timestamptz NOT NULL, device_id int, device_id_peer int, v0 int, v1 int, v2 float, v3 float ); SELECT create_hypertable('metrics', 'time'); create_hypertable ----------------------- (18,public,metrics,t) INSERT INTO metrics (time, device_id, device_id_peer, v0, v1, v2, v3) SELECT time, device_id, 0, device_id + 1, device_id + 2, 0.5, NULL FROM generate_series('2000-01-01 0:00:00+0'::timestamptz, '2000-01-02 23:55:00+0', '20m') gtime (time), generate_series(1, 2, 1) gdevice (device_id); ALTER TABLE metrics SET ( timescaledb.compress ); SELECT compress_chunk(ch) FROM show_chunks('metrics') ch; compress_chunk ------------------------------------------ _timescaledb_internal._hyper_18_19_chunk CREATE MATERIALIZED VIEW metrics_cagg WITH (timescaledb.continuous, timescaledb.materialized_only = true) AS SELECT time_bucket('1 day', time) as dayb, device_id, sum(v0), avg(v3) FROM metrics GROUP BY 1, 2 WITH NO DATA; -- this was previously crashing SELECT add_continuous_aggregate_policy('metrics_cagg', '7 day'::interval, NULL, '1 h'::interval, if_not_exists => true); add_continuous_aggregate_policy --------------------------------- 1058 \set ON_ERROR_STOP 0 SELECT add_continuous_aggregate_policy('metrics_cagg', '7 day'::interval, '1 day'::interval, '1 h'::interval, if_not_exists => true); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "metrics_cagg" SELECT remove_continuous_aggregate_policy('metrics_cagg'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('metrics_cagg', NULL, '1 day'::interval, '1h'::interval, if_not_exists=>true); add_continuous_aggregate_policy --------------------------------- 1059 SELECT add_continuous_aggregate_policy('metrics_cagg', NULL, '1 day'::interval, '1h'::interval, if_not_exists=>true); -- same param values, so we get a NOTICE NOTICE: continuous aggregate refresh policy already exists for "metrics_cagg", skipping add_continuous_aggregate_policy --------------------------------- -1 SELECT add_continuous_aggregate_policy('metrics_cagg', NULL, NULL, '1h'::interval, if_not_exists=>true); -- different values, so we get a WARNING ERROR: refresh interval overlaps with an existing continuous aggregate policy on "metrics_cagg" SELECT remove_continuous_aggregate_policy('metrics_cagg'); remove_continuous_aggregate_policy ------------------------------------ --can set compression policy only after setting up refresh policy -- SELECT add_compression_policy('metrics_cagg', '1 day'::interval); ERROR: setup a refresh policy for "metrics_cagg" before setting up a columnstore policy --can set compression policy only after enabling compression -- SELECT add_continuous_aggregate_policy('metrics_cagg', '7 day'::interval, '1 day'::interval, '1 h'::interval) as "REFRESH_JOB" \gset SELECT add_compression_policy('metrics_cagg', '8 day'::interval) AS "COMP_JOB" ; ERROR: columnstore not enabled on continuous aggregate "metrics_cagg" ALTER MATERIALIZED VIEW metrics_cagg SET (timescaledb.compress); NOTICE: defaulting compress_orderby to dayb,device_id --cannot use compress_created_before with cagg SELECT add_compression_policy('metrics_cagg', compress_created_before => '8 day'::interval) AS "COMP_JOB" ; ERROR: cannot use "compress_created_before" with continuous aggregate "metrics_cagg" \set ON_ERROR_STOP 1 SELECT add_compression_policy('metrics_cagg', '8 day'::interval) AS "COMP_JOB" ; COMP_JOB ---------- 1061 SELECT remove_compression_policy('metrics_cagg'); remove_compression_policy --------------------------- t SELECT add_compression_policy('metrics_cagg', '8 day'::interval) AS "COMP_JOB" \gset --verify that jobs were added for the policies --- SELECT materialization_hypertable_name AS "MAT_TABLE_NAME", view_name AS "VIEW_NAME" FROM timescaledb_information.continuous_aggregates WHERE view_name = 'metrics_cagg' \gset SELECT count(*) FROM timescaledb_information.jobs WHERE hypertable_name = :'VIEW_NAME'; count ------- 2 --exec the cagg compression job -- CALL refresh_continuous_aggregate('metrics_cagg', NULL, '2001-02-01 00:00:00+0'); CALL run_job(:COMP_JOB); SELECT count(*), count(*) FILTER ( WHERE is_compressed is TRUE ) FROM timescaledb_information.chunks WHERE hypertable_name = :'MAT_TABLE_NAME' ORDER BY 1; count | count -------+------- 1 | 1 --add some new data into metrics_cagg so that cagg policy job has something to do INSERT INTO metrics (time, device_id, device_id_peer, v0, v1, v2, v3) SELECT now() - '5 day'::interval, 102, 0, 10, 10, 10, 10; CALL run_job(:REFRESH_JOB); --now we have a new chunk and it is not compressed SELECT count(*), count(*) FILTER ( WHERE is_compressed is TRUE ) FROM timescaledb_information.chunks WHERE hypertable_name = :'MAT_TABLE_NAME' ORDER BY 1; count | count -------+------- 2 | 1 --verify that both jobs are dropped when view is dropped DROP MATERIALIZED VIEW metrics_cagg; NOTICE: drop cascades to 2 other objects SELECT count(*) FROM timescaledb_information.jobs WHERE hypertable_name = :'VIEW_NAME'; count ------- 0 -- add test case for issue 4252 CREATE TABLE IF NOT EXISTS sensor_data( time TIMESTAMPTZ NOT NULL, sensor_id INTEGER, temperature DOUBLE PRECISION, cpu DOUBLE PRECISION); SELECT create_hypertable('sensor_data','time'); create_hypertable --------------------------- (22,public,sensor_data,t) INSERT INTO sensor_data(time, sensor_id, cpu, temperature) SELECT time, sensor_id, extract(dow from time) AS cpu, extract(doy from time) AS temperature FROM generate_series('2022-05-05'::timestamp at time zone 'UTC' - interval '6 weeks', '2022-05-05'::timestamp at time zone 'UTC', interval '5 hours') as g1(time), generate_series(1,1000,1) as g2(sensor_id); CREATE materialized view deals_best_weekly WITH (timescaledb.continuous) AS SELECT time_bucket('7 days', "time") AS bucket, avg(temperature) AS avg_temp, max(cpu) AS max_rating FROM sensor_data GROUP BY bucket WITH NO DATA; CREATE materialized view deals_best_daily WITH (timescaledb.continuous) AS SELECT time_bucket('1 day', "time") AS bucket, avg(temperature) AS avg_temp, max(cpu) AS max_rating FROM sensor_data GROUP BY bucket WITH NO DATA; ALTER materialized view deals_best_weekly set (timescaledb.materialized_only=true); ALTER materialized view deals_best_daily set (timescaledb.materialized_only=true); -- we have data from 6 weeks before to May 5 2022 (Thu) CALL refresh_continuous_aggregate('deals_best_weekly', '2022-04-24', '2022-05-03'); SELECT * FROM deals_best_weekly ORDER BY bucket; bucket | avg_temp | max_rating ------------------------------+------------------+------------ Sun Apr 24 17:00:00 2022 PDT | 117.764705882353 | 6 CALL refresh_continuous_aggregate('deals_best_daily', '2022-04-20', '2022-05-04'); SELECT * FROM deals_best_daily ORDER BY bucket LIMIT 2; bucket | avg_temp | max_rating ------------------------------+----------+------------ Wed Apr 20 17:00:00 2022 PDT | 110.8 | 4 Thu Apr 21 17:00:00 2022 PDT | 111.75 | 5 -- expect to get an up-to-date notice CALL refresh_continuous_aggregate('deals_best_weekly', '2022-04-24', '2022-05-05'); NOTICE: continuous aggregate "deals_best_weekly" is already up-to-date SELECT * FROM deals_best_weekly ORDER BY bucket; bucket | avg_temp | max_rating ------------------------------+------------------+------------ Sun Apr 24 17:00:00 2022 PDT | 117.764705882353 | 6 -- github issue 5907: segfault when creating 1-step policies on cagg -- whose underlying hypertable has a retention policy setup CREATE TABLE t(a integer NOT NULL, b integer); SELECT create_hypertable('t', 'a', chunk_time_interval=> 10); create_hypertable ------------------- (25,public,t,t) CREATE OR REPLACE FUNCTION unix_now() returns int LANGUAGE SQL IMMUTABLE as $$ SELECT extract(epoch from now())::INT $$; SELECT set_integer_now_func('t', 'unix_now'); set_integer_now_func ---------------------- SELECT add_retention_policy('t', 20); add_retention_policy ---------------------- 1063 CREATE MATERIALIZED VIEW cagg(a, sumb) WITH (timescaledb.continuous) AS SELECT time_bucket(1, a), sum(b) FROM t GROUP BY time_bucket(1, a); NOTICE: continuous aggregate "cagg" is already up-to-date SELECT timescaledb_experimental.add_policies('cagg'); add_policies -------------- f -- Issue #6902 -- Fix timestamp out of range in a refresh policy when setting `end_offset=>NULL` -- for a CAgg with variable sized bucket (i.e: using `time_bucket` with timezone) CREATE TABLE issue_6902 ( ts TIMESTAMPTZ NOT NULL, temperature NUMERIC ) WITH ( timescaledb.hypertable, timescaledb.partition_column='ts', timescaledb.chunk_interval='1 day', timescaledb.compress='off' ); INSERT INTO issue_6902 SELECT t, 1 FROM generate_series(now() - interval '3 hours', now(), interval '1 minute') AS t; CREATE MATERIALIZED VIEW issue_6902_by_hour WITH (timescaledb.continuous) AS SELECT time_bucket(INTERVAL '1 hour', ts, 'America/Sao_Paulo') AS bucket, -- using timezone MAX(temperature), MIN(temperature), COUNT(*) FROM issue_6902 GROUP BY 1 WITH NO DATA; SELECT add_continuous_aggregate_policy ( 'issue_6902_by_hour', start_offset => INTERVAL '3 hours', end_offset => NULL, schedule_interval => INTERVAL '12 hour', initial_start => now() + INTERVAL '12 hour' ) AS job_id \gset -- 181 rows CALL run_job(:job_id); SELECT count(*) FROM issue_6902; count ------- 181 -- run again without any change, remain 181 rows CALL run_job(:job_id); SELECT count(*) FROM issue_6902; count ------- 181 -- change existing data UPDATE issue_6902 SET temperature = temperature + 1; -- run again without any change, remain 181 rows CALL run_job(:job_id); SELECT count(*) FROM issue_6902; count ------- 181 -- insert more data INSERT INTO issue_6902 SELECT t, 1 FROM generate_series(now() - interval '3 hours', now(), interval '1 minute') AS t; -- run again without and should have 362 rows CALL run_job(:job_id); SELECT count(*) FROM issue_6902; count ------- 362 -- test untyped interval error handling CREATE TABLE m(time timestamptz) WITH (tsdb.hypertable); NOTICE: using column "time" as partitioning column CREATE MATERIALIZED VIEW cagg_error WITH (tsdb.continuous) AS SELECT time_bucket('1 day', time) FROM m GROUP BY 1; NOTICE: continuous aggregate "cagg_error" is already up-to-date SELECT timescaledb_experimental.add_policies('cagg_error', drop_after => '20 days'); ERROR: unsupported interval argument type: unknown ================================================ FILE: tsl/test/expected/cagg_policy_concurrent.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- Test creation of multiple refresh policies SET timezone TO PST8PDT; SET timescaledb.current_timestamp_mock TO '2025-06-01 0:30:00+00'; SELECT setseed(1); setseed --------- -- test interval checking with bigint CREATE TABLE overlap_test_bigint ( time BIGINT NOT NULL, a INTEGER, b INTEGER ); SELECT create_hypertable('overlap_test_bigint', 'time', chunk_time_interval => 100); create_hypertable ---------------------------------- (1,public,overlap_test_bigint,t) CREATE OR REPLACE FUNCTION integer_now_overlap_test_bigint() RETURNS BIGINT LANGUAGE SQL STABLE AS $$ SELECT COALESCE(MAX(time), 0) FROM overlap_test_bigint $$; SELECT set_integer_now_func('overlap_test_bigint', 'integer_now_overlap_test_bigint'); set_integer_now_func ---------------------- INSERT INTO overlap_test_bigint SELECT i, (i % 5), random() * 100 FROM generate_series(1, 2000) i; CREATE MATERIALIZED VIEW mat_m1(time, counta) WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(10, time) AS bucket, count(a), sum(b) FROM overlap_test_bigint GROUP BY 1 WITH NO DATA; /* Test interval checking when multiple policies are created on the same cagg */ SELECT add_continuous_aggregate_policy('mat_m1', NULL, 1000::bigint, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1000 SELECT add_continuous_aggregate_policy('mat_m1', 1000::bigint, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1001 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ /* Creating policies in either order should work */ SELECT add_continuous_aggregate_policy('mat_m1', 1000::bigint, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1002 SELECT add_continuous_aggregate_policy('mat_m1', NULL, 1000::bigint, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1003 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', NULL, 3000::bigint, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1004 SELECT add_continuous_aggregate_policy('mat_m1', 2000::bigint, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1005 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ /* Test non-null offsets on both sides too */ SELECT add_continuous_aggregate_policy('mat_m1', 2000::bigint, 1000::bigint, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1006 SELECT add_continuous_aggregate_policy('mat_m1', 4000::bigint, 3000::bigint,'12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1007 SELECT add_continuous_aggregate_policy('mat_m1', 3000::bigint, 2000::bigint, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1008 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ /* Check overlap is detected correctly */ \set ON_ERROR_STOP 0 SELECT add_continuous_aggregate_policy('mat_m1', NULL, 1000::bigint, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1009 SELECT add_continuous_aggregate_policy('mat_m1', 2000::bigint, NULL, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', 2000::bigint, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1010 SELECT add_continuous_aggregate_policy('mat_m1', NULL, 1000::bigint, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', 5000::bigint, 1000::bigint, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1011 SELECT add_continuous_aggregate_policy('mat_m1', 4000::bigint, 2000::bigint, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', 4000::bigint, 2000::bigint, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1012 SELECT add_continuous_aggregate_policy('mat_m1', 5000::bigint, 1000::bigint, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', NULL, 2000::bigint, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1013 SELECT add_continuous_aggregate_policy('mat_m1', NULL, 1000::bigint, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', 2000::bigint, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1014 SELECT add_continuous_aggregate_policy('mat_m1', 1000::bigint, NULL, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ \set ON_ERROR_STOP 1 /* Check behaviour when exact policy is already defined */ \set ON_ERROR_STOP 0 /*if_not_exists=false*/ SELECT add_continuous_aggregate_policy('mat_m1', 4000::bigint, 2000::bigint, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1015 SELECT add_continuous_aggregate_policy('mat_m1', 2000::bigint, 1000::bigint, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1016 SELECT add_continuous_aggregate_policy('mat_m1', 2000::bigint, 1000::bigint, '12 h'::interval); ERROR: continuous aggregate refresh policy already exists for "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', NULL, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1017 SELECT add_continuous_aggregate_policy('mat_m1', NULL, NULL, '12 h'::interval); ERROR: continuous aggregate refresh policy already exists for "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ \set ON_ERROR_STOP 1 /*if_not_exists => true*/ SELECT add_continuous_aggregate_policy('mat_m1', 4000::bigint, 2000::bigint, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1018 SELECT add_continuous_aggregate_policy('mat_m1', 2000::bigint, 1000::bigint, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1019 SELECT add_continuous_aggregate_policy('mat_m1', 2000::bigint, 1000::bigint, '12 h'::interval, if_not_exists => true); NOTICE: continuous aggregate refresh policy already exists for "mat_m1", skipping add_continuous_aggregate_policy --------------------------------- -1 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', NULL, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1020 SELECT add_continuous_aggregate_policy('mat_m1', NULL, NULL, '12 h'::interval, if_not_exists => true); NOTICE: continuous aggregate refresh policy already exists for "mat_m1", skipping add_continuous_aggregate_policy --------------------------------- -1 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ /* Throw an error if there is an overlap even if if_not_exists => true */ SELECT add_continuous_aggregate_policy('mat_m1', 4000::bigint, 2000::bigint, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1021 \set ON_ERROR_STOP 0 SELECT add_continuous_aggregate_policy('mat_m1', 3000::bigint, 1000::bigint, '12 h'::interval, if_not_exists => true); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" \set ON_ERROR_STOP 1 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ /* Test `alter_job` changing the config */ SELECT add_continuous_aggregate_policy('mat_m1', NULL, 3000::bigint, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1022 SELECT id AS job_id, config AS config FROM _timescaledb_catalog.bgw_job WHERE proc_name = 'policy_refresh_continuous_aggregate' \gset SELECT add_continuous_aggregate_policy('mat_m1', 2000::bigint, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1023 /* Alter end offset but don't overlap */ SELECT jsonb_set(:'config', '{end_offset}', '2000') AS config \gset SELECT * FROM alter_job(:job_id, config := :'config'); job_id | schedule_interval | max_runtime | max_retries | retry_period | scheduled | config | next_start | check_config | fixed_schedule | initial_start | timezone | application_name --------+-------------------+-------------+-------------+--------------+-----------+--------------------------------------------------------------------+------------+------------------------------------------------------------------+----------------+---------------+----------+-------------------------------------------- 1022 | @ 12 hours | @ 0 | -1 | @ 12 hours | t | {"end_offset": 2000, "start_offset": null, "mat_hypertable_id": 2} | -infinity | _timescaledb_functions.policy_refresh_continuous_aggregate_check | f | | | Refresh Continuous Aggregate Policy [1022] \set ON_ERROR_STOP 0 /* Alter end offset to overlap with another job*/ SELECT jsonb_set(:'config', '{end_offset}', '1000') AS config \gset SELECT * FROM alter_job(:job_id, config := :'config'); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" /* Alter end offset to be null */ SELECT jsonb_set(:'config', '{end_offset}', 'null') AS config \gset SELECT * FROM alter_job(:job_id, config := :'config'); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" /* Alter job to be identical to existing job */ SELECT jsonb_set(:'config', '{start_offset}', '2000') AS config \gset SELECT * FROM alter_job(:job_id, config := :'config'); ERROR: continuous aggregate refresh policy already exists for "mat_m1" \set ON_ERROR_STOP 1 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', 2000::bigint, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1024 SELECT id AS job_id, config AS config FROM _timescaledb_catalog.bgw_job WHERE proc_name = 'policy_refresh_continuous_aggregate' \gset SELECT add_continuous_aggregate_policy('mat_m1', NULL, 3000::bigint, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1025 /* Alter end offset to null but no overlap */ SELECT jsonb_set(:'config', '{end_offset}', 'null') AS config \gset SELECT * FROM alter_job(:job_id, config := :'config'); job_id | schedule_interval | max_runtime | max_retries | retry_period | scheduled | config | next_start | check_config | fixed_schedule | initial_start | timezone | application_name --------+-------------------+-------------+-------------+--------------+-----------+--------------------------------------------------------------------+------------+------------------------------------------------------------------+----------------+---------------+----------+-------------------------------------------- 1024 | @ 12 hours | @ 0 | -1 | @ 12 hours | t | {"end_offset": null, "start_offset": 2000, "mat_hypertable_id": 2} | -infinity | _timescaledb_functions.policy_refresh_continuous_aggregate_check | f | | | Refresh Continuous Aggregate Policy [1024] SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ /* Test that refresh is done correctly even though multiple policies exist */ /* We do this by creating two CAggs on the same hypertable */ /* One will have a single policy while the other will have two policies with adjacent offsets */ CREATE MATERIALIZED VIEW mat_m2(time, counta) WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket(10, time) AS bucket, count(a), sum(b) FROM overlap_test_bigint GROUP BY 1 WITH NO DATA; /* Create two policies on mat_m1 */ SELECT add_continuous_aggregate_policy('mat_m1', 5000::bigint, 3000::bigint, '12 h'::interval) AS agg_m1_job_1 \gset SELECT add_continuous_aggregate_policy('mat_m1', 3000::bigint, 1000::bigint, '12 h'::interval) AS agg_m1_job_2 \gset /* Create single policy on mat_m2 */ SELECT add_continuous_aggregate_policy('mat_m2', 5000::bigint, 1000::bigint, '12 h'::interval) AS agg_m2_job \gset /* Cleanup any existing data */ TRUNCATE mat_m1; TRUNCATE mat_m2; /* Refresh both continuous aggs immediately */ CALL run_job(:agg_m1_job_1); CALL run_job(:agg_m1_job_2); CALL run_job(:agg_m2_job); /* Compare both outputs */ SELECT count(*) AS exp_row_count from mat_m1 \gset SELECT count(*) AS actual_row_count from ( SELECT * from mat_m1 UNION SELECT * from mat_m2) union_q \gset /* Row counts should be the same */ SELECT :exp_row_count = :actual_row_count, :exp_row_count, :actual_row_count; ?column? | ?column? | ?column? ----------+----------+---------- t | 100 | 100 SELECT * from mat_m2 EXCEPT SELECT * from mat_m1; time | counta | sum ------+--------+----- SELECT * from mat_m1 EXCEPT SELECT * from mat_m2; time | counta | sum ------+--------+----- DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to table _timescaledb_internal._hyper_2_22_chunk DROP MATERIALIZED VIEW mat_m2; NOTICE: drop cascades to table _timescaledb_internal._hyper_3_23_chunk CREATE TABLE overlap_test_timestamptz ( time timestamptz NOT NULL, a INTEGER, b INTEGER ); SELECT create_hypertable('overlap_test_timestamptz', 'time', chunk_time_interval => '1 day'::interval); create_hypertable --------------------------------------- (4,public,overlap_test_timestamptz,t) INSERT INTO overlap_test_timestamptz SELECT t, (i % 5), random() * 100 FROM generate_series('2025-01-01T01:01:01+00', '2025-06-01T01:01:01+00', INTERVAL '1 days') t, generate_series(1, 10) i; CREATE MATERIALIZED VIEW mat_m1(time, counta) WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1 day', time) AS bucket, count(a), sum(b) FROM overlap_test_timestamptz GROUP BY 1 WITH NO DATA; /* Test interval checking when multiple policies are created on the same cagg */ SELECT add_continuous_aggregate_policy('mat_m1', NULL, '30 days'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1029 SELECT add_continuous_aggregate_policy('mat_m1', '29 days'::interval, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1030 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ /* Creating policies in either order should work */ SELECT add_continuous_aggregate_policy('mat_m1', '30 days'::interval, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1031 SELECT add_continuous_aggregate_policy('mat_m1', NULL, '30 days'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1032 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', NULL, '30 days'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1033 SELECT add_continuous_aggregate_policy('mat_m1', '15 days'::interval, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1034 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ /* Test non-null offsets on both sides too */ SELECT add_continuous_aggregate_policy('mat_m1', '30 days'::interval, '20 days'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1035 SELECT add_continuous_aggregate_policy('mat_m1', '10 days'::interval, '5 days'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1036 SELECT add_continuous_aggregate_policy('mat_m1', '19 days'::interval, '11 days'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1037 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ /* Check overlap is detected correctly */ \set ON_ERROR_STOP 0 SELECT add_continuous_aggregate_policy('mat_m1', NULL, '30 days'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1038 SELECT add_continuous_aggregate_policy('mat_m1', '45 days'::interval, NULL, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', '45 days'::interval, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1039 SELECT add_continuous_aggregate_policy('mat_m1', NULL, '30 days'::interval, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', '30 days'::interval, '10 days'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1040 SELECT add_continuous_aggregate_policy('mat_m1', '20 days'::interval, '15 days'::interval, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', '20 days'::interval, '15 days'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1041 SELECT add_continuous_aggregate_policy('mat_m1', '30 days'::interval, '10 days'::interval, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', NULL, '30 days'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1042 SELECT add_continuous_aggregate_policy('mat_m1', NULL, '20 days'::interval, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', '30 days'::interval, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1043 SELECT add_continuous_aggregate_policy('mat_m1', '20 days'::interval, NULL, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ \set ON_ERROR_STOP 1 /* Check behaviour when exact policy is already defined */ \set ON_ERROR_STOP 0 /*if_not_exists=false*/ SELECT add_continuous_aggregate_policy('mat_m1', '45 days'::interval, '30 days', '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1044 SELECT add_continuous_aggregate_policy('mat_m1', '31 days'::interval, '15 days', '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT add_continuous_aggregate_policy('mat_m1', '31 days'::interval, '15 days', '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', NULL::interval, NULL::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1045 SELECT add_continuous_aggregate_policy('mat_m1', NULL::interval, NULL::interval, '12 h'::interval); ERROR: continuous aggregate refresh policy already exists for "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ \set ON_ERROR_STOP 1 /*if_not_exists => true*/ \set ON_ERROR_STOP 0 SELECT add_continuous_aggregate_policy('mat_m1', '45 days', '30 days', '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1046 SELECT add_continuous_aggregate_policy('mat_m1', '30 days'::interval, '15 days'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1047 SELECT add_continuous_aggregate_policy('mat_m1', '30 days'::interval, '15 days'::interval, '12 h'::interval, if_not_exists => true); NOTICE: continuous aggregate refresh policy already exists for "mat_m1", skipping add_continuous_aggregate_policy --------------------------------- -1 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', NULL::interval, NULL::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1048 SELECT add_continuous_aggregate_policy('mat_m1', NULL::interval, NULL::interval, '12 h'::interval, if_not_exists => true); NOTICE: continuous aggregate refresh policy already exists for "mat_m1", skipping add_continuous_aggregate_policy --------------------------------- -1 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ \set ON_ERROR_STOP 1 /* Throw an error if there is an overlap even if if_not_exists => true */ SELECT add_continuous_aggregate_policy('mat_m1', '30 days'::interval, '10 days'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1049 \set ON_ERROR_STOP 0 SELECT add_continuous_aggregate_policy('mat_m1', '15 days'::interval, NULL, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" \set ON_ERROR_STOP 1 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ /* Mixing different interval units should also work correctly*/ SELECT add_continuous_aggregate_policy('mat_m1', NULL, '1 month'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1050 SELECT add_continuous_aggregate_policy('mat_m1', '2 weeks'::interval, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1051 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', '1 year'::interval, '2 months'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1052 SELECT add_continuous_aggregate_policy('mat_m1', '5 weeks'::interval, '-7 days'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1053 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', '2 weeks'::interval, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1054 SELECT add_continuous_aggregate_policy('mat_m1', NULL, '1 month'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1055 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ \set ON_ERROR_STOP 0 SELECT add_continuous_aggregate_policy('mat_m1', NULL, '2 weeks'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1056 SELECT add_continuous_aggregate_policy('mat_m1', '1 month'::interval, NULL, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ \set ON_ERROR_STOP 1 /* Check overlap with negative offsets */ SELECT add_continuous_aggregate_policy('mat_m1', NULL, '2 weeks'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1057 SELECT add_continuous_aggregate_policy('mat_m1', '-1 month'::interval, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1058 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ \set ON_ERROR_STOP 0 SELECT add_continuous_aggregate_policy('mat_m1', NULL, '-2 weeks'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1059 SELECT add_continuous_aggregate_policy('mat_m1', '1 month'::interval, NULL, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ \set ON_ERROR_STOP 1 /* Test `alter_job` changing the config */ SELECT add_continuous_aggregate_policy('mat_m1', NULL, '2 months'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1060 SELECT id AS job_id, config AS config FROM _timescaledb_catalog.bgw_job WHERE proc_name = 'policy_refresh_continuous_aggregate' \gset SELECT add_continuous_aggregate_policy('mat_m1', '2 weeks'::interval, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1061 /* Alter end offset but don't overlap */ SELECT jsonb_set(:'config', '{end_offset}', '"30 days"') AS config \gset SELECT * FROM alter_job(:job_id, config := :'config'); job_id | schedule_interval | max_runtime | max_retries | retry_period | scheduled | config | next_start | check_config | fixed_schedule | initial_start | timezone | application_name --------+-------------------+-------------+-------------+--------------+-----------+-------------------------------------------------------------------------+------------+------------------------------------------------------------------+----------------+---------------+----------+-------------------------------------------- 1060 | @ 12 hours | @ 0 | -1 | @ 12 hours | t | {"end_offset": "30 days", "start_offset": null, "mat_hypertable_id": 5} | -infinity | _timescaledb_functions.policy_refresh_continuous_aggregate_check | f | | | Refresh Continuous Aggregate Policy [1060] \set ON_ERROR_STOP 0 /* Alter end offset to overlap with another job*/ SELECT jsonb_set(:'config', '{end_offset}', '"1 week"') AS config \gset SELECT * FROM alter_job(:job_id, config := :'config'); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" /* Alter end offset to be null */ SELECT jsonb_set(:'config', '{end_offset}', 'null') AS config \gset SELECT * FROM alter_job(:job_id, config := :'config'); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" /* Alter job to be identical to existing job */ SELECT jsonb_set(:'config', '{start_offset}', '"2 weeks"') AS config \gset SELECT * FROM alter_job(:job_id, config := :'config'); ERROR: continuous aggregate refresh policy already exists for "mat_m1" \set ON_ERROR_STOP 1 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', '2 weeks'::interval, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1062 SELECT id AS job_id, config AS config FROM _timescaledb_catalog.bgw_job WHERE proc_name = 'policy_refresh_continuous_aggregate' \gset SELECT add_continuous_aggregate_policy('mat_m1', NULL, '2 months'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1063 /* Alter end offset to null but no overlap */ SELECT jsonb_set(:'config', '{end_offset}', 'null') AS config \gset SELECT * FROM alter_job(:job_id, config := :'config'); job_id | schedule_interval | max_runtime | max_retries | retry_period | scheduled | config | next_start | check_config | fixed_schedule | initial_start | timezone | application_name --------+-------------------+-------------+-------------+--------------+-----------+---------------------------------------------------------------------------+------------+------------------------------------------------------------------+----------------+---------------+----------+-------------------------------------------- 1062 | @ 12 hours | @ 0 | -1 | @ 12 hours | t | {"end_offset": null, "start_offset": "@ 14 days", "mat_hypertable_id": 5} | -infinity | _timescaledb_functions.policy_refresh_continuous_aggregate_check | f | | | Refresh Continuous Aggregate Policy [1062] SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ CREATE MATERIALIZED VIEW mat_m2(time, counta) WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1 day', time) AS bucket, count(a), sum(b) FROM overlap_test_timestamptz GROUP BY 1 WITH NO DATA; /* Create two policies on mat_m1 */ SELECT add_continuous_aggregate_policy('mat_m1', NULL, '30 days'::interval, '12 h'::interval) AS agg_m1_job_1 \gset SELECT add_continuous_aggregate_policy('mat_m1', '30 days'::interval, NULL, '12 h'::interval) AS agg_m1_job_2 \gset /* Create single policy on mat_m2 */ SELECT add_continuous_aggregate_policy('mat_m2', NULL, NULL, '12 h'::interval) AS agg_m2_job \gset /* Cleanup any existing data */ TRUNCATE mat_m1; TRUNCATE mat_m2; /* Refresh both continuous aggs immediately */ CALL run_job(:agg_m1_job_1); CALL run_job(:agg_m1_job_2); CALL run_job(:agg_m2_job); /* Compare both outputs */ SELECT count(*) AS exp_row_count from mat_m1 \gset SELECT count(*) AS actual_row_count from ( SELECT * from mat_m1 UNION SELECT * from mat_m2) AS union_q \gset /* Row counts should be the same */ SELECT :exp_row_count = :actual_row_count, :exp_row_count, :actual_row_count; ?column? | ?column? | ?column? ----------+----------+---------- t | 152 | 152 SELECT * from mat_m2 EXCEPT SELECT * from mat_m1; time | counta | sum ------+--------+----- SELECT * from mat_m1 EXCEPT SELECT * from mat_m2; time | counta | sum ------+--------+----- DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 17 other objects DROP MATERIALIZED VIEW mat_m2; NOTICE: drop cascades to 17 other objects /* Test with variable sized buckets */ CREATE TABLE overlap_test_timestamptz_var ( time timestamptz NOT NULL, a INTEGER, b INTEGER ); SELECT create_hypertable('overlap_test_timestamptz_var', 'time', chunk_time_interval => '1 month'::interval); create_hypertable ------------------------------------------- (7,public,overlap_test_timestamptz_var,t) INSERT INTO overlap_test_timestamptz_var SELECT t, (i % 5), random() * 100 FROM generate_series('2024-01-01T01:01:01+00', '2025-06-01T01:01:01+00', INTERVAL '1 day') t, generate_series(1, 10) i; CREATE MATERIALIZED VIEW mat_m1(time, counta) WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1 month', time) AS bucket, count(a), sum(b) FROM overlap_test_timestamptz_var GROUP BY 1 WITH NO DATA; /* Test interval checking when multiple policies are created on the same cagg */ SELECT add_continuous_aggregate_policy('mat_m1', NULL, '3 months'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1067 SELECT add_continuous_aggregate_policy('mat_m1', '3 months'::interval, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1068 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ /* Creating policies in either order should work */ SELECT add_continuous_aggregate_policy('mat_m1', '3 months'::interval, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1069 SELECT add_continuous_aggregate_policy('mat_m1', NULL, '3 months'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1070 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', NULL, '3 months'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1071 SELECT add_continuous_aggregate_policy('mat_m1', '2 months'::interval, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1072 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ /* Test non-null offsets on both sides too */ SELECT add_continuous_aggregate_policy('mat_m1', '8 months'::interval, '6 months'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1073 SELECT add_continuous_aggregate_policy('mat_m1', '6 months'::interval, '12 weeks'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1074 SELECT add_continuous_aggregate_policy('mat_m1', '12 weeks'::interval, '1 days'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1075 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ /* Check overlap is detected correctly */ \set ON_ERROR_STOP 0 SELECT add_continuous_aggregate_policy('mat_m1', NULL, '2 months'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1076 SELECT add_continuous_aggregate_policy('mat_m1', '3 months'::interval, NULL, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', '3 months'::interval, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1077 SELECT add_continuous_aggregate_policy('mat_m1', NULL, '2 months'::interval, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', '6 months'::interval, '1 week'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1078 SELECT add_continuous_aggregate_policy('mat_m1', '4 months'::interval, '2 weeks'::interval, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', '4 months'::interval, '2 weeks'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1079 SELECT add_continuous_aggregate_policy('mat_m1', '6 months'::interval, '1 week'::interval, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', NULL, '30 days'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1080 SELECT add_continuous_aggregate_policy('mat_m1', NULL, '20 days'::interval, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', '30 days'::interval, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1081 SELECT add_continuous_aggregate_policy('mat_m1', '20 days'::interval, NULL, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ \set ON_ERROR_STOP 1 /* Check behaviour when exact policy is already defined */ \set ON_ERROR_STOP 0 /*if_not_exists=false*/ SELECT add_continuous_aggregate_policy('mat_m1', '1 year'::interval, '8 months', '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1082 SELECT add_continuous_aggregate_policy('mat_m1', '8 months'::interval, '2 weeks', '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1083 SELECT add_continuous_aggregate_policy('mat_m1', '8 months'::interval, '2 weeks', '12 h'::interval); ERROR: continuous aggregate refresh policy already exists for "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', NULL::interval, NULL::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1084 SELECT add_continuous_aggregate_policy('mat_m1', NULL::interval, NULL::interval, '12 h'::interval); ERROR: continuous aggregate refresh policy already exists for "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ \set ON_ERROR_STOP 1 /*if_not_exists => true*/ \set ON_ERROR_STOP 0 SELECT add_continuous_aggregate_policy('mat_m1', '1 year'::interval, '8 months', '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1085 SELECT add_continuous_aggregate_policy('mat_m1', '8 months'::interval, '2 weeks', '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1086 SELECT add_continuous_aggregate_policy('mat_m1', '8 months'::interval, '2 weeks', '12 h'::interval, if_not_exists => true); NOTICE: continuous aggregate refresh policy already exists for "mat_m1", skipping add_continuous_aggregate_policy --------------------------------- -1 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', NULL::interval, NULL::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1087 SELECT add_continuous_aggregate_policy('mat_m1', NULL::interval, NULL::interval, '12 h'::interval, if_not_exists => true); NOTICE: continuous aggregate refresh policy already exists for "mat_m1", skipping add_continuous_aggregate_policy --------------------------------- -1 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ \set ON_ERROR_STOP 1 /* Mixing different interval units should also work correctly*/ SELECT add_continuous_aggregate_policy('mat_m1', NULL, '1 month'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1088 SELECT add_continuous_aggregate_policy('mat_m1', '2 weeks'::interval, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1089 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', '1 year'::interval, '2 months'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1090 SELECT add_continuous_aggregate_policy('mat_m1', '8 weeks'::interval, '-7 days'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1091 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', '2 weeks'::interval, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1092 SELECT add_continuous_aggregate_policy('mat_m1', NULL, '1 month'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1093 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ \set ON_ERROR_STOP 0 SELECT add_continuous_aggregate_policy('mat_m1', NULL, '2 weeks'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1094 SELECT add_continuous_aggregate_policy('mat_m1', '1 month'::interval, NULL, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ \set ON_ERROR_STOP 1 /* Check overlap with negative offsets */ SELECT add_continuous_aggregate_policy('mat_m1', NULL, '2 weeks'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1095 SELECT add_continuous_aggregate_policy('mat_m1', '-1 month'::interval, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1096 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ \set ON_ERROR_STOP 0 SELECT add_continuous_aggregate_policy('mat_m1', NULL, '-2 weeks'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1097 SELECT add_continuous_aggregate_policy('mat_m1', '1 month'::interval, NULL, '12 h'::interval); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ \set ON_ERROR_STOP 1 /* Test `alter_job` changing the config */ SELECT add_continuous_aggregate_policy('mat_m1', NULL, '2 months'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1098 SELECT id AS job_id, config AS config FROM _timescaledb_catalog.bgw_job WHERE proc_name = 'policy_refresh_continuous_aggregate' \gset SELECT add_continuous_aggregate_policy('mat_m1', '2 weeks'::interval, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1099 /* Alter end offset but don't overlap */ SELECT jsonb_set(:'config', '{end_offset}', '"30 days"') AS config \gset SELECT * FROM alter_job(:job_id, config := :'config'); job_id | schedule_interval | max_runtime | max_retries | retry_period | scheduled | config | next_start | check_config | fixed_schedule | initial_start | timezone | application_name --------+-------------------+-------------+-------------+--------------+-----------+-------------------------------------------------------------------------+------------+------------------------------------------------------------------+----------------+---------------+----------+-------------------------------------------- 1098 | @ 12 hours | @ 0 | -1 | @ 12 hours | t | {"end_offset": "30 days", "start_offset": null, "mat_hypertable_id": 8} | -infinity | _timescaledb_functions.policy_refresh_continuous_aggregate_check | f | | | Refresh Continuous Aggregate Policy [1098] \set ON_ERROR_STOP 0 /* Alter end offset to overlap with another job*/ SELECT jsonb_set(:'config', '{end_offset}', '"1 week"') AS config \gset SELECT * FROM alter_job(:job_id, config := :'config'); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" /* Alter end offset to be null */ SELECT jsonb_set(:'config', '{end_offset}', 'null') AS config \gset SELECT * FROM alter_job(:job_id, config := :'config'); ERROR: refresh interval overlaps with an existing continuous aggregate policy on "mat_m1" /* Alter job to be identical to existing job */ SELECT jsonb_set(:'config', '{start_offset}', '"2 weeks"') AS config \gset SELECT * FROM alter_job(:job_id, config := :'config'); ERROR: continuous aggregate refresh policy already exists for "mat_m1" \set ON_ERROR_STOP 1 SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ SELECT add_continuous_aggregate_policy('mat_m1', '2 weeks'::interval, NULL, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1100 SELECT id AS job_id, config AS config FROM _timescaledb_catalog.bgw_job WHERE proc_name = 'policy_refresh_continuous_aggregate' \gset SELECT add_continuous_aggregate_policy('mat_m1', NULL, '2 months'::interval, '12 h'::interval); add_continuous_aggregate_policy --------------------------------- 1101 /* Alter end offset to null but no overlap */ SELECT jsonb_set(:'config', '{end_offset}', 'null') AS config \gset SELECT * FROM alter_job(:job_id, config := :'config'); job_id | schedule_interval | max_runtime | max_retries | retry_period | scheduled | config | next_start | check_config | fixed_schedule | initial_start | timezone | application_name --------+-------------------+-------------+-------------+--------------+-----------+---------------------------------------------------------------------------+------------+------------------------------------------------------------------+----------------+---------------+----------+-------------------------------------------- 1100 | @ 12 hours | @ 0 | -1 | @ 12 hours | t | {"end_offset": null, "start_offset": "@ 14 days", "mat_hypertable_id": 8} | -infinity | _timescaledb_functions.policy_refresh_continuous_aggregate_check | f | | | Refresh Continuous Aggregate Policy [1100] SELECT remove_continuous_aggregate_policy('mat_m1'); remove_continuous_aggregate_policy ------------------------------------ CREATE MATERIALIZED VIEW mat_m2(time, counta) WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1 month', time) AS bucket, count(a), sum(b) FROM overlap_test_timestamptz_var GROUP BY 1 WITH NO DATA; /* Create two policies on mat_m1 */ SELECT add_continuous_aggregate_policy('mat_m1', NULL, '30 days'::interval, '12 h'::interval, buckets_per_batch => 0) AS agg_m1_job_1 \gset SELECT add_continuous_aggregate_policy('mat_m1', '30 days'::interval, NULL, '12 h'::interval, buckets_per_batch => 0) AS agg_m1_job_2 \gset /* Create single policy on mat_m2 */ SELECT add_continuous_aggregate_policy('mat_m2', NULL, NULL, '12 h'::interval, buckets_per_batch => 0) AS agg_m2_job \gset /* Cleanup any existing data */ TRUNCATE mat_m1; TRUNCATE mat_m2; /* Refresh both continuous aggs immediately */ CALL run_job(:agg_m1_job_1); CALL run_job(:agg_m1_job_2); CALL run_job(:agg_m2_job); /* Compare both outputs */ SELECT count(*) AS exp_row_count from mat_m1 \gset SELECT count(*) AS actual_row_count from ( SELECT * from mat_m1 UNION SELECT * from mat_m2) AS union_q \gset /* Row counts should be the same */ SELECT :exp_row_count = :actual_row_count, :exp_row_count, :actual_row_count; ?column? | ?column? | ?column? ----------+----------+---------- t | 18 | 18 SELECT * from mat_m2 EXCEPT SELECT * from mat_m1; time | counta | sum ------+--------+----- SELECT * from mat_m1 EXCEPT SELECT * from mat_m2; time | counta | sum ------+--------+----- DROP MATERIALIZED VIEW mat_m1; NOTICE: drop cascades to 3 other objects DROP MATERIALIZED VIEW mat_m2; NOTICE: drop cascades to 3 other objects /* Concurrent policies aren't allowed on hierarchical continuous aggs */ CREATE MATERIALIZED VIEW mat_m1 WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1 day', time) AS bucket, count(a) AS counta, sum(b) AS sumb FROM overlap_test_timestamptz GROUP BY 1 WITH NO DATA; CREATE MATERIALIZED VIEW mat_m1_rollup WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1 month', bucket) AS bucket, sum(counta) AS counta, sum(sumb) AS sumb FROM mat_m1 GROUP BY 1 WITH NO DATA; CREATE MATERIALIZED VIEW mat_m1_rollup2 WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1 month', bucket) AS bucket, sum(counta) AS counta, sum(sumb) AS sumb FROM mat_m1 GROUP BY 1 WITH NO DATA; SELECT add_continuous_aggregate_policy('mat_m1_rollup', NULL, '30 days'::interval, '12 h'::interval) AS "JOB_ID" \gset -- alter_job should not be blocked SELECT alter_job(:JOB_ID, next_start => '2000-01-01'::timestamptz); alter_job ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1105,"@ 12 hours","@ 0",-1,"@ 12 hours",t,"{""end_offset"": ""@ 30 days"", ""start_offset"": null, ""mat_hypertable_id"": 11}","Sat Jan 01 00:00:00 2000 PST",_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1105]") \set ON_ERROR_STOP 0 -- Multiple policies on hierarchical cagg should not be allowed SELECT add_continuous_aggregate_policy('mat_m1_rollup', '29 days'::interval, NULL, '12 h'::interval); ERROR: multiple refresh policies are not supported for hierarchical continuous aggregates \set ON_ERROR_STOP 1 -- Adding the exact same policy with if_not_exists should succeed (not error) SELECT add_continuous_aggregate_policy('mat_m1_rollup', NULL, '30 days'::interval, '12 h'::interval, if_not_exists => true); NOTICE: continuous aggregate refresh policy already exists for "mat_m1_rollup", skipping add_continuous_aggregate_policy --------------------------------- -1 -- different hierarchical caggs should be allowed to have their own policies SELECT add_continuous_aggregate_policy('mat_m1_rollup2', NULL, '30 days'::interval, '12 h'::interval) AS "JOB_ID2" \gset SELECT alter_job(:JOB_ID2, next_start => '2000-01-01'::timestamptz); alter_job ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- (1106,"@ 12 hours","@ 0",-1,"@ 12 hours",t,"{""end_offset"": ""@ 30 days"", ""start_offset"": null, ""mat_hypertable_id"": 12}","Sat Jan 01 00:00:00 2000 PST",_timescaledb_functions.policy_refresh_continuous_aggregate_check,f,,,"Refresh Continuous Aggregate Policy [1106]") ================================================ FILE: tsl/test/expected/cagg_policy_incremental.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE OR REPLACE FUNCTION ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(timeout INT = -1, mock_start_time INT = 0) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_create() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_destroy() RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; CREATE OR REPLACE FUNCTION ts_bgw_params_reset_time(set_time BIGINT = 0, wait BOOLEAN = false) RETURNS VOID AS :MODULE_PATHNAME LANGUAGE C VOLATILE; -- Create a user with specific timezone and mock time CREATE ROLE test_cagg_refresh_policy_user WITH LOGIN; ALTER ROLE test_cagg_refresh_policy_user SET timezone TO 'UTC'; ALTER ROLE test_cagg_refresh_policy_user SET timescaledb.current_timestamp_mock TO '2025-03-11 00:00:00+00'; GRANT ALL ON SCHEMA public TO test_cagg_refresh_policy_user; \c :TEST_DBNAME test_cagg_refresh_policy_user CREATE TABLE public.bgw_log( msg_no INT, mock_time BIGINT, application_name TEXT, msg TEXT ); CREATE VIEW sorted_bgw_log AS SELECT msg_no, mock_time, application_name, regexp_replace(regexp_replace(msg, '(Wait until|started at|execution time) [0-9]+(\.[0-9]+)?', '\1 (RANDOM)', 'g'), 'background worker "[^"]+"','connection') AS msg FROM bgw_log ORDER BY mock_time, application_name COLLATE "C", msg_no; CREATE TABLE public.bgw_dsm_handle_store( handle BIGINT ); INSERT INTO public.bgw_dsm_handle_store VALUES (0); SELECT ts_bgw_params_create(); ts_bgw_params_create ---------------------- CREATE TABLE conditions ( time TIMESTAMP WITH TIME ZONE NOT NULL, device_id INTEGER, temperature NUMERIC ); SELECT FROM create_hypertable('conditions', by_range('time')); -- INSERT INTO conditions SELECT t, d, 10 FROM generate_series( '2025-02-05 00:00:00+00', '2025-03-05 00:00:00+00', '1 hour'::interval) AS t, generate_series(1,5) AS d; CREATE MATERIALIZED VIEW conditions_by_day WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1 day', time), device_id, count(*), min(temperature), max(temperature), avg(temperature), sum(temperature) FROM conditions GROUP BY 1, 2 WITH NO DATA; SELECT add_continuous_aggregate_policy( 'conditions_by_day', start_offset => NULL, end_offset => NULL, schedule_interval => INTERVAL '1 h', buckets_per_batch => 10 ) AS job_id \gset SELECT config FROM timescaledb_information.jobs WHERE job_id = :'job_id' \gset SELECT ts_bgw_params_reset_time(0, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+--------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Registered new background worker 2 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1000] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Sat Mar 01 00:00:00 2025 UTC, Thu Mar 06 00:00:00 2025 UTC ] (batch 1 of 4) 1 | 0 | Refresh Continuous Aggregate Policy [1000] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 2 | 0 | Refresh Continuous Aggregate Policy [1000] | inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" 3 | 0 | Refresh Continuous Aggregate Policy [1000] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Wed Feb 19 00:00:00 2025 UTC, Sat Mar 01 00:00:00 2025 UTC ] (batch 2 of 4) 4 | 0 | Refresh Continuous Aggregate Policy [1000] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 5 | 0 | Refresh Continuous Aggregate Policy [1000] | inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" 6 | 0 | Refresh Continuous Aggregate Policy [1000] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Sun Feb 09 00:00:00 2025 UTC, Wed Feb 19 00:00:00 2025 UTC ] (batch 3 of 4) 7 | 0 | Refresh Continuous Aggregate Policy [1000] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 8 | 0 | Refresh Continuous Aggregate Policy [1000] | inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" 9 | 0 | Refresh Continuous Aggregate Policy [1000] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Mon Nov 24 00:00:00 4714 UTC BC, Sun Feb 09 00:00:00 2025 UTC ] (batch 4 of 4) 10 | 0 | Refresh Continuous Aggregate Policy [1000] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 11 | 0 | Refresh Continuous Aggregate Policy [1000] | inserted 20 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_ranges; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- CREATE MATERIALIZED VIEW conditions_by_day_manual_refresh WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1 day', time), device_id, count(*), min(temperature), max(temperature), avg(temperature), sum(temperature) FROM conditions GROUP BY 1, 2 WITH NO DATA; CALL refresh_continuous_aggregate('conditions_by_day_manual_refresh', NULL, NULL); SELECT count(*) FROM conditions_by_day; count ------- 145 SELECT count(*) FROM conditions_by_day_manual_refresh; count ------- 145 -- Should have no differences SELECT count(*) > 0 AS has_diff FROM ((SELECT * FROM conditions_by_day_manual_refresh ORDER BY 1, 2) EXCEPT (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; has_diff ---------- f TRUNCATE bgw_log, conditions_by_day; SELECT config FROM alter_job( :'job_id', config => jsonb_set(:'config', '{max_batches_per_execution}', '2') ); config ----------------------------------------------------------------------------------------------------------------------------- {"end_offset": null, "start_offset": null, "buckets_per_batch": 10, "mat_hypertable_id": 2, "max_batches_per_execution": 2} -- advance time by 1h so that job runs one more time SELECT ts_bgw_params_reset_time(extract(epoch from interval '1 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+------------+--------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0 | 3600000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 3600000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 3600000000 | Refresh Continuous Aggregate Policy [1000] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Sat Mar 01 00:00:00 2025 UTC, Thu Mar 06 00:00:00 2025 UTC ] (batch 1 of 4) 1 | 3600000000 | Refresh Continuous Aggregate Policy [1000] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 2 | 3600000000 | Refresh Continuous Aggregate Policy [1000] | inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" 3 | 3600000000 | Refresh Continuous Aggregate Policy [1000] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Wed Feb 19 00:00:00 2025 UTC, Sat Mar 01 00:00:00 2025 UTC ] (batch 2 of 4) 4 | 3600000000 | Refresh Continuous Aggregate Policy [1000] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 5 | 3600000000 | Refresh Continuous Aggregate Policy [1000] | inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" 6 | 3600000000 | Refresh Continuous Aggregate Policy [1000] | reached maximum number of batches per execution (2), batches not processed (2) SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_ranges; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- SELECT count(*) FROM conditions_by_day; count ------- 75 SELECT count(*) FROM conditions_by_day_manual_refresh; count ------- 145 -- Should have differences SELECT count(*) > 0 AS has_diff FROM ((SELECT * FROM conditions_by_day_manual_refresh ORDER BY 1, 2) EXCEPT (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; has_diff ---------- t -- advance time by 2h so that job runs one more time SELECT ts_bgw_params_reset_time(extract(epoch from interval '2 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+------------+--------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0 | 3600000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 3600000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 3600000000 | Refresh Continuous Aggregate Policy [1000] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Sat Mar 01 00:00:00 2025 UTC, Thu Mar 06 00:00:00 2025 UTC ] (batch 1 of 4) 1 | 3600000000 | Refresh Continuous Aggregate Policy [1000] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 2 | 3600000000 | Refresh Continuous Aggregate Policy [1000] | inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" 3 | 3600000000 | Refresh Continuous Aggregate Policy [1000] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Wed Feb 19 00:00:00 2025 UTC, Sat Mar 01 00:00:00 2025 UTC ] (batch 2 of 4) 4 | 3600000000 | Refresh Continuous Aggregate Policy [1000] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 5 | 3600000000 | Refresh Continuous Aggregate Policy [1000] | inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" 6 | 3600000000 | Refresh Continuous Aggregate Policy [1000] | reached maximum number of batches per execution (2), batches not processed (2) 0 | 7200000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 7200000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 7200000000 | Refresh Continuous Aggregate Policy [1000] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Sun Feb 09 00:00:00 2025 UTC, Wed Feb 19 00:00:00 2025 UTC ] (batch 1 of 2) 1 | 7200000000 | Refresh Continuous Aggregate Policy [1000] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 2 | 7200000000 | Refresh Continuous Aggregate Policy [1000] | inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" 3 | 7200000000 | Refresh Continuous Aggregate Policy [1000] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Mon Nov 24 00:00:00 4714 UTC BC, Sun Feb 09 00:00:00 2025 UTC ] (batch 2 of 2) 4 | 7200000000 | Refresh Continuous Aggregate Policy [1000] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 5 | 7200000000 | Refresh Continuous Aggregate Policy [1000] | inserted 20 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_ranges; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- -- Should have no differences SELECT count(*) > 0 AS has_diff FROM ((SELECT * FROM conditions_by_day_manual_refresh ORDER BY 1, 2) EXCEPT (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; has_diff ---------- f -- Set max_batches_per_execution to 10 SELECT config FROM alter_job( :'job_id', config => jsonb_set(:'config', '{max_batches_per_execution}', '10') ); config ------------------------------------------------------------------------------------------------------------------------------ {"end_offset": null, "start_offset": null, "buckets_per_batch": 10, "mat_hypertable_id": 2, "max_batches_per_execution": 10} TRUNCATE bgw_log; -- Insert data into the past INSERT INTO conditions SELECT t, d, 10 FROM generate_series( '2020-02-05 00:00:00+00', '2020-03-05 00:00:00+00', '1 hour'::interval) AS t, generate_series(1,5) AS d; -- advance time by 3h so that job runs one more time SELECT ts_bgw_params_reset_time(extract(epoch from interval '3 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- -- Should process all four batches in the past SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-------------+--------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0 | 10800000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 10800000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 10800000000 | Refresh Continuous Aggregate Policy [1000] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Sat Feb 29 00:00:00 2020 UTC, Fri Mar 06 00:00:00 2020 UTC ] (batch 1 of 4) 1 | 10800000000 | Refresh Continuous Aggregate Policy [1000] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 2 | 10800000000 | Refresh Continuous Aggregate Policy [1000] | inserted 30 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" 3 | 10800000000 | Refresh Continuous Aggregate Policy [1000] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Wed Feb 19 00:00:00 2020 UTC, Sat Feb 29 00:00:00 2020 UTC ] (batch 2 of 4) 4 | 10800000000 | Refresh Continuous Aggregate Policy [1000] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 5 | 10800000000 | Refresh Continuous Aggregate Policy [1000] | inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" 6 | 10800000000 | Refresh Continuous Aggregate Policy [1000] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Sun Feb 09 00:00:00 2020 UTC, Wed Feb 19 00:00:00 2020 UTC ] (batch 3 of 4) 7 | 10800000000 | Refresh Continuous Aggregate Policy [1000] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 8 | 10800000000 | Refresh Continuous Aggregate Policy [1000] | inserted 50 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" 9 | 10800000000 | Refresh Continuous Aggregate Policy [1000] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Wed Feb 05 00:00:00 2020 UTC, Sun Feb 09 00:00:00 2020 UTC ] (batch 4 of 4) 10 | 10800000000 | Refresh Continuous Aggregate Policy [1000] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 11 | 10800000000 | Refresh Continuous Aggregate Policy [1000] | inserted 20 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_ranges; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- SELECT count(*) FROM conditions_by_day; count ------- 295 SELECT count(*) FROM conditions_by_day_manual_refresh; count ------- 145 CALL refresh_continuous_aggregate('conditions_by_day_manual_refresh', NULL, NULL); SELECT count(*) FROM conditions_by_day; count ------- 295 SELECT count(*) FROM conditions_by_day_manual_refresh; count ------- 295 -- Should have no differences SELECT count(*) > 0 AS has_diff FROM ((SELECT * FROM conditions_by_day_manual_refresh ORDER BY 1, 2) EXCEPT (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; has_diff ---------- f -- Check invalid configurations \set ON_ERROR_STOP 0 \set VERBOSITY default SELECT config FROM alter_job( :'job_id', config => jsonb_set(:'config', '{max_batches_per_execution}', '-1') ); ERROR: invalid max batches per execution DETAIL: max_batches_per_execution: -1 HINT: The max batches per execution should be greater than or equal to zero. SELECT config FROM alter_job( :'job_id', config => jsonb_set(:'config', '{buckets_per_batch}', '-1') ); ERROR: invalid buckets per batch DETAIL: buckets_per_batch: -1 HINT: The buckets per batch should be greater than or equal to zero. \set VERBOSITY terse \set ON_ERROR_STOP 1 -- Truncate all data from the original hypertable TRUNCATE bgw_log, conditions; -- advance time by 4h so that job runs one more time SELECT ts_bgw_params_reset_time(extract(epoch from interval '4 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- -- Should fallback to single batch processing because there's no data to be refreshed on the original hypertable SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-------------+--------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------- 0 | 14400000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 14400000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 14400000000 | Refresh Continuous Aggregate Policy [1000] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Mon Nov 24 00:00:00 4714 UTC BC, Thu Mar 06 00:00:00 2025 UTC ] 1 | 14400000000 | Refresh Continuous Aggregate Policy [1000] | deleted 295 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 2 | 14400000000 | Refresh Continuous Aggregate Policy [1000] | inserted 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_ranges; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- -- Should return zero rows SELECT count(*) FROM conditions_by_day; count ------- 0 -- 1 day of data INSERT INTO conditions SELECT t, d, 10 FROM generate_series( '2020-02-05 00:00:00+00', '2020-02-06 00:00:00+00', '1 hour'::interval) AS t, generate_series(1,5) AS d; TRUNCATE bgw_log; -- advance time by 5h so that job runs one more time SELECT ts_bgw_params_reset_time(extract(epoch from interval '5 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- -- Should fallback to single batch processing because the refresh size is too small SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-------------+--------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------- 0 | 18000000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 18000000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 18000000000 | Refresh Continuous Aggregate Policy [1000] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Wed Feb 05 00:00:00 2020 UTC, Fri Feb 07 00:00:00 2020 UTC ] 1 | 18000000000 | Refresh Continuous Aggregate Policy [1000] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 2 | 18000000000 | Refresh Continuous Aggregate Policy [1000] | inserted 10 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_ranges; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- -- Should return 10 rows because the bucket width is `1 day` and buckets per batch is `10` SELECT count(*) FROM conditions_by_day; count ------- 10 TRUNCATE conditions_by_day, conditions, bgw_log; -- Less than 1 day of data (smaller than the bucket width) INSERT INTO conditions VALUES ('2020-02-05 00:00:00+00', 1, 10); -- advance time by 6h so that job runs one more time SELECT ts_bgw_params_reset_time(extract(epoch from interval '6 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- -- Should fallback to single batch processing because the refresh size is too small SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-------------+--------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------- 0 | 21600000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 21600000000 | DB Scheduler | [TESTING] Registered new background worker 2 | 21600000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 21600000000 | Refresh Continuous Aggregate Policy [1000] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Mon Nov 24 00:00:00 4714 UTC BC, Thu Mar 06 00:00:00 2025 UTC ] 1 | 21600000000 | Refresh Continuous Aggregate Policy [1000] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 2 | 21600000000 | Refresh Continuous Aggregate Policy [1000] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_ranges; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- -- Should return 1 row SELECT count(*) FROM conditions_by_day; count ------- 1 SELECT delete_job(:job_id); delete_job ------------ SELECT add_continuous_aggregate_policy( 'conditions_by_day', start_offset => INTERVAL '15 days', end_offset => NULL, schedule_interval => INTERVAL '1 h', buckets_per_batch => 5, refresh_newest_first => true -- explicitly set to true to test the default behavior ) AS job_id \gset SELECT add_continuous_aggregate_policy( 'conditions_by_day_manual_refresh', start_offset => INTERVAL '15 days', end_offset => NULL, schedule_interval => INTERVAL '1 h', buckets_per_batch => 0 -- 0 means no batching, so it will refresh all buckets in one go ) AS job_id_manual \gset TRUNCATE bgw_log, conditions_by_day, conditions_by_day_manual_refresh, conditions; INSERT INTO conditions SELECT t, d, 10 FROM generate_series( '2025-03-11 00:00:00+00'::timestamptz - INTERVAL '30 days', '2025-03-11 00:00:00+00'::timestamptz, '1 hour'::interval) AS t, generate_series(1,5) AS d; SELECT ts_bgw_params_reset_time(0, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+--------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Registered new background worker 2 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Tue Mar 11 00:00:00 2025 UTC, Wed Mar 12 00:00:00 2025 UTC ] (batch 1 of 4) 1 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 2 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" 3 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Thu Mar 06 00:00:00 2025 UTC, Tue Mar 11 00:00:00 2025 UTC ] (batch 2 of 4) 4 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 5 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" 6 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Sat Mar 01 00:00:00 2025 UTC, Thu Mar 06 00:00:00 2025 UTC ] (batch 3 of 4) 7 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 8 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" 9 | 0 | Refresh Continuous Aggregate Policy [1001] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Mon Feb 24 00:00:00 2025 UTC, Sat Mar 01 00:00:00 2025 UTC ] (batch 4 of 4) 10 | 0 | Refresh Continuous Aggregate Policy [1001] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 11 | 0 | Refresh Continuous Aggregate Policy [1001] | inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" 0 | 0 | Refresh Continuous Aggregate Policy [1002] | continuous aggregate refresh (individual invalidation) on "conditions_by_day_manual_refresh" in window [ Mon Feb 24 00:00:00 2025 UTC, Wed Mar 12 00:00:00 2025 UTC ] 1 | 0 | Refresh Continuous Aggregate Policy [1002] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_3" 2 | 0 | Refresh Continuous Aggregate Policy [1002] | inserted 80 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_3" SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_ranges; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- -- Both continuous aggregates should have the same data SELECT count(*) FROM conditions_by_day; count ------- 80 SELECT count(*) FROM conditions_by_day_manual_refresh; count ------- 80 -- Should have no differences SELECT count(*) > 0 AS has_diff FROM ((SELECT * FROM conditions_by_day_manual_refresh ORDER BY 1, 2) EXCEPT (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; has_diff ---------- f -- Testing with explicit refresh_newest_first = false (from oldest to newest) SELECT delete_job(:job_id); delete_job ------------ SELECT delete_job(:job_id_manual); delete_job ------------ SELECT add_continuous_aggregate_policy( 'conditions_by_day', start_offset => INTERVAL '15 days', end_offset => NULL, schedule_interval => INTERVAL '1 h', buckets_per_batch => 5, refresh_newest_first => false ) AS job_id \gset SELECT config FROM timescaledb_information.jobs WHERE job_id = :'job_id'; config ---------------------------------------------------------------------------------------------------------------------------------- {"end_offset": null, "start_offset": "@ 15 days", "buckets_per_batch": 5, "mat_hypertable_id": 2, "refresh_newest_first": false} TRUNCATE bgw_log, conditions_by_day; SELECT ts_bgw_params_reset_time(0, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+--------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1003] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Mon Feb 24 00:00:00 2025 UTC, Sat Mar 01 00:00:00 2025 UTC ] (batch 1 of 4) 1 | 0 | Refresh Continuous Aggregate Policy [1003] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 2 | 0 | Refresh Continuous Aggregate Policy [1003] | inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" 3 | 0 | Refresh Continuous Aggregate Policy [1003] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Sat Mar 01 00:00:00 2025 UTC, Thu Mar 06 00:00:00 2025 UTC ] (batch 2 of 4) 4 | 0 | Refresh Continuous Aggregate Policy [1003] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 5 | 0 | Refresh Continuous Aggregate Policy [1003] | inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" 6 | 0 | Refresh Continuous Aggregate Policy [1003] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Thu Mar 06 00:00:00 2025 UTC, Tue Mar 11 00:00:00 2025 UTC ] (batch 3 of 4) 7 | 0 | Refresh Continuous Aggregate Policy [1003] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 8 | 0 | Refresh Continuous Aggregate Policy [1003] | inserted 25 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" 9 | 0 | Refresh Continuous Aggregate Policy [1003] | continuous aggregate refresh (individual invalidation) on "conditions_by_day" in window [ Tue Mar 11 00:00:00 2025 UTC, Wed Mar 12 00:00:00 2025 UTC ] (batch 4 of 4) 10 | 0 | Refresh Continuous Aggregate Policy [1003] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_2" 11 | 0 | Refresh Continuous Aggregate Policy [1003] | inserted 5 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_2" SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_ranges; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- -- Both continuous aggregates should have the same data SELECT count(*) FROM conditions_by_day; count ------- 80 SELECT count(*) FROM conditions_by_day_manual_refresh; count ------- 80 -- Should have no differences SELECT count(*) > 0 AS has_diff FROM ((SELECT * FROM conditions_by_day_manual_refresh ORDER BY 1, 2) EXCEPT (SELECT * FROM conditions_by_day ORDER BY 1, 2)) AS diff; has_diff ---------- f -- Tests with Variable sized bucket SELECT delete_job(:job_id); delete_job ------------ TRUNCATE conditions; INSERT INTO conditions SELECT t, d, 10 FROM generate_series( '2025-01-01 00:00:00+00', '2025-10-08 00:00:00+00', '1 hour'::interval) AS t, generate_series(1,5) AS d; CREATE MATERIALIZED VIEW conditions_by_month WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('1 month', time), device_id, count(*), min(temperature), max(temperature), avg(temperature), sum(temperature) FROM conditions GROUP BY 1, 2 WITH NO DATA; SELECT add_continuous_aggregate_policy( 'conditions_by_month', start_offset => INTERVAL '600 days', end_offset => INTERVAL '7 days', schedule_interval => INTERVAL '1 day', refresh_newest_first => false ) AS job_id \gset SELECT config FROM timescaledb_information.jobs WHERE job_id = :'job_id'; config ----------------------------------------------------------------------------------------------------------------- {"end_offset": "@ 7 days", "start_offset": "@ 600 days", "mat_hypertable_id": 4, "refresh_newest_first": false} TRUNCATE bgw_log, conditions_by_day; SELECT ts_bgw_params_reset_time(0, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+--------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1004] | continuous aggregate refresh (individual invalidation) on "conditions_by_month" in window [ Tue Aug 01 00:00:00 2023 UTC, Sat Mar 01 00:00:00 2025 UTC ] 1 | 0 | Refresh Continuous Aggregate Policy [1004] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_4" 2 | 0 | Refresh Continuous Aggregate Policy [1004] | inserted 10 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_4" SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_ranges; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- SELECT delete_job(:job_id); delete_job ------------ ------------------------------------------------------------------------------------------ --Test that batched refresh with variable-length buckets doesn't leave remainders ------------------------------------------------------------------------------------------- CREATE TABLE test_data ( time TIMESTAMPTZ NOT NULL, value INT ); SELECT public.create_hypertable( relation => 'test_data', time_column_name => 'time', chunk_time_interval => interval '1 months' ); create_hypertable ------------------------ (5,public,test_data,t) -- Insert initial data INSERT INTO test_data SELECT time, 1 FROM generate_series('2024-01-01'::timestamptz, '2024-12-31'::timestamptz, '1 day'::interval) time; -- Create continuous aggregate with monthly buckets and timezone (variable-length buckets) CREATE MATERIALIZED VIEW batch_test_cagg WITH (timescaledb.continuous) AS SELECT time_bucket('1 month'::interval, time) AS bucket, count(*) as count FROM test_data GROUP BY bucket WITH NO DATA; -- Add a policy to enable batched refresh (batch size is 30 days by default for monthly buckets) SELECT add_continuous_aggregate_policy('batch_test_cagg', start_offset =>null, end_offset => INTERVAL '1 month', schedule_interval => INTERVAL '1 hour', buckets_per_batch => 1 ) AS job_id \gset -- Run the policy job - this uses batched processing, 1 bucket per batch TRUNCATE bgw_log; SELECT ts_bgw_params_reset_time(0, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ -- Verify that invalidation log has no entries other than the left and right ends with -/+ infinity SELECT materialization_id, _timescaledb_functions.to_timestamp(lowest_modified_value) as low, _timescaledb_functions.to_timestamp(greatest_modified_value) as high FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log WHERE materialization_id IN (SELECT mat_hypertable_id FROM _timescaledb_catalog.continuous_agg WHERE user_view_name = 'batch_test_cagg') AND lowest_modified_value != -9223372036854775808 -- -infinity AND greatest_modified_value != 9223372036854775807 -- +infinity ORDER BY low; materialization_id | low | high --------------------+-----+------ --verify that there is no duplicate/overlapping refreshes. --Note that batch 1 and batch 12 contains 2 buckets instead of 1 bucket as set in the policy. --This is due to the fact that we currently cut a batch of 30 days for monthly cagg, --so first batch and batch containing February can have 2 buckets. After we have a cleaner solution to --cut an exact batch size for variable-length buckets, this should be fixed. SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+-----------+--------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------- 0 | 0 | DB Scheduler | [TESTING] Registered new background worker 1 | 0 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) 0 | 0 | Refresh Continuous Aggregate Policy [1005] | continuous aggregate refresh (individual invalidation) on "batch_test_cagg" in window [ Sun Dec 01 00:00:00 2024 UTC, Sat Feb 01 00:00:00 2025 UTC ] (batch 1 of 13) 1 | 0 | Refresh Continuous Aggregate Policy [1005] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" 2 | 0 | Refresh Continuous Aggregate Policy [1005] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" 3 | 0 | Refresh Continuous Aggregate Policy [1005] | continuous aggregate refresh (individual invalidation) on "batch_test_cagg" in window [ Fri Nov 01 00:00:00 2024 UTC, Sun Dec 01 00:00:00 2024 UTC ] (batch 2 of 13) 4 | 0 | Refresh Continuous Aggregate Policy [1005] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" 5 | 0 | Refresh Continuous Aggregate Policy [1005] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" 6 | 0 | Refresh Continuous Aggregate Policy [1005] | continuous aggregate refresh (individual invalidation) on "batch_test_cagg" in window [ Tue Oct 01 00:00:00 2024 UTC, Fri Nov 01 00:00:00 2024 UTC ] (batch 3 of 13) 7 | 0 | Refresh Continuous Aggregate Policy [1005] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" 8 | 0 | Refresh Continuous Aggregate Policy [1005] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" 9 | 0 | Refresh Continuous Aggregate Policy [1005] | continuous aggregate refresh (individual invalidation) on "batch_test_cagg" in window [ Sun Sep 01 00:00:00 2024 UTC, Tue Oct 01 00:00:00 2024 UTC ] (batch 4 of 13) 10 | 0 | Refresh Continuous Aggregate Policy [1005] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" 11 | 0 | Refresh Continuous Aggregate Policy [1005] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" 12 | 0 | Refresh Continuous Aggregate Policy [1005] | continuous aggregate refresh (individual invalidation) on "batch_test_cagg" in window [ Thu Aug 01 00:00:00 2024 UTC, Sun Sep 01 00:00:00 2024 UTC ] (batch 5 of 13) 13 | 0 | Refresh Continuous Aggregate Policy [1005] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" 14 | 0 | Refresh Continuous Aggregate Policy [1005] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" 15 | 0 | Refresh Continuous Aggregate Policy [1005] | continuous aggregate refresh (individual invalidation) on "batch_test_cagg" in window [ Mon Jul 01 00:00:00 2024 UTC, Thu Aug 01 00:00:00 2024 UTC ] (batch 6 of 13) 16 | 0 | Refresh Continuous Aggregate Policy [1005] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" 17 | 0 | Refresh Continuous Aggregate Policy [1005] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" 18 | 0 | Refresh Continuous Aggregate Policy [1005] | continuous aggregate refresh (individual invalidation) on "batch_test_cagg" in window [ Sat Jun 01 00:00:00 2024 UTC, Mon Jul 01 00:00:00 2024 UTC ] (batch 7 of 13) 19 | 0 | Refresh Continuous Aggregate Policy [1005] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" 20 | 0 | Refresh Continuous Aggregate Policy [1005] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" 21 | 0 | Refresh Continuous Aggregate Policy [1005] | continuous aggregate refresh (individual invalidation) on "batch_test_cagg" in window [ Wed May 01 00:00:00 2024 UTC, Sat Jun 01 00:00:00 2024 UTC ] (batch 8 of 13) 22 | 0 | Refresh Continuous Aggregate Policy [1005] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" 23 | 0 | Refresh Continuous Aggregate Policy [1005] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" 24 | 0 | Refresh Continuous Aggregate Policy [1005] | continuous aggregate refresh (individual invalidation) on "batch_test_cagg" in window [ Mon Apr 01 00:00:00 2024 UTC, Wed May 01 00:00:00 2024 UTC ] (batch 9 of 13) 25 | 0 | Refresh Continuous Aggregate Policy [1005] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" 26 | 0 | Refresh Continuous Aggregate Policy [1005] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" 27 | 0 | Refresh Continuous Aggregate Policy [1005] | continuous aggregate refresh (individual invalidation) on "batch_test_cagg" in window [ Fri Mar 01 00:00:00 2024 UTC, Mon Apr 01 00:00:00 2024 UTC ] (batch 10 of 13) 28 | 0 | Refresh Continuous Aggregate Policy [1005] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" 29 | 0 | Refresh Continuous Aggregate Policy [1005] | inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" 30 | 0 | Refresh Continuous Aggregate Policy [1005] | continuous aggregate refresh (individual invalidation) on "batch_test_cagg" in window [ Mon Jan 01 00:00:00 2024 UTC, Fri Mar 01 00:00:00 2024 UTC ] (batch 12 of 13) 31 | 0 | Refresh Continuous Aggregate Policy [1005] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" 32 | 0 | Refresh Continuous Aggregate Policy [1005] | inserted 2 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" 33 | 0 | Refresh Continuous Aggregate Policy [1005] | continuous aggregate refresh (individual invalidation) on "batch_test_cagg" in window [ -infinity, Mon Jan 01 00:00:00 2024 UTC ] (batch 13 of 13) 34 | 0 | Refresh Continuous Aggregate Policy [1005] | deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_6" 35 | 0 | Refresh Continuous Aggregate Policy [1005] | inserted 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_6" --now run the refresh again, should not do anything TRUNCATE bgw_log; SELECT ts_bgw_params_reset_time(extract(epoch from interval '1 hour')::bigint * 1000000, true); ts_bgw_params_reset_time -------------------------- SELECT ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish(25); ts_bgw_db_scheduler_test_run_and_wait_for_scheduler_finish ------------------------------------------------------------ SELECT * FROM sorted_bgw_log; msg_no | mock_time | application_name | msg --------+------------+------------------+---------------------------------------------------- 0 | 3600000000 | DB Scheduler | [TESTING] Registered new background worker 1 | 3600000000 | DB Scheduler | [TESTING] Wait until (RANDOM), started at (RANDOM) --clean up DROP TABLE test_data CASCADE; NOTICE: drop cascades to 2 other objects NOTICE: drop cascades to 2 other objects \c :TEST_DBNAME :ROLE_SUPERUSER REASSIGN OWNED BY test_cagg_refresh_policy_user TO :ROLE_SUPERUSER; REVOKE ALL ON SCHEMA public FROM test_cagg_refresh_policy_user; DROP ROLE test_cagg_refresh_policy_user; ================================================ FILE: tsl/test/expected/cagg_policy_move.out ================================================ [File too large to display: 16.0 KB] ================================================ FILE: tsl/test/expected/cagg_policy_run.out ================================================ [File too large to display: 3.0 KB] ================================================ FILE: tsl/test/expected/cagg_query-15.out ================================================ [File too large to display: 142.2 KB] ================================================ FILE: tsl/test/expected/cagg_query-16.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. -- Connect as superuser to use SET ROLE later \c :TEST_DBNAME :ROLE_SUPERUSER SET timezone TO PST8PDT; -- Run tests with default role SET ROLE :ROLE_DEFAULT_PERM_USER; \set TEST_BASE_NAME cagg_query \ir include/cagg_query_common.sql -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. SELECT format('%s/results/%s_results_view.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_VIEW", format('%s/results/%s_results_view_hashagg.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_VIEW_HASHAGG", format('%s/results/%s_results_table.out', :'TEST_OUTPUT_DIR', :'TEST_BASE_NAME') as "TEST_RESULTS_TABLE" \gset SELECT format('\! diff %s %s', :'TEST_RESULTS_VIEW', :'TEST_RESULTS_TABLE') as "DIFF_CMD", format('\! diff %s %s', :'TEST_RESULTS_VIEW_HASHAGG', :'TEST_RESULTS_TABLE') as "DIFF_CMD2" \gset \set EXPLAIN 'EXPLAIN (VERBOSE, BUFFERS OFF, COSTS OFF)' SET client_min_messages TO NOTICE; CREATE TABLE conditions ( timec TIMESTAMPTZ NOT NULL, location TEXT NOT NULL, temperature DOUBLE PRECISION NULL, humidity DOUBLE PRECISION NULL ); select table_name from create_hypertable( 'conditions', 'timec'); table_name ------------ conditions insert into conditions values ( '2018-01-01 09:20:00-08', 'SFO', 55, 45); insert into conditions values ( '2018-01-02 09:30:00-08', 'por', 100, 100); insert into conditions values ( '2018-01-02 09:20:00-08', 'SFO', 65, 45); insert into conditions values ( '2018-01-02 09:10:00-08', 'NYC', 65, 45); insert into conditions values ( '2018-11-01 09:20:00-08', 'NYC', 45, 30); insert into conditions values ( '2018-11-01 10:40:00-08', 'NYC', 55, 35); insert into conditions values ( '2018-11-01 11:50:00-08', 'NYC', 65, 40); insert into conditions values ( '2018-11-01 12:10:00-08', 'NYC', 75, 45); insert into conditions values ( '2018-11-01 13:10:00-08', 'NYC', 85, 50); insert into conditions values ( '2018-11-02 09:20:00-08', 'NYC', 10, 10); insert into conditions values ( '2018-11-02 10:30:00-08', 'NYC', 20, 15); insert into conditions values ( '2018-11-02 11:40:00-08', 'NYC', null, null); insert into conditions values ( '2018-11-03 09:50:00-08', 'NYC', null, null); create table location_tab( locid integer, locname text ); insert into location_tab values( 1, 'SFO'); insert into location_tab values( 2, 'NYC'); insert into location_tab values( 3, 'por'); create materialized view mat_m1( location, timec, minl, sumt , sumh) WITH (timescaledb.continuous, timescaledb.materialized_only=false) as select location, time_bucket('1day', timec), min(location), sum(temperature),sum(humidity) from conditions group by time_bucket('1day', timec), location WITH NO DATA; --compute time_bucketted max+bucket_width for the materialized view SELECT time_bucket('1day' , q.timeval+ '1day'::interval) FROM ( select max(timec)as timeval from conditions ) as q; time_bucket ------------------------------ Sat Nov 03 17:00:00 2018 PDT CALL refresh_continuous_aggregate('mat_m1', NULL, NULL); --test first/last create materialized view mat_m2(location, timec, firsth, lasth, maxtemp, mintemp) WITH (timescaledb.continuous, timescaledb.materialized_only=false) as select location, time_bucket('1day', timec), first(humidity, timec), last(humidity, timec), max(temperature), min(temperature) from conditions group by time_bucket('1day', timec), location WITH NO DATA; --time that refresh assumes as now() for repeatability SELECT time_bucket('1day' , q.timeval+ '1day'::interval) FROM ( select max(timec)as timeval from conditions ) as q; time_bucket ------------------------------ Sat Nov 03 17:00:00 2018 PDT CALL refresh_continuous_aggregate('mat_m2', NULL, NULL); --normal view -- create or replace view regview( location, timec, minl, sumt , sumh) as select location, time_bucket('1day', timec), min(location), sum(temperature),sum(humidity) from conditions group by location, time_bucket('1day', timec); set enable_hashagg = false; -- NO pushdown cases --- --when we have addl. attrs in order by that are not in the -- group by, we will still need a sort :EXPLAIN select * from mat_m1 order by sumh, sumt, minl, timec ; --- QUERY PLAN --- Sort Output: _materialized_hypertable_2.location, _materialized_hypertable_2.timec, _materialized_hypertable_2.minl, _materialized_hypertable_2.sumt, _materialized_hypertable_2.sumh Sort Key: _materialized_hypertable_2.sumh, _materialized_hypertable_2.sumt, _materialized_hypertable_2.minl, _materialized_hypertable_2.timec -> Append -> Append -> Seq Scan on _timescaledb_internal._hyper_2_3_chunk Output: _hyper_2_3_chunk.location, _hyper_2_3_chunk.timec, _hyper_2_3_chunk.minl, _hyper_2_3_chunk.sumt, _hyper_2_3_chunk.sumh -> Index Scan using _hyper_2_4_chunk__materialized_hypertable_2_timec_idx on _timescaledb_internal._hyper_2_4_chunk Output: _hyper_2_4_chunk.location, _hyper_2_4_chunk.timec, _hyper_2_4_chunk.minl, _hyper_2_4_chunk.sumt, _hyper_2_4_chunk.sumh Index Cond: (_hyper_2_4_chunk.timec < 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) -> GroupAggregate Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), min(_hyper_1_2_chunk.location), sum(_hyper_1_2_chunk.temperature), sum(_hyper_1_2_chunk.humidity) Group Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Sort Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity Sort Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Result Output: _hyper_1_2_chunk.location, time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec), _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity -> Index Scan using _hyper_1_2_chunk_conditions_timec_idx on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.location, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity Index Cond: (_hyper_1_2_chunk.timec >= 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) :EXPLAIN select * from regview order by timec desc; --- QUERY PLAN --- Sort Output: conditions.location, (time_bucket('@ 1 day'::interval, conditions.timec)), (min(conditions.location)), (sum(conditions.temperature)), (sum(conditions.humidity)) Sort Key: (time_bucket('@ 1 day'::interval, conditions.timec)) DESC -> Finalize GroupAggregate Output: conditions.location, (time_bucket('@ 1 day'::interval, conditions.timec)), min(conditions.location), sum(conditions.temperature), sum(conditions.humidity) Group Key: conditions.location, (time_bucket('@ 1 day'::interval, conditions.timec)) -> Sort Output: conditions.location, (time_bucket('@ 1 day'::interval, conditions.timec)), (PARTIAL min(conditions.location)), (PARTIAL sum(conditions.temperature)), (PARTIAL sum(conditions.humidity)) Sort Key: conditions.location, (time_bucket('@ 1 day'::interval, conditions.timec)) -> Append -> Partial GroupAggregate Output: _hyper_1_1_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_1_chunk.timec)), PARTIAL min(_hyper_1_1_chunk.location), PARTIAL sum(_hyper_1_1_chunk.temperature), PARTIAL sum(_hyper_1_1_chunk.humidity) Group Key: _hyper_1_1_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_1_chunk.timec)) -> Sort Output: _hyper_1_1_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_1_chunk.timec)), _hyper_1_1_chunk.temperature, _hyper_1_1_chunk.humidity Sort Key: _hyper_1_1_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_1_chunk.timec)) -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk.location, time_bucket('@ 1 day'::interval, _hyper_1_1_chunk.timec), _hyper_1_1_chunk.temperature, _hyper_1_1_chunk.humidity -> Partial GroupAggregate Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), PARTIAL min(_hyper_1_2_chunk.location), PARTIAL sum(_hyper_1_2_chunk.temperature), PARTIAL sum(_hyper_1_2_chunk.humidity) Group Key: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)) -> Sort Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity Sort Key: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)) -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.location, time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec), _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity -- PUSHDOWN cases -- -- all group by elts in order by , reorder group by elts to match -- group by order -- This should prevent an additional sort after GroupAggregate :EXPLAIN select * from mat_m1 order by timec desc, location; --- QUERY PLAN --- Sort Output: _materialized_hypertable_2.location, _materialized_hypertable_2.timec, _materialized_hypertable_2.minl, _materialized_hypertable_2.sumt, _materialized_hypertable_2.sumh Sort Key: _materialized_hypertable_2.timec DESC, _materialized_hypertable_2.location -> Append -> Append -> Seq Scan on _timescaledb_internal._hyper_2_3_chunk Output: _hyper_2_3_chunk.location, _hyper_2_3_chunk.timec, _hyper_2_3_chunk.minl, _hyper_2_3_chunk.sumt, _hyper_2_3_chunk.sumh -> Index Scan using _hyper_2_4_chunk__materialized_hypertable_2_timec_idx on _timescaledb_internal._hyper_2_4_chunk Output: _hyper_2_4_chunk.location, _hyper_2_4_chunk.timec, _hyper_2_4_chunk.minl, _hyper_2_4_chunk.sumt, _hyper_2_4_chunk.sumh Index Cond: (_hyper_2_4_chunk.timec < 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) -> GroupAggregate Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), min(_hyper_1_2_chunk.location), sum(_hyper_1_2_chunk.temperature), sum(_hyper_1_2_chunk.humidity) Group Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Sort Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity Sort Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Result Output: _hyper_1_2_chunk.location, time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec), _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity -> Index Scan using _hyper_1_2_chunk_conditions_timec_idx on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.location, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity Index Cond: (_hyper_1_2_chunk.timec >= 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) :EXPLAIN select * from mat_m1 order by location, timec desc; --- QUERY PLAN --- Sort Output: _materialized_hypertable_2.location, _materialized_hypertable_2.timec, _materialized_hypertable_2.minl, _materialized_hypertable_2.sumt, _materialized_hypertable_2.sumh Sort Key: _materialized_hypertable_2.location, _materialized_hypertable_2.timec DESC -> Append -> Append -> Seq Scan on _timescaledb_internal._hyper_2_3_chunk Output: _hyper_2_3_chunk.location, _hyper_2_3_chunk.timec, _hyper_2_3_chunk.minl, _hyper_2_3_chunk.sumt, _hyper_2_3_chunk.sumh -> Index Scan using _hyper_2_4_chunk__materialized_hypertable_2_timec_idx on _timescaledb_internal._hyper_2_4_chunk Output: _hyper_2_4_chunk.location, _hyper_2_4_chunk.timec, _hyper_2_4_chunk.minl, _hyper_2_4_chunk.sumt, _hyper_2_4_chunk.sumh Index Cond: (_hyper_2_4_chunk.timec < 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) -> GroupAggregate Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), min(_hyper_1_2_chunk.location), sum(_hyper_1_2_chunk.temperature), sum(_hyper_1_2_chunk.humidity) Group Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Sort Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity Sort Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Result Output: _hyper_1_2_chunk.location, time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec), _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity -> Index Scan using _hyper_1_2_chunk_conditions_timec_idx on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.location, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity Index Cond: (_hyper_1_2_chunk.timec >= 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) :EXPLAIN select * from mat_m1 order by location, timec asc; --- QUERY PLAN --- Sort Output: _materialized_hypertable_2.location, _materialized_hypertable_2.timec, _materialized_hypertable_2.minl, _materialized_hypertable_2.sumt, _materialized_hypertable_2.sumh Sort Key: _materialized_hypertable_2.location, _materialized_hypertable_2.timec -> Append -> Append -> Seq Scan on _timescaledb_internal._hyper_2_3_chunk Output: _hyper_2_3_chunk.location, _hyper_2_3_chunk.timec, _hyper_2_3_chunk.minl, _hyper_2_3_chunk.sumt, _hyper_2_3_chunk.sumh -> Index Scan using _hyper_2_4_chunk__materialized_hypertable_2_timec_idx on _timescaledb_internal._hyper_2_4_chunk Output: _hyper_2_4_chunk.location, _hyper_2_4_chunk.timec, _hyper_2_4_chunk.minl, _hyper_2_4_chunk.sumt, _hyper_2_4_chunk.sumh Index Cond: (_hyper_2_4_chunk.timec < 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) -> GroupAggregate Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), min(_hyper_1_2_chunk.location), sum(_hyper_1_2_chunk.temperature), sum(_hyper_1_2_chunk.humidity) Group Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Sort Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity Sort Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Result Output: _hyper_1_2_chunk.location, time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec), _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity -> Index Scan using _hyper_1_2_chunk_conditions_timec_idx on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.location, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity Index Cond: (_hyper_1_2_chunk.timec >= 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) :EXPLAIN select * from mat_m1 where timec > '2018-10-01' order by timec desc; --- QUERY PLAN --- Append -> GroupAggregate Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), min(_hyper_1_2_chunk.location), sum(_hyper_1_2_chunk.temperature), sum(_hyper_1_2_chunk.humidity) Group Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Sort Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity Sort Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)) DESC, _hyper_1_2_chunk.location -> Result Output: _hyper_1_2_chunk.location, time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec), _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity -> Index Scan using _hyper_1_2_chunk_conditions_timec_idx on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.location, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity Index Cond: ((_hyper_1_2_chunk.timec >= 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) AND (_hyper_1_2_chunk.timec > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone)) Filter: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec) > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone) -> Index Scan using _hyper_2_4_chunk__materialized_hypertable_2_timec_idx on _timescaledb_internal._hyper_2_4_chunk Output: _hyper_2_4_chunk.location, _hyper_2_4_chunk.timec, _hyper_2_4_chunk.minl, _hyper_2_4_chunk.sumt, _hyper_2_4_chunk.sumh Index Cond: ((_hyper_2_4_chunk.timec < 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) AND (_hyper_2_4_chunk.timec > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone)) -- outer sort is used by mat_m1 for grouping. But doesn't avoid a sort after the join --- :EXPLAIN select l.locid, mat_m1.* from mat_m1 , location_tab l where timec > '2018-10-01' and l.locname = mat_m1.location order by timec desc; --- QUERY PLAN --- Sort Output: l.locid, _hyper_2_4_chunk.location, _hyper_2_4_chunk.timec, _hyper_2_4_chunk.minl, _hyper_2_4_chunk.sumt, _hyper_2_4_chunk.sumh Sort Key: _hyper_2_4_chunk.timec DESC -> Hash Join Output: l.locid, _hyper_2_4_chunk.location, _hyper_2_4_chunk.timec, _hyper_2_4_chunk.minl, _hyper_2_4_chunk.sumt, _hyper_2_4_chunk.sumh Hash Cond: (l.locname = _hyper_2_4_chunk.location) -> Seq Scan on public.location_tab l Output: l.locid, l.locname -> Hash Output: _hyper_2_4_chunk.location, _hyper_2_4_chunk.timec, _hyper_2_4_chunk.minl, _hyper_2_4_chunk.sumt, _hyper_2_4_chunk.sumh -> Append -> Index Scan using _hyper_2_4_chunk__materialized_hypertable_2_timec_idx on _timescaledb_internal._hyper_2_4_chunk Output: _hyper_2_4_chunk.location, _hyper_2_4_chunk.timec, _hyper_2_4_chunk.minl, _hyper_2_4_chunk.sumt, _hyper_2_4_chunk.sumh Index Cond: ((_hyper_2_4_chunk.timec < 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) AND (_hyper_2_4_chunk.timec > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone)) -> GroupAggregate Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), min(_hyper_1_2_chunk.location), sum(_hyper_1_2_chunk.temperature), sum(_hyper_1_2_chunk.humidity) Group Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Sort Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity Sort Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Result Output: _hyper_1_2_chunk.location, time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec), _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity -> Index Scan using _hyper_1_2_chunk_conditions_timec_idx on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.location, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity Index Cond: ((_hyper_1_2_chunk.timec >= 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) AND (_hyper_1_2_chunk.timec > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone)) Filter: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec) > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone) :EXPLAIN select * from mat_m2 where timec > '2018-10-01' order by timec desc; --- QUERY PLAN --- Append -> GroupAggregate Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), first(_hyper_1_2_chunk.humidity, _hyper_1_2_chunk.timec), last(_hyper_1_2_chunk.humidity, _hyper_1_2_chunk.timec), max(_hyper_1_2_chunk.temperature), min(_hyper_1_2_chunk.temperature) Group Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Sort Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.humidity, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.temperature Sort Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)) DESC, _hyper_1_2_chunk.location -> Result Output: _hyper_1_2_chunk.location, time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec), _hyper_1_2_chunk.humidity, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.temperature -> Index Scan using _hyper_1_2_chunk_conditions_timec_idx on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.location, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.humidity, _hyper_1_2_chunk.temperature Index Cond: ((_hyper_1_2_chunk.timec >= 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) AND (_hyper_1_2_chunk.timec > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone)) Filter: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec) > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone) -> Index Scan using _hyper_3_6_chunk__materialized_hypertable_3_timec_idx on _timescaledb_internal._hyper_3_6_chunk Output: _hyper_3_6_chunk.location, _hyper_3_6_chunk.timec, _hyper_3_6_chunk.firsth, _hyper_3_6_chunk.lasth, _hyper_3_6_chunk.maxtemp, _hyper_3_6_chunk.mintemp Index Cond: ((_hyper_3_6_chunk.timec < 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) AND (_hyper_3_6_chunk.timec > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone)) :EXPLAIN select * from (select * from mat_m2 where timec > '2018-10-01' order by timec desc ) as q limit 1; --- QUERY PLAN --- Limit Output: _hyper_3_6_chunk.location, _hyper_3_6_chunk.timec, _hyper_3_6_chunk.firsth, _hyper_3_6_chunk.lasth, _hyper_3_6_chunk.maxtemp, _hyper_3_6_chunk.mintemp -> Sort Output: _hyper_3_6_chunk.location, _hyper_3_6_chunk.timec, _hyper_3_6_chunk.firsth, _hyper_3_6_chunk.lasth, _hyper_3_6_chunk.maxtemp, _hyper_3_6_chunk.mintemp Sort Key: _hyper_3_6_chunk.timec DESC -> Append -> Index Scan using _hyper_3_6_chunk__materialized_hypertable_3_timec_idx on _timescaledb_internal._hyper_3_6_chunk Output: _hyper_3_6_chunk.location, _hyper_3_6_chunk.timec, _hyper_3_6_chunk.firsth, _hyper_3_6_chunk.lasth, _hyper_3_6_chunk.maxtemp, _hyper_3_6_chunk.mintemp Index Cond: ((_hyper_3_6_chunk.timec < 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) AND (_hyper_3_6_chunk.timec > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone)) -> GroupAggregate Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), first(_hyper_1_2_chunk.humidity, _hyper_1_2_chunk.timec), last(_hyper_1_2_chunk.humidity, _hyper_1_2_chunk.timec), max(_hyper_1_2_chunk.temperature), min(_hyper_1_2_chunk.temperature) Group Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Sort Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.humidity, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.temperature Sort Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Result Output: _hyper_1_2_chunk.location, time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec), _hyper_1_2_chunk.humidity, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.temperature -> Index Scan using _hyper_1_2_chunk_conditions_timec_idx on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.location, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.humidity, _hyper_1_2_chunk.temperature Index Cond: ((_hyper_1_2_chunk.timec >= 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) AND (_hyper_1_2_chunk.timec > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone)) Filter: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec) > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone) :EXPLAIN select * from (select * from mat_m2 where timec > '2018-10-01' order by timec desc , location asc nulls first) as q limit 1; --- QUERY PLAN --- Limit Output: _hyper_3_6_chunk.location, _hyper_3_6_chunk.timec, _hyper_3_6_chunk.firsth, _hyper_3_6_chunk.lasth, _hyper_3_6_chunk.maxtemp, _hyper_3_6_chunk.mintemp -> Sort Output: _hyper_3_6_chunk.location, _hyper_3_6_chunk.timec, _hyper_3_6_chunk.firsth, _hyper_3_6_chunk.lasth, _hyper_3_6_chunk.maxtemp, _hyper_3_6_chunk.mintemp Sort Key: _hyper_3_6_chunk.timec DESC, _hyper_3_6_chunk.location NULLS FIRST -> Append -> Index Scan using _hyper_3_6_chunk__materialized_hypertable_3_timec_idx on _timescaledb_internal._hyper_3_6_chunk Output: _hyper_3_6_chunk.location, _hyper_3_6_chunk.timec, _hyper_3_6_chunk.firsth, _hyper_3_6_chunk.lasth, _hyper_3_6_chunk.maxtemp, _hyper_3_6_chunk.mintemp Index Cond: ((_hyper_3_6_chunk.timec < 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) AND (_hyper_3_6_chunk.timec > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone)) -> GroupAggregate Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), first(_hyper_1_2_chunk.humidity, _hyper_1_2_chunk.timec), last(_hyper_1_2_chunk.humidity, _hyper_1_2_chunk.timec), max(_hyper_1_2_chunk.temperature), min(_hyper_1_2_chunk.temperature) Group Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Sort Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.humidity, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.temperature Sort Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Result Output: _hyper_1_2_chunk.location, time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec), _hyper_1_2_chunk.humidity, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.temperature -> Index Scan using _hyper_1_2_chunk_conditions_timec_idx on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.location, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.humidity, _hyper_1_2_chunk.temperature Index Cond: ((_hyper_1_2_chunk.timec >= 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) AND (_hyper_1_2_chunk.timec > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone)) Filter: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec) > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone) --plans with CTE :EXPLAIN with m1 as ( Select * from mat_m2 where timec > '2018-10-01' order by timec desc ) select * from m1; --- QUERY PLAN --- Sort Output: _hyper_3_6_chunk.location, _hyper_3_6_chunk.timec, _hyper_3_6_chunk.firsth, _hyper_3_6_chunk.lasth, _hyper_3_6_chunk.maxtemp, _hyper_3_6_chunk.mintemp Sort Key: _hyper_3_6_chunk.timec DESC -> Append -> Index Scan using _hyper_3_6_chunk__materialized_hypertable_3_timec_idx on _timescaledb_internal._hyper_3_6_chunk Output: _hyper_3_6_chunk.location, _hyper_3_6_chunk.timec, _hyper_3_6_chunk.firsth, _hyper_3_6_chunk.lasth, _hyper_3_6_chunk.maxtemp, _hyper_3_6_chunk.mintemp Index Cond: ((_hyper_3_6_chunk.timec < 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) AND (_hyper_3_6_chunk.timec > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone)) -> GroupAggregate Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), first(_hyper_1_2_chunk.humidity, _hyper_1_2_chunk.timec), last(_hyper_1_2_chunk.humidity, _hyper_1_2_chunk.timec), max(_hyper_1_2_chunk.temperature), min(_hyper_1_2_chunk.temperature) Group Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Sort Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.humidity, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.temperature Sort Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Result Output: _hyper_1_2_chunk.location, time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec), _hyper_1_2_chunk.humidity, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.temperature -> Index Scan using _hyper_1_2_chunk_conditions_timec_idx on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.location, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.humidity, _hyper_1_2_chunk.temperature Index Cond: ((_hyper_1_2_chunk.timec >= 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) AND (_hyper_1_2_chunk.timec > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone)) Filter: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec) > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone) -- should reorder mat_m1 group by only based on mat_m1 order-by :EXPLAIN select * from mat_m1, mat_m2 where mat_m1.timec > '2018-10-01' and mat_m1.timec = mat_m2.timec order by mat_m1.timec desc; --- QUERY PLAN --- Sort Output: _hyper_2_4_chunk.location, _hyper_2_4_chunk.timec, _hyper_2_4_chunk.minl, _hyper_2_4_chunk.sumt, _hyper_2_4_chunk.sumh, _materialized_hypertable_3.location, _materialized_hypertable_3.timec, _materialized_hypertable_3.firsth, _materialized_hypertable_3.lasth, _materialized_hypertable_3.maxtemp, _materialized_hypertable_3.mintemp Sort Key: _hyper_2_4_chunk.timec DESC -> Hash Join Output: _hyper_2_4_chunk.location, _hyper_2_4_chunk.timec, _hyper_2_4_chunk.minl, _hyper_2_4_chunk.sumt, _hyper_2_4_chunk.sumh, _materialized_hypertable_3.location, _materialized_hypertable_3.timec, _materialized_hypertable_3.firsth, _materialized_hypertable_3.lasth, _materialized_hypertable_3.maxtemp, _materialized_hypertable_3.mintemp Hash Cond: (_materialized_hypertable_3.timec = _hyper_2_4_chunk.timec) -> Append -> Append -> Seq Scan on _timescaledb_internal._hyper_3_5_chunk Output: _hyper_3_5_chunk.location, _hyper_3_5_chunk.timec, _hyper_3_5_chunk.firsth, _hyper_3_5_chunk.lasth, _hyper_3_5_chunk.maxtemp, _hyper_3_5_chunk.mintemp -> Index Scan using _hyper_3_6_chunk__materialized_hypertable_3_timec_idx on _timescaledb_internal._hyper_3_6_chunk Output: _hyper_3_6_chunk.location, _hyper_3_6_chunk.timec, _hyper_3_6_chunk.firsth, _hyper_3_6_chunk.lasth, _hyper_3_6_chunk.maxtemp, _hyper_3_6_chunk.mintemp Index Cond: (_hyper_3_6_chunk.timec < 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) -> GroupAggregate Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), first(_hyper_1_2_chunk.humidity, _hyper_1_2_chunk.timec), last(_hyper_1_2_chunk.humidity, _hyper_1_2_chunk.timec), max(_hyper_1_2_chunk.temperature), min(_hyper_1_2_chunk.temperature) Group Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Sort Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.humidity, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.temperature Sort Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Result Output: _hyper_1_2_chunk.location, time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec), _hyper_1_2_chunk.humidity, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.temperature -> Index Scan using _hyper_1_2_chunk_conditions_timec_idx on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.location, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.humidity, _hyper_1_2_chunk.temperature Index Cond: (_hyper_1_2_chunk.timec >= 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) -> Hash Output: _hyper_2_4_chunk.location, _hyper_2_4_chunk.timec, _hyper_2_4_chunk.minl, _hyper_2_4_chunk.sumt, _hyper_2_4_chunk.sumh -> Append -> Index Scan using _hyper_2_4_chunk__materialized_hypertable_2_timec_idx on _timescaledb_internal._hyper_2_4_chunk Output: _hyper_2_4_chunk.location, _hyper_2_4_chunk.timec, _hyper_2_4_chunk.minl, _hyper_2_4_chunk.sumt, _hyper_2_4_chunk.sumh Index Cond: ((_hyper_2_4_chunk.timec < 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) AND (_hyper_2_4_chunk.timec > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone)) -> GroupAggregate Output: _hyper_1_2_chunk_1.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk_1.timec)), min(_hyper_1_2_chunk_1.location), sum(_hyper_1_2_chunk_1.temperature), sum(_hyper_1_2_chunk_1.humidity) Group Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk_1.timec)), _hyper_1_2_chunk_1.location -> Sort Output: _hyper_1_2_chunk_1.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk_1.timec)), _hyper_1_2_chunk_1.temperature, _hyper_1_2_chunk_1.humidity Sort Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk_1.timec)), _hyper_1_2_chunk_1.location -> Result Output: _hyper_1_2_chunk_1.location, time_bucket('@ 1 day'::interval, _hyper_1_2_chunk_1.timec), _hyper_1_2_chunk_1.temperature, _hyper_1_2_chunk_1.humidity -> Index Scan using _hyper_1_2_chunk_conditions_timec_idx on _timescaledb_internal._hyper_1_2_chunk _hyper_1_2_chunk_1 Output: _hyper_1_2_chunk_1.location, _hyper_1_2_chunk_1.timec, _hyper_1_2_chunk_1.temperature, _hyper_1_2_chunk_1.humidity Index Cond: ((_hyper_1_2_chunk_1.timec >= 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) AND (_hyper_1_2_chunk_1.timec > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone)) Filter: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk_1.timec) > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone) --should reorder only for mat_m1. :EXPLAIN select * from mat_m1, regview where mat_m1.timec > '2018-10-01' and mat_m1.timec = regview.timec order by mat_m1.timec desc; --- QUERY PLAN --- Sort Output: _hyper_2_4_chunk.location, _hyper_2_4_chunk.timec, _hyper_2_4_chunk.minl, _hyper_2_4_chunk.sumt, _hyper_2_4_chunk.sumh, conditions.location, (time_bucket('@ 1 day'::interval, conditions.timec)), (min(conditions.location)), (sum(conditions.temperature)), (sum(conditions.humidity)) Sort Key: _hyper_2_4_chunk.timec DESC -> Hash Join Output: _hyper_2_4_chunk.location, _hyper_2_4_chunk.timec, _hyper_2_4_chunk.minl, _hyper_2_4_chunk.sumt, _hyper_2_4_chunk.sumh, conditions.location, (time_bucket('@ 1 day'::interval, conditions.timec)), (min(conditions.location)), (sum(conditions.temperature)), (sum(conditions.humidity)) Hash Cond: ((time_bucket('@ 1 day'::interval, conditions.timec)) = _hyper_2_4_chunk.timec) -> Finalize GroupAggregate Output: conditions.location, (time_bucket('@ 1 day'::interval, conditions.timec)), min(conditions.location), sum(conditions.temperature), sum(conditions.humidity) Group Key: conditions.location, (time_bucket('@ 1 day'::interval, conditions.timec)) -> Sort Output: conditions.location, (time_bucket('@ 1 day'::interval, conditions.timec)), (PARTIAL min(conditions.location)), (PARTIAL sum(conditions.temperature)), (PARTIAL sum(conditions.humidity)) Sort Key: conditions.location, (time_bucket('@ 1 day'::interval, conditions.timec)) -> Append -> Partial GroupAggregate Output: _hyper_1_1_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_1_chunk.timec)), PARTIAL min(_hyper_1_1_chunk.location), PARTIAL sum(_hyper_1_1_chunk.temperature), PARTIAL sum(_hyper_1_1_chunk.humidity) Group Key: _hyper_1_1_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_1_chunk.timec)) -> Sort Output: _hyper_1_1_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_1_chunk.timec)), _hyper_1_1_chunk.temperature, _hyper_1_1_chunk.humidity Sort Key: _hyper_1_1_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_1_chunk.timec)) -> Seq Scan on _timescaledb_internal._hyper_1_1_chunk Output: _hyper_1_1_chunk.location, time_bucket('@ 1 day'::interval, _hyper_1_1_chunk.timec), _hyper_1_1_chunk.temperature, _hyper_1_1_chunk.humidity -> Partial GroupAggregate Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), PARTIAL min(_hyper_1_2_chunk.location), PARTIAL sum(_hyper_1_2_chunk.temperature), PARTIAL sum(_hyper_1_2_chunk.humidity) Group Key: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)) -> Sort Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity Sort Key: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)) -> Seq Scan on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.location, time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec), _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity -> Hash Output: _hyper_2_4_chunk.location, _hyper_2_4_chunk.timec, _hyper_2_4_chunk.minl, _hyper_2_4_chunk.sumt, _hyper_2_4_chunk.sumh -> Append -> Index Scan using _hyper_2_4_chunk__materialized_hypertable_2_timec_idx on _timescaledb_internal._hyper_2_4_chunk Output: _hyper_2_4_chunk.location, _hyper_2_4_chunk.timec, _hyper_2_4_chunk.minl, _hyper_2_4_chunk.sumt, _hyper_2_4_chunk.sumh Index Cond: ((_hyper_2_4_chunk.timec < 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) AND (_hyper_2_4_chunk.timec > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone)) -> GroupAggregate Output: _hyper_1_2_chunk_1.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk_1.timec)), min(_hyper_1_2_chunk_1.location), sum(_hyper_1_2_chunk_1.temperature), sum(_hyper_1_2_chunk_1.humidity) Group Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk_1.timec)), _hyper_1_2_chunk_1.location -> Sort Output: _hyper_1_2_chunk_1.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk_1.timec)), _hyper_1_2_chunk_1.temperature, _hyper_1_2_chunk_1.humidity Sort Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk_1.timec)), _hyper_1_2_chunk_1.location -> Result Output: _hyper_1_2_chunk_1.location, time_bucket('@ 1 day'::interval, _hyper_1_2_chunk_1.timec), _hyper_1_2_chunk_1.temperature, _hyper_1_2_chunk_1.humidity -> Index Scan using _hyper_1_2_chunk_conditions_timec_idx on _timescaledb_internal._hyper_1_2_chunk _hyper_1_2_chunk_1 Output: _hyper_1_2_chunk_1.location, _hyper_1_2_chunk_1.timec, _hyper_1_2_chunk_1.temperature, _hyper_1_2_chunk_1.humidity Index Cond: ((_hyper_1_2_chunk_1.timec >= 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) AND (_hyper_1_2_chunk_1.timec > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone)) Filter: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk_1.timec) > 'Mon Oct 01 00:00:00 2018 PDT'::timestamp with time zone) select l.locid, mat_m1.* from mat_m1 , location_tab l where timec > '2018-10-01' and l.locname = mat_m1.location order by timec desc; locid | location | timec | minl | sumt | sumh -------+----------+------------------------------+------+------+------ 2 | NYC | Fri Nov 02 17:00:00 2018 PDT | NYC | | 2 | NYC | Thu Nov 01 17:00:00 2018 PDT | NYC | 30 | 25 2 | NYC | Wed Oct 31 17:00:00 2018 PDT | NYC | 325 | 200 \set ECHO none ---- Run the same queries with hash agg enabled now set enable_hashagg = true; \set ECHO none --- Run the queries directly on the table now set enable_hashagg = true; \set ECHO none -- diff results view select and table select :DIFF_CMD :DIFF_CMD2 --check if the guc works , reordering will not work set timescaledb.enable_cagg_reorder_groupby = false; set enable_hashagg = false; :EXPLAIN select * from mat_m1 order by timec desc, location; --- QUERY PLAN --- Sort Output: _materialized_hypertable_2.location, _materialized_hypertable_2.timec, _materialized_hypertable_2.minl, _materialized_hypertable_2.sumt, _materialized_hypertable_2.sumh Sort Key: _materialized_hypertable_2.timec DESC, _materialized_hypertable_2.location -> Append -> Append -> Seq Scan on _timescaledb_internal._hyper_2_3_chunk Output: _hyper_2_3_chunk.location, _hyper_2_3_chunk.timec, _hyper_2_3_chunk.minl, _hyper_2_3_chunk.sumt, _hyper_2_3_chunk.sumh -> Index Scan using _hyper_2_4_chunk__materialized_hypertable_2_timec_idx on _timescaledb_internal._hyper_2_4_chunk Output: _hyper_2_4_chunk.location, _hyper_2_4_chunk.timec, _hyper_2_4_chunk.minl, _hyper_2_4_chunk.sumt, _hyper_2_4_chunk.sumh Index Cond: (_hyper_2_4_chunk.timec < 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) -> GroupAggregate Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), min(_hyper_1_2_chunk.location), sum(_hyper_1_2_chunk.temperature), sum(_hyper_1_2_chunk.humidity) Group Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Sort Output: _hyper_1_2_chunk.location, (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity Sort Key: (time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec)), _hyper_1_2_chunk.location -> Result Output: _hyper_1_2_chunk.location, time_bucket('@ 1 day'::interval, _hyper_1_2_chunk.timec), _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity -> Index Scan using _hyper_1_2_chunk_conditions_timec_idx on _timescaledb_internal._hyper_1_2_chunk Output: _hyper_1_2_chunk.location, _hyper_1_2_chunk.timec, _hyper_1_2_chunk.temperature, _hyper_1_2_chunk.humidity Index Cond: (_hyper_1_2_chunk.timec >= 'Sat Nov 03 17:00:00 2018 PDT'::timestamp with time zone) ----------------------------------------------------------------------- -- Test the cagg_watermark function. The watermark gives the point -- where to UNION raw and materialized data in real-time -- aggregation. Specifically, test that the watermark caching works as -- expected. ----------------------------------------------------------------------- -- Insert some more data so that there is something to UNION in -- real-time aggregation. insert into conditions values ( '2018-12-02 20:10:00-08', 'SFO', 55, 45); insert into conditions values ( '2018-12-02 21:20:00-08', 'SFO', 65, 45); insert into conditions values ( '2018-12-02 20:30:00-08', 'NYC', 65, 45); insert into conditions values ( '2018-12-02 21:50:00-08', 'NYC', 45, 30); -- Test join of two caggs. Joining two caggs will force the cache to -- reset every time the watermark function is invoked on a different -- cagg in the same query. SELECT mat_hypertable_id AS mat_id, raw_hypertable_id AS raw_id, schema_name AS mat_schema, table_name AS mat_name, format('%I.%I', schema_name, table_name) AS mat_table FROM _timescaledb_catalog.continuous_agg ca, _timescaledb_catalog.hypertable h WHERE user_view_name='mat_m1' AND h.id = ca.mat_hypertable_id \gset BEGIN; -- Query without join SELECT m1.location, m1.timec, sumt, sumh FROM mat_m1 m1 ORDER BY m1.location COLLATE "C", m1.timec DESC LIMIT 10; location | timec | sumt | sumh ----------+------------------------------+------+------ NYC | Sun Dec 02 16:00:00 2018 PST | 110 | 75 NYC | Fri Nov 02 17:00:00 2018 PDT | | NYC | Thu Nov 01 17:00:00 2018 PDT | 30 | 25 NYC | Wed Oct 31 17:00:00 2018 PDT | 325 | 200 NYC | Mon Jan 01 16:00:00 2018 PST | 65 | 45 SFO | Sun Dec 02 16:00:00 2018 PST | 120 | 90 SFO | Mon Jan 01 16:00:00 2018 PST | 65 | 45 SFO | Sun Dec 31 16:00:00 2017 PST | 55 | 45 por | Mon Jan 01 16:00:00 2018 PST | 100 | 100 -- Query that joins two caggs. This should force the watermark cache -- to reset when the materialized hypertable ID changes. A hash join -- could potentially read all values from mat_m1 then all values from -- mat_m2. This would be the optimal situation for cagg_watermark -- caching. We want to avoid it in tests to see that caching doesn't -- do anything wrong in worse situations (e.g., a nested loop join). SET enable_hashjoin=false; SELECT m1.location, m1.timec, sumt, sumh, firsth, lasth, maxtemp, mintemp FROM mat_m1 m1 RIGHT JOIN mat_m2 m2 ON (m1.location = m2.location AND m1.timec = m2.timec) ORDER BY m1.location COLLATE "C", m1.timec DESC LIMIT 10; location | timec | sumt | sumh | firsth | lasth | maxtemp | mintemp ----------+------------------------------+------+------+--------+-------+---------+--------- NYC | Sun Dec 02 16:00:00 2018 PST | 110 | 75 | 45 | 30 | 65 | 45 NYC | Fri Nov 02 17:00:00 2018 PDT | | | | | | NYC | Thu Nov 01 17:00:00 2018 PDT | 30 | 25 | 10 | | 20 | 10 NYC | Wed Oct 31 17:00:00 2018 PDT | 325 | 200 | 30 | 50 | 85 | 45 NYC | Mon Jan 01 16:00:00 2018 PST | 65 | 45 | 45 | 45 | 65 | 65 SFO | Sun Dec 02 16:00:00 2018 PST | 120 | 90 | 45 | 45 | 65 | 55 SFO | Mon Jan 01 16:00:00 2018 PST | 65 | 45 | 45 | 45 | 65 | 65 SFO | Sun Dec 31 16:00:00 2017 PST | 55 | 45 | 45 | 45 | 55 | 55 por | Mon Jan 01 16:00:00 2018 PST | 100 | 100 | 100 | 100 | 100 | 100 -- Show the current watermark SELECT _timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_id)); to_timestamp ------------------------------ Sat Nov 03 17:00:00 2018 PDT -- The watermark should, in this case, be the same as the invalidation -- threshold SELECT _timescaledb_functions.to_timestamp(watermark) FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id = :raw_id; to_timestamp ------------------------------ Sat Nov 03 17:00:00 2018 PDT -- The watermark is the end of materialization (end of last bucket) -- while the MAX is the start of the last bucket SELECT max(timec) FROM :mat_table; max ------------------------------ Fri Nov 02 17:00:00 2018 PDT -- Drop the most recent chunk SELECT chunk_name, range_start, range_end FROM timescaledb_information.chunks WHERE hypertable_name = :'mat_name'; chunk_name | range_start | range_end ------------------+------------------------------+------------------------------ _hyper_2_3_chunk | Wed Nov 29 16:00:00 2017 PST | Wed Feb 07 16:00:00 2018 PST _hyper_2_4_chunk | Wed Sep 05 17:00:00 2018 PDT | Wed Nov 14 16:00:00 2018 PST SELECT drop_chunks('mat_m1', newer_than=>'2018-01-01'::timestamptz); drop_chunks ---------------------------------------- _timescaledb_internal._hyper_2_4_chunk SELECT chunk_name, range_start, range_end FROM timescaledb_information.chunks WHERE hypertable_name = :'mat_name'; chunk_name | range_start | range_end ------------------+------------------------------+------------------------------ _hyper_2_3_chunk | Wed Nov 29 16:00:00 2017 PST | Wed Feb 07 16:00:00 2018 PST -- The watermark should be updated to reflect the dropped data (i.e., -- the cache should be reset) SELECT _timescaledb_functions.to_timestamp(_timescaledb_functions.cagg_watermark(:mat_id)); to_timestamp ------------------------------ Tue Jan 02 16:00:00 2018 PST -- Since we removed the last chunk, the invalidation threshold doesn't -- move back, while the watermark does. SELECT _timescaledb_functions.to_timestamp(watermark) FROM _timescaledb_catalog.continuous_aggs_invalidation_threshold WHERE hypertable_id = :raw_id; to_timestamp ------------------------------ Sat Nov 03 17:00:00 2018 PDT -- Compare the new watermark to the MAX time in the table SELECT max(timec) FROM :mat_table; max ------------------------------ Mon Jan 01 16:00:00 2018 PST -- Try a subtransaction SAVEPOINT clear_cagg; SELECT m1.location, m1.timec, sumt, sumh, firsth, lasth, maxtemp, mintemp FROM mat_m1 m1 RIGHT JOIN mat_m2 m2 ON (m1.location = m2.location AND m1.timec = m2.timec) ORDER BY m1.location COLLATE "C", m1.timec DESC LIMIT 10; location | timec | sumt | sumh | firsth | lasth | maxtemp | mintemp ----------+------------------------------+------+------+--------+-------+---------+--------- NYC | Sun Dec 02 16:00:00 2018 PST | 110 | 75 | 45 | 30 | 65 | 45 NYC | Fri Nov 02 17:00:00 2018 PDT | | | | | | NYC | Thu Nov 01 17:00:00 2018 PDT | 30 | 25 | 10 | | 20 | 10 NYC | Wed Oct 31 17:00:00 2018 PDT | 325 | 200 | 30 | 50 | 85 | 45 NYC | Mon Jan 01 16:00:00 2018 PST | 65 | 45 | 45 | 45 | 65 | 65 SFO | Sun Dec 02 16:00:00 2018 PST | 120 | 90 | 45 | 45 | 65 | 55 SFO | Mon Jan 01 16:00:00 2018 PST | 65 | 45 | 45 | 45 | 65 | 65 SFO | Sun Dec 31 16:00:00 2017 PST | 55 | 45 | 45 | 45 | 55 | 55 por | Mon Jan 01 16:00:00 2018 PST | 100 | 100 | 100 | 100 | 100 | 100 ALTER MATERIALIZED VIEW mat_m1 SET (timescaledb.materialized_only=true); SELECT m1.location, m1.timec, sumt, sumh, firsth, lasth, maxtemp, mintemp FROM mat_m1 m1 RIGHT JOIN mat_m2 m2 ON (m1.location = m2.location AND m1.timec = m2.timec) ORDER BY m1.location COLLATE "C" NULLS LAST, m1.timec DESC NULLS LAST, firsth NULLS LAST, lasth NULLS LAST, mintemp NULLS LAST, maxtemp NULLS LAST LIMIT 10; location | timec | sumt | sumh | firsth | lasth | maxtemp | mintemp ----------+------------------------------+------+------+--------+-------+---------+--------- NYC | Mon Jan 01 16:00:00 2018 PST | 65 | 45 | 45 | 45 | 65 | 65 SFO | Mon Jan 01 16:00:00 2018 PST | 65 | 45 | 45 | 45 | 65 | 65 SFO | Sun Dec 31 16:00:00 2017 PST | 55 | 45 | 45 | 45 | 55 | 55 por | Mon Jan 01 16:00:00 2018 PST | 100 | 100 | 100 | 100 | 100 | 100 | | | | 10 | | 20 | 10 | | | | 30 | 50 | 85 | 45 | | | | 45 | 30 | 65 | 45 | | | | 45 | 45 | 65 | 55 | | | | | | | ROLLBACK; ----- -- Tests with time_bucket and offset/origin ----- CREATE TABLE temperature ( time timestamptz NOT NULL, value float ); SELECT create_hypertable('temperature', 'time'); create_hypertable -------------------------- (4,public,temperature,t) INSERT INTO temperature VALUES ('2000-01-01 01:00:00'::timestamptz, 5); CREATE TABLE temperature_wo_tz ( time timestamp NOT NULL, value float ); SELECT create_hypertable('temperature_wo_tz', 'time'); psql:include/cagg_query_common.sql:316: WARNING: column type "timestamp without time zone" used for "time" does not follow best practices create_hypertable -------------------------------- (5,public,temperature_wo_tz,t) INSERT INTO temperature_wo_tz VALUES ('2000-01-01 01:00:00'::timestamp, 5); CREATE TABLE temperature_date ( time date NOT NULL, value float ); SELECT create_hypertable('temperature_date', 'time'); create_hypertable ------------------------------- (6,public,temperature_date,t) INSERT INTO temperature_date VALUES ('2000-01-01 01:00:00'::timestamp, 5); -- Integer based tables CREATE TABLE table_smallint ( time smallint, data smallint ); CREATE TABLE table_int ( time int, data int ); CREATE TABLE table_bigint ( time bigint, data bigint ); SELECT create_hypertable('table_smallint', 'time', chunk_time_interval => 10); create_hypertable ----------------------------- (7,public,table_smallint,t) SELECT create_hypertable('table_int', 'time', chunk_time_interval => 10); create_hypertable ------------------------ (8,public,table_int,t) SELECT create_hypertable('table_bigint', 'time', chunk_time_interval => 10); create_hypertable --------------------------- (9,public,table_bigint,t) CREATE OR REPLACE FUNCTION integer_now_smallint() returns smallint LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), 0) FROM table_smallint $$; CREATE OR REPLACE FUNCTION integer_now_int() returns int LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), 0) FROM table_int $$; CREATE OR REPLACE FUNCTION integer_now_bigint() returns bigint LANGUAGE SQL STABLE as $$ SELECT coalesce(max(time), 0) FROM table_bigint $$; SELECT set_integer_now_func('table_smallint', 'integer_now_smallint'); set_integer_now_func ---------------------- SELECT set_integer_now_func('table_int', 'integer_now_int'); set_integer_now_func ---------------------- SELECT set_integer_now_func('table_bigint', 'integer_now_bigint'); set_integer_now_func ---------------------- INSERT INTO table_smallint VALUES(1,2); INSERT INTO table_int VALUES(1,2); INSERT INTO table_bigint VALUES(1,2); CREATE VIEW caggs_info AS SELECT user_view_schema, user_view_name, bucket_func, bucket_width, bucket_origin, bucket_offset, bucket_timezone, bucket_fixed_width FROM _timescaledb_catalog.continuous_aggs_bucket_function NATURAL JOIN _timescaledb_catalog.continuous_agg; --- -- Tests with CAgg creation --- CREATE MATERIALIZED VIEW cagg_4_hours WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('4 hour', time), max(value) FROM temperature GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:372: NOTICE: refreshing continuous aggregate "cagg_4_hours" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_4_hours'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+----------------+-------------------------------------------------------+--------------+---------------+---------------+-----------------+-------------------- public | cagg_4_hours | public.time_bucket(interval,timestamp with time zone) | @ 4 hours | | | | t DROP MATERIALIZED VIEW cagg_4_hours; psql:include/cagg_query_common.sql:374: NOTICE: drop cascades to table _timescaledb_internal._hyper_10_14_chunk CREATE MATERIALIZED VIEW cagg_4_hours_offset WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('4 hour', time, '30m'::interval), max(value) FROM temperature GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:380: NOTICE: refreshing continuous aggregate "cagg_4_hours_offset" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_4_hours_offset'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+---------------------+----------------------------------------------------------------+--------------+---------------+---------------+-----------------+-------------------- public | cagg_4_hours_offset | public.time_bucket(interval,timestamp with time zone,interval) | @ 4 hours | | @ 30 mins | | t DROP MATERIALIZED VIEW cagg_4_hours_offset; psql:include/cagg_query_common.sql:382: NOTICE: drop cascades to table _timescaledb_internal._hyper_11_15_chunk CREATE MATERIALIZED VIEW cagg_4_hours_offset2 WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('4 hour', time, "offset"=>'30m'::interval), max(value) FROM temperature GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:388: NOTICE: refreshing continuous aggregate "cagg_4_hours_offset2" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_4_hours_offset2'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+----------------------+----------------------------------------------------------------+--------------+---------------+---------------+-----------------+-------------------- public | cagg_4_hours_offset2 | public.time_bucket(interval,timestamp with time zone,interval) | @ 4 hours | | @ 30 mins | | t DROP MATERIALIZED VIEW cagg_4_hours_offset2; psql:include/cagg_query_common.sql:390: NOTICE: drop cascades to table _timescaledb_internal._hyper_12_16_chunk -- Variable buckets (timezone is provided) with offset CREATE MATERIALIZED VIEW cagg_4_hours_offset_ts WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('4 hour', time, "offset"=>'30m'::interval, timezone=>'UTC'), max(value) FROM temperature GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:397: NOTICE: refreshing continuous aggregate "cagg_4_hours_offset_ts" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_4_hours_offset_ts'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+------------------------+---------------------------------------------------------------------------------------------------------+--------------+---------------+---------------+-----------------+-------------------- public | cagg_4_hours_offset_ts | public.time_bucket(interval,timestamp with time zone,pg_catalog.text,timestamp with time zone,interval) | @ 4 hours | | @ 30 mins | UTC | f DROP MATERIALIZED VIEW cagg_4_hours_offset_ts; psql:include/cagg_query_common.sql:399: NOTICE: drop cascades to table _timescaledb_internal._hyper_13_17_chunk CREATE MATERIALIZED VIEW cagg_4_hours_origin WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max(value) FROM temperature GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:405: NOTICE: refreshing continuous aggregate "cagg_4_hours_origin" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_4_hours_origin'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+---------------------+--------------------------------------------------------------------------------+--------------+------------------------------+---------------+-----------------+-------------------- public | cagg_4_hours_origin | public.time_bucket(interval,timestamp with time zone,timestamp with time zone) | @ 4 hours | Sat Jan 01 01:00:00 2000 PST | | | t DROP MATERIALIZED VIEW cagg_4_hours_origin; psql:include/cagg_query_common.sql:407: NOTICE: drop cascades to table _timescaledb_internal._hyper_14_18_chunk -- Using named parameter CREATE MATERIALIZED VIEW cagg_4_hours_origin2 WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('4 hour', time, origin=>'2000-01-01 01:00:00 PST'::timestamptz), max(value) FROM temperature GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:414: NOTICE: refreshing continuous aggregate "cagg_4_hours_origin2" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_4_hours_origin2'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+----------------------+--------------------------------------------------------------------------------+--------------+------------------------------+---------------+-----------------+-------------------- public | cagg_4_hours_origin2 | public.time_bucket(interval,timestamp with time zone,timestamp with time zone) | @ 4 hours | Sat Jan 01 01:00:00 2000 PST | | | t DROP MATERIALIZED VIEW cagg_4_hours_origin2; psql:include/cagg_query_common.sql:416: NOTICE: drop cascades to table _timescaledb_internal._hyper_15_19_chunk -- Variable buckets (timezone is provided) with origin CREATE MATERIALIZED VIEW cagg_4_hours_origin_ts WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('4 hour', time, origin=>'2000-01-01 01:00:00 PST'::timestamptz, timezone=>'UTC'), max(value) FROM temperature GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:423: NOTICE: refreshing continuous aggregate "cagg_4_hours_origin_ts" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_4_hours_origin_ts'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+------------------------+---------------------------------------------------------------------------------------------------------+--------------+------------------------------+---------------+-----------------+-------------------- public | cagg_4_hours_origin_ts | public.time_bucket(interval,timestamp with time zone,pg_catalog.text,timestamp with time zone,interval) | @ 4 hours | Sat Jan 01 01:00:00 2000 PST | | UTC | f DROP MATERIALIZED VIEW cagg_4_hours_origin_ts; psql:include/cagg_query_common.sql:425: NOTICE: drop cascades to table _timescaledb_internal._hyper_16_20_chunk -- Without named parameter CREATE MATERIALIZED VIEW cagg_4_hours_origin_ts2 WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('4 hour', time, 'UTC', '2000-01-01 01:00:00 PST'::timestamptz), max(value) FROM temperature GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:432: NOTICE: refreshing continuous aggregate "cagg_4_hours_origin_ts2" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_4_hours_origin_ts2'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+-------------------------+---------------------------------------------------------------------------------------------------------+--------------+------------------------------+---------------+-----------------+-------------------- public | cagg_4_hours_origin_ts2 | public.time_bucket(interval,timestamp with time zone,pg_catalog.text,timestamp with time zone,interval) | @ 4 hours | Sat Jan 01 01:00:00 2000 PST | | UTC | f DROP MATERIALIZED VIEW cagg_4_hours_origin_ts2; psql:include/cagg_query_common.sql:434: NOTICE: drop cascades to table _timescaledb_internal._hyper_17_21_chunk -- Timestamp based CAggs CREATE MATERIALIZED VIEW cagg_4_hours_wo_tz WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('4 hour', time), max(value) FROM temperature_wo_tz GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:441: NOTICE: refreshing continuous aggregate "cagg_4_hours_wo_tz" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_4_hours_wo_tz'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+--------------------+----------------------------------------------------------+--------------+---------------+---------------+-----------------+-------------------- public | cagg_4_hours_wo_tz | public.time_bucket(interval,timestamp without time zone) | @ 4 hours | | | | t CREATE MATERIALIZED VIEW cagg_4_hours_origin_ts_wo_tz WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00'::timestamp), max(value) FROM temperature_wo_tz GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:448: NOTICE: refreshing continuous aggregate "cagg_4_hours_origin_ts_wo_tz" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_4_hours_origin_ts_wo_tz'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+------------------------------+--------------------------------------------------------------------------------------+--------------+------------------------------+---------------+-----------------+-------------------- public | cagg_4_hours_origin_ts_wo_tz | public.time_bucket(interval,timestamp without time zone,timestamp without time zone) | @ 4 hours | Fri Dec 31 17:00:00 1999 PST | | | t DROP MATERIALIZED VIEW cagg_4_hours_origin_ts_wo_tz; psql:include/cagg_query_common.sql:450: NOTICE: drop cascades to table _timescaledb_internal._hyper_19_23_chunk -- Variable buckets (timezone is provided) with origin CREATE MATERIALIZED VIEW cagg_4_hours_origin_ts_wo_tz2 WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('4 hour', time, origin=>'2000-01-01 01:00:00'::timestamp), max(value) FROM temperature_wo_tz GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:457: NOTICE: refreshing continuous aggregate "cagg_4_hours_origin_ts_wo_tz2" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_4_hours_origin_ts_wo_tz2'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+-------------------------------+--------------------------------------------------------------------------------------+--------------+------------------------------+---------------+-----------------+-------------------- public | cagg_4_hours_origin_ts_wo_tz2 | public.time_bucket(interval,timestamp without time zone,timestamp without time zone) | @ 4 hours | Fri Dec 31 17:00:00 1999 PST | | | t DROP MATERIALIZED VIEW cagg_4_hours_origin_ts_wo_tz2; psql:include/cagg_query_common.sql:459: NOTICE: drop cascades to table _timescaledb_internal._hyper_20_24_chunk CREATE MATERIALIZED VIEW cagg_4_hours_offset_wo_tz WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('4 hour', time, "offset"=>'30m'::interval), max(value) FROM temperature_wo_tz GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:465: NOTICE: refreshing continuous aggregate "cagg_4_hours_offset_wo_tz" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_4_hours_offset_wo_tz'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+---------------------------+-------------------------------------------------------------------+--------------+---------------+---------------+-----------------+-------------------- public | cagg_4_hours_offset_wo_tz | public.time_bucket(interval,timestamp without time zone,interval) | @ 4 hours | | @ 30 mins | | t DROP MATERIALIZED VIEW cagg_4_hours_offset_wo_tz; psql:include/cagg_query_common.sql:467: NOTICE: drop cascades to table _timescaledb_internal._hyper_21_25_chunk DROP MATERIALIZED VIEW cagg_4_hours_wo_tz; psql:include/cagg_query_common.sql:468: NOTICE: drop cascades to table _timescaledb_internal._hyper_18_22_chunk -- Date based CAggs CREATE MATERIALIZED VIEW cagg_4_hours_date WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('4 days', time), max(value) FROM temperature_date GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:475: NOTICE: refreshing continuous aggregate "cagg_4_hours_date" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_4_hours_date'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+-------------------+----------------------------------------------+--------------+---------------+---------------+-----------------+-------------------- public | cagg_4_hours_date | public.time_bucket(interval,pg_catalog.date) | @ 4 days | | | | t DROP MATERIALIZED VIEW cagg_4_hours_date; psql:include/cagg_query_common.sql:477: NOTICE: drop cascades to table _timescaledb_internal._hyper_22_26_chunk CREATE MATERIALIZED VIEW cagg_4_hours_date_origin WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('4 days', time, '2000-01-01'::date), max(value) FROM temperature_date GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:483: NOTICE: refreshing continuous aggregate "cagg_4_hours_date_origin" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_4_hours_date_origin'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+--------------------------+--------------------------------------------------------------+--------------+------------------------------+---------------+-----------------+-------------------- public | cagg_4_hours_date_origin | public.time_bucket(interval,pg_catalog.date,pg_catalog.date) | @ 4 days | Sat Jan 01 00:00:00 2000 PST | | | t DROP MATERIALIZED VIEW cagg_4_hours_date_origin; psql:include/cagg_query_common.sql:485: NOTICE: drop cascades to table _timescaledb_internal._hyper_23_27_chunk CREATE MATERIALIZED VIEW cagg_4_hours_date_origin2 WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('4 days', time, origin=>'2000-01-01'::date), max(value) FROM temperature_date GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:491: NOTICE: refreshing continuous aggregate "cagg_4_hours_date_origin2" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_4_hours_date_origin2'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+---------------------------+--------------------------------------------------------------+--------------+------------------------------+---------------+-----------------+-------------------- public | cagg_4_hours_date_origin2 | public.time_bucket(interval,pg_catalog.date,pg_catalog.date) | @ 4 days | Sat Jan 01 00:00:00 2000 PST | | | t DROP MATERIALIZED VIEW cagg_4_hours_date_origin2; psql:include/cagg_query_common.sql:493: NOTICE: drop cascades to table _timescaledb_internal._hyper_24_28_chunk CREATE MATERIALIZED VIEW cagg_4_hours_date_offset WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('4 days', time, "offset"=>'30m'::interval), max(value) FROM temperature_date GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:499: NOTICE: refreshing continuous aggregate "cagg_4_hours_date_offset" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_4_hours_date_offset'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+--------------------------+-------------------------------------------------------+--------------+---------------+---------------+-----------------+-------------------- public | cagg_4_hours_date_offset | public.time_bucket(interval,pg_catalog.date,interval) | @ 4 days | | @ 30 mins | | t DROP MATERIALIZED VIEW cagg_4_hours_date_offset; psql:include/cagg_query_common.sql:501: NOTICE: drop cascades to table _timescaledb_internal._hyper_25_29_chunk -- Integer based CAggs CREATE MATERIALIZED VIEW cagg_smallint WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time), SUM(data) as value FROM table_smallint GROUP BY 1; psql:include/cagg_query_common.sql:508: NOTICE: refreshing continuous aggregate "cagg_smallint" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_smallint'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+----------------+---------------------------------------+--------------+---------------+---------------+-----------------+-------------------- public | cagg_smallint | public.time_bucket(smallint,smallint) | 2 | | | | t DROP MATERIALIZED VIEW cagg_smallint; psql:include/cagg_query_common.sql:510: NOTICE: drop cascades to table _timescaledb_internal._hyper_26_30_chunk CREATE MATERIALIZED VIEW cagg_smallint_offset WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time, "offset"=>1::smallint), SUM(data) as value FROM table_smallint GROUP BY 1; psql:include/cagg_query_common.sql:516: NOTICE: refreshing continuous aggregate "cagg_smallint_offset" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_smallint_offset'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+----------------------+------------------------------------------------+--------------+---------------+---------------+-----------------+-------------------- public | cagg_smallint_offset | public.time_bucket(smallint,smallint,smallint) | 2 | | 1 | | t DROP MATERIALIZED VIEW cagg_smallint_offset; psql:include/cagg_query_common.sql:518: NOTICE: drop cascades to table _timescaledb_internal._hyper_27_31_chunk CREATE MATERIALIZED VIEW cagg_int WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time), SUM(data) as value FROM table_int GROUP BY 1; psql:include/cagg_query_common.sql:524: NOTICE: refreshing continuous aggregate "cagg_int" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_int'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+----------------+-------------------------------------+--------------+---------------+---------------+-----------------+-------------------- public | cagg_int | public.time_bucket(integer,integer) | 2 | | | | t DROP MATERIALIZED VIEW cagg_int; psql:include/cagg_query_common.sql:526: NOTICE: drop cascades to table _timescaledb_internal._hyper_28_32_chunk CREATE MATERIALIZED VIEW cagg_int_offset WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time, "offset"=>1::int), SUM(data) as value FROM table_int GROUP BY 1; psql:include/cagg_query_common.sql:532: NOTICE: refreshing continuous aggregate "cagg_int_offset" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_int_offset'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+-----------------+---------------------------------------------+--------------+---------------+---------------+-----------------+-------------------- public | cagg_int_offset | public.time_bucket(integer,integer,integer) | 2 | | 1 | | t DROP MATERIALIZED VIEW cagg_int_offset; psql:include/cagg_query_common.sql:534: NOTICE: drop cascades to table _timescaledb_internal._hyper_29_33_chunk CREATE MATERIALIZED VIEW cagg_bigint WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time), SUM(data) as value FROM table_bigint GROUP BY 1 WITH NO DATA; SELECT * FROM caggs_info WHERE user_view_name = 'cagg_bigint'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+----------------+-----------------------------------+--------------+---------------+---------------+-----------------+-------------------- public | cagg_bigint | public.time_bucket(bigint,bigint) | 2 | | | | t DROP MATERIALIZED VIEW cagg_bigint; CREATE MATERIALIZED VIEW cagg_bigint_offset WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time, "offset"=>1::bigint), SUM(data) as value FROM table_bigint GROUP BY 1 WITH NO DATA; SELECT * FROM caggs_info WHERE user_view_name = 'cagg_bigint_offset'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+--------------------+------------------------------------------+--------------+---------------+---------------+-----------------+-------------------- public | cagg_bigint_offset | public.time_bucket(bigint,bigint,bigint) | 2 | | 1 | | t DROP MATERIALIZED VIEW cagg_bigint_offset; -- Without named parameter CREATE MATERIALIZED VIEW cagg_bigint_offset2 WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('2', time, 1::bigint), SUM(data) as value FROM table_bigint GROUP BY 1 WITH NO DATA; SELECT * FROM caggs_info WHERE user_view_name = 'cagg_bigint_offset2'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+---------------------+------------------------------------------+--------------+---------------+---------------+-----------------+-------------------- public | cagg_bigint_offset2 | public.time_bucket(bigint,bigint,bigint) | 2 | | 1 | | t -- mess with the bucket_func signature to make sure it will raise an exception SET ROLE :ROLE_SUPERUSER; \set ON_ERROR_STOP 0 BEGIN; UPDATE _timescaledb_catalog.continuous_aggs_bucket_function SET bucket_func = 'func_does_not_exist()'; -- should error because function does not exist CALL refresh_continuous_aggregate('cagg_bigint_offset2', NULL, NULL); psql:include/cagg_query_common.sql:566: ERROR: function "func_does_not_exist()" does not exist ROLLBACK; \set ON_ERROR_STOP 1 SET ROLE :ROLE_DEFAULT_PERM_USER; DROP MATERIALIZED VIEW cagg_bigint_offset2; -- Test invalid bucket definitions \set ON_ERROR_STOP 0 -- Offset and origin at the same time is not allowed (function does not exists) CREATE MATERIALIZED VIEW cagg_4_hours_offset_and_origin WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('4 hour', time, "offset"=>'30m'::interval, origin=>'2000-01-01 01:00:00 PST'::timestamptz), max(value) FROM temperature GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:580: ERROR: function time_bucket(unknown, timestamp with time zone, offset => interval, origin => timestamp with time zone) does not exist at character 140 -- Offset and origin at the same time is not allowed (function does exists but invalid parameter combination) CREATE MATERIALIZED VIEW cagg_4_hours_offset_and_origin WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('4 hour', time, "offset"=>'30m'::interval, origin=>'2000-01-01 01:00:00 PST'::timestamptz, timezone=>'UTC'), max(value) FROM temperature GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:587: ERROR: using offset and origin in a time_bucket function at the same time is not supported \set ON_ERROR_STOP 1 --- -- Tests with CAgg processing --- -- Check used timezone SHOW timezone; TimeZone ---------- PST8PDT -- Populate it INSERT INTO temperature SELECT time, 5 FROM generate_series('2000-01-01 01:00:00 PST'::timestamptz, '2000-01-01 23:59:59 PST','1m') time; INSERT INTO temperature SELECT time, 6 FROM generate_series('2020-01-01 00:00:00 PST'::timestamptz, '2020-01-01 23:59:59 PST','1m') time; -- Create CAggs CREATE MATERIALIZED VIEW cagg_4_hours WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('4 hour', time), max(value) FROM temperature GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:613: NOTICE: refreshing continuous aggregate "cagg_4_hours" CREATE MATERIALIZED VIEW cagg_4_hours_offset WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('4 hour', time, '30m'::interval), max(value) FROM temperature GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:619: NOTICE: refreshing continuous aggregate "cagg_4_hours_offset" -- Align origin with first value CREATE MATERIALIZED VIEW cagg_4_hours_origin WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max(value) FROM temperature GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:626: NOTICE: refreshing continuous aggregate "cagg_4_hours_origin" -- Query the CAggs and check that all buckets are materialized SELECT time_bucket('4 hour', time), max(value) FROM temperature GROUP BY 1 ORDER BY 1; time_bucket | max ------------------------------+----- Sat Jan 01 00:00:00 2000 PST | 5 Sat Jan 01 04:00:00 2000 PST | 5 Sat Jan 01 08:00:00 2000 PST | 5 Sat Jan 01 12:00:00 2000 PST | 5 Sat Jan 01 16:00:00 2000 PST | 5 Sat Jan 01 20:00:00 2000 PST | 5 Wed Jan 01 00:00:00 2020 PST | 6 Wed Jan 01 04:00:00 2020 PST | 6 Wed Jan 01 08:00:00 2020 PST | 6 Wed Jan 01 12:00:00 2020 PST | 6 Wed Jan 01 16:00:00 2020 PST | 6 Wed Jan 01 20:00:00 2020 PST | 6 SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+----- Sat Jan 01 00:00:00 2000 PST | 5 Sat Jan 01 04:00:00 2000 PST | 5 Sat Jan 01 08:00:00 2000 PST | 5 Sat Jan 01 12:00:00 2000 PST | 5 Sat Jan 01 16:00:00 2000 PST | 5 Sat Jan 01 20:00:00 2000 PST | 5 Wed Jan 01 00:00:00 2020 PST | 6 Wed Jan 01 04:00:00 2020 PST | 6 Wed Jan 01 08:00:00 2020 PST | 6 Wed Jan 01 12:00:00 2020 PST | 6 Wed Jan 01 16:00:00 2020 PST | 6 Wed Jan 01 20:00:00 2020 PST | 6 ALTER MATERIALIZED VIEW cagg_4_hours SET (timescaledb.materialized_only=true); SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+----- Sat Jan 01 00:00:00 2000 PST | 5 Sat Jan 01 04:00:00 2000 PST | 5 Sat Jan 01 08:00:00 2000 PST | 5 Sat Jan 01 12:00:00 2000 PST | 5 Sat Jan 01 16:00:00 2000 PST | 5 Sat Jan 01 20:00:00 2000 PST | 5 Wed Jan 01 00:00:00 2020 PST | 6 Wed Jan 01 04:00:00 2020 PST | 6 Wed Jan 01 08:00:00 2020 PST | 6 Wed Jan 01 12:00:00 2020 PST | 6 Wed Jan 01 16:00:00 2020 PST | 6 Wed Jan 01 20:00:00 2020 PST | 6 SELECT time_bucket('4 hour', time, '30m'::interval), max(value) FROM temperature GROUP BY 1 ORDER BY 1; time_bucket | max ------------------------------+----- Sat Jan 01 00:30:00 2000 PST | 5 Sat Jan 01 04:30:00 2000 PST | 5 Sat Jan 01 08:30:00 2000 PST | 5 Sat Jan 01 12:30:00 2000 PST | 5 Sat Jan 01 16:30:00 2000 PST | 5 Sat Jan 01 20:30:00 2000 PST | 5 Tue Dec 31 20:30:00 2019 PST | 6 Wed Jan 01 00:30:00 2020 PST | 6 Wed Jan 01 04:30:00 2020 PST | 6 Wed Jan 01 08:30:00 2020 PST | 6 Wed Jan 01 12:30:00 2020 PST | 6 Wed Jan 01 16:30:00 2020 PST | 6 Wed Jan 01 20:30:00 2020 PST | 6 SELECT * FROM cagg_4_hours_offset; time_bucket | max ------------------------------+----- Sat Jan 01 00:30:00 2000 PST | 5 Sat Jan 01 04:30:00 2000 PST | 5 Sat Jan 01 08:30:00 2000 PST | 5 Sat Jan 01 12:30:00 2000 PST | 5 Sat Jan 01 16:30:00 2000 PST | 5 Sat Jan 01 20:30:00 2000 PST | 5 Tue Dec 31 20:30:00 2019 PST | 6 Wed Jan 01 00:30:00 2020 PST | 6 Wed Jan 01 04:30:00 2020 PST | 6 Wed Jan 01 08:30:00 2020 PST | 6 Wed Jan 01 12:30:00 2020 PST | 6 Wed Jan 01 16:30:00 2020 PST | 6 Wed Jan 01 20:30:00 2020 PST | 6 ALTER MATERIALIZED VIEW cagg_4_hours_offset SET (timescaledb.materialized_only=true); SELECT * FROM cagg_4_hours_offset; time_bucket | max ------------------------------+----- Sat Jan 01 00:30:00 2000 PST | 5 Sat Jan 01 04:30:00 2000 PST | 5 Sat Jan 01 08:30:00 2000 PST | 5 Sat Jan 01 12:30:00 2000 PST | 5 Sat Jan 01 16:30:00 2000 PST | 5 Sat Jan 01 20:30:00 2000 PST | 5 Tue Dec 31 20:30:00 2019 PST | 6 Wed Jan 01 00:30:00 2020 PST | 6 Wed Jan 01 04:30:00 2020 PST | 6 Wed Jan 01 08:30:00 2020 PST | 6 Wed Jan 01 12:30:00 2020 PST | 6 Wed Jan 01 16:30:00 2020 PST | 6 Wed Jan 01 20:30:00 2020 PST | 6 SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max(value) FROM temperature GROUP BY 1 ORDER BY 1; time_bucket | max ------------------------------+----- Sat Jan 01 01:00:00 2000 PST | 5 Sat Jan 01 05:00:00 2000 PST | 5 Sat Jan 01 09:00:00 2000 PST | 5 Sat Jan 01 13:00:00 2000 PST | 5 Sat Jan 01 17:00:00 2000 PST | 5 Sat Jan 01 21:00:00 2000 PST | 5 Tue Dec 31 21:00:00 2019 PST | 6 Wed Jan 01 01:00:00 2020 PST | 6 Wed Jan 01 05:00:00 2020 PST | 6 Wed Jan 01 09:00:00 2020 PST | 6 Wed Jan 01 13:00:00 2020 PST | 6 Wed Jan 01 17:00:00 2020 PST | 6 Wed Jan 01 21:00:00 2020 PST | 6 SELECT * FROM cagg_4_hours_origin; time_bucket | max ------------------------------+----- Sat Jan 01 01:00:00 2000 PST | 5 Sat Jan 01 05:00:00 2000 PST | 5 Sat Jan 01 09:00:00 2000 PST | 5 Sat Jan 01 13:00:00 2000 PST | 5 Sat Jan 01 17:00:00 2000 PST | 5 Sat Jan 01 21:00:00 2000 PST | 5 Tue Dec 31 21:00:00 2019 PST | 6 Wed Jan 01 01:00:00 2020 PST | 6 Wed Jan 01 05:00:00 2020 PST | 6 Wed Jan 01 09:00:00 2020 PST | 6 Wed Jan 01 13:00:00 2020 PST | 6 Wed Jan 01 17:00:00 2020 PST | 6 Wed Jan 01 21:00:00 2020 PST | 6 ALTER MATERIALIZED VIEW cagg_4_hours_origin SET (timescaledb.materialized_only=true); SELECT * FROM cagg_4_hours_origin; time_bucket | max ------------------------------+----- Sat Jan 01 01:00:00 2000 PST | 5 Sat Jan 01 05:00:00 2000 PST | 5 Sat Jan 01 09:00:00 2000 PST | 5 Sat Jan 01 13:00:00 2000 PST | 5 Sat Jan 01 17:00:00 2000 PST | 5 Sat Jan 01 21:00:00 2000 PST | 5 Tue Dec 31 21:00:00 2019 PST | 6 Wed Jan 01 01:00:00 2020 PST | 6 Wed Jan 01 05:00:00 2020 PST | 6 Wed Jan 01 09:00:00 2020 PST | 6 Wed Jan 01 13:00:00 2020 PST | 6 Wed Jan 01 17:00:00 2020 PST | 6 Wed Jan 01 21:00:00 2020 PST | 6 -- Update the last bucket and re-materialize INSERT INTO temperature values('2020-01-01 23:55:00 PST', 10); CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+----- Sat Jan 01 00:00:00 2000 PST | 5 Sat Jan 01 04:00:00 2000 PST | 5 Sat Jan 01 08:00:00 2000 PST | 5 Sat Jan 01 12:00:00 2000 PST | 5 Sat Jan 01 16:00:00 2000 PST | 5 Sat Jan 01 20:00:00 2000 PST | 5 Wed Jan 01 00:00:00 2020 PST | 6 Wed Jan 01 04:00:00 2020 PST | 6 Wed Jan 01 08:00:00 2020 PST | 6 Wed Jan 01 12:00:00 2020 PST | 6 Wed Jan 01 16:00:00 2020 PST | 6 Wed Jan 01 20:00:00 2020 PST | 10 SELECT * FROM cagg_4_hours_offset; time_bucket | max ------------------------------+----- Sat Jan 01 00:30:00 2000 PST | 5 Sat Jan 01 04:30:00 2000 PST | 5 Sat Jan 01 08:30:00 2000 PST | 5 Sat Jan 01 12:30:00 2000 PST | 5 Sat Jan 01 16:30:00 2000 PST | 5 Sat Jan 01 20:30:00 2000 PST | 5 Tue Dec 31 20:30:00 2019 PST | 6 Wed Jan 01 00:30:00 2020 PST | 6 Wed Jan 01 04:30:00 2020 PST | 6 Wed Jan 01 08:30:00 2020 PST | 6 Wed Jan 01 12:30:00 2020 PST | 6 Wed Jan 01 16:30:00 2020 PST | 6 Wed Jan 01 20:30:00 2020 PST | 10 SELECT * FROM cagg_4_hours_origin; time_bucket | max ------------------------------+----- Sat Jan 01 01:00:00 2000 PST | 5 Sat Jan 01 05:00:00 2000 PST | 5 Sat Jan 01 09:00:00 2000 PST | 5 Sat Jan 01 13:00:00 2000 PST | 5 Sat Jan 01 17:00:00 2000 PST | 5 Sat Jan 01 21:00:00 2000 PST | 5 Tue Dec 31 21:00:00 2019 PST | 6 Wed Jan 01 01:00:00 2020 PST | 6 Wed Jan 01 05:00:00 2020 PST | 6 Wed Jan 01 09:00:00 2020 PST | 6 Wed Jan 01 13:00:00 2020 PST | 6 Wed Jan 01 17:00:00 2020 PST | 6 Wed Jan 01 21:00:00 2020 PST | 10 -- Check the real-time functionality ALTER MATERIALIZED VIEW cagg_4_hours SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_4_hours_offset SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_4_hours_origin SET (timescaledb.materialized_only=false); -- Check watermarks SELECT continuous_agg.user_view_name, continuous_aggs_watermark.watermark, _timescaledb_functions.to_timestamp(watermark) FROM _timescaledb_catalog.continuous_aggs_watermark JOIN _timescaledb_catalog.continuous_agg USING (mat_hypertable_id) WHERE user_view_name LIKE 'cagg_4_hours%' ORDER BY mat_hypertable_id, watermark; user_view_name | watermark | to_timestamp ---------------------+------------------+------------------------------ cagg_4_hours | 1577952000000000 | Thu Jan 02 00:00:00 2020 PST cagg_4_hours_offset | 1577953800000000 | Thu Jan 02 00:30:00 2020 PST cagg_4_hours_origin | 1577955600000000 | Thu Jan 02 01:00:00 2020 PST -- Insert new data INSERT INTO temperature values('2020-01-02 00:10:00 PST', 2222); INSERT INTO temperature values('2020-01-02 05:35:00 PST', 5555); INSERT INTO temperature values('2020-01-02 09:05:00 PST', 8888); -- Watermark is at Thu Jan 02 00:00:00 2020 PST - all inserted tuples should be seen SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+------ Sat Jan 01 00:00:00 2000 PST | 5 Sat Jan 01 04:00:00 2000 PST | 5 Sat Jan 01 08:00:00 2000 PST | 5 Sat Jan 01 12:00:00 2000 PST | 5 Sat Jan 01 16:00:00 2000 PST | 5 Sat Jan 01 20:00:00 2000 PST | 5 Wed Jan 01 00:00:00 2020 PST | 6 Wed Jan 01 04:00:00 2020 PST | 6 Wed Jan 01 08:00:00 2020 PST | 6 Wed Jan 01 12:00:00 2020 PST | 6 Wed Jan 01 16:00:00 2020 PST | 6 Wed Jan 01 20:00:00 2020 PST | 10 Thu Jan 02 00:00:00 2020 PST | 2222 Thu Jan 02 04:00:00 2020 PST | 5555 Thu Jan 02 08:00:00 2020 PST | 8888 -- Watermark is at Thu Jan 02 00:30:00 2020 PST - only two inserted tuples should be seen SELECT * FROM cagg_4_hours_offset; time_bucket | max ------------------------------+------ Sat Jan 01 00:30:00 2000 PST | 5 Sat Jan 01 04:30:00 2000 PST | 5 Sat Jan 01 08:30:00 2000 PST | 5 Sat Jan 01 12:30:00 2000 PST | 5 Sat Jan 01 16:30:00 2000 PST | 5 Sat Jan 01 20:30:00 2000 PST | 5 Tue Dec 31 20:30:00 2019 PST | 6 Wed Jan 01 00:30:00 2020 PST | 6 Wed Jan 01 04:30:00 2020 PST | 6 Wed Jan 01 08:30:00 2020 PST | 6 Wed Jan 01 12:30:00 2020 PST | 6 Wed Jan 01 16:30:00 2020 PST | 6 Wed Jan 01 20:30:00 2020 PST | 10 Thu Jan 02 04:30:00 2020 PST | 5555 Thu Jan 02 08:30:00 2020 PST | 8888 -- Watermark is at Thu Jan 02 01:00:00 2020 PST - only two inserted tuples should be seen SELECT * FROM cagg_4_hours_origin; time_bucket | max ------------------------------+------ Sat Jan 01 01:00:00 2000 PST | 5 Sat Jan 01 05:00:00 2000 PST | 5 Sat Jan 01 09:00:00 2000 PST | 5 Sat Jan 01 13:00:00 2000 PST | 5 Sat Jan 01 17:00:00 2000 PST | 5 Sat Jan 01 21:00:00 2000 PST | 5 Tue Dec 31 21:00:00 2019 PST | 6 Wed Jan 01 01:00:00 2020 PST | 6 Wed Jan 01 05:00:00 2020 PST | 6 Wed Jan 01 09:00:00 2020 PST | 6 Wed Jan 01 13:00:00 2020 PST | 6 Wed Jan 01 17:00:00 2020 PST | 6 Wed Jan 01 21:00:00 2020 PST | 10 Thu Jan 02 05:00:00 2020 PST | 5555 Thu Jan 02 09:00:00 2020 PST | 8888 -- Update materialized data SET client_min_messages TO DEBUG1; CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); psql:include/cagg_query_common.sql:683: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); psql:include/cagg_query_common.sql:683: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Thu Jan 02 00:00:00 2020 PST, Thu Jan 02 12:00:00 2020 PST ] psql:include/cagg_query_common.sql:683: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:683: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); psql:include/cagg_query_common.sql:684: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); psql:include/cagg_query_common.sql:684: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Wed Jan 01 20:30:00 2020 PST, Thu Jan 02 12:30:00 2020 PST ] psql:include/cagg_query_common.sql:684: LOG: deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:684: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); psql:include/cagg_query_common.sql:685: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); psql:include/cagg_query_common.sql:685: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Wed Jan 01 21:00:00 2020 PST, Thu Jan 02 13:00:00 2020 PST ] psql:include/cagg_query_common.sql:685: LOG: deleted 1 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" psql:include/cagg_query_common.sql:685: LOG: inserted 3 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" RESET client_min_messages; psql:include/cagg_query_common.sql:686: LOG: statement: RESET client_min_messages; -- Query the CAggs and check that all buckets are materialized SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+------ Sat Jan 01 00:00:00 2000 PST | 5 Sat Jan 01 04:00:00 2000 PST | 5 Sat Jan 01 08:00:00 2000 PST | 5 Sat Jan 01 12:00:00 2000 PST | 5 Sat Jan 01 16:00:00 2000 PST | 5 Sat Jan 01 20:00:00 2000 PST | 5 Wed Jan 01 00:00:00 2020 PST | 6 Wed Jan 01 04:00:00 2020 PST | 6 Wed Jan 01 08:00:00 2020 PST | 6 Wed Jan 01 12:00:00 2020 PST | 6 Wed Jan 01 16:00:00 2020 PST | 6 Wed Jan 01 20:00:00 2020 PST | 10 Thu Jan 02 00:00:00 2020 PST | 2222 Thu Jan 02 04:00:00 2020 PST | 5555 Thu Jan 02 08:00:00 2020 PST | 8888 ALTER MATERIALIZED VIEW cagg_4_hours SET (timescaledb.materialized_only=true); SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+------ Sat Jan 01 00:00:00 2000 PST | 5 Sat Jan 01 04:00:00 2000 PST | 5 Sat Jan 01 08:00:00 2000 PST | 5 Sat Jan 01 12:00:00 2000 PST | 5 Sat Jan 01 16:00:00 2000 PST | 5 Sat Jan 01 20:00:00 2000 PST | 5 Wed Jan 01 00:00:00 2020 PST | 6 Wed Jan 01 04:00:00 2020 PST | 6 Wed Jan 01 08:00:00 2020 PST | 6 Wed Jan 01 12:00:00 2020 PST | 6 Wed Jan 01 16:00:00 2020 PST | 6 Wed Jan 01 20:00:00 2020 PST | 10 Thu Jan 02 00:00:00 2020 PST | 2222 Thu Jan 02 04:00:00 2020 PST | 5555 Thu Jan 02 08:00:00 2020 PST | 8888 SELECT time_bucket('4 hour', time), max(value) FROM temperature GROUP BY 1 ORDER BY 1; time_bucket | max ------------------------------+------ Sat Jan 01 00:00:00 2000 PST | 5 Sat Jan 01 04:00:00 2000 PST | 5 Sat Jan 01 08:00:00 2000 PST | 5 Sat Jan 01 12:00:00 2000 PST | 5 Sat Jan 01 16:00:00 2000 PST | 5 Sat Jan 01 20:00:00 2000 PST | 5 Wed Jan 01 00:00:00 2020 PST | 6 Wed Jan 01 04:00:00 2020 PST | 6 Wed Jan 01 08:00:00 2020 PST | 6 Wed Jan 01 12:00:00 2020 PST | 6 Wed Jan 01 16:00:00 2020 PST | 6 Wed Jan 01 20:00:00 2020 PST | 10 Thu Jan 02 00:00:00 2020 PST | 2222 Thu Jan 02 04:00:00 2020 PST | 5555 Thu Jan 02 08:00:00 2020 PST | 8888 SELECT * FROM cagg_4_hours_offset; time_bucket | max ------------------------------+------ Sat Jan 01 00:30:00 2000 PST | 5 Sat Jan 01 04:30:00 2000 PST | 5 Sat Jan 01 08:30:00 2000 PST | 5 Sat Jan 01 12:30:00 2000 PST | 5 Sat Jan 01 16:30:00 2000 PST | 5 Sat Jan 01 20:30:00 2000 PST | 5 Tue Dec 31 20:30:00 2019 PST | 6 Wed Jan 01 00:30:00 2020 PST | 6 Wed Jan 01 04:30:00 2020 PST | 6 Wed Jan 01 08:30:00 2020 PST | 6 Wed Jan 01 12:30:00 2020 PST | 6 Wed Jan 01 16:30:00 2020 PST | 6 Wed Jan 01 20:30:00 2020 PST | 2222 Thu Jan 02 04:30:00 2020 PST | 5555 Thu Jan 02 08:30:00 2020 PST | 8888 ALTER MATERIALIZED VIEW cagg_4_hours_offset SET (timescaledb.materialized_only=true); SELECT * FROM cagg_4_hours_offset; time_bucket | max ------------------------------+------ Sat Jan 01 00:30:00 2000 PST | 5 Sat Jan 01 04:30:00 2000 PST | 5 Sat Jan 01 08:30:00 2000 PST | 5 Sat Jan 01 12:30:00 2000 PST | 5 Sat Jan 01 16:30:00 2000 PST | 5 Sat Jan 01 20:30:00 2000 PST | 5 Tue Dec 31 20:30:00 2019 PST | 6 Wed Jan 01 00:30:00 2020 PST | 6 Wed Jan 01 04:30:00 2020 PST | 6 Wed Jan 01 08:30:00 2020 PST | 6 Wed Jan 01 12:30:00 2020 PST | 6 Wed Jan 01 16:30:00 2020 PST | 6 Wed Jan 01 20:30:00 2020 PST | 2222 Thu Jan 02 04:30:00 2020 PST | 5555 Thu Jan 02 08:30:00 2020 PST | 8888 SELECT time_bucket('4 hour', time, '30m'::interval), max(value) FROM temperature GROUP BY 1 ORDER BY 1; time_bucket | max ------------------------------+------ Sat Jan 01 00:30:00 2000 PST | 5 Sat Jan 01 04:30:00 2000 PST | 5 Sat Jan 01 08:30:00 2000 PST | 5 Sat Jan 01 12:30:00 2000 PST | 5 Sat Jan 01 16:30:00 2000 PST | 5 Sat Jan 01 20:30:00 2000 PST | 5 Tue Dec 31 20:30:00 2019 PST | 6 Wed Jan 01 00:30:00 2020 PST | 6 Wed Jan 01 04:30:00 2020 PST | 6 Wed Jan 01 08:30:00 2020 PST | 6 Wed Jan 01 12:30:00 2020 PST | 6 Wed Jan 01 16:30:00 2020 PST | 6 Wed Jan 01 20:30:00 2020 PST | 2222 Thu Jan 02 04:30:00 2020 PST | 5555 Thu Jan 02 08:30:00 2020 PST | 8888 SELECT * FROM cagg_4_hours_origin; time_bucket | max ------------------------------+------ Sat Jan 01 01:00:00 2000 PST | 5 Sat Jan 01 05:00:00 2000 PST | 5 Sat Jan 01 09:00:00 2000 PST | 5 Sat Jan 01 13:00:00 2000 PST | 5 Sat Jan 01 17:00:00 2000 PST | 5 Sat Jan 01 21:00:00 2000 PST | 5 Tue Dec 31 21:00:00 2019 PST | 6 Wed Jan 01 01:00:00 2020 PST | 6 Wed Jan 01 05:00:00 2020 PST | 6 Wed Jan 01 09:00:00 2020 PST | 6 Wed Jan 01 13:00:00 2020 PST | 6 Wed Jan 01 17:00:00 2020 PST | 6 Wed Jan 01 21:00:00 2020 PST | 2222 Thu Jan 02 05:00:00 2020 PST | 5555 Thu Jan 02 09:00:00 2020 PST | 8888 ALTER MATERIALIZED VIEW cagg_4_hours_origin SET (timescaledb.materialized_only=true); SELECT * FROM cagg_4_hours_origin; time_bucket | max ------------------------------+------ Sat Jan 01 01:00:00 2000 PST | 5 Sat Jan 01 05:00:00 2000 PST | 5 Sat Jan 01 09:00:00 2000 PST | 5 Sat Jan 01 13:00:00 2000 PST | 5 Sat Jan 01 17:00:00 2000 PST | 5 Sat Jan 01 21:00:00 2000 PST | 5 Tue Dec 31 21:00:00 2019 PST | 6 Wed Jan 01 01:00:00 2020 PST | 6 Wed Jan 01 05:00:00 2020 PST | 6 Wed Jan 01 09:00:00 2020 PST | 6 Wed Jan 01 13:00:00 2020 PST | 6 Wed Jan 01 17:00:00 2020 PST | 6 Wed Jan 01 21:00:00 2020 PST | 2222 Thu Jan 02 05:00:00 2020 PST | 5555 Thu Jan 02 09:00:00 2020 PST | 8888 SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max(value) FROM temperature GROUP BY 1 ORDER BY 1; time_bucket | max ------------------------------+------ Sat Jan 01 01:00:00 2000 PST | 5 Sat Jan 01 05:00:00 2000 PST | 5 Sat Jan 01 09:00:00 2000 PST | 5 Sat Jan 01 13:00:00 2000 PST | 5 Sat Jan 01 17:00:00 2000 PST | 5 Sat Jan 01 21:00:00 2000 PST | 5 Tue Dec 31 21:00:00 2019 PST | 6 Wed Jan 01 01:00:00 2020 PST | 6 Wed Jan 01 05:00:00 2020 PST | 6 Wed Jan 01 09:00:00 2020 PST | 6 Wed Jan 01 13:00:00 2020 PST | 6 Wed Jan 01 17:00:00 2020 PST | 6 Wed Jan 01 21:00:00 2020 PST | 2222 Thu Jan 02 05:00:00 2020 PST | 5555 Thu Jan 02 09:00:00 2020 PST | 8888 -- Test invalidations TRUNCATE temperature; CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); INSERT INTO temperature SELECT time, 5 FROM generate_series('2000-01-01 01:00:00 PST'::timestamptz, '2000-01-01 23:59:59 PST','1m') time; INSERT INTO temperature SELECT time, 6 FROM generate_series('2020-01-01 00:00:00 PST'::timestamptz, '2020-01-01 23:59:59 PST','1m') time; INSERT INTO temperature values('2020-01-02 01:05:00+01', 2222); INSERT INTO temperature values('2020-01-02 01:35:00+01', 5555); INSERT INTO temperature values('2020-01-02 05:05:00+01', 8888); SET client_min_messages TO DEBUG1; CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); psql:include/cagg_query_common.sql:725: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours', NULL, NULL); psql:include/cagg_query_common.sql:725: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577952000000000 psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Sat Jan 01 00:00:00 2000 PST, Sun Jan 02 00:00:00 2000 PST ] psql:include/cagg_query_common.sql:725: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark >= new watermark 1577995200000000 946800000000000 psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Wed Jan 01 00:00:00 2020 PST, Thu Jan 02 00:00:00 2020 PST ] psql:include/cagg_query_common.sql:725: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: DEBUG: hypertable 33 existing watermark >= new watermark 1577995200000000 1577952000000000 psql:include/cagg_query_common.sql:725: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours" in window [ Thu Jan 02 12:00:00 2020 PST, Thu Jan 02 16:00:00 2020 PST ] psql:include/cagg_query_common.sql:725: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_33" psql:include/cagg_query_common.sql:725: LOG: inserted 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_33" CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); psql:include/cagg_query_common.sql:726: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_offset', NULL, NULL); psql:include/cagg_query_common.sql:726: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577953800000000 psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Sat Jan 01 00:30:00 2000 PST, Sun Jan 02 00:30:00 2000 PST ] psql:include/cagg_query_common.sql:726: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark >= new watermark 1577997000000000 946801800000000 psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Tue Dec 31 20:30:00 2019 PST, Thu Jan 02 00:30:00 2020 PST ] psql:include/cagg_query_common.sql:726: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: LOG: inserted 7 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: DEBUG: hypertable 34 existing watermark >= new watermark 1577997000000000 1577953800000000 psql:include/cagg_query_common.sql:726: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_offset" in window [ Thu Jan 02 12:30:00 2020 PST, Thu Jan 02 16:30:00 2020 PST ] psql:include/cagg_query_common.sql:726: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_34" psql:include/cagg_query_common.sql:726: LOG: inserted 0 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_34" CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); psql:include/cagg_query_common.sql:727: LOG: statement: CALL refresh_continuous_aggregate('cagg_4_hours_origin', NULL, NULL); psql:include/cagg_query_common.sql:727: DEBUG: hypertable 4 existing watermark >= new invalidation threshold 1577998800000000 1577955600000000 psql:include/cagg_query_common.sql:727: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Sat Jan 01 01:00:00 2000 PST, Sun Jan 02 01:00:00 2000 PST ] psql:include/cagg_query_common.sql:727: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" psql:include/cagg_query_common.sql:727: LOG: inserted 6 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" psql:include/cagg_query_common.sql:727: DEBUG: hypertable 35 existing watermark >= new watermark 1577998800000000 946803600000000 psql:include/cagg_query_common.sql:727: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_4_hours_origin" in window [ Tue Dec 31 21:00:00 2019 PST, Thu Jan 02 01:00:00 2020 PST ] psql:include/cagg_query_common.sql:727: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_35" psql:include/cagg_query_common.sql:727: LOG: inserted 7 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_35" psql:include/cagg_query_common.sql:727: DEBUG: hypertable 35 existing watermark >= new watermark 1577998800000000 1577955600000000 RESET client_min_messages; psql:include/cagg_query_common.sql:728: LOG: statement: RESET client_min_messages; ALTER MATERIALIZED VIEW cagg_4_hours SET (timescaledb.materialized_only=true); SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+------ Sat Jan 01 00:00:00 2000 PST | 5 Sat Jan 01 04:00:00 2000 PST | 5 Sat Jan 01 08:00:00 2000 PST | 5 Sat Jan 01 12:00:00 2000 PST | 5 Sat Jan 01 16:00:00 2000 PST | 5 Sat Jan 01 20:00:00 2000 PST | 5 Wed Jan 01 00:00:00 2020 PST | 6 Wed Jan 01 04:00:00 2020 PST | 6 Wed Jan 01 08:00:00 2020 PST | 6 Wed Jan 01 12:00:00 2020 PST | 6 Wed Jan 01 16:00:00 2020 PST | 5555 Wed Jan 01 20:00:00 2020 PST | 8888 ALTER MATERIALIZED VIEW cagg_4_hours SET (timescaledb.materialized_only=false); SELECT * FROM cagg_4_hours; time_bucket | max ------------------------------+------ Sat Jan 01 00:00:00 2000 PST | 5 Sat Jan 01 04:00:00 2000 PST | 5 Sat Jan 01 08:00:00 2000 PST | 5 Sat Jan 01 12:00:00 2000 PST | 5 Sat Jan 01 16:00:00 2000 PST | 5 Sat Jan 01 20:00:00 2000 PST | 5 Wed Jan 01 00:00:00 2020 PST | 6 Wed Jan 01 04:00:00 2020 PST | 6 Wed Jan 01 08:00:00 2020 PST | 6 Wed Jan 01 12:00:00 2020 PST | 6 Wed Jan 01 16:00:00 2020 PST | 5555 Wed Jan 01 20:00:00 2020 PST | 8888 SELECT time_bucket('4 hour', time), max(value) FROM temperature GROUP BY 1 ORDER BY 1; time_bucket | max ------------------------------+------ Sat Jan 01 00:00:00 2000 PST | 5 Sat Jan 01 04:00:00 2000 PST | 5 Sat Jan 01 08:00:00 2000 PST | 5 Sat Jan 01 12:00:00 2000 PST | 5 Sat Jan 01 16:00:00 2000 PST | 5 Sat Jan 01 20:00:00 2000 PST | 5 Wed Jan 01 00:00:00 2020 PST | 6 Wed Jan 01 04:00:00 2020 PST | 6 Wed Jan 01 08:00:00 2020 PST | 6 Wed Jan 01 12:00:00 2020 PST | 6 Wed Jan 01 16:00:00 2020 PST | 5555 Wed Jan 01 20:00:00 2020 PST | 8888 ALTER MATERIALIZED VIEW cagg_4_hours_offset SET (timescaledb.materialized_only=true); SELECT * FROM cagg_4_hours_offset; time_bucket | max ------------------------------+------ Sat Jan 01 00:30:00 2000 PST | 5 Sat Jan 01 04:30:00 2000 PST | 5 Sat Jan 01 08:30:00 2000 PST | 5 Sat Jan 01 12:30:00 2000 PST | 5 Sat Jan 01 16:30:00 2000 PST | 5 Sat Jan 01 20:30:00 2000 PST | 5 Tue Dec 31 20:30:00 2019 PST | 6 Wed Jan 01 00:30:00 2020 PST | 6 Wed Jan 01 04:30:00 2020 PST | 6 Wed Jan 01 08:30:00 2020 PST | 6 Wed Jan 01 12:30:00 2020 PST | 2222 Wed Jan 01 16:30:00 2020 PST | 8888 Wed Jan 01 20:30:00 2020 PST | 6 ALTER MATERIALIZED VIEW cagg_4_hours_offset SET (timescaledb.materialized_only=false); SELECT * FROM cagg_4_hours_offset; time_bucket | max ------------------------------+------ Sat Jan 01 00:30:00 2000 PST | 5 Sat Jan 01 04:30:00 2000 PST | 5 Sat Jan 01 08:30:00 2000 PST | 5 Sat Jan 01 12:30:00 2000 PST | 5 Sat Jan 01 16:30:00 2000 PST | 5 Sat Jan 01 20:30:00 2000 PST | 5 Tue Dec 31 20:30:00 2019 PST | 6 Wed Jan 01 00:30:00 2020 PST | 6 Wed Jan 01 04:30:00 2020 PST | 6 Wed Jan 01 08:30:00 2020 PST | 6 Wed Jan 01 12:30:00 2020 PST | 2222 Wed Jan 01 16:30:00 2020 PST | 8888 Wed Jan 01 20:30:00 2020 PST | 6 SELECT time_bucket('4 hour', time, '30m'::interval), max(value) FROM temperature GROUP BY 1 ORDER BY 1; time_bucket | max ------------------------------+------ Sat Jan 01 00:30:00 2000 PST | 5 Sat Jan 01 04:30:00 2000 PST | 5 Sat Jan 01 08:30:00 2000 PST | 5 Sat Jan 01 12:30:00 2000 PST | 5 Sat Jan 01 16:30:00 2000 PST | 5 Sat Jan 01 20:30:00 2000 PST | 5 Tue Dec 31 20:30:00 2019 PST | 6 Wed Jan 01 00:30:00 2020 PST | 6 Wed Jan 01 04:30:00 2020 PST | 6 Wed Jan 01 08:30:00 2020 PST | 6 Wed Jan 01 12:30:00 2020 PST | 2222 Wed Jan 01 16:30:00 2020 PST | 8888 Wed Jan 01 20:30:00 2020 PST | 6 ALTER MATERIALIZED VIEW cagg_4_hours_origin SET (timescaledb.materialized_only=true); SELECT * FROM cagg_4_hours_origin; time_bucket | max ------------------------------+------ Sat Jan 01 01:00:00 2000 PST | 5 Sat Jan 01 05:00:00 2000 PST | 5 Sat Jan 01 09:00:00 2000 PST | 5 Sat Jan 01 13:00:00 2000 PST | 5 Sat Jan 01 17:00:00 2000 PST | 5 Sat Jan 01 21:00:00 2000 PST | 5 Tue Dec 31 21:00:00 2019 PST | 6 Wed Jan 01 01:00:00 2020 PST | 6 Wed Jan 01 05:00:00 2020 PST | 6 Wed Jan 01 09:00:00 2020 PST | 6 Wed Jan 01 13:00:00 2020 PST | 5555 Wed Jan 01 17:00:00 2020 PST | 8888 Wed Jan 01 21:00:00 2020 PST | 6 ALTER MATERIALIZED VIEW cagg_4_hours_origin SET (timescaledb.materialized_only=false); SELECT * FROM cagg_4_hours_origin; time_bucket | max ------------------------------+------ Sat Jan 01 01:00:00 2000 PST | 5 Sat Jan 01 05:00:00 2000 PST | 5 Sat Jan 01 09:00:00 2000 PST | 5 Sat Jan 01 13:00:00 2000 PST | 5 Sat Jan 01 17:00:00 2000 PST | 5 Sat Jan 01 21:00:00 2000 PST | 5 Tue Dec 31 21:00:00 2019 PST | 6 Wed Jan 01 01:00:00 2020 PST | 6 Wed Jan 01 05:00:00 2020 PST | 6 Wed Jan 01 09:00:00 2020 PST | 6 Wed Jan 01 13:00:00 2020 PST | 5555 Wed Jan 01 17:00:00 2020 PST | 8888 Wed Jan 01 21:00:00 2020 PST | 6 SELECT time_bucket('4 hour', time, '2000-01-01 01:00:00 PST'::timestamptz), max(value) FROM temperature GROUP BY 1 ORDER BY 1; time_bucket | max ------------------------------+------ Sat Jan 01 01:00:00 2000 PST | 5 Sat Jan 01 05:00:00 2000 PST | 5 Sat Jan 01 09:00:00 2000 PST | 5 Sat Jan 01 13:00:00 2000 PST | 5 Sat Jan 01 17:00:00 2000 PST | 5 Sat Jan 01 21:00:00 2000 PST | 5 Tue Dec 31 21:00:00 2019 PST | 6 Wed Jan 01 01:00:00 2020 PST | 6 Wed Jan 01 05:00:00 2020 PST | 6 Wed Jan 01 09:00:00 2020 PST | 6 Wed Jan 01 13:00:00 2020 PST | 5555 Wed Jan 01 17:00:00 2020 PST | 8888 Wed Jan 01 21:00:00 2020 PST | 6 --- Test with variable width buckets (use February, since hourly origins are not supported with variable sized buckets) TRUNCATE temperature; INSERT INTO temperature SELECT time, 5 FROM generate_series('2000-02-01 01:00:00 PST'::timestamptz, '2000-02-01 23:59:59 PST','1m') time; INSERT INTO temperature SELECT time, 6 FROM generate_series('2020-02-01 01:00:00 PST'::timestamptz, '2020-02-01 23:59:59 PST','1m') time; SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log ORDER BY 1, 2, 3; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- 2 | -9223372036854775808 | -210866803200000001 2 | 1541289600000000 | 9223372036854775807 3 | -9223372036854775808 | -210866803200000001 3 | 1541289600000000 | 9223372036854775807 33 | -9223372036854775808 | -210866803200000001 33 | 1577998800000000 | 9223372036854775807 34 | -9223372036854775808 | -210866803200000001 34 | 1577998800000000 | 9223372036854775807 35 | -9223372036854775808 | -210866803200000001 35 | 1577998800000000 | 9223372036854775807 CREATE MATERIALIZED VIEW cagg_1_year WITH (timescaledb.continuous, timescaledb.materialized_only = false) AS SELECT time_bucket('1 year', time), max(value) FROM temperature GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:766: NOTICE: refreshing continuous aggregate "cagg_1_year" SELECT * FROM _timescaledb_catalog.continuous_aggs_materialization_invalidation_log ORDER BY 1, 2, 3; materialization_id | lowest_modified_value | greatest_modified_value --------------------+-----------------------+------------------------- 2 | -9223372036854775808 | -210866803200000001 2 | 1541289600000000 | 9223372036854775807 3 | -9223372036854775808 | -210866803200000001 3 | 1541289600000000 | 9223372036854775807 33 | -9223372036854775808 | -210866803200000001 33 | -9223372036854775808 | 9223372036854775807 33 | 1577998800000000 | 9223372036854775807 34 | -9223372036854775808 | -210866803200000001 34 | -9223372036854775808 | 9223372036854775807 34 | 1577998800000000 | 9223372036854775807 35 | -9223372036854775808 | -210866803200000001 35 | -9223372036854775808 | 9223372036854775807 35 | 1577998800000000 | 9223372036854775807 36 | 1609459200000000 | 9223372036854775807 --- -- Tests with integer based hypertables --- TRUNCATE table_int; INSERT INTO table_int SELECT time, 5 FROM generate_series(-50, 50) time; CREATE MATERIALIZED VIEW cagg_int WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('10', time), SUM(data) as value FROM table_int GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:783: NOTICE: refreshing continuous aggregate "cagg_int" CREATE MATERIALIZED VIEW cagg_int_offset WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('10', time, "offset"=>5), SUM(data) as value FROM table_int GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:789: NOTICE: refreshing continuous aggregate "cagg_int_offset" -- Compare bucketing results SELECT time_bucket('10', time), SUM(data) FROM table_int GROUP BY 1 ORDER BY 1; time_bucket | sum -------------+----- -50 | 50 -40 | 50 -30 | 50 -20 | 50 -10 | 50 0 | 50 10 | 50 20 | 50 30 | 50 40 | 50 50 | 5 SELECT * FROM cagg_int; time_bucket | value -------------+------- -50 | 50 -40 | 50 -30 | 50 -20 | 50 -10 | 50 0 | 50 10 | 50 20 | 50 30 | 50 40 | 50 50 | 5 SELECT time_bucket('10', time, "offset"=>5), SUM(data) FROM table_int GROUP BY 1 ORDER BY 1; time_bucket | sum -------------+----- -55 | 25 -45 | 50 -35 | 50 -25 | 50 -15 | 50 -5 | 50 5 | 50 15 | 50 25 | 50 35 | 50 45 | 30 SELECT * FROM cagg_int_offset; time_bucket | value -------------+------- -55 | 25 -45 | 50 -35 | 50 -25 | 50 -15 | 50 -5 | 50 5 | 50 15 | 50 25 | 50 35 | 50 45 | 30 -- Update table INSERT INTO table_int VALUES(51, 100); INSERT INTO table_int VALUES(100, 555); -- Compare bucketing results SELECT time_bucket('10', time), SUM(data) FROM table_int GROUP BY 1 ORDER BY 1; time_bucket | sum -------------+----- -50 | 50 -40 | 50 -30 | 50 -20 | 50 -10 | 50 0 | 50 10 | 50 20 | 50 30 | 50 40 | 50 50 | 105 100 | 555 SELECT * FROM cagg_int; time_bucket | value -------------+------- -50 | 50 -40 | 50 -30 | 50 -20 | 50 -10 | 50 0 | 50 10 | 50 20 | 50 30 | 50 40 | 50 50 | 5 100 | 555 CALL refresh_continuous_aggregate('cagg_int', NULL, NULL); SELECT * FROM cagg_int; time_bucket | value -------------+------- -50 | 50 -40 | 50 -30 | 50 -20 | 50 -10 | 50 0 | 50 10 | 50 20 | 50 30 | 50 40 | 50 50 | 105 100 | 555 SELECT time_bucket('10', time, "offset"=>5), SUM(data) FROM table_int GROUP BY 1 ORDER BY 1; time_bucket | sum -------------+----- -55 | 25 -45 | 50 -35 | 50 -25 | 50 -15 | 50 -5 | 50 5 | 50 15 | 50 25 | 50 35 | 50 45 | 130 95 | 555 SELECT * FROM cagg_int_offset; -- the value 100 is part of the already serialized bucket, so it should not be visible time_bucket | value -------------+------- -55 | 25 -45 | 50 -35 | 50 -25 | 50 -15 | 50 -5 | 50 5 | 50 15 | 50 25 | 50 35 | 50 45 | 30 95 | 555 CALL refresh_continuous_aggregate('cagg_int_offset', NULL, NULL); SELECT * FROM cagg_int_offset; time_bucket | value -------------+------- -55 | 25 -45 | 50 -35 | 50 -25 | 50 -15 | 50 -5 | 50 5 | 50 15 | 50 25 | 50 35 | 50 45 | 130 95 | 555 -- Ensure everything was materialized ALTER MATERIALIZED VIEW cagg_int SET (timescaledb.materialized_only=true); ALTER MATERIALIZED VIEW cagg_int_offset SET (timescaledb.materialized_only=true); SELECT * FROM cagg_int; time_bucket | value -------------+------- -50 | 50 -40 | 50 -30 | 50 -20 | 50 -10 | 50 0 | 50 10 | 50 20 | 50 30 | 50 40 | 50 50 | 105 100 | 555 SELECT * FROM cagg_int_offset; time_bucket | value -------------+------- -55 | 25 -45 | 50 -35 | 50 -25 | 50 -15 | 50 -5 | 50 5 | 50 15 | 50 25 | 50 35 | 50 45 | 130 95 | 555 -- Check that the refresh is properly aligned INSERT INTO table_int VALUES(114, 0); SET client_min_messages TO DEBUG1; CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); psql:include/cagg_query_common.sql:824: LOG: statement: CALL refresh_continuous_aggregate('cagg_int_offset', 100, 130); psql:include/cagg_query_common.sql:824: DEBUG: continuous aggregate refresh (individual invalidation) on "cagg_int_offset" in window [ 105, 125 ] psql:include/cagg_query_common.sql:824: LOG: deleted 0 row(s) from materialization table "_timescaledb_internal._materialized_hypertable_38" psql:include/cagg_query_common.sql:824: DEBUG: building index "_hyper_38_67_chunk__materialized_hypertable_38_time_bucket_idx" on table "_hyper_38_67_chunk" serially psql:include/cagg_query_common.sql:824: DEBUG: index "_hyper_38_67_chunk__materialized_hypertable_38_time_bucket_idx" can safely use deduplication psql:include/cagg_query_common.sql:824: LOG: inserted 1 row(s) into materialization table "_timescaledb_internal._materialized_hypertable_38" RESET client_min_messages; psql:include/cagg_query_common.sql:825: LOG: statement: RESET client_min_messages; SELECT * FROM cagg_int_offset; time_bucket | value -------------+------- -55 | 25 -45 | 50 -35 | 50 -25 | 50 -15 | 50 -5 | 50 5 | 50 15 | 50 25 | 50 35 | 50 45 | 130 95 | 555 105 | 0 SELECT time_bucket('10', time, "offset"=>5), SUM(data) FROM table_int GROUP BY 1 ORDER BY 1; time_bucket | sum -------------+----- -55 | 25 -45 | 50 -35 | 50 -25 | 50 -15 | 50 -5 | 50 5 | 50 15 | 50 25 | 50 35 | 50 45 | 130 95 | 555 105 | 0 -- Variable sized buckets with origin CREATE MATERIALIZED VIEW cagg_1_hour_variable_bucket_fixed_origin WITH (timescaledb.continuous) AS SELECT time_bucket('1 year', time, origin=>'2000-01-01 01:05:00 UTC'::timestamptz, timezone=>'UTC') AS hour_bucket, max(value) AS max_value FROM temperature GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:835: NOTICE: refreshing continuous aggregate "cagg_1_hour_variable_bucket_fixed_origin" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_1_hour_variable_bucket_fixed_origin'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+------------------------------------------+---------------------------------------------------------------------------------------------------------+--------------+------------------------------+---------------+-----------------+-------------------- public | cagg_1_hour_variable_bucket_fixed_origin | public.time_bucket(interval,timestamp with time zone,pg_catalog.text,timestamp with time zone,interval) | @ 1 year | Fri Dec 31 17:05:00 1999 PST | | UTC | f DROP MATERIALIZED VIEW cagg_1_hour_variable_bucket_fixed_origin; psql:include/cagg_query_common.sql:837: NOTICE: drop cascades to 2 other objects -- Variable due to the used timezone CREATE MATERIALIZED VIEW cagg_1_hour_variable_bucket_fixed_origin2 WITH (timescaledb.continuous) AS SELECT time_bucket('1 hour', time, origin=>'2000-01-01 01:05:00 UTC'::timestamptz, timezone=>'UTC') AS hour_bucket, max(value) AS max_value FROM temperature GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:844: NOTICE: refreshing continuous aggregate "cagg_1_hour_variable_bucket_fixed_origin2" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_1_hour_variable_bucket_fixed_origin2'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+-------------------------------------------+---------------------------------------------------------------------------------------------------------+--------------+------------------------------+---------------+-----------------+-------------------- public | cagg_1_hour_variable_bucket_fixed_origin2 | public.time_bucket(interval,timestamp with time zone,pg_catalog.text,timestamp with time zone,interval) | @ 1 hour | Fri Dec 31 17:05:00 1999 PST | | UTC | f DROP MATERIALIZED VIEW cagg_1_hour_variable_bucket_fixed_origin2; psql:include/cagg_query_common.sql:846: NOTICE: drop cascades to 2 other objects -- Variable with offset CREATE MATERIALIZED VIEW cagg_1_hour_variable_bucket_fixed_origin3 WITH (timescaledb.continuous) AS SELECT time_bucket('1 year', time, "offset"=>'5 minutes'::interval) AS hour_bucket, max(value) AS max_value FROM temperature GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:853: NOTICE: refreshing continuous aggregate "cagg_1_hour_variable_bucket_fixed_origin3" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_1_hour_variable_bucket_fixed_origin3'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+-------------------------------------------+----------------------------------------------------------------+--------------+---------------+---------------+-----------------+-------------------- public | cagg_1_hour_variable_bucket_fixed_origin3 | public.time_bucket(interval,timestamp with time zone,interval) | @ 1 year | | @ 5 mins | | f DROP MATERIALIZED VIEW cagg_1_hour_variable_bucket_fixed_origin3; psql:include/cagg_query_common.sql:855: NOTICE: drop cascades to 2 other objects --- -- Test with blocking a few broken configurations --- \set ON_ERROR_STOP 0 -- Unfortunately '\set VERBOSITY verbose' cannot be used here to check the error details -- since it also prints the line number of the location, which is depended on the build -- Different time origin CREATE MATERIALIZED VIEW cagg_1_hour_origin WITH (timescaledb.continuous) AS SELECT time_bucket('1 hour', time, origin=>'2000-01-02 01:00:00 PST'::timestamptz) AS hour_bucket, max(value) AS max_value FROM temperature GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:870: NOTICE: refreshing continuous aggregate "cagg_1_hour_origin" CREATE MATERIALIZED VIEW cagg_1_week_origin WITH (timescaledb.continuous) AS SELECT time_bucket('1 week', hour_bucket, origin=>'2022-01-02 01:00:00 PST'::timestamptz) AS week_bucket, max(max_value) AS max_value FROM cagg_1_hour_origin GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:876: ERROR: cannot create continuous aggregate with different bucket origin values -- Different time offset CREATE MATERIALIZED VIEW cagg_1_hour_offset WITH (timescaledb.continuous) AS SELECT time_bucket('1 hour', time, "offset"=>'30m'::interval) AS hour_bucket, max(value) AS max_value FROM temperature GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:883: NOTICE: refreshing continuous aggregate "cagg_1_hour_offset" CREATE MATERIALIZED VIEW cagg_1_week_offset WITH (timescaledb.continuous) AS SELECT time_bucket('1 week', hour_bucket, "offset"=>'35m'::interval) AS week_bucket, max(max_value) AS max_value FROM cagg_1_hour_offset GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:889: ERROR: cannot create continuous aggregate with different bucket offset values -- Cagg with NULL offset on top of cagg with non-NULL offset \set VERBOSITY default CREATE MATERIALIZED VIEW cagg_1_week_null_offset WITH (timescaledb.continuous) AS SELECT time_bucket('1 week', hour_bucket, "offset"=>NULL::interval) AS week_bucket, max(max_value) AS max_value FROM cagg_1_hour_offset GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:897: ERROR: cannot create continuous aggregate with different bucket offset values DETAIL: Time origin of "public.cagg_1_week_null_offset" [NULL] and "public.cagg_1_hour_offset" [@ 30 mins] should be the same. -- Cagg with non-NULL offset on top of cagg with NULL offset CREATE MATERIALIZED VIEW cagg_1_hour_null_offset WITH (timescaledb.continuous) AS SELECT time_bucket('1 hour', time, "offset"=>NULL::interval) AS hour_bucket, max(value) AS max_value FROM temperature GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:904: NOTICE: refreshing continuous aggregate "cagg_1_hour_null_offset" HINT: Use WITH NO DATA if you do not want to refresh the continuous aggregate on creation. CREATE MATERIALIZED VIEW cagg_1_week_non_null_offset WITH (timescaledb.continuous) AS SELECT time_bucket('1 week', hour_bucket, "offset"=>'35m'::interval) AS week_bucket, max(max_value) AS max_value FROM cagg_1_hour_null_offset GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:910: ERROR: cannot create continuous aggregate with different bucket offset values DETAIL: Time origin of "public.cagg_1_week_non_null_offset" [@ 35 mins] and "public.cagg_1_hour_null_offset" [NULL] should be the same. \set VERBOSITY terse -- Different integer offset CREATE MATERIALIZED VIEW cagg_int_offset_5 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('10', time, "offset"=>5) AS time, SUM(data) AS value FROM table_int GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:918: NOTICE: refreshing continuous aggregate "cagg_int_offset_5" CREATE MATERIALIZED VIEW cagg_int_offset_10 WITH (timescaledb.continuous, timescaledb.materialized_only=false) AS SELECT time_bucket('10', time, "offset"=>10) AS time, SUM(value) AS value FROM cagg_int_offset_5 GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:924: ERROR: cannot create continuous aggregate with different bucket offset values \set ON_ERROR_STOP 1 DROP MATERIALIZED VIEW cagg_1_hour_origin; psql:include/cagg_query_common.sql:928: NOTICE: drop cascades to 2 other objects DROP MATERIALIZED VIEW cagg_1_hour_offset; psql:include/cagg_query_common.sql:929: NOTICE: drop cascades to 2 other objects DROP MATERIALIZED VIEW cagg_int_offset_5; psql:include/cagg_query_common.sql:930: NOTICE: drop cascades to 3 other objects --- -- CAGGs on CAGGs tests --- CREATE MATERIALIZED VIEW cagg_1_hour_offset WITH (timescaledb.continuous) AS SELECT time_bucket('1 hour', time, origin=>'2000-01-02 01:00:00 PST'::timestamptz) AS hour_bucket, max(value) AS max_value FROM temperature GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:939: NOTICE: refreshing continuous aggregate "cagg_1_hour_offset" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_1_hour_offset'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+--------------------+--------------------------------------------------------------------------------+--------------+------------------------------+---------------+-----------------+-------------------- public | cagg_1_hour_offset | public.time_bucket(interval,timestamp with time zone,timestamp with time zone) | @ 1 hour | Sun Jan 02 01:00:00 2000 PST | | | t CREATE MATERIALIZED VIEW cagg_1_week_offset WITH (timescaledb.continuous) AS SELECT time_bucket('1 week', hour_bucket, origin=>'2000-01-02 01:00:00 PST'::timestamptz) AS week_bucket, max(max_value) AS max_value FROM cagg_1_hour_offset GROUP BY 1 ORDER BY 1; psql:include/cagg_query_common.sql:946: NOTICE: refreshing continuous aggregate "cagg_1_week_offset" SELECT * FROM caggs_info WHERE user_view_name = 'cagg_1_week_offset'; user_view_schema | user_view_name | bucket_func | bucket_width | bucket_origin | bucket_offset | bucket_timezone | bucket_fixed_width ------------------+--------------------+--------------------------------------------------------------------------------+--------------+------------------------------+---------------+-----------------+-------------------- public | cagg_1_week_offset | public.time_bucket(interval,timestamp with time zone,timestamp with time zone) | @ 7 days | Sun Jan 02 01:00:00 2000 PST | | | t -- Compare output SELECT * FROM cagg_1_week_offset; week_bucket | max_value ------------------------------+----------- Sun Jan 30 01:00:00 2000 PST | 5 Sun Jan 26 01:00:00 2020 PST | 6 SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestamptz), max(value) FROM temperature GROUP BY 1 ORDER BY 1; time_bucket | max ------------------------------+----- Sun Jan 30 01:00:00 2000 PST | 5 Sun Jan 26 01:00:00 2020 PST | 6 INSERT INTO temperature values('2030-01-01 05:05:00 PST', 22222); INSERT INTO temperature values('2030-01-03 05:05:00 PST', 55555); -- Compare real-time functionality ALTER MATERIALIZED VIEW cagg_1_hour_offset SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_1_week_offset SET (timescaledb.materialized_only=false); SELECT * FROM cagg_1_week_offset; week_bucket | max_value ------------------------------+----------- Sun Jan 30 01:00:00 2000 PST | 5 Sun Jan 26 01:00:00 2020 PST | 6 Sun Dec 30 01:00:00 2029 PST | 55555 SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestamptz), max(value) FROM temperature GROUP BY 1 ORDER BY 1; time_bucket | max ------------------------------+------- Sun Jan 30 01:00:00 2000 PST | 5 Sun Jan 26 01:00:00 2020 PST | 6 Sun Dec 30 01:00:00 2029 PST | 55555 -- Test refresh CALL refresh_continuous_aggregate('cagg_1_hour_offset', NULL, NULL); CALL refresh_continuous_aggregate('cagg_1_week_offset', NULL, NULL); -- Everything should be now materailized ALTER MATERIALIZED VIEW cagg_1_hour_offset SET (timescaledb.materialized_only=false); ALTER MATERIALIZED VIEW cagg_1_week_offset SET (timescaledb.materialized_only=false); SELECT * FROM cagg_1_week_offset; week_bucket | max_value ------------------------------+----------- Sun Jan 30 01:00:00 2000 PST | 5 Sun Jan 26 01:00:00 2020 PST | 6 Sun Dec 30 01:00:00 2029 PST | 55555 SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestamptz), max(value) FROM temperature GROUP BY 1 ORDER BY 1; time_bucket | max ------------------------------+------- Sun Jan 30 01:00:00 2000 PST | 5 Sun Jan 26 01:00:00 2020 PST | 6 Sun Dec 30 01:00:00 2029 PST | 55555 TRUNCATE temperature; SELECT * FROM cagg_1_week_offset; week_bucket | max_value ------------------------------+----------- Sun Jan 30 01:00:00 2000 PST | 5 Sun Jan 26 01:00:00 2020 PST | 6 Sun Dec 30 01:00:00 2029 PST | 55555 SELECT time_bucket('1 week', time, origin=>'2000-01-02 01:00:00 PST'::timestamptz), max(value) FROM temperature GROUP BY 1 ORDER BY 1; time_bucket | max -------------+----- DROP VIEW caggs_info; ================================================ FILE: tsl/test/expected/cagg_query-17.out ================================================ [File too large to display: 141.7 KB] ================================================ FILE: tsl/test/expected/cagg_query-18.out ================================================ [File too large to display: 141.7 KB] ================================================ FILE: tsl/test/expected/cagg_query_using_merge-15.out ================================================ [File too large to display: 141.0 KB] ================================================ FILE: tsl/test/expected/cagg_query_using_merge-16.out ================================================ [File too large to display: 140.4 KB] ================================================ FILE: tsl/test/expected/cagg_query_using_merge-17.out ================================================ [File too large to display: 140.4 KB] ================================================ FILE: tsl/test/expected/cagg_query_using_merge-18.out ================================================ [File too large to display: 140.4 KB] ================================================ FILE: tsl/test/expected/cagg_refresh_using_merge.out ================================================ [File too large to display: 49.1 KB] ================================================ FILE: tsl/test/expected/cagg_refresh_using_trigger.out ================================================ [File too large to display: 33.6 KB] ================================================ FILE: tsl/test/expected/cagg_tableam.out ================================================ -- This file and its contents are licensed under the Timescale License. -- Please see the included NOTICE for copyright information and -- LICENSE-TIMESCALE for a copy of the license. \c :TEST_DBNAME :ROLE_SUPERUSER CREATE ACCESS METHOD heap2 TYPE TABLE HANDLER heap_tableam_handler; SET ROLE :ROLE_DEFAULT_PERM_USER; CREATE VIEW cagg_info AS WITH caggs AS ( SELECT format('%I.%I', user_view_schema, user_view_name)::regclass AS user_view, format('%I.%I', ht.schema_name, ht.table_name)::regclass AS mat_relid FROM _timescaledb_catalog.hypertable ht, _timescaledb_catalog.continuous_agg cagg WHERE ht.id = cagg.mat_hypertable_id ) SELECT user_view, relname AS mat_table, (SELECT spcname FROM pg_tablespace WHERE oid = reltablespace) AS tablespace FROM pg_class JOIN caggs ON pg_class.oid = caggs.mat_relid; CREATE TABLE whatever(time BIGINT NOT NULL, data INTEGER); SELECT hypertable_id AS whatever_nid FROM create_hypertable('whatever', 'time', chunk_time_interval => 10) \gset CREATE OR REPLACE FUNCTION integer_now_test() RETURNS bigint LANGUAGE SQL STABLE AS $$ SELECT coalesce(max(time), bigint '0') FROM whatever $$; SELECT set_integer_now_func('whatever', 'integer_now_test'); set_integer_now_func ---------------------- INSERT INTO whatever SELECT i, i FROM generate_series(0, 29) AS i; -- Checking that the access method for the materialized hypertable is -- set to the correct access method. CREATE MATERIALIZED VIEW tableam_view USING heap2 WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('5', time), COUNT(data) FROM whatever GROUP BY 1 WITH NO DATA; CREATE MATERIALIZED VIEW notableam_view WITH (timescaledb.continuous, timescaledb.materialized_only=true) AS SELECT time_bucket('5', time), COUNT(data) FROM whatever GROUP BY 1 WITH NO DATA; SELECT user_view, mat_table, amname FROM cagg_info, LATERAL (SELECT relam FROM pg_class WHERE relname = mat_table) AS cls, LATERAL (SELECT amname FROM pg_am WHERE oid = relam) AS am WHERE user_view::text IN ('tableam_view', 'notableam_view'); user_view | mat_table | amname ----------------+----------------------------+-------- tableam_view | _materialized_hypertable_2 | heap2 notableam_view | _materialized_hypertable_3 | heap -- Check that the view with the other access method actually works. SELECT * FROM notableam_view ORDER BY 1; time_bucket | count -------------+------- SELECT * FROM tableam_view ORDER BY 1; time_bucket | count -------------+------- CALL refresh_continuous_aggregate('notableam_view', NULL, NULL); CALL refresh_continuous_aggregate('tableam_view', NULL, NULL); SELECT * FROM notableam_view ORDER BY 1; time_bucket | count -------------+------- 0 | 5 5 | 5 10 | 5 15 | 5 20 | 5 25 | 5 SELECT * FROM tableam_view ORDER BY 1; time_bucket | count -------------+------- 0 | 5 5 | 5 10 | 5 15 | 5 20 | 5 25 | 5 DROP MATERIALIZED VIEW notableam_view; NOTICE: drop cascades to table _timescaledb_internal._hyper_3_4_chunk DROP MATERIALIZED VIEW tableam_view; NOTICE: drop cascades to table _timescaledb_internal._hyper_2_5_chunk ================================================ FILE: tsl/test/expected/cagg_union_view-15.out ================================================ [File too large to display: 43.6 KB] ================================================ FILE: tsl/test/expected/cagg_union_view-16.out ================================================ [File too large to display: 43.6 KB] ================================================ FILE: tsl/test/expected/cagg_union_view-17.out ================================================ [File too large to display: 43.6 KB] ================================================ FILE: tsl/test/expected/cagg_union_view-18.out ================================================ [File too large to display: 43.6 KB] ================================================ FILE: tsl/test/expected/cagg_usage-15.out ================================================ [File too large to display: 20.6 KB] ================================================ FILE: tsl/test/expected/cagg_usage-16.out ================================================ [File too large to display: 20.3 KB] ================================================ FILE: tsl/test/expected/cagg_usage-17.out ================================================ [File too large to display: 20.3 KB] ================================================ FILE: tsl/test/expected/cagg_usage-18.out ================================================ [File too large to display: 20.3 KB] ================================================ FILE: tsl/test/expected/cagg_utils.out ================================================ [File too large to display: 25.5 KB] ================================================ FILE: tsl/test/expected/cagg_uuid.out ================================================ [File too large to display: 19.5 KB] ================================================ FILE: tsl/test/expected/cagg_watermark.out ================================================ [File too large to display: 119.9 KB] ================================================ FILE: tsl/test/expected/chunk_api.out ================================================ [File too large to display: 25.2 KB] ================================================ FILE: tsl/test/expected/chunk_column_stats.out ================================================ [File too large to display: 35.6 KB] ================================================ FILE: tsl/test/expected/chunk_merge.out ================================================ [File too large to display: 6.8 KB] ================================================ FILE: tsl/test/expected/chunk_publication_compression.out ================================================ [File too large to display: 3.1 KB] ================================================ FILE: tsl/test/expected/chunk_utils_compression.out ================================================ [File too large to display: 6.3 KB] ================================================ FILE: tsl/test/expected/chunk_utils_internal.out ================================================ [File too large to display: 74.3 KB] ================================================ FILE: tsl/test/expected/columnar_index_scan-15.out ================================================ [File too large to display: 105.8 KB] ================================================ FILE: tsl/test/expected/columnar_index_scan-16.out ================================================ [File too large to display: 105.9 KB] ================================================ FILE: tsl/test/expected/columnar_index_scan-17.out ================================================ [File too large to display: 113.1 KB] ================================================ FILE: tsl/test/expected/columnar_index_scan-18.out ================================================ [File too large to display: 113.1 KB] ================================================ FILE: tsl/test/expected/columnar_scan_cost.out ================================================ [File too large to display: 15.7 KB] ================================================ FILE: tsl/test/expected/columnstore_aliases.out ================================================ [File too large to display: 5.7 KB] ================================================ FILE: tsl/test/expected/compress_auto_sparse_index.out ================================================ [File too large to display: 4.9 KB] ================================================ FILE: tsl/test/expected/compress_batch_size.out ================================================ [File too large to display: 5.9 KB] ================================================ FILE: tsl/test/expected/compress_bgw_reorder_drop_chunks.out ================================================ [File too large to display: 11.9 KB] ================================================ FILE: tsl/test/expected/compress_bitmap_scan.out ================================================ [File too large to display: 15.9 KB] ================================================ FILE: tsl/test/expected/compress_bloom_dml.out ================================================ [File too large to display: 9.3 KB] ================================================ FILE: tsl/test/expected/compress_bloom_hash.out ================================================ [File too large to display: 1.7 KB] ================================================ FILE: tsl/test/expected/compress_bloom_hash_1.out ================================================ [File too large to display: 1.7 KB] ================================================ FILE: tsl/test/expected/compress_bloom_legacy_v1.out ================================================ [File too large to display: 4.6 KB] ================================================ FILE: tsl/test/expected/compress_bloom_sparse-15.out ================================================ [File too large to display: 53.8 KB] ================================================ FILE: tsl/test/expected/compress_bloom_sparse-16.out ================================================ [File too large to display: 53.8 KB] ================================================ FILE: tsl/test/expected/compress_bloom_sparse-17.out ================================================ [File too large to display: 53.8 KB] ================================================ FILE: tsl/test/expected/compress_bloom_sparse-18.out ================================================ [File too large to display: 53.8 KB] ================================================ FILE: tsl/test/expected/compress_bloom_sparse_debug.out ================================================ [File too large to display: 9.1 KB] ================================================ FILE: tsl/test/expected/compress_compbloom_basics.out ================================================ [File too large to display: 30.7 KB] ================================================ FILE: tsl/test/expected/compress_compbloom_config.out ================================================ [File too large to display: 15.8 KB] ================================================ FILE: tsl/test/expected/compress_compbloom_index_add.out ================================================ [File too large to display: 5.0 KB] ================================================ FILE: tsl/test/expected/compress_compbloom_index_drop.out ================================================ [File too large to display: 5.9 KB] ================================================ FILE: tsl/test/expected/compress_compbloom_manual_config.out ================================================ [File too large to display: 11.2 KB] ================================================ FILE: tsl/test/expected/compress_compbloom_upsert.out ================================================ [File too large to display: 11.0 KB] ================================================ FILE: tsl/test/expected/compress_composite_bloom_debug.out ================================================ [File too large to display: 19.0 KB] ================================================ FILE: tsl/test/expected/compress_default.out ================================================ [File too large to display: 1.4 KB] ================================================ FILE: tsl/test/expected/compress_dml_copy.out ================================================ [File too large to display: 1.7 KB] ================================================ FILE: tsl/test/expected/compress_explain.out ================================================ [File too large to display: 15.0 KB] ================================================ FILE: tsl/test/expected/compress_float8_corrupt.out ================================================ [File too large to display: 41.1 KB] ================================================ FILE: tsl/test/expected/compress_qualpushdown_complex.out ================================================ [File too large to display: 17.3 KB] ================================================ FILE: tsl/test/expected/compress_qualpushdown_saop.out ================================================ [File too large to display: 54.8 KB] ================================================ FILE: tsl/test/expected/compress_sort_transform.out ================================================ [File too large to display: 25.0 KB] ================================================ FILE: tsl/test/expected/compress_sparse_config.out ================================================ [File too large to display: 44.7 KB] ================================================ FILE: tsl/test/expected/compress_unordered_sort.out ================================================ [File too large to display: 19.7 KB] ================================================ FILE: tsl/test/expected/compressed_collation.out ================================================ [File too large to display: 4.4 KB] ================================================ FILE: tsl/test/expected/compressed_detoaster.out ================================================ [File too large to display: 1.6 KB] ================================================ FILE: tsl/test/expected/compression.out ================================================ [File too large to display: 147.8 KB] ================================================ FILE: tsl/test/expected/compression_algos.out ================================================ [File too large to display: 167.4 KB] ================================================ FILE: tsl/test/expected/compression_allocation.out ================================================ [File too large to display: 2.1 KB] ================================================ FILE: tsl/test/expected/compression_bgw.out ================================================ [File too large to display: 43.2 KB] ================================================ FILE: tsl/test/expected/compression_bool_vectorized.out ================================================ [File too large to display: 2.2 KB] ================================================ FILE: tsl/test/expected/compression_bools.out ================================================ [File too large to display: 8.5 KB] ================================================ FILE: tsl/test/expected/compression_conflicts.out ================================================ [File too large to display: 31.8 KB] ================================================ FILE: tsl/test/expected/compression_constraints.out ================================================ [File too large to display: 17.3 KB] ================================================ FILE: tsl/test/expected/compression_create_compressed_table.out ================================================ [File too large to display: 2.3 KB] ================================================ FILE: tsl/test/expected/compression_ddl.out ================================================ [File too large to display: 109.9 KB] ================================================ FILE: tsl/test/expected/compression_defaults.out ================================================ [File too large to display: 40.2 KB] ================================================ FILE: tsl/test/expected/compression_delete_bitmapscan-15.out ================================================ [File too large to display: 7.9 KB] ================================================ FILE: tsl/test/expected/compression_delete_bitmapscan-16.out ================================================ [File too large to display: 7.9 KB] ================================================ FILE: tsl/test/expected/compression_delete_bitmapscan-17.out ================================================ [File too large to display: 8.1 KB] ================================================ FILE: tsl/test/expected/compression_delete_bitmapscan-18.out ================================================ [File too large to display: 8.1 KB] ================================================ FILE: tsl/test/expected/compression_errors.out ================================================ [File too large to display: 48.2 KB] ================================================ FILE: tsl/test/expected/compression_fks.out ================================================ [File too large to display: 1.9 KB] ================================================ FILE: tsl/test/expected/compression_hypertable.out ================================================ [File too large to display: 22.0 KB] ================================================ FILE: tsl/test/expected/compression_indexcreate.out ================================================ [File too large to display: 6.1 KB] ================================================ FILE: tsl/test/expected/compression_indexscan.out ================================================ [File too large to display: 46.2 KB] ================================================ FILE: tsl/test/expected/compression_insert.out ================================================ [File too large to display: 69.8 KB] ================================================ FILE: tsl/test/expected/compression_merge.out ================================================ [File too large to display: 38.7 KB] ================================================ FILE: tsl/test/expected/compression_null_dump_restore.out ================================================ [File too large to display: 3.2 KB] ================================================ FILE: tsl/test/expected/compression_nulls_and_defaults.out ================================================ [File too large to display: 17.6 KB] ================================================ FILE: tsl/test/expected/compression_permissions-15.out ================================================ [File too large to display: 16.3 KB] ================================================ FILE: tsl/test/expected/compression_permissions-16.out ================================================ [File too large to display: 16.3 KB] ================================================ FILE: tsl/test/expected/compression_permissions-17.out ================================================ [File too large to display: 16.3 KB] ================================================ FILE: tsl/test/expected/compression_permissions-18.out ================================================ [File too large to display: 16.3 KB] ================================================ FILE: tsl/test/expected/compression_policy.out ================================================ [File too large to display: 3.4 KB] ================================================ FILE: tsl/test/expected/compression_qualpushdown.out ================================================ [File too large to display: 26.7 KB] ================================================ FILE: tsl/test/expected/compression_segment_meta.out ================================================ [File too large to display: 7.1 KB] ================================================ FILE: tsl/test/expected/compression_sequence_num_removal.out ================================================ [File too large to display: 48.2 KB] ================================================ FILE: tsl/test/expected/compression_settings.out ================================================ [File too large to display: 33.6 KB] ================================================ FILE: tsl/test/expected/compression_sorted_merge.out ================================================ [File too large to display: 110.3 KB] ================================================ FILE: tsl/test/expected/compression_sorted_merge_columns.out ================================================ [File too large to display: 8.0 KB] ================================================ FILE: tsl/test/expected/compression_sorted_merge_distinct.out ================================================ [File too large to display: 5.2 KB] ================================================ FILE: tsl/test/expected/compression_sorted_merge_filter.out ================================================ [File too large to display: 4.1 KB] ================================================ FILE: tsl/test/expected/compression_sorted_merge_unordered.out ================================================ [File too large to display: 103.6 KB] ================================================ FILE: tsl/test/expected/compression_trigger.out ================================================ [File too large to display: 8.1 KB] ================================================ FILE: tsl/test/expected/compression_update_delete-15.out ================================================ [File too large to display: 117.0 KB] ================================================ FILE: tsl/test/expected/compression_update_delete-16.out ================================================ [File too large to display: 117.0 KB] ================================================ FILE: tsl/test/expected/compression_update_delete-17.out ================================================ [File too large to display: 117.5 KB] ================================================ FILE: tsl/test/expected/compression_update_delete-18.out ================================================ [File too large to display: 117.5 KB] ================================================ FILE: tsl/test/expected/compression_uuid.out ================================================ [File too large to display: 21.4 KB] ================================================ FILE: tsl/test/expected/create_table_with.out ================================================ [File too large to display: 30.8 KB] ================================================ FILE: tsl/test/expected/decompress_index.out ================================================ [File too large to display: 9.4 KB] ================================================ FILE: tsl/test/expected/decompress_memory.out ================================================ [File too large to display: 3.9 KB] ================================================ FILE: tsl/test/expected/decompress_vector_qual.out ================================================ [File too large to display: 84.2 KB] ================================================ FILE: tsl/test/expected/detach_chunk.out ================================================ [File too large to display: 5.7 KB] ================================================ FILE: tsl/test/expected/direct_compress_copy.out ================================================ [File too large to display: 15.5 KB] ================================================ FILE: tsl/test/expected/direct_compress_insert.out ================================================ [File too large to display: 33.3 KB] ================================================ FILE: tsl/test/expected/feature_flags.out ================================================ [File too large to display: 7.6 KB] ================================================ FILE: tsl/test/expected/fixed_schedules.out ================================================ [File too large to display: 52.7 KB] ================================================ FILE: tsl/test/expected/foreign_keys_test-15.out ================================================ [File too large to display: 70.7 KB] ================================================ FILE: tsl/test/expected/foreign_keys_test-16.out ================================================ [File too large to display: 70.7 KB] ================================================ FILE: tsl/test/expected/foreign_keys_test-17.out ================================================ [File too large to display: 70.7 KB] ================================================ FILE: tsl/test/expected/foreign_keys_test-18.out ================================================ [File too large to display: 71.1 KB] ================================================ FILE: tsl/test/expected/hypertable_generalization.out ================================================ [File too large to display: 18.0 KB] ================================================ FILE: tsl/test/expected/information_view_chunk_count.out ================================================ [File too large to display: 2.2 KB] ================================================ FILE: tsl/test/expected/insert_memory_usage.out ================================================ [File too large to display: 5.3 KB] ================================================ FILE: tsl/test/expected/jit.out ================================================ [File too large to display: 8.5 KB] ================================================ FILE: tsl/test/expected/license_tsl.out ================================================ [File too large to display: 373 B] ================================================ FILE: tsl/test/expected/merge_append_partially_compressed.out ================================================ [File too large to display: 101.2 KB] ================================================ FILE: tsl/test/expected/merge_chunks.out ================================================ [File too large to display: 55.7 KB] ================================================ FILE: tsl/test/expected/merge_compress.out ================================================ [File too large to display: 4.6 KB] ================================================ FILE: tsl/test/expected/modify_exclusion-15.out ================================================ [File too large to display: 129.0 KB] ================================================ FILE: tsl/test/expected/modify_exclusion-16.out ================================================ [File too large to display: 129.0 KB] ================================================ FILE: tsl/test/expected/modify_exclusion-17.out ================================================ [File too large to display: 129.0 KB] ================================================ FILE: tsl/test/expected/modify_exclusion-18.out ================================================ [File too large to display: 129.3 KB] ================================================ FILE: tsl/test/expected/move.out ================================================ [File too large to display: 27.2 KB] ================================================ FILE: tsl/test/expected/ordered_append-15.out ================================================ [File too large to display: 295.0 KB] ================================================ FILE: tsl/test/expected/ordered_append-16.out ================================================ [File too large to display: 295.0 KB] ================================================ FILE: tsl/test/expected/ordered_append-17.out ================================================ [File too large to display: 288.7 KB] ================================================ FILE: tsl/test/expected/ordered_append-18.out ================================================ [File too large to display: 288.7 KB] ================================================ FILE: tsl/test/expected/plan_skip_scan-15.out ================================================ [File too large to display: 543.0 KB] ================================================ FILE: tsl/test/expected/plan_skip_scan-16.out ================================================ [File too large to display: 541.7 KB] ================================================ FILE: tsl/test/expected/plan_skip_scan-17.out ================================================ [File too large to display: 541.5 KB] ================================================ FILE: tsl/test/expected/plan_skip_scan-18.out ================================================ [File too large to display: 541.3 KB] ================================================ FILE: tsl/test/expected/plan_skip_scan_dagg-15.out ================================================ [File too large to display: 437.0 KB] ================================================ FILE: tsl/test/expected/plan_skip_scan_dagg-16.out ================================================ [File too large to display: 436.1 KB] ================================================ FILE: tsl/test/expected/plan_skip_scan_dagg-17.out ================================================ [File too large to display: 436.1 KB] ================================================ FILE: tsl/test/expected/plan_skip_scan_dagg-18.out ================================================ [File too large to display: 436.0 KB] ================================================ FILE: tsl/test/expected/plan_skip_scan_dagg.out ================================================ [File too large to display: 436.7 KB] ================================================ FILE: tsl/test/expected/plan_skip_scan_notnull.out ================================================ [File too large to display: 137.4 KB] ================================================ FILE: tsl/test/expected/policy_generalization.out ================================================ [File too large to display: 5.6 KB] ================================================ FILE: tsl/test/expected/privilege_maintain.out ================================================ [File too large to display: 8.1 KB] ================================================ FILE: tsl/test/expected/read_only.out ================================================ [File too large to display: 10.7 KB] ================================================ FILE: tsl/test/expected/rebuild_columnstore_tests.out ================================================ [File too large to display: 17.2 KB] ================================================ FILE: tsl/test/expected/recompress_chunk_segmentwise.out ================================================ [File too large to display: 46.7 KB] ================================================ FILE: tsl/test/expected/recompression_integrity_tests.out ================================================ [File too large to display: 28.9 KB] ================================================ FILE: tsl/test/expected/recompression_integrity_unordered.out ================================================ [File too large to display: 42.7 KB] ================================================ FILE: tsl/test/expected/reorder.out ================================================ [File too large to display: 52.5 KB] ================================================ FILE: tsl/test/expected/scheduler_fixed.out ================================================ [File too large to display: 10.8 KB] ================================================ FILE: tsl/test/expected/size_utils_tsl.out ================================================ [File too large to display: 4.2 KB] ================================================ FILE: tsl/test/expected/skip_scan.out ================================================ [File too large to display: 198.7 KB] ================================================ FILE: tsl/test/expected/skip_scan_dagg.out ================================================ [File too large to display: 103.0 KB] ================================================ FILE: tsl/test/expected/split_chunk.out ================================================ [File too large to display: 65.9 KB] ================================================ FILE: tsl/test/expected/telemetry_stats.out ================================================ [File too large to display: 28.6 KB] ================================================ FILE: tsl/test/expected/transparent_decompression-15.out ================================================ [File too large to display: 605.5 KB] ================================================ FILE: tsl/test/expected/transparent_decompression-16.out ================================================ [File too large to display: 606.9 KB] ================================================ FILE: tsl/test/expected/transparent_decompression-17.out ================================================ [File too large to display: 606.9 KB] ================================================ FILE: tsl/test/expected/transparent_decompression-18.out ================================================ [File too large to display: 612.7 KB] ================================================ FILE: tsl/test/expected/transparent_decompression_join_index.out ================================================ [File too large to display: 4.6 KB] ================================================ FILE: tsl/test/expected/transparent_decompression_ordered_index-15.out ================================================ [File too large to display: 71.2 KB] ================================================ FILE: tsl/test/expected/transparent_decompression_ordered_index-16.out ================================================ [File too large to display: 71.1 KB] ================================================ FILE: tsl/test/expected/transparent_decompression_ordered_index-17.out ================================================ [File too large to display: 71.1 KB] ================================================ FILE: tsl/test/expected/transparent_decompression_ordered_index-18.out ================================================ [File too large to display: 71.9 KB] ================================================ FILE: tsl/test/expected/transparent_decompression_queries-15.out ================================================ [File too large to display: 12.6 KB] ================================================ FILE: tsl/test/expected/transparent_decompression_queries-16.out ================================================ [File too large to display: 12.6 KB] ================================================ FILE: tsl/test/expected/transparent_decompression_queries-17.out ================================================ [File too large to display: 13.3 KB] ================================================ FILE: tsl/test/expected/transparent_decompression_queries-18.out ================================================ [File too large to display: 13.3 KB] ================================================ FILE: tsl/test/expected/tsl_tables.out ================================================ [File too large to display: 56.2 KB] ================================================ FILE: tsl/test/expected/uncompressed_size.out ================================================ [File too large to display: 2.9 KB] ================================================ FILE: tsl/test/expected/unlogged.out ================================================ [File too large to display: 3.9 KB] ================================================ FILE: tsl/test/expected/uuid_policies.out ================================================ [File too large to display: 10.6 KB] ================================================ FILE: tsl/test/expected/vacuum.out ================================================ [File too large to display: 7.4 KB] ================================================ FILE: tsl/test/expected/vector_agg_byte.out ================================================ [File too large to display: 4.3 KB] ================================================ FILE: tsl/test/expected/vector_agg_default.out ================================================ [File too large to display: 20.3 KB] ================================================ FILE: tsl/test/expected/vector_agg_expr.out ================================================ [File too large to display: 47.1 KB] ================================================ FILE: tsl/test/expected/vector_agg_filter.out ================================================ [File too large to display: 49.5 KB] ================================================ FILE: tsl/test/expected/vector_agg_functions.out ================================================ [File too large to display: 156.4 KB] ================================================ FILE: tsl/test/expected/vector_agg_groupagg.out ================================================ [File too large to display: 6.8 KB] ================================================ FILE: tsl/test/expected/vector_agg_grouping.out ================================================ [File too large to display: 133.6 KB] ================================================ FILE: tsl/test/expected/vector_agg_memory.out ================================================ [File too large to display: 5.4 KB] ================================================ FILE: tsl/test/expected/vector_agg_modify_hypertable.out ================================================ [File too large to display: 1.8 KB] ================================================ FILE: tsl/test/expected/vector_agg_param.out ================================================ [File too large to display: 5.0 KB] ================================================ FILE: tsl/test/expected/vector_agg_planning-15.out ================================================ [File too large to display: 20.0 KB] ================================================ FILE: tsl/test/expected/vector_agg_planning-16.out ================================================ [File too large to display: 19.9 KB] ================================================ FILE: tsl/test/expected/vector_agg_planning-17.out ================================================ [File too large to display: 20.0 KB] ================================================ FILE: tsl/test/expected/vector_agg_planning-18.out ================================================ [File too large to display: 20.0 KB] ================================================ FILE: tsl/test/expected/vector_agg_segmentby.out ================================================ [File too large to display: 7.6 KB] ================================================ FILE: tsl/test/expected/vector_agg_text.out ================================================ [File too large to display: 8.3 KB] ================================================ FILE: tsl/test/expected/vector_agg_uuid.out ================================================ [File too large to display: 7.6 KB] ================================================ FILE: tsl/test/expected/vector_qual_default.out ================================================ [File too large to display: 3.7 KB] ================================================ FILE: tsl/test/expected/vectorized_aggregation.out ================================================ [File too large to display: 275.3 KB] ================================================ FILE: tsl/test/fuzzing/compression/array-bool/empty ================================================ ================================================ FILE: tsl/test/fuzzing/compression/array-text/0dbf553220bcd27478f10999d679d564a11632a1 ================================================ [File too large to display: 2 B] ================================================ FILE: tsl/test/fuzzing/compression/array-text/5d1be7e9dda1ee8896be5b7e34a85ee16452a7b4 ================================================ [File too large to display: 1 B] ================================================ FILE: tsl/test/fuzzing/compression/array-text/9159cb8bcee7fcb95582f140960cdae72788d326 ================================================ [File too large to display: 2 B] ================================================ FILE: tsl/test/fuzzing/compression/array-text/a42c6cf1de3abfdea9b95f34687cbbe92b9a7383 ================================================ [File too large to display: 1 B] ================================================ FILE: tsl/test/fuzzing/compression/array-uuid/empty ================================================ ================================================ FILE: tsl/test/fuzzing/compression/bool-bool/03fa95c77415b6c8691a0dc1d13e195ccd1b897c ================================================ [File too large to display: 2 B] ================================================ FILE: tsl/test/fuzzing/compression/bool-bool/11f4de6b8b45cf8051b1d17fa4cde9ad935cea41 ================================================ [File too large to display: 1 B] ================================================ FILE: tsl/test/fuzzing/compression/bool-bool/6326b8e4ed85d653f9a043fca18c638dd4df6d43 ================================================ [File too large to display: 2 B] ================================================ FILE: tsl/test/fuzzing/compression/bool-bool/8dc00598417d4eb788a77ac6ccef3cb484905d8b ================================================ [File too large to display: 1 B] ================================================ FILE: tsl/test/fuzzing/compression/bool-bool/96ddb4dc6bf60a475735388b8da21dc601275c4c ================================================ [File too large to display: 6 B] ================================================ FILE: tsl/test/fuzzing/compression/bool-bool/empty ================================================ ================================================ FILE: tsl/test/fuzzing/compression/deltadelta-int8/015c684cdef94cdf964d0800d325c6849d014324 ================================================ [File too large to display: 2 B] ================================================ FILE: tsl/test/fuzzing/compression/deltadelta-int8/5958b905c02f53a22cdfabb51cd2823d13650241 ================================================ [File too large to display: 2 B] ================================================ FILE: tsl/test/fuzzing/compression/deltadelta-int8/9842926af7ca0a8cca12604f945414f07b01e13d ================================================ [File too large to display: 1 B] ================================================ FILE: tsl/test/fuzzing/compression/deltadelta-int8/a42c6cf1de3abfdea9b95f34687cbbe92b9a7383 ================================================ [File too large to display: 1 B] ================================================ FILE: tsl/test/fuzzing/compression/dictionary-bool/empty ================================================ ================================================ FILE: tsl/test/fuzzing/compression/dictionary-text/559b65125ca556ff1a57f82f9ae55a86b71c6296 ================================================ [File too large to display: 2 B] ================================================ FILE: tsl/test/fuzzing/compression/dictionary-text/85e53271e14006f0265921d02d4d736cdc580b0b ================================================ [File too large to display: 1 B] ================================================ FILE: tsl/test/fuzzing/compression/dictionary-text/9a78211436f6d425ec38f5c4e02270801f3524f8 ================================================ [File too large to display: 1 B] ================================================ FILE: tsl/test/fuzzing/compression/dictionary-text/bf8b4530d8d246dd74ac53a13471bba17941dff7 ================================================ [File too large to display: 1 B] ================================================ FILE: tsl/test/fuzzing/compression/dictionary-text/c92920944247d80c842eaa65fd01efec1c84c342 ================================================ [File too large to display: 2 B] ================================================ FILE: tsl/test/fuzzing/compression/dictionary-uuid/31bba5d620d39268ee0d0b4acaf0b48ab78e7376 ================================================ [File too large to display: 2 B] ================================================ FILE: tsl/test/fuzzing/compression/dictionary-uuid/c4ea21bb365bbeeaf5f2c654883e56d11e43c44e ================================================ [File too large to display: 1 B] ================================================ FILE: tsl/test/fuzzing/compression/dictionary-uuid/empty ================================================ ================================================ FILE: tsl/test/fuzzing/compression/gorilla-float8/070bd26d7cc8fcba32ef29da3ad625824c38d343 ================================================ [File too large to display: 2 B] ================================================ FILE: tsl/test/fuzzing/compression/uuid-uuid/39cd2aeafea74e6f19c4aec657fb7ffa73262927 ================================================ [File too large to display: 2 B] ================================================ FILE: tsl/test/fuzzing/compression/uuid-uuid/3cdf2936da2fc556bfa533ab1eb59ce710ac80e5 ================================================ [File too large to display: 1 B] ================================================ FILE: tsl/test/fuzzing/compression/uuid-uuid/5d1be7e9dda1ee8896be5b7e34a85ee16452a7b4 ================================================ [File too large to display: 1 B] ================================================ FILE: tsl/test/fuzzing/compression/uuid-uuid/empty ================================================ ================================================ FILE: tsl/test/isolation/CMakeLists.txt ================================================ [File too large to display: 86 B] ================================================ FILE: tsl/test/isolation/expected/attach_chunk_isolation.out ================================================ [File too large to display: 11.6 KB] ================================================ FILE: tsl/test/isolation/expected/bgw_job_duplicate_race.out ================================================ [File too large to display: 892 B] ================================================ FILE: tsl/test/isolation/expected/bgw_job_stat_history_retention_isolation.out ================================================ [File too large to display: 1.2 KB] ================================================ FILE: tsl/test/isolation/expected/cagg_concurrent_invalidation.out ================================================ [File too large to display: 2.2 KB] ================================================ FILE: tsl/test/isolation/expected/cagg_concurrent_move.out ================================================ [File too large to display: 2.2 KB] ================================================ FILE: tsl/test/isolation/expected/cagg_concurrent_refresh.out ================================================ [File too large to display: 33.1 KB] ================================================ FILE: tsl/test/isolation/expected/cagg_insert.out ================================================ [File too large to display: 10.5 KB] ================================================ FILE: tsl/test/isolation/expected/cagg_multi_iso.out ================================================ [File too large to display: 7.2 KB] ================================================ FILE: tsl/test/isolation/expected/cagg_watermark_concurrent_update.out ================================================ [File too large to display: 19.2 KB] ================================================ FILE: tsl/test/isolation/expected/cagg_watermark_concurrent_update_1.out ================================================ [File too large to display: 17.2 KB] ================================================ FILE: tsl/test/isolation/expected/compression_chunk_race.out ================================================ [File too large to display: 4.1 KB] ================================================ FILE: tsl/test/isolation/expected/compression_conflicts_iso.out ================================================ [File too large to display: 86.1 KB] ================================================ FILE: tsl/test/isolation/expected/compression_ddl_iso.out ================================================ [File too large to display: 19.0 KB] ================================================ FILE: tsl/test/isolation/expected/compression_dml_iso.out ================================================ [File too large to display: 20.8 KB] ================================================ FILE: tsl/test/isolation/expected/compression_freeze.out ================================================ [File too large to display: 8.4 KB] ================================================ FILE: tsl/test/isolation/expected/compression_merge_race.out ================================================ [File too large to display: 4.2 KB] ================================================ FILE: tsl/test/isolation/expected/compression_recompress.out ================================================ [File too large to display: 42.4 KB] ================================================ FILE: tsl/test/isolation/expected/concurrent_decompress_update.out ================================================ [File too large to display: 865 B] ================================================ FILE: tsl/test/isolation/expected/deadlock_drop_chunks_compress.out ================================================ [File too large to display: 6.8 KB] ================================================ FILE: tsl/test/isolation/expected/deadlock_drop_index_vacuum.out ================================================ [File too large to display: 644 B] ================================================ FILE: tsl/test/isolation/expected/deadlock_recompress_chunk.out ================================================ [File too large to display: 1.2 KB] ================================================ FILE: tsl/test/isolation/expected/decompression_chunk_and_parallel_query.out ================================================ [File too large to display: 1.1 KB] ================================================ FILE: tsl/test/isolation/expected/decompression_chunk_and_parallel_query_wo_idx.out ================================================ [File too large to display: 2.6 KB] ================================================ FILE: tsl/test/isolation/expected/delete_job_deadlock.out ================================================ [File too large to display: 428 B] ================================================ FILE: tsl/test/isolation/expected/detach_chunk_isolation.out ================================================ [File too large to display: 5.1 KB] ================================================ FILE: tsl/test/isolation/expected/direct_compress_copy.out ================================================ [File too large to display: 2.7 KB] ================================================ FILE: tsl/test/isolation/expected/fk_hypertable_lock.out ================================================ [File too large to display: 1.2 KB] ================================================ FILE: tsl/test/isolation/expected/freeze_chunk.out ================================================ [File too large to display: 16.4 KB] ================================================ FILE: tsl/test/isolation/expected/hypertable_row_lock.out ================================================ [File too large to display: 5.3 KB] ================================================ FILE: tsl/test/isolation/expected/merge_chunks_concurrent.out ================================================ [File too large to display: 21.2 KB] ================================================ FILE: tsl/test/isolation/expected/osm_range_updates_iso.out ================================================ [File too large to display: 6.9 KB] ================================================ FILE: tsl/test/isolation/expected/parallel_compression.out ================================================ [File too large to display: 3.4 KB] ================================================ FILE: tsl/test/isolation/expected/reorder_deadlock.out ================================================ [File too large to display: 1.1 KB] ================================================ FILE: tsl/test/isolation/expected/reorder_vs_insert.out ================================================ [File too large to display: 3.3 KB] ================================================ FILE: tsl/test/isolation/expected/reorder_vs_insert_other_chunk.out ================================================ [File too large to display: 1.7 KB] ================================================ FILE: tsl/test/isolation/expected/reorder_vs_select.out ================================================ [File too large to display: 4.2 KB] ================================================ FILE: tsl/test/isolation/expected/split_chunk_concurrent.out ================================================ [File too large to display: 24.7 KB] ================================================ FILE: tsl/test/isolation/specs/.gitignore ================================================ [File too large to display: 107 B] ================================================ FILE: tsl/test/isolation/specs/CMakeLists.txt ================================================ [File too large to display: 2.6 KB] ================================================ FILE: tsl/test/isolation/specs/attach_chunk_isolation.spec ================================================ [File too large to display: 4.8 KB] ================================================ FILE: tsl/test/isolation/specs/bgw_job_duplicate_race.spec ================================================ [File too large to display: 3.5 KB] ================================================ FILE: tsl/test/isolation/specs/bgw_job_stat_history_retention_isolation.spec ================================================ [File too large to display: 1.9 KB] ================================================ FILE: tsl/test/isolation/specs/cagg_concurrent_invalidation.spec ================================================ [File too large to display: 4.4 KB] ================================================ FILE: tsl/test/isolation/specs/cagg_concurrent_move.spec ================================================ [File too large to display: 3.6 KB] ================================================ FILE: tsl/test/isolation/specs/cagg_concurrent_refresh.spec ================================================ [File too large to display: 18.4 KB] ================================================ FILE: tsl/test/isolation/specs/cagg_insert.spec ================================================ [File too large to display: 7.8 KB] ================================================ FILE: tsl/test/isolation/specs/cagg_multi_iso.spec ================================================ [File too large to display: 4.2 KB] ================================================ FILE: tsl/test/isolation/specs/cagg_watermark_concurrent_update.spec ================================================ [File too large to display: 2.8 KB] ================================================ FILE: tsl/test/isolation/specs/compression_chunk_race.spec ================================================ [File too large to display: 4.1 KB] ================================================ FILE: tsl/test/isolation/specs/compression_conflicts_iso.spec ================================================ [File too large to display: 9.6 KB] ================================================ FILE: tsl/test/isolation/specs/compression_ddl_iso.spec ================================================ [File too large to display: 6.2 KB] ================================================ FILE: tsl/test/isolation/specs/compression_dml_iso.spec ================================================ [File too large to display: 4.5 KB] ================================================ FILE: tsl/test/isolation/specs/compression_freeze.spec ================================================ [File too large to display: 5.0 KB] ================================================ FILE: tsl/test/isolation/specs/compression_merge_race.spec ================================================ [File too large to display: 5.3 KB] ================================================ FILE: tsl/test/isolation/specs/compression_recompress.spec ================================================ [File too large to display: 12.1 KB] ================================================ FILE: tsl/test/isolation/specs/concurrent_decompress_update.spec ================================================ [File too large to display: 2.7 KB] ================================================ FILE: tsl/test/isolation/specs/deadlock_drop_chunks_compress.spec ================================================ [File too large to display: 1.6 KB] ================================================ FILE: tsl/test/isolation/specs/deadlock_drop_index_vacuum.spec ================================================ [File too large to display: 3.8 KB] ================================================ FILE: tsl/test/isolation/specs/deadlock_recompress_chunk.spec ================================================ [File too large to display: 3.1 KB] ================================================ FILE: tsl/test/isolation/specs/decompression_chunk_and_parallel_query.in ================================================ [File too large to display: 2.5 KB] ================================================ FILE: tsl/test/isolation/specs/decompression_chunk_and_parallel_query_wo_idx.spec ================================================ [File too large to display: 4.8 KB] ================================================ FILE: tsl/test/isolation/specs/delete_job_deadlock.spec ================================================ [File too large to display: 2.5 KB] ================================================ FILE: tsl/test/isolation/specs/detach_chunk_isolation.spec ================================================ [File too large to display: 2.8 KB] ================================================ FILE: tsl/test/isolation/specs/direct_compress_copy.spec ================================================ [File too large to display: 2.7 KB] ================================================ FILE: tsl/test/isolation/specs/fk_hypertable_lock.spec ================================================ [File too large to display: 3.4 KB] ================================================ FILE: tsl/test/isolation/specs/freeze_chunk.spec ================================================ [File too large to display: 6.7 KB] ================================================ FILE: tsl/test/isolation/specs/hypertable_row_lock.spec ================================================ [File too large to display: 4.4 KB] ================================================ FILE: tsl/test/isolation/specs/merge_chunks_concurrent.spec ================================================ [File too large to display: 12.1 KB] ================================================ FILE: tsl/test/isolation/specs/osm_range_updates_iso.spec ================================================ [File too large to display: 5.4 KB] ================================================ FILE: tsl/test/isolation/specs/parallel_compression.spec ================================================ [File too large to display: 2.3 KB] ================================================ FILE: tsl/test/isolation/specs/reorder_deadlock.spec.in ================================================ [File too large to display: 1.5 KB] ================================================ FILE: tsl/test/isolation/specs/reorder_vs_insert.spec.in ================================================ [File too large to display: 1.9 KB] ================================================ FILE: tsl/test/isolation/specs/reorder_vs_insert_other_chunk.spec.in ================================================ [File too large to display: 1.3 KB] ================================================ FILE: tsl/test/isolation/specs/reorder_vs_select.spec.in ================================================ [File too large to display: 2.0 KB] ================================================ FILE: tsl/test/isolation/specs/split_chunk_concurrent.spec ================================================ [File too large to display: 8.5 KB] ================================================ FILE: tsl/test/postgresql.conf.in ================================================ [File too large to display: 1.3 KB] ================================================ FILE: tsl/test/shared/CMakeLists.txt ================================================ [File too large to display: 22 B] ================================================ FILE: tsl/test/shared/expected/build_info.out ================================================ [File too large to display: 1.1 KB] ================================================ FILE: tsl/test/shared/expected/cagg_compression.out ================================================ [File too large to display: 18.5 KB] ================================================ FILE: tsl/test/shared/expected/chunk_append_merge_append.out ================================================ [File too large to display: 6.2 KB] ================================================ FILE: tsl/test/shared/expected/chunkwise_agg_gather_sort.out ================================================ [File too large to display: 2.3 KB] ================================================ FILE: tsl/test/shared/expected/classify_relation.out ================================================ [File too large to display: 586 B] ================================================ FILE: tsl/test/shared/expected/compat.out ================================================ [File too large to display: 17.4 KB] ================================================ FILE: tsl/test/shared/expected/compress_bloom_sparse_compat.out ================================================ [File too large to display: 542 B] ================================================ FILE: tsl/test/shared/expected/compress_unique_index.out ================================================ [File too large to display: 2.7 KB] ================================================ FILE: tsl/test/shared/expected/compression_dml.out ================================================ [File too large to display: 43.8 KB] ================================================ FILE: tsl/test/shared/expected/compression_nulls_not_distinct.out ================================================ [File too large to display: 2.4 KB] ================================================ FILE: tsl/test/shared/expected/constify_now-15.out ================================================ [File too large to display: 34.5 KB] ================================================ FILE: tsl/test/shared/expected/constify_now-16.out ================================================ [File too large to display: 34.5 KB] ================================================ FILE: tsl/test/shared/expected/constify_now-17.out ================================================ [File too large to display: 34.5 KB] ================================================ FILE: tsl/test/shared/expected/constify_now-18.out ================================================ [File too large to display: 33.9 KB] ================================================ FILE: tsl/test/shared/expected/constify_timestamptz_op_interval-15.out ================================================ [File too large to display: 11.9 KB] ================================================ FILE: tsl/test/shared/expected/constify_timestamptz_op_interval-16.out ================================================ [File too large to display: 11.9 KB] ================================================ FILE: tsl/test/shared/expected/constify_timestamptz_op_interval-17.out ================================================ [File too large to display: 11.9 KB] ================================================ FILE: tsl/test/shared/expected/constify_timestamptz_op_interval-18.out ================================================ [File too large to display: 12.0 KB] ================================================ FILE: tsl/test/shared/expected/constraint_aware_append.out ================================================ [File too large to display: 1.0 KB] ================================================ FILE: tsl/test/shared/expected/constraint_exclusion_prepared-15.out ================================================ [File too large to display: 236.5 KB] ================================================ FILE: tsl/test/shared/expected/constraint_exclusion_prepared-16.out ================================================ [File too large to display: 236.5 KB] ================================================ FILE: tsl/test/shared/expected/constraint_exclusion_prepared-17.out ================================================ [File too large to display: 236.5 KB] ================================================ FILE: tsl/test/shared/expected/constraint_exclusion_prepared-18.out ================================================ [File too large to display: 236.6 KB] ================================================ FILE: tsl/test/shared/expected/decompress_join.out ================================================ [File too large to display: 3.2 KB] ================================================ FILE: tsl/test/shared/expected/decompress_placeholdervar.out ================================================ [File too large to display: 8.0 KB] ================================================ FILE: tsl/test/shared/expected/decompress_tracking.out ================================================ [File too large to display: 9.0 KB] ================================================ FILE: tsl/test/shared/expected/extension.out ================================================ [File too large to display: 16.1 KB] ================================================ FILE: tsl/test/shared/expected/gapfill-15.out ================================================ [File too large to display: 197.4 KB] ================================================ FILE: tsl/test/shared/expected/gapfill-16.out ================================================ [File too large to display: 197.4 KB] ================================================ FILE: tsl/test/shared/expected/gapfill-17.out ================================================ [File too large to display: 197.4 KB] ================================================ FILE: tsl/test/shared/expected/gapfill-18.out ================================================ [File too large to display: 197.4 KB] ================================================ FILE: tsl/test/shared/expected/gapfill_bug.out ================================================ [File too large to display: 28.2 KB] ================================================ FILE: tsl/test/shared/expected/generated_columns.out ================================================ [File too large to display: 2.1 KB] ================================================ FILE: tsl/test/shared/expected/lateral_subquery-15.out ================================================ [File too large to display: 3.4 KB] ================================================ FILE: tsl/test/shared/expected/lateral_subquery-16.out ================================================ [File too large to display: 3.4 KB] ================================================ FILE: tsl/test/shared/expected/lateral_subquery-17.out ================================================ [File too large to display: 3.4 KB] ================================================ FILE: tsl/test/shared/expected/lateral_subquery-18.out ================================================ [File too large to display: 3.4 KB] ================================================ FILE: tsl/test/shared/expected/memoize.out ================================================ [File too large to display: 19.9 KB] ================================================ FILE: tsl/test/shared/expected/merge_dml.out ================================================ [File too large to display: 10.3 KB] ================================================ FILE: tsl/test/shared/expected/ordered_append_join-15.out ================================================ [File too large to display: 272.1 KB] ================================================ FILE: tsl/test/shared/expected/ordered_append_join-16.out ================================================ [File too large to display: 272.1 KB] ================================================ FILE: tsl/test/shared/expected/ordered_append_join-17.out ================================================ [File too large to display: 270.6 KB] ================================================ FILE: tsl/test/shared/expected/ordered_append_join-18.out ================================================ [File too large to display: 270.6 KB] ================================================ FILE: tsl/test/shared/expected/parameterized_chunkappend.out ================================================ [File too large to display: 2.7 KB] ================================================ FILE: tsl/test/shared/expected/security_barrier.out ================================================ [File too large to display: 2.2 KB] ================================================ FILE: tsl/test/shared/expected/space_constraint-15.out ================================================ [File too large to display: 45.9 KB] ================================================ FILE: tsl/test/shared/expected/space_constraint-16.out ================================================ [File too large to display: 45.9 KB] ================================================ FILE: tsl/test/shared/expected/space_constraint-17.out ================================================ [File too large to display: 45.9 KB] ================================================ FILE: tsl/test/shared/expected/space_constraint-18.out ================================================ [File too large to display: 40.4 KB] ================================================ FILE: tsl/test/shared/expected/subtract_integer_from_now.out ================================================ [File too large to display: 3.5 KB] ================================================ FILE: tsl/test/shared/expected/timestamp_limits.out ================================================ [File too large to display: 11.3 KB] ================================================ FILE: tsl/test/shared/expected/transparent_decompress_chunk-15.out ================================================ [File too large to display: 68.8 KB] ================================================ FILE: tsl/test/shared/expected/transparent_decompress_chunk-16.out ================================================ [File too large to display: 68.8 KB] ================================================ FILE: tsl/test/shared/expected/transparent_decompress_chunk-17.out ================================================ [File too large to display: 72.6 KB] ================================================ FILE: tsl/test/shared/expected/transparent_decompress_chunk-18.out ================================================ [File too large to display: 71.7 KB] ================================================ FILE: tsl/test/shared/expected/with_clause_parser.out ================================================ [File too large to display: 19.4 KB] ================================================ FILE: tsl/test/shared/sql/.gitignore ================================================ [File too large to display: 283 B] ================================================ FILE: tsl/test/shared/sql/CMakeLists.txt ================================================ [File too large to display: 2.9 KB] ================================================ FILE: tsl/test/shared/sql/build_info.sql ================================================ [File too large to display: 574 B] ================================================ FILE: tsl/test/shared/sql/cagg_compression.sql ================================================ [File too large to display: 3.5 KB] ================================================ FILE: tsl/test/shared/sql/chunk_append_merge_append.sql ================================================ [File too large to display: 1.1 KB] ================================================ FILE: tsl/test/shared/sql/chunkwise_agg_gather_sort.sql ================================================ [File too large to display: 756 B] ================================================ FILE: tsl/test/shared/sql/classify_relation.sql ================================================ [File too large to display: 431 B] ================================================ FILE: tsl/test/shared/sql/compat.sql ================================================ [File too large to display: 4.5 KB] ================================================ FILE: tsl/test/shared/sql/compress_bloom_sparse_compat.sql ================================================ [File too large to display: 504 B] ================================================ FILE: tsl/test/shared/sql/compress_unique_index.sql ================================================ [File too large to display: 1.5 KB] ================================================ FILE: tsl/test/shared/sql/compression_dml.sql ================================================ [File too large to display: 16.0 KB] ================================================ FILE: tsl/test/shared/sql/compression_nulls_not_distinct.sql ================================================ [File too large to display: 1.7 KB] ================================================ FILE: tsl/test/shared/sql/constify_now.sql.in ================================================ [File too large to display: 11.9 KB] ================================================ FILE: tsl/test/shared/sql/constify_timestamptz_op_interval.sql.in ================================================ [File too large to display: 4.6 KB] ================================================ FILE: tsl/test/shared/sql/constraint_aware_append.sql ================================================ [File too large to display: 881 B] ================================================ FILE: tsl/test/shared/sql/constraint_exclusion_prepared.sql.in ================================================ [File too large to display: 2.7 KB] ================================================ FILE: tsl/test/shared/sql/decompress_join.sql ================================================ [File too large to display: 1.5 KB] ================================================ FILE: tsl/test/shared/sql/decompress_placeholdervar.sql ================================================ [File too large to display: 5.2 KB] ================================================ FILE: tsl/test/shared/sql/decompress_tracking.sql ================================================ [File too large to display: 2.6 KB] ================================================ FILE: tsl/test/shared/sql/extension.sql ================================================ [File too large to display: 859 B] ================================================ FILE: tsl/test/shared/sql/gapfill.sql.in ================================================ [File too large to display: 45.8 KB] ================================================ FILE: tsl/test/shared/sql/gapfill_bug.sql ================================================ [File too large to display: 12.5 KB] ================================================ FILE: tsl/test/shared/sql/generated_columns.sql ================================================ [File too large to display: 1.5 KB] ================================================ FILE: tsl/test/shared/sql/include/cagg_compression_query.sql ================================================ [File too large to display: 948 B] ================================================ FILE: tsl/test/shared/sql/include/cagg_compression_setup.sql ================================================ [File too large to display: 3.2 KB] ================================================ FILE: tsl/test/shared/sql/include/constraint_exclusion_prepared_query.sql ================================================ [File too large to display: 2.4 KB] ================================================ FILE: tsl/test/shared/sql/include/gapfill_metrics_query.sql ================================================ [File too large to display: 7.3 KB] ================================================ FILE: tsl/test/shared/sql/include/memoize_query.sql ================================================ [File too large to display: 372 B] ================================================ FILE: tsl/test/shared/sql/include/ordered_append_join.sql ================================================ [File too large to display: 6.9 KB] ================================================ FILE: tsl/test/shared/sql/include/shared_setup.sql ================================================ [File too large to display: 10.7 KB] ================================================ FILE: tsl/test/shared/sql/lateral_subquery.sql.in ================================================ [File too large to display: 1.3 KB] ================================================ FILE: tsl/test/shared/sql/memoize.sql ================================================ [File too large to display: 1.8 KB] ================================================ FILE: tsl/test/shared/sql/merge_dml.sql ================================================ [File too large to display: 9.4 KB] ================================================ FILE: tsl/test/shared/sql/ordered_append_join.sql.in ================================================ [File too large to display: 4.5 KB] ================================================ FILE: tsl/test/shared/sql/parameterized_chunkappend.sql ================================================ [File too large to display: 1.0 KB] ================================================ FILE: tsl/test/shared/sql/security_barrier.sql ================================================ [File too large to display: 1.4 KB] ================================================ FILE: tsl/test/shared/sql/space_constraint.sql.in ================================================ [File too large to display: 4.4 KB] ================================================ FILE: tsl/test/shared/sql/subtract_integer_from_now.sql ================================================ [File too large to display: 2.8 KB] ================================================ FILE: tsl/test/shared/sql/timestamp_limits.sql ================================================ [File too large to display: 7.9 KB] ================================================ FILE: tsl/test/shared/sql/transparent_decompress_chunk.sql.in ================================================ [File too large to display: 11.7 KB] ================================================ FILE: tsl/test/shared/sql/with_clause_parser.sql ================================================ [File too large to display: 5.9 KB] ================================================ FILE: tsl/test/sql/.gitignore ================================================ [File too large to display: 325 B] ================================================ FILE: tsl/test/sql/CMakeLists.txt ================================================ [File too large to display: 7.6 KB] ================================================ FILE: tsl/test/sql/agg_partials_pushdown.sql ================================================ [File too large to display: 7.3 KB] ================================================ FILE: tsl/test/sql/attach_chunk.sql ================================================ [File too large to display: 10.5 KB] ================================================ FILE: tsl/test/sql/bgw_custom.sql ================================================ [File too large to display: 27.7 KB] ================================================ FILE: tsl/test/sql/bgw_db_scheduler.sql ================================================ [File too large to display: 26.8 KB] ================================================ FILE: tsl/test/sql/bgw_db_scheduler_fixed.sql ================================================ [File too large to display: 33.0 KB] ================================================ FILE: tsl/test/sql/bgw_job_ddl.sql ================================================ [File too large to display: 4.0 KB] ================================================ FILE: tsl/test/sql/bgw_job_stat_history.sql ================================================ [File too large to display: 20.8 KB] ================================================ FILE: tsl/test/sql/bgw_job_stat_history_errors.sql ================================================ [File too large to display: 5.9 KB] ================================================ FILE: tsl/test/sql/bgw_job_stat_history_errors_permissions.sql ================================================ [File too large to display: 3.2 KB] ================================================ FILE: tsl/test/sql/bgw_policy.sql ================================================ [File too large to display: 18.2 KB] ================================================ FILE: tsl/test/sql/bgw_reorder_drop_chunks.sql ================================================ [File too large to display: 14.2 KB] ================================================ FILE: tsl/test/sql/bgw_scheduler_control.sql ================================================ [File too large to display: 4.2 KB] ================================================ FILE: tsl/test/sql/bgw_scheduler_restart.sql ================================================ [File too large to display: 5.4 KB] ================================================ FILE: tsl/test/sql/bgw_security.sql ================================================ [File too large to display: 1.2 KB] ================================================ FILE: tsl/test/sql/bgw_telemetry.sql ================================================ [File too large to display: 1.3 KB] ================================================ FILE: tsl/test/sql/cagg.sql.in ================================================ [File too large to display: 52.1 KB] ================================================ FILE: tsl/test/sql/cagg_bgw.sql.in ================================================ [File too large to display: 14.4 KB] ================================================ FILE: tsl/test/sql/cagg_bgw_drop_chunks.sql ================================================ [File too large to display: 3.4 KB] ================================================ FILE: tsl/test/sql/cagg_ddl.sql.in ================================================ [File too large to display: 59.0 KB] ================================================ FILE: tsl/test/sql/cagg_direct_compress.sql ================================================ [File too large to display: 5.5 KB] ================================================ FILE: tsl/test/sql/cagg_drop_chunks.sql ================================================ [File too large to display: 4.2 KB] ================================================ FILE: tsl/test/sql/cagg_dump.sql ================================================ [File too large to display: 5.2 KB] ================================================ FILE: tsl/test/sql/cagg_errors.sql ================================================ [File too large to display: 20.0 KB] ================================================ FILE: tsl/test/sql/cagg_invalidation.sql ================================================ [File too large to display: 35.3 KB] ================================================ FILE: tsl/test/sql/cagg_invalidation_variable_bucket.sql ================================================ [File too large to display: 25.0 KB] ================================================ FILE: tsl/test/sql/cagg_joins.sql ================================================ [File too large to display: 21.4 KB] ================================================ FILE: tsl/test/sql/cagg_multi.sql ================================================ [File too large to display: 8.4 KB] ================================================ FILE: tsl/test/sql/cagg_on_cagg.sql ================================================ [File too large to display: 10.2 KB] ================================================ FILE: tsl/test/sql/cagg_on_cagg_joins.sql ================================================ [File too large to display: 8.9 KB] ================================================ FILE: tsl/test/sql/cagg_permissions.sql.in ================================================ [File too large to display: 10.5 KB] ================================================ FILE: tsl/test/sql/cagg_planning.sql ================================================ [File too large to display: 4.7 KB] ================================================ FILE: tsl/test/sql/cagg_policy.sql ================================================ [File too large to display: 32.8 KB] ================================================ FILE: tsl/test/sql/cagg_policy_concurrent.sql ================================================ [File too large to display: 29.7 KB] ================================================ FILE: tsl/test/sql/cagg_policy_incremental.sql ================================================ [File too large to display: 15.5 KB] ================================================ FILE: tsl/test/sql/cagg_policy_move.sql ================================================ [File too large to display: 9.3 KB] ================================================ FILE: tsl/test/sql/cagg_policy_run.sql ================================================ [File too large to display: 2.2 KB] ================================================ FILE: tsl/test/sql/cagg_query.sql.in ================================================ [File too large to display: 421 B] ================================================ FILE: tsl/test/sql/cagg_query_using_merge.sql.in ================================================ [File too large to display: 546 B] ================================================ FILE: tsl/test/sql/cagg_refresh_using_merge.sql ================================================ [File too large to display: 6.5 KB] ================================================ FILE: tsl/test/sql/cagg_refresh_using_trigger.sql ================================================ [File too large to display: 247 B] ================================================ FILE: tsl/test/sql/cagg_tableam.sql ================================================ [File too large to display: 2.5 KB] ================================================ FILE: tsl/test/sql/cagg_union_view.sql.in ================================================ [File too large to display: 19.8 KB] ================================================ FILE: tsl/test/sql/cagg_usage.sql.in ================================================ [File too large to display: 12.4 KB] ================================================ FILE: tsl/test/sql/cagg_utils.sql ================================================ [File too large to display: 10.5 KB] ================================================ FILE: tsl/test/sql/cagg_uuid.sql ================================================ [File too large to display: 10.9 KB] ================================================ FILE: tsl/test/sql/cagg_watermark.sql ================================================ [File too large to display: 37.5 KB] ================================================ FILE: tsl/test/sql/chunk_api.sql ================================================ [File too large to display: 13.8 KB] ================================================ FILE: tsl/test/sql/chunk_column_stats.sql ================================================ [File too large to display: 15.1 KB] ================================================ FILE: tsl/test/sql/chunk_merge.sql ================================================ [File too large to display: 4.8 KB] ================================================ FILE: tsl/test/sql/chunk_publication_compression.sql ================================================ [File too large to display: 1.9 KB] ================================================ FILE: tsl/test/sql/chunk_utils_compression.sql ================================================ [File too large to display: 3.7 KB] ================================================ FILE: tsl/test/sql/chunk_utils_internal.sql ================================================ [File too large to display: 36.2 KB] ================================================ FILE: tsl/test/sql/columnar_index_scan.sql.in ================================================ [File too large to display: 4.1 KB] ================================================ FILE: tsl/test/sql/columnar_scan_cost.sql ================================================ [File too large to display: 6.6 KB] ================================================ FILE: tsl/test/sql/columnstore_aliases.sql ================================================ [File too large to display: 2.7 KB] ================================================ FILE: tsl/test/sql/compress_auto_sparse_index.sql ================================================ [File too large to display: 2.7 KB] ================================================ FILE: tsl/test/sql/compress_batch_size.sql ================================================ [File too large to display: 3.7 KB] ================================================ FILE: tsl/test/sql/compress_bgw_reorder_drop_chunks.sql ================================================ [File too large to display: 7.1 KB] ================================================ FILE: tsl/test/sql/compress_bitmap_scan.sql ================================================ [File too large to display: 2.2 KB] ================================================ FILE: tsl/test/sql/compress_bloom_dml.sql ================================================ [File too large to display: 3.2 KB] ================================================ FILE: tsl/test/sql/compress_bloom_hash.sql ================================================ [File too large to display: 1.0 KB] ================================================ FILE: tsl/test/sql/compress_bloom_legacy_v1.sql ================================================ [File too large to display: 2.0 KB] ================================================ FILE: tsl/test/sql/compress_bloom_sparse.sql.in ================================================ [File too large to display: 16.4 KB] ================================================ FILE: tsl/test/sql/compress_bloom_sparse_debug.sql ================================================ [File too large to display: 6.4 KB] ================================================ FILE: tsl/test/sql/compress_compbloom_basics.sql ================================================ [File too large to display: 7.5 KB] ================================================ FILE: tsl/test/sql/compress_compbloom_config.sql ================================================ [File too large to display: 5.0 KB] ================================================ FILE: tsl/test/sql/compress_compbloom_index_add.sql ================================================ [File too large to display: 1.8 KB] ================================================ FILE: tsl/test/sql/compress_compbloom_index_drop.sql ================================================ [File too large to display: 1.9 KB] ================================================ FILE: tsl/test/sql/compress_compbloom_manual_config.sql ================================================ [File too large to display: 2.8 KB] ================================================ FILE: tsl/test/sql/compress_compbloom_upsert.sql ================================================ [File too large to display: 5.9 KB] ================================================ FILE: tsl/test/sql/compress_composite_bloom_debug.sql ================================================ [File too large to display: 13.3 KB] ================================================ FILE: tsl/test/sql/compress_default.sql ================================================ [File too large to display: 1.1 KB] ================================================ FILE: tsl/test/sql/compress_dml_copy.sql ================================================ [File too large to display: 1.4 KB] ================================================ FILE: tsl/test/sql/compress_explain.sql ================================================ [File too large to display: 3.8 KB] ================================================ FILE: tsl/test/sql/compress_float8_corrupt.sql ================================================ [File too large to display: 40.7 KB] ================================================ FILE: tsl/test/sql/compress_qualpushdown_complex.sql ================================================ [File too large to display: 7.9 KB] ================================================ FILE: tsl/test/sql/compress_qualpushdown_saop.sql ================================================ [File too large to display: 10.6 KB] ================================================ FILE: tsl/test/sql/compress_sort_transform.sql ================================================ [File too large to display: 2.9 KB] ================================================ FILE: tsl/test/sql/compress_sparse_config.sql ================================================ [File too large to display: 15.2 KB] ================================================ FILE: tsl/test/sql/compress_unordered_sort.sql ================================================ [File too large to display: 8.0 KB] ================================================ FILE: tsl/test/sql/compressed_collation.sql ================================================ [File too large to display: 3.6 KB] ================================================ FILE: tsl/test/sql/compressed_detoaster.sql ================================================ [File too large to display: 1.5 KB] ================================================ FILE: tsl/test/sql/compression.sql ================================================ [File too large to display: 54.3 KB] ================================================ FILE: tsl/test/sql/compression_algos.sql ================================================ [File too large to display: 25.9 KB] ================================================ FILE: tsl/test/sql/compression_allocation.sql ================================================ [File too large to display: 1.8 KB] ================================================ FILE: tsl/test/sql/compression_bgw.sql ================================================ [File too large to display: 16.1 KB] ================================================ FILE: tsl/test/sql/compression_bool_vectorized.sql ================================================ [File too large to display: 1.4 KB] ================================================ FILE: tsl/test/sql/compression_bools.sql ================================================ [File too large to display: 4.0 KB] ================================================ FILE: tsl/test/sql/compression_conflicts.sql ================================================ [File too large to display: 20.5 KB] ================================================ FILE: tsl/test/sql/compression_constraints.sql ================================================ [File too large to display: 13.4 KB] ================================================ FILE: tsl/test/sql/compression_create_compressed_table.sql ================================================ [File too large to display: 1.8 KB] ================================================ FILE: tsl/test/sql/compression_ddl.sql ================================================ [File too large to display: 51.9 KB] ================================================ FILE: tsl/test/sql/compression_defaults.sql ================================================ [File too large to display: 19.1 KB] ================================================ FILE: tsl/test/sql/compression_delete_bitmapscan.sql.in ================================================ [File too large to display: 5.9 KB] ================================================ FILE: tsl/test/sql/compression_errors.sql ================================================ [File too large to display: 24.4 KB] ================================================ FILE: tsl/test/sql/compression_fks.sql ================================================ [File too large to display: 1.3 KB] ================================================ FILE: tsl/test/sql/compression_hypertable.sql ================================================ [File too large to display: 10.6 KB] ================================================ FILE: tsl/test/sql/compression_indexcreate.sql ================================================ [File too large to display: 2.1 KB] ================================================ FILE: tsl/test/sql/compression_indexscan.sql ================================================ [File too large to display: 12.5 KB] ================================================ FILE: tsl/test/sql/compression_insert.sql ================================================ [File too large to display: 44.1 KB] ================================================ FILE: tsl/test/sql/compression_merge.sql ================================================ [File too large to display: 16.3 KB] ================================================ FILE: tsl/test/sql/compression_null_dump_restore.sql ================================================ [File too large to display: 1.7 KB] ================================================ FILE: tsl/test/sql/compression_nulls_and_defaults.sql ================================================ [File too large to display: 10.1 KB] ================================================ FILE: tsl/test/sql/compression_permissions.sql.in ================================================ [File too large to display: 7.7 KB] ================================================ FILE: tsl/test/sql/compression_policy.sql ================================================ [File too large to display: 3.1 KB] ================================================ FILE: tsl/test/sql/compression_qualpushdown.sql ================================================ [File too large to display: 10.7 KB] ================================================ FILE: tsl/test/sql/compression_segment_meta.sql ================================================ [File too large to display: 4.2 KB] ================================================ FILE: tsl/test/sql/compression_sequence_num_removal.sql ================================================ [File too large to display: 10.4 KB] ================================================ FILE: tsl/test/sql/compression_settings.sql ================================================ [File too large to display: 7.4 KB] ================================================ FILE: tsl/test/sql/compression_sorted_merge.sql ================================================ [File too large to display: 25.8 KB] ================================================ FILE: tsl/test/sql/compression_sorted_merge_columns.sql ================================================ [File too large to display: 3.4 KB] ================================================ FILE: tsl/test/sql/compression_sorted_merge_distinct.sql ================================================ [File too large to display: 2.6 KB] ================================================ FILE: tsl/test/sql/compression_sorted_merge_filter.sql ================================================ [File too large to display: 2.7 KB] ================================================ FILE: tsl/test/sql/compression_sorted_merge_unordered.sql ================================================ [File too large to display: 14.3 KB] ================================================ FILE: tsl/test/sql/compression_trigger.sql ================================================ [File too large to display: 4.5 KB] ================================================ FILE: tsl/test/sql/compression_update_delete.sql.in ================================================ [File too large to display: 68.9 KB] ================================================ FILE: tsl/test/sql/compression_uuid.sql ================================================ [File too large to display: 14.3 KB] ================================================ FILE: tsl/test/sql/create_table_with.sql ================================================ [File too large to display: 19.3 KB] ================================================ FILE: tsl/test/sql/data/copy_data.csv ================================================ [File too large to display: 6.8 KB] ================================================ FILE: tsl/test/sql/data/magic.csv ================================================ [File too large to display: 882 B] ================================================ FILE: tsl/test/sql/decompress_index.sql ================================================ [File too large to display: 5.2 KB] ================================================ FILE: tsl/test/sql/decompress_memory.sql ================================================ [File too large to display: 3.6 KB] ================================================ FILE: tsl/test/sql/decompress_vector_qual.sql ================================================ [File too large to display: 46.0 KB] ================================================ FILE: tsl/test/sql/detach_chunk.sql ================================================ [File too large to display: 4.1 KB] ================================================ FILE: tsl/test/sql/direct_compress_copy.sql ================================================ [File too large to display: 10.9 KB] ================================================ FILE: tsl/test/sql/direct_compress_insert.sql ================================================ [File too large to display: 20.8 KB] ================================================ FILE: tsl/test/sql/feature_flags.sql ================================================ [File too large to display: 3.7 KB] ================================================ FILE: tsl/test/sql/fixed_schedules.sql ================================================ [File too large to display: 3.6 KB] ================================================ FILE: tsl/test/sql/foreign_keys_test.sql.in ================================================ [File too large to display: 1.3 KB] ================================================ FILE: tsl/test/sql/hypertable_generalization.sql ================================================ [File too large to display: 11.7 KB] ================================================ FILE: tsl/test/sql/include/aggregate_queries.sql ================================================ [File too large to display: 3.4 KB] ================================================ FILE: tsl/test/sql/include/aggregate_table_create.sql ================================================ [File too large to display: 914 B] ================================================ FILE: tsl/test/sql/include/aggregate_table_populate.sql ================================================ [File too large to display: 3.4 KB] ================================================ FILE: tsl/test/sql/include/cagg_on_cagg_common.sql ================================================ [File too large to display: 7.6 KB] ================================================ FILE: tsl/test/sql/include/cagg_on_cagg_setup.sql ================================================ [File too large to display: 1.8 KB] ================================================ FILE: tsl/test/sql/include/cagg_on_cagg_validations.sql ================================================ [File too large to display: 2.6 KB] ================================================ FILE: tsl/test/sql/include/cagg_planning_query.sql ================================================ [File too large to display: 1.2 KB] ================================================ FILE: tsl/test/sql/include/cagg_query_common.sql ================================================ [File too large to display: 39.7 KB] ================================================ FILE: tsl/test/sql/include/cagg_refresh_common.sql ================================================ [File too large to display: 19.5 KB] ================================================ FILE: tsl/test/sql/include/chunk_utils_internal_orderedappend.sql ================================================ [File too large to display: 10.4 KB] ================================================ FILE: tsl/test/sql/include/cluster_test_setup.sql ================================================ [File too large to display: 3.1 KB] ================================================ FILE: tsl/test/sql/include/columnar_index_scan_query.sql ================================================ [File too large to display: 7.3 KB] ================================================ FILE: tsl/test/sql/include/compression_alter.sql ================================================ [File too large to display: 8.4 KB] ================================================ FILE: tsl/test/sql/include/compression_test.sql ================================================ [File too large to display: 3.6 KB] ================================================ FILE: tsl/test/sql/include/compression_test_hypertable.sql ================================================ [File too large to display: 1.9 KB] ================================================ FILE: tsl/test/sql/include/compression_test_hypertable_segment_meta.sql ================================================ [File too large to display: 1.0 KB] ================================================ FILE: tsl/test/sql/include/compression_test_merge.sql ================================================ [File too large to display: 1.4 KB] ================================================ FILE: tsl/test/sql/include/compression_test_segment_meta.sql ================================================ [File too large to display: 377 B] ================================================ FILE: tsl/test/sql/include/compression_utils.sql ================================================ [File too large to display: 5.5 KB] ================================================ FILE: tsl/test/sql/include/cont_agg_equal.sql ================================================ [File too large to display: 1.2 KB] ================================================ FILE: tsl/test/sql/include/cont_agg_equal_deprecated.sql ================================================ [File too large to display: 1.3 KB] ================================================ FILE: tsl/test/sql/include/cont_agg_test_equal.sql ================================================ [File too large to display: 658 B] ================================================ FILE: tsl/test/sql/include/debugsupport.sql ================================================ [File too large to display: 371 B] ================================================ FILE: tsl/test/sql/include/filter_exec.sql ================================================ [File too large to display: 783 B] ================================================ FILE: tsl/test/sql/include/foreign_keys.sql ================================================ [File too large to display: 28.2 KB] ================================================ FILE: tsl/test/sql/include/jit_cleanup.sql ================================================ [File too large to display: 319 B] ================================================ FILE: tsl/test/sql/include/jit_load.sql ================================================ [File too large to display: 1.5 KB] ================================================ FILE: tsl/test/sql/include/jit_query.sql ================================================ [File too large to display: 1.2 KB] ================================================ FILE: tsl/test/sql/include/modify_exclusion_load.sql ================================================ [File too large to display: 4.0 KB] ================================================ FILE: tsl/test/sql/include/ordered_append.sql ================================================ [File too large to display: 5.8 KB] ================================================ FILE: tsl/test/sql/include/ordered_append_load.sql ================================================ [File too large to display: 6.5 KB] ================================================ FILE: tsl/test/sql/include/rand_generator.sql ================================================ [File too large to display: 672 B] ================================================ FILE: tsl/test/sql/include/recompress_basic.sql ================================================ [File too large to display: 7.4 KB] ================================================ FILE: tsl/test/sql/include/recompression_integrity_check.sql ================================================ [File too large to display: 2.5 KB] ================================================ FILE: tsl/test/sql/include/scheduler_fixed_common.sql ================================================ [File too large to display: 4.6 KB] ================================================ FILE: tsl/test/sql/include/skip_scan_comp_query.sql ================================================ [File too large to display: 14.8 KB] ================================================ FILE: tsl/test/sql/include/skip_scan_dagg_comp_query.sql ================================================ [File too large to display: 14.1 KB] ================================================ FILE: tsl/test/sql/include/skip_scan_dagg_load_comp_query.sql ================================================ [File too large to display: 7.0 KB] ================================================ FILE: tsl/test/sql/include/skip_scan_dagg_query.sql ================================================ [File too large to display: 12.5 KB] ================================================ FILE: tsl/test/sql/include/skip_scan_dagg_query_ht.sql ================================================ [File too large to display: 1.8 KB] ================================================ FILE: tsl/test/sql/include/skip_scan_load.sql ================================================ [File too large to display: 7.5 KB] ================================================ FILE: tsl/test/sql/include/skip_scan_load_comp_query.sql ================================================ [File too large to display: 6.8 KB] ================================================ FILE: tsl/test/sql/include/skip_scan_multi_load.sql ================================================ [File too large to display: 4.3 KB] ================================================ FILE: tsl/test/sql/include/skip_scan_multi_query.sql ================================================ [File too large to display: 4.9 KB] ================================================ FILE: tsl/test/sql/include/skip_scan_notnull.sql ================================================ [File too large to display: 3.2 KB] ================================================ FILE: tsl/test/sql/include/skip_scan_notnull_setup.sql ================================================ [File too large to display: 1.1 KB] ================================================ FILE: tsl/test/sql/include/skip_scan_query.sql ================================================ [File too large to display: 15.2 KB] ================================================ FILE: tsl/test/sql/include/skip_scan_query_ht.sql ================================================ [File too large to display: 1.2 KB] ================================================ FILE: tsl/test/sql/include/transparent_decompression_constraintaware.sql ================================================ [File too large to display: 2.4 KB] ================================================ FILE: tsl/test/sql/include/transparent_decompression_ordered.sql ================================================ [File too large to display: 1.6 KB] ================================================ FILE: tsl/test/sql/include/transparent_decompression_ordered_index.sql ================================================ [File too large to display: 5.4 KB] ================================================ FILE: tsl/test/sql/include/transparent_decompression_ordered_indexplan.sql ================================================ [File too large to display: 1.7 KB] ================================================ FILE: tsl/test/sql/include/transparent_decompression_query.sql ================================================ [File too large to display: 14.9 KB] ================================================ FILE: tsl/test/sql/include/transparent_decompression_systemcolumns.sql ================================================ [File too large to display: 1.4 KB] ================================================ FILE: tsl/test/sql/include/transparent_decompression_undiffed.sql ================================================ [File too large to display: 2.3 KB] ================================================ FILE: tsl/test/sql/include/vector_agg_planning_query.sql ================================================ [File too large to display: 2.2 KB] ================================================ FILE: tsl/test/sql/information_view_chunk_count.sql ================================================ [File too large to display: 1.7 KB] ================================================ FILE: tsl/test/sql/insert_memory_usage.sql ================================================ [File too large to display: 5.2 KB] ================================================ FILE: tsl/test/sql/jit.sql ================================================ [File too large to display: 1.6 KB] ================================================ FILE: tsl/test/sql/license_tsl.sql ================================================ [File too large to display: 289 B] ================================================ FILE: tsl/test/sql/merge_append_partially_compressed.sql ================================================ [File too large to display: 14.5 KB] ================================================ FILE: tsl/test/sql/merge_chunks.sql ================================================ [File too large to display: 20.1 KB] ================================================ FILE: tsl/test/sql/merge_compress.sql ================================================ [File too large to display: 3.9 KB] ================================================ FILE: tsl/test/sql/modify_exclusion.sql.in ================================================ [File too large to display: 8.8 KB] ================================================ FILE: tsl/test/sql/move.sql ================================================ [File too large to display: 7.4 KB] ================================================ FILE: tsl/test/sql/ordered_append.sql.in ================================================ [File too large to display: 2.1 KB] ================================================ FILE: tsl/test/sql/plan_skip_scan.sql.in ================================================ [File too large to display: 2.0 KB] ================================================ FILE: tsl/test/sql/plan_skip_scan_dagg.sql.in ================================================ [File too large to display: 897 B] ================================================ FILE: tsl/test/sql/plan_skip_scan_notnull.sql ================================================ [File too large to display: 1.6 KB] ================================================ FILE: tsl/test/sql/policy_generalization.sql ================================================ [File too large to display: 4.0 KB] ================================================ FILE: tsl/test/sql/privilege_maintain.sql ================================================ [File too large to display: 1.3 KB] ================================================ FILE: tsl/test/sql/read_only.sql ================================================ [File too large to display: 7.5 KB] ================================================ FILE: tsl/test/sql/rebuild_columnstore_tests.sql ================================================ [File too large to display: 9.2 KB] ================================================ FILE: tsl/test/sql/recompress_chunk_segmentwise.sql ================================================ [File too large to display: 20.6 KB] ================================================ FILE: tsl/test/sql/recompression_integrity_tests.sql ================================================ [File too large to display: 7.9 KB] ================================================ FILE: tsl/test/sql/recompression_integrity_unordered.sql ================================================ [File too large to display: 8.8 KB] ================================================ FILE: tsl/test/sql/reorder.sql ================================================ [File too large to display: 11.2 KB] ================================================ FILE: tsl/test/sql/scheduler_fixed.sql ================================================ [File too large to display: 7.3 KB] ================================================ FILE: tsl/test/sql/size_utils_tsl.sql ================================================ [File too large to display: 2.0 KB] ================================================ FILE: tsl/test/sql/skip_scan.sql ================================================ [File too large to display: 4.0 KB] ================================================ FILE: tsl/test/sql/skip_scan_dagg.sql ================================================ [File too large to display: 2.5 KB] ================================================ FILE: tsl/test/sql/split_chunk.sql ================================================ [File too large to display: 29.3 KB] ================================================ FILE: tsl/test/sql/telemetry_stats.sql ================================================ [File too large to display: 11.7 KB] ================================================ FILE: tsl/test/sql/transparent_decompression.sql.in ================================================ [File too large to display: 10.6 KB] ================================================ FILE: tsl/test/sql/transparent_decompression_join_index.sql ================================================ [File too large to display: 2.3 KB] ================================================ FILE: tsl/test/sql/transparent_decompression_ordered_index.sql.in ================================================ [File too large to display: 9.8 KB] ================================================ FILE: tsl/test/sql/transparent_decompression_queries.sql.in ================================================ [File too large to display: 5.9 KB] ================================================ FILE: tsl/test/sql/tsl_tables.sql ================================================ [File too large to display: 14.8 KB] ================================================ FILE: tsl/test/sql/uncompressed_size.sql ================================================ [File too large to display: 1.7 KB] ================================================ FILE: tsl/test/sql/unlogged.sql ================================================ [File too large to display: 2.0 KB] ================================================ FILE: tsl/test/sql/uuid_policies.sql ================================================ [File too large to display: 8.0 KB] ================================================ FILE: tsl/test/sql/vacuum.sql ================================================ [File too large to display: 5.2 KB] ================================================ FILE: tsl/test/sql/vector_agg_byte.sql ================================================ [File too large to display: 1.8 KB] ================================================ FILE: tsl/test/sql/vector_agg_default.sql ================================================ [File too large to display: 3.0 KB] ================================================ FILE: tsl/test/sql/vector_agg_expr.sql ================================================ [File too large to display: 3.4 KB] ================================================ FILE: tsl/test/sql/vector_agg_filter.sql ================================================ [File too large to display: 5.8 KB] ================================================ FILE: tsl/test/sql/vector_agg_functions.sql ================================================ [File too large to display: 7.6 KB] ================================================ FILE: tsl/test/sql/vector_agg_groupagg.sql ================================================ [File too large to display: 4.3 KB] ================================================ FILE: tsl/test/sql/vector_agg_grouping.sql ================================================ [File too large to display: 7.7 KB] ================================================ FILE: tsl/test/sql/vector_agg_memory.sql ================================================ [File too large to display: 4.4 KB] ================================================ FILE: tsl/test/sql/vector_agg_modify_hypertable.sql ================================================ [File too large to display: 1.4 KB] ================================================ FILE: tsl/test/sql/vector_agg_param.sql ================================================ [File too large to display: 1.6 KB] ================================================ FILE: tsl/test/sql/vector_agg_planning.sql.in ================================================ [File too large to display: 2.4 KB] ================================================ FILE: tsl/test/sql/vector_agg_segmentby.sql ================================================ [File too large to display: 3.6 KB] ================================================ FILE: tsl/test/sql/vector_agg_text.sql ================================================ [File too large to display: 5.0 KB] ================================================ FILE: tsl/test/sql/vector_agg_uuid.sql ================================================ [File too large to display: 5.0 KB] ================================================ FILE: tsl/test/sql/vector_qual_default.sql ================================================ [File too large to display: 2.7 KB] ================================================ FILE: tsl/test/sql/vectorized_aggregation.sql ================================================ [File too large to display: 14.6 KB] ================================================ FILE: tsl/test/src/CMakeLists.txt ================================================ [File too large to display: 840 B] ================================================ FILE: tsl/test/src/compression_sql_test.c ================================================ [File too large to display: 14.0 KB] ================================================ FILE: tsl/test/src/compression_sql_test.h ================================================ [File too large to display: 477 B] ================================================ FILE: tsl/test/src/compression_unit_test.c ================================================ [File too large to display: 55.8 KB] ================================================ FILE: tsl/test/src/decompress_arithmetic_test_impl.c ================================================ [File too large to display: 6.3 KB] ================================================ FILE: tsl/test/src/decompress_text_test_impl.c ================================================ [File too large to display: 6.3 KB] ================================================ FILE: tsl/test/src/test_chunk_stats.c ================================================ [File too large to display: 2.6 KB] ================================================ FILE: tsl/test/src/test_continuous_agg.c ================================================ [File too large to display: 2.1 KB] ================================================ FILE: tsl/test/src/test_merge_chunk.c ================================================ [File too large to display: 941 B] ================================================ FILE: tsl/test/t/001_job_crash_log.pl ================================================ [File too large to display: 2.3 KB] ================================================ FILE: tsl/test/t/002_logrepl_decomp_marker.pl ================================================ [File too large to display: 16.4 KB] ================================================ FILE: tsl/test/t/003_mvcc_cagg.pl ================================================ [File too large to display: 4.7 KB] ================================================ FILE: tsl/test/t/004_truncate_or_delete_spin_lock.pl ================================================ [File too large to display: 5.5 KB] ================================================ FILE: tsl/test/t/005_recompression_spin_lock_test.pl ================================================ [File too large to display: 5.2 KB] ================================================ FILE: tsl/test/t/CMakeLists.txt ================================================ [File too large to display: 746 B] ================================================ FILE: version.config ================================================ [File too large to display: 47 B]